diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index 92b35e97baa05..6a4531f1bdefa 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -11,3 +11,4 @@ attention.
 - If submitting code, have you built your formula locally prior to submission with `gradle check`?
 - If submitting code, is your pull request against master? Unless there is a good reason otherwise, we prefer pull requests against master and will backport as needed.
 - If submitting code, have you checked that your submission is for an [OS that we support](https://www.elastic.co/support/matrix#show_os)?
+- If you are submitting this code for a class then read our [policy](https://github.com/elastic/elasticsearch/blob/master/CONTRIBUTING.md#contributing-as-part-of-a-class) for that.
diff --git a/.gitignore b/.gitignore
index d1810a5a83fc7..b4ec8795057e2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -33,6 +33,7 @@ dependency-reduced-pom.xml
 # testing stuff
 **/.local*
 .vagrant/
+/logs/
 
 # osx stuff
 .DS_Store
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index b0f1e054e4693..f38a0588a9c0d 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -38,6 +38,11 @@ If you have a bugfix or new feature that you would like to contribute to Elastic
 
 We enjoy working with contributors to get their code accepted. There are many approaches to fixing a problem and it is important to find the best approach before writing too much code.
 
+Note that it is unlikely the project will merge refactors for the sake of refactoring. These
+types of pull requests have a high cost to maintainers in reviewing and testing with little
+to no tangible benefit. This especially includes changes generated by tools. For example,
+converting all generic interface instances to use the diamond operator. 
+
 The process for contributing to any of the [Elastic repositories](https://github.com/elastic/) is similar. Details for individual projects can be found below.
 
 ### Fork and clone the repository
@@ -88,7 +93,8 @@ Contributing to the Elasticsearch codebase
 **Repository:** [https://github.com/elastic/elasticsearch](https://github.com/elastic/elasticsearch)
 
 Make sure you have [Gradle](http://gradle.org) installed, as
-Elasticsearch uses it as its build system.
+Elasticsearch uses it as its build system. Gradle must be at least
+version 3.3 in order to build successfully.
 
 Eclipse users can automatically configure their IDE: `gradle eclipse`
 then `File: Import: Existing Projects into Workspace`. Select the
@@ -137,3 +143,32 @@ Before submitting your changes, run the test suite to make sure that nothing is
 ```sh
 gradle check
 ```
+
+Contributing as part of a class
+-------------------------------
+In general Elasticsearch is happy to accept contributions that were created as
+part of a class but strongly advise against making the contribution as part of
+the class. So if you have code you wrote for a class feel free to submit it.
+
+Please, please, please do not assign contributing to Elasticsearch as part of a
+class. If you really want to assign writing code for Elasticsearch as an
+assignment then the code contributions should be made to your private clone and
+opening PRs against the primary Elasticsearch clone must be optional, fully
+voluntary, not for a grade, and without any deadlines.
+
+Because:
+
+* While the code review process is likely very educational, it can take wildly
+varying amounts of time depending on who is available, where the change is, and
+how deep the change is. There is no way to predict how long it will take unless
+we rush.
+* We do not rush reviews without a very, very good reason. Class deadlines
+aren't a good enough reason for us to rush reviews.
+* We deeply discourage opening a PR you don't intend to work through the entire
+code review process because it wastes our time.
+* We don't have the capacity to absorb an entire class full of new contributors,
+especially when they are unlikely to become long time contributors.
+
+Finally, we require that you run `gradle check` before submitting a
+non-documentation contribution. This is mentioned above, but it is worth
+repeating in this section because it has come up in this context.
diff --git a/GRADLE.CHEATSHEET b/GRADLE.CHEATSHEET
index 3362b8571e7b9..2c9c34fe1b512 100644
--- a/GRADLE.CHEATSHEET
+++ b/GRADLE.CHEATSHEET
@@ -4,4 +4,4 @@ test -> test
 verify -> check
 verify -Dskip.unit.tests -> integTest
 package -DskipTests -> assemble
-install -DskipTests -> install
+install -DskipTests -> publishToMavenLocal
diff --git a/NOTICE.txt b/NOTICE.txt
index c99b958193198..2126baed56547 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -1,5 +1,5 @@
 Elasticsearch
-Copyright 2009-2016 Elasticsearch
+Copyright 2009-2017 Elasticsearch
 
-This product includes software developed by The Apache Software
-Foundation (http://www.apache.org/).
+This product includes software developed by The Apache Software Foundation
+(http://www.apache.org/).
diff --git a/README.textile b/README.textile
index 69d3fd54767e5..91895df93b4f8 100644
--- a/README.textile
+++ b/README.textile
@@ -50,16 +50,16 @@ h3. Indexing
 Let's try and index some twitter like information. First, let's create a twitter user, and add some tweets (the @twitter@ index will be created automatically):
 
 <pre>
-curl -XPUT 'http://localhost:9200/twitter/user/kimchy?pretty' -d '{ "name" : "Shay Banon" }'
+curl -XPUT 'http://localhost:9200/twitter/user/kimchy?pretty' -H 'Content-Type: application/json' -d '{ "name" : "Shay Banon" }'
 
-curl -XPUT 'http://localhost:9200/twitter/tweet/1?pretty' -d '
+curl -XPUT 'http://localhost:9200/twitter/tweet/1?pretty' -H 'Content-Type: application/json' -d '
 {
     "user": "kimchy",
     "post_date": "2009-11-15T13:12:00",
     "message": "Trying out Elasticsearch, so far so good?"
 }'
 
-curl -XPUT 'http://localhost:9200/twitter/tweet/2?pretty' -d '
+curl -XPUT 'http://localhost:9200/twitter/tweet/2?pretty' -H 'Content-Type: application/json' -d '
 {
     "user": "kimchy",
     "post_date": "2009-11-15T14:12:12",
@@ -87,7 +87,7 @@ curl -XGET 'http://localhost:9200/twitter/tweet/_search?q=user:kimchy&pretty=tru
 We can also use the JSON query language Elasticsearch provides instead of a query string:
 
 <pre>
-curl -XGET 'http://localhost:9200/twitter/tweet/_search?pretty=true' -d '
+curl -XGET 'http://localhost:9200/twitter/tweet/_search?pretty=true' -H 'Content-Type: application/json' -d '
 {
     "query" : {
         "match" : { "user": "kimchy" }
@@ -98,7 +98,7 @@ curl -XGET 'http://localhost:9200/twitter/tweet/_search?pretty=true' -d '
 Just for kicks, let's get all the documents stored (we should see the user as well):
 
 <pre>
-curl -XGET 'http://localhost:9200/twitter/_search?pretty=true' -d '
+curl -XGET 'http://localhost:9200/twitter/_search?pretty=true' -H 'Content-Type: application/json' -d '
 {
     "query" : {
         "match_all" : {}
@@ -106,10 +106,10 @@ curl -XGET 'http://localhost:9200/twitter/_search?pretty=true' -d '
 }'
 </pre>
 
-We can also do range search (the @postDate@ was automatically identified as date)
+We can also do range search (the @post_date@ was automatically identified as date)
 
 <pre>
-curl -XGET 'http://localhost:9200/twitter/_search?pretty=true' -d '
+curl -XGET 'http://localhost:9200/twitter/_search?pretty=true' -H 'Content-Type: application/json' -d '
 {
     "query" : {
         "range" : {
@@ -130,16 +130,16 @@ Elasticsearch supports multiple indices, as well as multiple types per index. In
 Another way to define our simple twitter system is to have a different index per user (note, though that each index has an overhead). Here is the indexing curl's in this case:
 
 <pre>
-curl -XPUT 'http://localhost:9200/kimchy/info/1?pretty' -d '{ "name" : "Shay Banon" }'
+curl -XPUT 'http://localhost:9200/kimchy/info/1?pretty' -H 'Content-Type: application/json' -d '{ "name" : "Shay Banon" }'
 
-curl -XPUT 'http://localhost:9200/kimchy/tweet/1?pretty' -d '
+curl -XPUT 'http://localhost:9200/kimchy/tweet/1?pretty' -H 'Content-Type: application/json' -d '
 {
     "user": "kimchy",
     "post_date": "2009-11-15T13:12:00",
     "message": "Trying out Elasticsearch, so far so good?"
 }'
 
-curl -XPUT 'http://localhost:9200/kimchy/tweet/2?pretty' -d '
+curl -XPUT 'http://localhost:9200/kimchy/tweet/2?pretty' -H 'Content-Type: application/json' -d '
 {
     "user": "kimchy",
     "post_date": "2009-11-15T14:12:12",
@@ -152,7 +152,7 @@ The above will index information into the @kimchy@ index, with two types, @info@
 Complete control on the index level is allowed. As an example, in the above case, we would want to change from the default 5 shards with 1 replica per index, to only 1 shard with 1 replica per index (== per twitter user). Here is how this can be done (the configuration can be in yaml as well):
 
 <pre>
-curl -XPUT http://localhost:9200/another_user?pretty -d '
+curl -XPUT http://localhost:9200/another_user?pretty -H 'Content-Type: application/json' -d '
 {
     "index" : {
         "number_of_shards" : 1,
@@ -165,7 +165,7 @@ Search (and similar operations) are multi index aware. This means that we can ea
 index (twitter user), for example:
 
 <pre>
-curl -XGET 'http://localhost:9200/kimchy,another_user/_search?pretty=true' -d '
+curl -XGET 'http://localhost:9200/kimchy,another_user/_search?pretty=true' -H 'Content-Type: application/json' -d '
 {
     "query" : {
         "match_all" : {}
@@ -176,7 +176,7 @@ curl -XGET 'http://localhost:9200/kimchy,another_user/_search?pretty=true' -d '
 Or on all the indices:
 
 <pre>
-curl -XGET 'http://localhost:9200/_search?pretty=true' -d '
+curl -XGET 'http://localhost:9200/_search?pretty=true' -H 'Content-Type: application/json' -d '
 {
     "query" : {
         "match_all" : {}
@@ -200,7 +200,7 @@ We have just covered a very small portion of what Elasticsearch is all about. Fo
 
 h3. Building from Source
 
-Elasticsearch uses "Gradle":https://gradle.org for its build system. You'll need to have a modern version of Gradle installed - 2.13 should do.
+Elasticsearch uses "Gradle":https://gradle.org for its build system. You'll need to have at least version 3.3 of Gradle installed.
 
 In order to create a distribution, simply run the @gradle assemble@ command in the cloned directory.
 
diff --git a/TESTING.asciidoc b/TESTING.asciidoc
index 5046dc087b564..4b47f0409734d 100644
--- a/TESTING.asciidoc
+++ b/TESTING.asciidoc
@@ -41,6 +41,12 @@ run it using Gradle:
 gradle run
 -------------------------------------
 
+or to attach a remote debugger, run it as:
+
+-------------------------------------
+gradle run --debug-jvm
+-------------------------------------
+
 === Test case filtering.
 
 - `tests.class` is a class-filtering shell-like glob pattern,
@@ -335,7 +341,7 @@ vagrant plugin install vagrant-cachier
 . Validate your installed dependencies:
 
 -------------------------------------
-gradle :qa:vagrant:checkVagrantVersion
+gradle :qa:vagrant:vagrantCheckVersion
 -------------------------------------
 
 . Download and smoke test the VMs with `gradle vagrantSmokeTest` or
@@ -361,24 +367,25 @@ VM running trusty by running
 
 These are the linux flavors the Vagrantfile currently supports:
 
-* ubuntu-1204 aka precise
 * ubuntu-1404 aka trusty
-* ubuntu-1504 aka vivid
-* debian-8 aka jessie, the current debian stable distribution
+* ubuntu-1604 aka xenial
+* debian-8 aka jessie
+* debian-9 aka stretch, the current debian stable distribution
 * centos-6
 * centos-7
-* fedora-22
+* fedora-25
+* fedora-26
+* oel-6 aka Oracle Enterprise Linux 6
 * oel-7 aka Oracle Enterprise Linux 7
 * sles-12
-* opensuse-13
+* opensuse-42 aka Leap
 
 We're missing the following from the support matrix because there aren't high
 quality boxes available in vagrant atlas:
 
 * sles-11
-* oel-6
 
-We're missing the follow because our tests are very linux/bash centric:
+We're missing the following because our tests are very linux/bash centric:
 
 * Windows Server 2012
 
@@ -427,19 +434,46 @@ and in another window:
 
 ----------------------------------------------------
 vagrant up centos-7 --provider virtualbox && vagrant ssh centos-7
-cd $TESTROOT
-sudo bats $BATS/*rpm*.bats
+cd $BATS_ARCHIVES
+sudo -E bats $BATS_TESTS/*rpm*.bats
 ----------------------------------------------------
 
 If you wanted to retest all the release artifacts on a single VM you could:
 
 -------------------------------------------------
-gradle prepareTestRoot
-vagrant up ubuntu-1404 --provider virtualbox && vagrant ssh ubuntu-1404
-cd $TESTROOT
-sudo bats $BATS/*.bats
+gradle setupBats
+cd qa/vagrant; vagrant up ubuntu-1404 --provider virtualbox && vagrant ssh ubuntu-1404
+cd $BATS_ARCHIVES
+sudo -E bats $BATS_TESTS/*.bats
 -------------------------------------------------
 
+You can also use Gradle to prepare the test environment and then starts a single VM:
+
+-------------------------------------------------
+gradle vagrantFedora25#up
+-------------------------------------------------
+
+Or any of vagrantCentos6#up, vagrantCentos7#up, vagrantDebian8#up,
+vagrantFedora25#up, vagrantOel6#up, vagrantOel7#up, vagrantOpensuse13#up,
+vagrantSles12#up, vagrantUbuntu1404#up, vagrantUbuntu1604#up.
+
+Once up, you can then connect to the VM using SSH from the elasticsearch directory:
+
+-------------------------------------------------
+vagrant ssh fedora-25
+-------------------------------------------------
+
+Or from another directory:
+
+-------------------------------------------------
+VAGRANT_CWD=/path/to/elasticsearch vagrant ssh fedora-25
+-------------------------------------------------
+
+Note: Starting vagrant VM outside of the elasticsearch folder requires to
+indicates the folder that contains the Vagrantfile using the VAGRANT_CWD
+environment variable.
+
+
 == Coverage analysis
 
 Tests can be run instrumented with jacoco to produce a coverage report in
@@ -475,12 +509,11 @@ gradle run --debug-jvm
 == Building with extra plugins
 Additional plugins may be built alongside elasticsearch, where their
 dependency on elasticsearch will be substituted with the local elasticsearch
-build. To add your plugin, create a directory called x-plugins as a sibling
-of elasticsearch. Checkout your plugin underneath x-plugins and the build
-will automatically pick it up. You can verify the plugin is included as part
-of the build by checking the projects of the build.
+build. To add your plugin, create a directory called elasticsearch-extra as
+a sibling of elasticsearch. Checkout your plugin underneath elasticsearch-extra
+and the build will automatically pick it up. You can verify the plugin is
+included as part of the build by checking the projects of the build.
 
 ---------------------------------------------------------------------------
 gradle projects
 ---------------------------------------------------------------------------
-
diff --git a/Vagrantfile b/Vagrantfile
index fc148ee444319..487594bba8a1a 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -22,27 +22,27 @@
 # under the License.
 
 Vagrant.configure(2) do |config|
-  config.vm.define "ubuntu-1204" do |config|
-    config.vm.box = "elastic/ubuntu-12.04-x86_64"
-    ubuntu_common config
-  end
   config.vm.define "ubuntu-1404" do |config|
     config.vm.box = "elastic/ubuntu-14.04-x86_64"
     ubuntu_common config
   end
-  config.vm.define "ubuntu-1504" do |config|
-    config.vm.box = "elastic/ubuntu-15.04-x86_64"
+  config.vm.define "ubuntu-1604" do |config|
+    config.vm.box = "elastic/ubuntu-16.04-x86_64"
     ubuntu_common config, extra: <<-SHELL
       # Install Jayatana so we can work around it being present.
       [ -f /usr/share/java/jayatanaag.jar ] || install jayatana
     SHELL
   end
-  # Wheezy's backports don't contain Openjdk 8 and the backflips required to
-  # get the sun jdk on there just aren't worth it. We have jessie for testing
-  # debian and it works fine.
+  # Wheezy's backports don't contain Openjdk 8 and the backflips
+  # required to get the sun jdk on there just aren't worth it. We have
+  # jessie and stretch for testing debian and it works fine.
   config.vm.define "debian-8" do |config|
     config.vm.box = "elastic/debian-8-x86_64"
-    deb_common config, 'echo deb http://cloudfront.debian.net/debian jessie-backports main > /etc/apt/sources.list.d/backports.list', 'backports'
+    deb_common config
+  end
+  config.vm.define "debian-9" do |config|
+    config.vm.box = "elastic/debian-9-x86_64"
+    deb_common config
   end
   config.vm.define "centos-6" do |config|
     config.vm.box = "elastic/centos-6-x86_64"
@@ -60,12 +60,16 @@ Vagrant.configure(2) do |config|
     config.vm.box = "elastic/oraclelinux-7-x86_64"
     rpm_common config
   end
-  config.vm.define "fedora-24" do |config|
-    config.vm.box = "elastic/fedora-24-x86_64"
+  config.vm.define "fedora-25" do |config|
+    config.vm.box = "elastic/fedora-25-x86_64"
     dnf_common config
   end
-  config.vm.define "opensuse-13" do |config|
-    config.vm.box = "elastic/opensuse-13-x86_64"
+  config.vm.define "fedora-26" do |config|
+    config.vm.box = "elastic/fedora-26-x86_64"
+    dnf_common config
+  end
+  config.vm.define "opensuse-42" do |config|
+    config.vm.box = "elastic/opensuse-42-x86_64"
     opensuse_common config
   end
   config.vm.define "sles-12" do |config|
@@ -77,6 +81,9 @@ Vagrant.configure(2) do |config|
   # the elasticsearch project called vagrant....
   config.vm.synced_folder ".", "/vagrant", disabled: true
   config.vm.synced_folder ".", "/elasticsearch"
+  # Expose project directory
+  PROJECT_DIR = ENV['VAGRANT_PROJECT_DIR'] || Dir.pwd
+  config.vm.synced_folder PROJECT_DIR, "/project"
   config.vm.provider "virtualbox" do |v|
     # Give the boxes 3GB because Elasticsearch defaults to using 2GB
     v.memory = 3072
@@ -105,16 +112,22 @@ SOURCE_PROMPT
 source /etc/profile.d/elasticsearch_prompt.sh
 SOURCE_PROMPT
       SHELL
+      # Creates a file to mark the machine as created by vagrant. Tests check
+      # for this file and refuse to run if it is not present so that they can't
+      # be run unexpectedly.
+      config.vm.provision "markerfile", type: "shell", inline: <<-SHELL
+        touch /etc/is_vagrant_vm
+      SHELL
     end
     config.config_procs.push ['2', set_prompt]
   end
 end
 
 def ubuntu_common(config, extra: '')
-  deb_common config, 'apt-add-repository -y ppa:openjdk-r/ppa > /dev/null 2>&1', 'openjdk-r-*', extra: extra
+  deb_common config, extra: extra
 end
 
-def deb_common(config, add_openjdk_repository_command, openjdk_list, extra: '')
+def deb_common(config, extra: '')
   # http://foo-o-rama.com/vagrant--stdin-is-not-a-tty--fix.html
   config.vm.provision "fix-no-tty", type: "shell" do |s|
       s.privileged = false
@@ -124,24 +137,14 @@ def deb_common(config, add_openjdk_repository_command, openjdk_list, extra: '')
     update_command: "apt-get update",
     update_tracking_file: "/var/cache/apt/archives/last_update",
     install_command: "apt-get install -y",
-    java_package: "openjdk-8-jdk",
-    extra: <<-SHELL
-      export DEBIAN_FRONTEND=noninteractive
-      ls /etc/apt/sources.list.d/#{openjdk_list}.list > /dev/null 2>&1 ||
-        (echo "==> Importing java-8 ppa" &&
-          #{add_openjdk_repository_command} &&
-          apt-get update)
-      #{extra}
-SHELL
-  )
+    extra: extra)
 end
 
 def rpm_common(config)
   provision(config,
     update_command: "yum check-update",
     update_tracking_file: "/var/cache/yum/last_update",
-    install_command: "yum install -y",
-    java_package: "java-1.8.0-openjdk-devel")
+    install_command: "yum install -y")
 end
 
 def dnf_common(config)
@@ -149,7 +152,7 @@ def dnf_common(config)
     update_command: "dnf check-update",
     update_tracking_file: "/var/cache/dnf/last_update",
     install_command: "dnf install -y",
-    java_package: "java-1.8.0-openjdk-devel")
+    install_command_retries: 5)
   if Vagrant.has_plugin?("vagrant-cachier")
     # Autodetect doesn't work....
     config.cache.auto_detect = false
@@ -166,17 +169,12 @@ def suse_common(config, extra)
     update_command: "zypper --non-interactive list-updates",
     update_tracking_file: "/var/cache/zypp/packages/last_update",
     install_command: "zypper --non-interactive --quiet install --no-recommends",
-    java_package: "java-1_8_0-openjdk-devel",
     extra: extra)
 end
 
 def sles_common(config)
   extra = <<-SHELL
-    zypper rr systemsmanagement_puppet
-    zypper addrepo -t yast2 http://demeter.uni-regensburg.de/SLES12-x64/DVD1/ dvd1 || true
-    zypper addrepo -t yast2 http://demeter.uni-regensburg.de/SLES12-x64/DVD2/ dvd2 || true
-    zypper addrepo http://download.opensuse.org/repositories/Java:Factory/SLE_12/Java:Factory.repo || true
-    zypper --no-gpg-checks --non-interactive refresh
+    zypper rr systemsmanagement_puppet puppetlabs-pc1
     zypper --non-interactive install git-core
 SHELL
   suse_common config, extra
@@ -191,26 +189,50 @@ end
 #   is cached by vagrant-cachier.
 # @param install_command [String] The command used to install a package.
 #   Required. Think `apt-get install #{package}`.
-# @param java_package [String] The name of the java package. Required.
 # @param extra [String] Extra provisioning commands run before anything else.
 #   Optional. Used for things like setting up the ppa for Java 8.
 def provision(config,
     update_command: 'required',
     update_tracking_file: 'required',
     install_command: 'required',
-    java_package: 'required',
+    install_command_retries: 0,
     extra: '')
   # Vagrant run ruby 2.0.0 which doesn't have required named parameters....
   raise ArgumentError.new('update_command is required') if update_command == 'required'
   raise ArgumentError.new('update_tracking_file is required') if update_tracking_file == 'required'
   raise ArgumentError.new('install_command is required') if install_command == 'required'
-  raise ArgumentError.new('java_package is required') if java_package == 'required'
-  config.vm.provision "bats dependencies", type: "shell", inline: <<-SHELL
+  config.vm.provider "virtualbox" do |v|
+    # Give the box more memory and cpu because our tests are beasts!
+    v.memory = Integer(ENV['VAGRANT_MEMORY'] || 8192)
+    v.cpus = Integer(ENV['VAGRANT_CPUS'] || 4)
+  end
+  config.vm.synced_folder "#{Dir.home}/.gradle/caches", "/home/vagrant/.gradle/caches",
+    create: true,
+    owner: "vagrant"
+  config.vm.provision "dependencies", type: "shell", inline: <<-SHELL
     set -e
     set -o pipefail
+
+    # Retry install command up to $2 times, if failed
+    retry_installcommand() {
+      n=0
+      while true; do
+        #{install_command} $1 && break
+        let n=n+1
+        if [ $n -ge $2 ]; then
+          echo "==> Exhausted retries to install $1"
+          return 1
+        fi
+        echo "==> Retrying installing $1, attempt $((n+1))"
+        # Add a small delay to increase chance of metalink providing updated list of mirrors
+        sleep 5
+      done
+    }
+
     installed() {
       command -v $1 2>&1 >/dev/null
     }
+
     install() {
       # Only apt-get update if we haven't in the last day
       if [ ! -f #{update_tracking_file} ] || [ "x$(find #{update_tracking_file} -mtime +0)" == "x#{update_tracking_file}" ]; then
@@ -219,15 +241,24 @@ def provision(config,
         touch #{update_tracking_file}
       fi
       echo "==> Installing $1"
-      #{install_command} $1
+      if [ #{install_command_retries} -eq 0 ]
+      then
+            #{install_command} $1
+      else
+            retry_installcommand $1 #{install_command_retries}
+      fi
     }
+
     ensure() {
       installed $1 || install $1
     }
 
     #{extra}
 
-    installed java || install #{java_package}
+    installed java || {
+      echo "==> Java is not installed on vagrant box ${config.vm.box}"
+      return 1
+    }
     ensure tar
     ensure curl
     ensure unzip
@@ -241,13 +272,39 @@ def provision(config,
       /tmp/bats/install.sh /usr
       rm -rf /tmp/bats
     }
+
+    installed gradle || {
+      echo "==> Installing Gradle"
+      curl -sS -o /tmp/gradle.zip -L https://services.gradle.org/distributions/gradle-3.3-bin.zip
+      unzip -q /tmp/gradle.zip -d /opt
+      rm -rf /tmp/gradle.zip
+      ln -s /opt/gradle-3.3/bin/gradle /usr/bin/gradle
+      # make nfs mounted gradle home dir writeable
+      chown vagrant:vagrant /home/vagrant/.gradle
+    }
+
+
     cat \<\<VARS > /etc/profile.d/elasticsearch_vars.sh
 export ZIP=/elasticsearch/distribution/zip/build/distributions
 export TAR=/elasticsearch/distribution/tar/build/distributions
 export RPM=/elasticsearch/distribution/rpm/build/distributions
 export DEB=/elasticsearch/distribution/deb/build/distributions
-export TESTROOT=/elasticsearch/qa/vagrant/build/testroot
-export BATS=/elasticsearch/qa/vagrant/src/test/resources/packaging/scripts
+export BATS=/project/build/bats
+export BATS_UTILS=/project/build/bats/utils
+export BATS_TESTS=/project/build/bats/tests
+export BATS_ARCHIVES=/project/build/bats/archives
+export GRADLE_HOME=/opt/gradle-3.3
 VARS
+    cat \<\<SUDOERS_VARS > /etc/sudoers.d/elasticsearch_vars
+Defaults   env_keep += "ZIP"
+Defaults   env_keep += "TAR"
+Defaults   env_keep += "RPM"
+Defaults   env_keep += "DEB"
+Defaults   env_keep += "BATS"
+Defaults   env_keep += "BATS_UTILS"
+Defaults   env_keep += "BATS_TESTS"
+Defaults   env_keep += "BATS_ARCHIVES"
+SUDOERS_VARS
+    chmod 0440 /etc/sudoers.d/elasticsearch_vars
   SHELL
 end
diff --git a/benchmarks/build.gradle b/benchmarks/build.gradle
index 3b8b92328e149..35b7c1a163d3e 100644
--- a/benchmarks/build.gradle
+++ b/benchmarks/build.gradle
@@ -34,13 +34,14 @@ apply plugin: 'com.github.johnrengelman.shadow'
 // have the shadow plugin provide the runShadow task
 apply plugin: 'application'
 
+// Not published so no need to assemble
+tasks.remove(assemble)
+build.dependsOn.remove('assemble')
+
 archivesBaseName = 'elasticsearch-benchmarks'
 mainClassName = 'org.openjdk.jmh.Main'
 
-// never try to invoke tests on the benchmark project - there aren't any
-check.dependsOn.remove(test)
-// explicitly override the test task too in case somebody invokes 'gradle test' so it won't trip
-task test(type: Test, overwrite: true)
+test.enabled = false
 
 dependencies {
     compile("org.elasticsearch:elasticsearch:${version}") {
@@ -60,7 +61,6 @@ compileJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-u
 // enable the JMH's BenchmarkProcessor to generate the final benchmark classes
 // needs to be added separately otherwise Gradle will quote it and javac will fail
 compileJava.options.compilerArgs.addAll(["-processor", "org.openjdk.jmh.generators.BenchmarkProcessor"])
-compileTestJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-unchecked"
 
 forbiddenApis {
     // classes generated by JMH can use all sorts of forbidden APIs but we have no influence at all and cannot exclude these classes
diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java
index 86902b380c86e..39cfdb6582d74 100644
--- a/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java
+++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java
@@ -27,7 +27,6 @@
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
-import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.common.settings.Settings;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -160,11 +159,9 @@ private int toInt(String v) {
     public ClusterState measureAllocation() {
         ClusterState clusterState = initialClusterState;
         while (clusterState.getRoutingNodes().hasUnassignedShards()) {
-            RoutingAllocation.Result result = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes()
+            clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes()
                     .shardsWithState(ShardRoutingState.INITIALIZING));
-            clusterState = ClusterState.builder(clusterState).routingResult(result).build();
-            result = strategy.reroute(clusterState, "reroute");
-            clusterState = ClusterState.builder(clusterState).routingResult(result).build();
+            clusterState = strategy.reroute(clusterState, "reroute");
         }
         return clusterState;
     }
diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java
index ad06f75074d53..860137cf559a0 100644
--- a/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java
+++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java
@@ -22,10 +22,10 @@
 import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.EmptyClusterInfoService;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
-import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation;
+import org.elasticsearch.cluster.routing.allocation.FailedShard;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
-import org.elasticsearch.cluster.routing.allocation.StartedRerouteAllocation;
 import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
@@ -35,9 +35,7 @@
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.gateway.GatewayAllocator;
 
-import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
-import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
@@ -52,12 +50,12 @@ protected NoopGatewayAllocator() {
         }
 
         @Override
-        public void applyStartedShards(StartedRerouteAllocation allocation) {
+        public void applyStartedShards(RoutingAllocation allocation, List<ShardRouting> startedShards) {
             // noop
         }
 
         @Override
-        public void applyFailedShards(FailedRerouteAllocation allocation) {
+        public void applyFailedShards(RoutingAllocation allocation, List<FailedShard> failedShards) {
             // noop
         }
 
diff --git a/distribution/licenses/HdrHistogram-NOTICE.txt b/bootstrap.memory_lock
similarity index 100%
rename from distribution/licenses/HdrHistogram-NOTICE.txt
rename to bootstrap.memory_lock
diff --git a/build.gradle b/build.gradle
index f1b57d7857bfc..2c7fff04e53ea 100644
--- a/build.gradle
+++ b/build.gradle
@@ -17,17 +17,26 @@
  * under the License.
  */
 
-import com.bmuschko.gradle.nexus.NexusPlugin
+import java.util.regex.Matcher
+import java.nio.file.Path
 import org.eclipse.jgit.lib.Repository
 import org.eclipse.jgit.lib.RepositoryBuilder
 import org.gradle.plugins.ide.eclipse.model.SourceFolder
 import org.apache.tools.ant.taskdefs.condition.Os
+import org.elasticsearch.gradle.BuildPlugin
+import org.elasticsearch.gradle.VersionProperties
+import org.elasticsearch.gradle.Version
 
 // common maven publishing configuration
 subprojects {
   group = 'org.elasticsearch'
   version = org.elasticsearch.gradle.VersionProperties.elasticsearch
   description = "Elasticsearch subproject ${project.path}"
+}
+
+Path rootPath = rootDir.toPath()
+// setup pom license info, but only for artifacts that are part of elasticsearch
+configure(subprojects.findAll { it.projectDir.toPath().startsWith(rootPath) }) {
 
   // we only use maven publish to add tasks for pom generation
   plugins.withType(MavenPublishPlugin).whenPluginAdded {
@@ -52,89 +61,128 @@ subprojects {
       }
     }
   }
+  plugins.withType(BuildPlugin).whenPluginAdded {
+    project.licenseFile = project.rootProject.file('LICENSE.txt')
+    project.noticeFile = project.rootProject.file('NOTICE.txt')
+  }
+}
 
-  plugins.withType(NexusPlugin).whenPluginAdded {
-    modifyPom {
-      project {
-        url 'https://github.com/elastic/elasticsearch'
-        inceptionYear '2009'
-
-        scm {
-          url 'https://github.com/elastic/elasticsearch'
-          connection 'scm:https://elastic@github.com/elastic/elasticsearch'
-          developerConnection 'scm:git://github.com/elastic/elasticsearch.git'
-        }
-
-        licenses {
-          license {
-            name 'The Apache Software License, Version 2.0'
-            url 'http://www.apache.org/licenses/LICENSE-2.0.txt'
-            distribution 'repo'
-          }
-        }
-      }
+// introspect all versions of ES that may be tested agains for backwards compatibility
+Version currentVersion = Version.fromString(VersionProperties.elasticsearch.minus('-SNAPSHOT'))
+File versionFile = file('core/src/main/java/org/elasticsearch/Version.java')
+List<String> versionLines = versionFile.readLines('UTF-8')
+List<Version> versions = []
+// keep track of the major version's first version, so we know where wire compat begins
+int firstMajorIndex = -1 // index in the versions list of the first minor from this major
+for (String line : versionLines) {
+  Matcher match = line =~ /\W+public static final Version V_(\d+)_(\d+)_(\d+)(_UNRELEASED)? .*/
+  if (match.matches()) {
+    int major = Integer.parseInt(match.group(1))
+    int minor = Integer.parseInt(match.group(2))
+    int bugfix = Integer.parseInt(match.group(3))
+    boolean unreleased = match.group(4) != null
+    Version foundVersion = new Version(major, minor, bugfix, false, unreleased)
+    if (currentVersion != foundVersion) {
+      versions.add(foundVersion)
     }
-    extraArchive {
-      javadoc = true
-      tests = false
-    }
-    nexus {
-      String buildSnapshot = System.getProperty('build.snapshot', 'true')
-      if (buildSnapshot == 'false') {
-        Repository repo = new RepositoryBuilder().findGitDir(project.rootDir).build()
-        String shortHash = repo.resolve('HEAD')?.name?.substring(0,7)
-        repositoryUrl = project.hasProperty('build.repository') ? project.property('build.repository') : "file://${System.getenv('HOME')}/elasticsearch-releases/${version}-${shortHash}/"
-      }
-    }
-    // we have our own username/password prompts so that they only happen once
-    // TODO: add gpg signing prompts, which is tricky, as the buildDeb/buildRpm tasks are executed before this code block
-    project.gradle.taskGraph.whenReady { taskGraph ->
-      if (taskGraph.allTasks.any { it.name == 'uploadArchives' }) {
-        Console console = System.console()
-        // no need for username/password on local deploy
-        if (project.nexus.repositoryUrl.startsWith('file://')) {
-          project.rootProject.allprojects.each {
-            it.ext.nexusUsername = 'foo'
-            it.ext.nexusPassword = 'bar'
-          }
-        } else {
-          if (project.hasProperty('nexusUsername') == false) {
-            String nexusUsername = console.readLine('\nNexus username: ')
-            project.rootProject.allprojects.each {
-              it.ext.nexusUsername = nexusUsername
-            }
-          }
-          if (project.hasProperty('nexusPassword') == false) {
-            String nexusPassword = new String(console.readPassword('\nNexus password: '))
-            project.rootProject.allprojects.each {
-              it.ext.nexusPassword = nexusPassword
-            }
-          }
-        }
-      }
+    if (major == currentVersion.major && firstMajorIndex == -1) {
+      firstMajorIndex = versions.size() - 1
     }
   }
 }
+if (versions.toSorted { it.id } != versions) {
+  println "Versions: ${versions}"
+  throw new GradleException("Versions.java contains out of order version constants")
+}
+if (currentVersion.bugfix == 0) {
+  // If on a release branch, after the initial release of that branch, the bugfix version will
+  // be bumped, and will be != 0. On master and N.x branches, we want to test against the
+  // unreleased version of closest branch. So for those cases, the version includes -SNAPSHOT,
+  // and the bwc distribution will checkout and build that version.
+  Version last = versions[-1]
+  versions[-1] = new Version(last.major, last.minor, last.bugfix,
+      true, last.unreleased)
+}
+
+// build metadata from previous build, contains eg hashes for bwc builds
+String buildMetadataValue = System.getenv('BUILD_METADATA')
+if (buildMetadataValue == null) {
+  buildMetadataValue = ''
+}
+Map<String, String> buildMetadataMap = buildMetadataValue.tokenize(';').collectEntries {
+  def (String key, String value) = it.split('=')
+  return [key, value]
+}
 
+// injecting groovy property variables into all projects
 allprojects {
   // injecting groovy property variables into all projects
   project.ext {
     // for ide hacks...
     isEclipse = System.getProperty("eclipse.launcher") != null || gradle.startParameter.taskNames.contains('eclipse') || gradle.startParameter.taskNames.contains('cleanEclipse')
     isIdea = System.getProperty("idea.active") != null || gradle.startParameter.taskNames.contains('idea') || gradle.startParameter.taskNames.contains('cleanIdea')
+    // for backcompat testing
+    indexCompatVersions = versions
+    wireCompatVersions = versions.subList(firstMajorIndex, versions.size())
+    buildMetadata = buildMetadataMap
+  }
+}
+
+task verifyVersions {
+  doLast {
+    if (gradle.startParameter.isOffline()) {
+      throw new GradleException("Must run in online mode to verify versions")
+    }
+    // Read the list from maven central
+    Node xml
+    new URL('https://repo1.maven.org/maven2/org/elasticsearch/elasticsearch/maven-metadata.xml').openStream().withStream { s ->
+        xml = new XmlParser().parse(s)
+    }
+    Set<Version> knownVersions = new TreeSet<>(xml.versioning.versions.version.collect { it.text() }.findAll { it ==~ /\d\.\d\.\d/ }.collect { Version.fromString(it) })
+
+    // Limit the known versions to those that should be index compatible, and are not future versions
+    knownVersions = knownVersions.findAll { it.major >= 2 && it.before(VersionProperties.elasticsearch) }
+
+    /* Limit the listed versions to those that have been marked as released.
+     * Versions not marked as released don't get the same testing and we want
+     * to make sure that we flip all unreleased versions to released as soon
+     * as possible after release. */
+    Set<Version> actualVersions = new TreeSet<>(indexCompatVersions.findAll { false == it.snapshot })
+
+    // Finally, compare!
+    if (knownVersions.equals(actualVersions) == false) {
+      throw new GradleException("out-of-date released versions\nActual  :" + actualVersions + "\nExpected:" + knownVersions +
+        "\nUpdate Version.java. Note that Version.CURRENT doesn't count because it is not released.")
+    }
+  }
+}
+
+/*
+ * When adding backcompat behavior that spans major versions, temporarily
+ * disabling the backcompat tests is necessary. This flag controls
+ * the enabled state of every bwc task. It should be set back to true
+ * after the backport of the backcompat code is complete.
+ */
+allprojects {
+  ext.bwc_tests_enabled = true
+}
+
+task verifyBwcTestsEnabled {
+  doLast {
+    if (project.bwc_tests_enabled == false) {
+      throw new GradleException('Bwc tests are disabled. They must be re-enabled after completing backcompat behavior backporting.')
+    }
   }
 }
 
+task branchConsistency {
+  description 'Ensures this branch is internally consistent. For example, that versions constants match released versions.'
+  group 'Verification'
+  dependsOn verifyVersions, verifyBwcTestsEnabled
+}
+
 subprojects {
   project.afterEvaluate {
-    // include license and notice in jars
-    tasks.withType(Jar) {
-      into('META-INF') {
-        from project.rootProject.rootDir
-        include 'LICENSE.txt'
-        include 'NOTICE.txt'
-      }
-    }
     // ignore missing javadocs
     tasks.withType(Javadoc) { Javadoc javadoc ->
       // the -quiet here is because of a bug in gradle, in that adding a string option
@@ -163,8 +211,9 @@ subprojects {
     "org.elasticsearch.gradle:build-tools:${version}": ':build-tools',
     "org.elasticsearch:rest-api-spec:${version}": ':rest-api-spec',
     "org.elasticsearch:elasticsearch:${version}": ':core',
-    "org.elasticsearch.client:rest:${version}": ':client:rest',
-    "org.elasticsearch.client:sniffer:${version}": ':client:sniffer',
+    "org.elasticsearch.client:elasticsearch-rest-client:${version}": ':client:rest',
+    "org.elasticsearch.client:elasticsearch-rest-client-sniffer:${version}": ':client:sniffer',
+    "org.elasticsearch.client:elasticsearch-rest-high-level-client:${version}": ':client:rest-high-level',
     "org.elasticsearch.client:test:${version}": ':client:test',
     "org.elasticsearch.client:transport:${version}": ':client:transport',
     "org.elasticsearch.test:framework:${version}": ':test:framework',
@@ -179,12 +228,23 @@ subprojects {
     "org.elasticsearch.plugin:transport-netty4-client:${version}": ':modules:transport-netty4',
     "org.elasticsearch.plugin:reindex-client:${version}": ':modules:reindex',
     "org.elasticsearch.plugin:lang-mustache-client:${version}": ':modules:lang-mustache',
+    "org.elasticsearch.plugin:parent-join-client:${version}": ':modules:parent-join',
+    "org.elasticsearch.plugin:aggs-matrix-stats-client:${version}": ':modules:aggs-matrix-stats',
     "org.elasticsearch.plugin:percolator-client:${version}": ':modules:percolator',
   ]
-  configurations.all {
-    resolutionStrategy.dependencySubstitution { DependencySubstitutions subs ->
-      projectSubstitutions.each { k,v ->
-        subs.substitute(subs.module(k)).with(subs.project(v))
+  if (wireCompatVersions[-1].snapshot) {
+    // if the most previous version is a snapshot, we need to connect that version to the
+    // bwc project which will checkout and build that snapshot version
+    ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch:${wireCompatVersions[-1]}"] = ':distribution:bwc'
+    ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch:${wireCompatVersions[-1]}"] = ':distribution:bwc'
+    ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${wireCompatVersions[-1]}"] = ':distribution:bwc'
+  }
+  project.afterEvaluate {
+    configurations.all {
+      resolutionStrategy.dependencySubstitution { DependencySubstitutions subs ->
+        projectSubstitutions.each { k,v ->
+          subs.substitute(subs.module(k)).with(subs.project(v))
+        }
       }
     }
   }
@@ -341,3 +401,17 @@ task run(type: Run) {
   group = 'Verification'
   impliesSubProjects = true
 }
+
+/* Remove assemble on all qa projects because we don't need to publish
+ * artifacts for them. */
+gradle.projectsEvaluated {
+  subprojects {
+    if (project.path.startsWith(':qa')) {
+      Task assemble = project.tasks.findByName('assemble')
+      if (assemble) {
+        project.tasks.remove(assemble)
+        project.build.dependsOn.remove('assemble')
+      }
+    }
+  }
+}
diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle
index 1be5020f4f813..73115aab88fd3 100644
--- a/buildSrc/build.gradle
+++ b/buildSrc/build.gradle
@@ -23,14 +23,12 @@ apply plugin: 'groovy'
 
 group = 'org.elasticsearch.gradle'
 
-// TODO: remove this when upgrading to a version that supports ProgressLogger
-// gradle 2.14 made internal apis unavailable to plugins, and gradle considered
-// ProgressLogger to be an internal api. Until this is made available again,
-// we can't upgrade without losing our nice progress logging
-// NOTE that this check duplicates that in BuildPlugin, but we need to check
-// early here before trying to compile the broken classes in buildSrc
-if (GradleVersion.current() != GradleVersion.version('2.13')) {
-  throw new GradleException('Gradle 2.13 is required to build elasticsearch')
+if (GradleVersion.current() < GradleVersion.version('3.3')) {
+  throw new GradleException('Gradle 3.3+ is required to build elasticsearch')
+}
+
+if (JavaVersion.current() < JavaVersion.VERSION_1_8) {
+  throw new GradleException('Java 1.8 is required to build elasticsearch gradle tools')
 }
 
 if (project == rootProject) {
@@ -94,12 +92,18 @@ dependencies {
   compile 'com.netflix.nebula:gradle-info-plugin:3.0.3'
   compile 'org.eclipse.jgit:org.eclipse.jgit:3.2.0.201312181205-r'
   compile 'com.perforce:p4java:2012.3.551082' // THIS IS SUPPOSED TO BE OPTIONAL IN THE FUTURE....
-  compile 'de.thetaphi:forbiddenapis:2.2'
-  compile 'com.bmuschko:gradle-nexus-plugin:2.3.1'
+  compile 'de.thetaphi:forbiddenapis:2.3'
   compile 'org.apache.rat:apache-rat:0.11'
-  compile 'ru.vyarus:gradle-animalsniffer-plugin:1.0.1'
+  compile "org.elasticsearch:jna:4.4.0-1"
 }
 
+// Gradle 2.14+ removed ProgressLogger(-Factory) classes from the public APIs
+// Use logging dependency instead
+
+dependencies {
+  compileOnly "org.gradle:gradle-logging:${GradleVersion.current().getVersion()}"
+  compile 'ru.vyarus:gradle-animalsniffer-plugin:1.2.0' // Gradle 2.14 requires a version > 1.0.1
+}
 
 /*****************************************************************************
  *                         Bootstrap repositories                            *
@@ -108,11 +112,10 @@ dependencies {
 if (project == rootProject) {
 
   repositories {
-    mavenCentral()
-    maven {
-      name 'sonatype-snapshots'
-      url "https://oss.sonatype.org/content/repositories/snapshots/"
+    if (System.getProperty("repos.mavenLocal") != null) {
+      mavenLocal()
     }
+    mavenCentral()
   }
   test.exclude 'org/elasticsearch/test/NamingConventionsCheckBadClasses*'
 }
@@ -154,4 +157,11 @@ if (project != rootProject) {
     testClass = 'org.elasticsearch.test.NamingConventionsCheckBadClasses$UnitTestCase'
     integTestClass = 'org.elasticsearch.test.NamingConventionsCheckBadClasses$IntegTestCase'
   }
+
+  task namingConventionsMain(type: org.elasticsearch.gradle.precommit.NamingConventionsTask) {
+    checkForTestsInMain = true
+    testClass = namingConventions.testClass
+    integTestClass = namingConventions.integTestClass
+  }
+  precommit.dependsOn namingConventionsMain
 }
diff --git a/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingPlugin.groovy b/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingPlugin.groovy
index e2230b116c714..c811a4f6c268e 100644
--- a/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingPlugin.groovy
+++ b/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingPlugin.groovy
@@ -33,7 +33,7 @@ class RandomizedTestingPlugin implements Plugin<Project> {
         ]
         RandomizedTestingTask newTestTask = tasks.create(properties)
         newTestTask.classpath = oldTestTask.classpath
-        newTestTask.testClassesDir = oldTestTask.testClassesDir
+        newTestTask.testClassesDir = oldTestTask.project.sourceSets.test.output.classesDir
 
         // hack so check task depends on custom test
         Task checkTask = tasks.findByPath('check')
diff --git a/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingTask.groovy b/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingTask.groovy
index b28e7210ea41d..e24c226837d26 100644
--- a/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingTask.groovy
+++ b/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingTask.groovy
@@ -19,7 +19,7 @@ import org.gradle.api.tasks.Optional
 import org.gradle.api.tasks.TaskAction
 import org.gradle.api.tasks.util.PatternFilterable
 import org.gradle.api.tasks.util.PatternSet
-import org.gradle.logging.ProgressLoggerFactory
+import org.gradle.internal.logging.progress.ProgressLoggerFactory
 import org.gradle.util.ConfigureUtil
 
 import javax.inject.Inject
@@ -69,6 +69,10 @@ class RandomizedTestingTask extends DefaultTask {
     @Input
     String ifNoTests = 'ignore'
 
+    @Optional
+    @Input
+    String onNonEmptyWorkDirectory = 'fail'
+
     TestLoggingConfiguration testLoggingConfig = new TestLoggingConfiguration()
 
     BalancersConfiguration balancersConfig = new BalancersConfiguration(task: this)
@@ -81,6 +85,7 @@ class RandomizedTestingTask extends DefaultTask {
     String argLine = null
 
     Map<String, Object> systemProperties = new HashMap<>()
+    Map<String, Object> environmentVariables = new HashMap<>()
     PatternFilterable patternSet = new PatternSet()
 
     RandomizedTestingTask() {
@@ -91,7 +96,7 @@ class RandomizedTestingTask extends DefaultTask {
 
     @Inject
     ProgressLoggerFactory getProgressLoggerFactory() {
-        throw new UnsupportedOperationException();
+        throw new UnsupportedOperationException()
     }
 
     void jvmArgs(Iterable<String> arguments) {
@@ -106,6 +111,10 @@ class RandomizedTestingTask extends DefaultTask {
         systemProperties.put(property, value)
     }
 
+    void environment(String key, Object value) {
+        environmentVariables.put(key, value)
+    }
+
     void include(String... includes) {
         this.patternSet.include(includes);
     }
@@ -194,7 +203,9 @@ class RandomizedTestingTask extends DefaultTask {
             haltOnFailure: true, // we want to capture when a build failed, but will decide whether to rethrow later
             shuffleOnSlave: shuffleOnSlave,
             leaveTemporary: leaveTemporary,
-            ifNoTests: ifNoTests
+            ifNoTests: ifNoTests,
+            onNonEmptyWorkDirectory: onNonEmptyWorkDirectory,
+            newenvironment: true
         ]
 
         DefaultLogger listener = null
@@ -250,6 +261,9 @@ class RandomizedTestingTask extends DefaultTask {
                 for (Map.Entry<String, Object> prop : systemProperties) {
                     sysproperty key: prop.getKey(), value: prop.getValue().toString()
                 }
+                for (Map.Entry<String, Object> envvar : environmentVariables) {
+                    env key: envvar.getKey(), value: envvar.getValue().toString()
+                }
                 makeListeners()
             }
         } catch (BuildException e) {
diff --git a/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/TestProgressLogger.groovy b/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/TestProgressLogger.groovy
index 14f5d476be3cb..da25afa938916 100644
--- a/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/TestProgressLogger.groovy
+++ b/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/TestProgressLogger.groovy
@@ -25,8 +25,8 @@ import com.carrotsearch.ant.tasks.junit4.events.aggregated.AggregatedStartEvent
 import com.carrotsearch.ant.tasks.junit4.events.aggregated.AggregatedSuiteResultEvent
 import com.carrotsearch.ant.tasks.junit4.events.aggregated.AggregatedTestResultEvent
 import com.carrotsearch.ant.tasks.junit4.listeners.AggregatedEventListener
-import org.gradle.logging.ProgressLogger
-import org.gradle.logging.ProgressLoggerFactory
+import org.gradle.internal.logging.progress.ProgressLogger
+import org.gradle.internal.logging.progress.ProgressLoggerFactory
 
 import static com.carrotsearch.ant.tasks.junit4.FormattingUtils.formatDurationInSeconds
 import static com.carrotsearch.ant.tasks.junit4.events.aggregated.TestStatus.ERROR
@@ -77,7 +77,7 @@ class TestProgressLogger implements AggregatedEventListener {
     /** Have we finished a whole suite yet? */
     volatile boolean suiteFinished = false
     /* Note that we probably overuse volatile here but it isn't hurting us and
-      lets us move things around without worying about breaking things. */
+       lets us move things around without worrying about breaking things. */
 
     @Subscribe
     void onStart(AggregatedStartEvent e) throws IOException {
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
index 4d7bee866b824..c51fc0229eebe 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
@@ -18,9 +18,12 @@
  */
 package org.elasticsearch.gradle
 
+import com.carrotsearch.gradle.junit4.RandomizedTestingTask
 import nebula.plugin.extraconfigurations.ProvidedBasePlugin
+import org.apache.tools.ant.taskdefs.condition.Os
 import org.elasticsearch.gradle.precommit.PrecommitTasks
 import org.gradle.api.GradleException
+import org.gradle.api.InvalidUserDataException
 import org.gradle.api.JavaVersion
 import org.gradle.api.Plugin
 import org.gradle.api.Project
@@ -32,19 +35,20 @@ import org.gradle.api.artifacts.ModuleVersionIdentifier
 import org.gradle.api.artifacts.ProjectDependency
 import org.gradle.api.artifacts.ResolvedArtifact
 import org.gradle.api.artifacts.dsl.RepositoryHandler
-import org.gradle.api.artifacts.maven.MavenPom
+import org.gradle.api.file.CopySpec
+import org.gradle.api.plugins.JavaPlugin
 import org.gradle.api.publish.maven.MavenPublication
 import org.gradle.api.publish.maven.plugins.MavenPublishPlugin
 import org.gradle.api.publish.maven.tasks.GenerateMavenPom
 import org.gradle.api.tasks.bundling.Jar
 import org.gradle.api.tasks.compile.JavaCompile
+import org.gradle.api.tasks.javadoc.Javadoc
 import org.gradle.internal.jvm.Jvm
 import org.gradle.process.ExecResult
 import org.gradle.util.GradleVersion
 
 import java.time.ZoneOffset
 import java.time.ZonedDateTime
-
 /**
  * Encapsulates build configuration for elasticsearch projects.
  */
@@ -54,6 +58,11 @@ class BuildPlugin implements Plugin<Project> {
 
     @Override
     void apply(Project project) {
+        if (project.pluginManager.hasPlugin('elasticsearch.standalone-rest-test')) {
+              throw new InvalidUserDataException('elasticsearch.standalone-test, '
+                + 'elasticearch.standalone-rest-test, and elasticsearch.build '
+                + 'are mutually exclusive')
+        }
         project.pluginManager.apply('java')
         project.pluginManager.apply('carrotsearch.randomized-testing')
         // these plugins add lots of info to our jars
@@ -63,7 +72,6 @@ class BuildPlugin implements Plugin<Project> {
         project.pluginManager.apply('nebula.info-java')
         project.pluginManager.apply('nebula.info-scm')
         project.pluginManager.apply('nebula.info-jar')
-        project.pluginManager.apply('com.bmuschko.nexus')
         project.pluginManager.apply(ProvidedBasePlugin)
 
         globalBuildInfo(project)
@@ -71,6 +79,8 @@ class BuildPlugin implements Plugin<Project> {
         configureConfigurations(project)
         project.ext.versions = VersionProperties.versions
         configureCompile(project)
+        configureJavadoc(project)
+        configureSourcesJar(project)
         configurePomGeneration(project)
 
         configureTest(project)
@@ -113,7 +123,7 @@ class BuildPlugin implements Plugin<Project> {
             }
 
             // enforce gradle version
-            GradleVersion minGradle = GradleVersion.version('2.13')
+            GradleVersion minGradle = GradleVersion.version('3.3')
             if (GradleVersion.current() < minGradle) {
                 throw new GradleException("${minGradle} or above is required to build elasticsearch")
             }
@@ -157,7 +167,7 @@ class BuildPlugin implements Plugin<Project> {
     private static String findJavaHome() {
         String javaHome = System.getenv('JAVA_HOME')
         if (javaHome == null) {
-            if (System.getProperty("idea.active") != null) {
+            if (System.getProperty("idea.active") != null || System.getProperty("eclipse.launcher") != null) {
                 // intellij doesn't set JAVA_HOME, so we use the jdk gradle was run with
                 javaHome = Jvm.current().javaHome
             } else {
@@ -194,19 +204,28 @@ class BuildPlugin implements Plugin<Project> {
 
     /** Runs the given javascript using jjs from the jdk, and returns the output */
     private static String runJavascript(Project project, String javaHome, String script) {
-        File tmpScript = File.createTempFile('es-gradle-tmp', '.js')
-        tmpScript.setText(script, 'UTF-8')
-        ByteArrayOutputStream output = new ByteArrayOutputStream()
+        ByteArrayOutputStream stdout = new ByteArrayOutputStream()
+        ByteArrayOutputStream stderr = new ByteArrayOutputStream()
+        if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+            // gradle/groovy does not properly escape the double quote for windows
+            script = script.replace('"', '\\"')
+        }
+        File jrunscriptPath = new File(javaHome, 'bin/jrunscript')
         ExecResult result = project.exec {
-            executable = new File(javaHome, 'bin/jjs')
-            args tmpScript.toString()
-            standardOutput = output
-            errorOutput = new ByteArrayOutputStream()
-            ignoreExitValue = true // we do not fail so we can first cleanup the tmp file
+            executable = jrunscriptPath
+            args '-e', script
+            standardOutput = stdout
+            errorOutput = stderr
+            ignoreExitValue = true
         }
-        java.nio.file.Files.delete(tmpScript.toPath())
-        result.assertNormalExitValue()
-        return output.toString('UTF-8').trim()
+        if (result.exitValue != 0) {
+            project.logger.error("STDOUT:")
+            stdout.toString('UTF-8').eachLine { line -> project.logger.error(line) }
+            project.logger.error("STDERR:")
+            stderr.toString('UTF-8').eachLine { line -> project.logger.error(line) }
+            result.rethrowFailure()
+        }
+        return stdout.toString('UTF-8').trim()
     }
 
     /** Return the configuration name used for finding transitive deps of the given dependency. */
@@ -267,14 +286,9 @@ class BuildPlugin implements Plugin<Project> {
         project.configurations.compile.dependencies.all(disableTransitiveDeps)
         project.configurations.testCompile.dependencies.all(disableTransitiveDeps)
         project.configurations.provided.dependencies.all(disableTransitiveDeps)
-
-        // add exclusions to the pom directly, for each of the transitive deps of this project's deps
-        project.modifyPom { MavenPom pom ->
-            pom.withXml(fixupDependencies(project))
-        }
     }
 
-    /** Adds repositores used by ES dependencies */
+    /** Adds repositories used by ES dependencies */
     static void configureRepositories(Project project) {
         RepositoryHandler repos = project.repositories
         if (System.getProperty("repos.mavenlocal") != null) {
@@ -284,10 +298,6 @@ class BuildPlugin implements Plugin<Project> {
             repos.mavenLocal()
         }
         repos.mavenCentral()
-        repos.maven {
-            name 'sonatype-snapshots'
-            url 'http://oss.sonatype.org/content/repositories/snapshots/'
-        }
         String luceneVersion = VersionProperties.lucene
         if (luceneVersion.contains('-snapshot')) {
             // extract the revision number from the version with a regex matcher
@@ -303,12 +313,14 @@ class BuildPlugin implements Plugin<Project> {
      * Returns a closure which can be used with a MavenPom for fixing problems with gradle generated poms.
      *
      * <ul>
-     *     <li>Remove transitive dependencies (using wildcard exclusions, fixed in gradle 2.14)</li>
-     *     <li>Set compile time deps back to compile from runtime (known issue with maven-publish plugin)
+     *     <li>Remove transitive dependencies. We currently exclude all artifacts explicitly instead of using wildcards
+     *         as Ivy incorrectly translates POMs with * excludes to Ivy XML with * excludes which results in the main artifact
+     *         being excluded as well (see https://issues.apache.org/jira/browse/IVY-1531). Note that Gradle 2.14+ automatically
+     *         translates non-transitive dependencies to * excludes. We should revisit this when upgrading Gradle.</li>
+     *     <li>Set compile time deps back to compile from runtime (known issue with maven-publish plugin)</li>
      * </ul>
      */
     private static Closure fixupDependencies(Project project) {
-        // TODO: remove this when enforcing gradle 2.14+, it now properly handles exclusions
         return { XmlProvider xml ->
             // first find if we have dependencies at all, and grab the node
             NodeList depsNodes = xml.asNode().get('dependencies')
@@ -331,6 +343,13 @@ class BuildPlugin implements Plugin<Project> {
                     depNode.scope*.value = 'compile'
                 }
 
+                // remove any exclusions added by gradle, they contain wildcards and systems like ivy have bugs with wildcards
+                // see https://github.com/elastic/elasticsearch/issues/24490
+                NodeList exclusionsNode = depNode.get('exclusions')
+                if (exclusionsNode.size() > 0) {
+                    depNode.remove(exclusionsNode.get(0))
+                }
+
                 // collect the transitive deps now that we know what this dependency is
                 String depConfig = transitiveDepConfigName(groupId, artifactId, version)
                 Configuration configuration = project.configurations.findByName(depConfig)
@@ -343,10 +362,19 @@ class BuildPlugin implements Plugin<Project> {
                     continue
                 }
 
-                // we now know we have something to exclude, so add a wildcard exclusion element
-                Node exclusion = depNode.appendNode('exclusions').appendNode('exclusion')
-                exclusion.appendNode('groupId', '*')
-                exclusion.appendNode('artifactId', '*')
+                // we now know we have something to exclude, so add exclusions for all artifacts except the main one
+                Node exclusions = depNode.appendNode('exclusions')
+                for (ResolvedArtifact artifact : artifacts) {
+                    ModuleVersionIdentifier moduleVersionIdentifier = artifact.moduleVersion.id;
+                    String depGroupId = moduleVersionIdentifier.group
+                    String depArtifactId = moduleVersionIdentifier.name
+                    // add exclusions for all artifacts except the main one
+                    if (depGroupId != groupId || depArtifactId != artifactId) {
+                        Node exclusion = exclusions.appendNode('exclusion')
+                        exclusion.appendNode('groupId', depGroupId)
+                        exclusion.appendNode('artifactId', depArtifactId)
+                    }
+                }
             }
         }
     }
@@ -366,8 +394,11 @@ class BuildPlugin implements Plugin<Project> {
             project.tasks.withType(GenerateMavenPom.class) { GenerateMavenPom t ->
                 // place the pom next to the jar it is for
                 t.destination = new File(project.buildDir, "distributions/${project.archivesBaseName}-${project.version}.pom")
-                // build poms with assemble
-                project.assemble.dependsOn(t)
+                // build poms with assemble (if the assemble task exists)
+                Task assemble = project.tasks.findByName('assemble')
+                if (assemble) {
+                    assemble.dependsOn(t)
+                }
             }
         }
     }
@@ -376,8 +407,9 @@ class BuildPlugin implements Plugin<Project> {
     static void configureCompile(Project project) {
         project.ext.compactProfile = 'compact3'
         project.afterEvaluate {
-            // fail on all javac warnings
             project.tasks.withType(JavaCompile) {
+                File gradleJavaHome = Jvm.current().javaHome
+                // we fork because compiling lots of different classes in a shared jvm can eventually trigger GC overhead limitations
                 options.fork = true
                 options.forkOptions.executable = new File(project.javaHome, 'bin/javac')
                 options.forkOptions.memoryMaximumSize = "1g"
@@ -394,6 +426,7 @@ class BuildPlugin implements Plugin<Project> {
                  * -serial because we don't use java serialization.
                  */
                 // don't even think about passing args with -J-xxx, oracle will ask you to submit a bug report :)
+                // fail on all javac warnings
                 options.compilerArgs << '-Werror' << '-Xlint:all,-path,-serial,-options,-deprecation' << '-Xdoclint:all' << '-Xdoclint:-missing'
 
                 // either disable annotation processor completely (default) or allow to enable them if an annotation processor is explicitly defined
@@ -402,21 +435,74 @@ class BuildPlugin implements Plugin<Project> {
                 }
 
                 options.encoding = 'UTF-8'
-                //options.incremental = true
+                options.incremental = true
 
                 if (project.javaVersion == JavaVersion.VERSION_1_9) {
-                    // hack until gradle supports java 9's new "-release" arg
+                    // hack until gradle supports java 9's new "--release" arg
                     assert minimumJava == JavaVersion.VERSION_1_8
-                    options.compilerArgs << '-release' << '8'
-                    project.sourceCompatibility = null
-                    project.targetCompatibility = null
+                    options.compilerArgs << '--release' << '8'
+                    if (GradleVersion.current().getBaseVersion() < GradleVersion.version("4.1")) {
+                        // this hack is not needed anymore since Gradle 4.1, see https://github.com/gradle/gradle/pull/2474
+                        doFirst {
+                            sourceCompatibility = null
+                            targetCompatibility = null
+                        }
+                    }
                 }
             }
         }
     }
 
-    /** Adds additional manifest info to jars, and adds source and javadoc jars */
+    static void configureJavadoc(Project project) {
+        String artifactsHost = VersionProperties.elasticsearch.endsWith("-SNAPSHOT") ? "https://snapshots.elastic.co" : "https://artifacts.elastic.co"
+        project.afterEvaluate {
+            project.tasks.withType(Javadoc) {
+                executable = new File(project.javaHome, 'bin/javadoc')
+            }
+            /*
+             * Order matters, the linksOffline for org.elasticsearch:elasticsearch must be the last one
+             * or all the links for the other packages (e.g org.elasticsearch.client) will point to core rather than their own artifacts
+             */
+            Closure sortClosure = { a, b -> b.group <=> a.group }
+            Closure depJavadocClosure = { dep ->
+                if (dep.group != null && dep.group.startsWith('org.elasticsearch')) {
+                    String substitution = project.ext.projectSubstitutions.get("${dep.group}:${dep.name}:${dep.version}")
+                    if (substitution != null) {
+                        project.javadoc.dependsOn substitution + ':javadoc'
+                        String artifactPath = dep.group.replaceAll('\\.', '/') + '/' + dep.name.replaceAll('\\.', '/') + '/' + dep.version
+                        project.javadoc.options.linksOffline artifactsHost + "/javadoc/" + artifactPath, "${project.project(substitution).buildDir}/docs/javadoc/"
+                    }
+                }
+            }
+            project.configurations.compile.dependencies.findAll().toSorted(sortClosure).each(depJavadocClosure)
+            project.configurations.provided.dependencies.findAll().toSorted(sortClosure).each(depJavadocClosure)
+        }
+        configureJavadocJar(project)
+    }
+
+    /** Adds a javadocJar task to generate a jar containing javadocs. */
+    static void configureJavadocJar(Project project) {
+        Jar javadocJarTask = project.task('javadocJar', type: Jar)
+        javadocJarTask.classifier = 'javadoc'
+        javadocJarTask.group = 'build'
+        javadocJarTask.description = 'Assembles a jar containing javadocs.'
+        javadocJarTask.from(project.tasks.getByName(JavaPlugin.JAVADOC_TASK_NAME))
+        project.assemble.dependsOn(javadocJarTask)
+    }
+
+    static void configureSourcesJar(Project project) {
+        Jar sourcesJarTask = project.task('sourcesJar', type: Jar)
+        sourcesJarTask.classifier = 'sources'
+        sourcesJarTask.group = 'build'
+        sourcesJarTask.description = 'Assembles a jar containing source files.'
+        sourcesJarTask.from(project.sourceSets.main.allSource)
+        project.assemble.dependsOn(sourcesJarTask)
+    }
+
+    /** Adds additional manifest info to jars */
     static void configureJars(Project project) {
+        project.ext.licenseFile = null
+        project.ext.noticeFile = null
         project.tasks.withType(Jar) { Jar jarTask ->
             // we put all our distributable files under distributions
             jarTask.destinationDir = new File(project.buildDir, 'distributions')
@@ -437,7 +523,21 @@ class BuildPlugin implements Plugin<Project> {
                         'Build-Java-Version': project.javaVersion)
                 if (jarTask.manifest.attributes.containsKey('Change') == false) {
                     logger.warn('Building without git revision id.')
-                    jarTask.manifest.attributes('Change': 'N/A')
+                    jarTask.manifest.attributes('Change': 'Unknown')
+                }
+            }
+            // add license/notice files
+            project.afterEvaluate {
+                if (project.licenseFile == null || project.noticeFile == null) {
+                    throw new GradleException("Must specify license and notice file for project ${project.path}")
+                }
+                jarTask.into('META-INF') {
+                    from(project.licenseFile.parent) {
+                        include project.licenseFile.name
+                    }
+                    from(project.noticeFile.parent) {
+                        include project.noticeFile.name
+                    }
                 }
             }
         }
@@ -449,16 +549,12 @@ class BuildPlugin implements Plugin<Project> {
             jvm "${project.javaHome}/bin/java"
             parallelism System.getProperty('tests.jvms', 'auto')
             ifNoTests 'fail'
+            onNonEmptyWorkDirectory 'wipe'
             leaveTemporary true
 
             // TODO: why are we not passing maxmemory to junit4?
             jvmArg '-Xmx' + System.getProperty('tests.heap.size', '512m')
             jvmArg '-Xms' + System.getProperty('tests.heap.size', '512m')
-            if (JavaVersion.current().isJava7()) {
-                // some tests need a large permgen, but that only exists on java 7
-                jvmArg '-XX:MaxPermSize=128m'
-            }
-            jvmArg '-XX:MaxDirectMemorySize=512m'
             jvmArg '-XX:+HeapDumpOnOutOfMemoryError'
             File heapdumpDir = new File(project.buildDir, 'heapdump')
             heapdumpDir.mkdirs()
@@ -472,6 +568,8 @@ class BuildPlugin implements Plugin<Project> {
             systemProperty 'tests.artifact', project.name
             systemProperty 'tests.task', path
             systemProperty 'tests.security.manager', 'true'
+            // Breaking change in JDK-9, revert to JDK-8 behavior for now, see https://github.com/elastic/elasticsearch/issues/21534
+            systemProperty 'jdk.io.permissionsUseCanonicalPath', 'true'
             systemProperty 'jna.nosys', 'true'
             // default test sysprop values
             systemProperty 'tests.ifNoTests', 'fail'
@@ -484,11 +582,9 @@ class BuildPlugin implements Plugin<Project> {
                 }
             }
 
-            // System assertions (-esa) are disabled for now because of what looks like a
-            // JDK bug triggered by Groovy on JDK7. We should look at re-enabling system
-            // assertions when we upgrade to a new version of Groovy (currently 2.4.4) or
-            // require JDK8. See https://issues.apache.org/jira/browse/GROOVY-7528.
-            enableSystemAssertions false
+            boolean assertionsEnabled = Boolean.parseBoolean(System.getProperty('tests.asserts', 'true'))
+            enableSystemAssertions assertionsEnabled
+            enableAssertions assertionsEnabled
 
             testLogging {
                 showNumFailuresAtEnd 25
@@ -529,11 +625,22 @@ class BuildPlugin implements Plugin<Project> {
 
     /** Configures the test task */
     static Task configureTest(Project project) {
-        Task test = project.tasks.getByName('test')
+        RandomizedTestingTask test = project.tasks.getByName('test')
         test.configure(commonTestConfig(project))
         test.configure {
             include '**/*Tests.class'
         }
+
+        // Add a method to create additional unit tests for a project, which will share the same
+        // randomized testing setup, but by default run no tests.
+        project.extensions.add('additionalTest', { String name, Closure config ->
+            RandomizedTestingTask additionalTest = project.tasks.create(name, RandomizedTestingTask.class)
+            additionalTest.classpath = test.classpath
+            additionalTest.testClassesDir = test.testClassesDir
+            additionalTest.configure(commonTestConfig(project))
+            additionalTest.configure(config)
+            test.dependsOn(additionalTest)
+        });
         return test
     }
 
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/NoticeTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/NoticeTask.groovy
new file mode 100644
index 0000000000000..928298db7bfc2
--- /dev/null
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/NoticeTask.groovy
@@ -0,0 +1,99 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle
+
+import org.gradle.api.DefaultTask
+import org.gradle.api.Project
+import org.gradle.api.artifacts.Configuration
+import org.gradle.api.tasks.InputFile
+import org.gradle.api.tasks.OutputFile
+import org.gradle.api.tasks.TaskAction
+
+/**
+ * A task to create a notice file which includes dependencies' notices.
+ */
+public class NoticeTask extends DefaultTask {
+
+    @InputFile
+    File inputFile = project.rootProject.file('NOTICE.txt')
+
+    @OutputFile
+    File outputFile = new File(project.buildDir, "notices/${name}/NOTICE.txt")
+
+    /** Directories to include notices from */
+    private List<File> licensesDirs = new ArrayList<>()
+
+    public NoticeTask() {
+        description = 'Create a notice file from dependencies'
+        // Default licenses directory is ${projectDir}/licenses (if it exists)
+        File licensesDir = new File(project.projectDir, 'licenses')
+        if (licensesDir.exists()) {
+            licensesDirs.add(licensesDir)
+        }
+    }
+
+    /** Add notices from the specified directory. */
+    public void licensesDir(File licensesDir) {
+        licensesDirs.add(licensesDir)
+    }
+
+    @TaskAction
+    public void generateNotice() {
+        StringBuilder output = new StringBuilder()
+        output.append(inputFile.getText('UTF-8'))
+        output.append('\n\n')
+        // This is a map rather than a set so that the sort order is the 3rd
+        // party component names, unaffected by the full path to the various files
+        Map<String, File> seen = new TreeMap<>()
+        for (File licensesDir : licensesDirs) {
+            licensesDir.eachFileMatch({ it ==~ /.*-NOTICE\.txt/ }) { File file ->
+                String name = file.name.substring(0, file.name.length() - '-NOTICE.txt'.length())
+                if (seen.containsKey(name)) {
+                    File prevFile = seen.get(name)
+                    if (prevFile.text != file.text) {
+                        throw new RuntimeException("Two different notices exist for dependency '" +
+                                name + "': " + prevFile + " and " + file)
+                    }
+                } else {
+                    seen.put(name, file)
+                }
+            }
+        }
+        for (Map.Entry<String, File> entry : seen.entrySet()) {
+            String name = entry.getKey()
+            File file = entry.getValue()
+            appendFile(file, name, 'NOTICE', output)
+            appendFile(new File(file.parentFile, "${name}-LICENSE.txt"), name, 'LICENSE', output)
+        }
+        outputFile.setText(output.toString(), 'UTF-8')
+    }
+
+    static void appendFile(File file, String name, String type, StringBuilder output) {
+        String text = file.getText('UTF-8')
+        if (text.trim().isEmpty()) {
+            return
+        }
+        output.append('================================================================================\n')
+        output.append("${name} ${type}\n")
+        output.append('================================================================================\n')
+        output.append(text)
+        output.append('\n\n')
+    }
+}
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/Version.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/Version.groovy
new file mode 100644
index 0000000000000..ace8dd34fe9e9
--- /dev/null
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/Version.groovy
@@ -0,0 +1,84 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle
+
+import groovy.transform.Sortable
+
+/**
+ * Encapsulates comparison and printing logic for an x.y.z version.
+ */
+@Sortable(includes=['id'])
+public class Version {
+
+    final int major
+    final int minor
+    final int bugfix
+    final int id
+    final boolean snapshot
+    /**
+     * Is the vesion listed as {@code _UNRELEASED} in Version.java.
+     */
+    final boolean unreleased
+
+    public Version(int major, int minor, int bugfix, boolean snapshot,
+            boolean unreleased) {
+        this.major = major
+        this.minor = minor
+        this.bugfix = bugfix
+        this.snapshot = snapshot
+        this.id = major * 100000 + minor * 1000 + bugfix * 10 +
+            (snapshot ? 1 : 0)
+        this.unreleased = unreleased
+    }
+
+    public static Version fromString(String s) {
+        String[] parts = s.split('\\.')
+        String bugfix = parts[2]
+        boolean snapshot = false
+        if (bugfix.contains('-')) {
+            snapshot = bugfix.endsWith('-SNAPSHOT')
+            bugfix = bugfix.split('-')[0]
+        }
+        return new Version(parts[0] as int, parts[1] as int, bugfix as int,
+            snapshot, false)
+    }
+
+    @Override
+    public String toString() {
+        String snapshotStr = snapshot ? '-SNAPSHOT' : ''
+        return "${major}.${minor}.${bugfix}${snapshotStr}"
+    }
+
+    public boolean before(String compareTo) {
+        return id < fromString(compareTo).id
+    }
+
+    public boolean onOrBefore(String compareTo) {
+        return id <= fromString(compareTo).id
+    }
+
+    public boolean onOrAfter(String compareTo) {
+        return id >= fromString(compareTo).id
+    }
+
+    public boolean after(String compareTo) {
+        return id > fromString(compareTo).id
+    }
+}
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy
index 0fefecc144646..d2802638ce512 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy
@@ -30,7 +30,11 @@ public class DocsTestPlugin extends RestTestPlugin {
 
     @Override
     public void apply(Project project) {
+        project.pluginManager.apply('elasticsearch.standalone-rest-test')
         super.apply(project)
+        // Docs are published separately so no need to assemble
+        project.tasks.remove(project.assemble)
+        project.build.dependsOn.remove('assemble')
         Map<String, String> defaultSubstitutions = [
             /* These match up with the asciidoc syntax for substitutions but
              * the values may differ. In particular {version} needs to resolve
@@ -38,7 +42,7 @@ public class DocsTestPlugin extends RestTestPlugin {
              * the last released version for docs. */
             '\\{version\\}':
                 VersionProperties.elasticsearch.replace('-SNAPSHOT', ''),
-            '\\{lucene_version\\}' : VersionProperties.lucene,
+            '\\{lucene_version\\}' : VersionProperties.lucene.replaceAll('-snapshot-\\w+$', ''),
         ]
         Task listSnippets = project.tasks.create('listSnippets', SnippetsTask)
         listSnippets.group 'Docs'
@@ -53,17 +57,7 @@ public class DocsTestPlugin extends RestTestPlugin {
                 'List snippets that probably should be marked // CONSOLE'
         listConsoleCandidates.defaultSubstitutions = defaultSubstitutions
         listConsoleCandidates.perSnippet {
-            if (
-                       it.console != null // Already marked, nothing to do
-                    || it.testResponse    // It is a response
-                ) {
-                return
-            }
-            if (    // js almost always should be `// CONSOLE`
-                    it.language == 'js' ||
-                    // snippets containing `curl` *probably* should
-                    // be `// CONSOLE`
-                    it.curl) {
+            if (RestTestsFromSnippetsTask.isConsoleCandidate(it)) {
                 println(it.toString())
             }
         }
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy
index dc4e6f5f70af4..2ec12fe341f6b 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy
@@ -41,6 +41,16 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
     @Input
     Map<String, String> setups = new HashMap()
 
+    /**
+     * A list of files that contain snippets that *probably* should be
+     * converted to `// CONSOLE` but have yet to be converted. If a file is in
+     * this list and doesn't contain unconverted snippets this task will fail.
+     * If there are unconverted snippets not in this list then this task will
+     * fail. All files are paths relative to the docs dir.
+     */
+    @Input
+    List<String> expectedUnconvertedCandidates = []
+
     /**
      * Root directory of the tests being generated. To make rest tests happy
      * we generate them in a testRoot() which is contained in this directory.
@@ -56,6 +66,7 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
         TestBuilder builder = new TestBuilder()
         doFirst { outputRoot().delete() }
         perSnippet builder.&handleSnippet
+        doLast builder.&checkUnconverted
         doLast builder.&finishLastTest
     }
 
@@ -67,6 +78,27 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
         return new File(testRoot, '/rest-api-spec/test')
     }
 
+    /**
+     * Is this snippet a candidate for conversion to `// CONSOLE`?
+     */
+    static isConsoleCandidate(Snippet snippet) {
+        /* Snippets that are responses or already marked as `// CONSOLE` or
+         * `// NOTCONSOLE` are not candidates. */
+        if (snippet.console != null || snippet.testResponse) {
+            return false
+        }
+        /* js snippets almost always should be marked with `// CONSOLE`. js
+         * snippets that shouldn't be marked `// CONSOLE`, like examples for
+         * js client, should always be marked with `// NOTCONSOLE`.
+         *
+         * `sh` snippets that contain `curl` almost always should be marked
+         * with `// CONSOLE`. In the exceptionally rare cases where they are
+         * not communicating with Elasticsearch, like the xamples in the ec2
+         * and gce discovery plugins, the snippets should be marked
+         * `// NOTCONSOLE`. */
+        return snippet.language == 'js' || snippet.curl
+    }
+
     private class TestBuilder {
         private static final String SYNTAX = {
             String method = /(?<method>GET|PUT|POST|HEAD|OPTIONS|DELETE)/
@@ -88,17 +120,34 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
          */
         PrintWriter current
 
+        /**
+         * Files containing all snippets that *probably* should be converted
+         * to `// CONSOLE` but have yet to be converted. All files are paths
+         * relative to the docs dir.
+         */
+        Set<String> unconvertedCandidates = new HashSet<>()
+
+        /**
+         * The last non-TESTRESPONSE snippet.
+         */
+        Snippet previousTest
+
         /**
          * Called each time a snippet is encountered. Tracks the snippets and
          * calls buildTest to actually build the test.
          */
         void handleSnippet(Snippet snippet) {
+            if (RestTestsFromSnippetsTask.isConsoleCandidate(snippet)) {
+                unconvertedCandidates.add(snippet.path.toString()
+                    .replace('\\', '/'))
+            }
             if (BAD_LANGUAGES.contains(snippet.language)) {
                 throw new InvalidUserDataException(
                         "$snippet: Use `js` instead of `${snippet.language}`.")
             }
             if (snippet.testSetup) {
                 setup(snippet)
+                previousTest = snippet
                 return
             }
             if (snippet.testResponse) {
@@ -107,6 +156,7 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
             }
             if (snippet.test || snippet.console) {
                 test(snippet)
+                previousTest = snippet
                 return
             }
             // Must be an unmarked snippet....
@@ -115,13 +165,37 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
         private void test(Snippet test) {
             setupCurrent(test)
 
-            if (false == test.continued) {
+            if (test.continued) {
+                /* Catch some difficult to debug errors with // TEST[continued]
+                 * and throw a helpful error message. */
+                if (previousTest == null || previousTest.path != test.path) {
+                    throw new InvalidUserDataException("// TEST[continued] " +
+                        "cannot be on first snippet in a file: $test")
+                }
+                if (previousTest != null && previousTest.testSetup) {
+                    throw new InvalidUserDataException("// TEST[continued] " +
+                        "cannot immediately follow // TESTSETUP: $test")
+                }
+            } else {
                 current.println('---')
                 current.println("\"line_$test.start\":")
+                /* The Elasticsearch test runner doesn't support the warnings
+                 * construct unless you output this skip. Since we don't know
+                 * if this snippet will use the warnings construct we emit this
+                 * warning every time. */
+                current.println("  - skip:")
+                current.println("      features: ")
+                current.println("        - stash_in_key")
+                current.println("        - stash_in_path")
+                current.println("        - stash_path_replace")
+                current.println("        - warnings")
             }
             if (test.skipTest) {
-                current.println("  - skip:")
-                current.println("      features: always_skip")
+                if (test.continued) {
+                    throw new InvalidUserDataException("Continued snippets "
+                        + "can't be skipped")
+                }
+                current.println("        - always_skip")
                 current.println("      reason: $test.skipTest")
             }
             if (test.setup != null) {
@@ -148,6 +222,11 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
             def (String path, String query) = pathAndQuery.tokenize('?')
             if (path == null) {
                 path = '' // Catch requests to the root...
+            } else {
+                // Escape some characters that are also escaped by sense
+                path = path.replace('<', '%3C').replace('>', '%3E')
+                path = path.replace('{', '%7B').replace('}', '%7D')
+                path = path.replace('|', '%7C')
             }
             current.println("  - do:")
             if (catchPart != null) {
@@ -250,5 +329,35 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
                 current = null
             }
         }
+
+        void checkUnconverted() {
+            List<String> listedButNotFound = []
+            for (String listed : expectedUnconvertedCandidates) {
+                if (false == unconvertedCandidates.remove(listed)) {
+                    listedButNotFound.add(listed)
+                }
+            }
+            String message = ""
+            if (false == listedButNotFound.isEmpty()) {
+                Collections.sort(listedButNotFound)
+                listedButNotFound = listedButNotFound.collect {'    ' + it}
+                message += "Expected unconverted snippets but none found in:\n"
+                message += listedButNotFound.join("\n")
+            }
+            if (false == unconvertedCandidates.isEmpty()) {
+                List<String> foundButNotListed =
+                    new ArrayList<>(unconvertedCandidates)
+                Collections.sort(foundButNotListed)
+                foundButNotListed = foundButNotListed.collect {'    ' + it}
+                if (false == "".equals(message)) {
+                    message += "\n"
+                }
+                message += "Unexpected unconverted snippets:\n"
+                message += foundButNotListed.join("\n")
+            }
+            if (false == "".equals(message)) {
+                throw new InvalidUserDataException(message);
+            }
+        }
     }
 }
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy
index 41f74b45be143..94af22f4aa279 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy
@@ -39,6 +39,7 @@ public class SnippetsTask extends DefaultTask {
     private static final String SKIP = /skip:([^\]]+)/
     private static final String SETUP = /setup:([^ \]]+)/
     private static final String WARNING = /warning:(.+)/
+    private static final String CAT = /(_cat)/
     private static final String TEST_SYNTAX =
         /(?:$CATCH|$SUBSTITUTION|$SKIP|(continued)|$SETUP|$WARNING) ?/
 
@@ -89,6 +90,7 @@ public class SnippetsTask extends DefaultTask {
                      * tests cleaner.
                      */
                     subst = subst.replace('$body', '\\$body')
+                    subst = subst.replace('$_path', '\\$_path')
                     // \n is a new line....
                     subst = subst.replace('\\n', '\n')
                     snippet.contents = snippet.contents.replaceAll(
@@ -221,8 +223,17 @@ public class SnippetsTask extends DefaultTask {
                             substitutions = []
                         }
                         String loc = "$file:$lineNumber"
-                        parse(loc, matcher.group(2), /$SUBSTITUTION ?/) {
-                            substitutions.add([it.group(1), it.group(2)])
+                        parse(loc, matcher.group(2), /(?:$SUBSTITUTION|$CAT) ?/) {
+                            if (it.group(1) != null) {
+                                // TESTRESPONSE[s/adsf/jkl/]
+                                substitutions.add([it.group(1), it.group(2)])
+                            } else if (it.group(3) != null) {
+                                // TESTRESPONSE[_cat]
+                                substitutions.add(['^', '/'])
+                                substitutions.add(['\n$', '\\\\s*/'])
+                                substitutions.add(['( +)', '$1\\\\s+'])
+                                substitutions.add(['\n', '\\\\s*\n '])
+                            }
                         }
                     }
                     return
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy
index 0a454ee1006ff..2e11fdc2681bc 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy
@@ -19,6 +19,7 @@
 package org.elasticsearch.gradle.plugin
 
 import org.elasticsearch.gradle.BuildPlugin
+import org.elasticsearch.gradle.NoticeTask
 import org.elasticsearch.gradle.test.RestIntegTestTask
 import org.elasticsearch.gradle.test.RunTask
 import org.gradle.api.Project
@@ -49,33 +50,29 @@ public class PluginBuildPlugin extends BuildPlugin {
         project.afterEvaluate {
             boolean isModule = project.path.startsWith(':modules:')
             String name = project.pluginProperties.extension.name
-            project.jar.baseName = name
-            project.bundlePlugin.baseName = name
+            project.archivesBaseName = name
 
             if (project.pluginProperties.extension.hasClientJar) {
                 // for plugins which work with the transport client, we copy the jar
                 // file to a new name, copy the nebula generated pom to the same name,
                 // and generate a different pom for the zip
-                project.signArchives.enabled = false
                 addClientJarPomGeneration(project)
                 addClientJarTask(project)
-                if (isModule == false) {
-                    addZipPomGeneration(project)
-                }
             } else {
                 // no client plugin, so use the pom file from nebula, without jar, for the zip
                 project.ext.set("nebulaPublish.maven.jar", false)
             }
 
-            project.integTest.dependsOn(project.bundlePlugin)
+            project.integTestCluster.dependsOn(project.bundlePlugin)
             project.tasks.run.dependsOn(project.bundlePlugin)
             if (isModule) {
-                project.integTest.clusterConfig.module(project)
+                project.integTestCluster.module(project)
                 project.tasks.run.clusterConfig.module(project)
             } else {
-                project.integTest.clusterConfig.plugin(project.path)
+                project.integTestCluster.plugin(project.path)
                 project.tasks.run.clusterConfig.plugin(project.path)
                 addZipPomGeneration(project)
+                addNoticeGeneration(project)
             }
 
             project.namingConventions {
@@ -99,7 +96,7 @@ public class PluginBuildPlugin extends BuildPlugin {
             provided "com.vividsolutions:jts:${project.versions.jts}"
             provided "org.apache.logging.log4j:log4j-api:${project.versions.log4j}"
             provided "org.apache.logging.log4j:log4j-core:${project.versions.log4j}"
-            provided "net.java.dev.jna:jna:${project.versions.jna}"
+            provided "org.elasticsearch:jna:${project.versions.jna}"
         }
     }
 
@@ -123,12 +120,15 @@ public class PluginBuildPlugin extends BuildPlugin {
         // add the plugin properties and metadata to test resources, so unit tests can
         // know about the plugin (used by test security code to statically initialize the plugin in unit tests)
         SourceSet testSourceSet = project.sourceSets.test
-        testSourceSet.output.dir(buildProperties.generatedResourcesDir, builtBy: 'pluginProperties')
+        testSourceSet.output.dir(buildProperties.descriptorOutput.parentFile, builtBy: 'pluginProperties')
         testSourceSet.resources.srcDir(pluginMetadata)
 
         // create the actual bundle task, which zips up all the files for the plugin
         Zip bundle = project.tasks.create(name: 'bundlePlugin', type: Zip, dependsOn: [project.jar, buildProperties]) {
-            from buildProperties // plugin properties file
+            from(buildProperties.descriptorOutput.parentFile) {
+                // plugin properties file
+                include(buildProperties.descriptorOutput.name)
+            }
             from pluginMetadata // metadata (eg custom security policy)
             from project.jar // this plugin's jar
             from project.configurations.runtime - project.configurations.provided // the dep jars
@@ -152,7 +152,7 @@ public class PluginBuildPlugin extends BuildPlugin {
     /** Adds a task to move jar and associated files to a "-client" name. */
     protected static void addClientJarTask(Project project) {
         Task clientJar = project.tasks.create('clientJar')
-        clientJar.dependsOn('generatePomFileForJarPublication', project.jar, project.javadocJar, project.sourcesJar)
+        clientJar.dependsOn(project.jar, 'generatePomFileForClientJarPublication', project.javadocJar, project.sourcesJar)
         clientJar.doFirst {
             Path jarFile = project.jar.outputs.files.singleFile.toPath()
             String clientFileName = jarFile.fileName.toString().replace(project.version, "client-${project.version}")
@@ -179,7 +179,10 @@ public class PluginBuildPlugin extends BuildPlugin {
     static final Pattern GIT_PATTERN = Pattern.compile(/git@([^:]+):([^\.]+)\.git/)
 
     /** Find the reponame. */
-    protected static String urlFromOrigin(String origin) {
+    static String urlFromOrigin(String origin) {
+        if (origin == null) {
+            return null // best effort, the url doesnt really matter, it is just required by maven central
+        }
         if (origin.startsWith('https')) {
             return origin
         }
@@ -197,9 +200,9 @@ public class PluginBuildPlugin extends BuildPlugin {
 
         project.publishing {
             publications {
-                jar(MavenPublication) {
+                clientJar(MavenPublication) {
                     from project.components.java
-                    artifactId = artifactId + '-client'
+                    artifactId = project.pluginProperties.extension.name + '-client'
                     pom.withXml { XmlProvider xml ->
                         Node root = xml.asNode()
                         root.appendNode('name', project.pluginProperties.extension.name)
@@ -213,7 +216,7 @@ public class PluginBuildPlugin extends BuildPlugin {
         }
     }
 
-    /** Adds a task to generate a*/
+    /** Adds a task to generate a pom file for the zip distribution. */
     protected void addZipPomGeneration(Project project) {
         project.plugins.apply(MavenPublishPlugin.class)
 
@@ -221,7 +224,19 @@ public class PluginBuildPlugin extends BuildPlugin {
             publications {
                 zip(MavenPublication) {
                     artifact project.bundlePlugin
-                    pom.packaging = 'pom'
+                }
+                /* HUGE HACK: the underlying maven publication library refuses to deploy any attached artifacts
+                 * when the packaging type is set to 'pom'. But Sonatype's OSS repositories require source files
+                 * for artifacts that are of type 'zip'. We already publish the source and javadoc for Elasticsearch
+                 * under the various other subprojects. So here we create another publication using the same
+                 * name that has the "real" pom, and rely on the fact that gradle will execute the publish tasks
+                 * in alphabetical order. This lets us publish the zip file and even though the pom says the
+                 * type is 'pom' instead of 'zip'. We cannot setup a dependency between the tasks because the
+                 * publishing tasks are created *extremely* late in the configuration phase, so that we cannot get
+                 * ahold of the actual task. Furthermore, this entire hack only exists so we can make publishing to
+                 * maven local work, since we publish to maven central externally. */
+                zipReal(MavenPublication) {
+                    artifactId = project.pluginProperties.extension.name
                     pom.withXml { XmlProvider xml ->
                         Node root = xml.asNode()
                         root.appendNode('name', project.pluginProperties.extension.name)
@@ -234,4 +249,19 @@ public class PluginBuildPlugin extends BuildPlugin {
             }
         }
     }
+
+    protected void addNoticeGeneration(Project project) {
+        File licenseFile = project.pluginProperties.extension.licenseFile
+        if (licenseFile != null) {
+            project.bundlePlugin.from(licenseFile.parentFile) {
+                include(licenseFile.name)
+            }
+        }
+        File noticeFile = project.pluginProperties.extension.noticeFile
+        if (noticeFile != null) {
+            NoticeTask generateNotice = project.tasks.create('generateNotice', NoticeTask.class)
+            generateNotice.inputFile = noticeFile
+            project.bundlePlugin.from(generateNotice)
+        }
+    }
 }
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.groovy
index 5502266693653..1251be265da9a 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.groovy
@@ -39,10 +39,24 @@ class PluginPropertiesExtension {
     @Input
     String classname
 
+    @Input
+    boolean hasNativeController = false
+
     /** Indicates whether the plugin jar should be made available for the transport client. */
     @Input
     boolean hasClientJar = false
 
+    /** A license file that should be included in the built plugin zip. */
+    @Input
+    File licenseFile = null
+
+    /**
+     * A notice file that should be included in the built plugin zip. This will be
+     * extended with notices from the {@code licenses/} directory.
+     */
+    @Input
+    File noticeFile = null
+
     PluginPropertiesExtension(Project project) {
         name = project.name
         version = project.version
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesTask.groovy
index 7156c2650cbe0..91efe247a016b 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesTask.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesTask.groovy
@@ -22,6 +22,7 @@ import org.elasticsearch.gradle.VersionProperties
 import org.gradle.api.InvalidUserDataException
 import org.gradle.api.Task
 import org.gradle.api.tasks.Copy
+import org.gradle.api.tasks.OutputFile
 
 /**
  * Creates a plugin descriptor.
@@ -29,20 +30,22 @@ import org.gradle.api.tasks.Copy
 class PluginPropertiesTask extends Copy {
 
     PluginPropertiesExtension extension
-    File generatedResourcesDir = new File(project.buildDir, 'generated-resources')
+
+    @OutputFile
+    File descriptorOutput = new File(project.buildDir, 'generated-resources/plugin-descriptor.properties')
 
     PluginPropertiesTask() {
-        File templateFile = new File(project.buildDir, 'templates/plugin-descriptor.properties')
+        File templateFile = new File(project.buildDir, "templates/${descriptorOutput.name}")
         Task copyPluginPropertiesTemplate = project.tasks.create('copyPluginPropertiesTemplate') {
             doLast {
-                InputStream resourceTemplate = PluginPropertiesTask.getResourceAsStream('/plugin-descriptor.properties')
+                InputStream resourceTemplate = PluginPropertiesTask.getResourceAsStream("/${descriptorOutput.name}")
                 templateFile.parentFile.mkdirs()
                 templateFile.setText(resourceTemplate.getText('UTF-8'), 'UTF-8')
             }
         }
+
         dependsOn(copyPluginPropertiesTemplate)
         extension = project.extensions.create('esplugin', PluginPropertiesExtension, project)
-        project.clean.delete(generatedResourcesDir)
         project.afterEvaluate {
             // check require properties are set
             if (extension.name == null) {
@@ -55,8 +58,8 @@ class PluginPropertiesTask extends Copy {
                 throw new InvalidUserDataException('classname is a required setting for esplugin')
             }
             // configure property substitution
-            from(templateFile)
-            into(generatedResourcesDir)
+            from(templateFile.parentFile).include(descriptorOutput.name)
+            into(descriptorOutput.parentFile)
             Map<String, String> properties = generateSubstitutions()
             expand(properties)
             inputs.properties(properties)
@@ -76,7 +79,8 @@ class PluginPropertiesTask extends Copy {
             'version': stringSnap(extension.version),
             'elasticsearchVersion': stringSnap(VersionProperties.elasticsearch),
             'javaVersion': project.targetCompatibility as String,
-            'classname': extension.classname
+            'classname': extension.classname,
+            'hasNativeController': extension.hasNativeController
         ]
     }
 }
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/DependencyLicensesTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/DependencyLicensesTask.groovy
index 6fa37be309ec1..4d292d87ec39c 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/DependencyLicensesTask.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/DependencyLicensesTask.groovy
@@ -86,6 +86,9 @@ public class DependencyLicensesTask extends DefaultTask {
     /** A map of patterns to prefix, used to find the LICENSE and NOTICE file. */
     private LinkedHashMap<String, String> mappings = new LinkedHashMap<>()
 
+    /** Names of dependencies whose shas should not exist. */
+    private Set<String> ignoreShas = new HashSet<>()
+
     /**
      * Add a mapping from a regex pattern for the jar name, to a prefix to find
      * the LICENSE and NOTICE file for that jar.
@@ -106,6 +109,15 @@ public class DependencyLicensesTask extends DefaultTask {
         mappings.put(from, to)
     }
 
+    /**
+     * Add a rule which will skip SHA checking for the given dependency name. This should be used for
+     * locally build dependencies, which cause the sha to change constantly.
+     */
+    @Input
+    public void ignoreSha(String dep) {
+        ignoreShas.add(dep)
+    }
+
     @TaskAction
     public void checkDependencies() {
         if (dependencies.isEmpty()) {
@@ -139,19 +151,27 @@ public class DependencyLicensesTask extends DefaultTask {
 
         for (File dependency : dependencies) {
             String jarName = dependency.getName()
-            logger.info("Checking license/notice/sha for " + jarName)
-            checkSha(dependency, jarName, shaFiles)
+            String depName = jarName - ~/\-\d+.*/
+            if (ignoreShas.contains(depName)) {
+                // local deps should not have sha files!
+                if (getShaFile(jarName).exists()) {
+                    throw new GradleException("SHA file ${getShaFile(jarName)} exists for ignored dependency ${depName}")
+                }
+            } else {
+                logger.info("Checking sha for " + jarName)
+                checkSha(dependency, jarName, shaFiles)
+            }
 
-            String name = jarName - ~/\-\d+.*/
-            Matcher match = mappingsPattern.matcher(name)
+            logger.info("Checking license/notice for " + depName)
+            Matcher match = mappingsPattern.matcher(depName)
             if (match.matches()) {
                 int i = 0
                 while (i < match.groupCount() && match.group(i + 1) == null) ++i;
-                logger.info("Mapped dependency name ${name} to ${mapped.get(i)} for license check")
-                name = mapped.get(i)
+                logger.info("Mapped dependency name ${depName} to ${mapped.get(i)} for license check")
+                depName = mapped.get(i)
             }
-            checkFile(name, jarName, licenses, 'LICENSE')
-            checkFile(name, jarName, notices, 'NOTICE')
+            checkFile(depName, jarName, licenses, 'LICENSE')
+            checkFile(depName, jarName, notices, 'NOTICE')
         }
 
         licenses.each { license, count ->
@@ -169,8 +189,12 @@ public class DependencyLicensesTask extends DefaultTask {
         }
     }
 
+    private File getShaFile(String jarName) {
+        return new File(licensesDir, jarName + SHA_EXTENSION)
+    }
+
     private void checkSha(File jar, String jarName, Set<File> shaFiles) {
-        File shaFile = new File(licensesDir, jarName + SHA_EXTENSION)
+        File shaFile = getShaFile(jarName)
         if (shaFile.exists() == false) {
             throw new GradleException("Missing SHA for ${jarName}. Run 'gradle updateSHAs' to create")
         }
@@ -215,6 +239,10 @@ public class DependencyLicensesTask extends DefaultTask {
             }
             for (File dependency : parentTask.dependencies) {
                 String jarName = dependency.getName()
+                String depName = jarName - ~/\-\d+.*/
+                if (parentTask.ignoreShas.contains(depName)) {
+                    continue
+                }
                 File shaFile = new File(parentTask.licensesDir, jarName + SHA_EXTENSION)
                 if (shaFile.exists() == false) {
                     logger.lifecycle("Adding sha for ${jarName}")
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ForbiddenPatternsTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ForbiddenPatternsTask.groovy
index 7cb344bf47fc9..ed62e88c567fa 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ForbiddenPatternsTask.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ForbiddenPatternsTask.groovy
@@ -59,11 +59,16 @@ public class ForbiddenPatternsTask extends DefaultTask {
         filesFilter.exclude('**/*.png')
 
         // add mandatory rules
-        patterns.put('nocommit', /nocommit/)
+        patterns.put('nocommit', /nocommit|NOCOMMIT/)
+        patterns.put('nocommit should be all lowercase or all uppercase',
+            /((?i)nocommit)(?<!(nocommit|NOCOMMIT))/)
         patterns.put('tab', /\t/)
         if (System.getProperty('build.snapshot', 'true').equals('false')) {
-            patterns.put('norelease', /norelease/)
+            patterns.put('norelease', /norelease|NORELEASE/)
         }
+        patterns.put('norelease should be all lowercase or all uppercase',
+            /((?i)norelease)(?<!(norelease|NORELEASE))/)
+
 
         inputs.property("excludes", filesFilter.excludes)
         inputs.property("rules", patterns)
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/NamingConventionsTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/NamingConventionsTask.groovy
index 52de7dac2d5a3..2711a0e38f23b 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/NamingConventionsTask.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/NamingConventionsTask.groovy
@@ -38,17 +38,7 @@ public class NamingConventionsTask extends LoggedExec {
      * inputs (ie the jars/class files).
      */
     @OutputFile
-    File successMarker = new File(project.buildDir, 'markers/namingConventions')
-
-    /**
-     * The classpath to run the naming conventions checks against. Must contain the files in the test
-     * output directory and everything required to load those classes.
-     *
-     * We don't declare the actual test files as a dependency or input because if they change then
-     * this will change.
-     */
-    @InputFiles
-    FileCollection classpath = project.sourceSets.test.runtimeClasspath
+    File successMarker = new File(project.buildDir, "markers/${this.name}")
 
     /**
      * Should we skip the integ tests in disguise tests? Defaults to true because only core names its
@@ -69,18 +59,35 @@ public class NamingConventionsTask extends LoggedExec {
     @Input
     String integTestClass = 'org.elasticsearch.test.ESIntegTestCase'
 
+    /**
+     * Should the test also check the main classpath for test classes instead of
+     * doing the usual checks to the test classpath.
+     */
+    @Input
+    boolean checkForTestsInMain = false;
+
     public NamingConventionsTask() {
         // Extra classpath contains the actual test
-        project.configurations.create('namingConventions')
-        Dependency buildToolsDep = project.dependencies.add('namingConventions',
-                "org.elasticsearch.gradle:build-tools:${VersionProperties.elasticsearch}")
-        buildToolsDep.transitive = false // We don't need gradle in the classpath. It conflicts.
+        if (false == project.configurations.names.contains('namingConventions')) {
+            project.configurations.create('namingConventions')
+            Dependency buildToolsDep = project.dependencies.add('namingConventions',
+                    "org.elasticsearch.gradle:build-tools:${VersionProperties.elasticsearch}")
+            buildToolsDep.transitive = false // We don't need gradle in the classpath. It conflicts.
+        }
         FileCollection extraClasspath = project.configurations.namingConventions
         dependsOn(extraClasspath)
 
-        description = "Runs NamingConventionsCheck on ${classpath}"
+        FileCollection classpath = project.sourceSets.test.runtimeClasspath
+        inputs.files(classpath)
+        description = "Tests that test classes aren't misnamed or misplaced"
         executable = new File(project.javaHome, 'bin/java')
-        onlyIf { project.sourceSets.test.output.classesDir.exists() }
+        if (false == checkForTestsInMain) {
+            /* This task is created by default for all subprojects with this
+             * setting and there is no point in running it if the files don't
+             * exist. */
+            onlyIf { project.sourceSets.test.output.classesDir.exists() }
+        }
+
         /*
          * We build the arguments in a funny afterEvaluate/doFirst closure so that we can wait for the classpath to be
          * ready for us. Strangely neither one on their own are good enough.
@@ -104,7 +111,14 @@ public class NamingConventionsTask extends LoggedExec {
                 if (':build-tools'.equals(project.path)) {
                     args('--self-test')
                 }
-                args('--', project.sourceSets.test.output.classesDir.absolutePath)
+                if (checkForTestsInMain) {
+                    args('--main')
+                    args('--')
+                    args(project.sourceSets.main.output.classesDir.absolutePath)
+                } else {
+                    args('--')
+                    args(project.sourceSets.test.output.classesDir.absolutePath)
+                }
             }
         }
         doLast { successMarker.setText("", 'UTF-8') }
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy
index 44eb050dfb5ae..96b7ac425279a 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy
@@ -59,8 +59,7 @@ class PrecommitTasks {
              * use the NamingConventionsCheck we break the circular dependency
              * here.
              */
-            // https://github.com/elastic/elasticsearch/issues/20243
-            // precommitTasks.add(configureLoggerUsage(project))
+            precommitTasks.add(configureLoggerUsage(project))
         }
 
 
@@ -140,6 +139,7 @@ class PrecommitTasks {
             configProperties = [
                 suppressions: checkstyleSuppressions
             ]
+            toolVersion = 7.5
         }
         for (String taskName : ['checkstyleMain', 'checkstyleTest']) {
             Task task = project.tasks.findByName(taskName)
@@ -148,6 +148,9 @@ class PrecommitTasks {
                 checkstyleTask.dependsOn(task)
                 task.dependsOn(copyCheckstyleConf)
                 task.inputs.file(checkstyleSuppressions)
+                task.reports {
+                    html.enabled false
+                }
             }
         }
         return checkstyleTask
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.groovy
index 076a564f84a5d..33ca6dccfa32e 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.groovy
@@ -143,6 +143,10 @@ public class ThirdPartyAuditTask extends AntTask {
                     if (m.matches()) {
                         missingClasses.add(m.group(1).replace('.', '/') + ".class");
                     }
+
+                    // Reset the priority of the event to DEBUG, so it doesn't
+                    // pollute the build output
+                    event.setMessage(event.getMessage(), Project.MSG_DEBUG);
                 } else if (event.getPriority() == Project.MSG_ERR) {
                     Matcher m = VIOLATION_PATTERN.matcher(event.getMessage());
                     if (m.matches()) {
@@ -205,9 +209,11 @@ public class ThirdPartyAuditTask extends AntTask {
         try {
             ant.thirdPartyAudit(failOnUnsupportedJava: false,
                             failOnMissingClasses: false,
-                            signaturesFile: new File(getClass().getResource('/forbidden/third-party-audit.txt').toURI()),
                             classpath: classpath.asPath) {
                 fileset(dir: tmpDir)
+                signatures {
+                    string(value: getClass().getResourceAsStream('/forbidden/third-party-audit.txt').getText('UTF-8'))
+                }
             }
         } catch (BuildException ignore) {}
 
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/AntFixture.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/AntFixture.groovy
new file mode 100644
index 0000000000000..34c3046aa2b6b
--- /dev/null
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/AntFixture.groovy
@@ -0,0 +1,291 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle.test
+
+import org.apache.tools.ant.taskdefs.condition.Os
+import org.elasticsearch.gradle.AntTask
+import org.elasticsearch.gradle.LoggedExec
+import org.gradle.api.GradleException
+import org.gradle.api.Task
+import org.gradle.api.tasks.Exec
+import org.gradle.api.tasks.Input
+
+/**
+ * A fixture for integration tests which runs in a separate process launched by Ant.
+ */
+public class AntFixture extends AntTask implements Fixture {
+
+    /** The path to the executable that starts the fixture. */
+    @Input
+    String executable
+
+    private final List<Object> arguments = new ArrayList<>()
+
+    @Input
+    public void args(Object... args) {
+        arguments.addAll(args)
+    }
+
+    /**
+     * Environment variables for the fixture process. The value can be any object, which
+     * will have toString() called at execution time.
+     */
+    private final Map<String, Object> environment = new HashMap<>()
+
+    @Input
+    public void env(String key, Object value) {
+        environment.put(key, value)
+    }
+
+    /** A flag to indicate whether the command should be executed from a shell. */
+    @Input
+    boolean useShell = false
+
+    /**
+     * A flag to indicate whether the fixture should be run in the foreground, or spawned.
+     * It is protected so subclasses can override (eg RunTask).
+     */
+    protected boolean spawn = true
+
+    /**
+     * A closure to call before the fixture is considered ready. The closure is passed the fixture object,
+     * as well as a groovy AntBuilder, to enable running ant condition checks. The default wait
+     * condition is for http on the http port.
+     */
+    @Input
+    Closure waitCondition = { AntFixture fixture, AntBuilder ant ->
+        File tmpFile = new File(fixture.cwd, 'wait.success')
+        ant.get(src: "http://${fixture.addressAndPort}",
+                dest: tmpFile.toString(),
+                ignoreerrors: true, // do not fail on error, so logging information can be flushed
+                retries: 10)
+        return tmpFile.exists()
+    }
+
+    private final Task stopTask
+
+    public AntFixture() {
+        stopTask = createStopTask()
+        finalizedBy(stopTask)
+    }
+
+    @Override
+    public Task getStopTask() {
+        return stopTask
+    }
+
+    @Override
+    protected void runAnt(AntBuilder ant) {
+        project.delete(baseDir) // reset everything
+        cwd.mkdirs()
+        final String realExecutable
+        final List<Object> realArgs = new ArrayList<>()
+        final Map<String, Object> realEnv = environment
+        // We need to choose which executable we are using. In shell mode, or when we
+        // are spawning and thus using the wrapper script, the executable is the shell.
+        if (useShell || spawn) {
+            if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+                realExecutable = 'cmd'
+                realArgs.add('/C')
+                realArgs.add('"') // quote the entire command
+            } else {
+                realExecutable = 'sh'
+            }
+        } else {
+            realExecutable = executable
+            realArgs.addAll(arguments)
+        }
+        if (spawn) {
+            writeWrapperScript(executable)
+            realArgs.add(wrapperScript)
+            realArgs.addAll(arguments)
+        }
+        if (Os.isFamily(Os.FAMILY_WINDOWS) && (useShell || spawn)) {
+            realArgs.add('"')
+        }
+        commandString.eachLine { line -> logger.info(line) }
+
+        ant.exec(executable: realExecutable, spawn: spawn, dir: cwd, taskname: name) {
+            realEnv.each { key, value -> env(key: key, value: value) }
+            realArgs.each { arg(value: it) }
+        }
+
+        String failedProp = "failed${name}"
+        // first wait for resources, or the failure marker from the wrapper script
+        ant.waitfor(maxwait: '30', maxwaitunit: 'second', checkevery: '500', checkeveryunit: 'millisecond', timeoutproperty: failedProp) {
+            or {
+                resourceexists {
+                    file(file: failureMarker.toString())
+                }
+                and {
+                    resourceexists {
+                        file(file: pidFile.toString())
+                    }
+                    resourceexists {
+                        file(file: portsFile.toString())
+                    }
+                }
+            }
+        }
+
+        if (ant.project.getProperty(failedProp) || failureMarker.exists()) {
+            fail("Failed to start ${name}")
+        }
+
+        // the process is started (has a pid) and is bound to a network interface
+        // so now wait undil the waitCondition has been met
+        // TODO: change this to a loop?
+        boolean success
+        try {
+            success = waitCondition(this, ant) == false
+        } catch (Exception e) {
+            String msg = "Wait condition caught exception for ${name}"
+            logger.error(msg, e)
+            fail(msg, e)
+        }
+        if (success == false) {
+            fail("Wait condition failed for ${name}")
+        }
+    }
+
+    /** Returns a debug string used to log information about how the fixture was run. */
+    protected String getCommandString() {
+        String commandString = "\n${name} configuration:\n"
+        commandString += "-----------------------------------------\n"
+        commandString += "  cwd: ${cwd}\n"
+        commandString += "  command: ${executable} ${arguments.join(' ')}\n"
+        commandString += '  environment:\n'
+        environment.each { k, v -> commandString += "    ${k}: ${v}\n" }
+        if (spawn) {
+            commandString += "\n  [${wrapperScript.name}]\n"
+            wrapperScript.eachLine('UTF-8', { line -> commandString += "    ${line}\n"})
+        }
+        return commandString
+    }
+
+    /**
+     * Writes a script to run the real executable, so that stdout/stderr can be captured.
+     * TODO: this could be removed if we do use our own ProcessBuilder and pump output from the process
+     */
+    private void writeWrapperScript(String executable) {
+        wrapperScript.parentFile.mkdirs()
+        String argsPasser = '"$@"'
+        String exitMarker = "; if [ \$? != 0 ]; then touch run.failed; fi"
+        if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+            argsPasser = '%*'
+            exitMarker = "\r\n if \"%errorlevel%\" neq \"0\" ( type nul >> run.failed )"
+        }
+        wrapperScript.setText("\"${executable}\" ${argsPasser} > run.log 2>&1 ${exitMarker}", 'UTF-8')
+    }
+
+    /** Fail the build with the given message, and logging relevant info*/
+    private void fail(String msg, Exception... suppressed) {
+        if (logger.isInfoEnabled() == false) {
+            // We already log the command at info level. No need to do it twice.
+            commandString.eachLine { line -> logger.error(line) }
+        }
+        logger.error("${name} output:")
+        logger.error("-----------------------------------------")
+        logger.error("  failure marker exists: ${failureMarker.exists()}")
+        logger.error("  pid file exists: ${pidFile.exists()}")
+        logger.error("  ports file exists: ${portsFile.exists()}")
+        // also dump the log file for the startup script (which will include ES logging output to stdout)
+        if (runLog.exists()) {
+            logger.error("\n  [log]")
+            runLog.eachLine { line -> logger.error("    ${line}") }
+        }
+        logger.error("-----------------------------------------")
+        GradleException toThrow = new GradleException(msg)
+        for (Exception e : suppressed) {
+            toThrow.addSuppressed(e)
+        }
+        throw toThrow
+    }
+
+    /** Adds a task to kill an elasticsearch node with the given pidfile */
+    private Task createStopTask() {
+        final AntFixture fixture = this
+        final Object pid = "${ -> fixture.pid }"
+        Exec stop = project.tasks.create(name: "${name}#stop", type: LoggedExec)
+        stop.onlyIf { fixture.pidFile.exists() }
+        stop.doFirst {
+            logger.info("Shutting down ${fixture.name} with pid ${pid}")
+        }
+        if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+            stop.executable = 'Taskkill'
+            stop.args('/PID', pid, '/F')
+        } else {
+            stop.executable = 'kill'
+            stop.args('-9', pid)
+        }
+        stop.doLast {
+            project.delete(fixture.pidFile)
+        }
+        return stop
+    }
+
+    /**
+     * A path relative to the build dir that all configuration and runtime files
+     * will live in for this fixture
+     */
+    protected File getBaseDir() {
+        return new File(project.buildDir, "fixtures/${name}")
+    }
+
+    /** Returns the working directory for the process. Defaults to "cwd" inside baseDir. */
+    protected File getCwd() {
+        return new File(baseDir, 'cwd')
+    }
+
+    /** Returns the file the process writes its pid to. Defaults to "pid" inside baseDir. */
+    protected File getPidFile() {
+        return new File(baseDir, 'pid')
+    }
+
+    /** Reads the pid file and returns the process' pid */
+    public int getPid() {
+        return Integer.parseInt(pidFile.getText('UTF-8').trim())
+    }
+
+    /** Returns the file the process writes its bound ports to. Defaults to "ports" inside baseDir. */
+    protected File getPortsFile() {
+        return new File(baseDir, 'ports')
+    }
+
+    /** Returns an address and port suitable for a uri to connect to this node over http */
+    public String getAddressAndPort() {
+        return portsFile.readLines("UTF-8").get(0)
+    }
+
+    /** Returns a file that wraps around the actual command when {@code spawn == true}. */
+    protected File getWrapperScript() {
+        return new File(cwd, Os.isFamily(Os.FAMILY_WINDOWS) ? 'run.bat' : 'run')
+    }
+
+    /** Returns a file that the wrapper script writes when the command failed. */
+    protected File getFailureMarker() {
+        return new File(cwd, 'run.failed')
+    }
+
+    /** Returns a file that the wrapper script writes when the command failed. */
+    protected File getRunLog() {
+        return new File(cwd, 'run.log')
+    }
+}
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy
index 6a2375efc6299..c9965caa96e7d 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy
@@ -20,8 +20,6 @@ package org.elasticsearch.gradle.test
 
 import org.gradle.api.GradleException
 import org.gradle.api.Project
-import org.gradle.api.artifacts.Configuration
-import org.gradle.api.file.FileCollection
 import org.gradle.api.tasks.Input
 
 /** Configuration for an elasticsearch cluster, used for integration tests. */
@@ -47,25 +45,56 @@ class ClusterConfiguration {
     @Input
     int transportPort = 0
 
+    /**
+     * An override of the data directory. This may only be used with a single node.
+     * The value is lazily evaluated at runtime as a String path.
+     */
+    @Input
+    Object dataDir = null
+
+    /** Optional override of the cluster name. */
+    @Input
+    String clusterName = null
+
     @Input
     boolean daemonize = true
 
     @Input
     boolean debug = false
 
+    /**
+     * if <code>true</code> each node will be configured with <tt>discovery.zen.minimum_master_nodes</tt> set
+     * to the total number of nodes in the cluster. This will also cause that each node has `0s` state recovery
+     * timeout which can lead to issues if for instance an existing clusterstate is expected to be recovered
+     * before any tests start
+     */
+    @Input
+    boolean useMinimumMasterNodes = true
+
     @Input
     String jvmArgs = "-Xms" + System.getProperty('tests.heap.size', '512m') +
-            " " + "-Xmx" + System.getProperty('tests.heap.size', '512m') +
-            " " + System.getProperty('tests.jvm.argline', '')
+        " " + "-Xmx" + System.getProperty('tests.heap.size', '512m') +
+        " " + System.getProperty('tests.jvm.argline', '')
 
     /**
-     * The seed nodes port file. In the case the cluster has more than one node we use a seed node
-     * to form the cluster. The file is null if there is no seed node yet available.
+     * A closure to call which returns the unicast host to connect to for cluster formation.
      *
-     * Note: this can only be null if the cluster has only one node or if the first node is not yet
-     * configured. All nodes but the first node should see a non null value.
+     * This allows multi node clusters, or a new cluster to connect to an existing cluster.
+     * The closure takes two arguments, the NodeInfo for the first node in the cluster, and
+     * an AntBuilder which may be used to wait on conditions before returning.
      */
-    File seedNodePortsFile
+    @Input
+    Closure unicastTransportUri = { NodeInfo seedNode, NodeInfo node, AntBuilder ant ->
+        if (seedNode == node) {
+            return null
+        }
+        ant.waitfor(maxwait: '40', maxwaitunit: 'second', checkevery: '500', checkeveryunit: 'millisecond') {
+            resourceexists {
+                file(file: seedNode.transportPortsFile.toString())
+            }
+        }
+        return seedNode.transportUri()
+    }
 
     /**
      * A closure to call before the cluster is considered ready. The closure is passed the node info,
@@ -75,7 +104,13 @@ class ClusterConfiguration {
     @Input
     Closure waitCondition = { NodeInfo node, AntBuilder ant ->
         File tmpFile = new File(node.cwd, 'wait.success')
-        ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=${numNodes}",
+        String waitUrl = "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow"
+        ant.echo(message: "==> [${new Date()}] checking health: ${waitUrl}",
+                 level: 'info')
+        // checking here for wait_for_nodes to be >= the number of nodes because its possible
+        // this cluster is attempting to connect to nodes created by another task (same cluster name),
+        // so there will be more nodes in that case in the cluster state
+        ant.get(src: waitUrl,
                 dest: tmpFile.toString(),
                 ignoreerrors: true, // do not fail on error, so logging buffers can be flushed by the wait task
                 retries: 10)
@@ -88,7 +123,9 @@ class ClusterConfiguration {
 
     Map<String, String> systemProperties = new HashMap<>()
 
-    Map<String, String> settings = new HashMap<>()
+    Map<String, Object> settings = new HashMap<>()
+
+    Map<String, String> keystoreSettings = new HashMap<>()
 
     // map from destination path, to source file
     Map<String, Object> extraConfigFiles = new HashMap<>()
@@ -99,16 +136,23 @@ class ClusterConfiguration {
 
     LinkedHashMap<String, Object[]> setupCommands = new LinkedHashMap<>()
 
+    List<Object> dependencies = new ArrayList<>()
+
     @Input
     void systemProperty(String property, String value) {
         systemProperties.put(property, value)
     }
 
     @Input
-    void setting(String name, String value) {
+    void setting(String name, Object value) {
         settings.put(name, value)
     }
 
+    @Input
+    void keystoreSetting(String name, String value) {
+        keystoreSettings.put(name, value)
+    }
+
     @Input
     void plugin(String path) {
         Project pluginProject = project.project(path)
@@ -138,11 +182,9 @@ class ClusterConfiguration {
         extraConfigFiles.put(path, sourceFile)
     }
 
-    /** Returns an address and port suitable for a uri to connect to this clusters seed node over transport protocol*/
-    String seedNodeTransportUri() {
-        if (seedNodePortsFile != null) {
-            return seedNodePortsFile.readLines("UTF-8").get(0)
-        }
-        return null;
+    /** Add dependencies that must be run before the first task setting up the cluster. */
+    @Input
+    void dependsOn(Object... deps) {
+        dependencies.addAll(deps)
     }
 }
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
index 8819c63080a3b..ee225904cc93e 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
@@ -23,6 +23,7 @@ import org.apache.tools.ant.taskdefs.condition.Os
 import org.elasticsearch.gradle.LoggedExec
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.plugin.PluginBuildPlugin
+import org.elasticsearch.gradle.plugin.PluginPropertiesExtension
 import org.gradle.api.AntBuilder
 import org.gradle.api.DefaultTask
 import org.gradle.api.GradleException
@@ -30,13 +31,16 @@ import org.gradle.api.InvalidUserDataException
 import org.gradle.api.Project
 import org.gradle.api.Task
 import org.gradle.api.artifacts.Configuration
+import org.gradle.api.artifacts.Dependency
 import org.gradle.api.file.FileCollection
 import org.gradle.api.logging.Logger
 import org.gradle.api.tasks.Copy
 import org.gradle.api.tasks.Delete
 import org.gradle.api.tasks.Exec
 
+import java.nio.charset.StandardCharsets
 import java.nio.file.Paths
+import java.util.concurrent.TimeUnit
 
 /**
  * A helper for creating tasks to build a cluster that is used by a task, and tear down the cluster when the task is finished.
@@ -46,24 +50,20 @@ class ClusterFormationTasks {
     /**
      * Adds dependent tasks to the given task to start and stop a cluster with the given configuration.
      *
-     * Returns a NodeInfo object for the first node in the cluster.
+     * Returns a list of NodeInfo objects for each node in the cluster.
      */
-    static NodeInfo setup(Project project, Task task, ClusterConfiguration config) {
-        if (task.getEnabled() == false) {
-            // no need to add cluster formation tasks if the task won't run!
-            return
-        }
+    static List<NodeInfo> setup(Project project, String prefix, Task runner, ClusterConfiguration config) {
         File sharedDir = new File(project.buildDir, "cluster/shared")
         // first we remove everything in the shared cluster directory to ensure there are no leftovers in repos or anything
         // in theory this should not be necessary but repositories are only deleted in the cluster-state and not on-disk
         // such that snapshots survive failures / test runs and there is no simple way today to fix that.
-        Task cleanup = project.tasks.create(name: "${task.name}#prepareCluster.cleanShared", type: Delete, dependsOn: task.dependsOn.collect()) {
+        Task cleanup = project.tasks.create(name: "${prefix}#prepareCluster.cleanShared", type: Delete, dependsOn: config.dependencies) {
             delete sharedDir
             doLast {
                 sharedDir.mkdirs()
             }
         }
-        List<Task> startTasks = [cleanup]
+        List<Task> startTasks = []
         List<NodeInfo> nodes = []
         if (config.numNodes < config.numBwcNodes) {
             throw new GradleException("numNodes must be >= numBwcNodes [${config.numNodes} < ${config.numBwcNodes}]")
@@ -72,45 +72,45 @@ class ClusterFormationTasks {
             throw new GradleException("bwcVersion must not be null if numBwcNodes is > 0")
         }
         // this is our current version distribution configuration we use for all kinds of REST tests etc.
-        String distroConfigName = "${task.name}_elasticsearchDistro"
-        Configuration distro = project.configurations.create(distroConfigName)
-        configureDistributionDependency(project, config.distribution, distro, VersionProperties.elasticsearch)
-        if (config.bwcVersion != null && config.numBwcNodes > 0) {
+        Configuration currentDistro = project.configurations.create("${prefix}_elasticsearchDistro")
+        Configuration bwcDistro = project.configurations.create("${prefix}_elasticsearchBwcDistro")
+        Configuration bwcPlugins = project.configurations.create("${prefix}_elasticsearchBwcPlugins")
+        configureDistributionDependency(project, config.distribution, currentDistro, VersionProperties.elasticsearch)
+        if (config.numBwcNodes > 0) {
+            if (config.bwcVersion == null) {
+                throw new IllegalArgumentException("Must specify bwcVersion when numBwcNodes > 0")
+            }
             // if we have a cluster that has a BWC cluster we also need to configure a dependency on the BWC version
             // this version uses the same distribution etc. and only differs in the version we depend on.
             // from here on everything else works the same as if it's the current version, we fetch the BWC version
             // from mirrors using gradles built-in mechanism etc.
-            project.configurations {
-                elasticsearchBwcDistro
+
+            configureDistributionDependency(project, config.distribution, bwcDistro, config.bwcVersion)
+            for (Map.Entry<String, Project> entry : config.plugins.entrySet()) {
+                configureBwcPluginDependency("${prefix}_elasticsearchBwcPlugins", project, entry.getValue(), bwcPlugins, config.bwcVersion)
             }
-            configureDistributionDependency(project, config.distribution, project.configurations.elasticsearchBwcDistro, config.bwcVersion)
+            bwcDistro.resolutionStrategy.cacheChangingModulesFor(0, TimeUnit.SECONDS)
+            bwcPlugins.resolutionStrategy.cacheChangingModulesFor(0, TimeUnit.SECONDS)
         }
-
-        for (int i = 0; i < config.numNodes; ++i) {
+        for (int i = 0; i < config.numNodes; i++) {
             // we start N nodes and out of these N nodes there might be M bwc nodes.
-            // for each of those nodes we might have a different configuratioon
+            // for each of those nodes we might have a different configuration
             String elasticsearchVersion = VersionProperties.elasticsearch
+            Configuration distro = currentDistro
             if (i < config.numBwcNodes) {
                 elasticsearchVersion = config.bwcVersion
-                distro = project.configurations.elasticsearchBwcDistro
-            }
-            NodeInfo node = new NodeInfo(config, i, project, task, elasticsearchVersion, sharedDir)
-            if (i == 0) {
-                if (config.seedNodePortsFile != null) {
-                    // we might allow this in the future to be set but for now we are the only authority to set this!
-                    throw new GradleException("seedNodePortsFile has a non-null value but first node has not been intialized")
-                }
-                config.seedNodePortsFile = node.transportPortsFile;
+                distro = bwcDistro
             }
+            NodeInfo node = new NodeInfo(config, i, project, prefix, elasticsearchVersion, sharedDir)
             nodes.add(node)
-            startTasks.add(configureNode(project, task, cleanup, node, distro))
+            Task dependsOn = startTasks.empty ? cleanup : startTasks.get(0)
+            startTasks.add(configureNode(project, prefix, runner, dependsOn, node, config, distro, nodes.get(0)))
         }
 
-        Task wait = configureWaitTask("${task.name}#wait", project, nodes, startTasks)
-        task.dependsOn(wait)
+        Task wait = configureWaitTask("${prefix}#wait", project, nodes, startTasks)
+        runner.dependsOn(wait)
 
-        // delay the resolution of the uri by wrapping in a closure, so it is not used until read for tests
-        return nodes[0]
+        return nodes
     }
 
     /** Adds a dependency on the given distribution */
@@ -124,6 +124,13 @@ class ClusterFormationTasks {
         project.dependencies.add(configuration.name, "org.elasticsearch.distribution.${distro}:elasticsearch:${elasticsearchVersion}@${packaging}")
     }
 
+    /** Adds a dependency on a different version of the given plugin, which will be retrieved using gradle's dependency resolution */
+    static void configureBwcPluginDependency(String name, Project project, Project pluginProject, Configuration configuration, String elasticsearchVersion) {
+        verifyProjectHasBuildPlugin(name, elasticsearchVersion, project, pluginProject)
+        PluginPropertiesExtension extension = pluginProject.extensions.findByName('esplugin');
+        project.dependencies.add(configuration.name, "org.elasticsearch.plugin:${extension.name}:${elasticsearchVersion}@zip")
+    }
+
     /**
      * Adds dependent tasks to start an elasticsearch cluster before the given task is executed,
      * and stop it after it has finished executing.
@@ -141,49 +148,83 @@ class ClusterFormationTasks {
      *
      * @return a task which starts the node.
      */
-    static Task configureNode(Project project, Task task, Object dependsOn, NodeInfo node, Configuration configuration) {
+    static Task configureNode(Project project, String prefix, Task runner, Object dependsOn, NodeInfo node, ClusterConfiguration config,
+                              Configuration distribution, NodeInfo seedNode) {
 
         // tasks are chained so their execution order is maintained
-        Task setup = project.tasks.create(name: taskName(task, node, 'clean'), type: Delete, dependsOn: dependsOn) {
+        Task setup = project.tasks.create(name: taskName(prefix, node, 'clean'), type: Delete, dependsOn: dependsOn) {
             delete node.homeDir
             delete node.cwd
             doLast {
                 node.cwd.mkdirs()
             }
         }
-        setup = configureCheckPreviousTask(taskName(task, node, 'checkPrevious'), project, setup, node)
-        setup = configureStopTask(taskName(task, node, 'stopPrevious'), project, setup, node)
-        setup = configureExtractTask(taskName(task, node, 'extract'), project, setup, node, configuration)
-        setup = configureWriteConfigTask(taskName(task, node, 'configure'), project, setup, node)
-        setup = configureExtraConfigFilesTask(taskName(task, node, 'extraConfig'), project, setup, node)
-        setup = configureCopyPluginsTask(taskName(task, node, 'copyPlugins'), project, setup, node)
+
+        setup = configureCheckPreviousTask(taskName(prefix, node, 'checkPrevious'), project, setup, node)
+        setup = configureStopTask(taskName(prefix, node, 'stopPrevious'), project, setup, node)
+        setup = configureExtractTask(taskName(prefix, node, 'extract'), project, setup, node, distribution)
+        setup = configureWriteConfigTask(taskName(prefix, node, 'configure'), project, setup, node, seedNode)
+        setup = configureCreateKeystoreTask(taskName(prefix, node, 'createKeystore'), project, setup, node)
+        setup = configureAddKeystoreSettingTasks(prefix, project, setup, node)
+
+        if (node.config.plugins.isEmpty() == false) {
+            if (node.nodeVersion == VersionProperties.elasticsearch) {
+                setup = configureCopyPluginsTask(taskName(prefix, node, 'copyPlugins'), project, setup, node, prefix)
+            } else {
+                setup = configureCopyBwcPluginsTask(taskName(prefix, node, 'copyBwcPlugins'), project, setup, node, prefix)
+            }
+        }
 
         // install modules
         for (Project module : node.config.modules) {
             String actionName = pluginTaskName('install', module.name, 'Module')
-            setup = configureInstallModuleTask(taskName(task, node, actionName), project, setup, node, module)
+            setup = configureInstallModuleTask(taskName(prefix, node, actionName), project, setup, node, module)
         }
 
         // install plugins
         for (Map.Entry<String, Project> plugin : node.config.plugins.entrySet()) {
             String actionName = pluginTaskName('install', plugin.getKey(), 'Plugin')
-            setup = configureInstallPluginTask(taskName(task, node, actionName), project, setup, node, plugin.getValue())
+            setup = configureInstallPluginTask(taskName(prefix, node, actionName), project, setup, node, plugin.getValue(), prefix)
         }
 
+        // sets up any extra config files that need to be copied over to the ES instance;
+        // its run after plugins have been installed, as the extra config files may belong to plugins
+        setup = configureExtraConfigFilesTask(taskName(prefix, node, 'extraConfig'), project, setup, node)
+
         // extra setup commands
         for (Map.Entry<String, Object[]> command : node.config.setupCommands.entrySet()) {
             // the first argument is the actual script name, relative to home
             Object[] args = command.getValue().clone()
-            args[0] = new File(node.homeDir, args[0].toString())
-            setup = configureExecTask(taskName(task, node, command.getKey()), project, setup, node, args)
+            final Object commandPath
+            if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+                /*
+                 * We have to delay building the string as the path will not exist during configuration which will fail on Windows due to
+                 * getting the short name requiring the path to already exist. Note that we have to capture the value of arg[0] now
+                 * otherwise we would stack overflow later since arg[0] is replaced below.
+                 */
+                String argsZero = args[0]
+                commandPath = "${-> Paths.get(NodeInfo.getShortPathName(node.homeDir.toString())).resolve(argsZero.toString()).toString()}"
+            } else {
+                commandPath = node.homeDir.toPath().resolve(args[0].toString()).toString()
+            }
+            args[0] = commandPath
+            setup = configureExecTask(taskName(prefix, node, command.getKey()), project, setup, node, args)
         }
 
-        Task start = configureStartTask(taskName(task, node, 'start'), project, setup, node)
+        Task start = configureStartTask(taskName(prefix, node, 'start'), project, setup, node)
 
         if (node.config.daemonize) {
+            Task stop = configureStopTask(taskName(prefix, node, 'stop'), project, [], node)
             // if we are running in the background, make sure to stop the server when the task completes
-            Task stop = configureStopTask(taskName(task, node, 'stop'), project, [], node)
-            task.finalizedBy(stop)
+            runner.finalizedBy(stop)
+            start.finalizedBy(stop)
+            for (Object dependency : config.dependencies) {
+                if (dependency instanceof Fixture) {
+                    def depStop = ((Fixture)dependency).stopTask
+                    runner.finalizedBy(depStop)
+                    start.finalizedBy(depStop)
+                }
+            }
         }
         return start
     }
@@ -222,9 +263,9 @@ class ClusterFormationTasks {
                 Object rpm = "${ -> configuration.singleFile}"
                 extract = project.tasks.create(name: name, type: LoggedExec, dependsOn: extractDependsOn) {
                     commandLine 'rpm', '--badreloc', '--nodeps', '--noscripts', '--notriggers',
-                        '--dbpath', rpmDatabase,
-                        '--relocate', "/=${rpmExtracted}",
-                        '-i', rpm
+                            '--dbpath', rpmDatabase,
+                            '--relocate', "/=${rpmExtracted}",
+                            '-i', rpm
                     doFirst {
                         rpmDatabase.deleteDir()
                         rpmExtracted.deleteDir()
@@ -249,32 +290,37 @@ class ClusterFormationTasks {
     }
 
     /** Adds a task to write elasticsearch.yml for the given node configuration */
-    static Task configureWriteConfigTask(String name, Project project, Task setup, NodeInfo node) {
+    static Task configureWriteConfigTask(String name, Project project, Task setup, NodeInfo node, NodeInfo seedNode) {
         Map esConfig = [
                 'cluster.name'                 : node.clusterName,
                 'pidfile'                      : node.pidFile,
                 'path.repo'                    : "${node.sharedDir}/repo",
                 'path.shared_data'             : "${node.sharedDir}/",
                 // Define a node attribute so we can test that it exists
-                'node.attr.testattr'                : 'test',
+                'node.attr.testattr'           : 'test',
                 'repositories.url.allowed_urls': 'http://snapshot.test*'
         ]
+        // we set min master nodes to the total number of nodes in the cluster and
+        // basically skip initial state recovery to allow the cluster to form using a realistic master election
+        // this means all nodes must be up, join the seed node and do a master election. This will also allow new and
+        // old nodes in the BWC case to become the master
+        if (node.config.useMinimumMasterNodes && node.config.numNodes > 1) {
+            esConfig['discovery.zen.minimum_master_nodes'] = node.config.numNodes
+            esConfig['discovery.initial_state_timeout'] = '0s' // don't wait for state.. just start up quickly
+        }
         esConfig['node.max_local_storage_nodes'] = node.config.numNodes
         esConfig['http.port'] = node.config.httpPort
         esConfig['transport.tcp.port'] =  node.config.transportPort
+        // Default the watermarks to absurdly low to prevent the tests from failing on nodes without enough disk space
+        esConfig['cluster.routing.allocation.disk.watermark.low'] = '1b'
+        esConfig['cluster.routing.allocation.disk.watermark.high'] = '1b'
         esConfig.putAll(node.config.settings)
 
         Task writeConfig = project.tasks.create(name: name, type: DefaultTask, dependsOn: setup)
         writeConfig.doFirst {
-            if (node.nodeNum > 0) { // multi-node cluster case, we have to wait for the seed node to startup
-                ant.waitfor(maxwait: '20', maxwaitunit: 'second', checkevery: '500', checkeveryunit: 'millisecond') {
-                    resourceexists {
-                        file(file: node.config.seedNodePortsFile.toString())
-                    }
-                }
-                // the seed node is enough to form the cluster - all subsequent nodes will get the seed node as a unicast
-                // host and join the cluster via that.
-                esConfig['discovery.zen.ping.unicast.hosts'] = "\"${node.config.seedNodeTransportUri()}\""
+            String unicastTransportUri = node.config.unicastTransportUri(seedNode, node, project.ant)
+            if (unicastTransportUri != null) {
+                esConfig['discovery.zen.ping.unicast.hosts'] = "\"${unicastTransportUri}\""
             }
             File configFile = new File(node.confDir, 'elasticsearch.yml')
             logger.info("Configuring ${configFile}")
@@ -282,6 +328,42 @@ class ClusterFormationTasks {
         }
     }
 
+    /** Adds a task to create keystore */
+    static Task configureCreateKeystoreTask(String name, Project project, Task setup, NodeInfo node) {
+        if (node.config.keystoreSettings.isEmpty()) {
+            return setup
+        } else {
+            /*
+             * We have to delay building the string as the path will not exist during configuration which will fail on Windows due to
+             * getting the short name requiring the path to already exist.
+             */
+            final Object esKeystoreUtil = "${-> node.binPath().resolve('elasticsearch-keystore').toString()}"
+            return configureExecTask(name, project, setup, node, esKeystoreUtil, 'create')
+        }
+    }
+
+    /** Adds tasks to add settings to the keystore */
+    static Task configureAddKeystoreSettingTasks(String parent, Project project, Task setup, NodeInfo node) {
+        Map kvs = node.config.keystoreSettings
+        Task parentTask = setup
+        /*
+         * We have to delay building the string as the path will not exist during configuration which will fail on Windows due to getting
+         * the short name requiring the path to already exist.
+         */
+        final Object esKeystoreUtil = "${-> node.binPath().resolve('elasticsearch-keystore').toString()}"
+        for (Map.Entry<String, String> entry in kvs) {
+            String key = entry.getKey()
+            String name = taskName(parent, node, 'addToKeystore#' + key)
+            Task t = configureExecTask(name, project, parentTask, node, esKeystoreUtil, 'add', key, '-x')
+            String settingsValue = entry.getValue() // eval this early otherwise it will not use the right value
+            t.doFirst {
+                standardInput = new ByteArrayInputStream(settingsValue.getBytes(StandardCharsets.UTF_8))
+            }
+            parentTask = t
+        }
+        return parentTask
+    }
+
     static Task configureExtraConfigFilesTask(String name, Project project, Task setup, NodeInfo node) {
         if (node.config.extraConfigFiles.isEmpty()) {
             return setup
@@ -318,20 +400,15 @@ class ClusterFormationTasks {
      * For each plugin, if the plugin has rest spec apis in its tests, those api files are also copied
      * to the test resources for this project.
      */
-    static Task configureCopyPluginsTask(String name, Project project, Task setup, NodeInfo node) {
-        if (node.config.plugins.isEmpty()) {
-            return setup
-        }
+    static Task configureCopyPluginsTask(String name, Project project, Task setup, NodeInfo node, String prefix) {
         Copy copyPlugins = project.tasks.create(name: name, type: Copy, dependsOn: setup)
 
         List<FileCollection> pluginFiles = []
         for (Map.Entry<String, Project> plugin : node.config.plugins.entrySet()) {
 
             Project pluginProject = plugin.getValue()
-            if (pluginProject.plugins.hasPlugin(PluginBuildPlugin) == false) {
-                throw new GradleException("Task ${name} cannot project ${pluginProject.path} which is not an esplugin")
-            }
-            String configurationName = "_plugin_${pluginProject.path}"
+            verifyProjectHasBuildPlugin(name, node.nodeVersion, project, pluginProject)
+            String configurationName = "_plugin_${prefix}_${pluginProject.path}"
             Configuration configuration = project.configurations.findByName(configurationName)
             if (configuration == null) {
                 configuration = project.configurations.create(configurationName)
@@ -360,6 +437,33 @@ class ClusterFormationTasks {
         return copyPlugins
     }
 
+    /** Configures task to copy a plugin based on a zip file resolved using dependencies for an older version */
+    static Task configureCopyBwcPluginsTask(String name, Project project, Task setup, NodeInfo node, String prefix) {
+        Configuration bwcPlugins = project.configurations.getByName("${prefix}_elasticsearchBwcPlugins")
+        for (Map.Entry<String, Project> plugin : node.config.plugins.entrySet()) {
+            Project pluginProject = plugin.getValue()
+            verifyProjectHasBuildPlugin(name, node.nodeVersion, project, pluginProject)
+            String configurationName = "_plugin_bwc_${prefix}_${pluginProject.path}"
+            Configuration configuration = project.configurations.findByName(configurationName)
+            if (configuration == null) {
+                configuration = project.configurations.create(configurationName)
+            }
+
+            final String depName = pluginProject.extensions.findByName('esplugin').name
+
+            Dependency dep = bwcPlugins.dependencies.find {
+                it.name == depName
+            }
+            configuration.dependencies.add(dep)
+        }
+
+        Copy copyPlugins = project.tasks.create(name: name, type: Copy, dependsOn: setup) {
+            from bwcPlugins
+            into node.pluginsTmpDir
+        }
+        return copyPlugins
+    }
+
     static Task configureInstallModuleTask(String name, Project project, Task setup, NodeInfo node, Project module) {
         if (node.config.distribution != 'integ-test-zip') {
             throw new GradleException("Module ${module.path} not allowed be installed distributions other than integ-test-zip because they should already have all modules bundled!")
@@ -374,11 +478,21 @@ class ClusterFormationTasks {
         return installModule
     }
 
-    static Task configureInstallPluginTask(String name, Project project, Task setup, NodeInfo node, Project plugin) {
-        FileCollection pluginZip = project.configurations.getByName("_plugin_${plugin.path}")
+    static Task configureInstallPluginTask(String name, Project project, Task setup, NodeInfo node, Project plugin, String prefix) {
+        final FileCollection pluginZip;
+        if (node.nodeVersion != VersionProperties.elasticsearch) {
+            pluginZip = project.configurations.getByName("_plugin_bwc_${prefix}_${plugin.path}")
+        } else {
+            pluginZip = project.configurations.getByName("_plugin_${prefix}_${plugin.path}")
+        }
         // delay reading the file location until execution time by wrapping in a closure within a GString
-        Object file = "${-> new File(node.pluginsTmpDir, pluginZip.singleFile.getName()).toURI().toURL().toString()}"
-        Object[] args = [new File(node.homeDir, 'bin/elasticsearch-plugin'), 'install', file]
+        final Object file = "${-> new File(node.pluginsTmpDir, pluginZip.singleFile.getName()).toURI().toURL().toString()}"
+        /*
+         * We have to delay building the string as the path will not exist during configuration which will fail on Windows due to getting
+         * the short name requiring the path to already exist.
+         */
+        final Object esPluginUtil = "${-> node.binPath().resolve('elasticsearch-plugin').toString()}"
+        final Object[] args = [esPluginUtil, 'install', file]
         return configureExecTask(name, project, setup, node, args)
     }
 
@@ -490,7 +604,7 @@ class ClusterFormationTasks {
                 anyNodeFailed |= node.failedMarker.exists()
             }
             if (ant.properties.containsKey("failed${name}".toString()) || anyNodeFailed) {
-                waitFailed(nodes, logger, 'Failed to start elasticsearch')
+                waitFailed(project, nodes, logger, 'Failed to start elasticsearch')
             }
 
             // go through each node checking the wait condition
@@ -507,14 +621,14 @@ class ClusterFormationTasks {
                 }
 
                 if (success == false) {
-                    waitFailed(nodes, logger, 'Elasticsearch cluster failed to pass wait condition')
+                    waitFailed(project, nodes, logger, 'Elasticsearch cluster failed to pass wait condition')
                 }
             }
         }
         return wait
     }
 
-    static void waitFailed(List<NodeInfo> nodes, Logger logger, String msg) {
+    static void waitFailed(Project project, List<NodeInfo> nodes, Logger logger, String msg) {
         for (NodeInfo node : nodes) {
             if (logger.isInfoEnabled() == false) {
                 // We already log the command at info level. No need to do it twice.
@@ -534,6 +648,17 @@ class ClusterFormationTasks {
                 logger.error("|\n|  [log]")
                 node.startLog.eachLine { line -> logger.error("|    ${line}") }
             }
+            if (node.pidFile.exists() && node.failedMarker.exists() == false &&
+                (node.httpPortsFile.exists() == false || node.transportPortsFile.exists() == false)) {
+                logger.error("|\n|  [jstack]")
+                String pid = node.pidFile.getText('UTF-8')
+                ByteArrayOutputStream output = new ByteArrayOutputStream()
+                project.exec {
+                    commandLine = ["${project.javaHome}/bin/jstack", pid]
+                    standardOutput = output
+                }
+                output.toString('UTF-8').eachLine { line -> logger.error("|    ${line}") }
+            }
             logger.error("|-----------------------------------------")
         }
         throw new GradleException(msg)
@@ -558,11 +683,11 @@ class ClusterFormationTasks {
             standardOutput = new ByteArrayOutputStream()
             doLast {
                 String out = standardOutput.toString()
-                if (out.contains("${pid} org.elasticsearch.bootstrap.Elasticsearch") == false) {
+                if (out.contains("${ext.pid} org.elasticsearch.bootstrap.Elasticsearch") == false) {
                     logger.error('jps -l')
                     logger.error(out)
-                    logger.error("pid file: ${pidFile}")
-                    logger.error("pid: ${pid}")
+                    logger.error("pid file: ${node.pidFile}")
+                    logger.error("pid: ${ext.pid}")
                     throw new GradleException("jps -l did not report any process with org.elasticsearch.bootstrap.Elasticsearch\n" +
                             "Did you run gradle clean? Maybe an old pid file is still lying around.")
                 } else {
@@ -599,11 +724,11 @@ class ClusterFormationTasks {
     }
 
     /** Returns a unique task name for this task and node configuration */
-    static String taskName(Task parentTask, NodeInfo node, String action) {
+    static String taskName(String prefix, NodeInfo node, String action) {
         if (node.config.numNodes > 1) {
-            return "${parentTask.name}#node${node.nodeNum}.${action}"
+            return "${prefix}#node${node.nodeNum}.${action}"
         } else {
-            return "${parentTask.name}#${action}"
+            return "${prefix}#${action}"
         }
     }
 
@@ -625,4 +750,11 @@ class ClusterFormationTasks {
         project.ant.project.removeBuildListener(listener)
         return retVal
     }
+
+    static void verifyProjectHasBuildPlugin(String name, String version, Project project, Project pluginProject) {
+        if (pluginProject.plugins.hasPlugin(PluginBuildPlugin) == false) {
+            throw new GradleException("Task [${name}] cannot add plugin [${pluginProject.path}] with version [${version}] to project's " +
+                    "[${project.path}] dependencies: the plugin is not an esplugin")
+        }
+    }
 }
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/Fixture.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/Fixture.groovy
index 46b81624ba3fa..498a1627b3598 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/Fixture.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/Fixture.groovy
@@ -16,272 +16,15 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-
 package org.elasticsearch.gradle.test
 
-import org.apache.tools.ant.taskdefs.condition.Os
-import org.elasticsearch.gradle.AntTask
-import org.elasticsearch.gradle.LoggedExec
-import org.gradle.api.GradleException
-import org.gradle.api.Task
-import org.gradle.api.tasks.Exec
-import org.gradle.api.tasks.Input
-
 /**
- * A fixture for integration tests which runs in a separate process.
+ * Any object that can produce an accompanying stop task, meant to tear down
+ * a previously instantiated service.
  */
-public class Fixture extends AntTask {
-
-    /** The path to the executable that starts the fixture. */
-    @Input
-    String executable
-
-    private final List<Object> arguments = new ArrayList<>()
-
-    @Input
-    public void args(Object... args) {
-        arguments.addAll(args)
-    }
-
-    /**
-     * Environment variables for the fixture process. The value can be any object, which
-     * will have toString() called at execution time.
-     */
-    private final Map<String, Object> environment = new HashMap<>()
-
-    @Input
-    public void env(String key, Object value) {
-        environment.put(key, value)
-    }
-
-    /** A flag to indicate whether the command should be executed from a shell. */
-    @Input
-    boolean useShell = false
-
-    /**
-     * A flag to indicate whether the fixture should be run in the foreground, or spawned.
-     * It is protected so subclasses can override (eg RunTask).
-     */
-    protected boolean spawn = true
-
-    /**
-     * A closure to call before the fixture is considered ready. The closure is passed the fixture object,
-     * as well as a groovy AntBuilder, to enable running ant condition checks. The default wait
-     * condition is for http on the http port.
-     */
-    @Input
-    Closure waitCondition = { Fixture fixture, AntBuilder ant ->
-        File tmpFile = new File(fixture.cwd, 'wait.success')
-        ant.get(src: "http://${fixture.addressAndPort}",
-                dest: tmpFile.toString(),
-                ignoreerrors: true, // do not fail on error, so logging information can be flushed
-                retries: 10)
-        return tmpFile.exists()
-    }
+public interface Fixture {
 
     /** A task which will stop this fixture. This should be used as a finalizedBy for any tasks that use the fixture. */
-    public final Task stopTask
-
-    public Fixture() {
-        stopTask = createStopTask()
-        finalizedBy(stopTask)
-    }
-
-    @Override
-    protected void runAnt(AntBuilder ant) {
-        project.delete(baseDir) // reset everything
-        cwd.mkdirs()
-        final String realExecutable
-        final List<Object> realArgs = new ArrayList<>()
-        final Map<String, Object> realEnv = environment
-        // We need to choose which executable we are using. In shell mode, or when we
-        // are spawning and thus using the wrapper script, the executable is the shell.
-        if (useShell || spawn) {
-            if (Os.isFamily(Os.FAMILY_WINDOWS)) {
-                realExecutable = 'cmd'
-                realArgs.add('/C')
-                realArgs.add('"') // quote the entire command
-            } else {
-                realExecutable = 'sh'
-            }
-        } else {
-            realExecutable = executable
-            realArgs.addAll(arguments)
-        }
-        if (spawn) {
-            writeWrapperScript(executable)
-            realArgs.add(wrapperScript)
-            realArgs.addAll(arguments)
-        }
-        if (Os.isFamily(Os.FAMILY_WINDOWS) && (useShell || spawn)) {
-            realArgs.add('"')
-        }
-        commandString.eachLine { line -> logger.info(line) }
-
-        ant.exec(executable: realExecutable, spawn: spawn, dir: cwd, taskname: name) {
-            realEnv.each { key, value -> env(key: key, value: value) }
-            realArgs.each { arg(value: it) }
-        }
-
-        String failedProp = "failed${name}"
-        // first wait for resources, or the failure marker from the wrapper script
-        ant.waitfor(maxwait: '30', maxwaitunit: 'second', checkevery: '500', checkeveryunit: 'millisecond', timeoutproperty: failedProp) {
-            or {
-                resourceexists {
-                    file(file: failureMarker.toString())
-                }
-                and {
-                    resourceexists {
-                        file(file: pidFile.toString())
-                    }
-                    resourceexists {
-                        file(file: portsFile.toString())
-                    }
-                }
-            }
-        }
-
-        if (ant.project.getProperty(failedProp) || failureMarker.exists()) {
-            fail("Failed to start ${name}")
-        }
-
-        // the process is started (has a pid) and is bound to a network interface
-        // so now wait undil the waitCondition has been met
-        // TODO: change this to a loop?
-        boolean success
-        try {
-            success = waitCondition(this, ant) == false
-        } catch (Exception e) {
-            String msg = "Wait condition caught exception for ${name}"
-            logger.error(msg, e)
-            fail(msg, e)
-        }
-        if (success == false) {
-            fail("Wait condition failed for ${name}")
-        }
-    }
-
-    /** Returns a debug string used to log information about how the fixture was run. */
-    protected String getCommandString() {
-        String commandString = "\n${name} configuration:\n"
-        commandString += "-----------------------------------------\n"
-        commandString += "  cwd: ${cwd}\n"
-        commandString += "  command: ${executable} ${arguments.join(' ')}\n"
-        commandString += '  environment:\n'
-        environment.each { k, v -> commandString += "    ${k}: ${v}\n" }
-        if (spawn) {
-            commandString += "\n  [${wrapperScript.name}]\n"
-            wrapperScript.eachLine('UTF-8', { line -> commandString += "    ${line}\n"})
-        }
-        return commandString
-    }
-
-    /**
-     * Writes a script to run the real executable, so that stdout/stderr can be captured.
-     * TODO: this could be removed if we do use our own ProcessBuilder and pump output from the process
-     */
-    private void writeWrapperScript(String executable) {
-        wrapperScript.parentFile.mkdirs()
-        String argsPasser = '"$@"'
-        String exitMarker = "; if [ \$? != 0 ]; then touch run.failed; fi"
-        if (Os.isFamily(Os.FAMILY_WINDOWS)) {
-            argsPasser = '%*'
-            exitMarker = "\r\n if \"%errorlevel%\" neq \"0\" ( type nul >> run.failed )"
-        }
-        wrapperScript.setText("\"${executable}\" ${argsPasser} > run.log 2>&1 ${exitMarker}", 'UTF-8')
-    }
-
-    /** Fail the build with the given message, and logging relevant info*/
-    private void fail(String msg, Exception... suppressed) {
-        if (logger.isInfoEnabled() == false) {
-            // We already log the command at info level. No need to do it twice.
-            commandString.eachLine { line -> logger.error(line) }
-        }
-        logger.error("${name} output:")
-        logger.error("-----------------------------------------")
-        logger.error("  failure marker exists: ${failureMarker.exists()}")
-        logger.error("  pid file exists: ${pidFile.exists()}")
-        logger.error("  ports file exists: ${portsFile.exists()}")
-        // also dump the log file for the startup script (which will include ES logging output to stdout)
-        if (runLog.exists()) {
-            logger.error("\n  [log]")
-            runLog.eachLine { line -> logger.error("    ${line}") }
-        }
-        logger.error("-----------------------------------------")
-        GradleException toThrow = new GradleException(msg)
-        for (Exception e : suppressed) {
-            toThrow.addSuppressed(e)
-        }
-        throw toThrow
-    }
-
-    /** Adds a task to kill an elasticsearch node with the given pidfile */
-    private Task createStopTask() {
-        final Fixture fixture = this
-        final Object pid = "${ -> fixture.pid }"
-        Exec stop = project.tasks.create(name: "${name}#stop", type: LoggedExec)
-        stop.onlyIf { fixture.pidFile.exists() }
-        stop.doFirst {
-            logger.info("Shutting down ${fixture.name} with pid ${pid}")
-        }
-        if (Os.isFamily(Os.FAMILY_WINDOWS)) {
-            stop.executable = 'Taskkill'
-            stop.args('/PID', pid, '/F')
-        } else {
-            stop.executable = 'kill'
-            stop.args('-9', pid)
-        }
-        stop.doLast {
-            project.delete(fixture.pidFile)
-        }
-        return stop
-    }
-
-    /**
-     * A path relative to the build dir that all configuration and runtime files
-     * will live in for this fixture
-     */
-    protected File getBaseDir() {
-        return new File(project.buildDir, "fixtures/${name}")
-    }
-
-    /** Returns the working directory for the process. Defaults to "cwd" inside baseDir. */
-    protected File getCwd() {
-        return new File(baseDir, 'cwd')
-    }
-
-    /** Returns the file the process writes its pid to. Defaults to "pid" inside baseDir. */
-    protected File getPidFile() {
-        return new File(baseDir, 'pid')
-    }
-
-    /** Reads the pid file and returns the process' pid */
-    public int getPid() {
-        return Integer.parseInt(pidFile.getText('UTF-8').trim())
-    }
-
-    /** Returns the file the process writes its bound ports to. Defaults to "ports" inside baseDir. */
-    protected File getPortsFile() {
-        return new File(baseDir, 'ports')
-    }
-
-    /** Returns an address and port suitable for a uri to connect to this node over http */
-    public String getAddressAndPort() {
-        return portsFile.readLines("UTF-8").get(0)
-    }
-
-    /** Returns a file that wraps around the actual command when {@code spawn == true}. */
-    protected File getWrapperScript() {
-        return new File(cwd, Os.isFamily(Os.FAMILY_WINDOWS) ? 'run.bat' : 'run')
-    }
-
-    /** Returns a file that the wrapper script writes when the command failed. */
-    protected File getFailureMarker() {
-        return new File(cwd, 'run.failed')
-    }
+    public Object getStopTask()
 
-    /** Returns a file that the wrapper script writes when the command failed. */
-    protected File getRunLog() {
-        return new File(cwd, 'run.log')
-    }
 }
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/JNAKernel32Library.java b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/JNAKernel32Library.java
new file mode 100644
index 0000000000000..4d069cd434fc0
--- /dev/null
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/JNAKernel32Library.java
@@ -0,0 +1,25 @@
+package org.elasticsearch.gradle.test;
+
+import com.sun.jna.Native;
+import com.sun.jna.WString;
+import org.apache.tools.ant.taskdefs.condition.Os;
+
+public class JNAKernel32Library {
+
+    private static final class Holder {
+        private static final JNAKernel32Library instance = new JNAKernel32Library();
+    }
+
+    static JNAKernel32Library getInstance() {
+        return Holder.instance;
+    }
+
+    private JNAKernel32Library() {
+        if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+            Native.register("kernel32");
+        }
+    }
+
+    native int GetShortPathNameW(WString lpszLongPath, char[] lpszShortPath, int cchBuffer);
+
+}
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy
index 5d9961a0425ee..9a237c628de98 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy
@@ -18,10 +18,15 @@
  */
 package org.elasticsearch.gradle.test
 
+import com.sun.jna.Native
+import com.sun.jna.WString
 import org.apache.tools.ant.taskdefs.condition.Os
 import org.gradle.api.InvalidUserDataException
 import org.gradle.api.Project
-import org.gradle.api.Task
+
+import java.nio.file.Files
+import java.nio.file.Path
+import java.nio.file.Paths
 
 /**
  * A container for the files and configuration associated with a single node in a test cluster.
@@ -57,6 +62,9 @@ class NodeInfo {
     /** config directory */
     File confDir
 
+    /** data directory (as an Object, to allow lazy evaluation) */
+    Object dataDir
+
     /** THE config file */
     File configFile
 
@@ -82,24 +90,40 @@ class NodeInfo {
     String executable
 
     /** Path to the elasticsearch start script */
-    File esScript
+    private Object esScript
 
     /** script to run when running in the background */
-    File wrapperScript
+    private File wrapperScript
 
     /** buffer for ant output when starting this node */
     ByteArrayOutputStream buffer = new ByteArrayOutputStream()
 
-    /** Creates a node to run as part of a cluster for the given task */
-    NodeInfo(ClusterConfiguration config, int nodeNum, Project project, Task task, String nodeVersion, File sharedDir) {
+    /** the version of elasticsearch that this node runs */
+    String nodeVersion
+
+    /** Holds node configuration for part of a test cluster. */
+    NodeInfo(ClusterConfiguration config, int nodeNum, Project project, String prefix, String nodeVersion, File sharedDir) {
         this.config = config
         this.nodeNum = nodeNum
         this.sharedDir = sharedDir
-        clusterName = "${task.path.replace(':', '_').substring(1)}"
-        baseDir = new File(project.buildDir, "cluster/${task.name} node${nodeNum}")
+        if (config.clusterName != null) {
+            clusterName = config.clusterName
+        } else {
+            clusterName = project.path.replace(':', '_').substring(1) + '_' + prefix
+        }
+        baseDir = new File(project.buildDir, "cluster/${prefix} node${nodeNum}")
         pidFile = new File(baseDir, 'es.pid')
+        this.nodeVersion = nodeVersion
         homeDir = homeDir(baseDir, config.distribution, nodeVersion)
         confDir = confDir(baseDir, config.distribution, nodeVersion)
+        if (config.dataDir != null) {
+            if (config.numNodes != 1) {
+                throw new IllegalArgumentException("Cannot set data dir for integ test with more than one node")
+            }
+            dataDir = config.dataDir
+        } else {
+            dataDir = new File(homeDir, "data")
+        }
         configFile = new File(confDir, 'elasticsearch.yml')
         // even for rpm/deb, the logs are under home because we dont start with real services
         File logsDir = new File(homeDir, 'logs')
@@ -116,22 +140,37 @@ class NodeInfo {
             args.add('/C')
             args.add('"') // quote the entire command
             wrapperScript = new File(cwd, "run.bat")
-            esScript = new File(homeDir, 'bin/elasticsearch.bat')
+            /*
+             * We have to delay building the string as the path will not exist during configuration which will fail on Windows due to
+             * getting the short name requiring the path to already exist.
+             */
+            esScript = "${-> binPath().resolve('elasticsearch.bat').toString()}"
         } else {
-            executable = 'sh'
+            executable = 'bash'
             wrapperScript = new File(cwd, "run")
-            esScript = new File(homeDir, 'bin/elasticsearch')
+            esScript = binPath().resolve('elasticsearch')
         }
         if (config.daemonize) {
-            args.add("${wrapperScript}")
+            if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+                /*
+                 * We have to delay building the string as the path will not exist during configuration which will fail on Windows due to
+                 * getting the short name requiring the path to already exist.
+                 */
+                args.add("${-> getShortPathName(wrapperScript.toString())}")
+            } else {
+                args.add("${wrapperScript}")
+            }
         } else {
             args.add("${esScript}")
         }
 
-        env = [ 'JAVA_HOME' : project.javaHome ]
+        env = ['JAVA_HOME': project.javaHome]
         args.addAll("-E", "node.portsfile=true")
         String collectedSystemProperties = config.systemProperties.collect { key, value -> "-D${key}=${value}" }.join(" ")
         String esJavaOpts = config.jvmArgs.isEmpty() ? collectedSystemProperties : collectedSystemProperties + " " + config.jvmArgs
+        if (Boolean.parseBoolean(System.getProperty('tests.asserts', 'true'))) {
+            esJavaOpts += " -ea -esa"
+        }
         env.put('ES_JAVA_OPTS', esJavaOpts)
         for (Map.Entry<String, String> property : System.properties.entrySet()) {
             if (property.key.startsWith('tests.es.')) {
@@ -139,13 +178,68 @@ class NodeInfo {
                 args.add("${property.key.substring('tests.es.'.size())}=${property.value}")
             }
         }
-        env.put('ES_JVM_OPTIONS', new File(confDir, 'jvm.options'))
-        args.addAll("-E", "path.conf=${confDir}")
+        final File jvmOptionsFile = new File(confDir, 'jvm.options')
+        if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+            /*
+             * We have to delay building the string as the path will not exist during configuration which will fail on Windows due to
+             * getting the short name requiring the path to already exist.
+             */
+            env.put('ES_JVM_OPTIONS', "${-> getShortPathName(jvmOptionsFile.toString())}")
+        } else {
+            env.put('ES_JVM_OPTIONS', jvmOptionsFile)
+        }
+        if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+            /*
+             * We have to delay building the string as the path will not exist during configuration which will fail on Windows due to
+             * getting the short name requiring the path to already exist.
+             */
+            args.addAll("-E", "path.conf=${-> getShortPathName(confDir.toString())}")
+        } else {
+            args.addAll("-E", "path.conf=${confDir}")
+        }
+        if (!System.properties.containsKey("tests.es.path.data")) {
+            if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+                /*
+                 * We have to delay building the string as the path will not exist during configuration which will fail on Windows due to
+                 * getting the short name requiring the path to already exist. This one is extra tricky because usually we rely on the node
+                 * creating its data directory on startup but we simply can not do that here because getting the short path name requires
+                 * the directory to already exist. Therefore, we create this directory immediately before getting the short name.
+                 */
+                args.addAll("-E", "path.data=${-> Files.createDirectories(Paths.get(dataDir.toString())); getShortPathName(dataDir.toString())}")
+            } else {
+                args.addAll("-E", "path.data=${-> dataDir.toString()}")
+            }
+        }
         if (Os.isFamily(Os.FAMILY_WINDOWS)) {
             args.add('"') // end the entire command, quoted
         }
     }
 
+    Path binPath() {
+        if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+            return Paths.get(getShortPathName(new File(homeDir, 'bin').toString()))
+        } else {
+            return Paths.get(new File(homeDir, 'bin').toURI())
+        }
+    }
+
+    static String getShortPathName(String path) {
+        assert Os.isFamily(Os.FAMILY_WINDOWS)
+        final WString longPath = new WString("\\\\?\\" + path)
+        // first we get the length of the buffer needed
+        final int length = JNAKernel32Library.getInstance().GetShortPathNameW(longPath, null, 0)
+        if (length == 0) {
+            throw new IllegalStateException("path [" + path + "] encountered error [" + Native.getLastError() + "]")
+        }
+        final char[] shortPath = new char[length]
+        // knowing the length of the buffer, now we get the short name
+        if (JNAKernel32Library.getInstance().GetShortPathNameW(longPath, shortPath, length) == 0) {
+            throw new IllegalStateException("path [" + path + "] encountered error [" + Native.getLastError() + "]")
+        }
+        // we have to strip the \\?\ away from the path for cmd.exe
+        return Native.toString(shortPath).substring(4)
+    }
+
     /** Returns debug string for the command that started this node. */
     String getCommandString() {
         String esCommandString = "\nNode ${nodeNum} configuration:\n"
@@ -184,6 +278,19 @@ class NodeInfo {
         return transportPortsFile.readLines("UTF-8").get(0)
     }
 
+    /** Returns the file which contains the transport protocol ports for this node */
+    File getTransportPortsFile() {
+        return transportPortsFile
+    }
+
+    /** Returns the data directory for this node */
+    File getDataDir() {
+        if (!(dataDir instanceof File)) {
+            return new File(dataDir)
+        }
+        return dataDir
+    }
+
     /** Returns the directory elasticsearch home is contained in for the given distribution */
     static File homeDir(File baseDir, String distro, String nodeVersion) {
         String path
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy
index c6463d2881185..859c89c693806 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy
@@ -20,80 +20,114 @@ package org.elasticsearch.gradle.test
 
 import com.carrotsearch.gradle.junit4.RandomizedTestingTask
 import org.elasticsearch.gradle.BuildPlugin
+import org.gradle.api.DefaultTask
 import org.gradle.api.Task
+import org.gradle.api.execution.TaskExecutionAdapter
 import org.gradle.api.internal.tasks.options.Option
 import org.gradle.api.plugins.JavaBasePlugin
 import org.gradle.api.tasks.Input
-import org.gradle.util.ConfigureUtil
+import org.gradle.api.tasks.TaskState
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.util.stream.Stream
 
 /**
- * Runs integration tests, but first starts an ES cluster,
- * and passes the ES cluster info as parameters to the tests.
+ * A wrapper task around setting up a cluster and running rest tests.
  */
-public class RestIntegTestTask extends RandomizedTestingTask {
+public class RestIntegTestTask extends DefaultTask {
+
+    protected ClusterConfiguration clusterConfig
+
+    protected RandomizedTestingTask runner
 
-    ClusterConfiguration clusterConfig
+    protected Task clusterInit
+
+    /** Info about nodes in the integ test cluster. Note this is *not* available until runtime. */
+    List<NodeInfo> nodes
 
     /** Flag indicating whether the rest tests in the rest spec should be run. */
     @Input
     boolean includePackaged = false
 
     public RestIntegTestTask() {
-        description = 'Runs rest tests against an elasticsearch cluster.'
-        group = JavaBasePlugin.VERIFICATION_GROUP
-        dependsOn(project.testClasses)
-        classpath = project.sourceSets.test.runtimeClasspath
-        testClassesDir = project.sourceSets.test.output.classesDir
-        clusterConfig = new ClusterConfiguration(project)
+        runner = project.tasks.create("${name}Runner", RandomizedTestingTask.class)
+        super.dependsOn(runner)
+        clusterInit = project.tasks.create(name: "${name}Cluster#init", dependsOn: project.testClasses)
+        runner.dependsOn(clusterInit)
+        runner.classpath = project.sourceSets.test.runtimeClasspath
+        runner.testClassesDir = project.sourceSets.test.output.classesDir
+        clusterConfig = project.extensions.create("${name}Cluster", ClusterConfiguration.class, project)
 
         // start with the common test configuration
-        configure(BuildPlugin.commonTestConfig(project))
+        runner.configure(BuildPlugin.commonTestConfig(project))
         // override/add more for rest tests
-        parallelism = '1'
-        include('**/*IT.class')
-        systemProperty('tests.rest.load_packaged', 'false')
+        runner.parallelism = '1'
+        runner.include('**/*IT.class')
+        runner.systemProperty('tests.rest.load_packaged', 'false')
+        // we pass all nodes to the rest cluster to allow the clients to round-robin between them
+        // this is more realistic than just talking to a single node
+        runner.systemProperty('tests.rest.cluster', "${-> nodes.collect{it.httpUri()}.join(",")}")
+        runner.systemProperty('tests.config.dir', "${-> nodes[0].confDir}")
+        // TODO: our "client" qa tests currently use the rest-test plugin. instead they should have their own plugin
+        // that sets up the test cluster and passes this transport uri instead of http uri. Until then, we pass
+        // both as separate sysprops
+        runner.systemProperty('tests.cluster', "${-> nodes[0].transportUri()}")
+
+        // dump errors and warnings from cluster log on failure
+        TaskExecutionAdapter logDumpListener = new TaskExecutionAdapter() {
+            @Override
+            void afterExecute(Task task, TaskState state) {
+                if (state.failure != null) {
+                    for (NodeInfo nodeInfo : nodes) {
+                        printLogExcerpt(nodeInfo)
+                    }
+                }
+            }
+        }
+        runner.doFirst {
+            project.gradle.addListener(logDumpListener)
+        }
+        runner.doLast {
+            project.gradle.removeListener(logDumpListener)
+        }
 
         // copy the rest spec/tests into the test resources
         RestSpecHack.configureDependencies(project)
         project.afterEvaluate {
-            dependsOn(RestSpecHack.configureTask(project, includePackaged))
+            runner.dependsOn(RestSpecHack.configureTask(project, includePackaged))
         }
         // this must run after all projects have been configured, so we know any project
         // references can be accessed as a fully configured
         project.gradle.projectsEvaluated {
-            NodeInfo node = ClusterFormationTasks.setup(project, this, clusterConfig)
-            systemProperty('tests.rest.cluster', "${-> node.httpUri()}")
-            systemProperty('tests.config.dir', "${-> node.confDir}")
-            // TODO: our "client" qa tests currently use the rest-test plugin. instead they should have their own plugin
-            // that sets up the test cluster and passes this transport uri instead of http uri. Until then, we pass
-            // both as separate sysprops
-            systemProperty('tests.cluster', "${-> node.transportUri()}")
+            if (enabled == false) {
+                runner.enabled = false
+                clusterInit.enabled = false
+                return // no need to add cluster formation tasks if the task won't run!
+            }
+            nodes = ClusterFormationTasks.setup(project, "${name}Cluster", runner, clusterConfig)
+            super.dependsOn(runner.finalizedBy)
         }
     }
 
     @Option(
-        option = "debug-jvm",
-        description = "Enable debugging configuration, to allow attaching a debugger to elasticsearch."
+            option = "debug-jvm",
+            description = "Enable debugging configuration, to allow attaching a debugger to elasticsearch."
     )
     public void setDebug(boolean enabled) {
         clusterConfig.debug = enabled;
     }
 
-    @Input
-    public void cluster(Closure closure) {
-        ConfigureUtil.configure(closure, clusterConfig)
-    }
-
-    public ClusterConfiguration getCluster() {
-        return clusterConfig
+    public List<NodeInfo> getNodes() {
+        return nodes
     }
 
     @Override
     public Task dependsOn(Object... dependencies) {
-        super.dependsOn(dependencies)
+        runner.dependsOn(dependencies)
         for (Object dependency : dependencies) {
             if (dependency instanceof Fixture) {
-                finalizedBy(((Fixture)dependency).stopTask)
+                runner.finalizedBy(((Fixture)dependency).getStopTask())
             }
         }
         return this
@@ -101,11 +135,54 @@ public class RestIntegTestTask extends RandomizedTestingTask {
 
     @Override
     public void setDependsOn(Iterable<?> dependencies) {
-        super.setDependsOn(dependencies)
+        runner.setDependsOn(dependencies)
         for (Object dependency : dependencies) {
             if (dependency instanceof Fixture) {
-                finalizedBy(((Fixture)dependency).stopTask)
+                runner.finalizedBy(((Fixture)dependency).getStopTask())
+            }
+        }
+    }
+
+    @Override
+    public Task mustRunAfter(Object... tasks) {
+        clusterInit.mustRunAfter(tasks)
+    }
+
+    /** Print out an excerpt of the log from the given node. */
+    protected static void printLogExcerpt(NodeInfo nodeInfo) {
+        File logFile = new File(nodeInfo.homeDir, "logs/${nodeInfo.clusterName}.log")
+        println("\nCluster ${nodeInfo.clusterName} - node ${nodeInfo.nodeNum} log excerpt:")
+        println("(full log at ${logFile})")
+        println('-----------------------------------------')
+        Stream<String> stream = Files.lines(logFile.toPath(), StandardCharsets.UTF_8)
+        try {
+            boolean inStartup = true
+            boolean inExcerpt = false
+            int linesSkipped = 0
+            for (String line : stream) {
+                if (line.startsWith("[")) {
+                    inExcerpt = false // clear with the next log message
+                }
+                if (line =~ /(\[WARN\])|(\[ERROR\])/) {
+                    inExcerpt = true // show warnings and errors
+                }
+                if (inStartup || inExcerpt) {
+                    if (linesSkipped != 0) {
+                        println("... SKIPPED ${linesSkipped} LINES ...")
+                    }
+                    println(line)
+                    linesSkipped = 0
+                } else {
+                    ++linesSkipped
+                }
+                if (line =~ /recovered \[\d+\] indices into cluster_state/) {
+                    inStartup = false
+                }
             }
+        } finally {
+            stream.close()
         }
+        println('=========================================')
+
     }
 }
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestSpecHack.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestSpecHack.groovy
index 43b5c2f6f38d2..296ae7115789f 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestSpecHack.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestSpecHack.groovy
@@ -43,18 +43,22 @@ public class RestSpecHack {
     }
 
     /**
-     * Creates a task to copy the rest spec files.
+     * Creates a task (if necessary) to copy the rest spec files.
      *
      * @param project The project to add the copy task to
      * @param includePackagedTests true if the packaged tests should be copied, false otherwise
      */
     public static Task configureTask(Project project, boolean includePackagedTests) {
+        Task copyRestSpec = project.tasks.findByName('copyRestSpec')
+        if (copyRestSpec != null) {
+            return copyRestSpec
+        }
         Map copyRestSpecProps = [
                 name     : 'copyRestSpec',
                 type     : Copy,
                 dependsOn: [project.configurations.restSpec, 'processTestResources']
         ]
-        Task copyRestSpec = project.tasks.create(copyRestSpecProps) {
+        copyRestSpec = project.tasks.create(copyRestSpecProps) {
             from { project.zipTree(project.configurations.restSpec.singleFile) }
             include 'rest-api-spec/api/**'
             if (includePackagedTests) {
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestTestPlugin.groovy
index dc9aa7693889d..da1462412812a 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestTestPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestTestPlugin.groovy
@@ -18,18 +18,35 @@
  */
 package org.elasticsearch.gradle.test
 
+import org.elasticsearch.gradle.BuildPlugin
+import org.gradle.api.InvalidUserDataException
 import org.gradle.api.Plugin
 import org.gradle.api.Project
+import org.gradle.api.plugins.JavaBasePlugin
 
-/** A plugin to add rest integration tests. Used for qa projects.  */
+/**
+ * Adds support for starting an Elasticsearch cluster before running integration
+ * tests. Used in conjunction with {@link StandaloneRestTestPlugin} for qa
+ * projects and in conjunction with {@link BuildPlugin} for testing the rest
+ * client.
+ */
 public class RestTestPlugin implements Plugin<Project> {
+    List REQUIRED_PLUGINS = [
+        'elasticsearch.build',
+        'elasticsearch.standalone-rest-test']
 
     @Override
     public void apply(Project project) {
-        project.pluginManager.apply(StandaloneTestBasePlugin)
+        if (false == REQUIRED_PLUGINS.any {project.pluginManager.hasPlugin(it)}) {
+            throw new InvalidUserDataException('elasticsearch.rest-test '
+                + 'requires either elasticsearch.build or '
+                + 'elasticsearch.standalone-rest-test')
+        }
 
         RestIntegTestTask integTest = project.tasks.create('integTest', RestIntegTestTask.class)
-        integTest.cluster.distribution = 'zip' // rest tests should run with the real zip
+        integTest.description = 'Runs rest tests against an elasticsearch cluster.'
+        integTest.group = JavaBasePlugin.VERIFICATION_GROUP
+        integTest.clusterConfig.distribution = 'zip' // rest tests should run with the real zip
         integTest.mustRunAfter(project.precommit)
         project.check.dependsOn(integTest)
     }
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RunTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RunTask.groovy
index f045c95740ba6..a88152d7865ff 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RunTask.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RunTask.groovy
@@ -16,8 +16,9 @@ public class RunTask extends DefaultTask {
         clusterConfig.httpPort = 9200
         clusterConfig.transportPort = 9300
         clusterConfig.daemonize = false
+        clusterConfig.distribution = 'zip'
         project.afterEvaluate {
-            ClusterFormationTasks.setup(project, this, clusterConfig)
+            ClusterFormationTasks.setup(project, name, this, clusterConfig)
         }
     }
 
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy
new file mode 100644
index 0000000000000..66868aafac0b0
--- /dev/null
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy
@@ -0,0 +1,64 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+package org.elasticsearch.gradle.test
+
+import com.carrotsearch.gradle.junit4.RandomizedTestingPlugin
+import org.elasticsearch.gradle.BuildPlugin
+import org.elasticsearch.gradle.VersionProperties
+import org.elasticsearch.gradle.precommit.PrecommitTasks
+import org.gradle.api.InvalidUserDataException
+import org.gradle.api.Plugin
+import org.gradle.api.Project
+import org.gradle.api.plugins.JavaBasePlugin
+
+/**
+ * Configures the build to compile tests against Elasticsearch's test framework
+ * and run REST tests. Use BuildPlugin if you want to build main code as well
+ * as tests.
+ */
+public class StandaloneRestTestPlugin implements Plugin<Project> {
+
+    @Override
+    public void apply(Project project) {
+        if (project.pluginManager.hasPlugin('elasticsearch.build')) {
+            throw new InvalidUserDataException('elasticsearch.standalone-test '
+                + 'elasticsearch.standalone-rest-test, and elasticsearch.build '
+                + 'are mutually exclusive')
+        }
+        project.pluginManager.apply(JavaBasePlugin)
+        project.pluginManager.apply(RandomizedTestingPlugin)
+
+        BuildPlugin.globalBuildInfo(project)
+        BuildPlugin.configureRepositories(project)
+
+        // only setup tests to build
+        project.sourceSets.create('test')
+        project.dependencies.add('testCompile', "org.elasticsearch.test:framework:${VersionProperties.elasticsearch}")
+
+        project.eclipse.classpath.sourceSets = [project.sourceSets.test]
+        project.eclipse.classpath.plusConfigurations = [project.configurations.testRuntime]
+        project.idea.module.testSourceDirs += project.sourceSets.test.java.srcDirs
+        project.idea.module.scopes['TEST'] = [plus: [project.configurations.testRuntime]]
+
+        PrecommitTasks.create(project, false)
+        project.check.dependsOn(project.precommit)
+    }
+}
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneTestBasePlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneTestBasePlugin.groovy
deleted file mode 100644
index af2b20e4abfab..0000000000000
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneTestBasePlugin.groovy
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-package org.elasticsearch.gradle.test
-
-import com.carrotsearch.gradle.junit4.RandomizedTestingPlugin
-import org.elasticsearch.gradle.BuildPlugin
-import org.elasticsearch.gradle.VersionProperties
-import org.elasticsearch.gradle.precommit.PrecommitTasks
-import org.gradle.api.Plugin
-import org.gradle.api.Project
-import org.gradle.api.plugins.JavaBasePlugin
-
-/** Configures the build to have a rest integration test.  */
-public class StandaloneTestBasePlugin implements Plugin<Project> {
-
-    @Override
-    public void apply(Project project) {
-        project.pluginManager.apply(JavaBasePlugin)
-        project.pluginManager.apply(RandomizedTestingPlugin)
-
-        BuildPlugin.globalBuildInfo(project)
-        BuildPlugin.configureRepositories(project)
-
-        // only setup tests to build
-        project.sourceSets.create('test')
-        project.dependencies.add('testCompile', "org.elasticsearch.test:framework:${VersionProperties.elasticsearch}")
-
-        project.eclipse.classpath.sourceSets = [project.sourceSets.test]
-        project.eclipse.classpath.plusConfigurations = [project.configurations.testRuntime]
-        project.idea.module.testSourceDirs += project.sourceSets.test.java.srcDirs
-        project.idea.module.scopes['TEST'] = [plus: [project.configurations.testRuntime]]
-
-        PrecommitTasks.create(project, false)
-        project.check.dependsOn(project.precommit)
-    }
-}
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneTestPlugin.groovy
index fefd08fe4e525..de52d75c6008c 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneTestPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneTestPlugin.groovy
@@ -25,12 +25,15 @@ import org.gradle.api.Plugin
 import org.gradle.api.Project
 import org.gradle.api.plugins.JavaBasePlugin
 
-/** A plugin to add tests only. Used for QA tests that run arbitrary unit tests. */
+/**
+ * Configures the build to compile against Elasticsearch's test framework and
+ * run integration and unit tests. Use BuildPlugin if you want to build main
+ * code as well as tests. */
 public class StandaloneTestPlugin implements Plugin<Project> {
 
     @Override
     public void apply(Project project) {
-        project.pluginManager.apply(StandaloneTestBasePlugin)
+        project.pluginManager.apply(StandaloneRestTestPlugin)
 
         Map testOptions = [
             name: 'test',
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/VagrantFixture.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/VagrantFixture.groovy
new file mode 100644
index 0000000000000..fa08a8f9c6667
--- /dev/null
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/VagrantFixture.groovy
@@ -0,0 +1,54 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle.test
+
+import org.elasticsearch.gradle.vagrant.VagrantCommandTask
+import org.gradle.api.Task
+
+/**
+ * A fixture for integration tests which runs in a virtual machine launched by Vagrant.
+ */
+class VagrantFixture extends VagrantCommandTask implements Fixture {
+
+    private VagrantCommandTask stopTask
+
+    public VagrantFixture() {
+        this.stopTask = project.tasks.create(name: "${name}#stop", type: VagrantCommandTask) {
+            command 'halt'
+        }
+        finalizedBy this.stopTask
+    }
+
+    @Override
+    void setBoxName(String boxName) {
+        super.setBoxName(boxName)
+        this.stopTask.setBoxName(boxName)
+    }
+
+    @Override
+    void setEnvironmentVars(Map<String, String> environmentVars) {
+        super.setEnvironmentVars(environmentVars)
+        this.stopTask.setEnvironmentVars(environmentVars)
+    }
+
+    @Override
+    public Task getStopTask() {
+        return this.stopTask
+    }
+}
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/BatsOverVagrantTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/BatsOverVagrantTask.groovy
index c68e0528c9b67..110f2fc7e8461 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/BatsOverVagrantTask.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/BatsOverVagrantTask.groovy
@@ -18,14 +18,7 @@
  */
 package org.elasticsearch.gradle.vagrant
 
-import org.gradle.api.DefaultTask
 import org.gradle.api.tasks.Input
-import org.gradle.api.tasks.TaskAction
-import org.gradle.logging.ProgressLoggerFactory
-import org.gradle.process.internal.ExecAction
-import org.gradle.process.internal.ExecActionFactory
-
-import javax.inject.Inject
 
 /**
  * Runs bats over vagrant. Pretty much like running it using Exec but with a
@@ -34,12 +27,15 @@ import javax.inject.Inject
 public class BatsOverVagrantTask extends VagrantCommandTask {
 
     @Input
-    String command
+    String remoteCommand
 
     BatsOverVagrantTask() {
-        project.afterEvaluate {
-            args 'ssh', boxName, '--command', command
-        }
+        command = 'ssh'
+    }
+
+    void setRemoteCommand(String remoteCommand) {
+        this.remoteCommand = Objects.requireNonNull(remoteCommand)
+        setArgs(['--command', remoteCommand])
     }
 
     @Override
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/TapLoggerOutputStream.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/TapLoggerOutputStream.groovy
index 3f980c57a49a6..e15759a1fe588 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/TapLoggerOutputStream.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/TapLoggerOutputStream.groovy
@@ -19,11 +19,9 @@
 package org.elasticsearch.gradle.vagrant
 
 import com.carrotsearch.gradle.junit4.LoggingOutputStream
-import groovy.transform.PackageScope
 import org.gradle.api.GradleScriptException
 import org.gradle.api.logging.Logger
-import org.gradle.logging.ProgressLogger
-import org.gradle.logging.ProgressLoggerFactory
+import org.gradle.internal.logging.progress.ProgressLogger
 
 import java.util.regex.Matcher
 
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantCommandTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantCommandTask.groovy
index d79c2533fabaf..aab120e8d049a 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantCommandTask.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantCommandTask.groovy
@@ -21,9 +21,15 @@ package org.elasticsearch.gradle.vagrant
 import org.apache.commons.io.output.TeeOutputStream
 import org.elasticsearch.gradle.LoggedExec
 import org.gradle.api.tasks.Input
-import org.gradle.logging.ProgressLoggerFactory
+import org.gradle.api.tasks.Optional
+import org.gradle.api.tasks.TaskAction
+import org.gradle.internal.logging.progress.ProgressLoggerFactory
 
 import javax.inject.Inject
+import java.util.concurrent.CountDownLatch
+import java.util.concurrent.locks.Lock
+import java.util.concurrent.locks.ReadWriteLock
+import java.util.concurrent.locks.ReentrantLock
 
 /**
  * Runs a vagrant command. Pretty much like Exec task but with a nicer output
@@ -31,17 +37,51 @@ import javax.inject.Inject
  */
 public class VagrantCommandTask extends LoggedExec {
 
+    @Input
+    String command
+
+    @Input @Optional
+    String subcommand
+
     @Input
     String boxName
 
+    @Input
+    Map<String, String> environmentVars
+
     public VagrantCommandTask() {
         executable = 'vagrant'
+
+        // We're using afterEvaluate here to slot in some logic that captures configurations and
+        // modifies the command line right before the main execution happens. The reason that we
+        // call doFirst instead of just doing the work in the afterEvaluate is that the latter
+        // restricts how subclasses can extend functionality. Calling afterEvaluate is like having
+        // all the logic of a task happening at construction time, instead of at execution time
+        // where a subclass can override or extend the logic.
         project.afterEvaluate {
-            // It'd be nice if --machine-readable were, well, nice
-            standardOutput = new TeeOutputStream(standardOutput, createLoggerOutputStream())
+            doFirst {
+                if (environmentVars != null) {
+                    environment environmentVars
+                }
+
+                // Build our command line for vagrant
+                def vagrantCommand = [executable, command]
+                if (subcommand != null) {
+                    vagrantCommand = vagrantCommand + subcommand
+                }
+                commandLine([*vagrantCommand, boxName, *args])
+
+                // It'd be nice if --machine-readable were, well, nice
+                standardOutput = new TeeOutputStream(standardOutput, createLoggerOutputStream())
+            }
         }
     }
 
+    @Inject
+    ProgressLoggerFactory getProgressLoggerFactory() {
+        throw new UnsupportedOperationException()
+    }
+
     protected OutputStream createLoggerOutputStream() {
         return new VagrantLoggerOutputStream(
             command: commandLine.join(' '),
@@ -50,9 +90,4 @@ public class VagrantCommandTask extends LoggedExec {
               stuff starts with ==> $box */
             squashedPrefix: "==> $boxName: ")
     }
-
-    @Inject
-    ProgressLoggerFactory getProgressLoggerFactory() {
-        throw new UnsupportedOperationException();
-    }
 }
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantLoggerOutputStream.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantLoggerOutputStream.groovy
index 331a638b5cade..e899c0171298b 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantLoggerOutputStream.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantLoggerOutputStream.groovy
@@ -19,9 +19,7 @@
 package org.elasticsearch.gradle.vagrant
 
 import com.carrotsearch.gradle.junit4.LoggingOutputStream
-import org.gradle.api.logging.Logger
-import org.gradle.logging.ProgressLogger
-import org.gradle.logging.ProgressLoggerFactory
+import org.gradle.internal.logging.progress.ProgressLogger
 
 /**
  * Adapts an OutputStream being written to by vagrant into a ProcessLogger. It
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantPropertiesExtension.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantPropertiesExtension.groovy
new file mode 100644
index 0000000000000..f16913d5be64a
--- /dev/null
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantPropertiesExtension.groovy
@@ -0,0 +1,76 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle.vagrant
+
+import org.gradle.api.tasks.Input
+
+class VagrantPropertiesExtension {
+
+    @Input
+    List<String> boxes
+
+    @Input
+    Long testSeed
+
+    @Input
+    String formattedTestSeed
+
+    @Input
+    String upgradeFromVersion
+
+    @Input
+    List<String> upgradeFromVersions
+
+    @Input
+    String batsDir
+
+    @Input
+    Boolean inheritTests
+
+    @Input
+    Boolean inheritTestArchives
+
+    @Input
+    Boolean inheritTestUtils
+
+    VagrantPropertiesExtension(List<String> availableBoxes) {
+        this.boxes = availableBoxes
+        this.batsDir = 'src/test/resources/packaging'
+    }
+
+    void boxes(String... boxes) {
+        this.boxes = Arrays.asList(boxes)
+    }
+
+    void setBatsDir(String batsDir) {
+        this.batsDir = batsDir
+    }
+
+    void setInheritTests(Boolean inheritTests) {
+        this.inheritTests = inheritTests
+    }
+
+    void setInheritTestArchives(Boolean inheritTestArchives) {
+        this.inheritTestArchives = inheritTestArchives
+    }
+
+    void setInheritTestUtils(Boolean inheritTestUtils) {
+        this.inheritTestUtils = inheritTestUtils
+    }
+}
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantSupportPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantSupportPlugin.groovy
new file mode 100644
index 0000000000000..9dfe487e83018
--- /dev/null
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantSupportPlugin.groovy
@@ -0,0 +1,127 @@
+package org.elasticsearch.gradle.vagrant
+
+import org.gradle.api.GradleException
+import org.gradle.api.InvalidUserDataException
+import org.gradle.api.Plugin
+import org.gradle.api.Project
+import org.gradle.process.ExecResult
+import org.gradle.process.internal.ExecException
+
+/**
+ * Global configuration for if Vagrant tasks are supported in this
+ * build environment.
+ */
+class VagrantSupportPlugin implements Plugin<Project> {
+
+    @Override
+    void apply(Project project) {
+        if (project.rootProject.ext.has('vagrantEnvChecksDone') == false) {
+            Map vagrantInstallation = getVagrantInstallation(project)
+            Map virtualBoxInstallation = getVirtualBoxInstallation(project)
+
+            project.rootProject.ext.vagrantInstallation = vagrantInstallation
+            project.rootProject.ext.virtualBoxInstallation = virtualBoxInstallation
+            project.rootProject.ext.vagrantSupported = vagrantInstallation.supported && virtualBoxInstallation.supported
+            project.rootProject.ext.vagrantEnvChecksDone = true
+
+            // Finding that HOME needs to be set when performing vagrant updates
+            String homeLocation = System.getenv("HOME")
+            if (project.rootProject.ext.vagrantSupported && homeLocation == null) {
+                throw new GradleException("Could not locate \$HOME environment variable. Vagrant is enabled " +
+                        "and requires \$HOME to be set to function properly.")
+            }
+        }
+
+        addVerifyInstallationTasks(project)
+    }
+
+    private Map getVagrantInstallation(Project project) {
+        try {
+            ByteArrayOutputStream pipe = new ByteArrayOutputStream()
+            ExecResult runResult = project.exec {
+                commandLine 'vagrant', '--version'
+                standardOutput pipe
+                ignoreExitValue true
+            }
+            String version = pipe.toString().trim()
+            if (runResult.exitValue == 0) {
+                if (version ==~ /Vagrant 1\.(8\.[6-9]|9\.[0-9])+/ || version ==~ /Vagrant 2\.[0-9]+\.[0-9]+/) {
+                    return [ 'supported' : true ]
+                } else {
+                    return [ 'supported' : false,
+                             'info' : "Illegal version of vagrant [${version}]. Need [Vagrant 1.8.6+]" ]
+                }
+            } else {
+                return [ 'supported' : false,
+                         'info' : "Could not read installed vagrant version:\n" + version ]
+            }
+        } catch (ExecException e) {
+            // Exec still throws this if it cannot find the command, regardless if ignoreExitValue is set.
+            // Swallow error. Vagrant isn't installed. Don't halt the build here.
+            return [ 'supported' : false, 'info' : "Could not find vagrant: " + e.message ]
+        }
+    }
+
+    private Map getVirtualBoxInstallation(Project project) {
+        try {
+            ByteArrayOutputStream pipe = new ByteArrayOutputStream()
+            ExecResult runResult = project.exec {
+                commandLine 'vboxmanage', '--version'
+                standardOutput = pipe
+                ignoreExitValue true
+            }
+            String version = pipe.toString().trim()
+            if (runResult.exitValue == 0) {
+                try {
+                    String[] versions = version.split('\\.')
+                    int major = Integer.parseInt(versions[0])
+                    int minor = Integer.parseInt(versions[1])
+                    if ((major < 5) || (major == 5 && minor < 1)) {
+                        return [ 'supported' : false,
+                                 'info' : "Illegal version of virtualbox [${version}]. Need [5.1+]" ]
+                    } else {
+                        return [ 'supported' : true ]
+                    }
+                } catch (NumberFormatException | ArrayIndexOutOfBoundsException e) {
+                    return [ 'supported' : false,
+                             'info' : "Unable to parse version of virtualbox [${version}]. Required [5.1+]" ]
+                }
+            } else {
+                return [ 'supported': false, 'info': "Could not read installed virtualbox version:\n" + version ]
+            }
+        } catch (ExecException e) {
+            // Exec still throws this if it cannot find the command, regardless if ignoreExitValue is set.
+            // Swallow error. VirtualBox isn't installed. Don't halt the build here.
+            return [ 'supported' : false, 'info' : "Could not find virtualbox: " + e.message ]
+        }
+    }
+
+    private void addVerifyInstallationTasks(Project project) {
+        createCheckVagrantVersionTask(project)
+        createCheckVirtualBoxVersionTask(project)
+    }
+
+    private void createCheckVagrantVersionTask(Project project) {
+        project.tasks.create('vagrantCheckVersion') {
+            description 'Check the Vagrant version'
+            group 'Verification'
+            doLast {
+                if (project.rootProject.vagrantInstallation.supported == false) {
+                    throw new InvalidUserDataException(project.rootProject.vagrantInstallation.info)
+                }
+            }
+        }
+    }
+
+    private void createCheckVirtualBoxVersionTask(Project project) {
+        project.tasks.create('virtualboxCheckVersion') {
+            description 'Check the Virtualbox version'
+            group 'Verification'
+            doLast {
+                if (project.rootProject.virtualBoxInstallation.supported == false) {
+                    throw new InvalidUserDataException(project.rootProject.virtualBoxInstallation.info)
+                }
+            }
+        }
+    }
+}
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy
new file mode 100644
index 0000000000000..d0c529b263285
--- /dev/null
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy
@@ -0,0 +1,403 @@
+package org.elasticsearch.gradle.vagrant
+
+import org.elasticsearch.gradle.FileContentsTask
+import org.gradle.api.*
+import org.gradle.api.artifacts.dsl.RepositoryHandler
+import org.gradle.api.execution.TaskExecutionAdapter
+import org.gradle.api.internal.artifacts.dependencies.DefaultProjectDependency
+import org.gradle.api.tasks.Copy
+import org.gradle.api.tasks.Delete
+import org.gradle.api.tasks.Exec
+import org.gradle.api.tasks.TaskState
+
+class VagrantTestPlugin implements Plugin<Project> {
+
+    /** All available boxes **/
+    static List<String> BOXES = [
+            'centos-6',
+            'centos-7',
+            'debian-8',
+            'debian-9',
+            'fedora-25',
+            'fedora-26',
+            'oel-6',
+            'oel-7',
+            'opensuse-42',
+            'sles-12',
+            'ubuntu-1404',
+            'ubuntu-1604'
+    ]
+
+    /** Boxes used when sampling the tests **/
+    static List<String> SAMPLE = [
+            'centos-7',
+            'ubuntu-1404',
+    ]
+
+    /** All onboarded archives by default, available for Bats tests even if not used **/
+    static List<String> DISTRIBUTION_ARCHIVES = ['tar', 'rpm', 'deb']
+
+    /** Packages onboarded for upgrade tests **/
+    static List<String> UPGRADE_FROM_ARCHIVES = ['rpm', 'deb']
+
+    private static final BATS = 'bats'
+    private static final String BATS_TEST_COMMAND ="cd \$BATS_ARCHIVES && sudo bats --tap \$BATS_TESTS/*.$BATS"
+    private static final String PLATFORM_TEST_COMMAND ="rm -rf ~/elasticsearch && rsync -r /elasticsearch/ ~/elasticsearch && cd ~/elasticsearch && \$GRADLE_HOME/bin/gradle test integTest"
+
+    @Override
+    void apply(Project project) {
+
+        // Creates the Vagrant extension for the project
+        project.extensions.create('esvagrant', VagrantPropertiesExtension, listVagrantBoxes(project))
+
+        // Add required repositories for Bats tests
+        configureBatsRepositories(project)
+
+        // Creates custom configurations for Bats testing files (and associated scripts and archives)
+        createBatsConfiguration(project)
+
+        // Creates all the main Vagrant tasks
+        createVagrantTasks(project)
+
+        if (project.extensions.esvagrant.boxes == null || project.extensions.esvagrant.boxes.size() == 0) {
+            throw new InvalidUserDataException('Vagrant boxes cannot be null or empty for esvagrant')
+        }
+
+        for (String box : project.extensions.esvagrant.boxes) {
+            if (BOXES.contains(box) == false) {
+                throw new InvalidUserDataException("Vagrant box [${box}] not found, available virtual machines are ${BOXES}")
+            }
+        }
+
+        // Creates all tasks related to the Vagrant boxes
+        createVagrantBoxesTasks(project)
+    }
+
+    private List<String> listVagrantBoxes(Project project) {
+        String vagrantBoxes = project.getProperties().get('vagrant.boxes', 'sample')
+        if (vagrantBoxes == 'sample') {
+            return SAMPLE
+        } else if (vagrantBoxes == 'all') {
+            return BOXES
+        } else {
+            return vagrantBoxes.split(',')
+        }
+    }
+
+    private static void configureBatsRepositories(Project project) {
+        RepositoryHandler repos = project.repositories
+
+        // Try maven central first, it'll have releases before 5.0.0
+        repos.mavenCentral()
+
+        /* Setup a repository that tries to download from
+          https://artifacts.elastic.co/downloads/elasticsearch/[module]-[revision].[ext]
+          which should work for 5.0.0+. This isn't a real ivy repository but gradle
+          is fine with that */
+        repos.ivy {
+            artifactPattern "https://artifacts.elastic.co/downloads/elasticsearch/[module]-[revision].[ext]"
+        }
+    }
+
+    private static void createBatsConfiguration(Project project) {
+        project.configurations.create(BATS)
+
+        final long seed
+        final String formattedSeed
+        String maybeTestsSeed = System.getProperty("tests.seed")
+        if (maybeTestsSeed != null) {
+            if (maybeTestsSeed.trim().isEmpty()) {
+                throw new GradleException("explicit tests.seed cannot be empty")
+            }
+            String masterSeed = maybeTestsSeed.tokenize(':').get(0)
+            seed = new BigInteger(masterSeed, 16).longValue()
+            formattedSeed = maybeTestsSeed
+        } else {
+            seed = new Random().nextLong()
+            formattedSeed = String.format("%016X", seed)
+        }
+
+        String upgradeFromVersion = System.getProperty("tests.packaging.upgradeVersion");
+        if (upgradeFromVersion == null) {
+            upgradeFromVersion = project.indexCompatVersions[new Random(seed).nextInt(project.indexCompatVersions.size())]
+        }
+
+        DISTRIBUTION_ARCHIVES.each {
+            // Adds a dependency for the current version
+            project.dependencies.add(BATS, project.dependencies.project(path: ":distribution:${it}", configuration: 'archives'))
+        }
+
+        UPGRADE_FROM_ARCHIVES.each {
+            // The version of elasticsearch that we upgrade *from*
+            project.dependencies.add(BATS, "org.elasticsearch.distribution.${it}:elasticsearch:${upgradeFromVersion}@${it}")
+        }
+
+        project.extensions.esvagrant.testSeed = seed
+        project.extensions.esvagrant.formattedTestSeed = formattedSeed
+        project.extensions.esvagrant.upgradeFromVersion = upgradeFromVersion
+    }
+
+    private static void createCleanTask(Project project) {
+        project.tasks.create('clean', Delete.class) {
+            description 'Clean the project build directory'
+            group 'Build'
+            delete project.buildDir
+        }
+    }
+
+    private static void createStopTask(Project project) {
+        project.tasks.create('stop') {
+            description 'Stop any tasks from tests that still may be running'
+            group 'Verification'
+        }
+    }
+
+    private static void createSmokeTestTask(Project project) {
+        project.tasks.create('vagrantSmokeTest') {
+            description 'Smoke test the specified vagrant boxes'
+            group 'Verification'
+        }
+    }
+
+    private static void createPrepareVagrantTestEnvTask(Project project) {
+        File batsDir = new File("${project.buildDir}/${BATS}")
+
+        Task createBatsDirsTask = project.tasks.create('createBatsDirs')
+        createBatsDirsTask.outputs.dir batsDir
+        createBatsDirsTask.doLast {
+            batsDir.mkdirs()
+        }
+
+        Copy copyBatsArchives = project.tasks.create('copyBatsArchives', Copy) {
+            dependsOn createBatsDirsTask
+            into "${batsDir}/archives"
+            from project.configurations[BATS]
+        }
+
+        Copy copyBatsTests = project.tasks.create('copyBatsTests', Copy) {
+            dependsOn createBatsDirsTask
+            into "${batsDir}/tests"
+            from {
+                "${project.extensions.esvagrant.batsDir}/tests"
+            }
+        }
+
+        Copy copyBatsUtils = project.tasks.create('copyBatsUtils', Copy) {
+            dependsOn createBatsDirsTask
+            into "${batsDir}/utils"
+            from {
+                "${project.extensions.esvagrant.batsDir}/utils"
+            }
+        }
+
+        // Now we iterate over dependencies of the bats configuration. When a project dependency is found,
+        // we bring back its own archives, test files or test utils.
+        project.afterEvaluate {
+            project.configurations.bats.dependencies.findAll {it.targetConfiguration == BATS }.each { d ->
+                if (d instanceof DefaultProjectDependency) {
+                    DefaultProjectDependency externalBatsDependency = (DefaultProjectDependency) d
+                    Project externalBatsProject = externalBatsDependency.dependencyProject
+                    String externalBatsDir = externalBatsProject.extensions.esvagrant.batsDir
+
+                    if (project.extensions.esvagrant.inheritTests) {
+                        copyBatsTests.from(externalBatsProject.files("${externalBatsDir}/tests"))
+                    }
+                    if (project.extensions.esvagrant.inheritTestArchives) {
+                        copyBatsArchives.from(externalBatsDependency.projectConfiguration.files)
+                    }
+                    if (project.extensions.esvagrant.inheritTestUtils) {
+                        copyBatsUtils.from(externalBatsProject.files("${externalBatsDir}/utils"))
+                    }
+                }
+            }
+        }
+
+        Task createVersionFile = project.tasks.create('createVersionFile', FileContentsTask) {
+            dependsOn createBatsDirsTask
+            file "${batsDir}/archives/version"
+            contents project.version
+        }
+
+        Task createUpgradeFromFile = project.tasks.create('createUpgradeFromFile', FileContentsTask) {
+            dependsOn createBatsDirsTask
+            file "${batsDir}/archives/upgrade_from_version"
+            contents project.extensions.esvagrant.upgradeFromVersion
+        }
+
+        Task vagrantSetUpTask = project.tasks.create('setupBats')
+        vagrantSetUpTask.dependsOn 'vagrantCheckVersion'
+        vagrantSetUpTask.dependsOn copyBatsTests, copyBatsUtils, copyBatsArchives, createVersionFile, createUpgradeFromFile
+    }
+
+    private static void createPackagingTestTask(Project project) {
+        project.tasks.create('packagingTest') {
+            group 'Verification'
+            description "Tests yum/apt packages using vagrant and bats.\n" +
+                    "    Specify the vagrant boxes to test using the gradle property 'vagrant.boxes'.\n" +
+                    "    'sample' can be used to test a single yum and apt box. 'all' can be used to\n" +
+                    "    test all available boxes. The available boxes are: \n" +
+                    "    ${BOXES}"
+            dependsOn 'vagrantCheckVersion'
+        }
+    }
+
+    private static void createPlatformTestTask(Project project) {
+        project.tasks.create('platformTest') {
+            group 'Verification'
+            description "Test unit and integ tests on different platforms using vagrant.\n" +
+                    "    Specify the vagrant boxes to test using the gradle property 'vagrant.boxes'.\n" +
+                    "    'all' can be used to test all available boxes. The available boxes are: \n" +
+                    "    ${BOXES}"
+            dependsOn 'vagrantCheckVersion'
+        }
+    }
+
+    private static void createVagrantTasks(Project project) {
+        createCleanTask(project)
+        createStopTask(project)
+        createSmokeTestTask(project)
+        createPrepareVagrantTestEnvTask(project)
+        createPackagingTestTask(project)
+        createPlatformTestTask(project)
+    }
+
+    private static void createVagrantBoxesTasks(Project project) {
+        assert project.extensions.esvagrant.boxes != null
+
+        assert project.tasks.stop != null
+        Task stop = project.tasks.stop
+
+        assert project.tasks.vagrantSmokeTest != null
+        Task vagrantSmokeTest = project.tasks.vagrantSmokeTest
+
+        assert project.tasks.vagrantCheckVersion != null
+        Task vagrantCheckVersion = project.tasks.vagrantCheckVersion
+
+        assert project.tasks.virtualboxCheckVersion != null
+        Task virtualboxCheckVersion = project.tasks.virtualboxCheckVersion
+
+        assert project.tasks.setupBats != null
+        Task setupBats = project.tasks.setupBats
+
+        assert project.tasks.packagingTest != null
+        Task packagingTest = project.tasks.packagingTest
+
+        assert project.tasks.platformTest != null
+        Task platformTest = project.tasks.platformTest
+
+        /*
+         * We always use the main project.rootDir as Vagrant's current working directory (VAGRANT_CWD)
+         * so that boxes are not duplicated for every Gradle project that use this VagrantTestPlugin.
+         */
+        def vagrantEnvVars = [
+                'VAGRANT_CWD'           : "${project.rootDir.absolutePath}",
+                'VAGRANT_VAGRANTFILE'   : 'Vagrantfile',
+                'VAGRANT_PROJECT_DIR'   : "${project.projectDir.absolutePath}"
+        ]
+
+        // Each box gets it own set of tasks
+        for (String box : BOXES) {
+            String boxTask = box.capitalize().replace('-', '')
+
+            // always add a halt task for all boxes, so clean makes sure they are all shutdown
+            Task halt = project.tasks.create("vagrant${boxTask}#halt", VagrantCommandTask) {
+                command 'halt'
+                boxName box
+                environmentVars vagrantEnvVars
+            }
+            stop.dependsOn(halt)
+
+            Task update = project.tasks.create("vagrant${boxTask}#update", VagrantCommandTask) {
+                command 'box'
+                subcommand 'update'
+                boxName box
+                environmentVars vagrantEnvVars
+                dependsOn vagrantCheckVersion, virtualboxCheckVersion
+            }
+            update.mustRunAfter(setupBats)
+
+            Task up = project.tasks.create("vagrant${boxTask}#up", VagrantCommandTask) {
+                command 'up'
+                boxName box
+                environmentVars vagrantEnvVars
+                /* Its important that we try to reprovision the box even if it already
+                  exists. That way updates to the vagrant configuration take automatically.
+                  That isn't to say that the updates will always be compatible. Its ok to
+                  just destroy the boxes if they get busted but that is a manual step
+                  because its slow-ish. */
+                /* We lock the provider to virtualbox because the Vagrantfile specifies
+                  lots of boxes that only work properly in virtualbox. Virtualbox is
+                  vagrant's default but its possible to change that default and folks do.
+                  But the boxes that we use are unlikely to work properly with other
+                  virtualization providers. Thus the lock. */
+                args '--provision', '--provider', 'virtualbox'
+                /* It'd be possible to check if the box is already up here and output
+                  SKIPPED but that would require running vagrant status which is slow! */
+                dependsOn update
+            }
+
+            Task smoke = project.tasks.create("vagrant${boxTask}#smoketest", Exec) {
+                environment vagrantEnvVars
+                dependsOn up
+                finalizedBy halt
+                commandLine 'vagrant', 'ssh', box, '--command',
+                        "set -o pipefail && echo 'Hello from ${project.path}' | sed -ue 's/^/    ${box}: /'"
+            }
+            vagrantSmokeTest.dependsOn(smoke)
+
+            Task packaging = project.tasks.create("vagrant${boxTask}#packagingTest", BatsOverVagrantTask) {
+                remoteCommand BATS_TEST_COMMAND
+                boxName box
+                environmentVars vagrantEnvVars
+                dependsOn up, setupBats
+                finalizedBy halt
+            }
+
+            TaskExecutionAdapter packagingReproListener = new TaskExecutionAdapter() {
+                @Override
+                void afterExecute(Task task, TaskState state) {
+                    if (state.failure != null) {
+                        println "REPRODUCE WITH: gradle ${packaging.path} " +
+                            "-Dtests.seed=${project.extensions.esvagrant.formattedTestSeed} "
+                    }
+                }
+            }
+            packaging.doFirst {
+                project.gradle.addListener(packagingReproListener)
+            }
+            packaging.doLast {
+                project.gradle.removeListener(packagingReproListener)
+            }
+            if (project.extensions.esvagrant.boxes.contains(box)) {
+                packagingTest.dependsOn(packaging)
+            }
+
+            Task platform = project.tasks.create("vagrant${boxTask}#platformTest", VagrantCommandTask) {
+                command 'ssh'
+                boxName box
+                environmentVars vagrantEnvVars
+                dependsOn up
+                finalizedBy halt
+                args '--command', PLATFORM_TEST_COMMAND + " -Dtests.seed=${-> project.extensions.esvagrant.formattedTestSeed}"
+            }
+            TaskExecutionAdapter platformReproListener = new TaskExecutionAdapter() {
+                @Override
+                void afterExecute(Task task, TaskState state) {
+                    if (state.failure != null) {
+                        println "REPRODUCE WITH: gradle ${platform.path} " +
+                            "-Dtests.seed=${project.extensions.esvagrant.formattedTestSeed} "
+                    }
+                }
+            }
+            platform.doFirst {
+                project.gradle.addListener(platformReproListener)
+            }
+            platform.doLast {
+                project.gradle.removeListener(platformReproListener)
+            }
+            if (project.extensions.esvagrant.boxes.contains(box)) {
+                platformTest.dependsOn(platform)
+            }
+        }
+    }
+}
diff --git a/buildSrc/src/main/java/org/elasticsearch/test/NamingConventionsCheck.java b/buildSrc/src/main/java/org/elasticsearch/test/NamingConventionsCheck.java
index cbfa31d1aaf5b..9bd14675d34a4 100644
--- a/buildSrc/src/main/java/org/elasticsearch/test/NamingConventionsCheck.java
+++ b/buildSrc/src/main/java/org/elasticsearch/test/NamingConventionsCheck.java
@@ -28,6 +28,7 @@
 import java.nio.file.Paths;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.util.HashSet;
+import java.util.Objects;
 import java.util.Set;
 
 /**
@@ -49,6 +50,7 @@ public static void main(String[] args) throws IOException {
         Path rootPath = null;
         boolean skipIntegTestsInDisguise = false;
         boolean selfTest = false;
+        boolean checkMainClasses = false;
         for (int i = 0; i < args.length; i++) {
             String arg = args[i];
             switch (arg) {
@@ -64,6 +66,9 @@ public static void main(String[] args) throws IOException {
                 case "--self-test":
                     selfTest = true;
                     break;
+                case "--main":
+                    checkMainClasses = true;
+                    break;
                 case "--":
                     rootPath = Paths.get(args[++i]);
                     break;
@@ -73,28 +78,43 @@ public static void main(String[] args) throws IOException {
         }
 
         NamingConventionsCheck check = new NamingConventionsCheck(testClass, integTestClass);
-        check.check(rootPath, skipIntegTestsInDisguise);
+        if (checkMainClasses) {
+            check.checkMain(rootPath);
+        } else {
+            check.checkTests(rootPath, skipIntegTestsInDisguise);
+        }
 
         if (selfTest) {
-            assertViolation("WrongName", check.missingSuffix);
-            assertViolation("WrongNameTheSecond", check.missingSuffix);
-            assertViolation("DummyAbstractTests", check.notRunnable);
-            assertViolation("DummyInterfaceTests", check.notRunnable);
-            assertViolation("InnerTests", check.innerClasses);
-            assertViolation("NotImplementingTests", check.notImplementing);
-            assertViolation("PlainUnit", check.pureUnitTest);
+            if (checkMainClasses) {
+                assertViolation(NamingConventionsCheckInMainTests.class.getName(), check.testsInMain);
+                assertViolation(NamingConventionsCheckInMainIT.class.getName(), check.testsInMain);
+            } else {
+                assertViolation("WrongName", check.missingSuffix);
+                assertViolation("WrongNameTheSecond", check.missingSuffix);
+                assertViolation("DummyAbstractTests", check.notRunnable);
+                assertViolation("DummyInterfaceTests", check.notRunnable);
+                assertViolation("InnerTests", check.innerClasses);
+                assertViolation("NotImplementingTests", check.notImplementing);
+                assertViolation("PlainUnit", check.pureUnitTest);
+            }
         }
 
         // Now we should have no violations
-        assertNoViolations("Not all subclasses of " + check.testClass.getSimpleName()
-                + " match the naming convention. Concrete classes must end with [Tests]", check.missingSuffix);
+        assertNoViolations(
+                "Not all subclasses of " + check.testClass.getSimpleName()
+                    + " match the naming convention. Concrete classes must end with [Tests]",
+                check.missingSuffix);
         assertNoViolations("Classes ending with [Tests] are abstract or interfaces", check.notRunnable);
         assertNoViolations("Found inner classes that are tests, which are excluded from the test runner", check.innerClasses);
         assertNoViolations("Pure Unit-Test found must subclass [" + check.testClass.getSimpleName() + "]", check.pureUnitTest);
         assertNoViolations("Classes ending with [Tests] must subclass [" + check.testClass.getSimpleName() + "]", check.notImplementing);
+        assertNoViolations(
+                "Classes ending with [Tests] or [IT] or extending [" + check.testClass.getSimpleName() + "] must be in src/test/java",
+                check.testsInMain);
         if (skipIntegTestsInDisguise == false) {
-            assertNoViolations("Subclasses of " + check.integTestClass.getSimpleName() +
-                    " should end with IT as they are integration tests", check.integTestsInDisguise);
+            assertNoViolations(
+                    "Subclasses of " + check.integTestClass.getSimpleName() + " should end with IT as they are integration tests",
+                    check.integTestsInDisguise);
         }
     }
 
@@ -104,84 +124,76 @@ public static void main(String[] args) throws IOException {
     private final Set<Class<?>> integTestsInDisguise = new HashSet<>();
     private final Set<Class<?>> notRunnable = new HashSet<>();
     private final Set<Class<?>> innerClasses = new HashSet<>();
+    private final Set<Class<?>> testsInMain = new HashSet<>();
 
     private final Class<?> testClass;
     private final Class<?> integTestClass;
 
     public NamingConventionsCheck(Class<?> testClass, Class<?> integTestClass) {
-        this.testClass = testClass;
+        this.testClass = Objects.requireNonNull(testClass, "--test-class is required");
         this.integTestClass = integTestClass;
     }
 
-    public void check(Path rootPath, boolean skipTestsInDisguised) throws IOException {
-        Files.walkFileTree(rootPath, new FileVisitor<Path>() {
-            /**
-             * The package name of the directory we are currently visiting. Kept as a string rather than something fancy because we load
-             * just about every class and doing so requires building a string out of it anyway. At least this way we don't need to build the
-             * first part of the string over and over and over again.
-             */
-            private String packageName;
-
+    public void checkTests(Path rootPath, boolean skipTestsInDisguised) throws IOException {
+        Files.walkFileTree(rootPath, new TestClassVisitor() {
             @Override
-            public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
-                // First we visit the root directory
-                if (packageName == null) {
-                    // And it package is empty string regardless of the directory name
-                    packageName = "";
-                } else {
-                    packageName += dir.getFileName() + ".";
+            protected void visitTestClass(Class<?> clazz) {
+                if (skipTestsInDisguised == false && integTestClass.isAssignableFrom(clazz)) {
+                    integTestsInDisguise.add(clazz);
+                }
+                if (Modifier.isAbstract(clazz.getModifiers()) || Modifier.isInterface(clazz.getModifiers())) {
+                    notRunnable.add(clazz);
+                } else if (isTestCase(clazz) == false) {
+                    notImplementing.add(clazz);
+                } else if (Modifier.isStatic(clazz.getModifiers())) {
+                    innerClasses.add(clazz);
                 }
-                return FileVisitResult.CONTINUE;
             }
 
             @Override
-            public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
-                // Go up one package by jumping back to the second to last '.'
-                packageName = packageName.substring(0, 1 + packageName.lastIndexOf('.', packageName.length() - 2));
-                return FileVisitResult.CONTINUE;
+            protected void visitIntegrationTestClass(Class<?> clazz) {
+                if (isTestCase(clazz) == false) {
+                    notImplementing.add(clazz);
+                }
             }
 
             @Override
-            public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
-                String filename = file.getFileName().toString();
-                if (filename.endsWith(".class")) {
-                    String className = filename.substring(0, filename.length() - ".class".length());
-                    Class<?> clazz = loadClassWithoutInitializing(packageName + className);
-                    if (clazz.getName().endsWith("Tests")) {
-                        if (skipTestsInDisguised == false && integTestClass.isAssignableFrom(clazz)) {
-                            integTestsInDisguise.add(clazz);
-                        }
-                        if (Modifier.isAbstract(clazz.getModifiers()) || Modifier.isInterface(clazz.getModifiers())) {
-                            notRunnable.add(clazz);
-                        } else if (isTestCase(clazz) == false) {
-                            notImplementing.add(clazz);
-                        } else if (Modifier.isStatic(clazz.getModifiers())) {
-                            innerClasses.add(clazz);
-                        }
-                    } else if (clazz.getName().endsWith("IT")) {
-                        if (isTestCase(clazz) == false) {
-                            notImplementing.add(clazz);
-                        }
-                    } else if (Modifier.isAbstract(clazz.getModifiers()) == false && Modifier.isInterface(clazz.getModifiers()) == false) {
-                        if (isTestCase(clazz)) {
-                            missingSuffix.add(clazz);
-                        } else if (junit.framework.Test.class.isAssignableFrom(clazz)) {
-                            pureUnitTest.add(clazz);
-                        }
-                    }
+            protected void visitOtherClass(Class<?> clazz) {
+                if (Modifier.isAbstract(clazz.getModifiers()) || Modifier.isInterface(clazz.getModifiers())) {
+                    return;
+                }
+                if (isTestCase(clazz)) {
+                    missingSuffix.add(clazz);
+                } else if (junit.framework.Test.class.isAssignableFrom(clazz)) {
+                    pureUnitTest.add(clazz);
                 }
-                return FileVisitResult.CONTINUE;
+            }
+        });
+    }
+
+    public void checkMain(Path rootPath) throws IOException {
+        Files.walkFileTree(rootPath, new TestClassVisitor() {
+            @Override
+            protected void visitTestClass(Class<?> clazz) {
+                testsInMain.add(clazz);
             }
 
-            private boolean isTestCase(Class<?> clazz) {
-                return testClass.isAssignableFrom(clazz);
+            @Override
+            protected void visitIntegrationTestClass(Class<?> clazz) {
+                testsInMain.add(clazz);
             }
 
             @Override
-            public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
-                throw exc;
+            protected void visitOtherClass(Class<?> clazz) {
+                if (Modifier.isAbstract(clazz.getModifiers()) || Modifier.isInterface(clazz.getModifiers())) {
+                    return;
+                }
+                if (isTestCase(clazz)) {
+                    testsInMain.add(clazz);
+                }
             }
         });
+
     }
 
     /**
@@ -203,7 +215,7 @@ private static void assertNoViolations(String message, Set<Class<?>> set) {
      * similar enough.
      */
     private static void assertViolation(String className, Set<Class<?>> set) {
-        className = "org.elasticsearch.test.NamingConventionsCheckBadClasses$" + className;
+        className = className.startsWith("org") ? className : "org.elasticsearch.test.NamingConventionsCheckBadClasses$" + className;
         if (false == set.remove(loadClassWithoutInitializing(className))) {
             System.err.println("Error in NamingConventionsCheck! Expected [" + className + "] to be a violation but wasn't.");
             System.exit(1);
@@ -229,4 +241,74 @@ static Class<?> loadClassWithoutInitializing(String name) {
             throw new RuntimeException(e);
         }
     }
+
+    abstract class TestClassVisitor implements FileVisitor<Path> {
+        /**
+         * The package name of the directory we are currently visiting. Kept as a string rather than something fancy because we load
+         * just about every class and doing so requires building a string out of it anyway. At least this way we don't need to build the
+         * first part of the string over and over and over again.
+         */
+        private String packageName;
+
+        /**
+         * Visit classes named like a test.
+         */
+        protected abstract void visitTestClass(Class<?> clazz);
+        /**
+         * Visit classes named like an integration test.
+         */
+        protected abstract void visitIntegrationTestClass(Class<?> clazz);
+        /**
+         * Visit classes not named like a test at all.
+         */
+        protected abstract void visitOtherClass(Class<?> clazz);
+
+        @Override
+        public final FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
+            // First we visit the root directory
+            if (packageName == null) {
+                // And it package is empty string regardless of the directory name
+                packageName = "";
+            } else {
+                packageName += dir.getFileName() + ".";
+            }
+            return FileVisitResult.CONTINUE;
+        }
+
+        @Override
+        public final FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
+            // Go up one package by jumping back to the second to last '.'
+            packageName = packageName.substring(0, 1 + packageName.lastIndexOf('.', packageName.length() - 2));
+            return FileVisitResult.CONTINUE;
+        }
+
+        @Override
+        public final FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+            String filename = file.getFileName().toString();
+            if (filename.endsWith(".class")) {
+                String className = filename.substring(0, filename.length() - ".class".length());
+                Class<?> clazz = loadClassWithoutInitializing(packageName + className);
+                if (clazz.getName().endsWith("Tests")) {
+                    visitTestClass(clazz);
+                } else if (clazz.getName().endsWith("IT")) {
+                    visitIntegrationTestClass(clazz);
+                } else {
+                    visitOtherClass(clazz);
+                }
+            }
+            return FileVisitResult.CONTINUE;
+        }
+
+        /**
+         * Is this class a test case?
+         */
+        protected boolean isTestCase(Class<?> clazz) {
+            return testClass.isAssignableFrom(clazz);
+        }
+
+        @Override
+        public final FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
+            throw exc;
+        }
+    }
 }
diff --git a/buildSrc/src/main/java/org/elasticsearch/test/NamingConventionsCheckInMainIT.java b/buildSrc/src/main/java/org/elasticsearch/test/NamingConventionsCheckInMainIT.java
new file mode 100644
index 0000000000000..46adc7f065b16
--- /dev/null
+++ b/buildSrc/src/main/java/org/elasticsearch/test/NamingConventionsCheckInMainIT.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test;
+
+/**
+ * This class should fail the naming conventions self test.
+ */
+public class NamingConventionsCheckInMainIT {
+}
diff --git a/buildSrc/src/main/java/org/elasticsearch/test/NamingConventionsCheckInMainTests.java b/buildSrc/src/main/java/org/elasticsearch/test/NamingConventionsCheckInMainTests.java
new file mode 100644
index 0000000000000..27c0b41eb3f6a
--- /dev/null
+++ b/buildSrc/src/main/java/org/elasticsearch/test/NamingConventionsCheckInMainTests.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test;
+
+/**
+ * This class should fail the naming conventions self test.
+ */
+public class NamingConventionsCheckInMainTests {
+}
diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.standalone-rest-test.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.standalone-rest-test.properties
new file mode 100644
index 0000000000000..2daf4dc27c0ec
--- /dev/null
+++ b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.standalone-rest-test.properties
@@ -0,0 +1,20 @@
+#
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+implementation-class=org.elasticsearch.gradle.test.StandaloneRestTestPlugin
diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.vagrant.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.vagrant.properties
new file mode 100644
index 0000000000000..844310fa9d7fa
--- /dev/null
+++ b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.vagrant.properties
@@ -0,0 +1 @@
+implementation-class=org.elasticsearch.gradle.vagrant.VagrantTestPlugin
diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.vagrantsupport.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.vagrantsupport.properties
new file mode 100644
index 0000000000000..73a3f4123496c
--- /dev/null
+++ b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.vagrantsupport.properties
@@ -0,0 +1 @@
+implementation-class=org.elasticsearch.gradle.vagrant.VagrantSupportPlugin
\ No newline at end of file
diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml
index 14c2bc8ca5a5f..7774fd8b13fe5 100644
--- a/buildSrc/src/main/resources/checkstyle_suppressions.xml
+++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml
@@ -10,11 +10,12 @@
   <suppress files="org[/\\]elasticsearch[/\\]painless[/\\]antlr[/\\]PainlessLexer\.java" checks="." />
   <suppress files="org[/\\]elasticsearch[/\\]painless[/\\]antlr[/\\]PainlessParser(|BaseVisitor|Visitor)\.java" checks="." />
 
-  <!-- ThrowableProxy is a forked copy from Log4j to hack around a bug; this can be removed when the hack is removed -->
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]apache[/\\]logging[/\\]log4j[/\\]core[/\\]impl[/\\]ThrowableProxy.java" checks="RegexpSinglelineJava" />
+  <!-- JNA requires the no-argument constructor on JNAKernel32Library.SizeT to be public-->
+  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]bootstrap[/\\]JNAKernel32Library.java" checks="RedundantModifier" />
 
   <!-- Hopefully temporary suppression of LineLength on files that don't pass it. We should remove these when we the
     files start to pass. -->
+  <suppress files="client[/\\]rest[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]client[/\\]HeapBufferedAsyncResponseConsumerTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]apache[/\\]lucene[/\\]queries[/\\]BlendedTermQuery.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]apache[/\\]lucene[/\\]queries[/\\]ExtendedCommonTermsQuery.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]apache[/\\]lucene[/\\]search[/\\]postingshighlight[/\\]CustomPostingsHighlighter.java" checks="LineLength" />
@@ -29,7 +30,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]repositories[/\\]delete[/\\]TransportDeleteRepositoryAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]repositories[/\\]get[/\\]GetRepositoriesRequestBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]repositories[/\\]get[/\\]TransportGetRepositoriesAction.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]repositories[/\\]put[/\\]PutRepositoryRequest.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]repositories[/\\]put[/\\]PutRepositoryRequestBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]repositories[/\\]put[/\\]TransportPutRepositoryAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]repositories[/\\]verify[/\\]TransportVerifyRepositoryAction.java" checks="LineLength" />
@@ -38,7 +38,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]settings[/\\]ClusterUpdateSettingsAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]settings[/\\]ClusterUpdateSettingsRequestBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]settings[/\\]SettingsUpdater.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]settings[/\\]TransportClusterUpdateSettingsAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]shards[/\\]ClusterSearchShardsAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]shards[/\\]ClusterSearchShardsRequestBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]shards[/\\]TransportClusterSearchShardsAction.java" checks="LineLength" />
@@ -58,14 +57,8 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]stats[/\\]TransportClusterStatsAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]tasks[/\\]PendingClusterTasksAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]tasks[/\\]PendingClusterTasksRequestBuilder.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]tasks[/\\]PendingClusterTasksResponse.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]tasks[/\\]TransportPendingClusterTasksAction.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]validate[/\\]template[/\\]RenderSearchTemplateAction.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]validate[/\\]template[/\\]RenderSearchTemplateRequestBuilder.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]validate[/\\]template[/\\]TransportRenderSearchTemplateAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]indices[/\\]alias[/\\]Alias.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]indices[/\\]alias[/\\]IndicesAliasesRequest.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]indices[/\\]alias[/\\]IndicesAliasesRequestBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]indices[/\\]alias[/\\]TransportIndicesAliasesAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]indices[/\\]alias[/\\]exists[/\\]TransportAliasesExistAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]indices[/\\]alias[/\\]get[/\\]BaseAliasesRequestBuilder.java" checks="LineLength" />
@@ -117,8 +110,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]indices[/\\]template[/\\]delete[/\\]TransportDeleteIndexTemplateAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]indices[/\\]template[/\\]get[/\\]GetIndexTemplatesRequestBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]indices[/\\]template[/\\]get[/\\]TransportGetIndexTemplatesAction.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]indices[/\\]template[/\\]put[/\\]PutIndexTemplateRequest.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]indices[/\\]template[/\\]put[/\\]PutIndexTemplateRequestBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]indices[/\\]template[/\\]put[/\\]TransportPutIndexTemplateAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]indices[/\\]upgrade[/\\]get[/\\]IndexUpgradeStatus.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]indices[/\\]upgrade[/\\]get[/\\]TransportUpgradeStatusAction.java" checks="LineLength" />
@@ -141,50 +132,28 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]bulk[/\\]TransportBulkAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]bulk[/\\]TransportShardBulkAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]delete[/\\]DeleteRequest.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]delete[/\\]TransportDeleteAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]explain[/\\]TransportExplainAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]get[/\\]GetRequest.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]get[/\\]MultiGetRequest.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]get[/\\]TransportGetAction.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]get[/\\]TransportMultiGetAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]get[/\\]TransportShardMultiGetAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]index[/\\]IndexRequest.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]index[/\\]IndexRequestBuilder.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]index[/\\]TransportIndexAction.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]indexedscripts[/\\]delete[/\\]DeleteIndexedScriptAction.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]indexedscripts[/\\]delete[/\\]DeleteIndexedScriptRequest.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]indexedscripts[/\\]delete[/\\]DeleteIndexedScriptRequestBuilder.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]indexedscripts[/\\]delete[/\\]TransportDeleteIndexedScriptAction.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]indexedscripts[/\\]get[/\\]GetIndexedScriptRequest.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]indexedscripts[/\\]get[/\\]GetIndexedScriptRequestBuilder.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]indexedscripts[/\\]get[/\\]TransportGetIndexedScriptAction.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]indexedscripts[/\\]put[/\\]PutIndexedScriptRequest.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]indexedscripts[/\\]put[/\\]PutIndexedScriptRequestBuilder.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]indexedscripts[/\\]put[/\\]TransportPutIndexedScriptAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]ingest[/\\]DeletePipelineRequestBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]ingest[/\\]DeletePipelineTransportAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]ingest[/\\]GetPipelineRequestBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]ingest[/\\]GetPipelineTransportAction.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]ingest[/\\]IngestActionFilter.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]ingest[/\\]IngestProxyActionFilter.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]ingest[/\\]PutPipelineTransportAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]ingest[/\\]SimulatePipelineRequest.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]ingest[/\\]SimulatePipelineRequestBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]ingest[/\\]SimulatePipelineTransportAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]search[/\\]MultiSearchRequestBuilder.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]search[/\\]SearchPhaseExecutionException.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]search[/\\]SearchResponse.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]search[/\\]SearchPhaseController.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]search[/\\]ShardSearchFailure.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]search[/\\]TransportClearScrollAction.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]search[/\\]TransportMultiSearchAction.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]suggest[/\\]SuggestResponse.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]ActionFilter.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]AutoCreateIndex.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]DelegatingActionListener.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]IndicesOptions.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]ToXContentToBytes.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]broadcast[/\\]BroadcastOperationRequestBuilder.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]broadcast[/\\]BroadcastRequest.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]broadcast[/\\]BroadcastResponse.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]broadcast[/\\]TransportBroadcastAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]broadcast[/\\]node[/\\]TransportBroadcastByNodeAction.java" checks="LineLength" />
@@ -202,7 +171,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]single[/\\]instance[/\\]InstanceShardOperationRequestBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]single[/\\]instance[/\\]TransportInstanceSingleOperationAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]single[/\\]shard[/\\]SingleShardOperationRequestBuilder.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]single[/\\]shard[/\\]SingleShardRequest.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]single[/\\]shard[/\\]TransportSingleShardAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]termvectors[/\\]MultiTermVectorsRequest.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]termvectors[/\\]MultiTermVectorsRequestBuilder.java" checks="LineLength" />
@@ -212,33 +180,23 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]termvectors[/\\]TransportMultiTermVectorsAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]termvectors[/\\]TransportShardMultiTermsVectorAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]termvectors[/\\]TransportTermVectorsAction.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]termvectors[/\\]dfs[/\\]ShardDfsOnlyRequest.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]termvectors[/\\]dfs[/\\]TransportDfsOnlyAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]update[/\\]TransportUpdateAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]update[/\\]UpdateHelper.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]update[/\\]UpdateRequest.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]update[/\\]UpdateRequestBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]bootstrap[/\\]Bootstrap.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]bootstrap[/\\]JNAKernel32Library.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]bootstrap[/\\]JNANatives.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]bootstrap[/\\]JVMCheck.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]bootstrap[/\\]JarHell.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]bootstrap[/\\]Seccomp.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]bootstrap[/\\]Security.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]client[/\\]ElasticsearchClient.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]client[/\\]FilterClient.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]client[/\\]support[/\\]AbstractClient.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]client[/\\]transport[/\\]TransportClient.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]client[/\\]transport[/\\]support[/\\]TransportProxyClient.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]ClusterModule.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]ClusterState.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]ClusterStateObserver.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]ClusterStateUpdateTask.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]DiffableUtils.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]IncompatibleClusterStateVersionException.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]InternalClusterInfoService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]LocalNodeMasterListener.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]action[/\\]index[/\\]NodeIndexDeletedAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]action[/\\]index[/\\]NodeMappingRefreshAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]action[/\\]shard[/\\]ShardStateAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]block[/\\]ClusterBlock.java" checks="LineLength" />
@@ -246,7 +204,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]metadata[/\\]AutoExpandReplicas.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]metadata[/\\]IndexMetaData.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]metadata[/\\]IndexNameExpressionResolver.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]metadata[/\\]IndexTemplateMetaData.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]metadata[/\\]MappingMetaData.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]metadata[/\\]MetaData.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]metadata[/\\]MetaDataCreateIndexService.java" checks="LineLength" />
@@ -259,7 +216,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]routing[/\\]IndexRoutingTable.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]routing[/\\]IndexShardRoutingTable.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]routing[/\\]OperationRouting.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]routing[/\\]RoutingNode.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]routing[/\\]RoutingNodes.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]routing[/\\]RoutingService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]routing[/\\]RoutingTable.java" checks="LineLength" />
@@ -272,8 +228,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]routing[/\\]allocation[/\\]command[/\\]AllocationCommands.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]routing[/\\]allocation[/\\]command[/\\]MoveAllocationCommand.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]routing[/\\]allocation[/\\]decider[/\\]AllocationDeciders.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]service[/\\]InternalClusterService.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]Base64.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]Booleans.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]Numbers.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]blobstore[/\\]fs[/\\]FsBlobStore.java" checks="LineLength" />
@@ -281,9 +235,7 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]bytes[/\\]BytesArray.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]bytes[/\\]PagedBytesReference.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]cache[/\\]Cache.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]cli[/\\]CheckFileCommand.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]collect[/\\]ImmutableOpenIntMap.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]geo[/\\]GeoDistance.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]inject[/\\]DefaultConstructionProxyFactory.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]inject[/\\]internal[/\\]ConstructionContext.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]inject[/\\]multibindings[/\\]MapBinder.java" checks="LineLength" />
@@ -298,36 +250,22 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]lucene[/\\]search[/\\]function[/\\]FiltersFunctionScoreQuery.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]lucene[/\\]search[/\\]function[/\\]FunctionScoreQuery.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]lucene[/\\]store[/\\]ByteArrayIndexInput.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]lucene[/\\]uid[/\\]Versions.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]network[/\\]Cidrs.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]network[/\\]NetworkModule.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]network[/\\]NetworkService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]recycler[/\\]Recyclers.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]util[/\\]BigArrays.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]util[/\\]CancellableThreads.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]util[/\\]CollectionUtils.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]util[/\\]ExtensionPoint.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]util[/\\]concurrent[/\\]EsExecutors.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]util[/\\]concurrent[/\\]PrioritizedEsThreadPoolExecutor.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]util[/\\]concurrent[/\\]ThreadBarrier.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]util[/\\]concurrent[/\\]ThreadContext.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]xcontent[/\\]XContentBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]xcontent[/\\]XContentFactory.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]xcontent[/\\]XContentHelper.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]xcontent[/\\]smile[/\\]SmileXContent.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]xcontent[/\\]support[/\\]XContentMapValues.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]Discovery.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]DiscoveryModule.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]DiscoveryService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]DiscoverySettings.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]local[/\\]LocalDiscovery.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]zen[/\\]ZenDiscovery.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]zen[/\\]elect[/\\]ElectMasterService.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]zen[/\\]fd[/\\]FaultDetection.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]zen[/\\]membership[/\\]MembershipAction.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]zen[/\\]ping[/\\]ZenPing.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]zen[/\\]publish[/\\]PendingClusterStatesQueue.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]zen[/\\]publish[/\\]PublishClusterStateAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]env[/\\]ESFileStore.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]AsyncShardFetch.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]GatewayAllocator.java" checks="LineLength" />
@@ -339,14 +277,11 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]AlreadyExpiredException.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]CompositeIndexEventListener.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]IndexSettings.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]IndexingSlowLog.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]MergePolicyConfig.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]SearchSlowLog.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]AnalysisRegistry.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]AnalysisService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]CommonGramsTokenFilterFactory.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]CustomAnalyzerProvider.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]NumericDoubleAnalyzer.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]ShingleTokenFilterFactory.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]StemmerOverrideTokenFilterFactory.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]compound[/\\]HyphenationCompoundWordTokenFilterFactory.java" checks="LineLength" />
@@ -363,67 +298,55 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]fieldcomparator[/\\]FloatValuesComparatorSource.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]fieldcomparator[/\\]LongValuesComparatorSource.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]ordinals[/\\]GlobalOrdinalsBuilder.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]ordinals[/\\]InternalGlobalOrdinalsIndexFieldData.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]ordinals[/\\]MultiOrdinals.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]ordinals[/\\]OrdinalsBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]ordinals[/\\]SinglePackedOrdinals.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]plain[/\\]AbstractAtomicParentChildFieldData.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]plain[/\\]AbstractIndexOrdinalsFieldData.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]plain[/\\]AbstractLatLonPointDVIndexFieldData.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]plain[/\\]BinaryDVIndexFieldData.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]plain[/\\]IndexIndexFieldData.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]plain[/\\]PagedBytesIndexFieldData.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]plain[/\\]ParentChildIndexFieldData.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]plain[/\\]SortedNumericDVIndexFieldData.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]plain[/\\]SortedSetDVOrdinalsIndexFieldData.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]get[/\\]GetResult.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]get[/\\]ShardGetService.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]BaseGeoPointFieldMapper.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]CompletionFieldMapper.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]DocumentFieldMappers.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]DocumentMapper.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]DocumentMapperParser.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]DocumentParser.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]DynamicTemplate.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]FieldMapper.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]FieldNamesFieldMapper.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]FieldTypeLookup.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]MappedFieldType.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]Mapper.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]MapperService.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]Mapping.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]MetadataFieldMapper.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]CompletionFieldMapper.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]LegacyDateFieldMapper.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]LegacyDoubleFieldMapper.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]LegacyFloatFieldMapper.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]LegacyNumberFieldMapper.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]StringFieldMapper.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]LegacyTokenCountFieldMapper.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]TypeParsers.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]BaseGeoPointFieldMapper.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]GeoPointFieldMapper.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]GeoPointFieldMapperLegacy.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]GeoShapeFieldMapper.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]FieldNamesFieldMapper.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]IdFieldMapper.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]IndexFieldMapper.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]LegacyDateFieldMapper.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]LegacyDoubleFieldMapper.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]LegacyNumberFieldMapper.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]LegacyTokenCountFieldMapper.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]MappedFieldType.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]MapperService.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]Mapping.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]MetadataFieldMapper.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]ObjectMapper.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]ParentFieldMapper.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]RootObjectMapper.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]RoutingFieldMapper.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]SourceFieldMapper.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]StringFieldMapper.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]TTLFieldMapper.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]TimestampFieldMapper.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]TypeFieldMapper.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]TypeParsers.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]UidFieldMapper.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]VersionFieldMapper.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]DynamicTemplate.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]ObjectMapper.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]RootObjectMapper.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]merge[/\\]MergeStats.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]AbstractQueryBuilder.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]MatchQueryParser.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]QueryBuilders.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]QueryValidationException.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]support[/\\]InnerHitsQueryParserHelper.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]support[/\\]QueryParsers.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]search[/\\]MatchQuery.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]search[/\\]MultiMatchQuery.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]search[/\\]geo[/\\]GeoDistanceRangeQuery.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]search[/\\]geo[/\\]IndexedGeoBoundingBoxQuery.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]shard[/\\]CommitPoint.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]shard[/\\]IndexEventListener.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]shard[/\\]IndexSearcherWrapper.java" checks="LineLength" />
@@ -433,11 +356,8 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]shard[/\\]ShardStateMetaData.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]shard[/\\]StoreRecovery.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]shard[/\\]TranslogRecoveryPerformer.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]snapshots[/\\]blobstore[/\\]BlobStoreIndexShardRepository.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]snapshots[/\\]blobstore[/\\]BlobStoreIndexShardSnapshots.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]store[/\\]IndexStore.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]store[/\\]IndexStoreConfig.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]store[/\\]LegacyVerification.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]store[/\\]Store.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]termvectors[/\\]TermVectorsService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]translog[/\\]BaseTranslogReader.java" checks="LineLength" />
@@ -447,7 +367,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]translog[/\\]TranslogWriter.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]IndexingMemoryController.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]IndicesService.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]IndicesWarmer.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]analysis[/\\]PreBuiltCacheFactory.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]analysis[/\\]PreBuiltTokenFilters.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]breaker[/\\]HierarchyCircuitBreakerService.java" checks="LineLength" />
@@ -455,153 +374,103 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]fielddata[/\\]cache[/\\]IndicesFieldDataCacheListener.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]flush[/\\]ShardsSyncedFlushResult.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]flush[/\\]SyncedFlushService.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]recovery[/\\]RecoveriesCollection.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]recovery[/\\]PeerRecoverySourceService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]recovery[/\\]RecoveryFailedException.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]recovery[/\\]RecoverySettings.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]recovery[/\\]PeerRecoverySourceService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]recovery[/\\]RecoveryState.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]recovery[/\\]StartRecoveryRequest.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]store[/\\]IndicesStore.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]store[/\\]TransportNodesListShardStoreMetaData.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]ttl[/\\]IndicesTTLService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]monitor[/\\]jvm[/\\]GcNames.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]monitor[/\\]jvm[/\\]HotThreads.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]monitor[/\\]jvm[/\\]JvmStats.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]node[/\\]Node.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]node[/\\]internal[/\\]InternalSettingsPreparer.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]plugins[/\\]PluginsService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]RepositoriesService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]Repository.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]RepositoryModule.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]RepositoryTypesRegistry.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]VerifyNodeRepositoryAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]blobstore[/\\]BlobStoreRepository.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]blobstore[/\\]ChecksumBlobStoreFormat.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]fs[/\\]FsRepository.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]uri[/\\]URLIndexShardRepository.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]uri[/\\]URLRepository.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]BytesRestResponse.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]RestController.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]cat[/\\]RestCountAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]cat[/\\]RestIndicesAction.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]cat[/\\]RestNodesAction.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]cat[/\\]RestPendingClusterTasksAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]cat[/\\]RestShardsAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]cat[/\\]RestThreadPoolAction.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]AbstractScriptParser.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]ScriptContextRegistry.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]ScriptModes.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]ScriptParameterParser.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]ScriptService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]ScriptSettings.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]Template.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]MultiValueMode.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]SearchService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]AggregatorFactories.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]AggregatorFactory.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]BaseAggregationBuilder.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]BucketCollector.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]ParsedAggregation.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]InternalAggregation.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]InternalAggregations.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]InternalMultiBucketAggregation.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]ValuesSourceAggregationBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]BucketsAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]InternalSingleBucketAggregation.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]children[/\\]ChildrenParser.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]children[/\\]ParentToChildrenAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]filter[/\\]InternalFilter.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]filters[/\\]FiltersAggregator.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]filters[/\\]FiltersParser.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]filters[/\\]InternalFilters.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]geogrid[/\\]GeoHashGridAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]global[/\\]GlobalAggregator.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]global[/\\]InternalGlobal.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]histogram[/\\]HistogramAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]missing[/\\]InternalMissing.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]missing[/\\]MissingAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]nested[/\\]InternalReverseNested.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]nested[/\\]ReverseNestedAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]range[/\\]RangeAggregator.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]sampler[/\\]DiversifiedBytesHashSamplerAggregator.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]sampler[/\\]DiversifiedMapSamplerAggregator.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]sampler[/\\]DiversifiedNumericSamplerAggregator.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]sampler[/\\]DiversifiedOrdinalsSamplerAggregator.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]sampler[/\\]InternalSampler.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]sampler[/\\]SamplerAggregator.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]significant[/\\]GlobalOrdinalsSignificantTermsAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]significant[/\\]SignificantTermsAggregatorFactory.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]significant[/\\]SignificantTermsParametersParser.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]significant[/\\]heuristics[/\\]GND.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]significant[/\\]heuristics[/\\]NXYSignificanceHeuristic.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]significant[/\\]heuristics[/\\]PercentageScore.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]significant[/\\]heuristics[/\\]ScriptHeuristic.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]significant[/\\]heuristics[/\\]SignificanceHeuristic.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]terms[/\\]AbstractTermsParametersParser.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]terms[/\\]GlobalOrdinalsStringTermsAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]terms[/\\]InternalOrder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]terms[/\\]LongTermsAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]terms[/\\]StringTermsAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]terms[/\\]TermsAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]terms[/\\]TermsAggregatorFactory.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]terms[/\\]TermsParametersParser.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]terms[/\\]support[/\\]IncludeExclude.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]metrics[/\\]ValuesSourceMetricsAggregationBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]metrics[/\\]cardinality[/\\]CardinalityAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]metrics[/\\]cardinality[/\\]HyperLogLogPlusPlus.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]metrics[/\\]geobounds[/\\]GeoBoundsAggregator.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]metrics[/\\]percentiles[/\\]tdigest[/\\]AbstractTDigestPercentilesAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]metrics[/\\]percentiles[/\\]tdigest[/\\]TDigestPercentileRanksAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]metrics[/\\]percentiles[/\\]tdigest[/\\]TDigestPercentilesAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]metrics[/\\]scripted[/\\]ScriptedMetricAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]metrics[/\\]stats[/\\]extended[/\\]ExtendedStatsAggregator.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]metrics[/\\]tophits[/\\]TopHitsAggregator.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]pipeline[/\\]bucketscript[/\\]BucketScriptPipelineAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]support[/\\]AggregationPath.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]support[/\\]ValuesSourceParser.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]support[/\\]format[/\\]ValueFormat.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]support[/\\]format[/\\]ValueParser.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]builder[/\\]SearchSourceBuilder.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]controller[/\\]SearchPhaseController.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]dfs[/\\]AggregatedDfs.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]dfs[/\\]DfsSearchResult.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]fetch[/\\]FetchPhase.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]fetch[/\\]FetchSearchResult.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]fetch[/\\]FetchSubPhase.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]fetch[/\\]FetchSubPhaseParseElement.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]internal[/\\]DefaultSearchContext.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]internal[/\\]FilteredSearchContext.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]internal[/\\]InternalSearchHit.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]internal[/\\]SearchContext.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]internal[/\\]ShardSearchTransportRequest.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]lookup[/\\]FieldLookup.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]lookup[/\\]LeafDocLookup.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]lookup[/\\]LeafFieldsLookup.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]query[/\\]QueryPhase.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]rescore[/\\]QueryRescorer.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]rescore[/\\]RescoreParseElement.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]sort[/\\]GeoDistanceSortParser.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]sort[/\\]ScriptSortParser.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]sort[/\\]SortParseElement.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]SuggestContextParser.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]SuggestUtils.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]completion[/\\]CompletionSuggestParser.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]completion[/\\]context[/\\]CategoryContextMapping.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]completion[/\\]context[/\\]ContextMapping.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]completion[/\\]context[/\\]GeoContextMapping.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]completion[/\\]context[/\\]GeoQueryContext.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]phrase[/\\]CandidateScorer.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]phrase[/\\]NoisyChannelSpellChecker.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]phrase[/\\]PhraseSuggestParser.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]phrase[/\\]WordScorer.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]term[/\\]TermSuggestParser.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]snapshots[/\\]RestoreService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]snapshots[/\\]SnapshotShardFailure.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]snapshots[/\\]SnapshotShardsService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]snapshots[/\\]SnapshotsService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]threadpool[/\\]ThreadPool.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]tribe[/\\]TribeService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]apache[/\\]lucene[/\\]queries[/\\]BlendedTermQueryTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]apache[/\\]lucene[/\\]search[/\\]postingshighlight[/\\]CustomPostingsHighlighterTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ESExceptionTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]NamingConventionTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]VersionTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]RejectionActionIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]HotThreadsIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]health[/\\]ClusterHealthResponsesTests.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]node[/\\]stats[/\\]NodeStatsTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]repositories[/\\]RepositoryBlocksIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]settings[/\\]SettingsUpdaterTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]snapshots[/\\]SnapshotBlocksIT.java" checks="LineLength" />
@@ -613,20 +482,15 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]indices[/\\]get[/\\]GetIndexIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]indices[/\\]shards[/\\]IndicesShardStoreRequestIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]indices[/\\]shards[/\\]IndicesShardStoreResponseTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]indices[/\\]template[/\\]put[/\\]MetaDataIndexTemplateServiceTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]bulk[/\\]BulkProcessorIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]bulk[/\\]BulkRequestTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]bulk[/\\]RetryTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]get[/\\]MultiGetShardRequestTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]ingest[/\\]BulkRequestModifierTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]ingest[/\\]IngestProxyActionFilterTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]ingest[/\\]SimulateExecutionServiceTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]ingest[/\\]SimulatePipelineRequestParsingTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]ingest[/\\]SimulatePipelineResponseTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]ingest[/\\]WriteableIngestDocumentTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]search[/\\]SearchRequestBuilderTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]IndicesOptionsTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]IndicesOptionsTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]TransportActionFilterChainTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]WaitActiveShardCountIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]broadcast[/\\]node[/\\]TransportBroadcastByNodeActionTests.java" checks="LineLength" />
@@ -662,14 +526,12 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]allocation[/\\]AwarenessAllocationIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]allocation[/\\]ClusterRerouteIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]allocation[/\\]FilteringAllocationIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]allocation[/\\]ShardsAllocatorModuleIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]allocation[/\\]SimpleAllocationIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]health[/\\]ClusterIndexHealthTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]metadata[/\\]AutoExpandReplicasTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]metadata[/\\]DateMathExpressionResolverTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]metadata[/\\]HumanReadableIndexSettingsTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]metadata[/\\]IndexNameExpressionResolverTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]metadata[/\\]MetaDataIndexUpgradeServiceTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]metadata[/\\]MetaDataTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]metadata[/\\]ToAndFromJsonMetaDataTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]metadata[/\\]WildcardExpressionResolverTests.java" checks="LineLength" />
@@ -711,26 +573,22 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]routing[/\\]allocation[/\\]ThrottlingAllocationTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]routing[/\\]allocation[/\\]UpdateNumberOfReplicasTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]routing[/\\]allocation[/\\]decider[/\\]DiskThresholdDeciderUnitTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]routing[/\\]allocation[/\\]decider[/\\]EnableAllocationDeciderIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]routing[/\\]allocation[/\\]decider[/\\]EnableAllocationTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]serialization[/\\]ClusterSerializationTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]serialization[/\\]DiffableTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]settings[/\\]ClusterSettingsIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]shards[/\\]ClusterSearchShardsIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]structure[/\\]RoutingIteratorTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]BooleansTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]blobstore[/\\]FsBlobStoreContainerTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]blobstore[/\\]FsBlobStoreTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]breaker[/\\]MemoryCircuitBreakerTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]geo[/\\]ShapeBuilderTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]hash[/\\]MessageDigestsTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]inject[/\\]ModuleTestCase.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]lucene[/\\]index[/\\]FreqTermsEnumTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]network[/\\]CidrsTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]unit[/\\]DistanceUnitTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]unit[/\\]FuzzinessTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]util[/\\]BigArraysTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]util[/\\]ExtensionPointTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]util[/\\]LongObjectHashMapTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]util[/\\]concurrent[/\\]EsExecutorsTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]util[/\\]concurrent[/\\]PrioritizedExecutorsTests.java" checks="LineLength" />
@@ -739,28 +597,23 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]deps[/\\]joda[/\\]SimpleJodaTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]BlockingClusterStatePublishResponseHandlerTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]DiscoveryWithServiceDisruptionsIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]ZenUnicastDiscoveryIT.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]zen[/\\]ElectMasterServiceTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]zen[/\\]ZenDiscoveryUnitTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]zen[/\\]publish[/\\]PublishClusterStateActionTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]document[/\\]DocumentActionsIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]env[/\\]EnvironmentTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]env[/\\]NodeEnvironmentTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]explain[/\\]ExplainActionIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]GatewayModuleTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]GatewayServiceTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]GatewayTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]MetaDataStateFormatTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]MetaDataWriteDataNodesIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]PrimaryShardAllocatorTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]PriorityComparatorTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]QuorumGatewayIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]RecoverAfterNodesIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]RecoveryBackwardsCompatibilityIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]RecoveryFromGatewayIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]ReplicaShardAllocatorTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]ReusePeerRecoverySharedTest.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]get[/\\]GetActionIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]IndexModuleTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]IndexServiceTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]IndexWithShadowReplicasIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]IndexingSlowLogTests.java" checks="LineLength" />
@@ -768,86 +621,71 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]SearchSlowLogTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]NGramTokenizerFactoryTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]PatternCaptureTokenFilterTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]PreBuiltAnalyzerProviderFactoryTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]PreBuiltAnalyzerTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]PreBuiltCharFilterFactoryFactoryTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]PreBuiltTokenFilterFactoryFactoryTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]PreBuiltTokenizerFactoryFactoryTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]WordDelimiterTokenFilterFactoryTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]commongrams[/\\]CommonGramsTokenFilterFactoryTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]engine[/\\]InternalEngineMergeIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]engine[/\\]InternalEngineTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]engine[/\\]ShadowEngineTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]AbstractFieldDataImplTestCase.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]AbstractFieldDataTestCase.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]AbstractStringFieldDataTestCase.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]BinaryDVFieldDataTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]DuelFieldDataTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]FieldDataCacheTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]IndexFieldDataServiceTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]ParentChildFieldDataTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]DocumentFieldMapperTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]DynamicMappingTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]FieldTypeTestCase.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]MapperServiceTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]BinaryFieldMapperTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]CustomBoostMappingTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]FieldLevelBoostTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]CompletionFieldMapperTests.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]CompletionFieldTypeTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]CompoundTypesTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]CopyToMapperTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]BooleanFieldMapperTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]CompletionFieldTypeTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]MultiFieldCopyToMapperTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]LegacyTokenCountFieldMapperTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]LegacyDateFieldMapperTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]GenericStoreDynamicTemplateTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]PathMatchDynamicTemplateTests.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]CustomBoostMappingTests.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]DocumentFieldMapperTests.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]DocumentMapperMergeTests.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]DocumentParserTests.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]DynamicMappingTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]DynamicTemplatesTests.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]ExternalFieldMapperTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]ExternalMapper.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]ExternalMetadataMapper.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]ExternalFieldMapperTests.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]FieldLevelBoostTests.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]FieldNamesFieldMapperTests.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]FieldTypeTestCase.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]GenericStoreDynamicTemplateTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]GeoEncodingTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]GeoPointFieldMapperTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]GeoPointFieldTypeTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]GeoShapeFieldMapperTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]GeohashMappingGeoPointTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]IdFieldMapperTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]IndexFieldMapperTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]FieldNamesFieldMapperTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]TypeFieldMapperTests.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]JavaMultiFieldMergeTests.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]LegacyDateFieldMapperTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]LegacyIpFieldMapperTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]DocumentMapperMergeTests.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]LegacyNumberFieldMapperTests.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]LegacyStringMappingTests.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]LegacyTokenCountFieldMapperTests.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]MapperServiceTests.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]MultiFieldCopyToMapperTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]MultiFieldTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]JavaMultiFieldMergeTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]NestedObjectMapperTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]LegacyNumberFieldMapperTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]NullValueObjectMappingTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]ObjectMapperTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]ParentFieldMapperTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]PathMapperTests.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]PathMatchDynamicTemplateTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]RoutingFieldMapperTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]SourceFieldMapperTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]LegacyStringMappingTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]TimestampFieldMapperTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]TTLFieldMapperTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]DocumentParserTests.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]TimestampFieldMapperTests.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]TokenCountFieldMapperIntegrationIT.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]TypeFieldMapperTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]UpdateMappingOnClusterIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]UpdateMappingTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]BoolQueryBuilderTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]BoostingQueryBuilderTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]CommonTermsQueryBuilderTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]FieldMaskingSpanQueryBuilderTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]GeoDistanceQueryBuilderTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]HasChildQueryBuilderTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]HasParentQueryBuilderTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]MoreLikeThisQueryBuilderTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]MultiMatchQueryBuilderTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]RandomQueryBuilder.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]RangeQueryBuilderTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]SpanMultiTermQueryBuilderTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]SpanNotQueryBuilderTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]support[/\\]QueryInnerHitsTests.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]functionscore[/\\]FunctionScoreTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]search[/\\]MultiMatchQueryTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]search[/\\]geo[/\\]GeoUtilsTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]search[/\\]nested[/\\]AbstractNumberNestedSortingTestCase.java" checks="LineLength" />
@@ -856,7 +694,6 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]search[/\\]nested[/\\]LongNestedSortingTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]search[/\\]nested[/\\]NestedSortingTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]shard[/\\]IndexShardTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]shard[/\\]IndexingOperationListenerTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]shard[/\\]ShardPathTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]similarity[/\\]SimilarityTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]snapshots[/\\]blobstore[/\\]FileInfoTests.java" checks="LineLength" />
@@ -871,10 +708,8 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indexlifecycle[/\\]IndexLifecycleActionIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]IndexingMemoryControllerTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]IndicesLifecycleListenerIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]IndicesLifecycleListenerSingleNodeTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]IndicesOptionsIntegrationIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]analyze[/\\]AnalyzeActionIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]analyze[/\\]HunspellServiceIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]exists[/\\]indices[/\\]IndicesExistsIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]exists[/\\]types[/\\]TypesExistsIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]flush[/\\]FlushIT.java" checks="LineLength" />
@@ -889,29 +724,20 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]recovery[/\\]IndexPrimaryRelocationIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]recovery[/\\]IndexRecoveryIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]recovery[/\\]RecoverySourceHandlerTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]recovery[/\\]RecoveryStateTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]recovery[/\\]RecoveryStatusTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]settings[/\\]UpdateNumberOfReplicasIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]settings[/\\]UpdateSettingsIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]state[/\\]CloseIndexDisableCloseAllIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]state[/\\]OpenCloseIndexIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]state[/\\]RareClusterStateIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]stats[/\\]IndexStatsIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]store[/\\]IndicesStoreIntegrationIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]store[/\\]IndicesStoreTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]template[/\\]SimpleIndexTemplateIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]mget[/\\]SimpleMgetIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]monitor[/\\]jvm[/\\]JvmGcMonitorServiceSettingsTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]plugins[/\\]PluginInfoTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]plugins[/\\]PluginsServiceTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]recovery[/\\]FullRollingRestartIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]recovery[/\\]RecoverySettingsTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]recovery[/\\]RecoveryWhileUnderLoadIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]recovery[/\\]RelocationIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]recovery[/\\]TruncatedRecoveryIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]BytesRestResponseTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]CorsRegexDefaultIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]CorsRegexIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]routing[/\\]AliasResolveRoutingIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]routing[/\\]AliasRoutingIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]routing[/\\]SimpleRoutingIT.java" checks="LineLength" />
@@ -919,15 +745,28 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]ScriptContextRegistryTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]ScriptContextTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]ScriptModesTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]ScriptParameterParserTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]ScriptServiceTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]MultiValueModeTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]SearchWithRejectionsIT.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]StressSearchServiceReaperIT.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]AggregationCollectorTests.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]ParsedAggregationTests.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]AggregationTestScriptsPlugin.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]AggregationsIntegrationIT.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]AggregatorFactoriesTests.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]AggregatorTestCase.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]BaseAggregationTestCase.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]BasePipelineAggregationTestCase.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]CombiIT.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]EquivalenceIT.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]FiltersAggsRewriteIT.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]InternalAggregationTestCase.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]MetaDataIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]MissingValueIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]ChildrenIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]DateHistogramIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]DateHistogramOffsetIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]GeoDistanceIT.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]MissingIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]NaNSortingIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]NestedIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]ParentIdAggIT.java" checks="LineLength" />
@@ -950,28 +789,21 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]basic[/\\]SearchWithRandomExceptionsIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]basic[/\\]SearchWithRandomIOExceptionsIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]basic[/\\]TransportTwoNodesSearchIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]child[/\\]ChildQuerySearchIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]child[/\\]ParentFieldLoadingIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]fetch[/\\]FetchSubPhasePluginIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]geo[/\\]GeoBoundingBoxIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]geo[/\\]GeoFilterIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]geo[/\\]GeoShapeQueryTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]highlight[/\\]HighlighterSearchIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]innerhits[/\\]InnerHitsIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]matchedqueries[/\\]MatchedQueriesIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]morelikethis[/\\]MoreLikeThisIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]nested[/\\]SimpleNestedIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]preference[/\\]SearchPreferenceIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]profile[/\\]QueryProfilerIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]query[/\\]ExistsIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]query[/\\]MultiMatchQueryIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]query[/\\]SearchQueryIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]scroll[/\\]DuelScrollIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]scroll[/\\]SearchScrollIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]scroll[/\\]SearchScrollWithFailingNodesIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]searchafter[/\\]SearchAfterIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]simple[/\\]SimpleSearchIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]sort[/\\]SortParserTests.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]sort[/\\]GeoDistanceSortBuilderIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]CompletionSuggestSearchIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]ContextCompletionSuggestSearchIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]CustomSuggester.java" checks="LineLength" />
@@ -980,68 +812,50 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]phrase[/\\]NoisyChannelSpellCheckerTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]similarity[/\\]SimilarityIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]snapshots[/\\]AbstractSnapshotIntegTestCase.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]snapshots[/\\]BlobStoreFormatIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]snapshots[/\\]DedicatedClusterSnapshotRestoreIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]snapshots[/\\]RepositoriesIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]snapshots[/\\]SharedClusterSnapshotRestoreIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]snapshots[/\\]SnapshotBackwardsCompatibilityIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]snapshots[/\\]SnapshotUtilsTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]ESBlobStoreRepositoryIntegTestCase.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]geo[/\\]RandomShapeGenerator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]hamcrest[/\\]ElasticsearchGeoAssertions.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]timestamp[/\\]SimpleTimestampIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]tribe[/\\]TribeIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ttl[/\\]SimpleTTLIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]update[/\\]UpdateIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]validate[/\\]SimpleValidateQueryIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]versioning[/\\]SimpleVersioningIT.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-expression[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]expression[/\\]ExpressionScriptEngineService.java" checks="LineLength" />
-  <suppress files="modules[/\\]lang-expression[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]expression[/\\]ExpressionSearchScript.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-expression[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]expression[/\\]ExpressionTests.java" checks="LineLength" />
-  <suppress files="modules[/\\]lang-expression[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]expression[/\\]IndexedExpressionTests.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-expression[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]expression[/\\]MoreExpressionTests.java" checks="LineLength" />
+  <suppress files="modules[/\\]lang-expression[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]expression[/\\]StoredExpressionTests.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]groovy[/\\]GroovyScriptEngineService.java" checks="LineLength" />
-  <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]groovy[/\\]GroovyScriptTests.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]groovy[/\\]GroovySecurityTests.java" checks="LineLength" />
   <suppress files="modules[/\\]percolator[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]MultiPercolateRequest.java" checks="LineLength" />
   <suppress files="modules[/\\]percolator[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]MultiPercolateRequestBuilder.java" checks="LineLength" />
-  <suppress files="modules[/\\]percolator[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]PercolateShardResponse.java" checks="LineLength" />
   <suppress files="modules[/\\]percolator[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]TransportMultiPercolateAction.java" checks="LineLength" />
-  <suppress files="modules[/\\]percolator[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]TransportPercolateAction.java" checks="LineLength" />
-  <suppress files="modules[/\\]percolator[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]TransportShardMultiPercolateAction.java" checks="LineLength" />
   <suppress files="modules[/\\]percolator[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]MultiPercolatorIT.java" checks="LineLength" />
-  <suppress files="modules[/\\]percolator[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]PercolatorIT.java" checks="LineLength" />
   <suppress files="modules[/\\]percolator[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]MultiPercolatorRequestTests.java" checks="LineLength" />
+  <suppress files="modules[/\\]percolator[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]PercolatorIT.java" checks="LineLength" />
   <suppress files="plugins[/\\]analysis-icu[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]IcuCollationTokenFilterFactory.java" checks="LineLength" />
   <suppress files="plugins[/\\]analysis-icu[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]IcuFoldingTokenFilterFactory.java" checks="LineLength" />
   <suppress files="plugins[/\\]analysis-icu[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]IcuNormalizerTokenFilterFactory.java" checks="LineLength" />
   <suppress files="plugins[/\\]analysis-icu[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]IndexableBinaryStringTools.java" checks="LineLength" />
-  <suppress files="plugins[/\\]analysis-icu[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]AnalysisTestUtils.java" checks="LineLength" />
   <suppress files="plugins[/\\]analysis-kuromoji[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]JapaneseStopTokenFilterFactory.java" checks="LineLength" />
   <suppress files="plugins[/\\]analysis-kuromoji[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]KuromojiAnalysisTests.java" checks="LineLength" />
   <suppress files="plugins[/\\]analysis-phonetic[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]PhoneticTokenFilterFactory.java" checks="LineLength" />
-  <suppress files="plugins[/\\]discovery-azure-classic[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cloud[/\\]azure[/\\]AbstractAzureTestCase.java" checks="LineLength" />
-  <suppress files="plugins[/\\]discovery-azure-classic[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]azure[/\\]AzureMinimumMasterNodesTests.java" checks="LineLength" />
-  <suppress files="plugins[/\\]discovery-azure-classic[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]azure[/\\]AzureSimpleTests.java" checks="LineLength" />
-  <suppress files="plugins[/\\]discovery-azure-classic[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]azure[/\\]AzureTwoStartedNodesTests.java" checks="LineLength" />
-  <suppress files="plugins[/\\]discovery-ec2[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cloud[/\\]aws[/\\]AbstractAwsTestCase.java" checks="LineLength" />
+  <suppress files="plugins[/\\]discovery-ec2[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]ec2[/\\]AbstractAwsTestCase.java" checks="LineLength" />
   <suppress files="plugins[/\\]discovery-ec2[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]ec2[/\\]AmazonEC2Mock.java" checks="LineLength" />
   <suppress files="plugins[/\\]discovery-gce[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]gce[/\\]GceNetworkTests.java" checks="LineLength" />
   <suppress files="plugins[/\\]lang-javascript[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]javascript[/\\]JavaScriptScriptEngineService.java" checks="LineLength" />
   <suppress files="plugins[/\\]lang-javascript[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]javascript[/\\]JavaScriptScriptEngineTests.java" checks="LineLength" />
   <suppress files="plugins[/\\]lang-javascript[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]javascript[/\\]JavaScriptScriptMultiThreadedTests.java" checks="LineLength" />
-  <suppress files="plugins[/\\]lang-javascript[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]javascript[/\\]JavaScriptSecurityTests.java" checks="LineLength" />
-  <suppress files="plugins[/\\]lang-javascript[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]javascript[/\\]SimpleBench.java" checks="LineLength" />
   <suppress files="plugins[/\\]lang-python[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]python[/\\]PythonScriptEngineTests.java" checks="LineLength" />
-  <suppress files="plugins[/\\]lang-python[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]python[/\\]PythonScriptMultiThreadedTests.java" checks="LineLength" />
   <suppress files="plugins[/\\]lang-python[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]python[/\\]PythonSecurityTests.java" checks="LineLength" />
   <suppress files="plugins[/\\]mapper-attachments[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]mapper[/\\]attachments[/\\]AttachmentMapper.java" checks="LineLength" />
-  <suppress files="plugins[/\\]mapper-attachments[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]mapper[/\\]attachments[/\\]DateAttachmentMapperTests.java" checks="LineLength" />
   <suppress files="plugins[/\\]mapper-attachments[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]mapper[/\\]attachments[/\\]EncryptedDocMapperTests.java" checks="LineLength" />
   <suppress files="plugins[/\\]mapper-attachments[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]mapper[/\\]attachments[/\\]MetadataMapperTests.java" checks="LineLength" />
   <suppress files="plugins[/\\]mapper-attachments[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]mapper[/\\]attachments[/\\]MultifieldAttachmentMapperTests.java" checks="LineLength" />
   <suppress files="plugins[/\\]mapper-attachments[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]mapper[/\\]attachments[/\\]SimpleAttachmentMapperTests.java" checks="LineLength" />
-  <suppress files="plugins[/\\]mapper-attachments[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]mapper[/\\]attachments[/\\]StandaloneRunner.java" checks="LineLength" />
   <suppress files="plugins[/\\]mapper-attachments[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]mapper[/\\]attachments[/\\]VariousDocTests.java" checks="LineLength" />
   <suppress files="plugins[/\\]mapper-murmur3[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]murmur3[/\\]Murmur3FieldMapper.java" checks="LineLength" />
   <suppress files="plugins[/\\]mapper-murmur3[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]murmur3[/\\]Murmur3FieldMapperTests.java" checks="LineLength" />
@@ -1050,26 +864,21 @@
   <suppress files="plugins[/\\]repository-azure[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cloud[/\\]azure[/\\]blobstore[/\\]AzureBlobStore.java" checks="LineLength" />
   <suppress files="plugins[/\\]repository-azure[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cloud[/\\]azure[/\\]storage[/\\]AzureStorageServiceImpl.java" checks="LineLength" />
   <suppress files="plugins[/\\]repository-azure[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cloud[/\\]azure[/\\]storage[/\\]AzureStorageSettings.java" checks="LineLength" />
-  <suppress files="plugins[/\\]repository-azure[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]azure[/\\]AzureRepository.java" checks="LineLength" />
-  <suppress files="plugins[/\\]repository-azure[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cloud[/\\]azure[/\\]AbstractAzureWithThirdPartyTestCase.java" checks="LineLength" />
   <suppress files="plugins[/\\]repository-azure[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cloud[/\\]azure[/\\]storage[/\\]AzureStorageServiceMock.java" checks="LineLength" />
-  <suppress files="plugins[/\\]repository-azure[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]azure[/\\]AzureSnapshotRestoreServiceTests.java" checks="LineLength" />
   <suppress files="plugins[/\\]repository-hdfs[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]hdfs[/\\]HdfsRepository.java" checks="LineLength" />
   <suppress files="plugins[/\\]repository-hdfs[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]hdfs[/\\]HdfsTests.java" checks="LineLength" />
-  <suppress files="plugins[/\\]repository-s3[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cloud[/\\]aws[/\\]blobstore[/\\]DefaultS3OutputStream.java" checks="LineLength" />
-  <suppress files="plugins[/\\]repository-s3[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cloud[/\\]aws[/\\]blobstore[/\\]S3BlobContainer.java" checks="LineLength" />
-  <suppress files="plugins[/\\]repository-s3[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cloud[/\\]aws[/\\]blobstore[/\\]S3OutputStream.java" checks="LineLength" />
   <suppress files="plugins[/\\]repository-s3[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]s3[/\\]S3Repository.java" checks="LineLength" />
-  <suppress files="plugins[/\\]repository-s3[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cloud[/\\]aws[/\\]AbstractAwsTestCase.java" checks="LineLength" />
-  <suppress files="plugins[/\\]repository-s3[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cloud[/\\]aws[/\\]AmazonS3Wrapper.java" checks="LineLength" />
-  <suppress files="plugins[/\\]repository-s3[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cloud[/\\]aws[/\\]TestAmazonS3.java" checks="LineLength" />
-  <suppress files="plugins[/\\]repository-s3[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cloud[/\\]aws[/\\]blobstore[/\\]MockDefaultS3OutputStream.java" checks="LineLength" />
+  <suppress files="plugins[/\\]repository-s3[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]s3[/\\]AbstractAwsTestCase.java" checks="LineLength" />
   <suppress files="plugins[/\\]repository-s3[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]s3[/\\]AbstractS3SnapshotRestoreTest.java" checks="LineLength" />
-  <suppress files="plugins[/\\]store-smb[/\\]src[/\\]main[/\\]java[/\\]org[/\\]apache[/\\]lucene[/\\]store[/\\]SmbDirectoryWrapper.java" checks="LineLength" />
+  <suppress files="plugins[/\\]repository-s3[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]s3[/\\]AmazonS3Wrapper.java" checks="LineLength" />
+  <suppress files="plugins[/\\]repository-s3[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]s3[/\\]AwsS3ServiceImplTests.java" checks="LineLength" />
+  <suppress files="plugins[/\\]repository-s3[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]s3[/\\]MockDefaultS3OutputStream.java" checks="LineLength" />
+  <suppress files="plugins[/\\]repository-s3[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]s3[/\\]TestAmazonS3.java" checks="LineLength" />
+  <suppress files="plugins[/\\]store-smb[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]store[/\\]SmbDirectoryWrapper.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]bootstrap[/\\]BootstrapForTesting.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]MockInternalClusterInfoService.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]routing[/\\]TestShardRouting.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]cli[/\\]CliToolTestCase.java" checks="LineLength" />
+  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]inject[/\\]ModuleTestCase.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]util[/\\]MockBigArrays.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]script[/\\]NativeSignificanceScoreScriptWithParams.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]BackgroundIndexer.java" checks="LineLength" />
@@ -1077,29 +886,17 @@
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]CorruptionUtils.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]ESBackcompatTestCase.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]ESIntegTestCase.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]ESSingleNodeTestCase.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]ESTestCase.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]ExternalNode.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]ExternalTestCluster.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]IndexSettingsModule.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]InternalTestCluster.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]MockIndexEventListener.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]TestSearchContext.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]cluster[/\\]NoopClusterService.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]cluster[/\\]TestClusterService.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]discovery[/\\]ClusterDiscoveryConfiguration.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]disruption[/\\]IntermittentLongGCDisruption.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]disruption[/\\]SlowClusterStateProcessing.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]engine[/\\]AssertingSearcher.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]engine[/\\]MockEngineSupport.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]hamcrest[/\\]ElasticsearchAssertions.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]junit[/\\]listeners[/\\]LoggingListener.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]store[/\\]MockFSDirectoryService.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]store[/\\]MockFSIndexStore.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]cli[/\\]CliTool.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]tribe[/\\]TribeService.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]sort[/\\]GeoDistanceSortBuilderIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]CorsNotSetIT.java" checks="LineLength" />
-  <suppress files="plugins[/\\]store-smb[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]store[/\\]SmbDirectoryWrapper.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]inject[/\\]ModuleTestCase.java" checks="LineLength" />
 </suppressions>
diff --git a/buildSrc/src/main/resources/eclipse.settings/org.eclipse.jdt.core.prefs b/buildSrc/src/main/resources/eclipse.settings/org.eclipse.jdt.core.prefs
index 9bee5e587b03f..48c93f444ba2a 100644
--- a/buildSrc/src/main/resources/eclipse.settings/org.eclipse.jdt.core.prefs
+++ b/buildSrc/src/main/resources/eclipse.settings/org.eclipse.jdt.core.prefs
@@ -1,6 +1,5 @@
 eclipse.preferences.version=1
 
-# previous configuration from maven build
 # this is merged with gradle's generated properties during 'gradle eclipse'
 
 # NOTE: null pointer analysis etc is not enabled currently, it seems very unstable
diff --git a/buildSrc/src/main/resources/forbidden/es-all-signatures.txt b/buildSrc/src/main/resources/forbidden/es-all-signatures.txt
index 37f03f4c91c28..1b655004cec3e 100644
--- a/buildSrc/src/main/resources/forbidden/es-all-signatures.txt
+++ b/buildSrc/src/main/resources/forbidden/es-all-signatures.txt
@@ -36,3 +36,12 @@ org.apache.lucene.document.FieldType#numericType()
 java.lang.invoke.MethodHandle#invoke(java.lang.Object[])
 java.lang.invoke.MethodHandle#invokeWithArguments(java.lang.Object[])
 java.lang.invoke.MethodHandle#invokeWithArguments(java.util.List)
+
+# This method is misleading, and uses lenient boolean parsing under the hood. If you intend to parse
+# a system property as a boolean, use
+# org.elasticsearch.common.Booleans#parseBoolean(java.lang.String) on the result of
+# java.lang.SystemProperty#getProperty(java.lang.String) instead. If you were not intending to parse
+# a system property as a boolean, but instead parse a string to a boolean, use
+# org.elasticsearch.common.Booleans#parseBoolean(java.lang.String) directly on the string.
+@defaultMessage use org.elasticsearch.common.Booleans#parseBoolean(java.lang.String)
+java.lang.Boolean#getBoolean(java.lang.String)
diff --git a/buildSrc/src/main/resources/forbidden/http-signatures.txt b/buildSrc/src/main/resources/forbidden/http-signatures.txt
new file mode 100644
index 0000000000000..dcf20bbb09387
--- /dev/null
+++ b/buildSrc/src/main/resources/forbidden/http-signatures.txt
@@ -0,0 +1,45 @@
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance  with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on
+# an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+# either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+@defaultMessage Explicitly specify the ContentType of HTTP entities when creating
+org.apache.http.entity.StringEntity#<init>(java.lang.String)
+org.apache.http.entity.StringEntity#<init>(java.lang.String,java.lang.String)
+org.apache.http.entity.StringEntity#<init>(java.lang.String,java.nio.charset.Charset)
+org.apache.http.entity.ByteArrayEntity#<init>(byte[])
+org.apache.http.entity.ByteArrayEntity#<init>(byte[],int,int)
+org.apache.http.entity.FileEntity#<init>(java.io.File)
+org.apache.http.entity.InputStreamEntity#<init>(java.io.InputStream)
+org.apache.http.entity.InputStreamEntity#<init>(java.io.InputStream,long)
+org.apache.http.nio.entity.NByteArrayEntity#<init>(byte[])
+org.apache.http.nio.entity.NByteArrayEntity#<init>(byte[],int,int)
+org.apache.http.nio.entity.NFileEntity#<init>(java.io.File)
+org.apache.http.nio.entity.NStringEntity#<init>(java.lang.String)
+org.apache.http.nio.entity.NStringEntity#<init>(java.lang.String,java.lang.String)
+
+@defaultMessage Use non-deprecated constructors
+org.apache.http.nio.entity.NFileEntity#<init>(java.io.File,java.lang.String)
+org.apache.http.nio.entity.NFileEntity#<init>(java.io.File,java.lang.String,boolean)
+org.apache.http.entity.FileEntity#<init>(java.io.File,java.lang.String)
+org.apache.http.entity.StringEntity#<init>(java.lang.String,java.lang.String,java.lang.String)
+
+@defaultMessage BasicEntity is easy to mess up and forget to set content type
+org.apache.http.entity.BasicHttpEntity#<init>()
+
+@defaultMessage EntityTemplate is easy to mess up and forget to set content type
+org.apache.http.entity.EntityTemplate#<init>(org.apache.http.entity.ContentProducer)
+
+@defaultMessage SerializableEntity uses java serialization and makes it easy to forget to set content type
+org.apache.http.entity.SerializableEntity#<init>(java.io.Serializable)
diff --git a/buildSrc/src/main/resources/plugin-descriptor.properties b/buildSrc/src/main/resources/plugin-descriptor.properties
index ebde46d326ba9..67c6ee39968cd 100644
--- a/buildSrc/src/main/resources/plugin-descriptor.properties
+++ b/buildSrc/src/main/resources/plugin-descriptor.properties
@@ -30,11 +30,15 @@ name=${name}
 # 'classname': the name of the class to load, fully-qualified.
 classname=${classname}
 #
-# 'java.version' version of java the code is built against
+# 'java.version': version of java the code is built against
 # use the system property java.specification.version
 # version string must be a sequence of nonnegative decimal integers
 # separated by "."'s and may have leading zeros
 java.version=${javaVersion}
 #
-# 'elasticsearch.version' version of elasticsearch compiled against
+# 'elasticsearch.version': version of elasticsearch compiled against
 elasticsearch.version=${elasticsearchVersion}
+### optional elements for plugins:
+#
+# 'has.native.controller': whether or not the plugin has a native controller
+has.native.controller=${hasNativeController}
diff --git a/buildSrc/version.properties b/buildSrc/version.properties
index e96f98245958d..eab95c5ace916 100644
--- a/buildSrc/version.properties
+++ b/buildSrc/version.properties
@@ -1,18 +1,21 @@
-elasticsearch     = 5.0.0-alpha6
-lucene            = 6.2.0
+elasticsearch     = 5.6.4
+lucene            = 6.6.1
 
 # optional dependencies
 spatial4j         = 0.6
 jts               = 1.13
-jackson           = 2.8.1
+jackson           = 2.8.6
 snakeyaml         = 1.15
-log4j             = 2.6.2
+# when updating log4j, please update also docs/java-api/index.asciidoc
+log4j             = 2.9.1
 slf4j             = 1.6.2
-jna               = 4.2.2
+
+# when updating the JNA version, also update the version in buildSrc/build.gradle
+jna               = 4.4.0-1
 
 # test dependencies
-randomizedrunner  = 2.3.2
-junit             = 4.11
+randomizedrunner  = 2.5.0
+junit             = 4.12
 httpclient        = 4.5.2
 httpcore          = 4.4.5
 commonslogging    = 1.1.3
diff --git a/client/benchmark/build.gradle b/client/benchmark/build.gradle
index bd4abddbd1ddf..ec0a002b1ae45 100644
--- a/client/benchmark/build.gradle
+++ b/client/benchmark/build.gradle
@@ -37,6 +37,10 @@ apply plugin: 'application'
 
 group = 'org.elasticsearch.client'
 
+// Not published so no need to assemble
+tasks.remove(assemble)
+build.dependsOn.remove('assemble')
+
 archivesBaseName = 'client-benchmarks'
 mainClassName = 'org.elasticsearch.client.benchmark.BenchmarkMain'
 
@@ -49,7 +53,7 @@ task test(type: Test, overwrite: true)
 dependencies {
   compile 'org.apache.commons:commons-math3:3.2'
 
-  compile("org.elasticsearch.client:rest:${version}")
+  compile("org.elasticsearch.client:elasticsearch-rest-client:${version}")
   // bottleneck should be the client, not Elasticsearch
   compile project(path: ':client:client-benchmark-noop-api-plugin')
   // for transport client
@@ -64,7 +68,3 @@ dependencies {
 
 // No licenses for our benchmark deps (we don't ship benchmarks)
 dependencyLicenses.enabled = false
-
-extraArchive {
-  javadoc = false
-}
diff --git a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/ops/bulk/BulkBenchmarkTask.java b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/ops/bulk/BulkBenchmarkTask.java
index 214a75d12cc01..e9cde26e6c870 100644
--- a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/ops/bulk/BulkBenchmarkTask.java
+++ b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/ops/bulk/BulkBenchmarkTask.java
@@ -95,7 +95,7 @@ private static final class LoadGenerator {
         private final BlockingQueue<List<String>> bulkQueue;
         private final int bulkSize;
 
-        public LoadGenerator(Path bulkDataFile, BlockingQueue<List<String>> bulkQueue, int bulkSize) {
+        LoadGenerator(Path bulkDataFile, BlockingQueue<List<String>> bulkQueue, int bulkSize) {
             this.bulkDataFile = bulkDataFile;
             this.bulkQueue = bulkQueue;
             this.bulkSize = bulkSize;
@@ -143,7 +143,7 @@ private static final class BulkIndexer implements Runnable {
         private final BulkRequestExecutor bulkRequestExecutor;
         private final SampleRecorder sampleRecorder;
 
-        public BulkIndexer(BlockingQueue<List<String>> bulkData, int warmupIterations, int measurementIterations,
+        BulkIndexer(BlockingQueue<List<String>> bulkData, int warmupIterations, int measurementIterations,
                            SampleRecorder sampleRecorder, BulkRequestExecutor bulkRequestExecutor) {
             this.bulkData = bulkData;
             this.warmupIterations = warmupIterations;
diff --git a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/rest/RestClientBenchmark.java b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/rest/RestClientBenchmark.java
index b342d93fba5a1..9210526e7c81c 100644
--- a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/rest/RestClientBenchmark.java
+++ b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/rest/RestClientBenchmark.java
@@ -73,7 +73,7 @@ private static final class RestBulkRequestExecutor implements BulkRequestExecuto
         private final RestClient client;
         private final String actionMetaData;
 
-        public RestBulkRequestExecutor(RestClient client, String index, String type) {
+        RestBulkRequestExecutor(RestClient client, String index, String type) {
             this.client = client;
             this.actionMetaData = String.format(Locale.ROOT, "{ \"index\" : { \"_index\" : \"%s\", \"_type\" : \"%s\" } }%n", index, type);
         }
diff --git a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/transport/TransportClientBenchmark.java b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/transport/TransportClientBenchmark.java
index c38234ef30241..bf25fde24cfc4 100644
--- a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/transport/TransportClientBenchmark.java
+++ b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/transport/TransportClientBenchmark.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugin.noop.NoopPlugin;
 import org.elasticsearch.plugin.noop.action.bulk.NoopBulkAction;
@@ -70,7 +71,7 @@ private static final class TransportBulkRequestExecutor implements BulkRequestEx
         private final String indexName;
         private final String typeName;
 
-        public TransportBulkRequestExecutor(TransportClient client, String indexName, String typeName) {
+        TransportBulkRequestExecutor(TransportClient client, String indexName, String typeName) {
             this.client = client;
             this.indexName = indexName;
             this.typeName = typeName;
@@ -80,7 +81,7 @@ public TransportBulkRequestExecutor(TransportClient client, String indexName, St
         public boolean bulkIndex(List<String> bulkData) {
             NoopBulkRequestBuilder builder = NoopBulkAction.INSTANCE.newRequestBuilder(client);
             for (String bulkItem : bulkData) {
-                builder.add(new IndexRequest(indexName, typeName).source(bulkItem.getBytes(StandardCharsets.UTF_8)));
+                builder.add(new IndexRequest(indexName, typeName).source(bulkItem.getBytes(StandardCharsets.UTF_8), XContentType.JSON));
             }
             BulkResponse bulkResponse;
             try {
diff --git a/client/client-benchmark-noop-api-plugin/build.gradle b/client/client-benchmark-noop-api-plugin/build.gradle
index 9f3af3ce002ae..bee41034c3ce5 100644
--- a/client/client-benchmark-noop-api-plugin/build.gradle
+++ b/client/client-benchmark-noop-api-plugin/build.gradle
@@ -20,7 +20,6 @@
 group = 'org.elasticsearch.plugin'
 
 apply plugin: 'elasticsearch.esplugin'
-apply plugin: 'com.bmuschko.nexus'
 
 esplugin {
   name 'client-benchmark-noop-api'
@@ -28,9 +27,12 @@ esplugin {
   classname 'org.elasticsearch.plugin.noop.NoopPlugin'
 }
 
+// Not published so no need to assemble
+tasks.remove(assemble)
+build.dependsOn.remove('assemble')
+
 compileJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-unchecked"
 
 // no unit tests
 test.enabled = false
 integTest.enabled = false
-
diff --git a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/NoopPlugin.java b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/NoopPlugin.java
index 343d3cf613a04..e8ed27715c10a 100644
--- a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/NoopPlugin.java
+++ b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/NoopPlugin.java
@@ -23,19 +23,27 @@
 import org.elasticsearch.plugin.noop.action.bulk.TransportNoopBulkAction;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.IndexScopedSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.plugin.noop.action.search.NoopSearchAction;
 import org.elasticsearch.plugin.noop.action.search.RestNoopSearchAction;
 import org.elasticsearch.plugin.noop.action.search.TransportNoopSearchAction;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestHandler;
 
 import java.util.Arrays;
 import java.util.List;
+import java.util.function.Supplier;
 
 public class NoopPlugin extends Plugin implements ActionPlugin {
     @Override
-    public List<ActionHandler<? extends ActionRequest<?>, ? extends ActionResponse>> getActions() {
+    public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
         return Arrays.asList(
             new ActionHandler<>(NoopBulkAction.INSTANCE, TransportNoopBulkAction.class),
             new ActionHandler<>(NoopSearchAction.INSTANCE, TransportNoopSearchAction.class)
@@ -43,7 +51,11 @@ public class NoopPlugin extends Plugin implements ActionPlugin {
     }
 
     @Override
-    public List<Class<? extends RestHandler>> getRestHandlers() {
-        return Arrays.asList(RestNoopBulkAction.class, RestNoopSearchAction.class);
+    public List<RestHandler> getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings,
+            IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver,
+            Supplier<DiscoveryNodes> nodesInCluster) {
+        return Arrays.asList(
+                new RestNoopBulkAction(settings, restController),
+                new RestNoopSearchAction(settings, restController));
     }
 }
diff --git a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/NoopBulkRequestBuilder.java b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/NoopBulkRequestBuilder.java
index ceaf9f8cc9d17..1034e722e8789 100644
--- a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/NoopBulkRequestBuilder.java
+++ b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/NoopBulkRequestBuilder.java
@@ -33,6 +33,7 @@
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentType;
 
 public class NoopBulkRequestBuilder extends ActionRequestBuilder<BulkRequest, BulkResponse, NoopBulkRequestBuilder>
         implements WriteRequestBuilder<NoopBulkRequestBuilder> {
@@ -95,17 +96,17 @@ public NoopBulkRequestBuilder add(UpdateRequestBuilder request) {
     /**
      * Adds a framed data in binary format
      */
-    public NoopBulkRequestBuilder add(byte[] data, int from, int length) throws Exception {
-        request.add(data, from, length, null, null);
+    public NoopBulkRequestBuilder add(byte[] data, int from, int length, XContentType xContentType) throws Exception {
+        request.add(data, from, length, null, null, xContentType);
         return this;
     }
 
     /**
      * Adds a framed data in binary format
      */
-    public NoopBulkRequestBuilder add(byte[] data, int from, int length, @Nullable String defaultIndex, @Nullable String defaultType)
-        throws Exception {
-        request.add(data, from, length, defaultIndex, defaultType);
+    public NoopBulkRequestBuilder add(byte[] data, int from, int length, @Nullable String defaultIndex, @Nullable String defaultType,
+                                      XContentType xContentType) throws Exception {
+        request.add(data, from, length, defaultIndex, defaultType, xContentType);
         return this;
     }
 
diff --git a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/RestNoopBulkAction.java b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/RestNoopBulkAction.java
index 814c05889b2b7..df712f537e43e 100644
--- a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/RestNoopBulkAction.java
+++ b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/RestNoopBulkAction.java
@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.plugin.noop.action.bulk;
 
+import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.bulk.BulkItemResponse;
 import org.elasticsearch.action.bulk.BulkRequest;
@@ -27,7 +28,6 @@
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.shard.ShardId;
@@ -39,12 +39,13 @@
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.action.RestBuilderListener;
 
+import java.io.IOException;
+
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.rest.RestRequest.Method.PUT;
 import static org.elasticsearch.rest.RestStatus.OK;
 
 public class RestNoopBulkAction extends BaseRestHandler {
-    @Inject
     public RestNoopBulkAction(Settings settings, RestController controller) {
         super(settings);
 
@@ -57,7 +58,7 @@ public RestNoopBulkAction(Settings settings, RestController controller) {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) throws Exception {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         BulkRequest bulkRequest = Requests.bulkRequest();
         String defaultIndex = request.param("index");
         String defaultType = request.param("type");
@@ -72,22 +73,24 @@ public void handleRequest(final RestRequest request, final RestChannel channel,
         }
         bulkRequest.timeout(request.paramAsTime("timeout", BulkShardRequest.DEFAULT_TIMEOUT));
         bulkRequest.setRefreshPolicy(request.param("refresh"));
-        bulkRequest.add(request.content(), defaultIndex, defaultType, defaultRouting, defaultFields, defaultPipeline, null, true);
+        bulkRequest.add(request.requiredContent(), defaultIndex, defaultType, defaultRouting, defaultFields,
+            null, defaultPipeline, null, true, request.getXContentType());
 
         // short circuit the call to the transport layer
-        BulkRestBuilderListener listener = new BulkRestBuilderListener(channel, request);
-        listener.onResponse(bulkRequest);
-
+        return channel -> {
+            BulkRestBuilderListener listener = new BulkRestBuilderListener(channel, request);
+            listener.onResponse(bulkRequest);
+        };
     }
 
     private static class BulkRestBuilderListener extends RestBuilderListener<BulkRequest> {
-        private final BulkItemResponse ITEM_RESPONSE = new BulkItemResponse(1, "update",
+        private final BulkItemResponse ITEM_RESPONSE = new BulkItemResponse(1, DocWriteRequest.OpType.UPDATE,
             new UpdateResponse(new ShardId("mock", "", 1), "mock_type", "1", 1L, DocWriteResponse.Result.CREATED));
 
         private final RestRequest request;
 
 
-        public BulkRestBuilderListener(RestChannel channel, RestRequest request) {
+        BulkRestBuilderListener(RestChannel channel, RestRequest request) {
             super(channel);
             this.request = request;
         }
@@ -99,9 +102,7 @@ public RestResponse buildResponse(BulkRequest bulkRequest, XContentBuilder build
             builder.field(Fields.ERRORS, false);
             builder.startArray(Fields.ITEMS);
             for (int idx = 0; idx < bulkRequest.numberOfActions(); idx++) {
-                builder.startObject();
                 ITEM_RESPONSE.toXContent(builder, request);
-                builder.endObject();
             }
             builder.endArray();
             builder.endObject();
diff --git a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/TransportNoopBulkAction.java b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/TransportNoopBulkAction.java
index dcc225c260309..aff5863bd9298 100644
--- a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/TransportNoopBulkAction.java
+++ b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/TransportNoopBulkAction.java
@@ -19,6 +19,7 @@
 package org.elasticsearch.plugin.noop.action.bulk;
 
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.bulk.BulkItemResponse;
 import org.elasticsearch.action.bulk.BulkRequest;
@@ -34,7 +35,7 @@
 import org.elasticsearch.transport.TransportService;
 
 public class TransportNoopBulkAction extends HandledTransportAction<BulkRequest, BulkResponse> {
-    private static final BulkItemResponse ITEM_RESPONSE = new BulkItemResponse(1, "update",
+    private static final BulkItemResponse ITEM_RESPONSE = new BulkItemResponse(1, DocWriteRequest.OpType.UPDATE,
         new UpdateResponse(new ShardId("mock", "", 1), "mock_type", "1", 1L, DocWriteResponse.Result.CREATED));
 
     @Inject
@@ -45,7 +46,7 @@ public TransportNoopBulkAction(Settings settings, ThreadPool threadPool, Transpo
 
     @Override
     protected void doExecute(BulkRequest request, ActionListener<BulkResponse> listener) {
-        final int itemCount = request.subRequests().size();
+        final int itemCount = request.requests().size();
         // simulate at least a realistic amount of data that gets serialized
         BulkItemResponse[] bulkItemResponses = new BulkItemResponse[itemCount];
         for (int idx = 0; idx < itemCount; idx++) {
diff --git a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/RestNoopSearchAction.java b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/RestNoopSearchAction.java
index 3520876af04b4..48a453c372507 100644
--- a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/RestNoopSearchAction.java
+++ b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/RestNoopSearchAction.java
@@ -20,10 +20,8 @@
 
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestStatusToXContentListener;
@@ -34,8 +32,6 @@
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
 public class RestNoopSearchAction extends BaseRestHandler {
-
-    @Inject
     public RestNoopSearchAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_noop_search", this);
@@ -47,8 +43,8 @@ public RestNoopSearchAction(Settings settings, RestController controller) {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) throws IOException {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         SearchRequest searchRequest = new SearchRequest();
-        client.execute(NoopSearchAction.INSTANCE, searchRequest, new RestStatusToXContentListener<>(channel));
+        return channel -> client.execute(NoopSearchAction.INSTANCE, searchRequest, new RestStatusToXContentListener<>(channel));
     }
 }
diff --git a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/TransportNoopSearchAction.java b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/TransportNoopSearchAction.java
index c4397684bc41e..280e0b08f2c72 100644
--- a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/TransportNoopSearchAction.java
+++ b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/TransportNoopSearchAction.java
@@ -28,8 +28,8 @@
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.search.aggregations.InternalAggregations;
-import org.elasticsearch.search.internal.InternalSearchHit;
-import org.elasticsearch.search.internal.InternalSearchHits;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.internal.InternalSearchResponse;
 import org.elasticsearch.search.profile.SearchProfileShardResults;
 import org.elasticsearch.search.suggest.Suggest;
@@ -49,10 +49,10 @@ public TransportNoopSearchAction(Settings settings, ThreadPool threadPool, Trans
     @Override
     protected void doExecute(SearchRequest request, ActionListener<SearchResponse> listener) {
         listener.onResponse(new SearchResponse(new InternalSearchResponse(
-            new InternalSearchHits(
-                new InternalSearchHit[0], 0L, 0.0f),
+            new SearchHits(
+                new SearchHit[0], 0L, 0.0f),
             new InternalAggregations(Collections.emptyList()),
             new Suggest(Collections.emptyList()),
-            new SearchProfileShardResults(Collections.emptyMap()), false, false), "", 1, 1, 0, new ShardSearchFailure[0]));
+            new SearchProfileShardResults(Collections.emptyMap()), false, false, 1), "", 1, 1, 0, 0, new ShardSearchFailure[0]));
     }
 }
diff --git a/client/rest-high-level/build.gradle b/client/rest-high-level/build.gradle
new file mode 100644
index 0000000000000..ba97605dba82e
--- /dev/null
+++ b/client/rest-high-level/build.gradle
@@ -0,0 +1,63 @@
+import org.elasticsearch.gradle.precommit.PrecommitTasks
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+apply plugin: 'elasticsearch.build'
+apply plugin: 'elasticsearch.rest-test'
+apply plugin: 'nebula.maven-base-publish'
+apply plugin: 'nebula.maven-scm'
+
+group = 'org.elasticsearch.client'
+archivesBaseName = 'elasticsearch-rest-high-level-client'
+
+publishing {
+    publications {
+        nebula {
+            artifactId = archivesBaseName
+        }
+    }
+}
+
+dependencies {
+  compile "org.elasticsearch:elasticsearch:${version}"
+  compile "org.elasticsearch.client:elasticsearch-rest-client:${version}"
+  compile "org.elasticsearch.plugin:parent-join-client:${version}"
+  compile "org.elasticsearch.plugin:aggs-matrix-stats-client:${version}"
+
+  testCompile "org.elasticsearch.client:test:${version}"
+  testCompile "org.elasticsearch.test:framework:${version}"
+  testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
+  testCompile "junit:junit:${versions.junit}"
+  testCompile "org.hamcrest:hamcrest-all:${versions.hamcrest}"
+}
+
+dependencyLicenses {
+  // Don't check licenses for dependency that are part of the elasticsearch project
+  // But any other dependency should have its license/notice/sha1
+  dependencies = project.configurations.runtime.fileCollection {
+    it.group.startsWith('org.elasticsearch') == false
+  }
+}
+
+forbiddenApisMain {
+  // core does not depend on the httpclient for compile so we add the signatures here. We don't add them for test as they are already
+  // specified
+  signaturesURLs += [PrecommitTasks.getResource('/forbidden/http-signatures.txt')]
+  signaturesURLs += [file('src/main/resources/forbidden/rest-high-level-signatures.txt').toURI().toURL()]
+}
\ No newline at end of file
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java
new file mode 100644
index 0000000000000..7a95553c3c003
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java
@@ -0,0 +1,578 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import org.apache.http.HttpEntity;
+import org.apache.http.client.methods.HttpDelete;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpHead;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpPut;
+import org.apache.http.entity.ByteArrayEntity;
+import org.apache.http.entity.ContentType;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.bulk.BulkRequest;
+import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.get.GetRequest;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.search.ClearScrollRequest;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchScrollRequest;
+import org.elasticsearch.action.support.ActiveShardCount;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.action.update.UpdateRequest;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.lucene.uid.Versions;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.VersionType;
+import org.elasticsearch.rest.action.search.RestSearchAction;
+import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.StringJoiner;
+
+public final class Request {
+
+    static final XContentType REQUEST_BODY_CONTENT_TYPE = XContentType.JSON;
+
+    private final String method;
+    private final String endpoint;
+    private final Map<String, String> parameters;
+    private final HttpEntity entity;
+
+    public Request(String method, String endpoint, Map<String, String> parameters, HttpEntity entity) {
+        this.method = Objects.requireNonNull(method, "method cannot be null");
+        this.endpoint = Objects.requireNonNull(endpoint, "endpoint cannot be null");
+        this.parameters = Objects.requireNonNull(parameters, "parameters cannot be null");
+        this.entity = entity;
+    }
+
+    public String getMethod() {
+        return method;
+    }
+
+    public String getEndpoint() {
+        return endpoint;
+    }
+
+    public Map<String, String> getParameters() {
+        return parameters;
+    }
+
+    public HttpEntity getEntity() {
+        return entity;
+    }
+
+    @Override
+    public String toString() {
+        return "Request{" +
+                "method='" + method + '\'' +
+                ", endpoint='" + endpoint + '\'' +
+                ", params=" + parameters +
+                ", hasBody=" + (entity != null) +
+                '}';
+    }
+
+    static Request delete(DeleteRequest deleteRequest) {
+        String endpoint = endpoint(deleteRequest.index(), deleteRequest.type(), deleteRequest.id());
+
+        Params parameters = Params.builder();
+        parameters.withRouting(deleteRequest.routing());
+        parameters.withParent(deleteRequest.parent());
+        parameters.withTimeout(deleteRequest.timeout());
+        parameters.withVersion(deleteRequest.version());
+        parameters.withVersionType(deleteRequest.versionType());
+        parameters.withRefreshPolicy(deleteRequest.getRefreshPolicy());
+        parameters.withWaitForActiveShards(deleteRequest.waitForActiveShards());
+
+        return new Request(HttpDelete.METHOD_NAME, endpoint, parameters.getParams(), null);
+    }
+
+    static Request info() {
+        return new Request(HttpGet.METHOD_NAME, "/", Collections.emptyMap(), null);
+    }
+
+    static Request bulk(BulkRequest bulkRequest) throws IOException {
+        Params parameters = Params.builder();
+        parameters.withTimeout(bulkRequest.timeout());
+        parameters.withRefreshPolicy(bulkRequest.getRefreshPolicy());
+
+        // Bulk API only supports newline delimited JSON or Smile. Before executing
+        // the bulk, we need to check that all requests have the same content-type
+        // and this content-type is supported by the Bulk API.
+        XContentType bulkContentType = null;
+        for (int i = 0; i < bulkRequest.numberOfActions(); i++) {
+            DocWriteRequest<?> request = bulkRequest.requests().get(i);
+
+            DocWriteRequest.OpType opType = request.opType();
+            if (opType == DocWriteRequest.OpType.INDEX || opType == DocWriteRequest.OpType.CREATE) {
+                bulkContentType = enforceSameContentType((IndexRequest) request, bulkContentType);
+
+            } else if (opType == DocWriteRequest.OpType.UPDATE) {
+                UpdateRequest updateRequest = (UpdateRequest) request;
+                if (updateRequest.doc() != null) {
+                    bulkContentType = enforceSameContentType(updateRequest.doc(), bulkContentType);
+                }
+                if (updateRequest.upsertRequest() != null) {
+                    bulkContentType = enforceSameContentType(updateRequest.upsertRequest(), bulkContentType);
+                }
+            }
+        }
+
+        if (bulkContentType == null) {
+            bulkContentType = XContentType.JSON;
+        }
+
+        final byte separator = bulkContentType.xContent().streamSeparator();
+        final ContentType requestContentType = createContentType(bulkContentType);
+
+        ByteArrayOutputStream content = new ByteArrayOutputStream();
+        for (DocWriteRequest<?> request : bulkRequest.requests()) {
+            DocWriteRequest.OpType opType = request.opType();
+
+            try (XContentBuilder metadata = XContentBuilder.builder(bulkContentType.xContent())) {
+                metadata.startObject();
+                {
+                    metadata.startObject(opType.getLowercase());
+                    if (Strings.hasLength(request.index())) {
+                        metadata.field("_index", request.index());
+                    }
+                    if (Strings.hasLength(request.type())) {
+                        metadata.field("_type", request.type());
+                    }
+                    if (Strings.hasLength(request.id())) {
+                        metadata.field("_id", request.id());
+                    }
+                    if (Strings.hasLength(request.routing())) {
+                        metadata.field("_routing", request.routing());
+                    }
+                    if (Strings.hasLength(request.parent())) {
+                        metadata.field("_parent", request.parent());
+                    }
+                    if (request.version() != Versions.MATCH_ANY) {
+                        metadata.field("_version", request.version());
+                    }
+
+                    VersionType versionType = request.versionType();
+                    if (versionType != VersionType.INTERNAL) {
+                        if (versionType == VersionType.EXTERNAL) {
+                            metadata.field("_version_type", "external");
+                        } else if (versionType == VersionType.EXTERNAL_GTE) {
+                            metadata.field("_version_type", "external_gte");
+                        } else if (versionType == VersionType.FORCE) {
+                            metadata.field("_version_type", "force");
+                        }
+                    }
+
+                    if (opType == DocWriteRequest.OpType.INDEX || opType == DocWriteRequest.OpType.CREATE) {
+                        IndexRequest indexRequest = (IndexRequest) request;
+                        if (Strings.hasLength(indexRequest.getPipeline())) {
+                            metadata.field("pipeline", indexRequest.getPipeline());
+                        }
+                    } else if (opType == DocWriteRequest.OpType.UPDATE) {
+                        UpdateRequest updateRequest = (UpdateRequest) request;
+                        if (updateRequest.retryOnConflict() > 0) {
+                            metadata.field("_retry_on_conflict", updateRequest.retryOnConflict());
+                        }
+                        if (updateRequest.fetchSource() != null) {
+                            metadata.field("_source", updateRequest.fetchSource());
+                        }
+                    }
+                    metadata.endObject();
+                }
+                metadata.endObject();
+
+                BytesRef metadataSource = metadata.bytes().toBytesRef();
+                content.write(metadataSource.bytes, metadataSource.offset, metadataSource.length);
+                content.write(separator);
+            }
+
+            BytesRef source = null;
+            if (opType == DocWriteRequest.OpType.INDEX || opType == DocWriteRequest.OpType.CREATE) {
+                IndexRequest indexRequest = (IndexRequest) request;
+                BytesReference indexSource = indexRequest.source();
+                XContentType indexXContentType = indexRequest.getContentType();
+
+                try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, indexSource, indexXContentType)) {
+                    try (XContentBuilder builder = XContentBuilder.builder(bulkContentType.xContent())) {
+                        builder.copyCurrentStructure(parser);
+                        source = builder.bytes().toBytesRef();
+                    }
+                }
+            } else if (opType == DocWriteRequest.OpType.UPDATE) {
+                source = XContentHelper.toXContent((UpdateRequest) request, bulkContentType, false).toBytesRef();
+            }
+
+            if (source != null) {
+                content.write(source.bytes, source.offset, source.length);
+                content.write(separator);
+            }
+        }
+
+        HttpEntity entity = new ByteArrayEntity(content.toByteArray(), 0, content.size(), requestContentType);
+        return new Request(HttpPost.METHOD_NAME, "/_bulk", parameters.getParams(), entity);
+    }
+
+    static Request exists(GetRequest getRequest) {
+        Request request = get(getRequest);
+        return new Request(HttpHead.METHOD_NAME, request.endpoint, request.parameters, null);
+    }
+
+    static Request get(GetRequest getRequest) {
+        String endpoint = endpoint(getRequest.index(), getRequest.type(), getRequest.id());
+
+        Params parameters = Params.builder();
+        parameters.withPreference(getRequest.preference());
+        parameters.withRouting(getRequest.routing());
+        parameters.withParent(getRequest.parent());
+        parameters.withRefresh(getRequest.refresh());
+        parameters.withRealtime(getRequest.realtime());
+        parameters.withStoredFields(getRequest.storedFields());
+        parameters.withVersion(getRequest.version());
+        parameters.withVersionType(getRequest.versionType());
+        parameters.withFetchSourceContext(getRequest.fetchSourceContext());
+
+        return new Request(HttpGet.METHOD_NAME, endpoint, parameters.getParams(), null);
+    }
+
+    static Request index(IndexRequest indexRequest) {
+        String method = Strings.hasLength(indexRequest.id()) ? HttpPut.METHOD_NAME : HttpPost.METHOD_NAME;
+
+        boolean isCreate = (indexRequest.opType() == DocWriteRequest.OpType.CREATE);
+        String endpoint = endpoint(indexRequest.index(), indexRequest.type(), indexRequest.id(), isCreate ? "_create" : null);
+
+        Params parameters = Params.builder();
+        parameters.withRouting(indexRequest.routing());
+        parameters.withParent(indexRequest.parent());
+        parameters.withTimeout(indexRequest.timeout());
+        parameters.withVersion(indexRequest.version());
+        parameters.withVersionType(indexRequest.versionType());
+        parameters.withPipeline(indexRequest.getPipeline());
+        parameters.withRefreshPolicy(indexRequest.getRefreshPolicy());
+        parameters.withWaitForActiveShards(indexRequest.waitForActiveShards());
+
+        BytesRef source = indexRequest.source().toBytesRef();
+        ContentType contentType = createContentType(indexRequest.getContentType());
+        HttpEntity entity = new ByteArrayEntity(source.bytes, source.offset, source.length, contentType);
+
+        return new Request(method, endpoint, parameters.getParams(), entity);
+    }
+
+    static Request ping() {
+        return new Request(HttpHead.METHOD_NAME, "/", Collections.emptyMap(), null);
+    }
+
+    static Request update(UpdateRequest updateRequest) throws IOException {
+        String endpoint = endpoint(updateRequest.index(), updateRequest.type(), updateRequest.id(), "_update");
+
+        Params parameters = Params.builder();
+        parameters.withRouting(updateRequest.routing());
+        parameters.withParent(updateRequest.parent());
+        parameters.withTimeout(updateRequest.timeout());
+        parameters.withRefreshPolicy(updateRequest.getRefreshPolicy());
+        parameters.withWaitForActiveShards(updateRequest.waitForActiveShards());
+        parameters.withDocAsUpsert(updateRequest.docAsUpsert());
+        parameters.withFetchSourceContext(updateRequest.fetchSource());
+        parameters.withRetryOnConflict(updateRequest.retryOnConflict());
+        parameters.withVersion(updateRequest.version());
+        parameters.withVersionType(updateRequest.versionType());
+
+        // The Java API allows update requests with different content types
+        // set for the partial document and the upsert document. This client
+        // only accepts update requests that have the same content types set
+        // for both doc and upsert.
+        XContentType xContentType = null;
+        if (updateRequest.doc() != null) {
+            xContentType = updateRequest.doc().getContentType();
+        }
+        if (updateRequest.upsertRequest() != null) {
+            XContentType upsertContentType = updateRequest.upsertRequest().getContentType();
+            if ((xContentType != null) && (xContentType != upsertContentType)) {
+                throw new IllegalStateException("Update request cannot have different content types for doc [" + xContentType + "]" +
+                        " and upsert [" + upsertContentType + "] documents");
+            } else {
+                xContentType = upsertContentType;
+            }
+        }
+        if (xContentType == null) {
+            xContentType = Requests.INDEX_CONTENT_TYPE;
+        }
+
+        HttpEntity entity = createEntity(updateRequest, xContentType);
+        return new Request(HttpPost.METHOD_NAME, endpoint, parameters.getParams(), entity);
+    }
+
+    static Request search(SearchRequest searchRequest) throws IOException {
+        String endpoint = endpoint(searchRequest.indices(), searchRequest.types(), "_search");
+        Params params = Params.builder();
+        params.putParam(RestSearchAction.TYPED_KEYS_PARAM, "true");
+        params.withRouting(searchRequest.routing());
+        params.withPreference(searchRequest.preference());
+        params.withIndicesOptions(searchRequest.indicesOptions());
+        params.putParam("search_type", searchRequest.searchType().name().toLowerCase(Locale.ROOT));
+        if (searchRequest.requestCache() != null) {
+            params.putParam("request_cache", Boolean.toString(searchRequest.requestCache()));
+        }
+        params.putParam("batched_reduce_size", Integer.toString(searchRequest.getBatchedReduceSize()));
+        if (searchRequest.scroll() != null) {
+            params.putParam("scroll", searchRequest.scroll().keepAlive());
+        }
+        HttpEntity entity = null;
+        if (searchRequest.source() != null) {
+            entity = createEntity(searchRequest.source(), REQUEST_BODY_CONTENT_TYPE);
+        }
+        return new Request(HttpGet.METHOD_NAME, endpoint, params.getParams(), entity);
+    }
+
+    static Request searchScroll(SearchScrollRequest searchScrollRequest) throws IOException {
+        HttpEntity entity = createEntity(searchScrollRequest, REQUEST_BODY_CONTENT_TYPE);
+        return new Request("GET", "/_search/scroll", Collections.emptyMap(), entity);
+    }
+
+    static Request clearScroll(ClearScrollRequest clearScrollRequest) throws IOException {
+        HttpEntity entity = createEntity(clearScrollRequest, REQUEST_BODY_CONTENT_TYPE);
+        return new Request("DELETE", "/_search/scroll", Collections.emptyMap(), entity);
+    }
+
+    private static HttpEntity createEntity(ToXContent toXContent, XContentType xContentType) throws IOException {
+        BytesRef source = XContentHelper.toXContent(toXContent, xContentType, false).toBytesRef();
+        return new ByteArrayEntity(source.bytes, source.offset, source.length, createContentType(xContentType));
+    }
+
+    static String endpoint(String[] indices, String[] types, String endpoint) {
+        return endpoint(String.join(",", indices), String.join(",", types), endpoint);
+    }
+
+    /**
+     * Utility method to build request's endpoint.
+     */
+    static String endpoint(String... parts) {
+        StringJoiner joiner = new StringJoiner("/", "/", "");
+        for (String part : parts) {
+            if (Strings.hasLength(part)) {
+                joiner.add(part);
+            }
+        }
+        return joiner.toString();
+    }
+
+    /**
+     * Returns a {@link ContentType} from a given {@link XContentType}.
+     *
+     * @param xContentType the {@link XContentType}
+     * @return the {@link ContentType}
+     */
+    @SuppressForbidden(reason = "Only allowed place to convert a XContentType to a ContentType")
+    public static ContentType createContentType(final XContentType xContentType) {
+        return ContentType.create(xContentType.mediaTypeWithoutParameters(), (Charset) null);
+    }
+
+    /**
+     * Utility class to build request's parameters map and centralize all parameter names.
+     */
+    static class Params {
+        private final Map<String, String> params = new HashMap<>();
+
+        private Params() {
+        }
+
+        Params putParam(String key, String value) {
+            if (Strings.hasLength(value)) {
+                if (params.putIfAbsent(key, value) != null) {
+                    throw new IllegalArgumentException("Request parameter [" + key + "] is already registered");
+                }
+            }
+            return this;
+        }
+
+        Params putParam(String key, TimeValue value) {
+            if (value != null) {
+                return putParam(key, value.getStringRep());
+            }
+            return this;
+        }
+
+        Params withDocAsUpsert(boolean docAsUpsert) {
+            if (docAsUpsert) {
+                return putParam("doc_as_upsert", Boolean.TRUE.toString());
+            }
+            return this;
+        }
+
+        Params withFetchSourceContext(FetchSourceContext fetchSourceContext) {
+            if (fetchSourceContext != null) {
+                if (fetchSourceContext.fetchSource() == false) {
+                    putParam("_source", Boolean.FALSE.toString());
+                }
+                if (fetchSourceContext.includes() != null && fetchSourceContext.includes().length > 0) {
+                    putParam("_source_include", String.join(",", fetchSourceContext.includes()));
+                }
+                if (fetchSourceContext.excludes() != null && fetchSourceContext.excludes().length > 0) {
+                    putParam("_source_exclude", String.join(",", fetchSourceContext.excludes()));
+                }
+            }
+            return this;
+        }
+
+        Params withParent(String parent) {
+            return putParam("parent", parent);
+        }
+
+        Params withPipeline(String pipeline) {
+            return putParam("pipeline", pipeline);
+        }
+
+        Params withPreference(String preference) {
+            return putParam("preference", preference);
+        }
+
+        Params withRealtime(boolean realtime) {
+            if (realtime == false) {
+                return putParam("realtime", Boolean.FALSE.toString());
+            }
+            return this;
+        }
+
+        Params withRefresh(boolean refresh) {
+            if (refresh) {
+                return withRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
+            }
+            return this;
+        }
+
+        Params withRefreshPolicy(WriteRequest.RefreshPolicy refreshPolicy) {
+            if (refreshPolicy != WriteRequest.RefreshPolicy.NONE) {
+                return putParam("refresh", refreshPolicy.getValue());
+            }
+            return this;
+        }
+
+        Params withRetryOnConflict(int retryOnConflict) {
+            if (retryOnConflict > 0) {
+                return putParam("retry_on_conflict", String.valueOf(retryOnConflict));
+            }
+            return this;
+        }
+
+        Params withRouting(String routing) {
+            return putParam("routing", routing);
+        }
+
+        Params withStoredFields(String[] storedFields) {
+            if (storedFields != null && storedFields.length > 0) {
+                return putParam("stored_fields", String.join(",", storedFields));
+            }
+            return this;
+        }
+
+        Params withTimeout(TimeValue timeout) {
+            return putParam("timeout", timeout);
+        }
+
+        Params withVersion(long version) {
+            if (version != Versions.MATCH_ANY) {
+                return putParam("version", Long.toString(version));
+            }
+            return this;
+        }
+
+        Params withVersionType(VersionType versionType) {
+            if (versionType != VersionType.INTERNAL) {
+                return putParam("version_type", versionType.name().toLowerCase(Locale.ROOT));
+            }
+            return this;
+        }
+
+        Params withWaitForActiveShards(ActiveShardCount activeShardCount) {
+            if (activeShardCount != null && activeShardCount != ActiveShardCount.DEFAULT) {
+                return putParam("wait_for_active_shards", activeShardCount.toString().toLowerCase(Locale.ROOT));
+            }
+            return this;
+        }
+
+        Params withIndicesOptions(IndicesOptions indicesOptions) {
+            putParam("ignore_unavailable", Boolean.toString(indicesOptions.ignoreUnavailable()));
+            putParam("allow_no_indices", Boolean.toString(indicesOptions.allowNoIndices()));
+            String expandWildcards;
+            if (indicesOptions.expandWildcardsOpen() == false && indicesOptions.expandWildcardsClosed() == false) {
+                expandWildcards = "none";
+            } else {
+                StringJoiner joiner  = new StringJoiner(",");
+                if (indicesOptions.expandWildcardsOpen()) {
+                    joiner.add("open");
+                }
+                if (indicesOptions.expandWildcardsClosed()) {
+                    joiner.add("closed");
+                }
+                expandWildcards = joiner.toString();
+            }
+            putParam("expand_wildcards", expandWildcards);
+            return this;
+        }
+
+        Map<String, String> getParams() {
+            return Collections.unmodifiableMap(params);
+        }
+
+        static Params builder() {
+            return new Params();
+        }
+    }
+
+    /**
+     * Ensure that the {@link IndexRequest}'s content type is supported by the Bulk API and that it conforms
+     * to the current {@link BulkRequest}'s content type (if it's known at the time of this method get called).
+     *
+     * @return the {@link IndexRequest}'s content type
+     */
+    static XContentType enforceSameContentType(IndexRequest indexRequest, @Nullable XContentType xContentType) {
+        XContentType requestContentType = indexRequest.getContentType();
+        if (requestContentType != XContentType.JSON && requestContentType != XContentType.SMILE) {
+            throw new IllegalArgumentException("Unsupported content-type found for request with content-type [" + requestContentType
+                    + "], only JSON and SMILE are supported");
+        }
+        if (xContentType == null) {
+            return requestContentType;
+        }
+        if (requestContentType != xContentType) {
+            throw new IllegalArgumentException("Mismatching content-type found for request with content-type [" + requestContentType
+                    + "], previous requests have content-type [" + xContentType + "]");
+        }
+        return xContentType;
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java
new file mode 100644
index 0000000000000..59684b18508ee
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java
@@ -0,0 +1,601 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import org.apache.http.Header;
+import org.apache.http.HttpEntity;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.bulk.BulkRequest;
+import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.delete.DeleteResponse;
+import org.elasticsearch.action.get.GetRequest;
+import org.elasticsearch.action.get.GetResponse;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.action.main.MainRequest;
+import org.elasticsearch.action.main.MainResponse;
+import org.elasticsearch.action.search.ClearScrollRequest;
+import org.elasticsearch.action.search.ClearScrollResponse;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.SearchScrollRequest;
+import org.elasticsearch.action.update.UpdateRequest;
+import org.elasticsearch.action.update.UpdateResponse;
+import org.elasticsearch.common.CheckedFunction;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ContextParser;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.plugins.spi.NamedXContentProvider;
+import org.elasticsearch.rest.BytesRestResponse;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.search.aggregations.Aggregation;
+import org.elasticsearch.search.aggregations.bucket.adjacency.AdjacencyMatrixAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.adjacency.ParsedAdjacencyMatrix;
+import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.filter.ParsedFilter;
+import org.elasticsearch.search.aggregations.bucket.filters.FiltersAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.filters.ParsedFilters;
+import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.geogrid.ParsedGeoHashGrid;
+import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.global.ParsedGlobal;
+import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.histogram.ParsedDateHistogram;
+import org.elasticsearch.search.aggregations.bucket.histogram.ParsedHistogram;
+import org.elasticsearch.search.aggregations.bucket.missing.MissingAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.missing.ParsedMissing;
+import org.elasticsearch.search.aggregations.bucket.nested.NestedAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.nested.ParsedNested;
+import org.elasticsearch.search.aggregations.bucket.nested.ParsedReverseNested;
+import org.elasticsearch.search.aggregations.bucket.nested.ReverseNestedAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.range.ip.IpRangeAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.range.ParsedBinaryRange;
+import org.elasticsearch.search.aggregations.bucket.range.ParsedRange;
+import org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.range.date.DateRangeAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.range.date.ParsedDateRange;
+import org.elasticsearch.search.aggregations.bucket.range.geodistance.GeoDistanceAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.range.geodistance.ParsedGeoDistance;
+import org.elasticsearch.search.aggregations.bucket.sampler.InternalSampler;
+import org.elasticsearch.search.aggregations.bucket.sampler.ParsedSampler;
+import org.elasticsearch.search.aggregations.bucket.significant.ParsedSignificantLongTerms;
+import org.elasticsearch.search.aggregations.bucket.significant.ParsedSignificantStringTerms;
+import org.elasticsearch.search.aggregations.bucket.significant.SignificantLongTerms;
+import org.elasticsearch.search.aggregations.bucket.significant.SignificantStringTerms;
+import org.elasticsearch.search.aggregations.bucket.terms.DoubleTerms;
+import org.elasticsearch.search.aggregations.bucket.terms.LongTerms;
+import org.elasticsearch.search.aggregations.bucket.terms.ParsedDoubleTerms;
+import org.elasticsearch.search.aggregations.bucket.terms.ParsedLongTerms;
+import org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms;
+import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
+import org.elasticsearch.search.aggregations.metrics.avg.AvgAggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.avg.ParsedAvg;
+import org.elasticsearch.search.aggregations.metrics.cardinality.CardinalityAggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.cardinality.ParsedCardinality;
+import org.elasticsearch.search.aggregations.metrics.geobounds.GeoBoundsAggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.geobounds.ParsedGeoBounds;
+import org.elasticsearch.search.aggregations.metrics.geocentroid.GeoCentroidAggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.geocentroid.ParsedGeoCentroid;
+import org.elasticsearch.search.aggregations.metrics.max.MaxAggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.max.ParsedMax;
+import org.elasticsearch.search.aggregations.metrics.min.MinAggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.min.ParsedMin;
+import org.elasticsearch.search.aggregations.metrics.percentiles.hdr.InternalHDRPercentileRanks;
+import org.elasticsearch.search.aggregations.metrics.percentiles.hdr.InternalHDRPercentiles;
+import org.elasticsearch.search.aggregations.metrics.percentiles.hdr.ParsedHDRPercentileRanks;
+import org.elasticsearch.search.aggregations.metrics.percentiles.hdr.ParsedHDRPercentiles;
+import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.InternalTDigestPercentileRanks;
+import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.InternalTDigestPercentiles;
+import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.ParsedTDigestPercentileRanks;
+import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.ParsedTDigestPercentiles;
+import org.elasticsearch.search.aggregations.metrics.scripted.ParsedScriptedMetric;
+import org.elasticsearch.search.aggregations.metrics.scripted.ScriptedMetricAggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.stats.ParsedStats;
+import org.elasticsearch.search.aggregations.metrics.stats.StatsAggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStatsAggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.stats.extended.ParsedExtendedStats;
+import org.elasticsearch.search.aggregations.metrics.sum.ParsedSum;
+import org.elasticsearch.search.aggregations.metrics.sum.SumAggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.tophits.ParsedTopHits;
+import org.elasticsearch.search.aggregations.metrics.tophits.TopHitsAggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.valuecount.ParsedValueCount;
+import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCountAggregationBuilder;
+import org.elasticsearch.search.aggregations.pipeline.InternalSimpleValue;
+import org.elasticsearch.search.aggregations.pipeline.ParsedSimpleValue;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.InternalBucketMetricValue;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.ParsedBucketMetricValue;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.percentile.ParsedPercentilesBucket;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.percentile.PercentilesBucketPipelineAggregationBuilder;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.ParsedStatsBucket;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.StatsBucketPipelineAggregationBuilder;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.extended.ExtendedStatsBucketPipelineAggregationBuilder;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.extended.ParsedExtendedStatsBucket;
+import org.elasticsearch.search.aggregations.pipeline.derivative.DerivativePipelineAggregationBuilder;
+import org.elasticsearch.search.aggregations.pipeline.derivative.ParsedDerivative;
+import org.elasticsearch.search.suggest.Suggest;
+import org.elasticsearch.search.suggest.completion.CompletionSuggestion;
+import org.elasticsearch.search.suggest.phrase.PhraseSuggestion;
+import org.elasticsearch.search.suggest.term.TermSuggestion;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.ServiceLoader;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static java.util.Collections.emptySet;
+import static java.util.Collections.singleton;
+import static java.util.stream.Collectors.toList;
+
+/**
+ * High level REST client that wraps an instance of the low level {@link RestClient} and allows to build requests and read responses.
+ * The provided {@link RestClient} is externally built and closed.
+ * Can be sub-classed to expose additional client methods that make use of endpoints added to Elasticsearch through plugins, or to
+ * add support for custom response sections, again added to Elasticsearch through plugins.
+ */
+public class RestHighLevelClient {
+
+    private final RestClient client;
+    private final NamedXContentRegistry registry;
+
+    /**
+     * Creates a {@link RestHighLevelClient} given the low level {@link RestClient} that it should use to perform requests.
+     */
+    public RestHighLevelClient(RestClient restClient) {
+        this(restClient, Collections.emptyList());
+    }
+
+    /**
+     * Creates a {@link RestHighLevelClient} given the low level {@link RestClient} that it should use to perform requests and
+     * a list of entries that allow to parse custom response sections added to Elasticsearch through plugins.
+     */
+    protected RestHighLevelClient(RestClient restClient, List<NamedXContentRegistry.Entry> namedXContentEntries) {
+        this.client = Objects.requireNonNull(restClient);
+        this.registry = new NamedXContentRegistry(
+                Stream.of(getDefaultNamedXContents().stream(), getProvidedNamedXContents().stream(), namedXContentEntries.stream())
+                    .flatMap(Function.identity()).collect(toList()));
+    }
+
+    /**
+     * Executes a bulk request using the Bulk API
+     *
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html">Bulk API on elastic.co</a>
+     */
+    public BulkResponse bulk(BulkRequest bulkRequest, Header... headers) throws IOException {
+        return performRequestAndParseEntity(bulkRequest, Request::bulk, BulkResponse::fromXContent, emptySet(), headers);
+    }
+
+    /**
+     * Asynchronously executes a bulk request using the Bulk API
+     *
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html">Bulk API on elastic.co</a>
+     */
+    public void bulkAsync(BulkRequest bulkRequest, ActionListener<BulkResponse> listener, Header... headers) {
+        performRequestAsyncAndParseEntity(bulkRequest, Request::bulk, BulkResponse::fromXContent, listener, emptySet(), headers);
+    }
+
+    /**
+     * Pings the remote Elasticsearch cluster and returns true if the ping succeeded, false otherwise
+     */
+    public boolean ping(Header... headers) throws IOException {
+        return performRequest(new MainRequest(), (request) -> Request.ping(), RestHighLevelClient::convertExistsResponse,
+                emptySet(), headers);
+    }
+
+    /**
+     * Get the cluster info otherwise provided when sending an HTTP request to port 9200
+     */
+    public MainResponse info(Header... headers) throws IOException {
+        return performRequestAndParseEntity(new MainRequest(), (request) -> Request.info(), MainResponse::fromXContent, emptySet(),
+                headers);
+    }
+
+    /**
+     * Retrieves a document by id using the Get API
+     *
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-get.html">Get API on elastic.co</a>
+     */
+    public GetResponse get(GetRequest getRequest, Header... headers) throws IOException {
+        return performRequestAndParseEntity(getRequest, Request::get, GetResponse::fromXContent, singleton(404), headers);
+    }
+
+    /**
+     * Asynchronously retrieves a document by id using the Get API
+     *
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-get.html">Get API on elastic.co</a>
+     */
+    public void getAsync(GetRequest getRequest, ActionListener<GetResponse> listener, Header... headers) {
+        performRequestAsyncAndParseEntity(getRequest, Request::get, GetResponse::fromXContent, listener, singleton(404), headers);
+    }
+
+    /**
+     * Checks for the existence of a document. Returns true if it exists, false otherwise
+     *
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-get.html">Get API on elastic.co</a>
+     */
+    public boolean exists(GetRequest getRequest, Header... headers) throws IOException {
+        return performRequest(getRequest, Request::exists, RestHighLevelClient::convertExistsResponse, emptySet(), headers);
+    }
+
+    /**
+     * Asynchronously checks for the existence of a document. Returns true if it exists, false otherwise
+     *
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-get.html">Get API on elastic.co</a>
+     */
+    public void existsAsync(GetRequest getRequest, ActionListener<Boolean> listener, Header... headers) {
+        performRequestAsync(getRequest, Request::exists, RestHighLevelClient::convertExistsResponse, listener, emptySet(), headers);
+    }
+
+    /**
+     * Index a document using the Index API
+     *
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html">Index API on elastic.co</a>
+     */
+    public IndexResponse index(IndexRequest indexRequest, Header... headers) throws IOException {
+        return performRequestAndParseEntity(indexRequest, Request::index, IndexResponse::fromXContent, emptySet(), headers);
+    }
+
+    /**
+     * Asynchronously index a document using the Index API
+     *
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html">Index API on elastic.co</a>
+     */
+    public void indexAsync(IndexRequest indexRequest, ActionListener<IndexResponse> listener, Header... headers) {
+        performRequestAsyncAndParseEntity(indexRequest, Request::index, IndexResponse::fromXContent, listener, emptySet(), headers);
+    }
+
+    /**
+     * Updates a document using the Update API
+     * <p>
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-update.html">Update API on elastic.co</a>
+     */
+    public UpdateResponse update(UpdateRequest updateRequest, Header... headers) throws IOException {
+        return performRequestAndParseEntity(updateRequest, Request::update, UpdateResponse::fromXContent, emptySet(), headers);
+    }
+
+    /**
+     * Asynchronously updates a document using the Update API
+     * <p>
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-update.html">Update API on elastic.co</a>
+     */
+    public void updateAsync(UpdateRequest updateRequest, ActionListener<UpdateResponse> listener, Header... headers) {
+        performRequestAsyncAndParseEntity(updateRequest, Request::update, UpdateResponse::fromXContent, listener, emptySet(), headers);
+    }
+
+    /**
+     * Deletes a document by id using the Delete api
+     *
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-delete.html">Delete API on elastic.co</a>
+     */
+    public DeleteResponse delete(DeleteRequest deleteRequest, Header... headers) throws IOException {
+        return performRequestAndParseEntity(deleteRequest, Request::delete, DeleteResponse::fromXContent, Collections.singleton(404),
+            headers);
+    }
+
+    /**
+     * Asynchronously deletes a document by id using the Delete api
+     *
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-delete.html">Delete API on elastic.co</a>
+     */
+    public void deleteAsync(DeleteRequest deleteRequest, ActionListener<DeleteResponse> listener, Header... headers) {
+        performRequestAsyncAndParseEntity(deleteRequest, Request::delete, DeleteResponse::fromXContent, listener,
+            Collections.singleton(404), headers);
+    }
+
+    /**
+     * Executes a search using the Search api
+     *
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/search-search.html">Search API on elastic.co</a>
+     */
+    public SearchResponse search(SearchRequest searchRequest, Header... headers) throws IOException {
+        return performRequestAndParseEntity(searchRequest, Request::search, SearchResponse::fromXContent, emptySet(), headers);
+    }
+
+    /**
+     * Asynchronously executes a search using the Search api
+     *
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/search-search.html">Search API on elastic.co</a>
+     */
+    public void searchAsync(SearchRequest searchRequest, ActionListener<SearchResponse> listener, Header... headers) {
+        performRequestAsyncAndParseEntity(searchRequest, Request::search, SearchResponse::fromXContent, listener, emptySet(), headers);
+    }
+
+    /**
+     * Executes a search using the Search Scroll api
+     *
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-scroll.html">Search Scroll
+     * API on elastic.co</a>
+     */
+    public SearchResponse searchScroll(SearchScrollRequest searchScrollRequest, Header... headers) throws IOException {
+        return performRequestAndParseEntity(searchScrollRequest, Request::searchScroll, SearchResponse::fromXContent, emptySet(), headers);
+    }
+
+    /**
+     * Asynchronously executes a search using the Search Scroll api
+     *
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-scroll.html">Search Scroll
+     * API on elastic.co</a>
+     */
+    public void searchScrollAsync(SearchScrollRequest searchScrollRequest, ActionListener<SearchResponse> listener, Header... headers) {
+        performRequestAsyncAndParseEntity(searchScrollRequest, Request::searchScroll, SearchResponse::fromXContent,
+                listener, emptySet(), headers);
+    }
+
+    /**
+     * Clears one or more scroll ids using the Clear Scroll api
+     *
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-scroll.html#_clear_scroll_api">
+     * Clear Scroll API on elastic.co</a>
+     */
+    public ClearScrollResponse clearScroll(ClearScrollRequest clearScrollRequest, Header... headers) throws IOException {
+        return performRequestAndParseEntity(clearScrollRequest, Request::clearScroll, ClearScrollResponse::fromXContent,
+                emptySet(), headers);
+    }
+
+    /**
+     * Asynchronously clears one or more scroll ids using the Clear Scroll api
+     *
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-scroll.html#_clear_scroll_api">
+     * Clear Scroll API on elastic.co</a>
+     */
+    public void clearScrollAsync(ClearScrollRequest clearScrollRequest, ActionListener<ClearScrollResponse> listener, Header... headers) {
+        performRequestAsyncAndParseEntity(clearScrollRequest, Request::clearScroll, ClearScrollResponse::fromXContent,
+                listener, emptySet(), headers);
+    }
+
+    protected <Req extends ActionRequest, Resp> Resp performRequestAndParseEntity(Req request,
+                                                                            CheckedFunction<Req, Request, IOException> requestConverter,
+                                                                            CheckedFunction<XContentParser, Resp, IOException> entityParser,
+                                                                            Set<Integer> ignores, Header... headers) throws IOException {
+        return performRequest(request, requestConverter, (response) -> parseEntity(response.getEntity(), entityParser), ignores, headers);
+    }
+
+    protected <Req extends ActionRequest, Resp> Resp performRequest(Req request,
+                                                          CheckedFunction<Req, Request, IOException> requestConverter,
+                                                          CheckedFunction<Response, Resp, IOException> responseConverter,
+                                                          Set<Integer> ignores, Header... headers) throws IOException {
+        ActionRequestValidationException validationException = request.validate();
+        if (validationException != null) {
+            throw validationException;
+        }
+        Request req = requestConverter.apply(request);
+        Response response;
+        try {
+            response = client.performRequest(req.getMethod(), req.getEndpoint(), req.getParameters(), req.getEntity(), headers);
+        } catch (ResponseException e) {
+            if (ignores.contains(e.getResponse().getStatusLine().getStatusCode())) {
+                try {
+                    return responseConverter.apply(e.getResponse());
+                } catch (Exception innerException) {
+                    //the exception is ignored as we now try to parse the response as an error.
+                    //this covers cases like get where 404 can either be a valid document not found response,
+                    //or an error for which parsing is completely different. We try to consider the 404 response as a valid one
+                    //first. If parsing of the response breaks, we fall back to parsing it as an error.
+                    throw parseResponseException(e);
+                }
+            }
+            throw parseResponseException(e);
+        }
+
+        try {
+            return responseConverter.apply(response);
+        } catch(Exception e) {
+            throw new IOException("Unable to parse response body for " + response, e);
+        }
+    }
+
+    protected <Req extends ActionRequest, Resp> void performRequestAsyncAndParseEntity(Req request,
+                                                                 CheckedFunction<Req, Request, IOException> requestConverter,
+                                                                 CheckedFunction<XContentParser, Resp, IOException> entityParser,
+                                                                 ActionListener<Resp> listener, Set<Integer> ignores, Header... headers) {
+        performRequestAsync(request, requestConverter, (response) -> parseEntity(response.getEntity(), entityParser),
+                listener, ignores, headers);
+    }
+
+    protected <Req extends ActionRequest, Resp> void performRequestAsync(Req request,
+                                                               CheckedFunction<Req, Request, IOException> requestConverter,
+                                                               CheckedFunction<Response, Resp, IOException> responseConverter,
+                                                               ActionListener<Resp> listener, Set<Integer> ignores, Header... headers) {
+        ActionRequestValidationException validationException = request.validate();
+        if (validationException != null) {
+            listener.onFailure(validationException);
+            return;
+        }
+        Request req;
+        try {
+            req = requestConverter.apply(request);
+        } catch (Exception e) {
+            listener.onFailure(e);
+            return;
+        }
+
+        ResponseListener responseListener = wrapResponseListener(responseConverter, listener, ignores);
+        client.performRequestAsync(req.getMethod(), req.getEndpoint(), req.getParameters(), req.getEntity(), responseListener, headers);
+    }
+
+    <Resp> ResponseListener wrapResponseListener(CheckedFunction<Response, Resp, IOException> responseConverter,
+                                                        ActionListener<Resp> actionListener, Set<Integer> ignores) {
+        return new ResponseListener() {
+            @Override
+            public void onSuccess(Response response) {
+                try {
+                    actionListener.onResponse(responseConverter.apply(response));
+                } catch(Exception e) {
+                    IOException ioe = new IOException("Unable to parse response body for " + response, e);
+                    onFailure(ioe);
+                }
+            }
+
+            @Override
+            public void onFailure(Exception exception) {
+                if (exception instanceof ResponseException) {
+                    ResponseException responseException = (ResponseException) exception;
+                    Response response = responseException.getResponse();
+                    if (ignores.contains(response.getStatusLine().getStatusCode())) {
+                        try {
+                            actionListener.onResponse(responseConverter.apply(response));
+                        } catch (Exception innerException) {
+                            //the exception is ignored as we now try to parse the response as an error.
+                            //this covers cases like get where 404 can either be a valid document not found response,
+                            //or an error for which parsing is completely different. We try to consider the 404 response as a valid one
+                            //first. If parsing of the response breaks, we fall back to parsing it as an error.
+                            actionListener.onFailure(parseResponseException(responseException));
+                        }
+                    } else {
+                        actionListener.onFailure(parseResponseException(responseException));
+                    }
+                } else {
+                    actionListener.onFailure(exception);
+                }
+            }
+        };
+    }
+
+    /**
+     * Converts a {@link ResponseException} obtained from the low level REST client into an {@link ElasticsearchException}.
+     * If a response body was returned, tries to parse it as an error returned from Elasticsearch.
+     * If no response body was returned or anything goes wrong while parsing the error, returns a new {@link ElasticsearchStatusException}
+     * that wraps the original {@link ResponseException}. The potential exception obtained while parsing is added to the returned
+     * exception as a suppressed exception. This method is guaranteed to not throw any exception eventually thrown while parsing.
+     */
+    protected ElasticsearchStatusException parseResponseException(ResponseException responseException) {
+        Response response = responseException.getResponse();
+        HttpEntity entity = response.getEntity();
+        ElasticsearchStatusException elasticsearchException;
+        if (entity == null) {
+            elasticsearchException = new ElasticsearchStatusException(
+                    responseException.getMessage(), RestStatus.fromCode(response.getStatusLine().getStatusCode()), responseException);
+        } else {
+            try {
+                elasticsearchException = parseEntity(entity, BytesRestResponse::errorFromXContent);
+                elasticsearchException.addSuppressed(responseException);
+            } catch (Exception e) {
+                RestStatus restStatus = RestStatus.fromCode(response.getStatusLine().getStatusCode());
+                elasticsearchException = new ElasticsearchStatusException("Unable to parse response body", restStatus, responseException);
+                elasticsearchException.addSuppressed(e);
+            }
+        }
+        return elasticsearchException;
+    }
+
+    protected <Resp> Resp parseEntity(final HttpEntity entity,
+                                      final CheckedFunction<XContentParser, Resp, IOException> entityParser) throws IOException {
+        if (entity == null) {
+            throw new IllegalStateException("Response body expected but not returned");
+        }
+        if (entity.getContentType() == null) {
+            throw new IllegalStateException("Elasticsearch didn't return the [Content-Type] header, unable to parse response body");
+        }
+        XContentType xContentType = XContentType.fromMediaTypeOrFormat(entity.getContentType().getValue());
+        if (xContentType == null) {
+            throw new IllegalStateException("Unsupported Content-Type: " + entity.getContentType().getValue());
+        }
+        try (XContentParser parser = xContentType.xContent().createParser(registry, entity.getContent())) {
+            return entityParser.apply(parser);
+        }
+    }
+
+    static boolean convertExistsResponse(Response response) {
+        return response.getStatusLine().getStatusCode() == 200;
+    }
+
+    static List<NamedXContentRegistry.Entry> getDefaultNamedXContents() {
+        Map<String, ContextParser<Object, ? extends Aggregation>> map = new HashMap<>();
+        map.put(CardinalityAggregationBuilder.NAME, (p, c) -> ParsedCardinality.fromXContent(p, (String) c));
+        map.put(InternalHDRPercentiles.NAME, (p, c) -> ParsedHDRPercentiles.fromXContent(p, (String) c));
+        map.put(InternalHDRPercentileRanks.NAME, (p, c) -> ParsedHDRPercentileRanks.fromXContent(p, (String) c));
+        map.put(InternalTDigestPercentiles.NAME, (p, c) -> ParsedTDigestPercentiles.fromXContent(p, (String) c));
+        map.put(InternalTDigestPercentileRanks.NAME, (p, c) -> ParsedTDigestPercentileRanks.fromXContent(p, (String) c));
+        map.put(PercentilesBucketPipelineAggregationBuilder.NAME, (p, c) -> ParsedPercentilesBucket.fromXContent(p, (String) c));
+        map.put(MinAggregationBuilder.NAME, (p, c) -> ParsedMin.fromXContent(p, (String) c));
+        map.put(MaxAggregationBuilder.NAME, (p, c) -> ParsedMax.fromXContent(p, (String) c));
+        map.put(SumAggregationBuilder.NAME, (p, c) -> ParsedSum.fromXContent(p, (String) c));
+        map.put(AvgAggregationBuilder.NAME, (p, c) -> ParsedAvg.fromXContent(p, (String) c));
+        map.put(ValueCountAggregationBuilder.NAME, (p, c) -> ParsedValueCount.fromXContent(p, (String) c));
+        map.put(InternalSimpleValue.NAME, (p, c) -> ParsedSimpleValue.fromXContent(p, (String) c));
+        map.put(DerivativePipelineAggregationBuilder.NAME, (p, c) -> ParsedDerivative.fromXContent(p, (String) c));
+        map.put(InternalBucketMetricValue.NAME, (p, c) -> ParsedBucketMetricValue.fromXContent(p, (String) c));
+        map.put(StatsAggregationBuilder.NAME, (p, c) -> ParsedStats.fromXContent(p, (String) c));
+        map.put(StatsBucketPipelineAggregationBuilder.NAME, (p, c) -> ParsedStatsBucket.fromXContent(p, (String) c));
+        map.put(ExtendedStatsAggregationBuilder.NAME, (p, c) -> ParsedExtendedStats.fromXContent(p, (String) c));
+        map.put(ExtendedStatsBucketPipelineAggregationBuilder.NAME,
+                (p, c) -> ParsedExtendedStatsBucket.fromXContent(p, (String) c));
+        map.put(GeoBoundsAggregationBuilder.NAME, (p, c) -> ParsedGeoBounds.fromXContent(p, (String) c));
+        map.put(GeoCentroidAggregationBuilder.NAME, (p, c) -> ParsedGeoCentroid.fromXContent(p, (String) c));
+        map.put(HistogramAggregationBuilder.NAME, (p, c) -> ParsedHistogram.fromXContent(p, (String) c));
+        map.put(DateHistogramAggregationBuilder.NAME, (p, c) -> ParsedDateHistogram.fromXContent(p, (String) c));
+        map.put(StringTerms.NAME, (p, c) -> ParsedStringTerms.fromXContent(p, (String) c));
+        map.put(LongTerms.NAME, (p, c) -> ParsedLongTerms.fromXContent(p, (String) c));
+        map.put(DoubleTerms.NAME, (p, c) -> ParsedDoubleTerms.fromXContent(p, (String) c));
+        map.put(MissingAggregationBuilder.NAME, (p, c) -> ParsedMissing.fromXContent(p, (String) c));
+        map.put(NestedAggregationBuilder.NAME, (p, c) -> ParsedNested.fromXContent(p, (String) c));
+        map.put(ReverseNestedAggregationBuilder.NAME, (p, c) -> ParsedReverseNested.fromXContent(p, (String) c));
+        map.put(GlobalAggregationBuilder.NAME, (p, c) -> ParsedGlobal.fromXContent(p, (String) c));
+        map.put(FilterAggregationBuilder.NAME, (p, c) -> ParsedFilter.fromXContent(p, (String) c));
+        map.put(InternalSampler.PARSER_NAME, (p, c) -> ParsedSampler.fromXContent(p, (String) c));
+        map.put(GeoGridAggregationBuilder.NAME, (p, c) -> ParsedGeoHashGrid.fromXContent(p, (String) c));
+        map.put(RangeAggregationBuilder.NAME, (p, c) -> ParsedRange.fromXContent(p, (String) c));
+        map.put(DateRangeAggregationBuilder.NAME, (p, c) -> ParsedDateRange.fromXContent(p, (String) c));
+        map.put(GeoDistanceAggregationBuilder.NAME, (p, c) -> ParsedGeoDistance.fromXContent(p, (String) c));
+        map.put(FiltersAggregationBuilder.NAME, (p, c) -> ParsedFilters.fromXContent(p, (String) c));
+        map.put(AdjacencyMatrixAggregationBuilder.NAME, (p, c) -> ParsedAdjacencyMatrix.fromXContent(p, (String) c));
+        map.put(SignificantLongTerms.NAME, (p, c) -> ParsedSignificantLongTerms.fromXContent(p, (String) c));
+        map.put(SignificantStringTerms.NAME, (p, c) -> ParsedSignificantStringTerms.fromXContent(p, (String) c));
+        map.put(ScriptedMetricAggregationBuilder.NAME, (p, c) -> ParsedScriptedMetric.fromXContent(p, (String) c));
+        map.put(IpRangeAggregationBuilder.NAME, (p, c) -> ParsedBinaryRange.fromXContent(p, (String) c));
+        map.put(TopHitsAggregationBuilder.NAME, (p, c) -> ParsedTopHits.fromXContent(p, (String) c));
+        List<NamedXContentRegistry.Entry> entries = map.entrySet().stream()
+                .map(entry -> new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(entry.getKey()), entry.getValue()))
+                .collect(Collectors.toList());
+        entries.add(new NamedXContentRegistry.Entry(Suggest.Suggestion.class, new ParseField(TermSuggestion.NAME),
+                (parser, context) -> TermSuggestion.fromXContent(parser, (String)context)));
+        entries.add(new NamedXContentRegistry.Entry(Suggest.Suggestion.class, new ParseField(PhraseSuggestion.NAME),
+                (parser, context) -> PhraseSuggestion.fromXContent(parser, (String)context)));
+        entries.add(new NamedXContentRegistry.Entry(Suggest.Suggestion.class, new ParseField(CompletionSuggestion.NAME),
+                (parser, context) -> CompletionSuggestion.fromXContent(parser, (String)context)));
+        return entries;
+    }
+
+    /**
+     * Loads and returns the {@link NamedXContentRegistry.Entry} parsers provided by plugins.
+     */
+    static List<NamedXContentRegistry.Entry> getProvidedNamedXContents() {
+        List<NamedXContentRegistry.Entry> entries = new ArrayList<>();
+        for (NamedXContentProvider service : ServiceLoader.load(NamedXContentProvider.class)) {
+            entries.addAll(service.getNamedXContentParsers());
+        }
+        return entries;
+    }
+}
diff --git a/client/rest-high-level/src/main/resources/forbidden/rest-high-level-signatures.txt b/client/rest-high-level/src/main/resources/forbidden/rest-high-level-signatures.txt
new file mode 100644
index 0000000000000..fb2330f3f083c
--- /dev/null
+++ b/client/rest-high-level/src/main/resources/forbidden/rest-high-level-signatures.txt
@@ -0,0 +1,21 @@
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance  with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on
+# an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+# either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+@defaultMessage Use Request#createContentType(XContentType) to be sure to pass the right MIME type
+org.apache.http.entity.ContentType#create(java.lang.String)
+org.apache.http.entity.ContentType#create(java.lang.String,java.lang.String)
+org.apache.http.entity.ContentType#create(java.lang.String,java.nio.charset.Charset)
+org.apache.http.entity.ContentType#create(java.lang.String,org.apache.http.NameValuePair[])
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java
new file mode 100644
index 0000000000000..a28686b27aa0d
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java
@@ -0,0 +1,704 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.StringEntity;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.action.bulk.BulkItemResponse;
+import org.elasticsearch.action.bulk.BulkProcessor;
+import org.elasticsearch.action.bulk.BulkRequest;
+import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.delete.DeleteResponse;
+import org.elasticsearch.action.get.GetRequest;
+import org.elasticsearch.action.get.GetResponse;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.action.update.UpdateRequest;
+import org.elasticsearch.action.update.UpdateResponse;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.VersionType;
+import org.elasticsearch.index.get.GetResult;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptType;
+import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
+import org.elasticsearch.threadpool.ThreadPool;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+
+import java.util.concurrent.atomic.AtomicReference;
+
+import static java.util.Collections.singletonMap;
+
+public class CrudIT extends ESRestHighLevelClientTestCase {
+
+    public void testDelete() throws IOException {
+        {
+            // Testing deletion
+            String docId = "id";
+            highLevelClient().index(new IndexRequest("index", "type", docId).source(Collections.singletonMap("foo", "bar")));
+            DeleteRequest deleteRequest = new DeleteRequest("index", "type", docId);
+            if (randomBoolean()) {
+                deleteRequest.version(1L);
+            }
+            DeleteResponse deleteResponse = execute(deleteRequest, highLevelClient()::delete, highLevelClient()::deleteAsync);
+            assertEquals("index", deleteResponse.getIndex());
+            assertEquals("type", deleteResponse.getType());
+            assertEquals(docId, deleteResponse.getId());
+            assertEquals(DocWriteResponse.Result.DELETED, deleteResponse.getResult());
+        }
+        {
+            // Testing non existing document
+            String docId = "does_not_exist";
+            DeleteRequest deleteRequest = new DeleteRequest("index", "type", docId);
+            DeleteResponse deleteResponse = execute(deleteRequest, highLevelClient()::delete, highLevelClient()::deleteAsync);
+            assertEquals("index", deleteResponse.getIndex());
+            assertEquals("type", deleteResponse.getType());
+            assertEquals(docId, deleteResponse.getId());
+            assertEquals(DocWriteResponse.Result.NOT_FOUND, deleteResponse.getResult());
+        }
+        {
+            // Testing version conflict
+            String docId = "version_conflict";
+            highLevelClient().index(new IndexRequest("index", "type", docId).source(Collections.singletonMap("foo", "bar")));
+            DeleteRequest deleteRequest = new DeleteRequest("index", "type", docId).version(2);
+            ElasticsearchException exception = expectThrows(ElasticsearchException.class,
+                () -> execute(deleteRequest, highLevelClient()::delete, highLevelClient()::deleteAsync));
+            assertEquals(RestStatus.CONFLICT, exception.status());
+            assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, reason=[type][" + docId + "]: " +
+                "version conflict, current version [1] is different than the one provided [2]]", exception.getMessage());
+            assertEquals("index", exception.getMetadata("es.index").get(0));
+        }
+        {
+            // Testing version type
+            String docId = "version_type";
+            highLevelClient().index(new IndexRequest("index", "type", docId).source(Collections.singletonMap("foo", "bar"))
+                .versionType(VersionType.EXTERNAL).version(12));
+            DeleteRequest deleteRequest = new DeleteRequest("index", "type", docId).versionType(VersionType.EXTERNAL).version(13);
+            DeleteResponse deleteResponse = execute(deleteRequest, highLevelClient()::delete, highLevelClient()::deleteAsync);
+            assertEquals("index", deleteResponse.getIndex());
+            assertEquals("type", deleteResponse.getType());
+            assertEquals(docId, deleteResponse.getId());
+            assertEquals(DocWriteResponse.Result.DELETED, deleteResponse.getResult());
+        }
+        {
+            // Testing version type with a wrong version
+            String docId = "wrong_version";
+            highLevelClient().index(new IndexRequest("index", "type", docId).source(Collections.singletonMap("foo", "bar"))
+                .versionType(VersionType.EXTERNAL).version(12));
+            ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class, () -> {
+                DeleteRequest deleteRequest = new DeleteRequest("index", "type", docId).versionType(VersionType.EXTERNAL).version(10);
+                execute(deleteRequest, highLevelClient()::delete, highLevelClient()::deleteAsync);
+            });
+            assertEquals(RestStatus.CONFLICT, exception.status());
+            assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, reason=[type][" +
+                docId + "]: version conflict, current version [12] is higher or equal to the one provided [10]]", exception.getMessage());
+            assertEquals("index", exception.getMetadata("es.index").get(0));
+        }
+        {
+            // Testing routing
+            String docId = "routing";
+            highLevelClient().index(new IndexRequest("index", "type", docId).source(Collections.singletonMap("foo", "bar")).routing("foo"));
+            DeleteRequest deleteRequest = new DeleteRequest("index", "type", docId).routing("foo");
+            DeleteResponse deleteResponse = execute(deleteRequest, highLevelClient()::delete, highLevelClient()::deleteAsync);
+            assertEquals("index", deleteResponse.getIndex());
+            assertEquals("type", deleteResponse.getType());
+            assertEquals(docId, deleteResponse.getId());
+            assertEquals(DocWriteResponse.Result.DELETED, deleteResponse.getResult());
+        }
+    }
+
+    public void testExists() throws IOException {
+        {
+            GetRequest getRequest = new GetRequest("index", "type", "id");
+            assertFalse(execute(getRequest, highLevelClient()::exists, highLevelClient()::existsAsync));
+        }
+        String document = "{\"field1\":\"value1\",\"field2\":\"value2\"}";
+        StringEntity stringEntity = new StringEntity(document, ContentType.APPLICATION_JSON);
+        Response response = client().performRequest("PUT", "/index/type/id", Collections.singletonMap("refresh", "wait_for"), stringEntity);
+        assertEquals(201, response.getStatusLine().getStatusCode());
+        {
+            GetRequest getRequest = new GetRequest("index", "type", "id");
+            assertTrue(execute(getRequest, highLevelClient()::exists, highLevelClient()::existsAsync));
+        }
+        {
+            GetRequest getRequest = new GetRequest("index", "type", "does_not_exist");
+            assertFalse(execute(getRequest, highLevelClient()::exists, highLevelClient()::existsAsync));
+        }
+        {
+            GetRequest getRequest = new GetRequest("index", "type", "does_not_exist").version(1);
+            assertFalse(execute(getRequest, highLevelClient()::exists, highLevelClient()::existsAsync));
+        }
+    }
+
+    public void testGet() throws IOException {
+        {
+            GetRequest getRequest = new GetRequest("index", "type", "id");
+            ElasticsearchException exception = expectThrows(ElasticsearchException.class,
+                    () -> execute(getRequest, highLevelClient()::get, highLevelClient()::getAsync));
+            assertEquals(RestStatus.NOT_FOUND, exception.status());
+            assertEquals("Elasticsearch exception [type=index_not_found_exception, reason=no such index]", exception.getMessage());
+            assertEquals("index", exception.getMetadata("es.index").get(0));
+        }
+
+        String document = "{\"field1\":\"value1\",\"field2\":\"value2\"}";
+        StringEntity stringEntity = new StringEntity(document, ContentType.APPLICATION_JSON);
+        Response response = client().performRequest("PUT", "/index/type/id", Collections.singletonMap("refresh", "wait_for"), stringEntity);
+        assertEquals(201, response.getStatusLine().getStatusCode());
+        {
+            GetRequest getRequest = new GetRequest("index", "type", "id").version(2);
+            ElasticsearchException exception = expectThrows(ElasticsearchException.class,
+                    () -> execute(getRequest, highLevelClient()::get, highLevelClient()::getAsync));
+            assertEquals(RestStatus.CONFLICT, exception.status());
+            assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, " + "reason=[type][id]: " +
+                    "version conflict, current version [1] is different than the one provided [2]]", exception.getMessage());
+            assertEquals("index", exception.getMetadata("es.index").get(0));
+        }
+        {
+            GetRequest getRequest = new GetRequest("index", "type", "id");
+            if (randomBoolean()) {
+                getRequest.version(1L);
+            }
+            GetResponse getResponse = execute(getRequest, highLevelClient()::get, highLevelClient()::getAsync);
+            assertEquals("index", getResponse.getIndex());
+            assertEquals("type", getResponse.getType());
+            assertEquals("id", getResponse.getId());
+            assertTrue(getResponse.isExists());
+            assertFalse(getResponse.isSourceEmpty());
+            assertEquals(1L, getResponse.getVersion());
+            assertEquals(document, getResponse.getSourceAsString());
+        }
+        {
+            GetRequest getRequest = new GetRequest("index", "type", "does_not_exist");
+            GetResponse getResponse = execute(getRequest, highLevelClient()::get, highLevelClient()::getAsync);
+            assertEquals("index", getResponse.getIndex());
+            assertEquals("type", getResponse.getType());
+            assertEquals("does_not_exist", getResponse.getId());
+            assertFalse(getResponse.isExists());
+            assertEquals(-1, getResponse.getVersion());
+            assertTrue(getResponse.isSourceEmpty());
+            assertNull(getResponse.getSourceAsString());
+        }
+        {
+            GetRequest getRequest = new GetRequest("index", "type", "id");
+            getRequest.fetchSourceContext(new FetchSourceContext(false, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY));
+            GetResponse getResponse = execute(getRequest, highLevelClient()::get, highLevelClient()::getAsync);
+            assertEquals("index", getResponse.getIndex());
+            assertEquals("type", getResponse.getType());
+            assertEquals("id", getResponse.getId());
+            assertTrue(getResponse.isExists());
+            assertTrue(getResponse.isSourceEmpty());
+            assertEquals(1L, getResponse.getVersion());
+            assertNull(getResponse.getSourceAsString());
+        }
+        {
+            GetRequest getRequest = new GetRequest("index", "type", "id");
+            if (randomBoolean()) {
+                getRequest.fetchSourceContext(new FetchSourceContext(true, new String[]{"field1"}, Strings.EMPTY_ARRAY));
+            } else {
+                getRequest.fetchSourceContext(new FetchSourceContext(true, Strings.EMPTY_ARRAY, new String[]{"field2"}));
+            }
+            GetResponse getResponse = execute(getRequest, highLevelClient()::get, highLevelClient()::getAsync);
+            assertEquals("index", getResponse.getIndex());
+            assertEquals("type", getResponse.getType());
+            assertEquals("id", getResponse.getId());
+            assertTrue(getResponse.isExists());
+            assertFalse(getResponse.isSourceEmpty());
+            assertEquals(1L, getResponse.getVersion());
+            Map<String, Object> sourceAsMap = getResponse.getSourceAsMap();
+            assertEquals(1, sourceAsMap.size());
+            assertEquals("value1", sourceAsMap.get("field1"));
+        }
+    }
+
+    public void testIndex() throws IOException {
+        final XContentType xContentType = randomFrom(XContentType.values());
+        {
+            IndexRequest indexRequest = new IndexRequest("index", "type");
+            indexRequest.source(XContentBuilder.builder(xContentType.xContent()).startObject().field("test", "test").endObject());
+
+            IndexResponse indexResponse = execute(indexRequest, highLevelClient()::index, highLevelClient()::indexAsync);
+            assertEquals(RestStatus.CREATED, indexResponse.status());
+            assertEquals(DocWriteResponse.Result.CREATED, indexResponse.getResult());
+            assertEquals("index", indexResponse.getIndex());
+            assertEquals("type", indexResponse.getType());
+            assertTrue(Strings.hasLength(indexResponse.getId()));
+            assertEquals(1L, indexResponse.getVersion());
+            assertNotNull(indexResponse.getShardId());
+            assertEquals(-1, indexResponse.getShardId().getId());
+            assertEquals("index", indexResponse.getShardId().getIndexName());
+            assertEquals("index", indexResponse.getShardId().getIndex().getName());
+            assertEquals("_na_", indexResponse.getShardId().getIndex().getUUID());
+            assertNotNull(indexResponse.getShardInfo());
+            assertEquals(0, indexResponse.getShardInfo().getFailed());
+            assertTrue(indexResponse.getShardInfo().getSuccessful() > 0);
+            assertTrue(indexResponse.getShardInfo().getTotal() > 0);
+        }
+        {
+            IndexRequest indexRequest = new IndexRequest("index", "type", "id");
+            indexRequest.source(XContentBuilder.builder(xContentType.xContent()).startObject().field("version", 1).endObject());
+
+            IndexResponse indexResponse = execute(indexRequest, highLevelClient()::index, highLevelClient()::indexAsync);
+            assertEquals(RestStatus.CREATED, indexResponse.status());
+            assertEquals("index", indexResponse.getIndex());
+            assertEquals("type", indexResponse.getType());
+            assertEquals("id", indexResponse.getId());
+            assertEquals(1L, indexResponse.getVersion());
+
+            indexRequest = new IndexRequest("index", "type", "id");
+            indexRequest.source(XContentBuilder.builder(xContentType.xContent()).startObject().field("version", 2).endObject());
+
+            indexResponse = execute(indexRequest, highLevelClient()::index, highLevelClient()::indexAsync);
+            assertEquals(RestStatus.OK, indexResponse.status());
+            assertEquals("index", indexResponse.getIndex());
+            assertEquals("type", indexResponse.getType());
+            assertEquals("id", indexResponse.getId());
+            assertEquals(2L, indexResponse.getVersion());
+
+            ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class, () -> {
+                IndexRequest wrongRequest = new IndexRequest("index", "type", "id");
+                wrongRequest.source(XContentBuilder.builder(xContentType.xContent()).startObject().field("field", "test").endObject());
+                wrongRequest.version(5L);
+
+                execute(wrongRequest, highLevelClient()::index, highLevelClient()::indexAsync);
+            });
+            assertEquals(RestStatus.CONFLICT, exception.status());
+            assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, reason=[type][id]: " +
+                         "version conflict, current version [2] is different than the one provided [5]]", exception.getMessage());
+            assertEquals("index", exception.getMetadata("es.index").get(0));
+        }
+        {
+            ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class, () -> {
+                IndexRequest indexRequest = new IndexRequest("index", "type", "missing_parent");
+                indexRequest.source(XContentBuilder.builder(xContentType.xContent()).startObject().field("field", "test").endObject());
+                indexRequest.parent("missing");
+
+                execute(indexRequest, highLevelClient()::index, highLevelClient()::indexAsync);
+            });
+
+            assertEquals(RestStatus.BAD_REQUEST, exception.status());
+            assertEquals("Elasticsearch exception [type=illegal_argument_exception, " +
+                         "reason=can't specify parent if no parent field has been configured]", exception.getMessage());
+        }
+        {
+            ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class, () -> {
+                IndexRequest indexRequest = new IndexRequest("index", "type", "missing_pipeline");
+                indexRequest.source(XContentBuilder.builder(xContentType.xContent()).startObject().field("field", "test").endObject());
+                indexRequest.setPipeline("missing");
+
+                execute(indexRequest, highLevelClient()::index, highLevelClient()::indexAsync);
+            });
+
+            assertEquals(RestStatus.BAD_REQUEST, exception.status());
+            assertEquals("Elasticsearch exception [type=illegal_argument_exception, " +
+                         "reason=pipeline with id [missing] does not exist]", exception.getMessage());
+        }
+        {
+            IndexRequest indexRequest = new IndexRequest("index", "type", "external_version_type");
+            indexRequest.source(XContentBuilder.builder(xContentType.xContent()).startObject().field("field", "test").endObject());
+            indexRequest.version(12L);
+            indexRequest.versionType(VersionType.EXTERNAL);
+
+            IndexResponse indexResponse = execute(indexRequest, highLevelClient()::index, highLevelClient()::indexAsync);
+            assertEquals(RestStatus.CREATED, indexResponse.status());
+            assertEquals("index", indexResponse.getIndex());
+            assertEquals("type", indexResponse.getType());
+            assertEquals("external_version_type", indexResponse.getId());
+            assertEquals(12L, indexResponse.getVersion());
+        }
+        {
+            final IndexRequest indexRequest = new IndexRequest("index", "type", "with_create_op_type");
+            indexRequest.source(XContentBuilder.builder(xContentType.xContent()).startObject().field("field", "test").endObject());
+            indexRequest.opType(DocWriteRequest.OpType.CREATE);
+
+            IndexResponse indexResponse = execute(indexRequest, highLevelClient()::index, highLevelClient()::indexAsync);
+            assertEquals(RestStatus.CREATED, indexResponse.status());
+            assertEquals("index", indexResponse.getIndex());
+            assertEquals("type", indexResponse.getType());
+            assertEquals("with_create_op_type", indexResponse.getId());
+
+            ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class, () -> {
+                execute(indexRequest, highLevelClient()::index, highLevelClient()::indexAsync);
+            });
+
+            assertEquals(RestStatus.CONFLICT, exception.status());
+            assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, reason=[type][with_create_op_type]: " +
+                         "version conflict, document already exists (current version [1])]", exception.getMessage());
+        }
+    }
+
+    public void testUpdate() throws IOException {
+        {
+            UpdateRequest updateRequest = new UpdateRequest("index", "type", "does_not_exist");
+            updateRequest.doc(singletonMap("field", "value"), randomFrom(XContentType.values()));
+
+            ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class, () ->
+                    execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync));
+            assertEquals(RestStatus.NOT_FOUND, exception.status());
+            assertEquals("Elasticsearch exception [type=document_missing_exception, reason=[type][does_not_exist]: document missing]",
+                    exception.getMessage());
+        }
+        {
+            IndexRequest indexRequest = new IndexRequest("index", "type", "id");
+            indexRequest.source(singletonMap("field", "value"));
+            IndexResponse indexResponse = highLevelClient().index(indexRequest);
+            assertEquals(RestStatus.CREATED, indexResponse.status());
+
+            UpdateRequest updateRequest = new UpdateRequest("index", "type", "id");
+            updateRequest.doc(singletonMap("field", "updated"), randomFrom(XContentType.values()));
+
+            UpdateResponse updateResponse = execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync);
+            assertEquals(RestStatus.OK, updateResponse.status());
+            assertEquals(indexResponse.getVersion() + 1, updateResponse.getVersion());
+
+            UpdateRequest updateRequestConflict = new UpdateRequest("index", "type", "id");
+            updateRequestConflict.doc(singletonMap("field", "with_version_conflict"), randomFrom(XContentType.values()));
+            updateRequestConflict.version(indexResponse.getVersion());
+
+            ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class, () ->
+                    execute(updateRequestConflict, highLevelClient()::update, highLevelClient()::updateAsync));
+            assertEquals(RestStatus.CONFLICT, exception.status());
+            assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, reason=[type][id]: version conflict, " +
+                            "current version [2] is different than the one provided [1]]", exception.getMessage());
+        }
+        {
+            ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class, () -> {
+                UpdateRequest updateRequest = new UpdateRequest("index", "type", "id");
+                updateRequest.doc(singletonMap("field", "updated"), randomFrom(XContentType.values()));
+                if (randomBoolean()) {
+                    updateRequest.parent("missing");
+                } else {
+                    updateRequest.routing("missing");
+                }
+                execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync);
+            });
+
+            assertEquals(RestStatus.NOT_FOUND, exception.status());
+            assertEquals("Elasticsearch exception [type=document_missing_exception, reason=[type][id]: document missing]",
+                    exception.getMessage());
+        }
+        {
+            IndexRequest indexRequest = new IndexRequest("index", "type", "with_script");
+            indexRequest.source(singletonMap("counter", 12));
+            IndexResponse indexResponse = highLevelClient().index(indexRequest);
+            assertEquals(RestStatus.CREATED, indexResponse.status());
+
+            UpdateRequest updateRequest = new UpdateRequest("index", "type", "with_script");
+            Script script = new Script(ScriptType.INLINE, "painless", "ctx._source.counter += params.count", singletonMap("count", 8));
+            updateRequest.script(script);
+            updateRequest.fetchSource(true);
+
+            UpdateResponse updateResponse = execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync);
+            assertEquals(RestStatus.OK, updateResponse.status());
+            assertEquals(DocWriteResponse.Result.UPDATED, updateResponse.getResult());
+            assertEquals(2L, updateResponse.getVersion());
+            assertEquals(20, updateResponse.getGetResult().sourceAsMap().get("counter"));
+
+        }
+        {
+            IndexRequest indexRequest = new IndexRequest("index", "type", "with_doc");
+            indexRequest.source("field_1", "one", "field_3", "three");
+            indexRequest.version(12L);
+            indexRequest.versionType(VersionType.EXTERNAL);
+            IndexResponse indexResponse = highLevelClient().index(indexRequest);
+            assertEquals(RestStatus.CREATED, indexResponse.status());
+            assertEquals(12L, indexResponse.getVersion());
+
+            UpdateRequest updateRequest = new UpdateRequest("index", "type", "with_doc");
+            updateRequest.doc(singletonMap("field_2", "two"), randomFrom(XContentType.values()));
+            updateRequest.fetchSource("field_*", "field_3");
+
+            UpdateResponse updateResponse = execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync);
+            assertEquals(RestStatus.OK, updateResponse.status());
+            assertEquals(DocWriteResponse.Result.UPDATED, updateResponse.getResult());
+            assertEquals(13L, updateResponse.getVersion());
+            GetResult getResult = updateResponse.getGetResult();
+            assertEquals(13L, updateResponse.getVersion());
+            Map<String, Object> sourceAsMap = getResult.sourceAsMap();
+            assertEquals("one", sourceAsMap.get("field_1"));
+            assertEquals("two", sourceAsMap.get("field_2"));
+            assertFalse(sourceAsMap.containsKey("field_3"));
+        }
+        {
+            IndexRequest indexRequest = new IndexRequest("index", "type", "noop");
+            indexRequest.source("field", "value");
+            IndexResponse indexResponse = highLevelClient().index(indexRequest);
+            assertEquals(RestStatus.CREATED, indexResponse.status());
+            assertEquals(1L, indexResponse.getVersion());
+
+            UpdateRequest updateRequest = new UpdateRequest("index", "type", "noop");
+            updateRequest.doc(singletonMap("field", "value"), randomFrom(XContentType.values()));
+
+            UpdateResponse updateResponse = execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync);
+            assertEquals(RestStatus.OK, updateResponse.status());
+            assertEquals(DocWriteResponse.Result.NOOP, updateResponse.getResult());
+            assertEquals(1L, updateResponse.getVersion());
+
+            updateRequest.detectNoop(false);
+
+            updateResponse = execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync);
+            assertEquals(RestStatus.OK, updateResponse.status());
+            assertEquals(DocWriteResponse.Result.UPDATED, updateResponse.getResult());
+            assertEquals(2L, updateResponse.getVersion());
+        }
+        {
+            UpdateRequest updateRequest = new UpdateRequest("index", "type", "with_upsert");
+            updateRequest.upsert(singletonMap("doc_status", "created"));
+            updateRequest.doc(singletonMap("doc_status", "updated"));
+            updateRequest.fetchSource(true);
+
+            UpdateResponse updateResponse = execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync);
+            assertEquals(RestStatus.CREATED, updateResponse.status());
+            assertEquals("index", updateResponse.getIndex());
+            assertEquals("type", updateResponse.getType());
+            assertEquals("with_upsert", updateResponse.getId());
+            GetResult getResult = updateResponse.getGetResult();
+            assertEquals(1L, updateResponse.getVersion());
+            assertEquals("created", getResult.sourceAsMap().get("doc_status"));
+        }
+        {
+            UpdateRequest updateRequest = new UpdateRequest("index", "type", "with_doc_as_upsert");
+            updateRequest.doc(singletonMap("field", "initialized"));
+            updateRequest.fetchSource(true);
+            updateRequest.docAsUpsert(true);
+
+            UpdateResponse updateResponse = execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync);
+            assertEquals(RestStatus.CREATED, updateResponse.status());
+            assertEquals("index", updateResponse.getIndex());
+            assertEquals("type", updateResponse.getType());
+            assertEquals("with_doc_as_upsert", updateResponse.getId());
+            GetResult getResult = updateResponse.getGetResult();
+            assertEquals(1L, updateResponse.getVersion());
+            assertEquals("initialized", getResult.sourceAsMap().get("field"));
+        }
+        {
+            UpdateRequest updateRequest = new UpdateRequest("index", "type", "with_scripted_upsert");
+            updateRequest.fetchSource(true);
+            updateRequest.script(new Script(ScriptType.INLINE, "painless", "ctx._source.level = params.test", singletonMap("test", "C")));
+            updateRequest.scriptedUpsert(true);
+            updateRequest.upsert(singletonMap("level", "A"));
+
+            UpdateResponse updateResponse = execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync);
+            assertEquals(RestStatus.CREATED, updateResponse.status());
+            assertEquals("index", updateResponse.getIndex());
+            assertEquals("type", updateResponse.getType());
+            assertEquals("with_scripted_upsert", updateResponse.getId());
+
+            GetResult getResult = updateResponse.getGetResult();
+            assertEquals(1L, updateResponse.getVersion());
+            assertEquals("C", getResult.sourceAsMap().get("level"));
+        }
+        {
+            IllegalStateException exception = expectThrows(IllegalStateException.class, () -> {
+                UpdateRequest updateRequest = new UpdateRequest("index", "type", "id");
+                updateRequest.doc(new IndexRequest().source(Collections.singletonMap("field", "doc"), XContentType.JSON));
+                updateRequest.upsert(new IndexRequest().source(Collections.singletonMap("field", "upsert"), XContentType.YAML));
+                execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync);
+            });
+            assertEquals("Update request cannot have different content types for doc [JSON] and upsert [YAML] documents",
+                    exception.getMessage());
+        }
+    }
+
+    public void testBulk() throws IOException {
+        int nbItems = randomIntBetween(10, 100);
+        boolean[] errors = new boolean[nbItems];
+
+        XContentType xContentType = randomFrom(XContentType.JSON, XContentType.SMILE);
+
+        BulkRequest bulkRequest = new BulkRequest();
+        for (int i = 0; i < nbItems; i++) {
+            String id = String.valueOf(i);
+            boolean erroneous = randomBoolean();
+            errors[i] = erroneous;
+
+            DocWriteRequest.OpType opType = randomFrom(DocWriteRequest.OpType.values());
+            if (opType == DocWriteRequest.OpType.DELETE) {
+                if (erroneous == false) {
+                    assertEquals(RestStatus.CREATED,
+                            highLevelClient().index(new IndexRequest("index", "test", id).source("field", -1)).status());
+                }
+                DeleteRequest deleteRequest = new DeleteRequest("index", "test", id);
+                bulkRequest.add(deleteRequest);
+
+            } else {
+                BytesReference source = XContentBuilder.builder(xContentType.xContent()).startObject().field("id", i).endObject().bytes();
+                if (opType == DocWriteRequest.OpType.INDEX) {
+                    IndexRequest indexRequest = new IndexRequest("index", "test", id).source(source, xContentType);
+                    if (erroneous) {
+                        indexRequest.version(12L);
+                    }
+                    bulkRequest.add(indexRequest);
+
+                } else if (opType == DocWriteRequest.OpType.CREATE) {
+                    IndexRequest createRequest = new IndexRequest("index", "test", id).source(source, xContentType).create(true);
+                    if (erroneous) {
+                        assertEquals(RestStatus.CREATED, highLevelClient().index(createRequest).status());
+                    }
+                    bulkRequest.add(createRequest);
+
+                } else if (opType == DocWriteRequest.OpType.UPDATE) {
+                    UpdateRequest updateRequest = new UpdateRequest("index", "test", id)
+                            .doc(new IndexRequest().source(source, xContentType));
+                    if (erroneous == false) {
+                        assertEquals(RestStatus.CREATED,
+                                highLevelClient().index(new IndexRequest("index", "test", id).source("field", -1)).status());
+                    }
+                    bulkRequest.add(updateRequest);
+                }
+            }
+        }
+
+        BulkResponse bulkResponse = execute(bulkRequest, highLevelClient()::bulk, highLevelClient()::bulkAsync);
+        assertEquals(RestStatus.OK, bulkResponse.status());
+        assertTrue(bulkResponse.getTook().getMillis() > 0);
+        assertEquals(nbItems, bulkResponse.getItems().length);
+
+        validateBulkResponses(nbItems, errors, bulkResponse, bulkRequest);
+    }
+
+    public void testBulkProcessorIntegration() throws IOException, InterruptedException {
+        int nbItems = randomIntBetween(10, 100);
+        boolean[] errors = new boolean[nbItems];
+
+        XContentType xContentType = randomFrom(XContentType.JSON, XContentType.SMILE);
+
+        AtomicReference<BulkResponse> responseRef = new AtomicReference<>();
+        AtomicReference<BulkRequest> requestRef = new AtomicReference<>();
+        AtomicReference<Throwable> error = new AtomicReference<>();
+
+        BulkProcessor.Listener listener = new BulkProcessor.Listener() {
+            @Override
+            public void beforeBulk(long executionId, BulkRequest request) {
+
+            }
+
+            @Override
+            public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
+                responseRef.set(response);
+                requestRef.set(request);
+            }
+
+            @Override
+            public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
+                error.set(failure);
+            }
+        };
+
+        ThreadPool threadPool = new ThreadPool(Settings.builder().put("node.name", getClass().getName()).build());
+        try(BulkProcessor processor = new BulkProcessor.Builder(highLevelClient()::bulkAsync, listener, threadPool)
+            .setConcurrentRequests(0)
+            .setBulkSize(new ByteSizeValue(5, ByteSizeUnit.GB))
+            .setBulkActions(nbItems + 1)
+            .build()) {
+            for (int i = 0; i < nbItems; i++) {
+                String id = String.valueOf(i);
+                boolean erroneous = randomBoolean();
+                errors[i] = erroneous;
+
+                DocWriteRequest.OpType opType = randomFrom(DocWriteRequest.OpType.values());
+                if (opType == DocWriteRequest.OpType.DELETE) {
+                    if (erroneous == false) {
+                        assertEquals(RestStatus.CREATED,
+                            highLevelClient().index(new IndexRequest("index", "test", id).source("field", -1)).status());
+                    }
+                    DeleteRequest deleteRequest = new DeleteRequest("index", "test", id);
+                    processor.add(deleteRequest);
+
+                } else {
+                    if (opType == DocWriteRequest.OpType.INDEX) {
+                        IndexRequest indexRequest = new IndexRequest("index", "test", id).source(xContentType, "id", i);
+                        if (erroneous) {
+                            indexRequest.version(12L);
+                        }
+                        processor.add(indexRequest);
+
+                    } else if (opType == DocWriteRequest.OpType.CREATE) {
+                        IndexRequest createRequest = new IndexRequest("index", "test", id).source(xContentType, "id", i).create(true);
+                        if (erroneous) {
+                            assertEquals(RestStatus.CREATED, highLevelClient().index(createRequest).status());
+                        }
+                        processor.add(createRequest);
+
+                    } else if (opType == DocWriteRequest.OpType.UPDATE) {
+                        UpdateRequest updateRequest = new UpdateRequest("index", "test", id)
+                            .doc(new IndexRequest().source(xContentType, "id", i));
+                        if (erroneous == false) {
+                            assertEquals(RestStatus.CREATED,
+                                highLevelClient().index(new IndexRequest("index", "test", id).source("field", -1)).status());
+                        }
+                        processor.add(updateRequest);
+                    }
+                }
+            }
+            assertNull(responseRef.get());
+            assertNull(requestRef.get());
+        }
+
+
+        BulkResponse bulkResponse = responseRef.get();
+        BulkRequest bulkRequest = requestRef.get();
+
+        assertEquals(RestStatus.OK, bulkResponse.status());
+        assertTrue(bulkResponse.getTookInMillis() > 0);
+        assertEquals(nbItems, bulkResponse.getItems().length);
+        assertNull(error.get());
+
+        validateBulkResponses(nbItems, errors, bulkResponse, bulkRequest);
+
+        terminate(threadPool);
+    }
+
+    private void validateBulkResponses(int nbItems, boolean[] errors, BulkResponse bulkResponse, BulkRequest bulkRequest) {
+        for (int i = 0; i < nbItems; i++) {
+            BulkItemResponse bulkItemResponse = bulkResponse.getItems()[i];
+
+            assertEquals(i, bulkItemResponse.getItemId());
+            assertEquals("index", bulkItemResponse.getIndex());
+            assertEquals("test", bulkItemResponse.getType());
+            assertEquals(String.valueOf(i), bulkItemResponse.getId());
+
+            DocWriteRequest.OpType requestOpType = bulkRequest.requests().get(i).opType();
+            if (requestOpType == DocWriteRequest.OpType.INDEX || requestOpType == DocWriteRequest.OpType.CREATE) {
+                assertEquals(errors[i], bulkItemResponse.isFailed());
+                assertEquals(errors[i] ? RestStatus.CONFLICT : RestStatus.CREATED, bulkItemResponse.status());
+            } else if (requestOpType == DocWriteRequest.OpType.UPDATE) {
+                assertEquals(errors[i], bulkItemResponse.isFailed());
+                assertEquals(errors[i] ? RestStatus.NOT_FOUND : RestStatus.OK, bulkItemResponse.status());
+            } else if (requestOpType == DocWriteRequest.OpType.DELETE) {
+                assertFalse(bulkItemResponse.isFailed());
+                assertEquals(errors[i] ? RestStatus.NOT_FOUND : RestStatus.OK, bulkItemResponse.status());
+            }
+        }
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/CustomRestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/CustomRestHighLevelClientTests.java
new file mode 100644
index 0000000000000..18ca074dd58b1
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/CustomRestHighLevelClientTests.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import org.apache.http.Header;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpHost;
+import org.apache.http.ProtocolVersion;
+import org.apache.http.RequestLine;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.entity.ByteArrayEntity;
+import org.apache.http.entity.ContentType;
+import org.apache.http.message.BasicHeader;
+import org.apache.http.message.BasicRequestLine;
+import org.apache.http.message.BasicStatusLine;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.Build;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.main.MainRequest;
+import org.elasticsearch.action.main.MainResponse;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseListener;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.emptySet;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyMapOf;
+import static org.mockito.Matchers.anyObject;
+import static org.mockito.Matchers.anyVararg;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Test and demonstrates how {@link RestHighLevelClient} can be extended to support custom endpoints.
+ */
+public class CustomRestHighLevelClientTests extends ESTestCase {
+
+    private static final String ENDPOINT = "/_custom";
+
+    private CustomRestClient restHighLevelClient;
+
+    @Before
+    @SuppressWarnings("unchecked")
+    public void initClients() throws IOException {
+        if (restHighLevelClient == null) {
+            final RestClient restClient = mock(RestClient.class);
+            restHighLevelClient = new CustomRestClient(restClient);
+
+            doAnswer(mock -> mockPerformRequest((Header) mock.getArguments()[4]))
+                    .when(restClient)
+                    .performRequest(eq(HttpGet.METHOD_NAME), eq(ENDPOINT), anyMapOf(String.class, String.class), anyObject(), anyVararg());
+
+            doAnswer(mock -> mockPerformRequestAsync((Header) mock.getArguments()[5], (ResponseListener) mock.getArguments()[4]))
+                    .when(restClient)
+                    .performRequestAsync(eq(HttpGet.METHOD_NAME), eq(ENDPOINT), anyMapOf(String.class, String.class),
+                            any(HttpEntity.class), any(ResponseListener.class), anyVararg());
+        }
+    }
+
+    public void testCustomEndpoint() throws IOException {
+        final MainRequest request = new MainRequest();
+        final Header header = new BasicHeader("node_name", randomAlphaOfLengthBetween(1, 10));
+
+        MainResponse response = restHighLevelClient.custom(request, header);
+        assertEquals(header.getValue(), response.getNodeName());
+
+        response = restHighLevelClient.customAndParse(request, header);
+        assertEquals(header.getValue(), response.getNodeName());
+    }
+
+    public void testCustomEndpointAsync() throws Exception {
+        final MainRequest request = new MainRequest();
+        final Header header = new BasicHeader("node_name", randomAlphaOfLengthBetween(1, 10));
+
+        PlainActionFuture<MainResponse> future = PlainActionFuture.newFuture();
+        restHighLevelClient.customAsync(request, future, header);
+        assertEquals(header.getValue(), future.get().getNodeName());
+
+        future = PlainActionFuture.newFuture();
+        restHighLevelClient.customAndParseAsync(request, future, header);
+        assertEquals(header.getValue(), future.get().getNodeName());
+    }
+
+    /**
+     * The {@link RestHighLevelClient} must declare the following execution methods using the <code>protected</code> modifier
+     * so that they can be used by subclasses to implement custom logic.
+     */
+    @SuppressForbidden(reason = "We're forced to uses Class#getDeclaredMethods() here because this test checks protected methods")
+    public void testMethodsVisibility() throws ClassNotFoundException {
+        final String[] methodNames = new String[]{"performRequest",
+                                                  "performRequestAsync",
+                                                  "performRequestAndParseEntity",
+                                                  "performRequestAsyncAndParseEntity",
+                                                  "parseEntity",
+                                                  "parseResponseException"};
+
+        final List<String> protectedMethods =  Arrays.stream(RestHighLevelClient.class.getDeclaredMethods())
+                                                     .filter(method -> Modifier.isProtected(method.getModifiers()))
+                                                     .map(Method::getName)
+                                                     .collect(Collectors.toList());
+
+        assertThat(protectedMethods, containsInAnyOrder(methodNames));
+    }
+
+    /**
+     * Mocks the asynchronous request execution by calling the {@link #mockPerformRequest(Header)} method.
+     */
+    private Void mockPerformRequestAsync(Header httpHeader, ResponseListener responseListener) {
+        try {
+            responseListener.onSuccess(mockPerformRequest(httpHeader));
+        } catch (IOException e) {
+            responseListener.onFailure(e);
+        }
+        return null;
+    }
+
+    /**
+     * Mocks the synchronous request execution like if it was executed by Elasticsearch.
+     */
+    private Response mockPerformRequest(Header httpHeader) throws IOException {
+        final Response mockResponse = mock(Response.class);
+        when(mockResponse.getHost()).thenReturn(new HttpHost("localhost", 9200));
+
+        ProtocolVersion protocol = new ProtocolVersion("HTTP", 1, 1);
+        when(mockResponse.getStatusLine()).thenReturn(new BasicStatusLine(protocol, 200, "OK"));
+
+        MainResponse response = new MainResponse(httpHeader.getValue(), Version.CURRENT, ClusterName.DEFAULT, "_na", Build.CURRENT, true);
+        BytesRef bytesRef = XContentHelper.toXContent(response, XContentType.JSON, false).toBytesRef();
+        when(mockResponse.getEntity()).thenReturn(new ByteArrayEntity(bytesRef.bytes, ContentType.APPLICATION_JSON));
+
+        RequestLine requestLine = new BasicRequestLine(HttpGet.METHOD_NAME, ENDPOINT, protocol);
+        when(mockResponse.getRequestLine()).thenReturn(requestLine);
+
+        return mockResponse;
+    }
+
+    /**
+     * A custom high level client that provides custom methods to execute a request and get its associate response back.
+     */
+    static class CustomRestClient extends RestHighLevelClient {
+
+        private CustomRestClient(RestClient restClient) {
+            super(restClient);
+        }
+
+        MainResponse custom(MainRequest mainRequest, Header... headers) throws IOException {
+            return performRequest(mainRequest, this::toRequest, this::toResponse, emptySet(), headers);
+        }
+
+        MainResponse customAndParse(MainRequest mainRequest, Header... headers) throws IOException {
+            return performRequestAndParseEntity(mainRequest, this::toRequest, MainResponse::fromXContent, emptySet(), headers);
+        }
+
+        void customAsync(MainRequest mainRequest, ActionListener<MainResponse> listener, Header... headers) {
+            performRequestAsync(mainRequest, this::toRequest, this::toResponse, listener, emptySet(), headers);
+        }
+
+        void customAndParseAsync(MainRequest mainRequest, ActionListener<MainResponse> listener, Header... headers) {
+            performRequestAsyncAndParseEntity(mainRequest, this::toRequest, MainResponse::fromXContent, listener, emptySet(), headers);
+        }
+
+        Request toRequest(MainRequest mainRequest) throws IOException {
+            return new Request(HttpGet.METHOD_NAME, ENDPOINT, emptyMap(), null);
+        }
+
+        MainResponse toResponse(Response response) throws IOException {
+            return parseEntity(response.getEntity(), MainResponse::fromXContent);
+        }
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ESRestHighLevelClientTestCase.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ESRestHighLevelClientTestCase.java
new file mode 100644
index 0000000000000..cdd8317830909
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ESRestHighLevelClientTestCase.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import org.apache.http.Header;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.junit.AfterClass;
+import org.junit.Before;
+
+import java.io.IOException;
+
+public abstract class ESRestHighLevelClientTestCase extends ESRestTestCase {
+
+    private static RestHighLevelClient restHighLevelClient;
+
+    @Before
+    public void initHighLevelClient() throws IOException {
+        super.initClient();
+        if (restHighLevelClient == null) {
+            restHighLevelClient = new RestHighLevelClient(client());
+        }
+    }
+
+    @AfterClass
+    public static void cleanupClient() {
+        restHighLevelClient = null;
+    }
+
+    protected static RestHighLevelClient highLevelClient() {
+        return restHighLevelClient;
+    }
+
+    /**
+     * Executes the provided request using either the sync method or its async variant, both provided as functions
+     */
+    protected static <Req, Resp> Resp execute(Req request, SyncMethod<Req, Resp> syncMethod,
+                                       AsyncMethod<Req, Resp> asyncMethod, Header... headers) throws IOException {
+        if (randomBoolean()) {
+            return syncMethod.execute(request, headers);
+        } else {
+            PlainActionFuture<Resp> future = PlainActionFuture.newFuture();
+            asyncMethod.execute(request, future, headers);
+            return future.actionGet();
+        }
+    }
+
+    @FunctionalInterface
+    protected interface SyncMethod<Request, Response> {
+        Response execute(Request request, Header... headers) throws IOException;
+    }
+
+    @FunctionalInterface
+    protected interface AsyncMethod<Request, Response> {
+        void execute(Request request, ActionListener<Response> listener, Header... headers);
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/PingAndInfoIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/PingAndInfoIT.java
new file mode 100644
index 0000000000000..b22ded52655df
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/PingAndInfoIT.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import org.elasticsearch.action.main.MainResponse;
+
+import java.io.IOException;
+import java.util.Map;
+
+public class PingAndInfoIT extends ESRestHighLevelClientTestCase {
+
+    public void testPing() throws IOException {
+        assertTrue(highLevelClient().ping());
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testInfo() throws IOException {
+        MainResponse info = highLevelClient().info();
+        // compare with what the low level client outputs
+        Map<String, Object> infoAsMap = entityAsMap(adminClient().performRequest("GET", "/"));
+        assertEquals(infoAsMap.get("cluster_name"), info.getClusterName().value());
+        assertEquals(infoAsMap.get("cluster_uuid"), info.getClusterUuid());
+
+        // only check node name existence, might be a different one from what was hit by low level client in multi-node cluster
+        assertNotNull(info.getNodeName());
+        Map<String, Object> versionMap = (Map<String, Object>) infoAsMap.get("version");
+        assertEquals(versionMap.get("build_hash"), info.getBuild().shortHash());
+        assertEquals(versionMap.get("build_date"), info.getBuild().date());
+        assertEquals(versionMap.get("build_snapshot"), info.getBuild().isSnapshot());
+        assertEquals(versionMap.get("number"), info.getVersion().toString());
+        assertEquals(versionMap.get("lucene_version"), info.getVersion().luceneVersion.toString());
+    }
+
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java
new file mode 100644
index 0000000000000..8f52eb37fe95d
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java
@@ -0,0 +1,945 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import org.apache.http.HttpEntity;
+import org.apache.http.entity.ByteArrayEntity;
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.util.EntityUtils;
+import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.bulk.BulkRequest;
+import org.elasticsearch.action.bulk.BulkShardRequest;
+import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.get.GetRequest;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.search.ClearScrollRequest;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchScrollRequest;
+import org.elasticsearch.action.search.SearchType;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.action.support.replication.ReplicatedWriteRequest;
+import org.elasticsearch.action.support.replication.ReplicationRequest;
+import org.elasticsearch.action.update.UpdateRequest;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.Streams;
+import org.elasticsearch.common.lucene.uid.Versions;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.VersionType;
+import org.elasticsearch.index.query.TermQueryBuilder;
+import org.elasticsearch.rest.action.search.RestSearchAction;
+import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
+import org.elasticsearch.search.aggregations.support.ValueType;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.collapse.CollapseBuilder;
+import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
+import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
+import org.elasticsearch.search.rescore.QueryRescorerBuilder;
+import org.elasticsearch.search.suggest.SuggestBuilder;
+import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.RandomObjects;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Modifier;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.StringJoiner;
+import java.util.function.Consumer;
+import java.util.function.Function;
+
+import static java.util.Collections.singletonMap;
+import static org.elasticsearch.client.Request.enforceSameContentType;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
+
+public class RequestTests extends ESTestCase {
+
+    public void testConstructor() throws Exception {
+        final String method = randomFrom("GET", "PUT", "POST", "HEAD", "DELETE");
+        final String endpoint = randomAlphaOfLengthBetween(1, 10);
+        final Map<String, String> parameters = singletonMap(randomAlphaOfLength(5), randomAlphaOfLength(5));
+        final HttpEntity entity = randomBoolean() ? new StringEntity(randomAlphaOfLengthBetween(1, 100), ContentType.TEXT_PLAIN) : null;
+
+        NullPointerException e = expectThrows(NullPointerException.class, () -> new Request(null, endpoint, parameters, entity));
+        assertEquals("method cannot be null", e.getMessage());
+
+        e = expectThrows(NullPointerException.class, () -> new Request(method, null, parameters, entity));
+        assertEquals("endpoint cannot be null", e.getMessage());
+
+        e = expectThrows(NullPointerException.class, () -> new Request(method, endpoint, null, entity));
+        assertEquals("parameters cannot be null", e.getMessage());
+
+        final Request request = new Request(method, endpoint, parameters, entity);
+        assertEquals(method, request.getMethod());
+        assertEquals(endpoint, request.getEndpoint());
+        assertEquals(parameters, request.getParameters());
+        assertEquals(entity, request.getEntity());
+
+        final Constructor<?>[] constructors = Request.class.getConstructors();
+        assertEquals("Expected only 1 constructor", 1, constructors.length);
+        assertTrue("Request constructor is not public", Modifier.isPublic(constructors[0].getModifiers()));
+    }
+
+    public void testClassVisibility() throws Exception {
+        assertTrue("Request class is not public", Modifier.isPublic(Request.class.getModifiers()));
+    }
+
+    public void testPing() {
+        Request request = Request.ping();
+        assertEquals("/", request.getEndpoint());
+        assertEquals(0, request.getParameters().size());
+        assertNull(request.getEntity());
+        assertEquals("HEAD", request.getMethod());
+    }
+
+    public void testInfo() {
+        Request request = Request.info();
+        assertEquals("/", request.getEndpoint());
+        assertEquals(0, request.getParameters().size());
+        assertNull(request.getEntity());
+        assertEquals("GET", request.getMethod());
+    }
+
+    public void testGet() {
+        getAndExistsTest(Request::get, "GET");
+    }
+
+    public void testDelete() throws IOException {
+        String index = randomAlphaOfLengthBetween(3, 10);
+        String type = randomAlphaOfLengthBetween(3, 10);
+        String id = randomAlphaOfLengthBetween(3, 10);
+        DeleteRequest deleteRequest = new DeleteRequest(index, type, id);
+
+        Map<String, String> expectedParams = new HashMap<>();
+
+        setRandomTimeout(deleteRequest, expectedParams);
+        setRandomRefreshPolicy(deleteRequest, expectedParams);
+        setRandomVersion(deleteRequest, expectedParams);
+        setRandomVersionType(deleteRequest, expectedParams);
+
+        if (frequently()) {
+            if (randomBoolean()) {
+                String routing = randomAlphaOfLengthBetween(3, 10);
+                deleteRequest.routing(routing);
+                expectedParams.put("routing", routing);
+            }
+            if (randomBoolean()) {
+                String parent = randomAlphaOfLengthBetween(3, 10);
+                deleteRequest.parent(parent);
+                expectedParams.put("parent", parent);
+            }
+        }
+
+        Request request = Request.delete(deleteRequest);
+        assertEquals("/" + index + "/" + type + "/" + id, request.getEndpoint());
+        assertEquals(expectedParams, request.getParameters());
+        assertEquals("DELETE", request.getMethod());
+        assertNull(request.getEntity());
+    }
+
+    public void testExists() {
+        getAndExistsTest(Request::exists, "HEAD");
+    }
+
+    private static void getAndExistsTest(Function<GetRequest, Request> requestConverter, String method) {
+        String index = randomAlphaOfLengthBetween(3, 10);
+        String type = randomAlphaOfLengthBetween(3, 10);
+        String id = randomAlphaOfLengthBetween(3, 10);
+        GetRequest getRequest = new GetRequest(index, type, id);
+
+        Map<String, String> expectedParams = new HashMap<>();
+        if (randomBoolean()) {
+            if (randomBoolean()) {
+                String preference = randomAlphaOfLengthBetween(3, 10);
+                getRequest.preference(preference);
+                expectedParams.put("preference", preference);
+            }
+            if (randomBoolean()) {
+                String routing = randomAlphaOfLengthBetween(3, 10);
+                getRequest.routing(routing);
+                expectedParams.put("routing", routing);
+            }
+            if (randomBoolean()) {
+                boolean realtime = randomBoolean();
+                getRequest.realtime(realtime);
+                if (realtime == false) {
+                    expectedParams.put("realtime", "false");
+                }
+            }
+            if (randomBoolean()) {
+                boolean refresh = randomBoolean();
+                getRequest.refresh(refresh);
+                if (refresh) {
+                    expectedParams.put("refresh", "true");
+                }
+            }
+            if (randomBoolean()) {
+                long version = randomLong();
+                getRequest.version(version);
+                if (version != Versions.MATCH_ANY) {
+                    expectedParams.put("version", Long.toString(version));
+                }
+            }
+            if (randomBoolean()) {
+                VersionType versionType = randomFrom(VersionType.values());
+                getRequest.versionType(versionType);
+                if (versionType != VersionType.INTERNAL) {
+                    expectedParams.put("version_type", versionType.name().toLowerCase(Locale.ROOT));
+                }
+            }
+            if (randomBoolean()) {
+                int numStoredFields = randomIntBetween(1, 10);
+                String[] storedFields = new String[numStoredFields];
+                StringBuilder storedFieldsParam = new StringBuilder();
+                for (int i = 0; i < numStoredFields; i++) {
+                    String storedField = randomAlphaOfLengthBetween(3, 10);
+                    storedFields[i] = storedField;
+                    storedFieldsParam.append(storedField);
+                    if (i < numStoredFields - 1) {
+                        storedFieldsParam.append(",");
+                    }
+                }
+                getRequest.storedFields(storedFields);
+                expectedParams.put("stored_fields", storedFieldsParam.toString());
+            }
+            if (randomBoolean()) {
+                randomizeFetchSourceContextParams(getRequest::fetchSourceContext, expectedParams);
+            }
+        }
+        Request request = requestConverter.apply(getRequest);
+        assertEquals("/" + index + "/" + type + "/" + id, request.getEndpoint());
+        assertEquals(expectedParams, request.getParameters());
+        assertNull(request.getEntity());
+        assertEquals(method, request.getMethod());
+    }
+
+    public void testIndex() throws IOException {
+        String index = randomAlphaOfLengthBetween(3, 10);
+        String type = randomAlphaOfLengthBetween(3, 10);
+        IndexRequest indexRequest = new IndexRequest(index, type);
+
+        String id = randomBoolean() ? randomAlphaOfLengthBetween(3, 10) : null;
+        indexRequest.id(id);
+
+        Map<String, String> expectedParams = new HashMap<>();
+
+        String method = "POST";
+        if (id != null) {
+            method = "PUT";
+            if (randomBoolean()) {
+                indexRequest.opType(DocWriteRequest.OpType.CREATE);
+            }
+        }
+
+        setRandomTimeout(indexRequest, expectedParams);
+        setRandomRefreshPolicy(indexRequest, expectedParams);
+
+        // There is some logic around _create endpoint and version/version type
+        if (indexRequest.opType() == DocWriteRequest.OpType.CREATE) {
+            indexRequest.version(randomFrom(Versions.MATCH_ANY, Versions.MATCH_DELETED));
+            expectedParams.put("version", Long.toString(Versions.MATCH_DELETED));
+        } else {
+            setRandomVersion(indexRequest, expectedParams);
+            setRandomVersionType(indexRequest, expectedParams);
+        }
+
+        if (frequently()) {
+            if (randomBoolean()) {
+                String routing = randomAlphaOfLengthBetween(3, 10);
+                indexRequest.routing(routing);
+                expectedParams.put("routing", routing);
+            }
+            if (randomBoolean()) {
+                String parent = randomAlphaOfLengthBetween(3, 10);
+                indexRequest.parent(parent);
+                expectedParams.put("parent", parent);
+            }
+            if (randomBoolean()) {
+                String pipeline = randomAlphaOfLengthBetween(3, 10);
+                indexRequest.setPipeline(pipeline);
+                expectedParams.put("pipeline", pipeline);
+            }
+        }
+
+        XContentType xContentType = randomFrom(XContentType.values());
+        int nbFields = randomIntBetween(0, 10);
+        try (XContentBuilder builder = XContentBuilder.builder(xContentType.xContent())) {
+            builder.startObject();
+            for (int i = 0; i < nbFields; i++) {
+                builder.field("field_" + i, i);
+            }
+            builder.endObject();
+            indexRequest.source(builder);
+        }
+
+        Request request = Request.index(indexRequest);
+        if (indexRequest.opType() == DocWriteRequest.OpType.CREATE) {
+            assertEquals("/" + index + "/" + type + "/" + id + "/_create", request.getEndpoint());
+        } else if (id != null) {
+            assertEquals("/" + index + "/" + type + "/" + id, request.getEndpoint());
+        } else {
+            assertEquals("/" + index + "/" + type, request.getEndpoint());
+        }
+        assertEquals(expectedParams, request.getParameters());
+        assertEquals(method, request.getMethod());
+
+        HttpEntity entity = request.getEntity();
+        assertTrue(entity instanceof ByteArrayEntity);
+        assertEquals(indexRequest.getContentType().mediaTypeWithoutParameters(), entity.getContentType().getValue());
+        try (XContentParser parser = createParser(xContentType.xContent(), entity.getContent())) {
+            assertEquals(nbFields, parser.map().size());
+        }
+    }
+
+    public void testUpdate() throws IOException {
+        XContentType xContentType = randomFrom(XContentType.values());
+
+        Map<String, String> expectedParams = new HashMap<>();
+        String index = randomAlphaOfLengthBetween(3, 10);
+        String type = randomAlphaOfLengthBetween(3, 10);
+        String id = randomAlphaOfLengthBetween(3, 10);
+
+        UpdateRequest updateRequest = new UpdateRequest(index, type, id);
+        updateRequest.detectNoop(randomBoolean());
+
+        if (randomBoolean()) {
+            BytesReference source = RandomObjects.randomSource(random(), xContentType);
+            updateRequest.doc(new IndexRequest().source(source, xContentType));
+
+            boolean docAsUpsert = randomBoolean();
+            updateRequest.docAsUpsert(docAsUpsert);
+            if (docAsUpsert) {
+                expectedParams.put("doc_as_upsert", "true");
+            }
+        } else {
+            updateRequest.script(mockScript("_value + 1"));
+            updateRequest.scriptedUpsert(randomBoolean());
+        }
+        if (randomBoolean()) {
+            BytesReference source = RandomObjects.randomSource(random(), xContentType);
+            updateRequest.upsert(new IndexRequest().source(source, xContentType));
+        }
+        if (randomBoolean()) {
+            String routing = randomAlphaOfLengthBetween(3, 10);
+            updateRequest.routing(routing);
+            expectedParams.put("routing", routing);
+        }
+        if (randomBoolean()) {
+            String parent = randomAlphaOfLengthBetween(3, 10);
+            updateRequest.parent(parent);
+            expectedParams.put("parent", parent);
+        }
+        if (randomBoolean()) {
+            String timeout = randomTimeValue();
+            updateRequest.timeout(timeout);
+            expectedParams.put("timeout", timeout);
+        } else {
+            expectedParams.put("timeout", ReplicationRequest.DEFAULT_TIMEOUT.getStringRep());
+        }
+        if (randomBoolean()) {
+            WriteRequest.RefreshPolicy refreshPolicy = randomFrom(WriteRequest.RefreshPolicy.values());
+            updateRequest.setRefreshPolicy(refreshPolicy);
+            if (refreshPolicy != WriteRequest.RefreshPolicy.NONE) {
+                expectedParams.put("refresh", refreshPolicy.getValue());
+            }
+        }
+        if (randomBoolean()) {
+            int waitForActiveShards = randomIntBetween(0, 10);
+            updateRequest.waitForActiveShards(waitForActiveShards);
+            expectedParams.put("wait_for_active_shards", String.valueOf(waitForActiveShards));
+        }
+        if (randomBoolean()) {
+            long version = randomLong();
+            updateRequest.version(version);
+            if (version != Versions.MATCH_ANY) {
+                expectedParams.put("version", Long.toString(version));
+            }
+        }
+        if (randomBoolean()) {
+            VersionType versionType = randomFrom(VersionType.values());
+            updateRequest.versionType(versionType);
+            if (versionType != VersionType.INTERNAL) {
+                expectedParams.put("version_type", versionType.name().toLowerCase(Locale.ROOT));
+            }
+        }
+        if (randomBoolean()) {
+            int retryOnConflict = randomIntBetween(0, 5);
+            updateRequest.retryOnConflict(retryOnConflict);
+            if (retryOnConflict > 0) {
+                expectedParams.put("retry_on_conflict", String.valueOf(retryOnConflict));
+            }
+        }
+        if (randomBoolean()) {
+            randomizeFetchSourceContextParams(updateRequest::fetchSource, expectedParams);
+        }
+
+        Request request = Request.update(updateRequest);
+        assertEquals("/" + index + "/" + type + "/" + id + "/_update", request.getEndpoint());
+        assertEquals(expectedParams, request.getParameters());
+        assertEquals("POST", request.getMethod());
+
+        HttpEntity entity = request.getEntity();
+        assertTrue(entity instanceof ByteArrayEntity);
+
+        UpdateRequest parsedUpdateRequest = new UpdateRequest();
+
+        XContentType entityContentType = XContentType.fromMediaTypeOrFormat(entity.getContentType().getValue());
+        try (XContentParser parser = createParser(entityContentType.xContent(), entity.getContent())) {
+            parsedUpdateRequest.fromXContent(parser);
+        }
+
+        assertEquals(updateRequest.scriptedUpsert(), parsedUpdateRequest.scriptedUpsert());
+        assertEquals(updateRequest.docAsUpsert(), parsedUpdateRequest.docAsUpsert());
+        assertEquals(updateRequest.detectNoop(), parsedUpdateRequest.detectNoop());
+        assertEquals(updateRequest.fetchSource(), parsedUpdateRequest.fetchSource());
+        assertEquals(updateRequest.script(), parsedUpdateRequest.script());
+        if (updateRequest.doc() != null) {
+            assertToXContentEquivalent(updateRequest.doc().source(), parsedUpdateRequest.doc().source(), xContentType);
+        } else {
+            assertNull(parsedUpdateRequest.doc());
+        }
+        if (updateRequest.upsertRequest() != null) {
+            assertToXContentEquivalent(updateRequest.upsertRequest().source(), parsedUpdateRequest.upsertRequest().source(), xContentType);
+        } else {
+            assertNull(parsedUpdateRequest.upsertRequest());
+        }
+    }
+
+    public void testUpdateWithDifferentContentTypes() throws IOException {
+        IllegalStateException exception = expectThrows(IllegalStateException.class, () -> {
+            UpdateRequest updateRequest = new UpdateRequest();
+            updateRequest.doc(new IndexRequest().source(singletonMap("field", "doc"), XContentType.JSON));
+            updateRequest.upsert(new IndexRequest().source(singletonMap("field", "upsert"), XContentType.YAML));
+            Request.update(updateRequest);
+        });
+        assertEquals("Update request cannot have different content types for doc [JSON] and upsert [YAML] documents",
+                exception.getMessage());
+    }
+
+    public void testBulk() throws IOException {
+        Map<String, String> expectedParams = new HashMap<>();
+
+        BulkRequest bulkRequest = new BulkRequest();
+        if (randomBoolean()) {
+            String timeout = randomTimeValue();
+            bulkRequest.timeout(timeout);
+            expectedParams.put("timeout", timeout);
+        } else {
+            expectedParams.put("timeout", BulkShardRequest.DEFAULT_TIMEOUT.getStringRep());
+        }
+
+        if (randomBoolean()) {
+            WriteRequest.RefreshPolicy refreshPolicy = randomFrom(WriteRequest.RefreshPolicy.values());
+            bulkRequest.setRefreshPolicy(refreshPolicy);
+            if (refreshPolicy != WriteRequest.RefreshPolicy.NONE) {
+                expectedParams.put("refresh", refreshPolicy.getValue());
+            }
+        }
+
+        XContentType xContentType = randomFrom(XContentType.JSON, XContentType.SMILE);
+
+        int nbItems = randomIntBetween(10, 100);
+        for (int i = 0; i < nbItems; i++) {
+            String index = randomAlphaOfLength(5);
+            String type = randomAlphaOfLength(5);
+            String id = randomAlphaOfLength(5);
+
+            BytesReference source = RandomObjects.randomSource(random(), xContentType);
+            DocWriteRequest.OpType opType = randomFrom(DocWriteRequest.OpType.values());
+
+            DocWriteRequest<?> docWriteRequest = null;
+            if (opType == DocWriteRequest.OpType.INDEX) {
+                IndexRequest indexRequest = new IndexRequest(index, type, id).source(source, xContentType);
+                docWriteRequest = indexRequest;
+                if (randomBoolean()) {
+                    indexRequest.setPipeline(randomAlphaOfLength(5));
+                }
+                if (randomBoolean()) {
+                    indexRequest.parent(randomAlphaOfLength(5));
+                }
+            } else if (opType == DocWriteRequest.OpType.CREATE) {
+                IndexRequest createRequest = new IndexRequest(index, type, id).source(source, xContentType).create(true);
+                docWriteRequest = createRequest;
+                if (randomBoolean()) {
+                    createRequest.parent(randomAlphaOfLength(5));
+                }
+            } else if (opType == DocWriteRequest.OpType.UPDATE) {
+                final UpdateRequest updateRequest = new UpdateRequest(index, type, id).doc(new IndexRequest().source(source, xContentType));
+                docWriteRequest = updateRequest;
+                if (randomBoolean()) {
+                    updateRequest.retryOnConflict(randomIntBetween(1, 5));
+                }
+                if (randomBoolean()) {
+                    randomizeFetchSourceContextParams(updateRequest::fetchSource, new HashMap<>());
+                }
+                if (randomBoolean()) {
+                    updateRequest.parent(randomAlphaOfLength(5));
+                }
+            } else if (opType == DocWriteRequest.OpType.DELETE) {
+                docWriteRequest = new DeleteRequest(index, type, id);
+            }
+
+            if (randomBoolean()) {
+                docWriteRequest.routing(randomAlphaOfLength(10));
+            }
+            if (randomBoolean()) {
+                docWriteRequest.version(randomNonNegativeLong());
+            }
+            if (randomBoolean()) {
+                docWriteRequest.versionType(randomFrom(VersionType.values()));
+            }
+            bulkRequest.add(docWriteRequest);
+        }
+
+        Request request = Request.bulk(bulkRequest);
+        assertEquals("/_bulk", request.getEndpoint());
+        assertEquals(expectedParams, request.getParameters());
+        assertEquals("POST", request.getMethod());
+        assertEquals(xContentType.mediaTypeWithoutParameters(), request.getEntity().getContentType().getValue());
+        byte[] content = new byte[(int) request.getEntity().getContentLength()];
+        try (InputStream inputStream = request.getEntity().getContent()) {
+            Streams.readFully(inputStream, content);
+        }
+
+        BulkRequest parsedBulkRequest = new BulkRequest();
+        parsedBulkRequest.add(content, 0, content.length, xContentType);
+        assertEquals(bulkRequest.numberOfActions(), parsedBulkRequest.numberOfActions());
+
+        for (int i = 0; i < bulkRequest.numberOfActions(); i++) {
+            DocWriteRequest<?> originalRequest = bulkRequest.requests().get(i);
+            DocWriteRequest<?> parsedRequest = parsedBulkRequest.requests().get(i);
+
+            assertEquals(originalRequest.opType(), parsedRequest.opType());
+            assertEquals(originalRequest.index(), parsedRequest.index());
+            assertEquals(originalRequest.type(), parsedRequest.type());
+            assertEquals(originalRequest.id(), parsedRequest.id());
+            assertEquals(originalRequest.routing(), parsedRequest.routing());
+            assertEquals(originalRequest.parent(), parsedRequest.parent());
+            assertEquals(originalRequest.version(), parsedRequest.version());
+            assertEquals(originalRequest.versionType(), parsedRequest.versionType());
+
+            DocWriteRequest.OpType opType = originalRequest.opType();
+            if (opType == DocWriteRequest.OpType.INDEX) {
+                IndexRequest indexRequest = (IndexRequest) originalRequest;
+                IndexRequest parsedIndexRequest = (IndexRequest) parsedRequest;
+
+                assertEquals(indexRequest.getPipeline(), parsedIndexRequest.getPipeline());
+                assertToXContentEquivalent(indexRequest.source(), parsedIndexRequest.source(), xContentType);
+            } else if (opType == DocWriteRequest.OpType.UPDATE) {
+                UpdateRequest updateRequest = (UpdateRequest) originalRequest;
+                UpdateRequest parsedUpdateRequest = (UpdateRequest) parsedRequest;
+
+                assertEquals(updateRequest.retryOnConflict(), parsedUpdateRequest.retryOnConflict());
+                assertEquals(updateRequest.fetchSource(), parsedUpdateRequest.fetchSource());
+                if (updateRequest.doc() != null) {
+                    assertToXContentEquivalent(updateRequest.doc().source(), parsedUpdateRequest.doc().source(), xContentType);
+                } else {
+                    assertNull(parsedUpdateRequest.doc());
+                }
+            }
+        }
+    }
+
+    public void testBulkWithDifferentContentTypes() throws IOException {
+        {
+            BulkRequest bulkRequest = new BulkRequest();
+            bulkRequest.add(new DeleteRequest("index", "type", "0"));
+            bulkRequest.add(new UpdateRequest("index", "type", "1").script(mockScript("test")));
+            bulkRequest.add(new DeleteRequest("index", "type", "2"));
+
+            Request request = Request.bulk(bulkRequest);
+            assertEquals(XContentType.JSON.mediaTypeWithoutParameters(), request.getEntity().getContentType().getValue());
+        }
+        {
+            XContentType xContentType = randomFrom(XContentType.JSON, XContentType.SMILE);
+            BulkRequest bulkRequest = new BulkRequest();
+            bulkRequest.add(new DeleteRequest("index", "type", "0"));
+            bulkRequest.add(new IndexRequest("index", "type", "0").source(singletonMap("field", "value"), xContentType));
+            bulkRequest.add(new DeleteRequest("index", "type", "2"));
+
+            Request request = Request.bulk(bulkRequest);
+            assertEquals(xContentType.mediaTypeWithoutParameters(), request.getEntity().getContentType().getValue());
+        }
+        {
+            XContentType xContentType = randomFrom(XContentType.JSON, XContentType.SMILE);
+            UpdateRequest updateRequest = new UpdateRequest("index", "type", "0");
+            if (randomBoolean()) {
+                updateRequest.doc(new IndexRequest().source(singletonMap("field", "value"), xContentType));
+            } else {
+                updateRequest.upsert(new IndexRequest().source(singletonMap("field", "value"), xContentType));
+            }
+
+            Request request = Request.bulk(new BulkRequest().add(updateRequest));
+            assertEquals(xContentType.mediaTypeWithoutParameters(), request.getEntity().getContentType().getValue());
+        }
+        {
+            BulkRequest bulkRequest = new BulkRequest();
+            bulkRequest.add(new IndexRequest("index", "type", "0").source(singletonMap("field", "value"), XContentType.SMILE));
+            bulkRequest.add(new IndexRequest("index", "type", "1").source(singletonMap("field", "value"), XContentType.JSON));
+            IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> Request.bulk(bulkRequest));
+            assertEquals("Mismatching content-type found for request with content-type [JSON], " +
+                            "previous requests have content-type [SMILE]", exception.getMessage());
+        }
+        {
+            BulkRequest bulkRequest = new BulkRequest();
+            bulkRequest.add(new IndexRequest("index", "type", "0")
+                    .source(singletonMap("field", "value"), XContentType.JSON));
+            bulkRequest.add(new IndexRequest("index", "type", "1")
+                    .source(singletonMap("field", "value"), XContentType.JSON));
+            bulkRequest.add(new UpdateRequest("index", "type", "2")
+                    .doc(new IndexRequest().source(singletonMap("field", "value"), XContentType.JSON))
+                    .upsert(new IndexRequest().source(singletonMap("field", "value"), XContentType.SMILE))
+            );
+            IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> Request.bulk(bulkRequest));
+            assertEquals("Mismatching content-type found for request with content-type [SMILE], " +
+                            "previous requests have content-type [JSON]", exception.getMessage());
+        }
+        {
+            XContentType xContentType = randomFrom(XContentType.CBOR, XContentType.YAML);
+            BulkRequest bulkRequest = new BulkRequest();
+            bulkRequest.add(new DeleteRequest("index", "type", "0"));
+            bulkRequest.add(new IndexRequest("index", "type", "1").source(singletonMap("field", "value"), XContentType.JSON));
+            bulkRequest.add(new DeleteRequest("index", "type", "2"));
+            bulkRequest.add(new DeleteRequest("index", "type", "3"));
+            bulkRequest.add(new IndexRequest("index", "type", "4").source(singletonMap("field", "value"), XContentType.JSON));
+            bulkRequest.add(new IndexRequest("index", "type", "1").source(singletonMap("field", "value"), xContentType));
+            IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> Request.bulk(bulkRequest));
+            assertEquals("Unsupported content-type found for request with content-type [" + xContentType
+                    + "], only JSON and SMILE are supported", exception.getMessage());
+        }
+    }
+
+    public void testSearch() throws Exception {
+        SearchRequest searchRequest = new SearchRequest();
+        int numIndices = randomIntBetween(0, 5);
+        String[] indices = new String[numIndices];
+        for (int i = 0; i < numIndices; i++) {
+            indices[i] = "index-" + randomAlphaOfLengthBetween(2, 5);
+        }
+        searchRequest.indices(indices);
+        int numTypes = randomIntBetween(0, 5);
+        String[] types = new String[numTypes];
+        for (int i = 0; i < numTypes; i++) {
+            types[i] = "type-" + randomAlphaOfLengthBetween(2, 5);
+        }
+        searchRequest.types(types);
+
+        Map<String, String> expectedParams = new HashMap<>();
+        expectedParams.put(RestSearchAction.TYPED_KEYS_PARAM, "true");
+        if (randomBoolean()) {
+            searchRequest.routing(randomAlphaOfLengthBetween(3, 10));
+            expectedParams.put("routing", searchRequest.routing());
+        }
+        if (randomBoolean()) {
+            searchRequest.preference(randomAlphaOfLengthBetween(3, 10));
+            expectedParams.put("preference", searchRequest.preference());
+        }
+        if (randomBoolean()) {
+            searchRequest.searchType(randomFrom(SearchType.values()));
+        }
+        expectedParams.put("search_type", searchRequest.searchType().name().toLowerCase(Locale.ROOT));
+        if (randomBoolean()) {
+            searchRequest.requestCache(randomBoolean());
+            expectedParams.put("request_cache", Boolean.toString(searchRequest.requestCache()));
+        }
+        if (randomBoolean()) {
+            searchRequest.setBatchedReduceSize(randomIntBetween(2, Integer.MAX_VALUE));
+        }
+        expectedParams.put("batched_reduce_size", Integer.toString(searchRequest.getBatchedReduceSize()));
+        if (randomBoolean()) {
+            searchRequest.scroll(randomTimeValue());
+            expectedParams.put("scroll", searchRequest.scroll().keepAlive().getStringRep());
+        }
+
+        if (randomBoolean()) {
+            searchRequest.indicesOptions(IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()));
+        }
+        expectedParams.put("ignore_unavailable", Boolean.toString(searchRequest.indicesOptions().ignoreUnavailable()));
+        expectedParams.put("allow_no_indices", Boolean.toString(searchRequest.indicesOptions().allowNoIndices()));
+        if (searchRequest.indicesOptions().expandWildcardsOpen() && searchRequest.indicesOptions().expandWildcardsClosed()) {
+            expectedParams.put("expand_wildcards", "open,closed");
+        } else if (searchRequest.indicesOptions().expandWildcardsOpen()) {
+            expectedParams.put("expand_wildcards", "open");
+        } else if (searchRequest.indicesOptions().expandWildcardsClosed()) {
+            expectedParams.put("expand_wildcards", "closed");
+        } else {
+            expectedParams.put("expand_wildcards", "none");
+        }
+
+        SearchSourceBuilder searchSourceBuilder = null;
+        if (frequently()) {
+            searchSourceBuilder = new SearchSourceBuilder();
+            if (randomBoolean()) {
+                searchSourceBuilder.size(randomIntBetween(0, Integer.MAX_VALUE));
+            }
+            if (randomBoolean()) {
+                searchSourceBuilder.from(randomIntBetween(0, Integer.MAX_VALUE));
+            }
+            if (randomBoolean()) {
+                searchSourceBuilder.minScore(randomFloat());
+            }
+            if (randomBoolean()) {
+                searchSourceBuilder.explain(randomBoolean());
+            }
+            if (randomBoolean()) {
+                searchSourceBuilder.profile(randomBoolean());
+            }
+            if (randomBoolean()) {
+                searchSourceBuilder.highlighter(new HighlightBuilder().field(randomAlphaOfLengthBetween(3, 10)));
+            }
+            if (randomBoolean()) {
+                searchSourceBuilder.query(new TermQueryBuilder(randomAlphaOfLengthBetween(3, 10), randomAlphaOfLengthBetween(3, 10)));
+            }
+            if (randomBoolean()) {
+                searchSourceBuilder.aggregation(new TermsAggregationBuilder(randomAlphaOfLengthBetween(3, 10), ValueType.STRING)
+                        .field(randomAlphaOfLengthBetween(3, 10)));
+            }
+            if (randomBoolean()) {
+                searchSourceBuilder.suggest(new SuggestBuilder().addSuggestion(randomAlphaOfLengthBetween(3, 10),
+                        new CompletionSuggestionBuilder(randomAlphaOfLengthBetween(3, 10))));
+            }
+            if (randomBoolean()) {
+                searchSourceBuilder.addRescorer(new QueryRescorerBuilder(
+                        new TermQueryBuilder(randomAlphaOfLengthBetween(3, 10), randomAlphaOfLengthBetween(3, 10))));
+            }
+            if (randomBoolean()) {
+                searchSourceBuilder.collapse(new CollapseBuilder(randomAlphaOfLengthBetween(3, 10)));
+            }
+            searchRequest.source(searchSourceBuilder);
+        }
+
+        Request request = Request.search(searchRequest);
+        StringJoiner endpoint = new StringJoiner("/", "/", "");
+        String index = String.join(",", indices);
+        if (Strings.hasLength(index)) {
+            endpoint.add(index);
+        }
+        String type = String.join(",", types);
+        if (Strings.hasLength(type)) {
+            endpoint.add(type);
+        }
+        endpoint.add("_search");
+        assertEquals(endpoint.toString(), request.getEndpoint());
+        assertEquals(expectedParams, request.getParameters());
+        if (searchSourceBuilder == null) {
+            assertNull(request.getEntity());
+        } else {
+            assertToXContentBody(searchSourceBuilder, request.getEntity());
+        }
+    }
+
+    public void testSearchScroll() throws IOException {
+        SearchScrollRequest searchScrollRequest = new SearchScrollRequest();
+        searchScrollRequest.scrollId(randomAlphaOfLengthBetween(5, 10));
+        if (randomBoolean()) {
+            searchScrollRequest.scroll(randomPositiveTimeValue());
+        }
+        Request request = Request.searchScroll(searchScrollRequest);
+        assertEquals("GET", request.getMethod());
+        assertEquals("/_search/scroll", request.getEndpoint());
+        assertEquals(0, request.getParameters().size());
+        assertToXContentBody(searchScrollRequest, request.getEntity());
+        assertEquals(Request.REQUEST_BODY_CONTENT_TYPE.mediaTypeWithoutParameters(), request.getEntity().getContentType().getValue());
+    }
+
+    public void testClearScroll() throws IOException {
+        ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
+        int numScrolls = randomIntBetween(1, 10);
+        for (int i = 0; i < numScrolls; i++) {
+            clearScrollRequest.addScrollId(randomAlphaOfLengthBetween(5, 10));
+        }
+        Request request = Request.clearScroll(clearScrollRequest);
+        assertEquals("DELETE", request.getMethod());
+        assertEquals("/_search/scroll", request.getEndpoint());
+        assertEquals(0, request.getParameters().size());
+        assertToXContentBody(clearScrollRequest, request.getEntity());
+        assertEquals(Request.REQUEST_BODY_CONTENT_TYPE.mediaTypeWithoutParameters(), request.getEntity().getContentType().getValue());
+    }
+
+    private static void assertToXContentBody(ToXContent expectedBody, HttpEntity actualEntity) throws IOException {
+        BytesReference expectedBytes = XContentHelper.toXContent(expectedBody, Request.REQUEST_BODY_CONTENT_TYPE, false);
+        assertEquals(XContentType.JSON.mediaTypeWithoutParameters(), actualEntity.getContentType().getValue());
+        assertEquals(expectedBytes, new BytesArray(EntityUtils.toByteArray(actualEntity)));
+    }
+
+    public void testParams() {
+        final int nbParams = randomIntBetween(0, 10);
+        Request.Params params = Request.Params.builder();
+        Map<String, String> expectedParams = new HashMap<>();
+        for (int i = 0; i < nbParams; i++) {
+            String paramName = "p_" + i;
+            String paramValue = randomAlphaOfLength(5);
+            params.putParam(paramName, paramValue);
+            expectedParams.put(paramName, paramValue);
+        }
+
+        Map<String, String> requestParams = params.getParams();
+        assertEquals(nbParams, requestParams.size());
+        assertEquals(expectedParams, requestParams);
+    }
+
+    public void testParamsNoDuplicates() {
+        Request.Params params = Request.Params.builder();
+        params.putParam("test", "1");
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> params.putParam("test", "2"));
+        assertEquals("Request parameter [test] is already registered", e.getMessage());
+
+        Map<String, String> requestParams = params.getParams();
+        assertEquals(1L, requestParams.size());
+        assertEquals("1", requestParams.values().iterator().next());
+    }
+
+    public void testEndpoint() {
+        assertEquals("/", Request.endpoint());
+        assertEquals("/", Request.endpoint(Strings.EMPTY_ARRAY));
+        assertEquals("/", Request.endpoint(""));
+        assertEquals("/a/b", Request.endpoint("a", "b"));
+        assertEquals("/a/b/_create", Request.endpoint("a", "b", "_create"));
+        assertEquals("/a/b/c/_create", Request.endpoint("a", "b", "c", "_create"));
+        assertEquals("/a/_create", Request.endpoint("a", null, null, "_create"));
+    }
+
+    public void testCreateContentType() {
+        final XContentType xContentType = randomFrom(XContentType.values());
+        assertEquals(xContentType.mediaTypeWithoutParameters(), Request.createContentType(xContentType).getMimeType());
+    }
+
+    public void testEnforceSameContentType() {
+        XContentType xContentType = randomFrom(XContentType.JSON, XContentType.SMILE);
+        IndexRequest indexRequest = new IndexRequest().source(singletonMap("field", "value"), xContentType);
+        assertEquals(xContentType, enforceSameContentType(indexRequest, null));
+        assertEquals(xContentType, enforceSameContentType(indexRequest, xContentType));
+
+        XContentType bulkContentType = randomBoolean() ? xContentType : null;
+
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () ->
+                enforceSameContentType(new IndexRequest().source(singletonMap("field", "value"), XContentType.CBOR), bulkContentType));
+        assertEquals("Unsupported content-type found for request with content-type [CBOR], only JSON and SMILE are supported",
+                exception.getMessage());
+
+        exception = expectThrows(IllegalArgumentException.class, () ->
+                enforceSameContentType(new IndexRequest().source(singletonMap("field", "value"), XContentType.YAML), bulkContentType));
+        assertEquals("Unsupported content-type found for request with content-type [YAML], only JSON and SMILE are supported",
+                exception.getMessage());
+
+        XContentType requestContentType = xContentType == XContentType.JSON ? XContentType.SMILE : XContentType.JSON;
+
+        exception = expectThrows(IllegalArgumentException.class, () ->
+                enforceSameContentType(new IndexRequest().source(singletonMap("field", "value"), requestContentType), xContentType));
+        assertEquals("Mismatching content-type found for request with content-type [" + requestContentType + "], "
+                + "previous requests have content-type [" + xContentType + "]", exception.getMessage());
+    }
+
+    /**
+     * Randomize the {@link FetchSourceContext} request parameters.
+     */
+    private static void randomizeFetchSourceContextParams(Consumer<FetchSourceContext> consumer, Map<String, String> expectedParams) {
+        if (randomBoolean()) {
+            if (randomBoolean()) {
+                boolean fetchSource = randomBoolean();
+                consumer.accept(new FetchSourceContext(fetchSource));
+                if (fetchSource == false) {
+                    expectedParams.put("_source", "false");
+                }
+            } else {
+                int numIncludes = randomIntBetween(0, 5);
+                String[] includes = new String[numIncludes];
+                StringBuilder includesParam = new StringBuilder();
+                for (int i = 0; i < numIncludes; i++) {
+                    String include = randomAlphaOfLengthBetween(3, 10);
+                    includes[i] = include;
+                    includesParam.append(include);
+                    if (i < numIncludes - 1) {
+                        includesParam.append(",");
+                    }
+                }
+                if (numIncludes > 0) {
+                    expectedParams.put("_source_include", includesParam.toString());
+                }
+                int numExcludes = randomIntBetween(0, 5);
+                String[] excludes = new String[numExcludes];
+                StringBuilder excludesParam = new StringBuilder();
+                for (int i = 0; i < numExcludes; i++) {
+                    String exclude = randomAlphaOfLengthBetween(3, 10);
+                    excludes[i] = exclude;
+                    excludesParam.append(exclude);
+                    if (i < numExcludes - 1) {
+                        excludesParam.append(",");
+                    }
+                }
+                if (numExcludes > 0) {
+                    expectedParams.put("_source_exclude", excludesParam.toString());
+                }
+                consumer.accept(new FetchSourceContext(true, includes, excludes));
+            }
+        }
+    }
+
+    private static void setRandomTimeout(ReplicationRequest<?> request, Map<String, String> expectedParams) {
+        if (randomBoolean()) {
+            String timeout = randomTimeValue();
+            request.timeout(timeout);
+            expectedParams.put("timeout", timeout);
+        } else {
+            expectedParams.put("timeout", ReplicationRequest.DEFAULT_TIMEOUT.getStringRep());
+        }
+    }
+
+    private static void setRandomRefreshPolicy(ReplicatedWriteRequest<?> request, Map<String, String> expectedParams) {
+        if (randomBoolean()) {
+            WriteRequest.RefreshPolicy refreshPolicy = randomFrom(WriteRequest.RefreshPolicy.values());
+            request.setRefreshPolicy(refreshPolicy);
+            if (refreshPolicy != WriteRequest.RefreshPolicy.NONE) {
+                expectedParams.put("refresh", refreshPolicy.getValue());
+            }
+        }
+    }
+
+    private static void setRandomVersion(DocWriteRequest<?> request, Map<String, String> expectedParams) {
+        if (randomBoolean()) {
+            long version = randomFrom(Versions.MATCH_ANY, Versions.MATCH_DELETED, Versions.NOT_FOUND, randomNonNegativeLong());
+            request.version(version);
+            if (version != Versions.MATCH_ANY) {
+                expectedParams.put("version", Long.toString(version));
+            }
+        }
+    }
+
+    private static void setRandomVersionType(DocWriteRequest<?> request, Map<String, String> expectedParams) {
+        if (randomBoolean()) {
+            VersionType versionType = randomFrom(VersionType.values());
+            request.versionType(versionType);
+            if (versionType != VersionType.INTERNAL) {
+                expectedParams.put("version_type", versionType.name().toLowerCase(Locale.ROOT));
+            }
+        }
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientExtTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientExtTests.java
new file mode 100644
index 0000000000000..8c12cbeb1e563
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientExtTests.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import org.apache.http.HttpEntity;
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.StringEntity;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.mockito.Mockito.mock;
+
+/**
+ * This test works against a {@link RestHighLevelClient} subclass that simulates how custom response sections returned by
+ * Elasticsearch plugins can be parsed using the high level client.
+ */
+public class RestHighLevelClientExtTests extends ESTestCase {
+
+    private RestHighLevelClient restHighLevelClient;
+
+    @Before
+    public void initClient() throws IOException {
+        RestClient restClient = mock(RestClient.class);
+        restHighLevelClient = new RestHighLevelClientExt(restClient);
+    }
+
+    public void testParseEntityCustomResponseSection() throws IOException {
+        {
+            HttpEntity jsonEntity = new StringEntity("{\"custom1\":{ \"field\":\"value\"}}", ContentType.APPLICATION_JSON);
+            BaseCustomResponseSection customSection = restHighLevelClient.parseEntity(jsonEntity, BaseCustomResponseSection::fromXContent);
+            assertThat(customSection, instanceOf(CustomResponseSection1.class));
+            CustomResponseSection1 customResponseSection1 = (CustomResponseSection1) customSection;
+            assertEquals("value", customResponseSection1.value);
+        }
+        {
+            HttpEntity jsonEntity = new StringEntity("{\"custom2\":{ \"array\": [\"item1\", \"item2\"]}}", ContentType.APPLICATION_JSON);
+            BaseCustomResponseSection customSection = restHighLevelClient.parseEntity(jsonEntity, BaseCustomResponseSection::fromXContent);
+            assertThat(customSection, instanceOf(CustomResponseSection2.class));
+            CustomResponseSection2 customResponseSection2 = (CustomResponseSection2) customSection;
+            assertArrayEquals(new String[]{"item1", "item2"}, customResponseSection2.values);
+        }
+    }
+
+    private static class RestHighLevelClientExt extends RestHighLevelClient {
+
+        private RestHighLevelClientExt(RestClient restClient) {
+            super(restClient, getNamedXContentsExt());
+        }
+
+        private static List<NamedXContentRegistry.Entry> getNamedXContentsExt() {
+            List<NamedXContentRegistry.Entry> entries = new ArrayList<>();
+            entries.add(new NamedXContentRegistry.Entry(BaseCustomResponseSection.class, new ParseField("custom1"),
+                    CustomResponseSection1::fromXContent));
+            entries.add(new NamedXContentRegistry.Entry(BaseCustomResponseSection.class, new ParseField("custom2"),
+                    CustomResponseSection2::fromXContent));
+            return entries;
+        }
+    }
+
+    private abstract static class BaseCustomResponseSection {
+
+        static BaseCustomResponseSection fromXContent(XContentParser parser) throws IOException {
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
+            BaseCustomResponseSection custom = parser.namedObject(BaseCustomResponseSection.class, parser.currentName(), null);
+            assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
+            return custom;
+        }
+    }
+
+    private static class CustomResponseSection1 extends BaseCustomResponseSection {
+
+        private final String value;
+
+        private CustomResponseSection1(String value) {
+            this.value = value;
+        }
+
+        static CustomResponseSection1 fromXContent(XContentParser parser) throws IOException {
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
+            assertEquals("field", parser.currentName());
+            assertEquals(XContentParser.Token.VALUE_STRING, parser.nextToken());
+            CustomResponseSection1 responseSection1 = new CustomResponseSection1(parser.text());
+            assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
+            return responseSection1;
+        }
+    }
+
+    private static class CustomResponseSection2 extends BaseCustomResponseSection {
+
+        private final String[] values;
+
+        private CustomResponseSection2(String[] values) {
+            this.values = values;
+        }
+
+        static CustomResponseSection2 fromXContent(XContentParser parser) throws IOException {
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
+            assertEquals("array", parser.currentName());
+            assertEquals(XContentParser.Token.START_ARRAY, parser.nextToken());
+            List<String> values = new ArrayList<>();
+            while(parser.nextToken().isValue()) {
+                values.add(parser.text());
+            }
+            assertEquals(XContentParser.Token.END_ARRAY, parser.currentToken());
+            CustomResponseSection2 responseSection2 = new CustomResponseSection2(values.toArray(new String[values.size()]));
+            assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
+            return responseSection2;
+        }
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
new file mode 100644
index 0000000000000..a6d015afca713
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
@@ -0,0 +1,691 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import org.apache.http.Header;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpHost;
+import org.apache.http.HttpResponse;
+import org.apache.http.ProtocolVersion;
+import org.apache.http.RequestLine;
+import org.apache.http.StatusLine;
+import org.apache.http.entity.ByteArrayEntity;
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.message.BasicHttpResponse;
+import org.apache.http.message.BasicRequestLine;
+import org.apache.http.message.BasicStatusLine;
+import org.apache.http.nio.entity.NStringEntity;
+
+import org.elasticsearch.Build;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.main.MainRequest;
+import org.elasticsearch.action.main.MainResponse;
+import org.elasticsearch.action.search.ClearScrollRequest;
+import org.elasticsearch.action.search.ClearScrollResponse;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.SearchResponseSections;
+import org.elasticsearch.action.search.SearchScrollRequest;
+import org.elasticsearch.action.search.ShardSearchFailure;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.common.CheckedFunction;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.cbor.CborXContent;
+import org.elasticsearch.common.xcontent.smile.SmileXContent;
+import org.elasticsearch.join.aggregations.ChildrenAggregationBuilder;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.search.aggregations.Aggregation;
+import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.aggregations.matrix.stats.MatrixStatsAggregationBuilder;
+import org.elasticsearch.search.suggest.Suggest;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.InternalAggregationTestCase;
+import org.junit.Before;
+import org.mockito.ArgumentMatcher;
+import org.mockito.internal.matchers.ArrayEquals;
+import org.mockito.internal.matchers.VarargMatcher;
+
+import java.io.IOException;
+import java.net.SocketTimeoutException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.elasticsearch.client.RestClientTestUtil.randomHeaders;
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.mockito.Matchers.anyMapOf;
+import static org.mockito.Matchers.anyObject;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Matchers.anyVararg;
+import static org.mockito.Matchers.argThat;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Matchers.isNotNull;
+import static org.mockito.Matchers.isNull;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class RestHighLevelClientTests extends ESTestCase {
+
+    private static final ProtocolVersion HTTP_PROTOCOL = new ProtocolVersion("http", 1, 1);
+    private static final RequestLine REQUEST_LINE = new BasicRequestLine("GET", "/", HTTP_PROTOCOL);
+
+    private RestClient restClient;
+    private RestHighLevelClient restHighLevelClient;
+
+    @Before
+    public void initClient() {
+        restClient = mock(RestClient.class);
+        restHighLevelClient = new RestHighLevelClient(restClient);
+    }
+
+    public void testPingSuccessful() throws IOException {
+        Header[] headers = randomHeaders(random(), "Header");
+        Response response = mock(Response.class);
+        when(response.getStatusLine()).thenReturn(newStatusLine(RestStatus.OK));
+        when(restClient.performRequest(anyString(), anyString(), anyMapOf(String.class, String.class),
+                anyObject(), anyVararg())).thenReturn(response);
+        assertTrue(restHighLevelClient.ping(headers));
+        verify(restClient).performRequest(eq("HEAD"), eq("/"), eq(Collections.emptyMap()),
+                isNull(HttpEntity.class), argThat(new HeadersVarargMatcher(headers)));
+    }
+
+    public void testPing404NotFound() throws IOException {
+        Header[] headers = randomHeaders(random(), "Header");
+        Response response = mock(Response.class);
+        when(response.getStatusLine()).thenReturn(newStatusLine(RestStatus.NOT_FOUND));
+        when(restClient.performRequest(anyString(), anyString(), anyMapOf(String.class, String.class),
+                anyObject(), anyVararg())).thenReturn(response);
+        assertFalse(restHighLevelClient.ping(headers));
+        verify(restClient).performRequest(eq("HEAD"), eq("/"), eq(Collections.emptyMap()),
+                isNull(HttpEntity.class), argThat(new HeadersVarargMatcher(headers)));
+    }
+
+    public void testPingSocketTimeout() throws IOException {
+        Header[] headers = randomHeaders(random(), "Header");
+        when(restClient.performRequest(anyString(), anyString(), anyMapOf(String.class, String.class),
+                anyObject(), anyVararg())).thenThrow(new SocketTimeoutException());
+        expectThrows(SocketTimeoutException.class, () -> restHighLevelClient.ping(headers));
+        verify(restClient).performRequest(eq("HEAD"), eq("/"), eq(Collections.emptyMap()),
+                isNull(HttpEntity.class), argThat(new HeadersVarargMatcher(headers)));
+    }
+
+    public void testInfo() throws IOException {
+        Header[] headers = randomHeaders(random(), "Header");
+        MainResponse testInfo = new MainResponse("nodeName", Version.CURRENT, new ClusterName("clusterName"), "clusterUuid",
+                Build.CURRENT, true);
+        mockResponse(testInfo);
+        MainResponse receivedInfo = restHighLevelClient.info(headers);
+        assertEquals(testInfo, receivedInfo);
+        verify(restClient).performRequest(eq("GET"), eq("/"), eq(Collections.emptyMap()),
+                isNull(HttpEntity.class), argThat(new HeadersVarargMatcher(headers)));
+    }
+
+    public void testSearchScroll() throws IOException {
+        Header[] headers = randomHeaders(random(), "Header");
+        SearchResponse mockSearchResponse = new SearchResponse(new SearchResponseSections(SearchHits.empty(), InternalAggregations.EMPTY,
+                null, false, false, null, 1), randomAlphaOfLengthBetween(5, 10), 5, 5, 0, 100, new ShardSearchFailure[0]);
+        mockResponse(mockSearchResponse);
+        SearchResponse searchResponse = restHighLevelClient.searchScroll(new SearchScrollRequest(randomAlphaOfLengthBetween(5, 10)),
+                headers);
+        assertEquals(mockSearchResponse.getScrollId(), searchResponse.getScrollId());
+        assertEquals(0, searchResponse.getHits().totalHits);
+        assertEquals(5, searchResponse.getTotalShards());
+        assertEquals(5, searchResponse.getSuccessfulShards());
+        assertEquals(100, searchResponse.getTook().getMillis());
+        verify(restClient).performRequest(eq("GET"), eq("/_search/scroll"), eq(Collections.emptyMap()),
+                isNotNull(HttpEntity.class), argThat(new HeadersVarargMatcher(headers)));
+    }
+
+    public void testClearScroll() throws IOException {
+        Header[] headers = randomHeaders(random(), "Header");
+        ClearScrollResponse mockClearScrollResponse = new ClearScrollResponse(randomBoolean(), randomIntBetween(0, Integer.MAX_VALUE));
+        mockResponse(mockClearScrollResponse);
+        ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
+        clearScrollRequest.addScrollId(randomAlphaOfLengthBetween(5, 10));
+        ClearScrollResponse clearScrollResponse = restHighLevelClient.clearScroll(clearScrollRequest, headers);
+        assertEquals(mockClearScrollResponse.isSucceeded(), clearScrollResponse.isSucceeded());
+        assertEquals(mockClearScrollResponse.getNumFreed(), clearScrollResponse.getNumFreed());
+        verify(restClient).performRequest(eq("DELETE"), eq("/_search/scroll"), eq(Collections.emptyMap()),
+                isNotNull(HttpEntity.class), argThat(new HeadersVarargMatcher(headers)));
+    }
+
+    private void mockResponse(ToXContent toXContent) throws IOException {
+        Response response = mock(Response.class);
+        ContentType contentType = ContentType.parse(Request.REQUEST_BODY_CONTENT_TYPE.mediaType());
+        String requestBody = toXContent(toXContent, Request.REQUEST_BODY_CONTENT_TYPE, false).utf8ToString();
+        when(response.getEntity()).thenReturn(new NStringEntity(requestBody, contentType));
+        when(restClient.performRequest(anyString(), anyString(), anyMapOf(String.class, String.class),
+                anyObject(), anyVararg())).thenReturn(response);
+    }
+
+    public void testRequestValidation() {
+        ActionRequestValidationException validationException = new ActionRequestValidationException();
+        validationException.addValidationError("validation error");
+        ActionRequest request = new ActionRequest() {
+            @Override
+            public ActionRequestValidationException validate() {
+                return validationException;
+            }
+        };
+
+        {
+            ActionRequestValidationException actualException = expectThrows(ActionRequestValidationException.class,
+                    () -> restHighLevelClient.performRequest(request, null, null, null));
+            assertSame(validationException, actualException);
+        }
+        {
+            TrackingActionListener trackingActionListener = new TrackingActionListener();
+            restHighLevelClient.performRequestAsync(request, null, null, trackingActionListener, null);
+            assertSame(validationException, trackingActionListener.exception.get());
+        }
+    }
+
+    public void testParseEntity() throws IOException {
+        {
+            IllegalStateException ise = expectThrows(IllegalStateException.class, () -> restHighLevelClient.parseEntity(null, null));
+            assertEquals("Response body expected but not returned", ise.getMessage());
+        }
+        {
+            IllegalStateException ise = expectThrows(IllegalStateException.class,
+                    () -> restHighLevelClient.parseEntity(new StringEntity("", (ContentType) null), null));
+            assertEquals("Elasticsearch didn't return the [Content-Type] header, unable to parse response body", ise.getMessage());
+        }
+        {
+            StringEntity entity = new StringEntity("", ContentType.APPLICATION_SVG_XML);
+            IllegalStateException ise = expectThrows(IllegalStateException.class, () -> restHighLevelClient.parseEntity(entity, null));
+            assertEquals("Unsupported Content-Type: " + entity.getContentType().getValue(), ise.getMessage());
+        }
+        {
+            CheckedFunction<XContentParser, String, IOException> entityParser = parser -> {
+                assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+                assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
+                assertTrue(parser.nextToken().isValue());
+                String value = parser.text();
+                assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
+                return value;
+            };
+            HttpEntity jsonEntity = new StringEntity("{\"field\":\"value\"}", ContentType.APPLICATION_JSON);
+            assertEquals("value", restHighLevelClient.parseEntity(jsonEntity, entityParser));
+            HttpEntity yamlEntity = new StringEntity("---\nfield: value\n", ContentType.create("application/yaml"));
+            assertEquals("value", restHighLevelClient.parseEntity(yamlEntity, entityParser));
+            HttpEntity smileEntity = createBinaryEntity(SmileXContent.contentBuilder(), ContentType.create("application/smile"));
+            assertEquals("value", restHighLevelClient.parseEntity(smileEntity, entityParser));
+            HttpEntity cborEntity = createBinaryEntity(CborXContent.contentBuilder(), ContentType.create("application/cbor"));
+            assertEquals("value", restHighLevelClient.parseEntity(cborEntity, entityParser));
+        }
+    }
+
+    private static HttpEntity createBinaryEntity(XContentBuilder xContentBuilder, ContentType contentType) throws IOException {
+        try (XContentBuilder builder = xContentBuilder) {
+            builder.startObject();
+            builder.field("field", "value");
+            builder.endObject();
+            return new ByteArrayEntity(builder.bytes().toBytesRef().bytes, contentType);
+        }
+    }
+
+    public void testConvertExistsResponse() {
+        RestStatus restStatus = randomBoolean() ? RestStatus.OK : randomFrom(RestStatus.values());
+        HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus));
+        Response response = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse);
+        boolean result = RestHighLevelClient.convertExistsResponse(response);
+        assertEquals(restStatus == RestStatus.OK, result);
+    }
+
+    public void testParseResponseException() throws IOException {
+        {
+            RestStatus restStatus = randomFrom(RestStatus.values());
+            HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus));
+            Response response = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse);
+            ResponseException responseException = new ResponseException(response);
+            ElasticsearchException elasticsearchException = restHighLevelClient.parseResponseException(responseException);
+            assertEquals(responseException.getMessage(), elasticsearchException.getMessage());
+            assertEquals(restStatus, elasticsearchException.status());
+            assertSame(responseException, elasticsearchException.getCause());
+        }
+        {
+            RestStatus restStatus = randomFrom(RestStatus.values());
+            HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus));
+            httpResponse.setEntity(new StringEntity("{\"error\":\"test error message\",\"status\":" + restStatus.getStatus() + "}",
+                    ContentType.APPLICATION_JSON));
+            Response response = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse);
+            ResponseException responseException = new ResponseException(response);
+            ElasticsearchException elasticsearchException = restHighLevelClient.parseResponseException(responseException);
+            assertEquals("Elasticsearch exception [type=exception, reason=test error message]", elasticsearchException.getMessage());
+            assertEquals(restStatus, elasticsearchException.status());
+            assertSame(responseException, elasticsearchException.getSuppressed()[0]);
+        }
+        {
+            RestStatus restStatus = randomFrom(RestStatus.values());
+            HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus));
+            httpResponse.setEntity(new StringEntity("{\"error\":", ContentType.APPLICATION_JSON));
+            Response response = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse);
+            ResponseException responseException = new ResponseException(response);
+            ElasticsearchException elasticsearchException = restHighLevelClient.parseResponseException(responseException);
+            assertEquals("Unable to parse response body", elasticsearchException.getMessage());
+            assertEquals(restStatus, elasticsearchException.status());
+            assertSame(responseException, elasticsearchException.getCause());
+            assertThat(elasticsearchException.getSuppressed()[0], instanceOf(IOException.class));
+        }
+        {
+            RestStatus restStatus = randomFrom(RestStatus.values());
+            HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus));
+            httpResponse.setEntity(new StringEntity("{\"status\":" + restStatus.getStatus() + "}", ContentType.APPLICATION_JSON));
+            Response response = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse);
+            ResponseException responseException = new ResponseException(response);
+            ElasticsearchException elasticsearchException = restHighLevelClient.parseResponseException(responseException);
+            assertEquals("Unable to parse response body", elasticsearchException.getMessage());
+            assertEquals(restStatus, elasticsearchException.status());
+            assertSame(responseException, elasticsearchException.getCause());
+            assertThat(elasticsearchException.getSuppressed()[0], instanceOf(IllegalStateException.class));
+        }
+    }
+
+    public void testPerformRequestOnSuccess() throws IOException {
+        MainRequest mainRequest = new MainRequest();
+        CheckedFunction<MainRequest, Request, IOException> requestConverter = request ->
+                new Request("GET", "/", Collections.emptyMap(), null);
+        RestStatus restStatus = randomFrom(RestStatus.values());
+        HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus));
+        Response mockResponse = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse);
+        when(restClient.performRequest(anyString(), anyString(), anyMapOf(String.class, String.class),
+                anyObject(), anyVararg())).thenReturn(mockResponse);
+        {
+            Integer result = restHighLevelClient.performRequest(mainRequest, requestConverter,
+                    response -> response.getStatusLine().getStatusCode(), Collections.emptySet());
+            assertEquals(restStatus.getStatus(), result.intValue());
+        }
+        {
+            IOException ioe = expectThrows(IOException.class, () -> restHighLevelClient.performRequest(mainRequest,
+                    requestConverter, response -> {throw new IllegalStateException();}, Collections.emptySet()));
+            assertEquals("Unable to parse response body for Response{requestLine=GET / http/1.1, host=http://localhost:9200, " +
+                    "response=http/1.1 " + restStatus.getStatus() + " " + restStatus.name() + "}", ioe.getMessage());
+        }
+    }
+
+    public void testPerformRequestOnResponseExceptionWithoutEntity() throws IOException {
+        MainRequest mainRequest = new MainRequest();
+        CheckedFunction<MainRequest, Request, IOException> requestConverter = request ->
+                new Request("GET", "/", Collections.emptyMap(), null);
+        RestStatus restStatus = randomFrom(RestStatus.values());
+        HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus));
+        Response mockResponse = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse);
+        ResponseException responseException = new ResponseException(mockResponse);
+        when(restClient.performRequest(anyString(), anyString(), anyMapOf(String.class, String.class),
+                anyObject(), anyVararg())).thenThrow(responseException);
+        ElasticsearchException elasticsearchException = expectThrows(ElasticsearchException.class,
+                () -> restHighLevelClient.performRequest(mainRequest, requestConverter,
+                        response -> response.getStatusLine().getStatusCode(), Collections.emptySet()));
+        assertEquals(responseException.getMessage(), elasticsearchException.getMessage());
+        assertEquals(restStatus, elasticsearchException.status());
+        assertSame(responseException, elasticsearchException.getCause());
+    }
+
+    public void testPerformRequestOnResponseExceptionWithEntity() throws IOException {
+        MainRequest mainRequest = new MainRequest();
+        CheckedFunction<MainRequest, Request, IOException> requestConverter = request ->
+                new Request("GET", "/", Collections.emptyMap(), null);
+        RestStatus restStatus = randomFrom(RestStatus.values());
+        HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus));
+        httpResponse.setEntity(new StringEntity("{\"error\":\"test error message\",\"status\":" + restStatus.getStatus() + "}",
+                ContentType.APPLICATION_JSON));
+        Response mockResponse = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse);
+        ResponseException responseException = new ResponseException(mockResponse);
+        when(restClient.performRequest(anyString(), anyString(), anyMapOf(String.class, String.class),
+                anyObject(), anyVararg())).thenThrow(responseException);
+        ElasticsearchException elasticsearchException = expectThrows(ElasticsearchException.class,
+                () -> restHighLevelClient.performRequest(mainRequest, requestConverter,
+                        response -> response.getStatusLine().getStatusCode(), Collections.emptySet()));
+        assertEquals("Elasticsearch exception [type=exception, reason=test error message]", elasticsearchException.getMessage());
+        assertEquals(restStatus, elasticsearchException.status());
+        assertSame(responseException, elasticsearchException.getSuppressed()[0]);
+    }
+
+    public void testPerformRequestOnResponseExceptionWithBrokenEntity() throws IOException {
+        MainRequest mainRequest = new MainRequest();
+        CheckedFunction<MainRequest, Request, IOException> requestConverter = request ->
+                new Request("GET", "/", Collections.emptyMap(), null);
+        RestStatus restStatus = randomFrom(RestStatus.values());
+        HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus));
+        httpResponse.setEntity(new StringEntity("{\"error\":", ContentType.APPLICATION_JSON));
+        Response mockResponse = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse);
+        ResponseException responseException = new ResponseException(mockResponse);
+        when(restClient.performRequest(anyString(), anyString(), anyMapOf(String.class, String.class),
+                anyObject(), anyVararg())).thenThrow(responseException);
+        ElasticsearchException elasticsearchException = expectThrows(ElasticsearchException.class,
+                () -> restHighLevelClient.performRequest(mainRequest, requestConverter,
+                        response -> response.getStatusLine().getStatusCode(), Collections.emptySet()));
+        assertEquals("Unable to parse response body", elasticsearchException.getMessage());
+        assertEquals(restStatus, elasticsearchException.status());
+        assertSame(responseException, elasticsearchException.getCause());
+        assertThat(elasticsearchException.getSuppressed()[0], instanceOf(JsonParseException.class));
+    }
+
+    public void testPerformRequestOnResponseExceptionWithBrokenEntity2() throws IOException {
+        MainRequest mainRequest = new MainRequest();
+        CheckedFunction<MainRequest, Request, IOException> requestConverter = request ->
+                new Request("GET", "/", Collections.emptyMap(), null);
+        RestStatus restStatus = randomFrom(RestStatus.values());
+        HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus));
+        httpResponse.setEntity(new StringEntity("{\"status\":" + restStatus.getStatus() + "}", ContentType.APPLICATION_JSON));
+        Response mockResponse = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse);
+        ResponseException responseException = new ResponseException(mockResponse);
+        when(restClient.performRequest(anyString(), anyString(), anyMapOf(String.class, String.class),
+                anyObject(), anyVararg())).thenThrow(responseException);
+        ElasticsearchException elasticsearchException = expectThrows(ElasticsearchException.class,
+                () -> restHighLevelClient.performRequest(mainRequest, requestConverter,
+                        response -> response.getStatusLine().getStatusCode(), Collections.emptySet()));
+        assertEquals("Unable to parse response body", elasticsearchException.getMessage());
+        assertEquals(restStatus, elasticsearchException.status());
+        assertSame(responseException, elasticsearchException.getCause());
+        assertThat(elasticsearchException.getSuppressed()[0], instanceOf(IllegalStateException.class));
+    }
+
+    public void testPerformRequestOnResponseExceptionWithIgnores() throws IOException {
+        MainRequest mainRequest = new MainRequest();
+        CheckedFunction<MainRequest, Request, IOException> requestConverter = request ->
+                new Request("GET", "/", Collections.emptyMap(), null);
+        HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(RestStatus.NOT_FOUND));
+        Response mockResponse = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse);
+        ResponseException responseException = new ResponseException(mockResponse);
+        when(restClient.performRequest(anyString(), anyString(), anyMapOf(String.class, String.class),
+                anyObject(), anyVararg())).thenThrow(responseException);
+        //although we got an exception, we turn it into a successful response because the status code was provided among ignores
+        assertEquals(Integer.valueOf(404), restHighLevelClient.performRequest(mainRequest, requestConverter,
+                response -> response.getStatusLine().getStatusCode(), Collections.singleton(404)));
+    }
+
+    public void testPerformRequestOnResponseExceptionWithIgnoresErrorNoBody() throws IOException {
+        MainRequest mainRequest = new MainRequest();
+        CheckedFunction<MainRequest, Request, IOException> requestConverter = request ->
+                new Request("GET", "/", Collections.emptyMap(), null);
+        HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(RestStatus.NOT_FOUND));
+        Response mockResponse = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse);
+        ResponseException responseException = new ResponseException(mockResponse);
+        when(restClient.performRequest(anyString(), anyString(), anyMapOf(String.class, String.class),
+                anyObject(), anyVararg())).thenThrow(responseException);
+        ElasticsearchException elasticsearchException = expectThrows(ElasticsearchException.class,
+                () -> restHighLevelClient.performRequest(mainRequest, requestConverter,
+                        response -> {throw new IllegalStateException();}, Collections.singleton(404)));
+        assertEquals(RestStatus.NOT_FOUND, elasticsearchException.status());
+        assertSame(responseException, elasticsearchException.getCause());
+        assertEquals(responseException.getMessage(), elasticsearchException.getMessage());
+    }
+
+    public void testPerformRequestOnResponseExceptionWithIgnoresErrorValidBody() throws IOException {
+        MainRequest mainRequest = new MainRequest();
+        CheckedFunction<MainRequest, Request, IOException> requestConverter = request ->
+                new Request("GET", "/", Collections.emptyMap(), null);
+        HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(RestStatus.NOT_FOUND));
+        httpResponse.setEntity(new StringEntity("{\"error\":\"test error message\",\"status\":404}",
+                ContentType.APPLICATION_JSON));
+        Response mockResponse = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse);
+        ResponseException responseException = new ResponseException(mockResponse);
+        when(restClient.performRequest(anyString(), anyString(), anyMapOf(String.class, String.class),
+                anyObject(), anyVararg())).thenThrow(responseException);
+        ElasticsearchException elasticsearchException = expectThrows(ElasticsearchException.class,
+                () -> restHighLevelClient.performRequest(mainRequest, requestConverter,
+                        response -> {throw new IllegalStateException();}, Collections.singleton(404)));
+        assertEquals(RestStatus.NOT_FOUND, elasticsearchException.status());
+        assertSame(responseException, elasticsearchException.getSuppressed()[0]);
+        assertEquals("Elasticsearch exception [type=exception, reason=test error message]", elasticsearchException.getMessage());
+    }
+
+    public void testWrapResponseListenerOnSuccess() {
+        {
+            TrackingActionListener trackingActionListener = new TrackingActionListener();
+            ResponseListener responseListener = restHighLevelClient.wrapResponseListener(
+                    response -> response.getStatusLine().getStatusCode(), trackingActionListener, Collections.emptySet());
+            RestStatus restStatus = randomFrom(RestStatus.values());
+            HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus));
+            responseListener.onSuccess(new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse));
+            assertNull(trackingActionListener.exception.get());
+            assertEquals(restStatus.getStatus(), trackingActionListener.statusCode.get());
+        }
+        {
+            TrackingActionListener trackingActionListener = new TrackingActionListener();
+            ResponseListener responseListener = restHighLevelClient.wrapResponseListener(
+                    response -> {throw new IllegalStateException();}, trackingActionListener, Collections.emptySet());
+            RestStatus restStatus = randomFrom(RestStatus.values());
+            HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus));
+            responseListener.onSuccess(new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse));
+            assertThat(trackingActionListener.exception.get(), instanceOf(IOException.class));
+            IOException ioe = (IOException) trackingActionListener.exception.get();
+            assertEquals("Unable to parse response body for Response{requestLine=GET / http/1.1, host=http://localhost:9200, " +
+                    "response=http/1.1 " + restStatus.getStatus() + " " + restStatus.name() + "}", ioe.getMessage());
+            assertThat(ioe.getCause(), instanceOf(IllegalStateException.class));
+        }
+    }
+
+    public void testWrapResponseListenerOnException() {
+        TrackingActionListener trackingActionListener = new TrackingActionListener();
+        ResponseListener responseListener = restHighLevelClient.wrapResponseListener(
+                response -> response.getStatusLine().getStatusCode(), trackingActionListener, Collections.emptySet());
+        IllegalStateException exception = new IllegalStateException();
+        responseListener.onFailure(exception);
+        assertSame(exception, trackingActionListener.exception.get());
+    }
+
+    public void testWrapResponseListenerOnResponseExceptionWithoutEntity() throws IOException {
+        TrackingActionListener trackingActionListener = new TrackingActionListener();
+        ResponseListener responseListener = restHighLevelClient.wrapResponseListener(
+                response -> response.getStatusLine().getStatusCode(), trackingActionListener, Collections.emptySet());
+        RestStatus restStatus = randomFrom(RestStatus.values());
+        HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus));
+        Response response = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse);
+        ResponseException responseException = new ResponseException(response);
+        responseListener.onFailure(responseException);
+        assertThat(trackingActionListener.exception.get(), instanceOf(ElasticsearchException.class));
+        ElasticsearchException elasticsearchException = (ElasticsearchException) trackingActionListener.exception.get();
+        assertEquals(responseException.getMessage(), elasticsearchException.getMessage());
+        assertEquals(restStatus, elasticsearchException.status());
+        assertSame(responseException, elasticsearchException.getCause());
+    }
+
+    public void testWrapResponseListenerOnResponseExceptionWithEntity() throws IOException {
+        TrackingActionListener trackingActionListener = new TrackingActionListener();
+        ResponseListener responseListener = restHighLevelClient.wrapResponseListener(
+                response -> response.getStatusLine().getStatusCode(), trackingActionListener, Collections.emptySet());
+        RestStatus restStatus = randomFrom(RestStatus.values());
+        HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus));
+        httpResponse.setEntity(new StringEntity("{\"error\":\"test error message\",\"status\":" + restStatus.getStatus() + "}",
+                ContentType.APPLICATION_JSON));
+        Response response = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse);
+        ResponseException responseException = new ResponseException(response);
+        responseListener.onFailure(responseException);
+        assertThat(trackingActionListener.exception.get(), instanceOf(ElasticsearchException.class));
+        ElasticsearchException elasticsearchException = (ElasticsearchException)trackingActionListener.exception.get();
+        assertEquals("Elasticsearch exception [type=exception, reason=test error message]", elasticsearchException.getMessage());
+        assertEquals(restStatus, elasticsearchException.status());
+        assertSame(responseException, elasticsearchException.getSuppressed()[0]);
+    }
+
+    public void testWrapResponseListenerOnResponseExceptionWithBrokenEntity() throws IOException {
+        {
+            TrackingActionListener trackingActionListener = new TrackingActionListener();
+            ResponseListener responseListener = restHighLevelClient.wrapResponseListener(
+                    response -> response.getStatusLine().getStatusCode(), trackingActionListener, Collections.emptySet());
+            RestStatus restStatus = randomFrom(RestStatus.values());
+            HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus));
+            httpResponse.setEntity(new StringEntity("{\"error\":", ContentType.APPLICATION_JSON));
+            Response response = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse);
+            ResponseException responseException = new ResponseException(response);
+            responseListener.onFailure(responseException);
+            assertThat(trackingActionListener.exception.get(), instanceOf(ElasticsearchException.class));
+            ElasticsearchException elasticsearchException = (ElasticsearchException)trackingActionListener.exception.get();
+            assertEquals("Unable to parse response body", elasticsearchException.getMessage());
+            assertEquals(restStatus, elasticsearchException.status());
+            assertSame(responseException, elasticsearchException.getCause());
+            assertThat(elasticsearchException.getSuppressed()[0], instanceOf(JsonParseException.class));
+        }
+        {
+            TrackingActionListener trackingActionListener = new TrackingActionListener();
+            ResponseListener responseListener = restHighLevelClient.wrapResponseListener(
+                    response -> response.getStatusLine().getStatusCode(), trackingActionListener, Collections.emptySet());
+            RestStatus restStatus = randomFrom(RestStatus.values());
+            HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus));
+            httpResponse.setEntity(new StringEntity("{\"status\":" + restStatus.getStatus() + "}", ContentType.APPLICATION_JSON));
+            Response response = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse);
+            ResponseException responseException = new ResponseException(response);
+            responseListener.onFailure(responseException);
+            assertThat(trackingActionListener.exception.get(), instanceOf(ElasticsearchException.class));
+            ElasticsearchException elasticsearchException = (ElasticsearchException)trackingActionListener.exception.get();
+            assertEquals("Unable to parse response body", elasticsearchException.getMessage());
+            assertEquals(restStatus, elasticsearchException.status());
+            assertSame(responseException, elasticsearchException.getCause());
+            assertThat(elasticsearchException.getSuppressed()[0], instanceOf(IllegalStateException.class));
+        }
+    }
+
+    public void testWrapResponseListenerOnResponseExceptionWithIgnores() throws IOException {
+        TrackingActionListener trackingActionListener = new TrackingActionListener();
+        ResponseListener responseListener = restHighLevelClient.wrapResponseListener(
+                response -> response.getStatusLine().getStatusCode(), trackingActionListener, Collections.singleton(404));
+        HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(RestStatus.NOT_FOUND));
+        Response response = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse);
+        ResponseException responseException = new ResponseException(response);
+        responseListener.onFailure(responseException);
+        //although we got an exception, we turn it into a successful response because the status code was provided among ignores
+        assertNull(trackingActionListener.exception.get());
+        assertEquals(404, trackingActionListener.statusCode.get());
+    }
+
+    public void testWrapResponseListenerOnResponseExceptionWithIgnoresErrorNoBody() throws IOException {
+        TrackingActionListener trackingActionListener = new TrackingActionListener();
+        //response parsing throws exception while handling ignores. same as when GetResponse#fromXContent throws error when trying
+        //to parse a 404 response which contains an error rather than a valid document not found response.
+        ResponseListener responseListener = restHighLevelClient.wrapResponseListener(
+                response -> { throw new IllegalStateException(); }, trackingActionListener, Collections.singleton(404));
+        HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(RestStatus.NOT_FOUND));
+        Response response = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse);
+        ResponseException responseException = new ResponseException(response);
+        responseListener.onFailure(responseException);
+        assertThat(trackingActionListener.exception.get(), instanceOf(ElasticsearchException.class));
+        ElasticsearchException elasticsearchException = (ElasticsearchException)trackingActionListener.exception.get();
+        assertEquals(RestStatus.NOT_FOUND, elasticsearchException.status());
+        assertSame(responseException, elasticsearchException.getCause());
+        assertEquals(responseException.getMessage(), elasticsearchException.getMessage());
+    }
+
+    public void testWrapResponseListenerOnResponseExceptionWithIgnoresErrorValidBody() throws IOException {
+        TrackingActionListener trackingActionListener = new TrackingActionListener();
+        //response parsing throws exception while handling ignores. same as when GetResponse#fromXContent throws error when trying
+        //to parse a 404 response which contains an error rather than a valid document not found response.
+        ResponseListener responseListener = restHighLevelClient.wrapResponseListener(
+                response -> { throw new IllegalStateException(); }, trackingActionListener, Collections.singleton(404));
+        HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(RestStatus.NOT_FOUND));
+        httpResponse.setEntity(new StringEntity("{\"error\":\"test error message\",\"status\":404}",
+                ContentType.APPLICATION_JSON));
+        Response response = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse);
+        ResponseException responseException = new ResponseException(response);
+        responseListener.onFailure(responseException);
+        assertThat(trackingActionListener.exception.get(), instanceOf(ElasticsearchException.class));
+        ElasticsearchException elasticsearchException = (ElasticsearchException)trackingActionListener.exception.get();
+        assertEquals(RestStatus.NOT_FOUND, elasticsearchException.status());
+        assertSame(responseException, elasticsearchException.getSuppressed()[0]);
+        assertEquals("Elasticsearch exception [type=exception, reason=test error message]", elasticsearchException.getMessage());
+    }
+
+    public void testDefaultNamedXContents() {
+        List<NamedXContentRegistry.Entry> namedXContents = RestHighLevelClient.getDefaultNamedXContents();
+        int expectedInternalAggregations = InternalAggregationTestCase.getDefaultNamedXContents().size();
+        int expectedSuggestions = 3;
+        assertEquals(expectedInternalAggregations + expectedSuggestions, namedXContents.size());
+        Map<Class<?>, Integer> categories = new HashMap<>();
+        for (NamedXContentRegistry.Entry namedXContent : namedXContents) {
+            Integer counter = categories.putIfAbsent(namedXContent.categoryClass, 1);
+            if (counter != null) {
+                categories.put(namedXContent.categoryClass, counter + 1);
+            }
+        }
+        assertEquals(2, categories.size());
+        assertEquals(expectedInternalAggregations, categories.get(Aggregation.class).intValue());
+        assertEquals(expectedSuggestions, categories.get(Suggest.Suggestion.class).intValue());
+    }
+
+    public void testProvidedNamedXContents() {
+        List<NamedXContentRegistry.Entry> namedXContents = RestHighLevelClient.getProvidedNamedXContents();
+        assertEquals(2, namedXContents.size());
+        Map<Class<?>, Integer> categories = new HashMap<>();
+        List<String> names = new ArrayList<>();
+        for (NamedXContentRegistry.Entry namedXContent : namedXContents) {
+            names.add(namedXContent.name.getPreferredName());
+            Integer counter = categories.putIfAbsent(namedXContent.categoryClass, 1);
+            if (counter != null) {
+                categories.put(namedXContent.categoryClass, counter + 1);
+            }
+        }
+        assertEquals(1, categories.size());
+        assertEquals(Integer.valueOf(2), categories.get(Aggregation.class));
+        assertTrue(names.contains(ChildrenAggregationBuilder.NAME));
+        assertTrue(names.contains(MatrixStatsAggregationBuilder.NAME));
+    }
+
+    private static class TrackingActionListener implements ActionListener<Integer> {
+        private final AtomicInteger statusCode = new AtomicInteger(-1);
+        private final AtomicReference<Exception> exception = new AtomicReference<>();
+
+        @Override
+        public void onResponse(Integer statusCode) {
+            assertTrue(this.statusCode.compareAndSet(-1, statusCode));
+        }
+
+        @Override
+        public void onFailure(Exception e) {
+            assertTrue(exception.compareAndSet(null, e));
+        }
+    }
+
+    private static class HeadersVarargMatcher extends ArgumentMatcher<Header[]> implements VarargMatcher {
+        private Header[] expectedHeaders;
+
+        HeadersVarargMatcher(Header... expectedHeaders) {
+            this.expectedHeaders = expectedHeaders;
+        }
+
+        @Override
+        public boolean matches(Object varargArgument) {
+            if (varargArgument instanceof Header[]) {
+                Header[] actualHeaders = (Header[]) varargArgument;
+                return new ArrayEquals(expectedHeaders).matches(actualHeaders);
+            }
+            return false;
+        }
+    }
+
+    private static StatusLine newStatusLine(RestStatus restStatus) {
+        return new BasicStatusLine(HTTP_PROTOCOL, restStatus.getStatus(), restStatus.name());
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java
new file mode 100644
index 0000000000000..6c53d191dfc98
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java
@@ -0,0 +1,464 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import org.apache.http.HttpEntity;
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.nio.entity.NStringEntity;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.action.search.ClearScrollRequest;
+import org.elasticsearch.action.search.ClearScrollResponse;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.SearchScrollRequest;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.query.MatchQueryBuilder;
+import org.elasticsearch.join.aggregations.Children;
+import org.elasticsearch.join.aggregations.ChildrenAggregationBuilder;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.aggregations.bucket.range.Range;
+import org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.terms.Terms;
+import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
+import org.elasticsearch.search.aggregations.matrix.stats.MatrixStats;
+import org.elasticsearch.search.aggregations.matrix.stats.MatrixStatsAggregationBuilder;
+import org.elasticsearch.search.aggregations.support.ValueType;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.sort.SortOrder;
+import org.elasticsearch.search.suggest.Suggest;
+import org.elasticsearch.search.suggest.SuggestBuilder;
+import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.hamcrest.Matchers.both;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.either;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.lessThan;
+
+public class SearchIT extends ESRestHighLevelClientTestCase {
+
+    @Before
+    public void indexDocuments() throws IOException {
+        StringEntity doc1 = new StringEntity("{\"type\":\"type1\", \"num\":10, \"num2\":50}", ContentType.APPLICATION_JSON);
+        client().performRequest("PUT", "/index/type/1", Collections.emptyMap(), doc1);
+        StringEntity doc2 = new StringEntity("{\"type\":\"type1\", \"num\":20, \"num2\":40}", ContentType.APPLICATION_JSON);
+        client().performRequest("PUT", "/index/type/2", Collections.emptyMap(), doc2);
+        StringEntity doc3 = new StringEntity("{\"type\":\"type1\", \"num\":50, \"num2\":35}", ContentType.APPLICATION_JSON);
+        client().performRequest("PUT", "/index/type/3", Collections.emptyMap(), doc3);
+        StringEntity doc4 = new StringEntity("{\"type\":\"type2\", \"num\":100, \"num2\":10}", ContentType.APPLICATION_JSON);
+        client().performRequest("PUT", "/index/type/4", Collections.emptyMap(), doc4);
+        StringEntity doc5 = new StringEntity("{\"type\":\"type2\", \"num\":100, \"num2\":10}", ContentType.APPLICATION_JSON);
+        client().performRequest("PUT", "/index/type/5", Collections.emptyMap(), doc5);
+        client().performRequest("POST", "/index/_refresh");
+    }
+
+    public void testSearchNoQuery() throws IOException {
+        SearchRequest searchRequest = new SearchRequest();
+        SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
+        assertSearchHeader(searchResponse);
+        assertNull(searchResponse.getAggregations());
+        assertNull(searchResponse.getSuggest());
+        assertEquals(Collections.emptyMap(), searchResponse.getProfileResults());
+        assertEquals(5, searchResponse.getHits().totalHits);
+        assertEquals(5, searchResponse.getHits().getHits().length);
+        for (SearchHit searchHit : searchResponse.getHits().getHits()) {
+            assertEquals("index", searchHit.getIndex());
+            assertEquals("type", searchHit.getType());
+            assertThat(Integer.valueOf(searchHit.getId()), both(greaterThan(0)).and(lessThan(6)));
+            assertEquals(1.0f, searchHit.getScore(), 0);
+            assertEquals(-1L, searchHit.getVersion());
+            assertNotNull(searchHit.getSourceAsMap());
+            assertEquals(3, searchHit.getSourceAsMap().size());
+            assertTrue(searchHit.getSourceAsMap().containsKey("type"));
+            assertTrue(searchHit.getSourceAsMap().containsKey("num"));
+            assertTrue(searchHit.getSourceAsMap().containsKey("num2"));
+        }
+    }
+
+    public void testSearchMatchQuery() throws IOException {
+        SearchRequest searchRequest = new SearchRequest();
+        searchRequest.source(new SearchSourceBuilder().query(new MatchQueryBuilder("num", 10)));
+        SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
+        assertSearchHeader(searchResponse);
+        assertNull(searchResponse.getAggregations());
+        assertNull(searchResponse.getSuggest());
+        assertEquals(Collections.emptyMap(), searchResponse.getProfileResults());
+        assertEquals(1, searchResponse.getHits().totalHits);
+        assertEquals(1, searchResponse.getHits().getHits().length);
+        assertThat(searchResponse.getHits().getMaxScore(), greaterThan(0f));
+        SearchHit searchHit = searchResponse.getHits().getHits()[0];
+        assertEquals("index", searchHit.getIndex());
+        assertEquals("type", searchHit.getType());
+        assertEquals("1", searchHit.getId());
+        assertThat(searchHit.getScore(), greaterThan(0f));
+        assertEquals(-1L, searchHit.getVersion());
+        assertNotNull(searchHit.getSourceAsMap());
+        assertEquals(3, searchHit.getSourceAsMap().size());
+        assertEquals("type1", searchHit.getSourceAsMap().get("type"));
+        assertEquals(50, searchHit.getSourceAsMap().get("num2"));
+    }
+
+    public void testSearchWithTermsAgg() throws IOException {
+        SearchRequest searchRequest = new SearchRequest();
+        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+        searchSourceBuilder.aggregation(new TermsAggregationBuilder("agg1", ValueType.STRING).field("type.keyword"));
+        searchSourceBuilder.size(0);
+        searchRequest.source(searchSourceBuilder);
+        SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
+        assertSearchHeader(searchResponse);
+        assertNull(searchResponse.getSuggest());
+        assertEquals(Collections.emptyMap(), searchResponse.getProfileResults());
+        assertEquals(0, searchResponse.getHits().getHits().length);
+        assertEquals(0f, searchResponse.getHits().getMaxScore(), 0f);
+        Terms termsAgg = searchResponse.getAggregations().get("agg1");
+        assertEquals("agg1", termsAgg.getName());
+        assertEquals(2, termsAgg.getBuckets().size());
+        Terms.Bucket type1 = termsAgg.getBucketByKey("type1");
+        assertEquals(3, type1.getDocCount());
+        assertEquals(0, type1.getAggregations().asList().size());
+        Terms.Bucket type2 = termsAgg.getBucketByKey("type2");
+        assertEquals(2, type2.getDocCount());
+        assertEquals(0, type2.getAggregations().asList().size());
+    }
+
+    public void testSearchWithRangeAgg() throws IOException {
+        {
+            SearchRequest searchRequest = new SearchRequest();
+            SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+            searchSourceBuilder.aggregation(new RangeAggregationBuilder("agg1").field("num"));
+            searchSourceBuilder.size(0);
+            searchRequest.source(searchSourceBuilder);
+
+            ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class,
+                    () -> execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync));
+            assertEquals(RestStatus.BAD_REQUEST, exception.status());
+        }
+
+        SearchRequest searchRequest = new SearchRequest();
+        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+        searchSourceBuilder.aggregation(new RangeAggregationBuilder("agg1").field("num")
+                .addRange("first", 0, 30).addRange("second", 31, 200));
+        searchSourceBuilder.size(0);
+        searchRequest.source(searchSourceBuilder);
+        SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
+        assertSearchHeader(searchResponse);
+        assertNull(searchResponse.getSuggest());
+        assertEquals(Collections.emptyMap(), searchResponse.getProfileResults());
+        assertEquals(5, searchResponse.getHits().totalHits);
+        assertEquals(0, searchResponse.getHits().getHits().length);
+        assertEquals(0f, searchResponse.getHits().getMaxScore(), 0f);
+        Range rangeAgg = searchResponse.getAggregations().get("agg1");
+        assertEquals("agg1", rangeAgg.getName());
+        assertEquals(2, rangeAgg.getBuckets().size());
+        {
+            Range.Bucket bucket = rangeAgg.getBuckets().get(0);
+            assertEquals("first", bucket.getKeyAsString());
+            assertEquals(2, bucket.getDocCount());
+        }
+        {
+            Range.Bucket bucket = rangeAgg.getBuckets().get(1);
+            assertEquals("second", bucket.getKeyAsString());
+            assertEquals(3, bucket.getDocCount());
+        }
+    }
+
+    public void testSearchWithTermsAndRangeAgg() throws IOException {
+        SearchRequest searchRequest = new SearchRequest();
+        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+        TermsAggregationBuilder agg = new TermsAggregationBuilder("agg1", ValueType.STRING).field("type.keyword");
+        agg.subAggregation(new RangeAggregationBuilder("subagg").field("num")
+                .addRange("first", 0, 30).addRange("second", 31, 200));
+        searchSourceBuilder.aggregation(agg);
+        searchSourceBuilder.size(0);
+        searchRequest.source(searchSourceBuilder);
+        SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
+        assertSearchHeader(searchResponse);
+        assertNull(searchResponse.getSuggest());
+        assertEquals(Collections.emptyMap(), searchResponse.getProfileResults());
+        assertEquals(0, searchResponse.getHits().getHits().length);
+        assertEquals(0f, searchResponse.getHits().getMaxScore(), 0f);
+        Terms termsAgg = searchResponse.getAggregations().get("agg1");
+        assertEquals("agg1", termsAgg.getName());
+        assertEquals(2, termsAgg.getBuckets().size());
+        Terms.Bucket type1 = termsAgg.getBucketByKey("type1");
+        assertEquals(3, type1.getDocCount());
+        assertEquals(1, type1.getAggregations().asList().size());
+        {
+            Range rangeAgg = type1.getAggregations().get("subagg");
+            assertEquals(2, rangeAgg.getBuckets().size());
+            {
+                Range.Bucket bucket = rangeAgg.getBuckets().get(0);
+                assertEquals("first", bucket.getKeyAsString());
+                assertEquals(2, bucket.getDocCount());
+            }
+            {
+                Range.Bucket bucket = rangeAgg.getBuckets().get(1);
+                assertEquals("second", bucket.getKeyAsString());
+                assertEquals(1, bucket.getDocCount());
+            }
+        }
+        Terms.Bucket type2 = termsAgg.getBucketByKey("type2");
+        assertEquals(2, type2.getDocCount());
+        assertEquals(1, type2.getAggregations().asList().size());
+        {
+            Range rangeAgg = type2.getAggregations().get("subagg");
+            assertEquals(2, rangeAgg.getBuckets().size());
+            {
+                Range.Bucket bucket = rangeAgg.getBuckets().get(0);
+                assertEquals("first", bucket.getKeyAsString());
+                assertEquals(0, bucket.getDocCount());
+            }
+            {
+                Range.Bucket bucket = rangeAgg.getBuckets().get(1);
+                assertEquals("second", bucket.getKeyAsString());
+                assertEquals(2, bucket.getDocCount());
+            }
+        }
+    }
+
+    public void testSearchWithMatrixStats() throws IOException {
+        SearchRequest searchRequest = new SearchRequest();
+        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+        searchSourceBuilder.aggregation(new MatrixStatsAggregationBuilder("agg1").fields(Arrays.asList("num", "num2")));
+        searchSourceBuilder.size(0);
+        searchRequest.source(searchSourceBuilder);
+        SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
+        assertSearchHeader(searchResponse);
+        assertNull(searchResponse.getSuggest());
+        assertEquals(Collections.emptyMap(), searchResponse.getProfileResults());
+        assertEquals(5, searchResponse.getHits().totalHits);
+        assertEquals(0, searchResponse.getHits().getHits().length);
+        assertEquals(0f, searchResponse.getHits().getMaxScore(), 0f);
+        assertEquals(1, searchResponse.getAggregations().asList().size());
+        MatrixStats matrixStats = searchResponse.getAggregations().get("agg1");
+        assertEquals(5, matrixStats.getFieldCount("num"));
+        assertEquals(56d, matrixStats.getMean("num"), 0d);
+        assertEquals(1830d, matrixStats.getVariance("num"), 0d);
+        assertEquals(0.09340198804973057, matrixStats.getSkewness("num"), 0d);
+        assertEquals(1.2741646510794589, matrixStats.getKurtosis("num"), 0d);
+        assertEquals(5, matrixStats.getFieldCount("num2"));
+        assertEquals(29d, matrixStats.getMean("num2"), 0d);
+        assertEquals(330d, matrixStats.getVariance("num2"), 0d);
+        assertEquals(-0.13568039346585542, matrixStats.getSkewness("num2"), 1.0e-16);
+        assertEquals(1.3517561983471074, matrixStats.getKurtosis("num2"), 0d);
+        assertEquals(-767.5, matrixStats.getCovariance("num", "num2"), 0d);
+        assertEquals(-0.9876336291667923, matrixStats.getCorrelation("num", "num2"), 0d);
+    }
+
+    public void testSearchWithParentJoin() throws IOException {
+        StringEntity parentMapping = new StringEntity("{\n" +
+                "    \"mappings\": {\n" +
+                "        \"answer\" : {\n" +
+                "            \"_parent\" : {\n" +
+                "                \"type\" : \"question\"\n" +
+                "            }\n" +
+                "        }\n" +
+                "    },\n" +
+                "    \"settings\": {\n" +
+                "        \"index.mapping.single_type\": false" +
+                "    }\n" +
+                "}", ContentType.APPLICATION_JSON);
+        client().performRequest("PUT", "/child_example", Collections.emptyMap(), parentMapping);
+        StringEntity questionDoc = new StringEntity("{\n" +
+                "    \"body\": \"<p>I have Windows 2003 server and i bought a new Windows 2008 server...\",\n" +
+                "    \"title\": \"Whats the best way to file transfer my site from server to a newer one?\",\n" +
+                "    \"tags\": [\n" +
+                "        \"windows-server-2003\",\n" +
+                "        \"windows-server-2008\",\n" +
+                "        \"file-transfer\"\n" +
+                "    ]\n" +
+                "}", ContentType.APPLICATION_JSON);
+        client().performRequest("PUT", "/child_example/question/1", Collections.emptyMap(), questionDoc);
+        StringEntity answerDoc1 = new StringEntity("{\n" +
+                "    \"owner\": {\n" +
+                "        \"location\": \"Norfolk, United Kingdom\",\n" +
+                "        \"display_name\": \"Sam\",\n" +
+                "        \"id\": 48\n" +
+                "    },\n" +
+                "    \"body\": \"<p>Unfortunately you're pretty much limited to FTP...\",\n" +
+                "    \"creation_date\": \"2009-05-04T13:45:37.030\"\n" +
+                "}", ContentType.APPLICATION_JSON);
+        client().performRequest("PUT", "child_example/answer/1", Collections.singletonMap("parent", "1"), answerDoc1);
+        StringEntity answerDoc2 = new StringEntity("{\n" +
+                "    \"owner\": {\n" +
+                "        \"location\": \"Norfolk, United Kingdom\",\n" +
+                "        \"display_name\": \"Troll\",\n" +
+                "        \"id\": 49\n" +
+                "    },\n" +
+                "    \"body\": \"<p>Use Linux...\",\n" +
+                "    \"creation_date\": \"2009-05-05T13:45:37.030\"\n" +
+                "}", ContentType.APPLICATION_JSON);
+        client().performRequest("PUT", "/child_example/answer/2", Collections.singletonMap("parent", "1"), answerDoc2);
+        client().performRequest("POST", "/_refresh");
+
+        TermsAggregationBuilder leafTermAgg = new TermsAggregationBuilder("top-names", ValueType.STRING)
+                .field("owner.display_name.keyword").size(10);
+        ChildrenAggregationBuilder childrenAgg = new ChildrenAggregationBuilder("to-answers", "answer").subAggregation(leafTermAgg);
+        TermsAggregationBuilder termsAgg = new TermsAggregationBuilder("top-tags", ValueType.STRING).field("tags.keyword")
+                .size(10).subAggregation(childrenAgg);
+        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+        searchSourceBuilder.size(0).aggregation(termsAgg);
+        SearchRequest searchRequest = new SearchRequest("child_example");
+        searchRequest.source(searchSourceBuilder);
+
+        SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
+        assertSearchHeader(searchResponse);
+        assertNull(searchResponse.getSuggest());
+        assertEquals(Collections.emptyMap(), searchResponse.getProfileResults());
+        assertEquals(3, searchResponse.getHits().totalHits);
+        assertEquals(0, searchResponse.getHits().getHits().length);
+        assertEquals(0f, searchResponse.getHits().getMaxScore(), 0f);
+        assertEquals(1, searchResponse.getAggregations().asList().size());
+        Terms terms = searchResponse.getAggregations().get("top-tags");
+        assertEquals(0, terms.getDocCountError());
+        assertEquals(0, terms.getSumOfOtherDocCounts());
+        assertEquals(3, terms.getBuckets().size());
+        for (Terms.Bucket bucket : terms.getBuckets()) {
+            assertThat(bucket.getKeyAsString(),
+                    either(equalTo("file-transfer")).or(equalTo("windows-server-2003")).or(equalTo("windows-server-2008")));
+            assertEquals(1, bucket.getDocCount());
+            assertEquals(1, bucket.getAggregations().asList().size());
+            Children children = bucket.getAggregations().get("to-answers");
+            assertEquals(2, children.getDocCount());
+            assertEquals(1, children.getAggregations().asList().size());
+            Terms leafTerms = children.getAggregations().get("top-names");
+            assertEquals(0, leafTerms.getDocCountError());
+            assertEquals(0, leafTerms.getSumOfOtherDocCounts());
+            assertEquals(2, leafTerms.getBuckets().size());
+            assertEquals(2, leafTerms.getBuckets().size());
+            Terms.Bucket sam = leafTerms.getBucketByKey("Sam");
+            assertEquals(1, sam.getDocCount());
+            Terms.Bucket troll = leafTerms.getBucketByKey("Troll");
+            assertEquals(1, troll.getDocCount());
+        }
+    }
+
+    public void testSearchWithSuggest() throws IOException {
+        SearchRequest searchRequest = new SearchRequest();
+        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+        searchSourceBuilder.suggest(new SuggestBuilder().addSuggestion("sugg1", new PhraseSuggestionBuilder("type"))
+                .setGlobalText("type"));
+        searchSourceBuilder.size(0);
+        searchRequest.source(searchSourceBuilder);
+
+        SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
+        assertSearchHeader(searchResponse);
+        assertNull(searchResponse.getAggregations());
+        assertEquals(Collections.emptyMap(), searchResponse.getProfileResults());
+        assertEquals(0, searchResponse.getHits().totalHits);
+        assertEquals(0f, searchResponse.getHits().getMaxScore(), 0f);
+        assertEquals(0, searchResponse.getHits().getHits().length);
+        assertEquals(1, searchResponse.getSuggest().size());
+
+        Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>> sugg = searchResponse
+                .getSuggest().iterator().next();
+        assertEquals("sugg1", sugg.getName());
+        for (Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option> options : sugg) {
+            assertEquals("type", options.getText().string());
+            assertEquals(0, options.getOffset());
+            assertEquals(4, options.getLength());
+            assertEquals(2 ,options.getOptions().size());
+            for (Suggest.Suggestion.Entry.Option option : options) {
+                assertThat(option.getScore(), greaterThan(0f));
+                assertThat(option.getText().string(), either(equalTo("type1")).or(equalTo("type2")));
+            }
+        }
+    }
+
+    public void testSearchScroll() throws Exception {
+
+        for (int i = 0; i < 100; i++) {
+            XContentBuilder builder = jsonBuilder().startObject().field("field", i).endObject();
+            HttpEntity entity = new NStringEntity(builder.string(), ContentType.APPLICATION_JSON);
+            client().performRequest("PUT", "test/type1/" + Integer.toString(i), Collections.emptyMap(), entity);
+        }
+        client().performRequest("POST", "/test/_refresh");
+
+        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().size(35).sort("field", SortOrder.ASC);
+        SearchRequest searchRequest = new SearchRequest("test").scroll(TimeValue.timeValueMinutes(2)).source(searchSourceBuilder);
+        SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
+
+        try {
+            long counter = 0;
+            assertSearchHeader(searchResponse);
+            assertThat(searchResponse.getHits().getTotalHits(), equalTo(100L));
+            assertThat(searchResponse.getHits().getHits().length, equalTo(35));
+            for (SearchHit hit : searchResponse.getHits()) {
+                assertThat(((Number) hit.getSortValues()[0]).longValue(), equalTo(counter++));
+            }
+
+            searchResponse = execute(new SearchScrollRequest(searchResponse.getScrollId()).scroll(TimeValue.timeValueMinutes(2)),
+                    highLevelClient()::searchScroll, highLevelClient()::searchScrollAsync);
+
+            assertThat(searchResponse.getHits().getTotalHits(), equalTo(100L));
+            assertThat(searchResponse.getHits().getHits().length, equalTo(35));
+            for (SearchHit hit : searchResponse.getHits()) {
+                assertEquals(counter++, ((Number) hit.getSortValues()[0]).longValue());
+            }
+
+            searchResponse = execute(new SearchScrollRequest(searchResponse.getScrollId()).scroll(TimeValue.timeValueMinutes(2)),
+                    highLevelClient()::searchScroll, highLevelClient()::searchScrollAsync);
+
+            assertThat(searchResponse.getHits().getTotalHits(), equalTo(100L));
+            assertThat(searchResponse.getHits().getHits().length, equalTo(30));
+            for (SearchHit hit : searchResponse.getHits()) {
+                assertEquals(counter++, ((Number) hit.getSortValues()[0]).longValue());
+            }
+        } finally {
+            ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
+            clearScrollRequest.addScrollId(searchResponse.getScrollId());
+            ClearScrollResponse clearScrollResponse = execute(clearScrollRequest,
+                    // Not using a method reference to work around https://bugs.eclipse.org/bugs/show_bug.cgi?id=517951
+                    (request, headers) -> highLevelClient().clearScroll(request, headers),
+                    (request, listener, headers) -> highLevelClient().clearScrollAsync(request, listener, headers));
+            assertThat(clearScrollResponse.getNumFreed(), greaterThan(0));
+            assertTrue(clearScrollResponse.isSucceeded());
+
+            SearchScrollRequest scrollRequest = new SearchScrollRequest(searchResponse.getScrollId()).scroll(TimeValue.timeValueMinutes(2));
+            ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class, () -> execute(scrollRequest,
+                    highLevelClient()::searchScroll, highLevelClient()::searchScrollAsync));
+            assertEquals(RestStatus.NOT_FOUND, exception.status());
+            assertThat(exception.getRootCause(), instanceOf(ElasticsearchException.class));
+            ElasticsearchException rootCause = (ElasticsearchException) exception.getRootCause();
+            assertThat(rootCause.getMessage(), containsString("No search context found for"));
+        }
+    }
+
+    private static void assertSearchHeader(SearchResponse searchResponse) {
+        assertThat(searchResponse.getTook().nanos(), greaterThanOrEqualTo(0L));
+        assertEquals(0, searchResponse.getFailedShards());
+        assertThat(searchResponse.getTotalShards(), greaterThan(0));
+        assertEquals(searchResponse.getTotalShards(), searchResponse.getSuccessfulShards());
+        assertEquals(0, searchResponse.getShardFailures().length);
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java
new file mode 100644
index 0000000000000..30907cd050531
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java
@@ -0,0 +1,962 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.documentation;
+
+import org.apache.http.HttpEntity;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.entity.ContentType;
+import org.apache.http.nio.entity.NStringEntity;
+import org.elasticsearch.Build;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.action.bulk.BackoffPolicy;
+import org.elasticsearch.action.bulk.BulkItemResponse;
+import org.elasticsearch.action.bulk.BulkProcessor;
+import org.elasticsearch.action.bulk.BulkRequest;
+import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.delete.DeleteResponse;
+import org.elasticsearch.action.get.GetRequest;
+import org.elasticsearch.action.get.GetResponse;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.action.main.MainResponse;
+import org.elasticsearch.action.support.ActiveShardCount;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.action.support.replication.ReplicationResponse;
+import org.elasticsearch.action.update.UpdateRequest;
+import org.elasticsearch.action.update.UpdateResponse;
+import org.elasticsearch.client.ESRestHighLevelClientTestCase;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.VersionType;
+import org.elasticsearch.index.get.GetResult;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptType;
+import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
+import org.elasticsearch.threadpool.ThreadPool;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.singletonMap;
+
+/**
+ * This class is used to generate the Java CRUD API documentation.
+ * You need to wrap your code between two tags like:
+ * // tag::example[]
+ * // end::example[]
+ *
+ * Where example is your tag name.
+ *
+ * Then in the documentation, you can extract what is between tag and end tags with
+ * ["source","java",subs="attributes,callouts,macros"]
+ * --------------------------------------------------
+ * include-tagged::{doc-tests}/CRUDDocumentationIT.java[example]
+ * --------------------------------------------------
+ */
+public class CRUDDocumentationIT extends ESRestHighLevelClientTestCase {
+
+    public void testIndex() throws IOException {
+        RestHighLevelClient client = highLevelClient();
+
+        {
+            //tag::index-request-map
+            Map<String, Object> jsonMap = new HashMap<>();
+            jsonMap.put("user", "kimchy");
+            jsonMap.put("postDate", new Date());
+            jsonMap.put("message", "trying out Elasticsearch");
+            IndexRequest indexRequest = new IndexRequest("posts", "doc", "1")
+                    .source(jsonMap); // <1>
+            //end::index-request-map
+            IndexResponse indexResponse = client.index(indexRequest);
+            assertEquals(indexResponse.getResult(), DocWriteResponse.Result.CREATED);
+        }
+        {
+            //tag::index-request-xcontent
+            XContentBuilder builder = XContentFactory.jsonBuilder();
+            builder.startObject();
+            {
+                builder.field("user", "kimchy");
+                builder.field("postDate", new Date());
+                builder.field("message", "trying out Elasticsearch");
+            }
+            builder.endObject();
+            IndexRequest indexRequest = new IndexRequest("posts", "doc", "1")
+                    .source(builder);  // <1>
+            //end::index-request-xcontent
+            IndexResponse indexResponse = client.index(indexRequest);
+            assertEquals(indexResponse.getResult(), DocWriteResponse.Result.UPDATED);
+        }
+        {
+            //tag::index-request-shortcut
+            IndexRequest indexRequest = new IndexRequest("posts", "doc", "1")
+                    .source("user", "kimchy",
+                            "postDate", new Date(),
+                            "message", "trying out Elasticsearch"); // <1>
+            //end::index-request-shortcut
+            IndexResponse indexResponse = client.index(indexRequest);
+            assertEquals(indexResponse.getResult(), DocWriteResponse.Result.UPDATED);
+        }
+        {
+            //tag::index-request-string
+            IndexRequest request = new IndexRequest(
+                    "posts", // <1>
+                    "doc",  // <2>
+                    "1");   // <3>
+            String jsonString = "{" +
+                    "\"user\":\"kimchy\"," +
+                    "\"postDate\":\"2013-01-30\"," +
+                    "\"message\":\"trying out Elasticsearch\"" +
+                    "}";
+            request.source(jsonString, XContentType.JSON); // <4>
+            //end::index-request-string
+
+            // tag::index-execute
+            IndexResponse indexResponse = client.index(request);
+            // end::index-execute
+            assertEquals(indexResponse.getResult(), DocWriteResponse.Result.UPDATED);
+
+            // tag::index-response
+            String index = indexResponse.getIndex();
+            String type = indexResponse.getType();
+            String id = indexResponse.getId();
+            long version = indexResponse.getVersion();
+            if (indexResponse.getResult() == DocWriteResponse.Result.CREATED) {
+                // <1>
+            } else if (indexResponse.getResult() == DocWriteResponse.Result.UPDATED) {
+                // <2>
+            }
+            ReplicationResponse.ShardInfo shardInfo = indexResponse.getShardInfo();
+            if (shardInfo.getTotal() != shardInfo.getSuccessful()) {
+                // <3>
+            }
+            if (shardInfo.getFailed() > 0) {
+                for (ReplicationResponse.ShardInfo.Failure failure : shardInfo.getFailures()) {
+                    String reason = failure.reason(); // <4>
+                }
+            }
+            // end::index-response
+
+            // tag::index-execute-async
+            client.indexAsync(request, new ActionListener<IndexResponse>() {
+                @Override
+                public void onResponse(IndexResponse indexResponse) {
+                    // <1>
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    // <2>
+                }
+            });
+            // end::index-execute-async
+        }
+        {
+            IndexRequest request = new IndexRequest("posts", "doc", "1");
+            // tag::index-request-routing
+            request.routing("routing"); // <1>
+            // end::index-request-routing
+            // tag::index-request-parent
+            request.parent("parent"); // <1>
+            // end::index-request-parent
+            // tag::index-request-timeout
+            request.timeout(TimeValue.timeValueSeconds(1)); // <1>
+            request.timeout("1s"); // <2>
+            // end::index-request-timeout
+            // tag::index-request-refresh
+            request.setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL); // <1>
+            request.setRefreshPolicy("wait_for");                            // <2>
+            // end::index-request-refresh
+            // tag::index-request-version
+            request.version(2); // <1>
+            // end::index-request-version
+            // tag::index-request-version-type
+            request.versionType(VersionType.EXTERNAL); // <1>
+            // end::index-request-version-type
+            // tag::index-request-op-type
+            request.opType(DocWriteRequest.OpType.CREATE); // <1>
+            request.opType("create"); // <2>
+            // end::index-request-op-type
+            // tag::index-request-pipeline
+            request.setPipeline("pipeline"); // <1>
+            // end::index-request-pipeline
+        }
+        {
+            // tag::index-conflict
+            IndexRequest request = new IndexRequest("posts", "doc", "1")
+                    .source("field", "value")
+                    .version(1);
+            try {
+                IndexResponse response = client.index(request);
+            } catch(ElasticsearchException e) {
+                if (e.status() == RestStatus.CONFLICT) {
+                    // <1>
+                }
+            }
+            // end::index-conflict
+        }
+        {
+            // tag::index-optype
+            IndexRequest request = new IndexRequest("posts", "doc", "1")
+                    .source("field", "value")
+                    .opType(DocWriteRequest.OpType.CREATE);
+            try {
+                IndexResponse response = client.index(request);
+            } catch(ElasticsearchException e) {
+                if (e.status() == RestStatus.CONFLICT) {
+                    // <1>
+                }
+            }
+            // end::index-optype
+        }
+    }
+
+    public void testUpdate() throws IOException {
+        RestHighLevelClient client = highLevelClient();
+        {
+            IndexRequest indexRequest = new IndexRequest("posts", "doc", "1").source("field", 0);
+            IndexResponse indexResponse = client.index(indexRequest);
+            assertSame(indexResponse.status(), RestStatus.CREATED);
+
+            XContentType xContentType = XContentType.JSON;
+            String script = XContentBuilder.builder(xContentType.xContent())
+                    .startObject()
+                        .startObject("script")
+                            .field("lang", "painless")
+                            .field("code", "ctx._source.field += params.count")
+                        .endObject()
+                    .endObject().string();
+            HttpEntity body = new NStringEntity(script, ContentType.create(xContentType.mediaType()));
+            Response response = client().performRequest(HttpPost.METHOD_NAME, "/_scripts/increment-field", emptyMap(), body);
+            assertEquals(response.getStatusLine().getStatusCode(), RestStatus.OK.getStatus());
+        }
+        {
+            //tag::update-request
+            UpdateRequest request = new UpdateRequest(
+                    "posts", // <1>
+                    "doc",  // <2>
+                    "1");   // <3>
+            //end::update-request
+            request.fetchSource(true);
+            //tag::update-request-with-inline-script
+            Map<String, Object> parameters = singletonMap("count", 4); // <1>
+
+            Script inline = new Script(ScriptType.INLINE, "painless", "ctx._source.field += params.count", parameters);  // <2>
+            request.script(inline);  // <3>
+            //end::update-request-with-inline-script
+            UpdateResponse updateResponse = client.update(request);
+            assertEquals(updateResponse.getResult(), DocWriteResponse.Result.UPDATED);
+            assertEquals(4, updateResponse.getGetResult().getSource().get("field"));
+
+            request = new UpdateRequest("posts", "doc", "1").fetchSource(true);
+            //tag::update-request-with-stored-script
+            Script stored =
+                    new Script(ScriptType.STORED, "painless", "increment-field", parameters);  // <1>
+            request.script(stored);  // <2>
+            //end::update-request-with-stored-script
+            updateResponse = client.update(request);
+            assertEquals(updateResponse.getResult(), DocWriteResponse.Result.UPDATED);
+            assertEquals(8, updateResponse.getGetResult().getSource().get("field"));
+        }
+        {
+            //tag::update-request-with-doc-as-map
+            Map<String, Object> jsonMap = new HashMap<>();
+            jsonMap.put("updated", new Date());
+            jsonMap.put("reason", "daily update");
+            UpdateRequest request = new UpdateRequest("posts", "doc", "1")
+                    .doc(jsonMap); // <1>
+            //end::update-request-with-doc-as-map
+            UpdateResponse updateResponse = client.update(request);
+            assertEquals(updateResponse.getResult(), DocWriteResponse.Result.UPDATED);
+        }
+        {
+            //tag::update-request-with-doc-as-xcontent
+            XContentBuilder builder = XContentFactory.jsonBuilder();
+            builder.startObject();
+            {
+                builder.field("updated", new Date());
+                builder.field("reason", "daily update");
+            }
+            builder.endObject();
+            UpdateRequest request = new UpdateRequest("posts", "doc", "1")
+                    .doc(builder);  // <1>
+            //end::update-request-with-doc-as-xcontent
+            UpdateResponse updateResponse = client.update(request);
+            assertEquals(updateResponse.getResult(), DocWriteResponse.Result.UPDATED);
+        }
+        {
+            //tag::update-request-shortcut
+            UpdateRequest request = new UpdateRequest("posts", "doc", "1")
+                    .doc("updated", new Date(),
+                         "reason", "daily update"); // <1>
+            //end::update-request-shortcut
+            UpdateResponse updateResponse = client.update(request);
+            assertEquals(updateResponse.getResult(), DocWriteResponse.Result.UPDATED);
+        }
+        {
+            //tag::update-request-with-doc-as-string
+            UpdateRequest request = new UpdateRequest("posts", "doc", "1");
+            String jsonString = "{" +
+                    "\"updated\":\"2017-01-01\"," +
+                    "\"reason\":\"daily update\"" +
+                    "}";
+            request.doc(jsonString, XContentType.JSON); // <1>
+            //end::update-request-with-doc-as-string
+            request.fetchSource(true);
+            // tag::update-execute
+            UpdateResponse updateResponse = client.update(request);
+            // end::update-execute
+            assertEquals(updateResponse.getResult(), DocWriteResponse.Result.UPDATED);
+
+            // tag::update-response
+            String index = updateResponse.getIndex();
+            String type = updateResponse.getType();
+            String id = updateResponse.getId();
+            long version = updateResponse.getVersion();
+            if (updateResponse.getResult() == DocWriteResponse.Result.CREATED) {
+                // <1>
+            } else if (updateResponse.getResult() == DocWriteResponse.Result.UPDATED) {
+                // <2>
+            } else if (updateResponse.getResult() == DocWriteResponse.Result.DELETED) {
+                // <3>
+            } else if (updateResponse.getResult() == DocWriteResponse.Result.NOOP) {
+                // <4>
+            }
+            // end::update-response
+
+            // tag::update-getresult
+            GetResult result = updateResponse.getGetResult(); // <1>
+            if (result.isExists()) {
+                String sourceAsString = result.sourceAsString(); // <2>
+                Map<String, Object> sourceAsMap = result.sourceAsMap(); // <3>
+                byte[] sourceAsBytes = result.source(); // <4>
+            } else {
+                // <5>
+            }
+            // end::update-getresult
+            assertNotNull(result);
+            assertEquals(3, result.sourceAsMap().size());
+            // tag::update-failure
+            ReplicationResponse.ShardInfo shardInfo = updateResponse.getShardInfo();
+            if (shardInfo.getTotal() != shardInfo.getSuccessful()) {
+                // <1>
+            }
+            if (shardInfo.getFailed() > 0) {
+                for (ReplicationResponse.ShardInfo.Failure failure : shardInfo.getFailures()) {
+                    String reason = failure.reason(); // <2>
+                }
+            }
+            // end::update-failure
+
+            // tag::update-execute-async
+            client.updateAsync(request, new ActionListener<UpdateResponse>() {
+                @Override
+                public void onResponse(UpdateResponse updateResponse) {
+                    // <1>
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    // <2>
+                }
+            });
+            // end::update-execute-async
+        }
+        {
+            //tag::update-docnotfound
+            UpdateRequest request = new UpdateRequest("posts", "type", "does_not_exist").doc("field", "value");
+            try {
+                UpdateResponse updateResponse = client.update(request);
+            } catch (ElasticsearchException e) {
+                if (e.status() == RestStatus.NOT_FOUND) {
+                    // <1>
+                }
+            }
+            //end::update-docnotfound
+        }
+        {
+            // tag::update-conflict
+            UpdateRequest request = new UpdateRequest("posts", "doc", "1")
+                    .doc("field", "value")
+                    .version(1);
+            try {
+                UpdateResponse updateResponse = client.update(request);
+            } catch(ElasticsearchException e) {
+                if (e.status() == RestStatus.CONFLICT) {
+                    // <1>
+                }
+            }
+            // end::update-conflict
+        }
+        {
+            UpdateRequest request = new UpdateRequest("posts", "doc", "1").doc("reason", "no source");
+            //tag::update-request-no-source
+            request.fetchSource(true); // <1>
+            //end::update-request-no-source
+            UpdateResponse updateResponse = client.update(request);
+            assertEquals(updateResponse.getResult(), DocWriteResponse.Result.UPDATED);
+            assertNotNull(updateResponse.getGetResult());
+            assertEquals(3, updateResponse.getGetResult().sourceAsMap().size());
+        }
+        {
+            UpdateRequest request = new UpdateRequest("posts", "doc", "1").doc("reason", "source includes");
+            //tag::update-request-source-include
+            String[] includes = new String[]{"updated", "r*"};
+            String[] excludes = Strings.EMPTY_ARRAY;
+            request.fetchSource(new FetchSourceContext(true, includes, excludes)); // <1>
+            //end::update-request-source-include
+            UpdateResponse updateResponse = client.update(request);
+            assertEquals(updateResponse.getResult(), DocWriteResponse.Result.UPDATED);
+            Map<String, Object> sourceAsMap = updateResponse.getGetResult().sourceAsMap();
+            assertEquals(2, sourceAsMap.size());
+            assertEquals("source includes", sourceAsMap.get("reason"));
+            assertTrue(sourceAsMap.containsKey("updated"));
+        }
+        {
+            UpdateRequest request = new UpdateRequest("posts", "doc", "1").doc("reason", "source excludes");
+            //tag::update-request-source-exclude
+            String[] includes = Strings.EMPTY_ARRAY;
+            String[] excludes = new String[]{"updated"};
+            request.fetchSource(new FetchSourceContext(true, includes, excludes)); // <1>
+            //end::update-request-source-exclude
+            UpdateResponse updateResponse = client.update(request);
+            assertEquals(updateResponse.getResult(), DocWriteResponse.Result.UPDATED);
+            Map<String, Object> sourceAsMap = updateResponse.getGetResult().sourceAsMap();
+            assertEquals(2, sourceAsMap.size());
+            assertEquals("source excludes", sourceAsMap.get("reason"));
+            assertTrue(sourceAsMap.containsKey("field"));
+        }
+        {
+            UpdateRequest request = new UpdateRequest("posts", "doc", "id");
+            // tag::update-request-routing
+            request.routing("routing"); // <1>
+            // end::update-request-routing
+            // tag::update-request-parent
+            request.parent("parent"); // <1>
+            // end::update-request-parent
+            // tag::update-request-timeout
+            request.timeout(TimeValue.timeValueSeconds(1)); // <1>
+            request.timeout("1s"); // <2>
+            // end::update-request-timeout
+            // tag::update-request-retry
+            request.retryOnConflict(3); // <1>
+            // end::update-request-retry
+            // tag::update-request-refresh
+            request.setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL); // <1>
+            request.setRefreshPolicy("wait_for");                            // <2>
+            // end::update-request-refresh
+            // tag::update-request-version
+            request.version(2); // <1>
+            // end::update-request-version
+            // tag::update-request-detect-noop
+            request.detectNoop(false); // <1>
+            // end::update-request-detect-noop
+            // tag::update-request-upsert
+            String jsonString = "{\"created\":\"2017-01-01\"}";
+            request.upsert(jsonString, XContentType.JSON);  // <1>
+            // end::update-request-upsert
+            // tag::update-request-scripted-upsert
+            request.scriptedUpsert(true); // <1>
+            // end::update-request-scripted-upsert
+            // tag::update-request-doc-upsert
+            request.docAsUpsert(true); // <1>
+            // end::update-request-doc-upsert
+            // tag::update-request-active-shards
+            request.waitForActiveShards(2); // <1>
+            request.waitForActiveShards(ActiveShardCount.ALL); // <2>
+            // end::update-request-active-shards
+        }
+    }
+
+    public void testDelete() throws IOException {
+        RestHighLevelClient client = highLevelClient();
+
+        {
+            IndexRequest indexRequest = new IndexRequest("posts", "doc", "1").source("field", "value");
+            IndexResponse indexResponse = client.index(indexRequest);
+            assertSame(indexResponse.status(), RestStatus.CREATED);
+        }
+
+        {
+            // tag::delete-request
+            DeleteRequest request = new DeleteRequest(
+                    "posts",    // <1>
+                    "doc",     // <2>
+                    "1");      // <3>
+            // end::delete-request
+
+            // tag::delete-execute
+            DeleteResponse deleteResponse = client.delete(request);
+            // end::delete-execute
+            assertSame(deleteResponse.getResult(), DocWriteResponse.Result.DELETED);
+
+            // tag::delete-response
+            String index = deleteResponse.getIndex();
+            String type = deleteResponse.getType();
+            String id = deleteResponse.getId();
+            long version = deleteResponse.getVersion();
+            ReplicationResponse.ShardInfo shardInfo = deleteResponse.getShardInfo();
+            if (shardInfo.getTotal() != shardInfo.getSuccessful()) {
+                // <1>
+            }
+            if (shardInfo.getFailed() > 0) {
+                for (ReplicationResponse.ShardInfo.Failure failure : shardInfo.getFailures()) {
+                    String reason = failure.reason(); // <2>
+                }
+            }
+            // end::delete-response
+
+            // tag::delete-execute-async
+            client.deleteAsync(request, new ActionListener<DeleteResponse>() {
+                @Override
+                public void onResponse(DeleteResponse deleteResponse) {
+                    // <1>
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    // <2>
+                }
+            });
+            // end::delete-execute-async
+        }
+
+        {
+            DeleteRequest request = new DeleteRequest("posts", "doc", "1");
+            // tag::delete-request-routing
+            request.routing("routing"); // <1>
+            // end::delete-request-routing
+            // tag::delete-request-parent
+            request.parent("parent"); // <1>
+            // end::delete-request-parent
+            // tag::delete-request-timeout
+            request.timeout(TimeValue.timeValueMinutes(2)); // <1>
+            request.timeout("2m"); // <2>
+            // end::delete-request-timeout
+            // tag::delete-request-refresh
+            request.setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL); // <1>
+            request.setRefreshPolicy("wait_for");                            // <2>
+            // end::delete-request-refresh
+            // tag::delete-request-version
+            request.version(2); // <1>
+            // end::delete-request-version
+            // tag::delete-request-version-type
+            request.versionType(VersionType.EXTERNAL); // <1>
+            // end::delete-request-version-type
+        }
+
+        {
+            // tag::delete-notfound
+            DeleteRequest request = new DeleteRequest("posts", "doc", "does_not_exist");
+            DeleteResponse deleteResponse = client.delete(request);
+            if (deleteResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) {
+                // <1>
+            }
+            // end::delete-notfound
+        }
+
+        {
+            IndexResponse indexResponse = client.index(new IndexRequest("posts", "doc", "1").source("field", "value"));
+            assertSame(indexResponse.status(), RestStatus.CREATED);
+
+            // tag::delete-conflict
+            try {
+                DeleteRequest request = new DeleteRequest("posts", "doc", "1").version(2);
+                DeleteResponse deleteResponse = client.delete(request);
+            } catch (ElasticsearchException exception) {
+                if (exception.status() == RestStatus.CONFLICT) {
+                    // <1>
+                }
+            }
+            // end::delete-conflict
+        }
+    }
+
+    public void testBulk() throws IOException {
+        RestHighLevelClient client = highLevelClient();
+        {
+            // tag::bulk-request
+            BulkRequest request = new BulkRequest(); // <1>
+            request.add(new IndexRequest("posts", "doc", "1")  // <2>
+                    .source(XContentType.JSON,"field", "foo"));
+            request.add(new IndexRequest("posts", "doc", "2")  // <3>
+                    .source(XContentType.JSON,"field", "bar"));
+            request.add(new IndexRequest("posts", "doc", "3")  // <4>
+                    .source(XContentType.JSON,"field", "baz"));
+            // end::bulk-request
+            // tag::bulk-execute
+            BulkResponse bulkResponse = client.bulk(request);
+            // end::bulk-execute
+            assertSame(bulkResponse.status(), RestStatus.OK);
+            assertFalse(bulkResponse.hasFailures());
+        }
+        {
+            // tag::bulk-request-with-mixed-operations
+            BulkRequest request = new BulkRequest();
+            request.add(new DeleteRequest("posts", "doc", "3")); // <1>
+            request.add(new UpdateRequest("posts", "doc", "2") // <2>
+                    .doc(XContentType.JSON,"other", "test"));
+            request.add(new IndexRequest("posts", "doc", "4")  // <3>
+                    .source(XContentType.JSON,"field", "baz"));
+            // end::bulk-request-with-mixed-operations
+            BulkResponse bulkResponse = client.bulk(request);
+            assertSame(bulkResponse.status(), RestStatus.OK);
+            assertFalse(bulkResponse.hasFailures());
+
+            // tag::bulk-response
+            for (BulkItemResponse bulkItemResponse : bulkResponse) { // <1>
+                DocWriteResponse itemResponse = bulkItemResponse.getResponse(); // <2>
+
+                if (bulkItemResponse.getOpType() == DocWriteRequest.OpType.INDEX
+                        || bulkItemResponse.getOpType() == DocWriteRequest.OpType.CREATE) { // <3>
+                    IndexResponse indexResponse = (IndexResponse) itemResponse;
+
+                } else if (bulkItemResponse.getOpType() == DocWriteRequest.OpType.UPDATE) { // <4>
+                    UpdateResponse updateResponse = (UpdateResponse) itemResponse;
+
+                } else if (bulkItemResponse.getOpType() == DocWriteRequest.OpType.DELETE) { // <5>
+                    DeleteResponse deleteResponse = (DeleteResponse) itemResponse;
+                }
+            }
+            // end::bulk-response
+            // tag::bulk-has-failures
+            if (bulkResponse.hasFailures()) { // <1>
+
+            }
+            // end::bulk-has-failures
+            // tag::bulk-errors
+            for (BulkItemResponse bulkItemResponse : bulkResponse) {
+                if (bulkItemResponse.isFailed()) { // <1>
+                    BulkItemResponse.Failure failure = bulkItemResponse.getFailure(); // <2>
+
+                }
+            }
+            // end::bulk-errors
+        }
+        {
+            BulkRequest request = new BulkRequest();
+            // tag::bulk-request-timeout
+            request.timeout(TimeValue.timeValueMinutes(2)); // <1>
+            request.timeout("2m"); // <2>
+            // end::bulk-request-timeout
+            // tag::bulk-request-refresh
+            request.setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL); // <1>
+            request.setRefreshPolicy("wait_for");                            // <2>
+            // end::bulk-request-refresh
+            // tag::bulk-request-active-shards
+            request.waitForActiveShards(2); // <1>
+            request.waitForActiveShards(ActiveShardCount.ALL); // <2>
+            // end::bulk-request-active-shards
+
+            // tag::bulk-execute-async
+            client.bulkAsync(request, new ActionListener<BulkResponse>() {
+                @Override
+                public void onResponse(BulkResponse bulkResponse) {
+                    // <1>
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    // <2>
+                }
+            });
+            // end::bulk-execute-async
+        }
+    }
+
+    public void testGet() throws IOException {
+        RestHighLevelClient client = highLevelClient();
+        {
+            String mappings = "{\n" +
+                    "    \"mappings\" : {\n" +
+                    "        \"doc\" : {\n" +
+                    "            \"properties\" : {\n" +
+                    "                \"message\" : {\n" +
+                    "                    \"type\": \"text\",\n" +
+                    "                    \"store\": true\n" +
+                    "                }\n" +
+                    "            }\n" +
+                    "        }\n" +
+                    "    }\n" +
+                    "}";
+
+            NStringEntity entity = new NStringEntity(mappings, ContentType.APPLICATION_JSON);
+            Response response = client().performRequest("PUT", "/posts", Collections.emptyMap(), entity);
+            assertEquals(200, response.getStatusLine().getStatusCode());
+
+            IndexRequest indexRequest = new IndexRequest("posts", "doc", "1")
+                    .source("user", "kimchy",
+                            "postDate", new Date(),
+                            "message", "trying out Elasticsearch");
+            IndexResponse indexResponse = client.index(indexRequest);
+            assertEquals(indexResponse.getResult(), DocWriteResponse.Result.CREATED);
+        }
+        {
+            //tag::get-request
+            GetRequest getRequest = new GetRequest(
+                    "posts", // <1>
+                    "doc",  // <2>
+                    "1");   // <3>
+            //end::get-request
+
+            //tag::get-execute
+            GetResponse getResponse = client.get(getRequest);
+            //end::get-execute
+            assertTrue(getResponse.isExists());
+            assertEquals(3, getResponse.getSourceAsMap().size());
+            //tag::get-response
+            String index = getResponse.getIndex();
+            String type = getResponse.getType();
+            String id = getResponse.getId();
+            if (getResponse.isExists()) {
+                long version = getResponse.getVersion();
+                String sourceAsString = getResponse.getSourceAsString();        // <1>
+                Map<String, Object> sourceAsMap = getResponse.getSourceAsMap(); // <2>
+                byte[] sourceAsBytes = getResponse.getSourceAsBytes();          // <3>
+            } else {
+                // <4>
+            }
+            //end::get-response
+        }
+        {
+            GetRequest request = new GetRequest("posts", "doc", "1");
+            //tag::get-request-no-source
+            request.fetchSourceContext(new FetchSourceContext(false)); // <1>
+            //end::get-request-no-source
+            GetResponse getResponse = client.get(request);
+            assertNull(getResponse.getSourceInternal());
+        }
+        {
+            GetRequest request = new GetRequest("posts", "doc", "1");
+            //tag::get-request-source-include
+            String[] includes = new String[]{"message", "*Date"};
+            String[] excludes = Strings.EMPTY_ARRAY;
+            FetchSourceContext fetchSourceContext = new FetchSourceContext(true, includes, excludes);
+            request.fetchSourceContext(fetchSourceContext); // <1>
+            //end::get-request-source-include
+            GetResponse getResponse = client.get(request);
+            Map<String, Object> sourceAsMap = getResponse.getSourceAsMap();
+            assertEquals(2, sourceAsMap.size());
+            assertEquals("trying out Elasticsearch", sourceAsMap.get("message"));
+            assertTrue(sourceAsMap.containsKey("postDate"));
+        }
+        {
+            GetRequest request = new GetRequest("posts", "doc", "1");
+            //tag::get-request-source-exclude
+            String[] includes = Strings.EMPTY_ARRAY;
+            String[] excludes = new String[]{"message"};
+            FetchSourceContext fetchSourceContext = new FetchSourceContext(true, includes, excludes);
+            request.fetchSourceContext(fetchSourceContext); // <1>
+            //end::get-request-source-exclude
+            GetResponse getResponse = client.get(request);
+            Map<String, Object> sourceAsMap = getResponse.getSourceAsMap();
+            assertEquals(2, sourceAsMap.size());
+            assertEquals("kimchy", sourceAsMap.get("user"));
+            assertTrue(sourceAsMap.containsKey("postDate"));
+        }
+        {
+            GetRequest request = new GetRequest("posts", "doc", "1");
+            //tag::get-request-stored
+            request.storedFields("message"); // <1>
+            GetResponse getResponse = client.get(request);
+            String message = (String) getResponse.getField("message").getValue(); // <2>
+            //end::get-request-stored
+            assertEquals("trying out Elasticsearch", message);
+            assertEquals(1, getResponse.getFields().size());
+            assertNull(getResponse.getSourceInternal());
+        }
+        {
+            GetRequest request = new GetRequest("posts", "doc", "1");
+            //tag::get-request-routing
+            request.routing("routing"); // <1>
+            //end::get-request-routing
+            //tag::get-request-parent
+            request.parent("parent"); // <1>
+            //end::get-request-parent
+            //tag::get-request-preference
+            request.preference("preference"); // <1>
+            //end::get-request-preference
+            //tag::get-request-realtime
+            request.realtime(false); // <1>
+            //end::get-request-realtime
+            //tag::get-request-refresh
+            request.refresh(true); // <1>
+            //end::get-request-refresh
+            //tag::get-request-version
+            request.version(2); // <1>
+            //end::get-request-version
+            //tag::get-request-version-type
+            request.versionType(VersionType.EXTERNAL); // <1>
+            //end::get-request-version-type
+        }
+        {
+            GetRequest request = new GetRequest("posts", "doc", "1");
+            //tag::get-execute-async
+            client.getAsync(request, new ActionListener<GetResponse>() {
+                @Override
+                public void onResponse(GetResponse getResponse) {
+                    // <1>
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    // <2>
+                }
+            });
+            //end::get-execute-async
+        }
+        {
+            //tag::get-indexnotfound
+            GetRequest request = new GetRequest("does_not_exist", "doc", "1");
+            try {
+                GetResponse getResponse = client.get(request);
+            } catch (ElasticsearchException e) {
+                if (e.status() == RestStatus.NOT_FOUND) {
+                    // <1>
+                }
+            }
+            //end::get-indexnotfound
+        }
+        {
+            // tag::get-conflict
+            try {
+                GetRequest request = new GetRequest("posts", "doc", "1").version(2);
+                GetResponse getResponse = client.get(request);
+            } catch (ElasticsearchException exception) {
+                if (exception.status() == RestStatus.CONFLICT) {
+                    // <1>
+                }
+            }
+            // end::get-conflict
+        }
+    }
+
+    public void testBulkProcessor() throws InterruptedException, IOException {
+        Settings settings = Settings.builder().put("node.name", "my-application").build();
+        RestHighLevelClient client = highLevelClient();
+        {
+            // tag::bulk-processor-init
+            ThreadPool threadPool = new ThreadPool(settings); // <1>
+
+            BulkProcessor.Listener listener = new BulkProcessor.Listener() { // <2>
+                @Override
+                public void beforeBulk(long executionId, BulkRequest request) {
+                    // <3>
+                }
+
+                @Override
+                public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
+                    // <4>
+                }
+
+                @Override
+                public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
+                    // <5>
+                }
+            };
+
+            BulkProcessor bulkProcessor = new BulkProcessor.Builder(client::bulkAsync, listener, threadPool)
+                    .build(); // <6>
+            // end::bulk-processor-init
+            assertNotNull(bulkProcessor);
+
+            // tag::bulk-processor-add
+            IndexRequest one = new IndexRequest("posts", "doc", "1").
+                    source(XContentType.JSON, "title", "In which order are my Elasticsearch queries executed?");
+            IndexRequest two = new IndexRequest("posts", "doc", "2")
+                    .source(XContentType.JSON, "title", "Current status and upcoming changes in Elasticsearch");
+            IndexRequest three = new IndexRequest("posts", "doc", "3")
+                    .source(XContentType.JSON, "title", "The Future of Federated Search in Elasticsearch");
+
+            bulkProcessor.add(one);
+            bulkProcessor.add(two);
+            bulkProcessor.add(three);
+            // end::bulk-processor-add
+
+            // tag::bulk-processor-await
+            boolean terminated = bulkProcessor.awaitClose(30L, TimeUnit.SECONDS); // <1>
+            // end::bulk-processor-await
+            assertTrue(terminated);
+
+            // tag::bulk-processor-close
+            bulkProcessor.close();
+            // end::bulk-processor-close
+            terminate(threadPool);
+        }
+        {
+            // tag::bulk-processor-listener
+            BulkProcessor.Listener listener = new BulkProcessor.Listener() {
+                @Override
+                public void beforeBulk(long executionId, BulkRequest request) {
+                    int numberOfActions = request.numberOfActions(); // <1>
+                    logger.debug("Executing bulk [{}] with {} requests", executionId, numberOfActions);
+                }
+
+                @Override
+                public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
+                    if (response.hasFailures()) { // <2>
+                        logger.warn("Bulk [{}] executed with failures", executionId);
+                    } else {
+                        logger.debug("Bulk [{}] completed in {} milliseconds", executionId, response.getTook().getMillis());
+                    }
+                }
+
+                @Override
+                public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
+                    logger.error("Failed to execute bulk", failure); // <3>
+                }
+            };
+            // end::bulk-processor-listener
+
+            ThreadPool threadPool = new ThreadPool(settings);
+            try {
+                // tag::bulk-processor-options
+                BulkProcessor.Builder builder = new BulkProcessor.Builder(client::bulkAsync, listener, threadPool);
+                builder.setBulkActions(500); // <1>
+                builder.setBulkSize(new ByteSizeValue(1L, ByteSizeUnit.MB)); // <2>
+                builder.setConcurrentRequests(0); // <3>
+                builder.setFlushInterval(TimeValue.timeValueSeconds(10L)); // <4>
+                builder.setBackoffPolicy(BackoffPolicy.constantBackoff(TimeValue.timeValueSeconds(1L), 3)); // <5>
+                // end::bulk-processor-options
+            } finally {
+                terminate(threadPool);
+            }
+        }
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MainDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MainDocumentationIT.java
new file mode 100644
index 0000000000000..877ae910e83fa
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MainDocumentationIT.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.documentation;
+
+import org.elasticsearch.Build;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.main.MainResponse;
+import org.elasticsearch.client.ESRestHighLevelClientTestCase;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.cluster.ClusterName;
+
+import java.io.IOException;
+
+/**
+ * This class is used to generate the Java Main API documentation.
+ * You need to wrap your code between two tags like:
+ * // tag::example[]
+ * // end::example[]
+ *
+ * Where example is your tag name.
+ *
+ * Then in the documentation, you can extract what is between tag and end tags with
+ * ["source","java",subs="attributes,callouts,macros"]
+ * --------------------------------------------------
+ * include-tagged::{doc-tests}/MainDocumentationIT.java[example]
+ * --------------------------------------------------
+ */
+public class MainDocumentationIT extends ESRestHighLevelClientTestCase {
+
+    public void testMain() throws IOException {
+        RestHighLevelClient client = highLevelClient();
+        {
+            //tag::main-execute
+            MainResponse response = client.info();
+            //end::main-execute
+            assertTrue(response.isAvailable());
+            //tag::main-response
+            ClusterName clusterName = response.getClusterName(); // <1>
+            String clusterUuid = response.getClusterUuid(); // <2>
+            String nodeName = response.getNodeName(); // <3>
+            Version version = response.getVersion(); // <4>
+            Build build = response.getBuild(); // <5>
+            //end::main-response
+            assertNotNull(clusterName);
+            assertNotNull(clusterUuid);
+            assertNotNull(nodeName);
+            assertNotNull(version);
+            assertNotNull(build);
+        }
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MigrationDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MigrationDocumentationIT.java
new file mode 100644
index 0000000000000..0984e7d1c93fe
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MigrationDocumentationIT.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.documentation;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.delete.DeleteResponse;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.client.ESRestHighLevelClientTestCase;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpStatus;
+import org.apache.http.entity.ContentType;
+import org.apache.http.nio.entity.NStringEntity;
+import org.elasticsearch.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.rest.RestStatus;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Map;
+
+import static java.util.Collections.emptyMap;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
+
+/**
+ * This class is used to generate the documentation for the
+ * docs/java-rest/high-level/migration.asciidoc page.
+ *
+ * You need to wrap your code between two tags like:
+ * // tag::example[]
+ * // end::example[]
+ *
+ * Where example is your tag name.
+ *
+ * Then in the documentation, you can extract what is between tag and end tags with
+ * ["source","java",subs="attributes,callouts,macros"]
+ * --------------------------------------------------
+ * include-tagged::{doc-tests}/MigrationDocumentationIT.java[example]
+ * --------------------------------------------------
+ */
+public class MigrationDocumentationIT extends ESRestHighLevelClientTestCase {
+
+    public void testCreateIndex() throws IOException {
+        RestClient restClient = client();
+        {
+            //tag::migration-create-inded
+            Settings indexSettings = Settings.builder() // <1>
+                    .put(SETTING_NUMBER_OF_SHARDS, 1)
+                    .put(SETTING_NUMBER_OF_REPLICAS, 0)
+                    .build();
+
+            String payload = XContentFactory.jsonBuilder() // <2>
+                    .startObject()
+                        .startObject("settings") // <3>
+                            .value(indexSettings)
+                        .endObject()
+                        .startObject("mappings")  // <4>
+                            .startObject("doc")
+                                .startObject("properties")
+                                    .startObject("time")
+                                        .field("type", "date")
+                                    .endObject()
+                                .endObject()
+                            .endObject()
+                        .endObject()
+                    .endObject().string();
+
+            HttpEntity entity = new NStringEntity(payload, ContentType.APPLICATION_JSON); // <5>
+
+            Response response = restClient.performRequest("PUT", "my-index", emptyMap(), entity); // <6>
+            if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
+                // <7>
+            }
+            //end::migration-create-inded
+            assertEquals(200, response.getStatusLine().getStatusCode());
+        }
+    }
+
+    public void testClusterHealth() throws IOException {
+        RestClient restClient = client();
+        {
+            //tag::migration-cluster-health
+            Response response = restClient.performRequest("GET", "/_cluster/health"); // <1>
+
+            ClusterHealthStatus healthStatus;
+            try (InputStream is = response.getEntity().getContent()) { // <2>
+                Map<String, Object> map = XContentHelper.convertToMap(XContentType.JSON.xContent(), is, true); // <3>
+                healthStatus = ClusterHealthStatus.fromString((String) map.get("status")); // <4>
+            }
+
+            if (healthStatus == ClusterHealthStatus.GREEN) {
+                // <5>
+            }
+            //end::migration-cluster-health
+            assertSame(ClusterHealthStatus.GREEN, healthStatus);
+        }
+    }
+
+    public void testRequests() throws IOException {
+        RestHighLevelClient client = highLevelClient();
+        {
+            //tag::migration-request-ctor
+            IndexRequest request = new IndexRequest("index", "doc", "id"); // <1>
+            request.source("{\"field\":\"value\"}", XContentType.JSON);
+            //end::migration-request-ctor
+
+            //tag::migration-request-ctor-execution
+            IndexResponse response = client.index(request);
+            //end::migration-request-ctor-execution
+            assertEquals(RestStatus.CREATED, response.status());
+        }
+        {
+            //tag::migration-request-sync-execution
+            DeleteRequest request = new DeleteRequest("index", "doc", "id");
+            DeleteResponse response = client.delete(request); // <1>
+            //end::migration-request-sync-execution
+            assertEquals(RestStatus.OK, response.status());
+        }
+        {
+            //tag::migration-request-async-execution
+            DeleteRequest request = new DeleteRequest("index", "doc", "id"); // <1>
+            client.deleteAsync(request, new ActionListener<DeleteResponse>() { // <2>
+                @Override
+                public void onResponse(DeleteResponse deleteResponse) {
+                    // <3>
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    // <4>
+                }
+            });
+            //end::migration-request-async-execution
+        }
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/QueryDSLDocumentationTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/QueryDSLDocumentationTests.java
new file mode 100644
index 0000000000000..2459d2e587297
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/QueryDSLDocumentationTests.java
@@ -0,0 +1,452 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.documentation;
+
+import org.apache.lucene.search.join.ScoreMode;
+import org.elasticsearch.common.geo.GeoPoint;
+import org.elasticsearch.common.geo.ShapeRelation;
+import org.elasticsearch.common.geo.builders.CoordinatesBuilder;
+import org.elasticsearch.common.geo.builders.ShapeBuilders;
+import org.elasticsearch.common.unit.DistanceUnit;
+import org.elasticsearch.index.query.GeoShapeQueryBuilder;
+import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
+import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder.FilterFunctionBuilder;
+import org.elasticsearch.join.query.JoinQueryBuilders;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptType;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static java.util.Collections.singletonMap;
+import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
+import static org.elasticsearch.index.query.QueryBuilders.boostingQuery;
+import static org.elasticsearch.index.query.QueryBuilders.commonTermsQuery;
+import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery;
+import static org.elasticsearch.index.query.QueryBuilders.disMaxQuery;
+import static org.elasticsearch.index.query.QueryBuilders.existsQuery;
+import static org.elasticsearch.index.query.QueryBuilders.functionScoreQuery;
+import static org.elasticsearch.index.query.QueryBuilders.fuzzyQuery;
+import static org.elasticsearch.index.query.QueryBuilders.geoBoundingBoxQuery;
+import static org.elasticsearch.index.query.QueryBuilders.geoDistanceQuery;
+import static org.elasticsearch.index.query.QueryBuilders.geoPolygonQuery;
+import static org.elasticsearch.index.query.QueryBuilders.geoShapeQuery;
+import static org.elasticsearch.index.query.QueryBuilders.idsQuery;
+import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
+import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
+import static org.elasticsearch.index.query.QueryBuilders.moreLikeThisQuery;
+import static org.elasticsearch.index.query.QueryBuilders.multiMatchQuery;
+import static org.elasticsearch.index.query.QueryBuilders.nestedQuery;
+import static org.elasticsearch.index.query.QueryBuilders.prefixQuery;
+import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery;
+import static org.elasticsearch.index.query.QueryBuilders.rangeQuery;
+import static org.elasticsearch.index.query.QueryBuilders.regexpQuery;
+import static org.elasticsearch.index.query.QueryBuilders.scriptQuery;
+import static org.elasticsearch.index.query.QueryBuilders.simpleQueryStringQuery;
+import static org.elasticsearch.index.query.QueryBuilders.spanContainingQuery;
+import static org.elasticsearch.index.query.QueryBuilders.spanFirstQuery;
+import static org.elasticsearch.index.query.QueryBuilders.spanMultiTermQueryBuilder;
+import static org.elasticsearch.index.query.QueryBuilders.spanNearQuery;
+import static org.elasticsearch.index.query.QueryBuilders.spanNotQuery;
+import static org.elasticsearch.index.query.QueryBuilders.spanOrQuery;
+import static org.elasticsearch.index.query.QueryBuilders.spanTermQuery;
+import static org.elasticsearch.index.query.QueryBuilders.spanWithinQuery;
+import static org.elasticsearch.index.query.QueryBuilders.termQuery;
+import static org.elasticsearch.index.query.QueryBuilders.termsQuery;
+import static org.elasticsearch.index.query.QueryBuilders.typeQuery;
+import static org.elasticsearch.index.query.QueryBuilders.wildcardQuery;
+import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.exponentialDecayFunction;
+import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.randomFunction;
+
+/**
+ * Examples of using the transport client that are imported into the transport client documentation.
+ * There are no assertions here because we're mostly concerned with making sure that the examples
+ * compile and don't throw weird runtime exceptions. Assertions and example data would be nice, but
+ * that is secondary.
+ */
+public class QueryDSLDocumentationTests extends ESTestCase {
+    public void testBool() {
+        // tag::bool
+        boolQuery()
+                .must(termQuery("content", "test1"))                 // <1>
+                .must(termQuery("content", "test4"))                 // <1>
+                .mustNot(termQuery("content", "test2"))              // <2>
+                .should(termQuery("content", "test3"))               // <3>
+                .filter(termQuery("content", "test5"));              // <4>
+        // end::bool
+    }
+
+    public void testBoosting() {
+        // tag::boosting
+        boostingQuery(
+                    termQuery("name","kimchy"),                      // <1>
+                    termQuery("name","dadoonet"))                    // <2>
+                .negativeBoost(0.2f);                                // <3>
+        // end::boosting
+    }
+
+    public void testCommonTerms() {
+        // tag::common_terms
+        commonTermsQuery("name",                                     // <1>
+                         "kimchy");                                  // <2>
+        // end::common_terms
+    }
+
+    public void testConstantScore() {
+        // tag::constant_score
+        constantScoreQuery(
+                termQuery("name","kimchy"))                          // <1>
+            .boost(2.0f);                                            // <2>
+        // end::constant_score
+    }
+
+    public void testDisMax() {
+        // tag::dis_max
+        disMaxQuery()
+                .add(termQuery("name", "kimchy"))                    // <1>
+                .add(termQuery("name", "elasticsearch"))             // <2>
+                .boost(1.2f)                                         // <3>
+                .tieBreaker(0.7f);                                   // <4>
+        // end::dis_max
+    }
+
+    public void testExists() {
+        // tag::exists
+        existsQuery("name");                                         // <1>
+        // end::exists
+    }
+
+    public void testFunctionScore() {
+        // tag::function_score
+        FilterFunctionBuilder[] functions = {
+                new FunctionScoreQueryBuilder.FilterFunctionBuilder(
+                        matchQuery("name", "kimchy"),                // <1>
+                        randomFunction("ABCDEF")),                   // <2>
+                new FunctionScoreQueryBuilder.FilterFunctionBuilder(
+                        exponentialDecayFunction("age", 0L, 1L))     // <3>
+        };
+        functionScoreQuery(functions);
+        // end::function_score
+    }
+
+    public void testFuzzy() {
+        // tag::fuzzy
+        fuzzyQuery(
+                "name",                                              // <1>
+                "kimchy");                                           // <2>
+        // end::fuzzy
+    }
+
+    public void testGeoBoundingBox() {
+        // tag::geo_bounding_box
+        geoBoundingBoxQuery("pin.location")                          // <1>
+            .setCorners(40.73, -74.1,                                // <2>
+                        40.717, -73.99);                             // <3>
+        // end::geo_bounding_box
+    }
+
+    public void testGeoDistance() {
+        // tag::geo_distance
+        geoDistanceQuery("pin.location")                             // <1>
+            .point(40, -70)                                          // <2>
+            .distance(200, DistanceUnit.KILOMETERS);                 // <3>
+        // end::geo_distance
+    }
+
+    public void testGeoPolygon() {
+        // tag::geo_polygon
+        List<GeoPoint> points = new ArrayList<GeoPoint>();           // <1>
+        points.add(new GeoPoint(40, -70));
+        points.add(new GeoPoint(30, -80));
+        points.add(new GeoPoint(20, -90));
+        geoPolygonQuery("pin.location", points);                     // <2>
+        // end::geo_polygon
+    }
+
+    public void testGeoShape() throws IOException {
+        {
+            // tag::geo_shape
+            GeoShapeQueryBuilder qb = geoShapeQuery(
+                    "pin.location",                                      // <1>
+                    ShapeBuilders.newMultiPoint(                         // <2>
+                            new CoordinatesBuilder()
+                        .coordinate(0, 0)
+                        .coordinate(0, 10)
+                        .coordinate(10, 10)
+                        .coordinate(10, 0)
+                        .coordinate(0, 0)
+                        .build()));
+            qb.relation(ShapeRelation.WITHIN);                           // <3>
+            // end::geo_shape
+        }
+
+        {
+            // tag::indexed_geo_shape
+            // Using pre-indexed shapes
+            GeoShapeQueryBuilder qb = geoShapeQuery(
+                        "pin.location",                                  // <1>
+                        "DEU",                                           // <2>
+                        "countries");                                    // <3>
+            qb.relation(ShapeRelation.WITHIN)                            // <4>
+                .indexedShapeIndex("shapes")                             // <5>
+                .indexedShapePath("location");                           // <6>
+            // end::indexed_geo_shape
+        }
+    }
+
+    public void testHasChild() {
+        // tag::has_child
+        JoinQueryBuilders.hasChildQuery(
+                "blog_tag",                                          // <1>
+                termQuery("tag","something"),                        // <2>
+                ScoreMode.None);                                     // <3>
+        // end::has_child
+    }
+
+    public void testHasParent() {
+        // tag::has_parent
+        JoinQueryBuilders.hasParentQuery(
+            "blog",                                                  // <1>
+            termQuery("tag","something"),                            // <2>
+            false);                                                  // <3>
+        // end::has_parent
+    }
+
+    public void testIds() {
+        // tag::ids
+        idsQuery("my_type", "type2")
+                .addIds("1", "4", "100");
+
+        idsQuery()                                                   // <1>
+                .addIds("1", "4", "100");
+        // end::ids
+    }
+
+    public void testMatchAll() {
+        // tag::match_all
+        matchAllQuery();
+        // end::match_all
+    }
+
+    public void testMatch() {
+        // tag::match
+        matchQuery(
+                "name",                                              // <1>
+                "kimchy elasticsearch");                             // <2>
+        // end::match
+    }
+
+    public void testMoreLikeThis() {
+        // tag::more_like_this
+        String[] fields = {"name.first", "name.last"};               // <1>
+        String[] texts = {"text like this one"};                     // <2>
+
+        moreLikeThisQuery(fields, texts, null)
+            .minTermFreq(1)                                          // <3>
+            .maxQueryTerms(12);                                      // <4>
+        // end::more_like_this
+    }
+
+    public void testMultiMatch() {
+        // tag::multi_match
+        multiMatchQuery(
+                "kimchy elasticsearch",                              // <1>
+                "user", "message");                                  // <2>
+        // end::multi_match
+    }
+
+    public void testNested() {
+        // tag::nested
+        nestedQuery(
+                "obj1",                                              // <1>
+                boolQuery()                                          // <2>
+                        .must(matchQuery("obj1.name", "blue"))
+                        .must(rangeQuery("obj1.count").gt(5)),
+                ScoreMode.Avg);                                      // <3>
+        // end::nested
+    }
+
+    public void testPrefix() {
+        // tag::prefix
+        prefixQuery(
+                "brand",                                             // <1>
+                "heine");                                            // <2>
+        // end::prefix
+    }
+
+    public void testQueryString() {
+        // tag::query_string
+        queryStringQuery("+kimchy -elasticsearch");
+        // end::query_string
+    }
+
+    public void testRange() {
+        // tag::range
+        rangeQuery("price")                                          // <1>
+            .from(5)                                                 // <2>
+            .to(10)                                                  // <3>
+            .includeLower(true)                                      // <4>
+            .includeUpper(false);                                    // <5>
+        // end::range
+
+        // tag::range_simplified
+        // A simplified form using gte, gt, lt or lte
+        rangeQuery("age")                                             // <1>
+            .gte("10")                                                // <2>
+            .lt("20");                                                // <3>
+        // end::range_simplified
+    }
+
+    public void testRegExp() {
+        // tag::regexp
+        regexpQuery(
+                "name.first",                                        // <1>
+                "s.*y");                                             // <2>
+        // end::regexp
+    }
+
+    public void testScript() {
+        // tag::script_inline
+        scriptQuery(
+                new Script("doc['num1'].value > 1")                  // <1>
+            );
+        // end::script_inline
+
+        // tag::script_file
+        Map<String, Object> parameters = new HashMap<>();
+        parameters.put("param1", 5);
+        scriptQuery(new Script(
+                ScriptType.STORED,                                   // <1>
+                "painless",                                          // <2>
+                "myscript",                                          // <3>
+                singletonMap("param1", 5)));                         // <4>
+        // end::script_file
+    }
+
+    public void testSimpleQueryString() {
+        // tag::simple_query_string
+        simpleQueryStringQuery("+kimchy -elasticsearch");
+        // end::simple_query_string
+    }
+
+    public void testSpanContaining() {
+        // tag::span_containing
+        spanContainingQuery(
+                spanNearQuery(spanTermQuery("field1","bar"), 5)      // <1>
+                    .addClause(spanTermQuery("field1","baz"))
+                    .inOrder(true),
+                spanTermQuery("field1","foo"));                      // <2>
+        // end::span_containing
+    }
+
+    public void testSpanFirst() {
+        // tag::span_first
+        spanFirstQuery(
+                spanTermQuery("user", "kimchy"),                     // <1>
+                3                                                    // <2>
+            );
+        // end::span_first
+    }
+
+    public void testSpanMultiTerm() {
+        // tag::span_multi
+        spanMultiTermQueryBuilder(
+                prefixQuery("user", "ki"));                          // <1>
+        // end::span_multi
+    }
+
+    public void testSpanNear() {
+        // tag::span_near
+        spanNearQuery(
+                spanTermQuery("field","value1"),                     // <1>
+                12)                                                  // <2>
+                    .addClause(spanTermQuery("field","value2"))      // <1>
+                    .addClause(spanTermQuery("field","value3"))      // <1>
+                    .inOrder(false);                                 // <3>
+        // end::span_near
+    }
+
+    public void testSpanNot() {
+        // tag::span_not
+        spanNotQuery(
+                spanTermQuery("field","value1"),                     // <1>
+                spanTermQuery("field","value2"));                    // <2>
+        // end::span_not
+    }
+
+    public void testSpanOr() {
+        // tag::span_or
+        spanOrQuery(spanTermQuery("field","value1"))                 // <1>
+            .addClause(spanTermQuery("field","value2"))              // <1>
+            .addClause(spanTermQuery("field","value3"));             // <1>
+        // end::span_or
+    }
+
+    public void testSpanTerm() {
+        // tag::span_term
+        spanTermQuery(
+                "user",       // <1>
+                "kimchy");    // <2>
+        // end::span_term
+    }
+
+    public void testSpanWithin() {
+        // tag::span_within
+        spanWithinQuery(
+                spanNearQuery(spanTermQuery("field1", "bar"), 5)     // <1>
+                    .addClause(spanTermQuery("field1", "baz"))
+                    .inOrder(true),
+                spanTermQuery("field1", "foo"));                     // <2>
+        // end::span_within
+    }
+
+    public void testTerm() {
+        // tag::term
+        termQuery(
+                "name",                                              // <1>
+                "kimchy");                                           // <2>
+        // end::term
+    }
+
+    public void testTerms() {
+        // tag::terms
+        termsQuery("tags",                                           // <1>
+                "blue", "pill");                                     // <2>
+        // end::terms
+    }
+
+    public void testType() {
+        // tag::type
+        typeQuery("my_type");                                        // <1>
+        // end::type
+    }
+
+    public void testWildcard() {
+        // tag::wildcard
+        wildcardQuery(
+                "user",                                              // <1>
+                "k?mch*");                                           // <2>
+        // end::wildcard
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java
new file mode 100644
index 0000000000000..8c900275a50d5
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java
@@ -0,0 +1,674 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.documentation;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.bulk.BulkRequest;
+import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.action.search.ClearScrollRequest;
+import org.elasticsearch.action.search.ClearScrollResponse;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.SearchScrollRequest;
+import org.elasticsearch.action.search.ShardSearchFailure;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.client.ESRestHighLevelClientTestCase;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.common.text.Text;
+import org.elasticsearch.common.unit.Fuzziness;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.query.MatchQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.search.Scroll;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.search.aggregations.Aggregation;
+import org.elasticsearch.search.aggregations.AggregationBuilders;
+import org.elasticsearch.search.aggregations.Aggregations;
+import org.elasticsearch.search.aggregations.bucket.range.Range;
+import org.elasticsearch.search.aggregations.bucket.terms.Terms;
+import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket;
+import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.avg.Avg;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
+import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
+import org.elasticsearch.search.profile.ProfileResult;
+import org.elasticsearch.search.profile.ProfileShardResult;
+import org.elasticsearch.search.profile.aggregation.AggregationProfileShardResult;
+import org.elasticsearch.search.profile.query.CollectorResult;
+import org.elasticsearch.search.profile.query.QueryProfileShardResult;
+import org.elasticsearch.search.sort.FieldSortBuilder;
+import org.elasticsearch.search.sort.ScoreSortBuilder;
+import org.elasticsearch.search.sort.SortOrder;
+import org.elasticsearch.search.suggest.Suggest;
+import org.elasticsearch.search.suggest.SuggestBuilder;
+import org.elasticsearch.search.suggest.SuggestBuilders;
+import org.elasticsearch.search.suggest.SuggestionBuilder;
+import org.elasticsearch.search.suggest.term.TermSuggestion;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+
+/**
+ * This class is used to generate the Java High Level REST Client Search API documentation.
+ * <p>
+ * You need to wrap your code between two tags like:
+ * // tag::example[]
+ * // end::example[]
+ * <p>
+ * Where example is your tag name.
+ * <p>
+ * Then in the documentation, you can extract what is between tag and end tags with
+ * ["source","java",subs="attributes,callouts,macros"]
+ * --------------------------------------------------
+ * include-tagged::{doc-tests}/SearchDocumentationIT.java[example]
+ * --------------------------------------------------
+ */
+public class SearchDocumentationIT extends ESRestHighLevelClientTestCase {
+
+    @SuppressWarnings({ "unused", "unchecked" })
+    public void testSearch() throws IOException {
+        RestHighLevelClient client = highLevelClient();
+        {
+            BulkRequest request = new BulkRequest();
+            request.add(new IndexRequest("posts", "doc", "1")
+                    .source(XContentType.JSON, "title", "In which order are my Elasticsearch queries executed?", "user",
+                            Arrays.asList("kimchy", "luca"), "innerObject", Collections.singletonMap("key", "value")));
+            request.add(new IndexRequest("posts", "doc", "2")
+                    .source(XContentType.JSON, "title", "Current status and upcoming changes in Elasticsearch", "user",
+                            Arrays.asList("kimchy", "christoph"), "innerObject", Collections.singletonMap("key", "value")));
+            request.add(new IndexRequest("posts", "doc", "3")
+                    .source(XContentType.JSON, "title", "The Future of Federated Search in Elasticsearch", "user",
+                            Arrays.asList("kimchy", "tanguy"), "innerObject", Collections.singletonMap("key", "value")));
+            request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
+            BulkResponse bulkResponse = client.bulk(request);
+            assertSame(RestStatus.OK, bulkResponse.status());
+            assertFalse(bulkResponse.hasFailures());
+        }
+        {
+            // tag::search-request-basic
+            SearchRequest searchRequest = new SearchRequest(); // <1>
+            SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); // <2>
+            searchSourceBuilder.query(QueryBuilders.matchAllQuery()); // <3>
+            // end::search-request-basic
+        }
+        {
+            // tag::search-request-indices-types
+            SearchRequest searchRequest = new SearchRequest("posts"); // <1>
+            searchRequest.types("doc"); // <2>
+            // end::search-request-indices-types
+            // tag::search-request-routing
+            searchRequest.routing("routing"); // <1>
+            // end::search-request-routing
+            // tag::search-request-indicesOptions
+            searchRequest.indicesOptions(IndicesOptions.lenientExpandOpen()); // <1>
+            // end::search-request-indicesOptions
+            // tag::search-request-preference
+            searchRequest.preference("_local"); // <1>
+            // end::search-request-preference
+            assertNotNull(client.search(searchRequest));
+        }
+        {
+            // tag::search-source-basics
+            SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); // <1>
+            sourceBuilder.query(QueryBuilders.termQuery("user", "kimchy")); // <2>
+            sourceBuilder.from(0); // <3>
+            sourceBuilder.size(5); // <4>
+            sourceBuilder.timeout(new TimeValue(60, TimeUnit.SECONDS)); // <5>
+            // end::search-source-basics
+
+            // tag::search-source-sorting
+            sourceBuilder.sort(new ScoreSortBuilder().order(SortOrder.DESC)); // <1>
+            sourceBuilder.sort(new FieldSortBuilder("_uid").order(SortOrder.ASC));  // <2>
+            // end::search-source-sorting
+
+            // tag::search-source-filtering-off
+            sourceBuilder.fetchSource(false);
+            // end::search-source-filtering-off
+            // tag::search-source-filtering-includes
+            String[] includeFields = new String[] {"title", "user", "innerObject.*"};
+            String[] excludeFields = new String[] {"_type"};
+            sourceBuilder.fetchSource(includeFields, excludeFields);
+            // end::search-source-filtering-includes
+            sourceBuilder.fetchSource(true);
+
+            // tag::search-source-setter
+            SearchRequest searchRequest = new SearchRequest();
+            searchRequest.source(sourceBuilder);
+            // end::search-source-setter
+
+            // tag::search-execute
+            SearchResponse searchResponse = client.search(searchRequest);
+            // end::search-execute
+
+            // tag::search-execute-async
+            client.searchAsync(searchRequest, new ActionListener<SearchResponse>() {
+                @Override
+                public void onResponse(SearchResponse searchResponse) {
+                    // <1>
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    // <2>
+                }
+            });
+            // end::search-execute-async
+
+            // tag::search-response-1
+            RestStatus status = searchResponse.status();
+            TimeValue took = searchResponse.getTook();
+            Boolean terminatedEarly = searchResponse.isTerminatedEarly();
+            boolean timedOut = searchResponse.isTimedOut();
+            // end::search-response-1
+
+            // tag::search-response-2
+            int totalShards = searchResponse.getTotalShards();
+            int successfulShards = searchResponse.getSuccessfulShards();
+            int failedShards = searchResponse.getFailedShards();
+            for (ShardSearchFailure failure : searchResponse.getShardFailures()) {
+                // failures should be handled here
+            }
+            // end::search-response-2
+            assertNotNull(searchResponse);
+
+            // tag::search-hits-get
+            SearchHits hits = searchResponse.getHits();
+            // end::search-hits-get
+            // tag::search-hits-info
+            long totalHits = hits.getTotalHits();
+            float maxScore = hits.getMaxScore();
+            // end::search-hits-info
+            // tag::search-hits-singleHit
+            SearchHit[] searchHits = hits.getHits();
+            for (SearchHit hit : searchHits) {
+                // do something with the SearchHit
+            }
+            // end::search-hits-singleHit
+            for (SearchHit hit : searchHits) {
+                // tag::search-hits-singleHit-properties
+                String index = hit.getIndex();
+                String type = hit.getType();
+                String id = hit.getId();
+                float score = hit.getScore();
+                // end::search-hits-singleHit-properties
+                // tag::search-hits-singleHit-source
+                String sourceAsString = hit.getSourceAsString();
+                Map<String, Object> sourceAsMap = hit.getSourceAsMap();
+                String documentTitle = (String) sourceAsMap.get("title");
+                List<Object> users = (List<Object>) sourceAsMap.get("user");
+                Map<String, Object> innerObject = (Map<String, Object>) sourceAsMap.get("innerObject");
+                // end::search-hits-singleHit-source
+            }
+            assertEquals(3, totalHits);
+            assertNotNull(hits.getHits()[0].getSourceAsString());
+            assertNotNull(hits.getHits()[0].getSourceAsMap().get("title"));
+            assertNotNull(hits.getHits()[0].getSourceAsMap().get("user"));
+            assertNotNull(hits.getHits()[0].getSourceAsMap().get("innerObject"));
+        }
+    }
+
+    @SuppressWarnings("unused")
+    public void testBuildingSearchQueries() {
+        RestHighLevelClient client = highLevelClient();
+        {
+            // tag::search-query-builder-ctor
+            MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder("user", "kimchy"); // <1>
+            // end::search-query-builder-ctor
+            // tag::search-query-builder-options
+            matchQueryBuilder.fuzziness(Fuzziness.AUTO); // <1>
+            matchQueryBuilder.prefixLength(3); // <2>
+            matchQueryBuilder.maxExpansions(10); // <3>
+            // end::search-query-builder-options
+        }
+        {
+            // tag::search-query-builders
+            QueryBuilder matchQueryBuilder = QueryBuilders.matchQuery("user", "kimchy")
+                                                            .fuzziness(Fuzziness.AUTO)
+                                                            .prefixLength(3)
+                                                            .maxExpansions(10);
+            // end::search-query-builders
+            SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+            // tag::search-query-setter
+            searchSourceBuilder.query(matchQueryBuilder);
+            // end::search-query-setter
+        }
+    }
+
+    @SuppressWarnings({ "unused" })
+    public void testSearchRequestAggregations() throws IOException {
+        RestHighLevelClient client = highLevelClient();
+        {
+            BulkRequest request = new BulkRequest();
+            request.add(new IndexRequest("posts", "doc", "1")
+                    .source(XContentType.JSON, "company", "Elastic", "age", 20));
+            request.add(new IndexRequest("posts", "doc", "2")
+                    .source(XContentType.JSON, "company", "Elastic", "age", 30));
+            request.add(new IndexRequest("posts", "doc", "3")
+                    .source(XContentType.JSON, "company", "Elastic", "age", 40));
+            request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
+            BulkResponse bulkResponse = client.bulk(request);
+            assertSame(RestStatus.OK, bulkResponse.status());
+            assertFalse(bulkResponse.hasFailures());
+        }
+        {
+            SearchRequest searchRequest = new SearchRequest();
+            // tag::search-request-aggregations
+            SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+            TermsAggregationBuilder aggregation = AggregationBuilders.terms("by_company")
+                    .field("company.keyword");
+            aggregation.subAggregation(AggregationBuilders.avg("average_age")
+                    .field("age"));
+            searchSourceBuilder.aggregation(aggregation);
+            // end::search-request-aggregations
+            searchSourceBuilder.query(QueryBuilders.matchAllQuery());
+            searchRequest.source(searchSourceBuilder);
+            SearchResponse searchResponse = client.search(searchRequest);
+            {
+                // tag::search-request-aggregations-get
+                Aggregations aggregations = searchResponse.getAggregations();
+                Terms byCompanyAggregation = aggregations.get("by_company"); // <1>
+                Bucket elasticBucket = byCompanyAggregation.getBucketByKey("Elastic"); // <2>
+                Avg averageAge = elasticBucket.getAggregations().get("average_age"); // <3>
+                double avg = averageAge.getValue();
+                // end::search-request-aggregations-get
+
+                try {
+                    // tag::search-request-aggregations-get-wrongCast
+                    Range range = aggregations.get("by_company"); // <1>
+                    // end::search-request-aggregations-get-wrongCast
+                } catch (ClassCastException ex) {
+                    assertEquals("org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms"
+                            + " cannot be cast to org.elasticsearch.search.aggregations.bucket.range.Range", ex.getMessage());
+                }
+                assertEquals(3, elasticBucket.getDocCount());
+                assertEquals(30, avg, 0.0);
+            }
+            Aggregations aggregations = searchResponse.getAggregations();
+            {
+                // tag::search-request-aggregations-asMap
+                Map<String, Aggregation> aggregationMap = aggregations.getAsMap();
+                Terms companyAggregation = (Terms) aggregationMap.get("by_company");
+                // end::search-request-aggregations-asMap
+            }
+            {
+                // tag::search-request-aggregations-asList
+                List<Aggregation> aggregationList = aggregations.asList();
+                // end::search-request-aggregations-asList
+            }
+            {
+                // tag::search-request-aggregations-iterator
+                for (Aggregation agg : aggregations) {
+                    String type = agg.getType();
+                    if (type.equals(TermsAggregationBuilder.NAME)) {
+                        Bucket elasticBucket = ((Terms) agg).getBucketByKey("Elastic");
+                        long numberOfDocs = elasticBucket.getDocCount();
+                    }
+                }
+                // end::search-request-aggregations-iterator
+            }
+        }
+    }
+
+    @SuppressWarnings({ "unused", "rawtypes" })
+    public void testSearchRequestSuggestions() throws IOException {
+        RestHighLevelClient client = highLevelClient();
+        {
+            BulkRequest request = new BulkRequest();
+            request.add(new IndexRequest("posts", "doc", "1").source(XContentType.JSON, "user", "kimchy"));
+            request.add(new IndexRequest("posts", "doc", "2").source(XContentType.JSON, "user", "javanna"));
+            request.add(new IndexRequest("posts", "doc", "3").source(XContentType.JSON, "user", "tlrx"));
+            request.add(new IndexRequest("posts", "doc", "4").source(XContentType.JSON, "user", "cbuescher"));
+            request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
+            BulkResponse bulkResponse = client.bulk(request);
+            assertSame(RestStatus.OK, bulkResponse.status());
+            assertFalse(bulkResponse.hasFailures());
+        }
+        {
+            SearchRequest searchRequest = new SearchRequest();
+            // tag::search-request-suggestion
+            SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+            SuggestionBuilder termSuggestionBuilder =
+                SuggestBuilders.termSuggestion("user").text("kmichy"); // <1>
+            SuggestBuilder suggestBuilder = new SuggestBuilder();
+            suggestBuilder.addSuggestion("suggest_user", termSuggestionBuilder); // <2>
+            searchSourceBuilder.suggest(suggestBuilder);
+            // end::search-request-suggestion
+            searchRequest.source(searchSourceBuilder);
+            SearchResponse searchResponse = client.search(searchRequest);
+            {
+                // tag::search-request-suggestion-get
+                Suggest suggest = searchResponse.getSuggest(); // <1>
+                TermSuggestion termSuggestion = suggest.getSuggestion("suggest_user"); // <2>
+                for (TermSuggestion.Entry entry : termSuggestion.getEntries()) { // <3>
+                    for (TermSuggestion.Entry.Option option : entry) { // <4>
+                        String suggestText = option.getText().string();
+                    }
+                }
+                // end::search-request-suggestion-get
+                assertEquals(1, termSuggestion.getEntries().size());
+                assertEquals(1, termSuggestion.getEntries().get(0).getOptions().size());
+                assertEquals("kimchy", termSuggestion.getEntries().get(0).getOptions().get(0).getText().string());
+            }
+        }
+    }
+
+    public void testSearchRequestHighlighting() throws IOException {
+        RestHighLevelClient client = highLevelClient();
+        {
+            BulkRequest request = new BulkRequest();
+            request.add(new IndexRequest("posts", "doc", "1")
+                    .source(XContentType.JSON, "title", "In which order are my Elasticsearch queries executed?", "user",
+                            Arrays.asList("kimchy", "luca"), "innerObject", Collections.singletonMap("key", "value")));
+            request.add(new IndexRequest("posts", "doc", "2")
+                    .source(XContentType.JSON, "title", "Current status and upcoming changes in Elasticsearch", "user",
+                            Arrays.asList("kimchy", "christoph"), "innerObject", Collections.singletonMap("key", "value")));
+            request.add(new IndexRequest("posts", "doc", "3")
+                    .source(XContentType.JSON, "title", "The Future of Federated Search in Elasticsearch", "user",
+                            Arrays.asList("kimchy", "tanguy"), "innerObject", Collections.singletonMap("key", "value")));
+            request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
+            BulkResponse bulkResponse = client.bulk(request);
+            assertSame(RestStatus.OK, bulkResponse.status());
+            assertFalse(bulkResponse.hasFailures());
+        }
+        {
+            SearchRequest searchRequest = new SearchRequest();
+            // tag::search-request-highlighting
+            SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+            HighlightBuilder highlightBuilder = new HighlightBuilder(); // <1>
+            HighlightBuilder.Field highlightTitle =
+                    new HighlightBuilder.Field("title"); // <2>
+            highlightTitle.highlighterType("unified");  // <3>
+            highlightBuilder.field(highlightTitle);  // <4>
+            HighlightBuilder.Field highlightUser = new HighlightBuilder.Field("user");
+            highlightBuilder.field(highlightUser);
+            searchSourceBuilder.highlighter(highlightBuilder);
+            // end::search-request-highlighting
+            searchSourceBuilder.query(QueryBuilders.boolQuery()
+                    .should(matchQuery("title", "Elasticsearch"))
+                    .should(matchQuery("user", "kimchy")));
+            searchRequest.source(searchSourceBuilder);
+            SearchResponse searchResponse = client.search(searchRequest);
+            {
+                // tag::search-request-highlighting-get
+                SearchHits hits = searchResponse.getHits();
+                for (SearchHit hit : hits.getHits()) {
+                    Map<String, HighlightField> highlightFields = hit.getHighlightFields();
+                    HighlightField highlight = highlightFields.get("title"); // <1>
+                    Text[] fragments = highlight.fragments();  // <2>
+                    String fragmentString = fragments[0].string();
+                }
+                // end::search-request-highlighting-get
+                hits = searchResponse.getHits();
+                for (SearchHit hit : hits.getHits()) {
+                    Map<String, HighlightField> highlightFields = hit.getHighlightFields();
+                    HighlightField highlight = highlightFields.get("title");
+                    Text[] fragments = highlight.fragments();
+                    assertEquals(1, fragments.length);
+                    assertThat(fragments[0].string(), containsString("<em>Elasticsearch</em>"));
+                    highlight = highlightFields.get("user");
+                    fragments = highlight.fragments();
+                    assertEquals(1, fragments.length);
+                    assertThat(fragments[0].string(), containsString("<em>kimchy</em>"));
+                }
+            }
+
+        }
+    }
+
+    public void testSearchRequestProfiling() throws IOException {
+        RestHighLevelClient client = highLevelClient();
+        {
+            IndexRequest request = new IndexRequest("posts", "doc", "1")
+                    .source(XContentType.JSON, "tags", "elasticsearch", "comments", 123);
+            request.setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL);
+            IndexResponse indexResponse = client.index(request);
+            assertSame(RestStatus.CREATED, indexResponse.status());
+        }
+        {
+            SearchRequest searchRequest = new SearchRequest();
+            // tag::search-request-profiling
+            SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+            searchSourceBuilder.profile(true);
+            // end::search-request-profiling
+            searchSourceBuilder.query(QueryBuilders.termQuery("tags", "elasticsearch"));
+            searchSourceBuilder.aggregation(AggregationBuilders.histogram("by_comments").field("comments").interval(100));
+            searchRequest.source(searchSourceBuilder);
+
+            SearchResponse searchResponse = client.search(searchRequest);
+            // tag::search-request-profiling-get
+            Map<String, ProfileShardResult> profilingResults = searchResponse.getProfileResults(); // <1>
+            for (Map.Entry<String, ProfileShardResult> profilingResult : profilingResults.entrySet()) {  // <2>
+                String key = profilingResult.getKey(); // <3>
+                ProfileShardResult profileShardResult = profilingResult.getValue(); // <4>
+            }
+            // end::search-request-profiling-get
+
+            ProfileShardResult profileShardResult = profilingResults.values().iterator().next();
+            assertNotNull(profileShardResult);
+
+            // tag::search-request-profiling-queries
+            List<QueryProfileShardResult> queryProfileShardResults = profileShardResult.getQueryProfileResults(); // <1>
+            for (QueryProfileShardResult queryProfileResult : queryProfileShardResults) { // <2>
+
+            }
+            // end::search-request-profiling-queries
+            assertThat(queryProfileShardResults.size(), equalTo(1));
+
+            for (QueryProfileShardResult queryProfileResult : queryProfileShardResults) {
+                // tag::search-request-profiling-queries-results
+                for (ProfileResult profileResult : queryProfileResult.getQueryResults()) { // <1>
+                    String queryName = profileResult.getQueryName(); // <2>
+                    long queryTimeInMillis = profileResult.getTime(); // <3>
+                    List<ProfileResult> profiledChildren = profileResult.getProfiledChildren(); // <4>
+                }
+                // end::search-request-profiling-queries-results
+
+                // tag::search-request-profiling-queries-collectors
+                CollectorResult collectorResult = queryProfileResult.getCollectorResult();  // <1>
+                String collectorName = collectorResult.getName();  // <2>
+                Long collectorTimeInMillis = collectorResult.getTime(); // <3>
+                List<CollectorResult> profiledChildren = collectorResult.getProfiledChildren(); // <4>
+                // end::search-request-profiling-queries-collectors
+            }
+
+            // tag::search-request-profiling-aggs
+            AggregationProfileShardResult aggsProfileResults = profileShardResult.getAggregationProfileResults(); // <1>
+            for (ProfileResult profileResult : aggsProfileResults.getProfileResults()) { // <2>
+                String aggName = profileResult.getQueryName(); // <3>
+                long aggTimeInMillis = profileResult.getTime(); // <4>
+                List<ProfileResult> profiledChildren = profileResult.getProfiledChildren(); // <5>
+            }
+            // end::search-request-profiling-aggs
+            assertThat(aggsProfileResults.getProfileResults().size(), equalTo(1));
+        }
+    }
+
+    public void testScroll() throws IOException {
+        RestHighLevelClient client = highLevelClient();
+        {
+            BulkRequest request = new BulkRequest();
+            request.add(new IndexRequest("posts", "doc", "1")
+                    .source(XContentType.JSON, "title", "In which order are my Elasticsearch queries executed?"));
+            request.add(new IndexRequest("posts", "doc", "2")
+                    .source(XContentType.JSON, "title", "Current status and upcoming changes in Elasticsearch"));
+            request.add(new IndexRequest("posts", "doc", "3")
+                    .source(XContentType.JSON, "title", "The Future of Federated Search in Elasticsearch"));
+            request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
+            BulkResponse bulkResponse = client.bulk(request);
+            assertSame(RestStatus.OK, bulkResponse.status());
+            assertFalse(bulkResponse.hasFailures());
+        }
+        {
+            int size = 1;
+            // tag::search-scroll-init
+            SearchRequest searchRequest = new SearchRequest("posts");
+            SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+            searchSourceBuilder.query(matchQuery("title", "Elasticsearch"));
+            searchSourceBuilder.size(size); // <1>
+            searchRequest.source(searchSourceBuilder);
+            searchRequest.scroll(TimeValue.timeValueMinutes(1L)); // <2>
+            SearchResponse searchResponse = client.search(searchRequest);
+            String scrollId = searchResponse.getScrollId(); // <3>
+            SearchHits hits = searchResponse.getHits();  // <4>
+            // end::search-scroll-init
+            assertEquals(3, hits.getTotalHits());
+            assertEquals(1, hits.getHits().length);
+            assertNotNull(scrollId);
+
+            // tag::search-scroll2
+            SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); // <1>
+            scrollRequest.scroll(TimeValue.timeValueSeconds(30));
+            SearchResponse searchScrollResponse = client.searchScroll(scrollRequest);
+            scrollId = searchScrollResponse.getScrollId();  // <2>
+            hits = searchScrollResponse.getHits(); // <3>
+            assertEquals(3, hits.getTotalHits());
+            assertEquals(1, hits.getHits().length);
+            assertNotNull(scrollId);
+            // end::search-scroll2
+
+            ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
+            clearScrollRequest.addScrollId(scrollId);
+            ClearScrollResponse clearScrollResponse = client.clearScroll(clearScrollRequest);
+            assertTrue(clearScrollResponse.isSucceeded());
+        }
+        {
+            SearchRequest searchRequest = new SearchRequest();
+            searchRequest.scroll("60s");
+
+            SearchResponse initialSearchResponse = client.search(searchRequest);
+            String scrollId = initialSearchResponse.getScrollId();
+
+            SearchScrollRequest scrollRequest = new SearchScrollRequest();
+            scrollRequest.scrollId(scrollId);
+
+            // tag::scroll-request-arguments
+            scrollRequest.scroll(TimeValue.timeValueSeconds(60L)); // <1>
+            scrollRequest.scroll("60s"); // <2>
+            // end::scroll-request-arguments
+
+            // tag::search-scroll-execute-sync
+            SearchResponse searchResponse = client.searchScroll(scrollRequest);
+            // end::search-scroll-execute-sync
+
+            assertEquals(0, searchResponse.getFailedShards());
+            assertEquals(3L, searchResponse.getHits().getTotalHits());
+
+            // tag::search-scroll-execute-async
+            client.searchScrollAsync(scrollRequest, new ActionListener<SearchResponse>() {
+                @Override
+                public void onResponse(SearchResponse searchResponse) {
+                    // <1>
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    // <2>
+                }
+            });
+            // end::search-scroll-execute-async
+
+            // tag::clear-scroll-request
+            ClearScrollRequest request = new ClearScrollRequest(); // <1>
+            request.addScrollId(scrollId); // <2>
+            // end::clear-scroll-request
+
+            // tag::clear-scroll-add-scroll-id
+            request.addScrollId(scrollId);
+            // end::clear-scroll-add-scroll-id
+
+            List<String> scrollIds = Collections.singletonList(scrollId);
+
+            // tag::clear-scroll-add-scroll-ids
+            request.setScrollIds(scrollIds);
+            // end::clear-scroll-add-scroll-ids
+
+            // tag::clear-scroll-execute
+            ClearScrollResponse response = client.clearScroll(request);
+            // end::clear-scroll-execute
+
+            // tag::clear-scroll-response
+            boolean success = response.isSucceeded(); // <1>
+            int released = response.getNumFreed(); // <2>
+            // end::clear-scroll-response
+            assertTrue(success);
+            assertThat(released, greaterThan(0));
+
+            // tag::clear-scroll-execute-async
+            client.clearScrollAsync(request, new ActionListener<ClearScrollResponse>() {
+                @Override
+                public void onResponse(ClearScrollResponse clearScrollResponse) {
+                    // <1>
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    // <2>
+                }
+            });
+            // end::clear-scroll-execute-async
+        }
+        {
+            // tag::search-scroll-example
+            final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(1L));
+            SearchRequest searchRequest = new SearchRequest("posts");
+            searchRequest.scroll(scroll);
+            SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+            searchSourceBuilder.query(matchQuery("title", "Elasticsearch"));
+            searchRequest.source(searchSourceBuilder);
+
+            SearchResponse searchResponse = client.search(searchRequest); // <1>
+            String scrollId = searchResponse.getScrollId();
+            SearchHit[] searchHits = searchResponse.getHits().getHits();
+
+            while (searchHits != null && searchHits.length > 0) { // <2>
+                SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); // <3>
+                scrollRequest.scroll(scroll);
+                searchResponse = client.searchScroll(scrollRequest);
+                scrollId = searchResponse.getScrollId();
+                searchHits = searchResponse.getHits().getHits();
+                // <4>
+            }
+
+            ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); // <5>
+            clearScrollRequest.addScrollId(scrollId);
+            ClearScrollResponse clearScrollResponse = client.clearScroll(clearScrollRequest);
+            boolean succeeded = clearScrollResponse.isSucceeded();
+            // end::search-scroll-example
+            assertTrue(succeeded);
+        }
+    }
+}
diff --git a/client/rest/build.gradle b/client/rest/build.gradle
index 1c92013da9743..4f52152b8a88a 100644
--- a/client/rest/build.gradle
+++ b/client/rest/build.gradle
@@ -18,7 +18,6 @@
  */
 
 import org.elasticsearch.gradle.precommit.PrecommitTasks
-import org.gradle.api.JavaVersion
 
 apply plugin: 'elasticsearch.build'
 apply plugin: 'ru.vyarus.animalsniffer'
@@ -29,6 +28,15 @@ targetCompatibility = JavaVersion.VERSION_1_7
 sourceCompatibility = JavaVersion.VERSION_1_7
 
 group = 'org.elasticsearch.client'
+archivesBaseName = 'elasticsearch-rest-client'
+
+publishing {
+  publications {
+    nebula {
+      artifactId = archivesBaseName
+    }
+  }
+}
 
 dependencies {
   compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
@@ -60,11 +68,6 @@ forbiddenApisTest {
   signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
 }
 
-dependencyLicenses {
-  mapping from: /http.*/, to: 'httpclient'
-  mapping from: /commons-.*/, to: 'commons'
-}
-
 //JarHell is part of es core, which we don't want to pull in
 jarHell.enabled=false
 
diff --git a/client/rest/licenses/commons-LICENSE.txt b/client/rest/licenses/commons-codec-LICENSE.txt
similarity index 100%
rename from client/rest/licenses/commons-LICENSE.txt
rename to client/rest/licenses/commons-codec-LICENSE.txt
diff --git a/client/rest/licenses/commons-NOTICE.txt b/client/rest/licenses/commons-codec-NOTICE.txt
similarity index 100%
rename from client/rest/licenses/commons-NOTICE.txt
rename to client/rest/licenses/commons-codec-NOTICE.txt
diff --git a/distribution/licenses/securesm-LICENSE.txt b/client/rest/licenses/commons-logging-LICENSE.txt
similarity index 100%
rename from distribution/licenses/securesm-LICENSE.txt
rename to client/rest/licenses/commons-logging-LICENSE.txt
diff --git a/client/rest/licenses/commons-logging-NOTICE.txt b/client/rest/licenses/commons-logging-NOTICE.txt
new file mode 100644
index 0000000000000..a37977d45a168
--- /dev/null
+++ b/client/rest/licenses/commons-logging-NOTICE.txt
@@ -0,0 +1,6 @@
+Apache Commons Logging
+Copyright 2003-2013 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
diff --git a/client/rest/licenses/httpasyncclient-LICENSE.txt b/client/rest/licenses/httpasyncclient-LICENSE.txt
new file mode 100644
index 0000000000000..2c41ec88f61cf
--- /dev/null
+++ b/client/rest/licenses/httpasyncclient-LICENSE.txt
@@ -0,0 +1,182 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+   
+This project contains annotations derived from JCIP-ANNOTATIONS
+Copyright (c) 2005 Brian Goetz and Tim Peierls.
+See http://www.jcip.net and the Creative Commons Attribution License 
+(http://creativecommons.org/licenses/by/2.5)
+
diff --git a/client/rest/licenses/httpasyncclient-NOTICE.txt b/client/rest/licenses/httpasyncclient-NOTICE.txt
new file mode 100644
index 0000000000000..b45be98d168a4
--- /dev/null
+++ b/client/rest/licenses/httpasyncclient-NOTICE.txt
@@ -0,0 +1,5 @@
+Apache HttpComponents AsyncClient
+Copyright 2010-2016 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
diff --git a/client/rest/licenses/httpcore-LICENSE.txt b/client/rest/licenses/httpcore-LICENSE.txt
new file mode 100644
index 0000000000000..e454a52586f29
--- /dev/null
+++ b/client/rest/licenses/httpcore-LICENSE.txt
@@ -0,0 +1,178 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
diff --git a/client/rest/licenses/httpcore-NOTICE.txt b/client/rest/licenses/httpcore-NOTICE.txt
new file mode 100644
index 0000000000000..013448d3e9561
--- /dev/null
+++ b/client/rest/licenses/httpcore-NOTICE.txt
@@ -0,0 +1,5 @@
+Apache HttpComponents Core
+Copyright 2005-2016 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
diff --git a/distribution/licenses/spatial4j-LICENSE.txt b/client/rest/licenses/httpcore-nio-LICENSE.txt
similarity index 100%
rename from distribution/licenses/spatial4j-LICENSE.txt
rename to client/rest/licenses/httpcore-nio-LICENSE.txt
diff --git a/client/rest/licenses/httpcore-nio-NOTICE.txt b/client/rest/licenses/httpcore-nio-NOTICE.txt
new file mode 100644
index 0000000000000..a2e17bb60009f
--- /dev/null
+++ b/client/rest/licenses/httpcore-nio-NOTICE.txt
@@ -0,0 +1,8 @@
+
+Apache HttpCore NIO
+Copyright 2005-2016 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+
diff --git a/client/rest/src/main/java/org/elasticsearch/client/HeapBufferedAsyncResponseConsumer.java b/client/rest/src/main/java/org/elasticsearch/client/HeapBufferedAsyncResponseConsumer.java
index da7f5c79721bc..84753e6f75c8d 100644
--- a/client/rest/src/main/java/org/elasticsearch/client/HeapBufferedAsyncResponseConsumer.java
+++ b/client/rest/src/main/java/org/elasticsearch/client/HeapBufferedAsyncResponseConsumer.java
@@ -38,25 +38,15 @@
 /**
  * Default implementation of {@link org.apache.http.nio.protocol.HttpAsyncResponseConsumer}. Buffers the whole
  * response content in heap memory, meaning that the size of the buffer is equal to the content-length of the response.
- * Limits the size of responses that can be read to {@link #DEFAULT_BUFFER_LIMIT} by default, configurable value.
- * Throws an exception in case the entity is longer than the configured buffer limit.
+ * Limits the size of responses that can be read based on a configurable argument. Throws an exception in case the entity is longer
+ * than the configured buffer limit.
  */
 public class HeapBufferedAsyncResponseConsumer extends AbstractAsyncResponseConsumer<HttpResponse> {
 
-    //default buffer limit is 10MB
-    public static final int DEFAULT_BUFFER_LIMIT = 10 * 1024 * 1024;
-
-    private final int bufferLimit;
+    private final int bufferLimitBytes;
     private volatile HttpResponse response;
     private volatile SimpleInputBuffer buf;
 
-    /**
-     * Creates a new instance of this consumer with a buffer limit of {@link #DEFAULT_BUFFER_LIMIT}
-     */
-    public HeapBufferedAsyncResponseConsumer() {
-        this.bufferLimit = DEFAULT_BUFFER_LIMIT;
-    }
-
     /**
      * Creates a new instance of this consumer with the provided buffer limit
      */
@@ -64,7 +54,14 @@ public HeapBufferedAsyncResponseConsumer(int bufferLimit) {
         if (bufferLimit <= 0) {
             throw new IllegalArgumentException("bufferLimit must be greater than 0");
         }
-        this.bufferLimit = bufferLimit;
+        this.bufferLimitBytes = bufferLimit;
+    }
+
+    /**
+     * Get the limit of the buffer.
+     */
+    public int getBufferLimit() {
+        return bufferLimitBytes;
     }
 
     @Override
@@ -75,9 +72,9 @@ protected void onResponseReceived(HttpResponse response) throws HttpException, I
     @Override
     protected void onEntityEnclosed(HttpEntity entity, ContentType contentType) throws IOException {
         long len = entity.getContentLength();
-        if (len > bufferLimit) {
+        if (len > bufferLimitBytes) {
             throw new ContentTooLongException("entity content is too long [" + len +
-                    "] for the configured buffer limit [" + bufferLimit + "]");
+                    "] for the configured buffer limit [" + bufferLimitBytes + "]");
         }
         if (len < 0) {
             len = 4096;
diff --git a/client/rest/src/main/java/org/elasticsearch/client/HttpAsyncResponseConsumerFactory.java b/client/rest/src/main/java/org/elasticsearch/client/HttpAsyncResponseConsumerFactory.java
new file mode 100644
index 0000000000000..1af9e0dcf0fa4
--- /dev/null
+++ b/client/rest/src/main/java/org/elasticsearch/client/HttpAsyncResponseConsumerFactory.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import org.apache.http.HttpResponse;
+import org.apache.http.nio.protocol.HttpAsyncResponseConsumer;
+
+import static org.elasticsearch.client.HttpAsyncResponseConsumerFactory.HeapBufferedResponseConsumerFactory.DEFAULT_BUFFER_LIMIT;
+
+/**
+ * Factory used to create instances of {@link HttpAsyncResponseConsumer}. Each request retry needs its own instance of the
+ * consumer object. Users can implement this interface and pass their own instance to the specialized
+ * performRequest methods that accept an {@link HttpAsyncResponseConsumerFactory} instance as argument.
+ */
+public interface HttpAsyncResponseConsumerFactory {
+
+    /**
+     * Creates the default type of {@link HttpAsyncResponseConsumer}, based on heap buffering with a buffer limit of 100MB.
+     */
+    HttpAsyncResponseConsumerFactory DEFAULT = new HeapBufferedResponseConsumerFactory(DEFAULT_BUFFER_LIMIT);
+
+    /**
+     * Creates the {@link HttpAsyncResponseConsumer}, called once per request attempt.
+     */
+    HttpAsyncResponseConsumer<HttpResponse> createHttpAsyncResponseConsumer();
+
+    /**
+     * Default factory used to create instances of {@link HttpAsyncResponseConsumer}.
+     * Creates one instance of {@link HeapBufferedAsyncResponseConsumer} for each request attempt, with a configurable
+     * buffer limit which defaults to 100MB.
+     */
+    class HeapBufferedResponseConsumerFactory implements HttpAsyncResponseConsumerFactory {
+
+        //default buffer limit is 100MB
+        static final int DEFAULT_BUFFER_LIMIT = 100 * 1024 * 1024;
+
+        private final int bufferLimit;
+
+        public HeapBufferedResponseConsumerFactory(int bufferLimitBytes) {
+            this.bufferLimit = bufferLimitBytes;
+        }
+
+        @Override
+        public HttpAsyncResponseConsumer<HttpResponse> createHttpAsyncResponseConsumer() {
+            return new HeapBufferedAsyncResponseConsumer(bufferLimit);
+        }
+    }
+}
diff --git a/client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java b/client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java
index fd4c360023468..ad2348762dd07 100644
--- a/client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java
+++ b/client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java
@@ -59,6 +59,12 @@ static void logResponse(Log logger, HttpUriRequest request, HttpHost host, HttpR
             logger.debug("request [" + request.getMethod() + " " + host + getUri(request.getRequestLine()) +
                     "] returned [" + httpResponse.getStatusLine() + "]");
         }
+        if (logger.isWarnEnabled()) {
+            Header[] warnings = httpResponse.getHeaders("Warning");
+            if (warnings != null && warnings.length > 0) {
+                logger.warn(buildWarningMessage(request, host, warnings));
+            }
+        }
         if (tracer.isTraceEnabled()) {
             String requestLine;
             try {
@@ -97,6 +103,18 @@ static void logFailedRequest(Log logger, HttpUriRequest request, HttpHost host,
         }
     }
 
+    static String buildWarningMessage(HttpUriRequest request, HttpHost host, Header[] warnings) {
+        StringBuilder message = new StringBuilder("request [").append(request.getMethod()).append(" ").append(host)
+                .append(getUri(request.getRequestLine())).append("] returned ").append(warnings.length).append(" warnings: ");
+        for (int i = 0; i < warnings.length; i++) {
+            if (i > 0) {
+                message.append(",");
+            }
+            message.append("[").append(warnings[i].getValue()).append("]");
+        }
+        return message.toString();
+    }
+
     /**
      * Creates curl output for given request
      */
@@ -134,7 +152,7 @@ static String buildTraceResponse(HttpResponse httpResponse) throws IOException {
             httpResponse.setEntity(entity);
             ContentType contentType = ContentType.get(entity);
             Charset charset = StandardCharsets.UTF_8;
-            if (contentType != null) {
+            if (contentType != null && contentType.getCharset() != null) {
                 charset = contentType.getCharset();
             }
             try (BufferedReader reader = new BufferedReader(new InputStreamReader(entity.getContent(), charset))) {
diff --git a/client/rest/src/main/java/org/elasticsearch/client/ResponseListener.java b/client/rest/src/main/java/org/elasticsearch/client/ResponseListener.java
index ce948f6569b65..3d5873599009d 100644
--- a/client/rest/src/main/java/org/elasticsearch/client/ResponseListener.java
+++ b/client/rest/src/main/java/org/elasticsearch/client/ResponseListener.java
@@ -23,6 +23,10 @@
  * Listener to be provided when calling async performRequest methods provided by {@link RestClient}.
  * Those methods that do accept a listener will return immediately, execute asynchronously, and notify
  * the listener whenever the request yielded a response, or failed with an exception.
+ *
+ * <p>
+ * Note that it is <strong>not</strong> safe to call {@link RestClient#close()} from either of these
+ * callbacks.
  */
 public interface ResponseListener {
 
diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java
index d2301e1e8e755..cc0f1b3089638 100644
--- a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java
+++ b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java
@@ -25,6 +25,7 @@
 import org.apache.http.HttpHost;
 import org.apache.http.HttpRequest;
 import org.apache.http.HttpResponse;
+import org.apache.http.client.AuthCache;
 import org.apache.http.client.ClientProtocolException;
 import org.apache.http.client.methods.HttpEntityEnclosingRequestBase;
 import org.apache.http.client.methods.HttpHead;
@@ -34,8 +35,11 @@
 import org.apache.http.client.methods.HttpPut;
 import org.apache.http.client.methods.HttpRequestBase;
 import org.apache.http.client.methods.HttpTrace;
+import org.apache.http.client.protocol.HttpClientContext;
 import org.apache.http.client.utils.URIBuilder;
 import org.apache.http.concurrent.FutureCallback;
+import org.apache.http.impl.auth.BasicScheme;
+import org.apache.http.impl.client.BasicAuthCache;
 import org.apache.http.impl.nio.client.CloseableHttpAsyncClient;
 import org.apache.http.nio.client.methods.HttpAsyncMethods;
 import org.apache.http.nio.protocol.HttpAsyncRequestProducer;
@@ -49,6 +53,7 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
@@ -91,7 +96,7 @@ public class RestClient implements Closeable {
     private final long maxRetryTimeoutMillis;
     private final String pathPrefix;
     private final AtomicInteger lastHostIndex = new AtomicInteger(0);
-    private volatile Set<HttpHost> hosts;
+    private volatile HostTuple<Set<HttpHost>> hostTuple;
     private final ConcurrentMap<HttpHost, DeadHostState> blacklist = new ConcurrentHashMap<>();
     private final FailureListener failureListener;
 
@@ -121,11 +126,13 @@ public synchronized void setHosts(HttpHost... hosts) {
             throw new IllegalArgumentException("hosts must not be null nor empty");
         }
         Set<HttpHost> httpHosts = new HashSet<>();
+        AuthCache authCache = new BasicAuthCache();
         for (HttpHost host : hosts) {
             Objects.requireNonNull(host, "host cannot be null");
             httpHosts.add(host);
+            authCache.put(host, new BasicScheme());
         }
-        this.hosts = Collections.unmodifiableSet(httpHosts);
+        this.hostTuple = new HostTuple<>(Collections.unmodifiableSet(httpHosts), authCache);
         this.blacklist.clear();
     }
 
@@ -143,7 +150,7 @@ public synchronized void setHosts(HttpHost... hosts) {
      * @throws ResponseException in case Elasticsearch responded with a status code that indicated an error
      */
     public Response performRequest(String method, String endpoint, Header... headers) throws IOException {
-        return performRequest(method, endpoint, Collections.<String, String>emptyMap(), (HttpEntity)null, headers);
+        return performRequest(method, endpoint, Collections.<String, String>emptyMap(), null, headers);
     }
 
     /**
@@ -165,9 +172,9 @@ public Response performRequest(String method, String endpoint, Map<String, Strin
 
     /**
      * Sends a request to the Elasticsearch cluster that the client points to and waits for the corresponding response
-     * to be returned. Shortcut to {@link #performRequest(String, String, Map, HttpEntity, HttpAsyncResponseConsumer, Header...)}
-     * which doesn't require specifying an {@link HttpAsyncResponseConsumer} instance, {@link HeapBufferedAsyncResponseConsumer}
-     * will be used to consume the response body.
+     * to be returned. Shortcut to {@link #performRequest(String, String, Map, HttpEntity, HttpAsyncResponseConsumerFactory, Header...)}
+     * which doesn't require specifying an {@link HttpAsyncResponseConsumerFactory} instance,
+     * {@link HttpAsyncResponseConsumerFactory} will be used to create the needed instances of {@link HttpAsyncResponseConsumer}.
      *
      * @param method the http method
      * @param endpoint the path of the request (without host and port)
@@ -181,8 +188,7 @@ public Response performRequest(String method, String endpoint, Map<String, Strin
      */
     public Response performRequest(String method, String endpoint, Map<String, String> params,
                                    HttpEntity entity, Header... headers) throws IOException {
-        HttpAsyncResponseConsumer<HttpResponse> responseConsumer = new HeapBufferedAsyncResponseConsumer();
-        return performRequest(method, endpoint, params, entity, responseConsumer, headers);
+        return performRequest(method, endpoint, params, entity, HttpAsyncResponseConsumerFactory.DEFAULT, headers);
     }
 
     /**
@@ -196,8 +202,9 @@ public Response performRequest(String method, String endpoint, Map<String, Strin
      * @param endpoint the path of the request (without host and port)
      * @param params the query_string parameters
      * @param entity the body of the request, null if not applicable
-     * @param responseConsumer the {@link HttpAsyncResponseConsumer} callback. Controls how the response
-     * body gets streamed from a non-blocking HTTP connection on the client side.
+     * @param httpAsyncResponseConsumerFactory the {@link HttpAsyncResponseConsumerFactory} used to create one
+     * {@link HttpAsyncResponseConsumer} callback per retry. Controls how the response body gets streamed from a non-blocking HTTP
+     * connection on the client side.
      * @param headers the optional request headers
      * @return the response returned by Elasticsearch
      * @throws IOException in case of a problem or the connection was aborted
@@ -205,10 +212,10 @@ public Response performRequest(String method, String endpoint, Map<String, Strin
      * @throws ResponseException in case Elasticsearch responded with a status code that indicated an error
      */
     public Response performRequest(String method, String endpoint, Map<String, String> params,
-                                   HttpEntity entity, HttpAsyncResponseConsumer<HttpResponse> responseConsumer,
+                                   HttpEntity entity, HttpAsyncResponseConsumerFactory httpAsyncResponseConsumerFactory,
                                    Header... headers) throws IOException {
         SyncResponseListener listener = new SyncResponseListener(maxRetryTimeoutMillis);
-        performRequestAsync(method, endpoint, params, entity, responseConsumer, listener, headers);
+        performRequestAsync(method, endpoint, params, entity, httpAsyncResponseConsumerFactory, listener, headers);
         return listener.get();
     }
 
@@ -245,9 +252,9 @@ public void performRequestAsync(String method, String endpoint, Map<String, Stri
     /**
      * Sends a request to the Elasticsearch cluster that the client points to. Doesn't wait for the response, instead
      * the provided {@link ResponseListener} will be notified upon completion or failure.
-     * Shortcut to {@link #performRequestAsync(String, String, Map, HttpEntity, HttpAsyncResponseConsumer, ResponseListener, Header...)}
-     * which doesn't require specifying an {@link HttpAsyncResponseConsumer} instance, {@link HeapBufferedAsyncResponseConsumer}
-     * will be used to consume the response body.
+     * Shortcut to {@link #performRequestAsync(String, String, Map, HttpEntity, HttpAsyncResponseConsumerFactory, ResponseListener,
+     * Header...)} which doesn't require specifying an {@link HttpAsyncResponseConsumerFactory} instance,
+     * {@link HttpAsyncResponseConsumerFactory} will be used to create the needed instances of {@link HttpAsyncResponseConsumer}.
      *
      * @param method the http method
      * @param endpoint the path of the request (without host and port)
@@ -258,8 +265,7 @@ public void performRequestAsync(String method, String endpoint, Map<String, Stri
      */
     public void performRequestAsync(String method, String endpoint, Map<String, String> params,
                                     HttpEntity entity, ResponseListener responseListener, Header... headers) {
-        HttpAsyncResponseConsumer<HttpResponse> responseConsumer = new HeapBufferedAsyncResponseConsumer();
-        performRequestAsync(method, endpoint, params, entity, responseConsumer, responseListener, headers);
+        performRequestAsync(method, endpoint, params, entity, HttpAsyncResponseConsumerFactory.DEFAULT, responseListener, headers);
     }
 
     /**
@@ -274,36 +280,74 @@ public void performRequestAsync(String method, String endpoint, Map<String, Stri
      * @param endpoint the path of the request (without host and port)
      * @param params the query_string parameters
      * @param entity the body of the request, null if not applicable
-     * @param responseConsumer the {@link HttpAsyncResponseConsumer} callback. Controls how the response
-     * body gets streamed from a non-blocking HTTP connection on the client side.
+     * @param httpAsyncResponseConsumerFactory the {@link HttpAsyncResponseConsumerFactory} used to create one
+     * {@link HttpAsyncResponseConsumer} callback per retry. Controls how the response body gets streamed from a non-blocking HTTP
+     * connection on the client side.
      * @param responseListener the {@link ResponseListener} to notify when the request is completed or fails
      * @param headers the optional request headers
      */
     public void performRequestAsync(String method, String endpoint, Map<String, String> params,
-                                    HttpEntity entity, HttpAsyncResponseConsumer<HttpResponse> responseConsumer,
+                                    HttpEntity entity, HttpAsyncResponseConsumerFactory httpAsyncResponseConsumerFactory,
                                     ResponseListener responseListener, Header... headers) {
-        URI uri = buildUri(pathPrefix, endpoint, params);
-        HttpRequestBase request = createHttpRequest(method, uri, entity);
-        setHeaders(request, headers);
-        FailureTrackingResponseListener failureTrackingResponseListener = new FailureTrackingResponseListener(responseListener);
-        long startTime = System.nanoTime();
-        performRequestAsync(startTime, nextHost().iterator(), request, responseConsumer, failureTrackingResponseListener);
+        try {
+            Objects.requireNonNull(params, "params must not be null");
+            Map<String, String> requestParams = new HashMap<>(params);
+            //ignore is a special parameter supported by the clients, shouldn't be sent to es
+            String ignoreString = requestParams.remove("ignore");
+            Set<Integer> ignoreErrorCodes;
+            if (ignoreString == null) {
+                if (HttpHead.METHOD_NAME.equals(method)) {
+                    //404 never causes error if returned for a HEAD request
+                    ignoreErrorCodes = Collections.singleton(404);
+                } else {
+                    ignoreErrorCodes = Collections.emptySet();
+                }
+            } else {
+                String[] ignoresArray = ignoreString.split(",");
+                ignoreErrorCodes = new HashSet<>();
+                if (HttpHead.METHOD_NAME.equals(method)) {
+                    //404 never causes error if returned for a HEAD request
+                    ignoreErrorCodes.add(404);
+                }
+                for (String ignoreCode : ignoresArray) {
+                    try {
+                        ignoreErrorCodes.add(Integer.valueOf(ignoreCode));
+                    } catch (NumberFormatException e) {
+                        throw new IllegalArgumentException("ignore value should be a number, found [" + ignoreString + "] instead", e);
+                    }
+                }
+            }
+            URI uri = buildUri(pathPrefix, endpoint, requestParams);
+            HttpRequestBase request = createHttpRequest(method, uri, entity);
+            setHeaders(request, headers);
+            FailureTrackingResponseListener failureTrackingResponseListener = new FailureTrackingResponseListener(responseListener);
+            long startTime = System.nanoTime();
+            performRequestAsync(startTime, nextHost(), request, ignoreErrorCodes, httpAsyncResponseConsumerFactory,
+                    failureTrackingResponseListener);
+        } catch (Exception e) {
+            responseListener.onFailure(e);
+        }
     }
 
-    private void performRequestAsync(final long startTime, final Iterator<HttpHost> hosts, final HttpRequestBase request,
-                                     final HttpAsyncResponseConsumer<HttpResponse> responseConsumer,
+    private void performRequestAsync(final long startTime, final HostTuple<Iterator<HttpHost>> hostTuple, final HttpRequestBase request,
+                                     final Set<Integer> ignoreErrorCodes,
+                                     final HttpAsyncResponseConsumerFactory httpAsyncResponseConsumerFactory,
                                      final FailureTrackingResponseListener listener) {
-        final HttpHost host = hosts.next();
+        final HttpHost host = hostTuple.hosts.next();
         //we stream the request body if the entity allows for it
-        HttpAsyncRequestProducer requestProducer = HttpAsyncMethods.create(host, request);
-        client.execute(requestProducer, responseConsumer, new FutureCallback<HttpResponse>() {
+        final HttpAsyncRequestProducer requestProducer = HttpAsyncMethods.create(host, request);
+        final HttpAsyncResponseConsumer<HttpResponse> asyncResponseConsumer =
+            httpAsyncResponseConsumerFactory.createHttpAsyncResponseConsumer();
+        final HttpClientContext context = HttpClientContext.create();
+        context.setAuthCache(hostTuple.authCache);
+        client.execute(requestProducer, asyncResponseConsumer, context, new FutureCallback<HttpResponse>() {
             @Override
             public void completed(HttpResponse httpResponse) {
                 try {
                     RequestLogger.logResponse(logger, request, host, httpResponse);
                     int statusCode = httpResponse.getStatusLine().getStatusCode();
                     Response response = new Response(request.getRequestLine(), host, httpResponse);
-                    if (isSuccessfulResponse(request.getMethod(), statusCode)) {
+                    if (isSuccessfulResponse(statusCode) || ignoreErrorCodes.contains(response.getStatusLine().getStatusCode())) {
                         onResponse(host);
                         listener.onSuccess(response);
                     } else {
@@ -311,7 +355,7 @@ public void completed(HttpResponse httpResponse) {
                         if (isRetryStatus(statusCode)) {
                             //mark host dead and retry against next one
                             onFailure(host);
-                            retryIfPossible(responseException, hosts, request);
+                            retryIfPossible(responseException);
                         } else {
                             //mark host alive and don't retry, as the error should be a request problem
                             onResponse(host);
@@ -328,14 +372,14 @@ public void failed(Exception failure) {
                 try {
                     RequestLogger.logFailedRequest(logger, request, host, failure);
                     onFailure(host);
-                    retryIfPossible(failure, hosts, request);
+                    retryIfPossible(failure);
                 } catch(Exception e) {
                     listener.onDefinitiveFailure(e);
                 }
             }
 
-            private void retryIfPossible(Exception exception, Iterator<HttpHost> hosts, HttpRequestBase request) {
-                if (hosts.hasNext()) {
+            private void retryIfPossible(Exception exception) {
+                if (hostTuple.hosts.hasNext()) {
                     //in case we are retrying, check whether maxRetryTimeout has been reached
                     long timeElapsedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime);
                     long timeout = maxRetryTimeoutMillis - timeElapsedMillis;
@@ -346,7 +390,7 @@ private void retryIfPossible(Exception exception, Iterator<HttpHost> hosts, Http
                     } else {
                         listener.trackFailure(exception);
                         request.reset();
-                        performRequestAsync(startTime, hosts, request, responseConsumer, listener);
+                        performRequestAsync(startTime, hostTuple, request, ignoreErrorCodes, httpAsyncResponseConsumerFactory, listener);
                     }
                 } else {
                     listener.onDefinitiveFailure(exception);
@@ -384,17 +428,18 @@ private void setHeaders(HttpRequest httpRequest, Header[] requestHeaders) {
      * The iterator returned will never be empty. In case there are no healthy hosts available, or dead ones to be be retried,
      * one dead host gets returned so that it can be retried.
      */
-    private Iterable<HttpHost> nextHost() {
+    private HostTuple<Iterator<HttpHost>> nextHost() {
+        final HostTuple<Set<HttpHost>> hostTuple = this.hostTuple;
         Collection<HttpHost> nextHosts = Collections.emptySet();
         do {
-            Set<HttpHost> filteredHosts = new HashSet<>(hosts);
+            Set<HttpHost> filteredHosts = new HashSet<>(hostTuple.hosts);
             for (Map.Entry<HttpHost, DeadHostState> entry : blacklist.entrySet()) {
                 if (System.nanoTime() - entry.getValue().getDeadUntilNanos() < 0) {
                     filteredHosts.remove(entry.getKey());
                 }
             }
             if (filteredHosts.isEmpty()) {
-                //last resort: if there are no good hosts to use, return a single dead one, the one that's closest to being retried
+                //last resort: if there are no good host to use, return a single dead one, the one that's closest to being retried
                 List<Map.Entry<HttpHost, DeadHostState>> sortedHosts = new ArrayList<>(blacklist.entrySet());
                 if (sortedHosts.size() > 0) {
                     Collections.sort(sortedHosts, new Comparator<Map.Entry<HttpHost, DeadHostState>>() {
@@ -413,7 +458,7 @@ public int compare(Map.Entry<HttpHost, DeadHostState> o1, Map.Entry<HttpHost, De
                 nextHosts = rotatedHosts;
             }
         } while(nextHosts.isEmpty());
-        return nextHosts;
+        return new HostTuple<>(nextHosts.iterator(), hostTuple.authCache);
     }
 
     /**
@@ -451,8 +496,8 @@ public void close() throws IOException {
         client.close();
     }
 
-    private static boolean isSuccessfulResponse(String method, int statusCode) {
-        return statusCode < 300 || (HttpHead.METHOD_NAME.equals(method) && statusCode == 404);
+    private static boolean isSuccessfulResponse(int statusCode) {
+        return statusCode < 300;
     }
 
     private static boolean isRetryStatus(int statusCode) {
@@ -508,8 +553,8 @@ private static HttpRequestBase addRequestBody(HttpRequestBase httpRequest, HttpE
         return httpRequest;
     }
 
-    private static URI buildUri(String pathPrefix, String path, Map<String, String> params) {
-        Objects.requireNonNull(params, "params must not be null");
+    static URI buildUri(String pathPrefix, String path, Map<String, String> params) {
+        Objects.requireNonNull(path, "path must not be null");
         try {
             String fullPath;
             if (pathPrefix != null) {
@@ -655,4 +700,18 @@ public void onFailure(HttpHost host) {
 
         }
     }
+
+    /**
+     * {@code HostTuple} enables the {@linkplain HttpHost}s and {@linkplain AuthCache} to be set together in a thread
+     * safe, volatile way.
+     */
+    private static class HostTuple<T> {
+        public final T hosts;
+        public final AuthCache authCache;
+
+        HostTuple(final T hosts, final AuthCache authCache) {
+            this.hosts = hosts;
+            this.authCache = authCache;
+        }
+    }
 }
diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClientBuilder.java b/client/rest/src/main/java/org/elasticsearch/client/RestClientBuilder.java
index d342d59ade44d..e968039324daf 100644
--- a/client/rest/src/main/java/org/elasticsearch/client/RestClientBuilder.java
+++ b/client/rest/src/main/java/org/elasticsearch/client/RestClientBuilder.java
@@ -28,6 +28,8 @@
 import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
 import org.apache.http.nio.conn.SchemeIOSessionStrategy;
 
+import java.security.AccessController;
+import java.security.PrivilegedAction;
 import java.util.Objects;
 
 /**
@@ -37,7 +39,7 @@
  */
 public final class RestClientBuilder {
     public static final int DEFAULT_CONNECT_TIMEOUT_MILLIS = 1000;
-    public static final int DEFAULT_SOCKET_TIMEOUT_MILLIS = 10000;
+    public static final int DEFAULT_SOCKET_TIMEOUT_MILLIS = 30000;
     public static final int DEFAULT_MAX_RETRY_TIMEOUT_MILLIS = DEFAULT_SOCKET_TIMEOUT_MILLIS;
     public static final int DEFAULT_CONNECTION_REQUEST_TIMEOUT_MILLIS = 500;
     public static final int DEFAULT_MAX_CONN_PER_ROUTE = 10;
@@ -185,7 +187,8 @@ public RestClient build() {
 
     private CloseableHttpAsyncClient createHttpClient() {
         //default timeouts are all infinite
-        RequestConfig.Builder requestConfigBuilder = RequestConfig.custom().setConnectTimeout(DEFAULT_CONNECT_TIMEOUT_MILLIS)
+        RequestConfig.Builder requestConfigBuilder = RequestConfig.custom()
+                .setConnectTimeout(DEFAULT_CONNECT_TIMEOUT_MILLIS)
                 .setSocketTimeout(DEFAULT_SOCKET_TIMEOUT_MILLIS)
                 .setConnectionRequestTimeout(DEFAULT_CONNECTION_REQUEST_TIMEOUT_MILLIS);
         if (requestConfigCallback != null) {
@@ -194,11 +197,18 @@ private CloseableHttpAsyncClient createHttpClient() {
 
         HttpAsyncClientBuilder httpClientBuilder = HttpAsyncClientBuilder.create().setDefaultRequestConfig(requestConfigBuilder.build())
                 //default settings for connection pooling may be too constraining
-                .setMaxConnPerRoute(DEFAULT_MAX_CONN_PER_ROUTE).setMaxConnTotal(DEFAULT_MAX_CONN_TOTAL);
+                .setMaxConnPerRoute(DEFAULT_MAX_CONN_PER_ROUTE).setMaxConnTotal(DEFAULT_MAX_CONN_TOTAL).useSystemProperties();
         if (httpClientConfigCallback != null) {
             httpClientBuilder = httpClientConfigCallback.customizeHttpClient(httpClientBuilder);
         }
-        return httpClientBuilder.build();
+
+        final HttpAsyncClientBuilder finalBuilder = httpClientBuilder;
+        return AccessController.doPrivileged(new PrivilegedAction<CloseableHttpAsyncClient>() {
+            @Override
+            public CloseableHttpAsyncClient run() {
+                return finalBuilder.build();
+            }
+        });
     }
 
     /**
diff --git a/client/rest/src/test/java/org/elasticsearch/client/HeapBufferedAsyncResponseConsumerTests.java b/client/rest/src/test/java/org/elasticsearch/client/HeapBufferedAsyncResponseConsumerTests.java
index d30a9e00b53da..c1eb1044a2cab 100644
--- a/client/rest/src/test/java/org/elasticsearch/client/HeapBufferedAsyncResponseConsumerTests.java
+++ b/client/rest/src/test/java/org/elasticsearch/client/HeapBufferedAsyncResponseConsumerTests.java
@@ -30,11 +30,17 @@
 import org.apache.http.message.BasicStatusLine;
 import org.apache.http.nio.ContentDecoder;
 import org.apache.http.nio.IOControl;
+import org.apache.http.nio.protocol.HttpAsyncResponseConsumer;
 import org.apache.http.protocol.HttpContext;
 
-import static org.elasticsearch.client.HeapBufferedAsyncResponseConsumer.DEFAULT_BUFFER_LIMIT;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Modifier;
+
+import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.spy;
@@ -45,13 +51,14 @@ public class HeapBufferedAsyncResponseConsumerTests extends RestClientTestCase {
 
     //maximum buffer that this test ends up allocating is 50MB
     private static final int MAX_TEST_BUFFER_SIZE = 50 * 1024 * 1024;
+    private static final int TEST_BUFFER_LIMIT = 10 * 1024 * 1024;
 
     public void testResponseProcessing() throws Exception {
         ContentDecoder contentDecoder = mock(ContentDecoder.class);
         IOControl ioControl = mock(IOControl.class);
         HttpContext httpContext = mock(HttpContext.class);
 
-        HeapBufferedAsyncResponseConsumer consumer = spy(new HeapBufferedAsyncResponseConsumer());
+        HeapBufferedAsyncResponseConsumer consumer = spy(new HeapBufferedAsyncResponseConsumer(TEST_BUFFER_LIMIT));
 
         ProtocolVersion protocolVersion = new ProtocolVersion("HTTP", 1, 1);
         StatusLine statusLine = new BasicStatusLine(protocolVersion, 200, "OK");
@@ -74,8 +81,8 @@ public void testResponseProcessing() throws Exception {
     }
 
     public void testDefaultBufferLimit() throws Exception {
-        HeapBufferedAsyncResponseConsumer consumer = new HeapBufferedAsyncResponseConsumer();
-        bufferLimitTest(consumer, DEFAULT_BUFFER_LIMIT);
+        HeapBufferedAsyncResponseConsumer consumer = new HeapBufferedAsyncResponseConsumer(TEST_BUFFER_LIMIT);
+        bufferLimitTest(consumer, TEST_BUFFER_LIMIT);
     }
 
     public void testConfiguredBufferLimit() throws Exception {
@@ -94,6 +101,26 @@ public void testConfiguredBufferLimit() throws Exception {
         bufferLimitTest(consumer, bufferLimit);
     }
 
+    public void testCanConfigureHeapBufferLimitFromOutsidePackage() throws ClassNotFoundException, NoSuchMethodException,
+            IllegalAccessException, InvocationTargetException, InstantiationException {
+        int bufferLimit = randomIntBetween(1, Integer.MAX_VALUE);
+        //we use reflection to make sure that the class can be instantiated from the outside, and the constructor is public
+        Constructor<?> constructor = HttpAsyncResponseConsumerFactory.HeapBufferedResponseConsumerFactory.class.getConstructor(Integer.TYPE);
+        assertEquals(Modifier.PUBLIC, constructor.getModifiers() & Modifier.PUBLIC);
+        Object object = constructor.newInstance(bufferLimit);
+        assertThat(object, instanceOf(HttpAsyncResponseConsumerFactory.HeapBufferedResponseConsumerFactory.class));
+        HttpAsyncResponseConsumerFactory.HeapBufferedResponseConsumerFactory consumerFactory =
+                (HttpAsyncResponseConsumerFactory.HeapBufferedResponseConsumerFactory) object;
+        HttpAsyncResponseConsumer<HttpResponse> consumer = consumerFactory.createHttpAsyncResponseConsumer();
+        assertThat(consumer, instanceOf(HeapBufferedAsyncResponseConsumer.class));
+        HeapBufferedAsyncResponseConsumer bufferedAsyncResponseConsumer = (HeapBufferedAsyncResponseConsumer) consumer;
+        assertEquals(bufferLimit, bufferedAsyncResponseConsumer.getBufferLimit());
+    }
+
+    public void testHttpAsyncResponseConsumerFactoryVisibility() throws ClassNotFoundException {
+        assertEquals(Modifier.PUBLIC, HttpAsyncResponseConsumerFactory.class.getModifiers() & Modifier.PUBLIC);
+    }
+
     private static void bufferLimitTest(HeapBufferedAsyncResponseConsumer consumer, int bufferLimit) throws Exception {
         ProtocolVersion protocolVersion = new ProtocolVersion("HTTP", 1, 1);
         StatusLine statusLine = new BasicStatusLine(protocolVersion, 200, "OK");
diff --git a/client/rest/src/test/java/org/elasticsearch/client/RequestLoggerTests.java b/client/rest/src/test/java/org/elasticsearch/client/RequestLoggerTests.java
index 789f2bf6f6d65..68717dfe223cd 100644
--- a/client/rest/src/test/java/org/elasticsearch/client/RequestLoggerTests.java
+++ b/client/rest/src/test/java/org/elasticsearch/client/RequestLoggerTests.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.client;
 
-import com.carrotsearch.randomizedtesting.generators.RandomInts;
+import org.apache.http.Header;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpEntityEnclosingRequest;
 import org.apache.http.HttpHost;
@@ -29,10 +29,11 @@
 import org.apache.http.client.methods.HttpPatch;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.methods.HttpPut;
-import org.apache.http.client.methods.HttpRequestBase;
 import org.apache.http.client.methods.HttpTrace;
+import org.apache.http.client.methods.HttpUriRequest;
 import org.apache.http.entity.InputStreamEntity;
 import org.apache.http.entity.StringEntity;
+import org.apache.http.message.BasicHeader;
 import org.apache.http.message.BasicHttpResponse;
 import org.apache.http.message.BasicStatusLine;
 import org.apache.http.nio.entity.NByteArrayEntity;
@@ -43,16 +44,16 @@
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 
 import static org.hamcrest.CoreMatchers.equalTo;
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertThat;
 
 public class RequestLoggerTests extends RestClientTestCase {
-
     public void testTraceRequest() throws IOException, URISyntaxException {
-        HttpHost host = new HttpHost("localhost", 9200, getRandom().nextBoolean() ? "http" : "https");
-
+        HttpHost host = new HttpHost("localhost", 9200, randomBoolean() ? "http" : "https");
         String expectedEndpoint = "/index/type/_api";
         URI uri;
         if (randomBoolean()) {
@@ -60,46 +61,15 @@ public void testTraceRequest() throws IOException, URISyntaxException {
         } else {
             uri = new URI("index/type/_api");
         }
-
-        HttpRequestBase request;
-        int requestType = RandomInts.randomIntBetween(getRandom(), 0, 7);
-        switch(requestType) {
-            case 0:
-                request = new HttpGetWithEntity(uri);
-                break;
-            case 1:
-                request = new HttpPost(uri);
-                break;
-            case 2:
-                request = new HttpPut(uri);
-                break;
-            case 3:
-                request = new HttpDeleteWithEntity(uri);
-                break;
-            case 4:
-                request = new HttpHead(uri);
-                break;
-            case 5:
-                request = new HttpTrace(uri);
-                break;
-            case 6:
-                request = new HttpOptions(uri);
-                break;
-            case 7:
-                request = new HttpPatch(uri);
-                break;
-            default:
-                throw new UnsupportedOperationException();
-        }
-
+        HttpUriRequest request = randomHttpRequest(uri);
         String expected = "curl -iX " + request.getMethod() + " '" + host + expectedEndpoint + "'";
-        boolean hasBody = request instanceof HttpEntityEnclosingRequest && getRandom().nextBoolean();
+        boolean hasBody = request instanceof HttpEntityEnclosingRequest && randomBoolean();
         String requestBody = "{ \"field\": \"value\" }";
         if (hasBody) {
             expected += " -d '" + requestBody + "'";
             HttpEntityEnclosingRequest enclosingRequest = (HttpEntityEnclosingRequest) request;
             HttpEntity entity;
-            switch(RandomInts.randomIntBetween(getRandom(), 0, 3)) {
+            switch(randomIntBetween(0, 4)) {
                 case 0:
                     entity = new StringEntity(requestBody, StandardCharsets.UTF_8);
                     break;
@@ -112,6 +82,10 @@ public void testTraceRequest() throws IOException, URISyntaxException {
                 case 3:
                     entity = new NByteArrayEntity(requestBody.getBytes(StandardCharsets.UTF_8));
                     break;
+                case 4:
+                    // Evil entity without a charset
+                    entity = new StringEntity(requestBody, (Charset) null);
+                    break;
                 default:
                     throw new UnsupportedOperationException();
             }
@@ -128,12 +102,12 @@ public void testTraceRequest() throws IOException, URISyntaxException {
 
     public void testTraceResponse() throws IOException {
         ProtocolVersion protocolVersion = new ProtocolVersion("HTTP", 1, 1);
-        int statusCode = RandomInts.randomIntBetween(getRandom(), 200, 599);
+        int statusCode = randomIntBetween(200, 599);
         String reasonPhrase = "REASON";
         BasicStatusLine statusLine = new BasicStatusLine(protocolVersion, statusCode, reasonPhrase);
         String expected = "# " + statusLine.toString();
         BasicHttpResponse httpResponse = new BasicHttpResponse(statusLine);
-        int numHeaders = RandomInts.randomIntBetween(getRandom(), 0, 3);
+        int numHeaders = randomIntBetween(0, 3);
         for (int i = 0; i < numHeaders; i++) {
             httpResponse.setHeader("header" + i, "value");
             expected += "\n# header" + i + ": value";
@@ -146,11 +120,20 @@ public void testTraceResponse() throws IOException {
             expected += "\n#   \"field\": \"value\"";
             expected += "\n# }";
             HttpEntity entity;
-            if (getRandom().nextBoolean()) {
+            switch(randomIntBetween(0, 2)) {
+            case 0:
                 entity = new StringEntity(responseBody, StandardCharsets.UTF_8);
-            } else {
+                break;
+            case 1:
                 //test a non repeatable entity
                 entity = new InputStreamEntity(new ByteArrayInputStream(responseBody.getBytes(StandardCharsets.UTF_8)));
+                break;
+            case 2:
+                // Evil entity without a charset
+                entity = new StringEntity(responseBody, (Charset) null);
+                break;
+            default:
+                throw new UnsupportedOperationException();
             }
             httpResponse.setEntity(entity);
         }
@@ -162,4 +145,46 @@ public void testTraceResponse() throws IOException {
             assertThat(body, equalTo(responseBody));
         }
     }
+
+    public void testResponseWarnings() throws Exception {
+        HttpHost host = new HttpHost("localhost", 9200);
+        HttpUriRequest request = randomHttpRequest(new URI("/index/type/_api"));
+        int numWarnings = randomIntBetween(1, 5);
+        StringBuilder expected = new StringBuilder("request [").append(request.getMethod()).append(" ").append(host)
+                .append("/index/type/_api] returned ").append(numWarnings).append(" warnings: ");
+        Header[] warnings = new Header[numWarnings];
+        for (int i = 0; i < numWarnings; i++) {
+            String warning = "this is warning number " + i;
+            warnings[i] = new BasicHeader("Warning", warning);
+            if (i > 0) {
+                expected.append(",");
+            }
+            expected.append("[").append(warning).append("]");
+        }
+        assertEquals(expected.toString(), RequestLogger.buildWarningMessage(request, host, warnings));
+    }
+
+    private static HttpUriRequest randomHttpRequest(URI uri) {
+        int requestType = randomIntBetween(0, 7);
+        switch(requestType) {
+            case 0:
+                return new HttpGetWithEntity(uri);
+            case 1:
+                return new HttpPost(uri);
+            case 2:
+                return new HttpPut(uri);
+            case 3:
+                return new HttpDeleteWithEntity(uri);
+            case 4:
+                return new HttpHead(uri);
+            case 5:
+                return new HttpTrace(uri);
+            case 6:
+                return new HttpOptions(uri);
+            case 7:
+                return new HttpPatch(uri);
+            default:
+                throw new UnsupportedOperationException();
+        }
+    }
 }
diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderIntegTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderIntegTests.java
new file mode 100644
index 0000000000000..8fb33e6f395a8
--- /dev/null
+++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderIntegTests.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import com.sun.net.httpserver.HttpExchange;
+import com.sun.net.httpserver.HttpHandler;
+import com.sun.net.httpserver.HttpsConfigurator;
+import com.sun.net.httpserver.HttpsServer;
+import org.apache.http.HttpHost;
+import org.codehaus.mojo.animal_sniffer.IgnoreJRERequirement;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import javax.net.ssl.KeyManagerFactory;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManagerFactory;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.security.KeyStore;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.fail;
+
+/**
+ * Integration test to validate the builder builds a client with the correct configuration
+ */
+//animal-sniffer doesn't like our usage of com.sun.net.httpserver.* classes
+@IgnoreJRERequirement
+public class RestClientBuilderIntegTests extends RestClientTestCase {
+
+    private static HttpsServer httpsServer;
+
+    @BeforeClass
+    public static void startHttpServer() throws Exception {
+        httpsServer = HttpsServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0);
+        httpsServer.setHttpsConfigurator(new HttpsConfigurator(getSslContext()));
+        httpsServer.createContext("/", new ResponseHandler());
+        httpsServer.start();
+    }
+
+    //animal-sniffer doesn't like our usage of com.sun.net.httpserver.* classes
+    @IgnoreJRERequirement
+    private static class ResponseHandler implements HttpHandler {
+        @Override
+        public void handle(HttpExchange httpExchange) throws IOException {
+            httpExchange.sendResponseHeaders(200, -1);
+            httpExchange.close();
+        }
+    }
+
+    @AfterClass
+    public static void stopHttpServers() throws IOException {
+        httpsServer.stop(0);
+        httpsServer = null;
+    }
+
+    public void testBuilderUsesDefaultSSLContext() throws Exception {
+        final SSLContext defaultSSLContext = SSLContext.getDefault();
+        try {
+            try (RestClient client = buildRestClient()) {
+                try {
+                    client.performRequest("GET", "/");
+                    fail("connection should have been rejected due to SSL handshake");
+                } catch (Exception e) {
+                    assertThat(e.getMessage(), containsString("General SSLEngine problem"));
+                }
+            }
+
+            SSLContext.setDefault(getSslContext());
+            try (RestClient client = buildRestClient()) {
+                Response response = client.performRequest("GET", "/");
+                assertEquals(200, response.getStatusLine().getStatusCode());
+            }
+        } finally {
+            SSLContext.setDefault(defaultSSLContext);
+        }
+    }
+
+    private RestClient buildRestClient() {
+        InetSocketAddress address = httpsServer.getAddress();
+        return RestClient.builder(new HttpHost(address.getHostString(), address.getPort(), "https")).build();
+    }
+
+    private static SSLContext getSslContext() throws Exception {
+        SSLContext sslContext = SSLContext.getInstance("TLS");
+        try (InputStream in = RestClientBuilderIntegTests.class.getResourceAsStream("/testks.jks")) {
+            KeyStore keyStore = KeyStore.getInstance("JKS");
+            keyStore.load(in, "password".toCharArray());
+            KeyManagerFactory kmf = KeyManagerFactory.getInstance("SunX509");
+            kmf.init(keyStore, "password".toCharArray());
+            TrustManagerFactory tmf = TrustManagerFactory.getInstance("SunX509");
+            tmf.init(keyStore);
+            sslContext.init(kmf.getKeyManagers(), tmf.getTrustManagers(), null);
+        }
+        return sslContext;
+    }
+}
diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientIntegTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientIntegTests.java
deleted file mode 100644
index 9c5c50946d884..0000000000000
--- a/client/rest/src/test/java/org/elasticsearch/client/RestClientIntegTests.java
+++ /dev/null
@@ -1,298 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.client;
-
-import com.sun.net.httpserver.Headers;
-import com.sun.net.httpserver.HttpContext;
-import com.sun.net.httpserver.HttpExchange;
-import com.sun.net.httpserver.HttpHandler;
-import com.sun.net.httpserver.HttpServer;
-import org.apache.http.Consts;
-import org.apache.http.Header;
-import org.apache.http.HttpHost;
-import org.apache.http.entity.StringEntity;
-import org.apache.http.util.EntityUtils;
-import org.codehaus.mojo.animal_sniffer.IgnoreJRERequirement;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.OutputStream;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-
-import static org.elasticsearch.client.RestClientTestUtil.getAllStatusCodes;
-import static org.elasticsearch.client.RestClientTestUtil.getHttpMethods;
-import static org.elasticsearch.client.RestClientTestUtil.randomStatusCode;
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertThat;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-/**
- * Integration test to check interaction between {@link RestClient} and {@link org.apache.http.client.HttpClient}.
- * Works against a real http server, one single host.
- */
-//animal-sniffer doesn't like our usage of com.sun.net.httpserver.* classes
-@IgnoreJRERequirement
-public class RestClientIntegTests extends RestClientTestCase {
-
-    private static HttpServer httpServer;
-    private static RestClient restClient;
-    private static Header[] defaultHeaders;
-
-    @BeforeClass
-    public static void startHttpServer() throws Exception {
-        httpServer = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0);
-        httpServer.start();
-        //returns a different status code depending on the path
-        for (int statusCode : getAllStatusCodes()) {
-            createStatusCodeContext(httpServer, statusCode);
-        }
-        int numHeaders = randomIntBetween(0, 5);
-        defaultHeaders = generateHeaders("Header-default", "Header-array", numHeaders);
-        restClient = RestClient.builder(new HttpHost(httpServer.getAddress().getHostString(), httpServer.getAddress().getPort()))
-                .setDefaultHeaders(defaultHeaders).build();
-    }
-
-    private static void createStatusCodeContext(HttpServer httpServer, final int statusCode) {
-        httpServer.createContext("/" + statusCode, new ResponseHandler(statusCode));
-    }
-
-    //animal-sniffer doesn't like our usage of com.sun.net.httpserver.* classes
-    @IgnoreJRERequirement
-    private static class ResponseHandler implements HttpHandler {
-        private final int statusCode;
-
-        ResponseHandler(int statusCode) {
-            this.statusCode = statusCode;
-        }
-
-        @Override
-        public void handle(HttpExchange httpExchange) throws IOException {
-            StringBuilder body = new StringBuilder();
-            try (InputStreamReader reader = new InputStreamReader(httpExchange.getRequestBody(), Consts.UTF_8)) {
-                char[] buffer = new char[256];
-                int read;
-                while ((read = reader.read(buffer)) != -1) {
-                    body.append(buffer, 0, read);
-                }
-            }
-            Headers requestHeaders = httpExchange.getRequestHeaders();
-            Headers responseHeaders = httpExchange.getResponseHeaders();
-            for (Map.Entry<String, List<String>> header : requestHeaders.entrySet()) {
-                responseHeaders.put(header.getKey(), header.getValue());
-            }
-            httpExchange.getRequestBody().close();
-            httpExchange.sendResponseHeaders(statusCode, body.length() == 0 ? -1 : body.length());
-            if (body.length() > 0) {
-                try (OutputStream out = httpExchange.getResponseBody()) {
-                    out.write(body.toString().getBytes(Consts.UTF_8));
-                }
-            }
-            httpExchange.close();
-        }
-    }
-
-    @AfterClass
-    public static void stopHttpServers() throws IOException {
-        restClient.close();
-        restClient = null;
-        httpServer.stop(0);
-        httpServer = null;
-    }
-
-    /**
-     * End to end test for headers. We test it explicitly against a real http client as there are different ways
-     * to set/add headers to the {@link org.apache.http.client.HttpClient}.
-     * Exercises the test http server ability to send back whatever headers it received.
-     */
-    public void testHeaders() throws IOException {
-        for (String method : getHttpMethods()) {
-            final Set<String> standardHeaders = new HashSet<>(Arrays.asList("Connection", "Host", "User-agent", "Date"));
-            if (method.equals("HEAD") == false) {
-                standardHeaders.add("Content-length");
-            }
-
-            final int numHeaders = randomIntBetween(1, 5);
-            final Header[] headers = generateHeaders("Header", "Header-array", numHeaders);
-            final Map<String, List<String>> expectedHeaders = new HashMap<>();
-
-            addHeaders(expectedHeaders, defaultHeaders, headers);
-
-            final int statusCode = randomStatusCode(getRandom());
-            Response esResponse;
-            try {
-                esResponse = restClient.performRequest(method, "/" + statusCode, Collections.<String, String>emptyMap(), headers);
-            } catch(ResponseException e) {
-                esResponse = e.getResponse();
-            }
-            assertThat(esResponse.getStatusLine().getStatusCode(), equalTo(statusCode));
-            for (final Header responseHeader : esResponse.getHeaders()) {
-                final String name = responseHeader.getName();
-                final String value = responseHeader.getValue();
-                if (name.startsWith("Header")) {
-                    final List<String> values = expectedHeaders.get(name);
-                    assertNotNull("found response header [" + name + "] that wasn't originally sent: " + value, values);
-                    assertTrue("found incorrect response header [" + name + "]: " + value, values.remove(value));
-
-                    // we've collected them all
-                    if (values.isEmpty()) {
-                        expectedHeaders.remove(name);
-                    }
-                } else {
-                    assertTrue("unknown header was returned " + name, standardHeaders.remove(name));
-                }
-            }
-            assertTrue("some headers that were sent weren't returned: " + expectedHeaders, expectedHeaders.isEmpty());
-            assertTrue("some expected standard headers weren't returned: " + standardHeaders, standardHeaders.isEmpty());
-        }
-    }
-
-    /**
-     * End to end test for delete with body. We test it explicitly as it is not supported
-     * out of the box by {@link org.apache.http.client.HttpClient}.
-     * Exercises the test http server ability to send back whatever body it received.
-     */
-    public void testDeleteWithBody() throws IOException {
-        bodyTest("DELETE");
-    }
-
-    /**
-     * End to end test for get with body. We test it explicitly as it is not supported
-     * out of the box by {@link org.apache.http.client.HttpClient}.
-     * Exercises the test http server ability to send back whatever body it received.
-     */
-    public void testGetWithBody() throws IOException {
-        bodyTest("GET");
-    }
-
-    /**
-     * Ensure that pathPrefix works as expected.
-     */
-    public void testPathPrefix() throws IOException {
-        // guarantee no other test setup collides with this one and lets it sneak through
-        final String uniqueContextSuffix = "/testPathPrefix";
-        final String pathPrefix = "base/" + randomAsciiOfLengthBetween(1, 5) + "/";
-        final int statusCode = randomStatusCode(getRandom());
-
-        final HttpContext context =
-            httpServer.createContext("/" + pathPrefix + statusCode + uniqueContextSuffix, new ResponseHandler(statusCode));
-
-        try (final RestClient client =
-                RestClient.builder(new HttpHost(httpServer.getAddress().getHostString(), httpServer.getAddress().getPort()))
-                    .setPathPrefix((randomBoolean() ? "/" : "") + pathPrefix).build()) {
-
-            for (final String method : getHttpMethods()) {
-                Response esResponse;
-                try {
-                    esResponse = client.performRequest(method, "/" + statusCode + uniqueContextSuffix);
-                } catch(ResponseException e) {
-                    esResponse = e.getResponse();
-                }
-
-                assertThat(esResponse.getRequestLine().getUri(), equalTo("/" + pathPrefix + statusCode + uniqueContextSuffix));
-                assertThat(esResponse.getStatusLine().getStatusCode(), equalTo(statusCode));
-            }
-        } finally {
-            httpServer.removeContext(context);
-        }
-    }
-
-    private void bodyTest(String method) throws IOException {
-        String requestBody = "{ \"field\": \"value\" }";
-        StringEntity entity = new StringEntity(requestBody);
-        int statusCode = randomStatusCode(getRandom());
-        Response esResponse;
-        try {
-            esResponse = restClient.performRequest(method, "/" + statusCode, Collections.<String, String>emptyMap(), entity);
-        } catch(ResponseException e) {
-            esResponse = e.getResponse();
-        }
-        assertEquals(statusCode, esResponse.getStatusLine().getStatusCode());
-        assertEquals(requestBody, EntityUtils.toString(esResponse.getEntity()));
-    }
-
-    public void testAsyncRequests() throws Exception {
-        int numRequests = randomIntBetween(5, 20);
-        final CountDownLatch latch = new CountDownLatch(numRequests);
-        final List<TestResponse> responses = new CopyOnWriteArrayList<>();
-        for (int i = 0; i < numRequests; i++) {
-            final String method = RestClientTestUtil.randomHttpMethod(getRandom());
-            final int statusCode = randomStatusCode(getRandom());
-            restClient.performRequestAsync(method, "/" + statusCode, new ResponseListener() {
-                @Override
-                public void onSuccess(Response response) {
-                    responses.add(new TestResponse(method, statusCode, response));
-                    latch.countDown();
-                }
-
-                @Override
-                public void onFailure(Exception exception) {
-                    responses.add(new TestResponse(method, statusCode, exception));
-                    latch.countDown();
-                }
-            });
-        }
-        assertTrue(latch.await(5, TimeUnit.SECONDS));
-
-        assertEquals(numRequests, responses.size());
-        for (TestResponse response : responses) {
-            assertEquals(response.method, response.getResponse().getRequestLine().getMethod());
-            assertEquals(response.statusCode, response.getResponse().getStatusLine().getStatusCode());
-
-        }
-    }
-
-    private static class TestResponse {
-        private final String method;
-        private final int statusCode;
-        private final Object response;
-
-        TestResponse(String method, int statusCode, Object response) {
-            this.method = method;
-            this.statusCode = statusCode;
-            this.response = response;
-        }
-
-        Response getResponse() {
-            if (response instanceof Response) {
-                return (Response) response;
-            }
-            if (response instanceof ResponseException) {
-                return ((ResponseException) response).getResponse();
-            }
-            throw new AssertionError("unexpected response " + response.getClass());
-        }
-    }
-}
diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsIntegTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsIntegTests.java
new file mode 100644
index 0000000000000..f997f798712e6
--- /dev/null
+++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsIntegTests.java
@@ -0,0 +1,210 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import com.sun.net.httpserver.HttpExchange;
+import com.sun.net.httpserver.HttpHandler;
+import com.sun.net.httpserver.HttpServer;
+import org.apache.http.HttpHost;
+import org.codehaus.mojo.animal_sniffer.IgnoreJRERequirement;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import static org.elasticsearch.client.RestClientTestUtil.getAllStatusCodes;
+import static org.elasticsearch.client.RestClientTestUtil.randomErrorNoRetryStatusCode;
+import static org.elasticsearch.client.RestClientTestUtil.randomOkStatusCode;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Integration test to check interaction between {@link RestClient} and {@link org.apache.http.client.HttpClient}.
+ * Works against real http servers, multiple hosts. Also tests failover by randomly shutting down hosts.
+ */
+//animal-sniffer doesn't like our usage of com.sun.net.httpserver.* classes
+@IgnoreJRERequirement
+public class RestClientMultipleHostsIntegTests extends RestClientTestCase {
+
+    private static HttpServer[] httpServers;
+    private static RestClient restClient;
+    private static String pathPrefix;
+
+    @BeforeClass
+    public static void startHttpServer() throws Exception {
+        String pathPrefixWithoutLeadingSlash;
+        if (randomBoolean()) {
+            pathPrefixWithoutLeadingSlash = "testPathPrefix/" + randomAsciiOfLengthBetween(1, 5);
+            pathPrefix = "/" + pathPrefixWithoutLeadingSlash;
+        } else {
+            pathPrefix = pathPrefixWithoutLeadingSlash = "";
+        }
+        int numHttpServers = randomIntBetween(2, 4);
+        httpServers = new HttpServer[numHttpServers];
+        HttpHost[] httpHosts = new HttpHost[numHttpServers];
+        for (int i = 0; i < numHttpServers; i++) {
+            HttpServer httpServer = createHttpServer();
+            httpServers[i] = httpServer;
+            httpHosts[i] = new HttpHost(httpServer.getAddress().getHostString(), httpServer.getAddress().getPort());
+        }
+        RestClientBuilder restClientBuilder = RestClient.builder(httpHosts);
+        if (pathPrefix.length() > 0) {
+            restClientBuilder.setPathPrefix((randomBoolean() ? "/" : "") + pathPrefixWithoutLeadingSlash);
+        }
+        restClient = restClientBuilder.build();
+    }
+
+    private static HttpServer createHttpServer() throws Exception {
+        HttpServer httpServer = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0);
+        httpServer.start();
+        //returns a different status code depending on the path
+        for (int statusCode : getAllStatusCodes()) {
+            httpServer.createContext(pathPrefix + "/" + statusCode, new ResponseHandler(statusCode));
+        }
+        return httpServer;
+    }
+
+    //animal-sniffer doesn't like our usage of com.sun.net.httpserver.* classes
+    @IgnoreJRERequirement
+    private static class ResponseHandler implements HttpHandler {
+        private final int statusCode;
+
+        ResponseHandler(int statusCode) {
+            this.statusCode = statusCode;
+        }
+
+        @Override
+        public void handle(HttpExchange httpExchange) throws IOException {
+            httpExchange.getRequestBody().close();
+            httpExchange.sendResponseHeaders(statusCode, -1);
+            httpExchange.close();
+        }
+    }
+
+    @AfterClass
+    public static void stopHttpServers() throws IOException {
+        restClient.close();
+        restClient = null;
+        for (HttpServer httpServer : httpServers) {
+            httpServer.stop(0);
+        }
+        httpServers = null;
+    }
+
+    @Before
+    public void stopRandomHost() {
+        //verify that shutting down some hosts doesn't matter as long as one working host is left behind
+        if (httpServers.length > 1 && randomBoolean()) {
+            List<HttpServer> updatedHttpServers = new ArrayList<>(httpServers.length - 1);
+            int nodeIndex = randomInt(httpServers.length - 1);
+            for (int i = 0; i < httpServers.length; i++) {
+                HttpServer httpServer = httpServers[i];
+                if (i == nodeIndex) {
+                    httpServer.stop(0);
+                } else {
+                    updatedHttpServers.add(httpServer);
+                }
+            }
+            httpServers = updatedHttpServers.toArray(new HttpServer[updatedHttpServers.size()]);
+        }
+    }
+
+    public void testSyncRequests() throws IOException {
+        int numRequests = randomIntBetween(5, 20);
+        for (int i = 0; i < numRequests; i++) {
+            final String method = RestClientTestUtil.randomHttpMethod(getRandom());
+            //we don't test status codes that are subject to retries as they interfere with hosts being stopped
+            final int statusCode = randomBoolean() ? randomOkStatusCode(getRandom()) : randomErrorNoRetryStatusCode(getRandom());
+            Response response;
+            try {
+                response = restClient.performRequest(method, "/" + statusCode);
+            } catch(ResponseException responseException) {
+                response = responseException.getResponse();
+            }
+            assertEquals(method, response.getRequestLine().getMethod());
+            assertEquals(statusCode, response.getStatusLine().getStatusCode());
+            assertEquals((pathPrefix.length() > 0 ? pathPrefix : "") + "/" + statusCode, response.getRequestLine().getUri());
+        }
+    }
+
+    public void testAsyncRequests() throws Exception {
+        int numRequests = randomIntBetween(5, 20);
+        final CountDownLatch latch = new CountDownLatch(numRequests);
+        final List<TestResponse> responses = new CopyOnWriteArrayList<>();
+        for (int i = 0; i < numRequests; i++) {
+            final String method = RestClientTestUtil.randomHttpMethod(getRandom());
+            //we don't test status codes that are subject to retries as they interfere with hosts being stopped
+            final int statusCode = randomBoolean() ? randomOkStatusCode(getRandom()) : randomErrorNoRetryStatusCode(getRandom());
+            restClient.performRequestAsync(method, "/" + statusCode, new ResponseListener() {
+                @Override
+                public void onSuccess(Response response) {
+                    responses.add(new TestResponse(method, statusCode, response));
+                    latch.countDown();
+                }
+
+                @Override
+                public void onFailure(Exception exception) {
+                    responses.add(new TestResponse(method, statusCode, exception));
+                    latch.countDown();
+                }
+            });
+        }
+        assertTrue(latch.await(5, TimeUnit.SECONDS));
+
+        assertEquals(numRequests, responses.size());
+        for (TestResponse testResponse : responses) {
+            Response response = testResponse.getResponse();
+            assertEquals(testResponse.method, response.getRequestLine().getMethod());
+            assertEquals(testResponse.statusCode, response.getStatusLine().getStatusCode());
+            assertEquals((pathPrefix.length() > 0 ? pathPrefix : "") + "/" + testResponse.statusCode,
+                    response.getRequestLine().getUri());
+        }
+    }
+
+    private static class TestResponse {
+        private final String method;
+        private final int statusCode;
+        private final Object response;
+
+        TestResponse(String method, int statusCode, Object response) {
+            this.method = method;
+            this.statusCode = statusCode;
+            this.response = response;
+        }
+
+        Response getResponse() {
+            if (response instanceof Response) {
+                return (Response) response;
+            }
+            if (response instanceof ResponseException) {
+                return ((ResponseException) response).getResponse();
+            }
+            throw new AssertionError("unexpected response " + response.getClass());
+        }
+    }
+}
diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java
index 049a216936f7a..6f87a244ff59f 100644
--- a/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java
+++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java
@@ -19,15 +19,17 @@
 
 package org.elasticsearch.client;
 
-import com.carrotsearch.randomizedtesting.generators.RandomInts;
+import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
 import org.apache.http.Header;
 import org.apache.http.HttpHost;
 import org.apache.http.HttpResponse;
 import org.apache.http.ProtocolVersion;
 import org.apache.http.StatusLine;
 import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.client.protocol.HttpClientContext;
 import org.apache.http.concurrent.FutureCallback;
 import org.apache.http.conn.ConnectTimeoutException;
+import org.apache.http.impl.auth.BasicScheme;
 import org.apache.http.impl.nio.client.CloseableHttpAsyncClient;
 import org.apache.http.message.BasicHttpResponse;
 import org.apache.http.message.BasicStatusLine;
@@ -73,13 +75,15 @@ public class RestClientMultipleHostsTests extends RestClientTestCase {
     public void createRestClient() throws IOException {
         CloseableHttpAsyncClient httpClient = mock(CloseableHttpAsyncClient.class);
         when(httpClient.<HttpResponse>execute(any(HttpAsyncRequestProducer.class), any(HttpAsyncResponseConsumer.class),
-                any(FutureCallback.class))).thenAnswer(new Answer<Future<HttpResponse>>() {
+               any(HttpClientContext.class), any(FutureCallback.class))).thenAnswer(new Answer<Future<HttpResponse>>() {
             @Override
             public Future<HttpResponse> answer(InvocationOnMock invocationOnMock) throws Throwable {
                 HttpAsyncRequestProducer requestProducer = (HttpAsyncRequestProducer) invocationOnMock.getArguments()[0];
                 HttpUriRequest request = (HttpUriRequest)requestProducer.generateRequest();
                 HttpHost httpHost = requestProducer.getTarget();
-                FutureCallback<HttpResponse> futureCallback = (FutureCallback<HttpResponse>) invocationOnMock.getArguments()[2];
+                HttpClientContext context = (HttpClientContext) invocationOnMock.getArguments()[2];
+                assertThat(context.getAuthCache().get(httpHost), instanceOf(BasicScheme.class));
+                FutureCallback<HttpResponse> futureCallback = (FutureCallback<HttpResponse>) invocationOnMock.getArguments()[3];
                 //return the desired status code or exception depending on the path
                 if (request.getURI().getPath().equals("/soe")) {
                     futureCallback.failed(new SocketTimeoutException(httpHost.toString()));
@@ -95,7 +99,7 @@ public Future<HttpResponse> answer(InvocationOnMock invocationOnMock) throws Thr
                 return null;
             }
         });
-        int numHosts = RandomInts.randomIntBetween(getRandom(), 2, 5);
+        int numHosts = RandomNumbers.randomIntBetween(getRandom(), 2, 5);
         httpHosts = new HttpHost[numHosts];
         for (int i = 0; i < numHosts; i++) {
             httpHosts[i] = new HttpHost("localhost", 9200 + i);
@@ -105,7 +109,7 @@ public Future<HttpResponse> answer(InvocationOnMock invocationOnMock) throws Thr
     }
 
     public void testRoundRobinOkStatusCodes() throws IOException {
-        int numIters = RandomInts.randomIntBetween(getRandom(), 1, 5);
+        int numIters = RandomNumbers.randomIntBetween(getRandom(), 1, 5);
         for (int i = 0; i < numIters; i++) {
             Set<HttpHost> hostsSet = new HashSet<>();
             Collections.addAll(hostsSet, httpHosts);
@@ -121,7 +125,7 @@ public void testRoundRobinOkStatusCodes() throws IOException {
     }
 
     public void testRoundRobinNoRetryErrors() throws IOException {
-        int numIters = RandomInts.randomIntBetween(getRandom(), 1, 5);
+        int numIters = RandomNumbers.randomIntBetween(getRandom(), 1, 5);
         for (int i = 0; i < numIters; i++) {
             Set<HttpHost> hostsSet = new HashSet<>();
             Collections.addAll(hostsSet, httpHosts);
@@ -198,7 +202,7 @@ public void testRoundRobinRetryErrors() throws IOException {
             assertEquals("every host should have been used but some weren't: " + hostsSet, 0, hostsSet.size());
         }
 
-        int numIters = RandomInts.randomIntBetween(getRandom(), 2, 5);
+        int numIters = RandomNumbers.randomIntBetween(getRandom(), 2, 5);
         for (int i = 1; i <= numIters; i++) {
             //check that one different host is resurrected at each new attempt
             Set<HttpHost> hostsSet = new HashSet<>();
@@ -228,7 +232,7 @@ public void testRoundRobinRetryErrors() throws IOException {
             if (getRandom().nextBoolean()) {
                 //mark one host back alive through a successful request and check that all requests after that are sent to it
                 HttpHost selectedHost = null;
-                int iters = RandomInts.randomIntBetween(getRandom(), 2, 10);
+                int iters = RandomNumbers.randomIntBetween(getRandom(), 2, 10);
                 for (int y = 0; y < iters; y++) {
                     int statusCode = randomErrorNoRetryStatusCode(getRandom());
                     Response response;
@@ -269,7 +273,7 @@ public void testRoundRobinRetryErrors() throws IOException {
     }
 
     private static String randomErrorRetryEndpoint() {
-        switch(RandomInts.randomIntBetween(getRandom(), 0, 3)) {
+        switch(RandomNumbers.randomIntBetween(getRandom(), 0, 3)) {
             case 0:
                 return "/" + randomErrorRetryStatusCode(getRandom());
             case 1:
diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostIntegTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostIntegTests.java
new file mode 100644
index 0000000000000..9b63e3492fd80
--- /dev/null
+++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostIntegTests.java
@@ -0,0 +1,266 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import com.sun.net.httpserver.Headers;
+import com.sun.net.httpserver.HttpExchange;
+import com.sun.net.httpserver.HttpHandler;
+import com.sun.net.httpserver.HttpServer;
+import org.apache.http.Consts;
+import org.apache.http.Header;
+import org.apache.http.HttpHost;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
+import org.apache.http.util.EntityUtils;
+import org.codehaus.mojo.animal_sniffer.IgnoreJRERequirement;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.elasticsearch.client.RestClientTestUtil.getAllStatusCodes;
+import static org.elasticsearch.client.RestClientTestUtil.getHttpMethods;
+import static org.elasticsearch.client.RestClientTestUtil.randomStatusCode;
+import static org.hamcrest.Matchers.nullValue;
+import static org.hamcrest.Matchers.startsWith;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Integration test to check interaction between {@link RestClient} and {@link org.apache.http.client.HttpClient}.
+ * Works against a real http server, one single host.
+ */
+//animal-sniffer doesn't like our usage of com.sun.net.httpserver.* classes
+@IgnoreJRERequirement
+public class RestClientSingleHostIntegTests extends RestClientTestCase {
+
+    private static HttpServer httpServer;
+    private static RestClient restClient;
+    private static String pathPrefix;
+    private static Header[] defaultHeaders;
+
+    @BeforeClass
+    public static void startHttpServer() throws Exception {
+        pathPrefix = randomBoolean() ? "/testPathPrefix/" + randomAsciiOfLengthBetween(1, 5) : "";
+        httpServer = createHttpServer();
+        defaultHeaders = RestClientTestUtil.randomHeaders(getRandom(), "Header-default");
+        restClient = createRestClient(false, true);
+    }
+
+    private static HttpServer createHttpServer() throws Exception {
+        HttpServer httpServer = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0);
+        httpServer.start();
+        //returns a different status code depending on the path
+        for (int statusCode : getAllStatusCodes()) {
+            httpServer.createContext(pathPrefix + "/" + statusCode, new ResponseHandler(statusCode));
+        }
+        return httpServer;
+    }
+
+    //animal-sniffer doesn't like our usage of com.sun.net.httpserver.* classes
+    @IgnoreJRERequirement
+    private static class ResponseHandler implements HttpHandler {
+        private final int statusCode;
+
+        ResponseHandler(int statusCode) {
+            this.statusCode = statusCode;
+        }
+
+        @Override
+        public void handle(HttpExchange httpExchange) throws IOException {
+            StringBuilder body = new StringBuilder();
+            try (InputStreamReader reader = new InputStreamReader(httpExchange.getRequestBody(), Consts.UTF_8)) {
+                char[] buffer = new char[256];
+                int read;
+                while ((read = reader.read(buffer)) != -1) {
+                    body.append(buffer, 0, read);
+                }
+            }
+            Headers requestHeaders = httpExchange.getRequestHeaders();
+            Headers responseHeaders = httpExchange.getResponseHeaders();
+            for (Map.Entry<String, List<String>> header : requestHeaders.entrySet()) {
+                responseHeaders.put(header.getKey(), header.getValue());
+            }
+            httpExchange.getRequestBody().close();
+            httpExchange.sendResponseHeaders(statusCode, body.length() == 0 ? -1 : body.length());
+            if (body.length() > 0) {
+                try (OutputStream out = httpExchange.getResponseBody()) {
+                    out.write(body.toString().getBytes(Consts.UTF_8));
+                }
+            }
+            httpExchange.close();
+        }
+    }
+
+    private static RestClient createRestClient(final boolean useAuth, final boolean usePreemptiveAuth) {
+        // provide the username/password for every request
+        final BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider();
+        credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("user", "pass"));
+
+        final RestClientBuilder restClientBuilder = RestClient.builder(
+            new HttpHost(httpServer.getAddress().getHostString(), httpServer.getAddress().getPort())).setDefaultHeaders(defaultHeaders);
+        if (pathPrefix.length() > 0) {
+            // sometimes cut off the leading slash
+            restClientBuilder.setPathPrefix(randomBoolean() ? pathPrefix.substring(1) : pathPrefix);
+        }
+
+        if (useAuth) {
+            restClientBuilder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
+                @Override
+                public HttpAsyncClientBuilder customizeHttpClient(final HttpAsyncClientBuilder httpClientBuilder) {
+                    if (usePreemptiveAuth == false) {
+                        // disable preemptive auth by ignoring any authcache
+                        httpClientBuilder.disableAuthCaching();
+                    }
+
+                    return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
+                }
+            });
+        }
+
+        return restClientBuilder.build();
+    }
+
+    @AfterClass
+    public static void stopHttpServers() throws IOException {
+        restClient.close();
+        restClient = null;
+        httpServer.stop(0);
+        httpServer = null;
+    }
+
+    /**
+     * End to end test for headers. We test it explicitly against a real http client as there are different ways
+     * to set/add headers to the {@link org.apache.http.client.HttpClient}.
+     * Exercises the test http server ability to send back whatever headers it received.
+     */
+    public void testHeaders() throws IOException {
+        for (String method : getHttpMethods()) {
+            final Set<String> standardHeaders = new HashSet<>(Arrays.asList("Connection", "Host", "User-agent", "Date"));
+            if (method.equals("HEAD") == false) {
+                standardHeaders.add("Content-length");
+            }
+            final Header[] requestHeaders = RestClientTestUtil.randomHeaders(getRandom(), "Header");
+            final int statusCode = randomStatusCode(getRandom());
+            Response esResponse;
+            try {
+                esResponse = restClient.performRequest(method, "/" + statusCode, Collections.<String, String>emptyMap(), requestHeaders);
+            } catch(ResponseException e) {
+                esResponse = e.getResponse();
+            }
+
+            assertEquals(method, esResponse.getRequestLine().getMethod());
+            assertEquals(statusCode, esResponse.getStatusLine().getStatusCode());
+            assertEquals(pathPrefix + "/" + statusCode, esResponse.getRequestLine().getUri());
+            assertHeaders(defaultHeaders, requestHeaders, esResponse.getHeaders(), standardHeaders);
+            for (final Header responseHeader : esResponse.getHeaders()) {
+                String name = responseHeader.getName();
+                if (name.startsWith("Header") == false) {
+                    assertTrue("unknown header was returned " + name, standardHeaders.remove(name));
+                }
+            }
+            assertTrue("some expected standard headers weren't returned: " + standardHeaders, standardHeaders.isEmpty());
+        }
+    }
+
+    /**
+     * End to end test for delete with body. We test it explicitly as it is not supported
+     * out of the box by {@link org.apache.http.client.HttpClient}.
+     * Exercises the test http server ability to send back whatever body it received.
+     */
+    public void testDeleteWithBody() throws IOException {
+        bodyTest("DELETE");
+    }
+
+    /**
+     * End to end test for get with body. We test it explicitly as it is not supported
+     * out of the box by {@link org.apache.http.client.HttpClient}.
+     * Exercises the test http server ability to send back whatever body it received.
+     */
+    public void testGetWithBody() throws IOException {
+        bodyTest("GET");
+    }
+
+    /**
+     * Verify that credentials are sent on the first request with preemptive auth enabled (default when provided with credentials).
+     */
+    public void testPreemptiveAuthEnabled() throws IOException  {
+        final String[] methods = { "POST", "PUT", "GET", "DELETE" };
+
+        try (RestClient restClient = createRestClient(true, true)) {
+            for (final String method : methods) {
+                final Response response = bodyTest(restClient, method);
+
+                assertThat(response.getHeader("Authorization"), startsWith("Basic"));
+            }
+        }
+    }
+
+    /**
+     * Verify that credentials are <em>not</em> sent on the first request with preemptive auth disabled.
+     */
+    public void testPreemptiveAuthDisabled() throws IOException  {
+        final String[] methods = { "POST", "PUT", "GET", "DELETE" };
+
+        try (RestClient restClient = createRestClient(true, false)) {
+            for (final String method : methods) {
+                final Response response = bodyTest(restClient, method);
+
+                assertThat(response.getHeader("Authorization"), nullValue());
+            }
+        }
+    }
+
+    private Response bodyTest(final String method) throws IOException {
+        return bodyTest(restClient, method);
+    }
+
+    private Response bodyTest(final RestClient restClient, final String method) throws IOException {
+        String requestBody = "{ \"field\": \"value\" }";
+        StringEntity entity = new StringEntity(requestBody);
+        int statusCode = randomStatusCode(getRandom());
+        Response esResponse;
+        try {
+            esResponse = restClient.performRequest(method, "/" + statusCode, Collections.<String, String>emptyMap(), entity);
+        } catch(ResponseException e) {
+            esResponse = e.getResponse();
+        }
+        assertEquals(method, esResponse.getRequestLine().getMethod());
+        assertEquals(statusCode, esResponse.getStatusLine().getStatusCode());
+        assertEquals(pathPrefix + "/" + statusCode, esResponse.getRequestLine().getUri());
+        assertEquals(requestBody, EntityUtils.toString(esResponse.getEntity()));
+
+        return esResponse;
+    }
+}
diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java
index 92e2b0da971df..a74310daa0140 100644
--- a/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java
+++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java
@@ -34,10 +34,12 @@
 import org.apache.http.client.methods.HttpPut;
 import org.apache.http.client.methods.HttpTrace;
 import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.client.protocol.HttpClientContext;
 import org.apache.http.client.utils.URIBuilder;
 import org.apache.http.concurrent.FutureCallback;
 import org.apache.http.conn.ConnectTimeoutException;
 import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.auth.BasicScheme;
 import org.apache.http.impl.nio.client.CloseableHttpAsyncClient;
 import org.apache.http.message.BasicHttpResponse;
 import org.apache.http.message.BasicStatusLine;
@@ -56,7 +58,6 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.Future;
@@ -70,7 +71,6 @@
 import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
@@ -98,11 +98,13 @@ public class RestClientSingleHostTests extends RestClientTestCase {
     public void createRestClient() throws IOException {
         httpClient = mock(CloseableHttpAsyncClient.class);
         when(httpClient.<HttpResponse>execute(any(HttpAsyncRequestProducer.class), any(HttpAsyncResponseConsumer.class),
-                any(FutureCallback.class))).thenAnswer(new Answer<Future<HttpResponse>>() {
+                any(HttpClientContext.class), any(FutureCallback.class))).thenAnswer(new Answer<Future<HttpResponse>>() {
                     @Override
                     public Future<HttpResponse> answer(InvocationOnMock invocationOnMock) throws Throwable {
                         HttpAsyncRequestProducer requestProducer = (HttpAsyncRequestProducer) invocationOnMock.getArguments()[0];
-                        FutureCallback<HttpResponse> futureCallback = (FutureCallback<HttpResponse>) invocationOnMock.getArguments()[2];
+                        HttpClientContext context = (HttpClientContext) invocationOnMock.getArguments()[2];
+                        assertThat(context.getAuthCache().get(httpHost), instanceOf(BasicScheme.class));
+                        FutureCallback<HttpResponse> futureCallback = (FutureCallback<HttpResponse>) invocationOnMock.getArguments()[3];
                         HttpUriRequest request = (HttpUriRequest)requestProducer.generateRequest();
                         //return the desired status code or exception depending on the path
                         if (request.getURI().getPath().equals("/soe")) {
@@ -132,13 +134,23 @@ public Future<HttpResponse> answer(InvocationOnMock invocationOnMock) throws Thr
                 });
 
 
-        int numHeaders = randomIntBetween(0, 3);
-        defaultHeaders = generateHeaders("Header-default", "Header-array", numHeaders);
+        defaultHeaders = RestClientTestUtil.randomHeaders(getRandom(), "Header-default");
         httpHost = new HttpHost("localhost", 9200);
         failureListener = new HostsTrackingFailureListener();
         restClient = new RestClient(httpClient, 10000, defaultHeaders, new HttpHost[]{httpHost}, null, failureListener);
     }
 
+    public void testNullPath() throws IOException {
+        for (String method : getHttpMethods()) {
+            try {
+                restClient.performRequest(method, null);
+                fail("path set to null should fail!");
+            } catch (NullPointerException e) {
+                assertEquals("path must not be null", e.getMessage());
+            }
+        }
+    }
+
     /**
      * Verifies the content of the {@link HttpRequest} that's internally created and passed through to the http client
      */
@@ -149,7 +161,7 @@ public void testInternalHttpRequest() throws Exception {
         for (String httpMethod : getHttpMethods()) {
             HttpUriRequest expectedRequest = performRandomRequest(httpMethod);
             verify(httpClient, times(++times)).<HttpResponse>execute(requestArgumentCaptor.capture(),
-                    any(HttpAsyncResponseConsumer.class), any(FutureCallback.class));
+                    any(HttpAsyncResponseConsumer.class), any(HttpClientContext.class), any(FutureCallback.class));
             HttpUriRequest actualRequest = (HttpUriRequest)requestArgumentCaptor.getValue().generateRequest();
             assertEquals(expectedRequest.getURI(), actualRequest.getURI());
             assertEquals(expectedRequest.getClass(), actualRequest.getClass());
@@ -209,23 +221,45 @@ public void testOkStatusCodes() throws IOException {
      */
     public void testErrorStatusCodes() throws IOException {
         for (String method : getHttpMethods()) {
+            Set<Integer> expectedIgnores = new HashSet<>();
+            String ignoreParam = "";
+            if (HttpHead.METHOD_NAME.equals(method)) {
+                expectedIgnores.add(404);
+            }
+            if (randomBoolean()) {
+                int numIgnores = randomIntBetween(1, 3);
+                for (int i = 0; i < numIgnores; i++) {
+                    Integer code = randomFrom(getAllErrorStatusCodes());
+                    expectedIgnores.add(code);
+                    ignoreParam += code;
+                    if (i < numIgnores - 1) {
+                        ignoreParam += ",";
+                    }
+                }
+            }
             //error status codes should cause an exception to be thrown
             for (int errorStatusCode : getAllErrorStatusCodes()) {
                 try {
-                    Response response = performRequest(method, "/" + errorStatusCode);
-                    if (method.equals("HEAD") && errorStatusCode == 404) {
-                        //no exception gets thrown although we got a 404
-                        assertThat(response.getStatusLine().getStatusCode(), equalTo(errorStatusCode));
+                    Map<String, String> params;
+                    if (ignoreParam.isEmpty()) {
+                        params = Collections.emptyMap();
+                    } else {
+                        params = Collections.singletonMap("ignore", ignoreParam);
+                    }
+                    Response response = performRequest(method, "/" + errorStatusCode, params);
+                    if (expectedIgnores.contains(errorStatusCode)) {
+                        //no exception gets thrown although we got an error status code, as it was configured to be ignored
+                        assertEquals(errorStatusCode, response.getStatusLine().getStatusCode());
                     } else {
                         fail("request should have failed");
                     }
                 } catch(ResponseException e) {
-                    if (method.equals("HEAD") && errorStatusCode == 404) {
+                    if (expectedIgnores.contains(errorStatusCode)) {
                         throw e;
                     }
-                    assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(errorStatusCode));
+                    assertEquals(errorStatusCode, e.getResponse().getStatusLine().getStatusCode());
                 }
-                if (errorStatusCode <= 500) {
+                if (errorStatusCode <= 500 || expectedIgnores.contains(errorStatusCode)) {
                     failureListener.assertNotCalled();
                 } else {
                     failureListener.assertCalled(httpHost);
@@ -328,44 +362,27 @@ public void testNullParams() throws IOException {
      */
     public void testHeaders() throws IOException {
         for (String method : getHttpMethods()) {
-            final int numHeaders = randomIntBetween(1, 5);
-            final Header[] headers = generateHeaders("Header", null, numHeaders);
-            final Map<String, List<String>> expectedHeaders = new HashMap<>();
-
-            addHeaders(expectedHeaders, defaultHeaders, headers);
+            final Header[] requestHeaders = RestClientTestUtil.randomHeaders(getRandom(), "Header");
 
             final int statusCode = randomStatusCode(getRandom());
             Response esResponse;
             try {
-                esResponse = restClient.performRequest(method, "/" + statusCode, headers);
+                esResponse = restClient.performRequest(method, "/" + statusCode, requestHeaders);
             } catch(ResponseException e) {
                 esResponse = e.getResponse();
             }
             assertThat(esResponse.getStatusLine().getStatusCode(), equalTo(statusCode));
-            for (Header responseHeader : esResponse.getHeaders()) {
-                final String name = responseHeader.getName();
-                final String value = responseHeader.getValue();
-                final List<String> values = expectedHeaders.get(name);
-                assertNotNull("found response header [" + name + "] that wasn't originally sent: " + value, values);
-                assertTrue("found incorrect response header [" + name + "]: " + value, values.remove(value));
-
-                // we've collected them all
-                if (values.isEmpty()) {
-                    expectedHeaders.remove(name);
-                }
-            }
-            assertTrue("some headers that were sent weren't returned " + expectedHeaders, expectedHeaders.isEmpty());
+            assertHeaders(defaultHeaders, requestHeaders, esResponse.getHeaders(), Collections.<String>emptySet());
         }
     }
 
     private HttpUriRequest performRandomRequest(String method) throws Exception {
         String uriAsString = "/" + randomStatusCode(getRandom());
         URIBuilder uriBuilder = new URIBuilder(uriAsString);
-        Map<String, String> params = Collections.emptyMap();
+        final Map<String, String> params = new HashMap<>();
         boolean hasParams = randomBoolean();
         if (hasParams) {
             int numParams = randomIntBetween(1, 3);
-            params = new HashMap<>(numParams);
             for (int i = 0; i < numParams; i++) {
                 String paramKey = "param-" + i;
                 String paramValue = randomAsciiOfLengthBetween(3, 10);
@@ -373,6 +390,14 @@ private HttpUriRequest performRandomRequest(String method) throws Exception {
                 uriBuilder.addParameter(paramKey, paramValue);
             }
         }
+        if (randomBoolean()) {
+            //randomly add some ignore parameter, which doesn't get sent as part of the request
+            String ignore = Integer.toString(randomFrom(RestClientTestUtil.getAllErrorStatusCodes()));
+            if (randomBoolean()) {
+                ignore += "," + Integer.toString(randomFrom(RestClientTestUtil.getAllErrorStatusCodes()));
+            }
+            params.put("ignore", ignore);
+        }
         URI uri = uriBuilder.build();
 
         HttpUriRequest request;
@@ -413,10 +438,9 @@ private HttpUriRequest performRandomRequest(String method) throws Exception {
         }
 
         Header[] headers = new Header[0];
-        final int numHeaders = randomIntBetween(1, 5);
-        final Set<String> uniqueNames = new HashSet<>(numHeaders);
+        final Set<String> uniqueNames = new HashSet<>();
         if (randomBoolean()) {
-            headers = generateHeaders("Header", "Header-array", numHeaders);
+            headers = RestClientTestUtil.randomHeaders(getRandom(), "Header");
             for (Header header : headers) {
                 request.addHeader(header);
                 uniqueNames.add(header.getName());
@@ -444,16 +468,25 @@ private HttpUriRequest performRandomRequest(String method) throws Exception {
     }
 
     private Response performRequest(String method, String endpoint, Header... headers) throws IOException {
-        switch(randomIntBetween(0, 2)) {
+        return performRequest(method, endpoint, Collections.<String, String>emptyMap(), headers);
+    }
+
+    private Response performRequest(String method, String endpoint, Map<String, String> params, Header... headers) throws IOException {
+        int methodSelector;
+        if (params.isEmpty()) {
+            methodSelector = randomIntBetween(0, 2);
+        } else {
+            methodSelector = randomIntBetween(1, 2);
+        }
+        switch(methodSelector) {
             case 0:
                 return restClient.performRequest(method, endpoint, headers);
             case 1:
-                return restClient.performRequest(method, endpoint, Collections.<String, String>emptyMap(), headers);
+                return restClient.performRequest(method, endpoint, params, headers);
             case 2:
-                return restClient.performRequest(method, endpoint, Collections.<String, String>emptyMap(), (HttpEntity)null, headers);
+                return restClient.performRequest(method, endpoint, params, (HttpEntity)null, headers);
             default:
                 throw new UnsupportedOperationException();
         }
     }
-
 }
diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientTests.java
new file mode 100644
index 0000000000000..6978aab58fe71
--- /dev/null
+++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientTests.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import org.apache.http.Header;
+import org.apache.http.HttpHost;
+import org.apache.http.impl.nio.client.CloseableHttpAsyncClient;
+
+import java.net.URI;
+import java.util.Collections;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+
+public class RestClientTests extends RestClientTestCase {
+
+    public void testPerformAsyncWithUnsupportedMethod() throws Exception {
+        RestClient.SyncResponseListener listener = new RestClient.SyncResponseListener(10000);
+        try (RestClient restClient = createRestClient()) {
+            restClient.performRequestAsync("unsupported", randomAsciiOfLength(5), listener);
+            listener.get();
+
+            fail("should have failed because of unsupported method");
+        } catch (UnsupportedOperationException exception) {
+            assertEquals("http method not supported: unsupported", exception.getMessage());
+        }
+    }
+
+    public void testPerformAsyncWithNullParams() throws Exception {
+        RestClient.SyncResponseListener listener = new RestClient.SyncResponseListener(10000);
+        try (RestClient restClient = createRestClient()) {
+            restClient.performRequestAsync(randomAsciiOfLength(5), randomAsciiOfLength(5), null, listener);
+            listener.get();
+
+            fail("should have failed because of null parameters");
+        } catch (NullPointerException exception) {
+            assertEquals("params must not be null", exception.getMessage());
+        }
+    }
+
+    public void testPerformAsyncWithNullHeaders() throws Exception {
+        RestClient.SyncResponseListener listener = new RestClient.SyncResponseListener(10000);
+        try (RestClient restClient = createRestClient()) {
+            restClient.performRequestAsync("GET", randomAsciiOfLength(5), listener, (Header) null);
+            listener.get();
+
+            fail("should have failed because of null headers");
+        } catch (NullPointerException exception) {
+            assertEquals("request header must not be null", exception.getMessage());
+        }
+    }
+
+    public void testPerformAsyncWithWrongEndpoint() throws Exception {
+        RestClient.SyncResponseListener listener = new RestClient.SyncResponseListener(10000);
+        try (RestClient restClient = createRestClient()) {
+            restClient.performRequestAsync("GET", "::http:///", listener);
+            listener.get();
+
+            fail("should have failed because of wrong endpoint");
+        } catch (IllegalArgumentException exception) {
+            assertEquals("Expected scheme name at index 0: ::http:///", exception.getMessage());
+        }
+    }
+
+    public void testBuildUriLeavesPathUntouched() {
+        {
+            URI uri = RestClient.buildUri("/foo$bar", "/index/type/id", Collections.<String, String>emptyMap());
+            assertEquals("/foo$bar/index/type/id", uri.getPath());
+        }
+        {
+            URI uri = RestClient.buildUri(null, "/foo$bar/ty/pe/i/d", Collections.<String, String>emptyMap());
+            assertEquals("/foo$bar/ty/pe/i/d", uri.getPath());
+        }
+        {
+            URI uri = RestClient.buildUri(null, "/index/type/id", Collections.singletonMap("foo$bar", "x/y/z"));
+            assertEquals("/index/type/id", uri.getPath());
+            assertEquals("foo$bar=x/y/z", uri.getQuery());
+        }
+    }
+
+    private static RestClient createRestClient() {
+        HttpHost[] hosts = new HttpHost[]{new HttpHost("localhost", 9200)};
+        return new RestClient(mock(CloseableHttpAsyncClient.class), randomLongBetween(1_000, 30_000), new Header[]{}, hosts, null, null);
+    }
+}
diff --git a/client/rest/src/test/java/org/elasticsearch/client/documentation/RestClientDocumentation.java b/client/rest/src/test/java/org/elasticsearch/client/documentation/RestClientDocumentation.java
new file mode 100644
index 0000000000000..1bad6b5f6d6fd
--- /dev/null
+++ b/client/rest/src/test/java/org/elasticsearch/client/documentation/RestClientDocumentation.java
@@ -0,0 +1,337 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.documentation;
+
+import org.apache.http.Header;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpHost;
+import org.apache.http.RequestLine;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.CredentialsProvider;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.entity.ContentType;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
+import org.apache.http.impl.nio.reactor.IOReactorConfig;
+import org.apache.http.message.BasicHeader;
+import org.apache.http.nio.entity.NStringEntity;
+import org.apache.http.ssl.SSLContextBuilder;
+import org.apache.http.ssl.SSLContexts;
+import org.apache.http.util.EntityUtils;
+import org.elasticsearch.client.HttpAsyncResponseConsumerFactory;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseListener;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestClientBuilder;
+
+import javax.net.ssl.SSLContext;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.security.KeyStore;
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+
+/**
+ * This class is used to generate the Java low-level REST client documentation.
+ * You need to wrap your code between two tags like:
+ * // tag::example[]
+ * // end::example[]
+ *
+ * Where example is your tag name.
+ *
+ * Then in the documentation, you can extract what is between tag and end tags with
+ * ["source","java",subs="attributes,callouts,macros"]
+ * --------------------------------------------------
+ * include-tagged::{doc-tests}/RestClientDocumentation.java[example]
+ * --------------------------------------------------
+ *
+ * Note that this is not a test class as we are only interested in testing that docs snippets compile. We don't want
+ * to send requests to a node and we don't even have the tools to do it.
+ */
+@SuppressWarnings("unused")
+public class RestClientDocumentation {
+
+    @SuppressWarnings("unused")
+    public void testUsage() throws IOException, InterruptedException {
+
+        //tag::rest-client-init
+        RestClient restClient = RestClient.builder(
+                new HttpHost("localhost", 9200, "http"),
+                new HttpHost("localhost", 9201, "http")).build();
+        //end::rest-client-init
+
+        //tag::rest-client-close
+        restClient.close();
+        //end::rest-client-close
+
+        {
+            //tag::rest-client-init-default-headers
+            RestClientBuilder builder = RestClient.builder(new HttpHost("localhost", 9200, "http"));
+            Header[] defaultHeaders = new Header[]{new BasicHeader("header", "value")};
+            builder.setDefaultHeaders(defaultHeaders); // <1>
+            //end::rest-client-init-default-headers
+        }
+        {
+            //tag::rest-client-init-max-retry-timeout
+            RestClientBuilder builder = RestClient.builder(new HttpHost("localhost", 9200, "http"));
+            builder.setMaxRetryTimeoutMillis(10000); // <1>
+            //end::rest-client-init-max-retry-timeout
+        }
+        {
+            //tag::rest-client-init-failure-listener
+            RestClientBuilder builder = RestClient.builder(new HttpHost("localhost", 9200, "http"));
+            builder.setFailureListener(new RestClient.FailureListener() {
+                @Override
+                public void onFailure(HttpHost host) {
+                    // <1>
+                }
+            });
+            //end::rest-client-init-failure-listener
+        }
+        {
+            //tag::rest-client-init-request-config-callback
+            RestClientBuilder builder = RestClient.builder(new HttpHost("localhost", 9200, "http"));
+            builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
+                @Override
+                public RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
+                    return requestConfigBuilder.setSocketTimeout(10000); // <1>
+                }
+            });
+            //end::rest-client-init-request-config-callback
+        }
+        {
+            //tag::rest-client-init-client-config-callback
+            RestClientBuilder builder = RestClient.builder(new HttpHost("localhost", 9200, "http"));
+            builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
+                @Override
+                public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
+                    return httpClientBuilder.setProxy(new HttpHost("proxy", 9000, "http"));  // <1>
+                }
+            });
+            //end::rest-client-init-client-config-callback
+        }
+
+        {
+            //tag::rest-client-verb-endpoint
+            Response response = restClient.performRequest("GET", "/"); // <1>
+            //end::rest-client-verb-endpoint
+        }
+        {
+            //tag::rest-client-headers
+            Response response = restClient.performRequest("GET", "/", new BasicHeader("header", "value"));
+            //end::rest-client-headers
+        }
+        {
+            //tag::rest-client-verb-endpoint-params
+            Map<String, String> params = Collections.singletonMap("pretty", "true");
+            Response response = restClient.performRequest("GET", "/", params); // <1>
+            //end::rest-client-verb-endpoint-params
+        }
+        {
+            //tag::rest-client-verb-endpoint-params-body
+            Map<String, String> params = Collections.emptyMap();
+            String jsonString = "{" +
+                        "\"user\":\"kimchy\"," +
+                        "\"postDate\":\"2013-01-30\"," +
+                        "\"message\":\"trying out Elasticsearch\"" +
+                    "}";
+            HttpEntity entity = new NStringEntity(jsonString, ContentType.APPLICATION_JSON);
+            Response response = restClient.performRequest("PUT", "/posts/doc/1", params, entity); // <1>
+            //end::rest-client-verb-endpoint-params-body
+        }
+        {
+            //tag::rest-client-response-consumer
+            Map<String, String> params = Collections.emptyMap();
+            HttpAsyncResponseConsumerFactory.HeapBufferedResponseConsumerFactory consumerFactory =
+                    new HttpAsyncResponseConsumerFactory.HeapBufferedResponseConsumerFactory(30 * 1024 * 1024);
+            Response response = restClient.performRequest("GET", "/posts/_search", params, null, consumerFactory); // <1>
+            //end::rest-client-response-consumer
+        }
+        {
+            //tag::rest-client-verb-endpoint-async
+            ResponseListener responseListener = new ResponseListener() {
+                @Override
+                public void onSuccess(Response response) {
+                    // <1>
+                }
+
+                @Override
+                public void onFailure(Exception exception) {
+                    // <2>
+                }
+            };
+            restClient.performRequestAsync("GET", "/", responseListener); // <3>
+            //end::rest-client-verb-endpoint-async
+
+            //tag::rest-client-headers-async
+            Header[] headers = {
+                    new BasicHeader("header1", "value1"),
+                    new BasicHeader("header2", "value2")
+            };
+            restClient.performRequestAsync("GET", "/", responseListener, headers);
+            //end::rest-client-headers-async
+
+            //tag::rest-client-verb-endpoint-params-async
+            Map<String, String> params = Collections.singletonMap("pretty", "true");
+            restClient.performRequestAsync("GET", "/", params, responseListener); // <1>
+            //end::rest-client-verb-endpoint-params-async
+
+            //tag::rest-client-verb-endpoint-params-body-async
+            String jsonString = "{" +
+                    "\"user\":\"kimchy\"," +
+                    "\"postDate\":\"2013-01-30\"," +
+                    "\"message\":\"trying out Elasticsearch\"" +
+                    "}";
+            HttpEntity entity = new NStringEntity(jsonString, ContentType.APPLICATION_JSON);
+            restClient.performRequestAsync("PUT", "/posts/doc/1", params, entity, responseListener); // <1>
+            //end::rest-client-verb-endpoint-params-body-async
+
+            //tag::rest-client-response-consumer-async
+            HttpAsyncResponseConsumerFactory.HeapBufferedResponseConsumerFactory consumerFactory =
+                    new HttpAsyncResponseConsumerFactory.HeapBufferedResponseConsumerFactory(30 * 1024 * 1024);
+            restClient.performRequestAsync("GET", "/posts/_search", params, null, consumerFactory, responseListener); // <1>
+            //end::rest-client-response-consumer-async
+        }
+        {
+            //tag::rest-client-response2
+            Response response = restClient.performRequest("GET", "/");
+            RequestLine requestLine = response.getRequestLine(); // <1>
+            HttpHost host = response.getHost(); // <2>
+            int statusCode = response.getStatusLine().getStatusCode(); // <3>
+            Header[] headers = response.getHeaders(); // <4>
+            String responseBody = EntityUtils.toString(response.getEntity()); // <5>
+            //end::rest-client-response2
+        }
+        {
+            HttpEntity[] documents = new HttpEntity[10];
+            //tag::rest-client-async-example
+            final CountDownLatch latch = new CountDownLatch(documents.length);
+            for (int i = 0; i < documents.length; i++) {
+                restClient.performRequestAsync(
+                        "PUT",
+                        "/posts/doc/" + i,
+                        Collections.<String, String>emptyMap(),
+                        //let's assume that the documents are stored in an HttpEntity array
+                        documents[i],
+                        new ResponseListener() {
+                            @Override
+                            public void onSuccess(Response response) {
+                                // <1>
+                                latch.countDown();
+                            }
+
+                            @Override
+                            public void onFailure(Exception exception) {
+                                // <2>
+                                latch.countDown();
+                            }
+                        }
+                );
+            }
+            latch.await();
+            //end::rest-client-async-example
+        }
+
+    }
+
+    @SuppressWarnings("unused")
+    public void testCommonConfiguration() throws Exception {
+        {
+            //tag::rest-client-config-timeouts
+            RestClientBuilder builder = RestClient.builder(new HttpHost("localhost", 9200))
+                    .setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
+                        @Override
+                        public RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
+                            return requestConfigBuilder.setConnectTimeout(5000)
+                                    .setSocketTimeout(60000);
+                        }
+                    })
+                    .setMaxRetryTimeoutMillis(60000);
+            //end::rest-client-config-timeouts
+        }
+        {
+            //tag::rest-client-config-threads
+            RestClientBuilder builder = RestClient.builder(new HttpHost("localhost", 9200))
+                    .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
+                        @Override
+                        public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
+                            return httpClientBuilder.setDefaultIOReactorConfig(
+                                    IOReactorConfig.custom().setIoThreadCount(1).build());
+                        }
+                    });
+            //end::rest-client-config-threads
+        }
+        {
+            //tag::rest-client-config-basic-auth
+            final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
+            credentialsProvider.setCredentials(AuthScope.ANY,
+                    new UsernamePasswordCredentials("user", "password"));
+
+            RestClientBuilder builder = RestClient.builder(new HttpHost("localhost", 9200))
+                    .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
+                        @Override
+                        public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
+                            return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
+                        }
+                    });
+            //end::rest-client-config-basic-auth
+        }
+        {
+            //tag::rest-client-config-disable-preemptive-auth
+            final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
+            credentialsProvider.setCredentials(AuthScope.ANY,
+                    new UsernamePasswordCredentials("user", "password"));
+
+            RestClientBuilder builder = RestClient.builder(new HttpHost("localhost", 9200))
+                    .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
+                        @Override
+                        public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
+                            httpClientBuilder.disableAuthCaching(); // <1>
+                            return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
+                        }
+                    });
+            //end::rest-client-config-disable-preemptive-auth
+        }
+        {
+            Path keyStorePath = Paths.get("");
+            String keyStorePass = "";
+            //tag::rest-client-config-encrypted-communication
+            KeyStore truststore = KeyStore.getInstance("jks");
+            try (InputStream is = Files.newInputStream(keyStorePath)) {
+                truststore.load(is, keyStorePass.toCharArray());
+            }
+            SSLContextBuilder sslBuilder = SSLContexts.custom().loadTrustMaterial(truststore, null);
+            final SSLContext sslContext = sslBuilder.build();
+            RestClientBuilder builder = RestClient.builder(new HttpHost("localhost", 9200, "https"))
+                    .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
+                        @Override
+                        public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
+                            return httpClientBuilder.setSSLContext(sslContext);
+                        }
+                    });
+            //end::rest-client-config-encrypted-communication
+        }
+    }
+}
diff --git a/client/rest/src/test/resources/testks.jks b/client/rest/src/test/resources/testks.jks
new file mode 100644
index 0000000000000..cd706db5e3834
Binary files /dev/null and b/client/rest/src/test/resources/testks.jks differ
diff --git a/client/sniffer/build.gradle b/client/sniffer/build.gradle
index f35110e4f9e61..6d6541301ae6d 100644
--- a/client/sniffer/build.gradle
+++ b/client/sniffer/build.gradle
@@ -18,7 +18,6 @@
  */
 
 import org.elasticsearch.gradle.precommit.PrecommitTasks
-import org.gradle.api.JavaVersion
 
 apply plugin: 'elasticsearch.build'
 apply plugin: 'ru.vyarus.animalsniffer'
@@ -29,9 +28,18 @@ targetCompatibility = JavaVersion.VERSION_1_7
 sourceCompatibility = JavaVersion.VERSION_1_7
 
 group = 'org.elasticsearch.client'
+archivesBaseName = 'elasticsearch-rest-client-sniffer'
+
+publishing {
+  publications {
+    nebula {
+      artifactId = archivesBaseName
+    }
+  }
+}
 
 dependencies {
-  compile "org.elasticsearch.client:rest:${version}"
+  compile "org.elasticsearch.client:elasticsearch-rest-client:${version}"
   compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
   compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
   compile "commons-codec:commons-codec:${versions.commonscodec}"
@@ -92,4 +100,4 @@ thirdPartyAudit.excludes = [
   //commons-logging provided dependencies
   'javax.servlet.ServletContextEvent',
   'javax.servlet.ServletContextListener'
-]
+]
\ No newline at end of file
diff --git a/client/sniffer/licenses/jackson-core-2.8.1.jar.sha1 b/client/sniffer/licenses/jackson-core-2.8.1.jar.sha1
deleted file mode 100644
index b92131d6fab45..0000000000000
--- a/client/sniffer/licenses/jackson-core-2.8.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-fd13b1c033741d48291315c6370f7d475a42dccf
\ No newline at end of file
diff --git a/client/sniffer/licenses/jackson-core-2.8.6.jar.sha1 b/client/sniffer/licenses/jackson-core-2.8.6.jar.sha1
new file mode 100644
index 0000000000000..af7677d13c28c
--- /dev/null
+++ b/client/sniffer/licenses/jackson-core-2.8.6.jar.sha1
@@ -0,0 +1 @@
+2ef7b1cc34de149600f5e75bc2d5bf40de894e60
\ No newline at end of file
diff --git a/client/sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
index 89a7d9df8e60d..c655babd9ed3d 100644
--- a/client/sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
+++ b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
@@ -27,12 +27,16 @@
 
 import java.io.Closeable;
 import java.io.IOException;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
 import java.util.List;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
 
 /**
  * Class responsible for sniffing nodes from some source (default is elasticsearch itself) and setting them to a provided instance of
@@ -45,6 +49,7 @@
 public class Sniffer implements Closeable {
 
     private static final Log logger = LogFactory.getLog(Sniffer.class);
+    private static final String SNIFFER_THREAD_NAME = "es_rest_client_sniffer";
 
     private final Task task;
 
@@ -79,7 +84,8 @@ private Task(HostsSniffer hostsSniffer, RestClient restClient, long sniffInterva
             this.restClient = restClient;
             this.sniffIntervalMillis = sniffIntervalMillis;
             this.sniffAfterFailureDelayMillis = sniffAfterFailureDelayMillis;
-            this.scheduledExecutorService = Executors.newScheduledThreadPool(1);
+            SnifferThreadFactory threadFactory = new SnifferThreadFactory(SNIFFER_THREAD_NAME);
+            this.scheduledExecutorService = Executors.newScheduledThreadPool(1, threadFactory);
             scheduleNextRun(0);
         }
 
@@ -151,4 +157,34 @@ synchronized void shutdown() {
     public static SnifferBuilder builder(RestClient restClient) {
         return new SnifferBuilder(restClient);
     }
+
+    private static class SnifferThreadFactory implements ThreadFactory {
+
+        private final AtomicInteger threadNumber = new AtomicInteger(1);
+        private final String namePrefix;
+        private final ThreadFactory originalThreadFactory;
+
+        private SnifferThreadFactory(String namePrefix) {
+            this.namePrefix = namePrefix;
+            this.originalThreadFactory = AccessController.doPrivileged(new PrivilegedAction<ThreadFactory>() {
+                @Override
+                public ThreadFactory run() {
+                    return Executors.defaultThreadFactory();
+                }
+            });
+        }
+
+        @Override
+        public Thread newThread(final Runnable r) {
+            return AccessController.doPrivileged(new PrivilegedAction<Thread>() {
+                @Override
+                public Thread run() {
+                    Thread t = originalThreadFactory.newThread(r);
+                    t.setName(namePrefix + "[T#" + threadNumber.getAndIncrement() + "]");
+                    t.setDaemon(true);
+                    return t;
+                }
+            });
+        }
+    }
 }
diff --git a/client/sniffer/src/test/java/org/elasticsearch/client/sniff/ElasticsearchHostsSnifferTests.java b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/ElasticsearchHostsSnifferTests.java
index a926cabb87d7b..aeb0620134b55 100644
--- a/client/sniffer/src/test/java/org/elasticsearch/client/sniff/ElasticsearchHostsSnifferTests.java
+++ b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/ElasticsearchHostsSnifferTests.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.client.sniff;
 
-import com.carrotsearch.randomizedtesting.generators.RandomInts;
+import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
 import com.carrotsearch.randomizedtesting.generators.RandomStrings;
 import com.fasterxml.jackson.core.JsonFactory;
@@ -69,7 +69,7 @@ public class ElasticsearchHostsSnifferTests extends RestClientTestCase {
 
     @Before
     public void startHttpServer() throws IOException {
-        this.sniffRequestTimeout = RandomInts.randomIntBetween(getRandom(), 1000, 10000);
+        this.sniffRequestTimeout = RandomNumbers.randomIntBetween(getRandom(), 1000, 10000);
         this.scheme = RandomPicks.randomFrom(getRandom(), ElasticsearchHostsSniffer.Scheme.values());
         if (rarely()) {
             this.sniffResponse = SniffResponse.buildFailure();
@@ -101,7 +101,7 @@ public void testConstructorValidation() throws IOException {
                 assertEquals(e.getMessage(), "scheme cannot be null");
             }
             try {
-                new ElasticsearchHostsSniffer(restClient, RandomInts.randomIntBetween(getRandom(), Integer.MIN_VALUE, 0),
+                new ElasticsearchHostsSniffer(restClient, RandomNumbers.randomIntBetween(getRandom(), Integer.MIN_VALUE, 0),
                         ElasticsearchHostsSniffer.Scheme.HTTP);
                 fail("should have failed");
             } catch (IllegalArgumentException e) {
@@ -175,7 +175,7 @@ public void handle(HttpExchange httpExchange) throws IOException {
     }
 
     private static SniffResponse buildSniffResponse(ElasticsearchHostsSniffer.Scheme scheme) throws IOException {
-        int numNodes = RandomInts.randomIntBetween(getRandom(), 1, 5);
+        int numNodes = RandomNumbers.randomIntBetween(getRandom(), 1, 5);
         List<HttpHost> hosts = new ArrayList<>(numNodes);
         JsonFactory jsonFactory = new JsonFactory();
         StringWriter writer = new StringWriter();
@@ -205,7 +205,7 @@ private static SniffResponse buildSniffResponse(ElasticsearchHostsSniffer.Scheme
             boolean isHttpEnabled = rarely() == false;
             if (isHttpEnabled) {
                 String host = "host" + i;
-                int port = RandomInts.randomIntBetween(getRandom(), 9200, 9299);
+                int port = RandomNumbers.randomIntBetween(getRandom(), 9200, 9299);
                 HttpHost httpHost = new HttpHost(host, port, scheme.toString());
                 hosts.add(httpHost);
                 generator.writeObjectFieldStart("http");
@@ -228,7 +228,7 @@ private static SniffResponse buildSniffResponse(ElasticsearchHostsSniffer.Scheme
             }
             if (getRandom().nextBoolean()) {
                 String[] roles = {"master", "data", "ingest"};
-                int numRoles = RandomInts.randomIntBetween(getRandom(), 0, 3);
+                int numRoles = RandomNumbers.randomIntBetween(getRandom(), 0, 3);
                 Set<String> nodeRoles = new HashSet<>(numRoles);
                 for (int j = 0; j < numRoles; j++) {
                     String role;
@@ -242,7 +242,7 @@ private static SniffResponse buildSniffResponse(ElasticsearchHostsSniffer.Scheme
                 }
                 generator.writeEndArray();
             }
-            int numAttributes = RandomInts.randomIntBetween(getRandom(), 0, 3);
+            int numAttributes = RandomNumbers.randomIntBetween(getRandom(), 0, 3);
             Map<String, String> attributes = new HashMap<>(numAttributes);
             for (int j = 0; j < numAttributes; j++) {
                 attributes.put("attr" + j, "value" + j);
@@ -291,6 +291,6 @@ static SniffResponse buildResponse(String nodesInfoBody, List<HttpHost> hosts) {
     }
 
     private static int randomErrorResponseCode() {
-        return RandomInts.randomIntBetween(getRandom(), 400, 599);
+        return RandomNumbers.randomIntBetween(getRandom(), 400, 599);
     }
 }
diff --git a/client/sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java
index b0c387d733aad..9a7359e9c7215 100644
--- a/client/sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java
+++ b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.client.sniff;
 
-import com.carrotsearch.randomizedtesting.generators.RandomInts;
+import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
 import org.apache.http.HttpHost;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.client.RestClientTestCase;
@@ -31,7 +31,7 @@
 public class SnifferBuilderTests extends RestClientTestCase {
 
     public void testBuild() throws Exception {
-        int numNodes = RandomInts.randomIntBetween(getRandom(), 1, 5);
+        int numNodes = RandomNumbers.randomIntBetween(getRandom(), 1, 5);
         HttpHost[] hosts = new HttpHost[numNodes];
         for (int i = 0; i < numNodes; i++) {
             hosts[i] = new HttpHost("localhost", 9200 + i);
@@ -46,14 +46,14 @@ public void testBuild() throws Exception {
             }
 
             try {
-                Sniffer.builder(client).setSniffIntervalMillis(RandomInts.randomIntBetween(getRandom(), Integer.MIN_VALUE, 0));
+                Sniffer.builder(client).setSniffIntervalMillis(RandomNumbers.randomIntBetween(getRandom(), Integer.MIN_VALUE, 0));
                 fail("should have failed");
             } catch(IllegalArgumentException e) {
                 assertEquals("sniffIntervalMillis must be greater than 0", e.getMessage());
             }
 
             try {
-                Sniffer.builder(client).setSniffAfterFailureDelayMillis(RandomInts.randomIntBetween(getRandom(), Integer.MIN_VALUE, 0));
+                Sniffer.builder(client).setSniffAfterFailureDelayMillis(RandomNumbers.randomIntBetween(getRandom(), Integer.MIN_VALUE, 0));
                 fail("should have failed");
             } catch(IllegalArgumentException e) {
                 assertEquals("sniffAfterFailureDelayMillis must be greater than 0", e.getMessage());
@@ -74,10 +74,10 @@ public void testBuild() throws Exception {
 
             SnifferBuilder builder = Sniffer.builder(client);
             if (getRandom().nextBoolean()) {
-                builder.setSniffIntervalMillis(RandomInts.randomIntBetween(getRandom(), 1, Integer.MAX_VALUE));
+                builder.setSniffIntervalMillis(RandomNumbers.randomIntBetween(getRandom(), 1, Integer.MAX_VALUE));
             }
             if (getRandom().nextBoolean()) {
-                builder.setSniffAfterFailureDelayMillis(RandomInts.randomIntBetween(getRandom(), 1, Integer.MAX_VALUE));
+                builder.setSniffAfterFailureDelayMillis(RandomNumbers.randomIntBetween(getRandom(), 1, Integer.MAX_VALUE));
             }
             if (getRandom().nextBoolean()) {
                 builder.setHostsSniffer(new MockHostsSniffer());
diff --git a/client/sniffer/src/test/java/org/elasticsearch/client/sniff/documentation/SnifferDocumentation.java b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/documentation/SnifferDocumentation.java
new file mode 100644
index 0000000000000..199632d478f81
--- /dev/null
+++ b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/documentation/SnifferDocumentation.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.sniff.documentation;
+
+import org.apache.http.HttpHost;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.sniff.ElasticsearchHostsSniffer;
+import org.elasticsearch.client.sniff.HostsSniffer;
+import org.elasticsearch.client.sniff.SniffOnFailureListener;
+import org.elasticsearch.client.sniff.Sniffer;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * This class is used to generate the Java low-level REST client documentation.
+ * You need to wrap your code between two tags like:
+ * // tag::example[]
+ * // end::example[]
+ *
+ * Where example is your tag name.
+ *
+ * Then in the documentation, you can extract what is between tag and end tags with
+ * ["source","java",subs="attributes,callouts,macros"]
+ * --------------------------------------------------
+ * include-tagged::{doc-tests}/SnifferDocumentation.java[example]
+ * --------------------------------------------------
+ *
+ * Note that this is not a test class as we are only interested in testing that docs snippets compile. We don't want
+ * to send requests to a node and we don't even have the tools to do it.
+ */
+@SuppressWarnings("unused")
+public class SnifferDocumentation {
+
+    @SuppressWarnings("unused")
+    public void testUsage() throws IOException {
+        {
+            //tag::sniffer-init
+            RestClient restClient = RestClient.builder(
+                    new HttpHost("localhost", 9200, "http"))
+                    .build();
+            Sniffer sniffer = Sniffer.builder(restClient).build();
+            //end::sniffer-init
+
+            //tag::sniffer-close
+            sniffer.close();
+            restClient.close();
+            //end::sniffer-close
+        }
+        {
+            //tag::sniffer-interval
+            RestClient restClient = RestClient.builder(
+                    new HttpHost("localhost", 9200, "http"))
+                    .build();
+            Sniffer sniffer = Sniffer.builder(restClient)
+                    .setSniffIntervalMillis(60000).build();
+            //end::sniffer-interval
+        }
+        {
+            //tag::sniff-on-failure
+            SniffOnFailureListener sniffOnFailureListener = new SniffOnFailureListener();
+            RestClient restClient = RestClient.builder(new HttpHost("localhost", 9200))
+                    .setFailureListener(sniffOnFailureListener) // <1>
+                    .build();
+            Sniffer sniffer = Sniffer.builder(restClient)
+                    .setSniffAfterFailureDelayMillis(30000) // <2>
+                    .build();
+            sniffOnFailureListener.setSniffer(sniffer); // <3>
+            //end::sniff-on-failure
+        }
+        {
+            //tag::sniffer-https
+            RestClient restClient = RestClient.builder(
+                    new HttpHost("localhost", 9200, "http"))
+                    .build();
+            HostsSniffer hostsSniffer = new ElasticsearchHostsSniffer(
+                    restClient,
+                    ElasticsearchHostsSniffer.DEFAULT_SNIFF_REQUEST_TIMEOUT,
+                    ElasticsearchHostsSniffer.Scheme.HTTPS);
+            Sniffer sniffer = Sniffer.builder(restClient)
+                    .setHostsSniffer(hostsSniffer).build();
+            //end::sniffer-https
+        }
+        {
+            //tag::sniff-request-timeout
+            RestClient restClient = RestClient.builder(
+                    new HttpHost("localhost", 9200, "http"))
+                    .build();
+            HostsSniffer hostsSniffer = new ElasticsearchHostsSniffer(
+                    restClient,
+                    TimeUnit.SECONDS.toMillis(5),
+                    ElasticsearchHostsSniffer.Scheme.HTTP);
+            Sniffer sniffer = Sniffer.builder(restClient)
+                    .setHostsSniffer(hostsSniffer).build();
+            //end::sniff-request-timeout
+        }
+        {
+            //tag::custom-hosts-sniffer
+            RestClient restClient = RestClient.builder(
+                    new HttpHost("localhost", 9200, "http"))
+                    .build();
+            HostsSniffer hostsSniffer = new HostsSniffer() {
+                @Override
+                public List<HttpHost> sniffHosts() throws IOException {
+                    return null; // <1>
+                }
+            };
+            Sniffer sniffer = Sniffer.builder(restClient)
+                    .setHostsSniffer(hostsSniffer).build();
+            //end::custom-hosts-sniffer
+        }
+    }
+}
diff --git a/client/test/build.gradle b/client/test/build.gradle
index a7ffe79ac5c08..e57d415e9eaab 100644
--- a/client/test/build.gradle
+++ b/client/test/build.gradle
@@ -26,9 +26,6 @@ apply plugin: 'ru.vyarus.animalsniffer'
 targetCompatibility = JavaVersion.VERSION_1_7
 sourceCompatibility = JavaVersion.VERSION_1_7
 
-install.enabled = false
-uploadArchives.enabled = false
-
 dependencies {
   compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
   compile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
@@ -61,4 +58,4 @@ namingConventions.enabled = false
 
 //we aren't releasing this jar
 thirdPartyAudit.enabled = false
-test.enabled = false
\ No newline at end of file
+test.enabled = false
diff --git a/client/test/src/main/java/org/elasticsearch/client/RestClientTestCase.java b/client/test/src/main/java/org/elasticsearch/client/RestClientTestCase.java
index 4296932a00208..6a2a45ef2813c 100644
--- a/client/test/src/main/java/org/elasticsearch/client/RestClientTestCase.java
+++ b/client/test/src/main/java/org/elasticsearch/client/RestClientTestCase.java
@@ -30,16 +30,19 @@
 import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
 import com.carrotsearch.randomizedtesting.annotations.ThreadLeakZombies;
 import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
-
 import org.apache.http.Header;
-import org.apache.http.message.BasicHeader;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
 @TestMethodProviders({
         JUnit3MethodProvider.class
 })
@@ -53,70 +56,56 @@
 public abstract class RestClientTestCase extends RandomizedTest {
 
     /**
-     * Create the specified number of {@link Header}s.
-     * <p>
-     * Generated header names will be the {@code baseName} plus its index or, rarely, the {@code arrayName} if it's supplied.
+     * Assert that the actual headers are the expected ones given the original default and request headers. Some headers can be ignored,
+     * for instance in case the http client is adding its own automatically.
      *
-     * @param baseName The base name to use for all headers.
-     * @param arrayName The optional ({@code null}able) array name to use randomly.
-     * @param headers The number of headers to create.
-     * @return Never {@code null}.
+     * @param defaultHeaders the default headers set to the REST client instance
+     * @param requestHeaders the request headers sent with a particular request
+     * @param actualHeaders the actual headers as a result of the provided default and request headers
+     * @param ignoreHeaders header keys to be ignored as they are not part of default nor request headers, yet they
+     *                      will be part of the actual ones
      */
-    protected static Header[] generateHeaders(final String baseName, final String arrayName, final int headers) {
-        final Header[] generated = new Header[headers];
-        for (int i = 0; i < headers; i++) {
-            String headerName = baseName + i;
-            if (arrayName != null && rarely()) {
-                headerName = arrayName;
+    protected static void assertHeaders(final Header[] defaultHeaders, final Header[] requestHeaders,
+                                        final Header[] actualHeaders, final Set<String> ignoreHeaders) {
+        final Map<String, List<String>> expectedHeaders = new HashMap<>();
+        final Set<String> requestHeaderKeys = new HashSet<>();
+        for (final Header header : requestHeaders) {
+            final String name = header.getName();
+            addValueToListEntry(expectedHeaders, name, header.getValue());
+            requestHeaderKeys.add(name);
+        }
+        for (final Header defaultHeader : defaultHeaders) {
+            final String name = defaultHeader.getName();
+            if (requestHeaderKeys.contains(name) == false) {
+                addValueToListEntry(expectedHeaders, name, defaultHeader.getValue());
             }
-
-            generated[i] = new BasicHeader(headerName, randomAsciiOfLengthBetween(3, 10));
         }
-        return generated;
+        Set<String> actualIgnoredHeaders = new HashSet<>();
+        for (Header responseHeader : actualHeaders) {
+            final String name = responseHeader.getName();
+            if (ignoreHeaders.contains(name)) {
+                expectedHeaders.remove(name);
+                actualIgnoredHeaders.add(name);
+                continue;
+            }
+            final String value = responseHeader.getValue();
+            final List<String> values = expectedHeaders.get(name);
+            assertNotNull("found response header [" + name + "] that wasn't originally sent: " + value, values);
+            assertTrue("found incorrect response header [" + name + "]: " + value, values.remove(value));
+            if (values.isEmpty()) {
+                expectedHeaders.remove(name);
+            }
+        }
+        assertEquals("some headers meant to be ignored were not part of the actual headers", ignoreHeaders, actualIgnoredHeaders);
+        assertTrue("some headers that were sent weren't returned " + expectedHeaders, expectedHeaders.isEmpty());
     }
 
-    /**
-     * Create a new {@link List} within the {@code map} if none exists for {@code name} or append to the existing list.
-     *
-     * @param map The map to manipulate.
-     * @param name The name to create/append the list for.
-     * @param value The value to add.
-     */
-    private static void createOrAppendList(final Map<String, List<String>> map, final String name, final String value) {
+    private static void addValueToListEntry(final Map<String, List<String>> map, final String name, final String value) {
         List<String> values = map.get(name);
-
         if (values == null) {
             values = new ArrayList<>();
             map.put(name, values);
         }
-
         values.add(value);
     }
-
-    /**
-     * Add the {@code headers} to the {@code map} so that related tests can more easily assert that they exist.
-     * <p>
-     * If both the {@code defaultHeaders} and {@code headers} contain the same {@link Header}, based on its
-     * {@linkplain Header#getName() name}, then this will only use the {@code Header}(s) from {@code headers}.
-     *
-     * @param map The map to build with name/value(s) pairs.
-     * @param defaultHeaders The headers to add to the map representing default headers.
-     * @param headers The headers to add to the map representing request-level headers.
-     * @see #createOrAppendList(Map, String, String)
-     */
-    protected static void addHeaders(final Map<String, List<String>> map, final Header[] defaultHeaders, final Header[] headers) {
-        final Set<String> uniqueHeaders = new HashSet<>();
-        for (final Header header : headers) {
-            final String name = header.getName();
-            createOrAppendList(map, name, header.getValue());
-            uniqueHeaders.add(name);
-        }
-        for (final Header defaultHeader : defaultHeaders) {
-            final String name = defaultHeader.getName();
-            if (uniqueHeaders.contains(name) == false) {
-                createOrAppendList(map, name, defaultHeader.getValue());
-            }
-        }
-    }
-
 }
diff --git a/client/test/src/main/java/org/elasticsearch/client/RestClientTestUtil.java b/client/test/src/main/java/org/elasticsearch/client/RestClientTestUtil.java
index 4d4aa00f4929f..7cda8a71d6178 100644
--- a/client/test/src/main/java/org/elasticsearch/client/RestClientTestUtil.java
+++ b/client/test/src/main/java/org/elasticsearch/client/RestClientTestUtil.java
@@ -19,7 +19,11 @@
 
 package org.elasticsearch.client;
 
+import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+import com.carrotsearch.randomizedtesting.generators.RandomStrings;
+import org.apache.http.Header;
+import org.apache.http.message.BasicHeader;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -55,7 +59,7 @@ static String randomHttpMethod(Random random) {
     }
 
     static int randomStatusCode(Random random) {
-        return RandomPicks.randomFrom(random, ALL_ERROR_STATUS_CODES);
+        return RandomPicks.randomFrom(random, ALL_STATUS_CODES);
     }
 
     static int randomOkStatusCode(Random random) {
@@ -81,4 +85,23 @@ static List<Integer> getAllErrorStatusCodes() {
     static List<Integer> getAllStatusCodes() {
         return ALL_STATUS_CODES;
     }
+
+    /**
+     * Create a random number of {@link org.apache.http.Header}s.
+     * Generated header names will either be the {@code baseName} plus its index, or exactly the provided {@code baseName} so that the
+     * we test also support for multiple headers with same key and different values.
+     */
+    static Header[] randomHeaders(Random random, final String baseName) {
+        int numHeaders = RandomNumbers.randomIntBetween(random, 0, 5);
+        final Header[] headers = new Header[numHeaders];
+        for (int i = 0; i < numHeaders; i++) {
+            String headerName = baseName;
+            //randomly exercise the code path that supports multiple headers with same key
+            if (random.nextBoolean()) {
+                headerName = headerName + i;
+            }
+            headers[i] = new BasicHeader(headerName, RandomStrings.randomAsciiOfLengthBetween(random, 3, 10));
+        }
+        return headers;
+    }
 }
diff --git a/client/transport/build.gradle b/client/transport/build.gradle
index c3dc2d84982b1..91e463f84683e 100644
--- a/client/transport/build.gradle
+++ b/client/transport/build.gradle
@@ -32,6 +32,7 @@ dependencies {
   compile "org.elasticsearch.plugin:reindex-client:${version}"
   compile "org.elasticsearch.plugin:lang-mustache-client:${version}"
   compile "org.elasticsearch.plugin:percolator-client:${version}"
+  compile "org.elasticsearch.plugin:parent-join-client:${version}"
   testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
   testCompile "junit:junit:${versions.junit}"
   testCompile "org.hamcrest:hamcrest-all:${versions.hamcrest}"
@@ -54,4 +55,4 @@ namingConventions {
   testClass = 'com.carrotsearch.randomizedtesting.RandomizedTest'
   //we don't have integration tests
   skipIntegTestInDisguise = true
-}
+}
\ No newline at end of file
diff --git a/client/transport/src/main/java/org/elasticsearch/transport/client/PreBuiltTransportClient.java b/client/transport/src/main/java/org/elasticsearch/transport/client/PreBuiltTransportClient.java
index dbd04079d53b1..08ac14f3ca18b 100644
--- a/client/transport/src/main/java/org/elasticsearch/transport/client/PreBuiltTransportClient.java
+++ b/client/transport/src/main/java/org/elasticsearch/transport/client/PreBuiltTransportClient.java
@@ -22,10 +22,12 @@
 import io.netty.util.ThreadDeathWatcher;
 import io.netty.util.concurrent.GlobalEventExecutor;
 import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.reindex.ReindexPlugin;
+import org.elasticsearch.join.ParentJoinPlugin;
 import org.elasticsearch.percolator.PercolatorPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.mustache.MustachePlugin;
@@ -39,34 +41,93 @@
 import java.util.concurrent.TimeUnit;
 
 /**
- * A builder to create an instance of {@link TransportClient}
- * This class pre-installs the
- * {@link Netty3Plugin},
+ * A builder to create an instance of {@link TransportClient}. This class pre-installs the
+ * {@link Netty3Plugin}
  * {@link Netty4Plugin},
  * {@link ReindexPlugin},
  * {@link PercolatorPlugin},
- * and {@link MustachePlugin}
- * for the client. These plugins are all elasticsearch core modules required.
+ * {@link MustachePlugin},
+ * {@link ParentJoinPlugin}
+ * plugins for the client. These plugins are all the required modules for Elasticsearch.
  */
 @SuppressWarnings({"unchecked","varargs"})
 public class PreBuiltTransportClient extends TransportClient {
 
+    static {
+        // initialize Netty system properties before triggering any Netty class loads
+        initializeNetty();
+    }
+
+    /**
+     * Netty wants to do some unwelcome things like use unsafe and replace a private field, or use a poorly considered buffer recycler. This
+     * method disables these things by default, but can be overridden by setting the corresponding system properties.
+     */
+    private static void initializeNetty() {
+        /*
+         * We disable three pieces of Netty functionality here:
+         *  - we disable Netty from being unsafe
+         *  - we disable Netty from replacing the selector key set
+         *  - we disable Netty from using the recycler
+         *
+         * While permissions are needed to read and set these, the permissions needed here are innocuous and thus should simply be granted
+         * rather than us handling a security exception here.
+         */
+        setSystemPropertyIfUnset("io.netty.noUnsafe", Boolean.toString(true));
+        setSystemPropertyIfUnset("io.netty.noKeySetOptimization", Boolean.toString(true));
+        setSystemPropertyIfUnset("io.netty.recycler.maxCapacityPerThread", Integer.toString(0));
+    }
+
+    @SuppressForbidden(reason = "set system properties to configure Netty")
+    private static void setSystemPropertyIfUnset(final String key, final String value) {
+        final String currentValue = System.getProperty(key);
+        if (currentValue == null) {
+            System.setProperty(key, value);
+        }
+    }
+
     private static final Collection<Class<? extends Plugin>> PRE_INSTALLED_PLUGINS =
-            Collections.unmodifiableList(
-                    Arrays.asList(
-                            Netty3Plugin.class,
-                            Netty4Plugin.class,
-                            ReindexPlugin.class,
-                            PercolatorPlugin.class,
-                            MustachePlugin.class));
+        Collections.unmodifiableList(
+            Arrays.asList(
+                Netty3Plugin.class,
+                Netty4Plugin.class,
+                ReindexPlugin.class,
+                PercolatorPlugin.class,
+                MustachePlugin.class,
+                ParentJoinPlugin.class));
 
+    /**
+     * Creates a new transport client with pre-installed plugins.
+     *
+     * @param settings the settings passed to this transport client
+     * @param plugins  an optional array of additional plugins to run with this client
+     */
     @SafeVarargs
     public PreBuiltTransportClient(Settings settings, Class<? extends Plugin>... plugins) {
         this(settings, Arrays.asList(plugins));
     }
 
+    /**
+     * Creates a new transport client with pre-installed plugins.
+     *
+     * @param settings the settings passed to this transport client
+     * @param plugins  a collection of additional plugins to run with this client
+     */
     public PreBuiltTransportClient(Settings settings, Collection<Class<? extends Plugin>> plugins) {
-        super(settings, Settings.EMPTY, addPlugins(plugins, PRE_INSTALLED_PLUGINS));
+        this(settings, plugins, null);
+    }
+
+    /**
+     * Creates a new transport client with pre-installed plugins.
+     *
+     * @param settings            the settings passed to this transport client
+     * @param plugins             a collection of additional plugins to run with this client
+     * @param hostFailureListener a failure listener that is invoked if a node is disconnected; this can be <code>null</code>
+     */
+    public PreBuiltTransportClient(
+        Settings settings,
+        Collection<Class<? extends Plugin>> plugins,
+        HostFailureListener hostFailureListener) {
+        super(settings, Settings.EMPTY, addPlugins(plugins, PRE_INSTALLED_PLUGINS), hostFailureListener);
     }
 
     @Override
diff --git a/client/transport/src/test/java/org/elasticsearch/transport/client/PreBuiltTransportClientTests.java b/client/transport/src/test/java/org/elasticsearch/transport/client/PreBuiltTransportClientTests.java
index a1d95b68af703..dbcf3571125de 100644
--- a/client/transport/src/test/java/org/elasticsearch/transport/client/PreBuiltTransportClientTests.java
+++ b/client/transport/src/test/java/org/elasticsearch/transport/client/PreBuiltTransportClientTests.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.reindex.ReindexPlugin;
+import org.elasticsearch.join.ParentJoinPlugin;
 import org.elasticsearch.percolator.PercolatorPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.mustache.MustachePlugin;
@@ -41,8 +42,6 @@ public class PreBuiltTransportClientTests extends RandomizedTest {
 
     @Test
     public void testPluginInstalled() {
-        // TODO: remove when Netty 4.1.5 is upgraded to Netty 4.1.6 including https://github.com/netty/netty/pull/5778
-        assumeFalse(Constants.JRE_IS_MINIMUM_JAVA9);
         try (TransportClient client = new PreBuiltTransportClient(Settings.EMPTY)) {
             Settings settings = client.settings();
             assertEquals(Netty4Plugin.NETTY_TRANSPORT_NAME, NetworkModule.HTTP_DEFAULT_TYPE_SETTING.get(settings));
@@ -52,7 +51,8 @@ public void testPluginInstalled() {
 
     @Test
     public void testInstallPluginTwice() {
-        for (Class<? extends Plugin> plugin : Arrays.asList(ReindexPlugin.class, PercolatorPlugin.class, MustachePlugin.class)) {
+        for (Class<? extends Plugin> plugin :
+                Arrays.asList(ParentJoinPlugin.class, ReindexPlugin.class, PercolatorPlugin.class, MustachePlugin.class)) {
             try {
                 new PreBuiltTransportClient(Settings.EMPTY, plugin);
                 fail("exception expected");
diff --git a/core/build.gradle b/core/build.gradle
index 4eab7ed5d5833..a1265e2eeaa11 100644
--- a/core/build.gradle
+++ b/core/build.gradle
@@ -22,7 +22,6 @@ import com.carrotsearch.gradle.junit4.RandomizedTestingTask
 import org.elasticsearch.gradle.BuildPlugin
 
 apply plugin: 'elasticsearch.build'
-apply plugin: 'com.bmuschko.nexus'
 apply plugin: 'nebula.optional-base'
 apply plugin: 'nebula.maven-base-publish'
 apply plugin: 'nebula.maven-scm'
@@ -63,10 +62,7 @@ dependencies {
   compile 'com.carrotsearch:hppc:0.7.1'
 
   // time handling, remove with java 8 time
-  compile 'joda-time:joda-time:2.9.4'
-  // joda 2.0 moved to using volatile fields for datetime
-  // When updating to a new version, make sure to update our copy of BaseDateTime
-  compile 'org.joda:joda-convert:1.2'
+  compile 'joda-time:joda-time:2.9.5'
 
   // json and yaml
   compile "org.yaml:snakeyaml:${versions.snakeyaml}"
@@ -78,19 +74,20 @@ dependencies {
   // percentiles aggregation
   compile 'com.tdunning:t-digest:3.0'
   // precentil ranks aggregation
-  compile 'org.hdrhistogram:HdrHistogram:2.1.6'
+  compile 'org.hdrhistogram:HdrHistogram:2.1.9'
 
   // lucene spatial
   compile "org.locationtech.spatial4j:spatial4j:${versions.spatial4j}", optional
   compile "com.vividsolutions:jts:${versions.jts}", optional
 
   // logging
-  compile "org.apache.logging.log4j:log4j-api:${versions.log4j}", optional
+  compile "org.apache.logging.log4j:log4j-api:${versions.log4j}"
   compile "org.apache.logging.log4j:log4j-core:${versions.log4j}", optional
   // to bridge dependencies that are still on Log4j 1 to Log4j 2
   compile "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}", optional
 
-  compile "net.java.dev.jna:jna:${versions.jna}"
+  // repackaged jna with native bits linked against all elastic supported platforms
+  compile "org.elasticsearch:jna:${versions.jna}"
 
   if (isEclipse == false || project.path == ":core-tests") {
     testCompile("org.elasticsearch.test:framework:${version}") {
@@ -98,6 +95,8 @@ dependencies {
       exclude group: 'org.elasticsearch', module: 'elasticsearch'
     }
   }
+  testCompile 'com.google.jimfs:jimfs:1.1'
+  testCompile 'com.google.guava:guava:18.0'
 }
 
 if (isEclipse) {
@@ -125,8 +124,8 @@ forbiddenPatterns {
 task generateModulesList {
   List<String> modules = project(':modules').subprojects.collect { it.name }
   File modulesFile = new File(buildDir, 'generated-resources/modules.txt')
-  processResources.from(modulesFile) 
-  inputs.property('modules', modules) 
+  processResources.from(modulesFile)
+  inputs.property('modules', modules)
   outputs.file(modulesFile)
   doLast {
     modulesFile.parentFile.mkdirs()
@@ -139,8 +138,8 @@ task generatePluginsList {
     .findAll { it.name.contains('example') == false }
     .collect { it.name }
   File pluginsFile = new File(buildDir, 'generated-resources/plugins.txt')
-  processResources.from(pluginsFile) 
-  inputs.property('plugins', plugins) 
+  processResources.from(pluginsFile)
+  inputs.property('plugins', plugins)
   outputs.file(pluginsFile)
   doLast {
     pluginsFile.parentFile.mkdirs()
@@ -159,8 +158,11 @@ thirdPartyAudit.excludes = [
   'com.fasterxml.jackson.databind.ObjectMapper',
 
   // from log4j
+  'com.conversantmedia.util.concurrent.DisruptorBlockingQueue',
+  'com.conversantmedia.util.concurrent.SpinPolicy',
   'com.fasterxml.jackson.annotation.JsonInclude$Include',
   'com.fasterxml.jackson.databind.DeserializationContext',
+  'com.fasterxml.jackson.databind.DeserializationFeature',
   'com.fasterxml.jackson.databind.JsonMappingException',
   'com.fasterxml.jackson.databind.JsonNode',
   'com.fasterxml.jackson.databind.Module$SetupContext',
@@ -177,6 +179,10 @@ thirdPartyAudit.excludes = [
   'com.fasterxml.jackson.dataformat.xml.JacksonXmlModule',
   'com.fasterxml.jackson.dataformat.xml.XmlMapper',
   'com.fasterxml.jackson.dataformat.xml.util.DefaultXmlPrettyPrinter',
+  'com.fasterxml.jackson.databind.node.JsonNodeFactory',
+  'com.fasterxml.jackson.databind.node.ObjectNode',
+  'org.fusesource.jansi.Ansi',
+  'org.fusesource.jansi.AnsiRenderer$Code',
   'com.lmax.disruptor.BlockingWaitStrategy',
   'com.lmax.disruptor.BusySpinWaitStrategy',
   'com.lmax.disruptor.EventFactory',
@@ -197,11 +203,11 @@ thirdPartyAudit.excludes = [
   'javax.jms.Connection',
   'javax.jms.ConnectionFactory',
   'javax.jms.Destination',
+  'javax.jms.JMSException',
+  'javax.jms.MapMessage',
   'javax.jms.Message',
   'javax.jms.MessageConsumer',
-  'javax.jms.MessageListener',
   'javax.jms.MessageProducer',
-  'javax.jms.ObjectMessage',
   'javax.jms.Session',
   'javax.mail.Authenticator',
   'javax.mail.Message$RecipientType',
@@ -225,10 +231,14 @@ thirdPartyAudit.excludes = [
   'org.apache.commons.compress.utils.IOUtils',
   'org.apache.commons.csv.CSVFormat',
   'org.apache.commons.csv.QuoteMode',
+  'org.apache.kafka.clients.producer.Callback',
   'org.apache.kafka.clients.producer.KafkaProducer',
   'org.apache.kafka.clients.producer.Producer',
   'org.apache.kafka.clients.producer.ProducerRecord',
+  'org.apache.kafka.clients.producer.RecordMetadata',
   'org.codehaus.stax2.XMLStreamWriter2',
+  'org.jctools.queues.MessagePassingQueue$Consumer',
+  'org.jctools.queues.MpscArrayQueue',
   'org.osgi.framework.AdaptPermission',
   'org.osgi.framework.AdminPermission',
   'org.osgi.framework.Bundle',
@@ -237,6 +247,7 @@ thirdPartyAudit.excludes = [
   'org.osgi.framework.BundleEvent',
   'org.osgi.framework.BundleReference',
   'org.osgi.framework.FrameworkUtil',
+  'org.osgi.framework.ServiceRegistration',
   'org.osgi.framework.SynchronousBundleListener',
   'org.osgi.framework.wiring.BundleWire',
   'org.osgi.framework.wiring.BundleWiring',
@@ -245,11 +256,17 @@ thirdPartyAudit.excludes = [
   'org.zeromq.ZMQ',
 
   // from org.locationtech.spatial4j.io.GeoJSONReader (spatial4j)
-  'org.noggit.JSONParser', 
+  'org.noggit.JSONParser',
 ]
 
-// dependency license are currently checked in distribution
-dependencyLicenses.enabled = false
+if (JavaVersion.current() > JavaVersion.VERSION_1_8) {
+  thirdPartyAudit.excludes += ['javax.xml.bind.DatatypeConverter']
+}
+
+dependencyLicenses {
+  mapping from: /lucene-.*/, to: 'lucene'
+  mapping from: /jackson-.*/, to: 'jackson'
+}
 
 if (isEclipse == false || project.path == ":core-tests") {
   task integTest(type: RandomizedTestingTask,
diff --git a/core/licenses/HdrHistogram-2.1.9.jar.sha1 b/core/licenses/HdrHistogram-2.1.9.jar.sha1
new file mode 100644
index 0000000000000..2378df07b2c0c
--- /dev/null
+++ b/core/licenses/HdrHistogram-2.1.9.jar.sha1
@@ -0,0 +1 @@
+e4631ce165eb400edecfa32e03d3f1be53dee754
\ No newline at end of file
diff --git a/distribution/licenses/HdrHistogram-LICENSE.txt b/core/licenses/HdrHistogram-LICENSE.txt
similarity index 100%
rename from distribution/licenses/HdrHistogram-LICENSE.txt
rename to core/licenses/HdrHistogram-LICENSE.txt
diff --git a/distribution/licenses/jopt-simple-NOTICE.txt b/core/licenses/HdrHistogram-NOTICE.txt
similarity index 100%
rename from distribution/licenses/jopt-simple-NOTICE.txt
rename to core/licenses/HdrHistogram-NOTICE.txt
diff --git a/distribution/licenses/apache-log4j-extras-DEPENDENCIES b/core/licenses/apache-log4j-extras-DEPENDENCIES
similarity index 100%
rename from distribution/licenses/apache-log4j-extras-DEPENDENCIES
rename to core/licenses/apache-log4j-extras-DEPENDENCIES
diff --git a/core/licenses/hppc-0.7.1.jar.sha1 b/core/licenses/hppc-0.7.1.jar.sha1
new file mode 100644
index 0000000000000..aa191a6c93b99
--- /dev/null
+++ b/core/licenses/hppc-0.7.1.jar.sha1
@@ -0,0 +1 @@
+8b5057f74ea378c0150a1860874a3ebdcb713767
\ No newline at end of file
diff --git a/distribution/licenses/hppc-LICENSE.txt b/core/licenses/hppc-LICENSE.txt
similarity index 100%
rename from distribution/licenses/hppc-LICENSE.txt
rename to core/licenses/hppc-LICENSE.txt
diff --git a/distribution/licenses/hppc-NOTICE.txt b/core/licenses/hppc-NOTICE.txt
similarity index 100%
rename from distribution/licenses/hppc-NOTICE.txt
rename to core/licenses/hppc-NOTICE.txt
diff --git a/distribution/licenses/jackson-LICENSE b/core/licenses/jackson-LICENSE
similarity index 100%
rename from distribution/licenses/jackson-LICENSE
rename to core/licenses/jackson-LICENSE
diff --git a/distribution/licenses/jackson-NOTICE b/core/licenses/jackson-NOTICE
similarity index 100%
rename from distribution/licenses/jackson-NOTICE
rename to core/licenses/jackson-NOTICE
diff --git a/core/licenses/jackson-core-2.8.6.jar.sha1 b/core/licenses/jackson-core-2.8.6.jar.sha1
new file mode 100644
index 0000000000000..af7677d13c28c
--- /dev/null
+++ b/core/licenses/jackson-core-2.8.6.jar.sha1
@@ -0,0 +1 @@
+2ef7b1cc34de149600f5e75bc2d5bf40de894e60
\ No newline at end of file
diff --git a/core/licenses/jackson-dataformat-cbor-2.8.6.jar.sha1 b/core/licenses/jackson-dataformat-cbor-2.8.6.jar.sha1
new file mode 100644
index 0000000000000..6a2e980235381
--- /dev/null
+++ b/core/licenses/jackson-dataformat-cbor-2.8.6.jar.sha1
@@ -0,0 +1 @@
+b88721371cfa2d7242bb5e52fe70861aa061c050
\ No newline at end of file
diff --git a/core/licenses/jackson-dataformat-smile-2.8.6.jar.sha1 b/core/licenses/jackson-dataformat-smile-2.8.6.jar.sha1
new file mode 100644
index 0000000000000..19be9a2040bed
--- /dev/null
+++ b/core/licenses/jackson-dataformat-smile-2.8.6.jar.sha1
@@ -0,0 +1 @@
+71590ad45cee21249774e2f93e5eca66e446cef3
\ No newline at end of file
diff --git a/core/licenses/jackson-dataformat-yaml-2.8.6.jar.sha1 b/core/licenses/jackson-dataformat-yaml-2.8.6.jar.sha1
new file mode 100644
index 0000000000000..c61dad3bbcdd7
--- /dev/null
+++ b/core/licenses/jackson-dataformat-yaml-2.8.6.jar.sha1
@@ -0,0 +1 @@
+8bd44d50f9a6cdff9c7578ea39d524eb519e35ab
\ No newline at end of file
diff --git a/core/licenses/jna-4.4.0-1.jar.sha1 b/core/licenses/jna-4.4.0-1.jar.sha1
new file mode 100644
index 0000000000000..6b564834b5733
--- /dev/null
+++ b/core/licenses/jna-4.4.0-1.jar.sha1
@@ -0,0 +1 @@
+c9dfcec6f07ee4b1d7a6c09a7eaa9dd4fb6d2c79
\ No newline at end of file
diff --git a/distribution/licenses/jna-LICENSE.txt b/core/licenses/jna-LICENSE.txt
similarity index 100%
rename from distribution/licenses/jna-LICENSE.txt
rename to core/licenses/jna-LICENSE.txt
diff --git a/distribution/licenses/jna-NOTICE.txt b/core/licenses/jna-NOTICE.txt
similarity index 100%
rename from distribution/licenses/jna-NOTICE.txt
rename to core/licenses/jna-NOTICE.txt
diff --git a/core/licenses/joda-time-2.9.5.jar.sha1 b/core/licenses/joda-time-2.9.5.jar.sha1
new file mode 100644
index 0000000000000..ecf1c781556ee
--- /dev/null
+++ b/core/licenses/joda-time-2.9.5.jar.sha1
@@ -0,0 +1 @@
+5f01da7306363fad2028b916f3eab926262de928
\ No newline at end of file
diff --git a/distribution/licenses/joda-time-LICENSE.txt b/core/licenses/joda-time-LICENSE.txt
similarity index 100%
rename from distribution/licenses/joda-time-LICENSE.txt
rename to core/licenses/joda-time-LICENSE.txt
diff --git a/distribution/licenses/joda-time-NOTICE.txt b/core/licenses/joda-time-NOTICE.txt
similarity index 100%
rename from distribution/licenses/joda-time-NOTICE.txt
rename to core/licenses/joda-time-NOTICE.txt
diff --git a/distribution/licenses/jopt-simple-5.0.2.jar.sha1 b/core/licenses/jopt-simple-5.0.2.jar.sha1
similarity index 100%
rename from distribution/licenses/jopt-simple-5.0.2.jar.sha1
rename to core/licenses/jopt-simple-5.0.2.jar.sha1
diff --git a/distribution/licenses/jopt-simple-LICENSE.txt b/core/licenses/jopt-simple-LICENSE.txt
similarity index 100%
rename from distribution/licenses/jopt-simple-LICENSE.txt
rename to core/licenses/jopt-simple-LICENSE.txt
diff --git a/core/licenses/jopt-simple-NOTICE.txt b/core/licenses/jopt-simple-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/core/licenses/jts-1.13.jar.sha1 b/core/licenses/jts-1.13.jar.sha1
new file mode 100644
index 0000000000000..5b9e3902cf493
--- /dev/null
+++ b/core/licenses/jts-1.13.jar.sha1
@@ -0,0 +1 @@
+3ccfb9b60f04d71add996a666ceb8902904fd805
\ No newline at end of file
diff --git a/distribution/licenses/jts-LICENSE.txt b/core/licenses/jts-LICENSE.txt
similarity index 100%
rename from distribution/licenses/jts-LICENSE.txt
rename to core/licenses/jts-LICENSE.txt
diff --git a/distribution/licenses/jts-NOTICE.txt b/core/licenses/jts-NOTICE.txt
similarity index 100%
rename from distribution/licenses/jts-NOTICE.txt
rename to core/licenses/jts-NOTICE.txt
diff --git a/core/licenses/log4j-1.2-api-2.9.1.jar.sha1 b/core/licenses/log4j-1.2-api-2.9.1.jar.sha1
new file mode 100644
index 0000000000000..0b5acc62b7a13
--- /dev/null
+++ b/core/licenses/log4j-1.2-api-2.9.1.jar.sha1
@@ -0,0 +1 @@
+894f96d677880d4ab834a1356f62b875e579caaa
\ No newline at end of file
diff --git a/distribution/licenses/log4j-LICENSE.txt b/core/licenses/log4j-LICENSE.txt
similarity index 100%
rename from distribution/licenses/log4j-LICENSE.txt
rename to core/licenses/log4j-LICENSE.txt
diff --git a/distribution/licenses/log4j-NOTICE.txt b/core/licenses/log4j-NOTICE.txt
similarity index 100%
rename from distribution/licenses/log4j-NOTICE.txt
rename to core/licenses/log4j-NOTICE.txt
diff --git a/core/licenses/log4j-api-2.9.1.jar.sha1 b/core/licenses/log4j-api-2.9.1.jar.sha1
new file mode 100644
index 0000000000000..e1a89fadfed95
--- /dev/null
+++ b/core/licenses/log4j-api-2.9.1.jar.sha1
@@ -0,0 +1 @@
+7a2999229464e7a324aa503c0a52ec0f05efe7bd
\ No newline at end of file
diff --git a/distribution/licenses/log4j-api-LICENSE.txt b/core/licenses/log4j-api-LICENSE.txt
similarity index 100%
rename from distribution/licenses/log4j-api-LICENSE.txt
rename to core/licenses/log4j-api-LICENSE.txt
diff --git a/distribution/licenses/log4j-api-NOTICE.txt b/core/licenses/log4j-api-NOTICE.txt
similarity index 100%
rename from distribution/licenses/log4j-api-NOTICE.txt
rename to core/licenses/log4j-api-NOTICE.txt
diff --git a/core/licenses/log4j-core-2.9.1.jar.sha1 b/core/licenses/log4j-core-2.9.1.jar.sha1
new file mode 100644
index 0000000000000..990ea322a7613
--- /dev/null
+++ b/core/licenses/log4j-core-2.9.1.jar.sha1
@@ -0,0 +1 @@
+c041978c686866ee8534f538c6220238db3bb6be
\ No newline at end of file
diff --git a/distribution/licenses/log4j-core-LICENSE.txt b/core/licenses/log4j-core-LICENSE.txt
similarity index 100%
rename from distribution/licenses/log4j-core-LICENSE.txt
rename to core/licenses/log4j-core-LICENSE.txt
diff --git a/distribution/licenses/log4j-core-NOTICE.txt b/core/licenses/log4j-core-NOTICE.txt
similarity index 100%
rename from distribution/licenses/log4j-core-NOTICE.txt
rename to core/licenses/log4j-core-NOTICE.txt
diff --git a/distribution/licenses/lucene-LICENSE.txt b/core/licenses/lucene-LICENSE.txt
similarity index 100%
rename from distribution/licenses/lucene-LICENSE.txt
rename to core/licenses/lucene-LICENSE.txt
diff --git a/distribution/licenses/lucene-NOTICE.txt b/core/licenses/lucene-NOTICE.txt
similarity index 100%
rename from distribution/licenses/lucene-NOTICE.txt
rename to core/licenses/lucene-NOTICE.txt
diff --git a/core/licenses/lucene-analyzers-common-6.6.1.jar.sha1 b/core/licenses/lucene-analyzers-common-6.6.1.jar.sha1
new file mode 100644
index 0000000000000..b05e43efdf51d
--- /dev/null
+++ b/core/licenses/lucene-analyzers-common-6.6.1.jar.sha1
@@ -0,0 +1 @@
+52cb2bbc52221d33972faacf67e5da0ab92956bd
\ No newline at end of file
diff --git a/core/licenses/lucene-backward-codecs-6.6.1.jar.sha1 b/core/licenses/lucene-backward-codecs-6.6.1.jar.sha1
new file mode 100644
index 0000000000000..b570be50213a3
--- /dev/null
+++ b/core/licenses/lucene-backward-codecs-6.6.1.jar.sha1
@@ -0,0 +1 @@
+4ad390d10b0290af6dac83a519956b98b1fd18f0
\ No newline at end of file
diff --git a/core/licenses/lucene-core-6.6.1.jar.sha1 b/core/licenses/lucene-core-6.6.1.jar.sha1
new file mode 100644
index 0000000000000..d81efa2861d48
--- /dev/null
+++ b/core/licenses/lucene-core-6.6.1.jar.sha1
@@ -0,0 +1 @@
+b51e719d781e6ec2dbf6d6eacc20a9c2df30269a
\ No newline at end of file
diff --git a/core/licenses/lucene-grouping-6.6.1.jar.sha1 b/core/licenses/lucene-grouping-6.6.1.jar.sha1
new file mode 100644
index 0000000000000..3aad8585184a9
--- /dev/null
+++ b/core/licenses/lucene-grouping-6.6.1.jar.sha1
@@ -0,0 +1 @@
+fa9069bd2b75b219a295d15394607350195b0665
\ No newline at end of file
diff --git a/core/licenses/lucene-highlighter-6.6.1.jar.sha1 b/core/licenses/lucene-highlighter-6.6.1.jar.sha1
new file mode 100644
index 0000000000000..422105a06bf05
--- /dev/null
+++ b/core/licenses/lucene-highlighter-6.6.1.jar.sha1
@@ -0,0 +1 @@
+6cc18a6e4a60b8fca62fcfaf8b9fc3ff6bf1864d
\ No newline at end of file
diff --git a/core/licenses/lucene-join-6.6.1.jar.sha1 b/core/licenses/lucene-join-6.6.1.jar.sha1
new file mode 100644
index 0000000000000..0752b32b10807
--- /dev/null
+++ b/core/licenses/lucene-join-6.6.1.jar.sha1
@@ -0,0 +1 @@
+355dc2046a1574cf23d325171372531e687a72cb
\ No newline at end of file
diff --git a/core/licenses/lucene-memory-6.6.1.jar.sha1 b/core/licenses/lucene-memory-6.6.1.jar.sha1
new file mode 100644
index 0000000000000..f41087ddf8af9
--- /dev/null
+++ b/core/licenses/lucene-memory-6.6.1.jar.sha1
@@ -0,0 +1 @@
+4df5d3018bf7853b4f44eada0c3d823f25800fc3
\ No newline at end of file
diff --git a/core/licenses/lucene-misc-6.6.1.jar.sha1 b/core/licenses/lucene-misc-6.6.1.jar.sha1
new file mode 100644
index 0000000000000..8d53bf1a1a3c3
--- /dev/null
+++ b/core/licenses/lucene-misc-6.6.1.jar.sha1
@@ -0,0 +1 @@
+4a434f20c15a1e651ba9d3db1167fec695b557d4
\ No newline at end of file
diff --git a/core/licenses/lucene-queries-6.6.1.jar.sha1 b/core/licenses/lucene-queries-6.6.1.jar.sha1
new file mode 100644
index 0000000000000..8b675bba89333
--- /dev/null
+++ b/core/licenses/lucene-queries-6.6.1.jar.sha1
@@ -0,0 +1 @@
+e138ad9807b029ca3ee0276eeb0257812c9c9179
\ No newline at end of file
diff --git a/core/licenses/lucene-queryparser-6.6.1.jar.sha1 b/core/licenses/lucene-queryparser-6.6.1.jar.sha1
new file mode 100644
index 0000000000000..4ddb0f20e6c9b
--- /dev/null
+++ b/core/licenses/lucene-queryparser-6.6.1.jar.sha1
@@ -0,0 +1 @@
+f80e27fee9595ced0276e3caa53b6d12cc779b0e
\ No newline at end of file
diff --git a/core/licenses/lucene-sandbox-6.6.1.jar.sha1 b/core/licenses/lucene-sandbox-6.6.1.jar.sha1
new file mode 100644
index 0000000000000..bd79ca46cf3af
--- /dev/null
+++ b/core/licenses/lucene-sandbox-6.6.1.jar.sha1
@@ -0,0 +1 @@
+3a4d147697dfb27b3a0f01f67c0b61175c14b011
\ No newline at end of file
diff --git a/core/licenses/lucene-spatial-6.6.1.jar.sha1 b/core/licenses/lucene-spatial-6.6.1.jar.sha1
new file mode 100644
index 0000000000000..7f4593c67e80b
--- /dev/null
+++ b/core/licenses/lucene-spatial-6.6.1.jar.sha1
@@ -0,0 +1 @@
+a83dc0e68cc3aeb8835610022f8d2cff34096d40
\ No newline at end of file
diff --git a/core/licenses/lucene-spatial-extras-6.6.1.jar.sha1 b/core/licenses/lucene-spatial-extras-6.6.1.jar.sha1
new file mode 100644
index 0000000000000..e0846a30ac504
--- /dev/null
+++ b/core/licenses/lucene-spatial-extras-6.6.1.jar.sha1
@@ -0,0 +1 @@
+c1a3c9892f1d57b14adc4bcf30509c2bec2ebafb
\ No newline at end of file
diff --git a/core/licenses/lucene-spatial3d-6.6.1.jar.sha1 b/core/licenses/lucene-spatial3d-6.6.1.jar.sha1
new file mode 100644
index 0000000000000..f2e7580b53fef
--- /dev/null
+++ b/core/licenses/lucene-spatial3d-6.6.1.jar.sha1
@@ -0,0 +1 @@
+87ed4ef7f3b18bf106c6f780eea39c88c9a39ad1
\ No newline at end of file
diff --git a/core/licenses/lucene-suggest-6.6.1.jar.sha1 b/core/licenses/lucene-suggest-6.6.1.jar.sha1
new file mode 100644
index 0000000000000..d1d442f41baf2
--- /dev/null
+++ b/core/licenses/lucene-suggest-6.6.1.jar.sha1
@@ -0,0 +1 @@
+9c74240a249fd7c35fdeb6379ed2a929fb7c8acb
\ No newline at end of file
diff --git a/distribution/licenses/securesm-1.1.jar.sha1 b/core/licenses/securesm-1.1.jar.sha1
similarity index 100%
rename from distribution/licenses/securesm-1.1.jar.sha1
rename to core/licenses/securesm-1.1.jar.sha1
diff --git a/modules/transport-netty4/licenses/netty-buffer-LICENSE.txt b/core/licenses/securesm-LICENSE.txt
similarity index 100%
rename from modules/transport-netty4/licenses/netty-buffer-LICENSE.txt
rename to core/licenses/securesm-LICENSE.txt
diff --git a/distribution/licenses/securesm-NOTICE.txt b/core/licenses/securesm-NOTICE.txt
similarity index 100%
rename from distribution/licenses/securesm-NOTICE.txt
rename to core/licenses/securesm-NOTICE.txt
diff --git a/distribution/licenses/snakeyaml-1.15.jar.sha1 b/core/licenses/snakeyaml-1.15.jar.sha1
similarity index 100%
rename from distribution/licenses/snakeyaml-1.15.jar.sha1
rename to core/licenses/snakeyaml-1.15.jar.sha1
diff --git a/distribution/licenses/snakeyaml-LICENSE.txt b/core/licenses/snakeyaml-LICENSE.txt
similarity index 100%
rename from distribution/licenses/snakeyaml-LICENSE.txt
rename to core/licenses/snakeyaml-LICENSE.txt
diff --git a/distribution/licenses/snakeyaml-NOTICE.txt b/core/licenses/snakeyaml-NOTICE.txt
similarity index 100%
rename from distribution/licenses/snakeyaml-NOTICE.txt
rename to core/licenses/snakeyaml-NOTICE.txt
diff --git a/distribution/licenses/spatial4j-0.6.jar.sha1 b/core/licenses/spatial4j-0.6.jar.sha1
similarity index 100%
rename from distribution/licenses/spatial4j-0.6.jar.sha1
rename to core/licenses/spatial4j-0.6.jar.sha1
diff --git a/distribution/licenses/spatial4j-ABOUT.txt b/core/licenses/spatial4j-ABOUT.txt
similarity index 100%
rename from distribution/licenses/spatial4j-ABOUT.txt
rename to core/licenses/spatial4j-ABOUT.txt
diff --git a/modules/transport-netty4/licenses/netty-codec-LICENSE.txt b/core/licenses/spatial4j-LICENSE.txt
similarity index 100%
rename from modules/transport-netty4/licenses/netty-codec-LICENSE.txt
rename to core/licenses/spatial4j-LICENSE.txt
diff --git a/distribution/licenses/spatial4j-NOTICE.txt b/core/licenses/spatial4j-NOTICE.txt
similarity index 100%
rename from distribution/licenses/spatial4j-NOTICE.txt
rename to core/licenses/spatial4j-NOTICE.txt
diff --git a/core/licenses/t-digest-3.0.jar.sha1 b/core/licenses/t-digest-3.0.jar.sha1
new file mode 100644
index 0000000000000..ce2f2e2f04098
--- /dev/null
+++ b/core/licenses/t-digest-3.0.jar.sha1
@@ -0,0 +1 @@
+84ccf145ac2215e6bfa63baa3101c0af41017cfc
\ No newline at end of file
diff --git a/distribution/licenses/t-digest-LICENSE.txt b/core/licenses/t-digest-LICENSE.txt
similarity index 100%
rename from distribution/licenses/t-digest-LICENSE.txt
rename to core/licenses/t-digest-LICENSE.txt
diff --git a/distribution/licenses/t-digest-NOTICE.txt b/core/licenses/t-digest-NOTICE.txt
similarity index 100%
rename from distribution/licenses/t-digest-NOTICE.txt
rename to core/licenses/t-digest-NOTICE.txt
diff --git a/core/src/main/java/org/apache/logging/log4j/core/impl/ThrowableProxy.java b/core/src/main/java/org/apache/logging/log4j/core/impl/ThrowableProxy.java
deleted file mode 100644
index 37ab0a1539154..0000000000000
--- a/core/src/main/java/org/apache/logging/log4j/core/impl/ThrowableProxy.java
+++ /dev/null
@@ -1,665 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache license, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the license for the specific language governing permissions and
- * limitations under the license.
- */
-
-package org.apache.logging.log4j.core.impl;
-
-import java.io.Serializable;
-import java.net.URL;
-import java.security.CodeSource;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.Stack;
-
-import org.apache.logging.log4j.core.util.Loader;
-import org.apache.logging.log4j.status.StatusLogger;
-import org.apache.logging.log4j.util.ReflectionUtil;
-import org.apache.logging.log4j.util.Strings;
-
-/**
- * Wraps a Throwable to add packaging information about each stack trace element.
- *
- * <p>
- * A proxy is used to represent a throwable that may not exist in a different class loader or JVM. When an application
- * deserializes a ThrowableProxy, the throwable may not be set, but the throwable's information is preserved in other
- * fields of the proxy like the message and stack trace.
- * </p>
- *
- * <p>
- * TODO: Move this class to org.apache.logging.log4j.core because it is used from LogEvent.
- * </p>
- * <p>
- * TODO: Deserialize: Try to rebuild Throwable if the target exception is in this class loader?
- * </p>
- */
-public class ThrowableProxy implements Serializable {
-
-    private static final String CAUSED_BY_LABEL = "Caused by: ";
-    private static final String SUPPRESSED_LABEL = "Suppressed: ";
-    private static final String WRAPPED_BY_LABEL = "Wrapped by: ";
-
-    /**
-     * Cached StackTracePackageElement and ClassLoader.
-     * <p>
-     * Consider this class private.
-     * </p>
-     */
-    static class CacheEntry {
-        private final ExtendedClassInfo element;
-        private final ClassLoader loader;
-
-        public CacheEntry(final ExtendedClassInfo element, final ClassLoader loader) {
-            this.element = element;
-            this.loader = loader;
-        }
-    }
-
-    private static final ThrowableProxy[] EMPTY_THROWABLE_PROXY_ARRAY = new ThrowableProxy[0];
-
-    private static final char EOL = '\n';
-
-    private static final long serialVersionUID = -2752771578252251910L;
-
-    private final ThrowableProxy causeProxy;
-
-    private int commonElementCount;
-
-    private final ExtendedStackTraceElement[] extendedStackTrace;
-
-    private final String localizedMessage;
-
-    private final String message;
-
-    private final String name;
-
-    private final ThrowableProxy[] suppressedProxies;
-
-    private final transient Throwable throwable;
-
-    /**
-     * For JSON and XML IO via Jackson.
-     */
-    @SuppressWarnings("unused")
-    private ThrowableProxy() {
-        this.throwable = null;
-        this.name = null;
-        this.extendedStackTrace = null;
-        this.causeProxy = null;
-        this.message = null;
-        this.localizedMessage = null;
-        this.suppressedProxies = EMPTY_THROWABLE_PROXY_ARRAY;
-    }
-
-    /**
-     * Constructs the wrapper for the Throwable that includes packaging data.
-     *
-     * @param throwable
-     *        The Throwable to wrap, must not be null.
-     */
-    public ThrowableProxy(final Throwable throwable) {
-        this(throwable, null);
-    }
-
-    /**
-     * Constructs the wrapper for the Throwable that includes packaging data.
-     *
-     * @param throwable
-     *        The Throwable to wrap, must not be null.
-     * @param visited
-     *        The set of visited suppressed exceptions.
-     */
-    private ThrowableProxy(final Throwable throwable, final Set<Throwable> visited) {
-        this.throwable = throwable;
-        this.name = throwable.getClass().getName();
-        this.message = throwable.getMessage();
-        this.localizedMessage = throwable.getLocalizedMessage();
-        final Map<String, CacheEntry> map = new HashMap<>();
-        final Stack<Class<?>> stack = ReflectionUtil.getCurrentStackTrace();
-        this.extendedStackTrace = this.toExtendedStackTrace(stack, map, null, throwable.getStackTrace());
-        final Throwable throwableCause = throwable.getCause();
-        final Set<Throwable> causeVisited = new HashSet<>(1);
-        this.causeProxy = throwableCause == null ? null : new ThrowableProxy(throwable, stack, map, throwableCause, visited, causeVisited);
-        this.suppressedProxies = this.toSuppressedProxies(throwable, visited);
-    }
-
-    /**
-     * Constructs the wrapper for a Throwable that is referenced as the cause by another Throwable.
-     *
-     * @param parent
-     *        The Throwable referencing this Throwable.
-     * @param stack
-     *        The Class stack.
-     * @param map
-     *        The cache containing the packaging data.
-     * @param cause
-     *        The Throwable to wrap.
-     * @param suppressedVisited TODO
-     * @param causeVisited TODO
-     */
-    private ThrowableProxy(final Throwable parent, final Stack<Class<?>> stack, final Map<String, CacheEntry> map,
-                           final Throwable cause, final Set<Throwable> suppressedVisited, final Set<Throwable> causeVisited) {
-        causeVisited.add(cause);
-        this.throwable = cause;
-        this.name = cause.getClass().getName();
-        this.message = this.throwable.getMessage();
-        this.localizedMessage = this.throwable.getLocalizedMessage();
-        this.extendedStackTrace = this.toExtendedStackTrace(stack, map, parent.getStackTrace(), cause.getStackTrace());
-        final Throwable causeCause = cause.getCause();
-        this.causeProxy = causeCause == null || causeVisited.contains(causeCause) ? null : new ThrowableProxy(parent,
-            stack, map, causeCause, suppressedVisited, causeVisited);
-        this.suppressedProxies = this.toSuppressedProxies(cause, suppressedVisited);
-    }
-
-    @Override
-    public boolean equals(final Object obj) {
-        if (this == obj) {
-            return true;
-        }
-        if (obj == null) {
-            return false;
-        }
-        if (this.getClass() != obj.getClass()) {
-            return false;
-        }
-        final ThrowableProxy other = (ThrowableProxy) obj;
-        if (this.causeProxy == null) {
-            if (other.causeProxy != null) {
-                return false;
-            }
-        } else if (!this.causeProxy.equals(other.causeProxy)) {
-            return false;
-        }
-        if (this.commonElementCount != other.commonElementCount) {
-            return false;
-        }
-        if (this.name == null) {
-            if (other.name != null) {
-                return false;
-            }
-        } else if (!this.name.equals(other.name)) {
-            return false;
-        }
-        if (!Arrays.equals(this.extendedStackTrace, other.extendedStackTrace)) {
-            return false;
-        }
-        if (!Arrays.equals(this.suppressedProxies, other.suppressedProxies)) {
-            return false;
-        }
-        return true;
-    }
-
-    private void formatCause(final StringBuilder sb, final String prefix, final ThrowableProxy cause, final List<String> ignorePackages) {
-        formatThrowableProxy(sb, prefix, CAUSED_BY_LABEL, cause, ignorePackages);
-    }
-
-    private void formatThrowableProxy(final StringBuilder sb, final String prefix, final String causeLabel,
-                                      final ThrowableProxy throwableProxy, final List<String> ignorePackages) {
-        if (throwableProxy == null) {
-            return;
-        }
-        sb.append(prefix).append(causeLabel).append(throwableProxy).append(EOL);
-        this.formatElements(sb, prefix, throwableProxy.commonElementCount,
-            throwableProxy.getStackTrace(), throwableProxy.extendedStackTrace, ignorePackages);
-        this.formatSuppressed(sb, prefix + "\t", throwableProxy.suppressedProxies, ignorePackages);
-        this.formatCause(sb, prefix, throwableProxy.causeProxy, ignorePackages);
-    }
-
-    private void formatSuppressed(final StringBuilder sb, final String prefix, final ThrowableProxy[] suppressedProxies,
-                                  final List<String> ignorePackages) {
-        if (suppressedProxies == null) {
-            return;
-        }
-        for (final ThrowableProxy suppressedProxy : suppressedProxies) {
-            final ThrowableProxy cause = suppressedProxy;
-            formatThrowableProxy(sb, prefix, SUPPRESSED_LABEL, cause, ignorePackages);
-        }
-    }
-
-    private void formatElements(final StringBuilder sb, final String prefix, final int commonCount,
-                                final StackTraceElement[] causedTrace, final ExtendedStackTraceElement[] extStackTrace,
-                                final List<String> ignorePackages) {
-        if (ignorePackages == null || ignorePackages.isEmpty()) {
-            for (final ExtendedStackTraceElement element : extStackTrace) {
-                this.formatEntry(element, sb, prefix);
-            }
-        } else {
-            int count = 0;
-            for (int i = 0; i < extStackTrace.length; ++i) {
-                if (!this.ignoreElement(causedTrace[i], ignorePackages)) {
-                    if (count > 0) {
-                        appendSuppressedCount(sb, prefix, count);
-                        count = 0;
-                    }
-                    this.formatEntry(extStackTrace[i], sb, prefix);
-                } else {
-                    ++count;
-                }
-            }
-            if (count > 0) {
-                appendSuppressedCount(sb, prefix, count);
-            }
-        }
-        if (commonCount != 0) {
-            sb.append(prefix).append("\t... ").append(commonCount).append(" more").append(EOL);
-        }
-    }
-
-    private void appendSuppressedCount(final StringBuilder sb, final String prefix, final int count) {
-        sb.append(prefix);
-        if (count == 1) {
-            sb.append("\t....").append(EOL);
-        } else {
-            sb.append("\t... suppressed ").append(count).append(" lines").append(EOL);
-        }
-    }
-
-    private void formatEntry(final ExtendedStackTraceElement extStackTraceElement, final StringBuilder sb, final String prefix) {
-        sb.append(prefix);
-        sb.append("\tat ");
-        sb.append(extStackTraceElement);
-        sb.append(EOL);
-    }
-
-    /**
-     * Formats the specified Throwable.
-     *
-     * @param sb
-     *        StringBuilder to contain the formatted Throwable.
-     * @param cause
-     *        The Throwable to format.
-     */
-    public void formatWrapper(final StringBuilder sb, final ThrowableProxy cause) {
-        this.formatWrapper(sb, cause, null);
-    }
-
-    /**
-     * Formats the specified Throwable.
-     *
-     * @param sb
-     *        StringBuilder to contain the formatted Throwable.
-     * @param cause
-     *        The Throwable to format.
-     * @param packages
-     *        The List of packages to be suppressed from the trace.
-     */
-    @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
-    public void formatWrapper(final StringBuilder sb, final ThrowableProxy cause, final List<String> packages) {
-        final Throwable caused = cause.getCauseProxy() != null ? cause.getCauseProxy().getThrowable() : null;
-        if (caused != null) {
-            this.formatWrapper(sb, cause.causeProxy);
-            sb.append(WRAPPED_BY_LABEL);
-        }
-        sb.append(cause).append(EOL);
-        this.formatElements(sb, "", cause.commonElementCount,
-            cause.getThrowable().getStackTrace(), cause.extendedStackTrace, packages);
-    }
-
-    public ThrowableProxy getCauseProxy() {
-        return this.causeProxy;
-    }
-
-    /**
-     * Format the Throwable that is the cause of this Throwable.
-     *
-     * @return The formatted Throwable that caused this Throwable.
-     */
-    public String getCauseStackTraceAsString() {
-        return this.getCauseStackTraceAsString(null);
-    }
-
-    /**
-     * Format the Throwable that is the cause of this Throwable.
-     *
-     * @param packages
-     *        The List of packages to be suppressed from the trace.
-     * @return The formatted Throwable that caused this Throwable.
-     */
-    public String getCauseStackTraceAsString(final List<String> packages) {
-        final StringBuilder sb = new StringBuilder();
-        if (this.causeProxy != null) {
-            this.formatWrapper(sb, this.causeProxy);
-            sb.append(WRAPPED_BY_LABEL);
-        }
-        sb.append(this.toString());
-        sb.append(EOL);
-        this.formatElements(sb, "", 0, this.throwable.getStackTrace(), this.extendedStackTrace, packages);
-        return sb.toString();
-    }
-
-    /**
-     * Return the number of elements that are being omitted because they are common with the parent Throwable's stack
-     * trace.
-     *
-     * @return The number of elements omitted from the stack trace.
-     */
-    public int getCommonElementCount() {
-        return this.commonElementCount;
-    }
-
-    /**
-     * Gets the stack trace including packaging information.
-     *
-     * @return The stack trace including packaging information.
-     */
-    public ExtendedStackTraceElement[] getExtendedStackTrace() {
-        return this.extendedStackTrace;
-    }
-
-    /**
-     * Format the stack trace including packaging information.
-     *
-     * @return The formatted stack trace including packaging information.
-     */
-    public String getExtendedStackTraceAsString() {
-        return this.getExtendedStackTraceAsString(null);
-    }
-
-    /**
-     * Format the stack trace including packaging information.
-     *
-     * @param ignorePackages
-     *        List of packages to be ignored in the trace.
-     * @return The formatted stack trace including packaging information.
-     */
-    public String getExtendedStackTraceAsString(final List<String> ignorePackages) {
-        final StringBuilder sb = new StringBuilder(this.name);
-        final String msg = this.message;
-        if (msg != null) {
-            sb.append(": ").append(msg);
-        }
-        sb.append(EOL);
-        final StackTraceElement[] causedTrace = this.throwable != null ? this.throwable.getStackTrace() : null;
-        this.formatElements(sb, "", 0, causedTrace, this.extendedStackTrace, ignorePackages);
-        this.formatSuppressed(sb, "\t", this.suppressedProxies, ignorePackages);
-        this.formatCause(sb, "", this.causeProxy, ignorePackages);
-        return sb.toString();
-    }
-
-    public String getLocalizedMessage() {
-        return this.localizedMessage;
-    }
-
-    public String getMessage() {
-        return this.message;
-    }
-
-    /**
-     * Return the FQCN of the Throwable.
-     *
-     * @return The FQCN of the Throwable.
-     */
-    public String getName() {
-        return this.name;
-    }
-
-    public StackTraceElement[] getStackTrace() {
-        return this.throwable == null ? null : this.throwable.getStackTrace();
-    }
-
-    /**
-     * Gets proxies for suppressed exceptions.
-     *
-     * @return proxies for suppressed exceptions.
-     */
-    public ThrowableProxy[] getSuppressedProxies() {
-        return this.suppressedProxies;
-    }
-
-    /**
-     * Format the suppressed Throwables.
-     *
-     * @return The formatted suppressed Throwables.
-     */
-    public String getSuppressedStackTrace() {
-        final ThrowableProxy[] suppressed = this.getSuppressedProxies();
-        if (suppressed == null || suppressed.length == 0) {
-            return Strings.EMPTY;
-        }
-        final StringBuilder sb = new StringBuilder("Suppressed Stack Trace Elements:").append(EOL);
-        for (final ThrowableProxy proxy : suppressed) {
-            sb.append(proxy.getExtendedStackTraceAsString());
-        }
-        return sb.toString();
-    }
-
-    /**
-     * The throwable or null if this object is deserialized from XML or JSON.
-     *
-     * @return The throwable or null if this object is deserialized from XML or JSON.
-     */
-    public Throwable getThrowable() {
-        return this.throwable;
-    }
-
-    @Override
-    public int hashCode() {
-        final int prime = 31;
-        int result = 1;
-        result = prime * result + (this.causeProxy == null ? 0 : this.causeProxy.hashCode());
-        result = prime * result + this.commonElementCount;
-        result = prime * result + (this.extendedStackTrace == null ? 0 : Arrays.hashCode(this.extendedStackTrace));
-        result = prime * result + (this.suppressedProxies == null ? 0 : Arrays.hashCode(this.suppressedProxies));
-        result = prime * result + (this.name == null ? 0 : this.name.hashCode());
-        return result;
-    }
-
-    private boolean ignoreElement(final StackTraceElement element, final List<String> ignorePackages) {
-        final String className = element.getClassName();
-        for (final String pkg : ignorePackages) {
-            if (className.startsWith(pkg)) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-    /**
-     * Loads classes not located via Reflection.getCallerClass.
-     *
-     * @param lastLoader
-     *        The ClassLoader that loaded the Class that called this Class.
-     * @param className
-     *        The name of the Class.
-     * @return The Class object for the Class or null if it could not be located.
-     */
-    private Class<?> loadClass(final ClassLoader lastLoader, final String className) {
-        // XXX: this is overly complicated
-        Class<?> clazz;
-        if (lastLoader != null) {
-            try {
-                clazz = Loader.initializeClass(className, lastLoader);
-                if (clazz != null) {
-                    return clazz;
-                }
-            } catch (final Throwable ignore) {
-                // Ignore exception.
-            }
-        }
-        try {
-            clazz = Loader.loadClass(className);
-        } catch (final ClassNotFoundException ignored) {
-            return initializeClass(className);
-        } catch (final NoClassDefFoundError ignored) {
-            return initializeClass(className);
-        } catch (final SecurityException ignored) {
-            return initializeClass(className);
-        }
-        return clazz;
-    }
-
-    private Class<?> initializeClass(final String className) {
-        try {
-            return Loader.initializeClass(className, this.getClass().getClassLoader());
-        } catch (final ClassNotFoundException ignore) {
-            return null;
-        } catch (final NoClassDefFoundError ignore) {
-            return null;
-        } catch (final SecurityException ignore) {
-            return null;
-        }
-    }
-
-    /**
-     * Construct the CacheEntry from the Class's information.
-     *
-     * @param stackTraceElement
-     *        The stack trace element
-     * @param callerClass
-     *        The Class.
-     * @param exact
-     *        True if the class was obtained via Reflection.getCallerClass.
-     *
-     * @return The CacheEntry.
-     */
-    private CacheEntry toCacheEntry(final StackTraceElement stackTraceElement, final Class<?> callerClass,
-                                    final boolean exact) {
-        String location = "?";
-        String version = "?";
-        ClassLoader lastLoader = null;
-        if (callerClass != null) {
-            try {
-                final CodeSource source = callerClass.getProtectionDomain().getCodeSource();
-                if (source != null) {
-                    final URL locationURL = source.getLocation();
-                    if (locationURL != null) {
-                        final String str = locationURL.toString().replace('\\', '/');
-                        int index = str.lastIndexOf("/");
-                        if (index >= 0 && index == str.length() - 1) {
-                            index = str.lastIndexOf("/", index - 1);
-                            location = str.substring(index + 1);
-                        } else {
-                            location = str.substring(index + 1);
-                        }
-                    }
-                }
-            } catch (final Exception ex) {
-                // Ignore the exception.
-            }
-            final Package pkg = callerClass.getPackage();
-            if (pkg != null) {
-                final String ver = pkg.getImplementationVersion();
-                if (ver != null) {
-                    version = ver;
-                }
-            }
-            lastLoader = callerClass.getClassLoader();
-        }
-        return new CacheEntry(new ExtendedClassInfo(exact, location, version), lastLoader);
-    }
-
-    /**
-     * Resolve all the stack entries in this stack trace that are not common with the parent.
-     *
-     * @param stack
-     *        The callers Class stack.
-     * @param map
-     *        The cache of CacheEntry objects.
-     * @param rootTrace
-     *        The first stack trace resolve or null.
-     * @param stackTrace
-     *        The stack trace being resolved.
-     * @return The StackTracePackageElement array.
-     */
-    ExtendedStackTraceElement[] toExtendedStackTrace(final Stack<Class<?>> stack, final Map<String, CacheEntry> map,
-                                                     final StackTraceElement[] rootTrace, final StackTraceElement[] stackTrace) {
-        int stackLength;
-        if (rootTrace != null) {
-            int rootIndex = rootTrace.length - 1;
-            int stackIndex = stackTrace.length - 1;
-            while (rootIndex >= 0 && stackIndex >= 0 && rootTrace[rootIndex].equals(stackTrace[stackIndex])) {
-                --rootIndex;
-                --stackIndex;
-            }
-            this.commonElementCount = stackTrace.length - 1 - stackIndex;
-            stackLength = stackIndex + 1;
-        } else {
-            this.commonElementCount = 0;
-            stackLength = stackTrace.length;
-        }
-        final ExtendedStackTraceElement[] extStackTrace = new ExtendedStackTraceElement[stackLength];
-        Class<?> clazz = stack.isEmpty() ? null : stack.peek();
-        ClassLoader lastLoader = null;
-        for (int i = stackLength - 1; i >= 0; --i) {
-            final StackTraceElement stackTraceElement = stackTrace[i];
-            final String className = stackTraceElement.getClassName();
-            // The stack returned from getCurrentStack may be missing entries for java.lang.reflect.Method.invoke()
-            // and its implementation. The Throwable might also contain stack entries that are no longer
-            // present as those methods have returned.
-            ExtendedClassInfo extClassInfo;
-            if (clazz != null && className.equals(clazz.getName())) {
-                final CacheEntry entry = this.toCacheEntry(stackTraceElement, clazz, true);
-                extClassInfo = entry.element;
-                lastLoader = entry.loader;
-                stack.pop();
-                clazz = stack.isEmpty() ? null : stack.peek();
-            } else {
-                final CacheEntry cacheEntry = map.get(className);
-                if (cacheEntry != null) {
-                    final CacheEntry entry = cacheEntry;
-                    extClassInfo = entry.element;
-                    if (entry.loader != null) {
-                        lastLoader = entry.loader;
-                    }
-                } else {
-                    final CacheEntry entry = this.toCacheEntry(stackTraceElement,
-                        this.loadClass(lastLoader, className), false);
-                    extClassInfo = entry.element;
-                    map.put(stackTraceElement.toString(), entry);
-                    if (entry.loader != null) {
-                        lastLoader = entry.loader;
-                    }
-                }
-            }
-            extStackTrace[i] = new ExtendedStackTraceElement(stackTraceElement, extClassInfo);
-        }
-        return extStackTrace;
-    }
-
-    @Override
-    public String toString() {
-        final String msg = this.message;
-        return msg != null ? this.name + ": " + msg : this.name;
-    }
-
-    private ThrowableProxy[] toSuppressedProxies(final Throwable thrown, Set<Throwable> suppressedVisited) {
-        try {
-            final Throwable[] suppressed = thrown.getSuppressed();
-            if (suppressed == null) {
-                return EMPTY_THROWABLE_PROXY_ARRAY;
-            }
-            final List<ThrowableProxy> proxies = new ArrayList<>(suppressed.length);
-            if (suppressedVisited == null) {
-                suppressedVisited = new HashSet<>(proxies.size());
-            }
-            for (int i = 0; i < suppressed.length; i++) {
-                final Throwable candidate = suppressed[i];
-                if (!suppressedVisited.contains(candidate)) {
-                    suppressedVisited.add(candidate);
-                    proxies.add(new ThrowableProxy(candidate, suppressedVisited));
-                }
-            }
-            return proxies.toArray(new ThrowableProxy[proxies.size()]);
-        } catch (final Exception e) {
-            StatusLogger.getLogger().error(e);
-        }
-        return null;
-    }
-}
diff --git a/core/src/main/java/org/apache/lucene/analysis/miscellaneous/DisableGraphAttribute.java b/core/src/main/java/org/apache/lucene/analysis/miscellaneous/DisableGraphAttribute.java
new file mode 100644
index 0000000000000..16b9ecd1a5247
--- /dev/null
+++ b/core/src/main/java/org/apache/lucene/analysis/miscellaneous/DisableGraphAttribute.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lucene.analysis.miscellaneous;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.util.Attribute;
+import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute;
+
+/**
+ * This attribute can be used to indicate that the {@link PositionLengthAttribute}
+ * should not be taken in account in this {@link TokenStream}.
+ * Query parsers can extract this information to decide if this token stream should be analyzed
+ * as a graph or not.
+ */
+public interface DisableGraphAttribute extends Attribute {}
diff --git a/core/src/main/java/org/apache/lucene/analysis/miscellaneous/DisableGraphAttributeImpl.java b/core/src/main/java/org/apache/lucene/analysis/miscellaneous/DisableGraphAttributeImpl.java
new file mode 100644
index 0000000000000..5a4e7f79f238e
--- /dev/null
+++ b/core/src/main/java/org/apache/lucene/analysis/miscellaneous/DisableGraphAttributeImpl.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lucene.analysis.miscellaneous;
+
+import org.apache.lucene.util.AttributeImpl;
+import org.apache.lucene.util.AttributeReflector;
+
+/** Default implementation of {@link DisableGraphAttribute}. */
+public class DisableGraphAttributeImpl extends AttributeImpl implements DisableGraphAttribute {
+    public DisableGraphAttributeImpl() {}
+
+    @Override
+    public void clear() {}
+
+    @Override
+    public void reflectWith(AttributeReflector reflector) {
+    }
+
+    @Override
+    public void copyTo(AttributeImpl target) {}
+}
diff --git a/core/src/main/java/org/apache/lucene/index/OneMergeHelper.java b/core/src/main/java/org/apache/lucene/index/OneMergeHelper.java
index 99ef7f4dd7fef..f8b8c6178225b 100644
--- a/core/src/main/java/org/apache/lucene/index/OneMergeHelper.java
+++ b/core/src/main/java/org/apache/lucene/index/OneMergeHelper.java
@@ -19,6 +19,8 @@
 
 package org.apache.lucene.index;
 
+import java.io.IOException;
+
 /**
  * Allows pkg private access
  */
@@ -27,4 +29,33 @@ private OneMergeHelper() {}
     public static String getSegmentName(MergePolicy.OneMerge merge) {
         return merge.info != null ? merge.info.info.name : "_na_";
     }
+
+    /**
+     * The current MB per second rate limit for this merge.
+     **/
+    public static double getMbPerSec(Thread thread, MergePolicy.OneMerge merge) {
+        if (thread instanceof ConcurrentMergeScheduler.MergeThread) {
+            return ((ConcurrentMergeScheduler.MergeThread) thread).rateLimiter.getMBPerSec();
+        }
+        assert false: "this is not merge thread";
+        return Double.POSITIVE_INFINITY;
+    }
+
+    /**
+     * Returns total bytes written by this merge.
+     **/
+    public static long getTotalBytesWritten(Thread thread,
+                                            MergePolicy.OneMerge merge) throws IOException {
+        /**
+         * TODO: The number of bytes written during the merge should be accessible in OneMerge.
+         */
+        if (thread instanceof ConcurrentMergeScheduler.MergeThread) {
+            return ((ConcurrentMergeScheduler.MergeThread) thread).rateLimiter
+                .getTotalBytesWritten();
+        }
+        assert false: "this is not merge thread";
+        return merge.totalBytesSize();
+    }
+
+
 }
diff --git a/core/src/main/java/org/apache/lucene/index/XPointValues.java b/core/src/main/java/org/apache/lucene/index/XPointValues.java
deleted file mode 100644
index c4fa0b4d6232e..0000000000000
--- a/core/src/main/java/org/apache/lucene/index/XPointValues.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.lucene.index;
-import org.apache.lucene.util.StringHelper;
-
-import java.io.IOException;
-
-/**
- * Forked utility methods from Lucene's PointValues until LUCENE-7257 is released.
- */
-public class XPointValues {
-    /** Return the cumulated number of points across all leaves of the given
-     * {@link IndexReader}. Leaves that do not have points for the given field
-     * are ignored.
-     *  @see PointValues#size(String) */
-    public static long size(IndexReader reader, String field) throws IOException {
-        long size = 0;
-        for (LeafReaderContext ctx : reader.leaves()) {
-            FieldInfo info = ctx.reader().getFieldInfos().fieldInfo(field);
-            if (info == null || info.getPointDimensionCount() == 0) {
-                continue;
-            }
-            PointValues values = ctx.reader().getPointValues();
-            size += values.size(field);
-        }
-        return size;
-    }
-
-    /** Return the cumulated number of docs that have points across all leaves
-     * of the given {@link IndexReader}. Leaves that do not have points for the
-     * given field are ignored.
-     *  @see PointValues#getDocCount(String) */
-    public static int getDocCount(IndexReader reader, String field) throws IOException {
-        int count = 0;
-        for (LeafReaderContext ctx : reader.leaves()) {
-            FieldInfo info = ctx.reader().getFieldInfos().fieldInfo(field);
-            if (info == null || info.getPointDimensionCount() == 0) {
-                continue;
-            }
-            PointValues values = ctx.reader().getPointValues();
-            count += values.getDocCount(field);
-        }
-        return count;
-    }
-
-    /** Return the minimum packed values across all leaves of the given
-     * {@link IndexReader}. Leaves that do not have points for the given field
-     * are ignored.
-     *  @see PointValues#getMinPackedValue(String) */
-    public static byte[] getMinPackedValue(IndexReader reader, String field) throws IOException {
-        byte[] minValue = null;
-        for (LeafReaderContext ctx : reader.leaves()) {
-            FieldInfo info = ctx.reader().getFieldInfos().fieldInfo(field);
-            if (info == null || info.getPointDimensionCount() == 0) {
-                continue;
-            }
-            PointValues values = ctx.reader().getPointValues();
-            byte[] leafMinValue = values.getMinPackedValue(field);
-            if (leafMinValue == null) {
-                continue;
-            }
-            if (minValue == null) {
-                minValue = leafMinValue.clone();
-            } else {
-                final int numDimensions = values.getNumDimensions(field);
-                final int numBytesPerDimension = values.getBytesPerDimension(field);
-                for (int i = 0; i < numDimensions; ++i) {
-                    int offset = i * numBytesPerDimension;
-                    if (StringHelper.compare(numBytesPerDimension, leafMinValue, offset, minValue, offset) < 0) {
-                        System.arraycopy(leafMinValue, offset, minValue, offset, numBytesPerDimension);
-                    }
-                }
-            }
-        }
-        return minValue;
-    }
-
-    /** Return the maximum packed values across all leaves of the given
-     * {@link IndexReader}. Leaves that do not have points for the given field
-     * are ignored.
-     *  @see PointValues#getMaxPackedValue(String) */
-    public static byte[] getMaxPackedValue(IndexReader reader, String field) throws IOException {
-        byte[] maxValue = null;
-        for (LeafReaderContext ctx : reader.leaves()) {
-            FieldInfo info = ctx.reader().getFieldInfos().fieldInfo(field);
-            if (info == null || info.getPointDimensionCount() == 0) {
-                continue;
-            }
-            PointValues values = ctx.reader().getPointValues();
-            byte[] leafMaxValue = values.getMaxPackedValue(field);
-            if (leafMaxValue == null) {
-                continue;
-            }
-            if (maxValue == null) {
-                maxValue = leafMaxValue.clone();
-            } else {
-                final int numDimensions = values.getNumDimensions(field);
-                final int numBytesPerDimension = values.getBytesPerDimension(field);
-                for (int i = 0; i < numDimensions; ++i) {
-                    int offset = i * numBytesPerDimension;
-                    if (StringHelper.compare(numBytesPerDimension, leafMaxValue, offset, maxValue, offset) > 0) {
-                        System.arraycopy(leafMaxValue, offset, maxValue, offset, numBytesPerDimension);
-                    }
-                }
-            }
-        }
-        return maxValue;
-    }
-
-    /** Default constructor */
-    private XPointValues() {
-    }
-}
diff --git a/core/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java b/core/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java
index a4b94b007fd28..0b34a95710cfe 100644
--- a/core/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java
+++ b/core/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java
@@ -163,7 +163,7 @@ protected int compare(int i, int j) {
             if (prev > current) {
                 actualDf++;
             }
-            contexts[i] = ctx = adjustDF(ctx, Math.min(maxDoc, actualDf));
+            contexts[i] = ctx = adjustDF(reader.getContext(), ctx, Math.min(maxDoc, actualDf));
             prev = current;
             if (sumTTF >= 0 && ctx.totalTermFreq() >= 0) {
                 sumTTF += ctx.totalTermFreq();
@@ -179,16 +179,17 @@ protected int compare(int i, int j) {
             }
             // the blended sumTTF can't be greater than the sumTTTF on the field
             final long fixedTTF = sumTTF == -1 ? -1 : sumTTF;
-            contexts[i] = adjustTTF(contexts[i], fixedTTF);
+            contexts[i] = adjustTTF(reader.getContext(), contexts[i], fixedTTF);
         }
     }
 
-    private TermContext adjustTTF(TermContext termContext, long sumTTF) {
+    private TermContext adjustTTF(IndexReaderContext readerContext, TermContext termContext, long sumTTF) {
+        assert termContext.wasBuiltFor(readerContext);
         if (sumTTF == -1 && termContext.totalTermFreq() == -1) {
             return termContext;
         }
-        TermContext newTermContext = new TermContext(termContext.topReaderContext);
-        List<LeafReaderContext> leaves = termContext.topReaderContext.leaves();
+        TermContext newTermContext = new TermContext(readerContext);
+        List<LeafReaderContext> leaves = readerContext.leaves();
         final int len;
         if (leaves == null) {
             len = 1;
@@ -209,7 +210,8 @@ private TermContext adjustTTF(TermContext termContext, long sumTTF) {
         return newTermContext;
     }
 
-    private static TermContext adjustDF(TermContext ctx, int newDocFreq) {
+    private static TermContext adjustDF(IndexReaderContext readerContext, TermContext ctx, int newDocFreq) {
+        assert ctx.wasBuiltFor(readerContext);
         // Use a value of ttf that is consistent with the doc freq (ie. gte)
         long newTTF;
         if (ctx.totalTermFreq() < 0) {
@@ -217,14 +219,14 @@ private static TermContext adjustDF(TermContext ctx, int newDocFreq) {
         } else {
             newTTF = Math.max(ctx.totalTermFreq(), newDocFreq);
         }
-        List<LeafReaderContext> leaves = ctx.topReaderContext.leaves();
+        List<LeafReaderContext> leaves = readerContext.leaves();
         final int len;
         if (leaves == null) {
             len = 1;
         } else {
             len = leaves.size();
         }
-        TermContext newCtx = new TermContext(ctx.topReaderContext);
+        TermContext newCtx = new TermContext(readerContext);
         for (int i = 0; i < len; ++i) {
             TermState termState = ctx.get(i);
             if (termState == null) {
diff --git a/core/src/main/java/org/apache/lucene/queries/MinDocQuery.java b/core/src/main/java/org/apache/lucene/queries/MinDocQuery.java
index a8b7dc9299ff0..ea5d4e7e77be1 100644
--- a/core/src/main/java/org/apache/lucene/queries/MinDocQuery.java
+++ b/core/src/main/java/org/apache/lucene/queries/MinDocQuery.java
@@ -19,6 +19,7 @@
 
 package org.apache.lucene.queries;
 
+import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.ConstantScoreScorer;
 import org.apache.lucene.search.ConstantScoreWeight;
@@ -35,16 +36,26 @@
  *  to a configured doc ID. */
 public final class MinDocQuery extends Query {
 
+    // Matching documents depend on the sequence of segments that the index reader
+    // wraps. Yet matches must be cacheable per-segment, so we need to incorporate
+    // the reader id in the identity of the query so that a cache entry may only
+    // be reused if this query is run against the same index reader.
+    private final Object readerId;
     private final int minDoc;
 
     /** Sole constructor. */
     public MinDocQuery(int minDoc) {
+        this(minDoc, null);
+    }
+
+    MinDocQuery(int minDoc, Object readerId) {
         this.minDoc = minDoc;
+        this.readerId = readerId;
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(classHash(), minDoc);
+        return Objects.hash(classHash(), minDoc, readerId);
     }
 
     @Override
@@ -53,11 +64,24 @@ public boolean equals(Object obj) {
             return false;
         }
         MinDocQuery that = (MinDocQuery) obj;
-        return minDoc == that.minDoc;
+        return minDoc == that.minDoc && Objects.equals(readerId, that.readerId);
+    }
+
+    @Override
+    public Query rewrite(IndexReader reader) throws IOException {
+        if (Objects.equals(reader.getContext().id(), readerId) == false) {
+            return new MinDocQuery(minDoc, reader.getContext().id());
+        }
+        return this;
     }
 
     @Override
     public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException {
+        if (readerId == null) {
+            throw new IllegalStateException("Rewrite first");
+        } else if (Objects.equals(searcher.getIndexReader().getContext().id(), readerId) == false) {
+            throw new IllegalStateException("Executing against a different reader than the query has been rewritten against");
+        }
         return new ConstantScoreWeight(this) {
             @Override
             public Scorer scorer(LeafReaderContext context) throws IOException {
diff --git a/core/src/main/java/org/apache/lucene/queryparser/classic/ExistsFieldQueryExtension.java b/core/src/main/java/org/apache/lucene/queryparser/classic/ExistsFieldQueryExtension.java
index cb4bee30aaaf4..503fef4f18622 100644
--- a/core/src/main/java/org/apache/lucene/queryparser/classic/ExistsFieldQueryExtension.java
+++ b/core/src/main/java/org/apache/lucene/queryparser/classic/ExistsFieldQueryExtension.java
@@ -19,8 +19,11 @@
 
 package org.apache.lucene.queryparser.classic;
 
-import org.apache.lucene.search.ConstantScoreQuery;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.WildcardQuery;
+import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
 import org.elasticsearch.index.query.ExistsQueryBuilder;
 import org.elasticsearch.index.query.QueryShardContext;
 
@@ -33,6 +36,16 @@ public class ExistsFieldQueryExtension implements FieldQueryExtension {
 
     @Override
     public Query query(QueryShardContext context, String queryText) {
-        return new ConstantScoreQuery(ExistsQueryBuilder.newFilter(context, queryText));
+        final FieldNamesFieldMapper.FieldNamesFieldType fieldNamesFieldType =
+            (FieldNamesFieldMapper.FieldNamesFieldType) context.getMapperService().fullName(FieldNamesFieldMapper.NAME);
+        if (fieldNamesFieldType == null) {
+            return new MatchNoDocsQuery("No mappings yet");
+        }
+        if (fieldNamesFieldType.isEnabled() == false) {
+            // The field_names_field is disabled so we switch to a wildcard query that matches all terms
+            return new WildcardQuery(new Term(queryText, "*"));
+        }
+
+        return ExistsQueryBuilder.newFilter(context, queryText);
     }
 }
diff --git a/core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java b/core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java
index bcf0a2b201a31..130066788e71c 100644
--- a/core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java
+++ b/core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java
@@ -20,10 +20,12 @@
 package org.apache.lucene.queryparser.classic;
 
 import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.miscellaneous.DisableGraphAttribute;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.queryparser.analyzing.AnalyzingQueryParser;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.BoostQuery;
@@ -34,16 +36,23 @@
 import org.apache.lucene.search.PhraseQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.SynonymQuery;
+import org.apache.lucene.search.spans.SpanNearQuery;
+import org.apache.lucene.search.spans.SpanOrQuery;
+import org.apache.lucene.search.spans.SpanQuery;
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.automaton.RegExp;
 import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.unit.Fuzziness;
+import org.elasticsearch.index.mapper.AllFieldMapper;
 import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.index.mapper.LegacyDateFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.StringFieldType;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.query.support.QueryParsers;
+import org.elasticsearch.index.analysis.ShingleTokenFilterFactory;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -51,8 +60,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
-
+import java.util.Collections;
 import static java.util.Collections.unmodifiableMap;
 import static org.elasticsearch.common.lucene.search.Queries.fixNegativeQueryIfNeeded;
 
@@ -63,7 +71,7 @@
  * Also breaks fields with [type].[name] into a boolean query that must include the type
  * as well as the query on the name.
  */
-public class MapperQueryParser extends QueryParser {
+public class MapperQueryParser extends AnalyzingQueryParser {
 
     public static final Map<String, FieldQueryExtension> FIELD_QUERY_EXTENSIONS;
 
@@ -86,7 +94,8 @@ public MapperQueryParser(QueryShardContext context) {
 
     public void reset(QueryParserSettings settings) {
         this.settings = settings;
-        if (settings.fieldsAndWeights().isEmpty()) {
+        if (settings.fieldsAndWeights() == null) {
+            // this query has no explicit fields to query so we fallback to the default field
             this.field = settings.defaultField();
         } else if (settings.fieldsAndWeights().size() == 1) {
             this.field = settings.fieldsAndWeights().keySet().iterator().next();
@@ -99,11 +108,11 @@ public void reset(QueryParserSettings settings) {
         setAutoGeneratePhraseQueries(settings.autoGeneratePhraseQueries());
         setMaxDeterminizedStates(settings.maxDeterminizedStates());
         setAllowLeadingWildcard(settings.allowLeadingWildcard());
-        setLowercaseExpandedTerms(settings.lowercaseExpandedTerms());
+        setLowercaseExpandedTerms(false);
         setPhraseSlop(settings.phraseSlop());
         setDefaultOperator(settings.defaultOperator());
         setFuzzyPrefixLength(settings.fuzzyPrefixLength());
-        setLocale(settings.locale());
+        setSplitOnWhitespace(settings.splitOnWhitespace());
     }
 
     /**
@@ -144,6 +153,11 @@ public Query getFieldQuery(String field, String queryText, boolean quoted) throw
         if (fields != null) {
             if (fields.size() == 1) {
                 return getFieldQuerySingle(fields.iterator().next(), queryText, quoted);
+            } else if (fields.isEmpty()) {
+                // the requested fields do not match any field in the mapping
+                // happens for wildcard fields only since we cannot expand to a valid field name
+                // if there is no match in the mappings.
+                return new MatchNoDocsQuery("empty fields");
             }
             if (settings.useDisMax()) {
                 List<Query> queries = new ArrayList<>();
@@ -180,17 +194,17 @@ private Query getFieldQuerySingle(String field, String queryText, boolean quoted
             if (queryText.charAt(0) == '>') {
                 if (queryText.length() > 2) {
                     if (queryText.charAt(1) == '=') {
-                        return getRangeQuerySingle(field, queryText.substring(2), null, true, true);
+                        return getRangeQuerySingle(field, queryText.substring(2), null, true, true, context);
                     }
                 }
-                return getRangeQuerySingle(field, queryText.substring(1), null, false, true);
+                return getRangeQuerySingle(field, queryText.substring(1), null, false, true, context);
             } else if (queryText.charAt(0) == '<') {
                 if (queryText.length() > 2) {
                     if (queryText.charAt(1) == '=') {
-                        return getRangeQuerySingle(field, null, queryText.substring(2), true, true);
+                        return getRangeQuerySingle(field, null, queryText.substring(2), true, true, context);
                     }
                 }
-                return getRangeQuerySingle(field, null, queryText.substring(1), true, false);
+                return getRangeQuerySingle(field, null, queryText.substring(1), true, false, context);
             }
         }
         currentFieldType = null;
@@ -290,19 +304,19 @@ protected Query getRangeQuery(String field, String part1, String part2,
         Collection<String> fields = extractMultiFields(field);
 
         if (fields == null) {
-            return getRangeQuerySingle(field, part1, part2, startInclusive, endInclusive);
+            return getRangeQuerySingle(field, part1, part2, startInclusive, endInclusive, context);
         }
 
 
         if (fields.size() == 1) {
-            return getRangeQuerySingle(fields.iterator().next(), part1, part2, startInclusive, endInclusive);
+            return getRangeQuerySingle(fields.iterator().next(), part1, part2, startInclusive, endInclusive, context);
         }
 
         if (settings.useDisMax()) {
             List<Query> queries = new ArrayList<>();
             boolean added = false;
             for (String mField : fields) {
-                Query q = getRangeQuerySingle(mField, part1, part2, startInclusive, endInclusive);
+                Query q = getRangeQuerySingle(mField, part1, part2, startInclusive, endInclusive, context);
                 if (q != null) {
                     added = true;
                     queries.add(applyBoost(mField, q));
@@ -315,7 +329,7 @@ protected Query getRangeQuery(String field, String part1, String part2,
         } else {
             List<BooleanClause> clauses = new ArrayList<>();
             for (String mField : fields) {
-                Query q = getRangeQuerySingle(mField, part1, part2, startInclusive, endInclusive);
+                Query q = getRangeQuerySingle(mField, part1, part2, startInclusive, endInclusive, context);
                 if (q != null) {
                     clauses.add(new BooleanClause(applyBoost(mField, q), BooleanClause.Occur.SHOULD));
                 }
@@ -326,24 +340,23 @@ protected Query getRangeQuery(String field, String part1, String part2,
     }
 
     private Query getRangeQuerySingle(String field, String part1, String part2,
-                                      boolean startInclusive, boolean endInclusive) {
+            boolean startInclusive, boolean endInclusive, QueryShardContext context) {
         currentFieldType = context.fieldMapper(field);
         if (currentFieldType != null) {
-            if (lowercaseExpandedTerms && currentFieldType.tokenized()) {
-                part1 = part1 == null ? null : part1.toLowerCase(locale);
-                part2 = part2 == null ? null : part2.toLowerCase(locale);
-            }
-
             try {
+                BytesRef part1Binary = part1 == null ? null : getAnalyzer().normalize(field, part1);
+                BytesRef part2Binary = part2 == null ? null : getAnalyzer().normalize(field, part2);
                 Query rangeQuery;
                 if (currentFieldType instanceof LegacyDateFieldMapper.DateFieldType && settings.timeZone() != null) {
                     LegacyDateFieldMapper.DateFieldType dateFieldType = (LegacyDateFieldMapper.DateFieldType) this.currentFieldType;
-                    rangeQuery = dateFieldType.rangeQuery(part1, part2, startInclusive, endInclusive, settings.timeZone(), null);
+                    rangeQuery = dateFieldType.rangeQuery(part1Binary, part2Binary,
+                            startInclusive, endInclusive, settings.timeZone(), null, context);
                 } else if (currentFieldType instanceof DateFieldMapper.DateFieldType && settings.timeZone() != null) {
                     DateFieldMapper.DateFieldType dateFieldType = (DateFieldMapper.DateFieldType) this.currentFieldType;
-                    rangeQuery = dateFieldType.rangeQuery(part1, part2, startInclusive, endInclusive, settings.timeZone(), null);
+                    rangeQuery = dateFieldType.rangeQuery(part1Binary, part2Binary,
+                            startInclusive, endInclusive, settings.timeZone(), null, context);
                 } else {
-                    rangeQuery = currentFieldType.rangeQuery(part1, part2, startInclusive, endInclusive);
+                    rangeQuery = currentFieldType.rangeQuery(part1Binary, part2Binary, startInclusive, endInclusive, context);
                 }
                 return rangeQuery;
             } catch (RuntimeException e) {
@@ -357,9 +370,6 @@ private Query getRangeQuerySingle(String field, String part1, String part2,
     }
 
     protected Query getFuzzyQuery(String field, String termStr, String minSimilarity) throws ParseException {
-        if (lowercaseExpandedTerms) {
-            termStr = termStr.toLowerCase(locale);
-        }
         Collection<String> fields = extractMultiFields(field);
         if (fields != null) {
             if (fields.size() == 1) {
@@ -398,8 +408,9 @@ private Query getFuzzyQuerySingle(String field, String termStr, String minSimila
         currentFieldType = context.fieldMapper(field);
         if (currentFieldType != null) {
             try {
-                return currentFieldType.fuzzyQuery(termStr, Fuzziness.build(minSimilarity),
-                    fuzzyPrefixLength, settings.fuzzyMaxExpansions(), FuzzyQuery.defaultTranspositions);
+                BytesRef term = termStr == null ? null : getAnalyzer().normalize(field, termStr);
+                return currentFieldType.fuzzyQuery(term, Fuzziness.build(minSimilarity),
+                    getFuzzyPrefixLength(), settings.fuzzyMaxExpansions(), FuzzyQuery.defaultTranspositions);
             } catch (RuntimeException e) {
                 if (settings.lenient()) {
                     return null;
@@ -422,9 +433,6 @@ protected Query newFuzzyQuery(Term term, float minimumSimilarity, int prefixLeng
 
     @Override
     protected Query getPrefixQuery(String field, String termStr) throws ParseException {
-        if (lowercaseExpandedTerms) {
-            termStr = termStr.toLowerCase(locale);
-        }
         Collection<String> fields = extractMultiFields(field);
         if (fields != null) {
             if (fields.size() == 1) {
@@ -470,8 +478,8 @@ private Query getPrefixQuerySingle(String field, String termStr) throws ParseExc
                     setAnalyzer(context.getSearchAnalyzer(currentFieldType));
                 }
                 Query query = null;
-                if (currentFieldType.tokenized() == false) {
-                    query = currentFieldType.prefixQuery(termStr, multiTermRewriteMethod, context);
+                if (currentFieldType instanceof StringFieldType == false) {
+                    query = currentFieldType.prefixQuery(termStr, getMultiTermRewriteMethod(), context);
                 }
                 if (query == null) {
                     query = getPossiblyAnalyzedPrefixQuery(currentFieldType.name(), termStr);
@@ -572,25 +580,20 @@ private Query getPossiblyAnalyzedPrefixQuery(String field, String termStr) throw
 
     @Override
     protected Query getWildcardQuery(String field, String termStr) throws ParseException {
-        if (termStr.equals("*")) {
-            // we want to optimize for match all query for the "*:*", and "*" cases
-            if ("*".equals(field) || Objects.equals(field, this.field)) {
-                String actualField = field;
-                if (actualField == null) {
-                    actualField = this.field;
-                }
-                if (actualField == null) {
-                    return newMatchAllDocsQuery();
-                }
-                if ("*".equals(actualField) || "_all".equals(actualField)) {
-                    return newMatchAllDocsQuery();
-                }
-                // effectively, we check if a field exists or not
-                return FIELD_QUERY_EXTENSIONS.get(ExistsFieldQueryExtension.NAME).query(context, actualField);
+        if (termStr.equals("*") && field != null) {
+            /**
+             * We rewrite _all:* to a match all query.
+             * TODO: We can remove this special case when _all is completely removed.
+             */
+            if ("*".equals(field) || AllFieldMapper.NAME.equals(field)) {
+                return newMatchAllDocsQuery();
             }
-        }
-        if (lowercaseExpandedTerms) {
-            termStr = termStr.toLowerCase(locale);
+            String actualField = field;
+            if (actualField == null) {
+                actualField = this.field;
+            }
+            // effectively, we check if a field exists or not
+            return FIELD_QUERY_EXTENSIONS.get(ExistsFieldQueryExtension.NAME).query(context, actualField);
         }
         Collection<String> fields = extractMultiFields(field);
         if (fields != null) {
@@ -628,6 +631,10 @@ protected Query getWildcardQuery(String field, String termStr) throws ParseExcep
     }
 
     private Query getWildcardQuerySingle(String field, String termStr) throws ParseException {
+        if ("*".equals(termStr)) {
+            // effectively, we check if a field exists or not
+            return FIELD_QUERY_EXTENSIONS.get(ExistsFieldQueryExtension.NAME).query(context, field);
+        }
         String indexedNameField = field;
         currentFieldType = null;
         Analyzer oldAnalyzer = getAnalyzer();
@@ -638,9 +645,8 @@ private Query getWildcardQuerySingle(String field, String termStr) throws ParseE
                     setAnalyzer(context.getSearchAnalyzer(currentFieldType));
                 }
                 indexedNameField = currentFieldType.name();
-                return getPossiblyAnalyzedWildcardQuery(indexedNameField, termStr);
             }
-            return getPossiblyAnalyzedWildcardQuery(indexedNameField, termStr);
+            return super.getWildcardQuery(indexedNameField, termStr);
         } catch (RuntimeException e) {
             if (settings.lenient()) {
                 return null;
@@ -651,75 +657,8 @@ private Query getWildcardQuerySingle(String field, String termStr) throws ParseE
         }
     }
 
-    private Query getPossiblyAnalyzedWildcardQuery(String field, String termStr) throws ParseException {
-        if (!settings.analyzeWildcard()) {
-            return super.getWildcardQuery(field, termStr);
-        }
-        boolean isWithinToken = (!termStr.startsWith("?") && !termStr.startsWith("*"));
-        StringBuilder aggStr = new StringBuilder();
-        StringBuilder tmp = new StringBuilder();
-        for (int i = 0; i < termStr.length(); i++) {
-            char c = termStr.charAt(i);
-            if (c == '?' || c == '*') {
-                if (isWithinToken) {
-                    try (TokenStream source = getAnalyzer().tokenStream(field, tmp.toString())) {
-                        source.reset();
-                        CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class);
-                        if (source.incrementToken()) {
-                            String term = termAtt.toString();
-                            if (term.length() == 0) {
-                                // no tokens, just use what we have now
-                                aggStr.append(tmp);
-                            } else {
-                                aggStr.append(term);
-                            }
-                        } else {
-                            // no tokens, just use what we have now
-                            aggStr.append(tmp);
-                        }
-                    } catch (IOException e) {
-                        aggStr.append(tmp);
-                    }
-                    tmp.setLength(0);
-                }
-                isWithinToken = false;
-                aggStr.append(c);
-            } else {
-                tmp.append(c);
-                isWithinToken = true;
-            }
-        }
-        if (isWithinToken) {
-            try {
-                try (TokenStream source = getAnalyzer().tokenStream(field, tmp.toString())) {
-                    source.reset();
-                    CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class);
-                    if (source.incrementToken()) {
-                        String term = termAtt.toString();
-                        if (term.length() == 0) {
-                            // no tokens, just use what we have now
-                            aggStr.append(tmp);
-                        } else {
-                            aggStr.append(term);
-                        }
-                    } else {
-                        // no tokens, just use what we have now
-                        aggStr.append(tmp);
-                    }
-                }
-            } catch (IOException e) {
-                aggStr.append(tmp);
-            }
-        }
-
-        return super.getWildcardQuery(field, aggStr.toString());
-    }
-
     @Override
     protected Query getRegexpQuery(String field, String termStr) throws ParseException {
-        if (lowercaseExpandedTerms) {
-            termStr = termStr.toLowerCase(locale);
-        }
         Collection<String> fields = extractMultiFields(field);
         if (fields != null) {
             if (fields.size() == 1) {
@@ -767,7 +706,7 @@ private Query getRegexpQuerySingle(String field, String termStr) throws ParseExc
                 Query query = null;
                 if (currentFieldType.tokenized() == false) {
                     query = currentFieldType.regexpQuery(termStr, RegExp.ALL,
-                        maxDeterminizedStates, multiTermRewriteMethod, context);
+                        getMaxDeterminizedStates(), getMultiTermRewriteMethod(), context);
                 }
                 if (query == null) {
                     query = super.getRegexpQuery(field, termStr);
@@ -809,7 +748,7 @@ protected Query getBooleanQuery(List<BooleanClause> clauses) throws ParseExcepti
     }
 
     private Query applyBoost(String field, Query q) {
-        Float fieldBoost = settings.fieldsAndWeights().get(field);
+        Float fieldBoost = settings.fieldsAndWeights() == null ? null : settings.fieldsAndWeights().get(field);
         if (fieldBoost != null && fieldBoost != 1f) {
             return new BoostQuery(q, fieldBoost);
         }
@@ -818,33 +757,58 @@ private Query applyBoost(String field, Query q) {
 
     private Query applySlop(Query q, int slop) {
         if (q instanceof PhraseQuery) {
-            PhraseQuery pq = (PhraseQuery) q;
-            PhraseQuery.Builder builder = new PhraseQuery.Builder();
-            builder.setSlop(slop);
-            final Term[] terms = pq.getTerms();
-            final int[] positions = pq.getPositions();
-            for (int i = 0; i < terms.length; ++i) {
-                builder.add(terms[i], positions[i]);
-            }
-            pq = builder.build();
             //make sure that the boost hasn't been set beforehand, otherwise we'd lose it
             assert q instanceof BoostQuery == false;
-            return pq;
+            return addSlopToPhrase((PhraseQuery) q, slop);
         } else if (q instanceof MultiPhraseQuery) {
             MultiPhraseQuery.Builder builder = new MultiPhraseQuery.Builder((MultiPhraseQuery) q);
             builder.setSlop(slop);
             return builder.build();
+        } else if (q instanceof SpanQuery) {
+            return addSlopToSpan((SpanQuery) q, slop);
         } else {
             return q;
         }
     }
 
+    private Query addSlopToSpan(SpanQuery query, int slop) {
+        if (query instanceof SpanNearQuery) {
+            return new SpanNearQuery(((SpanNearQuery) query).getClauses(), slop,
+                ((SpanNearQuery) query).isInOrder());
+        } else if (query instanceof SpanOrQuery) {
+            SpanQuery[] clauses = new SpanQuery[((SpanOrQuery) query).getClauses().length];
+            int pos = 0;
+            for (SpanQuery clause : ((SpanOrQuery) query).getClauses()) {
+                clauses[pos++] = (SpanQuery) addSlopToSpan(clause, slop);
+            }
+            return new SpanOrQuery(clauses);
+        } else {
+            return query;
+        }
+    }
+
+    /**
+     * Rebuild a phrase query with a slop value
+     */
+    private PhraseQuery addSlopToPhrase(PhraseQuery query, int slop) {
+        PhraseQuery.Builder builder = new PhraseQuery.Builder();
+        builder.setSlop(slop);
+        final Term[] terms = query.getTerms();
+        final int[] positions = query.getPositions();
+        for (int i = 0; i < terms.length; ++i) {
+            builder.add(terms[i], positions[i]);
+        }
+
+        return builder.build();
+    }
+
     private Collection<String> extractMultiFields(String field) {
         Collection<String> fields;
         if (field != null) {
             fields = context.simpleMatchToIndexNames(field);
         } else {
-            fields = settings.fieldsAndWeights().keySet();
+            Map<String, Float> fieldsAndWeights = settings.fieldsAndWeights();
+            fields = fieldsAndWeights == null ? Collections.emptyList() : fieldsAndWeights.keySet();
         }
         return fields;
     }
@@ -859,4 +823,30 @@ public Query parse(String query) throws ParseException {
         }
         return super.parse(query);
     }
+
+    /**
+     * Checks if graph analysis should be enabled for the field depending
+     * on the provided {@link Analyzer}
+     */
+    protected Query createFieldQuery(Analyzer analyzer, BooleanClause.Occur operator, String field,
+                                     String queryText, boolean quoted, int phraseSlop) {
+        assert operator == BooleanClause.Occur.SHOULD || operator == BooleanClause.Occur.MUST;
+
+        // Use the analyzer to get all the tokens, and then build an appropriate
+        // query based on the analysis chain.
+        try (TokenStream source = analyzer.tokenStream(field, queryText)) {
+            if (source.hasAttribute(DisableGraphAttribute.class)) {
+                /**
+                 * A {@link TokenFilter} in this {@link TokenStream} disabled the graph analysis to avoid
+                 * paths explosion. See {@link ShingleTokenFilterFactory} for details.
+                 */
+                setEnableGraphQueries(false);
+            }
+            Query query = super.createFieldQuery(source, operator, field, quoted, phraseSlop);
+            setEnableGraphQueries(true);
+            return query;
+        } catch (IOException e) {
+            throw new RuntimeException("Error analyzing query text", e);
+        }
+    }
 }
diff --git a/core/src/main/java/org/apache/lucene/queryparser/classic/QueryParserSettings.java b/core/src/main/java/org/apache/lucene/queryparser/classic/QueryParserSettings.java
index c1fc2ae556ea7..295c1ace4f637 100644
--- a/core/src/main/java/org/apache/lucene/queryparser/classic/QueryParserSettings.java
+++ b/core/src/main/java/org/apache/lucene/queryparser/classic/QueryParserSettings.java
@@ -24,7 +24,6 @@
 import org.elasticsearch.common.unit.Fuzziness;
 import org.joda.time.DateTimeZone;
 
-import java.util.Locale;
 import java.util.Map;
 
 /**
@@ -53,12 +52,8 @@ public class QueryParserSettings {
 
     private boolean analyzeWildcard;
 
-    private boolean lowercaseExpandedTerms;
-
     private boolean enablePositionIncrements;
 
-    private Locale locale;
-
     private Fuzziness fuzziness;
     private int fuzzyPrefixLength;
     private int fuzzyMaxExpansions;
@@ -79,6 +74,8 @@ public class QueryParserSettings {
     /** To limit effort spent determinizing regexp queries. */
     private int maxDeterminizedStates;
 
+    private boolean splitOnWhitespace;
+
     public QueryParserSettings(String queryString) {
         this.queryString = queryString;
     }
@@ -135,14 +132,6 @@ public void allowLeadingWildcard(boolean allowLeadingWildcard) {
         this.allowLeadingWildcard = allowLeadingWildcard;
     }
 
-    public boolean lowercaseExpandedTerms() {
-        return lowercaseExpandedTerms;
-    }
-
-    public void lowercaseExpandedTerms(boolean lowercaseExpandedTerms) {
-        this.lowercaseExpandedTerms = lowercaseExpandedTerms;
-    }
-
     public boolean enablePositionIncrements() {
         return enablePositionIncrements;
     }
@@ -267,14 +256,6 @@ public void useDisMax(boolean useDisMax) {
         this.useDisMax = useDisMax;
     }
 
-    public void locale(Locale locale) {
-        this.locale = locale;
-    }
-
-    public Locale locale() {
-        return this.locale;
-    }
-
     public void timeZone(DateTimeZone timeZone) {
         this.timeZone = timeZone;
     }
@@ -290,4 +271,12 @@ public void fuzziness(Fuzziness fuzziness) {
     public Fuzziness fuzziness() {
         return fuzziness;
     }
+
+    public void splitOnWhitespace(boolean value) {
+        this.splitOnWhitespace = value;
+    }
+
+    public boolean splitOnWhitespace() {
+        return splitOnWhitespace;
+    }
 }
diff --git a/core/src/main/java/org/apache/lucene/search/grouping/CollapseTopFieldDocs.java b/core/src/main/java/org/apache/lucene/search/grouping/CollapseTopFieldDocs.java
new file mode 100644
index 0000000000000..b4d3c82343957
--- /dev/null
+++ b/core/src/main/java/org/apache/lucene/search/grouping/CollapseTopFieldDocs.java
@@ -0,0 +1,242 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lucene.search.grouping;
+
+import org.apache.lucene.search.FieldComparator;
+import org.apache.lucene.search.FieldDoc;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.TopFieldDocs;
+import org.apache.lucene.util.PriorityQueue;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Represents hits returned by {@link CollapsingTopDocsCollector#getTopDocs()}.
+ */
+public final class CollapseTopFieldDocs extends TopFieldDocs {
+    /** The field used for collapsing **/
+    public final String field;
+    /** The collapse value for each top doc */
+    public final Object[] collapseValues;
+
+    public CollapseTopFieldDocs(String field, int totalHits, ScoreDoc[] scoreDocs,
+                                SortField[] sortFields, Object[] values, float maxScore) {
+        super(totalHits, scoreDocs, sortFields, maxScore);
+        this.field = field;
+        this.collapseValues = values;
+    }
+
+    // Refers to one hit:
+    private static final class ShardRef {
+        // Which shard (index into shardHits[]):
+        final int shardIndex;
+
+        // True if we should use the incoming ScoreDoc.shardIndex for sort order
+        final boolean useScoreDocIndex;
+
+        // Which hit within the shard:
+        int hitIndex;
+
+        ShardRef(int shardIndex, boolean useScoreDocIndex) {
+            this.shardIndex = shardIndex;
+            this.useScoreDocIndex = useScoreDocIndex;
+        }
+
+        @Override
+        public String toString() {
+            return "ShardRef(shardIndex=" + shardIndex + " hitIndex=" + hitIndex + ")";
+        }
+
+        int getShardIndex(ScoreDoc scoreDoc) {
+            if (useScoreDocIndex) {
+                if (scoreDoc.shardIndex == -1) {
+                    throw new IllegalArgumentException("setShardIndex is false but TopDocs["
+                        + shardIndex + "].scoreDocs[" + hitIndex + "] is not set");
+                }
+                return scoreDoc.shardIndex;
+            } else {
+                // NOTE: we don't assert that shardIndex is -1 here, because caller could in fact have set it but asked us to ignore it now
+                return shardIndex;
+            }
+        }
+    }
+
+    /**
+     * if we need to tie-break since score / sort value are the same we first compare shard index (lower shard wins)
+     * and then iff shard index is the same we use the hit index.
+     */
+    static boolean tieBreakLessThan(ShardRef first, ScoreDoc firstDoc, ShardRef second, ScoreDoc secondDoc) {
+        final int firstShardIndex = first.getShardIndex(firstDoc);
+        final int secondShardIndex = second.getShardIndex(secondDoc);
+        // Tie break: earlier shard wins
+        if (firstShardIndex < secondShardIndex) {
+            return true;
+        } else if (firstShardIndex > secondShardIndex) {
+            return false;
+        } else {
+            // Tie break in same shard: resolve however the
+            // shard had resolved it:
+            assert first.hitIndex != second.hitIndex;
+            return first.hitIndex < second.hitIndex;
+        }
+    }
+
+    private static class MergeSortQueue extends PriorityQueue<ShardRef> {
+        // These are really FieldDoc instances:
+        final ScoreDoc[][] shardHits;
+        final FieldComparator<?>[] comparators;
+        final int[] reverseMul;
+
+        MergeSortQueue(Sort sort, CollapseTopFieldDocs[] shardHits) {
+            super(shardHits.length);
+            this.shardHits = new ScoreDoc[shardHits.length][];
+            for (int shardIDX = 0; shardIDX < shardHits.length; shardIDX++) {
+                final ScoreDoc[] shard = shardHits[shardIDX].scoreDocs;
+                if (shard != null) {
+                    this.shardHits[shardIDX] = shard;
+                    // Fail gracefully if API is misused:
+                    for (int hitIDX = 0; hitIDX < shard.length; hitIDX++) {
+                        final ScoreDoc sd = shard[hitIDX];
+                        final FieldDoc gd = (FieldDoc) sd;
+                        assert gd.fields != null;
+                    }
+                }
+            }
+
+            final SortField[] sortFields = sort.getSort();
+            comparators = new FieldComparator[sortFields.length];
+            reverseMul = new int[sortFields.length];
+            for (int compIDX = 0; compIDX < sortFields.length; compIDX++) {
+                final SortField sortField = sortFields[compIDX];
+                comparators[compIDX] = sortField.getComparator(1, compIDX);
+                reverseMul[compIDX] = sortField.getReverse() ? -1 : 1;
+            }
+        }
+
+        // Returns true if first is < second
+        @Override
+        public boolean lessThan(ShardRef first, ShardRef second) {
+            assert first != second;
+            final FieldDoc firstFD = (FieldDoc) shardHits[first.shardIndex][first.hitIndex];
+            final FieldDoc secondFD = (FieldDoc) shardHits[second.shardIndex][second.hitIndex];
+
+            for (int compIDX = 0; compIDX < comparators.length; compIDX++) {
+                final FieldComparator comp = comparators[compIDX];
+
+                final int cmp =
+                    reverseMul[compIDX] * comp.compareValues(firstFD.fields[compIDX], secondFD.fields[compIDX]);
+
+                if (cmp != 0) {
+                    return cmp < 0;
+                }
+            }
+            return tieBreakLessThan(first, firstFD, second, secondFD);
+        }
+    }
+
+    /**
+     * Returns a new CollapseTopDocs, containing topN collapsed results across
+     * the provided CollapseTopDocs, sorting by score. Each {@link CollapseTopFieldDocs} instance must be sorted.
+     **/
+    public static CollapseTopFieldDocs merge(Sort sort, int start, int size,
+                                             CollapseTopFieldDocs[] shardHits, boolean setShardIndex) {
+        String collapseField = shardHits[0].field;
+        for (int i = 1; i < shardHits.length; i++) {
+            if (collapseField.equals(shardHits[i].field) == false) {
+                throw new IllegalArgumentException("collapse field differ across shards [" +
+                    collapseField + "] != [" + shardHits[i].field + "]");
+            }
+        }
+        final PriorityQueue<ShardRef> queue = new MergeSortQueue(sort, shardHits);
+
+        int totalHitCount = 0;
+        int availHitCount = 0;
+        float maxScore = Float.MIN_VALUE;
+        for(int shardIDX=0;shardIDX<shardHits.length;shardIDX++) {
+            final CollapseTopFieldDocs shard = shardHits[shardIDX];
+            // totalHits can be non-zero even if no hits were
+            // collected, when searchAfter was used:
+            totalHitCount += shard.totalHits;
+            if (shard.scoreDocs != null && shard.scoreDocs.length > 0) {
+                availHitCount += shard.scoreDocs.length;
+                queue.add(new ShardRef(shardIDX, setShardIndex == false));
+                maxScore = Math.max(maxScore, shard.getMaxScore());
+            }
+        }
+
+        if (availHitCount == 0) {
+            maxScore = Float.NaN;
+        }
+
+        final ScoreDoc[] hits;
+        final Object[] values;
+        if (availHitCount <= start) {
+            hits = new ScoreDoc[0];
+            values = new Object[0];
+        } else {
+            List<ScoreDoc> hitList = new ArrayList<>();
+            List<Object> collapseList = new ArrayList<>();
+            int requestedResultWindow = start + size;
+            int numIterOnHits = Math.min(availHitCount, requestedResultWindow);
+            int hitUpto = 0;
+            Set<Object> seen = new HashSet<>();
+            while (hitUpto < numIterOnHits) {
+                if (queue.size() == 0) {
+                    break;
+                }
+                ShardRef ref = queue.top();
+                final ScoreDoc hit = shardHits[ref.shardIndex].scoreDocs[ref.hitIndex];
+                final Object collapseValue = shardHits[ref.shardIndex].collapseValues[ref.hitIndex++];
+                if (seen.contains(collapseValue)) {
+                    if (ref.hitIndex < shardHits[ref.shardIndex].scoreDocs.length) {
+                        queue.updateTop();
+                    } else {
+                        queue.pop();
+                    }
+                    continue;
+                }
+                seen.add(collapseValue);
+                if (setShardIndex) {
+                    hit.shardIndex = ref.shardIndex;
+                }
+                if (hitUpto >= start) {
+                    hitList.add(hit);
+                    collapseList.add(collapseValue);
+                }
+
+                hitUpto++;
+
+                if (ref.hitIndex < shardHits[ref.shardIndex].scoreDocs.length) {
+                    // Not done with this these TopDocs yet:
+                    queue.updateTop();
+                } else {
+                    queue.pop();
+                }
+            }
+            hits = hitList.toArray(new ScoreDoc[0]);
+            values = collapseList.toArray(new Object[0]);
+        }
+        return new CollapseTopFieldDocs(collapseField, totalHitCount, hits, sort.getSort(), values, maxScore);
+    }
+}
diff --git a/core/src/main/java/org/apache/lucene/search/grouping/CollapsingDocValuesSource.java b/core/src/main/java/org/apache/lucene/search/grouping/CollapsingDocValuesSource.java
new file mode 100644
index 0000000000000..5bc8afb347c71
--- /dev/null
+++ b/core/src/main/java/org/apache/lucene/search/grouping/CollapsingDocValuesSource.java
@@ -0,0 +1,211 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lucene.search.grouping;
+
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.DocValuesType;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
+
+import java.io.IOException;
+
+/**
+ * Utility class that ensures that a single collapse key is extracted per document.
+ */
+abstract class CollapsingDocValuesSource<T> {
+    protected final String field;
+
+    CollapsingDocValuesSource(String field) throws IOException {
+        this.field = field;
+    }
+
+    abstract T get(int doc);
+
+    abstract T copy(T value, T reuse);
+
+    abstract void setNextReader(LeafReader reader) throws IOException;
+
+    /**
+     * Implementation for {@link NumericDocValues} and {@link SortedNumericDocValues}.
+     * Fails with an {@link IllegalStateException} if a document contains multiple values for the specified field.
+     */
+    static class Numeric extends CollapsingDocValuesSource<Long> {
+        private NumericDocValues values;
+        private Bits docsWithField;
+
+        Numeric(String field) throws IOException {
+            super(field);
+        }
+
+        @Override
+        public Long get(int doc) {
+            if (docsWithField.get(doc)) {
+                return values.get(doc);
+            } else {
+                return null;
+            }
+        }
+
+        @Override
+        public Long copy(Long value, Long reuse) {
+            return value;
+        }
+
+        @Override
+        public void setNextReader(LeafReader reader) throws IOException {
+            DocValuesType type = getDocValuesType(reader, field);
+            if (type == null || type == DocValuesType.NONE) {
+                values = DocValues.emptyNumeric();
+                docsWithField = new Bits.MatchNoBits(reader.maxDoc());
+                return ;
+            }
+            docsWithField = DocValues.getDocsWithField(reader, field);
+            switch (type) {
+                case NUMERIC:
+                    values = DocValues.getNumeric(reader, field);
+                    break;
+
+                case SORTED_NUMERIC:
+                    final SortedNumericDocValues sorted = DocValues.getSortedNumeric(reader, field);
+                    values = DocValues.unwrapSingleton(sorted);
+                    if (values == null) {
+                        values = new NumericDocValues() {
+                            @Override
+                            public long get(int docID) {
+                                sorted.setDocument(docID);
+                                assert sorted.count() > 0;
+                                if (sorted.count() > 1) {
+                                    throw new IllegalStateException("failed to collapse " + docID +
+                                        ", the collapse field must be single valued");
+                                }
+                                return sorted.valueAt(0);
+                            }
+                        };
+                    }
+                    break;
+
+                default:
+                    throw new IllegalStateException("unexpected doc values type " +
+                        type + "` for field `" + field + "`");
+            }
+        }
+    }
+
+    /**
+     * Implementation for {@link SortedDocValues} and {@link SortedSetDocValues}.
+     * Fails with an {@link IllegalStateException} if a document contains multiple values for the specified field.
+     */
+    static class Keyword extends CollapsingDocValuesSource<BytesRef> {
+        private Bits docsWithField;
+        private SortedDocValues values;
+
+        Keyword(String field) throws IOException {
+            super(field);
+        }
+
+        @Override
+        public BytesRef get(int doc) {
+            if (docsWithField.get(doc)) {
+                return values.get(doc);
+            } else {
+                return null;
+            }
+        }
+
+        @Override
+        public BytesRef copy(BytesRef value, BytesRef reuse) {
+            if (value == null) {
+                return null;
+            }
+            if (reuse != null) {
+                reuse.bytes = ArrayUtil.grow(reuse.bytes, value.length);
+                reuse.offset = 0;
+                reuse.length = value.length;
+                System.arraycopy(value.bytes, value.offset, reuse.bytes, 0, value.length);
+                return reuse;
+            } else {
+                return BytesRef.deepCopyOf(value);
+            }
+        }
+
+        @Override
+        public void setNextReader(LeafReader reader) throws IOException {
+            DocValuesType type = getDocValuesType(reader, field);
+            if (type == null || type == DocValuesType.NONE) {
+                values = DocValues.emptySorted();
+                docsWithField = new Bits.MatchNoBits(reader.maxDoc());
+                return ;
+            }
+            docsWithField = DocValues.getDocsWithField(reader, field);
+            switch (type) {
+                case SORTED:
+                    values = DocValues.getSorted(reader, field);
+                    break;
+
+                case SORTED_SET:
+                    final SortedSetDocValues sorted = DocValues.getSortedSet(reader, field);
+                    values = DocValues.unwrapSingleton(sorted);
+                    if (values == null) {
+                        values = new SortedDocValues() {
+                            @Override
+                            public int getOrd(int docID) {
+                                sorted.setDocument(docID);
+                                int ord = (int) sorted.nextOrd();
+                                if (sorted.nextOrd() != SortedSetDocValues.NO_MORE_ORDS) {
+                                    throw new IllegalStateException("failed to collapse " + docID +
+                                        ", the collapse field must be single valued");
+                                }
+                                return ord;
+                            }
+
+                            @Override
+                            public BytesRef lookupOrd(int ord) {
+                                return sorted.lookupOrd(ord);
+                            }
+
+                            @Override
+                            public int getValueCount() {
+                                return (int) sorted.getValueCount();
+                            }
+                        };
+                    }
+                    break;
+
+                default:
+                    throw new IllegalStateException("unexpected doc values type "
+                        + type + "` for field `" + field + "`");
+            }
+        }
+    }
+
+    private static DocValuesType getDocValuesType(LeafReader in, String field) {
+        FieldInfo fi = in.getFieldInfos().fieldInfo(field);
+        if (fi != null) {
+            return fi.getDocValuesType();
+        }
+        return null;
+    }
+}
diff --git a/core/src/main/java/org/apache/lucene/search/grouping/CollapsingTopDocsCollector.java b/core/src/main/java/org/apache/lucene/search/grouping/CollapsingTopDocsCollector.java
new file mode 100644
index 0000000000000..955a63e5483b7
--- /dev/null
+++ b/core/src/main/java/org/apache/lucene/search/grouping/CollapsingTopDocsCollector.java
@@ -0,0 +1,214 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lucene.search.grouping;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.FieldDoc;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.util.BytesRef;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Iterator;
+
+import static org.apache.lucene.search.SortField.Type.SCORE;
+
+/**
+ * A collector that groups documents based on field values and returns {@link CollapseTopFieldDocs}
+ * output. The collapsing is done in a single pass by selecting only the top sorted document per collapse key.
+ * The value used for the collapse key of each group can be found in {@link CollapseTopFieldDocs#collapseValues}.
+ */
+public abstract class CollapsingTopDocsCollector<T> extends FirstPassGroupingCollector<T> {
+    protected final String collapseField;
+
+    protected final Sort sort;
+    protected Scorer scorer;
+
+    private int totalHitCount;
+    private float maxScore;
+    private final boolean trackMaxScore;
+
+    private CollapsingTopDocsCollector(String collapseField, Sort sort,
+                                       int topN, boolean trackMaxScore) throws IOException {
+        super(sort, topN);
+        this.collapseField = collapseField;
+        this.trackMaxScore = trackMaxScore;
+        if (trackMaxScore) {
+            maxScore = Float.NEGATIVE_INFINITY;
+        } else {
+            maxScore = Float.NaN;
+        }
+        this.sort = sort;
+    }
+
+    /**
+     * Transform {@link FirstPassGroupingCollector#getTopGroups(int, boolean)} output in
+     * {@link CollapseTopFieldDocs}. The collapsing needs only one pass so we can create the final top docs at the end
+     * of the first pass.
+     */
+    public CollapseTopFieldDocs getTopDocs() {
+        Collection<SearchGroup<T>> groups = super.getTopGroups(0, true);
+        if (groups == null) {
+            return new CollapseTopFieldDocs(collapseField, totalHitCount, new ScoreDoc[0],
+                sort.getSort(), new Object[0], Float.NaN);
+        }
+        FieldDoc[] docs = new FieldDoc[groups.size()];
+        Object[] collapseValues = new Object[groups.size()];
+        int scorePos = -1;
+        for (int index = 0; index < sort.getSort().length; index++) {
+            SortField sortField = sort.getSort()[index];
+            if (sortField.getType() == SCORE) {
+                scorePos = index;
+                break;
+            }
+        }
+        int pos = 0;
+        Iterator<CollectedSearchGroup<T>> it = orderedGroups.iterator();
+        for (SearchGroup<T> group : groups) {
+            assert it.hasNext();
+            CollectedSearchGroup<T> col = it.next();
+            float score = Float.NaN;
+            if (scorePos != -1) {
+                score = (float) group.sortValues[scorePos];
+            }
+            docs[pos] = new FieldDoc(col.topDoc, score, group.sortValues);
+            collapseValues[pos] = group.groupValue;
+            pos++;
+        }
+        return new CollapseTopFieldDocs(collapseField, totalHitCount, docs, sort.getSort(),
+            collapseValues, maxScore);
+    }
+
+    @Override
+    public boolean needsScores() {
+        if (super.needsScores() == false) {
+            return trackMaxScore;
+        }
+        return true;
+    }
+
+    @Override
+    public void setScorer(Scorer scorer) throws IOException {
+        super.setScorer(scorer);
+        this.scorer = scorer;
+    }
+
+    @Override
+    public void collect(int doc) throws IOException {
+        super.collect(doc);
+        if (trackMaxScore) {
+            maxScore = Math.max(maxScore, scorer.score());
+        }
+        totalHitCount++;
+    }
+
+    private static class Numeric extends CollapsingTopDocsCollector<Long> {
+        private final CollapsingDocValuesSource.Numeric source;
+
+        private Numeric(String collapseField, Sort sort, int topN, boolean trackMaxScore) throws IOException {
+            super(collapseField, sort, topN, trackMaxScore);
+            source = new CollapsingDocValuesSource.Numeric(collapseField);
+        }
+
+        @Override
+        protected void doSetNextReader(LeafReaderContext readerContext) throws IOException {
+            super.doSetNextReader(readerContext);
+            source.setNextReader(readerContext.reader());
+        }
+
+        @Override
+        protected Long getDocGroupValue(int doc) {
+            return source.get(doc);
+        }
+
+        @Override
+        protected Long copyDocGroupValue(Long groupValue, Long reuse) {
+            return source.copy(groupValue, reuse);
+        }
+    }
+
+    private static class Keyword extends CollapsingTopDocsCollector<BytesRef> {
+        private final CollapsingDocValuesSource.Keyword source;
+
+        private Keyword(String collapseField, Sort sort, int topN, boolean trackMaxScore) throws IOException {
+            super(collapseField, sort, topN, trackMaxScore);
+            source = new CollapsingDocValuesSource.Keyword(collapseField);
+
+        }
+
+        @Override
+        protected void doSetNextReader(LeafReaderContext readerContext) throws IOException {
+            super.doSetNextReader(readerContext);
+            source.setNextReader(readerContext.reader());
+        }
+
+        @Override
+        protected BytesRef getDocGroupValue(int doc) {
+            return source.get(doc);
+        }
+
+        @Override
+        protected BytesRef copyDocGroupValue(BytesRef groupValue, BytesRef reuse) {
+            return source.copy(groupValue, reuse);
+        }
+    }
+
+    /**
+     * Create a collapsing top docs collector on a {@link org.apache.lucene.index.NumericDocValues} field.
+     * It accepts also {@link org.apache.lucene.index.SortedNumericDocValues} field but
+     * the collect will fail with an {@link IllegalStateException} if a document contains more than one value for the
+     * field.
+     *
+     * @param collapseField The sort field used to group
+     *                      documents.
+     * @param sort          The {@link Sort} used to sort the collapsed hits.
+     *                      The collapsing keeps only the top sorted document per collapsed key.
+     *                      This must be non-null, ie, if you want to groupSort by relevance
+     *                      use Sort.RELEVANCE.
+     * @param topN          How many top groups to keep.
+     * @throws IOException When I/O related errors occur
+     */
+    public static CollapsingTopDocsCollector<?> createNumeric(String collapseField, Sort sort,
+                                                              int topN, boolean trackMaxScore) throws IOException {
+        return new Numeric(collapseField, sort, topN, trackMaxScore);
+    }
+
+    /**
+     * Create a collapsing top docs collector on a {@link org.apache.lucene.index.SortedDocValues} field.
+     * It accepts also {@link org.apache.lucene.index.SortedSetDocValues} field but
+     * the collect will fail with an {@link IllegalStateException} if a document contains more than one value for the
+     * field.
+     *
+     * @param collapseField The sort field used to group
+     *                      documents.
+     * @param sort          The {@link Sort} used to sort the collapsed hits. The collapsing keeps only the top sorted
+     *                      document per collapsed key.
+     *                      This must be non-null, ie, if you want to groupSort by relevance use Sort.RELEVANCE.
+     * @param topN          How many top groups to keep.
+     * @throws IOException When I/O related errors occur
+     */
+    public static CollapsingTopDocsCollector<?> createKeyword(String collapseField, Sort sort,
+                                                              int topN, boolean trackMaxScore) throws IOException {
+        return new Keyword(collapseField, sort, topN, trackMaxScore);
+    }
+}
+
diff --git a/core/src/main/java/org/apache/lucene/search/postingshighlight/Snippet.java b/core/src/main/java/org/apache/lucene/search/highlight/Snippet.java
similarity index 96%
rename from core/src/main/java/org/apache/lucene/search/postingshighlight/Snippet.java
rename to core/src/main/java/org/apache/lucene/search/highlight/Snippet.java
index f3bfa1b9c652a..81a3d406ea346 100644
--- a/core/src/main/java/org/apache/lucene/search/postingshighlight/Snippet.java
+++ b/core/src/main/java/org/apache/lucene/search/highlight/Snippet.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.apache.lucene.search.postingshighlight;
+package org.apache.lucene.search.highlight;
 
 /**
  * Represents a scored highlighted snippet.
diff --git a/core/src/main/java/org/apache/lucene/search/postingshighlight/CustomPassageFormatter.java b/core/src/main/java/org/apache/lucene/search/postingshighlight/CustomPassageFormatter.java
index 889e7f741ed80..a33bf16dee4c7 100644
--- a/core/src/main/java/org/apache/lucene/search/postingshighlight/CustomPassageFormatter.java
+++ b/core/src/main/java/org/apache/lucene/search/postingshighlight/CustomPassageFormatter.java
@@ -19,6 +19,7 @@
 
 package org.apache.lucene.search.postingshighlight;
 
+import org.apache.lucene.search.highlight.Snippet;
 import org.apache.lucene.search.highlight.Encoder;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightUtils;
 
@@ -46,10 +47,10 @@ public Snippet[] format(Passage[] passages, String content) {
         for (int j = 0; j < passages.length; j++) {
             Passage passage = passages[j];
             StringBuilder sb = new StringBuilder();
-            pos = passage.startOffset;
-            for (int i = 0; i < passage.numMatches; i++) {
-                int start = passage.matchStarts[i];
-                int end = passage.matchEnds[i];
+            pos = passage.getStartOffset();
+            for (int i = 0; i < passage.getNumMatches(); i++) {
+                int start = passage.getMatchStarts()[i];
+                int end = passage.getMatchEnds()[i];
                 // its possible to have overlapping terms
                 if (start > pos) {
                     append(sb, content, pos, start);
@@ -62,7 +63,7 @@ public Snippet[] format(Passage[] passages, String content) {
                 }
             }
             // its possible a "term" from the analyzer could span a sentence boundary.
-            append(sb, content, pos, Math.max(pos, passage.endOffset));
+            append(sb, content, pos, Math.max(pos, passage.getEndOffset()));
             //we remove the paragraph separator if present at the end of the snippet (we used it as separator between values)
             if (sb.charAt(sb.length() - 1) == HighlightUtils.PARAGRAPH_SEPARATOR) {
                 sb.deleteCharAt(sb.length() - 1);
@@ -70,7 +71,7 @@ public Snippet[] format(Passage[] passages, String content) {
                 sb.deleteCharAt(sb.length() - 1);
             }
             //and we trim the snippets too
-            snippets[j] = new Snippet(sb.toString().trim(), passage.score, passage.numMatches > 0);
+            snippets[j] = new Snippet(sb.toString().trim(), passage.getScore(), passage.getNumMatches() > 0);
         }
         return snippets;
     }
diff --git a/core/src/main/java/org/apache/lucene/search/postingshighlight/CustomPostingsHighlighter.java b/core/src/main/java/org/apache/lucene/search/postingshighlight/CustomPostingsHighlighter.java
index 30f57b2626c4b..ac90a3e57aee7 100644
--- a/core/src/main/java/org/apache/lucene/search/postingshighlight/CustomPostingsHighlighter.java
+++ b/core/src/main/java/org/apache/lucene/search/postingshighlight/CustomPostingsHighlighter.java
@@ -22,6 +22,7 @@
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.highlight.Snippet;
 
 import java.io.IOException;
 import java.text.BreakIterator;
diff --git a/core/src/main/java/org/apache/lucene/search/suggest/analyzing/XAnalyzingSuggester.java b/core/src/main/java/org/apache/lucene/search/suggest/analyzing/XAnalyzingSuggester.java
index c65f962dbb8b0..5eaf63369b962 100644
--- a/core/src/main/java/org/apache/lucene/search/suggest/analyzing/XAnalyzingSuggester.java
+++ b/core/src/main/java/org/apache/lucene/search/suggest/analyzing/XAnalyzingSuggester.java
@@ -392,7 +392,7 @@ private static final class  EscapingTokenStreamToAutomaton extends TokenStreamTo
     final BytesRefBuilder spare = new BytesRefBuilder();
     private char sepLabel;
 
-    public EscapingTokenStreamToAutomaton(char sepLabel) {
+    EscapingTokenStreamToAutomaton(char sepLabel) {
       this.sepLabel = sepLabel;
     }
 
@@ -432,7 +432,7 @@ private static class AnalyzingComparator implements Comparator<BytesRef> {
 
     private final boolean hasPayloads;
 
-    public AnalyzingComparator(boolean hasPayloads) {
+    AnalyzingComparator(boolean hasPayloads) {
       this.hasPayloads = hasPayloads;
     }
 
@@ -1114,7 +1114,7 @@ private static final class SurfaceFormAndPayload implements Comparable<SurfaceFo
             BytesRef payload;
             long weight;
 
-            public SurfaceFormAndPayload(BytesRef payload, long cost) {
+            SurfaceFormAndPayload(BytesRef payload, long cost) {
                 super();
                 this.payload = payload;
                 this.weight = cost;
diff --git a/core/src/main/java/org/apache/lucene/search/uhighlight/BoundedBreakIteratorScanner.java b/core/src/main/java/org/apache/lucene/search/uhighlight/BoundedBreakIteratorScanner.java
new file mode 100644
index 0000000000000..1cd5fb9340d7f
--- /dev/null
+++ b/core/src/main/java/org/apache/lucene/search/uhighlight/BoundedBreakIteratorScanner.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lucene.search.uhighlight;
+
+import java.text.BreakIterator;
+import java.text.CharacterIterator;
+import java.util.Locale;
+
+/**
+ * A custom break iterator that scans text to find break-delimited passages bounded by
+ * a provided maximum length. This class delegates the boundary search to a first level
+ * break iterator. When this break iterator finds a passage greater than the maximum length
+ * a secondary break iterator is used to re-split the passage at the first boundary after
+ * maximum length.
+ * This is useful to split passages created by {@link BreakIterator}s like `sentence` that
+ * can create big outliers on semi-structured text.
+ *
+ * WARNING: This break iterator is designed to work with the {@link UnifiedHighlighter}.
+ **/
+public class BoundedBreakIteratorScanner extends BreakIterator {
+    private final BreakIterator mainBreak;
+    private final BreakIterator innerBreak;
+    private final int maxLen;
+
+    private int lastPrecedingOffset = -1;
+    private int windowStart = -1;
+    private int windowEnd = -1;
+    private int innerStart = -1;
+    private int innerEnd = 0;
+
+    private BoundedBreakIteratorScanner(BreakIterator mainBreak,
+                                        BreakIterator innerBreak,
+                                        int maxLen) {
+        this.mainBreak = mainBreak;
+        this.innerBreak = innerBreak;
+        this.maxLen = maxLen;
+    }
+
+    @Override
+    public CharacterIterator getText() {
+        return mainBreak.getText();
+    }
+
+    @Override
+    public void setText(CharacterIterator newText) {
+        reset();
+        mainBreak.setText(newText);
+        innerBreak.setText(newText);
+    }
+
+    @Override
+    public void setText(String newText) {
+        reset();
+        mainBreak.setText(newText);
+        innerBreak.setText(newText);
+    }
+
+    private void reset() {
+        lastPrecedingOffset = -1;
+        windowStart = -1;
+        windowEnd = -1;
+        innerStart = -1;
+        innerEnd = 0;
+    }
+
+    /**
+     * Must be called with increasing offset. See {@link FieldHighlighter} for usage.
+     */
+    @Override
+    public int preceding(int offset) {
+        if (offset < lastPrecedingOffset) {
+            throw new IllegalArgumentException("offset < lastPrecedingOffset: " +
+                "usage doesn't look like UnifiedHighlighter");
+        }
+        if (offset > windowStart && offset < windowEnd) {
+            innerStart = innerEnd;
+            innerEnd = windowEnd;
+        } else {
+            windowStart = innerStart = mainBreak.preceding(offset);
+            windowEnd = innerEnd = mainBreak.following(offset-1);
+        }
+
+        if (innerEnd - innerStart > maxLen) {
+            // the current split is too big,
+            // so starting from the current term we try to find boundaries on the left first
+            if (offset - maxLen > innerStart) {
+                innerStart = Math.max(innerStart,
+                    innerBreak.preceding(offset - maxLen));
+            }
+            // and then we try to expand the passage to the right with the remaining size
+            int remaining = Math.max(0, maxLen - (offset - innerStart));
+            if (offset + remaining < windowEnd) {
+                innerEnd = Math.min(windowEnd,
+                    innerBreak.following(offset + remaining));
+            }
+        }
+        lastPrecedingOffset = offset - 1;
+        return innerStart;
+    }
+
+    /**
+     * Can be invoked only after a call to preceding(offset+1).
+     * See {@link FieldHighlighter} for usage.
+     */
+    @Override
+    public int following(int offset) {
+        if (offset != lastPrecedingOffset || innerEnd == -1) {
+            throw new IllegalArgumentException("offset != lastPrecedingOffset: " +
+                "usage doesn't look like UnifiedHighlighter");
+        }
+        return innerEnd;
+    }
+
+    /**
+     * Returns a {@link BreakIterator#getSentenceInstance(Locale)} bounded to maxLen.
+     * Secondary boundaries are found using a {@link BreakIterator#getWordInstance(Locale)}.
+     */
+    public static BreakIterator getSentence(Locale locale, int maxLen) {
+        final BreakIterator sBreak = BreakIterator.getSentenceInstance(locale);
+        final BreakIterator wBreak = BreakIterator.getWordInstance(locale);
+        return new BoundedBreakIteratorScanner(sBreak, wBreak, maxLen);
+    }
+
+
+    @Override
+    public int current() {
+        // Returns the last offset of the current split
+        return this.innerEnd;
+    }
+
+    @Override
+    public int first() {
+        throw new IllegalStateException("first() should not be called in this context");
+    }
+
+    @Override
+    public int next() {
+        throw new IllegalStateException("next() should not be called in this context");
+    }
+
+    @Override
+    public int last() {
+        throw new IllegalStateException("last() should not be called in this context");
+    }
+
+    @Override
+    public int next(int n) {
+        throw new IllegalStateException("next(n) should not be called in this context");
+    }
+
+    @Override
+    public int previous() {
+        throw new IllegalStateException("previous() should not be called in this context");
+    }
+}
diff --git a/core/src/main/java/org/apache/lucene/search/uhighlight/CustomFieldHighlighter.java b/core/src/main/java/org/apache/lucene/search/uhighlight/CustomFieldHighlighter.java
new file mode 100644
index 0000000000000..915e7cc153128
--- /dev/null
+++ b/core/src/main/java/org/apache/lucene/search/uhighlight/CustomFieldHighlighter.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lucene.search.uhighlight;
+
+import java.text.BreakIterator;
+import java.util.Locale;
+
+import static org.apache.lucene.search.uhighlight.CustomUnifiedHighlighter.MULTIVAL_SEP_CHAR;
+
+/**
+ * Custom {@link FieldHighlighter} that creates a single passage bounded to {@code noMatchSize} when
+ * no highlights were found.
+ */
+class CustomFieldHighlighter extends FieldHighlighter {
+    private static final Passage[] EMPTY_PASSAGE = new Passage[0];
+
+    private final Locale breakIteratorLocale;
+    private final int noMatchSize;
+    private final String fieldValue;
+
+    CustomFieldHighlighter(String field, FieldOffsetStrategy fieldOffsetStrategy,
+                           Locale breakIteratorLocale, BreakIterator breakIterator,
+                           PassageScorer passageScorer, int maxPassages, int maxNoHighlightPassages,
+                           PassageFormatter passageFormatter, int noMatchSize, String fieldValue) {
+        super(field, fieldOffsetStrategy, breakIterator, passageScorer, maxPassages,
+            maxNoHighlightPassages, passageFormatter);
+        this.breakIteratorLocale = breakIteratorLocale;
+        this.noMatchSize = noMatchSize;
+        this.fieldValue = fieldValue;
+    }
+
+    @Override
+    protected Passage[] getSummaryPassagesNoHighlight(int maxPassages) {
+        if (noMatchSize > 0) {
+            int pos = 0;
+            while (pos < fieldValue.length() && fieldValue.charAt(pos) == MULTIVAL_SEP_CHAR) {
+                pos ++;
+            }
+            if (pos < fieldValue.length()) {
+                int end = fieldValue.indexOf(MULTIVAL_SEP_CHAR, pos);
+                if (end == -1) {
+                    end = fieldValue.length();
+                }
+                if (noMatchSize+pos < end) {
+                    BreakIterator bi = BreakIterator.getWordInstance(breakIteratorLocale);
+                    bi.setText(fieldValue);
+                    // Finds the next word boundary **after** noMatchSize.
+                    end = bi.following(noMatchSize + pos);
+                    if (end == BreakIterator.DONE) {
+                        end = fieldValue.length();
+                    }
+                }
+                Passage passage = new Passage();
+                passage.setScore(Float.NaN);
+                passage.setStartOffset(pos);
+                passage.setEndOffset(end);
+                return new Passage[]{passage};
+            }
+        }
+        return EMPTY_PASSAGE;
+    }
+}
diff --git a/core/src/main/java/org/apache/lucene/search/uhighlight/CustomPassageFormatter.java b/core/src/main/java/org/apache/lucene/search/uhighlight/CustomPassageFormatter.java
new file mode 100644
index 0000000000000..7a34a805db623
--- /dev/null
+++ b/core/src/main/java/org/apache/lucene/search/uhighlight/CustomPassageFormatter.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lucene.search.uhighlight;
+
+import org.apache.lucene.search.highlight.Encoder;
+import org.apache.lucene.search.highlight.Snippet;
+import org.elasticsearch.search.fetch.subphase.highlight.HighlightUtils;
+
+/**
+ * Custom passage formatter that allows us to:
+ * 1) extract different snippets (instead of a single big string) together with their scores ({@link Snippet})
+ * 2) use the {@link Encoder} implementations that are already used with the other highlighters
+ */
+public class CustomPassageFormatter extends PassageFormatter {
+
+    private final String preTag;
+    private final String postTag;
+    private final Encoder encoder;
+
+    public CustomPassageFormatter(String preTag, String postTag, Encoder encoder) {
+        this.preTag = preTag;
+        this.postTag = postTag;
+        this.encoder = encoder;
+    }
+
+    @Override
+    public Snippet[] format(Passage[] passages, String content) {
+        Snippet[] snippets = new Snippet[passages.length];
+        int pos;
+        for (int j = 0; j < passages.length; j++) {
+            Passage passage = passages[j];
+            StringBuilder sb = new StringBuilder();
+            pos = passage.getStartOffset();
+            for (int i = 0; i < passage.getNumMatches(); i++) {
+                int start = passage.getMatchStarts()[i];
+                int end = passage.getMatchEnds()[i];
+                // its possible to have overlapping terms
+                if (start > pos) {
+                    append(sb, content, pos, start);
+                }
+                if (end > pos) {
+                    sb.append(preTag);
+                    append(sb, content, Math.max(pos, start), end);
+                    sb.append(postTag);
+                    pos = end;
+                }
+            }
+            // its possible a "term" from the analyzer could span a sentence boundary.
+            append(sb, content, pos, Math.max(pos, passage.getEndOffset()));
+            //we remove the paragraph separator if present at the end of the snippet (we used it as separator between values)
+            if (sb.charAt(sb.length() - 1) == HighlightUtils.PARAGRAPH_SEPARATOR) {
+                sb.deleteCharAt(sb.length() - 1);
+            } else if (sb.charAt(sb.length() - 1) == HighlightUtils.NULL_SEPARATOR) {
+                sb.deleteCharAt(sb.length() - 1);
+            }
+            //and we trim the snippets too
+            snippets[j] = new Snippet(sb.toString().trim(), passage.getScore(), passage.getNumMatches() > 0);
+        }
+        return snippets;
+    }
+
+    private void append(StringBuilder dest, String content, int start, int end) {
+        dest.append(encoder.encodeText(content.substring(start, end)));
+    }
+}
diff --git a/core/src/main/java/org/apache/lucene/search/uhighlight/CustomUnifiedHighlighter.java b/core/src/main/java/org/apache/lucene/search/uhighlight/CustomUnifiedHighlighter.java
new file mode 100644
index 0000000000000..ef570033d0c8e
--- /dev/null
+++ b/core/src/main/java/org/apache/lucene/search/uhighlight/CustomUnifiedHighlighter.java
@@ -0,0 +1,238 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lucene.search.uhighlight;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.queries.CommonTermsQuery;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.PrefixQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.highlight.Snippet;
+import org.apache.lucene.search.spans.SpanMultiTermQueryWrapper;
+import org.apache.lucene.search.spans.SpanNearQuery;
+import org.apache.lucene.search.spans.SpanOrQuery;
+import org.apache.lucene.search.spans.SpanQuery;
+import org.apache.lucene.search.spans.SpanTermQuery;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.automaton.CharacterRunAutomaton;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.lucene.all.AllTermQuery;
+import org.elasticsearch.common.lucene.search.MultiPhrasePrefixQuery;
+import org.elasticsearch.common.lucene.search.function.FiltersFunctionScoreQuery;
+import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery;
+import org.elasticsearch.index.search.ESToParentBlockJoinQuery;
+
+import java.io.IOException;
+import java.text.BreakIterator;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Subclass of the {@link UnifiedHighlighter} that works for a single field in a single document.
+ * Uses a custom {@link PassageFormatter}. Accepts field content as a constructor
+ * argument, given that loadings field value can be done reading from _source field.
+ * Supports using different {@link BreakIterator} to break the text into fragments. Considers every distinct field
+ * value as a discrete passage for highlighting (unless the whole content needs to be highlighted).
+ * Supports both returning empty snippets and non highlighted snippets when no highlighting can be performed.
+ */
+public class CustomUnifiedHighlighter extends UnifiedHighlighter {
+    public static final char MULTIVAL_SEP_CHAR = (char) 0;
+    private static final Snippet[] EMPTY_SNIPPET = new Snippet[0];
+
+    private final OffsetSource offsetSource;
+    private final String fieldValue;
+    private final PassageFormatter passageFormatter;
+    private final BreakIterator breakIterator;
+    private final Locale breakIteratorLocale;
+    private final int noMatchSize;
+
+    /**
+     * Creates a new instance of {@link CustomUnifiedHighlighter}
+     *
+     * @param analyzer the analyzer used for the field at index time, used for multi term queries internally.
+     * @param passageFormatter our own {@link CustomPassageFormatter}
+     *                    which generates snippets in forms of {@link Snippet} objects.
+     * @param offsetSource the {@link OffsetSource} to used for offsets retrieval.
+     * @param breakIteratorLocale the {@link Locale} to use for dividing text into passages.
+     *                    If null {@link Locale#ROOT} is used.
+     * @param breakIterator the {@link BreakIterator} to use for dividing text into passages.
+     *                    If null {@link BreakIterator#getSentenceInstance(Locale)} is used.
+     * @param fieldValue the original field values delimited by MULTIVAL_SEP_CHAR.
+     * @param noMatchSize The size of the text that should be returned when no highlighting can be performed.
+     */
+    public CustomUnifiedHighlighter(IndexSearcher searcher,
+                                    Analyzer analyzer,
+                                    OffsetSource offsetSource,
+                                    PassageFormatter passageFormatter,
+                                    @Nullable Locale breakIteratorLocale,
+                                    @Nullable BreakIterator breakIterator,
+                                    String fieldValue,
+                                    int noMatchSize) {
+        super(searcher, analyzer);
+        this.offsetSource = offsetSource;
+        this.breakIterator = breakIterator;
+        this.breakIteratorLocale = breakIteratorLocale == null ? Locale.ROOT : breakIteratorLocale;
+        this.passageFormatter = passageFormatter;
+        this.fieldValue = fieldValue;
+        this.noMatchSize = noMatchSize;
+    }
+
+    /**
+     * Highlights terms extracted from the provided query within the content of the provided field name
+     */
+    public Snippet[] highlightField(String field, Query query, int docId, int maxPassages) throws IOException {
+        Map<String, Object[]> fieldsAsObjects = super.highlightFieldsAsObjects(new String[]{field}, query,
+            new int[]{docId}, new int[]{maxPassages});
+        Object[] snippetObjects = fieldsAsObjects.get(field);
+        if (snippetObjects != null) {
+            //one single document at a time
+            assert snippetObjects.length == 1;
+            Object snippetObject = snippetObjects[0];
+            if (snippetObject != null && snippetObject instanceof Snippet[]) {
+                return (Snippet[]) snippetObject;
+            }
+        }
+        return EMPTY_SNIPPET;
+    }
+
+    @Override
+    protected List<CharSequence[]> loadFieldValues(String[] fields, DocIdSetIterator docIter,
+                                                   int cacheCharsThreshold) throws IOException {
+        // we only highlight one field, one document at a time
+        return Collections.singletonList(new String[]{fieldValue});
+    }
+
+    @Override
+    protected BreakIterator getBreakIterator(String field) {
+        return breakIterator;
+    }
+
+    @Override
+    protected PassageFormatter getFormatter(String field) {
+        return passageFormatter;
+    }
+
+    @Override
+    protected FieldHighlighter getFieldHighlighter(String field, Query query, Set<Term> allTerms, int maxPassages) {
+        BytesRef[] terms = filterExtractedTerms(getFieldMatcher(field), allTerms);
+        Set<HighlightFlag> highlightFlags = getFlags(field);
+        PhraseHelper phraseHelper = getPhraseHelper(field, query, highlightFlags);
+        CharacterRunAutomaton[] automata = getAutomata(field, query, highlightFlags);
+        OffsetSource offsetSource = getOptimizedOffsetSource(field, terms, phraseHelper, automata);
+        BreakIterator breakIterator = new SplittingBreakIterator(getBreakIterator(field),
+            UnifiedHighlighter.MULTIVAL_SEP_CHAR);
+        FieldOffsetStrategy strategy =
+            getOffsetStrategy(offsetSource, field, terms, phraseHelper, automata, highlightFlags);
+        return new CustomFieldHighlighter(field, strategy, breakIteratorLocale, breakIterator,
+            getScorer(field), maxPassages, (noMatchSize > 0 ? 1 : 0), getFormatter(field), noMatchSize, fieldValue);
+    }
+
+    @Override
+    protected Collection<Query> preMultiTermQueryRewrite(Query query) {
+        return rewriteCustomQuery(query);
+    }
+
+    @Override
+    protected Collection<Query> preSpanQueryRewrite(Query query) {
+        return rewriteCustomQuery(query);
+    }
+
+    /**
+     * Translate custom queries in queries that are supported by the unified highlighter.
+     */
+    private Collection<Query> rewriteCustomQuery(Query query) {
+        if (query instanceof MultiPhrasePrefixQuery) {
+            MultiPhrasePrefixQuery mpq = (MultiPhrasePrefixQuery) query;
+            Term[][] terms = mpq.getTerms();
+            int[] positions = mpq.getPositions();
+            SpanQuery[] positionSpanQueries = new SpanQuery[positions.length];
+            int sizeMinus1 = terms.length - 1;
+            for (int i = 0; i < positions.length; i++) {
+                SpanQuery[] innerQueries = new SpanQuery[terms[i].length];
+                for (int j = 0; j < terms[i].length; j++) {
+                    if (i == sizeMinus1) {
+                        innerQueries[j] = new SpanMultiTermQueryWrapper(new PrefixQuery(terms[i][j]));
+                    } else {
+                        innerQueries[j] = new SpanTermQuery(terms[i][j]);
+                    }
+                }
+                if (innerQueries.length > 1) {
+                    positionSpanQueries[i] = new SpanOrQuery(innerQueries);
+                } else {
+                    positionSpanQueries[i] = innerQueries[0];
+                }
+            }
+
+            if (positionSpanQueries.length == 1) {
+                return Collections.singletonList(positionSpanQueries[0]);
+            }
+            // sum position increments beyond 1
+            int positionGaps = 0;
+            if (positions.length >= 2) {
+                // positions are in increasing order.   max(0,...) is just a safeguard.
+                positionGaps = Math.max(0, positions[positions.length - 1] - positions[0] - positions.length + 1);
+            }
+            //if original slop is 0 then require inOrder
+            boolean inorder = (mpq.getSlop() == 0);
+            return Collections.singletonList(new SpanNearQuery(positionSpanQueries,
+                mpq.getSlop() + positionGaps, inorder));
+        } else if (query instanceof CommonTermsQuery) {
+            CommonTermsQuery ctq = (CommonTermsQuery) query;
+            List<Query> tqs = new ArrayList<> ();
+            for (Term term : ctq.getTerms()) {
+                tqs.add(new TermQuery(term));
+            }
+            return tqs;
+        } else if (query instanceof AllTermQuery) {
+            AllTermQuery atq = (AllTermQuery) query;
+            return Collections.singletonList(new TermQuery(atq.getTerm()));
+        } else if (query instanceof FunctionScoreQuery) {
+            return Collections.singletonList(((FunctionScoreQuery) query).getSubQuery());
+        } else if (query instanceof FiltersFunctionScoreQuery) {
+            return Collections.singletonList(((FiltersFunctionScoreQuery) query).getSubQuery());
+        } else if (query instanceof ESToParentBlockJoinQuery) {
+            return Collections.singletonList(((ESToParentBlockJoinQuery) query).getChildQuery());
+        } else {
+            return null;
+        }
+    }
+
+    /**
+     * Forces the offset source for this highlighter
+     */
+    @Override
+    protected OffsetSource getOffsetSource(String field) {
+        if (offsetSource == null) {
+            return super.getOffsetSource(field);
+        }
+        return offsetSource;
+    }
+
+}
diff --git a/core/src/main/java/org/apache/lucene/search/vectorhighlight/CustomFieldQuery.java b/core/src/main/java/org/apache/lucene/search/vectorhighlight/CustomFieldQuery.java
index 88b045d2a5e90..1339873d3180d 100644
--- a/core/src/main/java/org/apache/lucene/search/vectorhighlight/CustomFieldQuery.java
+++ b/core/src/main/java/org/apache/lucene/search/vectorhighlight/CustomFieldQuery.java
@@ -28,12 +28,14 @@
 import org.apache.lucene.search.MultiPhraseQuery;
 import org.apache.lucene.search.PhraseQuery;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.SynonymQuery;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.join.ToParentBlockJoinQuery;
 import org.apache.lucene.search.spans.SpanTermQuery;
 import org.elasticsearch.common.lucene.search.MultiPhrasePrefixQuery;
 import org.elasticsearch.common.lucene.search.function.FiltersFunctionScoreQuery;
 import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery;
+import org.elasticsearch.index.search.ESToParentBlockJoinQuery;
 
 import java.io.IOException;
 import java.util.Collection;
@@ -79,8 +81,8 @@ void flatten(Query sourceQuery, IndexReader reader, Collection<Query> flatQuerie
         } else if (sourceQuery instanceof BlendedTermQuery) {
             final BlendedTermQuery blendedTermQuery = (BlendedTermQuery) sourceQuery;
             flatten(blendedTermQuery.rewrite(reader), reader, flatQueries, boost);
-        } else if (sourceQuery instanceof ToParentBlockJoinQuery) {
-            ToParentBlockJoinQuery blockJoinQuery = (ToParentBlockJoinQuery) sourceQuery;
+        } else if (sourceQuery instanceof ESToParentBlockJoinQuery) {
+            ESToParentBlockJoinQuery blockJoinQuery = (ESToParentBlockJoinQuery) sourceQuery;
             flatten(blockJoinQuery.getChildQuery(), reader, flatQueries, boost);
         } else if (sourceQuery instanceof BoostingQuery) {
             BoostingQuery boostingQuery = (BoostingQuery) sourceQuery;
@@ -88,6 +90,18 @@ void flatten(Query sourceQuery, IndexReader reader, Collection<Query> flatQuerie
             flatten(boostingQuery.getMatch(), reader, flatQueries, boost);
             //flatten negative query with negative boost
             flatten(boostingQuery.getContext(), reader, flatQueries, boostingQuery.getBoost());
+        } else if (sourceQuery instanceof SynonymQuery) {
+            // SynonymQuery should be handled by the parent class directly.
+            // This statement should be removed when https://issues.apache.org/jira/browse/LUCENE-7484 is merged.
+            SynonymQuery synQuery = (SynonymQuery) sourceQuery;
+            for (Term term : synQuery.getTerms()) {
+                flatten(new TermQuery(term), reader, flatQueries, boost);
+            }
+        } else if (sourceQuery instanceof ESToParentBlockJoinQuery) {
+            Query childQuery = ((ESToParentBlockJoinQuery) sourceQuery).getChildQuery();
+            if (childQuery != null) {
+                flatten(childQuery, reader, flatQueries, boost);
+            }
         } else {
             super.flatten(sourceQuery, reader, flatQueries, boost);
         }
diff --git a/core/src/main/java/org/apache/lucene/spatial/geopoint/search/XGeoPointDistanceRangeQuery.java b/core/src/main/java/org/apache/lucene/spatial/geopoint/search/XGeoPointDistanceRangeQuery.java
index 3cf290e035ea8..2606b43af8c45 100644
--- a/core/src/main/java/org/apache/lucene/spatial/geopoint/search/XGeoPointDistanceRangeQuery.java
+++ b/core/src/main/java/org/apache/lucene/spatial/geopoint/search/XGeoPointDistanceRangeQuery.java
@@ -1,18 +1,20 @@
 /*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ *    http://www.apache.org/licenses/LICENSE-2.0
  *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
  */
 package org.apache.lucene.spatial.geopoint.search;
 
diff --git a/core/src/main/java/org/apache/lucene/store/StoreRateLimiting.java b/core/src/main/java/org/apache/lucene/store/StoreRateLimiting.java
index e1ae7b938b3de..2dff741392b61 100644
--- a/core/src/main/java/org/apache/lucene/store/StoreRateLimiting.java
+++ b/core/src/main/java/org/apache/lucene/store/StoreRateLimiting.java
@@ -20,12 +20,16 @@
 
 import org.apache.lucene.store.RateLimiter.SimpleRateLimiter;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.unit.ByteSizeValue;
 
 /**
  */
 public class StoreRateLimiting {
 
+    private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(StoreRateLimiting.class));
+
     public interface Provider {
 
         StoreRateLimiting rateLimiting();
@@ -68,14 +72,14 @@ public RateLimiter getRateLimiter() {
     }
 
     public void setMaxRate(ByteSizeValue rate) {
-        if (rate.bytes() <= 0) {
+        if (rate.getBytes() <= 0) {
             actualRateLimiter = null;
         } else if (actualRateLimiter == null) {
             actualRateLimiter = rateLimiter;
-            actualRateLimiter.setMBPerSec(rate.mbFrac());
+            actualRateLimiter.setMBPerSec(rate.getMbFrac());
         } else {
             assert rateLimiter == actualRateLimiter;
-            rateLimiter.setMBPerSec(rate.mbFrac());
+            rateLimiter.setMBPerSec(rate.getMbFrac());
         }
     }
 
@@ -85,9 +89,12 @@ public Type getType() {
 
     public void setType(Type type) {
         this.type = type;
+        if (type != Type.NONE) {
+            DEPRECATION_LOGGER.deprecated("Store rate limiting is deprecated and will be removed in 6.0");
+        }
     }
 
     public void setType(String type) {
-        this.type = Type.fromString(type);
+        setType(Type.fromString(type));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/Assertions.java b/core/src/main/java/org/elasticsearch/Assertions.java
new file mode 100644
index 0000000000000..8783101db0a88
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/Assertions.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch;
+
+/**
+ * Provides a static final field that can be used to check if assertions are enabled. Since this field might be used elsewhere to check if
+ * assertions are enabled, if you are running with assertions enabled for specific packages or classes, you should enable assertions on this
+ * class too (e.g., {@code -ea org.elasticsearch.Assertions -ea org.elasticsearch.cluster.service.MasterService}).
+ */
+public final class Assertions {
+
+    private Assertions() {
+
+    }
+
+    public static final boolean ENABLED;
+
+    static {
+        boolean enabled = false;
+        /*
+         * If assertions are enabled, the following line will be evaluated and enabled will have the value true, otherwise when assertions
+         * are disabled enabled will have the value false.
+         */
+        // noinspection ConstantConditions,AssertWithSideEffects
+        assert enabled = true;
+        // noinspection ConstantConditions
+        ENABLED = enabled;
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/Build.java b/core/src/main/java/org/elasticsearch/Build.java
index 25da5f281665f..7fbf09b649bfc 100644
--- a/core/src/main/java/org/elasticsearch/Build.java
+++ b/core/src/main/java/org/elasticsearch/Build.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch;
 
+import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 
@@ -42,8 +43,10 @@ public class Build {
         final String date;
         final boolean isSnapshot;
 
+        final String esPrefix = "elasticsearch-" + Version.CURRENT;
         final URL url = getElasticsearchCodebase();
-        if (url.toString().endsWith(".jar")) {
+        final String urlStr = url.toString();
+        if (urlStr.startsWith("file:/") && (urlStr.endsWith(esPrefix + ".jar") || urlStr.endsWith(esPrefix + "-SNAPSHOT.jar"))) {
             try (JarInputStream jar = new JarInputStream(url.openStream())) {
                 Manifest manifest = jar.getManifest();
                 shortHash = manifest.getMainAttributes().getValue("Change");
@@ -53,10 +56,21 @@ public class Build {
                 throw new RuntimeException(e);
             }
         } else {
-            // not running from a jar (unit tests, IDE)
+            // not running from the official elasticsearch jar file (unit tests, IDE, uber client jar, shadiness)
             shortHash = "Unknown";
             date = "Unknown";
-            isSnapshot = true;
+            final String buildSnapshot = System.getProperty("build.snapshot");
+            if (buildSnapshot != null) {
+                try {
+                    Class.forName("com.carrotsearch.randomizedtesting.RandomizedContext");
+                } catch (final ClassNotFoundException e) {
+                    // we are not in tests but build.snapshot is set, bail hard
+                    throw new IllegalStateException("build.snapshot set to [" + buildSnapshot + "] but not running tests");
+                }
+                isSnapshot = Booleans.parseBooleanExact(buildSnapshot);
+            } else {
+                isSnapshot = true;
+            }
         }
         if (shortHash == null) {
             throw new IllegalStateException("Error finding the build shortHash. " +
@@ -79,10 +93,10 @@ static URL getElasticsearchCodebase() {
         return Build.class.getProtectionDomain().getCodeSource().getLocation();
     }
 
-    private String shortHash;
-    private String date;
+    private final String shortHash;
+    private final String date;
 
-    Build(String shortHash, String date, boolean isSnapshot) {
+    public Build(String shortHash, String date, boolean isSnapshot) {
         this.shortHash = shortHash;
         this.date = date;
         this.isSnapshot = isSnapshot;
diff --git a/core/src/main/java/org/elasticsearch/ElasticsearchException.java b/core/src/main/java/org/elasticsearch/ElasticsearchException.java
index 750f133ea1705..4e2821019b8a8 100644
--- a/core/src/main/java/org/elasticsearch/ElasticsearchException.java
+++ b/core/src/main/java/org/elasticsearch/ElasticsearchException.java
@@ -19,49 +19,79 @@
 
 package org.elasticsearch;
 
-import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.elasticsearch.action.support.replication.ReplicationOperation;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
+import org.elasticsearch.common.CheckedFunction;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.logging.LoggerMessageFormat;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.transport.TcpTransport;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.singletonMap;
 import static java.util.Collections.unmodifiableMap;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.INDEX_UUID_NA_VALUE;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureFieldName;
 
 /**
  * A base class for all elasticsearch exceptions.
  */
 public class ElasticsearchException extends RuntimeException implements ToXContent, Writeable {
 
-    public static final String REST_EXCEPTION_SKIP_CAUSE = "rest.exception.cause.skip";
+    private static final Version UNKNOWN_VERSION_ADDED = Version.fromId(0);
+
+    /**
+     * Passed in the {@link Params} of {@link #generateThrowableXContent(XContentBuilder, Params, Throwable)}
+     * to control if the {@code caused_by} element should render. Unlike most parameters to {@code toXContent} methods this parameter is
+     * internal only and not available as a URL parameter.
+     */
+    private static final String REST_EXCEPTION_SKIP_CAUSE = "rest.exception.cause.skip";
+    /**
+     * Passed in the {@link Params} of {@link #generateThrowableXContent(XContentBuilder, Params, Throwable)}
+     * to control if the {@code stack_trace} element should render. Unlike most parameters to {@code toXContent} methods this parameter is
+     * internal only and not available as a URL parameter. Use the {@code error_trace} parameter instead.
+     */
     public static final String REST_EXCEPTION_SKIP_STACK_TRACE = "rest.exception.stacktrace.skip";
     public static final boolean REST_EXCEPTION_SKIP_STACK_TRACE_DEFAULT = true;
-    public static final boolean REST_EXCEPTION_SKIP_CAUSE_DEFAULT = false;
-    private static final String INDEX_HEADER_KEY = "es.index";
-    private static final String INDEX_HEADER_KEY_UUID = "es.index_uuid";
-    private static final String SHARD_HEADER_KEY = "es.shard";
-    private static final String RESOURCE_HEADER_TYPE_KEY = "es.resource.type";
-    private static final String RESOURCE_HEADER_ID_KEY = "es.resource.id";
-
-    private static final Map<Integer, FunctionThatThrowsIOException<StreamInput, ? extends ElasticsearchException>> ID_TO_SUPPLIER;
+    private static final boolean REST_EXCEPTION_SKIP_CAUSE_DEFAULT = false;
+    private static final String INDEX_METADATA_KEY = "es.index";
+    private static final String INDEX_METADATA_KEY_UUID = "es.index_uuid";
+    private static final String SHARD_METADATA_KEY = "es.shard";
+    private static final String RESOURCE_METADATA_TYPE_KEY = "es.resource.type";
+    private static final String RESOURCE_METADATA_ID_KEY = "es.resource.id";
+
+    private static final String TYPE = "type";
+    private static final String REASON = "reason";
+    private static final String CAUSED_BY = "caused_by";
+    private static final String STACK_TRACE = "stack_trace";
+    private static final String HEADER = "header";
+    private static final String ERROR = "error";
+    private static final String ROOT_CAUSE = "root_cause";
+
+    private static final Map<Integer, CheckedFunction<StreamInput, ? extends ElasticsearchException, IOException>> ID_TO_SUPPLIER;
     private static final Map<Class<? extends ElasticsearchException>, ElasticsearchExceptionHandle> CLASS_TO_ELASTICSEARCH_EXCEPTION_HANDLE;
+    private final Map<String, List<String>> metadata = new HashMap<>();
     private final Map<String, List<String>> headers = new HashMap<>();
 
     /**
@@ -103,14 +133,56 @@ public ElasticsearchException(StreamInput in) throws IOException {
         super(in.readOptionalString(), in.readException());
         readStackTrace(this, in);
         headers.putAll(in.readMapOfLists(StreamInput::readString, StreamInput::readString));
+        if (in.getVersion().onOrAfter(Version.V_5_3_0)) {
+            metadata.putAll(in.readMapOfLists(StreamInput::readString, StreamInput::readString));
+        } else {
+            for (Iterator<Map.Entry<String, List<String>>> iterator = headers.entrySet().iterator(); iterator.hasNext(); ) {
+                Map.Entry<String, List<String>> header = iterator.next();
+                if (header.getKey().startsWith("es.")) {
+                    metadata.put(header.getKey(), header.getValue());
+                    iterator.remove();
+                }
+            }
+        }
     }
 
     /**
-     * Adds a new header with the given key.
-     * This method will replace existing header if a header with the same key already exists
+     * Adds a new piece of metadata with the given key.
+     * If the provided key is already present, the corresponding metadata will be replaced
      */
-    public void addHeader(String key, String... value) {
-        this.headers.put(key, Arrays.asList(value));
+    public void addMetadata(String key, String... values) {
+        addMetadata(key, Arrays.asList(values));
+    }
+
+    /**
+     * Adds a new piece of metadata with the given key.
+     * If the provided key is already present, the corresponding metadata will be replaced
+     */
+    public void addMetadata(String key, List<String> values) {
+        //we need to enforce this otherwise bw comp doesn't work properly, as "es." was the previous criteria to split headers in two sets
+        if (key.startsWith("es.") == false) {
+            throw new IllegalArgumentException("exception metadata must start with [es.], found [" + key + "] instead");
+        }
+        this.metadata.put(key, values);
+    }
+
+    /**
+     * Returns a set of all metadata keys on this exception
+     */
+    public Set<String> getMetadataKeys() {
+        return metadata.keySet();
+    }
+
+    /**
+     * Returns the list of metadata values for the given key or {@code null} if no metadata for the
+     * given key exists.
+     */
+    public List<String> getMetadata(String key) {
+        return metadata.get(key);
+    }
+
+    protected Map<String, List<String>> getMetadata() {
+        return metadata;
     }
 
     /**
@@ -118,9 +190,20 @@ public void addHeader(String key, String... value) {
      * This method will replace existing header if a header with the same key already exists
      */
     public void addHeader(String key, List<String> value) {
+        //we need to enforce this otherwise bw comp doesn't work properly, as "es." was the previous criteria to split headers in two sets
+        if (key.startsWith("es.")) {
+            throw new IllegalArgumentException("exception headers must not start with [es.], found [" + key + "] instead");
+        }
         this.headers.put(key, value);
     }
 
+    /**
+     * Adds a new header with the given key.
+     * This method will replace existing header if a header with the same key already exists
+     */
+    public void addHeader(String key, String... value) {
+        addHeader(key, Arrays.asList(value));
+    }
 
     /**
      * Returns a set of all header keys on this exception
@@ -130,13 +213,17 @@ public Set<String> getHeaderKeys() {
     }
 
     /**
-     * Returns the list of header values for the given key or {@code null} if not header for the
+     * Returns the list of header values for the given key or {@code null} if no header for the
      * given key exists.
      */
     public List<String> getHeader(String key) {
         return headers.get(key);
     }
 
+    protected Map<String, List<String>> getHeaders() {
+        return headers;
+    }
+
     /**
      * Returns the rest status code associated with this exception.
      */
@@ -197,11 +284,19 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalString(this.getMessage());
         out.writeException(this.getCause());
         writeStackTraces(this, out);
-        out.writeMapOfLists(headers, StreamOutput::writeString, StreamOutput::writeString);
+        if (out.getVersion().onOrAfter(Version.V_5_3_0)) {
+            out.writeMapOfLists(headers, StreamOutput::writeString, StreamOutput::writeString);
+            out.writeMapOfLists(metadata, StreamOutput::writeString, StreamOutput::writeString);
+        } else {
+            HashMap<String, List<String>> finalHeaders = new HashMap<>(headers.size() + metadata.size());
+            finalHeaders.putAll(headers);
+            finalHeaders.putAll(metadata);
+            out.writeMapOfLists(finalHeaders, StreamOutput::writeString, StreamOutput::writeString);
+        }
     }
 
     public static ElasticsearchException readException(StreamInput input, int id) throws IOException {
-        FunctionThatThrowsIOException<StreamInput, ? extends ElasticsearchException> elasticsearchException = ID_TO_SUPPLIER.get(id);
+        CheckedFunction<StreamInput, ? extends ElasticsearchException, IOException> elasticsearchException = ID_TO_SUPPLIER.get(id);
         if (elasticsearchException == null) {
             throw new IllegalStateException("unknown exception for id: " + id);
         }
@@ -211,8 +306,12 @@ public static ElasticsearchException readException(StreamInput input, int id) th
     /**
      * Returns <code>true</code> iff the given class is a registered for an exception to be read.
      */
-    public static boolean isRegistered(Class<? extends Throwable> exception) {
-        return CLASS_TO_ELASTICSEARCH_EXCEPTION_HANDLE.containsKey(exception);
+    public static boolean isRegistered(Class<? extends Throwable> exception, Version version) {
+        ElasticsearchExceptionHandle elasticsearchExceptionHandle = CLASS_TO_ELASTICSEARCH_EXCEPTION_HANDLE.get(exception);
+        if (elasticsearchExceptionHandle != null) {
+            return version.onOrAfter(elasticsearchExceptionHandle.versionAdded);
+        }
+        return false;
     }
 
     static Set<Class<? extends ElasticsearchException>> getRegisteredKeys() { // for testing
@@ -230,64 +329,51 @@ public static int getId(Class<? extends ElasticsearchException> exception) {
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         Throwable ex = ExceptionsHelper.unwrapCause(this);
         if (ex != this) {
-            toXContent(builder, params, this);
+            generateThrowableXContent(builder, params, this);
         } else {
-            builder.field("type", getExceptionName());
-            builder.field("reason", getMessage());
-            for (String key : headers.keySet()) {
-                if (key.startsWith("es.")) {
-                    List<String> values = headers.get(key);
-                    xContentHeader(builder, key.substring("es.".length()), values);
-                }
-            }
-            innerToXContent(builder, params);
-            renderHeader(builder, params);
-            if (params.paramAsBoolean(REST_EXCEPTION_SKIP_STACK_TRACE, REST_EXCEPTION_SKIP_STACK_TRACE_DEFAULT) == false) {
-                builder.field("stack_trace", ExceptionsHelper.stackTrace(this));
-            }
+            innerToXContent(builder, params, this, getExceptionName(), getMessage(), headers, metadata, getCause());
         }
         return builder;
     }
 
-    /**
-     * Renders additional per exception information into the xcontent
-     */
-    protected void innerToXContent(XContentBuilder builder, Params params) throws IOException {
-        causeToXContent(builder, params);
-    }
+    protected static void innerToXContent(XContentBuilder builder, Params params,
+                                          Throwable throwable, String type, String message, Map<String, List<String>> headers,
+                                          Map<String, List<String>> metadata, Throwable cause) throws IOException {
+        builder.field(TYPE, type);
+        builder.field(REASON, message);
 
-    /**
-     * Renders a cause exception as xcontent
-     */
-    protected void causeToXContent(XContentBuilder builder, Params params) throws IOException {
-        final Throwable cause = getCause();
-        if (cause != null && params.paramAsBoolean(REST_EXCEPTION_SKIP_CAUSE, REST_EXCEPTION_SKIP_CAUSE_DEFAULT) == false) {
-            builder.field("caused_by");
-            builder.startObject();
-            toXContent(builder, params, cause);
-            builder.endObject();
+        for (Map.Entry<String, List<String>> entry : metadata.entrySet()) {
+            headerToXContent(builder, entry.getKey().substring("es.".length()), entry.getValue());
         }
-    }
 
-    protected final void renderHeader(XContentBuilder builder, Params params) throws IOException {
-        boolean hasHeader = false;
-        for (String key : headers.keySet()) {
-            if (key.startsWith("es.")) {
-                continue;
-            }
-            if (hasHeader == false) {
-                builder.startObject("header");
-                hasHeader = true;
+        if (throwable instanceof ElasticsearchException) {
+            ElasticsearchException exception = (ElasticsearchException) throwable;
+            exception.metadataToXContent(builder, params);
+        }
+
+        if (params.paramAsBoolean(REST_EXCEPTION_SKIP_CAUSE, REST_EXCEPTION_SKIP_CAUSE_DEFAULT) == false) {
+            if (cause != null) {
+                builder.field(CAUSED_BY);
+                builder.startObject();
+                generateThrowableXContent(builder, params, cause);
+                builder.endObject();
             }
-            List<String> values = headers.get(key);
-            xContentHeader(builder, key, values);
         }
-        if (hasHeader) {
+
+        if (headers.isEmpty() == false) {
+            builder.startObject(HEADER);
+            for (Map.Entry<String, List<String>> entry : headers.entrySet()) {
+                headerToXContent(builder, entry.getKey(), entry.getValue());
+            }
             builder.endObject();
         }
+
+        if (params.paramAsBoolean(REST_EXCEPTION_SKIP_STACK_TRACE, REST_EXCEPTION_SKIP_STACK_TRACE_DEFAULT) == false) {
+            builder.field(STACK_TRACE, ExceptionsHelper.stackTrace(throwable));
+        }
     }
 
-    private void xContentHeader(XContentBuilder builder, String key, List<String> values) throws IOException {
+    private static void headerToXContent(XContentBuilder builder, String key, List<String> values) throws IOException {
         if (values != null && values.isEmpty() == false) {
             if (values.size() == 1) {
                 builder.field(key, values.get(0));
@@ -302,25 +388,210 @@ private void xContentHeader(XContentBuilder builder, String key, List<String> va
     }
 
     /**
-     * Statis toXContent helper method that also renders non {@link org.elasticsearch.ElasticsearchException} instances as XContent.
+     * Renders additional per exception information into the XContent
      */
-    public static void toXContent(XContentBuilder builder, Params params, Throwable ex) throws IOException {
-        ex = ExceptionsHelper.unwrapCause(ex);
-        if (ex instanceof ElasticsearchException) {
-            ((ElasticsearchException) ex).toXContent(builder, params);
+    protected void metadataToXContent(XContentBuilder builder, Params params) throws IOException {
+    }
+
+    /**
+     * Generate a {@link ElasticsearchException} from a {@link XContentParser}. This does not
+     * return the original exception type (ie NodeClosedException for example) but just wraps
+     * the type, the reason and the cause of the exception. It also recursively parses the
+     * tree structure of the cause, returning it as a tree structure of {@link ElasticsearchException}
+     * instances.
+     */
+    public static ElasticsearchException fromXContent(XContentParser parser) throws IOException {
+        XContentParser.Token token = parser.nextToken();
+        ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser::getTokenLocation);
+        return innerFromXContent(parser, false);
+    }
+
+    private static ElasticsearchException innerFromXContent(XContentParser parser, boolean parseRootCauses) throws IOException {
+        XContentParser.Token token = parser.currentToken();
+        ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser::getTokenLocation);
+
+        String type = null, reason = null, stack = null;
+        ElasticsearchException cause = null;
+        Map<String, List<String>> metadata = new HashMap<>();
+        Map<String, List<String>> headers = new HashMap<>();
+        List<ElasticsearchException> rootCauses = new ArrayList<>();
+
+        for (; token == XContentParser.Token.FIELD_NAME; token = parser.nextToken()) {
+            String currentFieldName = parser.currentName();
+            token = parser.nextToken();
+
+            if (token.isValue()) {
+                if (TYPE.equals(currentFieldName)) {
+                    type = parser.text();
+                } else if (REASON.equals(currentFieldName)) {
+                    reason = parser.text();
+                } else if (STACK_TRACE.equals(currentFieldName)) {
+                    stack = parser.text();
+                } else if (token == XContentParser.Token.VALUE_STRING) {
+                    metadata.put(currentFieldName, Collections.singletonList(parser.text()));
+                }
+            } else if (token == XContentParser.Token.START_OBJECT) {
+                if (CAUSED_BY.equals(currentFieldName)) {
+                    cause = fromXContent(parser);
+                } else if (HEADER.equals(currentFieldName)) {
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                        if (token == XContentParser.Token.FIELD_NAME) {
+                            currentFieldName = parser.currentName();
+                        } else {
+                            List<String> values = headers.getOrDefault(currentFieldName, new ArrayList<>());
+                            if (token == XContentParser.Token.VALUE_STRING) {
+                                values.add(parser.text());
+                            } else if (token == XContentParser.Token.START_ARRAY) {
+                                while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                                    if (token == XContentParser.Token.VALUE_STRING) {
+                                        values.add(parser.text());
+                                    } else {
+                                        parser.skipChildren();
+                                    }
+                                }
+                            } else if (token == XContentParser.Token.START_OBJECT) {
+                                parser.skipChildren();
+                            }
+                            headers.put(currentFieldName, values);
+                        }
+                    }
+                } else {
+                    // Any additional metadata object added by the metadataToXContent method is ignored
+                    // and skipped, so that the parser does not fail on unknown fields. The parser only
+                    // support metadata key-pairs and metadata arrays of values.
+                    parser.skipChildren();
+                }
+            } else if (token == XContentParser.Token.START_ARRAY) {
+                if (parseRootCauses && ROOT_CAUSE.equals(currentFieldName)) {
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                        rootCauses.add(fromXContent(parser));
+                    }
+                } else {
+                    // Parse the array and add each item to the corresponding list of metadata.
+                    // Arrays of objects are not supported yet and just ignored and skipped.
+                    List<String> values = new ArrayList<>();
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                        if (token == XContentParser.Token.VALUE_STRING) {
+                            values.add(parser.text());
+                        } else {
+                            parser.skipChildren();
+                        }
+                    }
+                    if (values.size() > 0) {
+                        if (metadata.containsKey(currentFieldName)) {
+                            values.addAll(metadata.get(currentFieldName));
+                        }
+                        metadata.put(currentFieldName, values);
+                    }
+                }
+            }
+        }
+
+        ElasticsearchException e = new ElasticsearchException(buildMessage(type, reason, stack), cause);
+        for (Map.Entry<String, List<String>> entry : metadata.entrySet()) {
+            //subclasses can print out additional metadata through the metadataToXContent method. Simple key-value pairs will be
+            //parsed back and become part of this metadata set, while objects and arrays are not supported when parsing back.
+            //Those key-value pairs become part of the metadata set and inherit the "es." prefix as that is currently required
+            //by addMetadata. The prefix will get stripped out when printing metadata out so it will be effectively invisible.
+            //TODO move subclasses that print out simple metadata to using addMetadata directly and support also numbers and booleans.
+            //TODO rename metadataToXContent and have only SearchPhaseExecutionException use it, which prints out complex objects
+            e.addMetadata("es." + entry.getKey(), entry.getValue());
+        }
+        for (Map.Entry<String, List<String>> header : headers.entrySet()) {
+            e.addHeader(header.getKey(), header.getValue());
+        }
+
+        // Adds root causes as suppressed exception. This way they are not lost
+        // after parsing and can be retrieved using getSuppressed() method.
+        for (ElasticsearchException rootCause : rootCauses) {
+            e.addSuppressed(rootCause);
+        }
+        return e;
+    }
+
+    /**
+     * Static toXContent helper method that renders {@link org.elasticsearch.ElasticsearchException} or {@link Throwable} instances
+     * as XContent, delegating the rendering to {@link #toXContent(XContentBuilder, Params)}
+     * or {@link #innerToXContent(XContentBuilder, Params, Throwable, String, String, Map, Map, Throwable)}.
+     *
+     * This method is usually used when the {@link Throwable} is rendered as a part of another XContent object, and its result can
+     * be parsed back using the {@link #fromXContent(XContentParser)} method.
+     */
+    public static void generateThrowableXContent(XContentBuilder builder, Params params, Throwable t) throws IOException {
+        t = ExceptionsHelper.unwrapCause(t);
+
+        if (t instanceof ElasticsearchException) {
+            ((ElasticsearchException) t).toXContent(builder, params);
         } else {
-            builder.field("type", getExceptionName(ex));
-            builder.field("reason", ex.getMessage());
-            if (ex.getCause() != null) {
-                builder.field("caused_by");
+            innerToXContent(builder, params, t, getExceptionName(t), t.getMessage(), emptyMap(), emptyMap(), t.getCause());
+        }
+    }
+
+    /**
+     * Render any exception as a xcontent, encapsulated within a field or object named "error". The level of details that are rendered
+     * depends on the value of the "detailed" parameter: when it's false only a simple message based on the type and message of the
+     * exception is rendered. When it's true all detail are provided including guesses root causes, cause and potentially stack
+     * trace.
+     *
+     * This method is usually used when the {@link Exception} is rendered as a full XContent object, and its output can be parsed
+     * by the {@link #failureFromXContent(XContentParser)} method.
+     */
+    public static void generateFailureXContent(XContentBuilder builder, Params params, @Nullable Exception e, boolean detailed)
+            throws IOException {
+        // No exception to render as an error
+        if (e == null) {
+            builder.field(ERROR, "unknown");
+            return;
+        }
+
+        // Render the exception with a simple message
+        if (detailed == false) {
+            String message = "No ElasticsearchException found";
+            Throwable t = e;
+            for (int counter = 0; counter < 10 && t != null; counter++) {
+                if (t instanceof ElasticsearchException) {
+                    message = t.getClass().getSimpleName() + "[" + t.getMessage() + "]";
+                    break;
+                }
+                t = t.getCause();
+            }
+            builder.field(ERROR, message);
+            return;
+        }
+
+        // Render the exception with all details
+        final ElasticsearchException[] rootCauses = ElasticsearchException.guessRootCauses(e);
+        builder.startObject(ERROR);
+        {
+            builder.startArray(ROOT_CAUSE);
+            for (ElasticsearchException rootCause : rootCauses) {
                 builder.startObject();
-                toXContent(builder, params, ex.getCause());
+                rootCause.toXContent(builder, new DelegatingMapParams(singletonMap(REST_EXCEPTION_SKIP_CAUSE, "true"), params));
                 builder.endObject();
             }
-            if (params.paramAsBoolean(REST_EXCEPTION_SKIP_STACK_TRACE, REST_EXCEPTION_SKIP_STACK_TRACE_DEFAULT) == false) {
-                builder.field("stack_trace", ExceptionsHelper.stackTrace(ex));
-            }
+            builder.endArray();
         }
+        generateThrowableXContent(builder, params, e);
+        builder.endObject();
+    }
+
+    /**
+     * Parses the output of {@link #generateFailureXContent(XContentBuilder, Params, Exception, boolean)}
+     */
+    public static ElasticsearchException failureFromXContent(XContentParser parser) throws IOException {
+        XContentParser.Token token = parser.currentToken();
+        ensureFieldName(parser, token, ERROR);
+
+        token = parser.nextToken();
+        if (token.isValue()) {
+            return new ElasticsearchException(buildMessage("exception", parser.text(), null));
+        }
+
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser::getTokenLocation);
+        token = parser.nextToken();
+
+        // Root causes are parsed in the innerFromXContent() and are added as suppressed exceptions.
+        return innerFromXContent(parser, true);
     }
 
     /**
@@ -368,12 +639,23 @@ public static String getExceptionName(Throwable ex) {
         return toUnderscoreCase(simpleName);
     }
 
+    static String buildMessage(String type, String reason, String stack) {
+        StringBuilder message = new StringBuilder("Elasticsearch exception [");
+        message.append(TYPE).append('=').append(type).append(", ");
+        message.append(REASON).append('=').append(reason);
+        if (stack != null) {
+            message.append(", ").append(STACK_TRACE).append('=').append(stack);
+        }
+        message.append(']');
+        return message.toString();
+    }
+
     @Override
     public String toString() {
         StringBuilder builder = new StringBuilder();
-        if (headers.containsKey(INDEX_HEADER_KEY)) {
+        if (metadata.containsKey(INDEX_METADATA_KEY)) {
             builder.append(getIndex());
-            if (headers.containsKey(SHARD_HEADER_KEY)) {
+            if (metadata.containsKey(SHARD_METADATA_KEY)) {
                 builder.append('[').append(getShardId()).append(']');
             }
             builder.append(' ');
@@ -431,284 +713,327 @@ public static <T extends Throwable> T writeStackTraces(T throwable, StreamOutput
      * in id order below. If you want to remove an exception leave a tombstone comment and mark the id as null in
      * ExceptionSerializationTests.testIds.ids.
      */
-    enum ElasticsearchExceptionHandle {
+    private enum ElasticsearchExceptionHandle {
         INDEX_SHARD_SNAPSHOT_FAILED_EXCEPTION(org.elasticsearch.index.snapshots.IndexShardSnapshotFailedException.class,
-                org.elasticsearch.index.snapshots.IndexShardSnapshotFailedException::new, 0),
+                org.elasticsearch.index.snapshots.IndexShardSnapshotFailedException::new, 0, UNKNOWN_VERSION_ADDED),
         DFS_PHASE_EXECUTION_EXCEPTION(org.elasticsearch.search.dfs.DfsPhaseExecutionException.class,
-                org.elasticsearch.search.dfs.DfsPhaseExecutionException::new, 1),
+                org.elasticsearch.search.dfs.DfsPhaseExecutionException::new, 1, UNKNOWN_VERSION_ADDED),
         EXECUTION_CANCELLED_EXCEPTION(org.elasticsearch.common.util.CancellableThreads.ExecutionCancelledException.class,
-                org.elasticsearch.common.util.CancellableThreads.ExecutionCancelledException::new, 2),
+                org.elasticsearch.common.util.CancellableThreads.ExecutionCancelledException::new, 2, UNKNOWN_VERSION_ADDED),
         MASTER_NOT_DISCOVERED_EXCEPTION(org.elasticsearch.discovery.MasterNotDiscoveredException.class,
-                org.elasticsearch.discovery.MasterNotDiscoveredException::new, 3),
+                org.elasticsearch.discovery.MasterNotDiscoveredException::new, 3, UNKNOWN_VERSION_ADDED),
         ELASTICSEARCH_SECURITY_EXCEPTION(org.elasticsearch.ElasticsearchSecurityException.class,
-                org.elasticsearch.ElasticsearchSecurityException::new, 4),
+                org.elasticsearch.ElasticsearchSecurityException::new, 4, UNKNOWN_VERSION_ADDED),
         INDEX_SHARD_RESTORE_EXCEPTION(org.elasticsearch.index.snapshots.IndexShardRestoreException.class,
-                org.elasticsearch.index.snapshots.IndexShardRestoreException::new, 5),
+                org.elasticsearch.index.snapshots.IndexShardRestoreException::new, 5, UNKNOWN_VERSION_ADDED),
         INDEX_CLOSED_EXCEPTION(org.elasticsearch.indices.IndexClosedException.class,
-                org.elasticsearch.indices.IndexClosedException::new, 6),
+                org.elasticsearch.indices.IndexClosedException::new, 6, UNKNOWN_VERSION_ADDED),
         BIND_HTTP_EXCEPTION(org.elasticsearch.http.BindHttpException.class,
-                org.elasticsearch.http.BindHttpException::new, 7),
+                org.elasticsearch.http.BindHttpException::new, 7, UNKNOWN_VERSION_ADDED),
         REDUCE_SEARCH_PHASE_EXCEPTION(org.elasticsearch.action.search.ReduceSearchPhaseException.class,
-                org.elasticsearch.action.search.ReduceSearchPhaseException::new, 8),
+                org.elasticsearch.action.search.ReduceSearchPhaseException::new, 8, UNKNOWN_VERSION_ADDED),
         NODE_CLOSED_EXCEPTION(org.elasticsearch.node.NodeClosedException.class,
-                org.elasticsearch.node.NodeClosedException::new, 9),
+                org.elasticsearch.node.NodeClosedException::new, 9, UNKNOWN_VERSION_ADDED),
         SNAPSHOT_FAILED_ENGINE_EXCEPTION(org.elasticsearch.index.engine.SnapshotFailedEngineException.class,
-                org.elasticsearch.index.engine.SnapshotFailedEngineException::new, 10),
+                org.elasticsearch.index.engine.SnapshotFailedEngineException::new, 10, UNKNOWN_VERSION_ADDED),
         SHARD_NOT_FOUND_EXCEPTION(org.elasticsearch.index.shard.ShardNotFoundException.class,
-                org.elasticsearch.index.shard.ShardNotFoundException::new, 11),
+                org.elasticsearch.index.shard.ShardNotFoundException::new, 11, UNKNOWN_VERSION_ADDED),
         CONNECT_TRANSPORT_EXCEPTION(org.elasticsearch.transport.ConnectTransportException.class,
-                org.elasticsearch.transport.ConnectTransportException::new, 12),
+                org.elasticsearch.transport.ConnectTransportException::new, 12, UNKNOWN_VERSION_ADDED),
         NOT_SERIALIZABLE_TRANSPORT_EXCEPTION(org.elasticsearch.transport.NotSerializableTransportException.class,
-                org.elasticsearch.transport.NotSerializableTransportException::new, 13),
+                org.elasticsearch.transport.NotSerializableTransportException::new, 13, UNKNOWN_VERSION_ADDED),
         RESPONSE_HANDLER_FAILURE_TRANSPORT_EXCEPTION(org.elasticsearch.transport.ResponseHandlerFailureTransportException.class,
-                org.elasticsearch.transport.ResponseHandlerFailureTransportException::new, 14),
+                org.elasticsearch.transport.ResponseHandlerFailureTransportException::new, 14, UNKNOWN_VERSION_ADDED),
         INDEX_CREATION_EXCEPTION(org.elasticsearch.indices.IndexCreationException.class,
-                org.elasticsearch.indices.IndexCreationException::new, 15),
+                org.elasticsearch.indices.IndexCreationException::new, 15, UNKNOWN_VERSION_ADDED),
         INDEX_NOT_FOUND_EXCEPTION(org.elasticsearch.index.IndexNotFoundException.class,
-                org.elasticsearch.index.IndexNotFoundException::new, 16),
+                org.elasticsearch.index.IndexNotFoundException::new, 16, UNKNOWN_VERSION_ADDED),
         ILLEGAL_SHARD_ROUTING_STATE_EXCEPTION(org.elasticsearch.cluster.routing.IllegalShardRoutingStateException.class,
-                org.elasticsearch.cluster.routing.IllegalShardRoutingStateException::new, 17),
+                org.elasticsearch.cluster.routing.IllegalShardRoutingStateException::new, 17, UNKNOWN_VERSION_ADDED),
         BROADCAST_SHARD_OPERATION_FAILED_EXCEPTION(org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedException.class,
-                org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedException::new, 18),
+                org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedException::new, 18, UNKNOWN_VERSION_ADDED),
         RESOURCE_NOT_FOUND_EXCEPTION(org.elasticsearch.ResourceNotFoundException.class,
-                org.elasticsearch.ResourceNotFoundException::new, 19),
+                org.elasticsearch.ResourceNotFoundException::new, 19, UNKNOWN_VERSION_ADDED),
         ACTION_TRANSPORT_EXCEPTION(org.elasticsearch.transport.ActionTransportException.class,
-                org.elasticsearch.transport.ActionTransportException::new, 20),
+                org.elasticsearch.transport.ActionTransportException::new, 20, UNKNOWN_VERSION_ADDED),
         ELASTICSEARCH_GENERATION_EXCEPTION(org.elasticsearch.ElasticsearchGenerationException.class,
-                org.elasticsearch.ElasticsearchGenerationException::new, 21),
+                org.elasticsearch.ElasticsearchGenerationException::new, 21, UNKNOWN_VERSION_ADDED),
         //      22 was CreateFailedEngineException
         INDEX_SHARD_STARTED_EXCEPTION(org.elasticsearch.index.shard.IndexShardStartedException.class,
-                org.elasticsearch.index.shard.IndexShardStartedException::new, 23),
+                org.elasticsearch.index.shard.IndexShardStartedException::new, 23, UNKNOWN_VERSION_ADDED),
         SEARCH_CONTEXT_MISSING_EXCEPTION(org.elasticsearch.search.SearchContextMissingException.class,
-                org.elasticsearch.search.SearchContextMissingException::new, 24),
+                org.elasticsearch.search.SearchContextMissingException::new, 24, UNKNOWN_VERSION_ADDED),
         GENERAL_SCRIPT_EXCEPTION(org.elasticsearch.script.GeneralScriptException.class,
-                org.elasticsearch.script.GeneralScriptException::new, 25),
+                org.elasticsearch.script.GeneralScriptException::new, 25, UNKNOWN_VERSION_ADDED),
         BATCH_OPERATION_EXCEPTION(org.elasticsearch.index.shard.TranslogRecoveryPerformer.BatchOperationException.class,
-                org.elasticsearch.index.shard.TranslogRecoveryPerformer.BatchOperationException::new, 26),
+                org.elasticsearch.index.shard.TranslogRecoveryPerformer.BatchOperationException::new, 26, UNKNOWN_VERSION_ADDED),
         SNAPSHOT_CREATION_EXCEPTION(org.elasticsearch.snapshots.SnapshotCreationException.class,
-                org.elasticsearch.snapshots.SnapshotCreationException::new, 27),
+                org.elasticsearch.snapshots.SnapshotCreationException::new, 27, UNKNOWN_VERSION_ADDED),
         DELETE_FAILED_ENGINE_EXCEPTION(org.elasticsearch.index.engine.DeleteFailedEngineException.class,
-                org.elasticsearch.index.engine.DeleteFailedEngineException::new, 28),
+                org.elasticsearch.index.engine.DeleteFailedEngineException::new, 28, UNKNOWN_VERSION_ADDED),
         DOCUMENT_MISSING_EXCEPTION(org.elasticsearch.index.engine.DocumentMissingException.class,
-                org.elasticsearch.index.engine.DocumentMissingException::new, 29),
+                org.elasticsearch.index.engine.DocumentMissingException::new, 29, UNKNOWN_VERSION_ADDED),
         SNAPSHOT_EXCEPTION(org.elasticsearch.snapshots.SnapshotException.class,
-                org.elasticsearch.snapshots.SnapshotException::new, 30),
+                org.elasticsearch.snapshots.SnapshotException::new, 30, UNKNOWN_VERSION_ADDED),
         INVALID_ALIAS_NAME_EXCEPTION(org.elasticsearch.indices.InvalidAliasNameException.class,
-                org.elasticsearch.indices.InvalidAliasNameException::new, 31),
+                org.elasticsearch.indices.InvalidAliasNameException::new, 31, UNKNOWN_VERSION_ADDED),
         INVALID_INDEX_NAME_EXCEPTION(org.elasticsearch.indices.InvalidIndexNameException.class,
-                org.elasticsearch.indices.InvalidIndexNameException::new, 32),
+                org.elasticsearch.indices.InvalidIndexNameException::new, 32, UNKNOWN_VERSION_ADDED),
         INDEX_PRIMARY_SHARD_NOT_ALLOCATED_EXCEPTION(org.elasticsearch.indices.IndexPrimaryShardNotAllocatedException.class,
-                org.elasticsearch.indices.IndexPrimaryShardNotAllocatedException::new, 33),
+                org.elasticsearch.indices.IndexPrimaryShardNotAllocatedException::new, 33, UNKNOWN_VERSION_ADDED),
         TRANSPORT_EXCEPTION(org.elasticsearch.transport.TransportException.class,
-                org.elasticsearch.transport.TransportException::new, 34),
+                org.elasticsearch.transport.TransportException::new, 34, UNKNOWN_VERSION_ADDED),
         ELASTICSEARCH_PARSE_EXCEPTION(org.elasticsearch.ElasticsearchParseException.class,
-                org.elasticsearch.ElasticsearchParseException::new, 35),
+                org.elasticsearch.ElasticsearchParseException::new, 35, UNKNOWN_VERSION_ADDED),
         SEARCH_EXCEPTION(org.elasticsearch.search.SearchException.class,
-                org.elasticsearch.search.SearchException::new, 36),
+                org.elasticsearch.search.SearchException::new, 36, UNKNOWN_VERSION_ADDED),
         MAPPER_EXCEPTION(org.elasticsearch.index.mapper.MapperException.class,
-                org.elasticsearch.index.mapper.MapperException::new, 37),
+                org.elasticsearch.index.mapper.MapperException::new, 37, UNKNOWN_VERSION_ADDED),
         INVALID_TYPE_NAME_EXCEPTION(org.elasticsearch.indices.InvalidTypeNameException.class,
-                org.elasticsearch.indices.InvalidTypeNameException::new, 38),
+                org.elasticsearch.indices.InvalidTypeNameException::new, 38, UNKNOWN_VERSION_ADDED),
         SNAPSHOT_RESTORE_EXCEPTION(org.elasticsearch.snapshots.SnapshotRestoreException.class,
-                org.elasticsearch.snapshots.SnapshotRestoreException::new, 39),
-        PARSING_EXCEPTION(org.elasticsearch.common.ParsingException.class, org.elasticsearch.common.ParsingException::new, 40),
+                org.elasticsearch.snapshots.SnapshotRestoreException::new, 39, UNKNOWN_VERSION_ADDED),
+        PARSING_EXCEPTION(org.elasticsearch.common.ParsingException.class, org.elasticsearch.common.ParsingException::new, 40,
+            UNKNOWN_VERSION_ADDED),
         INDEX_SHARD_CLOSED_EXCEPTION(org.elasticsearch.index.shard.IndexShardClosedException.class,
-                org.elasticsearch.index.shard.IndexShardClosedException::new, 41),
+                org.elasticsearch.index.shard.IndexShardClosedException::new, 41, UNKNOWN_VERSION_ADDED),
         RECOVER_FILES_RECOVERY_EXCEPTION(org.elasticsearch.indices.recovery.RecoverFilesRecoveryException.class,
-                org.elasticsearch.indices.recovery.RecoverFilesRecoveryException::new, 42),
+                org.elasticsearch.indices.recovery.RecoverFilesRecoveryException::new, 42, UNKNOWN_VERSION_ADDED),
         TRUNCATED_TRANSLOG_EXCEPTION(org.elasticsearch.index.translog.TruncatedTranslogException.class,
-                org.elasticsearch.index.translog.TruncatedTranslogException::new, 43),
+                org.elasticsearch.index.translog.TruncatedTranslogException::new, 43, UNKNOWN_VERSION_ADDED),
         RECOVERY_FAILED_EXCEPTION(org.elasticsearch.indices.recovery.RecoveryFailedException.class,
-                org.elasticsearch.indices.recovery.RecoveryFailedException::new, 44),
+                org.elasticsearch.indices.recovery.RecoveryFailedException::new, 44, UNKNOWN_VERSION_ADDED),
         INDEX_SHARD_RELOCATED_EXCEPTION(org.elasticsearch.index.shard.IndexShardRelocatedException.class,
-                org.elasticsearch.index.shard.IndexShardRelocatedException::new, 45),
+                org.elasticsearch.index.shard.IndexShardRelocatedException::new, 45, UNKNOWN_VERSION_ADDED),
         NODE_SHOULD_NOT_CONNECT_EXCEPTION(org.elasticsearch.transport.NodeShouldNotConnectException.class,
-                org.elasticsearch.transport.NodeShouldNotConnectException::new, 46),
+        org.elasticsearch.transport.NodeShouldNotConnectException::new, 46, UNKNOWN_VERSION_ADDED),
         INDEX_TEMPLATE_ALREADY_EXISTS_EXCEPTION(org.elasticsearch.indices.IndexTemplateAlreadyExistsException.class,
-                org.elasticsearch.indices.IndexTemplateAlreadyExistsException::new, 47),
+                org.elasticsearch.indices.IndexTemplateAlreadyExistsException::new, 47, UNKNOWN_VERSION_ADDED),
         TRANSLOG_CORRUPTED_EXCEPTION(org.elasticsearch.index.translog.TranslogCorruptedException.class,
-                org.elasticsearch.index.translog.TranslogCorruptedException::new, 48),
+                org.elasticsearch.index.translog.TranslogCorruptedException::new, 48, UNKNOWN_VERSION_ADDED),
         CLUSTER_BLOCK_EXCEPTION(org.elasticsearch.cluster.block.ClusterBlockException.class,
-                org.elasticsearch.cluster.block.ClusterBlockException::new, 49),
+                org.elasticsearch.cluster.block.ClusterBlockException::new, 49, UNKNOWN_VERSION_ADDED),
         FETCH_PHASE_EXECUTION_EXCEPTION(org.elasticsearch.search.fetch.FetchPhaseExecutionException.class,
-                org.elasticsearch.search.fetch.FetchPhaseExecutionException::new, 50),
+        org.elasticsearch.search.fetch.FetchPhaseExecutionException::new, 50, UNKNOWN_VERSION_ADDED),
         INDEX_SHARD_ALREADY_EXISTS_EXCEPTION(org.elasticsearch.index.IndexShardAlreadyExistsException.class,
-                org.elasticsearch.index.IndexShardAlreadyExistsException::new, 51),
+                org.elasticsearch.index.IndexShardAlreadyExistsException::new, 51, UNKNOWN_VERSION_ADDED),
         VERSION_CONFLICT_ENGINE_EXCEPTION(org.elasticsearch.index.engine.VersionConflictEngineException.class,
-                org.elasticsearch.index.engine.VersionConflictEngineException::new, 52),
-        ENGINE_EXCEPTION(org.elasticsearch.index.engine.EngineException.class, org.elasticsearch.index.engine.EngineException::new, 53),
+                org.elasticsearch.index.engine.VersionConflictEngineException::new, 52, UNKNOWN_VERSION_ADDED),
+        ENGINE_EXCEPTION(org.elasticsearch.index.engine.EngineException.class, org.elasticsearch.index.engine.EngineException::new, 53,
+            UNKNOWN_VERSION_ADDED),
         // 54 was DocumentAlreadyExistsException, which is superseded by VersionConflictEngineException
-        NO_SUCH_NODE_EXCEPTION(org.elasticsearch.action.NoSuchNodeException.class, org.elasticsearch.action.NoSuchNodeException::new, 55),
+        NO_SUCH_NODE_EXCEPTION(org.elasticsearch.action.NoSuchNodeException.class, org.elasticsearch.action.NoSuchNodeException::new, 55,
+            UNKNOWN_VERSION_ADDED),
         SETTINGS_EXCEPTION(org.elasticsearch.common.settings.SettingsException.class,
-                org.elasticsearch.common.settings.SettingsException::new, 56),
+                org.elasticsearch.common.settings.SettingsException::new, 56, UNKNOWN_VERSION_ADDED),
         INDEX_TEMPLATE_MISSING_EXCEPTION(org.elasticsearch.indices.IndexTemplateMissingException.class,
-                org.elasticsearch.indices.IndexTemplateMissingException::new, 57),
+                org.elasticsearch.indices.IndexTemplateMissingException::new, 57, UNKNOWN_VERSION_ADDED),
         SEND_REQUEST_TRANSPORT_EXCEPTION(org.elasticsearch.transport.SendRequestTransportException.class,
-                org.elasticsearch.transport.SendRequestTransportException::new, 58),
+                org.elasticsearch.transport.SendRequestTransportException::new, 58, UNKNOWN_VERSION_ADDED),
         ES_REJECTED_EXECUTION_EXCEPTION(org.elasticsearch.common.util.concurrent.EsRejectedExecutionException.class,
-                org.elasticsearch.common.util.concurrent.EsRejectedExecutionException::new, 59),
+                org.elasticsearch.common.util.concurrent.EsRejectedExecutionException::new, 59, UNKNOWN_VERSION_ADDED),
         EARLY_TERMINATION_EXCEPTION(org.elasticsearch.common.lucene.Lucene.EarlyTerminationException.class,
-                org.elasticsearch.common.lucene.Lucene.EarlyTerminationException::new, 60),
+                org.elasticsearch.common.lucene.Lucene.EarlyTerminationException::new, 60, UNKNOWN_VERSION_ADDED),
         // 61 used to be for RoutingValidationException
         NOT_SERIALIZABLE_EXCEPTION_WRAPPER(org.elasticsearch.common.io.stream.NotSerializableExceptionWrapper.class,
-                org.elasticsearch.common.io.stream.NotSerializableExceptionWrapper::new, 62),
+                org.elasticsearch.common.io.stream.NotSerializableExceptionWrapper::new, 62, UNKNOWN_VERSION_ADDED),
         ALIAS_FILTER_PARSING_EXCEPTION(org.elasticsearch.indices.AliasFilterParsingException.class,
-                org.elasticsearch.indices.AliasFilterParsingException::new, 63),
-        // 64 was DeleteByQueryFailedEngineException, which was removed in 3.0
-        GATEWAY_EXCEPTION(org.elasticsearch.gateway.GatewayException.class, org.elasticsearch.gateway.GatewayException::new, 65),
+                org.elasticsearch.indices.AliasFilterParsingException::new, 63, UNKNOWN_VERSION_ADDED),
+        // 64 was DeleteByQueryFailedEngineException, which was removed in 5.0
+        GATEWAY_EXCEPTION(org.elasticsearch.gateway.GatewayException.class, org.elasticsearch.gateway.GatewayException::new, 65,
+            UNKNOWN_VERSION_ADDED),
         INDEX_SHARD_NOT_RECOVERING_EXCEPTION(org.elasticsearch.index.shard.IndexShardNotRecoveringException.class,
-                org.elasticsearch.index.shard.IndexShardNotRecoveringException::new, 66),
-        HTTP_EXCEPTION(org.elasticsearch.http.HttpException.class, org.elasticsearch.http.HttpException::new, 67),
+                org.elasticsearch.index.shard.IndexShardNotRecoveringException::new, 66, UNKNOWN_VERSION_ADDED),
+        HTTP_EXCEPTION(org.elasticsearch.http.HttpException.class, org.elasticsearch.http.HttpException::new, 67, UNKNOWN_VERSION_ADDED),
         ELASTICSEARCH_EXCEPTION(org.elasticsearch.ElasticsearchException.class,
-                org.elasticsearch.ElasticsearchException::new, 68),
+                org.elasticsearch.ElasticsearchException::new, 68, UNKNOWN_VERSION_ADDED),
         SNAPSHOT_MISSING_EXCEPTION(org.elasticsearch.snapshots.SnapshotMissingException.class,
-                org.elasticsearch.snapshots.SnapshotMissingException::new, 69),
+                org.elasticsearch.snapshots.SnapshotMissingException::new, 69, UNKNOWN_VERSION_ADDED),
         PRIMARY_MISSING_ACTION_EXCEPTION(org.elasticsearch.action.PrimaryMissingActionException.class,
-                org.elasticsearch.action.PrimaryMissingActionException::new, 70),
-        FAILED_NODE_EXCEPTION(org.elasticsearch.action.FailedNodeException.class, org.elasticsearch.action.FailedNodeException::new, 71),
-        SEARCH_PARSE_EXCEPTION(org.elasticsearch.search.SearchParseException.class, org.elasticsearch.search.SearchParseException::new, 72),
+                org.elasticsearch.action.PrimaryMissingActionException::new, 70, UNKNOWN_VERSION_ADDED),
+        FAILED_NODE_EXCEPTION(org.elasticsearch.action.FailedNodeException.class, org.elasticsearch.action.FailedNodeException::new, 71,
+            UNKNOWN_VERSION_ADDED),
+        SEARCH_PARSE_EXCEPTION(org.elasticsearch.search.SearchParseException.class, org.elasticsearch.search.SearchParseException::new, 72,
+            UNKNOWN_VERSION_ADDED),
         CONCURRENT_SNAPSHOT_EXECUTION_EXCEPTION(org.elasticsearch.snapshots.ConcurrentSnapshotExecutionException.class,
-                org.elasticsearch.snapshots.ConcurrentSnapshotExecutionException::new, 73),
+                org.elasticsearch.snapshots.ConcurrentSnapshotExecutionException::new, 73, UNKNOWN_VERSION_ADDED),
         BLOB_STORE_EXCEPTION(org.elasticsearch.common.blobstore.BlobStoreException.class,
-                org.elasticsearch.common.blobstore.BlobStoreException::new, 74),
+                org.elasticsearch.common.blobstore.BlobStoreException::new, 74, UNKNOWN_VERSION_ADDED),
         INCOMPATIBLE_CLUSTER_STATE_VERSION_EXCEPTION(org.elasticsearch.cluster.IncompatibleClusterStateVersionException.class,
-                org.elasticsearch.cluster.IncompatibleClusterStateVersionException::new, 75),
+                org.elasticsearch.cluster.IncompatibleClusterStateVersionException::new, 75, UNKNOWN_VERSION_ADDED),
         RECOVERY_ENGINE_EXCEPTION(org.elasticsearch.index.engine.RecoveryEngineException.class,
-                org.elasticsearch.index.engine.RecoveryEngineException::new, 76),
+                org.elasticsearch.index.engine.RecoveryEngineException::new, 76, UNKNOWN_VERSION_ADDED),
         UNCATEGORIZED_EXECUTION_EXCEPTION(org.elasticsearch.common.util.concurrent.UncategorizedExecutionException.class,
-                org.elasticsearch.common.util.concurrent.UncategorizedExecutionException::new, 77),
+                org.elasticsearch.common.util.concurrent.UncategorizedExecutionException::new, 77, UNKNOWN_VERSION_ADDED),
         TIMESTAMP_PARSING_EXCEPTION(org.elasticsearch.action.TimestampParsingException.class,
-                org.elasticsearch.action.TimestampParsingException::new, 78),
+                org.elasticsearch.action.TimestampParsingException::new, 78, UNKNOWN_VERSION_ADDED),
         ROUTING_MISSING_EXCEPTION(org.elasticsearch.action.RoutingMissingException.class,
-                org.elasticsearch.action.RoutingMissingException::new, 79),
+                org.elasticsearch.action.RoutingMissingException::new, 79, UNKNOWN_VERSION_ADDED),
         INDEX_FAILED_ENGINE_EXCEPTION(org.elasticsearch.index.engine.IndexFailedEngineException.class,
-                org.elasticsearch.index.engine.IndexFailedEngineException::new, 80),
+                org.elasticsearch.index.engine.IndexFailedEngineException::new, 80, UNKNOWN_VERSION_ADDED),
         INDEX_SHARD_RESTORE_FAILED_EXCEPTION(org.elasticsearch.index.snapshots.IndexShardRestoreFailedException.class,
-                org.elasticsearch.index.snapshots.IndexShardRestoreFailedException::new, 81),
+                org.elasticsearch.index.snapshots.IndexShardRestoreFailedException::new, 81, UNKNOWN_VERSION_ADDED),
         REPOSITORY_EXCEPTION(org.elasticsearch.repositories.RepositoryException.class,
-                org.elasticsearch.repositories.RepositoryException::new, 82),
+                org.elasticsearch.repositories.RepositoryException::new, 82, UNKNOWN_VERSION_ADDED),
         RECEIVE_TIMEOUT_TRANSPORT_EXCEPTION(org.elasticsearch.transport.ReceiveTimeoutTransportException.class,
-                org.elasticsearch.transport.ReceiveTimeoutTransportException::new, 83),
+                org.elasticsearch.transport.ReceiveTimeoutTransportException::new, 83, UNKNOWN_VERSION_ADDED),
         NODE_DISCONNECTED_EXCEPTION(org.elasticsearch.transport.NodeDisconnectedException.class,
-                org.elasticsearch.transport.NodeDisconnectedException::new, 84),
+                org.elasticsearch.transport.NodeDisconnectedException::new, 84, UNKNOWN_VERSION_ADDED),
         ALREADY_EXPIRED_EXCEPTION(org.elasticsearch.index.AlreadyExpiredException.class,
-                org.elasticsearch.index.AlreadyExpiredException::new, 85),
+                org.elasticsearch.index.AlreadyExpiredException::new, 85, UNKNOWN_VERSION_ADDED),
         AGGREGATION_EXECUTION_EXCEPTION(org.elasticsearch.search.aggregations.AggregationExecutionException.class,
-                org.elasticsearch.search.aggregations.AggregationExecutionException::new, 86),
+                org.elasticsearch.search.aggregations.AggregationExecutionException::new, 86, UNKNOWN_VERSION_ADDED),
         // 87 used to be for MergeMappingException
         INVALID_INDEX_TEMPLATE_EXCEPTION(org.elasticsearch.indices.InvalidIndexTemplateException.class,
-                org.elasticsearch.indices.InvalidIndexTemplateException::new, 88),
+                org.elasticsearch.indices.InvalidIndexTemplateException::new, 88, UNKNOWN_VERSION_ADDED),
         REFRESH_FAILED_ENGINE_EXCEPTION(org.elasticsearch.index.engine.RefreshFailedEngineException.class,
-                org.elasticsearch.index.engine.RefreshFailedEngineException::new, 90),
+                org.elasticsearch.index.engine.RefreshFailedEngineException::new, 90, UNKNOWN_VERSION_ADDED),
         AGGREGATION_INITIALIZATION_EXCEPTION(org.elasticsearch.search.aggregations.AggregationInitializationException.class,
-                org.elasticsearch.search.aggregations.AggregationInitializationException::new, 91),
+                org.elasticsearch.search.aggregations.AggregationInitializationException::new, 91, UNKNOWN_VERSION_ADDED),
         DELAY_RECOVERY_EXCEPTION(org.elasticsearch.indices.recovery.DelayRecoveryException.class,
-                org.elasticsearch.indices.recovery.DelayRecoveryException::new, 92),
+                org.elasticsearch.indices.recovery.DelayRecoveryException::new, 92, UNKNOWN_VERSION_ADDED),
         // 93 used to be for IndexWarmerMissingException
         NO_NODE_AVAILABLE_EXCEPTION(org.elasticsearch.client.transport.NoNodeAvailableException.class,
-                org.elasticsearch.client.transport.NoNodeAvailableException::new, 94),
+                org.elasticsearch.client.transport.NoNodeAvailableException::new, 94, UNKNOWN_VERSION_ADDED),
         INVALID_SNAPSHOT_NAME_EXCEPTION(org.elasticsearch.snapshots.InvalidSnapshotNameException.class,
-                org.elasticsearch.snapshots.InvalidSnapshotNameException::new, 96),
+                org.elasticsearch.snapshots.InvalidSnapshotNameException::new, 96, UNKNOWN_VERSION_ADDED),
         ILLEGAL_INDEX_SHARD_STATE_EXCEPTION(org.elasticsearch.index.shard.IllegalIndexShardStateException.class,
-                org.elasticsearch.index.shard.IllegalIndexShardStateException::new, 97),
+                org.elasticsearch.index.shard.IllegalIndexShardStateException::new, 97, UNKNOWN_VERSION_ADDED),
         INDEX_SHARD_SNAPSHOT_EXCEPTION(org.elasticsearch.index.snapshots.IndexShardSnapshotException.class,
-                org.elasticsearch.index.snapshots.IndexShardSnapshotException::new, 98),
+                org.elasticsearch.index.snapshots.IndexShardSnapshotException::new, 98, UNKNOWN_VERSION_ADDED),
         INDEX_SHARD_NOT_STARTED_EXCEPTION(org.elasticsearch.index.shard.IndexShardNotStartedException.class,
-                org.elasticsearch.index.shard.IndexShardNotStartedException::new, 99),
+                org.elasticsearch.index.shard.IndexShardNotStartedException::new, 99, UNKNOWN_VERSION_ADDED),
         SEARCH_PHASE_EXECUTION_EXCEPTION(org.elasticsearch.action.search.SearchPhaseExecutionException.class,
-                org.elasticsearch.action.search.SearchPhaseExecutionException::new, 100),
+                org.elasticsearch.action.search.SearchPhaseExecutionException::new, 100, UNKNOWN_VERSION_ADDED),
         ACTION_NOT_FOUND_TRANSPORT_EXCEPTION(org.elasticsearch.transport.ActionNotFoundTransportException.class,
-                org.elasticsearch.transport.ActionNotFoundTransportException::new, 101),
+                org.elasticsearch.transport.ActionNotFoundTransportException::new, 101, UNKNOWN_VERSION_ADDED),
         TRANSPORT_SERIALIZATION_EXCEPTION(org.elasticsearch.transport.TransportSerializationException.class,
-                org.elasticsearch.transport.TransportSerializationException::new, 102),
+                org.elasticsearch.transport.TransportSerializationException::new, 102, UNKNOWN_VERSION_ADDED),
         REMOTE_TRANSPORT_EXCEPTION(org.elasticsearch.transport.RemoteTransportException.class,
-                org.elasticsearch.transport.RemoteTransportException::new, 103),
+                org.elasticsearch.transport.RemoteTransportException::new, 103, UNKNOWN_VERSION_ADDED),
         ENGINE_CREATION_FAILURE_EXCEPTION(org.elasticsearch.index.engine.EngineCreationFailureException.class,
-                org.elasticsearch.index.engine.EngineCreationFailureException::new, 104),
+                org.elasticsearch.index.engine.EngineCreationFailureException::new, 104, UNKNOWN_VERSION_ADDED),
         ROUTING_EXCEPTION(org.elasticsearch.cluster.routing.RoutingException.class,
-                org.elasticsearch.cluster.routing.RoutingException::new, 105),
+                org.elasticsearch.cluster.routing.RoutingException::new, 105, UNKNOWN_VERSION_ADDED),
         INDEX_SHARD_RECOVERY_EXCEPTION(org.elasticsearch.index.shard.IndexShardRecoveryException.class,
-                org.elasticsearch.index.shard.IndexShardRecoveryException::new, 106),
+                org.elasticsearch.index.shard.IndexShardRecoveryException::new, 106, UNKNOWN_VERSION_ADDED),
         REPOSITORY_MISSING_EXCEPTION(org.elasticsearch.repositories.RepositoryMissingException.class,
-                org.elasticsearch.repositories.RepositoryMissingException::new, 107),
+                org.elasticsearch.repositories.RepositoryMissingException::new, 107, UNKNOWN_VERSION_ADDED),
         DOCUMENT_SOURCE_MISSING_EXCEPTION(org.elasticsearch.index.engine.DocumentSourceMissingException.class,
-                org.elasticsearch.index.engine.DocumentSourceMissingException::new, 109),
-        FLUSH_NOT_ALLOWED_ENGINE_EXCEPTION(org.elasticsearch.index.engine.FlushNotAllowedEngineException.class,
-                org.elasticsearch.index.engine.FlushNotAllowedEngineException::new, 110),
+                org.elasticsearch.index.engine.DocumentSourceMissingException::new, 109, UNKNOWN_VERSION_ADDED),
+        // 110 used to be FlushNotAllowedEngineException
         NO_CLASS_SETTINGS_EXCEPTION(org.elasticsearch.common.settings.NoClassSettingsException.class,
-                org.elasticsearch.common.settings.NoClassSettingsException::new, 111),
+                org.elasticsearch.common.settings.NoClassSettingsException::new, 111, UNKNOWN_VERSION_ADDED),
         BIND_TRANSPORT_EXCEPTION(org.elasticsearch.transport.BindTransportException.class,
-                org.elasticsearch.transport.BindTransportException::new, 112),
+                org.elasticsearch.transport.BindTransportException::new, 112, UNKNOWN_VERSION_ADDED),
         ALIASES_NOT_FOUND_EXCEPTION(org.elasticsearch.rest.action.admin.indices.AliasesNotFoundException.class,
-                org.elasticsearch.rest.action.admin.indices.AliasesNotFoundException::new, 113),
+                org.elasticsearch.rest.action.admin.indices.AliasesNotFoundException::new, 113, UNKNOWN_VERSION_ADDED),
         INDEX_SHARD_RECOVERING_EXCEPTION(org.elasticsearch.index.shard.IndexShardRecoveringException.class,
-                org.elasticsearch.index.shard.IndexShardRecoveringException::new, 114),
+                org.elasticsearch.index.shard.IndexShardRecoveringException::new, 114, UNKNOWN_VERSION_ADDED),
         TRANSLOG_EXCEPTION(org.elasticsearch.index.translog.TranslogException.class,
-                org.elasticsearch.index.translog.TranslogException::new, 115),
+                org.elasticsearch.index.translog.TranslogException::new, 115, UNKNOWN_VERSION_ADDED),
         PROCESS_CLUSTER_EVENT_TIMEOUT_EXCEPTION(org.elasticsearch.cluster.metadata.ProcessClusterEventTimeoutException.class,
-                org.elasticsearch.cluster.metadata.ProcessClusterEventTimeoutException::new, 116),
+                org.elasticsearch.cluster.metadata.ProcessClusterEventTimeoutException::new, 116, UNKNOWN_VERSION_ADDED),
         RETRY_ON_PRIMARY_EXCEPTION(ReplicationOperation.RetryOnPrimaryException.class,
-                ReplicationOperation.RetryOnPrimaryException::new, 117),
+                ReplicationOperation.RetryOnPrimaryException::new, 117, UNKNOWN_VERSION_ADDED),
         ELASTICSEARCH_TIMEOUT_EXCEPTION(org.elasticsearch.ElasticsearchTimeoutException.class,
-                org.elasticsearch.ElasticsearchTimeoutException::new, 118),
+                org.elasticsearch.ElasticsearchTimeoutException::new, 118, UNKNOWN_VERSION_ADDED),
         QUERY_PHASE_EXECUTION_EXCEPTION(org.elasticsearch.search.query.QueryPhaseExecutionException.class,
-                org.elasticsearch.search.query.QueryPhaseExecutionException::new, 119),
+                org.elasticsearch.search.query.QueryPhaseExecutionException::new, 119, UNKNOWN_VERSION_ADDED),
         REPOSITORY_VERIFICATION_EXCEPTION(org.elasticsearch.repositories.RepositoryVerificationException.class,
-                org.elasticsearch.repositories.RepositoryVerificationException::new, 120),
+                org.elasticsearch.repositories.RepositoryVerificationException::new, 120, UNKNOWN_VERSION_ADDED),
         INVALID_AGGREGATION_PATH_EXCEPTION(org.elasticsearch.search.aggregations.InvalidAggregationPathException.class,
-                org.elasticsearch.search.aggregations.InvalidAggregationPathException::new, 121),
-        INDEX_ALREADY_EXISTS_EXCEPTION(org.elasticsearch.indices.IndexAlreadyExistsException.class,
-                org.elasticsearch.indices.IndexAlreadyExistsException::new, 123),
+                org.elasticsearch.search.aggregations.InvalidAggregationPathException::new, 121, UNKNOWN_VERSION_ADDED),
+        // 123 used to be IndexAlreadyExistsException and was renamed
+        RESOURCE_ALREADY_EXISTS_EXCEPTION(ResourceAlreadyExistsException.class,
+            ResourceAlreadyExistsException::new, 123, UNKNOWN_VERSION_ADDED),
         // 124 used to be Script.ScriptParseException
         HTTP_ON_TRANSPORT_EXCEPTION(TcpTransport.HttpOnTransportException.class,
-                TcpTransport.HttpOnTransportException::new, 125),
+                TcpTransport.HttpOnTransportException::new, 125, UNKNOWN_VERSION_ADDED),
         MAPPER_PARSING_EXCEPTION(org.elasticsearch.index.mapper.MapperParsingException.class,
-                org.elasticsearch.index.mapper.MapperParsingException::new, 126),
+                org.elasticsearch.index.mapper.MapperParsingException::new, 126, UNKNOWN_VERSION_ADDED),
         SEARCH_CONTEXT_EXCEPTION(org.elasticsearch.search.SearchContextException.class,
-                org.elasticsearch.search.SearchContextException::new, 127),
+                org.elasticsearch.search.SearchContextException::new, 127, UNKNOWN_VERSION_ADDED),
         SEARCH_SOURCE_BUILDER_EXCEPTION(org.elasticsearch.search.builder.SearchSourceBuilderException.class,
-                org.elasticsearch.search.builder.SearchSourceBuilderException::new, 128),
+                org.elasticsearch.search.builder.SearchSourceBuilderException::new, 128, UNKNOWN_VERSION_ADDED),
         ENGINE_CLOSED_EXCEPTION(org.elasticsearch.index.engine.EngineClosedException.class,
-                org.elasticsearch.index.engine.EngineClosedException::new, 129),
+                org.elasticsearch.index.engine.EngineClosedException::new, 129, UNKNOWN_VERSION_ADDED),
         NO_SHARD_AVAILABLE_ACTION_EXCEPTION(org.elasticsearch.action.NoShardAvailableActionException.class,
-                org.elasticsearch.action.NoShardAvailableActionException::new, 130),
+                org.elasticsearch.action.NoShardAvailableActionException::new, 130, UNKNOWN_VERSION_ADDED),
         UNAVAILABLE_SHARDS_EXCEPTION(org.elasticsearch.action.UnavailableShardsException.class,
-                org.elasticsearch.action.UnavailableShardsException::new, 131),
+                org.elasticsearch.action.UnavailableShardsException::new, 131, UNKNOWN_VERSION_ADDED),
         FLUSH_FAILED_ENGINE_EXCEPTION(org.elasticsearch.index.engine.FlushFailedEngineException.class,
-                org.elasticsearch.index.engine.FlushFailedEngineException::new, 132),
+                org.elasticsearch.index.engine.FlushFailedEngineException::new, 132, UNKNOWN_VERSION_ADDED),
         CIRCUIT_BREAKING_EXCEPTION(org.elasticsearch.common.breaker.CircuitBreakingException.class,
-                org.elasticsearch.common.breaker.CircuitBreakingException::new, 133),
+                org.elasticsearch.common.breaker.CircuitBreakingException::new, 133, UNKNOWN_VERSION_ADDED),
         NODE_NOT_CONNECTED_EXCEPTION(org.elasticsearch.transport.NodeNotConnectedException.class,
-                org.elasticsearch.transport.NodeNotConnectedException::new, 134),
+                org.elasticsearch.transport.NodeNotConnectedException::new, 134, UNKNOWN_VERSION_ADDED),
         STRICT_DYNAMIC_MAPPING_EXCEPTION(org.elasticsearch.index.mapper.StrictDynamicMappingException.class,
-                org.elasticsearch.index.mapper.StrictDynamicMappingException::new, 135),
+                org.elasticsearch.index.mapper.StrictDynamicMappingException::new, 135, UNKNOWN_VERSION_ADDED),
         RETRY_ON_REPLICA_EXCEPTION(org.elasticsearch.action.support.replication.TransportReplicationAction.RetryOnReplicaException.class,
-                org.elasticsearch.action.support.replication.TransportReplicationAction.RetryOnReplicaException::new, 136),
+                org.elasticsearch.action.support.replication.TransportReplicationAction.RetryOnReplicaException::new, 136,
+            UNKNOWN_VERSION_ADDED),
         TYPE_MISSING_EXCEPTION(org.elasticsearch.indices.TypeMissingException.class,
-                org.elasticsearch.indices.TypeMissingException::new, 137),
+                org.elasticsearch.indices.TypeMissingException::new, 137, UNKNOWN_VERSION_ADDED),
         FAILED_TO_COMMIT_CLUSTER_STATE_EXCEPTION(org.elasticsearch.discovery.Discovery.FailedToCommitClusterStateException.class,
-                org.elasticsearch.discovery.Discovery.FailedToCommitClusterStateException::new, 140),
+                org.elasticsearch.discovery.Discovery.FailedToCommitClusterStateException::new, 140, UNKNOWN_VERSION_ADDED),
         QUERY_SHARD_EXCEPTION(org.elasticsearch.index.query.QueryShardException.class,
-                org.elasticsearch.index.query.QueryShardException::new, 141),
+                org.elasticsearch.index.query.QueryShardException::new, 141, UNKNOWN_VERSION_ADDED),
         NO_LONGER_PRIMARY_SHARD_EXCEPTION(ShardStateAction.NoLongerPrimaryShardException.class,
-                ShardStateAction.NoLongerPrimaryShardException::new, 142),
-        SCRIPT_EXCEPTION(org.elasticsearch.script.ScriptException.class, org.elasticsearch.script.ScriptException::new, 143),
-        NOT_MASTER_EXCEPTION(org.elasticsearch.cluster.NotMasterException.class, org.elasticsearch.cluster.NotMasterException::new, 144),
-        STATUS_EXCEPTION(org.elasticsearch.ElasticsearchStatusException.class, org.elasticsearch.ElasticsearchStatusException::new, 145);
+                ShardStateAction.NoLongerPrimaryShardException::new, 142, UNKNOWN_VERSION_ADDED),
+        SCRIPT_EXCEPTION(org.elasticsearch.script.ScriptException.class, org.elasticsearch.script.ScriptException::new, 143,
+            UNKNOWN_VERSION_ADDED),
+        NOT_MASTER_EXCEPTION(org.elasticsearch.cluster.NotMasterException.class, org.elasticsearch.cluster.NotMasterException::new, 144,
+            UNKNOWN_VERSION_ADDED),
+        STATUS_EXCEPTION(org.elasticsearch.ElasticsearchStatusException.class, org.elasticsearch.ElasticsearchStatusException::new, 145,
+            UNKNOWN_VERSION_ADDED),
+        TASK_CANCELLED_EXCEPTION(org.elasticsearch.tasks.TaskCancelledException.class,
+            org.elasticsearch.tasks.TaskCancelledException::new, 146, Version.V_5_1_1),
+        SHARD_LOCK_OBTAIN_FAILED_EXCEPTION(org.elasticsearch.env.ShardLockObtainFailedException.class,
+                                           org.elasticsearch.env.ShardLockObtainFailedException::new, 147, Version.V_5_0_2),
+        UNKNOWN_NAMED_OBJECT_EXCEPTION(org.elasticsearch.common.xcontent.NamedXContentRegistry.UnknownNamedObjectException.class,
+                org.elasticsearch.common.xcontent.NamedXContentRegistry.UnknownNamedObjectException::new, 148, Version.V_5_2_0);
 
         final Class<? extends ElasticsearchException> exceptionClass;
-        final FunctionThatThrowsIOException<StreamInput, ? extends ElasticsearchException> constructor;
+        final CheckedFunction<StreamInput, ? extends ElasticsearchException, IOException> constructor;
         final int id;
+        final Version versionAdded;
 
         <E extends ElasticsearchException> ElasticsearchExceptionHandle(Class<E> exceptionClass,
-                FunctionThatThrowsIOException<StreamInput, E> constructor, int id) {
+                                                                        CheckedFunction<StreamInput, E, IOException> constructor, int id,
+                                                                        Version versionAdded) {
             // We need the exceptionClass because you can't dig it out of the constructor reliably.
             this.exceptionClass = exceptionClass;
             this.constructor = constructor;
+            this.versionAdded = versionAdded;
             this.id = id;
         }
     }
 
+    /**
+     * Returns an array of all registered handle IDs. These are the IDs for every registered
+     * exception.
+     *
+     * @return an array of all registered handle IDs
+     */
+    static int[] ids() {
+        return Arrays.stream(ElasticsearchExceptionHandle.values()).mapToInt(h -> h.id).toArray();
+    }
+
+    /**
+     * Returns an array of all registered pairs of handle IDs and exception classes. These pairs are
+     * provided for every registered exception.
+     *
+     * @return an array of all registered pairs of handle IDs and exception classes
+     */
+    static Tuple<Integer, Class<? extends ElasticsearchException>>[] classes() {
+        @SuppressWarnings("unchecked")
+        final Tuple<Integer, Class<? extends ElasticsearchException>>[] ts =
+                Arrays.stream(ElasticsearchExceptionHandle.values())
+                        .map(h -> Tuple.tuple(h.id, h.exceptionClass)).toArray(Tuple[]::new);
+        return ts;
+    }
+
     static {
         ID_TO_SUPPLIER = unmodifiableMap(Arrays
                 .stream(ElasticsearchExceptionHandle.values()).collect(Collectors.toMap(e -> e.id, e -> e.constructor)));
@@ -717,9 +1042,9 @@ <E extends ElasticsearchException> ElasticsearchExceptionHandle(Class<E> excepti
     }
 
     public Index getIndex() {
-        List<String> index = getHeader(INDEX_HEADER_KEY);
+        List<String> index = getMetadata(INDEX_METADATA_KEY);
         if (index != null && index.isEmpty() == false) {
-            List<String> index_uuid = getHeader(INDEX_HEADER_KEY_UUID);
+            List<String> index_uuid = getMetadata(INDEX_METADATA_KEY_UUID);
             return new Index(index.get(0), index_uuid.get(0));
         }
 
@@ -727,7 +1052,7 @@ public Index getIndex() {
     }
 
     public ShardId getShardId() {
-        List<String> shard = getHeader(SHARD_HEADER_KEY);
+        List<String> shard = getMetadata(SHARD_METADATA_KEY);
         if (shard != null && shard.isEmpty() == false) {
             return new ShardId(getIndex(), Integer.parseInt(shard.get(0)));
         }
@@ -736,8 +1061,8 @@ public ShardId getShardId() {
 
     public void setIndex(Index index) {
         if (index != null) {
-            addHeader(INDEX_HEADER_KEY, index.getName());
-            addHeader(INDEX_HEADER_KEY_UUID, index.getUUID());
+            addMetadata(INDEX_METADATA_KEY, index.getName());
+            addMetadata(INDEX_METADATA_KEY_UUID, index.getUUID());
         }
     }
 
@@ -750,27 +1075,22 @@ public void setIndex(String index) {
     public void setShard(ShardId shardId) {
         if (shardId != null) {
             setIndex(shardId.getIndex());
-            addHeader(SHARD_HEADER_KEY, Integer.toString(shardId.id()));
+            addMetadata(SHARD_METADATA_KEY, Integer.toString(shardId.id()));
         }
     }
 
-    public void setShard(String index, int shardId) {
-            setIndex(index);
-            addHeader(SHARD_HEADER_KEY, Integer.toString(shardId));
-    }
-
     public void setResources(String type, String... id) {
         assert type != null;
-        addHeader(RESOURCE_HEADER_ID_KEY, id);
-        addHeader(RESOURCE_HEADER_TYPE_KEY, type);
+        addMetadata(RESOURCE_METADATA_ID_KEY, id);
+        addMetadata(RESOURCE_METADATA_TYPE_KEY, type);
     }
 
     public List<String> getResourceId() {
-        return getHeader(RESOURCE_HEADER_ID_KEY);
+        return getMetadata(RESOURCE_METADATA_ID_KEY);
     }
 
     public String getResourceType() {
-        List<String> header = getHeader(RESOURCE_HEADER_TYPE_KEY);
+        List<String> header = getMetadata(RESOURCE_METADATA_TYPE_KEY);
         if (header != null && header.isEmpty() == false) {
             assert header.size() == 1;
             return header.get(0);
@@ -778,26 +1098,6 @@ public String getResourceType() {
         return null;
     }
 
-    public static void renderException(XContentBuilder builder, Params params, Exception e) throws IOException {
-        builder.startObject("error");
-        final ElasticsearchException[] rootCauses = ElasticsearchException.guessRootCauses(e);
-        builder.field("root_cause");
-        builder.startArray();
-        for (ElasticsearchException rootCause : rootCauses) {
-            builder.startObject();
-            rootCause.toXContent(builder, new ToXContent.DelegatingMapParams(
-                    Collections.singletonMap(ElasticsearchException.REST_EXCEPTION_SKIP_CAUSE, "true"), params));
-            builder.endObject();
-        }
-        builder.endArray();
-        ElasticsearchException.toXContent(builder, params, e);
-        builder.endObject();
-    }
-
-    interface FunctionThatThrowsIOException<T, R> {
-        R apply(T t) throws IOException;
-    }
-
     // lower cases and adds underscores to transitions in a name
     private static String toUnderscoreCase(String value) {
         StringBuilder sb = new StringBuilder();
@@ -832,4 +1132,5 @@ private static String toUnderscoreCase(String value) {
         }
         return sb.toString();
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/ElasticsearchParseException.java b/core/src/main/java/org/elasticsearch/ElasticsearchParseException.java
index 1358ef54d9d89..1711e9a3aaf91 100644
--- a/core/src/main/java/org/elasticsearch/ElasticsearchParseException.java
+++ b/core/src/main/java/org/elasticsearch/ElasticsearchParseException.java
@@ -25,7 +25,7 @@
 import java.io.IOException;
 
 /**
- *
+ * Unchecked exception that is translated into a {@code 400 BAD REQUEST} error when it bubbles out over HTTP.
  */
 public class ElasticsearchParseException extends ElasticsearchException {
 
diff --git a/core/src/main/java/org/elasticsearch/ExceptionsHelper.java b/core/src/main/java/org/elasticsearch/ExceptionsHelper.java
index c30662a093479..e89e04a301da1 100644
--- a/core/src/main/java/org/elasticsearch/ExceptionsHelper.java
+++ b/core/src/main/java/org/elasticsearch/ExceptionsHelper.java
@@ -214,7 +214,7 @@ static class GroupBy {
         final String index;
         final Class<? extends Throwable> causeType;
 
-        public GroupBy(Throwable t) {
+        GroupBy(Throwable t) {
             if (t instanceof ElasticsearchException) {
                 final Index index = ((ElasticsearchException) t).getIndex();
                 if (index != null) {
diff --git a/core/src/main/java/org/elasticsearch/ResourceAlreadyExistsException.java b/core/src/main/java/org/elasticsearch/ResourceAlreadyExistsException.java
new file mode 100644
index 0000000000000..8ab5cb433f41d
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/ResourceAlreadyExistsException.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.rest.RestStatus;
+
+import java.io.IOException;
+
+public class ResourceAlreadyExistsException extends ElasticsearchException {
+
+    public ResourceAlreadyExistsException(Index index) {
+        this("index {} already exists", index.toString());
+        setIndex(index);
+    }
+
+    public ResourceAlreadyExistsException(String msg, Object... args) {
+        super(msg, args);
+    }
+
+    public ResourceAlreadyExistsException(StreamInput in) throws IOException{
+        super(in);
+    }
+
+    @Override
+    public RestStatus status() {
+        return RestStatus.BAD_REQUEST;
+    }
+
+    @Override
+    protected String getExceptionName() {
+        if (getIndex() != null) {
+            // This is only for BWC and is removed in 6.0.0
+            return "index_already_exists_exception";
+        } else {
+            return super.getExceptionName();
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/Version.java b/core/src/main/java/org/elasticsearch/Version.java
index 3b4282c9b5624..5c4295a9ff080 100644
--- a/core/src/main/java/org/elasticsearch/Version.java
+++ b/core/src/main/java/org/elasticsearch/Version.java
@@ -29,9 +29,7 @@
 
 import java.io.IOException;
 
-/**
- */
-public class Version {
+public class Version implements Comparable<Version> {
     /*
      * The logic for ID is: XXYYZZAA, where XX is major version, YY is minor version, ZZ is revision, and AA is alpha/beta/rc indicator AA
      * values below 25 are for alpha builder (since 5.0), and above 25 and below 50 are beta builds, and below 99 are RC builds, with 99
@@ -75,6 +73,18 @@ public class Version {
     public static final Version V_2_3_5 = new Version(V_2_3_5_ID, org.apache.lucene.util.Version.LUCENE_5_5_0);
     public static final int V_2_4_0_ID = 2040099;
     public static final Version V_2_4_0 = new Version(V_2_4_0_ID, org.apache.lucene.util.Version.LUCENE_5_5_2);
+    public static final int V_2_4_1_ID = 2040199;
+    public static final Version V_2_4_1 = new Version(V_2_4_1_ID, org.apache.lucene.util.Version.LUCENE_5_5_2);
+    public static final int V_2_4_2_ID = 2040299;
+    public static final Version V_2_4_2 = new Version(V_2_4_2_ID, org.apache.lucene.util.Version.LUCENE_5_5_2);
+    public static final int V_2_4_3_ID = 2040399;
+    public static final Version V_2_4_3 = new Version(V_2_4_3_ID, org.apache.lucene.util.Version.LUCENE_5_5_2);
+    public static final int V_2_4_4_ID = 2040499;
+    public static final Version V_2_4_4 = new Version(V_2_4_4_ID, org.apache.lucene.util.Version.LUCENE_5_5_2);
+    public static final int V_2_4_5_ID = 2040599;
+    public static final Version V_2_4_5 = new Version(V_2_4_5_ID, org.apache.lucene.util.Version.LUCENE_5_5_4);
+    public static final int V_2_4_6_ID = 2040699;
+    public static final Version V_2_4_6 = new Version(V_2_4_6_ID, org.apache.lucene.util.Version.LUCENE_5_5_4);
     public static final int V_5_0_0_alpha1_ID = 5000001;
     public static final Version V_5_0_0_alpha1 = new Version(V_5_0_0_alpha1_ID, org.apache.lucene.util.Version.LUCENE_6_0_0);
     public static final int V_5_0_0_alpha2_ID = 5000002;
@@ -85,9 +95,64 @@ public class Version {
     public static final Version V_5_0_0_alpha4 = new Version(V_5_0_0_alpha4_ID, org.apache.lucene.util.Version.LUCENE_6_1_0);
     public static final int V_5_0_0_alpha5_ID = 5000005;
     public static final Version V_5_0_0_alpha5 = new Version(V_5_0_0_alpha5_ID, org.apache.lucene.util.Version.LUCENE_6_1_0);
-    public static final int V_5_0_0_alpha6_ID = 5000006;
-    public static final Version V_5_0_0_alpha6 = new Version(V_5_0_0_alpha6_ID, org.apache.lucene.util.Version.LUCENE_6_2_0);
-    public static final Version CURRENT = V_5_0_0_alpha6;
+    public static final int V_5_0_0_beta1_ID = 5000026;
+    public static final Version V_5_0_0_beta1 = new Version(V_5_0_0_beta1_ID, org.apache.lucene.util.Version.LUCENE_6_2_0);
+    public static final int V_5_0_0_rc1_ID = 5000051;
+    public static final Version V_5_0_0_rc1 = new Version(V_5_0_0_rc1_ID, org.apache.lucene.util.Version.LUCENE_6_2_0);
+    public static final int V_5_0_0_ID = 5000099;
+    public static final Version V_5_0_0 = new Version(V_5_0_0_ID, org.apache.lucene.util.Version.LUCENE_6_2_0);
+    public static final int V_5_0_1_ID = 5000199;
+    public static final Version V_5_0_1 = new Version(V_5_0_1_ID, org.apache.lucene.util.Version.LUCENE_6_2_1);
+    public static final int V_5_0_2_ID = 5000299;
+    public static final Version V_5_0_2 = new Version(V_5_0_2_ID, org.apache.lucene.util.Version.LUCENE_6_2_1);
+    // no version constant for 5.1.0 due to inadvertent release
+    public static final int V_5_1_1_ID = 5010199;
+    public static final Version V_5_1_1 = new Version(V_5_1_1_ID, org.apache.lucene.util.Version.LUCENE_6_3_0);
+    public static final int V_5_1_2_ID = 5010299;
+    public static final Version V_5_1_2 = new Version(V_5_1_2_ID, org.apache.lucene.util.Version.LUCENE_6_3_0);
+    public static final int V_5_2_0_ID = 5020099;
+    public static final Version V_5_2_0 = new Version(V_5_2_0_ID, org.apache.lucene.util.Version.LUCENE_6_4_0);
+    public static final int V_5_2_1_ID = 5020199;
+    public static final Version V_5_2_1 = new Version(V_5_2_1_ID, org.apache.lucene.util.Version.LUCENE_6_4_1);
+    public static final int V_5_2_2_ID = 5020299;
+    public static final Version V_5_2_2 = new Version(V_5_2_2_ID, org.apache.lucene.util.Version.LUCENE_6_4_1);
+    public static final int V_5_3_0_ID = 5030099;
+    public static final Version V_5_3_0 = new Version(V_5_3_0_ID, org.apache.lucene.util.Version.LUCENE_6_4_1);
+    public static final int V_5_3_1_ID = 5030199;
+    public static final Version V_5_3_1 = new Version(V_5_3_1_ID, org.apache.lucene.util.Version.LUCENE_6_4_2);
+    public static final int V_5_3_2_ID = 5030299;
+    public static final Version V_5_3_2 = new Version(V_5_3_2_ID, org.apache.lucene.util.Version.LUCENE_6_4_2);
+    public static final int V_5_3_3_ID = 5030399;
+    public static final Version V_5_3_3 = new Version(V_5_3_3_ID, org.apache.lucene.util.Version.LUCENE_6_4_2);
+    public static final int V_5_4_0_ID = 5040099;
+    public static final Version V_5_4_0 = new Version(V_5_4_0_ID, org.apache.lucene.util.Version.LUCENE_6_5_0);
+    public static final int V_5_4_1_ID = 5040199;
+    public static final Version V_5_4_1 = new Version(V_5_4_1_ID, org.apache.lucene.util.Version.LUCENE_6_5_1);
+    public static final int V_5_4_2_ID = 5040299;
+    public static final Version V_5_4_2 = new Version(V_5_4_2_ID, org.apache.lucene.util.Version.LUCENE_6_5_1);
+    public static final int V_5_4_3_ID = 5040399;
+    public static final Version V_5_4_3 = new Version(V_5_4_3_ID, org.apache.lucene.util.Version.LUCENE_6_5_1);
+    public static final int V_5_5_0_ID = 5050099;
+    public static final Version V_5_5_0 = new Version(V_5_5_0_ID, org.apache.lucene.util.Version.LUCENE_6_6_0);
+    public static final int V_5_5_1_ID = 5050199;
+    public static final Version V_5_5_1 = new Version(V_5_5_1_ID, org.apache.lucene.util.Version.LUCENE_6_6_0);
+    public static final int V_5_5_2_ID = 5050299;
+    public static final Version V_5_5_2 = new Version(V_5_5_2_ID, org.apache.lucene.util.Version.LUCENE_6_6_0);
+    public static final int V_5_5_3_ID = 5050399;
+    public static final Version V_5_5_3 = new Version(V_5_5_3_ID, org.apache.lucene.util.Version.LUCENE_6_6_0);
+    public static final int V_5_6_0_ID = 5060099;
+    public static final Version V_5_6_0 = new Version(V_5_6_0_ID, org.apache.lucene.util.Version.LUCENE_6_6_0);
+    public static final int V_5_6_1_ID = 5060199;
+    public static final Version V_5_6_1 = new Version(V_5_6_1_ID, org.apache.lucene.util.Version.LUCENE_6_6_1);
+    public static final int V_5_6_2_ID = 5060299;
+    public static final Version V_5_6_2 = new Version(V_5_6_2_ID, org.apache.lucene.util.Version.LUCENE_6_6_1);
+    public static final int V_5_6_3_ID = 5060399;
+    public static final Version V_5_6_3 = new Version(V_5_6_3_ID, org.apache.lucene.util.Version.LUCENE_6_6_1);
+    public static final int V_5_6_4_ID_UNRELEASED = 5060499;
+    public static final Version V_5_6_4_UNRELEASED = new Version(V_5_6_4_ID_UNRELEASED, org.apache.lucene.util.Version.LUCENE_6_6_1);
+    public static final Version CURRENT = V_5_6_4_UNRELEASED;
+
+    // unreleased versions must be added to the above list with the suffix _UNRELEASED (with the exception of CURRENT)
 
     static {
         assert CURRENT.luceneVersion.equals(org.apache.lucene.util.Version.LATEST) : "Version must be upgraded to ["
@@ -100,8 +165,60 @@ public static Version readVersion(StreamInput in) throws IOException {
 
     public static Version fromId(int id) {
         switch (id) {
-            case V_5_0_0_alpha6_ID:
-                return V_5_0_0_alpha6;
+            case V_5_6_4_ID_UNRELEASED:
+                return V_5_6_4_UNRELEASED;
+            case V_5_6_3_ID:
+                return V_5_6_3;
+            case V_5_6_2_ID:
+                return V_5_6_2;
+            case V_5_6_1_ID:
+                return V_5_6_1;
+            case V_5_6_0_ID:
+                return V_5_6_0;
+            case V_5_5_3_ID:
+                return V_5_5_3;
+            case V_5_5_2_ID:
+                return V_5_5_2;
+            case V_5_5_1_ID:
+                return V_5_5_1;
+            case V_5_5_0_ID:
+                return V_5_5_0;
+            case V_5_4_3_ID:
+                return V_5_4_3;
+            case V_5_4_2_ID:
+                return V_5_4_2;
+            case V_5_4_1_ID:
+                return V_5_4_1;
+            case V_5_4_0_ID:
+                return V_5_4_0;
+            case V_5_3_3_ID:
+                return V_5_3_3;
+            case V_5_3_2_ID:
+                return V_5_3_2;
+            case V_5_3_1_ID:
+                return V_5_3_1;
+            case V_5_3_0_ID:
+                return V_5_3_0;
+            case V_5_2_2_ID:
+                return V_5_2_2;
+            case V_5_2_1_ID:
+                return V_5_2_1;
+            case V_5_2_0_ID:
+                return V_5_2_0;
+            case V_5_1_2_ID:
+                return V_5_1_2;
+            case V_5_1_1_ID:
+                return V_5_1_1;
+            case V_5_0_2_ID:
+                return V_5_0_2;
+            case V_5_0_1_ID:
+                return V_5_0_1;
+            case V_5_0_0_ID:
+                return V_5_0_0;
+            case V_5_0_0_rc1_ID:
+                return V_5_0_0_rc1;
+            case V_5_0_0_beta1_ID:
+                return V_5_0_0_beta1;
             case V_5_0_0_alpha5_ID:
                 return V_5_0_0_alpha5;
             case V_5_0_0_alpha4_ID:
@@ -112,6 +229,18 @@ public static Version fromId(int id) {
                 return V_5_0_0_alpha2;
             case V_5_0_0_alpha1_ID:
                 return V_5_0_0_alpha1;
+            case V_2_4_6_ID:
+                return V_2_4_6;
+            case V_2_4_5_ID:
+                return V_2_4_5;
+            case V_2_4_4_ID:
+                return V_2_4_4;
+            case V_2_4_3_ID:
+                return V_2_4_3;
+            case V_2_4_2_ID:
+                return V_2_4_2;
+            case V_2_4_1_ID:
+                return V_2_4_1;
             case V_2_4_0_ID:
                 return V_2_4_0;
             case V_2_3_5_ID:
@@ -176,12 +305,26 @@ public static void writeVersion(Version version, StreamOutput out) throws IOExce
     }
 
     /**
-     * Returns the smallest version between the 2.
+     * Returns the minimum version between the 2.
      */
-    public static Version smallest(Version version1, Version version2) {
+    public static Version min(Version version1, Version version2) {
         return version1.id < version2.id ? version1 : version2;
     }
 
+    /**
+     * Returns the minimum version between the 2.
+     * @deprecated use {@link #min(Version, Version)} instead
+     */
+    @Deprecated
+    public static Version smallest(Version version1, Version version2) {
+        return min(version1, version2);
+    }
+
+    /**
+     * Returns the maximum version between the 2
+     */
+    public static Version max(Version version1, Version version2) { return version1.id > version2.id ? version1 : version2; }
+
     /**
      * Returns the version given its string representation, current version if the argument is null or empty
      */
@@ -193,7 +336,7 @@ public static Version fromString(String version) {
         if (snapshot = version.endsWith("-SNAPSHOT")) {
             version = version.substring(0, version.length() - 9);
         }
-        String[] parts = version.split("\\.|\\-");
+        String[] parts = version.split("[.-]");
         if (parts.length < 3 || parts.length > 4) {
             throw new IllegalArgumentException(
                     "the version needs to contain major, minor, and revision, and optionally the build: " + version);
@@ -267,6 +410,11 @@ public boolean onOrBefore(Version version) {
         return version.id >= id;
     }
 
+    @Override
+    public int compareTo(Version other) {
+        return Integer.compare(this.id, other.id);
+    }
+
     /**
      * Returns the minimum compatible version based on the current
      * version. Ie a node needs to have at least the return version in order
@@ -275,7 +423,52 @@ public boolean onOrBefore(Version version) {
      * is a beta or RC release then the version itself is returned.
      */
     public Version minimumCompatibilityVersion() {
-        return Version.smallest(this, fromId(major * 1000000 + 99));
+        final int bwcMajor;
+        final int bwcMinor;
+        if (major >= 6) {
+            bwcMajor = major - 1;
+            bwcMinor = CURRENT.minor;
+        } else {
+            bwcMajor = major;
+            bwcMinor = 0;
+        }
+        return Version.min(this, fromId(bwcMajor * 1000000 + bwcMinor * 10000 + 99));
+    }
+
+    /**
+     * Returns the minimum created index version that this version supports. Indices created with lower versions
+     * can't be used with this version.
+     */
+    public Version minimumIndexCompatibilityVersion() {
+        final int bwcMajor;
+        if (major == 5) {
+            bwcMajor = 2; // we jumped from 2 to 5
+        } else {
+            bwcMajor = major - 1;
+        }
+        final int bwcMinor;
+        if (bwcMajor <= 2) {
+            // we do support beta1 prior to 5.x
+            // this allows clusters that have upgraded to 5.0 with an index created in 2.0.0.beta1 to go to 5.2 etc.
+            // otherwise the upgrade will fail and that is really not what we want. from 5 onwards we are supporting only GA
+            // releases
+            bwcMinor = 01;
+        } else {
+            bwcMinor = 99;
+        }
+
+        return Version.min(this, fromId(bwcMajor * 1000000 + bwcMinor));
+    }
+
+    /**
+     * Returns <code>true</code> iff both version are compatible. Otherwise <code>false</code>
+     */
+    public boolean isCompatible(Version version) {
+        boolean compatible = onOrAfter(version.minimumCompatibilityVersion())
+            && version.onOrAfter(minimumCompatibilityVersion());
+
+        assert compatible == false || Math.max(major, version.major) - Math.min(major, version.major) <= 1;
+        return compatible;
     }
 
     @SuppressForbidden(reason = "System.out.*")
@@ -348,4 +541,9 @@ public boolean isAlpha() {
     public boolean isRC() {
         return build > 50 && build < 99;
     }
+
+    public boolean isRelease() {
+        return build == 99;
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/action/ActionListener.java b/core/src/main/java/org/elasticsearch/action/ActionListener.java
index 6ce8592879e08..f9fafa9f95a2e 100644
--- a/core/src/main/java/org/elasticsearch/action/ActionListener.java
+++ b/core/src/main/java/org/elasticsearch/action/ActionListener.java
@@ -19,6 +19,11 @@
 
 package org.elasticsearch.action;
 
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.common.CheckedConsumer;
+
+import java.util.ArrayList;
+import java.util.List;
 import java.util.function.Consumer;
 
 /**
@@ -45,7 +50,8 @@ public interface ActionListener<Response> {
      * @param <Response> the type of the response
      * @return a listener that listens for responses and invokes the consumer when received
      */
-    static <Response> ActionListener<Response> wrap(Consumer<Response> onResponse, Consumer<Exception> onFailure) {
+    static <Response> ActionListener<Response> wrap(CheckedConsumer<Response, ? extends Exception> onResponse,
+            Consumer<Exception> onFailure) {
         return new ActionListener<Response>() {
             @Override
             public void onResponse(Response response) {
@@ -62,4 +68,41 @@ public void onFailure(Exception e) {
             }
         };
     }
+
+    /**
+     * Notifies every given listener with the response passed to {@link #onResponse(Object)}. If a listener itself throws an exception
+     * the exception is forwarded to {@link #onFailure(Exception)}. If in turn {@link #onFailure(Exception)} fails all remaining
+     * listeners will be processed and the caught exception will be re-thrown.
+     */
+    static <Response> void onResponse(Iterable<ActionListener<Response>> listeners, Response response) {
+        List<Exception> exceptionList = new ArrayList<>();
+        for (ActionListener<Response> listener : listeners) {
+            try {
+                listener.onResponse(response);
+            } catch (Exception ex) {
+                try {
+                    listener.onFailure(ex);
+                } catch (Exception ex1) {
+                    exceptionList.add(ex1);
+                }
+            }
+        }
+        ExceptionsHelper.maybeThrowRuntimeAndSuppress(exceptionList);
+    }
+
+    /**
+     * Notifies every given listener with the failure passed to {@link #onFailure(Exception)}. If a listener itself throws an exception
+     * all remaining listeners will be processed and the caught exception will be re-thrown.
+     */
+    static <Response> void onFailure(Iterable<ActionListener<Response>> listeners, Exception failure) {
+        List<Exception> exceptionList = new ArrayList<>();
+        for (ActionListener<Response> listener : listeners) {
+            try {
+                listener.onFailure(failure);
+            } catch (Exception ex) {
+                exceptionList.add(ex);
+            }
+        }
+        ExceptionsHelper.maybeThrowRuntimeAndSuppress(exceptionList);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/ActionModule.java b/core/src/main/java/org/elasticsearch/action/ActionModule.java
index 1be1ddda9a470..8aa52de08569d 100644
--- a/core/src/main/java/org/elasticsearch/action/ActionModule.java
+++ b/core/src/main/java/org/elasticsearch/action/ActionModule.java
@@ -19,13 +19,7 @@
 
 package org.elasticsearch.action;
 
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplainAction;
 import org.elasticsearch.action.admin.cluster.allocation.TransportClusterAllocationExplainAction;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
@@ -43,6 +37,8 @@
 import org.elasticsearch.action.admin.cluster.node.tasks.get.TransportGetTaskAction;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksAction;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.TransportListTasksAction;
+import org.elasticsearch.action.admin.cluster.remote.RemoteInfoAction;
+import org.elasticsearch.action.admin.cluster.remote.TransportRemoteInfoAction;
 import org.elasticsearch.action.admin.cluster.repositories.delete.DeleteRepositoryAction;
 import org.elasticsearch.action.admin.cluster.repositories.delete.TransportDeleteRepositoryAction;
 import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesAction;
@@ -155,6 +151,9 @@
 import org.elasticsearch.action.delete.TransportDeleteAction;
 import org.elasticsearch.action.explain.ExplainAction;
 import org.elasticsearch.action.explain.TransportExplainAction;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesAction;
+import org.elasticsearch.action.fieldcaps.TransportFieldCapabilitiesAction;
+import org.elasticsearch.action.fieldcaps.TransportFieldCapabilitiesIndexAction;
 import org.elasticsearch.action.fieldstats.FieldStatsAction;
 import org.elasticsearch.action.fieldstats.TransportFieldStatsAction;
 import org.elasticsearch.action.get.GetAction;
@@ -168,8 +167,6 @@
 import org.elasticsearch.action.ingest.DeletePipelineTransportAction;
 import org.elasticsearch.action.ingest.GetPipelineAction;
 import org.elasticsearch.action.ingest.GetPipelineTransportAction;
-import org.elasticsearch.action.ingest.IngestActionFilter;
-import org.elasticsearch.action.ingest.IngestProxyActionFilter;
 import org.elasticsearch.action.ingest.PutPipelineAction;
 import org.elasticsearch.action.ingest.PutPipelineTransportAction;
 import org.elasticsearch.action.ingest.SimulatePipelineAction;
@@ -196,18 +193,24 @@
 import org.elasticsearch.action.termvectors.TransportTermVectorsAction;
 import org.elasticsearch.action.update.TransportUpdateAction;
 import org.elasticsearch.action.update.UpdateAction;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.NamedRegistry;
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.inject.multibindings.MapBinder;
 import org.elasticsearch.common.inject.multibindings.Multibinder;
-import org.elasticsearch.common.network.NetworkModule;
+import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsFilter;
+import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.ActionPlugin.ActionHandler;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestHandler;
+import org.elasticsearch.rest.action.RestFieldCapabilitiesAction;
 import org.elasticsearch.rest.action.RestFieldStatsAction;
 import org.elasticsearch.rest.action.RestMainAction;
 import org.elasticsearch.rest.action.admin.cluster.RestCancelTasksAction;
@@ -234,10 +237,10 @@
 import org.elasticsearch.rest.action.admin.cluster.RestPendingClusterTasksAction;
 import org.elasticsearch.rest.action.admin.cluster.RestPutRepositoryAction;
 import org.elasticsearch.rest.action.admin.cluster.RestPutStoredScriptAction;
+import org.elasticsearch.rest.action.admin.cluster.RestRemoteClusterInfoAction;
 import org.elasticsearch.rest.action.admin.cluster.RestRestoreSnapshotAction;
 import org.elasticsearch.rest.action.admin.cluster.RestSnapshotsStatusAction;
 import org.elasticsearch.rest.action.admin.cluster.RestVerifyRepositoryAction;
-import org.elasticsearch.rest.action.admin.indices.RestAliasesExistAction;
 import org.elasticsearch.rest.action.admin.indices.RestAnalyzeAction;
 import org.elasticsearch.rest.action.admin.indices.RestClearIndicesCacheAction;
 import org.elasticsearch.rest.action.admin.indices.RestCloseIndexAction;
@@ -252,11 +255,9 @@
 import org.elasticsearch.rest.action.admin.indices.RestGetIndicesAction;
 import org.elasticsearch.rest.action.admin.indices.RestGetMappingAction;
 import org.elasticsearch.rest.action.admin.indices.RestGetSettingsAction;
-import org.elasticsearch.rest.action.admin.indices.RestHeadIndexTemplateAction;
 import org.elasticsearch.rest.action.admin.indices.RestIndexDeleteAliasesAction;
 import org.elasticsearch.rest.action.admin.indices.RestIndexPutAliasAction;
 import org.elasticsearch.rest.action.admin.indices.RestIndicesAliasesAction;
-import org.elasticsearch.rest.action.admin.indices.RestIndicesExistsAction;
 import org.elasticsearch.rest.action.admin.indices.RestIndicesSegmentsAction;
 import org.elasticsearch.rest.action.admin.indices.RestIndicesShardStoresAction;
 import org.elasticsearch.rest.action.admin.indices.RestIndicesStatsAction;
@@ -268,7 +269,6 @@
 import org.elasticsearch.rest.action.admin.indices.RestRolloverIndexAction;
 import org.elasticsearch.rest.action.admin.indices.RestShrinkIndexAction;
 import org.elasticsearch.rest.action.admin.indices.RestSyncedFlushAction;
-import org.elasticsearch.rest.action.admin.indices.RestTypesExistsAction;
 import org.elasticsearch.rest.action.admin.indices.RestUpdateSettingsAction;
 import org.elasticsearch.rest.action.admin.indices.RestUpgradeAction;
 import org.elasticsearch.rest.action.admin.indices.RestValidateQueryAction;
@@ -288,12 +288,12 @@
 import org.elasticsearch.rest.action.cat.RestShardsAction;
 import org.elasticsearch.rest.action.cat.RestSnapshotAction;
 import org.elasticsearch.rest.action.cat.RestTasksAction;
+import org.elasticsearch.rest.action.cat.RestTemplatesAction;
 import org.elasticsearch.rest.action.cat.RestThreadPoolAction;
 import org.elasticsearch.rest.action.document.RestBulkAction;
 import org.elasticsearch.rest.action.document.RestDeleteAction;
 import org.elasticsearch.rest.action.document.RestGetAction;
 import org.elasticsearch.rest.action.document.RestGetSourceAction;
-import org.elasticsearch.rest.action.document.RestHeadAction;
 import org.elasticsearch.rest.action.document.RestIndexAction;
 import org.elasticsearch.rest.action.document.RestMultiGetAction;
 import org.elasticsearch.rest.action.document.RestMultiTermVectorsAction;
@@ -309,6 +309,16 @@
 import org.elasticsearch.rest.action.search.RestSearchAction;
 import org.elasticsearch.rest.action.search.RestSearchScrollAction;
 import org.elasticsearch.rest.action.search.RestSuggestAction;
+import org.elasticsearch.threadpool.ThreadPool;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Consumer;
+import java.util.function.Supplier;
+import java.util.function.UnaryOperator;
+import java.util.stream.Collectors;
 
 import static java.util.Collections.unmodifiableList;
 import static java.util.Collections.unmodifiableMap;
@@ -318,8 +328,14 @@
  */
 public class ActionModule extends AbstractModule {
 
+    private static final Logger logger = ESLoggerFactory.getLogger(ActionModule.class);
+
     private final boolean transportClient;
     private final Settings settings;
+    private final IndexNameExpressionResolver indexNameExpressionResolver;
+    private final IndexScopedSettings indexScopedSettings;
+    private final ClusterSettings clusterSettings;
+    private final SettingsFilter settingsFilter;
     private final List<ActionPlugin> actionPlugins;
     private final Map<String, ActionHandler<?, ?>> actions;
     private final List<Class<? extends ActionFilter>> actionFilters;
@@ -327,19 +343,41 @@ public class ActionModule extends AbstractModule {
     private final DestructiveOperations destructiveOperations;
     private final RestController restController;
 
-    public ActionModule(boolean ingestEnabled, boolean transportClient, Settings settings, IndexNameExpressionResolver resolver,
-            ClusterSettings clusterSettings, List<ActionPlugin> actionPlugins) {
+    public ActionModule(boolean transportClient, Settings settings, IndexNameExpressionResolver indexNameExpressionResolver,
+                        IndexScopedSettings indexScopedSettings, ClusterSettings clusterSettings, SettingsFilter settingsFilter,
+                        ThreadPool threadPool, List<ActionPlugin> actionPlugins, NodeClient nodeClient,
+                        CircuitBreakerService circuitBreakerService) {
         this.transportClient = transportClient;
         this.settings = settings;
+        this.indexNameExpressionResolver = indexNameExpressionResolver;
+        this.indexScopedSettings = indexScopedSettings;
+        this.clusterSettings = clusterSettings;
+        this.settingsFilter = settingsFilter;
         this.actionPlugins = actionPlugins;
         actions = setupActions(actionPlugins);
-        actionFilters = setupActionFilters(actionPlugins, ingestEnabled);
-        autoCreateIndex = transportClient ? null : new AutoCreateIndex(settings, clusterSettings, resolver);
+        actionFilters = setupActionFilters(actionPlugins);
+        autoCreateIndex = transportClient ? null : new AutoCreateIndex(settings, clusterSettings, indexNameExpressionResolver);
         destructiveOperations = new DestructiveOperations(settings, clusterSettings);
         Set<String> headers = actionPlugins.stream().flatMap(p -> p.getRestHeaders().stream()).collect(Collectors.toSet());
-        restController = new RestController(settings, headers);
+        UnaryOperator<RestHandler> restWrapper = null;
+        for (ActionPlugin plugin : actionPlugins) {
+            UnaryOperator<RestHandler> newRestWrapper = plugin.getRestHandlerWrapper(threadPool.getThreadContext());
+            if (newRestWrapper != null) {
+                logger.debug("Using REST wrapper from plugin " + plugin.getClass().getName());
+                if (restWrapper != null) {
+                    throw new IllegalArgumentException("Cannot have more than one plugin implementing a REST wrapper");
+                }
+                restWrapper = newRestWrapper;
+            }
+        }
+        if (transportClient) {
+            restController = null;
+        } else {
+            restController = new RestController(settings, headers, restWrapper, nodeClient, circuitBreakerService);
+        }
     }
 
+
     public Map<String, ActionHandler<?, ?>> getActions() {
         return actions;
     }
@@ -347,7 +385,7 @@ public ActionModule(boolean ingestEnabled, boolean transportClient, Settings set
     static Map<String, ActionHandler<?, ?>> setupActions(List<ActionPlugin> actionPlugins) {
         // Subclass NamedRegistry for easy registration
         class ActionRegistry extends NamedRegistry<ActionHandler<?, ?>> {
-            public ActionRegistry() {
+            ActionRegistry() {
                 super("action");
             }
 
@@ -355,7 +393,7 @@ public void register(ActionHandler<?, ?> handler) {
                 register(handler.getAction().name(), handler);
             }
 
-            public <Request extends ActionRequest<Request>, Response extends ActionResponse> void register(
+            public <Request extends ActionRequest, Response extends ActionResponse> void register(
                     GenericAction<Request, Response> action, Class<? extends TransportAction<Request, Response>> transportAction,
                     Class<?>... supportTransportActions) {
                 register(new ActionHandler<>(action, transportAction, supportTransportActions));
@@ -365,6 +403,7 @@ public <Request extends ActionRequest<Request>, Response extends ActionResponse>
 
         actions.register(MainAction.INSTANCE, TransportMainAction.class);
         actions.register(NodesInfoAction.INSTANCE, TransportNodesInfoAction.class);
+        actions.register(RemoteInfoAction.INSTANCE, TransportRemoteInfoAction.class);
         actions.register(NodesStatsAction.INSTANCE, TransportNodesStatsAction.class);
         actions.register(NodesHotThreadsAction.INSTANCE, TransportNodesHotThreadsAction.class);
         actions.register(ListTasksAction.INSTANCE, TransportListTasksAction.class);
@@ -448,6 +487,8 @@ public <Request extends ActionRequest<Request>, Response extends ActionResponse>
         actions.register(DeleteStoredScriptAction.INSTANCE, TransportDeleteStoredScriptAction.class);
 
         actions.register(FieldStatsAction.INSTANCE, TransportFieldStatsAction.class);
+        actions.register(FieldCapabilitiesAction.INSTANCE, TransportFieldCapabilitiesAction.class,
+            TransportFieldCapabilitiesIndexAction.class);
 
         actions.register(PutPipelineAction.INSTANCE, PutPipelineTransportAction.class);
         actions.register(GetPipelineAction.INSTANCE, GetPipelineTransportAction.class);
@@ -459,163 +500,146 @@ public <Request extends ActionRequest<Request>, Response extends ActionResponse>
         return unmodifiableMap(actions.getRegistry());
     }
 
-    private List<Class<? extends ActionFilter>> setupActionFilters(List<ActionPlugin> actionPlugins, boolean ingestEnabled) {
-        List<Class<? extends ActionFilter>> filters = new ArrayList<>();
-        if (transportClient == false) {
-            if (ingestEnabled) {
-                filters.add(IngestActionFilter.class);
-            } else {
-                filters.add(IngestProxyActionFilter.class);
-            }
-        }
-
-        for (ActionPlugin plugin : actionPlugins) {
-            filters.addAll(plugin.getActionFilters());
-        }
-        return unmodifiableList(filters);
+    private List<Class<? extends ActionFilter>> setupActionFilters(List<ActionPlugin> actionPlugins) {
+        return unmodifiableList(actionPlugins.stream().flatMap(p -> p.getActionFilters().stream()).collect(Collectors.toList()));
     }
 
-    static Set<Class<? extends RestHandler>> setupRestHandlers(List<ActionPlugin> actionPlugins) {
-        Set<Class<? extends RestHandler>> handlers = new HashSet<>();
-        registerRestHandler(handlers, RestMainAction.class);
-        registerRestHandler(handlers, RestNodesInfoAction.class);
-        registerRestHandler(handlers, RestNodesStatsAction.class);
-        registerRestHandler(handlers, RestNodesHotThreadsAction.class);
-        registerRestHandler(handlers, RestClusterAllocationExplainAction.class);
-        registerRestHandler(handlers, RestClusterStatsAction.class);
-        registerRestHandler(handlers, RestClusterStateAction.class);
-        registerRestHandler(handlers, RestClusterHealthAction.class);
-        registerRestHandler(handlers, RestClusterUpdateSettingsAction.class);
-        registerRestHandler(handlers, RestClusterGetSettingsAction.class);
-        registerRestHandler(handlers, RestClusterRerouteAction.class);
-        registerRestHandler(handlers, RestClusterSearchShardsAction.class);
-        registerRestHandler(handlers, RestPendingClusterTasksAction.class);
-        registerRestHandler(handlers, RestPutRepositoryAction.class);
-        registerRestHandler(handlers, RestGetRepositoriesAction.class);
-        registerRestHandler(handlers, RestDeleteRepositoryAction.class);
-        registerRestHandler(handlers, RestVerifyRepositoryAction.class);
-        registerRestHandler(handlers, RestGetSnapshotsAction.class);
-        registerRestHandler(handlers, RestCreateSnapshotAction.class);
-        registerRestHandler(handlers, RestRestoreSnapshotAction.class);
-        registerRestHandler(handlers, RestDeleteSnapshotAction.class);
-        registerRestHandler(handlers, RestSnapshotsStatusAction.class);
-
-        registerRestHandler(handlers, RestIndicesExistsAction.class);
-        registerRestHandler(handlers, RestTypesExistsAction.class);
-        registerRestHandler(handlers, RestGetIndicesAction.class);
-        registerRestHandler(handlers, RestIndicesStatsAction.class);
-        registerRestHandler(handlers, RestIndicesSegmentsAction.class);
-        registerRestHandler(handlers, RestIndicesShardStoresAction.class);
-        registerRestHandler(handlers, RestGetAliasesAction.class);
-        registerRestHandler(handlers, RestAliasesExistAction.class);
-        registerRestHandler(handlers, RestIndexDeleteAliasesAction.class);
-        registerRestHandler(handlers, RestIndexPutAliasAction.class);
-        registerRestHandler(handlers, RestIndicesAliasesAction.class);
-        registerRestHandler(handlers, RestCreateIndexAction.class);
-        registerRestHandler(handlers, RestShrinkIndexAction.class);
-        registerRestHandler(handlers, RestRolloverIndexAction.class);
-        registerRestHandler(handlers, RestDeleteIndexAction.class);
-        registerRestHandler(handlers, RestCloseIndexAction.class);
-        registerRestHandler(handlers, RestOpenIndexAction.class);
-
-        registerRestHandler(handlers, RestUpdateSettingsAction.class);
-        registerRestHandler(handlers, RestGetSettingsAction.class);
-
-        registerRestHandler(handlers, RestAnalyzeAction.class);
-        registerRestHandler(handlers, RestGetIndexTemplateAction.class);
-        registerRestHandler(handlers, RestPutIndexTemplateAction.class);
-        registerRestHandler(handlers, RestDeleteIndexTemplateAction.class);
-        registerRestHandler(handlers, RestHeadIndexTemplateAction.class);
-
-        registerRestHandler(handlers, RestPutMappingAction.class);
-        registerRestHandler(handlers, RestGetMappingAction.class);
-        registerRestHandler(handlers, RestGetFieldMappingAction.class);
-
-        registerRestHandler(handlers, RestRefreshAction.class);
-        registerRestHandler(handlers, RestFlushAction.class);
-        registerRestHandler(handlers, RestSyncedFlushAction.class);
-        registerRestHandler(handlers, RestForceMergeAction.class);
-        registerRestHandler(handlers, RestUpgradeAction.class);
-        registerRestHandler(handlers, RestClearIndicesCacheAction.class);
-
-        registerRestHandler(handlers, RestIndexAction.class);
-        registerRestHandler(handlers, RestGetAction.class);
-        registerRestHandler(handlers, RestGetSourceAction.class);
-        registerRestHandler(handlers, RestHeadAction.Document.class);
-        registerRestHandler(handlers, RestHeadAction.Source.class);
-        registerRestHandler(handlers, RestMultiGetAction.class);
-        registerRestHandler(handlers, RestDeleteAction.class);
-        registerRestHandler(handlers, org.elasticsearch.rest.action.document.RestCountAction.class);
-        registerRestHandler(handlers, RestSuggestAction.class);
-        registerRestHandler(handlers, RestTermVectorsAction.class);
-        registerRestHandler(handlers, RestMultiTermVectorsAction.class);
-        registerRestHandler(handlers, RestBulkAction.class);
-        registerRestHandler(handlers, RestUpdateAction.class);
-
-        registerRestHandler(handlers, RestSearchAction.class);
-        registerRestHandler(handlers, RestSearchScrollAction.class);
-        registerRestHandler(handlers, RestClearScrollAction.class);
-        registerRestHandler(handlers, RestMultiSearchAction.class);
-
-        registerRestHandler(handlers, RestValidateQueryAction.class);
-
-        registerRestHandler(handlers, RestExplainAction.class);
-
-        registerRestHandler(handlers, RestRecoveryAction.class);
+    public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster) {
+        List<AbstractCatAction> catActions = new ArrayList<>();
+        Consumer<RestHandler> registerHandler = a -> {
+            if (a instanceof AbstractCatAction) {
+                catActions.add((AbstractCatAction) a);
+            }
+        };
+        registerHandler.accept(new RestMainAction(settings, restController));
+        registerHandler.accept(new RestNodesInfoAction(settings, restController, settingsFilter));
+        registerHandler.accept(new RestRemoteClusterInfoAction(settings, restController));
+        registerHandler.accept(new RestNodesStatsAction(settings, restController));
+        registerHandler.accept(new RestNodesHotThreadsAction(settings, restController));
+        registerHandler.accept(new RestClusterAllocationExplainAction(settings, restController));
+        registerHandler.accept(new RestClusterStatsAction(settings, restController));
+        registerHandler.accept(new RestClusterStateAction(settings, restController, settingsFilter));
+        registerHandler.accept(new RestClusterHealthAction(settings, restController));
+        registerHandler.accept(new RestClusterUpdateSettingsAction(settings, restController));
+        registerHandler.accept(new RestClusterGetSettingsAction(settings, restController, clusterSettings, settingsFilter));
+        registerHandler.accept(new RestClusterRerouteAction(settings, restController, settingsFilter));
+        registerHandler.accept(new RestClusterSearchShardsAction(settings, restController));
+        registerHandler.accept(new RestPendingClusterTasksAction(settings, restController));
+        registerHandler.accept(new RestPutRepositoryAction(settings, restController));
+        registerHandler.accept(new RestGetRepositoriesAction(settings, restController, settingsFilter));
+        registerHandler.accept(new RestDeleteRepositoryAction(settings, restController));
+        registerHandler.accept(new RestVerifyRepositoryAction(settings, restController));
+        registerHandler.accept(new RestGetSnapshotsAction(settings, restController));
+        registerHandler.accept(new RestCreateSnapshotAction(settings, restController));
+        registerHandler.accept(new RestRestoreSnapshotAction(settings, restController));
+        registerHandler.accept(new RestDeleteSnapshotAction(settings, restController));
+        registerHandler.accept(new RestSnapshotsStatusAction(settings, restController));
+
+        registerHandler.accept(new RestGetIndicesAction(settings, restController, indexScopedSettings, settingsFilter));
+        registerHandler.accept(new RestIndicesStatsAction(settings, restController));
+        registerHandler.accept(new RestIndicesSegmentsAction(settings, restController));
+        registerHandler.accept(new RestIndicesShardStoresAction(settings, restController));
+        registerHandler.accept(new RestGetAliasesAction(settings, restController));
+        registerHandler.accept(new RestIndexDeleteAliasesAction(settings, restController));
+        registerHandler.accept(new RestIndexPutAliasAction(settings, restController));
+        registerHandler.accept(new RestIndicesAliasesAction(settings, restController));
+        registerHandler.accept(new RestCreateIndexAction(settings, restController));
+        registerHandler.accept(new RestShrinkIndexAction(settings, restController));
+        registerHandler.accept(new RestRolloverIndexAction(settings, restController));
+        registerHandler.accept(new RestDeleteIndexAction(settings, restController));
+        registerHandler.accept(new RestCloseIndexAction(settings, restController));
+        registerHandler.accept(new RestOpenIndexAction(settings, restController));
+
+        registerHandler.accept(new RestUpdateSettingsAction(settings, restController));
+        registerHandler.accept(new RestGetSettingsAction(settings, restController, indexScopedSettings, settingsFilter));
+
+        registerHandler.accept(new RestAnalyzeAction(settings, restController));
+        registerHandler.accept(new RestGetIndexTemplateAction(settings, restController));
+        registerHandler.accept(new RestPutIndexTemplateAction(settings, restController));
+        registerHandler.accept(new RestDeleteIndexTemplateAction(settings, restController));
+
+        registerHandler.accept(new RestPutMappingAction(settings, restController));
+        registerHandler.accept(new RestGetMappingAction(settings, restController));
+        registerHandler.accept(new RestGetFieldMappingAction(settings, restController));
+
+        registerHandler.accept(new RestRefreshAction(settings, restController));
+        registerHandler.accept(new RestFlushAction(settings, restController));
+        registerHandler.accept(new RestSyncedFlushAction(settings, restController));
+        registerHandler.accept(new RestForceMergeAction(settings, restController));
+        registerHandler.accept(new RestUpgradeAction(settings, restController));
+        registerHandler.accept(new RestClearIndicesCacheAction(settings, restController));
+
+        registerHandler.accept(new RestIndexAction(settings, restController));
+        registerHandler.accept(new RestGetAction(settings, restController));
+        registerHandler.accept(new RestGetSourceAction(settings, restController));
+        registerHandler.accept(new RestMultiGetAction(settings, restController));
+        registerHandler.accept(new RestDeleteAction(settings, restController));
+        registerHandler.accept(new org.elasticsearch.rest.action.document.RestCountAction(settings, restController));
+        registerHandler.accept(new RestTermVectorsAction(settings, restController));
+        registerHandler.accept(new RestSuggestAction(settings, restController));
+        registerHandler.accept(new RestMultiTermVectorsAction(settings, restController));
+        registerHandler.accept(new RestBulkAction(settings, restController));
+        registerHandler.accept(new RestUpdateAction(settings, restController));
+
+        registerHandler.accept(new RestSearchAction(settings, restController));
+        registerHandler.accept(new RestSearchScrollAction(settings, restController));
+        registerHandler.accept(new RestClearScrollAction(settings, restController));
+        registerHandler.accept(new RestMultiSearchAction(settings, restController));
+
+        registerHandler.accept(new RestValidateQueryAction(settings, restController));
+
+        registerHandler.accept(new RestExplainAction(settings, restController));
+
+        registerHandler.accept(new RestRecoveryAction(settings, restController));
 
         // Scripts API
-        registerRestHandler(handlers, RestGetStoredScriptAction.class);
-        registerRestHandler(handlers, RestPutStoredScriptAction.class);
-        registerRestHandler(handlers, RestDeleteStoredScriptAction.class);
+        registerHandler.accept(new RestGetStoredScriptAction(settings, restController));
+        registerHandler.accept(new RestPutStoredScriptAction(settings, restController));
+        registerHandler.accept(new RestDeleteStoredScriptAction(settings, restController));
 
-        registerRestHandler(handlers, RestFieldStatsAction.class);
+        registerHandler.accept(new RestFieldStatsAction(settings, restController));
+        registerHandler.accept(new RestFieldCapabilitiesAction(settings, restController));
 
         // Tasks API
-        registerRestHandler(handlers, RestListTasksAction.class);
-        registerRestHandler(handlers, RestGetTaskAction.class);
-        registerRestHandler(handlers, RestCancelTasksAction.class);
+        registerHandler.accept(new RestListTasksAction(settings, restController, nodesInCluster));
+        registerHandler.accept(new RestGetTaskAction(settings, restController));
+        registerHandler.accept(new RestCancelTasksAction(settings, restController, nodesInCluster));
 
         // Ingest API
-        registerRestHandler(handlers, RestPutPipelineAction.class);
-        registerRestHandler(handlers, RestGetPipelineAction.class);
-        registerRestHandler(handlers, RestDeletePipelineAction.class);
-        registerRestHandler(handlers, RestSimulatePipelineAction.class);
+        registerHandler.accept(new RestPutPipelineAction(settings, restController));
+        registerHandler.accept(new RestGetPipelineAction(settings, restController));
+        registerHandler.accept(new RestDeletePipelineAction(settings, restController));
+        registerHandler.accept(new RestSimulatePipelineAction(settings, restController));
 
         // CAT API
-        registerRestHandler(handlers, RestCatAction.class);
-        registerRestHandler(handlers, RestAllocationAction.class);
-        registerRestHandler(handlers, RestShardsAction.class);
-        registerRestHandler(handlers, RestMasterAction.class);
-        registerRestHandler(handlers, RestNodesAction.class);
-        registerRestHandler(handlers, RestTasksAction.class);
-        registerRestHandler(handlers, RestIndicesAction.class);
-        registerRestHandler(handlers, RestSegmentsAction.class);
+        registerHandler.accept(new RestAllocationAction(settings, restController));
+        registerHandler.accept(new RestShardsAction(settings, restController));
+        registerHandler.accept(new RestMasterAction(settings, restController));
+        registerHandler.accept(new RestNodesAction(settings, restController));
+        registerHandler.accept(new RestTasksAction(settings, restController, nodesInCluster));
+        registerHandler.accept(new RestIndicesAction(settings, restController, indexNameExpressionResolver));
+        registerHandler.accept(new RestSegmentsAction(settings, restController));
         // Fully qualified to prevent interference with rest.action.count.RestCountAction
-        registerRestHandler(handlers, org.elasticsearch.rest.action.cat.RestCountAction.class);
+        registerHandler.accept(new org.elasticsearch.rest.action.cat.RestCountAction(settings, restController));
         // Fully qualified to prevent interference with rest.action.indices.RestRecoveryAction
-        registerRestHandler(handlers, org.elasticsearch.rest.action.cat.RestRecoveryAction.class);
-        registerRestHandler(handlers, RestHealthAction.class);
-        registerRestHandler(handlers, org.elasticsearch.rest.action.cat.RestPendingClusterTasksAction.class);
-        registerRestHandler(handlers, RestAliasAction.class);
-        registerRestHandler(handlers, RestThreadPoolAction.class);
-        registerRestHandler(handlers, RestPluginsAction.class);
-        registerRestHandler(handlers, RestFielddataAction.class);
-        registerRestHandler(handlers, RestNodeAttrsAction.class);
-        registerRestHandler(handlers, RestRepositoriesAction.class);
-        registerRestHandler(handlers, RestSnapshotAction.class);
+        registerHandler.accept(new org.elasticsearch.rest.action.cat.RestRecoveryAction(settings, restController));
+        registerHandler.accept(new RestHealthAction(settings, restController));
+        registerHandler.accept(new org.elasticsearch.rest.action.cat.RestPendingClusterTasksAction(settings, restController));
+        registerHandler.accept(new RestAliasAction(settings, restController));
+        registerHandler.accept(new RestThreadPoolAction(settings, restController));
+        registerHandler.accept(new RestPluginsAction(settings, restController));
+        registerHandler.accept(new RestFielddataAction(settings, restController));
+        registerHandler.accept(new RestNodeAttrsAction(settings, restController));
+        registerHandler.accept(new RestRepositoriesAction(settings, restController));
+        registerHandler.accept(new RestSnapshotAction(settings, restController));
+        registerHandler.accept(new RestTemplatesAction(settings, restController));
         for (ActionPlugin plugin : actionPlugins) {
-            for (Class<? extends RestHandler> handler : plugin.getRestHandlers()) {
-                registerRestHandler(handlers, handler);
+            for (RestHandler handler : plugin.getRestHandlers(settings, restController, clusterSettings, indexScopedSettings,
+                    settingsFilter, indexNameExpressionResolver, nodesInCluster)) {
+                registerHandler.accept(handler);
             }
         }
-        return handlers;
-    }
-
-    private static void registerRestHandler(Set<Class<? extends RestHandler>> handlers, Class<? extends RestHandler> handler) {
-        if (handlers.contains(handler)) {
-            throw new IllegalArgumentException("can't register the same [rest_handler] more than once for [" + handler.getName() + "]");
-        }
-        handlers.add(handler);
+        registerHandler.accept(new RestCatAction(settings, restController, catActions));
     }
 
     @Override
@@ -644,23 +668,10 @@ protected void configure() {
                     bind(supportAction).asEagerSingleton();
                 }
             }
-
-            // Bind the RestController which is required (by Node) even if rest isn't enabled.
-            bind(RestController.class).toInstance(restController);
-
-            // Setup the RestHandlers
-            if (NetworkModule.HTTP_ENABLED.get(settings)) {
-                Multibinder<RestHandler> restHandlers = Multibinder.newSetBinder(binder(), RestHandler.class);
-                Multibinder<AbstractCatAction> catHandlers = Multibinder.newSetBinder(binder(), AbstractCatAction.class);
-                for (Class<? extends RestHandler> handler : setupRestHandlers(actionPlugins)) {
-                    bind(handler).asEagerSingleton();
-                    if (AbstractCatAction.class.isAssignableFrom(handler)) {
-                        catHandlers.addBinding().to(handler.asSubclass(AbstractCatAction.class));
-                    } else {
-                        restHandlers.addBinding().to(handler);
-                    }
-                }
-            }
         }
     }
+
+    public RestController getRestController() {
+        return restController;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/ActionRequest.java b/core/src/main/java/org/elasticsearch/action/ActionRequest.java
index 970afa413cc49..379bbc8481b17 100644
--- a/core/src/main/java/org/elasticsearch/action/ActionRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/ActionRequest.java
@@ -28,7 +28,7 @@
 /**
  *
  */
-public abstract class ActionRequest<Request extends ActionRequest<Request>> extends TransportRequest {
+public abstract class ActionRequest extends TransportRequest {
 
     public ActionRequest() {
         super();
diff --git a/core/src/main/java/org/elasticsearch/action/CompositeIndicesRequest.java b/core/src/main/java/org/elasticsearch/action/CompositeIndicesRequest.java
index 126874d0c3ce7..9c661e93be8fe 100644
--- a/core/src/main/java/org/elasticsearch/action/CompositeIndicesRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/CompositeIndicesRequest.java
@@ -19,16 +19,11 @@
 
 package org.elasticsearch.action;
 
-import java.util.List;
-
 /**
- * Needs to be implemented by all {@link org.elasticsearch.action.ActionRequest} subclasses that are composed of
- * multiple subrequests which relate to one or more indices. Allows to retrieve those subrequests.
+ * Marker interface that needs to be implemented by all {@link org.elasticsearch.action.ActionRequest} subclasses that are composed of
+ * multiple sub-requests which relate to one or more indices.  A composite request is executed by its own transport action class
+ * (e.g. {@link org.elasticsearch.action.search.TransportMultiSearchAction}), which goes through all sub-requests and delegates their
+ * execution to the appropriate transport action (e.g. {@link org.elasticsearch.action.search.TransportSearchAction}) for each single item.
  */
 public interface CompositeIndicesRequest {
-
-    /**
-     * Returns the subrequests that a composite request is composed of
-     */
-    List<? extends IndicesRequest> subRequests();
 }
diff --git a/core/src/main/java/org/elasticsearch/action/DocWriteRequest.java b/core/src/main/java/org/elasticsearch/action/DocWriteRequest.java
new file mode 100644
index 0000000000000..09db7089ff629
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/DocWriteRequest.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action;
+
+import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.action.update.UpdateRequest;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.index.VersionType;
+
+import java.io.IOException;
+import java.util.Locale;
+
+/**
+ * Generic interface to group ActionRequest, which perform writes to a single document
+ * Action requests implementing this can be part of {@link org.elasticsearch.action.bulk.BulkRequest}
+ */
+public interface DocWriteRequest<T> extends IndicesRequest {
+
+    /**
+     * Get the index that this request operates on
+     * @return the index
+     */
+    String index();
+
+    /**
+     * Get the type that this request operates on
+     * @return the type
+     */
+    String type();
+
+    /**
+     * Get the id of the document for this request
+     * @return the id
+     */
+    String id();
+
+    /**
+     * Get the options for this request
+     * @return the indices options
+     */
+    IndicesOptions indicesOptions();
+
+    /**
+     * Set the routing for this request
+     * @return the Request
+     */
+    T routing(String routing);
+
+    /**
+     * Get the routing for this request
+     * @return the Routing
+     */
+    String routing();
+
+
+    /**
+     * Get the parent for this request
+     * @return the Parent
+     */
+    String parent();
+
+    /**
+     * Get the document version for this request
+     * @return the document version
+     */
+    long version();
+
+    /**
+     * Sets the version, which will perform the operation only if a matching
+     * version exists and no changes happened on the doc since then.
+     */
+    T version(long version);
+
+    /**
+     * Get the document version type for this request
+     * @return the document version type
+     */
+    VersionType versionType();
+
+    /**
+     * Sets the versioning type. Defaults to {@link VersionType#INTERNAL}.
+     */
+    T versionType(VersionType versionType);
+
+    /**
+     * Get the requested document operation type of the request
+     * @return the operation type {@link OpType}
+     */
+    OpType opType();
+
+    /**
+     * Requested operation type to perform on the document
+     */
+    enum OpType {
+        /**
+         * Index the source. If there an existing document with the id, it will
+         * be replaced.
+         */
+        INDEX(0),
+        /**
+         * Creates the resource. Simply adds it to the index, if there is an existing
+         * document with the id, then it won't be removed.
+         */
+        CREATE(1),
+        /** Updates a document */
+        UPDATE(2),
+        /** Deletes a document */
+        DELETE(3);
+
+        private final byte op;
+        private final String lowercase;
+
+        OpType(int op) {
+            this.op = (byte) op;
+            this.lowercase = this.toString().toLowerCase(Locale.ROOT);
+        }
+
+        public byte getId() {
+            return op;
+        }
+
+        public String getLowercase() {
+            return lowercase;
+        }
+
+        public static OpType fromId(byte id) {
+            switch (id) {
+                case 0: return INDEX;
+                case 1: return CREATE;
+                case 2: return UPDATE;
+                case 3: return DELETE;
+                default: throw new IllegalArgumentException("Unknown opType: [" + id + "]");
+            }
+        }
+
+        public static OpType fromString(String sOpType) {
+            String lowerCase = sOpType.toLowerCase(Locale.ROOT);
+            for (OpType opType : OpType.values()) {
+                if (opType.getLowercase().equals(lowerCase)) {
+                    return opType;
+                }
+            }
+            throw new IllegalArgumentException("Unknown opType: [" + sOpType + "]");
+        }
+    }
+
+    /** read a document write (index/delete/update) request */
+    static DocWriteRequest readDocumentRequest(StreamInput in) throws IOException {
+        byte type = in.readByte();
+        DocWriteRequest docWriteRequest;
+        if (type == 0) {
+            IndexRequest indexRequest = new IndexRequest();
+            indexRequest.readFrom(in);
+            docWriteRequest = indexRequest;
+        } else if (type == 1) {
+            DeleteRequest deleteRequest = new DeleteRequest();
+            deleteRequest.readFrom(in);
+            docWriteRequest = deleteRequest;
+        } else if (type == 2) {
+            UpdateRequest updateRequest = new UpdateRequest();
+            updateRequest.readFrom(in);
+            docWriteRequest = updateRequest;
+        } else {
+            throw new IllegalStateException("invalid request type [" + type+ " ]");
+        }
+        return docWriteRequest;
+    }
+
+    /** write a document write (index/delete/update) request*/
+    static void writeDocumentRequest(StreamOutput out, DocWriteRequest request)  throws IOException {
+        if (request instanceof IndexRequest) {
+            out.writeByte((byte) 0);
+            ((IndexRequest) request).writeTo(out);
+        } else if (request instanceof DeleteRequest) {
+            out.writeByte((byte) 1);
+            ((DeleteRequest) request).writeTo(out);
+        } else if (request instanceof UpdateRequest) {
+            out.writeByte((byte) 2);
+            ((UpdateRequest) request).writeTo(out);
+        } else {
+            throw new IllegalStateException("invalid request [" + request.getClass().getSimpleName() + " ]");
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/DocWriteResponse.java b/core/src/main/java/org/elasticsearch/action/DocWriteResponse.java
index 0b64f3afa7d09..089ee8bbadd3c 100644
--- a/core/src/main/java/org/elasticsearch/action/DocWriteResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/DocWriteResponse.java
@@ -19,26 +19,41 @@
 package org.elasticsearch.action;
 
 import org.elasticsearch.action.support.WriteRequest;
-import org.elasticsearch.action.support.WriteResponse;
 import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
+import org.elasticsearch.action.support.WriteResponse;
 import org.elasticsearch.action.support.replication.ReplicationResponse;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.StatusToXContent;
+import org.elasticsearch.common.xcontent.StatusToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.net.URLEncoder;
 import java.util.Locale;
 
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+
 /**
  * A base class for the response of a write operation that involves a single doc
  */
-public abstract class DocWriteResponse extends ReplicationResponse implements WriteResponse, StatusToXContent {
+public abstract class DocWriteResponse extends ReplicationResponse implements WriteResponse, StatusToXContentObject {
+
+    private static final String _SHARDS = "_shards";
+    private static final String _INDEX = "_index";
+    private static final String _TYPE = "_type";
+    private static final String _ID = "_id";
+    private static final String _VERSION = "_version";
+    private static final String RESULT = "result";
+    private static final String FORCED_REFRESH = "forced_refresh";
 
     /**
      * An enum that represents the the results of CRUD operations, primarily used to communicate the type of
@@ -124,7 +139,6 @@ public String getIndex() {
         return this.shardId.getIndexName();
     }
 
-
     /**
      * The exact shard the document was changed in.
      */
@@ -168,32 +182,48 @@ public void setForcedRefresh(boolean forcedRefresh) {
     }
 
     /** returns the rest status for this response (based on {@link ShardInfo#status()} */
+    @Override
     public RestStatus status() {
         return getShardInfo().status();
     }
 
     /**
-     * Gets the location of the written document as a string suitable for a {@code Location} header.
-     * @param routing any routing used in the request. If null the location doesn't include routing information.
+     * Return the relative URI for the location of the document suitable for use in the {@code Location} header. The use of relative URIs is
+     * permitted as of HTTP/1.1 (cf. https://tools.ietf.org/html/rfc7231#section-7.1.2).
+     *
+     * @param routing custom routing or {@code null} if custom routing is not used
+     * @return the relative URI for the location of the document
      */
     public String getLocation(@Nullable String routing) {
-        // Absolute path for the location of the document. This should be allowed as of HTTP/1.1:
-        // https://tools.ietf.org/html/rfc7231#section-7.1.2
-        String index = getIndex();
-        String type = getType();
-        String id = getId();
-        String routingStart = "?routing=";
-        int bufferSize = 3 + index.length() + type.length() + id.length();
-        if (routing != null) {
-            bufferSize += routingStart.length() + routing.length();
-        }
-        StringBuilder location = new StringBuilder(bufferSize);
-        location.append('/').append(index);
-        location.append('/').append(type);
-        location.append('/').append(id);
-        if (routing != null) {
-            location.append(routingStart).append(routing);
+        final String encodedIndex;
+        final String encodedType;
+        final String encodedId;
+        final String encodedRouting;
+        try {
+            // encode the path components separately otherwise the path separators will be encoded
+            encodedIndex = URLEncoder.encode(getIndex(), "UTF-8");
+            encodedType = URLEncoder.encode(getType(), "UTF-8");
+            encodedId = URLEncoder.encode(getId(), "UTF-8");
+            encodedRouting = routing == null ? null : URLEncoder.encode(routing, "UTF-8");
+        } catch (final UnsupportedEncodingException e) {
+            throw new AssertionError(e);
         }
+        final String routingStart = "?routing=";
+        final int bufferSizeExcludingRouting = 3 + encodedIndex.length() + encodedType.length() + encodedId.length();
+        final int bufferSize;
+        if (encodedRouting == null) {
+            bufferSize = bufferSizeExcludingRouting;
+        } else {
+            bufferSize = bufferSizeExcludingRouting + routingStart.length() + encodedRouting.length();
+        }
+        final StringBuilder location = new StringBuilder(bufferSize);
+        location.append('/').append(encodedIndex);
+        location.append('/').append(encodedType);
+        location.append('/').append(encodedId);
+        if (encodedRouting != null) {
+            location.append(routingStart).append(encodedRouting);
+        }
+
         return location.toString();
     }
 
@@ -220,17 +250,128 @@ public void writeTo(StreamOutput out) throws IOException {
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+    public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        innerToXContent(builder, params);
+        builder.endObject();
+        return builder;
+    }
+
+    public XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
         ReplicationResponse.ShardInfo shardInfo = getShardInfo();
-        builder.field("_index", shardId.getIndexName())
-            .field("_type", type)
-            .field("_id", id)
-            .field("_version", version)
-            .field("result", getResult().getLowercase());
+        builder.field(_INDEX, shardId.getIndexName())
+                .field(_TYPE, type)
+                .field(_ID, id)
+                .field(_VERSION, version)
+                .field(RESULT, getResult().getLowercase());
         if (forcedRefresh) {
-            builder.field("forced_refresh", forcedRefresh);
+            builder.field(FORCED_REFRESH, true);
         }
-        shardInfo.toXContent(builder, params);
+        builder.field(_SHARDS, shardInfo);
         return builder;
     }
+
+    /**
+     * Parse the output of the {@link #innerToXContent(XContentBuilder, Params)} method.
+     *
+     * This method is intended to be called by subclasses and must be called multiple times to parse all the information concerning
+     * {@link DocWriteResponse} objects. It always parses the current token, updates the given parsing context accordingly
+     * if needed and then immediately returns.
+     */
+    protected static void parseInnerToXContent(XContentParser parser, Builder context) throws IOException {
+        XContentParser.Token token = parser.currentToken();
+        ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser::getTokenLocation);
+
+        String currentFieldName = parser.currentName();
+        token = parser.nextToken();
+
+        if (token.isValue()) {
+            if (_INDEX.equals(currentFieldName)) {
+                // index uuid and shard id are unknown and can't be parsed back for now.
+                context.setShardId(new ShardId(new Index(parser.text(), IndexMetaData.INDEX_UUID_NA_VALUE), -1));
+            } else if (_TYPE.equals(currentFieldName)) {
+                context.setType(parser.text());
+            } else if (_ID.equals(currentFieldName)) {
+                context.setId(parser.text());
+            } else if (_VERSION.equals(currentFieldName)) {
+                context.setVersion(parser.longValue());
+            } else if (RESULT.equals(currentFieldName)) {
+                String result = parser.text();
+                for (Result r :  Result.values()) {
+                    if (r.getLowercase().equals(result)) {
+                        context.setResult(r);
+                        break;
+                    }
+                }
+            } else if (FORCED_REFRESH.equals(currentFieldName)) {
+                context.setForcedRefresh(parser.booleanValue());
+            }
+        } else if (token == XContentParser.Token.START_OBJECT) {
+            if (_SHARDS.equals(currentFieldName)) {
+                context.setShardInfo(ShardInfo.fromXContent(parser));
+            } else {
+                parser.skipChildren(); // skip potential inner objects for forward compatibility
+            }
+        } else if (token == XContentParser.Token.START_ARRAY) {
+            parser.skipChildren(); // skip potential inner arrays for forward compatibility
+        }
+    }
+
+    /**
+     * Base class of all {@link DocWriteResponse} builders. These {@link DocWriteResponse.Builder} are used during
+     * xcontent parsing to temporarily store the parsed values, then the {@link Builder#build()} method is called to
+     * instantiate the appropriate {@link DocWriteResponse} with the parsed values.
+     */
+    public abstract static class Builder {
+
+        protected ShardId shardId = null;
+        protected String type = null;
+        protected String id = null;
+        protected Long version = null;
+        protected Result result = null;
+        protected boolean forcedRefresh;
+        protected ShardInfo shardInfo = null;
+
+        public ShardId getShardId() {
+            return shardId;
+        }
+
+        public void setShardId(ShardId shardId) {
+            this.shardId = shardId;
+        }
+
+        public String getType() {
+            return type;
+        }
+
+        public void setType(String type) {
+            this.type = type;
+        }
+
+        public String getId() {
+            return id;
+        }
+
+        public void setId(String id) {
+            this.id = id;
+        }
+
+        public void setVersion(Long version) {
+            this.version = version;
+        }
+
+        public void setResult(Result result) {
+            this.result = result;
+        }
+
+        public void setForcedRefresh(boolean forcedRefresh) {
+            this.forcedRefresh = forcedRefresh;
+        }
+
+        public void setShardInfo(ShardInfo shardInfo) {
+            this.shardInfo = shardInfo;
+        }
+
+        public abstract DocWriteResponse build();
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/DocumentRequest.java b/core/src/main/java/org/elasticsearch/action/DocumentRequest.java
deleted file mode 100644
index a90f013a6b9ab..0000000000000
--- a/core/src/main/java/org/elasticsearch/action/DocumentRequest.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.action;
-
-import org.elasticsearch.action.support.IndicesOptions;
-
-/**
- * Generic interface to group ActionRequest, which work on single document level
- *
- * Forces this class return index/type/id getters
- */
-public interface DocumentRequest<T> extends IndicesRequest {
-
-    /**
-     * Get the index that this request operates on
-     * @return the index
-     */
-    String index();
-
-    /**
-     * Get the type that this request operates on
-     * @return the type
-     */
-    String type();
-
-    /**
-     * Get the id of the document for this request
-     * @return the id
-     */
-    String id();
-
-    /**
-     * Get the options for this request
-     * @return the indices options
-     */
-    IndicesOptions indicesOptions();
-
-    /**
-     * Set the routing for this request
-     * @return the Request
-     */
-    T routing(String routing);
-
-    /**
-     * Get the routing for this request
-     * @return the Routing
-     */
-    String routing();
-
-
-    /**
-     * Get the parent for this request
-     * @return the Parent
-     */
-    String parent();
-
-}
diff --git a/core/src/main/java/org/elasticsearch/action/ListenableActionFuture.java b/core/src/main/java/org/elasticsearch/action/ListenableActionFuture.java
index 29b5a2a877495..87e4df3bc7975 100644
--- a/core/src/main/java/org/elasticsearch/action/ListenableActionFuture.java
+++ b/core/src/main/java/org/elasticsearch/action/ListenableActionFuture.java
@@ -29,5 +29,5 @@ public interface ListenableActionFuture<T> extends ActionFuture<T> {
     /**
      * Add an action listener to be invoked when a response has received.
      */
-    void addListener(final ActionListener<T> listener);
+    void addListener(ActionListener<T> listener);
 }
diff --git a/core/src/main/java/org/elasticsearch/action/NotifyOnceListener.java b/core/src/main/java/org/elasticsearch/action/NotifyOnceListener.java
new file mode 100644
index 0000000000000..1b717dcc6c05a
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/NotifyOnceListener.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * A listener that ensures that only one of onResponse or onFailure is called. And the method
+ * the is called is only called once. Subclasses should implement notification logic with
+ * innerOnResponse and innerOnFailure.
+ */
+public abstract class NotifyOnceListener<Response> implements ActionListener<Response> {
+
+    private final AtomicBoolean hasBeenCalled = new AtomicBoolean(false);
+
+    protected abstract void innerOnResponse(Response response);
+
+    protected abstract void innerOnFailure(Exception e);
+
+    @Override
+    public final void onResponse(Response response) {
+        if (hasBeenCalled.compareAndSet(false, true)) {
+            innerOnResponse(response);
+        }
+    }
+
+    @Override
+    public final void onFailure(Exception e) {
+        if (hasBeenCalled.compareAndSet(false, true)) {
+            innerOnFailure(e);
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/OriginalIndices.java b/core/src/main/java/org/elasticsearch/action/OriginalIndices.java
index cc299f544b38f..0642326d2b48e 100644
--- a/core/src/main/java/org/elasticsearch/action/OriginalIndices.java
+++ b/core/src/main/java/org/elasticsearch/action/OriginalIndices.java
@@ -24,11 +24,15 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 
 import java.io.IOException;
+import java.util.Arrays;
 
 /**
  * Used to keep track of original indices within internal (e.g. shard level) requests
  */
-public class OriginalIndices implements IndicesRequest {
+public final class OriginalIndices implements IndicesRequest {
+
+    //constant to use when original indices are not applicable and will not be serialized across the wire
+    public static final OriginalIndices NONE = new OriginalIndices(null, null);
 
     private final String[] indices;
     private final IndicesOptions indicesOptions;
@@ -39,7 +43,6 @@ public OriginalIndices(IndicesRequest indicesRequest) {
 
     public OriginalIndices(String[] indices, IndicesOptions indicesOptions) {
         this.indices = indices;
-        assert indicesOptions != null;
         this.indicesOptions = indicesOptions;
     }
 
@@ -57,9 +60,17 @@ public static OriginalIndices readOriginalIndices(StreamInput in) throws IOExcep
         return new OriginalIndices(in.readStringArray(), IndicesOptions.readIndicesOptions(in));
     }
 
-
     public static void writeOriginalIndices(OriginalIndices originalIndices, StreamOutput out) throws IOException {
+        assert originalIndices != NONE;
         out.writeStringArrayNullable(originalIndices.indices);
         originalIndices.indicesOptions.writeIndicesOptions(out);
     }
+
+    @Override
+    public String toString() {
+        return "OriginalIndices{" +
+            "indices=" + Arrays.toString(indices) +
+            ", indicesOptions=" + indicesOptions +
+            '}';
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/TaskOperationFailure.java b/core/src/main/java/org/elasticsearch/action/TaskOperationFailure.java
index 6704f610ec0aa..8c8f263c34db0 100644
--- a/core/src/main/java/org/elasticsearch/action/TaskOperationFailure.java
+++ b/core/src/main/java/org/elasticsearch/action/TaskOperationFailure.java
@@ -88,7 +88,7 @@ public RestStatus getStatus() {
         return status;
     }
 
-    public Throwable getCause() {
+    public Exception getCause() {
         return reason;
     }
 
@@ -105,7 +105,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (reason != null) {
             builder.field("reason");
             builder.startObject();
-            ElasticsearchException.toXContent(builder, params, reason);
+            ElasticsearchException.generateThrowableXContent(builder, params, reason);
             builder.endObject();
         }
         return builder;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainRequest.java
index f31b1d3737684..aea1ee57dca87 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainRequest.java
@@ -19,13 +19,11 @@
 
 package org.elasticsearch.action.admin.cluster.allocation;
 
-import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.master.MasterNodeRequest;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ObjectParser;
@@ -40,36 +38,47 @@
  */
 public class ClusterAllocationExplainRequest extends MasterNodeRequest<ClusterAllocationExplainRequest> {
 
-    private static ObjectParser<ClusterAllocationExplainRequest, ParseFieldMatcherSupplier> PARSER = new ObjectParser(
-            "cluster/allocation/explain");
+    private static ObjectParser<ClusterAllocationExplainRequest, Void> PARSER = new ObjectParser<>("cluster/allocation/explain");
     static {
         PARSER.declareString(ClusterAllocationExplainRequest::setIndex, new ParseField("index"));
         PARSER.declareInt(ClusterAllocationExplainRequest::setShard, new ParseField("shard"));
         PARSER.declareBoolean(ClusterAllocationExplainRequest::setPrimary, new ParseField("primary"));
+        PARSER.declareString(ClusterAllocationExplainRequest::setCurrentNode, new ParseField("current_node"));
     }
 
+    @Nullable
     private String index;
+    @Nullable
     private Integer shard;
+    @Nullable
     private Boolean primary;
+    @Nullable
+    private String currentNode;
     private boolean includeYesDecisions = false;
     private boolean includeDiskInfo = false;
 
-    /** Explain the first unassigned shard */
+    /**
+     * Create a new allocation explain request to explain any unassigned shard in the cluster.
+     */
     public ClusterAllocationExplainRequest() {
         this.index = null;
         this.shard = null;
         this.primary = null;
+        this.currentNode = null;
     }
 
     /**
      * Create a new allocation explain request. If {@code primary} is false, the first unassigned replica
      * will be picked for explanation. If no replicas are unassigned, the first assigned replica will
      * be explained.
+     *
+     * Package private for testing.
      */
-    public ClusterAllocationExplainRequest(String index, int shard, boolean primary) {
+    ClusterAllocationExplainRequest(String index, int shard, boolean primary, @Nullable String currentNode) {
         this.index = index;
         this.shard = shard;
         this.primary = primary;
+        this.currentNode = currentNode;
     }
 
     @Override
@@ -93,54 +102,103 @@ public ActionRequestValidationException validate() {
      * Returns {@code true} iff the first unassigned shard is to be used
      */
     public boolean useAnyUnassignedShard() {
-        return this.index == null && this.shard == null && this.primary == null;
+        return this.index == null && this.shard == null && this.primary == null && this.currentNode == null;
     }
 
+    /**
+     * Sets the index name of the shard to explain.
+     */
     public ClusterAllocationExplainRequest setIndex(String index) {
         this.index = index;
         return this;
     }
 
+    /**
+     * Returns the index name of the shard to explain, or {@code null} to use any unassigned shard (see {@link #useAnyUnassignedShard()}).
+     */
     @Nullable
     public String getIndex() {
         return this.index;
     }
 
+    /**
+     * Sets the shard id of the shard to explain.
+     */
     public ClusterAllocationExplainRequest setShard(Integer shard) {
         this.shard = shard;
         return this;
     }
 
+    /**
+     * Returns the shard id of the shard to explain, or {@code null} to use any unassigned shard (see {@link #useAnyUnassignedShard()}).
+     */
     @Nullable
     public Integer getShard() {
         return this.shard;
     }
 
+    /**
+     * Sets whether to explain the allocation of the primary shard or a replica shard copy
+     * for the shard id (see {@link #getShard()}).
+     */
     public ClusterAllocationExplainRequest setPrimary(Boolean primary) {
         this.primary = primary;
         return this;
     }
 
+    /**
+     * Returns {@code true} if explaining the primary shard for the shard id (see {@link #getShard()}),
+     * {@code false} if explaining a replica shard copy for the shard id, or {@code null} to use any
+     * unassigned shard (see {@link #useAnyUnassignedShard()}).
+     */
     @Nullable
     public Boolean isPrimary() {
         return this.primary;
     }
 
+    /**
+     * Requests the explain API to explain an already assigned replica shard currently allocated to
+     * the given node.
+     */
+    public ClusterAllocationExplainRequest setCurrentNode(String currentNodeId) {
+        this.currentNode = currentNodeId;
+        return this;
+    }
+
+    /**
+     * Returns the node holding the replica shard to be explained.  Returns {@code null} if any replica shard
+     * can be explained.
+     */
+    @Nullable
+    public String getCurrentNode() {
+        return currentNode;
+    }
+
+    /**
+     * Set to {@code true} to include yes decisions for a particular node.
+     */
     public void includeYesDecisions(boolean includeYesDecisions) {
         this.includeYesDecisions = includeYesDecisions;
     }
 
-    /** Returns true if all decisions should be included. Otherwise only "NO" and "THROTTLE" decisions are returned */
+    /**
+     * Returns {@code true} if yes decisions should be included.  Otherwise only "no" and "throttle"
+     * decisions are returned.
+     */
     public boolean includeYesDecisions() {
         return this.includeYesDecisions;
     }
 
-    /** {@code true} to include information about the gathered disk information of nodes in the cluster */
+    /**
+     * Set to {@code true} to include information about the gathered disk information of nodes in the cluster.
+     */
     public void includeDiskInfo(boolean includeDiskInfo) {
         this.includeDiskInfo = includeDiskInfo;
     }
 
-    /** Returns true if information about disk usage and shard sizes should also be returned */
+    /**
+     * Returns {@code true} if information about disk usage and shard sizes should also be returned.
+     */
     public boolean includeDiskInfo() {
         return this.includeDiskInfo;
     }
@@ -154,37 +212,46 @@ public String toString() {
             sb.append("index=").append(index);
             sb.append(",shard=").append(shard);
             sb.append(",primary?=").append(primary);
+            if (currentNode != null) {
+                sb.append(",currentNode=").append(currentNode);
+            }
         }
         sb.append(",includeYesDecisions?=").append(includeYesDecisions);
         return sb.toString();
     }
 
     public static ClusterAllocationExplainRequest parse(XContentParser parser) throws IOException {
-        ClusterAllocationExplainRequest req = PARSER.parse(parser, new ClusterAllocationExplainRequest(), () -> ParseFieldMatcher.STRICT);
-        Exception e = req.validate();
-        if (e != null) {
-            throw new ElasticsearchParseException("'index', 'shard', and 'primary' must be specified in allocation explain request", e);
-        }
-        return req;
+        return PARSER.parse(parser, new ClusterAllocationExplainRequest(), null);
     }
 
     @Override
     public void readFrom(StreamInput in) throws IOException {
+        checkVersion(in.getVersion());
         super.readFrom(in);
         this.index = in.readOptionalString();
         this.shard = in.readOptionalVInt();
         this.primary = in.readOptionalBoolean();
+        this.currentNode = in.readOptionalString();
         this.includeYesDecisions = in.readBoolean();
         this.includeDiskInfo = in.readBoolean();
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
+        checkVersion(out.getVersion());
         super.writeTo(out);
         out.writeOptionalString(index);
         out.writeOptionalVInt(shard);
         out.writeOptionalBoolean(primary);
+        out.writeOptionalString(currentNode);
         out.writeBoolean(includeYesDecisions);
         out.writeBoolean(includeDiskInfo);
     }
+
+    private void checkVersion(Version version) {
+        if (version.before(Version.V_5_2_0)) {
+            throw new IllegalArgumentException("cannot explain shards in a mixed-cluster with pre-" + Version.V_5_2_0 +
+                                               " nodes, node version [" + version + "]");
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainRequestBuilder.java
index 343d1756d7524..038657a277cd5 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainRequestBuilder.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.action.admin.cluster.allocation;
 
-import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 
@@ -65,6 +64,15 @@ public ClusterAllocationExplainRequestBuilder setIncludeDiskInfo(boolean include
         return this;
     }
 
+    /**
+     * Requests the explain API to explain an already assigned replica shard currently allocated to
+     * the given node.
+     */
+    public ClusterAllocationExplainRequestBuilder setCurrentNode(String currentNode) {
+        request.setCurrentNode(currentNode);
+        return this;
+    }
+
     /**
      * Signal that the first unassigned shard should be used
      */
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainResponse.java
index cc586bd1a5813..83a847ba01e54 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainResponse.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.action.admin.cluster.allocation;
 
 import org.elasticsearch.action.ActionResponse;
-import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java
index fd06c685b95f8..e2911c227a868 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java
@@ -21,285 +21,184 @@
 
 import org.elasticsearch.cluster.ClusterInfo;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
+import org.elasticsearch.cluster.routing.allocation.AllocationDecision;
+import org.elasticsearch.cluster.routing.allocation.ShardAllocationDecision;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.shard.ShardId;
 
 import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
+import java.util.Locale;
+
+import static org.elasticsearch.cluster.routing.allocation.AbstractAllocationDecision.discoveryNodeToXContent;
 
 /**
- * A {@code ClusterAllocationExplanation} is an explanation of why a shard may or may not be allocated to nodes. It also includes weights
- * for where the shard is likely to be assigned. It is an immutable class
+ * A {@code ClusterAllocationExplanation} is an explanation of why a shard is unassigned,
+ * or if it is not unassigned, then which nodes it could possibly be relocated to.
+ * It is an immutable class.
  */
 public final class ClusterAllocationExplanation implements ToXContent, Writeable {
 
-    private final ShardId shard;
-    private final boolean primary;
-    private final boolean hasPendingAsyncFetch;
-    private final String assignedNodeId;
-    private final UnassignedInfo unassignedInfo;
-    private final long allocationDelayMillis;
-    private final long remainingDelayMillis;
-    private final Map<DiscoveryNode, NodeExplanation> nodeExplanations;
+    private final ShardRouting shardRouting;
+    private final DiscoveryNode currentNode;
+    private final DiscoveryNode relocationTargetNode;
     private final ClusterInfo clusterInfo;
-
-    public ClusterAllocationExplanation(ShardId shard, boolean primary, @Nullable String assignedNodeId, long allocationDelayMillis,
-                                        long remainingDelayMillis, @Nullable UnassignedInfo unassignedInfo, boolean hasPendingAsyncFetch,
-                                        Map<DiscoveryNode, NodeExplanation> nodeExplanations, @Nullable ClusterInfo clusterInfo) {
-        this.shard = shard;
-        this.primary = primary;
-        this.hasPendingAsyncFetch = hasPendingAsyncFetch;
-        this.assignedNodeId = assignedNodeId;
-        this.unassignedInfo = unassignedInfo;
-        this.allocationDelayMillis = allocationDelayMillis;
-        this.remainingDelayMillis = remainingDelayMillis;
-        this.nodeExplanations = nodeExplanations;
+    private final ShardAllocationDecision shardAllocationDecision;
+
+    public ClusterAllocationExplanation(ShardRouting shardRouting, @Nullable DiscoveryNode currentNode,
+                                        @Nullable DiscoveryNode relocationTargetNode, @Nullable ClusterInfo clusterInfo,
+                                        ShardAllocationDecision shardAllocationDecision) {
+        this.shardRouting = shardRouting;
+        this.currentNode = currentNode;
+        this.relocationTargetNode = relocationTargetNode;
         this.clusterInfo = clusterInfo;
+        this.shardAllocationDecision = shardAllocationDecision;
     }
 
     public ClusterAllocationExplanation(StreamInput in) throws IOException {
-        this.shard = ShardId.readShardId(in);
-        this.primary = in.readBoolean();
-        this.hasPendingAsyncFetch = in.readBoolean();
-        this.assignedNodeId = in.readOptionalString();
-        this.unassignedInfo = in.readOptionalWriteable(UnassignedInfo::new);
-        this.allocationDelayMillis = in.readVLong();
-        this.remainingDelayMillis = in.readVLong();
-
-        int mapSize = in.readVInt();
-        Map<DiscoveryNode, NodeExplanation> nodeToExplanation = new HashMap<>(mapSize);
-        for (int i = 0; i < mapSize; i++) {
-            NodeExplanation nodeExplanation = new NodeExplanation(in);
-            nodeToExplanation.put(nodeExplanation.getNode(), nodeExplanation);
-        }
-        this.nodeExplanations = nodeToExplanation;
-        if (in.readBoolean()) {
-            this.clusterInfo = new ClusterInfo(in);
-        } else {
-            this.clusterInfo = null;
-        }
+        this.shardRouting = new ShardRouting(in);
+        this.currentNode = in.readOptionalWriteable(DiscoveryNode::new);
+        this.relocationTargetNode = in.readOptionalWriteable(DiscoveryNode::new);
+        this.clusterInfo = in.readOptionalWriteable(ClusterInfo::new);
+        this.shardAllocationDecision = new ShardAllocationDecision(in);
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        this.getShard().writeTo(out);
-        out.writeBoolean(this.isPrimary());
-        out.writeBoolean(this.isStillFetchingShardData());
-        out.writeOptionalString(this.getAssignedNodeId());
-        out.writeOptionalWriteable(this.getUnassignedInfo());
-        out.writeVLong(allocationDelayMillis);
-        out.writeVLong(remainingDelayMillis);
-
-        out.writeVInt(this.nodeExplanations.size());
-        for (NodeExplanation explanation : this.nodeExplanations.values()) {
-            explanation.writeTo(out);
-        }
-        if (this.clusterInfo != null) {
-            out.writeBoolean(true);
-            this.clusterInfo.writeTo(out);
-        } else {
-            out.writeBoolean(false);
-        }
+        shardRouting.writeTo(out);
+        out.writeOptionalWriteable(currentNode);
+        out.writeOptionalWriteable(relocationTargetNode);
+        out.writeOptionalWriteable(clusterInfo);
+        shardAllocationDecision.writeTo(out);
     }
 
-    /** Return the shard that the explanation is about */
+    /**
+     * Returns the shard that the explanation is about.
+     */
     public ShardId getShard() {
-        return this.shard;
+        return shardRouting.shardId();
     }
 
-    /** Return true if the explained shard is primary, false otherwise */
+    /**
+     * Returns {@code true} if the explained shard is primary, {@code false} otherwise.
+     */
     public boolean isPrimary() {
-        return this.primary;
+        return shardRouting.primary();
     }
 
-    /** Return turn if shard data is still being fetched for the allocation */
-    public boolean isStillFetchingShardData() {
-        return this.hasPendingAsyncFetch;
+    /**
+     * Returns the current {@link ShardRoutingState} of the shard.
+     */
+    public ShardRoutingState getShardState() {
+        return shardRouting.state();
     }
 
-    /** Return turn if the shard is assigned to a node */
-    public boolean isAssigned() {
-        return this.assignedNodeId != null;
+    /**
+     * Returns the currently assigned node, or {@code null} if the shard is unassigned.
+     */
+    @Nullable
+    public DiscoveryNode getCurrentNode() {
+        return currentNode;
     }
 
-    /** Return the assigned node id or null if not assigned */
+    /**
+     * Returns the relocating target node, or {@code null} if the shard is not in the {@link ShardRoutingState#RELOCATING} state.
+     */
     @Nullable
-    public String getAssignedNodeId() {
-        return this.assignedNodeId;
+    public DiscoveryNode getRelocationTargetNode() {
+        return relocationTargetNode;
     }
 
-    /** Return the unassigned info for the shard or null if the shard is assigned */
+    /**
+     * Returns the unassigned info for the shard, or {@code null} if the shard is active.
+     */
     @Nullable
     public UnassignedInfo getUnassignedInfo() {
-        return this.unassignedInfo;
-    }
-
-    /** Return the configured delay before the shard can be allocated in milliseconds */
-    public long getAllocationDelayMillis() {
-        return this.allocationDelayMillis;
-    }
-
-    /** Return the remaining allocation delay for this shard in milliseconds */
-    public long getRemainingDelayMillis() {
-        return this.remainingDelayMillis;
-    }
-
-    /** Return a map of node to the explanation for that node */
-    public Map<DiscoveryNode, NodeExplanation> getNodeExplanations() {
-        return this.nodeExplanations;
+        return shardRouting.unassignedInfo();
     }
 
-    /** Return the cluster disk info for the cluster or null if none available */
+    /**
+     * Returns the cluster disk info for the cluster, or {@code null} if none available.
+     */
     @Nullable
     public ClusterInfo getClusterInfo() {
         return this.clusterInfo;
     }
 
+    /** \
+     * Returns the shard allocation decision for attempting to assign or move the shard.
+     */
+    public ShardAllocationDecision getShardAllocationDecision() {
+        return shardAllocationDecision;
+    }
+
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject(); {
-            builder.startObject("shard"); {
-                builder.field("index", shard.getIndexName());
-                builder.field("index_uuid", shard.getIndex().getUUID());
-                builder.field("id", shard.getId());
-                builder.field("primary", primary);
-            }
-            builder.endObject(); // end shard
-            builder.field("assigned", this.assignedNodeId != null);
-            // If assigned, show the node id of the node it's assigned to
-            if (assignedNodeId != null) {
-                builder.field("assigned_node_id", this.assignedNodeId);
+            builder.field("index", shardRouting.getIndexName());
+            builder.field("shard", shardRouting.getId());
+            builder.field("primary", shardRouting.primary());
+            builder.field("current_state", shardRouting.state().toString().toLowerCase(Locale.ROOT));
+            if (shardRouting.unassignedInfo() != null) {
+                unassignedInfoToXContent(shardRouting.unassignedInfo(), builder);
             }
-            builder.field("shard_state_fetch_pending", this.hasPendingAsyncFetch);
-            // If we have unassigned info, show that
-            if (unassignedInfo != null) {
-                unassignedInfo.toXContent(builder, params);
-                builder.timeValueField("allocation_delay_in_millis", "allocation_delay", TimeValue.timeValueMillis(allocationDelayMillis));
-                builder.timeValueField("remaining_delay_in_millis", "remaining_delay", TimeValue.timeValueMillis(remainingDelayMillis));
-            }
-            builder.startObject("nodes"); {
-                for (NodeExplanation explanation : nodeExplanations.values()) {
-                    explanation.toXContent(builder, params);
+            if (currentNode != null) {
+                builder.startObject("current_node");
+                {
+                    discoveryNodeToXContent(currentNode, true, builder);
+                    if (shardAllocationDecision.getMoveDecision().isDecisionTaken()
+                            && shardAllocationDecision.getMoveDecision().getCurrentNodeRanking() > 0) {
+                        builder.field("weight_ranking", shardAllocationDecision.getMoveDecision().getCurrentNodeRanking());
+                    }
                 }
+                builder.endObject();
             }
-            builder.endObject(); // end nodes
             if (this.clusterInfo != null) {
                 builder.startObject("cluster_info"); {
                     this.clusterInfo.toXContent(builder, params);
                 }
                 builder.endObject(); // end "cluster_info"
             }
+            if (shardAllocationDecision.isDecisionTaken()) {
+                shardAllocationDecision.toXContent(builder, params);
+            } else {
+                String explanation;
+                if (shardRouting.state() == ShardRoutingState.RELOCATING) {
+                    explanation = "the shard is in the process of relocating from node [" + currentNode.getName() + "] " +
+                                  "to node [" + relocationTargetNode.getName() + "], wait until relocation has completed";
+                } else {
+                    assert shardRouting.state() == ShardRoutingState.INITIALIZING;
+                    explanation = "the shard is in the process of initializing on node [" + currentNode.getName() + "], " +
+                                  "wait until initialization has completed";
+                }
+                builder.field("explanation", explanation);
+            }
         }
         builder.endObject(); // end wrapping object
         return builder;
     }
 
-    /** An Enum representing the final decision for a shard allocation on a node */
-    public enum FinalDecision {
-        // Yes, the shard can be assigned
-        YES((byte) 0),
-        // No, the shard cannot be assigned
-        NO((byte) 1),
-        // The shard is already assigned to this node
-        ALREADY_ASSIGNED((byte) 2);
+    private XContentBuilder unassignedInfoToXContent(UnassignedInfo unassignedInfo, XContentBuilder builder)
+        throws IOException {
 
-        private final byte id;
-
-        FinalDecision (byte id) {
-            this.id = id;
+        builder.startObject("unassigned_info");
+        builder.field("reason", unassignedInfo.getReason());
+        builder.field("at", UnassignedInfo.DATE_TIME_FORMATTER.printer().print(unassignedInfo.getUnassignedTimeInMillis()));
+        if (unassignedInfo.getNumFailedAllocations() >  0) {
+            builder.field("failed_allocation_attempts", unassignedInfo.getNumFailedAllocations());
         }
-
-        private static FinalDecision fromId(byte id) {
-            switch (id) {
-                case 0: return YES;
-                case 1: return NO;
-                case 2: return ALREADY_ASSIGNED;
-                default:
-                    throw new IllegalArgumentException("unknown id for final decision: [" + id + "]");
-            }
-        }
-
-        @Override
-        public String toString() {
-            switch (id) {
-                case 0: return "YES";
-                case 1: return "NO";
-                case 2: return "ALREADY_ASSIGNED";
-                default:
-                    throw new IllegalArgumentException("unknown id for final decision: [" + id + "]");
-            }
-        }
-
-        static FinalDecision readFrom(StreamInput in) throws IOException {
-            return fromId(in.readByte());
-        }
-
-        void writeTo(StreamOutput out) throws IOException {
-            out.writeByte(id);
-        }
-    }
-
-    /** An Enum representing the state of the shard store's copy of the data on a node */
-    public enum StoreCopy {
-        // No data for this shard is on the node
-        NONE((byte) 0),
-        // A copy of the data is available on this node
-        AVAILABLE((byte) 1),
-        // The copy of the data on the node is corrupt
-        CORRUPT((byte) 2),
-        // There was an error reading this node's copy of the data
-        IO_ERROR((byte) 3),
-        // The copy of the data on the node is stale
-        STALE((byte) 4),
-        // It's unknown what the copy of the data is
-        UNKNOWN((byte) 5);
-
-        private final byte id;
-
-        StoreCopy (byte id) {
-            this.id = id;
-        }
-
-        private static StoreCopy fromId(byte id) {
-            switch (id) {
-                case 0: return NONE;
-                case 1: return AVAILABLE;
-                case 2: return CORRUPT;
-                case 3: return IO_ERROR;
-                case 4: return STALE;
-                case 5: return UNKNOWN;
-                default:
-                    throw new IllegalArgumentException("unknown id for store copy: [" + id + "]");
-            }
-        }
-
-        @Override
-        public String toString() {
-            switch (id) {
-                case 0: return "NONE";
-                case 1: return "AVAILABLE";
-                case 2: return "CORRUPT";
-                case 3: return "IO_ERROR";
-                case 4: return "STALE";
-                case 5: return "UNKNOWN";
-                default:
-                    throw new IllegalArgumentException("unknown id for store copy: [" + id + "]");
-            }
-        }
-
-        static StoreCopy readFrom(StreamInput in) throws IOException {
-            return fromId(in.readByte());
-        }
-
-        void writeTo(StreamOutput out) throws IOException {
-            out.writeByte(id);
+        String details = unassignedInfo.getDetails();
+        if (details != null) {
+            builder.field("details", details);
         }
+        builder.field("last_allocation_status", AllocationDecision.fromAllocationStatus(unassignedInfo.getLastAllocationStatus()));
+        builder.endObject();
+        return builder;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/NodeExplanation.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/NodeExplanation.java
deleted file mode 100644
index e564711d4188d..0000000000000
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/NodeExplanation.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.action.admin.cluster.allocation;
-
-import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresResponse;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.routing.allocation.decider.Decision;
-import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-
-import java.io.IOException;
-import java.util.Map;
-/** The cluster allocation explanation for a single node */
-public class NodeExplanation implements Writeable, ToXContent {
-    private final DiscoveryNode node;
-    private final Decision nodeDecision;
-    private final Float nodeWeight;
-    private final IndicesShardStoresResponse.StoreStatus storeStatus;
-    private final ClusterAllocationExplanation.FinalDecision finalDecision;
-    private final ClusterAllocationExplanation.StoreCopy storeCopy;
-    private final String finalExplanation;
-
-    public NodeExplanation(final DiscoveryNode node, final Decision nodeDecision, final Float nodeWeight,
-                           @Nullable final IndicesShardStoresResponse.StoreStatus storeStatus,
-                           final ClusterAllocationExplanation.FinalDecision finalDecision,
-                           final String finalExplanation,
-                           final ClusterAllocationExplanation.StoreCopy storeCopy) {
-        this.node = node;
-        this.nodeDecision = nodeDecision;
-        this.nodeWeight = nodeWeight;
-        this.storeStatus = storeStatus;
-        this.finalDecision = finalDecision;
-        this.finalExplanation = finalExplanation;
-        this.storeCopy = storeCopy;
-    }
-
-    public NodeExplanation(StreamInput in) throws IOException {
-        this.node = new DiscoveryNode(in);
-        this.nodeDecision = Decision.readFrom(in);
-        this.nodeWeight = in.readFloat();
-        if (in.readBoolean()) {
-            this.storeStatus = IndicesShardStoresResponse.StoreStatus.readStoreStatus(in);
-        } else {
-            this.storeStatus = null;
-        }
-        this.finalDecision = ClusterAllocationExplanation.FinalDecision.readFrom(in);
-        this.finalExplanation = in.readString();
-        this.storeCopy = ClusterAllocationExplanation.StoreCopy.readFrom(in);
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        node.writeTo(out);
-        Decision.writeTo(nodeDecision, out);
-        out.writeFloat(nodeWeight);
-        if (storeStatus == null) {
-            out.writeBoolean(false);
-        } else {
-            out.writeBoolean(true);
-            storeStatus.writeTo(out);
-        }
-        finalDecision.writeTo(out);
-        out.writeString(finalExplanation);
-        storeCopy.writeTo(out);
-    }
-
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject(node.getId()); {
-            builder.field("node_name", node.getName());
-            builder.startObject("node_attributes"); {
-                for (Map.Entry<String, String> attrEntry : node.getAttributes().entrySet()) {
-                    builder.field(attrEntry.getKey(), attrEntry.getValue());
-                }
-            }
-            builder.endObject(); // end attributes
-            builder.startObject("store"); {
-                builder.field("shard_copy", storeCopy.toString());
-                if (storeStatus != null) {
-                    final Throwable storeErr = storeStatus.getStoreException();
-                    if (storeErr != null) {
-                        builder.field("store_exception", ExceptionsHelper.detailedMessage(storeErr));
-                    }
-                }
-            }
-            builder.endObject(); // end store
-            builder.field("final_decision", finalDecision.toString());
-            builder.field("final_explanation", finalExplanation.toString());
-            builder.field("weight", nodeWeight);
-            nodeDecision.toXContent(builder, params);
-        }
-        builder.endObject(); // end node <uuid>
-        return builder;
-    }
-
-    public DiscoveryNode getNode() {
-        return this.node;
-    }
-
-    public Decision getDecision() {
-        return this.nodeDecision;
-    }
-
-    public Float getWeight() {
-        return this.nodeWeight;
-    }
-
-    @Nullable
-    public IndicesShardStoresResponse.StoreStatus getStoreStatus() {
-        return this.storeStatus;
-    }
-
-    public ClusterAllocationExplanation.FinalDecision getFinalDecision() {
-        return this.finalDecision;
-    }
-
-    public String getFinalExplanation() {
-        return this.finalExplanation;
-    }
-
-    public ClusterAllocationExplanation.StoreCopy getStoreCopy() {
-        return this.storeCopy;
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java
index a304fa60cb798..77d4b24d8cc23 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java
@@ -19,13 +19,7 @@
 
 package org.elasticsearch.action.admin.cluster.allocation;
 
-import org.apache.lucene.index.CorruptIndexException;
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresRequest;
-import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresResponse;
-import org.elasticsearch.action.admin.indices.shards.TransportIndicesShardStoresAction;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
 import org.elasticsearch.cluster.ClusterInfo;
@@ -33,34 +27,25 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.routing.RecoverySource;
-import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.ShardRouting;
-import org.elasticsearch.cluster.routing.UnassignedInfo;
+import org.elasticsearch.cluster.routing.allocation.MoveDecision;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
+import org.elasticsearch.cluster.routing.allocation.AllocateUnassignedDecision;
+import org.elasticsearch.cluster.routing.allocation.RoutingAllocation.DebugMode;
+import org.elasticsearch.cluster.routing.allocation.ShardAllocationDecision;
 import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocator;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
-import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.collect.ImmutableOpenIntMap;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.gateway.GatewayAllocator;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import static org.elasticsearch.cluster.routing.UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING;
 
 /**
  * The {@code TransportClusterAllocationExplainAction} is responsible for actually executing the explanation of a shard's allocation on the
@@ -72,7 +57,6 @@ public class TransportClusterAllocationExplainAction
     private final ClusterInfoService clusterInfoService;
     private final AllocationDeciders allocationDeciders;
     private final ShardsAllocator shardAllocator;
-    private final TransportIndicesShardStoresAction shardStoresAction;
     private final GatewayAllocator gatewayAllocator;
 
     @Inject
@@ -80,14 +64,12 @@ public TransportClusterAllocationExplainAction(Settings settings, TransportServi
                                                    ThreadPool threadPool, ActionFilters actionFilters,
                                                    IndexNameExpressionResolver indexNameExpressionResolver,
                                                    ClusterInfoService clusterInfoService, AllocationDeciders allocationDeciders,
-                                                   ShardsAllocator shardAllocator, TransportIndicesShardStoresAction shardStoresAction,
-                                                   GatewayAllocator gatewayAllocator) {
+                                                   ShardsAllocator shardAllocator, GatewayAllocator gatewayAllocator) {
         super(settings, ClusterAllocationExplainAction.NAME, transportService, clusterService, threadPool, actionFilters,
                 indexNameExpressionResolver, ClusterAllocationExplainRequest::new);
         this.clusterInfoService = clusterInfoService;
         this.allocationDeciders = allocationDeciders;
         this.shardAllocator = shardAllocator;
-        this.shardStoresAction = shardStoresAction;
         this.gatewayAllocator = gatewayAllocator;
     }
 
@@ -106,240 +88,114 @@ protected ClusterAllocationExplainResponse newResponse() {
         return new ClusterAllocationExplainResponse();
     }
 
-    /**
-     * Return the decisions for the given {@code ShardRouting} on the given {@code RoutingNode}. If {@code includeYesDecisions} is not true,
-     * only non-YES (NO and THROTTLE) decisions are returned.
-     */
-    public static Decision tryShardOnNode(ShardRouting shard, RoutingNode node, RoutingAllocation allocation, boolean includeYesDecisions) {
-        Decision d = allocation.deciders().canAllocate(shard, node, allocation);
-        if (includeYesDecisions) {
-            return d;
-        } else {
-            Decision.Multi nonYesDecisions = new Decision.Multi();
-            List<Decision> decisions = d.getDecisions();
-            for (Decision decision : decisions) {
-                if (decision.type() != Decision.Type.YES) {
-                    nonYesDecisions.add(decision);
-                }
-            }
-            return nonYesDecisions;
-        }
-    }
-
-    /**
-     * Construct a {@code NodeExplanation} object for the given shard given all the metadata. This also attempts to construct the human
-     * readable FinalDecision and final explanation as part of the explanation.
-     */
-    public static NodeExplanation calculateNodeExplanation(ShardRouting shard,
-                                                           IndexMetaData indexMetaData,
-                                                           DiscoveryNode node,
-                                                           Decision nodeDecision,
-                                                           Float nodeWeight,
-                                                           IndicesShardStoresResponse.StoreStatus storeStatus,
-                                                           String assignedNodeId,
-                                                           Set<String> activeAllocationIds,
-                                                           boolean hasPendingAsyncFetch) {
-        final ClusterAllocationExplanation.FinalDecision finalDecision;
-        final ClusterAllocationExplanation.StoreCopy storeCopy;
-        final String finalExplanation;
+    @Override
+    protected void masterOperation(final ClusterAllocationExplainRequest request, final ClusterState state,
+                                   final ActionListener<ClusterAllocationExplainResponse> listener) {
+        final RoutingNodes routingNodes = state.getRoutingNodes();
+        final ClusterInfo clusterInfo = clusterInfoService.getClusterInfo();
+        final RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, routingNodes, state,
+                clusterInfo, System.nanoTime(), false);
 
-        if (storeStatus == null) {
-            // No copies of the data
-            storeCopy = ClusterAllocationExplanation.StoreCopy.NONE;
-        } else {
-            final Exception storeErr = storeStatus.getStoreException();
-            if (storeErr != null) {
-                if (ExceptionsHelper.unwrapCause(storeErr) instanceof CorruptIndexException) {
-                    storeCopy = ClusterAllocationExplanation.StoreCopy.CORRUPT;
-                } else {
-                    storeCopy = ClusterAllocationExplanation.StoreCopy.IO_ERROR;
-                }
-            } else if (activeAllocationIds.isEmpty()) {
-                // The ids are only empty if dealing with a legacy index
-                // TODO: fetch the shard state versions and display here?
-                storeCopy = ClusterAllocationExplanation.StoreCopy.UNKNOWN;
-            } else if (activeAllocationIds.contains(storeStatus.getAllocationId())) {
-                storeCopy = ClusterAllocationExplanation.StoreCopy.AVAILABLE;
-            } else {
-                // Otherwise, this is a stale copy of the data (allocation ids don't match)
-                storeCopy = ClusterAllocationExplanation.StoreCopy.STALE;
-            }
-        }
+        ShardRouting shardRouting = findShardToExplain(request, allocation);
+        logger.debug("explaining the allocation for [{}], found shard [{}]", request, shardRouting);
 
-        if (node.getId().equals(assignedNodeId)) {
-            finalDecision = ClusterAllocationExplanation.FinalDecision.ALREADY_ASSIGNED;
-            finalExplanation = "the shard is already assigned to this node";
-        } else if (shard.unassigned() && shard.primary() == false &&
-                shard.unassignedInfo().getReason() != UnassignedInfo.Reason.INDEX_CREATED && nodeDecision.type() != Decision.Type.YES) {
-            finalExplanation = "the shard cannot be assigned because allocation deciders return a " + nodeDecision.type().name() +
-                    " decision";
-            finalDecision = ClusterAllocationExplanation.FinalDecision.NO;
-        } else if (shard.unassigned() && shard.primary() == false &&
-                shard.unassignedInfo().getReason() != UnassignedInfo.Reason.INDEX_CREATED && hasPendingAsyncFetch) {
-            finalExplanation = "the shard's state is still being fetched so it cannot be allocated";
-            finalDecision = ClusterAllocationExplanation.FinalDecision.NO;
-        } else if (shard.primary() && shard.unassigned() &&
-                (shard.recoverySource().getType() == RecoverySource.Type.EXISTING_STORE ||
-                    shard.recoverySource().getType() == RecoverySource.Type.SNAPSHOT)
-                && hasPendingAsyncFetch) {
-            finalExplanation = "the shard's state is still being fetched so it cannot be allocated";
-            finalDecision = ClusterAllocationExplanation.FinalDecision.NO;
-        } else if (shard.primary() && shard.unassigned() && shard.recoverySource().getType() == RecoverySource.Type.EXISTING_STORE &&
-                storeCopy == ClusterAllocationExplanation.StoreCopy.STALE) {
-            finalExplanation = "the copy of the shard is stale, allocation ids do not match";
-            finalDecision = ClusterAllocationExplanation.FinalDecision.NO;
-        } else if (shard.primary() && shard.unassigned() && shard.recoverySource().getType() == RecoverySource.Type.EXISTING_STORE &&
-                storeCopy == ClusterAllocationExplanation.StoreCopy.NONE) {
-            finalExplanation = "there is no copy of the shard available";
-            finalDecision = ClusterAllocationExplanation.FinalDecision.NO;
-        } else if (shard.primary() && shard.unassigned() && shard.recoverySource().getType() == RecoverySource.Type.EXISTING_STORE &&
-                storeCopy == ClusterAllocationExplanation.StoreCopy.CORRUPT) {
-            finalExplanation = "the copy of the shard is corrupt";
-            finalDecision = ClusterAllocationExplanation.FinalDecision.NO;
-        } else if (shard.primary() && shard.unassigned() && shard.recoverySource().getType() == RecoverySource.Type.EXISTING_STORE &&
-                storeCopy == ClusterAllocationExplanation.StoreCopy.IO_ERROR) {
-            finalExplanation = "the copy of the shard cannot be read";
-            finalDecision = ClusterAllocationExplanation.FinalDecision.NO;
-        } else {
-            if (nodeDecision.type() == Decision.Type.NO) {
-                finalDecision = ClusterAllocationExplanation.FinalDecision.NO;
-                finalExplanation = "the shard cannot be assigned because one or more allocation decider returns a 'NO' decision";
-            } else {
-                // TODO: handle throttling decision better here
-                finalDecision = ClusterAllocationExplanation.FinalDecision.YES;
-                if (storeCopy == ClusterAllocationExplanation.StoreCopy.AVAILABLE) {
-                    finalExplanation = "the shard can be assigned and the node contains a valid copy of the shard data";
-                } else {
-                    finalExplanation = "the shard can be assigned";
-                }
-            }
-        }
-        return new NodeExplanation(node, nodeDecision, nodeWeight, storeStatus, finalDecision, finalExplanation, storeCopy);
+        ClusterAllocationExplanation cae = explainShard(shardRouting, allocation,
+            request.includeDiskInfo() ? clusterInfo : null, request.includeYesDecisions(), gatewayAllocator, shardAllocator);
+        listener.onResponse(new ClusterAllocationExplainResponse(cae));
     }
 
+    // public for testing
+    public static ClusterAllocationExplanation explainShard(ShardRouting shardRouting, RoutingAllocation allocation,
+                                                            ClusterInfo clusterInfo, boolean includeYesDecisions,
+                                                            GatewayAllocator gatewayAllocator, ShardsAllocator shardAllocator) {
+        allocation.setDebugMode(includeYesDecisions ? DebugMode.ON : DebugMode.EXCLUDE_YES_DECISIONS);
 
-    /**
-     * For the given {@code ShardRouting}, return the explanation of the allocation for that shard on all nodes. If {@code
-     * includeYesDecisions} is true, returns all decisions, otherwise returns only 'NO' and 'THROTTLE' decisions.
-     */
-    public static ClusterAllocationExplanation explainShard(ShardRouting shard, RoutingAllocation allocation, RoutingNodes routingNodes,
-                                                            boolean includeYesDecisions, ShardsAllocator shardAllocator,
-                                                            List<IndicesShardStoresResponse.StoreStatus> shardStores,
-                                                            GatewayAllocator gatewayAllocator, ClusterInfo clusterInfo) {
-        // don't short circuit deciders, we want a full explanation
-        allocation.debugDecision(true);
-        // get the existing unassigned info if available
-        UnassignedInfo ui = shard.unassignedInfo();
-
-        Map<DiscoveryNode, Decision> nodeToDecision = new HashMap<>();
-        for (RoutingNode node : routingNodes) {
-            DiscoveryNode discoNode = node.node();
-            if (discoNode.isDataNode()) {
-                Decision d = tryShardOnNode(shard, node, allocation, includeYesDecisions);
-                nodeToDecision.put(discoNode, d);
+        ShardAllocationDecision shardDecision;
+        if (shardRouting.initializing() || shardRouting.relocating()) {
+            shardDecision = ShardAllocationDecision.NOT_TAKEN;
+        } else {
+            AllocateUnassignedDecision allocateDecision = shardRouting.unassigned() ?
+                gatewayAllocator.decideUnassignedShardAllocation(shardRouting, allocation) : AllocateUnassignedDecision.NOT_TAKEN;
+            if (allocateDecision.isDecisionTaken() == false) {
+                shardDecision = shardAllocator.decideShardAllocation(shardRouting, allocation);
+            } else {
+                shardDecision = new ShardAllocationDecision(allocateDecision, MoveDecision.NOT_TAKEN);
             }
         }
-        long remainingDelayMillis = 0;
-        final MetaData metadata = allocation.metaData();
-        final IndexMetaData indexMetaData = metadata.index(shard.index());
-        long allocationDelayMillis = INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.get(indexMetaData.getSettings()).getMillis();
-        if (ui != null && ui.isDelayed()) {
-            long remainingDelayNanos = ui.getRemainingDelay(System.nanoTime(), indexMetaData.getSettings());
-            remainingDelayMillis = TimeValue.timeValueNanos(remainingDelayNanos).millis();
-        }
 
-        // Calculate weights for each of the nodes
-        Map<DiscoveryNode, Float> weights = shardAllocator.weighShard(allocation, shard);
-
-        Map<DiscoveryNode, IndicesShardStoresResponse.StoreStatus> nodeToStatus = new HashMap<>(shardStores.size());
-        for (IndicesShardStoresResponse.StoreStatus status : shardStores) {
-            nodeToStatus.put(status.getNode(), status);
-        }
-
-        Map<DiscoveryNode, NodeExplanation> explanations = new HashMap<>(shardStores.size());
-        for (Map.Entry<DiscoveryNode, Decision> entry : nodeToDecision.entrySet()) {
-            DiscoveryNode node = entry.getKey();
-            Decision decision = entry.getValue();
-            Float weight = weights.get(node);
-            IndicesShardStoresResponse.StoreStatus storeStatus = nodeToStatus.get(node);
-            NodeExplanation nodeExplanation = calculateNodeExplanation(shard, indexMetaData, node, decision, weight,
-                    storeStatus, shard.currentNodeId(), indexMetaData.inSyncAllocationIds(shard.getId()),
-                    allocation.hasPendingAsyncFetch());
-            explanations.put(node, nodeExplanation);
-        }
-        return new ClusterAllocationExplanation(shard.shardId(), shard.primary(),
-                shard.currentNodeId(), allocationDelayMillis, remainingDelayMillis, ui,
-                gatewayAllocator.hasFetchPending(shard.shardId(), shard.primary()), explanations, clusterInfo);
+        return new ClusterAllocationExplanation(shardRouting,
+            shardRouting.currentNodeId() != null ? allocation.nodes().get(shardRouting.currentNodeId()) : null,
+            shardRouting.relocatingNodeId() != null ? allocation.nodes().get(shardRouting.relocatingNodeId()) : null,
+            clusterInfo, shardDecision);
     }
 
-    @Override
-    protected void masterOperation(final ClusterAllocationExplainRequest request, final ClusterState state,
-                                   final ActionListener<ClusterAllocationExplainResponse> listener) {
-        final RoutingNodes routingNodes = state.getRoutingNodes();
-        final ClusterInfo clusterInfo = clusterInfoService.getClusterInfo();
-        final RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, routingNodes, state,
-                clusterInfo, System.nanoTime(), false);
-
+    // public for testing
+    public static ShardRouting findShardToExplain(ClusterAllocationExplainRequest request, RoutingAllocation allocation) {
         ShardRouting foundShard = null;
         if (request.useAnyUnassignedShard()) {
             // If we can use any shard, just pick the first unassigned one (if there are any)
-            RoutingNodes.UnassignedShards.UnassignedIterator ui = routingNodes.unassigned().iterator();
+            RoutingNodes.UnassignedShards.UnassignedIterator ui = allocation.routingNodes().unassigned().iterator();
             if (ui.hasNext()) {
                 foundShard = ui.next();
             }
+            if (foundShard == null) {
+                throw new IllegalArgumentException("unable to find any unassigned shards to explain [" + request + "]");
+            }
         } else {
             String index = request.getIndex();
             int shard = request.getShard();
             if (request.isPrimary()) {
                 // If we're looking for the primary shard, there's only one copy, so pick it directly
                 foundShard = allocation.routingTable().shardRoutingTable(index, shard).primaryShard();
+                if (request.getCurrentNode() != null) {
+                    DiscoveryNode primaryNode = allocation.nodes().resolveNode(request.getCurrentNode());
+                    // the primary is assigned to a node other than the node specified in the request
+                    if (primaryNode.getId().equals(foundShard.currentNodeId()) == false) {
+                        throw new IllegalArgumentException(
+                                "unable to find primary shard assigned to node [" + request.getCurrentNode() + "]");
+                    }
+                }
             } else {
                 // If looking for a replica, go through all the replica shards
                 List<ShardRouting> replicaShardRoutings = allocation.routingTable().shardRoutingTable(index, shard).replicaShards();
-                if (replicaShardRoutings.size() > 0) {
-                    // Pick the first replica at the very least
-                    foundShard = replicaShardRoutings.get(0);
-                    // In case there are multiple replicas where some are assigned and some aren't,
-                    // try to find one that is unassigned at least
+                if (request.getCurrentNode() != null) {
+                    // the request is to explain a replica shard already assigned on a particular node,
+                    // so find that shard copy
+                    DiscoveryNode replicaNode = allocation.nodes().resolveNode(request.getCurrentNode());
                     for (ShardRouting replica : replicaShardRoutings) {
-                        if (replica.unassigned()) {
+                        if (replicaNode.getId().equals(replica.currentNodeId())) {
                             foundShard = replica;
                             break;
                         }
                     }
+                    if (foundShard == null) {
+                        throw new IllegalArgumentException("unable to find a replica shard assigned to node [" +
+                                                            request.getCurrentNode() + "]");
+                    }
+                } else {
+                    if (replicaShardRoutings.size() > 0) {
+                        // Pick the first replica at the very least
+                        foundShard = replicaShardRoutings.get(0);
+                        for (ShardRouting replica : replicaShardRoutings) {
+                            // In case there are multiple replicas where some are assigned and some aren't,
+                            // try to find one that is unassigned at least
+                            if (replica.unassigned()) {
+                                foundShard = replica;
+                                break;
+                            } else if (replica.started() && (foundShard.initializing() || foundShard.relocating())) {
+                                // prefer started shards to initializing or relocating shards because started shards
+                                // can be explained
+                                foundShard = replica;
+                            }
+                        }
+                    }
                 }
             }
         }
 
         if (foundShard == null) {
-            listener.onFailure(new ElasticsearchException("unable to find any shards to explain [{}] in the routing table", request));
-            return;
+            throw new IllegalArgumentException("unable to find any shards to explain [" + request + "] in the routing table");
         }
-        final ShardRouting shardRouting = foundShard;
-        logger.debug("explaining the allocation for [{}], found shard [{}]", request, shardRouting);
-
-        getShardStores(shardRouting, new ActionListener<IndicesShardStoresResponse>() {
-            @Override
-            public void onResponse(IndicesShardStoresResponse shardStoreResponse) {
-                ImmutableOpenIntMap<List<IndicesShardStoresResponse.StoreStatus>> shardStatuses =
-                        shardStoreResponse.getStoreStatuses().get(shardRouting.getIndexName());
-                List<IndicesShardStoresResponse.StoreStatus> shardStoreStatus = shardStatuses.get(shardRouting.id());
-                ClusterAllocationExplanation cae = explainShard(shardRouting, allocation, routingNodes,
-                        request.includeYesDecisions(), shardAllocator, shardStoreStatus, gatewayAllocator,
-                        request.includeDiskInfo() ? clusterInfo : null);
-                listener.onResponse(new ClusterAllocationExplainResponse(cae));
-            }
-
-            @Override
-            public void onFailure(Exception e) {
-                listener.onFailure(e);
-            }
-        });
-    }
-
-    private void getShardStores(ShardRouting shard, final ActionListener<IndicesShardStoresResponse> listener) {
-        IndicesShardStoresRequest request = new IndicesShardStoresRequest(shard.getIndexName());
-        request.shardStatuses("all");
-        shardStoresAction.execute(request, listener);
+        return foundShard;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponse.java
index d483ae86bf76e..a9a2c36970ee4 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponse.java
@@ -24,22 +24,19 @@
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.health.ClusterIndexHealth;
 import org.elasticsearch.cluster.health.ClusterStateHealth;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.xcontent.StatusToXContent;
+import org.elasticsearch.common.xcontent.StatusToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
 import java.util.Locale;
 import java.util.Map;
 
-/**
- *
- */
-public class ClusterHealthResponse extends ActionResponse implements StatusToXContent {
+public class ClusterHealthResponse extends ActionResponse implements StatusToXContentObject {
     private String clusterName;
     private int numberOfPendingTasks = 0;
     private int numberOfInFlightFetch = 0;
@@ -203,18 +200,9 @@ public void writeTo(StreamOutput out) throws IOException {
         taskMaxWaitingTime.writeTo(out);
     }
 
-
     @Override
     public String toString() {
-        try {
-            XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint();
-            builder.startObject();
-            toXContent(builder, EMPTY_PARAMS);
-            builder.endObject();
-            return builder.string();
-        } catch (IOException e) {
-            return "{ \"error\" : \"" + e.getMessage() + "\"}";
-        }
+        return Strings.toString(this);
     }
 
     @Override
@@ -243,6 +231,7 @@ public RestStatus status() {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
         builder.field(CLUSTER_NAME, getClusterName());
         builder.field(STATUS, getStatus().name().toLowerCase(Locale.ROOT));
         builder.field(TIMED_OUT, isTimedOut());
@@ -271,6 +260,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             }
             builder.endObject();
         }
+        builder.endObject();
         return builder;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java
index 9314079424012..44c604dc8b845 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.TransportMasterNodeReadAction;
+import org.elasticsearch.cluster.LocalClusterUpdateTask;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateObserver;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
@@ -44,9 +45,8 @@
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
-/**
- *
- */
+import java.util.function.Predicate;
+
 public class TransportClusterHealthAction extends TransportMasterNodeReadAction<ClusterHealthRequest, ClusterHealthResponse> {
 
     private final GatewayAllocator gatewayAllocator;
@@ -86,37 +86,55 @@ protected final void masterOperation(ClusterHealthRequest request, ClusterState
     protected void masterOperation(Task task, final ClusterHealthRequest request, final ClusterState unusedState, final ActionListener<ClusterHealthResponse> listener) {
         if (request.waitForEvents() != null) {
             final long endTimeMS = TimeValue.nsecToMSec(System.nanoTime()) + request.timeout().millis();
-            clusterService.submitStateUpdateTask("cluster_health (wait_for_events [" + request.waitForEvents() + "])", new ClusterStateUpdateTask(request.waitForEvents()) {
-                @Override
-                public ClusterState execute(ClusterState currentState) {
-                    return currentState;
-                }
+            if (request.local()) {
+                clusterService.submitStateUpdateTask("cluster_health (wait_for_events [" + request.waitForEvents() + "])", new LocalClusterUpdateTask(request.waitForEvents()) {
+                    @Override
+                    public ClusterTasksResult<LocalClusterUpdateTask> execute(ClusterState currentState) {
+                        return unchanged();
+                    }
 
-                @Override
-                public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
-                    final long timeoutInMillis = Math.max(0, endTimeMS - TimeValue.nsecToMSec(System.nanoTime()));
-                    final TimeValue newTimeout = TimeValue.timeValueMillis(timeoutInMillis);
-                    request.timeout(newTimeout);
-                    executeHealth(request, listener);
-                }
+                    @Override
+                    public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                        final long timeoutInMillis = Math.max(0, endTimeMS - TimeValue.nsecToMSec(System.nanoTime()));
+                        final TimeValue newTimeout = TimeValue.timeValueMillis(timeoutInMillis);
+                        request.timeout(newTimeout);
+                        executeHealth(request, listener);
+                    }
 
-                @Override
-                public void onNoLongerMaster(String source) {
-                    logger.trace("stopped being master while waiting for events with priority [{}]. retrying.", request.waitForEvents());
-                    doExecute(task, request, listener);
-                }
+                    @Override
+                    public void onFailure(String source, Exception e) {
+                        logger.error((Supplier<?>) () -> new ParameterizedMessage("unexpected failure during [{}]", source), e);
+                        listener.onFailure(e);
+                    }
+                });
+            } else {
+                clusterService.submitStateUpdateTask("cluster_health (wait_for_events [" + request.waitForEvents() + "])", new ClusterStateUpdateTask(request.waitForEvents()) {
+                    @Override
+                    public ClusterState execute(ClusterState currentState) {
+                        return currentState;
+                    }
 
-                @Override
-                public void onFailure(String source, Exception e) {
-                    logger.error((Supplier<?>) () -> new ParameterizedMessage("unexpected failure during [{}]", source), e);
-                    listener.onFailure(e);
-                }
+                    @Override
+                    public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                        final long timeoutInMillis = Math.max(0, endTimeMS - TimeValue.nsecToMSec(System.nanoTime()));
+                        final TimeValue newTimeout = TimeValue.timeValueMillis(timeoutInMillis);
+                        request.timeout(newTimeout);
+                        executeHealth(request, listener);
+                    }
 
-                @Override
-                public boolean runOnlyOnMaster() {
-                    return !request.local();
-                }
-            });
+                    @Override
+                    public void onNoLongerMaster(String source) {
+                        logger.trace("stopped being master while waiting for events with priority [{}]. retrying.", request.waitForEvents());
+                        doExecute(task, request, listener);
+                    }
+
+                    @Override
+                    public void onFailure(String source, Exception e) {
+                        logger.error((Supplier<?>) () -> new ParameterizedMessage("unexpected failure during [{}]", source), e);
+                        listener.onFailure(e);
+                    }
+                });
+            }
         } else {
             executeHealth(request, listener);
         }
@@ -142,19 +160,14 @@ private void executeHealth(final ClusterHealthRequest request, final ActionListe
         }
 
         assert waitFor >= 0;
-        final ClusterStateObserver observer = new ClusterStateObserver(clusterService, logger, threadPool.getThreadContext());
-        final ClusterState state = observer.observedState();
+        final ClusterState state = clusterService.state();
+        final ClusterStateObserver observer = new ClusterStateObserver(state, clusterService, null, logger, threadPool.getThreadContext());
         if (request.timeout().millis() == 0) {
             listener.onResponse(getResponse(request, state, waitFor, request.timeout().millis() == 0));
             return;
         }
         final int concreteWaitFor = waitFor;
-        final ClusterStateObserver.ChangePredicate validationPredicate = new ClusterStateObserver.ValidationPredicate() {
-            @Override
-            protected boolean validate(ClusterState newState) {
-                return newState.status() == ClusterState.ClusterStateStatus.APPLIED && validateRequest(request, newState, concreteWaitFor);
-            }
-        };
+        final Predicate<ClusterState> validationPredicate = newState -> validateRequest(request, newState, concreteWaitFor);
 
         final ClusterStateObserver.Listener stateListener = new ClusterStateObserver.Listener() {
             @Override
@@ -169,12 +182,11 @@ public void onClusterServiceClose() {
 
             @Override
             public void onTimeout(TimeValue timeout) {
-                final ClusterState clusterState = clusterService.state();
-                final ClusterHealthResponse response = getResponse(request, clusterState, concreteWaitFor, true);
+                final ClusterHealthResponse response = getResponse(request, observer.setAndGetObservedState(), concreteWaitFor, true);
                 listener.onResponse(response);
             }
         };
-        if (state.status() == ClusterState.ClusterStateStatus.APPLIED && validateRequest(request, state, concreteWaitFor)) {
+        if (validationPredicate.test(state)) {
             stateListener.onNewClusterState(state);
         } else {
             observer.waitForNextChange(stateListener, validationPredicate, request.timeout());
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/TransportNodesHotThreadsAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/TransportNodesHotThreadsAction.java
index 73403f40318cb..53d74432bae62 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/TransportNodesHotThreadsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/TransportNodesHotThreadsAction.java
@@ -24,7 +24,6 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.nodes.BaseNodeRequest;
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
-import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
@@ -85,11 +84,6 @@ protected NodeHotThreads nodeOperation(NodeRequest request) {
         }
     }
 
-    @Override
-    protected boolean accumulateExceptions() {
-        return false;
-    }
-
     public static class NodeRequest extends BaseNodeRequest {
 
         NodesHotThreadsRequest request;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java
index dd243cf6d88eb..0ab12fe6c0912 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java
@@ -220,7 +220,7 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeBoolean(false);
         } else {
             out.writeBoolean(true);
-            out.writeLong(totalIndexingBuffer.bytes());
+            out.writeLong(totalIndexingBuffer.getBytes());
         }
         if (settings == null) {
             out.writeBoolean(false);
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java
index 028198cf83140..6b6b74165fb24 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java
@@ -29,7 +29,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.node.service.NodeService;
+import org.elasticsearch.node.NodeService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -79,11 +79,6 @@ protected NodeInfo nodeOperation(NodeInfoRequest nodeRequest) {
                 request.transport(), request.http(), request.plugins(), request.ingest(), request.indices());
     }
 
-    @Override
-    protected boolean accumulateExceptions() {
-        return false;
-    }
-
     public static class NodeInfoRequest extends BaseNodeRequest {
 
         NodesInfoRequest request;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/liveness/LivenessRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/liveness/LivenessRequest.java
index 033dd5957d9fe..d6441bb8e77f5 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/liveness/LivenessRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/liveness/LivenessRequest.java
@@ -25,7 +25,7 @@
  * Transport level private response for the transport handler registered under
  * {@value org.elasticsearch.action.admin.cluster.node.liveness.TransportLivenessAction#NAME}
  */
-public final class LivenessRequest extends ActionRequest<LivenessRequest> {
+public final class LivenessRequest extends ActionRequest {
     @Override
     public ActionRequestValidationException validate() {
         return null;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java
index 38e91e342413a..9e301368453eb 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java
@@ -249,26 +249,26 @@ public void writeTo(StreamOutput out) throws IOException {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        if (!params.param("node_info_format", "default").equals("none")) {
-            builder.field("name", getNode().getName());
-            builder.field("transport_address", getNode().getAddress().toString());
-            builder.field("host", getNode().getHostName());
-            builder.field("ip", getNode().getAddress());
-
-            builder.startArray("roles");
-            for (DiscoveryNode.Role role : getNode().getRoles()) {
-                builder.value(role.getRoleName());
-            }
-            builder.endArray();
-
-            if (!getNode().getAttributes().isEmpty()) {
-                builder.startObject("attributes");
-                for (Map.Entry<String, String> attrEntry : getNode().getAttributes().entrySet()) {
-                    builder.field(attrEntry.getKey(), attrEntry.getValue());
-                }
-                builder.endObject();
+
+        builder.field("name", getNode().getName());
+        builder.field("transport_address", getNode().getAddress().toString());
+        builder.field("host", getNode().getHostName());
+        builder.field("ip", getNode().getAddress());
+
+        builder.startArray("roles");
+        for (DiscoveryNode.Role role : getNode().getRoles()) {
+            builder.value(role.getRoleName());
+        }
+        builder.endArray();
+
+        if (!getNode().getAttributes().isEmpty()) {
+            builder.startObject("attributes");
+            for (Map.Entry<String, String> attrEntry : getNode().getAttributes().entrySet()) {
+                builder.field(attrEntry.getKey(), attrEntry.getValue());
             }
+            builder.endObject();
         }
+
         if (getIndices() != null) {
             getIndices().toXContent(builder, params);
         }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java
index 5863e54d08fa1..f5ae1a780c048 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java
@@ -29,7 +29,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.node.service.NodeService;
+import org.elasticsearch.node.NodeService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -79,11 +79,6 @@ protected NodeStats nodeOperation(NodeStatsRequest nodeStatsRequest) {
                 request.ingest());
     }
 
-    @Override
-    protected boolean accumulateExceptions() {
-        return false;
-    }
-
     public static class NodeStatsRequest extends BaseNodeRequest {
 
         NodesStatsRequest request;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/TransportCancelTasksAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/TransportCancelTasksAction.java
index dc52e4fd5080b..aca1be7adff4c 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/TransportCancelTasksAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/TransportCancelTasksAction.java
@@ -19,14 +19,16 @@
 
 package org.elasticsearch.action.admin.cluster.node.tasks.cancel;
 
+import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import org.elasticsearch.ResourceNotFoundException;
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.TaskOperationFailure;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.tasks.TransportTasksAction;
-import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -45,10 +47,9 @@
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
-import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
 
 /**
@@ -111,92 +112,109 @@ protected void processTasks(CancelTasksRequest request, Consumer<CancellableTask
     }
 
     @Override
-    protected synchronized TaskInfo taskOperation(CancelTasksRequest request, CancellableTask cancellableTask) {
-        final BanLock banLock = new BanLock(nodes -> removeBanOnNodes(cancellableTask, nodes));
-        Set<String> childNodes = taskManager.cancel(cancellableTask, request.getReason(), banLock::onTaskFinished);
-        if (childNodes != null) {
-            if (childNodes.isEmpty()) {
-                logger.trace("cancelling task {} with no children", cancellableTask.getId());
-                return cancellableTask.taskInfo(clusterService.localNode(), false);
-            } else {
-                logger.trace("cancelling task {} with children on nodes [{}]", cancellableTask.getId(), childNodes);
-                setBanOnNodes(request.getReason(), cancellableTask, childNodes, banLock);
-                return cancellableTask.taskInfo(clusterService.localNode(), false);
+    protected synchronized void taskOperation(CancelTasksRequest request, CancellableTask cancellableTask,
+            ActionListener<TaskInfo> listener) {
+        String nodeId = clusterService.localNode().getId();
+        final boolean canceled;
+        if (cancellableTask.shouldCancelChildrenOnCancellation()) {
+            DiscoveryNodes childNodes = clusterService.state().nodes();
+            final BanLock banLock = new BanLock(childNodes.getSize(), () -> removeBanOnNodes(cancellableTask, childNodes));
+            canceled = taskManager.cancel(cancellableTask, request.getReason(), banLock::onTaskFinished);
+            if (canceled) {
+                // /In case the task has some child tasks, we need to wait for until ban is set on all nodes
+                logger.trace("cancelling task {} on child nodes", cancellableTask.getId());
+                AtomicInteger responses = new AtomicInteger(childNodes.getSize());
+                List<Exception> failures = new ArrayList<>();
+                setBanOnNodes(request.getReason(), cancellableTask, childNodes, new ActionListener<Void>() {
+                    @Override
+                    public void onResponse(Void aVoid) {
+                        processResponse();
+                    }
+
+                    @Override
+                    public void onFailure(Exception e) {
+                        synchronized (failures) {
+                            failures.add(e);
+                        }
+                        processResponse();
+                    }
+
+                    private void processResponse() {
+                        banLock.onBanSet();
+                        if (responses.decrementAndGet() == 0) {
+                            if (failures.isEmpty() == false) {
+                                IllegalStateException exception = new IllegalStateException("failed to cancel children of the task [" +
+                                    cancellableTask.getId() + "]");
+                                failures.forEach(exception::addSuppressed);
+                                listener.onFailure(exception);
+                            } else {
+                                listener.onResponse(cancellableTask.taskInfo(nodeId, false));
+                            }
+                        }
+                    }
+                });
             }
-        } else {
+        }  else {
+            canceled = taskManager.cancel(cancellableTask, request.getReason(),
+                () -> listener.onResponse(cancellableTask.taskInfo(nodeId, false)));
+            if (canceled) {
+                logger.trace("task {} doesn't have any children that should be cancelled", cancellableTask.getId());
+            }
+        }
+        if (canceled == false) {
             logger.trace("task {} is already cancelled", cancellableTask.getId());
             throw new IllegalStateException("task with id " + cancellableTask.getId() + " is already cancelled");
         }
     }
 
-    @Override
-    protected boolean accumulateExceptions() {
-        return true;
-    }
-
-    private void setBanOnNodes(String reason, CancellableTask task, Set<String> nodes, BanLock banLock) {
+    private void setBanOnNodes(String reason, CancellableTask task, DiscoveryNodes nodes, ActionListener<Void> listener) {
         sendSetBanRequest(nodes,
             BanParentTaskRequest.createSetBanParentTaskRequest(new TaskId(clusterService.localNode().getId(), task.getId()), reason),
-            banLock);
+            listener);
     }
 
-    private void removeBanOnNodes(CancellableTask task, Set<String> nodes) {
+    private void removeBanOnNodes(CancellableTask task, DiscoveryNodes nodes) {
         sendRemoveBanRequest(nodes,
             BanParentTaskRequest.createRemoveBanParentTaskRequest(new TaskId(clusterService.localNode().getId(), task.getId())));
     }
 
-    private void sendSetBanRequest(Set<String> nodes, BanParentTaskRequest request, BanLock banLock) {
-        ClusterState clusterState = clusterService.state();
-        for (String node : nodes) {
-            DiscoveryNode discoveryNode = clusterState.getNodes().get(node);
-            if (discoveryNode != null) {
-                // Check if node still in the cluster
-                logger.debug("Sending ban for tasks with the parent [{}] to the node [{}], ban [{}]", request.parentTaskId, node,
-                    request.ban);
-                transportService.sendRequest(discoveryNode, BAN_PARENT_ACTION_NAME, request,
-                    new EmptyTransportResponseHandler(ThreadPool.Names.SAME) {
-                        @Override
-                        public void handleResponse(TransportResponse.Empty response) {
-                            banLock.onBanSet();
-                        }
-
-                        @Override
-                        public void handleException(TransportException exp) {
-                            banLock.onBanSet();
-                        }
-                    });
-            } else {
-                banLock.onBanSet();
-                logger.debug("Cannot send ban for tasks with the parent [{}] to the node [{}] - the node no longer in the cluster",
-                    request.parentTaskId, node);
-            }
+    private void sendSetBanRequest(DiscoveryNodes nodes, BanParentTaskRequest request, ActionListener<Void> listener) {
+        for (ObjectObjectCursor<String, DiscoveryNode> node : nodes.getNodes()) {
+            logger.trace("Sending ban for tasks with the parent [{}] to the node [{}], ban [{}]", request.parentTaskId, node.key,
+                request.ban);
+            transportService.sendRequest(node.value, BAN_PARENT_ACTION_NAME, request,
+                new EmptyTransportResponseHandler(ThreadPool.Names.SAME) {
+                    @Override
+                    public void handleResponse(TransportResponse.Empty response) {
+                        listener.onResponse(null);
+                    }
+
+                    @Override
+                    public void handleException(TransportException exp) {
+                        logger.warn("Cannot send ban for tasks with the parent [{}] to the node [{}]", request.parentTaskId, node.key);
+                        listener.onFailure(exp);
+                    }
+                });
         }
     }
 
-    private void sendRemoveBanRequest(Set<String> nodes, BanParentTaskRequest request) {
-        ClusterState clusterState = clusterService.state();
-        for (String node : nodes) {
-            DiscoveryNode discoveryNode = clusterState.getNodes().get(node);
-            if (discoveryNode != null) {
-                // Check if node still in the cluster
-                logger.debug("Sending remove ban for tasks with the parent [{}] to the node [{}]", request.parentTaskId, node);
-                transportService.sendRequest(discoveryNode, BAN_PARENT_ACTION_NAME, request, EmptyTransportResponseHandler
-                    .INSTANCE_SAME);
-            } else {
-                logger.debug("Cannot send remove ban request for tasks with the parent [{}] to the node [{}] - the node no longer in " +
-                    "the cluster", request.parentTaskId, node);
-            }
+    private void sendRemoveBanRequest(DiscoveryNodes nodes, BanParentTaskRequest request) {
+        for (ObjectObjectCursor<String, DiscoveryNode> node : nodes.getNodes()) {
+            logger.debug("Sending remove ban for tasks with the parent [{}] to the node [{}]", request.parentTaskId, node.key);
+            transportService.sendRequest(node.value, BAN_PARENT_ACTION_NAME, request, EmptyTransportResponseHandler
+                .INSTANCE_SAME);
         }
     }
 
     private static class BanLock {
-        private final Consumer<Set<String>> finish;
+        private final Runnable finish;
         private final AtomicInteger counter;
-        private final AtomicReference<Set<String>> nodes = new AtomicReference<>();
+        private final int nodesSize;
 
-        public BanLock(Consumer<Set<String>> finish) {
+        BanLock(int nodesSize, Runnable finish) {
             counter = new AtomicInteger(0);
             this.finish = finish;
+            this.nodesSize = nodesSize;
         }
 
         public void onBanSet() {
@@ -205,15 +223,14 @@ public void onBanSet() {
             }
         }
 
-        public void onTaskFinished(Set<String> nodes) {
-            this.nodes.set(nodes);
-            if (counter.addAndGet(nodes.size()) == 0) {
+        public void onTaskFinished() {
+            if (counter.addAndGet(nodesSize) == 0) {
                 finish();
             }
         }
 
         public void finish() {
-            finish.accept(nodes.get());
+            finish.run();
         }
 
     }
@@ -245,7 +262,7 @@ private BanParentTaskRequest(TaskId parentTaskId) {
             this.ban = false;
         }
 
-        public BanParentTaskRequest() {
+        BanParentTaskRequest() {
         }
 
         @Override
@@ -285,5 +302,4 @@ public void messageReceived(final BanParentTaskRequest request, final TransportC
         }
     }
 
-
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskRequest.java
index efbc9679e714e..07d40b5ffcaa0 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskRequest.java
@@ -33,7 +33,7 @@
 /**
  * A request to get node tasks
  */
-public class GetTaskRequest extends ActionRequest<GetTaskRequest> {
+public class GetTaskRequest extends ActionRequest {
     private TaskId taskId = TaskId.EMPTY_TASK_ID;
     private boolean waitForCompletion = false;
     private TimeValue timeout = null;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskResponse.java
index ffd4b35831421..72f26d2d57692 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskResponse.java
@@ -23,7 +23,7 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.tasks.TaskResult;
 
@@ -34,7 +34,7 @@
 /**
  * Returns the list of tasks currently running on the nodes
  */
-public class GetTaskResponse extends ActionResponse implements ToXContent {
+public class GetTaskResponse extends ActionResponse implements ToXContentObject {
     private TaskResult task;
 
     public GetTaskResponse() {
@@ -65,7 +65,10 @@ public TaskResult getTask() {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        return task.innerToXContent(builder, params);
+        builder.startObject();
+        task.innerToXContent(builder, params);
+        builder.endObject();
+        return builder;
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java
index dc77a1a6e8415..30d71a992fd95 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java
@@ -31,17 +31,17 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.IndexNotFoundException;
-import org.elasticsearch.tasks.TaskResult;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.tasks.TaskInfo;
+import org.elasticsearch.tasks.TaskResult;
 import org.elasticsearch.tasks.TaskResultsService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportException;
@@ -67,14 +67,17 @@ public class TransportGetTaskAction extends HandledTransportAction<GetTaskReques
     private final ClusterService clusterService;
     private final TransportService transportService;
     private final Client client;
+    private final NamedXContentRegistry xContentRegistry;
 
     @Inject
     public TransportGetTaskAction(Settings settings, ThreadPool threadPool, TransportService transportService, ActionFilters actionFilters,
-            IndexNameExpressionResolver indexNameExpressionResolver, ClusterService clusterService, Client client) {
+            IndexNameExpressionResolver indexNameExpressionResolver, ClusterService clusterService, Client client,
+            NamedXContentRegistry xContentRegistry) {
         super(settings, GetTaskAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, GetTaskRequest::new);
         this.clusterService = clusterService;
         this.transportService = transportService;
         this.client = client;
+        this.xContentRegistry = xContentRegistry;
     }
 
     @Override
@@ -105,11 +108,18 @@ private void runOnNodeWithTaskIfPossible(Task thisTask, GetTaskRequest request,
         DiscoveryNode node = clusterService.state().nodes().get(request.getTaskId().getNodeId());
         if (node == null) {
             // Node is no longer part of the cluster! Try and look the task up from the results index.
-            getFinishedTaskFromIndex(thisTask, request, listener);
+            getFinishedTaskFromIndex(thisTask, request, ActionListener.wrap(listener::onResponse, e -> {
+                if (e instanceof ResourceNotFoundException) {
+                    e = new ResourceNotFoundException(
+                            "task [" + request.getTaskId() + "] belongs to the node [" + request.getTaskId().getNodeId()
+                                    + "] which isn't part of the cluster and there is no record of the task",
+                            e);
+                }
+                listener.onFailure(e);
+            }));
             return;
         }
         GetTaskRequest nodeRequest = request.nodeRequest(clusterService.localNode().getId(), thisTask.getId());
-        taskManager.registerChildTask(thisTask, node.getId());
         transportService.sendRequest(node, GetTaskAction.NAME, nodeRequest, builder.build(),
                 new TransportResponseHandler<GetTaskResponse>() {
                     @Override
@@ -150,7 +160,7 @@ void getRunningTaskFromNode(Task thisTask, GetTaskRequest request, ActionListene
                     @Override
                     protected void doRun() throws Exception {
                         taskManager.waitForTaskCompletion(runningTask, waitForCompletionTimeout(request.getTimeout()));
-                        waitedForCompletion(thisTask, request, runningTask.taskInfo(clusterService.localNode(), true), listener);
+                        waitedForCompletion(thisTask, request, runningTask.taskInfo(clusterService.localNode().getId(), true), listener);
                     }
 
                     @Override
@@ -159,7 +169,7 @@ public void onFailure(Exception e) {
                     }
                 });
             } else {
-                TaskInfo info = runningTask.taskInfo(clusterService.localNode(), true);
+                TaskInfo info = runningTask.taskInfo(clusterService.localNode().getId(), true);
                 listener.onResponse(new GetTaskResponse(new TaskResult(false, info)));
             }
         }
@@ -216,7 +226,7 @@ public void onResponse(GetResponse getResponse) {
             public void onFailure(Exception e) {
                 if (ExceptionsHelper.unwrap(e, IndexNotFoundException.class) != null) {
                     // We haven't yet created the index for the task results so it can't be found.
-                    listener.onFailure(new ResourceNotFoundException("task [{}] isn't running or stored its results", e,
+                    listener.onFailure(new ResourceNotFoundException("task [{}] isn't running and hasn't stored its results", e,
                         request.getTaskId()));
                 } else {
                     listener.onFailure(e);
@@ -231,15 +241,15 @@ public void onFailure(Exception e) {
      */
     void onGetFinishedTaskFromIndex(GetResponse response, ActionListener<GetTaskResponse> listener) throws IOException {
         if (false == response.isExists()) {
-            listener.onFailure(new ResourceNotFoundException("task [{}] isn't running or stored its results", response.getId()));
+            listener.onFailure(new ResourceNotFoundException("task [{}] isn't running and hasn't stored its results", response.getId()));
             return;
         }
         if (response.isSourceEmpty()) {
             listener.onFailure(new ElasticsearchException("Stored task status for [{}] didn't contain any source!", response.getId()));
             return;
         }
-        try (XContentParser parser = XContentHelper.createParser(response.getSourceAsBytesRef())) {
-            TaskResult result = TaskResult.PARSER.apply(parser, () -> ParseFieldMatcher.STRICT);
+        try (XContentParser parser = XContentHelper.createParser(xContentRegistry, response.getSourceAsBytesRef())) {
+            TaskResult result = TaskResult.PARSER.apply(parser, null);
             listener.onResponse(new GetTaskResponse(result));
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java
index b33226b973ba7..a203dd35b47ff 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java
@@ -27,7 +27,7 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.tasks.TaskInfo;
@@ -43,7 +43,7 @@
 /**
  * Returns the list of tasks currently running on the nodes
  */
-public class ListTasksResponse extends BaseTasksResponse implements ToXContent {
+public class ListTasksResponse extends BaseTasksResponse implements ToXContentObject {
 
     private List<TaskInfo> tasks;
 
@@ -161,8 +161,9 @@ public XContentBuilder toXContentGroupedByNode(XContentBuilder builder, Params p
             }
             builder.startObject("tasks");
             for(TaskInfo task : entry.getValue()) {
-                builder.field(task.getTaskId().toString());
+                builder.startObject(task.getTaskId().toString());
                 task.toXContent(builder, params);
+                builder.endObject();
             }
             builder.endObject();
             builder.endObject();
@@ -187,7 +188,10 @@ public XContentBuilder toXContentGroupedByParents(XContentBuilder builder, Param
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        return toXContentGroupedByParents(builder, params);
+        builder.startObject();
+        toXContentGroupedByParents(builder, params);
+        builder.endObject();
+        return builder;
     }
 
     private void toXContentCommon(XContentBuilder builder, Params params) throws IOException {
@@ -214,6 +218,6 @@ private void toXContentCommon(XContentBuilder builder, Params params) throws IOE
 
     @Override
     public String toString() {
-        return Strings.toString(this, true);
+        return Strings.toString(this);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TaskGroup.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TaskGroup.java
index b254137163d75..87bf70acede44 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TaskGroup.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TaskGroup.java
@@ -81,7 +81,7 @@ public List<TaskGroup> getChildTasks() {
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
-        task.innerToXContent(builder, params);
+        task.toXContent(builder, params);
         if (childTasks.isEmpty() == false) {
             builder.startArray("children");
             for (TaskGroup taskGroup : childTasks) {
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java
index 32c0c3c1845c3..a3cca310d6ab2 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.action.admin.cluster.node.tasks.list;
 
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.TaskOperationFailure;
 import org.elasticsearch.action.support.ActionFilters;
@@ -72,8 +73,8 @@ protected TaskInfo readTaskResponse(StreamInput in) throws IOException {
     }
 
     @Override
-    protected TaskInfo taskOperation(ListTasksRequest request, Task task) {
-        return task.taskInfo(clusterService.localNode(), request.getDetailed());
+    protected void taskOperation(ListTasksRequest request, Task task, ActionListener<TaskInfo> listener) {
+        listener.onResponse(task.taskInfo(clusterService.localNode().getId(), request.getDetailed()));
     }
 
     @Override
@@ -92,8 +93,4 @@ protected void processTasks(ListTasksRequest request, Consumer<Task> operation)
         super.processTasks(request, operation);
     }
 
-    @Override
-    protected boolean accumulateExceptions() {
-        return true;
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/remote/RemoteInfoAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/remote/RemoteInfoAction.java
new file mode 100644
index 0000000000000..aa546c7dffd26
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/remote/RemoteInfoAction.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.cluster.remote;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.client.ElasticsearchClient;
+
+public final class RemoteInfoAction extends Action<RemoteInfoRequest, RemoteInfoResponse, RemoteInfoRequestBuilder> {
+
+    public static final String NAME = "cluster:monitor/remote/info";
+    public static final RemoteInfoAction INSTANCE = new RemoteInfoAction();
+
+    public RemoteInfoAction() {
+        super(NAME);
+    }
+
+    @Override
+    public RemoteInfoRequestBuilder newRequestBuilder(ElasticsearchClient client) {
+        return new RemoteInfoRequestBuilder(client, INSTANCE);
+    }
+
+    @Override
+    public RemoteInfoResponse newResponse() {
+        return new RemoteInfoResponse();
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/remote/RemoteInfoRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/remote/RemoteInfoRequest.java
new file mode 100644
index 0000000000000..6e41f145b65e7
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/remote/RemoteInfoRequest.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.cluster.remote;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+
+public final class RemoteInfoRequest extends ActionRequest {
+
+    @Override
+    public ActionRequestValidationException validate() {
+        return null;
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/remote/RemoteInfoRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/remote/RemoteInfoRequestBuilder.java
new file mode 100644
index 0000000000000..f46f5ecd2d3ca
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/remote/RemoteInfoRequestBuilder.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.cluster.remote;
+
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.client.ElasticsearchClient;
+
+public final class RemoteInfoRequestBuilder extends ActionRequestBuilder<RemoteInfoRequest, RemoteInfoResponse, RemoteInfoRequestBuilder> {
+
+    public RemoteInfoRequestBuilder(ElasticsearchClient client, RemoteInfoAction action) {
+        super(client, action, new RemoteInfoRequest());
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/remote/RemoteInfoResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/remote/RemoteInfoResponse.java
new file mode 100644
index 0000000000000..8e9360bdb1238
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/remote/RemoteInfoResponse.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.cluster.remote;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.transport.RemoteConnectionInfo;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+public final class RemoteInfoResponse extends ActionResponse implements ToXContentObject {
+
+    private List<RemoteConnectionInfo> infos;
+
+    RemoteInfoResponse() {
+    }
+
+    RemoteInfoResponse(Collection<RemoteConnectionInfo> infos) {
+        this.infos = Collections.unmodifiableList(new ArrayList<>(infos));
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeList(infos);
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        infos = in.readList(RemoteConnectionInfo::new);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        for (RemoteConnectionInfo info : infos) {
+            info.toXContent(builder, params);
+        }
+        builder.endObject();
+        return builder;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/remote/TransportRemoteInfoAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/remote/TransportRemoteInfoAction.java
new file mode 100644
index 0000000000000..33254a9aed9ab
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/remote/TransportRemoteInfoAction.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.cluster.remote;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.transport.RemoteClusterService;
+import org.elasticsearch.action.search.SearchTransportService;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+
+public final class TransportRemoteInfoAction extends HandledTransportAction<RemoteInfoRequest, RemoteInfoResponse> {
+
+    private final RemoteClusterService remoteClusterService;
+
+    @Inject
+    public TransportRemoteInfoAction(Settings settings, ThreadPool threadPool, TransportService transportService,
+                                     ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
+                                     SearchTransportService searchTransportService) {
+        super(settings, RemoteInfoAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver,
+            RemoteInfoRequest::new);
+        this.remoteClusterService = searchTransportService.getRemoteClusterService();
+    }
+
+    @Override
+    protected void doExecute(RemoteInfoRequest remoteInfoRequest, ActionListener<RemoteInfoResponse> listener) {
+        remoteClusterService.getRemoteConnectionInfos(ActionListener.wrap(remoteConnectionInfos
+                -> listener.onResponse(new RemoteInfoResponse(remoteConnectionInfos)), listener::onFailure));
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesResponse.java
index c933156fcb077..6d4cb83934548 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesResponse.java
@@ -60,7 +60,7 @@ public List<RepositoryMetaData> repositories() {
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
         int size = in.readVInt();
-        List<RepositoryMetaData> repositoryListBuilder = new ArrayList<>();
+        List<RepositoryMetaData> repositoryListBuilder = new ArrayList<>(size);
         for (int j = 0; j < size; j++) {
             repositoryListBuilder.add(new RepositoryMetaData(
                     in.readString(),
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java
index 83beb0016a972..f0f8d50b4c1c9 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java
@@ -22,22 +22,20 @@
 import org.elasticsearch.ElasticsearchGenerationException;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 
 import java.io.IOException;
 import java.util.Map;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
-import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS;
 import static org.elasticsearch.common.settings.Settings.readSettingsFromStream;
 import static org.elasticsearch.common.settings.Settings.writeSettingsToStream;
+import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS;
 
 /**
  * Register repository request.
@@ -144,14 +142,28 @@ public PutRepositoryRequest settings(Settings.Builder settings) {
     /**
      * Sets the repository settings.
      *
-     * @param source repository settings in json, yaml or properties format
+     * @param source repository settings in json or yaml format
      * @return this request
+     * @deprecated use {@link #settings(String, XContentType)} to avoid content type auto-detection
      */
+    @Deprecated
     public PutRepositoryRequest settings(String source) {
         this.settings = Settings.builder().loadFromSource(source).build();
         return this;
     }
 
+    /**
+     * Sets the repository settings.
+     *
+     * @param source repository settings in json or yaml format
+     * @param xContentType the content type of the source
+     * @return this request
+     */
+    public PutRepositoryRequest settings(String source, XContentType xContentType) {
+        this.settings = Settings.builder().loadFromSource(source, xContentType).build();
+        return this;
+    }
+
     /**
      * Sets the repository settings.
      *
@@ -162,7 +174,7 @@ public PutRepositoryRequest settings(Map<String, Object> source) {
         try {
             XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
             builder.map(source);
-            settings(builder.string());
+            settings(builder.string(), builder.contentType());
         } catch (IOException e) {
             throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e);
         }
@@ -198,18 +210,8 @@ public boolean verify() {
      *
      * @param repositoryDefinition repository definition
      */
-    public PutRepositoryRequest source(XContentBuilder repositoryDefinition) {
-        return source(repositoryDefinition.bytes());
-    }
-
-    /**
-     * Parses repository definition.
-     *
-     * @param repositoryDefinition repository definition
-     */
-    public PutRepositoryRequest source(Map repositoryDefinition) {
-        Map<String, Object> source = repositoryDefinition;
-        for (Map.Entry<String, Object> entry : source.entrySet()) {
+    public PutRepositoryRequest source(Map<String, Object> repositoryDefinition) {
+        for (Map.Entry<String, Object> entry : repositoryDefinition.entrySet()) {
             String name = entry.getKey();
             if (name.equals("type")) {
                 type(entry.getValue().toString());
@@ -217,64 +219,14 @@ public PutRepositoryRequest source(Map repositoryDefinition) {
                 if (!(entry.getValue() instanceof Map)) {
                     throw new IllegalArgumentException("Malformed settings section, should include an inner object");
                 }
-                settings((Map<String, Object>) entry.getValue());
+                @SuppressWarnings("unchecked")
+                Map<String, Object> sub = (Map<String, Object>) entry.getValue();
+                settings(sub);
             }
         }
         return this;
     }
 
-    /**
-     * Parses repository definition.
-     * JSON, Smile and YAML formats are supported
-     *
-     * @param repositoryDefinition repository definition
-     */
-    public PutRepositoryRequest source(String repositoryDefinition) {
-        try (XContentParser parser = XContentFactory.xContent(repositoryDefinition).createParser(repositoryDefinition)) {
-            return source(parser.mapOrdered());
-        } catch (IOException e) {
-            throw new IllegalArgumentException("failed to parse repository source [" + repositoryDefinition + "]", e);
-        }
-    }
-
-    /**
-     * Parses repository definition.
-     * JSON, Smile and YAML formats are supported
-     *
-     * @param repositoryDefinition repository definition
-     */
-    public PutRepositoryRequest source(byte[] repositoryDefinition) {
-        return source(repositoryDefinition, 0, repositoryDefinition.length);
-    }
-
-    /**
-     * Parses repository definition.
-     * JSON, Smile and YAML formats are supported
-     *
-     * @param repositoryDefinition repository definition
-     */
-    public PutRepositoryRequest source(byte[] repositoryDefinition, int offset, int length) {
-        try (XContentParser parser = XContentFactory.xContent(repositoryDefinition, offset, length).createParser(repositoryDefinition, offset, length)) {
-            return source(parser.mapOrdered());
-        } catch (IOException e) {
-            throw new IllegalArgumentException("failed to parse repository source", e);
-        }
-    }
-
-    /**
-     * Parses repository definition.
-     * JSON, Smile and YAML formats are supported
-     *
-     * @param repositoryDefinition repository definition
-     */
-    public PutRepositoryRequest source(BytesReference repositoryDefinition) {
-        try (XContentParser parser = XContentFactory.xContent(repositoryDefinition).createParser(repositoryDefinition)) {
-            return source(parser.mapOrdered());
-        } catch (IOException e) {
-            throw new IllegalArgumentException("failed to parse template source", e);
-        }
-    }
-
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequestBuilder.java
index 39cfa6af7f750..aed09daff2a86 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequestBuilder.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 
 import java.util.Map;
 
@@ -89,16 +90,30 @@ public PutRepositoryRequestBuilder setSettings(Settings.Builder settings) {
     }
 
     /**
-     * Sets the repository settings in Json, Yaml or properties format
+     * Sets the repository settings in Json or Yaml format
      *
      * @param source repository settings
      * @return this builder
+     * @deprecated use {@link #setSettings(String, XContentType)} instead to avoid content type auto detection
      */
+    @Deprecated
     public PutRepositoryRequestBuilder setSettings(String source) {
         request.settings(source);
         return this;
     }
 
+    /**
+     * Sets the repository settings in Json or Yaml format
+     *
+     * @param source repository settings
+     * @param xContentType the contenty type of the source
+     * @return this builder
+     */
+    public PutRepositoryRequestBuilder setSettings(String source, XContentType xContentType) {
+        request.settings(source, xContentType);
+        return this;
+    }
+
     /**
      * Sets the repository settings
      *
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryResponse.java
index 5de45fafa6df4..27612a3dab24b 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryResponse.java
@@ -22,18 +22,18 @@
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentHelper;
 
 import java.io.IOException;
 
 /**
  * Unregister repository response
  */
-public class VerifyRepositoryResponse extends ActionResponse implements ToXContent {
+public class VerifyRepositoryResponse extends ActionResponse implements ToXContentObject {
 
     private DiscoveryNode[] nodes;
 
@@ -83,6 +83,7 @@ static final class Fields {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
         builder.startObject(Fields.NODES);
         for (DiscoveryNode node : nodes) {
             builder.startObject(node.getId());
@@ -90,11 +91,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.endObject();
         }
         builder.endObject();
+        builder.endObject();
         return builder;
     }
 
     @Override
     public String toString() {
-        return XContentHelper.toString(this);
+        return Strings.toString(this);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java
index fbb6a8d18e8b6..74bf8f341f393 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java
@@ -59,7 +59,7 @@ public RoutingExplanations getExplanations() {
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
-        state = ClusterState.Builder.readFrom(in, null);
+        state = ClusterState.readFrom(in, null);
         readAcknowledged(in);
         explanations = RoutingExplanations.readFrom(in);
     }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java
index fbfe3ad37133b..7aade821f838a 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java
@@ -31,7 +31,6 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
-import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.routing.allocation.RoutingExplanations;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
@@ -111,15 +110,14 @@ public void onFailure(String source, Exception e) {
 
         @Override
         public ClusterState execute(ClusterState currentState) {
-            RoutingAllocation.Result routingResult = allocationService.reroute(currentState, request.getCommands(), request.explain(),
-                request.isRetryFailed());
-            ClusterState newState = ClusterState.builder(currentState).routingResult(routingResult).build();
-            clusterStateToSend = newState;
-            explanations = routingResult.explanations();
+            AllocationService.CommandsResult commandsResult =
+                allocationService.reroute(currentState, request.getCommands(), request.explain(), request.isRetryFailed());
+            clusterStateToSend = commandsResult.getClusterState();
+            explanations = commandsResult.explanations();
             if (request.dryRun()) {
                 return currentState;
             }
-            return newState;
+            return commandsResult.getClusterState();
         }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequest.java
index 16310b58cbc5b..bd0110e644cb6 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequest.java
@@ -51,7 +51,7 @@ public ClusterUpdateSettingsRequest() {
     @Override
     public ActionRequestValidationException validate() {
         ActionRequestValidationException validationException = null;
-        if (transientSettings.getAsMap().isEmpty() && persistentSettings.getAsMap().isEmpty()) {
+        if (transientSettings.isEmpty() && persistentSettings.isEmpty()) {
             validationException = addValidationError("no settings to update", validationException);
         }
         return validationException;
@@ -83,12 +83,22 @@ public ClusterUpdateSettingsRequest transientSettings(Settings.Builder settings)
 
     /**
      * Sets the source containing the transient settings to be updated. They will not survive a full cluster restart
+     * @deprecated use {@link #transientSettings(String, XContentType)} to avoid content type detection
      */
+    @Deprecated
     public ClusterUpdateSettingsRequest transientSettings(String source) {
         this.transientSettings = Settings.builder().loadFromSource(source).build();
         return this;
     }
 
+    /**
+     * Sets the source containing the transient settings to be updated. They will not survive a full cluster restart
+     */
+    public ClusterUpdateSettingsRequest transientSettings(String source, XContentType xContentType) {
+        this.transientSettings = Settings.builder().loadFromSource(source, xContentType).build();
+        return this;
+    }
+
     /**
      * Sets the transient settings to be updated. They will not survive a full cluster restart
      */
@@ -97,7 +107,7 @@ public ClusterUpdateSettingsRequest transientSettings(Map source) {
         try {
             XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
             builder.map(source);
-            transientSettings(builder.string());
+            transientSettings(builder.string(), builder.contentType());
         } catch (IOException e) {
             throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e);
         }
@@ -122,12 +132,22 @@ public ClusterUpdateSettingsRequest persistentSettings(Settings.Builder settings
 
     /**
      * Sets the source containing the persistent settings to be updated. They will get applied cross restarts
+     * @deprecated use {@link #persistentSettings(String, XContentType)} to avoid content type detection
      */
+    @Deprecated
     public ClusterUpdateSettingsRequest persistentSettings(String source) {
         this.persistentSettings = Settings.builder().loadFromSource(source).build();
         return this;
     }
 
+    /**
+     * Sets the source containing the persistent settings to be updated. They will get applied cross restarts
+     */
+    public ClusterUpdateSettingsRequest persistentSettings(String source, XContentType xContentType) {
+        this.persistentSettings = Settings.builder().loadFromSource(source, xContentType).build();
+        return this;
+    }
+
     /**
      * Sets the persistent settings to be updated. They will get applied cross restarts
      */
@@ -136,7 +156,7 @@ public ClusterUpdateSettingsRequest persistentSettings(Map source) {
         try {
             XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
             builder.map(source);
-            persistentSettings(builder.string());
+            persistentSettings(builder.string(), builder.contentType());
         } catch (IOException e) {
             throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e);
         }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestBuilder.java
index f0492edfeb19f..906b1867b1f09 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestBuilder.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 
 import java.util.Map;
 
@@ -52,12 +53,22 @@ public ClusterUpdateSettingsRequestBuilder setTransientSettings(Settings.Builder
 
     /**
      * Sets the source containing the transient settings to be updated. They will not survive a full cluster restart
+     * @deprecated use {@link #setTransientSettings(String, XContentType)} to avoid content type detection
      */
+    @Deprecated
     public ClusterUpdateSettingsRequestBuilder setTransientSettings(String settings) {
         request.transientSettings(settings);
         return this;
     }
 
+    /**
+     * Sets the source containing the transient settings to be updated. They will not survive a full cluster restart
+     */
+    public ClusterUpdateSettingsRequestBuilder setTransientSettings(String settings, XContentType xContentType) {
+        request.transientSettings(settings, xContentType);
+        return this;
+    }
+
     /**
      * Sets the transient settings to be updated. They will not survive a full cluster restart
      */
@@ -84,12 +95,22 @@ public ClusterUpdateSettingsRequestBuilder setPersistentSettings(Settings.Builde
 
     /**
      * Sets the source containing the persistent settings to be updated. They will get applied cross restarts
+     * @deprecated use {@link #setPersistentSettings(String, XContentType)} to avoid content type detection
      */
+    @Deprecated
     public ClusterUpdateSettingsRequestBuilder setPersistentSettings(String settings) {
         request.persistentSettings(settings);
         return this;
     }
 
+    /**
+     * Sets the source containing the persistent settings to be updated. They will get applied cross restarts
+     */
+    public ClusterUpdateSettingsRequestBuilder setPersistentSettings(String settings, XContentType xContentType) {
+        request.persistentSettings(settings, xContentType);
+        return this;
+    }
+
     /**
      * Sets the persistent settings to be updated. They will get applied cross restarts
      */
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/SettingsUpdater.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/SettingsUpdater.java
index 575fbcd3b9827..e9fec716a90c7 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/SettingsUpdater.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/SettingsUpdater.java
@@ -67,12 +67,20 @@ synchronized ClusterState updateSettings(final ClusterState currentState, Settin
             .transientSettings(transientSettings.build());
 
         ClusterBlocks.Builder blocks = ClusterBlocks.builder().blocks(currentState.blocks());
-        boolean updatedReadOnly = MetaData.SETTING_READ_ONLY_SETTING.get(metaData.persistentSettings()) || MetaData.SETTING_READ_ONLY_SETTING.get(metaData.transientSettings());
+        boolean updatedReadOnly = MetaData.SETTING_READ_ONLY_SETTING.get(metaData.persistentSettings())
+            || MetaData.SETTING_READ_ONLY_SETTING.get(metaData.transientSettings());
         if (updatedReadOnly) {
             blocks.addGlobalBlock(MetaData.CLUSTER_READ_ONLY_BLOCK);
         } else {
             blocks.removeGlobalBlock(MetaData.CLUSTER_READ_ONLY_BLOCK);
         }
+        boolean updatedReadOnlyAllowDelete = MetaData.SETTING_READ_ONLY_ALLOW_DELETE_SETTING.get(metaData.persistentSettings())
+            || MetaData.SETTING_READ_ONLY_ALLOW_DELETE_SETTING.get(metaData.transientSettings());
+        if (updatedReadOnlyAllowDelete) {
+            blocks.addGlobalBlock(MetaData.CLUSTER_READ_ONLY_ALLOW_DELETE_BLOCK);
+        } else {
+            blocks.removeGlobalBlock(MetaData.CLUSTER_READ_ONLY_ALLOW_DELETE_BLOCK);
+        }
         ClusterState build = builder(currentState).metaData(metaData).blocks(blocks).build();
         Settings settings = build.metaData().settings();
         // now we try to apply things and if they are invalid we fail
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java
index 2907716eaae0d..dae55b2fc048a 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java
@@ -33,7 +33,6 @@
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
-import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Priority;
@@ -43,19 +42,19 @@
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
-/**
- *
- */
-public class TransportClusterUpdateSettingsAction extends TransportMasterNodeAction<ClusterUpdateSettingsRequest, ClusterUpdateSettingsResponse> {
+public class TransportClusterUpdateSettingsAction extends
+    TransportMasterNodeAction<ClusterUpdateSettingsRequest, ClusterUpdateSettingsResponse> {
 
     private final AllocationService allocationService;
 
     private final ClusterSettings clusterSettings;
 
     @Inject
-    public TransportClusterUpdateSettingsAction(Settings settings, TransportService transportService, ClusterService clusterService, ThreadPool threadPool,
-                                                AllocationService allocationService, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, ClusterSettings clusterSettings) {
-        super(settings, ClusterUpdateSettingsAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver, ClusterUpdateSettingsRequest::new);
+    public TransportClusterUpdateSettingsAction(Settings settings, TransportService transportService, ClusterService clusterService,
+                                                ThreadPool threadPool, AllocationService allocationService, ActionFilters actionFilters,
+                                                IndexNameExpressionResolver indexNameExpressionResolver, ClusterSettings clusterSettings) {
+        super(settings, ClusterUpdateSettingsAction.NAME, false, transportService, clusterService, threadPool, actionFilters,
+            indexNameExpressionResolver, ClusterUpdateSettingsRequest::new);
         this.allocationService = allocationService;
         this.clusterSettings = clusterSettings;
     }
@@ -68,9 +67,15 @@ protected String executor() {
     @Override
     protected ClusterBlockException checkBlock(ClusterUpdateSettingsRequest request, ClusterState state) {
         // allow for dedicated changes to the metadata blocks, so we don't block those to allow to "re-enable" it
-        if ((request.transientSettings().getAsMap().isEmpty() && request.persistentSettings().getAsMap().size() == 1 && MetaData.SETTING_READ_ONLY_SETTING.exists(request.persistentSettings())) ||
-                request.persistentSettings().getAsMap().isEmpty() && request.transientSettings().getAsMap().size() == 1 && MetaData.SETTING_READ_ONLY_SETTING.exists(request.transientSettings())) {
-            return null;
+        if (request.transientSettings().size() + request.persistentSettings().size() == 1) {
+            // only one setting
+            if (MetaData.SETTING_READ_ONLY_SETTING.exists(request.persistentSettings())
+                || MetaData.SETTING_READ_ONLY_SETTING.exists(request.transientSettings())
+                || MetaData.SETTING_READ_ONLY_ALLOW_DELETE_SETTING.exists(request.transientSettings())
+                || MetaData.SETTING_READ_ONLY_ALLOW_DELETE_SETTING.exists(request.persistentSettings())) {
+                // one of the settings above as the only setting in the request means - resetting the block!
+                return null;
+            }
         }
         return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
     }
@@ -82,7 +87,8 @@ protected ClusterUpdateSettingsResponse newResponse() {
     }
 
     @Override
-    protected void masterOperation(final ClusterUpdateSettingsRequest request, final ClusterState state, final ActionListener<ClusterUpdateSettingsResponse> listener) {
+    protected void masterOperation(final ClusterUpdateSettingsRequest request, final ClusterState state,
+                                   final ActionListener<ClusterUpdateSettingsResponse> listener) {
         final SettingsUpdater updater = new SettingsUpdater(clusterSettings);
         clusterService.submitStateUpdateTask("cluster_update_settings",
                 new AckedClusterStateUpdateTask<ClusterUpdateSettingsResponse>(Priority.IMMEDIATE, request, listener) {
@@ -118,7 +124,8 @@ private void reroute(final boolean updateSettingsAcked) {
                 // so we should *not* execute the reroute.
                 if (!clusterService.state().nodes().isLocalNodeElectedMaster()) {
                     logger.debug("Skipping reroute after cluster update settings, because node is no longer master");
-                    listener.onResponse(new ClusterUpdateSettingsResponse(updateSettingsAcked, updater.getTransientUpdates(), updater.getPersistentUpdate()));
+                    listener.onResponse(new ClusterUpdateSettingsResponse(updateSettingsAcked, updater.getTransientUpdates(),
+                        updater.getPersistentUpdate()));
                     return;
                 }
 
@@ -136,15 +143,18 @@ public boolean mustAck(DiscoveryNode discoveryNode) {
                     }
 
                     @Override
-                    //we return when the cluster reroute is acked or it times out but the acknowledged flag depends on whether the update settings was acknowledged
+                    // we return when the cluster reroute is acked or it times out but the acknowledged flag depends on whether the
+                    // update settings was acknowledged
                     protected ClusterUpdateSettingsResponse newResponse(boolean acknowledged) {
-                        return new ClusterUpdateSettingsResponse(updateSettingsAcked && acknowledged, updater.getTransientUpdates(), updater.getPersistentUpdate());
+                        return new ClusterUpdateSettingsResponse(updateSettingsAcked && acknowledged, updater.getTransientUpdates(),
+                            updater.getPersistentUpdate());
                     }
 
                     @Override
                     public void onNoLongerMaster(String source) {
                         logger.debug("failed to preform reroute after cluster settings were updated - current node is no longer a master");
-                        listener.onResponse(new ClusterUpdateSettingsResponse(updateSettingsAcked, updater.getTransientUpdates(), updater.getPersistentUpdate()));
+                        listener.onResponse(new ClusterUpdateSettingsResponse(updateSettingsAcked, updater.getTransientUpdates(),
+                            updater.getPersistentUpdate()));
                     }
 
                     @Override
@@ -157,11 +167,7 @@ public void onFailure(String source, Exception e) {
                     @Override
                     public ClusterState execute(final ClusterState currentState) {
                         // now, reroute in case things that require it changed (e.g. number of replicas)
-                        RoutingAllocation.Result routingResult = allocationService.reroute(currentState, "reroute after cluster update settings");
-                        if (!routingResult.changed()) {
-                            return currentState;
-                        }
-                        return ClusterState.builder(currentState).routingResult(routingResult).build();
+                        return allocationService.reroute(currentState, "reroute after cluster update settings");
                     }
                 });
             }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsGroup.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsGroup.java
index ccb4d32465eda..a89e86f2a4fee 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsGroup.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsGroup.java
@@ -25,7 +25,6 @@
 import org.elasticsearch.common.io.stream.Streamable;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
 
 import java.io.IOException;
@@ -35,9 +34,9 @@
 public class ClusterSearchShardsGroup implements Streamable, ToXContent {
 
     private ShardId shardId;
-    ShardRouting[] shards;
+    private ShardRouting[] shards;
 
-    ClusterSearchShardsGroup() {
+    private ClusterSearchShardsGroup() {
 
     }
 
@@ -46,18 +45,14 @@ public ClusterSearchShardsGroup(ShardId shardId, ShardRouting[] shards) {
         this.shards = shards;
     }
 
-    public static ClusterSearchShardsGroup readSearchShardsGroupResponse(StreamInput in) throws IOException {
+    static ClusterSearchShardsGroup readSearchShardsGroupResponse(StreamInput in) throws IOException {
         ClusterSearchShardsGroup response = new ClusterSearchShardsGroup();
         response.readFrom(in);
         return response;
     }
 
-    public String getIndex() {
-        return shardId.getIndexName();
-    }
-
-    public int getShardId() {
-        return shardId.id();
+    public ShardId getShardId() {
+        return shardId;
     }
 
     public ShardRouting[] getShards() {
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequest.java
index 21ecf8a4c4feb..2f643bef7212d 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequest.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.action.admin.cluster.shards;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.support.IndicesOptions;
@@ -29,16 +30,17 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 
 import java.io.IOException;
+import java.util.Objects;
 
 /**
  */
 public class ClusterSearchShardsRequest extends MasterNodeReadRequest<ClusterSearchShardsRequest> implements IndicesRequest.Replaceable {
-    private String[] indices;
+
+    private String[] indices = Strings.EMPTY_ARRAY;
     @Nullable
     private String routing;
     @Nullable
     private String preference;
-    private String[] types = Strings.EMPTY_ARRAY;
     private IndicesOptions indicesOptions = IndicesOptions.lenientExpandOpen();
 
 
@@ -59,14 +61,9 @@ public ActionRequestValidationException validate() {
      */
     @Override
     public ClusterSearchShardsRequest indices(String... indices) {
-        if (indices == null) {
-            throw new IllegalArgumentException("indices must not be null");
-        } else {
-            for (int i = 0; i < indices.length; i++) {
-                if (indices[i] == null) {
-                    throw new IllegalArgumentException("indices[" + i + "] must not be null");
-                }
-            }
+        Objects.requireNonNull(indices, "indices must not be null");
+        for (int i = 0; i < indices.length; i++) {
+            Objects.requireNonNull(indices[i], "indices[" + i + "] must not be null");
         }
         this.indices = indices;
         return this;
@@ -90,23 +87,6 @@ public ClusterSearchShardsRequest indicesOptions(IndicesOptions indicesOptions)
         return this;
     }
 
-    /**
-     * The document types to execute the search against. Defaults to be executed against
-     * all types.
-     */
-    public String[] types() {
-        return types;
-    }
-
-    /**
-     * The document types to execute the search against. Defaults to be executed against
-     * all types.
-     */
-    public ClusterSearchShardsRequest types(String... types) {
-        this.types = types;
-        return this;
-    }
-
     /**
      * A comma separated list of routing values to control the shards the search will be executed on.
      */
@@ -156,7 +136,10 @@ public void readFrom(StreamInput in) throws IOException {
         routing = in.readOptionalString();
         preference = in.readOptionalString();
 
-        types = in.readStringArray();
+        if (in.getVersion().onOrBefore(Version.V_5_1_1)) {
+            //types
+            in.readStringArray();
+        }
         indicesOptions = IndicesOptions.readIndicesOptions(in);
     }
 
@@ -172,7 +155,10 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalString(routing);
         out.writeOptionalString(preference);
 
-        out.writeStringArray(types);
+        if (out.getVersion().onOrBefore(Version.V_5_1_1)) {
+            //types
+            out.writeStringArray(Strings.EMPTY_ARRAY);
+        }
         indicesOptions.writeIndicesOptions(out);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequestBuilder.java
index 0b9c9d044e7b6..100c9b258f2cc 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequestBuilder.java
@@ -39,15 +39,6 @@ public ClusterSearchShardsRequestBuilder setIndices(String... indices) {
         return this;
     }
 
-    /**
-     * The document types to execute the search against. Defaults to be executed against
-     * all types.
-     */
-    public ClusterSearchShardsRequestBuilder setTypes(String... types) {
-        request.types(types);
-        return this;
-    }
-
     /**
      * A comma separated list of routing values to control the shards the search will be executed on.
      */
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsResponse.java
index 5f45025351e65..b5b28e2b8f79a 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsResponse.java
@@ -19,24 +19,35 @@
 
 package org.elasticsearch.action.admin.cluster.shards;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.search.internal.AliasFilter;
 
 import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
 
-/**
- */
-public class ClusterSearchShardsResponse extends ActionResponse implements ToXContent {
+public class ClusterSearchShardsResponse extends ActionResponse implements ToXContentObject {
 
     private ClusterSearchShardsGroup[] groups;
     private DiscoveryNode[] nodes;
+    private Map<String, AliasFilter> indicesAndFilters;
+
+    public ClusterSearchShardsResponse() {
 
-    ClusterSearchShardsResponse() {
+    }
 
+    public ClusterSearchShardsResponse(ClusterSearchShardsGroup[] groups, DiscoveryNode[] nodes,
+                                       Map<String, AliasFilter> indicesAndFilters) {
+        this.groups = groups;
+        this.nodes = nodes;
+        this.indicesAndFilters = indicesAndFilters;
     }
 
     public ClusterSearchShardsGroup[] getGroups() {
@@ -47,9 +58,8 @@ public DiscoveryNode[] getNodes() {
         return nodes;
     }
 
-    public ClusterSearchShardsResponse(ClusterSearchShardsGroup[] groups, DiscoveryNode[] nodes) {
-        this.groups = groups;
-        this.nodes = nodes;
+    public Map<String, AliasFilter> getIndicesAndFilters() {
+        return indicesAndFilters;
     }
 
     @Override
@@ -63,7 +73,15 @@ public void readFrom(StreamInput in) throws IOException {
         for (int i = 0; i < nodes.length; i++) {
             nodes[i] = new DiscoveryNode(in);
         }
-
+        if (in.getVersion().onOrAfter(Version.V_5_1_1)) {
+            int size = in.readVInt();
+            indicesAndFilters = new HashMap<>();
+            for (int i = 0; i < size; i++) {
+                String index = in.readString();
+                AliasFilter aliasFilter = new AliasFilter(in);
+                indicesAndFilters.put(index, aliasFilter);
+            }
+        }
     }
 
     @Override
@@ -77,20 +95,48 @@ public void writeTo(StreamOutput out) throws IOException {
         for (DiscoveryNode node : nodes) {
             node.writeTo(out);
         }
+        if (out.getVersion().onOrAfter(Version.V_5_1_1)) {
+            out.writeVInt(indicesAndFilters.size());
+            for (Map.Entry<String, AliasFilter> entry : indicesAndFilters.entrySet()) {
+                out.writeString(entry.getKey());
+                entry.getValue().writeTo(out);
+            }
+        }
     }
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
         builder.startObject("nodes");
         for (DiscoveryNode node : nodes) {
             node.toXContent(builder, params);
         }
         builder.endObject();
+        if (indicesAndFilters != null) {
+            builder.startObject("indices");
+            for (Map.Entry<String, AliasFilter> entry : indicesAndFilters.entrySet()) {
+                String index = entry.getKey();
+                builder.startObject(index);
+                AliasFilter aliasFilter = entry.getValue();
+                String[] aliases = aliasFilter.getAliases();
+                if (aliases.length > 0) {
+                    Arrays.sort(aliases); // we want consistent ordering here and these values might be generated from a set / map
+                    builder.array("aliases", aliases);
+                    if (aliasFilter.getQueryBuilder() != null) { // might be null if we include non-filtering aliases
+                        builder.field("filter");
+                        aliasFilter.getQueryBuilder().toXContent(builder, params);
+                    }
+                }
+                builder.endObject();
+            }
+            builder.endObject();
+        }
         builder.startArray("shards");
         for (ClusterSearchShardsGroup group : groups) {
             group.toXContent(builder, params);
         }
         builder.endArray();
+        builder.endObject();
         return builder;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java
index 2f9a6e7dede56..20ed69ae5a92f 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java
@@ -33,23 +33,29 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 
-/**
- */
-public class TransportClusterSearchShardsAction extends TransportMasterNodeReadAction<ClusterSearchShardsRequest, ClusterSearchShardsResponse> {
+public class TransportClusterSearchShardsAction extends
+        TransportMasterNodeReadAction<ClusterSearchShardsRequest, ClusterSearchShardsResponse> {
+
+    private final IndicesService indicesService;
 
     @Inject
     public TransportClusterSearchShardsAction(Settings settings, TransportService transportService, ClusterService clusterService,
-                                              ThreadPool threadPool, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
-        super(settings, ClusterSearchShardsAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver, ClusterSearchShardsRequest::new);
+                                              IndicesService indicesService, ThreadPool threadPool, ActionFilters actionFilters,
+                                              IndexNameExpressionResolver indexNameExpressionResolver) {
+        super(settings, ClusterSearchShardsAction.NAME, transportService, clusterService, threadPool, actionFilters,
+                indexNameExpressionResolver, ClusterSearchShardsRequest::new);
+        this.indicesService = indicesService;
     }
 
     @Override
@@ -60,7 +66,8 @@ protected String executor() {
 
     @Override
     protected ClusterBlockException checkBlock(ClusterSearchShardsRequest request, ClusterState state) {
-        return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ, indexNameExpressionResolver.concreteIndexNames(state, request));
+        return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ,
+                indexNameExpressionResolver.concreteIndexNames(state, request));
     }
 
     @Override
@@ -69,12 +76,22 @@ protected ClusterSearchShardsResponse newResponse() {
     }
 
     @Override
-    protected void masterOperation(final ClusterSearchShardsRequest request, final ClusterState state, final ActionListener<ClusterSearchShardsResponse> listener) {
+    protected void masterOperation(final ClusterSearchShardsRequest request, final ClusterState state,
+                                   final ActionListener<ClusterSearchShardsResponse> listener) {
         ClusterState clusterState = clusterService.state();
         String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(clusterState, request);
         Map<String, Set<String>> routingMap = indexNameExpressionResolver.resolveSearchRouting(state, request.routing(), request.indices());
+        Map<String, AliasFilter> indicesAndFilters = new HashMap<>();
+        for (String index : concreteIndices) {
+            final AliasFilter aliasFilter = indicesService.buildAliasFilter(clusterState, index, request.indices());
+            final String[] aliases = indexNameExpressionResolver.indexAliases(clusterState, index, aliasMetadata -> true, true,
+                request.indices());
+            indicesAndFilters.put(index, new AliasFilter(aliasFilter.getQueryBuilder(), aliases));
+        }
+
         Set<String> nodeIds = new HashSet<>();
-        GroupShardsIterator groupShardsIterator = clusterService.operationRouting().searchShards(clusterState, concreteIndices, routingMap, request.preference());
+        GroupShardsIterator<ShardIterator> groupShardsIterator = clusterService.operationRouting().searchShards(clusterState, concreteIndices,
+                routingMap, request.preference());
         ShardRouting shard;
         ClusterSearchShardsGroup[] groupResponses = new ClusterSearchShardsGroup[groupShardsIterator.size()];
         int currentGroup = 0;
@@ -94,6 +111,6 @@ protected void masterOperation(final ClusterSearchShardsRequest request, final C
         for (String nodeId : nodeIds) {
             nodes[currentNode++] = clusterState.getNodes().get(nodeId);
         }
-        listener.onResponse(new ClusterSearchShardsResponse(groupResponses, nodes));
+        listener.onResponse(new ClusterSearchShardsResponse(groupResponses, nodes, indicesAndFilters));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java
index 2d0943845caeb..7c14f97482fb1 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java
@@ -25,13 +25,11 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.MasterNodeRequest;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 
 import java.io.IOException;
@@ -41,11 +39,11 @@
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 import static org.elasticsearch.common.Strings.EMPTY_ARRAY;
-import static org.elasticsearch.common.Strings.hasLength;
-import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS;
 import static org.elasticsearch.common.settings.Settings.readSettingsFromStream;
 import static org.elasticsearch.common.settings.Settings.writeSettingsToStream;
+import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS;
 import static org.elasticsearch.common.xcontent.support.XContentMapValues.lenientNodeBooleanValue;
+import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue;
 
 /**
  * Create snapshot request
@@ -291,18 +289,34 @@ public CreateSnapshotRequest settings(Settings.Builder settings) {
     }
 
     /**
-     * Sets repository-specific snapshot settings in JSON, YAML or properties format
+     * Sets repository-specific snapshot settings in JSON or YAML format
      * <p>
      * See repository documentation for more information.
      *
      * @param source repository-specific snapshot settings
      * @return this request
+     * @deprecated use {@link #settings(String, XContentType)} to avoid content type detection
      */
+    @Deprecated
     public CreateSnapshotRequest settings(String source) {
         this.settings = Settings.builder().loadFromSource(source).build();
         return this;
     }
 
+    /**
+     * Sets repository-specific snapshot settings in JSON or YAML format
+     * <p>
+     * See repository documentation for more information.
+     *
+     * @param source repository-specific snapshot settings
+     * @param xContentType the content type of the source
+     * @return this request
+     */
+    public CreateSnapshotRequest settings(String source, XContentType xContentType) {
+        this.settings = Settings.builder().loadFromSource(source, xContentType).build();
+        return this;
+    }
+
     /**
      * Sets repository-specific snapshot settings.
      * <p>
@@ -315,7 +329,7 @@ public CreateSnapshotRequest settings(Map<String, Object> source) {
         try {
             XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
             builder.map(source);
-            settings(builder.string());
+            settings(builder.string(), builder.contentType());
         } catch (IOException e) {
             throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e);
         }
@@ -357,17 +371,7 @@ public boolean includeGlobalState() {
      * @param source snapshot definition
      * @return this request
      */
-    public CreateSnapshotRequest source(XContentBuilder source) {
-        return source(source.bytes());
-    }
-
-    /**
-     * Parses snapshot definition.
-     *
-     * @param source snapshot definition
-     * @return this request
-     */
-    public CreateSnapshotRequest source(Map source) {
+    public CreateSnapshotRequest source(Map<String, Object> source) {
         for (Map.Entry<String, Object> entry : ((Map<String, Object>) source).entrySet()) {
             String name = entry.getKey();
             if (name.equals("indices")) {
@@ -379,80 +383,20 @@ public CreateSnapshotRequest source(Map source) {
                     throw new IllegalArgumentException("malformed indices section, should be an array of strings");
                 }
             } else if (name.equals("partial")) {
-                partial(lenientNodeBooleanValue(entry.getValue()));
+                partial(lenientNodeBooleanValue(entry.getValue(), name));
             } else if (name.equals("settings")) {
                 if (!(entry.getValue() instanceof Map)) {
                     throw new IllegalArgumentException("malformed settings section, should indices an inner object");
                 }
                 settings((Map<String, Object>) entry.getValue());
             } else if (name.equals("include_global_state")) {
-                includeGlobalState = lenientNodeBooleanValue(entry.getValue());
+                includeGlobalState = lenientNodeBooleanValue(entry.getValue(), name);
             }
         }
         indicesOptions(IndicesOptions.fromMap((Map<String, Object>) source, IndicesOptions.lenientExpandOpen()));
         return this;
     }
 
-    /**
-     * Parses snapshot definition. JSON, YAML and properties formats are supported
-     *
-     * @param source snapshot definition
-     * @return this request
-     */
-    public CreateSnapshotRequest source(String source) {
-        if (hasLength(source)) {
-            try (XContentParser parser = XContentFactory.xContent(source).createParser(source)) {
-                return source(parser.mapOrdered());
-            } catch (Exception e) {
-                throw new IllegalArgumentException("failed to parse repository source [" + source + "]", e);
-            }
-        }
-        return this;
-    }
-
-    /**
-     * Parses snapshot definition. JSON, YAML and properties formats are supported
-     *
-     * @param source snapshot definition
-     * @return this request
-     */
-    public CreateSnapshotRequest source(byte[] source) {
-        return source(source, 0, source.length);
-    }
-
-    /**
-     * Parses snapshot definition. JSON, YAML and properties formats are supported
-     *
-     * @param source snapshot definition
-     * @param offset offset
-     * @param length length
-     * @return this request
-     */
-    public CreateSnapshotRequest source(byte[] source, int offset, int length) {
-        if (length > 0) {
-            try (XContentParser parser = XContentFactory.xContent(source, offset, length).createParser(source, offset, length)) {
-                return source(parser.mapOrdered());
-            } catch (IOException e) {
-                throw new IllegalArgumentException("failed to parse repository source", e);
-            }
-        }
-        return this;
-    }
-
-    /**
-     * Parses snapshot definition. JSON, YAML and properties formats are supported
-     *
-     * @param source snapshot definition
-     * @return this request
-     */
-    public CreateSnapshotRequest source(BytesReference source) {
-        try (XContentParser parser = XContentFactory.xContent(source).createParser(source)) {
-            return source(parser.mapOrdered());
-        } catch (IOException e) {
-            throw new IllegalArgumentException("failed to parse snapshot source", e);
-        }
-    }
-
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
@@ -478,4 +422,9 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeBoolean(waitForCompletion);
         out.writeBoolean(partial);
     }
+
+    @Override
+    public String getDescription() {
+        return "snapshot [" + repository + ":" + snapshot + "]";
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestBuilder.java
index ebdd206b5c3e8..d3b5e12351c28 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestBuilder.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 
 import java.util.Map;
 
@@ -147,12 +148,28 @@ public CreateSnapshotRequestBuilder setSettings(Settings.Builder settings) {
      *
      * @param source repository-specific snapshot settings
      * @return this builder
+     * @deprecated use {@link #setSettings(String, XContentType)} to avoid content type detection
      */
+    @Deprecated
     public CreateSnapshotRequestBuilder setSettings(String source) {
         request.settings(source);
         return this;
     }
 
+    /**
+     * Sets repository-specific snapshot settings in YAML or JSON format
+     * <p>
+     * See repository documentation for more information.
+     *
+     * @param source repository-specific snapshot settings
+     * @param xContentType the content type of the source
+     * @return this builder
+     */
+    public CreateSnapshotRequestBuilder setSettings(String source, XContentType xContentType) {
+        request.settings(source, xContentType);
+        return this;
+    }
+
     /**
      * Sets repository-specific snapshot settings.
      * <p>
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponse.java
index efc2fbeb5b580..1f9f77f9ed3df 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponse.java
@@ -23,7 +23,7 @@
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.snapshots.SnapshotInfo;
@@ -33,7 +33,7 @@
 /**
  * Create snapshot response
  */
-public class CreateSnapshotResponse extends ActionResponse implements ToXContent {
+public class CreateSnapshotResponse extends ActionResponse implements ToXContentObject {
 
     @Nullable
     private SnapshotInfo snapshotInfo;
@@ -83,12 +83,14 @@ public RestStatus status() {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
         if (snapshotInfo != null) {
             builder.field("snapshot");
             snapshotInfo.toXContent(builder, params);
         } else {
             builder.field("accepted", true);
         }
+        builder.endObject();
         return builder;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java
index 3c37d1870e576..5cce5482ec508 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java
@@ -75,6 +75,6 @@ public void onResponse() {
             public void onFailure(Exception e) {
                 listener.onFailure(e);
             }
-        });
+        }, false);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java
index fd2c97ed5d43c..32c9493b440d5 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java
@@ -28,6 +28,7 @@
 import java.io.IOException;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
+import static org.elasticsearch.snapshots.SnapshotInfo.VERBOSE_INTRODUCED;
 
 /**
  * Get snapshot request
@@ -36,6 +37,7 @@ public class GetSnapshotsRequest extends MasterNodeRequest<GetSnapshotsRequest>
 
     public static final String ALL_SNAPSHOTS = "_all";
     public static final String CURRENT_SNAPSHOT = "_current";
+    public static final boolean DEFAULT_VERBOSE_MODE = true;
 
     private String repository;
 
@@ -43,6 +45,8 @@ public class GetSnapshotsRequest extends MasterNodeRequest<GetSnapshotsRequest>
 
     private boolean ignoreUnavailable;
 
+    private boolean verbose = DEFAULT_VERBOSE_MODE;
+
     public GetSnapshotsRequest() {
     }
 
@@ -123,6 +127,7 @@ public GetSnapshotsRequest ignoreUnavailable(boolean ignoreUnavailable) {
         this.ignoreUnavailable = ignoreUnavailable;
         return this;
     }
+
     /**
      * @return Whether snapshots should be ignored when unavailable (corrupt or temporarily not fetchable)
      */
@@ -130,12 +135,36 @@ public boolean ignoreUnavailable() {
         return ignoreUnavailable;
     }
 
+    /**
+     * Set to {@code false} to only show the snapshot names and the indices they contain.
+     * This is useful when the snapshots belong to a cloud-based repository where each
+     * blob read is a concern (cost wise and performance wise), as the snapshot names and
+     * indices they contain can be retrieved from a single index blob in the repository,
+     * whereas the rest of the information requires reading a snapshot metadata file for
+     * each snapshot requested.  Defaults to {@code true}, which returns all information
+     * about each requested snapshot.
+     */
+    public GetSnapshotsRequest verbose(boolean verbose) {
+        this.verbose = verbose;
+        return this;
+    }
+
+    /**
+     * Returns whether the request will return a verbose response.
+     */
+    public boolean verbose() {
+        return verbose;
+    }
+
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
         repository = in.readString();
         snapshots = in.readStringArray();
         ignoreUnavailable = in.readBoolean();
+        if (in.getVersion().onOrAfter(VERBOSE_INTRODUCED)) {
+            verbose = in.readBoolean();
+        }
     }
 
     @Override
@@ -144,5 +173,8 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeString(repository);
         out.writeStringArray(snapshots);
         out.writeBoolean(ignoreUnavailable);
+        if (out.getVersion().onOrAfter(VERBOSE_INTRODUCED)) {
+            out.writeBoolean(verbose);
+        }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequestBuilder.java
index 3b0ac47c69f9f..2115bd0bc3b81 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequestBuilder.java
@@ -96,4 +96,18 @@ public GetSnapshotsRequestBuilder setIgnoreUnavailable(boolean ignoreUnavailable
         return this;
     }
 
+    /**
+     * Set to {@code false} to only show the snapshot names and the indices they contain.
+     * This is useful when the snapshots belong to a cloud-based repository where each
+     * blob read is a concern (cost wise and performance wise), as the snapshot names and
+     * indices they contain can be retrieved from a single index blob in the repository,
+     * whereas the rest of the information requires reading a snapshot metadata file for
+     * each snapshot requested.  Defaults to {@code true}, which returns all information
+     * about each requested snapshot.
+     */
+    public GetSnapshotsRequestBuilder setVerbose(boolean verbose) {
+        request.verbose(verbose);
+        return this;
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java
index 924f5a90d4256..0d1e5eda7f2d2 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.snapshots.SnapshotInfo;
 
@@ -34,7 +35,7 @@
 /**
  * Get snapshots response
  */
-public class GetSnapshotsResponse extends ActionResponse implements ToXContent {
+public class GetSnapshotsResponse extends ActionResponse implements ToXContentObject {
 
     private List<SnapshotInfo> snapshots = Collections.emptyList();
 
@@ -58,7 +59,7 @@ public List<SnapshotInfo> getSnapshots() {
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
         int size = in.readVInt();
-        List<SnapshotInfo> builder = new ArrayList<>();
+        List<SnapshotInfo> builder = new ArrayList<>(size);
         for (int i = 0; i < size; i++) {
             builder.add(new SnapshotInfo(in));
         }
@@ -76,11 +77,13 @@ public void writeTo(StreamOutput out) throws IOException {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
+        builder.startObject();
         builder.startArray("snapshots");
         for (SnapshotInfo snapshotInfo : snapshots) {
             snapshotInfo.toXContent(builder, params);
         }
         builder.endArray();
+        builder.endObject();
         return builder;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java
index ad8cb1ae88e52..eec218a4119ba 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.action.admin.cluster.snapshots.get;
 
+import org.apache.lucene.util.CollectionUtil;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
@@ -30,6 +31,8 @@
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.repositories.IndexId;
+import org.elasticsearch.repositories.RepositoryData;
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.snapshots.SnapshotInfo;
 import org.elasticsearch.snapshots.SnapshotMissingException;
@@ -38,11 +41,13 @@
 import org.elasticsearch.transport.TransportService;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
-import java.util.LinkedHashSet;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 /**
  * Transport Action for get snapshots operation
@@ -75,30 +80,36 @@ protected ClusterBlockException checkBlock(GetSnapshotsRequest request, ClusterS
     }
 
     @Override
-    protected void masterOperation(final GetSnapshotsRequest request, ClusterState state,
+    protected void masterOperation(final GetSnapshotsRequest request, final ClusterState state,
                                    final ActionListener<GetSnapshotsResponse> listener) {
         try {
             final String repository = request.repository();
-            List<SnapshotInfo> snapshotInfoBuilder = new ArrayList<>();
-            if (isAllSnapshots(request.snapshots())) {
-                snapshotInfoBuilder.addAll(snapshotsService.currentSnapshots(repository));
-                snapshotInfoBuilder.addAll(snapshotsService.snapshots(repository,
-                                                                      snapshotsService.snapshotIds(repository),
-                                                                      request.ignoreUnavailable()));
-            } else if (isCurrentSnapshots(request.snapshots())) {
-                snapshotInfoBuilder.addAll(snapshotsService.currentSnapshots(repository));
-            } else {
-                final Map<String, SnapshotId> allSnapshotIds = new HashMap<>();
-                for (SnapshotInfo snapshotInfo : snapshotsService.currentSnapshots(repository)) {
-                    SnapshotId snapshotId = snapshotInfo.snapshotId();
-                    allSnapshotIds.put(snapshotId.getName(), snapshotId);
-                }
-                for (SnapshotId snapshotId : snapshotsService.snapshotIds(repository)) {
+            final Map<String, SnapshotId> allSnapshotIds = new HashMap<>();
+            final List<SnapshotInfo> currentSnapshots = new ArrayList<>();
+            for (SnapshotInfo snapshotInfo : snapshotsService.currentSnapshots(repository)) {
+                SnapshotId snapshotId = snapshotInfo.snapshotId();
+                allSnapshotIds.put(snapshotId.getName(), snapshotId);
+                currentSnapshots.add(snapshotInfo);
+            }
+
+            final RepositoryData repositoryData;
+            if (isCurrentSnapshotsOnly(request.snapshots()) == false) {
+                repositoryData = snapshotsService.getRepositoryData(repository);
+                for (SnapshotId snapshotId : repositoryData.getAllSnapshotIds()) {
                     allSnapshotIds.put(snapshotId.getName(), snapshotId);
                 }
-                final Set<SnapshotId> toResolve = new LinkedHashSet<>(); // maintain order
+            } else {
+                repositoryData = null;
+            }
+
+            final Set<SnapshotId> toResolve = new HashSet<>();
+            if (isAllSnapshots(request.snapshots())) {
+                toResolve.addAll(allSnapshotIds.values());
+            } else {
                 for (String snapshotOrPattern : request.snapshots()) {
-                    if (Regex.isSimpleMatchPattern(snapshotOrPattern) == false) {
+                    if (GetSnapshotsRequest.CURRENT_SNAPSHOT.equalsIgnoreCase(snapshotOrPattern)) {
+                        toResolve.addAll(currentSnapshots.stream().map(SnapshotInfo::snapshotId).collect(Collectors.toList()));
+                    } else if (Regex.isSimpleMatchPattern(snapshotOrPattern) == false) {
                         if (allSnapshotIds.containsKey(snapshotOrPattern)) {
                             toResolve.add(allSnapshotIds.get(snapshotOrPattern));
                         } else if (request.ignoreUnavailable() == false) {
@@ -113,13 +124,28 @@ protected void masterOperation(final GetSnapshotsRequest request, ClusterState s
                     }
                 }
 
-                if (toResolve.isEmpty() && request.ignoreUnavailable() == false) {
+                if (toResolve.isEmpty() && request.ignoreUnavailable() == false && isCurrentSnapshotsOnly(request.snapshots()) == false) {
                     throw new SnapshotMissingException(repository, request.snapshots()[0]);
                 }
+            }
 
-                snapshotInfoBuilder.addAll(snapshotsService.snapshots(repository, new ArrayList<>(toResolve), request.ignoreUnavailable()));
+            final List<SnapshotInfo> snapshotInfos;
+            if (request.verbose()) {
+                final Set<SnapshotId> incompatibleSnapshots = repositoryData != null ?
+                    new HashSet<>(repositoryData.getIncompatibleSnapshotIds()) : Collections.emptySet();
+                snapshotInfos = snapshotsService.snapshots(repository, new ArrayList<>(toResolve),
+                    incompatibleSnapshots, request.ignoreUnavailable());
+            } else {
+                if (repositoryData != null) {
+                    // want non-current snapshots as well, which are found in the repository data
+                    snapshotInfos = buildSimpleSnapshotInfos(toResolve, repositoryData, currentSnapshots);
+                } else {
+                    // only want current snapshots
+                    snapshotInfos = currentSnapshots.stream().map(SnapshotInfo::basic).collect(Collectors.toList());
+                    CollectionUtil.timSort(snapshotInfos);
+                }
             }
-            listener.onResponse(new GetSnapshotsResponse(snapshotInfoBuilder));
+            listener.onResponse(new GetSnapshotsResponse(snapshotInfos));
         } catch (Exception e) {
             listener.onFailure(e);
         }
@@ -129,7 +155,35 @@ private boolean isAllSnapshots(String[] snapshots) {
         return (snapshots.length == 0) || (snapshots.length == 1 && GetSnapshotsRequest.ALL_SNAPSHOTS.equalsIgnoreCase(snapshots[0]));
     }
 
-    private boolean isCurrentSnapshots(String[] snapshots) {
+    private boolean isCurrentSnapshotsOnly(String[] snapshots) {
         return (snapshots.length == 1 && GetSnapshotsRequest.CURRENT_SNAPSHOT.equalsIgnoreCase(snapshots[0]));
     }
+
+    private List<SnapshotInfo> buildSimpleSnapshotInfos(final Set<SnapshotId> toResolve,
+                                                        final RepositoryData repositoryData,
+                                                        final List<SnapshotInfo> currentSnapshots) {
+        List<SnapshotInfo> snapshotInfos = new ArrayList<>();
+        for (SnapshotInfo snapshotInfo : currentSnapshots) {
+            if (toResolve.remove(snapshotInfo.snapshotId())) {
+                snapshotInfos.add(snapshotInfo.basic());
+            }
+        }
+        Map<SnapshotId, List<String>> snapshotsToIndices = new HashMap<>();
+        for (IndexId indexId : repositoryData.getIndices().values()) {
+            for (SnapshotId snapshotId : repositoryData.getSnapshots(indexId)) {
+                if (toResolve.contains(snapshotId)) {
+                    snapshotsToIndices.computeIfAbsent(snapshotId, (k) -> new ArrayList<>())
+                                      .add(indexId.getName());
+                }
+            }
+        }
+        for (Map.Entry<SnapshotId, List<String>> entry : snapshotsToIndices.entrySet()) {
+            final List<String> indices = entry.getValue();
+            CollectionUtil.timSort(indices);
+            final SnapshotId snapshotId = entry.getKey();
+            snapshotInfos.add(new SnapshotInfo(snapshotId, indices, repositoryData.getSnapshotState(snapshotId)));
+        }
+        CollectionUtil.timSort(snapshotInfos);
+        return Collections.unmodifiableList(snapshotInfos);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java
index 1a41a776c7340..9561b50eaeee7 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java
@@ -24,13 +24,11 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.MasterNodeRequest;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 
 import java.io.IOException;
@@ -39,10 +37,9 @@
 import java.util.Map;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
-import static org.elasticsearch.common.Strings.hasLength;
-import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS;
 import static org.elasticsearch.common.settings.Settings.readSettingsFromStream;
 import static org.elasticsearch.common.settings.Settings.writeSettingsToStream;
+import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS;
 import static org.elasticsearch.common.xcontent.support.XContentMapValues.lenientNodeBooleanValue;
 
 /**
@@ -316,18 +313,34 @@ public RestoreSnapshotRequest settings(Settings.Builder settings) {
     }
 
     /**
-     * Sets repository-specific restore settings in JSON, YAML or properties format
+     * Sets repository-specific restore settings in JSON or YAML format
      * <p>
      * See repository documentation for more information.
      *
      * @param source repository-specific snapshot settings
      * @return this request
+     * @deprecated use {@link #settings(String, XContentType)} to avoid content type detection
      */
+    @Deprecated
     public RestoreSnapshotRequest settings(String source) {
         this.settings = Settings.builder().loadFromSource(source).build();
         return this;
     }
 
+    /**
+     * Sets repository-specific restore settings in JSON or YAML format
+     * <p>
+     * See repository documentation for more information.
+     *
+     * @param source repository-specific snapshot settings
+     * @param xContentType the content type of the source
+     * @return this request
+     */
+    public RestoreSnapshotRequest settings(String source, XContentType xContentType) {
+        this.settings = Settings.builder().loadFromSource(source, xContentType).build();
+        return this;
+    }
+
     /**
      * Sets repository-specific restore settings
      * <p>
@@ -340,7 +353,7 @@ public RestoreSnapshotRequest settings(Map<String, Object> source) {
         try {
             XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
             builder.map(source);
-            settings(builder.string());
+            settings(builder.string(), builder.contentType());
         } catch (IOException e) {
             throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e);
         }
@@ -439,12 +452,22 @@ public RestoreSnapshotRequest indexSettings(Settings.Builder settings) {
 
     /**
      * Sets settings that should be added/changed in all restored indices
+     * @deprecated use {@link #indexSettings(String, XContentType)} to avoid content type detection
      */
+    @Deprecated
     public RestoreSnapshotRequest indexSettings(String source) {
         this.indexSettings = Settings.builder().loadFromSource(source).build();
         return this;
     }
 
+    /**
+     * Sets settings that should be added/changed in all restored indices
+     */
+    public RestoreSnapshotRequest indexSettings(String source, XContentType xContentType) {
+        this.indexSettings = Settings.builder().loadFromSource(source, xContentType).build();
+        return this;
+    }
+
     /**
      * Sets settings that should be added/changed in all restored indices
      */
@@ -452,7 +475,7 @@ public RestoreSnapshotRequest indexSettings(Map<String, Object> source) {
         try {
             XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
             builder.map(source);
-            indexSettings(builder.string());
+            indexSettings(builder.string(), builder.contentType());
         } catch (IOException e) {
             throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e);
         }
@@ -472,22 +495,8 @@ public Settings indexSettings() {
      * @param source restore definition
      * @return this request
      */
-    public RestoreSnapshotRequest source(XContentBuilder source) {
-        try {
-            return source(source.bytes());
-        } catch (Exception e) {
-            throw new IllegalArgumentException("Failed to build json for repository request", e);
-        }
-    }
-
-    /**
-     * Parses restore definition
-     *
-     * @param source restore definition
-     * @return this request
-     */
-    public RestoreSnapshotRequest source(Map source) {
-        for (Map.Entry<String, Object> entry : ((Map<String, Object>) source).entrySet()) {
+    public RestoreSnapshotRequest source(Map<String, Object> source) {
+        for (Map.Entry<String, Object> entry : source.entrySet()) {
             String name = entry.getKey();
             if (name.equals("indices")) {
                 if (entry.getValue() instanceof String) {
@@ -498,16 +507,16 @@ public RestoreSnapshotRequest source(Map source) {
                     throw new IllegalArgumentException("malformed indices section, should be an array of strings");
                 }
             } else if (name.equals("partial")) {
-                partial(lenientNodeBooleanValue(entry.getValue()));
+                partial(lenientNodeBooleanValue(entry.getValue(), name));
             } else if (name.equals("settings")) {
                 if (!(entry.getValue() instanceof Map)) {
                     throw new IllegalArgumentException("malformed settings section");
                 }
                 settings((Map<String, Object>) entry.getValue());
             } else if (name.equals("include_global_state")) {
-                includeGlobalState = lenientNodeBooleanValue(entry.getValue());
+                includeGlobalState = lenientNodeBooleanValue(entry.getValue(), name);
             } else if (name.equals("include_aliases")) {
-                includeAliases = lenientNodeBooleanValue(entry.getValue());
+                includeAliases = lenientNodeBooleanValue(entry.getValue(), name);
             } else if (name.equals("rename_pattern")) {
                 if (entry.getValue() instanceof String) {
                     renamePattern((String) entry.getValue());
@@ -543,74 +552,6 @@ public RestoreSnapshotRequest source(Map source) {
         return this;
     }
 
-    /**
-     * Parses restore definition
-     * <p>
-     * JSON, YAML and properties formats are supported
-     *
-     * @param source restore definition
-     * @return this request
-     */
-    public RestoreSnapshotRequest source(String source) {
-        if (hasLength(source)) {
-            try (XContentParser parser = XContentFactory.xContent(source).createParser(source)) {
-                return source(parser.mapOrdered());
-            } catch (Exception e) {
-                throw new IllegalArgumentException("failed to parse repository source [" + source + "]", e);
-            }
-        }
-        return this;
-    }
-
-    /**
-     * Parses restore definition
-     * <p>
-     * JSON, YAML and properties formats are supported
-     *
-     * @param source restore definition
-     * @return this request
-     */
-    public RestoreSnapshotRequest source(byte[] source) {
-        return source(source, 0, source.length);
-    }
-
-    /**
-     * Parses restore definition
-     * <p>
-     * JSON, YAML and properties formats are supported
-     *
-     * @param source restore definition
-     * @param offset offset
-     * @param length length
-     * @return this request
-     */
-    public RestoreSnapshotRequest source(byte[] source, int offset, int length) {
-        if (length > 0) {
-            try (XContentParser parser = XContentFactory.xContent(source, offset, length).createParser(source, offset, length)) {
-                return source(parser.mapOrdered());
-            } catch (IOException e) {
-                throw new IllegalArgumentException("failed to parse repository source", e);
-            }
-        }
-        return this;
-    }
-
-    /**
-     * Parses restore definition
-     * <p>
-     * JSON, YAML and properties formats are supported
-     *
-     * @param source restore definition
-     * @return this request
-     */
-    public RestoreSnapshotRequest source(BytesReference source) {
-        try (XContentParser parser = XContentFactory.xContent(source).createParser(source)) {
-            return source(parser.mapOrdered());
-        } catch (IOException e) {
-            throw new IllegalArgumentException("failed to parse template source", e);
-        }
-    }
-
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
@@ -646,4 +587,10 @@ public void writeTo(StreamOutput out) throws IOException {
         writeSettingsToStream(indexSettings, out);
         out.writeStringArray(ignoreIndexSettings);
     }
+
+    @Override
+    public String getDescription() {
+        return "snapshot [" + repository + ":" + snapshot + "]";
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java
index 661a1a1d018af..807e238724364 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 
 import java.util.List;
 import java.util.Map;
@@ -153,18 +154,34 @@ public RestoreSnapshotRequestBuilder setSettings(Settings.Builder settings) {
     }
 
     /**
-     * Sets repository-specific restore settings in JSON, YAML or properties format
+     * Sets repository-specific restore settings in JSON or YAML format
      * <p>
      * See repository documentation for more information.
      *
      * @param source repository-specific snapshot settings
      * @return this builder
+     * @deprecated use {@link #setSettings(String, XContentType)} to avoid content type detection
      */
+    @Deprecated
     public RestoreSnapshotRequestBuilder setSettings(String source) {
         request.settings(source);
         return this;
     }
 
+    /**
+     * Sets repository-specific restore settings in JSON or YAML format
+     * <p>
+     * See repository documentation for more information.
+     *
+     * @param source repository-specific snapshot settings
+     * @param xContentType the content type of the source
+     * @return this builder
+     */
+    public RestoreSnapshotRequestBuilder setSettings(String source, XContentType xContentType) {
+        request.settings(source, xContentType);
+        return this;
+    }
+
     /**
      * Sets repository-specific restore settings
      * <p>
@@ -251,12 +268,26 @@ public RestoreSnapshotRequestBuilder setIndexSettings(Settings.Builder settings)
      *
      * @param source index settings
      * @return this builder
+     * @deprecated use {@link #setIndexSettings(String, XContentType)} to avoid content type detection
      */
+    @Deprecated
     public RestoreSnapshotRequestBuilder setIndexSettings(String source) {
         request.indexSettings(source);
         return this;
     }
 
+    /**
+     * Sets index settings that should be added or replaced during restore
+     *
+     * @param source index settings
+     * @param xContentType the content type of the source
+     * @return this builder
+     */
+    public RestoreSnapshotRequestBuilder setIndexSettings(String source, XContentType xContentType) {
+        request.indexSettings(source, xContentType);
+        return this;
+    }
+
     /**
      * Sets index settings that should be added or replaced during restore
      *
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotResponse.java
index 70f4f2aa4f24c..5a02e4bcb1387 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotResponse.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.snapshots.RestoreInfo;
@@ -33,7 +34,7 @@
 /**
  * Contains information about restores snapshot
  */
-public class RestoreSnapshotResponse extends ActionResponse implements ToXContent {
+public class RestoreSnapshotResponse extends ActionResponse implements ToXContentObject {
 
     @Nullable
     private RestoreInfo restoreInfo;
@@ -75,12 +76,14 @@ public RestStatus status() {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
+        builder.startObject();
         if (restoreInfo != null) {
             builder.field("snapshot");
             restoreInfo.toXContent(builder, params);
         } else {
             builder.field("accepted", true);
         }
+        builder.endObject();
         return builder;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java
index 070db6c5248d8..5ff5bd17fe5e6 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java
@@ -22,19 +22,27 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
+import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateListener;
+import org.elasticsearch.cluster.RestoreInProgress;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.snapshots.RestoreInfo;
 import org.elasticsearch.snapshots.RestoreService;
+import org.elasticsearch.snapshots.RestoreService.RestoreCompletionResponse;
 import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import static org.elasticsearch.snapshots.RestoreService.restoreInProgress;
+
 /**
  * Transport action for restore snapshot operation
  */
@@ -78,28 +86,44 @@ protected void masterOperation(final RestoreSnapshotRequest request, final Clust
                 request.settings(), request.masterNodeTimeout(), request.includeGlobalState(), request.partial(), request.includeAliases(),
                 request.indexSettings(), request.ignoreIndexSettings(), "restore_snapshot[" + request.snapshot() + "]");
 
-        restoreService.restoreSnapshot(restoreRequest, new ActionListener<RestoreInfo>() {
+        restoreService.restoreSnapshot(restoreRequest, new ActionListener<RestoreCompletionResponse>() {
             @Override
-            public void onResponse(RestoreInfo restoreInfo) {
-                if (restoreInfo == null && request.waitForCompletion()) {
-                    restoreService.addListener(new ActionListener<RestoreService.RestoreCompletionResponse>() {
+            public void onResponse(RestoreCompletionResponse restoreCompletionResponse) {
+                if (restoreCompletionResponse.getRestoreInfo() == null && request.waitForCompletion()) {
+                    final Snapshot snapshot = restoreCompletionResponse.getSnapshot();
+
+                    ClusterStateListener clusterStateListener = new ClusterStateListener() {
                         @Override
-                        public void onResponse(RestoreService.RestoreCompletionResponse restoreCompletionResponse) {
-                            final Snapshot snapshot = restoreCompletionResponse.getSnapshot();
-                            if (snapshot.getRepository().equals(request.repository()) &&
-                                    snapshot.getSnapshotId().getName().equals(request.snapshot())) {
-                                listener.onResponse(new RestoreSnapshotResponse(restoreCompletionResponse.getRestoreInfo()));
-                                restoreService.removeListener(this);
+                        public void clusterChanged(ClusterChangedEvent changedEvent) {
+                            final RestoreInProgress.Entry prevEntry = restoreInProgress(changedEvent.previousState(), snapshot);
+                            final RestoreInProgress.Entry newEntry = restoreInProgress(changedEvent.state(), snapshot);
+                            if (prevEntry == null) {
+                                // When there is a master failure after a restore has been started, this listener might not be registered
+                                // on the current master and as such it might miss some intermediary cluster states due to batching.
+                                // Clean up listener in that case and acknowledge completion of restore operation to client.
+                                clusterService.removeListener(this);
+                                listener.onResponse(new RestoreSnapshotResponse(null));
+                            } else if (newEntry == null) {
+                                clusterService.removeListener(this);
+                                ImmutableOpenMap<ShardId, RestoreInProgress.ShardRestoreStatus> shards = prevEntry.shards();
+                                assert prevEntry.state().completed() : "expected completed snapshot state but was " + prevEntry.state();
+                                assert RestoreService.completed(shards) : "expected all restore entries to be completed";
+                                RestoreInfo ri = new RestoreInfo(prevEntry.snapshot().getSnapshotId().getName(),
+                                    prevEntry.indices(),
+                                    shards.size(),
+                                    shards.size() - RestoreService.failedShards(shards));
+                                RestoreSnapshotResponse response = new RestoreSnapshotResponse(ri);
+                                logger.debug("restore of [{}] completed", snapshot);
+                                listener.onResponse(response);
+                            } else {
+                                // restore not completed yet, wait for next cluster state update
                             }
                         }
+                    };
 
-                        @Override
-                        public void onFailure(Exception e) {
-                            listener.onFailure(e);
-                        }
-                    });
+                    clusterService.addListener(clusterStateListener);
                 } else {
-                    listener.onResponse(new RestoreSnapshotResponse(restoreInfo));
+                    listener.onResponse(new RestoreSnapshotResponse(restoreCompletionResponse.getRestoreInfo()));
                 }
             }
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStage.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStage.java
index efbc82c9b6a83..a01fec42304cc 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStage.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStage.java
@@ -49,7 +49,7 @@ public enum SnapshotIndexShardStage {
 
     private boolean completed;
 
-    private SnapshotIndexShardStage(byte value, boolean completed) {
+    SnapshotIndexShardStage(byte value, boolean completed) {
         this.value = value;
         this.completed = completed;
     }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusResponse.java
index b9800a2d9edb8..d44a490680c9b 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusResponse.java
@@ -22,7 +22,7 @@
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
 import java.io.IOException;
@@ -33,7 +33,7 @@
 /**
  * Snapshot status response
  */
-public class SnapshotsStatusResponse extends ActionResponse implements ToXContent {
+public class SnapshotsStatusResponse extends ActionResponse implements ToXContentObject {
 
     private List<SnapshotStatus> snapshots = Collections.emptyList();
 
@@ -75,11 +75,13 @@ public void writeTo(StreamOutput out) throws IOException {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
         builder.startArray("snapshots");
         for (SnapshotStatus snapshot : snapshots) {
             snapshot.toXContent(builder, params);
         }
         builder.endArray();
+        builder.endObject();
         return builder;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java
index 71a709f0b5b40..872793f6ef21a 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java
@@ -122,11 +122,6 @@ protected NodeSnapshotStatus nodeOperation(NodeRequest request) {
         }
     }
 
-    @Override
-    protected boolean accumulateExceptions() {
-        return true;
-    }
-
     public static class Request extends BaseNodesRequest<Request> {
 
         private Snapshot[] snapshots;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java
index cf00784dc3f09..ba8132e8a2e94 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java
@@ -36,7 +36,9 @@
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus;
+import org.elasticsearch.repositories.RepositoryData;
 import org.elasticsearch.snapshots.Snapshot;
+import org.elasticsearch.snapshots.SnapshotException;
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.snapshots.SnapshotInfo;
 import org.elasticsearch.snapshots.SnapshotMissingException;
@@ -203,7 +205,8 @@ private SnapshotsStatusResponse buildResponse(SnapshotsStatusRequest request, Li
         final String repositoryName = request.repository();
         if (Strings.hasText(repositoryName) && request.snapshots() != null && request.snapshots().length > 0) {
             final Set<String> requestedSnapshotNames = Sets.newHashSet(request.snapshots());
-            final Map<String, SnapshotId> matchedSnapshotIds = snapshotsService.snapshotIds(repositoryName).stream()
+            final RepositoryData repositoryData = snapshotsService.getRepositoryData(repositoryName);
+            final Map<String, SnapshotId> matchedSnapshotIds = repositoryData.getAllSnapshotIds().stream()
                 .filter(s -> requestedSnapshotNames.contains(s.getName()))
                 .collect(Collectors.toMap(SnapshotId::getName, Function.identity()));
             for (final String snapshotName : request.snapshots()) {
@@ -222,6 +225,8 @@ private SnapshotsStatusResponse buildResponse(SnapshotsStatusRequest request, Li
                     } else {
                         throw new SnapshotMissingException(repositoryName, snapshotName);
                     }
+                } else if (repositoryData.getIncompatibleSnapshotIds().contains(snapshotId)) {
+                    throw new SnapshotException(repositoryName, snapshotName, "cannot get the status for an incompatible snapshot");
                 }
                 SnapshotInfo snapshotInfo = snapshotsService.snapshot(repositoryName, snapshotId);
                 List<SnapshotIndexShardStatus> shardStatusBuilder = new ArrayList<>();
@@ -245,7 +250,7 @@ private SnapshotsStatusResponse buildResponse(SnapshotsStatusRequest request, Li
                         default:
                             throw new IllegalArgumentException("Unknown snapshot state " + snapshotInfo.state());
                     }
-                    builder.add(new SnapshotStatus(new Snapshot(repositoryName, snapshotInfo.snapshotId()), state, Collections.unmodifiableList(shardStatusBuilder)));
+                    builder.add(new SnapshotStatus(new Snapshot(repositoryName, snapshotId), state, Collections.unmodifiableList(shardStatusBuilder)));
                 }
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateResponse.java
index 2a2f4707f69ba..7c4844ae5f090 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateResponse.java
@@ -55,7 +55,7 @@ public ClusterName getClusterName() {
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
         clusterName = new ClusterName(in);
-        clusterState = ClusterState.Builder.readFrom(in, null);
+        clusterState = ClusterState.readFrom(in, null);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java
index 3a6168315e03e..6347a027f4f89 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java
@@ -86,8 +86,8 @@ public void readFrom(StreamInput in) throws IOException {
         this.nodeStats = NodeStats.readNodeStats(in);
         int size = in.readVInt();
         shardsStats = new ShardStats[size];
-        for (size--; size >= 0; size--) {
-            shardsStats[size] = ShardStats.readShardStats(in);
+        for (int i = 0; i < size; i++) {
+            shardsStats[i] = ShardStats.readShardStats(in);
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java
index 62255df369bd9..d5a2912f549cf 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
 import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.InetSocketTransportAddress;
@@ -65,8 +66,8 @@ public class ClusterStatsNodes implements ToXContent {
         this.plugins = new HashSet<>();
 
         Set<InetAddress> seenAddresses = new HashSet<>(nodeResponses.size());
-        List<NodeInfo> nodeInfos = new ArrayList<>();
-        List<NodeStats> nodeStats = new ArrayList<>();
+        List<NodeInfo> nodeInfos = new ArrayList<>(nodeResponses.size());
+        List<NodeStats> nodeStats = new ArrayList<>(nodeResponses.size());
         for (ClusterStatsNodeResponse nodeResponse : nodeResponses) {
             nodeInfos.add(nodeResponse.nodeInfo());
             nodeStats.add(nodeResponse.nodeStats());
@@ -250,11 +251,11 @@ private OsStats(List<NodeInfo> nodeInfos, List<NodeStats> nodeStatsList) {
             long freeMemory = 0;
             for (NodeStats nodeStats : nodeStatsList) {
                 if (nodeStats.getOs() != null) {
-                    long total = nodeStats.getOs().getMem().getTotal().bytes();
+                    long total = nodeStats.getOs().getMem().getTotal().getBytes();
                     if (total > 0) {
                         totalMemory += total;
                     }
-                    long free = nodeStats.getOs().getMem().getFree().bytes();
+                    long free = nodeStats.getOs().getMem().getFree().getBytes();
                     if (free > 0) {
                         freeMemory += free;
                     }
@@ -423,8 +424,8 @@ private JvmStats(List<NodeInfo> nodeInfos, List<NodeStats> nodeStatsList) {
                 }
                 maxUptime = Math.max(maxUptime, js.getUptime().millis());
                 if (js.getMem() != null) {
-                    heapUsed += js.getMem().getHeapUsed().bytes();
-                    heapMax += js.getMem().getHeapMax().bytes();
+                    heapUsed += js.getMem().getHeapUsed().getBytes();
+                    heapMax += js.getMem().getHeapMax().getBytes();
                 }
             }
             this.threads = threads;
@@ -544,7 +545,7 @@ static class NetworkTypes implements ToXContent {
         private final Map<String, AtomicInteger> transportTypes;
         private final Map<String, AtomicInteger> httpTypes;
 
-        private NetworkTypes(final List<NodeInfo> nodeInfos) {
+        NetworkTypes(final List<NodeInfo> nodeInfos) {
             final Map<String, AtomicInteger> transportTypes = new HashMap<>();
             final Map<String, AtomicInteger> httpTypes = new HashMap<>();
             for (final NodeInfo nodeInfo : nodeInfos) {
@@ -553,8 +554,12 @@ private NetworkTypes(final List<NodeInfo> nodeInfos) {
                     settings.get(NetworkModule.TRANSPORT_TYPE_KEY, NetworkModule.TRANSPORT_DEFAULT_TYPE_SETTING.get(settings));
                 final String httpType =
                     settings.get(NetworkModule.HTTP_TYPE_KEY, NetworkModule.HTTP_DEFAULT_TYPE_SETTING.get(settings));
-                transportTypes.computeIfAbsent(transportType, k -> new AtomicInteger()).incrementAndGet();
-                httpTypes.computeIfAbsent(httpType, k -> new AtomicInteger()).incrementAndGet();
+                if (Strings.hasText(transportType)) {
+                    transportTypes.computeIfAbsent(transportType, k -> new AtomicInteger()).incrementAndGet();
+                }
+                if (Strings.hasText(httpType)) {
+                    httpTypes.computeIfAbsent(httpType, k -> new AtomicInteger()).incrementAndGet();
+                }
             }
             this.transportTypes = Collections.unmodifiableMap(transportTypes);
             this.httpTypes = Collections.unmodifiableMap(httpTypes);
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java
index efc72d104f86e..40c71c098c897 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java
@@ -40,19 +40,18 @@ public class ClusterStatsResponse extends BaseNodesResponse<ClusterStatsNodeResp
 
     ClusterStatsNodes nodesStats;
     ClusterStatsIndices indicesStats;
-    String clusterUUID;
     ClusterHealthStatus status;
     long timestamp;
 
-
     ClusterStatsResponse() {
     }
 
-    public ClusterStatsResponse(long timestamp, ClusterName clusterName, String clusterUUID,
-                                List<ClusterStatsNodeResponse> nodes, List<FailedNodeException> failures) {
+    public ClusterStatsResponse(long timestamp,
+                                ClusterName clusterName,
+                                List<ClusterStatsNodeResponse> nodes,
+                                List<FailedNodeException> failures) {
         super(clusterName, nodes, failures);
         this.timestamp = timestamp;
-        this.clusterUUID = clusterUUID;
         nodesStats = new ClusterStatsNodes(nodes);
         indicesStats = new ClusterStatsIndices(nodes);
         for (ClusterStatsNodeResponse response : nodes) {
@@ -84,7 +83,6 @@ public ClusterStatsIndices getIndicesStats() {
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
         timestamp = in.readVLong();
-        clusterUUID = in.readString();
         // it may be that the master switched on us while doing the operation. In this case the status may be null.
         status = in.readOptionalWriteable(ClusterHealthStatus::readFrom);
     }
@@ -93,7 +91,6 @@ public void readFrom(StreamInput in) throws IOException {
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         out.writeVLong(timestamp);
-        out.writeString(clusterUUID);
         out.writeOptionalWriteable(status);
     }
 
@@ -117,9 +114,6 @@ protected void writeNodesTo(StreamOutput out, List<ClusterStatsNodeResponse> nod
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.field("timestamp", getTimestamp());
-        if (params.paramAsBoolean("output_uuid", false)) {
-            builder.field("uuid", clusterUUID);
-        }
         if (status != null) {
             builder.field("status", status.name().toLowerCase(Locale.ROOT));
         }
@@ -144,4 +138,5 @@ public String toString() {
             return "{ \"error\" : \"" + e.getMessage() + "\"}";
         }
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java
index 3eb73273832e0..d5659c48723d6 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java
@@ -39,7 +39,7 @@
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.indices.IndicesService;
-import org.elasticsearch.node.service.NodeService;
+import org.elasticsearch.node.NodeService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -75,8 +75,11 @@ public TransportClusterStatsAction(Settings settings, ThreadPool threadPool,
     @Override
     protected ClusterStatsResponse newResponse(ClusterStatsRequest request,
                                                List<ClusterStatsNodeResponse> responses, List<FailedNodeException> failures) {
-        return new ClusterStatsResponse(System.currentTimeMillis(), clusterService.getClusterName(),
-            clusterService.state().metaData().clusterUUID(), responses, failures);
+        return new ClusterStatsResponse(
+            System.currentTimeMillis(),
+            clusterService.getClusterName(),
+            responses,
+            failures);
     }
 
     @Override
@@ -112,11 +115,6 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq
 
     }
 
-    @Override
-    protected boolean accumulateExceptions() {
-        return false;
-    }
-
     public static class ClusterStatsNodeRequest extends BaseNodeRequest {
 
         ClusterStatsRequest request;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/DeleteStoredScriptRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/DeleteStoredScriptRequest.java
index d01c128cf36b3..c30eae12a82ea 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/DeleteStoredScriptRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/DeleteStoredScriptRequest.java
@@ -31,66 +31,79 @@
 public class DeleteStoredScriptRequest extends AcknowledgedRequest<DeleteStoredScriptRequest> {
 
     private String id;
-    private String scriptLang;
+    private String lang;
 
     DeleteStoredScriptRequest() {
+        super();
     }
 
-    public DeleteStoredScriptRequest(String scriptLang, String id) {
-        this.scriptLang = scriptLang;
+    public DeleteStoredScriptRequest(String id, String lang) {
+        super();
+
         this.id = id;
+        this.lang = lang;
     }
 
     @Override
     public ActionRequestValidationException validate() {
         ActionRequestValidationException validationException = null;
-        if (id == null) {
-            validationException = addValidationError("id is missing", validationException);
+
+        if (id == null || id.isEmpty()) {
+            validationException = addValidationError("must specify id for stored script", validationException);
         } else if (id.contains("#")) {
-            validationException = addValidationError("id can't contain: '#'", validationException);
+            validationException = addValidationError("id cannot contain '#' for stored script", validationException);
         }
-        if (scriptLang == null) {
-            validationException = addValidationError("lang is missing", validationException);
-        } else if (scriptLang.contains("#")) {
-            validationException = addValidationError("lang can't contain: '#'", validationException);
+
+        if (lang != null && lang.contains("#")) {
+            validationException = addValidationError("lang cannot contain '#' for stored script", validationException);
         }
+
         return validationException;
     }
 
-    public String scriptLang() {
-        return scriptLang;
+    public String id() {
+        return id;
     }
 
-    public DeleteStoredScriptRequest scriptLang(String type) {
-        this.scriptLang = type;
+    public DeleteStoredScriptRequest id(String id) {
+        this.id = id;
+
         return this;
     }
 
-    public String id() {
-        return id;
+    public String lang() {
+        return lang;
     }
 
-    public DeleteStoredScriptRequest id(String id) {
-        this.id = id;
+    public DeleteStoredScriptRequest lang(String lang) {
+        this.lang = lang;
+
         return this;
     }
 
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
-        scriptLang = in.readString();
+
+        lang = in.readString();
+
+        if (lang.isEmpty()) {
+            lang = null;
+        }
+
         id = in.readString();
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        out.writeString(scriptLang);
+
+        out.writeString(lang == null ? "" : lang);
         out.writeString(id);
     }
 
     @Override
     public String toString() {
-        return "delete script {[" + scriptLang + "][" + id + "]}";
+        return "delete stored script {id [" + id + "]" + (lang != null ? ", lang [" + lang + "]" : "") + "}";
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/DeleteStoredScriptRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/DeleteStoredScriptRequestBuilder.java
index caf55a03f18e3..8a65506dabd34 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/DeleteStoredScriptRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/DeleteStoredScriptRequestBuilder.java
@@ -29,13 +29,15 @@ public DeleteStoredScriptRequestBuilder(ElasticsearchClient client, DeleteStored
         super(client, action, new DeleteStoredScriptRequest());
     }
 
-    public DeleteStoredScriptRequestBuilder setScriptLang(String scriptLang) {
-        request.scriptLang(scriptLang);
+    public DeleteStoredScriptRequestBuilder setLang(String lang) {
+        request.lang(lang);
+
         return this;
     }
 
     public DeleteStoredScriptRequestBuilder setId(String id) {
         request.id(id);
+
         return this;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptRequest.java
index bb7a9effd32eb..2bfd547362c80 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptRequest.java
@@ -28,61 +28,79 @@
 
 import java.io.IOException;
 
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
 public class GetStoredScriptRequest extends MasterNodeReadRequest<GetStoredScriptRequest> {
 
     protected String id;
     protected String lang;
 
     GetStoredScriptRequest() {
+        super();
     }
 
-    public GetStoredScriptRequest(String lang, String id) {
-        this.lang = lang;
+    public GetStoredScriptRequest(String id, String lang) {
+        super();
+
         this.id = id;
+        this.lang = lang;
     }
 
     @Override
     public ActionRequestValidationException validate() {
         ActionRequestValidationException validationException = null;
-        if (lang == null) {
-            validationException = ValidateActions.addValidationError("lang is missing", validationException);
+
+        if (id == null || id.isEmpty()) {
+            validationException = addValidationError("must specify id for stored script", validationException);
+        } else if (id.contains("#")) {
+            validationException = addValidationError("id cannot contain '#' for stored script", validationException);
         }
-        if (id == null) {
-            validationException = ValidateActions.addValidationError("id is missing", validationException);
+
+        if (lang != null && lang.contains("#")) {
+            validationException = addValidationError("lang cannot contain '#' for stored script", validationException);
         }
+
         return validationException;
     }
 
-    public GetStoredScriptRequest lang(@Nullable String type) {
-        this.lang = type;
-        return this;
+    public String id() {
+        return id;
     }
 
     public GetStoredScriptRequest id(String id) {
         this.id = id;
+
         return this;
     }
 
-
     public String lang() {
         return lang;
     }
 
-    public String id() {
-        return id;
+    public GetStoredScriptRequest lang(String lang) {
+        this.lang = lang;
+
+        return this;
     }
 
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
+
         lang = in.readString();
+
+        if (lang.isEmpty()) {
+            lang = null;
+        }
+
         id = in.readString();
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        out.writeString(lang);
+
+        out.writeString(lang == null ? "" : lang);
         out.writeString(id);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptResponse.java
index 36dd9beb38a7a..d543ac67e1d91 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptResponse.java
@@ -19,49 +19,70 @@
 
 package org.elasticsearch.action.admin.cluster.storedscripts;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionResponse;
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.script.Script;
+import org.elasticsearch.script.StoredScriptSource;
 
 import java.io.IOException;
 
 public class GetStoredScriptResponse extends ActionResponse implements ToXContent {
 
-    private String storedScript;
+    private StoredScriptSource source;
 
     GetStoredScriptResponse() {
     }
 
-    GetStoredScriptResponse(String storedScript) {
-        this.storedScript = storedScript;
+    GetStoredScriptResponse(StoredScriptSource source) {
+        this.source = source;
     }
 
     /**
      * @return if a stored script and if not found <code>null</code>
      */
-    public String getStoredScript() {
-        return storedScript;
+    public StoredScriptSource getSource() {
+        return source;
     }
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.value(storedScript);
+        source.toXContent(builder, params);
+
         return builder;
     }
 
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
-        storedScript = in.readOptionalString();
+
+        if (in.readBoolean()) {
+            if (in.getVersion().onOrAfter(Version.V_5_3_0)) {
+                source = new StoredScriptSource(in);
+            } else {
+                source = new StoredScriptSource(in.readString());
+            }
+        } else {
+            source = null;
+        }
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        out.writeOptionalString(storedScript);
+
+        if (source == null) {
+            out.writeBoolean(false);
+        } else {
+            out.writeBoolean(true);
+
+            if (out.getVersion().onOrAfter(Version.V_5_3_0)) {
+                source.writeTo(out);
+            } else {
+                out.writeString(source.getSource());
+            }
+        }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java
index cfe153d7d9641..1e3d251e7a284 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java
@@ -19,108 +19,153 @@
 
 package org.elasticsearch.action.admin.cluster.storedscripts;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentType;
 
 import java.io.IOException;
+import java.util.Objects;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 
 public class PutStoredScriptRequest extends AcknowledgedRequest<PutStoredScriptRequest> {
 
     private String id;
-    private String scriptLang;
-    private BytesReference script;
+    private String lang;
+    private BytesReference content;
+    private XContentType xContentType;
 
     public PutStoredScriptRequest() {
         super();
     }
 
-    public PutStoredScriptRequest(String scriptLang) {
-        super();
-        this.scriptLang = scriptLang;
+    @Deprecated
+    public PutStoredScriptRequest(String id, String lang, BytesReference content) {
+        this(id, lang, content, XContentFactory.xContentType(content));
     }
 
-    public PutStoredScriptRequest(String scriptLang, String id) {
+    public PutStoredScriptRequest(String id, String lang, BytesReference content, XContentType xContentType) {
         super();
-        this.scriptLang = scriptLang;
         this.id = id;
+        this.lang = lang;
+        this.content = content;
+        this.xContentType = Objects.requireNonNull(xContentType);
     }
 
     @Override
     public ActionRequestValidationException validate() {
         ActionRequestValidationException validationException = null;
-        if (id == null) {
-            validationException = addValidationError("id is missing", validationException);
+
+        if (id == null || id.isEmpty()) {
+            validationException = addValidationError("must specify id for stored script", validationException);
         } else if (id.contains("#")) {
-            validationException = addValidationError("id can't contain: '#'", validationException);
+            validationException = addValidationError("id cannot contain '#' for stored script", validationException);
         }
-        if (scriptLang == null) {
-            validationException = addValidationError("lang is missing", validationException);
-        } else if (scriptLang.contains("#")) {
-            validationException = addValidationError("lang can't contain: '#'", validationException);
+
+        if (lang != null && lang.contains("#")) {
+            validationException = addValidationError("lang cannot contain '#' for stored script", validationException);
         }
-        if (script == null) {
-            validationException = addValidationError("script is missing", validationException);
+
+        if (content == null) {
+            validationException = addValidationError("must specify code for stored script", validationException);
         }
+
         return validationException;
     }
 
-    public String scriptLang() {
-        return scriptLang;
+    public String id() {
+        return id;
     }
 
-    public PutStoredScriptRequest scriptLang(String scriptLang) {
-        this.scriptLang = scriptLang;
+    public PutStoredScriptRequest id(String id) {
+        this.id = id;
+
         return this;
     }
 
-    public String id() {
-        return id;
+    public String lang() {
+        return lang;
     }
 
-    public PutStoredScriptRequest id(String id) {
-        this.id = id;
+    public PutStoredScriptRequest lang(String lang) {
+        this.lang = lang;
+
         return this;
     }
 
-    public BytesReference script() {
-        return script;
+    public BytesReference content() {
+        return content;
+    }
+
+    public XContentType xContentType() {
+        return xContentType;
     }
 
-    public PutStoredScriptRequest script(BytesReference source) {
-        this.script = source;
+    /**
+     * Set the script source using bytes.
+     * @deprecated this method is deprecated as it relies on content type detection. Use {@link #content(BytesReference, XContentType)}
+     */
+    @Deprecated
+    public PutStoredScriptRequest content(BytesReference content) {
+        return content(content, XContentFactory.xContentType(content));
+    }
+
+    /**
+     * Set the script source and the content type of the bytes.
+     */
+    public PutStoredScriptRequest content(BytesReference content, XContentType xContentType) {
+        this.content = content;
+        this.xContentType = Objects.requireNonNull(xContentType);
         return this;
     }
 
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
-        scriptLang = in.readString();
+
+        lang = in.readString();
+
+        if (lang.isEmpty()) {
+            lang = null;
+        }
+
         id = in.readOptionalString();
-        script = in.readBytesReference();
+        content = in.readBytesReference();
+        if (in.getVersion().onOrAfter(Version.V_5_3_0)) {
+            xContentType = XContentType.readFrom(in);
+        } else {
+            xContentType = XContentFactory.xContentType(content);
+        }
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        out.writeString(scriptLang);
+
+        out.writeString(lang == null ? "" : lang);
         out.writeOptionalString(id);
-        out.writeBytesReference(script);
+        out.writeBytesReference(content);
+        if (out.getVersion().onOrAfter(Version.V_5_3_0)) {
+            xContentType.writeTo(out);
+        }
     }
 
     @Override
     public String toString() {
-        String sSource = "_na_";
+        String source = "_na_";
+
         try {
-            sSource = XContentHelper.convertToJson(script, false);
+            source = XContentHelper.convertToJson(content, false, xContentType);
         } catch (Exception e) {
             // ignore
         }
-        return "put script {[" + id + "][" + scriptLang + "], script[" + sSource + "]}";
+
+        return "put stored script {id [" + id + "]" + (lang != null ? ", lang [" + lang + "]" : "") + ", content [" + source + "]}";
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestBuilder.java
index 15c51c2ccd7e5..f8223d691999b 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestBuilder.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentType;
 
 public class PutStoredScriptRequestBuilder extends AcknowledgedRequestBuilder<PutStoredScriptRequest,
         PutStoredScriptResponse, PutStoredScriptRequestBuilder> {
@@ -30,19 +31,31 @@ public PutStoredScriptRequestBuilder(ElasticsearchClient client, PutStoredScript
         super(client, action, new PutStoredScriptRequest());
     }
 
-    public PutStoredScriptRequestBuilder setScriptLang(String scriptLang) {
-        request.scriptLang(scriptLang);
+    public PutStoredScriptRequestBuilder setId(String id) {
+        request.id(id);
         return this;
     }
 
-    public PutStoredScriptRequestBuilder setId(String id) {
-        request.id(id);
+    /**
+     * Set the source of the script.
+     * @deprecated this method requires content type detection. Use {@link #setContent(BytesReference, XContentType)} instead
+     */
+    @Deprecated
+    public PutStoredScriptRequestBuilder setContent(BytesReference content) {
+        request.content(content);
         return this;
     }
 
-    public PutStoredScriptRequestBuilder setSource(BytesReference source) {
-        request.script(source);
+    /**
+     * Set the source of the script along with the content type of the source
+     */
+    public PutStoredScriptRequestBuilder setContent(BytesReference source, XContentType xContentType) {
+        request.content(source, xContentType);
         return this;
     }
 
+    public PutStoredScriptRequestBuilder setLang(String lang) {
+        request.lang(lang);
+        return this;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportPutStoredScriptAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportPutStoredScriptAction.java
index a91dec8d9b30a..8b4079aee7379 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportPutStoredScriptAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportPutStoredScriptAction.java
@@ -59,7 +59,7 @@ protected PutStoredScriptResponse newResponse() {
     @Override
     protected void masterOperation(PutStoredScriptRequest request, ClusterState state,
                                    ActionListener<PutStoredScriptResponse> listener) throws Exception {
-        scriptService.storeScript(clusterService, request, listener);
+        scriptService.putStoredScript(clusterService, request, listener);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/tasks/PendingClusterTasksResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/tasks/PendingClusterTasksResponse.java
index 35d5b3efb7b7d..ec42e34ec9614 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/tasks/PendingClusterTasksResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/tasks/PendingClusterTasksResponse.java
@@ -23,18 +23,15 @@
 import org.elasticsearch.cluster.service.PendingClusterTask;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
 
-/**
- */
-public class PendingClusterTasksResponse extends ActionResponse implements Iterable<PendingClusterTask>, ToXContent {
+public class PendingClusterTasksResponse extends ActionResponse implements Iterable<PendingClusterTask>, ToXContentObject {
 
     private List<PendingClusterTask> pendingTasks;
 
@@ -61,30 +58,20 @@ public Iterator<PendingClusterTask> iterator() {
         return pendingTasks.iterator();
     }
 
-    public String prettyPrint() {
+    @Override
+    public String toString() {
         StringBuilder sb = new StringBuilder();
         sb.append("tasks: (").append(pendingTasks.size()).append("):\n");
         for (PendingClusterTask pendingClusterTask : this) {
-            sb.append(pendingClusterTask.getInsertOrder()).append("/").append(pendingClusterTask.getPriority()).append("/").append(pendingClusterTask.getSource()).append("/").append(pendingClusterTask.getTimeInQueue()).append("\n");
+            sb.append(pendingClusterTask.getInsertOrder()).append("/").append(pendingClusterTask.getPriority()).append("/")
+                    .append(pendingClusterTask.getSource()).append("/").append(pendingClusterTask.getTimeInQueue()).append("\n");
         }
         return sb.toString();
     }
 
-    @Override
-    public String toString() {
-        try {
-            XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint();
-            builder.startObject();
-            toXContent(builder, EMPTY_PARAMS);
-            builder.endObject();
-            return builder.string();
-        } catch (IOException e) {
-            return "{ \"error\" : \"" + e.getMessage() + "\"}";
-        }
-    }
-
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
         builder.startArray(Fields.TASKS);
         for (PendingClusterTask pendingClusterTask : this) {
             builder.startObject();
@@ -97,6 +84,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.endObject();
         }
         builder.endArray();
+        builder.endObject();
         return builder;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java
index 63493210f7c27..e7393efd01ce7 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java
@@ -19,23 +19,15 @@
 
 package org.elasticsearch.action.admin.indices.alias;
 
-import com.carrotsearch.hppc.cursors.ObjectCursor;
-
 import org.elasticsearch.ElasticsearchGenerationException;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.AliasesRequest;
-import org.elasticsearch.action.CompositeIndicesRequest;
-import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.cluster.metadata.AliasAction;
-import org.elasticsearch.cluster.metadata.AliasMetaData;
-import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -45,6 +37,7 @@
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.QueryBuilder;
 
@@ -63,7 +56,7 @@
 /**
  * A request to add/remove aliases for one or more indices.
  */
-public class IndicesAliasesRequest extends AcknowledgedRequest<IndicesAliasesRequest> implements CompositeIndicesRequest {
+public class IndicesAliasesRequest extends AcknowledgedRequest<IndicesAliasesRequest> {
     private List<AliasActions> allAliasActions = new ArrayList<>();
 
     //indices options that require every specified index to exist, expand wildcards only to open indices and
@@ -95,10 +88,10 @@ public byte value() {
 
             public static Type fromValue(byte value) {
                 switch (value) {
-                case 0: return ADD;
-                case 1: return REMOVE;
-                case 2: return REMOVE_INDEX;
-                default: throw new IllegalArgumentException("No type for action [" + value + "]");
+                    case 0: return ADD;
+                    case 1: return REMOVE;
+                    case 2: return REMOVE_INDEX;
+                    default: throw new IllegalArgumentException("No type for action [" + value + "]");
                 }
             }
         }
@@ -109,20 +102,23 @@ public static Type fromValue(byte value) {
         public static AliasActions add() {
             return new AliasActions(AliasActions.Type.ADD);
         }
+
         /**
          * Build a new {@link AliasAction} to remove aliases.
          */
         public static AliasActions remove() {
             return new AliasActions(AliasActions.Type.REMOVE);
         }
+
         /**
-         * Build a new {@link AliasAction} to remove aliases.
+         * Build a new {@link AliasAction} to remove an index.
          */
         public static AliasActions removeIndex() {
             return new AliasActions(AliasActions.Type.REMOVE_INDEX);
         }
-        private static ObjectParser<AliasActions, ParseFieldMatcherSupplier> parser(String name, Supplier<AliasActions> supplier) {
-            ObjectParser<AliasActions, ParseFieldMatcherSupplier> parser = new ObjectParser<>(name, supplier);
+
+        private static ObjectParser<AliasActions, Void> parser(String name, Supplier<AliasActions> supplier) {
+            ObjectParser<AliasActions, Void> parser = new ObjectParser<>(name, supplier);
             parser.declareString((action, index) -> {
                 if (action.indices() != null) {
                     throw new IllegalArgumentException("Only one of [index] and [indices] is supported");
@@ -150,7 +146,7 @@ private static ObjectParser<AliasActions, ParseFieldMatcherSupplier> parser(Stri
             return parser;
         }
 
-        private static final ObjectParser<AliasActions, ParseFieldMatcherSupplier> ADD_PARSER = parser("add", AliasActions::add);
+        private static final ObjectParser<AliasActions, Void> ADD_PARSER = parser("add", AliasActions::add);
         static {
             ADD_PARSER.declareObject(AliasActions::filter, (parser, m) -> {
                 try {
@@ -160,18 +156,17 @@ private static ObjectParser<AliasActions, ParseFieldMatcherSupplier> parser(Stri
                 }
             }, new ParseField("filter"));
             // Since we need to support numbers AND strings here we have to use ValueType.INT.
-            ADD_PARSER.declareField(AliasActions::routing, p -> p.text(), new ParseField("routing"), ValueType.INT);
-            ADD_PARSER.declareField(AliasActions::indexRouting, p -> p.text(), new ParseField("index_routing"), ValueType.INT);
-            ADD_PARSER.declareField(AliasActions::searchRouting, p -> p.text(), new ParseField("search_routing"), ValueType.INT);
+            ADD_PARSER.declareField(AliasActions::routing, XContentParser::text, new ParseField("routing"), ValueType.INT);
+            ADD_PARSER.declareField(AliasActions::indexRouting, XContentParser::text, new ParseField("index_routing"), ValueType.INT);
+            ADD_PARSER.declareField(AliasActions::searchRouting, XContentParser::text, new ParseField("search_routing"), ValueType.INT);
         }
-        private static final ObjectParser<AliasActions, ParseFieldMatcherSupplier> REMOVE_PARSER = parser("remove", AliasActions::remove);
-        private static final ObjectParser<AliasActions, ParseFieldMatcherSupplier> REMOVE_INDEX_PARSER = parser("remove_index",
-                AliasActions::removeIndex);
+        private static final ObjectParser<AliasActions, Void> REMOVE_PARSER = parser("remove", AliasActions::remove);
+        private static final ObjectParser<AliasActions, Void> REMOVE_INDEX_PARSER = parser("remove_index", AliasActions::removeIndex);
 
         /**
          * Parser for any one {@link AliasAction}.
          */
-        public static final ConstructingObjectParser<AliasActions, ParseFieldMatcherSupplier> PARSER = new ConstructingObjectParser<>(
+        public static final ConstructingObjectParser<AliasActions, Void> PARSER = new ConstructingObjectParser<>(
                 "alias_action", a -> {
                     // Take the first action and complain if there are more than one actions
                     AliasActions action = null;
@@ -406,24 +401,6 @@ public IndicesOptions indicesOptions() {
             return INDICES_OPTIONS;
         }
 
-        public String[] concreteAliases(MetaData metaData, String concreteIndex) {
-            if (expandAliasesWildcards()) {
-                //for DELETE we expand the aliases
-                String[] indexAsArray = {concreteIndex};
-                ImmutableOpenMap<String, List<AliasMetaData>> aliasMetaData = metaData.findAliases(aliases, indexAsArray);
-                List<String> finalAliases = new ArrayList<>();
-                for (ObjectCursor<List<AliasMetaData>> curAliases : aliasMetaData.values()) {
-                    for (AliasMetaData aliasMeta: curAliases.value) {
-                        finalAliases.add(aliasMeta.alias());
-                    }
-                }
-                return finalAliases.toArray(new String[finalAliases.size()]);
-            } else {
-                //for add we just return the current aliases
-                return aliases;
-            }
-        }
-
         @Override
         public String toString() {
             return "AliasActions["
@@ -502,9 +479,4 @@ public void writeTo(StreamOutput out) throws IOException {
     public IndicesOptions indicesOptions() {
         return INDICES_OPTIONS;
     }
-
-    @Override
-    public List<? extends IndicesRequest> subRequests() {
-        return allAliasActions;
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java
index 44de63c028db1..9dcd361ae6421 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.action.admin.indices.alias;
 
+import com.carrotsearch.hppc.cursors.ObjectCursor;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions;
 import org.elasticsearch.action.support.ActionFilters;
@@ -28,9 +29,12 @@
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.AliasAction;
+import org.elasticsearch.cluster.metadata.AliasMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.metadata.MetaDataIndexAliasesService;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.action.admin.indices.AliasesNotFoundException;
@@ -75,9 +79,7 @@ protected IndicesAliasesResponse newResponse() {
     protected ClusterBlockException checkBlock(IndicesAliasesRequest request, ClusterState state) {
         Set<String> indices = new HashSet<>();
         for (AliasActions aliasAction : request.aliasActions()) {
-            for (String index : aliasAction.indices()) {
-                indices.add(index);
-            }
+            Collections.addAll(indices, aliasAction.indices());
         }
         return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_WRITE, indices.toArray(new String[indices.size()]));
     }
@@ -97,12 +99,12 @@ protected void masterOperation(final IndicesAliasesRequest request, final Cluste
             for (String index : concreteIndices) {
                 switch (action.actionType()) {
                 case ADD:
-                    for (String alias : action.concreteAliases(state.metaData(), index)) {
+                    for (String alias : concreteAliases(action, state.metaData(), index)) {
                         finalActions.add(new AliasAction.Add(index, alias, action.filter(), action.indexRouting(), action.searchRouting()));
                     }
                     break;
                 case REMOVE:
-                    for (String alias : action.concreteAliases(state.metaData(), index)) {
+                    for (String alias : concreteAliases(action, state.metaData(), index)) {
                         finalActions.add(new AliasAction.Remove(index, alias));
                     }
                     break;
@@ -134,4 +136,22 @@ public void onFailure(Exception t) {
             }
         });
     }
+
+    private static String[] concreteAliases(AliasActions action, MetaData metaData, String concreteIndex) {
+        if (action.expandAliasesWildcards()) {
+            //for DELETE we expand the aliases
+            String[] indexAsArray = {concreteIndex};
+            ImmutableOpenMap<String, List<AliasMetaData>> aliasMetaData = metaData.findAliases(action.aliases(), indexAsArray);
+            List<String> finalAliases = new ArrayList<>();
+            for (ObjectCursor<List<AliasMetaData>> curAliases : aliasMetaData.values()) {
+                for (AliasMetaData aliasMeta: curAliases.value) {
+                    finalAliases.add(aliasMeta.alias());
+                }
+            }
+            return finalAliases.toArray(new String[finalAliases.size()]);
+        } else {
+            //for ADD and REMOVE_INDEX we just return the current aliases
+            return action.aliases();
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesResponse.java
index e23faa1cbbf9c..0c307d643246f 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesResponse.java
@@ -59,7 +59,7 @@ public void readFrom(StreamInput in) throws IOException {
             int valueSize = in.readVInt();
             List<AliasMetaData> value = new ArrayList<>(valueSize);
             for (int j = 0; j < valueSize; j++) {
-                value.add(AliasMetaData.Builder.readFrom(in));
+                value.add(new AliasMetaData(in));
             }
             aliasesBuilder.put(key, Collections.unmodifiableList(value));
         }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequest.java
index 6d0824eeb31c5..08f220e0199d8 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequest.java
@@ -75,7 +75,7 @@ public static class NameOrDefinition implements Writeable {
             try {
                 XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
                 builder.map(definition);
-                this.definition = Settings.builder().loadFromSource(builder.string()).build();
+                this.definition = Settings.builder().loadFromSource(builder.string(), builder.contentType()).build();
             } catch (IOException e) {
                 throw new IllegalArgumentException("Failed to parse [" + definition + "]", e);
             }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequestBuilder.java
index 78d06185423dd..67a7dca45b933 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequestBuilder.java
@@ -116,7 +116,7 @@ public AnalyzeRequestBuilder setExplain(boolean explain) {
     /**
      * Sets attributes that will include results
      */
-    public AnalyzeRequestBuilder setAttributes(String attributes){
+    public AnalyzeRequestBuilder setAttributes(String... attributes){
         request.attributes(attributes);
         return this;
     }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java
index 48db340a1c75e..07e14211a36ae 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java
@@ -23,7 +23,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
 import java.io.IOException;
@@ -32,28 +32,27 @@
 import java.util.List;
 import java.util.Map;
 
-/**
- *
- */
-public class AnalyzeResponse extends ActionResponse implements Iterable<AnalyzeResponse.AnalyzeToken>, ToXContent {
+public class AnalyzeResponse extends ActionResponse implements Iterable<AnalyzeResponse.AnalyzeToken>, ToXContentObject {
 
-    public static class AnalyzeToken implements Streamable, ToXContent {
+    public static class AnalyzeToken implements Streamable, ToXContentObject {
         private String term;
         private int startOffset;
         private int endOffset;
         private int position;
+        private int positionLength = 1;
         private Map<String, Object> attributes;
         private String type;
 
         AnalyzeToken() {
         }
 
-        public AnalyzeToken(String term, int position, int startOffset, int endOffset, String type,
-                            Map<String, Object> attributes) {
+        public AnalyzeToken(String term, int position, int startOffset, int endOffset, int positionLength,
+                            String type, Map<String, Object> attributes) {
             this.term = term;
             this.position = position;
             this.startOffset = startOffset;
             this.endOffset = endOffset;
+            this.positionLength = positionLength;
             this.type = type;
             this.attributes = attributes;
         }
@@ -74,6 +73,10 @@ public int getPosition() {
             return this.position;
         }
 
+        public int getPositionLength() {
+            return this.positionLength;
+        }
+
         public String getType() {
             return this.type;
         }
@@ -90,6 +93,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.field(Fields.END_OFFSET, endOffset);
             builder.field(Fields.TYPE, type);
             builder.field(Fields.POSITION, position);
+            if (positionLength > 1) {
+                builder.field(Fields.POSITION_LENGTH, positionLength);
+            }
             if (attributes != null && !attributes.isEmpty()) {
                 for (Map.Entry<String, Object> entity : attributes.entrySet()) {
                     builder.field(entity.getKey(), entity.getValue());
@@ -111,6 +117,14 @@ public void readFrom(StreamInput in) throws IOException {
             startOffset = in.readInt();
             endOffset = in.readInt();
             position = in.readVInt();
+            if (in.getVersion().onOrAfter(Version.V_5_2_0)) {
+                Integer len = in.readOptionalVInt();
+                if (len != null) {
+                    positionLength = len;
+                } else {
+                    positionLength = 1;
+                }
+            }
             type = in.readOptionalString();
             if (in.getVersion().onOrAfter(Version.V_2_2_0)) {
                 attributes = (Map<String, Object>) in.readGenericValue();
@@ -123,6 +137,9 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeInt(startOffset);
             out.writeInt(endOffset);
             out.writeVInt(position);
+            if (out.getVersion().onOrAfter(Version.V_5_2_0)) {
+                out.writeOptionalVInt(positionLength > 1 ? positionLength : null);
+            }
             out.writeOptionalString(type);
             if (out.getVersion().onOrAfter(Version.V_2_2_0)) {
                 out.writeGenericValue(attributes);
@@ -157,6 +174,7 @@ public Iterator<AnalyzeToken> iterator() {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
         if (tokens != null) {
             builder.startArray(Fields.TOKENS);
             for (AnalyzeToken token : tokens) {
@@ -170,6 +188,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             detail.toXContent(builder, params);
             builder.endObject();
         }
+        builder.endObject();
         return builder;
     }
 
@@ -209,6 +228,7 @@ static final class Fields {
         static final String END_OFFSET = "end_offset";
         static final String TYPE = "type";
         static final String POSITION = "position";
+        static final String POSITION_LENGTH = "positionLength";
         static final String DETAIL = "detail";
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/DetailAnalyzeResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/DetailAnalyzeResponse.java
index c67c036023097..2d1ba22b989e5 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/DetailAnalyzeResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/DetailAnalyzeResponse.java
@@ -292,7 +292,7 @@ public String[] getTexts() {
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.startObject();
             builder.field(Fields.NAME, name);
-            builder.field(Fields.FILTERED_TEXT, texts);
+            builder.array(Fields.FILTERED_TEXT, texts);
             builder.endObject();
             return builder;
         }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java
index f035bc0f4b7d1..d7e299b1cf1b5 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java
@@ -24,6 +24,7 @@
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute;
 import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.IOUtils;
@@ -45,13 +46,14 @@
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.AnalysisRegistry;
-import org.elasticsearch.index.analysis.AnalysisService;
 import org.elasticsearch.index.analysis.CharFilterFactory;
 import org.elasticsearch.index.analysis.CustomAnalyzer;
+import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.analysis.TokenFilterFactory;
 import org.elasticsearch.index.analysis.TokenizerFactory;
 import org.elasticsearch.index.mapper.AllFieldMapper;
+import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
@@ -130,10 +132,17 @@ protected AnalyzeResponse shardOperation(AnalyzeRequest request, ShardId shardId
                 }
                 MappedFieldType fieldType = indexService.mapperService().fullName(request.field());
                 if (fieldType != null) {
-                    if (fieldType.tokenized() == false) {
+                    if (fieldType.tokenized()) {
+                        analyzer = fieldType.indexAnalyzer();
+                    } else if (fieldType instanceof KeywordFieldMapper.KeywordFieldType) {
+                        analyzer = ((KeywordFieldMapper.KeywordFieldType) fieldType).normalizer();
+                        if (analyzer == null) {
+                            // this will be KeywordAnalyzer
+                            analyzer = fieldType.indexAnalyzer();
+                        }
+                    } else {
                         throw new IllegalArgumentException("Can't process field [" + request.field() + "], Analysis requests are only supported on tokenized fields");
                     }
-                    analyzer = fieldType.indexAnalyzer();
                     field = fieldType.name();
                 }
             }
@@ -145,45 +154,46 @@ protected AnalyzeResponse shardOperation(AnalyzeRequest request, ShardId shardId
                 }
             }
             final AnalysisRegistry analysisRegistry = indicesService.getAnalysis();
-            return analyze(request, field, analyzer, indexService != null ? indexService.analysisService() : null, analysisRegistry, environment);
+            return analyze(request, field, analyzer, indexService != null ? indexService.getIndexAnalyzers() : null, analysisRegistry, environment);
         } catch (IOException e) {
             throw new ElasticsearchException("analysis failed", e);
         }
 
     }
 
-    public static AnalyzeResponse analyze(AnalyzeRequest request, String field,  Analyzer analyzer, AnalysisService analysisService, AnalysisRegistry analysisRegistry, Environment environment) throws IOException {
+    public static AnalyzeResponse analyze(AnalyzeRequest request, String field, Analyzer analyzer, IndexAnalyzers indexAnalyzers, AnalysisRegistry analysisRegistry, Environment environment) throws IOException {
 
         boolean closeAnalyzer = false;
         if (analyzer == null && request.analyzer() != null) {
-            if (analysisService == null) {
+            if (indexAnalyzers == null) {
                 analyzer = analysisRegistry.getAnalyzer(request.analyzer());
                 if (analyzer == null) {
                     throw new IllegalArgumentException("failed to find global analyzer [" + request.analyzer() + "]");
                 }
             } else {
-                analyzer = analysisService.analyzer(request.analyzer());
+                analyzer = indexAnalyzers.get(request.analyzer());
                 if (analyzer == null) {
                     throw new IllegalArgumentException("failed to find analyzer [" + request.analyzer() + "]");
                 }
             }
 
         } else if (request.tokenizer() != null) {
-            TokenizerFactory tokenizerFactory = parseTokenizerFactory(request, analysisService, analysisRegistry, environment);
+            final IndexSettings indexSettings = indexAnalyzers == null ? null : indexAnalyzers.getIndexSettings();
+            TokenizerFactory tokenizerFactory = parseTokenizerFactory(request, indexAnalyzers, analysisRegistry, environment);
 
             TokenFilterFactory[] tokenFilterFactories = new TokenFilterFactory[0];
-            tokenFilterFactories = getTokenFilterFactories(request, analysisService, analysisRegistry, environment, tokenFilterFactories);
+            tokenFilterFactories = getTokenFilterFactories(request, indexSettings, analysisRegistry, environment, tokenFilterFactories);
 
             CharFilterFactory[] charFilterFactories = new CharFilterFactory[0];
-            charFilterFactories = getCharFilterFactories(request, analysisService, analysisRegistry, environment, charFilterFactories);
+            charFilterFactories = getCharFilterFactories(request, indexSettings, analysisRegistry, environment, charFilterFactories);
 
             analyzer = new CustomAnalyzer(tokenizerFactory, charFilterFactories, tokenFilterFactories);
             closeAnalyzer = true;
         } else if (analyzer == null) {
-            if (analysisService == null) {
+            if (indexAnalyzers == null) {
                 analyzer = analysisRegistry.getAnalyzer("standard");
             } else {
-                analyzer = analysisService.defaultIndexAnalyzer();
+                analyzer = indexAnalyzers.getDefaultIndexAnalyzer();
             }
         }
         if (analyzer == null) {
@@ -217,13 +227,15 @@ private static List<AnalyzeResponse.AnalyzeToken> simpleAnalyze(AnalyzeRequest r
                 PositionIncrementAttribute posIncr = stream.addAttribute(PositionIncrementAttribute.class);
                 OffsetAttribute offset = stream.addAttribute(OffsetAttribute.class);
                 TypeAttribute type = stream.addAttribute(TypeAttribute.class);
+                PositionLengthAttribute posLen = stream.addAttribute(PositionLengthAttribute.class);
 
                 while (stream.incrementToken()) {
                     int increment = posIncr.getPositionIncrement();
                     if (increment > 0) {
                         lastPosition = lastPosition + increment;
                     }
-                    tokens.add(new AnalyzeResponse.AnalyzeToken(term.toString(), lastPosition, lastOffset + offset.startOffset(), lastOffset + offset.endOffset(), type.type(), null));
+                    tokens.add(new AnalyzeResponse.AnalyzeToken(term.toString(), lastPosition, lastOffset + offset.startOffset(),
+                        lastOffset + offset.endOffset(), posLen.getPositionLength(), type.type(), null));
 
                 }
                 stream.end();
@@ -380,6 +392,7 @@ private void analyze(TokenStream stream, Analyzer analyzer, String field, Set<St
                 PositionIncrementAttribute posIncr = stream.addAttribute(PositionIncrementAttribute.class);
                 OffsetAttribute offset = stream.addAttribute(OffsetAttribute.class);
                 TypeAttribute type = stream.addAttribute(TypeAttribute.class);
+                PositionLengthAttribute posLen = stream.addAttribute(PositionLengthAttribute.class);
 
                 while (stream.incrementToken()) {
                     int increment = posIncr.getPositionIncrement();
@@ -387,7 +400,7 @@ private void analyze(TokenStream stream, Analyzer analyzer, String field, Set<St
                         lastPosition = lastPosition + increment;
                     }
                     tokens.add(new AnalyzeResponse.AnalyzeToken(term.toString(), lastPosition, lastOffset + offset.startOffset(),
-                        lastOffset + offset.endOffset(), type.type(), extractExtendedAttributes(stream, includeAttributes)));
+                        lastOffset + offset.endOffset(), posLen.getPositionLength(), type.type(), extractExtendedAttributes(stream, includeAttributes)));
 
                 }
                 stream.end();
@@ -446,7 +459,7 @@ private static Map<String, Object> extractExtendedAttributes(TokenStream stream,
         return extendedAttributes;
     }
 
-    private static CharFilterFactory[] getCharFilterFactories(AnalyzeRequest request, AnalysisService analysisService, AnalysisRegistry analysisRegistry,
+    private static CharFilterFactory[] getCharFilterFactories(AnalyzeRequest request, IndexSettings indexSettings, AnalysisRegistry analysisRegistry,
                                                               Environment environment, CharFilterFactory[] charFilterFactories) throws IOException {
         if (request.charFilters() != null && request.charFilters().size() > 0) {
             charFilterFactories = new CharFilterFactory[request.charFilters().size()];
@@ -468,19 +481,19 @@ private static CharFilterFactory[] getCharFilterFactories(AnalyzeRequest request
                     charFilterFactories[i] = charFilterFactoryFactory.get(getNaIndexSettings(settings), environment, "_anonymous_charfilter_[" + i + "]", settings);
                 } else {
                     AnalysisModule.AnalysisProvider<CharFilterFactory> charFilterFactoryFactory;
-                    if (analysisService == null) {
+                    if (indexSettings == null) {
                         charFilterFactoryFactory = analysisRegistry.getCharFilterProvider(charFilter.name);
                         if (charFilterFactoryFactory == null) {
                             throw new IllegalArgumentException("failed to find global char filter under [" + charFilter.name + "]");
                         }
                         charFilterFactories[i] = charFilterFactoryFactory.get(environment, charFilter.name);
                     } else {
-                        charFilterFactoryFactory = analysisRegistry.getCharFilterProvider(charFilter.name, analysisService.getIndexSettings());
+                        charFilterFactoryFactory = analysisRegistry.getCharFilterProvider(charFilter.name, indexSettings);
                         if (charFilterFactoryFactory == null) {
                             throw new IllegalArgumentException("failed to find char filter under [" + charFilter.name + "]");
                         }
-                        charFilterFactories[i] = charFilterFactoryFactory.get(analysisService.getIndexSettings(), environment, charFilter.name,
-                            AnalysisRegistry.getSettingsFromIndexSettings(analysisService.getIndexSettings(),
+                        charFilterFactories[i] = charFilterFactoryFactory.get(indexSettings, environment, charFilter.name,
+                            AnalysisRegistry.getSettingsFromIndexSettings(indexSettings,
                                 AnalysisRegistry.INDEX_ANALYSIS_CHAR_FILTER + "." + charFilter.name));
                     }
                 }
@@ -492,7 +505,7 @@ private static CharFilterFactory[] getCharFilterFactories(AnalyzeRequest request
         return charFilterFactories;
     }
 
-    private static TokenFilterFactory[] getTokenFilterFactories(AnalyzeRequest request, AnalysisService analysisService, AnalysisRegistry analysisRegistry,
+    private static TokenFilterFactory[] getTokenFilterFactories(AnalyzeRequest request, IndexSettings indexSettings, AnalysisRegistry analysisRegistry,
                                                                 Environment environment, TokenFilterFactory[] tokenFilterFactories) throws IOException {
         if (request.tokenFilters() != null && request.tokenFilters().size() > 0) {
             tokenFilterFactories = new TokenFilterFactory[request.tokenFilters().size()];
@@ -514,19 +527,19 @@ private static TokenFilterFactory[] getTokenFilterFactories(AnalyzeRequest reque
                     tokenFilterFactories[i] = tokenFilterFactoryFactory.get(getNaIndexSettings(settings), environment, "_anonymous_tokenfilter_[" + i + "]", settings);
                 } else {
                     AnalysisModule.AnalysisProvider<TokenFilterFactory> tokenFilterFactoryFactory;
-                    if (analysisService == null) {
+                    if (indexSettings == null) {
                         tokenFilterFactoryFactory = analysisRegistry.getTokenFilterProvider(tokenFilter.name);
                         if (tokenFilterFactoryFactory == null) {
                             throw new IllegalArgumentException("failed to find global token filter under [" + tokenFilter.name + "]");
                         }
                         tokenFilterFactories[i] = tokenFilterFactoryFactory.get(environment, tokenFilter.name);
                     } else {
-                        tokenFilterFactoryFactory = analysisRegistry.getTokenFilterProvider(tokenFilter.name, analysisService.getIndexSettings());
+                        tokenFilterFactoryFactory = analysisRegistry.getTokenFilterProvider(tokenFilter.name, indexSettings);
                        if (tokenFilterFactoryFactory == null) {
                             throw new IllegalArgumentException("failed to find token filter under [" + tokenFilter.name + "]");
                         }
-                        tokenFilterFactories[i] = tokenFilterFactoryFactory.get(analysisService.getIndexSettings(), environment, tokenFilter.name,
-                            AnalysisRegistry.getSettingsFromIndexSettings(analysisService.getIndexSettings(),
+                        tokenFilterFactories[i] = tokenFilterFactoryFactory.get(indexSettings, environment, tokenFilter.name,
+                            AnalysisRegistry.getSettingsFromIndexSettings(indexSettings,
                                 AnalysisRegistry.INDEX_ANALYSIS_FILTER + "." + tokenFilter.name));
                     }
                 }
@@ -538,7 +551,7 @@ private static TokenFilterFactory[] getTokenFilterFactories(AnalyzeRequest reque
         return tokenFilterFactories;
     }
 
-    private static TokenizerFactory parseTokenizerFactory(AnalyzeRequest request, AnalysisService analysisService,
+    private static TokenizerFactory parseTokenizerFactory(AnalyzeRequest request, IndexAnalyzers indexAnalzyers,
                                                           AnalysisRegistry analysisRegistry, Environment environment) throws IOException {
         TokenizerFactory tokenizerFactory;
         final AnalyzeRequest.NameOrDefinition tokenizer = request.tokenizer();
@@ -558,19 +571,19 @@ private static TokenizerFactory parseTokenizerFactory(AnalyzeRequest request, An
             tokenizerFactory = tokenizerFactoryFactory.get(getNaIndexSettings(settings), environment, "_anonymous_tokenizer", settings);
         } else {
             AnalysisModule.AnalysisProvider<TokenizerFactory> tokenizerFactoryFactory;
-            if (analysisService == null) {
+            if (indexAnalzyers == null) {
                 tokenizerFactoryFactory = analysisRegistry.getTokenizerProvider(tokenizer.name);
                 if (tokenizerFactoryFactory == null) {
                     throw new IllegalArgumentException("failed to find global tokenizer under [" + tokenizer.name + "]");
                 }
                 tokenizerFactory = tokenizerFactoryFactory.get(environment, tokenizer.name);
             } else {
-                tokenizerFactoryFactory = analysisRegistry.getTokenizerProvider(tokenizer.name, analysisService.getIndexSettings());
+                tokenizerFactoryFactory = analysisRegistry.getTokenizerProvider(tokenizer.name, indexAnalzyers.getIndexSettings());
                 if (tokenizerFactoryFactory == null) {
                     throw new IllegalArgumentException("failed to find tokenizer under [" + tokenizer.name + "]");
                 }
-                tokenizerFactory = tokenizerFactoryFactory.get(analysisService.getIndexSettings(), environment, tokenizer.name,
-                    AnalysisRegistry.getSettingsFromIndexSettings(analysisService.getIndexSettings(),
+                tokenizerFactory = tokenizerFactoryFactory.get(indexAnalzyers.getIndexSettings(), environment, tokenizer.name,
+                    AnalysisRegistry.getSettingsFromIndexSettings(indexAnalzyers.getIndexSettings(),
                         AnalysisRegistry.INDEX_ANALYSIS_TOKENIZER + "." + tokenizer.name));
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java
index d33f37defec1e..244b8a24b9b67 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java
@@ -22,6 +22,7 @@
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.open.OpenIndexResponse;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.DestructiveOperations;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
@@ -97,6 +98,10 @@ protected ClusterBlockException checkBlock(CloseIndexRequest request, ClusterSta
     @Override
     protected void masterOperation(final CloseIndexRequest request, final ClusterState state, final ActionListener<CloseIndexResponse> listener) {
         final Index[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request);
+        if (concreteIndices == null || concreteIndices.length == 0) {
+            listener.onResponse(new CloseIndexResponse(true));
+            return;
+        }
         CloseIndexClusterStateUpdateRequest updateRequest = new CloseIndexClusterStateUpdateRequest()
                 .ackTimeout(request.timeout()).masterNodeTimeout(request.masterNodeTimeout())
                 .indices(concreteIndices);
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java
index 7b3b2a0a2f0c6..a2290a5e2556e 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java
@@ -41,6 +41,7 @@ public class CreateIndexClusterStateUpdateRequest extends ClusterStateUpdateRequ
     private final TransportMessage originalMessage;
     private final String cause;
     private final String index;
+    private final String providedName;
     private final boolean updateAllTypes;
     private Index shrinkFrom;
 
@@ -59,11 +60,13 @@ public class CreateIndexClusterStateUpdateRequest extends ClusterStateUpdateRequ
     private ActiveShardCount waitForActiveShards = ActiveShardCount.DEFAULT;
 
 
-    public CreateIndexClusterStateUpdateRequest(TransportMessage originalMessage, String cause, String index, boolean updateAllTypes) {
+    public CreateIndexClusterStateUpdateRequest(TransportMessage originalMessage, String cause, String index, String providedName,
+                                                boolean updateAllTypes) {
         this.originalMessage = originalMessage;
         this.cause = cause;
         this.index = index;
         this.updateAllTypes = updateAllTypes;
+        this.providedName = providedName;
     }
 
     public CreateIndexClusterStateUpdateRequest settings(Settings settings) {
@@ -151,6 +154,14 @@ public boolean updateAllTypes() {
         return updateAllTypes;
     }
 
+    /**
+     * The name that was provided by the user. This might contain a date math expression.
+     * @see IndexMetaData#SETTING_INDEX_PROVIDED_NAME
+     */
+    public String getProvidedName() {
+        return providedName;
+    }
+
     public ActiveShardCount waitForActiveShards() {
         return waitForActiveShards;
     }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java
index 17df06dbf4b84..daf1415a0aad5 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java
@@ -21,6 +21,7 @@
 
 import org.elasticsearch.ElasticsearchGenerationException;
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.admin.indices.alias.Alias;
@@ -35,6 +36,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentHelper;
@@ -42,10 +44,11 @@
 import org.elasticsearch.common.xcontent.XContentType;
 
 import java.io.IOException;
-import java.nio.charset.StandardCharsets;
+import java.io.UncheckedIOException;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
@@ -168,19 +171,29 @@ public CreateIndexRequest settings(Settings.Builder settings) {
     }
 
     /**
-     * The settings to create the index with (either json/yaml/properties format)
+     * The settings to create the index with (either json or yaml format)
+     * @deprecated use {@link #source(String, XContentType)} instead to avoid content type detection
      */
+    @Deprecated
     public CreateIndexRequest settings(String source) {
         this.settings = Settings.builder().loadFromSource(source).build();
         return this;
     }
 
+    /**
+     * The settings to create the index with (either json or yaml format)
+     */
+    public CreateIndexRequest settings(String source, XContentType xContentType) {
+        this.settings = Settings.builder().loadFromSource(source, xContentType).build();
+        return this;
+    }
+
     /**
      * Allows to set the settings using a json builder.
      */
     public CreateIndexRequest settings(XContentBuilder builder) {
         try {
-            settings(builder.string());
+            settings(builder.string(), builder.contentType());
         } catch (IOException e) {
             throw new ElasticsearchGenerationException("Failed to generate json settings from builder", e);
         }
@@ -195,7 +208,7 @@ public CreateIndexRequest settings(Map source) {
         try {
             XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
             builder.map(source);
-            settings(builder.string());
+            settings(builder.string(), XContentType.JSON);
         } catch (IOException e) {
             throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e);
         }
@@ -207,13 +220,42 @@ public CreateIndexRequest settings(Map source) {
      *
      * @param type   The mapping type
      * @param source The mapping source
+     * @deprecated use {@link #mapping(String, String, XContentType)} to avoid content type detection
      */
+    @Deprecated
     public CreateIndexRequest mapping(String type, String source) {
+        return mapping(type, new BytesArray(source), XContentFactory.xContentType(source));
+    }
+
+    /**
+     * Adds mapping that will be added when the index gets created.
+     *
+     * @param type   The mapping type
+     * @param source The mapping source
+     * @param xContentType The content type of the source
+     */
+    public CreateIndexRequest mapping(String type, String source, XContentType xContentType) {
+        return mapping(type, new BytesArray(source), xContentType);
+    }
+
+    /**
+     * Adds mapping that will be added when the index gets created.
+     *
+     * @param type   The mapping type
+     * @param source The mapping source
+     * @param xContentType the content type of the mapping source
+     */
+    private CreateIndexRequest mapping(String type, BytesReference source, XContentType xContentType) {
         if (mappings.containsKey(type)) {
             throw new IllegalStateException("mappings for type \"" + type + "\" were already defined");
         }
-        mappings.put(type, source);
-        return this;
+        Objects.requireNonNull(xContentType);
+        try {
+            mappings.put(type, XContentHelper.convertToJson(source, false, false, xContentType));
+            return this;
+        } catch (IOException e) {
+            throw new UncheckedIOException("failed to convert to json", e);
+        }
     }
 
     /**
@@ -231,15 +273,7 @@ public CreateIndexRequest cause(String cause) {
      * @param source The mapping source
      */
     public CreateIndexRequest mapping(String type, XContentBuilder source) {
-        if (mappings.containsKey(type)) {
-            throw new IllegalStateException("mappings for type \"" + type + "\" were already defined");
-        }
-        try {
-            mappings.put(type, source.string());
-        } catch (IOException e) {
-            throw new IllegalArgumentException("Failed to build json for mapping request", e);
-        }
-        return this;
+        return mapping(type, source.bytes(), source.contentType());
     }
 
     /**
@@ -260,7 +294,7 @@ public CreateIndexRequest mapping(String type, Map source) {
         try {
             XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
             builder.map(source);
-            return mapping(type, builder.string());
+            return mapping(type, builder);
         } catch (IOException e) {
             throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e);
         }
@@ -307,7 +341,8 @@ public CreateIndexRequest aliases(String source) {
      * Sets the aliases that will be associated with the index when it gets created
      */
     public CreateIndexRequest aliases(BytesReference source) {
-        try (XContentParser parser = XContentHelper.createParser(source)) {
+        // EMPTY is safe here because we never call namedObject
+        try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, source)) {
             //move to the first alias
             parser.nextToken();
             while ((parser.nextToken()) != XContentParser.Token.END_OBJECT) {
@@ -329,9 +364,18 @@ public CreateIndexRequest alias(Alias alias) {
 
     /**
      * Sets the settings and mappings as a single source.
+     * @deprecated use {@link #source(String, XContentType)}
      */
+    @Deprecated
     public CreateIndexRequest source(String source) {
-        return source(source.getBytes(StandardCharsets.UTF_8));
+        return source(new BytesArray(source));
+    }
+
+    /**
+     * Sets the settings and mappings as a single source.
+     */
+    public CreateIndexRequest source(String source, XContentType xContentType) {
+        return source(new BytesArray(source), xContentType);
     }
 
     /**
@@ -343,7 +387,9 @@ public CreateIndexRequest source(XContentBuilder source) {
 
     /**
      * Sets the settings and mappings as a single source.
+     * @deprecated use {@link #source(byte[], XContentType)}
      */
+    @Deprecated
     public CreateIndexRequest source(byte[] source) {
         return source(source, 0, source.length);
     }
@@ -351,6 +397,15 @@ public CreateIndexRequest source(byte[] source) {
     /**
      * Sets the settings and mappings as a single source.
      */
+    public CreateIndexRequest source(byte[] source, XContentType xContentType) {
+        return source(source, 0, source.length, xContentType);
+    }
+
+    /**
+     * Sets the settings and mappings as a single source.
+     * @deprecated use {@link #source(byte[], int, int, XContentType)}
+     */
+    @Deprecated
     public CreateIndexRequest source(byte[] source, int offset, int length) {
         return source(new BytesArray(source, offset, length));
     }
@@ -358,17 +413,27 @@ public CreateIndexRequest source(byte[] source, int offset, int length) {
     /**
      * Sets the settings and mappings as a single source.
      */
+    public CreateIndexRequest source(byte[] source, int offset, int length, XContentType xContentType) {
+        return source(new BytesArray(source, offset, length), xContentType);
+    }
+
+    /**
+     * Sets the settings and mappings as a single source.
+     * @deprecated use {@link #source(BytesReference, XContentType)}
+     */
+    @Deprecated
     public CreateIndexRequest source(BytesReference source) {
         XContentType xContentType = XContentFactory.xContentType(source);
-        if (xContentType != null) {
-            try (XContentParser parser = XContentFactory.xContent(xContentType).createParser(source)) {
-                source(parser.map());
-            } catch (IOException e) {
-                throw new ElasticsearchParseException("failed to parse source for create index", e);
-            }
-        } else {
-            settings(source.utf8ToString());
-        }
+        source(source, xContentType);
+        return this;
+    }
+
+    /**
+     * Sets the settings and mappings as a single source.
+     */
+    public CreateIndexRequest source(BytesReference source, XContentType xContentType) {
+        Objects.requireNonNull(xContentType);
+        source(XContentHelper.convertToMap(source, false, xContentType).v2());
         return this;
     }
 
@@ -485,7 +550,13 @@ public void readFrom(StreamInput in) throws IOException {
         readTimeout(in);
         int size = in.readVInt();
         for (int i = 0; i < size; i++) {
-            mappings.put(in.readString(), in.readString());
+            final String type = in.readString();
+            String source = in.readString();
+            if (in.getVersion().before(Version.V_5_3_0)) {
+                // we do not know the content type that comes from earlier versions so we autodetect and convert
+                source = XContentHelper.convertToJson(new BytesArray(source), false, false, XContentFactory.xContentType(source));
+            }
+            mappings.put(type, source);
         }
         int customSize = in.readVInt();
         for (int i = 0; i < customSize; i++) {
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java
index eaae4d53b73fd..237c88244b4cd 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentType;
 
 import java.util.Map;
 
@@ -76,13 +77,23 @@ public CreateIndexRequestBuilder setSettings(XContentBuilder builder) {
     }
 
     /**
-     * The settings to create the index with (either json/yaml/properties format)
+     * The settings to create the index with (either json or yaml format)
+     * @deprecated use {@link #setSettings(String, XContentType)} to avoid content type detection
      */
+    @Deprecated
     public CreateIndexRequestBuilder setSettings(String source) {
         request.settings(source);
         return this;
     }
 
+    /**
+     * The settings to create the index with (either json or yaml format)
+     */
+    public CreateIndexRequestBuilder setSettings(String source, XContentType xContentType) {
+        request.settings(source, xContentType);
+        return this;
+    }
+
     /**
      * A simplified version of settings that takes key value pairs settings.
      */
@@ -104,12 +115,26 @@ public CreateIndexRequestBuilder setSettings(Map<String, ?> source) {
      *
      * @param type   The mapping type
      * @param source The mapping source
+     * @deprecated use {@link #addMapping(String, String, XContentType)} to avoid content type auto-detection
      */
+    @Deprecated
     public CreateIndexRequestBuilder addMapping(String type, String source) {
         request.mapping(type, source);
         return this;
     }
 
+    /**
+     * Adds mapping that will be added when the index gets created.
+     *
+     * @param type   The mapping type
+     * @param source The mapping source
+     * @param xContentType The content type of the source
+     */
+    public CreateIndexRequestBuilder addMapping(String type, String source, XContentType xContentType) {
+        request.mapping(type, source, xContentType);
+        return this;
+    }
+
     /**
      * The cause for this index creation.
      */
@@ -191,7 +216,9 @@ public CreateIndexRequestBuilder addAlias(Alias alias) {
 
     /**
      * Sets the settings and mappings as a single source.
+     * @deprecated use {@link #setSource(String, XContentType)}
      */
+    @Deprecated
     public CreateIndexRequestBuilder setSource(String source) {
         request.source(source);
         return this;
@@ -200,6 +227,16 @@ public CreateIndexRequestBuilder setSource(String source) {
     /**
      * Sets the settings and mappings as a single source.
      */
+    public CreateIndexRequestBuilder setSource(String source, XContentType xContentType) {
+        request.source(source, xContentType);
+        return this;
+    }
+
+    /**
+     * Sets the settings and mappings as a single source.
+     * @deprecated use {@link #setSource(BytesReference, XContentType)}
+     */
+    @Deprecated
     public CreateIndexRequestBuilder setSource(BytesReference source) {
         request.source(source);
         return this;
@@ -208,6 +245,16 @@ public CreateIndexRequestBuilder setSource(BytesReference source) {
     /**
      * Sets the settings and mappings as a single source.
      */
+    public CreateIndexRequestBuilder setSource(BytesReference source, XContentType xContentType) {
+        request.source(source, xContentType);
+        return this;
+    }
+
+    /**
+     * Sets the settings and mappings as a single source.
+     * @deprecated use {@link #setSource(byte[], XContentType)}
+     */
+    @Deprecated
     public CreateIndexRequestBuilder setSource(byte[] source) {
         request.source(source);
         return this;
@@ -216,11 +263,29 @@ public CreateIndexRequestBuilder setSource(byte[] source) {
     /**
      * Sets the settings and mappings as a single source.
      */
+    public CreateIndexRequestBuilder setSource(byte[] source, XContentType xContentType) {
+        request.source(source, xContentType);
+        return this;
+    }
+
+    /**
+     * Sets the settings and mappings as a single source.
+     * @deprecated use {@link #setSource(byte[], int, int, XContentType)}
+     */
+    @Deprecated
     public CreateIndexRequestBuilder setSource(byte[] source, int offset, int length) {
         request.source(source, offset, length);
         return this;
     }
 
+    /**
+     * Sets the settings and mappings as a single source.
+     */
+    public CreateIndexRequestBuilder setSource(byte[] source, int offset, int length, XContentType xContentType) {
+        request.source(source, offset, length, xContentType);
+        return this;
+    }
+
     /**
      * Sets the settings and mappings as a single source.
      */
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexResponse.java
index 35dd53276cd6d..7d948e7137ebf 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexResponse.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.action.admin.indices.create;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -32,14 +33,16 @@
 public class CreateIndexResponse extends AcknowledgedResponse {
 
     private boolean shardsAcked;
+    private String index;
 
     protected CreateIndexResponse() {
     }
 
-    protected CreateIndexResponse(boolean acknowledged, boolean shardsAcked) {
+    protected CreateIndexResponse(boolean acknowledged, boolean shardsAcked, String index) {
         super(acknowledged);
         assert acknowledged || shardsAcked == false; // if its not acknowledged, then shards acked should be false too
         this.shardsAcked = shardsAcked;
+        this.index = index;
     }
 
     @Override
@@ -47,6 +50,9 @@ public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
         readAcknowledged(in);
         shardsAcked = in.readBoolean();
+        if (in.getVersion().onOrAfter(Version.V_5_6_0)) {
+            index = in.readString();
+        }
     }
 
     @Override
@@ -54,6 +60,9 @@ public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         writeAcknowledged(out);
         out.writeBoolean(shardsAcked);
+        if (out.getVersion().onOrAfter(Version.V_5_6_0)) {
+            out.writeString(index);
+        }
     }
 
     /**
@@ -65,7 +74,12 @@ public boolean isShardsAcked() {
         return shardsAcked;
     }
 
+    public String index() {
+        return index;
+    }
+
     public void addCustomFields(XContentBuilder builder) throws IOException {
         builder.field("shards_acknowledged", isShardsAcked());
+        builder.field("index", index());
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java
index d3ce1975e8917..0ac8d02f97760 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java
@@ -72,14 +72,14 @@ protected void masterOperation(final CreateIndexRequest request, final ClusterSt
         }
 
         final String indexName = indexNameExpressionResolver.resolveDateMathExpression(request.index());
-        final CreateIndexClusterStateUpdateRequest updateRequest = new CreateIndexClusterStateUpdateRequest(request, cause, indexName, request.updateAllTypes())
+        final CreateIndexClusterStateUpdateRequest updateRequest = new CreateIndexClusterStateUpdateRequest(request, cause, indexName, request.index(), request.updateAllTypes())
                 .ackTimeout(request.timeout()).masterNodeTimeout(request.masterNodeTimeout())
                 .settings(request.settings()).mappings(request.mappings())
                 .aliases(request.aliases()).customs(request.customs())
                 .waitForActiveShards(request.waitForActiveShards());
 
         createIndexService.createIndex(updateRequest, ActionListener.wrap(response ->
-            listener.onResponse(new CreateIndexResponse(response.isAcknowledged(), response.isShardsAcked())),
+            listener.onResponse(new CreateIndexResponse(response.isAcknowledged(), response.isShardsAcked(), indexName)),
             listener::onFailure));
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexRequestBuilder.java
index 9e5dc88b983a0..e4c7a9fa21430 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexRequestBuilder.java
@@ -20,37 +20,18 @@
 package org.elasticsearch.action.admin.indices.delete;
 
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder;
+import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
-import org.elasticsearch.common.unit.TimeValue;
 
 /**
  *
  */
-public class DeleteIndexRequestBuilder extends MasterNodeOperationRequestBuilder<DeleteIndexRequest, DeleteIndexResponse, DeleteIndexRequestBuilder> {
+public class DeleteIndexRequestBuilder extends AcknowledgedRequestBuilder<DeleteIndexRequest, DeleteIndexResponse, DeleteIndexRequestBuilder> {
 
     public DeleteIndexRequestBuilder(ElasticsearchClient client, DeleteIndexAction action, String... indices) {
         super(client, action, new DeleteIndexRequest(indices));
     }
 
-    /**
-     * Timeout to wait for the index deletion to be acknowledged by current cluster nodes. Defaults
-     * to <tt>60s</tt>.
-     */
-    public DeleteIndexRequestBuilder setTimeout(TimeValue timeout) {
-        request.timeout(timeout);
-        return this;
-    }
-
-    /**
-     * Timeout to wait for the index deletion to be acknowledged by current cluster nodes. Defaults
-     * to <tt>10s</tt>.
-     */
-    public DeleteIndexRequestBuilder setTimeout(String timeout) {
-        request.timeout(timeout);
-        return this;
-    }
-
     /**
      * Specifies what type of requested indices to ignore and wildcard indices expressions.
      * <p>
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java
index 251eed8bdb88b..f5c63bd470d40 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java
@@ -78,7 +78,7 @@ protected void doExecute(Task task, DeleteIndexRequest request, ActionListener<D
 
     @Override
     protected ClusterBlockException checkBlock(DeleteIndexRequest request, ClusterState state) {
-        return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_WRITE, indexNameExpressionResolver.concreteIndexNames(state, request));
+        return state.blocks().indicesAllowReleaseResources(indexNameExpressionResolver.concreteIndexNames(state, request));
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/flush/FlushRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/flush/FlushRequest.java
index 7dc55c08fac46..f91b69755c14c 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/flush/FlushRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/flush/FlushRequest.java
@@ -40,7 +40,7 @@
 public class FlushRequest extends BroadcastRequest<FlushRequest> {
 
     private boolean force = false;
-    private boolean waitIfOngoing = false;
+    private boolean waitIfOngoing = true;
 
     /**
      * Constructs a new flush request against one or more indices. If nothing is provided, all indices will
@@ -61,6 +61,7 @@ public boolean waitIfOngoing() {
     /**
      * if set to <tt>true</tt> the flush will block
      * if a another flush operation is already running until the flush can be performed.
+     * The default is <code>true</code>
      */
     public FlushRequest waitIfOngoing(boolean waitIfOngoing) {
         this.waitIfOngoing = waitIfOngoing;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java
index 570307a717da2..d79668ea73ed9 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java
@@ -23,7 +23,7 @@
 import org.elasticsearch.action.support.replication.ReplicationResponse;
 import org.elasticsearch.action.support.replication.TransportReplicationAction;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
-import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
@@ -54,33 +54,21 @@ protected ReplicationResponse newResponseInstance() {
     }
 
     @Override
-    protected PrimaryResult shardOperationOnPrimary(ShardFlushRequest shardRequest) {
-        IndexShard indexShard = indicesService.indexServiceSafe(shardRequest.shardId().getIndex()).getShard(shardRequest.shardId().id());
-        indexShard.flush(shardRequest.getRequest());
-        logger.trace("{} flush request executed on primary", indexShard.shardId());
+    protected PrimaryResult shardOperationOnPrimary(ShardFlushRequest shardRequest, IndexShard primary) {
+        primary.flush(shardRequest.getRequest());
+        logger.trace("{} flush request executed on primary", primary.shardId());
         return new PrimaryResult(shardRequest, new ReplicationResponse());
     }
 
     @Override
-    protected ReplicaResult shardOperationOnReplica(ShardFlushRequest request) {
-        IndexShard indexShard = indicesService.indexServiceSafe(request.shardId().getIndex()).getShard(request.shardId().id());
-        indexShard.flush(request.getRequest());
-        logger.trace("{} flush request executed on replica", indexShard.shardId());
+    protected ReplicaResult shardOperationOnReplica(ShardFlushRequest request, IndexShard replica) {
+        replica.flush(request.getRequest());
+        logger.trace("{} flush request executed on replica", replica.shardId());
         return new ReplicaResult();
     }
 
     @Override
-    protected ClusterBlockLevel globalBlockLevel() {
-        return ClusterBlockLevel.METADATA_WRITE;
-    }
-
-    @Override
-    protected ClusterBlockLevel indexBlockLevel() {
-        return ClusterBlockLevel.METADATA_WRITE;
-    }
-
-    @Override
-    protected boolean shouldExecuteReplication(Settings settings) {
+    protected boolean shouldExecuteReplication(IndexMetaData indexMetaData) {
         return true;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexRequest.java
index 6887857925979..48886c38aa4bc 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexRequest.java
@@ -34,7 +34,7 @@
  */
 public class GetIndexRequest extends ClusterInfoRequest<GetIndexRequest> {
 
-    public static enum Feature {
+    public enum Feature {
         ALIASES((byte) 0, "_aliases", "_alias"),
         MAPPINGS((byte) 1, "_mappings", "_mapping"),
         SETTINGS((byte) 2, "_settings");
@@ -52,7 +52,7 @@ public static enum Feature {
         private final String preferredName;
         private final byte id;
 
-        private Feature(byte id, String... validNames) {
+        Feature(byte id, String... validNames) {
             assert validNames != null && validNames.length > 0;
             this.id = id;
             this.validNames = Arrays.asList(validNames);
@@ -77,7 +77,7 @@ public static Feature fromName(String name) {
                     return feature;
                 }
             }
-            throw new IllegalArgumentException("No feature for name [" + name + "]");
+            throw new IllegalArgumentException("No endpoint or operation is available at [" + name + "]");
         }
 
         public static Feature fromId(byte id) {
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java
index 3a29237faeb2b..36bfa81a33416 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java
@@ -104,7 +104,7 @@ public void readFrom(StreamInput in) throws IOException {
             int valueSize = in.readVInt();
             ImmutableOpenMap.Builder<String, MappingMetaData> mappingEntryBuilder = ImmutableOpenMap.builder();
             for (int j = 0; j < valueSize; j++) {
-                mappingEntryBuilder.put(in.readString(), MappingMetaData.PROTO.readFrom(in));
+                mappingEntryBuilder.put(in.readString(), new MappingMetaData(in));
             }
             mappingsMapBuilder.put(key, mappingEntryBuilder.build());
         }
@@ -114,9 +114,9 @@ public void readFrom(StreamInput in) throws IOException {
         for (int i = 0; i < aliasesSize; i++) {
             String key = in.readString();
             int valueSize = in.readVInt();
-            List<AliasMetaData> aliasEntryBuilder = new ArrayList<>();
+            List<AliasMetaData> aliasEntryBuilder = new ArrayList<>(valueSize);
             for (int j = 0; j < valueSize; j++) {
-                aliasEntryBuilder.add(AliasMetaData.Builder.readFrom(in));
+                aliasEntryBuilder.add(new AliasMetaData(in));
             }
             aliasesMapBuilder.put(key, Collections.unmodifiableList(aliasEntryBuilder));
         }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsRequest.java
index 967ea31c84a09..819d2de999ccf 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsRequest.java
@@ -30,7 +30,7 @@
 import java.io.IOException;
 
 /** Request the mappings of specific fields */
-public class GetFieldMappingsRequest extends ActionRequest<GetFieldMappingsRequest> implements IndicesRequest.Replaceable {
+public class GetFieldMappingsRequest extends ActionRequest implements IndicesRequest.Replaceable {
 
     protected boolean local = false;
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java
index e0cedcf841e47..3f4ddaf08db2c 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.mapper.Mapper;
 
 import java.io.IOException;
@@ -108,20 +109,25 @@ public String fullName() {
 
         /** Returns the mappings as a map. Note that the returned map has a single key which is always the field's {@link Mapper#name}. */
         public Map<String, Object> sourceAsMap() {
-            return XContentHelper.convertToMap(source, true).v2();
+            return XContentHelper.convertToMap(source, true, XContentType.JSON).v2();
         }
 
         public boolean isNull() {
             return NULL.fullName().equals(fullName) && NULL.source.length() == source.length();
         }
 
+        //pkg-private for testing
+        BytesReference getSource() {
+            return source;
+        }
+
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.field("full_name", fullName);
             if (params.paramAsBoolean("pretty", false)) {
                 builder.field("mapping", sourceAsMap());
             } else {
-                builder.rawField("mapping", source);
+                builder.rawField("mapping", source, XContentType.JSON);
             }
             return builder;
         }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java
index 30e9e24c4937b..04e257aad5b1a 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java
@@ -59,7 +59,7 @@ public void readFrom(StreamInput in) throws IOException {
             int valueSize = in.readVInt();
             ImmutableOpenMap.Builder<String, MappingMetaData> typeMapBuilder = ImmutableOpenMap.builder();
             for (int j = 0; j < valueSize; j++) {
-                typeMapBuilder.put(in.readString(), MappingMetaData.PROTO.readFrom(in));
+                typeMapBuilder.put(in.readString(), new MappingMetaData(in));
             }
             indexMapBuilder.put(key, typeMapBuilder.build());
         }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java
index 864c6703c48e0..92c23bb856865 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java
@@ -50,12 +50,10 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Iterator;
 import java.util.Map;
 import java.util.stream.Collectors;
 
 import static java.util.Collections.singletonMap;
-import static org.elasticsearch.common.util.CollectionUtils.newLinkedList;
 
 /**
  * Transport action used to retrieve the mappings related to fields that belong to a specific index
@@ -174,24 +172,12 @@ private Map<String, FieldMappingMetaData> findFieldMappingsByType(DocumentMapper
                     addFieldMapper(fieldMapper.fieldType().name(), fieldMapper, fieldMappings, request.includeDefaults());
                 }
             } else if (Regex.isSimpleMatchPattern(field)) {
-                // go through the field mappers 3 times, to make sure we give preference to the resolve order: full name, index name, name.
-                // also make sure we only store each mapper once.
-                Collection<FieldMapper> remainingFieldMappers = newLinkedList(allFieldMappers);
-                for (Iterator<FieldMapper> it = remainingFieldMappers.iterator(); it.hasNext(); ) {
-                    final FieldMapper fieldMapper = it.next();
-                    if (Regex.simpleMatch(field, fieldMapper.fieldType().name())) {
-                        addFieldMapper(fieldMapper.fieldType().name(), fieldMapper, fieldMappings, request.includeDefaults());
-                        it.remove();
-                    }
-                }
-                for (Iterator<FieldMapper> it = remainingFieldMappers.iterator(); it.hasNext(); ) {
-                    final FieldMapper fieldMapper = it.next();
+                for (FieldMapper fieldMapper : allFieldMappers) {
                     if (Regex.simpleMatch(field, fieldMapper.fieldType().name())) {
-                        addFieldMapper(fieldMapper.fieldType().name(), fieldMapper, fieldMappings, request.includeDefaults());
-                        it.remove();
+                        addFieldMapper(fieldMapper.fieldType().name(), fieldMapper, fieldMappings,
+                            request.includeDefaults());
                     }
                 }
-
             } else {
                 // not a pattern
                 FieldMapper fieldMapper = allFieldMappers.smartNameFieldMapper(field);
@@ -220,4 +206,4 @@ private void addFieldMapper(String field, FieldMapper fieldMapper, MapBuilder<St
         }
     }
 
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingClusterStateUpdateRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingClusterStateUpdateRequest.java
index c8b10af9a8f5d..0f396afa5513b 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingClusterStateUpdateRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingClusterStateUpdateRequest.java
@@ -32,7 +32,7 @@ public class PutMappingClusterStateUpdateRequest extends IndicesClusterStateUpda
 
     private boolean updateAllTypes = false;
 
-    PutMappingClusterStateUpdateRequest() {
+    public PutMappingClusterStateUpdateRequest() {
 
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java
index 152bc516549b9..8fb9736cbfc15 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java
@@ -22,19 +22,24 @@
 import com.carrotsearch.hppc.ObjectHashSet;
 
 import org.elasticsearch.ElasticsearchGenerationException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.Index;
 
 import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.util.Arrays;
 import java.util.Map;
 import java.util.Objects;
@@ -56,7 +61,7 @@ public class PutMappingRequest extends AcknowledgedRequest<PutMappingRequest> im
 
     private static ObjectHashSet<String> RESERVED_FIELDS = ObjectHashSet.from(
             "_uid", "_id", "_type", "_source",  "_all", "_analyzer", "_parent", "_routing", "_index",
-            "_size", "_timestamp", "_ttl"
+            "_size", "_timestamp", "_ttl", "_field_names"
     );
 
     private String[] indices;
@@ -245,7 +250,7 @@ public static XContentBuilder buildFromSimplifiedDef(String type, Object... sour
      */
     public PutMappingRequest source(XContentBuilder mappingBuilder) {
         try {
-            return source(mappingBuilder.string());
+            return source(mappingBuilder.string(), mappingBuilder.contentType());
         } catch (IOException e) {
             throw new IllegalArgumentException("Failed to build json for mapping request", e);
         }
@@ -259,7 +264,7 @@ public PutMappingRequest source(Map mappingSource) {
         try {
             XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
             builder.map(mappingSource);
-            return source(builder.string());
+            return source(builder.string(), XContentType.JSON);
         } catch (IOException e) {
             throw new ElasticsearchGenerationException("Failed to generate [" + mappingSource + "]", e);
         }
@@ -267,10 +272,31 @@ public PutMappingRequest source(Map mappingSource) {
 
     /**
      * The mapping source definition.
+     * @deprecated use {@link #source(String, XContentType)}
      */
+    @Deprecated
     public PutMappingRequest source(String mappingSource) {
-        this.source = mappingSource;
-        return this;
+        return source(mappingSource, XContentFactory.xContentType(mappingSource));
+    }
+
+    /**
+     * The mapping source definition.
+     */
+    public PutMappingRequest source(String mappingSource, XContentType xContentType) {
+        return source(new BytesArray(mappingSource), xContentType);
+    }
+
+    /**
+     * The mapping source definition.
+     */
+    public PutMappingRequest source(BytesReference mappingSource, XContentType xContentType) {
+        Objects.requireNonNull(xContentType);
+        try {
+            this.source = XContentHelper.convertToJson(mappingSource, false, false, xContentType);
+            return this;
+        } catch (IOException e) {
+            throw new UncheckedIOException("failed to convert source to json", e);
+        }
     }
 
     /** True if all fields that span multiple types should be updated, false otherwise */
@@ -291,6 +317,10 @@ public void readFrom(StreamInput in) throws IOException {
         indicesOptions = IndicesOptions.readIndicesOptions(in);
         type = in.readOptionalString();
         source = in.readString();
+        if (in.getVersion().before(Version.V_5_3_0)) {
+            // we do not know the format from earlier versions so convert if necessary
+            source = XContentHelper.convertToJson(new BytesArray(source), false, false, XContentFactory.xContentType(source));
+        }
         updateAllTypes = in.readBoolean();
         readTimeout(in);
         concreteIndex = in.readOptionalWriteable(Index::new);
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestBuilder.java
index c21c40cf041ea..012a593ebc473 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestBuilder.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.Index;
 
 import java.util.Map;
@@ -82,12 +83,22 @@ public PutMappingRequestBuilder setSource(Map mappingSource) {
 
     /**
      * The mapping source definition.
+     * @deprecated use {@link #setSource(String, XContentType)}
      */
+    @Deprecated
     public PutMappingRequestBuilder setSource(String mappingSource) {
         request.source(mappingSource);
         return this;
     }
 
+    /**
+     * The mapping source definition.
+     */
+    public PutMappingRequestBuilder setSource(String mappingSource, XContentType xContentType) {
+        request.source(mappingSource, xContentType);
+        return this;
+    }
+
     /**
      * A specialized simplified mapping source method, takes the form of simple properties definition:
      * ("field1", "type=string,store=true").
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/open/TransportOpenIndexAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/open/TransportOpenIndexAction.java
index 1128ebf9875fd..451b9a280be68 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/open/TransportOpenIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/open/TransportOpenIndexAction.java
@@ -22,6 +22,7 @@
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.DestructiveOperations;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
@@ -82,6 +83,10 @@ protected ClusterBlockException checkBlock(OpenIndexRequest request, ClusterStat
     @Override
     protected void masterOperation(final OpenIndexRequest request, final ClusterState state, final ActionListener<OpenIndexResponse> listener) {
         final Index[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request);
+        if (concreteIndices == null || concreteIndices.length == 0) {
+            listener.onResponse(new OpenIndexResponse(true));
+            return;
+        }
         OpenIndexClusterStateUpdateRequest updateRequest = new OpenIndexClusterStateUpdateRequest()
                 .ackTimeout(request.timeout()).masterNodeTimeout(request.masterNodeTimeout())
                 .indices(concreteIndices);
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java
index cf9f568195382..d1d8b4078b647 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java
@@ -24,13 +24,12 @@
 import org.elasticsearch.action.support.replication.ReplicationResponse;
 import org.elasticsearch.action.support.replication.TransportReplicationAction;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
-import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.shard.IndexShard;
-import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -55,34 +54,21 @@ protected ReplicationResponse newResponseInstance() {
     }
 
     @Override
-    protected PrimaryResult shardOperationOnPrimary(BasicReplicationRequest shardRequest) {
-        IndexShard indexShard = indicesService.indexServiceSafe(shardRequest.shardId().getIndex()).getShard(shardRequest.shardId().id());
-        indexShard.refresh("api");
-        logger.trace("{} refresh request executed on primary", indexShard.shardId());
+    protected PrimaryResult shardOperationOnPrimary(BasicReplicationRequest shardRequest, IndexShard primary) {
+        primary.refresh("api");
+        logger.trace("{} refresh request executed on primary", primary.shardId());
         return new PrimaryResult(shardRequest, new ReplicationResponse());
     }
 
     @Override
-    protected ReplicaResult shardOperationOnReplica(BasicReplicationRequest request) {
-        final ShardId shardId = request.shardId();
-        IndexShard indexShard = indicesService.indexServiceSafe(shardId.getIndex()).getShard(shardId.id());
-        indexShard.refresh("api");
-        logger.trace("{} refresh request executed on replica", indexShard.shardId());
+    protected ReplicaResult shardOperationOnReplica(BasicReplicationRequest request, IndexShard replica) {
+        replica.refresh("api");
+        logger.trace("{} refresh request executed on replica", replica.shardId());
         return new ReplicaResult();
     }
 
     @Override
-    protected ClusterBlockLevel globalBlockLevel() {
-        return ClusterBlockLevel.METADATA_WRITE;
-    }
-
-    @Override
-    protected ClusterBlockLevel indexBlockLevel() {
-        return ClusterBlockLevel.METADATA_WRITE;
-    }
-
-    @Override
-    protected boolean shouldExecuteReplication(Settings settings) {
+    protected boolean shouldExecuteReplication(IndexMetaData indexMetaData) {
         return true;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java
index 8d9b48f20008e..d6bfaf0a48cec 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.action.admin.indices.rollover;
 
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ObjectParser;
@@ -32,8 +31,7 @@
  */
 public abstract class Condition<T> implements NamedWriteable {
 
-    public static ObjectParser<Set<Condition>, ParseFieldMatcherSupplier> PARSER =
-        new ObjectParser<>("conditions", null);
+    public static ObjectParser<Set<Condition>, Void> PARSER = new ObjectParser<>("conditions", null);
     static {
         PARSER.declareString((conditions, s) ->
             conditions.add(new MaxAgeCondition(TimeValue.parseTimeValue(s, MaxAgeCondition.NAME))),
@@ -49,7 +47,7 @@ protected Condition(String name) {
         this.name = name;
     }
 
-    public abstract Result evaluate(final Stats stats);
+    public abstract Result evaluate(Stats stats);
 
     @Override
     public final String toString() {
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
index 854611658dfe5..4804bc577fc58 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
@@ -18,7 +18,6 @@
  */
 package org.elasticsearch.action.admin.indices.rollover;
 
-import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
@@ -26,16 +25,10 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ObjectParser;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentType;
 
 import java.io.IOException;
 import java.util.HashSet;
@@ -50,23 +43,19 @@
  */
 public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implements IndicesRequest {
 
-    public static ObjectParser<RolloverRequest, ParseFieldMatcherSupplier> PARSER =
-        new ObjectParser<>("conditions", null);
+    public static final ObjectParser<RolloverRequest, Void> PARSER = new ObjectParser<>("conditions", null);
     static {
-        PARSER.declareField((parser, request, parseFieldMatcherSupplier) ->
-                Condition.PARSER.parse(parser, request.conditions, parseFieldMatcherSupplier),
+        PARSER.declareField((parser, request, context) -> Condition.PARSER.parse(parser, request.conditions, null),
             new ParseField("conditions"), ObjectParser.ValueType.OBJECT);
-        PARSER.declareField((parser, request, parseFieldMatcherSupplier) ->
-                request.createIndexRequest.settings(parser.map()),
+        PARSER.declareField((parser, request, context) -> request.createIndexRequest.settings(parser.map()),
             new ParseField("settings"), ObjectParser.ValueType.OBJECT);
-        PARSER.declareField((parser, request, parseFieldMatcherSupplier) -> {
+        PARSER.declareField((parser, request, context) -> {
             for (Map.Entry<String, Object> mappingsEntry : parser.map().entrySet()) {
                 request.createIndexRequest.mapping(mappingsEntry.getKey(),
                     (Map<String, Object>) mappingsEntry.getValue());
             }
         }, new ParseField("mappings"), ObjectParser.ValueType.OBJECT);
-        PARSER.declareField((parser, request, parseFieldMatcherSupplier) ->
-                request.createIndexRequest.aliases(parser.map()),
+        PARSER.declareField((parser, request, context) -> request.createIndexRequest.aliases(parser.map()),
             new ParseField("aliases"), ObjectParser.ValueType.OBJECT);
     }
 
@@ -194,19 +183,6 @@ CreateIndexRequest getCreateIndexRequest() {
         return createIndexRequest;
     }
 
-    public void source(BytesReference source) {
-        XContentType xContentType = XContentFactory.xContentType(source);
-        if (xContentType != null) {
-            try (XContentParser parser = XContentFactory.xContent(xContentType).createParser(source)) {
-                PARSER.parse(parser, this, () -> ParseFieldMatcher.EMPTY);
-            } catch (IOException e) {
-                throw new ElasticsearchParseException("failed to parse source for rollover index", e);
-            }
-        } else {
-            throw new ElasticsearchParseException("failed to parse content type for rollover index source");
-        }
-    }
-
     /**
      * Sets the number of shard copies that should be active for creation of the
      * new rollover index to return. Defaults to {@link ActiveShardCount#DEFAULT}, which will
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java
index b495e3c6a0f32..8c1be3501a820 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java
@@ -22,7 +22,7 @@
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
 import java.io.IOException;
@@ -32,7 +32,7 @@
 import java.util.Set;
 import java.util.stream.Collectors;
 
-public final class RolloverResponse extends ActionResponse implements ToXContent {
+public final class RolloverResponse extends ActionResponse implements ToXContentObject {
 
     private static final String NEW_INDEX = "new_index";
     private static final String OLD_INDEX = "old_index";
@@ -157,6 +157,7 @@ public void writeTo(StreamOutput out) throws IOException {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
         builder.field(OLD_INDEX, oldIndex);
         builder.field(NEW_INDEX, newIndex);
         builder.field(ROLLED_OVER, rolledOver);
@@ -168,6 +169,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.field(entry.getKey(), entry.getValue());
         }
         builder.endObject();
+        builder.endObject();
         return builder;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
index eaf9025bf0440..2abe0dad74ee3 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
@@ -61,7 +61,7 @@
  */
 public class TransportRolloverAction extends TransportMasterNodeAction<RolloverRequest, RolloverResponse> {
 
-    private static final Pattern INDEX_NAME_PATTERN = Pattern.compile("^.*-(\\d)+$");
+    private static final Pattern INDEX_NAME_PATTERN = Pattern.compile("^.*-\\d+$");
     private final MetaDataCreateIndexService createIndexService;
     private final MetaDataIndexAliasesService indexAliasesService;
     private final ActiveShardsObserver activeShardsObserver;
@@ -106,23 +106,29 @@ protected void masterOperation(final RolloverRequest rolloverRequest, final Clus
         validate(metaData, rolloverRequest);
         final AliasOrIndex aliasOrIndex = metaData.getAliasAndIndexLookup().get(rolloverRequest.getAlias());
         final IndexMetaData indexMetaData = aliasOrIndex.getIndices().get(0);
+        final String sourceProvidedName = indexMetaData.getSettings().get(IndexMetaData.SETTING_INDEX_PROVIDED_NAME,
+            indexMetaData.getIndex().getName());
         final String sourceIndexName = indexMetaData.getIndex().getName();
+        final String unresolvedName = (rolloverRequest.getNewIndexName() != null)
+            ? rolloverRequest.getNewIndexName()
+            : generateRolloverIndexName(sourceProvidedName, indexNameExpressionResolver);
+        final String rolloverIndexName = indexNameExpressionResolver.resolveDateMathExpression(unresolvedName);
+        MetaDataCreateIndexService.validateIndexName(rolloverIndexName, state); // will fail if the index already exists
         client.admin().indices().prepareStats(sourceIndexName).clear().setDocs(true).execute(
             new ActionListener<IndicesStatsResponse>() {
                 @Override
                 public void onResponse(IndicesStatsResponse statsResponse) {
                     final Set<Condition.Result> conditionResults = evaluateConditions(rolloverRequest.getConditions(),
-                        statsResponse.getTotal().getDocs(), metaData.index(sourceIndexName));
-                    final String rolloverIndexName = (rolloverRequest.getNewIndexName() != null)
-                        ? rolloverRequest.getNewIndexName()
-                        : generateRolloverIndexName(sourceIndexName);
+                        metaData.index(sourceIndexName), statsResponse);
+
                     if (rolloverRequest.isDryRun()) {
                         listener.onResponse(
                             new RolloverResponse(sourceIndexName, rolloverIndexName, conditionResults, true, false, false, false));
                         return;
                     }
                     if (conditionResults.size() == 0 || conditionResults.stream().anyMatch(result -> result.matched)) {
-                        CreateIndexClusterStateUpdateRequest updateRequest = prepareCreateIndexRequest(rolloverIndexName, rolloverRequest);
+                        CreateIndexClusterStateUpdateRequest updateRequest = prepareCreateIndexRequest(unresolvedName, rolloverIndexName,
+                            rolloverRequest);
                         createIndexService.createIndex(updateRequest, ActionListener.wrap(createIndexClusterStateUpdateResponse -> {
                             // switch the alias to point to the newly created index
                             indexAliasesService.indicesAliases(
@@ -145,7 +151,7 @@ public void onResponse(IndicesStatsResponse statsResponse) {
                     } else {
                         // conditions not met
                         listener.onResponse(
-                            new RolloverResponse(sourceIndexName, sourceIndexName, conditionResults, false, false, false, false)
+                            new RolloverResponse(sourceIndexName, rolloverIndexName, conditionResults, false, false, false, false)
                         );
                     }
                 }
@@ -170,14 +176,19 @@ static IndicesAliasesClusterStateUpdateRequest prepareRolloverAliasesUpdateReque
     }
 
 
-    static String generateRolloverIndexName(String sourceIndexName) {
-        if (INDEX_NAME_PATTERN.matcher(sourceIndexName).matches()) {
+    static String generateRolloverIndexName(String sourceIndexName, IndexNameExpressionResolver indexNameExpressionResolver) {
+        String resolvedName = indexNameExpressionResolver.resolveDateMathExpression(sourceIndexName);
+        final boolean isDateMath = sourceIndexName.equals(resolvedName) == false;
+        if (INDEX_NAME_PATTERN.matcher(resolvedName).matches()) {
             int numberIndex = sourceIndexName.lastIndexOf("-");
             assert numberIndex != -1 : "no separator '-' found";
-            int counter = Integer.parseInt(sourceIndexName.substring(numberIndex + 1));
-            return String.join("-", sourceIndexName.substring(0, numberIndex), String.format(Locale.ROOT, "%06d", ++counter));
+            int counter = Integer.parseInt(sourceIndexName.substring(numberIndex + 1, isDateMath ? sourceIndexName.length()-1 :
+                sourceIndexName.length()));
+            String newName = sourceIndexName.substring(0, numberIndex) + "-" + String.format(Locale.ROOT, "%06d", ++counter)
+                + (isDateMath ? ">" : "");
+            return newName;
         } else {
-            throw new IllegalArgumentException("index name [" + sourceIndexName + "] does not match pattern '^.*-(\\d)+$'");
+            throw new IllegalArgumentException("index name [" + sourceIndexName + "] does not match pattern '^.*-\\d+$'");
         }
     }
 
@@ -190,6 +201,11 @@ static Set<Condition.Result> evaluateConditions(final Set<Condition> conditions,
             .collect(Collectors.toSet());
     }
 
+    static Set<Condition.Result> evaluateConditions(final Set<Condition> conditions, final IndexMetaData metaData,
+                                                    final IndicesStatsResponse statsResponse) {
+        return evaluateConditions(conditions, statsResponse.getPrimaries().getDocs(), metaData);
+    }
+
     static void validate(MetaData metaData, RolloverRequest request) {
         final AliasOrIndex aliasOrIndex = metaData.getAliasAndIndexLookup().get(request.getAlias());
         if (aliasOrIndex == null) {
@@ -203,14 +219,14 @@ static void validate(MetaData metaData, RolloverRequest request) {
         }
     }
 
-    static CreateIndexClusterStateUpdateRequest prepareCreateIndexRequest(final String targetIndexName,
+    static CreateIndexClusterStateUpdateRequest prepareCreateIndexRequest(final String providedIndexName, final String targetIndexName,
                                                                           final RolloverRequest rolloverRequest) {
 
         final CreateIndexRequest createIndexRequest = rolloverRequest.getCreateIndexRequest();
         createIndexRequest.cause("rollover_index");
         createIndexRequest.index(targetIndexName);
         return new CreateIndexClusterStateUpdateRequest(createIndexRequest,
-            "rollover_index", targetIndexName, true)
+            "rollover_index", targetIndexName, providedIndexName, true)
             .ackTimeout(createIndexRequest.timeout())
             .masterNodeTimeout(createIndexRequest.masterNodeTimeout())
             .settings(createIndexRequest.settings())
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java
index f9ebff0663617..8ae075c66340a 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java
@@ -65,7 +65,10 @@ protected ClusterBlockException checkBlock(UpdateSettingsRequest request, Cluste
         if (globalBlock != null) {
             return globalBlock;
         }
-        if (request.settings().getAsMap().size() == 1 && IndexMetaData.INDEX_BLOCKS_METADATA_SETTING.exists(request.settings()) || IndexMetaData.INDEX_READ_ONLY_SETTING.exists(request.settings())) {
+        if (request.settings().size() == 1 &&  // we have to allow resetting these settings otherwise users can't unblock an index
+            IndexMetaData.INDEX_BLOCKS_METADATA_SETTING.exists(request.settings())
+            || IndexMetaData.INDEX_READ_ONLY_SETTING.exists(request.settings())
+            || IndexMetaData.INDEX_BLOCKS_READ_ONLY_ALLOW_DELETE_SETTING.exists(request.settings())) {
             return null;
         }
         return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_WRITE, indexNameExpressionResolver.concreteIndexNames(state, request));
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequest.java
index 494b9df7bd37b..80f3fb1a0718d 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequest.java
@@ -70,7 +70,7 @@ public UpdateSettingsRequest(Settings settings, String... indices) {
     @Override
     public ActionRequestValidationException validate() {
         ActionRequestValidationException validationException = null;
-        if (settings.getAsMap().isEmpty()) {
+        if (settings.isEmpty()) {
             validationException = addValidationError("no settings to update", validationException);
         }
         return validationException;
@@ -121,13 +121,23 @@ public UpdateSettingsRequest settings(Settings.Builder settings) {
     }
 
     /**
-     * Sets the settings to be updated (either json/yaml/properties format)
+     * Sets the settings to be updated (either json or yaml format)
+     * @deprecated use {@link #settings(String, XContentType)} to avoid content type detection
      */
+    @Deprecated
     public UpdateSettingsRequest settings(String source) {
         this.settings = Settings.builder().loadFromSource(source).build();
         return this;
     }
 
+    /**
+     * Sets the settings to be updated (either json or yaml format)
+     */
+    public UpdateSettingsRequest settings(String source, XContentType xContentType) {
+        this.settings = Settings.builder().loadFromSource(source, xContentType).build();
+        return this;
+    }
+
     /**
      * Returns <code>true</code> iff the settings update should only add but not update settings. If the setting already exists
      * it should not be overwritten by this update. The default is <code>false</code>
@@ -146,14 +156,14 @@ public UpdateSettingsRequest setPreserveExisting(boolean preserveExisting) {
     }
 
     /**
-     * Sets the settings to be updated (either json/yaml/properties format)
+     * Sets the settings to be updated (either json or yaml format)
      */
     @SuppressWarnings("unchecked")
     public UpdateSettingsRequest settings(Map source) {
         try {
             XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
             builder.map(source);
-            settings(builder.string());
+            settings(builder.string(), builder.contentType());
         } catch (IOException e) {
             throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e);
         }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequestBuilder.java
index 36dfbf3b2d49b..a9cecbfc5a434 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequestBuilder.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 
 import java.util.Map;
 
@@ -70,15 +71,25 @@ public UpdateSettingsRequestBuilder setSettings(Settings.Builder settings) {
     }
 
     /**
-     * Sets the settings to be updated (either json/yaml/properties format)
+     * Sets the settings to be updated (either json or yaml format)
+     * @deprecated use {@link #setSettings(String, XContentType)} to avoid content type detection
      */
+    @Deprecated
     public UpdateSettingsRequestBuilder setSettings(String source) {
         request.settings(source);
         return this;
     }
 
     /**
-     * Sets the settings to be updated (either json/yaml/properties format)
+     * Sets the settings to be updated (either json or yaml format)
+     */
+    public UpdateSettingsRequestBuilder setSettings(String source, XContentType xContentType) {
+        request.settings(source, xContentType);
+        return this;
+    }
+
+    /**
+     * Sets the settings to be updated
      */
     public UpdateSettingsRequestBuilder setSettings(Map<String, Object> source) {
         request.settings(source);
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresResponse.java
index f1df6d53e185d..3657e3272658f 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresResponse.java
@@ -207,7 +207,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.field(Fields.ALLOCATED, allocationStatus.value());
             if (storeException != null) {
                 builder.startObject(Fields.STORE_EXCEPTION);
-                ElasticsearchException.toXContent(builder, params, storeException);
+                ElasticsearchException.generateThrowableXContent(builder, params, storeException);
                 builder.endObject();
             }
             return builder;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java
index e13578d66de63..27b89cd8c6297 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java
@@ -29,7 +29,6 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.health.ClusterShardHealth;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -155,7 +154,7 @@ private class InternalAsyncFetch extends AsyncShardFetch<NodeGatewayStartedShard
             }
 
             @Override
-            protected synchronized void processAsyncFetch(ShardId shardId, List<NodeGatewayStartedShards> responses, List<FailedNodeException> failures) {
+            protected synchronized void processAsyncFetch(List<NodeGatewayStartedShards> responses, List<FailedNodeException> failures, long fetchingRound) {
                 fetchResponses.add(new Response(shardId, responses, failures));
                 if (expectedOps.countDown()) {
                     finish();
@@ -226,7 +225,7 @@ public class Response {
                 private final List<NodeGatewayStartedShards> responses;
                 private final List<FailedNodeException> failures;
 
-                public Response(ShardId shardId, List<NodeGatewayStartedShards> responses, List<FailedNodeException> failures) {
+                Response(ShardId shardId, List<NodeGatewayStartedShards> responses, List<FailedNodeException> failures) {
                     this.shardId = shardId;
                     this.responses = responses;
                     this.failures = failures;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkRequest.java
index 9cb60415a12cc..faa0a63c54dcf 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkRequest.java
@@ -18,7 +18,6 @@
  */
 package org.elasticsearch.action.admin.indices.shrink;
 
-import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
@@ -26,15 +25,9 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ObjectParser;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentType;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -46,14 +39,11 @@
  */
 public class ShrinkRequest extends AcknowledgedRequest<ShrinkRequest> implements IndicesRequest {
 
-    public static ObjectParser<ShrinkRequest, ParseFieldMatcherSupplier> PARSER =
-        new ObjectParser<>("shrink_request", null);
+    public static final ObjectParser<ShrinkRequest, Void> PARSER = new ObjectParser<>("shrink_request", null);
     static {
-        PARSER.declareField((parser, request, parseFieldMatcherSupplier) ->
-                request.getShrinkIndexRequest().settings(parser.map()),
+        PARSER.declareField((parser, request, context) -> request.getShrinkIndexRequest().settings(parser.map()),
             new ParseField("settings"), ObjectParser.ValueType.OBJECT);
-        PARSER.declareField((parser, request, parseFieldMatcherSupplier) ->
-                request.getShrinkIndexRequest().aliases(parser.map()),
+        PARSER.declareField((parser, request, context) -> request.getShrinkIndexRequest().aliases(parser.map()),
             new ParseField("aliases"), ObjectParser.ValueType.OBJECT);
     }
 
@@ -152,17 +142,4 @@ public void setWaitForActiveShards(ActiveShardCount waitForActiveShards) {
     public void setWaitForActiveShards(final int waitForActiveShards) {
         setWaitForActiveShards(ActiveShardCount.from(waitForActiveShards));
     }
-
-    public void source(BytesReference source) {
-        XContentType xContentType = XContentFactory.xContentType(source);
-        if (xContentType != null) {
-            try (XContentParser parser = XContentFactory.xContent(xContentType).createParser(source)) {
-                PARSER.parse(parser, this, () -> ParseFieldMatcher.EMPTY);
-            } catch (IOException e) {
-                throw new ElasticsearchParseException("failed to parse source for shrink index", e);
-            }
-        } else {
-            throw new ElasticsearchParseException("failed to parse content type for shrink index source");
-        }
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkResponse.java
index e7ad0afe3aa17..0c5149f6bf353 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkResponse.java
@@ -25,7 +25,7 @@ public final class ShrinkResponse extends CreateIndexResponse {
     ShrinkResponse() {
     }
 
-    ShrinkResponse(boolean acknowledged, boolean shardsAcked) {
-        super(acknowledged, shardsAcked);
+    ShrinkResponse(boolean acknowledged, boolean shardsAcked, String index) {
+        super(acknowledged, shardsAcked, index);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkAction.java
index 4667f1e98255a..2555299709cda 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkAction.java
@@ -91,8 +91,13 @@ public void onResponse(IndicesStatsResponse indicesStatsResponse) {
                         IndexShardStats shard = indicesStatsResponse.getIndex(sourceIndex).getIndexShards().get(i);
                         return shard == null ? null : shard.getPrimary().getDocs();
                     }, indexNameExpressionResolver);
-                createIndexService.createIndex(updateRequest, ActionListener.wrap(response ->
-                    listener.onResponse(new ShrinkResponse(response.isAcknowledged(), response.isShardsAcked())), listener::onFailure));
+                createIndexService.createIndex(
+                    updateRequest,
+                    ActionListener.wrap(response ->
+                        listener.onResponse(new ShrinkResponse(response.isAcknowledged(), response.isShardsAcked(), updateRequest.index())),
+                        listener::onFailure
+                    )
+                );
             }
 
             @Override
@@ -104,10 +109,10 @@ public void onFailure(Exception e) {
     }
 
     // static for unittesting this method
-    static CreateIndexClusterStateUpdateRequest prepareCreateIndexRequest(final ShrinkRequest shrinkReqeust, final ClusterState state
+    static CreateIndexClusterStateUpdateRequest prepareCreateIndexRequest(final ShrinkRequest shrinkRequest, final ClusterState state
         , final IntFunction<DocsStats> perShardDocStats, IndexNameExpressionResolver indexNameExpressionResolver) {
-        final String sourceIndex = indexNameExpressionResolver.resolveDateMathExpression(shrinkReqeust.getSourceIndex());
-        final CreateIndexRequest targetIndex = shrinkReqeust.getShrinkIndexRequest();
+        final String sourceIndex = indexNameExpressionResolver.resolveDateMathExpression(shrinkRequest.getSourceIndex());
+        final CreateIndexRequest targetIndex = shrinkRequest.getShrinkIndexRequest();
         final String targetIndexName = indexNameExpressionResolver.resolveDateMathExpression(targetIndex.index());
         final IndexMetaData metaData = state.metaData().index(sourceIndex);
         final Settings targetIndexSettings = Settings.builder().put(targetIndex.settings())
@@ -131,13 +136,16 @@ static CreateIndexClusterStateUpdateRequest prepareCreateIndexRequest(final Shri
             }
 
         }
+        if (IndexMetaData.INDEX_ROUTING_PARTITION_SIZE_SETTING.exists(targetIndexSettings)) {
+            throw new IllegalArgumentException("cannot provide a routing partition size value when shrinking an index");
+        }
         targetIndex.cause("shrink_index");
         Settings.Builder settingsBuilder = Settings.builder().put(targetIndexSettings);
         settingsBuilder.put("index.number_of_shards", numShards);
         targetIndex.settings(settingsBuilder);
 
         return new CreateIndexClusterStateUpdateRequest(targetIndex,
-            "shrink_index", targetIndexName, true)
+            "shrink_index", targetIndex.index(), targetIndexName, true)
             // mappings are updated on the node when merging in the shards, this prevents race-conditions since all mapping must be
             // applied once we took the snapshot and if somebody fucks things up and switches the index read/write and adds docs we miss
             // the mappings for everything is corrupted and hard to debug
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java
index ce90858f49a9d..b5e91ddf2a104 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java
@@ -46,6 +46,9 @@
 import org.elasticsearch.search.suggest.completion.CompletionStats;
 
 import java.io.IOException;
+import java.util.Arrays;
+import java.util.Objects;
+import java.util.stream.Stream;
 
 public class CommonStats implements Writeable, ToXContent {
 
@@ -225,45 +228,19 @@ public CommonStats(IndicesQueryCache indicesQueryCache, IndexShard indexShard, C
     }
 
     public CommonStats(StreamInput in) throws IOException {
-        if (in.readBoolean()) {
-            docs = DocsStats.readDocStats(in);
-        }
-        if (in.readBoolean()) {
-            store = StoreStats.readStoreStats(in);
-        }
-        if (in.readBoolean()) {
-            indexing = IndexingStats.readIndexingStats(in);
-        }
-        if (in.readBoolean()) {
-            get = GetStats.readGetStats(in);
-        }
-        if (in.readBoolean()) {
-            search = SearchStats.readSearchStats(in);
-        }
-        if (in.readBoolean()) {
-            merge = MergeStats.readMergeStats(in);
-        }
-        if (in.readBoolean()) {
-            refresh = RefreshStats.readRefreshStats(in);
-        }
-        if (in.readBoolean()) {
-            flush = FlushStats.readFlushStats(in);
-        }
-        if (in.readBoolean()) {
-            warmer = WarmerStats.readWarmerStats(in);
-        }
-        if (in.readBoolean()) {
-            queryCache = QueryCacheStats.readQueryCacheStats(in);
-        }
-        if (in.readBoolean()) {
-            fieldData = FieldDataStats.readFieldDataStats(in);
-        }
-        if (in.readBoolean()) {
-            completion = CompletionStats.readCompletionStats(in);
-        }
-        if (in.readBoolean()) {
-            segments = SegmentsStats.readSegmentsStats(in);
-        }
+        docs = in.readOptionalStreamable(DocsStats::new);
+        store = in.readOptionalStreamable(StoreStats::new);
+        indexing = in.readOptionalStreamable(IndexingStats::new);
+        get = in.readOptionalStreamable(GetStats::new);
+        search = in.readOptionalStreamable(SearchStats::new);
+        merge = in.readOptionalStreamable(MergeStats::new);
+        refresh =  in.readOptionalStreamable(RefreshStats::new);
+        flush =  in.readOptionalStreamable(FlushStats::new);
+        warmer =  in.readOptionalStreamable(WarmerStats::new);
+        queryCache = in.readOptionalStreamable(QueryCacheStats::new);
+        fieldData =  in.readOptionalStreamable(FieldDataStats::new);
+        completion =  in.readOptionalStreamable(CompletionStats::new);
+        segments =  in.readOptionalStreamable(SegmentsStats::new);
         translog = in.readOptionalStreamable(TranslogStats::new);
         requestCache = in.readOptionalStreamable(RequestCacheStats::new);
         recoveryStats = in.readOptionalStreamable(RecoveryStats::new);
@@ -271,84 +248,19 @@ public CommonStats(StreamInput in) throws IOException {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        if (docs == null) {
-            out.writeBoolean(false);
-        } else {
-            out.writeBoolean(true);
-            docs.writeTo(out);
-        }
-        if (store == null) {
-            out.writeBoolean(false);
-        } else {
-            out.writeBoolean(true);
-            store.writeTo(out);
-        }
-        if (indexing == null) {
-            out.writeBoolean(false);
-        } else {
-            out.writeBoolean(true);
-            indexing.writeTo(out);
-        }
-        if (get == null) {
-            out.writeBoolean(false);
-        } else {
-            out.writeBoolean(true);
-            get.writeTo(out);
-        }
-        if (search == null) {
-            out.writeBoolean(false);
-        } else {
-            out.writeBoolean(true);
-            search.writeTo(out);
-        }
-        if (merge == null) {
-            out.writeBoolean(false);
-        } else {
-            out.writeBoolean(true);
-            merge.writeTo(out);
-        }
-        if (refresh == null) {
-            out.writeBoolean(false);
-        } else {
-            out.writeBoolean(true);
-            refresh.writeTo(out);
-        }
-        if (flush == null) {
-            out.writeBoolean(false);
-        } else {
-            out.writeBoolean(true);
-            flush.writeTo(out);
-        }
-        if (warmer == null) {
-            out.writeBoolean(false);
-        } else {
-            out.writeBoolean(true);
-            warmer.writeTo(out);
-        }
-        if (queryCache == null) {
-            out.writeBoolean(false);
-        } else {
-            out.writeBoolean(true);
-            queryCache.writeTo(out);
-        }
-        if (fieldData == null) {
-            out.writeBoolean(false);
-        } else {
-            out.writeBoolean(true);
-            fieldData.writeTo(out);
-        }
-        if (completion == null) {
-            out.writeBoolean(false);
-        } else {
-            out.writeBoolean(true);
-            completion.writeTo(out);
-        }
-        if (segments == null) {
-            out.writeBoolean(false);
-        } else {
-            out.writeBoolean(true);
-            segments.writeTo(out);
-        }
+        out.writeOptionalStreamable(docs);
+        out.writeOptionalStreamable(store);
+        out.writeOptionalStreamable(indexing);
+        out.writeOptionalStreamable(get);
+        out.writeOptionalStreamable(search);
+        out.writeOptionalStreamable(merge);
+        out.writeOptionalStreamable(refresh);
+        out.writeOptionalStreamable(flush);
+        out.writeOptionalStreamable(warmer);
+        out.writeOptionalStreamable(queryCache);
+        out.writeOptionalStreamable(fieldData);
+        out.writeOptionalStreamable(completion);
+        out.writeOptionalStreamable(segments);
         out.writeOptionalStreamable(translog);
         out.writeOptionalStreamable(requestCache);
         out.writeOptionalStreamable(recoveryStats);
@@ -590,53 +502,12 @@ public ByteSizeValue getTotalMemory() {
     // note, requires a wrapping object
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        if (docs != null) {
-            docs.toXContent(builder, params);
-        }
-        if (store != null) {
-            store.toXContent(builder, params);
-        }
-        if (indexing != null) {
-            indexing.toXContent(builder, params);
-        }
-        if (get != null) {
-            get.toXContent(builder, params);
-        }
-        if (search != null) {
-            search.toXContent(builder, params);
-        }
-        if (merge != null) {
-            merge.toXContent(builder, params);
-        }
-        if (refresh != null) {
-            refresh.toXContent(builder, params);
-        }
-        if (flush != null) {
-            flush.toXContent(builder, params);
-        }
-        if (warmer != null) {
-            warmer.toXContent(builder, params);
-        }
-        if (queryCache != null) {
-            queryCache.toXContent(builder, params);
-        }
-        if (fieldData != null) {
-            fieldData.toXContent(builder, params);
-        }
-        if (completion != null) {
-            completion.toXContent(builder, params);
-        }
-        if (segments != null) {
-            segments.toXContent(builder, params);
-        }
-        if (translog != null) {
-            translog.toXContent(builder, params);
-        }
-        if (requestCache != null) {
-            requestCache.toXContent(builder, params);
-        }
-        if (recoveryStats != null) {
-            recoveryStats.toXContent(builder, params);
+        final Stream<ToXContent> stream = Arrays.stream(new ToXContent[] {
+            docs, store, indexing, get, search, merge, refresh, flush, warmer, queryCache,
+            fieldData, completion, segments, translog, requestCache, recoveryStats})
+            .filter(Objects::nonNull);
+        for (ToXContent toXContent : ((Iterable<ToXContent>)stream::iterator)) {
+            toXContent.toXContent(builder, params);
         }
         return builder;
     }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponse.java
index 2caa0da956942..26c28da1ae63b 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponse.java
@@ -137,29 +137,25 @@ public CommonStats getPrimaries() {
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
-        shards = new ShardStats[in.readVInt()];
-        for (int i = 0; i < shards.length; i++) {
-            shards[i] = ShardStats.readShardStats(in);
-        }
+        shards = in.readArray(ShardStats::readShardStats, (size) -> new ShardStats[size]);
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        out.writeVInt(shards.length);
-        for (ShardStats shard : shards) {
-            shard.writeTo(out);
-        }
+        out.writeArray(shards);
     }
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        String level = params.param("level", "indices");
-        boolean isLevelValid = "indices".equalsIgnoreCase(level) || "shards".equalsIgnoreCase(level) || "cluster".equalsIgnoreCase(level);
+        final String level = params.param("level", "indices");
+        final boolean isLevelValid =
+            "cluster".equalsIgnoreCase(level) || "indices".equalsIgnoreCase(level) || "shards".equalsIgnoreCase(level);
         if (!isLevelValid) {
-            return builder;
+            throw new IllegalArgumentException("level parameter must be one of [cluster] or [indices] or [shards] but was [" + level + "]");
         }
 
+
         builder.startObject("_all");
 
         builder.startObject("primaries");
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java
index 5bc6ce8106459..794df89d233aa 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.engine.CommitStats;
@@ -31,9 +32,7 @@
 
 import java.io.IOException;
 
-/**
- */
-public class ShardStats implements Streamable, ToXContent {
+public class ShardStats implements Streamable, Writeable, ToXContent {
     private ShardRouting shardRouting;
     private CommonStats commonStats;
     @Nullable
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/template/delete/DeleteIndexTemplateResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/template/delete/DeleteIndexTemplateResponse.java
index 5c2a2b166bc3c..9519f0f9fcf4f 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/template/delete/DeleteIndexTemplateResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/template/delete/DeleteIndexTemplateResponse.java
@@ -32,7 +32,7 @@ public class DeleteIndexTemplateResponse extends AcknowledgedResponse {
     DeleteIndexTemplateResponse() {
     }
 
-    DeleteIndexTemplateResponse(boolean acknowledged) {
+    protected DeleteIndexTemplateResponse(boolean acknowledged) {
         super(acknowledged);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java
index 7693419df436d..3c5fb36d6c6aa 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java
@@ -23,17 +23,16 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 
 import static java.util.Collections.singletonMap;
 
-public class GetIndexTemplatesResponse extends ActionResponse implements ToXContent {
+public class GetIndexTemplatesResponse extends ActionResponse implements ToXContentObject {
 
     private List<IndexTemplateMetaData> indexTemplates;
 
@@ -54,7 +53,7 @@ public void readFrom(StreamInput in) throws IOException {
         int size = in.readVInt();
         indexTemplates = new ArrayList<>(size);
         for (int i = 0 ; i < size ; i++) {
-            indexTemplates.add(0, IndexTemplateMetaData.Builder.readFrom(in));
+            indexTemplates.add(0, IndexTemplateMetaData.readFrom(in));
         }
     }
 
@@ -70,10 +69,11 @@ public void writeTo(StreamOutput out) throws IOException {
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         params = new ToXContent.DelegatingMapParams(singletonMap("reduce_mappings", "true"), params);
-
+        builder.startObject();
         for (IndexTemplateMetaData indexTemplateMetaData : getIndexTemplates()) {
             IndexTemplateMetaData.Builder.toXContent(indexTemplateMetaData, builder, params);
         }
+        builder.endObject();
         return builder;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java
index 76c665c7b8fef..ccb02c664acc9 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java
@@ -20,6 +20,7 @@
 
 import org.elasticsearch.ElasticsearchGenerationException;
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.admin.indices.alias.Alias;
@@ -33,6 +34,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentHelper;
@@ -41,9 +43,11 @@
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 
 import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
@@ -74,6 +78,8 @@ public class PutIndexTemplateRequest extends MasterNodeRequest<PutIndexTemplateR
 
     private Map<String, IndexMetaData.Custom> customs = new HashMap<>();
 
+    private Integer version;
+
     public PutIndexTemplateRequest() {
     }
 
@@ -129,9 +135,18 @@ public int order() {
         return this.order;
     }
 
+    public PutIndexTemplateRequest version(Integer version) {
+        this.version = version;
+        return this;
+    }
+
+    public Integer version() {
+        return this.version;
+    }
+
     /**
      * Set to <tt>true</tt> to force only creation, not an update of an index template. If it already
-     * exists, it will fail with an {@link org.elasticsearch.indices.IndexTemplateAlreadyExistsException}.
+     * exists, it will fail with an {@link IllegalArgumentException}.
      */
     public PutIndexTemplateRequest create(boolean create) {
         this.create = create;
@@ -159,21 +174,31 @@ public PutIndexTemplateRequest settings(Settings.Builder settings) {
     }
 
     /**
-     * The settings to create the index template with (either json/yaml/properties format).
+     * The settings to create the index template with (either json/yaml format).
+     * @deprecated use {@link #settings(String, XContentType)}
      */
+    @Deprecated
     public PutIndexTemplateRequest settings(String source) {
         this.settings = Settings.builder().loadFromSource(source).build();
         return this;
     }
 
     /**
-     * The settings to crete the index template with (either json/yaml/properties format).
+     * The settings to create the index template with (either json/yaml format).
+     */
+    public PutIndexTemplateRequest settings(String source, XContentType xContentType) {
+        this.settings = Settings.builder().loadFromSource(source, xContentType).build();
+        return this;
+    }
+
+    /**
+     * The settings to create the index template with (either json or yaml format).
      */
     public PutIndexTemplateRequest settings(Map<String, Object> source) {
         try {
             XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
             builder.map(source);
-            settings(builder.string());
+            settings(builder.string(), XContentType.JSON);
         } catch (IOException e) {
             throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e);
         }
@@ -189,10 +214,23 @@ public Settings settings() {
      *
      * @param type   The mapping type
      * @param source The mapping source
+     * @deprecated use {@link #mapping(String, String, XContentType)}
      */
+    @Deprecated
     public PutIndexTemplateRequest mapping(String type, String source) {
-        mappings.put(type, source);
-        return this;
+        XContentType xContentType = XContentFactory.xContentType(source);
+        return mapping(type, source, xContentType);
+    }
+
+    /**
+     * Adds mapping that will be added when the index gets created.
+     *
+     * @param type   The mapping type
+     * @param source The mapping source
+     * @param xContentType The type of content contained within the source
+     */
+    public PutIndexTemplateRequest mapping(String type, String source, XContentType xContentType) {
+        return mapping(type, new BytesArray(source), xContentType);
     }
 
     /**
@@ -214,12 +252,24 @@ public String cause() {
      * @param source The mapping source
      */
     public PutIndexTemplateRequest mapping(String type, XContentBuilder source) {
+        return mapping(type, source.bytes(), source.contentType());
+    }
+
+    /**
+     * Adds mapping that will be added when the index gets created.
+     *
+     * @param type   The mapping type
+     * @param source The mapping source
+     * @param xContentType the source content type
+     */
+    public PutIndexTemplateRequest mapping(String type, BytesReference source, XContentType xContentType) {
+        Objects.requireNonNull(xContentType);
         try {
-            mappings.put(type, source.string());
+            mappings.put(type, XContentHelper.convertToJson(source, false, false, xContentType));
+            return this;
         } catch (IOException e) {
-            throw new IllegalArgumentException("Failed to build json for mapping request", e);
+            throw new UncheckedIOException("failed to convert source to json", e);
         }
-        return this;
     }
 
     /**
@@ -236,7 +286,7 @@ public PutIndexTemplateRequest mapping(String type, Map<String, Object> source)
         try {
             XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
             builder.map(source);
-            return mapping(type, builder.string());
+            return mapping(type, builder);
         } catch (IOException e) {
             throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e);
         }
@@ -260,7 +310,7 @@ public Map<String, String> mappings() {
      */
     public PutIndexTemplateRequest source(XContentBuilder templateBuilder) {
         try {
-            return source(templateBuilder.bytes());
+            return source(templateBuilder.bytes(), templateBuilder.contentType());
         } catch (Exception e) {
             throw new IllegalArgumentException("Failed to build json for template request", e);
         }
@@ -278,16 +328,23 @@ public PutIndexTemplateRequest source(Map templateSource) {
                 template(entry.getValue().toString());
             } else if (name.equals("order")) {
                 order(XContentMapValues.nodeIntegerValue(entry.getValue(), order()));
+            } else if ("version".equals(name)) {
+                if ((entry.getValue() instanceof Integer) == false) {
+                    throw new IllegalArgumentException("Malformed [version] value, should be an integer");
+                }
+                version((Integer)entry.getValue());
             } else if (name.equals("settings")) {
                 if (!(entry.getValue() instanceof Map)) {
-                    throw new IllegalArgumentException("Malformed settings section, should include an inner object");
+                    throw new IllegalArgumentException("Malformed [settings] section, should include an inner object");
                 }
                 settings((Map<String, Object>) entry.getValue());
             } else if (name.equals("mappings")) {
                 Map<String, Object> mappings = (Map<String, Object>) entry.getValue();
                 for (Map.Entry<String, Object> entry1 : mappings.entrySet()) {
                     if (!(entry1.getValue() instanceof Map)) {
-                        throw new IllegalArgumentException("Malformed mappings section for type [" + entry1.getKey() + "], should include an inner object describing the mapping");
+                        throw new IllegalArgumentException(
+                            "Malformed [mappings] section for type [" + entry1.getKey() +
+                                "], should include an inner object describing the mapping");
                     }
                     mapping(entry1.getKey(), (Map<String, Object>) entry1.getValue());
                 }
@@ -310,18 +367,25 @@ public PutIndexTemplateRequest source(Map templateSource) {
 
     /**
      * The template source definition.
+     * @deprecated use {@link #source(String, XContentType)}
      */
+    @Deprecated
     public PutIndexTemplateRequest source(String templateSource) {
-        try (XContentParser parser = XContentFactory.xContent(templateSource).createParser(templateSource)) {
-            return source(parser.mapOrdered());
-        } catch (Exception e) {
-            throw new IllegalArgumentException("failed to parse template source [" + templateSource + "]", e);
-        }
+        return source(XContentHelper.convertToMap(XContentFactory.xContent(templateSource), templateSource, true));
     }
 
     /**
      * The template source definition.
      */
+    public PutIndexTemplateRequest source(String templateSource, XContentType xContentType) {
+        return source(XContentHelper.convertToMap(xContentType.xContent(), templateSource, true));
+    }
+
+    /**
+     * The template source definition.
+     * @deprecated use {@link #source(byte[], XContentType)}
+     */
+    @Deprecated
     public PutIndexTemplateRequest source(byte[] source) {
         return source(source, 0, source.length);
     }
@@ -329,23 +393,40 @@ public PutIndexTemplateRequest source(byte[] source) {
     /**
      * The template source definition.
      */
+    public PutIndexTemplateRequest source(byte[] source, XContentType xContentType) {
+        return source(source, 0, source.length, xContentType);
+    }
+
+    /**
+     * The template source definition.
+     * @deprecated use {@link #source(byte[], int, int, XContentType)}
+     */
+    @Deprecated
     public PutIndexTemplateRequest source(byte[] source, int offset, int length) {
-        try (XContentParser parser = XContentFactory.xContent(source, offset, length).createParser(source, offset, length)) {
-            return source(parser.mapOrdered());
-        } catch (IOException e) {
-            throw new IllegalArgumentException("failed to parse template source", e);
-        }
+        return source(new BytesArray(source, offset, length));
     }
 
     /**
      * The template source definition.
      */
+    public PutIndexTemplateRequest source(byte[] source, int offset, int length, XContentType xContentType) {
+        return source(new BytesArray(source, offset, length), xContentType);
+    }
+
+    /**
+     * The template source definition.
+     * @deprecated use {@link #source(BytesReference, XContentType)}
+     */
+    @Deprecated
     public PutIndexTemplateRequest source(BytesReference source) {
-        try (XContentParser parser = XContentFactory.xContent(source).createParser(source)) {
-            return source(parser.mapOrdered());
-        } catch (IOException e) {
-            throw new IllegalArgumentException("failed to parse template source", e);
-        }
+        return source(XContentHelper.convertToMap(source, true).v2());
+    }
+
+    /**
+     * The template source definition.
+     */
+    public PutIndexTemplateRequest source(BytesReference source, XContentType xContentType) {
+        return source(XContentHelper.convertToMap(source, true, xContentType).v2());
     }
 
     public PutIndexTemplateRequest custom(IndexMetaData.Custom custom) {
@@ -393,7 +474,8 @@ public PutIndexTemplateRequest aliases(String source) {
      * Sets the aliases that will be associated with the index when it gets created
      */
     public PutIndexTemplateRequest aliases(BytesReference source) {
-        try (XContentParser parser = XContentHelper.createParser(source)) {
+        // EMPTY is safe here because we never call namedObject
+        try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, source)) {
             //move to the first alias
             parser.nextToken();
             while ((parser.nextToken()) != XContentParser.Token.END_OBJECT) {
@@ -437,7 +519,14 @@ public void readFrom(StreamInput in) throws IOException {
         settings = readSettingsFromStream(in);
         int size = in.readVInt();
         for (int i = 0; i < size; i++) {
-            mappings.put(in.readString(), in.readString());
+            final String type = in.readString();
+            String mappingSource = in.readString();
+            if (in.getVersion().before(Version.V_5_3_0)) {
+                // we do not know the incoming type so convert it if needed
+                mappingSource =
+                    XContentHelper.convertToJson(new BytesArray(mappingSource), false, false, XContentFactory.xContentType(mappingSource));
+            }
+            mappings.put(type, mappingSource);
         }
         int customSize = in.readVInt();
         for (int i = 0; i < customSize; i++) {
@@ -449,6 +538,7 @@ public void readFrom(StreamInput in) throws IOException {
         for (int i = 0; i < aliasesSize; i++) {
             aliases.add(Alias.read(in));
         }
+        version = in.readOptionalVInt();
     }
 
     @Override
@@ -474,5 +564,6 @@ public void writeTo(StreamOutput out) throws IOException {
         for (Alias alias : aliases) {
             alias.writeTo(out);
         }
+        out.writeOptionalVInt(version);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java
index 5207cacf6b080..0e4e17a956849 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java
@@ -24,13 +24,15 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentType;
 
 import java.util.Map;
 
 /**
  *
  */
-public class PutIndexTemplateRequestBuilder extends MasterNodeOperationRequestBuilder<PutIndexTemplateRequest, PutIndexTemplateResponse, PutIndexTemplateRequestBuilder> {
+public class PutIndexTemplateRequestBuilder
+    extends MasterNodeOperationRequestBuilder<PutIndexTemplateRequest, PutIndexTemplateResponse, PutIndexTemplateRequestBuilder> {
 
     public PutIndexTemplateRequestBuilder(ElasticsearchClient client, PutIndexTemplateAction action) {
         super(client, action, new PutIndexTemplateRequest());
@@ -56,9 +58,17 @@ public PutIndexTemplateRequestBuilder setOrder(int order) {
         return this;
     }
 
+    /**
+     * Sets the optional version of this template.
+     */
+    public PutIndexTemplateRequestBuilder setVersion(Integer version) {
+        request.version(version);
+        return this;
+    }
+
     /**
      * Set to <tt>true</tt> to force only creation, not an update of an index template. If it already
-     * exists, it will fail with an {@link org.elasticsearch.indices.IndexTemplateAlreadyExistsException}.
+     * exists, it will fail with an {@link IllegalArgumentException}.
      */
     public PutIndexTemplateRequestBuilder setCreate(boolean create) {
         request.create(create);
@@ -82,15 +92,25 @@ public PutIndexTemplateRequestBuilder setSettings(Settings.Builder settings) {
     }
 
     /**
-     * The settings to crete the index template with (either json/yaml/properties format)
+     * The settings to crete the index template with (either json or yaml format)
+     * @deprecated use {@link #setSettings(String, XContentType)}
      */
+    @Deprecated
     public PutIndexTemplateRequestBuilder setSettings(String source) {
         request.settings(source);
         return this;
     }
 
     /**
-     * The settings to crete the index template with (either json/yaml/properties format)
+     * The settings to crete the index template with (either json or yaml format)
+     */
+    public PutIndexTemplateRequestBuilder setSettings(String source, XContentType xContentType) {
+        request.settings(source, xContentType);
+        return this;
+    }
+
+    /**
+     * The settings to crete the index template with (either json or yaml format)
      */
     public PutIndexTemplateRequestBuilder setSettings(Map<String, Object> source) {
         request.settings(source);
@@ -102,12 +122,26 @@ public PutIndexTemplateRequestBuilder setSettings(Map<String, Object> source) {
      *
      * @param type   The mapping type
      * @param source The mapping source
+     * @deprecated use {@link #addMapping(String, String, XContentType)}
      */
+    @Deprecated
     public PutIndexTemplateRequestBuilder addMapping(String type, String source) {
         request.mapping(type, source);
         return this;
     }
 
+    /**
+     * Adds mapping that will be added when the index template gets created.
+     *
+     * @param type   The mapping type
+     * @param source The mapping source
+     * @param xContentType The type/format of the source
+     */
+    public PutIndexTemplateRequestBuilder addMapping(String type, String source, XContentType xContentType) {
+        request.mapping(type, source, xContentType);
+        return this;
+    }
+
     /**
      * A specialized simplified mapping source method, takes the form of simple properties definition:
      * ("field1", "type=string,store=true").
@@ -208,7 +242,9 @@ public PutIndexTemplateRequestBuilder setSource(Map templateSource) {
 
     /**
      * The template source definition.
+     * @deprecated use {@link #setSource(BytesReference, XContentType)}
      */
+    @Deprecated
     public PutIndexTemplateRequestBuilder setSource(String templateSource) {
         request.source(templateSource);
         return this;
@@ -217,6 +253,16 @@ public PutIndexTemplateRequestBuilder setSource(String templateSource) {
     /**
      * The template source definition.
      */
+    public PutIndexTemplateRequestBuilder setSource(BytesReference templateSource, XContentType xContentType) {
+        request.source(templateSource, xContentType);
+        return this;
+    }
+
+    /**
+     * The template source definition.
+     * @deprecated use {@link #setSource(BytesReference, XContentType)}
+     */
+    @Deprecated
     public PutIndexTemplateRequestBuilder setSource(BytesReference templateSource) {
         request.source(templateSource);
         return this;
@@ -224,7 +270,9 @@ public PutIndexTemplateRequestBuilder setSource(BytesReference templateSource) {
 
     /**
      * The template source definition.
+     * @deprecated use {@link #setSource(byte[], XContentType)}
      */
+    @Deprecated
     public PutIndexTemplateRequestBuilder setSource(byte[] templateSource) {
         request.source(templateSource);
         return this;
@@ -233,8 +281,26 @@ public PutIndexTemplateRequestBuilder setSource(byte[] templateSource) {
     /**
      * The template source definition.
      */
+    public PutIndexTemplateRequestBuilder setSource(byte[] templateSource, XContentType xContentType) {
+        request.source(templateSource, xContentType);
+        return this;
+    }
+
+    /**
+     * The template source definition.
+     * @deprecated use {@link #setSource(byte[], int, int, XContentType)}
+     */
+    @Deprecated
     public PutIndexTemplateRequestBuilder setSource(byte[] templateSource, int offset, int length) {
         request.source(templateSource, offset, length);
         return this;
     }
+
+    /**
+     * The template source definition.
+     */
+    public PutIndexTemplateRequestBuilder setSource(byte[] templateSource, int offset, int length, XContentType xContentType) {
+        request.source(templateSource, offset, length, xContentType);
+        return this;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java
index 64a01b90ac479..77746b395e15e 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java
@@ -86,7 +86,8 @@ protected void masterOperation(final PutIndexTemplateRequest request, final Clus
                 .aliases(request.aliases())
                 .customs(request.customs())
                 .create(request.create())
-                .masterTimeout(request.masterNodeTimeout()),
+                .masterTimeout(request.masterNodeTimeout())
+                .version(request.version()),
 
                 new MetaDataIndexTemplateService.PutListener() {
                     @Override
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/QueryExplanation.java b/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/QueryExplanation.java
index ea145ba15bc52..042857a097eb5 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/QueryExplanation.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/QueryExplanation.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.action.admin.indices.validate.query;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
@@ -30,20 +31,26 @@
  */
 public class QueryExplanation  implements Streamable {
 
+    public static final int RANDOM_SHARD = -1;
+
     private String index;
-    
+
+    private int shard = RANDOM_SHARD;
+
     private boolean valid;
-    
+
     private String explanation;
-    
+
     private String error;
 
     QueryExplanation() {
-        
+
     }
-    
-    public QueryExplanation(String index, boolean valid, String explanation, String error) {
+
+    public QueryExplanation(String index, int shard, boolean valid, String explanation,
+                            String error) {
         this.index = index;
+        this.shard = shard;
         this.valid = valid;
         this.explanation = explanation;
         this.error = error;
@@ -53,6 +60,10 @@ public String getIndex() {
         return this.index;
     }
 
+    public int getShard() {
+        return this.shard;
+    }
+
     public boolean isValid() {
         return this.valid;
     }
@@ -68,6 +79,11 @@ public String getExplanation() {
     @Override
     public void readFrom(StreamInput in) throws IOException {
         index = in.readString();
+        if (in.getVersion().onOrAfter(Version.V_5_4_0)) {
+            shard = in.readInt();
+        } else {
+            shard = RANDOM_SHARD;
+        }
         valid = in.readBoolean();
         explanation = in.readOptionalString();
         error = in.readOptionalString();
@@ -76,6 +92,9 @@ public void readFrom(StreamInput in) throws IOException {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(index);
+        if (out.getVersion().onOrAfter(Version.V_5_4_0)) {
+            out.writeInt(shard);
+        }
         out.writeBoolean(valid);
         out.writeOptionalString(explanation);
         out.writeOptionalString(error);
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ShardValidateQueryRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ShardValidateQueryRequest.java
index 831ef6e106015..2ccf2f1bd3e21 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ShardValidateQueryRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ShardValidateQueryRequest.java
@@ -20,14 +20,15 @@
 package org.elasticsearch.action.admin.indices.validate.query;
 
 import org.elasticsearch.action.support.broadcast.BroadcastShardRequest;
-import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.search.internal.AliasFilter;
 
 import java.io.IOException;
+import java.util.Objects;
 
 /**
  * Internal validate request executed directly against a specific index shard.
@@ -39,21 +40,18 @@ public class ShardValidateQueryRequest extends BroadcastShardRequest {
     private boolean explain;
     private boolean rewrite;
     private long nowInMillis;
-
-    @Nullable
-    private String[] filteringAliases;
+    private AliasFilter filteringAliases;
 
     public ShardValidateQueryRequest() {
-
     }
 
-    ShardValidateQueryRequest(ShardId shardId, @Nullable String[] filteringAliases, ValidateQueryRequest request) {
+    public ShardValidateQueryRequest(ShardId shardId, AliasFilter filteringAliases, ValidateQueryRequest request) {
         super(shardId, request);
         this.query = request.query();
         this.types = request.types();
         this.explain = request.explain();
         this.rewrite = request.rewrite();
-        this.filteringAliases = filteringAliases;
+        this.filteringAliases = Objects.requireNonNull(filteringAliases, "filteringAliases must not be null");
         this.nowInMillis = request.nowInMillis;
     }
 
@@ -69,11 +67,11 @@ public boolean explain() {
         return this.explain;
     }
 
-    public boolean rewrite() { 
-        return this.rewrite; 
+    public boolean rewrite() {
+        return this.rewrite;
     }
 
-    public String[] filteringAliases() {
+    public AliasFilter filteringAliases() {
         return filteringAliases;
     }
 
@@ -93,14 +91,7 @@ public void readFrom(StreamInput in) throws IOException {
                 types[i] = in.readString();
             }
         }
-        int aliasesSize = in.readVInt();
-        if (aliasesSize > 0) {
-            filteringAliases = new String[aliasesSize];
-            for (int i = 0; i < aliasesSize; i++) {
-                filteringAliases[i] = in.readString();
-            }
-        }
-
+        filteringAliases = new AliasFilter(in);
         explain = in.readBoolean();
         rewrite = in.readBoolean();
         nowInMillis = in.readVLong();
@@ -110,20 +101,11 @@ public void readFrom(StreamInput in) throws IOException {
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         out.writeNamedWriteable(query);
-
         out.writeVInt(types.length);
         for (String type : types) {
             out.writeString(type);
         }
-        if (filteringAliases != null) {
-            out.writeVInt(filteringAliases.length);
-            for (String alias : filteringAliases) {
-                out.writeString(alias);
-            }
-        } else {
-            out.writeVInt(0);
-        }
-
+        filteringAliases.writeTo(out);
         out.writeBoolean(explain);
         out.writeBoolean(rewrite);
         out.writeVLong(nowInMillis);
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java
index d1405e92e1c0d..7f9f69a3aae39 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.action.admin.indices.validate.query;
 
-import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.action.ActionListener;
@@ -38,17 +37,12 @@
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.BigArrays;
-import org.elasticsearch.index.IndexService;
-import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.query.ParsedQuery;
 import org.elasticsearch.index.query.QueryShardException;
-import org.elasticsearch.index.shard.IndexShard;
-import org.elasticsearch.indices.IndicesService;
-import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.SearchService;
-import org.elasticsearch.search.fetch.FetchPhase;
-import org.elasticsearch.search.internal.DefaultSearchContext;
+import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.internal.ShardSearchLocalRequest;
 import org.elasticsearch.tasks.Task;
@@ -67,25 +61,15 @@
  */
 public class TransportValidateQueryAction extends TransportBroadcastAction<ValidateQueryRequest, ValidateQueryResponse, ShardValidateQueryRequest, ShardValidateQueryResponse> {
 
-    private final IndicesService indicesService;
-
-    private final ScriptService scriptService;
-
-    private final BigArrays bigArrays;
-
-    private final FetchPhase fetchPhase;
+    private final SearchService searchService;
 
     @Inject
     public TransportValidateQueryAction(Settings settings, ThreadPool threadPool, ClusterService clusterService,
-            TransportService transportService, IndicesService indicesService, ScriptService scriptService,
-            BigArrays bigArrays, ActionFilters actionFilters,
-            IndexNameExpressionResolver indexNameExpressionResolver, FetchPhase fetchPhase) {
+            TransportService transportService, SearchService searchService, ActionFilters actionFilters,
+            IndexNameExpressionResolver indexNameExpressionResolver) {
         super(settings, ValidateQueryAction.NAME, threadPool, clusterService, transportService, actionFilters,
                 indexNameExpressionResolver, ValidateQueryRequest::new, ShardValidateQueryRequest::new, ThreadPool.Names.SEARCH);
-        this.indicesService = indicesService;
-        this.scriptService = scriptService;
-        this.bigArrays = bigArrays;
-        this.fetchPhase = fetchPhase;
+        this.searchService = searchService;
     }
 
     @Override
@@ -96,8 +80,9 @@ protected void doExecute(Task task, ValidateQueryRequest request, ActionListener
 
     @Override
     protected ShardValidateQueryRequest newShardRequest(int numShards, ShardRouting shard, ValidateQueryRequest request) {
-        String[] filteringAliases = indexNameExpressionResolver.filteringAliases(clusterService.state(), shard.getIndexName(), request.indices());
-        return new ShardValidateQueryRequest(shard.shardId(), filteringAliases, request);
+        final AliasFilter aliasFilter = searchService.buildAliasFilter(clusterService.state(), shard.getIndexName(),
+            request.indices());
+        return new ShardValidateQueryRequest(shard.shardId(), aliasFilter, request);
     }
 
     @Override
@@ -107,8 +92,14 @@ protected ShardValidateQueryResponse newShardResponse() {
 
     @Override
     protected GroupShardsIterator shards(ClusterState clusterState, ValidateQueryRequest request, String[] concreteIndices) {
-        // Hard-code routing to limit request to a single shard, but still, randomize it...
-        Map<String, Set<String>> routingMap = indexNameExpressionResolver.resolveSearchRouting(clusterState, Integer.toString(Randomness.get().nextInt(1000)), request.indices());
+        final String routing;
+        if (request.allShards()) {
+            routing = null;
+        } else {
+            // Random routing to limit request to a single shard
+            routing = Integer.toString(Randomness.get().nextInt(1000));
+        }
+        Map<String, Set<String>> routingMap = indexNameExpressionResolver.resolveSearchRouting(clusterState, routing, request.indices());
         return clusterService.operationRouting().searchShards(clusterState, concreteIndices, routingMap, "_local");
     }
 
@@ -142,12 +133,13 @@ protected ValidateQueryResponse newResponse(ValidateQueryRequest request, Atomic
             } else {
                 ShardValidateQueryResponse validateQueryResponse = (ShardValidateQueryResponse) shardResponse;
                 valid = valid && validateQueryResponse.isValid();
-                if (request.explain() || request.rewrite()) {
+                if (request.explain() || request.rewrite() || request.allShards()) {
                     if (queryExplanations == null) {
                         queryExplanations = new ArrayList<>();
                     }
                     queryExplanations.add(new QueryExplanation(
                             validateQueryResponse.getIndex(),
+                            request.allShards() ? validateQueryResponse.getShardId().getId() : QueryExplanation.RANDOM_SHARD,
                             validateQueryResponse.isValid(),
                             validateQueryResponse.getExplanation(),
                             validateQueryResponse.getError()
@@ -160,30 +152,19 @@ protected ValidateQueryResponse newResponse(ValidateQueryRequest request, Atomic
     }
 
     @Override
-    protected ShardValidateQueryResponse shardOperation(ShardValidateQueryRequest request) {
-        IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex());
-        IndexShard indexShard = indexService.getShard(request.shardId().id());
-
+    protected ShardValidateQueryResponse shardOperation(ShardValidateQueryRequest request) throws IOException {
         boolean valid;
         String explanation = null;
         String error = null;
-        Engine.Searcher searcher = indexShard.acquireSearcher("validate_query");
-
-        DefaultSearchContext searchContext = new DefaultSearchContext(0,
-                new ShardSearchLocalRequest(request.types(), request.nowInMillis(), request.filteringAliases()), null, searcher,
-                indexService, indexShard, scriptService, bigArrays, threadPool.estimatedTimeInMillisCounter(),
-                parseFieldMatcher, SearchService.NO_TIMEOUT, fetchPhase);
-        SearchContext.setCurrent(searchContext);
+        ShardSearchLocalRequest shardSearchLocalRequest = new ShardSearchLocalRequest(request.shardId(), request.types(),
+            request.nowInMillis(), request.filteringAliases());
+        SearchContext searchContext = searchService.createSearchContext(shardSearchLocalRequest, SearchService.NO_TIMEOUT, null);
         try {
-            searchContext.parsedQuery(searchContext.getQueryShardContext().toQuery(request.query()));
-            searchContext.preProcess();
-
+            ParsedQuery parsedQuery = searchContext.getQueryShardContext().toQuery(request.query());
+            searchContext.parsedQuery(parsedQuery);
+            searchContext.preProcess(request.rewrite());
             valid = true;
-            if (request.rewrite()) {
-                explanation = getRewrittenQuery(searcher.searcher(), searchContext.query());
-            } else if (request.explain()) {
-                explanation = searchContext.filteredQuery().query().toString();
-            }
+            explanation = explain(searchContext, request.rewrite());
         } catch (QueryShardException|ParsingException e) {
             valid = false;
             error = e.getDetailedMessage();
@@ -191,19 +172,18 @@ protected ShardValidateQueryResponse shardOperation(ShardValidateQueryRequest re
             valid = false;
             error = e.getMessage();
         } finally {
-            searchContext.close();
-            SearchContext.removeCurrent();
+            Releasables.close(searchContext);
         }
 
         return new ShardValidateQueryResponse(request.shardId(), valid, explanation, error);
     }
 
-    private String getRewrittenQuery(IndexSearcher searcher, Query query) throws IOException {
-        Query queryRewrite = searcher.rewrite(query);
-        if (queryRewrite instanceof MatchNoDocsQuery) {
-            return query.toString();
+    private String explain(SearchContext context, boolean rewritten) throws IOException {
+        Query query = context.query();
+        if (rewritten && query instanceof MatchNoDocsQuery) {
+            return context.parsedQuery().query().toString();
         } else {
-            return queryRewrite.toString();
+            return query.toString();
         }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryRequest.java
index 41ef37ad621f1..5953a5548c465 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryRequest.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.action.admin.indices.validate.query;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ValidateActions;
 import org.elasticsearch.action.support.IndicesOptions;
@@ -43,6 +44,7 @@ public class ValidateQueryRequest extends BroadcastRequest<ValidateQueryRequest>
 
     private boolean explain;
     private boolean rewrite;
+    private boolean allShards;
 
     private String[] types = Strings.EMPTY_ARRAY;
 
@@ -125,6 +127,20 @@ public boolean rewrite() {
         return rewrite;
     }
 
+    /**
+     * Indicates whether the query should be validated on all shards instead of one random shard
+     */
+    public void allShards(boolean allShards) {
+        this.allShards = allShards;
+    }
+
+    /**
+     * Indicates whether the query should be validated on all shards instead of one random shard
+     */
+    public boolean allShards() {
+        return allShards;
+    }
+
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
@@ -138,6 +154,9 @@ public void readFrom(StreamInput in) throws IOException {
         }
         explain = in.readBoolean();
         rewrite = in.readBoolean();
+        if (in.getVersion().onOrAfter(Version.V_5_4_0)) {
+            allShards = in.readBoolean();
+        }
     }
 
     @Override
@@ -150,11 +169,14 @@ public void writeTo(StreamOutput out) throws IOException {
         }
         out.writeBoolean(explain);
         out.writeBoolean(rewrite);
+        if (out.getVersion().onOrAfter(Version.V_5_4_0)) {
+            out.writeBoolean(allShards);
+        }
     }
 
     @Override
     public String toString() {
         return "[" + Arrays.toString(indices) + "]" + Arrays.toString(types) + ", query[" + query + "], explain:" + explain +
-                ", rewrite:" + rewrite;
+                ", rewrite:" + rewrite + ", all_shards:" + allShards;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryRequestBuilder.java
index bfee7ec6b99e6..9240357ad7184 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryRequestBuilder.java
@@ -67,4 +67,12 @@ public ValidateQueryRequestBuilder setRewrite(boolean rewrite) {
         request.rewrite(rewrite);
         return this;
     }
+
+    /**
+     * Indicates whether the query should be validated on all shards
+     */
+    public ValidateQueryRequestBuilder setAllShards(boolean rewrite) {
+        request.allShards(rewrite);
+        return this;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BackoffPolicy.java b/core/src/main/java/org/elasticsearch/action/bulk/BackoffPolicy.java
index bc8f8c347ab0e..81084e22377e5 100644
--- a/core/src/main/java/org/elasticsearch/action/bulk/BackoffPolicy.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/BackoffPolicy.java
@@ -171,7 +171,7 @@ private static final class ConstantBackoff extends BackoffPolicy {
 
         private final int numberOfElements;
 
-        public ConstantBackoff(TimeValue delay, int numberOfElements) {
+        ConstantBackoff(TimeValue delay, int numberOfElements) {
             assert numberOfElements >= 0;
             this.delay = delay;
             this.numberOfElements = numberOfElements;
@@ -188,7 +188,7 @@ private static final class ConstantBackoffIterator implements Iterator<TimeValue
         private final int numberOfElements;
         private int curr;
 
-        public ConstantBackoffIterator(TimeValue delay, int numberOfElements) {
+        ConstantBackoffIterator(TimeValue delay, int numberOfElements) {
             this.delay = delay;
             this.numberOfElements = numberOfElements;
         }
@@ -212,7 +212,7 @@ private static final class WrappedBackoffPolicy extends BackoffPolicy {
         private final BackoffPolicy delegate;
         private final Runnable onBackoff;
 
-        public WrappedBackoffPolicy(BackoffPolicy delegate, Runnable onBackoff) {
+        WrappedBackoffPolicy(BackoffPolicy delegate, Runnable onBackoff) {
             this.delegate = delegate;
             this.onBackoff = onBackoff;
         }
@@ -227,7 +227,7 @@ private static final class WrappedBackoffIterator implements Iterator<TimeValue>
         private final Iterator<TimeValue> delegate;
         private final Runnable onBackoff;
 
-        public WrappedBackoffIterator(Iterator<TimeValue> delegate, Runnable onBackoff) {
+        WrappedBackoffIterator(Iterator<TimeValue> delegate, Runnable onBackoff) {
             this.delegate = delegate;
             this.onBackoff = onBackoff;
         }
diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BulkItemRequest.java b/core/src/main/java/org/elasticsearch/action/bulk/BulkItemRequest.java
index 760c5781aea0a..96c35d661c0bc 100644
--- a/core/src/main/java/org/elasticsearch/action/bulk/BulkItemRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/BulkItemRequest.java
@@ -19,16 +19,17 @@
 
 package org.elasticsearch.action.bulk;
 
-import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.IndicesRequest;
-import org.elasticsearch.action.delete.DeleteRequest;
-import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.update.UpdateRequest;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
 
 import java.io.IOException;
+import java.util.Objects;
 
 /**
  *
@@ -36,15 +37,14 @@
 public class BulkItemRequest implements Streamable {
 
     private int id;
-    private ActionRequest request;
+    private DocWriteRequest request;
     private volatile BulkItemResponse primaryResponse;
-    private volatile boolean ignoreOnReplica;
 
     BulkItemRequest() {
 
     }
 
-    public BulkItemRequest(int id, ActionRequest request) {
+    public BulkItemRequest(int id, DocWriteRequest request) {
         assert request instanceof IndicesRequest;
         this.id = id;
         this.request = request;
@@ -54,14 +54,13 @@ public int id() {
         return id;
     }
 
-    public ActionRequest request() {
+    public DocWriteRequest request() {
         return request;
     }
 
     public String index() {
-        IndicesRequest indicesRequest = (IndicesRequest) request;
-        assert indicesRequest.indices().length == 1;
-        return indicesRequest.indices()[0];
+        assert request.indices().length == 1;
+        return request.indices()[0];
     }
 
     BulkItemResponse getPrimaryResponse() {
@@ -72,15 +71,34 @@ void setPrimaryResponse(BulkItemResponse primaryResponse) {
         this.primaryResponse = primaryResponse;
     }
 
-    /**
-     * Marks this request to be ignored and *not* execute on a replica.
-     */
-    void setIgnoreOnReplica() {
-        this.ignoreOnReplica = true;
-    }
 
     boolean isIgnoreOnReplica() {
-        return ignoreOnReplica;
+        return primaryResponse != null &&
+                (primaryResponse.isFailed() || primaryResponse.getResponse().getResult() == DocWriteResponse.Result.NOOP);
+    }
+
+    /**
+     * Abort this request, and store a {@link org.elasticsearch.action.bulk.BulkItemResponse.Failure} response.
+     *
+     * @param index The concrete index that was resolved for this request
+     * @param cause The cause of the rejection (may not be null)
+     * @throws IllegalStateException If a response already exists for this request
+     */
+    public void abort(String index, Exception cause) {
+        if (primaryResponse == null) {
+            final BulkItemResponse.Failure failure = new BulkItemResponse.Failure(index, request.type(), request.id(),
+                    Objects.requireNonNull(cause), true);
+            setPrimaryResponse(new BulkItemResponse(id, request.opType(), failure));
+        } else {
+            assert primaryResponse.isFailed() && primaryResponse.getFailure().isAborted()
+                    : "response [" + Strings.toString(primaryResponse) + "]; cause [" + cause + "]";
+            if (primaryResponse.isFailed() && primaryResponse.getFailure().isAborted()) {
+                primaryResponse.getFailure().getCause().addSuppressed(cause);
+            } else {
+                throw new IllegalStateException(
+                        "aborting item that with response [" + primaryResponse + "] that was previously processed", cause);
+            }
+        }
     }
 
     public static BulkItemRequest readBulkItem(StreamInput in) throws IOException {
@@ -92,33 +110,30 @@ public static BulkItemRequest readBulkItem(StreamInput in) throws IOException {
     @Override
     public void readFrom(StreamInput in) throws IOException {
         id = in.readVInt();
-        byte type = in.readByte();
-        if (type == 0) {
-            request = new IndexRequest();
-        } else if (type == 1) {
-            request = new DeleteRequest();
-        } else if (type == 2) {
-            request = new UpdateRequest();
-        }
-        request.readFrom(in);
+        request = DocWriteRequest.readDocumentRequest(in);
         if (in.readBoolean()) {
             primaryResponse = BulkItemResponse.readBulkItem(in);
+            // This is a bwc layer for 6.0 which no longer mutates the requests with these
+            // Since 5.x still requires it we do it here. Note that these are harmless
+            // as both operations are idempotent. This is something we rely on and assert on
+            // in InternalEngine.planIndexingAsNonPrimary()
+            request.version(primaryResponse.getVersion());
+            request.versionType(request.versionType().versionTypeForReplicationAndRecovery());
+        }
+        if (in.getVersion().before(Version.V_5_6_0)) {
+            boolean ignoreOnReplica = in.readBoolean();
+            assert ignoreOnReplica == isIgnoreOnReplica() :
+                "ignoreOnReplica mismatch. wire [" + ignoreOnReplica + "], ours [" + isIgnoreOnReplica() + "]";
         }
-        ignoreOnReplica = in.readBoolean();
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeVInt(id);
-        if (request instanceof IndexRequest) {
-            out.writeByte((byte) 0);
-        } else if (request instanceof DeleteRequest) {
-            out.writeByte((byte) 1);
-        } else if (request instanceof UpdateRequest) {
-            out.writeByte((byte) 2);
-        }
-        request.writeTo(out);
+        DocWriteRequest.writeDocumentRequest(out, request);
         out.writeOptionalStreamable(primaryResponse);
-        out.writeBoolean(ignoreOnReplica);
+        if (out.getVersion().before(Version.V_5_6_0)) {
+            out.writeBoolean(isIgnoreOnReplica());
+        }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java b/core/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java
index ad45ace84c909..69b9180fdef91 100644
--- a/core/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java
@@ -21,27 +21,40 @@
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.DocWriteRequest.OpType;
 import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.delete.DeleteResponse;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.action.update.UpdateResponse;
+import org.elasticsearch.common.CheckedConsumer;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.StatusToXContent;
+import org.elasticsearch.common.xcontent.StatusToXContentObject;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
 
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.throwUnknownField;
+
 /**
  * Represents a single item response for an action executed as part of the bulk API. Holds the index/type/id
  * of the relevant action, and if it has failed or not (with the failure message incase it failed).
  */
-public class BulkItemResponse implements Streamable, StatusToXContent {
+public class BulkItemResponse implements Streamable, StatusToXContentObject {
+
+    private static final String _INDEX = "_index";
+    private static final String _TYPE = "_type";
+    private static final String _ID = "_id";
+    private static final String STATUS = "status";
+    private static final String ERROR = "error";
 
     @Override
     public RestStatus status() {
@@ -50,29 +63,97 @@ public RestStatus status() {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject(opType);
+        builder.startObject();
+        builder.startObject(opType.getLowercase());
         if (failure == null) {
-            response.toXContent(builder, params);
-            builder.field(Fields.STATUS, response.status().getStatus());
+            response.innerToXContent(builder, params);
+            builder.field(STATUS, response.status().getStatus());
         } else {
-            builder.field(Fields._INDEX, failure.getIndex());
-            builder.field(Fields._TYPE, failure.getType());
-            builder.field(Fields._ID, failure.getId());
-            builder.field(Fields.STATUS, failure.getStatus().getStatus());
-            builder.startObject(Fields.ERROR);
-            ElasticsearchException.toXContent(builder, params, failure.getCause());
+            builder.field(_INDEX, failure.getIndex());
+            builder.field(_TYPE, failure.getType());
+            builder.field(_ID, failure.getId());
+            builder.field(STATUS, failure.getStatus().getStatus());
+            builder.startObject(ERROR);
+            ElasticsearchException.generateThrowableXContent(builder, params, failure.getCause());
             builder.endObject();
         }
         builder.endObject();
+        builder.endObject();
         return builder;
     }
 
-    static final class Fields {
-        static final String _INDEX = "_index";
-        static final String _TYPE = "_type";
-        static final String _ID = "_id";
-        static final String STATUS = "status";
-        static final String ERROR = "error";
+    /**
+     * Reads a {@link BulkItemResponse} from a {@link XContentParser}.
+     *
+     * @param parser the {@link XContentParser}
+     * @param id the id to assign to the parsed {@link BulkItemResponse}. It is usually the index of
+     *           the item in the {@link BulkResponse#getItems} array.
+     */
+    public static BulkItemResponse fromXContent(XContentParser parser, int id) throws IOException {
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation);
+
+        XContentParser.Token token = parser.nextToken();
+        ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser::getTokenLocation);
+
+        String currentFieldName = parser.currentName();
+        token = parser.nextToken();
+
+        final OpType opType = OpType.fromString(currentFieldName);
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser::getTokenLocation);
+
+        DocWriteResponse.Builder builder = null;
+        CheckedConsumer<XContentParser, IOException> itemParser = null;
+
+        if (opType == OpType.INDEX || opType == OpType.CREATE) {
+            final IndexResponse.Builder indexResponseBuilder = new IndexResponse.Builder();
+            builder = indexResponseBuilder;
+            itemParser = (indexParser) -> IndexResponse.parseXContentFields(indexParser, indexResponseBuilder);
+
+        } else if (opType == OpType.UPDATE) {
+            final UpdateResponse.Builder updateResponseBuilder = new UpdateResponse.Builder();
+            builder = updateResponseBuilder;
+            itemParser = (updateParser) -> UpdateResponse.parseXContentFields(updateParser, updateResponseBuilder);
+
+        } else if (opType == OpType.DELETE) {
+            final DeleteResponse.Builder deleteResponseBuilder = new DeleteResponse.Builder();
+            builder = deleteResponseBuilder;
+            itemParser = (deleteParser) -> DeleteResponse.parseXContentFields(deleteParser, deleteResponseBuilder);
+        } else {
+            throwUnknownField(currentFieldName, parser.getTokenLocation());
+        }
+
+        RestStatus status = null;
+        ElasticsearchException exception = null;
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            }
+
+            if (ERROR.equals(currentFieldName)) {
+                if (token == XContentParser.Token.START_OBJECT) {
+                    exception = ElasticsearchException.fromXContent(parser);
+                }
+            } else if (STATUS.equals(currentFieldName)) {
+                if (token == XContentParser.Token.VALUE_NUMBER) {
+                    status = RestStatus.fromCode(parser.intValue());
+                }
+            } else {
+                itemParser.accept(parser);
+            }
+        }
+
+        ensureExpectedToken(XContentParser.Token.END_OBJECT, token, parser::getTokenLocation);
+        token = parser.nextToken();
+        ensureExpectedToken(XContentParser.Token.END_OBJECT, token, parser::getTokenLocation);
+
+        BulkItemResponse bulkItemResponse;
+        if (exception != null) {
+            Failure failure = new Failure(builder.getShardId().getIndexName(), builder.getType(), builder.getId(), exception, status);
+            bulkItemResponse = new BulkItemResponse(id, opType, failure);
+        } else {
+            bulkItemResponse = new BulkItemResponse(id, opType, builder.build());
+        }
+        return bulkItemResponse;
     }
 
     /**
@@ -88,15 +169,29 @@ public static class Failure implements Writeable, ToXContent {
         private final String index;
         private final String type;
         private final String id;
-        private final Throwable cause;
+        private final Exception cause;
         private final RestStatus status;
+        private final boolean aborted;
+
+        public Failure(String index, String type, String id, Exception cause) {
+            this(index, type, id, cause, ExceptionsHelper.status(cause));
+        }
+
+        Failure(String index, String type, String id, Exception cause, RestStatus status) {
+            this(index, type, id, cause, status, false);
+        }
 
-        public Failure(String index, String type, String id, Throwable cause) {
+        Failure(String index, String type, String id, Exception cause, boolean aborted) {
+            this(index, type, id, cause, ExceptionsHelper.status(cause), aborted);
+        }
+
+        Failure(String index, String type, String id, Exception cause, RestStatus status, boolean aborted) {
             this.index = index;
             this.type = type;
             this.id = id;
             this.cause = cause;
-            this.status = ExceptionsHelper.status(cause);
+            this.status = status;
+            this.aborted = aborted;
         }
 
         /**
@@ -108,6 +203,11 @@ public Failure(StreamInput in) throws IOException {
             id = in.readOptionalString();
             cause = in.readException();
             status = ExceptionsHelper.status(cause);
+            if (supportsAbortedFlag(in.getVersion())) {
+                aborted = in.readBoolean();
+            } else {
+                aborted = false;
+            }
         }
 
         @Override
@@ -116,8 +216,16 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeString(getType());
             out.writeOptionalString(getId());
             out.writeException(getCause());
+            if (supportsAbortedFlag(out.getVersion())) {
+                out.writeBoolean(aborted);
+            }
         }
 
+        private static Version V_6_0_0_BETA_2 = Version.fromString("6.0.0-beta2");
+        private static boolean supportsAbortedFlag(Version version) {
+            // The "aborted" flag was added for 5.5.3 and 5.6.0, but was not in 6.0.0-beta2
+            return version.after(V_6_0_0_BETA_2) || (version.major == 5 && version.onOrAfter(Version.V_5_5_3));
+        }
 
         /**
          * The index name of the action.
@@ -157,10 +265,19 @@ public RestStatus getStatus() {
         /**
          * The actual cause of the failure.
          */
-        public Throwable getCause() {
+        public Exception getCause() {
             return cause;
         }
 
+        /**
+         * Whether this failure is the result of an <em>abort</em>.
+         * If {@code true}, the request to which this failure relates should never be retried, regardless of the {@link #getCause() cause}.
+         * @see BulkItemRequest#abort(String, Exception)
+         */
+        public boolean isAborted() {
+            return aborted;
+        }
+
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.field(INDEX_FIELD, index);
@@ -169,7 +286,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                 builder.field(ID_FIELD, id);
             }
             builder.startObject(CAUSE_FIELD);
-            ElasticsearchException.toXContent(builder, params, cause);
+            ElasticsearchException.generateThrowableXContent(builder, params, cause);
             builder.endObject();
             builder.field(STATUS_FIELD, status.getStatus());
             return builder;
@@ -177,13 +294,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
 
         @Override
         public String toString() {
-            return Strings.toString(this, true);
+            return Strings.toString(this);
         }
     }
 
     private int id;
 
-    private String opType;
+    private OpType opType;
 
     private DocWriteResponse response;
 
@@ -193,13 +310,13 @@ public String toString() {
 
     }
 
-    public BulkItemResponse(int id, String opType, DocWriteResponse response) {
+    public BulkItemResponse(int id, OpType opType, DocWriteResponse response) {
         this.id = id;
         this.opType = opType;
         this.response = response;
     }
 
-    public BulkItemResponse(int id, String opType, Failure failure) {
+    public BulkItemResponse(int id, OpType opType, Failure failure) {
         this.id = id;
         this.opType = opType;
         this.failure = failure;
@@ -215,7 +332,7 @@ public int getItemId() {
     /**
      * The operation type ("index", "create" or "delete").
      */
-    public String getOpType() {
+    public OpType getOpType() {
         return this.opType;
     }
 
@@ -300,8 +417,11 @@ public static BulkItemResponse readBulkItem(StreamInput in) throws IOException {
     @Override
     public void readFrom(StreamInput in) throws IOException {
         id = in.readVInt();
-        opType = in.readString();
-
+        if (in.getVersion().onOrAfter(Version.V_5_3_0)) {
+            opType = OpType.fromId(in.readByte());
+        } else {
+            opType = OpType.fromString(in.readString());
+        }
         byte type = in.readByte();
         if (type == 0) {
             response = new IndexResponse();
@@ -322,8 +442,11 @@ public void readFrom(StreamInput in) throws IOException {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeVInt(id);
-        out.writeString(opType);
-
+        if (out.getVersion().onOrAfter(Version.V_5_3_0)) {
+            out.writeByte(opType.getId());
+        } else {
+            out.writeString(opType.getLowercase());
+        }
         if (response == null) {
             out.writeByte((byte) 2);
         } else {
diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BulkProcessor.java b/core/src/main/java/org/elasticsearch/action/bulk/BulkProcessor.java
index c54b3588c178d..02ba2f39048bb 100644
--- a/core/src/main/java/org/elasticsearch/action/bulk/BulkProcessor.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/BulkProcessor.java
@@ -19,7 +19,8 @@
 
 package org.elasticsearch.action.bulk;
 
-import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.client.Client;
@@ -28,16 +29,14 @@
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.util.concurrent.EsExecutors;
-import org.elasticsearch.common.util.concurrent.FutureUtils;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.Closeable;
 import java.util.Objects;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.ScheduledThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.BiConsumer;
 
 /**
  * A bulk processor is a thread safe bulk processing class, allowing to easily set when to "flush" a new bulk request
@@ -65,7 +64,7 @@ public interface Listener {
 
         /**
          * Callback after a failed execution of bulk request.
-         *
+         * <p>
          * Note that in case an instance of <code>InterruptedException</code> is passed, which means that request processing has been
          * cancelled externally, the thread's interruption status has been restored prior to calling this method.
          */
@@ -77,10 +76,10 @@ public interface Listener {
      */
     public static class Builder {
 
-        private final Client client;
+        private final BiConsumer<BulkRequest, ActionListener<BulkResponse>> consumer;
         private final Listener listener;
+        private final ThreadPool threadPool;
 
-        private String name;
         private int concurrentRequests = 1;
         private int bulkActions = 1000;
         private ByteSizeValue bulkSize = new ByteSizeValue(5, ByteSizeUnit.MB);
@@ -91,17 +90,10 @@ public static class Builder {
          * Creates a builder of bulk processor with the client to use and the listener that will be used
          * to be notified on the completion of bulk requests.
          */
-        public Builder(Client client, Listener listener) {
-            this.client = client;
+        public Builder(BiConsumer<BulkRequest, ActionListener<BulkResponse>> consumer, Listener listener, ThreadPool threadPool) {
+            this.consumer = consumer;
             this.listener = listener;
-        }
-
-        /**
-         * Sets an optional name to identify this bulk processor.
-         */
-        public Builder setName(String name) {
-            this.name = name;
-            return this;
+            this.threadPool = threadPool;
         }
 
         /**
@@ -163,7 +155,7 @@ public Builder setBackoffPolicy(BackoffPolicy backoffPolicy) {
          * Builds a new bulk processor.
          */
         public BulkProcessor build() {
-            return new BulkProcessor(client, backoffPolicy, listener, name, concurrentRequests, bulkActions, bulkSize, flushInterval);
+            return new BulkProcessor(consumer, backoffPolicy, listener, concurrentRequests, bulkActions, bulkSize, flushInterval, threadPool);
         }
     }
 
@@ -171,15 +163,13 @@ public static Builder builder(Client client, Listener listener) {
         Objects.requireNonNull(client, "client");
         Objects.requireNonNull(listener, "listener");
 
-        return new Builder(client, listener);
+        return new Builder(client::bulk, listener, client.threadPool());
     }
 
     private final int bulkActions;
     private final long bulkSize;
 
-
-    private final ScheduledThreadPoolExecutor scheduler;
-    private final ScheduledFuture<?> scheduledFuture;
+    private final ThreadPool.Cancellable cancellableFlushTask;
 
     private final AtomicLong executionIdGen = new AtomicLong();
 
@@ -188,22 +178,21 @@ public static Builder builder(Client client, Listener listener) {
 
     private volatile boolean closed = false;
 
-    BulkProcessor(Client client, BackoffPolicy backoffPolicy, Listener listener, @Nullable String name, int concurrentRequests, int bulkActions, ByteSizeValue bulkSize, @Nullable TimeValue flushInterval) {
+    BulkProcessor(BiConsumer<BulkRequest, ActionListener<BulkResponse>> consumer, BackoffPolicy backoffPolicy, Listener listener,
+                  int concurrentRequests, int bulkActions, ByteSizeValue bulkSize, @Nullable TimeValue flushInterval,
+                  ThreadPool threadPool) {
         this.bulkActions = bulkActions;
-        this.bulkSize = bulkSize.bytes();
-
+        this.bulkSize = bulkSize.getBytes();
         this.bulkRequest = new BulkRequest();
-        this.bulkRequestHandler = (concurrentRequests == 0) ? BulkRequestHandler.syncHandler(client, backoffPolicy, listener) : BulkRequestHandler.asyncHandler(client, backoffPolicy, listener, concurrentRequests);
 
-        if (flushInterval != null) {
-            this.scheduler = (ScheduledThreadPoolExecutor) Executors.newScheduledThreadPool(1, EsExecutors.daemonThreadFactory(client.settings(), (name != null ? "[" + name + "]" : "") + "bulk_processor"));
-            this.scheduler.setExecuteExistingDelayedTasksAfterShutdownPolicy(false);
-            this.scheduler.setContinueExistingPeriodicTasksAfterShutdownPolicy(false);
-            this.scheduledFuture = this.scheduler.scheduleWithFixedDelay(new Flush(), flushInterval.millis(), flushInterval.millis(), TimeUnit.MILLISECONDS);
+        if (concurrentRequests == 0) {
+            this.bulkRequestHandler = BulkRequestHandler.syncHandler(consumer, backoffPolicy, listener, threadPool);
         } else {
-            this.scheduler = null;
-            this.scheduledFuture = null;
+            this.bulkRequestHandler = BulkRequestHandler.asyncHandler(consumer, backoffPolicy, listener, threadPool, concurrentRequests);
         }
+
+        // Start period flushing task after everything is setup
+        this.cancellableFlushTask = startFlushTask(flushInterval, threadPool);
     }
 
     /**
@@ -213,20 +202,20 @@ public static Builder builder(Client client, Listener listener) {
     public void close() {
         try {
             awaitClose(0, TimeUnit.NANOSECONDS);
-        } catch(InterruptedException exc) {
+        } catch (InterruptedException exc) {
             Thread.currentThread().interrupt();
         }
     }
 
     /**
      * Closes the processor. If flushing by time is enabled, then it's shutdown. Any remaining bulk actions are flushed.
-     *
+     * <p>
      * If concurrent requests are not enabled, returns {@code true} immediately.
      * If concurrent requests are enabled, waits for up to the specified timeout for all bulk requests to complete then returns {@code true},
      * If the specified waiting time elapses before all bulk requests complete, {@code false} is returned.
      *
      * @param timeout The maximum time to wait for the bulk requests to complete
-     * @param unit The time unit of the {@code timeout} argument
+     * @param unit    The time unit of the {@code timeout} argument
      * @return {@code true} if all bulk requests completed and {@code false} if the waiting time elapsed before all the bulk requests completed
      * @throws InterruptedException If the current thread is interrupted
      */
@@ -235,10 +224,9 @@ public synchronized boolean awaitClose(long timeout, TimeUnit unit) throws Inter
             return true;
         }
         closed = true;
-        if (this.scheduledFuture != null) {
-            FutureUtils.cancel(this.scheduledFuture);
-            this.scheduler.shutdown();
-        }
+
+        this.cancellableFlushTask.cancel();
+
         if (bulkRequest.numberOfActions() > 0) {
             execute();
         }
@@ -250,24 +238,24 @@ public synchronized boolean awaitClose(long timeout, TimeUnit unit) throws Inter
      * (for example, if no id is provided, one will be generated, or usage of the create flag).
      */
     public BulkProcessor add(IndexRequest request) {
-        return add((ActionRequest<?>) request);
+        return add((DocWriteRequest) request);
     }
 
     /**
      * Adds an {@link DeleteRequest} to the list of actions to execute.
      */
     public BulkProcessor add(DeleteRequest request) {
-        return add((ActionRequest<?>) request);
+        return add((DocWriteRequest) request);
     }
 
     /**
      * Adds either a delete or an index request.
      */
-    public BulkProcessor add(ActionRequest<?> request) {
+    public BulkProcessor add(DocWriteRequest request) {
         return add(request, null);
     }
 
-    public BulkProcessor add(ActionRequest<?> request, @Nullable Object payload) {
+    public BulkProcessor add(DocWriteRequest request, @Nullable Object payload) {
         internalAdd(request, payload);
         return this;
     }
@@ -282,22 +270,69 @@ protected void ensureOpen() {
         }
     }
 
-    private synchronized void internalAdd(ActionRequest<?> request, @Nullable Object payload) {
+    private synchronized void internalAdd(DocWriteRequest request, @Nullable Object payload) {
         ensureOpen();
         bulkRequest.add(request, payload);
         executeIfNeeded();
     }
 
+    /**
+     * Adds the data from the bytes to be processed by the bulk processor
+     * @deprecated use {@link #add(BytesReference, String, String, XContentType)} instead to avoid content type auto-detection
+     */
+    @Deprecated
     public BulkProcessor add(BytesReference data, @Nullable String defaultIndex, @Nullable String defaultType) throws Exception {
         return add(data, defaultIndex, defaultType, null, null);
     }
 
-    public synchronized BulkProcessor add(BytesReference data, @Nullable String defaultIndex, @Nullable String defaultType, @Nullable String defaultPipeline, @Nullable Object payload) throws Exception {
-        bulkRequest.add(data, defaultIndex, defaultType, null, null, defaultPipeline, payload, true);
+    /**
+     * Adds the data from the bytes to be processed by the bulk processor
+     */
+    public BulkProcessor add(BytesReference data, @Nullable String defaultIndex, @Nullable String defaultType,
+                             XContentType xContentType) throws Exception {
+        return add(data, defaultIndex, defaultType, null, null, xContentType);
+    }
+
+    /**
+     * Adds the data from the bytes to be processed by the bulk processor
+     * @deprecated use {@link #add(BytesReference, String, String, String, Object, XContentType)} instead to avoid content type
+     * auto-detection
+     */
+    @Deprecated
+    public synchronized BulkProcessor add(BytesReference data, @Nullable String defaultIndex, @Nullable String defaultType,
+                                          @Nullable String defaultPipeline, @Nullable Object payload) throws Exception {
+        bulkRequest.add(data, defaultIndex, defaultType, null, null, null, defaultPipeline, payload, true);
         executeIfNeeded();
         return this;
     }
 
+    /**
+     * Adds the data from the bytes to be processed by the bulk processor
+     */
+    public synchronized BulkProcessor add(BytesReference data, @Nullable String defaultIndex, @Nullable String defaultType,
+                                          @Nullable String defaultPipeline, @Nullable Object payload, XContentType xContentType) throws Exception {
+        bulkRequest.add(data, defaultIndex, defaultType, null, null, null, defaultPipeline, payload, true, xContentType);
+        executeIfNeeded();
+        return this;
+    }
+
+    private ThreadPool.Cancellable startFlushTask(TimeValue flushInterval, ThreadPool threadPool) {
+        if (flushInterval == null) {
+            return new ThreadPool.Cancellable() {
+                @Override
+                public void cancel() {}
+
+                @Override
+                public boolean isCancelled() {
+                    return true;
+                }
+            };
+        }
+
+        final Runnable flushRunnable = threadPool.getThreadContext().preserveContext(new Flush());
+        return threadPool.scheduleWithFixedDelay(flushRunnable, flushInterval, ThreadPool.Names.GENERIC);
+    }
+
     private void executeIfNeeded() {
         ensureOpen();
         if (!isOverTheLimit()) {
diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java
index 7e7aa4ce603be..d31c4b1464edf 100644
--- a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java
@@ -22,7 +22,7 @@
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.CompositeIndicesRequest;
-import org.elasticsearch.action.IndicesRequest;
+import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.support.ActiveShardCount;
@@ -35,12 +35,17 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.VersionType;
+import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -56,7 +61,9 @@
  * Note that we only support refresh on the bulk request not per item.
  * @see org.elasticsearch.client.Client#bulk(BulkRequest)
  */
-public class BulkRequest extends ActionRequest<BulkRequest> implements CompositeIndicesRequest, WriteRequest<BulkRequest> {
+public class BulkRequest extends ActionRequest implements CompositeIndicesRequest, WriteRequest<BulkRequest> {
+    private static final DeprecationLogger DEPRECATION_LOGGER =
+        new DeprecationLogger(Loggers.getLogger(BulkRequest.class));
 
     private static final int REQUEST_OVERHEAD = 50;
 
@@ -65,7 +72,7 @@ public class BulkRequest extends ActionRequest<BulkRequest> implements Composite
      * {@link WriteRequest}s to this but java doesn't support syntax to declare that everything in the array has both types so we declare
      * the one with the least casts.
      */
-    final List<ActionRequest<?>> requests = new ArrayList<>();
+    final List<DocWriteRequest> requests = new ArrayList<>();
     List<Object> payloads = null;
 
     protected TimeValue timeout = BulkShardRequest.DEFAULT_TIMEOUT;
@@ -80,14 +87,14 @@ public BulkRequest() {
     /**
      * Adds a list of requests to be executed. Either index or delete requests.
      */
-    public BulkRequest add(ActionRequest<?>... requests) {
-        for (ActionRequest<?> request : requests) {
+    public BulkRequest add(DocWriteRequest... requests) {
+        for (DocWriteRequest request : requests) {
             add(request, null);
         }
         return this;
     }
 
-    public BulkRequest add(ActionRequest<?> request) {
+    public BulkRequest add(DocWriteRequest request) {
         return add(request, null);
     }
 
@@ -97,7 +104,7 @@ public BulkRequest add(ActionRequest<?> request) {
      * @param payload Optional payload
      * @return the current bulk request
      */
-    public BulkRequest add(ActionRequest<?> request, @Nullable Object payload) {
+    public BulkRequest add(DocWriteRequest request, @Nullable Object payload) {
         if (request instanceof IndexRequest) {
             add((IndexRequest) request, payload);
         } else if (request instanceof DeleteRequest) {
@@ -113,8 +120,8 @@ public BulkRequest add(ActionRequest<?> request, @Nullable Object payload) {
     /**
      * Adds a list of requests to be executed. Either index or delete requests.
      */
-    public BulkRequest add(Iterable<ActionRequest<?>> requests) {
-        for (ActionRequest<?> request : requests) {
+    public BulkRequest add(Iterable<DocWriteRequest> requests) {
+        for (DocWriteRequest request : requests) {
             add(request);
         }
         return this;
@@ -163,7 +170,7 @@ BulkRequest internalAdd(UpdateRequest request, @Nullable Object payload) {
             sizeInBytes += request.upsertRequest().source().length();
         }
         if (request.script() != null) {
-            sizeInBytes += request.script().getScript().length() * 2;
+            sizeInBytes += request.script().getIdOrCode().length() * 2;
         }
         return this;
     }
@@ -200,20 +207,10 @@ private void addPayload(Object payload) {
     /**
      * The list of requests in this bulk request.
      */
-    public List<ActionRequest<?>> requests() {
+    public List<DocWriteRequest> requests() {
         return this.requests;
     }
 
-    @Override
-    public List<? extends IndicesRequest> subRequests() {
-        List<IndicesRequest> indicesRequests = new ArrayList<>();
-        for (ActionRequest<?> request : requests) {
-            assert request instanceof IndicesRequest;
-            indicesRequests.add((IndicesRequest) request);
-        }
-        return indicesRequests;
-    }
-
     /**
      * The list of optional payloads associated with requests in the same order as the requests. Note, elements within
      * it might be null if no payload has been provided.
@@ -241,34 +238,84 @@ public long estimatedSizeInBytes() {
 
     /**
      * Adds a framed data in binary format
+     * @deprecated use {@link #add(byte[], int, int, XContentType)}
      */
-    public BulkRequest add(byte[] data, int from, int length) throws Exception {
+    @Deprecated
+    public BulkRequest add(byte[] data, int from, int length) throws IOException {
         return add(data, from, length, null, null);
     }
 
     /**
      * Adds a framed data in binary format
      */
-    public BulkRequest add(byte[] data, int from, int length, @Nullable String defaultIndex, @Nullable String defaultType) throws Exception {
+    public BulkRequest add(byte[] data, int from, int length, XContentType xContentType) throws IOException {
+        return add(data, from, length, null, null, xContentType);
+    }
+
+    /**
+     * Adds a framed data in binary format
+     * @deprecated use {@link #add(byte[], int, int, String, String, XContentType)}
+     */
+    @Deprecated
+    public BulkRequest add(byte[] data, int from, int length, @Nullable String defaultIndex, @Nullable String defaultType) throws IOException {
         return add(new BytesArray(data, from, length), defaultIndex, defaultType);
     }
 
     /**
      * Adds a framed data in binary format
      */
-    public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Nullable String defaultType) throws Exception {
-        return add(data, defaultIndex, defaultType, null, null, null, null, true);
+    public BulkRequest add(byte[] data, int from, int length, @Nullable String defaultIndex, @Nullable String defaultType,
+                           XContentType xContentType) throws IOException {
+        return add(new BytesArray(data, from, length), defaultIndex, defaultType, xContentType);
+    }
+
+    /**
+     * Adds a framed data in binary format
+     *
+     * @deprecated use {@link #add(BytesReference, String, String, XContentType)}
+     */
+    @Deprecated
+    public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Nullable String defaultType) throws IOException {
+        return add(data, defaultIndex, defaultType, null, null, null, null, null, true);
+    }
+
+    /**
+     * Adds a framed data in binary format
+     */
+    public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Nullable String defaultType,
+                           XContentType xContentType) throws IOException {
+        return add(data, defaultIndex, defaultType, null, null, null, null, null, true, xContentType);
     }
 
     /**
      * Adds a framed data in binary format
+     *
+     * @deprecated use {@link #add(BytesReference, String, String, boolean, XContentType)}
      */
-    public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Nullable String defaultType, boolean allowExplicitIndex) throws Exception {
-        return add(data, defaultIndex, defaultType, null, null, null, null, allowExplicitIndex);
+    @Deprecated
+    public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Nullable String defaultType, boolean allowExplicitIndex) throws IOException {
+        return add(data, defaultIndex, defaultType, null, null, null, null, null, allowExplicitIndex);
     }
 
-    public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Nullable String defaultType, @Nullable String defaultRouting, @Nullable String[] defaultFields, @Nullable String defaultPipeline, @Nullable Object payload, boolean allowExplicitIndex) throws Exception {
-        XContent xContent = XContentFactory.xContent(data);
+    /**
+     * Adds a framed data in binary format
+     */
+    public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Nullable String defaultType, boolean allowExplicitIndex,
+                           XContentType xContentType) throws IOException {
+        return add(data, defaultIndex, defaultType, null, null, null, null, null, allowExplicitIndex, xContentType);
+    }
+
+    @Deprecated
+    public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Nullable String defaultType, @Nullable String defaultRouting, @Nullable String[] defaultFields, @Nullable FetchSourceContext defaultFetchSourceContext, @Nullable String defaultPipeline, @Nullable Object payload, boolean allowExplicitIndex) throws IOException {
+        XContentType xContentType = XContentFactory.xContentType(data);
+        return add(data, defaultIndex, defaultType, defaultRouting, defaultFields, defaultFetchSourceContext, defaultPipeline, payload,
+            allowExplicitIndex, xContentType);
+    }
+
+    public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Nullable String defaultType, @Nullable String
+        defaultRouting, @Nullable String[] defaultFields, @Nullable FetchSourceContext defaultFetchSourceContext, @Nullable String
+        defaultPipeline, @Nullable Object payload, boolean allowExplicitIndex, XContentType xContentType) throws IOException {
+        XContent xContent = xContentType.xContent();
         int line = 0;
         int from = 0;
         int length = data.length();
@@ -281,7 +328,8 @@ public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Null
             line++;
 
             // now parse the action
-            try (XContentParser parser = xContent.createParser(data.slice(from, nextMarker - from))) {
+            // EMPTY is safe here because we never call namedObject
+            try (XContentParser parser = xContent.createParser(NamedXContentRegistry.EMPTY, data.slice(from, nextMarker - from))) {
                 // move pointers
                 from = nextMarker + 1;
 
@@ -290,10 +338,16 @@ public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Null
                 if (token == null) {
                     continue;
                 }
-                assert token == XContentParser.Token.START_OBJECT;
+                if (token != XContentParser.Token.START_OBJECT) {
+                    throw new IllegalArgumentException("Malformed action/metadata line [" + line + "], expected "
+                        + XContentParser.Token.START_OBJECT + " but found [" + token + "]");
+                }
                 // Move to FIELD_NAME, that's the action
                 token = parser.nextToken();
-                assert token == XContentParser.Token.FIELD_NAME;
+                if (token != XContentParser.Token.FIELD_NAME) {
+                    throw new IllegalArgumentException("Malformed action/metadata line [" + line + "], expected "
+                        + XContentParser.Token.FIELD_NAME + " but found [" + token + "]");
+                }
                 String action = parser.currentName();
 
                 String index = defaultIndex;
@@ -301,6 +355,7 @@ public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Null
                 String id = null;
                 String routing = defaultRouting;
                 String parent = null;
+                FetchSourceContext fetchSourceContext = defaultFetchSourceContext;
                 String[] fields = defaultFields;
                 String timestamp = null;
                 TimeValue ttl = null;
@@ -334,8 +389,10 @@ public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Null
                             } else if ("_parent".equals(currentFieldName) || "parent".equals(currentFieldName)) {
                                 parent = parser.text();
                             } else if ("_timestamp".equals(currentFieldName) || "timestamp".equals(currentFieldName)) {
+                                DEPRECATION_LOGGER.deprecated("The [timestamp] parameter of index requests is deprecated");
                                 timestamp = parser.text();
                             } else if ("_ttl".equals(currentFieldName) || "ttl".equals(currentFieldName)) {
+                                DEPRECATION_LOGGER.deprecated("The [ttl] parameter of index requests is deprecated");
                                 if (parser.currentToken() == XContentParser.Token.VALUE_STRING) {
                                     ttl = TimeValue.parseTimeValue(parser.text(), null, currentFieldName);
                                 } else {
@@ -353,16 +410,21 @@ public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Null
                                 pipeline = parser.text();
                             } else if ("fields".equals(currentFieldName)) {
                                 throw new IllegalArgumentException("Action/metadata line [" + line + "] contains a simple value for parameter [fields] while a list is expected");
+                            } else if ("_source".equals(currentFieldName)) {
+                                fetchSourceContext = FetchSourceContext.fromXContent(parser);
                             } else {
                                 throw new IllegalArgumentException("Action/metadata line [" + line + "] contains an unknown parameter [" + currentFieldName + "]");
                             }
                         } else if (token == XContentParser.Token.START_ARRAY) {
                             if ("fields".equals(currentFieldName)) {
+                                DEPRECATION_LOGGER.deprecated("Deprecated field [fields] used, expected [_source] instead");
                                 List<Object> values = parser.list();
                                 fields = values.toArray(new String[values.size()]);
                             } else {
                                 throw new IllegalArgumentException("Malformed action/metadata line [" + line + "], expected a simple value for field [" + currentFieldName + "] but found [" + token + "]");
                             }
+                        } else if (token == XContentParser.Token.START_OBJECT && "_source".equals(currentFieldName)) {
+                            fetchSourceContext = FetchSourceContext.fromXContent(parser);
                         } else if (token != XContentParser.Token.VALUE_NULL) {
                             throw new IllegalArgumentException("Malformed action/metadata line [" + line + "], expected a simple value for field [" + currentFieldName + "] but found [" + token + "]");
                         }
@@ -387,22 +449,30 @@ public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Null
                     if ("index".equals(action)) {
                         if (opType == null) {
                             internalAdd(new IndexRequest(index, type, id).routing(routing).parent(parent).timestamp(timestamp).ttl(ttl).version(version).versionType(versionType)
-                                    .setPipeline(pipeline).source(data.slice(from, nextMarker - from)), payload);
+                                    .setPipeline(pipeline)
+                                    .source(sliceTrimmingCarriageReturn(data, from, nextMarker, xContentType), xContentType), payload);
                         } else {
                             internalAdd(new IndexRequest(index, type, id).routing(routing).parent(parent).timestamp(timestamp).ttl(ttl).version(version).versionType(versionType)
                                     .create("create".equals(opType)).setPipeline(pipeline)
-                                    .source(data.slice(from, nextMarker - from)), payload);
+                                    .source(sliceTrimmingCarriageReturn(data, from, nextMarker, xContentType), xContentType), payload);
                         }
                     } else if ("create".equals(action)) {
                         internalAdd(new IndexRequest(index, type, id).routing(routing).parent(parent).timestamp(timestamp).ttl(ttl).version(version).versionType(versionType)
                                 .create(true).setPipeline(pipeline)
-                                .source(data.slice(from, nextMarker - from)), payload);
+                                .source(sliceTrimmingCarriageReturn(data, from, nextMarker, xContentType), xContentType), payload);
                     } else if ("update".equals(action)) {
                         UpdateRequest updateRequest = new UpdateRequest(index, type, id).routing(routing).parent(parent).retryOnConflict(retryOnConflict)
                                 .version(version).versionType(versionType)
                                 .routing(routing)
-                                .parent(parent)
-                                .source(data.slice(from, nextMarker - from));
+                                .parent(parent);
+                        // EMPTY is safe here because we never call namedObject
+                        try (XContentParser sliceParser = xContent.createParser(NamedXContentRegistry.EMPTY,
+                                                        sliceTrimmingCarriageReturn(data, from, nextMarker, xContentType))) {
+                            updateRequest.fromXContent(sliceParser);
+                        }
+                        if (fetchSourceContext != null) {
+                            updateRequest.fetchSource(fetchSourceContext);
+                        }
                         if (fields != null) {
                             updateRequest.fields(fields);
                         }
@@ -432,6 +502,20 @@ public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Null
         return this;
     }
 
+    /**
+     * Returns the sliced {@link BytesReference}. If the {@link XContentType} is JSON, the byte preceding the marker is checked to see
+     * if it is a carriage return and if so, the BytesReference is sliced so that the carriage return is ignored
+     */
+    private BytesReference sliceTrimmingCarriageReturn(BytesReference bytesReference, int from, int nextMarker, XContentType xContentType) {
+        final int length;
+        if (XContentType.JSON == xContentType && bytesReference.get(nextMarker - 1) == (byte) '\r') {
+            length = nextMarker - from - 1;
+        } else {
+            length = nextMarker - from;
+        }
+        return bytesReference.slice(from, length);
+    }
+
     /**
      * Sets the number of shard copies that must be active before proceeding with the write.
      * See {@link ReplicationRequest#waitForActiveShards(ActiveShardCount)} for details.
@@ -497,7 +581,7 @@ private int findNextMarker(byte marker, int from, BytesReference data, int lengt
      * @return Whether this bulk request contains index request with an ingest pipeline enabled.
      */
     public boolean hasIndexRequestsWithPipelines() {
-        for (ActionRequest<?> actionRequest : requests) {
+        for (DocWriteRequest actionRequest : requests) {
             if (actionRequest instanceof IndexRequest) {
                 IndexRequest indexRequest = (IndexRequest) actionRequest;
                 if (Strings.hasText(indexRequest.getPipeline())) {
@@ -515,13 +599,13 @@ public ActionRequestValidationException validate() {
         if (requests.isEmpty()) {
             validationException = addValidationError("no requests added", validationException);
         }
-        for (ActionRequest<?> request : requests) {
+        for (DocWriteRequest request : requests) {
             // We first check if refresh has been set
             if (((WriteRequest<?>) request).getRefreshPolicy() != RefreshPolicy.NONE) {
                 validationException = addValidationError(
                         "RefreshPolicy is not supported on an item request. Set it on the BulkRequest instead.", validationException);
             }
-            ActionRequestValidationException ex = request.validate();
+            ActionRequestValidationException ex = ((WriteRequest<?>) request).validate();
             if (ex != null) {
                 if (validationException == null) {
                     validationException = new ActionRequestValidationException();
@@ -539,20 +623,7 @@ public void readFrom(StreamInput in) throws IOException {
         waitForActiveShards = ActiveShardCount.readFrom(in);
         int size = in.readVInt();
         for (int i = 0; i < size; i++) {
-            byte type = in.readByte();
-            if (type == 0) {
-                IndexRequest request = new IndexRequest();
-                request.readFrom(in);
-                requests.add(request);
-            } else if (type == 1) {
-                DeleteRequest request = new DeleteRequest();
-                request.readFrom(in);
-                requests.add(request);
-            } else if (type == 2) {
-                UpdateRequest request = new UpdateRequest();
-                request.readFrom(in);
-                requests.add(request);
-            }
+            requests.add(DocWriteRequest.readDocumentRequest(in));
         }
         refreshPolicy = RefreshPolicy.readFrom(in);
         timeout = new TimeValue(in);
@@ -563,15 +634,8 @@ public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         waitForActiveShards.writeTo(out);
         out.writeVInt(requests.size());
-        for (ActionRequest<?> request : requests) {
-            if (request instanceof IndexRequest) {
-                out.writeByte((byte) 0);
-            } else if (request instanceof DeleteRequest) {
-                out.writeByte((byte) 1);
-            } else if (request instanceof UpdateRequest) {
-                out.writeByte((byte) 2);
-            }
-            request.writeTo(out);
+        for (DocWriteRequest request : requests) {
+            DocWriteRequest.writeDocumentRequest(out, request);
         }
         refreshPolicy.writeTo(out);
         timeout.writeTo(out);
diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequestBuilder.java
index c48a8f507b862..8f634fa28a41b 100644
--- a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequestBuilder.java
@@ -32,6 +32,7 @@
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentType;
 
 /**
  * A bulk request holds an ordered {@link IndexRequest}s and {@link DeleteRequest}s and allows to executes
@@ -97,7 +98,9 @@ public BulkRequestBuilder add(UpdateRequestBuilder request) {
 
     /**
      * Adds a framed data in binary format
+     * @deprecated use {@link #add(byte[], int, int, XContentType)}
      */
+    @Deprecated
     public BulkRequestBuilder add(byte[] data, int from, int length) throws Exception {
         request.add(data, from, length, null, null);
         return this;
@@ -106,11 +109,30 @@ public BulkRequestBuilder add(byte[] data, int from, int length) throws Exceptio
     /**
      * Adds a framed data in binary format
      */
+    public BulkRequestBuilder add(byte[] data, int from, int length, XContentType xContentType) throws Exception {
+        request.add(data, from, length, null, null, xContentType);
+        return this;
+    }
+
+    /**
+     * Adds a framed data in binary format
+     * @deprecated use {@link #add(byte[], int, int, String, String, XContentType)}
+     */
+    @Deprecated
     public BulkRequestBuilder add(byte[] data, int from, int length, @Nullable String defaultIndex, @Nullable String defaultType) throws Exception {
         request.add(data, from, length, defaultIndex, defaultType);
         return this;
     }
 
+    /**
+     * Adds a framed data in binary format
+     */
+    public BulkRequestBuilder add(byte[] data, int from, int length, @Nullable String defaultIndex, @Nullable String defaultType,
+                                  XContentType xContentType) throws Exception {
+        request.add(data, from, length, defaultIndex, defaultType, xContentType);
+        return this;
+    }
+
     /**
      * Sets the number of shard copies that must be active before proceeding with the write.
      * See {@link ReplicationRequest#waitForActiveShards(ActiveShardCount)} for details.
diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequestHandler.java b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequestHandler.java
index 6ad566ca50019..e1755bfb8bf1e 100644
--- a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequestHandler.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequestHandler.java
@@ -22,23 +22,27 @@
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.client.Client;
 import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
+import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
+import java.util.function.BiConsumer;
 
 /**
  * Abstracts the low-level details of bulk request handling
  */
 abstract class BulkRequestHandler {
     protected final Logger logger;
-    protected final Client client;
+    protected final BiConsumer<BulkRequest, ActionListener<BulkResponse>> consumer;
+    protected final ThreadPool threadPool;
 
-    protected BulkRequestHandler(Client client) {
-        this.client = client;
-        this.logger = Loggers.getLogger(getClass(), client.settings());
+    protected BulkRequestHandler(BiConsumer<BulkRequest, ActionListener<BulkResponse>> consumer, ThreadPool threadPool) {
+        this.logger = Loggers.getLogger(getClass());
+        this.consumer = consumer;
+        this.threadPool = threadPool;
     }
 
 
@@ -47,20 +51,25 @@ protected BulkRequestHandler(Client client) {
     public abstract boolean awaitClose(long timeout, TimeUnit unit) throws InterruptedException;
 
 
-    public static BulkRequestHandler syncHandler(Client client, BackoffPolicy backoffPolicy, BulkProcessor.Listener listener) {
-        return new SyncBulkRequestHandler(client, backoffPolicy, listener);
+    public static BulkRequestHandler syncHandler(BiConsumer<BulkRequest, ActionListener<BulkResponse>> consumer,
+                                                 BackoffPolicy backoffPolicy, BulkProcessor.Listener listener,
+                                                 ThreadPool threadPool) {
+        return new SyncBulkRequestHandler(consumer, backoffPolicy, listener, threadPool);
     }
 
-    public static BulkRequestHandler asyncHandler(Client client, BackoffPolicy backoffPolicy, BulkProcessor.Listener listener, int concurrentRequests) {
-        return new AsyncBulkRequestHandler(client, backoffPolicy, listener, concurrentRequests);
+    public static BulkRequestHandler asyncHandler(BiConsumer<BulkRequest, ActionListener<BulkResponse>> consumer,
+                                                  BackoffPolicy backoffPolicy, BulkProcessor.Listener listener,
+                                                  ThreadPool threadPool, int concurrentRequests) {
+        return new AsyncBulkRequestHandler(consumer, backoffPolicy, listener, threadPool, concurrentRequests);
     }
 
     private static class SyncBulkRequestHandler extends BulkRequestHandler {
         private final BulkProcessor.Listener listener;
         private final BackoffPolicy backoffPolicy;
 
-        public SyncBulkRequestHandler(Client client, BackoffPolicy backoffPolicy, BulkProcessor.Listener listener) {
-            super(client);
+        SyncBulkRequestHandler(BiConsumer<BulkRequest, ActionListener<BulkResponse>> consumer, BackoffPolicy backoffPolicy,
+                               BulkProcessor.Listener listener, ThreadPool threadPool) {
+            super(consumer, threadPool);
             this.backoffPolicy = backoffPolicy;
             this.listener = listener;
         }
@@ -71,9 +80,10 @@ public void execute(BulkRequest bulkRequest, long executionId) {
             try {
                 listener.beforeBulk(executionId, bulkRequest);
                 BulkResponse bulkResponse = Retry
-                        .on(EsRejectedExecutionException.class)
-                        .policy(backoffPolicy)
-                        .withSyncBackoff(client, bulkRequest);
+                    .on(EsRejectedExecutionException.class)
+                    .policy(backoffPolicy)
+                    .using(threadPool)
+                    .withSyncBackoff(consumer, bulkRequest, Settings.EMPTY);
                 afterCalled = true;
                 listener.afterBulk(executionId, bulkRequest, bulkResponse);
             } catch (InterruptedException e) {
@@ -103,8 +113,10 @@ private static class AsyncBulkRequestHandler extends BulkRequestHandler {
         private final Semaphore semaphore;
         private final int concurrentRequests;
 
-        private AsyncBulkRequestHandler(Client client, BackoffPolicy backoffPolicy, BulkProcessor.Listener listener, int concurrentRequests) {
-            super(client);
+        private AsyncBulkRequestHandler(BiConsumer<BulkRequest, ActionListener<BulkResponse>> consumer, BackoffPolicy backoffPolicy,
+                                        BulkProcessor.Listener listener, ThreadPool threadPool,
+                                        int concurrentRequests) {
+            super(consumer, threadPool);
             this.backoffPolicy = backoffPolicy;
             assert concurrentRequests > 0;
             this.listener = listener;
@@ -121,26 +133,27 @@ public void execute(BulkRequest bulkRequest, long executionId) {
                 semaphore.acquire();
                 acquired = true;
                 Retry.on(EsRejectedExecutionException.class)
-                        .policy(backoffPolicy)
-                        .withAsyncBackoff(client, bulkRequest, new ActionListener<BulkResponse>() {
-                            @Override
-                            public void onResponse(BulkResponse response) {
-                                try {
-                                    listener.afterBulk(executionId, bulkRequest, response);
-                                } finally {
-                                    semaphore.release();
-                                }
+                    .policy(backoffPolicy)
+                    .using(threadPool)
+                    .withAsyncBackoff(consumer, bulkRequest, new ActionListener<BulkResponse>() {
+                        @Override
+                        public void onResponse(BulkResponse response) {
+                            try {
+                                listener.afterBulk(executionId, bulkRequest, response);
+                            } finally {
+                                semaphore.release();
                             }
-
-                            @Override
-                            public void onFailure(Exception e) {
-                                try {
-                                    listener.afterBulk(executionId, bulkRequest, e);
-                                } finally {
-                                    semaphore.release();
-                                }
+                        }
+
+                        @Override
+                        public void onFailure(Exception e) {
+                            try {
+                                listener.afterBulk(executionId, bulkRequest, e);
+                            } finally {
+                                semaphore.release();
                             }
-                        });
+                        }
+                    }, Settings.EMPTY);
                 bulkRequestSetupSuccessful = true;
             } catch (InterruptedException e) {
                 Thread.currentThread().interrupt();
diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BulkResponse.java b/core/src/main/java/org/elasticsearch/action/bulk/BulkResponse.java
index e214f87ddb63b..8e0b48143dc92 100644
--- a/core/src/main/java/org/elasticsearch/action/bulk/BulkResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/BulkResponse.java
@@ -23,17 +23,32 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.StatusToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Iterator;
+import java.util.List;
+
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.throwUnknownField;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.throwUnknownToken;
 
 /**
  * A response of a bulk execution. Holding a response for each item responding (in order) of the
  * bulk requests. Each item holds the index/type/id is operated on, and if it failed or not (with the
  * failure message).
  */
-public class BulkResponse extends ActionResponse implements Iterable<BulkItemResponse> {
+public class BulkResponse extends ActionResponse implements Iterable<BulkItemResponse>, StatusToXContentObject {
+
+    private static final String ITEMS = "items";
+    private static final String ERRORS = "errors";
+    private static final String TOOK = "took";
+    private static final String INGEST_TOOK = "ingest_took";
 
     public static final long NO_INGEST_TOOK = -1L;
 
@@ -141,4 +156,61 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeVLong(tookInMillis);
         out.writeZLong(ingestTookInMillis);
     }
+
+    @Override
+    public RestStatus status() {
+        return RestStatus.OK;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(TOOK, tookInMillis);
+        if (ingestTookInMillis != BulkResponse.NO_INGEST_TOOK) {
+            builder.field(INGEST_TOOK, ingestTookInMillis);
+        }
+        builder.field(ERRORS, hasFailures());
+        builder.startArray(ITEMS);
+        for (BulkItemResponse item : this) {
+            item.toXContent(builder, params);
+        }
+        builder.endArray();
+        builder.endObject();
+        return builder;
+    }
+
+    public static BulkResponse fromXContent(XContentParser parser) throws IOException {
+        XContentParser.Token token = parser.nextToken();
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser::getTokenLocation);
+
+        long took = -1L;
+        long ingestTook = NO_INGEST_TOOK;
+        List<BulkItemResponse> items = new ArrayList<>();
+
+        String currentFieldName = parser.currentName();
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token.isValue()) {
+                if (TOOK.equals(currentFieldName)) {
+                    took = parser.longValue();
+                } else if (INGEST_TOOK.equals(currentFieldName)) {
+                    ingestTook = parser.longValue();
+                } else if (ERRORS.equals(currentFieldName) == false) {
+                    throwUnknownField(currentFieldName, parser.getTokenLocation());
+                }
+            } else if (token == XContentParser.Token.START_ARRAY) {
+                if (ITEMS.equals(currentFieldName)) {
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                        items.add(BulkItemResponse.fromXContent(parser, items.size()));
+                    }
+                } else {
+                    throwUnknownField(currentFieldName, parser.getTokenLocation());
+                }
+            } else {
+                throwUnknownToken(token, parser.getTokenLocation());
+            }
+        }
+        return new BulkResponse(items.toArray(new BulkItemResponse[items.size()]), took, ingestTook);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BulkShardRequest.java b/core/src/main/java/org/elasticsearch/action/bulk/BulkShardRequest.java
index ffc2407b8a478..b888c5670c86b 100644
--- a/core/src/main/java/org/elasticsearch/action/bulk/BulkShardRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/BulkShardRequest.java
@@ -39,13 +39,13 @@ public class BulkShardRequest extends ReplicatedWriteRequest<BulkShardRequest> {
     public BulkShardRequest() {
     }
 
-    BulkShardRequest(ShardId shardId, RefreshPolicy refreshPolicy, BulkItemRequest[] items) {
+    public BulkShardRequest(ShardId shardId, RefreshPolicy refreshPolicy, BulkItemRequest[] items) {
         super(shardId);
         this.items = items;
         setRefreshPolicy(refreshPolicy);
     }
 
-    BulkItemRequest[] items() {
+    public BulkItemRequest[] items() {
         return items;
     }
 
@@ -88,8 +88,14 @@ public void readFrom(StreamInput in) throws IOException {
     @Override
     public String toString() {
         // This is included in error messages so we'll try to make it somewhat user friendly.
-        StringBuilder b = new StringBuilder("BulkShardRequest to [");
-        b.append(index).append("] containing [").append(items.length).append("] requests");
+        StringBuilder b = new StringBuilder("BulkShardRequest [");
+        b.append(shardId).append("] containing [");
+        if (items.length > 1) {
+          b.append(items.length).append("] requests");
+        } else {
+            b.append(items[0].request()).append("]");
+        }
+
         switch (getRefreshPolicy()) {
         case IMMEDIATE:
             b.append(" and a refresh");
diff --git a/core/src/main/java/org/elasticsearch/action/bulk/Retry.java b/core/src/main/java/org/elasticsearch/action/bulk/Retry.java
index 375796ae8017d..e1ba1a6bee112 100644
--- a/core/src/main/java/org/elasticsearch/action/bulk/Retry.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/Retry.java
@@ -20,19 +20,25 @@
 
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.action.ActionFuture;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.PlainActionFuture;
-import org.elasticsearch.client.Client;
 import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.FutureUtils;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.function.BiConsumer;
+import java.util.function.BiFunction;
 import java.util.function.Predicate;
 
 /**
@@ -42,11 +48,16 @@ public class Retry {
     private final Class<? extends Throwable> retryOnThrowable;
 
     private BackoffPolicy backoffPolicy;
+    private ThreadPool threadPool;
 
     public static Retry on(Class<? extends Throwable> retryOnThrowable) {
         return new Retry(retryOnThrowable);
     }
 
+    Retry(Class<? extends Throwable> retryOnThrowable) {
+        this.retryOnThrowable = retryOnThrowable;
+    }
+
     /**
      * @param backoffPolicy The backoff policy that defines how long and how often to wait for retries.
      */
@@ -55,42 +66,48 @@ public Retry policy(BackoffPolicy backoffPolicy) {
         return this;
     }
 
-    Retry(Class<? extends Throwable> retryOnThrowable) {
-        this.retryOnThrowable = retryOnThrowable;
+    /**
+     * @param threadPool The threadPool that will be used to schedule retries.
+     */
+    public Retry using(ThreadPool threadPool) {
+        this.threadPool = threadPool;
+        return this;
     }
 
     /**
-     * Invokes #bulk(BulkRequest, ActionListener) on the provided client. Backs off on the provided exception and delegates results to the
-     * provided listener.
-     *
-     * @param client      Client invoking the bulk request.
+     * Invokes #apply(BulkRequest, ActionListener). Backs off on the provided exception and delegates results to the
+     * provided listener. Retries will be attempted using the provided schedule function
+     * @param consumer The consumer to which apply the request and listener
      * @param bulkRequest The bulk request that should be executed.
-     * @param listener    A listener that is invoked when the bulk request finishes or completes with an exception. The listener is not
+     * @param listener A listener that is invoked when the bulk request finishes or completes with an exception. The listener is not
+     * @param settings settings
      */
-    public void withAsyncBackoff(Client client, BulkRequest bulkRequest, ActionListener<BulkResponse> listener) {
-        AsyncRetryHandler r = new AsyncRetryHandler(retryOnThrowable, backoffPolicy, client, listener);
+    public void withAsyncBackoff(BiConsumer<BulkRequest, ActionListener<BulkResponse>> consumer, BulkRequest bulkRequest, ActionListener<BulkResponse> listener, Settings settings) {
+        RetryHandler r = new RetryHandler(retryOnThrowable, backoffPolicy, consumer, listener, settings, threadPool);
         r.execute(bulkRequest);
-
     }
 
     /**
-     * Invokes #bulk(BulkRequest) on the provided client. Backs off on the provided exception.
+     * Invokes #apply(BulkRequest, ActionListener). Backs off on the provided exception. Retries will be attempted using
+     * the provided schedule function.
      *
-     * @param client      Client invoking the bulk request.
+     * @param consumer The consumer to which apply the request and listener
      * @param bulkRequest The bulk request that should be executed.
+     * @param settings settings
      * @return the bulk response as returned by the client.
      * @throws Exception Any exception thrown by the callable.
      */
-    public BulkResponse withSyncBackoff(Client client, BulkRequest bulkRequest) throws Exception {
-        return SyncRetryHandler
-                .create(retryOnThrowable, backoffPolicy, client)
-                .executeBlocking(bulkRequest)
-                .actionGet();
+    public BulkResponse withSyncBackoff(BiConsumer<BulkRequest, ActionListener<BulkResponse>> consumer, BulkRequest bulkRequest, Settings settings) throws Exception {
+        PlainActionFuture<BulkResponse> actionFuture = PlainActionFuture.newFuture();
+        RetryHandler r = new RetryHandler(retryOnThrowable, backoffPolicy, consumer, actionFuture, settings, threadPool);
+        r.execute(bulkRequest);
+        return actionFuture.actionGet();
     }
 
-    static class AbstractRetryHandler implements ActionListener<BulkResponse> {
+    static class RetryHandler implements ActionListener<BulkResponse> {
         private final Logger logger;
-        private final Client client;
+        private final ThreadPool threadPool;
+        private final BiConsumer<BulkRequest, ActionListener<BulkResponse>> consumer;
         private final ActionListener<BulkResponse> listener;
         private final Iterator<TimeValue> backoff;
         private final Class<? extends Throwable> retryOnThrowable;
@@ -102,12 +119,15 @@ static class AbstractRetryHandler implements ActionListener<BulkResponse> {
         private volatile BulkRequest currentBulkRequest;
         private volatile ScheduledFuture<?> scheduledRequestFuture;
 
-        public AbstractRetryHandler(Class<? extends Throwable> retryOnThrowable, BackoffPolicy backoffPolicy, Client client, ActionListener<BulkResponse> listener) {
+        RetryHandler(Class<? extends Throwable> retryOnThrowable, BackoffPolicy backoffPolicy,
+                     BiConsumer<BulkRequest, ActionListener<BulkResponse>> consumer, ActionListener<BulkResponse> listener,
+                     Settings settings, ThreadPool threadPool) {
             this.retryOnThrowable = retryOnThrowable;
             this.backoff = backoffPolicy.iterator();
-            this.client = client;
+            this.consumer = consumer;
             this.listener = listener;
-            this.logger = Loggers.getLogger(getClass(), client.settings());
+            this.logger = Loggers.getLogger(getClass(), settings);
+            this.threadPool = threadPool;
             // in contrast to System.currentTimeMillis(), nanoTime() uses a monotonic clock under the hood
             this.startTimestampNanos = System.nanoTime();
         }
@@ -142,7 +162,8 @@ private void retry(BulkRequest bulkRequestForRetry) {
             assert backoff.hasNext();
             TimeValue next = backoff.next();
             logger.trace("Retry of bulk request scheduled in {} ms.", next.millis());
-            scheduledRequestFuture = client.threadPool().schedule(next, ThreadPool.Names.SAME, (() -> this.execute(bulkRequestForRetry)));
+            Runnable command = threadPool.getThreadContext().preserveContext(() -> this.execute(bulkRequestForRetry));
+            scheduledRequestFuture = threadPool.schedule(next, ThreadPool.Names.SAME, command);
         }
 
         private BulkRequest createBulkRequestForRetry(BulkResponse bulkItemResponses) {
@@ -206,32 +227,7 @@ private BulkResponse getAccumulatedResponse() {
 
         public void execute(BulkRequest bulkRequest) {
             this.currentBulkRequest = bulkRequest;
-            client.bulk(bulkRequest, this);
-        }
-    }
-
-    static class AsyncRetryHandler extends AbstractRetryHandler {
-        public AsyncRetryHandler(Class<? extends Throwable> retryOnThrowable, BackoffPolicy backoffPolicy, Client client, ActionListener<BulkResponse> listener) {
-            super(retryOnThrowable, backoffPolicy, client, listener);
-        }
-    }
-
-    static class SyncRetryHandler extends AbstractRetryHandler {
-        private final PlainActionFuture<BulkResponse> actionFuture;
-
-        public static SyncRetryHandler create(Class<? extends Throwable> retryOnThrowable, BackoffPolicy backoffPolicy, Client client) {
-            PlainActionFuture<BulkResponse> actionFuture = PlainActionFuture.newFuture();
-            return new SyncRetryHandler(retryOnThrowable, backoffPolicy, client, actionFuture);
-        }
-
-        public SyncRetryHandler(Class<? extends Throwable> retryOnThrowable, BackoffPolicy backoffPolicy, Client client, PlainActionFuture<BulkResponse> actionFuture) {
-            super(retryOnThrowable, backoffPolicy, client, actionFuture);
-            this.actionFuture = actionFuture;
-        }
-
-        public ActionFuture<BulkResponse> executeBlocking(BulkRequest bulkRequest) {
-            super.execute(bulkRequest);
-            return actionFuture;
+            consumer.accept(bulkRequest, this);
         }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/core/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
index da080b54b2531..ecc9fc35fe1e2 100644
--- a/core/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
@@ -19,25 +19,28 @@
 
 package org.elasticsearch.action.bulk;
 
-import org.elasticsearch.ElasticsearchException;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.apache.logging.log4j.util.Supplier;
+import org.apache.lucene.util.SparseFixedBitSet;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.ResourceAlreadyExistsException;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ActionRequest;
-import org.elasticsearch.action.DocumentRequest;
+import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.RoutingMissingException;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
 import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction;
-import org.elasticsearch.action.delete.DeleteRequest;
-import org.elasticsearch.action.delete.TransportDeleteAction;
 import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.ingest.IngestActionForwarder;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.AutoCreateIndex;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.action.update.TransportUpdateAction;
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateObserver;
+import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
@@ -46,12 +49,16 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.indices.IndexAlreadyExistsException;
 import org.elasticsearch.indices.IndexClosedException;
+import org.elasticsearch.ingest.IngestService;
+import org.elasticsearch.node.NodeClosedException;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -59,52 +66,62 @@
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.LongSupplier;
+import java.util.stream.Collectors;
+
+import static java.util.Collections.emptyMap;
 
 /**
- *
+ * Groups bulk request items by shard, optionally creating non-existent indices and
+ * delegates to {@link TransportShardBulkAction} for shard-level bulk execution
  */
 public class TransportBulkAction extends HandledTransportAction<BulkRequest, BulkResponse> {
 
     private final AutoCreateIndex autoCreateIndex;
     private final boolean allowIdGeneration;
     private final ClusterService clusterService;
+    private final IngestService ingestService;
     private final TransportShardBulkAction shardBulkAction;
     private final TransportCreateIndexAction createIndexAction;
     private final LongSupplier relativeTimeProvider;
+    private final IngestActionForwarder ingestForwarder;
 
     @Inject
-    public TransportBulkAction(Settings settings, ThreadPool threadPool, TransportService transportService, ClusterService clusterService,
+    public TransportBulkAction(Settings settings, ThreadPool threadPool, TransportService transportService,
+                               ClusterService clusterService, IngestService ingestService,
                                TransportShardBulkAction shardBulkAction, TransportCreateIndexAction createIndexAction,
                                ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
                                AutoCreateIndex autoCreateIndex) {
-        this(settings, threadPool, transportService, clusterService,
+        this(settings, threadPool, transportService, clusterService, ingestService,
                 shardBulkAction, createIndexAction,
                 actionFilters, indexNameExpressionResolver,
                 autoCreateIndex,
                 System::nanoTime);
     }
 
-    public TransportBulkAction(Settings settings, ThreadPool threadPool, TransportService transportService, ClusterService clusterService,
+    public TransportBulkAction(Settings settings, ThreadPool threadPool, TransportService transportService,
+                               ClusterService clusterService, IngestService ingestService,
                                TransportShardBulkAction shardBulkAction, TransportCreateIndexAction createIndexAction,
                                ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
                                AutoCreateIndex autoCreateIndex, LongSupplier relativeTimeProvider) {
         super(settings, BulkAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, BulkRequest::new);
         Objects.requireNonNull(relativeTimeProvider);
         this.clusterService = clusterService;
+        this.ingestService = ingestService;
         this.shardBulkAction = shardBulkAction;
         this.createIndexAction = createIndexAction;
-
         this.autoCreateIndex = autoCreateIndex;
         this.allowIdGeneration = this.settings.getAsBoolean("action.bulk.action.allow_id_generation", true);
         this.relativeTimeProvider = relativeTimeProvider;
+        this.ingestForwarder = new IngestActionForwarder(transportService);
+        clusterService.addStateApplier(this.ingestForwarder);
     }
 
     @Override
@@ -114,69 +131,78 @@ protected final void doExecute(final BulkRequest bulkRequest, final ActionListen
 
     @Override
     protected void doExecute(Task task, BulkRequest bulkRequest, ActionListener<BulkResponse> listener) {
+        if (bulkRequest.hasIndexRequestsWithPipelines()) {
+            if (clusterService.localNode().isIngestNode()) {
+                processBulkIndexIngestRequest(task, bulkRequest, listener);
+            } else {
+                ingestForwarder.forwardIngestRequest(BulkAction.INSTANCE, bulkRequest, listener);
+            }
+            return;
+        }
+
         final long startTime = relativeTime();
         final AtomicArray<BulkItemResponse> responses = new AtomicArray<>(bulkRequest.requests.size());
 
         if (needToCheck()) {
-            // Keep track of all unique indices and all unique types per index for the create index requests:
-            final Set<String> autoCreateIndices = new HashSet<>();
-            for (ActionRequest request : bulkRequest.requests) {
-                if (request instanceof DocumentRequest) {
-                    DocumentRequest req = (DocumentRequest) request;
-                    autoCreateIndices.add(req.index());
-                } else {
-                    throw new ElasticsearchException("Parsed unknown request in bulk actions: " + request.getClass().getSimpleName());
+            // Attempt to create all the indices that we're going to need during the bulk before we start.
+            // Step 1: collect all the indices in the request
+            final Set<String> indices = bulkRequest.requests.stream()
+                .map(DocWriteRequest::index)
+                .collect(Collectors.toSet());
+            /* Step 2: filter that to indices that don't exist and we can create. At the same time build a map of indices we can't create
+             * that we'll use when we try to run the requests. */
+            final Map<String, IndexNotFoundException> indicesThatCannotBeCreated = new HashMap<>();
+            Set<String> autoCreateIndices = new HashSet<>();
+            ClusterState state = clusterService.state();
+            for (String index : indices) {
+                boolean shouldAutoCreate;
+                try {
+                    shouldAutoCreate = shouldAutoCreate(index, state);
+                } catch (IndexNotFoundException e) {
+                    shouldAutoCreate = false;
+                    indicesThatCannotBeCreated.put(index, e);
+                }
+                if (shouldAutoCreate) {
+                    autoCreateIndices.add(index);
                 }
             }
-            final AtomicInteger counter = new AtomicInteger(autoCreateIndices.size());
-            ClusterState state = clusterService.state();
-            for (String index : autoCreateIndices) {
-                if (shouldAutoCreate(index, state)) {
-                    CreateIndexRequest createIndexRequest = new CreateIndexRequest();
-                    createIndexRequest.index(index);
-                    createIndexRequest.cause("auto(bulk api)");
-                    createIndexRequest.masterNodeTimeout(bulkRequest.timeout());
-                    createIndexAction.execute(createIndexRequest, new ActionListener<CreateIndexResponse>() {
+            // Step 3: create all the indices that are missing, if there are any missing. start the bulk after all the creates come back.
+            if (autoCreateIndices.isEmpty()) {
+                executeBulk(task, bulkRequest, startTime, listener, responses, indicesThatCannotBeCreated);
+            } else {
+                final AtomicInteger counter = new AtomicInteger(autoCreateIndices.size());
+                for (String index : autoCreateIndices) {
+                    createIndex(index, bulkRequest.timeout(), new ActionListener<CreateIndexResponse>() {
                         @Override
                         public void onResponse(CreateIndexResponse result) {
                             if (counter.decrementAndGet() == 0) {
-                                try {
-                                    executeBulk(task, bulkRequest, startTime, listener, responses);
-                                } catch (Exception e) {
-                                    listener.onFailure(e);
-                                }
+                                executeBulk(task, bulkRequest, startTime, listener, responses, indicesThatCannotBeCreated);
                             }
                         }
 
                         @Override
                         public void onFailure(Exception e) {
-                            if (!(ExceptionsHelper.unwrapCause(e) instanceof IndexAlreadyExistsException)) {
-                                // fail all requests involving this index, if create didnt work
+                            if (!(ExceptionsHelper.unwrapCause(e) instanceof ResourceAlreadyExistsException)) {
+                                // fail all requests involving this index, if create didn't work
                                 for (int i = 0; i < bulkRequest.requests.size(); i++) {
-                                    ActionRequest request = bulkRequest.requests.get(i);
+                                    DocWriteRequest request = bulkRequest.requests.get(i);
                                     if (request != null && setResponseFailureIfIndexMatches(responses, i, request, index, e)) {
                                         bulkRequest.requests.set(i, null);
                                     }
                                 }
                             }
                             if (counter.decrementAndGet() == 0) {
-                                try {
-                                    executeBulk(task, bulkRequest, startTime, listener, responses);
-                                } catch (Exception inner) {
+                                executeBulk(task, bulkRequest, startTime, ActionListener.wrap(listener::onResponse, inner -> {
                                     inner.addSuppressed(e);
                                     listener.onFailure(inner);
-                                }
+                                }), responses, indicesThatCannotBeCreated);
                             }
                         }
                     });
-                } else {
-                    if (counter.decrementAndGet() == 0) {
-                        executeBulk(task, bulkRequest, startTime, listener, responses);
-                    }
                 }
             }
         } else {
-            executeBulk(task, bulkRequest, startTime, listener, responses);
+            executeBulk(task, bulkRequest, startTime, listener, responses, emptyMap());
         }
     }
 
@@ -188,30 +214,21 @@ boolean shouldAutoCreate(String index, ClusterState state) {
         return autoCreateIndex.shouldAutoCreate(index, state);
     }
 
-    private boolean setResponseFailureIfIndexMatches(AtomicArray<BulkItemResponse> responses, int idx, ActionRequest request, String index, Exception e) {
-        if (request instanceof IndexRequest) {
-            IndexRequest indexRequest = (IndexRequest) request;
-            if (index.equals(indexRequest.index())) {
-                responses.set(idx, new BulkItemResponse(idx, "index", new BulkItemResponse.Failure(indexRequest.index(), indexRequest.type(), indexRequest.id(), e)));
-                return true;
-            }
-        } else if (request instanceof DeleteRequest) {
-            DeleteRequest deleteRequest = (DeleteRequest) request;
-            if (index.equals(deleteRequest.index())) {
-                responses.set(idx, new BulkItemResponse(idx, "delete", new BulkItemResponse.Failure(deleteRequest.index(), deleteRequest.type(), deleteRequest.id(), e)));
-                return true;
-            }
-        } else if (request instanceof UpdateRequest) {
-            UpdateRequest updateRequest = (UpdateRequest) request;
-            if (index.equals(updateRequest.index())) {
-                responses.set(idx, new BulkItemResponse(idx, "update", new BulkItemResponse.Failure(updateRequest.index(), updateRequest.type(), updateRequest.id(), e)));
-                return true;
-            }
-        } else {
-            throw new ElasticsearchException("Parsed unknown request in bulk actions: " + request.getClass().getSimpleName());
+    void createIndex(String index, TimeValue timeout, ActionListener<CreateIndexResponse> listener) {
+        CreateIndexRequest createIndexRequest = new CreateIndexRequest();
+        createIndexRequest.index(index);
+        createIndexRequest.cause("auto(bulk api)");
+        createIndexRequest.masterNodeTimeout(timeout);
+        createIndexAction.execute(createIndexRequest, listener);
+    }
+
+    private boolean setResponseFailureIfIndexMatches(AtomicArray<BulkItemResponse> responses, int idx, DocWriteRequest request, String index, Exception e) {
+        if (index.equals(request.index())) {
+            responses.set(idx, new BulkItemResponse(idx, request.opType(), new BulkItemResponse.Failure(request.index(), request.type(), request.id(), e)));
+            return true;
         }
         return false;
-    }
+     }
 
     /**
      * This method executes the {@link BulkRequest} and calls the given listener once the request returns.
@@ -221,213 +238,240 @@ private boolean setResponseFailureIfIndexMatches(AtomicArray<BulkItemResponse> r
      */
     public void executeBulk(final BulkRequest bulkRequest, final ActionListener<BulkResponse> listener) {
         final long startTimeNanos = relativeTime();
-        executeBulk(null, bulkRequest, startTimeNanos, listener, new AtomicArray<>(bulkRequest.requests.size()));
+        executeBulk(null, bulkRequest, startTimeNanos, listener, new AtomicArray<>(bulkRequest.requests.size()), emptyMap());
     }
 
     private long buildTookInMillis(long startTimeNanos) {
         return TimeUnit.NANOSECONDS.toMillis(relativeTime() - startTimeNanos);
     }
 
-    void executeBulk(Task task, final BulkRequest bulkRequest, final long startTimeNanos, final ActionListener<BulkResponse> listener, final AtomicArray<BulkItemResponse> responses ) {
-        final ClusterState clusterState = clusterService.state();
-        // TODO use timeout to wait here if its blocked...
-        clusterState.blocks().globalBlockedRaiseException(ClusterBlockLevel.WRITE);
-
-        final ConcreteIndices concreteIndices = new ConcreteIndices(clusterState, indexNameExpressionResolver);
-        MetaData metaData = clusterState.metaData();
-        for (int i = 0; i < bulkRequest.requests.size(); i++) {
-            ActionRequest request = bulkRequest.requests.get(i);
-            //the request can only be null because we set it to null in the previous step, so it gets ignored
-            if (request == null) {
-                continue;
-            }
-            DocumentRequest documentRequest = (DocumentRequest) request;
-            if (addFailureIfIndexIsUnavailable(documentRequest, bulkRequest, responses, i, concreteIndices, metaData)) {
-                continue;
+    /**
+     * retries on retryable cluster blocks, resolves item requests,
+     * constructs shard bulk requests and delegates execution to shard bulk action
+     * */
+    private final class BulkOperation extends AbstractRunnable {
+        private final Task task;
+        private final BulkRequest bulkRequest;
+        private final ActionListener<BulkResponse> listener;
+        private final AtomicArray<BulkItemResponse> responses;
+        private final long startTimeNanos;
+        private final ClusterStateObserver observer;
+        private final Map<String, IndexNotFoundException> indicesThatCannotBeCreated;
+
+        BulkOperation(Task task, BulkRequest bulkRequest, ActionListener<BulkResponse> listener, AtomicArray<BulkItemResponse> responses,
+                long startTimeNanos, Map<String, IndexNotFoundException> indicesThatCannotBeCreated) {
+            this.task = task;
+            this.bulkRequest = bulkRequest;
+            this.listener = listener;
+            this.responses = responses;
+            this.startTimeNanos = startTimeNanos;
+            this.indicesThatCannotBeCreated = indicesThatCannotBeCreated;
+            this.observer = new ClusterStateObserver(clusterService, bulkRequest.timeout(), logger, threadPool.getThreadContext());
+        }
+
+        @Override
+        public void onFailure(Exception e) {
+            listener.onFailure(e);
+        }
+
+        @Override
+        protected void doRun() throws Exception {
+            final ClusterState clusterState = observer.setAndGetObservedState();
+            if (handleBlockExceptions(clusterState)) {
+                return;
             }
-            Index concreteIndex = concreteIndices.resolveIfAbsent(documentRequest);
-            if (request instanceof IndexRequest) {
-                IndexRequest indexRequest = (IndexRequest) request;
-                MappingMetaData mappingMd = null;
-                final IndexMetaData indexMetaData = metaData.index(concreteIndex);
-                if (indexMetaData != null) {
-                    mappingMd = indexMetaData.mappingOrDefault(indexRequest.type());
+            final ConcreteIndices concreteIndices = new ConcreteIndices(clusterState, indexNameExpressionResolver);
+            MetaData metaData = clusterState.metaData();
+            for (int i = 0; i < bulkRequest.requests.size(); i++) {
+                DocWriteRequest docWriteRequest = bulkRequest.requests.get(i);
+                //the request can only be null because we set it to null in the previous step, so it gets ignored
+                if (docWriteRequest == null) {
+                    continue;
                 }
-                try {
-                    indexRequest.resolveRouting(metaData);
-                    indexRequest.process(mappingMd, allowIdGeneration, concreteIndex.getName());
-                } catch (ElasticsearchParseException | RoutingMissingException e) {
-                    BulkItemResponse.Failure failure = new BulkItemResponse.Failure(concreteIndex.getName(), indexRequest.type(), indexRequest.id(), e);
-                    BulkItemResponse bulkItemResponse = new BulkItemResponse(i, "index", failure);
-                    responses.set(i, bulkItemResponse);
-                    // make sure the request gets never processed again
-                    bulkRequest.requests.set(i, null);
+                if (addFailureIfIndexIsUnavailable(docWriteRequest, i, concreteIndices, metaData)) {
+                    continue;
                 }
-            } else if (request instanceof DeleteRequest) {
+                Index concreteIndex = concreteIndices.resolveIfAbsent(docWriteRequest);
                 try {
-                    TransportDeleteAction.resolveAndValidateRouting(metaData, concreteIndex.getName(), (DeleteRequest)request);
-                } catch(RoutingMissingException e) {
-                    BulkItemResponse.Failure failure = new BulkItemResponse.Failure(concreteIndex.getName(), documentRequest.type(), documentRequest.id(), e);
-                    BulkItemResponse bulkItemResponse = new BulkItemResponse(i, "delete", failure);
+                    switch (docWriteRequest.opType()) {
+                        case CREATE:
+                        case INDEX:
+                            IndexRequest indexRequest = (IndexRequest) docWriteRequest;
+                            MappingMetaData mappingMd = null;
+                            final IndexMetaData indexMetaData = metaData.index(concreteIndex);
+                            if (indexMetaData != null) {
+                                mappingMd = indexMetaData.mappingOrDefault(indexRequest.type());
+                            }
+                            indexRequest.resolveRouting(metaData);
+                            indexRequest.process(mappingMd, allowIdGeneration, concreteIndex.getName());
+                            break;
+                        case UPDATE:
+                            TransportUpdateAction.resolveAndValidateRouting(metaData, concreteIndex.getName(), (UpdateRequest) docWriteRequest);
+                            break;
+                        case DELETE:
+                            docWriteRequest.routing(metaData.resolveIndexRouting(docWriteRequest.parent(), docWriteRequest.routing(), docWriteRequest.index()));
+                            // check if routing is required, if so, throw error if routing wasn't specified
+                            if (docWriteRequest.routing() == null && metaData.routingRequired(concreteIndex.getName(), docWriteRequest.type())) {
+                                throw new RoutingMissingException(concreteIndex.getName(), docWriteRequest.type(), docWriteRequest.id());
+                            }
+                            break;
+                        default: throw new AssertionError("request type not supported: [" + docWriteRequest.opType() + "]");
+                    }
+                } catch (ElasticsearchParseException | IllegalArgumentException | RoutingMissingException e) {
+                    BulkItemResponse.Failure failure = new BulkItemResponse.Failure(concreteIndex.getName(), docWriteRequest.type(), docWriteRequest.id(), e);
+                    BulkItemResponse bulkItemResponse = new BulkItemResponse(i, docWriteRequest.opType(), failure);
                     responses.set(i, bulkItemResponse);
                     // make sure the request gets never processed again
                     bulkRequest.requests.set(i, null);
                 }
+            }
 
-            } else if (request instanceof UpdateRequest) {
-                try {
-                    TransportUpdateAction.resolveAndValidateRouting(metaData, concreteIndex.getName(), (UpdateRequest)request);
-                } catch(RoutingMissingException e) {
-                    BulkItemResponse.Failure failure = new BulkItemResponse.Failure(concreteIndex.getName(), documentRequest.type(), documentRequest.id(), e);
-                    BulkItemResponse bulkItemResponse = new BulkItemResponse(i, "update", failure);
-                    responses.set(i, bulkItemResponse);
-                    // make sure the request gets never processed again
-                    bulkRequest.requests.set(i, null);
-                }
-            } else {
-                throw new AssertionError("request type not supported: [" + request.getClass().getName() + "]");
+            // first, go over all the requests and create a ShardId -> Operations mapping
+            Map<ShardId, List<BulkItemRequest>> requestsByShard = new HashMap<>();
+            for (int i = 0; i < bulkRequest.requests.size(); i++) {
+                DocWriteRequest request = bulkRequest.requests.get(i);
+                if (request == null) {
+                    continue;
+                 }
+                String concreteIndex = concreteIndices.getConcreteIndex(request.index()).getName();
+                ShardId shardId = clusterService.operationRouting().indexShards(clusterState, concreteIndex, request.id(), request.routing()).shardId();
+                List<BulkItemRequest> shardRequests = requestsByShard.computeIfAbsent(shardId, shard -> new ArrayList<>());
+                shardRequests.add(new BulkItemRequest(i, request));
+             }
+
+            if (requestsByShard.isEmpty()) {
+                listener.onResponse(new BulkResponse(responses.toArray(new BulkItemResponse[responses.length()]), buildTookInMillis(startTimeNanos)));
+                return;
             }
-        }
 
-        // first, go over all the requests and create a ShardId -> Operations mapping
-        Map<ShardId, List<BulkItemRequest>> requestsByShard = new HashMap<>();
-
-        for (int i = 0; i < bulkRequest.requests.size(); i++) {
-            ActionRequest request = bulkRequest.requests.get(i);
-            if (request instanceof IndexRequest) {
-                IndexRequest indexRequest = (IndexRequest) request;
-                String concreteIndex = concreteIndices.getConcreteIndex(indexRequest.index()).getName();
-                ShardId shardId = clusterService.operationRouting().indexShards(clusterState, concreteIndex, indexRequest.id(), indexRequest.routing()).shardId();
-                List<BulkItemRequest> list = requestsByShard.get(shardId);
-                if (list == null) {
-                    list = new ArrayList<>();
-                    requestsByShard.put(shardId, list);
-                }
-                list.add(new BulkItemRequest(i, request));
-            } else if (request instanceof DeleteRequest) {
-                DeleteRequest deleteRequest = (DeleteRequest) request;
-                String concreteIndex = concreteIndices.getConcreteIndex(deleteRequest.index()).getName();
-                ShardId shardId = clusterService.operationRouting().indexShards(clusterState, concreteIndex, deleteRequest.id(), deleteRequest.routing()).shardId();
-                List<BulkItemRequest> list = requestsByShard.get(shardId);
-                if (list == null) {
-                    list = new ArrayList<>();
-                    requestsByShard.put(shardId, list);
-                }
-                list.add(new BulkItemRequest(i, request));
-            } else if (request instanceof UpdateRequest) {
-                UpdateRequest updateRequest = (UpdateRequest) request;
-                String concreteIndex = concreteIndices.getConcreteIndex(updateRequest.index()).getName();
-                ShardId shardId = clusterService.operationRouting().indexShards(clusterState, concreteIndex, updateRequest.id(), updateRequest.routing()).shardId();
-                List<BulkItemRequest> list = requestsByShard.get(shardId);
-                if (list == null) {
-                    list = new ArrayList<>();
-                    requestsByShard.put(shardId, list);
+            final AtomicInteger counter = new AtomicInteger(requestsByShard.size());
+            String nodeId = clusterService.localNode().getId();
+            for (Map.Entry<ShardId, List<BulkItemRequest>> entry : requestsByShard.entrySet()) {
+                final ShardId shardId = entry.getKey();
+                final List<BulkItemRequest> requests = entry.getValue();
+                BulkShardRequest bulkShardRequest = new BulkShardRequest(shardId, bulkRequest.getRefreshPolicy(),
+                        requests.toArray(new BulkItemRequest[requests.size()]));
+                bulkShardRequest.waitForActiveShards(bulkRequest.waitForActiveShards());
+                bulkShardRequest.timeout(bulkRequest.timeout());
+                if (task != null) {
+                    bulkShardRequest.setParentTask(nodeId, task.getId());
                 }
-                list.add(new BulkItemRequest(i, request));
+                shardBulkAction.execute(bulkShardRequest, new ActionListener<BulkShardResponse>() {
+                    @Override
+                    public void onResponse(BulkShardResponse bulkShardResponse) {
+                        for (BulkItemResponse bulkItemResponse : bulkShardResponse.getResponses()) {
+                            // we may have no response if item failed
+                            if (bulkItemResponse.getResponse() != null) {
+                                bulkItemResponse.getResponse().setShardInfo(bulkShardResponse.getShardInfo());
+                            }
+                            responses.set(bulkItemResponse.getItemId(), bulkItemResponse);
+                        }
+                        if (counter.decrementAndGet() == 0) {
+                            finishHim();
+                        }
+                    }
+
+                    @Override
+                    public void onFailure(Exception e) {
+                        // create failures for all relevant requests
+                        for (BulkItemRequest request : requests) {
+                            final String indexName = concreteIndices.getConcreteIndex(request.index()).getName();
+                            DocWriteRequest docWriteRequest = request.request();
+                            responses.set(request.id(), new BulkItemResponse(request.id(), docWriteRequest.opType(),
+                                    new BulkItemResponse.Failure(indexName, docWriteRequest.type(), docWriteRequest.id(), e)));
+                        }
+                        if (counter.decrementAndGet() == 0) {
+                            finishHim();
+                        }
+                    }
+
+                    private void finishHim() {
+                        listener.onResponse(new BulkResponse(responses.toArray(new BulkItemResponse[responses.length()]), buildTookInMillis(startTimeNanos)));
+                    }
+                });
             }
         }
 
-        if (requestsByShard.isEmpty()) {
-            listener.onResponse(new BulkResponse(responses.toArray(new BulkItemResponse[responses.length()]), buildTookInMillis(startTimeNanos)));
-            return;
+        private boolean handleBlockExceptions(ClusterState state) {
+            ClusterBlockException blockException = state.blocks().globalBlockedException(ClusterBlockLevel.WRITE);
+            if (blockException != null) {
+                if (blockException.retryable()) {
+                    logger.trace("cluster is blocked, scheduling a retry", blockException);
+                    retry(blockException);
+                } else {
+                    onFailure(blockException);
+                }
+                return true;
+            }
+            return false;
         }
 
-        final AtomicInteger counter = new AtomicInteger(requestsByShard.size());
-        String nodeId = clusterService.localNode().getId();
-        for (Map.Entry<ShardId, List<BulkItemRequest>> entry : requestsByShard.entrySet()) {
-            final ShardId shardId = entry.getKey();
-            final List<BulkItemRequest> requests = entry.getValue();
-            BulkShardRequest bulkShardRequest = new BulkShardRequest(shardId, bulkRequest.getRefreshPolicy(),
-                    requests.toArray(new BulkItemRequest[requests.size()]));
-            bulkShardRequest.waitForActiveShards(bulkRequest.waitForActiveShards());
-            bulkShardRequest.timeout(bulkRequest.timeout());
-            if (task != null) {
-                bulkShardRequest.setParentTask(nodeId, task.getId());
+        void retry(Exception failure) {
+            assert failure != null;
+            if (observer.isTimedOut()) {
+                // we running as a last attempt after a timeout has happened. don't retry
+                onFailure(failure);
+                return;
             }
-            shardBulkAction.execute(bulkShardRequest, new ActionListener<BulkShardResponse>() {
+            final ThreadContext.StoredContext context = threadPool.getThreadContext().newStoredContext(true);
+            observer.waitForNextChange(new ClusterStateObserver.Listener() {
                 @Override
-                public void onResponse(BulkShardResponse bulkShardResponse) {
-                    for (BulkItemResponse bulkItemResponse : bulkShardResponse.getResponses()) {
-                        // we may have no response if item failed
-                        if (bulkItemResponse.getResponse() != null) {
-                            bulkItemResponse.getResponse().setShardInfo(bulkShardResponse.getShardInfo());
-                        }
-                        responses.set(bulkItemResponse.getItemId(), bulkItemResponse);
-                    }
-                    if (counter.decrementAndGet() == 0) {
-                        finishHim();
-                    }
+                public void onNewClusterState(ClusterState state) {
+                    context.close();
+                    run();
                 }
 
                 @Override
-                public void onFailure(Exception e) {
-                    // create failures for all relevant requests
-                    for (BulkItemRequest request : requests) {
-                        final String indexName = concreteIndices.getConcreteIndex(request.index()).getName();
-                        if (request.request() instanceof IndexRequest) {
-                            IndexRequest indexRequest = (IndexRequest) request.request();
-                            responses.set(request.id(), new BulkItemResponse(request.id(), indexRequest.opType().toString().toLowerCase(Locale.ENGLISH),
-                                    new BulkItemResponse.Failure(indexName, indexRequest.type(), indexRequest.id(), e)));
-                        } else if (request.request() instanceof DeleteRequest) {
-                            DeleteRequest deleteRequest = (DeleteRequest) request.request();
-                            responses.set(request.id(), new BulkItemResponse(request.id(), "delete",
-                                    new BulkItemResponse.Failure(indexName, deleteRequest.type(), deleteRequest.id(), e)));
-                        } else if (request.request() instanceof UpdateRequest) {
-                            UpdateRequest updateRequest = (UpdateRequest) request.request();
-                            responses.set(request.id(), new BulkItemResponse(request.id(), "update",
-                                    new BulkItemResponse.Failure(indexName, updateRequest.type(), updateRequest.id(), e)));
-                        }
-                    }
-                    if (counter.decrementAndGet() == 0) {
-                        finishHim();
-                    }
+                public void onClusterServiceClose() {
+                    onFailure(new NodeClosedException(clusterService.localNode()));
                 }
 
-                private void finishHim() {
-                    listener.onResponse(new BulkResponse(responses.toArray(new BulkItemResponse[responses.length()]), buildTookInMillis(startTimeNanos)));
+                @Override
+                public void onTimeout(TimeValue timeout) {
+                    context.close();
+                    // Try one more time...
+                    run();
                 }
             });
         }
-    }
 
-    private boolean addFailureIfIndexIsUnavailable(DocumentRequest request, BulkRequest bulkRequest, AtomicArray<BulkItemResponse> responses, int idx,
-                                              final ConcreteIndices concreteIndices,
-                                              final MetaData metaData) {
-        Index concreteIndex = concreteIndices.getConcreteIndex(request.index());
-        Exception unavailableException = null;
-        if (concreteIndex == null) {
-            try {
-                concreteIndex = concreteIndices.resolveIfAbsent(request);
-            } catch (IndexClosedException | IndexNotFoundException ex) {
-                // Fix for issue where bulk request references an index that
-                // cannot be auto-created see issue #8125
-                unavailableException = ex;
+        private boolean addFailureIfIndexIsUnavailable(DocWriteRequest request, int idx, final ConcreteIndices concreteIndices,
+                final MetaData metaData) {
+            IndexNotFoundException cannotCreate = indicesThatCannotBeCreated.get(request.index());
+            if (cannotCreate != null) {
+                addFailure(request, idx, cannotCreate);
+                return true;
+            }
+            Index concreteIndex = concreteIndices.getConcreteIndex(request.index());
+            if (concreteIndex == null) {
+                try {
+                    concreteIndex = concreteIndices.resolveIfAbsent(request);
+                } catch (IndexClosedException | IndexNotFoundException ex) {
+                    addFailure(request, idx, ex);
+                    return true;
+                }
             }
-        }
-        if (unavailableException == null) {
             IndexMetaData indexMetaData = metaData.getIndexSafe(concreteIndex);
             if (indexMetaData.getState() == IndexMetaData.State.CLOSE) {
-                unavailableException = new IndexClosedException(concreteIndex);
+                addFailure(request, idx, new IndexClosedException(concreteIndex));
+                return true;
             }
+            return false;
         }
-        if (unavailableException != null) {
+
+        private void addFailure(DocWriteRequest request, int idx, Exception unavailableException) {
             BulkItemResponse.Failure failure = new BulkItemResponse.Failure(request.index(), request.type(), request.id(),
                     unavailableException);
-            String operationType = "unknown";
-            if (request instanceof IndexRequest) {
-                operationType = "index";
-            } else if (request instanceof DeleteRequest) {
-                operationType = "delete";
-            } else if (request instanceof UpdateRequest) {
-                operationType = "update";
-            }
-            BulkItemResponse bulkItemResponse = new BulkItemResponse(idx, operationType, failure);
+            BulkItemResponse bulkItemResponse = new BulkItemResponse(idx, request.opType(), failure);
             responses.set(idx, bulkItemResponse);
             // make sure the request gets never processed again
             bulkRequest.requests.set(idx, null);
-            return true;
         }
-        return false;
+    }
+
+    void executeBulk(Task task, final BulkRequest bulkRequest, final long startTimeNanos, final ActionListener<BulkResponse> listener,
+            final AtomicArray<BulkItemResponse> responses, Map<String, IndexNotFoundException> indicesThatCannotBeCreated) {
+        new BulkOperation(task, bulkRequest, listener, responses, startTimeNanos, indicesThatCannotBeCreated).run();
     }
 
     private static class ConcreteIndices  {
@@ -444,7 +488,7 @@ Index getConcreteIndex(String indexOrAlias) {
             return indices.get(indexOrAlias);
         }
 
-        Index resolveIfAbsent(DocumentRequest request) {
+        Index resolveIfAbsent(DocWriteRequest request) {
             Index concreteIndex = indices.get(request.index());
             if (concreteIndex == null) {
                 concreteIndex = indexNameExpressionResolver.concreteSingleIndex(state, request);
@@ -458,4 +502,131 @@ private long relativeTime() {
         return relativeTimeProvider.getAsLong();
     }
 
+    void processBulkIndexIngestRequest(Task task, BulkRequest original, ActionListener<BulkResponse> listener) {
+        long ingestStartTimeInNanos = System.nanoTime();
+        BulkRequestModifier bulkRequestModifier = new BulkRequestModifier(original);
+        ingestService.getPipelineExecutionService().executeBulkRequest(() -> bulkRequestModifier, (indexRequest, exception) -> {
+            logger.debug((Supplier<?>) () -> new ParameterizedMessage("failed to execute pipeline [{}] for document [{}/{}/{}]",
+                indexRequest.getPipeline(), indexRequest.index(), indexRequest.type(), indexRequest.id()), exception);
+            bulkRequestModifier.markCurrentItemAsFailed(exception);
+        }, (exception) -> {
+            if (exception != null) {
+                logger.error("failed to execute pipeline for a bulk request", exception);
+                listener.onFailure(exception);
+            } else {
+                long ingestTookInMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - ingestStartTimeInNanos);
+                BulkRequest bulkRequest = bulkRequestModifier.getBulkRequest();
+                ActionListener<BulkResponse> actionListener = bulkRequestModifier.wrapActionListenerIfNeeded(ingestTookInMillis, listener);
+                if (bulkRequest.requests().isEmpty()) {
+                    // at this stage, the transport bulk action can't deal with a bulk request with no requests,
+                    // so we stop and send an empty response back to the client.
+                    // (this will happen if pre-processing all items in the bulk failed)
+                    actionListener.onResponse(new BulkResponse(new BulkItemResponse[0], 0));
+                } else {
+                    doExecute(task, bulkRequest, actionListener);
+                }
+            }
+        });
+    }
+
+    static final class BulkRequestModifier implements Iterator<DocWriteRequest> {
+
+        final BulkRequest bulkRequest;
+        final SparseFixedBitSet failedSlots;
+        final List<BulkItemResponse> itemResponses;
+
+        int currentSlot = -1;
+        int[] originalSlots;
+
+        BulkRequestModifier(BulkRequest bulkRequest) {
+            this.bulkRequest = bulkRequest;
+            this.failedSlots = new SparseFixedBitSet(bulkRequest.requests().size());
+            this.itemResponses = new ArrayList<>(bulkRequest.requests().size());
+        }
+
+        @Override
+        public DocWriteRequest next() {
+            return bulkRequest.requests().get(++currentSlot);
+        }
+
+        @Override
+        public boolean hasNext() {
+            return (currentSlot + 1) < bulkRequest.requests().size();
+        }
+
+        BulkRequest getBulkRequest() {
+            if (itemResponses.isEmpty()) {
+                return bulkRequest;
+            } else {
+                BulkRequest modifiedBulkRequest = new BulkRequest();
+                modifiedBulkRequest.setRefreshPolicy(bulkRequest.getRefreshPolicy());
+                modifiedBulkRequest.waitForActiveShards(bulkRequest.waitForActiveShards());
+                modifiedBulkRequest.timeout(bulkRequest.timeout());
+
+                int slot = 0;
+                List<DocWriteRequest> requests = bulkRequest.requests();
+                originalSlots = new int[requests.size()]; // oversize, but that's ok
+                for (int i = 0; i < requests.size(); i++) {
+                    DocWriteRequest request = requests.get(i);
+                    if (failedSlots.get(i) == false) {
+                        modifiedBulkRequest.add(request);
+                        originalSlots[slot++] = i;
+                    }
+                }
+                return modifiedBulkRequest;
+            }
+        }
+
+        ActionListener<BulkResponse> wrapActionListenerIfNeeded(long ingestTookInMillis, ActionListener<BulkResponse> actionListener) {
+            if (itemResponses.isEmpty()) {
+                return ActionListener.wrap(
+                    response -> actionListener.onResponse(
+                        new BulkResponse(response.getItems(), response.getTookInMillis(), ingestTookInMillis)),
+                    actionListener::onFailure);
+            } else {
+                return new IngestBulkResponseListener(ingestTookInMillis, originalSlots, itemResponses, actionListener);
+            }
+        }
+
+        void markCurrentItemAsFailed(Exception e) {
+            IndexRequest indexRequest = (IndexRequest) bulkRequest.requests().get(currentSlot);
+            // We hit a error during preprocessing a request, so we:
+            // 1) Remember the request item slot from the bulk, so that we're done processing all requests we know what failed
+            // 2) Add a bulk item failure for this request
+            // 3) Continue with the next request in the bulk.
+            failedSlots.set(currentSlot);
+            BulkItemResponse.Failure failure = new BulkItemResponse.Failure(indexRequest.index(), indexRequest.type(), indexRequest.id(), e);
+            itemResponses.add(new BulkItemResponse(currentSlot, indexRequest.opType(), failure));
+        }
+
+    }
+
+    static final class IngestBulkResponseListener implements ActionListener<BulkResponse> {
+
+        private final long ingestTookInMillis;
+        private final int[] originalSlots;
+        private final List<BulkItemResponse> itemResponses;
+        private final ActionListener<BulkResponse> actionListener;
+
+        IngestBulkResponseListener(long ingestTookInMillis, int[] originalSlots, List<BulkItemResponse> itemResponses, ActionListener<BulkResponse> actionListener) {
+            this.ingestTookInMillis = ingestTookInMillis;
+            this.itemResponses = itemResponses;
+            this.actionListener = actionListener;
+            this.originalSlots = originalSlots;
+        }
+
+        @Override
+        public void onResponse(BulkResponse response) {
+            BulkItemResponse[] items = response.getItems();
+            for (int i = 0; i < items.length; i++) {
+                itemResponses.add(originalSlots[i], response.getItems()[i]);
+            }
+            actionListener.onResponse(new BulkResponse(itemResponses.toArray(new BulkItemResponse[itemResponses.size()]), response.getTookInMillis(), ingestTookInMillis));
+        }
+
+        @Override
+        public void onFailure(Exception e) {
+            actionListener.onFailure(e);
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java b/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java
index 754316f3de008..dfa18a2deb65d 100644
--- a/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java
@@ -21,17 +21,16 @@
 
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
-import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.delete.DeleteResponse;
-import org.elasticsearch.action.delete.TransportDeleteAction;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.index.IndexResponse;
-import org.elasticsearch.action.index.TransportIndexAction;
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.replication.ReplicationRequest;
+import org.elasticsearch.action.support.TransportActions;
+import org.elasticsearch.action.support.replication.ReplicationOperation;
 import org.elasticsearch.action.support.replication.ReplicationResponse.ShardInfo;
 import org.elasticsearch.action.support.replication.TransportWriteAction;
 import org.elasticsearch.action.update.UpdateHelper;
@@ -49,32 +48,26 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.engine.VersionConflictEngineException;
+import org.elasticsearch.index.mapper.MapperParsingException;
+import org.elasticsearch.index.get.GetResult;
+import org.elasticsearch.index.mapper.Mapping;
+import org.elasticsearch.index.mapper.SourceToParse;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.translog.Translog;
-import org.elasticsearch.index.translog.Translog.Location;
 import org.elasticsearch.indices.IndicesService;
-import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportService;
 
+import java.io.IOException;
 import java.util.Map;
 
-import static org.elasticsearch.action.support.replication.ReplicationOperation.ignoreReplicaException;
-import static org.elasticsearch.action.support.replication.ReplicationOperation.isConflictException;
-
-/**
- * Performs the index operation.
- */
-public class TransportShardBulkAction extends TransportWriteAction<BulkShardRequest, BulkShardResponse> {
-
-    private static final String OP_TYPE_UPDATE = "update";
-    private static final String OP_TYPE_DELETE = "delete";
+/** Performs shard-level bulk (index, delete or update) operations */
+public class TransportShardBulkAction extends TransportWriteAction<BulkShardRequest, BulkShardRequest, BulkShardResponse> {
 
     public static final String ACTION_NAME = BulkAction.NAME + "[s]";
 
@@ -88,7 +81,7 @@ public TransportShardBulkAction(Settings settings, TransportService transportSer
                                     MappingUpdatedAction mappingUpdatedAction, UpdateHelper updateHelper, ActionFilters actionFilters,
                                     IndexNameExpressionResolver indexNameExpressionResolver) {
         super(settings, ACTION_NAME, transportService, clusterService, indicesService, threadPool, shardStateAction, actionFilters,
-                indexNameExpressionResolver, BulkShardRequest::new, ThreadPool.Names.BULK);
+                indexNameExpressionResolver, BulkShardRequest::new, BulkShardRequest::new, ThreadPool.Names.BULK);
         this.updateHelper = updateHelper;
         this.allowIdGeneration = settings.getAsBoolean("action.allow_id_generation", true);
         this.mappingUpdatedAction = mappingUpdatedAction;
@@ -110,17 +103,17 @@ protected boolean resolveIndex() {
     }
 
     @Override
-    protected WriteResult<BulkShardResponse> onPrimaryShard(BulkShardRequest request, IndexShard indexShard) throws Exception {
-        ShardId shardId = request.shardId();
-        final IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
-        final IndexMetaData metaData = indexService.getIndexSettings().getIndexMetaData();
+    public WritePrimaryResult<BulkShardRequest, BulkShardResponse> shardOperationOnPrimary(
+            BulkShardRequest request, IndexShard primary) throws Exception {
+        final IndexMetaData metaData = primary.indexSettings().getIndexMetaData();
 
         long[] preVersions = new long[request.items().length];
         VersionType[] preVersionTypes = new VersionType[request.items().length];
         Translog.Location location = null;
         for (int requestIndex = 0; requestIndex < request.items().length; requestIndex++) {
-            BulkItemRequest item = request.items()[requestIndex];
-            location = handleItem(metaData, request, indexShard, preVersions, preVersionTypes, location, requestIndex, item);
+            if (isAborted(request.items()[requestIndex].getPrimaryResponse()) == false) {
+                location = executeBulkItemRequest(metaData, primary, request, preVersions, preVersionTypes, location, requestIndex);
+            }
         }
 
         BulkItemResponse[] responses = new BulkItemResponse[request.items().length];
@@ -129,369 +122,296 @@ protected WriteResult<BulkShardResponse> onPrimaryShard(BulkShardRequest request
             responses[i] = items[i].getPrimaryResponse();
         }
         BulkShardResponse response = new BulkShardResponse(request.shardId(), responses);
-        return new WriteResult<>(response, location);
-    }
-
-    private Translog.Location handleItem(IndexMetaData metaData, BulkShardRequest request, IndexShard indexShard, long[] preVersions, VersionType[] preVersionTypes, Translog.Location location, int requestIndex, BulkItemRequest item) {
-        if (item.request() instanceof IndexRequest) {
-            location = index(metaData, request, indexShard, preVersions, preVersionTypes, location, requestIndex, item);
-        } else if (item.request() instanceof DeleteRequest) {
-            location = delete(request, indexShard, preVersions, preVersionTypes, location, requestIndex, item);
-        } else if (item.request() instanceof UpdateRequest) {
-            Tuple<Translog.Location, BulkItemRequest> tuple = update(metaData, request, indexShard, preVersions, preVersionTypes, location, requestIndex, item);
-            location = tuple.v1();
-            item = tuple.v2();
-        } else {
-            throw new IllegalStateException("Unexpected index operation: " + item.request());
-        }
-
-        assert item.getPrimaryResponse() != null;
-        assert preVersionTypes[requestIndex] != null;
-        return location;
+        return new WritePrimaryResult<>(request, response, location, null, primary, logger);
     }
 
-    private Translog.Location index(IndexMetaData metaData, BulkShardRequest request, IndexShard indexShard, long[] preVersions, VersionType[] preVersionTypes, Translog.Location location, int requestIndex, BulkItemRequest item) {
-        IndexRequest indexRequest = (IndexRequest) item.request();
-        preVersions[requestIndex] = indexRequest.version();
-        preVersionTypes[requestIndex] = indexRequest.versionType();
+    /** Executes bulk item requests and handles request execution exceptions */
+    private Translog.Location executeBulkItemRequest(IndexMetaData metaData, IndexShard primary,
+                                                     BulkShardRequest request,
+                                                     long[] preVersions, VersionType[] preVersionTypes,
+                                                     Translog.Location location, int requestIndex) throws Exception {
+        final DocWriteRequest itemRequest = request.items()[requestIndex].request();
+        preVersions[requestIndex] = itemRequest.version();
+        preVersionTypes[requestIndex] = itemRequest.versionType();
+        DocWriteRequest.OpType opType = itemRequest.opType();
         try {
-            WriteResult<IndexResponse> result = shardIndexOperation(request, indexRequest, metaData, indexShard, true);
-            location = locationToSync(location, result.getLocation());
-            // add the response
-            IndexResponse indexResponse = result.getResponse();
-            setResponse(item, new BulkItemResponse(item.id(), indexRequest.opType().lowercase(), indexResponse));
-        } catch (Exception e) {
-            // rethrow the failure if we are going to retry on primary and let parent failure to handle it
-            if (retryPrimaryException(e)) {
-                // restore updated versions...
-                for (int j = 0; j < requestIndex; j++) {
-                    applyVersion(request.items()[j], preVersions[j], preVersionTypes[j]);
-                }
-                throw (ElasticsearchException) e;
+            // execute item request
+            final Engine.Result operationResult;
+            final DocWriteResponse response;
+            final BulkItemRequest replicaRequest;
+            switch (itemRequest.opType()) {
+                case CREATE:
+                case INDEX:
+                    final IndexRequest indexRequest = (IndexRequest) itemRequest;
+                    Engine.IndexResult indexResult = executeIndexRequestOnPrimary(indexRequest, primary, mappingUpdatedAction);
+                    if (indexResult.hasFailure()) {
+                        response = null;
+                    } else {
+                        // update the version on request so it will happen on the replicas
+                        final long version = indexResult.getVersion();
+                        indexRequest.version(version);
+                        indexRequest.versionType(indexRequest.versionType().versionTypeForReplicationAndRecovery());
+                        assert indexRequest.versionType().validateVersionForWrites(indexRequest.version());
+                        response = new IndexResponse(primary.shardId(), indexRequest.type(), indexRequest.id(),
+                                indexResult.getVersion(), indexResult.isCreated());
+                    }
+                    operationResult = indexResult;
+                    replicaRequest = request.items()[requestIndex];
+                    break;
+                case UPDATE:
+                    UpdateResultHolder updateResultHolder = executeUpdateRequest(((UpdateRequest) itemRequest),
+                            primary, metaData, request, requestIndex);
+                    operationResult = updateResultHolder.operationResult;
+                    response = updateResultHolder.response;
+                    replicaRequest = updateResultHolder.replicaRequest;
+                    break;
+                case DELETE:
+                    final DeleteRequest deleteRequest = (DeleteRequest) itemRequest;
+                    Engine.DeleteResult deleteResult = executeDeleteRequestOnPrimary(deleteRequest, primary, mappingUpdatedAction);
+                    if (deleteResult.hasFailure()) {
+                        response = null;
+                    } else {
+                        // update the request with the version so it will go to the replicas
+                        deleteRequest.versionType(deleteRequest.versionType().versionTypeForReplicationAndRecovery());
+                        deleteRequest.version(deleteResult.getVersion());
+                        assert deleteRequest.versionType().validateVersionForWrites(deleteRequest.version());
+                        response = new DeleteResponse(request.shardId(), deleteRequest.type(), deleteRequest.id(),
+                                deleteResult.getVersion(), deleteResult.isFound());
+                    }
+                    operationResult = deleteResult;
+                    replicaRequest = request.items()[requestIndex];
+                    break;
+                default:
+                    throw new IllegalStateException("unexpected opType [" + itemRequest.opType() + "] found");
             }
-            logFailure(e, "index", request.shardId(), indexRequest);
-            // if its a conflict failure, and we already executed the request on a primary (and we execute it
-            // again, due to primary relocation and only processing up to N bulk items when the shard gets closed)
-            // then just use the response we got from the successful execution
-            if (item.getPrimaryResponse() != null && isConflictException(e)) {
-                setResponse(item, item.getPrimaryResponse());
+            // update the bulk item request because update request execution can mutate the bulk item request
+            request.items()[requestIndex] = replicaRequest;
+            if (operationResult == null) { // in case of noop update operation
+                assert response.getResult() == DocWriteResponse.Result.NOOP
+                        : "only noop update can have null operation";
+                replicaRequest.setPrimaryResponse(new BulkItemResponse(replicaRequest.id(), opType, response));
+                assert replicaRequest.isIgnoreOnReplica();
+            } else if (operationResult.hasFailure() == false) {
+                location = locationToSync(location, operationResult.getTranslogLocation());
+                BulkItemResponse primaryResponse = new BulkItemResponse(replicaRequest.id(), opType, response);
+                replicaRequest.setPrimaryResponse(primaryResponse);
+                // set an empty ShardInfo to indicate no shards participated in the request execution
+                // so we can safely send it to the replicas. We won't use it in the real response though.
+                primaryResponse.getResponse().setShardInfo(new ShardInfo());
+                assert replicaRequest.isIgnoreOnReplica() == false;
             } else {
-                setResponse(item, new BulkItemResponse(item.id(), indexRequest.opType().lowercase(),
-                        new BulkItemResponse.Failure(request.index(), indexRequest.type(), indexRequest.id(), e)));
+                DocWriteRequest docWriteRequest = replicaRequest.request();
+                Exception failure = operationResult.getFailure();
+                if (isConflictException(failure)) {
+                    logger.trace((Supplier<?>) () -> new ParameterizedMessage("{} failed to execute bulk item ({}) {}",
+                            request.shardId(), docWriteRequest.opType().getLowercase(), request), failure);
+                } else {
+                    logger.debug((Supplier<?>) () -> new ParameterizedMessage("{} failed to execute bulk item ({}) {}",
+                            request.shardId(), docWriteRequest.opType().getLowercase(), request), failure);
+                }
+                // if its a conflict failure, and we already executed the request on a primary (and we execute it
+                // again, due to primary relocation and only processing up to N bulk items when the shard gets closed)
+                // then just use the response we got from the successful execution
+                if (replicaRequest.getPrimaryResponse() == null || isConflictException(failure) == false) {
+                    replicaRequest.setPrimaryResponse(new BulkItemResponse(replicaRequest.id(), docWriteRequest.opType(),
+                            new BulkItemResponse.Failure(request.index(), docWriteRequest.type(), docWriteRequest.id(), failure)));
+                    assert replicaRequest.isIgnoreOnReplica();
+                }
             }
-        }
-        return location;
-    }
-
-    private <ReplicationRequestT extends ReplicationRequest<ReplicationRequestT>> void logFailure(Throwable t, String operation, ShardId shardId, ReplicationRequest<ReplicationRequestT> request) {
-        if (ExceptionsHelper.status(t) == RestStatus.CONFLICT) {
-            logger.trace((Supplier<?>) () -> new ParameterizedMessage("{} failed to execute bulk item ({}) {}", shardId, operation, request), t);
-        } else {
-            logger.debug((Supplier<?>) () -> new ParameterizedMessage("{} failed to execute bulk item ({}) {}", shardId, operation, request), t);
-        }
-    }
-
-    private Translog.Location delete(BulkShardRequest request, IndexShard indexShard, long[] preVersions, VersionType[] preVersionTypes, Translog.Location location, int requestIndex, BulkItemRequest item) {
-        DeleteRequest deleteRequest = (DeleteRequest) item.request();
-        preVersions[requestIndex] = deleteRequest.version();
-        preVersionTypes[requestIndex] = deleteRequest.versionType();
-
-        try {
-            // add the response
-            final WriteResult<DeleteResponse> writeResult = TransportDeleteAction.executeDeleteRequestOnPrimary(deleteRequest, indexShard);
-            DeleteResponse deleteResponse = writeResult.getResponse();
-            location = locationToSync(location, writeResult.getLocation());
-            setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_DELETE, deleteResponse));
+            assert replicaRequest.getPrimaryResponse() != null;
+            assert preVersionTypes[requestIndex] != null;
         } catch (Exception e) {
             // rethrow the failure if we are going to retry on primary and let parent failure to handle it
             if (retryPrimaryException(e)) {
                 // restore updated versions...
                 for (int j = 0; j < requestIndex; j++) {
-                    applyVersion(request.items()[j], preVersions[j], preVersionTypes[j]);
+                    DocWriteRequest docWriteRequest = request.items()[j].request();
+                    docWriteRequest.version(preVersions[j]);
+                    docWriteRequest.versionType(preVersionTypes[j]);
                 }
-                throw (ElasticsearchException) e;
-            }
-            logFailure(e, "delete", request.shardId(), deleteRequest);
-            // if its a conflict failure, and we already executed the request on a primary (and we execute it
-            // again, due to primary relocation and only processing up to N bulk items when the shard gets closed)
-            // then just use the response we got from the successful execution
-            if (item.getPrimaryResponse() != null && isConflictException(e)) {
-                setResponse(item, item.getPrimaryResponse());
-            } else {
-                setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_DELETE,
-                        new BulkItemResponse.Failure(request.index(), deleteRequest.type(), deleteRequest.id(), e)));
             }
+            throw e;
         }
         return location;
     }
 
-    private Tuple<Translog.Location, BulkItemRequest> update(IndexMetaData metaData, BulkShardRequest request, IndexShard indexShard, long[] preVersions, VersionType[] preVersionTypes, Translog.Location location, int requestIndex, BulkItemRequest item) {
-        UpdateRequest updateRequest = (UpdateRequest) item.request();
-        preVersions[requestIndex] = updateRequest.version();
-        preVersionTypes[requestIndex] = updateRequest.versionType();
-        //  We need to do the requested retries plus the initial attempt. We don't do < 1+retry_on_conflict because retry_on_conflict may be Integer.MAX_VALUE
-        for (int updateAttemptsCount = 0; updateAttemptsCount <= updateRequest.retryOnConflict(); updateAttemptsCount++) {
-            UpdateResult updateResult;
-            try {
-                updateResult = shardUpdateOperation(metaData, request, updateRequest, indexShard);
-            } catch (Exception t) {
-                updateResult = new UpdateResult(null, null, false, t, null);
-            }
-            if (updateResult.success()) {
-                if (updateResult.writeResult != null) {
-                    location = locationToSync(location, updateResult.writeResult.getLocation());
-                }
-                switch (updateResult.result.getResponseResult()) {
-                    case CREATED:
-                    case UPDATED:
-                        @SuppressWarnings("unchecked")
-                        WriteResult<IndexResponse> result = updateResult.writeResult;
-                        IndexRequest indexRequest = updateResult.request();
-                        BytesReference indexSourceAsBytes = indexRequest.source();
-                        // add the response
-                        IndexResponse indexResponse = result.getResponse();
-                        UpdateResponse updateResponse = new UpdateResponse(indexResponse.getShardInfo(), indexResponse.getShardId(), indexResponse.getType(), indexResponse.getId(), indexResponse.getVersion(), indexResponse.getResult());
-                        if (updateRequest.fields() != null && updateRequest.fields().length > 0) {
-                            Tuple<XContentType, Map<String, Object>> sourceAndContent = XContentHelper.convertToMap(indexSourceAsBytes, true);
-                            updateResponse.setGetResult(updateHelper.extractGetResult(updateRequest, request.index(), indexResponse.getVersion(), sourceAndContent.v2(), sourceAndContent.v1(), indexSourceAsBytes));
-                        }
-                        item = request.items()[requestIndex] = new BulkItemRequest(request.items()[requestIndex].id(), indexRequest);
-                        setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, updateResponse));
-                        break;
-                    case DELETED:
-                        @SuppressWarnings("unchecked")
-                        WriteResult<DeleteResponse> writeResult = updateResult.writeResult;
-                        DeleteResponse response = writeResult.getResponse();
-                        DeleteRequest deleteRequest = updateResult.request();
-                        updateResponse = new UpdateResponse(response.getShardInfo(), response.getShardId(), response.getType(), response.getId(), response.getVersion(), response.getResult());
-                        updateResponse.setGetResult(updateHelper.extractGetResult(updateRequest, request.index(), response.getVersion(), updateResult.result.updatedSourceAsMap(), updateResult.result.updateSourceContentType(), null));
-                        // Replace the update request to the translated delete request to execute on the replica.
-                        item = request.items()[requestIndex] = new BulkItemRequest(request.items()[requestIndex].id(), deleteRequest);
-                        setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, updateResponse));
-                        break;
-                    case NOOP:
-                        setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, updateResult.noopResult));
-                        item.setIgnoreOnReplica(); // no need to go to the replica
-                        break;
-                    default:
-                        throw new IllegalStateException("Illegal operation " + updateResult.result.getResponseResult());
-                }
-                // NOTE: Breaking out of the retry_on_conflict loop!
-                break;
-            } else if (updateResult.failure()) {
-                Throwable e = updateResult.error;
-                if (updateResult.retry) {
-                    // updateAttemptCount is 0 based and marks current attempt, if it's equal to retryOnConflict we are going out of the iteration
-                    if (updateAttemptsCount >= updateRequest.retryOnConflict()) {
-                        setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE,
-                            new BulkItemResponse.Failure(request.index(), updateRequest.type(), updateRequest.id(), e)));
-                    }
-                } else {
-                    // rethrow the failure if we are going to retry on primary and let parent failure to handle it
-                    if (retryPrimaryException(e)) {
-                        // restore updated versions...
-                        for (int j = 0; j < requestIndex; j++) {
-                            applyVersion(request.items()[j], preVersions[j], preVersionTypes[j]);
-                        }
-                        throw (ElasticsearchException) e;
-                    }
-                    // if its a conflict failure, and we already executed the request on a primary (and we execute it
-                    // again, due to primary relocation and only processing up to N bulk items when the shard gets closed)
-                    // then just use the response we got from the successful execution
-                    if (item.getPrimaryResponse() != null && isConflictException(e)) {
-                        setResponse(item, item.getPrimaryResponse());
-                    } else if (updateResult.result == null) {
-                        setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, new BulkItemResponse.Failure(request.index(), updateRequest.type(), updateRequest.id(), e)));
-                    } else {
-                        switch (updateResult.result.getResponseResult()) {
-                            case CREATED:
-                            case UPDATED:
-                                IndexRequest indexRequest = updateResult.request();
-                                logFailure(e, "index", request.shardId(), indexRequest);
-                                setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE,
-                                    new BulkItemResponse.Failure(request.index(), indexRequest.type(), indexRequest.id(), e)));
-                                break;
-                            case DELETED:
-                                DeleteRequest deleteRequest = updateResult.request();
-                                logFailure(e, "delete", request.shardId(), deleteRequest);
-                                setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_DELETE,
-                                    new BulkItemResponse.Failure(request.index(), deleteRequest.type(), deleteRequest.id(), e)));
-                                break;
-                            default:
-                                throw new IllegalStateException("Illegal operation " + updateResult.result.getResponseResult());
-                        }
-                    }
-                    // NOTE: Breaking out of the retry_on_conflict loop!
-                    break;
-                }
-
-            }
-        }
-        return Tuple.tuple(location, item);
+    private static boolean isAborted(BulkItemResponse response) {
+        return response != null && response.isFailed() && response.getFailure().isAborted();
     }
 
-    private void setResponse(BulkItemRequest request, BulkItemResponse response) {
-        request.setPrimaryResponse(response);
-        if (response.isFailed()) {
-            request.setIgnoreOnReplica();
-        } else {
-            // Set the ShardInfo to 0 so we can safely send it to the replicas. We won't use it in the real response though.
-            response.getResponse().setShardInfo(new ShardInfo());
-        }
+    private static boolean isConflictException(final Exception e) {
+        return ExceptionsHelper.unwrapCause(e) instanceof VersionConflictEngineException;
     }
 
-    private WriteResult<IndexResponse> shardIndexOperation(BulkShardRequest request, IndexRequest indexRequest, IndexMetaData metaData,
-            IndexShard indexShard, boolean processed) throws Exception {
+    private static class UpdateResultHolder {
+        final BulkItemRequest replicaRequest;
+        final Engine.Result operationResult;
+        final DocWriteResponse response;
 
-        MappingMetaData mappingMd = metaData.mappingOrDefault(indexRequest.type());
-        if (!processed) {
-            indexRequest.process(mappingMd, allowIdGeneration, request.index());
+        private UpdateResultHolder(BulkItemRequest replicaRequest, Engine.Result operationResult,
+                                   DocWriteResponse response) {
+            this.replicaRequest = replicaRequest;
+            this.operationResult = operationResult;
+            this.response = response;
         }
-        return TransportIndexAction.executeIndexRequestOnPrimary(indexRequest, indexShard, mappingUpdatedAction);
     }
 
-    static class UpdateResult {
-
-        final UpdateHelper.Result result;
-        final ActionRequest actionRequest;
-        final boolean retry;
-        final Throwable error;
-        final WriteResult writeResult;
-        final UpdateResponse noopResult;
-
-        UpdateResult(UpdateHelper.Result result, ActionRequest actionRequest, boolean retry, Throwable error, WriteResult writeResult) {
-            this.result = result;
-            this.actionRequest = actionRequest;
-            this.retry = retry;
-            this.error = error;
-            this.writeResult = writeResult;
-            this.noopResult = null;
-        }
-
-        UpdateResult(UpdateHelper.Result result, ActionRequest actionRequest, WriteResult writeResult) {
-            this.result = result;
-            this.actionRequest = actionRequest;
-            this.writeResult = writeResult;
-            this.retry = false;
-            this.error = null;
-            this.noopResult = null;
-        }
-
-        public UpdateResult(UpdateHelper.Result result, UpdateResponse updateResponse) {
-            this.result = result;
-            this.noopResult = updateResponse;
-            this.actionRequest = null;
-            this.writeResult = null;
-            this.retry = false;
-            this.error = null;
-        }
-
-
-        boolean failure() {
-            return error != null;
-        }
-
-        boolean success() {
-            return noopResult != null || writeResult != null;
-        }
-
-        @SuppressWarnings("unchecked")
-        <T extends ActionRequest> T request() {
-            return (T) actionRequest;
-        }
-
-
-    }
-
-    private UpdateResult shardUpdateOperation(IndexMetaData metaData, BulkShardRequest bulkShardRequest, UpdateRequest updateRequest, IndexShard indexShard) {
-        UpdateHelper.Result translate = updateHelper.prepare(updateRequest, indexShard);
-        switch (translate.getResponseResult()) {
-            case CREATED:
-            case UPDATED:
-                IndexRequest indexRequest = translate.action();
-                try {
-                    WriteResult result = shardIndexOperation(bulkShardRequest, indexRequest, metaData, indexShard, false);
-                    return new UpdateResult(translate, indexRequest, result);
-                } catch (Exception e) {
-                    final Throwable cause = ExceptionsHelper.unwrapCause(e);
-                    boolean retry = false;
-                    if (cause instanceof VersionConflictEngineException) {
-                        retry = true;
+    /**
+     * Executes update request, delegating to a index or delete operation after translation,
+     * handles retries on version conflict and constructs update response
+     * NOTE: reassigns bulk item request at <code>requestIndex</code> for replicas to
+     * execute translated update request (NOOP update is an exception). NOOP updates are
+     * indicated by returning a <code>null</code> operation in {@link UpdateResultHolder}
+     * */
+    private UpdateResultHolder executeUpdateRequest(UpdateRequest updateRequest, IndexShard primary,
+                                                    IndexMetaData metaData, BulkShardRequest request,
+                                                    int requestIndex) throws Exception {
+        Engine.Result updateOperationResult = null;
+        UpdateResponse updateResponse = null;
+        BulkItemRequest replicaRequest = request.items()[requestIndex];
+        int maxAttempts = updateRequest.retryOnConflict();
+        for (int attemptCount = 0; attemptCount <= maxAttempts; attemptCount++) {
+            final UpdateHelper.Result translate;
+            // translate update request
+            try {
+                translate = updateHelper.prepare(updateRequest, primary, threadPool::absoluteTimeInMillis);
+            } catch (Exception failure) {
+                // we may fail translating a update to index or delete operation
+                // we use index result to communicate failure while translating update request
+                updateOperationResult = new Engine.IndexResult(failure, updateRequest.version());
+                break; // out of retry loop
+            }
+            // execute translated update request
+            switch (translate.getResponseResult()) {
+                case CREATED:
+                case UPDATED:
+                    IndexRequest indexRequest = translate.action();
+                    MappingMetaData mappingMd = metaData.mappingOrDefault(indexRequest.type());
+                    indexRequest.process(mappingMd, allowIdGeneration, request.index());
+                    updateOperationResult = executeIndexRequestOnPrimary(indexRequest, primary, mappingUpdatedAction);
+                    if (updateOperationResult.hasFailure() == false) {
+                        // update the version on request so it will happen on the replicas
+                        final long version = updateOperationResult.getVersion();
+                        indexRequest.version(version);
+                        indexRequest.versionType(indexRequest.versionType().versionTypeForReplicationAndRecovery());
+                        assert indexRequest.versionType().validateVersionForWrites(indexRequest.version());
                     }
-                    return new UpdateResult(translate, indexRequest, retry, cause, null);
-                }
-            case DELETED:
-                DeleteRequest deleteRequest = translate.action();
-                try {
-                    WriteResult<DeleteResponse> result = TransportDeleteAction.executeDeleteRequestOnPrimary(deleteRequest, indexShard);
-                    return new UpdateResult(translate, deleteRequest, result);
-                } catch (Exception e) {
-                    final Throwable cause = ExceptionsHelper.unwrapCause(e);
-                    boolean retry = false;
-                    if (cause instanceof VersionConflictEngineException) {
-                        retry = true;
+                    break;
+                case DELETED:
+                    DeleteRequest deleteRequest = translate.action();
+                    updateOperationResult = executeDeleteRequestOnPrimary(deleteRequest, primary, mappingUpdatedAction);
+                    if (updateOperationResult.hasFailure() == false) {
+                        // update the request with the version so it will go to the replicas
+                        deleteRequest.versionType(deleteRequest.versionType().versionTypeForReplicationAndRecovery());
+                        deleteRequest.version(updateOperationResult.getVersion());
+                        assert deleteRequest.versionType().validateVersionForWrites(deleteRequest.version());
                     }
-                    return new UpdateResult(translate, deleteRequest, retry, cause, null);
+                    break;
+                case NOOP:
+                    primary.noopUpdate(updateRequest.type());
+                    break;
+                default: throw new IllegalStateException("Illegal update operation " + translate.getResponseResult());
+            }
+            if (updateOperationResult == null) {
+                // this is a noop operation
+                updateResponse = translate.action();
+                break; // out of retry loop
+            } else if (updateOperationResult.hasFailure() == false) {
+                // enrich update response and
+                // set translated update (index/delete) request for replica execution in bulk items
+                switch (updateOperationResult.getOperationType()) {
+                    case INDEX:
+                        IndexRequest updateIndexRequest = translate.action();
+                        final IndexResponse indexResponse = new IndexResponse(primary.shardId(),
+                                updateIndexRequest.type(), updateIndexRequest.id(),
+                                updateOperationResult.getVersion(), ((Engine.IndexResult) updateOperationResult).isCreated());
+                        BytesReference indexSourceAsBytes = updateIndexRequest.source();
+                        updateResponse = new UpdateResponse(indexResponse.getShardInfo(),
+                                indexResponse.getShardId(), indexResponse.getType(), indexResponse.getId(),
+                                indexResponse.getVersion(), indexResponse.getResult());
+                        if ((updateRequest.fetchSource() != null && updateRequest.fetchSource().fetchSource()) ||
+                                (updateRequest.fields() != null && updateRequest.fields().length > 0)) {
+                            Tuple<XContentType, Map<String, Object>> sourceAndContent =
+                                XContentHelper.convertToMap(indexSourceAsBytes, true, updateIndexRequest.getContentType());
+                            updateResponse.setGetResult(updateHelper.extractGetResult(updateRequest, request.index(),
+                                    indexResponse.getVersion(), sourceAndContent.v2(), sourceAndContent.v1(), indexSourceAsBytes));
+                        }
+                        // set translated request as replica request
+                        replicaRequest = new BulkItemRequest(request.items()[requestIndex].id(), updateIndexRequest);
+                        break;
+                    case DELETE:
+                        DeleteRequest updateDeleteRequest = translate.action();
+                        DeleteResponse deleteResponse = new DeleteResponse(primary.shardId(),
+                                updateDeleteRequest.type(), updateDeleteRequest.id(),
+                                updateOperationResult.getVersion(), ((Engine.DeleteResult) updateOperationResult).isFound());
+                        updateResponse = new UpdateResponse(deleteResponse.getShardInfo(),
+                                deleteResponse.getShardId(), deleteResponse.getType(), deleteResponse.getId(),
+                                deleteResponse.getVersion(), deleteResponse.getResult());
+                        updateResponse.setGetResult(updateHelper.extractGetResult(updateRequest,
+                                request.index(), deleteResponse.getVersion(), translate.updatedSourceAsMap(),
+                                translate.updateSourceContentType(), null));
+                        // set translated request as replica request
+                        replicaRequest = new BulkItemRequest(request.items()[requestIndex].id(), updateDeleteRequest);
+                        break;
+                    default: throw new IllegalStateException("Illegal update operation " +
+                            updateOperationResult.getOperationType().getLowercase());
                 }
-            case NOOP:
-                UpdateResponse updateResponse = translate.action();
-                indexShard.noopUpdate(updateRequest.type());
-                return new UpdateResult(translate, updateResponse);
-            default:
-                throw new IllegalStateException("Illegal update operation " + translate.getResponseResult());
+                // successful operation
+                break; // out of retry loop
+            } else if (updateOperationResult.getFailure() instanceof VersionConflictEngineException == false) {
+                // not a version conflict exception
+                break; // out of retry loop
+            }
         }
+        return new UpdateResultHolder(replicaRequest, updateOperationResult, updateResponse);
     }
 
     @Override
-    protected Location onReplicaShard(BulkShardRequest request, IndexShard indexShard) {
+    public WriteReplicaResult<BulkShardRequest> shardOperationOnReplica(BulkShardRequest request, IndexShard replica) throws Exception {
         Translog.Location location = null;
         for (int i = 0; i < request.items().length; i++) {
             BulkItemRequest item = request.items()[i];
-            if (item == null || item.isIgnoreOnReplica()) {
-                continue;
-            }
-            if (item.request() instanceof IndexRequest) {
-                IndexRequest indexRequest = (IndexRequest) item.request();
+            if (item.isIgnoreOnReplica() == false) {
+                DocWriteRequest docWriteRequest = item.request();
+                // ensure request version is updated for replica operation during request execution in the primary
+                assert docWriteRequest.versionType() == docWriteRequest.versionType().versionTypeForReplicationAndRecovery()
+                        : "unexpected version in replica " + docWriteRequest.version();
+                final Engine.Result operationResult;
                 try {
-                    Engine.Index operation = TransportIndexAction.executeIndexRequestOnReplica(indexRequest, indexShard);
-                    location = locationToSync(location, operation.getTranslogLocation());
-                } catch (Exception e) {
-                    // if its not an ignore replica failure, we need to make sure to bubble up the failure
-                    // so we will fail the shard
-                    if (!ignoreReplicaException(e)) {
-                        throw e;
+                    switch (docWriteRequest.opType()) {
+                        case CREATE:
+                        case INDEX:
+                            operationResult = executeIndexRequestOnReplica(((IndexRequest) docWriteRequest), replica);
+                            break;
+                        case DELETE:
+                            operationResult = executeDeleteRequestOnReplica(((DeleteRequest) docWriteRequest), replica);
+                            break;
+                        default:
+                            throw new IllegalStateException("Unexpected request operation type on replica: "
+                                    + docWriteRequest.opType().getLowercase());
+                    }
+                    if (operationResult.hasFailure()) {
+                        // check if any transient write operation failures should be bubbled up
+                        Exception failure = operationResult.getFailure();
+                        assert failure instanceof VersionConflictEngineException
+                                || failure instanceof MapperParsingException
+                                : "expected version conflict or mapper parsing failures";
+                        if (!TransportActions.isShardNotAvailableException(failure)) {
+                            throw failure;
+                        }
+                    } else if (operationResult.getTranslogLocation() != null) { // out of order ops are not added to the translog
+                        location = locationToSync(location, operationResult.getTranslogLocation());
                     }
-                }
-            } else if (item.request() instanceof DeleteRequest) {
-                DeleteRequest deleteRequest = (DeleteRequest) item.request();
-                try {
-                    Engine.Delete delete = TransportDeleteAction.executeDeleteRequestOnReplica(deleteRequest, indexShard);
-                    indexShard.delete(delete);
-                    location = locationToSync(location, delete.getTranslogLocation());
                 } catch (Exception e) {
                     // if its not an ignore replica failure, we need to make sure to bubble up the failure
                     // so we will fail the shard
-                    if (!ignoreReplicaException(e)) {
+                    if (!TransportActions.isShardNotAvailableException(e)) {
                         throw e;
                     }
                 }
-            } else {
-                throw new IllegalStateException("Unexpected index operation: " + item.request());
             }
         }
-        return location;
-    }
-
-    private void applyVersion(BulkItemRequest item, long version, VersionType versionType) {
-        if (item.request() instanceof IndexRequest) {
-            ((IndexRequest) item.request()).version(version).versionType(versionType);
-        } else if (item.request() instanceof DeleteRequest) {
-            ((DeleteRequest) item.request()).version(version).versionType();
-        } else if (item.request() instanceof UpdateRequest) {
-            ((UpdateRequest) item.request()).version(version).versionType();
-        } else {
-            // log?
-        }
+        return new WriteReplicaResult<>(request, location, null, replica, logger);
     }
 
     private Translog.Location locationToSync(Translog.Location current, Translog.Location next) {
@@ -504,4 +424,124 @@ private Translog.Location locationToSync(Translog.Location current, Translog.Loc
         assert current == null || current.compareTo(next) < 0 : "translog locations are not increasing";
         return next;
     }
+
+    /**
+     * Execute the given {@link IndexRequest} on a replica shard, throwing a
+     * {@link RetryOnReplicaException} if the operation needs to be re-tried.
+     */
+    public static Engine.IndexResult executeIndexRequestOnReplica(IndexRequest request, IndexShard replica) throws IOException {
+        final ShardId shardId = replica.shardId();
+        SourceToParse sourceToParse =
+            SourceToParse.source(SourceToParse.Origin.REPLICA, shardId.getIndexName(), request.type(), request.id(), request.source(),
+                request.getContentType()).routing(request.routing()).parent(request.parent())
+                .timestamp(request.timestamp()).ttl(request.ttl());
+
+        final Engine.Index operation;
+        try {
+            operation = replica.prepareIndexOnReplica(sourceToParse, request.version(), request.versionType(), request.getAutoGeneratedTimestamp(), request.isRetry());
+        } catch (MapperParsingException e) {
+            return new Engine.IndexResult(e, request.version());
+        }
+        Mapping update = operation.parsedDoc().dynamicMappingsUpdate();
+        if (update != null) {
+            throw new RetryOnReplicaException(shardId, "Mappings are not available on the replica yet, triggered update: " + update);
+        }
+        return replica.index(operation);
+    }
+
+    /** Utility method to prepare an index operation on primary shards */
+    static Engine.Index prepareIndexOperationOnPrimary(IndexRequest request, IndexShard primary) {
+        SourceToParse sourceToParse =
+            SourceToParse.source(SourceToParse.Origin.PRIMARY, request.index(), request.type(), request.id(), request.source(),
+                request.getContentType()).routing(request.routing()).parent(request.parent())
+                .timestamp(request.timestamp()).ttl(request.ttl());
+        return primary.prepareIndexOnPrimary(sourceToParse, request.version(), request.versionType(), request.getAutoGeneratedTimestamp(), request.isRetry());
+    }
+
+    /** Executes index operation on primary shard after updates mapping if dynamic mappings are found */
+    public static Engine.IndexResult executeIndexRequestOnPrimary(IndexRequest request, IndexShard primary,
+                                                                  MappingUpdatedAction mappingUpdatedAction) throws Exception {
+        Engine.Index operation;
+        try {
+            operation = prepareIndexOperationOnPrimary(request, primary);
+        } catch (MapperParsingException | IllegalArgumentException e) {
+            return new Engine.IndexResult(e, request.version());
+        }
+        Mapping update = operation.parsedDoc().dynamicMappingsUpdate();
+        final ShardId shardId = primary.shardId();
+        if (update != null) {
+            // can throw timeout exception when updating mappings or ISE for attempting to update default mappings
+            // which are bubbled up
+            try {
+                mappingUpdatedAction.updateMappingOnMaster(shardId.getIndex(), request.type(), update);
+            } catch (IllegalArgumentException e) {
+                // throws IAE on conflicts merging dynamic mappings
+                return new Engine.IndexResult(e, request.version());
+            }
+            try {
+                operation = prepareIndexOperationOnPrimary(request, primary);
+            } catch (MapperParsingException | IllegalArgumentException e) {
+                return new Engine.IndexResult(e, request.version());
+            }
+            update = operation.parsedDoc().dynamicMappingsUpdate();
+            if (update != null) {
+                throw new ReplicationOperation.RetryOnPrimaryException(shardId,
+                        "Dynamic mappings are not available on the node that holds the primary yet");
+            }
+        }
+        return primary.index(operation);
+    }
+
+    public static Engine.DeleteResult executeDeleteRequestOnPrimary(DeleteRequest request, IndexShard primary,
+            MappingUpdatedAction mappingUpdatedAction) throws Exception {
+        boolean mappingUpdateNeeded = false;
+        final ShardId shardId = primary.shardId();
+        if (primary.indexSettings().isSingleType()) {
+            // When there is a single type, the unique identifier is only composed of the _id,
+            // so there is no way to differenciate foo#1 from bar#1. This is especially an issue
+            // if a user first deletes foo#1 and then indexes bar#1: since we do not encode the
+            // _type in the uid it might look like we are reindexing the same document, which
+            // would fail if bar#1 is indexed with a lower version than foo#1 was deleted with.
+            // In order to work around this issue, we make deletions create types. This way, we
+            // fail if index and delete operations do not use the same type.
+            try {
+                Mapping update = primary.mapperService().documentMapperWithAutoCreate(request.type()).getMapping();
+                if (update != null) {
+                    mappingUpdateNeeded = true;
+                    mappingUpdatedAction.updateMappingOnMaster(shardId.getIndex(), request.type(), update);
+                }
+            } catch (MapperParsingException | IllegalArgumentException e) {
+                return new Engine.DeleteResult(e, request.version(), false);
+            }
+        }
+        if (mappingUpdateNeeded) {
+            Mapping update = primary.mapperService().documentMapperWithAutoCreate(request.type()).getMapping();
+            if (update != null) {
+                throw new ReplicationOperation.RetryOnPrimaryException(shardId,
+                        "Dynamic mappings are not available on the node that holds the primary yet");
+            }
+        }
+        final Engine.Delete delete = primary.prepareDeleteOnPrimary(request.type(), request.id(), request.version(), request.versionType());
+        return primary.delete(delete);
+    }
+
+    public static Engine.DeleteResult executeDeleteRequestOnReplica(DeleteRequest request, IndexShard replica) throws IOException {
+        if (replica.indexSettings().isSingleType()) {
+            // We need to wait for the replica to have the mappings
+            Mapping update;
+            try {
+                update = replica.mapperService().documentMapperWithAutoCreate(request.type()).getMapping();
+            } catch (MapperParsingException | IllegalArgumentException e) {
+                return new Engine.DeleteResult(e, request.version(), false);
+            }
+            if (update != null) {
+                final ShardId shardId = replica.shardId();
+                throw new RetryOnReplicaException(shardId,
+                        "Mappings are not available on the replica yet, triggered update: " + update);
+            }
+        }
+        final Engine.Delete delete = replica.prepareDeleteOnReplica(request.type(), request.id(),
+                request.version(), request.versionType());
+        return replica.delete(delete);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/bulk/TransportSingleItemBulkWriteAction.java b/core/src/main/java/org/elasticsearch/action/bulk/TransportSingleItemBulkWriteAction.java
new file mode 100644
index 0000000000000..fd71f504ea9b1
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/bulk/TransportSingleItemBulkWriteAction.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.bulk;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.action.support.WriteResponse;
+import org.elasticsearch.action.support.replication.ReplicatedWriteRequest;
+import org.elasticsearch.action.support.replication.ReplicationResponse;
+import org.elasticsearch.action.support.replication.TransportWriteAction;
+import org.elasticsearch.cluster.action.shard.ShardStateAction;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.shard.IndexShard;
+import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+
+import java.util.function.Supplier;
+
+/** use transport bulk action directly */
+@Deprecated
+public abstract class TransportSingleItemBulkWriteAction<
+    Request extends ReplicatedWriteRequest<Request>,
+    Response extends ReplicationResponse & WriteResponse
+    > extends TransportWriteAction<Request, Request, Response> {
+
+    private final TransportBulkAction bulkAction;
+    private final TransportShardBulkAction shardBulkAction;
+
+
+    protected TransportSingleItemBulkWriteAction(Settings settings, String actionName, TransportService transportService,
+                                                 ClusterService clusterService, IndicesService indicesService, ThreadPool threadPool,
+                                                 ShardStateAction shardStateAction, ActionFilters actionFilters,
+                                                 IndexNameExpressionResolver indexNameExpressionResolver, Supplier<Request> request,
+                                                 Supplier<Request> replicaRequest, String executor,
+                                                 TransportBulkAction bulkAction, TransportShardBulkAction shardBulkAction) {
+        super(settings, actionName, transportService, clusterService, indicesService, threadPool, shardStateAction, actionFilters,
+            indexNameExpressionResolver, request, replicaRequest, executor);
+        this.bulkAction = bulkAction;
+        this.shardBulkAction = shardBulkAction;
+    }
+
+
+    @Override
+    protected void doExecute(Task task, final Request request, final ActionListener<Response> listener) {
+        bulkAction.execute(task, toSingleItemBulkRequest(request), wrapBulkResponse(listener));
+    }
+
+    @Override
+    protected WritePrimaryResult<Request, Response> shardOperationOnPrimary(
+        Request request, final IndexShard primary) throws Exception {
+        BulkItemRequest[] itemRequests = new BulkItemRequest[1];
+        WriteRequest.RefreshPolicy refreshPolicy = request.getRefreshPolicy();
+        request.setRefreshPolicy(WriteRequest.RefreshPolicy.NONE);
+        itemRequests[0] = new BulkItemRequest(0, ((DocWriteRequest) request));
+        BulkShardRequest bulkShardRequest = new BulkShardRequest(request.shardId(), refreshPolicy, itemRequests);
+        WritePrimaryResult<BulkShardRequest, BulkShardResponse> bulkResult =
+            shardBulkAction.shardOperationOnPrimary(bulkShardRequest, primary);
+        assert bulkResult.finalResponseIfSuccessful.getResponses().length == 1 : "expected only one bulk shard response";
+        BulkItemResponse itemResponse = bulkResult.finalResponseIfSuccessful.getResponses()[0];
+        final Response response;
+        final Exception failure;
+        if (itemResponse.isFailed()) {
+            failure = itemResponse.getFailure().getCause();
+            response = null;
+        } else {
+            response = (Response) itemResponse.getResponse();
+            failure = null;
+        }
+        return new WritePrimaryResult<>(request, response, bulkResult.location, failure, primary, logger);
+    }
+
+    @Override
+    protected WriteReplicaResult<Request> shardOperationOnReplica(
+        Request replicaRequest, IndexShard replica) throws Exception {
+        BulkItemRequest[] itemRequests = new BulkItemRequest[1];
+        WriteRequest.RefreshPolicy refreshPolicy = replicaRequest.getRefreshPolicy();
+        itemRequests[0] = new BulkItemRequest(0, ((DocWriteRequest) replicaRequest));
+        BulkShardRequest bulkShardRequest = new BulkShardRequest(replicaRequest.shardId(), refreshPolicy, itemRequests);
+        WriteReplicaResult<BulkShardRequest> result = shardBulkAction.shardOperationOnReplica(bulkShardRequest, replica);
+        // a replica operation can never throw a document-level failure,
+        // as the same document has been already indexed successfully in the primary
+        return new WriteReplicaResult<>(replicaRequest, result.location, null, replica, logger);
+    }
+
+
+    private ActionListener<BulkResponse> wrapBulkResponse(ActionListener<Response> listener) {
+        return ActionListener.wrap(bulkItemResponses -> {
+            assert bulkItemResponses.getItems().length == 1 : "expected only one item in bulk request";
+            BulkItemResponse bulkItemResponse = bulkItemResponses.getItems()[0];
+            if (bulkItemResponse.isFailed() == false) {
+                final DocWriteResponse response = bulkItemResponse.getResponse();
+                listener.onResponse((Response) response);
+            } else {
+                listener.onFailure(bulkItemResponse.getFailure().getCause());
+            }
+        }, listener::onFailure);
+    }
+
+    public static BulkRequest toSingleItemBulkRequest(ReplicatedWriteRequest request) {
+        BulkRequest bulkRequest = new BulkRequest();
+        bulkRequest.add(((DocWriteRequest) request));
+        bulkRequest.setRefreshPolicy(request.getRefreshPolicy());
+        bulkRequest.timeout(request.timeout());
+        bulkRequest.waitForActiveShards(request.waitForActiveShards());
+        request.setRefreshPolicy(WriteRequest.RefreshPolicy.NONE);
+        return bulkRequest;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java b/core/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java
index bdf09e3e532fd..5d44fc87e43ec 100644
--- a/core/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java
@@ -20,11 +20,14 @@
 package org.elasticsearch.action.delete;
 
 import org.elasticsearch.action.ActionRequestValidationException;
-import org.elasticsearch.action.DocumentRequest;
+import org.elasticsearch.action.CompositeIndicesRequest;
+import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.support.replication.ReplicatedWriteRequest;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.index.VersionType;
 
@@ -43,7 +46,7 @@
  * @see org.elasticsearch.client.Client#delete(DeleteRequest)
  * @see org.elasticsearch.client.Requests#deleteRequest(String)
  */
-public class DeleteRequest extends ReplicatedWriteRequest<DeleteRequest> implements DocumentRequest<DeleteRequest> {
+public class DeleteRequest extends ReplicatedWriteRequest<DeleteRequest> implements DocWriteRequest<DeleteRequest>, CompositeIndicesRequest {
 
     private String type;
     private String id;
@@ -53,6 +56,7 @@ public class DeleteRequest extends ReplicatedWriteRequest<DeleteRequest> impleme
     private String parent;
     private long version = Versions.MATCH_ANY;
     private VersionType versionType = VersionType.INTERNAL;
+    private static DeprecationLogger deprecationLogger = new DeprecationLogger(Loggers.getLogger(DeleteRequest.class));
 
     public DeleteRequest() {
     }
@@ -90,6 +94,9 @@ public ActionRequestValidationException validate() {
         if (!versionType.validateVersionForWrites(version)) {
             validationException = addValidationError("illegal version value [" + version + "] for version type [" + versionType.name() + "]", validationException);
         }
+        if (versionType == VersionType.FORCE) {
+            deprecationLogger.deprecated("version type FORCE is deprecated and will be removed in the next major version");
+        }
         return validationException;
     }
 
@@ -164,28 +171,33 @@ public String routing() {
         return this.routing;
     }
 
-    /**
-     * Sets the version, which will cause the delete operation to only be performed if a matching
-     * version exists and no changes happened on the doc since then.
-     */
+    @Override
     public DeleteRequest version(long version) {
         this.version = version;
         return this;
     }
 
+    @Override
     public long version() {
         return this.version;
     }
 
+    @Override
     public DeleteRequest versionType(VersionType versionType) {
         this.versionType = versionType;
         return this;
     }
 
+    @Override
     public VersionType versionType() {
         return this.versionType;
     }
 
+    @Override
+    public OpType opType() {
+        return OpType.DELETE;
+    }
+
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
diff --git a/core/src/main/java/org/elasticsearch/action/delete/DeleteResponse.java b/core/src/main/java/org/elasticsearch/action/delete/DeleteResponse.java
index 4742c573e686c..bf43978582ad4 100644
--- a/core/src/main/java/org/elasticsearch/action/delete/DeleteResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/delete/DeleteResponse.java
@@ -21,11 +21,14 @@
 
 import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
 
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+
 /**
  * The response of the delete action.
  *
@@ -34,8 +37,9 @@
  */
 public class DeleteResponse extends DocWriteResponse {
 
-    public DeleteResponse() {
+    private static final String FOUND = "found";
 
+    public DeleteResponse() {
     }
 
     public DeleteResponse(ShardId shardId, String type, String id, long version, boolean found) {
@@ -47,13 +51,6 @@ public RestStatus status() {
         return result == Result.DELETED ? super.status() : RestStatus.NOT_FOUND;
     }
 
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.field("found", result == Result.DELETED);
-        super.toXContent(builder, params);
-        return builder;
-    }
-
     @Override
     public String toString() {
         StringBuilder builder = new StringBuilder();
@@ -66,4 +63,61 @@ public String toString() {
         builder.append(",shards=").append(getShardInfo());
         return builder.append("]").toString();
     }
+
+    @Override
+    public XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.field(FOUND, result == Result.DELETED);
+        super.innerToXContent(builder, params);
+        return builder;
+    }
+
+    public static DeleteResponse fromXContent(XContentParser parser) throws IOException {
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
+
+        Builder context = new Builder();
+        while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
+            parseXContentFields(parser, context);
+        }
+        return context.build();
+    }
+
+    /**
+     * Parse the current token and update the parsing context appropriately.
+     */
+    public static void parseXContentFields(XContentParser parser, Builder context) throws IOException {
+        XContentParser.Token token = parser.currentToken();
+        String currentFieldName = parser.currentName();
+
+        if (FOUND.equals(currentFieldName)) {
+            if (token.isValue()) {
+                context.setFound(parser.booleanValue());
+            }
+        } else {
+            DocWriteResponse.parseInnerToXContent(parser, context);
+        }
+    }
+
+    /**
+     * Builder class for {@link DeleteResponse}. This builder is usually used during xcontent parsing to
+     * temporarily store the parsed values, then the {@link DocWriteResponse.Builder#build()} method is called to
+     * instantiate the {@link DeleteResponse}.
+     */
+    public static class Builder extends DocWriteResponse.Builder {
+
+        private boolean found = false;
+
+        public void setFound(boolean found) {
+            this.found = found;
+        }
+
+        @Override
+        public DeleteResponse build() {
+            DeleteResponse deleteResponse = new DeleteResponse(shardId, type, id, version, found);
+            deleteResponse.setForcedRefresh(forcedRefresh);
+            if (shardInfo != null) {
+                deleteResponse.setShardInfo(shardInfo);
+            }
+            return deleteResponse;
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java b/core/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java
index 6f3d27ea36908..3aaf4a472facf 100644
--- a/core/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java
+++ b/core/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java
@@ -19,130 +19,39 @@
 
 package org.elasticsearch.action.delete;
 
-import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.RoutingMissingException;
-import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
-import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
-import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction;
+import org.elasticsearch.action.bulk.TransportBulkAction;
+import org.elasticsearch.action.bulk.TransportShardBulkAction;
+import org.elasticsearch.action.bulk.TransportSingleItemBulkWriteAction;
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.AutoCreateIndex;
-import org.elasticsearch.action.support.replication.TransportWriteAction;
-import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.engine.Engine;
-import org.elasticsearch.index.shard.IndexShard;
-import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.index.translog.Translog.Location;
-import org.elasticsearch.indices.IndexAlreadyExistsException;
 import org.elasticsearch.indices.IndicesService;
-import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
 /**
  * Performs the delete operation.
+ *
+ * Deprecated use TransportBulkAction with a single item instead
  */
-public class TransportDeleteAction extends TransportWriteAction<DeleteRequest, DeleteResponse> {
-
-    private final AutoCreateIndex autoCreateIndex;
-    private final TransportCreateIndexAction createIndexAction;
+@Deprecated
+public class TransportDeleteAction extends TransportSingleItemBulkWriteAction<DeleteRequest, DeleteResponse> {
 
     @Inject
     public TransportDeleteAction(Settings settings, TransportService transportService, ClusterService clusterService,
                                  IndicesService indicesService, ThreadPool threadPool, ShardStateAction shardStateAction,
-                                 TransportCreateIndexAction createIndexAction, ActionFilters actionFilters,
-                                 IndexNameExpressionResolver indexNameExpressionResolver,
-                                 AutoCreateIndex autoCreateIndex) {
-        super(settings, DeleteAction.NAME, transportService, clusterService, indicesService, threadPool, shardStateAction, actionFilters,
-                indexNameExpressionResolver, DeleteRequest::new, ThreadPool.Names.INDEX);
-        this.createIndexAction = createIndexAction;
-        this.autoCreateIndex = autoCreateIndex;
-    }
-
-    @Override
-    protected void doExecute(Task task, final DeleteRequest request, final ActionListener<DeleteResponse> listener) {
-        ClusterState state = clusterService.state();
-        if (autoCreateIndex.shouldAutoCreate(request.index(), state)) {
-            createIndexAction.execute(task, new CreateIndexRequest().index(request.index()).cause("auto(delete api)").masterNodeTimeout(request.timeout()), new ActionListener<CreateIndexResponse>() {
-                @Override
-                public void onResponse(CreateIndexResponse result) {
-                    innerExecute(task, request, listener);
-                }
-
-                @Override
-                public void onFailure(Exception e) {
-                    if (ExceptionsHelper.unwrapCause(e) instanceof IndexAlreadyExistsException) {
-                        // we have the index, do it
-                        innerExecute(task, request, listener);
-                    } else {
-                        listener.onFailure(e);
-                    }
-                }
-            });
-        } else {
-            innerExecute(task, request, listener);
-        }
-    }
-
-    @Override
-    protected void resolveRequest(final MetaData metaData, IndexMetaData indexMetaData, DeleteRequest request) {
-        super.resolveRequest(metaData, indexMetaData, request);
-        resolveAndValidateRouting(metaData, indexMetaData.getIndex().getName(), request);
-        ShardId shardId = clusterService.operationRouting().shardId(clusterService.state(),
-            indexMetaData.getIndex().getName(), request.id(), request.routing());
-        request.setShardId(shardId);
-    }
-
-    public static void resolveAndValidateRouting(final MetaData metaData, final String concreteIndex,
-                                                 DeleteRequest request) {
-        request.routing(metaData.resolveIndexRouting(request.parent(), request.routing(), request.index()));
-        // check if routing is required, if so, throw error if routing wasn't specified
-        if (request.routing() == null && metaData.routingRequired(concreteIndex, request.type())) {
-            throw new RoutingMissingException(concreteIndex, request.type(), request.id());
-        }
-    }
-
-    private void innerExecute(Task task, final DeleteRequest request, final ActionListener<DeleteResponse> listener) {
-        super.doExecute(task, request, listener);
+                                 ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
+                                 TransportBulkAction bulkAction, TransportShardBulkAction shardBulkAction) {
+        super(settings, DeleteAction.NAME, transportService, clusterService, indicesService, threadPool, shardStateAction,
+            actionFilters, indexNameExpressionResolver, DeleteRequest::new, DeleteRequest::new, ThreadPool.Names.INDEX,
+            bulkAction, shardBulkAction);
     }
 
     @Override
     protected DeleteResponse newResponseInstance() {
         return new DeleteResponse();
     }
-
-    @Override
-    protected WriteResult<DeleteResponse> onPrimaryShard(DeleteRequest request, IndexShard indexShard) {
-        return executeDeleteRequestOnPrimary(request, indexShard);
-    }
-
-    @Override
-    protected Location onReplicaShard(DeleteRequest request, IndexShard indexShard) {
-        return executeDeleteRequestOnReplica(request, indexShard).getTranslogLocation();
-    }
-
-    public static WriteResult<DeleteResponse> executeDeleteRequestOnPrimary(DeleteRequest request, IndexShard indexShard) {
-        Engine.Delete delete = indexShard.prepareDeleteOnPrimary(request.type(), request.id(), request.version(), request.versionType());
-        indexShard.delete(delete);
-        // update the request with the version so it will go to the replicas
-        request.versionType(delete.versionType().versionTypeForReplicationAndRecovery());
-        request.version(delete.version());
-
-        assert request.versionType().validateVersionForWrites(request.version());
-        DeleteResponse response = new DeleteResponse(indexShard.shardId(), request.type(), request.id(), delete.version(), delete.found());
-        return new WriteResult<>(response, delete.getTranslogLocation());
-    }
-
-    public static Engine.Delete executeDeleteRequestOnReplica(DeleteRequest request, IndexShard indexShard) {
-        Engine.Delete delete = indexShard.prepareDeleteOnReplica(request.type(), request.id(), request.version(), request.versionType());
-        indexShard.delete(delete);
-        return delete;
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/explain/ExplainRequest.java b/core/src/main/java/org/elasticsearch/action/explain/ExplainRequest.java
index 851d9e6573d0d..5d8ca27657f92 100644
--- a/core/src/main/java/org/elasticsearch/action/explain/ExplainRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/explain/ExplainRequest.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
+import org.elasticsearch.search.internal.AliasFilter;
 
 import java.io.IOException;
 
@@ -40,10 +41,10 @@ public class ExplainRequest extends SingleShardRequest<ExplainRequest> {
     private String routing;
     private String preference;
     private QueryBuilder query;
-    private String[] fields;
+    private String[] storedFields;
     private FetchSourceContext fetchSourceContext;
 
-    private String[] filteringAlias = Strings.EMPTY_ARRAY;
+    private AliasFilter filteringAlias = new AliasFilter(null, Strings.EMPTY_ARRAY);
 
     long nowInMillis;
 
@@ -122,20 +123,20 @@ public FetchSourceContext fetchSourceContext() {
     }
 
 
-    public String[] fields() {
-        return fields;
+    public String[] storedFields() {
+        return storedFields;
     }
 
-    public ExplainRequest fields(String[] fields) {
-        this.fields = fields;
+    public ExplainRequest storedFields(String[] fields) {
+        this.storedFields = fields;
         return this;
     }
 
-    public String[] filteringAlias() {
+    public AliasFilter filteringAlias() {
         return filteringAlias;
     }
 
-    public ExplainRequest filteringAlias(String[] filteringAlias) {
+    public ExplainRequest filteringAlias(AliasFilter filteringAlias) {
         if (filteringAlias != null) {
             this.filteringAlias = filteringAlias;
         }
@@ -166,9 +167,9 @@ public void readFrom(StreamInput in) throws IOException {
         routing = in.readOptionalString();
         preference = in.readOptionalString();
         query = in.readNamedWriteable(QueryBuilder.class);
-        filteringAlias = in.readStringArray();
-        fields = in.readOptionalStringArray();
-        fetchSourceContext = in.readOptionalStreamable(FetchSourceContext::new);
+        filteringAlias = new AliasFilter(in);
+        storedFields = in.readOptionalStringArray();
+        fetchSourceContext = in.readOptionalWriteable(FetchSourceContext::new);
         nowInMillis = in.readVLong();
     }
 
@@ -180,9 +181,9 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalString(routing);
         out.writeOptionalString(preference);
         out.writeNamedWriteable(query);
-        out.writeStringArray(filteringAlias);
-        out.writeOptionalStringArray(fields);
-        out.writeOptionalStreamable(fetchSourceContext);
+        filteringAlias.writeTo(out);
+        out.writeOptionalStringArray(storedFields);
+        out.writeOptionalWriteable(fetchSourceContext);
         out.writeVLong(nowInMillis);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/explain/ExplainRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/explain/ExplainRequestBuilder.java
index c201315cbd832..d2d9bb3b820a8 100644
--- a/core/src/main/java/org/elasticsearch/action/explain/ExplainRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/explain/ExplainRequestBuilder.java
@@ -88,10 +88,10 @@ public ExplainRequestBuilder setQuery(QueryBuilder query) {
     }
 
     /**
-     * Explicitly specify the fields that will be returned for the explained document. By default, nothing is returned.
+     * Explicitly specify the stored fields that will be returned for the explained document. By default, nothing is returned.
      */
-    public ExplainRequestBuilder setFields(String... fields) {
-        request.fields(fields);
+    public ExplainRequestBuilder setStoredFields(String... fields) {
+        request.storedFields(fields);
         return this;
     }
 
@@ -99,12 +99,9 @@ public ExplainRequestBuilder setFields(String... fields) {
      * Indicates whether the response should contain the stored _source
      */
     public ExplainRequestBuilder setFetchSource(boolean fetch) {
-        FetchSourceContext context = request.fetchSourceContext();
-        if (context == null) {
-            request.fetchSourceContext(new FetchSourceContext(fetch));
-        } else {
-            context.fetchSource(fetch);
-        }
+        FetchSourceContext fetchSourceContext = request.fetchSourceContext() != null ? request.fetchSourceContext()
+            : FetchSourceContext.FETCH_SOURCE;
+        request.fetchSourceContext(new FetchSourceContext(fetch, fetchSourceContext.includes(), fetchSourceContext.excludes()));
         return this;
     }
 
@@ -129,14 +126,9 @@ public ExplainRequestBuilder setFetchSource(@Nullable String include, @Nullable
      * @param excludes An optional list of exclude (optionally wildcarded) pattern to filter the returned _source
      */
     public ExplainRequestBuilder setFetchSource(@Nullable String[] includes, @Nullable String[] excludes) {
-        FetchSourceContext context = request.fetchSourceContext();
-        if (context == null) {
-            request.fetchSourceContext(new FetchSourceContext(includes, excludes));
-        } else {
-            context.fetchSource(true);
-            context.includes(includes);
-            context.excludes(excludes);
-        }
+        FetchSourceContext fetchSourceContext = request.fetchSourceContext() != null ? request.fetchSourceContext()
+            : FetchSourceContext.FETCH_SOURCE;
+        request.fetchSourceContext(new FetchSourceContext(fetchSourceContext.fetchSource(), includes, excludes));
         return this;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java b/core/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java
index 95177853d4198..72aaeb9eb371a 100644
--- a/core/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java
+++ b/core/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java
@@ -31,20 +31,13 @@
 import org.elasticsearch.cluster.routing.ShardIterator;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.BigArrays;
-import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.get.GetResult;
-import org.elasticsearch.index.mapper.Uid;
-import org.elasticsearch.index.mapper.UidFieldMapper;
-import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.indices.IndicesService;
-import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.SearchService;
-import org.elasticsearch.search.fetch.FetchPhase;
-import org.elasticsearch.search.internal.DefaultSearchContext;
+import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.internal.ShardSearchLocalRequest;
 import org.elasticsearch.search.rescore.RescoreSearchContext;
@@ -60,26 +53,15 @@
 // TODO: AggregatedDfs. Currently the idf can be different then when executing a normal search with explain.
 public class TransportExplainAction extends TransportSingleShardAction<ExplainRequest, ExplainResponse> {
 
-    private final IndicesService indicesService;
-
-    private final ScriptService scriptService;
-
-
-    private final BigArrays bigArrays;
-
-    private final FetchPhase fetchPhase;
+    private final SearchService searchService;
 
     @Inject
     public TransportExplainAction(Settings settings, ThreadPool threadPool, ClusterService clusterService,
-            TransportService transportService, IndicesService indicesService, ScriptService scriptService,
-            BigArrays bigArrays, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
-            FetchPhase fetchPhase) {
+                                  TransportService transportService, SearchService searchService, ActionFilters actionFilters,
+                                  IndexNameExpressionResolver indexNameExpressionResolver) {
         super(settings, ExplainAction.NAME, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver,
                 ExplainRequest::new, ThreadPool.Names.GET);
-        this.indicesService = indicesService;
-        this.scriptService = scriptService;
-        this.bigArrays = bigArrays;
-        this.fetchPhase = fetchPhase;
+        this.searchService = searchService;
     }
 
     @Override
@@ -95,7 +77,9 @@ protected boolean resolveIndex(ExplainRequest request) {
 
     @Override
     protected void resolveRequest(ClusterState state, InternalRequest request) {
-        request.request().filteringAlias(indexNameExpressionResolver.filteringAliases(state, request.concreteIndex(), request.request().index()));
+        final AliasFilter aliasFilter = searchService.buildAliasFilter(state, request.concreteIndex(),
+            request.request().index());
+        request.request().filteringAlias(aliasFilter);
         // Fail fast on the node that received the request.
         if (request.request().routing() == null && state.getMetaData().routingRequired(request.concreteIndex(), request.request().type())) {
             throw new RoutingMissingException(request.concreteIndex(), request.request().type(), request.request().id());
@@ -103,35 +87,33 @@ protected void resolveRequest(ClusterState state, InternalRequest request) {
     }
 
     @Override
-    protected ExplainResponse shardOperation(ExplainRequest request, ShardId shardId) {
-        IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
-        IndexShard indexShard = indexService.getShard(shardId.id());
-        Term uidTerm = new Term(UidFieldMapper.NAME, Uid.createUidAsBytes(request.type(), request.id()));
-        Engine.GetResult result = indexShard.get(new Engine.Get(false, uidTerm));
-        if (!result.exists()) {
-            return new ExplainResponse(shardId.getIndexName(), request.type(), request.id(), false);
-        }
-
-        SearchContext context = new DefaultSearchContext(0,
-                new ShardSearchLocalRequest(new String[] { request.type() }, request.nowInMillis, request.filteringAlias()), null,
-                result.searcher(), indexService, indexShard, scriptService, bigArrays,
-                threadPool.estimatedTimeInMillisCounter(), parseFieldMatcher, SearchService.NO_TIMEOUT, fetchPhase);
-        SearchContext.setCurrent(context);
-
+    protected ExplainResponse shardOperation(ExplainRequest request, ShardId shardId) throws IOException {
+        ShardSearchLocalRequest shardSearchLocalRequest = new ShardSearchLocalRequest(shardId,
+            new String[]{request.type()}, request.nowInMillis, request.filteringAlias());
+        SearchContext context = searchService.createSearchContext(shardSearchLocalRequest, SearchService.NO_TIMEOUT, null);
+        Engine.GetResult result = null;
         try {
+            Term uidTerm = context.mapperService().createUidTerm(request.type(), request.id());
+            if (uidTerm == null) {
+                return new ExplainResponse(shardId.getIndexName(), request.type(), request.id(), false);
+            }
+            result = context.indexShard().get(new Engine.Get(false, request.type(), request.id(), uidTerm));
+            if (!result.exists()) {
+                return new ExplainResponse(shardId.getIndexName(), request.type(), request.id(), false);
+            }
             context.parsedQuery(context.getQueryShardContext().toQuery(request.query()));
-            context.preProcess();
+            context.preProcess(true);
             int topLevelDocId = result.docIdAndVersion().docId + result.docIdAndVersion().context.docBase;
             Explanation explanation = context.searcher().explain(context.query(), topLevelDocId);
             for (RescoreSearchContext ctx : context.rescore()) {
                 Rescorer rescorer = ctx.rescorer();
                 explanation = rescorer.explain(topLevelDocId, context, ctx, explanation);
             }
-            if (request.fields() != null || (request.fetchSourceContext() != null && request.fetchSourceContext().fetchSource())) {
+            if (request.storedFields() != null || (request.fetchSourceContext() != null && request.fetchSourceContext().fetchSource())) {
                 // Advantage is that we're not opening a second searcher to retrieve the _source. Also
                 // because we are working in the same searcher in engineGetResult we can be sure that a
                 // doc isn't deleted between the initial get and this call.
-                GetResult getResult = indexShard.getService().get(result, request.id(), request.type(), request.fields(), request.fetchSourceContext());
+                GetResult getResult = context.indexShard().getService().get(result, request.id(), request.type(), request.storedFields(), request.fetchSourceContext());
                 return new ExplainResponse(shardId.getIndexName(), request.type(), request.id(), true, explanation, getResult);
             } else {
                 return new ExplainResponse(shardId.getIndexName(), request.type(), request.id(), true, explanation);
@@ -139,8 +121,7 @@ protected ExplainResponse shardOperation(ExplainRequest request, ShardId shardId
         } catch (IOException e) {
             throw new ElasticsearchException("Could not explain", e);
         } finally {
-            context.close();
-            SearchContext.removeCurrent();
+            Releasables.close(result, context);
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java b/core/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java
new file mode 100644
index 0000000000000..ef7513f38abc2
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java
@@ -0,0 +1,282 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.fieldcaps;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Arrays;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Comparator;
+
+/**
+ * Describes the capabilities of a field optionally merged across multiple indices.
+ */
+public class FieldCapabilities implements Writeable, ToXContent {
+    private final String name;
+    private final String type;
+    private final boolean isSearchable;
+    private final boolean isAggregatable;
+
+    private final String[] indices;
+    private final String[] nonSearchableIndices;
+    private final String[] nonAggregatableIndices;
+
+    /**
+     * Constructor
+     * @param name The name of the field.
+     * @param type The type associated with the field.
+     * @param isSearchable Whether this field is indexed for search.
+     * @param isAggregatable Whether this field can be aggregated on.
+     */
+    FieldCapabilities(String name, String type, boolean isSearchable, boolean isAggregatable) {
+        this(name, type, isSearchable, isAggregatable, null, null, null);
+    }
+
+    /**
+     * Constructor
+     * @param name The name of the field
+     * @param type The type associated with the field.
+     * @param isSearchable Whether this field is indexed for search.
+     * @param isAggregatable Whether this field can be aggregated on.
+     * @param indices The list of indices where this field name is defined as {@code type},
+     *                or null if all indices have the same {@code type} for the field.
+     * @param nonSearchableIndices The list of indices where this field is not searchable,
+     *                             or null if the field is searchable in all indices.
+     * @param nonAggregatableIndices The list of indices where this field is not aggregatable,
+     *                               or null if the field is aggregatable in all indices.
+     */
+    FieldCapabilities(String name, String type,
+                      boolean isSearchable, boolean isAggregatable,
+                      String[] indices,
+                      String[] nonSearchableIndices,
+                      String[] nonAggregatableIndices) {
+        this.name = name;
+        this.type = type;
+        this.isSearchable = isSearchable;
+        this.isAggregatable = isAggregatable;
+        this.indices = indices;
+        this.nonSearchableIndices = nonSearchableIndices;
+        this.nonAggregatableIndices = nonAggregatableIndices;
+    }
+
+    FieldCapabilities(StreamInput in) throws IOException {
+        this.name = in.readString();
+        this.type = in.readString();
+        this.isSearchable = in.readBoolean();
+        this.isAggregatable = in.readBoolean();
+        this.indices = in.readOptionalStringArray();
+        this.nonSearchableIndices = in.readOptionalStringArray();
+        this.nonAggregatableIndices = in.readOptionalStringArray();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(name);
+        out.writeString(type);
+        out.writeBoolean(isSearchable);
+        out.writeBoolean(isAggregatable);
+        out.writeOptionalStringArray(indices);
+        out.writeOptionalStringArray(nonSearchableIndices);
+        out.writeOptionalStringArray(nonAggregatableIndices);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field("type", type);
+        builder.field("searchable", isSearchable);
+        builder.field("aggregatable", isAggregatable);
+        if (indices != null) {
+            builder.field("indices", indices);
+        }
+        if (nonSearchableIndices != null) {
+            builder.field("non_searchable_indices", nonSearchableIndices);
+        }
+        if (nonAggregatableIndices != null) {
+            builder.field("non_aggregatable_indices", nonAggregatableIndices);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    /**
+     * The name of the field.
+     */
+    public String getName() {
+        return name;
+    }
+
+    /**
+     * Whether this field is indexed for search on all indices.
+     */
+    public boolean isAggregatable() {
+        return isAggregatable;
+    }
+
+    /**
+     * Whether this field can be aggregated on all indices.
+     */
+    public boolean isSearchable() {
+        return isSearchable;
+    }
+
+    /**
+     * The type of the field.
+     */
+    public String getType() {
+        return type;
+    }
+
+    /**
+     * The list of indices where this field name is defined as {@code type},
+     * or null if all indices have the same {@code type} for the field.
+     */
+    public String[] indices() {
+        return indices;
+    }
+
+    /**
+     * The list of indices where this field is not searchable,
+     * or null if the field is searchable in all indices.
+     */
+    public String[] nonSearchableIndices() {
+        return nonSearchableIndices;
+    }
+
+    /**
+     * The list of indices where this field is not aggregatable,
+     * or null if the field is aggregatable in all indices.
+     */
+    public String[] nonAggregatableIndices() {
+        return nonAggregatableIndices;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        FieldCapabilities that = (FieldCapabilities) o;
+
+        if (isSearchable != that.isSearchable) return false;
+        if (isAggregatable != that.isAggregatable) return false;
+        if (!name.equals(that.name)) return false;
+        if (!type.equals(that.type)) return false;
+        if (!Arrays.equals(indices, that.indices)) return false;
+        if (!Arrays.equals(nonSearchableIndices, that.nonSearchableIndices)) return false;
+        return Arrays.equals(nonAggregatableIndices, that.nonAggregatableIndices);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = name.hashCode();
+        result = 31 * result + type.hashCode();
+        result = 31 * result + (isSearchable ? 1 : 0);
+        result = 31 * result + (isAggregatable ? 1 : 0);
+        result = 31 * result + Arrays.hashCode(indices);
+        result = 31 * result + Arrays.hashCode(nonSearchableIndices);
+        result = 31 * result + Arrays.hashCode(nonAggregatableIndices);
+        return result;
+    }
+
+    static class Builder {
+        private String name;
+        private String type;
+        private boolean isSearchable;
+        private boolean isAggregatable;
+        private List<IndexCaps> indiceList;
+
+        Builder(String name, String type) {
+            this.name = name;
+            this.type = type;
+            this.isSearchable = true;
+            this.isAggregatable = true;
+            this.indiceList = new ArrayList<>();
+        }
+
+        void add(String index, boolean search, boolean agg) {
+            IndexCaps indexCaps = new IndexCaps(index, search, agg);
+            indiceList.add(indexCaps);
+            this.isSearchable &= search;
+            this.isAggregatable &= agg;
+        }
+
+        FieldCapabilities build(boolean withIndices) {
+            final String[] indices;
+            /* Eclipse can't deal with o -> o.name, maybe because of
+             * https://bugs.eclipse.org/bugs/show_bug.cgi?id=511750 */
+            Collections.sort(indiceList, Comparator.comparing((IndexCaps o) -> o.name));
+            if (withIndices) {
+                indices = indiceList.stream()
+                    .map(caps -> caps.name)
+                    .toArray(String[]::new);
+            } else {
+                indices = null;
+            }
+
+            final String[] nonSearchableIndices;
+            if (isSearchable == false &&
+                indiceList.stream().anyMatch((caps) -> caps.isSearchable)) {
+                // Iff this field is searchable in some indices AND non-searchable in others
+                // we record the list of non-searchable indices
+                nonSearchableIndices = indiceList.stream()
+                    .filter((caps) -> caps.isSearchable == false)
+                    .map(caps -> caps.name)
+                    .toArray(String[]::new);
+            } else {
+                nonSearchableIndices = null;
+            }
+
+            final String[] nonAggregatableIndices;
+            if (isAggregatable == false &&
+                indiceList.stream().anyMatch((caps) -> caps.isAggregatable)) {
+                // Iff this field is aggregatable in some indices AND non-searchable in others
+                // we keep the list of non-aggregatable indices
+                nonAggregatableIndices = indiceList.stream()
+                    .filter((caps) -> caps.isAggregatable == false)
+                    .map(caps -> caps.name)
+                    .toArray(String[]::new);
+            } else {
+                nonAggregatableIndices = null;
+            }
+            return new FieldCapabilities(name, type, isSearchable, isAggregatable,
+                indices, nonSearchableIndices, nonAggregatableIndices);
+        }
+    }
+
+    private static class IndexCaps {
+        final String name;
+        final boolean isSearchable;
+        final boolean isAggregatable;
+
+        IndexCaps(String name, boolean isSearchable, boolean isAggregatable) {
+            this.name = name;
+            this.isSearchable = isSearchable;
+            this.isAggregatable = isAggregatable;
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesAction.java b/core/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesAction.java
new file mode 100644
index 0000000000000..93d67f3fc3cc4
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesAction.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.fieldcaps;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.client.ElasticsearchClient;
+
+public class FieldCapabilitiesAction extends Action<FieldCapabilitiesRequest,
+    FieldCapabilitiesResponse, FieldCapabilitiesRequestBuilder> {
+
+    public static final FieldCapabilitiesAction INSTANCE = new FieldCapabilitiesAction();
+    public static final String NAME = "indices:data/read/field_caps";
+
+    private FieldCapabilitiesAction() {
+        super(NAME);
+    }
+
+    @Override
+    public FieldCapabilitiesResponse newResponse() {
+        return new FieldCapabilitiesResponse();
+    }
+
+    @Override
+    public FieldCapabilitiesRequestBuilder newRequestBuilder(ElasticsearchClient client) {
+        return new FieldCapabilitiesRequestBuilder(client, this);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexRequest.java b/core/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexRequest.java
new file mode 100644
index 0000000000000..460a21ae866aa
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexRequest.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.fieldcaps;
+
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.support.single.shard.SingleShardRequest;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+
+public class FieldCapabilitiesIndexRequest
+    extends SingleShardRequest<FieldCapabilitiesIndexRequest> {
+
+    private String[] fields;
+
+    // For serialization
+    FieldCapabilitiesIndexRequest() {}
+
+    FieldCapabilitiesIndexRequest(String[] fields, String index) {
+        super(index);
+        if (fields == null || fields.length == 0) {
+            throw new IllegalArgumentException("specified fields can't be null or empty");
+        }
+        this.fields = fields;
+    }
+
+    public String[] fields() {
+        return fields;
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        fields = in.readStringArray();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeStringArray(fields);
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        return null;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexResponse.java b/core/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexResponse.java
new file mode 100644
index 0000000000000..1e4686245165b
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexResponse.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.fieldcaps;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Response for {@link FieldCapabilitiesIndexRequest} requests.
+ */
+public class FieldCapabilitiesIndexResponse extends ActionResponse implements Writeable {
+    private String indexName;
+    private Map<String, FieldCapabilities> responseMap;
+
+    FieldCapabilitiesIndexResponse(String indexName, Map<String, FieldCapabilities> responseMap) {
+        this.indexName = indexName;
+        this.responseMap = responseMap;
+    }
+
+    FieldCapabilitiesIndexResponse() {
+    }
+
+    FieldCapabilitiesIndexResponse(StreamInput input) throws IOException {
+        this.readFrom(input);
+    }
+
+
+    /**
+     * Get the index name
+     */
+    public String getIndexName() {
+        return indexName;
+    }
+
+    /**
+     * Get the field capabilities map
+     */
+    public Map<String, FieldCapabilities> get() {
+        return responseMap;
+    }
+
+    /**
+     *
+     * Get the field capabilities for the provided {@code field}
+     */
+    public FieldCapabilities getField(String field) {
+        return responseMap.get(field);
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        this.indexName = in.readString();
+        this.responseMap =
+            in.readMap(StreamInput::readString, FieldCapabilities::new);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeString(indexName);
+        out.writeMap(responseMap,
+            StreamOutput::writeString, (valueOut, fc) -> fc.writeTo(valueOut));
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        FieldCapabilitiesIndexResponse that = (FieldCapabilitiesIndexResponse) o;
+
+        return responseMap.equals(that.responseMap);
+    }
+
+    @Override
+    public int hashCode() {
+        return responseMap.hashCode();
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java b/core/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java
new file mode 100644
index 0000000000000..b04f882076326
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.fieldcaps;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.IndicesRequest;
+import org.elasticsearch.action.ValidateActions;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Objects;
+import java.util.Set;
+
+import static org.elasticsearch.common.xcontent.ObjectParser.fromList;
+
+public final class FieldCapabilitiesRequest extends ActionRequest implements IndicesRequest.Replaceable {
+    public static final ParseField FIELDS_FIELD = new ParseField("fields");
+    public static final String NAME = "field_caps_request";
+    private String[] indices = Strings.EMPTY_ARRAY;
+    private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpen();
+    private String[] fields = Strings.EMPTY_ARRAY;
+    // pkg private API mainly for cross cluster search to signal that we do multiple reductions ie. the results should not be merged
+    private boolean mergeResults = true;
+
+    private static ObjectParser<FieldCapabilitiesRequest, Void> PARSER =
+        new ObjectParser<>(NAME, FieldCapabilitiesRequest::new);
+
+    static {
+        PARSER.declareStringArray(fromList(String.class, FieldCapabilitiesRequest::fields),
+            FIELDS_FIELD);
+    }
+
+    public FieldCapabilitiesRequest() {}
+
+    /**
+     * Returns <code>true</code> iff the results should be merged.
+     */
+    boolean isMergeResults() {
+        return mergeResults;
+    }
+
+    /**
+     * if set to <code>true</code> the response will contain only a merged view of the per index field capabilities. Otherwise only
+     * unmerged per index field capabilities are returned.
+     */
+    void setMergeResults(boolean mergeResults) {
+        this.mergeResults = mergeResults;
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        fields = in.readStringArray();
+        if (in.getVersion().onOrAfter(Version.V_5_5_0)) {
+            indices = in.readStringArray();
+            indicesOptions = IndicesOptions.readIndicesOptions(in);
+            mergeResults = in.readBoolean();
+        } else {
+            mergeResults = true;
+        }
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeStringArray(fields);
+        if (out.getVersion().onOrAfter(Version.V_5_5_0)) {
+            out.writeStringArray(indices);
+            indicesOptions.writeIndicesOptions(out);
+            out.writeBoolean(mergeResults);
+        }
+    }
+
+    public static FieldCapabilitiesRequest parseFields(XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
+    }
+
+    /**
+     * The list of field names to retrieve
+     */
+    public FieldCapabilitiesRequest fields(String... fields) {
+        if (fields == null || fields.length == 0) {
+            throw new IllegalArgumentException("specified fields can't be null or empty");
+        }
+        Set<String> fieldSet = new HashSet<>(Arrays.asList(fields));
+        this.fields = fieldSet.toArray(new String[0]);
+        return this;
+    }
+
+    public String[] fields() {
+        return fields;
+    }
+
+    /**
+     *
+     * The list of indices to lookup
+     */
+    public FieldCapabilitiesRequest indices(String... indices) {
+        this.indices = Objects.requireNonNull(indices, "indices must not be null");
+        return this;
+    }
+
+    public FieldCapabilitiesRequest indicesOptions(IndicesOptions indicesOptions) {
+        this.indicesOptions = Objects.requireNonNull(indicesOptions, "indices options must not be null");
+        return this;
+    }
+
+    @Override
+    public String[] indices() {
+        return indices;
+    }
+
+    @Override
+    public IndicesOptions indicesOptions() {
+        return indicesOptions;
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        ActionRequestValidationException validationException = null;
+        if (fields == null || fields.length == 0) {
+            validationException =
+                ValidateActions.addValidationError("no fields specified", validationException);
+        }
+        return validationException;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        FieldCapabilitiesRequest that = (FieldCapabilitiesRequest) o;
+
+        if (!Arrays.equals(indices, that.indices)) return false;
+        if (!indicesOptions.equals(that.indicesOptions)) return false;
+        return Arrays.equals(fields, that.fields);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = Arrays.hashCode(indices);
+        result = 31 * result + indicesOptions.hashCode();
+        result = 31 * result + Arrays.hashCode(fields);
+        return result;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestBuilder.java
new file mode 100644
index 0000000000000..742d5b3ee3297
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestBuilder.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.fieldcaps;
+
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.client.ElasticsearchClient;
+
+public class FieldCapabilitiesRequestBuilder extends
+    ActionRequestBuilder<FieldCapabilitiesRequest, FieldCapabilitiesResponse,
+        FieldCapabilitiesRequestBuilder> {
+    public FieldCapabilitiesRequestBuilder(ElasticsearchClient client,
+                                           FieldCapabilitiesAction action,
+                                           String... indices) {
+        super(client, action, new FieldCapabilitiesRequest().indices(indices));
+    }
+
+    /**
+     * The list of field names to retrieve.
+     */
+    public FieldCapabilitiesRequestBuilder setFields(String... fields) {
+        request().fields(fields);
+        return this;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java b/core/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java
new file mode 100644
index 0000000000000..ae5db5835670a
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.fieldcaps;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Response for {@link FieldCapabilitiesRequest} requests.
+ */
+public class FieldCapabilitiesResponse extends ActionResponse implements ToXContent {
+    private Map<String, Map<String, FieldCapabilities>> responseMap;
+    private List<FieldCapabilitiesIndexResponse> indexResponses;
+
+    FieldCapabilitiesResponse(Map<String, Map<String, FieldCapabilities>> responseMap) {
+        this(responseMap, Collections.emptyList());
+    }
+
+    FieldCapabilitiesResponse(List<FieldCapabilitiesIndexResponse> indexResponses) {
+        this(Collections.emptyMap(), indexResponses);
+    }
+
+    private FieldCapabilitiesResponse(Map<String, Map<String, FieldCapabilities>> responseMap,
+                                      List<FieldCapabilitiesIndexResponse> indexResponses) {
+        this.responseMap = responseMap;
+        this.indexResponses = indexResponses;
+    }
+
+    /**
+     * Used for serialization
+     */
+    FieldCapabilitiesResponse() {
+        this.responseMap = Collections.emptyMap();
+    }
+
+    /**
+     * Get the field capabilities map.
+     */
+    public Map<String, Map<String, FieldCapabilities>> get() {
+        return responseMap;
+    }
+
+
+    /**
+     * Returns the actual per-index field caps responses
+     */
+    List<FieldCapabilitiesIndexResponse> getIndexResponses() {
+        return indexResponses;
+    }
+    /**
+     *
+     * Get the field capabilities per type for the provided {@code field}.
+     */
+    public Map<String, FieldCapabilities> getField(String field) {
+        return responseMap.get(field);
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        this.responseMap =
+            in.readMap(StreamInput::readString, FieldCapabilitiesResponse::readField);
+        if (in.getVersion().onOrAfter(Version.V_5_5_0)) {
+            indexResponses = in.readList(FieldCapabilitiesIndexResponse::new);
+        } else {
+            indexResponses = Collections.emptyList();
+        }
+    }
+
+    private static Map<String, FieldCapabilities> readField(StreamInput in) throws IOException {
+        return in.readMap(StreamInput::readString, FieldCapabilities::new);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeMap(responseMap, StreamOutput::writeString, FieldCapabilitiesResponse::writeField);
+        if (out.getVersion().onOrAfter(Version.V_5_5_0)) {
+            out.writeList(indexResponses);
+        }
+
+    }
+
+    private static void writeField(StreamOutput out,
+                           Map<String, FieldCapabilities> map) throws IOException {
+        out.writeMap(map, StreamOutput::writeString, (valueOut, fc) -> fc.writeTo(valueOut));
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.field("fields", responseMap);
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        FieldCapabilitiesResponse that = (FieldCapabilitiesResponse) o;
+
+        return responseMap.equals(that.responseMap);
+    }
+
+    @Override
+    public int hashCode() {
+        return responseMap.hashCode();
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java b/core/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java
new file mode 100644
index 0000000000000..3f0fb77781bdd
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java
@@ -0,0 +1,197 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.fieldcaps;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.OriginalIndices;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.CountDown;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.RemoteClusterAware;
+import org.elasticsearch.transport.RemoteClusterService;
+import org.elasticsearch.transport.Transport;
+import org.elasticsearch.transport.TransportException;
+import org.elasticsearch.transport.TransportRequestOptions;
+import org.elasticsearch.transport.TransportResponseHandler;
+import org.elasticsearch.transport.TransportService;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class TransportFieldCapabilitiesAction extends HandledTransportAction<FieldCapabilitiesRequest, FieldCapabilitiesResponse> {
+    private final ClusterService clusterService;
+    private final TransportFieldCapabilitiesIndexAction shardAction;
+    private final RemoteClusterService remoteClusterService;
+    private final TransportService transportService;
+
+    @Inject
+    public TransportFieldCapabilitiesAction(Settings settings, TransportService transportService,
+                                            ClusterService clusterService, ThreadPool threadPool,
+                                            TransportFieldCapabilitiesIndexAction shardAction,
+                                            ActionFilters actionFilters,
+                                            IndexNameExpressionResolver
+                                                    indexNameExpressionResolver) {
+        super(settings, FieldCapabilitiesAction.NAME, threadPool, transportService,
+            actionFilters, indexNameExpressionResolver, FieldCapabilitiesRequest::new);
+        this.clusterService = clusterService;
+        this.remoteClusterService = transportService.getRemoteClusterService();
+        this.transportService = transportService;
+        this.shardAction = shardAction;
+    }
+
+    @Override
+    protected void doExecute(FieldCapabilitiesRequest request,
+                             final ActionListener<FieldCapabilitiesResponse> listener) {
+        final ClusterState clusterState = clusterService.state();
+        final Map<String, OriginalIndices> remoteClusterIndices = remoteClusterService.groupIndices(request.indicesOptions(),
+            request.indices(), idx -> indexNameExpressionResolver.hasIndexOrAlias(idx, clusterState));
+        final OriginalIndices localIndices = remoteClusterIndices.remove(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY);
+        final String[] concreteIndices;
+        if (remoteClusterIndices.isEmpty() == false && localIndices.indices().length == 0) {
+            // in the case we have one or more remote indices but no local we don't expand to all local indices and just do remote
+            // indices
+            concreteIndices = Strings.EMPTY_ARRAY;
+        } else {
+            concreteIndices = indexNameExpressionResolver.concreteIndexNames(clusterState, localIndices);
+        }
+        final int totalNumRequest = concreteIndices.length + remoteClusterIndices.size();
+        final CountDown completionCounter = new CountDown(totalNumRequest);
+        final List<FieldCapabilitiesIndexResponse> indexResponses = Collections.synchronizedList(new ArrayList<>());
+        final Runnable onResponse = () -> {
+            if (completionCounter.countDown()) {
+                if (request.isMergeResults()) {
+                    listener.onResponse(merge(indexResponses));
+                } else {
+                    listener.onResponse(new FieldCapabilitiesResponse(indexResponses));
+                }
+            }
+        };
+        if (totalNumRequest == 0) {
+            listener.onResponse(new FieldCapabilitiesResponse());
+        } else {
+            ActionListener<FieldCapabilitiesIndexResponse> innerListener = new ActionListener<FieldCapabilitiesIndexResponse>() {
+                @Override
+                public void onResponse(FieldCapabilitiesIndexResponse result) {
+                    indexResponses.add(result);
+                    onResponse.run();
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    // TODO we should somehow inform the user that we failed
+                    onResponse.run();
+                }
+            };
+            for (String index : concreteIndices) {
+                shardAction.execute(new FieldCapabilitiesIndexRequest(request.fields(), index), innerListener);
+            }
+
+            // this is the cross cluster part of this API - we force the other cluster to not merge the results but instead
+            // send us back all individual index results.
+            for (Map.Entry<String, OriginalIndices> remoteIndices : remoteClusterIndices.entrySet()) {
+                String clusterAlias = remoteIndices.getKey();
+                OriginalIndices originalIndices = remoteIndices.getValue();
+                // if we are connected this is basically a no-op, if we are not we try to connect in parallel in a non-blocking fashion
+                remoteClusterService.ensureConnected(clusterAlias, ActionListener.wrap(v -> {
+                    Transport.Connection connection = remoteClusterService.getConnection(clusterAlias);
+                    FieldCapabilitiesRequest remoteRequest = new FieldCapabilitiesRequest();
+                    remoteRequest.setMergeResults(false); // we need to merge on this node
+                    remoteRequest.indicesOptions(originalIndices.indicesOptions());
+                    remoteRequest.indices(originalIndices.indices());
+                    remoteRequest.fields(request.fields());
+                    transportService.sendRequest(connection, FieldCapabilitiesAction.NAME, remoteRequest, TransportRequestOptions.EMPTY,
+                        new TransportResponseHandler<FieldCapabilitiesResponse>() {
+
+                            @Override
+                            public FieldCapabilitiesResponse newInstance() {
+                                return new FieldCapabilitiesResponse();
+                            }
+
+                            @Override
+                            public void handleResponse(FieldCapabilitiesResponse response) {
+                                try {
+                                    for (FieldCapabilitiesIndexResponse res : response.getIndexResponses()) {
+                                        indexResponses.add(new FieldCapabilitiesIndexResponse(RemoteClusterAware.
+                                            buildRemoteIndexName(clusterAlias, res.getIndexName()), res.get()));
+                                    }
+                                } finally {
+                                    onResponse.run();
+                                }
+                            }
+
+                            @Override
+                            public void handleException(TransportException exp) {
+                                onResponse.run();
+                            }
+
+                            @Override
+                            public String executor() {
+                                return ThreadPool.Names.SAME;
+                            }
+                        });
+                }, e -> onResponse.run()));
+            }
+
+        }
+    }
+
+    private FieldCapabilitiesResponse merge(List<FieldCapabilitiesIndexResponse> indexResponses) {
+        Map<String, Map<String, FieldCapabilities.Builder>> responseMapBuilder = new HashMap<> ();
+        for (FieldCapabilitiesIndexResponse response : indexResponses) {
+            innerMerge(responseMapBuilder, response.getIndexName(), response.get());
+        }
+
+        Map<String, Map<String, FieldCapabilities>> responseMap = new HashMap<>();
+        for (Map.Entry<String, Map<String, FieldCapabilities.Builder>> entry :
+            responseMapBuilder.entrySet()) {
+            Map<String, FieldCapabilities> typeMap = new HashMap<>();
+            boolean multiTypes = entry.getValue().size() > 1;
+            for (Map.Entry<String, FieldCapabilities.Builder> fieldEntry :
+                entry.getValue().entrySet()) {
+                typeMap.put(fieldEntry.getKey(), fieldEntry.getValue().build(multiTypes));
+            }
+            responseMap.put(entry.getKey(), typeMap);
+        }
+
+        return new FieldCapabilitiesResponse(responseMap);
+    }
+
+    private void innerMerge(Map<String, Map<String, FieldCapabilities.Builder>> responseMapBuilder, String indexName,
+                            Map<String, FieldCapabilities> map) {
+        for (Map.Entry<String, FieldCapabilities> entry : map.entrySet()) {
+            final String field = entry.getKey();
+            final FieldCapabilities fieldCap = entry.getValue();
+            Map<String, FieldCapabilities.Builder> typeMap = responseMapBuilder.computeIfAbsent(field, f -> new HashMap<>());
+            FieldCapabilities.Builder builder = typeMap.computeIfAbsent(fieldCap.getType(), key -> new FieldCapabilities.Builder(field,
+                key));
+            builder.add(indexName, fieldCap.isSearchable(), fieldCap.isAggregatable());
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java b/core/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java
new file mode 100644
index 0000000000000..b9e6f56b6d7ad
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.fieldcaps;
+
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.single.shard.TransportSingleShardAction;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.block.ClusterBlockException;
+import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.routing.ShardsIterator;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+public class TransportFieldCapabilitiesIndexAction extends TransportSingleShardAction<FieldCapabilitiesIndexRequest,
+    FieldCapabilitiesIndexResponse> {
+
+    private static final String ACTION_NAME = FieldCapabilitiesAction.NAME + "[index]";
+
+    private final IndicesService indicesService;
+
+    @Inject
+    public TransportFieldCapabilitiesIndexAction(Settings settings, ClusterService clusterService, TransportService transportService,
+                                                 IndicesService indicesService, ThreadPool threadPool, ActionFilters actionFilters,
+                                                 IndexNameExpressionResolver indexNameExpressionResolver) {
+        super(settings, ACTION_NAME, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver,
+            FieldCapabilitiesIndexRequest::new, ThreadPool.Names.MANAGEMENT);
+        this.indicesService = indicesService;
+    }
+
+    @Override
+    protected boolean resolveIndex(FieldCapabilitiesIndexRequest request) {
+        //internal action, index already resolved
+        return false;
+    }
+
+    @Override
+    protected ShardsIterator shards(ClusterState state, InternalRequest request) {
+        // Will balance requests between shards
+        // Resolve patterns and deduplicate
+        return state.routingTable().index(request.concreteIndex()).randomAllActiveShardsIt();
+    }
+
+    @Override
+    protected FieldCapabilitiesIndexResponse shardOperation(final FieldCapabilitiesIndexRequest request, ShardId shardId) {
+        MapperService mapperService = indicesService.indexServiceSafe(shardId.getIndex()).mapperService();
+        Set<String> fieldNames = new HashSet<>();
+        for (String field : request.fields()) {
+            fieldNames.addAll(mapperService.simpleMatchToIndexNames(field));
+        }
+        Map<String, FieldCapabilities> responseMap = new HashMap<>();
+        for (String field : fieldNames) {
+            MappedFieldType ft = mapperService.fullName(field);
+            if (ft != null) {
+                FieldCapabilities fieldCap = new FieldCapabilities(field, ft.typeName(), ft.isSearchable(), ft.isAggregatable());
+                responseMap.put(field, fieldCap);
+            }
+        }
+        return new FieldCapabilitiesIndexResponse(shardId.getIndexName(), responseMap);
+    }
+
+    @Override
+    protected FieldCapabilitiesIndexResponse newResponse() {
+        return new FieldCapabilitiesIndexResponse();
+    }
+
+    @Override
+    protected ClusterBlockException checkRequestBlock(ClusterState state, InternalRequest request) {
+        return state.blocks().indexBlockedException(ClusterBlockLevel.METADATA_READ, request.concreteIndex());
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStats.java b/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStats.java
index 4a4f106b08575..6c6d0da2e36d3 100644
--- a/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStats.java
+++ b/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStats.java
@@ -20,8 +20,10 @@
 package org.elasticsearch.action.fieldstats;
 
 import org.apache.lucene.document.InetAddressPoint;
+import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.StringHelper;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -34,6 +36,7 @@
 
 import java.io.IOException;
 import java.net.InetAddress;
+import java.util.Objects;
 
 public abstract class FieldStats<T> implements Writeable, ToXContent {
     private final byte type;
@@ -43,16 +46,55 @@ public abstract class FieldStats<T> implements Writeable, ToXContent {
     private long sumTotalTermFreq;
     private boolean isSearchable;
     private boolean isAggregatable;
+    private boolean hasMinMax;
     protected T minValue;
     protected T maxValue;
 
-    FieldStats(byte type, long maxDoc, boolean isSearchable, boolean isAggregatable) {
-        this(type, maxDoc, 0, 0, 0, isSearchable, isAggregatable, null, null);
+    /**
+     * Builds a FieldStats where min and max value are not available for the field.
+     * @param type The native type of this FieldStats
+     * @param maxDoc Max number of docs
+     * @param docCount  the number of documents that have at least one term for this field,
+     *                  or -1 if this information isn't available for this field.
+     * @param sumDocFreq  the sum of {@link TermsEnum#docFreq()} for all terms in this field,
+     *                    or -1 if this information isn't available for this field.
+     * @param sumTotalTermFreq the sum of {@link TermsEnum#totalTermFreq} for all terms in this field,
+     *                         or -1 if this measure isn't available for this field.
+     * @param isSearchable true if this field is searchable
+     * @param isAggregatable true if this field is aggregatable
+     */
+    FieldStats(byte type, long maxDoc, long docCount, long sumDocFreq, long sumTotalTermFreq,
+               boolean isSearchable, boolean isAggregatable) {
+        this.type = type;
+        this.maxDoc = maxDoc;
+        this.docCount = docCount;
+        this.sumDocFreq = sumDocFreq;
+        this.sumTotalTermFreq = sumTotalTermFreq;
+        this.isSearchable = isSearchable;
+        this.isAggregatable = isAggregatable;
+        this.hasMinMax = false;
     }
 
+    /**
+     * Builds a FieldStats with min and max value for the field.
+     * @param type The native type of this FieldStats
+     * @param maxDoc Max number of docs
+     * @param docCount  the number of documents that have at least one term for this field,
+     *                  or -1 if this information isn't available for this field.
+     * @param sumDocFreq  the sum of {@link TermsEnum#docFreq()} for all terms in this field,
+     *                    or -1 if this information isn't available for this field.
+     * @param sumTotalTermFreq the sum of {@link TermsEnum#totalTermFreq} for all terms in this field,
+     *                         or -1 if this measure isn't available for this field.
+     * @param isSearchable true if this field is searchable
+     * @param isAggregatable true if this field is aggregatable
+     * @param minValue the minimum value indexed in this field
+     * @param maxValue the maximum value indexed in this field
+     */
     FieldStats(byte type,
                long maxDoc, long docCount, long sumDocFreq, long sumTotalTermFreq,
                boolean isSearchable, boolean isAggregatable, T minValue, T maxValue) {
+        Objects.requireNonNull(minValue, "minValue must not be null");
+        Objects.requireNonNull(maxValue, "maxValue must not be null");
         this.type = type;
         this.maxDoc = maxDoc;
         this.docCount = docCount;
@@ -60,6 +102,7 @@ public abstract class FieldStats<T> implements Writeable, ToXContent {
         this.sumTotalTermFreq = sumTotalTermFreq;
         this.isSearchable = isSearchable;
         this.isAggregatable = isAggregatable;
+        this.hasMinMax = true;
         this.minValue = minValue;
         this.maxValue = maxValue;
     }
@@ -80,11 +123,20 @@ public String getDisplayType() {
                 return "string";
             case 4:
                 return "ip";
+            case 5:
+                return "geo_point";
             default:
                 throw new IllegalArgumentException("Unknown type.");
         }
     }
 
+    /**
+     * @return true if min/max informations are available for this field
+     */
+    public boolean hasMinMax() {
+        return hasMinMax;
+    }
+
     /**
      * @return the total number of documents.
      *
@@ -216,18 +268,20 @@ public final void accumulate(FieldStats other) {
         isAggregatable |= other.isAggregatable;
 
         assert type == other.getType();
-        updateMinMax((T) other.minValue, (T) other.maxValue);
+        if (hasMinMax && other.hasMinMax) {
+            updateMinMax((T) other.minValue, (T) other.maxValue);
+        } else {
+            hasMinMax = false;
+            minValue = null;
+            maxValue = null;
+        }
     }
 
-    private void updateMinMax(T min, T max) {
-        if (minValue == null) {
-            minValue = min;
-        } else if (min != null && compare(minValue, min) > 0) {
+    protected void updateMinMax(T min, T max) {
+        if (compare(minValue, min) > 0) {
             minValue = min;
         }
-        if (maxValue == null) {
-            maxValue = max;
-        } else if (max != null && compare(maxValue, max) < 0) {
+        if (compare(maxValue, max) < 0) {
             maxValue = max;
         }
     }
@@ -245,7 +299,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.field(SUM_TOTAL_TERM_FREQ_FIELD, sumTotalTermFreq);
         builder.field(SEARCHABLE_FIELD, isSearchable);
         builder.field(AGGREGATABLE_FIELD, isAggregatable);
-        toInnerXContent(builder);
+        if (hasMinMax) {
+            toInnerXContent(builder);
+        }
         builder.endObject();
         return builder;
     }
@@ -266,9 +322,14 @@ public final void writeTo(StreamOutput out) throws IOException {
         out.writeLong(sumTotalTermFreq);
         out.writeBoolean(isSearchable);
         out.writeBoolean(isAggregatable);
-        boolean hasMinMax = minValue != null;
-        out.writeBoolean(hasMinMax);
-        if (hasMinMax) {
+        if (out.getVersion().onOrAfter(Version.V_5_2_0)) {
+            out.writeBoolean(hasMinMax);
+            if (hasMinMax) {
+                writeMinMax(out);
+            }
+        } else {
+            assert hasMinMax : "cannot serialize null min/max fieldstats in a mixed-cluster " +
+                    "with pre-" + Version.V_5_2_0 + " nodes, remote version [" + out.getVersion() + "]";
             writeMinMax(out);
         }
     }
@@ -280,7 +341,7 @@ public final void writeTo(StreamOutput out) throws IOException {
      * otherwise <code>false</code> is returned
      */
     public boolean match(IndexConstraint constraint) {
-        if (minValue == null) {
+        if (hasMinMax == false) {
             return false;
         }
         int cmp;
@@ -307,23 +368,47 @@ public boolean match(IndexConstraint constraint) {
         }
     }
 
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        FieldStats<?> that = (FieldStats<?>) o;
+
+        if (type != that.type) return false;
+        if (maxDoc != that.maxDoc) return false;
+        if (docCount != that.docCount) return false;
+        if (sumDocFreq != that.sumDocFreq) return false;
+        if (sumTotalTermFreq != that.sumTotalTermFreq) return false;
+        if (isSearchable != that.isSearchable) return false;
+        if (isAggregatable != that.isAggregatable) return false;
+        if (hasMinMax != that.hasMinMax) return false;
+        if (hasMinMax == false) {
+            return true;
+        }
+        if (!minValue.equals(that.minValue)) return false;
+        return maxValue.equals(that.maxValue);
+
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(type, maxDoc, docCount, sumDocFreq, sumTotalTermFreq, isSearchable, isAggregatable,
+            hasMinMax, minValue, maxValue);
+    }
+
     public static class Long extends FieldStats<java.lang.Long> {
         public Long(long maxDoc, long docCount, long sumDocFreq, long sumTotalTermFreq,
-                    boolean isSearchable, boolean isAggregatable,
-                    long minValue, long maxValue) {
+                    boolean isSearchable, boolean isAggregatable) {
             super((byte) 0, maxDoc, docCount, sumDocFreq, sumTotalTermFreq,
-                isSearchable, isAggregatable, minValue, maxValue);
+                isSearchable, isAggregatable);
         }
 
         public Long(long maxDoc, long docCount, long sumDocFreq, long sumTotalTermFreq,
-                    boolean isSearchable, boolean isAggregatable) {
+                    boolean isSearchable, boolean isAggregatable,
+                    long minValue, long maxValue) {
             super((byte) 0, maxDoc, docCount, sumDocFreq, sumTotalTermFreq,
-                isSearchable, isAggregatable, null, null);
-        }
-
-        public Long(long maxDoc,
-                    boolean isSearchable, boolean isAggregatable) {
-            super((byte) 0, maxDoc, isSearchable, isAggregatable);
+                isSearchable, isAggregatable, minValue, maxValue);
         }
 
         @Override
@@ -344,16 +429,21 @@ public java.lang.Long valueOf(String value, String optionalFormat) {
 
         @Override
         public String getMinValueAsString() {
-            return minValue != null ? java.lang.Long.toString(minValue) : null;
+            return java.lang.Long.toString(minValue);
         }
 
         @Override
         public String getMaxValueAsString() {
-            return maxValue != null ? java.lang.Long.toString(maxValue) : null;
+            return java.lang.Long.toString(maxValue);
         }
     }
 
     public static class Double extends FieldStats<java.lang.Double> {
+        public Double(long maxDoc, long docCount, long sumDocFreq, long sumTotalTermFreq,
+                      boolean isSearchable, boolean isAggregatable) {
+            super((byte) 1, maxDoc, docCount, sumDocFreq, sumTotalTermFreq, isSearchable, isAggregatable);
+        }
+
         public Double(long maxDoc, long docCount, long sumDocFreq, long sumTotalTermFreq,
                       boolean isSearchable, boolean isAggregatable,
                       double minValue, double maxValue) {
@@ -361,15 +451,6 @@ public Double(long maxDoc, long docCount, long sumDocFreq, long sumTotalTermFreq
                 minValue, maxValue);
         }
 
-        public Double(long maxDoc, long docCount, long sumDocFreq, long sumTotalTermFreq,
-                      boolean isSearchable, boolean isAggregatable) {
-            super((byte) 1, maxDoc, docCount, sumDocFreq, sumTotalTermFreq, isSearchable, isAggregatable, null, null);
-        }
-
-        public Double(long maxDoc, boolean isSearchable, boolean isAggregatable) {
-            super((byte) 1, maxDoc, isSearchable, isAggregatable);
-        }
-
         @Override
         public int compare(java.lang.Double o1, java.lang.Double o2) {
             return o1.compareTo(o2);
@@ -391,12 +472,12 @@ public java.lang.Double valueOf(String value, String optionalFormat) {
 
         @Override
         public String getMinValueAsString() {
-            return minValue != null ? java.lang.Double.toString(minValue) : null;
+            return java.lang.Double.toString(minValue);
         }
 
         @Override
         public String getMaxValueAsString() {
-            return maxValue != null ? java.lang.Double.toString(maxValue) : null;
+            return java.lang.Double.toString(maxValue);
         }
     }
 
@@ -404,25 +485,17 @@ public static class Date extends FieldStats<java.lang.Long> {
         private FormatDateTimeFormatter formatter;
 
         public Date(long maxDoc, long docCount, long sumDocFreq, long sumTotalTermFreq,
-                    boolean isSearchable, boolean isAggregatable,
-                    FormatDateTimeFormatter formatter,
-                    long minValue, long maxValue) {
-            super((byte) 2, maxDoc, docCount, sumDocFreq, sumTotalTermFreq, isSearchable, isAggregatable,
-                minValue, maxValue);
-            this.formatter = formatter;
+                    boolean isSearchable, boolean isAggregatable) {
+            super((byte) 2, maxDoc, docCount, sumDocFreq, sumTotalTermFreq, isSearchable, isAggregatable);
+            this.formatter = null;
         }
 
         public Date(long maxDoc, long docCount, long sumDocFreq, long sumTotalTermFreq,
                     boolean isSearchable, boolean isAggregatable,
-                    FormatDateTimeFormatter formatter) {
+                    FormatDateTimeFormatter formatter,
+                    long minValue, long maxValue) {
             super((byte) 2, maxDoc, docCount, sumDocFreq, sumTotalTermFreq, isSearchable, isAggregatable,
-                null, null);
-            this.formatter = formatter;
-        }
-
-        public Date(long maxDoc, boolean isSearchable, boolean isAggregatable,
-                    FormatDateTimeFormatter formatter) {
-            super((byte) 2, maxDoc, isSearchable, isAggregatable);
+                minValue, maxValue);
             this.formatter = formatter;
         }
 
@@ -449,16 +522,37 @@ public java.lang.Long valueOf(String value, String fmt) {
 
         @Override
         public String getMinValueAsString() {
-            return minValue != null ? formatter.printer().print(minValue) : null;
+            return formatter.printer().print(minValue);
         }
 
         @Override
         public String getMaxValueAsString() {
-            return maxValue != null ? formatter.printer().print(maxValue) : null;
+            return formatter.printer().print(maxValue);
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (!super.equals(o)) return false;
+            Date that = (Date) o;
+            return Objects.equals(formatter == null ? null : formatter.format(),
+                that.formatter == null ? null : that.formatter.format());
+        }
+
+        @Override
+        public int hashCode() {
+            int result = super.hashCode();
+            result = 31 * result + (formatter == null ? 0 : formatter.format().hashCode());
+            return result;
         }
     }
 
     public static class Text extends FieldStats<BytesRef> {
+        public Text(long maxDoc, long docCount, long sumDocFreq, long sumTotalTermFreq,
+                    boolean isSearchable, boolean isAggregatable) {
+            super((byte) 3, maxDoc, docCount, sumDocFreq, sumTotalTermFreq,
+                isSearchable, isAggregatable);
+        }
+
         public Text(long maxDoc, long docCount, long sumDocFreq, long sumTotalTermFreq,
                     boolean isSearchable, boolean isAggregatable,
                     BytesRef minValue, BytesRef maxValue) {
@@ -467,10 +561,6 @@ public Text(long maxDoc, long docCount, long sumDocFreq, long sumTotalTermFreq,
                 minValue, maxValue);
         }
 
-        public Text(long maxDoc, boolean isSearchable, boolean isAggregatable) {
-            super((byte) 3, maxDoc, isSearchable, isAggregatable);
-        }
-
         @Override
         public int compare(BytesRef o1, BytesRef o2) {
             return o1.compareTo(o2);
@@ -492,12 +582,12 @@ protected BytesRef valueOf(String value, String optionalFormat) {
 
         @Override
         public String getMinValueAsString() {
-            return minValue != null ? minValue.utf8ToString() : null;
+            return minValue.utf8ToString();
         }
 
         @Override
         public String getMaxValueAsString() {
-            return maxValue != null ? maxValue.utf8ToString() : null;
+            return maxValue.utf8ToString();
         }
 
         @Override
@@ -508,6 +598,13 @@ protected void toInnerXContent(XContentBuilder builder) throws IOException {
     }
 
     public static class Ip extends FieldStats<InetAddress> {
+        public Ip(long maxDoc, long docCount, long sumDocFreq, long sumTotalTermFreq,
+                  boolean isSearchable, boolean isAggregatable) {
+            super((byte) 4, maxDoc, docCount, sumDocFreq, sumTotalTermFreq,
+                isSearchable, isAggregatable);
+        }
+
+
         public Ip(long maxDoc, long docCount, long sumDocFreq, long sumTotalTermFreq,
                   boolean isSearchable, boolean isAggregatable,
                   InetAddress minValue, InetAddress maxValue) {
@@ -516,10 +613,6 @@ public Ip(long maxDoc, long docCount, long sumDocFreq, long sumTotalTermFreq,
                 minValue, maxValue);
         }
 
-        public Ip(long maxDoc, boolean isSearchable, boolean isAggregatable) {
-            super((byte) 4, maxDoc, isSearchable, isAggregatable);
-        }
-
         @Override
         public int compare(InetAddress o1, InetAddress o2) {
             byte[] b1 = InetAddressPoint.encode(o1);
@@ -544,12 +637,61 @@ public InetAddress valueOf(String value, String fmt) {
 
         @Override
         public String getMinValueAsString() {
-            return  minValue != null ? NetworkAddress.format(minValue) : null;
+            return NetworkAddress.format(minValue);
+        }
+
+        @Override
+        public String getMaxValueAsString() {
+            return NetworkAddress.format(maxValue);
+        }
+    }
+
+    public static class GeoPoint extends FieldStats<org.elasticsearch.common.geo.GeoPoint> {
+        public GeoPoint(long maxDoc, long docCount, long sumDocFreq, long sumTotalTermFreq,
+                  boolean isSearchable, boolean isAggregatable) {
+            super((byte) 5, maxDoc, docCount, sumDocFreq, sumTotalTermFreq,
+                isSearchable, isAggregatable);
+        }
+
+        public GeoPoint(long maxDoc, long docCount, long sumDocFreq, long sumTotalTermFreq,
+                        boolean isSearchable, boolean isAggregatable,
+                        org.elasticsearch.common.geo.GeoPoint minValue, org.elasticsearch.common.geo.GeoPoint maxValue) {
+            super((byte) 5, maxDoc, docCount, sumDocFreq, sumTotalTermFreq, isSearchable, isAggregatable,
+                minValue, maxValue);
+        }
+
+        @Override
+        public org.elasticsearch.common.geo.GeoPoint valueOf(String value, String fmt) {
+            return org.elasticsearch.common.geo.GeoPoint.parseFromLatLon(value);
+        }
+
+        @Override
+        protected void updateMinMax(org.elasticsearch.common.geo.GeoPoint min, org.elasticsearch.common.geo.GeoPoint max) {
+            minValue.reset(Math.min(min.lat(), minValue.lat()), Math.min(min.lon(), minValue.lon()));
+            maxValue.reset(Math.max(max.lat(), maxValue.lat()), Math.max(max.lon(), maxValue.lon()));
+        }
+
+        @Override
+        public int compare(org.elasticsearch.common.geo.GeoPoint p1, org.elasticsearch.common.geo.GeoPoint p2) {
+            throw new IllegalArgumentException("compare is not supported for geo_point field stats");
+        }
+
+        @Override
+        public void writeMinMax(StreamOutput out) throws IOException {
+            out.writeDouble(minValue.lat());
+            out.writeDouble(minValue.lon());
+            out.writeDouble(maxValue.lat());
+            out.writeDouble(maxValue.lon());
+        }
+
+        @Override
+        public String getMinValueAsString() {
+            return minValue.toString();
         }
 
         @Override
         public String getMaxValueAsString() {
-            return  maxValue != null ? NetworkAddress.format(maxValue) : null;
+            return maxValue.toString();
         }
     }
 
@@ -561,56 +703,71 @@ public static FieldStats readFrom(StreamInput in) throws IOException {
         long sumTotalTermFreq = in.readLong();
         boolean isSearchable = in.readBoolean();
         boolean isAggregatable = in.readBoolean();
-        boolean hasMinMax = in.readBoolean();
-
+        boolean hasMinMax = true;
+        if (in.getVersion().onOrAfter(Version.V_5_2_0)) {
+            hasMinMax = in.readBoolean();
+        }
         switch (type) {
             case 0:
                 if (hasMinMax) {
                     return new Long(maxDoc, docCount, sumDocFreq, sumTotalTermFreq,
                         isSearchable, isAggregatable, in.readLong(), in.readLong());
+                } else {
+                    return new Long(maxDoc, docCount, sumDocFreq, sumTotalTermFreq,
+                        isSearchable, isAggregatable);
                 }
-                return new Long(maxDoc, docCount, sumDocFreq, sumTotalTermFreq,
-                    isSearchable, isAggregatable);
-
             case 1:
                 if (hasMinMax) {
                     return new Double(maxDoc, docCount, sumDocFreq, sumTotalTermFreq,
                         isSearchable, isAggregatable, in.readDouble(), in.readDouble());
+                } else {
+                    return new Double(maxDoc, docCount, sumDocFreq, sumTotalTermFreq,
+                        isSearchable, isAggregatable);
                 }
-                return new Double(maxDoc, docCount, sumDocFreq, sumTotalTermFreq,
-                    isSearchable, isAggregatable);
-
             case 2:
-                FormatDateTimeFormatter formatter = Joda.forPattern(in.readString());
                 if (hasMinMax) {
+                    FormatDateTimeFormatter formatter = Joda.forPattern(in.readString());
                     return new Date(maxDoc, docCount, sumDocFreq, sumTotalTermFreq,
                         isSearchable, isAggregatable, formatter, in.readLong(), in.readLong());
+                } else {
+                    return new Date(maxDoc, docCount, sumDocFreq, sumTotalTermFreq,
+                        isSearchable, isAggregatable);
                 }
-                return new Date(maxDoc, docCount, sumDocFreq, sumTotalTermFreq,
-                    isSearchable, isAggregatable, formatter);
-
             case 3:
                 if (hasMinMax) {
                     return new Text(maxDoc, docCount, sumDocFreq, sumTotalTermFreq,
                         isSearchable, isAggregatable, in.readBytesRef(), in.readBytesRef());
+                } else {
+                    return new Text(maxDoc, docCount, sumDocFreq, sumTotalTermFreq,
+                        isSearchable, isAggregatable);
                 }
-                return new Text(maxDoc, docCount, sumDocFreq, sumTotalTermFreq,
-                        isSearchable, isAggregatable, null, null);
 
-            case 4:
-                InetAddress min = null;
-                InetAddress max = null;
-                if (hasMinMax) {
-                    int l1 = in.readByte();
-                    byte[] b1 = new byte[l1];
-                    int l2 = in.readByte();
-                    byte[] b2 = new byte[l2];
-                    min = InetAddressPoint.decode(b1);
-                    max = InetAddressPoint.decode(b2);
+            case 4: {
+                if (hasMinMax == false) {
+                    return new Ip(maxDoc, docCount, sumDocFreq, sumTotalTermFreq,
+                        isSearchable, isAggregatable);
                 }
+                int l1 = in.readByte();
+                byte[] b1 = new byte[l1];
+                in.readBytes(b1, 0, l1);
+                int l2 = in.readByte();
+                byte[] b2 = new byte[l2];
+                in.readBytes(b2, 0, l2);
+                InetAddress min = InetAddressPoint.decode(b1);
+                InetAddress max = InetAddressPoint.decode(b2);
                 return new Ip(maxDoc, docCount, sumDocFreq, sumTotalTermFreq,
                     isSearchable, isAggregatable, min, max);
-
+            }
+            case 5: {
+                if (hasMinMax == false) {
+                    return new GeoPoint(maxDoc, docCount, sumDocFreq, sumTotalTermFreq,
+                        isSearchable, isAggregatable);
+                }
+                org.elasticsearch.common.geo.GeoPoint min = new org.elasticsearch.common.geo.GeoPoint(in.readDouble(), in.readDouble());
+                org.elasticsearch.common.geo.GeoPoint max = new org.elasticsearch.common.geo.GeoPoint(in.readDouble(), in.readDouble());
+                return new GeoPoint(maxDoc, docCount, sumDocFreq, sumTotalTermFreq,
+                    isSearchable, isAggregatable, min, max);
+            }
             default:
                 throw new IllegalArgumentException("Unknown type.");
         }
diff --git a/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsRequest.java b/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsRequest.java
index d0b40374d6bb3..7d1bc0c56f09b 100644
--- a/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsRequest.java
@@ -24,10 +24,8 @@
 import org.elasticsearch.action.ValidateActions;
 import org.elasticsearch.action.support.broadcast.BroadcastRequest;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentParser.Token;
 
@@ -76,41 +74,39 @@ public void setIndexConstraints(IndexConstraint[] indexConstraints) {
         this.indexConstraints = indexConstraints;
     }
 
-    public void source(BytesReference content) throws IOException {
+    public void source(XContentParser parser) throws IOException {
         List<IndexConstraint> indexConstraints = new ArrayList<>();
         List<String> fields = new ArrayList<>();
-        try (XContentParser parser = XContentHelper.createParser(content)) {
-            String fieldName = null;
-            Token token = parser.nextToken();
-            assert token == Token.START_OBJECT;
-            for (token = parser.nextToken(); token != Token.END_OBJECT; token = parser.nextToken()) {
-                switch (token) {
-                    case FIELD_NAME:
-                        fieldName = parser.currentName();
-                        break;
-                    case START_OBJECT:
-                        if ("index_constraints".equals(fieldName)) {
-                            parseIndexConstraints(indexConstraints, parser);
-                        } else {
-                            throw new IllegalArgumentException("unknown field [" + fieldName + "]");
-                        }
-                        break;
-                    case START_ARRAY:
-                        if ("fields".equals(fieldName)) {
-                            while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
-                                if (token.isValue()) {
-                                    fields.add(parser.text());
-                                } else {
-                                    throw new IllegalArgumentException("unexpected token [" + token + "]");
-                                }
+        String fieldName = null;
+        Token token = parser.nextToken();
+        assert token == Token.START_OBJECT;
+        for (token = parser.nextToken(); token != Token.END_OBJECT; token = parser.nextToken()) {
+            switch (token) {
+                case FIELD_NAME:
+                    fieldName = parser.currentName();
+                    break;
+                case START_OBJECT:
+                    if ("index_constraints".equals(fieldName)) {
+                        parseIndexConstraints(indexConstraints, parser);
+                    } else {
+                        throw new IllegalArgumentException("unknown field [" + fieldName + "]");
+                    }
+                    break;
+                case START_ARRAY:
+                    if ("fields".equals(fieldName)) {
+                        while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                            if (token.isValue()) {
+                                fields.add(parser.text());
+                            } else {
+                                throw new IllegalArgumentException("unexpected token [" + token + "]");
                             }
-                        } else {
-                            throw new IllegalArgumentException("unknown field [" + fieldName + "]");
                         }
-                        break;
-                    default:
-                        throw new IllegalArgumentException("unexpected token [" + token + "]");
-                }
+                    } else {
+                        throw new IllegalArgumentException("unknown field [" + fieldName + "]");
+                    }
+                    break;
+                default:
+                    throw new IllegalArgumentException("unexpected token [" + token + "]");
             }
         }
         this.fields = fields.toArray(new String[fields.size()]);
diff --git a/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsResponse.java b/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsResponse.java
index 14e2f13d4ff32..aeeacbf5a26f2 100644
--- a/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsResponse.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.action.fieldstats;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ShardOperationFailedException;
 import org.elasticsearch.action.support.broadcast.BroadcastResponse;
 import org.elasticsearch.common.Nullable;
@@ -93,10 +94,21 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeVInt(indicesMergedFieldStats.size());
         for (Map.Entry<String, Map<String, FieldStats>> entry1 : indicesMergedFieldStats.entrySet()) {
             out.writeString(entry1.getKey());
-            out.writeVInt(entry1.getValue().size());
+            int size = entry1.getValue().size();
+            if (out.getVersion().before(Version.V_5_2_0)) {
+                // filter fieldstats without min/max information
+                for (FieldStats stats : entry1.getValue().values()) {
+                    if (stats.hasMinMax() == false) {
+                        size--;
+                    }
+                }
+            }
+            out.writeVInt(size);
             for (Map.Entry<String, FieldStats> entry2 : entry1.getValue().entrySet()) {
-                out.writeString(entry2.getKey());
-                entry2.getValue().writeTo(out);
+                if (entry2.getValue().hasMinMax() || out.getVersion().onOrAfter(Version.V_5_2_0)) {
+                    out.writeString(entry2.getKey());
+                    entry2.getValue().writeTo(out);
+                }
             }
         }
         out.writeVInt(conflicts.size());
diff --git a/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsShardRequest.java b/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsShardRequest.java
index 85a0d469541fe..9152d64f2d527 100644
--- a/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsShardRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsShardRequest.java
@@ -41,8 +41,7 @@ public FieldStatsShardRequest() {
 
     public FieldStatsShardRequest(ShardId shardId, FieldStatsRequest request) {
         super(shardId, request);
-        Set<String> fields = new HashSet<>();
-        fields.addAll(Arrays.asList(request.getFields()));
+        Set<String> fields = new HashSet<>(Arrays.asList(request.getFields()));
         for (IndexConstraint indexConstraint : request.getIndexConstraints()) {
             fields.add(indexConstraint.getField());
         }
diff --git a/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsShardResponse.java b/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsShardResponse.java
index 7cc298729f03c..58598540ef45a 100644
--- a/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsShardResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsShardResponse.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.action.fieldstats;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.support.broadcast.BroadcastShardResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -27,6 +28,7 @@
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 /**
  */
@@ -46,6 +48,12 @@ public Map<String, FieldStats<?>> getFieldStats() {
         return fieldStats;
     }
 
+    Map<String, FieldStats<?> > filterNullMinMax() {
+        return fieldStats.entrySet().stream()
+            .filter((e) -> e.getValue().hasMinMax())
+            .collect(Collectors.toMap(p -> p.getKey(), p -> p.getValue()));
+    }
+
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
@@ -61,8 +69,17 @@ public void readFrom(StreamInput in) throws IOException {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        out.writeVInt(fieldStats.size());
-        for (Map.Entry<String, FieldStats<?>> entry : fieldStats.entrySet()) {
+        final Map<String, FieldStats<?> > stats;
+        if (out.getVersion().before(Version.V_5_2_0)) {
+            /**
+             * FieldStats with null min/max are not (de)serializable in versions prior to {@link Version.V_5_2_0_UNRELEASED}
+             */
+            stats = filterNullMinMax();
+        } else {
+            stats = getFieldStats();
+        }
+        out.writeVInt(stats.size());
+        for (Map.Entry<String, FieldStats<?>> entry : stats.entrySet()) {
             out.writeString(entry.getKey());
             entry.getValue().writeTo(out);
         }
diff --git a/core/src/main/java/org/elasticsearch/action/fieldstats/TransportFieldStatsAction.java b/core/src/main/java/org/elasticsearch/action/fieldstats/TransportFieldStatsAction.java
index e65f69514320f..9ee72223a6684 100644
--- a/core/src/main/java/org/elasticsearch/action/fieldstats/TransportFieldStatsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/fieldstats/TransportFieldStatsAction.java
@@ -36,8 +36,6 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.engine.Engine;
-import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
diff --git a/core/src/main/java/org/elasticsearch/action/get/GetRequest.java b/core/src/main/java/org/elasticsearch/action/get/GetRequest.java
index 42c4ccc701d26..abcf6aa671155 100644
--- a/core/src/main/java/org/elasticsearch/action/get/GetRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/get/GetRequest.java
@@ -26,6 +26,8 @@
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
@@ -51,7 +53,7 @@ public class GetRequest extends SingleShardRequest<GetRequest> implements Realti
     private String parent;
     private String preference;
 
-    private String[] fields;
+    private String[] storedFields;
 
     private FetchSourceContext fetchSourceContext;
 
@@ -61,7 +63,7 @@ public class GetRequest extends SingleShardRequest<GetRequest> implements Realti
 
     private VersionType versionType = VersionType.INTERNAL;
     private long version = Versions.MATCH_ANY;
-    private boolean ignoreErrorsOnGeneratedFields;
+    private static DeprecationLogger deprecationLogger = new DeprecationLogger(Loggers.getLogger(GetRequest.class));
 
     public GetRequest() {
         type = "_all";
@@ -102,6 +104,9 @@ public ActionRequestValidationException validate() {
             validationException = ValidateActions.addValidationError("illegal version value [" + version + "] for version type [" + versionType.name() + "]",
                     validationException);
         }
+        if (versionType == VersionType.FORCE) {
+            deprecationLogger.deprecated("version type FORCE is deprecated and will be removed in the next major version");
+        }
         return validationException;
     }
 
@@ -187,20 +192,20 @@ public FetchSourceContext fetchSourceContext() {
     }
 
     /**
-     * Explicitly specify the fields that will be returned. By default, the <tt>_source</tt>
+     * Explicitly specify the stored fields that will be returned. By default, the <tt>_source</tt>
      * field will be returned.
      */
-    public GetRequest fields(String... fields) {
-        this.fields = fields;
+    public GetRequest storedFields(String... fields) {
+        this.storedFields = fields;
         return this;
     }
 
     /**
-     * Explicitly specify the fields that will be returned. By default, the <tt>_source</tt>
+     * Explicitly specify the stored fields that will be returned. By default, the <tt>_source</tt>
      * field will be returned.
      */
-    public String[] fields() {
-        return this.fields;
+    public String[] storedFields() {
+        return this.storedFields;
     }
 
     /**
@@ -248,19 +253,10 @@ public GetRequest versionType(VersionType versionType) {
         return this;
     }
 
-    public GetRequest ignoreErrorsOnGeneratedFields(boolean ignoreErrorsOnGeneratedFields) {
-        this.ignoreErrorsOnGeneratedFields = ignoreErrorsOnGeneratedFields;
-        return this;
-    }
-
     public VersionType versionType() {
         return this.versionType;
     }
 
-    public boolean ignoreErrorsOnGeneratedFields() {
-        return ignoreErrorsOnGeneratedFields;
-    }
-
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
@@ -270,19 +266,12 @@ public void readFrom(StreamInput in) throws IOException {
         parent = in.readOptionalString();
         preference = in.readOptionalString();
         refresh = in.readBoolean();
-        int size = in.readInt();
-        if (size >= 0) {
-            fields = new String[size];
-            for (int i = 0; i < size; i++) {
-                fields[i] = in.readString();
-            }
-        }
+        storedFields = in.readOptionalStringArray();
         realtime = in.readBoolean();
-        this.ignoreErrorsOnGeneratedFields = in.readBoolean();
 
         this.versionType = VersionType.fromValue(in.readByte());
         this.version = in.readLong();
-        fetchSourceContext = in.readOptionalStreamable(FetchSourceContext::new);
+        fetchSourceContext = in.readOptionalWriteable(FetchSourceContext::new);
     }
 
     @Override
@@ -295,19 +284,11 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalString(preference);
 
         out.writeBoolean(refresh);
-        if (fields == null) {
-            out.writeInt(-1);
-        } else {
-            out.writeInt(fields.length);
-            for (String field : fields) {
-                out.writeString(field);
-            }
-        }
+        out.writeOptionalStringArray(storedFields);
         out.writeBoolean(realtime);
-        out.writeBoolean(ignoreErrorsOnGeneratedFields);
         out.writeByte(versionType.getValue());
         out.writeLong(version);
-        out.writeOptionalStreamable(fetchSourceContext);
+        out.writeOptionalWriteable(fetchSourceContext);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/action/get/GetRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/get/GetRequestBuilder.java
index 7827de12eac30..973b130bedbd2 100644
--- a/core/src/main/java/org/elasticsearch/action/get/GetRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/get/GetRequestBuilder.java
@@ -88,8 +88,8 @@ public GetRequestBuilder setPreference(String preference) {
      * Explicitly specify the fields that will be returned. By default, the <tt>_source</tt>
      * field will be returned.
      */
-    public GetRequestBuilder setFields(String... fields) {
-        request.fields(fields);
+    public GetRequestBuilder setStoredFields(String... fields) {
+        request.storedFields(fields);
         return this;
     }
 
@@ -99,12 +99,8 @@ public GetRequestBuilder setFields(String... fields) {
      * @return this for chaining
      */
     public GetRequestBuilder setFetchSource(boolean fetch) {
-        FetchSourceContext context = request.fetchSourceContext();
-        if (context == null) {
-            request.fetchSourceContext(new FetchSourceContext(fetch));
-        } else {
-            context.fetchSource(fetch);
-        }
+        FetchSourceContext context = request.fetchSourceContext() == null ? FetchSourceContext.FETCH_SOURCE : request.fetchSourceContext();
+        request.fetchSourceContext(new FetchSourceContext(fetch, context.includes(), context.excludes()));
         return this;
     }
 
@@ -129,14 +125,8 @@ public GetRequestBuilder setFetchSource(@Nullable String include, @Nullable Stri
      * @param excludes An optional list of exclude (optionally wildcarded) pattern to filter the returned _source
      */
     public GetRequestBuilder setFetchSource(@Nullable String[] includes, @Nullable String[] excludes) {
-        FetchSourceContext context = request.fetchSourceContext();
-        if (context == null) {
-            request.fetchSourceContext(new FetchSourceContext(includes, excludes));
-        } else {
-            context.fetchSource(true);
-            context.includes(includes);
-            context.excludes(excludes);
-        }
+        FetchSourceContext context = request.fetchSourceContext() == null ? FetchSourceContext.FETCH_SOURCE : request.fetchSourceContext();
+        request.fetchSourceContext(new FetchSourceContext(context.fetchSource(), includes, excludes));
         return this;
     }
 
@@ -155,11 +145,6 @@ public GetRequestBuilder setRealtime(boolean realtime) {
         return this;
     }
 
-    public GetRequestBuilder setIgnoreErrorsOnGeneratedFields(Boolean ignoreErrorsOnGeneratedFields) {
-        request.ignoreErrorsOnGeneratedFields(ignoreErrorsOnGeneratedFields);
-        return this;
-    }
-
     /**
      * Sets the version, which will cause the get operation to only be performed if a matching
      * version exists and no changes happened on the doc since then.
diff --git a/core/src/main/java/org/elasticsearch/action/get/GetResponse.java b/core/src/main/java/org/elasticsearch/action/get/GetResponse.java
index 5741984d35f39..156005fab243e 100644
--- a/core/src/main/java/org/elasticsearch/action/get/GetResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/get/GetResponse.java
@@ -21,18 +21,22 @@
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.get.GetField;
 import org.elasticsearch.index.get.GetResult;
 
 import java.io.IOException;
 import java.util.Iterator;
+import java.util.Locale;
 import java.util.Map;
+import java.util.Objects;
 
 /**
  * The response of a get action.
@@ -40,9 +44,9 @@
  * @see GetRequest
  * @see org.elasticsearch.client.Client#get(GetRequest)
  */
-public class GetResponse extends ActionResponse implements Iterable<GetField>, ToXContent {
+public class GetResponse extends ActionResponse implements Iterable<GetField>, ToXContentObject {
 
-    private GetResult getResult;
+    GetResult getResult;
 
     GetResponse() {
     }
@@ -142,6 +146,10 @@ public GetField getField(String name) {
         return getResult.field(name);
     }
 
+    /**
+     * @deprecated Use {@link GetResponse#getSource()} instead
+     */
+    @Deprecated
     @Override
     public Iterator<GetField> iterator() {
         return getResult.iterator();
@@ -152,10 +160,33 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         return getResult.toXContent(builder, params);
     }
 
-    public static GetResponse readGetResponse(StreamInput in) throws IOException {
-        GetResponse result = new GetResponse();
-        result.readFrom(in);
-        return result;
+    /**
+     * This method can be used to parse a {@link GetResponse} object when it has been printed out
+     * as a xcontent using the {@link #toXContent(XContentBuilder, Params)} method.
+     * <p>
+     * For forward compatibility reason this method might not fail if it tries to parse a field it
+     * doesn't know. But before returning the result it will check that enough information were
+     * parsed to return a valid {@link GetResponse} instance and throws a {@link ParsingException}
+     * otherwise. This is the case when we get a 404 back, which can be parsed as a normal
+     * {@link GetResponse} with found set to false, or as an elasticsearch exception. The caller
+     * of this method needs a way to figure out whether we got back a valid get response, which
+     * can be done by catching ParsingException.
+     *
+     * @param parser {@link XContentParser} to parse the response from
+     * @return a {@link GetResponse}
+     * @throws IOException is an I/O exception occurs during the parsing
+     */
+    public static GetResponse fromXContent(XContentParser parser) throws IOException {
+        GetResult getResult = GetResult.fromXContent(parser);
+
+        // At this stage we ensure that we parsed enough information to return
+        // a valid GetResponse instance. If it's not the case, we throw an
+        // exception so that callers know it and can handle it correctly.
+        if (getResult.getIndex() == null && getResult.getType() == null && getResult.getId() == null) {
+            throw new ParsingException(parser.getTokenLocation(),
+                    String.format(Locale.ROOT, "Missing required fields [%s,%s,%s]", GetResult._INDEX, GetResult._TYPE, GetResult._ID));
+        }
+        return new GetResponse(getResult);
     }
 
     @Override
@@ -170,8 +201,25 @@ public void writeTo(StreamOutput out) throws IOException {
         getResult.writeTo(out);
     }
 
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        GetResponse getResponse = (GetResponse) o;
+        return Objects.equals(getResult, getResponse.getResult);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(getResult);
+    }
+
     @Override
     public String toString() {
-        return Strings.toString(this, true);
+        return Strings.toString(this);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java b/core/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java
index 001e4ebd7a031..dc007cf141b18 100644
--- a/core/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java
@@ -28,14 +28,12 @@
 import org.elasticsearch.action.ValidateActions;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
 import org.elasticsearch.common.lucene.uid.Versions;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
@@ -46,8 +44,9 @@
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Locale;
 
-public class MultiGetRequest extends ActionRequest<MultiGetRequest> implements Iterable<MultiGetRequest.Item>, CompositeIndicesRequest, RealtimeRequest {
+public class MultiGetRequest extends ActionRequest implements Iterable<MultiGetRequest.Item>, CompositeIndicesRequest, RealtimeRequest {
 
     /**
      * A single get item.
@@ -58,7 +57,7 @@ public static class Item implements Streamable, IndicesRequest {
         private String id;
         private String routing;
         private String parent;
-        private String[] fields;
+        private String[] storedFields;
         private long version = Versions.MATCH_ANY;
         private VersionType versionType = VersionType.INTERNAL;
         private FetchSourceContext fetchSourceContext;
@@ -136,13 +135,13 @@ public String parent() {
             return parent;
         }
 
-        public Item fields(String... fields) {
-            this.fields = fields;
+        public Item storedFields(String... fields) {
+            this.storedFields = fields;
             return this;
         }
 
-        public String[] fields() {
-            return this.fields;
+        public String[] storedFields() {
+            return this.storedFields;
         }
 
         public long version() {
@@ -188,17 +187,11 @@ public void readFrom(StreamInput in) throws IOException {
             id = in.readString();
             routing = in.readOptionalString();
             parent = in.readOptionalString();
-            int size = in.readVInt();
-            if (size > 0) {
-                fields = new String[size];
-                for (int i = 0; i < size; i++) {
-                    fields[i] = in.readString();
-                }
-            }
+            storedFields = in.readOptionalStringArray();
             version = in.readLong();
             versionType = VersionType.fromValue(in.readByte());
 
-            fetchSourceContext = in.readOptionalStreamable(FetchSourceContext::new);
+            fetchSourceContext = in.readOptionalWriteable(FetchSourceContext::new);
         }
 
         @Override
@@ -208,19 +201,11 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeString(id);
             out.writeOptionalString(routing);
             out.writeOptionalString(parent);
-            if (fields == null) {
-                out.writeVInt(0);
-            } else {
-                out.writeVInt(fields.length);
-                for (String field : fields) {
-                    out.writeString(field);
-                }
-            }
-
+            out.writeOptionalStringArray(storedFields);
             out.writeLong(version);
             out.writeByte(versionType.getValue());
 
-            out.writeOptionalStreamable(fetchSourceContext);
+            out.writeOptionalWriteable(fetchSourceContext);
         }
 
         @Override
@@ -233,7 +218,7 @@ public boolean equals(Object o) {
             if (version != item.version) return false;
             if (fetchSourceContext != null ? !fetchSourceContext.equals(item.fetchSourceContext) : item.fetchSourceContext != null)
                 return false;
-            if (!Arrays.equals(fields, item.fields)) return false;
+            if (!Arrays.equals(storedFields, item.storedFields)) return false;
             if (!id.equals(item.id)) return false;
             if (!index.equals(item.index)) return false;
             if (routing != null ? !routing.equals(item.routing) : item.routing != null) return false;
@@ -251,7 +236,7 @@ public int hashCode() {
             result = 31 * result + id.hashCode();
             result = 31 * result + (routing != null ? routing.hashCode() : 0);
             result = 31 * result + (parent != null ? parent.hashCode() : 0);
-            result = 31 * result + (fields != null ? Arrays.hashCode(fields) : 0);
+            result = 31 * result + (storedFields != null ? Arrays.hashCode(storedFields) : 0);
             result = 31 * result + Long.hashCode(version);
             result = 31 * result + versionType.hashCode();
             result = 31 * result + (fetchSourceContext != null ? fetchSourceContext.hashCode() : 0);
@@ -262,8 +247,6 @@ public int hashCode() {
     String preference;
     boolean realtime = true;
     boolean refresh;
-    public boolean ignoreErrorsOnGeneratedFields = false;
-
     List<Item> items = new ArrayList<>();
 
     public List<Item> getItems() {
@@ -299,11 +282,6 @@ public ActionRequestValidationException validate() {
         return validationException;
     }
 
-    @Override
-    public List<? extends IndicesRequest> subRequests() {
-        return items;
-    }
-
     /**
      * Sets the preference to execute the search. Defaults to randomize across shards. Can be set to
      * <tt>_local</tt> to prefer local shards, <tt>_primary</tt> to execute only on primary shards, or
@@ -337,38 +315,43 @@ public MultiGetRequest refresh(boolean refresh) {
         return this;
     }
 
-
-    public MultiGetRequest ignoreErrorsOnGeneratedFields(boolean ignoreErrorsOnGeneratedFields) {
-        this.ignoreErrorsOnGeneratedFields = ignoreErrorsOnGeneratedFields;
-        return this;
-    }
-
-    public MultiGetRequest add(@Nullable String defaultIndex, @Nullable String defaultType, @Nullable String[] defaultFields, @Nullable FetchSourceContext defaultFetchSource, byte[] data, int from, int length) throws Exception {
-        return add(defaultIndex, defaultType, defaultFields, defaultFetchSource, new BytesArray(data, from, length), true);
-    }
-
-    public MultiGetRequest add(@Nullable String defaultIndex, @Nullable String defaultType, @Nullable String[] defaultFields, @Nullable FetchSourceContext defaultFetchSource, BytesReference data) throws Exception {
-        return add(defaultIndex, defaultType, defaultFields, defaultFetchSource, data, true);
-    }
-
-    public MultiGetRequest add(@Nullable String defaultIndex, @Nullable String defaultType, @Nullable String[] defaultFields, @Nullable FetchSourceContext defaultFetchSource, BytesReference data, boolean allowExplicitIndex) throws Exception {
-        return add(defaultIndex, defaultType, defaultFields, defaultFetchSource, null, data, allowExplicitIndex);
-    }
-
-    public MultiGetRequest add(@Nullable String defaultIndex, @Nullable String defaultType, @Nullable String[] defaultFields, @Nullable FetchSourceContext defaultFetchSource, @Nullable String defaultRouting, BytesReference data, boolean allowExplicitIndex) throws Exception {
-        try (XContentParser parser = XContentFactory.xContent(data).createParser(data)) {
-            XContentParser.Token token;
-            String currentFieldName = null;
-            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                if (token == XContentParser.Token.FIELD_NAME) {
-                    currentFieldName = parser.currentName();
-                } else if (token == XContentParser.Token.START_ARRAY) {
-                    if ("docs".equals(currentFieldName)) {
-                        parseDocuments(parser, this.items, defaultIndex, defaultType, defaultFields, defaultFetchSource, defaultRouting, allowExplicitIndex);
-                    } else if ("ids".equals(currentFieldName)) {
-                        parseIds(parser, this.items, defaultIndex, defaultType, defaultFields, defaultFetchSource, defaultRouting);
-                    }
+    public MultiGetRequest add(@Nullable String defaultIndex, @Nullable String defaultType, @Nullable String[] defaultFields,
+            @Nullable FetchSourceContext defaultFetchSource, @Nullable String defaultRouting, XContentParser parser,
+            boolean allowExplicitIndex) throws IOException {
+        XContentParser.Token token;
+        String currentFieldName = null;
+        if ((token = parser.nextToken()) != XContentParser.Token.START_OBJECT) {
+            final String message = String.format(
+                    Locale.ROOT,
+                    "unexpected token [%s], expected [%s]",
+                    token,
+                    XContentParser.Token.START_OBJECT);
+            throw new ParsingException(parser.getTokenLocation(), message);
+        }
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token == XContentParser.Token.START_ARRAY) {
+                if ("docs".equals(currentFieldName)) {
+                    parseDocuments(parser, this.items, defaultIndex, defaultType, defaultFields, defaultFetchSource, defaultRouting, allowExplicitIndex);
+                } else if ("ids".equals(currentFieldName)) {
+                    parseIds(parser, this.items, defaultIndex, defaultType, defaultFields, defaultFetchSource, defaultRouting);
+                } else {
+                    final String message = String.format(
+                            Locale.ROOT,
+                            "unknown key [%s] for a %s, expected [docs] or [ids]",
+                            currentFieldName,
+                            token);
+                    throw new ParsingException(parser.getTokenLocation(), message);
                 }
+            } else {
+                final String message = String.format(
+                        Locale.ROOT,
+                        "unexpected token [%s], expected [%s] or [%s]",
+                        token,
+                        XContentParser.Token.FIELD_NAME,
+                        XContentParser.Token.START_ARRAY);
+                throw new ParsingException(parser.getTokenLocation(), message);
             }
         }
         return this;
@@ -386,11 +369,11 @@ public static void parseDocuments(XContentParser parser, List<Item> items, @Null
             String id = null;
             String routing = defaultRouting;
             String parent = null;
-            List<String> fields = null;
+            List<String> storedFields = null;
             long version = Versions.MATCH_ANY;
             VersionType versionType = VersionType.INTERNAL;
 
-            FetchSourceContext fetchSourceContext = null;
+            FetchSourceContext fetchSourceContext = FetchSourceContext.FETCH_SOURCE;
 
             while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
                 if (token == XContentParser.Token.FIELD_NAME) {
@@ -410,33 +393,42 @@ public static void parseDocuments(XContentParser parser, List<Item> items, @Null
                     } else if ("_parent".equals(currentFieldName) || "parent".equals(currentFieldName)) {
                         parent = parser.text();
                     } else if ("fields".equals(currentFieldName)) {
-                        fields = new ArrayList<>();
-                        fields.add(parser.text());
+                        throw new ParsingException(parser.getTokenLocation(),
+                            "Unsupported field [fields] used, expected [stored_fields] instead");
+                    } else if ("stored_fields".equals(currentFieldName)) {
+                        storedFields = new ArrayList<>();
+                        storedFields.add(parser.text());
                     } else if ("_version".equals(currentFieldName) || "version".equals(currentFieldName)) {
                         version = parser.longValue();
                     } else if ("_version_type".equals(currentFieldName) || "_versionType".equals(currentFieldName) || "version_type".equals(currentFieldName) || "versionType".equals(currentFieldName)) {
                         versionType = VersionType.fromString(parser.text());
                     } else if ("_source".equals(currentFieldName)) {
                         if (parser.isBooleanValue()) {
-                            fetchSourceContext = new FetchSourceContext(parser.booleanValue());
+                            fetchSourceContext = new FetchSourceContext(parser.booleanValue(), fetchSourceContext.includes(),
+                                fetchSourceContext.excludes());
                         } else if (token == XContentParser.Token.VALUE_STRING) {
-                            fetchSourceContext = new FetchSourceContext(new String[]{parser.text()});
+                            fetchSourceContext = new FetchSourceContext(fetchSourceContext.fetchSource(),
+                                new String[]{parser.text()}, fetchSourceContext.excludes());
                         } else {
                             throw new ElasticsearchParseException("illegal type for _source: [{}]", token);
                         }
                     }
                 } else if (token == XContentParser.Token.START_ARRAY) {
                     if ("fields".equals(currentFieldName)) {
-                        fields = new ArrayList<>();
+                        throw new ParsingException(parser.getTokenLocation(),
+                            "Unsupported field [fields] used, expected [stored_fields] instead");
+                    } else if ("stored_fields".equals(currentFieldName)) {
+                        storedFields = new ArrayList<>();
                         while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
-                            fields.add(parser.text());
+                            storedFields.add(parser.text());
                         }
                     } else if ("_source".equals(currentFieldName)) {
                         ArrayList<String> includes = new ArrayList<>();
                         while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                             includes.add(parser.text());
                         }
-                        fetchSourceContext = new FetchSourceContext(includes.toArray(Strings.EMPTY_ARRAY));
+                        fetchSourceContext = new FetchSourceContext(fetchSourceContext.fetchSource(), includes.toArray(Strings.EMPTY_ARRAY)
+                            , fetchSourceContext.excludes());
                     }
 
                 } else if (token == XContentParser.Token.START_OBJECT) {
@@ -464,20 +456,20 @@ public static void parseDocuments(XContentParser parser, List<Item> items, @Null
                             }
                         }
 
-                        fetchSourceContext = new FetchSourceContext(
+                        fetchSourceContext = new FetchSourceContext(fetchSourceContext.fetchSource(),
                                 includes == null ? Strings.EMPTY_ARRAY : includes.toArray(new String[includes.size()]),
                                 excludes == null ? Strings.EMPTY_ARRAY : excludes.toArray(new String[excludes.size()]));
                     }
                 }
             }
             String[] aFields;
-            if (fields != null) {
-                aFields = fields.toArray(new String[fields.size()]);
+            if (storedFields != null) {
+                aFields = storedFields.toArray(new String[storedFields.size()]);
             } else {
                 aFields = defaultFields;
             }
-            items.add(new Item(index, type, id).routing(routing).fields(aFields).parent(parent).version(version).versionType(versionType)
-                    .fetchSourceContext(fetchSourceContext == null ? defaultFetchSource : fetchSourceContext));
+            items.add(new Item(index, type, id).routing(routing).storedFields(aFields).parent(parent).version(version).versionType(versionType)
+                    .fetchSourceContext(fetchSourceContext == FetchSourceContext.FETCH_SOURCE ? defaultFetchSource : fetchSourceContext));
         }
     }
 
@@ -491,7 +483,7 @@ public static void parseIds(XContentParser parser, List<Item> items, @Nullable S
             if (!token.isValue()) {
                 throw new IllegalArgumentException("ids array element should only contain ids");
             }
-            items.add(new Item(defaultIndex, defaultType, parser.text()).fields(defaultFields).fetchSourceContext(defaultFetchSource).routing(defaultRouting));
+            items.add(new Item(defaultIndex, defaultType, parser.text()).storedFields(defaultFields).fetchSourceContext(defaultFetchSource).routing(defaultRouting));
         }
     }
 
@@ -510,7 +502,6 @@ public void readFrom(StreamInput in) throws IOException {
         preference = in.readOptionalString();
         refresh = in.readBoolean();
         realtime = in.readBoolean();
-        ignoreErrorsOnGeneratedFields = in.readBoolean();
 
         int size = in.readVInt();
         items = new ArrayList<>(size);
@@ -525,7 +516,6 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalString(preference);
         out.writeBoolean(refresh);
         out.writeBoolean(realtime);
-        out.writeBoolean(ignoreErrorsOnGeneratedFields);
 
         out.writeVInt(items.size());
         for (Item item : items) {
diff --git a/core/src/main/java/org/elasticsearch/action/get/MultiGetRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/get/MultiGetRequestBuilder.java
index 6e32e1caf3099..a2cb204d5eabf 100644
--- a/core/src/main/java/org/elasticsearch/action/get/MultiGetRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/get/MultiGetRequestBuilder.java
@@ -80,9 +80,4 @@ public MultiGetRequestBuilder setRealtime(boolean realtime) {
         request.realtime(realtime);
         return this;
     }
-
-    public MultiGetRequestBuilder setIgnoreErrorsOnGeneratedFields(boolean ignoreErrorsOnGeneratedFields) {
-        request.ignoreErrorsOnGeneratedFields(ignoreErrorsOnGeneratedFields);
-        return this;
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/get/MultiGetResponse.java b/core/src/main/java/org/elasticsearch/action/get/MultiGetResponse.java
index e1fe435fd107d..93e4272bd956c 100644
--- a/core/src/main/java/org/elasticsearch/action/get/MultiGetResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/get/MultiGetResponse.java
@@ -24,14 +24,14 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Iterator;
 
-public class MultiGetResponse extends ActionResponse implements Iterable<MultiGetItemResponse>, ToXContent {
+public class MultiGetResponse extends ActionResponse implements Iterable<MultiGetItemResponse>, ToXContentObject {
 
     /**
      * Represents a failure.
@@ -128,6 +128,7 @@ public Iterator<MultiGetItemResponse> iterator() {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
         builder.startArray(Fields.DOCS);
         for (MultiGetItemResponse response : responses) {
             if (response.isFailed()) {
@@ -136,16 +137,15 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                 builder.field(Fields._INDEX, failure.getIndex());
                 builder.field(Fields._TYPE, failure.getType());
                 builder.field(Fields._ID, failure.getId());
-                ElasticsearchException.renderException(builder, params, failure.getFailure());
+                ElasticsearchException.generateFailureXContent(builder, params, failure.getFailure(), true);
                 builder.endObject();
             } else {
                 GetResponse getResponse = response.getResponse();
-                builder.startObject();
                 getResponse.toXContent(builder, params);
-                builder.endObject();
             }
         }
         builder.endArray();
+        builder.endObject();
         return builder;
     }
 
@@ -154,9 +154,6 @@ static final class Fields {
         static final String _INDEX = "_index";
         static final String _TYPE = "_type";
         static final String _ID = "_id";
-        static final String ERROR = "error";
-        static final String ROOT_CAUSE = "root_cause";
-
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/action/get/MultiGetShardRequest.java b/core/src/main/java/org/elasticsearch/action/get/MultiGetShardRequest.java
index 47f07c524888a..25a624b2eb558 100644
--- a/core/src/main/java/org/elasticsearch/action/get/MultiGetShardRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/get/MultiGetShardRequest.java
@@ -35,7 +35,6 @@ public class MultiGetShardRequest extends SingleShardRequest<MultiGetShardReques
     private String preference;
     boolean realtime = true;
     boolean refresh;
-    boolean ignoreErrorsOnGeneratedFields = false;
 
     IntArrayList locations;
     List<MultiGetRequest.Item> items;
@@ -52,7 +51,6 @@ public MultiGetShardRequest() {
         preference = multiGetRequest.preference;
         realtime = multiGetRequest.realtime;
         refresh = multiGetRequest.refresh;
-        ignoreErrorsOnGeneratedFields = multiGetRequest.ignoreErrorsOnGeneratedFields;
     }
 
     @Override
@@ -87,11 +85,6 @@ public MultiGetShardRequest realtime(boolean realtime) {
         return this;
     }
 
-    public MultiGetShardRequest ignoreErrorsOnGeneratedFields(Boolean ignoreErrorsOnGeneratedFields) {
-        this.ignoreErrorsOnGeneratedFields = ignoreErrorsOnGeneratedFields;
-        return this;
-    }
-
     public boolean refresh() {
         return this.refresh;
     }
@@ -130,7 +123,6 @@ public void readFrom(StreamInput in) throws IOException {
         preference = in.readOptionalString();
         refresh = in.readBoolean();
         realtime = in.readBoolean();
-        ignoreErrorsOnGeneratedFields = in.readBoolean();
     }
 
     @Override
@@ -146,11 +138,5 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalString(preference);
         out.writeBoolean(refresh);
         out.writeBoolean(realtime);
-        out.writeBoolean(ignoreErrorsOnGeneratedFields);
-
-    }
-
-    public boolean ignoreErrorsOnGeneratedFields() {
-        return ignoreErrorsOnGeneratedFields;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/get/TransportGetAction.java b/core/src/main/java/org/elasticsearch/action/get/TransportGetAction.java
index 240035aee2aeb..6b9de7ecf64e3 100644
--- a/core/src/main/java/org/elasticsearch/action/get/TransportGetAction.java
+++ b/core/src/main/java/org/elasticsearch/action/get/TransportGetAction.java
@@ -92,8 +92,8 @@ protected GetResponse shardOperation(GetRequest request, ShardId shardId) {
             indexShard.refresh("refresh_flag_get");
         }
 
-        GetResult result = indexShard.getService().get(request.type(), request.id(), request.fields(),
-                request.realtime(), request.version(), request.versionType(), request.fetchSourceContext(), request.ignoreErrorsOnGeneratedFields());
+        GetResult result = indexShard.getService().get(request.type(), request.id(), request.storedFields(),
+                request.realtime(), request.version(), request.versionType(), request.fetchSourceContext());
         return new GetResponse(result);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java b/core/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java
index 4e62030d3298e..bea65283cc034 100644
--- a/core/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java
+++ b/core/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java
@@ -47,8 +47,8 @@ public class TransportMultiGetAction extends HandledTransportAction<MultiGetRequ
     @Inject
     public TransportMultiGetAction(Settings settings, ThreadPool threadPool, TransportService transportService,
                                    ClusterService clusterService, TransportShardMultiGetAction shardAction,
-                                   ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
-        super(settings, MultiGetAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, MultiGetRequest::new);
+                                   ActionFilters actionFilters, IndexNameExpressionResolver resolver) {
+        super(settings, MultiGetAction.NAME, threadPool, transportService, actionFilters, resolver, MultiGetRequest::new);
         this.clusterService = clusterService;
         this.shardAction = shardAction;
     }
@@ -56,36 +56,42 @@ public TransportMultiGetAction(Settings settings, ThreadPool threadPool, Transpo
     @Override
     protected void doExecute(final MultiGetRequest request, final ActionListener<MultiGetResponse> listener) {
         ClusterState clusterState = clusterService.state();
-
         clusterState.blocks().globalBlockedRaiseException(ClusterBlockLevel.READ);
 
         final AtomicArray<MultiGetItemResponse> responses = new AtomicArray<>(request.items.size());
+        final Map<ShardId, MultiGetShardRequest> shardRequests = new HashMap<>();
 
-        Map<ShardId, MultiGetShardRequest> shardRequests = new HashMap<>();
         for (int i = 0; i < request.items.size(); i++) {
             MultiGetRequest.Item item = request.items.get(i);
-            if (!clusterState.metaData().hasConcreteIndex(item.index())) {
-                responses.set(i, new MultiGetItemResponse(null, new MultiGetResponse.Failure(item.index(), item.type(), item.id(), new IndexNotFoundException(item.index()))));
-                continue;
-            }
-            item.routing(clusterState.metaData().resolveIndexRouting(item.parent(), item.routing(), item.index()));
-            String concreteSingleIndex = indexNameExpressionResolver.concreteSingleIndex(clusterState, item).getName();
-            if (item.routing() == null && clusterState.getMetaData().routingRequired(concreteSingleIndex, item.type())) {
-                responses.set(i, new MultiGetItemResponse(null, new MultiGetResponse.Failure(concreteSingleIndex, item.type(), item.id(),
-                        new IllegalArgumentException("routing is required for [" + concreteSingleIndex + "]/[" + item.type() + "]/[" + item.id() + "]"))));
+
+            String concreteSingleIndex;
+            try {
+                concreteSingleIndex = indexNameExpressionResolver.concreteSingleIndex(clusterState, item).getName();
+
+                item.routing(clusterState.metaData().resolveIndexRouting(item.parent(), item.routing(), item.index()));
+                if ((item.routing() == null) && (clusterState.getMetaData().routingRequired(concreteSingleIndex, item.type()))) {
+                    String message = "routing is required for [" + concreteSingleIndex + "]/[" + item.type() + "]/[" + item.id() + "]";
+                    responses.set(i, newItemFailure(concreteSingleIndex, item.type(), item.id(), new IllegalArgumentException(message)));
+                    continue;
+                }
+            } catch (Exception e) {
+                responses.set(i, newItemFailure(item.index(), item.type(), item.id(), e));
                 continue;
             }
+
             ShardId shardId = clusterService.operationRouting()
-                    .getShards(clusterState, concreteSingleIndex, item.id(), item.routing(), null).shardId();
+                    .getShards(clusterState, concreteSingleIndex, item.id(), item.routing(), null)
+                    .shardId();
+
             MultiGetShardRequest shardRequest = shardRequests.get(shardId);
             if (shardRequest == null) {
-                shardRequest = new MultiGetShardRequest(request, shardId.getIndexName(), shardId.id());
+                shardRequest = new MultiGetShardRequest(request, shardId.getIndexName(), shardId.getId());
                 shardRequests.put(shardId, shardRequest);
             }
             shardRequest.add(i, item);
         }
 
-        if (shardRequests.size() == 0) {
+        if (shardRequests.isEmpty()) {
             // only failures..
             listener.onResponse(new MultiGetResponse(responses.toArray(new MultiGetItemResponse[responses.length()])));
         }
@@ -97,7 +103,8 @@ protected void doExecute(final MultiGetRequest request, final ActionListener<Mul
                 @Override
                 public void onResponse(MultiGetShardResponse response) {
                     for (int i = 0; i < response.locations.size(); i++) {
-                        responses.set(response.locations.get(i), new MultiGetItemResponse(response.responses.get(i), response.failures.get(i)));
+                        MultiGetItemResponse itemResponse = new MultiGetItemResponse(response.responses.get(i), response.failures.get(i));
+                        responses.set(response.locations.get(i), itemResponse);
                     }
                     if (counter.decrementAndGet() == 0) {
                         finishHim();
@@ -109,8 +116,7 @@ public void onFailure(Exception e) {
                     // create failures for all relevant requests
                     for (int i = 0; i < shardRequest.locations.size(); i++) {
                         MultiGetRequest.Item item = shardRequest.items.get(i);
-                        responses.set(shardRequest.locations.get(i), new MultiGetItemResponse(null,
-                                new MultiGetResponse.Failure(shardRequest.index(), item.type(), item.id(), e)));
+                        responses.set(shardRequest.locations.get(i), newItemFailure(shardRequest.index(), item.type(), item.id(), e));
                     }
                     if (counter.decrementAndGet() == 0) {
                         finishHim();
@@ -123,4 +129,8 @@ private void finishHim() {
             });
         }
     }
+
+    private static MultiGetItemResponse newItemFailure(String index, String type, String id, Exception exception) {
+        return new MultiGetItemResponse(null, new MultiGetResponse.Failure(index, type, id, exception));
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java b/core/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java
index ec3d182de49c2..8353c5dc389d9 100644
--- a/core/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java
+++ b/core/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java
@@ -88,13 +88,15 @@ protected MultiGetShardResponse shardOperation(MultiGetShardRequest request, Sha
         for (int i = 0; i < request.locations.size(); i++) {
             MultiGetRequest.Item item = request.items.get(i);
             try {
-                GetResult getResult = indexShard.getService().get(item.type(), item.id(), item.fields(), request.realtime(), item.version(), item.versionType(), item.fetchSourceContext(), request.ignoreErrorsOnGeneratedFields());
+                GetResult getResult = indexShard.getService().get(item.type(), item.id(), item.storedFields(), request.realtime(), item.version(),
+                    item.versionType(), item.fetchSourceContext());
                 response.add(request.locations.get(i), new GetResponse(getResult));
             } catch (Exception e) {
                 if (TransportActions.isShardNotAvailableException(e)) {
                     throw (ElasticsearchException) e;
                 } else {
-                    logger.debug((Supplier<?>) () -> new ParameterizedMessage("{} failed to execute multi_get for [{}]/[{}]", shardId, item.type(), item.id()), e);
+                    logger.debug((Supplier<?>) () -> new ParameterizedMessage("{} failed to execute multi_get for [{}]/[{}]", shardId,
+                        item.type(), item.id()), e);
                     response.add(request.locations.get(i), new MultiGetResponse.Failure(request.index(), item.type(), item.id(), e));
                 }
             }
diff --git a/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java
index 1f6bc9108c22c..11dda29637857 100644
--- a/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java
@@ -20,11 +20,14 @@
 package org.elasticsearch.action.index;
 
 import org.elasticsearch.ElasticsearchGenerationException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequestValidationException;
-import org.elasticsearch.action.DocumentRequest;
+import org.elasticsearch.action.CompositeIndicesRequest;
+import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.RoutingMissingException;
 import org.elasticsearch.action.TimestampParsingException;
 import org.elasticsearch.action.support.replication.ReplicatedWriteRequest;
+import org.elasticsearch.action.support.replication.ReplicationRequest;
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
@@ -34,7 +37,10 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.lucene.uid.Versions;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -47,6 +53,7 @@
 import java.nio.charset.StandardCharsets;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Objects;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 
@@ -55,10 +62,10 @@
  * created using {@link org.elasticsearch.client.Requests#indexRequest(String)}.
  *
  * The index requires the {@link #index()}, {@link #type(String)}, {@link #id(String)} and
- * {@link #source(byte[])} to be set.
+ * {@link #source(byte[], XContentType)} to be set.
  *
- * The source (content to index) can be set in its bytes form using ({@link #source(byte[])}),
- * its string form ({@link #source(String)}) or using a {@link org.elasticsearch.common.xcontent.XContentBuilder}
+ * The source (content to index) can be set in its bytes form using ({@link #source(byte[], XContentType)}),
+ * its string form ({@link #source(String, XContentType)}) or using a {@link org.elasticsearch.common.xcontent.XContentBuilder}
  * ({@link #source(org.elasticsearch.common.xcontent.XContentBuilder)}).
  *
  * If the {@link #id(String)} is not set, it will be automatically generated.
@@ -67,68 +74,14 @@
  * @see org.elasticsearch.client.Requests#indexRequest(String)
  * @see org.elasticsearch.client.Client#index(IndexRequest)
  */
-public class IndexRequest extends ReplicatedWriteRequest<IndexRequest> implements DocumentRequest<IndexRequest> {
+public class IndexRequest extends ReplicatedWriteRequest<IndexRequest> implements DocWriteRequest<IndexRequest>, CompositeIndicesRequest {
 
     /**
-     * Operation type controls if the type of the index operation.
+     * Max length of the source document to include into toString()
+     *
+     * @see ReplicationRequest#createTask(long, java.lang.String, java.lang.String, org.elasticsearch.tasks.TaskId)
      */
-    public enum OpType {
-        /**
-         * Index the source. If there an existing document with the id, it will
-         * be replaced.
-         */
-        INDEX((byte) 0),
-        /**
-         * Creates the resource. Simply adds it to the index, if there is an existing
-         * document with the id, then it won't be removed.
-         */
-        CREATE((byte) 1);
-
-        private final byte id;
-        private final String lowercase;
-
-        OpType(byte id) {
-            this.id = id;
-            this.lowercase = this.toString().toLowerCase(Locale.ENGLISH);
-        }
-
-        /**
-         * The internal representation of the operation type.
-         */
-        public byte id() {
-            return id;
-        }
-
-        public String lowercase() {
-            return this.lowercase;
-        }
-
-        /**
-         * Constructs the operation type from its internal representation.
-         */
-        public static OpType fromId(byte id) {
-            if (id == 0) {
-                return INDEX;
-            } else if (id == 1) {
-                return CREATE;
-            } else {
-                throw new IllegalArgumentException("No type match for [" + id + "]");
-            }
-        }
-
-        public static OpType fromString(String sOpType) {
-            String lowersOpType = sOpType.toLowerCase(Locale.ROOT);
-            switch (lowersOpType) {
-                case "create":
-                    return OpType.CREATE;
-                case "index":
-                    return OpType.INDEX;
-                default:
-                    throw new IllegalArgumentException("opType [" + sOpType + "] not allowed, either [index] or [create] are allowed");
-            }
-        }
-
-    }
+    static final int MAX_SOURCE_LENGTH_IN_TOSTRING = 2048;
 
     private String type;
     private String id;
@@ -148,7 +101,7 @@ public static OpType fromString(String sOpType) {
     private long version = Versions.MATCH_ANY;
     private VersionType versionType = VersionType.INTERNAL;
 
-    private XContentType contentType = Requests.INDEX_CONTENT_TYPE;
+    private XContentType contentType;
 
     private String pipeline;
 
@@ -161,6 +114,7 @@ public static OpType fromString(String sOpType) {
     private long autoGeneratedTimestamp = UNSET_AUTO_GENERATED_TIMESTAMP;
 
     private boolean isRetry = false;
+    private static DeprecationLogger deprecationLogger = new DeprecationLogger(Loggers.getLogger(IndexRequest.class));
 
 
     public IndexRequest() {
@@ -168,7 +122,7 @@ public IndexRequest() {
 
     /**
      * Constructs a new index request against the specific index. The {@link #type(String)}
-     * {@link #source(byte[])} must be set.
+     * {@link #source(byte[], XContentType)} must be set.
      */
     public IndexRequest(String index) {
         this.index = index;
@@ -205,10 +159,18 @@ public ActionRequestValidationException validate() {
         if (source == null) {
             validationException = addValidationError("source is missing", validationException);
         }
-
+        if (contentType == null) {
+            validationException = addValidationError("content type is missing", validationException);
+        }
+        final long resolvedVersion = resolveVersionDefaults();
         if (opType() == OpType.CREATE) {
-            if (versionType != VersionType.INTERNAL || version != Versions.MATCH_DELETED) {
-                validationException = addValidationError("create operations do not support versioning. use index instead", validationException);
+            if (versionType != VersionType.INTERNAL) {
+                validationException = addValidationError("create operations only support internal versioning. use index instead", validationException);
+                return validationException;
+            }
+
+            if (resolvedVersion != Versions.MATCH_DELETED) {
+                validationException = addValidationError("create operations do not support explicit versions. use index instead", validationException);
                 return validationException;
             }
         }
@@ -217,8 +179,8 @@ public ActionRequestValidationException validate() {
             addValidationError("an id is required for a " + opType() + " operation", validationException);
         }
 
-        if (!versionType.validateVersionForWrites(version)) {
-            validationException = addValidationError("illegal version value [" + version + "] for version type [" + versionType.name() + "]", validationException);
+        if (!versionType.validateVersionForWrites(resolvedVersion)) {
+            validationException = addValidationError("illegal version value [" + resolvedVersion + "] for version type [" + versionType.name() + "]", validationException);
         }
 
         if (ttl != null) {
@@ -232,28 +194,24 @@ public ActionRequestValidationException validate() {
                             id.getBytes(StandardCharsets.UTF_8).length, validationException);
         }
 
-        if (id == null && (versionType == VersionType.INTERNAL && version == Versions.MATCH_ANY) == false) {
+        if (id == null && (versionType == VersionType.INTERNAL && resolvedVersion == Versions.MATCH_ANY) == false) {
             validationException = addValidationError("an id must be provided if version type or value are set", validationException);
         }
+        if (versionType == VersionType.FORCE) {
+            deprecationLogger.deprecated("version type FORCE is deprecated and will be removed in the next major version");
+        }
 
         return validationException;
     }
 
     /**
-     * The content type that will be used when generating a document from user provided objects like Maps.
+     * The content type. This will be used when generating a document from user provided objects like Maps and when parsing the
+     * source at index time
      */
     public XContentType getContentType() {
         return contentType;
     }
 
-    /**
-     * Sets the content type that will be used when generating a document from user provided objects (like Map).
-     */
-    public IndexRequest contentType(XContentType contentType) {
-        this.contentType = contentType;
-        return this;
-    }
-
     /**
      * The type of the indexed document.
      */
@@ -325,11 +283,13 @@ public String parent() {
     /**
      * Sets the timestamp either as millis since the epoch, or, in the configured date format.
      */
+    @Deprecated
     public IndexRequest timestamp(String timestamp) {
         this.timestamp = timestamp;
         return this;
     }
 
+    @Deprecated
     public String timestamp() {
         return this.timestamp;
     }
@@ -337,6 +297,7 @@ public String timestamp() {
     /**
      * Sets the ttl value as a time value expression.
      */
+    @Deprecated
     public IndexRequest ttl(String ttl) {
         this.ttl = TimeValue.parseTimeValue(ttl, null, "ttl");
         return this;
@@ -345,6 +306,7 @@ public IndexRequest ttl(String ttl) {
     /**
      * Sets the ttl as a {@link TimeValue} instance.
      */
+    @Deprecated
     public IndexRequest ttl(TimeValue ttl) {
         this.ttl = ttl;
         return this;
@@ -353,6 +315,7 @@ public IndexRequest ttl(TimeValue ttl) {
     /**
      * Sets the relative ttl value in milliseconds. It musts be greater than 0 as it makes little sense otherwise.
      */
+    @Deprecated
     public IndexRequest ttl(long ttl) {
         this.ttl = new TimeValue(ttl);
         return this;
@@ -361,6 +324,7 @@ public IndexRequest ttl(long ttl) {
     /**
      * Returns the ttl as a {@link TimeValue}
      */
+    @Deprecated
     public TimeValue ttl() {
         return this.ttl;
     }
@@ -388,16 +352,16 @@ public BytesReference source() {
     }
 
     public Map<String, Object> sourceAsMap() {
-        return XContentHelper.convertToMap(source, false).v2();
+        return XContentHelper.convertToMap(source, false, contentType).v2();
     }
 
     /**
-     * Index the Map as a {@link org.elasticsearch.client.Requests#INDEX_CONTENT_TYPE}.
+     * Index the Map in {@link Requests#INDEX_CONTENT_TYPE} format
      *
      * @param source The map to index
      */
     public IndexRequest source(Map source) throws ElasticsearchGenerationException {
-        return source(source, contentType);
+        return source(source, Requests.INDEX_CONTENT_TYPE);
     }
 
     /**
@@ -418,63 +382,51 @@ public IndexRequest source(Map source, XContentType contentType) throws Elastics
     /**
      * Sets the document source to index.
      *
-     * Note, its preferable to either set it using {@link #source(org.elasticsearch.common.xcontent.XContentBuilder)}
-     * or using the {@link #source(byte[])}.
+     * @deprecated use {@link #source(String, XContentType)}
      */
+    @Deprecated
     public IndexRequest source(String source) {
-        this.source = new BytesArray(source.getBytes(StandardCharsets.UTF_8));
-        return this;
+        return source(new BytesArray(source), XContentFactory.xContentType(source));
     }
 
     /**
-     * Sets the content source to index.
+     * Sets the document source to index.
+     *
+     * Note, its preferable to either set it using {@link #source(org.elasticsearch.common.xcontent.XContentBuilder)}
+     * or using the {@link #source(byte[], XContentType)}.
      */
-    public IndexRequest source(XContentBuilder sourceBuilder) {
-        source = sourceBuilder.bytes();
-        return this;
-    }
-
-    public IndexRequest source(String field1, Object value1) {
-        try {
-            XContentBuilder builder = XContentFactory.contentBuilder(contentType);
-            builder.startObject().field(field1, value1).endObject();
-            return source(builder);
-        } catch (IOException e) {
-            throw new ElasticsearchGenerationException("Failed to generate", e);
-        }
-    }
-
-    public IndexRequest source(String field1, Object value1, String field2, Object value2) {
-        try {
-            XContentBuilder builder = XContentFactory.contentBuilder(contentType);
-            builder.startObject().field(field1, value1).field(field2, value2).endObject();
-            return source(builder);
-        } catch (IOException e) {
-            throw new ElasticsearchGenerationException("Failed to generate", e);
-        }
+    public IndexRequest source(String source, XContentType xContentType) {
+        return source(new BytesArray(source), xContentType);
     }
 
-    public IndexRequest source(String field1, Object value1, String field2, Object value2, String field3, Object value3) {
-        try {
-            XContentBuilder builder = XContentFactory.contentBuilder(contentType);
-            builder.startObject().field(field1, value1).field(field2, value2).field(field3, value3).endObject();
-            return source(builder);
-        } catch (IOException e) {
-            throw new ElasticsearchGenerationException("Failed to generate", e);
-        }
+    /**
+     * Sets the content source to index.
+     */
+    public IndexRequest source(XContentBuilder sourceBuilder) {
+        return source(sourceBuilder.bytes(), sourceBuilder.contentType());
     }
 
-    public IndexRequest source(String field1, Object value1, String field2, Object value2, String field3, Object value3, String field4, Object value4) {
-        try {
-            XContentBuilder builder = XContentFactory.contentBuilder(contentType);
-            builder.startObject().field(field1, value1).field(field2, value2).field(field3, value3).field(field4, value4).endObject();
-            return source(builder);
-        } catch (IOException e) {
-            throw new ElasticsearchGenerationException("Failed to generate", e);
-        }
+    /**
+     * Sets the content source to index using the default content type ({@link Requests#INDEX_CONTENT_TYPE})
+     * <p>
+     * <b>Note: the number of objects passed to this method must be an even
+     * number. Also the first argument in each pair (the field name) must have a
+     * valid String representation.</b>
+     * </p>
+     */
+    public IndexRequest source(Object... source) {
+        return source(Requests.INDEX_CONTENT_TYPE, source);
     }
 
-    public IndexRequest source(Object... source) {
+    /**
+     * Sets the content source to index.
+     * <p>
+     * <b>Note: the number of objects passed to this method as varargs must be an even
+     * number. Also the first argument in each pair (the field name) must have a
+     * valid String representation.</b>
+     * </p>
+     */
+    public IndexRequest source(XContentType xContentType, Object... source) {
         if (source.length % 2 != 0) {
             throw new IllegalArgumentException("The number of object passed must be even but was [" + source.length + "]");
         }
@@ -482,7 +434,7 @@ public IndexRequest source(Object... source) {
             throw new IllegalArgumentException("you are using the removed method for source with bytes and unsafe flag, the unsafe flag was removed, please just use source(BytesReference)");
         }
         try {
-            XContentBuilder builder = XContentFactory.contentBuilder(contentType);
+            XContentBuilder builder = XContentFactory.contentBuilder(xContentType);
             builder.startObject();
             for (int i = 0; i < source.length; i++) {
                 builder.field(source[i++].toString(), source[i]);
@@ -496,19 +448,39 @@ public IndexRequest source(Object... source) {
 
     /**
      * Sets the document to index in bytes form.
+     * @deprecated use {@link #source(BytesReference, XContentType)}
      */
+    @Deprecated
     public IndexRequest source(BytesReference source) {
-        this.source = source;
+        return source(source, XContentFactory.xContentType(source));
+
+    }
+
+    /**
+     * Sets the document to index in bytes form.
+     */
+    public IndexRequest source(BytesReference source, XContentType xContentType) {
+        this.source = Objects.requireNonNull(source);
+        this.contentType = Objects.requireNonNull(xContentType);
         return this;
     }
 
     /**
      * Sets the document to index in bytes form.
+     * @deprecated use {@link #source(byte[], XContentType)}
      */
+    @Deprecated
     public IndexRequest source(byte[] source) {
         return source(source, 0, source.length);
     }
 
+    /**
+     * Sets the document to index in bytes form.
+     */
+    public IndexRequest source(byte[] source, XContentType xContentType) {
+        return source(source, 0, source.length, xContentType);
+    }
+
     /**
      * Sets the document to index in bytes form (assumed to be safe to be used from different
      * threads).
@@ -516,30 +488,50 @@ public IndexRequest source(byte[] source) {
      * @param source The source to index
      * @param offset The offset in the byte array
      * @param length The length of the data
+     * @deprecated use {@link #source(byte[], int, int, XContentType)}
      */
+    @Deprecated
     public IndexRequest source(byte[] source, int offset, int length) {
-        this.source = new BytesArray(source, offset, length);
-        return this;
+        return source(new BytesArray(source, offset, length), XContentFactory.xContentType(source));
+    }
+
+    /**
+     * Sets the document to index in bytes form (assumed to be safe to be used from different
+     * threads).
+     *
+     * @param source The source to index
+     * @param offset The offset in the byte array
+     * @param length The length of the data
+     */
+    public IndexRequest source(byte[] source, int offset, int length, XContentType xContentType) {
+        return source(new BytesArray(source, offset, length), xContentType);
     }
 
     /**
      * Sets the type of operation to perform.
      */
     public IndexRequest opType(OpType opType) {
-        this.opType = opType;
-        if (opType == OpType.CREATE) {
-            version(Versions.MATCH_DELETED);
-            versionType(VersionType.INTERNAL);
+        if (opType != OpType.CREATE && opType != OpType.INDEX) {
+            throw new IllegalArgumentException("opType must be 'create' or 'index', found: [" + opType + "]");
         }
+        this.opType = opType;
         return this;
     }
 
     /**
-     * Sets a string representation of the {@link #opType(org.elasticsearch.action.index.IndexRequest.OpType)}. Can
+     * Sets a string representation of the {@link #opType(OpType)}. Can
      * be either "index" or "create".
      */
     public IndexRequest opType(String opType) {
-        return opType(OpType.fromString(opType));
+        String op = opType.toLowerCase(Locale.ROOT);
+        if (op.equals("create")) {
+            opType(OpType.CREATE);
+        } else if (op.equals("index")) {
+            opType(OpType.INDEX);
+        } else {
+            throw new IllegalArgumentException("opType must be 'create' or 'index', found: [" + opType + "]");
+        }
+        return this;
     }
 
 
@@ -554,34 +546,44 @@ public IndexRequest create(boolean create) {
         }
     }
 
-    /**
-     * The type of operation to perform.
-     */
+    @Override
     public OpType opType() {
         return this.opType;
     }
 
-    /**
-     * Sets the version, which will cause the index operation to only be performed if a matching
-     * version exists and no changes happened on the doc since then.
-     */
+    @Override
     public IndexRequest version(long version) {
         this.version = version;
         return this;
     }
 
+    /**
+     * Returns stored version. If currently stored version is {@link Versions#MATCH_ANY} and
+     * opType is {@link OpType#CREATE}, returns {@link Versions#MATCH_DELETED}.
+     */
+    @Override
     public long version() {
-        return this.version;
+        return resolveVersionDefaults();
     }
 
     /**
-     * Sets the versioning type. Defaults to {@link VersionType#INTERNAL}.
+     * Resolves the version based on operation type {@link #opType()}.
      */
+    private long resolveVersionDefaults() {
+        if (opType == OpType.CREATE && version == Versions.MATCH_ANY) {
+            return Versions.MATCH_DELETED;
+        } else {
+            return version;
+        }
+    }
+
+    @Override
     public IndexRequest versionType(VersionType versionType) {
         this.versionType = versionType;
         return this;
     }
 
+    @Override
     public VersionType versionType() {
         return this.versionType;
     }
@@ -600,14 +602,18 @@ public void process(@Nullable MappingMetaData mappingMd, boolean allowIdGenerati
             }
 
             if (parent != null && !mappingMd.hasParentField()) {
-                throw new IllegalArgumentException("Can't specify parent if no parent field has been configured");
+                throw new IllegalArgumentException("can't specify parent if no parent field has been configured");
             }
         } else {
             if (parent != null) {
-                throw new IllegalArgumentException("Can't specify parent if no parent field has been configured");
+                throw new IllegalArgumentException("can't specify parent if no parent field has been configured");
             }
         }
 
+        if ("".equals(id)) {
+            throw new IllegalArgumentException("if _id is specified it must not be empty");
+        }
+
         // generate id if not already provided and id generation is allowed
         if (allowIdGeneration && id == null) {
             assert autoGeneratedTimestamp == -1;
@@ -661,6 +667,11 @@ public void readFrom(StreamInput in) throws IOException {
         pipeline = in.readOptionalString();
         isRetry = in.readBoolean();
         autoGeneratedTimestamp = in.readLong();
+        if (in.getVersion().onOrAfter(Version.V_5_3_0)) {
+            contentType = in.readOptionalWriteable(XContentType::readFrom);
+        } else {
+            contentType = XContentFactory.xContentType(source);
+        }
     }
 
     @Override
@@ -673,19 +684,32 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalString(timestamp);
         out.writeOptionalWriteable(ttl);
         out.writeBytesReference(source);
-        out.writeByte(opType.id());
-        out.writeLong(version);
+        out.writeByte(opType.getId());
+        // ES versions below 5.1.2 don't know about resolveVersionDefaults but resolve the version eagerly (which messes with validation).
+        if (out.getVersion().before(Version.V_5_1_2)) {
+            out.writeLong(resolveVersionDefaults());
+        } else {
+            out.writeLong(version);
+        }
         out.writeByte(versionType.getValue());
         out.writeOptionalString(pipeline);
         out.writeBoolean(isRetry);
         out.writeLong(autoGeneratedTimestamp);
+        if (out.getVersion().onOrAfter(Version.V_5_3_0)) {
+            out.writeOptionalWriteable(contentType);
+        }
     }
 
     @Override
     public String toString() {
         String sSource = "_na_";
         try {
-            sSource = XContentHelper.convertToJson(source, false);
+            if (source.length() > MAX_SOURCE_LENGTH_IN_TOSTRING) {
+                sSource = "n/a, actual length: [" + new ByteSizeValue(source.length()).toString() + "], max length: " +
+                    new ByteSizeValue(MAX_SOURCE_LENGTH_IN_TOSTRING).toString();
+            } else {
+                sSource = XContentHelper.convertToJson(source, false);
+            }
         } catch (Exception e) {
             // ignore
         }
diff --git a/core/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java
index c4609e03aa5ed..309712ed23ee1 100644
--- a/core/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.action.index;
 
+import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.support.WriteRequestBuilder;
 import org.elasticsearch.action.support.replication.ReplicationRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
@@ -82,12 +83,22 @@ public IndexRequestBuilder setParent(String parent) {
 
     /**
      * Sets the source.
+     * @deprecated use {@link #setSource(BytesReference, XContentType)}
      */
+    @Deprecated
     public IndexRequestBuilder setSource(BytesReference source) {
         request.source(source);
         return this;
     }
 
+    /**
+     * Sets the source.
+     */
+    public IndexRequestBuilder setSource(BytesReference source, XContentType xContentType) {
+        request.source(source, xContentType);
+        return this;
+    }
+
     /**
      * Index the Map as a JSON.
      *
@@ -112,13 +123,26 @@ public IndexRequestBuilder setSource(Map<String, ?> source, XContentType content
      * Sets the document source to index.
      * <p>
      * Note, its preferable to either set it using {@link #setSource(org.elasticsearch.common.xcontent.XContentBuilder)}
-     * or using the {@link #setSource(byte[])}.
+     * or using the {@link #setSource(byte[], XContentType)}.
+     * @deprecated use {@link #setSource(String, XContentType)}
      */
+    @Deprecated
     public IndexRequestBuilder setSource(String source) {
         request.source(source);
         return this;
     }
 
+    /**
+     * Sets the document source to index.
+     * <p>
+     * Note, its preferable to either set it using {@link #setSource(org.elasticsearch.common.xcontent.XContentBuilder)}
+     * or using the {@link #setSource(byte[], XContentType)}.
+     */
+    public IndexRequestBuilder setSource(String source, XContentType xContentType) {
+        request.source(source, xContentType);
+        return this;
+    }
+
     /**
      * Sets the content source to index.
      */
@@ -129,12 +153,22 @@ public IndexRequestBuilder setSource(XContentBuilder sourceBuilder) {
 
     /**
      * Sets the document to index in bytes form.
+     * @deprecated use {@link #setSource(byte[], XContentType)}
      */
+    @Deprecated
     public IndexRequestBuilder setSource(byte[] source) {
         request.source(source);
         return this;
     }
 
+    /**
+     * Sets the document to index in bytes form.
+     */
+    public IndexRequestBuilder setSource(byte[] source, XContentType xContentType) {
+        request.source(source, xContentType);
+        return this;
+    }
+
     /**
      * Sets the document to index in bytes form (assumed to be safe to be used from different
      * threads).
@@ -142,47 +176,35 @@ public IndexRequestBuilder setSource(byte[] source) {
      * @param source The source to index
      * @param offset The offset in the byte array
      * @param length The length of the data
+     * @deprecated use {@link #setSource(byte[], int, int, XContentType)}
      */
+    @Deprecated
     public IndexRequestBuilder setSource(byte[] source, int offset, int length) {
         request.source(source, offset, length);
         return this;
     }
 
     /**
-     * Constructs a simple document with a field and a value.
-     */
-    public IndexRequestBuilder setSource(String field1, Object value1) {
-        request.source(field1, value1);
-        return this;
-    }
-
-    /**
-     * Constructs a simple document with a field and value pairs.
-     */
-    public IndexRequestBuilder setSource(String field1, Object value1, String field2, Object value2) {
-        request.source(field1, value1, field2, value2);
-        return this;
-    }
-
-    /**
-     * Constructs a simple document with a field and value pairs.
-     */
-    public IndexRequestBuilder setSource(String field1, Object value1, String field2, Object value2, String field3, Object value3) {
-        request.source(field1, value1, field2, value2, field3, value3);
-        return this;
-    }
-
-    /**
-     * Constructs a simple document with a field and value pairs.
+     * Sets the document to index in bytes form (assumed to be safe to be used from different
+     * threads).
+     *
+     * @param source The source to index
+     * @param offset The offset in the byte array
+     * @param length The length of the data
+     * @param xContentType The type/format of the source
      */
-    public IndexRequestBuilder setSource(String field1, Object value1, String field2, Object value2, String field3, Object value3, String field4, Object value4) {
-        request.source(field1, value1, field2, value2, field3, value3, field4, value4);
+    public IndexRequestBuilder setSource(byte[] source, int offset, int length, XContentType xContentType) {
+        request.source(source, offset, length, xContentType);
         return this;
     }
 
     /**
      * Constructs a simple document with a field name and value pairs.
-     * <b>Note: the number of objects passed to this method must be an even number.</b>
+     * <p>
+     * <b>Note: the number of objects passed to this method must be an even
+     * number. Also the first argument in each pair (the field name) must have a
+     * valid String representation.</b>
+     * </p>
      */
     public IndexRequestBuilder setSource(Object... source) {
         request.source(source);
@@ -190,17 +212,22 @@ public IndexRequestBuilder setSource(Object... source) {
     }
 
     /**
-     * The content type that will be used to generate a document from user provided objects (like Map).
+     * Constructs a simple document with a field name and value pairs.
+     * <p>
+     * <b>Note: the number of objects passed as varargs to this method must be an even
+     * number. Also the first argument in each pair (the field name) must have a
+     * valid String representation.</b>
+     * </p>
      */
-    public IndexRequestBuilder setContentType(XContentType contentType) {
-        request.contentType(contentType);
+    public IndexRequestBuilder setSource(XContentType xContentType, Object... source) {
+        request.source(xContentType, source);
         return this;
     }
 
     /**
      * Sets the type of operation to perform.
      */
-    public IndexRequestBuilder setOpType(IndexRequest.OpType opType) {
+    public IndexRequestBuilder setOpType(DocWriteRequest.OpType opType) {
         request.opType(opType);
         return this;
     }
@@ -233,6 +260,7 @@ public IndexRequestBuilder setVersionType(VersionType versionType) {
     /**
      * Sets the timestamp either as millis since the epoch, or, in the configured date format.
      */
+    @Deprecated
     public IndexRequestBuilder setTimestamp(String timestamp) {
         request.timestamp(timestamp);
         return this;
@@ -241,6 +269,7 @@ public IndexRequestBuilder setTimestamp(String timestamp) {
     /**
      * Sets the ttl value as a time value expression.
      */
+    @Deprecated
     public IndexRequestBuilder setTTL(String ttl) {
         request.ttl(ttl);
         return this;
@@ -249,6 +278,7 @@ public IndexRequestBuilder setTTL(String ttl) {
     /**
      * Sets the relative ttl value in milliseconds. It musts be greater than 0 as it makes little sense otherwise.
      */
+    @Deprecated
     public IndexRequestBuilder setTTL(long ttl) {
         request.ttl(ttl);
         return this;
@@ -257,6 +287,7 @@ public IndexRequestBuilder setTTL(long ttl) {
     /**
      * Sets the ttl as a {@link TimeValue} instance.
      */
+    @Deprecated
     public IndexRequestBuilder setTTL(TimeValue ttl) {
         request.ttl(ttl);
         return this;
diff --git a/core/src/main/java/org/elasticsearch/action/index/IndexResponse.java b/core/src/main/java/org/elasticsearch/action/index/IndexResponse.java
index 9167740567c12..682005480720c 100644
--- a/core/src/main/java/org/elasticsearch/action/index/IndexResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/index/IndexResponse.java
@@ -20,12 +20,16 @@
 package org.elasticsearch.action.index;
 
 import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
 
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+
 /**
  * A response of an index operation,
  *
@@ -34,8 +38,9 @@
  */
 public class IndexResponse extends DocWriteResponse {
 
-    public IndexResponse() {
+    private static final String CREATED = "created";
 
+    public IndexResponse() {
     }
 
     public IndexResponse(ShardId shardId, String type, String id, long version, boolean created) {
@@ -56,14 +61,64 @@ public String toString() {
         builder.append(",id=").append(getId());
         builder.append(",version=").append(getVersion());
         builder.append(",result=").append(getResult().getLowercase());
-        builder.append(",shards=").append(getShardInfo());
+        builder.append(",shards=").append(Strings.toString(getShardInfo()));
         return builder.append("]").toString();
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        super.toXContent(builder, params);
-        builder.field("created", result == Result.CREATED);
+    public XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
+        super.innerToXContent(builder, params);
+        builder.field(CREATED, result == Result.CREATED);
         return builder;
     }
+
+    public static IndexResponse fromXContent(XContentParser parser) throws IOException {
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
+
+        Builder context = new Builder();
+        while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
+            parseXContentFields(parser, context);
+        }
+        return context.build();
+    }
+
+    /**
+     * Parse the current token and update the parsing context appropriately.
+     */
+    public static void parseXContentFields(XContentParser parser, Builder context) throws IOException {
+        XContentParser.Token token = parser.currentToken();
+        String currentFieldName = parser.currentName();
+
+        if (CREATED.equals(currentFieldName)) {
+            if (token.isValue()) {
+                context.setCreated(parser.booleanValue());
+            }
+        } else {
+            DocWriteResponse.parseInnerToXContent(parser, context);
+        }
+    }
+
+    /**
+     * Builder class for {@link IndexResponse}. This builder is usually used during xcontent parsing to
+     * temporarily store the parsed values, then the {@link Builder#build()} method is called to
+     * instantiate the {@link IndexResponse}.
+     */
+    public static class Builder extends DocWriteResponse.Builder {
+
+        private boolean created = false;
+
+        public void setCreated(boolean created) {
+            this.created = created;
+        }
+
+        @Override
+        public IndexResponse build() {
+            IndexResponse indexResponse = new IndexResponse(shardId, type, id, version, created);
+            indexResponse.setForcedRefresh(forcedRefresh);
+            if (shardInfo != null) {
+                indexResponse.setShardInfo(shardInfo);
+            }
+            return indexResponse;
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java b/core/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java
index cc3fbb7906d2e..c925f86df973d 100644
--- a/core/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java
@@ -19,34 +19,16 @@
 
 package org.elasticsearch.action.index;
 
-import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
-import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
-import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction;
+import org.elasticsearch.action.bulk.TransportBulkAction;
+import org.elasticsearch.action.bulk.TransportShardBulkAction;
+import org.elasticsearch.action.bulk.TransportSingleItemBulkWriteAction;
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.AutoCreateIndex;
-import org.elasticsearch.action.support.replication.ReplicationOperation;
-import org.elasticsearch.action.support.replication.TransportWriteAction;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.action.index.MappingUpdatedAction;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.metadata.MappingMetaData;
-import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.engine.Engine;
-import org.elasticsearch.index.mapper.Mapping;
-import org.elasticsearch.index.mapper.SourceToParse;
-import org.elasticsearch.index.shard.IndexShard;
-import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.index.translog.Translog.Location;
-import org.elasticsearch.indices.IndexAlreadyExistsException;
 import org.elasticsearch.indices.IndicesService;
-import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -59,146 +41,26 @@
  * Defaults to <tt>true</tt>.
  * <li><b>allowIdGeneration</b>: If the id is set not, should it be generated. Defaults to <tt>true</tt>.
  * </ul>
+ *
+ * Deprecated use TransportBulkAction with a single item instead
  */
-public class TransportIndexAction extends TransportWriteAction<IndexRequest, IndexResponse> {
-
-    private final AutoCreateIndex autoCreateIndex;
-    private final boolean allowIdGeneration;
-    private final TransportCreateIndexAction createIndexAction;
-
-    private final ClusterService clusterService;
-    private final MappingUpdatedAction mappingUpdatedAction;
+@Deprecated
+public class TransportIndexAction extends TransportSingleItemBulkWriteAction<IndexRequest, IndexResponse> {
 
     @Inject
     public TransportIndexAction(Settings settings, TransportService transportService, ClusterService clusterService,
-                                IndicesService indicesService, ThreadPool threadPool, ShardStateAction shardStateAction,
-                                TransportCreateIndexAction createIndexAction, MappingUpdatedAction mappingUpdatedAction,
+                                IndicesService indicesService,
+                                ThreadPool threadPool, ShardStateAction shardStateAction,
                                 ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
-                                AutoCreateIndex autoCreateIndex) {
+                                TransportBulkAction bulkAction, TransportShardBulkAction shardBulkAction) {
         super(settings, IndexAction.NAME, transportService, clusterService, indicesService, threadPool, shardStateAction,
-            actionFilters, indexNameExpressionResolver, IndexRequest::new, ThreadPool.Names.INDEX);
-        this.mappingUpdatedAction = mappingUpdatedAction;
-        this.createIndexAction = createIndexAction;
-        this.autoCreateIndex = autoCreateIndex;
-        this.allowIdGeneration = settings.getAsBoolean("action.allow_id_generation", true);
-        this.clusterService = clusterService;
-    }
-
-    @Override
-    protected void doExecute(Task task, final IndexRequest request, final ActionListener<IndexResponse> listener) {
-        // if we don't have a master, we don't have metadata, that's fine, let it find a master using create index API
-        ClusterState state = clusterService.state();
-        if (autoCreateIndex.shouldAutoCreate(request.index(), state)) {
-            CreateIndexRequest createIndexRequest = new CreateIndexRequest();
-            createIndexRequest.index(request.index());
-            createIndexRequest.cause("auto(index api)");
-            createIndexRequest.masterNodeTimeout(request.timeout());
-            createIndexAction.execute(task, createIndexRequest, new ActionListener<CreateIndexResponse>() {
-                @Override
-                public void onResponse(CreateIndexResponse result) {
-                    innerExecute(task, request, listener);
-                }
-
-                @Override
-                public void onFailure(Exception e) {
-                    if (ExceptionsHelper.unwrapCause(e) instanceof IndexAlreadyExistsException) {
-                        // we have the index, do it
-                        try {
-                            innerExecute(task, request, listener);
-                        } catch (Exception inner) {
-                            inner.addSuppressed(e);
-                            listener.onFailure(inner);
-                        }
-                    } else {
-                        listener.onFailure(e);
-                    }
-                }
-            });
-        } else {
-            innerExecute(task, request, listener);
-        }
-    }
-
-    @Override
-    protected void resolveRequest(MetaData metaData, IndexMetaData indexMetaData, IndexRequest request) {
-        super.resolveRequest(metaData, indexMetaData, request);
-        MappingMetaData mappingMd =indexMetaData.mappingOrDefault(request.type());
-        request.resolveRouting(metaData);
-        request.process(mappingMd, allowIdGeneration, indexMetaData.getIndex().getName());
-        ShardId shardId = clusterService.operationRouting().shardId(clusterService.state(),
-            indexMetaData.getIndex().getName(), request.id(), request.routing());
-        request.setShardId(shardId);
-    }
-
-    private void innerExecute(Task task, final IndexRequest request, final ActionListener<IndexResponse> listener) {
-        super.doExecute(task, request, listener);
+            actionFilters, indexNameExpressionResolver, IndexRequest::new, IndexRequest::new, ThreadPool.Names.INDEX,
+            bulkAction, shardBulkAction);
     }
 
     @Override
     protected IndexResponse newResponseInstance() {
         return new IndexResponse();
     }
-
-    @Override
-    protected WriteResult<IndexResponse> onPrimaryShard(IndexRequest request, IndexShard indexShard) throws Exception {
-        return executeIndexRequestOnPrimary(request, indexShard, mappingUpdatedAction);
-    }
-
-    @Override
-    protected Location onReplicaShard(IndexRequest request, IndexShard indexShard) {
-        return executeIndexRequestOnReplica(request, indexShard).getTranslogLocation();
-    }
-
-    /**
-     * Execute the given {@link IndexRequest} on a replica shard, throwing a
-     * {@link RetryOnReplicaException} if the operation needs to be re-tried.
-     */
-    public static Engine.Index executeIndexRequestOnReplica(IndexRequest request, IndexShard indexShard) {
-        final ShardId shardId = indexShard.shardId();
-        SourceToParse sourceToParse = SourceToParse.source(SourceToParse.Origin.REPLICA, shardId.getIndexName(), request.type(), request.id(), request.source())
-                .routing(request.routing()).parent(request.parent()).timestamp(request.timestamp()).ttl(request.ttl());
-
-        final Engine.Index operation = indexShard.prepareIndexOnReplica(sourceToParse, request.version(), request.versionType(), request.getAutoGeneratedTimestamp(), request.isRetry());
-        Mapping update = operation.parsedDoc().dynamicMappingsUpdate();
-        if (update != null) {
-            throw new RetryOnReplicaException(shardId, "Mappings are not available on the replica yet, triggered update: " + update);
-        }
-        indexShard.index(operation);
-        return operation;
-    }
-
-    /** Utility method to prepare an index operation on primary shards */
-    public static Engine.Index prepareIndexOperationOnPrimary(IndexRequest request, IndexShard indexShard) {
-        SourceToParse sourceToParse = SourceToParse.source(SourceToParse.Origin.PRIMARY, request.index(), request.type(), request.id(), request.source())
-            .routing(request.routing()).parent(request.parent()).timestamp(request.timestamp()).ttl(request.ttl());
-        return indexShard.prepareIndexOnPrimary(sourceToParse, request.version(), request.versionType(), request.getAutoGeneratedTimestamp(), request.isRetry());
-    }
-
-    public static WriteResult<IndexResponse> executeIndexRequestOnPrimary(IndexRequest request, IndexShard indexShard,
-            MappingUpdatedAction mappingUpdatedAction) throws Exception {
-        Engine.Index operation = prepareIndexOperationOnPrimary(request, indexShard);
-        Mapping update = operation.parsedDoc().dynamicMappingsUpdate();
-        final ShardId shardId = indexShard.shardId();
-        if (update != null) {
-            mappingUpdatedAction.updateMappingOnMaster(shardId.getIndex(), request.type(), update);
-            operation = prepareIndexOperationOnPrimary(request, indexShard);
-            update = operation.parsedDoc().dynamicMappingsUpdate();
-            if (update != null) {
-                throw new ReplicationOperation.RetryOnPrimaryException(shardId,
-                    "Dynamic mappings are not available on the node that holds the primary yet");
-            }
-        }
-        indexShard.index(operation);
-
-        // update the version on request so it will happen on the replicas
-        final long version = operation.version();
-        request.version(version);
-        request.versionType(request.versionType().versionTypeForReplicationAndRecovery());
-
-        assert request.versionType().validateVersionForWrites(request.version());
-
-        IndexResponse response = new IndexResponse(shardId, request.type(), request.id(), request.version(), operation.isCreated());
-        return new WriteResult<>(response, operation.getTranslogLocation());
-    }
 }
 
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/DeletePipelineRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/ingest/DeletePipelineRequestBuilder.java
index fc14e0de2dfaa..90cbce135af2c 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/DeletePipelineRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/DeletePipelineRequestBuilder.java
@@ -32,4 +32,12 @@ public DeletePipelineRequestBuilder(ElasticsearchClient client, DeletePipelineAc
         super(client, action, new DeletePipelineRequest(id));
     }
 
+    /**
+     * Sets the id of the pipeline to delete.
+     */
+    public DeletePipelineRequestBuilder setId(String id) {
+        request.setId(id);
+        return this;
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/DeletePipelineTransportAction.java b/core/src/main/java/org/elasticsearch/action/ingest/DeletePipelineTransportAction.java
index 74ce894b05321..45cb83634f84f 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/DeletePipelineTransportAction.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/DeletePipelineTransportAction.java
@@ -30,7 +30,7 @@
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.ingest.PipelineStore;
-import org.elasticsearch.node.service.NodeService;
+import org.elasticsearch.node.NodeService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/GetPipelineResponse.java b/core/src/main/java/org/elasticsearch/action/ingest/GetPipelineResponse.java
index f603a354f4b03..30843bdff9b28 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/GetPipelineResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/GetPipelineResponse.java
@@ -22,7 +22,7 @@
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.StatusToXContent;
+import org.elasticsearch.common.xcontent.StatusToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.ingest.PipelineConfiguration;
 import org.elasticsearch.rest.RestStatus;
@@ -31,7 +31,7 @@
 import java.util.ArrayList;
 import java.util.List;
 
-public class GetPipelineResponse extends ActionResponse implements StatusToXContent {
+public class GetPipelineResponse extends ActionResponse implements StatusToXContentObject {
 
     private List<PipelineConfiguration> pipelines;
 
@@ -52,7 +52,7 @@ public void readFrom(StreamInput in) throws IOException {
         int size = in.readVInt();
         pipelines = new ArrayList<>(size);
         for (int i = 0; i < size; i++) {
-            pipelines.add(PipelineConfiguration.readPipelineConfiguration(in));
+            pipelines.add(PipelineConfiguration.readFrom(in));
         }
     }
 
@@ -76,9 +76,11 @@ public RestStatus status() {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
         for (PipelineConfiguration pipeline : pipelines) {
             builder.field(pipeline.getId(), pipeline.getConfigAsMap());
         }
+        builder.endObject();
         return builder;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/GetPipelineTransportAction.java b/core/src/main/java/org/elasticsearch/action/ingest/GetPipelineTransportAction.java
index 8bac5c7b80434..f64b36d47aedb 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/GetPipelineTransportAction.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/GetPipelineTransportAction.java
@@ -30,7 +30,7 @@
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.ingest.PipelineStore;
-import org.elasticsearch.node.service.NodeService;
+import org.elasticsearch.node.NodeService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/IngestActionFilter.java b/core/src/main/java/org/elasticsearch/action/ingest/IngestActionFilter.java
deleted file mode 100644
index 68983ccfd22e7..0000000000000
--- a/core/src/main/java/org/elasticsearch/action/ingest/IngestActionFilter.java
+++ /dev/null
@@ -1,242 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.action.ingest;
-
-import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.apache.logging.log4j.util.Supplier;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ActionRequest;
-import org.elasticsearch.action.ActionResponse;
-import org.elasticsearch.action.bulk.BulkAction;
-import org.elasticsearch.action.bulk.BulkItemResponse;
-import org.elasticsearch.action.bulk.BulkRequest;
-import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.action.index.IndexAction;
-import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.support.ActionFilter;
-import org.elasticsearch.action.support.ActionFilterChain;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.ingest.PipelineExecutionService;
-import org.elasticsearch.node.service.NodeService;
-import org.elasticsearch.tasks.Task;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-
-public final class IngestActionFilter extends AbstractComponent implements ActionFilter {
-
-    private final PipelineExecutionService executionService;
-
-    @Inject
-    public IngestActionFilter(Settings settings, NodeService nodeService) {
-        super(settings);
-        this.executionService = nodeService.getIngestService().getPipelineExecutionService();
-    }
-
-    @Override
-    public <Request extends ActionRequest<Request>, Response extends ActionResponse> void apply(Task task, String action, Request request, ActionListener<Response> listener, ActionFilterChain<Request, Response> chain) {
-        switch (action) {
-            case IndexAction.NAME:
-                IndexRequest indexRequest = (IndexRequest) request;
-                if (Strings.hasText(indexRequest.getPipeline())) {
-                    processIndexRequest(task, action, listener, chain, (IndexRequest) request);
-                } else {
-                    chain.proceed(task, action, request, listener);
-                }
-                break;
-            case BulkAction.NAME:
-                BulkRequest bulkRequest = (BulkRequest) request;
-                if (bulkRequest.hasIndexRequestsWithPipelines()) {
-                    @SuppressWarnings("unchecked")
-                    ActionListener<BulkResponse> actionListener = (ActionListener<BulkResponse>) listener;
-                    processBulkIndexRequest(task, bulkRequest, action, chain, actionListener);
-                } else {
-                    chain.proceed(task, action, request, listener);
-                }
-                break;
-            default:
-                chain.proceed(task, action, request, listener);
-                break;
-        }
-    }
-
-    @Override
-    public <Response extends ActionResponse> void apply(String action, Response response, ActionListener<Response> listener, ActionFilterChain<?, Response> chain) {
-        chain.proceed(action, response, listener);
-    }
-
-    void processIndexRequest(Task task, String action, ActionListener listener, ActionFilterChain chain, IndexRequest indexRequest) {
-
-        executionService.executeIndexRequest(indexRequest, t -> {
-            logger.error((Supplier<?>) () -> new ParameterizedMessage("failed to execute pipeline [{}]", indexRequest.getPipeline()), t);
-            listener.onFailure(t);
-        }, success -> {
-            // TransportIndexAction uses IndexRequest and same action name on the node that receives the request and the node that
-            // processes the primary action. This could lead to a pipeline being executed twice for the same
-            // index request, hence we set the pipeline to null once its execution completed.
-            indexRequest.setPipeline(null);
-            chain.proceed(task, action, indexRequest, listener);
-        });
-    }
-
-    void processBulkIndexRequest(Task task, BulkRequest original, String action, ActionFilterChain chain, ActionListener<BulkResponse> listener) {
-        long ingestStartTimeInNanos = System.nanoTime();
-        BulkRequestModifier bulkRequestModifier = new BulkRequestModifier(original);
-        executionService.executeBulkRequest(() -> bulkRequestModifier, (indexRequest, exception) -> {
-            logger.debug((Supplier<?>) () -> new ParameterizedMessage("failed to execute pipeline [{}] for document [{}/{}/{}]", indexRequest.getPipeline(), indexRequest.index(), indexRequest.type(), indexRequest.id()), exception);
-            bulkRequestModifier.markCurrentItemAsFailed(exception);
-        }, (exception) -> {
-            if (exception != null) {
-                logger.error("failed to execute pipeline for a bulk request", exception);
-                listener.onFailure(exception);
-            } else {
-                long ingestTookInMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - ingestStartTimeInNanos);
-                BulkRequest bulkRequest = bulkRequestModifier.getBulkRequest();
-                ActionListener<BulkResponse> actionListener = bulkRequestModifier.wrapActionListenerIfNeeded(ingestTookInMillis, listener);
-                if (bulkRequest.requests().isEmpty()) {
-                    // at this stage, the transport bulk action can't deal with a bulk request with no requests,
-                    // so we stop and send an empty response back to the client.
-                    // (this will happen if pre-processing all items in the bulk failed)
-                    actionListener.onResponse(new BulkResponse(new BulkItemResponse[0], 0));
-                } else {
-                    chain.proceed(task, action, bulkRequest, actionListener);
-                }
-            }
-        });
-    }
-
-    @Override
-    public int order() {
-        return Integer.MAX_VALUE;
-    }
-
-    static final class BulkRequestModifier implements Iterator<ActionRequest<?>> {
-
-        final BulkRequest bulkRequest;
-        final Set<Integer> failedSlots;
-        final List<BulkItemResponse> itemResponses;
-
-        int currentSlot = -1;
-        int[] originalSlots;
-
-        BulkRequestModifier(BulkRequest bulkRequest) {
-            this.bulkRequest = bulkRequest;
-            this.failedSlots = new HashSet<>();
-            this.itemResponses = new ArrayList<>(bulkRequest.requests().size());
-        }
-
-        @Override
-        public ActionRequest next() {
-            return bulkRequest.requests().get(++currentSlot);
-        }
-
-        @Override
-        public boolean hasNext() {
-            return (currentSlot + 1) < bulkRequest.requests().size();
-        }
-
-        BulkRequest getBulkRequest() {
-            if (itemResponses.isEmpty()) {
-                return bulkRequest;
-            } else {
-                BulkRequest modifiedBulkRequest = new BulkRequest();
-                modifiedBulkRequest.setRefreshPolicy(bulkRequest.getRefreshPolicy());
-                modifiedBulkRequest.waitForActiveShards(bulkRequest.waitForActiveShards());
-                modifiedBulkRequest.timeout(bulkRequest.timeout());
-
-                int slot = 0;
-                originalSlots = new int[bulkRequest.requests().size() - failedSlots.size()];
-                for (int i = 0; i < bulkRequest.requests().size(); i++) {
-                    ActionRequest request = bulkRequest.requests().get(i);
-                    if (failedSlots.contains(i) == false) {
-                        modifiedBulkRequest.add(request);
-                        originalSlots[slot++] = i;
-                    }
-                }
-                return modifiedBulkRequest;
-            }
-        }
-
-        ActionListener<BulkResponse> wrapActionListenerIfNeeded(long ingestTookInMillis, ActionListener<BulkResponse> actionListener) {
-            if (itemResponses.isEmpty()) {
-                return new ActionListener<BulkResponse>() {
-                    @Override
-                    public void onResponse(BulkResponse response) {
-                        actionListener.onResponse(new BulkResponse(response.getItems(), response.getTookInMillis(), ingestTookInMillis));
-                    }
-
-                    @Override
-                    public void onFailure(Exception e) {
-                        actionListener.onFailure(e);
-                    }
-                };
-            } else {
-                return new IngestBulkResponseListener(ingestTookInMillis, originalSlots, itemResponses, actionListener);
-            }
-        }
-
-        void markCurrentItemAsFailed(Exception e) {
-            IndexRequest indexRequest = (IndexRequest) bulkRequest.requests().get(currentSlot);
-            // We hit a error during preprocessing a request, so we:
-            // 1) Remember the request item slot from the bulk, so that we're done processing all requests we know what failed
-            // 2) Add a bulk item failure for this request
-            // 3) Continue with the next request in the bulk.
-            failedSlots.add(currentSlot);
-            BulkItemResponse.Failure failure = new BulkItemResponse.Failure(indexRequest.index(), indexRequest.type(), indexRequest.id(), e);
-            itemResponses.add(new BulkItemResponse(currentSlot, indexRequest.opType().lowercase(), failure));
-        }
-
-    }
-
-    static final class IngestBulkResponseListener implements ActionListener<BulkResponse> {
-
-        private final long ingestTookInMillis;
-        private final int[] originalSlots;
-        private final List<BulkItemResponse> itemResponses;
-        private final ActionListener<BulkResponse> actionListener;
-
-        IngestBulkResponseListener(long ingestTookInMillis, int[] originalSlots, List<BulkItemResponse> itemResponses, ActionListener<BulkResponse> actionListener) {
-            this.ingestTookInMillis = ingestTookInMillis;
-            this.itemResponses = itemResponses;
-            this.actionListener = actionListener;
-            this.originalSlots = originalSlots;
-        }
-
-        @Override
-        public void onResponse(BulkResponse response) {
-            for (int i = 0; i < response.getItems().length; i++) {
-                itemResponses.add(originalSlots[i], response.getItems()[i]);
-            }
-            actionListener.onResponse(new BulkResponse(itemResponses.toArray(new BulkItemResponse[itemResponses.size()]), response.getTookInMillis(), ingestTookInMillis));
-        }
-
-        @Override
-        public void onFailure(Exception e) {
-            actionListener.onFailure(e);
-        }
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/IngestActionForwarder.java b/core/src/main/java/org/elasticsearch/action/ingest/IngestActionForwarder.java
new file mode 100644
index 0000000000000..8b163eb1eedf8
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/ingest/IngestActionForwarder.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.ingest;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionListenerResponseHandler;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterStateApplier;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.Randomness;
+import org.elasticsearch.transport.TransportService;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * A utility for forwarding ingest requests to ingest nodes in a round-robin fashion.
+ *
+ * TODO: move this into IngestService and make index/bulk actions call that
+ */
+public final class IngestActionForwarder implements ClusterStateApplier {
+
+    private final TransportService transportService;
+    private final AtomicInteger ingestNodeGenerator = new AtomicInteger(Randomness.get().nextInt());
+    private DiscoveryNode[] ingestNodes;
+
+    public IngestActionForwarder(TransportService transportService) {
+        this.transportService = transportService;
+        ingestNodes = new DiscoveryNode[0];
+    }
+
+    public void forwardIngestRequest(Action<?, ?, ?> action, ActionRequest request, ActionListener<?> listener) {
+        transportService.sendRequest(randomIngestNode(), action.name(), request,
+            new ActionListenerResponseHandler(listener, action::newResponse));
+    }
+
+    private DiscoveryNode randomIngestNode() {
+        final DiscoveryNode[] nodes = ingestNodes;
+        if (nodes.length == 0) {
+            throw new IllegalStateException("There are no ingest nodes in this cluster, unable to forward request to an ingest node.");
+        }
+
+        return nodes[Math.floorMod(ingestNodeGenerator.incrementAndGet(), nodes.length)];
+    }
+
+    @Override
+    public void applyClusterState(ClusterChangedEvent event) {
+        ingestNodes = event.state().getNodes().getIngestNodes().values().toArray(DiscoveryNode.class);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/IngestProxyActionFilter.java b/core/src/main/java/org/elasticsearch/action/ingest/IngestProxyActionFilter.java
deleted file mode 100644
index 5d2aea389dceb..0000000000000
--- a/core/src/main/java/org/elasticsearch/action/ingest/IngestProxyActionFilter.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.action.ingest;
-
-import org.elasticsearch.action.Action;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ActionListenerResponseHandler;
-import org.elasticsearch.action.ActionRequest;
-import org.elasticsearch.action.ActionResponse;
-import org.elasticsearch.action.bulk.BulkAction;
-import org.elasticsearch.action.bulk.BulkRequest;
-import org.elasticsearch.action.index.IndexAction;
-import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.support.ActionFilter;
-import org.elasticsearch.action.support.ActionFilterChain;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.Randomness;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.tasks.Task;
-import org.elasticsearch.transport.TransportResponse;
-import org.elasticsearch.transport.TransportService;
-
-import java.util.concurrent.atomic.AtomicInteger;
-
-public final class IngestProxyActionFilter implements ActionFilter {
-
-    private final ClusterService clusterService;
-    private final TransportService transportService;
-    private final AtomicInteger randomNodeGenerator = new AtomicInteger(Randomness.get().nextInt());
-
-    @Inject
-    public IngestProxyActionFilter(ClusterService clusterService, TransportService transportService) {
-        this.clusterService = clusterService;
-        this.transportService = transportService;
-    }
-
-    @Override
-    public <Request extends ActionRequest<Request>, Response extends ActionResponse> void apply(Task task, String action, Request request, ActionListener<Response> listener, ActionFilterChain<Request, Response> chain) {
-        Action ingestAction;
-        switch (action) {
-            case IndexAction.NAME:
-                ingestAction = IndexAction.INSTANCE;
-                IndexRequest indexRequest = (IndexRequest) request;
-                if (Strings.hasText(indexRequest.getPipeline())) {
-                    forwardIngestRequest(ingestAction, request, listener);
-                } else {
-                    chain.proceed(task, action, request, listener);
-                }
-                break;
-            case BulkAction.NAME:
-                ingestAction = BulkAction.INSTANCE;
-                BulkRequest bulkRequest = (BulkRequest) request;
-                if (bulkRequest.hasIndexRequestsWithPipelines()) {
-                    forwardIngestRequest(ingestAction, request, listener);
-                } else {
-                    chain.proceed(task, action, request, listener);
-                }
-                break;
-            default:
-                chain.proceed(task, action, request, listener);
-                break;
-        }
-    }
-
-    @SuppressWarnings("unchecked")
-    private void forwardIngestRequest(Action<?, ?, ?> action, ActionRequest request, ActionListener<?> listener) {
-        transportService.sendRequest(randomIngestNode(), action.name(), request, new ActionListenerResponseHandler(listener, action::newResponse));
-    }
-
-    @Override
-    public <Response extends ActionResponse> void apply(String action, Response response, ActionListener<Response> listener, ActionFilterChain<?, Response> chain) {
-        chain.proceed(action, response, listener);
-    }
-
-    @Override
-    public int order() {
-        return Integer.MAX_VALUE;
-    }
-
-    private DiscoveryNode randomIngestNode() {
-        assert clusterService.localNode().isIngestNode() == false;
-        DiscoveryNodes nodes = clusterService.state().getNodes();
-        DiscoveryNode[] ingestNodes = nodes.getIngestNodes().values().toArray(DiscoveryNode.class);
-        if (ingestNodes.length == 0) {
-            throw new IllegalStateException("There are no ingest nodes in this cluster, unable to forward request to an ingest node.");
-        }
-
-        int index = getNodeNumber();
-        return ingestNodes[(index) % ingestNodes.length];
-    }
-
-    private int getNodeNumber() {
-        int index = randomNodeGenerator.incrementAndGet();
-        if (index < 0) {
-            index = 0;
-            randomNodeGenerator.set(0);
-        }
-        return index;
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java b/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java
index 10416146ba853..394349ca01691 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java
@@ -19,32 +19,40 @@
 
 package org.elasticsearch.action.ingest;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 
 import java.io.IOException;
 import java.util.Objects;
 
-import static org.elasticsearch.action.ValidateActions.addValidationError;
-
 public class PutPipelineRequest extends AcknowledgedRequest<PutPipelineRequest> {
 
     private String id;
     private BytesReference source;
+    private XContentType xContentType;
 
+    /**
+     * Create a new pipeline request
+     * @deprecated use {@link #PutPipelineRequest(String, BytesReference, XContentType)} to avoid content type auto-detection
+     */
+    @Deprecated
     public PutPipelineRequest(String id, BytesReference source) {
-        if (id == null) {
-            throw new IllegalArgumentException("id is missing");
-        }
-        if (source == null) {
-            throw new IllegalArgumentException("source is missing");
-        }
+        this(id, source, XContentFactory.xContentType(source));
+    }
 
-        this.id = id;
-        this.source = source;
+    /**
+     * Create a new pipeline request with the id and source along with the content type of the source
+     */
+    public PutPipelineRequest(String id, BytesReference source, XContentType xContentType) {
+        this.id = Objects.requireNonNull(id);
+        this.source = Objects.requireNonNull(source);
+        this.xContentType = Objects.requireNonNull(xContentType);
     }
 
     PutPipelineRequest() {
@@ -63,11 +71,20 @@ public BytesReference getSource() {
         return source;
     }
 
+    public XContentType getXContentType() {
+        return xContentType;
+    }
+
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
         id = in.readString();
         source = in.readBytesReference();
+        if (in.getVersion().onOrAfter(Version.V_5_3_0)) {
+            xContentType = XContentType.readFrom(in);
+        } else {
+            xContentType = XContentFactory.xContentType(source);
+        }
     }
 
     @Override
@@ -75,5 +92,8 @@ public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         out.writeString(id);
         out.writeBytesReference(source);
+        if (out.getVersion().onOrAfter(Version.V_5_3_0)) {
+            xContentType.writeTo(out);
+        }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequestBuilder.java
index bd927115fb5ff..c03b3b84f8b5b 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequestBuilder.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentType;
 
 public class PutPipelineRequestBuilder extends ActionRequestBuilder<PutPipelineRequest, WritePipelineResponse, PutPipelineRequestBuilder> {
 
@@ -29,8 +30,13 @@ public PutPipelineRequestBuilder(ElasticsearchClient client, PutPipelineAction a
         super(client, action, new PutPipelineRequest());
     }
 
+    @Deprecated
     public PutPipelineRequestBuilder(ElasticsearchClient client, PutPipelineAction action, String id, BytesReference source) {
         super(client, action, new PutPipelineRequest(id, source));
     }
 
+    public PutPipelineRequestBuilder(ElasticsearchClient client, PutPipelineAction action, String id, BytesReference source,
+                                     XContentType xContentType) {
+        super(client, action, new PutPipelineRequest(id, source, xContentType));
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java b/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java
index 82cd8d8eb7b32..7dde981804939 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java
@@ -36,7 +36,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.ingest.PipelineStore;
 import org.elasticsearch.ingest.IngestInfo;
-import org.elasticsearch.node.service.NodeService;
+import org.elasticsearch.node.NodeService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResult.java b/core/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResult.java
index 82b39ac897242..c6252feea276c 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResult.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResult.java
@@ -84,7 +84,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (failure == null) {
             ingestDocument.toXContent(builder, params);
         } else {
-            ElasticsearchException.renderException(builder, params, failure);
+            ElasticsearchException.generateFailureXContent(builder, params, failure, true);
         }
         builder.endObject();
         return builder;
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java
index a63f7a30dbeac..8d93fa5747558 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java
@@ -19,11 +19,14 @@
 
 package org.elasticsearch.action.ingest;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.ingest.Pipeline;
@@ -34,20 +37,32 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 
 import static org.elasticsearch.ingest.IngestDocument.MetaData;
 
-public class SimulatePipelineRequest extends ActionRequest<SimulatePipelineRequest> {
+public class SimulatePipelineRequest extends ActionRequest {
 
     private String id;
     private boolean verbose;
     private BytesReference source;
+    private XContentType xContentType;
 
+    /**
+     * Create a new request
+     * @deprecated use {@link #SimulatePipelineRequest(BytesReference, XContentType)} that does not attempt content autodetection
+     */
+    @Deprecated
     public SimulatePipelineRequest(BytesReference source) {
-        if (source == null) {
-            throw new IllegalArgumentException("source is missing");
-        }
-        this.source = source;
+        this(source, XContentFactory.xContentType(source));
+    }
+
+    /**
+     * Creates a new request with the given source and its content type
+     */
+    public SimulatePipelineRequest(BytesReference source, XContentType xContentType) {
+        this.source = Objects.requireNonNull(source);
+        this.xContentType = Objects.requireNonNull(xContentType);
     }
 
     SimulatePipelineRequest() {
@@ -78,12 +93,21 @@ public BytesReference getSource() {
         return source;
     }
 
+    public XContentType getXContentType() {
+        return xContentType;
+    }
+
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
         id = in.readOptionalString();
         verbose = in.readBoolean();
         source = in.readBytesReference();
+        if (in.getVersion().onOrAfter(Version.V_5_3_0)) {
+            xContentType = XContentType.readFrom(in);
+        } else {
+            xContentType = XContentFactory.xContentType(source);
+        }
     }
 
     @Override
@@ -92,6 +116,9 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalString(id);
         out.writeBoolean(verbose);
         out.writeBytesReference(source);
+        if (out.getVersion().onOrAfter(Version.V_5_3_0)) {
+            xContentType.writeTo(out);
+        }
     }
 
     public static final class Fields {
@@ -135,18 +162,18 @@ static Parsed parseWithPipelineId(String pipelineId, Map<String, Object> config,
         if (pipeline == null) {
             throw new IllegalArgumentException("pipeline [" + pipelineId + "] does not exist");
         }
-        List<IngestDocument> ingestDocumentList = parseDocs(config);
+        List<IngestDocument> ingestDocumentList = parseDocs(config, pipelineStore.isNewIngestDateFormat());
         return new Parsed(pipeline, ingestDocumentList, verbose);
     }
 
     static Parsed parse(Map<String, Object> config, boolean verbose, PipelineStore pipelineStore) throws Exception {
         Map<String, Object> pipelineConfig = ConfigurationUtils.readMap(null, null, config, Fields.PIPELINE);
         Pipeline pipeline = PIPELINE_FACTORY.create(SIMULATED_PIPELINE_ID, pipelineConfig, pipelineStore.getProcessorFactories());
-        List<IngestDocument> ingestDocumentList = parseDocs(config);
+        List<IngestDocument> ingestDocumentList = parseDocs(config, pipelineStore.isNewIngestDateFormat());
         return new Parsed(pipeline, ingestDocumentList, verbose);
     }
 
-    private static List<IngestDocument> parseDocs(Map<String, Object> config) {
+    private static List<IngestDocument> parseDocs(Map<String, Object> config, boolean newDateFormat) {
         List<Map<String, Object>> docs = ConfigurationUtils.readList(null, null, config, Fields.DOCS);
         List<IngestDocument> ingestDocumentList = new ArrayList<>();
         for (Map<String, Object> dataMap : docs) {
@@ -158,7 +185,7 @@ private static List<IngestDocument> parseDocs(Map<String, Object> config) {
                     ConfigurationUtils.readOptionalStringProperty(null, null, dataMap, MetaData.PARENT.getFieldName()),
                     ConfigurationUtils.readOptionalStringProperty(null, null, dataMap, MetaData.TIMESTAMP.getFieldName()),
                     ConfigurationUtils.readOptionalStringProperty(null, null, dataMap, MetaData.TTL.getFieldName()),
-                    document);
+                    document, newDateFormat);
             ingestDocumentList.add(ingestDocument);
         }
         return ingestDocumentList;
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequestBuilder.java
index 4a13fa111e6a2..bb5d0e4e40003 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequestBuilder.java
@@ -22,22 +22,46 @@
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentType;
 
 public class SimulatePipelineRequestBuilder extends ActionRequestBuilder<SimulatePipelineRequest, SimulatePipelineResponse, SimulatePipelineRequestBuilder> {
 
+    /**
+     * Create a new builder for {@link SimulatePipelineRequest}s
+     */
     public SimulatePipelineRequestBuilder(ElasticsearchClient client, SimulatePipelineAction action) {
         super(client, action, new SimulatePipelineRequest());
     }
 
+    /**
+     * Create a new builder for {@link SimulatePipelineRequest}s
+     * @deprecated use {@link #SimulatePipelineRequestBuilder(ElasticsearchClient, SimulatePipelineAction, BytesReference, XContentType)} to
+     *             avoid content type auto-detection on the source bytes
+     */
+    @Deprecated
     public SimulatePipelineRequestBuilder(ElasticsearchClient client, SimulatePipelineAction action, BytesReference source) {
         super(client, action, new SimulatePipelineRequest(source));
     }
 
+    /**
+     * Create a new builder for {@link SimulatePipelineRequest}s
+     */
+    public SimulatePipelineRequestBuilder(ElasticsearchClient client, SimulatePipelineAction action, BytesReference source,
+                                          XContentType xContentType) {
+        super(client, action, new SimulatePipelineRequest(source, xContentType));
+    }
+
+    /**
+     * Set the id for the pipeline to simulate
+     */
     public SimulatePipelineRequestBuilder setId(String id) {
         request.setId(id);
         return this;
     }
 
+    /**
+     * Enable or disable verbose mode
+     */
     public SimulatePipelineRequestBuilder setVerbose(boolean verbose) {
         request.setVerbose(verbose);
         return this;
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineResponse.java b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineResponse.java
index 83029a1aab502..e9ea1a7750738 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineResponse.java
@@ -22,7 +22,7 @@
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
 import java.io.IOException;
@@ -30,7 +30,7 @@
 import java.util.Collections;
 import java.util.List;
 
-public class SimulatePipelineResponse extends ActionResponse implements ToXContent {
+public class SimulatePipelineResponse extends ActionResponse implements ToXContentObject {
     private String pipelineId;
     private boolean verbose;
     private List<SimulateDocumentResult> results;
@@ -88,11 +88,13 @@ public void readFrom(StreamInput in) throws IOException {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
         builder.startArray(Fields.DOCUMENTS);
         for (SimulateDocumentResult response : results) {
             response.toXContent(builder, params);
         }
         builder.endArray();
+        builder.endObject();
         return builder;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineTransportAction.java b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineTransportAction.java
index 4f9a219c8ad9e..3f67007df690d 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineTransportAction.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineTransportAction.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.action.ingest;
 
-import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
@@ -28,7 +27,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.ingest.PipelineStore;
-import org.elasticsearch.node.service.NodeService;
+import org.elasticsearch.node.NodeService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -48,7 +47,7 @@ public SimulatePipelineTransportAction(Settings settings, ThreadPool threadPool,
 
     @Override
     protected void doExecute(SimulatePipelineRequest request, ActionListener<SimulatePipelineResponse> listener) {
-        final Map<String, Object> source = XContentHelper.convertToMap(request.getSource(), false).v2();
+        final Map<String, Object> source = XContentHelper.convertToMap(request.getSource(), false, request.getXContentType()).v2();
 
         final SimulatePipelineRequest.Parsed simulateRequest;
         try {
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java b/core/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java
index c978cc56d9ef1..3ebcb6cb6f373 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java
@@ -19,7 +19,6 @@
 package org.elasticsearch.action.ingest;
 
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -99,10 +98,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
 
         if (failure != null && ingestDocument != null) {
             builder.startObject("ignored_error");
-            ElasticsearchException.renderException(builder, params, failure);
+            ElasticsearchException.generateFailureXContent(builder, params, failure, true);
             builder.endObject();
         } else if (failure != null) {
-            ElasticsearchException.renderException(builder, params, failure);
+            ElasticsearchException.generateFailureXContent(builder, params, failure, true);
         }
 
         if (ingestDocument != null) {
diff --git a/core/src/main/java/org/elasticsearch/action/main/MainRequest.java b/core/src/main/java/org/elasticsearch/action/main/MainRequest.java
index 1484bc2a3e9c1..1736e56a8dc06 100644
--- a/core/src/main/java/org/elasticsearch/action/main/MainRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/main/MainRequest.java
@@ -22,7 +22,7 @@
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 
-public class MainRequest extends ActionRequest<MainRequest> {
+public class MainRequest extends ActionRequest {
 
     @Override
     public ActionRequestValidationException validate() {
diff --git a/core/src/main/java/org/elasticsearch/action/main/MainResponse.java b/core/src/main/java/org/elasticsearch/action/main/MainResponse.java
index 2403c3ee49cfd..39d4f31a1939a 100644
--- a/core/src/main/java/org/elasticsearch/action/main/MainResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/main/MainResponse.java
@@ -23,28 +23,34 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
+import java.util.Objects;
 
-public class MainResponse extends ActionResponse implements ToXContent {
+public class MainResponse extends ActionResponse implements ToXContentObject {
 
     private String nodeName;
     private Version version;
     private ClusterName clusterName;
+    private String clusterUuid;
     private Build build;
     private boolean available;
 
     MainResponse() {
     }
 
-    public MainResponse(String nodeName, Version version, ClusterName clusterName, Build build, boolean available) {
+    public MainResponse(String nodeName, Version version, ClusterName clusterName, String clusterUuid, Build build, boolean available) {
         this.nodeName = nodeName;
         this.version = version;
         this.clusterName = clusterName;
+        this.clusterUuid = clusterUuid;
         this.build = build;
         this.available = available;
     }
@@ -61,6 +67,10 @@ public ClusterName getClusterName() {
         return clusterName;
     }
 
+    public String getClusterUuid() {
+        return clusterUuid;
+    }
+
     public Build getBuild() {
         return build;
     }
@@ -75,6 +85,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeString(nodeName);
         Version.writeVersion(version, out);
         clusterName.writeTo(out);
+        out.writeString(clusterUuid);
         Build.writeBuild(build, out);
         out.writeBoolean(available);
     }
@@ -85,6 +96,7 @@ public void readFrom(StreamInput in) throws IOException {
         nodeName = in.readString();
         version = Version.readVersion(in);
         clusterName = new ClusterName(in);
+        clusterUuid = in.readString();
         build = Build.readBuild(in);
         available = in.readBoolean();
     }
@@ -94,6 +106,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.startObject();
         builder.field("name", nodeName);
         builder.field("cluster_name", clusterName.value());
+        builder.field("cluster_uuid", clusterUuid);
         builder.startObject("version")
             .field("number", version.toString())
             .field("build_hash", build.shortHash())
@@ -105,4 +118,46 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.endObject();
         return builder;
     }
+
+    private static final ObjectParser<MainResponse, Void> PARSER = new ObjectParser<>(MainResponse.class.getName(), true,
+            () -> new MainResponse());
+
+    static {
+        PARSER.declareString((response, value) -> response.nodeName = value, new ParseField("name"));
+        PARSER.declareString((response, value) -> response.clusterName = new ClusterName(value), new ParseField("cluster_name"));
+        PARSER.declareString((response, value) -> response.clusterUuid = value, new ParseField("cluster_uuid"));
+        PARSER.declareString((response, value) -> {}, new ParseField("tagline"));
+        PARSER.declareObject((response, value) -> {
+            response.build = new Build((String) value.get("build_hash"), (String) value.get("build_date"),
+                    (boolean) value.get("build_snapshot"));
+            response.version = Version.fromString((String) value.get("number"));
+            response.available = true;
+        }, (parser, context) -> parser.map(), new ParseField("version"));
+    }
+
+    public static MainResponse fromXContent(XContentParser parser) {
+        return PARSER.apply(parser, null);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        MainResponse other = (MainResponse) o;
+        return Objects.equals(nodeName, other.nodeName) &&
+                Objects.equals(version, other.version) &&
+                Objects.equals(clusterUuid, other.clusterUuid) &&
+                Objects.equals(build, other.build) &&
+                Objects.equals(available, other.available) &&
+                Objects.equals(clusterName, other.clusterName);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(nodeName, version, clusterUuid, build, clusterName, available);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/main/TransportMainAction.java b/core/src/main/java/org/elasticsearch/action/main/TransportMainAction.java
index c37268a52de8b..368696a9553d9 100644
--- a/core/src/main/java/org/elasticsearch/action/main/TransportMainAction.java
+++ b/core/src/main/java/org/elasticsearch/action/main/TransportMainAction.java
@@ -52,7 +52,7 @@ protected void doExecute(MainRequest request, ActionListener<MainResponse> liste
         assert Node.NODE_NAME_SETTING.exists(settings);
         final boolean available = clusterState.getBlocks().hasGlobalBlock(RestStatus.SERVICE_UNAVAILABLE) == false;
         listener.onResponse(
-            new MainResponse(Node.NODE_NAME_SETTING.get(settings), Version.CURRENT, clusterState.getClusterName(), Build.CURRENT,
-                available));
+            new MainResponse(Node.NODE_NAME_SETTING.get(settings), Version.CURRENT, clusterState.getClusterName(),
+                    clusterState.metaData().clusterUUID(), Build.CURRENT, available));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/search/AbstractAsyncAction.java b/core/src/main/java/org/elasticsearch/action/search/AbstractAsyncAction.java
deleted file mode 100644
index 3ce14d8dacd87..0000000000000
--- a/core/src/main/java/org/elasticsearch/action/search/AbstractAsyncAction.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.action.search;
-
-/**
- * Base implementation for an async action.
- */
-abstract class AbstractAsyncAction {
-
-    private final long startTime;
-
-    protected AbstractAsyncAction() {
-        this.startTime = System.currentTimeMillis();
-    }
-
-    /**
-     * Return the time when the action started.
-     */
-    protected final long startTime() {
-        return startTime;
-    }
-
-    /**
-     * Builds how long it took to execute the search.
-     */
-    protected final long buildTookInMillis() {
-        // protect ourselves against time going backwards
-        // negative values don't make sense and we want to be able to serialize that thing as a vLong
-        return Math.max(1, System.currentTimeMillis() - startTime);
-    }
-
-    abstract void start();
-}
diff --git a/core/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java b/core/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
index d86134c7d4467..89be2ecabeb24 100644
--- a/core/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
@@ -19,397 +19,305 @@
 
 package org.elasticsearch.action.search;
 
-import com.carrotsearch.hppc.IntArrayList;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
-import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.NoShardAvailableActionException;
+import org.elasticsearch.action.ShardOperationFailedException;
 import org.elasticsearch.action.support.TransportActions;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.block.ClusterBlockLevel;
-import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
-import org.elasticsearch.cluster.routing.ShardIterator;
-import org.elasticsearch.cluster.routing.ShardRouting;
-import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
-import org.elasticsearch.search.action.SearchTransportService;
-import org.elasticsearch.search.controller.SearchPhaseController;
-import org.elasticsearch.search.fetch.ShardFetchSearchRequest;
+import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.search.internal.InternalSearchResponse;
 import org.elasticsearch.search.internal.ShardSearchTransportRequest;
-import org.elasticsearch.search.query.QuerySearchResult;
-import org.elasticsearch.search.query.QuerySearchResultProvider;
-import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.Transport;
 
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
+import java.util.concurrent.Executor;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
-
-import static org.elasticsearch.action.search.TransportSearchHelper.internalSearchRequest;
-
-abstract class AbstractSearchAsyncAction<FirstResult extends SearchPhaseResult> extends AbstractAsyncAction {
-
-    protected final Logger logger;
-    protected final SearchTransportService searchTransportService;
-    private final IndexNameExpressionResolver indexNameExpressionResolver;
-    protected final SearchPhaseController searchPhaseController;
-    protected final ThreadPool threadPool;
-    protected final ActionListener<SearchResponse> listener;
-    protected final GroupShardsIterator shardsIts;
-    protected final SearchRequest request;
-    protected final ClusterState clusterState;
-    protected final DiscoveryNodes nodes;
-    protected final int expectedSuccessfulOps;
-    private final int expectedTotalOps;
-    protected final AtomicInteger successfulOps = new AtomicInteger();
-    private final AtomicInteger totalOps = new AtomicInteger();
-    protected final AtomicArray<FirstResult> firstResults;
-    private volatile AtomicArray<ShardSearchFailure> shardFailures;
+import java.util.function.BiFunction;
+import java.util.stream.Collectors;
+
+abstract class AbstractSearchAsyncAction<Result extends SearchPhaseResult> extends InitialSearchPhase<Result>
+    implements SearchPhaseContext {
+    private static final float DEFAULT_INDEX_BOOST = 1.0f;
+    private final Logger logger;
+    private final SearchTransportService searchTransportService;
+    private final Executor executor;
+    private final ActionListener<SearchResponse> listener;
+    private final SearchRequest request;
+    /**
+     * Used by subclasses to resolve node ids to DiscoveryNodes.
+     **/
+    private final BiFunction<String, String, Transport.Connection> nodeIdToConnection;
+    private final SearchTask task;
+    private final SearchPhaseResults<Result> results;
+    private final long clusterStateVersion;
+    private final Map<String, AliasFilter> aliasFilter;
+    private final Map<String, Float> concreteIndexBoosts;
+    private final SetOnce<AtomicArray<ShardSearchFailure>> shardFailures = new SetOnce<>();
     private final Object shardFailuresMutex = new Object();
-    protected volatile ScoreDoc[] sortedShardDocs;
-
-    protected AbstractSearchAsyncAction(Logger logger, SearchTransportService searchTransportService, ClusterService clusterService,
-                                        IndexNameExpressionResolver indexNameExpressionResolver,
-                                        SearchPhaseController searchPhaseController, ThreadPool threadPool, SearchRequest request,
-                                        ActionListener<SearchResponse> listener) {
+    private final AtomicInteger successfulOps = new AtomicInteger();
+    private final AtomicInteger skippedOps = new AtomicInteger();
+    private final TransportSearchAction.SearchTimeProvider timeProvider;
+
+
+    protected AbstractSearchAsyncAction(String name, Logger logger, SearchTransportService searchTransportService,
+                                        BiFunction<String, String, Transport.Connection> nodeIdToConnection,
+                                        Map<String, AliasFilter> aliasFilter, Map<String, Float> concreteIndexBoosts,
+                                        Executor executor, SearchRequest request,
+                                        ActionListener<SearchResponse> listener, GroupShardsIterator<SearchShardIterator> shardsIts,
+                                        TransportSearchAction.SearchTimeProvider timeProvider, long clusterStateVersion,
+                                        SearchTask task, SearchPhaseResults<Result> resultConsumer) {
+        super(name, request, shardsIts, logger);
+        this.timeProvider = timeProvider;
         this.logger = logger;
         this.searchTransportService = searchTransportService;
-        this.indexNameExpressionResolver = indexNameExpressionResolver;
-        this.searchPhaseController = searchPhaseController;
-        this.threadPool = threadPool;
+        this.executor = executor;
         this.request = request;
+        this.task = task;
         this.listener = listener;
-
-        this.clusterState = clusterService.state();
-        nodes = clusterState.nodes();
-
-        clusterState.blocks().globalBlockedRaiseException(ClusterBlockLevel.READ);
-
-        // TODO: I think startTime() should become part of ActionRequest and that should be used both for index name
-        // date math expressions and $now in scripts. This way all apis will deal with now in the same way instead
-        // of just for the _search api
-        String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(clusterState, request.indicesOptions(),
-            startTime(), request.indices());
-
-        for (String index : concreteIndices) {
-            clusterState.blocks().indexBlockedRaiseException(ClusterBlockLevel.READ, index);
-        }
-
-        Map<String, Set<String>> routingMap = indexNameExpressionResolver.resolveSearchRouting(clusterState, request.routing(),
-            request.indices());
-
-        shardsIts = clusterService.operationRouting().searchShards(clusterState, concreteIndices, routingMap, request.preference());
-        final int shardCount = shardsIts.size();
-        failIfOverShardCountLimit(clusterService, shardCount);
-        expectedSuccessfulOps = shardCount;
-        // we need to add 1 for non active partition, since we count it in the total!
-        expectedTotalOps = shardsIts.totalSizeWith1ForEmpty();
-
-        firstResults = new AtomicArray<>(shardsIts.size());
+        this.nodeIdToConnection = nodeIdToConnection;
+        this.clusterStateVersion = clusterStateVersion;
+        this.concreteIndexBoosts = concreteIndexBoosts;
+        this.aliasFilter = aliasFilter;
+        this.results = resultConsumer;
     }
 
-    private void failIfOverShardCountLimit(ClusterService clusterService, int shardCount) {
-        final long shardCountLimit = clusterService.getClusterSettings().get(TransportSearchAction.SHARD_COUNT_LIMIT_SETTING);
-        if (shardCount > shardCountLimit) {
-            throw new IllegalArgumentException("Trying to query " + shardCount + " shards, which is over the limit of "
-                    + shardCountLimit + ". This limit exists because querying many shards at the same time can make the "
-                    + "job of the coordinating node very CPU and/or memory intensive. It is usually a better idea to "
-                    + "have a smaller number of larger shards. Update [" + TransportSearchAction.SHARD_COUNT_LIMIT_SETTING.getKey()
-                    + "] to a greater value if you really want to query that many shards at the same time.");
-        }
+    /**
+     * Builds how long it took to execute the search.
+     */
+    long buildTookInMillis() {
+        return TimeUnit.NANOSECONDS.toMillis(
+                timeProvider.getRelativeCurrentNanos() - timeProvider.getRelativeStartNanos());
     }
 
-    public void start() {
-        if (expectedSuccessfulOps == 0) {
+    /**
+     * This is the main entry point for a search. This method starts the search execution of the initial phase.
+     */
+    public final void start() {
+        if (getNumShards() == 0) {
             //no search shards to search on, bail with empty response
             //(it happens with search across _all with no indices around and consistent with broadcast operations)
-            listener.onResponse(new SearchResponse(InternalSearchResponse.empty(), null, 0, 0, buildTookInMillis(),
+            listener.onResponse(new SearchResponse(InternalSearchResponse.empty(), null, 0, 0, 0, buildTookInMillis(),
                 ShardSearchFailure.EMPTY_ARRAY));
             return;
         }
-        int shardIndex = -1;
-        for (final ShardIterator shardIt : shardsIts) {
-            shardIndex++;
-            final ShardRouting shard = shardIt.nextOrNull();
-            if (shard != null) {
-                performFirstPhase(shardIndex, shardIt, shard);
-            } else {
-                // really, no shards active in this group
-                onFirstPhaseResult(shardIndex, null, null, shardIt, new NoShardAvailableActionException(shardIt.shardId()));
-            }
-        }
+        executePhase(this);
     }
 
-    void performFirstPhase(final int shardIndex, final ShardIterator shardIt, final ShardRouting shard) {
-        if (shard == null) {
-            // no more active shards... (we should not really get here, but just for safety)
-            onFirstPhaseResult(shardIndex, null, null, shardIt, new NoShardAvailableActionException(shardIt.shardId()));
-        } else {
-            final DiscoveryNode node = nodes.get(shard.currentNodeId());
-            if (node == null) {
-                onFirstPhaseResult(shardIndex, shard, null, shardIt, new NoShardAvailableActionException(shardIt.shardId()));
-            } else {
-                String[] filteringAliases = indexNameExpressionResolver.filteringAliases(clusterState,
-                    shard.index().getName(), request.indices());
-                sendExecuteFirstPhase(node, internalSearchRequest(shard, shardsIts.size(), request, filteringAliases,
-                    startTime()), new ActionListener<FirstResult>() {
-                        @Override
-                        public void onResponse(FirstResult result) {
-                            onFirstPhaseResult(shardIndex, shard, result, shardIt);
-                        }
-
-                        @Override
-                        public void onFailure(Exception t) {
-                            onFirstPhaseResult(shardIndex, shard, node.getId(), shardIt, t);
-                        }
-                    });
+    @Override
+    public final void executeNextPhase(SearchPhase currentPhase, SearchPhase nextPhase) {
+        /* This is the main search phase transition where we move to the next phase. At this point we check if there is
+         * at least one successful operation left and if so we move to the next phase. If not we immediately fail the
+         * search phase as "all shards failed"*/
+        if (successfulOps.get() == 0) { // we have 0 successful results that means we shortcut stuff and return a failure
+            if (logger.isDebugEnabled()) {
+                final ShardOperationFailedException[] shardSearchFailures = ExceptionsHelper.groupBy(buildShardFailures());
+                Throwable cause = shardSearchFailures.length == 0 ? null :
+                    ElasticsearchException.guessRootCauses(shardSearchFailures[0].getCause())[0];
+                logger.debug((Supplier<?>) () -> new ParameterizedMessage("All shards failed for phase: [{}]", getName()),
+                    cause);
             }
-        }
-    }
-
-    void onFirstPhaseResult(int shardIndex, ShardRouting shard, FirstResult result, ShardIterator shardIt) {
-        result.shardTarget(new SearchShardTarget(shard.currentNodeId(), shard.index(), shard.id()));
-        processFirstPhaseResult(shardIndex, result);
-        // we need to increment successful ops first before we compare the exit condition otherwise if we
-        // are fast we could concurrently update totalOps but then preempt one of the threads which can
-        // cause the successor to read a wrong value from successfulOps if second phase is very fast ie. count etc.
-        successfulOps.incrementAndGet();
-        // increment all the "future" shards to update the total ops since we some may work and some may not...
-        // and when that happens, we break on total ops, so we must maintain them
-        final int xTotalOps = totalOps.addAndGet(shardIt.remaining() + 1);
-        if (xTotalOps == expectedTotalOps) {
-            try {
-                innerMoveToSecondPhase();
-            } catch (Exception e) {
-                if (logger.isDebugEnabled()) {
-                    logger.debug(
-                        (Supplier<?>) () -> new ParameterizedMessage(
-                            "{}: Failed to execute [{}] while moving to second phase",
-                            shardIt.shardId(),
-                            request),
-                        e);
-                }
-                raiseEarlyFailure(new ReduceSearchPhaseException(firstPhaseName(), "", e, buildShardFailures()));
+            onPhaseFailure(currentPhase, "all shards failed", null);
+        } else {
+            if (logger.isTraceEnabled()) {
+                final String resultsFrom = results.getSuccessfulResults()
+                    .map(r -> r.getSearchShardTarget().toString()).collect(Collectors.joining(","));
+                logger.trace("[{}] Moving to next phase: [{}], based on results from: {} (cluster state version: {})",
+                    currentPhase.getName(), nextPhase.getName(), resultsFrom, clusterStateVersion);
             }
-        } else if (xTotalOps > expectedTotalOps) {
-            raiseEarlyFailure(new IllegalStateException("unexpected higher total ops [" + xTotalOps + "] compared " +
-                "to expected [" + expectedTotalOps + "]"));
+            executePhase(nextPhase);
         }
     }
 
-    void onFirstPhaseResult(final int shardIndex, @Nullable ShardRouting shard, @Nullable String nodeId,
-                            final ShardIterator shardIt, Exception e) {
-        // we always add the shard failure for a specific shard instance
-        // we do make sure to clean it on a successful response from a shard
-        SearchShardTarget shardTarget = new SearchShardTarget(nodeId, shardIt.shardId().getIndex(), shardIt.shardId().getId());
-        addShardFailure(shardIndex, shardTarget, e);
-
-        if (totalOps.incrementAndGet() == expectedTotalOps) {
+    private void executePhase(SearchPhase phase) {
+        try {
+            phase.run();
+        } catch (Exception e) {
             if (logger.isDebugEnabled()) {
-                if (e != null && !TransportActions.isShardNotAvailableException(e)) {
-                    logger.debug(
-                        (Supplier<?>) () -> new ParameterizedMessage(
-                            "{}: Failed to execute [{}]",
-                            shard != null ? shard.shortSummary() :
-                                shardIt.shardId(),
-                            request),
-                        e);
-                } else if (logger.isTraceEnabled()) {
-                    logger.trace((Supplier<?>) () -> new ParameterizedMessage("{}: Failed to execute [{}]", shard, request), e);
-                }
-            }
-            final ShardSearchFailure[] shardSearchFailures = buildShardFailures();
-            if (successfulOps.get() == 0) {
-                if (logger.isDebugEnabled()) {
-                    logger.debug((Supplier<?>) () -> new ParameterizedMessage("All shards failed for phase: [{}]", firstPhaseName()), e);
-                }
-
-                // no successful ops, raise an exception
-                raiseEarlyFailure(new SearchPhaseExecutionException(firstPhaseName(), "all shards failed", e, shardSearchFailures));
-            } else {
-                try {
-                    innerMoveToSecondPhase();
-                } catch (Exception inner) {
-                    inner.addSuppressed(e);
-                    raiseEarlyFailure(new ReduceSearchPhaseException(firstPhaseName(), "", inner, shardSearchFailures));
-                }
-            }
-        } else {
-            final ShardRouting nextShard = shardIt.nextOrNull();
-            final boolean lastShard = nextShard == null;
-            // trace log this exception
-            logger.trace(
-                (Supplier<?>) () -> new ParameterizedMessage(
-                    "{}: Failed to execute [{}] lastShard [{}]",
-                    shard != null ? shard.shortSummary() : shardIt.shardId(),
-                    request,
-                    lastShard),
-                e);
-            if (!lastShard) {
-                try {
-                    performFirstPhase(shardIndex, shardIt, nextShard);
-                } catch (Exception inner) {
-                    inner.addSuppressed(e);
-                    onFirstPhaseResult(shardIndex, shard, shard.currentNodeId(), shardIt, inner);
-                }
-            } else {
-                // no more shards active, add a failure
-                if (logger.isDebugEnabled() && !logger.isTraceEnabled()) { // do not double log this exception
-                    if (e != null && !TransportActions.isShardNotAvailableException(e)) {
-                        logger.debug(
-                            (Supplier<?>) () -> new ParameterizedMessage(
-                                "{}: Failed to execute [{}] lastShard [{}]",
-                                shard != null ? shard.shortSummary() :
-                                    shardIt.shardId(),
-                                request,
-                                lastShard),
-                            e);
-                    }
-                }
+                logger.debug(
+                    (Supplier<?>) () -> new ParameterizedMessage(
+                        "Failed to execute [{}] while moving to [{}] phase", request, phase.getName()),
+                    e);
             }
+            onPhaseFailure(phase, "", e);
         }
     }
 
-    protected final ShardSearchFailure[] buildShardFailures() {
-        AtomicArray<ShardSearchFailure> shardFailures = this.shardFailures;
+
+    private ShardSearchFailure[] buildShardFailures() {
+        AtomicArray<ShardSearchFailure> shardFailures = this.shardFailures.get();
         if (shardFailures == null) {
             return ShardSearchFailure.EMPTY_ARRAY;
         }
-        List<AtomicArray.Entry<ShardSearchFailure>> entries = shardFailures.asList();
+        List<ShardSearchFailure> entries = shardFailures.asList();
         ShardSearchFailure[] failures = new ShardSearchFailure[entries.size()];
         for (int i = 0; i < failures.length; i++) {
-            failures[i] = entries.get(i).value;
+            failures[i] = entries.get(i);
         }
         return failures;
     }
 
-    protected final void addShardFailure(final int shardIndex, @Nullable SearchShardTarget shardTarget, Exception e) {
+    public final void onShardFailure(final int shardIndex, @Nullable SearchShardTarget shardTarget, Exception e) {
         // we don't aggregate shard failures on non active shards (but do keep the header counts right)
-        if (TransportActions.isShardNotAvailableException(e)) {
-            return;
-        }
-
-        // lazily create shard failures, so we can early build the empty shard failure list in most cases (no failures)
-        if (shardFailures == null) {
-            synchronized (shardFailuresMutex) {
-                if (shardFailures == null) {
-                    shardFailures = new AtomicArray<>(shardsIts.size());
+        if (TransportActions.isShardNotAvailableException(e) == false) {
+            AtomicArray<ShardSearchFailure> shardFailures = this.shardFailures.get();
+            // lazily create shard failures, so we can early build the empty shard failure list in most cases (no failures)
+            if (shardFailures == null) { // this is double checked locking but it's fine since SetOnce uses a volatile read internally
+                synchronized (shardFailuresMutex) {
+                    shardFailures = this.shardFailures.get(); // read again otherwise somebody else has created it?
+                    if (shardFailures == null) { // still null so we are the first and create a new instance
+                        shardFailures = new AtomicArray<>(getNumShards());
+                        this.shardFailures.set(shardFailures);
+                    }
                 }
             }
-        }
-        ShardSearchFailure failure = shardFailures.get(shardIndex);
-        if (failure == null) {
-            shardFailures.set(shardIndex, new ShardSearchFailure(e, shardTarget));
-        } else {
-            // the failure is already present, try and not override it with an exception that is less meaningless
-            // for example, getting illegal shard state
-            if (TransportActions.isReadOverrideException(e)) {
+            ShardSearchFailure failure = shardFailures.get(shardIndex);
+            if (failure == null) {
                 shardFailures.set(shardIndex, new ShardSearchFailure(e, shardTarget));
+            } else {
+                // the failure is already present, try and not override it with an exception that is less meaningless
+                // for example, getting illegal shard state
+                if (TransportActions.isReadOverrideException(e)) {
+                    shardFailures.set(shardIndex, new ShardSearchFailure(e, shardTarget));
+                }
+            }
+
+            if (results.hasResult(shardIndex)) {
+                assert failure == null : "shard failed before but shouldn't: " + failure;
+                successfulOps.decrementAndGet(); // if this shard was successful before (initial phase) we have to adjust the counter
             }
         }
+        results.consumeShardFailure(shardIndex);
     }
 
-    private void raiseEarlyFailure(Exception e) {
-        for (AtomicArray.Entry<FirstResult> entry : firstResults.asList()) {
+    /**
+     * This method should be called if a search phase failed to ensure all relevant search contexts and resources are released.
+     * this method will also notify the listener and sends back a failure to the user.
+     *
+     * @param exception the exception explaining or causing the phase failure
+     */
+    private void raisePhaseFailure(SearchPhaseExecutionException exception) {
+        results.getSuccessfulResults().forEach((entry) -> {
             try {
-                DiscoveryNode node = nodes.get(entry.value.shardTarget().nodeId());
-                sendReleaseSearchContext(entry.value.id(), node);
+                SearchShardTarget searchShardTarget = entry.getSearchShardTarget();
+                Transport.Connection connection = getConnection(null, searchShardTarget.getNodeId());
+                sendReleaseSearchContext(entry.getRequestId(), connection, searchShardTarget.getOriginalIndices());
             } catch (Exception inner) {
-                inner.addSuppressed(e);
+                inner.addSuppressed(exception);
                 logger.trace("failed to release context", inner);
             }
-        }
-        listener.onFailure(e);
+        });
+        listener.onFailure(exception);
     }
 
-    /**
-     * Releases shard targets that are not used in the docsIdsToLoad.
-     */
-    protected void releaseIrrelevantSearchContexts(AtomicArray<? extends QuerySearchResultProvider> queryResults,
-                                                   AtomicArray<IntArrayList> docIdsToLoad) {
-        if (docIdsToLoad == null) {
-            return;
+    @Override
+    public final void onShardSuccess(Result result) {
+        successfulOps.incrementAndGet();
+        results.consumeResult(result);
+        if (logger.isTraceEnabled()) {
+            logger.trace("got first-phase result from {}", result != null ? result.getSearchShardTarget() : null);
         }
-        // we only release search context that we did not fetch from if we are not scrolling
-        if (request.scroll() == null) {
-            for (AtomicArray.Entry<? extends QuerySearchResultProvider> entry : queryResults.asList()) {
-                QuerySearchResult queryResult = entry.value.queryResult();
-                if (queryResult.hasHits()
-                    && docIdsToLoad.get(entry.index) == null) { // but none of them made it to the global top docs
-                    try {
-                        DiscoveryNode node = nodes.get(entry.value.queryResult().shardTarget().nodeId());
-                        sendReleaseSearchContext(entry.value.queryResult().id(), node);
-                    } catch (Exception e) {
-                        logger.trace("failed to release context", e);
-                    }
-                }
-            }
+        // clean a previous error on this shard group (note, this code will be serialized on the same shardIndex value level
+        // so its ok concurrency wise to miss potentially the shard failures being created because of another failure
+        // in the #addShardFailure, because by definition, it will happen on *another* shardIndex
+        AtomicArray<ShardSearchFailure> shardFailures = this.shardFailures.get();
+        if (shardFailures != null) {
+            shardFailures.set(result.getShardIndex(), null);
         }
     }
 
-    protected void sendReleaseSearchContext(long contextId, DiscoveryNode node) {
-        if (node != null) {
-            searchTransportService.sendFreeContext(node, contextId, request);
-        }
+    @Override
+    public final void onPhaseDone() {
+        executeNextPhase(this, getNextPhase(results, this));
     }
 
-    protected ShardFetchSearchRequest createFetchRequest(QuerySearchResult queryResult, AtomicArray.Entry<IntArrayList> entry,
-                                                         ScoreDoc[] lastEmittedDocPerShard) {
-        final ScoreDoc lastEmittedDoc = (lastEmittedDocPerShard != null) ? lastEmittedDocPerShard[entry.index] : null;
-        return new ShardFetchSearchRequest(request, queryResult.id(), entry.value, lastEmittedDoc);
+    @Override
+    public final int getNumShards() {
+        return results.getNumShards();
     }
 
-    protected abstract void sendExecuteFirstPhase(DiscoveryNode node, ShardSearchTransportRequest request,
-                                                  ActionListener<FirstResult> listener);
+    @Override
+    public final Logger getLogger() {
+        return logger;
+    }
 
-    protected final void processFirstPhaseResult(int shardIndex, FirstResult result) {
-        firstResults.set(shardIndex, result);
+    @Override
+    public final SearchTask getTask() {
+        return task;
+    }
 
-        if (logger.isTraceEnabled()) {
-            logger.trace("got first-phase result from {}", result != null ? result.shardTarget() : null);
-        }
+    @Override
+    public final SearchRequest getRequest() {
+        return request;
+    }
 
-        // clean a previous error on this shard group (note, this code will be serialized on the same shardIndex value level
-        // so its ok concurrency wise to miss potentially the shard failures being created because of another failure
-        // in the #addShardFailure, because by definition, it will happen on *another* shardIndex
-        AtomicArray<ShardSearchFailure> shardFailures = this.shardFailures;
-        if (shardFailures != null) {
-            shardFailures.set(shardIndex, null);
-        }
+    @Override
+    public final SearchResponse buildSearchResponse(InternalSearchResponse internalSearchResponse, String scrollId) {
+        return new SearchResponse(internalSearchResponse, scrollId, getNumShards(), successfulOps.get(),
+            skippedOps.get(), buildTookInMillis(), buildShardFailures());
     }
 
-    final void innerMoveToSecondPhase() throws Exception {
-        if (logger.isTraceEnabled()) {
-            StringBuilder sb = new StringBuilder();
-            boolean hadOne = false;
-            for (int i = 0; i < firstResults.length(); i++) {
-                FirstResult result = firstResults.get(i);
-                if (result == null) {
-                    continue; // failure
-                }
-                if (hadOne) {
-                    sb.append(",");
-                } else {
-                    hadOne = true;
-                }
-                sb.append(result.shardTarget());
-            }
+    @Override
+    public final void onPhaseFailure(SearchPhase phase, String msg, Throwable cause) {
+        raisePhaseFailure(new SearchPhaseExecutionException(phase.getName(), msg, cause, buildShardFailures()));
+    }
 
-            logger.trace("Moving to second phase, based on results from: {} (cluster state version: {})", sb, clusterState.version());
-        }
-        moveToSecondPhase();
+    @Override
+    public final Transport.Connection getConnection(String clusterAlias, String nodeId) {
+        return nodeIdToConnection.apply(clusterAlias, nodeId);
+    }
+
+    @Override
+    public final SearchTransportService getSearchTransport() {
+        return searchTransportService;
     }
 
-    protected abstract void moveToSecondPhase() throws Exception;
+    @Override
+    public final void execute(Runnable command) {
+        executor.execute(command);
+    }
+
+    @Override
+    public final void onResponse(SearchResponse response) {
+        listener.onResponse(response);
+    }
 
-    protected abstract String firstPhaseName();
+    @Override
+    public final void onFailure(Exception e) {
+        listener.onFailure(e);
+    }
+
+    public final ShardSearchTransportRequest buildShardSearchRequest(SearchShardIterator shardIt) {
+        String clusterAlias = shardIt.getClusterAlias();
+        AliasFilter filter = aliasFilter.get(shardIt.shardId().getIndex().getUUID());
+        assert filter != null;
+        float indexBoost = concreteIndexBoosts.getOrDefault(shardIt.shardId().getIndex().getUUID(), DEFAULT_INDEX_BOOST);
+        return new ShardSearchTransportRequest(shardIt.getOriginalIndices(), request, shardIt.shardId(), getNumShards(),
+            filter, indexBoost, timeProvider.getAbsoluteStartMillis(), clusterAlias);
+    }
+
+    /**
+     * Returns the next phase based on the results of the initial search phase
+     * @param results the results of the initial search phase. Each non null element in the result array represent a successfully
+     *                executed shard request
+     * @param context the search context for the next phase
+     */
+    protected abstract SearchPhase getNextPhase(SearchPhaseResults<Result> results, SearchPhaseContext context);
+
+    @Override
+    protected void skipShard(SearchShardIterator iterator) {
+        successfulOps.incrementAndGet();
+        skippedOps.incrementAndGet();
+        super.skipShard(iterator);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java b/core/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java
new file mode 100644
index 0000000000000..ea5cf831859de
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.search;
+
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.util.FixedBitSet;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.cluster.routing.GroupShardsIterator;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.search.internal.AliasFilter;
+import org.elasticsearch.transport.Transport;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Executor;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+import java.util.stream.Stream;
+
+/**
+ * This search phrase can be used as an initial search phase to pre-filter search shards based on query rewriting.
+ * The queries are rewritten against the shards and based on the rewrite result shards might be able to be excluded
+ * from the search. The extra round trip to the search shards is very cheap and is not subject to rejections
+ * which allows to fan out to more shards at the same time without running into rejections even if we are hitting a
+ * large portion of the clusters indices.
+ */
+final class CanMatchPreFilterSearchPhase extends AbstractSearchAsyncAction<SearchTransportService.CanMatchResponse> {
+
+    private final Function<GroupShardsIterator<SearchShardIterator>, SearchPhase> phaseFactory;
+    private final GroupShardsIterator<SearchShardIterator> shardsIts;
+
+    CanMatchPreFilterSearchPhase(Logger logger, SearchTransportService searchTransportService,
+                                        BiFunction<String, String, Transport.Connection> nodeIdToConnection,
+                                        Map<String, AliasFilter> aliasFilter, Map<String, Float> concreteIndexBoosts,
+                                        Executor executor, SearchRequest request,
+                                        ActionListener<SearchResponse> listener, GroupShardsIterator<SearchShardIterator> shardsIts,
+                                        TransportSearchAction.SearchTimeProvider timeProvider, long clusterStateVersion,
+                                        SearchTask task, Function<GroupShardsIterator<SearchShardIterator>, SearchPhase> phaseFactory) {
+        super("can_match", logger, searchTransportService, nodeIdToConnection, aliasFilter, concreteIndexBoosts, executor, request,
+            listener,
+            shardsIts, timeProvider, clusterStateVersion, task, new BitSetSearchPhaseResults(shardsIts.size()));
+        this.phaseFactory = phaseFactory;
+        this.shardsIts = shardsIts;
+    }
+
+    @Override
+    protected void executePhaseOnShard(SearchShardIterator shardIt, ShardRouting shard,
+                                       SearchActionListener<SearchTransportService.CanMatchResponse> listener) {
+        getSearchTransport().sendCanMatch(getConnection(shardIt.getClusterAlias(), shard.currentNodeId()),
+            buildShardSearchRequest(shardIt), getTask(), listener);
+    }
+
+    @Override
+    protected SearchPhase getNextPhase(SearchPhaseResults<SearchTransportService.CanMatchResponse> results,
+                                       SearchPhaseContext context) {
+
+        return phaseFactory.apply(getIterator((BitSetSearchPhaseResults) results, shardsIts));
+    }
+
+    private GroupShardsIterator<SearchShardIterator> getIterator(BitSetSearchPhaseResults results,
+                                                                 GroupShardsIterator<SearchShardIterator> shardsIts) {
+        int cardinality = results.getNumPossibleMatches();
+        FixedBitSet possibleMatches = results.getPossibleMatches();
+        if (cardinality == 0) {
+            // this is a special case where we have no hit but we need to get at least one search response in order
+            // to produce a valid search result with all the aggs etc.
+            possibleMatches.set(0);
+        }
+        int i = 0;
+        for (SearchShardIterator iter : shardsIts) {
+            if (possibleMatches.get(i++)) {
+                iter.reset();
+            } else {
+                iter.resetAndSkip();
+            }
+        }
+        return shardsIts;
+    }
+
+    private static final class BitSetSearchPhaseResults extends InitialSearchPhase.
+        SearchPhaseResults<SearchTransportService.CanMatchResponse> {
+
+        private final FixedBitSet possibleMatches;
+        private int numPossibleMatches;
+
+        BitSetSearchPhaseResults(int size) {
+            super(size);
+            possibleMatches = new FixedBitSet(size);
+        }
+
+        @Override
+        void consumeResult(SearchTransportService.CanMatchResponse result) {
+            if (result.canMatch()) {
+                consumeShardFailure(result.getShardIndex());
+            }
+        }
+
+        @Override
+        boolean hasResult(int shardIndex) {
+            return false; // unneeded
+        }
+
+        @Override
+        synchronized void consumeShardFailure(int shardIndex) {
+            // we have to carry over shard failures in order to account for them in the response.
+            possibleMatches.set(shardIndex);
+            numPossibleMatches++;
+        }
+
+
+        synchronized int getNumPossibleMatches() {
+            return numPossibleMatches;
+        }
+
+        synchronized FixedBitSet getPossibleMatches() {
+            return possibleMatches;
+        }
+
+        @Override
+        Stream<SearchTransportService.CanMatchResponse> getSuccessfulResults() {
+            return Stream.empty();
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/search/ClearScrollController.java b/core/src/main/java/org/elasticsearch/action/search/ClearScrollController.java
new file mode 100644
index 0000000000000..ac708d9b6b0c7
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/search/ClearScrollController.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.search;
+
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.apache.logging.log4j.util.Supplier;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.util.concurrent.CountDown;
+import org.elasticsearch.transport.Transport;
+import org.elasticsearch.transport.TransportResponse;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.elasticsearch.action.search.TransportSearchHelper.parseScrollId;
+
+final class ClearScrollController implements Runnable {
+    private final DiscoveryNodes nodes;
+    private final SearchTransportService searchTransportService;
+    private final CountDown expectedOps;
+    private final ActionListener<ClearScrollResponse> listener;
+    private final AtomicBoolean hasFailed = new AtomicBoolean(false);
+    private final AtomicInteger freedSearchContexts = new AtomicInteger(0);
+    private final Logger logger;
+    private final Runnable runner;
+
+    ClearScrollController(ClearScrollRequest request, ActionListener<ClearScrollResponse> listener, DiscoveryNodes nodes, Logger logger,
+                          SearchTransportService searchTransportService) {
+        this.nodes = nodes;
+        this.logger = logger;
+        this.searchTransportService = searchTransportService;
+        this.listener = listener;
+        List<String> scrollIds = request.getScrollIds();
+        final int expectedOps;
+        if (scrollIds.size() == 1 && "_all".equals(scrollIds.get(0))) {
+            expectedOps = nodes.getSize();
+            runner = this::cleanAllScrolls;
+        } else {
+            List<ScrollIdForNode> parsedScrollIds = new ArrayList<>();
+            for (String parsedScrollId : request.getScrollIds()) {
+                ScrollIdForNode[] context = parseScrollId(parsedScrollId).getContext();
+                for (ScrollIdForNode id : context) {
+                    parsedScrollIds.add(id);
+                }
+            }
+            if (parsedScrollIds.isEmpty()) {
+                expectedOps = 0;
+                runner = () -> listener.onResponse(new ClearScrollResponse(true, 0));
+            } else {
+                expectedOps = parsedScrollIds.size();
+                runner = () -> cleanScrollIds(parsedScrollIds);
+            }
+        }
+        this.expectedOps = new CountDown(expectedOps);
+
+    }
+
+    @Override
+    public void run() {
+        runner.run();
+    }
+
+    void cleanAllScrolls() {
+        for (final DiscoveryNode node : nodes) {
+            try {
+                Transport.Connection connection = searchTransportService.getConnection(null, node);
+                searchTransportService.sendClearAllScrollContexts(connection, new ActionListener<TransportResponse>() {
+                    @Override
+                    public void onResponse(TransportResponse response) {
+                        onFreedContext(true);
+                    }
+
+                    @Override
+                    public void onFailure(Exception e) {
+                        onFailedFreedContext(e, node);
+                    }
+                });
+            } catch (Exception e) {
+                onFailedFreedContext(e, node);
+            }
+        }
+    }
+
+    void cleanScrollIds(List<ScrollIdForNode> parsedScrollIds) {
+        SearchScrollAsyncAction.collectNodesAndRun(parsedScrollIds, nodes, searchTransportService, ActionListener.wrap(
+            lookup -> {
+                for (ScrollIdForNode target : parsedScrollIds) {
+                    final DiscoveryNode node = lookup.apply(target.getClusterAlias(), target.getNode());
+                    if (node == null) {
+                        onFreedContext(false);
+                    } else {
+                        try {
+                            Transport.Connection connection = searchTransportService.getConnection(target.getClusterAlias(), node);
+                            searchTransportService.sendFreeContext(connection, target.getScrollId(),
+                                ActionListener.wrap(freed -> onFreedContext(freed.isFreed()), e -> onFailedFreedContext(e, node)));
+                        } catch (Exception e) {
+                            onFailedFreedContext(e, node);
+                        }
+                    }
+                }
+            }, listener::onFailure));
+    }
+
+    private void onFreedContext(boolean freed) {
+        if (freed) {
+            freedSearchContexts.incrementAndGet();
+        }
+        if (expectedOps.countDown()) {
+            boolean succeeded = hasFailed.get() == false;
+            listener.onResponse(new ClearScrollResponse(succeeded, freedSearchContexts.get()));
+        }
+    }
+
+    private void onFailedFreedContext(Throwable e, DiscoveryNode node) {
+        logger.warn((Supplier<?>) () -> new ParameterizedMessage("Clear SC failed on node[{}]", node), e);
+        if (expectedOps.countDown()) {
+            listener.onResponse(new ClearScrollResponse(false, freedSearchContexts.get()));
+        } else {
+            hasFailed.set(true);
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/search/ClearScrollRequest.java b/core/src/main/java/org/elasticsearch/action/search/ClearScrollRequest.java
index 17343e8691255..4770818867c84 100644
--- a/core/src/main/java/org/elasticsearch/action/search/ClearScrollRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/search/ClearScrollRequest.java
@@ -23,6 +23,9 @@
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -31,9 +34,7 @@
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 
-/**
- */
-public class ClearScrollRequest extends ActionRequest<ClearScrollRequest> {
+public class ClearScrollRequest extends ActionRequest implements ToXContentObject {
 
     private List<String> scrollIds;
 
@@ -85,4 +86,47 @@ public void writeTo(StreamOutput out) throws IOException {
         }
     }
 
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.startArray("scroll_id");
+        for (String scrollId : scrollIds) {
+            builder.value(scrollId);
+        }
+        builder.endArray();
+        builder.endObject();
+        return builder;
+    }
+
+    public void fromXContent(XContentParser parser) throws IOException {
+        scrollIds = null;
+        if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
+            throw new IllegalArgumentException("Malformed content, must start with an object");
+        } else {
+            XContentParser.Token token;
+            String currentFieldName = null;
+            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                if (token == XContentParser.Token.FIELD_NAME) {
+                    currentFieldName = parser.currentName();
+                } else if ("scroll_id".equals(currentFieldName)){
+                    if (token == XContentParser.Token.START_ARRAY) {
+                        while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                            if (token.isValue() == false) {
+                                throw new IllegalArgumentException("scroll_id array element should only contain scroll_id");
+                            }
+                            addScrollId(parser.text());
+                        }
+                    } else {
+                        if (token.isValue() == false) {
+                            throw new IllegalArgumentException("scroll_id element should only contain scroll_id");
+                        }
+                        addScrollId(parser.text());
+                    }
+                } else {
+                    throw new IllegalArgumentException("Unknown parameter [" + currentFieldName
+                            + "] in request body or parameter is of the wrong type[" + token + "] ");
+                }
+            }
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/search/ClearScrollResponse.java b/core/src/main/java/org/elasticsearch/action/search/ClearScrollResponse.java
index 0887d2681992d..d5e2c754a2000 100644
--- a/core/src/main/java/org/elasticsearch/action/search/ClearScrollResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/search/ClearScrollResponse.java
@@ -20,20 +20,33 @@
 package org.elasticsearch.action.search;
 
 import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.StatusToXContent;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.StatusToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
 
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
 import static org.elasticsearch.rest.RestStatus.NOT_FOUND;
 import static org.elasticsearch.rest.RestStatus.OK;
 
-/**
- */
-public class ClearScrollResponse extends ActionResponse implements StatusToXContent {
+public class ClearScrollResponse extends ActionResponse implements StatusToXContentObject {
+
+    private static final ParseField SUCCEEDED = new ParseField("succeeded");
+    private static final ParseField NUMFREED = new ParseField("num_freed");
+
+    private static final ConstructingObjectParser<ClearScrollResponse, Void> PARSER = new ConstructingObjectParser<>("clear_scroll",
+            true, a -> new ClearScrollResponse((boolean)a[0], (int)a[1]));
+    static {
+        PARSER.declareField(constructorArg(), (parser, context) -> parser.booleanValue(), SUCCEEDED, ObjectParser.ValueType.BOOLEAN);
+        PARSER.declareField(constructorArg(), (parser, context) -> parser.intValue(), NUMFREED, ObjectParser.ValueType.INT);
+    }
 
     private boolean succeeded;
     private int numFreed;
@@ -68,11 +81,20 @@ public RestStatus status() {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.field(Fields.SUCCEEDED, succeeded);
-        builder.field(Fields.NUMFREED, numFreed);
+        builder.startObject();
+        builder.field(SUCCEEDED.getPreferredName(), succeeded);
+        builder.field(NUMFREED.getPreferredName(), numFreed);
+        builder.endObject();
         return builder;
     }
 
+    /**
+     * Parse the clear scroll response body into a new {@link ClearScrollResponse} object
+     */
+    public static ClearScrollResponse fromXContent(XContentParser parser) throws IOException {
+        return PARSER.apply(parser, null);
+    }
+
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
@@ -86,10 +108,4 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeBoolean(succeeded);
         out.writeVInt(numFreed);
     }
-
-    static final class Fields {
-        static final String SUCCEEDED = "succeeded";
-        static final String NUMFREED = "num_freed";
-    }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/action/search/CountedCollector.java b/core/src/main/java/org/elasticsearch/action/search/CountedCollector.java
new file mode 100644
index 0000000000000..2dd255aa14c69
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/search/CountedCollector.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.search;
+
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.util.concurrent.CountDown;
+import org.elasticsearch.search.SearchPhaseResult;
+import org.elasticsearch.search.SearchShardTarget;
+
+import java.util.function.Consumer;
+
+/**
+ * This is a simple base class to simplify fan out to shards and collect their results. Each results passed to
+ * {@link #onResult(SearchPhaseResult)} will be set to the provided result array
+ * where the given index is used to set the result on the array.
+ */
+final class CountedCollector<R extends SearchPhaseResult> {
+    private final Consumer<R> resultConsumer;
+    private final CountDown counter;
+    private final Runnable onFinish;
+    private final SearchPhaseContext context;
+
+    CountedCollector(Consumer<R> resultConsumer, int expectedOps, Runnable onFinish, SearchPhaseContext context) {
+        this.resultConsumer = resultConsumer;
+        this.counter = new CountDown(expectedOps);
+        this.onFinish = onFinish;
+        this.context = context;
+    }
+
+    /**
+     * Forcefully counts down an operation and executes the provided runnable
+     * if all expected operations where executed
+     */
+    void countDown() {
+        assert counter.isCountedDown() == false : "more operations executed than specified";
+        if (counter.countDown()) {
+            onFinish.run();
+        }
+    }
+
+    /**
+     * Sets the result to the given array index and then runs {@link #countDown()}
+     */
+    void onResult(R result) {
+        try {
+            resultConsumer.accept(result);
+        } finally {
+            countDown();
+        }
+    }
+
+    /**
+     * Escalates the failure via {@link SearchPhaseContext#onShardFailure(int, SearchShardTarget, Exception)}
+     * and then runs {@link #countDown()}
+     */
+    void onFailure(final int shardIndex, @Nullable SearchShardTarget shardTarget, Exception e) {
+        try {
+            context.onShardFailure(shardIndex, shardTarget, e);
+        } finally {
+            countDown();
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java b/core/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java
new file mode 100644
index 0000000000000..db0425db7c320
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.search;
+
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.apache.logging.log4j.util.Supplier;
+import org.elasticsearch.common.util.concurrent.AtomicArray;
+import org.elasticsearch.search.SearchPhaseResult;
+import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.search.dfs.AggregatedDfs;
+import org.elasticsearch.search.dfs.DfsSearchResult;
+import org.elasticsearch.search.query.QuerySearchRequest;
+import org.elasticsearch.search.query.QuerySearchResult;
+import org.elasticsearch.transport.Transport;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.function.Function;
+
+/**
+ * This search phase fans out to every shards to execute a distributed search with a pre-collected distributed frequencies for all
+ * search terms used in the actual search query. This phase is very similar to a the default query-then-fetch search phase but it doesn't
+ * retry on another shard if any of the shards are failing. Failures are treated as shard failures and are counted as a non-successful
+ * operation.
+ * @see CountedCollector#onFailure(int, SearchShardTarget, Exception)
+ */
+final class DfsQueryPhase extends SearchPhase {
+    private final InitialSearchPhase.ArraySearchPhaseResults<SearchPhaseResult> queryResult;
+    private final SearchPhaseController searchPhaseController;
+    private final AtomicArray<DfsSearchResult> dfsSearchResults;
+    private final Function<InitialSearchPhase.ArraySearchPhaseResults<SearchPhaseResult>, SearchPhase> nextPhaseFactory;
+    private final SearchPhaseContext context;
+    private final SearchTransportService searchTransportService;
+
+    DfsQueryPhase(AtomicArray<DfsSearchResult> dfsSearchResults,
+                  SearchPhaseController searchPhaseController,
+                  Function<InitialSearchPhase.ArraySearchPhaseResults<SearchPhaseResult>, SearchPhase> nextPhaseFactory,
+                  SearchPhaseContext context) {
+        super("dfs_query");
+        this.queryResult = searchPhaseController.newSearchPhaseResults(context.getRequest(), context.getNumShards());
+        this.searchPhaseController = searchPhaseController;
+        this.dfsSearchResults = dfsSearchResults;
+        this.nextPhaseFactory = nextPhaseFactory;
+        this.context = context;
+        this.searchTransportService = context.getSearchTransport();
+    }
+
+    @Override
+    public void run() throws IOException {
+        // TODO we can potentially also consume the actual per shard results from the initial phase here in the aggregateDfs
+        // to free up memory early
+        final List<DfsSearchResult> resultList = dfsSearchResults.asList();
+        final AggregatedDfs dfs = searchPhaseController.aggregateDfs(resultList);
+        final CountedCollector<SearchPhaseResult> counter = new CountedCollector<>(queryResult::consumeResult,
+            resultList.size(),
+            () -> context.executeNextPhase(this, nextPhaseFactory.apply(queryResult)), context);
+        for (final DfsSearchResult dfsResult : resultList) {
+            final SearchShardTarget searchShardTarget = dfsResult.getSearchShardTarget();
+            Transport.Connection connection = context.getConnection(searchShardTarget.getClusterAlias(), searchShardTarget.getNodeId());
+            QuerySearchRequest querySearchRequest = new QuerySearchRequest(searchShardTarget.getOriginalIndices(),
+                    dfsResult.getRequestId(), dfs);
+            final int shardIndex = dfsResult.getShardIndex();
+            searchTransportService.sendExecuteQuery(connection, querySearchRequest, context.getTask(),
+                new SearchActionListener<QuerySearchResult>(searchShardTarget, shardIndex) {
+
+                    @Override
+                    protected void innerOnResponse(QuerySearchResult response) {
+                        counter.onResult(response);
+                    }
+
+                    @Override
+                    public void onFailure(Exception exception) {
+                        try {
+                            if (context.getLogger().isDebugEnabled()) {
+                                context.getLogger().debug((Supplier<?>) () -> new ParameterizedMessage("[{}] Failed to execute query phase",
+                                    querySearchRequest.id()), exception);
+                            }
+                            counter.onFailure(shardIndex, searchShardTarget, exception);
+                        } finally {
+                            // the query might not have been executed at all (for example because thread pool rejected
+                            // execution) and the search context that was created in dfs phase might not be released.
+                            // release it again to be in the safe side
+                            context.sendReleaseSearchContext(querySearchRequest.id(), connection, searchShardTarget.getOriginalIndices());
+                        }
+                    }
+                });
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java b/core/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java
new file mode 100644
index 0000000000000..bc673644a0683
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.search;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.InnerHitBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.collapse.CollapseBuilder;
+import org.elasticsearch.search.internal.InternalSearchResponse;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.function.Function;
+
+/**
+ * This search phase is an optional phase that will be executed once all hits are fetched from the shards that executes
+ * field-collapsing on the inner hits. This phase only executes if field collapsing is requested in the search request and otherwise
+ * forwards to the next phase immediately.
+ */
+final class ExpandSearchPhase extends SearchPhase {
+    private final SearchPhaseContext context;
+    private final InternalSearchResponse searchResponse;
+    private final Function<InternalSearchResponse, SearchPhase> nextPhaseFactory;
+
+    ExpandSearchPhase(SearchPhaseContext context, InternalSearchResponse searchResponse,
+                      Function<InternalSearchResponse, SearchPhase> nextPhaseFactory) {
+        super("expand");
+        this.context = context;
+        this.searchResponse = searchResponse;
+        this.nextPhaseFactory = nextPhaseFactory;
+    }
+
+    /**
+     * Returns <code>true</code> iff the search request has inner hits and needs field collapsing
+     */
+    private boolean isCollapseRequest() {
+        final SearchRequest searchRequest = context.getRequest();
+        return searchRequest.source() != null &&
+            searchRequest.source().collapse() != null &&
+            searchRequest.source().collapse().getInnerHits().isEmpty() == false;
+    }
+
+    @Override
+    public void run() throws IOException {
+        if (isCollapseRequest() && searchResponse.hits().getHits().length > 0) {
+            SearchRequest searchRequest = context.getRequest();
+            CollapseBuilder collapseBuilder = searchRequest.source().collapse();
+            final List<InnerHitBuilder> innerHitBuilders = collapseBuilder.getInnerHits();
+            MultiSearchRequest multiRequest = new MultiSearchRequest();
+            if (collapseBuilder.getMaxConcurrentGroupRequests() > 0) {
+                multiRequest.maxConcurrentSearchRequests(collapseBuilder.getMaxConcurrentGroupRequests());
+            }
+            for (SearchHit hit : searchResponse.hits().getHits()) {
+                BoolQueryBuilder groupQuery = new BoolQueryBuilder();
+                Object collapseValue = hit.field(collapseBuilder.getField()).getValue();
+                if (collapseValue != null) {
+                    groupQuery.filter(QueryBuilders.matchQuery(collapseBuilder.getField(), collapseValue));
+                } else {
+                    groupQuery.mustNot(QueryBuilders.existsQuery(collapseBuilder.getField()));
+                }
+                QueryBuilder origQuery = searchRequest.source().query();
+                if (origQuery != null) {
+                    groupQuery.must(origQuery);
+                }
+                for (InnerHitBuilder innerHitBuilder : innerHitBuilders) {
+                    SearchSourceBuilder sourceBuilder = buildExpandSearchSourceBuilder(innerHitBuilder)
+                        .query(groupQuery);
+                    SearchRequest groupRequest = new SearchRequest(searchRequest.indices())
+                        .types(searchRequest.types())
+                        .source(sourceBuilder);
+                    multiRequest.add(groupRequest);
+                }
+            }
+            context.getSearchTransport().sendExecuteMultiSearch(multiRequest, context.getTask(),
+                ActionListener.wrap(response -> {
+                    Iterator<MultiSearchResponse.Item> it = response.iterator();
+                    for (SearchHit hit : searchResponse.hits.getHits()) {
+                        for (InnerHitBuilder innerHitBuilder : innerHitBuilders) {
+                            MultiSearchResponse.Item item = it.next();
+                            if (item.isFailure()) {
+                                context.onPhaseFailure(this, "failed to expand hits", item.getFailure());
+                                return;
+                            }
+                            SearchHits innerHits = item.getResponse().getHits();
+                            if (hit.getInnerHits() == null) {
+                                hit.setInnerHits(new HashMap<>(innerHitBuilders.size()));
+                            }
+                            hit.getInnerHits().put(innerHitBuilder.getName(), innerHits);
+                        }
+                    }
+                    context.executeNextPhase(this, nextPhaseFactory.apply(searchResponse));
+                }, context::onFailure)
+            );
+        } else {
+            context.executeNextPhase(this, nextPhaseFactory.apply(searchResponse));
+        }
+    }
+
+    private SearchSourceBuilder buildExpandSearchSourceBuilder(InnerHitBuilder options) {
+        SearchSourceBuilder groupSource = new SearchSourceBuilder();
+        groupSource.from(options.getFrom());
+        groupSource.size(options.getSize());
+        if (options.getSorts() != null) {
+            options.getSorts().forEach(groupSource::sort);
+        }
+        if (options.getFetchSourceContext() != null) {
+            if (options.getFetchSourceContext().includes() == null && options.getFetchSourceContext().excludes() == null) {
+                groupSource.fetchSource(options.getFetchSourceContext().fetchSource());
+            } else {
+                groupSource.fetchSource(options.getFetchSourceContext().includes(),
+                    options.getFetchSourceContext().excludes());
+            }
+        }
+        if (options.getDocValueFields() != null) {
+            options.getDocValueFields().forEach(groupSource::docValueField);
+        }
+        if (options.getStoredFieldsContext() != null && options.getStoredFieldsContext().fieldNames() != null) {
+            options.getStoredFieldsContext().fieldNames().forEach(groupSource::storedField);
+        }
+        if (options.getScriptFields() != null) {
+            for (SearchSourceBuilder.ScriptField field : options.getScriptFields()) {
+                groupSource.scriptField(field.fieldName(), field.script());
+            }
+        }
+        if (options.getHighlightBuilder() != null) {
+            groupSource.highlighter(options.getHighlightBuilder());
+        }
+        groupSource.explain(options.isExplain());
+        groupSource.trackScores(options.isTrackScores());
+        return groupSource;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java b/core/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java
new file mode 100644
index 0000000000000..4712496bc37ec
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java
@@ -0,0 +1,220 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.search;
+
+import com.carrotsearch.hppc.IntArrayList;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.apache.logging.log4j.util.Supplier;
+import org.apache.lucene.search.ScoreDoc;
+import org.elasticsearch.action.ActionRunnable;
+import org.elasticsearch.action.OriginalIndices;
+import org.elasticsearch.common.util.concurrent.AtomicArray;
+import org.elasticsearch.search.SearchPhaseResult;
+import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.search.fetch.FetchSearchResult;
+import org.elasticsearch.search.fetch.ShardFetchSearchRequest;
+import org.elasticsearch.search.internal.InternalSearchResponse;
+import org.elasticsearch.search.query.QuerySearchResult;
+import org.elasticsearch.transport.Transport;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.function.BiFunction;
+
+/**
+ * This search phase merges the query results from the previous phase together and calculates the topN hits for this search.
+ * Then it reaches out to all relevant shards to fetch the topN hits.
+ */
+final class FetchSearchPhase extends SearchPhase {
+    private final AtomicArray<FetchSearchResult> fetchResults;
+    private final SearchPhaseController searchPhaseController;
+    private final AtomicArray<SearchPhaseResult> queryResults;
+    private final BiFunction<InternalSearchResponse, String, SearchPhase> nextPhaseFactory;
+    private final SearchPhaseContext context;
+    private final Logger logger;
+    private final InitialSearchPhase.SearchPhaseResults<SearchPhaseResult> resultConsumer;
+
+    FetchSearchPhase(InitialSearchPhase.SearchPhaseResults<SearchPhaseResult> resultConsumer,
+                     SearchPhaseController searchPhaseController,
+                     SearchPhaseContext context) {
+        this(resultConsumer, searchPhaseController, context,
+            (response, scrollId) -> new ExpandSearchPhase(context, response, // collapse only happens if the request has inner hits
+                (finalResponse) -> sendResponsePhase(finalResponse, scrollId, context)));
+    }
+
+    FetchSearchPhase(InitialSearchPhase.SearchPhaseResults<SearchPhaseResult> resultConsumer,
+                     SearchPhaseController searchPhaseController,
+                     SearchPhaseContext context, BiFunction<InternalSearchResponse, String, SearchPhase> nextPhaseFactory) {
+        super("fetch");
+        if (context.getNumShards() != resultConsumer.getNumShards()) {
+            throw new IllegalStateException("number of shards must match the length of the query results but doesn't:"
+                + context.getNumShards() + "!=" + resultConsumer.getNumShards());
+        }
+        this.fetchResults = new AtomicArray<>(resultConsumer.getNumShards());
+        this.searchPhaseController = searchPhaseController;
+        this.queryResults = resultConsumer.getAtomicArray();
+        this.nextPhaseFactory =  nextPhaseFactory;
+        this.context = context;
+        this.logger = context.getLogger();
+        this.resultConsumer = resultConsumer;
+    }
+
+    @Override
+    public void run() throws IOException {
+        context.execute(new ActionRunnable<SearchResponse>(context) {
+            @Override
+            public void doRun() throws IOException {
+                // we do the heavy lifting in this inner run method where we reduce aggs etc. that's why we fork this phase
+                // off immediately instead of forking when we send back the response to the user since there we only need
+                // to merge together the fetched results which is a linear operation.
+                innerRun();
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                context.onPhaseFailure(FetchSearchPhase.this, "", e);
+            }
+        });
+    }
+
+    private void innerRun() throws IOException {
+        final int numShards = context.getNumShards();
+        final boolean isScrollSearch = context.getRequest().scroll() != null;
+        List<SearchPhaseResult> phaseResults = queryResults.asList();
+        String scrollId = isScrollSearch ? TransportSearchHelper.buildScrollId(queryResults) : null;
+        final SearchPhaseController.ReducedQueryPhase reducedQueryPhase = resultConsumer.reduce();
+        final boolean queryAndFetchOptimization = queryResults.length() == 1;
+        final Runnable finishPhase = ()
+            -> moveToNextPhase(searchPhaseController, scrollId, reducedQueryPhase, queryAndFetchOptimization ?
+            queryResults : fetchResults);
+        if (queryAndFetchOptimization) {
+            assert phaseResults.isEmpty() || phaseResults.get(0).fetchResult() != null : "phaseResults empty [" + phaseResults.isEmpty()
+                + "], single result: " +  phaseResults.get(0).fetchResult();
+            // query AND fetch optimization
+            finishPhase.run();
+        } else {
+            final IntArrayList[] docIdsToLoad = searchPhaseController.fillDocIdsToLoad(numShards, reducedQueryPhase.scoreDocs);
+            if (reducedQueryPhase.scoreDocs.length == 0) { // no docs to fetch -- sidestep everything and return
+                phaseResults.stream()
+                    .map(SearchPhaseResult::queryResult)
+                    .forEach(this::releaseIrrelevantSearchContext); // we have to release contexts here to free up resources
+                finishPhase.run();
+            } else {
+                final ScoreDoc[] lastEmittedDocPerShard = isScrollSearch ?
+                    searchPhaseController.getLastEmittedDocPerShard(reducedQueryPhase, numShards)
+                    : null;
+                final CountedCollector<FetchSearchResult> counter = new CountedCollector<>(r -> fetchResults.set(r.getShardIndex(), r),
+                    docIdsToLoad.length, // we count down every shard in the result no matter if we got any results or not
+                    finishPhase, context);
+                for (int i = 0; i < docIdsToLoad.length; i++) {
+                    IntArrayList entry = docIdsToLoad[i];
+                    SearchPhaseResult queryResult = queryResults.get(i);
+                    if (entry == null) { // no results for this shard ID
+                        if (queryResult != null) {
+                            // if we got some hits from this shard we have to release the context there
+                            // we do this as we go since it will free up resources and passing on the request on the
+                            // transport layer is cheap.
+                            releaseIrrelevantSearchContext(queryResult.queryResult());
+                        }
+                        // in any case we count down this result since we don't talk to this shard anymore
+                        counter.countDown();
+                    } else {
+                        SearchShardTarget searchShardTarget = queryResult.getSearchShardTarget();
+                        Transport.Connection connection = context.getConnection(searchShardTarget.getClusterAlias(),
+                            searchShardTarget.getNodeId());
+                        ShardFetchSearchRequest fetchSearchRequest = createFetchRequest(queryResult.queryResult().getRequestId(), i, entry,
+                            lastEmittedDocPerShard, searchShardTarget.getOriginalIndices());
+                        executeFetch(i, searchShardTarget, counter, fetchSearchRequest, queryResult.queryResult(),
+                            connection);
+                    }
+                }
+            }
+        }
+    }
+
+    protected ShardFetchSearchRequest createFetchRequest(long queryId, int index, IntArrayList entry,
+                                                               ScoreDoc[] lastEmittedDocPerShard, OriginalIndices originalIndices) {
+        final ScoreDoc lastEmittedDoc = (lastEmittedDocPerShard != null) ? lastEmittedDocPerShard[index] : null;
+        return new ShardFetchSearchRequest(originalIndices, queryId, entry, lastEmittedDoc);
+    }
+
+    private void executeFetch(final int shardIndex, final SearchShardTarget shardTarget,
+                              final CountedCollector<FetchSearchResult> counter,
+                              final ShardFetchSearchRequest fetchSearchRequest, final QuerySearchResult querySearchResult,
+                              final Transport.Connection connection) {
+        context.getSearchTransport().sendExecuteFetch(connection, fetchSearchRequest, context.getTask(),
+            new SearchActionListener<FetchSearchResult>(shardTarget, shardIndex) {
+                @Override
+                public void innerOnResponse(FetchSearchResult result) {
+                    counter.onResult(result);
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    try {
+                        if (logger.isDebugEnabled()) {
+                            logger.debug((Supplier<?>) () -> new ParameterizedMessage("[{}] Failed to execute fetch phase",
+                                fetchSearchRequest.id()), e);
+                        }
+                        counter.onFailure(shardIndex, shardTarget, e);
+                    } finally {
+                        // the search context might not be cleared on the node where the fetch was executed for example
+                        // because the action was rejected by the thread pool. in this case we need to send a dedicated
+                        // request to clear the search context.
+                        releaseIrrelevantSearchContext(querySearchResult);
+                    }
+                }
+            });
+    }
+
+    /**
+     * Releases shard targets that are not used in the docsIdsToLoad.
+     */
+    private void releaseIrrelevantSearchContext(QuerySearchResult queryResult) {
+        // we only release search context that we did not fetch from if we are not scrolling
+        // and if it has at lease one hit that didn't make it to the global topDocs
+        if (context.getRequest().scroll() == null && queryResult.hasSearchContext()) {
+            try {
+                SearchShardTarget searchShardTarget = queryResult.getSearchShardTarget();
+                Transport.Connection connection = context.getConnection(searchShardTarget.getClusterAlias(), searchShardTarget.getNodeId());
+                context.sendReleaseSearchContext(queryResult.getRequestId(), connection, searchShardTarget.getOriginalIndices());
+            } catch (Exception e) {
+                context.getLogger().trace("failed to release context", e);
+            }
+        }
+    }
+
+    private void moveToNextPhase(SearchPhaseController searchPhaseController,
+                                 String scrollId, SearchPhaseController.ReducedQueryPhase reducedQueryPhase,
+                                 AtomicArray<? extends SearchPhaseResult> fetchResultsArr) {
+        final InternalSearchResponse internalResponse = searchPhaseController.merge(context.getRequest().scroll() != null,
+            reducedQueryPhase, fetchResultsArr.asList(), fetchResultsArr::get);
+        context.executeNextPhase(this, nextPhaseFactory.apply(internalResponse, scrollId));
+    }
+
+    private static SearchPhase sendResponsePhase(InternalSearchResponse response, String scrollId, SearchPhaseContext context) {
+        return new SearchPhase("response") {
+            @Override
+            public void run() throws IOException {
+                context.onResponse(context.buildSearchResponse(response, scrollId));
+            }
+        };
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/search/InitialSearchPhase.java b/core/src/main/java/org/elasticsearch/action/search/InitialSearchPhase.java
new file mode 100644
index 0000000000000..fcee980379bf1
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/search/InitialSearchPhase.java
@@ -0,0 +1,326 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.search;
+
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.apache.logging.log4j.util.Supplier;
+import org.elasticsearch.action.NoShardAvailableActionException;
+import org.elasticsearch.action.support.TransportActions;
+import org.elasticsearch.cluster.routing.GroupShardsIterator;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.util.concurrent.AtomicArray;
+import org.elasticsearch.search.SearchPhaseResult;
+import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.transport.ConnectTransportException;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Stream;
+
+/**
+ * This is an abstract base class that encapsulates the logic to fan out to all shards in provided {@link GroupShardsIterator}
+ * and collect the results. If a shard request returns a failure this class handles the advance to the next replica of the shard until
+ * the shards replica iterator is exhausted. Each shard is referenced by position in the {@link GroupShardsIterator} which is later
+ * referred to as the <tt>shardIndex</tt>.
+ * The fan out and collect algorithm is traditionally used as the initial phase which can either be a query execution or collection
+ * distributed frequencies
+ */
+abstract class InitialSearchPhase<FirstResult extends SearchPhaseResult> extends SearchPhase {
+    private final SearchRequest request;
+    private final GroupShardsIterator<SearchShardIterator> shardsIts;
+    private final Logger logger;
+    private final int expectedTotalOps;
+    private final AtomicInteger totalOps = new AtomicInteger();
+    private final AtomicInteger shardExecutionIndex = new AtomicInteger(0);
+    private final int maxConcurrentShardRequests;
+
+    InitialSearchPhase(String name, SearchRequest request, GroupShardsIterator<SearchShardIterator> shardsIts, Logger logger) {
+        super(name);
+        this.request = request;
+        this.shardsIts = shardsIts;
+        this.logger = logger;
+        // we need to add 1 for non active partition, since we count it in the total. This means for each shard in the iterator we sum up
+        // it's number of active shards but use 1 as the default if no replica of a shard is active at this point.
+        // on a per shards level we use shardIt.remaining() to increment the totalOps pointer but add 1 for the current shard result
+        // we process hence we add one for the non active partition here.
+        this.expectedTotalOps = shardsIts.totalSizeWith1ForEmpty();
+        maxConcurrentShardRequests = Math.min(request.getMaxConcurrentShardRequests(), shardsIts.size());
+    }
+
+    private void onShardFailure(final int shardIndex, @Nullable ShardRouting shard, @Nullable String nodeId,
+                                final SearchShardIterator shardIt, Exception e) {
+        // we always add the shard failure for a specific shard instance
+        // we do make sure to clean it on a successful response from a shard
+        SearchShardTarget shardTarget = new SearchShardTarget(nodeId, shardIt.shardId(), shardIt.getClusterAlias(),
+            shardIt.getOriginalIndices());
+        onShardFailure(shardIndex, shardTarget, e);
+
+        if (totalOps.incrementAndGet() == expectedTotalOps) {
+            if (logger.isDebugEnabled()) {
+                if (e != null && !TransportActions.isShardNotAvailableException(e)) {
+                    logger.debug(
+                        (Supplier<?>) () -> new ParameterizedMessage(
+                            "{}: Failed to execute [{}]",
+                            shard != null ? shard.shortSummary() :
+                                shardIt.shardId(),
+                            request),
+                        e);
+                } else if (logger.isTraceEnabled()) {
+                    logger.trace((Supplier<?>) () -> new ParameterizedMessage("{}: Failed to execute [{}]", shard, request), e);
+                }
+            }
+            onPhaseDone();
+        } else {
+            final ShardRouting nextShard = shardIt.nextOrNull();
+            final boolean lastShard = nextShard == null;
+            // trace log this exception
+            logger.trace(
+                (Supplier<?>) () -> new ParameterizedMessage(
+                    "{}: Failed to execute [{}] lastShard [{}]",
+                    shard != null ? shard.shortSummary() : shardIt.shardId(),
+                    request,
+                    lastShard),
+                e);
+            if (!lastShard) {
+                try {
+                    performPhaseOnShard(shardIndex, shardIt, nextShard);
+                } catch (Exception inner) {
+                    inner.addSuppressed(e);
+                    onShardFailure(shardIndex, shard, shard.currentNodeId(), shardIt, inner);
+                }
+            } else {
+                maybeExecuteNext(); // move to the next execution if needed
+                // no more shards active, add a failure
+                if (logger.isDebugEnabled() && !logger.isTraceEnabled()) { // do not double log this exception
+                    if (e != null && !TransportActions.isShardNotAvailableException(e)) {
+                        logger.debug(
+                            (Supplier<?>) () -> new ParameterizedMessage(
+                                "{}: Failed to execute [{}] lastShard [{}]",
+                                shard != null ? shard.shortSummary() :
+                                    shardIt.shardId(),
+                                request,
+                                lastShard),
+                            e);
+                    }
+                }
+            }
+        }
+    }
+
+    @Override
+    public final void run() throws IOException {
+        boolean success = shardExecutionIndex.compareAndSet(0, maxConcurrentShardRequests);
+        assert success;
+        for (int i = 0; i < maxConcurrentShardRequests; i++) {
+            SearchShardIterator shardRoutings = shardsIts.get(i);
+            if (shardRoutings.skip()) {
+                skipShard(shardRoutings);
+            } else {
+                performPhaseOnShard(i, shardRoutings, shardRoutings.nextOrNull());
+            }
+        }
+    }
+
+    private void maybeExecuteNext() {
+        final int index = shardExecutionIndex.getAndIncrement();
+        if (index < shardsIts.size()) {
+            SearchShardIterator shardRoutings = shardsIts.get(index);
+            if (shardRoutings.skip()) {
+                skipShard(shardRoutings);
+            } else {
+                performPhaseOnShard(index, shardRoutings, shardRoutings.nextOrNull());
+            }
+        }
+    }
+
+
+    private void performPhaseOnShard(final int shardIndex, final SearchShardIterator shardIt, final ShardRouting shard) {
+        if (shard == null) {
+            onShardFailure(shardIndex, null, null, shardIt, new NoShardAvailableActionException(shardIt.shardId()));
+        } else {
+            try {
+                executePhaseOnShard(shardIt, shard, new SearchActionListener<FirstResult>(new SearchShardTarget(shard.currentNodeId(),
+                    shardIt.shardId(), shardIt.getClusterAlias(), shardIt.getOriginalIndices()), shardIndex) {
+                    @Override
+                    public void innerOnResponse(FirstResult result) {
+                        onShardResult(result, shardIt);
+                    }
+
+                    @Override
+                    public void onFailure(Exception t) {
+                        onShardFailure(shardIndex, shard, shard.currentNodeId(), shardIt, t);
+                    }
+                });
+            } catch (ConnectTransportException | IllegalArgumentException ex) {
+                // we are getting the connection early here so we might run into nodes that are not connected. in that case we move on to
+                // the next shard. previously when using discovery nodes here we had a special case for null when a node was not connected
+                // at all which is not not needed anymore.
+                onShardFailure(shardIndex, shard, shard.currentNodeId(), shardIt, ex);
+            }
+        }
+    }
+
+    private void onShardResult(FirstResult result, SearchShardIterator shardIt) {
+        assert result.getShardIndex() != -1 : "shard index is not set";
+        assert result.getSearchShardTarget() != null : "search shard target must not be null";
+        onShardSuccess(result);
+        // we need to increment successful ops first before we compare the exit condition otherwise if we
+        // are fast we could concurrently update totalOps but then preempt one of the threads which can
+        // cause the successor to read a wrong value from successfulOps if second phase is very fast ie. count etc.
+        // increment all the "future" shards to update the total ops since we some may work and some may not...
+        // and when that happens, we break on total ops, so we must maintain them
+        successfulShardExecution(shardIt);
+    }
+
+    private void successfulShardExecution(SearchShardIterator shardsIt) {
+        final int remainingOpsOnIterator;
+        if (shardsIt.skip()) {
+            remainingOpsOnIterator = shardsIt.remaining();
+        } else {
+            remainingOpsOnIterator = shardsIt.remaining() + 1;
+        }
+        final int xTotalOps = totalOps.addAndGet(remainingOpsOnIterator);
+        if (xTotalOps == expectedTotalOps) {
+            onPhaseDone();
+        } else if (xTotalOps > expectedTotalOps) {
+            throw new AssertionError("unexpected higher total ops [" + xTotalOps + "] compared to expected ["
+                + expectedTotalOps + "]");
+        } else {
+            maybeExecuteNext();
+        }
+    }
+
+
+    /**
+     * Executed once all shard results have been received and processed
+     * @see #onShardFailure(int, SearchShardTarget, Exception)
+     * @see #onShardSuccess(SearchPhaseResult)
+     */
+    abstract void onPhaseDone(); // as a tribute to @kimchy aka. finishHim()
+
+    /**
+     * Executed once for every failed shard level request. This method is invoked before the next replica is tried for the given
+     * shard target.
+     * @param shardIndex the internal index for this shard. Each shard has an index / ordinal assigned that is used to reference
+     *                   it's results
+     * @param shardTarget the shard target for this failure
+     * @param ex the failure reason
+     */
+    abstract void onShardFailure(int shardIndex, SearchShardTarget shardTarget, Exception ex);
+
+    /**
+     * Executed once for every successful shard level request.
+     * @param result the result returned form the shard
+     *
+     */
+    abstract void onShardSuccess(FirstResult result);
+
+    /**
+     * Sends the request to the actual shard.
+     * @param shardIt the shards iterator
+     * @param shard the shard routing to send the request for
+     * @param listener the listener to notify on response
+     */
+    protected abstract void executePhaseOnShard(SearchShardIterator shardIt, ShardRouting shard,
+                                                SearchActionListener<FirstResult> listener);
+
+    /**
+     * This class acts as a basic result collection that can be extended to do on-the-fly reduction or result processing
+     */
+    abstract static class SearchPhaseResults<Result extends SearchPhaseResult> {
+        private final int numShards;
+
+        protected SearchPhaseResults(int numShards) {
+            this.numShards = numShards;
+        }
+        /**
+         * Returns the number of expected results this class should collect
+         */
+        final int getNumShards() {
+            return numShards;
+        }
+
+        /**
+         * A stream of all non-null (successful) shard results
+         */
+        abstract Stream<Result> getSuccessfulResults();
+
+        /**
+         * Consumes a single shard result
+         * @param result the shards result
+         */
+        abstract void consumeResult(Result result);
+
+        /**
+         * Returns <code>true</code> iff a result if present for the given shard ID.
+         */
+        abstract boolean hasResult(int shardIndex);
+
+        void consumeShardFailure(int shardIndex) {}
+
+        AtomicArray<Result> getAtomicArray() {
+            throw new UnsupportedOperationException();
+        }
+
+        /**
+         * Reduces the collected results
+         */
+        SearchPhaseController.ReducedQueryPhase reduce() {
+            throw new UnsupportedOperationException("reduce is not supported");
+        }
+    }
+
+    /**
+     * This class acts as a basic result collection that can be extended to do on-the-fly reduction or result processing
+     */
+    static class ArraySearchPhaseResults<Result extends SearchPhaseResult> extends SearchPhaseResults<Result> {
+        final AtomicArray<Result> results;
+
+        ArraySearchPhaseResults(int size) {
+            super(size);
+            this.results = new AtomicArray<>(size);
+        }
+
+        Stream<Result> getSuccessfulResults() {
+            return results.asList().stream();
+        }
+
+        void consumeResult(Result result) {
+            assert results.get(result.getShardIndex()) == null : "shardIndex: " + result.getShardIndex() + " is already set";
+            results.set(result.getShardIndex(), result);
+        }
+
+        boolean hasResult(int shardIndex) {
+            return results.get(shardIndex) != null;
+        }
+
+        @Override
+        AtomicArray<Result> getAtomicArray() {
+            return results;
+        }
+    }
+
+    protected void skipShard(SearchShardIterator iterator) {
+        assert iterator.skip();
+        successfulShardExecution(iterator);
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java b/core/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java
index 08a1ec5b3de21..7ab97f9bc570e 100644
--- a/core/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java
@@ -22,7 +22,6 @@
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.CompositeIndicesRequest;
-import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -36,7 +35,7 @@
 /**
  * A multi search API request.
  */
-public class MultiSearchRequest extends ActionRequest<MultiSearchRequest> implements CompositeIndicesRequest {
+public class MultiSearchRequest extends ActionRequest implements CompositeIndicesRequest {
 
     private int maxConcurrentSearchRequests = 0;
     private List<SearchRequest> requests = new ArrayList<>();
@@ -84,11 +83,6 @@ public List<SearchRequest> requests() {
         return this.requests;
     }
 
-    @Override
-    public List<? extends IndicesRequest> subRequests() {
-        return this.requests;
-    }
-
     @Override
     public ActionRequestValidationException validate() {
         ActionRequestValidationException validationException = null;
diff --git a/core/src/main/java/org/elasticsearch/action/search/MultiSearchResponse.java b/core/src/main/java/org/elasticsearch/action/search/MultiSearchResponse.java
index 317b775a40369..4d42ad334a9f0 100644
--- a/core/src/main/java/org/elasticsearch/action/search/MultiSearchResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/search/MultiSearchResponse.java
@@ -23,12 +23,12 @@
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -37,7 +37,7 @@
 /**
  * A multi search response.
  */
-public class MultiSearchResponse extends ActionResponse implements Iterable<MultiSearchResponse.Item>, ToXContent {
+public class MultiSearchResponse extends ActionResponse implements Iterable<MultiSearchResponse.Item>, ToXContentObject {
 
     /**
      * A search response item, holding the actual search response, or an error message if it failed.
@@ -151,39 +151,31 @@ public void writeTo(StreamOutput out) throws IOException {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
         builder.startArray(Fields.RESPONSES);
         for (Item item : items) {
             builder.startObject();
             if (item.isFailure()) {
-                ElasticsearchException.renderException(builder, params, item.getFailure());
+                ElasticsearchException.generateFailureXContent(builder, params, item.getFailure(), true);
                 builder.field(Fields.STATUS, ExceptionsHelper.status(item.getFailure()).getStatus());
             } else {
-                item.getResponse().toXContent(builder, params);
+                item.getResponse().innerToXContent(builder, params);
                 builder.field(Fields.STATUS, item.getResponse().status().getStatus());
             }
             builder.endObject();
         }
         builder.endArray();
+        builder.endObject();
         return builder;
     }
 
     static final class Fields {
         static final String RESPONSES = "responses";
         static final String STATUS = "status";
-        static final String ERROR = "error";
-        static final String ROOT_CAUSE = "root_cause";
     }
 
     @Override
     public String toString() {
-        try {
-            XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint();
-            builder.startObject();
-            toXContent(builder, EMPTY_PARAMS);
-            builder.endObject();
-            return builder.string();
-        } catch (IOException e) {
-            return "{ \"error\" : \"" + e.getMessage() + "\"}";
-        }
+        return Strings.toString(this);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/search/ParsedScrollId.java b/core/src/main/java/org/elasticsearch/action/search/ParsedScrollId.java
index 2ddb35e135736..1f34cca8e461c 100644
--- a/core/src/main/java/org/elasticsearch/action/search/ParsedScrollId.java
+++ b/core/src/main/java/org/elasticsearch/action/search/ParsedScrollId.java
@@ -34,7 +34,7 @@ class ParsedScrollId {
 
     private final ScrollIdForNode[] context;
 
-    public ParsedScrollId(String source, String type, ScrollIdForNode[] context) {
+    ParsedScrollId(String source, String type, ScrollIdForNode[] context) {
         this.source = source;
         this.type = type;
         this.context = context;
diff --git a/core/src/main/java/org/elasticsearch/action/search/ScrollIdForNode.java b/core/src/main/java/org/elasticsearch/action/search/ScrollIdForNode.java
index 488132fdda23d..59e1a3310672b 100644
--- a/core/src/main/java/org/elasticsearch/action/search/ScrollIdForNode.java
+++ b/core/src/main/java/org/elasticsearch/action/search/ScrollIdForNode.java
@@ -19,12 +19,16 @@
 
 package org.elasticsearch.action.search;
 
+import org.elasticsearch.common.inject.internal.Nullable;
+
 class ScrollIdForNode {
     private final String node;
     private final long scrollId;
+    private final String clusterAlias;
 
-    public ScrollIdForNode(String node, long scrollId) {
+    ScrollIdForNode(@Nullable String clusterAlias, String node, long scrollId) {
         this.node = node;
+        this.clusterAlias = clusterAlias;
         this.scrollId = scrollId;
     }
 
@@ -32,7 +36,20 @@ public String getNode() {
         return node;
     }
 
+    public String getClusterAlias() {
+        return clusterAlias;
+    }
+
     public long getScrollId() {
         return scrollId;
     }
+
+    @Override
+    public String toString() {
+        return "ScrollIdForNode{" +
+            "node='" + node + '\'' +
+            ", scrollId=" + scrollId +
+            ", clusterAlias='" + clusterAlias + '\'' +
+            '}';
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchActionListener.java b/core/src/main/java/org/elasticsearch/action/search/SearchActionListener.java
new file mode 100644
index 0000000000000..67de87b1bb173
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchActionListener.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.search;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.search.SearchPhaseResult;
+import org.elasticsearch.search.SearchShardTarget;
+
+/**
+ * An base action listener that ensures shard target and shard index is set on all responses
+ * received by this listener.
+ */
+abstract class SearchActionListener<T extends SearchPhaseResult> implements ActionListener<T> {
+    private final int requestIndex;
+    private final SearchShardTarget searchShardTarget;
+
+    protected SearchActionListener(SearchShardTarget searchShardTarget,
+                                   int shardIndex) {
+        assert shardIndex >= 0 : "shard index must be positive";
+        this.searchShardTarget = searchShardTarget;
+        this.requestIndex = shardIndex;
+    }
+
+    @Override
+    public final void onResponse(T response) {
+        response.setShardIndex(requestIndex);
+        setSearchShardTarget(response);
+        innerOnResponse(response);
+    }
+
+    protected void setSearchShardTarget(T response) { // some impls need to override this
+        response.setSearchShardTarget(searchShardTarget);
+    }
+
+    protected abstract void innerOnResponse(T response);
+}
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchDfsQueryAndFetchAsyncAction.java b/core/src/main/java/org/elasticsearch/action/search/SearchDfsQueryAndFetchAsyncAction.java
deleted file mode 100644
index 367832afab340..0000000000000
--- a/core/src/main/java/org/elasticsearch/action/search/SearchDfsQueryAndFetchAsyncAction.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.action.search;
-
-import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.apache.logging.log4j.util.Supplier;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ActionRunnable;
-import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.util.concurrent.AtomicArray;
-import org.elasticsearch.search.action.SearchTransportService;
-import org.elasticsearch.search.controller.SearchPhaseController;
-import org.elasticsearch.search.dfs.AggregatedDfs;
-import org.elasticsearch.search.dfs.DfsSearchResult;
-import org.elasticsearch.search.fetch.QueryFetchSearchResult;
-import org.elasticsearch.search.internal.InternalSearchResponse;
-import org.elasticsearch.search.internal.ShardSearchTransportRequest;
-import org.elasticsearch.search.query.QuerySearchRequest;
-import org.elasticsearch.threadpool.ThreadPool;
-
-import java.io.IOException;
-import java.util.concurrent.atomic.AtomicInteger;
-
-class SearchDfsQueryAndFetchAsyncAction extends AbstractSearchAsyncAction<DfsSearchResult> {
-
-    private final AtomicArray<QueryFetchSearchResult> queryFetchResults;
-
-    SearchDfsQueryAndFetchAsyncAction(Logger logger, SearchTransportService searchTransportService,
-                                              ClusterService clusterService, IndexNameExpressionResolver indexNameExpressionResolver,
-                                              SearchPhaseController searchPhaseController, ThreadPool threadPool,
-                                              SearchRequest request, ActionListener<SearchResponse> listener) {
-        super(logger, searchTransportService, clusterService, indexNameExpressionResolver, searchPhaseController, threadPool,
-                request, listener);
-        queryFetchResults = new AtomicArray<>(firstResults.length());
-    }
-
-    @Override
-    protected String firstPhaseName() {
-        return "dfs";
-    }
-
-    @Override
-    protected void sendExecuteFirstPhase(DiscoveryNode node, ShardSearchTransportRequest request,
-                                         ActionListener<DfsSearchResult> listener) {
-        searchTransportService.sendExecuteDfs(node, request, listener);
-    }
-
-    @Override
-    protected void moveToSecondPhase() {
-        final AggregatedDfs dfs = searchPhaseController.aggregateDfs(firstResults);
-        final AtomicInteger counter = new AtomicInteger(firstResults.asList().size());
-
-        for (final AtomicArray.Entry<DfsSearchResult> entry : firstResults.asList()) {
-            DfsSearchResult dfsResult = entry.value;
-            DiscoveryNode node = nodes.get(dfsResult.shardTarget().nodeId());
-            QuerySearchRequest querySearchRequest = new QuerySearchRequest(request, dfsResult.id(), dfs);
-            executeSecondPhase(entry.index, dfsResult, counter, node, querySearchRequest);
-        }
-    }
-
-    void executeSecondPhase(final int shardIndex, final DfsSearchResult dfsResult, final AtomicInteger counter,
-                            final DiscoveryNode node, final QuerySearchRequest querySearchRequest) {
-        searchTransportService.sendExecuteFetch(node, querySearchRequest, new ActionListener<QueryFetchSearchResult>() {
-            @Override
-            public void onResponse(QueryFetchSearchResult result) {
-                result.shardTarget(dfsResult.shardTarget());
-                queryFetchResults.set(shardIndex, result);
-                if (counter.decrementAndGet() == 0) {
-                    finishHim();
-                }
-            }
-
-            @Override
-            public void onFailure(Exception t) {
-                try {
-                    onSecondPhaseFailure(t, querySearchRequest, shardIndex, dfsResult, counter);
-                } finally {
-                    // the query might not have been executed at all (for example because thread pool rejected execution)
-                    // and the search context that was created in dfs phase might not be released.
-                    // release it again to be in the safe side
-                    sendReleaseSearchContext(querySearchRequest.id(), node);
-                }
-            }
-        });
-    }
-
-    void onSecondPhaseFailure(Exception e, QuerySearchRequest querySearchRequest, int shardIndex, DfsSearchResult dfsResult,
-                              AtomicInteger counter) {
-        if (logger.isDebugEnabled()) {
-            logger.debug((Supplier<?>) () -> new ParameterizedMessage("[{}] Failed to execute query phase", querySearchRequest.id()), e);
-        }
-        this.addShardFailure(shardIndex, dfsResult.shardTarget(), e);
-        successfulOps.decrementAndGet();
-        if (counter.decrementAndGet() == 0) {
-            finishHim();
-        }
-    }
-
-    private void finishHim() {
-        threadPool.executor(ThreadPool.Names.SEARCH).execute(new ActionRunnable<SearchResponse>(listener) {
-            @Override
-            public void doRun() throws IOException {
-                sortedShardDocs = searchPhaseController.sortDocs(true, queryFetchResults);
-                final InternalSearchResponse internalResponse = searchPhaseController.merge(true, sortedShardDocs, queryFetchResults,
-                    queryFetchResults);
-                String scrollId = null;
-                if (request.scroll() != null) {
-                    scrollId = TransportSearchHelper.buildScrollId(request.searchType(), firstResults);
-                }
-                listener.onResponse(new SearchResponse(internalResponse, scrollId, expectedSuccessfulOps, successfulOps.get(),
-                    buildTookInMillis(), buildShardFailures()));
-            }
-
-            @Override
-            public void onFailure(Exception e) {
-                ReduceSearchPhaseException failure = new ReduceSearchPhaseException("query_fetch", "", e, buildShardFailures());
-                if (logger.isDebugEnabled()) {
-                    logger.debug("failed to reduce search", failure);
-                }
-                super.onFailure(e);
-            }
-        });
-
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java b/core/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java
index 7ceefb1998c7c..a901d71157137 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java
@@ -19,205 +19,43 @@
 
 package org.elasticsearch.action.search;
 
-import com.carrotsearch.hppc.IntArrayList;
 import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.apache.logging.log4j.util.Supplier;
-import org.apache.lucene.search.ScoreDoc;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ActionRunnable;
-import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.util.concurrent.AtomicArray;
-import org.elasticsearch.search.SearchShardTarget;
-import org.elasticsearch.search.action.SearchTransportService;
-import org.elasticsearch.search.controller.SearchPhaseController;
-import org.elasticsearch.search.dfs.AggregatedDfs;
+import org.elasticsearch.cluster.routing.GroupShardsIterator;
+import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.search.dfs.DfsSearchResult;
-import org.elasticsearch.search.fetch.FetchSearchResult;
-import org.elasticsearch.search.fetch.ShardFetchSearchRequest;
-import org.elasticsearch.search.internal.InternalSearchResponse;
-import org.elasticsearch.search.internal.ShardSearchTransportRequest;
-import org.elasticsearch.search.query.QuerySearchRequest;
-import org.elasticsearch.search.query.QuerySearchResult;
-import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.search.internal.AliasFilter;
+import org.elasticsearch.transport.Transport;
 
-import java.io.IOException;
-import java.util.concurrent.atomic.AtomicInteger;
+import java.util.Map;
+import java.util.concurrent.Executor;
+import java.util.function.BiFunction;
 
-class SearchDfsQueryThenFetchAsyncAction extends AbstractSearchAsyncAction<DfsSearchResult> {
+final class SearchDfsQueryThenFetchAsyncAction extends AbstractSearchAsyncAction<DfsSearchResult> {
 
-    final AtomicArray<QuerySearchResult> queryResults;
-    final AtomicArray<FetchSearchResult> fetchResults;
-    final AtomicArray<IntArrayList> docIdsToLoad;
+    private final SearchPhaseController searchPhaseController;
 
-    SearchDfsQueryThenFetchAsyncAction(Logger logger, SearchTransportService searchTransportService,
-                                               ClusterService clusterService, IndexNameExpressionResolver indexNameExpressionResolver,
-                                               SearchPhaseController searchPhaseController, ThreadPool threadPool,
-                                               SearchRequest request, ActionListener<SearchResponse> listener) {
-        super(logger, searchTransportService, clusterService, indexNameExpressionResolver, searchPhaseController, threadPool,
-                request, listener);
-        queryResults = new AtomicArray<>(firstResults.length());
-        fetchResults = new AtomicArray<>(firstResults.length());
-        docIdsToLoad = new AtomicArray<>(firstResults.length());
+    SearchDfsQueryThenFetchAsyncAction(final Logger logger, final SearchTransportService searchTransportService,
+            final BiFunction<String, String, Transport.Connection> nodeIdToConnection, final Map<String, AliasFilter> aliasFilter,
+            final Map<String, Float> concreteIndexBoosts, final SearchPhaseController searchPhaseController, final Executor executor,
+            final SearchRequest request, final ActionListener<SearchResponse> listener,
+            final GroupShardsIterator<SearchShardIterator> shardsIts, final TransportSearchAction.SearchTimeProvider timeProvider,
+            final long clusterStateVersion, final SearchTask task) {
+        super("dfs", logger, searchTransportService, nodeIdToConnection, aliasFilter, concreteIndexBoosts, executor, request, listener,
+                shardsIts, timeProvider, clusterStateVersion, task, new ArraySearchPhaseResults<>(shardsIts.size()));
+        this.searchPhaseController = searchPhaseController;
     }
 
     @Override
-    protected String firstPhaseName() {
-        return "dfs";
+    protected void executePhaseOnShard(final SearchShardIterator shardIt, final ShardRouting shard,
+                                       final SearchActionListener<DfsSearchResult> listener) {
+        getSearchTransport().sendExecuteDfs(getConnection(shardIt.getClusterAlias(), shard.currentNodeId()),
+            buildShardSearchRequest(shardIt) , getTask(), listener);
     }
 
     @Override
-    protected void sendExecuteFirstPhase(DiscoveryNode node, ShardSearchTransportRequest request,
-                                         ActionListener<DfsSearchResult> listener) {
-        searchTransportService.sendExecuteDfs(node, request, listener);
-    }
-
-    @Override
-    protected void moveToSecondPhase() {
-        final AggregatedDfs dfs = searchPhaseController.aggregateDfs(firstResults);
-        final AtomicInteger counter = new AtomicInteger(firstResults.asList().size());
-        for (final AtomicArray.Entry<DfsSearchResult> entry : firstResults.asList()) {
-            DfsSearchResult dfsResult = entry.value;
-            DiscoveryNode node = nodes.get(dfsResult.shardTarget().nodeId());
-            QuerySearchRequest querySearchRequest = new QuerySearchRequest(request, dfsResult.id(), dfs);
-            executeQuery(entry.index, dfsResult, counter, querySearchRequest, node);
-        }
-    }
-
-    void executeQuery(final int shardIndex, final DfsSearchResult dfsResult, final AtomicInteger counter,
-                      final QuerySearchRequest querySearchRequest, final DiscoveryNode node) {
-        searchTransportService.sendExecuteQuery(node, querySearchRequest, new ActionListener<QuerySearchResult>() {
-            @Override
-            public void onResponse(QuerySearchResult result) {
-                result.shardTarget(dfsResult.shardTarget());
-                queryResults.set(shardIndex, result);
-                if (counter.decrementAndGet() == 0) {
-                    executeFetchPhase();
-                }
-            }
-
-            @Override
-            public void onFailure(Exception t) {
-                try {
-                    onQueryFailure(t, querySearchRequest, shardIndex, dfsResult, counter);
-                } finally {
-                    // the query might not have been executed at all (for example because thread pool rejected
-                    // execution) and the search context that was created in dfs phase might not be released.
-                    // release it again to be in the safe side
-                    sendReleaseSearchContext(querySearchRequest.id(), node);
-                }
-            }
-        });
-    }
-
-    void onQueryFailure(Exception e, QuerySearchRequest querySearchRequest, int shardIndex, DfsSearchResult dfsResult,
-                        AtomicInteger counter) {
-        if (logger.isDebugEnabled()) {
-            logger.debug((Supplier<?>) () -> new ParameterizedMessage("[{}] Failed to execute query phase", querySearchRequest.id()), e);
-        }
-        this.addShardFailure(shardIndex, dfsResult.shardTarget(), e);
-        successfulOps.decrementAndGet();
-        if (counter.decrementAndGet() == 0) {
-            if (successfulOps.get() == 0) {
-                listener.onFailure(new SearchPhaseExecutionException("query", "all shards failed", buildShardFailures()));
-            } else {
-                executeFetchPhase();
-            }
-        }
-    }
-
-    void executeFetchPhase() {
-        try {
-            innerExecuteFetchPhase();
-        } catch (Exception e) {
-            listener.onFailure(new ReduceSearchPhaseException("query", "", e, buildShardFailures()));
-        }
-    }
-
-    void innerExecuteFetchPhase() throws Exception {
-        final boolean isScrollRequest = request.scroll() != null;
-        sortedShardDocs = searchPhaseController.sortDocs(isScrollRequest, queryResults);
-        searchPhaseController.fillDocIdsToLoad(docIdsToLoad, sortedShardDocs);
-
-        if (docIdsToLoad.asList().isEmpty()) {
-            finishHim();
-            return;
-        }
-
-        final ScoreDoc[] lastEmittedDocPerShard = (request.scroll() != null) ?
-            searchPhaseController.getLastEmittedDocPerShard(queryResults.asList(), sortedShardDocs, firstResults.length()) : null;
-        final AtomicInteger counter = new AtomicInteger(docIdsToLoad.asList().size());
-        for (final AtomicArray.Entry<IntArrayList> entry : docIdsToLoad.asList()) {
-            QuerySearchResult queryResult = queryResults.get(entry.index);
-            DiscoveryNode node = nodes.get(queryResult.shardTarget().nodeId());
-            ShardFetchSearchRequest fetchSearchRequest = createFetchRequest(queryResult, entry, lastEmittedDocPerShard);
-            executeFetch(entry.index, queryResult.shardTarget(), counter, fetchSearchRequest, node);
-        }
-    }
-
-    void executeFetch(final int shardIndex, final SearchShardTarget shardTarget, final AtomicInteger counter,
-                      final ShardFetchSearchRequest fetchSearchRequest, DiscoveryNode node) {
-        searchTransportService.sendExecuteFetch(node, fetchSearchRequest, new ActionListener<FetchSearchResult>() {
-            @Override
-            public void onResponse(FetchSearchResult result) {
-                result.shardTarget(shardTarget);
-                fetchResults.set(shardIndex, result);
-                if (counter.decrementAndGet() == 0) {
-                    finishHim();
-                }
-            }
-
-            @Override
-            public void onFailure(Exception t) {
-                // the search context might not be cleared on the node where the fetch was executed for example
-                // because the action was rejected by the thread pool. in this case we need to send a dedicated
-                // request to clear the search context. by setting docIdsToLoad to null, the context will be cleared
-                // in TransportSearchTypeAction.releaseIrrelevantSearchContexts() after the search request is done.
-                docIdsToLoad.set(shardIndex, null);
-                onFetchFailure(t, fetchSearchRequest, shardIndex, shardTarget, counter);
-            }
-        });
-    }
-
-    void onFetchFailure(Exception e, ShardFetchSearchRequest fetchSearchRequest, int shardIndex,
-                        SearchShardTarget shardTarget, AtomicInteger counter) {
-        if (logger.isDebugEnabled()) {
-            logger.debug((Supplier<?>) () -> new ParameterizedMessage("[{}] Failed to execute fetch phase", fetchSearchRequest.id()), e);
-        }
-        this.addShardFailure(shardIndex, shardTarget, e);
-        successfulOps.decrementAndGet();
-        if (counter.decrementAndGet() == 0) {
-            finishHim();
-        }
-    }
-
-    private void finishHim() {
-        threadPool.executor(ThreadPool.Names.SEARCH).execute(new ActionRunnable<SearchResponse>(listener) {
-            @Override
-            public void doRun() throws IOException {
-                final boolean isScrollRequest = request.scroll() != null;
-                final InternalSearchResponse internalResponse = searchPhaseController.merge(isScrollRequest, sortedShardDocs, queryResults,
-                    fetchResults);
-                String scrollId = isScrollRequest ? TransportSearchHelper.buildScrollId(request.searchType(), firstResults) : null;
-                listener.onResponse(new SearchResponse(internalResponse, scrollId, expectedSuccessfulOps, successfulOps.get(),
-                    buildTookInMillis(), buildShardFailures()));
-                releaseIrrelevantSearchContexts(queryResults, docIdsToLoad);
-            }
-
-            @Override
-            public void onFailure(Exception e) {
-                try {
-                    ReduceSearchPhaseException failure = new ReduceSearchPhaseException("merge", "", e, buildShardFailures());
-                    if (logger.isDebugEnabled()) {
-                        logger.debug("failed to reduce search", failure);
-                    }
-                    super.onFailure(failure);
-                } finally {
-                    releaseIrrelevantSearchContexts(queryResults, docIdsToLoad);
-                }
-            }
-        });
+    protected SearchPhase getNextPhase(final SearchPhaseResults<DfsSearchResult> results, final SearchPhaseContext context) {
+        return new DfsQueryPhase(results.getAtomicArray(), searchPhaseController, (queryResults) ->
+            new FetchSearchPhase(queryResults, searchPhaseController, context), context);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchPhase.java b/core/src/main/java/org/elasticsearch/action/search/SearchPhase.java
new file mode 100644
index 0000000000000..7bb9c2ba28a89
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchPhase.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.search;
+
+import org.elasticsearch.common.CheckedRunnable;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * Base class for all individual search phases like collecting distributed frequencies, fetching documents, querying shards.
+ */
+abstract class SearchPhase implements CheckedRunnable<IOException> {
+    private final String name;
+
+    protected SearchPhase(String name) {
+        this.name = Objects.requireNonNull(name, "name must not be null");
+    }
+
+    /**
+     * Returns the phases name.
+     */
+    public String getName() {
+        return name;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchPhaseContext.java b/core/src/main/java/org/elasticsearch/action/search/SearchPhaseContext.java
new file mode 100644
index 0000000000000..9829ff6a98337
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchPhaseContext.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.search;
+
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.OriginalIndices;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.search.internal.InternalSearchResponse;
+import org.elasticsearch.search.internal.ShardSearchTransportRequest;
+import org.elasticsearch.transport.Transport;
+
+import java.util.concurrent.Executor;
+
+/**
+ * This class provide contextual state and access to resources across multiple search phases.
+ */
+interface SearchPhaseContext extends ActionListener<SearchResponse>, Executor {
+    // TODO maybe we can make this concrete later - for now we just implement this in the base class for all initial phases
+
+    /**
+     * Returns the total number of shards to the current search across all indices
+     */
+    int getNumShards();
+
+    /**
+     * Returns a logger for this context to prevent each individual phase to create their own logger.
+     */
+    Logger getLogger();
+
+    /**
+     * Returns the currently executing search task
+     */
+    SearchTask getTask();
+
+    /**
+     * Returns the currently executing search request
+     */
+    SearchRequest getRequest();
+
+    /**
+     * Builds the final search response that should be send back to the user.
+     * @param internalSearchResponse the internal search response
+     * @param scrollId an optional scroll ID if this search is a scroll search
+     */
+    SearchResponse buildSearchResponse(InternalSearchResponse internalSearchResponse, String scrollId);
+
+    /**
+     * This method will communicate a fatal phase failure back to the user. In contrast to a shard failure
+     * will this method immediately fail the search request and return the failure to the issuer of the request
+     * @param phase the phase that failed
+     * @param msg an optional message
+     * @param cause the cause of the phase failure
+     */
+    void onPhaseFailure(SearchPhase phase, String msg, Throwable cause);
+
+    /**
+     * This method will record a shard failure for the given shard index. In contrast to a phase failure
+     * ({@link #onPhaseFailure(SearchPhase, String, Throwable)}) this method will immediately return to the user but will record
+     * a shard failure for the given shard index. This should be called if a shard failure happens after we successfully retrieved
+     * a result from that shard in a previous phase.
+     */
+    void onShardFailure(int shardIndex, @Nullable SearchShardTarget shardTarget, Exception e);
+
+    /**
+     * Returns a connection to the node if connected otherwise and {@link org.elasticsearch.transport.ConnectTransportException} will be
+     * thrown.
+     */
+    Transport.Connection getConnection(String clusterAlias, String nodeId);
+
+    /**
+     * Returns the {@link SearchTransportService} to send shard request to other nodes
+     */
+    SearchTransportService getSearchTransport();
+
+    /**
+     * Releases a search context with the given context ID on the node the given connection is connected to.
+     * @see org.elasticsearch.search.query.QuerySearchResult#getRequestId()
+     * @see org.elasticsearch.search.fetch.FetchSearchResult#getRequestId()
+     *
+     */
+    default void sendReleaseSearchContext(long contextId, Transport.Connection connection, OriginalIndices originalIndices) {
+        if (connection != null) {
+            getSearchTransport().sendFreeContext(connection, contextId, originalIndices);
+        }
+    }
+
+    /**
+     * Builds an request for the initial search phase.
+     */
+    ShardSearchTransportRequest buildShardSearchRequest(SearchShardIterator shardIt);
+
+    /**
+     * Processes the phase transition from on phase to another. This method handles all errors that happen during the initial run execution
+     * of the next phase. If there are no successful operations in the context when this method is executed the search is aborted and
+     * a response is returned to the user indicating that all shards have failed.
+     */
+    void executeNextPhase(SearchPhase currentPhase, SearchPhase nextPhase);
+
+}
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/core/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
new file mode 100644
index 0000000000000..afcb8656af631
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
@@ -0,0 +1,756 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.search;
+
+import com.carrotsearch.hppc.IntArrayList;
+import com.carrotsearch.hppc.ObjectObjectHashMap;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.CollectionStatistics;
+import org.apache.lucene.search.FieldDoc;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.TermStatistics;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.TopFieldDocs;
+import org.apache.lucene.search.grouping.CollapseTopFieldDocs;
+import org.elasticsearch.common.collect.HppcMaps;
+import org.elasticsearch.common.component.AbstractComponent;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.search.SearchPhaseResult;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext;
+import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.aggregations.pipeline.SiblingPipelineAggregator;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.dfs.AggregatedDfs;
+import org.elasticsearch.search.dfs.DfsSearchResult;
+import org.elasticsearch.search.fetch.FetchSearchResult;
+import org.elasticsearch.search.internal.InternalSearchResponse;
+import org.elasticsearch.search.profile.ProfileShardResult;
+import org.elasticsearch.search.profile.SearchProfileShardResults;
+import org.elasticsearch.search.query.QuerySearchResult;
+import org.elasticsearch.search.suggest.Suggest;
+import org.elasticsearch.search.suggest.Suggest.Suggestion;
+import org.elasticsearch.search.suggest.Suggest.Suggestion.Entry;
+import org.elasticsearch.search.suggest.completion.CompletionSuggestion;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.IntFunction;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
+
+public final class SearchPhaseController extends AbstractComponent {
+
+    private static final ScoreDoc[] EMPTY_DOCS = new ScoreDoc[0];
+
+    private final BigArrays bigArrays;
+    private final ScriptService scriptService;
+
+    public SearchPhaseController(Settings settings, BigArrays bigArrays, ScriptService scriptService) {
+        super(settings);
+        this.bigArrays = bigArrays;
+        this.scriptService = scriptService;
+    }
+
+    public AggregatedDfs aggregateDfs(Collection<DfsSearchResult> results) {
+        ObjectObjectHashMap<Term, TermStatistics> termStatistics = HppcMaps.newNoNullKeysMap();
+        ObjectObjectHashMap<String, CollectionStatistics> fieldStatistics = HppcMaps.newNoNullKeysMap();
+        long aggMaxDoc = 0;
+        for (DfsSearchResult lEntry : results) {
+            final Term[] terms = lEntry.terms();
+            final TermStatistics[] stats = lEntry.termStatistics();
+            assert terms.length == stats.length;
+            for (int i = 0; i < terms.length; i++) {
+                assert terms[i] != null;
+                TermStatistics existing = termStatistics.get(terms[i]);
+                if (existing != null) {
+                    assert terms[i].bytes().equals(existing.term());
+                    // totalTermFrequency is an optional statistic we need to check if either one or both
+                    // are set to -1 which means not present and then set it globally to -1
+                    termStatistics.put(terms[i], new TermStatistics(existing.term(),
+                            existing.docFreq() + stats[i].docFreq(),
+                            optionalSum(existing.totalTermFreq(), stats[i].totalTermFreq())));
+                } else {
+                    termStatistics.put(terms[i], stats[i]);
+                }
+
+            }
+
+            assert !lEntry.fieldStatistics().containsKey(null);
+            final Object[] keys = lEntry.fieldStatistics().keys;
+            final Object[] values = lEntry.fieldStatistics().values;
+            for (int i = 0; i < keys.length; i++) {
+                if (keys[i] != null) {
+                    String key = (String) keys[i];
+                    CollectionStatistics value = (CollectionStatistics) values[i];
+                    assert key != null;
+                    CollectionStatistics existing = fieldStatistics.get(key);
+                    if (existing != null) {
+                        CollectionStatistics merged = new CollectionStatistics(
+                                key, existing.maxDoc() + value.maxDoc(),
+                                optionalSum(existing.docCount(), value.docCount()),
+                                optionalSum(existing.sumTotalTermFreq(), value.sumTotalTermFreq()),
+                                optionalSum(existing.sumDocFreq(), value.sumDocFreq())
+                        );
+                        fieldStatistics.put(key, merged);
+                    } else {
+                        fieldStatistics.put(key, value);
+                    }
+                }
+            }
+            aggMaxDoc += lEntry.maxDoc();
+        }
+        return new AggregatedDfs(termStatistics, fieldStatistics, aggMaxDoc);
+    }
+
+    private static long optionalSum(long left, long right) {
+        return Math.min(left, right) == -1 ? -1 : left + right;
+    }
+
+    /**
+     * Returns a score doc array of top N search docs across all shards, followed by top suggest docs for each
+     * named completion suggestion across all shards. If more than one named completion suggestion is specified in the
+     * request, the suggest docs for a named suggestion are ordered by the suggestion name.
+     *
+     * Note: The order of the sorted score docs depends on the shard index in the result array if the merge process needs to disambiguate
+     * the result. In oder to obtain stable results the shard index (index of the result in the result array) must be the same.
+     *
+     * @param ignoreFrom Whether to ignore the from and sort all hits in each shard result.
+     *                   Enabled only for scroll search, because that only retrieves hits of length 'size' in the query phase.
+     * @param results the search phase results to obtain the sort docs from
+     * @param bufferedTopDocs the pre-consumed buffered top docs
+     * @param topDocsStats the top docs stats to fill
+     * @param from the offset into the search results top docs
+     * @param size the number of hits to return from the merged top docs
+     */
+    public SortedTopDocs sortDocs(boolean ignoreFrom, Collection<? extends SearchPhaseResult> results,
+                               final Collection<TopDocs> bufferedTopDocs, final TopDocsStats topDocsStats, int from, int size) {
+        if (results.isEmpty()) {
+            return SortedTopDocs.EMPTY;
+        }
+        final Collection<TopDocs> topDocs = bufferedTopDocs == null ? new ArrayList<>() : bufferedTopDocs;
+        final Map<String, List<Suggestion<CompletionSuggestion.Entry>>> groupedCompletionSuggestions = new HashMap<>();
+        for (SearchPhaseResult sortedResult : results) { // TODO we can move this loop into the reduce call to only loop over this once
+            /* We loop over all results once, group together the completion suggestions if there are any and collect relevant
+             * top docs results. Each top docs gets it's shard index set on all top docs to simplify top docs merging down the road
+             * this allowed to remove a single shared optimization code here since now we don't materialized a dense array of
+             * top docs anymore but instead only pass relevant results / top docs to the merge method*/
+            QuerySearchResult queryResult = sortedResult.queryResult();
+            if (queryResult.hasConsumedTopDocs() == false) { // already consumed?
+                final TopDocs td = queryResult.consumeTopDocs();
+                assert td != null;
+                topDocsStats.add(td);
+                if (td.scoreDocs.length > 0) { // make sure we set the shard index before we add it - the consumer didn't do that yet
+                    setShardIndex(td, queryResult.getShardIndex());
+                    topDocs.add(td);
+                }
+            }
+            if (queryResult.hasSuggestHits()) {
+                Suggest shardSuggest = queryResult.suggest();
+                for (CompletionSuggestion suggestion : shardSuggest.filter(CompletionSuggestion.class)) {
+                    suggestion.setShardIndex(sortedResult.getShardIndex());
+                    List<Suggestion<CompletionSuggestion.Entry>> suggestions =
+                        groupedCompletionSuggestions.computeIfAbsent(suggestion.getName(), s -> new ArrayList<>());
+                    suggestions.add(suggestion);
+                }
+            }
+        }
+        final boolean hasHits = (groupedCompletionSuggestions.isEmpty() && topDocs.isEmpty()) == false;
+        if (hasHits) {
+            final TopDocs mergedTopDocs = mergeTopDocs(topDocs, size, ignoreFrom ? 0 : from);
+            final ScoreDoc[] mergedScoreDocs = mergedTopDocs == null ? EMPTY_DOCS : mergedTopDocs.scoreDocs;
+            ScoreDoc[] scoreDocs = mergedScoreDocs;
+            if (groupedCompletionSuggestions.isEmpty() == false) {
+                int numSuggestDocs = 0;
+                List<Suggestion<? extends Entry<? extends Entry.Option>>> completionSuggestions =
+                    new ArrayList<>(groupedCompletionSuggestions.size());
+                for (List<Suggestion<CompletionSuggestion.Entry>> groupedSuggestions : groupedCompletionSuggestions.values()) {
+                    final CompletionSuggestion completionSuggestion = CompletionSuggestion.reduceTo(groupedSuggestions);
+                    assert completionSuggestion != null;
+                    numSuggestDocs += completionSuggestion.getOptions().size();
+                    completionSuggestions.add(completionSuggestion);
+                }
+                scoreDocs = new ScoreDoc[mergedScoreDocs.length + numSuggestDocs];
+                System.arraycopy(mergedScoreDocs, 0, scoreDocs, 0, mergedScoreDocs.length);
+                int offset = mergedScoreDocs.length;
+                Suggest suggestions = new Suggest(completionSuggestions);
+                for (CompletionSuggestion completionSuggestion : suggestions.filter(CompletionSuggestion.class)) {
+                    for (CompletionSuggestion.Entry.Option option : completionSuggestion.getOptions()) {
+                        scoreDocs[offset++] = option.getDoc();
+                    }
+                }
+            }
+            final boolean isSortedByField;
+            final SortField[] sortFields;
+            if (mergedTopDocs != null && mergedTopDocs instanceof TopFieldDocs) {
+                TopFieldDocs fieldDocs = (TopFieldDocs) mergedTopDocs;
+                isSortedByField = (fieldDocs instanceof CollapseTopFieldDocs &&
+                    fieldDocs.fields.length == 1 && fieldDocs.fields[0].getType() == SortField.Type.SCORE) == false;
+                sortFields = fieldDocs.fields;
+            } else {
+                isSortedByField = false;
+                sortFields = null;
+            }
+            return new SortedTopDocs(scoreDocs, isSortedByField, sortFields);
+        } else {
+            // no relevant docs
+            return SortedTopDocs.EMPTY;
+        }
+    }
+
+    TopDocs mergeTopDocs(Collection<TopDocs> results, int topN, int from) {
+        if (results.isEmpty()) {
+            return null;
+        }
+        assert results.isEmpty() == false;
+        final boolean setShardIndex = false;
+        final TopDocs topDocs = results.stream().findFirst().get();
+        final TopDocs mergedTopDocs;
+        final int numShards = results.size();
+        if (numShards == 1 && from == 0) { // only one shard and no pagination we can just return the topDocs as we got them.
+            return topDocs;
+        } else if (topDocs instanceof CollapseTopFieldDocs) {
+            CollapseTopFieldDocs firstTopDocs = (CollapseTopFieldDocs) topDocs;
+            final Sort sort = new Sort(firstTopDocs.fields);
+            final CollapseTopFieldDocs[] shardTopDocs = results.toArray(new CollapseTopFieldDocs[numShards]);
+            mergedTopDocs = CollapseTopFieldDocs.merge(sort, from, topN, shardTopDocs, setShardIndex);
+        } else if (topDocs instanceof TopFieldDocs) {
+            TopFieldDocs firstTopDocs = (TopFieldDocs) topDocs;
+            final Sort sort = new Sort(firstTopDocs.fields);
+            final TopFieldDocs[] shardTopDocs = results.toArray(new TopFieldDocs[numShards]);
+            mergedTopDocs = TopDocs.merge(sort, from, topN, shardTopDocs, setShardIndex);
+        } else {
+            final TopDocs[] shardTopDocs = results.toArray(new TopDocs[numShards]);
+            mergedTopDocs = TopDocs.merge(from, topN, shardTopDocs, setShardIndex);
+        }
+        return mergedTopDocs;
+    }
+
+    private static void setShardIndex(TopDocs topDocs, int shardIndex) {
+        for (ScoreDoc doc : topDocs.scoreDocs) {
+            if (doc.shardIndex != -1) {
+                // once there is a single shard index initialized all others will be initialized too
+                // there are many asserts down in lucene land that this is actually true. we can shortcut it here.
+                return;
+            }
+            doc.shardIndex = shardIndex;
+        }
+    }
+
+    public ScoreDoc[] getLastEmittedDocPerShard(ReducedQueryPhase reducedQueryPhase, int numShards) {
+        final ScoreDoc[] lastEmittedDocPerShard = new ScoreDoc[numShards];
+        if (reducedQueryPhase.isEmptyResult == false) {
+            final ScoreDoc[] sortedScoreDocs = reducedQueryPhase.scoreDocs;
+            // from is always zero as when we use scroll, we ignore from
+            long size = Math.min(reducedQueryPhase.fetchHits, reducedQueryPhase.size);
+            // with collapsing we can have more hits than sorted docs
+            size = Math.min(sortedScoreDocs.length, size);
+            for (int sortedDocsIndex = 0; sortedDocsIndex < size; sortedDocsIndex++) {
+                ScoreDoc scoreDoc = sortedScoreDocs[sortedDocsIndex];
+                lastEmittedDocPerShard[scoreDoc.shardIndex] = scoreDoc;
+            }
+        }
+        return lastEmittedDocPerShard;
+
+    }
+
+    /**
+     * Builds an array, with potential null elements, with docs to load.
+     */
+    public IntArrayList[] fillDocIdsToLoad(int numShards, ScoreDoc[] shardDocs) {
+        IntArrayList[] docIdsToLoad = new IntArrayList[numShards];
+        for (ScoreDoc shardDoc : shardDocs) {
+            IntArrayList shardDocIdsToLoad = docIdsToLoad[shardDoc.shardIndex];
+            if (shardDocIdsToLoad == null) {
+                shardDocIdsToLoad = docIdsToLoad[shardDoc.shardIndex] = new IntArrayList();
+            }
+            shardDocIdsToLoad.add(shardDoc.doc);
+        }
+        return docIdsToLoad;
+    }
+
+    /**
+     * Enriches search hits and completion suggestion hits from <code>sortedDocs</code> using <code>fetchResultsArr</code>,
+     * merges suggestions, aggregations and profile results
+     *
+     * Expects sortedDocs to have top search docs across all shards, optionally followed by top suggest docs for each named
+     * completion suggestion ordered by suggestion name
+     */
+    public InternalSearchResponse merge(boolean ignoreFrom, ReducedQueryPhase reducedQueryPhase,
+                                        Collection<? extends SearchPhaseResult> fetchResults, IntFunction<SearchPhaseResult> resultsLookup) {
+        if (reducedQueryPhase.isEmptyResult) {
+            return InternalSearchResponse.empty();
+        }
+        ScoreDoc[] sortedDocs = reducedQueryPhase.scoreDocs;
+        SearchHits hits = getHits(reducedQueryPhase, ignoreFrom, fetchResults, resultsLookup);
+        if (reducedQueryPhase.suggest != null) {
+            if (!fetchResults.isEmpty()) {
+                int currentOffset = hits.getHits().length;
+                for (CompletionSuggestion suggestion : reducedQueryPhase.suggest.filter(CompletionSuggestion.class)) {
+                    final List<CompletionSuggestion.Entry.Option> suggestionOptions = suggestion.getOptions();
+                    for (int scoreDocIndex = currentOffset; scoreDocIndex < currentOffset + suggestionOptions.size(); scoreDocIndex++) {
+                        ScoreDoc shardDoc = sortedDocs[scoreDocIndex];
+                        SearchPhaseResult searchResultProvider = resultsLookup.apply(shardDoc.shardIndex);
+                        if (searchResultProvider == null) {
+                            // this can happen if we are hitting a shard failure during the fetch phase
+                            // in this case we referenced the shard result via teh ScoreDoc but never got a
+                            // result from fetch.
+                            // TODO it would be nice to assert this in the future
+                            continue;
+                        }
+                        FetchSearchResult fetchResult = searchResultProvider.fetchResult();
+                        final int index = fetchResult.counterGetAndIncrement();
+                        assert index < fetchResult.hits().internalHits().length : "not enough hits fetched. index [" + index + "] length: "
+                            + fetchResult.hits().internalHits().length;
+                        SearchHit hit = fetchResult.hits().internalHits()[index];
+                        CompletionSuggestion.Entry.Option suggestOption =
+                            suggestionOptions.get(scoreDocIndex - currentOffset);
+                        hit.score(shardDoc.score);
+                        hit.shard(fetchResult.getSearchShardTarget());
+                        suggestOption.setHit(hit);
+                    }
+                    currentOffset += suggestionOptions.size();
+                }
+                assert currentOffset == sortedDocs.length : "expected no more score doc slices";
+            }
+        }
+        return reducedQueryPhase.buildResponse(hits);
+    }
+
+    private SearchHits getHits(ReducedQueryPhase reducedQueryPhase, boolean ignoreFrom,
+                               Collection<? extends SearchPhaseResult> fetchResults, IntFunction<SearchPhaseResult> resultsLookup) {
+        final boolean sorted = reducedQueryPhase.isSortedByField;
+        ScoreDoc[] sortedDocs = reducedQueryPhase.scoreDocs;
+        int sortScoreIndex = -1;
+        if (sorted) {
+            for (int i = 0; i < reducedQueryPhase.sortField.length; i++) {
+                if (reducedQueryPhase.sortField[i].getType() == SortField.Type.SCORE) {
+                    sortScoreIndex = i;
+                }
+            }
+        }
+        // clean the fetch counter
+        for (SearchPhaseResult entry : fetchResults) {
+            entry.fetchResult().initCounter();
+        }
+        int from = ignoreFrom ? 0 : reducedQueryPhase.from;
+        int numSearchHits = (int) Math.min(reducedQueryPhase.fetchHits - from, reducedQueryPhase.size);
+        // with collapsing we can have more fetch hits than sorted docs
+        numSearchHits = Math.min(sortedDocs.length, numSearchHits);
+        // merge hits
+        List<SearchHit> hits = new ArrayList<>();
+        if (!fetchResults.isEmpty()) {
+            for (int i = 0; i < numSearchHits; i++) {
+                ScoreDoc shardDoc = sortedDocs[i];
+                SearchPhaseResult fetchResultProvider = resultsLookup.apply(shardDoc.shardIndex);
+                if (fetchResultProvider == null) {
+                    // this can happen if we are hitting a shard failure during the fetch phase
+                    // in this case we referenced the shard result via teh ScoreDoc but never got a
+                    // result from fetch.
+                    // TODO it would be nice to assert this in the future
+                    continue;
+                }
+                FetchSearchResult fetchResult = fetchResultProvider.fetchResult();
+                final int index = fetchResult.counterGetAndIncrement();
+                assert index < fetchResult.hits().internalHits().length : "not enough hits fetched. index [" + index + "] length: "
+                    + fetchResult.hits().internalHits().length;
+                SearchHit searchHit = fetchResult.hits().internalHits()[index];
+                searchHit.score(shardDoc.score);
+                searchHit.shard(fetchResult.getSearchShardTarget());
+                if (sorted) {
+                    FieldDoc fieldDoc = (FieldDoc) shardDoc;
+                    searchHit.sortValues(fieldDoc.fields, reducedQueryPhase.sortValueFormats);
+                    if (sortScoreIndex != -1) {
+                        searchHit.score(((Number) fieldDoc.fields[sortScoreIndex]).floatValue());
+                    }
+                }
+                hits.add(searchHit);
+            }
+        }
+        return new SearchHits(hits.toArray(new SearchHit[hits.size()]), reducedQueryPhase.totalHits,
+            reducedQueryPhase.maxScore);
+    }
+
+    /**
+     * Reduces the given query results and consumes all aggregations and profile results.
+     * @param queryResults a list of non-null query shard results
+     */
+    public ReducedQueryPhase reducedQueryPhase(Collection<? extends SearchPhaseResult> queryResults, boolean isScrollRequest) {
+        return reducedQueryPhase(queryResults, null, new ArrayList<>(), new TopDocsStats(), 0, isScrollRequest);
+    }
+
+    /**
+     * Reduces the given query results and consumes all aggregations and profile results.
+     * @param queryResults a list of non-null query shard results
+     * @param bufferedAggs a list of pre-collected / buffered aggregations. if this list is non-null all aggregations have been consumed
+     *                    from all non-null query results.
+     * @param bufferedTopDocs a list of pre-collected / buffered top docs. if this list is non-null all top docs have been consumed
+     *                    from all non-null query results.
+     * @param numReducePhases the number of non-final reduce phases applied to the query results.
+     * @see QuerySearchResult#consumeAggs()
+     * @see QuerySearchResult#consumeProfileResult()
+     */
+    private ReducedQueryPhase reducedQueryPhase(Collection<? extends SearchPhaseResult> queryResults,
+                                                List<InternalAggregations> bufferedAggs, List<TopDocs> bufferedTopDocs,
+                                                TopDocsStats topDocsStats, int numReducePhases, boolean isScrollRequest) {
+        assert numReducePhases >= 0 : "num reduce phases must be >= 0 but was: " + numReducePhases;
+        numReducePhases++; // increment for this phase
+        boolean timedOut = false;
+        Boolean terminatedEarly = null;
+        if (queryResults.isEmpty()) { // early terminate we have nothing to reduce
+            return new ReducedQueryPhase(topDocsStats.totalHits, topDocsStats.fetchHits, topDocsStats.maxScore,
+                timedOut, terminatedEarly, null, null, null, EMPTY_DOCS, null, null, numReducePhases, false, 0, 0, true);
+        }
+        final QuerySearchResult firstResult = queryResults.stream().findFirst().get().queryResult();
+        final boolean hasSuggest = firstResult.suggest() != null;
+        final boolean hasProfileResults = firstResult.hasProfileResults();
+        final boolean consumeAggs;
+        final List<InternalAggregations> aggregationsList;
+        if (bufferedAggs != null) {
+            consumeAggs = false;
+            // we already have results from intermediate reduces and just need to perform the final reduce
+            assert firstResult.hasAggs() : "firstResult has no aggs but we got non null buffered aggs?";
+            aggregationsList = bufferedAggs;
+        } else if (firstResult.hasAggs()) {
+            // the number of shards was less than the buffer size so we reduce agg results directly
+            aggregationsList = new ArrayList<>(queryResults.size());
+            consumeAggs = true;
+        } else {
+            // no aggregations
+            aggregationsList = Collections.emptyList();
+            consumeAggs = false;
+        }
+
+        // count the total (we use the query result provider here, since we might not get any hits (we scrolled past them))
+        final Map<String, List<Suggestion>> groupedSuggestions = hasSuggest ? new HashMap<>() : Collections.emptyMap();
+        final Map<String, ProfileShardResult> profileResults = hasProfileResults ? new HashMap<>(queryResults.size())
+            : Collections.emptyMap();
+        int from = 0;
+        int size = 0;
+        for (SearchPhaseResult entry : queryResults) {
+            QuerySearchResult result = entry.queryResult();
+            from = result.from();
+            size = result.size();
+            if (result.searchTimedOut()) {
+                timedOut = true;
+            }
+            if (result.terminatedEarly() != null) {
+                if (terminatedEarly == null) {
+                    terminatedEarly = result.terminatedEarly();
+                } else if (result.terminatedEarly()) {
+                    terminatedEarly = true;
+                }
+            }
+            if (hasSuggest) {
+                assert result.suggest() != null;
+                for (Suggestion<? extends Suggestion.Entry<? extends Suggestion.Entry.Option>> suggestion : result.suggest()) {
+                    List<Suggestion> suggestionList = groupedSuggestions.computeIfAbsent(suggestion.getName(), s -> new ArrayList<>());
+                    suggestionList.add(suggestion);
+                }
+            }
+            if (consumeAggs) {
+                aggregationsList.add((InternalAggregations) result.consumeAggs());
+            }
+            if (hasProfileResults) {
+                String key = result.getSearchShardTarget().toString();
+                profileResults.put(key, result.consumeProfileResult());
+            }
+        }
+        final Suggest suggest = groupedSuggestions.isEmpty() ? null : new Suggest(Suggest.reduce(groupedSuggestions));
+        ReduceContext reduceContext = new ReduceContext(bigArrays, scriptService, true);
+        final InternalAggregations aggregations = aggregationsList.isEmpty() ? null : reduceAggs(aggregationsList,
+            firstResult.pipelineAggregators(), reduceContext);
+        final SearchProfileShardResults shardResults = profileResults.isEmpty() ? null : new SearchProfileShardResults(profileResults);
+        final SortedTopDocs scoreDocs = this.sortDocs(isScrollRequest, queryResults, bufferedTopDocs, topDocsStats, from, size);
+        return new ReducedQueryPhase(topDocsStats.totalHits, topDocsStats.fetchHits, topDocsStats.maxScore,
+            timedOut, terminatedEarly, suggest, aggregations, shardResults, scoreDocs.scoreDocs, scoreDocs.sortFields,
+            firstResult != null ? firstResult.sortValueFormats() : null,
+            numReducePhases, scoreDocs.isSortedByField, size, from, firstResult == null);
+    }
+
+
+    /**
+     * Performs an intermediate reduce phase on the aggregations. For instance with this reduce phase never prune information
+     * that relevant for the final reduce step. For final reduce see {@link #reduceAggs(List, List, ReduceContext)}
+     */
+    private InternalAggregations reduceAggsIncrementally(List<InternalAggregations> aggregationsList) {
+        ReduceContext reduceContext = new ReduceContext(bigArrays, scriptService, false);
+        return aggregationsList.isEmpty() ? null : reduceAggs(aggregationsList,
+            null, reduceContext);
+    }
+
+    private InternalAggregations reduceAggs(List<InternalAggregations> aggregationsList,
+                                            List<SiblingPipelineAggregator> pipelineAggregators, ReduceContext reduceContext) {
+        InternalAggregations aggregations = InternalAggregations.reduce(aggregationsList, reduceContext);
+        if (pipelineAggregators != null) {
+            List<InternalAggregation> newAggs = StreamSupport.stream(aggregations.spliterator(), false)
+                .map((p) -> (InternalAggregation) p)
+                .collect(Collectors.toList());
+            for (SiblingPipelineAggregator pipelineAggregator : pipelineAggregators) {
+                InternalAggregation newAgg = pipelineAggregator.doReduce(new InternalAggregations(newAggs), reduceContext);
+                newAggs.add(newAgg);
+            }
+            return new InternalAggregations(newAggs);
+        }
+        return aggregations;
+    }
+
+    public static final class ReducedQueryPhase {
+        // the sum of all hits across all reduces shards
+        final long totalHits;
+        // the number of returned hits (doc IDs) across all reduces shards
+        final long fetchHits;
+        // the max score across all reduces hits or {@link Float#NaN} if no hits returned
+        final float maxScore;
+        // <code>true</code> if at least one reduced result timed out
+        final boolean timedOut;
+        // non null and true if at least one reduced result was terminated early
+        final Boolean terminatedEarly;
+        // the reduced suggest results
+        final Suggest suggest;
+        // the reduced internal aggregations
+        final InternalAggregations aggregations;
+        // the reduced profile results
+        final SearchProfileShardResults shardResults;
+        // the number of reduces phases
+        final int numReducePhases;
+        // the searches merged top docs
+        final ScoreDoc[] scoreDocs;
+        // the top docs sort fields used to sort the score docs, <code>null</code> if the results are not sorted
+        final SortField[] sortField;
+        // <code>true</code> iff the result score docs is sorted by a field (not score), this implies that <code>sortField</code> is set.
+        final boolean isSortedByField;
+        // the size of the top hits to return
+        final int size;
+        // <code>true</code> iff the query phase had no results. Otherwise <code>false</code>
+        final boolean isEmptyResult;
+        // the offset into the merged top hits
+        final int from;
+        // sort value formats used to sort / format the result
+        final DocValueFormat[] sortValueFormats;
+
+        ReducedQueryPhase(long totalHits, long fetchHits, float maxScore, boolean timedOut, Boolean terminatedEarly, Suggest suggest,
+                          InternalAggregations aggregations, SearchProfileShardResults shardResults, ScoreDoc[] scoreDocs,
+                          SortField[] sortFields, DocValueFormat[] sortValueFormats, int numReducePhases, boolean isSortedByField, int size,
+                          int from, boolean isEmptyResult) {
+            if (numReducePhases <= 0) {
+                throw new IllegalArgumentException("at least one reduce phase must have been applied but was: " + numReducePhases);
+            }
+            this.totalHits = totalHits;
+            this.fetchHits = fetchHits;
+            if (Float.isInfinite(maxScore)) {
+                this.maxScore = Float.NaN;
+            } else {
+                this.maxScore = maxScore;
+            }
+            this.timedOut = timedOut;
+            this.terminatedEarly = terminatedEarly;
+            this.suggest = suggest;
+            this.aggregations = aggregations;
+            this.shardResults = shardResults;
+            this.numReducePhases = numReducePhases;
+            this.scoreDocs = scoreDocs;
+            this.sortField = sortFields;
+            this.isSortedByField = isSortedByField;
+            this.size = size;
+            this.from = from;
+            this.isEmptyResult = isEmptyResult;
+            this.sortValueFormats = sortValueFormats;
+        }
+
+        /**
+         * Creates a new search response from the given merged hits.
+         * @see #merge(boolean, ReducedQueryPhase, Collection, IntFunction)
+         */
+        public InternalSearchResponse buildResponse(SearchHits hits) {
+            return new InternalSearchResponse(hits, aggregations, suggest, shardResults, timedOut, terminatedEarly, numReducePhases);
+        }
+    }
+
+    /**
+     * A {@link InitialSearchPhase.ArraySearchPhaseResults} implementation
+     * that incrementally reduces aggregation results as shard results are consumed.
+     * This implementation can be configured to batch up a certain amount of results and only reduce them
+     * iff the buffer is exhausted.
+     */
+    static final class QueryPhaseResultConsumer extends InitialSearchPhase.ArraySearchPhaseResults<SearchPhaseResult> {
+        private final InternalAggregations[] aggsBuffer;
+        private final TopDocs[] topDocsBuffer;
+        private final boolean hasAggs;
+        private final boolean hasTopDocs;
+        private final int bufferSize;
+        private int index;
+        private final SearchPhaseController controller;
+        private int numReducePhases = 0;
+        private final TopDocsStats topDocsStats = new TopDocsStats();
+
+        /**
+         * Creates a new {@link QueryPhaseResultConsumer}
+         * @param controller a controller instance to reduce the query response objects
+         * @param expectedResultSize the expected number of query results. Corresponds to the number of shards queried
+         * @param bufferSize the size of the reduce buffer. if the buffer size is smaller than the number of expected results
+         *                   the buffer is used to incrementally reduce aggregation results before all shards responded.
+         */
+        private QueryPhaseResultConsumer(SearchPhaseController controller, int expectedResultSize, int bufferSize,
+                                         boolean hasTopDocs, boolean hasAggs) {
+            super(expectedResultSize);
+            if (expectedResultSize != 1 && bufferSize < 2) {
+                throw new IllegalArgumentException("buffer size must be >= 2 if there is more than one expected result");
+            }
+            if (expectedResultSize <= bufferSize) {
+                throw new IllegalArgumentException("buffer size must be less than the expected result size");
+            }
+            if (hasAggs == false && hasTopDocs == false) {
+                throw new IllegalArgumentException("either aggs or top docs must be present");
+            }
+            this.controller = controller;
+            // no need to buffer anything if we have less expected results. in this case we don't consume any results ahead of time.
+            this.aggsBuffer = new InternalAggregations[hasAggs ? bufferSize : 0];
+            this.topDocsBuffer = new TopDocs[hasTopDocs ? bufferSize : 0];
+            this.hasTopDocs = hasTopDocs;
+            this.hasAggs = hasAggs;
+            this.bufferSize = bufferSize;
+
+        }
+
+        @Override
+        public void consumeResult(SearchPhaseResult result) {
+            super.consumeResult(result);
+            QuerySearchResult queryResult = result.queryResult();
+            consumeInternal(queryResult);
+        }
+
+        private synchronized void consumeInternal(QuerySearchResult querySearchResult) {
+            if (index == bufferSize) {
+                if (hasAggs) {
+                    InternalAggregations reducedAggs = controller.reduceAggsIncrementally(Arrays.asList(aggsBuffer));
+                    Arrays.fill(aggsBuffer, null);
+                    aggsBuffer[0] = reducedAggs;
+                }
+                if (hasTopDocs) {
+                    TopDocs reducedTopDocs = controller.mergeTopDocs(Arrays.asList(topDocsBuffer),
+                        querySearchResult.from() + querySearchResult.size() // we have to merge here in the same way we collect on a shard
+                        , 0);
+                    Arrays.fill(topDocsBuffer, null);
+                    topDocsBuffer[0] = reducedTopDocs;
+                }
+                numReducePhases++;
+                index = 1;
+            }
+            final int i = index++;
+            if (hasAggs) {
+                aggsBuffer[i] = (InternalAggregations) querySearchResult.consumeAggs();
+            }
+            if (hasTopDocs) {
+                final TopDocs topDocs = querySearchResult.consumeTopDocs(); // can't be null
+                topDocsStats.add(topDocs);
+                SearchPhaseController.setShardIndex(topDocs, querySearchResult.getShardIndex());
+                topDocsBuffer[i] = topDocs;
+            }
+        }
+
+        private synchronized List<InternalAggregations> getRemainingAggs() {
+            return hasAggs ? Arrays.asList(aggsBuffer).subList(0, index) : null;
+        }
+
+        private synchronized List<TopDocs> getRemainingTopDocs() {
+            return hasTopDocs ? Arrays.asList(topDocsBuffer).subList(0, index) : null;
+        }
+
+
+        @Override
+        public ReducedQueryPhase reduce() {
+            return controller.reducedQueryPhase(results.asList(), getRemainingAggs(), getRemainingTopDocs(), topDocsStats,
+                numReducePhases, false);
+        }
+
+        /**
+         * Returns the number of buffered results
+         */
+        int getNumBuffered() {
+            return index;
+        }
+
+        int getNumReducePhases() { return numReducePhases; }
+    }
+
+    /**
+     * Returns a new ArraySearchPhaseResults instance. This might return an instance that reduces search responses incrementally.
+     */
+    InitialSearchPhase.ArraySearchPhaseResults<SearchPhaseResult> newSearchPhaseResults(SearchRequest request, int numShards) {
+        SearchSourceBuilder source = request.source();
+        boolean isScrollRequest = request.scroll() != null;
+        final boolean hasAggs = source != null && source.aggregations() != null;
+        final boolean hasTopDocs = source == null || source.size() != 0;
+
+        if (isScrollRequest == false && (hasAggs || hasTopDocs)) {
+            // no incremental reduce if scroll is used - we only hit a single shard or sometimes more...
+            if (request.getBatchedReduceSize() < numShards) {
+                // only use this if there are aggs and if there are more shards than we should reduce at once
+                return new QueryPhaseResultConsumer(this, numShards, request.getBatchedReduceSize(), hasTopDocs, hasAggs);
+            }
+        }
+        return new InitialSearchPhase.ArraySearchPhaseResults(numShards) {
+            @Override
+            public ReducedQueryPhase reduce() {
+                return reducedQueryPhase(results.asList(), isScrollRequest);
+            }
+        };
+    }
+
+    static final class TopDocsStats {
+        long totalHits;
+        long fetchHits;
+        float maxScore = Float.NEGATIVE_INFINITY;
+
+        void add(TopDocs topDocs) {
+            totalHits += topDocs.totalHits;
+            fetchHits += topDocs.scoreDocs.length;
+            if (!Float.isNaN(topDocs.getMaxScore())) {
+                maxScore = Math.max(maxScore, topDocs.getMaxScore());
+            }
+        }
+    }
+
+    static final class SortedTopDocs {
+        static final SortedTopDocs EMPTY = new SortedTopDocs(EMPTY_DOCS, false, null);
+        final ScoreDoc[] scoreDocs;
+        final boolean isSortedByField;
+        final SortField[] sortFields;
+
+        SortedTopDocs(ScoreDoc[] scoreDocs, boolean isSortedByField, SortField[] sortFields) {
+            this.scoreDocs = scoreDocs;
+            this.isSortedByField = isSortedByField;
+            this.sortFields = sortFields;
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchPhaseExecutionException.java b/core/src/main/java/org/elasticsearch/action/search/SearchPhaseExecutionException.java
index 515d3204fb6a8..c6e0b21dffd5d 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchPhaseExecutionException.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchPhaseExecutionException.java
@@ -103,6 +103,7 @@ public ShardSearchFailure[] shardFailures() {
         return shardFailures;
     }
 
+    @Override
     public Throwable getCause() {
         Throwable cause = super.getCause();
         if (cause == null) {
@@ -131,28 +132,34 @@ private static String buildMessage(String phaseName, String msg, ShardSearchFail
     }
 
     @Override
-    protected void innerToXContent(XContentBuilder builder, Params params) throws IOException {
+    protected void metadataToXContent(XContentBuilder builder, Params params) throws IOException {
         builder.field("phase", phaseName);
         final boolean group = params.paramAsBoolean("group_shard_failures", true); // we group by default
         builder.field("grouped", group); // notify that it's grouped
         builder.field("failed_shards");
         builder.startArray();
-        ShardOperationFailedException[] failures = params.paramAsBoolean("group_shard_failures", true) ? ExceptionsHelper.groupBy(shardFailures) : shardFailures;
+        ShardOperationFailedException[] failures = params.paramAsBoolean("group_shard_failures", true) ?
+                ExceptionsHelper.groupBy(shardFailures) : shardFailures;
         for (ShardOperationFailedException failure : failures) {
             builder.startObject();
             failure.toXContent(builder, params);
             builder.endObject();
         }
         builder.endArray();
-        super.innerToXContent(builder, params);
     }
 
     @Override
-    protected void causeToXContent(XContentBuilder builder, Params params) throws IOException {
-        if (super.getCause() != null) {
-            // if the cause is null we inject a guessed root cause that will then be rendered twice so wi disable it manually
-            super.causeToXContent(builder, params);
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        Throwable ex = ExceptionsHelper.unwrapCause(this);
+        if (ex != this) {
+            generateThrowableXContent(builder, params, this);
+        } else {
+            // We don't have a cause when all shards failed, but we do have shards failures so we can "guess" a cause
+            // (see {@link #getCause()}). Here, we use super.getCause() because we don't want the guessed exception to
+            // be rendered twice (one in the "cause" field, one in "failed_shards")
+            innerToXContent(builder, params, this, getExceptionName(), getMessage(), getHeaders(), getMetadata(), super.getCause());
         }
+        return builder;
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchQueryAndFetchAsyncAction.java b/core/src/main/java/org/elasticsearch/action/search/SearchQueryAndFetchAsyncAction.java
deleted file mode 100644
index 2e13a0d26e870..0000000000000
--- a/core/src/main/java/org/elasticsearch/action/search/SearchQueryAndFetchAsyncAction.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.action.search;
-
-import org.apache.logging.log4j.Logger;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ActionRunnable;
-import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.search.action.SearchTransportService;
-import org.elasticsearch.search.controller.SearchPhaseController;
-import org.elasticsearch.search.fetch.QueryFetchSearchResult;
-import org.elasticsearch.search.internal.InternalSearchResponse;
-import org.elasticsearch.search.internal.ShardSearchTransportRequest;
-import org.elasticsearch.threadpool.ThreadPool;
-
-import java.io.IOException;
-
-class SearchQueryAndFetchAsyncAction extends AbstractSearchAsyncAction<QueryFetchSearchResult> {
-
-    SearchQueryAndFetchAsyncAction(Logger logger, SearchTransportService searchTransportService,
-                                           ClusterService clusterService, IndexNameExpressionResolver indexNameExpressionResolver,
-                                           SearchPhaseController searchPhaseController, ThreadPool threadPool,
-                                           SearchRequest request, ActionListener<SearchResponse> listener) {
-        super(logger, searchTransportService, clusterService, indexNameExpressionResolver, searchPhaseController, threadPool,
-                request, listener);
-    }
-
-    @Override
-    protected String firstPhaseName() {
-        return "query_fetch";
-    }
-
-    @Override
-    protected void sendExecuteFirstPhase(DiscoveryNode node, ShardSearchTransportRequest request,
-                                         ActionListener<QueryFetchSearchResult> listener) {
-        searchTransportService.sendExecuteFetch(node, request, listener);
-    }
-
-    @Override
-    protected void moveToSecondPhase() throws Exception {
-        threadPool.executor(ThreadPool.Names.SEARCH).execute(new ActionRunnable<SearchResponse>(listener) {
-            @Override
-            public void doRun() throws IOException {
-                final boolean isScrollRequest = request.scroll() != null;
-                sortedShardDocs = searchPhaseController.sortDocs(isScrollRequest, firstResults);
-                final InternalSearchResponse internalResponse = searchPhaseController.merge(isScrollRequest, sortedShardDocs, firstResults,
-                    firstResults);
-                String scrollId = isScrollRequest ? TransportSearchHelper.buildScrollId(request.searchType(), firstResults) : null;
-                listener.onResponse(new SearchResponse(internalResponse, scrollId, expectedSuccessfulOps, successfulOps.get(),
-                    buildTookInMillis(), buildShardFailures()));
-            }
-
-            @Override
-            public void onFailure(Exception e) {
-                ReduceSearchPhaseException failure = new ReduceSearchPhaseException("merge", "", e, buildShardFailures());
-                if (logger.isDebugEnabled()) {
-                    logger.debug("failed to reduce search", failure);
-                }
-                super.onFailure(failure);
-            }
-        });
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java b/core/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java
index 3987b48c5615a..de8109aadd8fe 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java
@@ -19,138 +19,41 @@
 
 package org.elasticsearch.action.search;
 
-import com.carrotsearch.hppc.IntArrayList;
 import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.apache.logging.log4j.util.Supplier;
-import org.apache.lucene.search.ScoreDoc;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ActionRunnable;
-import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.util.concurrent.AtomicArray;
-import org.elasticsearch.search.SearchShardTarget;
-import org.elasticsearch.search.action.SearchTransportService;
-import org.elasticsearch.search.controller.SearchPhaseController;
-import org.elasticsearch.search.fetch.FetchSearchResult;
-import org.elasticsearch.search.fetch.ShardFetchSearchRequest;
-import org.elasticsearch.search.internal.InternalSearchResponse;
-import org.elasticsearch.search.internal.ShardSearchTransportRequest;
-import org.elasticsearch.search.query.QuerySearchResultProvider;
-import org.elasticsearch.threadpool.ThreadPool;
-
-import java.io.IOException;
-import java.util.concurrent.atomic.AtomicInteger;
-
-class SearchQueryThenFetchAsyncAction extends AbstractSearchAsyncAction<QuerySearchResultProvider> {
-
-    final AtomicArray<FetchSearchResult> fetchResults;
-    final AtomicArray<IntArrayList> docIdsToLoad;
-
-    SearchQueryThenFetchAsyncAction(Logger logger, SearchTransportService searchService,
-                                            ClusterService clusterService, IndexNameExpressionResolver indexNameExpressionResolver,
-                                            SearchPhaseController searchPhaseController, ThreadPool threadPool,
-                                            SearchRequest request, ActionListener<SearchResponse> listener) {
-        super(logger, searchService, clusterService, indexNameExpressionResolver, searchPhaseController, threadPool, request, listener);
-        fetchResults = new AtomicArray<>(firstResults.length());
-        docIdsToLoad = new AtomicArray<>(firstResults.length());
+import org.elasticsearch.cluster.routing.GroupShardsIterator;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.search.SearchPhaseResult;
+import org.elasticsearch.search.internal.AliasFilter;
+import org.elasticsearch.transport.Transport;
+
+import java.util.Map;
+import java.util.concurrent.Executor;
+import java.util.function.BiFunction;
+
+final class SearchQueryThenFetchAsyncAction extends AbstractSearchAsyncAction<SearchPhaseResult> {
+
+    private final SearchPhaseController searchPhaseController;
+
+    SearchQueryThenFetchAsyncAction(final Logger logger, final SearchTransportService searchTransportService,
+            final BiFunction<String, String, Transport.Connection> nodeIdToConnection, final Map<String, AliasFilter> aliasFilter,
+            final Map<String, Float> concreteIndexBoosts, final SearchPhaseController searchPhaseController, final Executor executor,
+            final SearchRequest request, final ActionListener<SearchResponse> listener,
+            final GroupShardsIterator<SearchShardIterator> shardsIts, final TransportSearchAction.SearchTimeProvider timeProvider,
+            long clusterStateVersion, SearchTask task) {
+        super("query", logger, searchTransportService, nodeIdToConnection, aliasFilter, concreteIndexBoosts, executor, request, listener,
+            shardsIts, timeProvider, clusterStateVersion, task, searchPhaseController.newSearchPhaseResults(request, shardsIts.size()));
+        this.searchPhaseController = searchPhaseController;
     }
 
-    @Override
-    protected String firstPhaseName() {
-        return "query";
+    protected void executePhaseOnShard(final SearchShardIterator shardIt, final ShardRouting shard,
+                                       final SearchActionListener<SearchPhaseResult> listener) {
+        getSearchTransport().sendExecuteQuery(getConnection(shardIt.getClusterAlias(), shard.currentNodeId()),
+            buildShardSearchRequest(shardIt), getTask(), listener);
     }
 
     @Override
-    protected void sendExecuteFirstPhase(DiscoveryNode node, ShardSearchTransportRequest request,
-                                         ActionListener<QuerySearchResultProvider> listener) {
-        searchTransportService.sendExecuteQuery(node, request, listener);
-    }
-
-    @Override
-    protected void moveToSecondPhase() throws Exception {
-        final boolean isScrollRequest = request.scroll() != null;
-        sortedShardDocs = searchPhaseController.sortDocs(isScrollRequest, firstResults);
-        searchPhaseController.fillDocIdsToLoad(docIdsToLoad, sortedShardDocs);
-
-        if (docIdsToLoad.asList().isEmpty()) {
-            finishHim();
-            return;
-        }
-
-        final ScoreDoc[] lastEmittedDocPerShard = isScrollRequest ?
-            searchPhaseController.getLastEmittedDocPerShard(firstResults.asList(), sortedShardDocs, firstResults.length()) : null;
-        final AtomicInteger counter = new AtomicInteger(docIdsToLoad.asList().size());
-        for (AtomicArray.Entry<IntArrayList> entry : docIdsToLoad.asList()) {
-            QuerySearchResultProvider queryResult = firstResults.get(entry.index);
-            DiscoveryNode node = nodes.get(queryResult.shardTarget().nodeId());
-            ShardFetchSearchRequest fetchSearchRequest = createFetchRequest(queryResult.queryResult(), entry, lastEmittedDocPerShard);
-            executeFetch(entry.index, queryResult.shardTarget(), counter, fetchSearchRequest, node);
-        }
-    }
-
-    void executeFetch(final int shardIndex, final SearchShardTarget shardTarget, final AtomicInteger counter,
-                      final ShardFetchSearchRequest fetchSearchRequest, DiscoveryNode node) {
-        searchTransportService.sendExecuteFetch(node, fetchSearchRequest, new ActionListener<FetchSearchResult>() {
-            @Override
-            public void onResponse(FetchSearchResult result) {
-                result.shardTarget(shardTarget);
-                fetchResults.set(shardIndex, result);
-                if (counter.decrementAndGet() == 0) {
-                    finishHim();
-                }
-            }
-
-            @Override
-            public void onFailure(Exception t) {
-                // the search context might not be cleared on the node where the fetch was executed for example
-                // because the action was rejected by the thread pool. in this case we need to send a dedicated
-                // request to clear the search context. by setting docIdsToLoad to null, the context will be cleared
-                // in TransportSearchTypeAction.releaseIrrelevantSearchContexts() after the search request is done.
-                docIdsToLoad.set(shardIndex, null);
-                onFetchFailure(t, fetchSearchRequest, shardIndex, shardTarget, counter);
-            }
-        });
-    }
-
-    void onFetchFailure(Exception e, ShardFetchSearchRequest fetchSearchRequest, int shardIndex, SearchShardTarget shardTarget,
-                        AtomicInteger counter) {
-        if (logger.isDebugEnabled()) {
-            logger.debug((Supplier<?>) () -> new ParameterizedMessage("[{}] Failed to execute fetch phase", fetchSearchRequest.id()), e);
-        }
-        this.addShardFailure(shardIndex, shardTarget, e);
-        successfulOps.decrementAndGet();
-        if (counter.decrementAndGet() == 0) {
-            finishHim();
-        }
-    }
-
-    private void finishHim() {
-        threadPool.executor(ThreadPool.Names.SEARCH).execute(new ActionRunnable<SearchResponse>(listener) {
-            @Override
-            public void doRun() throws IOException {
-                final boolean isScrollRequest = request.scroll() != null;
-                final InternalSearchResponse internalResponse = searchPhaseController.merge(isScrollRequest, sortedShardDocs, firstResults,
-                    fetchResults);
-                String scrollId = isScrollRequest ? TransportSearchHelper.buildScrollId(request.searchType(), firstResults) : null;
-                listener.onResponse(new SearchResponse(internalResponse, scrollId, expectedSuccessfulOps,
-                    successfulOps.get(), buildTookInMillis(), buildShardFailures()));
-                releaseIrrelevantSearchContexts(firstResults, docIdsToLoad);
-            }
-
-            @Override
-            public void onFailure(Exception e) {
-                try {
-                    ReduceSearchPhaseException failure = new ReduceSearchPhaseException("fetch", "", e, buildShardFailures());
-                    if (logger.isDebugEnabled()) {
-                        logger.debug("failed to reduce search", failure);
-                    }
-                    super.onFailure(failure);
-                } finally {
-                    releaseIrrelevantSearchContexts(firstResults, docIdsToLoad);
-                }
-            }
-        });
+    protected SearchPhase getNextPhase(final SearchPhaseResults<SearchPhaseResult> results, final SearchPhaseContext context) {
+        return new FetchSearchPhase(results, searchPhaseController, context);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/core/src/main/java/org/elasticsearch/action/search/SearchRequest.java
index a1b1a02a97e5b..73a4a074637d5 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchRequest.java
@@ -19,21 +19,25 @@
 
 package org.elasticsearch.action.search;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskId;
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.Objects;
 
 /**
@@ -43,11 +47,16 @@
  * Note, the search {@link #source(org.elasticsearch.search.builder.SearchSourceBuilder)}
  * is required. The search source is the different search options, including aggregations and such.
  * </p>
+ *
  * @see org.elasticsearch.client.Requests#searchRequest(String...)
  * @see org.elasticsearch.client.Client#search(SearchRequest)
  * @see SearchResponse
  */
-public final class SearchRequest extends ActionRequest<SearchRequest> implements IndicesRequest.Replaceable {
+public final class SearchRequest extends ActionRequest implements IndicesRequest.Replaceable {
+
+    private static final ToXContent.Params FORMAT_PARAMS = new ToXContent.MapParams(Collections.singletonMap("pretty", "false"));
+
+    public static final int DEFAULT_PRE_FILTER_SHARD_SIZE = 128;
 
     private SearchType searchType = SearchType.DEFAULT;
 
@@ -64,6 +73,12 @@ public final class SearchRequest extends ActionRequest<SearchRequest> implements
 
     private Scroll scroll;
 
+    private int batchedReduceSize = 512;
+
+    private int maxConcurrentShardRequests = 0;
+
+    private int preFilterShardSize = DEFAULT_PRE_FILTER_SHARD_SIZE;
+
     private String[] types = Strings.EMPTY_ARRAY;
 
     public static final IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.strictExpandOpenAndForbidClosed();
@@ -192,7 +207,7 @@ public SearchRequest searchType(SearchType searchType) {
      * "query_then_fetch"/"queryThenFetch", and "query_and_fetch"/"queryAndFetch".
      */
     public SearchRequest searchType(String searchType) {
-        return searchType(SearchType.fromString(searchType, ParseFieldMatcher.EMPTY));
+        return searchType(SearchType.fromString(searchType));
     }
 
     /**
@@ -268,6 +283,75 @@ public Boolean requestCache() {
         return this.requestCache;
     }
 
+    /**
+     * Sets the number of shard results that should be reduced at once on the coordinating node. This value should be used as a protection
+     * mechanism to reduce the memory overhead per search request if the potential number of shards in the request can be large.
+     */
+    public void setBatchedReduceSize(int batchedReduceSize) {
+        if (batchedReduceSize <= 1) {
+            throw new IllegalArgumentException("batchedReduceSize must be >= 2");
+        }
+        this.batchedReduceSize = batchedReduceSize;
+    }
+
+    /**
+     * Returns the number of shard results that should be reduced at once on the coordinating node. This value should be used as a
+     * protection mechanism to reduce the memory overhead per search request if the potential number of shards in the request can be large.
+     */
+    public int getBatchedReduceSize() {
+        return batchedReduceSize;
+    }
+
+    /**
+     * Returns the number of shard requests that should be executed concurrently. This value should be used as a protection mechanism to
+     * reduce the number of shard reqeusts fired per high level search request. Searches that hit the entire cluster can be throttled
+     * with this number to reduce the cluster load. The default grows with the number of nodes in the cluster but is at most <tt>256</tt>.
+     */
+    public int getMaxConcurrentShardRequests() {
+        return maxConcurrentShardRequests == 0 ? 256 : maxConcurrentShardRequests;
+    }
+
+    /**
+     * Sets the number of shard requests that should be executed concurrently. This value should be used as a protection mechanism to
+     * reduce the number of shard requests fired per high level search request. Searches that hit the entire cluster can be throttled
+     * with this number to reduce the cluster load. The default grows with the number of nodes in the cluster but is at most <tt>256</tt>.
+     */
+    public void setMaxConcurrentShardRequests(int maxConcurrentShardRequests) {
+        if (maxConcurrentShardRequests < 1) {
+            throw new IllegalArgumentException("maxConcurrentShardRequests must be >= 1");
+        }
+        this.maxConcurrentShardRequests = maxConcurrentShardRequests;
+    }
+    /**
+     * Sets a threshold that enforces a pre-filter roundtrip to pre-filter search shards based on query rewriting if the number of shards
+     * the search request expands to exceeds the threshold. This filter roundtrip can limit the number of shards significantly if for
+     * instance a shard can not match any documents based on it's rewrite method ie. if date filters are mandatory to match but the shard
+     * bounds and the query are disjoint. The default is <tt>128</tt>
+     */
+    public void setPreFilterShardSize(int preFilterShardSize) {
+        if (preFilterShardSize < 1) {
+            throw new IllegalArgumentException("preFilterShardSize must be >= 1");
+        }
+        this.preFilterShardSize = preFilterShardSize;
+    }
+
+    /**
+     * Returns a threshold that enforces a pre-filter roundtrip to pre-filter search shards based on query rewriting if the number of shards
+     * the search request expands to exceeds the threshold. This filter roundtrip can limit the number of shards significantly if for
+     * instance a shard can not match any documents based on it's rewrite method ie. if date filters are mandatory to match but the shard
+     * bounds and the query are disjoint. The default is <tt>128</tt>
+     */
+    public int getPreFilterShardSize() {
+        return preFilterShardSize;
+    }
+
+    /**
+     * Returns <code>true</code> iff the maxConcurrentShardRequest is set.
+     */
+    boolean isMaxConcurrentShardRequestsSet() {
+        return maxConcurrentShardRequests != 0;
+    }
+
     /**
      * @return true if the request only has suggest
      */
@@ -275,6 +359,30 @@ public boolean isSuggestOnly() {
         return source != null && source.isSuggestOnly();
     }
 
+    @Override
+    public Task createTask(long id, String type, String action, TaskId parentTaskId) {
+        // generating description in a lazy way since source can be quite big
+        return new SearchTask(id, type, action, null, parentTaskId) {
+            @Override
+            public String getDescription() {
+                StringBuilder sb = new StringBuilder();
+                sb.append("indices[");
+                Strings.arrayToDelimitedString(indices, ",", sb);
+                sb.append("], ");
+                sb.append("types[");
+                Strings.arrayToDelimitedString(types, ",", sb);
+                sb.append("], ");
+                sb.append("search_type[").append(searchType).append("], ");
+                if (source != null) {
+                    sb.append("source[").append(source.toString(FORMAT_PARAMS)).append("]");
+                } else {
+                    sb.append("source[]");
+                }
+                return sb.toString();
+            }
+        };
+    }
+
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
@@ -290,6 +398,14 @@ public void readFrom(StreamInput in) throws IOException {
         types = in.readStringArray();
         indicesOptions = IndicesOptions.readIndicesOptions(in);
         requestCache = in.readOptionalBoolean();
+        if (in.getVersion().onOrAfter(Version.V_5_4_0)) {
+            batchedReduceSize = in.readVInt();
+            if (in.getVersion().onOrAfter(Version.V_5_6_0)) {
+                maxConcurrentShardRequests = in.readVInt();
+                preFilterShardSize = in.readVInt();
+            }
+        }
+
     }
 
     @Override
@@ -307,6 +423,14 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeStringArray(types);
         indicesOptions.writeIndicesOptions(out);
         out.writeOptionalBoolean(requestCache);
+        if (out.getVersion().onOrAfter(Version.V_5_4_0)) {
+            out.writeVInt(batchedReduceSize);
+            if (out.getVersion().onOrAfter(Version.V_5_6_0)) {
+                out.writeVInt(maxConcurrentShardRequests);
+                out.writeVInt(preFilterShardSize);
+            }
+        }
+
     }
 
     @Override
@@ -326,13 +450,16 @@ public boolean equals(Object o) {
                 Objects.equals(requestCache, that.requestCache)  &&
                 Objects.equals(scroll, that.scroll) &&
                 Arrays.equals(types, that.types) &&
+                Objects.equals(batchedReduceSize, that.batchedReduceSize) &&
+                Objects.equals(maxConcurrentShardRequests, that.maxConcurrentShardRequests) &&
+                Objects.equals(preFilterShardSize, that.preFilterShardSize) &&
                 Objects.equals(indicesOptions, that.indicesOptions);
     }
 
     @Override
     public int hashCode() {
         return Objects.hash(searchType, Arrays.hashCode(indices), routing, preference, source, requestCache,
-                scroll, Arrays.hashCode(types), indicesOptions);
+                scroll, Arrays.hashCode(types), indicesOptions, batchedReduceSize, maxConcurrentShardRequests, preFilterShardSize);
     }
 
     @Override
@@ -346,6 +473,9 @@ public String toString() {
                 ", preference='" + preference + '\'' +
                 ", requestCache=" + requestCache +
                 ", scroll=" + scroll +
+                ", maxConcurrentShardRequests=" + maxConcurrentShardRequests +
+                ", batchedReduceSize=" + batchedReduceSize +
+                ", preFilterShardSize=" + preFilterShardSize +
                 ", source=" + source + '}';
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
index 3c320447fe833..42241477493cb 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
@@ -26,13 +26,14 @@
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.script.Script;
+import org.elasticsearch.search.collapse.CollapseBuilder;
 import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
-import org.elasticsearch.search.slice.SliceBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
 import org.elasticsearch.search.rescore.RescoreBuilder;
+import org.elasticsearch.search.slice.SliceBuilder;
 import org.elasticsearch.search.sort.SortBuilder;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.search.suggest.SuggestBuilder;
@@ -503,6 +504,11 @@ public SearchRequestBuilder setProfile(boolean profile) {
         return this;
     }
 
+    public SearchRequestBuilder setCollapse(CollapseBuilder collapse) {
+        sourceBuilder().collapse(collapse);
+        return this;
+    }
+
     @Override
     public String toString() {
         if (request.source() != null) {
@@ -517,4 +523,34 @@ private SearchSourceBuilder sourceBuilder() {
         }
         return request.source();
     }
+
+    /**
+     * Sets the number of shard results that should be reduced at once on the coordinating node. This value should be used as a protection
+     * mechanism to reduce the memory overhead per search request if the potential number of shards in the request can be large.
+     */
+    public SearchRequestBuilder setBatchedReduceSize(int batchedReduceSize) {
+        this.request.setBatchedReduceSize(batchedReduceSize);
+        return this;
+    }
+
+    /**
+     * Sets the number of shard requests that should be executed concurrently. This value should be used as a protection mechanism to
+     * reduce the number of shard requests fired per high level search request. Searches that hit the entire cluster can be throttled
+     * with this number to reduce the cluster load. The default grows with the number of nodes in the cluster but is at most <tt>256</tt>.
+     */
+    public SearchRequestBuilder setMaxConcurrentShardRequests(int maxConcurrentShardRequests) {
+        this.request.setMaxConcurrentShardRequests(maxConcurrentShardRequests);
+        return this;
+    }
+
+    /**
+     * Sets a threshold that enforces a pre-filter roundtrip to pre-filter search shards based on query rewriting if the number of shards
+     * the search request expands to exceeds the threshold. This filter roundtrip can limit the number of shards significantly if for
+     * instance a shard can not match any documents based on it's rewrite method ie. if date filters are mandatory to match but the shard
+     * bounds and the query are disjoint. The default is <tt>128</tt>
+     */
+    public SearchRequestBuilder setPreFilterShardSize(int preFilterShardSize) {
+        this.request.setPreFilterShardSize(preFilterShardSize);
+        return this;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchResponse.java b/core/src/main/java/org/elasticsearch/action/search/SearchResponse.java
index 3135d2c8f53b0..aace8c12460e7 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchResponse.java
@@ -19,34 +19,47 @@
 
 package org.elasticsearch.action.search;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.xcontent.StatusToXContent;
+import org.elasticsearch.common.xcontent.StatusToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.RestActions;
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.aggregations.Aggregations;
 import org.elasticsearch.search.internal.InternalSearchResponse;
 import org.elasticsearch.search.profile.ProfileShardResult;
+import org.elasticsearch.search.profile.SearchProfileShardResults;
 import org.elasticsearch.search.suggest.Suggest;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Map;
 
 import static org.elasticsearch.action.search.ShardSearchFailure.readShardSearchFailure;
-import static org.elasticsearch.search.internal.InternalSearchResponse.readInternalSearchResponse;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+
 
 /**
  * A response of a search request.
  */
-public class SearchResponse extends ActionResponse implements StatusToXContent {
+public class SearchResponse extends ActionResponse implements StatusToXContentObject {
+
+    private static final ParseField SCROLL_ID = new ParseField("_scroll_id");
+    private static final ParseField TOOK = new ParseField("took");
+    private static final ParseField TIMED_OUT = new ParseField("timed_out");
+    private static final ParseField TERMINATED_EARLY = new ParseField("terminated_early");
+    private static final ParseField NUM_REDUCE_PHASES = new ParseField("num_reduce_phases");
 
-    private InternalSearchResponse internalResponse;
+    private SearchResponseSections internalResponse;
 
     private String scrollId;
 
@@ -54,6 +67,8 @@ public class SearchResponse extends ActionResponse implements StatusToXContent {
 
     private int successfulShards;
 
+    private int skippedShards;
+
     private ShardSearchFailure[] shardFailures;
 
     private long tookInMillis;
@@ -61,13 +76,16 @@ public class SearchResponse extends ActionResponse implements StatusToXContent {
     public SearchResponse() {
     }
 
-    public SearchResponse(InternalSearchResponse internalResponse, String scrollId, int totalShards, int successfulShards, long tookInMillis, ShardSearchFailure[] shardFailures) {
+    public SearchResponse(SearchResponseSections internalResponse, String scrollId, int totalShards, int successfulShards,
+                          int skippedShards, long tookInMillis, ShardSearchFailure[] shardFailures) {
         this.internalResponse = internalResponse;
         this.scrollId = scrollId;
         this.totalShards = totalShards;
         this.successfulShards = successfulShards;
+        this.skippedShards = skippedShards;
         this.tookInMillis = tookInMillis;
         this.shardFailures = shardFailures;
+        assert skippedShards <= totalShards : "skipped: " + skippedShards + " total: " + totalShards;
     }
 
     @Override
@@ -106,6 +124,13 @@ public Boolean isTerminatedEarly() {
         return internalResponse.terminatedEarly();
     }
 
+    /**
+     * Returns the number of reduce phases applied to obtain this search response
+     */
+    public int getNumReducePhases() {
+        return internalResponse.getNumReducePhases();
+    }
+
     /**
      * How long the search took.
      */
@@ -134,6 +159,14 @@ public int getSuccessfulShards() {
         return successfulShards;
     }
 
+
+    /**
+     * The number of shards skipped due to pre-filtering
+     */
+    public int getSkippedShards() {
+        return skippedShards;
+    }
+
     /**
      * The failed number of shards the search was executed on.
      */
@@ -168,36 +201,123 @@ public void scrollId(String scrollId) {
      *
      * @return The profile results or an empty map
      */
-    @Nullable public Map<String, ProfileShardResult> getProfileResults() {
+    @Nullable
+    public Map<String, ProfileShardResult> getProfileResults() {
         return internalResponse.profile();
     }
 
-    static final class Fields {
-        static final String _SCROLL_ID = "_scroll_id";
-        static final String TOOK = "took";
-        static final String TIMED_OUT = "timed_out";
-        static final String TERMINATED_EARLY = "terminated_early";
-    }
-
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        innerToXContent(builder, params);
+        builder.endObject();
+        return builder;
+    }
+
+    public XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
         if (scrollId != null) {
-            builder.field(Fields._SCROLL_ID, scrollId);
+            builder.field(SCROLL_ID.getPreferredName(), scrollId);
         }
-        builder.field(Fields.TOOK, tookInMillis);
-        builder.field(Fields.TIMED_OUT, isTimedOut());
+        builder.field(TOOK.getPreferredName(), tookInMillis);
+        builder.field(TIMED_OUT.getPreferredName(), isTimedOut());
         if (isTerminatedEarly() != null) {
-            builder.field(Fields.TERMINATED_EARLY, isTerminatedEarly());
+            builder.field(TERMINATED_EARLY.getPreferredName(), isTerminatedEarly());
+        }
+        if (getNumReducePhases() != 1) {
+            builder.field(NUM_REDUCE_PHASES.getPreferredName(), getNumReducePhases());
         }
-        RestActions.buildBroadcastShardsHeader(builder, params, getTotalShards(), getSuccessfulShards(), getFailedShards(), getShardFailures());
+        RestActions.buildBroadcastShardsHeader(builder, params, getTotalShards(), getSuccessfulShards(), getSkippedShards(),
+            getFailedShards(), getShardFailures());
         internalResponse.toXContent(builder, params);
         return builder;
     }
 
+    public static SearchResponse fromXContent(XContentParser parser) throws IOException {
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
+        XContentParser.Token token;
+        String currentFieldName = null;
+        SearchHits hits = null;
+        Aggregations aggs = null;
+        Suggest suggest = null;
+        SearchProfileShardResults profile = null;
+        boolean timedOut = false;
+        Boolean terminatedEarly = null;
+        int numReducePhases = 1;
+        long tookInMillis = -1;
+        int successfulShards = -1;
+        int totalShards = -1;
+        int skippedShards = 0; // 0 for BWC
+        String scrollId = null;
+        List<ShardSearchFailure> failures = new ArrayList<>();
+        while((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token.isValue()) {
+                if (SCROLL_ID.match(currentFieldName)) {
+                    scrollId = parser.text();
+                } else if (TOOK.match(currentFieldName)) {
+                    tookInMillis = parser.longValue();
+                } else if (TIMED_OUT.match(currentFieldName)) {
+                    timedOut = parser.booleanValue();
+                } else if (TERMINATED_EARLY.match(currentFieldName)) {
+                    terminatedEarly = parser.booleanValue();
+                } else if (NUM_REDUCE_PHASES.match(currentFieldName)) {
+                    numReducePhases = parser.intValue();
+                } else {
+                    parser.skipChildren();
+                }
+            } else if (token == XContentParser.Token.START_OBJECT) {
+                if (SearchHits.Fields.HITS.equals(currentFieldName)) {
+                    hits = SearchHits.fromXContent(parser);
+                } else if (Aggregations.AGGREGATIONS_FIELD.equals(currentFieldName)) {
+                    aggs = Aggregations.fromXContent(parser);
+                } else if (Suggest.NAME.equals(currentFieldName)) {
+                    suggest = Suggest.fromXContent(parser);
+                } else if (SearchProfileShardResults.PROFILE_FIELD.equals(currentFieldName)) {
+                    profile = SearchProfileShardResults.fromXContent(parser);
+                } else if (RestActions._SHARDS_FIELD.match(currentFieldName)) {
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                        if (token == XContentParser.Token.FIELD_NAME) {
+                            currentFieldName = parser.currentName();
+                        } else if (token.isValue()) {
+                            if (RestActions.FAILED_FIELD.match(currentFieldName)) {
+                                parser.intValue(); // we don't need it but need to consume it
+                            } else if (RestActions.SUCCESSFUL_FIELD.match(currentFieldName)) {
+                                successfulShards = parser.intValue();
+                            } else if (RestActions.TOTAL_FIELD.match(currentFieldName)) {
+                                totalShards = parser.intValue();
+                            } else if (RestActions.SKIPPED_FIELD.match(currentFieldName)) {
+                                skippedShards = parser.intValue();
+                            } else {
+                                parser.skipChildren();
+                            }
+                        } else if (token == XContentParser.Token.START_ARRAY) {
+                            if (RestActions.FAILURES_FIELD.match(currentFieldName)) {
+                                while((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                                    failures.add(ShardSearchFailure.fromXContent(parser));
+                                }
+                            } else {
+                                parser.skipChildren();
+                            }
+                        } else {
+                            parser.skipChildren();
+                        }
+                    }
+                } else {
+                    parser.skipChildren();
+                }
+            }
+        }
+        SearchResponseSections searchResponseSections = new SearchResponseSections(hits, aggs, suggest, timedOut, terminatedEarly,
+                profile, numReducePhases);
+        return new SearchResponse(searchResponseSections, scrollId, totalShards, successfulShards, skippedShards, tookInMillis,
+                failures.toArray(new ShardSearchFailure[failures.size()]));
+    }
+
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
-        internalResponse = readInternalSearchResponse(in);
+        internalResponse = new InternalSearchResponse(in);
         totalShards = in.readVInt();
         successfulShards = in.readVInt();
         int size = in.readVInt();
@@ -211,6 +331,9 @@ public void readFrom(StreamInput in) throws IOException {
         }
         scrollId = in.readOptionalString();
         tookInMillis = in.readVLong();
+        if (in.getVersion().onOrAfter(Version.V_5_6_0)) {
+            skippedShards = in.readVInt();
+        }
     }
 
     @Override
@@ -227,10 +350,14 @@ public void writeTo(StreamOutput out) throws IOException {
 
         out.writeOptionalString(scrollId);
         out.writeVLong(tookInMillis);
+        if(out.getVersion().onOrAfter(Version.V_5_6_0)) {
+            out.writeVInt(skippedShards);
+        }
     }
 
     @Override
     public String toString() {
-        return Strings.toString(this, true);
+        return Strings.toString(this);
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchResponseSections.java b/core/src/main/java/org/elasticsearch/action/search/SearchResponseSections.java
new file mode 100644
index 0000000000000..1757acbfd6d93
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchResponseSections.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.search;
+
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.search.aggregations.Aggregations;
+import org.elasticsearch.search.profile.ProfileShardResult;
+import org.elasticsearch.search.profile.SearchProfileShardResults;
+import org.elasticsearch.search.suggest.Suggest;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+
+/**
+ * Base class that holds the various sections which a search response is
+ * composed of (hits, aggs, suggestions etc.) and allows to retrieve them.
+ *
+ * The reason why this class exists is that the high level REST client uses its own classes
+ * to parse aggregations into, which are not serializable. This is the common part that can be
+ * shared between core and client.
+ */
+public class SearchResponseSections implements ToXContent {
+
+    protected final SearchHits hits;
+    protected final Aggregations aggregations;
+    protected final Suggest suggest;
+    protected final SearchProfileShardResults profileResults;
+    protected final boolean timedOut;
+    protected final Boolean terminatedEarly;
+    protected final int numReducePhases;
+
+    public SearchResponseSections(SearchHits hits, Aggregations aggregations, Suggest suggest, boolean timedOut, Boolean terminatedEarly,
+                                  SearchProfileShardResults profileResults,  int numReducePhases) {
+        this.hits = hits;
+        this.aggregations = aggregations;
+        this.suggest = suggest;
+        this.profileResults = profileResults;
+        this.timedOut = timedOut;
+        this.terminatedEarly = terminatedEarly;
+        this.numReducePhases = numReducePhases;
+    }
+
+    public final boolean timedOut() {
+        return this.timedOut;
+    }
+
+    public final Boolean terminatedEarly() {
+        return this.terminatedEarly;
+    }
+
+    public final SearchHits hits() {
+        return hits;
+    }
+
+    public final Aggregations aggregations() {
+        return aggregations;
+    }
+
+    public final Suggest suggest() {
+        return suggest;
+    }
+
+    /**
+     * Returns the number of reduce phases applied to obtain this search response
+     */
+    public final int getNumReducePhases() {
+        return numReducePhases;
+    }
+
+    /**
+     * Returns the profile results for this search response (including all shards).
+     * An empty map is returned if profiling was not enabled
+     *
+     * @return Profile results
+     */
+    public final Map<String, ProfileShardResult> profile() {
+        if (profileResults == null) {
+            return Collections.emptyMap();
+        }
+        return profileResults.getShardResults();
+    }
+
+    @Override
+    public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        hits.toXContent(builder, params);
+        if (aggregations != null) {
+            aggregations.toXContent(builder, params);
+        }
+        if (suggest != null) {
+            suggest.toXContent(builder, params);
+        }
+        if (profileResults != null) {
+            profileResults.toXContent(builder, params);
+        }
+        return builder;
+    }
+
+    protected void writeTo(StreamOutput out) throws IOException {
+        throw new UnsupportedOperationException();
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchScrollAsyncAction.java b/core/src/main/java/org/elasticsearch/action/search/SearchScrollAsyncAction.java
new file mode 100644
index 0000000000000..109a1f30ffc8e
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchScrollAsyncAction.java
@@ -0,0 +1,286 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.search;
+
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.apache.logging.log4j.util.Supplier;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.util.concurrent.AtomicArray;
+import org.elasticsearch.common.util.concurrent.CountDown;
+import org.elasticsearch.search.SearchPhaseResult;
+import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.search.internal.InternalScrollSearchRequest;
+import org.elasticsearch.search.internal.InternalSearchResponse;
+import org.elasticsearch.transport.RemoteClusterService;
+import org.elasticsearch.transport.Transport;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.BiFunction;
+
+import static org.elasticsearch.action.search.TransportSearchHelper.internalScrollSearchRequest;
+
+/**
+ * Abstract base class for scroll execution modes. This class encapsulates the basic logic to
+ * fan out to nodes and execute the query part of the scroll request. Subclasses can for instance
+ * run separate fetch phases etc.
+ */
+abstract class SearchScrollAsyncAction<T extends SearchPhaseResult> implements Runnable {
+    /*
+     * Some random TODO:
+     * Today we still have a dedicated executing mode for scrolls while we could simplify this by implementing
+     * scroll like functionality (mainly syntactic sugar) as an ordinary search with search_after. We could even go further and
+     * make the scroll entirely stateless and encode the state per shard in the scroll ID.
+     *
+     * Today we also hold a context per shard but maybe
+     * we want the context per coordinating node such that we route the scroll to the same coordinator all the time and hold the context
+     * here? This would have the advantage that if we loose that node the entire scroll is deal not just one shard.
+     *
+     * Additionally there is the possibility to associate the scroll with a seq. id. such that we can talk to any replica as long as
+     * the shards engine hasn't advanced that seq. id yet. Such a resume is possible and best effort, it could be even a safety net since
+     * if you rely on indices being read-only things can change in-between without notification or it's hard to detect if there where any
+     * changes while scrolling. These are all options to improve the current situation which we can look into down the road
+     */
+    protected final Logger logger;
+    protected final ActionListener<SearchResponse> listener;
+    protected final ParsedScrollId scrollId;
+    protected final DiscoveryNodes nodes;
+    protected final SearchPhaseController searchPhaseController;
+    protected final SearchScrollRequest request;
+    protected final SearchTransportService searchTransportService;
+    private final long startTime;
+    private final List<ShardSearchFailure> shardFailures = new ArrayList<>();
+    private final AtomicInteger successfulOps;
+
+    protected SearchScrollAsyncAction(ParsedScrollId scrollId, Logger logger, DiscoveryNodes nodes,
+                                      ActionListener<SearchResponse> listener, SearchPhaseController searchPhaseController,
+                                      SearchScrollRequest request,
+                                      SearchTransportService searchTransportService) {
+        this.startTime = System.currentTimeMillis();
+        this.scrollId = scrollId;
+        this.successfulOps = new AtomicInteger(scrollId.getContext().length);
+        this.logger = logger;
+        this.listener = listener;
+        this.nodes = nodes;
+        this.searchPhaseController = searchPhaseController;
+        this.request = request;
+        this.searchTransportService = searchTransportService;
+    }
+
+    /**
+     * Builds how long it took to execute the search.
+     */
+    private long buildTookInMillis() {
+        // protect ourselves against time going backwards
+        // negative values don't make sense and we want to be able to serialize that thing as a vLong
+        return Math.max(1, System.currentTimeMillis() - startTime);
+    }
+
+    public final void run() {
+        final ScrollIdForNode[] context = scrollId.getContext();
+        if (context.length == 0) {
+            listener.onFailure(new SearchPhaseExecutionException("query", "no nodes to search on", ShardSearchFailure.EMPTY_ARRAY));
+        } else {
+            collectNodesAndRun(Arrays.asList(context), nodes, searchTransportService, ActionListener.wrap(lookup -> run(lookup, context),
+                listener::onFailure));
+        }
+    }
+
+    /**
+     * This method collects nodes from the remote clusters asynchronously if any of the scroll IDs references a remote cluster.
+     * Otherwise the action listener will be invoked immediately with a function based on the given discovery nodes.
+     */
+    static void collectNodesAndRun(final Iterable<ScrollIdForNode> scrollIds, DiscoveryNodes nodes,
+                                   SearchTransportService searchTransportService,
+                                   ActionListener<BiFunction<String, String, DiscoveryNode>> listener) {
+        Set<String> clusters = new HashSet<>();
+        for (ScrollIdForNode target : scrollIds) {
+            if (target.getClusterAlias() != null) {
+                clusters.add(target.getClusterAlias());
+            }
+        }
+        if (clusters.isEmpty()) { // no remote clusters
+            listener.onResponse((cluster, node) -> nodes.get(node));
+        } else {
+            RemoteClusterService remoteClusterService = searchTransportService.getRemoteClusterService();
+            remoteClusterService.collectNodes(clusters, ActionListener.wrap(nodeFunction -> {
+                final BiFunction<String, String, DiscoveryNode> clusterNodeLookup = (clusterAlias, node) -> {
+                    if (clusterAlias == null) {
+                        return nodes.get(node);
+                    } else {
+                        return nodeFunction.apply(clusterAlias, node);
+                    }
+                };
+                listener.onResponse(clusterNodeLookup);
+            }, listener::onFailure));
+        }
+    }
+
+    private void run(BiFunction<String, String, DiscoveryNode> clusterNodeLookup, final ScrollIdForNode[] context) {
+        final CountDown counter = new CountDown(scrollId.getContext().length);
+        for (int i = 0; i < context.length; i++) {
+            ScrollIdForNode target = context[i];
+            final int shardIndex = i;
+            final Transport.Connection connection;
+            try {
+                DiscoveryNode node = clusterNodeLookup.apply(target.getClusterAlias(), target.getNode());
+                if (node == null) {
+                    throw  new IllegalStateException("node [" + target.getNode() + "] is not available");
+                }
+                connection = getConnection(target.getClusterAlias(), node);
+            } catch (Exception ex) {
+                onShardFailure("query", counter, target.getScrollId(),
+                    ex, null, () -> SearchScrollAsyncAction.this.moveToNextPhase(clusterNodeLookup));
+                continue;
+            }
+            final InternalScrollSearchRequest internalRequest = internalScrollSearchRequest(target.getScrollId(), request);
+            // we can't create a SearchShardTarget here since we don't know the index and shard ID we are talking to
+            // we only know the node and the search context ID. Yet, the response will contain the SearchShardTarget
+            // from the target node instead...that's why we pass null here
+            SearchActionListener<T> searchActionListener = new SearchActionListener<T>(null, shardIndex) {
+
+                @Override
+                protected void setSearchShardTarget(T response) {
+                    // don't do this - it's part of the response...
+                    assert response.getSearchShardTarget() != null : "search shard target must not be null";
+                    if (target.getClusterAlias() != null) {
+                        // re-create the search target and add the cluster alias if there is any,
+                        // we need this down the road for subseq. phases
+                        SearchShardTarget searchShardTarget = response.getSearchShardTarget();
+                        response.setSearchShardTarget(new SearchShardTarget(searchShardTarget.getNodeId(), searchShardTarget.getShardId(),
+                            target.getClusterAlias(), null));
+                    }
+                }
+
+                @Override
+                protected void innerOnResponse(T result) {
+                    assert shardIndex == result.getShardIndex() : "shard index mismatch: " + shardIndex + " but got: "
+                        + result.getShardIndex();
+                    onFirstPhaseResult(shardIndex, result);
+                    if (counter.countDown()) {
+                        SearchPhase phase = moveToNextPhase(clusterNodeLookup);
+                        try {
+                            phase.run();
+                        } catch (Exception e) {
+                            // we need to fail the entire request here - the entire phase just blew up
+                            // don't call onShardFailure or onFailure here since otherwise we'd countDown the counter
+                            // again which would result in an exception
+                            listener.onFailure(new SearchPhaseExecutionException(phase.getName(), "Phase failed", e,
+                                ShardSearchFailure.EMPTY_ARRAY));
+                        }
+                    }
+                }
+
+                @Override
+                public void onFailure(Exception t) {
+                    onShardFailure("query", counter, target.getScrollId(), t, null,
+                        () -> SearchScrollAsyncAction.this.moveToNextPhase(clusterNodeLookup));
+                }
+            };
+            executeInitialPhase(connection, internalRequest, searchActionListener);
+        }
+    }
+
+    synchronized ShardSearchFailure[] buildShardFailures() { // pkg private for testing
+        if (shardFailures.isEmpty()) {
+            return ShardSearchFailure.EMPTY_ARRAY;
+        }
+        return shardFailures.toArray(new ShardSearchFailure[shardFailures.size()]);
+    }
+
+    // we do our best to return the shard failures, but its ok if its not fully concurrently safe
+    // we simply try and return as much as possible
+    private synchronized void addShardFailure(ShardSearchFailure failure) {
+        shardFailures.add(failure);
+    }
+
+    protected abstract void executeInitialPhase(Transport.Connection connection, InternalScrollSearchRequest internalRequest,
+                                                SearchActionListener<T> searchActionListener);
+
+    protected abstract SearchPhase moveToNextPhase(BiFunction<String, String, DiscoveryNode> clusterNodeLookup);
+
+    protected abstract void onFirstPhaseResult(int shardId, T result);
+
+    protected SearchPhase sendResponsePhase(SearchPhaseController.ReducedQueryPhase queryPhase,
+                                            final AtomicArray<? extends SearchPhaseResult> fetchResults) {
+        return new SearchPhase("fetch") {
+            @Override
+            public void run() throws IOException {
+                sendResponse(queryPhase, fetchResults);
+            }
+        };
+    }
+
+    protected final void sendResponse(SearchPhaseController.ReducedQueryPhase queryPhase,
+                                      final AtomicArray<? extends SearchPhaseResult> fetchResults) {
+        try {
+            final InternalSearchResponse internalResponse = searchPhaseController.merge(true, queryPhase, fetchResults.asList(),
+                fetchResults::get);
+            // the scroll ID never changes we always return the same ID. This ID contains all the shards and their context ids
+            // such that we can talk to them abgain in the next roundtrip.
+            String scrollId = null;
+            if (request.scroll() != null) {
+                scrollId = request.scrollId();
+            }
+            listener.onResponse(new SearchResponse(internalResponse, scrollId, this.scrollId.getContext().length, successfulOps.get(),
+                0, buildTookInMillis(), buildShardFailures()));
+        } catch (Exception e) {
+            listener.onFailure(new ReduceSearchPhaseException("fetch", "inner finish failed", e, buildShardFailures()));
+        }
+    }
+
+    protected void onShardFailure(String phaseName, final CountDown counter, final long searchId, Exception failure,
+                                  @Nullable SearchShardTarget searchShardTarget,
+                                  Supplier<SearchPhase> nextPhaseSupplier) {
+        if (logger.isDebugEnabled()) {
+            logger.debug((Supplier<?>) () -> new ParameterizedMessage("[{}] Failed to execute {} phase", searchId, phaseName), failure);
+        }
+        addShardFailure(new ShardSearchFailure(failure, searchShardTarget));
+        int successfulOperations = successfulOps.decrementAndGet();
+        assert successfulOperations >= 0 : "successfulOperations must be >= 0 but was: " + successfulOperations;
+        if (counter.countDown()) {
+            if (successfulOps.get() == 0) {
+                listener.onFailure(new SearchPhaseExecutionException(phaseName, "all shards failed", failure, buildShardFailures()));
+            } else {
+                SearchPhase phase = nextPhaseSupplier.get();
+                try {
+                    phase.run();
+                } catch (Exception e) {
+                    e.addSuppressed(failure);
+                    listener.onFailure(new SearchPhaseExecutionException(phase.getName(), "Phase failed", e,
+                        ShardSearchFailure.EMPTY_ARRAY));
+                }
+            }
+        }
+    }
+
+    protected Transport.Connection getConnection(String clusterAlias, DiscoveryNode node) {
+        return searchTransportService.getConnection(clusterAlias, node);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchScrollQueryAndFetchAsyncAction.java b/core/src/main/java/org/elasticsearch/action/search/SearchScrollQueryAndFetchAsyncAction.java
index 24e497954a7ab..7f36d71ae256b 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchScrollQueryAndFetchAsyncAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchScrollQueryAndFetchAsyncAction.java
@@ -20,164 +20,43 @@
 package org.elasticsearch.action.search;
 
 import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.apache.logging.log4j.util.Supplier;
-import org.apache.lucene.search.ScoreDoc;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
-import org.elasticsearch.search.action.SearchTransportService;
-import org.elasticsearch.search.controller.SearchPhaseController;
 import org.elasticsearch.search.fetch.QueryFetchSearchResult;
 import org.elasticsearch.search.fetch.ScrollQueryFetchSearchResult;
 import org.elasticsearch.search.internal.InternalScrollSearchRequest;
-import org.elasticsearch.search.internal.InternalSearchResponse;
+import org.elasticsearch.transport.Transport;
 
-import java.util.List;
-import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.BiFunction;
 
-import static org.elasticsearch.action.search.TransportSearchHelper.internalScrollSearchRequest;
+final class SearchScrollQueryAndFetchAsyncAction extends SearchScrollAsyncAction<ScrollQueryFetchSearchResult> {
 
-class SearchScrollQueryAndFetchAsyncAction extends AbstractAsyncAction {
-
-    private final Logger logger;
-    private final SearchPhaseController searchPhaseController;
-    private final SearchTransportService searchTransportService;
-    private final SearchScrollRequest request;
-    private final ActionListener<SearchResponse> listener;
-    private final ParsedScrollId scrollId;
-    private final DiscoveryNodes nodes;
-    private volatile AtomicArray<ShardSearchFailure> shardFailures;
+    private final SearchTask task;
     private final AtomicArray<QueryFetchSearchResult> queryFetchResults;
-    private final AtomicInteger successfulOps;
-    private final AtomicInteger counter;
-
-    SearchScrollQueryAndFetchAsyncAction(Logger logger, ClusterService clusterService,
-                                         SearchTransportService searchTransportService, SearchPhaseController searchPhaseController,
-                                         SearchScrollRequest request, ParsedScrollId scrollId, ActionListener<SearchResponse> listener) {
-        this.logger = logger;
-        this.searchPhaseController = searchPhaseController;
-        this.searchTransportService = searchTransportService;
-        this.request = request;
-        this.listener = listener;
-        this.scrollId = scrollId;
-        this.nodes = clusterService.state().nodes();
-        this.successfulOps = new AtomicInteger(scrollId.getContext().length);
-        this.counter = new AtomicInteger(scrollId.getContext().length);
 
+    SearchScrollQueryAndFetchAsyncAction(Logger logger, ClusterService clusterService, SearchTransportService searchTransportService,
+                                         SearchPhaseController searchPhaseController, SearchScrollRequest request, SearchTask task,
+                                         ParsedScrollId scrollId, ActionListener<SearchResponse> listener) {
+        super(scrollId, logger, clusterService.state().nodes(), listener, searchPhaseController, request, searchTransportService);
+        this.task = task;
         this.queryFetchResults = new AtomicArray<>(scrollId.getContext().length);
     }
 
-    protected final ShardSearchFailure[] buildShardFailures() {
-        if (shardFailures == null) {
-            return ShardSearchFailure.EMPTY_ARRAY;
-        }
-        List<AtomicArray.Entry<ShardSearchFailure>> entries = shardFailures.asList();
-        ShardSearchFailure[] failures = new ShardSearchFailure[entries.size()];
-        for (int i = 0; i < failures.length; i++) {
-            failures[i] = entries.get(i).value;
-        }
-        return failures;
-    }
-
-    // we do our best to return the shard failures, but its ok if its not fully concurrently safe
-    // we simply try and return as much as possible
-    protected final void addShardFailure(final int shardIndex, ShardSearchFailure failure) {
-        if (shardFailures == null) {
-            shardFailures = new AtomicArray<>(scrollId.getContext().length);
-        }
-        shardFailures.set(shardIndex, failure);
-    }
-
-    public void start() {
-        if (scrollId.getContext().length == 0) {
-            listener.onFailure(new SearchPhaseExecutionException("query", "no nodes to search on", ShardSearchFailure.EMPTY_ARRAY));
-            return;
-        }
-
-        ScrollIdForNode[] context = scrollId.getContext();
-        for (int i = 0; i < context.length; i++) {
-            ScrollIdForNode target = context[i];
-            DiscoveryNode node = nodes.get(target.getNode());
-            if (node != null) {
-                executePhase(i, node, target.getScrollId());
-            } else {
-                if (logger.isDebugEnabled()) {
-                    logger.debug("Node [{}] not available for scroll request [{}]", target.getNode(), scrollId.getSource());
-                }
-                successfulOps.decrementAndGet();
-                if (counter.decrementAndGet() == 0) {
-                    finishHim();
-                }
-            }
-        }
-
-        for (ScrollIdForNode target : scrollId.getContext()) {
-            DiscoveryNode node = nodes.get(target.getNode());
-            if (node == null) {
-                if (logger.isDebugEnabled()) {
-                    logger.debug("Node [{}] not available for scroll request [{}]", target.getNode(), scrollId.getSource());
-                }
-                successfulOps.decrementAndGet();
-                if (counter.decrementAndGet() == 0) {
-                    finishHim();
-                }
-            }
-        }
-    }
-
-    void executePhase(final int shardIndex, DiscoveryNode node, final long searchId) {
-        InternalScrollSearchRequest internalRequest = internalScrollSearchRequest(searchId, request);
-        searchTransportService.sendExecuteFetch(node, internalRequest, new ActionListener<ScrollQueryFetchSearchResult>() {
-            @Override
-            public void onResponse(ScrollQueryFetchSearchResult result) {
-                queryFetchResults.set(shardIndex, result.result());
-                if (counter.decrementAndGet() == 0) {
-                    finishHim();
-                }
-            }
-
-            @Override
-            public void onFailure(Exception t) {
-                onPhaseFailure(t, searchId, shardIndex);
-            }
-        });
-    }
-
-    private void onPhaseFailure(Exception e, long searchId, int shardIndex) {
-        if (logger.isDebugEnabled()) {
-            logger.debug((Supplier<?>) () -> new ParameterizedMessage("[{}] Failed to execute query phase", searchId), e);
-        }
-        addShardFailure(shardIndex, new ShardSearchFailure(e));
-        successfulOps.decrementAndGet();
-        if (counter.decrementAndGet() == 0) {
-            if (successfulOps.get() == 0) {
-                listener.onFailure(new SearchPhaseExecutionException("query_fetch", "all shards failed", e, buildShardFailures()));
-            } else {
-                finishHim();
-            }
-        }
+    @Override
+    protected void executeInitialPhase(Transport.Connection connection, InternalScrollSearchRequest internalRequest,
+                                       SearchActionListener<ScrollQueryFetchSearchResult> searchActionListener) {
+        searchTransportService.sendExecuteScrollFetch(connection, internalRequest, task, searchActionListener);
     }
 
-    private void finishHim() {
-        try {
-            innerFinishHim();
-        } catch (Exception e) {
-            listener.onFailure(new ReduceSearchPhaseException("fetch", "", e, buildShardFailures()));
-        }
+    @Override
+    protected SearchPhase moveToNextPhase(BiFunction<String, String, DiscoveryNode> clusterNodeLookup) {
+        return sendResponsePhase(searchPhaseController.reducedQueryPhase(queryFetchResults.asList(), true), queryFetchResults);
     }
 
-    private void innerFinishHim() throws Exception {
-        ScoreDoc[] sortedShardDocs = searchPhaseController.sortDocs(true, queryFetchResults);
-        final InternalSearchResponse internalResponse = searchPhaseController.merge(true, sortedShardDocs, queryFetchResults,
-            queryFetchResults);
-        String scrollId = null;
-        if (request.scroll() != null) {
-            scrollId = request.scrollId();
-        }
-        listener.onResponse(new SearchResponse(internalResponse, scrollId, this.scrollId.getContext().length, successfulOps.get(),
-            buildTookInMillis(), buildShardFailures()));
+    @Override
+    protected void onFirstPhaseResult(int shardId, ScrollQueryFetchSearchResult result) {
+        queryFetchResults.setOnce(shardId, result.result());
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java b/core/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java
index 21f1c4ce68a34..7363ca41c20ad 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java
@@ -21,210 +21,104 @@
 
 import com.carrotsearch.hppc.IntArrayList;
 import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.apache.logging.log4j.util.Supplier;
 import org.apache.lucene.search.ScoreDoc;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
-import org.elasticsearch.search.action.SearchTransportService;
-import org.elasticsearch.search.controller.SearchPhaseController;
+import org.elasticsearch.common.util.concurrent.CountDown;
+import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.fetch.FetchSearchResult;
 import org.elasticsearch.search.fetch.ShardFetchRequest;
 import org.elasticsearch.search.internal.InternalScrollSearchRequest;
-import org.elasticsearch.search.internal.InternalSearchResponse;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.search.query.ScrollQuerySearchResult;
+import org.elasticsearch.transport.Transport;
 
-import java.util.List;
-import java.util.concurrent.atomic.AtomicInteger;
+import java.io.IOException;
+import java.util.function.BiFunction;
 
-import static org.elasticsearch.action.search.TransportSearchHelper.internalScrollSearchRequest;
+final class SearchScrollQueryThenFetchAsyncAction extends SearchScrollAsyncAction<ScrollQuerySearchResult> {
 
-class SearchScrollQueryThenFetchAsyncAction extends AbstractAsyncAction {
+    private final SearchTask task;
+    private final AtomicArray<FetchSearchResult> fetchResults;
+    private final AtomicArray<QuerySearchResult> queryResults;
 
-    private final Logger logger;
-    private final SearchTransportService searchTransportService;
-    private final SearchPhaseController searchPhaseController;
-    private final SearchScrollRequest request;
-    private final ActionListener<SearchResponse> listener;
-    private final ParsedScrollId scrollId;
-    private final DiscoveryNodes nodes;
-    private volatile AtomicArray<ShardSearchFailure> shardFailures;
-    final AtomicArray<QuerySearchResult> queryResults;
-    final AtomicArray<FetchSearchResult> fetchResults;
-    private volatile ScoreDoc[] sortedShardDocs;
-    private final AtomicInteger successfulOps;
-
-    SearchScrollQueryThenFetchAsyncAction(Logger logger, ClusterService clusterService,
-                                          SearchTransportService searchTransportService, SearchPhaseController searchPhaseController,
-                                          SearchScrollRequest request, ParsedScrollId scrollId, ActionListener<SearchResponse> listener) {
-        this.logger = logger;
-        this.searchTransportService = searchTransportService;
-        this.searchPhaseController = searchPhaseController;
-        this.request = request;
-        this.listener = listener;
-        this.scrollId = scrollId;
-        this.nodes = clusterService.state().nodes();
-        this.successfulOps = new AtomicInteger(scrollId.getContext().length);
-        this.queryResults = new AtomicArray<>(scrollId.getContext().length);
+    SearchScrollQueryThenFetchAsyncAction(Logger logger, ClusterService clusterService, SearchTransportService searchTransportService,
+                                          SearchPhaseController searchPhaseController, SearchScrollRequest request, SearchTask task,
+                                          ParsedScrollId scrollId, ActionListener<SearchResponse> listener) {
+        super(scrollId, logger, clusterService.state().nodes(), listener, searchPhaseController, request,
+            searchTransportService);
+        this.task = task;
         this.fetchResults = new AtomicArray<>(scrollId.getContext().length);
+        this.queryResults = new AtomicArray<>(scrollId.getContext().length);
     }
 
-    protected final ShardSearchFailure[] buildShardFailures() {
-        if (shardFailures == null) {
-            return ShardSearchFailure.EMPTY_ARRAY;
-        }
-        List<AtomicArray.Entry<ShardSearchFailure>> entries = shardFailures.asList();
-        ShardSearchFailure[] failures = new ShardSearchFailure[entries.size()];
-        for (int i = 0; i < failures.length; i++) {
-            failures[i] = entries.get(i).value;
-        }
-        return failures;
-    }
-
-    // we do our best to return the shard failures, but its ok if its not fully concurrently safe
-    // we simply try and return as much as possible
-    protected final void addShardFailure(final int shardIndex, ShardSearchFailure failure) {
-        if (shardFailures == null) {
-            shardFailures = new AtomicArray<>(scrollId.getContext().length);
-        }
-        shardFailures.set(shardIndex, failure);
+    protected void onFirstPhaseResult(int shardId, ScrollQuerySearchResult result) {
+        queryResults.setOnce(shardId, result.queryResult());
     }
 
-    public void start() {
-        if (scrollId.getContext().length == 0) {
-            listener.onFailure(new SearchPhaseExecutionException("query", "no nodes to search on", ShardSearchFailure.EMPTY_ARRAY));
-            return;
-        }
-        final AtomicInteger counter = new AtomicInteger(scrollId.getContext().length);
-
-        ScrollIdForNode[] context = scrollId.getContext();
-        for (int i = 0; i < context.length; i++) {
-            ScrollIdForNode target = context[i];
-            DiscoveryNode node = nodes.get(target.getNode());
-            if (node != null) {
-                executeQueryPhase(i, counter, node, target.getScrollId());
-            } else {
-                if (logger.isDebugEnabled()) {
-                    logger.debug("Node [{}] not available for scroll request [{}]", target.getNode(), scrollId.getSource());
-                }
-                successfulOps.decrementAndGet();
-                if (counter.decrementAndGet() == 0) {
-                    try {
-                        executeFetchPhase();
-                    } catch (Exception e) {
-                        listener.onFailure(new SearchPhaseExecutionException("query", "Fetch failed", e, ShardSearchFailure.EMPTY_ARRAY));
-                        return;
-                    }
-                }
-            }
-        }
+    @Override
+    protected void executeInitialPhase(Transport.Connection connection, InternalScrollSearchRequest internalRequest,
+                                       SearchActionListener<ScrollQuerySearchResult> searchActionListener) {
+        searchTransportService.sendExecuteScrollQuery(connection, internalRequest, task, searchActionListener);
     }
 
-    private void executeQueryPhase(final int shardIndex, final AtomicInteger counter, DiscoveryNode node, final long searchId) {
-        InternalScrollSearchRequest internalRequest = internalScrollSearchRequest(searchId, request);
-        searchTransportService.sendExecuteQuery(node, internalRequest, new ActionListener<ScrollQuerySearchResult>() {
-            @Override
-            public void onResponse(ScrollQuerySearchResult result) {
-                queryResults.set(shardIndex, result.queryResult());
-                if (counter.decrementAndGet() == 0) {
-                    try {
-                        executeFetchPhase();
-                    } catch (Exception e) {
-                        onFailure(e);
-                    }
-                }
-            }
-
+    @Override
+    protected SearchPhase moveToNextPhase(BiFunction<String, String, DiscoveryNode> clusterNodeLookup) {
+        return new SearchPhase("fetch") {
             @Override
-            public void onFailure(Exception t) {
-                onQueryPhaseFailure(shardIndex, counter, searchId, t);
-            }
-        });
-    }
-
-    void onQueryPhaseFailure(final int shardIndex, final AtomicInteger counter, final long searchId, Exception failure) {
-        if (logger.isDebugEnabled()) {
-            logger.debug((Supplier<?>) () -> new ParameterizedMessage("[{}] Failed to execute query phase", searchId), failure);
-        }
-        addShardFailure(shardIndex, new ShardSearchFailure(failure));
-        successfulOps.decrementAndGet();
-        if (counter.decrementAndGet() == 0) {
-            if (successfulOps.get() == 0) {
-                listener.onFailure(new SearchPhaseExecutionException("query", "all shards failed", failure, buildShardFailures()));
-            } else {
-                try {
-                    executeFetchPhase();
-                } catch (Exception e) {
-                    e.addSuppressed(failure);
-                    listener.onFailure(new SearchPhaseExecutionException("query", "Fetch failed", e, ShardSearchFailure.EMPTY_ARRAY));
-                }
-            }
-        }
-    }
-
-    private void executeFetchPhase() throws Exception {
-        sortedShardDocs = searchPhaseController.sortDocs(true, queryResults);
-        AtomicArray<IntArrayList> docIdsToLoad = new AtomicArray<>(queryResults.length());
-        searchPhaseController.fillDocIdsToLoad(docIdsToLoad, sortedShardDocs);
-
-        if (docIdsToLoad.asList().isEmpty()) {
-            finishHim();
-            return;
-        }
-
-
-        final ScoreDoc[] lastEmittedDocPerShard = searchPhaseController.getLastEmittedDocPerShard(queryResults.asList(),
-            sortedShardDocs, queryResults.length());
-        final AtomicInteger counter = new AtomicInteger(docIdsToLoad.asList().size());
-        for (final AtomicArray.Entry<IntArrayList> entry : docIdsToLoad.asList()) {
-            IntArrayList docIds = entry.value;
-            final QuerySearchResult querySearchResult = queryResults.get(entry.index);
-            ScoreDoc lastEmittedDoc = lastEmittedDocPerShard[entry.index];
-            ShardFetchRequest shardFetchRequest = new ShardFetchRequest(querySearchResult.id(), docIds, lastEmittedDoc);
-            DiscoveryNode node = nodes.get(querySearchResult.shardTarget().nodeId());
-            searchTransportService.sendExecuteFetchScroll(node, shardFetchRequest, new ActionListener<FetchSearchResult>() {
-                @Override
-                public void onResponse(FetchSearchResult result) {
-                    result.shardTarget(querySearchResult.shardTarget());
-                    fetchResults.set(entry.index, result);
-                    if (counter.decrementAndGet() == 0) {
-                        finishHim();
-                    }
+            public void run() throws IOException {
+                final SearchPhaseController.ReducedQueryPhase reducedQueryPhase = searchPhaseController.reducedQueryPhase(
+                    queryResults.asList(), true);
+                if (reducedQueryPhase.scoreDocs.length == 0) {
+                    sendResponse(reducedQueryPhase, fetchResults);
+                    return;
                 }
 
-                @Override
-                public void onFailure(Exception t) {
-                    if (logger.isDebugEnabled()) {
-                        logger.debug("Failed to execute fetch phase", t);
-                    }
-                    successfulOps.decrementAndGet();
-                    if (counter.decrementAndGet() == 0) {
-                        finishHim();
+                final IntArrayList[] docIdsToLoad = searchPhaseController.fillDocIdsToLoad(queryResults.length(),
+                    reducedQueryPhase.scoreDocs);
+                final ScoreDoc[] lastEmittedDocPerShard = searchPhaseController.getLastEmittedDocPerShard(reducedQueryPhase,
+                    queryResults.length());
+                final CountDown counter = new CountDown(docIdsToLoad.length);
+                for (int i = 0; i < docIdsToLoad.length; i++) {
+                    final int index = i;
+                    final IntArrayList docIds = docIdsToLoad[index];
+                    if (docIds != null) {
+                        final QuerySearchResult querySearchResult = queryResults.get(index);
+                        ScoreDoc lastEmittedDoc = lastEmittedDocPerShard[index];
+                        ShardFetchRequest shardFetchRequest = new ShardFetchRequest(querySearchResult.getRequestId(), docIds,
+                            lastEmittedDoc);
+                        SearchShardTarget searchShardTarget = querySearchResult.getSearchShardTarget();
+                        DiscoveryNode node = clusterNodeLookup.apply(searchShardTarget.getClusterAlias(), searchShardTarget.getNodeId());
+                        assert node != null : "target node is null in secondary phase";
+                        Transport.Connection connection = getConnection(searchShardTarget.getClusterAlias(), node);
+                        searchTransportService.sendExecuteFetchScroll(connection, shardFetchRequest, task,
+                            new SearchActionListener<FetchSearchResult>(querySearchResult.getSearchShardTarget(), index) {
+                                @Override
+                                protected void innerOnResponse(FetchSearchResult response) {
+                                    fetchResults.setOnce(response.getShardIndex(), response);
+                                    if (counter.countDown()) {
+                                        sendResponse(reducedQueryPhase, fetchResults);
+                                    }
+                                }
+
+                                @Override
+                                public void onFailure(Exception t) {
+                                    onShardFailure(getName(), counter, querySearchResult.getRequestId(),
+                                        t, querySearchResult.getSearchShardTarget(),
+                                        () -> sendResponsePhase(reducedQueryPhase, fetchResults));
+                                }
+                            });
+                    } else {
+                        // the counter is set to the total size of docIdsToLoad
+                        // which can have null values so we have to count them down too
+                        if (counter.countDown()) {
+                            sendResponse(reducedQueryPhase, fetchResults);
+                        }
                     }
                 }
-            });
-        }
-    }
-
-    private void finishHim() {
-        try {
-            innerFinishHim();
-        } catch (Exception e) {
-            listener.onFailure(new ReduceSearchPhaseException("fetch", "inner finish failed", e, buildShardFailures()));
-        }
-    }
-
-    private void innerFinishHim() {
-        InternalSearchResponse internalResponse = searchPhaseController.merge(true, sortedShardDocs, queryResults, fetchResults);
-        String scrollId = null;
-        if (request.scroll() != null) {
-            scrollId = request.scrollId();
-        }
-        listener.onResponse(new SearchResponse(internalResponse, scrollId, this.scrollId.getContext().length, successfulOps.get(),
-            buildTookInMillis(), buildShardFailures()));
+            }
+        };
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchScrollRequest.java b/core/src/main/java/org/elasticsearch/action/search/SearchScrollRequest.java
index 9ab2a4cf5605a..7bd6677abbd89 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchScrollRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchScrollRequest.java
@@ -24,18 +24,19 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.search.Scroll;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskId;
 
 import java.io.IOException;
 import java.util.Objects;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 
-/**
- *
- */
-public class SearchScrollRequest extends ActionRequest<SearchScrollRequest> {
-
+public class SearchScrollRequest extends ActionRequest implements ToXContentObject {
     private String scrollId;
     private Scroll scroll;
 
@@ -110,6 +111,11 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalWriteable(scroll);
     }
 
+    @Override
+    public Task createTask(long id, String type, String action, TaskId parentTaskId) {
+        return new SearchTask(id, type, action, getDescription(), parentTaskId);
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) {
@@ -135,4 +141,45 @@ public String toString() {
                 ", scroll=" + scroll +
                 '}';
     }
+
+    @Override
+    public String getDescription() {
+        return "scrollId[" + scrollId + "], scroll[" + scroll + "]";
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field("scroll_id", scrollId);
+        if (scroll != null) {
+            builder.field("scroll", scroll.keepAlive().getStringRep());
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    /**
+     * Parse a search scroll request from a request body provided through the REST layer.
+     * Values that are already be set and are also found while parsing will be overridden.
+     */
+    public void fromXContent(XContentParser parser) throws IOException {
+        if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
+            throw new IllegalArgumentException("Malformed content, must start with an object");
+        } else {
+            XContentParser.Token token;
+            String currentFieldName = null;
+            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                if (token == XContentParser.Token.FIELD_NAME) {
+                    currentFieldName = parser.currentName();
+                } else if ("scroll_id".equals(currentFieldName) && token == XContentParser.Token.VALUE_STRING) {
+                    scrollId(parser.text());
+                } else if ("scroll".equals(currentFieldName) && token == XContentParser.Token.VALUE_STRING) {
+                    scroll(new Scroll(TimeValue.parseTimeValue(parser.text(), null, "scroll")));
+                } else {
+                    throw new IllegalArgumentException("Unknown parameter [" + currentFieldName
+                            + "] in request body or parameter is of the wrong type[" + token + "] ");
+                }
+            }
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchShardIterator.java b/core/src/main/java/org/elasticsearch/action/search/SearchShardIterator.java
new file mode 100644
index 0000000000000..c36d2b7908f78
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchShardIterator.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.search;
+
+import org.elasticsearch.action.OriginalIndices;
+import org.elasticsearch.cluster.routing.PlainShardIterator;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.index.shard.ShardId;
+
+import java.util.List;
+
+/**
+ * Extension of {@link PlainShardIterator} used in the search api, which also holds the {@link OriginalIndices}
+ * of the search request. Useful especially with cross cluster search, as each cluster has its own set of original indices.
+ */
+public final class SearchShardIterator extends PlainShardIterator {
+
+    private final OriginalIndices originalIndices;
+    private String clusterAlias;
+    private boolean skip = false;
+
+    /**
+     * Creates a {@link PlainShardIterator} instance that iterates over a subset of the given shards
+     * this the a given <code>shardId</code>.
+     *
+     * @param shardId shard id of the group
+     * @param shards  shards to iterate
+     */
+    public SearchShardIterator(String clusterAlias, ShardId shardId, List<ShardRouting> shards, OriginalIndices originalIndices) {
+        super(shardId, shards);
+        this.originalIndices = originalIndices;
+        this.clusterAlias = clusterAlias;
+    }
+
+    /**
+     * Returns the original indices associated with this shard iterator, specifically with the cluster that this shard belongs to.
+     */
+    public OriginalIndices getOriginalIndices() {
+        return originalIndices;
+    }
+
+    public String getClusterAlias() {
+        return clusterAlias;
+    }
+
+    /**
+     * Reset the iterator and mark it as skippable
+     * @see #skip()
+     */
+    void resetAndSkip() {
+        reset();
+        skip = true;
+    }
+
+    /**
+     * Returns <code>true</code> if the search execution should skip this shard since it can not match any documents given the query.
+     */
+    boolean skip() {
+        return skip;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchTask.java b/core/src/main/java/org/elasticsearch/action/search/SearchTask.java
new file mode 100644
index 0000000000000..d0a1cdd456f47
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchTask.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.search;
+
+import org.elasticsearch.tasks.CancellableTask;
+import org.elasticsearch.tasks.TaskId;
+
+/**
+ * Task storing information about a currently running search request.
+ */
+public class SearchTask extends CancellableTask {
+
+    public SearchTask(long id, String type, String action, String description, TaskId parentTaskId) {
+        super(id, type, action, description, parentTaskId);
+    }
+
+    @Override
+    public boolean shouldCancelChildrenOnCancellation() {
+        return true;
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchTransportService.java b/core/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
new file mode 100644
index 0000000000000..052f2bb0fdeba
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
@@ -0,0 +1,469 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.search;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionListenerResponseHandler;
+import org.elasticsearch.action.IndicesRequest;
+import org.elasticsearch.action.OriginalIndices;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.component.AbstractComponent;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.search.SearchPhaseResult;
+import org.elasticsearch.search.SearchService;
+import org.elasticsearch.search.dfs.DfsSearchResult;
+import org.elasticsearch.search.fetch.FetchSearchResult;
+import org.elasticsearch.search.fetch.QueryFetchSearchResult;
+import org.elasticsearch.search.fetch.ScrollQueryFetchSearchResult;
+import org.elasticsearch.search.fetch.ShardFetchRequest;
+import org.elasticsearch.search.fetch.ShardFetchSearchRequest;
+import org.elasticsearch.search.internal.InternalScrollSearchRequest;
+import org.elasticsearch.search.internal.ShardSearchRequest;
+import org.elasticsearch.search.internal.ShardSearchTransportRequest;
+import org.elasticsearch.search.query.QuerySearchRequest;
+import org.elasticsearch.search.query.QuerySearchResult;
+import org.elasticsearch.search.query.ScrollQuerySearchResult;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.RemoteClusterService;
+import org.elasticsearch.transport.Transport;
+import org.elasticsearch.transport.TransportActionProxy;
+import org.elasticsearch.transport.TaskAwareTransportRequestHandler;
+import org.elasticsearch.transport.TransportChannel;
+import org.elasticsearch.transport.TransportRequest;
+import org.elasticsearch.transport.TransportRequestOptions;
+import org.elasticsearch.transport.TransportResponse;
+import org.elasticsearch.transport.TransportService;
+
+import java.io.IOException;
+import java.util.function.Supplier;
+
+/**
+ * An encapsulation of {@link org.elasticsearch.search.SearchService} operations exposed through
+ * transport.
+ */
+public class SearchTransportService extends AbstractComponent {
+
+    public static final String FREE_CONTEXT_SCROLL_ACTION_NAME = "indices:data/read/search[free_context/scroll]";
+    public static final String FREE_CONTEXT_ACTION_NAME = "indices:data/read/search[free_context]";
+    public static final String CLEAR_SCROLL_CONTEXTS_ACTION_NAME = "indices:data/read/search[clear_scroll_contexts]";
+    public static final String DFS_ACTION_NAME = "indices:data/read/search[phase/dfs]";
+    public static final String QUERY_ACTION_NAME = "indices:data/read/search[phase/query]";
+    public static final String QUERY_ID_ACTION_NAME = "indices:data/read/search[phase/query/id]";
+    public static final String QUERY_SCROLL_ACTION_NAME = "indices:data/read/search[phase/query/scroll]";
+    @Deprecated
+    public static final String QUERY_FETCH_ACTION_NAME = "indices:data/read/search[phase/query+fetch]";
+    public static final String QUERY_FETCH_SCROLL_ACTION_NAME = "indices:data/read/search[phase/query+fetch/scroll]";
+    public static final String FETCH_ID_SCROLL_ACTION_NAME = "indices:data/read/search[phase/fetch/id/scroll]";
+    public static final String FETCH_ID_ACTION_NAME = "indices:data/read/search[phase/fetch/id]";
+    public static final String QUERY_CAN_MATCH_NAME = "indices:data/read/search[can_match]";
+
+    private final TransportService transportService;
+
+    public SearchTransportService(Settings settings, TransportService transportService) {
+        super(settings);
+        this.transportService = transportService;
+    }
+
+    public void sendFreeContext(Transport.Connection connection, final long contextId, OriginalIndices originalIndices) {
+        transportService.sendRequest(connection, FREE_CONTEXT_ACTION_NAME, new SearchFreeContextRequest(originalIndices, contextId),
+            TransportRequestOptions.EMPTY, new ActionListenerResponseHandler<>(new ActionListener<SearchFreeContextResponse>() {
+                @Override
+                public void onResponse(SearchFreeContextResponse response) {
+                    // no need to respond if it was freed or not
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+
+                }
+            }, SearchFreeContextResponse::new));
+    }
+
+    public void sendFreeContext(Transport.Connection connection, long contextId, final ActionListener<SearchFreeContextResponse> listener) {
+        transportService.sendRequest(connection, FREE_CONTEXT_SCROLL_ACTION_NAME, new ScrollFreeContextRequest(contextId),
+            TransportRequestOptions.EMPTY, new ActionListenerResponseHandler<>(listener, SearchFreeContextResponse::new));
+    }
+
+    public void sendCanMatch(Transport.Connection connection, final ShardSearchTransportRequest request, SearchTask task, final
+                            ActionListener<CanMatchResponse> listener) {
+        if (connection.getNode().getVersion().onOrAfter(Version.V_5_6_0)) {
+            transportService.sendChildRequest(connection, QUERY_CAN_MATCH_NAME, request, task,
+                TransportRequestOptions.EMPTY, new ActionListenerResponseHandler<>(listener, CanMatchResponse::new));
+        } else {
+            throw new IllegalArgumentException("can_match is not supported on pre 5.6.0 nodes");
+        }
+    }
+
+    public void sendClearAllScrollContexts(Transport.Connection connection, final ActionListener<TransportResponse> listener) {
+        transportService.sendRequest(connection, CLEAR_SCROLL_CONTEXTS_ACTION_NAME, TransportRequest.Empty.INSTANCE,
+            TransportRequestOptions.EMPTY, new ActionListenerResponseHandler<>(listener, () -> TransportResponse.Empty.INSTANCE));
+    }
+
+    public void sendExecuteDfs(Transport.Connection connection, final ShardSearchTransportRequest request, SearchTask task,
+                               final SearchActionListener<DfsSearchResult> listener) {
+        transportService.sendChildRequest(connection, DFS_ACTION_NAME, request, task,
+            new ActionListenerResponseHandler<>(listener, DfsSearchResult::new));
+    }
+
+    public void sendExecuteQuery(Transport.Connection connection, final ShardSearchTransportRequest request, SearchTask task,
+                                 final SearchActionListener<SearchPhaseResult> listener) {
+        // we optimize this and expect a QueryFetchSearchResult if we only have a single shard in the search request
+        // this used to be the QUERY_AND_FETCH which doesn't exists anymore.
+        final boolean fetchDocuments = request.numberOfShards() == 1;
+        Supplier<SearchPhaseResult> supplier = fetchDocuments ? QueryFetchSearchResult::new : QuerySearchResult::new;
+        if (connection.getVersion().before(Version.V_5_3_0) && fetchDocuments) {
+            // this is a BWC layer for pre 5.3 indices
+            if (request.scroll() != null) {
+                /**
+                 * This is needed for nodes pre 5.3 when the single shard optimization is used.
+                 * These nodes will set the last emitted doc only if the removed `query_and_fetch` search type is set
+                 * in the request. See {@link SearchType}.
+                 */
+                request.searchType(SearchType.QUERY_AND_FETCH);
+            }
+            transportService.sendChildRequest(connection, QUERY_FETCH_ACTION_NAME, request, task,
+                new ActionListenerResponseHandler<>(listener, supplier));
+        } else {
+            transportService.sendChildRequest(connection, QUERY_ACTION_NAME, request, task,
+                new ActionListenerResponseHandler<>(listener, supplier));
+        }
+    }
+
+    public void sendExecuteQuery(Transport.Connection connection, final QuerySearchRequest request, SearchTask task,
+                                 final SearchActionListener<QuerySearchResult> listener) {
+        transportService.sendChildRequest(connection, QUERY_ID_ACTION_NAME, request, task,
+            new ActionListenerResponseHandler<>(listener, QuerySearchResult::new));
+    }
+
+    public void sendExecuteScrollQuery(Transport.Connection connection, final InternalScrollSearchRequest request, SearchTask task,
+                                       final SearchActionListener<ScrollQuerySearchResult> listener) {
+        transportService.sendChildRequest(connection, QUERY_SCROLL_ACTION_NAME, request, task,
+            new ActionListenerResponseHandler<>(listener, ScrollQuerySearchResult::new));
+    }
+
+    public void sendExecuteScrollFetch(Transport.Connection connection, final InternalScrollSearchRequest request, SearchTask task,
+                                       final SearchActionListener<ScrollQueryFetchSearchResult> listener) {
+        transportService.sendChildRequest(connection, QUERY_FETCH_SCROLL_ACTION_NAME, request, task,
+            new ActionListenerResponseHandler<>(listener, ScrollQueryFetchSearchResult::new));
+    }
+
+    public void sendExecuteFetch(Transport.Connection connection, final ShardFetchSearchRequest request, SearchTask task,
+                                 final SearchActionListener<FetchSearchResult> listener) {
+        sendExecuteFetch(connection, FETCH_ID_ACTION_NAME, request, task, listener);
+    }
+
+    public void sendExecuteFetchScroll(Transport.Connection connection, final ShardFetchRequest request, SearchTask task,
+                                       final SearchActionListener<FetchSearchResult> listener) {
+        sendExecuteFetch(connection, FETCH_ID_SCROLL_ACTION_NAME, request, task, listener);
+    }
+
+    private void sendExecuteFetch(Transport.Connection connection, String action, final ShardFetchRequest request, SearchTask task,
+                                  final SearchActionListener<FetchSearchResult> listener) {
+        transportService.sendChildRequest(connection, action, request, task,
+            new ActionListenerResponseHandler<>(listener, FetchSearchResult::new));
+    }
+
+    /**
+     * Used by {@link TransportSearchAction} to send the expand queries (field collapsing).
+     */
+    void sendExecuteMultiSearch(final MultiSearchRequest request, SearchTask task,
+                                       final ActionListener<MultiSearchResponse> listener) {
+        transportService.sendChildRequest(transportService.getConnection(transportService.getLocalNode()), MultiSearchAction.NAME, request,
+            task, new ActionListenerResponseHandler<>(listener, MultiSearchResponse::new));
+    }
+
+    public RemoteClusterService getRemoteClusterService() {
+        return transportService.getRemoteClusterService();
+    }
+
+    static class ScrollFreeContextRequest extends TransportRequest {
+        private long id;
+
+        ScrollFreeContextRequest() {
+        }
+
+        ScrollFreeContextRequest(long id) {
+            this.id = id;
+        }
+
+        public long id() {
+            return this.id;
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            super.readFrom(in);
+            id = in.readLong();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeLong(id);
+        }
+    }
+
+    static class SearchFreeContextRequest extends ScrollFreeContextRequest implements IndicesRequest {
+        private OriginalIndices originalIndices;
+
+        SearchFreeContextRequest() {
+        }
+
+        SearchFreeContextRequest(OriginalIndices originalIndices, long id) {
+            super(id);
+            this.originalIndices = originalIndices;
+        }
+
+        @Override
+        public String[] indices() {
+            if (originalIndices == null) {
+                return null;
+            }
+            return originalIndices.indices();
+        }
+
+        @Override
+        public IndicesOptions indicesOptions() {
+            if (originalIndices == null) {
+                return null;
+            }
+            return originalIndices.indicesOptions();
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            super.readFrom(in);
+            originalIndices = OriginalIndices.readOriginalIndices(in);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            OriginalIndices.writeOriginalIndices(originalIndices, out);
+        }
+    }
+
+    public static class SearchFreeContextResponse extends TransportResponse {
+
+        private boolean freed;
+
+        SearchFreeContextResponse() {
+        }
+
+        SearchFreeContextResponse(boolean freed) {
+            this.freed = freed;
+        }
+
+        public boolean isFreed() {
+            return freed;
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            super.readFrom(in);
+            freed = in.readBoolean();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeBoolean(freed);
+        }
+    }
+
+    public static void registerRequestHandler(TransportService transportService, SearchService searchService) {
+        transportService.registerRequestHandler(FREE_CONTEXT_SCROLL_ACTION_NAME, ScrollFreeContextRequest::new, ThreadPool.Names.SAME,
+            new TaskAwareTransportRequestHandler<ScrollFreeContextRequest>() {
+                @Override
+                public void messageReceived(ScrollFreeContextRequest request, TransportChannel channel, Task task) throws Exception {
+                    boolean freed = searchService.freeContext(request.id());
+                    channel.sendResponse(new SearchFreeContextResponse(freed));
+                }
+            });
+        TransportActionProxy.registerProxyAction(transportService, FREE_CONTEXT_SCROLL_ACTION_NAME,
+                (Supplier<TransportResponse>) SearchFreeContextResponse::new);
+        transportService.registerRequestHandler(FREE_CONTEXT_ACTION_NAME, SearchFreeContextRequest::new, ThreadPool.Names.SAME,
+                new TaskAwareTransportRequestHandler<SearchFreeContextRequest>() {
+                @Override
+                public void messageReceived(SearchFreeContextRequest request, TransportChannel channel, Task task) throws Exception {
+                    boolean freed = searchService.freeContext(request.id());
+                    channel.sendResponse(new SearchFreeContextResponse(freed));
+                }
+            });
+        TransportActionProxy.registerProxyAction(transportService, FREE_CONTEXT_ACTION_NAME,
+                (Supplier<TransportResponse>) SearchFreeContextResponse::new);
+        transportService.registerRequestHandler(CLEAR_SCROLL_CONTEXTS_ACTION_NAME, () -> TransportRequest.Empty.INSTANCE,
+            ThreadPool.Names.SAME, new TaskAwareTransportRequestHandler<TransportRequest.Empty>() {
+                @Override
+                public void messageReceived(TransportRequest.Empty request, TransportChannel channel, Task task) throws Exception {
+                    searchService.freeAllScrollContexts();
+                    channel.sendResponse(TransportResponse.Empty.INSTANCE);
+                }
+            });
+        TransportActionProxy.registerProxyAction(transportService, CLEAR_SCROLL_CONTEXTS_ACTION_NAME,
+                () -> TransportResponse.Empty.INSTANCE);
+
+        transportService.registerRequestHandler(DFS_ACTION_NAME, ShardSearchTransportRequest::new, ThreadPool.Names.SEARCH,
+            new TaskAwareTransportRequestHandler<ShardSearchTransportRequest>() {
+                @Override
+                public void messageReceived(ShardSearchTransportRequest request, TransportChannel channel, Task task) throws Exception {
+                    DfsSearchResult result = searchService.executeDfsPhase(request, (SearchTask)task);
+                    channel.sendResponse(result);
+
+                }
+            });
+        TransportActionProxy.registerProxyAction(transportService, DFS_ACTION_NAME, DfsSearchResult::new);
+
+        transportService.registerRequestHandler(QUERY_ACTION_NAME, ShardSearchTransportRequest::new, ThreadPool.Names.SEARCH,
+            new TaskAwareTransportRequestHandler<ShardSearchTransportRequest>() {
+                @Override
+                public void messageReceived(ShardSearchTransportRequest request, TransportChannel channel, Task task) throws Exception {
+                    SearchPhaseResult result = searchService.executeQueryPhase(request, (SearchTask)task);
+                    channel.sendResponse(result);
+                }
+            });
+        TransportActionProxy.registerProxyAction(transportService, QUERY_ACTION_NAME,
+                (request) -> ((ShardSearchRequest)request).numberOfShards() == 1 ? QueryFetchSearchResult::new : QuerySearchResult::new);
+
+        transportService.registerRequestHandler(QUERY_ID_ACTION_NAME, QuerySearchRequest::new, ThreadPool.Names.SEARCH,
+            new TaskAwareTransportRequestHandler<QuerySearchRequest>() {
+                @Override
+                public void messageReceived(QuerySearchRequest request, TransportChannel channel, Task task) throws Exception {
+                    QuerySearchResult result = searchService.executeQueryPhase(request, (SearchTask)task);
+                    channel.sendResponse(result);
+                }
+            });
+        TransportActionProxy.registerProxyAction(transportService, QUERY_ID_ACTION_NAME, QuerySearchResult::new);
+
+        transportService.registerRequestHandler(QUERY_SCROLL_ACTION_NAME, InternalScrollSearchRequest::new, ThreadPool.Names.SEARCH,
+            new TaskAwareTransportRequestHandler<InternalScrollSearchRequest>() {
+                @Override
+                public void messageReceived(InternalScrollSearchRequest request, TransportChannel channel, Task task) throws Exception {
+                    ScrollQuerySearchResult result = searchService.executeQueryPhase(request, (SearchTask)task);
+                    channel.sendResponse(result);
+                }
+            });
+        TransportActionProxy.registerProxyAction(transportService, QUERY_SCROLL_ACTION_NAME, ScrollQuerySearchResult::new);
+
+        // this is for BWC with pre 5.3 indices in 5.3 we will only execute a `indices:data/read/search[phase/query+fetch]`
+        // if the node is pre 5.3
+        transportService.registerRequestHandler(QUERY_FETCH_ACTION_NAME, ShardSearchTransportRequest::new, ThreadPool.Names.SEARCH,
+            new TaskAwareTransportRequestHandler<ShardSearchTransportRequest>() {
+                @Override
+                public void messageReceived(ShardSearchTransportRequest request, TransportChannel channel, Task task) throws Exception {
+                    assert request.numberOfShards() == 1 : "expected single shard request but got: " + request.numberOfShards();
+                    SearchPhaseResult result = searchService.executeQueryPhase(request, (SearchTask)task);
+                    channel.sendResponse(result);
+                }
+            });
+        TransportActionProxy.registerProxyAction(transportService, QUERY_FETCH_ACTION_NAME, QueryFetchSearchResult::new);
+
+        transportService.registerRequestHandler(QUERY_FETCH_SCROLL_ACTION_NAME, InternalScrollSearchRequest::new, ThreadPool.Names.SEARCH,
+            new TaskAwareTransportRequestHandler<InternalScrollSearchRequest>() {
+                @Override
+                public void messageReceived(InternalScrollSearchRequest request, TransportChannel channel, Task task) throws Exception {
+                    ScrollQueryFetchSearchResult result = searchService.executeFetchPhase(request, (SearchTask)task);
+                    channel.sendResponse(result);
+                }
+            });
+        TransportActionProxy.registerProxyAction(transportService, QUERY_FETCH_SCROLL_ACTION_NAME, ScrollQueryFetchSearchResult::new);
+
+        transportService.registerRequestHandler(FETCH_ID_SCROLL_ACTION_NAME, ShardFetchRequest::new, ThreadPool.Names.SEARCH,
+            new TaskAwareTransportRequestHandler<ShardFetchRequest>() {
+                @Override
+                public void messageReceived(ShardFetchRequest request, TransportChannel channel, Task task) throws Exception {
+                    FetchSearchResult result = searchService.executeFetchPhase(request, (SearchTask)task);
+                    channel.sendResponse(result);
+                }
+            });
+        TransportActionProxy.registerProxyAction(transportService, FETCH_ID_SCROLL_ACTION_NAME, FetchSearchResult::new);
+
+        transportService.registerRequestHandler(FETCH_ID_ACTION_NAME, ShardFetchSearchRequest::new, ThreadPool.Names.SEARCH,
+            new TaskAwareTransportRequestHandler<ShardFetchSearchRequest>() {
+                @Override
+                public void messageReceived(ShardFetchSearchRequest request, TransportChannel channel, Task task) throws Exception {
+                    FetchSearchResult result = searchService.executeFetchPhase(request, (SearchTask)task);
+                    channel.sendResponse(result);
+                }
+            });
+        TransportActionProxy.registerProxyAction(transportService, FETCH_ID_ACTION_NAME, FetchSearchResult::new);
+
+        // this is super cheap and should not hit thread-pool rejections
+        transportService.registerRequestHandler(QUERY_CAN_MATCH_NAME, ShardSearchTransportRequest::new, ThreadPool.Names.SEARCH,
+            false, true, new TaskAwareTransportRequestHandler<ShardSearchTransportRequest>() {
+                @Override
+                public void messageReceived(ShardSearchTransportRequest request, TransportChannel channel, Task task) throws Exception {
+                    boolean canMatch = searchService.canMatch(request);
+                    channel.sendResponse(new CanMatchResponse(canMatch));
+                }
+            });
+        TransportActionProxy.registerProxyAction(transportService, QUERY_CAN_MATCH_NAME,
+                (Supplier<TransportResponse>) CanMatchResponse::new);
+    }
+
+
+    public static final class CanMatchResponse extends SearchPhaseResult {
+        private boolean canMatch;
+
+        public CanMatchResponse() {
+        }
+
+        public CanMatchResponse(boolean canMatch) {
+            this.canMatch = canMatch;
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            super.readFrom(in);
+            canMatch = in.readBoolean();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeBoolean(canMatch);
+        }
+
+        public boolean canMatch() {
+            return canMatch;
+        }
+    }
+
+
+    /**
+     * Returns a connection to the given node on the provided cluster. If the cluster alias is <code>null</code> the node will be resolved
+     * against the local cluster.
+     * @param clusterAlias the cluster alias the node should be resolve against
+     * @param node the node to resolve
+     * @return a connection to the given node belonging to the cluster with the provided alias.
+     */
+    Transport.Connection getConnection(String clusterAlias, DiscoveryNode node) {
+        if (clusterAlias == null) {
+            return transportService.getConnection(node);
+        } else {
+            return transportService.getRemoteClusterService().getConnection(node, clusterAlias);
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchType.java b/core/src/main/java/org/elasticsearch/action/search/SearchType.java
index 31535736957cd..cbbeef60f6c04 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchType.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchType.java
@@ -19,8 +19,6 @@
 
 package org.elasticsearch.action.search;
 
-import org.elasticsearch.common.ParseFieldMatcher;
-
 /**
  * Search type represent the manner at which the search operation is executed.
  *
@@ -39,16 +37,12 @@ public enum SearchType {
      * are fetched. This is very handy when the index has a lot of shards (not replicas, shard id groups).
      */
     QUERY_THEN_FETCH((byte) 1),
+    // 2 used to be DFS_QUERY_AND_FETCH
+
     /**
-     * Same as {@link #QUERY_AND_FETCH}, except for an initial scatter phase which goes and computes the distributed
-     * term frequencies for more accurate scoring.
-     */
-    DFS_QUERY_AND_FETCH((byte) 2),
-    /**
-     * The most naive (and possibly fastest) implementation is to simply execute the query on all relevant shards
-     * and return the results. Each shard returns size results. Since each shard already returns size hits, this
-     * type actually returns size times number of shards results back to the caller.
+     * Only used for pre 5.3 request where this type is still needed
      */
+    @Deprecated
     QUERY_AND_FETCH((byte) 3);
 
     /**
@@ -75,12 +69,9 @@ public byte id() {
     public static SearchType fromId(byte id) {
         if (id == 0) {
             return DFS_QUERY_THEN_FETCH;
-        } else if (id == 1) {
+        } else if (id == 1
+            || id == 3) { // This is a BWC layer for pre 5.3 indices where QUERY_AND_FETCH was id 3 but we don't have it anymore from 5.3 on
             return QUERY_THEN_FETCH;
-        } else if (id == 2) {
-            return DFS_QUERY_AND_FETCH;
-        } else if (id == 3) {
-            return QUERY_AND_FETCH;
         } else {
             throw new IllegalArgumentException("No search type for [" + id + "]");
         }
@@ -91,18 +82,14 @@ public static SearchType fromId(byte id) {
      * one of "dfs_query_then_fetch"/"dfsQueryThenFetch", "dfs_query_and_fetch"/"dfsQueryAndFetch",
      * "query_then_fetch"/"queryThenFetch" and "query_and_fetch"/"queryAndFetch".
      */
-    public static SearchType fromString(String searchType, ParseFieldMatcher parseFieldMatcher) {
+    public static SearchType fromString(String searchType) {
         if (searchType == null) {
             return SearchType.DEFAULT;
         }
         if ("dfs_query_then_fetch".equals(searchType)) {
             return SearchType.DFS_QUERY_THEN_FETCH;
-        } else if ("dfs_query_and_fetch".equals(searchType)) {
-            return SearchType.DFS_QUERY_AND_FETCH;
         } else if ("query_then_fetch".equals(searchType)) {
             return SearchType.QUERY_THEN_FETCH;
-        } else if ("query_and_fetch".equals(searchType)) {
-            return SearchType.QUERY_AND_FETCH;
         } else {
             throw new IllegalArgumentException("No search type for [" + searchType + "]");
         }
diff --git a/core/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java b/core/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java
index 8070081dcd865..7eb939ca8274e 100644
--- a/core/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java
+++ b/core/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java
@@ -21,22 +21,34 @@
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.action.OriginalIndices;
 import org.elasticsearch.action.ShardOperationFailedException;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.SearchException;
 import org.elasticsearch.search.SearchShardTarget;
 
 import java.io.IOException;
 
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+
 /**
  * Represents a failure to search on a specific shard.
  */
 public class ShardSearchFailure implements ShardOperationFailedException {
 
+    private static final String REASON_FIELD = "reason";
+    private static final String NODE_FIELD = "node";
+    private static final String INDEX_FIELD = "index";
+    private static final String SHARD_FIELD = "shard";
+
     public static final ShardSearchFailure[] EMPTY_ARRAY = new ShardSearchFailure[0];
 
     private SearchShardTarget shardTarget;
@@ -68,7 +80,7 @@ public ShardSearchFailure(String reason, SearchShardTarget shardTarget) {
         this(reason, shardTarget, RestStatus.INTERNAL_SERVER_ERROR);
     }
 
-    public ShardSearchFailure(String reason, SearchShardTarget shardTarget, RestStatus status) {
+    private ShardSearchFailure(String reason, SearchShardTarget shardTarget, RestStatus status) {
         this.shardTarget = shardTarget;
         this.reason = reason;
         this.status = status;
@@ -93,7 +105,7 @@ public RestStatus status() {
     @Override
     public String index() {
         if (shardTarget != null) {
-            return shardTarget.index();
+            return shardTarget.getIndex();
         }
         return null;
     }
@@ -104,7 +116,7 @@ public String index() {
     @Override
     public int shardId() {
         if (shardTarget != null) {
-            return shardTarget.shardId().id();
+            return shardTarget.getShardId().id();
         }
         return -1;
     }
@@ -153,20 +165,56 @@ public void writeTo(StreamOutput out) throws IOException {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.field("shard", shardId());
-        builder.field("index", index());
+        builder.field(SHARD_FIELD, shardId());
+        builder.field(INDEX_FIELD, index());
         if (shardTarget != null) {
-            builder.field("node", shardTarget.nodeId());
+            builder.field(NODE_FIELD, shardTarget.getNodeId());
         }
         if (cause != null) {
-            builder.field("reason");
+            builder.field(REASON_FIELD);
             builder.startObject();
-            ElasticsearchException.toXContent(builder, params, cause);
+            ElasticsearchException.generateThrowableXContent(builder, params, cause);
             builder.endObject();
         }
         return builder;
     }
 
+    public static ShardSearchFailure fromXContent(XContentParser parser) throws IOException {
+        XContentParser.Token token;
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation);
+        String currentFieldName = null;
+        int shardId = -1;
+        String indexName = null;
+        String nodeId = null;
+        ElasticsearchException exception = null;
+        while((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token.isValue()) {
+                if (SHARD_FIELD.equals(currentFieldName)) {
+                    shardId  = parser.intValue();
+                } else if (INDEX_FIELD.equals(currentFieldName)) {
+                    indexName  = parser.text();
+                } else if (NODE_FIELD.equals(currentFieldName)) {
+                    nodeId  = parser.text();
+                } else {
+                    parser.skipChildren();
+                }
+            } else if (token == XContentParser.Token.START_OBJECT) {
+                if (REASON_FIELD.equals(currentFieldName)) {
+                    exception = ElasticsearchException.fromXContent(parser);
+                } else {
+                    parser.skipChildren();
+                }
+            } else {
+                parser.skipChildren();
+            }
+        }
+        return new ShardSearchFailure(exception,
+                new SearchShardTarget(nodeId,
+                        new ShardId(new Index(indexName, IndexMetaData.INDEX_UUID_NA_VALUE), shardId), null, OriginalIndices.NONE));
+    }
+
     @Override
     public Throwable getCause() {
         return cause;
diff --git a/core/src/main/java/org/elasticsearch/action/search/TransportClearScrollAction.java b/core/src/main/java/org/elasticsearch/action/search/TransportClearScrollAction.java
index 220e7f5b25054..d9afbdacafe3c 100644
--- a/core/src/main/java/org/elasticsearch/action/search/TransportClearScrollAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/TransportClearScrollAction.java
@@ -19,33 +19,16 @@
 
 package org.elasticsearch.action.search;
 
-import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.apache.logging.log4j.util.Supplier;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.CountDown;
-import org.elasticsearch.search.action.SearchTransportService;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.TransportResponse;
 import org.elasticsearch.transport.TransportService;
 
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
-
-import static org.elasticsearch.action.search.TransportSearchHelper.parseScrollId;
-
-/**
- */
 public class TransportClearScrollAction extends HandledTransportAction<ClearScrollRequest, ClearScrollResponse> {
 
     private final ClusterService clusterService;
@@ -53,107 +36,19 @@ public class TransportClearScrollAction extends HandledTransportAction<ClearScro
 
     @Inject
     public TransportClearScrollAction(Settings settings, TransportService transportService, ThreadPool threadPool,
-                                      ClusterService clusterService, SearchTransportService searchTransportService,
-                                      ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
-        super(settings, ClearScrollAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, ClearScrollRequest::new);
+                                      ClusterService clusterService, ActionFilters actionFilters,
+                                      IndexNameExpressionResolver indexNameExpressionResolver,
+                                      SearchTransportService searchTransportService) {
+        super(settings, ClearScrollAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver,
+            ClearScrollRequest::new);
         this.clusterService = clusterService;
         this.searchTransportService = searchTransportService;
     }
 
     @Override
     protected void doExecute(ClearScrollRequest request, final ActionListener<ClearScrollResponse> listener) {
-        new Async(request, listener, clusterService.state()).run();
-    }
-
-    private class Async {
-        final DiscoveryNodes nodes;
-        final CountDown expectedOps;
-        final List<ScrollIdForNode[]> contexts = new ArrayList<>();
-        final ActionListener<ClearScrollResponse> listener;
-        final AtomicReference<Throwable> expHolder;
-        final AtomicInteger numberOfFreedSearchContexts = new AtomicInteger(0);
-
-        private Async(ClearScrollRequest request, ActionListener<ClearScrollResponse> listener, ClusterState clusterState) {
-            int expectedOps = 0;
-            this.nodes = clusterState.nodes();
-            if (request.getScrollIds().size() == 1 && "_all".equals(request.getScrollIds().get(0))) {
-                expectedOps = nodes.getSize();
-            } else {
-                for (String parsedScrollId : request.getScrollIds()) {
-                    ScrollIdForNode[] context = parseScrollId(parsedScrollId).getContext();
-                    expectedOps += context.length;
-                    this.contexts.add(context);
-                }
-            }
-            this.listener = listener;
-            this.expHolder = new AtomicReference<>();
-            this.expectedOps = new CountDown(expectedOps);
-        }
-
-        public void run() {
-            if (expectedOps.isCountedDown()) {
-                listener.onResponse(new ClearScrollResponse(true, 0));
-                return;
-            }
-
-            if (contexts.isEmpty()) {
-                for (final DiscoveryNode node : nodes) {
-                    searchTransportService.sendClearAllScrollContexts(node, new ActionListener<TransportResponse>() {
-                        @Override
-                        public void onResponse(TransportResponse response) {
-                            onFreedContext(true);
-                        }
-
-                        @Override
-                        public void onFailure(Exception e) {
-                            onFailedFreedContext(e, node);
-                        }
-                    });
-                }
-            } else {
-                for (ScrollIdForNode[] context : contexts) {
-                    for (ScrollIdForNode target : context) {
-                        final DiscoveryNode node = nodes.get(target.getNode());
-                        if (node == null) {
-                            onFreedContext(false);
-                            continue;
-                        }
-
-                        searchTransportService.sendFreeContext(node, target.getScrollId(), new ActionListener<SearchTransportService.SearchFreeContextResponse>() {
-                            @Override
-                            public void onResponse(SearchTransportService.SearchFreeContextResponse freed) {
-                                onFreedContext(freed.isFreed());
-                            }
-
-                            @Override
-                            public void onFailure(Exception e) {
-                                onFailedFreedContext(e, node);
-                            }
-                        });
-                    }
-                }
-            }
-        }
-
-        void onFreedContext(boolean freed) {
-            if (freed) {
-                numberOfFreedSearchContexts.incrementAndGet();
-            }
-            if (expectedOps.countDown()) {
-                boolean succeeded = expHolder.get() == null;
-                listener.onResponse(new ClearScrollResponse(succeeded, numberOfFreedSearchContexts.get()));
-            }
-        }
-
-        void onFailedFreedContext(Throwable e, DiscoveryNode node) {
-            logger.warn((Supplier<?>) () -> new ParameterizedMessage("Clear SC failed on node[{}]", node), e);
-            if (expectedOps.countDown()) {
-                listener.onResponse(new ClearScrollResponse(false, numberOfFreedSearchContexts.get()));
-            } else {
-                expHolder.set(e);
-            }
-        }
-
+        Runnable runnable = new ClearScrollController(request, listener, clusterService.state().nodes(), logger, searchTransportService);
+        runnable.run();
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/action/search/TransportMultiSearchAction.java b/core/src/main/java/org/elasticsearch/action/search/TransportMultiSearchAction.java
index efd0403527657..cc2638070a94f 100644
--- a/core/src/main/java/org/elasticsearch/action/search/TransportMultiSearchAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/TransportMultiSearchAction.java
@@ -34,6 +34,7 @@
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.util.List;
 import java.util.Queue;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -46,19 +47,18 @@ public class TransportMultiSearchAction extends HandledTransportAction<MultiSear
 
     @Inject
     public TransportMultiSearchAction(Settings settings, ThreadPool threadPool, TransportService transportService,
-                                ClusterService clusterService, TransportSearchAction searchAction,
-                                ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
-        super(settings, MultiSearchAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, MultiSearchRequest::new);
+                                      ClusterService clusterService, TransportSearchAction searchAction,
+                                      ActionFilters actionFilters, IndexNameExpressionResolver resolver) {
+        super(settings, MultiSearchAction.NAME, threadPool, transportService, actionFilters, resolver, MultiSearchRequest::new);
         this.clusterService = clusterService;
         this.searchAction = searchAction;
         this.availableProcessors = EsExecutors.boundedNumberOfProcessors(settings);
     }
 
-    // For testing only:
     TransportMultiSearchAction(ThreadPool threadPool, ActionFilters actionFilters, TransportService transportService,
                                ClusterService clusterService, TransportAction<SearchRequest, SearchResponse> searchAction,
-                               IndexNameExpressionResolver indexNameExpressionResolver, int availableProcessors) {
-        super(Settings.EMPTY, MultiSearchAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, MultiSearchRequest::new);
+                               IndexNameExpressionResolver resolver, int availableProcessors) {
+        super(Settings.EMPTY, MultiSearchAction.NAME, threadPool, transportService, actionFilters, resolver, MultiSearchRequest::new);
         this.clusterService = clusterService;
         this.searchAction = searchAction;
         this.availableProcessors = availableProcessors;
@@ -90,10 +90,9 @@ protected void doExecute(MultiSearchRequest request, ActionListener<MultiSearchR
     }
 
     /*
-     * This is not perfect and makes a big assumption, that all nodes have the same thread pool size / have the number
-     * of processors and that shard of the indices the search requests go to are more or less evenly distributed across
-     * all nodes in the cluster. But I think it is a good enough default for most cases, if not then the default should be
-     * overwritten in the request itself.
+     * This is not perfect and makes a big assumption, that all nodes have the same thread pool size / have the number of processors and
+     * that shard of the indices the search requests go to are more or less evenly distributed across all nodes in the cluster. But I think
+     * it is a good enough default for most cases, if not then the default should be overwritten in the request itself.
      */
     static int defaultMaxConcurrentSearches(int availableProcessors, ClusterState state) {
         int numDateNodes = state.getNodes().getDataNodes().size();
@@ -103,33 +102,70 @@ static int defaultMaxConcurrentSearches(int availableProcessors, ClusterState st
         return Math.max(1, numDateNodes * defaultSearchThreadPoolSize);
     }
 
-    void executeSearch(Queue<SearchRequestSlot> requests, AtomicArray<MultiSearchResponse.Item> responses,
-                       AtomicInteger responseCounter, ActionListener<MultiSearchResponse> listener) {
+    /**
+     * Executes a single request from the queue of requests. When a request finishes, another request is taken from the queue. When a
+     * request is executed, a permit is taken on the specified semaphore, and released as each request completes.
+     *
+     * @param requests the queue of multi-search requests to execute
+     * @param responses atomic array to hold the responses corresponding to each search request slot
+     * @param responseCounter incremented on each response
+     * @param listener the listener attached to the multi-search request
+     */
+    private void executeSearch(
+            final Queue<SearchRequestSlot> requests,
+            final AtomicArray<MultiSearchResponse.Item> responses,
+            final AtomicInteger responseCounter,
+            final ActionListener<MultiSearchResponse> listener) {
         SearchRequestSlot request = requests.poll();
         if (request == null) {
-            // Ok... so there're no more requests then this is ok, we're then waiting for running requests to complete
+            /*
+             * The number of times that we poll an item from the queue here is the minimum of the number of requests and the maximum number
+             * of concurrent requests. At first glance, it appears that we should never poll from the queue and not obtain a request given
+             * that we only poll here no more times than the number of requests. However, this is not the only consumer of this queue as
+             * earlier requests that have already completed will poll from the queue too and they could complete before later polls are
+             * invoked here. Thus, it can be the case that we poll here and and the queue was empty.
+             */
             return;
         }
+
+        /*
+         * With a request in hand, we are now prepared to execute the search request. There are two possibilities, either we go asynchronous
+         * or we do not (this can happen if the request does not resolve to any shards). If we do not go asynchronous, we are going to come
+         * back on the same thread that attempted to execute the search request. At this point, or any other point where we come back on the
+         * same thread as when the request was submitted, we should not recurse lest we might descend into a stack overflow. To avoid this,
+         * when we handle the response rather than going recursive, we fork to another thread, otherwise we recurse.
+         */
+        final Thread thread = Thread.currentThread();
         searchAction.execute(request.request, new ActionListener<SearchResponse>() {
             @Override
-            public void onResponse(SearchResponse searchResponse) {
-                responses.set(request.responseSlot, new MultiSearchResponse.Item(searchResponse, null));
-                handleResponse();
+            public void onResponse(final SearchResponse searchResponse) {
+                handleResponse(request.responseSlot, new MultiSearchResponse.Item(searchResponse, null));
             }
 
             @Override
-            public void onFailure(Exception e) {
-                responses.set(request.responseSlot, new MultiSearchResponse.Item(null, e));
-                handleResponse();
+            public void onFailure(final Exception e) {
+                handleResponse(request.responseSlot, new MultiSearchResponse.Item(null, e));
             }
 
-            private void handleResponse() {
+            private void handleResponse(final int responseSlot, final MultiSearchResponse.Item item) {
+                responses.set(responseSlot, item);
                 if (responseCounter.decrementAndGet() == 0) {
-                    listener.onResponse(new MultiSearchResponse(responses.toArray(new MultiSearchResponse.Item[responses.length()])));
+                    assert requests.isEmpty();
+                    finish();
                 } else {
-                    executeSearch(requests, responses, responseCounter, listener);
+                    if (thread == Thread.currentThread()) {
+                        // we are on the same thread, we need to fork to another thread to avoid recursive stack overflow on a single thread
+                        threadPool.generic().execute(() -> executeSearch(requests, responses, responseCounter, listener));
+                    } else {
+                        // we are on a different thread (we went asynchronous), it's safe to recurse
+                        executeSearch(requests, responses, responseCounter, listener);
+                    }
                 }
             }
+
+            private void finish() {
+                listener.onResponse(new MultiSearchResponse(responses.toArray(new MultiSearchResponse.Item[responses.length()])));
+            }
         });
     }
 
@@ -142,5 +178,7 @@ static final class SearchRequestSlot {
             this.request = request;
             this.responseSlot = responseSlot;
         }
+
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/core/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
index 8a33bff8f0e9a..21a1097f8fbbd 100644
--- a/core/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
@@ -20,105 +20,380 @@
 package org.elasticsearch.action.search;
 
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.OriginalIndices;
+import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsGroup;
+import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsResponse;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.GroupShardsIterator;
+import org.elasticsearch.cluster.routing.ShardIterator;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.IndexNotFoundException;
-import org.elasticsearch.indices.IndexClosedException;
-import org.elasticsearch.search.action.SearchTransportService;
-import org.elasticsearch.search.controller.SearchPhaseController;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.search.SearchService;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.internal.AliasFilter;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.RemoteClusterAware;
+import org.elasticsearch.transport.RemoteClusterService;
+import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportService;
 
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.Executor;
+import java.util.function.BiFunction;
+import java.util.function.LongSupplier;
 
-import static org.elasticsearch.action.search.SearchType.QUERY_AND_FETCH;
 import static org.elasticsearch.action.search.SearchType.QUERY_THEN_FETCH;
 
 public class TransportSearchAction extends HandledTransportAction<SearchRequest, SearchResponse> {
 
     /** The maximum number of shards for a single search request. */
     public static final Setting<Long> SHARD_COUNT_LIMIT_SETTING = Setting.longSetting(
-            "action.search.shard_count.limit", 1000L, 1L, Property.Dynamic, Property.NodeScope);
+            "action.search.shard_count.limit", Long.MAX_VALUE, 1L, Property.Dynamic, Property.NodeScope);
 
     private final ClusterService clusterService;
     private final SearchTransportService searchTransportService;
+    private final RemoteClusterService remoteClusterService;
     private final SearchPhaseController searchPhaseController;
+    private final SearchService searchService;
 
     @Inject
-    public TransportSearchAction(Settings settings, ThreadPool threadPool, SearchPhaseController searchPhaseController,
-                                 TransportService transportService, SearchTransportService searchTransportService,
-                                 ClusterService clusterService, ActionFilters actionFilters, IndexNameExpressionResolver
-                                             indexNameExpressionResolver) {
+    public TransportSearchAction(Settings settings, ThreadPool threadPool, TransportService transportService, SearchService searchService,
+                                 SearchTransportService searchTransportService, SearchPhaseController searchPhaseController,
+                                 ClusterService clusterService, ActionFilters actionFilters,
+                                 IndexNameExpressionResolver indexNameExpressionResolver) {
         super(settings, SearchAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, SearchRequest::new);
         this.searchPhaseController = searchPhaseController;
         this.searchTransportService = searchTransportService;
+        this.remoteClusterService = searchTransportService.getRemoteClusterService();
+        SearchTransportService.registerRequestHandler(transportService, searchService);
         this.clusterService = clusterService;
+        this.searchService = searchService;
+    }
+
+    private Map<String, AliasFilter> buildPerIndexAliasFilter(SearchRequest request, ClusterState clusterState,
+                                                              Index[] concreteIndices, Map<String, AliasFilter> remoteAliasMap) {
+        final Map<String, AliasFilter> aliasFilterMap = new HashMap<>();
+        for (Index index : concreteIndices) {
+            clusterState.blocks().indexBlockedRaiseException(ClusterBlockLevel.READ, index.getName());
+            AliasFilter aliasFilter = searchService.buildAliasFilter(clusterState, index.getName(), request.indices());
+            assert aliasFilter != null;
+            aliasFilterMap.put(index.getUUID(), aliasFilter);
+        }
+        aliasFilterMap.putAll(remoteAliasMap);
+        return aliasFilterMap;
+    }
+
+    private Map<String, Float> resolveIndexBoosts(SearchRequest searchRequest, ClusterState clusterState) {
+        if (searchRequest.source() == null) {
+            return Collections.emptyMap();
+        }
+
+        SearchSourceBuilder source = searchRequest.source();
+        if (source.indexBoosts() == null) {
+            return Collections.emptyMap();
+        }
+
+        Map<String, Float> concreteIndexBoosts = new HashMap<>();
+        for (SearchSourceBuilder.IndexBoost ib : source.indexBoosts()) {
+            Index[] concreteIndices =
+                indexNameExpressionResolver.concreteIndices(clusterState, searchRequest.indicesOptions(), ib.getIndex());
+
+            for (Index concreteIndex : concreteIndices) {
+                concreteIndexBoosts.putIfAbsent(concreteIndex.getUUID(), ib.getBoost());
+            }
+        }
+        return Collections.unmodifiableMap(concreteIndexBoosts);
+    }
+
+    /**
+     * Search operations need two clocks. One clock is to fulfill real clock needs (e.g., resolving
+     * "now" to an index name). Another clock is needed for measuring how long a search operation
+     * took. These two uses are at odds with each other. There are many issues with using a real
+     * clock for measuring how long an operation took (they often lack precision, they are subject
+     * to moving backwards due to NTP and other such complexities, etc.). There are also issues with
+     * using a relative clock for reporting real time. Thus, we simply separate these two uses.
+     */
+    static class SearchTimeProvider {
+
+        private final long absoluteStartMillis;
+        private final long relativeStartNanos;
+        private final LongSupplier relativeCurrentNanosProvider;
+
+        /**
+         * Instantiates a new search time provider. The absolute start time is the real clock time
+         * used for resolving index expressions that include dates. The relative start time is the
+         * start of the search operation according to a relative clock. The total time the search
+         * operation took can be measured against the provided relative clock and the relative start
+         * time.
+         *
+         * @param absoluteStartMillis the absolute start time in milliseconds since the epoch
+         * @param relativeStartNanos the relative start time in nanoseconds
+         * @param relativeCurrentNanosProvider provides the current relative time
+         */
+        SearchTimeProvider(
+                final long absoluteStartMillis,
+                final long relativeStartNanos,
+                final LongSupplier relativeCurrentNanosProvider) {
+            this.absoluteStartMillis = absoluteStartMillis;
+            this.relativeStartNanos = relativeStartNanos;
+            this.relativeCurrentNanosProvider = relativeCurrentNanosProvider;
+        }
+
+        long getAbsoluteStartMillis() {
+            return absoluteStartMillis;
+        }
+
+        long getRelativeStartNanos() {
+            return relativeStartNanos;
+        }
+
+        long getRelativeCurrentNanos() {
+            return relativeCurrentNanosProvider.getAsLong();
+        }
     }
 
     @Override
-    protected void doExecute(SearchRequest searchRequest, ActionListener<SearchResponse> listener) {
-        // optimize search type for cases where there is only one shard group to search on
-        try {
-            ClusterState clusterState = clusterService.state();
-            String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(clusterState, searchRequest);
-            Map<String, Set<String>> routingMap = indexNameExpressionResolver.resolveSearchRouting(clusterState,
-                    searchRequest.routing(), searchRequest.indices());
-            int shardCount = clusterService.operationRouting().searchShardsCount(clusterState, concreteIndices, routingMap);
-            if (shardCount == 1) {
-                // if we only have one group, then we always want Q_A_F, no need for DFS, and no need to do THEN since we hit one shard
-                searchRequest.searchType(QUERY_AND_FETCH);
+    protected void doExecute(Task task, SearchRequest searchRequest, ActionListener<SearchResponse> listener) {
+        final long absoluteStartMillis = System.currentTimeMillis();
+        final long relativeStartNanos = System.nanoTime();
+        final SearchTimeProvider timeProvider =
+                new SearchTimeProvider(absoluteStartMillis, relativeStartNanos, System::nanoTime);
+
+
+        final ClusterState clusterState = clusterService.state();
+        final Map<String, OriginalIndices> remoteClusterIndices = remoteClusterService.groupIndices(searchRequest.indicesOptions(),
+            searchRequest.indices(), idx -> indexNameExpressionResolver.hasIndexOrAlias(idx, clusterState));
+        OriginalIndices localIndices = remoteClusterIndices.remove(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY);
+        if (remoteClusterIndices.isEmpty()) {
+            executeSearch((SearchTask)task, timeProvider, searchRequest, localIndices, remoteClusterIndices, Collections.emptyList(),
+                (clusterName, nodeId) -> null, clusterState, Collections.emptyMap(), listener, clusterState.getNodes()
+                    .getDataNodes().size());
+        } else {
+            remoteClusterService.collectSearchShards(searchRequest.indicesOptions(), searchRequest.preference(), searchRequest.routing(),
+                remoteClusterIndices, ActionListener.wrap((searchShardsResponses) -> {
+                    List<SearchShardIterator> remoteShardIterators = new ArrayList<>();
+                    Map<String, AliasFilter> remoteAliasFilters = new HashMap<>();
+                    BiFunction<String, String, DiscoveryNode> clusterNodeLookup = processRemoteShards(searchShardsResponses,
+                        remoteClusterIndices, remoteShardIterators, remoteAliasFilters);
+                    int numNodesInvovled = searchShardsResponses.values().stream().mapToInt(r -> r.getNodes().length).sum()
+                        + clusterState.getNodes().getDataNodes().size();
+                    executeSearch((SearchTask) task, timeProvider, searchRequest, localIndices, remoteClusterIndices, remoteShardIterators,
+                        clusterNodeLookup, clusterState, remoteAliasFilters, listener, numNodesInvovled);
+                }, listener::onFailure));
+        }
+    }
+
+    static BiFunction<String, String, DiscoveryNode> processRemoteShards(Map<String, ClusterSearchShardsResponse> searchShardsResponses,
+                                                                      Map<String, OriginalIndices> remoteIndicesByCluster,
+                                                                      List<SearchShardIterator> remoteShardIterators,
+                                                                      Map<String, AliasFilter> aliasFilterMap) {
+        Map<String, Map<String, DiscoveryNode>> clusterToNode = new HashMap<>();
+        for (Map.Entry<String, ClusterSearchShardsResponse> entry : searchShardsResponses.entrySet()) {
+            String clusterAlias = entry.getKey();
+            ClusterSearchShardsResponse searchShardsResponse = entry.getValue();
+            HashMap<String, DiscoveryNode> idToDiscoveryNode = new HashMap<>();
+            clusterToNode.put(clusterAlias, idToDiscoveryNode);
+            for (DiscoveryNode remoteNode : searchShardsResponse.getNodes()) {
+                idToDiscoveryNode.put(remoteNode.getId(), remoteNode);
             }
-            if (searchRequest.isSuggestOnly()) {
-                // disable request cache if we have only suggest
-                searchRequest.requestCache(false);
-                switch (searchRequest.searchType()) {
-                    case DFS_QUERY_AND_FETCH:
-                    case DFS_QUERY_THEN_FETCH:
-                        // convert to Q_T_F if we have only suggest
-                        searchRequest.searchType(QUERY_THEN_FETCH);
-                        break;
+            final Map<String, AliasFilter> indicesAndFilters = searchShardsResponse.getIndicesAndFilters();
+            for (ClusterSearchShardsGroup clusterSearchShardsGroup : searchShardsResponse.getGroups()) {
+                //add the cluster name to the remote index names for indices disambiguation
+                //this ends up in the hits returned with the search response
+                ShardId shardId = clusterSearchShardsGroup.getShardId();
+                final AliasFilter aliasFilter;
+                if (indicesAndFilters == null) {
+                    aliasFilter = AliasFilter.EMPTY;
+                } else {
+                    aliasFilter = indicesAndFilters.get(shardId.getIndexName());
+                    assert aliasFilter != null : "alias filter must not be null for index: " + shardId.getIndex();
                 }
+                String[] aliases = aliasFilter.getAliases();
+                String[] finalIndices = aliases.length == 0 ? new String[] {shardId.getIndexName()} : aliases;
+                // here we have to map the filters to the UUID since from now on we use the uuid for the lookup
+                aliasFilterMap.put(shardId.getIndex().getUUID(), aliasFilter);
+                final OriginalIndices originalIndices = remoteIndicesByCluster.get(clusterAlias);
+                assert originalIndices != null : "original indices are null for clusterAlias: " + clusterAlias;
+                SearchShardIterator shardIterator = new SearchShardIterator(clusterAlias, shardId,
+                    Arrays.asList(clusterSearchShardsGroup.getShards()), new OriginalIndices(finalIndices,
+                    originalIndices.indicesOptions()));
+                remoteShardIterators.add(shardIterator);
             }
-        } catch (IndexNotFoundException | IndexClosedException e) {
-            // ignore these failures, we will notify the search response if its really the case from the actual action
-        } catch (Exception e) {
-            logger.debug("failed to optimize search type, continue as normal", e);
         }
+        return (clusterAlias, nodeId) -> {
+                Map<String, DiscoveryNode> clusterNodes = clusterToNode.get(clusterAlias);
+                if (clusterNodes == null) {
+                    throw new IllegalArgumentException("unknown remote cluster: " + clusterAlias);
+                }
+            return clusterNodes.get(nodeId);
+        };
+    }
+
+    private void executeSearch(SearchTask task, SearchTimeProvider timeProvider, SearchRequest searchRequest, OriginalIndices localIndices,
+                               Map<String, OriginalIndices> remoteClusterIndices, List<SearchShardIterator> remoteShardIterators,
+                               BiFunction<String, String, DiscoveryNode> remoteConnections, ClusterState clusterState,
+                               Map<String, AliasFilter> remoteAliasMap, ActionListener<SearchResponse> listener, int nodeCount) {
+
+        clusterState.blocks().globalBlockedRaiseException(ClusterBlockLevel.READ);
+        // TODO: I think startTime() should become part of ActionRequest and that should be used both for index name
+        // date math expressions and $now in scripts. This way all apis will deal with now in the same way instead
+        // of just for the _search api
+        final Index[] indices;
+        if (localIndices.indices().length == 0 && remoteClusterIndices.isEmpty() == false) {
+            indices = Index.EMPTY_ARRAY; // don't search on _all if only remote indices were specified
+        } else {
+            indices = indexNameExpressionResolver.concreteIndices(clusterState, searchRequest.indicesOptions(),
+                timeProvider.getAbsoluteStartMillis(), localIndices.indices());
+        }
+        Map<String, AliasFilter> aliasFilter = buildPerIndexAliasFilter(searchRequest, clusterState, indices, remoteAliasMap);
+        Map<String, Set<String>> routingMap = indexNameExpressionResolver.resolveSearchRouting(clusterState, searchRequest.routing(),
+            searchRequest.indices());
+        String[] concreteIndices = new String[indices.length];
+        for (int i = 0; i < indices.length; i++) {
+            concreteIndices[i] = indices[i].getName();
+        }
+        GroupShardsIterator<ShardIterator> localShardsIterator = clusterService.operationRouting().searchShards(clusterState,
+            concreteIndices, routingMap, searchRequest.preference());
+        GroupShardsIterator<SearchShardIterator> shardIterators = mergeShardsIterators(localShardsIterator, localIndices,
+            remoteShardIterators);
+
+        failIfOverShardCountLimit(clusterService, shardIterators.size());
+
+        Map<String, Float> concreteIndexBoosts = resolveIndexBoosts(searchRequest, clusterState);
+
+        // optimize search type for cases where there is only one shard group to search on
+        if (shardIterators.size() == 1) {
+            // if we only have one group, then we always want Q_A_F, no need for DFS, and no need to do THEN since we hit one shard
+            searchRequest.searchType(QUERY_THEN_FETCH);
+        }
+        if (searchRequest.isSuggestOnly()) {
+            // disable request cache if we have only suggest
+            searchRequest.requestCache(false);
+            switch (searchRequest.searchType()) {
+                case DFS_QUERY_THEN_FETCH:
+                    // convert to Q_T_F if we have only suggest
+                    searchRequest.searchType(QUERY_THEN_FETCH);
+                    break;
+            }
+        }
+
+        final DiscoveryNodes nodes = clusterState.nodes();
+        BiFunction<String, String, Transport.Connection> connectionLookup = (clusterName, nodeId) -> {
+            final DiscoveryNode discoveryNode = clusterName == null ? nodes.get(nodeId) : remoteConnections.apply(clusterName, nodeId);
+            if (discoveryNode == null) {
+                throw new IllegalStateException("no node found for id: " + nodeId);
+            }
+            return searchTransportService.getConnection(clusterName, discoveryNode);
+        };
+        if (searchRequest.isMaxConcurrentShardRequestsSet() == false) {
+            // we try to set a default of max concurrent shard requests based on
+            // the node count but upper-bound it by 256 by default to keep it sane. A single
+            // search request that fans out lots of shards should hit a cluster too hard while 256 is already a lot
+            // we multiply is by the default number of shards such that a single request in a cluster of 1 would hit all shards of a
+            // default index.
+            searchRequest.setMaxConcurrentShardRequests(Math.min(256, nodeCount
+                * IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getDefault(Settings.EMPTY)));
+        }
+        boolean preFilterSearchShards = shouldPreFilterSearchShards(searchRequest, shardIterators);
+        searchAsyncAction(task, searchRequest, shardIterators, timeProvider, connectionLookup, clusterState.version(),
+            Collections.unmodifiableMap(aliasFilter), concreteIndexBoosts, listener, preFilterSearchShards).start();
+    }
+
+    private boolean shouldPreFilterSearchShards(SearchRequest searchRequest, GroupShardsIterator<SearchShardIterator> shardIterators) {
+        SearchSourceBuilder source = searchRequest.source();
+        return searchRequest.searchType() == QUERY_THEN_FETCH && // we can't do this for DFS it needs to fan out to all shards all the time
+                SearchService.canRewriteToMatchNone(source) &&
+                searchRequest.getPreFilterShardSize() < shardIterators.size();
+    }
+
+    static GroupShardsIterator<SearchShardIterator> mergeShardsIterators(GroupShardsIterator<ShardIterator> localShardsIterator,
+                                                             OriginalIndices localIndices,
+                                                             List<SearchShardIterator> remoteShardIterators) {
+        List<SearchShardIterator> shards = new ArrayList<>();
+        for (SearchShardIterator shardIterator : remoteShardIterators) {
+            shards.add(shardIterator);
+        }
+        for (ShardIterator shardIterator : localShardsIterator) {
+            shards.add(new SearchShardIterator(null, shardIterator.shardId(), shardIterator.getShardRoutings(), localIndices));
+        }
+        return new GroupShardsIterator<>(shards);
+    }
 
-        searchAsyncAction(searchRequest, listener).start();
+    @Override
+    protected final void doExecute(SearchRequest searchRequest, ActionListener<SearchResponse> listener) {
+        throw new UnsupportedOperationException("the task parameter is required");
     }
 
-    private AbstractSearchAsyncAction searchAsyncAction(SearchRequest searchRequest, ActionListener<SearchResponse> listener) {
-        AbstractSearchAsyncAction searchAsyncAction;
-        switch(searchRequest.searchType()) {
-            case DFS_QUERY_THEN_FETCH:
-                searchAsyncAction = new SearchDfsQueryThenFetchAsyncAction(logger, searchTransportService, clusterService,
-                        indexNameExpressionResolver, searchPhaseController, threadPool, searchRequest, listener);
-                break;
-            case QUERY_THEN_FETCH:
-                searchAsyncAction = new SearchQueryThenFetchAsyncAction(logger, searchTransportService, clusterService,
-                        indexNameExpressionResolver, searchPhaseController, threadPool, searchRequest, listener);
-                break;
-            case DFS_QUERY_AND_FETCH:
-                searchAsyncAction = new SearchDfsQueryAndFetchAsyncAction(logger, searchTransportService, clusterService,
-                        indexNameExpressionResolver, searchPhaseController, threadPool, searchRequest, listener);
-                break;
-            case QUERY_AND_FETCH:
-                searchAsyncAction = new SearchQueryAndFetchAsyncAction(logger, searchTransportService, clusterService,
-                        indexNameExpressionResolver, searchPhaseController, threadPool, searchRequest, listener);
-                break;
-            default:
-                throw new IllegalStateException("Unknown search type: [" + searchRequest.searchType() + "]");
-        }
-        return searchAsyncAction;
+    private AbstractSearchAsyncAction searchAsyncAction(SearchTask task, SearchRequest searchRequest,
+                                                        GroupShardsIterator<SearchShardIterator> shardIterators,
+                                                        SearchTimeProvider timeProvider,
+                                                        BiFunction<String, String, Transport.Connection> connectionLookup,
+                                                        long clusterStateVersion, Map<String, AliasFilter> aliasFilter,
+                                                        Map<String, Float> concreteIndexBoosts,
+                                                        ActionListener<SearchResponse> listener, boolean preFilter) {
+        Executor executor = threadPool.executor(ThreadPool.Names.SEARCH);
+        if (preFilter) {
+            return new CanMatchPreFilterSearchPhase(logger, searchTransportService, connectionLookup,
+                aliasFilter, concreteIndexBoosts, executor, searchRequest, listener, shardIterators,
+                timeProvider, clusterStateVersion, task, (iter) -> {
+                AbstractSearchAsyncAction action = searchAsyncAction(task, searchRequest, iter, timeProvider, connectionLookup,
+                    clusterStateVersion, aliasFilter, concreteIndexBoosts, listener, false);
+                return new SearchPhase(action.getName()) {
+                    @Override
+                    public void run() throws IOException {
+                        action.start();
+                    }
+                };
+            });
+        } else {
+            AbstractSearchAsyncAction searchAsyncAction;
+            switch (searchRequest.searchType()) {
+                case DFS_QUERY_THEN_FETCH:
+                    searchAsyncAction = new SearchDfsQueryThenFetchAsyncAction(logger, searchTransportService, connectionLookup,
+                        aliasFilter, concreteIndexBoosts, searchPhaseController, executor, searchRequest, listener, shardIterators,
+                        timeProvider, clusterStateVersion, task);
+                    break;
+                case QUERY_AND_FETCH:
+                case QUERY_THEN_FETCH:
+                    searchAsyncAction = new SearchQueryThenFetchAsyncAction(logger, searchTransportService, connectionLookup,
+                        aliasFilter, concreteIndexBoosts, searchPhaseController, executor, searchRequest, listener, shardIterators,
+                        timeProvider, clusterStateVersion, task);
+                    break;
+                default:
+                    throw new IllegalStateException("Unknown search type: [" + searchRequest.searchType() + "]");
+            }
+            return searchAsyncAction;
+        }
     }
 
+    private static void failIfOverShardCountLimit(ClusterService clusterService, int shardCount) {
+        final long shardCountLimit = clusterService.getClusterSettings().get(SHARD_COUNT_LIMIT_SETTING);
+        if (shardCount > shardCountLimit) {
+            throw new IllegalArgumentException("Trying to query " + shardCount + " shards, which is over the limit of "
+                + shardCountLimit + ". This limit exists because querying many shards at the same time can make the "
+                + "job of the coordinating node very CPU and/or memory intensive. It is usually a better idea to "
+                + "have a smaller number of larger shards. Update [" + SHARD_COUNT_LIMIT_SETTING.getKey()
+                + "] to a greater value if you really want to query that many shards at the same time.");
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java b/core/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java
index db42527b125fa..94632973984b5 100644
--- a/core/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java
+++ b/core/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java
@@ -21,11 +21,11 @@
 
 import org.apache.lucene.store.ByteArrayDataInput;
 import org.apache.lucene.store.RAMOutputStream;
-import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.search.SearchPhaseResult;
+import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.internal.InternalScrollSearchRequest;
-import org.elasticsearch.search.internal.ShardSearchTransportRequest;
+import org.elasticsearch.transport.RemoteClusterAware;
 
 import java.io.IOException;
 import java.util.Base64;
@@ -35,33 +35,23 @@
  */
 final class TransportSearchHelper {
 
-    static ShardSearchTransportRequest internalSearchRequest(ShardRouting shardRouting, int numberOfShards, SearchRequest request,
-                                                             String[] filteringAliases, long nowInMillis) {
-        return new ShardSearchTransportRequest(request, shardRouting, numberOfShards, filteringAliases, nowInMillis);
-    }
-
     static InternalScrollSearchRequest internalScrollSearchRequest(long id, SearchScrollRequest request) {
         return new InternalScrollSearchRequest(request, id);
     }
 
-    static String buildScrollId(SearchType searchType, AtomicArray<? extends SearchPhaseResult> searchPhaseResults) throws IOException {
-        if (searchType == SearchType.DFS_QUERY_THEN_FETCH || searchType == SearchType.QUERY_THEN_FETCH) {
-            return buildScrollId(ParsedScrollId.QUERY_THEN_FETCH_TYPE, searchPhaseResults);
-        } else if (searchType == SearchType.QUERY_AND_FETCH || searchType == SearchType.DFS_QUERY_AND_FETCH) {
-            return buildScrollId(ParsedScrollId.QUERY_AND_FETCH_TYPE, searchPhaseResults);
-        } else {
-            throw new IllegalStateException("search_type [" + searchType + "] not supported");
-        }
-    }
-
-    static String buildScrollId(String type, AtomicArray<? extends SearchPhaseResult> searchPhaseResults) throws IOException {
+    static String buildScrollId(AtomicArray<? extends SearchPhaseResult> searchPhaseResults) throws IOException {
         try (RAMOutputStream out = new RAMOutputStream()) {
-            out.writeString(type);
+            out.writeString(searchPhaseResults.length() == 1 ? ParsedScrollId.QUERY_AND_FETCH_TYPE : ParsedScrollId.QUERY_THEN_FETCH_TYPE);
             out.writeVInt(searchPhaseResults.asList().size());
-            for (AtomicArray.Entry<? extends SearchPhaseResult> entry : searchPhaseResults.asList()) {
-                SearchPhaseResult searchPhaseResult = entry.value;
-                out.writeLong(searchPhaseResult.id());
-                out.writeString(searchPhaseResult.shardTarget().nodeId());
+            for (SearchPhaseResult searchPhaseResult : searchPhaseResults.asList()) {
+                out.writeLong(searchPhaseResult.getRequestId());
+                SearchShardTarget searchShardTarget = searchPhaseResult.getSearchShardTarget();
+                if (searchShardTarget.getClusterAlias() != null) {
+                    out.writeString(RemoteClusterAware.buildRemoteIndexName(searchShardTarget.getClusterAlias(),
+                        searchShardTarget.getNodeId()));
+                } else {
+                    out.writeString(searchShardTarget.getNodeId());
+                }
             }
             byte[] bytes = new byte[(int) out.getFilePointer()];
             out.writeTo(bytes, 0);
@@ -78,7 +68,15 @@ static ParsedScrollId parseScrollId(String scrollId) {
             for (int i = 0; i < context.length; ++i) {
                 long id = in.readLong();
                 String target = in.readString();
-                context[i] = new ScrollIdForNode(target, id);
+                String clusterAlias;
+                final int index = target.indexOf(RemoteClusterAware.REMOTE_CLUSTER_INDEX_SEPARATOR);
+                if (index == -1) {
+                    clusterAlias = null;
+                } else {
+                    clusterAlias = target.substring(0, index);
+                    target = target.substring(index+1);
+                }
+                context[i] = new ScrollIdForNode(clusterAlias, target, id);
             }
             if (in.getPosition() != bytes.length) {
                 throw new IllegalArgumentException("Not all bytes were read");
diff --git a/core/src/main/java/org/elasticsearch/action/search/TransportSearchScrollAction.java b/core/src/main/java/org/elasticsearch/action/search/TransportSearchScrollAction.java
index 485baaa022b40..448dd9d636763 100644
--- a/core/src/main/java/org/elasticsearch/action/search/TransportSearchScrollAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/TransportSearchScrollAction.java
@@ -26,8 +26,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.search.action.SearchTransportService;
-import org.elasticsearch.search.controller.SearchPhaseController;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -46,9 +45,9 @@ public class TransportSearchScrollAction extends HandledTransportAction<SearchSc
 
     @Inject
     public TransportSearchScrollAction(Settings settings, ThreadPool threadPool, TransportService transportService,
-                                       ClusterService clusterService, SearchTransportService searchTransportService,
-                                       SearchPhaseController searchPhaseController,
-                                       ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
+                                       ClusterService clusterService, ActionFilters actionFilters,
+                                       IndexNameExpressionResolver indexNameExpressionResolver,
+                                       SearchTransportService searchTransportService, SearchPhaseController searchPhaseController) {
         super(settings, SearchScrollAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver,
                 SearchScrollRequest::new);
         this.clusterService = clusterService;
@@ -57,23 +56,27 @@ public TransportSearchScrollAction(Settings settings, ThreadPool threadPool, Tra
     }
 
     @Override
-    protected void doExecute(SearchScrollRequest request, ActionListener<SearchResponse> listener) {
+    protected final void doExecute(SearchScrollRequest request, ActionListener<SearchResponse> listener) {
+        throw new UnsupportedOperationException("the task parameter is required");
+    }
+    @Override
+    protected void doExecute(Task task, SearchScrollRequest request, ActionListener<SearchResponse> listener) {
         try {
             ParsedScrollId scrollId = parseScrollId(request.scrollId());
-            AbstractAsyncAction action;
+            Runnable action;
             switch (scrollId.getType()) {
                 case QUERY_THEN_FETCH_TYPE:
                     action = new SearchScrollQueryThenFetchAsyncAction(logger, clusterService, searchTransportService,
-                            searchPhaseController, request, scrollId, listener);
+                        searchPhaseController, request, (SearchTask)task, scrollId, listener);
                     break;
-                case QUERY_AND_FETCH_TYPE:
+                case QUERY_AND_FETCH_TYPE: // TODO can we get rid of this?
                     action = new SearchScrollQueryAndFetchAsyncAction(logger, clusterService, searchTransportService,
-                            searchPhaseController, request, scrollId, listener);
+                        searchPhaseController, request, (SearchTask)task, scrollId, listener);
                     break;
                 default:
                     throw new IllegalArgumentException("Scroll id type [" + scrollId.getType() + "] unrecognized");
             }
-            action.start();
+            action.run();
         } catch (Exception e) {
             listener.onFailure(e);
         }
diff --git a/core/src/main/java/org/elasticsearch/action/support/ActionFilter.java b/core/src/main/java/org/elasticsearch/action/support/ActionFilter.java
index f536d9e0ceb69..3e12d0cc84223 100644
--- a/core/src/main/java/org/elasticsearch/action/support/ActionFilter.java
+++ b/core/src/main/java/org/elasticsearch/action/support/ActionFilter.java
@@ -40,29 +40,21 @@ public interface ActionFilter {
      * Enables filtering the execution of an action on the request side, either by sending a response through the
      * {@link ActionListener} or by continuing the execution through the given {@link ActionFilterChain chain}
      */
-    <Request extends ActionRequest<Request>, Response extends ActionResponse> void apply(Task task, String action, Request request,
+    <Request extends ActionRequest, Response extends ActionResponse> void apply(Task task, String action, Request request,
             ActionListener<Response> listener, ActionFilterChain<Request, Response> chain);
-
-    /**
-     * Enables filtering the execution of an action on the response side, either by sending a response through the
-     * {@link ActionListener} or by continuing the execution through the given {@link ActionFilterChain chain}
-     */
-    <Response extends ActionResponse> void apply(String action, Response response, ActionListener<Response> listener,
-            ActionFilterChain<?, Response> chain);
-
     /**
      * A simple base class for injectable action filters that spares the implementation from handling the
      * filter chain. This base class should serve any action filter implementations that doesn't require
      * to apply async filtering logic.
      */
-    public abstract static class Simple extends AbstractComponent implements ActionFilter {
+    abstract class Simple extends AbstractComponent implements ActionFilter {
 
         protected Simple(Settings settings) {
             super(settings);
         }
 
         @Override
-        public final <Request extends ActionRequest<Request>, Response extends ActionResponse> void apply(Task task, String action, Request request,
+        public final <Request extends ActionRequest, Response extends ActionResponse> void apply(Task task, String action, Request request,
                 ActionListener<Response> listener, ActionFilterChain<Request, Response> chain) {
             if (apply(action, request, listener)) {
                 chain.proceed(task, action, request, listener);
@@ -73,20 +65,6 @@ public final <Request extends ActionRequest<Request>, Response extends ActionRes
          * Applies this filter and returns {@code true} if the execution chain should proceed, or {@code false}
          * if it should be aborted since the filter already handled the request and called the given listener.
          */
-        protected abstract boolean apply(String action, ActionRequest<?> request, ActionListener<?> listener);
-
-        @Override
-        public final <Response extends ActionResponse> void apply(String action, Response response, ActionListener<Response> listener,
-                ActionFilterChain<?, Response> chain) {
-            if (apply(action, response, listener)) {
-                chain.proceed(action, response, listener);
-            }
-        }
-
-        /**
-         * Applies this filter and returns {@code true} if the execution chain should proceed, or {@code false}
-         * if it should be aborted since the filter already handled the response by calling the given listener.
-         */
-        protected abstract boolean apply(String action, ActionResponse response, ActionListener<?> listener);
+        protected abstract boolean apply(String action, ActionRequest request, ActionListener<?> listener);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/support/ActionFilterChain.java b/core/src/main/java/org/elasticsearch/action/support/ActionFilterChain.java
index 54f55e187a995..97e0c535bffdf 100644
--- a/core/src/main/java/org/elasticsearch/action/support/ActionFilterChain.java
+++ b/core/src/main/java/org/elasticsearch/action/support/ActionFilterChain.java
@@ -27,17 +27,11 @@
 /**
  * A filter chain allowing to continue and process the transport action request
  */
-public interface ActionFilterChain<Request extends ActionRequest<Request>, Response extends ActionResponse> {
+public interface ActionFilterChain<Request extends ActionRequest, Response extends ActionResponse> {
 
     /**
      * Continue processing the request. Should only be called if a response has not been sent through
      * the given {@link ActionListener listener}
      */
-    void proceed(Task task, final String action, final Request request, final ActionListener<Response> listener);
-
-    /**
-     * Continue processing the response. Should only be called if a response has not been sent through
-     * the given {@link ActionListener listener}
-     */
-    void proceed(final String action, final Response response, final ActionListener<Response> listener);
+    void proceed(Task task, String action, Request request, ActionListener<Response> listener);
 }
diff --git a/core/src/main/java/org/elasticsearch/action/support/ActiveShardsObserver.java b/core/src/main/java/org/elasticsearch/action/support/ActiveShardsObserver.java
index 7217961d899c4..280ba6ac94dc6 100644
--- a/core/src/main/java/org/elasticsearch/action/support/ActiveShardsObserver.java
+++ b/core/src/main/java/org/elasticsearch/action/support/ActiveShardsObserver.java
@@ -30,6 +30,7 @@
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.function.Consumer;
+import java.util.function.Predicate;
 
 /**
  * This class provides primitives for waiting for a configured number of shards
@@ -68,17 +69,12 @@ public void waitForActiveShards(final String indexName,
             return;
         }
 
-        final ClusterStateObserver observer = new ClusterStateObserver(clusterService, logger, threadPool.getThreadContext());
-        if (activeShardCount.enoughShardsActive(observer.observedState(), indexName)) {
-                onResult.accept(true);
+        final ClusterState state = clusterService.state();
+        final ClusterStateObserver observer = new ClusterStateObserver(state, clusterService, null, logger, threadPool.getThreadContext());
+        if (activeShardCount.enoughShardsActive(state, indexName)) {
+            onResult.accept(true);
         } else {
-            final ClusterStateObserver.ChangePredicate shardsAllocatedPredicate =
-                new ClusterStateObserver.ValidationPredicate() {
-                    @Override
-                    protected boolean validate(final ClusterState newState) {
-                        return activeShardCount.enoughShardsActive(newState, indexName);
-                    }
-                };
+            final Predicate<ClusterState> shardsAllocatedPredicate = newState -> activeShardCount.enoughShardsActive(newState, indexName);
 
             final ClusterStateObserver.Listener observerListener = new ClusterStateObserver.Listener() {
                 @Override
diff --git a/core/src/main/java/org/elasticsearch/action/support/AdapterActionFuture.java b/core/src/main/java/org/elasticsearch/action/support/AdapterActionFuture.java
index eab486f492934..b0f914e1a6298 100644
--- a/core/src/main/java/org/elasticsearch/action/support/AdapterActionFuture.java
+++ b/core/src/main/java/org/elasticsearch/action/support/AdapterActionFuture.java
@@ -41,6 +41,7 @@ public T actionGet() {
         try {
             return get();
         } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
             throw new IllegalStateException("Future got interrupted", e);
         } catch (ExecutionException e) {
             throw rethrowExecutionException(e);
@@ -69,6 +70,7 @@ public T actionGet(long timeout, TimeUnit unit) {
         } catch (TimeoutException e) {
             throw new ElasticsearchTimeoutException(e);
         } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
             throw new IllegalStateException("Future got interrupted", e);
         } catch (ExecutionException e) {
             throw rethrowExecutionException(e);
@@ -103,4 +105,5 @@ public void onFailure(Exception e) {
     }
 
     protected abstract T convert(L listenerResponse);
+
 }
diff --git a/core/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java b/core/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java
index 5c9152b475179..6a53b456166bf 100644
--- a/core/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java
+++ b/core/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java
@@ -24,11 +24,14 @@
 import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.mapper.MapperService;
 
 import java.util.ArrayList;
@@ -39,6 +42,7 @@
  * a write operation is about to happen in a non existing index.
  */
 public final class AutoCreateIndex {
+    private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(AutoCreateIndex.class));
 
     public static final Setting<AutoCreate> AUTO_CREATE_INDEX_SETTING =
         new Setting<>("action.auto_create_index", "true", AutoCreate::new, Property.NodeScope, Setting.Property.Dynamic);
@@ -63,18 +67,20 @@ public boolean needToCheck() {
 
     /**
      * Should the index be auto created?
+     * @throws IndexNotFoundException if the the index doesn't exist and shouldn't be auto created
      */
     public boolean shouldAutoCreate(String index, ClusterState state) {
+        if (resolver.hasIndexOrAlias(index, state)) {
+            return false;
+        }
         // One volatile read, so that all checks are done against the same instance:
         final AutoCreate autoCreate = this.autoCreate;
         if (autoCreate.autoCreateIndex == false) {
-            return false;
+            throw new IndexNotFoundException("no such index and [" + AUTO_CREATE_INDEX_SETTING.getKey() + "] is [false]", index);
         }
         if (dynamicMappingDisabled) {
-            return false;
-        }
-        if (resolver.hasIndexOrAlias(index, state)) {
-            return false;
+            throw new IndexNotFoundException("no such index and [" + MapperService.INDEX_MAPPER_DYNAMIC_SETTING.getKey() + "] is [false]",
+                    index);
         }
         // matches not set, default value of "true"
         if (autoCreate.expressions.isEmpty()) {
@@ -84,10 +90,15 @@ public boolean shouldAutoCreate(String index, ClusterState state) {
             String indexExpression = expression.v1();
             boolean include = expression.v2();
             if (Regex.simpleMatch(indexExpression, index)) {
-                return include;
+                if (include) {
+                    return true;
+                }
+                throw new IndexNotFoundException("no such index and [" + AUTO_CREATE_INDEX_SETTING.getKey() + "] contains [-"
+                        + indexExpression + "] which forbids automatic creation of the index", index);
             }
         }
-        return false;
+        throw new IndexNotFoundException("no such index and [" + AUTO_CREATE_INDEX_SETTING.getKey() + "] ([" + autoCreate
+                + "]) doesn't match", index);
     }
 
     AutoCreate getAutoCreate() {
@@ -101,28 +112,37 @@ void setAutoCreate(AutoCreate autoCreate) {
     static class AutoCreate {
         private final boolean autoCreateIndex;
         private final List<Tuple<String, Boolean>> expressions;
+        private final String string;
 
         private AutoCreate(String value) {
             boolean autoCreateIndex;
             List<Tuple<String, Boolean>> expressions = new ArrayList<>();
             try {
                 autoCreateIndex = Booleans.parseBooleanExact(value);
+                if (Booleans.isStrictlyBoolean(value) == false) {
+                    DEPRECATION_LOGGER.deprecated("Expected a boolean [true/false] or index name pattern for setting [{}] but got [{}]",
+                        AUTO_CREATE_INDEX_SETTING.getKey(), value);
+                }
             } catch (IllegalArgumentException ex) {
                 try {
                     String[] patterns = Strings.commaDelimitedListToStringArray(value);
                     for (String pattern : patterns) {
-                        if (pattern == null || pattern.length() == 0) {
-                            throw new IllegalArgumentException("Can't parse [" + value + "] for setting [action.auto_create_index] must be either [true, false, or a comma separated list of index patterns]");
+                        if (pattern == null || pattern.trim().length() == 0) {
+                            throw new IllegalArgumentException("Can't parse [" + value + "] for setting [action.auto_create_index] must "
+                                    + "be either [true, false, or a comma separated list of index patterns]");
                         }
+                        pattern = pattern.trim();
                         Tuple<String, Boolean> expression;
                         if (pattern.startsWith("-")) {
                             if (pattern.length() == 1) {
-                                throw new IllegalArgumentException("Can't parse [" + value + "] for setting [action.auto_create_index] must contain an index name after [-]");
+                                throw new IllegalArgumentException("Can't parse [" + value + "] for setting [action.auto_create_index] "
+                                        + "must contain an index name after [-]");
                             }
                             expression = new Tuple<>(pattern.substring(1), false);
                         } else if(pattern.startsWith("+")) {
                             if (pattern.length() == 1) {
-                                throw new IllegalArgumentException("Can't parse [" + value + "] for setting [action.auto_create_index] must contain an index name after [+]");
+                                throw new IllegalArgumentException("Can't parse [" + value + "] for setting [action.auto_create_index] "
+                                        + "must contain an index name after [+]");
                             }
                             expression = new Tuple<>(pattern.substring(1), true);
                         } else {
@@ -138,6 +158,7 @@ private AutoCreate(String value) {
             }
             this.expressions = expressions;
             this.autoCreateIndex = autoCreateIndex;
+            this.string = value;
         }
 
         boolean isAutoCreateIndex() {
@@ -147,5 +168,10 @@ boolean isAutoCreateIndex() {
         List<Tuple<String, Boolean>> getExpressions() {
             return expressions;
         }
+
+        @Override
+        public String toString() {
+            return string;
+        }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/support/ContextPreservingActionListener.java b/core/src/main/java/org/elasticsearch/action/support/ContextPreservingActionListener.java
new file mode 100644
index 0000000000000..72f1e7c1d6643
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/support/ContextPreservingActionListener.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.support;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+
+import java.util.function.Supplier;
+
+/**
+ * Restores the given {@link org.elasticsearch.common.util.concurrent.ThreadContext.StoredContext}
+ * once the listener is invoked
+ */
+public final class ContextPreservingActionListener<R> implements ActionListener<R> {
+
+    private final ActionListener<R> delegate;
+    private final Supplier<ThreadContext.StoredContext> context;
+
+    public ContextPreservingActionListener(Supplier<ThreadContext.StoredContext> contextSupplier, ActionListener<R> delegate) {
+        this.delegate = delegate;
+        this.context = contextSupplier;
+    }
+
+    @Override
+    public void onResponse(R r) {
+        try (ThreadContext.StoredContext ignore = context.get()) {
+            delegate.onResponse(r);
+        }
+    }
+
+    @Override
+    public void onFailure(Exception e) {
+        try (ThreadContext.StoredContext ignore = context.get()) {
+            delegate.onFailure(e);
+        }
+    }
+
+    /**
+     * Wraps the provided action listener in a {@link ContextPreservingActionListener} that will
+     * also copy the response headers when the {@link ThreadContext.StoredContext} is closed
+     */
+    public static <R> ContextPreservingActionListener<R> wrapPreservingContext(ActionListener<R> listener, ThreadContext threadContext) {
+        return new ContextPreservingActionListener<>(threadContext.newRestorableContext(true), listener);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/support/DefaultShardOperationFailedException.java b/core/src/main/java/org/elasticsearch/action/support/DefaultShardOperationFailedException.java
index 0fe3be1ad634b..656a0eb90d9d3 100644
--- a/core/src/main/java/org/elasticsearch/action/support/DefaultShardOperationFailedException.java
+++ b/core/src/main/java/org/elasticsearch/action/support/DefaultShardOperationFailedException.java
@@ -128,7 +128,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (reason != null) {
             builder.field("reason");
             builder.startObject();
-            ElasticsearchException.toXContent(builder, params, reason);
+            ElasticsearchException.generateThrowableXContent(builder, params, reason);
             builder.endObject();
         }
         return builder;
diff --git a/core/src/main/java/org/elasticsearch/action/support/GroupedActionListener.java b/core/src/main/java/org/elasticsearch/action/support/GroupedActionListener.java
new file mode 100644
index 0000000000000..ed9b7c8d15d60
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/support/GroupedActionListener.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.support;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.common.util.concurrent.AtomicArray;
+import org.elasticsearch.common.util.concurrent.CountDown;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ * An action listener that delegates it's results to another listener once
+ * it has received one or more failures or N results. This allows synchronous
+ * tasks to be forked off in a loop with the same listener and respond to a
+ * higher level listener once all tasks responded.
+ */
+public final class GroupedActionListener<T> implements ActionListener<T> {
+    private final CountDown countDown;
+    private final AtomicInteger pos = new AtomicInteger();
+    private final AtomicArray<T> results;
+    private final ActionListener<Collection<T>> delegate;
+    private final Collection<T> defaults;
+    private final AtomicReference<Exception> failure = new AtomicReference<>();
+
+    /**
+     * Creates a new listener
+     * @param delegate the delegate listener
+     * @param groupSize the group size
+     */
+    public GroupedActionListener(ActionListener<Collection<T>> delegate, int groupSize,
+                                 Collection<T> defaults) {
+        results = new AtomicArray<>(groupSize);
+        countDown = new CountDown(groupSize);
+        this.delegate = delegate;
+        this.defaults = defaults;
+    }
+
+    @Override
+    public void onResponse(T element) {
+        results.setOnce(pos.incrementAndGet() - 1, element);
+        if (countDown.countDown()) {
+            if (failure.get() != null) {
+                delegate.onFailure(failure.get());
+            } else {
+                List<T> collect = this.results.asList();
+                collect.addAll(defaults);
+                delegate.onResponse(Collections.unmodifiableList(collect));
+            }
+        }
+    }
+
+    @Override
+    public void onFailure(Exception e) {
+        if (failure.compareAndSet(null, e) == false) {
+            failure.get().addSuppressed(e);
+        }
+        if (countDown.countDown()) {
+            delegate.onFailure(failure.get());
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java b/core/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java
index 0a53b63b662af..68b699cb110ba 100644
--- a/core/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java
@@ -35,7 +35,7 @@
 /**
  * A TransportAction that self registers a handler into the transport service
  */
-public abstract class HandledTransportAction<Request extends ActionRequest<Request>, Response extends ActionResponse>
+public abstract class HandledTransportAction<Request extends ActionRequest, Response extends ActionResponse>
         extends TransportAction<Request, Response> {
     protected HandledTransportAction(Settings settings, String actionName, ThreadPool threadPool, TransportService transportService,
                                      ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
diff --git a/core/src/main/java/org/elasticsearch/action/support/IndicesOptions.java b/core/src/main/java/org/elasticsearch/action/support/IndicesOptions.java
index 2bc49f7e9f869..a79044f2d6923 100644
--- a/core/src/main/java/org/elasticsearch/action/support/IndicesOptions.java
+++ b/core/src/main/java/org/elasticsearch/action/support/IndicesOptions.java
@@ -195,8 +195,8 @@ public static IndicesOptions fromParameters(Object wildcardsString, Object ignor
 
         //note that allowAliasesToMultipleIndices is not exposed, always true (only for internal use)
         return fromOptions(
-                lenientNodeBooleanValue(ignoreUnavailableString, defaultSettings.ignoreUnavailable()),
-                lenientNodeBooleanValue(allowNoIndicesString, defaultSettings.allowNoIndices()),
+                lenientNodeBooleanValue(ignoreUnavailableString, "ignore_unavailable", defaultSettings.ignoreUnavailable()),
+                lenientNodeBooleanValue(allowNoIndicesString, "allow_no_indices", defaultSettings.allowNoIndices()),
                 expandWildcardsOpen,
                 expandWildcardsClosed,
                 defaultSettings.allowAliasesToMultipleIndices(),
diff --git a/core/src/main/java/org/elasticsearch/action/support/ToXContentToBytes.java b/core/src/main/java/org/elasticsearch/action/support/ToXContentToBytes.java
index 445967252a237..741b197dbc836 100644
--- a/core/src/main/java/org/elasticsearch/action/support/ToXContentToBytes.java
+++ b/core/src/main/java/org/elasticsearch/action/support/ToXContentToBytes.java
@@ -69,10 +69,16 @@ public final BytesReference buildAsBytes(XContentType contentType) {
 
     @Override
     public final String toString() {
+        return toString(EMPTY_PARAMS);
+    }
+
+    public final String toString(Params params) {
         try {
             XContentBuilder builder = XContentFactory.jsonBuilder();
-            builder.prettyPrint();
-            toXContent(builder, EMPTY_PARAMS);
+            if (params.paramAsBoolean("pretty", true)) {
+                builder.prettyPrint();
+            }
+            toXContent(builder, params);
             return builder.string();
         } catch (Exception e) {
             // So we have a stack trace logged somewhere
diff --git a/core/src/main/java/org/elasticsearch/action/support/TransportAction.java b/core/src/main/java/org/elasticsearch/action/support/TransportAction.java
index 7d1a091d6b3c8..ea606a15d7cc0 100644
--- a/core/src/main/java/org/elasticsearch/action/support/TransportAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/TransportAction.java
@@ -26,7 +26,6 @@
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.tasks.Task;
@@ -41,12 +40,11 @@
 /**
  *
  */
-public abstract class TransportAction<Request extends ActionRequest<Request>, Response extends ActionResponse> extends AbstractComponent {
+public abstract class TransportAction<Request extends ActionRequest, Response extends ActionResponse> extends AbstractComponent {
 
     protected final ThreadPool threadPool;
     protected final String actionName;
     private final ActionFilter[] filters;
-    protected final ParseFieldMatcher parseFieldMatcher;
     protected final IndexNameExpressionResolver indexNameExpressionResolver;
     protected final TaskManager taskManager;
 
@@ -56,7 +54,6 @@ protected TransportAction(Settings settings, String actionName, ThreadPool threa
         this.threadPool = threadPool;
         this.actionName = actionName;
         this.filters = actionFilters.filters();
-        this.parseFieldMatcher = new ParseFieldMatcher(settings);
         this.indexNameExpressionResolver = indexNameExpressionResolver;
         this.taskManager = taskManager;
     }
@@ -141,17 +138,8 @@ public final void execute(Task task, Request request, ActionListener<Response> l
             listener = new TaskResultStoringActionListener<>(taskManager, task, listener);
         }
 
-        if (filters.length == 0) {
-            try {
-                doExecute(task, request, listener);
-            } catch(Exception e) {
-                logger.trace("Error during transport action execution.", e);
-                listener.onFailure(e);
-            }
-        } else {
-            RequestFilterChain<Request, Response> requestFilterChain = new RequestFilterChain<>(this, logger);
-            requestFilterChain.proceed(task, actionName, request, listener);
-        }
+        RequestFilterChain<Request, Response> requestFilterChain = new RequestFilterChain<>(this, logger);
+        requestFilterChain.proceed(task, actionName, request, listener);
     }
 
     protected void doExecute(Task task, Request request, ActionListener<Response> listener) {
@@ -160,7 +148,7 @@ protected void doExecute(Task task, Request request, ActionListener<Response> li
 
     protected abstract void doExecute(Request request, ActionListener<Response> listener);
 
-    private static class RequestFilterChain<Request extends ActionRequest<Request>, Response extends ActionResponse>
+    private static class RequestFilterChain<Request extends ActionRequest, Response extends ActionResponse>
             implements ActionFilterChain<Request, Response> {
 
         private final TransportAction<Request, Response> action;
@@ -179,8 +167,7 @@ public void proceed(Task task, String actionName, Request request, ActionListene
                 if (i < this.action.filters.length) {
                     this.action.filters[i].apply(task, actionName, request, listener, this);
                 } else if (i == this.action.filters.length) {
-                    this.action.doExecute(task, request, new FilteredActionListener<>(actionName, listener,
-                            new ResponseFilterChain<>(this.action.filters, logger)));
+                    this.action.doExecute(task, request, listener);
                 } else {
                     listener.onFailure(new IllegalStateException("proceed was called too many times"));
                 }
@@ -190,69 +177,6 @@ public void proceed(Task task, String actionName, Request request, ActionListene
             }
         }
 
-        @Override
-        public void proceed(String action, Response response, ActionListener<Response> listener) {
-            assert false : "request filter chain should never be called on the response side";
-        }
-    }
-
-    private static class ResponseFilterChain<Request extends ActionRequest<Request>, Response extends ActionResponse>
-            implements ActionFilterChain<Request, Response> {
-
-        private final ActionFilter[] filters;
-        private final AtomicInteger index;
-        private final Logger logger;
-
-        private ResponseFilterChain(ActionFilter[] filters, Logger logger) {
-            this.filters = filters;
-            this.index = new AtomicInteger(filters.length);
-            this.logger = logger;
-        }
-
-        @Override
-        public void proceed(Task task, String action, Request request, ActionListener<Response> listener) {
-            assert false : "response filter chain should never be called on the request side";
-        }
-
-        @Override
-        public void proceed(String action, Response response, ActionListener<Response> listener) {
-            int i = index.decrementAndGet();
-            try {
-                if (i >= 0) {
-                    filters[i].apply(action, response, listener, this);
-                } else if (i == -1) {
-                    listener.onResponse(response);
-                } else {
-                    listener.onFailure(new IllegalStateException("proceed was called too many times"));
-                }
-            } catch (Exception e) {
-                logger.trace("Error during transport action execution.", e);
-                listener.onFailure(e);
-            }
-        }
-    }
-
-    private static class FilteredActionListener<Response extends ActionResponse> implements ActionListener<Response> {
-
-        private final String actionName;
-        private final ActionListener<Response> listener;
-        private final ResponseFilterChain<?, Response> chain;
-
-        private FilteredActionListener(String actionName, ActionListener<Response> listener, ResponseFilterChain<?, Response> chain) {
-            this.actionName = actionName;
-            this.listener = listener;
-            this.chain = chain;
-        }
-
-        @Override
-        public void onResponse(Response response) {
-            chain.proceed(actionName, response, listener);
-        }
-
-        @Override
-        public void onFailure(Exception e) {
-            listener.onFailure(e);
-        }
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/action/support/WriteRequest.java b/core/src/main/java/org/elasticsearch/action/support/WriteRequest.java
index 6379a4fb259c3..50edcd39bd16e 100644
--- a/core/src/main/java/org/elasticsearch/action/support/WriteRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/support/WriteRequest.java
@@ -65,36 +65,43 @@ enum RefreshPolicy implements Writeable {
         /**
          * Don't refresh after this request. The default.
          */
-        NONE,
+        NONE("false"),
         /**
          * Force a refresh as part of this request. This refresh policy does not scale for high indexing or search throughput but is useful
          * to present a consistent view to for indices with very low traffic. And it is wonderful for tests!
          */
-        IMMEDIATE,
+        IMMEDIATE("true"),
         /**
          * Leave this request open until a refresh has made the contents of this request visible to search. This refresh policy is
          * compatible with high indexing and search throughput but it causes the request to wait to reply until a refresh occurs.
          */
-        WAIT_UNTIL;
+        WAIT_UNTIL("wait_for");
+
+        private final String value;
+
+        RefreshPolicy(String value) {
+            this.value = value;
+        }
+
+        public String getValue() {
+            return value;
+        }
 
         /**
          * Parse the string representation of a refresh policy, usually from a request parameter.
          */
-        public static RefreshPolicy parse(String string) {
-            switch (string) {
-            case "false":
-                return NONE;
-            /*
-             * Empty string is IMMEDIATE because that makes "POST /test/test/1?refresh" perform a refresh which reads well and is what folks
-             * are used to.
-             */
-            case "":
-            case "true":
+        public static RefreshPolicy parse(String value) {
+            for (RefreshPolicy policy : values()) {
+                if (policy.getValue().equals(value)) {
+                    return policy;
+                }
+            }
+            if ("".equals(value)) {
+                // Empty string is IMMEDIATE because that makes "POST /test/test/1?refresh" perform
+                // a refresh which reads well and is what folks are used to.
                 return IMMEDIATE;
-            case "wait_for":
-                return WAIT_UNTIL;
             }
-            throw new IllegalArgumentException("Unknown value for refresh: [" + string + "].");
+            throw new IllegalArgumentException("Unknown value for refresh: [" + value + "].");
         }
 
         public static RefreshPolicy readFrom(StreamInput in) throws IOException {
diff --git a/core/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastRequest.java b/core/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastRequest.java
index 508581050a6eb..1a4a3b39ef066 100644
--- a/core/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastRequest.java
@@ -31,7 +31,7 @@
 /**
  *
  */
-public class BroadcastRequest<Request extends BroadcastRequest<Request>> extends ActionRequest<Request> implements IndicesRequest.Replaceable {
+public class BroadcastRequest<Request extends BroadcastRequest<Request>> extends ActionRequest implements IndicesRequest.Replaceable {
 
     protected String[] indices;
     private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpenAndForbidClosed();
diff --git a/core/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java b/core/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java
index 87ef385a243c8..f8b0c087ea7e2 100644
--- a/core/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java
@@ -44,6 +44,7 @@
 import org.elasticsearch.transport.TransportResponseHandler;
 import org.elasticsearch.transport.TransportService;
 
+import java.io.IOException;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReferenceArray;
 import java.util.function.Supplier;
@@ -86,9 +87,9 @@ protected final void doExecute(Request request, ActionListener<Response> listene
 
     protected abstract ShardResponse newShardResponse();
 
-    protected abstract ShardResponse shardOperation(ShardRequest request);
+    protected abstract ShardResponse shardOperation(ShardRequest request) throws IOException;
 
-    protected ShardResponse shardOperation(ShardRequest request, Task task) {
+    protected ShardResponse shardOperation(ShardRequest request, Task task) throws IOException {
         return shardOperation(request);
     }
 
@@ -96,7 +97,7 @@ protected ShardResponse shardOperation(ShardRequest request, Task task) {
      * Determines the shards this operation will be executed on. The operation is executed once per shard iterator, typically
      * on the first shard in it. If the operation fails, it will be retried on the next shard in the iterator.
      */
-    protected abstract GroupShardsIterator shards(ClusterState clusterState, Request request, String[] concreteIndices);
+    protected abstract GroupShardsIterator<ShardIterator> shards(ClusterState clusterState, Request request, String[] concreteIndices);
 
     protected abstract ClusterBlockException checkGlobalBlock(ClusterState state, Request request);
 
@@ -109,7 +110,7 @@ protected class AsyncBroadcastAction {
         private final ActionListener<Response> listener;
         private final ClusterState clusterState;
         private final DiscoveryNodes nodes;
-        private final GroupShardsIterator shardsIts;
+        private final GroupShardsIterator<ShardIterator> shardsIts;
         private final int expectedOps;
         private final AtomicInteger counterOps = new AtomicInteger();
         private final AtomicReferenceArray shardsResponses;
@@ -177,7 +178,6 @@ protected void performOperation(final ShardIterator shardIt, final ShardRouting
                         // no node connected, act as failure
                         onOperation(shard, shardIt, shardIndex, new NoShardAvailableActionException(shardIt.shardId()));
                     } else {
-                        taskManager.registerChildTask(task, node.getId());
                         transportService.sendRequest(node, transportShardAction, shardRequest, new TransportResponseHandler<ShardResponse>() {
                             @Override
                             public ShardResponse newInstance() {
diff --git a/core/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java b/core/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java
index 98c962b3eec69..b0c539be39e4f 100644
--- a/core/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java
@@ -270,7 +270,7 @@ protected AsyncAction(Task task, Request request, ActionListener<Response> liste
             ShardsIterator shardIt = shards(clusterState, request, concreteIndices);
             nodeIds = new HashMap<>();
 
-            for (ShardRouting shard : shardIt.asUnordered()) {
+            for (ShardRouting shard : shardIt) {
                 // send a request to the shard only if it is assigned to a node that is in the local node's cluster state
                 // a scenario in which a shard can be assigned but to a node that is not in the local node's cluster state
                 // is when the shard is assigned to the master node, the local node has detected the master as failed
@@ -318,7 +318,6 @@ private void sendNodeRequest(final DiscoveryNode node, List<ShardRouting> shards
                 NodeRequest nodeRequest = new NodeRequest(node.getId(), request, shards);
                 if (task != null) {
                     nodeRequest.setParentTask(clusterService.localNode().getId(), task.getId());
-                    taskManager.registerChildTask(task, node.getId());
                 }
                 transportService.sendRequest(node, transportNodeBroadcastAction, nodeRequest, new TransportResponseHandler<NodeResponse>() {
                     @Override
@@ -439,7 +438,6 @@ private void onShardOperation(final NodeRequest request, final Object[] shardRes
             } catch (Exception e) {
                 BroadcastShardOperationFailedException failure =
                     new BroadcastShardOperationFailedException(shardRouting.shardId(), "operation " + actionName + " failed", e);
-                failure.setIndex(shardRouting.getIndexName());
                 failure.setShard(shardRouting.shardId());
                 shardResults[shardIndex] = failure;
                 if (TransportActions.isShardNotAvailableException(e)) {
@@ -505,11 +503,7 @@ public IndicesOptions indicesOptions() {
         public void readFrom(StreamInput in) throws IOException {
             super.readFrom(in);
             indicesLevelRequest = readRequestFrom(in);
-            int size = in.readVInt();
-            shards = new ArrayList<>(size);
-            for (int i = 0; i < size; i++) {
-                shards.add(new ShardRouting(in));
-            }
+            shards = in.readList(ShardRouting::new);
             nodeId = in.readString();
         }
 
@@ -517,11 +511,7 @@ public void readFrom(StreamInput in) throws IOException {
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
             indicesLevelRequest.writeTo(out);
-            int size = shards.size();
-            out.writeVInt(size);
-            for (int i = 0; i < size; i++) {
-                shards.get(i).writeTo(out);
-            }
+            out.writeList(shards);
             out.writeString(nodeId);
         }
     }
@@ -532,10 +522,10 @@ class NodeResponse extends TransportResponse {
         protected List<BroadcastShardOperationFailedException> exceptions;
         protected List<ShardOperationResult> results;
 
-        public NodeResponse() {
+        NodeResponse() {
         }
 
-        public NodeResponse(String nodeId,
+        NodeResponse(String nodeId,
                             int totalShards,
                             List<ShardOperationResult> results,
                             List<BroadcastShardOperationFailedException> exceptions) {
@@ -566,18 +556,9 @@ public void readFrom(StreamInput in) throws IOException {
             super.readFrom(in);
             nodeId = in.readString();
             totalShards = in.readVInt();
-            int resultsSize = in.readVInt();
-            results = new ArrayList<>(resultsSize);
-            for (; resultsSize > 0; resultsSize--) {
-                final ShardOperationResult result = in.readBoolean() ? readShardResult(in) : null;
-                results.add(result);
-            }
+            results = in.readList((stream) -> stream.readBoolean() ? readShardResult(stream) : null);
             if (in.readBoolean()) {
-                int failureShards = in.readVInt();
-                exceptions = new ArrayList<>(failureShards);
-                for (int i = 0; i < failureShards; i++) {
-                    exceptions.add(new BroadcastShardOperationFailedException(in));
-                }
+                exceptions = in.readList(BroadcastShardOperationFailedException::new);
             } else {
                 exceptions = null;
             }
@@ -594,11 +575,7 @@ public void writeTo(StreamOutput out) throws IOException {
             }
             out.writeBoolean(exceptions != null);
             if (exceptions != null) {
-                int failureShards = exceptions.size();
-                out.writeVInt(failureShards);
-                for (int i = 0; i < failureShards; i++) {
-                    exceptions.get(i).writeTo(out);
-                }
+                out.writeList(exceptions);
             }
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequest.java b/core/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequest.java
index e3f32543bf2c3..54c1ee6675390 100644
--- a/core/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequest.java
@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.action.support.master;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ack.AckedRequest;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -73,19 +74,45 @@ public final TimeValue timeout() {
     /**
      * Reads the timeout value
      */
+    @Deprecated
     protected void readTimeout(StreamInput in) throws IOException {
-        timeout = new TimeValue(in);
+        // in older ES versions, we would explicitly call this method in subclasses
+        // now we properly serialize the timeout value as part of the readFrom method
+        if (in.getVersion().before(Version.V_5_6_0)) {
+            timeout = new TimeValue(in);
+        }
     }
 
     /**
      * writes the timeout value
      */
+    @Deprecated
     protected void writeTimeout(StreamOutput out) throws IOException {
-        timeout.writeTo(out);
+        // in older ES versions, we would explicitly call this method in subclasses
+        // now we properly serialize the timeout value as part of the writeTo method
+        if (out.getVersion().before(Version.V_5_6_0)) {
+            timeout.writeTo(out);
+        }
     }
 
     @Override
     public TimeValue ackTimeout() {
         return timeout;
     }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        if (in.getVersion().onOrAfter(Version.V_5_6_0)) {
+            timeout = new TimeValue(in);
+        }
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        if (out.getVersion().onOrAfter(Version.V_5_6_0)) {
+            timeout.writeTo(out);
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/support/master/MasterNodeRequest.java b/core/src/main/java/org/elasticsearch/action/support/master/MasterNodeRequest.java
index efbcadf445ff4..6f2ce6c4ef96d 100644
--- a/core/src/main/java/org/elasticsearch/action/support/master/MasterNodeRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/support/master/MasterNodeRequest.java
@@ -29,7 +29,7 @@
 /**
  * A based request for master based operation.
  */
-public abstract class MasterNodeRequest<Request extends MasterNodeRequest<Request>> extends ActionRequest<Request> {
+public abstract class MasterNodeRequest<Request extends MasterNodeRequest<Request>> extends ActionRequest {
 
     public static final TimeValue DEFAULT_MASTER_NODE_TIMEOUT = TimeValue.timeValueSeconds(30);
 
diff --git a/core/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java b/core/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java
index a664c325a4b1d..f2bc4da423dea 100644
--- a/core/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java
@@ -45,6 +45,7 @@
 import org.elasticsearch.transport.TransportException;
 import org.elasticsearch.transport.TransportService;
 
+import java.util.function.Predicate;
 import java.util.function.Supplier;
 
 /**
@@ -110,14 +111,6 @@ class AsyncSingleAction {
         private volatile ClusterStateObserver observer;
         private final Task task;
 
-        private final ClusterStateObserver.ChangePredicate retryableOrNoBlockPredicate = new ClusterStateObserver.ValidationPredicate() {
-            @Override
-            protected boolean validate(ClusterState newState) {
-                ClusterBlockException blockException = checkBlock(request, newState);
-                return (blockException == null || !blockException.retryable());
-            }
-        };
-
         AsyncSingleAction(Task task, Request request, ActionListener<Response> listener) {
             this.task = task;
             this.request = request;
@@ -128,12 +121,13 @@ protected boolean validate(ClusterState newState) {
         }
 
         public void start() {
-            this.observer = new ClusterStateObserver(clusterService, request.masterNodeTimeout(), logger, threadPool.getThreadContext());
-            doStart();
+            ClusterState state = clusterService.state();
+            this.observer = new ClusterStateObserver(state, clusterService, request.masterNodeTimeout(), logger, threadPool.getThreadContext());
+            doStart(state);
         }
 
-        protected void doStart() {
-            final ClusterState clusterState = observer.observedState();
+        protected void doStart(ClusterState clusterState) {
+            final Predicate<ClusterState> masterChangePredicate = MasterNodeChangePredicate.build(clusterState);
             final DiscoveryNodes nodes = clusterState.nodes();
             if (nodes.isLocalNodeElectedMaster() || localExecute(request)) {
                 // check for block, if blocked, retry, else, execute locally
@@ -143,7 +137,10 @@ protected void doStart() {
                         listener.onFailure(blockException);
                     } else {
                         logger.trace("can't execute due to a cluster block, retrying", blockException);
-                        retry(blockException, retryableOrNoBlockPredicate);
+                        retry(blockException, newState -> {
+                            ClusterBlockException newException = checkBlock(request, newState);
+                            return (newException == null || !newException.retryable());
+                        });
                     }
                 } else {
                     ActionListener<Response> delegate = new ActionListener<Response>() {
@@ -157,26 +154,24 @@ public void onFailure(Exception t) {
                             if (t instanceof Discovery.FailedToCommitClusterStateException
                                     || (t instanceof NotMasterException)) {
                                 logger.debug((org.apache.logging.log4j.util.Supplier<?>) () -> new ParameterizedMessage("master could not publish cluster state or stepped down before publishing action [{}], scheduling a retry", actionName), t);
-                                retry(t, MasterNodeChangePredicate.INSTANCE);
+                                retry(t, masterChangePredicate);
                             } else {
                                 listener.onFailure(t);
                             }
                         }
                     };
-                    taskManager.registerChildTask(task, nodes.getLocalNodeId());
                     threadPool.executor(executor).execute(new ActionRunnable(delegate) {
                         @Override
                         protected void doRun() throws Exception {
-                            masterOperation(task, request, clusterService.state(), delegate);
+                            masterOperation(task, request, clusterState, delegate);
                         }
                     });
                 }
             } else {
                 if (nodes.getMasterNode() == null) {
                     logger.debug("no known master node, scheduling a retry");
-                    retry(null, MasterNodeChangePredicate.INSTANCE);
+                    retry(null, masterChangePredicate);
                 } else {
-                    taskManager.registerChildTask(task, nodes.getMasterNode().getId());
                     transportService.sendRequest(nodes.getMasterNode(), actionName, request, new ActionListenerResponseHandler<Response>(listener, TransportMasterNodeAction.this::newResponse) {
                         @Override
                         public void handleException(final TransportException exp) {
@@ -185,7 +180,7 @@ public void handleException(final TransportException exp) {
                                 // we want to retry here a bit to see if a new master is elected
                                 logger.debug("connection exception while trying to forward request with action name [{}] to master node [{}], scheduling a retry. Error: [{}]",
                                         actionName, nodes.getMasterNode(), exp.getDetailedMessage());
-                                retry(cause, MasterNodeChangePredicate.INSTANCE);
+                                retry(cause, masterChangePredicate);
                             } else {
                                 listener.onFailure(exp);
                             }
@@ -195,12 +190,12 @@ public void handleException(final TransportException exp) {
             }
         }
 
-        private void retry(final Throwable failure, final ClusterStateObserver.ChangePredicate changePredicate) {
+        private void retry(final Throwable failure, final Predicate<ClusterState> statePredicate) {
             observer.waitForNextChange(
                 new ClusterStateObserver.Listener() {
                     @Override
                     public void onNewClusterState(ClusterState state) {
-                        doStart();
+                        doStart(state);
                     }
 
                     @Override
@@ -213,7 +208,7 @@ public void onTimeout(TimeValue timeout) {
                         logger.debug((org.apache.logging.log4j.util.Supplier<?>) () -> new ParameterizedMessage("timed out while retrying [{}] after failure (timeout [{}])", actionName, timeout), failure);
                         listener.onFailure(new MasterNotDiscoveredException(failure));
                     }
-                }, changePredicate
+                }, statePredicate
             );
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/action/support/master/info/TransportClusterInfoAction.java b/core/src/main/java/org/elasticsearch/action/support/master/info/TransportClusterInfoAction.java
index 66b9fce5d711d..8894704c2ea02 100644
--- a/core/src/main/java/org/elasticsearch/action/support/master/info/TransportClusterInfoAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/master/info/TransportClusterInfoAction.java
@@ -54,5 +54,5 @@ protected final void masterOperation(final Request request, final ClusterState s
         doMasterOperation(request, concreteIndices, state, listener);
     }
 
-    protected abstract void doMasterOperation(Request request, String[] concreteIndices, ClusterState state, final ActionListener<Response> listener);
+    protected abstract void doMasterOperation(Request request, String[] concreteIndices, ClusterState state, ActionListener<Response> listener);
 }
diff --git a/core/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesRequest.java b/core/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesRequest.java
index 663537f25da59..b6760c1cfcba9 100644
--- a/core/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesRequest.java
@@ -32,7 +32,7 @@
 /**
  *
  */
-public abstract class BaseNodesRequest<Request extends BaseNodesRequest<Request>> extends ActionRequest<Request> {
+public abstract class BaseNodesRequest<Request extends BaseNodesRequest<Request>> extends ActionRequest {
 
     /**
      * the list of nodesIds that will be used to resolve this request and {@link #concreteNodes}
diff --git a/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java b/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java
index 3582f5f5aaf8f..4583e47bc1db7 100644
--- a/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java
@@ -109,14 +109,10 @@ protected NodesResponse newResponse(NodesRequest request, AtomicReferenceArray n
         for (int i = 0; i < nodesResponses.length(); ++i) {
             Object response = nodesResponses.get(i);
 
-            if (nodeResponseClass.isInstance(response)) {
-                responses.add(nodeResponseClass.cast(response));
-            } else if (response instanceof FailedNodeException) {
+            if (response instanceof FailedNodeException) {
                 failures.add((FailedNodeException)response);
             } else {
-                logger.warn("ignoring unexpected response [{}] of type [{}], expected [{}] or [{}]",
-                            response, response != null ? response.getClass().getName() : null,
-                            nodeResponseClass.getSimpleName(), FailedNodeException.class.getSimpleName());
+                responses.add(nodeResponseClass.cast(response));
             }
         }
 
@@ -144,8 +140,6 @@ protected NodeResponse nodeOperation(NodeRequest request, Task task) {
         return nodeOperation(request);
     }
 
-    protected abstract boolean accumulateExceptions();
-
     /**
      * resolve node ids to concrete nodes of the incoming request
      **/
@@ -198,7 +192,6 @@ void start() {
                         TransportRequest nodeRequest = newNodeRequest(nodeId, request);
                         if (task != null) {
                             nodeRequest.setParentTask(clusterService.localNode().getId(), task.getId());
-                            taskManager.registerChildTask(task, node.getId());
                         }
 
                         transportService.sendRequest(node, transportNodeAction, nodeRequest, builder.build(),
@@ -243,9 +236,7 @@ private void onFailure(int idx, String nodeId, Throwable t) {
                     (org.apache.logging.log4j.util.Supplier<?>)
                         () -> new ParameterizedMessage("failed to execute on node [{}]", nodeId), t);
             }
-            if (accumulateExceptions()) {
-                responses.set(idx, new FailedNodeException(nodeId, "Failed node [" + nodeId + "]", t));
-            }
+            responses.set(idx, new FailedNodeException(nodeId, "Failed node [" + nodeId + "]", t));
             if (counter.incrementAndGet() == responses.length()) {
                 finishHim();
             }
diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java
index d541ef6a35c11..daa180bb49de8 100644
--- a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java
+++ b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java
@@ -32,9 +32,9 @@
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.util.set.Sets;
-import org.elasticsearch.index.engine.VersionConflictEngineException;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.transport.TransportResponse;
@@ -112,21 +112,22 @@ public void execute() throws Exception {
         pendingActions.incrementAndGet();
         primaryResult = primary.perform(request);
         final ReplicaRequest replicaRequest = primaryResult.replicaRequest();
-        assert replicaRequest.primaryTerm() > 0 : "replicaRequest doesn't have a primary term";
-        if (logger.isTraceEnabled()) {
-            logger.trace("[{}] op [{}] completed on primary for request [{}]", primaryId, opType, request);
-        }
+        if (replicaRequest != null) {
+            if (logger.isTraceEnabled()) {
+                logger.trace("[{}] op [{}] completed on primary for request [{}]", primaryId, opType, request);
+            }
 
-        // we have to get a new state after successfully indexing into the primary in order to honour recovery semantics.
-        // we have to make sure that every operation indexed into the primary after recovery start will also be replicated
-        // to the recovery target. If we use an old cluster state, we may miss a relocation that has started since then.
-        ClusterState clusterState = clusterStateSupplier.get();
-        final List<ShardRouting> shards = getShards(primaryId, clusterState);
-        Set<String> inSyncAllocationIds = getInSyncAllocationIds(primaryId, clusterState);
+            // we have to get a new state after successfully indexing into the primary in order to honour recovery semantics.
+            // we have to make sure that every operation indexed into the primary after recovery start will also be replicated
+            // to the recovery target. If we use an old cluster state, we may miss a relocation that has started since then.
+            ClusterState clusterState = clusterStateSupplier.get();
+            final List<ShardRouting> shards = getShards(primaryId, clusterState);
+            Set<String> inSyncAllocationIds = getInSyncAllocationIds(primaryId, clusterState);
 
-        markUnavailableShardsAsStale(replicaRequest, inSyncAllocationIds, shards);
+            markUnavailableShardsAsStale(replicaRequest, inSyncAllocationIds, shards);
 
-        performOnReplicas(replicaRequest, shards);
+            performOnReplicas(replicaRequest, shards);
+        }
 
         successfulShards.incrementAndGet();
         decPendingAndFinishIfNeeded();
@@ -144,7 +145,7 @@ private void markUnavailableShardsAsStale(ReplicaRequest replicaRequest, Set<Str
             for (String allocationId : Sets.difference(inSyncAllocationIds, availableAllocationIds)) {
                 // mark copy as stale
                 pendingActions.incrementAndGet();
-                replicasProxy.markShardCopyAsStale(replicaRequest.shardId(), allocationId, replicaRequest.primaryTerm(),
+                replicasProxy.markShardCopyAsStale(replicaRequest.shardId(), allocationId,
                     ReplicationOperation.this::decPendingAndFinishIfNeeded,
                     ReplicationOperation.this::onPrimaryDemoted,
                     throwable -> decPendingAndFinishIfNeeded()
@@ -198,7 +199,7 @@ public void onFailure(Exception replicaException) {
                         shard,
                         replicaRequest),
                     replicaException);
-                if (ignoreReplicaException(replicaException)) {
+                if (TransportActions.isShardNotAvailableException(replicaException)) {
                     decPendingAndFinishIfNeeded();
                 } else {
                     RestStatus restStatus = ExceptionsHelper.status(replicaException);
@@ -208,7 +209,7 @@ public void onFailure(Exception replicaException) {
                     logger.warn(
                         (org.apache.logging.log4j.util.Supplier<?>)
                             () -> new ParameterizedMessage("[{}] {}", shard.shardId(), message), replicaException);
-                    replicasProxy.failShard(shard, replicaRequest.primaryTerm(), message, replicaException,
+                    replicasProxy.failShard(shard, message, replicaException,
                         ReplicationOperation.this::decPendingAndFinishIfNeeded,
                         ReplicationOperation.this::onPrimaryDemoted,
                         throwable -> decPendingAndFinishIfNeeded()
@@ -310,30 +311,6 @@ private void finishAsFailed(Exception exception) {
         }
     }
 
-
-    /**
-     * Should an exception be ignored when the operation is performed on the replica.
-     */
-    public static boolean ignoreReplicaException(Exception e) {
-        if (TransportActions.isShardNotAvailableException(e)) {
-            return true;
-        }
-        // on version conflict or document missing, it means
-        // that a new change has crept into the replica, and it's fine
-        if (isConflictException(e)) {
-            return true;
-        }
-        return false;
-    }
-
-    public static boolean isConflictException(Throwable t) {
-        final Throwable cause = ExceptionsHelper.unwrapCause(t);
-        // on version conflict or document missing, it means
-        // that a new change has crept into the replica, and it's fine
-        return cause instanceof VersionConflictEngineException;
-    }
-
-
     public interface Primary<
                 Request extends ReplicationRequest<Request>,
                 ReplicaRequest extends ReplicationRequest<ReplicaRequest>,
@@ -359,7 +336,6 @@ public interface Primary<
          * @return the request to send to the repicas
          */
         PrimaryResultT perform(Request request) throws Exception;
-
     }
 
     public interface Replicas<ReplicaRequest extends ReplicationRequest<ReplicaRequest>> {
@@ -371,12 +347,12 @@ public interface Replicas<ReplicaRequest extends ReplicationRequest<ReplicaReque
          * @param replicaRequest operation to peform
          * @param listener       a callback to call once the operation has been complicated, either successfully or with an error.
          */
-        void performOn(ShardRouting replica, ReplicaRequest replicaRequest, ActionListener<TransportResponse.Empty> listener);
+        void performOn(ShardRouting replica, ReplicaRequest replicaRequest,
+                       ActionListener<TransportResponse.Empty> listener);
 
         /**
          * Fail the specified shard, removing it from the current set of active shards
          * @param replica          shard to fail
-         * @param primaryTerm      the primary term of the primary shard when requesting the failure
          * @param message          a (short) description of the reason
          * @param exception        the original exception which caused the ReplicationOperation to request the shard to be failed
          * @param onSuccess        a callback to call when the shard has been successfully removed from the active set.
@@ -384,7 +360,7 @@ public interface Replicas<ReplicaRequest extends ReplicationRequest<ReplicaReque
 *                         by the master.
          * @param onIgnoredFailure a callback to call when failing a shard has failed, but it that failure can be safely ignored and the
          */
-        void failShard(ShardRouting replica, long primaryTerm, String message, Exception exception, Runnable onSuccess,
+        void failShard(ShardRouting replica, String message, Exception exception, Runnable onSuccess,
                        Consumer<Exception> onPrimaryDemoted, Consumer<Exception> onIgnoredFailure);
 
         /**
@@ -392,13 +368,12 @@ void failShard(ShardRouting replica, long primaryTerm, String message, Exception
          *
          * @param shardId          shard id
          * @param allocationId     allocation id to remove from the set of in-sync allocation ids
-         * @param primaryTerm      the primary term of the primary shard when requesting the failure
          * @param onSuccess        a callback to call when the allocation id has been successfully removed from the in-sync set.
          * @param onPrimaryDemoted a callback to call when the request failed because the current primary was already demoted
          *                         by the master.
          * @param onIgnoredFailure a callback to call when the request failed, but the failure can be safely ignored.
          */
-        void markShardCopyAsStale(ShardId shardId, String allocationId, long primaryTerm, Runnable onSuccess,
+        void markShardCopyAsStale(ShardId shardId, String allocationId, Runnable onSuccess,
                                   Consumer<Exception> onPrimaryDemoted, Consumer<Exception> onIgnoredFailure);
     }
 
@@ -419,7 +394,11 @@ public RetryOnPrimaryException(StreamInput in) throws IOException {
 
     public interface PrimaryResult<R extends ReplicationRequest<R>> {
 
-        R replicaRequest();
+        /**
+         * @return null if no operation needs to be sent to a replica
+         * (for example when the operation failed on the primary due to a parsing exception)
+         */
+        @Nullable R replicaRequest();
 
         void setShardInfo(ReplicationResponse.ShardInfo shardInfo);
     }
diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequest.java b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequest.java
index 596d2581a7953..d50909b62c564 100644
--- a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequest.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.action.support.replication;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.IndicesRequest;
@@ -43,7 +44,7 @@
  * Requests that are run on a particular replica, first on the primary and then on the replicas like {@link IndexRequest} or
  * {@link TransportShardRefreshAction}.
  */
-public abstract class ReplicationRequest<Request extends ReplicationRequest<Request>> extends ActionRequest<Request>
+public abstract class ReplicationRequest<Request extends ReplicationRequest<Request>> extends ActionRequest
         implements IndicesRequest {
 
     public static final TimeValue DEFAULT_TIMEOUT = new TimeValue(1, TimeUnit.MINUTES);
@@ -55,7 +56,7 @@ public abstract class ReplicationRequest<Request extends ReplicationRequest<Requ
      */
     protected ShardId shardId;
 
-    long primaryTerm;
+    private long primaryTerm;
 
     protected TimeValue timeout = DEFAULT_TIMEOUT;
     protected String index;
@@ -170,13 +171,25 @@ long routedBasedOnClusterVersion() {
         return routedBasedOnClusterVersion;
     }
 
-    /** returns the primary term active at the time the operation was performed on the primary shard */
-    public long primaryTerm() {
+    /**
+     * Returns the primary term active at the time the operation was performed on the primary shard.
+     * This method should not be used anymore, as its functionality has moved to ConcreteShardRequest.
+     * It is purely here for interoperability with ES v5.5 and below.
+     */
+    @Deprecated
+    long primaryTerm() {
         return primaryTerm;
     }
 
-    /** marks the primary term in which the operation was performed */
-    public void primaryTerm(long term) {
+    /**
+     * marks the primary term in which the operation was performed
+     * This method should not be used anymore, as its functionality has moved to ConcreteShardRequest.
+     * It is purely here for interoperability with ES v5.5 and below.
+     */
+    @Deprecated
+    void primaryTerm(long term) {
+        assert term == 0L || primaryTerm == 0L || primaryTerm == term :
+            "primary term should only be set once on a request, was " + primaryTerm + " now set to "  + term;
         primaryTerm = term;
     }
 
@@ -201,7 +214,11 @@ public void readFrom(StreamInput in) throws IOException {
         timeout = new TimeValue(in);
         index = in.readString();
         routedBasedOnClusterVersion = in.readVLong();
-        primaryTerm = in.readVLong();
+        if (in.getVersion().before(Version.V_5_6_0)) {
+            primaryTerm = in.readVLong();
+        } else {
+            primaryTerm = 0L;
+        }
     }
 
     @Override
@@ -217,7 +234,9 @@ public void writeTo(StreamOutput out) throws IOException {
         timeout.writeTo(out);
         out.writeString(index);
         out.writeVLong(routedBasedOnClusterVersion);
-        out.writeVLong(primaryTerm);
+        if (out.getVersion().before(Version.V_5_6_0)) {
+            out.writeVLong(primaryTerm);
+        }
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationResponse.java b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationResponse.java
index e6ce0a5aad404..b8a5f3782bd64 100644
--- a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationResponse.java
@@ -23,17 +23,23 @@
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ShardOperationFailedException;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
 
 /**
  * Base class for write action responses.
@@ -64,7 +70,12 @@ public void setShardInfo(ShardInfo shardInfo) {
         this.shardInfo = shardInfo;
     }
 
-    public static class ShardInfo implements Streamable, ToXContent {
+    public static class ShardInfo implements Streamable, ToXContentObject {
+
+        private static final String TOTAL = "total";
+        private static final String SUCCESSFUL = "successful";
+        private static final String FAILED = "failed";
+        private static final String FAILURES = "failures";
 
         private int total;
         private int successful;
@@ -145,12 +156,12 @@ public void writeTo(StreamOutput out) throws IOException {
 
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startObject(Fields._SHARDS);
-            builder.field(Fields.TOTAL, total);
-            builder.field(Fields.SUCCESSFUL, successful);
-            builder.field(Fields.FAILED, getFailed());
+            builder.startObject();
+            builder.field(TOTAL, total);
+            builder.field(SUCCESSFUL, successful);
+            builder.field(FAILED, getFailed());
             if (failures.length > 0) {
-                builder.startArray(Fields.FAILURES);
+                builder.startArray(FAILURES);
                 for (Failure failure : failures) {
                     failure.toXContent(builder, params);
                 }
@@ -160,9 +171,51 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             return builder;
         }
 
+        public static ShardInfo fromXContent(XContentParser parser) throws IOException {
+            XContentParser.Token token = parser.currentToken();
+            ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser::getTokenLocation);
+
+            int total = 0, successful = 0;
+            List<Failure> failuresList = null;
+            String currentFieldName = null;
+            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                if (token == XContentParser.Token.FIELD_NAME) {
+                    currentFieldName = parser.currentName();
+                } else if (token.isValue()) {
+                    if (TOTAL.equals(currentFieldName)) {
+                        total = parser.intValue();
+                    } else if (SUCCESSFUL.equals(currentFieldName)) {
+                        successful = parser.intValue();
+                    } else {
+                        parser.skipChildren();
+                    }
+                } else if (token == XContentParser.Token.START_ARRAY) {
+                    if (FAILURES.equals(currentFieldName)) {
+                        failuresList = new ArrayList<>();
+                        while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                            failuresList.add(Failure.fromXContent(parser));
+                        }
+                    } else {
+                        parser.skipChildren(); // skip potential inner arrays for forward compatibility
+                    }
+                } else if (token == XContentParser.Token.START_OBJECT) {
+                    parser.skipChildren(); // skip potential inner arrays for forward compatibility
+                }
+            }
+            Failure[] failures = EMPTY;
+            if (failuresList != null) {
+                failures = failuresList.toArray(new Failure[failuresList.size()]);
+            }
+            return new ShardInfo(total, successful, failures);
+        }
+
         @Override
         public String toString() {
-            return Strings.toString(this);
+            return "ShardInfo{" +
+                "total=" + total +
+                ", successful=" + successful +
+                ", failures=" + Arrays.toString(failures) +
+                '}';
         }
 
         public static ShardInfo readShardInfo(StreamInput in) throws IOException {
@@ -171,7 +224,14 @@ public static ShardInfo readShardInfo(StreamInput in) throws IOException {
             return shardInfo;
         }
 
-        public static class Failure implements ShardOperationFailedException, ToXContent {
+        public static class Failure implements ShardOperationFailedException, ToXContentObject {
+
+            private static final String _INDEX = "_index";
+            private static final String _SHARD = "_shard";
+            private static final String _NODE = "_node";
+            private static final String REASON = "reason";
+            private static final String STATUS = "status";
+            private static final String PRIMARY = "primary";
 
             private ShardId shardId;
             private String nodeId;
@@ -268,39 +328,57 @@ public void writeTo(StreamOutput out) throws IOException {
             @Override
             public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
                 builder.startObject();
-                builder.field(Fields._INDEX, shardId.getIndexName());
-                builder.field(Fields._SHARD, shardId.id());
-                builder.field(Fields._NODE, nodeId);
-                builder.field(Fields.REASON);
+                builder.field(_INDEX, shardId.getIndexName());
+                builder.field(_SHARD, shardId.id());
+                builder.field(_NODE, nodeId);
+                builder.field(REASON);
                 builder.startObject();
-                ElasticsearchException.toXContent(builder, params, cause);
+                ElasticsearchException.generateThrowableXContent(builder, params, cause);
                 builder.endObject();
-                builder.field(Fields.STATUS, status);
-                builder.field(Fields.PRIMARY, primary);
+                builder.field(STATUS, status);
+                builder.field(PRIMARY, primary);
                 builder.endObject();
                 return builder;
             }
 
-            private static class Fields {
-
-                private static final String _INDEX = "_index";
-                private static final String _SHARD = "_shard";
-                private static final String _NODE = "_node";
-                private static final String REASON = "reason";
-                private static final String STATUS = "status";
-                private static final String PRIMARY = "primary";
-
+            public static Failure fromXContent(XContentParser parser) throws IOException {
+                XContentParser.Token token = parser.currentToken();
+                ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser::getTokenLocation);
+
+                String shardIndex = null, nodeId = null;
+                int shardId = -1;
+                boolean primary = false;
+                RestStatus status = null;
+                ElasticsearchException reason = null;
+
+                String currentFieldName = null;
+                while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                    if (token == XContentParser.Token.FIELD_NAME) {
+                        currentFieldName = parser.currentName();
+                    } else if (token.isValue()) {
+                        if (_INDEX.equals(currentFieldName)) {
+                            shardIndex = parser.text();
+                        } else if (_SHARD.equals(currentFieldName)) {
+                            shardId = parser.intValue();
+                        } else if (_NODE.equals(currentFieldName)) {
+                            nodeId = parser.text();
+                        } else if (STATUS.equals(currentFieldName)) {
+                            status = RestStatus.valueOf(parser.text());
+                        } else if (PRIMARY.equals(currentFieldName)) {
+                            primary = parser.booleanValue();
+                        }
+                    } else if (token == XContentParser.Token.START_OBJECT) {
+                        if (REASON.equals(currentFieldName)) {
+                            reason = ElasticsearchException.fromXContent(parser);
+                        } else {
+                            parser.skipChildren(); // skip potential inner objects for forward compatibility
+                        }
+                    } else if (token == XContentParser.Token.START_ARRAY) {
+                        parser.skipChildren(); // skip potential inner arrays for forward compatibility
+                    }
+                }
+                return new Failure(new ShardId(shardIndex, IndexMetaData.INDEX_UUID_NA_VALUE, shardId), nodeId, reason, status, primary);
             }
         }
-
-        private static class Fields {
-
-            private static final String _SHARDS = "_shards";
-            private static final String TOTAL = "total";
-            private static final String SUCCESSFUL = "successful";
-            private static final String FAILED = "failed";
-            private static final String FAILURES = "failures";
-
-        }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/TransportBroadcastReplicationAction.java b/core/src/main/java/org/elasticsearch/action/support/replication/TransportBroadcastReplicationAction.java
index e33d10eaa25ad..8193cf77cebef 100644
--- a/core/src/main/java/org/elasticsearch/action/support/replication/TransportBroadcastReplicationAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/replication/TransportBroadcastReplicationAction.java
@@ -119,7 +119,6 @@ public void onFailure(Exception e) {
     protected void shardExecute(Task task, Request request, ShardId shardId, ActionListener<ShardResponse> shardActionListener) {
         ShardRequest shardRequest = newShardRequest(request, shardId);
         shardRequest.setParentTask(clusterService.localNode().getId(), task.getId());
-        taskManager.registerChildTask(task, clusterService.localNode().getId());
         replicatedBroadcastShardAction.execute(shardRequest, shardActionListener);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java b/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java
index 9587b4e6b2cc5..65471e366fea2 100644
--- a/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java
@@ -21,6 +21,7 @@
 
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionListenerResponseHandler;
 import org.elasticsearch.action.UnavailableShardsException;
@@ -42,6 +43,7 @@
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lease.Releasable;
@@ -49,13 +51,13 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.IndexShardState;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.ShardNotFoundException;
+import org.elasticsearch.indices.IndexClosedException;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.node.NodeClosedException;
 import org.elasticsearch.tasks.Task;
@@ -93,17 +95,16 @@ public abstract class TransportReplicationAction<
             Response extends ReplicationResponse
         > extends TransportAction<Request, Response> {
 
-    protected final TransportService transportService;
+    private final TransportService transportService;
     protected final ClusterService clusterService;
-    protected final IndicesService indicesService;
+    private final IndicesService indicesService;
     private final ShardStateAction shardStateAction;
     private final TransportRequestOptions transportOptions;
     private final String executor;
 
     // package private for testing
-    final String transportReplicaAction;
-    final String transportPrimaryAction;
-    private final ReplicasProxy replicasProxy;
+    private final String transportReplicaAction;
+    private final String transportPrimaryAction;
 
     protected TransportReplicationAction(Settings settings, String actionName, TransportService transportService,
                                          ClusterService clusterService, IndicesService indicesService,
@@ -130,8 +131,6 @@ protected TransportReplicationAction(Settings settings, String actionName, Trans
             new ReplicaOperationTransportHandler());
 
         this.transportOptions = transportOptions();
-
-        this.replicasProxy = new ReplicasProxy();
     }
 
     @Override
@@ -167,27 +166,34 @@ protected void resolveRequest(MetaData metaData, IndexMetaData indexMetaData, Re
      * Primary operation on node with primary copy.
      *
      * @param shardRequest the request to the primary shard
+     * @param primary      the primary shard to perform the operation on
      */
-    protected abstract PrimaryResult shardOperationOnPrimary(Request shardRequest) throws Exception;
+    protected abstract PrimaryResult<ReplicaRequest, Response> shardOperationOnPrimary(
+            Request shardRequest, IndexShard primary) throws Exception;
 
     /**
      * Synchronous replica operation on nodes with replica copies. This is done under the lock form
-     * {@link #acquireReplicaOperationLock(ShardId, long, String, ActionListener)}.
+     * {@link IndexShard#acquireReplicaOperationLock(long, ActionListener, String)}
+     *
+     * @param shardRequest the request to the replica shard
+     * @param replica      the replica shard to perform the operation on
      */
-    protected abstract ReplicaResult shardOperationOnReplica(ReplicaRequest shardRequest);
+    protected abstract ReplicaResult shardOperationOnReplica(ReplicaRequest shardRequest, IndexShard replica) throws Exception;
 
     /**
-     * Cluster level block to check before request execution
+     * Cluster level block to check before request execution. Returning null means that no blocks need to be checked.
      */
+    @Nullable
     protected ClusterBlockLevel globalBlockLevel() {
-        return ClusterBlockLevel.WRITE;
+        return null;
     }
 
     /**
-     * Index level block to check before request execution
+     * Index level block to check before request execution. Returning null means that no blocks need to be checked.
      */
+    @Nullable
     protected ClusterBlockLevel indexBlockLevel() {
-        return ClusterBlockLevel.WRITE;
+        return null;
     }
 
     /**
@@ -203,7 +209,7 @@ protected TransportRequestOptions transportOptions() {
 
     protected boolean retryPrimaryException(final Throwable e) {
         return e.getClass() == ReplicationOperation.RetryOnPrimaryException.class
-            || TransportActions.isShardNotAvailableException(e);
+                || TransportActions.isShardNotAvailableException(e);
     }
 
     class OperationTransportHandler implements TransportRequestHandler<Request> {
@@ -247,28 +253,42 @@ public void messageReceived(final ConcreteShardRequest<Request> request, final T
 
         @Override
         public void messageReceived(ConcreteShardRequest<Request> request, TransportChannel channel, Task task) {
-            new AsyncPrimaryAction(request.request, request.targetAllocationID, channel, (ReplicationTask) task).run();
+            // incoming primary term can be 0 if request is coming from a < 5.6 node (relocated primary)
+            // just use as speculative term the one from the current cluster state, it's validated against the actual primary term
+            // within acquirePrimaryShardReference
+            final long primaryTerm;
+            if (request.primaryTerm > 0L) {
+                primaryTerm = request.primaryTerm;
+            } else {
+                ShardId shardId = request.request.shardId();
+                primaryTerm = clusterService.state().metaData().getIndexSafe(shardId.getIndex()).primaryTerm(shardId.id());
+            }
+            new AsyncPrimaryAction(request.request, request.targetAllocationID, primaryTerm, channel, (ReplicationTask) task).run();
         }
     }
 
     class AsyncPrimaryAction extends AbstractRunnable implements ActionListener<PrimaryShardReference> {
 
         private final Request request;
-        /** targetAllocationID of the shard this request is meant for */
+        // targetAllocationID of the shard this request is meant for
         private final String targetAllocationID;
+        // primary term of the shard this request is meant for
+        private final long primaryTerm;
         private final TransportChannel channel;
         private final ReplicationTask replicationTask;
 
-        AsyncPrimaryAction(Request request, String targetAllocationID, TransportChannel channel, ReplicationTask replicationTask) {
+        AsyncPrimaryAction(Request request, String targetAllocationID, long primaryTerm, TransportChannel channel,
+                           ReplicationTask replicationTask) {
             this.request = request;
             this.targetAllocationID = targetAllocationID;
+            this.primaryTerm = primaryTerm;
             this.channel = channel;
             this.replicationTask = replicationTask;
         }
 
         @Override
         protected void doRun() throws Exception {
-            acquirePrimaryShardReference(request.shardId(), targetAllocationID, this);
+            acquirePrimaryShardReference(request.shardId(), targetAllocationID, primaryTerm, this);
         }
 
         @Override
@@ -283,8 +303,21 @@ public void onResponse(PrimaryShardReference primaryShardReference) {
                     final ShardRouting primary = primaryShardReference.routingEntry();
                     assert primary.relocating() : "indexShard is marked as relocated but routing isn't" + primary;
                     DiscoveryNode relocatingNode = clusterService.state().nodes().get(primary.relocatingNodeId());
+                    if (relocatingNode != null && relocatingNode.getVersion().major > Version.CURRENT.major) {
+                        // ES 6.x requires a primary context hand-off during primary relocation which is not implemented on ES 5.x,
+                        // otherwise it might not be aware of a replica that finished recovery and became activated on the master before
+                        // the new primary became in charge of replicating operations, as the cluster state with that in-sync information
+                        // might not be applied yet on the primary relocation target before it would be in charge of replicating operations.
+                        // This would mean that the new primary could advance the global checkpoint too quickly, not taking into account
+                        // the newly in-sync replica.
+                        // ES 6.x detects that the primary is relocating from a 5.x node, and activates the primary mode of the global
+                        // checkpoint tracker only after activation of the relocation target, which means, however, that requests cannot
+                        // be handled as long as the relocation target shard has not been activated.
+                        throw new ReplicationOperation.RetryOnPrimaryException(request.shardId(),
+                            "waiting for 6.x primary to be activated");
+                    }
                     transportService.sendRequest(relocatingNode, transportPrimaryAction,
-                        new ConcreteShardRequest<>(request, primary.allocationId().getRelocationId()),
+                        new ConcreteShardRequest<>(request, primary.allocationId().getRelocationId(), primaryTerm),
                         transportOptions,
                         new TransportChannelResponseHandler<Response>(logger, channel, "rerouting indexing to target primary " + primary,
                             TransportReplicationAction.this::newResponseInstance) {
@@ -304,19 +337,12 @@ public void handleException(TransportException exp) {
                 } else {
                     setPhase(replicationTask, "primary");
                     final IndexMetaData indexMetaData = clusterService.state().getMetaData().index(request.shardId().getIndex());
-                    final boolean executeOnReplicas = (indexMetaData == null) || shouldExecuteReplication(indexMetaData.getSettings());
+                    final boolean executeOnReplicas = (indexMetaData == null) || shouldExecuteReplication(indexMetaData);
                     final ActionListener<Response> listener = createResponseListener(primaryShardReference);
-                    createReplicatedOperation(request, new ActionListener<PrimaryResult>() {
-                        @Override
-                        public void onResponse(PrimaryResult result) {
-                            result.respond(listener);
-                        }
-
-                        @Override
-                        public void onFailure(Exception e) {
-                            listener.onFailure(e);
-                        }
-                    }, primaryShardReference, executeOnReplicas).execute();
+                    createReplicatedOperation(request,
+                            ActionListener.wrap(result -> result.respond(listener), listener::onFailure),
+                            primaryShardReference, executeOnReplicas)
+                            .execute();
                 }
             } catch (Exception e) {
                 Releasables.closeWhileHandlingException(primaryShardReference); // release shard operation lock before responding to caller
@@ -361,22 +387,37 @@ public void onFailure(Exception e) {
             };
         }
 
-        protected ReplicationOperation<Request, ReplicaRequest, PrimaryResult> createReplicatedOperation(
-            Request request, ActionListener<PrimaryResult> listener,
+        protected ReplicationOperation<Request, ReplicaRequest, PrimaryResult<ReplicaRequest, Response>> createReplicatedOperation(
+            Request request, ActionListener<PrimaryResult<ReplicaRequest, Response>> listener,
             PrimaryShardReference primaryShardReference, boolean executeOnReplicas) {
             return new ReplicationOperation<>(request, primaryShardReference, listener,
-                executeOnReplicas, replicasProxy, clusterService::state, logger, actionName
+                executeOnReplicas, new ReplicasProxy(primaryTerm), clusterService::state, logger, actionName
             );
         }
     }
 
-    protected class PrimaryResult implements ReplicationOperation.PrimaryResult<ReplicaRequest> {
+    protected static class PrimaryResult<ReplicaRequest extends ReplicationRequest<ReplicaRequest>,
+            Response extends ReplicationResponse>
+            implements ReplicationOperation.PrimaryResult<ReplicaRequest> {
         final ReplicaRequest replicaRequest;
-        final Response finalResponse;
+        public final Response finalResponseIfSuccessful;
+        public final Exception finalFailure;
 
-        public PrimaryResult(ReplicaRequest replicaRequest, Response finalResponse) {
+        /**
+         * Result of executing a primary operation
+         * expects <code>finalResponseIfSuccessful</code> or <code>finalFailure</code> to be not-null
+         */
+        public PrimaryResult(ReplicaRequest replicaRequest, Response finalResponseIfSuccessful, Exception finalFailure) {
+            assert finalFailure != null ^ finalResponseIfSuccessful != null
+                    : "either a response or a failure has to be not null, " +
+                    "found [" + finalFailure + "] failure and ["+ finalResponseIfSuccessful + "] response";
             this.replicaRequest = replicaRequest;
-            this.finalResponse = finalResponse;
+            this.finalResponseIfSuccessful = finalResponseIfSuccessful;
+            this.finalFailure = finalFailure;
+        }
+
+        public PrimaryResult(ReplicaRequest replicaRequest, Response replicationResponse) {
+            this(replicaRequest, replicationResponse, null);
         }
 
         @Override
@@ -386,22 +427,37 @@ public ReplicaRequest replicaRequest() {
 
         @Override
         public void setShardInfo(ReplicationResponse.ShardInfo shardInfo) {
-            finalResponse.setShardInfo(shardInfo);
+            if (finalResponseIfSuccessful != null) {
+                finalResponseIfSuccessful.setShardInfo(shardInfo);
+            }
         }
 
         public void respond(ActionListener<Response> listener) {
-            listener.onResponse(finalResponse);
+            if (finalResponseIfSuccessful != null) {
+                listener.onResponse(finalResponseIfSuccessful);
+            } else {
+                listener.onFailure(finalFailure);
+            }
         }
     }
 
-    protected class ReplicaResult {
-        /**
-         * Public constructor so subclasses can call it.
-         */
-        public ReplicaResult() {}
+    protected static class ReplicaResult {
+        final Exception finalFailure;
+
+        public ReplicaResult(Exception finalFailure) {
+            this.finalFailure = finalFailure;
+        }
+
+        public ReplicaResult() {
+            this(null);
+        }
 
         public void respond(ActionListener<TransportResponse.Empty> listener) {
-            listener.onResponse(TransportResponse.Empty.INSTANCE);
+            if (finalFailure == null) {
+                listener.onResponse(TransportResponse.Empty.INSTANCE);
+            } else {
+                listener.onFailure(finalFailure);
+            }
         }
     }
 
@@ -415,7 +471,8 @@ public void messageReceived(final ConcreteShardRequest<ReplicaRequest> request,
         @Override
         public void messageReceived(ConcreteShardRequest<ReplicaRequest> requestWithAID, TransportChannel channel, Task task)
             throws Exception {
-            new AsyncReplicaAction(requestWithAID.request, requestWithAID.targetAllocationID, channel, (ReplicationTask) task).run();
+            new AsyncReplicaAction(requestWithAID.request, requestWithAID.targetAllocationID, requestWithAID.getPrimaryTerm(), channel,
+                (ReplicationTask) task).run();
         }
     }
 
@@ -435,7 +492,9 @@ private final class AsyncReplicaAction extends AbstractRunnable implements Actio
         private final ReplicaRequest request;
         // allocation id of the replica this request is meant for
         private final String targetAllocationID;
+        private final long primaryTerm;
         private final TransportChannel channel;
+        private final IndexShard replica;
         /**
          * The task on the node with the replica shard.
          */
@@ -444,17 +503,23 @@ private final class AsyncReplicaAction extends AbstractRunnable implements Actio
         // something we want to avoid at all costs
         private final ClusterStateObserver observer = new ClusterStateObserver(clusterService, null, logger, threadPool.getThreadContext());
 
-        AsyncReplicaAction(ReplicaRequest request, String targetAllocationID, TransportChannel channel, ReplicationTask task) {
+        AsyncReplicaAction(ReplicaRequest request, String targetAllocationID, long primaryTerm, TransportChannel channel,
+                           ReplicationTask task) {
             this.request = request;
             this.channel = channel;
             this.task = task;
             this.targetAllocationID = targetAllocationID;
+            assert primaryTerm > 0L : "primary term can't be zero";
+            this.primaryTerm = primaryTerm;
+            final ShardId shardId = request.shardId();
+            assert shardId != null : "request shardId must be set";
+            this.replica = getIndexShard(shardId);
         }
 
         @Override
         public void onResponse(Releasable releasable) {
             try {
-                ReplicaResult replicaResult = shardOperationOnReplica(request);
+                ReplicaResult replicaResult = shardOperationOnReplica(request, replica);
                 releasable.close(); // release shard operation lock before responding to caller
                 replicaResult.respond(new ResponseListener());
             } catch (Exception e) {
@@ -473,18 +538,18 @@ public void onFailure(Exception e) {
                             transportReplicaAction,
                             request),
                     e);
-                final ThreadContext.StoredContext context = threadPool.getThreadContext().newStoredContext();
+                request.onRetry();
                 observer.waitForNextChange(new ClusterStateObserver.Listener() {
                     @Override
                     public void onNewClusterState(ClusterState state) {
-                        context.close();
                         // Forking a thread on local node via transport service so that custom transport service have an
                         // opportunity to execute custom logic before the replica operation begins
                         String extraMessage = "action [" + transportReplicaAction + "], request[" + request + "]";
                         TransportChannelResponseHandler<TransportResponse.Empty> handler =
-                            new TransportChannelResponseHandler<>(logger, channel, extraMessage, () -> TransportResponse.Empty.INSTANCE);
+                            new TransportChannelResponseHandler<>(logger, channel, extraMessage,
+                                () -> TransportResponse.Empty.INSTANCE);
                         transportService.sendRequest(clusterService.localNode(), transportReplicaAction,
-                            new ConcreteShardRequest<>(request, targetAllocationID),
+                            new ConcreteShardRequest<>(request, targetAllocationID, primaryTerm),
                             handler);
                     }
 
@@ -521,8 +586,12 @@ protected void responseWithFailure(Exception e) {
         @Override
         protected void doRun() throws Exception {
             setPhase(task, "replica");
-            assert request.shardId() != null : "request shardId must be set";
-            acquireReplicaOperationLock(request.shardId(), request.primaryTerm(), targetAllocationID, this);
+            final String actualAllocationId = this.replica.routingEntry().allocationId().getId();
+            if (actualAllocationId.equals(targetAllocationID) == false) {
+                throw new ShardNotFoundException(this.replica.shardId(), "expected aID [{}] but found [{}]", targetAllocationID,
+                    actualAllocationId);
+            }
+            replica.acquireReplicaOperationLock(primaryTerm, this, executor);
         }
 
         /**
@@ -550,6 +619,11 @@ public void onFailure(Exception e) {
         }
     }
 
+    private IndexShard getIndexShard(ShardId shardId) {
+        IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
+        return indexService.getShard(shardId.id());
+    }
+
     /**
      * Responsible for routing and retrying failed operations on the primary.
      * The actual primary operation is done in {@link ReplicationOperation} on the
@@ -582,7 +656,7 @@ public void onFailure(Exception e) {
         @Override
         protected void doRun() {
             setPhase(task, "routing");
-            final ClusterState state = observer.observedState();
+            final ClusterState state = observer.setAndGetObservedState();
             if (handleBlockExceptions(state)) {
                 return;
             }
@@ -594,6 +668,9 @@ protected void doRun() {
                 retry(new IndexNotFoundException(concreteIndex));
                 return;
             }
+            if (indexMetaData.getState() == IndexMetaData.State.CLOSE) {
+                throw new IndexClosedException(indexMetaData.getIndex());
+            }
 
             // resolve all derived request fields, so we can route and apply it
             resolveRequest(state.metaData(), indexMetaData, request);
@@ -605,21 +682,21 @@ protected void doRun() {
                 return;
             }
             final DiscoveryNode node = state.nodes().get(primary.currentNodeId());
-            taskManager.registerChildTask(task, node.getId());
             if (primary.currentNodeId().equals(state.nodes().getLocalNodeId())) {
-                performLocalAction(state, primary, node);
+                performLocalAction(state, primary, node, indexMetaData);
             } else {
                 performRemoteAction(state, primary, node);
             }
         }
 
-        private void performLocalAction(ClusterState state, ShardRouting primary, DiscoveryNode node) {
+        private void performLocalAction(ClusterState state, ShardRouting primary, DiscoveryNode node, IndexMetaData indexMetaData) {
             setPhase(task, "waiting_on_primary");
             if (logger.isTraceEnabled()) {
                 logger.trace("send action [{}] on primary [{}] for request [{}] with cluster state version [{}] to [{}] ",
                     transportPrimaryAction, request.shardId(), request, state.version(), primary.currentNodeId());
             }
-            performAction(node, transportPrimaryAction, true, new ConcreteShardRequest<>(request, primary.allocationId().getId()));
+            performAction(node, transportPrimaryAction, true,
+                new ConcreteShardRequest<>(request, primary.allocationId().getId(), indexMetaData.primaryTerm(primary.id())));
         }
 
         private void performRemoteAction(ClusterState state, ShardRouting primary, DiscoveryNode node) {
@@ -670,15 +747,21 @@ private ShardRouting primary(ClusterState state) {
         }
 
         private boolean handleBlockExceptions(ClusterState state) {
-            ClusterBlockException blockException = state.blocks().globalBlockedException(globalBlockLevel());
-            if (blockException != null) {
-                handleBlockException(blockException);
-                return true;
+            ClusterBlockLevel globalBlockLevel = globalBlockLevel();
+            if (globalBlockLevel != null) {
+                ClusterBlockException blockException = state.blocks().globalBlockedException(globalBlockLevel);
+                if (blockException != null) {
+                    handleBlockException(blockException);
+                    return true;
+                }
             }
-            blockException = state.blocks().indexBlockedException(indexBlockLevel(), concreteIndex(state));
-            if (blockException != null) {
-                handleBlockException(blockException);
-                return true;
+            ClusterBlockLevel indexBlockLevel = indexBlockLevel();
+            if (indexBlockLevel != null) {
+                ClusterBlockException blockException = state.blocks().indexBlockedException(indexBlockLevel, concreteIndex(state));
+                if (blockException != null) {
+                    handleBlockException(blockException);
+                    return true;
+                }
             }
             return false;
         }
@@ -745,11 +828,10 @@ void retry(Exception failure) {
             }
             setPhase(task, "waiting_for_retry");
             request.onRetry();
-            final ThreadContext.StoredContext context = threadPool.getThreadContext().newStoredContext();
+            request.primaryTerm(0L);
             observer.waitForNextChange(new ClusterStateObserver.Listener() {
                 @Override
                 public void onNewClusterState(ClusterState state) {
-                    context.close();
                     run();
                 }
 
@@ -760,7 +842,6 @@ public void onClusterServiceClose() {
 
                 @Override
                 public void onTimeout(TimeValue timeout) {
-                    context.close();
                     // Try one more time...
                     run();
                 }
@@ -816,10 +897,9 @@ void retryBecauseUnavailable(ShardId shardId, String message) {
      * tries to acquire reference to {@link IndexShard} to perform a primary operation. Released after performing primary operation locally
      * and replication of the operation to all replica shards is completed / failed (see {@link ReplicationOperation}).
      */
-    protected void acquirePrimaryShardReference(ShardId shardId, String allocationId,
-                                                ActionListener<PrimaryShardReference> onReferenceAcquired) {
-        IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
-        IndexShard indexShard = indexService.getShard(shardId.id());
+    private void acquirePrimaryShardReference(ShardId shardId, String allocationId, long primaryTerm,
+                                              ActionListener<PrimaryShardReference> onReferenceAcquired) {
+        IndexShard indexShard = getIndexShard(shardId);
         // we may end up here if the cluster state used to route the primary is so stale that the underlying
         // index shard was replaced with a replica. For example - in a two node cluster, if the primary fails
         // the replica will take over and a replica will be assigned to the first node.
@@ -827,10 +907,16 @@ protected void acquirePrimaryShardReference(ShardId shardId, String allocationId
             throw new ReplicationOperation.RetryOnPrimaryException(indexShard.shardId(),
                 "actual shard is not a primary " + indexShard.routingEntry());
         }
+
         final String actualAllocationId = indexShard.routingEntry().allocationId().getId();
         if (actualAllocationId.equals(allocationId) == false) {
             throw new ShardNotFoundException(shardId, "expected aID [{}] but found [{}]", allocationId, actualAllocationId);
         }
+        final long actualTerm = indexShard.getPrimaryTerm();
+        if (actualTerm != primaryTerm) {
+            throw new ShardNotFoundException(shardId, "expected aID [{}] with term [{}] but found [{}]", allocationId,
+                primaryTerm, actualTerm);
+        }
 
         ActionListener<Releasable> onAcquired = new ActionListener<Releasable>() {
             @Override
@@ -847,29 +933,16 @@ public void onFailure(Exception e) {
         indexShard.acquirePrimaryOperationLock(onAcquired, executor);
     }
 
-    /**
-     * tries to acquire an operation on replicas. The lock is closed as soon as replication is completed on the node.
-     */
-    protected void acquireReplicaOperationLock(ShardId shardId, long primaryTerm, final String allocationId,
-                                               ActionListener<Releasable> onLockAcquired) {
-        IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
-        IndexShard indexShard = indexService.getShard(shardId.id());
-        final String actualAllocationId = indexShard.routingEntry().allocationId().getId();
-        if (actualAllocationId.equals(allocationId) == false) {
-            throw new ShardNotFoundException(shardId, "expected aID [{}] but found [{}]", allocationId, actualAllocationId);
-        }
-        indexShard.acquireReplicaOperationLock(primaryTerm, onLockAcquired, executor);
-    }
-
     /**
      * Indicated whether this operation should be replicated to shadow replicas or not. If this method returns true the replication phase
      * will be skipped. For example writes such as index and delete don't need to be replicated on shadow replicas but refresh and flush do.
      */
-    protected boolean shouldExecuteReplication(Settings settings) {
-        return IndexMetaData.isIndexUsingShadowReplicas(settings) == false;
+    protected boolean shouldExecuteReplication(IndexMetaData indexMetaData) {
+        return IndexMetaData.isIndexUsingShadowReplicas(indexMetaData.getSettings()) == false;
     }
 
-    class PrimaryShardReference implements ReplicationOperation.Primary<Request, ReplicaRequest, PrimaryResult>, Releasable {
+    class PrimaryShardReference implements
+            ReplicationOperation.Primary<Request, ReplicaRequest, PrimaryResult<ReplicaRequest, Response>>, Releasable {
 
         private final IndexShard indexShard;
         private final Releasable operationLock;
@@ -899,9 +972,7 @@ public void failShard(String reason, Exception e) {
 
         @Override
         public PrimaryResult perform(Request request) throws Exception {
-            PrimaryResult result = shardOperationOnPrimary(request);
-            result.replicaRequest().primaryTerm(indexShard.getPrimaryTerm());
-            return result;
+            return shardOperationOnPrimary(request, indexShard);
         }
 
         @Override
@@ -912,8 +983,15 @@ public ShardRouting routingEntry() {
 
     final class ReplicasProxy implements ReplicationOperation.Replicas<ReplicaRequest> {
 
+        private final long primaryTerm;
+
+        ReplicasProxy(long primaryTerm) {
+            this.primaryTerm = primaryTerm;
+        }
+
         @Override
-        public void performOn(ShardRouting replica, ReplicaRequest request, ActionListener<TransportResponse.Empty> listener) {
+        public void performOn(ShardRouting replica, ReplicaRequest request,
+                              ActionListener<TransportResponse.Empty> listener) {
             String nodeId = replica.currentNodeId();
             final DiscoveryNode node = clusterService.state().nodes().get(nodeId);
             if (node == null) {
@@ -921,19 +999,19 @@ public void performOn(ShardRouting replica, ReplicaRequest request, ActionListen
                 return;
             }
             transportService.sendRequest(node, transportReplicaAction,
-                new ConcreteShardRequest<>(request, replica.allocationId().getId()), transportOptions,
+                new ConcreteShardRequest<>(request, replica.allocationId().getId(), primaryTerm), transportOptions,
                 new ActionListenerResponseHandler<>(listener, () -> TransportResponse.Empty.INSTANCE));
         }
 
         @Override
-        public void failShard(ShardRouting replica, long primaryTerm, String message, Exception exception,
+        public void failShard(ShardRouting replica, String message, Exception exception,
                               Runnable onSuccess, Consumer<Exception> onPrimaryDemoted, Consumer<Exception> onIgnoredFailure) {
             shardStateAction.remoteShardFailed(replica.shardId(), replica.allocationId().getId(), primaryTerm, message, exception,
                 createListener(onSuccess, onPrimaryDemoted, onIgnoredFailure));
         }
 
         @Override
-        public void markShardCopyAsStale(ShardId shardId, String allocationId, long primaryTerm, Runnable onSuccess,
+        public void markShardCopyAsStale(ShardId shardId, String allocationId, Runnable onSuccess,
                                          Consumer<Exception> onPrimaryDemoted, Consumer<Exception> onIgnoredFailure) {
             shardStateAction.remoteShardFailed(shardId, allocationId, primaryTerm, "mark copy as stale", null,
                 createListener(onSuccess, onPrimaryDemoted, onIgnoredFailure));
@@ -964,24 +1042,29 @@ public void onFailure(Exception shardFailedError) {
     }
 
     /** a wrapper class to encapsulate a request when being sent to a specific allocation id **/
-    public static final class ConcreteShardRequest<R extends TransportRequest> extends TransportRequest {
+    public static final class ConcreteShardRequest<R extends ReplicationRequest<?>> extends TransportRequest {
 
         /** {@link AllocationId#getId()} of the shard this request is sent to **/
         private String targetAllocationID;
 
+        private long primaryTerm;
+
         private R request;
 
         ConcreteShardRequest(Supplier<R> requestSupplier) {
             request = requestSupplier.get();
             // null now, but will be populated by reading from the streams
             targetAllocationID = null;
+            primaryTerm = 0L;
         }
 
-        ConcreteShardRequest(R request, String targetAllocationID) {
+        ConcreteShardRequest(R request, String targetAllocationID, long primaryTerm) {
             Objects.requireNonNull(request);
             Objects.requireNonNull(targetAllocationID);
             this.request = request;
             this.targetAllocationID = targetAllocationID;
+            this.primaryTerm = primaryTerm;
+            this.request.primaryTerm(primaryTerm); // for bwc with ES < v5.6, that has the primary term on the inner ReplicationRequest
         }
 
         @Override
@@ -1005,18 +1088,30 @@ public Task createTask(long id, String type, String action, TaskId parentTaskId)
 
         @Override
         public String getDescription() {
-            return "[" + request.getDescription() + "] for aID [" + targetAllocationID + "]";
+            return "[" + request.getDescription() + "] for aID [" + targetAllocationID + "] and term [" + primaryTerm + "]";
         }
 
         @Override
         public void readFrom(StreamInput in) throws IOException {
             targetAllocationID = in.readString();
+            if (in.getVersion().onOrAfter(Version.V_5_6_0)) {
+                primaryTerm = in.readVLong();
+            }
             request.readFrom(in);
+            if (in.getVersion().before(Version.V_5_6_0)) {
+                primaryTerm = request.primaryTerm(); // for bwc with ES < v5.6, that has the primary term on the inner ReplicationRequest
+            }
         }
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             out.writeString(targetAllocationID);
+            if (out.getVersion().onOrAfter(Version.V_5_6_0)) {
+                out.writeVLong(primaryTerm);
+            } else {
+                // ensure that inner ReplicationRequest has primary term set
+                assert request.primaryTerm() == primaryTerm : "term on inner replication request not properly set";
+            }
             request.writeTo(out);
         }
 
@@ -1027,6 +1122,15 @@ public R getRequest() {
         public String getTargetAllocationID() {
             return targetAllocationID;
         }
+
+        public long getPrimaryTerm() {
+            return primaryTerm;
+        }
+
+        @Override
+        public String toString() {
+            return "request: " + request + ", target allocation id: " + targetAllocationID + ", primary term: " + primaryTerm;
+        }
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/TransportWriteAction.java b/core/src/main/java/org/elasticsearch/action/support/replication/TransportWriteAction.java
index 39b49a4a409d9..0322b2d2d12f3 100644
--- a/core/src/main/java/org/elasticsearch/action/support/replication/TransportWriteAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/replication/TransportWriteAction.java
@@ -21,17 +21,18 @@
 
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.bulk.BulkRequest;
+import org.elasticsearch.action.bulk.BulkShardRequest;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.action.support.WriteResponse;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
+import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.shard.IndexShard;
-import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.index.translog.Translog.Location;
 import org.elasticsearch.indices.IndicesService;
@@ -46,92 +47,69 @@
 
 /**
  * Base class for transport actions that modify data in some shard like index, delete, and shardBulk.
+ * Allows performing async actions (e.g. refresh) after performing write operations on primary and replica shards
  */
 public abstract class TransportWriteAction<
             Request extends ReplicatedWriteRequest<Request>,
+            ReplicaRequest extends ReplicatedWriteRequest<ReplicaRequest>,
             Response extends ReplicationResponse & WriteResponse
-        > extends TransportReplicationAction<Request, Request, Response> {
+        > extends TransportReplicationAction<Request, ReplicaRequest, Response> {
 
     protected TransportWriteAction(Settings settings, String actionName, TransportService transportService,
             ClusterService clusterService, IndicesService indicesService, ThreadPool threadPool, ShardStateAction shardStateAction,
             ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, Supplier<Request> request,
-            String executor) {
+                                   Supplier<ReplicaRequest> replicaRequest, String executor) {
         super(settings, actionName, transportService, clusterService, indicesService, threadPool, shardStateAction, actionFilters,
-                indexNameExpressionResolver, request, request, executor);
+                indexNameExpressionResolver, request, replicaRequest, executor);
     }
 
     /**
-     * Called on the primary with a reference to the {@linkplain IndexShard} to modify.
-     */
-    protected abstract WriteResult<Response> onPrimaryShard(Request request, IndexShard indexShard) throws Exception;
-
-    /**
-     * Called once per replica with a reference to the {@linkplain IndexShard} to modify.
+     * Called on the primary with a reference to the primary {@linkplain IndexShard} to modify.
      *
-     * @return the translog location of the {@linkplain IndexShard} after the write was completed or null if no write occurred
+     * @return the result of the operation on primary, including current translog location and operation response and failure
+     * async refresh is performed on the <code>primary</code> shard according to the <code>Request</code> refresh policy
      */
-    protected abstract Translog.Location onReplicaShard(Request request, IndexShard indexShard);
-
     @Override
-    protected final WritePrimaryResult shardOperationOnPrimary(Request request) throws Exception {
-        IndexShard indexShard = indexShard(request);
-        WriteResult<Response> result = onPrimaryShard(request, indexShard);
-        return new WritePrimaryResult(request, result.getResponse(), result.getLocation(), indexShard);
-    }
-
-    @Override
-    protected final WriteReplicaResult shardOperationOnReplica(Request request) {
-        IndexShard indexShard = indexShard(request);
-        Translog.Location location = onReplicaShard(request, indexShard);
-        return new WriteReplicaResult(indexShard, request, location);
-    }
+    protected abstract WritePrimaryResult<ReplicaRequest, Response> shardOperationOnPrimary(
+            Request request, IndexShard primary) throws Exception;
 
     /**
-     * Fetch the IndexShard for the request. Protected so it can be mocked in tests.
-     */
-    protected IndexShard indexShard(Request request) {
-        final ShardId shardId = request.shardId();
-        IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
-        return indexService.getShard(shardId.id());
-    }
-
-    /**
-     * Simple result from a write action. Write actions have static method to return these so they can integrate with bulk.
+     * Called once per replica with a reference to the replica {@linkplain IndexShard} to modify.
+     *
+     * @return the result of the operation on replica, including current translog location and operation response and failure
+     * async refresh is performed on the <code>replica</code> shard according to the <code>ReplicaRequest</code> refresh policy
      */
-    public static class WriteResult<Response extends ReplicationResponse> {
-        private final Response response;
-        private final Translog.Location location;
-
-        public WriteResult(Response response, @Nullable Location location) {
-            this.response = response;
-            this.location = location;
-        }
-
-        public Response getResponse() {
-            return response;
-        }
-
-        public Translog.Location getLocation() {
-            return location;
-        }
-    }
+    @Override
+    protected abstract WriteReplicaResult<ReplicaRequest> shardOperationOnReplica(
+            ReplicaRequest request, IndexShard replica) throws Exception;
 
     /**
      * Result of taking the action on the primary.
      */
-    class WritePrimaryResult extends PrimaryResult implements RespondingWriteResult {
+    protected static class WritePrimaryResult<ReplicaRequest extends ReplicatedWriteRequest<ReplicaRequest>,
+            Response extends ReplicationResponse & WriteResponse> extends PrimaryResult<ReplicaRequest, Response>
+            implements RespondingWriteResult {
         boolean finishedAsyncActions;
+        public final Location location;
         ActionListener<Response> listener = null;
 
-        public WritePrimaryResult(Request request, Response finalResponse,
-                                  @Nullable Translog.Location location,
-                                  IndexShard indexShard) {
-            super(request, finalResponse);
-            /*
-             * We call this before replication because this might wait for a refresh and that can take a while. This way we wait for the
-             * refresh in parallel on the primary and on the replica.
-             */
-            new AsyncAfterWriteAction(indexShard, request, location, this, logger).run();
+        public WritePrimaryResult(ReplicaRequest request, @Nullable Response finalResponse,
+                                  @Nullable Location location, @Nullable Exception operationFailure,
+                                  IndexShard primary, Logger logger) {
+            super(request, finalResponse, operationFailure);
+            this.location = location;
+            assert location == null || operationFailure == null
+                    : "expected either failure to be null or translog location to be null, " +
+                    "but found: [" + location + "] translog location and [" + operationFailure + "] failure";
+            if (operationFailure != null) {
+                this.finishedAsyncActions = true;
+            } else {
+                /*
+                 * We call this before replication because this might wait for a refresh and that can take a while.
+                 * This way we wait for the refresh in parallel on the primary and on the replica.
+                 */
+                new AsyncAfterWriteAction(primary, request, location, this, logger).run();
+            }
         }
 
         @Override
@@ -160,7 +138,7 @@ public synchronized void onFailure(Exception exception) {
 
         @Override
         public synchronized void onSuccess(boolean forcedRefresh) {
-            finalResponse.setForcedRefresh(forcedRefresh);
+            finalResponseIfSuccessful.setForcedRefresh(forcedRefresh);
             finishedAsyncActions = true;
             respondIfPossible(null);
         }
@@ -169,12 +147,21 @@ public synchronized void onSuccess(boolean forcedRefresh) {
     /**
      * Result of taking the action on the replica.
      */
-    class WriteReplicaResult extends ReplicaResult implements RespondingWriteResult {
+    protected static class WriteReplicaResult<ReplicaRequest extends ReplicatedWriteRequest<ReplicaRequest>>
+            extends ReplicaResult implements RespondingWriteResult {
+        public final Location location;
         boolean finishedAsyncActions;
         private ActionListener<TransportResponse.Empty> listener;
 
-        public WriteReplicaResult(IndexShard indexShard, ReplicatedWriteRequest<?> request, Translog.Location location) {
-            new AsyncAfterWriteAction(indexShard, request, location, this, logger).run();
+        public WriteReplicaResult(ReplicaRequest request, @Nullable Location location,
+                                  @Nullable Exception operationFailure, IndexShard replica, Logger logger) {
+            super(operationFailure);
+            this.location = location;
+            if (operationFailure != null) {
+                this.finishedAsyncActions = true;
+            } else {
+                new AsyncAfterWriteAction(replica, request, location, this, logger).run();
+            }
         }
 
         @Override
@@ -209,6 +196,16 @@ public synchronized void onSuccess(boolean forcedRefresh) {
         }
     }
 
+    @Override
+    protected ClusterBlockLevel globalBlockLevel() {
+        return ClusterBlockLevel.WRITE;
+    }
+
+    @Override
+    protected ClusterBlockLevel indexBlockLevel() {
+        return ClusterBlockLevel.WRITE;
+    }
+
     /**
      * callback used by {@link AsyncAfterWriteAction} to notify that all post
      * process actions have been executed
diff --git a/core/src/main/java/org/elasticsearch/action/support/single/instance/InstanceShardOperationRequest.java b/core/src/main/java/org/elasticsearch/action/support/single/instance/InstanceShardOperationRequest.java
index cb9a6ab9f6984..d91de908311a9 100644
--- a/core/src/main/java/org/elasticsearch/action/support/single/instance/InstanceShardOperationRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/support/single/instance/InstanceShardOperationRequest.java
@@ -35,7 +35,7 @@
 /**
  *
  */
-public abstract class InstanceShardOperationRequest<Request extends InstanceShardOperationRequest<Request>> extends ActionRequest<Request>
+public abstract class InstanceShardOperationRequest<Request extends InstanceShardOperationRequest<Request>> extends ActionRequest
         implements IndicesRequest {
 
     public static final TimeValue DEFAULT_TIMEOUT = new TimeValue(1, TimeUnit.MINUTES);
diff --git a/core/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java b/core/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java
index 81da5ec9a86ff..f05d254c266af 100644
--- a/core/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java
@@ -39,12 +39,12 @@
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.node.NodeClosedException;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.TransportResponseHandler;
 import org.elasticsearch.transport.ConnectTransportException;
 import org.elasticsearch.transport.TransportChannel;
 import org.elasticsearch.transport.TransportException;
 import org.elasticsearch.transport.TransportRequestHandler;
 import org.elasticsearch.transport.TransportRequestOptions;
+import org.elasticsearch.transport.TransportResponseHandler;
 import org.elasticsearch.transport.TransportService;
 
 import java.util.function.Supplier;
@@ -114,7 +114,6 @@ class AsyncSingleAction {
         private final Request request;
         private volatile ClusterStateObserver observer;
         private ShardIterator shardIt;
-        private DiscoveryNodes nodes;
 
         AsyncSingleAction(Request request, ActionListener<Response> listener) {
             this.request = request;
@@ -122,14 +121,14 @@ class AsyncSingleAction {
         }
 
         public void start() {
-            this.observer = new ClusterStateObserver(clusterService, request.timeout(), logger, threadPool.getThreadContext());
-            doStart();
+            ClusterState state = clusterService.state();
+            this.observer = new ClusterStateObserver(state, clusterService, request.timeout(), logger, threadPool.getThreadContext());
+            doStart(state);
         }
 
-        protected void doStart() {
-            nodes = observer.observedState().nodes();
+        protected void doStart(ClusterState clusterState) {
             try {
-                ClusterBlockException blockException = checkGlobalBlock(observer.observedState());
+                ClusterBlockException blockException = checkGlobalBlock(clusterState);
                 if (blockException != null) {
                     if (blockException.retryable()) {
                         retry(blockException);
@@ -138,9 +137,9 @@ protected void doStart() {
                         throw blockException;
                     }
                 }
-                request.concreteIndex(indexNameExpressionResolver.concreteSingleIndex(observer.observedState(), request).getName());
-                resolveRequest(observer.observedState(), request);
-                blockException = checkRequestBlock(observer.observedState(), request);
+                request.concreteIndex(indexNameExpressionResolver.concreteSingleIndex(clusterState, request).getName());
+                resolveRequest(clusterState, request);
+                blockException = checkRequestBlock(clusterState, request);
                 if (blockException != null) {
                     if (blockException.retryable()) {
                         retry(blockException);
@@ -149,7 +148,7 @@ protected void doStart() {
                         throw blockException;
                     }
                 }
-                shardIt = shards(observer.observedState(), request);
+                shardIt = shards(clusterState, request);
             } catch (Exception e) {
                 listener.onFailure(e);
                 return;
@@ -173,7 +172,7 @@ protected void doStart() {
             }
 
             request.shardId = shardIt.shardId();
-            DiscoveryNode node = nodes.get(shard.currentNodeId());
+            DiscoveryNode node = clusterState.nodes().get(shard.currentNodeId());
             transportService.sendRequest(node, shardActionName, request, transportOptions(), new TransportResponseHandler<Response>() {
 
                 @Override
@@ -223,18 +222,18 @@ void retry(@Nullable final Exception failure) {
             observer.waitForNextChange(new ClusterStateObserver.Listener() {
                 @Override
                 public void onNewClusterState(ClusterState state) {
-                    doStart();
+                    doStart(state);
                 }
 
                 @Override
                 public void onClusterServiceClose() {
-                    listener.onFailure(new NodeClosedException(nodes.getLocalNode()));
+                    listener.onFailure(new NodeClosedException(clusterService.localNode()));
                 }
 
                 @Override
                 public void onTimeout(TimeValue timeout) {
                     // just to be on the safe side, see if we can start it now?
-                    doStart();
+                    doStart(observer.setAndGetObservedState());
                 }
             }, request.timeout());
         }
diff --git a/core/src/main/java/org/elasticsearch/action/support/single/shard/SingleShardRequest.java b/core/src/main/java/org/elasticsearch/action/support/single/shard/SingleShardRequest.java
index 499932fce687e..f3b7a7e5fe241 100644
--- a/core/src/main/java/org/elasticsearch/action/support/single/shard/SingleShardRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/support/single/shard/SingleShardRequest.java
@@ -34,7 +34,7 @@
 /**
  *
  */
-public abstract class SingleShardRequest<Request extends SingleShardRequest<Request>> extends ActionRequest<Request> implements IndicesRequest {
+public abstract class SingleShardRequest<Request extends SingleShardRequest<Request>> extends ActionRequest implements IndicesRequest {
 
     public static final IndicesOptions INDICES_OPTIONS = IndicesOptions.strictSingleIndexNoExpandForbidClosed();
 
diff --git a/core/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java b/core/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java
index 8981caa60f760..811dcbed3dcf9 100644
--- a/core/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java
@@ -46,6 +46,7 @@
 import org.elasticsearch.transport.TransportResponseHandler;
 import org.elasticsearch.transport.TransportService;
 
+import java.io.IOException;
 import java.util.function.Supplier;
 
 import static org.elasticsearch.action.support.TransportActions.isShardNotAvailableException;
@@ -94,7 +95,7 @@ protected void doExecute(Request request, ActionListener<Response> listener) {
         new AsyncSingleAction(request, listener).start();
     }
 
-    protected abstract Response shardOperation(Request request, ShardId shardId);
+    protected abstract Response shardOperation(Request request, ShardId shardId) throws IOException;
 
     protected abstract Response newResponse();
 
diff --git a/core/src/main/java/org/elasticsearch/action/support/tasks/BaseTasksRequest.java b/core/src/main/java/org/elasticsearch/action/support/tasks/BaseTasksRequest.java
index abc081dbc071f..e912eebb4fb39 100644
--- a/core/src/main/java/org/elasticsearch/action/support/tasks/BaseTasksRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/support/tasks/BaseTasksRequest.java
@@ -36,13 +36,13 @@
 /**
  * A base class for task requests
  */
-public class BaseTasksRequest<Request extends BaseTasksRequest<Request>> extends ActionRequest<Request> {
+public class BaseTasksRequest<Request extends BaseTasksRequest<Request>> extends ActionRequest {
 
     public static final String[] ALL_ACTIONS = Strings.EMPTY_ARRAY;
 
     public static final String[] ALL_NODES = Strings.EMPTY_ARRAY;
 
-    private String[] nodesIds = ALL_NODES;
+    private String[] nodes = ALL_NODES;
 
     private TimeValue timeout;
 
@@ -58,7 +58,7 @@ public BaseTasksRequest() {
     @Override
     public ActionRequestValidationException validate() {
         ActionRequestValidationException validationException = null;
-        if (taskId.isSet() && nodesIds.length > 0) {
+        if (taskId.isSet() && nodes.length > 0) {
             validationException = addValidationError("task id cannot be used together with node ids",
                 validationException);
         }
@@ -81,13 +81,13 @@ public String[] getActions() {
         return actions;
     }
 
-    public final String[] getNodesIds() {
-        return nodesIds;
+    public final String[] getNodes() {
+        return nodes;
     }
 
     @SuppressWarnings("unchecked")
-    public final Request setNodesIds(String... nodesIds) {
-        this.nodesIds = nodesIds;
+    public final Request setNodes(String... nodes) {
+        this.nodes = nodes;
         return (Request) this;
     }
 
@@ -142,7 +142,7 @@ public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
         taskId = TaskId.readFromStream(in);
         parentTaskId = TaskId.readFromStream(in);
-        nodesIds = in.readStringArray();
+        nodes = in.readStringArray();
         actions = in.readStringArray();
         timeout = in.readOptionalWriteable(TimeValue::new);
     }
@@ -152,7 +152,7 @@ public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         taskId.writeTo(out);
         parentTaskId.writeTo(out);
-        out.writeStringArrayNullable(nodesIds);
+        out.writeStringArrayNullable(nodes);
         out.writeStringArrayNullable(actions);
         out.writeOptionalWriteable(timeout);
     }
diff --git a/core/src/main/java/org/elasticsearch/action/support/tasks/BaseTasksResponse.java b/core/src/main/java/org/elasticsearch/action/support/tasks/BaseTasksResponse.java
index 43be2b46db1c4..fdbd8e6fe708f 100644
--- a/core/src/main/java/org/elasticsearch/action/support/tasks/BaseTasksResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/support/tasks/BaseTasksResponse.java
@@ -19,16 +19,22 @@
 
 package org.elasticsearch.action.support.tasks;
 
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.TaskOperationFailure;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.tasks.TaskId;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.stream.Stream;
+
+import static java.util.stream.Collectors.toList;
+import static org.elasticsearch.ExceptionsHelper.rethrowAndSuppress;
 
 
 /**
@@ -38,9 +44,6 @@ public class BaseTasksResponse extends ActionResponse {
     private List<TaskOperationFailure> taskFailures;
     private List<FailedNodeException> nodeFailures;
 
-    public BaseTasksResponse() {
-    }
-
     public BaseTasksResponse(List<TaskOperationFailure> taskFailures, List<? extends FailedNodeException> nodeFailures) {
         this.taskFailures = taskFailures == null ? Collections.emptyList() : Collections.unmodifiableList(new ArrayList<>(taskFailures));
         this.nodeFailures = nodeFailures == null ? Collections.emptyList() : Collections.unmodifiableList(new ArrayList<>(nodeFailures));
@@ -60,17 +63,28 @@ public List<FailedNodeException> getNodeFailures() {
         return nodeFailures;
     }
 
+    /**
+     * Rethrow task failures if there are any.
+     */
+    public void rethrowFailures(String operationName) {
+        rethrowAndSuppress(Stream.concat(
+                    getNodeFailures().stream(),
+                    getTaskFailures().stream().map(f -> new ElasticsearchException(
+                            "{} of [{}] failed", f.getCause(), operationName, new TaskId(f.getNodeId(), f.getTaskId()))))
+                .collect(toList()));
+    }
+
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
         int size = in.readVInt();
-        List<TaskOperationFailure> taskFailures = new ArrayList<>();
+        List<TaskOperationFailure> taskFailures = new ArrayList<>(size);
         for (int i = 0; i < size; i++) {
             taskFailures.add(new TaskOperationFailure(in));
         }
         size = in.readVInt();
         this.taskFailures = Collections.unmodifiableList(taskFailures);
-        List<FailedNodeException> nodeFailures = new ArrayList<>();
+        List<FailedNodeException> nodeFailures = new ArrayList<>(size);
         for (int i = 0; i < size; i++) {
             nodeFailures.add(new FailedNodeException(in));
         }
diff --git a/core/src/main/java/org/elasticsearch/action/support/tasks/TasksRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/support/tasks/TasksRequestBuilder.java
index a3528cb75c4d3..656dae9992889 100644
--- a/core/src/main/java/org/elasticsearch/action/support/tasks/TasksRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/support/tasks/TasksRequestBuilder.java
@@ -48,7 +48,7 @@ public final RequestBuilder setTaskId(TaskId taskId) {
 
     @SuppressWarnings("unchecked")
     public final RequestBuilder setNodesIds(String... nodesIds) {
-        request.setNodesIds(nodesIds);
+        request.setNodes(nodesIds);
         return (RequestBuilder) this;
     }
 
@@ -63,5 +63,14 @@ public final RequestBuilder setTimeout(TimeValue timeout) {
         request.setTimeout(timeout);
         return (RequestBuilder) this;
     }
+
+    /**
+     * Match all children of the provided task. 
+     */
+    @SuppressWarnings("unchecked")
+    public final RequestBuilder setParentTaskId(TaskId taskId) {
+        request.setParentTaskId(taskId);
+        return (RequestBuilder) this;
+    }
 }
 
diff --git a/core/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java b/core/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java
index 6752ccd7293a7..35b2b41dfda6e 100644
--- a/core/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java
@@ -33,10 +33,12 @@
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.NodeShouldNotConnectException;
@@ -57,6 +59,8 @@
 import java.util.function.Consumer;
 import java.util.function.Supplier;
 
+import static java.util.Collections.emptyList;
+
 /**
  * The base class for transport actions that are interacting with currently running tasks.
  */
@@ -100,21 +104,56 @@ protected void doExecute(Task task, TasksRequest request, ActionListener<TasksRe
         new AsyncAction(task, request, listener).start();
     }
 
-    private NodeTasksResponse nodeOperation(NodeTaskRequest nodeTaskRequest) {
+    private void nodeOperation(NodeTaskRequest nodeTaskRequest, ActionListener<NodeTasksResponse> listener) {
         TasksRequest request = nodeTaskRequest.tasksRequest;
-        List<TaskResponse> results = new ArrayList<>();
-        List<TaskOperationFailure> exceptions = new ArrayList<>();
-        processTasks(request, task -> {
-            try {
-                TaskResponse response = taskOperation(request, task);
-                if (response != null) {
-                    results.add(response);
+        List<OperationTask> tasks = new ArrayList<>();
+        processTasks(request, tasks::add);
+        if (tasks.isEmpty()) {
+            listener.onResponse(new NodeTasksResponse(clusterService.localNode().getId(), emptyList(), emptyList()));
+            return;
+        }
+        AtomicArray<Tuple<TaskResponse, Exception>> responses = new AtomicArray<>(tasks.size());
+        final AtomicInteger counter = new AtomicInteger(tasks.size());
+        for (int i = 0; i < tasks.size(); i++) {
+            final int taskIndex = i;
+            ActionListener<TaskResponse> taskListener = new ActionListener<TaskResponse>() {
+                @Override
+                public void onResponse(TaskResponse response) {
+                    responses.setOnce(taskIndex, response == null ? null : new Tuple<>(response, null));
+                    respondIfFinished();
                 }
-            } catch (Exception ex) {
-                exceptions.add(new TaskOperationFailure(clusterService.localNode().getId(), task.getId(), ex));
+
+                @Override
+                public void onFailure(Exception e) {
+                    responses.setOnce(taskIndex, new Tuple<>(null, e));
+                    respondIfFinished();
+                }
+
+                private void respondIfFinished() {
+                    if (counter.decrementAndGet() != 0) {
+                        return;
+                    }
+                    List<TaskResponse> results = new ArrayList<>();
+                    List<TaskOperationFailure> exceptions = new ArrayList<>();
+                    for (Tuple<TaskResponse, Exception> response : responses.asList()) {
+                        if (response.v1() == null) {
+                            assert response.v2() != null;
+                            exceptions.add(new TaskOperationFailure(clusterService.localNode().getId(), tasks.get(taskIndex).getId(),
+                                    response.v2()));
+                        } else {
+                            assert response.v2() == null;
+                            results.add(response.v1());
+                        }
+                    }
+                    listener.onResponse(new NodeTasksResponse(clusterService.localNode().getId(), results, exceptions));
+                }
+            };
+            try {
+                taskOperation(request, tasks.get(taskIndex), taskListener);
+            } catch (Exception e) {
+                taskListener.onFailure(e);
             }
-        });
-        return new NodeTasksResponse(clusterService.localNode().getId(), results, exceptions);
+        }
     }
 
     protected String[] filterNodeIds(DiscoveryNodes nodes, String[] nodesIds) {
@@ -125,7 +164,7 @@ protected String[] resolveNodes(TasksRequest request, ClusterState clusterState)
         if (request.getTaskId().isSet()) {
             return new String[]{request.getTaskId().getNodeId()};
         } else {
-            return clusterState.nodes().resolveNodes(request.getNodesIds());
+            return clusterState.nodes().resolveNodes(request.getNodes());
         }
     }
 
@@ -178,14 +217,15 @@ protected TasksResponse newResponse(TasksRequest request, AtomicReferenceArray r
 
     protected abstract TaskResponse readTaskResponse(StreamInput in) throws IOException;
 
-    protected abstract TaskResponse taskOperation(TasksRequest request, OperationTask task);
+    /**
+     * Perform the required operation on the task. It is OK start an asynchronous operation or to throw an exception but not both.
+     */
+    protected abstract void taskOperation(TasksRequest request, OperationTask task, ActionListener<TaskResponse> listener);
 
     protected boolean transportCompress() {
         return false;
     }
 
-    protected abstract boolean accumulateExceptions();
-
     private class AsyncAction {
 
         private final TasksRequest request;
@@ -236,7 +276,6 @@ private void start() {
                         } else {
                             NodeTaskRequest nodeRequest = new NodeTaskRequest(request);
                             nodeRequest.setParentTask(clusterService.localNode().getId(), task.getId());
-                            taskManager.registerChildTask(task, node.getId());
                             transportService.sendRequest(node, transportNodeAction, nodeRequest, builder.build(),
                                 new TransportResponseHandler<NodeTasksResponse>() {
                                     @Override
@@ -280,9 +319,9 @@ private void onFailure(int idx, String nodeId, Throwable t) {
                     (org.apache.logging.log4j.util.Supplier<?>)
                         () -> new ParameterizedMessage("failed to execute on node [{}]", nodeId), t);
             }
-            if (accumulateExceptions()) {
-                responses.set(idx, new FailedNodeException(nodeId, "Failed node [" + nodeId + "]", t));
-            }
+
+            responses.set(idx, new FailedNodeException(nodeId, "Failed node [" + nodeId + "]", t));
+
             if (counter.incrementAndGet() == responses.length()) {
                 finishHim();
             }
@@ -305,7 +344,27 @@ class NodeTransportHandler implements TransportRequestHandler<NodeTaskRequest> {
 
         @Override
         public void messageReceived(final NodeTaskRequest request, final TransportChannel channel) throws Exception {
-            channel.sendResponse(nodeOperation(request));
+            nodeOperation(request, new ActionListener<NodeTasksResponse>() {
+                @Override
+                public void onResponse(
+                        TransportTasksAction<OperationTask, TasksRequest, TasksResponse, TaskResponse>.NodeTasksResponse response) {
+                    try {
+                        channel.sendResponse(response);
+                    } catch (Exception e) {
+                        onFailure(e);
+                    }
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    try {
+                        channel.sendResponse(e);
+                    } catch (IOException e1) {
+                        e1.addSuppressed(e);
+                        logger.warn("Failed to send failure", e1);
+                    }
+                }
+            });
         }
     }
 
@@ -341,10 +400,10 @@ private class NodeTasksResponse extends TransportResponse {
         protected List<TaskOperationFailure> exceptions;
         protected List<TaskResponse> results;
 
-        public NodeTasksResponse() {
+        NodeTasksResponse() {
         }
 
-        public NodeTasksResponse(String nodeId,
+        NodeTasksResponse(String nodeId,
                                  List<TaskResponse> results,
                                  List<TaskOperationFailure> exceptions) {
             this.nodeId = nodeId;
diff --git a/core/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsRequest.java b/core/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsRequest.java
index af192cea600c6..fda393e375c9e 100644
--- a/core/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsRequest.java
@@ -23,14 +23,11 @@
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.CompositeIndicesRequest;
-import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.RealtimeRequest;
 import org.elasticsearch.action.ValidateActions;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
@@ -41,7 +38,7 @@
 import java.util.List;
 import java.util.Set;
 
-public class MultiTermVectorsRequest extends ActionRequest<MultiTermVectorsRequest> implements Iterable<TermVectorsRequest>, CompositeIndicesRequest, RealtimeRequest {
+public class MultiTermVectorsRequest extends ActionRequest implements Iterable<TermVectorsRequest>, CompositeIndicesRequest, RealtimeRequest {
 
     String preference;
     List<TermVectorsRequest> requests = new ArrayList<>();
@@ -76,11 +73,6 @@ public ActionRequestValidationException validate() {
         return validationException;
     }
 
-    @Override
-    public List<? extends IndicesRequest> subRequests() {
-        return requests;
-    }
-
     @Override
     public Iterator<TermVectorsRequest> iterator() {
         return Collections.unmodifiableCollection(requests).iterator();
@@ -94,43 +86,41 @@ public List<TermVectorsRequest> getRequests() {
         return requests;
     }
 
-    public void add(TermVectorsRequest template, BytesReference data) throws Exception {
+    public void add(TermVectorsRequest template, @Nullable XContentParser parser) throws IOException {
         XContentParser.Token token;
         String currentFieldName = null;
-        if (data.length() > 0) {
-            try (XContentParser parser = XContentFactory.xContent(data).createParser(data)) {
-                while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                    if (token == XContentParser.Token.FIELD_NAME) {
-                        currentFieldName = parser.currentName();
-                    } else if (token == XContentParser.Token.START_ARRAY) {
-                        if ("docs".equals(currentFieldName)) {
-                            while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
-                                if (token != XContentParser.Token.START_OBJECT) {
-                                    throw new IllegalArgumentException("docs array element should include an object");
-                                }
-                                TermVectorsRequest termVectorsRequest = new TermVectorsRequest(template);
-                                TermVectorsRequest.parseRequest(termVectorsRequest, parser);
-                                add(termVectorsRequest);
+        if (parser != null) {
+            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                if (token == XContentParser.Token.FIELD_NAME) {
+                    currentFieldName = parser.currentName();
+                } else if (token == XContentParser.Token.START_ARRAY) {
+                    if ("docs".equals(currentFieldName)) {
+                        while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                            if (token != XContentParser.Token.START_OBJECT) {
+                                throw new IllegalArgumentException("docs array element should include an object");
                             }
-                        } else if ("ids".equals(currentFieldName)) {
-                            while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
-                                if (!token.isValue()) {
-                                    throw new IllegalArgumentException("ids array element should only contain ids");
-                                }
-                                ids.add(parser.text());
-                            }
-                        } else {
-                            throw new ElasticsearchParseException("no parameter named [{}] and type ARRAY", currentFieldName);
+                            TermVectorsRequest termVectorsRequest = new TermVectorsRequest(template);
+                            TermVectorsRequest.parseRequest(termVectorsRequest, parser);
+                            add(termVectorsRequest);
                         }
-                    } else if (token == XContentParser.Token.START_OBJECT && currentFieldName != null) {
-                        if ("parameters".equals(currentFieldName)) {
-                            TermVectorsRequest.parseRequest(template, parser);
-                        } else {
-                            throw new ElasticsearchParseException("no parameter named [{}] and type OBJECT", currentFieldName);
+                    } else if ("ids".equals(currentFieldName)) {
+                        while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                            if (!token.isValue()) {
+                                throw new IllegalArgumentException("ids array element should only contain ids");
+                            }
+                            ids.add(parser.text());
                         }
-                    } else if (currentFieldName != null) {
-                        throw new ElasticsearchParseException("_mtermvectors: Parameter [{}] not supported", currentFieldName);
+                    } else {
+                        throw new ElasticsearchParseException("no parameter named [{}] and type ARRAY", currentFieldName);
+                    }
+                } else if (token == XContentParser.Token.START_OBJECT && currentFieldName != null) {
+                    if ("parameters".equals(currentFieldName)) {
+                        TermVectorsRequest.parseRequest(template, parser);
+                    } else {
+                        throw new ElasticsearchParseException("no parameter named [{}] and type OBJECT", currentFieldName);
                     }
+                } else if (currentFieldName != null) {
+                    throw new ElasticsearchParseException("_mtermvectors: Parameter [{}] not supported", currentFieldName);
                 }
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsResponse.java b/core/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsResponse.java
index 233d4b0c63884..8508c834a9f36 100644
--- a/core/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsResponse.java
@@ -24,14 +24,14 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Iterator;
 
-public class MultiTermVectorsResponse extends ActionResponse implements Iterable<MultiTermVectorsItemResponse>, ToXContent {
+public class MultiTermVectorsResponse extends ActionResponse implements Iterable<MultiTermVectorsItemResponse>, ToXContentObject {
 
     /**
      * Represents a failure.
@@ -124,6 +124,7 @@ public Iterator<MultiTermVectorsItemResponse> iterator() {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
         builder.startArray(Fields.DOCS);
         for (MultiTermVectorsItemResponse response : responses) {
             if (response.isFailed()) {
@@ -132,16 +133,15 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                 builder.field(Fields._INDEX, failure.getIndex());
                 builder.field(Fields._TYPE, failure.getType());
                 builder.field(Fields._ID, failure.getId());
-                ElasticsearchException.renderException(builder, params, failure.getCause());
+                ElasticsearchException.generateFailureXContent(builder, params, failure.getCause(), true);
                 builder.endObject();
             } else {
                 TermVectorsResponse getResponse = response.getResponse();
-                builder.startObject();
                 getResponse.toXContent(builder, params);
-                builder.endObject();
             }
         }
         builder.endArray();
+        builder.endObject();
         return builder;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsFields.java b/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsFields.java
index 0ae8824ce8d7f..088691a5c9c19 100644
--- a/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsFields.java
+++ b/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsFields.java
@@ -105,13 +105,13 @@
  * <li>vint: frequency (always returned)</li>
  * <li>
  * <ul>
- * <li>vint: position_1 (if positions == true)</li>
- * <li>vint: startOffset_1 (if offset == true)</li>
- * <li>vint: endOffset_1 (if offset == true)</li>
- * <li>BytesRef: payload_1 (if payloads == true)</li>
+ * <li>vint: position_1 (if positions)</li>
+ * <li>vint: startOffset_1 (if offset)</li>
+ * <li>vint: endOffset_1 (if offset)</li>
+ * <li>BytesRef: payload_1 (if payloads)</li>
  * <li>...</li>
- * <li>vint: endOffset_freqency (if offset == true)</li>
- * <li>BytesRef: payload_freqency (if payloads == true)</li>
+ * <li>vint: endOffset_freqency (if offset)</li>
+ * <li>BytesRef: payload_freqency (if payloads)</li>
  * </ul></li>
  * </ul>
  */
@@ -200,7 +200,7 @@ private final class TermVector extends Terms {
         private long sumDocFreq;
         private int docCount;
 
-        public TermVector(BytesReference termVectors, long readOffset) throws IOException {
+        TermVector(BytesReference termVectors, long readOffset) throws IOException {
             this.perFieldTermVectorInput = termVectors.streamInput();
             this.readOffset = readOffset;
             reset();
diff --git a/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java b/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java
index 3f33b2e3901e7..0fe83e214463a 100644
--- a/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java
@@ -20,8 +20,8 @@
 package org.elasticsearch.action.termvectors;
 
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequestValidationException;
-import org.elasticsearch.action.DocumentRequest;
 import org.elasticsearch.action.RealtimeRequest;
 import org.elasticsearch.action.ValidateActions;
 import org.elasticsearch.action.get.MultiGetRequest;
@@ -34,7 +34,9 @@
 import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.VersionType;
 
 import java.io.IOException;
@@ -56,7 +58,7 @@
  * Note, the {@link #index()}, {@link #type(String)} and {@link #id(String)} are
  * required.
  */
-public class TermVectorsRequest extends SingleShardRequest<TermVectorsRequest> implements DocumentRequest<TermVectorsRequest>, RealtimeRequest {
+public class TermVectorsRequest extends SingleShardRequest<TermVectorsRequest> implements RealtimeRequest {
 
     private String type;
 
@@ -64,6 +66,8 @@ public class TermVectorsRequest extends SingleShardRequest<TermVectorsRequest> i
 
     private BytesReference doc;
 
+    private XContentType xContentType;
+
     private String routing;
 
     private String parent;
@@ -157,8 +161,9 @@ public TermVectorsRequest(TermVectorsRequest other) {
         super(other.index());
         this.id = other.id();
         this.type = other.type();
-        if (this.doc != null) {
+        if (other.doc != null) {
             this.doc = new BytesArray(other.doc().toBytesRef(), true);
+            this.xContentType = other.xContentType;
         }
         this.flagsEnum = other.getFlags().clone();
         this.preference = other.preference();
@@ -180,7 +185,7 @@ public TermVectorsRequest(MultiGetRequest.Item item) {
         super(item.index());
         this.id = item.id();
         this.type = item.type();
-        this.selectedFields(item.fields());
+        this.selectedFields(item.storedFields());
         this.routing(item.routing());
         this.parent(item.parent());
     }
@@ -200,7 +205,6 @@ public TermVectorsRequest type(String type) {
     /**
      * Returns the type of document to get the term vector for.
      */
-    @Override
     public String type() {
         return type;
     }
@@ -208,7 +212,6 @@ public String type() {
     /**
      * Returns the id of document the term vector is requested for.
      */
-    @Override
     public String id() {
         return id;
     }
@@ -228,40 +231,51 @@ public BytesReference doc() {
         return doc;
     }
 
+    public XContentType xContentType() {
+        return xContentType;
+    }
+
     /**
      * Sets an artificial document from which term vectors are requested for.
      */
     public TermVectorsRequest doc(XContentBuilder documentBuilder) {
-        return this.doc(documentBuilder.bytes(), true);
+        return this.doc(documentBuilder.bytes(), true, documentBuilder.contentType());
     }
 
     /**
      * Sets an artificial document from which term vectors are requested for.
+     * @deprecated use {@link #doc(BytesReference, boolean, XContentType)} to avoid content auto detection
      */
+    @Deprecated
     public TermVectorsRequest doc(BytesReference doc, boolean generateRandomId) {
+        return this.doc(doc, generateRandomId, XContentFactory.xContentType(doc));
+    }
+
+    /**
+     * Sets an artificial document from which term vectors are requested for.
+     */
+    public TermVectorsRequest doc(BytesReference doc, boolean generateRandomId, XContentType xContentType) {
         // assign a random id to this artificial document, for routing
         if (generateRandomId) {
             this.id(String.valueOf(randomInt.getAndAdd(1)));
         }
         this.doc = doc;
+        this.xContentType = xContentType;
         return this;
     }
 
     /**
      * @return The routing for this request.
      */
-    @Override
     public String routing() {
         return routing;
     }
 
-    @Override
     public TermVectorsRequest routing(String routing) {
         this.routing = routing;
         return this;
     }
 
-    @Override
     public String parent() {
         return parent;
     }
@@ -485,6 +499,11 @@ public void readFrom(StreamInput in) throws IOException {
 
         if (in.readBoolean()) {
             doc = in.readBytesReference();
+            if (in.getVersion().onOrAfter(Version.V_5_3_0)) {
+                xContentType = XContentType.readFrom(in);
+            } else {
+                xContentType = XContentFactory.xContentType(doc);
+            }
         }
         routing = in.readOptionalString();
         parent = in.readOptionalString();
@@ -525,6 +544,9 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeBoolean(doc != null);
         if (doc != null) {
             out.writeBytesReference(doc);
+            if (out.getVersion().onOrAfter(Version.V_5_3_0)) {
+                xContentType.writeTo(out);
+            }
         }
         out.writeOptionalString(routing);
         out.writeOptionalString(parent);
@@ -555,7 +577,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeLong(version);
     }
 
-    public static enum Flag {
+    public enum Flag {
         // Do not change the order of these flags we use
         // the ordinal for encoding! Only append to the end!
         Positions, Offsets, Payloads, FieldStatistics, TermStatistics
diff --git a/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsResponse.java b/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsResponse.java
index 964aa00b5c35d..c63400be7e941 100644
--- a/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsResponse.java
@@ -36,7 +36,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.dfs.AggregatedDfs;
 
@@ -46,7 +46,7 @@
 import java.util.Iterator;
 import java.util.Set;
 
-public class TermVectorsResponse extends ActionResponse implements ToXContent {
+public class TermVectorsResponse extends ActionResponse implements ToXContentObject {
 
     private static class FieldStrings {
         // term statistics strings
@@ -174,6 +174,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         assert index != null;
         assert type != null;
         assert id != null;
+        builder.startObject();
         builder.field(FieldStrings._INDEX, index);
         builder.field(FieldStrings._TYPE, type);
         if (!isArtificial()) {
@@ -182,15 +183,15 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.field(FieldStrings._VERSION, docVersion);
         builder.field(FieldStrings.FOUND, isExists());
         builder.field(FieldStrings.TOOK, tookInMillis);
-        if (!isExists()) {
-            return builder;
-        }
-        builder.startObject(FieldStrings.TERM_VECTORS);
-        final CharsRefBuilder spare = new CharsRefBuilder();
-        Fields theFields = getFields();
-        Iterator<String> fieldIter = theFields.iterator();
-        while (fieldIter.hasNext()) {
-            buildField(builder, spare, theFields, fieldIter);
+        if (isExists()) {
+            builder.startObject(FieldStrings.TERM_VECTORS);
+            final CharsRefBuilder spare = new CharsRefBuilder();
+            Fields theFields = getFields();
+            Iterator<String> fieldIter = theFields.iterator();
+            while (fieldIter.hasNext()) {
+                buildField(builder, spare, theFields, fieldIter);
+            }
+            builder.endObject();
         }
         builder.endObject();
         return builder;
diff --git a/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsWriter.java b/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsWriter.java
index 6b5e497b8e512..06eea6367edca 100644
--- a/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsWriter.java
+++ b/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsWriter.java
@@ -71,7 +71,7 @@ void setFields(Fields termVectorsByField, Set<String> selectedFields, EnumSet<Fl
 
             // if no terms found, take the retrieved term vector fields for stats
             if (topLevelTerms == null) {
-                topLevelTerms = fieldTermVector;
+                topLevelTerms = EMPTY_TERMS;
             }
 
             TermsEnum topLevelIterator = topLevelTerms.iterator();
@@ -292,4 +292,18 @@ private void writePotentiallyNegativeVLong(long value) throws IOException {
         // further...
         output.writeVLong(Math.max(0, value + 1));
     }
+
+    /** Implements an empty {@link Terms}. */
+    private static final Terms EMPTY_TERMS = new Terms() {
+        @Override public TermsEnum iterator() throws IOException { return TermsEnum.EMPTY; }
+        @Override public long size() throws IOException { return 0; }
+        @Override public long getSumTotalTermFreq() throws IOException { return 0; }
+        @Override public long getSumDocFreq() throws IOException { return 0; }
+        @Override public int getDocCount() throws IOException { return 0; }
+        @Override public boolean hasFreqs() { return false; }
+        @Override public boolean hasOffsets() { return false; }
+        @Override public boolean hasPositions() { return false; }
+        @Override public boolean hasPayloads() { return false; }
+    };
+
 }
diff --git a/core/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java b/core/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java
index 55885b0369671..5ff55a6fa552a 100644
--- a/core/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.action.support.single.shard.TransportSingleShardAction;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.routing.ShardIterator;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
@@ -55,6 +56,13 @@ public TransportTermVectorsAction(Settings settings, ClusterService clusterServi
 
     @Override
     protected ShardIterator shards(ClusterState state, InternalRequest request) {
+        if (request.request().doc() != null && request.request().routing() == null) {
+            // artificial document without routing specified, ignore its "id" and use either random shard or according to preference
+            GroupShardsIterator<ShardIterator> groupShardsIter = clusterService.operationRouting().searchShards(state,
+                    new String[] { request.concreteIndex() }, null, request.request().preference());
+            return groupShardsIter.iterator().next();
+        }
+
         return clusterService.operationRouting().getShards(state, request.concreteIndex(), request.request().id(),
                 request.request().routing(), request.request().preference());
     }
diff --git a/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java b/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java
index d35c7bdb5847c..6a8501878e3af 100644
--- a/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java
+++ b/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java
@@ -54,7 +54,7 @@
 import org.elasticsearch.index.engine.VersionConflictEngineException;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.indices.IndexAlreadyExistsException;
+import org.elasticsearch.ResourceAlreadyExistsException;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -129,7 +129,7 @@ public void onResponse(CreateIndexResponse result) {
 
                 @Override
                 public void onFailure(Exception e) {
-                    if (unwrapCause(e) instanceof IndexAlreadyExistsException) {
+                    if (unwrapCause(e) instanceof ResourceAlreadyExistsException) {
                         // we have the index, do it
                         try {
                             innerExecute(request, listener);
@@ -176,7 +176,7 @@ protected void shardOperation(final UpdateRequest request, final ActionListener<
         final ShardId shardId = request.getShardId();
         final IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
         final IndexShard indexShard = indexService.getShard(shardId.getId());
-        final UpdateHelper.Result result = updateHelper.prepare(request, indexShard);
+        final UpdateHelper.Result result = updateHelper.prepare(request, indexShard, threadPool::absoluteTimeInMillis);
         switch (result.getResponseResult()) {
             case CREATED:
                 IndexRequest upsertRequest = result.action();
@@ -186,8 +186,10 @@ protected void shardOperation(final UpdateRequest request, final ActionListener<
                     @Override
                     public void onResponse(IndexResponse response) {
                         UpdateResponse update = new UpdateResponse(response.getShardInfo(), response.getShardId(), response.getType(), response.getId(), response.getVersion(), response.getResult());
-                        if (request.fields() != null && request.fields().length > 0) {
-                            Tuple<XContentType, Map<String, Object>> sourceAndContent = XContentHelper.convertToMap(upsertSourceBytes, true);
+                        if ((request.fetchSource() != null && request.fetchSource().fetchSource()) ||
+                            (request.fields() != null && request.fields().length > 0)) {
+                            Tuple<XContentType, Map<String, Object>> sourceAndContent =
+                                XContentHelper.convertToMap(upsertSourceBytes, true, upsertRequest.getContentType());
                             update.setGetResult(updateHelper.extractGetResult(request, request.concreteIndex(), response.getVersion(), sourceAndContent.v2(), sourceAndContent.v1(), upsertSourceBytes));
                         } else {
                             update.setGetResult(null);
diff --git a/core/src/main/java/org/elasticsearch/action/update/UpdateHelper.java b/core/src/main/java/org/elasticsearch/action/update/UpdateHelper.java
index 081cfd951b363..d2bbdc2bd8026 100644
--- a/core/src/main/java/org/elasticsearch/action/update/UpdateHelper.java
+++ b/core/src/main/java/org/elasticsearch/action/update/UpdateHelper.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.action.update;
 
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.index.IndexRequest;
@@ -27,10 +28,11 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.VersionType;
@@ -44,6 +46,7 @@
 import org.elasticsearch.index.mapper.TimestampFieldMapper;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptContext;
@@ -51,19 +54,19 @@
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 import org.elasticsearch.search.lookup.SourceLookup;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.function.LongSupplier;
 
 /**
  * Helper for translating an update request to an index, delete request or update response.
  */
 public class UpdateHelper extends AbstractComponent {
-
     private final ScriptService scriptService;
 
-    @Inject
     public UpdateHelper(Settings settings, ScriptService scriptService) {
         super(settings);
         this.scriptService = scriptService;
@@ -72,19 +75,18 @@ public UpdateHelper(Settings settings, ScriptService scriptService) {
     /**
      * Prepares an update request by converting it into an index or delete request or an update response (no action).
      */
-    @SuppressWarnings("unchecked")
-    public Result prepare(UpdateRequest request, IndexShard indexShard) {
+    public Result prepare(UpdateRequest request, IndexShard indexShard, LongSupplier nowInMillis) {
         final GetResult getResult = indexShard.getService().get(request.type(), request.id(),
                 new String[]{RoutingFieldMapper.NAME, ParentFieldMapper.NAME, TTLFieldMapper.NAME, TimestampFieldMapper.NAME},
-                true, request.version(), request.versionType(), FetchSourceContext.FETCH_SOURCE, false);
-        return prepare(indexShard.shardId(), request, getResult);
+                true, request.version(), request.versionType(), FetchSourceContext.FETCH_SOURCE);
+        return prepare(indexShard.shardId(), request, getResult, nowInMillis);
     }
 
     /**
      * Prepares an update request by converting it into an index or delete request or an update response (no action).
      */
     @SuppressWarnings("unchecked")
-    protected Result prepare(ShardId shardId, UpdateRequest request, final GetResult getResult) {
+    protected Result prepare(ShardId shardId, UpdateRequest request, final GetResult getResult, LongSupplier nowInMillis) {
         long getDateNS = System.nanoTime();
         if (!getResult.isExists()) {
             if (request.upsertRequest() == null && !request.docAsUpsert()) {
@@ -100,6 +102,7 @@ protected Result prepare(ShardId shardId, UpdateRequest request, final GetResult
                 // Tell the script that this is a create and not an update
                 ctx.put("op", "create");
                 ctx.put("_source", upsertDoc);
+                ctx.put("_now", nowInMillis.getAsLong());
                 ctx = executeScript(request.script, ctx);
                 //Allow the script to set TTL using ctx._ttl
                 if (ttl == null) {
@@ -114,7 +117,7 @@ protected Result prepare(ShardId shardId, UpdateRequest request, final GetResult
                 if (!"create".equals(scriptOpChoice)) {
                     if (!"none".equals(scriptOpChoice)) {
                         logger.warn("Used upsert operation [{}] for script [{}], doing nothing...", scriptOpChoice,
-                                request.script.getScript());
+                                request.script.getIdOrCode());
                     }
                     UpdateResponse update = new UpdateResponse(shardId, getResult.getType(), getResult.getId(),
                             getResult.getVersion(), DocWriteResponse.Result.NOOP);
@@ -131,6 +134,7 @@ protected Result prepare(ShardId shardId, UpdateRequest request, final GetResult
                     .setRefreshPolicy(request.getRefreshPolicy())
                     .routing(request.routing())
                     .parent(request.parent())
+                    .timeout(request.timeout())
                     .waitForActiveShards(request.waitForActiveShards());
             if (request.versionType() != VersionType.INTERNAL) {
                 // in all but the internal versioning mode, we want to create the new document using the given version.
@@ -139,7 +143,12 @@ protected Result prepare(ShardId shardId, UpdateRequest request, final GetResult
             return new Result(indexRequest, DocWriteResponse.Result.CREATED, null, null);
         }
 
-        final long updateVersion = getResult.getVersion();
+        long updateVersion = getResult.getVersion();
+
+        if (request.versionType() != VersionType.INTERNAL) {
+            assert request.versionType() == VersionType.FORCE;
+            updateVersion = request.version(); // remember, match_any is excluded by the conflict test
+        }
 
         if (getResult.internalSourceRef() == null) {
             // no source, we can't do nothing, through a failure...
@@ -188,6 +197,7 @@ protected Result prepare(ShardId shardId, UpdateRequest request, final GetResult
             ctx.put("_timestamp", originalTimestamp);
             ctx.put("_ttl", originalTtl);
             ctx.put("_source", sourceAndContent.v2());
+            ctx.put("_now", nowInMillis.getAsLong());
 
             ctx = executeScript(request.script, ctx);
 
@@ -221,12 +231,14 @@ protected Result prepare(ShardId shardId, UpdateRequest request, final GetResult
                     .version(updateVersion).versionType(request.versionType())
                     .waitForActiveShards(request.waitForActiveShards())
                     .timestamp(timestamp).ttl(ttl)
+                    .timeout(request.timeout())
                     .setRefreshPolicy(request.getRefreshPolicy());
             return new Result(indexRequest, DocWriteResponse.Result.UPDATED, updatedSourceAsMap, updateSourceContentType);
         } else if ("delete".equals(operation)) {
             DeleteRequest deleteRequest = Requests.deleteRequest(request.index()).type(request.type()).id(request.id()).routing(routing).parent(parent)
                     .version(updateVersion).versionType(request.versionType())
                     .waitForActiveShards(request.waitForActiveShards())
+                    .timeout(request.timeout())
                     .setRefreshPolicy(request.getRefreshPolicy());
             return new Result(deleteRequest, DocWriteResponse.Result.DELETED, updatedSourceAsMap, updateSourceContentType);
         } else if ("none".equals(operation)) {
@@ -234,7 +246,7 @@ protected Result prepare(ShardId shardId, UpdateRequest request, final GetResult
             update.setGetResult(extractGetResult(request, request.index(), getResult.getVersion(), updatedSourceAsMap, updateSourceContentType, getResult.internalSourceRef()));
             return new Result(update, DocWriteResponse.Result.NOOP, updatedSourceAsMap, updateSourceContentType);
         } else {
-            logger.warn("Used update operation [{}] for script [{}], doing nothing...", operation, request.script.getScript());
+            logger.warn("Used update operation [{}] for script [{}], doing nothing...", operation, request.script.getIdOrCode());
             UpdateResponse update = new UpdateResponse(shardId, getResult.getType(), getResult.getId(), getResult.getVersion(), DocWriteResponse.Result.NOOP);
             return new Result(update, DocWriteResponse.Result.NOOP, updatedSourceAsMap, updateSourceContentType);
         }
@@ -243,7 +255,8 @@ protected Result prepare(ShardId shardId, UpdateRequest request, final GetResult
     private Map<String, Object> executeScript(Script script, Map<String, Object> ctx) {
         try {
             if (scriptService != null) {
-                ExecutableScript executableScript = scriptService.executable(script, ScriptContext.Standard.UPDATE, Collections.emptyMap());
+                CompiledScript compiledScript = scriptService.compile(script, ScriptContext.Standard.UPDATE);
+                ExecutableScript executableScript = scriptService.executable(compiledScript, script.getParams());
                 executableScript.setNextVar("ctx", ctx);
                 executableScript.run();
                 // we need to unwrap the ctx...
@@ -267,17 +280,19 @@ private TimeValue getTTLFromScriptContext(Map<String, Object> ctx) {
     }
 
     /**
-     * Extracts the fields from the updated document to be returned in a update response
+     * Applies {@link UpdateRequest#fetchSource()} to the _source of the updated document to be returned in a update response.
+     * For BWC this function also extracts the {@link UpdateRequest#fields()} from the updated document to be returned in a update response
      */
     public GetResult extractGetResult(final UpdateRequest request, String concreteIndex, long version, final Map<String, Object> source, XContentType sourceContentType, @Nullable final BytesReference sourceAsBytes) {
-        if (request.fields() == null || request.fields().length == 0) {
+        if ((request.fields() == null || request.fields().length == 0) &&
+            (request.fetchSource() == null || request.fetchSource().fetchSource() == false)) {
             return null;
         }
+        SourceLookup sourceLookup = new SourceLookup();
+        sourceLookup.setSource(source);
         boolean sourceRequested = false;
         Map<String, GetField> fields = null;
         if (request.fields() != null && request.fields().length > 0) {
-            SourceLookup sourceLookup = new SourceLookup();
-            sourceLookup.setSource(source);
             for (String field : request.fields()) {
                 if (field.equals("_source")) {
                     sourceRequested = true;
@@ -298,8 +313,26 @@ public GetResult extractGetResult(final UpdateRequest request, String concreteIn
             }
         }
 
+        BytesReference sourceFilteredAsBytes = sourceAsBytes;
+        if (request.fetchSource() != null && request.fetchSource().fetchSource()) {
+            sourceRequested = true;
+            if (request.fetchSource().includes().length > 0 || request.fetchSource().excludes().length > 0) {
+                Object value = sourceLookup.filter(request.fetchSource());
+                try {
+                    final int initialCapacity = Math.min(1024, sourceAsBytes.length());
+                    BytesStreamOutput streamOutput = new BytesStreamOutput(initialCapacity);
+                    try (XContentBuilder builder = new XContentBuilder(sourceContentType.xContent(), streamOutput)) {
+                        builder.value(value);
+                        sourceFilteredAsBytes = builder.bytes();
+                    }
+                } catch (IOException e) {
+                    throw new ElasticsearchException("Error filtering source", e);
+                }
+            }
+        }
+
         // TODO when using delete/none, we can still return the source as bytes by generating it (using the sourceContentType)
-        return new GetResult(concreteIndex, request.type(), request.id(), version, true, sourceRequested ? sourceAsBytes : null, fields);
+        return new GetResult(concreteIndex, request.type(), request.id(), version, true, sourceRequested ? sourceFilteredAsBytes : null, fields);
     }
 
     public static class Result {
diff --git a/core/src/main/java/org/elasticsearch/action/update/UpdateRequest.java b/core/src/main/java/org/elasticsearch/action/update/UpdateRequest.java
index 3f2dde6784aea..416b9e2c68508 100644
--- a/core/src/main/java/org/elasticsearch/action/update/UpdateRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/update/UpdateRequest.java
@@ -20,28 +20,30 @@
 package org.elasticsearch.action.update;
 
 import org.elasticsearch.action.ActionRequestValidationException;
-import org.elasticsearch.action.DocumentRequest;
+import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.action.support.replication.ReplicationRequest;
 import org.elasticsearch.action.support.single.instance.InstanceShardOperationRequest;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.lucene.uid.Versions;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.script.ScriptType;
+import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -54,7 +56,7 @@
 /**
  */
 public class UpdateRequest extends InstanceShardOperationRequest<UpdateRequest>
-        implements DocumentRequest<UpdateRequest>, WriteRequest<UpdateRequest> {
+        implements DocWriteRequest<UpdateRequest>, WriteRequest<UpdateRequest>, ToXContentObject {
 
     private String type;
     private String id;
@@ -68,6 +70,7 @@ public class UpdateRequest extends InstanceShardOperationRequest<UpdateRequest>
     Script script;
 
     private String[] fields;
+    private FetchSourceContext fetchSourceContext;
 
     private long version = Versions.MATCH_ANY;
     private VersionType versionType = VersionType.INTERNAL;
@@ -82,6 +85,7 @@ public class UpdateRequest extends InstanceShardOperationRequest<UpdateRequest>
     private boolean scriptedUpsert = false;
     private boolean docAsUpsert = false;
     private boolean detectNoop = true;
+    private static DeprecationLogger deprecationLogger = new DeprecationLogger(Loggers.getLogger(UpdateRequest.class));
 
     @Nullable
     private IndexRequest doc;
@@ -106,9 +110,8 @@ public ActionRequestValidationException validate() {
             validationException = addValidationError("id is missing", validationException);
         }
 
-        if (versionType != VersionType.INTERNAL) {
-            validationException = addValidationError("version type [" + versionType + "] is not supported by the update API",
-                    validationException);
+        if (!(versionType == VersionType.INTERNAL || versionType == VersionType.FORCE)) {
+            validationException = addValidationError("version type [" + versionType + "] is not supported by the update API", validationException);
         } else {
 
             if (version != Versions.MATCH_ANY && retryOnConflict > 0) {
@@ -129,6 +132,9 @@ public ActionRequestValidationException validate() {
         if (doc == null && docAsUpsert) {
             validationException = addValidationError("doc must be specified if doc_as_upsert is enabled", validationException);
         }
+        if (versionType == VersionType.FORCE) {
+            deprecationLogger.deprecated("version type FORCE is deprecated and will be removed in the next major version");
+        }
         return validationException;
     }
 
@@ -221,14 +227,14 @@ public UpdateRequest script(Script script) {
      */
     @Deprecated
     public String scriptString() {
-        return this.script == null ? null : this.script.getScript();
+        return this.script == null ? null : this.script.getIdOrCode();
     }
 
     /**
      * @deprecated Use {@link #script()} instead
      */
     @Deprecated
-    public ScriptService.ScriptType scriptType() {
+    public ScriptType scriptType() {
         return this.script == null ? null : this.script.getType();
     }
 
@@ -248,7 +254,7 @@ public Map<String, Object> scriptParams() {
      * @deprecated Use {@link #script(Script)} instead
      */
     @Deprecated
-    public UpdateRequest script(String script, ScriptService.ScriptType scriptType) {
+    public UpdateRequest script(String script, ScriptType scriptType) {
         updateOrCreateScript(script, scriptType, null, null);
         return this;
     }
@@ -324,13 +330,13 @@ public UpdateRequest scriptParams(Map<String, Object> scriptParams) {
     private void updateOrCreateScript(String scriptContent, ScriptType type, String lang, Map<String, Object> params) {
         Script script = script();
         if (script == null) {
-            script = new Script(scriptContent == null ? "" : scriptContent, type == null ? ScriptType.INLINE : type, lang, params);
+            script = new Script(type == null ? ScriptType.INLINE : type, lang, scriptContent == null ? "" : scriptContent, params);
         } else {
-            String newScriptContent = scriptContent == null ? script.getScript() : scriptContent;
+            String newScriptContent = scriptContent == null ? script.getIdOrCode() : scriptContent;
             ScriptType newScriptType = type == null ? script.getType() : type;
             String newScriptLang = lang == null ? script.getLang() : lang;
             Map<String, Object> newScriptParams = params == null ? script.getParams() : params;
-            script = new Script(newScriptContent, newScriptType, newScriptLang, newScriptParams);
+            script = new Script(newScriptType, newScriptLang, newScriptContent, newScriptParams);
         }
         script(script);
     }
@@ -343,8 +349,8 @@ private void updateOrCreateScript(String scriptContent, ScriptType type, String
      * @deprecated Use {@link #script(Script)} instead
      */
     @Deprecated
-    public UpdateRequest script(String script, ScriptService.ScriptType scriptType, @Nullable Map<String, Object> scriptParams) {
-        this.script = new Script(script, scriptType, null, scriptParams);
+    public UpdateRequest script(String script, ScriptType scriptType, @Nullable Map<String, Object> scriptParams) {
+        this.script = new Script(scriptType, Script.DEFAULT_SCRIPT_LANG, script, scriptParams);
         return this;
     }
 
@@ -365,25 +371,91 @@ public UpdateRequest script(String script, ScriptService.ScriptType scriptType,
      * @deprecated Use {@link #script(Script)} instead
      */
     @Deprecated
-    public UpdateRequest script(String script, @Nullable String scriptLang, ScriptService.ScriptType scriptType,
+    public UpdateRequest script(String script, @Nullable String scriptLang, ScriptType scriptType,
             @Nullable Map<String, Object> scriptParams) {
-        this.script = new Script(script, scriptType, scriptLang, scriptParams);
+        this.script = new Script(scriptType, scriptLang, script, scriptParams);
         return this;
     }
 
     /**
      * Explicitly specify the fields that will be returned. By default, nothing is returned.
+     * @deprecated Use {@link UpdateRequest#fetchSource(String[], String[])} instead
      */
+    @Deprecated
     public UpdateRequest fields(String... fields) {
         this.fields = fields;
         return this;
     }
 
+    /**
+     * Indicate that _source should be returned with every hit, with an
+     * "include" and/or "exclude" set which can include simple wildcard
+     * elements.
+     *
+     * @param include
+     *            An optional include (optionally wildcarded) pattern to filter
+     *            the returned _source
+     * @param exclude
+     *            An optional exclude (optionally wildcarded) pattern to filter
+     *            the returned _source
+     */
+    public UpdateRequest fetchSource(@Nullable String include, @Nullable String exclude) {
+        FetchSourceContext context = this.fetchSourceContext == null ? FetchSourceContext.FETCH_SOURCE : this.fetchSourceContext;
+        this.fetchSourceContext = new FetchSourceContext(context.fetchSource(), new String[] {include}, new String[]{exclude});
+        return this;
+    }
+
+    /**
+     * Indicate that _source should be returned, with an
+     * "include" and/or "exclude" set which can include simple wildcard
+     * elements.
+     *
+     * @param includes
+     *            An optional list of include (optionally wildcarded) pattern to
+     *            filter the returned _source
+     * @param excludes
+     *            An optional list of exclude (optionally wildcarded) pattern to
+     *            filter the returned _source
+     */
+    public UpdateRequest fetchSource(@Nullable String[] includes, @Nullable String[] excludes) {
+        FetchSourceContext context = this.fetchSourceContext == null ? FetchSourceContext.FETCH_SOURCE : this.fetchSourceContext;
+        this.fetchSourceContext = new FetchSourceContext(context.fetchSource(), includes, excludes);
+        return this;
+    }
+
+    /**
+     * Indicates whether the response should contain the updated _source.
+     */
+    public UpdateRequest fetchSource(boolean fetchSource) {
+        FetchSourceContext context = this.fetchSourceContext == null ? FetchSourceContext.FETCH_SOURCE : this.fetchSourceContext;
+        this.fetchSourceContext = new FetchSourceContext(fetchSource, context.includes(), context.excludes());
+        return this;
+    }
+
+    /**
+     * Explicitely set the fetch source context for this request
+     */
+    public UpdateRequest fetchSource(FetchSourceContext context) {
+        this.fetchSourceContext = context;
+        return this;
+    }
+
+
     /**
      * Get the fields to be returned.
+     * @deprecated Use {@link UpdateRequest#fetchSource()} instead
      */
+    @Deprecated
     public String[] fields() {
-        return this.fields;
+        return fields;
+    }
+
+    /**
+     * Gets the {@link FetchSourceContext} which defines how the _source should
+     * be fetched.
+     */
+    public FetchSourceContext fetchSource() {
+        return fetchSourceContext;
     }
 
     /**
@@ -399,31 +471,33 @@ public int retryOnConflict() {
         return this.retryOnConflict;
     }
 
-    /**
-     * Sets the version, which will cause the index operation to only be performed if a matching
-     * version exists and no changes happened on the doc since then.
-     */
+    @Override
     public UpdateRequest version(long version) {
         this.version = version;
         return this;
     }
 
+    @Override
     public long version() {
         return this.version;
     }
 
-    /**
-     * Sets the versioning type. Defaults to {@link VersionType#INTERNAL}.
-     */
+    @Override
     public UpdateRequest versionType(VersionType versionType) {
         this.versionType = versionType;
         return this;
     }
 
+    @Override
     public VersionType versionType() {
         return this.versionType;
     }
 
+    @Override
+    public OpType opType() {
+        return OpType.UPDATE;
+    }
+
     @Override
     public UpdateRequest setRefreshPolicy(RefreshPolicy refreshPolicy) {
         this.refreshPolicy = refreshPolicy;
@@ -491,7 +565,9 @@ public UpdateRequest doc(Map source, XContentType contentType) {
 
     /**
      * Sets the doc to use for updates when a script is not specified.
+     * @deprecated use {@link #doc(String, XContentType)}
      */
+    @Deprecated
     public UpdateRequest doc(String source) {
         safeDoc().source(source);
         return this;
@@ -500,6 +576,16 @@ public UpdateRequest doc(String source) {
     /**
      * Sets the doc to use for updates when a script is not specified.
      */
+    public UpdateRequest doc(String source, XContentType xContentType) {
+        safeDoc().source(source, xContentType);
+        return this;
+    }
+
+    /**
+     * Sets the doc to use for updates when a script is not specified.
+     * @deprecated use {@link #doc(byte[], XContentType)}
+     */
+    @Deprecated
     public UpdateRequest doc(byte[] source) {
         safeDoc().source(source);
         return this;
@@ -508,11 +594,29 @@ public UpdateRequest doc(byte[] source) {
     /**
      * Sets the doc to use for updates when a script is not specified.
      */
+    public UpdateRequest doc(byte[] source, XContentType xContentType) {
+        safeDoc().source(source, xContentType);
+        return this;
+    }
+
+    /**
+     * Sets the doc to use for updates when a script is not specified.
+     * @deprecated use {@link #doc(byte[], int, int, XContentType)}
+     */
+    @Deprecated
     public UpdateRequest doc(byte[] source, int offset, int length) {
         safeDoc().source(source, offset, length);
         return this;
     }
 
+    /**
+     * Sets the doc to use for updates when a script is not specified.
+     */
+    public UpdateRequest doc(byte[] source, int offset, int length, XContentType xContentType) {
+        safeDoc().source(source, offset, length, xContentType);
+        return this;
+    }
+
     /**
      * Sets the doc to use for updates when a script is not specified, the doc provided
      * is a field and value pairs.
@@ -523,10 +627,11 @@ public UpdateRequest doc(Object... source) {
     }
 
     /**
-     * Sets the doc to use for updates when a script is not specified.
+     * Sets the doc to use for updates when a script is not specified, the doc provided
+     * is a field and value pairs.
      */
-    public UpdateRequest doc(String field, Object value) {
-        safeDoc().source(field, value);
+    public UpdateRequest doc(XContentType xContentType, Object... source) {
+        safeDoc().source(xContentType, source);
         return this;
     }
 
@@ -576,7 +681,9 @@ public UpdateRequest upsert(Map source, XContentType contentType) {
 
     /**
      * Sets the doc source of the update request to be used when the document does not exists.
+     * @deprecated use {@link #upsert(String, XContentType)}
      */
+    @Deprecated
     public UpdateRequest upsert(String source) {
         safeUpsertRequest().source(source);
         return this;
@@ -585,6 +692,16 @@ public UpdateRequest upsert(String source) {
     /**
      * Sets the doc source of the update request to be used when the document does not exists.
      */
+    public UpdateRequest upsert(String source, XContentType xContentType) {
+        safeUpsertRequest().source(source, xContentType);
+        return this;
+    }
+
+    /**
+     * Sets the doc source of the update request to be used when the document does not exists.
+     * @deprecated use {@link #upsert(byte[], XContentType)}
+     */
+    @Deprecated
     public UpdateRequest upsert(byte[] source) {
         safeUpsertRequest().source(source);
         return this;
@@ -593,11 +710,29 @@ public UpdateRequest upsert(byte[] source) {
     /**
      * Sets the doc source of the update request to be used when the document does not exists.
      */
+    public UpdateRequest upsert(byte[] source, XContentType xContentType) {
+        safeUpsertRequest().source(source, xContentType);
+        return this;
+    }
+
+    /**
+     * Sets the doc source of the update request to be used when the document does not exists.
+     * @deprecated use {@link #upsert(byte[], int, int, XContentType)}
+     */
+    @Deprecated
     public UpdateRequest upsert(byte[] source, int offset, int length) {
         safeUpsertRequest().source(source, offset, length);
         return this;
     }
 
+    /**
+     * Sets the doc source of the update request to be used when the document does not exists.
+     */
+    public UpdateRequest upsert(byte[] source, int offset, int length, XContentType xContentType) {
+        safeUpsertRequest().source(source, offset, length, xContentType);
+        return this;
+    }
+
     /**
      * Sets the doc source of the update request to be used when the document does not exists. The doc
      * includes field and value pairs.
@@ -607,6 +742,15 @@ public UpdateRequest upsert(Object... source) {
         return this;
     }
 
+    /**
+     * Sets the doc source of the update request to be used when the document does not exists. The doc
+     * includes field and value pairs.
+     */
+    public UpdateRequest upsert(XContentType xContentType, Object... source) {
+        safeUpsertRequest().source(xContentType, source);
+        return this;
+    }
+
     public IndexRequest upsertRequest() {
         return this.upsertRequest;
     }
@@ -618,18 +762,6 @@ private IndexRequest safeUpsertRequest() {
         return upsertRequest;
     }
 
-    public UpdateRequest source(XContentBuilder source) throws Exception {
-        return source(source.bytes());
-    }
-
-    public UpdateRequest source(byte[] source) throws Exception {
-        return source(source, 0, source.length);
-    }
-
-    public UpdateRequest source(byte[] source, int offset, int length) throws Exception {
-        return source(new BytesArray(source, offset, length));
-    }
-
     /**
      * Should this update attempt to detect if it is a noop? Defaults to true.
      * @return this for chaining
@@ -646,51 +778,49 @@ public boolean detectNoop() {
         return detectNoop;
     }
 
-    public UpdateRequest source(BytesReference source) throws Exception {
+    public UpdateRequest fromXContent(XContentParser parser) throws IOException {
         Script script = null;
-        try (XContentParser parser = XContentFactory.xContent(source).createParser(source)) {
-            XContentParser.Token token = parser.nextToken();
-            if (token == null) {
-                return this;
-            }
-            String currentFieldName = null;
-            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                if (token == XContentParser.Token.FIELD_NAME) {
-                    currentFieldName = parser.currentName();
-                } else if ("script".equals(currentFieldName)) {
-                    script = Script.parse(parser, ParseFieldMatcher.EMPTY);
-                } else if ("scripted_upsert".equals(currentFieldName)) {
-                    scriptedUpsert = parser.booleanValue();
-                } else if ("upsert".equals(currentFieldName)) {
-                    XContentType xContentType = XContentFactory.xContentType(source);
-                    XContentBuilder builder = XContentFactory.contentBuilder(xContentType);
-                    builder.copyCurrentStructure(parser);
-                    safeUpsertRequest().source(builder);
-                } else if ("doc".equals(currentFieldName)) {
-                    XContentType xContentType = XContentFactory.xContentType(source);
-                    XContentBuilder docBuilder = XContentFactory.contentBuilder(xContentType);
-                    docBuilder.copyCurrentStructure(parser);
-                    safeDoc().source(docBuilder);
-                } else if ("doc_as_upsert".equals(currentFieldName)) {
-                    docAsUpsert(parser.booleanValue());
-                } else if ("detect_noop".equals(currentFieldName)) {
-                    detectNoop(parser.booleanValue());
-                } else if ("fields".equals(currentFieldName)) {
-                    List<Object> fields = null;
-                    if (token == XContentParser.Token.START_ARRAY) {
-                        fields = (List) parser.list();
-                    } else if (token.isValue()) {
-                        fields = Collections.singletonList(parser.text());
-                    }
-                    if (fields != null) {
-                        fields(fields.toArray(new String[fields.size()]));
-                    }
+        XContentParser.Token token = parser.nextToken();
+        if (token == null) {
+            return this;
+        }
+        String currentFieldName = null;
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if ("script".equals(currentFieldName)) {
+                script = Script.parse(parser);
+            } else if ("scripted_upsert".equals(currentFieldName)) {
+                scriptedUpsert = parser.booleanValue();
+            } else if ("upsert".equals(currentFieldName)) {
+                XContentBuilder builder = XContentFactory.contentBuilder(parser.contentType());
+                builder.copyCurrentStructure(parser);
+                safeUpsertRequest().source(builder);
+            } else if ("doc".equals(currentFieldName)) {
+                XContentBuilder docBuilder = XContentFactory.contentBuilder(parser.contentType());
+                docBuilder.copyCurrentStructure(parser);
+                safeDoc().source(docBuilder);
+            } else if ("doc_as_upsert".equals(currentFieldName)) {
+                docAsUpsert(parser.booleanValue());
+            } else if ("detect_noop".equals(currentFieldName)) {
+                detectNoop(parser.booleanValue());
+            } else if ("fields".equals(currentFieldName)) {
+                List<Object> fields = null;
+                if (token == XContentParser.Token.START_ARRAY) {
+                    fields = (List) parser.list();
+                } else if (token.isValue()) {
+                    fields = Collections.singletonList(parser.text());
                 }
-            }
-            if (script != null) {
-                this.script = script;
+                if (fields != null) {
+                    fields(fields.toArray(new String[fields.size()]));
+                }
+            } else if ("_source".equals(currentFieldName)) {
+                fetchSourceContext = FetchSourceContext.fromXContent(parser);
             }
         }
+        if (script != null) {
+            this.script = script;
+        }
         return this;
     }
 
@@ -729,13 +859,8 @@ public void readFrom(StreamInput in) throws IOException {
             doc = new IndexRequest();
             doc.readFrom(in);
         }
-        int size = in.readInt();
-        if (size >= 0) {
-            fields = new String[size];
-            for (int i = 0; i < size; i++) {
-                fields[i] = in.readString();
-            }
-        }
+        fields = in.readOptionalStringArray();
+        fetchSourceContext = in.readOptionalWriteable(FetchSourceContext::new);
         if (in.readBoolean()) {
             upsertRequest = new IndexRequest();
             upsertRequest.readFrom(in);
@@ -772,14 +897,8 @@ public void writeTo(StreamOutput out) throws IOException {
             doc.id(id);
             doc.writeTo(out);
         }
-        if (fields == null) {
-            out.writeInt(-1);
-        } else {
-            out.writeInt(fields.length);
-            for (String field : fields) {
-                out.writeString(field);
-            }
-        }
+        out.writeOptionalStringArray(fields);
+        out.writeOptionalWriteable(fetchSourceContext);
         if (upsertRequest == null) {
             out.writeBoolean(false);
         } else {
@@ -797,4 +916,42 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeBoolean(scriptedUpsert);
     }
 
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        if (docAsUpsert) {
+            builder.field("doc_as_upsert", docAsUpsert);
+        }
+        if (doc != null) {
+            XContentType xContentType = doc.getContentType();
+            try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, doc.source(), xContentType)) {
+                builder.field("doc");
+                builder.copyCurrentStructure(parser);
+            }
+        }
+        if (script != null) {
+            builder.field("script", script);
+        }
+        if (upsertRequest != null) {
+            XContentType xContentType = upsertRequest.getContentType();
+            try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, upsertRequest.source(), xContentType)) {
+                builder.field("upsert");
+                builder.copyCurrentStructure(parser);
+            }
+        }
+        if (scriptedUpsert) {
+            builder.field("scripted_upsert", scriptedUpsert);
+        }
+        if (detectNoop == false) {
+            builder.field("detect_noop", detectNoop);
+        }
+        if (fields != null) {
+            builder.array("fields", fields);
+        }
+        if (fetchSourceContext != null) {
+            builder.field("_source", fetchSourceContext);
+        }
+        builder.endObject();
+        return builder;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java
index f2d80bfe66e8f..2846ac4977fc5 100644
--- a/core/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java
@@ -25,17 +25,22 @@
 import org.elasticsearch.action.support.replication.ReplicationRequest;
 import org.elasticsearch.action.support.single.instance.InstanceShardOperationRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
-import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.VersionType;
+import org.elasticsearch.rest.action.document.RestUpdateAction;
 import org.elasticsearch.script.Script;
 
 import java.util.Map;
 
 public class UpdateRequestBuilder extends InstanceShardOperationRequestBuilder<UpdateRequest, UpdateResponse, UpdateRequestBuilder>
         implements WriteRequestBuilder<UpdateRequestBuilder> {
+    private static final DeprecationLogger DEPRECATION_LOGGER =
+        new DeprecationLogger(Loggers.getLogger(RestUpdateAction.class));
 
     public UpdateRequestBuilder(ElasticsearchClient client, UpdateAction action) {
         super(client, action, new UpdateRequest());
@@ -90,12 +95,57 @@ public UpdateRequestBuilder setScript(Script script) {
 
     /**
      * Explicitly specify the fields that will be returned. By default, nothing is returned.
+     * @deprecated Use {@link UpdateRequestBuilder#setFetchSource(String[], String[])} instead
      */
+    @Deprecated
     public UpdateRequestBuilder setFields(String... fields) {
+        DEPRECATION_LOGGER.deprecated("Deprecated field [fields] used, expected [_source] instead");
         request.fields(fields);
         return this;
     }
 
+    /**
+     * Indicate that _source should be returned with every hit, with an
+     * "include" and/or "exclude" set which can include simple wildcard
+     * elements.
+     *
+     * @param include
+     *            An optional include (optionally wildcarded) pattern to filter
+     *            the returned _source
+     * @param exclude
+     *            An optional exclude (optionally wildcarded) pattern to filter
+     *            the returned _source
+     */
+    public UpdateRequestBuilder setFetchSource(@Nullable String include, @Nullable String exclude) {
+        request.fetchSource(include, exclude);
+        return this;
+    }
+
+    /**
+     * Indicate that _source should be returned, with an
+     * "include" and/or "exclude" set which can include simple wildcard
+     * elements.
+     *
+     * @param includes
+     *            An optional list of include (optionally wildcarded) pattern to
+     *            filter the returned _source
+     * @param excludes
+     *            An optional list of exclude (optionally wildcarded) pattern to
+     *            filter the returned _source
+     */
+    public UpdateRequestBuilder setFetchSource(@Nullable String[] includes, @Nullable String[] excludes) {
+        request.fetchSource(includes, excludes);
+        return this;
+    }
+
+    /**
+     * Indicates whether the response should contain the updated _source.
+     */
+    public UpdateRequestBuilder setFetchSource(boolean fetchSource) {
+        request.fetchSource(fetchSource);
+        return this;
+    }
+
     /**
      * Sets the number of retries of a version conflict occurs because the document was updated between
      * getting it and updating it. Defaults to 0.
@@ -174,7 +224,9 @@ public UpdateRequestBuilder setDoc(Map source, XContentType contentType) {
 
     /**
      * Sets the doc to use for updates when a script is not specified.
+     * @deprecated use {@link #setDoc(String, XContentType)}
      */
+    @Deprecated
     public UpdateRequestBuilder setDoc(String source) {
         request.doc(source);
         return this;
@@ -183,6 +235,16 @@ public UpdateRequestBuilder setDoc(String source) {
     /**
      * Sets the doc to use for updates when a script is not specified.
      */
+    public UpdateRequestBuilder setDoc(String source, XContentType xContentType) {
+        request.doc(source, xContentType);
+        return this;
+    }
+
+    /**
+     * Sets the doc to use for updates when a script is not specified.
+     * @deprecated use {@link #setDoc(byte[], XContentType)}
+     */
+    @Deprecated
     public UpdateRequestBuilder setDoc(byte[] source) {
         request.doc(source);
         return this;
@@ -191,6 +253,16 @@ public UpdateRequestBuilder setDoc(byte[] source) {
     /**
      * Sets the doc to use for updates when a script is not specified.
      */
+    public UpdateRequestBuilder setDoc(byte[] source, XContentType xContentType) {
+        request.doc(source, xContentType);
+        return this;
+    }
+
+    /**
+     * Sets the doc to use for updates when a script is not specified.
+     * @deprecated use {@link #setDoc(byte[], int, int, XContentType)}
+     */
+    @Deprecated
     public UpdateRequestBuilder setDoc(byte[] source, int offset, int length) {
         request.doc(source, offset, length);
         return this;
@@ -199,8 +271,8 @@ public UpdateRequestBuilder setDoc(byte[] source, int offset, int length) {
     /**
      * Sets the doc to use for updates when a script is not specified.
      */
-    public UpdateRequestBuilder setDoc(String field, Object value) {
-        request.doc(field, value);
+    public UpdateRequestBuilder setDoc(byte[] source, int offset, int length, XContentType xContentType) {
+        request.doc(source, offset, length, xContentType);
         return this;
     }
 
@@ -213,6 +285,15 @@ public UpdateRequestBuilder setDoc(Object... source) {
         return this;
     }
 
+    /**
+     * Sets the doc to use for updates when a script is not specified, the doc provided
+     * is a field and value pairs.
+     */
+    public UpdateRequestBuilder setDoc(XContentType xContentType, Object... source) {
+        request.doc(xContentType, source);
+        return this;
+    }
+
     /**
      * Sets the index request to be used if the document does not exists. Otherwise, a {@link org.elasticsearch.index.engine.DocumentMissingException}
      * is thrown.
@@ -248,7 +329,9 @@ public UpdateRequestBuilder setUpsert(Map source, XContentType contentType) {
 
     /**
      * Sets the doc source of the update request to be used when the document does not exists.
+     * @deprecated use {@link #setUpsert(String, XContentType)}
      */
+    @Deprecated
     public UpdateRequestBuilder setUpsert(String source) {
         request.upsert(source);
         return this;
@@ -257,45 +340,62 @@ public UpdateRequestBuilder setUpsert(String source) {
     /**
      * Sets the doc source of the update request to be used when the document does not exists.
      */
-    public UpdateRequestBuilder setUpsert(byte[] source) {
-        request.upsert(source);
+    public UpdateRequestBuilder setUpsert(String source, XContentType xContentType) {
+        request.upsert(source, xContentType);
         return this;
     }
 
     /**
      * Sets the doc source of the update request to be used when the document does not exists.
+     * @deprecated use {@link #setDoc(byte[], XContentType)}
      */
-    public UpdateRequestBuilder setUpsert(byte[] source, int offset, int length) {
-        request.upsert(source, offset, length);
+    @Deprecated
+    public UpdateRequestBuilder setUpsert(byte[] source) {
+        request.upsert(source);
         return this;
     }
 
     /**
-     * Sets the doc source of the update request to be used when the document does not exists. The doc
-     * includes field and value pairs.
+     * Sets the doc source of the update request to be used when the document does not exists.
      */
-    public UpdateRequestBuilder setUpsert(Object... source) {
-        request.upsert(source);
+    public UpdateRequestBuilder setUpsert(byte[] source, XContentType xContentType) {
+        request.upsert(source, xContentType);
         return this;
     }
 
-    public UpdateRequestBuilder setSource(XContentBuilder source) throws Exception {
-        request.source(source);
+    /**
+     * Sets the doc source of the update request to be used when the document does not exists.
+     * @deprecated use {@link #setUpsert(byte[], int, int, XContentType)}
+     */
+    @Deprecated
+    public UpdateRequestBuilder setUpsert(byte[] source, int offset, int length) {
+        request.upsert(source, offset, length);
         return this;
     }
 
-    public UpdateRequestBuilder setSource(byte[] source) throws Exception {
-        request.source(source);
+    /**
+     * Sets the doc source of the update request to be used when the document does not exists.
+     */
+    public UpdateRequestBuilder setUpsert(byte[] source, int offset, int length, XContentType xContentType) {
+        request.upsert(source, offset, length, xContentType);
         return this;
     }
 
-    public UpdateRequestBuilder setSource(byte[] source, int offset, int length) throws Exception {
-        request.source(source, offset, length);
+    /**
+     * Sets the doc source of the update request to be used when the document does not exists. The doc
+     * includes field and value pairs.
+     */
+    public UpdateRequestBuilder setUpsert(Object... source) {
+        request.upsert(source);
         return this;
     }
 
-    public UpdateRequestBuilder setSource(BytesReference source) throws Exception {
-        request.source(source);
+    /**
+     * Sets the doc source of the update request to be used when the document does not exists. The doc
+     * includes field and value pairs.
+     */
+    public UpdateRequestBuilder setUpsert(XContentType xContentType, Object... source) {
+        request.upsert(xContentType, source);
         return this;
     }
 
@@ -330,6 +430,7 @@ public UpdateRequestBuilder setScriptedUpsert(boolean scriptedUpsert) {
      * and the source of the document isn't changed then the ttl update won't take
      * effect.
      */
+    @Deprecated
     public UpdateRequestBuilder setTtl(Long ttl) {
         request.doc().ttl(ttl);
         return this;
@@ -340,6 +441,7 @@ public UpdateRequestBuilder setTtl(Long ttl) {
      * and the source of the document isn't changed then the ttl update won't take
      * effect.
      */
+    @Deprecated
     public UpdateRequestBuilder setTtl(String ttl) {
         request.doc().ttl(ttl);
         return this;
@@ -350,6 +452,7 @@ public UpdateRequestBuilder setTtl(String ttl) {
      * and the source of the document isn't changed then the ttl update won't take
      * effect.
      */
+    @Deprecated
     public UpdateRequestBuilder setTtl(TimeValue ttl) {
         request.doc().ttl(ttl);
         return this;
diff --git a/core/src/main/java/org/elasticsearch/action/update/UpdateResponse.java b/core/src/main/java/org/elasticsearch/action/update/UpdateResponse.java
index 8061174d091a1..bc7ed479eaab3 100644
--- a/core/src/main/java/org/elasticsearch/action/update/UpdateResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/update/UpdateResponse.java
@@ -23,14 +23,19 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.get.GetResult;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
 
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+
 public class UpdateResponse extends DocWriteResponse {
 
+    private static final String GET = "get";
+
     private GetResult getResult;
 
     public UpdateResponse() {
@@ -82,16 +87,11 @@ public void writeTo(StreamOutput out) throws IOException {
         }
     }
 
-
-    static final class Fields {
-        static final String GET = "get";
-    }
-
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        super.toXContent(builder, params);
+    public XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
+        super.innerToXContent(builder, params);
         if (getGetResult() != null) {
-            builder.startObject(Fields.GET);
+            builder.startObject(GET);
             getGetResult().toXContentEmbedded(builder, params);
             builder.endObject();
         }
@@ -111,4 +111,59 @@ public String toString() {
         return builder.append("]").toString();
     }
 
+    public static UpdateResponse fromXContent(XContentParser parser) throws IOException {
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
+
+        Builder context = new Builder();
+        while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
+            parseXContentFields(parser, context);
+        }
+        return context.build();
+    }
+
+    /**
+     * Parse the current token and update the parsing context appropriately.
+     */
+    public static void parseXContentFields(XContentParser parser, Builder context) throws IOException {
+        XContentParser.Token token = parser.currentToken();
+        String currentFieldName = parser.currentName();
+
+        if (GET.equals(currentFieldName)) {
+            if (token == XContentParser.Token.START_OBJECT) {
+                context.setGetResult(GetResult.fromXContentEmbedded(parser));
+            }
+        } else {
+            DocWriteResponse.parseInnerToXContent(parser, context);
+        }
+    }
+
+    /**
+     * Builder class for {@link UpdateResponse}. This builder is usually used during xcontent parsing to
+     * temporarily store the parsed values, then the {@link DocWriteResponse.Builder#build()} method is called to
+     * instantiate the {@link UpdateResponse}.
+     */
+    public static class Builder extends DocWriteResponse.Builder {
+
+        private GetResult getResult = null;
+
+        public void setGetResult(GetResult getResult) {
+            this.getResult = getResult;
+        }
+
+        @Override
+        public UpdateResponse build() {
+            UpdateResponse update;
+            if (shardInfo != null) {
+                update = new UpdateResponse(shardInfo, shardId, type, id, version, result);
+            } else {
+                update = new UpdateResponse(shardId, type, id, version, result);
+            }
+            if (getResult != null) {
+                update.setGetResult(new GetResult(update.getIndex(), update.getType(), update.getId(), update.getVersion(),
+                        getResult.isExists(),getResult.internalSourceRef(), getResult.getFields()));
+            }
+            update.setForcedRefresh(forcedRefresh);
+            return update;
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java b/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
index 2a38a020fee56..1d6cef6fffed2 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
@@ -19,18 +19,29 @@
 
 package org.elasticsearch.bootstrap;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.core.Appender;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.appender.ConsoleAppender;
+import org.apache.logging.log4j.core.config.Configurator;
 import org.apache.lucene.util.Constants;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.StringHelper;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
+import org.elasticsearch.cli.ExitCodes;
 import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cli.UserException;
 import org.elasticsearch.common.PidFile;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.inject.CreationException;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.common.logging.LogConfigurator;
 import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.settings.KeyStoreWrapper;
+import org.elasticsearch.common.settings.SecureSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.env.Environment;
@@ -38,13 +49,19 @@
 import org.elasticsearch.monitor.os.OsProbe;
 import org.elasticsearch.monitor.process.ProcessProbe;
 import org.elasticsearch.node.Node;
-import org.elasticsearch.node.internal.InternalSettingsPreparer;
+import org.elasticsearch.node.NodeValidationException;
+import org.elasticsearch.node.InternalSettingsPreparer;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.PrintStream;
+import java.io.UnsupportedEncodingException;
+import java.net.URISyntaxException;
 import java.nio.file.Path;
-import java.util.Map;
+import java.security.NoSuchAlgorithmException;
+import java.util.List;
+import java.util.Collections;
+import java.util.Locale;
 import java.util.concurrent.CountDownLatch;
 
 /**
@@ -56,6 +73,7 @@ final class Bootstrap {
     private volatile Node node;
     private final CountDownLatch keepAliveLatch = new CountDownLatch(1);
     private final Thread keepAliveThread;
+    private final Spawner spawner = new Spawner();
 
     /** creates a new instance */
     Bootstrap() {
@@ -80,7 +98,7 @@ public void run() {
     }
 
     /** initialize native resources */
-    public static void initializeNatives(Path tmpFile, boolean mlockAll, boolean seccomp, boolean ctrlHandler) {
+    public static void initializeNatives(Path tmpFile, boolean mlockAll, boolean systemCallFilter, boolean ctrlHandler) {
         final Logger logger = Loggers.getLogger(Bootstrap.class);
 
         // check if the user is running as root, and bail
@@ -88,9 +106,9 @@ public static void initializeNatives(Path tmpFile, boolean mlockAll, boolean sec
             throw new RuntimeException("can not run elasticsearch as root");
         }
 
-        // enable secure computing mode
-        if (seccomp) {
-            Natives.trySeccomp(tmpFile);
+        // enable system call filter
+        if (systemCallFilter) {
+            Natives.tryInstallSystemCallFilter(tmpFile);
         }
 
         // mlockall if requested
@@ -130,6 +148,7 @@ public boolean handle(int code) {
 
         Natives.trySetMaxNumberOfThreads();
         Natives.trySetMaxSizeVirtualMemory();
+        Natives.trySetMaxFileSize();
 
         // init lucene random seed. it will use /dev/urandom where available:
         StringHelper.randomId();
@@ -142,12 +161,41 @@ static void initializeProbes() {
         JvmInfo.jvmInfo();
     }
 
-    private void setup(boolean addShutdownHook, Environment environment) throws Exception {
+    private void setup(boolean addShutdownHook, Environment environment) throws BootstrapException {
         Settings settings = environment.settings();
+
+        try {
+            spawner.spawnNativePluginControllers(environment);
+        } catch (IOException e) {
+            throw new BootstrapException(e);
+        }
+
+        // TODO: remove in 6.0.0
+        if (BootstrapSettings.SECCOMP_SETTING.exists(settings)) {
+            final DeprecationLogger deprecationLogger = new DeprecationLogger(Loggers.getLogger(Bootstrap.class));
+            deprecationLogger.deprecated(
+                "[{}] is deprecated, use [{}]",
+                BootstrapSettings.SECCOMP_SETTING.getKey(),
+                BootstrapSettings.SYSTEM_CALL_FILTER_SETTING.getKey());
+
+            if (!BootstrapSettings.SECCOMP_SETTING.get(settings).equals(BootstrapSettings.SYSTEM_CALL_FILTER_SETTING.get(settings))) {
+                final String message = String.format(
+                    Locale.ROOT,
+                    "[%s=%s] and [%s=%s] are set to inconsistent values; as [%s] is deprecated, prefer [%s]",
+                    BootstrapSettings.SECCOMP_SETTING.getKey(),
+                    Boolean.toString(BootstrapSettings.SECCOMP_SETTING.get(settings)),
+                    BootstrapSettings.SYSTEM_CALL_FILTER_SETTING.getKey(),
+                    Boolean.toString(BootstrapSettings.SYSTEM_CALL_FILTER_SETTING.get(settings)),
+                    BootstrapSettings.SECCOMP_SETTING.getKey(),
+                    BootstrapSettings.SYSTEM_CALL_FILTER_SETTING.getKey());
+                throw new BootstrapException(new UserException(ExitCodes.CONFIG, message));
+            }
+        }
+
         initializeNatives(
                 environment.tmpFile(),
                 BootstrapSettings.MEMORY_LOCK_SETTING.get(settings),
-                BootstrapSettings.SECCOMP_SETTING.get(settings),
+                BootstrapSettings.SYSTEM_CALL_FILTER_SETTING.get(settings),
                 BootstrapSettings.CTRLHANDLER_SETTING.get(settings));
 
         // initialize probes before the security manager is installed
@@ -158,7 +206,9 @@ private void setup(boolean addShutdownHook, Environment environment) throws Exce
                 @Override
                 public void run() {
                     try {
-                        IOUtils.close(node);
+                        IOUtils.close(node, spawner);
+                        LoggerContext context = (LoggerContext) LogManager.getContext(false);
+                        Configurator.shutdown(context);
                     } catch (IOException ex) {
                         throw new ElasticsearchException("failed to stop node", ex);
                     }
@@ -166,83 +216,120 @@ public void run() {
             });
         }
 
-        // look for jar hell
-        JarHell.checkJarHell();
+        try {
+            // look for jar hell
+            JarHell.checkJarHell();
+        } catch (IOException | URISyntaxException e) {
+            throw new BootstrapException(e);
+        }
 
         // install SM after natives, shutdown hooks, etc.
-        Security.configure(environment, BootstrapSettings.SECURITY_FILTER_BAD_DEFAULTS_SETTING.get(settings));
+        try {
+            Security.configure(environment, BootstrapSettings.SECURITY_FILTER_BAD_DEFAULTS_SETTING.get(settings));
+        } catch (IOException | NoSuchAlgorithmException e) {
+            throw new BootstrapException(e);
+        }
 
         node = new Node(environment) {
             @Override
-            protected void validateNodeBeforeAcceptingRequests(Settings settings, BoundTransportAddress boundTransportAddress) {
-                BootstrapCheck.check(settings, boundTransportAddress);
+            protected void validateNodeBeforeAcceptingRequests(
+                final Settings settings,
+                final BoundTransportAddress boundTransportAddress, List<BootstrapCheck> checks) throws NodeValidationException {
+                BootstrapChecks.check(settings, boundTransportAddress, checks);
             }
         };
     }
 
-    private static Environment initialEnvironment(boolean foreground, Path pidFile, Map<String, String> esSettings) {
+    private static SecureSettings loadSecureSettings(Environment initialEnv) throws BootstrapException {
+        final KeyStoreWrapper keystore;
+        try {
+            keystore = KeyStoreWrapper.load(initialEnv.configFile());
+        } catch (IOException e) {
+            throw new BootstrapException(e);
+        }
+        if (keystore == null) {
+            return null; // no keystore
+        }
+
+        try {
+            keystore.decrypt(new char[0] /* TODO: read password from stdin */);
+        } catch (Exception e) {
+            throw new BootstrapException(e);
+        }
+        return keystore;
+    }
+
+
+    private static Environment createEnvironment(boolean foreground, Path pidFile,
+                                                 SecureSettings secureSettings, Settings initialSettings) {
         Terminal terminal = foreground ? Terminal.DEFAULT : null;
         Settings.Builder builder = Settings.builder();
         if (pidFile != null) {
             builder.put(Environment.PIDFILE_SETTING.getKey(), pidFile);
         }
-        return InternalSettingsPreparer.prepareEnvironment(builder.build(), terminal, esSettings);
+        builder.put(initialSettings);
+        if (secureSettings != null) {
+            builder.setSecureSettings(secureSettings);
+        }
+        return InternalSettingsPreparer.prepareEnvironment(builder.build(), terminal, Collections.emptyMap());
     }
 
-    private void start() {
+    private void start() throws NodeValidationException {
         node.start();
         keepAliveThread.start();
     }
 
     static void stop() throws IOException {
         try {
-            IOUtils.close(INSTANCE.node);
+            IOUtils.close(INSTANCE.node, INSTANCE.spawner);
         } finally {
             INSTANCE.keepAliveLatch.countDown();
         }
     }
 
-    /** Set the system property before anything has a chance to trigger its use */
-    // TODO: why? is it just a bad default somewhere? or is it some BS around 'but the client' garbage <-- my guess
-    @SuppressForbidden(reason = "sets logger prefix on initialization")
-    static void initLoggerPrefix() {
-        System.setProperty("es.logger.prefix", "");
-    }
-
     /**
-     * This method is invoked by {@link Elasticsearch#main(String[])}
-     * to startup elasticsearch.
+     * This method is invoked by {@link Elasticsearch#main(String[])} to startup elasticsearch.
      */
     static void init(
             final boolean foreground,
             final Path pidFile,
-            final Map<String, String> esSettings) throws Exception {
-        // Set the system property before anything has a chance to trigger its use
-        initLoggerPrefix();
-
+            final boolean quiet,
+            final Environment initialEnv) throws BootstrapException, NodeValidationException, UserException {
         // force the class initializer for BootstrapInfo to run before
         // the security manager is installed
         BootstrapInfo.init();
 
         INSTANCE = new Bootstrap();
 
-        Environment environment = initialEnvironment(foreground, pidFile, esSettings);
-        LogConfigurator.configure(environment, true);
+        final SecureSettings keystore = loadSecureSettings(initialEnv);
+        Environment environment = createEnvironment(foreground, pidFile, keystore, initialEnv.settings());
+        try {
+            LogConfigurator.configure(environment);
+        } catch (IOException e) {
+            throw new BootstrapException(e);
+        }
         checkForCustomConfFile();
+        checkConfigExtension(environment.configExtension());
 
         if (environment.pidFile() != null) {
-            PidFile.create(environment.pidFile(), true);
+            try {
+                PidFile.create(environment.pidFile(), true);
+            } catch (IOException e) {
+                throw new BootstrapException(e);
+            }
         }
 
+        final boolean closeStandardStreams = (foreground == false) || quiet;
         try {
-            if (!foreground) {
-                Loggers.disableConsoleLogging();
+            if (closeStandardStreams) {
+                final Logger rootLogger = ESLoggerFactory.getRootLogger();
+                final Appender maybeConsoleAppender = Loggers.findAppender(rootLogger, ConsoleAppender.class);
+                if (maybeConsoleAppender != null) {
+                    Loggers.removeAppender(rootLogger, maybeConsoleAppender);
+                }
                 closeSystOut();
             }
 
-            // fail if using broken version
-            JVMCheck.check();
-
             // fail if somebody replaced the lucene jars
             checkLucene();
 
@@ -254,15 +341,24 @@ static void init(
 
             INSTANCE.setup(true, environment);
 
+            try {
+                // any secure settings must be read during node construction
+                IOUtils.close(keystore);
+            } catch (IOException e) {
+                throw new BootstrapException(e);
+            }
+
             INSTANCE.start();
 
-            if (!foreground) {
+            if (closeStandardStreams) {
                 closeSysError();
             }
-        } catch (Exception e) {
+        } catch (NodeValidationException | RuntimeException e) {
             // disable console logging, so user does not see the exception twice (jvm will show it already)
-            if (foreground) {
-                Loggers.disableConsoleLogging();
+            final Logger rootLogger = ESLoggerFactory.getRootLogger();
+            final Appender maybeConsoleAppender = Loggers.findAppender(rootLogger, ConsoleAppender.class);
+            if (foreground && maybeConsoleAppender != null) {
+                Loggers.removeAppender(rootLogger, maybeConsoleAppender);
             }
             Logger logger = Loggers.getLogger(Bootstrap.class);
             if (INSTANCE.node != null) {
@@ -272,17 +368,30 @@ static void init(
             if (e instanceof CreationException) {
                 // guice: log the shortened exc to the log file
                 ByteArrayOutputStream os = new ByteArrayOutputStream();
-                PrintStream ps = new PrintStream(os, false, "UTF-8");
+                PrintStream ps = null;
+                try {
+                    ps = new PrintStream(os, false, "UTF-8");
+                } catch (UnsupportedEncodingException uee) {
+                    assert false;
+                    e.addSuppressed(uee);
+                }
                 new StartupException(e).printStackTrace(ps);
                 ps.flush();
-                logger.error("Guice Exception: {}", os.toString("UTF-8"));
+                try {
+                    logger.error("Guice Exception: {}", os.toString("UTF-8"));
+                } catch (UnsupportedEncodingException uee) {
+                    assert false;
+                    e.addSuppressed(uee);
+                }
+            } else if (e instanceof NodeValidationException) {
+                logger.error("node validation exception\n{}", e.getMessage());
             } else {
                 // full exception
                 logger.error("Exception", e);
             }
             // re-enable it if appropriate, so they can see any logging during the shutdown process
-            if (foreground) {
-                Loggers.enableConsoleLogging();
+            if (foreground && maybeConsoleAppender != null) {
+                Loggers.addAppender(rootLogger, maybeConsoleAppender);
             }
 
             throw e;
@@ -316,6 +425,14 @@ private static void checkUnsetAndMaybeExit(String confFileSetting, String settin
         }
     }
 
+    // pkg private for tests
+    static void checkConfigExtension(String extension) {
+        if (".yaml".equals(extension) || ".json".equals(extension)) {
+            final DeprecationLogger deprecationLogger = new DeprecationLogger(Loggers.getLogger(Bootstrap.class));
+            deprecationLogger.deprecated("elasticsearch{} is deprecated; rename your configuration file to elasticsearch.yml", extension);
+        }
+    }
+
     @SuppressForbidden(reason = "Allowed to exit explicitly in bootstrap phase")
     private static void exit(int status) {
         System.exit(status);
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java
index 2091b4f3d3582..ffe52dfe5b957 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java
@@ -19,555 +19,27 @@
 
 package org.elasticsearch.bootstrap;
 
-import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.apache.logging.log4j.util.Supplier;
-import org.apache.lucene.util.Constants;
-import org.elasticsearch.common.SuppressForbidden;
-import org.elasticsearch.common.io.PathUtils;
-import org.elasticsearch.common.logging.Loggers;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.BoundTransportAddress;
-import org.elasticsearch.common.transport.TransportAddress;
-import org.elasticsearch.monitor.jvm.JvmInfo;
-import org.elasticsearch.monitor.process.ProcessProbe;
-import org.elasticsearch.node.Node;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Locale;
-
 /**
- * We enforce limits once any network host is configured. In this case we assume the node is running in production
- * and all production limit checks must pass. This should be extended as we go to settings like:
- * - discovery.zen.ping.unicast.hosts is set if we use zen disco
- * - ensure we can write in all data directories
- * - fail if the default cluster.name is used, if this is setup on network a real clustername should be used?
+ * Encapsulates a bootstrap check.
  */
-final class BootstrapCheck {
-
-    private BootstrapCheck() {
-    }
+public interface BootstrapCheck {
 
     /**
-     * checks the current limits against the snapshot or release build
-     * checks
+     * Test if the node fails the check.
      *
-     * @param settings              the current node settings
-     * @param boundTransportAddress the node network bindings
+     * @return {@code true} if the node failed the check
      */
-    static void check(final Settings settings, final BoundTransportAddress boundTransportAddress) {
-        check(
-                enforceLimits(boundTransportAddress),
-                BootstrapSettings.IGNORE_SYSTEM_BOOTSTRAP_CHECKS.get(settings),
-                checks(settings),
-                Node.NODE_NAME_SETTING.get(settings));
-    }
-
-    /**
-     * executes the provided checks and fails the node if
-     * enforceLimits is true, otherwise logs warnings
-     *
-     * @param enforceLimits      true if the checks should be enforced or
-     *                           otherwise warned
-     * @param ignoreSystemChecks true if system checks should be enforced
-     *                           or otherwise warned
-     * @param checks             the checks to execute
-     * @param nodeName           the node name to be used as a logging prefix
-     */
-    // visible for testing
-    static void check(final boolean enforceLimits, final boolean ignoreSystemChecks, final List<Check> checks, final String nodeName) {
-        check(enforceLimits, ignoreSystemChecks, checks, Loggers.getLogger(BootstrapCheck.class, nodeName));
-    }
-
-    /**
-     * executes the provided checks and fails the node if
-     * enforceLimits is true, otherwise logs warnings
-     *
-     * @param enforceLimits      true if the checks should be enforced or
-     *                           otherwise warned
-     * @param ignoreSystemChecks true if system checks should be enforced
-     *                           or otherwise warned
-     * @param checks             the checks to execute
-     * @param logger             the logger to
-     */
-    static void check(
-            final boolean enforceLimits,
-            final boolean ignoreSystemChecks,
-            final List<Check> checks,
-            final Logger logger) {
-        final List<String> errors = new ArrayList<>();
-        final List<String> ignoredErrors = new ArrayList<>();
-
-        if (enforceLimits) {
-            logger.info("bound or publishing to a non-loopback or non-link-local address, enforcing bootstrap checks");
-        }
-        if (enforceLimits && ignoreSystemChecks) {
-            logger.warn("enforcing bootstrap checks but ignoring system bootstrap checks, consider not ignoring system checks");
-        }
-
-        for (final Check check : checks) {
-            if (check.check()) {
-                if ((!enforceLimits || (check.isSystemCheck() && ignoreSystemChecks)) && !check.alwaysEnforce()) {
-                    ignoredErrors.add(check.errorMessage());
-                } else {
-                    errors.add(check.errorMessage());
-                }
-            }
-        }
-
-        if (!ignoredErrors.isEmpty()) {
-            ignoredErrors.forEach(error -> log(logger, error));
-        }
-
-        if (!errors.isEmpty()) {
-            final List<String> messages = new ArrayList<>(1 + errors.size());
-            messages.add("bootstrap checks failed");
-            messages.addAll(errors);
-            final RuntimeException re = new RuntimeException(String.join("\n", messages));
-            errors.stream().map(IllegalStateException::new).forEach(re::addSuppressed);
-            throw re;
-        }
-
-    }
-
-    static void log(final Logger logger, final String error) {
-        logger.warn(error);
-    }
+    boolean check();
 
     /**
-     * Tests if the checks should be enforced
+     * The error message for a failed check.
      *
-     * @param boundTransportAddress the node network bindings
-     * @return true if the checks should be enforced
+     * @return the error message on check failure
      */
-    // visible for testing
-    static boolean enforceLimits(BoundTransportAddress boundTransportAddress) {
-        return !(Arrays.stream(boundTransportAddress.boundAddresses()).allMatch(TransportAddress::isLoopbackOrLinkLocalAddress) &&
-                boundTransportAddress.publishAddress().isLoopbackOrLinkLocalAddress());
-    }
-
-    // the list of checks to execute
-    static List<Check> checks(final Settings settings) {
-        final List<Check> checks = new ArrayList<>();
-        checks.add(new HeapSizeCheck());
-        final FileDescriptorCheck fileDescriptorCheck
-            = Constants.MAC_OS_X ? new OsXFileDescriptorCheck() : new FileDescriptorCheck();
-        checks.add(fileDescriptorCheck);
-        checks.add(new MlockallCheck(BootstrapSettings.MEMORY_LOCK_SETTING.get(settings)));
-        if (Constants.LINUX) {
-            checks.add(new MaxNumberOfThreadsCheck());
-        }
-        if (Constants.LINUX || Constants.MAC_OS_X) {
-            checks.add(new MaxSizeVirtualMemoryCheck());
-        }
-        if (Constants.LINUX) {
-            checks.add(new MaxMapCountCheck());
-        }
-        checks.add(new ClientJvmCheck());
-        checks.add(new OnErrorCheck());
-        checks.add(new OnOutOfMemoryErrorCheck());
-        return Collections.unmodifiableList(checks);
-    }
-
-    /**
-     * Encapsulates a limit check
-     */
-    interface Check {
-
-        /**
-         * test if the node fails the check
-         *
-         * @return true if the node failed the check
-         */
-        boolean check();
-
-        /**
-         * the message for a failed check
-         *
-         * @return the error message on check failure
-         */
-        String errorMessage();
-
-        /**
-         * test if the check is a system-level check
-         *
-         * @return true if the check is a system-level check as opposed
-         * to an Elasticsearch-level check
-         */
-        boolean isSystemCheck();
-
-        default boolean alwaysEnforce() {
-            return false;
-        }
-
-    }
-
-    static class HeapSizeCheck implements BootstrapCheck.Check {
-
-        @Override
-        public boolean check() {
-            final long initialHeapSize = getInitialHeapSize();
-            final long maxHeapSize = getMaxHeapSize();
-            return initialHeapSize != 0 && maxHeapSize != 0 && initialHeapSize != maxHeapSize;
-        }
-
-        @Override
-        public String errorMessage() {
-            return String.format(
-                    Locale.ROOT,
-                    "initial heap size [%d] not equal to maximum heap size [%d]; " +
-                            "this can cause resize pauses and prevents mlockall from locking the entire heap",
-                    getInitialHeapSize(),
-                    getMaxHeapSize()
-            );
-        }
-
-        // visible for testing
-        long getInitialHeapSize() {
-            return JvmInfo.jvmInfo().getConfiguredInitialHeapSize();
-        }
-
-        // visible for testing
-        long getMaxHeapSize() {
-            return JvmInfo.jvmInfo().getConfiguredMaxHeapSize();
-        }
-
-        @Override
-        public final boolean isSystemCheck() {
-            return false;
-        }
-
-    }
-
-    static class OsXFileDescriptorCheck extends FileDescriptorCheck {
-
-        public OsXFileDescriptorCheck() {
-            // see constant OPEN_MAX defined in
-            // /usr/include/sys/syslimits.h on OS X and its use in JVM
-            // initialization in int os:init_2(void) defined in the JVM
-            // code for BSD (contains OS X)
-            super(10240);
-        }
-
-    }
-
-    static class FileDescriptorCheck implements Check {
-
-        private final int limit;
-
-        FileDescriptorCheck() {
-            this(1 << 16);
-        }
-
-        protected FileDescriptorCheck(final int limit) {
-            if (limit <= 0) {
-                throw new IllegalArgumentException("limit must be positive but was [" + limit + "]");
-            }
-            this.limit = limit;
-        }
-
-        public final boolean check() {
-            final long maxFileDescriptorCount = getMaxFileDescriptorCount();
-            return maxFileDescriptorCount != -1 && maxFileDescriptorCount < limit;
-        }
-
-        @Override
-        public final String errorMessage() {
-            return String.format(
-                Locale.ROOT,
-                "max file descriptors [%d] for elasticsearch process likely too low, increase to at least [%d]",
-                getMaxFileDescriptorCount(),
-                limit
-            );
-        }
-
-        // visible for testing
-        long getMaxFileDescriptorCount() {
-            return ProcessProbe.getInstance().getMaxFileDescriptorCount();
-        }
-
-        @Override
-        public final boolean isSystemCheck() {
-            return true;
-        }
-
-    }
-
-    static class MlockallCheck implements Check {
-
-        private final boolean mlockallSet;
-
-        public MlockallCheck(final boolean mlockAllSet) {
-            this.mlockallSet = mlockAllSet;
-        }
-
-        @Override
-        public boolean check() {
-            return mlockallSet && !isMemoryLocked();
-        }
-
-        @Override
-        public String errorMessage() {
-            return "memory locking requested for elasticsearch process but memory is not locked";
-        }
-
-        // visible for testing
-        boolean isMemoryLocked() {
-            return Natives.isMemoryLocked();
-        }
-
-        @Override
-        public final boolean isSystemCheck() {
-            return true;
-        }
-
-    }
-
-    static class MaxNumberOfThreadsCheck implements Check {
-
-        private final long maxNumberOfThreadsThreshold = 1 << 11;
-
-        @Override
-        public boolean check() {
-            return getMaxNumberOfThreads() != -1 && getMaxNumberOfThreads() < maxNumberOfThreadsThreshold;
-        }
-
-        @Override
-        public String errorMessage() {
-            return String.format(
-                Locale.ROOT,
-                "max number of threads [%d] for user [%s] likely too low, increase to at least [%d]",
-                getMaxNumberOfThreads(),
-                BootstrapInfo.getSystemProperties().get("user.name"),
-                maxNumberOfThreadsThreshold);
-        }
-
-        // visible for testing
-        long getMaxNumberOfThreads() {
-            return JNANatives.MAX_NUMBER_OF_THREADS;
-        }
-
-        @Override
-        public final boolean isSystemCheck() {
-            return true;
-        }
-
-    }
-
-    static class MaxSizeVirtualMemoryCheck implements Check {
-
-        @Override
-        public boolean check() {
-            return getMaxSizeVirtualMemory() != Long.MIN_VALUE && getMaxSizeVirtualMemory() != getRlimInfinity();
-        }
-
-        @Override
-        public String errorMessage() {
-            return String.format(
-                Locale.ROOT,
-                "max size virtual memory [%d] for user [%s] likely too low, increase to [unlimited]",
-                getMaxSizeVirtualMemory(),
-                BootstrapInfo.getSystemProperties().get("user.name"));
-        }
-
-        // visible for testing
-        long getRlimInfinity() {
-            return JNACLibrary.RLIM_INFINITY;
-        }
-
-        // visible for testing
-        long getMaxSizeVirtualMemory() {
-            return JNANatives.MAX_SIZE_VIRTUAL_MEMORY;
-        }
-
-        @Override
-        public final boolean isSystemCheck() {
-            return true;
-        }
-
-    }
-
-    static class MaxMapCountCheck implements Check {
-
-        private final long limit = 1 << 18;
-
-        @Override
-        public boolean check() {
-            return getMaxMapCount() != -1 && getMaxMapCount() < limit;
-        }
-
-        @Override
-        public String errorMessage() {
-            return String.format(
-                    Locale.ROOT,
-                    "max virtual memory areas vm.max_map_count [%d] likely too low, increase to at least [%d]",
-                    getMaxMapCount(),
-                    limit);
-        }
-
-        // visible for testing
-        long getMaxMapCount() {
-            return getMaxMapCount(Loggers.getLogger(BootstrapCheck.class));
-        }
-
-        // visible for testing
-        long getMaxMapCount(Logger logger) {
-            final Path path = getProcSysVmMaxMapCountPath();
-            try (final BufferedReader bufferedReader = getBufferedReader(path)) {
-                final String rawProcSysVmMaxMapCount = readProcSysVmMaxMapCount(bufferedReader);
-                if (rawProcSysVmMaxMapCount != null) {
-                    try {
-                        return parseProcSysVmMaxMapCount(rawProcSysVmMaxMapCount);
-                    } catch (final NumberFormatException e) {
-                        logger.warn(
-                            (Supplier<?>) () -> new ParameterizedMessage(
-                                "unable to parse vm.max_map_count [{}]",
-                                rawProcSysVmMaxMapCount),
-                            e);
-                    }
-                }
-            } catch (final IOException e) {
-                logger.warn((Supplier<?>) () -> new ParameterizedMessage("I/O exception while trying to read [{}]", path), e);
-            }
-            return -1;
-        }
-
-        @SuppressForbidden(reason = "access /proc/sys/vm/max_map_count")
-        private Path getProcSysVmMaxMapCountPath() {
-            return PathUtils.get("/proc/sys/vm/max_map_count");
-        }
-
-        // visible for testing
-        BufferedReader getBufferedReader(final Path path) throws IOException {
-            return Files.newBufferedReader(path);
-        }
-
-        // visible for testing
-        String readProcSysVmMaxMapCount(final BufferedReader bufferedReader) throws IOException {
-            return bufferedReader.readLine();
-        }
-
-        // visible for testing
-        long parseProcSysVmMaxMapCount(final String procSysVmMaxMapCount) throws NumberFormatException {
-            return Long.parseLong(procSysVmMaxMapCount);
-        }
-
-        @Override
-        public final boolean isSystemCheck() {
-            return true;
-        }
-
-    }
-
-    static class ClientJvmCheck implements BootstrapCheck.Check {
-
-        @Override
-        public boolean check() {
-            return getVmName().toLowerCase(Locale.ROOT).contains("client");
-        }
-
-        // visible for testing
-        String getVmName() {
-            return JvmInfo.jvmInfo().getVmName();
-        }
-
-        @Override
-        public String errorMessage() {
-            return String.format(
-                    Locale.ROOT,
-                    "JVM is using the client VM [%s] but should be using a server VM for the best performance",
-                    getVmName());
-        }
-
-        @Override
-        public final boolean isSystemCheck() {
-            return false;
-        }
-
-    }
-
-    abstract static class MightForkCheck implements BootstrapCheck.Check {
-
-        @Override
-        public boolean check() {
-            return isSeccompInstalled() && mightFork();
-        }
-
-        // visible for testing
-        boolean isSeccompInstalled() {
-            return Natives.isSeccompInstalled();
-        }
-
-        // visible for testing
-        abstract boolean mightFork();
-
-        @Override
-        public final boolean isSystemCheck() {
-            return false;
-        }
-
-        @Override
-        public final boolean alwaysEnforce() {
-            return true;
-        }
-
-    }
-
-    static class OnErrorCheck extends MightForkCheck {
-
-        @Override
-        boolean mightFork() {
-            final String onError = onError();
-            return onError != null && !onError.equals("");
-        }
-
-        // visible for testing
-        String onError() {
-            return JvmInfo.jvmInfo().onError();
-        }
-
-        @Override
-        public String errorMessage() {
-            return String.format(
-                Locale.ROOT,
-                "OnError [%s] requires forking but is prevented by system call filters ([%s=true]);" +
-                    " upgrade to at least Java 8u92 and use ExitOnOutOfMemoryError",
-                onError(),
-                BootstrapSettings.SECCOMP_SETTING.getKey());
-        }
-
-    }
-
-    static class OnOutOfMemoryErrorCheck extends MightForkCheck {
-
-        @Override
-        boolean mightFork() {
-            final String onOutOfMemoryError = onOutOfMemoryError();
-            return onOutOfMemoryError != null && !onOutOfMemoryError.equals("");
-        }
-
-        // visible for testing
-        String onOutOfMemoryError() {
-            return JvmInfo.jvmInfo().onOutOfMemoryError();
-        }
-
-        @Override
-        public String errorMessage() {
-            return String.format(
-                Locale.ROOT,
-                "OnOutOfMemoryError [%s] requires forking but is prevented by system call filters ([%s=true]);" +
-                    " upgrade to at least Java 8u92 and use ExitOnOutOfMemoryError",
-                onOutOfMemoryError(),
-                BootstrapSettings.SECCOMP_SETTING.getKey());
-        }
+    String errorMessage();
 
+    default boolean alwaysEnforce() {
+        return false;
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java
new file mode 100644
index 0000000000000..45cd04e794225
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java
@@ -0,0 +1,701 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.bootstrap;
+
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.apache.logging.log4j.util.Supplier;
+import org.apache.lucene.util.Constants;
+import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.common.io.PathUtils;
+import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.BoundTransportAddress;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.discovery.DiscoveryModule;
+import org.elasticsearch.monitor.jvm.JvmInfo;
+import org.elasticsearch.monitor.process.ProcessProbe;
+import org.elasticsearch.node.Node;
+import org.elasticsearch.node.NodeValidationException;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Locale;
+import java.util.function.Predicate;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * We enforce bootstrap checks once a node has the transport protocol bound to a non-loopback interface or if the system property {@code
+ * es.enforce.bootstrap.checks} is set to {@true}. In this case we assume the node is running in production and all bootstrap checks must
+ * pass.
+ */
+final class BootstrapChecks {
+
+    private BootstrapChecks() {
+    }
+
+    static final String ES_ENFORCE_BOOTSTRAP_CHECKS = "es.enforce.bootstrap.checks";
+
+    /**
+     * Executes the bootstrap checks if the node has the transport protocol bound to a non-loopback interface. If the system property
+     * {@code es.enforce.bootstrap.checks} is set to {@code true} then the bootstrap checks will be enforced regardless of whether or not
+     * the transport protocol is bound to a non-loopback interface.
+     *
+     * @param settings              the current node settings
+     * @param boundTransportAddress the node network bindings
+     */
+    static void check(final Settings settings, final BoundTransportAddress boundTransportAddress, List<BootstrapCheck> additionalChecks)
+        throws NodeValidationException {
+        final List<BootstrapCheck> builtInChecks = checks(settings);
+        final List<BootstrapCheck> combinedChecks = new ArrayList<>(builtInChecks);
+        combinedChecks.addAll(additionalChecks);
+        check(
+                enforceLimits(boundTransportAddress, DiscoveryModule.DISCOVERY_TYPE_SETTING.get(settings)),
+                Collections.unmodifiableList(combinedChecks),
+                Node.NODE_NAME_SETTING.get(settings));
+    }
+
+    /**
+     * Executes the provided checks and fails the node if {@code enforceLimits} is {@code true}, otherwise logs warnings. If the system
+     * property {@code es.enforce.bootstrap.checks} is set to {@code true} then the bootstrap checks will be enforced regardless of whether
+     * or not the transport protocol is bound to a non-loopback interface.
+     *
+     * @param enforceLimits {@code true} if the checks should be enforced or otherwise warned
+     * @param checks        the checks to execute
+     * @param nodeName      the node name to be used as a logging prefix
+     */
+    static void check(
+        final boolean enforceLimits,
+        final List<BootstrapCheck> checks,
+        final String nodeName) throws NodeValidationException {
+        check(enforceLimits, checks, Loggers.getLogger(BootstrapChecks.class, nodeName));
+    }
+
+    /**
+     * Executes the provided checks and fails the node if {@code enforceLimits} is {@code true}, otherwise logs warnings. If the system
+     * property {@code es.enforce.bootstrap.checks }is set to {@code true} then the bootstrap checks will be enforced regardless of whether
+     * or not the transport protocol is bound to a non-loopback interface.
+     *
+     * @param enforceLimits {@code true} if the checks should be enforced or otherwise warned
+     * @param checks        the checks to execute
+     * @param logger        the logger to
+     */
+    static void check(
+            final boolean enforceLimits,
+            final List<BootstrapCheck> checks,
+            final Logger logger) throws NodeValidationException {
+        final List<String> errors = new ArrayList<>();
+        final List<String> ignoredErrors = new ArrayList<>();
+
+        final String esEnforceBootstrapChecks = System.getProperty(ES_ENFORCE_BOOTSTRAP_CHECKS);
+        final boolean enforceBootstrapChecks;
+        if (esEnforceBootstrapChecks == null) {
+            enforceBootstrapChecks = false;
+        } else if (Boolean.TRUE.toString().equals(esEnforceBootstrapChecks)) {
+            enforceBootstrapChecks = true;
+        } else {
+            final String message =
+                    String.format(
+                            Locale.ROOT,
+                            "[%s] must be [true] but was [%s]",
+                            ES_ENFORCE_BOOTSTRAP_CHECKS,
+                            esEnforceBootstrapChecks);
+            throw new IllegalArgumentException(message);
+        }
+
+        if (enforceLimits) {
+            logger.info("bound or publishing to a non-loopback or non-link-local address, enforcing bootstrap checks");
+        } else if (enforceBootstrapChecks) {
+            logger.info("explicitly enforcing bootstrap checks");
+        }
+
+        for (final BootstrapCheck check : checks) {
+            if (check.check()) {
+                if (!(enforceLimits || enforceBootstrapChecks) && !check.alwaysEnforce()) {
+                    ignoredErrors.add(check.errorMessage());
+                } else {
+                    errors.add(check.errorMessage());
+                }
+            }
+        }
+
+        if (!ignoredErrors.isEmpty()) {
+            ignoredErrors.forEach(error -> log(logger, error));
+        }
+
+        if (!errors.isEmpty()) {
+            final List<String> messages = new ArrayList<>(1 + errors.size());
+            messages.add("[" + errors.size() + "] bootstrap checks failed");
+            for (int i = 0; i < errors.size(); i++) {
+                messages.add("[" + (i + 1) + "]: " + errors.get(i));
+            }
+            final NodeValidationException ne = new NodeValidationException(String.join("\n", messages));
+            errors.stream().map(IllegalStateException::new).forEach(ne::addSuppressed);
+            throw ne;
+        }
+    }
+
+    static void log(final Logger logger, final String error) {
+        logger.warn(error);
+    }
+
+    /**
+     * Tests if the checks should be enforced.
+     *
+     * @param boundTransportAddress the node network bindings
+     * @param discoveryType the discovery type
+     * @return {@code true} if the checks should be enforced
+     */
+    static boolean enforceLimits(final BoundTransportAddress boundTransportAddress, final String discoveryType) {
+        final boolean bound =
+                !(Arrays.stream(boundTransportAddress.boundAddresses()).allMatch(TransportAddress::isLoopbackOrLinkLocalAddress) &&
+                boundTransportAddress.publishAddress().isLoopbackOrLinkLocalAddress());
+        return bound && !"single-node".equals(discoveryType);
+    }
+
+    // the list of checks to execute
+    static List<BootstrapCheck> checks(final Settings settings) {
+        final List<BootstrapCheck> checks = new ArrayList<>();
+        checks.add(new HeapSizeCheck());
+        final FileDescriptorCheck fileDescriptorCheck
+            = Constants.MAC_OS_X ? new OsXFileDescriptorCheck() : new FileDescriptorCheck();
+        checks.add(fileDescriptorCheck);
+        checks.add(new MlockallCheck(BootstrapSettings.MEMORY_LOCK_SETTING.get(settings)));
+        if (Constants.LINUX) {
+            checks.add(new MaxNumberOfThreadsCheck());
+        }
+        if (Constants.LINUX || Constants.MAC_OS_X) {
+            checks.add(new MaxSizeVirtualMemoryCheck());
+        }
+        if (Constants.LINUX || Constants.MAC_OS_X) {
+            checks.add(new MaxFileSizeCheck());
+        }
+        if (Constants.LINUX) {
+            checks.add(new MaxMapCountCheck());
+        }
+        checks.add(new ClientJvmCheck());
+        checks.add(new UseSerialGCCheck());
+        checks.add(new SystemCallFilterCheck(BootstrapSettings.SYSTEM_CALL_FILTER_SETTING.get(settings)));
+        checks.add(new OnErrorCheck());
+        checks.add(new OnOutOfMemoryErrorCheck());
+        checks.add(new EarlyAccessCheck());
+        checks.add(new G1GCCheck());
+        return Collections.unmodifiableList(checks);
+    }
+
+    static class HeapSizeCheck implements BootstrapCheck {
+
+        @Override
+        public boolean check() {
+            final long initialHeapSize = getInitialHeapSize();
+            final long maxHeapSize = getMaxHeapSize();
+            return initialHeapSize != 0 && maxHeapSize != 0 && initialHeapSize != maxHeapSize;
+        }
+
+        @Override
+        public String errorMessage() {
+            return String.format(
+                    Locale.ROOT,
+                    "initial heap size [%d] not equal to maximum heap size [%d]; " +
+                            "this can cause resize pauses and prevents mlockall from locking the entire heap",
+                    getInitialHeapSize(),
+                    getMaxHeapSize()
+            );
+        }
+
+        // visible for testing
+        long getInitialHeapSize() {
+            return JvmInfo.jvmInfo().getConfiguredInitialHeapSize();
+        }
+
+        // visible for testing
+        long getMaxHeapSize() {
+            return JvmInfo.jvmInfo().getConfiguredMaxHeapSize();
+        }
+
+    }
+
+    static class OsXFileDescriptorCheck extends FileDescriptorCheck {
+
+        OsXFileDescriptorCheck() {
+            // see constant OPEN_MAX defined in
+            // /usr/include/sys/syslimits.h on OS X and its use in JVM
+            // initialization in int os:init_2(void) defined in the JVM
+            // code for BSD (contains OS X)
+            super(10240);
+        }
+
+    }
+
+    static class FileDescriptorCheck implements BootstrapCheck {
+
+        private final int limit;
+
+        FileDescriptorCheck() {
+            this(1 << 16);
+        }
+
+        protected FileDescriptorCheck(final int limit) {
+            if (limit <= 0) {
+                throw new IllegalArgumentException("limit must be positive but was [" + limit + "]");
+            }
+            this.limit = limit;
+        }
+
+        public final boolean check() {
+            final long maxFileDescriptorCount = getMaxFileDescriptorCount();
+            return maxFileDescriptorCount != -1 && maxFileDescriptorCount < limit;
+        }
+
+        @Override
+        public final String errorMessage() {
+            return String.format(
+                Locale.ROOT,
+                "max file descriptors [%d] for elasticsearch process is too low, increase to at least [%d]",
+                getMaxFileDescriptorCount(),
+                limit
+            );
+        }
+
+        // visible for testing
+        long getMaxFileDescriptorCount() {
+            return ProcessProbe.getInstance().getMaxFileDescriptorCount();
+        }
+
+    }
+
+    static class MlockallCheck implements BootstrapCheck {
+
+        private final boolean mlockallSet;
+
+        MlockallCheck(final boolean mlockAllSet) {
+            this.mlockallSet = mlockAllSet;
+        }
+
+        @Override
+        public boolean check() {
+            return mlockallSet && !isMemoryLocked();
+        }
+
+        @Override
+        public String errorMessage() {
+            return "memory locking requested for elasticsearch process but memory is not locked";
+        }
+
+        // visible for testing
+        boolean isMemoryLocked() {
+            return Natives.isMemoryLocked();
+        }
+
+    }
+
+    static class MaxNumberOfThreadsCheck implements BootstrapCheck {
+
+        // this should be plenty for machines up to 256 cores
+        private static final long MAX_NUMBER_OF_THREADS_THRESHOLD = 1 << 11;
+
+        @Override
+        public boolean check() {
+            return getMaxNumberOfThreads() != -1 && getMaxNumberOfThreads() < MAX_NUMBER_OF_THREADS_THRESHOLD;
+        }
+
+        @Override
+        public String errorMessage() {
+            return String.format(
+                Locale.ROOT,
+                "max number of threads [%d] for user [%s] is too low, increase to at least [%d]",
+                getMaxNumberOfThreads(),
+                BootstrapInfo.getSystemProperties().get("user.name"),
+                MAX_NUMBER_OF_THREADS_THRESHOLD);
+        }
+
+        // visible for testing
+        long getMaxNumberOfThreads() {
+            return JNANatives.MAX_NUMBER_OF_THREADS;
+        }
+
+    }
+
+    static class MaxSizeVirtualMemoryCheck implements BootstrapCheck {
+
+        @Override
+        public boolean check() {
+            return getMaxSizeVirtualMemory() != Long.MIN_VALUE && getMaxSizeVirtualMemory() != getRlimInfinity();
+        }
+
+        @Override
+        public String errorMessage() {
+            return String.format(
+                Locale.ROOT,
+                "max size virtual memory [%d] for user [%s] is too low, increase to [unlimited]",
+                getMaxSizeVirtualMemory(),
+                BootstrapInfo.getSystemProperties().get("user.name"));
+        }
+
+        // visible for testing
+        long getRlimInfinity() {
+            return JNACLibrary.RLIM_INFINITY;
+        }
+
+        // visible for testing
+        long getMaxSizeVirtualMemory() {
+            return JNANatives.MAX_SIZE_VIRTUAL_MEMORY;
+        }
+
+    }
+
+    /**
+     * Bootstrap check that the maximum file size is unlimited (otherwise Elasticsearch could run in to an I/O exception writing files).
+     */
+    static class MaxFileSizeCheck implements BootstrapCheck {
+
+        @Override
+        public boolean check() {
+            final long maxFileSize = getMaxFileSize();
+            return maxFileSize != Long.MIN_VALUE && maxFileSize != getRlimInfinity();
+        }
+
+        @Override
+        public String errorMessage() {
+            return String.format(
+                    Locale.ROOT,
+                    "max file size [%d] for user [%s] is too low, increase to [unlimited]",
+                    getMaxFileSize(),
+                    BootstrapInfo.getSystemProperties().get("user.name"));
+        }
+
+        long getRlimInfinity() {
+            return JNACLibrary.RLIM_INFINITY;
+        }
+
+        long getMaxFileSize() {
+            return JNANatives.MAX_FILE_SIZE;
+        }
+
+    }
+
+    static class MaxMapCountCheck implements BootstrapCheck {
+
+        private static final long LIMIT = 1 << 18;
+
+        @Override
+        public boolean check() {
+            return getMaxMapCount() != -1 && getMaxMapCount() < LIMIT;
+        }
+
+        @Override
+        public String errorMessage() {
+            return String.format(
+                    Locale.ROOT,
+                    "max virtual memory areas vm.max_map_count [%d] is too low, increase to at least [%d]",
+                    getMaxMapCount(),
+                    LIMIT);
+        }
+
+        // visible for testing
+        long getMaxMapCount() {
+            return getMaxMapCount(Loggers.getLogger(BootstrapChecks.class));
+        }
+
+        // visible for testing
+        long getMaxMapCount(Logger logger) {
+            final Path path = getProcSysVmMaxMapCountPath();
+            try (BufferedReader bufferedReader = getBufferedReader(path)) {
+                final String rawProcSysVmMaxMapCount = readProcSysVmMaxMapCount(bufferedReader);
+                if (rawProcSysVmMaxMapCount != null) {
+                    try {
+                        return parseProcSysVmMaxMapCount(rawProcSysVmMaxMapCount);
+                    } catch (final NumberFormatException e) {
+                        logger.warn(
+                            (Supplier<?>) () -> new ParameterizedMessage(
+                                "unable to parse vm.max_map_count [{}]",
+                                rawProcSysVmMaxMapCount),
+                            e);
+                    }
+                }
+            } catch (final IOException e) {
+                logger.warn((Supplier<?>) () -> new ParameterizedMessage("I/O exception while trying to read [{}]", path), e);
+            }
+            return -1;
+        }
+
+        @SuppressForbidden(reason = "access /proc/sys/vm/max_map_count")
+        private Path getProcSysVmMaxMapCountPath() {
+            return PathUtils.get("/proc/sys/vm/max_map_count");
+        }
+
+        // visible for testing
+        BufferedReader getBufferedReader(final Path path) throws IOException {
+            return Files.newBufferedReader(path);
+        }
+
+        // visible for testing
+        String readProcSysVmMaxMapCount(final BufferedReader bufferedReader) throws IOException {
+            return bufferedReader.readLine();
+        }
+
+        // visible for testing
+        long parseProcSysVmMaxMapCount(final String procSysVmMaxMapCount) throws NumberFormatException {
+            return Long.parseLong(procSysVmMaxMapCount);
+        }
+
+    }
+
+    static class ClientJvmCheck implements BootstrapCheck {
+
+        @Override
+        public boolean check() {
+            return getVmName().toLowerCase(Locale.ROOT).contains("client");
+        }
+
+        // visible for testing
+        String getVmName() {
+            return JvmInfo.jvmInfo().getVmName();
+        }
+
+        @Override
+        public String errorMessage() {
+            return String.format(
+                    Locale.ROOT,
+                    "JVM is using the client VM [%s] but should be using a server VM for the best performance",
+                    getVmName());
+        }
+
+    }
+
+    /**
+     * Checks if the serial collector is in use. This collector is single-threaded and devastating
+     * for performance and should not be used for a server application like Elasticsearch.
+     */
+    static class UseSerialGCCheck implements BootstrapCheck {
+
+        @Override
+        public boolean check() {
+            return getUseSerialGC().equals("true");
+        }
+
+        // visible for testing
+        String getUseSerialGC() {
+            return JvmInfo.jvmInfo().useSerialGC();
+        }
+
+        @Override
+        public String errorMessage() {
+            return String.format(
+                Locale.ROOT,
+                "JVM is using the serial collector but should not be for the best performance; " +
+                    "either it's the default for the VM [%s] or -XX:+UseSerialGC was explicitly specified",
+                JvmInfo.jvmInfo().getVmName());
+        }
+
+    }
+
+    /**
+     * Bootstrap check that if system call filters are enabled, then system call filters must have installed successfully.
+     */
+    static class SystemCallFilterCheck implements BootstrapCheck {
+
+        private final boolean areSystemCallFiltersEnabled;
+
+        SystemCallFilterCheck(final boolean areSystemCallFiltersEnabled) {
+            this.areSystemCallFiltersEnabled = areSystemCallFiltersEnabled;
+        }
+
+        @Override
+        public boolean check() {
+            return areSystemCallFiltersEnabled && !isSystemCallFilterInstalled();
+        }
+
+        // visible for testing
+        boolean isSystemCallFilterInstalled() {
+            return Natives.isSystemCallFilterInstalled();
+        }
+
+        @Override
+        public String errorMessage() {
+            return "system call filters failed to install; " +
+                "check the logs and fix your configuration or disable system call filters at your own risk";
+        }
+
+    }
+
+    abstract static class MightForkCheck implements BootstrapCheck {
+
+        @Override
+        public boolean check() {
+            return isSystemCallFilterInstalled() && mightFork();
+        }
+
+        // visible for testing
+        boolean isSystemCallFilterInstalled() {
+            return Natives.isSystemCallFilterInstalled();
+        }
+
+        // visible for testing
+        abstract boolean mightFork();
+
+        @Override
+        public final boolean alwaysEnforce() {
+            return true;
+        }
+
+    }
+
+    static class OnErrorCheck extends MightForkCheck {
+
+        @Override
+        boolean mightFork() {
+            final String onError = onError();
+            return onError != null && !onError.equals("");
+        }
+
+        // visible for testing
+        String onError() {
+            return JvmInfo.jvmInfo().onError();
+        }
+
+        @Override
+        public String errorMessage() {
+            return String.format(
+                Locale.ROOT,
+                "OnError [%s] requires forking but is prevented by system call filters ([%s=true]);" +
+                    " upgrade to at least Java 8u92 and use ExitOnOutOfMemoryError",
+                onError(),
+                BootstrapSettings.SYSTEM_CALL_FILTER_SETTING.getKey());
+        }
+
+    }
+
+    static class OnOutOfMemoryErrorCheck extends MightForkCheck {
+
+        @Override
+        boolean mightFork() {
+            final String onOutOfMemoryError = onOutOfMemoryError();
+            return onOutOfMemoryError != null && !onOutOfMemoryError.equals("");
+        }
+
+        // visible for testing
+        String onOutOfMemoryError() {
+            return JvmInfo.jvmInfo().onOutOfMemoryError();
+        }
+
+        @Override
+        public String errorMessage() {
+            return String.format(
+                Locale.ROOT,
+                "OnOutOfMemoryError [%s] requires forking but is prevented by system call filters ([%s=true]);" +
+                    " upgrade to at least Java 8u92 and use ExitOnOutOfMemoryError",
+                onOutOfMemoryError(),
+                BootstrapSettings.SYSTEM_CALL_FILTER_SETTING.getKey());
+        }
+
+    }
+
+    /**
+     * Bootstrap check for early-access builds from OpenJDK.
+     */
+    static class EarlyAccessCheck implements BootstrapCheck {
+
+        @Override
+        public boolean check() {
+            return "Oracle Corporation".equals(jvmVendor()) && javaVersion().endsWith("-ea");
+        }
+
+        String jvmVendor() {
+            return Constants.JVM_VENDOR;
+        }
+
+        String javaVersion() {
+            return Constants.JAVA_VERSION;
+        }
+
+        @Override
+        public String errorMessage() {
+            return String.format(
+                    Locale.ROOT,
+                    "Java version [%s] is an early-access build, only use release builds",
+                    javaVersion());
+        }
+
+    }
+
+    /**
+     * Bootstrap check for versions of HotSpot that are known to have issues that can lead to index corruption when G1GC is enabled.
+     */
+    static class G1GCCheck implements BootstrapCheck {
+
+        @Override
+        public boolean check() {
+            if ("Oracle Corporation".equals(jvmVendor()) && isJava8() && isG1GCEnabled()) {
+                final String jvmVersion = jvmVersion();
+                // HotSpot versions on Java 8 match this regular expression; note that this changes with Java 9 after JEP-223
+                final Pattern pattern = Pattern.compile("(\\d+)\\.(\\d+)-b\\d+");
+                final Matcher matcher = pattern.matcher(jvmVersion);
+                final boolean matches = matcher.matches();
+                assert matches : jvmVersion;
+                final int major = Integer.parseInt(matcher.group(1));
+                final int update = Integer.parseInt(matcher.group(2));
+                // HotSpot versions for Java 8 have major version 25, the bad versions are all versions prior to update 40
+                return major == 25 && update < 40;
+            } else {
+                return false;
+            }
+        }
+
+        // visible for testing
+        String jvmVendor() {
+            return Constants.JVM_VENDOR;
+        }
+
+        // visible for testing
+        boolean isG1GCEnabled() {
+            assert "Oracle Corporation".equals(jvmVendor());
+            return JvmInfo.jvmInfo().useG1GC().equals("true");
+        }
+
+        // visible for testing
+        String jvmVersion() {
+            assert "Oracle Corporation".equals(jvmVendor());
+            return Constants.JVM_VERSION;
+        }
+
+        // visible for testing
+        boolean isJava8() {
+            assert "Oracle Corporation".equals(jvmVendor());
+            return JavaVersion.current().equals(JavaVersion.parse("1.8"));
+        }
+
+        @Override
+        public String errorMessage() {
+           return String.format(
+               Locale.ROOT,
+               "JVM version [%s] can cause data corruption when used with G1GC; upgrade to at least Java 8u40", jvmVersion());
+        }
+
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapException.java b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapException.java
new file mode 100644
index 0000000000000..635afaf959967
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapException.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.bootstrap;
+
+import java.nio.file.Path;
+
+/**
+ * Wrapper exception for checked exceptions thrown during the bootstrap process. Methods invoked
+ * during bootstrap should explicitly declare the checked exceptions that they can throw, rather
+ * than declaring the top-level checked exception {@link Exception}. This exception exists to wrap
+ * these checked exceptions so that
+ * {@link Bootstrap#init(boolean, Path, boolean, org.elasticsearch.env.Environment)}
+ * does not have to declare all of these checked exceptions.
+ */
+class BootstrapException extends Exception {
+
+    /**
+     * Wraps an existing exception.
+     *
+     * @param cause the underlying cause of bootstrap failing
+     */
+    BootstrapException(final Exception cause) {
+        super(cause);
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapInfo.java b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapInfo.java
index 791836bf8a4a7..3ff6639de9aad 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapInfo.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapInfo.java
@@ -24,16 +24,16 @@
 import java.util.Dictionary;
 import java.util.Enumeration;
 
-/** 
- * Exposes system startup information 
+/**
+ * Exposes system startup information
  */
 @SuppressForbidden(reason = "exposes read-only view of system properties")
 public final class BootstrapInfo {
 
     /** no instantiation */
     private BootstrapInfo() {}
-    
-    /** 
+
+    /**
      * Returns true if we successfully loaded native libraries.
      * <p>
      * If this returns false, then native operations such as locking
@@ -42,19 +42,19 @@ private BootstrapInfo() {}
     public static boolean isNativesAvailable() {
         return Natives.JNA_AVAILABLE;
     }
-    
-    /** 
+
+    /**
      * Returns true if we were able to lock the process's address space.
      */
     public static boolean isMemoryLocked() {
         return Natives.isMemoryLocked();
     }
-    
+
     /**
-     * Returns true if secure computing mode is enabled (supported systems only)
+     * Returns true if system call filter is installed (supported systems only)
      */
-    public static boolean isSeccompInstalled() {
-        return Natives.isSeccompInstalled();
+    public static boolean isSystemCallFilterInstalled() {
+        return Natives.isSystemCallFilterInstalled();
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapSettings.java b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapSettings.java
index ad37916881bfd..2d907088f1f9b 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapSettings.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapSettings.java
@@ -33,11 +33,12 @@ private BootstrapSettings() {
 
     public static final Setting<Boolean> MEMORY_LOCK_SETTING =
         Setting.boolSetting("bootstrap.memory_lock", false, Property.NodeScope);
+    // TODO: remove in 6.0.0
     public static final Setting<Boolean> SECCOMP_SETTING =
         Setting.boolSetting("bootstrap.seccomp", true, Property.NodeScope);
+    public static final Setting<Boolean> SYSTEM_CALL_FILTER_SETTING =
+        Setting.boolSetting("bootstrap.system_call_filter", SECCOMP_SETTING, Property.NodeScope);
     public static final Setting<Boolean> CTRLHANDLER_SETTING =
         Setting.boolSetting("bootstrap.ctrlhandler", true, Property.NodeScope);
-    public static final Setting<Boolean> IGNORE_SYSTEM_BOOTSTRAP_CHECKS =
-            Setting.boolSetting("bootstrap.ignore_system_bootstrap_checks", false, Property.NodeScope);
 
 }
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/ESPolicy.java b/core/src/main/java/org/elasticsearch/bootstrap/ESPolicy.java
index ddc25c8853519..74fa7e0c1d5ac 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/ESPolicy.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/ESPolicy.java
@@ -21,17 +21,19 @@
 
 import org.elasticsearch.common.SuppressForbidden;
 
-import java.net.SocketPermission;
-import java.net.URL;
 import java.io.FilePermission;
 import java.io.IOException;
+import java.net.SocketPermission;
+import java.net.URL;
 import java.security.CodeSource;
 import java.security.Permission;
 import java.security.PermissionCollection;
 import java.security.Permissions;
 import java.security.Policy;
 import java.security.ProtectionDomain;
+import java.util.Collections;
 import java.util.Map;
+import java.util.function.Predicate;
 
 /** custom policy for union of static and dynamic permissions */
 final class ESPolicy extends Policy {
@@ -47,9 +49,9 @@ final class ESPolicy extends Policy {
     final PermissionCollection dynamic;
     final Map<String,Policy> plugins;
 
-    public ESPolicy(PermissionCollection dynamic, Map<String,Policy> plugins, boolean filterBadDefaults) {
+    ESPolicy(PermissionCollection dynamic, Map<String,Policy> plugins, boolean filterBadDefaults) {
         this.template = Security.readPolicy(getClass().getResource(POLICY_RESOURCE), JarHell.parseClassPath());
-        this.untrusted = Security.readPolicy(getClass().getResource(UNTRUSTED_RESOURCE), new URL[0]);
+        this.untrusted = Security.readPolicy(getClass().getResource(UNTRUSTED_RESOURCE), Collections.emptySet());
         if (filterBadDefaults) {
             this.system = new SystemPolicy(Policy.getPolicy());
         } else {
@@ -133,18 +135,66 @@ public PermissionCollection getPermissions(CodeSource codesource) {
 
     // TODO: remove this hack when insecure defaults are removed from java
 
+    /**
+     * Wraps a bad default permission, applying a pre-implies to any permissions before checking if the wrapped bad default permission
+     * implies a permission.
+     */
+    private static class BadDefaultPermission extends Permission {
+
+        private final Permission badDefaultPermission;
+        private final Predicate<Permission> preImplies;
+
+        /**
+         * Construct an instance with a pre-implies check to apply to desired permissions.
+         *
+         * @param badDefaultPermission the bad default permission to wrap
+         * @param preImplies           a test that is applied to a desired permission before checking if the bad default permission that
+         *                             this instance wraps implies the desired permission
+         */
+        BadDefaultPermission(final Permission badDefaultPermission, final Predicate<Permission> preImplies) {
+            super(badDefaultPermission.getName());
+            this.badDefaultPermission = badDefaultPermission;
+            this.preImplies = preImplies;
+        }
+
+        @Override
+        public final boolean implies(Permission permission) {
+            return preImplies.test(permission) && badDefaultPermission.implies(permission);
+        }
+
+        @Override
+        public final boolean equals(Object obj) {
+            return badDefaultPermission.equals(obj);
+        }
+
+        @Override
+        public int hashCode() {
+            return badDefaultPermission.hashCode();
+        }
+
+        @Override
+        public String getActions() {
+            return badDefaultPermission.getActions();
+        }
+
+    }
+
     // default policy file states:
     // "It is strongly recommended that you either remove this permission
     //  from this policy file or further restrict it to code sources
     //  that you specify, because Thread.stop() is potentially unsafe."
     // not even sure this method still works...
-    static final Permission BAD_DEFAULT_NUMBER_ONE = new RuntimePermission("stopThread");
+    private static final Permission BAD_DEFAULT_NUMBER_ONE = new BadDefaultPermission(new RuntimePermission("stopThread"), p -> true);
 
     // default policy file states:
     // "allows anyone to listen on dynamic ports"
     // specified exactly because that is what we want, and fastest since it won't imply any
     // expensive checks for the implicit "resolve"
-    static final Permission BAD_DEFAULT_NUMBER_TWO = new SocketPermission("localhost:0", "listen");
+    private static final Permission BAD_DEFAULT_NUMBER_TWO =
+        new BadDefaultPermission(
+            new SocketPermission("localhost:0", "listen"),
+            // we apply this pre-implies test because some SocketPermission#implies calls do expensive reverse-DNS resolves
+            p -> p instanceof SocketPermission && p.getActions().contains("listen"));
 
     /**
      * Wraps the Java system policy, filtering out bad default permissions that
@@ -159,7 +209,7 @@ static class SystemPolicy extends Policy {
 
         @Override
         public boolean implies(ProtectionDomain domain, Permission permission) {
-            if (BAD_DEFAULT_NUMBER_ONE.equals(permission) || BAD_DEFAULT_NUMBER_TWO.equals(permission)) {
+            if (BAD_DEFAULT_NUMBER_ONE.implies(permission) || BAD_DEFAULT_NUMBER_TWO.implies(permission)) {
                 return false;
             }
             return delegate.implies(domain, permission);
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java b/core/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java
index fa9b153047730..a8152569083e0 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java
@@ -24,25 +24,29 @@
 import joptsimple.OptionSpecBuilder;
 import joptsimple.util.PathConverter;
 import org.elasticsearch.Build;
+import org.elasticsearch.cli.EnvironmentAwareCommand;
 import org.elasticsearch.cli.ExitCodes;
-import org.elasticsearch.cli.SettingCommand;
 import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.cli.UserException;
+import org.elasticsearch.common.logging.LogConfigurator;
+import org.elasticsearch.env.Environment;
 import org.elasticsearch.monitor.jvm.JvmInfo;
+import org.elasticsearch.node.NodeValidationException;
 
 import java.io.IOException;
 import java.nio.file.Path;
+import java.security.Permission;
 import java.util.Arrays;
-import java.util.Map;
 
 /**
  * This class starts elasticsearch.
  */
-class Elasticsearch extends SettingCommand {
+class Elasticsearch extends EnvironmentAwareCommand {
 
     private final OptionSpecBuilder versionOption;
     private final OptionSpecBuilder daemonizeOption;
     private final OptionSpec<Path> pidfileOption;
+    private final OptionSpecBuilder quietOption;
 
     // visible for testing
     Elasticsearch() {
@@ -57,12 +61,25 @@ class Elasticsearch extends SettingCommand {
             .availableUnless(versionOption)
             .withRequiredArg()
             .withValuesConvertedBy(new PathConverter());
+        quietOption = parser.acceptsAll(Arrays.asList("q", "quiet"),
+            "Turns off standard ouput/error streams logging in console")
+            .availableUnless(versionOption)
+            .availableUnless(daemonizeOption);
     }
 
     /**
      * Main entry point for starting elasticsearch
      */
     public static void main(final String[] args) throws Exception {
+        // we want the JVM to think there is a security manager installed so that if internal policy decisions that would be based on the
+        // presence of a security manager or lack thereof act as if there is a security manager present (e.g., DNS cache policy)
+        System.setSecurityManager(new SecurityManager() {
+            @Override
+            public void checkPermission(Permission perm) {
+                // grant all permissions so that we can later set the security manager to the one that we want
+            }
+        });
+        LogConfigurator.registerErrorListener();
         final Elasticsearch elasticsearch = new Elasticsearch();
         int status = main(args, elasticsearch, Terminal.DEFAULT);
         if (status != ExitCodes.OK) {
@@ -75,7 +92,16 @@ static int main(final String[] args, final Elasticsearch elasticsearch, final Te
     }
 
     @Override
-    protected void execute(Terminal terminal, OptionSet options, Map<String, String> settings) throws Exception {
+    protected boolean shouldConfigureLoggingWithoutConfig() {
+        /*
+         * If we allow logging to be configured without a config before we are ready to read the log4j2.properties file, then we will fail
+         * to detect uses of logging before it is properly configured.
+         */
+        return false;
+    }
+
+    @Override
+    protected void execute(Terminal terminal, OptionSet options, Environment env) throws UserException {
         if (options.nonOptionArguments().isEmpty() == false) {
             throw new UserException(ExitCodes.USAGE, "Positional arguments not allowed, found " + options.nonOptionArguments());
         }
@@ -91,17 +117,23 @@ protected void execute(Terminal terminal, OptionSet options, Map<String, String>
 
         final boolean daemonize = options.has(daemonizeOption);
         final Path pidFile = pidfileOption.value(options);
+        final boolean quiet = options.has(quietOption);
 
-        init(daemonize, pidFile, settings);
+        try {
+            init(daemonize, pidFile, quiet, env);
+        } catch (NodeValidationException e) {
+            throw new UserException(ExitCodes.CONFIG, e.getMessage());
+        }
     }
 
-    void init(final boolean daemonize, final Path pidFile, final Map<String, String> esSettings) {
+    void init(final boolean daemonize, final Path pidFile, final boolean quiet, Environment initialEnv)
+        throws NodeValidationException, UserException {
         try {
-            Bootstrap.init(!daemonize, pidFile, esSettings);
-        } catch (final Throwable t) {
+            Bootstrap.init(!daemonize, pidFile, quiet, initialEnv);
+        } catch (BootstrapException | RuntimeException e) {
             // format exceptions to the console in a special way
             // to avoid 2MB stacktraces from guice, etc.
-            throw new StartupException(t);
+            throw new StartupException(e);
         }
     }
 
@@ -111,7 +143,8 @@ void init(final boolean daemonize, final Path pidFile, final Map<String, String>
      *
      * http://commons.apache.org/proper/commons-daemon/procrun.html
      *
-     * NOTE: If this method is renamed and/or moved, make sure to update service.bat!
+     * NOTE: If this method is renamed and/or moved, make sure to
+     * update elasticsearch-service.bat!
      */
     static void close(String[] args) throws IOException {
         Bootstrap.stop();
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/JNACLibrary.java b/core/src/main/java/org/elasticsearch/bootstrap/JNACLibrary.java
index fe0f400698fdb..64dabe9236306 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/JNACLibrary.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/JNACLibrary.java
@@ -40,6 +40,7 @@ final class JNACLibrary {
     public static final int ENOMEM = 12;
     public static final int RLIMIT_MEMLOCK = Constants.MAC_OS_X ? 6 : 8;
     public static final int RLIMIT_AS = Constants.MAC_OS_X ? 5 : 9;
+    public static final int RLIMIT_FSIZE = Constants.MAC_OS_X ? 1 : 1;
     public static final long RLIM_INFINITY = Constants.MAC_OS_X ? 9223372036854775807L : -1L;
 
     static {
@@ -61,7 +62,7 @@ public static final class Rlimit extends Structure implements Structure.ByRefere
 
         @Override
         protected List<String> getFieldOrder() {
-            return Arrays.asList(new String[] { "rlim_cur", "rlim_max" });
+            return Arrays.asList("rlim_cur", "rlim_max");
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/JNAKernel32Library.java b/core/src/main/java/org/elasticsearch/bootstrap/JNAKernel32Library.java
index 747ba2e458f4b..99574c2b39bb6 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/JNAKernel32Library.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/JNAKernel32Library.java
@@ -24,6 +24,7 @@
 import com.sun.jna.NativeLong;
 import com.sun.jna.Pointer;
 import com.sun.jna.Structure;
+import com.sun.jna.WString;
 import com.sun.jna.win32.StdCallLibrary;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.Constants;
@@ -109,7 +110,7 @@ class NativeHandlerCallback implements StdCallLibrary.StdCallCallback {
 
         private final ConsoleCtrlHandler handler;
 
-        public NativeHandlerCallback(ConsoleCtrlHandler handler) {
+        NativeHandlerCallback(ConsoleCtrlHandler handler) {
             this.handler = handler;
         }
 
@@ -149,17 +150,19 @@ public static class MemoryBasicInformation extends Structure {
 
         @Override
         protected List<String> getFieldOrder() {
-            return Arrays.asList(new String[]{"BaseAddress", "AllocationBase", "AllocationProtect", "RegionSize", "State", "Protect", "Type"});
+            return Arrays.asList("BaseAddress", "AllocationBase", "AllocationProtect", "RegionSize", "State", "Protect", "Type");
         }
     }
 
     public static class SizeT extends IntegerType {
 
+        // JNA requires this no-arg constructor to be public,
+        // otherwise it fails to register kernel32 library
         public SizeT() {
             this(0);
         }
 
-        public SizeT(long value) {
+        SizeT(long value) {
             super(Native.SIZE_T_SIZE, value);
         }
 
@@ -221,6 +224,17 @@ public SizeT(long value) {
      */
     native boolean CloseHandle(Pointer handle);
 
+    /**
+     * Retrieves the short path form of the specified path. See
+     * <a href="https://msdn.microsoft.com/en-us/library/windows/desktop/aa364989.aspx">{@code GetShortPathName}</a>.
+     *
+     * @param lpszLongPath  the path string
+     * @param lpszShortPath a buffer to receive the short name
+     * @param cchBuffer     the size of the buffer
+     * @return the length of the string copied into {@code lpszShortPath}, otherwise zero for failure
+     */
+    native int GetShortPathNameW(WString lpszLongPath, char[] lpszShortPath, int cchBuffer);
+
     /**
      * Creates or opens a new job object
      *
@@ -261,10 +275,8 @@ public static class JOBOBJECT_BASIC_LIMIT_INFORMATION extends Structure implemen
 
       @Override
       protected List<String> getFieldOrder() {
-          return Arrays.asList(new String[] {
-                  "PerProcessUserTimeLimit", "PerJobUserTimeLimit", "LimitFlags", "MinimumWorkingSetSize",
-                  "MaximumWorkingSetSize", "ActiveProcessLimit", "Affinity", "PriorityClass", "SchedulingClass"
-          });
+          return Arrays.asList("PerProcessUserTimeLimit", "PerJobUserTimeLimit", "LimitFlags", "MinimumWorkingSetSize",
+              "MaximumWorkingSetSize", "ActiveProcessLimit", "Affinity", "PriorityClass", "SchedulingClass");
       }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/JNANatives.java b/core/src/main/java/org/elasticsearch/bootstrap/JNANatives.java
index 5f3e357ff5f72..4a40db846e0df 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/JNANatives.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/JNANatives.java
@@ -21,6 +21,7 @@
 
 import com.sun.jna.Native;
 import com.sun.jna.Pointer;
+import com.sun.jna.WString;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.Constants;
 import org.elasticsearch.common.logging.Loggers;
@@ -43,17 +44,19 @@ private JNANatives() {}
 
     // Set to true, in case native mlockall call was successful
     static boolean LOCAL_MLOCKALL = false;
-    // Set to true, in case native seccomp call was successful
-    static boolean LOCAL_SECCOMP = false;
+    // Set to true, in case native system call filter install was successful
+    static boolean LOCAL_SYSTEM_CALL_FILTER = false;
     // Set to true, in case policy can be applied to all threads of the process (even existing ones)
     // otherwise they are only inherited for new threads (ES app threads)
-    static boolean LOCAL_SECCOMP_ALL = false;
+    static boolean LOCAL_SYSTEM_CALL_FILTER_ALL = false;
     // set to the maximum number of threads that can be created for
     // the user ID that owns the running Elasticsearch process
     static long MAX_NUMBER_OF_THREADS = -1;
 
     static long MAX_SIZE_VIRTUAL_MEMORY = Long.MIN_VALUE;
 
+    static long MAX_FILE_SIZE = Long.MIN_VALUE;
+
     static void tryMlockall() {
         int errno = Integer.MIN_VALUE;
         String errMsg = null;
@@ -137,6 +140,17 @@ static void trySetMaxSizeVirtualMemory() {
         }
     }
 
+    static void trySetMaxFileSize() {
+        if (Constants.LINUX || Constants.MAC_OS_X) {
+            final JNACLibrary.Rlimit rlimit = new JNACLibrary.Rlimit();
+            if (JNACLibrary.getrlimit(JNACLibrary.RLIMIT_FSIZE, rlimit) == 0) {
+                MAX_FILE_SIZE = rlimit.rlim_cur.longValue();
+            } else {
+                logger.warn("unable to retrieve max file size [" + JNACLibrary.strerror(Native.getLastError()) + "]");
+            }
+        }
+    }
+
     static String rlimitToString(long value) {
         assert Constants.LINUX || Constants.MAC_OS_X;
         if (value == JNACLibrary.RLIM_INFINITY) {
@@ -194,6 +208,35 @@ static void tryVirtualLock() {
         }
     }
 
+    /**
+     * Retrieves the short path form of the specified path.
+     *
+     * @param path the path
+     * @return the short path name (or the original path if getting the short path name fails for any reason)
+     */
+    static String getShortPathName(String path) {
+        assert Constants.WINDOWS;
+        try {
+            final WString longPath = new WString("\\\\?\\" + path);
+            // first we get the length of the buffer needed
+            final int length = JNAKernel32Library.getInstance().GetShortPathNameW(longPath, null, 0);
+            if (length == 0) {
+                logger.warn("failed to get short path name: {}", Native.getLastError());
+                return path;
+            }
+            final char[] shortPath = new char[length];
+            // knowing the length of the buffer, now we get the short name
+            if (JNAKernel32Library.getInstance().GetShortPathNameW(longPath, shortPath, length) > 0) {
+                return Native.toString(shortPath);
+            } else {
+                logger.warn("failed to get short path name: {}", Native.getLastError());
+                return path;
+            }
+        } catch (final UnsatisfiedLinkError e) {
+            return path;
+        }
+    }
+
     static void addConsoleCtrlHandler(ConsoleCtrlHandler handler) {
         // The console Ctrl handler is necessary on Windows platforms only.
         if (Constants.WINDOWS) {
@@ -210,12 +253,12 @@ static void addConsoleCtrlHandler(ConsoleCtrlHandler handler) {
         }
     }
 
-    static void trySeccomp(Path tmpFile) {
+    static void tryInstallSystemCallFilter(Path tmpFile) {
         try {
-            int ret = Seccomp.init(tmpFile);
-            LOCAL_SECCOMP = true;
+            int ret = SystemCallFilter.init(tmpFile);
+            LOCAL_SYSTEM_CALL_FILTER = true;
             if (ret == 1) {
-                LOCAL_SECCOMP_ALL = true;
+                LOCAL_SYSTEM_CALL_FILTER_ALL = true;
             }
         } catch (Exception e) {
             // this is likely to happen unless the kernel is newish, its a best effort at the moment
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/JVMCheck.java b/core/src/main/java/org/elasticsearch/bootstrap/JVMCheck.java
deleted file mode 100644
index c367b38a79baa..0000000000000
--- a/core/src/main/java/org/elasticsearch/bootstrap/JVMCheck.java
+++ /dev/null
@@ -1,246 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.bootstrap;
-
-import org.apache.lucene.util.Constants;
-import org.elasticsearch.common.logging.Loggers;
-import org.elasticsearch.monitor.jvm.JvmInfo;
-
-import java.lang.management.ManagementFactory;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Optional;
-
-/** Checks that the JVM is ok and won't cause index corruption */
-final class JVMCheck {
-    /** no instantiation */
-    private JVMCheck() {}
-
-    /**
-     * URL with latest JVM recommendations
-     */
-    static final String JVM_RECOMMENDATIONS = "http://www.elastic.co/guide/en/elasticsearch/reference/current/_installation.html";
-
-    /**
-     * System property which if set causes us to bypass the check completely (but issues a warning in doing so)
-     */
-    static final String JVM_BYPASS = "es.bypass.vm.check";
-
-    /**
-     * Metadata and messaging for checking and reporting HotSpot
-     * issues.
-     */
-    interface HotSpotCheck {
-        /**
-         * If this HotSpot check should be executed.
-         *
-         * @return true if this HotSpot check should be executed
-         */
-        boolean check();
-
-        /**
-         * The error message to display when this HotSpot issue is
-         * present.
-         *
-         * @return the error message for this HotSpot issue
-         */
-        String getErrorMessage();
-
-        /**
-         * The warning message for this HotSpot issue if a workaround
-         * exists and is used.
-         *
-         * @return the warning message for this HotSpot issue
-         */
-        Optional<String> getWarningMessage();
-
-        /**
-         * The workaround for this HotSpot issue, if one exists.
-         *
-         * @return the workaround for this HotSpot issue, if one exists
-         */
-        Optional<String> getWorkaround();
-    }
-
-    /**
-     * Metadata and messaging for hotspot bugs.
-     */
-    static class HotspotBug implements HotSpotCheck {
-
-        /** OpenJDK bug URL */
-        final String bugUrl;
-
-        /** Compiler workaround flag (null if there is no workaround) */
-        final String workAround;
-
-        HotspotBug(String bugUrl, String workAround) {
-            this.bugUrl = bugUrl;
-            this.workAround = workAround;
-        }
-
-        /** Returns an error message to the user for a broken version */
-        public String getErrorMessage() {
-            StringBuilder sb = new StringBuilder();
-            sb.append("Java version: ").append(fullVersion());
-            sb.append(" suffers from critical bug ").append(bugUrl);
-            sb.append(" which can cause data corruption.");
-            sb.append(System.lineSeparator());
-            sb.append("Please upgrade the JVM, see ").append(JVM_RECOMMENDATIONS);
-            sb.append(" for current recommendations.");
-            if (workAround != null) {
-                sb.append(System.lineSeparator());
-                sb.append("If you absolutely cannot upgrade, please add ").append(workAround);
-                sb.append(" to the ES_JAVA_OPTS environment variable.");
-                sb.append(System.lineSeparator());
-                sb.append("Upgrading is preferred, this workaround will result in degraded performance.");
-            }
-            return sb.toString();
-        }
-
-        /** Warns the user when a workaround is being used to dodge the bug */
-        public Optional<String> getWarningMessage() {
-            StringBuilder sb = new StringBuilder();
-            sb.append("Workaround flag ").append(workAround);
-            sb.append(" for bug ").append(bugUrl);
-            sb.append(" found. ");
-            sb.append(System.lineSeparator());
-            sb.append("This will result in degraded performance!");
-            sb.append(System.lineSeparator());
-            sb.append("Upgrading is preferred, see ").append(JVM_RECOMMENDATIONS);
-            sb.append(" for current recommendations.");
-            return Optional.of(sb.toString());
-        }
-
-        public boolean check() {
-            return true;
-        }
-
-        @Override
-        public Optional<String> getWorkaround() {
-            return Optional.of(workAround);
-        }
-    }
-
-    static class G1GCCheck implements HotSpotCheck {
-        @Override
-        public boolean check() {
-            return JvmInfo.jvmInfo().useG1GC().equals("true");
-        }
-
-        /** Returns an error message to the user for a broken version */
-        public String getErrorMessage() {
-            StringBuilder sb = new StringBuilder();
-            sb.append("Java version: ").append(fullVersion());
-            sb.append(" can cause data corruption");
-            sb.append(" when used with G1GC.");
-            sb.append(System.lineSeparator());
-            sb.append("Please upgrade the JVM, see ").append(JVM_RECOMMENDATIONS);
-            sb.append(" for current recommendations.");
-            return sb.toString();
-        }
-
-        @Override
-        public Optional<String> getWarningMessage() {
-            return Optional.empty();
-        }
-
-        @Override
-        public Optional<String> getWorkaround() {
-            return Optional.empty();
-        }
-    }
-
-    /** mapping of hotspot version to hotspot bug information for the most serious bugs */
-    static final Map<String, HotSpotCheck> JVM_BROKEN_HOTSPOT_VERSIONS;
-
-    static {
-        Map<String, HotSpotCheck> bugs = new HashMap<>();
-
-        // 1.7.0: loop optimizer bug
-        bugs.put("21.0-b17",  new HotspotBug("https://bugs.openjdk.java.net/browse/JDK-7070134", "-XX:-UseLoopPredicate"));
-        // register allocation issues (technically only x86/amd64). This impacted update 40, 45, and 51
-        bugs.put("24.0-b56",  new HotspotBug("https://bugs.openjdk.java.net/browse/JDK-8024830", "-XX:-UseSuperWord"));
-        bugs.put("24.45-b08", new HotspotBug("https://bugs.openjdk.java.net/browse/JDK-8024830", "-XX:-UseSuperWord"));
-        bugs.put("24.51-b03", new HotspotBug("https://bugs.openjdk.java.net/browse/JDK-8024830", "-XX:-UseSuperWord"));
-        G1GCCheck g1GcCheck = new G1GCCheck();
-        bugs.put("25.0-b70", g1GcCheck);
-        bugs.put("25.11-b03", g1GcCheck);
-        bugs.put("25.20-b23", g1GcCheck);
-        bugs.put("25.25-b02", g1GcCheck);
-        bugs.put("25.31-b07", g1GcCheck);
-
-        JVM_BROKEN_HOTSPOT_VERSIONS = Collections.unmodifiableMap(bugs);
-    }
-
-    /**
-     * Checks that the current JVM is "ok". This means it doesn't have severe bugs that cause data corruption.
-     */
-    static void check() {
-        if (Boolean.parseBoolean(System.getProperty(JVM_BYPASS))) {
-            Loggers.getLogger(JVMCheck.class).warn("bypassing jvm version check for version [{}], this can result in data corruption!", fullVersion());
-        } else if ("Oracle Corporation".equals(Constants.JVM_VENDOR)) {
-            HotSpotCheck bug = JVM_BROKEN_HOTSPOT_VERSIONS.get(Constants.JVM_VERSION);
-            if (bug != null && bug.check()) {
-                if (bug.getWorkaround().isPresent() && ManagementFactory.getRuntimeMXBean().getInputArguments().contains(bug.getWorkaround().get())) {
-                    Loggers.getLogger(JVMCheck.class).warn("{}", bug.getWarningMessage().get());
-                } else {
-                    throw new RuntimeException(bug.getErrorMessage());
-                }
-            }
-        } else if ("IBM Corporation".equals(Constants.JVM_VENDOR)) {
-            // currently some old JVM versions from IBM will easily result in index corruption.
-            // 2.8+ seems ok for ES from testing.
-            float version = Float.POSITIVE_INFINITY;
-            try {
-                version = Float.parseFloat(Constants.JVM_VERSION);
-            } catch (NumberFormatException ignored) {
-                // this is just a simple best-effort to detect old runtimes,
-                // if we cannot parse it, we don't fail.
-            }
-            if (version < 2.8f) {
-                StringBuilder sb = new StringBuilder();
-                sb.append("IBM J9 runtimes < 2.8 suffer from several bugs which can cause data corruption.");
-                sb.append(System.lineSeparator());
-                sb.append("Your version: " + fullVersion());
-                sb.append(System.lineSeparator());
-                sb.append("Please upgrade the JVM to a recent IBM JDK");
-                throw new RuntimeException(sb.toString());
-            }
-        }
-    }
-
-    /**
-     * Returns java + jvm version, looks like this:
-     * {@code Oracle Corporation 1.8.0_45 [Java HotSpot(TM) 64-Bit Server VM 25.45-b02]}
-     */
-    static String fullVersion() {
-        StringBuilder sb = new StringBuilder();
-        sb.append(Constants.JAVA_VENDOR);
-        sb.append(" ");
-        sb.append(Constants.JAVA_VERSION);
-        sb.append(" [");
-        sb.append(Constants.JVM_NAME);
-        sb.append(" ");
-        sb.append(Constants.JVM_VERSION);
-        sb.append("]");
-        return sb.toString();
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/JarHell.java b/core/src/main/java/org/elasticsearch/bootstrap/JarHell.java
index 4a0aaec68c250..1959e5e81394b 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/JarHell.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/JarHell.java
@@ -27,6 +27,7 @@
 
 import java.io.IOException;
 import java.net.MalformedURLException;
+import java.net.URISyntaxException;
 import java.net.URL;
 import java.net.URLClassLoader;
 import java.nio.file.FileVisitResult;
@@ -35,9 +36,11 @@
 import java.nio.file.SimpleFileVisitor;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.Enumeration;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedHashSet;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
@@ -74,7 +77,7 @@ public static void main(String args[]) throws Exception {
      * Checks the current classpath for duplicate classes
      * @throws IllegalStateException if jar hell was found
      */
-    public static void checkJarHell() throws Exception {
+    public static void checkJarHell() throws IOException, URISyntaxException {
         ClassLoader loader = JarHell.class.getClassLoader();
         Logger logger = Loggers.getLogger(JarHell.class);
         if (logger.isDebugEnabled()) {
@@ -92,7 +95,7 @@ public static void checkJarHell() throws Exception {
      * @return array of URLs
      * @throws IllegalStateException if the classpath contains empty elements
      */
-    public static URL[] parseClassPath()  {
+    public static Set<URL> parseClassPath()  {
         return parseClassPath(System.getProperty("java.class.path"));
     }
 
@@ -103,13 +106,12 @@ public static URL[] parseClassPath()  {
      * @throws IllegalStateException if the classpath contains empty elements
      */
     @SuppressForbidden(reason = "resolves against CWD because that is how classpaths work")
-    static URL[] parseClassPath(String classPath) {
+    static Set<URL> parseClassPath(String classPath) {
         String pathSeparator = System.getProperty("path.separator");
         String fileSeparator = System.getProperty("file.separator");
         String elements[] = classPath.split(pathSeparator);
-        URL urlElements[] = new URL[elements.length];
-        for (int i = 0; i < elements.length; i++) {
-            String element = elements[i];
+        Set<URL> urlElements = new LinkedHashSet<>(); // order is already lost, but some filesystems have it
+        for (String element : elements) {
             // Technically empty classpath element behaves like CWD.
             // So below is the "correct" code, however in practice with ES, this is usually just a misconfiguration,
             // from old shell scripts left behind or something:
@@ -135,13 +137,17 @@ static URL[] parseClassPath(String classPath) {
             }
             // now just parse as ordinary file
             try {
-                urlElements[i] = PathUtils.get(element).toUri().toURL();
+                URL url = PathUtils.get(element).toUri().toURL();
+                if (urlElements.add(url) == false) {
+                    throw new IllegalStateException("jar hell!" + System.lineSeparator() +
+                        "duplicate jar [" + element + "] on classpath: " + classPath);
+                }
             } catch (MalformedURLException e) {
                 // should not happen, as we use the filesystem API
                 throw new RuntimeException(e);
             }
         }
-        return urlElements;
+        return Collections.unmodifiableSet(urlElements);
     }
 
     /**
@@ -149,7 +155,7 @@ static URL[] parseClassPath(String classPath) {
      * @throws IllegalStateException if jar hell was found
      */
     @SuppressForbidden(reason = "needs JarFile for speed, just reading entries")
-    public static void checkJarHell(URL urls[]) throws Exception {
+    public static void checkJarHell(Set<URL> urls) throws URISyntaxException, IOException {
         Logger logger = Loggers.getLogger(JarHell.class);
         // we don't try to be sneaky and use deprecated/internal/not portable stuff
         // like sun.boot.class.path, and with jigsaw we don't yet have a way to get
@@ -167,8 +173,8 @@ public static void checkJarHell(URL urls[]) throws Exception {
             }
             if (path.toString().endsWith(".jar")) {
                 if (!seenJars.add(path)) {
-                    logger.debug("excluding duplicate classpath element: {}", path);
-                    continue;
+                    throw new IllegalStateException("jar hell!" + System.lineSeparator() +
+                                                    "duplicate jar on classpath: " + path);
                 }
                 logger.debug("examining jar: {}", path);
                 try (JarFile file = new JarFile(path.toString())) {
@@ -197,8 +203,8 @@ public static void checkJarHell(URL urls[]) throws Exception {
                     public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
                         String entry = root.relativize(file).toString();
                         if (entry.endsWith(".class")) {
-                            // normalize with the os separator
-                            entry = entry.replace(sep, ".").substring(0,  entry.length() - 6);
+                            // normalize with the os separator, remove '.class'
+                            entry = entry.replace(sep, ".").substring(0,  entry.length() - ".class".length());
                             checkClass(clazzes, entry, path);
                         }
                         return super.visitFile(file, attrs);
@@ -271,14 +277,6 @@ static void checkClass(Map<String,Path> clazzes, String clazz, Path jarpath) {
                         "class: " + clazz + System.lineSeparator() +
                         "exists multiple times in jar: " + jarpath + " !!!!!!!!!");
             } else {
-                if (clazz.startsWith("org.apache.logging.log4j.core.impl.ThrowableProxy")) {
-                    /*
-                     * deliberate to hack around a bug in Log4j
-                     * cf. https://github.com/elastic/elasticsearch/issues/20304
-                     * cf. https://issues.apache.org/jira/browse/LOG4J2-1560
-                     */
-                    return;
-                }
                 throw new IllegalStateException("jar hell!" + System.lineSeparator() +
                         "class: " + clazz + System.lineSeparator() +
                         "jar1: " + previous + System.lineSeparator() +
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/JavaVersion.java b/core/src/main/java/org/elasticsearch/bootstrap/JavaVersion.java
index 63de83d88d0f8..03722e03060a7 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/JavaVersion.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/JavaVersion.java
@@ -33,9 +33,7 @@ public List<Integer> getVersion() {
     }
 
     private JavaVersion(List<Integer> version) {
-        if (version.size() >= 2
-                && version.get(0).intValue() == 1
-                && version.get(1).intValue() == 8) {
+        if (version.size() >= 2 && version.get(0) == 1 && version.get(1) == 8) {
             // for Java 8 there is ambiguity since both 1.8 and 8 are supported,
             // so we rewrite the former to the latter
             version = new ArrayList<>(version.subList(1, version.size()));
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Natives.java b/core/src/main/java/org/elasticsearch/bootstrap/Natives.java
index 9fad34e329f58..ef40b5b862e26 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/Natives.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/Natives.java
@@ -76,6 +76,20 @@ static void tryVirtualLock() {
         JNANatives.tryVirtualLock();
     }
 
+    /**
+     * Retrieves the short path form of the specified path.
+     *
+     * @param path the path
+     * @return the short path name (or the original path if getting the short path name fails for any reason)
+     */
+    static String getShortPathName(final String path) {
+        if (!JNA_AVAILABLE) {
+            logger.warn("cannot obtain short path for [{}] because JNA is not avilable", path);
+            return path;
+        }
+        return JNANatives.getShortPathName(path);
+    }
+
     static void addConsoleCtrlHandler(ConsoleCtrlHandler handler) {
         if (!JNA_AVAILABLE) {
             logger.warn("cannot register console handler because JNA is not available");
@@ -91,12 +105,12 @@ static boolean isMemoryLocked() {
         return JNANatives.LOCAL_MLOCKALL;
     }
 
-    static void trySeccomp(Path tmpFile) {
+    static void tryInstallSystemCallFilter(Path tmpFile) {
         if (!JNA_AVAILABLE) {
-            logger.warn("cannot install syscall filters because JNA is not available");
+            logger.warn("cannot install system call filter because JNA is not available");
             return;
         }
-        JNANatives.trySeccomp(tmpFile);
+        JNANatives.tryInstallSystemCallFilter(tmpFile);
     }
 
     static void trySetMaxNumberOfThreads() {
@@ -115,10 +129,18 @@ static void trySetMaxSizeVirtualMemory() {
         JNANatives.trySetMaxSizeVirtualMemory();
     }
 
-    static boolean isSeccompInstalled() {
+    static void trySetMaxFileSize() {
+        if (!JNA_AVAILABLE) {
+            logger.warn("cannot getrlimit RLIMIT_FSIZE because JNA is not available");
+            return;
+        }
+        JNANatives.trySetMaxFileSize();
+    }
+
+    static boolean isSystemCallFilterInstalled() {
         if (!JNA_AVAILABLE) {
             return false;
         }
-        return JNANatives.LOCAL_SECCOMP;
+        return JNANatives.LOCAL_SYSTEM_CALL_FILTER;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Seccomp.java b/core/src/main/java/org/elasticsearch/bootstrap/Seccomp.java
deleted file mode 100644
index 88c618d445cd0..0000000000000
--- a/core/src/main/java/org/elasticsearch/bootstrap/Seccomp.java
+++ /dev/null
@@ -1,649 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.bootstrap;
-
-import com.sun.jna.Library;
-import com.sun.jna.Memory;
-import com.sun.jna.Native;
-import com.sun.jna.NativeLong;
-import com.sun.jna.Pointer;
-import com.sun.jna.Structure;
-import com.sun.jna.ptr.PointerByReference;
-import org.apache.logging.log4j.Logger;
-import org.apache.lucene.util.Constants;
-import org.apache.lucene.util.IOUtils;
-import org.elasticsearch.common.logging.Loggers;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.ByteOrder;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Installs a limited form of secure computing mode,
- * to filters system calls to block process execution.
- * <p>
- * This is supported on Linux, Solaris, FreeBSD, OpenBSD, Mac OS X, and Windows.
- * <p>
- * On Linux it currently supports amd64 and i386 architectures, requires Linux kernel 3.5 or above, and requires
- * {@code CONFIG_SECCOMP} and {@code CONFIG_SECCOMP_FILTER} compiled into the kernel.
- * <p>
- * On Linux BPF Filters are installed using either {@code seccomp(2)} (3.17+) or {@code prctl(2)} (3.5+). {@code seccomp(2)}
- * is preferred, as it allows filters to be applied to any existing threads in the process, and one motivation
- * here is to protect against bugs in the JVM. Otherwise, code will fall back to the {@code prctl(2)} method
- * which will at least protect elasticsearch application threads.
- * <p>
- * Linux BPF filters will return {@code EACCES} (Access Denied) for the following system calls:
- * <ul>
- *   <li>{@code execve}</li>
- *   <li>{@code fork}</li>
- *   <li>{@code vfork}</li>
- *   <li>{@code execveat}</li>
- * </ul>
- * <p>
- * On Solaris 10 or higher, the following privileges are dropped with {@code priv_set(3C)}:
- * <ul>
- *   <li>{@code PRIV_PROC_FORK}</li>
- *   <li>{@code PRIV_PROC_EXEC}</li>
- * </ul>
- * <p>
- * On BSD systems, process creation is restricted with {@code setrlimit(RLIMIT_NPROC)}.
- * <p>
- * On Mac OS X Leopard or above, a custom {@code sandbox(7)} ("Seatbelt") profile is installed that
- * denies the following rules:
- * <ul>
- *   <li>{@code process-fork}</li>
- *   <li>{@code process-exec}</li>
- * </ul>
- * <p>
- * On Windows, process creation is restricted with {@code SetInformationJobObject/ActiveProcessLimit}.
- * <p>
- * This is not intended as a sandbox. It is another level of security, mostly intended to annoy
- * security researchers and make their lives more difficult in achieving "remote execution" exploits.
- * @see <a href="http://www.kernel.org/doc/Documentation/prctl/seccomp_filter.txt">
- *      http://www.kernel.org/doc/Documentation/prctl/seccomp_filter.txt</a>
- * @see <a href="https://reverse.put.as/wp-content/uploads/2011/06/The-Apple-Sandbox-BHDC2011-Paper.pdf">
- *      https://reverse.put.as/wp-content/uploads/2011/06/The-Apple-Sandbox-BHDC2011-Paper.pdf</a>
- * @see <a href="https://docs.oracle.com/cd/E23824_01/html/821-1456/prbac-2.html">
- *      https://docs.oracle.com/cd/E23824_01/html/821-1456/prbac-2.html</a>
- */
-// not an example of how to write code!!!
-final class Seccomp {
-    private static final Logger logger = Loggers.getLogger(Seccomp.class);
-
-    // Linux implementation, based on seccomp(2) or prctl(2) with bpf filtering
-
-    /** Access to non-standard Linux libc methods */
-    interface LinuxLibrary extends Library {
-        /**
-         * maps to prctl(2)
-         */
-        int prctl(int option, NativeLong arg2, NativeLong arg3, NativeLong arg4, NativeLong arg5);
-        /**
-         * used to call seccomp(2), its too new...
-         * this is the only way, DON'T use it on some other architecture unless you know wtf you are doing
-         */
-        NativeLong syscall(NativeLong number, Object... args);
-    }
-
-    // null if unavailable or something goes wrong.
-    private static final LinuxLibrary linux_libc;
-
-    static {
-        LinuxLibrary lib = null;
-        if (Constants.LINUX) {
-            try {
-                lib = (LinuxLibrary) Native.loadLibrary("c", LinuxLibrary.class);
-            } catch (UnsatisfiedLinkError e) {
-                logger.warn("unable to link C library. native methods (seccomp) will be disabled.", e);
-            }
-        }
-        linux_libc = lib;
-    }
-
-    /** the preferred method is seccomp(2), since we can apply to all threads of the process */
-    static final int SECCOMP_SET_MODE_FILTER   =   1;   // since Linux 3.17
-    static final int SECCOMP_FILTER_FLAG_TSYNC =   1;   // since Linux 3.17
-
-    /** otherwise, we can use prctl(2), which will at least protect ES application threads */
-    static final int PR_GET_NO_NEW_PRIVS       =  39;   // since Linux 3.5
-    static final int PR_SET_NO_NEW_PRIVS       =  38;   // since Linux 3.5
-    static final int PR_GET_SECCOMP            =  21;   // since Linux 2.6.23
-    static final int PR_SET_SECCOMP            =  22;   // since Linux 2.6.23
-    static final long SECCOMP_MODE_FILTER      =   2;   // since Linux Linux 3.5
-
-    /** corresponds to struct sock_filter */
-    static final class SockFilter {
-        short code; // insn
-        byte jt;    // number of insn to jump (skip) if true
-        byte jf;    // number of insn to jump (skip) if false
-        int k;      // additional data
-
-        SockFilter(short code, byte jt, byte jf, int k) {
-            this.code = code;
-            this.jt = jt;
-            this.jf = jf;
-            this.k = k;
-        }
-    }
-
-    /** corresponds to struct sock_fprog */
-    public static final class SockFProg extends Structure implements Structure.ByReference {
-        public short   len;           // number of filters
-        public Pointer filter;        // filters
-
-        public SockFProg(SockFilter filters[]) {
-            len = (short) filters.length;
-            // serialize struct sock_filter * explicitly, its less confusing than the JNA magic we would need
-            Memory filter = new Memory(len * 8);
-            ByteBuffer bbuf = filter.getByteBuffer(0, len * 8);
-            bbuf.order(ByteOrder.nativeOrder()); // little endian
-            for (SockFilter f : filters) {
-                bbuf.putShort(f.code);
-                bbuf.put(f.jt);
-                bbuf.put(f.jf);
-                bbuf.putInt(f.k);
-            }
-            this.filter = filter;
-        }
-
-        @Override
-        protected List<String> getFieldOrder() {
-            return Arrays.asList(new String[] { "len", "filter" });
-        }
-    }
-
-    // BPF "macros" and constants
-    static final int BPF_LD  = 0x00;
-    static final int BPF_W   = 0x00;
-    static final int BPF_ABS = 0x20;
-    static final int BPF_JMP = 0x05;
-    static final int BPF_JEQ = 0x10;
-    static final int BPF_JGE = 0x30;
-    static final int BPF_JGT = 0x20;
-    static final int BPF_RET = 0x06;
-    static final int BPF_K   = 0x00;
-
-    static SockFilter BPF_STMT(int code, int k) {
-        return new SockFilter((short) code, (byte) 0, (byte) 0, k);
-    }
-
-    static SockFilter BPF_JUMP(int code, int k, int jt, int jf) {
-        return new SockFilter((short) code, (byte) jt, (byte) jf, k);
-    }
-
-    static final int SECCOMP_RET_ERRNO = 0x00050000;
-    static final int SECCOMP_RET_DATA  = 0x0000FFFF;
-    static final int SECCOMP_RET_ALLOW = 0x7FFF0000;
-
-    // some errno constants for error checking/handling
-    static final int EPERM  = 0x01;
-    static final int EACCES = 0x0D;
-    static final int EFAULT = 0x0E;
-    static final int EINVAL = 0x16;
-    static final int ENOSYS = 0x26;
-
-    // offsets that our BPF checks
-    // check with offsetof() when adding a new arch, move to Arch if different.
-    static final int SECCOMP_DATA_NR_OFFSET   = 0x00;
-    static final int SECCOMP_DATA_ARCH_OFFSET = 0x04;
-
-    static class Arch {
-        /** AUDIT_ARCH_XXX constant from linux/audit.h */
-        final int audit;
-        /** syscall limit (necessary for blacklisting on amd64, to ban 32-bit syscalls) */
-        final int limit;
-        /** __NR_fork */
-        final int fork;
-        /** __NR_vfork */
-        final int vfork;
-        /** __NR_execve */
-        final int execve;
-        /**  __NR_execveat */
-        final int execveat;
-        /** __NR_seccomp */
-        final int seccomp;
-
-        Arch(int audit, int limit, int fork, int vfork, int execve, int execveat, int seccomp) {
-            this.audit = audit;
-            this.limit = limit;
-            this.fork = fork;
-            this.vfork = vfork;
-            this.execve = execve;
-            this.execveat = execveat;
-            this.seccomp = seccomp;
-        }
-    }
-
-    /** supported architectures map keyed by os.arch */
-    private static final Map<String,Arch> ARCHITECTURES;
-    static {
-        Map<String,Arch> m = new HashMap<>();
-        m.put("amd64", new Arch(0xC000003E, 0x3FFFFFFF, 57, 58, 59, 322, 317));
-        m.put("i386",  new Arch(0x40000003, 0xFFFFFFFF, 2, 190, 11, 358, 354));
-        ARCHITECTURES = Collections.unmodifiableMap(m);
-    }
-
-    /** invokes prctl() from linux libc library */
-    private static int linux_prctl(int option, long arg2, long arg3, long arg4, long arg5) {
-        return linux_libc.prctl(option, new NativeLong(arg2), new NativeLong(arg3), new NativeLong(arg4), new NativeLong(arg5));
-    }
-
-    /** invokes syscall() from linux libc library */
-    private static long linux_syscall(long number, Object... args) {
-        return linux_libc.syscall(new NativeLong(number), args).longValue();
-    }
-
-    /** try to install our BPF filters via seccomp() or prctl() to block execution */
-    private static int linuxImpl() {
-        // first be defensive: we can give nice errors this way, at the very least.
-        // also, some of these security features get backported to old versions, checking kernel version here is a big no-no!
-        final Arch arch = ARCHITECTURES.get(Constants.OS_ARCH);
-        boolean supported = Constants.LINUX && arch != null;
-        if (supported == false) {
-            throw new UnsupportedOperationException("seccomp unavailable: '" + Constants.OS_ARCH + "' architecture unsupported");
-        }
-
-        // we couldn't link methods, could be some really ancient kernel (e.g. < 2.1.57) or some bug
-        if (linux_libc == null) {
-            throw new UnsupportedOperationException("seccomp unavailable: could not link methods. requires kernel 3.5+ with CONFIG_SECCOMP and CONFIG_SECCOMP_FILTER compiled in");
-        }
-
-        // pure paranoia:
-
-        // check that unimplemented syscalls actually return ENOSYS
-        // you never know (e.g. https://code.google.com/p/chromium/issues/detail?id=439795)
-        if (linux_syscall(999) >= 0) {
-            throw new UnsupportedOperationException("seccomp unavailable: your kernel is buggy and you should upgrade");
-        }
-
-        switch (Native.getLastError()) {
-            case ENOSYS:
-                break; // ok
-            case EPERM:
-                // NOT ok, but likely a docker container
-                if (logger.isDebugEnabled()) {
-                    logger.debug("syscall(BOGUS) bogusly gets EPERM instead of ENOSYS");
-                }
-                break;
-            default:
-                throw new UnsupportedOperationException("seccomp unavailable: your kernel is buggy and you should upgrade");
-        }
-
-        // try to check system calls really are who they claim
-        // you never know (e.g. https://chromium.googlesource.com/chromium/src.git/+/master/sandbox/linux/seccomp-bpf/sandbox_bpf.cc#57)
-        final int bogusArg = 0xf7a46a5c;
-
-        // test seccomp(BOGUS)
-        long ret = linux_syscall(arch.seccomp, bogusArg);
-        if (ret != -1) {
-            throw new UnsupportedOperationException("seccomp unavailable: seccomp(BOGUS_OPERATION) returned " + ret);
-        } else {
-            int errno = Native.getLastError();
-            switch (errno) {
-                case ENOSYS: break; // ok
-                case EINVAL: break; // ok
-                default: throw new UnsupportedOperationException("seccomp(BOGUS_OPERATION): " + JNACLibrary.strerror(errno));
-            }
-        }
-
-        // test seccomp(VALID, BOGUS)
-        ret = linux_syscall(arch.seccomp, SECCOMP_SET_MODE_FILTER, bogusArg);
-        if (ret != -1) {
-            throw new UnsupportedOperationException("seccomp unavailable: seccomp(SECCOMP_SET_MODE_FILTER, BOGUS_FLAG) returned " + ret);
-        } else {
-            int errno = Native.getLastError();
-            switch (errno) {
-                case ENOSYS: break; // ok
-                case EINVAL: break; // ok
-                default: throw new UnsupportedOperationException("seccomp(SECCOMP_SET_MODE_FILTER, BOGUS_FLAG): " + JNACLibrary.strerror(errno));
-            }
-        }
-
-        // test prctl(BOGUS)
-        ret = linux_prctl(bogusArg, 0, 0, 0, 0);
-        if (ret != -1) {
-            throw new UnsupportedOperationException("seccomp unavailable: prctl(BOGUS_OPTION) returned " + ret);
-        } else {
-            int errno = Native.getLastError();
-            switch (errno) {
-                case ENOSYS: break; // ok
-                case EINVAL: break; // ok
-                default: throw new UnsupportedOperationException("prctl(BOGUS_OPTION): " + JNACLibrary.strerror(errno));
-            }
-        }
-
-        // now just normal defensive checks
-
-        // check for GET_NO_NEW_PRIVS
-        switch (linux_prctl(PR_GET_NO_NEW_PRIVS, 0, 0, 0, 0)) {
-            case 0: break; // not yet set
-            case 1: break; // already set by caller
-            default:
-                int errno = Native.getLastError();
-                if (errno == EINVAL) {
-                    // friendly error, this will be the typical case for an old kernel
-                    throw new UnsupportedOperationException("seccomp unavailable: requires kernel 3.5+ with CONFIG_SECCOMP and CONFIG_SECCOMP_FILTER compiled in");
-                } else {
-                    throw new UnsupportedOperationException("prctl(PR_GET_NO_NEW_PRIVS): " + JNACLibrary.strerror(errno));
-                }
-        }
-        // check for SECCOMP
-        switch (linux_prctl(PR_GET_SECCOMP, 0, 0, 0, 0)) {
-            case 0: break; // not yet set
-            case 2: break; // already in filter mode by caller
-            default:
-                int errno = Native.getLastError();
-                if (errno == EINVAL) {
-                    throw new UnsupportedOperationException("seccomp unavailable: CONFIG_SECCOMP not compiled into kernel, CONFIG_SECCOMP and CONFIG_SECCOMP_FILTER are needed");
-                } else {
-                    throw new UnsupportedOperationException("prctl(PR_GET_SECCOMP): " + JNACLibrary.strerror(errno));
-                }
-        }
-        // check for SECCOMP_MODE_FILTER
-        if (linux_prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, 0, 0, 0) != 0) {
-            int errno = Native.getLastError();
-            switch (errno) {
-                case EFAULT: break; // available
-                case EINVAL: throw new UnsupportedOperationException("seccomp unavailable: CONFIG_SECCOMP_FILTER not compiled into kernel, CONFIG_SECCOMP and CONFIG_SECCOMP_FILTER are needed");
-                default: throw new UnsupportedOperationException("prctl(PR_SET_SECCOMP): " + JNACLibrary.strerror(errno));
-            }
-        }
-
-        // ok, now set PR_SET_NO_NEW_PRIVS, needed to be able to set a seccomp filter as ordinary user
-        if (linux_prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) != 0) {
-            throw new UnsupportedOperationException("prctl(PR_SET_NO_NEW_PRIVS): " + JNACLibrary.strerror(Native.getLastError()));
-        }
-
-        // check it worked
-        if (linux_prctl(PR_GET_NO_NEW_PRIVS, 0, 0, 0, 0) != 1) {
-            throw new UnsupportedOperationException("seccomp filter did not really succeed: prctl(PR_GET_NO_NEW_PRIVS): " + JNACLibrary.strerror(Native.getLastError()));
-        }
-
-        // BPF installed to check arch, limit, then syscall. See https://www.kernel.org/doc/Documentation/prctl/seccomp_filter.txt for details.
-        SockFilter insns[] = {
-          /* 1  */ BPF_STMT(BPF_LD  + BPF_W   + BPF_ABS, SECCOMP_DATA_ARCH_OFFSET),             //
-          /* 2  */ BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K,   arch.audit,     0, 7),                 // if (arch != audit) goto fail;
-          /* 3  */ BPF_STMT(BPF_LD  + BPF_W   + BPF_ABS, SECCOMP_DATA_NR_OFFSET),               //
-          /* 4  */ BPF_JUMP(BPF_JMP + BPF_JGT + BPF_K,   arch.limit,     5, 0),                 // if (syscall > LIMIT) goto fail;
-          /* 5  */ BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K,   arch.fork,      4, 0),                 // if (syscall == FORK) goto fail;
-          /* 6  */ BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K,   arch.vfork,     3, 0),                 // if (syscall == VFORK) goto fail;
-          /* 7  */ BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K,   arch.execve,    2, 0),                 // if (syscall == EXECVE) goto fail;
-          /* 8  */ BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K,   arch.execveat,  1, 0),                 // if (syscall == EXECVEAT) goto fail;
-          /* 9  */ BPF_STMT(BPF_RET + BPF_K, SECCOMP_RET_ALLOW),                                // pass: return OK;
-          /* 10 */ BPF_STMT(BPF_RET + BPF_K, SECCOMP_RET_ERRNO | (EACCES & SECCOMP_RET_DATA)),  // fail: return EACCES;
-        };
-        // seccomp takes a long, so we pass it one explicitly to keep the JNA simple
-        SockFProg prog = new SockFProg(insns);
-        prog.write();
-        long pointer = Pointer.nativeValue(prog.getPointer());
-
-        int method = 1;
-        // install filter, if this works, after this there is no going back!
-        // first try it with seccomp(SECCOMP_SET_MODE_FILTER), falling back to prctl()
-        if (linux_syscall(arch.seccomp, SECCOMP_SET_MODE_FILTER, SECCOMP_FILTER_FLAG_TSYNC, new NativeLong(pointer)) != 0) {
-            method = 0;
-            int errno1 = Native.getLastError();
-            if (logger.isDebugEnabled()) {
-                logger.debug("seccomp(SECCOMP_SET_MODE_FILTER): {}, falling back to prctl(PR_SET_SECCOMP)...", JNACLibrary.strerror(errno1));
-            }
-            if (linux_prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, pointer, 0, 0) != 0) {
-                int errno2 = Native.getLastError();
-                throw new UnsupportedOperationException("seccomp(SECCOMP_SET_MODE_FILTER): " + JNACLibrary.strerror(errno1) +
-                                                        ", prctl(PR_SET_SECCOMP): " + JNACLibrary.strerror(errno2));
-            }
-        }
-
-        // now check that the filter was really installed, we should be in filter mode.
-        if (linux_prctl(PR_GET_SECCOMP, 0, 0, 0, 0) != 2) {
-            throw new UnsupportedOperationException("seccomp filter installation did not really succeed. seccomp(PR_GET_SECCOMP): " + JNACLibrary.strerror(Native.getLastError()));
-        }
-
-        logger.debug("Linux seccomp filter installation successful, threads: [{}]", method == 1 ? "all" : "app" );
-        return method;
-    }
-
-    // OS X implementation via sandbox(7)
-
-    /** Access to non-standard OS X libc methods */
-    interface MacLibrary extends Library {
-        /**
-         * maps to sandbox_init(3), since Leopard
-         */
-        int sandbox_init(String profile, long flags, PointerByReference errorbuf);
-
-        /**
-         * releases memory when an error occurs during initialization (e.g. syntax bug)
-         */
-        void sandbox_free_error(Pointer errorbuf);
-    }
-
-    // null if unavailable, or something goes wrong.
-    private static final MacLibrary libc_mac;
-
-    static {
-        MacLibrary lib = null;
-        if (Constants.MAC_OS_X) {
-            try {
-                lib = (MacLibrary) Native.loadLibrary("c", MacLibrary.class);
-            } catch (UnsatisfiedLinkError e) {
-                logger.warn("unable to link C library. native methods (seatbelt) will be disabled.", e);
-            }
-        }
-        libc_mac = lib;
-    }
-
-    /** The only supported flag... */
-    static final int SANDBOX_NAMED = 1;
-    /** Allow everything except process fork and execution */
-    static final String SANDBOX_RULES = "(version 1) (allow default) (deny process-fork) (deny process-exec)";
-
-    /** try to install our custom rule profile into sandbox_init() to block execution */
-    private static void macImpl(Path tmpFile) throws IOException {
-        // first be defensive: we can give nice errors this way, at the very least.
-        boolean supported = Constants.MAC_OS_X;
-        if (supported == false) {
-            throw new IllegalStateException("bug: should not be trying to initialize seatbelt for an unsupported OS");
-        }
-
-        // we couldn't link methods, could be some really ancient OS X (< Leopard) or some bug
-        if (libc_mac == null) {
-            throw new UnsupportedOperationException("seatbelt unavailable: could not link methods. requires Leopard or above.");
-        }
-
-        // write rules to a temporary file, which will be passed to sandbox_init()
-        Path rules = Files.createTempFile(tmpFile, "es", "sb");
-        Files.write(rules, Collections.singleton(SANDBOX_RULES));
-
-        boolean success = false;
-        try {
-            PointerByReference errorRef = new PointerByReference();
-            int ret = libc_mac.sandbox_init(rules.toAbsolutePath().toString(), SANDBOX_NAMED, errorRef);
-            // if sandbox_init() fails, add the message from the OS (e.g. syntax error) and free the buffer
-            if (ret != 0) {
-                Pointer errorBuf = errorRef.getValue();
-                RuntimeException e = new UnsupportedOperationException("sandbox_init(): " + errorBuf.getString(0));
-                libc_mac.sandbox_free_error(errorBuf);
-                throw e;
-            }
-            logger.debug("OS X seatbelt initialization successful");
-            success = true;
-        } finally {
-            if (success) {
-                Files.delete(rules);
-            } else {
-                IOUtils.deleteFilesIgnoringExceptions(rules);
-            }
-        }
-    }
-
-    // Solaris implementation via priv_set(3C)
-
-    /** Access to non-standard Solaris libc methods */
-    interface SolarisLibrary extends Library {
-        /**
-         * see priv_set(3C), a convenience method for setppriv(2).
-         */
-        int priv_set(int op, String which, String... privs);
-    }
-
-    // null if unavailable, or something goes wrong.
-    private static final SolarisLibrary libc_solaris;
-
-    static {
-        SolarisLibrary lib = null;
-        if (Constants.SUN_OS) {
-            try {
-                lib = (SolarisLibrary) Native.loadLibrary("c", SolarisLibrary.class);
-            } catch (UnsatisfiedLinkError e) {
-                logger.warn("unable to link C library. native methods (priv_set) will be disabled.", e);
-            }
-        }
-        libc_solaris = lib;
-    }
-
-    // constants for priv_set(2)
-    static final int PRIV_OFF = 1;
-    static final String PRIV_ALLSETS = null;
-    // see privileges(5) for complete list of these
-    static final String PRIV_PROC_FORK = "proc_fork";
-    static final String PRIV_PROC_EXEC = "proc_exec";
-
-    static void solarisImpl() {
-        // first be defensive: we can give nice errors this way, at the very least.
-        boolean supported = Constants.SUN_OS;
-        if (supported == false) {
-            throw new IllegalStateException("bug: should not be trying to initialize priv_set for an unsupported OS");
-        }
-
-        // we couldn't link methods, could be some really ancient Solaris or some bug
-        if (libc_solaris == null) {
-            throw new UnsupportedOperationException("priv_set unavailable: could not link methods. requires Solaris 10+");
-        }
-
-        // drop a null-terminated list of privileges
-        if (libc_solaris.priv_set(PRIV_OFF, PRIV_ALLSETS, PRIV_PROC_FORK, PRIV_PROC_EXEC, null) != 0) {
-            throw new UnsupportedOperationException("priv_set unavailable: priv_set(): " + JNACLibrary.strerror(Native.getLastError()));
-        }
-
-        logger.debug("Solaris priv_set initialization successful");
-    }
-
-    // BSD implementation via setrlimit(2)
-
-    // TODO: add OpenBSD to Lucene Constants
-    // TODO: JNA doesn't have netbsd support, but this mechanism should work there too.
-    static final boolean OPENBSD = Constants.OS_NAME.startsWith("OpenBSD");
-
-    // not a standard limit, means something different on linux, etc!
-    static final int RLIMIT_NPROC = 7;
-
-    static void bsdImpl() {
-        boolean supported = Constants.FREE_BSD || OPENBSD || Constants.MAC_OS_X;
-        if (supported == false) {
-            throw new IllegalStateException("bug: should not be trying to initialize RLIMIT_NPROC for an unsupported OS");
-        }
-
-        JNACLibrary.Rlimit limit = new JNACLibrary.Rlimit();
-        limit.rlim_cur.setValue(0);
-        limit.rlim_max.setValue(0);
-        if (JNACLibrary.setrlimit(RLIMIT_NPROC, limit) != 0) {
-            throw new UnsupportedOperationException("RLIMIT_NPROC unavailable: " + JNACLibrary.strerror(Native.getLastError()));
-        }
-
-        logger.debug("BSD RLIMIT_NPROC initialization successful");
-    }
-
-    // windows impl via job ActiveProcessLimit
-
-    static void windowsImpl() {
-        if (!Constants.WINDOWS) {
-            throw new IllegalStateException("bug: should not be trying to initialize ActiveProcessLimit for an unsupported OS");
-        }
-
-        JNAKernel32Library lib = JNAKernel32Library.getInstance();
-
-        // create a new Job
-        Pointer job = lib.CreateJobObjectW(null, null);
-        if (job == null) {
-            throw new UnsupportedOperationException("CreateJobObject: " + Native.getLastError());
-        }
-
-        try {
-            // retrieve the current basic limits of the job
-            int clazz = JNAKernel32Library.JOBOBJECT_BASIC_LIMIT_INFORMATION_CLASS;
-            JNAKernel32Library.JOBOBJECT_BASIC_LIMIT_INFORMATION limits = new JNAKernel32Library.JOBOBJECT_BASIC_LIMIT_INFORMATION();
-            limits.write();
-            if (!lib.QueryInformationJobObject(job, clazz, limits.getPointer(), limits.size(), null)) {
-                throw new UnsupportedOperationException("QueryInformationJobObject: " + Native.getLastError());
-            }
-            limits.read();
-            // modify the number of active processes to be 1 (exactly the one process we will add to the job).
-            limits.ActiveProcessLimit = 1;
-            limits.LimitFlags = JNAKernel32Library.JOB_OBJECT_LIMIT_ACTIVE_PROCESS;
-            limits.write();
-            if (!lib.SetInformationJobObject(job, clazz, limits.getPointer(), limits.size())) {
-                throw new UnsupportedOperationException("SetInformationJobObject: " + Native.getLastError());
-            }
-            // assign ourselves to the job
-            if (!lib.AssignProcessToJobObject(job, lib.GetCurrentProcess())) {
-                throw new UnsupportedOperationException("AssignProcessToJobObject: " + Native.getLastError());
-            }
-        } finally {
-            lib.CloseHandle(job);
-        }
-
-        logger.debug("Windows ActiveProcessLimit initialization successful");
-    }
-
-    /**
-     * Attempt to drop the capability to execute for the process.
-     * <p>
-     * This is best effort and OS and architecture dependent. It may throw any Throwable.
-     * @return 0 if we can do this for application threads, 1 for the entire process
-     */
-    static int init(Path tmpFile) throws Exception {
-        if (Constants.LINUX) {
-            return linuxImpl();
-        } else if (Constants.MAC_OS_X) {
-            // try to enable both mechanisms if possible
-            bsdImpl();
-            macImpl(tmpFile);
-            return 1;
-        } else if (Constants.SUN_OS) {
-            solarisImpl();
-            return 1;
-        } else if (Constants.FREE_BSD || OPENBSD) {
-            bsdImpl();
-            return 1;
-        } else if (Constants.WINDOWS) {
-            windowsImpl();
-            return 1;
-        } else {
-            throw new UnsupportedOperationException("syscall filtering not supported for OS: '" + Constants.OS_NAME + "'");
-        }
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Security.java b/core/src/main/java/org/elasticsearch/bootstrap/Security.java
index 909d3dc153c7d..72b4ae15feb02 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/Security.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/Security.java
@@ -19,11 +19,12 @@
 
 package org.elasticsearch.bootstrap;
 
+import org.elasticsearch.Build;
 import org.elasticsearch.SecureSM;
 import org.elasticsearch.Version;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.io.PathUtils;
+import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.http.HttpTransportSettings;
@@ -48,8 +49,11 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 /**
  * Initializes SecurityManager with necessary permissions.
@@ -85,12 +89,6 @@
  * cleanups to the scripting apis). But still it can provide some defense for users
  * that enable dynamic scripting without being fully aware of the consequences.
  * <br>
- * <h1>Disabling Security</h1>
- * SecurityManager can be disabled completely with this setting:
- * <pre>
- * es.security.manager.enabled = false
- * </pre>
- * <br>
  * <h1>Debugging Security</h1>
  * A good place to start when there is a problem is to turn on security debugging:
  * <pre>
@@ -114,13 +112,13 @@ private Security() {}
      * @param environment configuration for generating dynamic permissions
      * @param filterBadDefaults true if we should filter out bad java defaults in the system policy.
      */
-    static void configure(Environment environment, boolean filterBadDefaults) throws Exception {
+    static void configure(Environment environment, boolean filterBadDefaults) throws IOException, NoSuchAlgorithmException {
 
         // enable security policy: union of template and environment-based paths, and possibly plugin permissions
         Policy.setPolicy(new ESPolicy(createPermissions(environment), getPluginPermissions(environment), filterBadDefaults));
 
         // enable security manager
-        System.setSecurityManager(new SecureSM(new String[] { "org.elasticsearch.bootstrap." }));
+        System.setSecurityManager(new SecureSM(new String[] { "org.elasticsearch.bootstrap.", "org.elasticsearch.cli" }));
 
         // do some basic tests
         selfTest();
@@ -133,19 +131,23 @@ static void configure(Environment environment, boolean filterBadDefaults) throws
     @SuppressForbidden(reason = "proper use of URL")
     static Map<String,Policy> getPluginPermissions(Environment environment) throws IOException, NoSuchAlgorithmException {
         Map<String,Policy> map = new HashMap<>();
-        // collect up lists of plugins and modules
-        List<Path> pluginsAndModules = new ArrayList<>();
+        // collect up set of plugins and modules by listing directories.
+        Set<Path> pluginsAndModules = new LinkedHashSet<>(); // order is already lost, but some filesystems have it
         if (Files.exists(environment.pluginsFile())) {
             try (DirectoryStream<Path> stream = Files.newDirectoryStream(environment.pluginsFile())) {
                 for (Path plugin : stream) {
-                    pluginsAndModules.add(plugin);
+                    if (pluginsAndModules.add(plugin) == false) {
+                        throw new IllegalStateException("duplicate plugin: " + plugin);
+                    }
                 }
             }
         }
         if (Files.exists(environment.modulesFile())) {
             try (DirectoryStream<Path> stream = Files.newDirectoryStream(environment.modulesFile())) {
-                for (Path plugin : stream) {
-                    pluginsAndModules.add(plugin);
+                for (Path module : stream) {
+                    if (pluginsAndModules.add(module) == false) {
+                        throw new IllegalStateException("duplicate module: " + module);
+                    }
                 }
             }
         }
@@ -155,15 +157,18 @@ static Map<String,Policy> getPluginPermissions(Environment environment) throws I
             if (Files.exists(policyFile)) {
                 // first get a list of URLs for the plugins' jars:
                 // we resolve symlinks so map is keyed on the normalize codebase name
-                List<URL> codebases = new ArrayList<>();
+                Set<URL> codebases = new LinkedHashSet<>(); // order is already lost, but some filesystems have it
                 try (DirectoryStream<Path> jarStream = Files.newDirectoryStream(plugin, "*.jar")) {
                     for (Path jar : jarStream) {
-                        codebases.add(jar.toRealPath().toUri().toURL());
+                        URL url = jar.toRealPath().toUri().toURL();
+                        if (codebases.add(url) == false) {
+                            throw new IllegalStateException("duplicate module/plugin: " + url);
+                        }
                     }
                 }
 
                 // parse the plugin's policy file into a set of permissions
-                Policy policy = readPolicy(policyFile.toUri().toURL(), codebases.toArray(new URL[codebases.size()]));
+                Policy policy = readPolicy(policyFile.toUri().toURL(), codebases);
 
                 // consult this policy for each of the plugin's jars:
                 for (URL url : codebases) {
@@ -181,25 +186,46 @@ static Map<String,Policy> getPluginPermissions(Environment environment) throws I
     /**
      * Reads and returns the specified {@code policyFile}.
      * <p>
-     * Resources (e.g. jar files and directories) listed in {@code codebases} location
-     * will be provided to the policy file via a system property of the short name:
-     * e.g. <code>${codebase.joda-convert-1.2.jar}</code> would map to full URL.
+     * Jar files listed in {@code codebases} location will be provided to the policy file via
+     * a system property of the short name: e.g. <code>${codebase.joda-convert-1.2.jar}</code>
+     * would map to full URL.
      */
     @SuppressForbidden(reason = "accesses fully qualified URLs to configure security")
-    static Policy readPolicy(URL policyFile, URL codebases[]) {
+    static Policy readPolicy(URL policyFile, Set<URL> codebases) {
         try {
+            List<String> propertiesSet = new ArrayList<>();
             try {
                 // set codebase properties
                 for (URL url : codebases) {
                     String shortName = PathUtils.get(url.toURI()).getFileName().toString();
-                    System.setProperty("codebase." + shortName, url.toString());
+                    if (shortName.endsWith(".jar") == false) {
+                        continue; // tests :(
+                    }
+                    String property = "codebase." + shortName;
+                    if (shortName.startsWith("elasticsearch-rest-client")) {
+                        // The rest client is currently the only example where we have an elasticsearch built artifact
+                        // which needs special permissions in policy files when used. This temporary solution is to
+                        // pass in an extra system property that omits the -version.jar suffix the other properties have.
+                        // That allows the snapshots to reference snapshot builds of the client, and release builds to
+                        // referenced release builds of the client, all with the same grant statements.
+                        final String esVersion = Version.CURRENT + (Build.CURRENT.isSnapshot() ? "-SNAPSHOT" : "");
+                        final int index = property.indexOf("-" + esVersion + ".jar");
+                        assert index >= 0;
+                        String restClientAlias = property.substring(0, index);
+                        propertiesSet.add(restClientAlias);
+                        System.setProperty(restClientAlias, url.toString());
+                    }
+                    propertiesSet.add(property);
+                    String previous = System.setProperty(property, url.toString());
+                    if (previous != null) {
+                        throw new IllegalStateException("codebase property already set: " + shortName + "->" + previous);
+                    }
                 }
                 return Policy.getInstance("JavaPolicy", new URIParameter(policyFile.toURI()));
             } finally {
                 // clear codebase properties
-                for (URL url : codebases) {
-                    String shortName = PathUtils.get(url.toURI()).getFileName().toString();
-                    System.clearProperty("codebase." + shortName);
+                for (String property : propertiesSet) {
+                    System.clearProperty(property);
                 }
             }
         } catch (NoSuchAlgorithmException | URISyntaxException e) {
@@ -254,14 +280,48 @@ static void addFilePermissions(Permissions policy, Environment environment) {
         if (environment.sharedDataFile() != null) {
             addPath(policy, Environment.PATH_SHARED_DATA_SETTING.getKey(), environment.sharedDataFile(), "read,readlink,write,delete");
         }
+        final Set<Path> dataFilesPaths = new HashSet<>();
         for (Path path : environment.dataFiles()) {
             addPath(policy, Environment.PATH_DATA_SETTING.getKey(), path, "read,readlink,write,delete");
+            /*
+             * We have to do this after adding the path because a side effect of that is that the directory is created; the Path#toRealPath
+             * invocation will fail if the directory does not already exist. We use Path#toRealPath to follow symlinks and handle issues
+             * like unicode normalization or case-insensitivity on some filesystems (e.g., the case-insensitive variant of HFS+ on macOS).
+             */
+            try {
+                final Path realPath = path.toRealPath();
+                if (!dataFilesPaths.add(realPath)) {
+                    throw new IllegalStateException("path [" + realPath + "] is duplicated by [" + path + "]");
+                }
+            } catch (final IOException e) {
+                throw new IllegalStateException("unable to access [" + path + "]", e);
+            }
         }
         // TODO: this should be removed in ES 6.0! We will no longer support data paths with the cluster as a folder
         assert Version.CURRENT.major < 6 : "cluster name is no longer used in data path";
         for (Path path : environment.dataWithClusterFiles()) {
             addPathIfExists(policy, Environment.PATH_DATA_SETTING.getKey(), path, "read,readlink,write,delete");
         }
+        /*
+         * If path.data and default.path.data are set, we need read access to the paths in default.path.data to check for the existence of
+         * index directories there that could have arisen from a bug in the handling of simultaneous configuration of path.data and
+         * default.path.data that was introduced in Elasticsearch 5.3.0.
+         *
+         * If path.data is not set then default.path.data would take precedence in setting the data paths for the environment and
+         * permissions would have been granted above.
+         *
+         * If path.data is not set and default.path.data is not set, then we would fallback to the default data directory under
+         * Elasticsearch home and again permissions would have been granted above.
+         *
+         * If path.data is set and default.path.data is not set, there is nothing to do here.
+         */
+        if (Environment.PATH_DATA_SETTING.exists(environment.settings())
+                && Environment.DEFAULT_PATH_DATA_SETTING.exists(environment.settings())) {
+            for (final String path : Environment.DEFAULT_PATH_DATA_SETTING.get(environment.settings())) {
+                // write permissions are not needed here, we are not going to be writing to any paths here
+                addPath(policy, Environment.DEFAULT_PATH_DATA_SETTING.getKey(), getPath(path), "read,readlink");
+            }
+        }
         for (Path path : environment.repoFiles()) {
             addPath(policy, Environment.PATH_REPO_SETTING.getKey(), path, "read,readlink,write,delete");
         }
@@ -271,42 +331,110 @@ static void addFilePermissions(Permissions policy, Environment environment) {
         }
     }
 
-    static void addBindPermissions(Permissions policy, Settings settings) throws IOException {
+    @SuppressForbidden(reason = "read path that is not configured in environment")
+    private static Path getPath(final String path) {
+        return PathUtils.get(path);
+    }
+
+    /**
+     * Add dynamic {@link SocketPermission}s based on HTTP and transport settings.
+     *
+     * @param policy the {@link Permissions} instance to apply the dynamic {@link SocketPermission}s to.
+     * @param settings the {@link Settings} instance to read the HTTP and transport settings from
+     */
+    private static void addBindPermissions(Permissions policy, Settings settings) {
+        addSocketPermissionForHttp(policy, settings);
+        addSocketPermissionForTransportProfiles(policy, settings);
+        addSocketPermissionForTribeNodes(policy, settings);
+    }
+
+    /**
+     * Add dynamic {@link SocketPermission} based on HTTP settings.
+     *
+     * @param policy the {@link Permissions} instance to apply the dynamic {@link SocketPermission}s to.
+     * @param settings the {@link Settings} instance to read the HTTP settingsfrom
+     */
+    private static void addSocketPermissionForHttp(final Permissions policy, final Settings settings) {
         // http is simple
-        String httpRange = HttpTransportSettings.SETTING_HTTP_PORT.get(settings).getPortRangeString();
-        // listen is always called with 'localhost' but use wildcard to be sure, no name service is consulted.
-        // see SocketPermission implies() code
-        policy.add(new SocketPermission("*:" + httpRange, "listen,resolve"));
-        // transport is waaaay overengineered
-        Map<String, Settings> profiles = TransportSettings.TRANSPORT_PROFILES_SETTING.get(settings).getAsGroups();
-        if (!profiles.containsKey(TransportSettings.DEFAULT_PROFILE)) {
-            profiles = new HashMap<>(profiles);
-            profiles.put(TransportSettings.DEFAULT_PROFILE, Settings.EMPTY);
-        }
+        final String httpRange = HttpTransportSettings.SETTING_HTTP_PORT.get(settings).getPortRangeString();
+        addSocketPermissionForPortRange(policy, httpRange);
+    }
+
+    /**
+     * Add dynamic {@link SocketPermission} based on transport settings. This method will first check if there is a port range specified in
+     * the transport profile specified by {@code profileSettings} and will fall back to {@code settings}.
+     *
+     * @param policy          the {@link Permissions} instance to apply the dynamic {@link SocketPermission}s to
+     * @param settings        the {@link Settings} instance to read the transport settings from
+     */
+    private static void addSocketPermissionForTransportProfiles(
+        final Permissions policy,
+        final Settings settings) {
+        // transport is way over-engineered
+        final Map<String, Settings> profiles = new HashMap<>(TransportSettings.TRANSPORT_PROFILES_SETTING.get(settings).getAsGroups());
+        profiles.putIfAbsent(TransportSettings.DEFAULT_PROFILE, Settings.EMPTY);
 
-        // loop through all profiles and add permissions for each one, if its valid.
-        // (otherwise Netty transports are lenient and ignores it)
-        for (Map.Entry<String, Settings> entry : profiles.entrySet()) {
-            Settings profileSettings = entry.getValue();
-            String name = entry.getKey();
-            String transportRange = profileSettings.get("port", TransportSettings.PORT.get(settings));
+        // loop through all profiles and add permissions for each one, if it's valid; otherwise Netty transports are lenient and ignores it
+        for (final Map.Entry<String, Settings> entry : profiles.entrySet()) {
+            final Settings profileSettings = entry.getValue();
+            final String name = entry.getKey();
 
-            // a profile is only valid if its the default profile, or if it has an actual name and specifies a port
-            boolean valid = TransportSettings.DEFAULT_PROFILE.equals(name) || (Strings.hasLength(name) && profileSettings.get("port") != null);
+            // a profile is only valid if it's the default profile, or if it has an actual name and specifies a port
+            // TODO: can this leniency be removed?
+            final boolean valid =
+                TransportSettings.DEFAULT_PROFILE.equals(name) ||
+                    (name != null && name.length() > 0 && profileSettings.get("port") != null);
             if (valid) {
-                // listen is always called with 'localhost' but use wildcard to be sure, no name service is consulted.
-                // see SocketPermission implies() code
-                policy.add(new SocketPermission("*:" + transportRange, "listen,resolve"));
+                final String transportRange = profileSettings.get("port");
+                if (transportRange != null) {
+                    addSocketPermissionForPortRange(policy, transportRange);
+                } else {
+                    addSocketPermissionForTransport(policy, settings);
+                }
             }
         }
     }
 
     /**
-     * Add access to path (and all files underneath it)
-     * @param policy current policy to add permissions to
+     * Add dynamic {@link SocketPermission} based on transport settings.
+     *
+     * @param policy          the {@link Permissions} instance to apply the dynamic {@link SocketPermission}s to
+     * @param settings        the {@link Settings} instance to read the transport settings from
+     */
+    private static void addSocketPermissionForTransport(final Permissions policy, final Settings settings) {
+        final String transportRange = TransportSettings.PORT.get(settings);
+        addSocketPermissionForPortRange(policy, transportRange);
+    }
+
+    private static void addSocketPermissionForTribeNodes(final Permissions policy, final Settings settings) {
+        for (final Settings tribeNodeSettings : settings.getGroups("tribe", true).values()) {
+            // tribe nodes have HTTP disabled by default, so we check if HTTP is enabled before granting
+            if (NetworkModule.HTTP_ENABLED.exists(tribeNodeSettings) && NetworkModule.HTTP_ENABLED.get(tribeNodeSettings)) {
+                addSocketPermissionForHttp(policy, tribeNodeSettings);
+            }
+            addSocketPermissionForTransport(policy, tribeNodeSettings);
+        }
+    }
+
+    /**
+     * Add dynamic {@link SocketPermission} for the specified port range.
+     *
+     * @param policy the {@link Permissions} instance to apply the dynamic {@link SocketPermission} to.
+     * @param portRange the port range
+     */
+    private static void addSocketPermissionForPortRange(final Permissions policy, final String portRange) {
+        // listen is always called with 'localhost' but use wildcard to be sure, no name service is consulted.
+        // see SocketPermission implies() code
+        policy.add(new SocketPermission("*:" + portRange, "listen,resolve"));
+    }
+
+    /**
+     * Add access to path (and all files underneath it); this also creates the directory if it does not exist.
+     *
+     * @param policy            current policy to add permissions to
      * @param configurationName the configuration name associated with the path (for error messages only)
-     * @param path the path itself
-     * @param permissions set of filepermissions to grant to the path
+     * @param path              the path itself
+     * @param permissions       set of file permissions to grant to the path
      */
     static void addPath(Permissions policy, String configurationName, Path path, String permissions) {
         // paths may not exist yet, this also checks accessibility
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Spawner.java b/core/src/main/java/org/elasticsearch/bootstrap/Spawner.java
new file mode 100644
index 0000000000000..f1616ba0eea09
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/bootstrap/Spawner.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.bootstrap;
+
+import org.apache.lucene.util.Constants;
+import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.plugins.Platforms;
+import org.elasticsearch.plugins.PluginInfo;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Locale;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * Spawns native plugin controller processes if present. Will only work prior to a system call
+ * filter being installed.
+ */
+final class Spawner implements Closeable {
+
+    /*
+     * References to the processes that have been spawned, so that we can destroy them.
+     */
+    private final List<Process> processes = new ArrayList<>();
+    private AtomicBoolean spawned = new AtomicBoolean();
+
+    @Override
+    public void close() throws IOException {
+        IOUtils.close(() -> processes.stream().map(s -> (Closeable) s::destroy).iterator());
+    }
+
+    /**
+     * Spawns the native controllers for each plugin
+     *
+     * @param environment the node environment
+     * @throws IOException if an I/O error occurs reading the plugins or spawning a native process
+     */
+    void spawnNativePluginControllers(final Environment environment) throws IOException {
+        if (!spawned.compareAndSet(false, true)) {
+            throw new IllegalStateException("native controllers already spawned");
+        }
+        final Path pluginsFile = environment.pluginsFile();
+        if (!Files.exists(pluginsFile)) {
+            throw new IllegalStateException("plugins directory [" + pluginsFile + "] not found");
+        }
+        /*
+         * For each plugin, attempt to spawn the controller daemon. Silently ignore any plugin that
+         * don't include a controller for the correct platform.
+         */
+        try (DirectoryStream<Path> stream = Files.newDirectoryStream(pluginsFile)) {
+            for (final Path plugin : stream) {
+                final PluginInfo info = PluginInfo.readFromProperties(plugin);
+                final Path spawnPath = Platforms.nativeControllerPath(plugin);
+                if (!Files.isRegularFile(spawnPath)) {
+                    continue;
+                }
+                if (!info.hasNativeController()) {
+                    final String message = String.format(
+                            Locale.ROOT,
+                            "plugin [%s] does not have permission to fork native controller",
+                            plugin.getFileName());
+                    throw new IllegalArgumentException(message);
+                }
+                final Process process =
+                        spawnNativePluginController(spawnPath, environment.tmpFile());
+                processes.add(process);
+            }
+        }
+    }
+
+    /**
+     * Attempt to spawn the controller daemon for a given plugin. The spawned process will remain
+     * connected to this JVM via its stdin, stdout, and stderr streams, but the references to these
+     * streams are not available to code outside this package.
+     */
+    private Process spawnNativePluginController(
+            final Path spawnPath,
+            final Path tmpPath) throws IOException {
+        final String command;
+        if (Constants.WINDOWS) {
+            /*
+             * We have to get the short path name or starting the process could fail due to max path limitations. The underlying issue here
+             * is that starting the process on Windows ultimately involves the use of CreateProcessW. CreateProcessW has a limitation that
+             * if its first argument (the application name) is null, then its second argument (the command line for the process to start) is
+             * restricted in length to 260 characters (cf. https://msdn.microsoft.com/en-us/library/windows/desktop/ms682425.aspx). Since
+             * this is exactly how the JDK starts the process on Windows (cf.
+             * http://hg.openjdk.java.net/jdk8/jdk8/jdk/file/687fd7c7986d/src/windows/native/java/lang/ProcessImpl_md.c#l319), this
+             * limitation is in force. As such, we use the short name to avoid any such problems.
+             */
+            command = Natives.getShortPathName(spawnPath.toString());
+        } else {
+            command = spawnPath.toString();
+        }
+        final ProcessBuilder pb = new ProcessBuilder(command);
+
+        // the only environment variable passes on the path to the temporary directory
+        pb.environment().clear();
+        pb.environment().put("TMPDIR", tmpPath.toString());
+
+        // the output stream of the process object corresponds to the daemon's stdin
+        return pb.start();
+    }
+
+    /**
+     * The collection of processes representing spawned native controllers.
+     *
+     * @return the processes
+     */
+    List<Process> getProcesses() {
+        return Collections.unmodifiableList(processes);
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/StartupException.java b/core/src/main/java/org/elasticsearch/bootstrap/StartupException.java
index a78f82ef3e732..59629eb5b3b19 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/StartupException.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/StartupException.java
@@ -104,7 +104,7 @@ private void printStackTrace(Consumer<String> consumer) {
                     continue;
                 }
 
-                consumer.accept("\tat " + line.toString());
+                consumer.accept("\tat " + line);
                 linesWritten++;
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/SystemCallFilter.java b/core/src/main/java/org/elasticsearch/bootstrap/SystemCallFilter.java
new file mode 100644
index 0000000000000..2be2d768ab346
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/bootstrap/SystemCallFilter.java
@@ -0,0 +1,658 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.bootstrap;
+
+import com.sun.jna.Library;
+import com.sun.jna.Memory;
+import com.sun.jna.Native;
+import com.sun.jna.NativeLong;
+import com.sun.jna.Pointer;
+import com.sun.jna.Structure;
+import com.sun.jna.ptr.PointerByReference;
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.util.Constants;
+import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.common.logging.Loggers;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Installs a system call filter to block process execution.
+ * <p>
+ * This is supported on Linux, Solaris, FreeBSD, OpenBSD, Mac OS X, and Windows.
+ * <p>
+ * On Linux it currently supports amd64 and i386 architectures, requires Linux kernel 3.5 or above, and requires
+ * {@code CONFIG_SECCOMP} and {@code CONFIG_SECCOMP_FILTER} compiled into the kernel.
+ * <p>
+ * On Linux BPF Filters are installed using either {@code seccomp(2)} (3.17+) or {@code prctl(2)} (3.5+). {@code seccomp(2)}
+ * is preferred, as it allows filters to be applied to any existing threads in the process, and one motivation
+ * here is to protect against bugs in the JVM. Otherwise, code will fall back to the {@code prctl(2)} method
+ * which will at least protect elasticsearch application threads.
+ * <p>
+ * Linux BPF filters will return {@code EACCES} (Access Denied) for the following system calls:
+ * <ul>
+ *   <li>{@code execve}</li>
+ *   <li>{@code fork}</li>
+ *   <li>{@code vfork}</li>
+ *   <li>{@code execveat}</li>
+ * </ul>
+ * <p>
+ * On Solaris 10 or higher, the following privileges are dropped with {@code priv_set(3C)}:
+ * <ul>
+ *   <li>{@code PRIV_PROC_FORK}</li>
+ *   <li>{@code PRIV_PROC_EXEC}</li>
+ * </ul>
+ * <p>
+ * On BSD systems, process creation is restricted with {@code setrlimit(RLIMIT_NPROC)}.
+ * <p>
+ * On Mac OS X Leopard or above, a custom {@code sandbox(7)} ("Seatbelt") profile is installed that
+ * denies the following rules:
+ * <ul>
+ *   <li>{@code process-fork}</li>
+ *   <li>{@code process-exec}</li>
+ * </ul>
+ * <p>
+ * On Windows, process creation is restricted with {@code SetInformationJobObject/ActiveProcessLimit}.
+ * <p>
+ * This is not intended as a sandbox. It is another level of security, mostly intended to annoy
+ * security researchers and make their lives more difficult in achieving "remote execution" exploits.
+ * @see <a href="http://www.kernel.org/doc/Documentation/prctl/seccomp_filter.txt">
+ *      http://www.kernel.org/doc/Documentation/prctl/seccomp_filter.txt</a>
+ * @see <a href="https://reverse.put.as/wp-content/uploads/2011/06/The-Apple-Sandbox-BHDC2011-Paper.pdf">
+ *      https://reverse.put.as/wp-content/uploads/2011/06/The-Apple-Sandbox-BHDC2011-Paper.pdf</a>
+ * @see <a href="https://docs.oracle.com/cd/E23824_01/html/821-1456/prbac-2.html">
+ *      https://docs.oracle.com/cd/E23824_01/html/821-1456/prbac-2.html</a>
+ */
+// not an example of how to write code!!!
+final class SystemCallFilter {
+    private static final Logger logger = Loggers.getLogger(SystemCallFilter.class);
+
+    // Linux implementation, based on seccomp(2) or prctl(2) with bpf filtering
+
+    /** Access to non-standard Linux libc methods */
+    interface LinuxLibrary extends Library {
+        /**
+         * maps to prctl(2)
+         */
+        int prctl(int option, NativeLong arg2, NativeLong arg3, NativeLong arg4, NativeLong arg5);
+        /**
+         * used to call seccomp(2), its too new...
+         * this is the only way, DON'T use it on some other architecture unless you know wtf you are doing
+         */
+        NativeLong syscall(NativeLong number, Object... args);
+    }
+
+    // null if unavailable or something goes wrong.
+    private static final LinuxLibrary linux_libc;
+
+    static {
+        LinuxLibrary lib = null;
+        if (Constants.LINUX) {
+            try {
+                lib = (LinuxLibrary) Native.loadLibrary("c", LinuxLibrary.class);
+            } catch (UnsatisfiedLinkError e) {
+                logger.warn("unable to link C library. native methods (seccomp) will be disabled.", e);
+            }
+        }
+        linux_libc = lib;
+    }
+
+    /** the preferred method is seccomp(2), since we can apply to all threads of the process */
+    static final int SECCOMP_SET_MODE_FILTER   =   1;   // since Linux 3.17
+    static final int SECCOMP_FILTER_FLAG_TSYNC =   1;   // since Linux 3.17
+
+    /** otherwise, we can use prctl(2), which will at least protect ES application threads */
+    static final int PR_GET_NO_NEW_PRIVS       =  39;   // since Linux 3.5
+    static final int PR_SET_NO_NEW_PRIVS       =  38;   // since Linux 3.5
+    static final int PR_GET_SECCOMP            =  21;   // since Linux 2.6.23
+    static final int PR_SET_SECCOMP            =  22;   // since Linux 2.6.23
+    static final long SECCOMP_MODE_FILTER      =   2;   // since Linux Linux 3.5
+
+    /** corresponds to struct sock_filter */
+    static final class SockFilter {
+        short code; // insn
+        byte jt;    // number of insn to jump (skip) if true
+        byte jf;    // number of insn to jump (skip) if false
+        int k;      // additional data
+
+        SockFilter(short code, byte jt, byte jf, int k) {
+            this.code = code;
+            this.jt = jt;
+            this.jf = jf;
+            this.k = k;
+        }
+    }
+
+    /** corresponds to struct sock_fprog */
+    public static final class SockFProg extends Structure implements Structure.ByReference {
+        public short   len;           // number of filters
+        public Pointer filter;        // filters
+
+        SockFProg(SockFilter filters[]) {
+            len = (short) filters.length;
+            // serialize struct sock_filter * explicitly, its less confusing than the JNA magic we would need
+            Memory filter = new Memory(len * 8);
+            ByteBuffer bbuf = filter.getByteBuffer(0, len * 8);
+            bbuf.order(ByteOrder.nativeOrder()); // little endian
+            for (SockFilter f : filters) {
+                bbuf.putShort(f.code);
+                bbuf.put(f.jt);
+                bbuf.put(f.jf);
+                bbuf.putInt(f.k);
+            }
+            this.filter = filter;
+        }
+
+        @Override
+        protected List<String> getFieldOrder() {
+            return Arrays.asList("len", "filter");
+        }
+    }
+
+    // BPF "macros" and constants
+    static final int BPF_LD  = 0x00;
+    static final int BPF_W   = 0x00;
+    static final int BPF_ABS = 0x20;
+    static final int BPF_JMP = 0x05;
+    static final int BPF_JEQ = 0x10;
+    static final int BPF_JGE = 0x30;
+    static final int BPF_JGT = 0x20;
+    static final int BPF_RET = 0x06;
+    static final int BPF_K   = 0x00;
+
+    static SockFilter BPF_STMT(int code, int k) {
+        return new SockFilter((short) code, (byte) 0, (byte) 0, k);
+    }
+
+    static SockFilter BPF_JUMP(int code, int k, int jt, int jf) {
+        return new SockFilter((short) code, (byte) jt, (byte) jf, k);
+    }
+
+    static final int SECCOMP_RET_ERRNO = 0x00050000;
+    static final int SECCOMP_RET_DATA  = 0x0000FFFF;
+    static final int SECCOMP_RET_ALLOW = 0x7FFF0000;
+
+    // some errno constants for error checking/handling
+    static final int EPERM  = 0x01;
+    static final int EACCES = 0x0D;
+    static final int EFAULT = 0x0E;
+    static final int EINVAL = 0x16;
+    static final int ENOSYS = 0x26;
+
+    // offsets that our BPF checks
+    // check with offsetof() when adding a new arch, move to Arch if different.
+    static final int SECCOMP_DATA_NR_OFFSET   = 0x00;
+    static final int SECCOMP_DATA_ARCH_OFFSET = 0x04;
+
+    static class Arch {
+        /** AUDIT_ARCH_XXX constant from linux/audit.h */
+        final int audit;
+        /** syscall limit (necessary for blacklisting on amd64, to ban 32-bit syscalls) */
+        final int limit;
+        /** __NR_fork */
+        final int fork;
+        /** __NR_vfork */
+        final int vfork;
+        /** __NR_execve */
+        final int execve;
+        /**  __NR_execveat */
+        final int execveat;
+        /** __NR_seccomp */
+        final int seccomp;
+
+        Arch(int audit, int limit, int fork, int vfork, int execve, int execveat, int seccomp) {
+            this.audit = audit;
+            this.limit = limit;
+            this.fork = fork;
+            this.vfork = vfork;
+            this.execve = execve;
+            this.execveat = execveat;
+            this.seccomp = seccomp;
+        }
+    }
+
+    /** supported architectures map keyed by os.arch */
+    private static final Map<String,Arch> ARCHITECTURES;
+    static {
+        Map<String,Arch> m = new HashMap<>();
+        m.put("amd64", new Arch(0xC000003E, 0x3FFFFFFF, 57, 58, 59, 322, 317));
+        m.put("i386",  new Arch(0x40000003, 0xFFFFFFFF, 2, 190, 11, 358, 354));
+        m.put("aarch64",  new Arch(0xC00000B7, 0xFFFFFFFF, 1079, 1071, 221, 281, 277));
+        ARCHITECTURES = Collections.unmodifiableMap(m);
+    }
+
+    /** invokes prctl() from linux libc library */
+    private static int linux_prctl(int option, long arg2, long arg3, long arg4, long arg5) {
+        return linux_libc.prctl(option, new NativeLong(arg2), new NativeLong(arg3), new NativeLong(arg4), new NativeLong(arg5));
+    }
+
+    /** invokes syscall() from linux libc library */
+    private static long linux_syscall(long number, Object... args) {
+        return linux_libc.syscall(new NativeLong(number), args).longValue();
+    }
+
+    /** try to install our BPF filters via seccomp() or prctl() to block execution */
+    private static int linuxImpl() {
+        // first be defensive: we can give nice errors this way, at the very least.
+        // also, some of these security features get backported to old versions, checking kernel version here is a big no-no!
+        final Arch arch = ARCHITECTURES.get(Constants.OS_ARCH);
+        boolean supported = Constants.LINUX && arch != null;
+        if (supported == false) {
+            throw new UnsupportedOperationException("seccomp unavailable: '" + Constants.OS_ARCH + "' architecture unsupported");
+        }
+
+        // we couldn't link methods, could be some really ancient kernel (e.g. < 2.1.57) or some bug
+        if (linux_libc == null) {
+            throw new UnsupportedOperationException("seccomp unavailable: could not link methods. requires kernel 3.5+ " +
+                "with CONFIG_SECCOMP and CONFIG_SECCOMP_FILTER compiled in");
+        }
+
+        // pure paranoia:
+
+        // check that unimplemented syscalls actually return ENOSYS
+        // you never know (e.g. https://code.google.com/p/chromium/issues/detail?id=439795)
+        if (linux_syscall(999) >= 0) {
+            throw new UnsupportedOperationException("seccomp unavailable: your kernel is buggy and you should upgrade");
+        }
+
+        switch (Native.getLastError()) {
+            case ENOSYS:
+                break; // ok
+            case EPERM:
+                // NOT ok, but likely a docker container
+                if (logger.isDebugEnabled()) {
+                    logger.debug("syscall(BOGUS) bogusly gets EPERM instead of ENOSYS");
+                }
+                break;
+            default:
+                throw new UnsupportedOperationException("seccomp unavailable: your kernel is buggy and you should upgrade");
+        }
+
+        // try to check system calls really are who they claim
+        // you never know (e.g. https://chromium.googlesource.com/chromium/src.git/+/master/sandbox/linux/seccomp-bpf/sandbox_bpf.cc#57)
+        final int bogusArg = 0xf7a46a5c;
+
+        // test seccomp(BOGUS)
+        long ret = linux_syscall(arch.seccomp, bogusArg);
+        if (ret != -1) {
+            throw new UnsupportedOperationException("seccomp unavailable: seccomp(BOGUS_OPERATION) returned " + ret);
+        } else {
+            int errno = Native.getLastError();
+            switch (errno) {
+                case ENOSYS: break; // ok
+                case EINVAL: break; // ok
+                default: throw new UnsupportedOperationException("seccomp(BOGUS_OPERATION): " + JNACLibrary.strerror(errno));
+            }
+        }
+
+        // test seccomp(VALID, BOGUS)
+        ret = linux_syscall(arch.seccomp, SECCOMP_SET_MODE_FILTER, bogusArg);
+        if (ret != -1) {
+            throw new UnsupportedOperationException("seccomp unavailable: seccomp(SECCOMP_SET_MODE_FILTER, BOGUS_FLAG) returned " + ret);
+        } else {
+            int errno = Native.getLastError();
+            switch (errno) {
+                case ENOSYS: break; // ok
+                case EINVAL: break; // ok
+                default: throw new UnsupportedOperationException("seccomp(SECCOMP_SET_MODE_FILTER, BOGUS_FLAG): "
+                                                                 + JNACLibrary.strerror(errno));
+            }
+        }
+
+        // test prctl(BOGUS)
+        ret = linux_prctl(bogusArg, 0, 0, 0, 0);
+        if (ret != -1) {
+            throw new UnsupportedOperationException("seccomp unavailable: prctl(BOGUS_OPTION) returned " + ret);
+        } else {
+            int errno = Native.getLastError();
+            switch (errno) {
+                case ENOSYS: break; // ok
+                case EINVAL: break; // ok
+                default: throw new UnsupportedOperationException("prctl(BOGUS_OPTION): " + JNACLibrary.strerror(errno));
+            }
+        }
+
+        // now just normal defensive checks
+
+        // check for GET_NO_NEW_PRIVS
+        switch (linux_prctl(PR_GET_NO_NEW_PRIVS, 0, 0, 0, 0)) {
+            case 0: break; // not yet set
+            case 1: break; // already set by caller
+            default:
+                int errno = Native.getLastError();
+                if (errno == EINVAL) {
+                    // friendly error, this will be the typical case for an old kernel
+                    throw new UnsupportedOperationException("seccomp unavailable: requires kernel 3.5+ with" +
+                                                            " CONFIG_SECCOMP and CONFIG_SECCOMP_FILTER compiled in");
+                } else {
+                    throw new UnsupportedOperationException("prctl(PR_GET_NO_NEW_PRIVS): " + JNACLibrary.strerror(errno));
+                }
+        }
+        // check for SECCOMP
+        switch (linux_prctl(PR_GET_SECCOMP, 0, 0, 0, 0)) {
+            case 0: break; // not yet set
+            case 2: break; // already in filter mode by caller
+            default:
+                int errno = Native.getLastError();
+                if (errno == EINVAL) {
+                    throw new UnsupportedOperationException("seccomp unavailable: CONFIG_SECCOMP not compiled into kernel," +
+                                                            " CONFIG_SECCOMP and CONFIG_SECCOMP_FILTER are needed");
+                } else {
+                    throw new UnsupportedOperationException("prctl(PR_GET_SECCOMP): " + JNACLibrary.strerror(errno));
+                }
+        }
+        // check for SECCOMP_MODE_FILTER
+        if (linux_prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, 0, 0, 0) != 0) {
+            int errno = Native.getLastError();
+            switch (errno) {
+                case EFAULT: break; // available
+                case EINVAL: throw new UnsupportedOperationException("seccomp unavailable: CONFIG_SECCOMP_FILTER not" +
+                    " compiled into kernel, CONFIG_SECCOMP and CONFIG_SECCOMP_FILTER are needed");
+                default: throw new UnsupportedOperationException("prctl(PR_SET_SECCOMP): " + JNACLibrary.strerror(errno));
+            }
+        }
+
+        // ok, now set PR_SET_NO_NEW_PRIVS, needed to be able to set a seccomp filter as ordinary user
+        if (linux_prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) != 0) {
+            throw new UnsupportedOperationException("prctl(PR_SET_NO_NEW_PRIVS): " + JNACLibrary.strerror(Native.getLastError()));
+        }
+
+        // check it worked
+        if (linux_prctl(PR_GET_NO_NEW_PRIVS, 0, 0, 0, 0) != 1) {
+            throw new UnsupportedOperationException("seccomp filter did not really succeed: prctl(PR_GET_NO_NEW_PRIVS): " +
+                                                    JNACLibrary.strerror(Native.getLastError()));
+        }
+
+        // BPF installed to check arch, limit, then syscall.
+        // See https://www.kernel.org/doc/Documentation/prctl/seccomp_filter.txt for details.
+        SockFilter insns[] = {
+          /* 1  */ BPF_STMT(BPF_LD  + BPF_W   + BPF_ABS, SECCOMP_DATA_ARCH_OFFSET),             //
+          /* 2  */ BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K,   arch.audit,     0, 7),                 // if (arch != audit) goto fail;
+          /* 3  */ BPF_STMT(BPF_LD  + BPF_W   + BPF_ABS, SECCOMP_DATA_NR_OFFSET),               //
+          /* 4  */ BPF_JUMP(BPF_JMP + BPF_JGT + BPF_K,   arch.limit,     5, 0),                 // if (syscall > LIMIT) goto fail;
+          /* 5  */ BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K,   arch.fork,      4, 0),                 // if (syscall == FORK) goto fail;
+          /* 6  */ BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K,   arch.vfork,     3, 0),                 // if (syscall == VFORK) goto fail;
+          /* 7  */ BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K,   arch.execve,    2, 0),                 // if (syscall == EXECVE) goto fail;
+          /* 8  */ BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K,   arch.execveat,  1, 0),                 // if (syscall == EXECVEAT) goto fail;
+          /* 9  */ BPF_STMT(BPF_RET + BPF_K, SECCOMP_RET_ALLOW),                                // pass: return OK;
+          /* 10 */ BPF_STMT(BPF_RET + BPF_K, SECCOMP_RET_ERRNO | (EACCES & SECCOMP_RET_DATA)),  // fail: return EACCES;
+        };
+        // seccomp takes a long, so we pass it one explicitly to keep the JNA simple
+        SockFProg prog = new SockFProg(insns);
+        prog.write();
+        long pointer = Pointer.nativeValue(prog.getPointer());
+
+        int method = 1;
+        // install filter, if this works, after this there is no going back!
+        // first try it with seccomp(SECCOMP_SET_MODE_FILTER), falling back to prctl()
+        if (linux_syscall(arch.seccomp, SECCOMP_SET_MODE_FILTER, SECCOMP_FILTER_FLAG_TSYNC, new NativeLong(pointer)) != 0) {
+            method = 0;
+            int errno1 = Native.getLastError();
+            if (logger.isDebugEnabled()) {
+                logger.debug("seccomp(SECCOMP_SET_MODE_FILTER): {}, falling back to prctl(PR_SET_SECCOMP)...",
+                             JNACLibrary.strerror(errno1));
+            }
+            if (linux_prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, pointer, 0, 0) != 0) {
+                int errno2 = Native.getLastError();
+                throw new UnsupportedOperationException("seccomp(SECCOMP_SET_MODE_FILTER): " + JNACLibrary.strerror(errno1) +
+                                                        ", prctl(PR_SET_SECCOMP): " + JNACLibrary.strerror(errno2));
+            }
+        }
+
+        // now check that the filter was really installed, we should be in filter mode.
+        if (linux_prctl(PR_GET_SECCOMP, 0, 0, 0, 0) != 2) {
+            throw new UnsupportedOperationException("seccomp filter installation did not really succeed. seccomp(PR_GET_SECCOMP): "
+                                                    + JNACLibrary.strerror(Native.getLastError()));
+        }
+
+        logger.debug("Linux seccomp filter installation successful, threads: [{}]", method == 1 ? "all" : "app" );
+        return method;
+    }
+
+    // OS X implementation via sandbox(7)
+
+    /** Access to non-standard OS X libc methods */
+    interface MacLibrary extends Library {
+        /**
+         * maps to sandbox_init(3), since Leopard
+         */
+        int sandbox_init(String profile, long flags, PointerByReference errorbuf);
+
+        /**
+         * releases memory when an error occurs during initialization (e.g. syntax bug)
+         */
+        void sandbox_free_error(Pointer errorbuf);
+    }
+
+    // null if unavailable, or something goes wrong.
+    private static final MacLibrary libc_mac;
+
+    static {
+        MacLibrary lib = null;
+        if (Constants.MAC_OS_X) {
+            try {
+                lib = (MacLibrary) Native.loadLibrary("c", MacLibrary.class);
+            } catch (UnsatisfiedLinkError e) {
+                logger.warn("unable to link C library. native methods (seatbelt) will be disabled.", e);
+            }
+        }
+        libc_mac = lib;
+    }
+
+    /** The only supported flag... */
+    static final int SANDBOX_NAMED = 1;
+    /** Allow everything except process fork and execution */
+    static final String SANDBOX_RULES = "(version 1) (allow default) (deny process-fork) (deny process-exec)";
+
+    /** try to install our custom rule profile into sandbox_init() to block execution */
+    private static void macImpl(Path tmpFile) throws IOException {
+        // first be defensive: we can give nice errors this way, at the very least.
+        boolean supported = Constants.MAC_OS_X;
+        if (supported == false) {
+            throw new IllegalStateException("bug: should not be trying to initialize seatbelt for an unsupported OS");
+        }
+
+        // we couldn't link methods, could be some really ancient OS X (< Leopard) or some bug
+        if (libc_mac == null) {
+            throw new UnsupportedOperationException("seatbelt unavailable: could not link methods. requires Leopard or above.");
+        }
+
+        // write rules to a temporary file, which will be passed to sandbox_init()
+        Path rules = Files.createTempFile(tmpFile, "es", "sb");
+        Files.write(rules, Collections.singleton(SANDBOX_RULES));
+
+        boolean success = false;
+        try {
+            PointerByReference errorRef = new PointerByReference();
+            int ret = libc_mac.sandbox_init(rules.toAbsolutePath().toString(), SANDBOX_NAMED, errorRef);
+            // if sandbox_init() fails, add the message from the OS (e.g. syntax error) and free the buffer
+            if (ret != 0) {
+                Pointer errorBuf = errorRef.getValue();
+                RuntimeException e = new UnsupportedOperationException("sandbox_init(): " + errorBuf.getString(0));
+                libc_mac.sandbox_free_error(errorBuf);
+                throw e;
+            }
+            logger.debug("OS X seatbelt initialization successful");
+            success = true;
+        } finally {
+            if (success) {
+                Files.delete(rules);
+            } else {
+                IOUtils.deleteFilesIgnoringExceptions(rules);
+            }
+        }
+    }
+
+    // Solaris implementation via priv_set(3C)
+
+    /** Access to non-standard Solaris libc methods */
+    interface SolarisLibrary extends Library {
+        /**
+         * see priv_set(3C), a convenience method for setppriv(2).
+         */
+        int priv_set(int op, String which, String... privs);
+    }
+
+    // null if unavailable, or something goes wrong.
+    private static final SolarisLibrary libc_solaris;
+
+    static {
+        SolarisLibrary lib = null;
+        if (Constants.SUN_OS) {
+            try {
+                lib = (SolarisLibrary) Native.loadLibrary("c", SolarisLibrary.class);
+            } catch (UnsatisfiedLinkError e) {
+                logger.warn("unable to link C library. native methods (priv_set) will be disabled.", e);
+            }
+        }
+        libc_solaris = lib;
+    }
+
+    // constants for priv_set(2)
+    static final int PRIV_OFF = 1;
+    static final String PRIV_ALLSETS = null;
+    // see privileges(5) for complete list of these
+    static final String PRIV_PROC_FORK = "proc_fork";
+    static final String PRIV_PROC_EXEC = "proc_exec";
+
+    static void solarisImpl() {
+        // first be defensive: we can give nice errors this way, at the very least.
+        boolean supported = Constants.SUN_OS;
+        if (supported == false) {
+            throw new IllegalStateException("bug: should not be trying to initialize priv_set for an unsupported OS");
+        }
+
+        // we couldn't link methods, could be some really ancient Solaris or some bug
+        if (libc_solaris == null) {
+            throw new UnsupportedOperationException("priv_set unavailable: could not link methods. requires Solaris 10+");
+        }
+
+        // drop a null-terminated list of privileges
+        if (libc_solaris.priv_set(PRIV_OFF, PRIV_ALLSETS, PRIV_PROC_FORK, PRIV_PROC_EXEC, null) != 0) {
+            throw new UnsupportedOperationException("priv_set unavailable: priv_set(): " + JNACLibrary.strerror(Native.getLastError()));
+        }
+
+        logger.debug("Solaris priv_set initialization successful");
+    }
+
+    // BSD implementation via setrlimit(2)
+
+    // TODO: add OpenBSD to Lucene Constants
+    // TODO: JNA doesn't have netbsd support, but this mechanism should work there too.
+    static final boolean OPENBSD = Constants.OS_NAME.startsWith("OpenBSD");
+
+    // not a standard limit, means something different on linux, etc!
+    static final int RLIMIT_NPROC = 7;
+
+    static void bsdImpl() {
+        boolean supported = Constants.FREE_BSD || OPENBSD || Constants.MAC_OS_X;
+        if (supported == false) {
+            throw new IllegalStateException("bug: should not be trying to initialize RLIMIT_NPROC for an unsupported OS");
+        }
+
+        JNACLibrary.Rlimit limit = new JNACLibrary.Rlimit();
+        limit.rlim_cur.setValue(0);
+        limit.rlim_max.setValue(0);
+        if (JNACLibrary.setrlimit(RLIMIT_NPROC, limit) != 0) {
+            throw new UnsupportedOperationException("RLIMIT_NPROC unavailable: " + JNACLibrary.strerror(Native.getLastError()));
+        }
+
+        logger.debug("BSD RLIMIT_NPROC initialization successful");
+    }
+
+    // windows impl via job ActiveProcessLimit
+
+    static void windowsImpl() {
+        if (!Constants.WINDOWS) {
+            throw new IllegalStateException("bug: should not be trying to initialize ActiveProcessLimit for an unsupported OS");
+        }
+
+        JNAKernel32Library lib = JNAKernel32Library.getInstance();
+
+        // create a new Job
+        Pointer job = lib.CreateJobObjectW(null, null);
+        if (job == null) {
+            throw new UnsupportedOperationException("CreateJobObject: " + Native.getLastError());
+        }
+
+        try {
+            // retrieve the current basic limits of the job
+            int clazz = JNAKernel32Library.JOBOBJECT_BASIC_LIMIT_INFORMATION_CLASS;
+            JNAKernel32Library.JOBOBJECT_BASIC_LIMIT_INFORMATION limits = new JNAKernel32Library.JOBOBJECT_BASIC_LIMIT_INFORMATION();
+            limits.write();
+            if (!lib.QueryInformationJobObject(job, clazz, limits.getPointer(), limits.size(), null)) {
+                throw new UnsupportedOperationException("QueryInformationJobObject: " + Native.getLastError());
+            }
+            limits.read();
+            // modify the number of active processes to be 1 (exactly the one process we will add to the job).
+            limits.ActiveProcessLimit = 1;
+            limits.LimitFlags = JNAKernel32Library.JOB_OBJECT_LIMIT_ACTIVE_PROCESS;
+            limits.write();
+            if (!lib.SetInformationJobObject(job, clazz, limits.getPointer(), limits.size())) {
+                throw new UnsupportedOperationException("SetInformationJobObject: " + Native.getLastError());
+            }
+            // assign ourselves to the job
+            if (!lib.AssignProcessToJobObject(job, lib.GetCurrentProcess())) {
+                throw new UnsupportedOperationException("AssignProcessToJobObject: " + Native.getLastError());
+            }
+        } finally {
+            lib.CloseHandle(job);
+        }
+
+        logger.debug("Windows ActiveProcessLimit initialization successful");
+    }
+
+    /**
+     * Attempt to drop the capability to execute for the process.
+     * <p>
+     * This is best effort and OS and architecture dependent. It may throw any Throwable.
+     * @return 0 if we can do this for application threads, 1 for the entire process
+     */
+    static int init(Path tmpFile) throws Exception {
+        if (Constants.LINUX) {
+            return linuxImpl();
+        } else if (Constants.MAC_OS_X) {
+            // try to enable both mechanisms if possible
+            bsdImpl();
+            macImpl(tmpFile);
+            return 1;
+        } else if (Constants.SUN_OS) {
+            solarisImpl();
+            return 1;
+        } else if (Constants.FREE_BSD || OPENBSD) {
+            bsdImpl();
+            return 1;
+        } else if (Constants.WINDOWS) {
+            windowsImpl();
+            return 1;
+        } else {
+            throw new UnsupportedOperationException("syscall filtering not supported for OS: '" + Constants.OS_NAME + "'");
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/cli/Command.java b/core/src/main/java/org/elasticsearch/cli/Command.java
index 2e896759ebbfa..a60dece26113a 100644
--- a/core/src/main/java/org/elasticsearch/cli/Command.java
+++ b/core/src/main/java/org/elasticsearch/cli/Command.java
@@ -23,15 +23,22 @@
 import joptsimple.OptionParser;
 import joptsimple.OptionSet;
 import joptsimple.OptionSpec;
+import org.apache.logging.log4j.Level;
+import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.common.logging.LogConfigurator;
+import org.elasticsearch.common.settings.Settings;
 
+import java.io.Closeable;
 import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
 import java.util.Arrays;
 
 /**
  * An action to execute within a cli.
  */
-public abstract class Command {
+public abstract class Command implements Closeable {
 
     /** A description of the command, used in the help output. */
     protected final String description;
@@ -41,15 +48,44 @@ public abstract class Command {
 
     private final OptionSpec<Void> helpOption = parser.acceptsAll(Arrays.asList("h", "help"), "show help").forHelp();
     private final OptionSpec<Void> silentOption = parser.acceptsAll(Arrays.asList("s", "silent"), "show minimal output");
-    private final OptionSpec<Void> verboseOption = parser.acceptsAll(Arrays.asList("v", "verbose"), "show verbose output")
-            .availableUnless(silentOption);
+    private final OptionSpec<Void> verboseOption =
+        parser.acceptsAll(Arrays.asList("v", "verbose"), "show verbose output").availableUnless(silentOption);
 
     public Command(String description) {
         this.description = description;
     }
 
+    final SetOnce<Thread> shutdownHookThread = new SetOnce<>();
+
     /** Parses options for this command from args and executes it. */
     public final int main(String[] args, Terminal terminal) throws Exception {
+        if (addShutdownHook()) {
+            shutdownHookThread.set(new Thread(() -> {
+                try {
+                    this.close();
+                } catch (final IOException e) {
+                    try (
+                        StringWriter sw = new StringWriter();
+                        PrintWriter pw = new PrintWriter(sw)) {
+                        e.printStackTrace(pw);
+                        terminal.println(sw.toString());
+                    } catch (final IOException impossible) {
+                        // StringWriter#close declares a checked IOException from the Closeable interface but the Javadocs for StringWriter
+                        // say that an exception here is impossible
+                        throw new AssertionError(impossible);
+                    }
+                }
+            }));
+            Runtime.getRuntime().addShutdownHook(shutdownHookThread.get());
+        }
+
+        if (shouldConfigureLoggingWithoutConfig()) {
+            // initialize default for es.logger.level because we will not read the log4j2.properties
+            final String loggerLevel = System.getProperty("es.logger.level", Level.INFO.name());
+            final Settings settings = Settings.builder().put("logger.level", loggerLevel).build();
+            LogConfigurator.configureWithoutConfig(settings);
+        }
+
         try {
             mainWithoutErrorHandling(args, terminal);
         } catch (OptionException e) {
@@ -66,6 +102,16 @@ public final int main(String[] args, Terminal terminal) throws Exception {
         return ExitCodes.OK;
     }
 
+    /**
+     * Indicate whether or not logging should be configured without reading a log4j2.properties. Most commands should do this because we do
+     * not configure logging for CLI tools. Only commands that configure logging on their own should not do this.
+     *
+     * @return true if logging should be configured without reading a log4j2.properties file
+     */
+    protected boolean shouldConfigureLoggingWithoutConfig() {
+        return true;
+    }
+
     /**
      * Executes the command, but all errors are thrown.
      */
@@ -110,4 +156,19 @@ protected static void exit(int status) {
      * Any runtime user errors (like an input file that does not exist), should throw a {@link UserException}. */
     protected abstract void execute(Terminal terminal, OptionSet options) throws Exception;
 
+    /**
+     * Return whether or not to install the shutdown hook to cleanup resources on exit. This method should only be overridden in test
+     * classes.
+     *
+     * @return whether or not to install the shutdown hook
+     */
+    protected boolean addShutdownHook() {
+        return true;
+    }
+
+    @Override
+    public void close() throws IOException {
+
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/cli/EnvironmentAwareCommand.java b/core/src/main/java/org/elasticsearch/cli/EnvironmentAwareCommand.java
new file mode 100644
index 0000000000000..df063de18d3d2
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/cli/EnvironmentAwareCommand.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cli;
+
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+import joptsimple.util.KeyValuePair;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.node.InternalSettingsPreparer;
+
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+
+/** A cli command which requires an {@link org.elasticsearch.env.Environment} to use current paths and settings. */
+public abstract class EnvironmentAwareCommand extends Command {
+
+    private final OptionSpec<KeyValuePair> settingOption;
+
+    public EnvironmentAwareCommand(String description) {
+        super(description);
+        this.settingOption = parser.accepts("E", "Configure a setting").withRequiredArg().ofType(KeyValuePair.class);
+    }
+
+    @Override
+    protected void execute(Terminal terminal, OptionSet options) throws Exception {
+        final Map<String, String> settings = new HashMap<>();
+        for (final KeyValuePair kvp : settingOption.values(options)) {
+            if (kvp.value.isEmpty()) {
+                throw new UserException(ExitCodes.USAGE, "setting [" + kvp.key + "] must not be empty");
+            }
+            if (settings.containsKey(kvp.key)) {
+                final String message = String.format(
+                        Locale.ROOT,
+                        "setting [%s] already set, saw [%s] and [%s]",
+                        kvp.key,
+                        settings.get(kvp.key),
+                        kvp.value);
+                throw new UserException(ExitCodes.USAGE, message);
+            }
+            settings.put(kvp.key, kvp.value);
+        }
+
+        putSystemPropertyIfSettingIsMissing(settings, "default.path.conf", "es.default.path.conf");
+        putSystemPropertyIfSettingIsMissing(settings, "default.path.data", "es.default.path.data");
+        putSystemPropertyIfSettingIsMissing(settings, "default.path.logs", "es.default.path.logs");
+        putSystemPropertyIfSettingIsMissing(settings, "path.conf", "es.path.conf");
+        putSystemPropertyIfSettingIsMissing(settings, "path.data", "es.path.data");
+        putSystemPropertyIfSettingIsMissing(settings, "path.home", "es.path.home");
+        putSystemPropertyIfSettingIsMissing(settings, "path.logs", "es.path.logs");
+
+        execute(terminal, options, createEnv(terminal, settings));
+    }
+
+    /** Create an {@link Environment} for the command to use. Overrideable for tests. */
+    protected Environment createEnv(Terminal terminal, Map<String, String> settings) {
+        return InternalSettingsPreparer.prepareEnvironment(Settings.EMPTY, terminal, settings);
+    }
+
+    /** Ensure the given setting exists, reading it from system properties if not already set. */
+    protected static void putSystemPropertyIfSettingIsMissing(final Map<String, String> settings, final String setting, final String key) {
+        final String value = System.getProperty(key);
+        if (value != null) {
+            if (settings.containsKey(setting)) {
+                final String message =
+                        String.format(
+                                Locale.ROOT,
+                                "duplicate setting [%s] found via command-line [%s] and system property [%s]",
+                                setting,
+                                settings.get(setting),
+                                value);
+                throw new IllegalArgumentException(message);
+            } else {
+                settings.put(setting, value);
+            }
+        }
+    }
+
+    /** Execute the command with the initialized {@link Environment}. */
+    protected abstract void execute(Terminal terminal, OptionSet options, Environment env) throws Exception;
+
+}
diff --git a/core/src/main/java/org/elasticsearch/cli/SettingCommand.java b/core/src/main/java/org/elasticsearch/cli/SettingCommand.java
deleted file mode 100644
index 17f7c9e5204cb..0000000000000
--- a/core/src/main/java/org/elasticsearch/cli/SettingCommand.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.cli;
-
-import joptsimple.OptionSet;
-import joptsimple.OptionSpec;
-import joptsimple.util.KeyValuePair;
-
-import java.util.HashMap;
-import java.util.Locale;
-import java.util.Map;
-
-public abstract class SettingCommand extends Command {
-
-    private final OptionSpec<KeyValuePair> settingOption;
-
-    public SettingCommand(String description) {
-        super(description);
-        this.settingOption = parser.accepts("E", "Configure a setting").withRequiredArg().ofType(KeyValuePair.class);
-    }
-
-    @Override
-    protected void execute(Terminal terminal, OptionSet options) throws Exception {
-        final Map<String, String> settings = new HashMap<>();
-        for (final KeyValuePair kvp : settingOption.values(options)) {
-            if (kvp.value.isEmpty()) {
-                throw new UserException(ExitCodes.USAGE, "Setting [" + kvp.key + "] must not be empty");
-            }
-            settings.put(kvp.key, kvp.value);
-        }
-
-        putSystemPropertyIfSettingIsMissing(settings, "path.conf", "es.path.conf");
-        putSystemPropertyIfSettingIsMissing(settings, "path.data", "es.path.data");
-        putSystemPropertyIfSettingIsMissing(settings, "path.home", "es.path.home");
-        putSystemPropertyIfSettingIsMissing(settings, "path.logs", "es.path.logs");
-
-        execute(terminal, options, settings);
-    }
-
-    protected static void putSystemPropertyIfSettingIsMissing(final Map<String, String> settings, final String setting, final String key) {
-        final String value = System.getProperty(key);
-        if (value != null) {
-            if (settings.containsKey(setting)) {
-                final String message =
-                        String.format(
-                                Locale.ROOT,
-                                "duplicate setting [%s] found via command-line [%s] and system property [%s]",
-                                setting,
-                                settings.get(setting),
-                                value);
-                throw new IllegalArgumentException(message);
-            } else {
-                settings.put(setting, value);
-            }
-        }
-    }
-
-    protected abstract void execute(Terminal terminal, OptionSet options, Map<String, String> settings) throws Exception;
-
-}
diff --git a/core/src/main/java/org/elasticsearch/cli/Terminal.java b/core/src/main/java/org/elasticsearch/cli/Terminal.java
index 58eb5012d07e0..d42e3475dc491 100644
--- a/core/src/main/java/org/elasticsearch/cli/Terminal.java
+++ b/core/src/main/java/org/elasticsearch/cli/Terminal.java
@@ -27,6 +27,7 @@
 import java.io.InputStreamReader;
 import java.io.PrintWriter;
 import java.nio.charset.Charset;
+import java.util.Locale;
 
 /**
  * A Terminal wraps access to reading input and writing output for a cli.
@@ -92,6 +93,27 @@ public final void print(Verbosity verbosity, String msg) {
         }
     }
 
+    /**
+     * Prompt for a yes or no answer from the user. This method will loop until 'y' or 'n'
+     * (or the default empty value) is entered.
+     */
+    public final boolean promptYesNo(String prompt, boolean defaultYes) {
+        String answerPrompt = defaultYes ? " [Y/n]" : " [y/N]";
+        while (true) {
+            String answer = readText(prompt + answerPrompt);
+            if (answer == null || answer.isEmpty()) {
+                return defaultYes;
+            }
+            answer = answer.toLowerCase(Locale.ROOT);
+            boolean answerYes = answer.equals("y");
+            if (answerYes == false && answer.equals("n") == false) {
+                println("Did not understand answer '" + answer + "'");
+                continue;
+            }
+            return answerYes;
+        }
+    }
+
     private static class ConsoleTerminal extends Terminal {
 
         private static final Console CONSOLE = System.console();
diff --git a/core/src/main/java/org/elasticsearch/client/Client.java b/core/src/main/java/org/elasticsearch/client/Client.java
index 0cf22d7a2c4fc..2080856e30204 100644
--- a/core/src/main/java/org/elasticsearch/client/Client.java
+++ b/core/src/main/java/org/elasticsearch/client/Client.java
@@ -30,6 +30,10 @@
 import org.elasticsearch.action.explain.ExplainRequest;
 import org.elasticsearch.action.explain.ExplainRequestBuilder;
 import org.elasticsearch.action.explain.ExplainResponse;
+import org.elasticsearch.action.fieldcaps.FieldCapabilities;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequestBuilder;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse;
 import org.elasticsearch.action.fieldstats.FieldStatsRequest;
 import org.elasticsearch.action.fieldstats.FieldStatsRequestBuilder;
 import org.elasticsearch.action.fieldstats.FieldStatsResponse;
@@ -452,12 +456,39 @@ public interface Client extends ElasticsearchClient, Releasable {
      */
     void clearScroll(ClearScrollRequest request, ActionListener<ClearScrollResponse> listener);
 
+    /**
+     * @deprecated Use _field_caps instead or run a min/max aggregations on the desired fields
+     */
+    @Deprecated
     FieldStatsRequestBuilder prepareFieldStats();
 
+    /**
+     * @deprecated Use _field_caps instead or run a min/max aggregations on the desired fields
+     */
+    @Deprecated
     ActionFuture<FieldStatsResponse> fieldStats(FieldStatsRequest request);
 
+    /**
+     * @deprecated Use _field_caps instead or run a min/max aggregations on the desired fields
+     */
+    @Deprecated
     void fieldStats(FieldStatsRequest request, ActionListener<FieldStatsResponse> listener);
 
+    /**
+     * Builder for the field capabilities request.
+     */
+    FieldCapabilitiesRequestBuilder prepareFieldCaps();
+
+    /**
+     * An action that returns the field capabilities from the provided request
+     */
+    ActionFuture<FieldCapabilitiesResponse> fieldCaps(FieldCapabilitiesRequest request);
+
+    /**
+     * An action that returns the field capabilities from the provided request
+     */
+    void fieldCaps(FieldCapabilitiesRequest request, ActionListener<FieldCapabilitiesResponse> listener);
+
     /**
      * Returns this clients settings
      */
diff --git a/core/src/main/java/org/elasticsearch/client/ClusterAdminClient.java b/core/src/main/java/org/elasticsearch/client/ClusterAdminClient.java
index 9e0d1a941192c..3f705a215ea11 100644
--- a/core/src/main/java/org/elasticsearch/client/ClusterAdminClient.java
+++ b/core/src/main/java/org/elasticsearch/client/ClusterAdminClient.java
@@ -112,6 +112,7 @@
 import org.elasticsearch.action.ingest.WritePipelineResponse;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.tasks.TaskId;
 
 /**
@@ -545,9 +546,16 @@ public interface ClusterAdminClient extends ElasticsearchClient {
 
     /**
      * Stores an ingest pipeline
+     * @deprecated use {@link #preparePutPipeline(String, BytesReference, XContentType)}
      */
+    @Deprecated
     PutPipelineRequestBuilder preparePutPipeline(String id, BytesReference source);
 
+    /**
+     * Stores an ingest pipeline
+     */
+    PutPipelineRequestBuilder preparePutPipeline(String id, BytesReference source, XContentType xContentType);
+
     /**
      * Deletes a stored ingest pipeline
      */
@@ -563,6 +571,11 @@ public interface ClusterAdminClient extends ElasticsearchClient {
      */
     DeletePipelineRequestBuilder prepareDeletePipeline();
 
+    /**
+     * Deletes a stored ingest pipeline
+     */
+    DeletePipelineRequestBuilder prepareDeletePipeline(String id);
+
     /**
      * Returns a stored ingest pipeline
      */
@@ -591,8 +604,14 @@ public interface ClusterAdminClient extends ElasticsearchClient {
     /**
      * Simulates an ingest pipeline
      */
+    @Deprecated
     SimulatePipelineRequestBuilder prepareSimulatePipeline(BytesReference source);
 
+    /**
+     * Simulates an ingest pipeline
+     */
+    SimulatePipelineRequestBuilder prepareSimulatePipeline(BytesReference source, XContentType xContentType);
+
     /**
      * Explain the allocation of a shard
      */
diff --git a/core/src/main/java/org/elasticsearch/client/ElasticsearchClient.java b/core/src/main/java/org/elasticsearch/client/ElasticsearchClient.java
index d9ddc56d48a07..c6aa4991a8232 100644
--- a/core/src/main/java/org/elasticsearch/client/ElasticsearchClient.java
+++ b/core/src/main/java/org/elasticsearch/client/ElasticsearchClient.java
@@ -40,8 +40,8 @@ public interface ElasticsearchClient {
      * @param <RequestBuilder> The request builder type.
      * @return A future allowing to get back the response.
      */
-    <Request extends ActionRequest<Request>, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> ActionFuture<Response> execute(
-            final Action<Request, Response, RequestBuilder> action, final Request request);
+    <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> ActionFuture<Response> execute(
+            Action<Request, Response, RequestBuilder> action, Request request);
 
     /**
      * Executes a generic action, denoted by an {@link Action}.
@@ -53,8 +53,8 @@ <Request extends ActionRequest<Request>, Response extends ActionResponse, Reques
      * @param <Response>       The response type.
      * @param <RequestBuilder> The request builder type.
      */
-    <Request extends ActionRequest<Request>, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> void execute(
-            final Action<Request, Response, RequestBuilder> action, final Request request, ActionListener<Response> listener);
+    <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> void execute(
+            Action<Request, Response, RequestBuilder> action, Request request, ActionListener<Response> listener);
 
     /**
      * Prepares a request builder to execute, specified by {@link Action}.
@@ -65,8 +65,8 @@ <Request extends ActionRequest<Request>, Response extends ActionResponse, Reques
      * @param <RequestBuilder> The request builder.
      * @return The request builder, that can, at a later stage, execute the request.
      */
-    <Request extends ActionRequest<Request>, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> RequestBuilder prepareExecute(
-            final Action<Request, Response, RequestBuilder> action);
+    <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> RequestBuilder prepareExecute(
+            Action<Request, Response, RequestBuilder> action);
 
     /**
      * Returns the threadpool used to execute requests on this client
diff --git a/core/src/main/java/org/elasticsearch/client/FilterClient.java b/core/src/main/java/org/elasticsearch/client/FilterClient.java
index d0f52282c7683..23d3c2c3d0c2f 100644
--- a/core/src/main/java/org/elasticsearch/client/FilterClient.java
+++ b/core/src/main/java/org/elasticsearch/client/FilterClient.java
@@ -62,7 +62,7 @@ public void close() {
     }
 
     @Override
-    protected <Request extends ActionRequest<Request>, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> void doExecute(
+    protected <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> void doExecute(
             Action<Request, Response, RequestBuilder> action, Request request, ActionListener<Response> listener) {
         in().execute(action, request, listener);
     }
diff --git a/core/src/main/java/org/elasticsearch/client/IndicesAdminClient.java b/core/src/main/java/org/elasticsearch/client/IndicesAdminClient.java
index 24d190c68a10d..176cbf60b10c1 100644
--- a/core/src/main/java/org/elasticsearch/client/IndicesAdminClient.java
+++ b/core/src/main/java/org/elasticsearch/client/IndicesAdminClient.java
@@ -50,6 +50,9 @@
 import org.elasticsearch.action.admin.indices.exists.types.TypesExistsRequest;
 import org.elasticsearch.action.admin.indices.exists.types.TypesExistsRequestBuilder;
 import org.elasticsearch.action.admin.indices.exists.types.TypesExistsResponse;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequestBuilder;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse;
 import org.elasticsearch.action.admin.indices.flush.FlushRequest;
 import org.elasticsearch.action.admin.indices.flush.FlushRequestBuilder;
 import org.elasticsearch.action.admin.indices.flush.FlushResponse;
@@ -817,5 +820,4 @@ public interface IndicesAdminClient extends ElasticsearchClient {
      * Swaps the index pointed to by an alias given all provided conditions are satisfied
      */
     void rolloverIndex(RolloverRequest request, ActionListener<RolloverResponse> listener);
-
 }
diff --git a/core/src/main/java/org/elasticsearch/client/ParentTaskAssigningClient.java b/core/src/main/java/org/elasticsearch/client/ParentTaskAssigningClient.java
index 44ba2b76e43c0..62843c41b7027 100644
--- a/core/src/main/java/org/elasticsearch/client/ParentTaskAssigningClient.java
+++ b/core/src/main/java/org/elasticsearch/client/ParentTaskAssigningClient.java
@@ -58,7 +58,7 @@ public Client unwrap() {
     }
 
     @Override
-    protected <     Request extends ActionRequest<Request>,
+    protected <     Request extends ActionRequest,
                     Response extends ActionResponse,
                     RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>
               > void doExecute(Action<Request, Response, RequestBuilder> action, Request request, ActionListener<Response> listener) {
diff --git a/core/src/main/java/org/elasticsearch/client/node/NodeClient.java b/core/src/main/java/org/elasticsearch/client/node/NodeClient.java
index e68b902e2592d..e4f26b157026d 100644
--- a/core/src/main/java/org/elasticsearch/client/node/NodeClient.java
+++ b/core/src/main/java/org/elasticsearch/client/node/NodeClient.java
@@ -28,12 +28,14 @@
 import org.elasticsearch.action.support.TransportAction;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.support.AbstractClient;
+import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskListener;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.Map;
+import java.util.function.Supplier;
 
 /**
  * Client that executes actions on the local node.
@@ -41,13 +43,19 @@
 public class NodeClient extends AbstractClient {
 
     private Map<GenericAction, TransportAction> actions;
+    /**
+     * The id of the local {@link DiscoveryNode}. Useful for generating task ids from tasks returned by
+     * {@link #executeLocally(GenericAction, ActionRequest, TaskListener)}.
+     */
+    private Supplier<String> localNodeId;
 
     public NodeClient(Settings settings, ThreadPool threadPool) {
         super(settings, threadPool);
     }
 
-    public void intialize(Map<GenericAction, TransportAction> actions) {
+    public void initialize(Map<GenericAction, TransportAction> actions, Supplier<String> localNodeId) {
         this.actions = actions;
+        this.localNodeId = localNodeId;
     }
 
     @Override
@@ -56,7 +64,7 @@ public void close() {
     }
 
     @Override
-    public <    Request extends ActionRequest<Request>,
+    public <    Request extends ActionRequest,
                 Response extends ActionResponse,
                 RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>
             > void doExecute(Action<Request, Response, RequestBuilder> action, Request request, ActionListener<Response> listener) {
@@ -69,7 +77,7 @@ > void doExecute(Action<Request, Response, RequestBuilder> action, Request reque
      * method if you don't need access to the task when listening for the response. This is the method used to implement the {@link Client}
      * interface.
      */
-    public <    Request extends ActionRequest<Request>,
+    public <    Request extends ActionRequest,
                 Response extends ActionResponse
             > Task executeLocally(GenericAction<Request, Response> action, Request request, ActionListener<Response> listener) {
         return transportAction(action).execute(request, listener);
@@ -79,17 +87,25 @@ > Task executeLocally(GenericAction<Request, Response> action, Request request,
      * Execute an {@link Action} locally, returning that {@link Task} used to track it, and linking an {@link TaskListener}. Prefer this
      * method if you need access to the task when listening for the response.
      */
-    public <    Request extends ActionRequest<Request>,
+    public <    Request extends ActionRequest,
                 Response extends ActionResponse
             > Task executeLocally(GenericAction<Request, Response> action, Request request, TaskListener<Response> listener) {
         return transportAction(action).execute(request, listener);
     }
 
+    /**
+     * The id of the local {@link DiscoveryNode}. Useful for generating task ids from tasks returned by
+     * {@link #executeLocally(GenericAction, ActionRequest, TaskListener)}.
+     */
+    public String getLocalNodeId() {
+        return localNodeId.get();
+    }
+
     /**
      * Get the {@link TransportAction} for an {@link Action}, throwing exceptions if the action isn't available.
      */
     @SuppressWarnings("unchecked")
-    private <    Request extends ActionRequest<Request>,
+    private <    Request extends ActionRequest,
                 Response extends ActionResponse
             > TransportAction<Request, Response> transportAction(GenericAction<Request, Response> action) {
         if (actions == null) {
diff --git a/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java b/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java
index c3816d8d37fad..fa050af77cb16 100644
--- a/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java
+++ b/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java
@@ -272,6 +272,10 @@
 import org.elasticsearch.action.explain.ExplainRequest;
 import org.elasticsearch.action.explain.ExplainRequestBuilder;
 import org.elasticsearch.action.explain.ExplainResponse;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesAction;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequestBuilder;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse;
 import org.elasticsearch.action.fieldstats.FieldStatsAction;
 import org.elasticsearch.action.fieldstats.FieldStatsRequest;
 import org.elasticsearch.action.fieldstats.FieldStatsRequestBuilder;
@@ -343,6 +347,7 @@
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.threadpool.ThreadPool;
 
@@ -380,13 +385,13 @@ public final AdminClient admin() {
     }
 
     @Override
-    public final <Request extends ActionRequest<Request>, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> RequestBuilder prepareExecute(
+    public final <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> RequestBuilder prepareExecute(
             final Action<Request, Response, RequestBuilder> action) {
         return action.newRequestBuilder(this);
     }
 
     @Override
-    public final <Request extends ActionRequest<Request>, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> ActionFuture<Response> execute(
+    public final <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> ActionFuture<Response> execute(
             Action<Request, Response, RequestBuilder> action, Request request) {
         PlainActionFuture<Response> actionFuture = PlainActionFuture.newFuture();
         execute(action, request, actionFuture);
@@ -397,13 +402,13 @@ public final <Request extends ActionRequest<Request>, Response extends ActionRes
      * This is the single execution point of *all* clients.
      */
     @Override
-    public final <Request extends ActionRequest<Request>, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> void execute(
+    public final <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> void execute(
             Action<Request, Response, RequestBuilder> action, Request request, ActionListener<Response> listener) {
         listener = threadedWrapper.wrap(listener);
         doExecute(action, request, listener);
     }
 
-    protected abstract <Request extends ActionRequest<Request>, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> void doExecute(final Action<Request, Response, RequestBuilder> action, final Request request, ActionListener<Response> listener);
+    protected abstract <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> void doExecute(Action<Request, Response, RequestBuilder> action, Request request, ActionListener<Response> listener);
 
     @Override
     public ActionFuture<IndexResponse> index(final IndexRequest request) {
@@ -669,12 +674,27 @@ public FieldStatsRequestBuilder prepareFieldStats() {
         return new FieldStatsRequestBuilder(this, FieldStatsAction.INSTANCE);
     }
 
+    @Override
+    public void fieldCaps(FieldCapabilitiesRequest request, ActionListener<FieldCapabilitiesResponse> listener) {
+        execute(FieldCapabilitiesAction.INSTANCE, request, listener);
+    }
+
+    @Override
+    public ActionFuture<FieldCapabilitiesResponse> fieldCaps(FieldCapabilitiesRequest request) {
+        return execute(FieldCapabilitiesAction.INSTANCE, request);
+    }
+
+    @Override
+    public FieldCapabilitiesRequestBuilder prepareFieldCaps() {
+        return new FieldCapabilitiesRequestBuilder(this, FieldCapabilitiesAction.INSTANCE);
+    }
+
     static class Admin implements AdminClient {
 
         private final ClusterAdmin clusterAdmin;
         private final IndicesAdmin indicesAdmin;
 
-        public Admin(ElasticsearchClient client) {
+        Admin(ElasticsearchClient client) {
             this.clusterAdmin = new ClusterAdmin(client);
             this.indicesAdmin = new IndicesAdmin(client);
         }
@@ -694,24 +714,24 @@ static class ClusterAdmin implements ClusterAdminClient {
 
         private final ElasticsearchClient client;
 
-        public ClusterAdmin(ElasticsearchClient client) {
+        ClusterAdmin(ElasticsearchClient client) {
             this.client = client;
         }
 
         @Override
-        public <Request extends ActionRequest<Request>, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> ActionFuture<Response> execute(
+        public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> ActionFuture<Response> execute(
                 Action<Request, Response, RequestBuilder> action, Request request) {
             return client.execute(action, request);
         }
 
         @Override
-        public <Request extends ActionRequest<Request>, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> void execute(
+        public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> void execute(
                 Action<Request, Response, RequestBuilder> action, Request request, ActionListener<Response> listener) {
             client.execute(action, request, listener);
         }
 
         @Override
-        public <Request extends ActionRequest<Request>, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> RequestBuilder prepareExecute(
+        public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> RequestBuilder prepareExecute(
                 Action<Request, Response, RequestBuilder> action) {
             return client.prepareExecute(action);
         }
@@ -1084,6 +1104,11 @@ public PutPipelineRequestBuilder preparePutPipeline(String id, BytesReference so
             return new PutPipelineRequestBuilder(this, PutPipelineAction.INSTANCE, id, source);
         }
 
+        @Override
+        public PutPipelineRequestBuilder preparePutPipeline(String id, BytesReference source, XContentType xContentType) {
+            return new PutPipelineRequestBuilder(this, PutPipelineAction.INSTANCE, id, source, xContentType);
+        }
+
         @Override
         public void deletePipeline(DeletePipelineRequest request, ActionListener<WritePipelineResponse> listener) {
             execute(DeletePipelineAction.INSTANCE, request, listener);
@@ -1099,6 +1124,11 @@ public DeletePipelineRequestBuilder prepareDeletePipeline() {
             return new DeletePipelineRequestBuilder(this, DeletePipelineAction.INSTANCE);
         }
 
+        @Override
+        public DeletePipelineRequestBuilder prepareDeletePipeline(String id) {
+            return new DeletePipelineRequestBuilder(this, DeletePipelineAction.INSTANCE, id);
+        }
+
         @Override
         public void getPipeline(GetPipelineRequest request, ActionListener<GetPipelineResponse> listener) {
             execute(GetPipelineAction.INSTANCE, request, listener);
@@ -1129,6 +1159,11 @@ public SimulatePipelineRequestBuilder prepareSimulatePipeline(BytesReference sou
             return new SimulatePipelineRequestBuilder(this, SimulatePipelineAction.INSTANCE, source);
         }
 
+        @Override
+        public SimulatePipelineRequestBuilder prepareSimulatePipeline(BytesReference source, XContentType xContentType) {
+            return new SimulatePipelineRequestBuilder(this, SimulatePipelineAction.INSTANCE, source, xContentType);
+        }
+
         @Override
         public void allocationExplain(ClusterAllocationExplainRequest request, ActionListener<ClusterAllocationExplainResponse> listener) {
             execute(ClusterAllocationExplainAction.INSTANCE, request, listener);
@@ -1197,7 +1232,7 @@ public DeleteStoredScriptRequestBuilder prepareDeleteStoredScript(){
 
         @Override
         public DeleteStoredScriptRequestBuilder prepareDeleteStoredScript(@Nullable String scriptLang, String id){
-            return prepareDeleteStoredScript().setScriptLang(scriptLang).setId(id);
+            return prepareDeleteStoredScript().setLang(scriptLang).setId(id);
         }
     }
 
@@ -1205,24 +1240,24 @@ static class IndicesAdmin implements IndicesAdminClient {
 
         private final ElasticsearchClient client;
 
-        public IndicesAdmin(ElasticsearchClient client) {
+        IndicesAdmin(ElasticsearchClient client) {
             this.client = client;
         }
 
         @Override
-        public <Request extends ActionRequest<Request>, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> ActionFuture<Response> execute(
+        public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> ActionFuture<Response> execute(
                 Action<Request, Response, RequestBuilder> action, Request request) {
             return client.execute(action, request);
         }
 
         @Override
-        public <Request extends ActionRequest<Request>, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> void execute(
+        public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> void execute(
                 Action<Request, Response, RequestBuilder> action, Request request, ActionListener<Response> listener) {
             client.execute(action, request, listener);
         }
 
         @Override
-        public <Request extends ActionRequest<Request>, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> RequestBuilder prepareExecute(
+        public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> RequestBuilder prepareExecute(
                 Action<Request, Response, RequestBuilder> action) {
             return client.prepareExecute(action);
         }
@@ -1743,7 +1778,7 @@ public void getSettings(GetSettingsRequest request, ActionListener<GetSettingsRe
     public Client filterWithHeader(Map<String, String> headers) {
         return new FilterClient(this) {
             @Override
-            protected <Request extends ActionRequest<Request>, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> void doExecute(Action<Request, Response, RequestBuilder> action, Request request, ActionListener<Response> listener) {
+            protected <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> void doExecute(Action<Request, Response, RequestBuilder> action, Request request, ActionListener<Response> listener) {
                 ThreadContext threadContext = threadPool().getThreadContext();
                 try (ThreadContext.StoredContext ctx = threadContext.stashAndMergeHeaders(headers)) {
                     super.doExecute(action, request, listener);
diff --git a/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java b/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
index f7ce9f929bdc0..d748af3685035 100644
--- a/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
+++ b/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
@@ -27,8 +27,9 @@
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.client.support.AbstractClient;
-import org.elasticsearch.client.transport.support.TransportProxyClient;
+import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.component.LifecycleComponent;
 import org.elasticsearch.common.inject.Injector;
 import org.elasticsearch.common.inject.Module;
@@ -40,11 +41,14 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.node.Node;
-import org.elasticsearch.node.internal.InternalSettingsPreparer;
+import org.elasticsearch.node.InternalSettingsPreparer;
 import org.elasticsearch.plugins.ActionPlugin;
+import org.elasticsearch.plugins.NetworkPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.PluginsService;
 import org.elasticsearch.plugins.SearchPlugin;
@@ -52,16 +56,23 @@
 import org.elasticsearch.threadpool.ExecutorBuilder;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TcpTransport;
+import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportService;
 
 import java.io.Closeable;
+import java.net.UnknownHostException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static java.util.stream.Collectors.toList;
+import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
 
 /**
  * The transport client allows to create a client that is not part of the cluster, but simply connects to one
@@ -72,6 +83,15 @@
  */
 public abstract class TransportClient extends AbstractClient {
 
+    public static final Setting<TimeValue> CLIENT_TRANSPORT_NODES_SAMPLER_INTERVAL =
+        Setting.positiveTimeSetting("client.transport.nodes_sampler_interval", timeValueSeconds(5), Setting.Property.NodeScope);
+    public static final Setting<TimeValue> CLIENT_TRANSPORT_PING_TIMEOUT =
+        Setting.positiveTimeSetting("client.transport.ping_timeout", timeValueSeconds(5), Setting.Property.NodeScope);
+    public static final Setting<Boolean> CLIENT_TRANSPORT_IGNORE_CLUSTER_NAME =
+        Setting.boolSetting("client.transport.ignore_cluster_name", false, Setting.Property.NodeScope);
+    public static final Setting<Boolean> CLIENT_TRANSPORT_SNIFF =
+        Setting.boolSetting("client.transport.sniff", false, Setting.Property.NodeScope);
+
     private static PluginsService newPluginService(final Settings settings, Collection<Class<? extends Plugin>> plugins) {
         final Settings.Builder settingsBuilder = Settings.builder()
                 .put(TcpTransport.PING_SCHEDULE.getKey(), "5s") // enable by default the transport schedule ping interval
@@ -99,7 +119,7 @@ protected static Collection<Class<? extends Plugin>> addPlugins(Collection<Class
     }
 
     private static ClientTemplate buildTemplate(Settings providedSettings, Settings defaultSettings,
-                                                Collection<Class<? extends Plugin>> plugins) {
+                                                Collection<Class<? extends Plugin>> plugins, HostFailureListener failureListner) {
         if (Node.NODE_NAME_SETTING.exists(providedSettings) == false) {
             providedSettings = Settings.builder().put(providedSettings).put(Node.NODE_NAME_SETTING.getKey(), "_client_").build();
         }
@@ -110,60 +130,75 @@ private static ClientTemplate buildTemplate(Settings providedSettings, Settings
         resourcesToClose.add(() -> ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS));
         final NetworkService networkService = new NetworkService(settings, Collections.emptyList());
         try {
-            final List<Setting<?>> additionalSettings = new ArrayList<>();
-            final List<String> additionalSettingsFilter = new ArrayList<>();
-            additionalSettings.addAll(pluginsService.getPluginSettings());
-            additionalSettingsFilter.addAll(pluginsService.getPluginSettingsFilter());
+            final List<Setting<?>> additionalSettings = new ArrayList<>(pluginsService.getPluginSettings());
+            final List<String> additionalSettingsFilter = new ArrayList<>(pluginsService.getPluginSettingsFilter());
             for (final ExecutorBuilder<?> builder : threadPool.builders()) {
                 additionalSettings.addAll(builder.getRegisteredSettings());
             }
             SettingsModule settingsModule = new SettingsModule(settings, additionalSettings, additionalSettingsFilter);
 
-            NetworkModule networkModule = new NetworkModule(networkService, settings, true);
             SearchModule searchModule = new SearchModule(settings, true, pluginsService.filterPlugins(SearchPlugin.class));
             List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
-            entries.addAll(networkModule.getNamedWriteables());
+            entries.addAll(NetworkModule.getNamedWriteables());
             entries.addAll(searchModule.getNamedWriteables());
+            entries.addAll(ClusterModule.getNamedWriteables());
             entries.addAll(pluginsService.filterPlugins(Plugin.class).stream()
                                          .flatMap(p -> p.getNamedWriteables().stream())
                                          .collect(Collectors.toList()));
             NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(entries);
+            NamedXContentRegistry xContentRegistry = new NamedXContentRegistry(Stream.of(
+                    searchModule.getNamedXContents().stream(),
+                    pluginsService.filterPlugins(Plugin.class).stream()
+                            .flatMap(p -> p.getNamedXContent().stream())
+                    ).flatMap(Function.identity()).collect(toList()));
 
             ModulesBuilder modules = new ModulesBuilder();
             // plugin modules must be added here, before others or we can get crazy injection errors...
             for (Module pluginModule : pluginsService.createGuiceModules()) {
                 modules.add(pluginModule);
             }
-            modules.add(networkModule);
             modules.add(b -> b.bind(ThreadPool.class).toInstance(threadPool));
-            modules.add(searchModule);
-            ActionModule actionModule = new ActionModule(false, true, settings, null, settingsModule.getClusterSettings(),
-                pluginsService.filterPlugins(ActionPlugin.class));
+            ActionModule actionModule = new ActionModule(true, settings, null, settingsModule.getIndexScopedSettings(),
+                    settingsModule.getClusterSettings(), settingsModule.getSettingsFilter(), threadPool,
+                    pluginsService.filterPlugins(ActionPlugin.class), null, null);
             modules.add(actionModule);
 
-            pluginsService.processModules(modules);
             CircuitBreakerService circuitBreakerService = Node.createCircuitBreakerService(settingsModule.getSettings(),
                 settingsModule.getClusterSettings());
             resourcesToClose.add(circuitBreakerService);
             BigArrays bigArrays = new BigArrays(settings, circuitBreakerService);
             resourcesToClose.add(bigArrays);
             modules.add(settingsModule);
+            NetworkModule networkModule = new NetworkModule(settings, true, pluginsService.filterPlugins(NetworkPlugin.class), threadPool,
+                bigArrays, circuitBreakerService, namedWriteableRegistry, xContentRegistry, networkService, null);
+            final Transport transport = networkModule.getTransportSupplier().get();
+            final TransportAddress address;
+            try {
+                address = transport.addressesFromString("0.0.0.0:0", 1)[0]; // this is just a dummy transport address
+            } catch (UnknownHostException e) {
+                throw new RuntimeException(e);
+            }
+            final TransportService transportService = new TransportService(settings, transport, threadPool,
+                networkModule.getTransportInterceptor(),
+                boundTransportAddress -> DiscoveryNode.createLocal(settings, address, UUIDs.randomBase64UUID()), null);
             modules.add((b -> {
                 b.bind(BigArrays.class).toInstance(bigArrays);
                 b.bind(PluginsService.class).toInstance(pluginsService);
                 b.bind(CircuitBreakerService.class).toInstance(circuitBreakerService);
                 b.bind(NamedWriteableRegistry.class).toInstance(namedWriteableRegistry);
+                b.bind(Transport.class).toInstance(transport);
+                b.bind(TransportService.class).toInstance(transportService);
+                b.bind(NetworkService.class).toInstance(networkService);
             }));
 
             Injector injector = modules.createInjector();
-            final TransportService transportService = injector.getInstance(TransportService.class);
             final TransportClientNodesService nodesService =
-                new TransportClientNodesService(settings, transportService, threadPool);
+                new TransportClientNodesService(settings, transportService, threadPool, failureListner == null
+                    ? (t, e) -> {} : failureListner);
             final TransportProxyClient proxy = new TransportProxyClient(settings, transportService, nodesService,
                 actionModule.getActions().values().stream().map(x -> x.getAction()).collect(Collectors.toList()));
 
-            List<LifecycleComponent> pluginLifecycleComponents = new ArrayList<>();
-            pluginLifecycleComponents.addAll(pluginsService.getGuiceServiceClasses().stream()
+            List<LifecycleComponent> pluginLifecycleComponents = new ArrayList<>(pluginsService.getGuiceServiceClasses().stream()
                 .map(injector::getInstance).collect(Collectors.toList()));
             resourcesToClose.addAll(pluginLifecycleComponents);
 
@@ -216,7 +251,7 @@ ThreadPool getThreadPool() {
      * Creates a new TransportClient with the given settings and plugins
      */
     public TransportClient(Settings settings, Collection<Class<? extends Plugin>> plugins) {
-        this(buildTemplate(settings, Settings.EMPTY, plugins));
+        this(buildTemplate(settings, Settings.EMPTY, plugins, null));
     }
 
     /**
@@ -225,8 +260,9 @@ public TransportClient(Settings settings, Collection<Class<? extends Plugin>> pl
      * @param defaultSettings default settings that are merged after the plugins have added it's additional settings.
      * @param plugins the client plugins
      */
-    protected TransportClient(Settings settings, Settings defaultSettings, Collection<Class<? extends Plugin>> plugins) {
-        this(buildTemplate(settings, defaultSettings, plugins));
+    protected TransportClient(Settings settings, Settings defaultSettings, Collection<Class<? extends Plugin>> plugins,
+                              HostFailureListener hostFailureListener) {
+        this(buildTemplate(settings, defaultSettings, plugins, hostFailureListener));
     }
 
     private TransportClient(ClientTemplate template) {
@@ -323,7 +359,25 @@ public void close() {
     }
 
     @Override
-    protected <Request extends ActionRequest<Request>, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> void doExecute(Action<Request, Response, RequestBuilder> action, Request request, ActionListener<Response> listener) {
+    protected <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> void doExecute(Action<Request, Response, RequestBuilder> action, Request request, ActionListener<Response> listener) {
         proxy.execute(action, request, listener);
     }
+
+    /**
+     * Listener that allows to be notified whenever a node failure / disconnect happens
+     */
+    @FunctionalInterface
+    public interface HostFailureListener {
+        /**
+         * Called once a node disconnect is detected.
+         * @param node the node that has been disconnected
+         * @param ex the exception causing the disconnection
+         */
+        void onNodeDisconnected(DiscoveryNode node, Exception ex);
+    }
+
+    // pkg private for testing
+    TransportClientNodesService getNodesService() {
+        return nodesService;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java b/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
index 18c2d15ec390f..3e50b4d74c916 100644
--- a/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
+++ b/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
@@ -22,6 +22,7 @@
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
+import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
@@ -35,16 +36,20 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.common.util.concurrent.FutureUtils;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.ConnectTransportException;
+import org.elasticsearch.transport.ConnectionProfile;
 import org.elasticsearch.transport.FutureTransportResponseHandler;
+import org.elasticsearch.transport.NodeDisconnectedException;
+import org.elasticsearch.transport.NodeNotConnectedException;
+import org.elasticsearch.transport.PlainTransportFuture;
+import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportException;
 import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportResponseHandler;
@@ -64,12 +69,7 @@
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.atomic.AtomicInteger;
 
-import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
-
-/**
- *
- */
-public class TransportClientNodesService extends AbstractComponent implements Closeable {
+final class TransportClientNodesService extends AbstractComponent implements Closeable {
 
     private final TimeValue nodesSamplerInterval;
 
@@ -103,37 +103,45 @@ public class TransportClientNodesService extends AbstractComponent implements Cl
 
     private volatile boolean closed;
 
+    private final TransportClient.HostFailureListener hostFailureListener;
+
+    // TODO: migrate this to use low level connections and single type channels
+    /** {@link ConnectionProfile} to use when to connecting to the listed nodes and doing a liveness check */
+    private static final ConnectionProfile LISTED_NODES_PROFILE;
+
+    static {
+        ConnectionProfile.Builder builder = new ConnectionProfile.Builder();
+        builder.addConnections(1,
+            TransportRequestOptions.Type.BULK,
+            TransportRequestOptions.Type.PING,
+            TransportRequestOptions.Type.RECOVERY,
+            TransportRequestOptions.Type.REG,
+            TransportRequestOptions.Type.STATE);
+        LISTED_NODES_PROFILE = builder.build();
+    }
 
-    public static final Setting<TimeValue> CLIENT_TRANSPORT_NODES_SAMPLER_INTERVAL =
-        Setting.positiveTimeSetting("client.transport.nodes_sampler_interval", timeValueSeconds(5), Property.NodeScope);
-    public static final Setting<TimeValue> CLIENT_TRANSPORT_PING_TIMEOUT =
-        Setting.positiveTimeSetting("client.transport.ping_timeout", timeValueSeconds(5), Property.NodeScope);
-    public static final Setting<Boolean> CLIENT_TRANSPORT_IGNORE_CLUSTER_NAME =
-        Setting.boolSetting("client.transport.ignore_cluster_name", false, Property.NodeScope);
-    public static final Setting<Boolean> CLIENT_TRANSPORT_SNIFF =
-        Setting.boolSetting("client.transport.sniff", false, Property.NodeScope);
-
-    public TransportClientNodesService(Settings settings,TransportService transportService,
-                                       ThreadPool threadPool) {
+    TransportClientNodesService(Settings settings, TransportService transportService,
+                                       ThreadPool threadPool, TransportClient.HostFailureListener hostFailureListener) {
         super(settings);
         this.clusterName = ClusterName.CLUSTER_NAME_SETTING.get(settings);
         this.transportService = transportService;
         this.threadPool = threadPool;
         this.minCompatibilityVersion = Version.CURRENT.minimumCompatibilityVersion();
 
-        this.nodesSamplerInterval = CLIENT_TRANSPORT_NODES_SAMPLER_INTERVAL.get(this.settings);
-        this.pingTimeout = CLIENT_TRANSPORT_PING_TIMEOUT.get(this.settings).millis();
-        this.ignoreClusterName = CLIENT_TRANSPORT_IGNORE_CLUSTER_NAME.get(this.settings);
+        this.nodesSamplerInterval = TransportClient.CLIENT_TRANSPORT_NODES_SAMPLER_INTERVAL.get(this.settings);
+        this.pingTimeout = TransportClient.CLIENT_TRANSPORT_PING_TIMEOUT.get(this.settings).millis();
+        this.ignoreClusterName = TransportClient.CLIENT_TRANSPORT_IGNORE_CLUSTER_NAME.get(this.settings);
 
         if (logger.isDebugEnabled()) {
             logger.debug("node_sampler_interval[{}]", nodesSamplerInterval);
         }
 
-        if (CLIENT_TRANSPORT_SNIFF.get(this.settings)) {
+        if (TransportClient.CLIENT_TRANSPORT_SNIFF.get(this.settings)) {
             this.nodesSampler = new SniffNodesSampler();
         } else {
             this.nodesSampler = new SimpleNodeSampler();
         }
+        this.hostFailureListener = hostFailureListener;
         this.nodesSamplerFuture = threadPool.schedule(nodesSamplerInterval, ThreadPool.Names.GENERIC, new ScheduledNodeSampler());
     }
 
@@ -179,8 +187,7 @@ public TransportClientNodesService addTransportAddresses(TransportAddress... tra
             if (filtered.isEmpty()) {
                 return this;
             }
-            List<DiscoveryNode> builder = new ArrayList<>();
-            builder.addAll(listedNodes());
+            List<DiscoveryNode> builder = new ArrayList<>(listedNodes);
             for (TransportAddress transportAddress : filtered) {
                 DiscoveryNode node = new DiscoveryNode("#transport#-" + tempNodeIdGenerator.incrementAndGet(),
                         transportAddress, Collections.emptyMap(), Collections.emptySet(), minCompatibilityVersion);
@@ -198,15 +205,25 @@ public TransportClientNodesService removeTransportAddress(TransportAddress trans
             if (closed) {
                 throw new IllegalStateException("transport client is closed, can't remove an address");
             }
-            List<DiscoveryNode> builder = new ArrayList<>();
+            List<DiscoveryNode> listNodesBuilder = new ArrayList<>();
             for (DiscoveryNode otherNode : listedNodes) {
                 if (!otherNode.getAddress().equals(transportAddress)) {
-                    builder.add(otherNode);
+                    listNodesBuilder.add(otherNode);
                 } else {
-                    logger.debug("removing address [{}]", otherNode);
+                    logger.debug("removing address [{}] from listed nodes", otherNode);
                 }
             }
-            listedNodes = Collections.unmodifiableList(builder);
+            listedNodes = Collections.unmodifiableList(listNodesBuilder);
+            List<DiscoveryNode> nodesBuilder = new ArrayList<>();
+            for (DiscoveryNode otherNode : nodes) {
+                if (!otherNode.getAddress().equals(transportAddress)) {
+                    nodesBuilder.add(otherNode);
+                } else {
+                    logger.debug("disconnecting from node with address [{}]", otherNode);
+                    transportService.disconnectFromNode(otherNode);
+                }
+            }
+            nodes = Collections.unmodifiableList(nodesBuilder);
             nodesSampler.sample();
         }
         return this;
@@ -227,13 +244,17 @@ public <Response> void execute(NodeListenerCallback<Response> callback, ActionLi
         }
         ensureNodesAreAvailable(nodes);
         int index = getNodeNumber();
-        RetryListener<Response> retryListener = new RetryListener<>(callback, listener, nodes, index);
-        DiscoveryNode node = nodes.get((index) % nodes.size());
+        RetryListener<Response> retryListener = new RetryListener<>(callback, listener, nodes, index, hostFailureListener);
+        DiscoveryNode node = retryListener.getNode(0);
         try {
             callback.doWithNode(node, retryListener);
         } catch (Exception e) {
-            //this exception can't come from the TransportService as it doesn't throw exception at all
-            listener.onFailure(e);
+            try {
+                //this exception can't come from the TransportService as it doesn't throw exception at all
+                listener.onFailure(e);
+            } finally {
+                retryListener.maybeNodeFailed(node, e);
+            }
         }
     }
 
@@ -242,15 +263,17 @@ public static class RetryListener<Response> implements ActionListener<Response>
         private final ActionListener<Response> listener;
         private final List<DiscoveryNode> nodes;
         private final int index;
+        private final TransportClient.HostFailureListener hostFailureListener;
 
         private volatile int i;
 
-        public RetryListener(NodeListenerCallback<Response> callback, ActionListener<Response> listener,
-                             List<DiscoveryNode> nodes, int index) {
+        RetryListener(NodeListenerCallback<Response> callback, ActionListener<Response> listener,
+                             List<DiscoveryNode> nodes, int index, TransportClient.HostFailureListener hostFailureListener) {
             this.callback = callback;
             this.listener = listener;
             this.nodes = nodes;
             this.index = index;
+            this.hostFailureListener = hostFailureListener;
         }
 
         @Override
@@ -260,13 +283,15 @@ public void onResponse(Response response) {
 
         @Override
         public void onFailure(Exception e) {
-            if (ExceptionsHelper.unwrapCause(e) instanceof ConnectTransportException) {
+            Throwable throwable = ExceptionsHelper.unwrapCause(e);
+            if (throwable instanceof ConnectTransportException) {
+                maybeNodeFailed(getNode(this.i), (ConnectTransportException) throwable);
                 int i = ++this.i;
                 if (i >= nodes.size()) {
                     listener.onFailure(new NoNodeAvailableException("None of the configured nodes were available: " + nodes, e));
                 } else {
                     try {
-                        callback.doWithNode(nodes.get((index + i) % nodes.size()), this);
+                        callback.doWithNode(getNode(i), this);
                     } catch(final Exception inner) {
                         inner.addSuppressed(e);
                         // this exception can't come from the TransportService as it doesn't throw exceptions at all
@@ -278,7 +303,15 @@ public void onFailure(Exception e) {
             }
         }
 
+        final DiscoveryNode getNode(int i) {
+            return nodes.get((index + i) % nodes.size());
+        }
 
+        final void maybeNodeFailed(DiscoveryNode node, Exception ex) {
+            if (ex instanceof NodeDisconnectedException || ex instanceof NodeNotConnectedException) {
+                hostFailureListener.onNodeDisconnected(node, ex);
+            }
+        }
     }
 
     @Override
@@ -371,49 +404,37 @@ protected void doSample() {
             HashSet<DiscoveryNode> newNodes = new HashSet<>();
             HashSet<DiscoveryNode> newFilteredNodes = new HashSet<>();
             for (DiscoveryNode listedNode : listedNodes) {
-                if (!transportService.nodeConnected(listedNode)) {
-                    try {
-                        // its a listed node, light connect to it...
-                        logger.trace("connecting to listed node (light) [{}]", listedNode);
-                        transportService.connectToNodeLight(listedNode);
-                    } catch (Exception e) {
-                        logger.debug(
-                            (Supplier<?>)
-                                () -> new ParameterizedMessage("failed to connect to node [{}], removed from nodes list", listedNode), e);
-                        newFilteredNodes.add(listedNode);
-                        continue;
-                    }
-                }
-                try {
-                    LivenessResponse livenessResponse = transportService.submitRequest(listedNode, TransportLivenessAction.NAME,
-                            new LivenessRequest(),
-                            TransportRequestOptions.builder().withType(TransportRequestOptions.Type.STATE).withTimeout(pingTimeout).build(),
-                            new FutureTransportResponseHandler<LivenessResponse>() {
-                                @Override
-                                public LivenessResponse newInstance() {
-                                    return new LivenessResponse();
-                                }
-                            }).txGet();
+                try (Transport.Connection connection = transportService.openConnection(listedNode, LISTED_NODES_PROFILE)){
+                    final PlainTransportFuture<LivenessResponse> handler = new PlainTransportFuture<>(
+                        new FutureTransportResponseHandler<LivenessResponse>() {
+                            @Override
+                            public LivenessResponse newInstance() {
+                                return new LivenessResponse();
+                            }
+                        });
+                    transportService.sendRequest(connection, TransportLivenessAction.NAME, new LivenessRequest(),
+                        TransportRequestOptions.builder().withType(TransportRequestOptions.Type.STATE).withTimeout(pingTimeout).build(),
+                        handler);
+                    final LivenessResponse livenessResponse = handler.txGet();
                     if (!ignoreClusterName && !clusterName.equals(livenessResponse.getClusterName())) {
                         logger.warn("node {} not part of the cluster {}, ignoring...", listedNode, clusterName);
                         newFilteredNodes.add(listedNode);
-                    } else if (livenessResponse.getDiscoveryNode() != null) {
+                    } else {
                         // use discovered information but do keep the original transport address,
                         // so people can control which address is exactly used.
                         DiscoveryNode nodeWithInfo = livenessResponse.getDiscoveryNode();
                         newNodes.add(new DiscoveryNode(nodeWithInfo.getName(), nodeWithInfo.getId(), nodeWithInfo.getEphemeralId(),
                             nodeWithInfo.getHostName(), nodeWithInfo.getHostAddress(), listedNode.getAddress(),
                             nodeWithInfo.getAttributes(), nodeWithInfo.getRoles(), nodeWithInfo.getVersion()));
-                    } else {
-                        // although we asked for one node, our target may not have completed
-                        // initialization yet and doesn't have cluster nodes
-                        logger.debug("node {} didn't return any discovery info, temporarily using transport discovery node", listedNode);
-                        newNodes.add(listedNode);
                     }
+                } catch (ConnectTransportException e) {
+                    logger.debug(
+                        (Supplier<?>)
+                            () -> new ParameterizedMessage("failed to connect to node [{}], ignoring...", listedNode), e);
+                    hostFailureListener.onNodeDisconnected(listedNode, e);
                 } catch (Exception e) {
                     logger.info(
                         (Supplier<?>) () -> new ParameterizedMessage("failed to get node info for {}, disconnecting...", listedNode), e);
-                    transportService.disconnectFromNode(listedNode);
                 }
             }
 
@@ -438,76 +459,93 @@ protected void doSample() {
 
             final CountDownLatch latch = new CountDownLatch(nodesToPing.size());
             final ConcurrentMap<DiscoveryNode, ClusterStateResponse> clusterStateResponses = ConcurrentCollections.newConcurrentMap();
-            for (final DiscoveryNode listedNode : nodesToPing) {
-                threadPool.executor(ThreadPool.Names.MANAGEMENT).execute(new Runnable() {
-                    @Override
-                    public void run() {
-                        try {
-                            if (!transportService.nodeConnected(listedNode)) {
-                                try {
+            try {
+                for (final DiscoveryNode nodeToPing : nodesToPing) {
+                    threadPool.executor(ThreadPool.Names.MANAGEMENT).execute(new AbstractRunnable() {
+
+                        /**
+                         * we try to reuse existing connections but if needed we will open a temporary connection
+                         * that will be closed at the end of the execution.
+                         */
+                        Transport.Connection connectionToClose = null;
+
+                        void onDone() {
+                            try {
+                                IOUtils.closeWhileHandlingException(connectionToClose);
+                            } finally {
+                                latch.countDown();
+                            }
+                        }
 
-                                    // if its one of the actual nodes we will talk to, not to listed nodes, fully connect
-                                    if (nodes.contains(listedNode)) {
-                                        logger.trace("connecting to cluster node [{}]", listedNode);
-                                        transportService.connectToNode(listedNode);
-                                    } else {
-                                        // its a listed node, light connect to it...
-                                        logger.trace("connecting to listed node (light) [{}]", listedNode);
-                                        transportService.connectToNodeLight(listedNode);
-                                    }
-                                } catch (Exception e) {
-                                    logger.debug(
-                                        (Supplier<?>)
-                                            () -> new ParameterizedMessage("failed to connect to node [{}], ignoring...", listedNode), e);
-                                    latch.countDown();
-                                    return;
+                        @Override
+                        public void onFailure(Exception e) {
+                            onDone();
+                            if (e instanceof ConnectTransportException) {
+                                logger.debug((Supplier<?>)
+                                    () -> new ParameterizedMessage("failed to connect to node [{}], ignoring...", nodeToPing), e);
+                                hostFailureListener.onNodeDisconnected(nodeToPing, e);
+                            } else {
+                                logger.info(
+                                    (Supplier<?>) () -> new ParameterizedMessage(
+                                        "failed to get local cluster state info for {}, disconnecting...", nodeToPing), e);
+                            }
+                        }
+
+                        @Override
+                        protected void doRun() throws Exception {
+                            Transport.Connection pingConnection = null;
+                            if (nodes.contains(nodeToPing)) {
+                                try {
+                                    pingConnection = transportService.getConnection(nodeToPing);
+                                } catch (NodeNotConnectedException e) {
+                                    // will use a temp connection
                                 }
                             }
-                            transportService.sendRequest(listedNode, ClusterStateAction.NAME,
-                                    Requests.clusterStateRequest().clear().nodes(true).local(true),
-                                    TransportRequestOptions.builder().withType(TransportRequestOptions.Type.STATE)
-                                            .withTimeout(pingTimeout).build(),
-                                    new TransportResponseHandler<ClusterStateResponse>() {
-
-                                        @Override
-                                        public ClusterStateResponse newInstance() {
-                                            return new ClusterStateResponse();
-                                        }
+                            if (pingConnection == null) {
+                                logger.trace("connecting to cluster node [{}]", nodeToPing);
+                                connectionToClose = transportService.openConnection(nodeToPing, LISTED_NODES_PROFILE);
+                                pingConnection = connectionToClose;
+                            }
+                            transportService.sendRequest(pingConnection, ClusterStateAction.NAME,
+                                Requests.clusterStateRequest().clear().nodes(true).local(true),
+                                TransportRequestOptions.builder().withType(TransportRequestOptions.Type.STATE)
+                                    .withTimeout(pingTimeout).build(),
+                                new TransportResponseHandler<ClusterStateResponse>() {
+
+                                    @Override
+                                    public ClusterStateResponse newInstance() {
+                                        return new ClusterStateResponse();
+                                    }
 
-                                        @Override
-                                        public String executor() {
-                                            return ThreadPool.Names.SAME;
-                                        }
+                                    @Override
+                                    public String executor() {
+                                        return ThreadPool.Names.SAME;
+                                    }
 
-                                        @Override
-                                        public void handleResponse(ClusterStateResponse response) {
-                                            clusterStateResponses.put(listedNode, response);
-                                            latch.countDown();
-                                        }
+                                    @Override
+                                    public void handleResponse(ClusterStateResponse response) {
+                                        clusterStateResponses.put(nodeToPing, response);
+                                        onDone();
+                                    }
 
-                                        @Override
-                                        public void handleException(TransportException e) {
-                                            logger.info(
-                                                (Supplier<?>) () -> new ParameterizedMessage(
-                                                    "failed to get local cluster state for {}, disconnecting...", listedNode), e);
-                                            transportService.disconnectFromNode(listedNode);
-                                            latch.countDown();
+                                    @Override
+                                    public void handleException(TransportException e) {
+                                        logger.info(
+                                            (Supplier<?>) () -> new ParameterizedMessage(
+                                                "failed to get local cluster state for {}, disconnecting...", nodeToPing), e);
+                                        try {
+                                            hostFailureListener.onNodeDisconnected(nodeToPing, e);
+                                        } finally {
+                                            onDone();
                                         }
-                                    });
-                        } catch (Exception e) {
-                            logger.info(
-                                (Supplier<?>)() -> new ParameterizedMessage(
-                                    "failed to get local cluster state info for {}, disconnecting...", listedNode), e);
-                            transportService.disconnectFromNode(listedNode);
-                            latch.countDown();
+                                    }
+                                });
                         }
-                    }
-                });
-            }
-
-            try {
+                    });
+                }
                 latch.await();
             } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
                 return;
             }
 
@@ -534,4 +572,9 @@ public interface NodeListenerCallback<Response> {
 
         void doWithNode(DiscoveryNode node, ActionListener<Response> listener);
     }
+
+    // pkg private for testing
+    void doSample() {
+        nodesSampler.doSample();
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/client/transport/support/TransportProxyClient.java b/core/src/main/java/org/elasticsearch/client/transport/TransportProxyClient.java
similarity index 79%
rename from core/src/main/java/org/elasticsearch/client/transport/support/TransportProxyClient.java
rename to core/src/main/java/org/elasticsearch/client/transport/TransportProxyClient.java
index 34833e9400a8e..5436bef172a47 100644
--- a/core/src/main/java/org/elasticsearch/client/transport/support/TransportProxyClient.java
+++ b/core/src/main/java/org/elasticsearch/client/transport/TransportProxyClient.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.client.transport.support;
+package org.elasticsearch.client.transport;
 
 import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionListener;
@@ -26,9 +26,6 @@
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.GenericAction;
 import org.elasticsearch.action.TransportActionNodeProxy;
-import org.elasticsearch.client.transport.TransportClientNodesService;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.transport.TransportService;
 
@@ -38,15 +35,12 @@
 
 import static java.util.Collections.unmodifiableMap;
 
-/**
- *
- */
-public class TransportProxyClient {
+final class TransportProxyClient {
 
     private final TransportClientNodesService nodesService;
     private final Map<Action, TransportActionNodeProxy> proxies;
 
-    public TransportProxyClient(Settings settings, TransportService transportService,
+    TransportProxyClient(Settings settings, TransportService transportService,
                                 TransportClientNodesService nodesService, List<GenericAction> actions) {
         this.nodesService = nodesService;
         Map<Action, TransportActionNodeProxy> proxies = new HashMap<>();
@@ -58,7 +52,9 @@ public TransportProxyClient(Settings settings, TransportService transportService
         this.proxies = unmodifiableMap(proxies);
     }
 
-    public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> void execute(final Action<Request, Response, RequestBuilder> action, final Request request, ActionListener<Response> listener) {
+    public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends
+        ActionRequestBuilder<Request, Response, RequestBuilder>> void execute(final Action<Request, Response, RequestBuilder> action,
+                                                                              final Request request, ActionListener<Response> listener) {
         final TransportActionNodeProxy<Request, Response> proxy = proxies.get(action);
         nodesService.execute((n, l) -> proxy.execute(n, request, l), listener);
     }
diff --git a/core/src/main/java/org/elasticsearch/cluster/AbstractDiffable.java b/core/src/main/java/org/elasticsearch/cluster/AbstractDiffable.java
index f9d5f33cad6ca..8e63bc2b9d700 100644
--- a/core/src/main/java/org/elasticsearch/cluster/AbstractDiffable.java
+++ b/core/src/main/java/org/elasticsearch/cluster/AbstractDiffable.java
@@ -40,12 +40,7 @@ public Diff<T> diff(T previousState) {
         }
     }
 
-    @Override
-    public Diff<T> readDiffFrom(StreamInput in) throws IOException {
-        return new CompleteDiff<>(this, in);
-    }
-
-    public static <T extends Diffable<T>> Diff<T> readDiffFrom(T reader, StreamInput in) throws IOException {
+    public static <T extends Diffable<T>> Diff<T> readDiffFrom(Reader<T> reader, StreamInput in) throws IOException {
         return new CompleteDiff<T>(reader, in);
     }
 
@@ -57,23 +52,23 @@ private static class CompleteDiff<T extends Diffable<T>> implements Diff<T> {
         /**
          * Creates simple diff with changes
          */
-        public CompleteDiff(T part) {
+        CompleteDiff(T part) {
             this.part = part;
         }
 
         /**
          * Creates simple diff without changes
          */
-        public CompleteDiff() {
+        CompleteDiff() {
             this.part = null;
         }
 
         /**
          * Read simple diff from the stream
          */
-        public CompleteDiff(Diffable<T> reader, StreamInput in) throws IOException {
+        CompleteDiff(Reader<T> reader, StreamInput in) throws IOException {
             if (in.readBoolean()) {
-                this.part = reader.readFrom(in);
+                this.part = reader.read(in);
             } else {
                 this.part = null;
             }
diff --git a/core/src/main/java/org/elasticsearch/cluster/AbstractNamedDiffable.java b/core/src/main/java/org/elasticsearch/cluster/AbstractNamedDiffable.java
new file mode 100644
index 0000000000000..fb253a1a5df69
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/cluster/AbstractNamedDiffable.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.io.stream.NamedWriteable;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+
+/**
+ * Abstract diffable object with simple diffs implementation that sends the entire object if object has changed or
+ * nothing is object remained the same. Comparing to AbstractDiffable, this class also works with NamedWriteables
+ */
+public abstract class AbstractNamedDiffable<T extends NamedDiffable<T>> implements Diffable<T>, NamedWriteable {
+
+    @Override
+    public Diff<T> diff(T previousState) {
+        if (this.get().equals(previousState)) {
+            return new CompleteNamedDiff<>(previousState.getWriteableName(), previousState.getMinimalSupportedVersion());
+        } else {
+            return new CompleteNamedDiff<>(get());
+        }
+    }
+
+    public static <T extends NamedDiffable<T>> NamedDiff<T> readDiffFrom(Class<? extends T> tClass, String name, StreamInput in)
+        throws IOException {
+        return new CompleteNamedDiff<>(tClass, name, in);
+    }
+
+    private static class CompleteNamedDiff<T extends NamedDiffable<T>> implements NamedDiff<T> {
+
+        @Nullable
+        private final T part;
+
+        private final String name;
+
+        /**
+         * A non-null value is only required for write operation, if the diff was just read from the stream the version
+         * is unnecessary.
+         */
+        @Nullable
+        private final Version minimalSupportedVersion;
+
+        /**
+         * Creates simple diff with changes
+         */
+        CompleteNamedDiff(T part) {
+            this.part = part;
+            this.name = part.getWriteableName();
+            this.minimalSupportedVersion = part.getMinimalSupportedVersion();
+        }
+
+        /**
+         * Creates simple diff without changes
+         */
+        CompleteNamedDiff(String name, Version minimalSupportedVersion) {
+            this.part = null;
+            this.name = name;
+            this.minimalSupportedVersion = minimalSupportedVersion;
+        }
+
+        /**
+         * Read simple diff from the stream
+         */
+        CompleteNamedDiff(Class<? extends T> tClass, String name, StreamInput in) throws IOException {
+            if (in.readBoolean()) {
+                this.part = in.readNamedWriteable(tClass, name);
+                this.minimalSupportedVersion = part.getMinimalSupportedVersion();
+            } else {
+                this.part = null;
+                this.minimalSupportedVersion = null; // We just read this diff, so it's not going to be written
+            }
+            this.name = name;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            assert minimalSupportedVersion != null : "shouldn't be called on diff that was de-serialized from the stream";
+            if (part != null) {
+                out.writeBoolean(true);
+                part.writeTo(out);
+            } else {
+                out.writeBoolean(false);
+            }
+        }
+
+        @Override
+        public T apply(T part) {
+            if (this.part != null) {
+                return this.part;
+            } else {
+                return part;
+            }
+        }
+
+        @Override
+        public String getWriteableName() {
+            return name;
+        }
+
+        @Override
+        public Version getMinimalSupportedVersion() {
+            assert minimalSupportedVersion != null : "shouldn't be called on the diff that was de-serialized from the stream";
+            return minimalSupportedVersion;
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    public T get() {
+        return (T) this;
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterChangedEvent.java b/core/src/main/java/org/elasticsearch/cluster/ClusterChangedEvent.java
index e3164eacdbb6b..701656db9ce8f 100644
--- a/core/src/main/java/org/elasticsearch/cluster/ClusterChangedEvent.java
+++ b/core/src/main/java/org/elasticsearch/cluster/ClusterChangedEvent.java
@@ -20,17 +20,21 @@
 package org.elasticsearch.cluster;
 
 import com.carrotsearch.hppc.cursors.ObjectCursor;
+import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexGraveyard;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.index.Index;
 
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Objects;
+import java.util.Set;
 import java.util.stream.Collectors;
 
 /**
@@ -143,6 +147,33 @@ public boolean metaDataChanged() {
         return state.metaData() != previousState.metaData();
     }
 
+    /**
+     * Returns a set of custom meta data types when any custom metadata for the cluster has changed
+     * between the previous cluster state and the new cluster state. custom meta data types are
+     * returned iff they have been added, updated or removed between the previous and the current state
+     */
+    public Set<String> changedCustomMetaDataSet() {
+        Set<String> result = new HashSet<>();
+        ImmutableOpenMap<String, MetaData.Custom> currentCustoms = state.metaData().customs();
+        ImmutableOpenMap<String, MetaData.Custom> previousCustoms = previousState.metaData().customs();
+        if (currentCustoms.equals(previousCustoms) == false) {
+            for (ObjectObjectCursor<String, MetaData.Custom> currentCustomMetaData : currentCustoms) {
+                // new custom md added or existing custom md changed
+                if (previousCustoms.containsKey(currentCustomMetaData.key) == false
+                        || currentCustomMetaData.value.equals(previousCustoms.get(currentCustomMetaData.key)) == false) {
+                    result.add(currentCustomMetaData.key);
+                }
+            }
+            // existing custom md deleted
+            for (ObjectObjectCursor<String, MetaData.Custom> previousCustomMetaData : previousCustoms) {
+                if (currentCustoms.containsKey(previousCustomMetaData.key) == false) {
+                    result.add(previousCustomMetaData.key);
+                }
+            }
+        }
+        return result;
+    }
+
     /**
      * Returns <code>true</code> iff the {@link IndexMetaData} for a given index
      * has changed between the previous cluster state and the new cluster state.
diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java b/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java
index 4e582cb32ca03..2d59abdeb9eab 100644
--- a/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java
+++ b/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java
@@ -19,12 +19,12 @@
 
 package org.elasticsearch.cluster;
 
-import org.apache.logging.log4j.Logger;
 import org.elasticsearch.cluster.action.index.MappingUpdatedAction;
 import org.elasticsearch.cluster.action.index.NodeMappingRefreshAction;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
+import org.elasticsearch.cluster.metadata.IndexGraveyard;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.metadata.IndexTemplateFilter;
+import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.metadata.MetaDataCreateIndexService;
 import org.elasticsearch.cluster.metadata.MetaDataDeleteIndexService;
 import org.elasticsearch.cluster.metadata.MetaDataIndexAliasesService;
@@ -32,6 +32,7 @@
 import org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService;
 import org.elasticsearch.cluster.metadata.MetaDataMappingService;
 import org.elasticsearch.cluster.metadata.MetaDataUpdateSettingsService;
+import org.elasticsearch.cluster.metadata.RepositoriesMetaData;
 import org.elasticsearch.cluster.routing.DelayedAllocationService;
 import org.elasticsearch.cluster.routing.RoutingService;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
@@ -54,19 +55,28 @@
 import org.elasticsearch.cluster.routing.allocation.decider.SnapshotInProgressAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.inject.AbstractModule;
-import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.io.stream.NamedWriteable;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.io.stream.Writeable.Reader;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.ExtensionPoint;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.gateway.GatewayAllocator;
+import org.elasticsearch.ingest.IngestMetadata;
 import org.elasticsearch.plugins.ClusterPlugin;
+import org.elasticsearch.script.ScriptMetaData;
 import org.elasticsearch.tasks.TaskResultsService;
 
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -89,9 +99,6 @@ public class ClusterModule extends AbstractModule {
     final Collection<AllocationDecider> allocationDeciders;
     final ShardsAllocator shardsAllocator;
 
-    // pkg private so tests can mock
-    Class<? extends ClusterInfoService> clusterInfoServiceImpl = InternalClusterInfoService.class;
-
     public ClusterModule(Settings settings, ClusterService clusterService, List<ClusterPlugin> clusterPlugins) {
         this.settings = settings;
         this.allocationDeciders = createAllocationDeciders(settings, clusterService.getClusterSettings(), clusterPlugins);
@@ -100,6 +107,52 @@ public ClusterModule(Settings settings, ClusterService clusterService, List<Clus
         indexNameExpressionResolver = new IndexNameExpressionResolver(settings);
     }
 
+
+    public static List<Entry> getNamedWriteables() {
+        List<Entry> entries = new ArrayList<>();
+        // Cluster State
+        registerClusterCustom(entries, SnapshotsInProgress.TYPE, SnapshotsInProgress::new, SnapshotsInProgress::readDiffFrom);
+        registerClusterCustom(entries, RestoreInProgress.TYPE, RestoreInProgress::new, RestoreInProgress::readDiffFrom);
+        registerClusterCustom(entries, SnapshotDeletionsInProgress.TYPE, SnapshotDeletionsInProgress::new,
+            SnapshotDeletionsInProgress::readDiffFrom);
+        // Metadata
+        registerMetaDataCustom(entries, RepositoriesMetaData.TYPE, RepositoriesMetaData::new, RepositoriesMetaData::readDiffFrom);
+        registerMetaDataCustom(entries, IngestMetadata.TYPE, IngestMetadata::new, IngestMetadata::readDiffFrom);
+        registerMetaDataCustom(entries, ScriptMetaData.TYPE, ScriptMetaData::new, ScriptMetaData::readDiffFrom);
+        registerMetaDataCustom(entries, IndexGraveyard.TYPE, IndexGraveyard::new, IndexGraveyard::readDiffFrom);
+        return entries;
+    }
+
+    public static List<NamedXContentRegistry.Entry> getNamedXWriteables() {
+        List<NamedXContentRegistry.Entry> entries = new ArrayList<>();
+        // Metadata
+        entries.add(new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(RepositoriesMetaData.TYPE),
+            RepositoriesMetaData::fromXContent));
+        entries.add(new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(IngestMetadata.TYPE),
+            IngestMetadata::fromXContent));
+        entries.add(new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(ScriptMetaData.TYPE),
+            ScriptMetaData::fromXContent));
+        entries.add(new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(IndexGraveyard.TYPE),
+            IndexGraveyard::fromXContent));
+        return entries;
+    }
+
+    private static <T extends ClusterState.Custom> void registerClusterCustom(List<Entry> entries, String name, Reader<? extends T> reader,
+                                                                       Reader<NamedDiff> diffReader) {
+        registerCustom(entries, ClusterState.Custom.class, name, reader, diffReader);
+    }
+
+    private static <T extends MetaData.Custom> void registerMetaDataCustom(List<Entry> entries, String name, Reader<? extends T> reader,
+                                                                       Reader<NamedDiff> diffReader) {
+        registerCustom(entries, MetaData.Custom.class, name, reader, diffReader);
+    }
+
+    private static <T extends NamedWriteable> void registerCustom(List<Entry> entries, Class<T> category, String name,
+                                                                  Reader<? extends T> reader, Reader<NamedDiff> diffReader) {
+        entries.add(new Entry(category, name, reader));
+        entries.add(new Entry(NamedDiff.class, name, diffReader));
+    }
+
     public IndexNameExpressionResolver getIndexNameExpressionResolver() {
         return indexNameExpressionResolver;
     }
@@ -109,21 +162,21 @@ public IndexNameExpressionResolver getIndexNameExpressionResolver() {
     public static Collection<AllocationDecider> createAllocationDeciders(Settings settings, ClusterSettings clusterSettings,
                                                                          List<ClusterPlugin> clusterPlugins) {
         // collect deciders by class so that we can detect duplicates
-        Map<Class, AllocationDecider> deciders = new HashMap<>();
+        Map<Class, AllocationDecider> deciders = new LinkedHashMap<>();
         addAllocationDecider(deciders, new MaxRetryAllocationDecider(settings));
-        addAllocationDecider(deciders, new SameShardAllocationDecider(settings));
-        addAllocationDecider(deciders, new FilterAllocationDecider(settings, clusterSettings));
         addAllocationDecider(deciders, new ReplicaAfterPrimaryActiveAllocationDecider(settings));
-        addAllocationDecider(deciders, new ThrottlingAllocationDecider(settings, clusterSettings));
         addAllocationDecider(deciders, new RebalanceOnlyWhenActiveAllocationDecider(settings));
         addAllocationDecider(deciders, new ClusterRebalanceAllocationDecider(settings, clusterSettings));
         addAllocationDecider(deciders, new ConcurrentRebalanceAllocationDecider(settings, clusterSettings));
         addAllocationDecider(deciders, new EnableAllocationDecider(settings, clusterSettings));
-        addAllocationDecider(deciders, new AwarenessAllocationDecider(settings, clusterSettings));
-        addAllocationDecider(deciders, new ShardsLimitAllocationDecider(settings, clusterSettings));
         addAllocationDecider(deciders, new NodeVersionAllocationDecider(settings));
-        addAllocationDecider(deciders, new DiskThresholdDecider(settings, clusterSettings));
         addAllocationDecider(deciders, new SnapshotInProgressAllocationDecider(settings, clusterSettings));
+        addAllocationDecider(deciders, new FilterAllocationDecider(settings, clusterSettings));
+        addAllocationDecider(deciders, new SameShardAllocationDecider(settings, clusterSettings));
+        addAllocationDecider(deciders, new DiskThresholdDecider(settings, clusterSettings));
+        addAllocationDecider(deciders, new ThrottlingAllocationDecider(settings, clusterSettings));
+        addAllocationDecider(deciders, new ShardsLimitAllocationDecider(settings, clusterSettings));
+        addAllocationDecider(deciders, new AwarenessAllocationDecider(settings, clusterSettings));
 
         clusterPlugins.stream()
             .flatMap(p -> p.createAllocationDeciders(settings, clusterSettings).stream())
@@ -162,7 +215,6 @@ private static ShardsAllocator createShardsAllocator(Settings settings, ClusterS
 
     @Override
     protected void configure() {
-        bind(ClusterInfoService.class).to(clusterInfoServiceImpl).asEagerSingleton();
         bind(GatewayAllocator.class).asEagerSingleton();
         bind(AllocationService.class).asEagerSingleton();
         bind(ClusterService.class).toInstance(clusterService);
diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterState.java b/core/src/main/java/org/elasticsearch/cluster/ClusterState.java
index 202fc0f7bad1f..f0fd06c2d28c9 100644
--- a/core/src/main/java/org/elasticsearch/cluster/ClusterState.java
+++ b/core/src/main/java/org/elasticsearch/cluster/ClusterState.java
@@ -22,7 +22,6 @@
 import com.carrotsearch.hppc.cursors.IntObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
-
 import org.elasticsearch.cluster.block.ClusterBlock;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
@@ -37,25 +36,24 @@
 import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
-import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.discovery.Discovery;
-import org.elasticsearch.discovery.local.LocalDiscovery;
-import org.elasticsearch.discovery.zen.publish.PublishClusterStateAction;
+import org.elasticsearch.discovery.zen.PublishClusterStateAction;
 
 import java.io.IOException;
 import java.util.EnumSet;
@@ -67,83 +65,34 @@
 /**
  * Represents the current state of the cluster.
  * <p>
- * The cluster state object is immutable with an
- * exception of the {@link RoutingNodes} structure, which is built on demand from the {@link RoutingTable},
- * and cluster state {@link #status}, which is updated during cluster state publishing and applying
- * processing.  The cluster state can be updated only on the master node. All updates are performed by on a
+ * The cluster state object is immutable with the exception of the {@link RoutingNodes} structure, which is
+ * built on demand from the {@link RoutingTable}.
+ * The cluster state can be updated only on the master node. All updates are performed by on a
  * single thread and controlled by the {@link ClusterService}. After every update the
- * {@link Discovery#publish} method publishes new version of the cluster state to all other nodes in the
+ * {@link Discovery#publish} method publishes a new version of the cluster state to all other nodes in the
  * cluster.  The actual publishing mechanism is delegated to the {@link Discovery#publish} method and depends on
- * the type of discovery. For example, for local discovery it is implemented by the {@link LocalDiscovery#publish}
- * method. In the Zen Discovery it is handled in the {@link PublishClusterStateAction#publish} method. The
+ * the type of discovery. In the Zen Discovery it is handled in the {@link PublishClusterStateAction#publish} method. The
  * publishing mechanism can be overridden by other discovery.
  * <p>
  * The cluster state implements the {@link Diffable} interface in order to support publishing of cluster state
  * differences instead of the entire state on each change. The publishing mechanism should only send differences
- * to a node if this node was present in the previous version of the cluster state. If a node is not present was
- * not present in the previous version of the cluster state, such node is unlikely to have the previous cluster
- * state version and should be sent a complete version. In order to make sure that the differences are applied to
+ * to a node if this node was present in the previous version of the cluster state. If a node was
+ * not present in the previous version of the cluster state, this node is unlikely to have the previous cluster
+ * state version and should be sent a complete version. In order to make sure that the differences are applied to the
  * correct version of the cluster state, each cluster state version update generates {@link #stateUUID} that uniquely
  * identifies this version of the state. This uuid is verified by the {@link ClusterStateDiff#apply} method to
- * makes sure that the correct diffs are applied. If uuids don’t match, the {@link ClusterStateDiff#apply} method
- * throws the {@link IncompatibleClusterStateVersionException}, which should cause the publishing mechanism to send
+ * make sure that the correct diffs are applied. If uuids don’t match, the {@link ClusterStateDiff#apply} method
+ * throws the {@link IncompatibleClusterStateVersionException}, which causes the publishing mechanism to send
  * a full version of the cluster state to the node on which this exception was thrown.
  */
 public class ClusterState implements ToXContent, Diffable<ClusterState> {
 
-    public static final ClusterState PROTO = builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).build();
-
-    public static enum ClusterStateStatus {
-        UNKNOWN((byte) 0),
-        RECEIVED((byte) 1),
-        BEING_APPLIED((byte) 2),
-        APPLIED((byte) 3);
-
-        private final byte id;
-
-        ClusterStateStatus(byte id) {
-            this.id = id;
-        }
+    public static final ClusterState EMPTY_STATE = builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).build();
 
-        public byte id() {
-            return this.id;
-        }
+    public interface Custom extends NamedDiffable<Custom>, ToXContent {
     }
 
-    public interface Custom extends Diffable<Custom>, ToXContent {
-
-        String type();
-    }
-
-    private static final Map<String, Custom> customPrototypes = new HashMap<>();
-
-    /**
-     * Register a custom index meta data factory. Make sure to call it from a static block.
-     */
-    public static void registerPrototype(String type, Custom proto) {
-        customPrototypes.put(type, proto);
-    }
-
-    static {
-        // register non plugin custom parts
-        registerPrototype(SnapshotsInProgress.TYPE, SnapshotsInProgress.PROTO);
-        registerPrototype(RestoreInProgress.TYPE, RestoreInProgress.PROTO);
-    }
-
-    @Nullable
-    public static <T extends Custom> T lookupPrototype(String type) {
-        //noinspection unchecked
-        return (T) customPrototypes.get(type);
-    }
-
-    public static <T extends Custom> T lookupPrototypeSafe(String type) {
-        @SuppressWarnings("unchecked")
-        T proto = (T) customPrototypes.get(type);
-        if (proto == null) {
-            throw new IllegalArgumentException("No custom state prototype registered for type [" + type + "], node likely missing plugins");
-        }
-        return proto;
-    }
+    private static final NamedDiffableValueSerializer<Custom> CUSTOM_VALUE_SERIALIZER = new NamedDiffableValueSerializer<>(Custom.class);
 
     public static final String UNKNOWN_UUID = "_na_";
 
@@ -170,13 +119,13 @@ public static <T extends Custom> T lookupPrototypeSafe(String type) {
     // built on demand
     private volatile RoutingNodes routingNodes;
 
-    private volatile ClusterStateStatus status;
-
     public ClusterState(long version, String stateUUID, ClusterState state) {
-        this(state.clusterName, version, stateUUID, state.metaData(), state.routingTable(), state.nodes(), state.blocks(), state.customs(), false);
+        this(state.clusterName, version, stateUUID, state.metaData(), state.routingTable(), state.nodes(), state.blocks(), state.customs(),
+            false);
     }
 
-    public ClusterState(ClusterName clusterName, long version, String stateUUID, MetaData metaData, RoutingTable routingTable, DiscoveryNodes nodes, ClusterBlocks blocks, ImmutableOpenMap<String, Custom> customs, boolean wasReadFromDiff) {
+    public ClusterState(ClusterName clusterName, long version, String stateUUID, MetaData metaData, RoutingTable routingTable,
+                        DiscoveryNodes nodes, ClusterBlocks blocks, ImmutableOpenMap<String, Custom> customs, boolean wasReadFromDiff) {
         this.version = version;
         this.stateUUID = stateUUID;
         this.clusterName = clusterName;
@@ -185,19 +134,9 @@ public ClusterState(ClusterName clusterName, long version, String stateUUID, Met
         this.nodes = nodes;
         this.blocks = blocks;
         this.customs = customs;
-        this.status = ClusterStateStatus.UNKNOWN;
         this.wasReadFromDiff = wasReadFromDiff;
     }
 
-    public ClusterStateStatus status() {
-        return status;
-    }
-
-    public ClusterState status(ClusterStateStatus newStatus) {
-        this.status = newStatus;
-        return this;
-    }
-
     public long version() {
         return this.version;
     }
@@ -278,50 +217,48 @@ public RoutingNodes getRoutingNodes() {
         return routingNodes;
     }
 
-    public String prettyPrint() {
+    @Override
+    public String toString() {
         StringBuilder sb = new StringBuilder();
         sb.append("cluster uuid: ").append(metaData.clusterUUID()).append("\n");
         sb.append("version: ").append(version).append("\n");
         sb.append("state uuid: ").append(stateUUID).append("\n");
         sb.append("from_diff: ").append(wasReadFromDiff).append("\n");
         sb.append("meta data version: ").append(metaData.version()).append("\n");
+        final String TAB = "   ";
         for (IndexMetaData indexMetaData : metaData) {
-            final String TAB = "   ";
             sb.append(TAB).append(indexMetaData.getIndex());
             sb.append(": v[").append(indexMetaData.getVersion()).append("]\n");
             for (int shard = 0; shard < indexMetaData.getNumberOfShards(); shard++) {
                 sb.append(TAB).append(TAB).append(shard).append(": ");
                 sb.append("p_term [").append(indexMetaData.primaryTerm(shard)).append("], ");
-                sb.append("a_ids ").append(indexMetaData.inSyncAllocationIds(shard)).append("\n");
+                sb.append("isa_ids ").append(indexMetaData.inSyncAllocationIds(shard)).append("\n");
             }
         }
-        sb.append(blocks().prettyPrint());
-        sb.append(nodes().prettyPrint());
-        sb.append(routingTable().prettyPrint());
-        sb.append(getRoutingNodes().prettyPrint());
-        return sb.toString();
-    }
-
-    @Override
-    public String toString() {
-        try {
-            XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint();
-            builder.startObject();
-            toXContent(builder, EMPTY_PARAMS);
-            builder.endObject();
-            return builder.string();
-        } catch (IOException e) {
-            return "{ \"error\" : \"" + e.getMessage() + "\"}";
+        sb.append(blocks());
+        sb.append(nodes());
+        sb.append(routingTable());
+        sb.append(getRoutingNodes());
+        if (customs.isEmpty() == false) {
+            sb.append("customs:\n");
+            for (ObjectObjectCursor<String, Custom> cursor : customs) {
+                final String type = cursor.key;
+                final Custom custom = cursor.value;
+                sb.append(TAB).append(type).append(": ").append(custom);
+            }
         }
+        return sb.toString();
     }
 
     /**
-     * a cluster state supersedes another state iff they are from the same master and the version this state is higher thant the other state.
+     * a cluster state supersedes another state if they are from the same master and the version of this state is higher than that of the
+     * other state.
      * <p>
      * In essence that means that all the changes from the other cluster state are also reflected by the current one
      */
     public boolean supersedes(ClusterState other) {
-        return this.nodes().getMasterNodeId() != null && this.nodes().getMasterNodeId().equals(other.nodes().getMasterNodeId()) && this.version() > other.version();
+        return this.nodes().getMasterNodeId() != null && this.nodes().getMasterNodeId().equals(other.nodes().getMasterNodeId())
+            && this.version() > other.version();
 
     }
 
@@ -346,7 +283,7 @@ public enum Metric {
 
         private final String value;
 
-        private Metric(String value) {
+        Metric(String value) {
             this.value = value;
         }
 
@@ -443,11 +380,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
 
                 builder.startObject("mappings");
                 for (ObjectObjectCursor<String, CompressedXContent> cursor1 : templateMetaData.mappings()) {
-                    byte[] mappingSource = cursor1.value.uncompressed();
-                    Map<String, Object> mapping;
-                    try (XContentParser parser = XContentFactory.xContent(mappingSource).createParser(mappingSource)) {
-                        mapping = parser.map();
-                    }
+                    Map<String, Object> mapping = XContentHelper.convertToMap(new BytesArray(cursor1.value.uncompressed()), false).v2();
                     if (mapping.size() == 1 && mapping.containsKey(cursor1.key)) {
                         // the type name is the root value, reduce it
                         mapping = (Map<String, Object>) mapping.get(cursor1.key);
@@ -475,11 +408,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
 
                 builder.startObject("mappings");
                 for (ObjectObjectCursor<String, MappingMetaData> cursor : indexMetaData.getMappings()) {
-                    byte[] mappingSource = cursor.value.source().uncompressed();
-                    Map<String, Object> mapping;
-                    try (XContentParser parser = XContentFactory.xContent(mappingSource).createParser(mappingSource)) {
-                        mapping = parser.map();
-                    }
+                    Map<String, Object> mapping = XContentHelper
+                            .convertToMap(new BytesArray(cursor.value.source().uncompressed()), false).v2();
                     if (mapping.size() == 1 && mapping.containsKey(cursor.key)) {
                         // the type name is the root value, reduce it
                         mapping = (Map<String, Object>) mapping.get(cursor.key);
@@ -629,12 +559,6 @@ public DiscoveryNodes nodes() {
             return nodes;
         }
 
-        public Builder routingResult(RoutingAllocation.Result routingResult) {
-            this.routingTable = routingResult.routingTable();
-            this.metaData = routingResult.metaData();
-            return this;
-        }
-
         public Builder routingTable(RoutingTable routingTable) {
             this.routingTable = routingTable;
             return this;
@@ -674,10 +598,6 @@ public Builder stateUUID(String uuid) {
             return this;
         }
 
-        public Custom getCustom(String type) {
-            return customs.get(type);
-        }
-
         public Builder putCustom(String type, Custom custom) {
             customs.put(type, custom);
             return this;
@@ -715,53 +635,39 @@ public static byte[] toBytes(ClusterState state) throws IOException {
          * @param data      input bytes
          * @param localNode used to set the local node in the cluster state.
          */
-        public static ClusterState fromBytes(byte[] data, DiscoveryNode localNode) throws IOException {
-            return readFrom(StreamInput.wrap(data), localNode);
+        public static ClusterState fromBytes(byte[] data, DiscoveryNode localNode, NamedWriteableRegistry registry) throws IOException {
+            StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(data), registry);
+            return readFrom(in, localNode);
 
         }
-
-        /**
-         * @param in        input stream
-         * @param localNode used to set the local node in the cluster state. can be null.
-         */
-        public static ClusterState readFrom(StreamInput in, @Nullable DiscoveryNode localNode) throws IOException {
-            return PROTO.readFrom(in, localNode);
-        }
     }
 
     @Override
-    public Diff diff(ClusterState previousState) {
+    public Diff<ClusterState> diff(ClusterState previousState) {
         return new ClusterStateDiff(previousState, this);
     }
 
-    @Override
-    public Diff<ClusterState> readDiffFrom(StreamInput in) throws IOException {
-        return new ClusterStateDiff(in, this);
+    public static Diff<ClusterState> readDiffFrom(StreamInput in, DiscoveryNode localNode) throws IOException {
+        return new ClusterStateDiff(in, localNode);
     }
 
-    public ClusterState readFrom(StreamInput in, DiscoveryNode localNode) throws IOException {
+    public static ClusterState readFrom(StreamInput in, DiscoveryNode localNode) throws IOException {
         ClusterName clusterName = new ClusterName(in);
         Builder builder = new Builder(clusterName);
         builder.version = in.readLong();
         builder.uuid = in.readString();
-        builder.metaData = MetaData.Builder.readFrom(in);
-        builder.routingTable = RoutingTable.Builder.readFrom(in);
-        builder.nodes = DiscoveryNodes.Builder.readFrom(in, localNode);
-        builder.blocks = ClusterBlocks.Builder.readClusterBlocks(in);
+        builder.metaData = MetaData.readFrom(in);
+        builder.routingTable = RoutingTable.readFrom(in);
+        builder.nodes = DiscoveryNodes.readFrom(in, localNode);
+        builder.blocks = new ClusterBlocks(in);
         int customSize = in.readVInt();
         for (int i = 0; i < customSize; i++) {
-            String type = in.readString();
-            Custom customIndexMetaData = lookupPrototypeSafe(type).readFrom(in);
-            builder.putCustom(type, customIndexMetaData);
+            Custom customIndexMetaData = in.readNamedWriteable(Custom.class);
+            builder.putCustom(customIndexMetaData.getWriteableName(), customIndexMetaData);
         }
         return builder.build();
     }
 
-    @Override
-    public ClusterState readFrom(StreamInput in) throws IOException {
-        return readFrom(in, nodes.getLocalNode());
-    }
-
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         clusterName.writeTo(out);
@@ -771,10 +677,18 @@ public void writeTo(StreamOutput out) throws IOException {
         routingTable.writeTo(out);
         nodes.writeTo(out);
         blocks.writeTo(out);
-        out.writeVInt(customs.size());
-        for (ObjectObjectCursor<String, Custom> cursor : customs) {
-            out.writeString(cursor.key);
-            cursor.value.writeTo(out);
+        // filter out custom states not supported by the other node
+        int numberOfCustoms = 0;
+        for (ObjectCursor<Custom> cursor : customs.values()) {
+            if (out.getVersion().onOrAfter(cursor.value.getMinimalSupportedVersion())) {
+                numberOfCustoms++;
+            }
+        }
+        out.writeVInt(numberOfCustoms);
+        for (ObjectCursor<Custom> cursor : customs.values()) {
+            if (out.getVersion().onOrAfter(cursor.value.getMinimalSupportedVersion())) {
+                out.writeNamedWriteable(cursor.value);
+            }
         }
     }
 
@@ -798,7 +712,7 @@ private static class ClusterStateDiff implements Diff<ClusterState> {
 
         private final Diff<ImmutableOpenMap<String, Custom>> customs;
 
-        public ClusterStateDiff(ClusterState before, ClusterState after) {
+        ClusterStateDiff(ClusterState before, ClusterState after) {
             fromUuid = before.stateUUID;
             toUuid = after.stateUUID;
             toVersion = after.version;
@@ -807,30 +721,19 @@ public ClusterStateDiff(ClusterState before, ClusterState after) {
             nodes = after.nodes.diff(before.nodes);
             metaData = after.metaData.diff(before.metaData);
             blocks = after.blocks.diff(before.blocks);
-            customs = DiffableUtils.diff(before.customs, after.customs, DiffableUtils.getStringKeySerializer());
+            customs = DiffableUtils.diff(before.customs, after.customs, DiffableUtils.getStringKeySerializer(), CUSTOM_VALUE_SERIALIZER);
         }
 
-        public ClusterStateDiff(StreamInput in, ClusterState proto) throws IOException {
+        ClusterStateDiff(StreamInput in, DiscoveryNode localNode) throws IOException {
             clusterName = new ClusterName(in);
             fromUuid = in.readString();
             toUuid = in.readString();
             toVersion = in.readLong();
-            routingTable = proto.routingTable.readDiffFrom(in);
-            nodes = proto.nodes.readDiffFrom(in);
-            metaData = proto.metaData.readDiffFrom(in);
-            blocks = proto.blocks.readDiffFrom(in);
-            customs = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(),
-                new DiffableUtils.DiffableValueSerializer<String, Custom>() {
-                    @Override
-                    public Custom read(StreamInput in, String key) throws IOException {
-                        return lookupPrototypeSafe(key).readFrom(in);
-                    }
-
-                    @Override
-                    public Diff<Custom> readDiff(StreamInput in, String key) throws IOException {
-                        return lookupPrototypeSafe(key).readDiffFrom(in);
-                    }
-                });
+            routingTable = RoutingTable.readDiffFrom(in);
+            nodes = DiscoveryNodes.readDiffFrom(in, localNode);
+            metaData = MetaData.readDiffFrom(in);
+            blocks = ClusterBlocks.readDiffFrom(in);
+            customs = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), CUSTOM_VALUE_SERIALIZER);
         }
 
         @Override
diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterStateApplier.java b/core/src/main/java/org/elasticsearch/cluster/ClusterStateApplier.java
new file mode 100644
index 0000000000000..c339a8ed97e7a
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/cluster/ClusterStateApplier.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster;
+
+import org.elasticsearch.cluster.service.ClusterService;
+
+/**
+ * A component that is in charge of applying an incoming cluster state to the node internal data structures.
+ * The single apply method is called before the cluster state becomes visible via {@link ClusterService#state()}.
+ */
+public interface ClusterStateApplier {
+
+    /**
+     * Called when a new cluster state ({@link ClusterChangedEvent#state()} needs to be applied
+     */
+    void applyClusterState(ClusterChangedEvent event);
+}
diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterStateObserver.java b/core/src/main/java/org/elasticsearch/cluster/ClusterStateObserver.java
index e18ec5543d944..fd4783c9bd8b6 100644
--- a/core/src/main/java/org/elasticsearch/cluster/ClusterStateObserver.java
+++ b/core/src/main/java/org/elasticsearch/cluster/ClusterStateObserver.java
@@ -26,7 +26,10 @@
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 
+import java.util.Objects;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Predicate;
+import java.util.function.Supplier;
 
 /**
  * A utility class which simplifies interacting with the cluster state in cases where
@@ -37,20 +40,14 @@ public class ClusterStateObserver {
 
     protected final Logger logger;
 
-    public final ChangePredicate MATCH_ALL_CHANGES_PREDICATE = new EventPredicate() {
-
-        @Override
-        public boolean apply(ClusterChangedEvent changedEvent) {
-            return changedEvent.previousState().version() != changedEvent.state().version();
-        }
-    };
+    private final Predicate<ClusterState> MATCH_ALL_CHANGES_PREDICATE = state -> true;
 
     private final ClusterService clusterService;
     private final ThreadContext contextHolder;
     volatile TimeValue timeOutValue;
 
 
-    final AtomicReference<ObservedState> lastObservedState;
+    final AtomicReference<StoredState> lastObservedState;
     final TimeoutClusterStateListener clusterStateListener = new ObserverClusterStateListener();
     // observingContext is not null when waiting on cluster state changes
     final AtomicReference<ObservingContext> observingContext = new AtomicReference<>(null);
@@ -68,8 +65,17 @@ public ClusterStateObserver(ClusterService clusterService, Logger logger, Thread
      *                       to wait indefinitely
      */
     public ClusterStateObserver(ClusterService clusterService, @Nullable TimeValue timeout, Logger logger, ThreadContext contextHolder) {
+        this(clusterService.state(), clusterService, timeout, logger, contextHolder);
+    }
+    /**
+     * @param timeout        a global timeout for this observer. After it has expired the observer
+     *                       will fail any existing or new #waitForNextChange calls. Set to null
+     *                       to wait indefinitely
+     */
+    public ClusterStateObserver(ClusterState initialState, ClusterService clusterService, @Nullable TimeValue timeout, Logger logger,
+                                ThreadContext contextHolder) {
         this.clusterService = clusterService;
-        this.lastObservedState = new AtomicReference<>(new ObservedState(clusterService.state()));
+        this.lastObservedState = new AtomicReference<>(new StoredState(initialState));
         this.timeOutValue = timeout;
         if (timeOutValue != null) {
             this.startTimeNS = System.nanoTime();
@@ -78,11 +84,14 @@ public ClusterStateObserver(ClusterService clusterService, @Nullable TimeValue t
         this.contextHolder = contextHolder;
     }
 
-    /** last cluster state observer by this observer. Note that this may not be the current one */
-    public ClusterState observedState() {
-        ObservedState state = lastObservedState.get();
-        assert state != null;
-        return state.clusterState;
+    /** sets the last observed state to the currently applied cluster state and returns it */
+    public ClusterState setAndGetObservedState() {
+        if (observingContext.get() != null) {
+            throw new ElasticsearchException("cannot set current cluster state while waiting for a cluster state change");
+        }
+        ClusterState clusterState = clusterService.state();
+        lastObservedState.set(new StoredState(clusterState));
+        return clusterState;
     }
 
     /** indicates whether this observer has timedout */
@@ -98,19 +107,19 @@ public void waitForNextChange(Listener listener, @Nullable TimeValue timeOutValu
         waitForNextChange(listener, MATCH_ALL_CHANGES_PREDICATE, timeOutValue);
     }
 
-    public void waitForNextChange(Listener listener, ChangePredicate changePredicate) {
-        waitForNextChange(listener, changePredicate, null);
+    public void waitForNextChange(Listener listener, Predicate<ClusterState> statePredicate) {
+        waitForNextChange(listener, statePredicate, null);
     }
 
     /**
-     * Wait for the next cluster state which satisfies changePredicate
+     * Wait for the next cluster state which satisfies statePredicate
      *
      * @param listener        callback listener
-     * @param changePredicate predicate to check whether cluster state changes are relevant and the callback should be called
+     * @param statePredicate predicate to check whether cluster state changes are relevant and the callback should be called
      * @param timeOutValue    a timeout for waiting. If null the global observer timeout will be used.
      */
-    public void waitForNextChange(Listener listener, ChangePredicate changePredicate, @Nullable TimeValue timeOutValue) {
-
+    public void waitForNextChange(Listener listener, Predicate<ClusterState> statePredicate, @Nullable TimeValue timeOutValue) {
+        listener = new ContextPreservingListener(listener, contextHolder.newRestorableContext(false));
         if (observingContext.get() != null) {
             throw new ElasticsearchException("already waiting for a cluster state change");
         }
@@ -126,7 +135,7 @@ public void waitForNextChange(Listener listener, ChangePredicate changePredicate
                     logger.trace("observer timed out. notifying listener. timeout setting [{}], time since start [{}]", timeOutValue, new TimeValue(timeSinceStartMS));
                     // update to latest, in case people want to retry
                     timedOut = true;
-                    lastObservedState.set(new ObservedState(clusterService.state()));
+                    lastObservedState.set(new StoredState(clusterService.state()));
                     listener.onTimeout(timeOutValue);
                     return;
                 }
@@ -140,34 +149,24 @@ public void waitForNextChange(Listener listener, ChangePredicate changePredicate
             timedOut = false;
         }
 
-        // sample a new state
-        ObservedState newState = new ObservedState(clusterService.state());
-        ObservedState lastState = lastObservedState.get();
-        if (changePredicate.apply(lastState.clusterState, lastState.status, newState.clusterState, newState.status)) {
+        // sample a new state. This state maybe *older* than the supplied state if we are called from an applier,
+        // which wants to wait for something else to happen
+        ClusterState newState = clusterService.state();
+        if (lastObservedState.get().isOlderOrDifferentMaster(newState) && statePredicate.test(newState)) {
             // good enough, let's go.
             logger.trace("observer: sampled state accepted by predicate ({})", newState);
-            lastObservedState.set(newState);
-            listener.onNewClusterState(newState.clusterState);
+            lastObservedState.set(new StoredState(newState));
+            listener.onNewClusterState(newState);
         } else {
             logger.trace("observer: sampled state rejected by predicate ({}). adding listener to ClusterService", newState);
-            ObservingContext context = new ObservingContext(new ContextPreservingListener(listener, contextHolder.newStoredContext()), changePredicate);
+            final ObservingContext context = new ObservingContext(listener, statePredicate);
             if (!observingContext.compareAndSet(null, context)) {
                 throw new ElasticsearchException("already waiting for a cluster state change");
             }
-            clusterService.add(timeoutTimeLeftMS == null ? null : new TimeValue(timeoutTimeLeftMS), clusterStateListener);
+            clusterService.addTimeoutListener(timeoutTimeLeftMS == null ? null : new TimeValue(timeoutTimeLeftMS), clusterStateListener);
         }
     }
 
-    /**
-     * reset this observer to the give cluster state. Any pending waits will be canceled.
-     */
-    public void reset(ClusterState toState) {
-        if (observingContext.getAndSet(null) != null) {
-            clusterService.remove(clusterStateListener);
-        }
-        lastObservedState.set(new ObservedState(toState));
-    }
-
     class ObserverClusterStateListener implements TimeoutClusterStateListener {
 
         @Override
@@ -177,18 +176,18 @@ public void clusterChanged(ClusterChangedEvent event) {
                 // No need to remove listener as it is the responsibility of the thread that set observingContext to null
                 return;
             }
-            if (context.changePredicate.apply(event)) {
+            final ClusterState state = event.state();
+            if (context.statePredicate.test(state)) {
                 if (observingContext.compareAndSet(context, null)) {
-                    clusterService.remove(this);
-                    ObservedState state = new ObservedState(event.state());
+                    clusterService.removeTimeoutListener(this);
                     logger.trace("observer: accepting cluster state change ({})", state);
-                    lastObservedState.set(state);
-                    context.listener.onNewClusterState(state.clusterState);
+                    lastObservedState.set(new StoredState(state));
+                    context.listener.onNewClusterState(state);
                 } else {
-                    logger.trace("observer: predicate approved change but observing context has changed - ignoring (new cluster state version [{}])", event.state().version());
+                    logger.trace("observer: predicate approved change but observing context has changed - ignoring (new cluster state version [{}])", state.version());
                 }
             } else {
-                logger.trace("observer: predicate rejected change (new cluster state version [{}])", event.state().version());
+                logger.trace("observer: predicate rejected change (new cluster state version [{}])", state.version());
             }
         }
 
@@ -199,15 +198,14 @@ public void postAdded() {
                 // No need to remove listener as it is the responsibility of the thread that set observingContext to null
                 return;
             }
-            ObservedState newState = new ObservedState(clusterService.state());
-            ObservedState lastState = lastObservedState.get();
-            if (context.changePredicate.apply(lastState.clusterState, lastState.status, newState.clusterState, newState.status)) {
+            ClusterState newState = clusterService.state();
+            if (lastObservedState.get().isOlderOrDifferentMaster(newState) && context.statePredicate.test(newState)) {
                 // double check we're still listening
                 if (observingContext.compareAndSet(context, null)) {
                     logger.trace("observer: post adding listener: accepting current cluster state ({})", newState);
-                    clusterService.remove(this);
-                    lastObservedState.set(newState);
-                    context.listener.onNewClusterState(newState.clusterState);
+                    clusterService.removeTimeoutListener(this);
+                    lastObservedState.set(new StoredState(newState));
+                    context.listener.onNewClusterState(newState);
                 } else {
                     logger.trace("observer: postAdded - predicate approved state but observing context has changed - ignoring ({})", newState);
                 }
@@ -222,7 +220,7 @@ public void onClose() {
 
             if (context != null) {
                 logger.trace("observer: cluster service closed. notifying listener.");
-                clusterService.remove(this);
+                clusterService.removeTimeoutListener(this);
                 context.listener.onClusterServiceClose();
             }
         }
@@ -231,17 +229,37 @@ public void onClose() {
         public void onTimeout(TimeValue timeout) {
             ObservingContext context = observingContext.getAndSet(null);
             if (context != null) {
-                clusterService.remove(this);
+                clusterService.removeTimeoutListener(this);
                 long timeSinceStartMS = TimeValue.nsecToMSec(System.nanoTime() - startTimeNS);
                 logger.trace("observer: timeout notification from cluster service. timeout setting [{}], time since start [{}]", timeOutValue, new TimeValue(timeSinceStartMS));
                 // update to latest, in case people want to retry
-                lastObservedState.set(new ObservedState(clusterService.state()));
+                lastObservedState.set(new StoredState(clusterService.state()));
                 timedOut = true;
                 context.listener.onTimeout(timeOutValue);
             }
         }
     }
 
+    /**
+     * The observer considers two cluster states to be the same if they have the same version and master node id (i.e. null or set)
+     */
+    private static class StoredState {
+        private final String masterNodeId;
+        private final long version;
+
+        StoredState(ClusterState clusterState) {
+            this.masterNodeId = clusterState.nodes().getMasterNodeId();
+            this.version = clusterState.version();
+        }
+
+        /**
+         * returns true if stored state is older then given state or they are from a different master, meaning they can't be compared
+         * */
+        public boolean isOlderOrDifferentMaster(ClusterState clusterState) {
+            return version < clusterState.version() || Objects.equals(masterNodeId, clusterState.nodes().getMasterNodeId()) == false;
+        }
+    }
+
     public interface Listener {
 
         /** called when a new state is observed */
@@ -253,101 +271,45 @@ public interface Listener {
         void onTimeout(TimeValue timeout);
     }
 
-    public interface ChangePredicate {
-
-        /**
-         * a rough check used when starting to monitor for a new change. Called infrequently can be less accurate.
-         *
-         * @return true if newState should be accepted
-         */
-        boolean apply(ClusterState previousState,
-                      ClusterState.ClusterStateStatus previousStatus,
-                      ClusterState newState,
-                      ClusterState.ClusterStateStatus newStatus);
-
-        /**
-         * called to see whether a cluster change should be accepted
-         *
-         * @return true if changedEvent.state() should be accepted
-         */
-        boolean apply(ClusterChangedEvent changedEvent);
-    }
-
-
-    public abstract static class ValidationPredicate implements ChangePredicate {
-
-        @Override
-        public boolean apply(ClusterState previousState, ClusterState.ClusterStateStatus previousStatus, ClusterState newState, ClusterState.ClusterStateStatus newStatus) {
-            return (previousState != newState || previousStatus != newStatus) && validate(newState);
-        }
-
-        protected abstract boolean validate(ClusterState newState);
-
-        @Override
-        public boolean apply(ClusterChangedEvent changedEvent) {
-            return changedEvent.previousState().version() != changedEvent.state().version() && validate(changedEvent.state());
-        }
-    }
-
-    public abstract static class EventPredicate implements ChangePredicate {
-        @Override
-        public boolean apply(ClusterState previousState, ClusterState.ClusterStateStatus previousStatus, ClusterState newState, ClusterState.ClusterStateStatus newStatus) {
-            return previousState != newState || previousStatus != newStatus;
-        }
-
-    }
-
     static class ObservingContext {
         public final Listener listener;
-        public final ChangePredicate changePredicate;
+        public final Predicate<ClusterState> statePredicate;
 
-        public ObservingContext(Listener listener, ChangePredicate changePredicate) {
+        ObservingContext(Listener listener, Predicate<ClusterState> statePredicate) {
             this.listener = listener;
-            this.changePredicate = changePredicate;
-        }
-    }
-
-    static class ObservedState {
-        public final ClusterState clusterState;
-        public final ClusterState.ClusterStateStatus status;
-
-        public ObservedState(ClusterState clusterState) {
-            this.clusterState = clusterState;
-            this.status = clusterState.status();
-        }
-
-        @Override
-        public String toString() {
-            return "version [" + clusterState.version() + "], status [" + status + "]";
+            this.statePredicate = statePredicate;
         }
     }
 
     private static final class ContextPreservingListener implements Listener {
         private final Listener delegate;
-        private final ThreadContext.StoredContext tempContext;
+        private final Supplier<ThreadContext.StoredContext> contextSupplier;
 
 
-        private ContextPreservingListener(Listener delegate, ThreadContext.StoredContext storedContext) {
-            this.tempContext = storedContext;
+        private ContextPreservingListener(Listener delegate, Supplier<ThreadContext.StoredContext> contextSupplier) {
+            this.contextSupplier = contextSupplier;
             this.delegate = delegate;
         }
 
         @Override
         public void onNewClusterState(ClusterState state) {
-            tempContext.restore();
-            delegate.onNewClusterState(state);
+            try (ThreadContext.StoredContext context  = contextSupplier.get()) {
+                delegate.onNewClusterState(state);
+            }
         }
 
         @Override
         public void onClusterServiceClose() {
-            tempContext.restore();
-            delegate.onClusterServiceClose();
+            try (ThreadContext.StoredContext context  = contextSupplier.get()) {
+                delegate.onClusterServiceClose();
+            }
         }
 
         @Override
         public void onTimeout(TimeValue timeout) {
-            tempContext.restore();
-            delegate.onTimeout(timeout);
+            try (ThreadContext.StoredContext context  = contextSupplier.get()) {
+                delegate.onTimeout(timeout);
+            }
         }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterStateTaskExecutor.java b/core/src/main/java/org/elasticsearch/cluster/ClusterStateTaskExecutor.java
index 9324e075e90d6..3693447cfb6d2 100644
--- a/core/src/main/java/org/elasticsearch/cluster/ClusterStateTaskExecutor.java
+++ b/core/src/main/java/org/elasticsearch/cluster/ClusterStateTaskExecutor.java
@@ -18,20 +18,21 @@
  */
 package org.elasticsearch.cluster;
 
+import org.elasticsearch.common.Nullable;
+
 import java.util.IdentityHashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.function.Consumer;
 
 public interface ClusterStateTaskExecutor<T> {
     /**
      * Update the cluster state based on the current state and the given tasks. Return the *same instance* if no state
      * should be changed.
      */
-    BatchResult<T> execute(ClusterState currentState, List<T> tasks) throws Exception;
+    ClusterTasksResult<T> execute(ClusterState currentState, List<T> tasks) throws Exception;
 
     /**
-     * indicates whether this task should only run if current node is master
+     * indicates whether this executor should only run if the current node is master
      */
     default boolean runOnlyOnMaster() {
         return true;
@@ -69,18 +70,22 @@ default String describeTasks(List<T> tasks) {
      * Represents the result of a batched execution of cluster state update tasks
      * @param <T> the type of the cluster state update task
      */
-    class BatchResult<T> {
+    class ClusterTasksResult<T> {
+        public final boolean noMaster;
+        @Nullable
         public final ClusterState resultingState;
         public final Map<T, TaskResult> executionResults;
 
         /**
          * Construct an execution result instance with a correspondence between the tasks and their execution result
+         * @param noMaster whether this node steps down as master or has lost connection to the master
          * @param resultingState the resulting cluster state
          * @param executionResults the correspondence between tasks and their outcome
          */
-        BatchResult(ClusterState resultingState, Map<T, TaskResult> executionResults) {
+        ClusterTasksResult(boolean noMaster, ClusterState resultingState, Map<T, TaskResult> executionResults) {
             this.resultingState = resultingState;
             this.executionResults = executionResults;
+            this.noMaster = noMaster;
         }
 
         public static <T> Builder<T> builder() {
@@ -118,8 +123,13 @@ private Builder<T> result(T task, TaskResult executionResult) {
                 return this;
             }
 
-            public BatchResult<T> build(ClusterState resultingState) {
-                return new BatchResult<>(resultingState, executionResults);
+            public ClusterTasksResult<T> build(ClusterState resultingState) {
+                return new ClusterTasksResult<>(false, resultingState, executionResults);
+            }
+
+            ClusterTasksResult<T> build(ClusterTasksResult<T> result, ClusterState previousState) {
+                return new ClusterTasksResult<>(result.noMaster, result.resultingState == null ? previousState : result.resultingState,
+                    executionResults);
             }
         }
     }
@@ -149,18 +159,5 @@ public Exception getFailure() {
             assert !isSuccess();
             return failure;
         }
-
-        /**
-         * Handle the execution result with the provided consumers
-         * @param onSuccess handler to invoke on success
-         * @param onFailure handler to invoke on failure; the throwable passed through will not be null
-         */
-        public void handle(Runnable onSuccess, Consumer<Exception> onFailure) {
-            if (failure == null) {
-                onSuccess.run();
-            } else {
-                onFailure.accept(failure);
-            }
-        }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterStateUpdateTask.java b/core/src/main/java/org/elasticsearch/cluster/ClusterStateUpdateTask.java
index a679d09861679..b298e7e915dea 100644
--- a/core/src/main/java/org/elasticsearch/cluster/ClusterStateUpdateTask.java
+++ b/core/src/main/java/org/elasticsearch/cluster/ClusterStateUpdateTask.java
@@ -28,7 +28,7 @@
 /**
  * A task that can update the cluster state.
  */
-public abstract  class ClusterStateUpdateTask implements ClusterStateTaskConfig, ClusterStateTaskExecutor<ClusterStateUpdateTask>, ClusterStateTaskListener {
+public abstract class ClusterStateUpdateTask implements ClusterStateTaskConfig, ClusterStateTaskExecutor<ClusterStateUpdateTask>, ClusterStateTaskListener {
 
     private final Priority priority;
 
@@ -41,9 +41,9 @@ public ClusterStateUpdateTask(Priority priority) {
     }
 
     @Override
-    public final BatchResult<ClusterStateUpdateTask> execute(ClusterState currentState, List<ClusterStateUpdateTask> tasks) throws Exception {
+    public final ClusterTasksResult<ClusterStateUpdateTask> execute(ClusterState currentState, List<ClusterStateUpdateTask> tasks) throws Exception {
         ClusterState result = execute(currentState);
-        return BatchResult.<ClusterStateUpdateTask>builder().successes(tasks).build(result);
+        return ClusterTasksResult.<ClusterStateUpdateTask>builder().successes(tasks).build(result);
     }
 
     @Override
@@ -75,4 +75,13 @@ public TimeValue timeout() {
     public Priority priority() {
         return priority;
     }
+
+    /**
+     * Marked as final as cluster state update tasks should only run on master.
+     * For local requests, use {@link LocalClusterUpdateTask} instead.
+     */
+    @Override
+    public final boolean runOnlyOnMaster() {
+        return true;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/Diffable.java b/core/src/main/java/org/elasticsearch/cluster/Diffable.java
index b039f5e9b8bf7..57d5ea9ed1f75 100644
--- a/core/src/main/java/org/elasticsearch/cluster/Diffable.java
+++ b/core/src/main/java/org/elasticsearch/cluster/Diffable.java
@@ -34,13 +34,4 @@ public interface Diffable<T> extends Writeable {
      */
     Diff<T> diff(T previousState);
 
-    /**
-     * Reads the {@link org.elasticsearch.cluster.Diff} from StreamInput
-     */
-    Diff<T> readDiffFrom(StreamInput in) throws IOException;
-
-    /**
-     * Reads an object of this type from the provided {@linkplain StreamInput}. The receiving instance remains unchanged.
-     */
-    T readFrom(StreamInput in) throws IOException;
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/DiffableUtils.java b/core/src/main/java/org/elasticsearch/cluster/DiffableUtils.java
index 1a3557890dd38..f7bb42b8dc368 100644
--- a/core/src/main/java/org/elasticsearch/cluster/DiffableUtils.java
+++ b/core/src/main/java/org/elasticsearch/cluster/DiffableUtils.java
@@ -23,10 +23,12 @@
 import com.carrotsearch.hppc.cursors.IntObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.collect.ImmutableOpenIntMap;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable.Reader;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -74,7 +76,7 @@ public static <K, T extends Diffable<T>> MapDiff<K, T, ImmutableOpenMap<K, T>> d
     /**
      * Calculates diff between two ImmutableOpenMaps of non-diffable objects
      */
-    public static <K, T> MapDiff<K, T, ImmutableOpenMap<K, T>> diff(ImmutableOpenMap<K, T> before, ImmutableOpenMap<K, T> after, KeySerializer<K> keySerializer, NonDiffableValueSerializer<K, T> valueSerializer) {
+    public static <K, T> MapDiff<K, T, ImmutableOpenMap<K, T>> diff(ImmutableOpenMap<K, T> before, ImmutableOpenMap<K, T> after, KeySerializer<K> keySerializer, ValueSerializer<K, T> valueSerializer) {
         assert after != null && before != null;
         return new ImmutableOpenMapDiff<>(before, after, keySerializer, valueSerializer);
     }
@@ -90,7 +92,7 @@ public static <T extends Diffable<T>> MapDiff<Integer, T, ImmutableOpenIntMap<T>
     /**
      * Calculates diff between two ImmutableOpenIntMaps of non-diffable objects
      */
-    public static <T> MapDiff<Integer, T, ImmutableOpenIntMap<T>> diff(ImmutableOpenIntMap<T> before, ImmutableOpenIntMap<T> after, KeySerializer<Integer> keySerializer, NonDiffableValueSerializer<Integer, T> valueSerializer) {
+    public static <T> MapDiff<Integer, T, ImmutableOpenIntMap<T>> diff(ImmutableOpenIntMap<T> before, ImmutableOpenIntMap<T> after, KeySerializer<Integer> keySerializer, ValueSerializer<Integer, T> valueSerializer) {
         assert after != null && before != null;
         return new ImmutableOpenIntMapDiff<>(before, after, keySerializer, valueSerializer);
     }
@@ -106,7 +108,7 @@ public static <K, T extends Diffable<T>> MapDiff<K, T, Map<K, T>> diff(Map<K, T>
     /**
      * Calculates diff between two Maps of non-diffable objects
      */
-    public static <K, T> MapDiff<K, T, Map<K, T>> diff(Map<K, T> before, Map<K, T> after, KeySerializer<K> keySerializer, NonDiffableValueSerializer<K, T> valueSerializer) {
+    public static <K, T> MapDiff<K, T, Map<K, T>> diff(Map<K, T> before, Map<K, T> after, KeySerializer<K> keySerializer, ValueSerializer<K, T> valueSerializer) {
         assert after != null && before != null;
         return new JdkMapDiff<>(before, after, keySerializer, valueSerializer);
     }
@@ -135,22 +137,22 @@ public static <K, T> MapDiff<K, T, Map<K, T>> readJdkMapDiff(StreamInput in, Key
     /**
      * Loads an object that represents difference between two ImmutableOpenMaps of Diffable objects using Diffable proto object
      */
-    public static <K, T extends Diffable<T>> MapDiff<K, T, ImmutableOpenMap<K, T>> readImmutableOpenMapDiff(StreamInput in, KeySerializer<K> keySerializer, T proto) throws IOException {
-        return new ImmutableOpenMapDiff<>(in, keySerializer, new DiffablePrototypeValueReader<>(proto));
+    public static <K, T extends Diffable<T>> MapDiff<K, T, ImmutableOpenMap<K, T>> readImmutableOpenMapDiff(StreamInput in, KeySerializer<K> keySerializer, Reader<T> reader, Reader<Diff<T>> diffReader) throws IOException {
+        return new ImmutableOpenMapDiff<>(in, keySerializer, new DiffableValueReader<>(reader, diffReader));
     }
 
     /**
      * Loads an object that represents difference between two ImmutableOpenIntMaps of Diffable objects using Diffable proto object
      */
-    public static <T extends Diffable<T>> MapDiff<Integer, T, ImmutableOpenIntMap<T>> readImmutableOpenIntMapDiff(StreamInput in, KeySerializer<Integer> keySerializer, T proto) throws IOException {
-        return new ImmutableOpenIntMapDiff<>(in, keySerializer, new DiffablePrototypeValueReader<>(proto));
+    public static <T extends Diffable<T>> MapDiff<Integer, T, ImmutableOpenIntMap<T>> readImmutableOpenIntMapDiff(StreamInput in, KeySerializer<Integer> keySerializer, Reader<T> reader, Reader<Diff<T>> diffReader) throws IOException {
+        return new ImmutableOpenIntMapDiff<>(in, keySerializer, new DiffableValueReader<>(reader, diffReader));
     }
 
     /**
      * Loads an object that represents difference between two Maps of Diffable objects using Diffable proto object
      */
-    public static <K, T extends Diffable<T>> MapDiff<K, T, Map<K, T>> readJdkMapDiff(StreamInput in, KeySerializer<K> keySerializer, T proto) throws IOException {
-        return new JdkMapDiff<>(in, keySerializer, new DiffablePrototypeValueReader<>(proto));
+    public static <K, T extends Diffable<T>> MapDiff<K, T, Map<K, T>> readJdkMapDiff(StreamInput in, KeySerializer<K> keySerializer, Reader<T> reader, Reader<Diff<T>> diffReader) throws IOException {
+        return new JdkMapDiff<>(in, keySerializer, new DiffableValueReader<>(reader, diffReader));
     }
 
     /**
@@ -164,7 +166,7 @@ protected JdkMapDiff(StreamInput in, KeySerializer<K> keySerializer, ValueSerial
             super(in, keySerializer, valueSerializer);
         }
 
-        public JdkMapDiff(Map<K, T> before, Map<K, T> after,
+        JdkMapDiff(Map<K, T> before, Map<K, T> after,
                           KeySerializer<K> keySerializer, ValueSerializer<K, T> valueSerializer) {
             super(keySerializer, valueSerializer);
             assert after != null && before != null;
@@ -191,8 +193,7 @@ public JdkMapDiff(Map<K, T> before, Map<K, T> after,
 
         @Override
         public Map<K, T> apply(Map<K, T> map) {
-            Map<K, T> builder = new HashMap<>();
-            builder.putAll(map);
+            Map<K, T> builder = new HashMap<>(map);
 
             for (K part : deletes) {
                 builder.remove(part);
@@ -214,12 +215,17 @@ public Map<K, T> apply(Map<K, T> map) {
      *
      * @param <T> the object type
      */
-    private static class ImmutableOpenMapDiff<K, T> extends MapDiff<K, T, ImmutableOpenMap<K, T>> {
+    public static class ImmutableOpenMapDiff<K, T> extends MapDiff<K, T, ImmutableOpenMap<K, T>> {
 
         protected ImmutableOpenMapDiff(StreamInput in, KeySerializer<K> keySerializer, ValueSerializer<K, T> valueSerializer) throws IOException {
             super(in, keySerializer, valueSerializer);
         }
 
+        private ImmutableOpenMapDiff(KeySerializer<K> keySerializer, ValueSerializer<K, T> valueSerializer,
+                                     List<K> deletes, Map<K, Diff<T>> diffs, Map<K, T> upserts) {
+            super(keySerializer, valueSerializer, deletes, diffs, upserts);
+        }
+
         public ImmutableOpenMapDiff(ImmutableOpenMap<K, T> before, ImmutableOpenMap<K, T> after,
                                     KeySerializer<K> keySerializer, ValueSerializer<K, T> valueSerializer) {
             super(keySerializer, valueSerializer);
@@ -245,6 +251,21 @@ public ImmutableOpenMapDiff(ImmutableOpenMap<K, T> before, ImmutableOpenMap<K, T
             }
         }
 
+        /**
+         * Returns a new diff map with the given key removed, does not modify the invoking instance.
+         * If the key does not exist in the diff map, the same instance is returned.
+         */
+        public ImmutableOpenMapDiff<K, T> withKeyRemoved(K key) {
+            if (this.diffs.containsKey(key) == false && this.upserts.containsKey(key) == false) {
+                return this;
+            }
+            Map<K, Diff<T>> newDiffs = new HashMap<>(this.diffs);
+            newDiffs.remove(key);
+            Map<K, T> newUpserts = new HashMap<>(this.upserts);
+            newUpserts.remove(key);
+            return new ImmutableOpenMapDiff<>(this.keySerializer, this.valueSerializer, this.deletes, newDiffs, newUpserts);
+        }
+
         @Override
         public ImmutableOpenMap<K, T> apply(ImmutableOpenMap<K, T> map) {
             ImmutableOpenMap.Builder<K, T> builder = ImmutableOpenMap.builder();
@@ -276,7 +297,7 @@ protected ImmutableOpenIntMapDiff(StreamInput in, KeySerializer<Integer> keySeri
             super(in, keySerializer, valueSerializer);
         }
 
-        public ImmutableOpenIntMapDiff(ImmutableOpenIntMap<T> before, ImmutableOpenIntMap<T> after,
+        ImmutableOpenIntMapDiff(ImmutableOpenIntMap<T> before, ImmutableOpenIntMap<T> after,
                                        KeySerializer<Integer> keySerializer, ValueSerializer<Integer, T> valueSerializer) {
             super(keySerializer, valueSerializer);
             assert after != null && before != null;
@@ -346,6 +367,15 @@ protected MapDiff(KeySerializer<K> keySerializer, ValueSerializer<K, T> valueSer
             upserts = new HashMap<>();
         }
 
+        protected MapDiff(KeySerializer<K> keySerializer, ValueSerializer<K, T> valueSerializer,
+                          List<K> deletes, Map<K, Diff<T>> diffs, Map<K, T> upserts) {
+            this.keySerializer = keySerializer;
+            this.valueSerializer = valueSerializer;
+            this.deletes = deletes;
+            this.diffs = diffs;
+            this.upserts = upserts;
+        }
+
         protected MapDiff(StreamInput in, KeySerializer<K> keySerializer, ValueSerializer<K, T> valueSerializer) throws IOException {
             this.keySerializer = keySerializer;
             this.valueSerializer = valueSerializer;
@@ -406,12 +436,29 @@ public void writeTo(StreamOutput out) throws IOException {
             for (K delete : deletes) {
                 keySerializer.writeKey(delete, out);
             }
-            out.writeVInt(diffs.size());
+            Version version = out.getVersion();
+            // filter out custom states not supported by the other node
+            int diffCount = 0;
+            for (Diff<T> diff : diffs.values()) {
+                if(valueSerializer.supportsVersion(diff, version)) {
+                    diffCount++;
+                }
+            }
+            out.writeVInt(diffCount);
             for (Map.Entry<K, Diff<T>> entry : diffs.entrySet()) {
-                keySerializer.writeKey(entry.getKey(), out);
-                valueSerializer.writeDiff(entry.getValue(), out);
+                if(valueSerializer.supportsVersion(entry.getValue(), version)) {
+                    keySerializer.writeKey(entry.getKey(), out);
+                    valueSerializer.writeDiff(entry.getValue(), out);
+                }
+            }
+            // filter out custom states not supported by the other node
+            int upsertsCount = 0;
+            for (T upsert : upserts.values()) {
+                if(valueSerializer.supportsVersion(upsert, version)) {
+                    upsertsCount++;
+                }
             }
-            out.writeVInt(upserts.size());
+            out.writeVInt(upsertsCount);
             for (Map.Entry<K, T> entry : upserts.entrySet()) {
                 keySerializer.writeKey(entry.getKey(), out);
                 valueSerializer.write(entry.getValue(), out);
@@ -511,6 +558,20 @@ public interface ValueSerializer<K, V> {
          */
         boolean supportsDiffableValues();
 
+        /**
+         * Whether this serializer supports the version of the output stream
+         */
+        default boolean supportsVersion(Diff<V> value, Version version) {
+            return true;
+        }
+
+        /**
+         * Whether this serializer supports the version of the output stream
+         */
+        default boolean supportsVersion(V value, Version version) {
+            return true;
+        }
+
         /**
          * Computes diff if this serializer supports diffable values
          */
@@ -600,25 +661,27 @@ public Diff<V> readDiff(StreamInput in, K key) throws IOException {
     }
 
     /**
-     * Implementation of the ValueSerializer that uses a prototype object for reading operations
+     * Implementation of the ValueSerializer that wraps value and diff readers.
      *
      * Note: this implementation is ignoring the key.
      */
-    public static class DiffablePrototypeValueReader<K, V extends Diffable<V>> extends DiffableValueSerializer<K, V> {
-        private final V proto;
+    public static class DiffableValueReader<K, V extends Diffable<V>> extends DiffableValueSerializer<K, V> {
+        private final Reader<V> reader;
+        private final Reader<Diff<V>> diffReader;
 
-        public DiffablePrototypeValueReader(V proto) {
-            this.proto = proto;
+        public DiffableValueReader(Reader<V> reader, Reader<Diff<V>> diffReader) {
+            this.reader = reader;
+            this.diffReader = diffReader;
         }
 
         @Override
         public V read(StreamInput in, K key) throws IOException {
-            return proto.readFrom(in);
+            return reader.read(in);
         }
 
         @Override
         public Diff<V> readDiff(StreamInput in, K key) throws IOException {
-            return proto.readDiffFrom(in);
+            return diffReader.read(in);
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java b/core/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java
index c2cdaf90bc440..b8ac2a5eb50c3 100644
--- a/core/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java
@@ -25,11 +25,10 @@
 import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequest;
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
-import org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.elasticsearch.action.admin.indices.stats.ShardStats;
-import org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
@@ -39,7 +38,6 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
@@ -66,7 +64,8 @@
  * Every time the timer runs, gathers information about the disk usage and
  * shard sizes across the cluster.
  */
-public class InternalClusterInfoService extends AbstractComponent implements ClusterInfoService, LocalNodeMasterListener, ClusterStateListener {
+public class InternalClusterInfoService extends AbstractComponent
+    implements ClusterInfoService, LocalNodeMasterListener, ClusterStateListener {
 
     public static final Setting<TimeValue> INTERNAL_CLUSTER_INFO_UPDATE_INTERVAL_SETTING =
         Setting.timeSetting("cluster.info.update.interval", TimeValue.timeValueSeconds(30), TimeValue.timeValueSeconds(10),
@@ -84,37 +83,32 @@ public class InternalClusterInfoService extends AbstractComponent implements Clu
     private volatile boolean isMaster = false;
     private volatile boolean enabled;
     private volatile TimeValue fetchTimeout;
-    private final TransportNodesStatsAction transportNodesStatsAction;
-    private final TransportIndicesStatsAction transportIndicesStatsAction;
     private final ClusterService clusterService;
     private final ThreadPool threadPool;
+    private final NodeClient client;
     private final List<Listener> listeners = new CopyOnWriteArrayList<>();
 
-    @Inject
-    public InternalClusterInfoService(Settings settings, ClusterSettings clusterSettings,
-                                      TransportNodesStatsAction transportNodesStatsAction,
-                                      TransportIndicesStatsAction transportIndicesStatsAction, ClusterService clusterService,
-                                      ThreadPool threadPool) {
+    public InternalClusterInfoService(Settings settings, ClusterService clusterService, ThreadPool threadPool, NodeClient client) {
         super(settings);
         this.leastAvailableSpaceUsages = ImmutableOpenMap.of();
         this.mostAvailableSpaceUsages = ImmutableOpenMap.of();
         this.shardRoutingToDataPath = ImmutableOpenMap.of();
         this.shardSizes = ImmutableOpenMap.of();
-        this.transportNodesStatsAction = transportNodesStatsAction;
-        this.transportIndicesStatsAction = transportIndicesStatsAction;
         this.clusterService = clusterService;
         this.threadPool = threadPool;
+        this.client = client;
         this.updateFrequency = INTERNAL_CLUSTER_INFO_UPDATE_INTERVAL_SETTING.get(settings);
         this.fetchTimeout = INTERNAL_CLUSTER_INFO_TIMEOUT_SETTING.get(settings);
         this.enabled = DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.get(settings);
+        ClusterSettings clusterSettings = clusterService.getClusterSettings();
         clusterSettings.addSettingsUpdateConsumer(INTERNAL_CLUSTER_INFO_TIMEOUT_SETTING, this::setFetchTimeout);
         clusterSettings.addSettingsUpdateConsumer(INTERNAL_CLUSTER_INFO_UPDATE_INTERVAL_SETTING, this::setUpdateFrequency);
         clusterSettings.addSettingsUpdateConsumer(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING, this::setEnabled);
 
         // Add InternalClusterInfoService to listen for Master changes
-        this.clusterService.add((LocalNodeMasterListener)this);
+        this.clusterService.addLocalNodeMasterListener(this);
         // Add to listen for state changes (when nodes are added)
-        this.clusterService.add((ClusterStateListener)this);
+        this.clusterService.addListener(this);
     }
 
     private void setEnabled(boolean enabled) {
@@ -174,7 +168,7 @@ public void clusterChanged(ClusterChangedEvent event) {
             }
         }
 
-        if (this.isMaster && dataNodeAdded && clusterService.state().getNodes().getDataNodes().size() > 1) {
+        if (this.isMaster && dataNodeAdded && event.state().getNodes().getDataNodes().size() > 1) {
             if (logger.isDebugEnabled()) {
                 logger.debug("data node was added, retrieving new cluster info");
             }
@@ -259,8 +253,7 @@ protected CountDownLatch updateNodeStats(final ActionListener<NodesStatsResponse
         nodesStatsRequest.clear();
         nodesStatsRequest.fs(true);
         nodesStatsRequest.timeout(fetchTimeout);
-
-        transportNodesStatsAction.execute(nodesStatsRequest, new LatchedActionListener<>(listener, latch));
+        client.admin().cluster().nodesStats(nodesStatsRequest, new LatchedActionListener<>(listener, latch));
         return latch;
     }
 
@@ -274,7 +267,7 @@ protected CountDownLatch updateIndicesStats(final ActionListener<IndicesStatsRes
         indicesStatsRequest.clear();
         indicesStatsRequest.store(true);
 
-        transportIndicesStatsAction.execute(indicesStatsRequest, new LatchedActionListener<>(listener, latch));
+        client.admin().indices().stats(indicesStatsRequest, new LatchedActionListener<>(listener, latch));
         return latch;
     }
 
@@ -415,9 +408,9 @@ static void fillDiskUsagePerNode(Logger logger, List<NodeStats> nodeStatsArray,
                     if (leastAvailablePath == null) {
                         assert mostAvailablePath == null;
                         mostAvailablePath = leastAvailablePath = info;
-                    } else if (leastAvailablePath.getAvailable().bytes() > info.getAvailable().bytes()){
+                    } else if (leastAvailablePath.getAvailable().getBytes() > info.getAvailable().getBytes()){
                         leastAvailablePath = info;
-                    } else if (mostAvailablePath.getAvailable().bytes() < info.getAvailable().bytes()) {
+                    } else if (mostAvailablePath.getAvailable().getBytes() < info.getAvailable().getBytes()) {
                         mostAvailablePath = info;
                     }
                 }
@@ -428,21 +421,21 @@ static void fillDiskUsagePerNode(Logger logger, List<NodeStats> nodeStatsArray,
                             nodeId, mostAvailablePath.getTotal(), leastAvailablePath.getAvailable(),
                             leastAvailablePath.getTotal(), leastAvailablePath.getAvailable());
                 }
-                if (leastAvailablePath.getTotal().bytes() < 0) {
+                if (leastAvailablePath.getTotal().getBytes() < 0) {
                     if (logger.isTraceEnabled()) {
                         logger.trace("node: [{}] least available path has less than 0 total bytes of disk [{}], skipping",
-                                nodeId, leastAvailablePath.getTotal().bytes());
+                                nodeId, leastAvailablePath.getTotal().getBytes());
                     }
                 } else {
-                    newLeastAvaiableUsages.put(nodeId, new DiskUsage(nodeId, nodeName, leastAvailablePath.getPath(), leastAvailablePath.getTotal().bytes(), leastAvailablePath.getAvailable().bytes()));
+                    newLeastAvaiableUsages.put(nodeId, new DiskUsage(nodeId, nodeName, leastAvailablePath.getPath(), leastAvailablePath.getTotal().getBytes(), leastAvailablePath.getAvailable().getBytes()));
                 }
-                if (mostAvailablePath.getTotal().bytes() < 0) {
+                if (mostAvailablePath.getTotal().getBytes() < 0) {
                     if (logger.isTraceEnabled()) {
                         logger.trace("node: [{}] most available path has less than 0 total bytes of disk [{}], skipping",
-                                nodeId, mostAvailablePath.getTotal().bytes());
+                                nodeId, mostAvailablePath.getTotal().getBytes());
                     }
                 } else {
-                    newMostAvaiableUsages.put(nodeId, new DiskUsage(nodeId, nodeName, mostAvailablePath.getPath(), mostAvailablePath.getTotal().bytes(), mostAvailablePath.getAvailable().bytes()));
+                    newMostAvaiableUsages.put(nodeId, new DiskUsage(nodeId, nodeName, mostAvailablePath.getPath(), mostAvailablePath.getTotal().getBytes(), mostAvailablePath.getAvailable().getBytes()));
                 }
 
             }
diff --git a/core/src/main/java/org/elasticsearch/cluster/LocalClusterUpdateTask.java b/core/src/main/java/org/elasticsearch/cluster/LocalClusterUpdateTask.java
new file mode 100644
index 0000000000000..9692ff8d4e1ed
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/cluster/LocalClusterUpdateTask.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.cluster;
+
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Priority;
+import org.elasticsearch.common.unit.TimeValue;
+
+import java.util.List;
+
+/**
+ * Used to apply state updates on nodes that are not necessarily master
+ */
+public abstract class LocalClusterUpdateTask implements ClusterStateTaskConfig, ClusterStateTaskExecutor<LocalClusterUpdateTask>,
+    ClusterStateTaskListener {
+
+    private final Priority priority;
+
+    public LocalClusterUpdateTask() {
+        this(Priority.NORMAL);
+    }
+
+    public LocalClusterUpdateTask(Priority priority) {
+        this.priority = priority;
+    }
+
+    public abstract ClusterTasksResult<LocalClusterUpdateTask> execute(ClusterState currentState) throws Exception;
+
+    @Override
+    public final ClusterTasksResult<LocalClusterUpdateTask> execute(ClusterState currentState,
+                                                                    List<LocalClusterUpdateTask> tasks) throws Exception {
+        assert tasks.size() == 1 && tasks.get(0) == this : "expected one-element task list containing current object but was " + tasks;
+        ClusterTasksResult<LocalClusterUpdateTask> result = execute(currentState);
+        return ClusterTasksResult.<LocalClusterUpdateTask>builder().successes(tasks).build(result, currentState);
+    }
+
+    /**
+     * node stepped down as master or has lost connection to the master
+     */
+    public static ClusterTasksResult<LocalClusterUpdateTask> noMaster() {
+        return new ClusterTasksResult(true, null, null);
+    }
+
+    /**
+     * no changes were made to the cluster state. Useful to execute a runnable on the cluster state applier thread
+     */
+    public static ClusterTasksResult<LocalClusterUpdateTask> unchanged() {
+        return new ClusterTasksResult(false, null, null);
+    }
+
+    /**
+     * locally apply cluster state received from a master
+     */
+    public static ClusterTasksResult<LocalClusterUpdateTask> newState(ClusterState clusterState) {
+        return new ClusterTasksResult(false, clusterState, null);
+    }
+
+    @Override
+    public String describeTasks(List<LocalClusterUpdateTask> tasks) {
+        return ""; // one of task, source is enough
+    }
+
+    @Nullable
+    public TimeValue timeout() {
+        return null;
+    }
+
+    @Override
+    public Priority priority() {
+        return priority;
+    }
+
+    @Override
+    public final boolean runOnlyOnMaster() {
+        return false;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/cluster/MasterNodeChangePredicate.java b/core/src/main/java/org/elasticsearch/cluster/MasterNodeChangePredicate.java
index afb557e5bbcd5..5bcfecaebafba 100644
--- a/core/src/main/java/org/elasticsearch/cluster/MasterNodeChangePredicate.java
+++ b/core/src/main/java/org/elasticsearch/cluster/MasterNodeChangePredicate.java
@@ -19,22 +19,35 @@
 
 package org.elasticsearch.cluster;
 
-public enum MasterNodeChangePredicate implements ClusterStateObserver.ChangePredicate {
-    INSTANCE;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+
+import java.util.function.Predicate;
+
+public final class MasterNodeChangePredicate {
+
+    private MasterNodeChangePredicate() {
 
-    @Override
-    public boolean apply(
-        ClusterState previousState,
-        ClusterState.ClusterStateStatus previousStatus,
-        ClusterState newState,
-        ClusterState.ClusterStateStatus newStatus) {
-        // checking if the masterNodeId changed is insufficient as the
-        // same master node might get re-elected after a disruption
-        return newState.nodes().getMasterNodeId() != null && newState != previousState;
     }
 
-    @Override
-    public boolean apply(ClusterChangedEvent changedEvent) {
-        return changedEvent.nodesDelta().masterNodeChanged();
+    /**
+     * builds a predicate that will accept a cluster state only if it was generated after the current has
+     * (re-)joined the master
+     */
+    public static Predicate<ClusterState> build(ClusterState currentState) {
+        final long currentVersion = currentState.version();
+        final DiscoveryNode masterNode = currentState.nodes().getMasterNode();
+        final String currentMasterId = masterNode == null ? null : masterNode.getEphemeralId();
+        return newState -> {
+            final DiscoveryNode newMaster = newState.nodes().getMasterNode();
+            final boolean accept;
+            if (newMaster == null) {
+                accept = false;
+            } else if (newMaster.getEphemeralId().equals(currentMasterId) == false) {
+                accept = true;
+            } else {
+                accept = newState.version() > currentVersion;
+            }
+            return accept;
+        };
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/NamedDiff.java b/core/src/main/java/org/elasticsearch/cluster/NamedDiff.java
new file mode 100644
index 0000000000000..9da3167ae88c5
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/cluster/NamedDiff.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.NamedWriteable;
+
+/**
+ * Diff that also support NamedWriteable interface
+ */
+public interface NamedDiff<T extends Diffable<T>> extends Diff<T>, NamedWriteable {
+    /**
+     * The minimal version of the recipient this custom object can be sent to
+     */
+    default Version getMinimalSupportedVersion() {
+        return Version.CURRENT.minimumCompatibilityVersion();
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/cluster/NamedDiffable.java b/core/src/main/java/org/elasticsearch/cluster/NamedDiffable.java
new file mode 100644
index 0000000000000..0797442209689
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/cluster/NamedDiffable.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.NamedWriteable;
+
+/**
+ * Diff that also support NamedWriteable interface
+ */
+public interface NamedDiffable<T> extends Diffable<T>, NamedWriteable {
+    /**
+     * The minimal version of the recipient this custom object can be sent to
+     */
+    default Version getMinimalSupportedVersion() {
+        return Version.CURRENT.minimumCompatibilityVersion();
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/cluster/NamedDiffableValueSerializer.java b/core/src/main/java/org/elasticsearch/cluster/NamedDiffableValueSerializer.java
new file mode 100644
index 0000000000000..c6434db9e8777
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/cluster/NamedDiffableValueSerializer.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.StreamInput;
+
+import java.io.IOException;
+
+/**
+ * Value Serializer for named diffables
+ */
+public class NamedDiffableValueSerializer<T extends NamedDiffable<T>> extends DiffableUtils.DiffableValueSerializer<String, T> {
+
+    private final Class<T> tClass;
+
+    public NamedDiffableValueSerializer(Class<T> tClass) {
+        this.tClass = tClass;
+    }
+
+    @Override
+    public T read(StreamInput in, String key) throws IOException {
+        return in.readNamedWriteable(tClass, key);
+    }
+
+    @Override
+    public boolean supportsVersion(Diff<T> value, Version version) {
+        return version.onOrAfter(((NamedDiff<?>)value).getMinimalSupportedVersion());
+    }
+
+    @Override
+    public boolean supportsVersion(T value, Version version) {
+        return version.onOrAfter(value.getMinimalSupportedVersion());
+    }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public Diff<T> readDiff(StreamInput in, String key) throws IOException {
+        return in.readNamedWriteable(NamedDiff.class, key);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java b/core/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java
index 99f161b9da5fa..aab75eb2aad7b 100644
--- a/core/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java
@@ -21,6 +21,7 @@
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.lease.Releasable;
@@ -31,10 +32,15 @@
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.common.util.concurrent.FutureUtils;
 import org.elasticsearch.common.util.concurrent.KeyedLock;
+import org.elasticsearch.discovery.zen.MasterFaultDetection;
+import org.elasticsearch.discovery.zen.NodesFaultDetection;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.util.HashSet;
+import java.util.Set;
 import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ScheduledFuture;
 
 import static org.elasticsearch.common.settings.Setting.Property;
@@ -45,8 +51,8 @@
  * This component is responsible for connecting to nodes once they are added to the cluster state, and disconnect when they are
  * removed. Also, it periodically checks that all connections are still open and if needed restores them.
  * Note that this component is *not* responsible for removing nodes from the cluster if they disconnect / do not respond
- * to pings. This is done by {@link org.elasticsearch.discovery.zen.fd.NodesFaultDetection}. Master fault detection
- * is done by {@link org.elasticsearch.discovery.zen.fd.MasterFaultDetection}.
+ * to pings. This is done by {@link NodesFaultDetection}. Master fault detection
+ * is done by {@link MasterFaultDetection}.
  */
 public class NodeConnectionsService extends AbstractLifecycleComponent {
 
@@ -73,20 +79,58 @@ public NodeConnectionsService(Settings settings, ThreadPool threadPool, Transpor
         this.reconnectInterval = NodeConnectionsService.CLUSTER_NODE_RECONNECT_INTERVAL_SETTING.get(settings);
     }
 
-    public void connectToAddedNodes(ClusterChangedEvent event) {
-
-        // TODO: do this in parallel (and wait)
-        for (final DiscoveryNode node : event.nodesDelta().addedNodes()) {
+    public void connectToNodes(DiscoveryNodes discoveryNodes) {
+        CountDownLatch latch = new CountDownLatch(discoveryNodes.getSize());
+        for (final DiscoveryNode node : discoveryNodes) {
+            final boolean connected;
             try (Releasable ignored = nodeLocks.acquire(node)) {
-                Integer current = nodes.put(node, 0);
-                assert current == null : "node " + node + " was added in event but already in internal nodes";
-                validateNodeConnected(node);
+                nodes.putIfAbsent(node, 0);
+                connected = transportService.nodeConnected(node);
+            }
+            if (connected) {
+                latch.countDown();
+            } else {
+                // spawn to another thread to do in parallel
+                threadPool.executor(ThreadPool.Names.MANAGEMENT).execute(new AbstractRunnable() {
+                    @Override
+                    public void onFailure(Exception e) {
+                        // both errors and rejections are logged here. the service
+                        // will try again after `cluster.nodes.reconnect_interval` on all nodes but the current master.
+                        // On the master, node fault detection will remove these nodes from the cluster as their are not
+                        // connected. Note that it is very rare that we end up here on the master.
+                        logger.warn((Supplier<?>) () -> new ParameterizedMessage("failed to connect to {}", node), e);
+                    }
+
+                    @Override
+                    protected void doRun() throws Exception {
+                        try (Releasable ignored = nodeLocks.acquire(node)) {
+                            validateAndConnectIfNeeded(node);
+                        }
+                    }
+
+                    @Override
+                    public void onAfter() {
+                        latch.countDown();
+                    }
+                });
             }
         }
+        try {
+            latch.await();
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+        }
     }
 
-    public void disconnectFromRemovedNodes(ClusterChangedEvent event) {
-        for (final DiscoveryNode node : event.nodesDelta().removedNodes()) {
+    /**
+     * Disconnects from all nodes except the ones provided as parameter
+     */
+    public void disconnectFromNodesExcept(DiscoveryNodes nodesToKeep) {
+        Set<DiscoveryNode> currentNodes = new HashSet<>(nodes.keySet());
+        for (DiscoveryNode node : nodesToKeep) {
+            currentNodes.remove(node);
+        }
+        for (final DiscoveryNode node : currentNodes) {
             try (Releasable ignored = nodeLocks.acquire(node)) {
                 Integer current = nodes.remove(node);
                 assert current != null : "node " + node + " was removed in event but not in internal nodes";
@@ -99,8 +143,8 @@ public void disconnectFromRemovedNodes(ClusterChangedEvent event) {
         }
     }
 
-    void validateNodeConnected(DiscoveryNode node) {
-        assert nodeLocks.isHeldByCurrentThread(node) : "validateNodeConnected must be called under lock";
+    void validateAndConnectIfNeeded(DiscoveryNode node) {
+        assert nodeLocks.isHeldByCurrentThread(node) : "validateAndConnectIfNeeded must be called under lock";
         if (lifecycle.stoppedOrClosed() ||
                 nodes.containsKey(node) == false) { // we double check existence of node since connectToNode might take time...
             // nothing to do
@@ -136,7 +180,7 @@ public void onFailure(Exception e) {
         protected void doRun() {
             for (DiscoveryNode node : nodes.keySet()) {
                 try (Releasable ignored = nodeLocks.acquire(node)) {
-                    validateNodeConnected(node);
+                    validateAndConnectIfNeeded(node);
                 }
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/cluster/RestoreInProgress.java b/core/src/main/java/org/elasticsearch/cluster/RestoreInProgress.java
index 55a09f87f759a..55e70dbe644ff 100644
--- a/core/src/main/java/org/elasticsearch/cluster/RestoreInProgress.java
+++ b/core/src/main/java/org/elasticsearch/cluster/RestoreInProgress.java
@@ -39,12 +39,10 @@
 /**
  * Meta data about restore processes that are currently executing
  */
-public class RestoreInProgress extends AbstractDiffable<Custom> implements Custom {
+public class RestoreInProgress extends AbstractNamedDiffable<Custom> implements Custom {
 
     public static final String TYPE = "restore";
 
-    public static final RestoreInProgress PROTO = new RestoreInProgress();
-
     private final List<Entry> entries;
 
     /**
@@ -377,15 +375,15 @@ public static State fromValue(byte value) {
      * {@inheritDoc}
      */
     @Override
-    public String type() {
+    public String getWriteableName() {
         return TYPE;
     }
 
-    /**
-     * {@inheritDoc}
-     */
-    @Override
-    public RestoreInProgress readFrom(StreamInput in) throws IOException {
+    public static NamedDiff<Custom> readDiffFrom(StreamInput in) throws IOException {
+        return readDiffFrom(Custom.class, TYPE, in);
+    }
+
+    public RestoreInProgress(StreamInput in) throws IOException {
         Entry[] entries = new Entry[in.readVInt()];
         for (int i = 0; i < entries.length; i++) {
             Snapshot snapshot = new Snapshot(in);
@@ -404,7 +402,7 @@ public RestoreInProgress readFrom(StreamInput in) throws IOException {
             }
             entries[i] = new Entry(snapshot, state, Collections.unmodifiableList(indexBuilder), builder.build());
         }
-        return new RestoreInProgress(entries);
+        this.entries = Arrays.asList(entries);
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/cluster/SnapshotDeletionsInProgress.java b/core/src/main/java/org/elasticsearch/cluster/SnapshotDeletionsInProgress.java
new file mode 100644
index 0000000000000..9641c95a827a1
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/cluster/SnapshotDeletionsInProgress.java
@@ -0,0 +1,220 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterState.Custom;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.snapshots.Snapshot;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * A class that represents the snapshot deletions that are in progress in the cluster.
+ */
+public class SnapshotDeletionsInProgress extends AbstractNamedDiffable<Custom> implements Custom {
+
+    public static final String TYPE = "snapshot_deletions";
+    // the version where SnapshotDeletionsInProgress was introduced
+    public static final Version VERSION_INTRODUCED = Version.V_5_2_0;
+
+    // the list of snapshot deletion request entries
+    private final List<Entry> entries;
+
+    private SnapshotDeletionsInProgress(List<Entry> entries) {
+        this.entries = Collections.unmodifiableList(entries);
+    }
+
+    public SnapshotDeletionsInProgress(StreamInput in) throws IOException {
+        this.entries = Collections.unmodifiableList(in.readList(Entry::new));
+    }
+
+    /**
+     * Returns a new instance of {@link SnapshotDeletionsInProgress} with the given
+     * {@link Entry} added.
+     */
+    public static SnapshotDeletionsInProgress newInstance(Entry entry) {
+        return new SnapshotDeletionsInProgress(Collections.singletonList(entry));
+    }
+
+    /**
+     * Returns a new instance of {@link SnapshotDeletionsInProgress} which adds
+     * the given {@link Entry} to the invoking instance.
+     */
+    public SnapshotDeletionsInProgress withAddedEntry(Entry entry) {
+        List<Entry> entries = new ArrayList<>(getEntries());
+        entries.add(entry);
+        return new SnapshotDeletionsInProgress(entries);
+    }
+
+    /**
+     * Returns a new instance of {@link SnapshotDeletionsInProgress} which removes
+     * the given entry from the invoking instance.
+     */
+    public SnapshotDeletionsInProgress withRemovedEntry(Entry entry) {
+        List<Entry> entries = new ArrayList<>(getEntries());
+        entries.remove(entry);
+        return new SnapshotDeletionsInProgress(entries);
+    }
+
+    /**
+     * Returns an unmodifiable list of snapshot deletion entries.
+     */
+    public List<Entry> getEntries() {
+        return entries;
+    }
+
+    /**
+     * Returns {@code true} if there are snapshot deletions in progress in the cluster,
+     * returns {@code false} otherwise.
+     */
+    public boolean hasDeletionsInProgress() {
+        return entries.isEmpty() == false;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return TYPE;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+
+        SnapshotDeletionsInProgress that = (SnapshotDeletionsInProgress) o;
+        return entries.equals(that.entries);
+    }
+
+    @Override
+    public int hashCode() {
+        return 31 + entries.hashCode();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeList(entries);
+    }
+
+    public static NamedDiff<Custom> readDiffFrom(StreamInput in) throws IOException {
+        return readDiffFrom(Custom.class, TYPE, in);
+    }
+
+    @Override
+    public Version getMinimalSupportedVersion() {
+        return VERSION_INTRODUCED;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startArray(TYPE);
+        for (Entry entry : entries) {
+            builder.startObject();
+            {
+                builder.field("repository", entry.snapshot.getRepository());
+                builder.field("snapshot", entry.snapshot.getSnapshotId().getName());
+                builder.timeValueField("start_time_millis", "start_time", entry.startTime);
+                builder.field("repository_state_id", entry.repositoryStateId);
+            }
+            builder.endObject();
+        }
+        builder.endArray();
+        return builder;
+    }
+
+    /**
+     * A class representing a snapshot deletion request entry in the cluster state.
+     */
+    public static final class Entry implements Writeable {
+        private final Snapshot snapshot;
+        private final long startTime;
+        private final long repositoryStateId;
+
+        public Entry(Snapshot snapshot, long startTime, long repositoryStateId) {
+            this.snapshot = snapshot;
+            this.startTime = startTime;
+            this.repositoryStateId = repositoryStateId;
+        }
+
+        public Entry(StreamInput in) throws IOException {
+            this.snapshot = new Snapshot(in);
+            this.startTime = in.readVLong();
+            this.repositoryStateId = in.readLong();
+        }
+
+        /**
+         * The snapshot to delete.
+         */
+        public Snapshot getSnapshot() {
+            return snapshot;
+        }
+
+        /**
+         * The start time in milliseconds for deleting the snapshots.
+         */
+        public long getStartTime() {
+            return startTime;
+        }
+
+        /**
+         * The repository state id at the time the snapshot deletion began.
+         */
+        public long getRepositoryStateId() {
+            return repositoryStateId;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) {
+                return true;
+            }
+            if (o == null || getClass() != o.getClass()) {
+                return false;
+            }
+            Entry that = (Entry) o;
+            return snapshot.equals(that.snapshot)
+                       && startTime == that.startTime
+                       && repositoryStateId == that.repositoryStateId;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(snapshot, startTime, repositoryStateId);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            snapshot.writeTo(out);
+            out.writeVLong(startTime);
+            out.writeLong(repositoryStateId);
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java b/core/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java
index 7088037353096..6e085fe7ebecc 100644
--- a/core/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java
+++ b/core/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java
@@ -22,6 +22,7 @@
 import com.carrotsearch.hppc.ObjectContainer;
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
+import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState.Custom;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -43,10 +44,14 @@
 /**
  * Meta data about snapshots that are currently executing
  */
-public class SnapshotsInProgress extends AbstractDiffable<Custom> implements Custom {
+public class SnapshotsInProgress extends AbstractNamedDiffable<Custom> implements Custom {
     public static final String TYPE = "snapshots";
 
-    public static final SnapshotsInProgress PROTO = new SnapshotsInProgress();
+    // denotes an undefined repository state id, which will happen when receiving a cluster state with
+    // a snapshot in progress from a pre 5.2.x node
+    public static final long UNDEFINED_REPOSITORY_STATE_ID = -2L;
+    // the version where repository state ids were introduced
+    private static final Version REPOSITORY_ID_INTRODUCED_VERSION = Version.V_5_2_0;
 
     @Override
     public boolean equals(Object o) {
@@ -74,9 +79,10 @@ public static class Entry {
         private final List<IndexId> indices;
         private final ImmutableOpenMap<String, List<ShardId>> waitingIndices;
         private final long startTime;
+        private final long repositoryStateId;
 
         public Entry(Snapshot snapshot, boolean includeGlobalState, boolean partial, State state, List<IndexId> indices,
-                     long startTime, ImmutableOpenMap<ShardId, ShardSnapshotStatus> shards) {
+                     long startTime, long repositoryStateId, ImmutableOpenMap<ShardId, ShardSnapshotStatus> shards) {
             this.state = state;
             this.snapshot = snapshot;
             this.includeGlobalState = includeGlobalState;
@@ -90,10 +96,12 @@ public Entry(Snapshot snapshot, boolean includeGlobalState, boolean partial, Sta
                 this.shards = shards;
                 this.waitingIndices = findWaitingIndices(shards);
             }
+            this.repositoryStateId = repositoryStateId;
         }
 
         public Entry(Entry entry, State state, ImmutableOpenMap<ShardId, ShardSnapshotStatus> shards) {
-            this(entry.snapshot, entry.includeGlobalState, entry.partial, state, entry.indices, entry.startTime, shards);
+            this(entry.snapshot, entry.includeGlobalState, entry.partial, state, entry.indices, entry.startTime,
+                 entry.repositoryStateId, shards);
         }
 
         public Entry(Entry entry, ImmutableOpenMap<ShardId, ShardSnapshotStatus> shards) {
@@ -132,6 +140,10 @@ public long startTime() {
             return startTime;
         }
 
+        public long getRepositoryStateId() {
+            return repositoryStateId;
+        }
+
         @Override
         public boolean equals(Object o) {
             if (this == o) return true;
@@ -147,6 +159,7 @@ public boolean equals(Object o) {
             if (!snapshot.equals(entry.snapshot)) return false;
             if (state != entry.state) return false;
             if (!waitingIndices.equals(entry.waitingIndices)) return false;
+            if (repositoryStateId != entry.repositoryStateId) return false;
 
             return true;
         }
@@ -161,6 +174,7 @@ public int hashCode() {
             result = 31 * result + indices.hashCode();
             result = 31 * result + waitingIndices.hashCode();
             result = 31 * result + Long.hashCode(startTime);
+            result = 31 * result + Long.hashCode(repositoryStateId);
             return result;
         }
 
@@ -169,14 +183,16 @@ public String toString() {
             return snapshot.toString();
         }
 
-        private ImmutableOpenMap<String, List<ShardId>> findWaitingIndices(ImmutableOpenMap<ShardId, ShardSnapshotStatus> shards) {
+        // package private for testing
+        ImmutableOpenMap<String, List<ShardId>> findWaitingIndices(ImmutableOpenMap<ShardId, ShardSnapshotStatus> shards) {
             Map<String, List<ShardId>> waitingIndicesMap = new HashMap<>();
             for (ObjectObjectCursor<ShardId, ShardSnapshotStatus> entry : shards) {
                 if (entry.value.state() == State.WAITING) {
-                    List<ShardId> waitingShards = waitingIndicesMap.get(entry.key.getIndex());
+                    final String indexName = entry.key.getIndexName();
+                    List<ShardId> waitingShards = waitingIndicesMap.get(indexName);
                     if (waitingShards == null) {
                         waitingShards = new ArrayList<>();
-                        waitingIndicesMap.put(entry.key.getIndexName(), waitingShards);
+                        waitingIndicesMap.put(indexName, waitingShards);
                     }
                     waitingShards.add(entry.key);
                 }
@@ -190,7 +206,6 @@ private ImmutableOpenMap<String, List<ShardId>> findWaitingIndices(ImmutableOpen
             }
             return waitingIndicesBuilder.build();
         }
-
     }
 
     /**
@@ -210,12 +225,9 @@ public static boolean completed(ObjectContainer<ShardSnapshotStatus> shards) {
 
 
     public static class ShardSnapshotStatus {
-        private State state;
-        private String nodeId;
-        private String reason;
-
-        private ShardSnapshotStatus() {
-        }
+        private final State state;
+        private final String nodeId;
+        private final String reason;
 
         public ShardSnapshotStatus(String nodeId) {
             this(nodeId, State.INIT);
@@ -229,6 +241,14 @@ public ShardSnapshotStatus(String nodeId, State state, String reason) {
             this.nodeId = nodeId;
             this.state = state;
             this.reason = reason;
+            // If the state is failed we have to have a reason for this failure
+            assert state.failed() == false || reason != null;
+        }
+
+        public ShardSnapshotStatus(StreamInput in) throws IOException {
+            nodeId = in.readOptionalString();
+            state = State.fromValue(in.readByte());
+            reason = in.readOptionalString();
         }
 
         public State state() {
@@ -243,18 +263,6 @@ public String reason() {
             return reason;
         }
 
-        public static ShardSnapshotStatus readShardSnapshotStatus(StreamInput in) throws IOException {
-            ShardSnapshotStatus shardSnapshotStatus = new ShardSnapshotStatus();
-            shardSnapshotStatus.readFrom(in);
-            return shardSnapshotStatus;
-        }
-
-        public void readFrom(StreamInput in) throws IOException {
-            nodeId = in.readOptionalString();
-            state = State.fromValue(in.readByte());
-            reason = in.readOptionalString();
-        }
-
         public void writeTo(StreamOutput out) throws IOException {
             out.writeOptionalString(nodeId);
             out.writeByte(state.value);
@@ -282,6 +290,11 @@ public int hashCode() {
             result = 31 * result + (reason != null ? reason.hashCode() : 0);
             return result;
         }
+
+        @Override
+        public String toString() {
+            return "ShardSnapshotStatus[state=" + state + ", nodeId=" + nodeId + ", reason=" + reason + "]";
+        }
     }
 
     public enum State {
@@ -365,12 +378,15 @@ public Entry snapshot(final Snapshot snapshot) {
     }
 
     @Override
-    public String type() {
+    public String getWriteableName() {
         return TYPE;
     }
 
-    @Override
-    public SnapshotsInProgress readFrom(StreamInput in) throws IOException {
+    public static NamedDiff<Custom> readDiffFrom(StreamInput in) throws IOException {
+        return readDiffFrom(Custom.class, TYPE, in);
+    }
+
+    public SnapshotsInProgress(StreamInput in) throws IOException {
         Entry[] entries = new Entry[in.readVInt()];
         for (int i = 0; i < entries.length; i++) {
             Snapshot snapshot = new Snapshot(in);
@@ -389,7 +405,13 @@ public SnapshotsInProgress readFrom(StreamInput in) throws IOException {
                 ShardId shardId = ShardId.readShardId(in);
                 String nodeId = in.readOptionalString();
                 State shardState = State.fromValue(in.readByte());
-                builder.put(shardId, new ShardSnapshotStatus(nodeId, shardState));
+                // Workaround for https://github.com/elastic/elasticsearch/issues/25878
+                String reason = shardState.failed() ? "" : null;
+                builder.put(shardId, new ShardSnapshotStatus(nodeId, shardState, reason));
+            }
+            long repositoryStateId = UNDEFINED_REPOSITORY_STATE_ID;
+            if (in.getVersion().onOrAfter(REPOSITORY_ID_INTRODUCED_VERSION)) {
+                repositoryStateId = in.readLong();
             }
             entries[i] = new Entry(snapshot,
                                    includeGlobalState,
@@ -397,9 +419,10 @@ public SnapshotsInProgress readFrom(StreamInput in) throws IOException {
                                    state,
                                    Collections.unmodifiableList(indexBuilder),
                                    startTime,
+                                   repositoryStateId,
                                    builder.build());
         }
-        return new SnapshotsInProgress(entries);
+        this.entries = Arrays.asList(entries);
     }
 
     @Override
@@ -421,6 +444,9 @@ public void writeTo(StreamOutput out) throws IOException {
                 out.writeOptionalString(shardEntry.value.nodeId());
                 out.writeByte(shardEntry.value.state().value());
             }
+            if (out.getVersion().onOrAfter(REPOSITORY_ID_INTRODUCED_VERSION)) {
+                out.writeLong(entry.repositoryStateId);
+            }
         }
     }
 
@@ -434,6 +460,7 @@ public void writeTo(StreamOutput out) throws IOException {
     private static final String INDICES = "indices";
     private static final String START_TIME_MILLIS = "start_time_millis";
     private static final String START_TIME = "start_time";
+    private static final String REPOSITORY_STATE_ID = "repository_state_id";
     private static final String SHARDS = "shards";
     private static final String INDEX = "index";
     private static final String SHARD = "shard";
@@ -465,6 +492,7 @@ public void toXContent(Entry entry, XContentBuilder builder, ToXContent.Params p
         }
         builder.endArray();
         builder.timeValueField(START_TIME_MILLIS, START_TIME, entry.startTime());
+        builder.field(REPOSITORY_STATE_ID, entry.getRepositoryStateId());
         builder.startArray(SHARDS);
         {
             for (ObjectObjectCursor<ShardId, ShardSnapshotStatus> shardEntry : entry.shards) {
diff --git a/core/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java b/core/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java
index 13aa148f8b19f..543118a172f95 100644
--- a/core/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java
+++ b/core/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java
@@ -25,10 +25,10 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterStateTaskExecutor;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateObserver;
 import org.elasticsearch.cluster.ClusterStateTaskConfig;
-import org.elasticsearch.cluster.ClusterStateTaskExecutor;
 import org.elasticsearch.cluster.ClusterStateTaskListener;
 import org.elasticsearch.cluster.MasterNodeChangePredicate;
 import org.elasticsearch.cluster.NotMasterException;
@@ -37,8 +37,8 @@
 import org.elasticsearch.cluster.routing.RoutingService;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
-import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation;
-import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
+import org.elasticsearch.cluster.routing.allocation.FailedShard;
+import org.elasticsearch.cluster.routing.allocation.StaleShard;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Priority;
@@ -69,6 +69,7 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Set;
+import java.util.function.Predicate;
 
 public class ShardStateAction extends AbstractComponent {
 
@@ -91,11 +92,13 @@ public ShardStateAction(Settings settings, ClusterService clusterService, Transp
         transportService.registerRequestHandler(SHARD_FAILED_ACTION_NAME, ShardEntry::new, ThreadPool.Names.SAME, new ShardFailedTransportHandler(clusterService, new ShardFailedClusterStateTaskExecutor(allocationService, routingService, logger), logger));
     }
 
-    private void sendShardAction(final String actionName, final ClusterStateObserver observer, final ShardEntry shardEntry, final Listener listener) {
-        DiscoveryNode masterNode = observer.observedState().nodes().getMasterNode();
+    private void sendShardAction(final String actionName, final ClusterState currentState, final ShardEntry shardEntry, final Listener listener) {
+        ClusterStateObserver observer = new ClusterStateObserver(currentState, clusterService, null, logger, threadPool.getThreadContext());
+        DiscoveryNode masterNode = currentState.nodes().getMasterNode();
+        Predicate<ClusterState> changePredicate = MasterNodeChangePredicate.build(currentState);
         if (masterNode == null) {
             logger.warn("{} no master known for action [{}] for shard entry [{}]", shardEntry.shardId, actionName, shardEntry);
-            waitForNewMasterAndRetry(actionName, observer, shardEntry, listener);
+            waitForNewMasterAndRetry(actionName, observer, shardEntry, listener, changePredicate);
         } else {
             logger.debug("{} sending [{}] to [{}] for shard entry [{}]", shardEntry.shardId, actionName, masterNode.getId(), shardEntry);
             transportService.sendRequest(masterNode,
@@ -108,7 +111,7 @@ public void handleResponse(TransportResponse.Empty response) {
                     @Override
                     public void handleException(TransportException exp) {
                         if (isMasterChannelException(exp)) {
-                            waitForNewMasterAndRetry(actionName, observer, shardEntry, listener);
+                            waitForNewMasterAndRetry(actionName, observer, shardEntry, listener, changePredicate);
                         } else {
                             logger.warn((Supplier<?>) () -> new ParameterizedMessage("{} unexpected failure while sending request [{}] to [{}] for shard entry [{}]", shardEntry.shardId, actionName, masterNode, shardEntry), exp);
                             listener.onFailure(exp instanceof RemoteTransportException ? (Exception) (exp.getCause() instanceof Exception ? exp.getCause() : new ElasticsearchException(exp.getCause())) : exp);
@@ -142,31 +145,39 @@ private static boolean isMasterChannelException(TransportException exp) {
      */
     public void remoteShardFailed(final ShardId shardId, String allocationId, long primaryTerm, final String message, @Nullable final Exception failure, Listener listener) {
         assert primaryTerm > 0L : "primary term should be strictly positive";
-        shardFailed(shardId, allocationId, primaryTerm, message, failure, listener);
+        shardFailed(shardId, allocationId, primaryTerm, message, failure, listener, clusterService.state());
     }
 
     /**
      * Send a shard failed request to the master node to update the cluster state when a shard on the local node failed.
      */
     public void localShardFailed(final ShardRouting shardRouting, final String message, @Nullable final Exception failure, Listener listener) {
-        shardFailed(shardRouting.shardId(), shardRouting.allocationId().getId(), 0L, message, failure, listener);
+        localShardFailed(shardRouting, message, failure, listener, clusterService.state());
+    }
+
+    /**
+     * Send a shard failed request to the master node to update the cluster state when a shard on the local node failed.
+     */
+    public void localShardFailed(final ShardRouting shardRouting, final String message, @Nullable final Exception failure, Listener listener,
+                                 final ClusterState currentState) {
+        shardFailed(shardRouting.shardId(), shardRouting.allocationId().getId(), 0L, message, failure, listener, currentState);
     }
 
-    private void shardFailed(final ShardId shardId, String allocationId, long primaryTerm, final String message, @Nullable final Exception failure, Listener listener) {
-        ClusterStateObserver observer = new ClusterStateObserver(clusterService, null, logger, threadPool.getThreadContext());
+    private void shardFailed(final ShardId shardId, String allocationId, long primaryTerm, final String message,
+                             @Nullable final Exception failure, Listener listener, ClusterState currentState) {
         ShardEntry shardEntry = new ShardEntry(shardId, allocationId, primaryTerm, message, failure);
-        sendShardAction(SHARD_FAILED_ACTION_NAME, observer, shardEntry, listener);
+        sendShardAction(SHARD_FAILED_ACTION_NAME, currentState, shardEntry, listener);
     }
 
     // visible for testing
-    protected void waitForNewMasterAndRetry(String actionName, ClusterStateObserver observer, ShardEntry shardEntry, Listener listener) {
+    protected void waitForNewMasterAndRetry(String actionName, ClusterStateObserver observer, ShardEntry shardEntry, Listener listener, Predicate<ClusterState> changePredicate) {
         observer.waitForNextChange(new ClusterStateObserver.Listener() {
             @Override
             public void onNewClusterState(ClusterState state) {
                 if (logger.isTraceEnabled()) {
-                    logger.trace("new cluster state [{}] after waiting for master election to fail shard entry [{}]", state.prettyPrint(), shardEntry);
+                    logger.trace("new cluster state [{}] after waiting for master election to fail shard entry [{}]", state, shardEntry);
                 }
-                sendShardAction(actionName, observer, shardEntry, listener);
+                sendShardAction(actionName, state, shardEntry, listener);
             }
 
             @Override
@@ -180,7 +191,7 @@ public void onTimeout(TimeValue timeout) {
                 // we wait indefinitely for a new master
                 assert false;
             }
-        }, MasterNodeChangePredicate.INSTANCE);
+        }, changePredicate);
     }
 
     private static class ShardFailedTransportHandler implements TransportRequestHandler<ShardEntry> {
@@ -188,7 +199,7 @@ private static class ShardFailedTransportHandler implements TransportRequestHand
         private final ShardFailedClusterStateTaskExecutor shardFailedClusterStateTaskExecutor;
         private final Logger logger;
 
-        public ShardFailedTransportHandler(ClusterService clusterService, ShardFailedClusterStateTaskExecutor shardFailedClusterStateTaskExecutor, Logger logger) {
+        ShardFailedTransportHandler(ClusterService clusterService, ShardFailedClusterStateTaskExecutor shardFailedClusterStateTaskExecutor, Logger logger) {
             this.clusterService = clusterService;
             this.shardFailedClusterStateTaskExecutor = shardFailedClusterStateTaskExecutor;
             this.logger = logger;
@@ -249,11 +260,11 @@ public ShardFailedClusterStateTaskExecutor(AllocationService allocationService,
         }
 
         @Override
-        public BatchResult<ShardEntry> execute(ClusterState currentState, List<ShardEntry> tasks) throws Exception {
-            BatchResult.Builder<ShardEntry> batchResultBuilder = BatchResult.builder();
+        public ClusterTasksResult<ShardEntry> execute(ClusterState currentState, List<ShardEntry> tasks) throws Exception {
+            ClusterTasksResult.Builder<ShardEntry> batchResultBuilder = ClusterTasksResult.builder();
             List<ShardEntry> tasksToBeApplied = new ArrayList<>();
-            List<FailedRerouteAllocation.FailedShard> shardRoutingsToBeApplied = new ArrayList<>();
-            List<FailedRerouteAllocation.StaleShard> staleShardsToBeApplied = new ArrayList<>();
+            List<FailedShard> failedShardsToBeApplied = new ArrayList<>();
+            List<StaleShard> staleShardsToBeApplied = new ArrayList<>();
 
             for (ShardEntry task : tasks) {
                 IndexMetaData indexMetaData = currentState.metaData().index(task.shardId.getIndex());
@@ -293,7 +304,7 @@ public BatchResult<ShardEntry> execute(ClusterState currentState, List<ShardEntr
                         if (task.primaryTerm > 0 && inSyncAllocationIds.contains(task.allocationId)) {
                             logger.debug("{} marking shard {} as stale (shard failed task: [{}])", task.shardId, task.allocationId, task);
                             tasksToBeApplied.add(task);
-                            staleShardsToBeApplied.add(new FailedRerouteAllocation.StaleShard(task.shardId, task.allocationId));
+                            staleShardsToBeApplied.add(new StaleShard(task.shardId, task.allocationId));
                         } else {
                             // tasks that correspond to non-existent shards are marked as successful
                             logger.debug("{} ignoring shard failed task [{}] (shard does not exist anymore)", task.shardId, task);
@@ -303,21 +314,18 @@ public BatchResult<ShardEntry> execute(ClusterState currentState, List<ShardEntr
                         // failing a shard also possibly marks it as stale (see IndexMetaDataUpdater)
                         logger.debug("{} failing shard {} (shard failed task: [{}])", task.shardId, matched, task);
                         tasksToBeApplied.add(task);
-                        shardRoutingsToBeApplied.add(new FailedRerouteAllocation.FailedShard(matched, task.message, task.failure));
+                        failedShardsToBeApplied.add(new FailedShard(matched, task.message, task.failure));
                     }
                 }
             }
-            assert tasksToBeApplied.size() == shardRoutingsToBeApplied.size() + staleShardsToBeApplied.size();
+            assert tasksToBeApplied.size() == failedShardsToBeApplied.size() + staleShardsToBeApplied.size();
 
             ClusterState maybeUpdatedState = currentState;
             try {
-                RoutingAllocation.Result result = applyFailedShards(currentState, shardRoutingsToBeApplied, staleShardsToBeApplied);
-                if (result.changed()) {
-                    maybeUpdatedState = ClusterState.builder(currentState).routingResult(result).build();
-                }
+                maybeUpdatedState = applyFailedShards(currentState, failedShardsToBeApplied, staleShardsToBeApplied);
                 batchResultBuilder.successes(tasksToBeApplied);
             } catch (Exception e) {
-                logger.warn((Supplier<?>) () -> new ParameterizedMessage("failed to apply failed shards {}", shardRoutingsToBeApplied), e);
+                logger.warn((Supplier<?>) () -> new ParameterizedMessage("failed to apply failed shards {}", failedShardsToBeApplied), e);
                 // failures are communicated back to the requester
                 // cluster state will not be updated in this case
                 batchResultBuilder.failures(tasksToBeApplied, e);
@@ -327,8 +335,7 @@ public BatchResult<ShardEntry> execute(ClusterState currentState, List<ShardEntr
         }
 
         // visible for testing
-        RoutingAllocation.Result applyFailedShards(ClusterState currentState, List<FailedRerouteAllocation.FailedShard> failedShards,
-                                                   List<FailedRerouteAllocation.StaleShard> staleShards) {
+        ClusterState applyFailedShards(ClusterState currentState, List<FailedShard> failedShards, List<StaleShard> staleShards) {
             return allocationService.applyFailedShards(currentState, failedShards, staleShards);
         }
 
@@ -346,9 +353,11 @@ public void clusterStatePublished(ClusterChangedEvent clusterChangedEvent) {
     }
 
     public void shardStarted(final ShardRouting shardRouting, final String message, Listener listener) {
-        ClusterStateObserver observer = new ClusterStateObserver(clusterService, null, logger, threadPool.getThreadContext());
+        shardStarted(shardRouting, message, listener, clusterService.state());
+    }
+    public void shardStarted(final ShardRouting shardRouting, final String message, Listener listener, ClusterState currentState) {
         ShardEntry shardEntry = new ShardEntry(shardRouting.shardId(), shardRouting.allocationId().getId(), 0L, message, null);
-        sendShardAction(SHARD_STARTED_ACTION_NAME, observer, shardEntry, listener);
+        sendShardAction(SHARD_STARTED_ACTION_NAME, currentState, shardEntry, listener);
     }
 
     private static class ShardStartedTransportHandler implements TransportRequestHandler<ShardEntry> {
@@ -356,7 +365,7 @@ private static class ShardStartedTransportHandler implements TransportRequestHan
         private final ShardStartedClusterStateTaskExecutor shardStartedClusterStateTaskExecutor;
         private final Logger logger;
 
-        public ShardStartedTransportHandler(ClusterService clusterService, ShardStartedClusterStateTaskExecutor shardStartedClusterStateTaskExecutor, Logger logger) {
+        ShardStartedTransportHandler(ClusterService clusterService, ShardStartedClusterStateTaskExecutor shardStartedClusterStateTaskExecutor, Logger logger) {
             this.clusterService = clusterService;
             this.shardStartedClusterStateTaskExecutor = shardStartedClusterStateTaskExecutor;
             this.logger = logger;
@@ -366,7 +375,7 @@ public ShardStartedTransportHandler(ClusterService clusterService, ShardStartedC
         public void messageReceived(ShardEntry request, TransportChannel channel) throws Exception {
             logger.debug("{} received shard started for [{}]", request.shardId, request);
             clusterService.submitStateUpdateTask(
-                "shard-started",
+                "shard-started " + request,
                 request,
                 ClusterStateTaskConfig.build(Priority.URGENT),
                 shardStartedClusterStateTaskExecutor,
@@ -385,8 +394,8 @@ public ShardStartedClusterStateTaskExecutor(AllocationService allocationService,
         }
 
         @Override
-        public BatchResult<ShardEntry> execute(ClusterState currentState, List<ShardEntry> tasks) throws Exception {
-            BatchResult.Builder<ShardEntry> builder = BatchResult.builder();
+        public ClusterTasksResult<ShardEntry> execute(ClusterState currentState, List<ShardEntry> tasks) throws Exception {
+            ClusterTasksResult.Builder<ShardEntry> builder = ClusterTasksResult.builder();
             List<ShardEntry> tasksToBeApplied = new ArrayList<>();
             List<ShardRouting> shardRoutingsToBeApplied = new ArrayList<>(tasks.size());
             Set<ShardRouting> seenShardRoutings = new HashSet<>(); // to prevent duplicates
@@ -426,11 +435,7 @@ public BatchResult<ShardEntry> execute(ClusterState currentState, List<ShardEntr
 
             ClusterState maybeUpdatedState = currentState;
             try {
-                RoutingAllocation.Result result =
-                    allocationService.applyStartedShards(currentState, shardRoutingsToBeApplied, true);
-                if (result.changed()) {
-                    maybeUpdatedState = ClusterState.builder(currentState).routingResult(result).build();
-                }
+                maybeUpdatedState = allocationService.applyStartedShards(currentState, shardRoutingsToBeApplied);
                 builder.successes(tasksToBeApplied);
             } catch (Exception e) {
                 logger.warn((Supplier<?>) () -> new ParameterizedMessage("failed to apply started shards {}", shardRoutingsToBeApplied), e);
diff --git a/core/src/main/java/org/elasticsearch/cluster/block/ClusterBlock.java b/core/src/main/java/org/elasticsearch/cluster/block/ClusterBlock.java
index 5a7f8f7c0a934..9f4dfde7022f8 100644
--- a/core/src/main/java/org/elasticsearch/cluster/block/ClusterBlock.java
+++ b/core/src/main/java/org/elasticsearch/cluster/block/ClusterBlock.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.cluster.block;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
@@ -46,18 +47,22 @@ public class ClusterBlock implements Streamable, ToXContent {
 
     private boolean disableStatePersistence = false;
 
+    private boolean allowReleaseResources;
+
     private RestStatus status;
 
     ClusterBlock() {
     }
 
-    public ClusterBlock(int id, String description, boolean retryable, boolean disableStatePersistence, RestStatus status, EnumSet<ClusterBlockLevel> levels) {
+    public ClusterBlock(int id, String description, boolean retryable, boolean disableStatePersistence, boolean allowReleaseResources, RestStatus status,
+                        EnumSet<ClusterBlockLevel> levels) {
         this.id = id;
         this.description = description;
         this.retryable = retryable;
         this.disableStatePersistence = disableStatePersistence;
         this.status = status;
         this.levels = levels;
+        this.allowReleaseResources = allowReleaseResources;
     }
 
     public int id() {
@@ -128,14 +133,19 @@ public void readFrom(StreamInput in) throws IOException {
         id = in.readVInt();
         description = in.readString();
         final int len = in.readVInt();
-        ArrayList<ClusterBlockLevel> levels = new ArrayList<>();
+        ArrayList<ClusterBlockLevel> levels = new ArrayList<>(len);
         for (int i = 0; i < len; i++) {
-            levels.add(ClusterBlockLevel.fromId(in.readVInt()));
+            levels.add(in.readEnum(ClusterBlockLevel.class));
         }
         this.levels = EnumSet.copyOf(levels);
         retryable = in.readBoolean();
         disableStatePersistence = in.readBoolean();
         status = RestStatus.readFrom(in);
+        if (in.getVersion().onOrAfter(Version.V_5_5_0)) {
+            allowReleaseResources = in.readBoolean();
+        } else {
+            allowReleaseResources = false;
+        }
     }
 
     @Override
@@ -144,11 +154,14 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeString(description);
         out.writeVInt(levels.size());
         for (ClusterBlockLevel level : levels) {
-            out.writeVInt(level.id());
+            out.writeEnum(level);
         }
         out.writeBoolean(retryable);
         out.writeBoolean(disableStatePersistence);
         RestStatus.writeTo(out, status);
+        if (out.getVersion().onOrAfter(Version.V_5_5_0)) {
+            out.writeBoolean(allowReleaseResources);
+        }
     }
 
     @Override
@@ -179,4 +192,8 @@ public boolean equals(Object o) {
     public int hashCode() {
         return id;
     }
+
+    public boolean isAllowReleaseResources() {
+        return allowReleaseResources;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/block/ClusterBlockLevel.java b/core/src/main/java/org/elasticsearch/cluster/block/ClusterBlockLevel.java
index 45ff1d3707bcf..127c5eef744c6 100644
--- a/core/src/main/java/org/elasticsearch/cluster/block/ClusterBlockLevel.java
+++ b/core/src/main/java/org/elasticsearch/cluster/block/ClusterBlockLevel.java
@@ -26,34 +26,11 @@
  *
  */
 public enum ClusterBlockLevel {
-    READ(0),
-    WRITE(1),
-    METADATA_READ(2),
-    METADATA_WRITE(3);
+    READ,
+    WRITE,
+    METADATA_READ,
+    METADATA_WRITE;
 
-    public static final EnumSet<ClusterBlockLevel> ALL = EnumSet.of(READ, WRITE, METADATA_READ, METADATA_WRITE);
+    public static final EnumSet<ClusterBlockLevel> ALL = EnumSet.allOf(ClusterBlockLevel.class);
     public static final EnumSet<ClusterBlockLevel> READ_WRITE = EnumSet.of(READ, WRITE);
-
-    private final int id;
-
-    ClusterBlockLevel(int id) {
-        this.id = id;
-    }
-
-    public int id() {
-        return this.id;
-    }
-
-    static ClusterBlockLevel fromId(int id) {
-        if (id == 0) {
-            return READ;
-        } else if (id == 1) {
-            return WRITE;
-        } else if (id == 2) {
-            return METADATA_READ;
-        } else if (id == 3) {
-            return METADATA_WRITE;
-        }
-        throw new IllegalArgumentException("No cluster block level matching [" + id + "]");
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java b/core/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java
index e6f04c8702cf4..9e05d50831882 100644
--- a/core/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java
+++ b/core/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java
@@ -21,6 +21,7 @@
 
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import org.elasticsearch.cluster.AbstractDiffable;
+import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaDataIndexStateService;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
@@ -48,8 +49,6 @@
 public class ClusterBlocks extends AbstractDiffable<ClusterBlocks> {
     public static final ClusterBlocks EMPTY_CLUSTER_BLOCK = new ClusterBlocks(emptySet(), ImmutableOpenMap.of());
 
-    public static final ClusterBlocks PROTO = EMPTY_CLUSTER_BLOCK;
-
     private final Set<ClusterBlock> global;
 
     private final ImmutableOpenMap<String, Set<ClusterBlock>> indicesBlocks;
@@ -59,23 +58,7 @@ public class ClusterBlocks extends AbstractDiffable<ClusterBlocks> {
     ClusterBlocks(Set<ClusterBlock> global, ImmutableOpenMap<String, Set<ClusterBlock>> indicesBlocks) {
         this.global = global;
         this.indicesBlocks = indicesBlocks;
-
-        levelHolders = new ImmutableLevelHolder[ClusterBlockLevel.values().length];
-        for (final ClusterBlockLevel level : ClusterBlockLevel.values()) {
-            Predicate<ClusterBlock> containsLevel = block -> block.contains(level);
-            Set<ClusterBlock> newGlobal = unmodifiableSet(global.stream()
-                    .filter(containsLevel)
-                    .collect(toSet()));
-
-            ImmutableOpenMap.Builder<String, Set<ClusterBlock>> indicesBuilder = ImmutableOpenMap.builder();
-            for (ObjectObjectCursor<String, Set<ClusterBlock>> entry : indicesBlocks) {
-                indicesBuilder.put(entry.key, unmodifiableSet(entry.value.stream()
-                        .filter(containsLevel)
-                        .collect(toSet())));
-            }
-
-            levelHolders[level.id()] = new ImmutableLevelHolder(newGlobal, indicesBuilder.build());
-        }
+        levelHolders = generateLevelHolders(global, indicesBlocks);
     }
 
     public Set<ClusterBlock> global() {
@@ -87,17 +70,38 @@ public ImmutableOpenMap<String, Set<ClusterBlock>> indices() {
     }
 
     public Set<ClusterBlock> global(ClusterBlockLevel level) {
-        return levelHolders[level.id()].global();
+        return levelHolders[level.ordinal()].global();
     }
 
     public ImmutableOpenMap<String, Set<ClusterBlock>> indices(ClusterBlockLevel level) {
-        return levelHolders[level.id()].indices();
+        return levelHolders[level.ordinal()].indices();
     }
 
     private Set<ClusterBlock> blocksForIndex(ClusterBlockLevel level, String index) {
         return indices(level).getOrDefault(index, emptySet());
     }
 
+    private static ImmutableLevelHolder[] generateLevelHolders(Set<ClusterBlock> global,
+                                                               ImmutableOpenMap<String, Set<ClusterBlock>> indicesBlocks) {
+        ImmutableLevelHolder[] levelHolders = new ImmutableLevelHolder[ClusterBlockLevel.values().length];
+        for (final ClusterBlockLevel level : ClusterBlockLevel.values()) {
+            Predicate<ClusterBlock> containsLevel = block -> block.contains(level);
+            Set<ClusterBlock> newGlobal = unmodifiableSet(global.stream()
+                .filter(containsLevel)
+                .collect(toSet()));
+
+            ImmutableOpenMap.Builder<String, Set<ClusterBlock>> indicesBuilder = ImmutableOpenMap.builder();
+            for (ObjectObjectCursor<String, Set<ClusterBlock>> entry : indicesBlocks) {
+                indicesBuilder.put(entry.key, unmodifiableSet(entry.value.stream()
+                    .filter(containsLevel)
+                    .collect(toSet())));
+            }
+
+            levelHolders[level.ordinal()] = new ImmutableLevelHolder(newGlobal, indicesBuilder.build());
+        }
+        return levelHolders;
+    }
+
     /**
      * Returns <tt>true</tt> if one of the global blocks as its disable state persistence flag set.
      */
@@ -199,7 +203,28 @@ public ClusterBlockException indicesBlockedException(ClusterBlockLevel level, St
         return new ClusterBlockException(unmodifiableSet(blocks.collect(toSet())));
     }
 
-    public String prettyPrint() {
+    /**
+     * Returns <code>true</code> iff non of the given have a {@link ClusterBlockLevel#METADATA_WRITE} in place where the
+     * {@link ClusterBlock#isAllowReleaseResources()} returns <code>false</code>. This is used in places where resources will be released
+     * like the deletion of an index to free up resources on nodes.
+     * @param indices the indices to check
+     */
+    public ClusterBlockException indicesAllowReleaseResources(String[] indices) {
+        final Function<String, Stream<ClusterBlock>> blocksForIndexAtLevel = index ->
+            blocksForIndex(ClusterBlockLevel.METADATA_WRITE, index).stream();
+        Stream<ClusterBlock> blocks = concat(
+            global(ClusterBlockLevel.METADATA_WRITE).stream(),
+            Stream.of(indices).flatMap(blocksForIndexAtLevel)).filter(clusterBlock -> clusterBlock.isAllowReleaseResources() == false);
+        Set<ClusterBlock> clusterBlocks = unmodifiableSet(blocks.collect(toSet()));
+        if (clusterBlocks.isEmpty()) {
+            return null;
+        }
+        return new ClusterBlockException(clusterBlocks);
+    }
+
+
+    @Override
+    public String toString() {
         if (global.isEmpty() && indices().isEmpty()) {
             return "";
         }
@@ -238,15 +263,16 @@ private static void writeBlockSet(Set<ClusterBlock> blocks, StreamOutput out) th
         }
     }
 
-    @Override
-    public ClusterBlocks readFrom(StreamInput in) throws IOException {
+    public ClusterBlocks(StreamInput in) throws IOException {
         Set<ClusterBlock> global = readBlockSet(in);
         int size = in.readVInt();
         ImmutableOpenMap.Builder<String, Set<ClusterBlock>> indicesBuilder = ImmutableOpenMap.builder(size);
         for (int j = 0; j < size; j++) {
             indicesBuilder.put(in.readString().intern(), readBlockSet(in));
         }
-        return new ClusterBlocks(global, indicesBuilder.build());
+        this.global = global;
+        this.indicesBlocks = indicesBuilder.build();
+        levelHolders = generateLevelHolders(global, indicesBlocks);
     }
 
     private static Set<ClusterBlock> readBlockSet(StreamInput in) throws IOException {
@@ -258,9 +284,11 @@ private static Set<ClusterBlock> readBlockSet(StreamInput in) throws IOException
         return unmodifiableSet(blocks);
     }
 
-    static class ImmutableLevelHolder {
+    public static Diff<ClusterBlocks> readDiffFrom(StreamInput in) throws IOException {
+        return AbstractDiffable.readDiffFrom(ClusterBlocks::new, in);
+    }
 
-        static final ImmutableLevelHolder EMPTY = new ImmutableLevelHolder(emptySet(), ImmutableOpenMap.of());
+    static class ImmutableLevelHolder {
 
         private final Set<ClusterBlock> global;
         private final ImmutableOpenMap<String, Set<ClusterBlock>> indices;
@@ -304,30 +332,31 @@ public Builder blocks(ClusterBlocks blocks) {
         }
 
         public Builder addBlocks(IndexMetaData indexMetaData) {
+            String indexName = indexMetaData.getIndex().getName();
             if (indexMetaData.getState() == IndexMetaData.State.CLOSE) {
-                addIndexBlock(indexMetaData.getIndex().getName(), MetaDataIndexStateService.INDEX_CLOSED_BLOCK);
+                addIndexBlock(indexName, MetaDataIndexStateService.INDEX_CLOSED_BLOCK);
             }
             if (IndexMetaData.INDEX_READ_ONLY_SETTING.get(indexMetaData.getSettings())) {
-                addIndexBlock(indexMetaData.getIndex().getName(), IndexMetaData.INDEX_READ_ONLY_BLOCK);
+                addIndexBlock(indexName, IndexMetaData.INDEX_READ_ONLY_BLOCK);
             }
             if (IndexMetaData.INDEX_BLOCKS_READ_SETTING.get(indexMetaData.getSettings())) {
-                addIndexBlock(indexMetaData.getIndex().getName(), IndexMetaData.INDEX_READ_BLOCK);
+                addIndexBlock(indexName, IndexMetaData.INDEX_READ_BLOCK);
             }
             if (IndexMetaData.INDEX_BLOCKS_WRITE_SETTING.get(indexMetaData.getSettings())) {
-                addIndexBlock(indexMetaData.getIndex().getName(), IndexMetaData.INDEX_WRITE_BLOCK);
+                addIndexBlock(indexName, IndexMetaData.INDEX_WRITE_BLOCK);
             }
             if (IndexMetaData.INDEX_BLOCKS_METADATA_SETTING.get(indexMetaData.getSettings())) {
-                addIndexBlock(indexMetaData.getIndex().getName(), IndexMetaData.INDEX_METADATA_BLOCK);
+                addIndexBlock(indexName, IndexMetaData.INDEX_METADATA_BLOCK);
+            }
+            if (IndexMetaData.INDEX_BLOCKS_READ_ONLY_ALLOW_DELETE_SETTING.get(indexMetaData.getSettings())) {
+                addIndexBlock(indexName, IndexMetaData.INDEX_READ_ONLY_ALLOW_DELETE_BLOCK);
             }
             return this;
         }
 
         public Builder updateBlocks(IndexMetaData indexMetaData) {
-            removeIndexBlock(indexMetaData.getIndex().getName(), MetaDataIndexStateService.INDEX_CLOSED_BLOCK);
-            removeIndexBlock(indexMetaData.getIndex().getName(), IndexMetaData.INDEX_READ_ONLY_BLOCK);
-            removeIndexBlock(indexMetaData.getIndex().getName(), IndexMetaData.INDEX_READ_BLOCK);
-            removeIndexBlock(indexMetaData.getIndex().getName(), IndexMetaData.INDEX_WRITE_BLOCK);
-            removeIndexBlock(indexMetaData.getIndex().getName(), IndexMetaData.INDEX_METADATA_BLOCK);
+            // let's remove all blocks for this index and add them back -- no need to remove all individual blocks....
+            indices.remove(indexMetaData.getIndex().getName());
             return addBlocks(indexMetaData);
         }
 
@@ -382,9 +411,5 @@ public ClusterBlocks build() {
             }
             return new ClusterBlocks(unmodifiableSet(new HashSet<>(global)), indicesBuilder.build());
         }
-
-        public static ClusterBlocks readClusterBlocks(StreamInput in) throws IOException {
-            return PROTO.readFrom(in);
-        }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java
index 7e4d19174852a..f8c360bed39f2 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java
@@ -21,13 +21,16 @@
 
 import org.elasticsearch.ElasticsearchGenerationException;
 import org.elasticsearch.cluster.AbstractDiffable;
+import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
@@ -42,8 +45,6 @@
  */
 public class AliasMetaData extends AbstractDiffable<AliasMetaData> {
 
-    public static final AliasMetaData PROTO = new AliasMetaData("", null, null, null);
-
     private final String alias;
 
     private final CompressedXContent filter;
@@ -174,22 +175,29 @@ public void writeTo(StreamOutput out) throws IOException {
 
     }
 
-    @Override
-    public AliasMetaData readFrom(StreamInput in) throws IOException {
-        String alias = in.readString();
-        CompressedXContent filter = null;
+    public AliasMetaData(StreamInput in) throws IOException {
+        alias = in.readString();
         if (in.readBoolean()) {
             filter = CompressedXContent.readCompressedString(in);
+        } else {
+            filter = null;
         }
-        String indexRouting = null;
         if (in.readBoolean()) {
             indexRouting = in.readString();
+        } else {
+            indexRouting = null;
         }
-        String searchRouting = null;
         if (in.readBoolean()) {
             searchRouting = in.readString();
+            searchRoutingValues = Collections.unmodifiableSet(Strings.splitStringByCommaToSet(searchRouting));
+        } else {
+            searchRouting = null;
+            searchRoutingValues = emptySet();
         }
-        return new AliasMetaData(alias, filter, indexRouting, searchRouting);
+    }
+
+    public static Diff<AliasMetaData> readDiffFrom(StreamInput in) throws IOException {
+        return readDiffFrom(AliasMetaData::new, in);
     }
 
     public static class Builder {
@@ -228,14 +236,7 @@ public Builder filter(String filter) {
                 this.filter = null;
                 return this;
             }
-            try {
-                try (XContentParser parser = XContentFactory.xContent(filter).createParser(filter)) {
-                    filter(parser.mapOrdered());
-                }
-                return this;
-            } catch (IOException e) {
-                throw new ElasticsearchGenerationException("Failed to generate [" + filter + "]", e);
-            }
+            return filter(XContentHelper.convertToMap(XContentFactory.xContent(filter), filter, true));
         }
 
         public Builder filter(Map<String, Object> filter) {
@@ -289,11 +290,7 @@ public static void toXContent(AliasMetaData aliasMetaData, XContentBuilder build
                 if (binary) {
                     builder.field("filter", aliasMetaData.filter.compressed());
                 } else {
-                    byte[] data = aliasMetaData.filter().uncompressed();
-                    try (XContentParser parser = XContentFactory.xContent(data).createParser(data)) {
-                        Map<String, Object> filter = parser.mapOrdered();
-                        builder.field("filter", filter);
-                    }
+                    builder.field("filter", XContentHelper.convertToMap(new BytesArray(aliasMetaData.filter().uncompressed()), true).v2());
                 }
             }
             if (aliasMetaData.indexRouting() != null) {
@@ -339,14 +336,6 @@ public static AliasMetaData fromXContent(XContentParser parser) throws IOExcepti
             }
             return builder.build();
         }
-
-        public void writeTo(AliasMetaData aliasMetaData, StreamOutput out) throws IOException {
-            aliasMetaData.writeTo(out);
-        }
-
-        public static AliasMetaData readFrom(StreamInput in) throws IOException {
-            return PROTO.readFrom(in);
-        }
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/AliasOrIndex.java b/core/src/main/java/org/elasticsearch/cluster/metadata/AliasOrIndex.java
index 4ad9b7e5317e7..786bd9af78a4c 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/AliasOrIndex.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/AliasOrIndex.java
@@ -121,7 +121,7 @@ public Tuple<String, AliasMetaData> next() {
                         }
 
                         @Override
-                        public final void remove() {
+                        public void remove() {
                             throw new UnsupportedOperationException();
                         }
 
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java b/core/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java
index cb46b22fe7e43..1a28117cf70c2 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java
@@ -25,7 +25,9 @@
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
@@ -75,8 +77,8 @@ public void validateAliasMetaData(AliasMetaData aliasMetaData, String index, Met
     public void validateAliasStandalone(Alias alias) {
         validateAliasStandalone(alias.name(), alias.indexRouting());
         if (Strings.hasLength(alias.filter())) {
-            try (XContentParser parser = XContentFactory.xContent(alias.filter()).createParser(alias.filter())) {
-                parser.map();
+            try {
+                XContentHelper.convertToMap(XContentFactory.xContent(alias.filter()), alias.filter(), false);
             } catch (Exception e) {
                 throw new IllegalArgumentException("failed to parse filter for alias [" + alias.name() + "]", e);
             }
@@ -99,10 +101,11 @@ public void validateAlias(String alias, String index, @Nullable String indexRout
         }
     }
 
-    private void validateAliasStandalone(String alias, String indexRouting) {
+    void validateAliasStandalone(String alias, String indexRouting) {
         if (!Strings.hasText(alias)) {
             throw new IllegalArgumentException("alias name is required");
         }
+        MetaDataCreateIndexService.validateIndexOrAliasName(alias, InvalidAliasNameException::new);
         if (indexRouting != null && indexRouting.indexOf(',') != -1) {
             throw new IllegalArgumentException("alias [" + alias + "] has several index routing values associated with it");
         }
@@ -113,9 +116,10 @@ private void validateAliasStandalone(String alias, String indexRouting) {
      * provided {@link org.elasticsearch.index.query.QueryShardContext}
      * @throws IllegalArgumentException if the filter is not valid
      */
-    public void validateAliasFilter(String alias, String filter, QueryShardContext queryShardContext) {
+    public void validateAliasFilter(String alias, String filter, QueryShardContext queryShardContext,
+            NamedXContentRegistry xContentRegistry) {
         assert queryShardContext != null;
-        try (XContentParser parser = XContentFactory.xContent(filter).createParser(filter)) {
+        try (XContentParser parser = XContentFactory.xContent(filter).createParser(xContentRegistry, filter)) {
             validateAliasFilter(parser, queryShardContext);
         } catch (Exception e) {
             throw new IllegalArgumentException("failed to parse filter for alias [" + alias + "]", e);
@@ -127,9 +131,10 @@ public void validateAliasFilter(String alias, String filter, QueryShardContext q
      * provided {@link org.elasticsearch.index.query.QueryShardContext}
      * @throws IllegalArgumentException if the filter is not valid
      */
-    public void validateAliasFilter(String alias, byte[] filter, QueryShardContext queryShardContext) {
+    public void validateAliasFilter(String alias, byte[] filter, QueryShardContext queryShardContext,
+            NamedXContentRegistry xContentRegistry) {
         assert queryShardContext != null;
-        try (XContentParser parser = XContentFactory.xContent(filter).createParser(filter)) {
+        try (XContentParser parser = XContentFactory.xContent(filter).createParser(xContentRegistry, filter)) {
             validateAliasFilter(parser, queryShardContext);
         } catch (Exception e) {
             throw new IllegalArgumentException("failed to parse filter for alias [" + alias + "]", e);
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/AutoExpandReplicas.java b/core/src/main/java/org/elasticsearch/cluster/metadata/AutoExpandReplicas.java
index 4b4a8e54d7c6c..0031aaf19de02 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/AutoExpandReplicas.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/AutoExpandReplicas.java
@@ -19,6 +19,8 @@
 package org.elasticsearch.cluster.metadata;
 
 import org.elasticsearch.common.Booleans;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 
@@ -28,12 +30,17 @@
  * based on the number of datanodes in the cluster. This class handles all the parsing and streamlines the access to these values.
  */
 final class AutoExpandReplicas {
+    private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(AutoExpandReplicas.class));
+
     // the value we recognize in the "max" position to mean all the nodes
     private static final String ALL_NODES_VALUE = "all";
     public static final Setting<AutoExpandReplicas> SETTING = new Setting<>(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "false", (value) -> {
         final int min;
         final int max;
-        if (Booleans.parseBoolean(value, true) == false) {
+        if (Booleans.isExplicitFalse(value)) {
+            if (Booleans.isStrictlyBoolean(value) == false) {
+                DEPRECATION_LOGGER.deprecated("Expected [false] for setting [{}] but got [{}]", IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, value);
+            }
             return new AutoExpandReplicas(0, 0, false);
         }
         final int dash = value.indexOf('-');
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/ClusterNameExpressionResolver.java b/core/src/main/java/org/elasticsearch/cluster/metadata/ClusterNameExpressionResolver.java
new file mode 100644
index 0000000000000..2032c2f4ef3ba
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/ClusterNameExpressionResolver.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.metadata;
+
+import org.elasticsearch.common.component.AbstractComponent;
+import org.elasticsearch.common.regex.Regex;
+import org.elasticsearch.common.settings.Settings;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Resolves cluster names from an expression. The expression must be the exact match of a cluster
+ * name or must be a wildcard expression.
+ */
+public final class ClusterNameExpressionResolver extends AbstractComponent {
+
+    private final WildcardExpressionResolver wildcardResolver = new WildcardExpressionResolver();
+
+    public ClusterNameExpressionResolver(Settings settings) {
+        super(settings);
+    }
+
+    /**
+     * Resolves the provided cluster expression to matching cluster names. This method only
+     * supports exact or wildcard matches.
+     *
+     * @param remoteClusters    the aliases for remote clusters
+     * @param clusterExpression the expressions that can be resolved to cluster names.
+     * @return the resolved cluster aliases.
+     */
+    public List<String> resolveClusterNames(Set<String> remoteClusters, String clusterExpression) {
+        if (remoteClusters.contains(clusterExpression)) {
+            return Collections.singletonList(clusterExpression);
+        } else if (Regex.isSimpleMatchPattern(clusterExpression)) {
+            return wildcardResolver.resolve(remoteClusters, clusterExpression);
+        } else {
+            return Collections.emptyList();
+        }
+    }
+
+    private static class WildcardExpressionResolver {
+
+        private List<String> resolve(Set<String> remoteClusters, String clusterExpression) {
+            if (isTrivialWildcard(clusterExpression)) {
+                return resolveTrivialWildcard(remoteClusters);
+            }
+
+            Set<String> matches = matches(remoteClusters, clusterExpression);
+            if (matches.isEmpty()) {
+                return Collections.emptyList();
+            } else {
+                return new ArrayList<>(matches);
+            }
+        }
+
+        private boolean isTrivialWildcard(String clusterExpression) {
+            return Regex.isMatchAllPattern(clusterExpression);
+        }
+
+        private List<String> resolveTrivialWildcard(Set<String> remoteClusters) {
+            return new ArrayList<>(remoteClusters);
+        }
+
+        private static Set<String> matches(Set<String> remoteClusters, String expression) {
+            if (expression.indexOf("*") == expression.length() - 1) {
+                return otherWildcard(remoteClusters, expression);
+            } else {
+                return otherWildcard(remoteClusters, expression);
+            }
+        }
+
+        private static Set<String> otherWildcard(Set<String> remoteClusters, String expression) {
+            final String pattern = expression;
+            return remoteClusters.stream()
+                .filter(n -> Regex.simpleMatch(pattern, n))
+                .collect(Collectors.toSet());
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java
index f6a54fc82d72f..d60617ea6423a 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java
@@ -20,15 +20,15 @@
 package org.elasticsearch.cluster.metadata;
 
 import org.elasticsearch.cluster.Diff;
+import org.elasticsearch.cluster.NamedDiff;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.joda.Joda;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ContextParser;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -43,7 +43,6 @@
 import java.util.List;
 import java.util.Objects;
 import java.util.concurrent.TimeUnit;
-import java.util.function.BiFunction;
 
 /**
  * A collection of tombstones for explicitly marking indices as deleted in the cluster state.
@@ -67,10 +66,9 @@ public final class IndexGraveyard implements MetaData.Custom {
                                                                                      500, // the default maximum number of tombstones
                                                                                      Setting.Property.NodeScope);
 
-    public static final IndexGraveyard PROTO = new IndexGraveyard(new ArrayList<>());
     public static final String TYPE = "index-graveyard";
     private static final ParseField TOMBSTONES_FIELD = new ParseField("tombstones");
-    private static final ObjectParser<List<Tombstone>, ParseFieldMatcherSupplier> GRAVEYARD_PARSER;
+    private static final ObjectParser<List<Tombstone>, Void> GRAVEYARD_PARSER;
     static {
         GRAVEYARD_PARSER = new ObjectParser<>("index_graveyard", ArrayList::new);
         GRAVEYARD_PARSER.declareObjectArray(List::addAll, Tombstone.getParser(), TOMBSTONES_FIELD);
@@ -83,7 +81,7 @@ private IndexGraveyard(final List<Tombstone> list) {
         tombstones = Collections.unmodifiableList(list);
     }
 
-    private IndexGraveyard(final StreamInput in) throws IOException {
+    public IndexGraveyard(final StreamInput in) throws IOException {
         final int queueSize = in.readVInt();
         List<Tombstone> tombstones = new ArrayList<>(queueSize);
         for (int i = 0; i < queueSize; i++) {
@@ -92,12 +90,8 @@ private IndexGraveyard(final StreamInput in) throws IOException {
         this.tombstones = Collections.unmodifiableList(tombstones);
     }
 
-    public static IndexGraveyard fromStream(final StreamInput in) throws IOException {
-        return new IndexGraveyard(in);
-    }
-
     @Override
-    public String type() {
+    public String getWriteableName() {
         return TYPE;
     }
 
@@ -144,8 +138,8 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa
         return builder.endArray();
     }
 
-    public IndexGraveyard fromXContent(final XContentParser parser) throws IOException {
-        return new IndexGraveyard(GRAVEYARD_PARSER.parse(parser, () -> ParseFieldMatcher.STRICT));
+    public static IndexGraveyard fromXContent(final XContentParser parser) throws IOException {
+        return new IndexGraveyard(GRAVEYARD_PARSER.parse(parser, null));
     }
 
     @Override
@@ -161,19 +155,13 @@ public void writeTo(final StreamOutput out) throws IOException {
         }
     }
 
-    @Override
-    public IndexGraveyard readFrom(final StreamInput in) throws IOException {
-        return new IndexGraveyard(in);
-    }
-
     @Override
     @SuppressWarnings("unchecked")
     public Diff<MetaData.Custom> diff(final MetaData.Custom previous) {
         return new IndexGraveyardDiff((IndexGraveyard) previous, this);
     }
 
-    @Override
-    public Diff<MetaData.Custom> readDiffFrom(final StreamInput in) throws IOException {
+    public static NamedDiff<MetaData.Custom> readDiffFrom(final StreamInput in) throws IOException {
         return new IndexGraveyardDiff(in);
     }
 
@@ -273,7 +261,7 @@ public IndexGraveyard build(final Settings settings) {
     /**
      * A class representing a diff of two IndexGraveyard objects.
      */
-    public static final class IndexGraveyardDiff implements Diff<MetaData.Custom> {
+    public static final class IndexGraveyardDiff implements NamedDiff<MetaData.Custom> {
 
         private final List<Tombstone> added;
         private final int removedCount;
@@ -349,6 +337,11 @@ public List<Tombstone> getAdded() {
         public int getRemovedCount() {
             return removedCount;
         }
+
+        @Override
+        public String getWriteableName() {
+            return TYPE;
+        }
     }
 
     /**
@@ -359,16 +352,17 @@ public static final class Tombstone implements ToXContent, Writeable {
         private static final String INDEX_KEY = "index";
         private static final String DELETE_DATE_IN_MILLIS_KEY = "delete_date_in_millis";
         private static final String DELETE_DATE_KEY = "delete_date";
-        private static final ObjectParser<Tombstone.Builder, ParseFieldMatcherSupplier> TOMBSTONE_PARSER;
+        private static final ObjectParser<Tombstone.Builder, Void> TOMBSTONE_PARSER;
         static {
             TOMBSTONE_PARSER = new ObjectParser<>("tombstoneEntry", Tombstone.Builder::new);
-            TOMBSTONE_PARSER.declareObject(Tombstone.Builder::index, Index::parseIndex, new ParseField(INDEX_KEY));
+            TOMBSTONE_PARSER.declareObject(Tombstone.Builder::index, (parser, context) -> Index.fromXContent(parser),
+                    new ParseField(INDEX_KEY));
             TOMBSTONE_PARSER.declareLong(Tombstone.Builder::deleteDateInMillis, new ParseField(DELETE_DATE_IN_MILLIS_KEY));
             TOMBSTONE_PARSER.declareString((b, s) -> {}, new ParseField(DELETE_DATE_KEY));
         }
 
-        static BiFunction<XContentParser, ParseFieldMatcherSupplier, Tombstone> getParser() {
-            return (p, c) -> TOMBSTONE_PARSER.apply(p, c).build();
+        static ContextParser<Void, Tombstone> getParser() {
+            return (parser, context) -> TOMBSTONE_PARSER.apply(parser, null).build();
         }
 
         private final Index index;
@@ -443,7 +437,7 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa
         }
 
         public static Tombstone fromXContent(final XContentParser parser) throws IOException {
-            return TOMBSTONE_PARSER.parse(parser, () -> ParseFieldMatcher.STRICT).build();
+            return TOMBSTONE_PARSER.parse(parser, null).build();
         }
 
         /**
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
index 4ab3b85e46ac3..57a2876128342 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
@@ -33,7 +33,7 @@
 import org.elasticsearch.cluster.node.DiscoveryNodeFilters;
 import org.elasticsearch.cluster.routing.allocation.IndexMetaDataUpdater;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.collect.ImmutableOpenIntMap;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.collect.MapBuilder;
@@ -44,10 +44,10 @@
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.loader.SettingsLoader;
-import org.elasticsearch.common.xcontent.FromXContentBuilder;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.gateway.MetaDataStateFormat;
@@ -59,7 +59,6 @@
 import org.joda.time.DateTimeZone;
 
 import java.io.IOException;
-import java.text.ParseException;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.EnumSet;
@@ -70,16 +69,18 @@
 import java.util.Set;
 import java.util.function.Function;
 
+import static org.elasticsearch.cluster.node.DiscoveryNodeFilters.IP_VALIDATOR;
 import static org.elasticsearch.cluster.node.DiscoveryNodeFilters.OpType.AND;
 import static org.elasticsearch.cluster.node.DiscoveryNodeFilters.OpType.OR;
 import static org.elasticsearch.common.settings.Settings.readSettingsFromStream;
 import static org.elasticsearch.common.settings.Settings.writeSettingsToStream;
 
-/**
- *
- */
-public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuilder<IndexMetaData>, ToXContent {
+public class IndexMetaData implements Diffable<IndexMetaData>, ToXContent {
 
+    /**
+     * This class will be removed in v7.0
+     */
+    @Deprecated
     public interface Custom extends Diffable<Custom>, ToXContent {
 
         String type();
@@ -88,6 +89,16 @@ public interface Custom extends Diffable<Custom>, ToXContent {
 
         Custom fromXContent(XContentParser parser) throws IOException;
 
+        /**
+         * Reads the {@link org.elasticsearch.cluster.Diff} from StreamInput
+         */
+        Diff<Custom> readDiffFrom(StreamInput in) throws IOException;
+
+        /**
+         * Reads an object of this type from the provided {@linkplain StreamInput}. The receiving instance remains unchanged.
+         */
+        Custom readFrom(StreamInput in) throws IOException;
+
         /**
          * Merges from this to another, with this being more important, i.e., if something exists in this and another,
          * this will prevail.
@@ -119,12 +130,13 @@ public static <T extends Custom> T lookupPrototypeSafe(String type) {
         return proto;
     }
 
-    public static final ClusterBlock INDEX_READ_ONLY_BLOCK = new ClusterBlock(5, "index read-only (api)", false, false, RestStatus.FORBIDDEN, EnumSet.of(ClusterBlockLevel.WRITE, ClusterBlockLevel.METADATA_WRITE));
-    public static final ClusterBlock INDEX_READ_BLOCK = new ClusterBlock(7, "index read (api)", false, false, RestStatus.FORBIDDEN, EnumSet.of(ClusterBlockLevel.READ));
-    public static final ClusterBlock INDEX_WRITE_BLOCK = new ClusterBlock(8, "index write (api)", false, false, RestStatus.FORBIDDEN, EnumSet.of(ClusterBlockLevel.WRITE));
-    public static final ClusterBlock INDEX_METADATA_BLOCK = new ClusterBlock(9, "index metadata (api)", false, false, RestStatus.FORBIDDEN, EnumSet.of(ClusterBlockLevel.METADATA_WRITE, ClusterBlockLevel.METADATA_READ));
+    public static final ClusterBlock INDEX_READ_ONLY_BLOCK = new ClusterBlock(5, "index read-only (api)", false, false, false, RestStatus.FORBIDDEN, EnumSet.of(ClusterBlockLevel.WRITE, ClusterBlockLevel.METADATA_WRITE));
+    public static final ClusterBlock INDEX_READ_BLOCK = new ClusterBlock(7, "index read (api)", false, false, false, RestStatus.FORBIDDEN, EnumSet.of(ClusterBlockLevel.READ));
+    public static final ClusterBlock INDEX_WRITE_BLOCK = new ClusterBlock(8, "index write (api)", false, false, false, RestStatus.FORBIDDEN, EnumSet.of(ClusterBlockLevel.WRITE));
+    public static final ClusterBlock INDEX_METADATA_BLOCK = new ClusterBlock(9, "index metadata (api)", false, false, false, RestStatus.FORBIDDEN, EnumSet.of(ClusterBlockLevel.METADATA_WRITE, ClusterBlockLevel.METADATA_READ));
+    public static final ClusterBlock INDEX_READ_ONLY_ALLOW_DELETE_BLOCK = new ClusterBlock(12, "index read-only / allow delete (api)", false, false, true, RestStatus.FORBIDDEN, EnumSet.of(ClusterBlockLevel.METADATA_WRITE, ClusterBlockLevel.WRITE));
 
-    public static enum State {
+    public enum State {
         OPEN((byte) 0),
         CLOSE((byte) 1);
 
@@ -157,20 +169,37 @@ public static State fromString(String state) {
         }
     }
 
+    static Setting<Integer> buildNumberOfShardsSetting() {
+        /* This is a safety limit that should only be exceeded in very rare and special cases. The assumption is that
+         * 99% of the users have less than 1024 shards per index. We also make it a hard check that requires restart of nodes
+         * if a cluster should allow to create more than 1024 shards per index. NOTE: this does not limit the number of shards per cluster.
+         * this also prevents creating stuff like a new index with millions of shards by accident which essentially kills the entire cluster
+         * with OOM on the spot.*/
+        final int maxNumShards = Integer.parseInt(System.getProperty("es.index.max_number_of_shards", "1024"));
+        if (maxNumShards < 1) {
+            throw new IllegalArgumentException("es.index.max_number_of_shards must be > 0");
+        }
+        return Setting.intSetting(SETTING_NUMBER_OF_SHARDS, Math.min(5, maxNumShards), 1, maxNumShards,
+            Property.IndexScope, Property.Final);
+    }
+
     public static final String INDEX_SETTING_PREFIX = "index.";
     public static final String SETTING_NUMBER_OF_SHARDS = "index.number_of_shards";
-    public static final Setting<Integer> INDEX_NUMBER_OF_SHARDS_SETTING =
-        Setting.intSetting(SETTING_NUMBER_OF_SHARDS, 5, 1, Property.IndexScope);
+    public static final Setting<Integer> INDEX_NUMBER_OF_SHARDS_SETTING = buildNumberOfShardsSetting();
     public static final String SETTING_NUMBER_OF_REPLICAS = "index.number_of_replicas";
     public static final Setting<Integer> INDEX_NUMBER_OF_REPLICAS_SETTING =
         Setting.intSetting(SETTING_NUMBER_OF_REPLICAS, 1, 0, Property.Dynamic, Property.IndexScope);
     public static final String SETTING_SHADOW_REPLICAS = "index.shadow_replicas";
     public static final Setting<Boolean> INDEX_SHADOW_REPLICAS_SETTING =
-        Setting.boolSetting(SETTING_SHADOW_REPLICAS, false, Property.IndexScope);
+        Setting.boolSetting(SETTING_SHADOW_REPLICAS, false, Property.IndexScope, Property.Deprecated);
+
+    public static final String SETTING_ROUTING_PARTITION_SIZE = "index.routing_partition_size";
+    public static final Setting<Integer> INDEX_ROUTING_PARTITION_SIZE_SETTING =
+            Setting.intSetting(SETTING_ROUTING_PARTITION_SIZE, 1, 1, Property.IndexScope);
 
     public static final String SETTING_SHARED_FILESYSTEM = "index.shared_filesystem";
     public static final Setting<Boolean> INDEX_SHARED_FILESYSTEM_SETTING =
-        Setting.boolSetting(SETTING_SHARED_FILESYSTEM, false, Property.IndexScope);
+        Setting.boolSetting(SETTING_SHARED_FILESYSTEM, INDEX_SHADOW_REPLICAS_SETTING, Property.IndexScope, Property.Deprecated);
 
     public static final String SETTING_AUTO_EXPAND_REPLICAS = "index.auto_expand_replicas";
     public static final Setting<AutoExpandReplicas> INDEX_AUTO_EXPAND_REPLICAS_SETTING = AutoExpandReplicas.SETTING;
@@ -190,12 +219,20 @@ public static State fromString(String state) {
     public static final Setting<Boolean> INDEX_BLOCKS_METADATA_SETTING =
         Setting.boolSetting(SETTING_BLOCKS_METADATA, false, Property.Dynamic, Property.IndexScope);
 
+    public static final String SETTING_READ_ONLY_ALLOW_DELETE = "index.blocks.read_only_allow_delete";
+    public static final Setting<Boolean> INDEX_BLOCKS_READ_ONLY_ALLOW_DELETE_SETTING =
+        Setting.boolSetting(SETTING_READ_ONLY_ALLOW_DELETE, false, Property.Dynamic, Property.IndexScope);
+
     public static final String SETTING_VERSION_CREATED = "index.version.created";
     public static final String SETTING_VERSION_CREATED_STRING = "index.version.created_string";
     public static final String SETTING_VERSION_UPGRADED = "index.version.upgraded";
     public static final String SETTING_VERSION_UPGRADED_STRING = "index.version.upgraded_string";
-    public static final String SETTING_VERSION_MINIMUM_COMPATIBLE = "index.version.minimum_compatible";
     public static final String SETTING_CREATION_DATE = "index.creation_date";
+    /**
+     * The user provided name for an index. This is the plain string provided by the user when the index was created.
+     * It might still contain date math expressions etc. (added in 5.0)
+     */
+    public static final String SETTING_INDEX_PROVIDED_NAME = "index.provided_name";
     public static final String SETTING_PRIORITY = "index.priority";
     public static final Setting<Integer> INDEX_PRIORITY_SETTING =
         Setting.intSetting("index.priority", 1, 0, Property.Dynamic, Property.IndexScope);
@@ -206,15 +243,19 @@ public static State fromString(String state) {
         new Setting<>(SETTING_DATA_PATH, "", Function.identity(), Property.IndexScope);
     public static final String SETTING_SHARED_FS_ALLOW_RECOVERY_ON_ANY_NODE = "index.shared_filesystem.recover_on_any_node";
     public static final Setting<Boolean> INDEX_SHARED_FS_ALLOW_RECOVERY_ON_ANY_NODE_SETTING =
-        Setting.boolSetting(SETTING_SHARED_FS_ALLOW_RECOVERY_ON_ANY_NODE, false, Property.Dynamic, Property.IndexScope);
+        Setting.boolSetting(SETTING_SHARED_FS_ALLOW_RECOVERY_ON_ANY_NODE, false,
+            Property.Dynamic, Property.IndexScope, Property.Deprecated);
     public static final String INDEX_UUID_NA_VALUE = "_na_";
 
+    public static final String INDEX_ROUTING_REQUIRE_GROUP_PREFIX = "index.routing.allocation.require";
+    public static final String INDEX_ROUTING_INCLUDE_GROUP_PREFIX = "index.routing.allocation.include";
+    public static final String INDEX_ROUTING_EXCLUDE_GROUP_PREFIX = "index.routing.allocation.exclude";
     public static final Setting<Settings> INDEX_ROUTING_REQUIRE_GROUP_SETTING =
-        Setting.groupSetting("index.routing.allocation.require.", Property.Dynamic, Property.IndexScope);
+        Setting.groupSetting(INDEX_ROUTING_REQUIRE_GROUP_PREFIX + ".", IP_VALIDATOR, Property.Dynamic, Property.IndexScope);
     public static final Setting<Settings> INDEX_ROUTING_INCLUDE_GROUP_SETTING =
-        Setting.groupSetting("index.routing.allocation.include.", Property.Dynamic, Property.IndexScope);
+        Setting.groupSetting(INDEX_ROUTING_INCLUDE_GROUP_PREFIX + ".", IP_VALIDATOR, Property.Dynamic, Property.IndexScope);
     public static final Setting<Settings> INDEX_ROUTING_EXCLUDE_GROUP_SETTING =
-        Setting.groupSetting("index.routing.allocation.exclude.", Property.Dynamic, Property.IndexScope);
+        Setting.groupSetting(INDEX_ROUTING_EXCLUDE_GROUP_PREFIX + ".", IP_VALIDATOR, Property.Dynamic, Property.IndexScope);
     public static final Setting<Settings> INDEX_ROUTING_INITIAL_RECOVERY_GROUP_SETTING =
         Setting.groupSetting("index.routing.allocation.initial_recovery."); // this is only setable internally not a registered setting!!
 
@@ -228,9 +269,12 @@ public static State fromString(String state) {
                       Setting.Property.Dynamic,
                       Setting.Property.IndexScope);
 
-    public static final IndexMetaData PROTO = IndexMetaData.builder("")
-            .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT))
-            .numberOfShards(1).numberOfReplicas(0).build();
+    /**
+     * an internal index format description, allowing us to find out if this index is upgraded or needs upgrading
+     */
+    private static final String INDEX_FORMAT = "index.format";
+    public static final Setting<Integer> INDEX_FORMAT_SETTING =
+            Setting.intSetting(INDEX_FORMAT, 0, Setting.Property.IndexScope, Setting.Property.Final);
 
     public static final String KEY_IN_SYNC_ALLOCATIONS = "in_sync_allocations";
     static final String KEY_VERSION = "version";
@@ -244,6 +288,7 @@ public static State fromString(String state) {
     public static final String INDEX_STATE_FILE_PREFIX = "state-";
     private final int routingNumShards;
     private final int routingFactor;
+    private final int routingPartitionSize;
 
     private final int numberOfShards;
     private final int numberOfReplicas;
@@ -273,7 +318,6 @@ public static State fromString(String state) {
 
     private final Version indexCreatedVersion;
     private final Version indexUpgradedVersion;
-    private final org.apache.lucene.util.Version minimumCompatibleLuceneVersion;
 
     private final ActiveShardCount waitForActiveShards;
 
@@ -281,8 +325,8 @@ private IndexMetaData(Index index, long version, long[] primaryTerms, State stat
                           ImmutableOpenMap<String, MappingMetaData> mappings, ImmutableOpenMap<String, AliasMetaData> aliases,
                           ImmutableOpenMap<String, Custom> customs, ImmutableOpenIntMap<Set<String>> inSyncAllocationIds,
                           DiscoveryNodeFilters requireFilters, DiscoveryNodeFilters initialRecoveryFilters, DiscoveryNodeFilters includeFilters, DiscoveryNodeFilters excludeFilters,
-                          Version indexCreatedVersion, Version indexUpgradedVersion, org.apache.lucene.util.Version minimumCompatibleLuceneVersion,
-                          int routingNumShards, ActiveShardCount waitForActiveShards) {
+                          Version indexCreatedVersion, Version indexUpgradedVersion,
+                          int routingNumShards, int routingPartitionSize, ActiveShardCount waitForActiveShards) {
 
         this.index = index;
         this.version = version;
@@ -303,9 +347,9 @@ private IndexMetaData(Index index, long version, long[] primaryTerms, State stat
         this.initialRecoveryFilters = initialRecoveryFilters;
         this.indexCreatedVersion = indexCreatedVersion;
         this.indexUpgradedVersion = indexUpgradedVersion;
-        this.minimumCompatibleLuceneVersion = minimumCompatibleLuceneVersion;
         this.routingNumShards = routingNumShards;
         this.routingFactor = routingNumShards / numberOfShards;
+        this.routingPartitionSize = routingPartitionSize;
         this.waitForActiveShards = waitForActiveShards;
         assert numberOfShards * routingFactor == routingNumShards :  routingNumShards + " must be a multiple of " + numberOfShards;
     }
@@ -362,13 +406,6 @@ public Version getUpgradedVersion() {
         return indexUpgradedVersion;
     }
 
-    /**
-     * Return the {@link org.apache.lucene.util.Version} of the oldest lucene segment in the index
-     */
-    public org.apache.lucene.util.Version getMinimumCompatibleVersion() {
-        return minimumCompatibleLuceneVersion;
-    }
-
     public long getCreationDate() {
         return settings.getAsLong(SETTING_CREATION_DATE, -1L);
     }
@@ -385,6 +422,14 @@ public int getNumberOfReplicas() {
         return numberOfReplicas;
     }
 
+    public int getRoutingPartitionSize() {
+        return routingPartitionSize;
+    }
+
+    public boolean isRoutingPartitionedIndex() {
+        return routingPartitionSize != 1;
+    }
+
     public int getTotalNumberOfShards() {
         return totalNumberOfShards;
     }
@@ -414,12 +459,14 @@ public MappingMetaData mapping(String mappingType) {
         return mappings.get(mappingType);
     }
 
-    public static final Setting<String> INDEX_SHRINK_SOURCE_UUID = Setting.simpleString("index.shrink.source.uuid");
-    public static final Setting<String> INDEX_SHRINK_SOURCE_NAME = Setting.simpleString("index.shrink.source.name");
+    public static final String INDEX_SHRINK_SOURCE_UUID_KEY = "index.shrink.source.uuid";
+    public static final String INDEX_SHRINK_SOURCE_NAME_KEY = "index.shrink.source.name";
+    public static final Setting<String> INDEX_SHRINK_SOURCE_UUID = Setting.simpleString(INDEX_SHRINK_SOURCE_UUID_KEY);
+    public static final Setting<String> INDEX_SHRINK_SOURCE_NAME = Setting.simpleString(INDEX_SHRINK_SOURCE_NAME_KEY);
 
 
     public Index getMergeSourceIndex() {
-        return INDEX_SHRINK_SOURCE_UUID.exists(settings) ? new Index(INDEX_SHRINK_SOURCE_NAME.get(settings),  INDEX_SHRINK_SOURCE_UUID.get(settings)) : null;
+        return INDEX_SHRINK_SOURCE_UUID.exists(settings) ? new Index(INDEX_SHRINK_SOURCE_NAME.get(settings), INDEX_SHRINK_SOURCE_UUID.get(settings)) : null;
     }
 
     /**
@@ -546,13 +593,11 @@ public Diff<IndexMetaData> diff(IndexMetaData previousState) {
         return new IndexMetaDataDiff(previousState, this);
     }
 
-    @Override
-    public Diff<IndexMetaData> readDiffFrom(StreamInput in) throws IOException {
+    public static Diff<IndexMetaData> readDiffFrom(StreamInput in) throws IOException {
         return new IndexMetaDataDiff(in);
     }
 
-    @Override
-    public IndexMetaData fromXContent(XContentParser parser, ParseFieldMatcher parseFieldMatcher) throws IOException {
+    public static IndexMetaData fromXContent(XContentParser parser) throws IOException {
         return Builder.fromXContent(parser);
     }
 
@@ -575,7 +620,7 @@ private static class IndexMetaDataDiff implements Diff<IndexMetaData> {
         private final Diff<ImmutableOpenMap<String, Custom>> customs;
         private final Diff<ImmutableOpenIntMap<Set<String>>> inSyncAllocationIds;
 
-        public IndexMetaDataDiff(IndexMetaData before, IndexMetaData after) {
+        IndexMetaDataDiff(IndexMetaData before, IndexMetaData after) {
             index = after.index.getName();
             version = after.version;
             routingNumShards = after.routingNumShards;
@@ -589,15 +634,17 @@ public IndexMetaDataDiff(IndexMetaData before, IndexMetaData after) {
                 DiffableUtils.getVIntKeySerializer(), DiffableUtils.StringSetValueSerializer.getInstance());
         }
 
-        public IndexMetaDataDiff(StreamInput in) throws IOException {
+        IndexMetaDataDiff(StreamInput in) throws IOException {
             index = in.readString();
             routingNumShards = in.readInt();
             version = in.readLong();
             state = State.fromId(in.readByte());
             settings = Settings.readSettingsFromStream(in);
             primaryTerms = in.readVLongArray();
-            mappings = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), MappingMetaData.PROTO);
-            aliases = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), AliasMetaData.PROTO);
+            mappings = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), MappingMetaData::new,
+                MappingMetaData::readDiffFrom);
+            aliases = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), AliasMetaData::new,
+                AliasMetaData::readDiffFrom);
             customs = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(),
                 new DiffableUtils.DiffableValueSerializer<String, Custom>() {
                     @Override
@@ -605,6 +652,7 @@ public Custom read(StreamInput in, String key) throws IOException {
                         return lookupPrototypeSafe(key).readFrom(in);
                     }
 
+                    @SuppressWarnings("unchecked")
                     @Override
                     public Diff<Custom> readDiff(StreamInput in, String key) throws IOException {
                         return lookupPrototypeSafe(key).readDiffFrom(in);
@@ -644,8 +692,7 @@ public IndexMetaData apply(IndexMetaData part) {
         }
     }
 
-    @Override
-    public IndexMetaData readFrom(StreamInput in) throws IOException {
+    public static IndexMetaData readFrom(StreamInput in) throws IOException {
         Builder builder = new Builder(in.readString());
         builder.version(in.readLong());
         builder.setRoutingNumShards(in.readInt());
@@ -654,12 +701,12 @@ public IndexMetaData readFrom(StreamInput in) throws IOException {
         builder.primaryTerms(in.readVLongArray());
         int mappingsSize = in.readVInt();
         for (int i = 0; i < mappingsSize; i++) {
-            MappingMetaData mappingMd = MappingMetaData.PROTO.readFrom(in);
+            MappingMetaData mappingMd = new MappingMetaData(in);
             builder.putMapping(mappingMd);
         }
         int aliasesSize = in.readVInt();
         for (int i = 0; i < aliasesSize; i++) {
-            AliasMetaData aliasMd = AliasMetaData.Builder.readFrom(in);
+            AliasMetaData aliasMd = new AliasMetaData(in);
             builder.putAlias(aliasMd);
         }
         int customSize = in.readVInt();
@@ -781,6 +828,11 @@ public int getRoutingNumShards() {
             return routingNumShards == null ? numberOfShards() : routingNumShards;
         }
 
+        /**
+         * Returns the number of shards.
+         *
+         * @return the provided value or -1 if it has not been set.
+         */
         public int numberOfShards() {
             return settings.getAsInt(SETTING_NUMBER_OF_SHARDS, -1);
         }
@@ -790,10 +842,29 @@ public Builder numberOfReplicas(int numberOfReplicas) {
             return this;
         }
 
+        /**
+         * Returns the number of replicas.
+         *
+         * @return the provided value or -1 if it has not been set.
+         */
         public int numberOfReplicas() {
             return settings.getAsInt(SETTING_NUMBER_OF_REPLICAS, -1);
         }
 
+        public Builder routingPartitionSize(int routingPartitionSize) {
+            settings = Settings.builder().put(settings).put(SETTING_ROUTING_PARTITION_SIZE, routingPartitionSize).build();
+            return this;
+        }
+
+        /**
+         * Returns the routing partition size.
+         *
+         * @return the provided value or -1 if it has not been set.
+         */
+        public int routingPartitionSize() {
+            return settings.getAsInt(SETTING_ROUTING_PARTITION_SIZE, -1);
+        }
+
         public Builder creationDate(long creationDate) {
             settings = Settings.builder().put(settings).put(SETTING_CREATION_DATE, creationDate).build();
             return this;
@@ -813,9 +884,7 @@ public MappingMetaData mapping(String type) {
         }
 
         public Builder putMapping(String type, String source) throws IOException {
-            try (XContentParser parser = XContentFactory.xContent(source).createParser(source)) {
-                putMapping(new MappingMetaData(type, parser.mapOrdered()));
-            }
+            putMapping(new MappingMetaData(type, XContentHelper.convertToMap(XContentFactory.xContent(source), source, true)));
             return this;
         }
 
@@ -859,7 +928,7 @@ public Set<String> getInSyncAllocationIds(int shardId) {
         }
 
         public Builder putInSyncAllocationIds(int shardId, Set<String> allocationIds) {
-            inSyncAllocationIds.put(shardId, new HashSet(allocationIds));
+            inSyncAllocationIds.put(shardId, new HashSet<>(allocationIds));
             return this;
         }
 
@@ -938,6 +1007,12 @@ public IndexMetaData build() {
                 throw new IllegalArgumentException("must specify non-negative number of shards for index [" + index + "]");
             }
 
+            int routingPartitionSize = INDEX_ROUTING_PARTITION_SIZE_SETTING.get(settings);
+            if (routingPartitionSize != 1 && routingPartitionSize >= getRoutingNumShards()) {
+                throw new IllegalArgumentException("routing partition size [" + routingPartitionSize + "] should be a positive number"
+                        + " less than the number of shards [" + getRoutingNumShards() + "] for [" + index + "]");
+            }
+
             // fill missing slots in inSyncAllocationIds with empty set if needed and make all entries immutable
             ImmutableOpenIntMap.Builder<Set<String>> filledInSyncAllocationIds = ImmutableOpenIntMap.builder();
             for (int i = 0; i < numberOfShards; i++) {
@@ -977,17 +1052,6 @@ public IndexMetaData build() {
             }
             Version indexCreatedVersion = Version.indexCreated(settings);
             Version indexUpgradedVersion = settings.getAsVersion(IndexMetaData.SETTING_VERSION_UPGRADED, indexCreatedVersion);
-            String stringLuceneVersion = settings.get(SETTING_VERSION_MINIMUM_COMPATIBLE);
-            final org.apache.lucene.util.Version minimumCompatibleLuceneVersion;
-            if (stringLuceneVersion != null) {
-                try {
-                    minimumCompatibleLuceneVersion = org.apache.lucene.util.Version.parse(stringLuceneVersion);
-                } catch (ParseException ex) {
-                    throw new IllegalStateException("Cannot parse lucene version [" + stringLuceneVersion + "] in the [" + SETTING_VERSION_MINIMUM_COMPATIBLE + "] setting", ex);
-                }
-            } else {
-                minimumCompatibleLuceneVersion = null;
-            }
 
             if (primaryTerms == null) {
                 initializePrimaryTerms();
@@ -1004,9 +1068,10 @@ public IndexMetaData build() {
             }
 
             final String uuid = settings.get(SETTING_INDEX_UUID, INDEX_UUID_NA_VALUE);
+
             return new IndexMetaData(new Index(index, uuid), version, primaryTerms, state, numberOfShards, numberOfReplicas, tmpSettings, mappings.build(),
                 tmpAliases.build(), customs.build(), filledInSyncAllocationIds.build(), requireFilters, initialRecoveryFilters, includeFilters, excludeFilters,
-                indexCreatedVersion, indexUpgradedVersion, minimumCompatibleLuceneVersion, getRoutingNumShards(), waitForActiveShards);
+                indexCreatedVersion, indexUpgradedVersion, getRoutingNumShards(), routingPartitionSize, waitForActiveShards);
         }
 
         public static void toXContent(IndexMetaData indexMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException {
@@ -1029,11 +1094,7 @@ public static void toXContent(IndexMetaData indexMetaData, XContentBuilder build
                 if (binary) {
                     builder.value(cursor.value.source().compressed());
                 } else {
-                    byte[] data = cursor.value.source().uncompressed();
-                    try (XContentParser parser = XContentFactory.xContent(data).createParser(data)) {
-                        Map<String, Object> mapping = parser.mapOrdered();
-                        builder.map(mapping);
-                    }
+                    builder.map(XContentHelper.convertToMap(new BytesArray(cursor.value.source().uncompressed()), true).v2());
                 }
             }
             builder.endArray();
@@ -1185,10 +1246,6 @@ public static IndexMetaData fromXContent(XContentParser parser) throws IOExcepti
             }
             return builder.build();
         }
-
-        public static IndexMetaData readFrom(StreamInput in) throws IOException {
-            return PROTO.readFrom(in);
-        }
     }
 
     /**
@@ -1199,6 +1256,7 @@ public static IndexMetaData readFrom(StreamInput in) throws IOException {
      * {@link #isIndexUsingShadowReplicas(org.elasticsearch.common.settings.Settings)}.
      */
     public static boolean isOnSharedFilesystem(Settings settings) {
+        // don't use the setting directly, not to trigger verbose deprecation logging
         return settings.getAsBoolean(SETTING_SHARED_FILESYSTEM, isIndexUsingShadowReplicas(settings));
     }
 
@@ -1208,6 +1266,7 @@ public static boolean isOnSharedFilesystem(Settings settings) {
      * setting for this is <code>false</code>.
      */
     public static boolean isIndexUsingShadowReplicas(Settings settings) {
+        // don't use the setting directly, not to trigger verbose deprecation logging
         return settings.getAsBoolean(SETTING_SHADOW_REPLICAS, false);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
index df53395fe27e1..711d685c1d668 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
@@ -29,6 +29,8 @@
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.joda.DateMathParser;
 import org.elasticsearch.common.joda.FormatDateTimeFormatter;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
@@ -48,13 +50,15 @@
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
-import java.util.concurrent.Callable;
+import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
 public class IndexNameExpressionResolver extends AbstractComponent {
 
     private final List<ExpressionResolver> expressionResolvers;
     private final DateMathExpressionResolver dateMathExpressionResolver;
+    private static final DeprecationLogger DEPRECATION_LOGGER =
+        new DeprecationLogger(Loggers.getLogger(IndexNameExpressionResolver.class));
 
     public IndexNameExpressionResolver(Settings settings) {
         super(settings);
@@ -131,9 +135,9 @@ public Index[] concreteIndices(ClusterState state, IndicesOptions options, Strin
      * @throws IllegalArgumentException if one of the aliases resolve to multiple indices and the provided
      * indices options in the context don't allow such a case.
      */
-    public String[] concreteIndexNames(ClusterState state, IndicesOptions options, long startTime, String... indexExpressions) {
+    public Index[] concreteIndices(ClusterState state, IndicesOptions options, long startTime, String... indexExpressions) {
         Context context = new Context(state, options, startTime);
-        return concreteIndexNames(context, indexExpressions);
+        return concreteIndices(context, indexExpressions);
     }
 
     String[] concreteIndexNames(Context context, String... indexExpressions) {
@@ -159,7 +163,6 @@ Index[] concreteIndices(Context context, String... indexExpressions) {
         if (indexExpressions.length == 1) {
             failNoIndices = options.allowNoIndices() == false;
         }
-
         List<String> expressions = Arrays.asList(indexExpressions);
         for (ExpressionResolver expressionResolver : expressionResolvers) {
             expressions = expressionResolver.resolve(context, expressions);
@@ -269,8 +272,19 @@ public String resolveDateMathExpression(String dateExpression) {
      * the index itself - null is returned. Returns <tt>null</tt> if no filtering is required.
      */
     public String[] filteringAliases(ClusterState state, String index, String... expressions) {
+        return indexAliases(state, index, AliasMetaData::filteringRequired, false, expressions);
+    }
+
+    /**
+     * Iterates through the list of indices and selects the effective list of required aliases for the
+     * given index.
+     * <p>Only aliases where the given predicate tests successfully are returned. If the indices list contains a non-required reference to
+     * the index itself - null is returned. Returns <tt>null</tt> if no filtering is required.
+     */
+    public String[] indexAliases(ClusterState state, String index, Predicate<AliasMetaData> requiredAlias, boolean skipIdentity,
+                                 String... expressions) {
         // expand the aliases wildcard
-        List<String> resolvedExpressions = expressions != null ? Arrays.asList(expressions) : Collections.<String>emptyList();
+        List<String> resolvedExpressions = expressions != null ? Arrays.asList(expressions) : Collections.emptyList();
         Context context = new Context(state, IndicesOptions.lenientExpandOpen(), true);
         for (ExpressionResolver expressionResolver : expressionResolvers) {
             resolvedExpressions = expressionResolver.resolve(context, resolvedExpressions);
@@ -279,54 +293,50 @@ public String[] filteringAliases(ClusterState state, String index, String... exp
         if (isAllIndices(resolvedExpressions)) {
             return null;
         }
+        final IndexMetaData indexMetaData = state.metaData().getIndices().get(index);
+        if (indexMetaData == null) {
+            // Shouldn't happen
+            throw new IndexNotFoundException(index);
+        }
         // optimize for the most common single index/alias scenario
         if (resolvedExpressions.size() == 1) {
             String alias = resolvedExpressions.get(0);
-            IndexMetaData indexMetaData = state.metaData().getIndices().get(index);
-            if (indexMetaData == null) {
-                // Shouldn't happen
-                throw new IndexNotFoundException(index);
-            }
+
             AliasMetaData aliasMetaData = indexMetaData.getAliases().get(alias);
-            boolean filteringRequired = aliasMetaData != null && aliasMetaData.filteringRequired();
-            if (!filteringRequired) {
+            if (aliasMetaData == null || requiredAlias.test(aliasMetaData) == false) {
                 return null;
             }
             return new String[]{alias};
         }
-        List<String> filteringAliases = null;
+        List<String> aliases = null;
         for (String alias : resolvedExpressions) {
             if (alias.equals(index)) {
-                return null;
-            }
-
-            IndexMetaData indexMetaData = state.metaData().getIndices().get(index);
-            if (indexMetaData == null) {
-                // Shouldn't happen
-                throw new IndexNotFoundException(index);
+                if (skipIdentity) {
+                    continue;
+                } else {
+                    return null;
+                }
             }
-
             AliasMetaData aliasMetaData = indexMetaData.getAliases().get(alias);
             // Check that this is an alias for the current index
             // Otherwise - skip it
             if (aliasMetaData != null) {
-                boolean filteringRequired = aliasMetaData.filteringRequired();
-                if (filteringRequired) {
-                    // If filtering required - add it to the list of filters
-                    if (filteringAliases == null) {
-                        filteringAliases = new ArrayList<>();
+                if (requiredAlias.test(aliasMetaData)) {
+                    // If required - add it to the list of aliases
+                    if (aliases == null) {
+                        aliases = new ArrayList<>();
                     }
-                    filteringAliases.add(alias);
+                    aliases.add(alias);
                 } else {
-                    // If not, we have a non filtering alias for this index - no filtering needed
+                    // If not, we have a non required alias for this index - no futher checking needed
                     return null;
                 }
             }
         }
-        if (filteringAliases == null) {
+        if (aliases == null) {
             return null;
         }
-        return filteringAliases.toArray(new String[filteringAliases.size()]);
+        return aliases.toArray(new String[aliases.size()]);
     }
 
     /**
@@ -503,11 +513,11 @@ static final class Context {
             this(state, options, System.currentTimeMillis(), preserveAliases);
         }
 
-        public Context(ClusterState state, IndicesOptions options, long startTime) {
+        Context(ClusterState state, IndicesOptions options, long startTime) {
            this(state, options, startTime, false);
         }
 
-        public Context(ClusterState state, IndicesOptions options, long startTime, boolean preserveAliases) {
+        Context(ClusterState state, IndicesOptions options, long startTime, boolean preserveAliases) {
             this.state = state;
             this.options = options;
             this.startTime = startTime;
@@ -580,6 +590,8 @@ public List<String> resolve(Context context, List<String> expressions) {
 
         private Set<String> innerResolve(Context context, List<String> expressions, IndicesOptions options, MetaData metaData) {
             Set<String> result = null;
+            boolean wildcardSeen = false;
+            boolean plusSeen = false;
             for (int i = 0; i < expressions.size(); i++) {
                 String expression = expressions.get(i);
                 if (aliasOrIndexExists(metaData, expression)) {
@@ -594,36 +606,36 @@ private Set<String> innerResolve(Context context, List<String> expressions, Indi
                 boolean add = true;
                 if (expression.charAt(0) == '+') {
                     // if its the first, add empty result set
+                    plusSeen = true;
                     if (i == 0) {
                         result = new HashSet<>();
                     }
                     expression = expression.substring(1);
                 } else if (expression.charAt(0) == '-') {
-                    // if its the first, fill it with all the indices...
-                    if (i == 0) {
-                        List<String> concreteIndices = resolveEmptyOrTrivialWildcard(options, metaData, false);
-                        result = new HashSet<>(concreteIndices);
+                    // if there is a negation without a wildcard being previously seen, add it verbatim,
+                    // otherwise return the expression
+                    if (wildcardSeen) {
+                        add = false;
+                        expression = expression.substring(1);
+                    } else {
+                        add = true;
                     }
-                    add = false;
-                    expression = expression.substring(1);
+                }
+                if (result == null) {
+                    // add all the previous ones...
+                    result = new HashSet<>(expressions.subList(0, i));
                 }
                 if (!Regex.isSimpleMatchPattern(expression)) {
                     if (!unavailableIgnoredOrExists(options, metaData, expression)) {
                         throw infe(expression);
                     }
-                    if (result != null) {
-                        if (add) {
-                            result.add(expression);
-                        } else {
-                            result.remove(expression);
-                        }
+                    if (add) {
+                        result.add(expression);
+                    } else {
+                        result.remove(expression);
                     }
                     continue;
                 }
-                if (result == null) {
-                    // add all the previous ones...
-                    result = new HashSet<>(expressions.subList(0, i));
-                }
 
                 final IndexMetaData.State excludeState = excludeState(options);
                 final Map<String, AliasOrIndex> matches = matches(metaData, expression);
@@ -637,6 +649,13 @@ private Set<String> innerResolve(Context context, List<String> expressions, Indi
                 if (!noIndicesAllowedOrMatches(options, matches)) {
                     throw infe(expression);
                 }
+
+                if (Regex.isSimpleMatchPattern(expression)) {
+                    wildcardSeen = true;
+                }
+            }
+            if (plusSeen) {
+              DEPRECATION_LOGGER.deprecated("support for '+' as part of index expressions is deprecated");
             }
             return result;
         }
@@ -751,7 +770,7 @@ static final class DateMathExpressionResolver implements ExpressionResolver {
         private final String defaultDateFormatterPattern;
         private final DateTimeFormatter defaultDateFormatter;
 
-        public DateMathExpressionResolver(Settings settings) {
+        DateMathExpressionResolver(Settings settings) {
             String defaultTimeZoneId = settings.get("date_math_expression_resolver.default_time_zone", "UTC");
             this.defaultTimeZone = DateTimeZone.forID(defaultTimeZoneId);
             defaultDateFormatterPattern = settings.get("date_math_expression_resolver.default_date_format", "YYYY.MM.dd");
@@ -850,12 +869,7 @@ String resolveExpression(String expression, final Context context) {
                                 DateTimeFormatter parser = dateFormatter.withZone(timeZone);
                                 FormatDateTimeFormatter formatter = new FormatDateTimeFormatter(dateFormatterPattern, parser, Locale.ROOT);
                                 DateMathParser dateMathParser = new DateMathParser(formatter);
-                                long millis = dateMathParser.parse(mathExpression, new Callable<Long>() {
-                                    @Override
-                                    public Long call() throws Exception {
-                                        return context.getStartTime();
-                                    }
-                                }, false, timeZone);
+                            long millis = dateMathParser.parse(mathExpression, context::getStartTime, false, timeZone);
 
                                 String time = formatter.printer().print(millis);
                                 beforePlaceHolderSb.append(time);
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java
index 13d7e152ba672..50535519e1edd 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java
@@ -21,7 +21,11 @@
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.cluster.AbstractDiffable;
+import org.elasticsearch.cluster.Diff;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.compress.CompressedXContent;
@@ -33,10 +37,12 @@
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 
 /**
@@ -44,12 +50,30 @@
  */
 public class IndexTemplateMetaData extends AbstractDiffable<IndexTemplateMetaData> {
 
-    public static final IndexTemplateMetaData PROTO = IndexTemplateMetaData.builder("").build();
-
     private final String name;
 
     private final int order;
 
+    /**
+     * The version is an arbitrary number managed by the user so that they can easily and quickly verify the existence of a given template.
+     * Expected usage:
+     * <pre><code>
+     * PUT /_template/my_template
+     * {
+     *   "template": "my_index-*",
+     *   "mappings": { ... },
+     *   "version": 1
+     * }
+     * </code></pre>
+     * Then, some process from the user can occasionally verify that the template exists with the appropriate version without having to
+     * check the template's content:
+     * <pre><code>
+     * GET /_template/my_template?filter_path=*.version
+     * </code></pre>
+     */
+    @Nullable
+    private final Integer version;
+
     private final String template;
 
     private final Settings settings;
@@ -61,10 +85,14 @@ public class IndexTemplateMetaData extends AbstractDiffable<IndexTemplateMetaDat
 
     private final ImmutableOpenMap<String, IndexMetaData.Custom> customs;
 
-    public IndexTemplateMetaData(String name, int order, String template, Settings settings, ImmutableOpenMap<String, CompressedXContent> mappings,
-                                 ImmutableOpenMap<String, AliasMetaData> aliases, ImmutableOpenMap<String, IndexMetaData.Custom> customs) {
+    public IndexTemplateMetaData(String name, int order, Integer version,
+                                 String template, Settings settings,
+                                 ImmutableOpenMap<String, CompressedXContent> mappings,
+                                 ImmutableOpenMap<String, AliasMetaData> aliases,
+                                 ImmutableOpenMap<String, IndexMetaData.Custom> customs) {
         this.name = name;
         this.order = order;
+        this.version = version;
         this.template = template;
         this.settings = settings;
         this.mappings = mappings;
@@ -84,6 +112,16 @@ public int getOrder() {
         return order();
     }
 
+    @Nullable
+    public Integer getVersion() {
+        return version();
+    }
+
+    @Nullable
+    public Integer version() {
+        return version;
+    }
+
     public String getName() {
         return this.name;
     }
@@ -150,21 +188,21 @@ public boolean equals(Object o) {
         if (!settings.equals(that.settings)) return false;
         if (!template.equals(that.template)) return false;
 
-        return true;
+        return Objects.equals(version, that.version);
     }
 
     @Override
     public int hashCode() {
         int result = name.hashCode();
         result = 31 * result + order;
+        result = 31 * result + Objects.hashCode(version);
         result = 31 * result + template.hashCode();
         result = 31 * result + settings.hashCode();
         result = 31 * result + mappings.hashCode();
         return result;
     }
 
-    @Override
-    public IndexTemplateMetaData readFrom(StreamInput in) throws IOException {
+    public static IndexTemplateMetaData readFrom(StreamInput in) throws IOException {
         Builder builder = new Builder(in.readString());
         builder.order(in.readInt());
         builder.template(in.readString());
@@ -175,7 +213,7 @@ public IndexTemplateMetaData readFrom(StreamInput in) throws IOException {
         }
         int aliasesSize = in.readVInt();
         for (int i = 0; i < aliasesSize; i++) {
-            AliasMetaData aliasMd = AliasMetaData.Builder.readFrom(in);
+            AliasMetaData aliasMd = new AliasMetaData(in);
             builder.putAlias(aliasMd);
         }
         int customSize = in.readVInt();
@@ -184,9 +222,16 @@ public IndexTemplateMetaData readFrom(StreamInput in) throws IOException {
             IndexMetaData.Custom customIndexMetaData = IndexMetaData.lookupPrototypeSafe(type).readFrom(in);
             builder.putCustom(type, customIndexMetaData);
         }
+        if (in.getVersion().onOrAfter(Version.V_5_0_0_beta1)) {
+            builder.version(in.readOptionalVInt());
+        }
         return builder.build();
     }
 
+    public static Diff<IndexTemplateMetaData> readDiffFrom(StreamInput in) throws IOException {
+        return readDiffFrom(IndexTemplateMetaData::readFrom, in);
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(name);
@@ -207,6 +252,9 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeString(cursor.key);
             cursor.value.writeTo(out);
         }
+        if (out.getVersion().onOrAfter(Version.V_5_0_0_beta1)) {
+            out.writeOptionalVInt(version);
+        }
     }
 
     public static class Builder {
@@ -220,6 +268,8 @@ public static class Builder {
 
         private int order;
 
+        private Integer version;
+
         private String template;
 
         private Settings settings = Settings.Builder.EMPTY_SETTINGS;
@@ -240,6 +290,7 @@ public Builder(String name) {
         public Builder(IndexTemplateMetaData indexTemplateMetaData) {
             this.name = indexTemplateMetaData.name();
             order(indexTemplateMetaData.order());
+            version(indexTemplateMetaData.version());
             template(indexTemplateMetaData.template());
             settings(indexTemplateMetaData.settings());
 
@@ -253,6 +304,11 @@ public Builder order(int order) {
             return this;
         }
 
+        public Builder version(Integer version) {
+            this.version = version;
+            return this;
+        }
+
         public Builder template(String template) {
             this.template = template;
             return this;
@@ -312,14 +368,26 @@ public IndexMetaData.Custom getCustom(String type) {
         }
 
         public IndexTemplateMetaData build() {
-            return new IndexTemplateMetaData(name, order, template, settings, mappings.build(), aliases.build(), customs.build());
+            return new IndexTemplateMetaData(name, order, version, template, settings, mappings.build(), aliases.build(), customs.build());
         }
 
         @SuppressWarnings("unchecked")
-        public static void toXContent(IndexTemplateMetaData indexTemplateMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException {
+        public static void toXContent(IndexTemplateMetaData indexTemplateMetaData, XContentBuilder builder, ToXContent.Params params)
+                throws IOException {
             builder.startObject(indexTemplateMetaData.name());
 
+            toInnerXContent(indexTemplateMetaData, builder, params);
+
+            builder.endObject();
+        }
+
+        public static void toInnerXContent(IndexTemplateMetaData indexTemplateMetaData, XContentBuilder builder, ToXContent.Params params)
+            throws IOException {
+
             builder.field("order", indexTemplateMetaData.order());
+            if (indexTemplateMetaData.version() != null) {
+                builder.field("version", indexTemplateMetaData.version());
+            }
             builder.field("template", indexTemplateMetaData.template());
 
             builder.startObject("settings");
@@ -330,10 +398,7 @@ public static void toXContent(IndexTemplateMetaData indexTemplateMetaData, XCont
                 builder.startObject("mappings");
                 for (ObjectObjectCursor<String, CompressedXContent> cursor : indexTemplateMetaData.mappings()) {
                     byte[] mappingSource = cursor.value.uncompressed();
-                    Map<String, Object> mapping;
-                    try (XContentParser parser = XContentFactory.xContent(mappingSource).createParser(mappingSource)) {;
-                        mapping = parser.map();
-                    }
+                    Map<String, Object> mapping = XContentHelper.convertToMap(new BytesArray(mappingSource), true).v2();
                     if (mapping.size() == 1 && mapping.containsKey(cursor.key)) {
                         // the type name is the root value, reduce it
                         mapping = (Map<String, Object>) mapping.get(cursor.key);
@@ -346,10 +411,7 @@ public static void toXContent(IndexTemplateMetaData indexTemplateMetaData, XCont
                 builder.startArray("mappings");
                 for (ObjectObjectCursor<String, CompressedXContent> cursor : indexTemplateMetaData.mappings()) {
                     byte[] data = cursor.value.uncompressed();
-                    try (XContentParser parser = XContentFactory.xContent(data).createParser(data)) {
-                        Map<String, Object> mapping = parser.mapOrdered();
-                        builder.map(mapping);
-                    }
+                    builder.map(XContentHelper.convertToMap(new BytesArray(data), true).v2());
                 }
                 builder.endArray();
             }
@@ -365,8 +427,6 @@ public static void toXContent(IndexTemplateMetaData indexTemplateMetaData, XCont
                 AliasMetaData.Builder.toXContent(cursor.value, builder, params);
             }
             builder.endObject();
-
-            builder.endObject();
         }
 
         public static IndexTemplateMetaData fromXContent(XContentParser parser, String templateName) throws IOException {
@@ -380,7 +440,9 @@ public static IndexTemplateMetaData fromXContent(XContentParser parser, String t
                 } else if (token == XContentParser.Token.START_OBJECT) {
                     if ("settings".equals(currentFieldName)) {
                         Settings.Builder templateSettingsBuilder = Settings.builder();
-                        templateSettingsBuilder.put(SettingsLoader.Helper.loadNestedFromMap(parser.mapOrdered())).normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX);
+                        templateSettingsBuilder.put(
+                            SettingsLoader.Helper.loadNestedFromMap(parser.mapOrdered()))
+                                                 .normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX);
                         builder.settings(templateSettingsBuilder.build());
                     } else if ("mappings".equals(currentFieldName)) {
                         while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
@@ -388,7 +450,8 @@ public static IndexTemplateMetaData fromXContent(XContentParser parser, String t
                                 currentFieldName = parser.currentName();
                             } else if (token == XContentParser.Token.START_OBJECT) {
                                 String mappingType = currentFieldName;
-                                Map<String, Object> mappingSource = MapBuilder.<String, Object>newMapBuilder().put(mappingType, parser.mapOrdered()).map();
+                                Map<String, Object> mappingSource =
+                                    MapBuilder.<String, Object>newMapBuilder().put(mappingType, parser.mapOrdered()).map();
                                 builder.putMapping(mappingType, XContentFactory.jsonBuilder().map(mappingSource).string());
                             }
                         }
@@ -428,6 +491,8 @@ public static IndexTemplateMetaData fromXContent(XContentParser parser, String t
                         builder.template(parser.text());
                     } else if ("order".equals(currentFieldName)) {
                         builder.order(parser.intValue());
+                    } else if ("version".equals(currentFieldName)) {
+                        builder.version(parser.intValue());
                     }
                 }
             }
@@ -451,10 +516,6 @@ private static String skipTemplateName(XContentParser parser) throws IOException
 
             return null;
         }
-
-        public static IndexTemplateMetaData readFrom(StreamInput in) throws IOException {
-            return PROTO.readFrom(in);
-        }
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java
index 0798dff1c9315..06dfbafaeb040 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java
@@ -19,8 +19,10 @@
 
 package org.elasticsearch.cluster.metadata;
 
+import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.action.TimestampParsingException;
 import org.elasticsearch.cluster.AbstractDiffable;
+import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -29,7 +31,6 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.TimestampFieldMapper;
 
@@ -43,8 +44,6 @@
  */
 public class MappingMetaData extends AbstractDiffable<MappingMetaData> {
 
-    public static final MappingMetaData PROTO = new MappingMetaData();
-
     public static class Routing {
 
         public static final Routing EMPTY = new Routing(false);
@@ -177,10 +176,7 @@ public MappingMetaData(DocumentMapper docMapper) {
 
     public MappingMetaData(CompressedXContent mapping) throws IOException {
         this.source = mapping;
-        Map<String, Object> mappingMap;
-        try (XContentParser parser = XContentHelper.createParser(mapping.compressedReference())) {
-            mappingMap = parser.mapOrdered();
-        }
+        Map<String, Object> mappingMap = XContentHelper.convertToMap(mapping.compressedReference(), true).v2();
         if (mappingMap.size() != 1) {
             throw new IllegalStateException("Can't derive type from mapping, no root type: " + mapping.string());
         }
@@ -220,7 +216,7 @@ private void initMappers(Map<String, Object> withoutType) {
                 String fieldName = entry.getKey();
                 Object fieldNode = entry.getValue();
                 if (fieldName.equals("required")) {
-                    required = lenientNodeBooleanValue(fieldNode);
+                    required = lenientNodeBooleanValue(fieldNode, fieldName);
                 }
             }
             this.routing = new Routing(required);
@@ -237,13 +233,13 @@ private void initMappers(Map<String, Object> withoutType) {
                 String fieldName = entry.getKey();
                 Object fieldNode = entry.getValue();
                 if (fieldName.equals("enabled")) {
-                    enabled = lenientNodeBooleanValue(fieldNode);
+                    enabled = lenientNodeBooleanValue(fieldNode, fieldName);
                 } else if (fieldName.equals("format")) {
                     format = fieldNode.toString();
                 } else if (fieldName.equals("default") && fieldNode != null) {
                     defaultTimestamp = fieldNode.toString();
                 } else if (fieldName.equals("ignore_missing")) {
-                    ignoreMissing = lenientNodeBooleanValue(fieldNode);
+                    ignoreMissing = lenientNodeBooleanValue(fieldNode, fieldName);
                 }
             }
             this.timestamp = new Timestamp(enabled, format, defaultTimestamp, ignoreMissing);
@@ -289,7 +285,7 @@ public boolean hasParentField() {
     /**
      * Converts the serialized compressed form of the mappings into a parsed map.
      */
-    public Map<String, Object> sourceAsMap() throws IOException {
+    public Map<String, Object> sourceAsMap() throws ElasticsearchParseException {
         Map<String, Object> mapping = XContentHelper.convertToMap(source.compressedReference(), true).v2();
         if (mapping.size() == 1 && mapping.containsKey(type())) {
             // the type name is the root value, reduce it
@@ -301,7 +297,7 @@ public Map<String, Object> sourceAsMap() throws IOException {
     /**
      * Converts the serialized compressed form of the mappings into a parsed map.
      */
-    public Map<String, Object> getSourceAsMap() throws IOException {
+    public Map<String, Object> getSourceAsMap() throws ElasticsearchParseException {
         return sourceAsMap();
     }
 
@@ -351,11 +347,11 @@ public int hashCode() {
         return result;
     }
 
-    public MappingMetaData readFrom(StreamInput in) throws IOException {
-        String type = in.readString();
-        CompressedXContent source = CompressedXContent.readCompressedString(in);
+    public MappingMetaData(StreamInput in) throws IOException {
+        type = in.readString();
+        source = CompressedXContent.readCompressedString(in);
         // routing
-        Routing routing = new Routing(in.readBoolean());
+        routing = new Routing(in.readBoolean());
         // timestamp
 
         boolean enabled = in.readBoolean();
@@ -365,9 +361,12 @@ public MappingMetaData readFrom(StreamInput in) throws IOException {
 
         ignoreMissing = in.readOptionalBoolean();
 
-        final Timestamp timestamp = new Timestamp(enabled, format, defaultTimestamp, ignoreMissing);
-        final boolean hasParentField = in.readBoolean();
-        return new MappingMetaData(type, source, routing, timestamp, hasParentField);
+        timestamp = new Timestamp(enabled, format, defaultTimestamp, ignoreMissing);
+        hasParentField = in.readBoolean();
+    }
+
+    public static Diff<MappingMetaData> readDiffFrom(StreamInput in) throws IOException {
+        return readDiffFrom(MappingMetaData::new, in);
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
index fd7e08fec3117..847f520d767ec 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
@@ -28,23 +28,26 @@
 import org.elasticsearch.cluster.Diffable;
 import org.elasticsearch.cluster.DiffableUtils;
 import org.elasticsearch.cluster.InternalClusterInfoService;
+import org.elasticsearch.cluster.NamedDiffable;
+import org.elasticsearch.cluster.NamedDiffableValueSerializer;
 import org.elasticsearch.cluster.block.ClusterBlock;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.collect.HppcMaps;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.loader.SettingsLoader;
-import org.elasticsearch.common.xcontent.FromXContentBuilder;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry.UnknownNamedObjectException;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -57,9 +60,7 @@
 import org.elasticsearch.index.store.IndexStoreConfig;
 import org.elasticsearch.indices.recovery.RecoverySettings;
 import org.elasticsearch.indices.ttl.IndicesTTLService;
-import org.elasticsearch.ingest.IngestMetadata;
 import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.script.ScriptMetaData;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -68,6 +69,7 @@
 import java.util.Comparator;
 import java.util.EnumSet;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -80,9 +82,9 @@
 import static org.elasticsearch.common.settings.Settings.writeSettingsToStream;
 import static org.elasticsearch.common.util.set.Sets.newHashSet;
 
-public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, FromXContentBuilder<MetaData>, ToXContent {
+public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, ToXContent {
 
-    public static final MetaData PROTO = builder().build();
+    private static final Logger logger = Loggers.getLogger(MetaData.class);
 
     public static final String ALL = "_all";
 
@@ -97,56 +99,45 @@ public enum XContentContext {
         SNAPSHOT
     }
 
+    /**
+     * Indicates that this custom metadata will be returned as part of an API call but will not be persisted
+     */
     public static EnumSet<XContentContext> API_ONLY = EnumSet.of(XContentContext.API);
-    public static EnumSet<XContentContext> API_AND_GATEWAY = EnumSet.of(XContentContext.API, XContentContext.GATEWAY);
-    public static EnumSet<XContentContext> API_AND_SNAPSHOT = EnumSet.of(XContentContext.API, XContentContext.SNAPSHOT);
-
-    public interface Custom extends Diffable<Custom>, ToXContent {
-
-        String type();
-
-        Custom fromXContent(XContentParser parser) throws IOException;
-
-        EnumSet<XContentContext> context();
-    }
 
-    public static Map<String, Custom> customPrototypes = new HashMap<>();
+    /**
+     * Indicates that this custom metadata will be returned as part of an API call and will be persisted between
+     * node restarts, but will not be a part of a snapshot global state
+     */
+    public static EnumSet<XContentContext> API_AND_GATEWAY = EnumSet.of(XContentContext.API, XContentContext.GATEWAY);
 
-    static {
-        // register non plugin custom metadata
-        registerPrototype(RepositoriesMetaData.TYPE, RepositoriesMetaData.PROTO);
-        registerPrototype(IngestMetadata.TYPE, IngestMetadata.PROTO);
-        registerPrototype(ScriptMetaData.TYPE, ScriptMetaData.PROTO);
-        registerPrototype(IndexGraveyard.TYPE, IndexGraveyard.PROTO);
-    }
+    /**
+     * Indicates that this custom metadata will be returned as part of an API call and stored as a part of
+     * a snapshot global state, but will not be persisted between node restarts
+     */
+    public static EnumSet<XContentContext> API_AND_SNAPSHOT = EnumSet.of(XContentContext.API, XContentContext.SNAPSHOT);
 
     /**
-     * Register a custom index meta data factory. Make sure to call it from a static block.
+     * Indicates that this custom metadata will be returned as part of an API call, stored as a part of
+     * a snapshot global state, and will be persisted between node restarts
      */
-    public static void registerPrototype(String type, Custom proto) {
-        customPrototypes.put(type, proto);
-    }
+    public static EnumSet<XContentContext> ALL_CONTEXTS = EnumSet.allOf(XContentContext.class);
 
-    @Nullable
-    public static <T extends Custom> T lookupPrototype(String type) {
-        //noinspection unchecked
-        return (T) customPrototypes.get(type);
-    }
+    public interface Custom extends NamedDiffable<Custom>, ToXContent {
 
-    public static <T extends Custom> T lookupPrototypeSafe(String type) {
-        //noinspection unchecked
-        T proto = (T) customPrototypes.get(type);
-        if (proto == null) {
-            throw new IllegalArgumentException("No custom metadata prototype registered for type [" + type + "], node likely missing plugins");
-        }
-        return proto;
+        EnumSet<XContentContext> context();
     }
 
-
     public static final Setting<Boolean> SETTING_READ_ONLY_SETTING =
         Setting.boolSetting("cluster.blocks.read_only", false, Property.Dynamic, Property.NodeScope);
 
-    public static final ClusterBlock CLUSTER_READ_ONLY_BLOCK = new ClusterBlock(6, "cluster read-only (api)", false, false, RestStatus.FORBIDDEN, EnumSet.of(ClusterBlockLevel.WRITE, ClusterBlockLevel.METADATA_WRITE));
+    public static final ClusterBlock CLUSTER_READ_ONLY_BLOCK = new ClusterBlock(6, "cluster read-only (api)", false, false,
+        false, RestStatus.FORBIDDEN, EnumSet.of(ClusterBlockLevel.WRITE, ClusterBlockLevel.METADATA_WRITE));
+
+    public static final Setting<Boolean> SETTING_READ_ONLY_ALLOW_DELETE_SETTING =
+        Setting.boolSetting("cluster.blocks.read_only_allow_delete", false, Property.Dynamic, Property.NodeScope);
+
+    public static final ClusterBlock CLUSTER_READ_ONLY_ALLOW_DELETE_BLOCK = new ClusterBlock(13, "cluster read-only / allow delete (api)",
+        false, false, true, RestStatus.FORBIDDEN, EnumSet.of(ClusterBlockLevel.WRITE, ClusterBlockLevel.METADATA_WRITE));
 
     public static final MetaData EMPTY_META_DATA = builder().build();
 
@@ -158,6 +149,8 @@ public static <T extends Custom> T lookupPrototypeSafe(String type) {
 
     public static final String GLOBAL_STATE_FILE_PREFIX = "global-";
 
+    private static final NamedDiffableValueSerializer<Custom> CUSTOM_VALUE_SERIALIZER = new NamedDiffableValueSerializer<>(Custom.class);
+
     private final String clusterUUID;
     private final long version;
 
@@ -579,14 +572,14 @@ public static boolean isGlobalStateEquals(MetaData metaData1, MetaData metaData2
         // Check if any persistent metadata needs to be saved
         int customCount1 = 0;
         for (ObjectObjectCursor<String, Custom> cursor : metaData1.customs) {
-            if (customPrototypes.get(cursor.key).context().contains(XContentContext.GATEWAY)) {
+            if (cursor.value.context().contains(XContentContext.GATEWAY)) {
                 if (!cursor.value.equals(metaData2.custom(cursor.key))) return false;
                 customCount1++;
             }
         }
         int customCount2 = 0;
-        for (ObjectObjectCursor<String, Custom> cursor : metaData2.customs) {
-            if (customPrototypes.get(cursor.key).context().contains(XContentContext.GATEWAY)) {
+        for (ObjectCursor<Custom> cursor : metaData2.customs.values()) {
+            if (cursor.value.context().contains(XContentContext.GATEWAY)) {
                 customCount2++;
             }
         }
@@ -599,13 +592,11 @@ public Diff<MetaData> diff(MetaData previousState) {
         return new MetaDataDiff(previousState, this);
     }
 
-    @Override
-    public Diff<MetaData> readDiffFrom(StreamInput in) throws IOException {
+    public static Diff<MetaData> readDiffFrom(StreamInput in) throws IOException {
         return new MetaDataDiff(in);
     }
 
-    @Override
-    public MetaData fromXContent(XContentParser parser, ParseFieldMatcher parseFieldMatcher) throws IOException {
+    public static MetaData fromXContent(XContentParser parser) throws IOException {
         return Builder.fromXContent(parser);
     }
 
@@ -627,35 +618,26 @@ private static class MetaDataDiff implements Diff<MetaData> {
         private Diff<ImmutableOpenMap<String, IndexTemplateMetaData>> templates;
         private Diff<ImmutableOpenMap<String, Custom>> customs;
 
-        public MetaDataDiff(MetaData before, MetaData after) {
+        MetaDataDiff(MetaData before, MetaData after) {
             clusterUUID = after.clusterUUID;
             version = after.version;
             transientSettings = after.transientSettings;
             persistentSettings = after.persistentSettings;
             indices = DiffableUtils.diff(before.indices, after.indices, DiffableUtils.getStringKeySerializer());
             templates = DiffableUtils.diff(before.templates, after.templates, DiffableUtils.getStringKeySerializer());
-            customs = DiffableUtils.diff(before.customs, after.customs, DiffableUtils.getStringKeySerializer());
+            customs = DiffableUtils.diff(before.customs, after.customs, DiffableUtils.getStringKeySerializer(), CUSTOM_VALUE_SERIALIZER);
         }
 
-        public MetaDataDiff(StreamInput in) throws IOException {
+        MetaDataDiff(StreamInput in) throws IOException {
             clusterUUID = in.readString();
             version = in.readLong();
             transientSettings = Settings.readSettingsFromStream(in);
             persistentSettings = Settings.readSettingsFromStream(in);
-            indices = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), IndexMetaData.PROTO);
-            templates = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), IndexTemplateMetaData.PROTO);
-            customs = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(),
-                    new DiffableUtils.DiffableValueSerializer<String, Custom>() {
-                @Override
-                public Custom read(StreamInput in, String key) throws IOException {
-                    return lookupPrototypeSafe(key).readFrom(in);
-                }
-
-                @Override
-                public Diff<Custom> readDiff(StreamInput in, String key) throws IOException {
-                    return lookupPrototypeSafe(key).readDiffFrom(in);
-                }
-            });
+            indices = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), IndexMetaData::readFrom,
+                IndexMetaData::readDiffFrom);
+            templates = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), IndexTemplateMetaData::readFrom,
+                IndexTemplateMetaData::readDiffFrom);
+            customs = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), CUSTOM_VALUE_SERIALIZER);
         }
 
         @Override
@@ -683,8 +665,7 @@ public MetaData apply(MetaData part) {
         }
     }
 
-    @Override
-    public MetaData readFrom(StreamInput in) throws IOException {
+    public static MetaData readFrom(StreamInput in) throws IOException {
         Builder builder = new Builder();
         builder.version = in.readLong();
         builder.clusterUUID = in.readString();
@@ -692,17 +673,16 @@ public MetaData readFrom(StreamInput in) throws IOException {
         builder.persistentSettings(readSettingsFromStream(in));
         int size = in.readVInt();
         for (int i = 0; i < size; i++) {
-            builder.put(IndexMetaData.Builder.readFrom(in), false);
+            builder.put(IndexMetaData.readFrom(in), false);
         }
         size = in.readVInt();
         for (int i = 0; i < size; i++) {
-            builder.put(IndexTemplateMetaData.Builder.readFrom(in));
+            builder.put(IndexTemplateMetaData.readFrom(in));
         }
         int customSize = in.readVInt();
         for (int i = 0; i < customSize; i++) {
-            String type = in.readString();
-            Custom customIndexMetaData = lookupPrototypeSafe(type).readFrom(in);
-            builder.putCustom(type, customIndexMetaData);
+            Custom customIndexMetaData = in.readNamedWriteable(Custom.class);
+            builder.putCustom(customIndexMetaData.getWriteableName(), customIndexMetaData);
         }
         return builder.build();
     }
@@ -721,10 +701,18 @@ public void writeTo(StreamOutput out) throws IOException {
         for (ObjectCursor<IndexTemplateMetaData> cursor : templates.values()) {
             cursor.value.writeTo(out);
         }
-        out.writeVInt(customs.size());
-        for (ObjectObjectCursor<String, Custom> cursor : customs) {
-            out.writeString(cursor.key);
-            cursor.value.writeTo(out);
+        // filter out custom states not supported by the other node
+        int numberOfCustoms = 0;
+        for (ObjectCursor<Custom> cursor : customs.values()) {
+            if (out.getVersion().onOrAfter(cursor.value.getMinimalSupportedVersion())) {
+                numberOfCustoms++;
+            }
+        }
+        out.writeVInt(numberOfCustoms);
+        for (ObjectCursor<Custom> cursor : customs.values()) {
+            if (out.getVersion().onOrAfter(cursor.value.getMinimalSupportedVersion())) {
+                out.writeNamedWriteable(cursor.value);
+            }
         }
     }
 
@@ -1012,55 +1000,70 @@ public MetaData build() {
             //    while these datastructures aren't even used.
             // 2) The aliasAndIndexLookup can be updated instead of rebuilding it all the time.
 
-            // build all concrete indices arrays:
-            // TODO: I think we can remove these arrays. it isn't worth the effort, for operations on all indices.
-            // When doing an operation across all indices, most of the time is spent on actually going to all shards and
-            // do the required operations, the bottleneck isn't resolving expressions into concrete indices.
-            List<String> allIndicesLst = new ArrayList<>();
-            for (ObjectCursor<IndexMetaData> cursor : indices.values()) {
-                allIndicesLst.add(cursor.value.getIndex().getName());
-            }
-            String[] allIndices = allIndicesLst.toArray(new String[allIndicesLst.size()]);
-
-            List<String> allOpenIndicesLst = new ArrayList<>();
-            List<String> allClosedIndicesLst = new ArrayList<>();
+            final Set<String> allIndices = new HashSet<>(indices.size());
+            final List<String> allOpenIndices = new ArrayList<>();
+            final List<String> allClosedIndices = new ArrayList<>();
+            final Set<String> duplicateAliasesIndices = new HashSet<>();
             for (ObjectCursor<IndexMetaData> cursor : indices.values()) {
-                IndexMetaData indexMetaData = cursor.value;
+                final IndexMetaData indexMetaData = cursor.value;
+                final String name = indexMetaData.getIndex().getName();
+                boolean added = allIndices.add(name);
+                assert added : "double index named [" + name + "]";
                 if (indexMetaData.getState() == IndexMetaData.State.OPEN) {
-                    allOpenIndicesLst.add(indexMetaData.getIndex().getName());
+                    allOpenIndices.add(indexMetaData.getIndex().getName());
                 } else if (indexMetaData.getState() == IndexMetaData.State.CLOSE) {
-                    allClosedIndicesLst.add(indexMetaData.getIndex().getName());
+                    allClosedIndices.add(indexMetaData.getIndex().getName());
+                }
+                indexMetaData.getAliases().keysIt().forEachRemaining(duplicateAliasesIndices::add);
+            }
+            duplicateAliasesIndices.retainAll(allIndices);
+            if (duplicateAliasesIndices.isEmpty() == false) {
+                // iterate again and constructs a helpful message
+                ArrayList<String> duplicates = new ArrayList<>();
+                for (ObjectCursor<IndexMetaData> cursor : indices.values()) {
+                    for (String alias: duplicateAliasesIndices) {
+                        if (cursor.value.getAliases().containsKey(alias)) {
+                            duplicates.add(alias + " (alias of " + cursor.value.getIndex() + ")");
+                        }
+                    }
                 }
+                assert duplicates.size() > 0;
+                throw new IllegalStateException("index and alias names need to be unique, but the following duplicates were found ["
+                    + Strings.collectionToCommaDelimitedString(duplicates)+ "]");
+
             }
-            String[] allOpenIndices = allOpenIndicesLst.toArray(new String[allOpenIndicesLst.size()]);
-            String[] allClosedIndices = allClosedIndicesLst.toArray(new String[allClosedIndicesLst.size()]);
 
             // build all indices map
             SortedMap<String, AliasOrIndex> aliasAndIndexLookup = new TreeMap<>();
             for (ObjectCursor<IndexMetaData> cursor : indices.values()) {
                 IndexMetaData indexMetaData = cursor.value;
-                aliasAndIndexLookup.put(indexMetaData.getIndex().getName(), new AliasOrIndex.Index(indexMetaData));
+                AliasOrIndex existing = aliasAndIndexLookup.put(indexMetaData.getIndex().getName(), new AliasOrIndex.Index(indexMetaData));
+                assert existing == null : "duplicate for " + indexMetaData.getIndex();
 
                 for (ObjectObjectCursor<String, AliasMetaData> aliasCursor : indexMetaData.getAliases()) {
                     AliasMetaData aliasMetaData = aliasCursor.value;
-                    AliasOrIndex aliasOrIndex = aliasAndIndexLookup.get(aliasMetaData.getAlias());
-                    if (aliasOrIndex == null) {
-                        aliasOrIndex = new AliasOrIndex.Alias(aliasMetaData, indexMetaData);
-                        aliasAndIndexLookup.put(aliasMetaData.getAlias(), aliasOrIndex);
-                    } else if (aliasOrIndex instanceof AliasOrIndex.Alias) {
-                        AliasOrIndex.Alias alias = (AliasOrIndex.Alias) aliasOrIndex;
-                        alias.addIndex(indexMetaData);
-                    } else if (aliasOrIndex instanceof AliasOrIndex.Index) {
-                        AliasOrIndex.Index index = (AliasOrIndex.Index) aliasOrIndex;
-                        throw new IllegalStateException("index and alias names need to be unique, but alias [" + aliasMetaData.getAlias() + "] and index " + index.getIndex().getIndex() + " have the same name");
-                    } else {
-                        throw new IllegalStateException("unexpected alias [" + aliasMetaData.getAlias() + "][" + aliasOrIndex + "]");
-                    }
+                    aliasAndIndexLookup.compute(aliasMetaData.getAlias(), (aliasName, alias) -> {
+                        if (alias == null) {
+                            return new AliasOrIndex.Alias(aliasMetaData, indexMetaData);
+                        } else {
+                            assert alias instanceof AliasOrIndex.Alias : alias.getClass().getName();
+                            ((AliasOrIndex.Alias) alias).addIndex(indexMetaData);
+                            return alias;
+                        }
+                    });
                 }
             }
             aliasAndIndexLookup = Collections.unmodifiableSortedMap(aliasAndIndexLookup);
+            // build all concrete indices arrays:
+            // TODO: I think we can remove these arrays. it isn't worth the effort, for operations on all indices.
+            // When doing an operation across all indices, most of the time is spent on actually going to all shards and
+            // do the required operations, the bottleneck isn't resolving expressions into concrete indices.
+            String[] allIndicesArray = allIndices.toArray(new String[allIndices.size()]);
+            String[] allOpenIndicesArray = allOpenIndices.toArray(new String[allOpenIndices.size()]);
+            String[] allClosedIndicesArray = allClosedIndices.toArray(new String[allClosedIndices.size()]);
+
             return new MetaData(clusterUUID, version, transientSettings, persistentSettings, indices.build(), templates.build(),
-                                customs.build(), allIndices, allOpenIndices, allClosedIndices, aliasAndIndexLookup);
+                                customs.build(), allIndicesArray, allOpenIndicesArray, allClosedIndicesArray, aliasAndIndexLookup);
         }
 
         public static String toXContent(MetaData metaData) throws IOException {
@@ -1079,7 +1082,7 @@ public static void toXContent(MetaData metaData, XContentBuilder builder, ToXCon
             builder.field("version", metaData.version());
             builder.field("cluster_uuid", metaData.clusterUUID);
 
-            if (!metaData.persistentSettings().getAsMap().isEmpty()) {
+            if (!metaData.persistentSettings().isEmpty()) {
                 builder.startObject("settings");
                 for (Map.Entry<String, String> entry : metaData.persistentSettings().getAsMap().entrySet()) {
                     builder.field(entry.getKey(), entry.getValue());
@@ -1087,7 +1090,7 @@ public static void toXContent(MetaData metaData, XContentBuilder builder, ToXCon
                 builder.endObject();
             }
 
-            if (context == XContentContext.API && !metaData.transientSettings().getAsMap().isEmpty()) {
+            if (context == XContentContext.API && !metaData.transientSettings().isEmpty()) {
                 builder.startObject("transient_settings");
                 for (Map.Entry<String, String> entry : metaData.transientSettings().getAsMap().entrySet()) {
                     builder.field(entry.getKey(), entry.getValue());
@@ -1110,8 +1113,7 @@ public static void toXContent(MetaData metaData, XContentBuilder builder, ToXCon
             }
 
             for (ObjectObjectCursor<String, Custom> cursor : metaData.customs()) {
-                Custom proto = lookupPrototypeSafe(cursor.key);
-                if (proto.context().contains(context)) {
+                if (cursor.value.context().contains(context)) {
                     builder.startObject(cursor.key);
                     cursor.value.toXContent(builder, params);
                     builder.endObject();
@@ -1162,14 +1164,12 @@ public static MetaData fromXContent(XContentParser parser) throws IOException {
                             builder.put(IndexTemplateMetaData.Builder.fromXContent(parser, parser.currentName()));
                         }
                     } else {
-                        // check if its a custom index metadata
-                        Custom proto = lookupPrototype(currentFieldName);
-                        if (proto == null) {
-                            //TODO warn
+                        try {
+                            Custom custom = parser.namedObject(Custom.class, currentFieldName, null);
+                            builder.putCustom(custom.getWriteableName(), custom);
+                        } catch (UnknownNamedObjectException ex) {
+                            logger.warn("Skipping unknown custom object with type {}", currentFieldName);
                             parser.skipChildren();
-                        } else {
-                            Custom custom = proto.fromXContent(parser);
-                            builder.putCustom(custom.type(), custom);
                         }
                     }
                 } else if (token.isValue()) {
@@ -1186,10 +1186,6 @@ public static MetaData fromXContent(XContentParser parser) throws IOException {
             }
             return builder.build();
         }
-
-        public static MetaData readFrom(StreamInput in) throws IOException {
-            return PROTO.readFrom(in);
-        }
     }
 
     private static final ToXContent.Params FORMAT_PARAMS;
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
index 8b3cbec0ebded..a27c42f55dcb9 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
@@ -25,6 +25,7 @@
 import org.apache.logging.log4j.util.Supplier;
 import org.apache.lucene.util.CollectionUtil;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ResourceAlreadyExistsException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.alias.Alias;
@@ -46,7 +47,6 @@
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
-import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.Strings;
@@ -59,20 +59,20 @@
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.IndexService;
-import org.elasticsearch.index.NodeServicesProvider;
 import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.MapperService.MergeReason;
 import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.indices.IndexAlreadyExistsException;
 import org.elasticsearch.indices.IndexCreationException;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.InvalidIndexNameException;
+import org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
@@ -89,8 +89,10 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.BiFunction;
 import java.util.function.Predicate;
 
+import static org.elasticsearch.action.support.ContextPreservingActionListener.wrapPreservingContext;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_CREATION_DATE;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_INDEX_UUID;
@@ -110,45 +112,60 @@ public class MetaDataCreateIndexService extends AbstractComponent {
     private final AllocationService allocationService;
     private final AliasValidator aliasValidator;
     private final Environment env;
-    private final NodeServicesProvider nodeServicesProvider;
     private final IndexScopedSettings indexScopedSettings;
     private final ActiveShardsObserver activeShardsObserver;
+    private final NamedXContentRegistry xContentRegistry;
+    private final ThreadPool threadPool;
 
     @Inject
     public MetaDataCreateIndexService(Settings settings, ClusterService clusterService,
                                       IndicesService indicesService, AllocationService allocationService,
                                       AliasValidator aliasValidator, Environment env,
-                                      NodeServicesProvider nodeServicesProvider, IndexScopedSettings indexScopedSettings,
-                                      ThreadPool threadPool) {
+                                      IndexScopedSettings indexScopedSettings, ThreadPool threadPool,
+                                      NamedXContentRegistry xContentRegistry) {
         super(settings);
         this.clusterService = clusterService;
         this.indicesService = indicesService;
         this.allocationService = allocationService;
         this.aliasValidator = aliasValidator;
         this.env = env;
-        this.nodeServicesProvider = nodeServicesProvider;
         this.indexScopedSettings = indexScopedSettings;
         this.activeShardsObserver = new ActiveShardsObserver(settings, clusterService, threadPool);
+        this.threadPool = threadPool;
+        this.xContentRegistry = xContentRegistry;
     }
 
+    /**
+     * Validate the name for an index against some static rules and a cluster state.
+     */
     public static void validateIndexName(String index, ClusterState state) {
+        validateIndexOrAliasName(index, InvalidIndexNameException::new);
+        if (!index.toLowerCase(Locale.ROOT).equals(index)) {
+            throw new InvalidIndexNameException(index, "must be lowercase");
+        }
         if (state.routingTable().hasIndex(index)) {
-            throw new IndexAlreadyExistsException(state.routingTable().index(index).getIndex());
+            throw new ResourceAlreadyExistsException(state.routingTable().index(index).getIndex());
         }
         if (state.metaData().hasIndex(index)) {
-            throw new IndexAlreadyExistsException(state.metaData().index(index).getIndex());
+            throw new ResourceAlreadyExistsException(state.metaData().index(index).getIndex());
+        }
+        if (state.metaData().hasAlias(index)) {
+            throw new InvalidIndexNameException(index, "already exists as alias");
         }
+    }
+
+    /**
+     * Validate the name for an index or alias against some static rules.
+     */
+    public static void validateIndexOrAliasName(String index, BiFunction<String, String, ? extends RuntimeException> exceptionCtor) {
         if (!Strings.validFileName(index)) {
-            throw new InvalidIndexNameException(index, "must not contain the following characters " + Strings.INVALID_FILENAME_CHARS);
+            throw exceptionCtor.apply(index, "must not contain the following characters " + Strings.INVALID_FILENAME_CHARS);
         }
         if (index.contains("#")) {
-            throw new InvalidIndexNameException(index, "must not contain '#'");
+            throw exceptionCtor.apply(index, "must not contain '#'");
         }
         if (index.charAt(0) == '_' || index.charAt(0) == '-' || index.charAt(0) == '+') {
-            throw new InvalidIndexNameException(index, "must not start with '_', '-', or '+'");
-        }
-        if (!index.toLowerCase(Locale.ROOT).equals(index)) {
-            throw new InvalidIndexNameException(index, "must be lowercase");
+            throw exceptionCtor.apply(index, "must not start with '_', '-', or '+'");
         }
         int byteCount = 0;
         try {
@@ -158,15 +175,10 @@ public static void validateIndexName(String index, ClusterState state) {
             throw new ElasticsearchException("Unable to determine length of index name", e);
         }
         if (byteCount > MAX_INDEX_NAME_BYTES) {
-            throw new InvalidIndexNameException(index,
-                    "index name is too long, (" + byteCount +
-                            " > " + MAX_INDEX_NAME_BYTES + ")");
-        }
-        if (state.metaData().hasAlias(index)) {
-            throw new InvalidIndexNameException(index, "already exists as alias");
+            throw exceptionCtor.apply(index, "index name is too long, (" + byteCount + " > " + MAX_INDEX_NAME_BYTES + ")");
         }
         if (index.equals(".") || index.equals("..")) {
-            throw new InvalidIndexNameException(index, "must not be '.' or '..'");
+            throw exceptionCtor.apply(index, "must not be '.' or '..'");
         }
     }
 
@@ -210,7 +222,9 @@ private void onlyCreateIndex(final CreateIndexClusterStateUpdateRequest request,
         request.settings(updatedSettingsBuilder.build());
 
         clusterService.submitStateUpdateTask("create-index [" + request.index() + "], cause [" + request.cause() + "]",
-                new AckedClusterStateUpdateTask<ClusterStateUpdateResponse>(Priority.URGENT, request, listener) {
+                new AckedClusterStateUpdateTask<ClusterStateUpdateResponse>(Priority.URGENT, request,
+                    wrapPreservingContext(listener, threadPool.getThreadContext())) {
+
                     @Override
                     protected ClusterStateUpdateResponse newResponse(boolean acknowledged) {
                         return new ClusterStateUpdateResponse(acknowledged);
@@ -219,7 +233,8 @@ protected ClusterStateUpdateResponse newResponse(boolean acknowledged) {
                     @Override
                     public ClusterState execute(ClusterState currentState) throws Exception {
                         Index createdIndex = null;
-                        String removalReason = null;
+                        String removalExtraInfo = null;
+                        IndexRemovalReason removalReason = IndexRemovalReason.FAILURE;
                         try {
                             validate(request, currentState);
 
@@ -241,62 +256,71 @@ public ClusterState execute(ClusterState currentState) throws Exception {
                             List<String> templateNames = new ArrayList<>();
 
                             for (Map.Entry<String, String> entry : request.mappings().entrySet()) {
-                                mappings.put(entry.getKey(), MapperService.parseMapping(entry.getValue()));
+                                mappings.put(entry.getKey(), MapperService.parseMapping(xContentRegistry, entry.getValue()));
                             }
 
                             for (Map.Entry<String, Custom> entry : request.customs().entrySet()) {
                                 customs.put(entry.getKey(), entry.getValue());
                             }
 
-                            // apply templates, merging the mappings into the request mapping if exists
-                            for (IndexTemplateMetaData template : templates) {
-                                templateNames.add(template.getName());
-                                for (ObjectObjectCursor<String, CompressedXContent> cursor : template.mappings()) {
-                                    if (mappings.containsKey(cursor.key)) {
-                                        XContentHelper.mergeDefaults(mappings.get(cursor.key), MapperService.parseMapping(cursor.value.string()));
-                                    } else {
-                                        mappings.put(cursor.key, MapperService.parseMapping(cursor.value.string()));
-                                    }
-                                }
-                                // handle custom
-                                for (ObjectObjectCursor<String, Custom> cursor : template.customs()) {
-                                    String type = cursor.key;
-                                    IndexMetaData.Custom custom = cursor.value;
-                                    IndexMetaData.Custom existing = customs.get(type);
-                                    if (existing == null) {
-                                        customs.put(type, custom);
-                                    } else {
-                                        IndexMetaData.Custom merged = existing.mergeWith(custom);
-                                        customs.put(type, merged);
-                                    }
-                                }
-                                //handle aliases
-                                for (ObjectObjectCursor<String, AliasMetaData> cursor : template.aliases()) {
-                                    AliasMetaData aliasMetaData = cursor.value;
-                                    //if an alias with same name came with the create index request itself,
-                                    // ignore this one taken from the index template
-                                    if (request.aliases().contains(new Alias(aliasMetaData.alias()))) {
-                                        continue;
+                            final Index shrinkFromIndex = request.shrinkFrom();
+
+                            if (shrinkFromIndex == null) {
+                                // apply templates, merging the mappings into the request mapping if exists
+                                for (IndexTemplateMetaData template : templates) {
+                                    templateNames.add(template.getName());
+                                    for (ObjectObjectCursor<String, CompressedXContent> cursor : template.mappings()) {
+                                        String mappingString = cursor.value.string();
+                                        if (mappings.containsKey(cursor.key)) {
+                                            XContentHelper.mergeDefaults(mappings.get(cursor.key),
+                                                MapperService.parseMapping(xContentRegistry, mappingString));
+                                        } else {
+                                            mappings.put(cursor.key,
+                                                MapperService.parseMapping(xContentRegistry, mappingString));
+                                        }
                                     }
-                                    //if an alias with same name was already processed, ignore this one
-                                    if (templatesAliases.containsKey(cursor.key)) {
-                                        continue;
+                                    // handle custom
+                                    for (ObjectObjectCursor<String, Custom> cursor : template.customs()) {
+                                        String type = cursor.key;
+                                        IndexMetaData.Custom custom = cursor.value;
+                                        IndexMetaData.Custom existing = customs.get(type);
+                                        if (existing == null) {
+                                            customs.put(type, custom);
+                                        } else {
+                                            IndexMetaData.Custom merged = existing.mergeWith(custom);
+                                            customs.put(type, merged);
+                                        }
                                     }
-
-                                    //Allow templatesAliases to be templated by replacing a token with the name of the index that we are applying it to
-                                    if (aliasMetaData.alias().contains("{index}")) {
-                                        String templatedAlias = aliasMetaData.alias().replace("{index}", request.index());
-                                        aliasMetaData = AliasMetaData.newAliasMetaData(aliasMetaData, templatedAlias);
+                                    //handle aliases
+                                    for (ObjectObjectCursor<String, AliasMetaData> cursor : template.aliases()) {
+                                        AliasMetaData aliasMetaData = cursor.value;
+                                        //if an alias with same name came with the create index request itself,
+                                        // ignore this one taken from the index template
+                                        if (request.aliases().contains(new Alias(aliasMetaData.alias()))) {
+                                            continue;
+                                        }
+                                        //if an alias with same name was already processed, ignore this one
+                                        if (templatesAliases.containsKey(cursor.key)) {
+                                            continue;
+                                        }
+
+                                        //Allow templatesAliases to be templated by replacing a token with the name of the index that we are applying it to
+                                        if (aliasMetaData.alias().contains("{index}")) {
+                                            String templatedAlias = aliasMetaData.alias().replace("{index}", request.index());
+                                            aliasMetaData = AliasMetaData.newAliasMetaData(aliasMetaData, templatedAlias);
+                                        }
+
+                                        aliasValidator.validateAliasMetaData(aliasMetaData, request.index(), currentState.metaData());
+                                        templatesAliases.put(aliasMetaData.alias(), aliasMetaData);
                                     }
-
-                                    aliasValidator.validateAliasMetaData(aliasMetaData, request.index(), currentState.metaData());
-                                    templatesAliases.put(aliasMetaData.alias(), aliasMetaData);
                                 }
                             }
                             Settings.Builder indexSettingsBuilder = Settings.builder();
-                            // apply templates, here, in reverse order, since first ones are better matching
-                            for (int i = templates.size() - 1; i >= 0; i--) {
-                                indexSettingsBuilder.put(templates.get(i).settings());
+                            if (shrinkFromIndex == null) {
+                                // apply templates, here, in reverse order, since first ones are better matching
+                                for (int i = templates.size() - 1; i >= 0; i--) {
+                                    indexSettingsBuilder.put(templates.get(i).settings());
+                                }
                             }
                             // now, put the request settings, so they override templates
                             indexSettingsBuilder.put(request.settings());
@@ -312,29 +336,35 @@ public ClusterState execute(ClusterState currentState) throws Exception {
 
                             if (indexSettingsBuilder.get(SETTING_VERSION_CREATED) == null) {
                                 DiscoveryNodes nodes = currentState.nodes();
-                                final Version createdVersion = Version.smallest(Version.CURRENT, nodes.getSmallestNonClientNodeVersion());
+                                final Version createdVersion = Version.min(Version.CURRENT, nodes.getSmallestNonClientNodeVersion());
                                 indexSettingsBuilder.put(SETTING_VERSION_CREATED, createdVersion);
                             }
 
                             if (indexSettingsBuilder.get(SETTING_CREATION_DATE) == null) {
                                 indexSettingsBuilder.put(SETTING_CREATION_DATE, new DateTime(DateTimeZone.UTC).getMillis());
                             }
-
+                            indexSettingsBuilder.put(IndexMetaData.SETTING_INDEX_PROVIDED_NAME, request.getProvidedName());
                             indexSettingsBuilder.put(SETTING_INDEX_UUID, UUIDs.randomBase64UUID());
-                            final Index shrinkFromIndex = request.shrinkFrom();
-                            int routingNumShards = IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(indexSettingsBuilder.build());;
-                            if (shrinkFromIndex != null) {
-                                prepareShrinkIndexSettings(currentState, mappings.keySet(), indexSettingsBuilder, shrinkFromIndex,
-                                    request.index());
-                                IndexMetaData sourceMetaData = currentState.metaData().getIndexSafe(shrinkFromIndex);
+                            final IndexMetaData.Builder tmpImdBuilder = IndexMetaData.builder(request.index());
+
+                            final int routingNumShards;
+                            if (shrinkFromIndex == null) {
+                                routingNumShards = IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(indexSettingsBuilder.build());
+                            } else {
+                                final IndexMetaData sourceMetaData = currentState.metaData().getIndexSafe(shrinkFromIndex);
                                 routingNumShards = sourceMetaData.getRoutingNumShards();
                             }
+                            tmpImdBuilder.setRoutingNumShards(routingNumShards);
+
+                            if (shrinkFromIndex != null) {
+                                prepareShrinkIndexSettings(
+                                        currentState, mappings.keySet(), indexSettingsBuilder, shrinkFromIndex, request.index());
+                            }
+                            final Settings actualIndexSettings = indexSettingsBuilder.build();
+                            tmpImdBuilder.settings(actualIndexSettings);
 
-                            Settings actualIndexSettings = indexSettingsBuilder.build();
-                            IndexMetaData.Builder tmpImdBuilder = IndexMetaData.builder(request.index())
-                                .setRoutingNumShards(routingNumShards);
                             // Set up everything, now locally create the index to see that things are ok, and apply
-                            final IndexMetaData tmpImd = tmpImdBuilder.settings(actualIndexSettings).build();
+                            final IndexMetaData tmpImd = tmpImdBuilder.build();
                             ActiveShardCount waitForActiveShards = request.waitForActiveShards();
                             if (waitForActiveShards == ActiveShardCount.DEFAULT) {
                                 waitForActiveShards = tmpImd.getWaitForActiveShards();
@@ -345,26 +375,29 @@ public ClusterState execute(ClusterState currentState) throws Exception {
                                                                    (tmpImd.getNumberOfReplicas() + 1) + "]");
                             }
                             // create the index here (on the master) to validate it can be created, as well as adding the mapping
-                            final IndexService indexService = indicesService.createIndex(nodeServicesProvider, tmpImd, Collections.emptyList());
+                            final IndexService indexService = indicesService.createIndex(tmpImd, Collections.emptyList());
                             createdIndex = indexService.index();
                             // now add the mappings
                             MapperService mapperService = indexService.mapperService();
                             try {
-                                mapperService.merge(mappings, request.updateAllTypes());
-                            } catch (MapperParsingException mpe) {
-                                removalReason = "failed on parsing default mapping/mappings on index creation";
-                                throw mpe;
+                                mapperService.merge(mappings, MergeReason.MAPPING_UPDATE, request.updateAllTypes());
+                            } catch (Exception e) {
+                                removalExtraInfo = "failed on parsing default mapping/mappings on index creation";
+                                throw e;
                             }
 
-                            final QueryShardContext queryShardContext = indexService.newQueryShardContext();
+                            // the context is only used for validation so it's fine to pass fake values for the shard id and the current
+                            // timestamp
+                            final QueryShardContext queryShardContext = indexService.newQueryShardContext(0, null, () -> 0L);
                             for (Alias alias : request.aliases()) {
                                 if (Strings.hasLength(alias.filter())) {
-                                    aliasValidator.validateAliasFilter(alias.name(), alias.filter(), queryShardContext);
+                                    aliasValidator.validateAliasFilter(alias.name(), alias.filter(), queryShardContext, xContentRegistry);
                                 }
                             }
                             for (AliasMetaData aliasMetaData : templatesAliases.values()) {
                                 if (aliasMetaData.filter() != null) {
-                                    aliasValidator.validateAliasFilter(aliasMetaData.alias(), aliasMetaData.filter().uncompressed(), queryShardContext);
+                                    aliasValidator.validateAliasFilter(aliasMetaData.alias(), aliasMetaData.filter().uncompressed(),
+                                            queryShardContext, xContentRegistry);
                                 }
                             }
 
@@ -378,6 +411,11 @@ public ClusterState execute(ClusterState currentState) throws Exception {
                             final IndexMetaData.Builder indexMetaDataBuilder = IndexMetaData.builder(request.index())
                                 .settings(actualIndexSettings)
                                 .setRoutingNumShards(routingNumShards);
+
+                            for (int shardId = 0; shardId < tmpImd.getNumberOfShards(); shardId++) {
+                                indexMetaDataBuilder.primaryTerm(shardId, tmpImd.primaryTerm(shardId));
+                            }
+
                             for (MappingMetaData mappingMd : mappingsMetaData.values()) {
                                 indexMetaDataBuilder.putMapping(mappingMd);
                             }
@@ -401,7 +439,7 @@ public ClusterState execute(ClusterState currentState) throws Exception {
                             try {
                                 indexMetaData = indexMetaDataBuilder.build();
                             } catch (Exception e) {
-                                removalReason = "failed to build index metadata";
+                                removalExtraInfo = "failed to build index metadata";
                                 throw e;
                             }
 
@@ -430,24 +468,24 @@ public ClusterState execute(ClusterState currentState) throws Exception {
                             if (request.state() == State.OPEN) {
                                 RoutingTable.Builder routingTableBuilder = RoutingTable.builder(updatedState.routingTable())
                                         .addAsNew(updatedState.metaData().index(request.index()));
-                                RoutingAllocation.Result routingResult = allocationService.reroute(
+                                updatedState = allocationService.reroute(
                                         ClusterState.builder(updatedState).routingTable(routingTableBuilder.build()).build(),
                                         "index [" + request.index() + "] created");
-                                updatedState = ClusterState.builder(updatedState).routingResult(routingResult).build();
                             }
-                            removalReason = "cleaning up after validating index on master";
+                            removalExtraInfo = "cleaning up after validating index on master";
+                            removalReason = IndexRemovalReason.NO_LONGER_ASSIGNED;
                             return updatedState;
                         } finally {
                             if (createdIndex != null) {
                                 // Index was already partially created - need to clean up
-                                indicesService.removeIndex(createdIndex, removalReason != null ? removalReason : "failed to create index");
+                                indicesService.removeIndex(createdIndex, removalReason, removalExtraInfo);
                             }
                         }
                     }
 
                     @Override
                     public void onFailure(String source, Exception e) {
-                        if (e instanceof IndexAlreadyExistsException) {
+                        if (e instanceof ResourceAlreadyExistsException) {
                             logger.trace((Supplier<?>) () -> new ParameterizedMessage("[{}] failed to create", request.index()), e);
                         } else {
                             logger.debug((Supplier<?>) () -> new ParameterizedMessage("[{}] failed to create", request.index()), e);
@@ -466,12 +504,7 @@ private List<IndexTemplateMetaData> findTemplates(CreateIndexClusterStateUpdateR
             }
         }
 
-        CollectionUtil.timSort(templates, new Comparator<IndexTemplateMetaData>() {
-            @Override
-            public int compare(IndexTemplateMetaData o1, IndexTemplateMetaData o2) {
-                return o2.order() - o1.order();
-            }
-        });
+        CollectionUtil.timSort(templates, Comparator.comparingInt(IndexTemplateMetaData::order).reversed());
         return templates;
     }
 
@@ -500,15 +533,6 @@ List<String> getIndexSettingsValidationErrors(Settings settings) {
                 validationErrors.add("custom path [" + customPath + "] is not a sub-path of path.shared_data [" + env.sharedDataFile() + "]");
             }
         }
-        //norelease - this can be removed?
-        Integer number_of_primaries = settings.getAsInt(IndexMetaData.SETTING_NUMBER_OF_SHARDS, null);
-        Integer number_of_replicas = settings.getAsInt(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, null);
-        if (number_of_primaries != null && number_of_primaries <= 0) {
-            validationErrors.add("index must have 1 or more primary shards");
-        }
-        if (number_of_replicas != null && number_of_replicas < 0) {
-            validationErrors.add("index must have 0 or more replica shards");
-        }
         return validationErrors;
     }
 
@@ -520,7 +544,7 @@ static List<String> validateShrinkIndex(ClusterState state, String sourceIndex,
                                         Set<String> targetIndexMappingsTypes, String targetIndexName,
                                         Settings targetIndexSettings) {
         if (state.metaData().hasIndex(targetIndexName)) {
-            throw new IndexAlreadyExistsException(state.metaData().index(targetIndexName).getIndex());
+            throw new ResourceAlreadyExistsException(state.metaData().index(targetIndexName).getIndex());
         }
         final IndexMetaData sourceMetaData = state.metaData().index(sourceIndex);
         if (sourceMetaData == null) {
@@ -541,6 +565,7 @@ static List<String> validateShrinkIndex(ClusterState state, String sourceIndex,
             throw new IllegalArgumentException("mappings are not allowed when shrinking indices" +
                 ", all mappings are copied from the source index");
         }
+
         if (IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.exists(targetIndexSettings)) {
             // this method applies all necessary checks ie. if the target shards are less than the source shards
             // of if the source shards are divisible by the number of target shards
@@ -578,13 +603,16 @@ static void prepareShrinkIndexSettings(ClusterState currentState, Set<String> ma
         indexSettingsBuilder
             // we use "i.r.a.initial_recovery" rather than "i.r.a.require|include" since we want the replica to allocate right away
             // once we are allocated.
-            .put("index.routing.allocation.initial_recovery._id",
+            .put(IndexMetaData.INDEX_ROUTING_INITIAL_RECOVERY_GROUP_SETTING.getKey() + "_id",
                 Strings.arrayToCommaDelimitedString(nodesToAllocateOn.toArray()))
             // we only try once and then give up with a shrink index
             .put("index.allocation.max_retries", 1)
             // now copy all similarity / analysis settings - this overrides all settings from the user unless they
             // wanna add extra settings
+            .put(IndexMetaData.SETTING_VERSION_CREATED, sourceMetaData.getCreationVersion())
+            .put(IndexMetaData.SETTING_VERSION_UPGRADED, sourceMetaData.getUpgradedVersion())
             .put(sourceMetaData.getSettings().filter(analysisSimilarityPredicate))
+            .put(IndexMetaData.SETTING_ROUTING_PARTITION_SIZE, sourceMetaData.getRoutingPartitionSize())
             .put(IndexMetaData.INDEX_SHRINK_SOURCE_NAME.getKey(), shrinkFromIndex.getName())
             .put(IndexMetaData.INDEX_SHRINK_SOURCE_UUID.getKey(), shrinkFromIndex.getUUID());
     }
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataDeleteIndexService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataDeleteIndexService.java
index 07e64eac619ea..a2212b5c3f01c 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataDeleteIndexService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataDeleteIndexService.java
@@ -23,21 +23,23 @@
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexClusterStateUpdateRequest;
 import org.elasticsearch.cluster.AckedClusterStateUpdateTask;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.RestoreInProgress;
 import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
-import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.snapshots.RestoreService;
 import org.elasticsearch.snapshots.SnapshotsService;
 
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.Set;
 
 import static java.util.stream.Collectors.toSet;
@@ -64,7 +66,7 @@ public void deleteIndices(final DeleteIndexClusterStateUpdateRequest request,
             throw new IllegalArgumentException("Index name is required");
         }
 
-        clusterService.submitStateUpdateTask("delete-index " + request.indices(),
+        clusterService.submitStateUpdateTask("delete-index " + Arrays.toString(request.indices()),
             new AckedClusterStateUpdateTask<ClusterStateUpdateResponse>(Priority.URGENT, request, listener) {
 
             @Override
@@ -74,7 +76,7 @@ protected ClusterStateUpdateResponse newResponse(boolean acknowledged) {
 
             @Override
             public ClusterState execute(final ClusterState currentState) {
-                return deleteIndices(currentState, Arrays.asList(request.indices()));
+                return deleteIndices(currentState, Sets.newHashSet(request.indices()));
             }
         });
     }
@@ -82,7 +84,7 @@ public ClusterState execute(final ClusterState currentState) {
     /**
      * Delete some indices from the cluster state.
      */
-    public ClusterState deleteIndices(ClusterState currentState, Collection<Index> indices) {
+    public ClusterState deleteIndices(ClusterState currentState, Set<Index> indices) {
         final MetaData meta = currentState.metaData();
         final Set<IndexMetaData> metaDatas = indices.stream().map(i -> meta.getIndexSafe(i)).collect(toSet());
         // Check if index deletion conflicts with any running snapshots
@@ -95,7 +97,7 @@ public ClusterState deleteIndices(ClusterState currentState, Collection<Index> i
         final int previousGraveyardSize = graveyardBuilder.tombstones().size();
         for (final Index index : indices) {
             String indexName = index.getName();
-            logger.debug("[{}] deleting index", index);
+            logger.info("{} deleting index", index);
             routingTableBuilder.remove(indexName);
             clusterBlocksBuilder.removeIndexBlocks(indexName);
             metaDataBuilder.remove(indexName);
@@ -108,9 +110,26 @@ public ClusterState deleteIndices(ClusterState currentState, Collection<Index> i
 
         MetaData newMetaData = metaDataBuilder.build();
         ClusterBlocks blocks = clusterBlocksBuilder.build();
-        RoutingAllocation.Result routingResult = allocationService.reroute(
-                ClusterState.builder(currentState).routingTable(routingTableBuilder.build()).metaData(newMetaData).build(),
+
+        // update snapshot restore entries
+        ImmutableOpenMap<String, ClusterState.Custom> customs = currentState.getCustoms();
+        final RestoreInProgress restoreInProgress = currentState.custom(RestoreInProgress.TYPE);
+        if (restoreInProgress != null) {
+            RestoreInProgress updatedRestoreInProgress = RestoreService.updateRestoreStateWithDeletedIndices(restoreInProgress, indices);
+            if (updatedRestoreInProgress != restoreInProgress) {
+                ImmutableOpenMap.Builder<String, ClusterState.Custom> builder = ImmutableOpenMap.builder(customs);
+                builder.put(RestoreInProgress.TYPE, updatedRestoreInProgress);
+                customs = builder.build();
+            }
+        }
+
+        return allocationService.reroute(
+                ClusterState.builder(currentState)
+                    .routingTable(routingTableBuilder.build())
+                    .metaData(newMetaData)
+                    .blocks(blocks)
+                    .customs(customs)
+                    .build(),
                 "deleted indices [" + indices + "]");
-        return ClusterState.builder(currentState).routingResult(routingResult).metaData(newMetaData).blocks(blocks).build();
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesService.java
index c21454a09a0f0..feee41d742326 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesService.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.cluster.metadata;
 
-import com.carrotsearch.hppc.cursors.ObjectCursor;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesClusterStateUpdateRequest;
@@ -33,10 +32,10 @@
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.IndexService;
-import org.elasticsearch.index.NodeServicesProvider;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.indices.IndicesService;
 
@@ -50,6 +49,7 @@
 import java.util.function.Function;
 
 import static java.util.Collections.emptyList;
+import static org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.NO_LONGER_ASSIGNED;
 
 /**
  * Service responsible for submitting add and remove aliases requests
@@ -62,19 +62,19 @@ public class MetaDataIndexAliasesService extends AbstractComponent {
 
     private final AliasValidator aliasValidator;
 
-    private final NodeServicesProvider nodeServicesProvider;
-
     private final MetaDataDeleteIndexService deleteIndexService;
 
+    private final NamedXContentRegistry xContentRegistry;
+
     @Inject
     public MetaDataIndexAliasesService(Settings settings, ClusterService clusterService, IndicesService indicesService,
-            AliasValidator aliasValidator, NodeServicesProvider nodeServicesProvider, MetaDataDeleteIndexService deleteIndexService) {
+            AliasValidator aliasValidator, MetaDataDeleteIndexService deleteIndexService, NamedXContentRegistry xContentRegistry) {
         super(settings);
         this.clusterService = clusterService;
         this.indicesService = indicesService;
         this.aliasValidator = aliasValidator;
-        this.nodeServicesProvider = nodeServicesProvider;
         this.deleteIndexService = deleteIndexService;
+        this.xContentRegistry = xContentRegistry;
     }
 
     public void indicesAliases(final IndicesAliasesClusterStateUpdateRequest request,
@@ -139,20 +139,19 @@ ClusterState innerExecute(ClusterState currentState, Iterable<AliasAction> actio
                             if (indexService == null) {
                                 // temporarily create the index and add mappings so we can parse the filter
                                 try {
-                                    indexService = indicesService.createIndex(nodeServicesProvider, index, emptyList());
+                                    indexService = indicesService.createIndex(index, emptyList());
+                                    indicesToClose.add(index.getIndex());
                                 } catch (IOException e) {
                                     throw new ElasticsearchException("Failed to create temporary index for parsing the alias", e);
                                 }
-                                for (ObjectCursor<MappingMetaData> cursor : index.getMappings().values()) {
-                                    MappingMetaData mappingMetaData = cursor.value;
-                                    indexService.mapperService().merge(mappingMetaData.type(), mappingMetaData.source(),
-                                            MapperService.MergeReason.MAPPING_RECOVERY, false);
-                                }
-                                indicesToClose.add(index.getIndex());
+                                indexService.mapperService().merge(index, MapperService.MergeReason.MAPPING_RECOVERY, false);
                             }
                             indices.put(action.getIndex(), indexService);
                         }
-                        aliasValidator.validateAliasFilter(alias, filter, indexService.newQueryShardContext());
+                        // the context is only used for validation so it's fine to pass fake values for the shard id and the current
+                        // timestamp
+                        aliasValidator.validateAliasFilter(alias, filter, indexService.newQueryShardContext(0, null, () -> 0L),
+                                xContentRegistry);
                     }
                 };
                 changed |= action.apply(newAliasValidator, metadata, index);
@@ -169,7 +168,7 @@ ClusterState innerExecute(ClusterState currentState, Iterable<AliasAction> actio
             return currentState;
         } finally {
             for (Index index : indicesToClose) {
-                indicesService.removeIndex(index, "created for alias processing");
+                indicesService.removeIndex(index, NO_LONGER_ASSIGNED, "created for alias processing");
             }
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateService.java
index 53a0ede809a55..7f8a176243aa6 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateService.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.cluster.metadata;
 
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.close.CloseIndexClusterStateUpdateRequest;
 import org.elasticsearch.action.admin.indices.open.OpenIndexClusterStateUpdateRequest;
@@ -31,14 +32,12 @@
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
-import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
-import org.elasticsearch.index.NodeServicesProvider;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.snapshots.RestoreService;
@@ -55,22 +54,20 @@
  */
 public class MetaDataIndexStateService extends AbstractComponent {
 
-    public static final ClusterBlock INDEX_CLOSED_BLOCK = new ClusterBlock(4, "index closed", false, false, RestStatus.FORBIDDEN, ClusterBlockLevel.READ_WRITE);
+    public static final ClusterBlock INDEX_CLOSED_BLOCK = new ClusterBlock(4, "index closed", false, false, false, RestStatus.FORBIDDEN, ClusterBlockLevel.READ_WRITE);
 
     private final ClusterService clusterService;
 
     private final AllocationService allocationService;
 
     private final MetaDataIndexUpgradeService metaDataIndexUpgradeService;
-    private final NodeServicesProvider nodeServiceProvider;
     private final IndicesService indicesService;
 
     @Inject
     public MetaDataIndexStateService(Settings settings, ClusterService clusterService, AllocationService allocationService,
                                      MetaDataIndexUpgradeService metaDataIndexUpgradeService,
-                                     NodeServicesProvider nodeServicesProvider, IndicesService indicesService) {
+                                     IndicesService indicesService) {
         super(settings);
-        this.nodeServiceProvider = nodeServicesProvider;
         this.indicesService = indicesService;
         this.clusterService = clusterService;
         this.allocationService = allocationService;
@@ -125,11 +122,10 @@ public ClusterState execute(ClusterState currentState) {
                     rtBuilder.remove(index.getIndex().getName());
                 }
 
-                RoutingAllocation.Result routingResult = allocationService.reroute(
+                //no explicit wait for other nodes needed as we use AckedClusterStateUpdateTask
+                return  allocationService.reroute(
                         ClusterState.builder(updatedState).routingTable(rtBuilder.build()).build(),
                         "indices closed [" + indicesAsString + "]");
-                //no explicit wait for other nodes needed as we use AckedClusterStateUpdateTask
-                return ClusterState.builder(updatedState).routingResult(routingResult).build();
             }
         });
     }
@@ -165,14 +161,16 @@ public ClusterState execute(ClusterState currentState) {
                 MetaData.Builder mdBuilder = MetaData.builder(currentState.metaData());
                 ClusterBlocks.Builder blocksBuilder = ClusterBlocks.builder()
                         .blocks(currentState.blocks());
+                final Version minIndexCompatibilityVersion = currentState.getNodes().getMaxNodeVersion()
+                    .minimumIndexCompatibilityVersion();
                 for (IndexMetaData closedMetaData : indicesToOpen) {
                     final String indexName = closedMetaData.getIndex().getName();
                     IndexMetaData indexMetaData = IndexMetaData.builder(closedMetaData).state(IndexMetaData.State.OPEN).build();
                     // The index might be closed because we couldn't import it due to old incompatible version
                     // We need to check that this index can be upgraded to the current version
-                    indexMetaData = metaDataIndexUpgradeService.upgradeIndexMetaData(indexMetaData);
+                    indexMetaData = metaDataIndexUpgradeService.upgradeIndexMetaData(indexMetaData, minIndexCompatibilityVersion);
                     try {
-                        indicesService.verifyIndexMetadata(nodeServiceProvider, indexMetaData, indexMetaData);
+                        indicesService.verifyIndexMetadata(indexMetaData, indexMetaData);
                     } catch (Exception e) {
                         throw new ElasticsearchException("Failed to verify index " + indexMetaData.getIndex(), e);
                     }
@@ -188,11 +186,10 @@ public ClusterState execute(ClusterState currentState) {
                     rtBuilder.addAsFromCloseToOpen(updatedState.metaData().getIndexSafe(index.getIndex()));
                 }
 
-                RoutingAllocation.Result routingResult = allocationService.reroute(
+                //no explicit wait for other nodes needed as we use AckedClusterStateUpdateTask
+                return allocationService.reroute(
                         ClusterState.builder(updatedState).routingTable(rtBuilder.build()).build(),
                         "indices opened [" + indicesAsString + "]");
-                //no explicit wait for other nodes needed as we use AckedClusterStateUpdateTask
-                return ClusterState.builder(updatedState).routingResult(routingResult).build();
             }
         });
     }
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
index 101f59f3aecbe..0a16a97a62d23 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
@@ -19,6 +19,7 @@
 package org.elasticsearch.cluster.metadata;
 
 import com.carrotsearch.hppc.cursors.ObjectCursor;
+
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.action.support.master.MasterNodeRequest;
@@ -32,14 +33,15 @@
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.regex.Regex;
+import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexService;
-import org.elasticsearch.index.NodeServicesProvider;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.indices.IndexTemplateAlreadyExistsException;
+import org.elasticsearch.index.mapper.MapperService.MergeReason;
 import org.elasticsearch.indices.IndexTemplateMissingException;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.InvalidIndexTemplateException;
@@ -53,6 +55,8 @@
 import java.util.Map;
 import java.util.Set;
 
+import static org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.NO_LONGER_ASSIGNED;
+
 /**
  * Service responsible for submitting index templates updates
  */
@@ -62,16 +66,21 @@ public class MetaDataIndexTemplateService extends AbstractComponent {
     private final AliasValidator aliasValidator;
     private final IndicesService indicesService;
     private final MetaDataCreateIndexService metaDataCreateIndexService;
-    private final NodeServicesProvider nodeServicesProvider;
+    private final IndexScopedSettings indexScopedSettings;
+    private final NamedXContentRegistry xContentRegistry;
 
     @Inject
-    public MetaDataIndexTemplateService(Settings settings, ClusterService clusterService, MetaDataCreateIndexService metaDataCreateIndexService, AliasValidator aliasValidator, IndicesService indicesService, NodeServicesProvider nodeServicesProvider) {
+    public MetaDataIndexTemplateService(Settings settings, ClusterService clusterService,
+                                        MetaDataCreateIndexService metaDataCreateIndexService,
+                                        AliasValidator aliasValidator, IndicesService indicesService,
+                                        IndexScopedSettings indexScopedSettings, NamedXContentRegistry xContentRegistry) {
         super(settings);
         this.clusterService = clusterService;
         this.aliasValidator = aliasValidator;
         this.indicesService = indicesService;
         this.metaDataCreateIndexService = metaDataCreateIndexService;
-        this.nodeServicesProvider = nodeServicesProvider;
+        this.indexScopedSettings = indexScopedSettings;
+        this.xContentRegistry = xContentRegistry;
     }
 
     public void removeTemplates(final RemoveRequest request, final RemoveListener listener) {
@@ -157,10 +166,10 @@ public void onFailure(String source, Exception e) {
             @Override
             public ClusterState execute(ClusterState currentState) throws Exception {
                 if (request.create && currentState.metaData().templates().containsKey(request.name)) {
-                    throw new IndexTemplateAlreadyExistsException(request.name);
+                    throw new IllegalArgumentException("index_template [" + request.name + "] already exists");
                 }
 
-                validateAndAddTemplate(request, templateBuilder, indicesService, nodeServicesProvider);
+                validateAndAddTemplate(request, templateBuilder, indicesService, xContentRegistry);
 
                 for (Alias alias : request.aliases) {
                     AliasMetaData aliasMetaData = AliasMetaData.builder(alias.name()).filter(alias.filter())
@@ -184,26 +193,31 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS
         });
     }
 
-    private static void validateAndAddTemplate(final PutRequest request, IndexTemplateMetaData.Builder templateBuilder, IndicesService indicesService,
-                                               NodeServicesProvider nodeServicesProvider) throws Exception {
+    private static void validateAndAddTemplate(final PutRequest request, IndexTemplateMetaData.Builder templateBuilder,
+            IndicesService indicesService, NamedXContentRegistry xContentRegistry) throws Exception {
         Index createdIndex = null;
         final String temporaryIndexName = UUIDs.randomBase64UUID();
         try {
+            // use the provided values, otherwise just pick valid dummy values
+            int dummyPartitionSize = IndexMetaData.INDEX_ROUTING_PARTITION_SIZE_SETTING.get(request.settings);
+            int dummyShards = request.settings.getAsInt(IndexMetaData.SETTING_NUMBER_OF_SHARDS,
+                    dummyPartitionSize == 1 ? 1 : dummyPartitionSize + 1);
 
             //create index service for parsing and validating "mappings"
             Settings dummySettings = Settings.builder()
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
                 .put(request.settings)
-                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, dummyShards)
                 .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
                 .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
                 .build();
 
             final IndexMetaData tmpIndexMetadata = IndexMetaData.builder(temporaryIndexName).settings(dummySettings).build();
-            IndexService dummyIndexService = indicesService.createIndex(nodeServicesProvider, tmpIndexMetadata, Collections.emptyList());
+            IndexService dummyIndexService = indicesService.createIndex(tmpIndexMetadata, Collections.emptyList());
             createdIndex = dummyIndexService.index();
 
             templateBuilder.order(request.order);
+            templateBuilder.version(request.version);
             templateBuilder.template(request.template);
             templateBuilder.settings(request.settings);
 
@@ -214,14 +228,14 @@ private static void validateAndAddTemplate(final PutRequest request, IndexTempla
                 } catch (Exception e) {
                     throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, entry.getKey(), e.getMessage());
                 }
-                mappingsForValidation.put(entry.getKey(), MapperService.parseMapping(entry.getValue()));
+                mappingsForValidation.put(entry.getKey(), MapperService.parseMapping(xContentRegistry, entry.getValue()));
             }
 
-            dummyIndexService.mapperService().merge(mappingsForValidation, false);
+            dummyIndexService.mapperService().merge(mappingsForValidation, MergeReason.MAPPING_UPDATE, false);
 
         } finally {
             if (createdIndex != null) {
-                indicesService.removeIndex(createdIndex, " created for parsing template mapping");
+                indicesService.removeIndex(createdIndex, NO_LONGER_ASSIGNED, " created for parsing template mapping");
             }
         }
     }
@@ -259,6 +273,14 @@ private void validate(PutRequest request) {
             validationErrors.add("template must not contain the following characters " + Strings.INVALID_FILENAME_CHARS);
         }
 
+        try {
+            indexScopedSettings.validate(request.settings);
+        } catch (IllegalArgumentException iae) {
+            validationErrors.add(iae.getMessage());
+            for (Throwable t : iae.getSuppressed()) {
+                validationErrors.add(t.getMessage());
+            }
+        }
         List<String> indexSettingsValidation = metaDataCreateIndexService.getIndexSettingsValidationErrors(request.settings);
         validationErrors.addAll(indexSettingsValidation);
         if (!validationErrors.isEmpty()) {
@@ -288,6 +310,7 @@ public static class PutRequest {
         final String cause;
         boolean create;
         int order;
+        Integer version;
         String template;
         Settings settings = Settings.Builder.EMPTY_SETTINGS;
         Map<String, String> mappings = new HashMap<>();
@@ -345,6 +368,11 @@ public PutRequest masterTimeout(TimeValue masterTimeout) {
             this.masterTimeout = masterTimeout;
             return this;
         }
+
+        public PutRequest version(Integer version) {
+            this.version = version;
+            return this;
+        }
     }
 
     public static class PutResponse {
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java
index d1141aeb9f477..909f60f1addd3 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java
@@ -19,20 +19,30 @@
 package org.elasticsearch.cluster.metadata;
 
 import com.carrotsearch.hppc.cursors.ObjectCursor;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.apache.logging.log4j.util.Supplier;
 import org.apache.lucene.analysis.Analyzer;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.analysis.AnalysisService;
+import org.elasticsearch.index.analysis.IndexAnalyzers;
+import org.elasticsearch.index.analysis.AnalyzerScope;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.indices.mapper.MapperRegistry;
+import org.elasticsearch.plugins.Plugin;
 
+import java.util.AbstractMap;
+import java.util.Collection;
 import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.UnaryOperator;
 
 /**
  * This service is responsible for upgrading legacy index metadata to the current version
@@ -44,14 +54,26 @@
  */
 public class MetaDataIndexUpgradeService extends AbstractComponent {
 
+    private final NamedXContentRegistry xContentRegistry;
     private final MapperRegistry mapperRegistry;
     private final IndexScopedSettings indexScopedSettings;
+    private final UnaryOperator<IndexMetaData> upgraders;
 
     @Inject
-    public MetaDataIndexUpgradeService(Settings settings, MapperRegistry mapperRegistry, IndexScopedSettings indexScopedSettings) {
+    public MetaDataIndexUpgradeService(Settings settings, NamedXContentRegistry xContentRegistry, MapperRegistry mapperRegistry,
+                                       IndexScopedSettings indexScopedSettings,
+                                       Collection<UnaryOperator<IndexMetaData>> indexMetaDataUpgraders) {
         super(settings);
+        this.xContentRegistry = xContentRegistry;
         this.mapperRegistry = mapperRegistry;
         this.indexScopedSettings = indexScopedSettings;
+        this.upgraders = indexMetaData -> {
+            IndexMetaData newIndexMetaData = indexMetaData;
+            for (UnaryOperator<IndexMetaData> upgrader : indexMetaDataUpgraders) {
+                newIndexMetaData = upgrader.apply(newIndexMetaData);
+            }
+            return newIndexMetaData;
+        };
     }
 
     /**
@@ -61,19 +83,21 @@ public MetaDataIndexUpgradeService(Settings settings, MapperRegistry mapperRegis
      * If the index does not need upgrade it returns the index metadata unchanged, otherwise it returns a modified index metadata. If index
      * cannot be updated the method throws an exception.
      */
-    public IndexMetaData upgradeIndexMetaData(IndexMetaData indexMetaData) {
+    public IndexMetaData upgradeIndexMetaData(IndexMetaData indexMetaData, Version minimumIndexCompatibilityVersion) {
         // Throws an exception if there are too-old segments:
         if (isUpgraded(indexMetaData)) {
             assert indexMetaData == archiveBrokenIndexSettings(indexMetaData) : "all settings must have been upgraded before";
             return indexMetaData;
         }
-        checkSupportedVersion(indexMetaData);
+        checkSupportedVersion(indexMetaData, minimumIndexCompatibilityVersion);
         IndexMetaData newMetaData = indexMetaData;
         // we have to run this first otherwise in we try to create IndexSettings
         // with broken settings and fail in checkMappingsCompatibility
         newMetaData = archiveBrokenIndexSettings(newMetaData);
         // only run the check with the upgraded settings!!
         checkMappingsCompatibility(newMetaData);
+        // apply plugin checks
+        newMetaData = upgraders.apply(newMetaData);
         return markAsUpgraded(newMetaData);
     }
 
@@ -86,30 +110,26 @@ boolean isUpgraded(IndexMetaData indexMetaData) {
     }
 
     /**
-     * Elasticsearch 5.0 no longer supports indices with pre Lucene v5.0 (Elasticsearch v2.0.0.beta1) segments. All indices
-     * that were created before Elasticsearch v2.0.0.beta1 should be reindexed in Elasticsearch 2.x
-     * before they can be opened by this version of elasticsearch.     */
-    private void checkSupportedVersion(IndexMetaData indexMetaData) {
-        if (indexMetaData.getState() == IndexMetaData.State.OPEN && isSupportedVersion(indexMetaData) == false) {
-            throw new IllegalStateException("The index [" + indexMetaData.getIndex() + "] was created before v2.0.0.beta1."
-                    + " It should be reindexed in Elasticsearch 2.x before upgrading to " + Version.CURRENT + ".");
+     * Elasticsearch v6.0 no longer supports indices created pre v5.0. All indices
+     * that were created before Elasticsearch v5.0 should be re-indexed in Elasticsearch 5.x
+     * before they can be opened by this version of elasticsearch.
+     */
+    private void checkSupportedVersion(IndexMetaData indexMetaData, Version minimumIndexCompatibilityVersion) {
+        if (indexMetaData.getState() == IndexMetaData.State.OPEN && isSupportedVersion(indexMetaData,
+            minimumIndexCompatibilityVersion) == false) {
+            throw new IllegalStateException("The index [" + indexMetaData.getIndex() + "] was created with version ["
+                + indexMetaData.getCreationVersion() + "] but the minimum compatible version is ["
+
+                + minimumIndexCompatibilityVersion + "]. It should be re-indexed in Elasticsearch " + minimumIndexCompatibilityVersion.major
+                + ".x before upgrading to " + Version.CURRENT + ".");
         }
     }
 
     /*
      * Returns true if this index can be supported by the current version of elasticsearch
      */
-    private static boolean isSupportedVersion(IndexMetaData indexMetaData) {
-        if (indexMetaData.getCreationVersion().onOrAfter(Version.V_2_0_0_beta1)) {
-            // The index was created with elasticsearch that was using Lucene 5.2.1
-            return true;
-        }
-        if (indexMetaData.getMinimumCompatibleVersion() != null &&
-                indexMetaData.getMinimumCompatibleVersion().onOrAfter(org.apache.lucene.util.Version.LUCENE_5_0_0)) {
-            //The index was upgraded we can work with it
-            return true;
-        }
-        return false;
+    private static boolean isSupportedVersion(IndexMetaData indexMetaData, Version minimumIndexCompatibilityVersion) {
+        return indexMetaData.getCreationVersion().onOrAfter(minimumIndexCompatibilityVersion);
     }
 
     /**
@@ -121,13 +141,31 @@ private void checkMappingsCompatibility(IndexMetaData indexMetaData) {
             // been started yet. However, we don't really need real analyzers at this stage - so we can fake it
             IndexSettings indexSettings = new IndexSettings(indexMetaData, this.settings);
             SimilarityService similarityService = new SimilarityService(indexSettings, Collections.emptyMap());
+            final NamedAnalyzer fakeDefault = new NamedAnalyzer("fake_default", AnalyzerScope.INDEX, new Analyzer() {
+                @Override
+                protected TokenStreamComponents createComponents(String fieldName) {
+                    throw new UnsupportedOperationException("shouldn't be here");
+                }
+            });
+            // this is just a fake map that always returns the same value for any possible string key
+            // also the entrySet impl isn't fully correct but we implement it since internally
+            // IndexAnalyzers will iterate over all analyzers to close them.
+            final Map<String, NamedAnalyzer> analyzerMap = new AbstractMap<String, NamedAnalyzer>() {
+                @Override
+                public NamedAnalyzer get(Object key) {
+                    assert key instanceof String : "key must be a string but was: " + key.getClass();
+                    return new NamedAnalyzer((String)key, AnalyzerScope.INDEX, fakeDefault.analyzer());
+                }
 
-            try (AnalysisService analysisService = new FakeAnalysisService(indexSettings)) {
-                MapperService mapperService = new MapperService(indexSettings, analysisService, similarityService, mapperRegistry, () -> null);
-                for (ObjectCursor<MappingMetaData> cursor : indexMetaData.getMappings().values()) {
-                    MappingMetaData mappingMetaData = cursor.value;
-                    mapperService.merge(mappingMetaData.type(), mappingMetaData.source(), MapperService.MergeReason.MAPPING_RECOVERY, false);
+                @Override
+                public Set<Entry<String, NamedAnalyzer>> entrySet() {
+                    return Collections.emptySet();
                 }
+            };
+            try (IndexAnalyzers fakeIndexAnalzyers = new IndexAnalyzers(indexSettings, fakeDefault, fakeDefault, fakeDefault, analyzerMap, analyzerMap)) {
+                MapperService mapperService = new MapperService(indexSettings, fakeIndexAnalzyers, xContentRegistry, similarityService,
+                        mapperRegistry, () -> null);
+                mapperService.merge(indexMetaData, MapperService.MergeReason.MAPPING_RECOVERY, false);
             }
         } catch (Exception ex) {
             // Wrap the inner exception so we have the index name in the exception message
@@ -143,37 +181,12 @@ private IndexMetaData markAsUpgraded(IndexMetaData indexMetaData) {
         return IndexMetaData.builder(indexMetaData).settings(settings).build();
     }
 
-    /**
-     * A fake analysis server that returns the same keyword analyzer for all requests
-     */
-    private static class FakeAnalysisService extends AnalysisService {
-
-        private Analyzer fakeAnalyzer = new Analyzer() {
-            @Override
-            protected TokenStreamComponents createComponents(String fieldName) {
-                throw new UnsupportedOperationException("shouldn't be here");
-            }
-        };
-
-        public FakeAnalysisService(IndexSettings indexSettings) {
-            super(indexSettings, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
-        }
-
-        @Override
-        public NamedAnalyzer analyzer(String name) {
-            return new NamedAnalyzer(name, fakeAnalyzer);
-        }
-
-        @Override
-        public void close() {
-            fakeAnalyzer.close();
-            super.close();
-        }
-    }
-
     IndexMetaData archiveBrokenIndexSettings(IndexMetaData indexMetaData) {
         final Settings settings = indexMetaData.getSettings();
-        final Settings upgrade = indexScopedSettings.archiveUnknownOrBrokenSettings(settings);
+        final Settings upgrade = indexScopedSettings.archiveUnknownOrInvalidSettings(
+            settings,
+            e -> logger.warn("{} ignoring unknown index setting: [{}] with value [{}]; archiving", indexMetaData.getIndex(), e.getKey(), e.getValue()),
+            (e, ex) -> logger.warn((Supplier<?>) () -> new ParameterizedMessage("{} ignoring invalid index setting: [{}] with value [{}]; archiving", indexMetaData.getIndex(), e.getKey(), e.getValue()), ex));
         if (upgrade != settings) {
             return IndexMetaData.builder(indexMetaData).settings(upgrade).build();
         } else {
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java
index 8ce58637b1550..865b58c468a52 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java
@@ -22,18 +22,18 @@
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
+import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingClusterStateUpdateRequest;
 import org.elasticsearch.cluster.AckedClusterStateTaskListener;
+import org.elasticsearch.cluster.ClusterStateTaskExecutor;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateTaskConfig;
-import org.elasticsearch.cluster.ClusterStateTaskExecutor;
 import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Priority;
-import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.inject.Inject;
@@ -41,9 +41,9 @@
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexService;
-import org.elasticsearch.index.NodeServicesProvider;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.MapperService.MergeReason;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.InvalidTypeNameException;
 
@@ -51,10 +51,11 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
+
+import static org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.NO_LONGER_ASSIGNED;
+
 /**
  * Service responsible for submitting mapping changes
  */
@@ -63,17 +64,15 @@ public class MetaDataMappingService extends AbstractComponent {
     private final ClusterService clusterService;
     private final IndicesService indicesService;
 
-    final ClusterStateTaskExecutor<RefreshTask> refreshExecutor = new RefreshTaskExecutor();
-    final ClusterStateTaskExecutor<PutMappingClusterStateUpdateRequest> putMappingExecutor = new PutMappingExecutor();
-    private final NodeServicesProvider nodeServicesProvider;
+    final RefreshTaskExecutor refreshExecutor = new RefreshTaskExecutor();
+    final PutMappingExecutor putMappingExecutor = new PutMappingExecutor();
 
 
     @Inject
-    public MetaDataMappingService(Settings settings, ClusterService clusterService, IndicesService indicesService, NodeServicesProvider nodeServicesProvider) {
+    public MetaDataMappingService(Settings settings, ClusterService clusterService, IndicesService indicesService) {
         super(settings);
         this.clusterService = clusterService;
         this.indicesService = indicesService;
-        this.nodeServicesProvider = nodeServicesProvider;
     }
 
     static class RefreshTask {
@@ -93,9 +92,9 @@ public String toString() {
 
     class RefreshTaskExecutor implements ClusterStateTaskExecutor<RefreshTask> {
         @Override
-        public BatchResult<RefreshTask> execute(ClusterState currentState, List<RefreshTask> tasks) throws Exception {
+        public ClusterTasksResult<RefreshTask> execute(ClusterState currentState, List<RefreshTask> tasks) throws Exception {
             ClusterState newClusterState = executeRefresh(currentState, tasks);
-            return BatchResult.<RefreshTask>builder().successes(tasks).build(newClusterState);
+            return ClusterTasksResult.<RefreshTask>builder().successes(tasks).build(newClusterState);
         }
     }
 
@@ -146,12 +145,9 @@ ClusterState executeRefresh(final ClusterState currentState, final List<RefreshT
             IndexService indexService = indicesService.indexService(indexMetaData.getIndex());
             if (indexService == null) {
                 // we need to create the index here, and add the current mapping to it, so we can merge
-                indexService = indicesService.createIndex(nodeServicesProvider, indexMetaData, Collections.emptyList());
+                indexService = indicesService.createIndex(indexMetaData, Collections.emptyList());
                 removeIndex = true;
-                for (ObjectCursor<MappingMetaData> metaData : indexMetaData.getMappings().values()) {
-                    // don't apply the default mapping, it has been applied when the mapping was created
-                    indexService.mapperService().merge(metaData.value.type(), metaData.value.source(), MapperService.MergeReason.MAPPING_RECOVERY, true);
-                }
+                indexService.mapperService().merge(indexMetaData, MergeReason.MAPPING_RECOVERY, true);
             }
 
             IndexMetaData.Builder builder = IndexMetaData.builder(indexMetaData);
@@ -163,7 +159,7 @@ ClusterState executeRefresh(final ClusterState currentState, final List<RefreshT
                 }
             } finally {
                 if (removeIndex) {
-                    indicesService.removeIndex(index, "created for mapping processing");
+                    indicesService.removeIndex(index, NO_LONGER_ASSIGNED, "created for mapping processing");
                 }
             }
         }
@@ -215,30 +211,23 @@ public void refreshMapping(final String index, final String indexUUID) {
 
     class PutMappingExecutor implements ClusterStateTaskExecutor<PutMappingClusterStateUpdateRequest> {
         @Override
-        public BatchResult<PutMappingClusterStateUpdateRequest> execute(ClusterState currentState,
-                                                                        List<PutMappingClusterStateUpdateRequest> tasks) throws Exception {
-            Set<Index> indicesToClose = new HashSet<>();
-            BatchResult.Builder<PutMappingClusterStateUpdateRequest> builder = BatchResult.builder();
+        public ClusterTasksResult<PutMappingClusterStateUpdateRequest> execute(ClusterState currentState,
+                                                                               List<PutMappingClusterStateUpdateRequest> tasks) throws Exception {
+            Map<Index, MapperService> indexMapperServices = new HashMap<>();
+            ClusterTasksResult.Builder<PutMappingClusterStateUpdateRequest> builder = ClusterTasksResult.builder();
             try {
-                // precreate incoming indices;
                 for (PutMappingClusterStateUpdateRequest request : tasks) {
                     try {
                         for (Index index : request.indices()) {
                             final IndexMetaData indexMetaData = currentState.metaData().getIndexSafe(index);
-                            if (indicesService.hasIndex(indexMetaData.getIndex()) == false) {
-                                // if the index does not exists we create it once, add all types to the mapper service and
-                                // close it later once we are done with mapping update
-                                indicesToClose.add(indexMetaData.getIndex());
-                                IndexService indexService = indicesService.createIndex(nodeServicesProvider, indexMetaData,
-                                    Collections.emptyList());
+                            if (indexMapperServices.containsKey(indexMetaData.getIndex()) == false) {
+                                MapperService mapperService = indicesService.createIndexMapperService(indexMetaData);
+                                indexMapperServices.put(index, mapperService);
                                 // add mappings for all types, we need them for cross-type validation
-                                for (ObjectCursor<MappingMetaData> mapping : indexMetaData.getMappings().values()) {
-                                    indexService.mapperService().merge(mapping.value.type(), mapping.value.source(),
-                                        MapperService.MergeReason.MAPPING_RECOVERY, request.updateAllTypes());
-                                }
+                                mapperService.merge(indexMetaData, MergeReason.MAPPING_RECOVERY, request.updateAllTypes());
                             }
                         }
-                        currentState = applyRequest(currentState, request);
+                        currentState = applyRequest(currentState, request, indexMapperServices);
                         builder.success(request);
                     } catch (Exception e) {
                         builder.failure(request, e);
@@ -246,34 +235,33 @@ public BatchResult<PutMappingClusterStateUpdateRequest> execute(ClusterState cur
                 }
                 return builder.build(currentState);
             } finally {
-                for (Index index : indicesToClose) {
-                    indicesService.removeIndex(index, "created for mapping processing");
-                }
+                IOUtils.close(indexMapperServices.values());
             }
         }
 
-        private ClusterState applyRequest(ClusterState currentState, PutMappingClusterStateUpdateRequest request) throws IOException {
+        private ClusterState applyRequest(ClusterState currentState, PutMappingClusterStateUpdateRequest request,
+                                          Map<Index, MapperService> indexMapperServices) throws IOException {
             String mappingType = request.type();
             CompressedXContent mappingUpdateSource = new CompressedXContent(request.source());
             final MetaData metaData = currentState.metaData();
-            final List<Tuple<IndexService, IndexMetaData>> updateList = new ArrayList<>();
+            final List<IndexMetaData> updateList = new ArrayList<>();
             for (Index index : request.indices()) {
-                IndexService indexService = indicesService.indexServiceSafe(index);
+                MapperService mapperService = indexMapperServices.get(index);
                 // IMPORTANT: always get the metadata from the state since it get's batched
                 // and if we pull it from the indexService we might miss an update etc.
                 final IndexMetaData indexMetaData = currentState.getMetaData().getIndexSafe(index);
 
-                // this is paranoia... just to be sure we use the exact same indexService and metadata tuple on the update that
+                // this is paranoia... just to be sure we use the exact same metadata tuple on the update that
                 // we used for the validation, it makes this mechanism little less scary (a little)
-                updateList.add(new Tuple<>(indexService, indexMetaData));
+                updateList.add(indexMetaData);
                 // try and parse it (no need to add it here) so we can bail early in case of parsing exception
                 DocumentMapper newMapper;
-                DocumentMapper existingMapper = indexService.mapperService().documentMapper(request.type());
+                DocumentMapper existingMapper = mapperService.documentMapper(request.type());
                 if (MapperService.DEFAULT_MAPPING.equals(request.type())) {
                     // _default_ types do not go through merging, but we do test the new settings. Also don't apply the old default
-                    newMapper = indexService.mapperService().parse(request.type(), mappingUpdateSource, false);
+                    newMapper = mapperService.parse(request.type(), mappingUpdateSource, false);
                 } else {
-                    newMapper = indexService.mapperService().parse(request.type(), mappingUpdateSource, existingMapper == null);
+                    newMapper = mapperService.parse(request.type(), mappingUpdateSource, existingMapper == null);
                     if (existingMapper != null) {
                         // first, simulate: just call merge and ignore the result
                         existingMapper.merge(newMapper.mapping(), request.updateAllTypes());
@@ -289,9 +277,9 @@ private ClusterState applyRequest(ClusterState currentState, PutMappingClusterSt
                             for (ObjectCursor<MappingMetaData> mapping : indexMetaData.getMappings().values()) {
                                 String parentType = newMapper.parentFieldMapper().type();
                                 if (parentType.equals(mapping.value.type()) &&
-                                        indexService.mapperService().getParentTypes().contains(parentType) == false) {
+                                        mapperService.getParentTypes().contains(parentType) == false) {
                                     throw new IllegalArgumentException("can't add a _parent field that points to an " +
-                                            "already existing type, that isn't already a parent");
+                                        "already existing type, that isn't already a parent");
                                 }
                             }
                         }
@@ -309,24 +297,25 @@ private ClusterState applyRequest(ClusterState currentState, PutMappingClusterSt
                 throw new InvalidTypeNameException("Document mapping type name can't start with '_', found: [" + mappingType + "]");
             }
             MetaData.Builder builder = MetaData.builder(metaData);
-            for (Tuple<IndexService, IndexMetaData> toUpdate : updateList) {
+            boolean updated = false;
+            for (IndexMetaData indexMetaData : updateList) {
                 // do the actual merge here on the master, and update the mapping source
                 // we use the exact same indexService and metadata we used to validate above here to actually apply the update
-                final IndexService indexService = toUpdate.v1();
-                final IndexMetaData indexMetaData = toUpdate.v2();
                 final Index index = indexMetaData.getIndex();
+                final MapperService mapperService = indexMapperServices.get(index);
                 CompressedXContent existingSource = null;
-                DocumentMapper existingMapper = indexService.mapperService().documentMapper(mappingType);
+                DocumentMapper existingMapper = mapperService.documentMapper(mappingType);
                 if (existingMapper != null) {
                     existingSource = existingMapper.mappingSource();
                 }
-                DocumentMapper mergedMapper = indexService.mapperService().merge(mappingType, mappingUpdateSource, MapperService.MergeReason.MAPPING_UPDATE, request.updateAllTypes());
+                DocumentMapper mergedMapper = mapperService.merge(mappingType, mappingUpdateSource, MergeReason.MAPPING_UPDATE, request.updateAllTypes());
                 CompressedXContent updatedSource = mergedMapper.mappingSource();
 
                 if (existingSource != null) {
                     if (existingSource.equals(updatedSource)) {
                         // same source, no changes, ignore it
                     } else {
+                        updated = true;
                         // use the merged mapping source
                         if (logger.isDebugEnabled()) {
                             logger.debug("{} update_mapping [{}] with source [{}]", index, mergedMapper.type(), updatedSource);
@@ -336,6 +325,7 @@ private ClusterState applyRequest(ClusterState currentState, PutMappingClusterSt
 
                     }
                 } else {
+                    updated = true;
                     if (logger.isDebugEnabled()) {
                         logger.debug("{} create_mapping [{}] with source [{}]", index, mappingType, updatedSource);
                     } else if (logger.isInfoEnabled()) {
@@ -346,13 +336,16 @@ private ClusterState applyRequest(ClusterState currentState, PutMappingClusterSt
                 IndexMetaData.Builder indexMetaDataBuilder = IndexMetaData.builder(indexMetaData);
                 // Mapping updates on a single type may have side-effects on other types so we need to
                 // update mapping metadata on all types
-                for (DocumentMapper mapper : indexService.mapperService().docMappers(true)) {
+                for (DocumentMapper mapper : mapperService.docMappers(true)) {
                     indexMetaDataBuilder.putMapping(new MappingMetaData(mapper.mappingSource()));
                 }
                 builder.put(indexMetaDataBuilder);
             }
-
-            return ClusterState.builder(currentState).metaData(builder).build();
+            if (updated) {
+                return ClusterState.builder(currentState).metaData(builder).build();
+            } else {
+                return currentState;
+            }
         }
 
         @Override
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java
index 9db777a4794b9..47a986d3b1eea 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java
@@ -33,7 +33,6 @@
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
-import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.collect.Tuple;
@@ -44,8 +43,8 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.Index;
-import org.elasticsearch.index.NodeServicesProvider;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -56,6 +55,8 @@
 import java.util.Map;
 import java.util.Set;
 
+import static org.elasticsearch.action.support.ContextPreservingActionListener.wrapPreservingContext;
+
 /**
  * Service responsible for submitting update index settings requests
  */
@@ -67,18 +68,18 @@ public class MetaDataUpdateSettingsService extends AbstractComponent implements
 
     private final IndexScopedSettings indexScopedSettings;
     private final IndicesService indicesService;
-    private final NodeServicesProvider nodeServiceProvider;
+    private final ThreadPool threadPool;
 
     @Inject
     public MetaDataUpdateSettingsService(Settings settings, ClusterService clusterService, AllocationService allocationService,
-                                         IndexScopedSettings indexScopedSettings, IndicesService indicesService, NodeServicesProvider nodeServicesProvider) {
+                                         IndexScopedSettings indexScopedSettings, IndicesService indicesService, ThreadPool threadPool) {
         super(settings);
         this.clusterService = clusterService;
-        this.clusterService.add(this);
+        this.threadPool = threadPool;
+        this.clusterService.addListener(this);
         this.allocationService = allocationService;
         this.indexScopedSettings = indexScopedSettings;
         this.indicesService = indicesService;
-        this.nodeServiceProvider = nodeServicesProvider;
     }
 
     @Override
@@ -165,10 +166,6 @@ public void updateSettings(final UpdateSettingsClusterStateUpdateRequest request
         indexScopedSettings.validate(normalizedSettings);
         // never allow to change the number of shards
         for (Map.Entry<String, String> entry : normalizedSettings.getAsMap().entrySet()) {
-            if (entry.getKey().equals(IndexMetaData.SETTING_NUMBER_OF_SHARDS)) {
-                listener.onFailure(new IllegalArgumentException("can't change the number of shards for an index"));
-                return;
-            }
             Setting setting = indexScopedSettings.get(entry.getKey());
             assert setting != null; // we already validated the normalized settings
             settingsForClosedIndices.put(entry.getKey(), entry.getValue());
@@ -184,7 +181,8 @@ public void updateSettings(final UpdateSettingsClusterStateUpdateRequest request
         final boolean preserveExisting = request.isPreserveExisting();
 
         clusterService.submitStateUpdateTask("update-settings",
-                new AckedClusterStateUpdateTask<ClusterStateUpdateResponse>(Priority.URGENT, request, listener) {
+                new AckedClusterStateUpdateTask<ClusterStateUpdateResponse>(Priority.URGENT, request,
+                    wrapPreservingContext(listener, threadPool.getThreadContext())) {
 
             @Override
             protected ClusterStateUpdateResponse newResponse(boolean acknowledged) {
@@ -218,7 +216,7 @@ public ClusterState execute(ClusterState currentState) {
                             closeIndices
                     ));
                 }
-                if (!skippedSettigns.getAsMap().isEmpty() && !openIndices.isEmpty()) {
+                if (!skippedSettigns.isEmpty() && !openIndices.isEmpty()) {
                     throw new IllegalArgumentException(String.format(Locale.ROOT,
                             "Can't update non dynamic settings [%s] for open indices %s",
                             skippedSettigns.getAsMap().keySet(),
@@ -228,6 +226,9 @@ public ClusterState execute(ClusterState currentState) {
 
                 int updatedNumberOfReplicas = openSettings.getAsInt(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, -1);
                 if (updatedNumberOfReplicas != -1 && preserveExisting == false) {
+                    // we do *not* update the in sync allocation ids as they will be removed upon the first index
+                    // operation which make these copies stale
+                    // TODO: update the list once the data is deleted by the node?
                     routingTableBuilder.updateNumberOfReplicas(updatedNumberOfReplicas, actualIndices);
                     metaDataBuilder.updateNumberOfReplicas(updatedNumberOfReplicas, actualIndices);
                     logger.info("updating number_of_replicas to [{}] for indices {}", updatedNumberOfReplicas, actualIndices);
@@ -235,6 +236,7 @@ public ClusterState execute(ClusterState currentState) {
 
                 ClusterBlocks.Builder blocks = ClusterBlocks.builder().blocks(currentState.blocks());
                 maybeUpdateClusterBlock(actualIndices, blocks, IndexMetaData.INDEX_READ_ONLY_BLOCK, IndexMetaData.INDEX_READ_ONLY_SETTING, openSettings);
+                maybeUpdateClusterBlock(actualIndices, blocks, IndexMetaData.INDEX_READ_ONLY_ALLOW_DELETE_BLOCK, IndexMetaData.INDEX_BLOCKS_READ_ONLY_ALLOW_DELETE_SETTING, openSettings);
                 maybeUpdateClusterBlock(actualIndices, blocks, IndexMetaData.INDEX_METADATA_BLOCK, IndexMetaData.INDEX_BLOCKS_METADATA_SETTING, openSettings);
                 maybeUpdateClusterBlock(actualIndices, blocks, IndexMetaData.INDEX_WRITE_BLOCK, IndexMetaData.INDEX_BLOCKS_WRITE_SETTING, openSettings);
                 maybeUpdateClusterBlock(actualIndices, blocks, IndexMetaData.INDEX_READ_BLOCK, IndexMetaData.INDEX_BLOCKS_READ_SETTING, openSettings);
@@ -271,18 +273,21 @@ public ClusterState execute(ClusterState currentState) {
                 ClusterState updatedState = ClusterState.builder(currentState).metaData(metaDataBuilder).routingTable(routingTableBuilder.build()).blocks(blocks).build();
 
                 // now, reroute in case things change that require it (like number of replicas)
-                RoutingAllocation.Result routingResult = allocationService.reroute(updatedState, "settings update");
-                updatedState = ClusterState.builder(updatedState).routingResult(routingResult).build();
+                updatedState = allocationService.reroute(updatedState, "settings update");
                 try {
                     for (Index index : openIndices) {
                         final IndexMetaData currentMetaData = currentState.getMetaData().getIndexSafe(index);
                         final IndexMetaData updatedMetaData = updatedState.metaData().getIndexSafe(index);
-                        indicesService.verifyIndexMetadata(nodeServiceProvider, currentMetaData, updatedMetaData);
+                        indicesService.verifyIndexMetadata(currentMetaData, updatedMetaData);
                     }
                     for (Index index : closeIndices) {
                         final IndexMetaData currentMetaData = currentState.getMetaData().getIndexSafe(index);
                         final IndexMetaData updatedMetaData = updatedState.metaData().getIndexSafe(index);
-                        indicesService.verifyIndexMetadata(nodeServiceProvider, currentMetaData, updatedMetaData);
+                        // Verifies that the current index settings can be updated with the updated dynamic settings.
+                        indicesService.verifyIndexMetadata(currentMetaData, updatedMetaData);
+                        // Now check that we can create the index with the updated settings (dynamic and non-dynamic).
+                        // This step is mandatory since we allow to update non-dynamic settings on closed indices.
+                        indicesService.verifyIndexMetadata(updatedMetaData, updatedMetaData);
                     }
                 } catch (IOException ex) {
                     throw ExceptionsHelper.convertToElastic(ex);
@@ -310,9 +315,9 @@ private static void maybeUpdateClusterBlock(String[] actualIndices, ClusterBlock
 
 
     public void upgradeIndexSettings(final UpgradeSettingsClusterStateUpdateRequest request, final ActionListener<ClusterStateUpdateResponse> listener) {
-
-
-        clusterService.submitStateUpdateTask("update-index-compatibility-versions", new AckedClusterStateUpdateTask<ClusterStateUpdateResponse>(Priority.URGENT, request, listener) {
+        clusterService.submitStateUpdateTask("update-index-compatibility-versions",
+            new AckedClusterStateUpdateTask<ClusterStateUpdateResponse>(Priority.URGENT, request,
+                wrapPreservingContext(listener, threadPool.getThreadContext())) {
 
             @Override
             protected ClusterStateUpdateResponse newResponse(boolean acknowledged) {
@@ -330,7 +335,6 @@ public ClusterState execute(ClusterState currentState) {
                             // No reason to pollute the settings, we didn't really upgrade anything
                             metaDataBuilder.put(IndexMetaData.builder(indexMetaData)
                                             .settings(Settings.builder().put(indexMetaData.getSettings())
-                                                            .put(IndexMetaData.SETTING_VERSION_MINIMUM_COMPATIBLE, entry.getValue().v2())
                                                             .put(IndexMetaData.SETTING_VERSION_UPGRADED, entry.getValue().v1())
                                             )
                             );
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetaData.java
index 2dc842ceaae3e..67909bff61449 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetaData.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetaData.java
@@ -21,6 +21,9 @@
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.cluster.AbstractDiffable;
+import org.elasticsearch.cluster.AbstractNamedDiffable;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.NamedDiff;
 import org.elasticsearch.cluster.metadata.MetaData.Custom;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -39,12 +42,10 @@
 /**
  * Contains metadata about registered snapshot repositories
  */
-public class RepositoriesMetaData extends AbstractDiffable<Custom> implements MetaData.Custom {
+public class RepositoriesMetaData extends AbstractNamedDiffable<Custom> implements Custom {
 
     public static final String TYPE = "repositories";
 
-    public static final RepositoriesMetaData PROTO = new RepositoriesMetaData();
-
     private final List<RepositoryMetaData> repositories;
 
     /**
@@ -100,20 +101,20 @@ public int hashCode() {
      * {@inheritDoc}
      */
     @Override
-    public String type() {
+    public String getWriteableName() {
         return TYPE;
     }
 
-    /**
-     * {@inheritDoc}
-     */
-    @Override
-    public Custom readFrom(StreamInput in) throws IOException {
+    public RepositoriesMetaData(StreamInput in) throws IOException {
         RepositoryMetaData[] repository = new RepositoryMetaData[in.readVInt()];
         for (int i = 0; i < repository.length; i++) {
-            repository[i] = RepositoryMetaData.readFrom(in);
+            repository[i] = new RepositoryMetaData(in);
         }
-        return new RepositoriesMetaData(repository);
+        this.repositories = Arrays.asList(repository);
+    }
+
+    public static NamedDiff<Custom> readDiffFrom(StreamInput in) throws  IOException {
+        return readDiffFrom(Custom.class, TYPE, in);
     }
 
     /**
@@ -127,11 +128,7 @@ public void writeTo(StreamOutput out) throws IOException {
         }
     }
 
-    /**
-     * {@inheritDoc}
-     */
-    @Override
-    public RepositoriesMetaData fromXContent(XContentParser parser) throws IOException {
+    public static RepositoriesMetaData fromXContent(XContentParser parser) throws IOException {
         XContentParser.Token token;
         List<RepositoryMetaData> repository = new ArrayList<>();
         while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/RepositoryMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/RepositoryMetaData.java
index 3c13a10c1cfd8..847db915b8bce 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/RepositoryMetaData.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/RepositoryMetaData.java
@@ -73,17 +73,10 @@ public Settings settings() {
     }
 
 
-    /**
-     * Reads repository metadata from stream input
-     *
-     * @param in stream input
-     * @return repository metadata
-     */
-    public static RepositoryMetaData readFrom(StreamInput in) throws IOException {
-        String name = in.readString();
-        String type = in.readString();
-        Settings settings = Settings.readSettingsFromStream(in);
-        return new RepositoryMetaData(name, type, settings);
+    public RepositoryMetaData(StreamInput in) throws IOException {
+        name = in.readString();
+        type = in.readString();
+        settings = Settings.readSettingsFromStream(in);
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/TemplateUpgradeService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/TemplateUpgradeService.java
new file mode 100644
index 0000000000000..8e8f0c594bc71
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/TemplateUpgradeService.java
@@ -0,0 +1,268 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.metadata;
+
+import com.carrotsearch.hppc.cursors.ObjectCursor;
+import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest;
+import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateResponse;
+import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
+import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateListener;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.component.AbstractComponent;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.gateway.GatewayService;
+import org.elasticsearch.indices.IndexTemplateMissingException;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.threadpool.ThreadPool;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.UnaryOperator;
+
+import static java.util.Collections.singletonMap;
+
+/**
+ * Upgrades Templates on behalf of installed {@link Plugin}s when a node joins the cluster
+ */
+public class TemplateUpgradeService extends AbstractComponent implements ClusterStateListener {
+    private final UnaryOperator<Map<String, IndexTemplateMetaData>> indexTemplateMetaDataUpgraders;
+
+    public final ClusterService clusterService;
+
+    public final ThreadPool threadPool;
+
+    public final Client client;
+
+    private final AtomicInteger updatesInProgress = new AtomicInteger();
+
+    private ImmutableOpenMap<String, IndexTemplateMetaData> lastTemplateMetaData;
+
+    public TemplateUpgradeService(Settings settings, Client client, ClusterService clusterService, ThreadPool threadPool,
+                                  Collection<UnaryOperator<Map<String, IndexTemplateMetaData>>> indexTemplateMetaDataUpgraders) {
+        super(settings);
+        this.client = client;
+        this.clusterService = clusterService;
+        this.threadPool = threadPool;
+        this.indexTemplateMetaDataUpgraders = templates -> {
+            Map<String, IndexTemplateMetaData> upgradedTemplates = new HashMap<>(templates);
+            for (UnaryOperator<Map<String, IndexTemplateMetaData>> upgrader : indexTemplateMetaDataUpgraders) {
+                upgradedTemplates = upgrader.apply(upgradedTemplates);
+            }
+            return upgradedTemplates;
+        };
+        clusterService.addListener(this);
+    }
+
+    @Override
+    public void clusterChanged(ClusterChangedEvent event) {
+        ClusterState state = event.state();
+        if (state.blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)) {
+            // wait until the gateway has recovered from disk, otherwise we think may not have the index templates,
+            // while they actually do exist
+            return;
+        }
+
+        if (updatesInProgress.get() > 0) {
+            // we are already running some updates - skip this cluster state update
+            return;
+        }
+
+        ImmutableOpenMap<String, IndexTemplateMetaData> templates = state.getMetaData().getTemplates();
+
+        if (templates == lastTemplateMetaData) {
+            // we already checked these sets of templates - no reason to check it again
+            // we can do identity check here because due to cluster state diffs the actual map will not change
+            // if there were no changes
+            return;
+        }
+
+        if (shouldLocalNodeUpdateTemplates(state.nodes()) == false) {
+            return;
+        }
+
+        lastTemplateMetaData = templates;
+        Optional<Tuple<Map<String, BytesReference>, Set<String>>> changes = calculateTemplateChanges(templates);
+        if (changes.isPresent()) {
+            if (updatesInProgress.compareAndSet(0, changes.get().v1().size() + changes.get().v2().size())) {
+                logger.info("Starting template upgrade to version {}, {} templates will be updated and {} will be removed",
+                    Version.CURRENT,
+                    changes.get().v1().size(),
+                    changes.get().v2().size());
+                threadPool.generic().execute(() -> updateTemplates(changes.get().v1(), changes.get().v2()));
+            }
+        }
+    }
+
+    /**
+     * Checks if the current node should update the templates
+     *
+     * If the master has the newest verison in the cluster - it will be dedicated template updater.
+     * Otherwise the node with the highest id among nodes with the highest version should update the templates
+     */
+    boolean shouldLocalNodeUpdateTemplates(DiscoveryNodes nodes) {
+        DiscoveryNode localNode = nodes.getLocalNode();
+        // Only data and master nodes should update the template
+        if (localNode.isDataNode() || localNode.isMasterNode()) {
+            DiscoveryNode masterNode = nodes.getMasterNode();
+            if (masterNode == null) {
+                return false;
+            }
+            Version maxVersion = nodes.getLargestNonClientNodeVersion();
+            if (maxVersion.equals(masterNode.getVersion())) {
+                // If the master has the latest version - we will allow it to handle the update
+                return nodes.isLocalNodeElectedMaster();
+            } else {
+                if (maxVersion.equals(localNode.getVersion()) == false) {
+                    // The localhost node doesn't have the latest version - not going to update
+                    return false;
+                }
+                for (ObjectCursor<DiscoveryNode> node : nodes.getMasterAndDataNodes().values()) {
+                    if (node.value.getVersion().equals(maxVersion) && node.value.getId().compareTo(localNode.getId()) > 0) {
+                        // We have a node with higher id then mine - it should update
+                        return false;
+                    }
+                }
+                // We have the highest version and highest id - we should perform the update
+                return true;
+            }
+        } else {
+            return false;
+        }
+    }
+
+    void updateTemplates(Map<String, BytesReference> changes, Set<String> deletions) {
+        for (Map.Entry<String, BytesReference> change : changes.entrySet()) {
+            PutIndexTemplateRequest request =
+                new PutIndexTemplateRequest(change.getKey()).source(change.getValue(), XContentType.JSON);
+            request.masterNodeTimeout(TimeValue.timeValueMinutes(1));
+            client.admin().indices().putTemplate(request, new ActionListener<PutIndexTemplateResponse>() {
+                @Override
+                public void onResponse(PutIndexTemplateResponse response) {
+                    if(updatesInProgress.decrementAndGet() == 0) {
+                        logger.info("Finished upgrading templates to version {}", Version.CURRENT);
+                    }
+                    if (response.isAcknowledged() == false) {
+                        logger.warn("Error updating template [{}], request was not acknowledged", change.getKey());
+                    }
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    if(updatesInProgress.decrementAndGet() == 0) {
+                        logger.info("Templates were upgraded to version {}", Version.CURRENT);
+                    }
+                    logger.warn(new ParameterizedMessage("Error updating template [{}]", change.getKey()), e);
+                }
+            });
+        }
+
+        for (String template : deletions) {
+            DeleteIndexTemplateRequest request = new DeleteIndexTemplateRequest(template);
+            request.masterNodeTimeout(TimeValue.timeValueMinutes(1));
+            client.admin().indices().deleteTemplate(request, new ActionListener<DeleteIndexTemplateResponse>() {
+                @Override
+                public void onResponse(DeleteIndexTemplateResponse response) {
+                    updatesInProgress.decrementAndGet();
+                    if (response.isAcknowledged() == false) {
+                        logger.warn("Error deleting template [{}], request was not acknowledged", template);
+                    }
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    updatesInProgress.decrementAndGet();
+                    if (e instanceof IndexTemplateMissingException == false) {
+                        // we might attempt to delete the same template from different nodes - so that's ok if template doesn't exist
+                        // otherwise we need to warn
+                        logger.warn(new ParameterizedMessage("Error deleting template [{}]", template), e);
+                    }
+                }
+            });
+        }
+    }
+
+    int getUpdatesInProgress() {
+        return updatesInProgress.get();
+    }
+
+    Optional<Tuple<Map<String, BytesReference>, Set<String>>> calculateTemplateChanges(
+        ImmutableOpenMap<String, IndexTemplateMetaData> templates) {
+        // collect current templates
+        Map<String, IndexTemplateMetaData> existingMap = new HashMap<>();
+        for (ObjectObjectCursor<String, IndexTemplateMetaData> customCursor : templates) {
+            existingMap.put(customCursor.key, customCursor.value);
+        }
+        // upgrade global custom meta data
+        Map<String, IndexTemplateMetaData> upgradedMap = indexTemplateMetaDataUpgraders.apply(existingMap);
+        if (upgradedMap.equals(existingMap) == false) {
+            Set<String> deletes = new HashSet<>();
+            Map<String, BytesReference> changes = new HashMap<>();
+            // remove templates if needed
+            existingMap.keySet().forEach(s -> {
+                if (upgradedMap.containsKey(s) == false) {
+                    deletes.add(s);
+                }
+            });
+            upgradedMap.forEach((key, value) -> {
+                if (value.equals(existingMap.get(key)) == false) {
+                    changes.put(key, toBytesReference(value));
+                }
+            });
+            return Optional.of(new Tuple<>(changes, deletes));
+        }
+        return Optional.empty();
+    }
+
+    private static final ToXContent.Params PARAMS = new ToXContent.MapParams(singletonMap("reduce_mappings", "true"));
+
+    private BytesReference toBytesReference(IndexTemplateMetaData templateMetaData) {
+        try {
+            return XContentHelper.toXContent((builder, params) -> {
+                IndexTemplateMetaData.Builder.toInnerXContent(templateMetaData, builder, params);
+                return builder;
+            }, XContentType.JSON, PARAMS, false);
+        } catch (IOException ex) {
+            throw new IllegalStateException("Cannot serialize template [" + templateMetaData.getName() + "]", ex);
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java b/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java
index 0c39c43bc9fe0..31b59b9f281fc 100644
--- a/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java
+++ b/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java
@@ -35,7 +35,6 @@
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 import java.util.function.Predicate;
@@ -96,7 +95,7 @@ public static boolean isIngestNode(Settings settings) {
      * @param version          the version of the node
      */
     public DiscoveryNode(final String id, TransportAddress address, Version version) {
-        this(id, address, Collections.emptyMap(), Collections.emptySet(), version);
+        this(id, address, Collections.emptyMap(), EnumSet.allOf(Role.class), version);
     }
 
     /**
@@ -192,18 +191,24 @@ public DiscoveryNode(String nodeName, String nodeId, String ephemeralId, String
     /** Creates a DiscoveryNode representing the local node. */
     public static DiscoveryNode createLocal(Settings settings, TransportAddress publishAddress, String nodeId) {
         Map<String, String> attributes = new HashMap<>(Node.NODE_ATTRIBUTES.get(settings).getAsMap());
-        Set<DiscoveryNode.Role> roles = new HashSet<>();
+        Set<Role> roles = getRolesFromSettings(settings);
+
+        return new DiscoveryNode(Node.NODE_NAME_SETTING.get(settings), nodeId, publishAddress, attributes, roles, Version.CURRENT);
+    }
+
+    /** extract node roles from the given settings */
+    public static Set<Role> getRolesFromSettings(Settings settings) {
+        Set<Role> roles = EnumSet.noneOf(Role.class);
         if (Node.NODE_INGEST_SETTING.get(settings)) {
-            roles.add(DiscoveryNode.Role.INGEST);
+            roles.add(Role.INGEST);
         }
         if (Node.NODE_MASTER_SETTING.get(settings)) {
-            roles.add(DiscoveryNode.Role.MASTER);
+            roles.add(Role.MASTER);
         }
         if (Node.NODE_DATA_SETTING.get(settings)) {
-            roles.add(DiscoveryNode.Role.DATA);
+            roles.add(Role.DATA);
         }
-
-        return new DiscoveryNode(Node.NODE_NAME_SETTING.get(settings), nodeId, publishAddress,attributes, roles, Version.CURRENT);
+        return roles;
     }
 
     /**
@@ -217,7 +222,14 @@ public DiscoveryNode(StreamInput in) throws IOException {
         this.ephemeralId = in.readString().intern();
         this.hostName = in.readString().intern();
         this.hostAddress = in.readString().intern();
-        this.address = TransportAddressSerializers.addressFromStream(in);
+        if (in.getVersion().after(Version.V_5_0_2)) {
+            this.address = TransportAddressSerializers.addressFromStream(in);
+        } else {
+            // we need to do this to preserve the host information during pinging and joining of a master. Since the version of the
+            // DiscoveryNode is set to Version#minimumCompatibilityVersion(), the host information gets lost as we do not serialize the
+            // hostString for the address
+            this.address = TransportAddressSerializers.addressFromStream(in, hostName);
+        }
         int size = in.readVInt();
         this.attributes = new HashMap<>(size);
         for (int i = 0; i < size; i++) {
@@ -226,11 +238,7 @@ public DiscoveryNode(StreamInput in) throws IOException {
         int rolesSize = in.readVInt();
         this.roles = EnumSet.noneOf(Role.class);
         for (int i = 0; i < rolesSize; i++) {
-            int ordinal = in.readVInt();
-            if (ordinal < 0 || ordinal >= Role.values().length) {
-                throw new IOException("Unknown Role ordinal [" + ordinal + "]");
-            }
-            this.roles.add(Role.values()[ordinal]);
+            this.roles.add(in.readEnum(Role.class));
         }
         this.version = Version.readVersion(in);
     }
@@ -250,7 +258,7 @@ public void writeTo(StreamOutput out) throws IOException {
         }
         out.writeVInt(roles.size());
         for (Role role : roles) {
-            out.writeVInt(role.ordinal());
+            out.writeEnum(role);
         }
         Version.writeVersion(version, out);
     }
diff --git a/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeFilters.java b/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeFilters.java
index e8ede54f4a2d1..23f8257227776 100644
--- a/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeFilters.java
+++ b/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeFilters.java
@@ -21,6 +21,7 @@
 
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.network.InetAddresses;
 import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
@@ -28,6 +29,7 @@
 
 import java.util.HashMap;
 import java.util.Map;
+import java.util.function.Consumer;
 
 /**
  */
@@ -38,6 +40,28 @@ public enum OpType {
         OR
     }
 
+    /**
+     * Validates the IP addresses in a group of {@link Settings} by looking for the keys
+     * "_ip", "_host_ip", and "_publish_ip" and ensuring each of their comma separated values
+     * that has no wildcards is a valid IP address.
+     */
+    public static final Consumer<Settings> IP_VALIDATOR = (settings) -> {
+        Map<String, String> settingsMap = settings.getAsMap();
+        for (Map.Entry<String, String> entry : settingsMap.entrySet()) {
+            String propertyKey = entry.getKey();
+            if (entry.getValue() == null) {
+                continue; // this setting gets reset
+            }
+            if ("_ip".equals(propertyKey) || "_host_ip".equals(propertyKey) || "_publish_ip".equals(propertyKey)) {
+                for (String value : Strings.tokenizeToStringArray(entry.getValue(), ",")) {
+                    if (Regex.isSimpleMatchPattern(value) == false && InetAddresses.isInetAddress(value) == false) {
+                        throw new IllegalArgumentException("invalid IP address [" + value + "] for [" + propertyKey + "]");
+                    }
+                }
+            }
+        }
+    };
+
     public static DiscoveryNodeFilters buildFromSettings(OpType opType, String prefix, Settings settings) {
         return buildFromKeyValue(opType, settings.getByPrefix(prefix).getAsMap());
     }
@@ -45,7 +69,7 @@ public static DiscoveryNodeFilters buildFromSettings(OpType opType, String prefi
     public static DiscoveryNodeFilters buildFromKeyValue(OpType opType, Map<String, String> filters) {
         Map<String, String[]> bFilters = new HashMap<>();
         for (Map.Entry<String, String> entry : filters.entrySet()) {
-            String[] values = Strings.splitStringByCommaToArray(entry.getValue());
+            String[] values = Strings.tokenizeToStringArray(entry.getValue(), ",");
             if (values.length > 0) {
                 bFilters.put(entry.getKey(), values);
             }
diff --git a/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java b/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java
index 9d0edf7b910db..b0f6604cac4f0 100644
--- a/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java
+++ b/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java
@@ -24,6 +24,7 @@
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.AbstractDiffable;
+import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
@@ -47,7 +48,6 @@
 public class DiscoveryNodes extends AbstractDiffable<DiscoveryNodes> implements Iterable<DiscoveryNode> {
 
     public static final DiscoveryNodes EMPTY_NODES = builder().build();
-    public static final DiscoveryNodes PROTO = EMPTY_NODES;
 
     private final ImmutableOpenMap<String, DiscoveryNode> nodes;
     private final ImmutableOpenMap<String, DiscoveryNode> dataNodes;
@@ -56,20 +56,25 @@ public class DiscoveryNodes extends AbstractDiffable<DiscoveryNodes> implements
 
     private final String masterNodeId;
     private final String localNodeId;
-    private final Version minNodeVersion;
     private final Version minNonClientNodeVersion;
+    private final Version maxNonClientNodeVersion;
+    private final Version maxNodeVersion;
+    private final Version minNodeVersion;
 
     private DiscoveryNodes(ImmutableOpenMap<String, DiscoveryNode> nodes, ImmutableOpenMap<String, DiscoveryNode> dataNodes,
                            ImmutableOpenMap<String, DiscoveryNode> masterNodes, ImmutableOpenMap<String, DiscoveryNode> ingestNodes,
-                           String masterNodeId, String localNodeId, Version minNodeVersion, Version minNonClientNodeVersion) {
+                           String masterNodeId, String localNodeId, Version minNonClientNodeVersion, Version maxNonClientNodeVersion,
+                           Version maxNodeVersion, Version minNodeVersion) {
         this.nodes = nodes;
         this.dataNodes = dataNodes;
         this.masterNodes = masterNodes;
         this.ingestNodes = ingestNodes;
         this.masterNodeId = masterNodeId;
         this.localNodeId = localNodeId;
-        this.minNodeVersion = minNodeVersion;
         this.minNonClientNodeVersion = minNonClientNodeVersion;
+        this.maxNonClientNodeVersion = maxNonClientNodeVersion;
+        this.minNodeVersion = minNodeVersion;
+        this.maxNodeVersion = maxNodeVersion;
     }
 
     @Override
@@ -173,7 +178,6 @@ public boolean nodeExists(DiscoveryNode node) {
         return existing != null && existing.equals(node);
     }
 
-
     /**
      * Get the id of the master node
      *
@@ -230,23 +234,44 @@ public boolean isAllNodes(String... nodesIds) {
         return nodesIds == null || nodesIds.length == 0 || (nodesIds.length == 1 && nodesIds[0].equals("_all"));
     }
 
+    /**
+     * Returns the version of the node with the oldest version in the cluster that is not a client node
+     *
+     * If there are no non-client nodes, Version.CURRENT will be returned.
+     *
+     * @return the oldest version in the cluster
+     */
+    public Version getSmallestNonClientNodeVersion() {
+        return minNonClientNodeVersion;
+    }
 
     /**
-     * Returns the version of the node with the oldest version in the cluster
+     * Returns the version of the node with the youngest version in the cluster that is not a client node.
+     *
+     * If there are no non-client nodes, Version.CURRENT will be returned.
+     *
+     * @return the youngest version in the cluster
+     */
+    public Version getLargestNonClientNodeVersion() {
+        return maxNonClientNodeVersion;
+    }
+
+    /**
+     * Returns the version of the node with the oldest version in the cluster.
      *
      * @return the oldest version in the cluster
      */
-    public Version getSmallestVersion() {
+    public Version getMinNodeVersion() {
         return minNodeVersion;
     }
 
     /**
-     * Returns the version of the node with the oldest version in the cluster that is not a client node
+     * Returns the version of the node with the yougest version in the cluster
      *
-     * @return the oldest version in the cluster
+     * @return the youngest version in the cluster
      */
-    public Version getSmallestNonClientNodeVersion() {
-        return minNonClientNodeVersion;
+    public Version getMaxNodeVersion() {
+        return maxNodeVersion;
     }
 
     /**
@@ -397,16 +422,6 @@ public Delta delta(DiscoveryNodes other) {
 
     @Override
     public String toString() {
-        StringBuilder sb = new StringBuilder();
-        sb.append("{");
-        for (DiscoveryNode node : this) {
-            sb.append(node).append(',');
-        }
-        sb.append("}");
-        return sb.toString();
-    }
-
-    public String prettyPrint() {
         StringBuilder sb = new StringBuilder();
         sb.append("nodes: \n");
         for (DiscoveryNode node : this) {
@@ -430,11 +445,7 @@ public static class Delta {
         private final List<DiscoveryNode> removed;
         private final List<DiscoveryNode> added;
 
-        public Delta(String localNodeId, List<DiscoveryNode> removed, List<DiscoveryNode> added) {
-            this(null, null, localNodeId, removed, added);
-        }
-
-        public Delta(@Nullable DiscoveryNode previousMasterNode, @Nullable DiscoveryNode newMasterNode, String localNodeId,
+        private Delta(@Nullable DiscoveryNode previousMasterNode, @Nullable DiscoveryNode newMasterNode, String localNodeId,
                      List<DiscoveryNode> removed, List<DiscoveryNode> added) {
             this.previousMasterNode = previousMasterNode;
             this.newMasterNode = newMasterNode;
@@ -538,7 +549,7 @@ public void writeTo(StreamOutput out) throws IOException {
         }
     }
 
-    private DiscoveryNodes readFrom(StreamInput in, DiscoveryNode localNode) throws IOException {
+    public static DiscoveryNodes readFrom(StreamInput in, DiscoveryNode localNode) throws IOException {
         Builder builder = new Builder();
         if (in.readBoolean()) {
             builder.masterNodeId(in.readString());
@@ -561,9 +572,8 @@ private DiscoveryNodes readFrom(StreamInput in, DiscoveryNode localNode) throws
         return builder.build();
     }
 
-    @Override
-    public DiscoveryNodes readFrom(StreamInput in) throws IOException {
-        return readFrom(in, getLocalNode());
+    public static Diff<DiscoveryNodes> readDiffFrom(StreamInput in, DiscoveryNode localNode) throws IOException {
+        return AbstractDiffable.readDiffFrom(in1 -> readFrom(in1, localNode), in);
     }
 
     public static Builder builder() {
@@ -668,33 +678,43 @@ public DiscoveryNodes build() {
             ImmutableOpenMap.Builder<String, DiscoveryNode> dataNodesBuilder = ImmutableOpenMap.builder();
             ImmutableOpenMap.Builder<String, DiscoveryNode> masterNodesBuilder = ImmutableOpenMap.builder();
             ImmutableOpenMap.Builder<String, DiscoveryNode> ingestNodesBuilder = ImmutableOpenMap.builder();
-            Version minNodeVersion = Version.CURRENT;
-            Version minNonClientNodeVersion = Version.CURRENT;
+            Version minNodeVersion = null;
+            Version maxNodeVersion = null;
+            Version minNonClientNodeVersion = null;
+            Version maxNonClientNodeVersion = null;
             for (ObjectObjectCursor<String, DiscoveryNode> nodeEntry : nodes) {
                 if (nodeEntry.value.isDataNode()) {
                     dataNodesBuilder.put(nodeEntry.key, nodeEntry.value);
-                    minNonClientNodeVersion = Version.smallest(minNonClientNodeVersion, nodeEntry.value.getVersion());
                 }
                 if (nodeEntry.value.isMasterNode()) {
                     masterNodesBuilder.put(nodeEntry.key, nodeEntry.value);
-                    minNonClientNodeVersion = Version.smallest(minNonClientNodeVersion, nodeEntry.value.getVersion());
+                }
+                final Version version = nodeEntry.value.getVersion();
+                if (nodeEntry.value.isDataNode() || nodeEntry.value.isMasterNode()) {
+                    if (minNonClientNodeVersion == null) {
+                        minNonClientNodeVersion = version;
+                        maxNonClientNodeVersion = version;
+                    } else {
+                        minNonClientNodeVersion = Version.min(minNonClientNodeVersion, version);
+                        maxNonClientNodeVersion = Version.max(maxNonClientNodeVersion, version);
+                    }
                 }
                 if (nodeEntry.value.isIngestNode()) {
                     ingestNodesBuilder.put(nodeEntry.key, nodeEntry.value);
                 }
-                minNodeVersion = Version.smallest(minNodeVersion, nodeEntry.value.getVersion());
+                minNodeVersion = minNodeVersion == null ? version : Version.min(minNodeVersion, version);
+                maxNodeVersion = maxNodeVersion == null ? version : Version.max(maxNodeVersion, version);
             }
 
             return new DiscoveryNodes(
                 nodes.build(), dataNodesBuilder.build(), masterNodesBuilder.build(), ingestNodesBuilder.build(),
-                masterNodeId, localNodeId, minNodeVersion, minNonClientNodeVersion
+                masterNodeId, localNodeId, minNonClientNodeVersion == null ? Version.CURRENT : minNonClientNodeVersion,
+                maxNonClientNodeVersion == null ? Version.CURRENT : maxNonClientNodeVersion,
+                maxNodeVersion == null ? Version.CURRENT : maxNodeVersion,
+                minNodeVersion == null ? Version.CURRENT : minNodeVersion
             );
         }
 
-        public static DiscoveryNodes readFrom(StreamInput in, @Nullable DiscoveryNode localNode) throws IOException {
-            return PROTO.readFrom(in, localNode);
-        }
-
         public boolean isLocalNodeElectedMaster() {
             return masterNodeId != null && masterNodeId.equals(localNodeId);
         }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/AllocationId.java b/core/src/main/java/org/elasticsearch/cluster/routing/AllocationId.java
index cb0fb487693e2..9ab91f9f8de2b 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/AllocationId.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/AllocationId.java
@@ -21,8 +21,6 @@
 
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -48,7 +46,7 @@ public class AllocationId implements ToXContent, Writeable {
     private static final String ID_KEY = "id";
     private static final String RELOCATION_ID_KEY = "relocation_id";
 
-    private static final ObjectParser<AllocationId.Builder, ParseFieldMatcherSupplier> ALLOCATION_ID_PARSER = new ObjectParser<>(
+    private static final ObjectParser<AllocationId.Builder, Void> ALLOCATION_ID_PARSER = new ObjectParser<>(
             "allocationId");
 
     static {
@@ -203,6 +201,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     }
 
     public static AllocationId fromXContent(XContentParser parser) throws IOException {
-        return ALLOCATION_ID_PARSER.parse(parser, new AllocationId.Builder(), () -> ParseFieldMatcher.STRICT).build();
+        return ALLOCATION_ID_PARSER.parse(parser, new AllocationId.Builder(), null).build();
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/DelayedAllocationService.java b/core/src/main/java/org/elasticsearch/cluster/routing/DelayedAllocationService.java
index 29d74dd893351..4522dfcf98f18 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/DelayedAllocationService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/DelayedAllocationService.java
@@ -106,12 +106,7 @@ public void onFailure(Exception e) {
         @Override
         public ClusterState execute(ClusterState currentState) throws Exception {
             removeIfSameTask(this);
-            RoutingAllocation.Result routingResult = allocationService.reroute(currentState, "assign delayed unassigned shards");
-            if (routingResult.changed()) {
-                return ClusterState.builder(currentState).routingResult(routingResult).build();
-            } else {
-                return currentState;
-            }
+            return allocationService.reroute(currentState, "assign delayed unassigned shards");
         }
 
         @Override
@@ -138,7 +133,7 @@ public DelayedAllocationService(Settings settings, ThreadPool threadPool, Cluste
         this.threadPool = threadPool;
         this.clusterService = clusterService;
         this.allocationService = allocationService;
-        clusterService.addFirst(this);
+        clusterService.addListener(this);
     }
 
     @Override
@@ -151,7 +146,7 @@ protected void doStop() {
 
     @Override
     protected void doClose() {
-        clusterService.remove(this);
+        clusterService.removeListener(this);
         removeTaskAndCancel();
     }
 
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/GroupShardsIterator.java b/core/src/main/java/org/elasticsearch/cluster/routing/GroupShardsIterator.java
index 9cf429383fdb7..21b02043a2249 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/GroupShardsIterator.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/GroupShardsIterator.java
@@ -30,14 +30,14 @@
  * ShardsIterators are always returned in ascending order independently of their order at construction
  * time. The incoming iterators are sorted to ensure consistent iteration behavior across Nodes / JVMs.
 */
-public class GroupShardsIterator implements Iterable<ShardIterator> {
+public final class GroupShardsIterator<ShardIt extends ShardIterator> implements Iterable<ShardIt> {
 
-    private final List<ShardIterator> iterators;
+    private final List<ShardIt> iterators;
 
     /**
      * Constructs a enw GroupShardsIterator from the given list.
      */
-    public GroupShardsIterator(List<ShardIterator> iterators) {
+    public GroupShardsIterator(List<ShardIt> iterators) {
         CollectionUtil.timSort(iterators);
         this.iterators = iterators;
     }
@@ -60,13 +60,8 @@ public int totalSize() {
      */
     public int totalSizeWith1ForEmpty() {
         int size = 0;
-        for (ShardIterator shard : iterators) {
-            int sizeActive = shard.size();
-            if (sizeActive == 0) {
-                size += 1;
-            } else {
-                size += sizeActive;
-            }
+        for (ShardIt shard : iterators) {
+            size += Math.max(1, shard.size());
         }
         return size;
     }
@@ -80,7 +75,11 @@ public int size() {
     }
 
     @Override
-    public Iterator<ShardIterator> iterator() {
+    public Iterator<ShardIt> iterator() {
         return iterators.iterator();
     }
+
+    public ShardIt get(int index) {
+        return  iterators.get(index);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java b/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java
index fd8e7d02b28a6..9de95fab72695 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java
@@ -25,13 +25,13 @@
 import org.apache.lucene.util.CollectionUtil;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.AbstractDiffable;
+import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.routing.RecoverySource.LocalShardsRecoverySource;
 import org.elasticsearch.cluster.routing.RecoverySource.PeerRecoverySource;
 import org.elasticsearch.cluster.routing.RecoverySource.SnapshotRecoverySource;
 import org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource;
-import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.collect.ImmutableOpenIntMap;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -64,8 +64,6 @@
  */
 public class IndexRoutingTable extends AbstractDiffable<IndexRoutingTable> implements Iterable<IndexShardRoutingTable> {
 
-    public static final IndexRoutingTable PROTO = builder(new Index("", "_na_")).build();
-
     private final Index index;
     private final ShardShuffler shuffler;
 
@@ -134,11 +132,22 @@ boolean validate(MetaData metaData) {
                     throw new IllegalStateException("shard routing has an index [" + shardRouting.index() + "] that is different " +
                                                     "from the routing table");
                 }
+                final Set<String> inSyncAllocationIds = indexMetaData.inSyncAllocationIds(shardRouting.id());
                 if (shardRouting.active() &&
-                    indexMetaData.inSyncAllocationIds(shardRouting.id()).contains(shardRouting.allocationId().getId()) == false) {
+                    inSyncAllocationIds.contains(shardRouting.allocationId().getId()) == false) {
                     throw new IllegalStateException("active shard routing " + shardRouting + " has no corresponding entry in the in-sync " +
-                        "allocation set " + indexMetaData.inSyncAllocationIds(shardRouting.id()));
+                        "allocation set " + inSyncAllocationIds);
                 }
+
+                if (indexMetaData.getCreationVersion().onOrAfter(Version.V_5_0_0_alpha1) &&
+                    IndexMetaData.isIndexUsingShadowReplicas(indexMetaData.getSettings()) == false && // see #20650
+                    shardRouting.primary() && shardRouting.initializing() &&
+                    shardRouting.recoverySource().getType() == RecoverySource.Type.SNAPSHOT &&
+                    inSyncAllocationIds.contains(shardRouting.allocationId().getId()) == false)
+                    throw new IllegalStateException("a primary shard routing " + shardRouting + " is a primary that is recovering from " +
+                        "a known allocation id but has no corresponding entry in the in-sync " +
+                        "allocation set " + inSyncAllocationIds);
+
             }
         }
         return true;
@@ -258,37 +267,6 @@ public ShardsIterator randomAllActiveShardsIt() {
         return new PlainShardsIterator(shuffler.shuffle(allActiveShards));
     }
 
-    /**
-     * A group shards iterator where each group ({@link ShardIterator}
-     * is an iterator across shard replication group.
-     */
-    public GroupShardsIterator groupByShardsIt() {
-        // use list here since we need to maintain identity across shards
-        ArrayList<ShardIterator> set = new ArrayList<>(shards.size());
-        for (IndexShardRoutingTable indexShard : this) {
-            set.add(indexShard.shardsIt());
-        }
-        return new GroupShardsIterator(set);
-    }
-
-    /**
-     * A groups shards iterator where each groups is a single {@link ShardRouting} and a group
-     * is created for each shard routing.
-     * <p>
-     * This basically means that components that use the {@link GroupShardsIterator} will iterate
-     * over *all* the shards (all the replicas) within the index.</p>
-     */
-    public GroupShardsIterator groupByAllIt() {
-        // use list here since we need to maintain identity across shards
-        ArrayList<ShardIterator> set = new ArrayList<>();
-        for (IndexShardRoutingTable indexShard : this) {
-            for (ShardRouting shardRouting : indexShard) {
-                set.add(shardRouting.shardsIt());
-            }
-        }
-        return new GroupShardsIterator(set);
-    }
-
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
@@ -309,8 +287,7 @@ public int hashCode() {
         return result;
     }
 
-    @Override
-    public IndexRoutingTable readFrom(StreamInput in) throws IOException {
+    public static IndexRoutingTable readFrom(StreamInput in) throws IOException {
         Index index = new Index(in);
         Builder builder = new Builder(index);
 
@@ -322,6 +299,10 @@ public IndexRoutingTable readFrom(StreamInput in) throws IOException {
         return builder.build();
     }
 
+    public static Diff<IndexRoutingTable> readDiffFrom(StreamInput in) throws IOException {
+        return readDiffFrom(IndexRoutingTable::readFrom, in);
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         index.writeTo(out);
@@ -344,46 +325,32 @@ public Builder(Index index) {
             this.index = index;
         }
 
-        /**
-         * Reads an {@link IndexRoutingTable} from an {@link StreamInput}
-         *
-         * @param in {@link StreamInput} to read the {@link IndexRoutingTable} from
-         * @return {@link IndexRoutingTable} read
-         * @throws IOException if something happens during read
-         */
-        public static IndexRoutingTable readFrom(StreamInput in) throws IOException {
-            return PROTO.readFrom(in);
-        }
-
         /**
          * Initializes a new empty index, as if it was created from an API.
          */
         public Builder initializeAsNew(IndexMetaData indexMetaData) {
-            RecoverySource primaryRecoverySource = indexMetaData.getMergeSourceIndex() != null ?
-                LocalShardsRecoverySource.INSTANCE :
-                StoreRecoverySource.EMPTY_STORE_INSTANCE;
-            return initializeEmpty(indexMetaData, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null), primaryRecoverySource);
+            return initializeEmpty(indexMetaData, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null));
         }
 
         /**
          * Initializes an existing index.
          */
         public Builder initializeAsRecovery(IndexMetaData indexMetaData) {
-            return initializeEmpty(indexMetaData, new UnassignedInfo(UnassignedInfo.Reason.CLUSTER_RECOVERED, null), null);
+            return initializeEmpty(indexMetaData, new UnassignedInfo(UnassignedInfo.Reason.CLUSTER_RECOVERED, null));
         }
 
         /**
          * Initializes a new index caused by dangling index imported.
          */
         public Builder initializeAsFromDangling(IndexMetaData indexMetaData) {
-            return initializeEmpty(indexMetaData, new UnassignedInfo(UnassignedInfo.Reason.DANGLING_INDEX_IMPORTED, null), null);
+            return initializeEmpty(indexMetaData, new UnassignedInfo(UnassignedInfo.Reason.DANGLING_INDEX_IMPORTED, null));
         }
 
         /**
          * Initializes a new empty index, as as a result of opening a closed index.
          */
         public Builder initializeAsFromCloseToOpen(IndexMetaData indexMetaData) {
-            return initializeEmpty(indexMetaData, new UnassignedInfo(UnassignedInfo.Reason.INDEX_REOPENED, null), null);
+            return initializeEmpty(indexMetaData, new UnassignedInfo(UnassignedInfo.Reason.INDEX_REOPENED, null));
         }
 
         /**
@@ -435,28 +402,36 @@ private Builder initializeAsRestore(IndexMetaData indexMetaData, SnapshotRecover
 
         /**
          * Initializes a new empty index, with an option to control if its from an API or not.
-         *
-         * @param primaryRecoverySource recovery source for primary shards. If null, it is automatically determined based on active
-         *                              allocation ids
          */
-        private Builder initializeEmpty(IndexMetaData indexMetaData, UnassignedInfo unassignedInfo, @Nullable RecoverySource primaryRecoverySource) {
+        private Builder initializeEmpty(IndexMetaData indexMetaData, UnassignedInfo unassignedInfo) {
             assert indexMetaData.getIndex().equals(index);
             if (!shards.isEmpty()) {
                 throw new IllegalStateException("trying to initialize an index with fresh shards, but already has shards created");
             }
             for (int shardNumber = 0; shardNumber < indexMetaData.getNumberOfShards(); shardNumber++) {
                 ShardId shardId = new ShardId(index, shardNumber);
-                if (primaryRecoverySource == null) {
-                    if (indexMetaData.inSyncAllocationIds(shardNumber).isEmpty() && indexMetaData.getCreationVersion().onOrAfter(Version.V_5_0_0_alpha1)) {
-                        primaryRecoverySource = indexMetaData.getMergeSourceIndex() != null ? LocalShardsRecoverySource.INSTANCE : StoreRecoverySource.EMPTY_STORE_INSTANCE;
-                    } else {
-                        primaryRecoverySource = StoreRecoverySource.EXISTING_STORE_INSTANCE;
-                    }
+                final RecoverySource primaryRecoverySource;
+                if (indexMetaData.inSyncAllocationIds(shardNumber).isEmpty() == false) {
+                    // we have previous valid copies for this shard. use them for recovery
+                    primaryRecoverySource = StoreRecoverySource.EXISTING_STORE_INSTANCE;
+                } else if (indexMetaData.getCreationVersion().before(Version.V_5_0_0_alpha1) &&
+                    unassignedInfo.getReason() != UnassignedInfo.Reason.INDEX_CREATED // tests can create old indices
+                    ) {
+                    // the index is old and didn't maintain inSyncAllocationIds. Fall back to old behavior and require
+                    // finding existing copies
+                    primaryRecoverySource = StoreRecoverySource.EXISTING_STORE_INSTANCE;
+                } else if (indexMetaData.getMergeSourceIndex() != null) {
+                    // this is a new index but the initial shards should merged from another index
+                    primaryRecoverySource = LocalShardsRecoverySource.INSTANCE;
+                } else {
+                    // a freshly created index with no restriction
+                    primaryRecoverySource = StoreRecoverySource.EMPTY_STORE_INSTANCE;
                 }
                 IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(shardId);
                 for (int i = 0; i <= indexMetaData.getNumberOfReplicas(); i++) {
                     boolean primary = i == 0;
-                    indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(shardId, primary, primary ? primaryRecoverySource : PeerRecoverySource.INSTANCE, unassignedInfo));
+                    indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(shardId, primary,
+                        primary ? primaryRecoverySource : PeerRecoverySource.INSTANCE, unassignedInfo));
                 }
                 shards.put(shardNumber, indexShardRoutingBuilder.build());
             }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java b/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java
index 619959923e942..2028cc5b8c943 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java
@@ -33,6 +33,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
@@ -184,6 +185,15 @@ public List<ShardRouting> activeShards() {
         return this.activeShards;
     }
 
+    /**
+     * Returns a {@link List} of all initializing shards, including target shards of relocations
+     *
+     * @return a {@link List} of shards
+     */
+    public List<ShardRouting> getAllInitializingShards() {
+        return this.allInitializingShards;
+    }
+
     /**
      * Returns a {@link List} of active shards
      *
@@ -572,14 +582,6 @@ public Builder(ShardId shardId) {
         }
 
         public Builder addShard(ShardRouting shardEntry) {
-            for (ShardRouting shard : shards) {
-                // don't add two that map to the same node id
-                // we rely on the fact that a node does not have primary and backup of the same shard
-                if (shard.assignedToNode() && shardEntry.assignedToNode()
-                        && shard.currentNodeId().equals(shardEntry.currentNodeId())) {
-                    return this;
-                }
-            }
             shards.add(shardEntry);
             return this;
         }
@@ -590,9 +592,28 @@ public Builder removeShard(ShardRouting shardEntry) {
         }
 
         public IndexShardRoutingTable build() {
+            // don't allow more than one shard copy with same id to be allocated to same node
+            assert distinctNodes(shards) : "more than one shard with same id assigned to same node (shards: " + shards + ")";
             return new IndexShardRoutingTable(shardId, Collections.unmodifiableList(new ArrayList<>(shards)));
         }
 
+        static boolean distinctNodes(List<ShardRouting> shards) {
+            Set<String> nodes = new HashSet<>();
+            for (ShardRouting shard : shards) {
+                if (shard.assignedToNode()) {
+                    if (nodes.add(shard.currentNodeId()) == false) {
+                        return false;
+                    }
+                    if (shard.relocating()) {
+                        if (nodes.add(shard.relocatingNodeId()) == false) {
+                            return false;
+                        }
+                    }
+                }
+            }
+            return true;
+        }
+
         public static IndexShardRoutingTable readFrom(StreamInput in) throws IOException {
             Index index = new Index(in);
             return readFromThin(in, index);
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java b/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java
index ef3fae4830137..5280725169980 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java
@@ -68,12 +68,7 @@ public ShardIterator getShards(ClusterState clusterState, String index, int shar
         return preferenceActiveShardIterator(indexShard, clusterState.nodes().getLocalNodeId(), clusterState.nodes(), preference);
     }
 
-    public int searchShardsCount(ClusterState clusterState, String[] concreteIndices, @Nullable Map<String, Set<String>> routing) {
-        final Set<IndexShardRoutingTable> shards = computeTargetedShards(clusterState, concreteIndices, routing);
-        return shards.size();
-    }
-
-    public GroupShardsIterator searchShards(ClusterState clusterState, String[] concreteIndices, @Nullable Map<String, Set<String>> routing, @Nullable String preference) {
+    public GroupShardsIterator<ShardIterator> searchShards(ClusterState clusterState, String[] concreteIndices, @Nullable Map<String, Set<String>> routing, @Nullable String preference) {
         final Set<IndexShardRoutingTable> shards = computeTargetedShards(clusterState, concreteIndices, routing);
         final Set<ShardIterator> set = new HashSet<>(shards.size());
         for (IndexShardRoutingTable shard : shards) {
@@ -82,7 +77,7 @@ public GroupShardsIterator searchShards(ClusterState clusterState, String[] conc
                 set.add(iterator);
             }
         }
-        return new GroupShardsIterator(new ArrayList<>(set));
+        return new GroupShardsIterator<>(new ArrayList<>(set));
     }
 
     private static final Map<String, Set<String>> EMPTY_ROUTING = Collections.emptyMap();
@@ -97,13 +92,10 @@ private Set<IndexShardRoutingTable> computeTargetedShards(ClusterState clusterSt
             final Set<String> effectiveRouting = routing.get(index);
             if (effectiveRouting != null) {
                 for (String r : effectiveRouting) {
-                    int shardId = generateShardId(indexMetaData, null, r);
-                    IndexShardRoutingTable indexShard = indexRouting.shard(shardId);
-                    if (indexShard == null) {
-                        throw new ShardNotFoundException(new ShardId(indexRouting.getIndex(), shardId));
+                    final int routingPartitionSize = indexMetaData.getRoutingPartitionSize();
+                    for (int partitionOffset = 0; partitionOffset < routingPartitionSize; partitionOffset++) {
+                        set.add(shardRoutingTable(indexRouting, calculateScaledShardId(indexMetaData, r, partitionOffset)));
                     }
-                    // we might get duplicates, but that's ok, they will override one another
-                    set.add(indexShard);
                 }
             } else {
                 for (IndexShardRoutingTable indexShard : indexRouting) {
@@ -126,7 +118,7 @@ private ShardIterator preferenceActiveShardIterator(IndexShardRoutingTable index
             Preference preferenceType = Preference.parse(preference);
             if (preferenceType == Preference.SHARDS) {
                 // starts with _shards, so execute on specific ones
-                int index = preference.indexOf(';');
+                int index = preference.indexOf('|');
 
                 String shards;
                 if (index == -1) {
@@ -192,6 +184,14 @@ private ShardIterator preferenceActiveShardIterator(IndexShardRoutingTable index
         }
     }
 
+    private IndexShardRoutingTable shardRoutingTable(IndexRoutingTable indexRouting, int shardId) {
+        IndexShardRoutingTable indexShard = indexRouting.shard(shardId);
+        if (indexShard == null) {
+            throw new ShardNotFoundException(new ShardId(indexRouting.getIndex(), shardId));
+        }
+        return indexShard;
+    }
+
     protected IndexRoutingTable indexRoutingTable(ClusterState clusterState, String index) {
         IndexRoutingTable indexRouting = clusterState.routingTable().index(index);
         if (indexRouting == null) {
@@ -218,15 +218,33 @@ public ShardId shardId(ClusterState clusterState, String index, String id, @Null
         return new ShardId(indexMetaData.getIndex(), generateShardId(indexMetaData, id, routing));
     }
 
-    static int generateShardId(IndexMetaData indexMetaData, String id, @Nullable String routing) {
-        final int hash;
+    static int generateShardId(IndexMetaData indexMetaData, @Nullable String id, @Nullable String routing) {
+        final String effectiveRouting;
+        final int partitionOffset;
+
         if (routing == null) {
-            hash = Murmur3HashFunction.hash(id);
+            assert(indexMetaData.isRoutingPartitionedIndex() == false) : "A routing value is required for gets from a partitioned index";
+            effectiveRouting = id;
         } else {
-            hash = Murmur3HashFunction.hash(routing);
+            effectiveRouting = routing;
         }
+
+        if (indexMetaData.isRoutingPartitionedIndex()) {
+            partitionOffset = Math.floorMod(Murmur3HashFunction.hash(id), indexMetaData.getRoutingPartitionSize());
+        } else {
+            // we would have still got 0 above but this check just saves us an unnecessary hash calculation
+            partitionOffset = 0;
+        }
+
+        return calculateScaledShardId(indexMetaData, effectiveRouting, partitionOffset);
+    }
+
+    private static int calculateScaledShardId(IndexMetaData indexMetaData, String effectiveRouting, int partitionOffset) {
+        final int hash = Murmur3HashFunction.hash(effectiveRouting) + partitionOffset;
+
         // we don't use IMD#getNumberOfShards since the index might have been shrunk such that we need to use the size
         // of original index to hash documents
         return Math.floorMod(hash, indexMetaData.getRoutingNumShards()) / indexMetaData.getRoutingFactor();
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/PlainShardIterator.java b/core/src/main/java/org/elasticsearch/cluster/routing/PlainShardIterator.java
index 5950bd35d37f0..bb45ca66956f8 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/PlainShardIterator.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/PlainShardIterator.java
@@ -43,7 +43,6 @@ public PlainShardIterator(ShardId shardId, List<ShardRouting> shards) {
         this.shardId = shardId;
     }
 
-
     @Override
     public ShardId shardId() {
         return this.shardId;
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/PlainShardsIterator.java b/core/src/main/java/org/elasticsearch/cluster/routing/PlainShardsIterator.java
index 3c2a338ae6467..6cb1989a8dd02 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/PlainShardsIterator.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/PlainShardsIterator.java
@@ -18,6 +18,8 @@
  */
 package org.elasticsearch.cluster.routing;
 
+import java.util.Collections;
+import java.util.Iterator;
 import java.util.List;
 
 /**
@@ -74,33 +76,12 @@ public int sizeActive() {
     }
 
     @Override
-    public int assignedReplicasIncludingRelocating() {
-        int count = 0;
-        for (ShardRouting shard : shards) {
-            if (shard.unassigned()) {
-                continue;
-            }
-            // if the shard is primary and relocating, add one to the counter since we perform it on the replica as well
-            // (and we already did it on the primary)
-            if (shard.primary()) {
-                if (shard.relocating()) {
-                    count++;
-                }
-            } else {
-                count++;
-                // if we are relocating the replica, we want to perform the index operation on both the relocating
-                // shard and the target shard. This means that we won't loose index operations between end of recovery
-                // and reassignment of the shard by the master node
-                if (shard.relocating()) {
-                    count++;
-                }
-            }
-        }
-        return count;
+    public List<ShardRouting> getShardRoutings() {
+        return Collections.unmodifiableList(shards);
     }
 
     @Override
-    public Iterable<ShardRouting> asUnordered() {
-        return shards;
+    public Iterator<ShardRouting> iterator() {
+        return shards.iterator();
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingChangesObserver.java b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingChangesObserver.java
index 0f3a8c6f214c6..883b4c22f7fc0 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingChangesObserver.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingChangesObserver.java
@@ -69,6 +69,12 @@ public interface RoutingChangesObserver {
      */
     void replicaPromoted(ShardRouting replicaShard);
 
+    /**
+     * Called when an initializing replica is reinitialized. This happens when a primary relocation completes, which
+     * reinitializes all currently initializing replicas as their recovery source node changes
+     */
+    void initializedReplicaReinitialized(ShardRouting oldReplica, ShardRouting reinitializedReplica);
+
 
     /**
      * Abstract implementation of {@link RoutingChangesObserver} that does not take any action. Useful for subclasses that only override
@@ -120,6 +126,11 @@ public void startedPrimaryReinitialized(ShardRouting startedPrimaryShard, ShardR
         public void replicaPromoted(ShardRouting replicaShard) {
 
         }
+
+        @Override
+        public void initializedReplicaReinitialized(ShardRouting oldReplica, ShardRouting reinitializedReplica) {
+
+        }
     }
 
     class DelegatingRoutingChangesObserver implements RoutingChangesObserver {
@@ -192,5 +203,12 @@ public void replicaPromoted(ShardRouting replicaShard) {
                 routingChangesObserver.replicaPromoted(replicaShard);
             }
         }
+
+        @Override
+        public void initializedReplicaReinitialized(ShardRouting oldReplica, ShardRouting reinitializedReplica) {
+            for (RoutingChangesObserver routingChangesObserver : routingChangesObservers) {
+                routingChangesObserver.initializedReplicaReinitialized(oldReplica, reinitializedReplica);
+            }
+        }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNode.java b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNode.java
index 8403f45a550b1..4ba277d99cd5d 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNode.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNode.java
@@ -19,18 +19,15 @@
 
 package org.elasticsearch.cluster.routing;
 
-import org.apache.lucene.util.CollectionUtil;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.index.shard.ShardId;
 
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.Comparator;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
-import java.util.Map;
 
 /**
  * A {@link RoutingNode} represents a cluster node associated with a single {@link DiscoveryNode} including all shards
@@ -103,7 +100,8 @@ public int size() {
      */
     void add(ShardRouting shard) {
         if (shards.containsKey(shard.shardId())) {
-            throw new IllegalStateException("Trying to add a shard " + shard.shardId() + " to a node [" + nodeId + "] where it already exists");
+            throw new IllegalStateException("Trying to add a shard " + shard.shardId() + " to a node [" + nodeId
+                + "] where it already exists. current [" + shards.get(shard.shardId()) + "]. new [" + shard + "]");
         }
         shards.put(shard.shardId(), shard);
     }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java
index 553e3b7324769..5016330e92bc8 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java
@@ -23,6 +23,8 @@
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.CollectionUtil;
+import org.elasticsearch.Assertions;
+import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
@@ -319,14 +321,35 @@ public ShardRouting activePrimary(ShardId shardId) {
     /**
      * Returns one active replica shard for the given shard id or <code>null</code> if
      * no active replica is found.
+     *
+     * Since replicas could possibly be on nodes with a older version of ES than
+     * the primary is, this will return replicas on the highest version of ES.
+     *
      */
-    public ShardRouting activeReplica(ShardId shardId) {
+    public ShardRouting activeReplicaWithHighestVersion(ShardId shardId) {
+        Version highestVersionSeen = null;
+        ShardRouting candidate = null;
         for (ShardRouting shardRouting : assignedShards(shardId)) {
             if (!shardRouting.primary() && shardRouting.active()) {
-                return shardRouting;
+                // It's possible for replicaNodeVersion to be null, when deassociating dead nodes
+                // that have been removed, the shards are failed, and part of the shard failing
+                // calls this method with an out-of-date RoutingNodes, where the version might not
+                // be accessible. Therefore, we need to protect against the version being null
+                // (meaning the node will be going away).
+                RoutingNode replicaNode = node(shardRouting.currentNodeId());
+                if (replicaNode != null && replicaNode.node() != null) {
+                    Version replicaNodeVersion = replicaNode.node().getVersion();
+                    if (highestVersionSeen == null || replicaNodeVersion.after(highestVersionSeen)) {
+                        highestVersionSeen = replicaNodeVersion;
+                        candidate = shardRouting;
+                    } else if (candidate == null) {
+                        // Only use this replica if there are no other candidates
+                        candidate = shardRouting;
+                    }
+                }
             }
         }
-        return null;
+        return candidate;
     }
 
     /**
@@ -391,7 +414,8 @@ public List<ShardRouting> shardsWithState(String index, ShardRoutingState... sta
         return shards;
     }
 
-    public String prettyPrint() {
+    @Override
+    public String toString() {
         StringBuilder sb = new StringBuilder("routing_nodes:\n");
         for (RoutingNode routingNode : this) {
             sb.append(routingNode.prettyPrint());
@@ -450,6 +474,9 @@ public Tuple<ShardRouting,ShardRouting> relocateShard(ShardRouting startedShard,
      *
      * Moves the initializing shard to started. If the shard is a relocation target, also removes the relocation source.
      *
+     * If the started shard is a primary relocation target, this also reinitializes currently initializing replicas as their
+     * recovery source changes
+     *
      * @return the started shard
      */
     public ShardRouting startShard(Logger logger, ShardRouting initializingShard, RoutingChangesObserver routingChangesObserver) {
@@ -467,6 +494,30 @@ public ShardRouting startShard(Logger logger, ShardRouting initializingShard, Ro
                 + initializingShard + " but was: " + relocationSourceShard.getTargetRelocatingShard();
             remove(relocationSourceShard);
             routingChangesObserver.relocationCompleted(relocationSourceShard);
+
+            // if this is a primary shard with ongoing replica recoveries, reinitialize them as their recovery source changed
+            if (startedShard.primary()) {
+                List<ShardRouting> assignedShards = assignedShards(startedShard.shardId());
+                // copy list to prevent ConcurrentModificationException
+                for (ShardRouting routing : new ArrayList<>(assignedShards)) {
+                    if (routing.initializing() && routing.primary() == false) {
+                        if (routing.isRelocationTarget()) {
+                            // find the relocation source
+                            ShardRouting sourceShard = getByAllocationId(routing.shardId(), routing.allocationId().getRelocationId());
+                            // cancel relocation and start relocation to same node again
+                            ShardRouting startedReplica = cancelRelocation(sourceShard);
+                            remove(routing);
+                            routingChangesObserver.shardFailed(routing,
+                                new UnassignedInfo(UnassignedInfo.Reason.REINITIALIZED, "primary changed"));
+                            relocateShard(startedReplica, sourceShard.relocatingNodeId(),
+                                sourceShard.getExpectedShardSize(), routingChangesObserver);
+                        } else {
+                            ShardRouting reinitializedReplica = reinitReplica(routing);
+                            routingChangesObserver.initializedReplicaReinitialized(routing, reinitializedReplica);
+                        }
+                    }
+                }
+            }
         }
         return startedShard;
     }
@@ -536,8 +587,22 @@ assert getByAllocationId(failedShard.shardId(), failedShard.allocationId().getId
         // fail actual shard
         if (failedShard.initializing()) {
             if (failedShard.relocatingNodeId() == null) {
-                // initializing shard that is not relocation target, just move to unassigned
-                moveToUnassigned(failedShard, unassignedInfo);
+                if (failedShard.primary()) {
+                    // promote active replica to primary if active replica exists (only the case for shadow replicas)
+                    ShardRouting activeReplica = activeReplicaWithHighestVersion(failedShard.shardId());
+                    assert activeReplica == null || IndexMetaData.isIndexUsingShadowReplicas(indexMetaData.getSettings()) :
+                        "initializing primary [" + failedShard + "] with active replicas [" + activeReplica + "] only expected when " +
+                            "using shadow replicas";
+                    if (activeReplica == null) {
+                        moveToUnassigned(failedShard, unassignedInfo);
+                    } else {
+                        movePrimaryToUnassignedAndDemoteToReplica(failedShard, unassignedInfo);
+                        promoteReplicaToPrimary(activeReplica, indexMetaData, routingChangesObserver);
+                    }
+                } else {
+                    // initializing shard that is not relocation target, just move to unassigned
+                    moveToUnassigned(failedShard, unassignedInfo);
+                }
             } else {
                 // The shard is a target of a relocating shard. In that case we only need to remove the target shard and cancel the source
                 // relocation. No shard is left unassigned
@@ -556,20 +621,12 @@ assert getByAllocationId(failedShard.shardId(), failedShard.allocationId().getId
             assert failedShard.active();
             if (failedShard.primary()) {
                 // promote active replica to primary if active replica exists
-                ShardRouting activeReplica = activeReplica(failedShard.shardId());
+                ShardRouting activeReplica = activeReplicaWithHighestVersion(failedShard.shardId());
                 if (activeReplica == null) {
                     moveToUnassigned(failedShard, unassignedInfo);
                 } else {
-                    // if the activeReplica was relocating before this call to failShard, its relocation was cancelled above when we
-                    // failed initializing replica shards (and moved replica relocation source back to started)
-                    assert activeReplica.started() : "replica relocation should have been cancelled: " + activeReplica;
                     movePrimaryToUnassignedAndDemoteToReplica(failedShard, unassignedInfo);
-                    ShardRouting primarySwappedCandidate = promoteActiveReplicaShardToPrimary(activeReplica);
-                    routingChangesObserver.replicaPromoted(activeReplica);
-                    if (IndexMetaData.isIndexUsingShadowReplicas(indexMetaData.getSettings())) {
-                        ShardRouting initializedShard = reinitShadowPrimary(primarySwappedCandidate);
-                        routingChangesObserver.startedPrimaryReinitialized(primarySwappedCandidate, initializedShard);
-                    }
+                    promoteReplicaToPrimary(activeReplica, indexMetaData, routingChangesObserver);
                 }
             } else {
                 assert failedShard.primary() == false;
@@ -585,6 +642,19 @@ assert node(failedShard.currentNodeId()).getByShardId(failedShard.shardId()) ==
             " was matched but wasn't removed";
     }
 
+    private void promoteReplicaToPrimary(ShardRouting activeReplica, IndexMetaData indexMetaData,
+                                         RoutingChangesObserver routingChangesObserver) {
+        // if the activeReplica was relocating before this call to failShard, its relocation was cancelled earlier when we
+        // failed initializing replica shards (and moved replica relocation source back to started)
+        assert activeReplica.started() : "replica relocation should have been cancelled: " + activeReplica;
+        ShardRouting primarySwappedCandidate = promoteActiveReplicaShardToPrimary(activeReplica);
+        routingChangesObserver.replicaPromoted(activeReplica);
+        if (IndexMetaData.isIndexUsingShadowReplicas(indexMetaData.getSettings())) {
+            ShardRouting initializedShard = reinitShadowPrimary(primarySwappedCandidate);
+            routingChangesObserver.startedPrimaryReinitialized(primarySwappedCandidate, initializedShard);
+        }
+    }
+
     /**
      * Mark a shard as started and adjusts internal statistics.
      *
@@ -706,6 +776,16 @@ private ShardRouting reinitShadowPrimary(ShardRouting candidate) {
         updateAssigned(candidate, reinitializedShard);
         inactivePrimaryCount++;
         inactiveShardCount++;
+        addRecovery(reinitializedShard);
+        return reinitializedShard;
+    }
+
+    private ShardRouting reinitReplica(ShardRouting shard) {
+        assert shard.primary() == false : "shard must be a replica: " + shard;
+        assert shard.initializing() : "can only reinitialize an initializing replica: " + shard;
+        assert shard.isRelocationTarget() == false : "replication target cannot be reinitialized: " + shard;
+        ShardRouting reinitializedShard = shard.reinitializeReplicaShard();
+        updateAssigned(shard, reinitializedShard);
         return reinitializedShard;
     }
 
@@ -968,9 +1048,7 @@ public ShardRouting[] drain() {
      *         this method does nothing.
      */
     public static boolean assertShardStats(RoutingNodes routingNodes) {
-        boolean run = false;
-        assert (run = true); // only run if assertions are enabled!
-        if (!run) {
+        if (!Assertions.ENABLED) {
             return true;
         }
         int unassignedPrimaryCount = 0;
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java
index 8300d3e37fd64..1c3d629a72fea 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java
@@ -25,7 +25,6 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
-import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
@@ -96,12 +95,7 @@ protected void performReroute(String reason) {
                 @Override
                 public ClusterState execute(ClusterState currentState) {
                     rerouting.set(false);
-                    RoutingAllocation.Result routingResult = allocationService.reroute(currentState, reason);
-                    if (!routingResult.changed()) {
-                        // no state changed
-                        return currentState;
-                    }
-                    return ClusterState.builder(currentState).routingResult(routingResult).build();
+                    return allocationService.reroute(currentState, reason);
                 }
 
                 @Override
@@ -115,7 +109,7 @@ public void onFailure(String source, Exception e) {
                     rerouting.set(false);
                     ClusterState state = clusterService.state();
                     if (logger.isTraceEnabled()) {
-                        logger.error((Supplier<?>) () -> new ParameterizedMessage("unexpected failure during [{}], current state:\n{}", source, state.prettyPrint()), e);
+                        logger.error((Supplier<?>) () -> new ParameterizedMessage("unexpected failure during [{}], current state:\n{}", source, state), e);
                     } else {
                         logger.error((Supplier<?>) () -> new ParameterizedMessage("unexpected failure during [{}], current state version [{}]", source, state.version()), e);
                     }
@@ -124,7 +118,7 @@ public void onFailure(String source, Exception e) {
         } catch (Exception e) {
             rerouting.set(false);
             ClusterState state = clusterService.state();
-            logger.warn((Supplier<?>) () -> new ParameterizedMessage("failed to reroute routing table, current state:\n{}", state.prettyPrint()), e);
+            logger.warn((Supplier<?>) () -> new ParameterizedMessage("failed to reroute routing table, current state:\n{}", state), e);
         }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java
index 2d960ce0450bb..a248d6a939a0c 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java
@@ -56,8 +56,6 @@
  */
 public class RoutingTable implements Iterable<IndexRoutingTable>, Diffable<RoutingTable> {
 
-    public static RoutingTable PROTO = builder().build();
-
     public static final RoutingTable EMPTY_ROUTING_TABLE = builder().build();
 
     private final long version;
@@ -240,7 +238,7 @@ public GroupShardsIterator allActiveShardsGrouped(String[] indices, boolean incl
         return allSatisfyingPredicateShardsGrouped(indices, includeEmpty, includeRelocationTargets, ACTIVE_PREDICATE);
     }
 
-    public GroupShardsIterator allAssignedShardsGrouped(String[] indices, boolean includeEmpty) {
+    public GroupShardsIterator<ShardIterator> allAssignedShardsGrouped(String[] indices, boolean includeEmpty) {
         return allAssignedShardsGrouped(indices, includeEmpty, false);
     }
 
@@ -251,14 +249,14 @@ public GroupShardsIterator allAssignedShardsGrouped(String[] indices, boolean in
      * @param includeRelocationTargets if true, an <b>extra</b> shard iterator will be added for relocating shards. The extra
      *                                 iterator contains a single ShardRouting pointing at the relocating target
      */
-    public GroupShardsIterator allAssignedShardsGrouped(String[] indices, boolean includeEmpty, boolean includeRelocationTargets) {
+    public GroupShardsIterator<ShardIterator> allAssignedShardsGrouped(String[] indices, boolean includeEmpty, boolean includeRelocationTargets) {
         return allSatisfyingPredicateShardsGrouped(indices, includeEmpty, includeRelocationTargets, ASSIGNED_PREDICATE);
     }
 
-    private static Predicate<ShardRouting> ACTIVE_PREDICATE = shardRouting -> shardRouting.active();
-    private static Predicate<ShardRouting> ASSIGNED_PREDICATE = shardRouting -> shardRouting.assignedToNode();
+    private static Predicate<ShardRouting> ACTIVE_PREDICATE = ShardRouting::active;
+    private static Predicate<ShardRouting> ASSIGNED_PREDICATE = ShardRouting::assignedToNode;
 
-    private GroupShardsIterator allSatisfyingPredicateShardsGrouped(String[] indices, boolean includeEmpty, boolean includeRelocationTargets, Predicate<ShardRouting> predicate) {
+    private GroupShardsIterator<ShardIterator> allSatisfyingPredicateShardsGrouped(String[] indices, boolean includeEmpty, boolean includeRelocationTargets, Predicate<ShardRouting> predicate) {
         // use list here since we need to maintain identity across shards
         ArrayList<ShardIterator> set = new ArrayList<>();
         for (String index : indices) {
@@ -280,7 +278,7 @@ private GroupShardsIterator allSatisfyingPredicateShardsGrouped(String[] indices
                 }
             }
         }
-        return new GroupShardsIterator(set);
+        return new GroupShardsIterator<>(set);
     }
 
     public ShardsIterator allShards(String[] indices) {
@@ -322,9 +320,8 @@ private ShardsIterator allShardsSatisfyingPredicate(String[] indices, Predicate<
      * @param indices The indices to return all the shards (replicas)
      * @return All the primary shards grouped into a single shard element group each
      * @throws IndexNotFoundException If an index passed does not exists
-     * @see IndexRoutingTable#groupByAllIt()
      */
-    public GroupShardsIterator activePrimaryShardsGrouped(String[] indices, boolean includeEmpty) {
+    public GroupShardsIterator<ShardIterator> activePrimaryShardsGrouped(String[] indices, boolean includeEmpty) {
         // use list here since we need to maintain identity across shards
         ArrayList<ShardIterator> set = new ArrayList<>();
         for (String index : indices) {
@@ -341,7 +338,7 @@ public GroupShardsIterator activePrimaryShardsGrouped(String[] indices, boolean
                 }
             }
         }
-        return new GroupShardsIterator(set);
+        return new GroupShardsIterator<>(set);
     }
 
     @Override
@@ -349,18 +346,16 @@ public Diff<RoutingTable> diff(RoutingTable previousState) {
         return new RoutingTableDiff(previousState, this);
     }
 
-    @Override
-    public Diff<RoutingTable> readDiffFrom(StreamInput in) throws IOException {
+    public static Diff<RoutingTable> readDiffFrom(StreamInput in) throws IOException {
         return new RoutingTableDiff(in);
     }
 
-    @Override
-    public RoutingTable readFrom(StreamInput in) throws IOException {
+    public static RoutingTable readFrom(StreamInput in) throws IOException {
         Builder builder = new Builder();
         builder.version = in.readLong();
         int size = in.readVInt();
         for (int i = 0; i < size; i++) {
-            IndexRoutingTable index = IndexRoutingTable.Builder.readFrom(in);
+            IndexRoutingTable index = IndexRoutingTable.readFrom(in);
             builder.add(index);
         }
 
@@ -382,14 +377,15 @@ private static class RoutingTableDiff implements Diff<RoutingTable> {
 
         private final Diff<ImmutableOpenMap<String, IndexRoutingTable>> indicesRouting;
 
-        public RoutingTableDiff(RoutingTable before, RoutingTable after) {
+        RoutingTableDiff(RoutingTable before, RoutingTable after) {
             version = after.version;
             indicesRouting = DiffableUtils.diff(before.indicesRouting, after.indicesRouting, DiffableUtils.getStringKeySerializer());
         }
 
-        public RoutingTableDiff(StreamInput in) throws IOException {
+        RoutingTableDiff(StreamInput in) throws IOException {
             version = in.readLong();
-            indicesRouting = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), IndexRoutingTable.PROTO);
+            indicesRouting = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), IndexRoutingTable::readFrom,
+                IndexRoutingTable::readDiffFrom);
         }
 
         @Override
@@ -607,13 +603,10 @@ public RoutingTable build() {
             indicesRouting = null;
             return table;
         }
-
-        public static RoutingTable readFrom(StreamInput in) throws IOException {
-            return PROTO.readFrom(in);
-        }
     }
 
-    public String prettyPrint() {
+    @Override
+    public String toString() {
         StringBuilder sb = new StringBuilder("routing_table (version ").append(version).append("):\n");
         for (ObjectObjectCursor<String, IndexRoutingTable> entry : indicesRouting) {
             sb.append(entry.value.prettyPrint()).append('\n');
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java b/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java
index e441fd8111335..3a60e5338d798 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java
@@ -390,7 +390,18 @@ public ShardRouting reinitializePrimaryShard() {
         assert primary : this;
         return new ShardRouting(shardId, currentNodeId, null, primary, ShardRoutingState.INITIALIZING,
             StoreRecoverySource.EXISTING_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.REINITIALIZED, null),
-            AllocationId.newInitializing(), UNAVAILABLE_EXPECTED_SHARD_SIZE);
+            allocationId, UNAVAILABLE_EXPECTED_SHARD_SIZE);
+    }
+
+    /**
+     * Reinitializes a replica shard, giving it a fresh allocation id
+     */
+    public ShardRouting reinitializeReplicaShard() {
+        assert state == ShardRoutingState.INITIALIZING : this;
+        assert primary == false : this;
+        assert isRelocationTarget() == false : this;
+        return new ShardRouting(shardId, currentNodeId, null, primary, ShardRoutingState.INITIALIZING,
+            recoverySource, unassignedInfo, AllocationId.newInitializing(), expectedShardSize);
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/ShardsIterator.java b/core/src/main/java/org/elasticsearch/cluster/routing/ShardsIterator.java
index bdfe35ad2b9fb..638875ea07138 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/ShardsIterator.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/ShardsIterator.java
@@ -18,10 +18,12 @@
  */
 package org.elasticsearch.cluster.routing;
 
+import java.util.List;
+
 /**
  * Allows to iterate over unrelated shards.
  */
-public interface ShardsIterator {
+public interface ShardsIterator extends Iterable<ShardRouting> {
 
     /**
      * Resets the iterator to its initial state.
@@ -42,13 +44,6 @@ public interface ShardsIterator {
      */
     int sizeActive();
 
-    /**
-     * Returns the number of replicas in this iterator that are not in the
-     * {@link ShardRoutingState#UNASSIGNED}. The returned double-counts replicas
-     * that are in the state {@link ShardRoutingState#RELOCATING}
-     */
-    int assignedReplicasIncludingRelocating();
-
     /**
      * Returns the next shard, or <tt>null</tt> if none available.
      */
@@ -67,6 +62,9 @@ public interface ShardsIterator {
     @Override
     boolean equals(Object other);
 
-    Iterable<ShardRouting> asUnordered();
+    /**
+     * Returns the {@link ShardRouting}s that this shards iterator holds.
+     */
+    List<ShardRouting> getShardRoutings();
 }
 
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java b/core/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java
index 4670e1e473671..3726bac781e3c 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java
@@ -185,15 +185,15 @@ public static AllocationStatus readFrom(StreamInput in) throws IOException {
             }
         }
 
-        public static AllocationStatus fromDecision(Decision decision) {
+        public static AllocationStatus fromDecision(Decision.Type decision) {
             Objects.requireNonNull(decision);
-            switch (decision.type()) {
+            switch (decision) {
                 case NO:
                     return DECIDERS_NO;
                 case THROTTLE:
                     return DECIDERS_THROTTLED;
                 default:
-                    throw new IllegalArgumentException("no allocation attempt from decision[" + decision.type() + "]");
+                    throw new IllegalArgumentException("no allocation attempt from decision[" + decision + "]");
             }
         }
 
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AbstractAllocationDecision.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AbstractAllocationDecision.java
new file mode 100644
index 0000000000000..7ee17c558f70d
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AbstractAllocationDecision.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.routing.allocation;
+
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.allocation.decider.Decision.Type;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+/**
+ * An abstract class for representing various types of allocation decisions.
+ */
+public abstract class AbstractAllocationDecision implements ToXContent, Writeable {
+
+    @Nullable
+    protected final DiscoveryNode targetNode;
+    @Nullable
+    protected final List<NodeAllocationResult> nodeDecisions;
+
+    protected AbstractAllocationDecision(@Nullable DiscoveryNode targetNode, @Nullable List<NodeAllocationResult> nodeDecisions) {
+        this.targetNode = targetNode;
+        this.nodeDecisions = nodeDecisions != null ? sortNodeDecisions(nodeDecisions) : null;
+    }
+
+    protected AbstractAllocationDecision(StreamInput in) throws IOException {
+        targetNode = in.readOptionalWriteable(DiscoveryNode::new);
+        nodeDecisions = in.readBoolean() ? Collections.unmodifiableList(in.readList(NodeAllocationResult::new)) : null;
+    }
+
+    /**
+     * Returns {@code true} if a decision was taken by the allocator, {@code false} otherwise.
+     * If no decision was taken, then the rest of the fields in this object cannot be accessed and will
+     * throw an {@code IllegalStateException}.
+     */
+    public abstract boolean isDecisionTaken();
+
+    /**
+     * Get the node that the allocator will assign the shard to, returning {@code null} if there is no node to
+     * which the shard will be assigned or moved.  If {@link #isDecisionTaken()} returns {@code false}, then
+     * invoking this method will throw an {@code IllegalStateException}.
+     */
+    @Nullable
+    public DiscoveryNode getTargetNode() {
+        checkDecisionState();
+        return targetNode;
+    }
+
+    /**
+     * Gets the sorted list of individual node-level decisions that went into making the ultimate decision whether
+     * to allocate or move the shard.  If {@link #isDecisionTaken()} returns {@code false}, then
+     * invoking this method will throw an {@code IllegalStateException}.
+     */
+    @Nullable
+    public List<NodeAllocationResult> getNodeDecisions() {
+        checkDecisionState();
+        return nodeDecisions;
+    }
+
+    /**
+     * Gets the explanation for the decision.  If {@link #isDecisionTaken()} returns {@code false}, then invoking
+     * this method will throw an {@code IllegalStateException}.
+     */
+    public abstract String getExplanation();
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeOptionalWriteable(targetNode);
+        if (nodeDecisions != null) {
+            out.writeBoolean(true);
+            out.writeList(nodeDecisions);
+        } else {
+            out.writeBoolean(false);
+        }
+    }
+
+    protected void checkDecisionState() {
+        if (isDecisionTaken() == false) {
+            throw new IllegalStateException("decision was not taken, individual object fields cannot be accessed");
+        }
+    }
+
+    /**
+     * Generates X-Content for a {@link DiscoveryNode} that leaves off some of the non-critical fields.
+     */
+    public static XContentBuilder discoveryNodeToXContent(DiscoveryNode node, boolean outerObjectWritten, XContentBuilder builder)
+        throws IOException {
+
+        builder.field(outerObjectWritten ? "id" : "node_id", node.getId());
+        builder.field(outerObjectWritten ? "name" : "node_name", node.getName());
+        builder.field("transport_address", node.getAddress().toString());
+        if (node.getAttributes().isEmpty() == false) {
+            builder.startObject(outerObjectWritten ? "attributes" : "node_attributes");
+            for (Map.Entry<String, String> entry : node.getAttributes().entrySet()) {
+                builder.field(entry.getKey(), entry.getValue());
+            }
+            builder.endObject();
+        }
+        return builder;
+    }
+
+    /**
+     * Sorts a list of node level decisions by the decision type, then by weight ranking, and finally by node id.
+     */
+    public List<NodeAllocationResult> sortNodeDecisions(List<NodeAllocationResult> nodeDecisions) {
+        return Collections.unmodifiableList(nodeDecisions.stream().sorted().collect(Collectors.toList()));
+    }
+
+    /**
+     * Generates X-Content for the node-level decisions, creating the outer "node_decisions" object
+     * in which they are serialized.
+     */
+    public XContentBuilder nodeDecisionsToXContent(List<NodeAllocationResult> nodeDecisions, XContentBuilder builder, Params params)
+        throws IOException {
+
+        if (nodeDecisions != null && nodeDecisions.isEmpty() == false) {
+            builder.startArray("node_allocation_decisions");
+            {
+                for (NodeAllocationResult explanation : nodeDecisions) {
+                    explanation.toXContent(builder, params);
+                }
+            }
+            builder.endArray();
+        }
+        return builder;
+    }
+
+    /**
+     * Returns {@code true} if there is at least one node that returned a {@link Type#YES} decision for allocating this shard.
+     */
+    protected boolean atLeastOneNodeWithYesDecision() {
+        if (nodeDecisions == null) {
+            return false;
+        }
+        for (NodeAllocationResult result : nodeDecisions) {
+            if (result.getNodeDecision() == AllocationDecision.YES) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (this == other) {
+            return true;
+        }
+        if (other == null || other instanceof AbstractAllocationDecision == false) {
+            return false;
+        }
+        @SuppressWarnings("unchecked") AbstractAllocationDecision that = (AbstractAllocationDecision) other;
+        return Objects.equals(targetNode, that.targetNode) && Objects.equals(nodeDecisions, that.nodeDecisions);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(targetNode, nodeDecisions);
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocateUnassignedDecision.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocateUnassignedDecision.java
new file mode 100644
index 0000000000000..decdafd724c7d
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocateUnassignedDecision.java
@@ -0,0 +1,331 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.routing.allocation;
+
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.UnassignedInfo.AllocationStatus;
+import org.elasticsearch.cluster.routing.allocation.decider.Decision;
+import org.elasticsearch.cluster.routing.allocation.decider.Decision.Type;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.EnumMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Represents the allocation decision by an allocator for an unassigned shard.
+ */
+public class AllocateUnassignedDecision extends AbstractAllocationDecision {
+    /** a constant representing a shard decision where no decision was taken */
+    public static final AllocateUnassignedDecision NOT_TAKEN =
+        new AllocateUnassignedDecision(AllocationStatus.NO_ATTEMPT, null, null, null, false, 0L, 0L);
+    /**
+     * a map of cached common no/throttle decisions that don't need explanations,
+     * this helps prevent unnecessary object allocations for the non-explain API case
+     */
+    private static final Map<AllocationStatus, AllocateUnassignedDecision> CACHED_DECISIONS;
+    static {
+        Map<AllocationStatus, AllocateUnassignedDecision> cachedDecisions = new EnumMap<>(AllocationStatus.class);
+        cachedDecisions.put(AllocationStatus.FETCHING_SHARD_DATA,
+            new AllocateUnassignedDecision(AllocationStatus.FETCHING_SHARD_DATA, null, null, null, false, 0L, 0L));
+        cachedDecisions.put(AllocationStatus.NO_VALID_SHARD_COPY,
+            new AllocateUnassignedDecision(AllocationStatus.NO_VALID_SHARD_COPY, null, null, null, false, 0L, 0L));
+        cachedDecisions.put(AllocationStatus.DECIDERS_NO,
+            new AllocateUnassignedDecision(AllocationStatus.DECIDERS_NO, null, null, null, false, 0L, 0L));
+        cachedDecisions.put(AllocationStatus.DECIDERS_THROTTLED,
+            new AllocateUnassignedDecision(AllocationStatus.DECIDERS_THROTTLED, null, null, null, false, 0L, 0L));
+        cachedDecisions.put(AllocationStatus.DELAYED_ALLOCATION,
+            new AllocateUnassignedDecision(AllocationStatus.DELAYED_ALLOCATION, null, null, null, false, 0L, 0L));
+        CACHED_DECISIONS = Collections.unmodifiableMap(cachedDecisions);
+    }
+
+    @Nullable
+    private final AllocationStatus allocationStatus;
+    @Nullable
+    private final String allocationId;
+    private final boolean reuseStore;
+    private final long remainingDelayInMillis;
+    private final long configuredDelayInMillis;
+
+    private AllocateUnassignedDecision(AllocationStatus allocationStatus,
+                                       DiscoveryNode assignedNode,
+                                       String allocationId,
+                                       List<NodeAllocationResult> nodeDecisions,
+                                       boolean reuseStore,
+                                       long remainingDelayInMillis,
+                                       long configuredDelayInMillis) {
+        super(assignedNode, nodeDecisions);
+        assert assignedNode != null || allocationStatus != null :
+            "a yes decision must have a node to assign the shard to";
+        assert allocationId == null || assignedNode != null :
+            "allocation id can only be null if the assigned node is null";
+        this.allocationStatus = allocationStatus;
+        this.allocationId = allocationId;
+        this.reuseStore = reuseStore;
+        this.remainingDelayInMillis = remainingDelayInMillis;
+        this.configuredDelayInMillis = configuredDelayInMillis;
+    }
+
+    public AllocateUnassignedDecision(StreamInput in) throws IOException {
+        super(in);
+        allocationStatus = in.readOptionalWriteable(AllocationStatus::readFrom);
+        allocationId = in.readOptionalString();
+        reuseStore = in.readBoolean();
+        remainingDelayInMillis = in.readVLong();
+        configuredDelayInMillis = in.readVLong();
+    }
+
+    /**
+     * Returns a NO decision with the given {@link AllocationStatus}, and the individual node-level
+     * decisions that comprised the final NO decision if in explain mode.
+     */
+    public static AllocateUnassignedDecision no(AllocationStatus allocationStatus, @Nullable List<NodeAllocationResult> decisions) {
+        return no(allocationStatus, decisions, false);
+    }
+
+    /**
+     * Returns a NO decision for a delayed shard allocation on a replica shard, with the individual node-level
+     * decisions that comprised the final NO decision, if in explain mode.  Instances created with this
+     * method will return {@link AllocationStatus#DELAYED_ALLOCATION} for {@link #getAllocationStatus()}.
+     */
+    public static AllocateUnassignedDecision delayed(long remainingDelay, long totalDelay,
+                                                     @Nullable List<NodeAllocationResult> decisions) {
+        return no(AllocationStatus.DELAYED_ALLOCATION, decisions, false, remainingDelay, totalDelay);
+    }
+
+    /**
+     * Returns a NO decision with the given {@link AllocationStatus}, and the individual node-level
+     * decisions that comprised the final NO decision if in explain mode.
+     */
+    public static AllocateUnassignedDecision no(AllocationStatus allocationStatus, @Nullable List<NodeAllocationResult> decisions,
+                                                boolean reuseStore) {
+        return no(allocationStatus, decisions, reuseStore, 0L, 0L);
+    }
+
+    private static AllocateUnassignedDecision no(AllocationStatus allocationStatus, @Nullable List<NodeAllocationResult> decisions,
+                                                 boolean reuseStore, long remainingDelay, long totalDelay) {
+        if (decisions != null) {
+            return new AllocateUnassignedDecision(allocationStatus, null, null, decisions, reuseStore, remainingDelay, totalDelay);
+        } else {
+            return getCachedDecision(allocationStatus);
+        }
+    }
+
+    /**
+     * Returns a THROTTLE decision, with the individual node-level decisions that
+     * comprised the final THROTTLE decision if in explain mode.
+     */
+    public static AllocateUnassignedDecision throttle(@Nullable List<NodeAllocationResult> decisions) {
+        if (decisions != null) {
+            return new AllocateUnassignedDecision(AllocationStatus.DECIDERS_THROTTLED, null, null, decisions, false, 0L, 0L);
+        } else {
+            return getCachedDecision(AllocationStatus.DECIDERS_THROTTLED);
+        }
+    }
+
+    /**
+     * Creates a YES decision with the given individual node-level decisions that
+     * comprised the final YES decision, along with the node id to which the shard is assigned and
+     * the allocation id for the shard, if available.
+     */
+    public static AllocateUnassignedDecision yes(DiscoveryNode assignedNode, @Nullable String allocationId,
+                                                 @Nullable List<NodeAllocationResult> decisions, boolean reuseStore) {
+        return new AllocateUnassignedDecision(null, assignedNode, allocationId, decisions, reuseStore, 0L, 0L);
+    }
+
+    /**
+     * Creates a {@link AllocateUnassignedDecision} from the given {@link Decision} and the assigned node, if any.
+     */
+    public static AllocateUnassignedDecision fromDecision(Decision decision, @Nullable DiscoveryNode assignedNode,
+                                                          @Nullable List<NodeAllocationResult> nodeDecisions) {
+        final Type decisionType = decision.type();
+        AllocationStatus allocationStatus = decisionType != Type.YES ? AllocationStatus.fromDecision(decisionType) : null;
+        return new AllocateUnassignedDecision(allocationStatus, assignedNode, null, nodeDecisions, false, 0L, 0L);
+    }
+
+    private static AllocateUnassignedDecision getCachedDecision(AllocationStatus allocationStatus) {
+        AllocateUnassignedDecision decision = CACHED_DECISIONS.get(allocationStatus);
+        return Objects.requireNonNull(decision, "precomputed decision not found for " + allocationStatus);
+    }
+
+    @Override
+    public boolean isDecisionTaken() {
+        return allocationStatus != AllocationStatus.NO_ATTEMPT;
+    }
+
+    /**
+     * Returns the {@link AllocationDecision} denoting the result of an allocation attempt.
+     * If {@link #isDecisionTaken()} returns {@code false}, then invoking this method will
+     * throw an {@code IllegalStateException}.
+     */
+    public AllocationDecision getAllocationDecision() {
+        checkDecisionState();
+        return AllocationDecision.fromAllocationStatus(allocationStatus);
+    }
+
+    /**
+     * Returns the status of an unsuccessful allocation attempt.  This value will be {@code null} if
+     * no decision was taken or if the decision was {@link Decision.Type#YES}.  If {@link #isDecisionTaken()}
+     * returns {@code false}, then invoking this method will throw an {@code IllegalStateException}.
+     */
+    @Nullable
+    public AllocationStatus getAllocationStatus() {
+        checkDecisionState();
+        return allocationStatus;
+    }
+
+    /**
+     * Gets the allocation id for the existing shard copy that the allocator is assigning the shard to.
+     * This method returns a non-null value iff {@link #getTargetNode()} returns a non-null value
+     * and the node on which the shard is assigned already has a shard copy with an in-sync allocation id
+     * that we can re-use.  If {@link #isDecisionTaken()} returns {@code false}, then invoking this method
+     * will throw an {@code IllegalStateException}.
+     */
+    @Nullable
+    public String getAllocationId() {
+        checkDecisionState();
+        return allocationId;
+    }
+
+    /**
+     * Gets the remaining delay for allocating the replica shard when a node holding the replica left
+     * the cluster and the deciders are waiting to see if the node returns before allocating the replica
+     * elsewhere.  Only returns a meaningful positive value if {@link #getAllocationStatus()} returns
+     * {@link AllocationStatus#DELAYED_ALLOCATION}.  If {@link #isDecisionTaken()} returns {@code false},
+     * then invoking this method will throw an {@code IllegalStateException}.
+     */
+    public long getRemainingDelayInMillis() {
+        checkDecisionState();
+        return remainingDelayInMillis;
+    }
+
+    /**
+     * Gets the total configured delay for allocating the replica shard when a node holding the replica left
+     * the cluster and the deciders are waiting to see if the node returns before allocating the replica
+     * elsewhere.  Only returns a meaningful positive value if {@link #getAllocationStatus()} returns
+     * {@link AllocationStatus#DELAYED_ALLOCATION}.  If {@link #isDecisionTaken()} returns {@code false},
+     * then invoking this method will throw an {@code IllegalStateException}.
+     */
+    public long getConfiguredDelayInMillis() {
+        checkDecisionState();
+        return configuredDelayInMillis;
+    }
+
+    @Override
+    public String getExplanation() {
+        checkDecisionState();
+        AllocationDecision allocationDecision = getAllocationDecision();
+        if (allocationDecision == AllocationDecision.YES) {
+            return "can allocate the shard";
+        } else if (allocationDecision == AllocationDecision.THROTTLED) {
+            return "allocation temporarily throttled";
+        } else if (allocationDecision == AllocationDecision.AWAITING_INFO) {
+            return "cannot allocate because information about existing shard data is still being retrieved from some of the nodes";
+        } else if (allocationDecision == AllocationDecision.NO_VALID_SHARD_COPY) {
+            if (hasNodeWithStaleOrCorruptShard()) {
+                return "cannot allocate because all found copies of the shard are either stale or corrupt";
+            } else {
+                return "cannot allocate because a previous copy of the primary shard existed but can no longer be found on " +
+                       "the nodes in the cluster";
+            }
+        } else if (allocationDecision == AllocationDecision.ALLOCATION_DELAYED) {
+            return "cannot allocate because the cluster is still waiting " +
+                              TimeValue.timeValueMillis(remainingDelayInMillis) +
+                              " for the departed node holding a replica to rejoin" +
+                              (atLeastOneNodeWithYesDecision() ?
+                                   ", despite being allowed to allocate the shard to at least one other node" : "");
+        } else {
+            assert allocationDecision == AllocationDecision.NO;
+            if (reuseStore) {
+                return "cannot allocate because allocation is not permitted to any of the nodes that hold an in-sync shard copy";
+            } else {
+                return "cannot allocate because allocation is not permitted to any of the nodes";
+            }
+        }
+    }
+
+    private boolean hasNodeWithStaleOrCorruptShard() {
+        return getNodeDecisions() != null && getNodeDecisions().stream().anyMatch(result ->
+                result.getShardStoreInfo() != null
+                    && (result.getShardStoreInfo().getAllocationId() != null
+                            || result.getShardStoreInfo().getStoreException() != null));
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        checkDecisionState();
+        builder.field("can_allocate", getAllocationDecision());
+        builder.field("allocate_explanation", getExplanation());
+        if (targetNode != null) {
+            builder.startObject("target_node");
+            discoveryNodeToXContent(targetNode, true, builder);
+            builder.endObject();
+        }
+        if (allocationId != null) {
+            builder.field("allocation_id", allocationId);
+        }
+        if (allocationStatus == AllocationStatus.DELAYED_ALLOCATION) {
+            builder.timeValueField("configured_delay_in_millis", "configured_delay", TimeValue.timeValueMillis(configuredDelayInMillis));
+            builder.timeValueField("remaining_delay_in_millis", "remaining_delay", TimeValue.timeValueMillis(remainingDelayInMillis));
+        }
+        nodeDecisionsToXContent(nodeDecisions, builder, params);
+        return builder;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeOptionalWriteable(allocationStatus);
+        out.writeOptionalString(allocationId);
+        out.writeBoolean(reuseStore);
+        out.writeVLong(remainingDelayInMillis);
+        out.writeVLong(configuredDelayInMillis);
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (super.equals(other) == false) {
+            return false;
+        }
+        if (other instanceof AllocateUnassignedDecision == false) {
+            return false;
+        }
+        @SuppressWarnings("unchecked") AllocateUnassignedDecision that = (AllocateUnassignedDecision) other;
+        return Objects.equals(allocationStatus, that.allocationStatus)
+                   && Objects.equals(allocationId, that.allocationId)
+                   && reuseStore == that.reuseStore
+                   && configuredDelayInMillis == that.configuredDelayInMillis
+                   && remainingDelayInMillis == that.remainingDelayInMillis;
+    }
+
+    @Override
+    public int hashCode() {
+        return 31 * super.hashCode() + Objects.hash(allocationStatus, allocationId, reuseStore,
+            configuredDelayInMillis, remainingDelayInMillis);
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationDecision.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationDecision.java
new file mode 100644
index 0000000000000..0fe9549635ecd
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationDecision.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.routing.allocation;
+
+import org.elasticsearch.cluster.routing.UnassignedInfo.AllocationStatus;
+import org.elasticsearch.cluster.routing.allocation.decider.Decision;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+
+import java.io.IOException;
+import java.util.Locale;
+
+/**
+ * An enum which represents the various decision types that can be taken by the
+ * allocators and deciders for allocating a shard to a node.
+ */
+public enum AllocationDecision implements Writeable {
+    /**
+     * The shard can be allocated to a node.
+     */
+    YES((byte) 0),
+    /**
+     * The allocation attempt was throttled for the shard.
+     */
+    THROTTLED((byte) 1),
+    /**
+     * The shard cannot be allocated, which can happen for any number of reasons,
+     * including the allocation deciders gave a NO decision for allocating.
+     */
+    NO((byte) 2),
+    /**
+     * The shard could not be rebalanced to another node despite rebalancing
+     * being allowed, because moving the shard to the other node would not form
+     * a better cluster balance.
+     */
+    WORSE_BALANCE((byte) 3),
+    /**
+     * Waiting on getting shard data from all nodes before making a decision
+     * about where to allocate the shard.
+     */
+    AWAITING_INFO((byte) 4),
+    /**
+     * The allocation decision has been delayed waiting for a replica with a shard copy
+     * that left the cluster to rejoin.
+     */
+    ALLOCATION_DELAYED((byte) 5),
+    /**
+     * The shard was denied allocation because there were no valid shard copies
+     * found for it amongst the nodes in the cluster.
+     */
+    NO_VALID_SHARD_COPY((byte) 6),
+    /**
+     * No attempt was made to allocate the shard
+     */
+    NO_ATTEMPT((byte) 7);
+
+
+    private final byte id;
+
+    AllocationDecision(byte id) {
+        this.id = id;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeByte(id);
+    }
+
+    public static AllocationDecision readFrom(StreamInput in) throws IOException {
+        byte id = in.readByte();
+        switch (id) {
+            case 0:
+                return YES;
+            case 1:
+                return THROTTLED;
+            case 2:
+                return NO;
+            case 3:
+                return WORSE_BALANCE;
+            case 4:
+                return AWAITING_INFO;
+            case 5:
+                return ALLOCATION_DELAYED;
+            case 6:
+                return NO_VALID_SHARD_COPY;
+            case 7:
+                return NO_ATTEMPT;
+            default:
+                throw new IllegalArgumentException("Unknown value [" + id + "]");
+        }
+    }
+
+    /**
+     * Gets an {@link AllocationDecision} from a {@link AllocationStatus}.
+     */
+    public static AllocationDecision fromAllocationStatus(AllocationStatus allocationStatus) {
+        if (allocationStatus == null) {
+            return YES;
+        } else {
+            switch (allocationStatus) {
+                case DECIDERS_THROTTLED:
+                    return THROTTLED;
+                case FETCHING_SHARD_DATA:
+                    return AWAITING_INFO;
+                case DELAYED_ALLOCATION:
+                    return ALLOCATION_DELAYED;
+                case NO_VALID_SHARD_COPY:
+                    return NO_VALID_SHARD_COPY;
+                case NO_ATTEMPT:
+                    return NO_ATTEMPT;
+                default:
+                    assert allocationStatus == AllocationStatus.DECIDERS_NO : "unhandled AllocationStatus type [" + allocationStatus + "]";
+                    return NO;
+            }
+        }
+    }
+
+    /**
+     * Gets an {@link AllocationDecision} from a {@link Decision.Type}
+     */
+    public static AllocationDecision fromDecisionType(Decision.Type type) {
+        switch (type) {
+            case YES:
+                return YES;
+            case THROTTLE:
+                return THROTTLED;
+            default:
+                assert type == Decision.Type.NO;
+                return NO;
+        }
+    }
+
+    @Override
+    public String toString() {
+        return super.toString().toLowerCase(Locale.ROOT);
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationExplanation.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationExplanation.java
deleted file mode 100644
index e2b5de9b52dae..0000000000000
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationExplanation.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.cluster.routing.allocation;
-
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Streamable;
-import org.elasticsearch.index.shard.ShardId;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Instances of this class keeps explanations of decisions that have been made by allocation.
- * An {@link AllocationExplanation} consists of a set of per node explanations.
- * Since {@link NodeExplanation}s are related to shards an {@link AllocationExplanation} maps
- * a shards id to a set of {@link NodeExplanation}s.  
- */
-public class AllocationExplanation implements Streamable {
-
-    public static final AllocationExplanation EMPTY = new AllocationExplanation();
-
-    /**
-     * Instances of this class keep messages and informations about nodes of an allocation
-     */
-    public static class NodeExplanation {
-        private final DiscoveryNode node;
-
-        private final String description;
-
-        /**
-         * Creates a new {@link NodeExplanation}
-         *  
-         * @param node node referenced by this {@link NodeExplanation}
-         * @param description a message associated with the given node 
-         */
-        public NodeExplanation(DiscoveryNode node, String description) {
-            this.node = node;
-            this.description = description;
-        }
-
-        /**
-         * The node referenced by the explanation
-         * @return referenced node
-         */
-        public DiscoveryNode node() {
-            return node;
-        }
-
-        /**
-         * Get the explanation for the node
-         * @return explanation for the node
-         */
-        public String description() {
-            return description;
-        }
-    }
-
-    private final Map<ShardId, List<NodeExplanation>> explanations = new HashMap<>();
-
-    /**
-     * Create and add a node explanation to this explanation referencing a shard  
-     * @param shardId id the of the referenced shard
-     * @param nodeExplanation Explanation itself
-     * @return AllocationExplanation involving the explanation 
-     */
-    public AllocationExplanation add(ShardId shardId, NodeExplanation nodeExplanation) {
-        List<NodeExplanation> list = explanations.get(shardId);
-        if (list == null) {
-            list = new ArrayList<>();
-            explanations.put(shardId, list);
-        }
-        list.add(nodeExplanation);
-        return this;
-    }
-
-    /**
-     * List of explanations involved by this AllocationExplanation
-     * @return Map of shard ids and corresponding explanations  
-     */
-    public Map<ShardId, List<NodeExplanation>> explanations() {
-        return this.explanations;
-    }
-
-    /**
-     * Read an {@link AllocationExplanation} from an {@link StreamInput}
-     * @param in {@link StreamInput} to read from
-     * @return a new {@link AllocationExplanation} read from the stream 
-     * @throws IOException if something bad happened while reading
-     */
-    public static AllocationExplanation readAllocationExplanation(StreamInput in) throws IOException {
-        AllocationExplanation e = new AllocationExplanation();
-        e.readFrom(in);
-        return e;
-    }
-
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        int size = in.readVInt();
-        for (int i = 0; i < size; i++) {
-            ShardId shardId = ShardId.readShardId(in);
-            int size2 = in.readVInt();
-            List<NodeExplanation> ne = new ArrayList<>(size2);
-            for (int j = 0; j < size2; j++) {
-                DiscoveryNode node = null;
-                if (in.readBoolean()) {
-                    node = new DiscoveryNode(in);
-                }
-                ne.add(new NodeExplanation(node, in.readString()));
-            }
-            explanations.put(shardId, ne);
-        }
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeVInt(explanations.size());
-        for (Map.Entry<ShardId, List<NodeExplanation>> entry : explanations.entrySet()) {
-            entry.getKey().writeTo(out);
-            out.writeVInt(entry.getValue().size());
-            for (NodeExplanation nodeExplanation : entry.getValue()) {
-                if (nodeExplanation.node() == null) {
-                    out.writeBoolean(false);
-                } else {
-                    out.writeBoolean(true);
-                    nodeExplanation.node().writeTo(out);
-                }
-                out.writeString(nodeExplanation.description());
-            }
-        }
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
index 214bedc324c66..3eaac2a2057d8 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
@@ -20,8 +20,8 @@
 package org.elasticsearch.cluster.routing.allocation;
 
 import org.elasticsearch.cluster.ClusterInfoService;
-import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.RestoreInProgress;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.health.ClusterStateHealth;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
@@ -32,16 +32,18 @@
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.routing.UnassignedInfo.AllocationStatus;
-import org.elasticsearch.cluster.routing.allocation.RoutingAllocation.Result;
 import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocator;
 import org.elasticsearch.cluster.routing.allocation.command.AllocationCommands;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.gateway.GatewayAllocator;
 
+import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.Iterator;
 import java.util.List;
 import java.util.function.Function;
@@ -62,7 +64,6 @@ public class AllocationService extends AbstractComponent {
     private final GatewayAllocator gatewayAllocator;
     private final ShardsAllocator shardsAllocator;
     private final ClusterInfoService clusterInfoService;
-    private final ClusterName clusterName;
 
     @Inject
     public AllocationService(Settings settings, AllocationDeciders allocationDeciders, GatewayAllocator gatewayAllocator,
@@ -72,63 +73,66 @@ public AllocationService(Settings settings, AllocationDeciders allocationDecider
         this.gatewayAllocator = gatewayAllocator;
         this.shardsAllocator = shardsAllocator;
         this.clusterInfoService = clusterInfoService;
-        clusterName = ClusterName.CLUSTER_NAME_SETTING.get(settings);
     }
 
     /**
      * Applies the started shards. Note, only initializing ShardRouting instances that exist in the routing table should be
      * provided as parameter and no duplicates should be contained.
      * <p>
-     * If the same instance of the routing table is returned, then no change has been made.</p>
+     * If the same instance of the {@link ClusterState} is returned, then no change has been made.</p>
      */
-    public Result applyStartedShards(ClusterState clusterState, List<ShardRouting> startedShards) {
-        return applyStartedShards(clusterState, startedShards, true);
-    }
-
-    public Result applyStartedShards(ClusterState clusterState, List<ShardRouting> startedShards, boolean withReroute) {
+    public ClusterState applyStartedShards(ClusterState clusterState, List<ShardRouting> startedShards) {
         if (startedShards.isEmpty()) {
-            return Result.unchanged(clusterState);
+            return clusterState;
         }
         RoutingNodes routingNodes = getMutableRoutingNodes(clusterState);
         // shuffle the unassigned nodes, just so we won't have things like poison failed shards
         routingNodes.unassigned().shuffle();
-        StartedRerouteAllocation allocation = new StartedRerouteAllocation(allocationDeciders, routingNodes, clusterState, startedShards,
-            clusterInfoService.getClusterInfo(), currentNanoTime());
+        RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, routingNodes, clusterState,
+            clusterInfoService.getClusterInfo(), currentNanoTime(), false);
+        // as starting a primary relocation target can reinitialize replica shards, start replicas first
+        startedShards = new ArrayList<>(startedShards);
+        Collections.sort(startedShards, Comparator.comparing(ShardRouting::primary));
         applyStartedShards(allocation, startedShards);
-        gatewayAllocator.applyStartedShards(allocation);
-        if (withReroute) {
-            reroute(allocation);
-        }
+        gatewayAllocator.applyStartedShards(allocation, startedShards);
+        reroute(allocation);
         String startedShardsAsString = firstListElementsToCommaDelimitedString(startedShards, s -> s.shardId().toString());
-        return buildResultAndLogHealthChange(allocation, "shards started [" + startedShardsAsString + "] ...");
-    }
-
-    protected Result buildResultAndLogHealthChange(RoutingAllocation allocation, String reason) {
-        return buildResultAndLogHealthChange(allocation, reason, new RoutingExplanations());
+        return buildResultAndLogHealthChange(clusterState, allocation, "shards started [" + startedShardsAsString + "] ...");
     }
 
-    protected Result buildResultAndLogHealthChange(RoutingAllocation allocation, String reason, RoutingExplanations explanations) {
-        RoutingTable oldRoutingTable = allocation.routingTable();
+    protected ClusterState buildResultAndLogHealthChange(ClusterState oldState, RoutingAllocation allocation, String reason) {
+        RoutingTable oldRoutingTable = oldState.routingTable();
         RoutingNodes newRoutingNodes = allocation.routingNodes();
         final RoutingTable newRoutingTable = new RoutingTable.Builder().updateNodes(oldRoutingTable.version(), newRoutingNodes).build();
         MetaData newMetaData = allocation.updateMetaDataWithRoutingChanges(newRoutingTable);
         assert newRoutingTable.validate(newMetaData); // validates the routing table is coherent with the cluster state metadata
+        final ClusterState.Builder newStateBuilder = ClusterState.builder(oldState)
+            .routingTable(newRoutingTable)
+            .metaData(newMetaData);
+        final RestoreInProgress restoreInProgress = allocation.custom(RestoreInProgress.TYPE);
+        if (restoreInProgress != null) {
+            RestoreInProgress updatedRestoreInProgress = allocation.updateRestoreInfoWithRoutingChanges(restoreInProgress);
+            if (updatedRestoreInProgress != restoreInProgress) {
+                ImmutableOpenMap.Builder<String, ClusterState.Custom> customsBuilder = ImmutableOpenMap.builder(allocation.getCustoms());
+                customsBuilder.put(RestoreInProgress.TYPE, updatedRestoreInProgress);
+                newStateBuilder.customs(customsBuilder.build());
+            }
+        }
+        final ClusterState newState = newStateBuilder.build();
         logClusterHealthStateChange(
-            new ClusterStateHealth(ClusterState.builder(clusterName).
-                metaData(allocation.metaData()).routingTable(oldRoutingTable).build()),
-            new ClusterStateHealth(ClusterState.builder(clusterName).
-                metaData(newMetaData).routingTable(newRoutingTable).build()),
+            new ClusterStateHealth(oldState),
+            new ClusterStateHealth(newState),
             reason
         );
-        return Result.changed(newRoutingTable, newMetaData, explanations);
+        return newState;
     }
 
-    public Result applyFailedShard(ClusterState clusterState, ShardRouting failedShard) {
-        return applyFailedShards(clusterState, Collections.singletonList(new FailedRerouteAllocation.FailedShard(failedShard, null, null)),
+    public ClusterState applyFailedShard(ClusterState clusterState, ShardRouting failedShard) {
+        return applyFailedShards(clusterState, Collections.singletonList(new FailedShard(failedShard, null, null)),
             Collections.emptyList());
     }
 
-    public Result applyFailedShards(ClusterState clusterState, List<FailedRerouteAllocation.FailedShard> failedShards) {
+    public ClusterState applyFailedShards(ClusterState clusterState, List<FailedShard> failedShards) {
         return applyFailedShards(clusterState, failedShards, Collections.emptyList());
     }
 
@@ -138,24 +142,24 @@ public Result applyFailedShards(ClusterState clusterState, List<FailedRerouteAll
      * are no routing entries in the routing table.
      *
      * <p>
-     * If the same instance of the routing table is returned, then no change has been made.</p>
+     * If the same instance of ClusterState is returned, then no change has been made.</p>
      */
-    public Result applyFailedShards(ClusterState clusterState, List<FailedRerouteAllocation.FailedShard> failedShards,
-                                    List<FailedRerouteAllocation.StaleShard> staleShards) {
+    public ClusterState applyFailedShards(final ClusterState clusterState, final List<FailedShard> failedShards,
+                                          final List<StaleShard> staleShards) {
         if (staleShards.isEmpty() && failedShards.isEmpty()) {
-            return Result.unchanged(clusterState);
+            return clusterState;
         }
-        clusterState = IndexMetaDataUpdater.removeStaleIdsWithoutRoutings(clusterState, staleShards);
+        ClusterState tmpState = IndexMetaDataUpdater.removeStaleIdsWithoutRoutings(clusterState, staleShards);
 
-        RoutingNodes routingNodes = getMutableRoutingNodes(clusterState);
+        RoutingNodes routingNodes = getMutableRoutingNodes(tmpState);
         // shuffle the unassigned nodes, just so we won't have things like poison failed shards
         routingNodes.unassigned().shuffle();
         long currentNanoTime = currentNanoTime();
-        FailedRerouteAllocation allocation = new FailedRerouteAllocation(allocationDeciders, routingNodes, clusterState, failedShards,
-            clusterInfoService.getClusterInfo(), currentNanoTime);
+        RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, routingNodes, tmpState,
+            clusterInfoService.getClusterInfo(), currentNanoTime, false);
 
-        for (FailedRerouteAllocation.FailedShard failedShardEntry : failedShards) {
-            ShardRouting shardToFail = failedShardEntry.routingEntry;
+        for (FailedShard failedShardEntry : failedShards) {
+            ShardRouting shardToFail = failedShardEntry.getRoutingEntry();
             IndexMetaData indexMetaData = allocation.metaData().getIndexSafe(shardToFail.shardId().getIndex());
             allocation.addIgnoreShardForNode(shardToFail.shardId(), shardToFail.currentNodeId());
             // failing a primary also fails initializing replica shards, re-resolve ShardRouting
@@ -166,26 +170,26 @@ public Result applyFailedShards(ClusterState clusterState, List<FailedRerouteAll
                         shardToFail.shardId(), shardToFail, failedShard);
                 }
                 int failedAllocations = failedShard.unassignedInfo() != null ? failedShard.unassignedInfo().getNumFailedAllocations() : 0;
-                UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.ALLOCATION_FAILED, failedShardEntry.message,
-                    failedShardEntry.failure, failedAllocations + 1, currentNanoTime, System.currentTimeMillis(), false,
+                UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.ALLOCATION_FAILED, failedShardEntry.getMessage(),
+                    failedShardEntry.getFailure(), failedAllocations + 1, currentNanoTime, System.currentTimeMillis(), false,
                     AllocationStatus.NO_ATTEMPT);
                 routingNodes.failShard(logger, failedShard, unassignedInfo, indexMetaData, allocation.changes());
             } else {
                 logger.trace("{} shard routing failed in an earlier iteration (routing: {})", shardToFail.shardId(), shardToFail);
             }
         }
-        gatewayAllocator.applyFailedShards(allocation);
+        gatewayAllocator.applyFailedShards(allocation, failedShards);
 
         reroute(allocation);
-        String failedShardsAsString = firstListElementsToCommaDelimitedString(failedShards, s -> s.routingEntry.shardId().toString());
-        return buildResultAndLogHealthChange(allocation, "shards failed [" + failedShardsAsString + "] ...");
+        String failedShardsAsString = firstListElementsToCommaDelimitedString(failedShards, s -> s.getRoutingEntry().shardId().toString());
+        return buildResultAndLogHealthChange(clusterState, allocation, "shards failed [" + failedShardsAsString + "] ...");
     }
 
     /**
      * unassigned an shards that are associated with nodes that are no longer part of the cluster, potentially promoting replicas
      * if needed.
      */
-    public RoutingAllocation.Result deassociateDeadNodes(ClusterState clusterState, boolean reroute, String reason) {
+    public ClusterState deassociateDeadNodes(final ClusterState clusterState, boolean reroute, String reason) {
         RoutingNodes routingNodes = getMutableRoutingNodes(clusterState);
         // shuffle the unassigned nodes, just so we won't have things like poison failed shards
         routingNodes.unassigned().shuffle();
@@ -200,9 +204,9 @@ public RoutingAllocation.Result deassociateDeadNodes(ClusterState clusterState,
         }
 
         if (allocation.routingNodesChanged() == false) {
-            return Result.unchanged(clusterState);
+            return clusterState;
         }
-        return buildResultAndLogHealthChange(allocation, reason);
+        return buildResultAndLogHealthChange(clusterState, allocation, reason);
     }
 
     /**
@@ -244,7 +248,7 @@ private <T> String firstListElementsToCommaDelimitedString(List<T> elements, Fun
                 .collect(Collectors.joining(", "));
     }
 
-    public Result reroute(ClusterState clusterState, AllocationCommands commands, boolean explain, boolean retryFailed) {
+    public CommandsResult reroute(final ClusterState clusterState, AllocationCommands commands, boolean explain, boolean retryFailed) {
         RoutingNodes routingNodes = getMutableRoutingNodes(clusterState);
         // we don't shuffle the unassigned shards here, to try and get as close as possible to
         // a consistent result of the effect the commands have on the routing
@@ -261,25 +265,25 @@ public Result reroute(ClusterState clusterState, AllocationCommands commands, bo
         // the assumption is that commands will move / act on shards (or fail through exceptions)
         // so, there will always be shard "movements", so no need to check on reroute
         reroute(allocation);
-        return buildResultAndLogHealthChange(allocation, "reroute commands", explanations);
+        return new CommandsResult(explanations, buildResultAndLogHealthChange(clusterState, allocation, "reroute commands"));
     }
 
 
     /**
      * Reroutes the routing table based on the live nodes.
      * <p>
-     * If the same instance of the routing table is returned, then no change has been made.
+     * If the same instance of ClusterState is returned, then no change has been made.
      */
-    public Result reroute(ClusterState clusterState, String reason) {
+    public ClusterState reroute(ClusterState clusterState, String reason) {
         return reroute(clusterState, reason, false);
     }
 
     /**
      * Reroutes the routing table based on the live nodes.
      * <p>
-     * If the same instance of the routing table is returned, then no change has been made.
+     * If the same instance of ClusterState is returned, then no change has been made.
      */
-    protected Result reroute(ClusterState clusterState, String reason, boolean debug) {
+    protected ClusterState reroute(final ClusterState clusterState, String reason, boolean debug) {
         RoutingNodes routingNodes = getMutableRoutingNodes(clusterState);
         // shuffle the unassigned nodes, just so we won't have things like poison failed shards
         routingNodes.unassigned().shuffle();
@@ -288,9 +292,9 @@ protected Result reroute(ClusterState clusterState, String reason, boolean debug
         allocation.debugDecision(debug);
         reroute(allocation);
         if (allocation.routingNodesChanged() == false) {
-            return Result.unchanged(clusterState);
+            return clusterState;
         }
-        return buildResultAndLogHealthChange(allocation, reason);
+        return buildResultAndLogHealthChange(clusterState, allocation, reason);
     }
 
     private void logClusterHealthStateChange(ClusterStateHealth previousStateHealth, ClusterStateHealth newStateHealth, String reason) {
@@ -368,4 +372,39 @@ private RoutingNodes getMutableRoutingNodes(ClusterState clusterState) {
     protected long currentNanoTime() {
         return System.nanoTime();
     }
+
+    /**
+     * this class is used to describe results of applying a set of
+     * {@link org.elasticsearch.cluster.routing.allocation.command.AllocationCommand}
+     */
+    public static class CommandsResult {
+
+        private final RoutingExplanations explanations;
+
+        private final ClusterState clusterState;
+
+        /**
+         * Creates a new {@link CommandsResult}
+         * @param explanations Explanation for the reroute actions
+         * @param clusterState Resulting cluster state
+         */
+        private CommandsResult(RoutingExplanations explanations, ClusterState clusterState) {
+            this.clusterState = clusterState;
+            this.explanations = explanations;
+        }
+
+        /**
+         * Get the explanation of this result
+         */
+        public RoutingExplanations explanations() {
+            return explanations;
+        }
+
+        /**
+         * thre resulting cluster state, after the commands were applied
+         */
+        public ClusterState getClusterState() {
+            return clusterState;
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java
index 103aa87dcd38f..390acda0fa519 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java
@@ -62,10 +62,10 @@ public DiskThresholdMonitor(Settings settings, ClusterSettings clusterSettings,
      */
     private void warnAboutDiskIfNeeded(DiskUsage usage) {
         // Check absolute disk values
-        if (usage.getFreeBytes() < diskThresholdSettings.getFreeBytesThresholdHigh().bytes()) {
+        if (usage.getFreeBytes() < diskThresholdSettings.getFreeBytesThresholdHigh().getBytes()) {
             logger.warn("high disk watermark [{}] exceeded on {}, shards will be relocated away from this node",
                 diskThresholdSettings.getFreeBytesThresholdHigh(), usage);
-        } else if (usage.getFreeBytes() < diskThresholdSettings.getFreeBytesThresholdLow().bytes()) {
+        } else if (usage.getFreeBytes() < diskThresholdSettings.getFreeBytesThresholdLow().getBytes()) {
             logger.info("low disk watermark [{}] exceeded on {}, replicas will not be assigned to this node",
                 diskThresholdSettings.getFreeBytesThresholdLow(), usage);
         }
@@ -100,7 +100,7 @@ public void onNewInfo(ClusterInfo info) {
                 String node = entry.key;
                 DiskUsage usage = entry.value;
                 warnAboutDiskIfNeeded(usage);
-                if (usage.getFreeBytes() < diskThresholdSettings.getFreeBytesThresholdHigh().bytes() ||
+                if (usage.getFreeBytes() < diskThresholdSettings.getFreeBytesThresholdHigh().getBytes() ||
                     usage.getFreeDiskAsPercentage() < diskThresholdSettings.getFreeDiskThresholdHigh()) {
                     if ((System.nanoTime() - lastRunNS) > diskThresholdSettings.getRerouteInterval().nanos()) {
                         lastRunNS = System.nanoTime();
@@ -112,7 +112,7 @@ public void onNewInfo(ClusterInfo info) {
                             node, diskThresholdSettings.getRerouteInterval());
                     }
                     nodeHasPassedWatermark.add(node);
-                } else if (usage.getFreeBytes() < diskThresholdSettings.getFreeBytesThresholdLow().bytes() ||
+                } else if (usage.getFreeBytes() < diskThresholdSettings.getFreeBytesThresholdLow().getBytes() ||
                     usage.getFreeDiskAsPercentage() < diskThresholdSettings.getFreeDiskThresholdLow()) {
                     nodeHasPassedWatermark.add(node);
                 } else {
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdSettings.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdSettings.java
index 81b9042fb3363..b87add57ce75a 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdSettings.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdSettings.java
@@ -49,6 +49,8 @@ public class DiskThresholdSettings {
         Setting.positiveTimeSetting("cluster.routing.allocation.disk.reroute_interval", TimeValue.timeValueSeconds(60),
             Setting.Property.Dynamic, Setting.Property.NodeScope);
 
+    private volatile String lowWatermarkRaw;
+    private volatile String highWatermarkRaw;
     private volatile Double freeDiskThresholdLow;
     private volatile Double freeDiskThresholdHigh;
     private volatile ByteSizeValue freeBytesThresholdLow;
@@ -86,6 +88,7 @@ private void setEnabled(boolean enabled) {
 
     private void setLowWatermark(String lowWatermark) {
         // Watermark is expressed in terms of used data, but we need "free" data watermark
+        this.lowWatermarkRaw = lowWatermark;
         this.freeDiskThresholdLow = 100.0 - thresholdPercentageFromWatermark(lowWatermark);
         this.freeBytesThresholdLow = thresholdBytesFromWatermark(lowWatermark,
             CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey());
@@ -93,11 +96,26 @@ private void setLowWatermark(String lowWatermark) {
 
     private void setHighWatermark(String highWatermark) {
         // Watermark is expressed in terms of used data, but we need "free" data watermark
+        this.highWatermarkRaw = highWatermark;
         this.freeDiskThresholdHigh = 100.0 - thresholdPercentageFromWatermark(highWatermark);
         this.freeBytesThresholdHigh = thresholdBytesFromWatermark(highWatermark,
             CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey());
     }
 
+    /**
+     * Gets the raw (uninterpreted) low watermark value as found in the settings.
+     */
+    public String getLowWatermarkRaw() {
+        return lowWatermarkRaw;
+    }
+
+    /**
+     * Gets the raw (uninterpreted) high watermark value as found in the settings.
+     */
+    public String getHighWatermarkRaw() {
+        return highWatermarkRaw;
+    }
+
     public Double getFreeDiskThresholdLow() {
         return freeDiskThresholdLow;
     }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/FailedRerouteAllocation.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/FailedRerouteAllocation.java
deleted file mode 100644
index 8a31998a7dd99..0000000000000
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/FailedRerouteAllocation.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.cluster.routing.allocation;
-
-import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.cluster.ClusterInfo;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.routing.RoutingNodes;
-import org.elasticsearch.cluster.routing.ShardRouting;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
-import org.elasticsearch.index.shard.ShardId;
-
-import java.util.List;
-
-/**
- * This {@link RoutingAllocation} keeps a shard which routing
- * allocation has failed.
- */
-public class FailedRerouteAllocation extends RoutingAllocation {
-
-    /**
-     * A failed shard with the shard routing itself and an optional
-     * details on why it failed.
-     */
-    public static class FailedShard {
-        public final ShardRouting routingEntry;
-        public final String message;
-        public final Exception failure;
-
-        public FailedShard(ShardRouting routingEntry, String message, Exception failure) {
-            assert routingEntry.assignedToNode() : "only assigned shards can be failed " + routingEntry;
-            this.routingEntry = routingEntry;
-            this.message = message;
-            this.failure = failure;
-        }
-
-        @Override
-        public String toString() {
-            return "failed shard, shard " + routingEntry + ", message [" + message + "], failure [" +
-                ExceptionsHelper.detailedMessage(failure) + "]";
-        }
-    }
-
-    public static class StaleShard {
-        public final ShardId shardId;
-        public final String allocationId;
-
-        public StaleShard(ShardId shardId, String allocationId) {
-            this.shardId = shardId;
-            this.allocationId = allocationId;
-        }
-
-        @Override
-        public String toString() {
-            return "stale shard, shard " + shardId + ", alloc. id [" + allocationId + "]";
-        }
-    }
-
-    private final List<FailedShard> failedShards;
-
-    public FailedRerouteAllocation(AllocationDeciders deciders, RoutingNodes routingNodes, ClusterState clusterState,
-                                   List<FailedShard> failedShards, ClusterInfo clusterInfo, long currentNanoTime) {
-        super(deciders, routingNodes, clusterState, clusterInfo, currentNanoTime, false);
-        this.failedShards = failedShards;
-    }
-
-    public List<FailedShard> failedShards() {
-        return failedShards;
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/FailedShard.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/FailedShard.java
new file mode 100644
index 0000000000000..9bf9fa86d1814
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/FailedShard.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.routing.allocation;
+
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.common.Nullable;
+
+/**
+ * A class representing a failed shard.
+ */
+public class FailedShard {
+    private final ShardRouting routingEntry;
+    private final String message;
+    private final Exception failure;
+
+    public FailedShard(ShardRouting routingEntry, String message, Exception failure) {
+        assert routingEntry.assignedToNode() : "only assigned shards can be failed " + routingEntry;
+        this.routingEntry = routingEntry;
+        this.message = message;
+        this.failure = failure;
+    }
+
+    @Override
+    public String toString() {
+        return "failed shard, shard " + routingEntry + ", message [" + message + "], failure [" +
+                   ExceptionsHelper.detailedMessage(failure) + "]";
+    }
+
+    /**
+     * The shard routing entry for the failed shard.
+     */
+    public ShardRouting getRoutingEntry() {
+        return routingEntry;
+    }
+
+    /**
+     * The failure message, if available, explaining why the shard failed.
+     */
+    @Nullable
+    public String getMessage() {
+        return message;
+    }
+
+    /**
+     * The exception, if present, causing the shard to fail.
+     */
+    @Nullable
+    public Exception getFailure() {
+        return failure;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/IndexMetaDataUpdater.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/IndexMetaDataUpdater.java
index edeeeea224a4f..b24a961829d15 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/IndexMetaDataUpdater.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/IndexMetaDataUpdater.java
@@ -22,12 +22,12 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.RoutingChangesObserver;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
-import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation.StaleShard;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
@@ -94,11 +94,6 @@ public void relocationCompleted(ShardRouting removedRelocationSource) {
         removeAllocationId(removedRelocationSource);
     }
 
-    @Override
-    public void startedPrimaryReinitialized(ShardRouting startedPrimaryShard, ShardRouting initializedShard) {
-        removeAllocationId(startedPrimaryShard);
-    }
-
     /**
      * Updates the current {@link MetaData} based on the changes of this RoutingChangesObserver. Specifically
      * we update {@link IndexMetaData#getInSyncAllocationIds()} and {@link IndexMetaData#primaryTerm(int)} based on
@@ -180,10 +175,13 @@ private IndexMetaData.Builder updateInSyncAllocations(RoutingTable newRoutingTab
             // Prevent set of inSyncAllocationIds to grow unboundedly. This can happen for example if we don't write to a primary
             // but repeatedly shut down nodes that have active replicas.
             // We use number_of_replicas + 1 (= possible active shard copies) to bound the inSyncAllocationIds set
+            // Only trim the set of allocation ids when it grows, otherwise we might trim too eagerly when the number
+            // of replicas was decreased while shards were unassigned.
             int maxActiveShards = oldIndexMetaData.getNumberOfReplicas() + 1; // +1 for the primary
-            if (inSyncAllocationIds.size() > maxActiveShards) {
+            IndexShardRoutingTable newShardRoutingTable = newRoutingTable.shardRoutingTable(shardId);
+            if (inSyncAllocationIds.size() > oldInSyncAllocationIds.size() && inSyncAllocationIds.size() > maxActiveShards) {
                 // trim entries that have no corresponding shard routing in the cluster state (i.e. trim unavailable copies)
-                List<ShardRouting> assignedShards = newRoutingTable.shardRoutingTable(shardId).assignedShards();
+                List<ShardRouting> assignedShards = newShardRoutingTable.assignedShards();
                 assert assignedShards.size() <= maxActiveShards :
                     "cannot have more assigned shards " + assignedShards + " than maximum possible active shards " + maxActiveShards;
                 Set<String> assignedAllocations = assignedShards.stream().map(s -> s.allocationId().getId()).collect(Collectors.toSet());
@@ -193,16 +191,12 @@ private IndexMetaData.Builder updateInSyncAllocations(RoutingTable newRoutingTab
                     .collect(Collectors.toSet());
             }
 
-            // only update in-sync allocation ids if there is at least one entry remaining. Assume for example that there only
-            // ever was a primary active and now it failed. If we were to remove the allocation id from the in-sync set, this would
-            // create an empty primary on the next allocation (see ShardRouting#allocatedPostIndexCreate)
-            if (inSyncAllocationIds.isEmpty() && oldInSyncAllocationIds.isEmpty() == false) {
-                assert updates.firstFailedPrimary != null :
-                    "in-sync set became empty but active primary wasn't failed: " + oldInSyncAllocationIds;
-                if (updates.firstFailedPrimary != null) {
-                    // add back allocation id of failed primary
-                    inSyncAllocationIds.add(updates.firstFailedPrimary.allocationId().getId());
-                }
+            // only remove allocation id of failed active primary if there is at least one active shard remaining. Assume for example that
+            // the primary fails but there is no new primary to fail over to. If we were to remove the allocation id of the primary from the
+            // in-sync set, this could create an empty primary on the next allocation.
+            if (newShardRoutingTable.activeShards().isEmpty() && updates.firstFailedPrimary != null) {
+                // add back allocation id of failed primary
+                inSyncAllocationIds.add(updates.firstFailedPrimary.allocationId().getId());
             }
 
             assert inSyncAllocationIds.isEmpty() == false || oldInSyncAllocationIds.isEmpty() :
@@ -229,17 +223,17 @@ public static ClusterState removeStaleIdsWithoutRoutings(ClusterState clusterSta
         MetaData.Builder metaDataBuilder = null;
         // group staleShards entries by index
         for (Map.Entry<Index, List<StaleShard>> indexEntry : staleShards.stream().collect(
-            Collectors.groupingBy(fs -> fs.shardId.getIndex())).entrySet()) {
+            Collectors.groupingBy(fs -> fs.getShardId().getIndex())).entrySet()) {
             final IndexMetaData oldIndexMetaData = oldMetaData.getIndexSafe(indexEntry.getKey());
             IndexMetaData.Builder indexMetaDataBuilder = null;
             // group staleShards entries by shard id
             for (Map.Entry<ShardId, List<StaleShard>> shardEntry : indexEntry.getValue().stream().collect(
-                Collectors.groupingBy(staleShard -> staleShard.shardId)).entrySet()) {
+                Collectors.groupingBy(staleShard -> staleShard.getShardId())).entrySet()) {
                 int shardNumber = shardEntry.getKey().getId();
                 Set<String> oldInSyncAllocations = oldIndexMetaData.inSyncAllocationIds(shardNumber);
-                Set<String> idsToRemove = shardEntry.getValue().stream().map(e -> e.allocationId).collect(Collectors.toSet());
+                Set<String> idsToRemove = shardEntry.getValue().stream().map(e -> e.getAllocationId()).collect(Collectors.toSet());
                 assert idsToRemove.stream().allMatch(id -> oldRoutingTable.getByAllocationId(shardEntry.getKey(), id) == null) :
-                    "removing stale ids: " + idsToRemove + ", some of which have still a routing entry: " + oldRoutingTable.prettyPrint();
+                    "removing stale ids: " + idsToRemove + ", some of which have still a routing entry: " + oldRoutingTable;
                 Set<String> remainingInSyncAllocations = Sets.difference(oldInSyncAllocations, idsToRemove);
                 assert remainingInSyncAllocations.isEmpty() == false : "Set of in-sync ids cannot become empty for shard " +
                     shardEntry.getKey() + " (before: " + oldInSyncAllocations + ", ids to remove: " + idsToRemove + ")";
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/MoveDecision.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/MoveDecision.java
new file mode 100644
index 0000000000000..de9795ff4c253
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/MoveDecision.java
@@ -0,0 +1,315 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.routing.allocation;
+
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.allocation.decider.Decision;
+import org.elasticsearch.cluster.routing.allocation.decider.Decision.Type;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Represents a decision to move a started shard, either because it is no longer allowed to remain on its current node
+ * or because moving it to another node will form a better cluster balance.
+ */
+public final class MoveDecision extends AbstractAllocationDecision {
+    /** a constant representing no decision taken */
+    public static final MoveDecision NOT_TAKEN = new MoveDecision(null, null, AllocationDecision.NO_ATTEMPT, null, null, 0);
+    /** cached decisions so we don't have to recreate objects for common decisions when not in explain mode. */
+    private static final MoveDecision CACHED_STAY_DECISION =
+        new MoveDecision(Decision.YES, null, AllocationDecision.NO_ATTEMPT, null, null, 0);
+    private static final MoveDecision CACHED_CANNOT_MOVE_DECISION =
+        new MoveDecision(Decision.NO, null, AllocationDecision.NO, null, null, 0);
+
+    @Nullable
+    AllocationDecision allocationDecision;
+    @Nullable
+    private final Decision canRemainDecision;
+    @Nullable
+    private final Decision clusterRebalanceDecision;
+    private final int currentNodeRanking;
+
+    private MoveDecision(Decision canRemainDecision, Decision clusterRebalanceDecision, AllocationDecision allocationDecision,
+                         DiscoveryNode assignedNode, List<NodeAllocationResult> nodeDecisions, int currentNodeRanking) {
+        super(assignedNode, nodeDecisions);
+        this.allocationDecision = allocationDecision;
+        this.canRemainDecision = canRemainDecision;
+        this.clusterRebalanceDecision = clusterRebalanceDecision;
+        this.currentNodeRanking = currentNodeRanking;
+    }
+
+    public MoveDecision(StreamInput in) throws IOException {
+        super(in);
+        allocationDecision = in.readOptionalWriteable(AllocationDecision::readFrom);
+        canRemainDecision = in.readOptionalWriteable(Decision::readFrom);
+        clusterRebalanceDecision = in.readOptionalWriteable(Decision::readFrom);
+        currentNodeRanking = in.readVInt();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeOptionalWriteable(allocationDecision);
+        out.writeOptionalWriteable(canRemainDecision);
+        out.writeOptionalWriteable(clusterRebalanceDecision);
+        out.writeVInt(currentNodeRanking);
+    }
+
+    /**
+     * Creates a move decision for the shard being able to remain on its current node, so the shard won't
+     * be forced to move to another node.
+     */
+    public static MoveDecision stay(Decision canRemainDecision) {
+        if (canRemainDecision != null) {
+            assert canRemainDecision.type() != Type.NO;
+            return new MoveDecision(canRemainDecision, null, AllocationDecision.NO_ATTEMPT, null, null, 0);
+        } else {
+            return CACHED_STAY_DECISION;
+        }
+    }
+
+    /**
+     * Creates a move decision for the shard not being allowed to remain on its current node.
+     *
+     * @param canRemainDecision the decision for whether the shard is allowed to remain on its current node
+     * @param allocationDecision the {@link AllocationDecision} for moving the shard to another node
+     * @param assignedNode the node where the shard should move to
+     * @param nodeDecisions the node-level decisions that comprised the final decision, non-null iff explain is true
+     * @return the {@link MoveDecision} for moving the shard to another node
+     */
+    public static MoveDecision cannotRemain(Decision canRemainDecision, AllocationDecision allocationDecision, DiscoveryNode assignedNode,
+                                            List<NodeAllocationResult> nodeDecisions) {
+        assert canRemainDecision != null;
+        assert canRemainDecision.type() != Type.YES : "create decision with MoveDecision#stay instead";
+        if (nodeDecisions == null && allocationDecision == AllocationDecision.NO) {
+            // the final decision is NO (no node to move the shard to) and we are not in explain mode, return a cached version
+            return CACHED_CANNOT_MOVE_DECISION;
+        } else {
+            assert ((assignedNode == null) == (allocationDecision != AllocationDecision.YES));
+            return new MoveDecision(canRemainDecision, null, allocationDecision, assignedNode, nodeDecisions, 0);
+        }
+    }
+
+    /**
+     * Creates a move decision for when rebalancing the shard is not allowed.
+     */
+    public static MoveDecision cannotRebalance(Decision canRebalanceDecision, AllocationDecision allocationDecision, int currentNodeRanking,
+                                               List<NodeAllocationResult> nodeDecisions) {
+        return new MoveDecision(null, canRebalanceDecision, allocationDecision, null, nodeDecisions, currentNodeRanking);
+    }
+
+    /**
+     * Creates a decision for whether to move the shard to a different node to form a better cluster balance.
+     */
+    public static MoveDecision rebalance(Decision canRebalanceDecision, AllocationDecision allocationDecision,
+                                         @Nullable DiscoveryNode assignedNode, int currentNodeRanking,
+                                         List<NodeAllocationResult> nodeDecisions) {
+        return new MoveDecision(null, canRebalanceDecision, allocationDecision, assignedNode, nodeDecisions, currentNodeRanking);
+    }
+
+    @Override
+    public boolean isDecisionTaken() {
+        return canRemainDecision != null || clusterRebalanceDecision != null;
+    }
+
+    /**
+     * Creates a new move decision from this decision, plus adding a remain decision.
+     */
+    public MoveDecision withRemainDecision(Decision canRemainDecision) {
+        return new MoveDecision(canRemainDecision, clusterRebalanceDecision, allocationDecision,
+                                   targetNode, nodeDecisions, currentNodeRanking);
+    }
+
+    /**
+     * Returns {@code true} if the shard cannot remain on its current node and can be moved,
+     * returns {@code false} otherwise.  If {@link #isDecisionTaken()} returns {@code false},
+     * then invoking this method will throw an {@code IllegalStateException}.
+     */
+    public boolean forceMove() {
+        checkDecisionState();
+        return canRemain() == false && allocationDecision == AllocationDecision.YES;
+    }
+
+    /**
+     * Returns {@code true} if the shard can remain on its current node, returns {@code false} otherwise.
+     * If {@link #isDecisionTaken()} returns {@code false}, then invoking this method will throw an {@code IllegalStateException}.
+     */
+    public boolean canRemain() {
+        checkDecisionState();
+        return canRemainDecision.type() == Type.YES;
+    }
+
+    /**
+     * Returns the decision for the shard being allowed to remain on its current node.  If {@link #isDecisionTaken()}
+     * returns {@code false}, then invoking this method will throw an {@code IllegalStateException}.
+     */
+    public Decision getCanRemainDecision() {
+        checkDecisionState();
+        return canRemainDecision;
+    }
+
+    /**
+     * Returns {@code true} if the shard is allowed to be rebalanced to another node in the cluster,
+     * returns {@code false} otherwise.  If {@link #getClusterRebalanceDecision()} returns {@code null}, then
+     * the result of this method is meaningless, as no rebalance decision was taken.  If {@link #isDecisionTaken()}
+     * returns {@code false}, then invoking this method will throw an {@code IllegalStateException}.
+     */
+    public boolean canRebalanceCluster() {
+        checkDecisionState();
+        return clusterRebalanceDecision != null && clusterRebalanceDecision.type() == Type.YES;
+    }
+
+    /**
+     * Returns the decision for being allowed to rebalance the shard.  Invoking this method will return
+     * {@code null} if {@link #canRemain()} ()} returns {@code false}, which means the node is not allowed to
+     * remain on its current node, so the cluster is forced to attempt to move the shard to a different node,
+     * as opposed to attempting to rebalance the shard if a better cluster balance is possible by moving it.
+     * If {@link #isDecisionTaken()} returns {@code false}, then invoking this method will throw an
+     * {@code IllegalStateException}.
+     */
+    @Nullable
+    public Decision getClusterRebalanceDecision() {
+        checkDecisionState();
+        return clusterRebalanceDecision;
+    }
+
+    /**
+     * Returns the {@link AllocationDecision} for moving this shard to another node.  If {@link #isDecisionTaken()} returns
+     * {@code false}, then invoking this method will throw an {@code IllegalStateException}.
+     */
+    @Nullable
+    public AllocationDecision getAllocationDecision() {
+        return allocationDecision;
+    }
+
+    /**
+     * Gets the current ranking of the node to which the shard is currently assigned, relative to the
+     * other nodes in the cluster as reported in {@link NodeAllocationResult#getWeightRanking()}.  The
+     * ranking will only return a meaningful positive integer if {@link #getClusterRebalanceDecision()} returns
+     * a non-null value; otherwise, 0 will be returned.  If {@link #isDecisionTaken()} returns
+     * {@code false}, then invoking this method will throw an {@code IllegalStateException}.
+     */
+    public int getCurrentNodeRanking() {
+        checkDecisionState();
+        return currentNodeRanking;
+    }
+
+    @Override
+    public String getExplanation() {
+        checkDecisionState();
+        String explanation;
+        if (clusterRebalanceDecision != null) {
+            // it was a decision to rebalance the shard, because the shard was allowed to remain on its current node
+            if (allocationDecision == AllocationDecision.AWAITING_INFO) {
+                explanation = "cannot rebalance as information about existing copies of this shard in the cluster is still being gathered";
+            } else if (clusterRebalanceDecision.type() == Type.NO) {
+                explanation = "rebalancing is not allowed" + (atLeastOneNodeWithYesDecision() ? ", even though there " +
+                              "is at least one node on which the shard can be allocated" : "");
+            } else if (clusterRebalanceDecision.type() == Type.THROTTLE) {
+                explanation = "rebalancing is throttled";
+            } else {
+                assert clusterRebalanceDecision.type() == Type.YES;
+                if (getTargetNode() != null) {
+                    if (allocationDecision == AllocationDecision.THROTTLED) {
+                        explanation = "shard rebalancing throttled";
+                    } else {
+                        explanation = "can rebalance shard";
+                    }
+                } else {
+                    explanation = "cannot rebalance as no target node exists that can both allocate this shard " +
+                                      "and improve the cluster balance";
+                }
+            }
+        } else {
+            // it was a decision to force move the shard
+            assert canRemain() == false;
+            if (allocationDecision == AllocationDecision.YES) {
+                explanation = "shard cannot remain on this node and is force-moved to another node";
+            } else if (allocationDecision == AllocationDecision.THROTTLED) {
+                explanation = "shard cannot remain on this node but is throttled on moving to another node";
+            } else {
+                assert allocationDecision == AllocationDecision.NO;
+                explanation = "cannot move shard to another node, even though it is not allowed to remain on its current node";
+            }
+        }
+        return explanation;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        checkDecisionState();
+        if (targetNode != null) {
+            builder.startObject("target_node");
+            discoveryNodeToXContent(targetNode, true, builder);
+            builder.endObject();
+        }
+        builder.field("can_remain_on_current_node", canRemain() ? "yes" : "no");
+        if (canRemain() == false && canRemainDecision.getDecisions().isEmpty() == false) {
+            builder.startArray("can_remain_decisions");
+            canRemainDecision.toXContent(builder, params);
+            builder.endArray();
+        }
+        if (clusterRebalanceDecision != null) {
+            AllocationDecision rebalanceDecision = AllocationDecision.fromDecisionType(clusterRebalanceDecision.type());
+            builder.field("can_rebalance_cluster", rebalanceDecision);
+            if (rebalanceDecision != AllocationDecision.YES && clusterRebalanceDecision.getDecisions().isEmpty() == false) {
+                builder.startArray("can_rebalance_cluster_decisions");
+                clusterRebalanceDecision.toXContent(builder, params);
+                builder.endArray();
+            }
+        }
+        if (clusterRebalanceDecision != null) {
+            builder.field("can_rebalance_to_other_node", allocationDecision);
+            builder.field("rebalance_explanation", getExplanation());
+        } else {
+            builder.field("can_move_to_other_node", forceMove() ? "yes" : "no");
+            builder.field("move_explanation", getExplanation());
+        }
+        nodeDecisionsToXContent(nodeDecisions, builder, params);
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (super.equals(other) == false) {
+            return false;
+        }
+        if (other instanceof MoveDecision == false) {
+            return false;
+        }
+        @SuppressWarnings("unchecked") MoveDecision that = (MoveDecision) other;
+        return Objects.equals(allocationDecision, that.allocationDecision)
+                   && Objects.equals(canRemainDecision, that.canRemainDecision)
+                   && Objects.equals(clusterRebalanceDecision, that.clusterRebalanceDecision)
+                   && currentNodeRanking == that.currentNodeRanking;
+    }
+
+    @Override
+    public int hashCode() {
+        return 31 * super.hashCode() + Objects.hash(allocationDecision, canRemainDecision, clusterRebalanceDecision, currentNodeRanking);
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/NodeAllocationResult.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/NodeAllocationResult.java
new file mode 100644
index 0000000000000..0d3fe2df920f6
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/NodeAllocationResult.java
@@ -0,0 +1,305 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.routing.allocation;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.allocation.decider.Decision;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Comparator;
+
+import static org.elasticsearch.cluster.routing.allocation.AbstractAllocationDecision.discoveryNodeToXContent;
+
+/**
+ * This class represents the shard allocation decision and its explanation for a single node.
+ */
+public class NodeAllocationResult implements ToXContent, Writeable, Comparable<NodeAllocationResult> {
+
+    private static final Comparator<NodeAllocationResult> nodeResultComparator =
+        Comparator.comparing(NodeAllocationResult::getNodeDecision)
+            .thenComparingInt(NodeAllocationResult::getWeightRanking)
+            .thenComparing(r -> r.getNode().getId());
+
+    private final DiscoveryNode node;
+    @Nullable
+    private final ShardStoreInfo shardStoreInfo;
+    private final AllocationDecision nodeDecision;
+    @Nullable
+    private final Decision canAllocateDecision;
+    private final int weightRanking;
+
+    public NodeAllocationResult(DiscoveryNode node, ShardStoreInfo shardStoreInfo, @Nullable Decision decision) {
+        this.node = node;
+        this.shardStoreInfo = shardStoreInfo;
+        this.canAllocateDecision = decision;
+        this.nodeDecision = decision != null ? AllocationDecision.fromDecisionType(canAllocateDecision.type()) : AllocationDecision.NO;
+        this.weightRanking = 0;
+    }
+
+    public NodeAllocationResult(DiscoveryNode node, AllocationDecision nodeDecision, Decision canAllocate, int weightRanking) {
+        this.node = node;
+        this.shardStoreInfo = null;
+        this.canAllocateDecision = canAllocate;
+        this.nodeDecision = nodeDecision;
+        this.weightRanking = weightRanking;
+    }
+
+    public NodeAllocationResult(DiscoveryNode node, Decision decision, int weightRanking) {
+        this.node = node;
+        this.shardStoreInfo = null;
+        this.canAllocateDecision = decision;
+        this.nodeDecision = AllocationDecision.fromDecisionType(decision.type());
+        this.weightRanking = weightRanking;
+    }
+
+    public NodeAllocationResult(StreamInput in) throws IOException {
+        node = new DiscoveryNode(in);
+        shardStoreInfo = in.readOptionalWriteable(ShardStoreInfo::new);
+        if (in.getVersion().before(Version.V_5_2_1)) {
+            canAllocateDecision = Decision.readFrom(in);
+        } else {
+            canAllocateDecision = in.readOptionalWriteable(Decision::readFrom);
+        }
+        nodeDecision = AllocationDecision.readFrom(in);
+        weightRanking = in.readVInt();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        node.writeTo(out);
+        out.writeOptionalWriteable(shardStoreInfo);
+        if (out.getVersion().before(Version.V_5_2_1)) {
+            if (canAllocateDecision == null) {
+                Decision.NO.writeTo(out);
+            } else {
+                canAllocateDecision.writeTo(out);
+            }
+        } else {
+            out.writeOptionalWriteable(canAllocateDecision);
+        }
+        nodeDecision.writeTo(out);
+        out.writeVInt(weightRanking);
+    }
+
+    /**
+     * Get the node that this decision is for.
+     */
+    public DiscoveryNode getNode() {
+        return node;
+    }
+
+    /**
+     * Get the shard store information for the node, if it exists.
+     */
+    @Nullable
+    public ShardStoreInfo getShardStoreInfo() {
+        return shardStoreInfo;
+    }
+
+    /**
+     * The decision details for allocating to this node.  Returns {@code null} if
+     * no allocation decision was taken on the node; in this case, {@link #getNodeDecision()}
+     * will return {@link AllocationDecision#NO}.
+     */
+    @Nullable
+    public Decision getCanAllocateDecision() {
+        return canAllocateDecision;
+    }
+
+    /**
+     * Is the weight assigned for the node?
+     */
+    public boolean isWeightRanked() {
+        return weightRanking > 0;
+    }
+
+    /**
+     * The weight ranking for allocating a shard to the node.  Each node will have
+     * a unique weight ranking that is relative to the other nodes against which the
+     * deciders ran.  For example, suppose there are 3 nodes which the allocation deciders
+     * decided upon: node1, node2, and node3.  If node2 had the best weight for holding the
+     * shard, followed by node3, followed by node1, then node2's weight will be 1, node3's
+     * weight will be 2, and node1's weight will be 1.  A value of 0 means the weight was
+     * not calculated or factored into the decision.
+     */
+    public int getWeightRanking() {
+        return weightRanking;
+    }
+
+    /**
+     * Gets the {@link AllocationDecision} for allocating to this node.
+     */
+    public AllocationDecision getNodeDecision() {
+        return nodeDecision;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        {
+            discoveryNodeToXContent(node, false, builder);
+            builder.field("node_decision", nodeDecision);
+            if (shardStoreInfo != null) {
+                shardStoreInfo.toXContent(builder, params);
+            }
+            if (isWeightRanked()) {
+                builder.field("weight_ranking", getWeightRanking());
+            }
+            if (canAllocateDecision != null && canAllocateDecision.getDecisions().isEmpty() == false) {
+                builder.startArray("deciders");
+                canAllocateDecision.toXContent(builder, params);
+                builder.endArray();
+            }
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public int compareTo(NodeAllocationResult other) {
+        return nodeResultComparator.compare(this, other);
+    }
+
+    /** A class that captures metadata about a shard store on a node. */
+    public static final class ShardStoreInfo implements ToXContent, Writeable {
+        private final boolean inSync;
+        @Nullable
+        private final String allocationId;
+        private final long matchingBytes;
+        @Nullable
+        private final Exception storeException;
+
+        public ShardStoreInfo(String allocationId, boolean inSync, Exception storeException) {
+            this.inSync = inSync;
+            this.allocationId = allocationId;
+            this.matchingBytes = -1;
+            this.storeException = storeException;
+        }
+
+        public ShardStoreInfo(long matchingBytes) {
+            this.inSync = false;
+            this.allocationId = null;
+            this.matchingBytes = matchingBytes;
+            this.storeException = null;
+        }
+
+        public ShardStoreInfo(StreamInput in) throws IOException {
+            this.inSync = in.readBoolean();
+            this.allocationId = in.readOptionalString();
+            this.matchingBytes = in.readLong();
+            this.storeException = in.readException();
+        }
+
+        /**
+         * Returns {@code true} if the shard copy is in-sync and contains the latest data.
+         * Returns {@code false} if the shard copy is stale or if the shard copy being examined
+         * is for a replica shard allocation.
+         */
+        public boolean isInSync() {
+            return inSync;
+        }
+
+        /**
+         * Gets the allocation id for the shard copy, if it exists.
+         */
+        @Nullable
+        public String getAllocationId() {
+            return allocationId;
+        }
+
+        /**
+         * Returns {@code true} if the shard copy has a matching sync id with the primary shard.
+         * Returns {@code false} if the shard copy does not have a matching sync id with the primary
+         * shard, or this explanation pertains to the allocation of a primary shard, in which case
+         * matching sync ids are irrelevant.
+         */
+        public boolean hasMatchingSyncId() {
+            return matchingBytes == Long.MAX_VALUE;
+        }
+
+        /**
+         * Gets the number of matching bytes the shard copy has with the primary shard.
+         * Returns {@code Long.MAX_VALUE} if {@link #hasMatchingSyncId()} returns {@code true}.
+         * Returns -1 if not applicable (this value only applies to assigning replica shards).
+         */
+        public long getMatchingBytes() {
+            return matchingBytes;
+        }
+
+        /**
+         * Gets the store exception when trying to read the store, if there was an error.  If
+         * there was no error, returns {@code null}.
+         */
+        @Nullable
+        public Exception getStoreException() {
+            return storeException;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeBoolean(inSync);
+            out.writeOptionalString(allocationId);
+            out.writeLong(matchingBytes);
+            out.writeException(storeException);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject("store");
+            {
+                if (matchingBytes < 0) {
+                    // dealing with a primary shard
+                    if (allocationId == null && storeException == null) {
+                        // there was no information we could obtain of any shard data on the node
+                        builder.field("found", false);
+                    } else {
+                        builder.field("in_sync", inSync);
+                    }
+                }
+                if (allocationId != null) {
+                    builder.field("allocation_id", allocationId);
+                }
+                if (matchingBytes >= 0) {
+                    if (hasMatchingSyncId()) {
+                        builder.field("matching_sync_id", true);
+                    } else {
+                        builder.byteSizeField("matching_size_in_bytes", "matching_size", matchingBytes);
+                    }
+                }
+                if (storeException != null) {
+                    builder.startObject("store_exception");
+                    ElasticsearchException.generateThrowableXContent(builder, params, storeException);
+                    builder.endObject();
+                }
+            }
+            builder.endObject();
+            return builder;
+        }
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RerouteExplanation.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RerouteExplanation.java
index d591cc131b590..36dbadec49e1b 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RerouteExplanation.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RerouteExplanation.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.cluster.routing.allocation;
 
 import org.elasticsearch.cluster.routing.allocation.command.AllocationCommand;
-import org.elasticsearch.cluster.routing.allocation.command.AllocationCommands;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -59,7 +58,7 @@ public static RerouteExplanation readFrom(StreamInput in) throws IOException {
 
     public static void writeTo(RerouteExplanation explanation, StreamOutput out) throws IOException {
         out.writeNamedWriteable(explanation.command);
-        Decision.writeTo(explanation.decisions, out);
+        explanation.decisions.writeTo(out);
     }
 
     @Override
@@ -67,15 +66,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.startObject();
         builder.field("command", command.name());
         builder.field("parameters", command);
-        // The Decision could be a Multi or Single decision, and they should
-        // both be encoded the same, so check and wrap in an array if necessary
-        if (decisions instanceof Decision.Multi) {
-            decisions.toXContent(builder, params);
-        } else {
-            builder.startArray("decisions");
-            decisions.toXContent(builder, params);
-            builder.endArray();
-        }
+        builder.startArray("decisions");
+        decisions.toXContent(builder, params);
+        builder.endArray();
         builder.endObject();
         return builder;
     }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java
index f2d40b38a4673..e1ae367bebf76 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java
@@ -21,6 +21,7 @@
 
 import org.elasticsearch.cluster.ClusterInfo;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.RestoreInProgress;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RoutingChangesObserver;
@@ -30,6 +31,7 @@
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.snapshots.RestoreService.RestoreInProgressUpdater;
 
 import java.util.HashMap;
 import java.util.HashSet;
@@ -46,83 +48,6 @@
  */
 public class RoutingAllocation {
 
-    /**
-     * this class is used to describe results of a {@link RoutingAllocation}
-     */
-    public static class Result {
-
-        private final boolean changed;
-
-        private final RoutingTable routingTable;
-
-        private final MetaData metaData;
-
-        private final RoutingExplanations explanations;
-
-        /**
-         * Creates a new {@link RoutingAllocation.Result} where no change to the routing table was made.
-         * @param clusterState the unchanged {@link ClusterState}
-         */
-        public static Result unchanged(ClusterState clusterState) {
-            return new Result(false, clusterState.routingTable(), clusterState.metaData(), new RoutingExplanations());
-        }
-
-        /**
-         * Creates a new {@link RoutingAllocation.Result} where changes were made to the routing table.
-         * @param routingTable the {@link RoutingTable} this Result references
-         * @param metaData the {@link MetaData} this Result references
-         * @param explanations Explanation for the reroute actions
-         */
-        public static Result changed(RoutingTable routingTable, MetaData metaData, RoutingExplanations explanations) {
-            return new Result(true, routingTable, metaData, explanations);
-        }
-
-        /**
-         * Creates a new {@link RoutingAllocation.Result}
-         * @param changed a flag to determine whether the actual {@link RoutingTable} has been changed
-         * @param routingTable the {@link RoutingTable} this Result references
-         * @param metaData the {@link MetaData} this Result references
-         * @param explanations Explanation for the reroute actions
-         */
-        private Result(boolean changed, RoutingTable routingTable, MetaData metaData, RoutingExplanations explanations) {
-            this.changed = changed;
-            this.routingTable = routingTable;
-            this.metaData = metaData;
-            this.explanations = explanations;
-        }
-
-        /** determine whether the actual {@link RoutingTable} has been changed
-         * @return <code>true</code> if the {@link RoutingTable} has been changed by allocation. Otherwise <code>false</code>
-         */
-        public boolean changed() {
-            return this.changed;
-        }
-
-        /**
-         * Get the {@link MetaData} referenced by this result
-         * @return referenced {@link MetaData}
-         */
-        public MetaData metaData() {
-            return metaData;
-        }
-
-        /**
-         * Get the {@link RoutingTable} referenced by this result
-         * @return referenced {@link RoutingTable}
-         */
-        public RoutingTable routingTable() {
-            return routingTable;
-        }
-
-        /**
-         * Get the explanation of this result
-         * @return explanation
-         */
-        public RoutingExplanations explanations() {
-            return explanations;
-        }
-    }
-
     private final AllocationDeciders deciders;
 
     private final RoutingNodes routingNodes;
@@ -135,8 +60,6 @@ public RoutingExplanations explanations() {
 
     private final ImmutableOpenMap<String, ClusterState.Custom> customs;
 
-    private final AllocationExplanation explanation = new AllocationExplanation();
-
     private final ClusterInfo clusterInfo;
 
     private Map<ShardId, Set<String>> ignoredShardToNodes = null;
@@ -145,7 +68,7 @@ public RoutingExplanations explanations() {
 
     private final boolean retryFailed;
 
-    private boolean debugDecision = false;
+    private DebugMode debugDecision = DebugMode.OFF;
 
     private boolean hasPendingAsyncFetch = false;
 
@@ -153,8 +76,9 @@ public RoutingExplanations explanations() {
 
     private final IndexMetaDataUpdater indexMetaDataUpdater = new IndexMetaDataUpdater();
     private final RoutingNodesChangedObserver nodesChangedObserver = new RoutingNodesChangedObserver();
+    private final RestoreInProgressUpdater restoreInProgressUpdater = new RestoreInProgressUpdater();
     private final RoutingChangesObserver routingChangesObserver = new RoutingChangesObserver.DelegatingRoutingChangesObserver(
-        nodesChangedObserver, indexMetaDataUpdater
+        nodesChangedObserver, indexMetaDataUpdater, restoreInProgressUpdater
     );
 
 
@@ -231,12 +155,8 @@ public <T extends ClusterState.Custom> T custom(String key) {
         return (T)customs.get(key);
     }
 
-    /**
-     * Get explanations of current routing
-     * @return explanation of routing
-     */
-    public AllocationExplanation explanation() {
-        return explanation;
+    public ImmutableOpenMap<String, ClusterState.Custom> getCustoms() {
+        return customs;
     }
 
     public void ignoreDisable(boolean ignoreDisable) {
@@ -247,11 +167,19 @@ public boolean ignoreDisable() {
         return this.ignoreDisable;
     }
 
-    public void debugDecision(boolean debug) {
+    public void setDebugMode(DebugMode debug) {
         this.debugDecision = debug;
     }
 
+    public void debugDecision(boolean debug) {
+        this.debugDecision = debug ? DebugMode.ON : DebugMode.OFF;
+    }
+
     public boolean debugDecision() {
+        return this.debugDecision != DebugMode.OFF;
+    }
+
+    public DebugMode getDebugMode() {
         return this.debugDecision;
     }
 
@@ -311,6 +239,13 @@ public MetaData updateMetaDataWithRoutingChanges(RoutingTable newRoutingTable) {
         return indexMetaDataUpdater.applyChanges(metaData, newRoutingTable);
     }
 
+    /**
+     * Returns updated {@link RestoreInProgress} based on the changes that were made to the routing nodes
+     */
+    public RestoreInProgress updateRestoreInfoWithRoutingChanges(RestoreInProgress restoreInProgress) {
+        return restoreInProgressUpdater.applyChanges(restoreInProgress);
+    }
+
     /**
      * Returns true iff changes were made to the routing nodes
      */
@@ -353,4 +288,20 @@ public void setHasPendingAsyncFetch() {
     public boolean isRetryFailed() {
         return retryFailed;
     }
+
+    public enum DebugMode {
+        /**
+         * debug mode is off
+         */
+        OFF,
+        /**
+         * debug mode is on
+         */
+        ON,
+        /**
+         * debug mode is on, but YES decisions from a {@link org.elasticsearch.cluster.routing.allocation.decider.Decision.Multi}
+         * are not included.
+         */
+        EXCLUDE_YES_DECISIONS
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingNodesChangedObserver.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingNodesChangedObserver.java
index 42e80689eeca5..3e465e42b4453 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingNodesChangedObserver.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingNodesChangedObserver.java
@@ -96,6 +96,17 @@ public void replicaPromoted(ShardRouting replicaShard) {
         setChanged();
     }
 
+    @Override
+    public void initializedReplicaReinitialized(ShardRouting oldReplica, ShardRouting reinitializedReplica) {
+        assert oldReplica.initializing() && oldReplica.primary() == false :
+            "expected initializing replica shard " + oldReplica;
+        assert reinitializedReplica.initializing() && reinitializedReplica.primary() == false :
+            "expected reinitialized replica shard " + reinitializedReplica;
+        assert oldReplica.allocationId().getId().equals(reinitializedReplica.allocationId().getId()) == false :
+            "expected allocation id to change for reinitialized replica shard (old: " + oldReplica + " new: " + reinitializedReplica + ")";
+        setChanged();
+    }
+
     /**
      * Marks the allocation as changed.
      */
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/ShardAllocationDecision.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/ShardAllocationDecision.java
new file mode 100644
index 0000000000000..557ce9300c610
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/ShardAllocationDecision.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.routing.allocation;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+
+/**
+ * Represents the decision taken for the allocation of a single shard.  If
+ * the shard is unassigned, {@link #getAllocateDecision()} will return an
+ * object containing the decision and its explanation, and {@link #getMoveDecision()}
+ * will return an object for which {@link MoveDecision#isDecisionTaken()} returns
+ * {@code false}.  If the shard is in the started state, then {@link #getMoveDecision()}
+ * will return an object containing the decision to move/rebalance the shard, and
+ * {@link #getAllocateDecision()} will return an object for which
+ * {@link AllocateUnassignedDecision#isDecisionTaken()} returns {@code false}.  If
+ * the shard is neither unassigned nor started (i.e. it is initializing or relocating),
+ * then both {@link #getAllocateDecision()} and {@link #getMoveDecision()} will return
+ * objects whose {@code isDecisionTaken()} method returns {@code false}.
+ */
+public final class ShardAllocationDecision implements ToXContent, Writeable {
+    public static final ShardAllocationDecision NOT_TAKEN =
+        new ShardAllocationDecision(AllocateUnassignedDecision.NOT_TAKEN, MoveDecision.NOT_TAKEN);
+
+    private final AllocateUnassignedDecision allocateDecision;
+    private final MoveDecision moveDecision;
+
+    public ShardAllocationDecision(AllocateUnassignedDecision allocateDecision,
+                                   MoveDecision moveDecision) {
+        this.allocateDecision = allocateDecision;
+        this.moveDecision = moveDecision;
+    }
+
+    public ShardAllocationDecision(StreamInput in) throws IOException {
+        allocateDecision = new AllocateUnassignedDecision(in);
+        moveDecision = new MoveDecision(in);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        allocateDecision.writeTo(out);
+        moveDecision.writeTo(out);
+    }
+
+    /**
+     * Returns {@code true} if either an allocation decision or a move decision was taken
+     * for the shard.  If no decision was taken, as in the case of initializing or relocating
+     * shards, then this method returns {@code false}.
+     */
+    public boolean isDecisionTaken() {
+        return allocateDecision.isDecisionTaken() || moveDecision.isDecisionTaken();
+    }
+
+    /**
+     * Gets the unassigned allocation decision for the shard.  If the shard was not in the unassigned state,
+     * the instance of {@link AllocateUnassignedDecision} that is returned will have {@link AllocateUnassignedDecision#isDecisionTaken()}
+     * return {@code false}.
+     */
+    public AllocateUnassignedDecision getAllocateDecision() {
+        return allocateDecision;
+    }
+
+    /**
+     * Gets the move decision for the shard.  If the shard was not in the started state,
+     * the instance of {@link MoveDecision} that is returned will have {@link MoveDecision#isDecisionTaken()}
+     * return {@code false}.
+     */
+    public MoveDecision getMoveDecision() {
+        return moveDecision;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        if (allocateDecision.isDecisionTaken()) {
+            allocateDecision.toXContent(builder, params);
+        }
+        if (moveDecision.isDecisionTaken()) {
+            moveDecision.toXContent(builder, params);
+        }
+        return builder;
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/StaleShard.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/StaleShard.java
new file mode 100644
index 0000000000000..9454f62db9732
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/StaleShard.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.routing.allocation;
+
+import org.elasticsearch.index.shard.ShardId;
+
+/**
+ * A class that represents a stale shard copy.
+ */
+public class StaleShard {
+    private final ShardId shardId;
+    private final String allocationId;
+
+    public StaleShard(ShardId shardId, String allocationId) {
+        this.shardId = shardId;
+        this.allocationId = allocationId;
+    }
+
+    @Override
+    public String toString() {
+        return "stale shard, shard " + shardId + ", alloc. id [" + allocationId + "]";
+    }
+
+    /**
+     * The shard id of the stale shard.
+     */
+    public ShardId getShardId() {
+        return shardId;
+    }
+
+    /**
+     * The allocation id of the stale shard.
+     */
+    public String getAllocationId() {
+        return allocationId;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/StartedRerouteAllocation.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/StartedRerouteAllocation.java
deleted file mode 100644
index e63ce2b19e923..0000000000000
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/StartedRerouteAllocation.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.cluster.routing.allocation;
-
-import org.elasticsearch.cluster.ClusterInfo;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.routing.RoutingNodes;
-import org.elasticsearch.cluster.routing.ShardRouting;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
-
-import java.util.List;
-
-/**
- * This {@link RoutingAllocation} holds a list of started shards within a
- * cluster
- */
-public class StartedRerouteAllocation extends RoutingAllocation {
-
-    private final List<ShardRouting> startedShards;
-
-    public StartedRerouteAllocation(AllocationDeciders deciders, RoutingNodes routingNodes, ClusterState clusterState,
-                                    List<ShardRouting> startedShards, ClusterInfo clusterInfo, long currentNanoTime) {
-        super(deciders, routingNodes, clusterState, clusterInfo, currentNanoTime, false);
-        this.startedShards = startedShards;
-    }
-
-    /**
-     * Get started shards
-     * @return list of started shards
-     */
-    public List<ShardRouting> startedShards() {
-        return startedShards;
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java
index 7087ae57c4b3f..a60287a8487a3 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java
@@ -29,8 +29,13 @@
 import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
-import org.elasticsearch.cluster.routing.UnassignedInfo;
+import org.elasticsearch.cluster.routing.UnassignedInfo.AllocationStatus;
+import org.elasticsearch.cluster.routing.allocation.AllocateUnassignedDecision;
+import org.elasticsearch.cluster.routing.allocation.AllocationDecision;
+import org.elasticsearch.cluster.routing.allocation.MoveDecision;
+import org.elasticsearch.cluster.routing.allocation.NodeAllocationResult;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
+import org.elasticsearch.cluster.routing.allocation.ShardAllocationDecision;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision.Type;
@@ -42,14 +47,17 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.gateway.PriorityComparator;
 
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.IdentityHashMap;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
@@ -73,9 +81,9 @@
 public class BalancedShardsAllocator extends AbstractComponent implements ShardsAllocator {
 
     public static final Setting<Float> INDEX_BALANCE_FACTOR_SETTING =
-        Setting.floatSetting("cluster.routing.allocation.balance.index", 0.55f, Property.Dynamic, Property.NodeScope);
+        Setting.floatSetting("cluster.routing.allocation.balance.index", 0.55f, 0.0f, Property.Dynamic, Property.NodeScope);
     public static final Setting<Float> SHARD_BALANCE_FACTOR_SETTING =
-        Setting.floatSetting("cluster.routing.allocation.balance.shard", 0.45f, Property.Dynamic, Property.NodeScope);
+        Setting.floatSetting("cluster.routing.allocation.balance.shard", 0.45f, 0.0f, Property.Dynamic, Property.NodeScope);
     public static final Setting<Float> THRESHOLD_SETTING =
         Setting.floatSetting("cluster.routing.allocation.balance.threshold", 1.0f, 0.0f,
             Property.Dynamic, Property.NodeScope);
@@ -104,12 +112,6 @@ private void setThreshold(float threshold) {
         this.threshold = threshold;
     }
 
-    @Override
-    public Map<DiscoveryNode, Float> weighShard(RoutingAllocation allocation, ShardRouting shard) {
-        final Balancer balancer = new Balancer(logger, allocation, weightFunction, threshold);
-        return balancer.weighShard(shard);
-    }
-
     @Override
     public void allocate(RoutingAllocation allocation) {
         if (allocation.routingNodes().size() == 0) {
@@ -122,6 +124,23 @@ public void allocate(RoutingAllocation allocation) {
         balancer.balance();
     }
 
+    @Override
+    public ShardAllocationDecision decideShardAllocation(final ShardRouting shard, final RoutingAllocation allocation) {
+        Balancer balancer = new Balancer(logger, allocation, weightFunction, threshold);
+        AllocateUnassignedDecision allocateUnassignedDecision = AllocateUnassignedDecision.NOT_TAKEN;
+        MoveDecision moveDecision = MoveDecision.NOT_TAKEN;
+        if (shard.unassigned()) {
+            allocateUnassignedDecision = balancer.decideAllocateUnassigned(shard, Sets.newHashSet());
+        } else {
+            moveDecision = balancer.decideMove(shard);
+            if (moveDecision.isDecisionTaken() && moveDecision.canRemain()) {
+                MoveDecision rebalanceDecision = balancer.decideRebalance(shard);
+                moveDecision = rebalanceDecision.withRemainDecision(moveDecision.getCanRemainDecision());
+            }
+        }
+        return new ShardAllocationDecision(allocateUnassignedDecision, moveDecision);
+    }
+
     /**
      * Returns the currently configured delta threshold
      */
@@ -210,7 +229,7 @@ private float weight(Balancer balancer, ModelNode node, String index, int numAdd
      */
     public static class Balancer {
         private final Logger logger;
-        private final Map<String, ModelNode> nodes = new HashMap<>();
+        private final Map<String, ModelNode> nodes;
         private final RoutingAllocation allocation;
         private final RoutingNodes routingNodes;
         private final WeightFunction weight;
@@ -218,6 +237,7 @@ public static class Balancer {
         private final float threshold;
         private final MetaData metaData;
         private final float avgShardsPerNode;
+        private final NodeSorter sorter;
 
         public Balancer(Logger logger, RoutingAllocation allocation, WeightFunction weight, float threshold) {
             this.logger = logger;
@@ -227,7 +247,8 @@ public Balancer(Logger logger, RoutingAllocation allocation, WeightFunction weig
             this.routingNodes = allocation.routingNodes();
             this.metaData = allocation.metaData();
             avgShardsPerNode = ((float) metaData.getTotalNumberOfShards()) / routingNodes.size();
-            buildModelFromAssigned();
+            nodes = Collections.unmodifiableMap(buildModelFromAssigned());
+            sorter = newNodeSorter();
         }
 
         /**
@@ -261,11 +282,18 @@ private NodeSorter newNodeSorter() {
             return new NodeSorter(nodesArray(), weight, this);
         }
 
+        /**
+         * The absolute value difference between two weights.
+         */
         private static float absDelta(float lower, float higher) {
             assert higher >= lower : higher + " lt " + lower +" but was expected to be gte";
             return Math.abs(higher - lower);
         }
 
+        /**
+         * Returns {@code true} iff the weight delta between two nodes is under a defined threshold.
+         * See {@link #THRESHOLD_SETTING} for defining the threshold.
+         */
         private static boolean lessThan(float delta, float threshold) {
             /* deltas close to the threshold are "rounded" to the threshold manually
                to prevent floating point problems if the delta is very close to the
@@ -303,12 +331,127 @@ private void balance() {
             balanceByWeights();
         }
 
+        /**
+         * Makes a decision about moving a single shard to a different node to form a more
+         * optimally balanced cluster.  This method is invoked from the cluster allocation
+         * explain API only.
+         */
+        private MoveDecision decideRebalance(final ShardRouting shard) {
+            if (shard.started() == false) {
+                // we can only rebalance started shards
+                return MoveDecision.NOT_TAKEN;
+            }
+
+            Decision canRebalance = allocation.deciders().canRebalance(shard, allocation);
+
+            sorter.reset(shard.getIndexName());
+            ModelNode[] modelNodes = sorter.modelNodes;
+            final String currentNodeId = shard.currentNodeId();
+            // find currently assigned node
+            ModelNode currentNode = null;
+            for (ModelNode node : modelNodes) {
+                if (node.getNodeId().equals(currentNodeId)) {
+                    currentNode = node;
+                    break;
+                }
+            }
+            assert currentNode != null : "currently assigned node could not be found";
+
+            // balance the shard, if a better node can be found
+            final float currentWeight = sorter.weight(currentNode);
+            final AllocationDeciders deciders = allocation.deciders();
+            final String idxName = shard.getIndexName();
+            Type rebalanceDecisionType = Type.NO;
+            ModelNode assignedNode = null;
+            List<Tuple<ModelNode, Decision>> betterBalanceNodes = new ArrayList<>();
+            List<Tuple<ModelNode, Decision>> sameBalanceNodes = new ArrayList<>();
+            List<Tuple<ModelNode, Decision>> worseBalanceNodes = new ArrayList<>();
+            for (ModelNode node : modelNodes) {
+                if (node == currentNode) {
+                    continue; // skip over node we're currently allocated to
+                }
+                final Decision canAllocate = deciders.canAllocate(shard, node.getRoutingNode(), allocation);
+                // the current weight of the node in the cluster, as computed by the weight function;
+                // this is a comparison of the number of shards on this node to the number of shards
+                // that should be on each node on average (both taking the cluster as a whole into account
+                // as well as shards per index)
+                final float nodeWeight = sorter.weight(node);
+                // if the node we are examining has a worse (higher) weight than the node the shard is
+                // assigned to, then there is no way moving the shard to the node with the worse weight
+                // can make the balance of the cluster better, so we check for that here
+                final boolean betterWeightThanCurrent = nodeWeight <= currentWeight;
+                boolean rebalanceConditionsMet = false;
+                if (betterWeightThanCurrent) {
+                    // get the delta between the weights of the node we are checking and the node that holds the shard
+                    float currentDelta = absDelta(nodeWeight, currentWeight);
+                    // checks if the weight delta is above a certain threshold; if it is not above a certain threshold,
+                    // then even though the node we are examining has a better weight and may make the cluster balance
+                    // more even, it doesn't make sense to execute the heavyweight operation of relocating a shard unless
+                    // the gains make it worth it, as defined by the threshold
+                    boolean deltaAboveThreshold = lessThan(currentDelta, threshold) == false;
+                    // simulate the weight of the node if we were to relocate the shard to it
+                    float weightWithShardAdded = weight.weightShardAdded(this, node, idxName);
+                    // calculate the delta of the weights of the two nodes if we were to add the shard to the
+                    // node in question and move it away from the node that currently holds it.
+                    float proposedDelta = weightWithShardAdded - weight.weightShardRemoved(this, currentNode, idxName);
+                    boolean betterWeightWithShardAdded = proposedDelta < currentDelta;
+                    rebalanceConditionsMet = deltaAboveThreshold && betterWeightWithShardAdded;
+                    // if the simulated weight delta with the shard moved away is better than the weight delta
+                    // with the shard remaining on the current node, and we are allowed to allocate to the
+                    // node in question, then allow the rebalance
+                    if (rebalanceConditionsMet && canAllocate.type().higherThan(rebalanceDecisionType)) {
+                        // rebalance to the node, only will get overwritten if the decision here is to
+                        // THROTTLE and we get a decision with YES on another node
+                        rebalanceDecisionType = canAllocate.type();
+                        assignedNode = node;
+                    }
+                }
+                Tuple<ModelNode, Decision> nodeResult = Tuple.tuple(node, canAllocate);
+                if (rebalanceConditionsMet) {
+                    betterBalanceNodes.add(nodeResult);
+                } else if (betterWeightThanCurrent) {
+                    sameBalanceNodes.add(nodeResult);
+                } else {
+                    worseBalanceNodes.add(nodeResult);
+                }
+            }
+
+            int weightRanking = 0;
+            List<NodeAllocationResult> nodeDecisions = new ArrayList<>(modelNodes.length - 1);
+            for (Tuple<ModelNode, Decision> result : betterBalanceNodes) {
+                nodeDecisions.add(new NodeAllocationResult(
+                    result.v1().routingNode.node(), AllocationDecision.fromDecisionType(result.v2().type()), result.v2(), ++weightRanking)
+                );
+            }
+            int currentNodeWeightRanking = ++weightRanking;
+            for (Tuple<ModelNode, Decision> result : sameBalanceNodes) {
+                AllocationDecision nodeDecision = result.v2().type() == Type.NO ? AllocationDecision.NO : AllocationDecision.WORSE_BALANCE;
+                nodeDecisions.add(new NodeAllocationResult(
+                    result.v1().routingNode.node(), nodeDecision, result.v2(), currentNodeWeightRanking)
+                );
+            }
+            for (Tuple<ModelNode, Decision> result : worseBalanceNodes) {
+                AllocationDecision nodeDecision = result.v2().type() == Type.NO ? AllocationDecision.NO : AllocationDecision.WORSE_BALANCE;
+                nodeDecisions.add(new NodeAllocationResult(
+                    result.v1().routingNode.node(), nodeDecision, result.v2(), ++weightRanking)
+                );
+            }
+
+            if (canRebalance.type() != Type.YES || allocation.hasPendingAsyncFetch()) {
+                AllocationDecision allocationDecision = allocation.hasPendingAsyncFetch() ? AllocationDecision.AWAITING_INFO :
+                                                            AllocationDecision.fromDecisionType(canRebalance.type());
+                return MoveDecision.cannotRebalance(canRebalance, allocationDecision, currentNodeWeightRanking, nodeDecisions);
+            } else {
+                return MoveDecision.rebalance(canRebalance, AllocationDecision.fromDecisionType(rebalanceDecisionType),
+                    assignedNode != null ? assignedNode.routingNode.node() : null, currentNodeWeightRanking, nodeDecisions);
+            }
+        }
+
         public Map<DiscoveryNode, Float> weighShard(ShardRouting shard) {
-            final NodeSorter sorter = newNodeSorter();
             final ModelNode[] modelNodes = sorter.modelNodes;
             final float[] weights = sorter.weights;
 
-            buildWeightOrderedIndices(sorter);
+            buildWeightOrderedIndices();
             Map<DiscoveryNode, Float> nodes = new HashMap<>(modelNodes.length);
             float currentNodeWeight = 0.0f;
             for (int i = 0; i < modelNodes.length; i++) {
@@ -332,20 +475,19 @@ public Map<DiscoveryNode, Float> weighShard(ShardRouting shard) {
          * weight of the maximum node and the minimum node according to the
          * {@link WeightFunction}. This weight is calculated per index to
          * distribute shards evenly per index. The balancer tries to relocate
-         * shards only if the delta exceeds the threshold. If the default case
+         * shards only if the delta exceeds the threshold. In the default case
          * the threshold is set to <tt>1.0</tt> to enforce gaining relocation
          * only, or in other words relocations that move the weight delta closer
          * to <tt>0.0</tt>
          */
         private void balanceByWeights() {
-            final NodeSorter sorter = newNodeSorter();
             final AllocationDeciders deciders = allocation.deciders();
             final ModelNode[] modelNodes = sorter.modelNodes;
             final float[] weights = sorter.weights;
-            for (String index : buildWeightOrderedIndices(sorter)) {
+            for (String index : buildWeightOrderedIndices()) {
                 IndexMetaData indexMetaData = metaData.index(index);
 
-                // find nodes that have a shard of this index or where shards of this index are allowed to stay
+                // find nodes that have a shard of this index or where shards of this index are allowed to be allocated to,
                 // move these nodes to the front of modelNodes so that we can only balance based on these nodes
                 int relevantNodes = 0;
                 for (int i = 0; i < modelNodes.length; i++) {
@@ -440,14 +582,14 @@ private void balanceByWeights() {
          * allocations on added nodes from one index when the weight parameters
          * for global balance overrule the index balance at an intermediate
          * state. For example this can happen if we have 3 nodes and 3 indices
-         * with 3 shards and 1 shard. At the first stage all three nodes hold
-         * 2 shard for each index. now we add another node and the first index
-         * is balanced moving 3 two of the nodes over to the new node since it
+         * with 3 primary and 1 replica shards. At the first stage all three nodes hold
+         * 2 shard for each index. Now we add another node and the first index
+         * is balanced moving three shards from two of the nodes over to the new node since it
          * has no shards yet and global balance for the node is way below
          * average. To re-balance we need to move shards back eventually likely
          * to the nodes we relocated them from.
          */
-        private String[] buildWeightOrderedIndices(NodeSorter sorter) {
+        private String[] buildWeightOrderedIndices() {
             final String[] indices = allocation.routingTable().indicesRouting().keys().toArray(String.class);
             final float[] deltas = new float[indices.length];
             for (int i = 0; i < deltas.length; i++) {
@@ -501,27 +643,52 @@ public void moveShards() {
             // Iterate over the started shards interleaving between nodes, and check if they can remain. In the presence of throttling
             // shard movements, the goal of this iteration order is to achieve a fairer movement of shards from the nodes that are
             // offloading the shards.
-            final NodeSorter sorter = newNodeSorter();
             for (Iterator<ShardRouting> it = allocation.routingNodes().nodeInterleavedShardIterator(); it.hasNext(); ) {
                 ShardRouting shardRouting = it.next();
-                // we can only move started shards...
-                if (shardRouting.started()) {
+                final MoveDecision moveDecision = decideMove(shardRouting);
+                if (moveDecision.isDecisionTaken() && moveDecision.forceMove()) {
                     final ModelNode sourceNode = nodes.get(shardRouting.currentNodeId());
-                    assert sourceNode != null && sourceNode.containsShard(shardRouting);
-                    RoutingNode routingNode = sourceNode.getRoutingNode();
-                    Decision decision = allocation.deciders().canRemain(shardRouting, routingNode, allocation);
-                    if (decision.type() == Decision.Type.NO) {
-                        moveShard(sorter, shardRouting, sourceNode, routingNode);
+                    final ModelNode targetNode = nodes.get(moveDecision.getTargetNode().getId());
+                    sourceNode.removeShard(shardRouting);
+                    Tuple<ShardRouting, ShardRouting> relocatingShards = routingNodes.relocateShard(shardRouting, targetNode.getNodeId(),
+                        allocation.clusterInfo().getShardSize(shardRouting, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE), allocation.changes());
+                    targetNode.addShard(relocatingShards.v2());
+                    if (logger.isTraceEnabled()) {
+                        logger.trace("Moved shard [{}] to node [{}]", shardRouting, targetNode.getRoutingNode());
                     }
+                } else if (moveDecision.isDecisionTaken() && moveDecision.canRemain() == false) {
+                    logger.trace("[{}][{}] can't move", shardRouting.index(), shardRouting.id());
                 }
             }
         }
 
         /**
-         * Move started shard to the minimal eligible node with respect to the weight function
+         * Makes a decision on whether to move a started shard to another node.  The following rules apply
+         * to the {@link MoveDecision} return object:
+         *   1. If the shard is not started, no decision will be taken and {@link MoveDecision#isDecisionTaken()} will return false.
+         *   2. If the shard is allowed to remain on its current node, no attempt will be made to move the shard and
+         *      {@link MoveDecision#canRemainDecision} will have a decision type of YES.  All other fields in the object will be null.
+         *   3. If the shard is not allowed to remain on its current node, then {@link MoveDecision#getAllocationDecision()} will be
+         *      populated with the decision of moving to another node.  If {@link MoveDecision#forceMove()} ()} returns {@code true}, then
+         *      {@link MoveDecision#targetNode} will return a non-null value, otherwise the assignedNodeId will be null.
+         *   4. If the method is invoked in explain mode (e.g. from the cluster allocation explain APIs), then
+         *      {@link MoveDecision#nodeDecisions} will have a non-null value.
          */
-        private void moveShard(NodeSorter sorter, ShardRouting shardRouting, ModelNode sourceNode, RoutingNode routingNode) {
-            logger.debug("[{}][{}] allocated on [{}], but can no longer be allocated on it, moving...", shardRouting.index(), shardRouting.id(), routingNode.node());
+        public MoveDecision decideMove(final ShardRouting shardRouting) {
+            if (shardRouting.started() == false) {
+                // we can only move started shards
+                return MoveDecision.NOT_TAKEN;
+            }
+
+            final boolean explain = allocation.debugDecision();
+            final ModelNode sourceNode = nodes.get(shardRouting.currentNodeId());
+            assert sourceNode != null && sourceNode.containsShard(shardRouting);
+            RoutingNode routingNode = sourceNode.getRoutingNode();
+            Decision canRemain = allocation.deciders().canRemain(shardRouting, routingNode, allocation);
+            if (canRemain.type() != Decision.Type.NO) {
+                return MoveDecision.stay(canRemain);
+            }
+
             sorter.reset(shardRouting.getIndexName());
             /*
              * the sorter holds the minimum weight node first for the shards index.
@@ -529,23 +696,36 @@ private void moveShard(NodeSorter sorter, ShardRouting shardRouting, ModelNode s
              * This is not guaranteed to be balanced after this operation we still try best effort to
              * allocate on the minimal eligible node.
              */
+            Type bestDecision = Type.NO;
+            RoutingNode targetNode = null;
+            final List<NodeAllocationResult> nodeExplanationMap = explain ? new ArrayList<>() : null;
+            int weightRanking = 0;
             for (ModelNode currentNode : sorter.modelNodes) {
                 if (currentNode != sourceNode) {
                     RoutingNode target = currentNode.getRoutingNode();
                     // don't use canRebalance as we want hard filtering rules to apply. See #17698
                     Decision allocationDecision = allocation.deciders().canAllocate(shardRouting, target, allocation);
-                    if (allocationDecision.type() == Type.YES) { // TODO maybe we can respect throttling here too?
-                        sourceNode.removeShard(shardRouting);
-                        Tuple<ShardRouting, ShardRouting> relocatingShards = routingNodes.relocateShard(shardRouting, target.nodeId(), allocation.clusterInfo().getShardSize(shardRouting, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE), allocation.changes());
-                        currentNode.addShard(relocatingShards.v2());
-                        if (logger.isTraceEnabled()) {
-                            logger.trace("Moved shard [{}] to node [{}]", shardRouting, routingNode.node());
+                    if (explain) {
+                        nodeExplanationMap.add(new NodeAllocationResult(
+                            currentNode.getRoutingNode().node(), allocationDecision, ++weightRanking));
+                    }
+                    // TODO maybe we can respect throttling here too?
+                    if (allocationDecision.type().higherThan(bestDecision)) {
+                        bestDecision = allocationDecision.type();
+                        if (bestDecision == Type.YES) {
+                            targetNode = target;
+                            if (explain == false) {
+                                // we are not in explain mode and already have a YES decision on the best weighted node,
+                                // no need to continue iterating
+                                break;
+                            }
                         }
-                        return;
                     }
                 }
             }
-            logger.debug("[{}][{}] can't move", shardRouting.index(), shardRouting.id());
+
+            return MoveDecision.cannotRemain(canRemain, AllocationDecision.fromDecisionType(bestDecision),
+                targetNode != null ? targetNode.node() : null, nodeExplanationMap);
         }
 
         /**
@@ -557,7 +737,8 @@ private void moveShard(NodeSorter sorter, ShardRouting shardRouting, ModelNode s
          * on the target node which we respect during the allocation / balancing
          * process. In short, this method recreates the status-quo in the cluster.
          */
-        private void buildModelFromAssigned() {
+        private Map<String, ModelNode> buildModelFromAssigned() {
+            Map<String, ModelNode> nodes = new HashMap<>();
             for (RoutingNode rn : routingNodes) {
                 ModelNode node = new ModelNode(rn);
                 nodes.put(rn.nodeId(), node);
@@ -572,6 +753,7 @@ private void buildModelFromAssigned() {
                     }
                 }
             }
+            return nodes;
         }
 
         /**
@@ -626,116 +808,61 @@ private void allocateUnassigned() {
             do {
                 for (int i = 0; i < primaryLength; i++) {
                     ShardRouting shard = primary[i];
-                    if (!shard.primary()) {
-                        final Decision decision = deciders.canAllocate(shard, allocation);
-                        if (decision.type() == Type.NO) {
-                            UnassignedInfo.AllocationStatus allocationStatus = UnassignedInfo.AllocationStatus.fromDecision(decision);
-                            unassigned.ignoreShard(shard, allocationStatus, allocation.changes());
-                            while(i < primaryLength-1 && comparator.compare(primary[i], primary[i+1]) == 0) {
-                                unassigned.ignoreShard(primary[++i], allocationStatus, allocation.changes());
-                            }
-                            continue;
-                        } else {
+                    AllocateUnassignedDecision allocationDecision = decideAllocateUnassigned(shard, throttledNodes);
+                    final String assignedNodeId = allocationDecision.getTargetNode() != null ?
+                                                      allocationDecision.getTargetNode().getId() : null;
+                    final ModelNode minNode = assignedNodeId != null ? nodes.get(assignedNodeId) : null;
+
+                    if (allocationDecision.getAllocationDecision() == AllocationDecision.YES) {
+                        if (logger.isTraceEnabled()) {
+                            logger.trace("Assigned shard [{}] to [{}]", shard, minNode.getNodeId());
+                        }
+
+                        final long shardSize = DiskThresholdDecider.getExpectedShardSize(shard, allocation,
+                            ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE);
+                        shard = routingNodes.initializeShard(shard, minNode.getNodeId(), null, shardSize, allocation.changes());
+                        minNode.addShard(shard);
+                        if (!shard.primary()) {
+                            // copy over the same replica shards to the secondary array so they will get allocated
+                            // in a subsequent iteration, allowing replicas of other shards to be allocated first
                             while(i < primaryLength-1 && comparator.compare(primary[i], primary[i+1]) == 0) {
                                 secondary[secondaryLength++] = primary[++i];
                             }
                         }
-                    }
-                    assert !shard.assignedToNode() : shard;
-                    /* find an node with minimal weight we can allocate on*/
-                    float minWeight = Float.POSITIVE_INFINITY;
-                    ModelNode minNode = null;
-                    Decision decision = null;
-                    if (throttledNodes.size() < nodes.size()) {
-                        /* Don't iterate over an identity hashset here the
-                         * iteration order is different for each run and makes testing hard */
-                        for (ModelNode node : nodes.values()) {
-                            if (throttledNodes.contains(node)) {
-                                continue;
-                            }
-                            if (!node.containsShard(shard)) {
-                                // simulate weight if we would add shard to node
-                                float currentWeight = weight.weightShardAdded(this, node, shard.getIndexName());
-                                /*
-                                 * Unless the operation is not providing any gains we
-                                 * don't check deciders
-                                 */
-                                if (currentWeight <= minWeight) {
-                                    Decision currentDecision = deciders.canAllocate(shard, node.getRoutingNode(), allocation);
-                                    NOUPDATE:
-                                    if (currentDecision.type() == Type.YES || currentDecision.type() == Type.THROTTLE) {
-                                        if (currentWeight == minWeight) {
-                                            /*  we have an equal weight tie breaking:
-                                             *  1. if one decision is YES prefer it
-                                             *  2. prefer the node that holds the primary for this index with the next id in the ring ie.
-                                             *  for the 3 shards 2 replica case we try to build up:
-                                             *    1 2 0
-                                             *    2 0 1
-                                             *    0 1 2
-                                             *  such that if we need to tie-break we try to prefer the node holding a shard with the minimal id greater
-                                             *  than the id of the shard we need to assign. This works find when new indices are created since
-                                             *  primaries are added first and we only add one shard set a time in this algorithm.
-                                             */
-                                            if (currentDecision.type() == decision.type()) {
-                                                final int repId = shard.id();
-                                                final int nodeHigh = node.highestPrimary(shard.index().getName());
-                                                final int minNodeHigh = minNode.highestPrimary(shard.getIndexName());
-                                                if ((((nodeHigh > repId && minNodeHigh > repId) || (nodeHigh < repId && minNodeHigh < repId)) && (nodeHigh < minNodeHigh))
-                                                        || (nodeHigh > minNodeHigh && nodeHigh > repId && minNodeHigh < repId)) {
-                                                    // nothing to set here; the minNode, minWeight, and decision get set below
-                                                } else {
-                                                    break NOUPDATE;
-                                                }
-                                            } else if (currentDecision.type() != Type.YES) {
-                                                break NOUPDATE;
-                                            }
-                                        }
-                                        minNode = node;
-                                        minWeight = currentWeight;
-                                        decision = currentDecision;
-                                    }
-                                }
-                            }
+                    } else {
+                        // did *not* receive a YES decision
+                        if (logger.isTraceEnabled()) {
+                            logger.trace("No eligible node found to assign shard [{}] allocation_status [{}]", shard,
+                                allocationDecision.getAllocationStatus());
                         }
-                    }
-                    assert (decision == null) == (minNode == null);
-                    if (minNode != null) {
-                        final long shardSize = DiskThresholdDecider.getExpectedShardSize(shard, allocation,
-                            ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE);
-                        if (decision.type() == Type.YES) {
-                            if (logger.isTraceEnabled()) {
-                                logger.trace("Assigned shard [{}] to [{}]", shard, minNode.getNodeId());
-                            }
 
-                            shard = routingNodes.initializeShard(shard, minNode.getNodeId(), null, shardSize, allocation.changes());
-                            minNode.addShard(shard);
-                            continue; // don't add to ignoreUnassigned
-                        } else {
+                        if (minNode != null) {
+                            // throttle decision scenario
+                            assert allocationDecision.getAllocationStatus() == AllocationStatus.DECIDERS_THROTTLED;
+                            final long shardSize = DiskThresholdDecider.getExpectedShardSize(shard, allocation,
+                                ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE);
                             minNode.addShard(shard.initialize(minNode.getNodeId(), null, shardSize));
                             final RoutingNode node = minNode.getRoutingNode();
                             final Decision.Type nodeLevelDecision = deciders.canAllocate(node, allocation).type();
                             if (nodeLevelDecision != Type.YES) {
                                 if (logger.isTraceEnabled()) {
-                                    logger.trace("Can not allocate on node [{}] remove from round decision [{}]", node, decision.type());
+                                    logger.trace("Can not allocate on node [{}] remove from round decision [{}]", node,
+                                        allocationDecision.getAllocationStatus());
                                 }
                                 assert nodeLevelDecision == Type.NO;
                                 throttledNodes.add(minNode);
                             }
+                        } else {
+                            if (logger.isTraceEnabled()) {
+                                logger.trace("No Node found to assign shard [{}]", shard);
+                            }
                         }
-                        if (logger.isTraceEnabled()) {
-                            logger.trace("No eligible node found to assign shard [{}] decision [{}]", shard, decision.type());
-                        }
-                    } else if (logger.isTraceEnabled()) {
-                        logger.trace("No Node found to assign shard [{}]", shard);
-                    }
-                    assert decision == null || decision.type() == Type.THROTTLE;
-                    UnassignedInfo.AllocationStatus allocationStatus =
-                        decision == null ? UnassignedInfo.AllocationStatus.DECIDERS_NO :
-                                           UnassignedInfo.AllocationStatus.fromDecision(decision);
-                    unassigned.ignoreShard(shard, allocationStatus, allocation.changes());
-                    if (!shard.primary()) { // we could not allocate it and we are a replica - check if we can ignore the other replicas
-                        while(secondaryLength > 0 && comparator.compare(shard, secondary[secondaryLength-1]) == 0) {
-                            unassigned.ignoreShard(secondary[--secondaryLength], allocationStatus, allocation.changes());
+
+                        unassigned.ignoreShard(shard, allocationDecision.getAllocationStatus(), allocation.changes());
+                        if (!shard.primary()) { // we could not allocate it and we are a replica - check if we can ignore the other replicas
+                            while(i < primaryLength-1 && comparator.compare(primary[i], primary[i+1]) == 0) {
+                                unassigned.ignoreShard(primary[++i], allocationDecision.getAllocationStatus(), allocation.changes());
+                            }
                         }
                     }
                 }
@@ -748,6 +875,114 @@ private void allocateUnassigned() {
             // clear everything we have either added it or moved to ignoreUnassigned
         }
 
+        /**
+         * Make a decision for allocating an unassigned shard.  This method returns a two values in a tuple: the
+         * first value is the {@link Decision} taken to allocate the unassigned shard, the second value is the
+         * {@link ModelNode} representing the node that the shard should be assigned to.  If the decision returned
+         * is of type {@link Type#NO}, then the assigned node will be null.
+         */
+        private AllocateUnassignedDecision decideAllocateUnassigned(final ShardRouting shard, final Set<ModelNode> throttledNodes) {
+            if (shard.assignedToNode()) {
+                // we only make decisions for unassigned shards here
+                return AllocateUnassignedDecision.NOT_TAKEN;
+            }
+
+            final boolean explain = allocation.debugDecision();
+            Decision shardLevelDecision = allocation.deciders().canAllocate(shard, allocation);
+            if (shardLevelDecision.type() == Type.NO && explain == false) {
+                // NO decision for allocating the shard, irrespective of any particular node, so exit early
+                return AllocateUnassignedDecision.no(AllocationStatus.DECIDERS_NO, null);
+            }
+
+            /* find an node with minimal weight we can allocate on*/
+            float minWeight = Float.POSITIVE_INFINITY;
+            ModelNode minNode = null;
+            Decision decision = null;
+            if (throttledNodes.size() >= nodes.size() && explain == false) {
+                // all nodes are throttled, so we know we won't be able to allocate this round,
+                // so if we are not in explain mode, short circuit
+                return AllocateUnassignedDecision.no(AllocationStatus.DECIDERS_NO, null);
+            }
+            /* Don't iterate over an identity hashset here the
+             * iteration order is different for each run and makes testing hard */
+            Map<String, NodeAllocationResult> nodeExplanationMap = explain ? new HashMap<>() : null;
+            List<Tuple<String, Float>> nodeWeights = explain ? new ArrayList<>() : null;
+            for (ModelNode node : nodes.values()) {
+                if ((throttledNodes.contains(node) || node.containsShard(shard)) && explain == false) {
+                    // decision is NO without needing to check anything further, so short circuit
+                    continue;
+                }
+
+                // simulate weight if we would add shard to node
+                float currentWeight = weight.weightShardAdded(this, node, shard.getIndexName());
+                // moving the shard would not improve the balance, and we are not in explain mode, so short circuit
+                if (currentWeight > minWeight && explain == false) {
+                    continue;
+                }
+
+                Decision currentDecision = allocation.deciders().canAllocate(shard, node.getRoutingNode(), allocation);
+                if (explain) {
+                    nodeExplanationMap.put(node.getNodeId(),
+                        new NodeAllocationResult(node.getRoutingNode().node(), currentDecision, 0));
+                    nodeWeights.add(Tuple.tuple(node.getNodeId(), currentWeight));
+                }
+                if (currentDecision.type() == Type.YES || currentDecision.type() == Type.THROTTLE) {
+                    final boolean updateMinNode;
+                    if (currentWeight == minWeight) {
+                        /*  we have an equal weight tie breaking:
+                         *  1. if one decision is YES prefer it
+                         *  2. prefer the node that holds the primary for this index with the next id in the ring ie.
+                         *  for the 3 shards 2 replica case we try to build up:
+                         *    1 2 0
+                         *    2 0 1
+                         *    0 1 2
+                         *  such that if we need to tie-break we try to prefer the node holding a shard with the minimal id greater
+                         *  than the id of the shard we need to assign. This works find when new indices are created since
+                         *  primaries are added first and we only add one shard set a time in this algorithm.
+                         */
+                        if (currentDecision.type() == decision.type()) {
+                            final int repId = shard.id();
+                            final int nodeHigh = node.highestPrimary(shard.index().getName());
+                            final int minNodeHigh = minNode.highestPrimary(shard.getIndexName());
+                            updateMinNode = ((((nodeHigh > repId && minNodeHigh > repId)
+                                                   || (nodeHigh < repId && minNodeHigh < repId))
+                                                  && (nodeHigh < minNodeHigh))
+                                                 || (nodeHigh > minNodeHigh && nodeHigh > repId && minNodeHigh < repId));
+                        } else {
+                            updateMinNode = currentDecision.type() == Type.YES;
+                        }
+                    } else {
+                        updateMinNode = true;
+                    }
+                    if (updateMinNode) {
+                        minNode = node;
+                        minWeight = currentWeight;
+                        decision = currentDecision;
+                    }
+                }
+            }
+            if (decision == null) {
+                // decision was not set and a node was not assigned, so treat it as a NO decision
+                decision = Decision.NO;
+            }
+            List<NodeAllocationResult> nodeDecisions = null;
+            if (explain) {
+                nodeDecisions = new ArrayList<>();
+                // fill in the correct weight ranking, once we've been through all nodes
+                nodeWeights.sort((nodeWeight1, nodeWeight2) -> Float.compare(nodeWeight1.v2(), nodeWeight2.v2()));
+                int weightRanking = 0;
+                for (Tuple<String, Float> nodeWeight : nodeWeights) {
+                    NodeAllocationResult current = nodeExplanationMap.get(nodeWeight.v1());
+                    nodeDecisions.add(new NodeAllocationResult(current.getNode(), current.getCanAllocateDecision(), ++weightRanking));
+                }
+            }
+            return AllocateUnassignedDecision.fromDecision(
+                decision,
+                minNode != null ? minNode.routingNode.node() : null,
+                nodeDecisions
+            );
+        }
+
         /**
          * Tries to find a relocation from the max node to the minimal node for an arbitrary shard of the given index on the
          * balance model. Iff this method returns a <code>true</code> the relocation has already been executed on the
@@ -792,10 +1027,8 @@ private boolean tryRelocateShard(ModelNode minNode, ModelNode maxNode, String id
                     long shardSize = allocation.clusterInfo().getShardSize(candidate, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE);
 
                     if (decision.type() == Type.YES) { /* only allocate on the cluster if we are not throttled */
-                        if (logger.isTraceEnabled()) {
-                            logger.trace("Relocate shard [{}] from node [{}] to node [{}]", candidate, maxNode.getNodeId(),
+                        logger.debug("Relocate shard [{}] from node [{}] to node [{}]", candidate, maxNode.getNodeId(),
                                     minNode.getNodeId());
-                        }
                         /* now allocate on the cluster */
                         minNode.addShard(routingNodes.relocateShard(candidate, minNode.getNodeId(), shardSize, allocation.changes()).v1());
                         return true;
@@ -819,7 +1052,7 @@ static class ModelNode implements Iterable<ModelIndex> {
         private int numShards = 0;
         private final RoutingNode routingNode;
 
-        public ModelNode(RoutingNode routingNode) {
+        ModelNode(RoutingNode routingNode) {
             this.routingNode = routingNode;
         }
 
@@ -897,7 +1130,7 @@ static final class ModelIndex implements Iterable<ShardRouting> {
         private final Set<ShardRouting> shards = new HashSet<>(4); // expect few shards of same index to be allocated on same node
         private int highestPrimary = -1;
 
-        public ModelIndex(String id) {
+        ModelIndex(String id) {
             this.id = id;
         }
 
@@ -954,7 +1187,7 @@ static final class NodeSorter extends IntroSorter {
         private final Balancer balancer;
         private float pivotWeight;
 
-        public NodeSorter(ModelNode[] modelNodes, WeightFunction function, Balancer balancer) {
+        NodeSorter(ModelNode[] modelNodes, WeightFunction function, Balancer balancer) {
             this.function = function;
             this.balancer = balancer;
             this.modelNodes = modelNodes;
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardsAllocator.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardsAllocator.java
index 35f3b2654181e..7e9d15b45282f 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardsAllocator.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardsAllocator.java
@@ -19,11 +19,12 @@
 
 package org.elasticsearch.cluster.routing.allocation.allocator;
 
-import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.allocation.AllocateUnassignedDecision;
+import org.elasticsearch.cluster.routing.allocation.MoveDecision;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
+import org.elasticsearch.cluster.routing.allocation.ShardAllocationDecision;
 
-import java.util.Map;
 /**
  * <p>
  * A {@link ShardsAllocator} is the main entry point for shard allocation on nodes in the cluster.
@@ -44,13 +45,17 @@ public interface ShardsAllocator {
     void allocate(RoutingAllocation allocation);
 
     /**
-     * Returns a map of node to a float "weight" of where the allocator would like to place the shard.
-     * Higher weights signify greater desire to place the shard on that node.
-     * Does not modify the allocation at all.
+     * Returns the decision for where a shard should reside in the cluster.  If the shard is unassigned,
+     * then the {@link AllocateUnassignedDecision} will be non-null.  If the shard is not in the unassigned
+     * state, then the {@link MoveDecision} will be non-null.
      *
-     * @param allocation current node allocation
-     * @param shard shard to weigh
-     * @return map of nodes to float weights
+     * This method is primarily used by the cluster allocation explain API to provide detailed explanations
+     * for the allocation of a single shard.  Implementations of the {@link #allocate(RoutingAllocation)} method
+     * may use the results of this method implementation to decide on allocating shards in the routing table
+     * to the cluster.
+     *
+     * If an implementation of this interface does not support explaining decisions for a single shard through
+     * the cluster explain API, then this method should throw a {@code UnsupportedOperationException}.
      */
-    Map<DiscoveryNode, Float> weighShard(RoutingAllocation allocation, ShardRouting shard);
+    ShardAllocationDecision decideShardAllocation(ShardRouting shard, RoutingAllocation allocation);
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AbstractAllocateAllocationCommand.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AbstractAllocateAllocationCommand.java
index 90a591b119938..4ffd70aee1cd8 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AbstractAllocateAllocationCommand.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AbstractAllocateAllocationCommand.java
@@ -30,7 +30,6 @@
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ObjectParser;
@@ -50,8 +49,8 @@ public abstract class AbstractAllocateAllocationCommand implements AllocationCom
     private static final String SHARD_FIELD = "shard";
     private static final String NODE_FIELD = "node";
 
-    protected static <T extends Builder<?>> ObjectParser<T, ParseFieldMatcherSupplier> createAllocateParser(String command) {
-        ObjectParser<T, ParseFieldMatcherSupplier> parser = new ObjectParser<>(command);
+    protected static <T extends Builder<?>> ObjectParser<T, Void> createAllocateParser(String command) {
+        ObjectParser<T, Void> parser = new ObjectParser<>(command);
         parser.declareString(Builder::setIndex, new ParseField(INDEX_FIELD));
         parser.declareInt(Builder::setShard, new ParseField(SHARD_FIELD));
         parser.declareString(Builder::setNode, new ParseField(NODE_FIELD));
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateEmptyPrimaryAllocationCommand.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateEmptyPrimaryAllocationCommand.java
index 82d6f436d2a70..157acc0e537b6 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateEmptyPrimaryAllocationCommand.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateEmptyPrimaryAllocationCommand.java
@@ -30,8 +30,6 @@
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -49,8 +47,7 @@ public class AllocateEmptyPrimaryAllocationCommand extends BasePrimaryAllocation
     public static final String NAME = "allocate_empty_primary";
     public static final ParseField COMMAND_NAME_FIELD = new ParseField(NAME);
 
-    private static final ObjectParser<Builder, ParseFieldMatcherSupplier> EMPTY_PRIMARY_PARSER = BasePrimaryAllocationCommand
-            .createAllocatePrimaryParser(NAME);
+    private static final ObjectParser<Builder, Void> EMPTY_PRIMARY_PARSER = BasePrimaryAllocationCommand.createAllocatePrimaryParser(NAME);
 
     /**
      * Creates a new {@link AllocateEmptyPrimaryAllocationCommand}
@@ -83,7 +80,7 @@ public static class Builder extends BasePrimaryAllocationCommand.Builder<Allocat
 
         @Override
         public Builder parse(XContentParser parser) throws IOException {
-            return EMPTY_PRIMARY_PARSER.parse(parser, this, () -> ParseFieldMatcher.STRICT);
+            return EMPTY_PRIMARY_PARSER.parse(parser, this, null);
         }
 
         @Override
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateReplicaAllocationCommand.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateReplicaAllocationCommand.java
index 8c47deee66fe6..6ec09a9bbbbe9 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateReplicaAllocationCommand.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateReplicaAllocationCommand.java
@@ -28,8 +28,6 @@
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -46,8 +44,7 @@ public class AllocateReplicaAllocationCommand extends AbstractAllocateAllocation
     public static final String NAME = "allocate_replica";
     public static final ParseField COMMAND_NAME_FIELD = new ParseField(NAME);
 
-    private static final ObjectParser<AllocateReplicaAllocationCommand.Builder, ParseFieldMatcherSupplier> REPLICA_PARSER =
-            createAllocateParser(NAME);
+    private static final ObjectParser<AllocateReplicaAllocationCommand.Builder, Void> REPLICA_PARSER = createAllocateParser(NAME);
 
     /**
      * Creates a new {@link AllocateReplicaAllocationCommand}
@@ -80,7 +77,7 @@ protected static class Builder extends AbstractAllocateAllocationCommand.Builder
 
         @Override
         public Builder parse(XContentParser parser) throws IOException {
-            return REPLICA_PARSER.parse(parser, this, () -> ParseFieldMatcher.STRICT);
+            return REPLICA_PARSER.parse(parser, this, null);
         }
 
         @Override
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateStalePrimaryAllocationCommand.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateStalePrimaryAllocationCommand.java
index acdd5cae30b49..c643fb5c948ae 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateStalePrimaryAllocationCommand.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateStalePrimaryAllocationCommand.java
@@ -28,8 +28,6 @@
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -46,8 +44,7 @@ public class AllocateStalePrimaryAllocationCommand extends BasePrimaryAllocation
     public static final String NAME = "allocate_stale_primary";
     public static final ParseField COMMAND_NAME_FIELD = new ParseField(NAME);
 
-    private static final ObjectParser<Builder, ParseFieldMatcherSupplier> STALE_PRIMARY_PARSER = BasePrimaryAllocationCommand
-            .createAllocatePrimaryParser(NAME);
+    private static final ObjectParser<Builder, Void> STALE_PRIMARY_PARSER = BasePrimaryAllocationCommand.createAllocatePrimaryParser(NAME);
 
     /**
      * Creates a new {@link AllocateStalePrimaryAllocationCommand}
@@ -81,7 +78,7 @@ public static class Builder extends BasePrimaryAllocationCommand.Builder<Allocat
 
         @Override
         public Builder parse(XContentParser parser) throws IOException {
-            return STALE_PRIMARY_PARSER.parse(parser, this, () -> ParseFieldMatcher.STRICT);
+            return STALE_PRIMARY_PARSER.parse(parser, this, null);
         }
 
         @Override
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocationCommandRegistry.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocationCommandRegistry.java
deleted file mode 100644
index 27c1f074c40a7..0000000000000
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocationCommandRegistry.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.cluster.routing.allocation.command;
-
-import org.elasticsearch.common.xcontent.ParseFieldRegistry;
-
-/**
- * Registry of allocation commands. This is it's own class just to make Guice happy.
- */
-public class AllocationCommandRegistry extends ParseFieldRegistry<AllocationCommand.Parser<?>> {
-    public AllocationCommandRegistry() {
-        super("allocation_command");
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocationCommands.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocationCommands.java
index 10ba3f5594458..5098b027f611a 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocationCommands.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocationCommands.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.action.support.ToXContentToBytes;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.routing.allocation.RoutingExplanations;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -126,12 +125,10 @@ public static void writeTo(AllocationCommands commands, StreamOutput out) throws
      *     }
      * </pre>
      * @param parser {@link XContentParser} to read the commands from
-     * @param registry of allocation command parsers
      * @return {@link AllocationCommands} read
      * @throws IOException if something bad happens while reading the stream
      */
-    public static AllocationCommands fromXContent(XContentParser parser, ParseFieldMatcher parseFieldMatcher,
-            AllocationCommandRegistry registry) throws IOException {
+    public static AllocationCommands fromXContent(XContentParser parser) throws IOException {
         AllocationCommands commands = new AllocationCommands();
 
         XContentParser.Token token = parser.currentToken();
@@ -160,7 +157,7 @@ public static AllocationCommands fromXContent(XContentParser parser, ParseFieldM
                 token = parser.nextToken();
                 String commandName = parser.currentName();
                 token = parser.nextToken();
-                commands.add(registry.lookup(commandName, parseFieldMatcher, parser.getTokenLocation()).fromXContent(parser));
+                commands.add(parser.namedObject(AllocationCommand.class, commandName, null));
                 // move to the end object one
                 if (parser.nextToken() != XContentParser.Token.END_OBJECT) {
                     throw new ElasticsearchParseException("allocation command is malformed, done parsing a command, but didn't get END_OBJECT, got [{}] instead", token);
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/BasePrimaryAllocationCommand.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/BasePrimaryAllocationCommand.java
index f4dc4fba4b877..2cb0426012524 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/BasePrimaryAllocationCommand.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/BasePrimaryAllocationCommand.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.cluster.routing.allocation.command;
 
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ObjectParser;
@@ -35,8 +34,8 @@ public abstract class BasePrimaryAllocationCommand extends AbstractAllocateAlloc
 
     private static final String ACCEPT_DATA_LOSS_FIELD = "accept_data_loss";
 
-    protected static <T extends Builder<?>> ObjectParser<T, ParseFieldMatcherSupplier> createAllocatePrimaryParser(String command) {
-        ObjectParser<T, ParseFieldMatcherSupplier> parser = AbstractAllocateAllocationCommand.createAllocateParser(command);
+    protected static <T extends Builder<?>> ObjectParser<T, Void> createAllocatePrimaryParser(String command) {
+        ObjectParser<T, Void> parser = AbstractAllocateAllocationCommand.createAllocateParser(command);
         parser.declareBoolean(Builder::setAcceptDataLoss, new ParseField(ACCEPT_DATA_LOSS_FIELD));
         return parser;
     }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java
index 5b6f145fe8f2b..53e67ba25a429 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java
@@ -23,13 +23,12 @@
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 
 import java.util.Collection;
 import java.util.Collections;
-import java.util.List;
-import java.util.Set;
+
+import static org.elasticsearch.cluster.routing.allocation.RoutingAllocation.DebugMode.EXCLUDE_YES_DECISIONS;
 
 /**
  * A composite {@link AllocationDecider} combining the "decision" of multiple
@@ -56,7 +55,8 @@ public Decision canRebalance(ShardRouting shardRouting, RoutingAllocation alloca
                 } else {
                     ret.add(decision);
                 }
-            } else if (decision != Decision.ALWAYS) {
+            } else if (decision != Decision.ALWAYS
+                        && (allocation.getDebugMode() != EXCLUDE_YES_DECISIONS || decision.type() != Decision.Type.YES)) {
                 ret.add(decision);
             }
         }
@@ -74,7 +74,7 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
             // short track if a NO is returned.
             if (decision == Decision.NO) {
                 if (logger.isTraceEnabled()) {
-                    logger.trace("Can not allocate [{}] on node [{}] due to [{}]", shardRouting, node.nodeId(), allocationDecider.getClass().getSimpleName());
+                    logger.trace("Can not allocate [{}] on node [{}] due to [{}]", shardRouting, node.node(), allocationDecider.getClass().getSimpleName());
                 }
                 // short circuit only if debugging is not enabled
                 if (!allocation.debugDecision()) {
@@ -82,7 +82,8 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
                 } else {
                     ret.add(decision);
                 }
-            } else if (decision != Decision.ALWAYS) {
+            } else if (decision != Decision.ALWAYS
+                        && (allocation.getDebugMode() != EXCLUDE_YES_DECISIONS || decision.type() != Decision.Type.YES)) {
                 // the assumption is that a decider that returns the static instance Decision#ALWAYS
                 // does not really implements canAllocate
                 ret.add(decision);
@@ -112,7 +113,8 @@ public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAl
                 } else {
                     ret.add(decision);
                 }
-            } else if (decision != Decision.ALWAYS) {
+            } else if (decision != Decision.ALWAYS
+                        && (allocation.getDebugMode() != EXCLUDE_YES_DECISIONS || decision.type() != Decision.Type.YES)) {
                 ret.add(decision);
             }
         }
@@ -131,7 +133,8 @@ public Decision canAllocate(IndexMetaData indexMetaData, RoutingNode node, Routi
                 } else {
                     ret.add(decision);
                 }
-            } else if (decision != Decision.ALWAYS) {
+            } else if (decision != Decision.ALWAYS
+                        && (allocation.getDebugMode() != EXCLUDE_YES_DECISIONS || decision.type() != Decision.Type.YES)) {
                 ret.add(decision);
             }
         }
@@ -150,7 +153,8 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingAllocation allocat
                 } else {
                     ret.add(decision);
                 }
-            } else if (decision != Decision.ALWAYS) {
+            } else if (decision != Decision.ALWAYS
+                        && (allocation.getDebugMode() != EXCLUDE_YES_DECISIONS || decision.type() != Decision.Type.YES)) {
                 ret.add(decision);
             }
         }
@@ -169,7 +173,8 @@ public Decision canAllocate(RoutingNode node, RoutingAllocation allocation) {
                 } else {
                     ret.add(decision);
                 }
-            } else if (decision != Decision.ALWAYS) {
+            } else if (decision != Decision.ALWAYS
+                        && (allocation.getDebugMode() != EXCLUDE_YES_DECISIONS || decision.type() != Decision.Type.YES)) {
                 ret.add(decision);
             }
         }
@@ -188,7 +193,8 @@ public Decision canRebalance(RoutingAllocation allocation) {
                 } else {
                     ret.add(decision);
                 }
-            } else if (decision != Decision.ALWAYS) {
+            } else if (decision != Decision.ALWAYS
+                        && (allocation.getDebugMode() != EXCLUDE_YES_DECISIONS || decision.type() != Decision.Type.YES)) {
                 ret.add(decision);
             }
         }
@@ -216,7 +222,8 @@ public Decision canForceAllocatePrimary(ShardRouting shardRouting, RoutingNode n
                 } else {
                     ret.add(decision);
                 }
-            } else if (decision != Decision.ALWAYS) {
+            } else if (decision != Decision.ALWAYS
+                        && (allocation.getDebugMode() != EXCLUDE_YES_DECISIONS || decision.type() != Decision.Type.YES)) {
                 ret.add(decision);
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java
index ceeb23bab1a4f..4160fd224aa14 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java
@@ -49,7 +49,7 @@
  * To enable allocation awareness in this example nodes should contain a value
  * for the <tt>rack_id</tt> key like:
  * <pre>
- * node.rack_id:1
+ * node.attr.rack_id:1
  * </pre>
  * <p>
  * Awareness can also be used to prevent over-allocation in the case of node or
@@ -78,31 +78,15 @@ public class AwarenessAllocationDecider extends AllocationDecider {
     public static final String NAME = "awareness";
 
     public static final Setting<String[]> CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING =
-        new Setting<>("cluster.routing.allocation.awareness.attributes", "", Strings::splitStringByCommaToArray , Property.Dynamic,
+        new Setting<>("cluster.routing.allocation.awareness.attributes", "", s -> Strings.tokenizeToStringArray(s, ","), Property.Dynamic,
             Property.NodeScope);
     public static final Setting<Settings> CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING =
         Setting.groupSetting("cluster.routing.allocation.awareness.force.", Property.Dynamic, Property.NodeScope);
 
-    private String[] awarenessAttributes;
+    private volatile String[] awarenessAttributes;
 
     private volatile Map<String, String[]> forcedAwarenessAttributes;
 
-    /**
-     * Creates a new {@link AwarenessAllocationDecider} instance
-     */
-    public AwarenessAllocationDecider() {
-        this(Settings.Builder.EMPTY_SETTINGS);
-    }
-
-    /**
-     * Creates a new {@link AwarenessAllocationDecider} instance from given settings
-     *
-     * @param settings {@link Settings} to use
-     */
-    public AwarenessAllocationDecider(Settings settings) {
-        this(settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
-    }
-
     public AwarenessAllocationDecider(Settings settings, ClusterSettings clusterSettings) {
         super(settings);
         this.awarenessAttributes = CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.get(settings);
@@ -140,7 +124,9 @@ public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAl
 
     private Decision underCapacity(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation, boolean moveToNode) {
         if (awarenessAttributes.length == 0) {
-            return allocation.decision(Decision.YES, NAME, "allocation awareness is not enabled");
+            return allocation.decision(Decision.YES, NAME,
+                "allocation awareness is not enabled, set cluster setting [%s] to enable it",
+                CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.getKey());
         }
 
         IndexMetaData indexMetaData = allocation.metaData().getIndexSafe(shardRouting.index());
@@ -148,7 +134,10 @@ private Decision underCapacity(ShardRouting shardRouting, RoutingNode node, Rout
         for (String awarenessAttribute : awarenessAttributes) {
             // the node the shard exists on must be associated with an awareness attribute
             if (!node.node().getAttributes().containsKey(awarenessAttribute)) {
-                return allocation.decision(Decision.NO, NAME, "node does not contain the awareness attribute: [%s]", awarenessAttribute);
+                return allocation.decision(Decision.NO, NAME,
+                    "node does not contain the awareness attribute [%s]; required attributes cluster setting [%s=%s]",
+                    awarenessAttribute, CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.getKey(),
+                    allocation.debugDecision() ? Strings.arrayToCommaDelimitedString(awarenessAttributes) : null);
             }
 
             // build attr_value -> nodes map
@@ -206,15 +195,14 @@ private Decision underCapacity(ShardRouting shardRouting, RoutingNode node, Rout
             // if we are above with leftover, then we know we are not good, even with mod
             if (currentNodeCount > (requiredCountPerAttribute + leftoverPerAttribute)) {
                 return allocation.decision(Decision.NO, NAME,
-                        "there are too many shards on the node for attribute [%s], there are [%d] total shards for the index " +
-                        " and [%d] total attributes values, expected the node count [%d] to be lower or equal to the required " +
-                        "number of shards per attribute [%d] plus leftover [%d]",
+                        "there are too many copies of the shard allocated to nodes with attribute [%s], there are [%d] total configured " +
+                        "shard copies for this shard id and [%d] total attribute values, expected the allocated shard count per " +
+                        "attribute [%d] to be less than or equal to the upper bound of the required number of shards per attribute [%d]",
                         awarenessAttribute,
                         shardCount,
                         numberOfAttributes,
                         currentNodeCount,
-                        requiredCountPerAttribute,
-                        leftoverPerAttribute);
+                        requiredCountPerAttribute + leftoverPerAttribute);
             }
             // all is well, we are below or same as average
             if (currentNodeCount <= requiredCountPerAttribute) {
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDecider.java
index c343d4254c8a5..281f6a603c3f3 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDecider.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDecider.java
@@ -48,14 +48,15 @@
 public class ClusterRebalanceAllocationDecider extends AllocationDecider {
 
     public static final String NAME = "cluster_rebalance";
+    private static final String CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE = "cluster.routing.allocation.allow_rebalance";
     public static final Setting<ClusterRebalanceType> CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING =
-        new Setting<>("cluster.routing.allocation.allow_rebalance", ClusterRebalanceType.INDICES_ALL_ACTIVE.name().toLowerCase(Locale.ROOT),
+        new Setting<>(CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, ClusterRebalanceType.INDICES_ALL_ACTIVE.toString(),
             ClusterRebalanceType::parseString, Property.Dynamic, Property.NodeScope);
 
     /**
      * An enum representation for the configured re-balance type.
      */
-    public static enum ClusterRebalanceType {
+    public enum ClusterRebalanceType {
         /**
          * Re-balancing is allowed once a shard replication group is active
          */
@@ -80,6 +81,11 @@ public static ClusterRebalanceType parseString(String typeString) {
             throw new IllegalArgumentException("Illegal value for " +
                             CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING + ": " + typeString);
         }
+
+        @Override
+        public String toString() {
+            return name().toLowerCase(Locale.ROOT);
+        }
     }
 
     private volatile ClusterRebalanceType type;
@@ -94,8 +100,7 @@ public ClusterRebalanceAllocationDecider(Settings settings, ClusterSettings clus
                     CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getRaw(settings));
             type = ClusterRebalanceType.INDICES_ALL_ACTIVE;
         }
-        logger.debug("using [{}] with [{}]", CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(),
-                type.toString().toLowerCase(Locale.ROOT));
+        logger.debug("using [{}] with [{}]", CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, type);
 
         clusterSettings.addSettingsUpdateConsumer(CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING, this::setType);
     }
@@ -115,12 +120,14 @@ public Decision canRebalance(RoutingAllocation allocation) {
             // check if there are unassigned primaries.
             if ( allocation.routingNodes().hasUnassignedPrimaries() ) {
                 return allocation.decision(Decision.NO, NAME,
-                        "the cluster has unassigned primary shards and rebalance type is set to [%s]", type);
+                        "the cluster has unassigned primary shards and cluster setting [%s] is set to [%s]",
+                        CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, type);
             }
             // check if there are initializing primaries that don't have a relocatingNodeId entry.
             if ( allocation.routingNodes().hasInactivePrimaries() ) {
                 return allocation.decision(Decision.NO, NAME,
-                        "the cluster has inactive primary shards and rebalance type is set to [%s]", type);
+                        "the cluster has inactive primary shards and cluster setting [%s] is set to [%s]",
+                        CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, type);
             }
 
             return allocation.decision(Decision.YES, NAME, "all primary shards are active");
@@ -129,16 +136,18 @@ public Decision canRebalance(RoutingAllocation allocation) {
             // check if there are unassigned shards.
             if (allocation.routingNodes().hasUnassignedShards() ) {
                 return allocation.decision(Decision.NO, NAME,
-                        "the cluster has unassigned shards and rebalance type is set to [%s]", type);
+                        "the cluster has unassigned shards and cluster setting [%s] is set to [%s]",
+                        CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, type);
             }
             // in case all indices are assigned, are there initializing shards which
             // are not relocating?
             if ( allocation.routingNodes().hasInactiveShards() ) {
                 return allocation.decision(Decision.NO, NAME,
-                        "the cluster has inactive shards and rebalance type is set to [%s]", type);
+                        "the cluster has inactive shards and cluster setting [%s] is set to [%s]",
+                        CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, type);
             }
         }
         // type == Type.ALWAYS
-        return allocation.decision(Decision.YES, NAME, "all shards are active, rebalance type is [%s]", type);
+        return allocation.decision(Decision.YES, NAME, "all shards are active");
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ConcurrentRebalanceAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ConcurrentRebalanceAllocationDecider.java
index dd3ece10dd5f1..63fbad59b922a 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ConcurrentRebalanceAllocationDecider.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ConcurrentRebalanceAllocationDecider.java
@@ -66,9 +66,11 @@ public Decision canRebalance(ShardRouting shardRouting, RoutingAllocation alloca
         }
         int relocatingShards = allocation.routingNodes().getRelocatingShardCount();
         if (relocatingShards >= clusterConcurrentRebalance) {
-            return allocation.decision(Decision.NO, NAME,
-                    "too many shards are concurrently rebalancing [%d], limit: [%d]",
-                    relocatingShards, clusterConcurrentRebalance);
+            return allocation.decision(Decision.THROTTLE, NAME,
+                    "reached the limit of concurrently rebalancing shards [%d], cluster setting [%s=%d]",
+                    relocatingShards,
+                    CLUSTER_ROUTING_ALLOCATION_CLUSTER_CONCURRENT_REBALANCE_SETTING.getKey(),
+                    clusterConcurrentRebalance);
         }
         return allocation.decision(Decision.YES, NAME,
                 "below threshold [%d] for concurrent rebalances, current rebalance shard count [%d]",
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/Decision.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/Decision.java
index 3792f536f2f1d..a2198ad90d9b0 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/Decision.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/Decision.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
@@ -38,7 +39,7 @@
  *
  * @see AllocationDecider
  */
-public abstract class Decision implements ToXContent {
+public abstract class Decision implements ToXContent, Writeable {
 
     public static final Decision ALWAYS = new Single(Type.YES);
     public static final Decision YES = new Single(Type.YES);
@@ -57,26 +58,6 @@ public static Decision single(Type type, @Nullable String label, @Nullable Strin
         return new Single(type, label, explanation, explanationParams);
     }
 
-    public static void writeTo(Decision decision, StreamOutput out) throws IOException {
-        if (decision instanceof Multi) {
-            // Flag specifying whether it is a Multi or Single Decision
-            out.writeBoolean(true);
-            out.writeVInt(((Multi) decision).decisions.size());
-            for (Decision d : ((Multi) decision).decisions) {
-                writeTo(d, out);
-            }
-        } else {
-            // Flag specifying whether it is a Multi or Single Decision
-            out.writeBoolean(false);
-            Single d = ((Single) decision);
-            Type.writeTo(d.type, out);
-            out.writeOptionalString(d.label);
-            // Flatten explanation on serialization, so that explanationParams
-            // do not need to be serialized
-            out.writeOptionalString(d.getExplanation());
-        }
-    }
-
     public static Decision readFrom(StreamInput in) throws IOException {
         // Determine whether to read a Single or Multi Decision
         if (in.readBoolean()) {
@@ -100,10 +81,16 @@ public static Decision readFrom(StreamInput in) throws IOException {
      * This enumeration defines the
      * possible types of decisions
      */
-    public enum Type {
-        YES,
-        NO,
-        THROTTLE;
+    public enum Type implements Writeable {
+        YES(1),
+        THROTTLE(2),
+        NO(0);
+
+        private final int id;
+
+        Type(int id) {
+            this.id = id;
+        }
 
         public static Type resolve(String s) {
             return Type.valueOf(s.toUpperCase(Locale.ROOT));
@@ -123,21 +110,22 @@ public static Type readFrom(StreamInput in) throws IOException {
             }
         }
 
-        public static void writeTo(Type type, StreamOutput out) throws IOException {
-            switch (type) {
-                case NO:
-                    out.writeVInt(0);
-                    break;
-                case YES:
-                    out.writeVInt(1);
-                    break;
-                case THROTTLE:
-                    out.writeVInt(2);
-                    break;
-                default:
-                    throw new IllegalArgumentException("Invalid Type [" + type + "]");
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeVInt(id);
+        }
+
+        public boolean higherThan(Type other) {
+            if (this == NO) {
+                return false;
+            } else if (other == NO) {
+                return true;
+            } else if (other == THROTTLE && this == YES) {
+                return true;
             }
+            return false;
         }
+
     }
 
     /**
@@ -152,6 +140,12 @@ public static void writeTo(Type type, StreamOutput out) throws IOException {
     @Nullable
     public abstract String label();
 
+    /**
+     * Get the explanation for this decision.
+     */
+    @Nullable
+    public abstract String getExplanation();
+
     /**
      * Return the list of all decisions that make up this decision
      */
@@ -210,8 +204,9 @@ public List<Decision> getDecisions() {
         }
 
         /**
-         * Returns the explanation string, fully formatted. Only formats the string once
+         * Returns the explanation string, fully formatted.  Only formats the string once.
          */
+        @Override
         @Nullable
         public String getExplanation() {
             if (explanationString == null && explanation != null) {
@@ -263,6 +258,16 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.endObject();
             return builder;
         }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeBoolean(false); // flag specifying its a single decision
+            type.writeTo(out);
+            out.writeOptionalString(label);
+            // Flatten explanation on serialization, so that explanationParams
+            // do not need to be serialized
+            out.writeOptionalString(getExplanation());
+        }
     }
 
     /**
@@ -303,6 +308,12 @@ public String label() {
             return null;
         }
 
+        @Override
+        @Nullable
+        public String getExplanation() {
+            throw new UnsupportedOperationException("multi-level decisions do not have an explanation");
+        }
+
         @Override
         public List<Decision> getDecisions() {
             return Collections.unmodifiableList(this.decisions);
@@ -339,12 +350,19 @@ public String toString() {
 
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startArray("decisions");
             for (Decision d : decisions) {
                 d.toXContent(builder, params);
             }
-            builder.endArray();
             return builder;
         }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeBoolean(true); // flag indicating it is a multi decision
+            out.writeVInt(getDecisions().size());
+            for (Decision d : getDecisions()) {
+                d.writeTo(out);
+            }
+        }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java
index b64b74cc9cb27..56663be1ef427 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java
@@ -40,6 +40,9 @@
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
 
+import static org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING;
+import static org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING;
+
 /**
  * The {@link DiskThresholdDecider} checks that the node a shard is potentially
  * being allocated to has enough disk space.
@@ -128,16 +131,19 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
             shardRouting.active() == false && shardRouting.recoverySource().getType() == RecoverySource.Type.EMPTY_STORE;
 
         // checks for exact byte comparisons
-        if (freeBytes < diskThresholdSettings.getFreeBytesThresholdLow().bytes()) {
+        if (freeBytes < diskThresholdSettings.getFreeBytesThresholdLow().getBytes()) {
             if (skipLowTresholdChecks == false) {
                 if (logger.isDebugEnabled()) {
                     logger.debug("less than the required {} free bytes threshold ({} bytes free) on node {}, preventing allocation",
                             diskThresholdSettings.getFreeBytesThresholdLow(), freeBytes, node.nodeId());
                 }
                 return allocation.decision(Decision.NO, NAME,
-                        "the node is above the low watermark and has less than required [%s] free, free: [%s]",
-                        diskThresholdSettings.getFreeBytesThresholdLow(), new ByteSizeValue(freeBytes));
-            } else if (freeBytes > diskThresholdSettings.getFreeBytesThresholdHigh().bytes()) {
+                    "the node is above the low watermark cluster setting [%s=%s], having less than the minimum required [%s] free " +
+                    "space, actual free: [%s]",
+                    CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(),
+                    diskThresholdSettings.getLowWatermarkRaw(),
+                    diskThresholdSettings.getFreeBytesThresholdLow(), new ByteSizeValue(freeBytes));
+            } else if (freeBytes > diskThresholdSettings.getFreeBytesThresholdHigh().getBytes()) {
                 // Allow the shard to be allocated because it is primary that
                 // has never been allocated if it's under the high watermark
                 if (logger.isDebugEnabled()) {
@@ -146,7 +152,8 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
                             diskThresholdSettings.getFreeBytesThresholdLow(), freeBytes, node.nodeId());
                 }
                 return allocation.decision(Decision.YES, NAME,
-                        "the node is above the low watermark, but this primary shard has never been allocated before");
+                        "the node is above the low watermark, but less than the high watermark, and this primary shard has " +
+                        "never been allocated before");
             } else {
                 // Even though the primary has never been allocated, the node is
                 // above the high watermark, so don't allow allocating the shard
@@ -156,9 +163,11 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
                             diskThresholdSettings.getFreeBytesThresholdHigh(), freeBytes, node.nodeId());
                 }
                 return allocation.decision(Decision.NO, NAME,
-                        "the node is above the high watermark even though this shard has never been allocated " +
-                                "and has less than required [%s] free on node, free: [%s]",
-                        diskThresholdSettings.getFreeBytesThresholdHigh(), new ByteSizeValue(freeBytes));
+                    "the node is above the high watermark cluster setting [%s=%s], having less than the minimum required [%s] free " +
+                    "space, actual free: [%s]",
+                    CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(),
+                    diskThresholdSettings.getHighWatermarkRaw(),
+                    diskThresholdSettings.getFreeBytesThresholdHigh(), new ByteSizeValue(freeBytes));
             }
         }
 
@@ -172,8 +181,10 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
                             Strings.format1Decimals(usedDiskPercentage, "%"), node.nodeId());
                 }
                 return allocation.decision(Decision.NO, NAME,
-                        "the node is above the low watermark and has more than allowed [%s%%] used disk, free: [%s%%]",
-                        usedDiskThresholdLow, freeDiskPercentage);
+                    "the node is above the low watermark cluster setting [%s=%s], using more disk space than the maximum allowed " +
+                    "[%s%%], actual free: [%s%%]",
+                    CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(),
+                    diskThresholdSettings.getLowWatermarkRaw(), usedDiskThresholdLow, freeDiskPercentage);
             } else if (freeDiskPercentage > diskThresholdSettings.getFreeDiskThresholdHigh()) {
                 // Allow the shard to be allocated because it is primary that
                 // has never been allocated if it's under the high watermark
@@ -184,7 +195,8 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
                             Strings.format1Decimals(usedDiskPercentage, "%"), node.nodeId());
                 }
                 return allocation.decision(Decision.YES, NAME,
-                        "the node is above the low watermark, but this primary shard has never been allocated before");
+                    "the node is above the low watermark, but less than the high watermark, and this primary shard has " +
+                    "never been allocated before");
             } else {
                 // Even though the primary has never been allocated, the node is
                 // above the high watermark, so don't allow allocating the shard
@@ -195,9 +207,10 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
                             Strings.format1Decimals(freeDiskPercentage, "%"), node.nodeId());
                 }
                 return allocation.decision(Decision.NO, NAME,
-                        "the node is above the high watermark even though this shard has never been allocated " +
-                                "and has more than allowed [%s%%] used disk, free: [%s%%]",
-                        usedDiskThresholdHigh, freeDiskPercentage);
+                    "the node is above the high watermark cluster setting [%s=%s], using more disk space than the maximum allowed " +
+                    "[%s%%], actual free: [%s%%]",
+                    CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(),
+                    diskThresholdSettings.getHighWatermarkRaw(), usedDiskThresholdHigh, freeDiskPercentage);
             }
         }
 
@@ -205,14 +218,16 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
         final long shardSize = getExpectedShardSize(shardRouting, allocation, 0);
         double freeSpaceAfterShard = freeDiskPercentageAfterShardAssigned(usage, shardSize);
         long freeBytesAfterShard = freeBytes - shardSize;
-        if (freeBytesAfterShard < diskThresholdSettings.getFreeBytesThresholdHigh().bytes()) {
+        if (freeBytesAfterShard < diskThresholdSettings.getFreeBytesThresholdHigh().getBytes()) {
             logger.warn("after allocating, node [{}] would have less than the required " +
                     "{} free bytes threshold ({} bytes free), preventing allocation",
                     node.nodeId(), diskThresholdSettings.getFreeBytesThresholdHigh(), freeBytesAfterShard);
             return allocation.decision(Decision.NO, NAME,
-                    "after allocating the shard to this node, it would be above the high watermark " +
-                            "and have less than required [%s] free, free: [%s]",
-                    diskThresholdSettings.getFreeBytesThresholdLow(), new ByteSizeValue(freeBytesAfterShard));
+                "allocating the shard to this node will bring the node above the high watermark cluster setting [%s=%s] " +
+                    "and cause it to have less than the minimum required [%s] of free space (free bytes after shard added: [%s])",
+                CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(),
+                diskThresholdSettings.getHighWatermarkRaw(),
+                diskThresholdSettings.getFreeBytesThresholdHigh(), new ByteSizeValue(freeBytesAfterShard));
         }
         if (freeSpaceAfterShard < diskThresholdSettings.getFreeDiskThresholdHigh()) {
             logger.warn("after allocating, node [{}] would have more than the allowed " +
@@ -220,9 +235,10 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
                     node.nodeId(), Strings.format1Decimals(diskThresholdSettings.getFreeDiskThresholdHigh(), "%"),
                                                            Strings.format1Decimals(freeSpaceAfterShard, "%"));
             return allocation.decision(Decision.NO, NAME,
-                    "after allocating the shard to this node, it would be above the high watermark " +
-                            "and have more than allowed [%s%%] used disk, free: [%s%%]",
-                    usedDiskThresholdLow, freeSpaceAfterShard);
+                "allocating the shard to this node will bring the node above the high watermark cluster setting [%s=%s] " +
+                    "and cause it to use more disk space than the maximum allowed [%s%%] (free space after shard added: [%s%%])",
+                CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(),
+                diskThresholdSettings.getHighWatermarkRaw(), usedDiskThresholdHigh, freeSpaceAfterShard);
         }
 
         return allocation.decision(Decision.YES, NAME,
@@ -258,15 +274,17 @@ public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAl
             return allocation.decision(Decision.YES, NAME,
                     "this shard is not allocated on the most utilized disk and can remain");
         }
-        if (freeBytes < diskThresholdSettings.getFreeBytesThresholdHigh().bytes()) {
+        if (freeBytes < diskThresholdSettings.getFreeBytesThresholdHigh().getBytes()) {
             if (logger.isDebugEnabled()) {
                 logger.debug("less than the required {} free bytes threshold ({} bytes free) on node {}, shard cannot remain",
                         diskThresholdSettings.getFreeBytesThresholdHigh(), freeBytes, node.nodeId());
             }
             return allocation.decision(Decision.NO, NAME,
-                    "after allocating this shard this node would be above the high watermark " +
-                            "and there would be less than required [%s] free on node, free: [%s]",
-                    diskThresholdSettings.getFreeBytesThresholdHigh(), new ByteSizeValue(freeBytes));
+                "the shard cannot remain on this node because it is above the high watermark cluster setting [%s=%s] " +
+                    "and there is less than the required [%s] free space on node, actual free: [%s]",
+                CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(),
+                diskThresholdSettings.getHighWatermarkRaw(),
+                diskThresholdSettings.getFreeBytesThresholdHigh(), new ByteSizeValue(freeBytes));
         }
         if (freeDiskPercentage < diskThresholdSettings.getFreeDiskThresholdHigh()) {
             if (logger.isDebugEnabled()) {
@@ -274,9 +292,11 @@ public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAl
                         diskThresholdSettings.getFreeDiskThresholdHigh(), freeDiskPercentage, node.nodeId());
             }
             return allocation.decision(Decision.NO, NAME,
-                    "after allocating this shard this node would be above the high watermark " +
-                            "and there would be less than required [%s%%] free disk on node, free: [%s%%]",
-                    diskThresholdSettings.getFreeDiskThresholdHigh(), freeDiskPercentage);
+                "the shard cannot remain on this node because it is above the high watermark cluster setting [%s=%s] " +
+                    "and there is less than the required [%s%%] free disk on node, actual free: [%s%%]",
+                CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(),
+                diskThresholdSettings.getHighWatermarkRaw(),
+                diskThresholdSettings.getFreeDiskThresholdHigh(), freeDiskPercentage);
         }
 
         return allocation.decision(Decision.YES, NAME,
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationDecider.java
index 64bf594214252..7bb073a4c4561 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationDecider.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationDecider.java
@@ -98,33 +98,39 @@ public void setEnableAllocation(Allocation enableAllocation) {
     @Override
     public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
         if (allocation.ignoreDisable()) {
-            return allocation.decision(Decision.YES, NAME, "allocation is explicitly ignoring any disabling of allocation");
+            return allocation.decision(Decision.YES, NAME,
+                "explicitly ignoring any disabling of allocation due to manual allocation commands via the reroute API");
         }
 
         final IndexMetaData indexMetaData = allocation.metaData().getIndexSafe(shardRouting.index());
         final Allocation enable;
+        final boolean usedIndexSetting;
         if (INDEX_ROUTING_ALLOCATION_ENABLE_SETTING.exists(indexMetaData.getSettings())) {
             enable = INDEX_ROUTING_ALLOCATION_ENABLE_SETTING.get(indexMetaData.getSettings());
+            usedIndexSetting = true;
         } else {
             enable = this.enableAllocation;
+            usedIndexSetting = false;
         }
         switch (enable) {
             case ALL:
                 return allocation.decision(Decision.YES, NAME, "all allocations are allowed");
             case NONE:
-                return allocation.decision(Decision.NO, NAME, "no allocations are allowed");
+                return allocation.decision(Decision.NO, NAME, "no allocations are allowed due to {}", setting(enable, usedIndexSetting));
             case NEW_PRIMARIES:
                 if (shardRouting.primary() && shardRouting.active() == false &&
                     shardRouting.recoverySource().getType() != RecoverySource.Type.EXISTING_STORE) {
                     return allocation.decision(Decision.YES, NAME, "new primary allocations are allowed");
                 } else {
-                    return allocation.decision(Decision.NO, NAME, "non-new primary allocations are forbidden");
+                    return allocation.decision(Decision.NO, NAME, "non-new primary allocations are forbidden due to {}",
+                                                setting(enable, usedIndexSetting));
                 }
             case PRIMARIES:
                 if (shardRouting.primary()) {
                     return allocation.decision(Decision.YES, NAME, "primary allocations are allowed");
                 } else {
-                    return allocation.decision(Decision.NO, NAME, "replica allocations are forbidden");
+                    return allocation.decision(Decision.NO, NAME, "replica allocations are forbidden due to {}",
+                                                setting(enable, usedIndexSetting));
                 }
             default:
                 throw new IllegalStateException("Unknown allocation option");
@@ -139,33 +145,64 @@ public Decision canRebalance(ShardRouting shardRouting, RoutingAllocation alloca
 
         Settings indexSettings = allocation.metaData().getIndexSafe(shardRouting.index()).getSettings();
         final Rebalance enable;
+        final boolean usedIndexSetting;
         if (INDEX_ROUTING_REBALANCE_ENABLE_SETTING.exists(indexSettings)) {
             enable = INDEX_ROUTING_REBALANCE_ENABLE_SETTING.get(indexSettings);
+            usedIndexSetting = true;
         } else {
             enable = this.enableRebalance;
+            usedIndexSetting = false;
         }
         switch (enable) {
             case ALL:
                 return allocation.decision(Decision.YES, NAME, "all rebalancing is allowed");
             case NONE:
-                return allocation.decision(Decision.NO, NAME, "no rebalancing is allowed");
+                return allocation.decision(Decision.NO, NAME, "no rebalancing is allowed due to %s", setting(enable, usedIndexSetting));
             case PRIMARIES:
                 if (shardRouting.primary()) {
                     return allocation.decision(Decision.YES, NAME, "primary rebalancing is allowed");
                 } else {
-                    return allocation.decision(Decision.NO, NAME, "replica rebalancing is forbidden");
+                    return allocation.decision(Decision.NO, NAME, "replica rebalancing is forbidden due to %s",
+                                                setting(enable, usedIndexSetting));
                 }
             case REPLICAS:
                 if (shardRouting.primary() == false) {
                     return allocation.decision(Decision.YES, NAME, "replica rebalancing is allowed");
                 } else {
-                    return allocation.decision(Decision.NO, NAME, "primary rebalancing is forbidden");
+                    return allocation.decision(Decision.NO, NAME, "primary rebalancing is forbidden due to %s",
+                                                setting(enable, usedIndexSetting));
                 }
             default:
                 throw new IllegalStateException("Unknown rebalance option");
         }
     }
 
+    private static String setting(Allocation allocation, boolean usedIndexSetting) {
+        StringBuilder buf = new StringBuilder();
+        if (usedIndexSetting) {
+            buf.append("index setting [");
+            buf.append(INDEX_ROUTING_ALLOCATION_ENABLE_SETTING.getKey());
+        } else {
+            buf.append("cluster setting [");
+            buf.append(CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey());
+        }
+        buf.append("=").append(allocation.toString().toLowerCase(Locale.ROOT)).append("]");
+        return buf.toString();
+    }
+
+    private static String setting(Rebalance rebalance, boolean usedIndexSetting) {
+        StringBuilder buf = new StringBuilder();
+        if (usedIndexSetting) {
+            buf.append("index setting [");
+            buf.append(INDEX_ROUTING_REBALANCE_ENABLE_SETTING.getKey());
+        } else {
+            buf.append("cluster setting [");
+            buf.append(CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey());
+        }
+        buf.append("=").append(rebalance.toString().toLowerCase(Locale.ROOT)).append("]");
+        return buf.toString();
+    }
+
     /**
      * Allocation values or rather their string representation to be used used with
      * {@link EnableAllocationDecider#CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING} /
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDecider.java
index 4071007c79123..933b0a829d569 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDecider.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDecider.java
@@ -30,6 +30,9 @@
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 
+import java.util.EnumSet;
+
+import static org.elasticsearch.cluster.node.DiscoveryNodeFilters.IP_VALIDATOR;
 import static org.elasticsearch.cluster.node.DiscoveryNodeFilters.OpType.AND;
 import static org.elasticsearch.cluster.node.DiscoveryNodeFilters.OpType.OR;
 
@@ -64,12 +67,26 @@ public class FilterAllocationDecider extends AllocationDecider {
 
     public static final String NAME = "filter";
 
+    private static final String CLUSTER_ROUTING_REQUIRE_GROUP_PREFIX = "cluster.routing.allocation.require";
+    private static final String CLUSTER_ROUTING_INCLUDE_GROUP_PREFIX = "cluster.routing.allocation.include";
+    private static final String CLUSTER_ROUTING_EXCLUDE_GROUP_PREFIX = "cluster.routing.allocation.exclude";
     public static final Setting<Settings> CLUSTER_ROUTING_REQUIRE_GROUP_SETTING =
-        Setting.groupSetting("cluster.routing.allocation.require.", Property.Dynamic, Property.NodeScope);
+        Setting.groupSetting(CLUSTER_ROUTING_REQUIRE_GROUP_PREFIX + ".", IP_VALIDATOR, Property.Dynamic, Property.NodeScope);
     public static final Setting<Settings> CLUSTER_ROUTING_INCLUDE_GROUP_SETTING =
-        Setting.groupSetting("cluster.routing.allocation.include.", Property.Dynamic, Property.NodeScope);
+        Setting.groupSetting(CLUSTER_ROUTING_INCLUDE_GROUP_PREFIX + ".", IP_VALIDATOR, Property.Dynamic, Property.NodeScope);
     public static final Setting<Settings> CLUSTER_ROUTING_EXCLUDE_GROUP_SETTING =
-        Setting.groupSetting("cluster.routing.allocation.exclude.", Property.Dynamic, Property.NodeScope);
+        Setting.groupSetting(CLUSTER_ROUTING_EXCLUDE_GROUP_PREFIX + ".", IP_VALIDATOR, Property.Dynamic, Property.NodeScope);
+
+    /**
+     * The set of {@link RecoverySource.Type} values for which the
+     * {@link IndexMetaData#INDEX_ROUTING_INITIAL_RECOVERY_GROUP_SETTING} should apply.
+     * Note that we do not include the {@link RecoverySource.Type#SNAPSHOT} type here
+     * because if the snapshot is restored to a different cluster that does not contain
+     * the initial recovery node id, or to the same cluster where the initial recovery node
+     * id has been decommissioned, then the primary shards will never be allocated.
+     */
+    static EnumSet<RecoverySource.Type> INITIAL_RECOVERY_TYPES =
+        EnumSet.of(RecoverySource.Type.EMPTY_STORE, RecoverySource.Type.LOCAL_SHARDS);
 
     private volatile DiscoveryNodeFilters clusterRequireFilters;
     private volatile DiscoveryNodeFilters clusterIncludeFilters;
@@ -93,11 +110,13 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
             // this is a setting that can only be set within the system!
             IndexMetaData indexMd = allocation.metaData().getIndexSafe(shardRouting.index());
             DiscoveryNodeFilters initialRecoveryFilters = indexMd.getInitialRecoveryFilters();
-            if (shardRouting.recoverySource().getType() != RecoverySource.Type.EXISTING_STORE &&
-                initialRecoveryFilters != null &&
+            if (initialRecoveryFilters != null  &&
+                INITIAL_RECOVERY_TYPES.contains(shardRouting.recoverySource().getType()) &&
                 initialRecoveryFilters.match(node.node()) == false) {
-                return allocation.decision(Decision.NO, NAME, "node does not match index initial recovery filters [%s]",
-                    indexMd.includeFilters());
+                String explanation = (shardRouting.recoverySource().getType() == RecoverySource.Type.LOCAL_SHARDS) ?
+                    "initial allocation of the shrunken index is only allowed on nodes [%s] that hold a copy of every shard in the index" :
+                    "initial allocation of the index is only allowed on nodes [%s]";
+                return allocation.decision(Decision.NO, NAME, explanation, initialRecoveryFilters);
             }
         }
         return shouldFilter(shardRouting, node, allocation);
@@ -136,17 +155,20 @@ private Decision shouldFilter(IndexMetaData indexMd, RoutingNode node, RoutingAl
     private Decision shouldIndexFilter(IndexMetaData indexMd, RoutingNode node, RoutingAllocation allocation) {
         if (indexMd.requireFilters() != null) {
             if (!indexMd.requireFilters().match(node.node())) {
-                return allocation.decision(Decision.NO, NAME, "node does not match index required filters [%s]", indexMd.requireFilters());
+                return allocation.decision(Decision.NO, NAME, "node does not match index setting [%s] filters [%s]",
+                    IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_PREFIX, indexMd.requireFilters());
             }
         }
         if (indexMd.includeFilters() != null) {
             if (!indexMd.includeFilters().match(node.node())) {
-                return allocation.decision(Decision.NO, NAME, "node does not match index include filters [%s]", indexMd.includeFilters());
+                return allocation.decision(Decision.NO, NAME, "node does not match index setting [%s] filters [%s]",
+                    IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_PREFIX, indexMd.includeFilters());
             }
         }
         if (indexMd.excludeFilters() != null) {
             if (indexMd.excludeFilters().match(node.node())) {
-                return allocation.decision(Decision.NO, NAME, "node matches index exclude filters [%s]", indexMd.excludeFilters());
+                return allocation.decision(Decision.NO, NAME, "node matches index setting [%s] filters [%s]",
+                    IndexMetaData.INDEX_ROUTING_EXCLUDE_GROUP_SETTING.getKey(), indexMd.excludeFilters());
             }
         }
         return null;
@@ -155,17 +177,20 @@ private Decision shouldIndexFilter(IndexMetaData indexMd, RoutingNode node, Rout
     private Decision shouldClusterFilter(RoutingNode node, RoutingAllocation allocation) {
         if (clusterRequireFilters != null) {
             if (!clusterRequireFilters.match(node.node())) {
-                return allocation.decision(Decision.NO, NAME, "node does not match global required filters [%s]", clusterRequireFilters);
+                return allocation.decision(Decision.NO, NAME, "node does not match cluster setting [%s] filters [%s]",
+                    CLUSTER_ROUTING_REQUIRE_GROUP_PREFIX, clusterRequireFilters);
             }
         }
         if (clusterIncludeFilters != null) {
             if (!clusterIncludeFilters.match(node.node())) {
-                return allocation.decision(Decision.NO, NAME, "node does not match global include filters [%s]", clusterIncludeFilters);
+                return allocation.decision(Decision.NO, NAME, "node does not cluster setting [%s] filters [%s]",
+                    CLUSTER_ROUTING_INCLUDE_GROUP_PREFIX, clusterIncludeFilters);
             }
         }
         if (clusterExcludeFilters != null) {
             if (clusterExcludeFilters.match(node.node())) {
-                return allocation.decision(Decision.NO, NAME, "node matches global exclude filters [%s]", clusterExcludeFilters);
+                return allocation.decision(Decision.NO, NAME, "node matches cluster setting [%s] filters [%s]",
+                    CLUSTER_ROUTING_EXCLUDE_GROUP_PREFIX, clusterExcludeFilters);
             }
         }
         return null;
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/MaxRetryAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/MaxRetryAllocationDecider.java
index 395d347232939..59a836abece37 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/MaxRetryAllocationDecider.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/MaxRetryAllocationDecider.java
@@ -54,7 +54,8 @@ public MaxRetryAllocationDecider(Settings settings) {
 
     @Override
     public Decision canAllocate(ShardRouting shardRouting, RoutingAllocation allocation) {
-        UnassignedInfo unassignedInfo = shardRouting.unassignedInfo();
+        final UnassignedInfo unassignedInfo = shardRouting.unassignedInfo();
+        final Decision decision;
         if (unassignedInfo != null && unassignedInfo.getNumFailedAllocations() > 0) {
             final IndexMetaData indexMetaData = allocation.metaData().getIndexSafe(shardRouting.index());
             final int maxRetry = SETTING_ALLOCATION_MAX_RETRY.get(indexMetaData.getSettings());
@@ -62,16 +63,21 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingAllocation allocat
                 // if we are called via the _reroute API we ignore the failure counter and try to allocate
                 // this improves the usability since people don't need to raise the limits to issue retries since a simple _reroute call is
                 // enough to manually retry.
-                return allocation.decision(Decision.YES, NAME, "shard has already failed allocating ["
-                    + unassignedInfo.getNumFailedAllocations() + "] times vs. [" + maxRetry + "] retries allowed "
-                    + unassignedInfo.toString() + " - retrying once on manual allocation");
+                decision = allocation.decision(Decision.YES, NAME, "shard has exceeded the maximum number of retries [%d] on " +
+                    "failed allocation attempts - retrying once due to a manual reroute command, [%s]",
+                    maxRetry, unassignedInfo.toString());
             } else if (unassignedInfo.getNumFailedAllocations() >= maxRetry) {
-                return allocation.decision(Decision.NO, NAME, "shard has already failed allocating ["
-                    + unassignedInfo.getNumFailedAllocations() + "] times vs. [" + maxRetry + "] retries allowed "
-                    + unassignedInfo.toString() + " - manually call [/_cluster/reroute?retry_failed=true] to retry");
+                decision = allocation.decision(Decision.NO, NAME, "shard has exceeded the maximum number of retries [%d] on " +
+                    "failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [%s]",
+                    maxRetry, unassignedInfo.toString());
+            } else {
+                decision = allocation.decision(Decision.YES, NAME, "shard has failed allocating [%d] times but [%d] retries are allowed",
+                    unassignedInfo.getNumFailedAllocations(), maxRetry);
             }
+        } else {
+            decision = allocation.decision(Decision.YES, NAME, "shard has no previous failures");
         }
-        return allocation.decision(Decision.YES, NAME, "shard has no previous failures");
+        return decision;
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/RebalanceOnlyWhenActiveAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/RebalanceOnlyWhenActiveAllocationDecider.java
index d8042f18a2731..c4cd2ecf50dda 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/RebalanceOnlyWhenActiveAllocationDecider.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/RebalanceOnlyWhenActiveAllocationDecider.java
@@ -37,8 +37,8 @@ public RebalanceOnlyWhenActiveAllocationDecider(Settings settings) {
     @Override
     public Decision canRebalance(ShardRouting shardRouting, RoutingAllocation allocation) {
         if (!allocation.routingNodes().allReplicasActive(shardRouting.shardId(), allocation.metaData())) {
-            return allocation.decision(Decision.NO, NAME, "rebalancing can not occur if not all replicas are active in the cluster");
+            return allocation.decision(Decision.NO, NAME, "rebalancing is not allowed until all replicas in the cluster are active");
         }
-        return allocation.decision(Decision.YES, NAME, "all replicas are active in the cluster, rebalancing can occur");
+        return allocation.decision(Decision.YES, NAME, "rebalancing is allowed as all replicas are active in the cluster");
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SameShardAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SameShardAllocationDecider.java
index 3f2921dfcdcf4..1bf776422521c 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SameShardAllocationDecider.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SameShardAllocationDecider.java
@@ -23,7 +23,9 @@
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 
 /**
@@ -46,53 +48,90 @@ public class SameShardAllocationDecider extends AllocationDecider {
     public static final String NAME = "same_shard";
 
     public static final Setting<Boolean> CLUSTER_ROUTING_ALLOCATION_SAME_HOST_SETTING =
-        Setting.boolSetting("cluster.routing.allocation.same_shard.host", false, Setting.Property.NodeScope);
+        Setting.boolSetting("cluster.routing.allocation.same_shard.host", false, Property.Dynamic, Property.NodeScope);
 
-    private final boolean sameHost;
+    private volatile boolean sameHost;
 
-    public SameShardAllocationDecider(Settings settings) {
+    public SameShardAllocationDecider(Settings settings, ClusterSettings clusterSettings) {
         super(settings);
-
         this.sameHost = CLUSTER_ROUTING_ALLOCATION_SAME_HOST_SETTING.get(settings);
+        clusterSettings.addSettingsUpdateConsumer(CLUSTER_ROUTING_ALLOCATION_SAME_HOST_SETTING, this::setSameHost);
+    }
+
+    /**
+     * Sets the same host setting.  {@code true} if allocating the same shard copy to the same host
+     * should not be allowed, even when multiple nodes are being run on the same host.  {@code false}
+     * otherwise.
+     */
+    private void setSameHost(boolean sameHost) {
+        this.sameHost = sameHost;
     }
 
     @Override
     public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
         Iterable<ShardRouting> assignedShards = allocation.routingNodes().assignedShards(shardRouting.shardId());
-        for (ShardRouting assignedShard : assignedShards) {
-            if (node.nodeId().equals(assignedShard.currentNodeId())) {
-                return allocation.decision(Decision.NO, NAME,
-                        "the shard cannot be allocated on the same node id [%s] on which it already exists", node.nodeId());
-            }
+        Decision decision = decideSameNode(shardRouting, node, allocation, assignedShards);
+        if (decision.type() == Decision.Type.NO || sameHost == false) {
+            // if its already a NO decision looking at the node, or we aren't configured to look at the host, return the decision
+            return decision;
         }
-        if (sameHost) {
-            if (node.node() != null) {
-                for (RoutingNode checkNode : allocation.routingNodes()) {
-                    if (checkNode.node() == null) {
-                        continue;
+        if (node.node() != null) {
+            for (RoutingNode checkNode : allocation.routingNodes()) {
+                if (checkNode.node() == null) {
+                    continue;
+                }
+                // check if its on the same host as the one we want to allocate to
+                boolean checkNodeOnSameHostName = false;
+                boolean checkNodeOnSameHostAddress = false;
+                if (Strings.hasLength(checkNode.node().getHostAddress()) && Strings.hasLength(node.node().getHostAddress())) {
+                    if (checkNode.node().getHostAddress().equals(node.node().getHostAddress())) {
+                        checkNodeOnSameHostAddress = true;
                     }
-                    // check if its on the same host as the one we want to allocate to
-                    boolean checkNodeOnSameHost = false;
-                    if (Strings.hasLength(checkNode.node().getHostAddress()) && Strings.hasLength(node.node().getHostAddress())) {
-                        if (checkNode.node().getHostAddress().equals(node.node().getHostAddress())) {
-                            checkNodeOnSameHost = true;
-                        }
-                    } else if (Strings.hasLength(checkNode.node().getHostName()) && Strings.hasLength(node.node().getHostName())) {
-                        if (checkNode.node().getHostName().equals(node.node().getHostName())) {
-                            checkNodeOnSameHost = true;
-                        }
+                } else if (Strings.hasLength(checkNode.node().getHostName()) && Strings.hasLength(node.node().getHostName())) {
+                    if (checkNode.node().getHostName().equals(node.node().getHostName())) {
+                        checkNodeOnSameHostName = true;
                     }
-                    if (checkNodeOnSameHost) {
-                        for (ShardRouting assignedShard : assignedShards) {
-                            if (checkNode.nodeId().equals(assignedShard.currentNodeId())) {
-                                return allocation.decision(Decision.NO, NAME,
-                                        "shard cannot be allocated on the same host [%s] on which it already exists", node.nodeId());
-                            }
+                }
+                if (checkNodeOnSameHostAddress || checkNodeOnSameHostName) {
+                    for (ShardRouting assignedShard : assignedShards) {
+                        if (checkNode.nodeId().equals(assignedShard.currentNodeId())) {
+                            String hostType = checkNodeOnSameHostAddress ? "address" : "name";
+                            String host = checkNodeOnSameHostAddress ? node.node().getHostAddress() : node.node().getHostName();
+                            return allocation.decision(Decision.NO, NAME,
+                                "the shard cannot be allocated on host %s [%s], where it already exists on node [%s]; " +
+                                    "set cluster setting [%s] to false to allow multiple nodes on the same host to hold the same " +
+                                    "shard copies",
+                                hostType, host, node.nodeId(), CLUSTER_ROUTING_ALLOCATION_SAME_HOST_SETTING.getKey());
                         }
                     }
                 }
             }
         }
-        return allocation.decision(Decision.YES, NAME, "shard is not allocated to same node or host");
+        return allocation.decision(Decision.YES, NAME, "the shard does not exist on the same host");
+    }
+
+    @Override
+    public Decision canForceAllocatePrimary(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
+        assert shardRouting.primary() : "must not call force allocate on a non-primary shard";
+        Iterable<ShardRouting> assignedShards = allocation.routingNodes().assignedShards(shardRouting.shardId());
+        return decideSameNode(shardRouting, node, allocation, assignedShards);
+    }
+
+    private Decision decideSameNode(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation,
+                                    Iterable<ShardRouting> assignedShards) {
+        for (ShardRouting assignedShard : assignedShards) {
+            if (node.nodeId().equals(assignedShard.currentNodeId())) {
+                if (assignedShard.isSameAllocation(shardRouting)) {
+                    return allocation.decision(Decision.NO, NAME,
+                        "the shard cannot be allocated to the node on which it already exists [%s]",
+                        shardRouting.toString());
+                } else {
+                    return allocation.decision(Decision.NO, NAME,
+                        "the shard cannot be allocated to the same node on which a copy of the shard already exists [%s]",
+                        assignedShard.toString());
+                }
+            }
+        }
+        return allocation.decision(Decision.YES, NAME, "the shard does not exist on the same node");
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java
index aa4fe3d593df8..2118d37fe4717 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java
@@ -29,6 +29,8 @@
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 
+import java.util.function.BiPredicate;
+
 /**
  * This {@link AllocationDecider} limits the number of shards per node on a per
  * index or node-wide basis. The allocator prevents a single node to hold more
@@ -83,45 +85,17 @@ private void setClusterShardLimit(int clusterShardLimit) {
 
     @Override
     public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
-        IndexMetaData indexMd = allocation.metaData().getIndexSafe(shardRouting.index());
-        final int indexShardLimit = INDEX_TOTAL_SHARDS_PER_NODE_SETTING.get(indexMd.getSettings(), settings);
-        // Capture the limit here in case it changes during this method's
-        // execution
-        final int clusterShardLimit = this.clusterShardLimit;
-
-        if (indexShardLimit <= 0 && clusterShardLimit <= 0) {
-            return allocation.decision(Decision.YES, NAME, "total shard limits are disabled: [index: %d, cluster: %d] <= 0",
-                    indexShardLimit, clusterShardLimit);
-        }
-
-        int indexShardCount = 0;
-        int nodeShardCount = 0;
-        for (ShardRouting nodeShard : node) {
-            // don't count relocating shards...
-            if (nodeShard.relocating()) {
-                continue;
-            }
-            nodeShardCount++;
-            if (nodeShard.index().equals(shardRouting.index())) {
-                indexShardCount++;
-            }
-        }
-        if (clusterShardLimit > 0 && nodeShardCount >= clusterShardLimit) {
-            return allocation.decision(Decision.NO, NAME, "too many shards for this node [%d], cluster-level limit per node: [%d]",
-                    nodeShardCount, clusterShardLimit);
-        }
-        if (indexShardLimit > 0 && indexShardCount >= indexShardLimit) {
-            return allocation.decision(Decision.NO, NAME,
-                    "too many shards for this index [%s] on node [%d], index-level limit per node: [%d]",
-                    shardRouting.index(), indexShardCount, indexShardLimit);
-        }
-        return allocation.decision(Decision.YES, NAME,
-                "the shard count is under index limit [%d] and cluster level node limit [%d] of total shards per node",
-                indexShardLimit, clusterShardLimit);
+        return doDecide(shardRouting, node, allocation, (count, limit) -> count >= limit);
     }
 
     @Override
     public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
+        return doDecide(shardRouting, node, allocation, (count, limit) -> count > limit);
+
+    }
+
+    private Decision doDecide(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation,
+                              BiPredicate<Integer, Integer> decider) {
         IndexMetaData indexMd = allocation.metaData().getIndexSafe(shardRouting.index());
         final int indexShardLimit = INDEX_TOTAL_SHARDS_PER_NODE_SETTING.get(indexMd.getSettings(), settings);
         // Capture the limit here in case it changes during this method's
@@ -145,20 +119,20 @@ public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAl
                 indexShardCount++;
             }
         }
-        // Subtle difference between the `canAllocate` and `canRemain` is that
-        // this checks > while canAllocate checks >=
-        if (clusterShardLimit > 0 && nodeShardCount > clusterShardLimit) {
-            return allocation.decision(Decision.NO, NAME, "too many shards for this node [%d], cluster-level limit per node: [%d]",
-                    nodeShardCount, clusterShardLimit);
+
+        if (clusterShardLimit > 0 && decider.test(nodeShardCount, clusterShardLimit)) {
+            return allocation.decision(Decision.NO, NAME,
+                "too many shards [%d] allocated to this node, cluster setting [%s=%d]",
+                nodeShardCount, CLUSTER_TOTAL_SHARDS_PER_NODE_SETTING.getKey(), clusterShardLimit);
         }
-        if (indexShardLimit > 0 && indexShardCount > indexShardLimit) {
+        if (indexShardLimit > 0 && decider.test(indexShardCount, indexShardLimit)) {
             return allocation.decision(Decision.NO, NAME,
-                    "too many shards for this index [%s] on node [%d], index-level limit per node: [%d]",
-                    shardRouting.index(), indexShardCount, indexShardLimit);
+                "too many shards [%d] allocated to this node for index [%s], index setting [%s=%d]",
+                indexShardCount, shardRouting.getIndexName(), INDEX_TOTAL_SHARDS_PER_NODE_SETTING.getKey(), indexShardLimit);
         }
         return allocation.decision(Decision.YES, NAME,
-                "the shard count is under index limit [%d] and cluster level node limit [%d] of total shards per node",
-                indexShardLimit, clusterShardLimit);
+            "the shard count [%d] for this node is under the index limit [%d] and cluster level node limit [%d]",
+            nodeShardCount, indexShardLimit, clusterShardLimit);
     }
 
     @Override
@@ -182,10 +156,12 @@ public Decision canAllocate(RoutingNode node, RoutingAllocation allocation) {
             nodeShardCount++;
         }
         if (clusterShardLimit >= 0 && nodeShardCount >= clusterShardLimit) {
-            return allocation.decision(Decision.NO, NAME, "too many shards for this node [%d], cluster-level limit per node: [%d]",
-                    nodeShardCount, clusterShardLimit);
+            return allocation.decision(Decision.NO, NAME,
+                "too many shards [%d] allocated to this node, cluster setting [%s=%d]",
+                nodeShardCount, CLUSTER_TOTAL_SHARDS_PER_NODE_SETTING.getKey(), clusterShardLimit);
         }
-        return allocation.decision(Decision.YES, NAME, "the shard count is under node limit [%d] of total shards per node",
-                clusterShardLimit);
+        return allocation.decision(Decision.YES, NAME,
+            "the shard count [%d] for this node is under the cluster level node limit [%d]",
+            nodeShardCount, clusterShardLimit);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SnapshotInProgressAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SnapshotInProgressAllocationDecider.java
index bd9bf35a68ed5..d7f2c04f26d71 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SnapshotInProgressAllocationDecider.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SnapshotInProgressAllocationDecider.java
@@ -107,15 +107,16 @@ private Decision canMove(ShardRouting shardRouting, RoutingAllocation allocation
                 if (shardSnapshotStatus != null && !shardSnapshotStatus.state().completed() && shardSnapshotStatus.nodeId() != null &&
                         shardSnapshotStatus.nodeId().equals(shardRouting.currentNodeId())) {
                     if (logger.isTraceEnabled()) {
-                        logger.trace("Preventing snapshotted shard [{}] to be moved from node [{}]",
+                        logger.trace("Preventing snapshotted shard [{}] from being moved away from node [{}]",
                                 shardRouting.shardId(), shardSnapshotStatus.nodeId());
                     }
-                    return allocation.decision(Decision.NO, NAME, "snapshot for shard [%s] is currently running on node [%s]",
-                            shardRouting.shardId(), shardSnapshotStatus.nodeId());
+                    return allocation.decision(Decision.THROTTLE, NAME,
+                        "waiting for snapshotting of shard [%s] to complete on this node [%s]",
+                        shardRouting.shardId(), shardSnapshotStatus.nodeId());
                 }
             }
         }
-        return allocation.decision(Decision.YES, NAME, "the shard is not primary or relocation is disabled");
+        return allocation.decision(Decision.YES, NAME, "the shard is not being snapshotted");
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ThrottlingAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ThrottlingAllocationDecider.java
index df2e1d122347a..721de71435def 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ThrottlingAllocationDecider.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ThrottlingAllocationDecider.java
@@ -126,8 +126,10 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
             }
             if (primariesInRecovery >= primariesInitialRecoveries) {
                 // TODO: Should index creation not be throttled for primary shards?
-                return allocation.decision(THROTTLE, NAME, "too many primaries are currently recovering [%d], limit: [%d]",
-                    primariesInRecovery, primariesInitialRecoveries);
+                return allocation.decision(THROTTLE, NAME,
+                    "reached the limit of ongoing initial primary recoveries [%d], cluster setting [%s=%d]",
+                    primariesInRecovery, CLUSTER_ROUTING_ALLOCATION_NODE_INITIAL_PRIMARIES_RECOVERIES_SETTING.getKey(),
+                    primariesInitialRecoveries);
             } else {
                 return allocation.decision(YES, NAME, "below primary recovery limit of [%d]", primariesInitialRecoveries);
             }
@@ -138,8 +140,11 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
             // Allocating a shard to this node will increase the incoming recoveries
             int currentInRecoveries = allocation.routingNodes().getIncomingRecoveries(node.nodeId());
             if (currentInRecoveries >= concurrentIncomingRecoveries) {
-                return allocation.decision(THROTTLE, NAME, "too many incoming shards are currently recovering [%d], limit: [%d]",
-                    currentInRecoveries, concurrentIncomingRecoveries);
+                return allocation.decision(THROTTLE, NAME,
+                    "reached the limit of incoming shard recoveries [%d], cluster setting [%s=%d] (can also be set via [%s])",
+                    currentInRecoveries, CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_INCOMING_RECOVERIES_SETTING.getKey(),
+                    concurrentIncomingRecoveries,
+                    CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_RECOVERIES_SETTING.getKey());
             } else {
                 // search for corresponding recovery source (= primary shard) and check number of outgoing recoveries on that node
                 ShardRouting primaryShard = allocation.routingNodes().activePrimary(shardRouting.shardId());
@@ -148,8 +153,13 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
                 }
                 int primaryNodeOutRecoveries = allocation.routingNodes().getOutgoingRecoveries(primaryShard.currentNodeId());
                 if (primaryNodeOutRecoveries >= concurrentOutgoingRecoveries) {
-                    return allocation.decision(THROTTLE, NAME, "too many outgoing shards are currently recovering [%d], limit: [%d]",
-                        primaryNodeOutRecoveries, concurrentOutgoingRecoveries);
+                    return allocation.decision(THROTTLE, NAME,
+                        "reached the limit of outgoing shard recoveries [%d] on the node [%s] which holds the primary, " +
+                        "cluster setting [%s=%d] (can also be set via [%s])",
+                        primaryNodeOutRecoveries, node.nodeId(),
+                        CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_OUTGOING_RECOVERIES_SETTING.getKey(),
+                        concurrentOutgoingRecoveries,
+                        CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_RECOVERIES_SETTING.getKey());
                 } else {
                     return allocation.decision(YES, NAME, "below shard recovery limit of outgoing: [%d < %d] incoming: [%d < %d]",
                         primaryNodeOutRecoveries,
diff --git a/core/src/main/java/org/elasticsearch/cluster/service/ClusterService.java b/core/src/main/java/org/elasticsearch/cluster/service/ClusterService.java
index e981313f4024e..c1fdbc4706f30 100644
--- a/core/src/main/java/org/elasticsearch/cluster/service/ClusterService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/service/ClusterService.java
@@ -22,16 +22,19 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
+import org.elasticsearch.Assertions;
 import org.elasticsearch.cluster.AckedClusterStateTaskListener;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterState.Builder;
+import org.elasticsearch.cluster.ClusterStateApplier;
 import org.elasticsearch.cluster.ClusterStateListener;
+import org.elasticsearch.cluster.ClusterStateObserver;
 import org.elasticsearch.cluster.ClusterStateTaskConfig;
 import org.elasticsearch.cluster.ClusterStateTaskExecutor;
+import org.elasticsearch.cluster.ClusterStateTaskExecutor.ClusterTasksResult;
 import org.elasticsearch.cluster.ClusterStateTaskListener;
-import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.LocalNodeMasterListener;
 import org.elasticsearch.cluster.NodeConnectionsService;
 import org.elasticsearch.cluster.TimeoutClusterStateListener;
@@ -59,30 +62,32 @@
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.common.util.concurrent.FutureUtils;
 import org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor;
-import org.elasticsearch.common.util.concurrent.PrioritizedRunnable;
 import org.elasticsearch.common.util.iterable.Iterables;
 import org.elasticsearch.discovery.Discovery;
+import org.elasticsearch.discovery.DiscoverySettings;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.HashMap;
-import java.util.IdentityHashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Queue;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.Executor;
 import java.util.concurrent.Future;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.BiConsumer;
+import java.util.function.UnaryOperator;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import static org.elasticsearch.common.util.concurrent.EsExecutors.daemonThreadFactory;
 
@@ -95,6 +100,7 @@ public class ClusterService extends AbstractLifecycleComponent {
     public static final String UPDATE_THREAD_NAME = "clusterService#updateTask";
     private final ThreadPool threadPool;
     private final ClusterName clusterName;
+    private final Supplier<DiscoveryNode> localNodeSupplier;
 
     private BiConsumer<ClusterChangedEvent, Discovery.AckListener> clusterStatePublisher;
 
@@ -104,39 +110,44 @@ public class ClusterService extends AbstractLifecycleComponent {
 
     private TimeValue slowTaskLoggingThreshold;
 
-    private volatile PrioritizedEsThreadPoolExecutor updateTasksExecutor;
+    private volatile PrioritizedEsThreadPoolExecutor threadPoolExecutor;
+    private volatile ClusterServiceTaskBatcher taskBatcher;
 
     /**
      * Those 3 state listeners are changing infrequently - CopyOnWriteArrayList is just fine
      */
-    private final Collection<ClusterStateListener> priorityClusterStateListeners = new CopyOnWriteArrayList<>();
+    private final Collection<ClusterStateApplier> highPriorityStateAppliers = new CopyOnWriteArrayList<>();
+    private final Collection<ClusterStateApplier> normalPriorityStateAppliers = new CopyOnWriteArrayList<>();
+    private final Collection<ClusterStateApplier> lowPriorityStateAppliers = new CopyOnWriteArrayList<>();
+    private final Iterable<ClusterStateApplier> clusterStateAppliers = Iterables.concat(highPriorityStateAppliers,
+        normalPriorityStateAppliers, lowPriorityStateAppliers);
+
     private final Collection<ClusterStateListener> clusterStateListeners = new CopyOnWriteArrayList<>();
-    private final Collection<ClusterStateListener> lastClusterStateListeners = new CopyOnWriteArrayList<>();
-    private final Map<ClusterStateTaskExecutor, List<UpdateTask>> updateTasksPerExecutor = new HashMap<>();
-    // TODO this is rather frequently changing I guess a Synced Set would be better here and a dedicated remove API
-    private final Collection<ClusterStateListener> postAppliedListeners = new CopyOnWriteArrayList<>();
-    private final Iterable<ClusterStateListener> preAppliedListeners = Iterables.concat(priorityClusterStateListeners,
-            clusterStateListeners, lastClusterStateListeners);
+    private final Collection<TimeoutClusterStateListener> timeoutClusterStateListeners =
+        Collections.newSetFromMap(new ConcurrentHashMap<TimeoutClusterStateListener, Boolean>());
 
     private final LocalNodeMasterListeners localNodeMasterListeners;
 
     private final Queue<NotifyTimeout> onGoingTimeouts = ConcurrentCollections.newQueue();
 
-    private volatile ClusterState clusterState;
+    private final AtomicReference<ClusterState> state;
 
     private final ClusterBlocks.Builder initialBlocks;
 
     private NodeConnectionsService nodeConnectionsService;
 
+    private DiscoverySettings discoverySettings;
+
     public ClusterService(Settings settings,
-                          ClusterSettings clusterSettings, ThreadPool threadPool) {
+                          ClusterSettings clusterSettings, ThreadPool threadPool, Supplier<DiscoveryNode> localNodeSupplier) {
         super(settings);
+        this.localNodeSupplier = localNodeSupplier;
         this.operationRouting = new OperationRouting(settings, clusterSettings);
         this.threadPool = threadPool;
         this.clusterSettings = clusterSettings;
         this.clusterName = ClusterName.CLUSTER_NAME_SETTING.get(settings);
         // will be replaced on doStart.
-        this.clusterState = ClusterState.builder(clusterName).build();
+        this.state = new AtomicReference<>(ClusterState.builder(clusterName).build());
 
         this.clusterSettings.addSettingsUpdateConsumer(CLUSTER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING,
                 this::setSlowTaskLoggingThreshold);
@@ -156,10 +167,8 @@ public synchronized void setClusterStatePublisher(BiConsumer<ClusterChangedEvent
         clusterStatePublisher = publisher;
     }
 
-    public synchronized void setLocalNode(DiscoveryNode localNode) {
-        assert clusterState.nodes().getLocalNodeId() == null : "local node is already set";
-        DiscoveryNodes.Builder nodeBuilder = DiscoveryNodes.builder(clusterState.nodes()).add(localNode).localNodeId(localNode.getId());
-        this.clusterState = ClusterState.builder(clusterState).nodes(nodeBuilder).build();
+    private void updateState(UnaryOperator<ClusterState> updateFunction) {
+        this.state.getAndUpdate(updateFunction);
     }
 
     public synchronized void setNodeConnectionsService(NodeConnectionsService nodeConnectionsService) {
@@ -197,13 +206,19 @@ public synchronized void removeInitialStateBlock(int blockId) throws IllegalStat
     @Override
     protected synchronized void doStart() {
         Objects.requireNonNull(clusterStatePublisher, "please set a cluster state publisher before starting");
-        Objects.requireNonNull(clusterState.nodes().getLocalNode(), "please set the local node before starting");
         Objects.requireNonNull(nodeConnectionsService, "please set the node connection service before starting");
-        add(localNodeMasterListeners);
-        this.clusterState = ClusterState.builder(clusterState).blocks(initialBlocks).build();
-        this.updateTasksExecutor = EsExecutors.newSinglePrioritizing(UPDATE_THREAD_NAME, daemonThreadFactory(settings, UPDATE_THREAD_NAME),
-                threadPool.getThreadContext());
-        this.clusterState = ClusterState.builder(clusterState).blocks(initialBlocks).build();
+        Objects.requireNonNull(discoverySettings, "please set discovery settings before starting");
+        addListener(localNodeMasterListeners);
+        DiscoveryNode localNode = localNodeSupplier.get();
+        assert localNode != null;
+        updateState(state -> {
+            assert state.nodes().getLocalNodeId() == null : "local node is already set";
+            DiscoveryNodes nodes = DiscoveryNodes.builder(state.nodes()).add(localNode).localNodeId(localNode.getId()).build();
+            return ClusterState.builder(state).nodes(nodes).blocks(initialBlocks).build();
+        });
+        this.threadPoolExecutor = EsExecutors.newSinglePrioritizing(UPDATE_THREAD_NAME,
+            daemonThreadFactory(settings, UPDATE_THREAD_NAME), threadPool.getThreadContext(), threadPool.scheduler());
+        this.taskBatcher = new ClusterServiceTaskBatcher(logger, threadPoolExecutor);
     }
 
     @Override
@@ -217,25 +232,59 @@ protected synchronized void doStop() {
                 logger.debug("failed to notify listeners on shutdown", ex);
             }
         }
-        ThreadPool.terminate(updateTasksExecutor, 10, TimeUnit.SECONDS);
+        ThreadPool.terminate(threadPoolExecutor, 10, TimeUnit.SECONDS);
         // close timeout listeners that did not have an ongoing timeout
-        postAppliedListeners
-                .stream()
-                .filter(listener -> listener instanceof TimeoutClusterStateListener)
-                .map(listener -> (TimeoutClusterStateListener)listener)
-                .forEach(TimeoutClusterStateListener::onClose);
-        remove(localNodeMasterListeners);
+        timeoutClusterStateListeners.forEach(TimeoutClusterStateListener::onClose);
+        removeListener(localNodeMasterListeners);
     }
 
     @Override
     protected synchronized void doClose() {
     }
 
+    class ClusterServiceTaskBatcher extends TaskBatcher {
+
+        ClusterServiceTaskBatcher(Logger logger, PrioritizedEsThreadPoolExecutor threadExecutor) {
+            super(logger, threadExecutor);
+        }
+
+        @Override
+        protected void onTimeout(List<? extends BatchedTask> tasks, TimeValue timeout) {
+            threadPool.generic().execute(
+                () -> tasks.forEach(
+                    task -> ((UpdateTask) task).listener.onFailure(task.source,
+                        new ProcessClusterEventTimeoutException(timeout, task.source))));
+        }
+
+        @Override
+        protected void run(Object batchingKey, List<? extends BatchedTask> tasks, String tasksSummary) {
+            ClusterStateTaskExecutor<Object> taskExecutor = (ClusterStateTaskExecutor<Object>) batchingKey;
+            List<UpdateTask> updateTasks = (List<UpdateTask>) tasks;
+            runTasks(new ClusterService.TaskInputs(taskExecutor, updateTasks, tasksSummary));
+        }
+
+        class UpdateTask extends BatchedTask {
+            final ClusterStateTaskListener listener;
+
+            UpdateTask(Priority priority, String source, Object task, ClusterStateTaskListener listener,
+                       ClusterStateTaskExecutor<?> executor) {
+                super(priority, source, executor, task);
+                this.listener = listener;
+            }
+
+            @Override
+            public String describeTasks(List<? extends BatchedTask> tasks) {
+                return ((ClusterStateTaskExecutor<Object>) batchingKey).describeTasks(
+                    tasks.stream().map(BatchedTask::getTask).collect(Collectors.toList()));
+            }
+        }
+    }
+
     /**
      * The local node.
      */
     public DiscoveryNode localNode() {
-        DiscoveryNode localNode = clusterState.getNodes().getLocalNode();
+        DiscoveryNode localNode = state().getNodes().getLocalNode();
         if (localNode == null) {
             throw new IllegalStateException("No local node found. Is the node started?");
         }
@@ -247,41 +296,62 @@ public OperationRouting operationRouting() {
     }
 
     /**
-     * The current state.
+     * The current cluster state.
      */
     public ClusterState state() {
-        return this.clusterState;
+        assert assertNotCalledFromClusterStateApplier("the applied cluster state is not yet available");
+        return this.state.get();
     }
 
     /**
-     * Adds a priority listener for updated cluster states.
+     * Adds a high priority applier of updated cluster states.
      */
-    public void addFirst(ClusterStateListener listener) {
-        priorityClusterStateListeners.add(listener);
+    public void addHighPriorityApplier(ClusterStateApplier applier) {
+        highPriorityStateAppliers.add(applier);
     }
 
     /**
-     * Adds last listener.
+     * Adds an applier which will be called after all high priority and normal appliers have been called.
      */
-    public void addLast(ClusterStateListener listener) {
-        lastClusterStateListeners.add(listener);
+    public void addLowPriorityApplier(ClusterStateApplier applier) {
+        lowPriorityStateAppliers.add(applier);
     }
 
     /**
-     * Adds a listener for updated cluster states.
+     * Adds a applier of updated cluster states.
      */
-    public void add(ClusterStateListener listener) {
+    public void addStateApplier(ClusterStateApplier applier) {
+        normalPriorityStateAppliers.add(applier);
+    }
+
+    /**
+     * Removes an applier of updated cluster states.
+     */
+    public void removeApplier(ClusterStateApplier applier) {
+        normalPriorityStateAppliers.remove(applier);
+        highPriorityStateAppliers.remove(applier);
+        lowPriorityStateAppliers.remove(applier);
+    }
+
+    /**
+     * Add a listener for updated cluster states
+     */
+    public void addListener(ClusterStateListener listener) {
         clusterStateListeners.add(listener);
     }
 
     /**
      * Removes a listener for updated cluster states.
      */
-    public void remove(ClusterStateListener listener) {
+    public void removeListener(ClusterStateListener listener) {
         clusterStateListeners.remove(listener);
-        priorityClusterStateListeners.remove(listener);
-        lastClusterStateListeners.remove(listener);
-        postAppliedListeners.remove(listener);
+    }
+
+    /**
+     * Removes a timeout listener for updated cluster states.
+     */
+    public void removeTimeoutListener(TimeoutClusterStateListener listener) {
+        timeoutClusterStateListeners.remove(listener);
         for (Iterator<NotifyTimeout> it = onGoingTimeouts.iterator(); it.hasNext(); ) {
             NotifyTimeout timeout = it.next();
             if (timeout.listener.equals(listener)) {
@@ -294,32 +364,32 @@ public void remove(ClusterStateListener listener) {
     /**
      * Add a listener for on/off local node master events
      */
-    public void add(LocalNodeMasterListener listener) {
+    public void addLocalNodeMasterListener(LocalNodeMasterListener listener) {
         localNodeMasterListeners.add(listener);
     }
 
     /**
      * Remove the given listener for on/off local master events
      */
-    public void remove(LocalNodeMasterListener listener) {
+    public void removeLocalNodeMasterListener(LocalNodeMasterListener listener) {
         localNodeMasterListeners.remove(listener);
     }
 
     /**
-     * Adds a cluster state listener that will timeout after the provided timeout,
-     * and is executed after the clusterstate has been successfully applied ie. is
-     * in state {@link org.elasticsearch.cluster.ClusterState.ClusterStateStatus#APPLIED}
-     * NOTE: a {@code null} timeout means that the listener will never be removed
-     * automatically
+     * Adds a cluster state listener that is expected to be removed during a short period of time.
+     * If provided, the listener will be notified once a specific time has elapsed.
+     *
+     * NOTE: the listener is not remmoved on timeout. This is the responsibility of the caller.
      */
-    public void add(@Nullable final TimeValue timeout, final TimeoutClusterStateListener listener) {
+    public void addTimeoutListener(@Nullable final TimeValue timeout, final TimeoutClusterStateListener listener) {
         if (lifecycle.stoppedOrClosed()) {
             listener.onClose();
             return;
         }
+
         // call the post added notification on the same event thread
         try {
-            updateTasksExecutor.execute(new SourcePrioritizedRunnable(Priority.HIGH, "_add_listener_") {
+            threadPoolExecutor.execute(new SourcePrioritizedRunnable(Priority.HIGH, "_add_listener_") {
                 @Override
                 public void run() {
                     if (timeout != null) {
@@ -327,7 +397,7 @@ public void run() {
                         notifyTimeout.future = threadPool.schedule(timeout, ThreadPool.Names.GENERIC, notifyTimeout);
                         onGoingTimeouts.add(notifyTimeout);
                     }
-                    postAppliedListeners.add(listener);
+                    timeoutClusterStateListeners.add(listener);
                     listener.postAdded();
                 }
             });
@@ -349,11 +419,11 @@ public void run() {
      *                   task
      *
      */
-    public void submitStateUpdateTask(final String source, final ClusterStateUpdateTask updateTask) {
+    public <T extends ClusterStateTaskConfig & ClusterStateTaskExecutor<T> & ClusterStateTaskListener> void submitStateUpdateTask(
+        final String source, final T updateTask) {
         submitStateUpdateTask(source, updateTask, updateTask, updateTask, updateTask);
     }
 
-
     /**
      * Submits a cluster state update task; submitted updates will be
      * batched across the same instance of executor. The exact batching
@@ -399,41 +469,11 @@ public <T> void submitStateUpdateTasks(final String source,
         if (!lifecycle.started()) {
             return;
         }
-        if (tasks.isEmpty()) {
-            return;
-        }
         try {
-            // convert to an identity map to check for dups based on update tasks semantics of using identity instead of equal
-            final IdentityHashMap<T, ClusterStateTaskListener> tasksIdentity = new IdentityHashMap<>(tasks);
-            final List<UpdateTask<T>> updateTasks = tasksIdentity.entrySet().stream().map(
-                entry -> new UpdateTask<>(source, entry.getKey(), config, executor, safe(entry.getValue(), logger))
-            ).collect(Collectors.toList());
-
-            synchronized (updateTasksPerExecutor) {
-                List<UpdateTask> existingTasks = updateTasksPerExecutor.computeIfAbsent(executor, k -> new ArrayList<>());
-                for (@SuppressWarnings("unchecked") UpdateTask<T> existing : existingTasks) {
-                    if (tasksIdentity.containsKey(existing.task)) {
-                        throw new IllegalStateException("task [" + executor.describeTasks(Collections.singletonList(existing.task)) +
-                            "] with source [" + source + "] is already queued");
-                    }
-                }
-                existingTasks.addAll(updateTasks);
-            }
-
-            final UpdateTask<T> firstTask = updateTasks.get(0);
-
-            if (config.timeout() != null) {
-                updateTasksExecutor.execute(firstTask, threadPool.scheduler(), config.timeout(), () -> threadPool.generic().execute(() -> {
-                    for (UpdateTask<T> task : updateTasks) {
-                        if (task.processed.getAndSet(true) == false) {
-                            logger.debug("cluster state update task [{}] timed out after [{}]", source, config.timeout());
-                            task.listener.onFailure(source, new ProcessClusterEventTimeoutException(config.timeout(), source));
-                        }
-                    }
-                }));
-            } else {
-                updateTasksExecutor.execute(firstTask);
-            }
+            List<ClusterServiceTaskBatcher.UpdateTask> safeTasks = tasks.entrySet().stream()
+                .map(e -> taskBatcher.new UpdateTask(config.priority(), source, e.getKey(), safe(e.getValue(), logger), executor))
+                .collect(Collectors.toList());
+            taskBatcher.submitTasks(safeTasks, config.timeout());
         } catch (EsRejectedExecutionException e) {
             // ignore cases where we are shutting down..., there is really nothing interesting
             // to be done here...
@@ -447,36 +487,20 @@ public <T> void submitStateUpdateTasks(final String source,
      * Returns the tasks that are pending.
      */
     public List<PendingClusterTask> pendingTasks() {
-        PrioritizedEsThreadPoolExecutor.Pending[] pendings = updateTasksExecutor.getPending();
-        List<PendingClusterTask> pendingClusterTasks = new ArrayList<>(pendings.length);
-        for (PrioritizedEsThreadPoolExecutor.Pending pending : pendings) {
-            final String source;
-            final long timeInQueue;
-            // we have to capture the task as it will be nulled after execution and we don't want to change while we check things here.
-            final Object task = pending.task;
-            if (task == null) {
-                continue;
-            } else if (task instanceof SourcePrioritizedRunnable) {
-                SourcePrioritizedRunnable runnable = (SourcePrioritizedRunnable) task;
-                source = runnable.source();
-                timeInQueue = runnable.getAgeInMillis();
-            } else {
-                assert false : "expected SourcePrioritizedRunnable got " + task.getClass();
-                source = "unknown [" + task.getClass() + "]";
-                timeInQueue = 0;
-            }
-
-            pendingClusterTasks.add(
-                    new PendingClusterTask(pending.insertionOrder, pending.priority, new Text(source), timeInQueue, pending.executing));
-        }
-        return pendingClusterTasks;
+        return Arrays.stream(threadPoolExecutor.getPending()).map(pending -> {
+            assert pending.task instanceof SourcePrioritizedRunnable :
+                "thread pool executor should only use SourcePrioritizedRunnable instances but found: " + pending.task.getClass().getName();
+            SourcePrioritizedRunnable task = (SourcePrioritizedRunnable) pending.task;
+            return new PendingClusterTask(pending.insertionOrder, pending.priority, new Text(task.source()),
+                task.getAgeInMillis(), pending.executing);
+        }).collect(Collectors.toList());
     }
 
     /**
      * Returns the number of currently pending tasks.
      */
     public int numberOfPendingTasks() {
-        return updateTasksExecutor.getNumberOfPendingTasks();
+        return threadPoolExecutor.getNumberOfPendingTasks();
     }
 
     /**
@@ -485,7 +509,7 @@ public int numberOfPendingTasks() {
      * @return A zero time value if the queue is empty, otherwise the time value oldest task waiting in the queue
      */
     public TimeValue getMaxTaskWaitTime() {
-        return updateTasksExecutor.getMaxTaskWaitTime();
+        return threadPoolExecutor.getMaxTaskWaitTime();
     }
 
     /** asserts that the current thread is the cluster state update thread */
@@ -495,64 +519,119 @@ public static boolean assertClusterStateThread() {
         return true;
     }
 
-    public ClusterName getClusterName() {
-        return clusterName;
+    /** asserts that the current thread is <b>NOT</b> the cluster state update thread */
+    public static boolean assertNotClusterStateUpdateThread(String reason) {
+        assert Thread.currentThread().getName().contains(UPDATE_THREAD_NAME) == false :
+            "Expected current thread [" + Thread.currentThread() + "] to not be the cluster state update thread. Reason: [" + reason + "]";
+        return true;
     }
 
-    abstract static class SourcePrioritizedRunnable extends PrioritizedRunnable {
-        protected final String source;
-
-        public SourcePrioritizedRunnable(Priority priority, String source) {
-            super(priority);
-            this.source = source;
+    /** asserts that the current stack trace does <b>NOT</b> invlove a cluster state applier */
+    private static boolean assertNotCalledFromClusterStateApplier(String reason) {
+        if (Thread.currentThread().getName().contains(UPDATE_THREAD_NAME)) {
+            for (StackTraceElement element : Thread.currentThread().getStackTrace()) {
+                final String className = element.getClassName();
+                final String methodName = element.getMethodName();
+                if (className.equals(ClusterStateObserver.class.getName())) {
+                    // people may start an observer from an applier
+                    return true;
+                } else if (className.equals(ClusterService.class.getName())
+                    && methodName.equals("callClusterStateAppliers")) {
+                    throw new AssertionError("should not be called by a cluster state applier. reason [" + reason + "]");
+                }
+            }
         }
+        return true;
+    }
 
-        public String source() {
-            return source;
-        }
+    public ClusterName getClusterName() {
+        return clusterName;
     }
 
-    <T> void runTasksForExecutor(ClusterStateTaskExecutor<T> executor) {
-        final ArrayList<UpdateTask<T>> toExecute = new ArrayList<>();
-        final Map<String, ArrayList<T>> processTasksBySource = new HashMap<>();
-        synchronized (updateTasksPerExecutor) {
-            List<UpdateTask> pending = updateTasksPerExecutor.remove(executor);
-            if (pending != null) {
-                for (UpdateTask<T> task : pending) {
-                    if (task.processed.getAndSet(true) == false) {
-                        logger.trace("will process {}", task.toString(executor));
-                        toExecute.add(task);
-                        processTasksBySource.computeIfAbsent(task.source, s -> new ArrayList<>()).add(task.task);
-                    } else {
-                        logger.trace("skipping {}, already processed", task.toString(executor));
-                    }
-                }
-            }
-        }
-        if (toExecute.isEmpty()) {
-            return;
-        }
-        final String tasksSummary = processTasksBySource.entrySet().stream().map(entry -> {
-            String tasks = executor.describeTasks(entry.getValue());
-            return tasks.isEmpty() ? entry.getKey() : entry.getKey() + "[" + tasks + "]";
-        }).reduce((s1, s2) -> s1 + ", " + s2).orElse("");
+    public void setDiscoverySettings(DiscoverySettings discoverySettings) {
+        this.discoverySettings = discoverySettings;
+    }
 
+    void runTasks(TaskInputs taskInputs) {
         if (!lifecycle.started()) {
-            logger.debug("processing [{}]: ignoring, cluster_service not started", tasksSummary);
+            logger.debug("processing [{}]: ignoring, cluster service not started", taskInputs.summary);
             return;
         }
-        logger.debug("processing [{}]: execute", tasksSummary);
-        ClusterState previousClusterState = clusterState;
-        if (!previousClusterState.nodes().isLocalNodeElectedMaster() && executor.runOnlyOnMaster()) {
-            logger.debug("failing [{}]: local node is no longer master", tasksSummary);
-            toExecute.stream().forEach(task -> task.listener.onNoLongerMaster(task.source));
+
+        logger.debug("processing [{}]: execute", taskInputs.summary);
+        ClusterState previousClusterState = state();
+
+        if (!previousClusterState.nodes().isLocalNodeElectedMaster() && taskInputs.runOnlyOnMaster()) {
+            logger.debug("failing [{}]: local node is no longer master", taskInputs.summary);
+            taskInputs.onNoLongerMaster();
             return;
         }
-        ClusterStateTaskExecutor.BatchResult<T> batchResult;
+
         long startTimeNS = currentTimeInNanos();
+        TaskOutputs taskOutputs = calculateTaskOutputs(taskInputs, previousClusterState, startTimeNS);
+        taskOutputs.notifyFailedTasks();
+
+        if (taskOutputs.clusterStateUnchanged()) {
+            taskOutputs.notifySuccessfulTasksOnUnchangedClusterState();
+            TimeValue executionTime = TimeValue.timeValueMillis(Math.max(0, TimeValue.nsecToMSec(currentTimeInNanos() - startTimeNS)));
+            logger.debug("processing [{}]: took [{}] no change in cluster_state", taskInputs.summary, executionTime);
+            warnAboutSlowTaskIfNeeded(executionTime, taskInputs.summary);
+        } else {
+            ClusterState newClusterState = taskOutputs.newClusterState;
+            if (logger.isTraceEnabled()) {
+                logger.trace("cluster state updated, source [{}]\n{}", taskInputs.summary, newClusterState);
+            } else if (logger.isDebugEnabled()) {
+                logger.debug("cluster state updated, version [{}], source [{}]", newClusterState.version(), taskInputs.summary);
+            }
+            try {
+                publishAndApplyChanges(taskInputs, taskOutputs);
+                TimeValue executionTime = TimeValue.timeValueMillis(Math.max(0, TimeValue.nsecToMSec(currentTimeInNanos() - startTimeNS)));
+                logger.debug("processing [{}]: took [{}] done applying updated cluster_state (version: {}, uuid: {})", taskInputs.summary,
+                    executionTime, newClusterState.version(), newClusterState.stateUUID());
+                warnAboutSlowTaskIfNeeded(executionTime, taskInputs.summary);
+            } catch (Exception e) {
+                TimeValue executionTime = TimeValue.timeValueMillis(Math.max(0, TimeValue.nsecToMSec(currentTimeInNanos() - startTimeNS)));
+                final long version = newClusterState.version();
+                final String stateUUID = newClusterState.stateUUID();
+                final String fullState = newClusterState.toString();
+                logger.warn(
+                    (Supplier<?>) () -> new ParameterizedMessage(
+                        "failed to apply updated cluster state in [{}]:\nversion [{}], uuid [{}], source [{}]\n{}",
+                        executionTime,
+                        version,
+                        stateUUID,
+                        taskInputs.summary,
+                        fullState),
+                    e);
+                // TODO: do we want to call updateTask.onFailure here?
+            }
+        }
+    }
+
+    public TaskOutputs calculateTaskOutputs(TaskInputs taskInputs, ClusterState previousClusterState, long startTimeNS) {
+        ClusterTasksResult<Object> clusterTasksResult = executeTasks(taskInputs, startTimeNS, previousClusterState);
+        // extract those that are waiting for results
+        List<ClusterServiceTaskBatcher.UpdateTask> nonFailedTasks = new ArrayList<>();
+        for (ClusterServiceTaskBatcher.UpdateTask updateTask : taskInputs.updateTasks) {
+            assert clusterTasksResult.executionResults.containsKey(updateTask.task) : "missing " + updateTask;
+            final ClusterStateTaskExecutor.TaskResult taskResult =
+                clusterTasksResult.executionResults.get(updateTask.task);
+            if (taskResult.isSuccess()) {
+                nonFailedTasks.add(updateTask);
+            }
+        }
+        ClusterState newClusterState = patchVersionsAndNoMasterBlocks(previousClusterState, clusterTasksResult);
+
+        return new TaskOutputs(taskInputs, previousClusterState, newClusterState, nonFailedTasks,
+                                  clusterTasksResult.executionResults);
+    }
+
+    private ClusterTasksResult<Object> executeTasks(TaskInputs taskInputs, long startTimeNS, ClusterState previousClusterState) {
+        ClusterTasksResult<Object> clusterTasksResult;
         try {
-            List<T> inputs = toExecute.stream().map(tUpdateTask -> tUpdateTask.task).collect(Collectors.toList());
-            batchResult = executor.execute(previousClusterState, inputs);
+            List<Object> inputs = taskInputs.updateTasks.stream()
+                .map(ClusterServiceTaskBatcher.UpdateTask::getTask).collect(Collectors.toList());
+            clusterTasksResult = taskInputs.executor.execute(previousClusterState, inputs);
         } catch (Exception e) {
             TimeValue executionTime = TimeValue.timeValueMillis(Math.max(0, TimeValue.nsecToMSec(currentTimeInNanos() - startTimeNS)));
             if (logger.isTraceEnabled()) {
@@ -561,219 +640,294 @@ <T> void runTasksForExecutor(ClusterStateTaskExecutor<T> executor) {
                         "failed to execute cluster state update in [{}], state:\nversion [{}], source [{}]\n{}{}{}",
                         executionTime,
                         previousClusterState.version(),
-                        tasksSummary,
-                        previousClusterState.nodes().prettyPrint(),
-                        previousClusterState.routingTable().prettyPrint(),
-                        previousClusterState.getRoutingNodes().prettyPrint()),
+                        taskInputs.summary,
+                        previousClusterState.nodes(),
+                        previousClusterState.routingTable(),
+                        previousClusterState.getRoutingNodes()),
                     e);
             }
-            warnAboutSlowTaskIfNeeded(executionTime, tasksSummary);
-            batchResult = ClusterStateTaskExecutor.BatchResult.<T>builder()
-                    .failures(toExecute.stream().map(updateTask -> updateTask.task)::iterator, e)
-                    .build(previousClusterState);
-        }
-
-        assert batchResult.executionResults != null;
-        assert batchResult.executionResults.size() == toExecute.size()
-                : String.format(Locale.ROOT, "expected [%d] task result%s but was [%d]", toExecute.size(),
-                toExecute.size() == 1 ? "" : "s", batchResult.executionResults.size());
-        boolean assertsEnabled = false;
-        assert (assertsEnabled = true);
-        if (assertsEnabled) {
-            for (UpdateTask<T> updateTask : toExecute) {
-                assert batchResult.executionResults.containsKey(updateTask.task) :
-                    "missing task result for " + updateTask.toString(executor);
-            }
+            warnAboutSlowTaskIfNeeded(executionTime, taskInputs.summary);
+            clusterTasksResult = ClusterTasksResult.builder()
+                .failures(taskInputs.updateTasks.stream().map(ClusterServiceTaskBatcher.UpdateTask::getTask)::iterator, e)
+                .build(previousClusterState);
         }
 
-        ClusterState newClusterState = batchResult.resultingState;
-        final ArrayList<UpdateTask<T>> proccessedListeners = new ArrayList<>();
-        // fail all tasks that have failed and extract those that are waiting for results
-        for (UpdateTask<T> updateTask : toExecute) {
-            assert batchResult.executionResults.containsKey(updateTask.task) : "missing " + updateTask.toString(executor);
-            final ClusterStateTaskExecutor.TaskResult executionResult =
-                    batchResult.executionResults.get(updateTask.task);
-            executionResult.handle(
-                    () -> proccessedListeners.add(updateTask),
-                    ex -> {
-                        logger.debug(
-                            (Supplier<?>)
-                                () -> new ParameterizedMessage("cluster state update task {} failed", updateTask.toString(executor)), ex);
-                        updateTask.listener.onFailure(updateTask.source, ex);
-                    }
-            );
+        assert clusterTasksResult.executionResults != null;
+        assert clusterTasksResult.executionResults.size() == taskInputs.updateTasks.size()
+                : String.format(Locale.ROOT, "expected [%d] task result%s but was [%d]", taskInputs.updateTasks.size(),
+                taskInputs.updateTasks.size() == 1 ? "" : "s", clusterTasksResult.executionResults.size());
+        if (Assertions.ENABLED) {
+            for (ClusterServiceTaskBatcher.UpdateTask updateTask : taskInputs.updateTasks) {
+                assert clusterTasksResult.executionResults.containsKey(updateTask.task) :
+                    "missing task result for " + updateTask;
+            }
         }
 
-        if (previousClusterState == newClusterState) {
-            for (UpdateTask<T> task : proccessedListeners) {
-                if (task.listener instanceof AckedClusterStateTaskListener) {
-                    //no need to wait for ack if nothing changed, the update can be counted as acknowledged
-                    ((AckedClusterStateTaskListener) task.listener).onAllNodesAcked(null);
-                }
-                task.listener.clusterStateProcessed(task.source, previousClusterState, newClusterState);
+        return clusterTasksResult;
+    }
+
+    private ClusterState patchVersionsAndNoMasterBlocks(ClusterState previousClusterState, ClusterTasksResult<Object> executionResult) {
+        ClusterState newClusterState = executionResult.resultingState;
+
+        if (executionResult.noMaster) {
+            assert newClusterState == previousClusterState : "state can only be changed by ClusterService when noMaster = true";
+            if (previousClusterState.nodes().getMasterNodeId() != null) {
+                // remove block if it already exists before adding new one
+                assert previousClusterState.blocks().hasGlobalBlock(discoverySettings.getNoMasterBlock().id()) == false :
+                    "NO_MASTER_BLOCK should only be added by ClusterService";
+                ClusterBlocks clusterBlocks = ClusterBlocks.builder().blocks(previousClusterState.blocks())
+                    .addGlobalBlock(discoverySettings.getNoMasterBlock())
+                    .build();
+
+                DiscoveryNodes discoveryNodes = new DiscoveryNodes.Builder(previousClusterState.nodes()).masterNodeId(null).build();
+                newClusterState = ClusterState.builder(previousClusterState)
+                    .blocks(clusterBlocks)
+                    .nodes(discoveryNodes)
+                    .build();
+            }
+        } else if (newClusterState.nodes().isLocalNodeElectedMaster() && previousClusterState != newClusterState) {
+            // only the master controls the version numbers
+            Builder builder = ClusterState.builder(newClusterState).incrementVersion();
+            if (previousClusterState.routingTable() != newClusterState.routingTable()) {
+                builder.routingTable(RoutingTable.builder(newClusterState.routingTable())
+                    .version(newClusterState.routingTable().version() + 1).build());
+            }
+            if (previousClusterState.metaData() != newClusterState.metaData()) {
+                builder.metaData(MetaData.builder(newClusterState.metaData()).version(newClusterState.metaData().version() + 1));
             }
-            TimeValue executionTime = TimeValue.timeValueMillis(Math.max(0, TimeValue.nsecToMSec(currentTimeInNanos() - startTimeNS)));
-            logger.debug("processing [{}]: took [{}] no change in cluster_state", tasksSummary, executionTime);
-            warnAboutSlowTaskIfNeeded(executionTime, tasksSummary);
-            return;
-        }
 
-        try {
-            ArrayList<Discovery.AckListener> ackListeners = new ArrayList<>();
-            if (newClusterState.nodes().isLocalNodeElectedMaster()) {
-                // only the master controls the version numbers
-                Builder builder = ClusterState.builder(newClusterState).incrementVersion();
-                if (previousClusterState.routingTable() != newClusterState.routingTable()) {
-                    builder.routingTable(RoutingTable.builder(newClusterState.routingTable())
-                            .version(newClusterState.routingTable().version() + 1).build());
-                }
-                if (previousClusterState.metaData() != newClusterState.metaData()) {
-                    builder.metaData(MetaData.builder(newClusterState.metaData()).version(newClusterState.metaData().version() + 1));
-                }
-                newClusterState = builder.build();
-                for (UpdateTask<T> task : proccessedListeners) {
-                    if (task.listener instanceof AckedClusterStateTaskListener) {
-                        final AckedClusterStateTaskListener ackedListener = (AckedClusterStateTaskListener) task.listener;
-                        if (ackedListener.ackTimeout() == null || ackedListener.ackTimeout().millis() == 0) {
-                            ackedListener.onAckTimeout();
-                        } else {
-                            try {
-                                ackListeners.add(new AckCountDownListener(ackedListener, newClusterState.version(), newClusterState.nodes(),
-                                        threadPool));
-                            } catch (EsRejectedExecutionException ex) {
-                                if (logger.isDebugEnabled()) {
-                                    logger.debug("Couldn't schedule timeout thread - node might be shutting down", ex);
-                                }
-                                //timeout straightaway, otherwise we could wait forever as the timeout thread has not started
-                                ackedListener.onAckTimeout();
-                            }
-                        }
-                    }
-                }
+            // remove the no master block, if it exists
+            if (newClusterState.blocks().hasGlobalBlock(discoverySettings.getNoMasterBlock().id())) {
+                builder.blocks(ClusterBlocks.builder().blocks(newClusterState.blocks())
+                    .removeGlobalBlock(discoverySettings.getNoMasterBlock().id()));
             }
-            final Discovery.AckListener ackListener = new DelegetingAckListener(ackListeners);
 
-            newClusterState.status(ClusterState.ClusterStateStatus.BEING_APPLIED);
+            newClusterState = builder.build();
+        }
 
-            if (logger.isTraceEnabled()) {
-                logger.trace("cluster state updated, source [{}]\n{}", tasksSummary, newClusterState.prettyPrint());
-            } else if (logger.isDebugEnabled()) {
-                logger.debug("cluster state updated, version [{}], source [{}]", newClusterState.version(), tasksSummary);
-            }
+        assert newClusterState.nodes().getMasterNodeId() == null ||
+            newClusterState.blocks().hasGlobalBlock(discoverySettings.getNoMasterBlock().id()) == false :
+            "cluster state with master node must not have NO_MASTER_BLOCK";
 
-            ClusterChangedEvent clusterChangedEvent = new ClusterChangedEvent(tasksSummary, newClusterState, previousClusterState);
-            // new cluster state, notify all listeners
-            final DiscoveryNodes.Delta nodesDelta = clusterChangedEvent.nodesDelta();
-            if (nodesDelta.hasChanges() && logger.isInfoEnabled()) {
-                String summary = nodesDelta.shortSummary();
-                if (summary.length() > 0) {
-                    logger.info("{}, reason: {}", summary, tasksSummary);
-                }
-            }
+        return newClusterState;
+    }
 
-            nodeConnectionsService.connectToAddedNodes(clusterChangedEvent);
-
-            // if we are the master, publish the new state to all nodes
-            // we publish here before we send a notification to all the listeners, since if it fails
-            // we don't want to notify
-            if (newClusterState.nodes().isLocalNodeElectedMaster()) {
-                logger.debug("publishing cluster state version [{}]", newClusterState.version());
-                try {
-                    clusterStatePublisher.accept(clusterChangedEvent, ackListener);
-                } catch (Discovery.FailedToCommitClusterStateException t) {
-                    final long version = newClusterState.version();
-                    logger.warn(
-                        (Supplier<?>) () -> new ParameterizedMessage(
-                            "failing [{}]: failed to commit cluster state version [{}]", tasksSummary, version),
-                        t);
-                    proccessedListeners.forEach(task -> task.listener.onFailure(task.source, t));
-                    return;
-                }
+    private void publishAndApplyChanges(TaskInputs taskInputs, TaskOutputs taskOutputs) {
+        ClusterState previousClusterState = taskOutputs.previousClusterState;
+        ClusterState newClusterState = taskOutputs.newClusterState;
+
+        ClusterChangedEvent clusterChangedEvent = new ClusterChangedEvent(taskInputs.summary, newClusterState, previousClusterState);
+        // new cluster state, notify all listeners
+        final DiscoveryNodes.Delta nodesDelta = clusterChangedEvent.nodesDelta();
+        if (nodesDelta.hasChanges() && logger.isInfoEnabled()) {
+            String summary = nodesDelta.shortSummary();
+            if (summary.length() > 0) {
+                logger.info("{}, reason: {}", summary, taskInputs.summary);
             }
+        }
+
+        final Discovery.AckListener ackListener = newClusterState.nodes().isLocalNodeElectedMaster() ?
+            taskOutputs.createAckListener(threadPool, newClusterState) :
+            null;
 
-            // update the current cluster state
-            clusterState = newClusterState;
-            logger.debug("set local cluster state to version {}", newClusterState.version());
+        nodeConnectionsService.connectToNodes(newClusterState.nodes());
+
+        // if we are the master, publish the new state to all nodes
+        // we publish here before we send a notification to all the listeners, since if it fails
+        // we don't want to notify
+        if (newClusterState.nodes().isLocalNodeElectedMaster()) {
+            logger.debug("publishing cluster state version [{}]", newClusterState.version());
             try {
-                // nothing to do until we actually recover from the gateway or any other block indicates we need to disable persistency
-                if (clusterChangedEvent.state().blocks().disableStatePersistence() == false && clusterChangedEvent.metaDataChanged()) {
-                    final Settings incomingSettings = clusterChangedEvent.state().metaData().settings();
-                    clusterSettings.applySettings(incomingSettings);
-                }
-            } catch (Exception ex) {
-                logger.warn("failed to apply cluster settings", ex);
+                clusterStatePublisher.accept(clusterChangedEvent, ackListener);
+            } catch (Discovery.FailedToCommitClusterStateException t) {
+                final long version = newClusterState.version();
+                logger.warn(
+                    (Supplier<?>) () -> new ParameterizedMessage(
+                        "failing [{}]: failed to commit cluster state version [{}]", taskInputs.summary, version),
+                    t);
+                // ensure that list of connected nodes in NodeConnectionsService is in-sync with the nodes of the current cluster state
+                nodeConnectionsService.connectToNodes(previousClusterState.nodes());
+                nodeConnectionsService.disconnectFromNodesExcept(previousClusterState.nodes());
+                taskOutputs.publishingFailed(t);
+                return;
             }
-            for (ClusterStateListener listener : preAppliedListeners) {
-                try {
-                    listener.clusterChanged(clusterChangedEvent);
-                } catch (Exception ex) {
-                    logger.warn("failed to notify ClusterStateListener", ex);
-                }
+        }
+
+        logger.debug("applying cluster state version {}", newClusterState.version());
+        try {
+            // nothing to do until we actually recover from the gateway or any other block indicates we need to disable persistency
+            if (clusterChangedEvent.state().blocks().disableStatePersistence() == false && clusterChangedEvent.metaDataChanged()) {
+                final Settings incomingSettings = clusterChangedEvent.state().metaData().settings();
+                clusterSettings.applySettings(incomingSettings);
             }
+        } catch (Exception ex) {
+            logger.warn("failed to apply cluster settings", ex);
+        }
 
-            nodeConnectionsService.disconnectFromRemovedNodes(clusterChangedEvent);
+        logger.debug("set local cluster state to version {}", newClusterState.version());
+        callClusterStateAppliers(newClusterState, clusterChangedEvent);
 
-            newClusterState.status(ClusterState.ClusterStateStatus.APPLIED);
+        nodeConnectionsService.disconnectFromNodesExcept(newClusterState.nodes());
 
-            for (ClusterStateListener listener : postAppliedListeners) {
-                try {
-                    listener.clusterChanged(clusterChangedEvent);
-                } catch (Exception ex) {
-                    logger.warn("failed to notify ClusterStateListener", ex);
-                }
-            }
+        updateState(css -> newClusterState);
 
-            //manual ack only from the master at the end of the publish
-            if (newClusterState.nodes().isLocalNodeElectedMaster()) {
-                try {
-                    ackListener.onNodeAck(newClusterState.nodes().getLocalNode(), null);
-                } catch (Exception e) {
-                    final DiscoveryNode localNode = newClusterState.nodes().getLocalNode();
-                    logger.debug(
-                        (Supplier<?>) () -> new ParameterizedMessage("error while processing ack for master node [{}]", localNode),
-                        e);
-                }
+        Stream.concat(clusterStateListeners.stream(), timeoutClusterStateListeners.stream()).forEach(listener -> {
+            try {
+                logger.trace("calling [{}] with change to version [{}]", listener, newClusterState.version());
+                listener.clusterChanged(clusterChangedEvent);
+            } catch (Exception ex) {
+                logger.warn("failed to notify ClusterStateListener", ex);
             }
+        });
 
-            for (UpdateTask<T> task : proccessedListeners) {
-                task.listener.clusterStateProcessed(task.source, previousClusterState, newClusterState);
+        //manual ack only from the master at the end of the publish
+        if (newClusterState.nodes().isLocalNodeElectedMaster()) {
+            try {
+                ackListener.onNodeAck(newClusterState.nodes().getLocalNode(), null);
+            } catch (Exception e) {
+                final DiscoveryNode localNode = newClusterState.nodes().getLocalNode();
+                logger.debug(
+                    (Supplier<?>) () -> new ParameterizedMessage("error while processing ack for master node [{}]", localNode),
+                    e);
             }
+        }
 
+        taskOutputs.processedDifferentClusterState(previousClusterState, newClusterState);
+
+        if (newClusterState.nodes().isLocalNodeElectedMaster()) {
             try {
-                executor.clusterStatePublished(clusterChangedEvent);
+                taskOutputs.clusterStatePublished(clusterChangedEvent);
             } catch (Exception e) {
                 logger.error(
                     (Supplier<?>) () -> new ParameterizedMessage(
                         "exception thrown while notifying executor of new cluster state publication [{}]",
-                        tasksSummary),
+                        taskInputs.summary),
                     e);
             }
+        }
+    }
 
-            TimeValue executionTime = TimeValue.timeValueMillis(Math.max(0, TimeValue.nsecToMSec(currentTimeInNanos() - startTimeNS)));
-            logger.debug("processing [{}]: took [{}] done applying updated cluster_state (version: {}, uuid: {})", tasksSummary,
-                executionTime, newClusterState.version(), newClusterState.stateUUID());
-            warnAboutSlowTaskIfNeeded(executionTime, tasksSummary);
-        } catch (Exception e) {
-            TimeValue executionTime = TimeValue.timeValueMillis(Math.max(0, TimeValue.nsecToMSec(currentTimeInNanos() - startTimeNS)));
-            final long version = newClusterState.version();
-            final String stateUUID = newClusterState.stateUUID();
-            final String prettyPrint = newClusterState.prettyPrint();
-            logger.warn(
-                (Supplier<?>) () -> new ParameterizedMessage(
-                    "failed to apply updated cluster state in [{}]:\nversion [{}], uuid [{}], source [{}]\n{}",
-                    executionTime,
-                    version,
-                    stateUUID,
-                    tasksSummary,
-                    prettyPrint),
-                e);
-            // TODO: do we want to call updateTask.onFailure here?
+    private void callClusterStateAppliers(ClusterState newClusterState, ClusterChangedEvent clusterChangedEvent) {
+        for (ClusterStateApplier applier : clusterStateAppliers) {
+            try {
+                logger.trace("calling [{}] with change to version [{}]", applier, newClusterState.version());
+                applier.applyClusterState(clusterChangedEvent);
+            } catch (Exception ex) {
+                logger.warn("failed to notify ClusterStateApplier", ex);
+            }
+        }
+    }
+
+    /**
+     * Represents a set of tasks to be processed together with their executor
+     */
+    class TaskInputs {
+        public final String summary;
+        public final List<ClusterServiceTaskBatcher.UpdateTask> updateTasks;
+        public final ClusterStateTaskExecutor<Object> executor;
+
+        TaskInputs(ClusterStateTaskExecutor<Object> executor, List<ClusterServiceTaskBatcher.UpdateTask> updateTasks, String summary) {
+            this.summary = summary;
+            this.executor = executor;
+            this.updateTasks = updateTasks;
+        }
+
+        public boolean runOnlyOnMaster() {
+            return executor.runOnlyOnMaster();
+        }
+
+        public void onNoLongerMaster() {
+            updateTasks.stream().forEach(task -> task.listener.onNoLongerMaster(task.source));
+        }
+    }
+
+    /**
+     * Output created by executing a set of tasks provided as TaskInputs
+     */
+    class TaskOutputs {
+        public final TaskInputs taskInputs;
+        public final ClusterState previousClusterState;
+        public final ClusterState newClusterState;
+        public final List<ClusterServiceTaskBatcher.UpdateTask> nonFailedTasks;
+        public final Map<Object, ClusterStateTaskExecutor.TaskResult> executionResults;
+
+        TaskOutputs(TaskInputs taskInputs, ClusterState previousClusterState,
+                           ClusterState newClusterState, List<ClusterServiceTaskBatcher.UpdateTask> nonFailedTasks,
+                           Map<Object, ClusterStateTaskExecutor.TaskResult> executionResults) {
+            this.taskInputs = taskInputs;
+            this.previousClusterState = previousClusterState;
+            this.newClusterState = newClusterState;
+            this.nonFailedTasks = nonFailedTasks;
+            this.executionResults = executionResults;
+        }
+
+        public void publishingFailed(Discovery.FailedToCommitClusterStateException t) {
+            nonFailedTasks.forEach(task -> task.listener.onFailure(task.source, t));
+        }
+
+        public void processedDifferentClusterState(ClusterState previousClusterState, ClusterState newClusterState) {
+            nonFailedTasks.forEach(task -> task.listener.clusterStateProcessed(task.source, previousClusterState, newClusterState));
+        }
+
+        public void clusterStatePublished(ClusterChangedEvent clusterChangedEvent) {
+            taskInputs.executor.clusterStatePublished(clusterChangedEvent);
+        }
+
+        public Discovery.AckListener createAckListener(ThreadPool threadPool, ClusterState newClusterState) {
+            ArrayList<Discovery.AckListener> ackListeners = new ArrayList<>();
+
+            //timeout straightaway, otherwise we could wait forever as the timeout thread has not started
+            nonFailedTasks.stream().filter(task -> task.listener instanceof AckedClusterStateTaskListener).forEach(task -> {
+                final AckedClusterStateTaskListener ackedListener = (AckedClusterStateTaskListener) task.listener;
+                if (ackedListener.ackTimeout() == null || ackedListener.ackTimeout().millis() == 0) {
+                    ackedListener.onAckTimeout();
+                } else {
+                    try {
+                        ackListeners.add(new AckCountDownListener(ackedListener, newClusterState.version(), newClusterState.nodes(),
+                            threadPool));
+                    } catch (EsRejectedExecutionException ex) {
+                        if (logger.isDebugEnabled()) {
+                            logger.debug("Couldn't schedule timeout thread - node might be shutting down", ex);
+                        }
+                        //timeout straightaway, otherwise we could wait forever as the timeout thread has not started
+                        ackedListener.onAckTimeout();
+                    }
+                }
+            });
+
+            return new DelegetingAckListener(ackListeners);
+        }
+
+        public boolean clusterStateUnchanged() {
+            return previousClusterState == newClusterState;
+        }
+
+        public void notifyFailedTasks() {
+            // fail all tasks that have failed
+            for (ClusterServiceTaskBatcher.UpdateTask updateTask : taskInputs.updateTasks) {
+                assert executionResults.containsKey(updateTask.task) : "missing " + updateTask;
+                final ClusterStateTaskExecutor.TaskResult taskResult = executionResults.get(updateTask.task);
+                if (taskResult.isSuccess() == false) {
+                    updateTask.listener.onFailure(updateTask.source, taskResult.getFailure());
+                }
+            }
         }
 
+        public void notifySuccessfulTasksOnUnchangedClusterState() {
+            nonFailedTasks.forEach(task -> {
+                if (task.listener instanceof AckedClusterStateTaskListener) {
+                    //no need to wait for ack if nothing changed, the update can be counted as acknowledged
+                    ((AckedClusterStateTaskListener) task.listener).onAllNodesAcked(null);
+                }
+                task.listener.clusterStateProcessed(task.source, newClusterState, newClusterState);
+            });
+        }
     }
 
     // this one is overridden in tests so we can control time
-    protected long currentTimeInNanos() {return System.nanoTime();}
+    protected long currentTimeInNanos() {
+        return System.nanoTime();
+    }
 
     private static SafeClusterStateTaskListener safe(ClusterStateTaskListener listener, Logger logger) {
         if (listener instanceof AckedClusterStateTaskListener) {
@@ -787,7 +941,7 @@ private static class SafeClusterStateTaskListener implements ClusterStateTaskLis
         private final ClusterStateTaskListener listener;
         private final Logger logger;
 
-        public SafeClusterStateTaskListener(ClusterStateTaskListener listener, Logger logger) {
+        SafeClusterStateTaskListener(ClusterStateTaskListener listener, Logger logger) {
             this.listener = listener;
             this.logger = logger;
         }
@@ -824,9 +978,7 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS
                     (Supplier<?>) () -> new ParameterizedMessage(
                         "exception thrown by listener while notifying of cluster state processed from [{}], old cluster state:\n" +
                             "{}\nnew cluster state:\n{}",
-                        source,
-                        oldState.prettyPrint(),
-                        newState.prettyPrint()),
+                        source, oldState, newState),
                     e);
             }
         }
@@ -836,7 +988,7 @@ private static class SafeAckedClusterStateTaskListener extends SafeClusterStateT
         private final AckedClusterStateTaskListener listener;
         private final Logger logger;
 
-        public SafeAckedClusterStateTaskListener(AckedClusterStateTaskListener listener, Logger logger) {
+        SafeAckedClusterStateTaskListener(AckedClusterStateTaskListener listener, Logger logger) {
             super(listener, logger);
             this.listener = listener;
             this.logger = logger;
@@ -872,38 +1024,6 @@ public TimeValue ackTimeout() {
         }
     }
 
-    class UpdateTask<T> extends SourcePrioritizedRunnable {
-
-        public final T task;
-        public final ClusterStateTaskConfig config;
-        public final ClusterStateTaskExecutor<T> executor;
-        public final ClusterStateTaskListener listener;
-        public final AtomicBoolean processed = new AtomicBoolean();
-
-        UpdateTask(String source, T task, ClusterStateTaskConfig config, ClusterStateTaskExecutor<T> executor,
-                   ClusterStateTaskListener listener) {
-            super(config.priority(), source);
-            this.task = task;
-            this.config = config;
-            this.executor = executor;
-            this.listener = listener;
-        }
-
-        @Override
-        public void run() {
-            runTasksForExecutor(executor);
-        }
-
-        public String toString(ClusterStateTaskExecutor<T> executor) {
-            String taskDescription = executor.describeTasks(Collections.singletonList(task));
-            if (taskDescription.isEmpty()) {
-                return "[" + source + "]";
-            } else {
-                return "[" + source + "[" + taskDescription + "]]";
-            }
-        }
-    }
-
     private void warnAboutSlowTaskIfNeeded(TimeValue executionTime, String source) {
         if (executionTime.getMillis() > slowTaskLoggingThreshold.getMillis()) {
             logger.warn("cluster state update task [{}] took [{}] above the warn threshold of {}", source, executionTime,
@@ -1057,12 +1177,7 @@ private static class AckCountDownListener implements Discovery.AckListener {
             countDown = Math.max(1, countDown);
             logger.trace("expecting {} acknowledgements for cluster_state update (version: {})", countDown, clusterStateVersion);
             this.countDown = new CountDown(countDown);
-            this.ackTimeoutCallback = threadPool.schedule(ackedTaskListener.ackTimeout(), ThreadPool.Names.GENERIC, new Runnable() {
-                @Override
-                public void run() {
-                    onTimeout();
-                }
-            });
+            this.ackTimeoutCallback = threadPool.schedule(ackedTaskListener.ackTimeout(), ThreadPool.Names.GENERIC, () -> onTimeout());
         }
 
         @Override
diff --git a/core/src/main/java/org/elasticsearch/cluster/service/SourcePrioritizedRunnable.java b/core/src/main/java/org/elasticsearch/cluster/service/SourcePrioritizedRunnable.java
new file mode 100644
index 0000000000000..6358acf7e1c7c
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/cluster/service/SourcePrioritizedRunnable.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.service;
+
+import org.elasticsearch.common.Priority;
+import org.elasticsearch.common.util.concurrent.PrioritizedRunnable;
+
+/**
+ * PrioritizedRunnable that also has a source string
+ */
+public abstract class SourcePrioritizedRunnable extends PrioritizedRunnable {
+    protected final String source;
+
+    public SourcePrioritizedRunnable(Priority priority, String source) {
+        super(priority);
+        this.source = source;
+    }
+
+    public String source() {
+        return source;
+    }
+
+    @Override
+    public String toString() {
+        return "[" + source + "]";
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/cluster/service/TaskBatcher.java b/core/src/main/java/org/elasticsearch/cluster/service/TaskBatcher.java
new file mode 100644
index 0000000000000..867d4191f800f
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/cluster/service/TaskBatcher.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.service;
+
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Priority;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
+import org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.IdentityHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+/**
+ * Batching support for {@link PrioritizedEsThreadPoolExecutor}
+ * Tasks that share the same batching key are batched (see {@link BatchedTask#batchingKey})
+ */
+public abstract class TaskBatcher {
+
+    private final Logger logger;
+    private final PrioritizedEsThreadPoolExecutor threadExecutor;
+    // package visible for tests
+    final Map<Object, LinkedHashSet<BatchedTask>> tasksPerBatchingKey = new HashMap<>();
+
+    public TaskBatcher(Logger logger, PrioritizedEsThreadPoolExecutor threadExecutor) {
+        this.logger = logger;
+        this.threadExecutor = threadExecutor;
+    }
+
+    public void submitTasks(List<? extends BatchedTask> tasks, @Nullable TimeValue timeout) throws EsRejectedExecutionException {
+        if (tasks.isEmpty()) {
+            return;
+        }
+        final BatchedTask firstTask = tasks.get(0);
+        assert tasks.stream().allMatch(t -> t.batchingKey == firstTask.batchingKey) :
+            "tasks submitted in a batch should share the same batching key: " + tasks;
+        // convert to an identity map to check for dups based on task identity
+        final Map<Object, BatchedTask> tasksIdentity = tasks.stream().collect(Collectors.toMap(
+            BatchedTask::getTask,
+            Function.identity(),
+            (a, b) -> { throw new IllegalStateException("cannot add duplicate task: " + a); },
+            IdentityHashMap::new));
+
+        synchronized (tasksPerBatchingKey) {
+            LinkedHashSet<BatchedTask> existingTasks = tasksPerBatchingKey.computeIfAbsent(firstTask.batchingKey,
+                k -> new LinkedHashSet<>(tasks.size()));
+            for (BatchedTask existing : existingTasks) {
+                // check that there won't be two tasks with the same identity for the same batching key
+                BatchedTask duplicateTask = tasksIdentity.get(existing.getTask());
+                if (duplicateTask != null) {
+                    throw new IllegalStateException("task [" + duplicateTask.describeTasks(
+                        Collections.singletonList(existing)) + "] with source [" + duplicateTask.source + "] is already queued");
+                }
+            }
+            existingTasks.addAll(tasks);
+        }
+
+        if (timeout != null) {
+            threadExecutor.execute(firstTask, timeout, () -> onTimeoutInternal(tasks, timeout));
+        } else {
+            threadExecutor.execute(firstTask);
+        }
+    }
+
+    private void onTimeoutInternal(List<? extends BatchedTask> tasks, TimeValue timeout) {
+        final ArrayList<BatchedTask> toRemove = new ArrayList<>();
+        for (BatchedTask task : tasks) {
+            if (task.processed.getAndSet(true) == false) {
+                logger.debug("task [{}] timed out after [{}]", task.source, timeout);
+                toRemove.add(task);
+            }
+        }
+        if (toRemove.isEmpty() == false) {
+            BatchedTask firstTask = toRemove.get(0);
+            Object batchingKey = firstTask.batchingKey;
+            assert tasks.stream().allMatch(t -> t.batchingKey == batchingKey) :
+                "tasks submitted in a batch should share the same batching key: " + tasks;
+            synchronized (tasksPerBatchingKey) {
+                LinkedHashSet<BatchedTask> existingTasks = tasksPerBatchingKey.get(batchingKey);
+                if (existingTasks != null) {
+                    existingTasks.removeAll(toRemove);
+                    if (existingTasks.isEmpty()) {
+                        tasksPerBatchingKey.remove(batchingKey);
+                    }
+                }
+            }
+            onTimeout(toRemove, timeout);
+        }
+    }
+
+    /**
+     * Action to be implemented by the specific batching implementation.
+     * All tasks have the same batching key.
+     */
+    protected abstract void onTimeout(List<? extends BatchedTask> tasks, TimeValue timeout);
+
+    void runIfNotProcessed(BatchedTask updateTask) {
+        // if this task is already processed, it shouldn't execute other tasks with same batching key that arrived later,
+        // to give other tasks with different batching key a chance to execute.
+        if (updateTask.processed.get() == false) {
+            final List<BatchedTask> toExecute = new ArrayList<>();
+            final Map<String, List<BatchedTask>> processTasksBySource = new HashMap<>();
+            synchronized (tasksPerBatchingKey) {
+                LinkedHashSet<BatchedTask> pending = tasksPerBatchingKey.remove(updateTask.batchingKey);
+                if (pending != null) {
+                    for (BatchedTask task : pending) {
+                        if (task.processed.getAndSet(true) == false) {
+                            logger.trace("will process {}", task);
+                            toExecute.add(task);
+                            processTasksBySource.computeIfAbsent(task.source, s -> new ArrayList<>()).add(task);
+                        } else {
+                            logger.trace("skipping {}, already processed", task);
+                        }
+                    }
+                }
+            }
+
+            if (toExecute.isEmpty() == false) {
+                final String tasksSummary = processTasksBySource.entrySet().stream().map(entry -> {
+                    String tasks = updateTask.describeTasks(entry.getValue());
+                    return tasks.isEmpty() ? entry.getKey() : entry.getKey() + "[" + tasks + "]";
+                }).reduce((s1, s2) -> s1 + ", " + s2).orElse("");
+
+                run(updateTask.batchingKey, toExecute, tasksSummary);
+            }
+        }
+    }
+
+    /**
+     * Action to be implemented by the specific batching implementation
+     * All tasks have the given batching key.
+     */
+    protected abstract void run(Object batchingKey, List<? extends BatchedTask> tasks, String tasksSummary);
+
+    /**
+     * Represents a runnable task that supports batching.
+     * Implementors of TaskBatcher can subclass this to add a payload to the task.
+     */
+    protected abstract class BatchedTask extends SourcePrioritizedRunnable {
+        /**
+         * whether the task has been processed already
+         */
+        protected final AtomicBoolean processed = new AtomicBoolean();
+
+        /**
+         * the object that is used as batching key
+         */
+        protected final Object batchingKey;
+        /**
+         * the task object that is wrapped
+         */
+        protected final Object task;
+
+        protected BatchedTask(Priority priority, String source, Object batchingKey, Object task) {
+            super(priority, source);
+            this.batchingKey = batchingKey;
+            this.task = task;
+        }
+
+        @Override
+        public void run() {
+            runIfNotProcessed(this);
+        }
+
+        @Override
+        public String toString() {
+            String taskDescription = describeTasks(Collections.singletonList(this));
+            if (taskDescription.isEmpty()) {
+                return "[" + source + "]";
+            } else {
+                return "[" + source + "[" + taskDescription + "]]";
+            }
+        }
+
+        public abstract String describeTasks(List<? extends BatchedTask> tasks);
+
+        public Object getTask() {
+            return task;
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/common/Booleans.java b/core/src/main/java/org/elasticsearch/common/Booleans.java
index 9c5f574663363..90303545cfb60 100644
--- a/core/src/main/java/org/elasticsearch/common/Booleans.java
+++ b/core/src/main/java/org/elasticsearch/common/Booleans.java
@@ -24,30 +24,6 @@
  */
 public class Booleans {
 
-    /**
-     * Returns <code>false</code> if text is in <tt>false</tt>, <tt>0</tt>, <tt>off</tt>, <tt>no</tt>; else, true
-     */
-    public static boolean parseBoolean(char[] text, int offset, int length, boolean defaultValue) {
-        // TODO: the leniency here is very dangerous: a simple typo will be misinterpreted and the user won't know.
-        // We should remove it and cutover to https://github.com/rmuir/booleanparser
-        if (text == null || length == 0) {
-            return defaultValue;
-        }
-        if (length == 1) {
-            return text[offset] != '0';
-        }
-        if (length == 2) {
-            return !(text[offset] == 'n' && text[offset + 1] == 'o');
-        }
-        if (length == 3) {
-            return !(text[offset] == 'o' && text[offset + 1] == 'f' && text[offset + 2] == 'f');
-        }
-        if (length == 5) {
-            return !(text[offset] == 'f' && text[offset + 1] == 'a' && text[offset + 2] == 'l' && text[offset + 3] == 's' && text[offset + 4] == 'e');
-        }
-        return true;
-    }
-
     /**
      * returns true if the a sequence of chars is one of "true","false","on","off","yes","no","0","1"
      *
@@ -78,6 +54,17 @@ public static boolean isBoolean(char[] text, int offset, int length) {
         return false;
     }
 
+    public static boolean isBoolean(String text) {
+        return isExplicitTrue(text) || isExplicitFalse(text);
+    }
+
+    public static Boolean parseBooleanExact(String value, Boolean defaultValue) {
+        if (Strings.hasText(value)) {
+            return parseBooleanExact(value);
+        }
+        return defaultValue;
+    }
+
     /***
      *
      * @return true/false
@@ -96,6 +83,13 @@ public static Boolean parseBooleanExact(String value) {
         throw new IllegalArgumentException("Failed to parse value [" + value + "] cannot be parsed to boolean [ true/1/on/yes OR false/0/off/no ]");
     }
 
+    /**
+     * @return true iff the provided value is either "true" or "false".
+     */
+    public static boolean isStrictlyBoolean(String value) {
+        return "false".equals(value) || "true".equals(value);
+    }
+
     public static Boolean parseBoolean(String value, Boolean defaultValue) {
         if (value == null) { // only for the null case we do that here!
             return defaultValue;
diff --git a/core/src/main/java/org/elasticsearch/common/CheckedBiConsumer.java b/core/src/main/java/org/elasticsearch/common/CheckedBiConsumer.java
new file mode 100644
index 0000000000000..3f8b76bf3653f
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/CheckedBiConsumer.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common;
+
+import java.util.function.BiConsumer;
+
+/**
+ * A {@link BiConsumer}-like interface which allows throwing checked exceptions.
+ */
+@FunctionalInterface
+public interface CheckedBiConsumer<T, U, E extends Exception> {
+    void accept(T t, U u) throws E;
+}
diff --git a/core/src/main/java/org/elasticsearch/common/CheckedConsumer.java b/core/src/main/java/org/elasticsearch/common/CheckedConsumer.java
new file mode 100644
index 0000000000000..1806f42eff0dd
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/CheckedConsumer.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common;
+
+import java.util.function.Consumer;
+
+/**
+ * A {@link Consumer}-like interface which allows throwing checked exceptions.
+ */
+@FunctionalInterface
+public interface CheckedConsumer<T, E extends Exception> {
+    void accept(T t) throws E;
+}
diff --git a/core/src/main/java/org/elasticsearch/common/CheckedFunction.java b/core/src/main/java/org/elasticsearch/common/CheckedFunction.java
new file mode 100644
index 0000000000000..4a2d222db0b7a
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/CheckedFunction.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common;
+
+import java.util.function.Function;
+
+/**
+ * A {@link Function}-like interface which allows throwing checked exceptions.
+ */
+@FunctionalInterface
+public interface CheckedFunction<T, R, E extends Exception> {
+    R apply(T t) throws E;
+}
diff --git a/core/src/main/java/org/elasticsearch/common/CheckedRunnable.java b/core/src/main/java/org/elasticsearch/common/CheckedRunnable.java
new file mode 100644
index 0000000000000..196eb53a878d5
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/CheckedRunnable.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common;
+
+import java.lang.Runnable;
+
+/**
+ * A {@link Runnable}-like interface which allows throwing checked exceptions.
+ */
+@FunctionalInterface
+public interface CheckedRunnable<E extends Exception> {
+    void run() throws E;
+}
diff --git a/core/src/main/java/org/elasticsearch/common/CheckedSupplier.java b/core/src/main/java/org/elasticsearch/common/CheckedSupplier.java
new file mode 100644
index 0000000000000..8f4d2edea0ceb
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/CheckedSupplier.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common;
+
+import java.util.function.Supplier;
+
+/**
+ * A {@link Supplier}-like interface which allows throwing checked exceptions.
+ */
+@FunctionalInterface
+public interface CheckedSupplier<R, E extends Exception> {
+    R get() throws E;
+}
diff --git a/core/src/main/java/org/elasticsearch/common/FieldMemoryStats.java b/core/src/main/java/org/elasticsearch/common/FieldMemoryStats.java
new file mode 100644
index 0000000000000..a09895fdbedce
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/FieldMemoryStats.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common;
+
+import com.carrotsearch.hppc.ObjectLongHashMap;
+import com.carrotsearch.hppc.cursors.ObjectLongCursor;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Objects;
+
+/**
+ * A reusable class to encode <tt>field -&gt; memory size</tt> mappings
+ */
+public final class FieldMemoryStats implements Writeable, Iterable<ObjectLongCursor<String>>{
+
+    private final ObjectLongHashMap<String> stats;
+
+    /**
+     * Creates a new FieldMemoryStats instance
+     */
+    public FieldMemoryStats(ObjectLongHashMap<String> stats) {
+        this.stats = Objects.requireNonNull(stats, "status must be non-null");
+        assert !stats.containsKey(null);
+    }
+
+    /**
+     * Creates a new FieldMemoryStats instance from a stream
+     */
+    public FieldMemoryStats(StreamInput input) throws IOException {
+        int size = input.readVInt();
+        stats = new ObjectLongHashMap<>(size);
+        for (int i = 0; i < size; i++) {
+            stats.put(input.readString(), input.readVLong());
+        }
+    }
+
+    /**
+     * Adds / merges the given field memory stats into this stats instance
+     */
+    public void add(FieldMemoryStats fieldMemoryStats) {
+        for (ObjectLongCursor<String> entry : fieldMemoryStats.stats) {
+            stats.addTo(entry.key, entry.value);
+        }
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeVInt(stats.size());
+        for (ObjectLongCursor<String> entry : stats) {
+            out.writeString(entry.key);
+            out.writeVLong(entry.value);
+        }
+    }
+
+    /**
+     * Generates x-content into the given builder for each of the fields in this stats instance
+     * @param builder the builder to generated on
+     * @param key the top level key for this stats object
+     * @param rawKey the raw byte key for each of the fields byte sizes
+     * @param readableKey the readable key for each of the fields byte sizes
+     */
+    public void toXContent(XContentBuilder builder, String key, String rawKey, String readableKey) throws IOException {
+        builder.startObject(key);
+        for (ObjectLongCursor<String> entry : stats) {
+            builder.startObject(entry.key);
+            builder.byteSizeField(rawKey, readableKey, entry.value);
+            builder.endObject();
+        }
+        builder.endObject();
+    }
+
+    /**
+     * Creates a deep copy of this stats instance
+     */
+    public FieldMemoryStats copy() {
+        return new FieldMemoryStats(stats.clone());
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        FieldMemoryStats that = (FieldMemoryStats) o;
+        return Objects.equals(stats, that.stats);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(stats);
+    }
+
+    @Override
+    public Iterator<ObjectLongCursor<String>> iterator() {
+        return stats.iterator();
+    }
+
+    /**
+     * Returns the fields value in bytes or <code>0</code> if it's not present in the stats
+     */
+    public long get(String field) {
+        return stats.get(field);
+    }
+
+    /**
+     * Returns <code>true</code> iff the given field is in the stats
+     */
+    public boolean containsField(String field) {
+        return stats.containsKey(field);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/common/Numbers.java b/core/src/main/java/org/elasticsearch/common/Numbers.java
index 52d0337ef7300..1735a0dfa6570 100644
--- a/core/src/main/java/org/elasticsearch/common/Numbers.java
+++ b/core/src/main/java/org/elasticsearch/common/Numbers.java
@@ -21,6 +21,9 @@
 
 import org.apache.lucene.util.BytesRef;
 
+import java.math.BigDecimal;
+import java.math.BigInteger;
+
 /**
  * A set of utilities for numbers.
  */
@@ -178,4 +181,56 @@ public static boolean isValidDouble(double value) {
         }
         return true;
     }
+
+    /** Return the long that {@code n} stores, or throws an exception if the
+     *  stored value cannot be converted to a long that stores the exact same
+     *  value. */
+    public static long toLongExact(Number n) {
+        if (n instanceof Byte || n instanceof Short || n instanceof Integer
+                || n instanceof Long) {
+            return n.longValue();
+        } else if (n instanceof Float || n instanceof Double) {
+            double d = n.doubleValue();
+            if (d != Math.round(d)) {
+                throw new IllegalArgumentException(n + " is not an integer value");
+            }
+            return n.longValue();
+        } else if (n instanceof BigDecimal) {
+            return ((BigDecimal) n).toBigIntegerExact().longValueExact();
+        } else if (n instanceof BigInteger) {
+            return ((BigInteger) n).longValueExact();
+        } else {
+            throw new IllegalArgumentException("Cannot check whether [" + n + "] of class [" + n.getClass().getName()
+                    + "] is actually a long");
+        }
+    }
+
+    /** Return the int that {@code n} stores, or throws an exception if the
+     *  stored value cannot be converted to an int that stores the exact same
+     *  value. */
+    public static int toIntExact(Number n) {
+        return Math.toIntExact(toLongExact(n));
+    }
+
+    /** Return the short that {@code n} stores, or throws an exception if the
+     *  stored value cannot be converted to a short that stores the exact same
+     *  value. */
+    public static short toShortExact(Number n) {
+        long l = toLongExact(n);
+        if (l != (short) l) {
+            throw new ArithmeticException("short overflow: " + l);
+        }
+        return (short) l;
+    }
+
+    /** Return the byte that {@code n} stores, or throws an exception if the
+     *  stored value cannot be converted to a byte that stores the exact same
+     *  value. */
+    public static byte toByteExact(Number n) {
+        long l = toLongExact(n);
+        if (l != (byte) l) {
+            throw new ArithmeticException("byte overflow: " + l);
+        }
+        return (byte) l;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/ParseField.java b/core/src/main/java/org/elasticsearch/common/ParseField.java
index 7121be7d1d880..fc9377eeb2f20 100644
--- a/core/src/main/java/org/elasticsearch/common/ParseField.java
+++ b/core/src/main/java/org/elasticsearch/common/ParseField.java
@@ -101,14 +101,10 @@ public ParseField withAllDeprecated(String allReplacedWith) {
     /**
      * @param fieldName
      *            the field name to match against this {@link ParseField}
-     * @param strict
-     *            if true an exception will be thrown if a deprecated field name
-     *            is given. If false the deprecated name will be matched but a
-     *            message will also be logged to the {@link DeprecationLogger}
      * @return true if <code>fieldName</code> matches any of the acceptable
      *         names for this {@link ParseField}.
      */
-    boolean match(String fieldName, boolean strict) {
+    public boolean match(String fieldName) {
         Objects.requireNonNull(fieldName, "fieldName cannot be null");
         // if this parse field has not been completely deprecated then try to
         // match the preferred name
@@ -128,11 +124,7 @@ boolean match(String fieldName, boolean strict) {
                     // message to indicate what should be used instead
                     msg = "Deprecated field [" + fieldName + "] used, replaced by [" + allReplacedWith + "]";
                 }
-                if (strict) {
-                    throw new IllegalArgumentException(msg);
-                } else {
-                    DEPRECATION_LOGGER.deprecated(msg);
-                }
+                DEPRECATION_LOGGER.deprecated(msg);
                 return true;
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/common/ParseFieldMatcher.java b/core/src/main/java/org/elasticsearch/common/ParseFieldMatcher.java
deleted file mode 100644
index 9866694a230ef..0000000000000
--- a/core/src/main/java/org/elasticsearch/common/ParseFieldMatcher.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.common;
-
-import org.elasticsearch.common.settings.Settings;
-
-/**
- * Matcher to use in combination with {@link ParseField} while parsing requests. Matches a {@link ParseField}
- * against a field name and throw deprecation exception depending on the current value of the {@link #PARSE_STRICT} setting.
- */
-public class ParseFieldMatcher {
-    public static final String PARSE_STRICT = "index.query.parse.strict";
-    public static final ParseFieldMatcher EMPTY = new ParseFieldMatcher(false);
-    public static final ParseFieldMatcher STRICT = new ParseFieldMatcher(true);
-
-    private final boolean strict;
-
-    public ParseFieldMatcher(Settings settings) {
-        this(settings.getAsBoolean(PARSE_STRICT, false));
-    }
-
-    public ParseFieldMatcher(boolean strict) {
-        this.strict = strict;
-    }
-
-    /** Should deprecated settings be rejected? */
-    public boolean isStrict() {
-        return strict;
-    }
-
-    /**
-     * Matches a {@link ParseField} against a field name, and throws deprecation exception depending on the current
-     * value of the {@link #PARSE_STRICT} setting.
-     * @param fieldName the field name found in the request while parsing
-     * @param parseField the parse field that we are looking for
-     * @throws IllegalArgumentException whenever we are in strict mode and the request contained a deprecated field
-     * @return true whenever the parse field that we are looking for was found, false otherwise
-     */
-    public boolean match(String fieldName, ParseField parseField) {
-        return parseField.match(fieldName, strict);
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/common/ParseFieldMatcherSupplier.java b/core/src/main/java/org/elasticsearch/common/ParseFieldMatcherSupplier.java
deleted file mode 100644
index 672890c2b9725..0000000000000
--- a/core/src/main/java/org/elasticsearch/common/ParseFieldMatcherSupplier.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.common;
-
-import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.index.query.QueryShardContext;
-
-/**
- * This interface should be implemented by classes like {@link QueryParseContext} or {@link QueryShardContext} that
- * are able to carry a {@link ParseFieldMatcher}.
- */
-@FunctionalInterface
-public interface ParseFieldMatcherSupplier {
-
-    /**
-     * @return the parseFieldMatcher
-     */
-    ParseFieldMatcher getParseFieldMatcher();
-}
diff --git a/core/src/main/java/org/elasticsearch/common/ParsingException.java b/core/src/main/java/org/elasticsearch/common/ParsingException.java
index 0519ab38339c3..5dc2c8a74e4c5 100644
--- a/core/src/main/java/org/elasticsearch/common/ParsingException.java
+++ b/core/src/main/java/org/elasticsearch/common/ParsingException.java
@@ -95,12 +95,11 @@ public RestStatus status() {
     }
 
     @Override
-    protected void innerToXContent(XContentBuilder builder, Params params) throws IOException {
+    protected void metadataToXContent(XContentBuilder builder, Params params) throws IOException {
         if (lineNumber != UNKNOWN_POSITION) {
             builder.field("line", lineNumber);
             builder.field("col", columnNumber);
         }
-        super.innerToXContent(builder, params);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/common/Strings.java b/core/src/main/java/org/elasticsearch/common/Strings.java
index 955b836ca1cbf..ce1cda593c82c 100644
--- a/core/src/main/java/org/elasticsearch/common/Strings.java
+++ b/core/src/main/java/org/elasticsearch/common/Strings.java
@@ -734,10 +734,12 @@ public static Set<String> commaDelimitedListToSet(String str) {
      * @return the delimited String
      */
     public static String collectionToDelimitedString(Iterable<?> coll, String delim, String prefix, String suffix) {
-        return collectionToDelimitedString(coll, delim, prefix, suffix, new StringBuilder());
+        StringBuilder sb = new StringBuilder();
+        collectionToDelimitedString(coll, delim, prefix, suffix, sb);
+        return sb.toString();
     }
 
-    public static String collectionToDelimitedString(Iterable<?> coll, String delim, String prefix, String suffix, StringBuilder sb) {
+    public static void collectionToDelimitedString(Iterable<?> coll, String delim, String prefix, String suffix, StringBuilder sb) {
         Iterator<?> it = coll.iterator();
         while (it.hasNext()) {
             sb.append(prefix).append(it.next()).append(suffix);
@@ -745,7 +747,6 @@ public static String collectionToDelimitedString(Iterable<?> coll, String delim,
                 sb.append(delim);
             }
         }
-        return sb.toString();
     }
 
     /**
@@ -780,12 +781,14 @@ public static String collectionToCommaDelimitedString(Iterable<?> coll) {
      * @return the delimited String
      */
     public static String arrayToDelimitedString(Object[] arr, String delim) {
-        return arrayToDelimitedString(arr, delim, new StringBuilder());
+        StringBuilder sb = new StringBuilder();
+        arrayToDelimitedString(arr, delim, sb);
+        return sb.toString();
     }
 
-    public static String arrayToDelimitedString(Object[] arr, String delim, StringBuilder sb) {
+    public static void arrayToDelimitedString(Object[] arr, String delim, StringBuilder sb) {
         if (isEmpty(arr)) {
-            return "";
+            return;
         }
         for (int i = 0; i < arr.length; i++) {
             if (i > 0) {
@@ -793,7 +796,6 @@ public static String arrayToDelimitedString(Object[] arr, String delim, StringBu
             }
             sb.append(arr[i]);
         }
-        return sb.toString();
     }
 
     /**
@@ -880,26 +882,17 @@ public static boolean isAllOrWildcard(String[] data) {
     }
 
     /**
-     * Return a {@link String} that is the json representation of the provided
-     * {@link ToXContent}.
+     * Return a {@link String} that is the json representation of the provided {@link ToXContent}.
+     * Wraps the output into an anonymous object.
      */
     public static String toString(ToXContent toXContent) {
-        return toString(toXContent, false);
-    }
-
-    /**
-     * Return a {@link String} that is the json representation of the provided
-     * {@link ToXContent}.
-     * @param wrapInObject set this to true if the ToXContent instance expects to be inside an object
-     */
-    public static String toString(ToXContent toXContent, boolean wrapInObject) {
         try {
             XContentBuilder builder = JsonXContent.contentBuilder();
-            if (wrapInObject) {
+            if (toXContent.isFragment()) {
                 builder.startObject();
             }
             toXContent.toXContent(builder, ToXContent.EMPTY_PARAMS);
-            if (wrapInObject) {
+            if (toXContent.isFragment()) {
                 builder.endObject();
             }
             return builder.string();
diff --git a/core/src/main/java/org/elasticsearch/common/Table.java b/core/src/main/java/org/elasticsearch/common/Table.java
index ab0252b11dc33..430070ee19cca 100644
--- a/core/src/main/java/org/elasticsearch/common/Table.java
+++ b/core/src/main/java/org/elasticsearch/common/Table.java
@@ -30,8 +30,6 @@
 
 import static java.util.Collections.emptyMap;
 
-/**
- */
 public class Table {
 
     private List<Cell> headers = new ArrayList<>();
@@ -197,6 +195,22 @@ public Cell findHeaderByName(String header) {
         return null;
     }
 
+    public Map<String, String> getAliasMap() {
+        Map<String, String> headerAliasMap = new HashMap<>();
+        for (int i = 0; i < headers.size(); i++) {
+            Cell headerCell = headers.get(i);
+            String headerName = headerCell.value.toString();
+            if (headerCell.attr.containsKey("alias")) {
+                String[] aliases = Strings.splitStringByCommaToArray(headerCell.attr.get("alias"));
+                for (String alias : aliases) {
+                    headerAliasMap.put(alias, headerName);
+                }
+            }
+            headerAliasMap.put(headerName, headerName);
+        }
+        return headerAliasMap;
+    }
+
     public static class Cell {
         public final Object value;
         public final Map<String, String> attr;
diff --git a/core/src/main/java/org/elasticsearch/common/blobstore/BlobContainer.java b/core/src/main/java/org/elasticsearch/common/blobstore/BlobContainer.java
index 1427f34a642a7..a04c75941e7b6 100644
--- a/core/src/main/java/org/elasticsearch/common/blobstore/BlobContainer.java
+++ b/core/src/main/java/org/elasticsearch/common/blobstore/BlobContainer.java
@@ -105,8 +105,11 @@ public interface BlobContainer {
     Map<String, BlobMetaData> listBlobsByPrefix(String blobNamePrefix) throws IOException;
 
     /**
-     * Atomically renames the source blob into the target blob.  If the source blob does not exist or the
-     * target blob already exists, an exception is thrown.
+     * Renames the source blob into the target blob.  If the source blob does not exist or the
+     * target blob already exists, an exception is thrown.  Atomicity of the move operation
+     * can only be guaranteed on an implementation-by-implementation basis.  The only current
+     * implementation of {@link BlobContainer} for which atomicity can be guaranteed is the
+     * {@link org.elasticsearch.common.blobstore.fs.FsBlobContainer}.
      *
      * @param   sourceBlobName
      *          The blob to rename.
diff --git a/core/src/main/java/org/elasticsearch/common/blobstore/BlobPath.java b/core/src/main/java/org/elasticsearch/common/blobstore/BlobPath.java
index 9092e13eb1b40..ea02aebb0aaad 100644
--- a/core/src/main/java/org/elasticsearch/common/blobstore/BlobPath.java
+++ b/core/src/main/java/org/elasticsearch/common/blobstore/BlobPath.java
@@ -55,15 +55,14 @@ public String[] toArray() {
     }
 
     public BlobPath add(String path) {
-        List<String> paths = new ArrayList<>();
-        paths.addAll(this.paths);
+        List<String> paths = new ArrayList<>(this.paths);
         paths.add(path);
         return new BlobPath(Collections.unmodifiableList(paths));
     }
 
     public String buildAsString() {
         String p = String.join(SEPARATOR, paths);
-        if (p.isEmpty()) {
+        if (p.isEmpty() || p.endsWith(SEPARATOR)) {
             return p;
         }
         return p + SEPARATOR;
diff --git a/core/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobStore.java b/core/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobStore.java
index ad9fe257cf77f..725535ecadbd8 100644
--- a/core/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobStore.java
+++ b/core/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobStore.java
@@ -46,7 +46,7 @@ public FsBlobStore(Settings settings, Path path) throws IOException {
         super(settings);
         this.path = path;
         Files.createDirectories(path);
-        this.bufferSizeInBytes = (int) settings.getAsBytesSize("repositories.fs.buffer_size", new ByteSizeValue(100, ByteSizeUnit.KB)).bytes();
+        this.bufferSizeInBytes = (int) settings.getAsBytesSize("repositories.fs.buffer_size", new ByteSizeValue(100, ByteSizeUnit.KB)).getBytes();
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java b/core/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java
index 537031ef7783a..ede57d461a13d 100644
--- a/core/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java
+++ b/core/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java
@@ -24,9 +24,11 @@
 import org.elasticsearch.common.blobstore.support.AbstractBlobContainer;
 
 import java.io.BufferedInputStream;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.URL;
+import java.nio.file.NoSuchFileException;
 import java.util.Map;
 
 /**
@@ -99,7 +101,11 @@ public boolean blobExists(String blobName) {
 
     @Override
     public InputStream readBlob(String name) throws IOException {
-        return new BufferedInputStream(new URL(path, name).openStream(), blobStore.bufferSizeInBytes());
+        try {
+            return new BufferedInputStream(new URL(path, name).openStream(), blobStore.bufferSizeInBytes());
+        } catch (FileNotFoundException fnfe) {
+            throw new NoSuchFileException("[" + name + "] blob not found");
+        }
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java b/core/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java
index 813a12571ec35..2386f79c076f1 100644
--- a/core/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java
+++ b/core/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java
@@ -55,7 +55,7 @@ public class URLBlobStore extends AbstractComponent implements BlobStore {
     public URLBlobStore(Settings settings, URL path) {
         super(settings);
         this.path = path;
-        this.bufferSizeInBytes = (int) settings.getAsBytesSize("repositories.uri.buffer_size", new ByteSizeValue(100, ByteSizeUnit.KB)).bytes();
+        this.bufferSizeInBytes = (int) settings.getAsBytesSize("repositories.uri.buffer_size", new ByteSizeValue(100, ByteSizeUnit.KB)).getBytes();
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/common/breaker/ChildMemoryCircuitBreaker.java b/core/src/main/java/org/elasticsearch/common/breaker/ChildMemoryCircuitBreaker.java
index 68bf52e9e0dba..e4019f9f6656d 100644
--- a/core/src/main/java/org/elasticsearch/common/breaker/ChildMemoryCircuitBreaker.java
+++ b/core/src/main/java/org/elasticsearch/common/breaker/ChildMemoryCircuitBreaker.java
@@ -90,12 +90,12 @@ public ChildMemoryCircuitBreaker(BreakerSettings settings, ChildMemoryCircuitBre
     @Override
     public void circuitBreak(String fieldName, long bytesNeeded) {
         this.trippedCount.incrementAndGet();
-        final String message = "[" + this.name + "] Data too large, data for [" +
-                fieldName + "] would be larger than limit of [" +
+        final String message = "[" + this.name + "] Data too large, data for [" + fieldName + "]" +
+                " would be [" + bytesNeeded + "/" + new ByteSizeValue(bytesNeeded) + "]" +
+                ", which is larger than the limit of [" +
                 memoryBytesLimit + "/" + new ByteSizeValue(memoryBytesLimit) + "]";
         logger.debug("{}", message);
-        throw new CircuitBreakingException(message,
-                bytesNeeded, this.memoryBytesLimit);
+        throw new CircuitBreakingException(message, bytesNeeded, memoryBytesLimit);
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/common/breaker/CircuitBreakingException.java b/core/src/main/java/org/elasticsearch/common/breaker/CircuitBreakingException.java
index e700d30164480..e01fe1beee224 100644
--- a/core/src/main/java/org/elasticsearch/common/breaker/CircuitBreakingException.java
+++ b/core/src/main/java/org/elasticsearch/common/breaker/CircuitBreakingException.java
@@ -73,9 +73,8 @@ public RestStatus status() {
     }
 
     @Override
-    protected void innerToXContent(XContentBuilder builder, Params params) throws IOException {
+    protected void metadataToXContent(XContentBuilder builder, Params params) throws IOException {
         builder.field("bytes_wanted", bytesWanted);
         builder.field("bytes_limit", byteLimit);
-        super.innerToXContent(builder, params);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/breaker/MemoryCircuitBreaker.java b/core/src/main/java/org/elasticsearch/common/breaker/MemoryCircuitBreaker.java
index 3ac4a52994d26..dbd1fe92ffe9a 100644
--- a/core/src/main/java/org/elasticsearch/common/breaker/MemoryCircuitBreaker.java
+++ b/core/src/main/java/org/elasticsearch/common/breaker/MemoryCircuitBreaker.java
@@ -57,7 +57,7 @@ public MemoryCircuitBreaker(ByteSizeValue limit, double overheadConstant, Logger
      * @param oldBreaker the previous circuit breaker to inherit the used value from (starting offset)
      */
     public MemoryCircuitBreaker(ByteSizeValue limit, double overheadConstant, MemoryCircuitBreaker oldBreaker, Logger logger) {
-        this.memoryBytesLimit = limit.bytes();
+        this.memoryBytesLimit = limit.getBytes();
         this.overheadConstant = overheadConstant;
         if (oldBreaker == null) {
             this.used = new AtomicLong(0);
@@ -79,7 +79,9 @@ public MemoryCircuitBreaker(ByteSizeValue limit, double overheadConstant, Memory
     @Override
     public void circuitBreak(String fieldName, long bytesNeeded) throws CircuitBreakingException {
         this.trippedCount.incrementAndGet();
-        final String message = "Data too large, data for field [" + fieldName + "] would be larger than limit of [" +
+        final String message = "[" + getName() + "] Data too large, data for field [" + fieldName + "]" +
+                " would be [" + bytesNeeded + "/" + new ByteSizeValue(bytesNeeded) + "]" +
+                ", which is larger than the limit of [" +
                 memoryBytesLimit + "/" + new ByteSizeValue(memoryBytesLimit) + "]";
         logger.debug("{}", message);
         throw new CircuitBreakingException(message, bytesNeeded, memoryBytesLimit);
diff --git a/core/src/main/java/org/elasticsearch/common/bytes/BytesArray.java b/core/src/main/java/org/elasticsearch/common/bytes/BytesArray.java
index 43c1df588b170..9b78c2fe5a788 100644
--- a/core/src/main/java/org/elasticsearch/common/bytes/BytesArray.java
+++ b/core/src/main/java/org/elasticsearch/common/bytes/BytesArray.java
@@ -20,12 +20,6 @@
 package org.elasticsearch.common.bytes;
 
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.io.stream.StreamInput;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.nio.charset.StandardCharsets;
-import java.util.Arrays;
 
 public final class BytesArray extends BytesReference {
 
diff --git a/core/src/main/java/org/elasticsearch/common/bytes/BytesReference.java b/core/src/main/java/org/elasticsearch/common/bytes/BytesReference.java
index f31ea2bbf8214..92632ad7874fd 100644
--- a/core/src/main/java/org/elasticsearch/common/bytes/BytesReference.java
+++ b/core/src/main/java/org/elasticsearch/common/bytes/BytesReference.java
@@ -23,6 +23,7 @@
 import org.apache.lucene.util.BytesRefIterator;
 import org.elasticsearch.common.io.stream.StreamInput;
 
+import java.io.EOFException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -215,6 +216,7 @@ private static void advance(final BytesRef ref, final int length) {
      * that way.
      */
     private static final class MarkSupportingStreamInputWrapper extends StreamInput {
+        // can't use FilterStreamInput it needs to reset the delegate
         private final BytesReference reference;
         private BytesReferenceStreamInput input;
         private int mark = 0;
@@ -254,6 +256,11 @@ public int available() throws IOException {
             return input.available();
         }
 
+        @Override
+        protected void ensureCanReadBytes(int length) throws EOFException {
+            input.ensureCanReadBytes(length);
+        }
+
         @Override
         public void reset() throws IOException {
             input = new BytesReferenceStreamInput(reference.iterator(), reference.length());
diff --git a/core/src/main/java/org/elasticsearch/common/bytes/BytesReferenceStreamInput.java b/core/src/main/java/org/elasticsearch/common/bytes/BytesReferenceStreamInput.java
index 4426ea53efabd..f7f1cdc650199 100644
--- a/core/src/main/java/org/elasticsearch/common/bytes/BytesReferenceStreamInput.java
+++ b/core/src/main/java/org/elasticsearch/common/bytes/BytesReferenceStreamInput.java
@@ -32,17 +32,17 @@
  */
 final class BytesReferenceStreamInput extends StreamInput {
     private final BytesRefIterator iterator;
-    private int sliceOffset;
+    private int sliceIndex;
     private BytesRef slice;
     private final int length; // the total size of the stream
     private int offset; // the current position of the stream
 
-    public BytesReferenceStreamInput(BytesRefIterator iterator, final int length) throws IOException {
+    BytesReferenceStreamInput(BytesRefIterator iterator, final int length) throws IOException {
         this.iterator = iterator;
         this.slice = iterator.next();
         this.length = length;
         this.offset = 0;
-        this.sliceOffset = 0;
+        this.sliceIndex = 0;
     }
 
     @Override
@@ -51,15 +51,15 @@ public byte readByte() throws IOException {
             throw new EOFException();
         }
         maybeNextSlice();
-        byte b = slice.bytes[slice.offset + (sliceOffset++)];
+        byte b = slice.bytes[slice.offset + (sliceIndex++)];
         offset++;
         return b;
     }
 
     private void maybeNextSlice() throws IOException {
-        while (sliceOffset == slice.length) {
+        while (sliceIndex == slice.length) {
             slice = iterator.next();
-            sliceOffset = 0;
+            sliceIndex = 0;
             if (slice == null) {
                 throw new EOFException();
             }
@@ -92,12 +92,12 @@ public int read(final byte[] b, final int bOffset, final int len) throws IOExcep
         int destOffset = bOffset;
         while (remaining > 0) {
             maybeNextSlice();
-            final int currentLen = Math.min(remaining, slice.length - sliceOffset);
+            final int currentLen = Math.min(remaining, slice.length - sliceIndex);
             assert currentLen > 0 : "length has to be > 0 to make progress but was: " + currentLen;
-            System.arraycopy(slice.bytes, slice.offset + sliceOffset, b, destOffset, currentLen);
+            System.arraycopy(slice.bytes, slice.offset + sliceIndex, b, destOffset, currentLen);
             destOffset += currentLen;
             remaining -= currentLen;
-            sliceOffset += currentLen;
+            sliceIndex += currentLen;
             offset += currentLen;
             assert remaining >= 0 : "remaining: " + remaining;
         }
@@ -114,6 +114,14 @@ public int available() throws IOException {
         return length - offset;
     }
 
+    @Override
+    protected void ensureCanReadBytes(int bytesToRead) throws EOFException {
+        int bytesAvailable = length - offset;
+        if (bytesAvailable < bytesToRead) {
+            throw new EOFException("tried to read: " + bytesToRead + " bytes but only " + bytesAvailable + " remaining");
+        }
+    }
+
     @Override
     public long skip(long n) throws IOException {
         final int skip = (int) Math.min(Integer.MAX_VALUE, n);
@@ -121,9 +129,9 @@ public long skip(long n) throws IOException {
         int remaining = numBytesSkipped;
         while (remaining > 0) {
             maybeNextSlice();
-            int currentLen = Math.min(remaining, slice.length - (slice.offset + sliceOffset));
+            int currentLen = Math.min(remaining, slice.length - sliceIndex);
             remaining -= currentLen;
-            sliceOffset += currentLen;
+            sliceIndex += currentLen;
             offset += currentLen;
             assert remaining >= 0 : "remaining: " + remaining;
         }
diff --git a/core/src/main/java/org/elasticsearch/common/bytes/ReleasablePagedBytesReference.java b/core/src/main/java/org/elasticsearch/common/bytes/ReleasablePagedBytesReference.java
index 2700ea4dc135e..ac90e546f7eb5 100644
--- a/core/src/main/java/org/elasticsearch/common/bytes/ReleasablePagedBytesReference.java
+++ b/core/src/main/java/org/elasticsearch/common/bytes/ReleasablePagedBytesReference.java
@@ -30,13 +30,17 @@
  */
 public final class ReleasablePagedBytesReference extends PagedBytesReference implements Releasable {
 
-    public ReleasablePagedBytesReference(BigArrays bigarrays, ByteArray byteArray, int length) {
+    private final Releasable releasable;
+
+    public ReleasablePagedBytesReference(BigArrays bigarrays, ByteArray byteArray, int length,
+                                         Releasable releasable) {
         super(bigarrays, byteArray, length);
+        this.releasable = releasable;
     }
 
     @Override
     public void close() {
-        Releasables.close(byteArray);
+        Releasables.close(releasable);
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/common/cache/Cache.java b/core/src/main/java/org/elasticsearch/common/cache/Cache.java
index a42d01ccf723c..91d011ba03cad 100644
--- a/core/src/main/java/org/elasticsearch/common/cache/Cache.java
+++ b/core/src/main/java/org/elasticsearch/common/cache/Cache.java
@@ -34,6 +34,8 @@
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.function.BiFunction;
+import java.util.function.Consumer;
+import java.util.function.Predicate;
 import java.util.function.ToLongBiFunction;
 
 /**
@@ -67,13 +69,13 @@
  */
 public class Cache<K, V> {
     // positive if entries have an expiration
-    private long expireAfterAccess = -1;
+    private long expireAfterAccessNanos = -1;
 
     // true if entries can expire after access
     private boolean entriesExpireAfterAccess;
 
     // positive if entries have an expiration after write
-    private long expireAfterWrite = -1;
+    private long expireAfterWriteNanos = -1;
 
     // true if entries can expire after initial insertion
     private boolean entriesExpireAfterWrite;
@@ -98,22 +100,32 @@ public class Cache<K, V> {
     Cache() {
     }
 
-    void setExpireAfterAccess(long expireAfterAccess) {
-        if (expireAfterAccess <= 0) {
-            throw new IllegalArgumentException("expireAfterAccess <= 0");
+    void setExpireAfterAccessNanos(long expireAfterAccessNanos) {
+        if (expireAfterAccessNanos <= 0) {
+            throw new IllegalArgumentException("expireAfterAccessNanos <= 0");
         }
-        this.expireAfterAccess = expireAfterAccess;
+        this.expireAfterAccessNanos = expireAfterAccessNanos;
         this.entriesExpireAfterAccess = true;
     }
 
-    void setExpireAfterWrite(long expireAfterWrite) {
-        if (expireAfterWrite <= 0) {
-            throw new IllegalArgumentException("expireAfterWrite <= 0");
+    // pkg-private for testing
+    long getExpireAfterAccessNanos() {
+        return this.expireAfterAccessNanos;
+    }
+
+    void setExpireAfterWriteNanos(long expireAfterWriteNanos) {
+        if (expireAfterWriteNanos <= 0) {
+            throw new IllegalArgumentException("expireAfterWriteNanos <= 0");
         }
-        this.expireAfterWrite = expireAfterWrite;
+        this.expireAfterWriteNanos = expireAfterWriteNanos;
         this.entriesExpireAfterWrite = true;
     }
 
+    // pkg-private for testing
+    long getExpireAfterWriteNanos() {
+        return this.expireAfterWriteNanos;
+    }
+
     void setMaximumWeight(long maximumWeight) {
         if (maximumWeight < 0) {
             throw new IllegalArgumentException("maximumWeight < 0");
@@ -156,7 +168,7 @@ static class Entry<K, V> {
         Entry<K, V> after;
         State state = State.NEW;
 
-        public Entry(K key, V value, long writeTime) {
+        Entry(K key, V value, long writeTime) {
             this.key = key;
             this.value = value;
             this.writeTime = this.accessTime = writeTime;
@@ -183,33 +195,40 @@ private static class CacheSegment<K, V> {
         SegmentStats segmentStats = new SegmentStats();
 
         /**
-         * get an entry from the segment
+         * get an entry from the segment; expired entries will be returned as null but not removed from the cache until the LRU list is
+         * pruned or a manual {@link Cache#refresh()} is performed however a caller can take action using the provided callback
          *
-         * @param key the key of the entry to get from the cache
-         * @param now the access time of this entry
+         * @param key       the key of the entry to get from the cache
+         * @param now       the access time of this entry
+         * @param isExpired test if the entry is expired
+         * @param onExpiration a callback if the entry associated to the key is expired
          * @return the entry if there was one, otherwise null
          */
-        Entry<K, V> get(K key, long now) {
+        Entry<K, V> get(K key, long now, Predicate<Entry<K, V>> isExpired, Consumer<Entry<K, V>> onExpiration) {
             CompletableFuture<Entry<K, V>> future;
             Entry<K, V> entry = null;
             try (ReleasableLock ignored = readLock.acquire()) {
                 future = map.get(key);
             }
             if (future != null) {
-              try {
-                  entry = future.handle((ok, ex) -> {
-                      if (ok != null) {
-                          segmentStats.hit();
-                          ok.accessTime = now;
-                          return ok;
-                      } else {
-                          segmentStats.miss();
-                          return null;
-                      }
-                  }).get();
-              } catch (ExecutionException | InterruptedException e) {
-                  throw new IllegalStateException(e);
-              }
+                try {
+                    entry = future.handle((ok, ex) -> {
+                        if (ok != null && !isExpired.test(ok)) {
+                            segmentStats.hit();
+                            ok.accessTime = now;
+                            return ok;
+                        } else {
+                            segmentStats.miss();
+                            if (ok != null) {
+                                assert isExpired.test(ok);
+                                onExpiration.accept(ok);
+                            }
+                            return null;
+                        }
+                    }).get();
+                } catch (ExecutionException | InterruptedException e) {
+                    throw new IllegalStateException(e);
+                }
             }
             else {
                 segmentStats.miss();
@@ -317,13 +336,13 @@ void eviction() {
      * @return the value to which the specified key is mapped, or null if this map contains no mapping for the key
      */
     public V get(K key) {
-        return get(key, now());
+        return get(key, now(), e -> {});
     }
 
-    private V get(K key, long now) {
+    private V get(K key, long now, Consumer<Entry<K, V>> onExpiration) {
         CacheSegment<K, V> segment = getCacheSegment(key);
-        Entry<K, V> entry = segment.get(key, now);
-        if (entry == null || isExpired(entry, now)) {
+        Entry<K, V> entry = segment.get(key, now, e -> isExpired(e, now), onExpiration);
+        if (entry == null) {
             return null;
         } else {
             promote(entry, now);
@@ -336,15 +355,23 @@ private V get(K key, long now) {
      * value using the given mapping function and enters it into this map unless null. The load method for a given key
      * will be invoked at most once.
      *
+     * Use of different {@link CacheLoader} implementations on the same key concurrently may result in only the first
+     * loader function being called and the second will be returned the result provided by the first including any exceptions
+     * thrown during the execution of the first.
+     *
      * @param key    the key whose associated value is to be returned or computed for if non-existent
      * @param loader the function to compute a value given a key
-     * @return the current (existing or computed) value associated with the specified key, or null if the computed
-     * value is null
-     * @throws ExecutionException thrown if loader throws an exception
+     * @return the current (existing or computed) non-null value associated with the specified key
+     * @throws ExecutionException thrown if loader throws an exception or returns a null value
      */
     public V computeIfAbsent(K key, CacheLoader<K, V> loader) throws ExecutionException {
         long now = now();
-        V value = get(key, now);
+        // we have to eagerly evict expired entries or our putIfAbsent call below will fail
+        V value = get(key, now, e -> {
+            try (ReleasableLock ignored = lruLock.acquire()) {
+                evictEntry(e);
+            }
+        });
         if (value == null) {
             // we need to synchronize loading of a value for a given key; however, holding the segment lock while
             // invoking load can lead to deadlock against another thread due to dependent key loading; therefore, we
@@ -400,6 +427,11 @@ public V computeIfAbsent(K key, CacheLoader<K, V> loader) throws ExecutionExcept
 
             try {
                 value = completableValue.get();
+                // check to ensure the future hasn't been completed with an exception
+                if (future.isCompletedExceptionally()) {
+                    future.get(); // call get to force the exception to be thrown for other concurrent callers
+                    throw new IllegalStateException("the future was completed exceptionally but no exception was thrown");
+                }
             } catch (InterruptedException e) {
                 throw new IllegalStateException(e);
             }
@@ -670,13 +702,18 @@ private void evict(long now) {
         assert lruLock.isHeldByCurrentThread();
 
         while (tail != null && shouldPrune(tail, now)) {
-            CacheSegment<K, V> segment = getCacheSegment(tail.key);
-            Entry<K, V> entry = tail;
-            if (segment != null) {
-                segment.remove(tail.key);
-            }
-            delete(entry, RemovalNotification.RemovalReason.EVICTED);
+            evictEntry(tail);
+        }
+    }
+
+    private void evictEntry(Entry<K, V> entry) {
+        assert lruLock.isHeldByCurrentThread();
+
+        CacheSegment<K, V> segment = getCacheSegment(entry.key);
+        if (segment != null) {
+            segment.remove(entry.key);
         }
+        delete(entry, RemovalNotification.RemovalReason.EVICTED);
     }
 
     private void delete(Entry<K, V> entry, RemovalNotification.RemovalReason removalReason) {
@@ -696,8 +733,8 @@ private boolean exceedsWeight() {
     }
 
     private boolean isExpired(Entry<K, V> entry, long now) {
-        return (entriesExpireAfterAccess && now - entry.accessTime > expireAfterAccess) ||
-                (entriesExpireAfterWrite && now - entry.writeTime > expireAfterWrite);
+        return (entriesExpireAfterAccess && now - entry.accessTime > expireAfterAccessNanos) ||
+                (entriesExpireAfterWrite && now - entry.writeTime > expireAfterWriteNanos);
     }
 
     private boolean unlink(Entry<K, V> entry) {
diff --git a/core/src/main/java/org/elasticsearch/common/cache/CacheBuilder.java b/core/src/main/java/org/elasticsearch/common/cache/CacheBuilder.java
index ffb0e591180ea..67c8d508ba572 100644
--- a/core/src/main/java/org/elasticsearch/common/cache/CacheBuilder.java
+++ b/core/src/main/java/org/elasticsearch/common/cache/CacheBuilder.java
@@ -19,13 +19,15 @@
 
 package org.elasticsearch.common.cache;
 
+import org.elasticsearch.common.unit.TimeValue;
+
 import java.util.Objects;
 import java.util.function.ToLongBiFunction;
 
 public class CacheBuilder<K, V> {
     private long maximumWeight = -1;
-    private long expireAfterAccess = -1;
-    private long expireAfterWrite = -1;
+    private long expireAfterAccessNanos = -1;
+    private long expireAfterWriteNanos = -1;
     private ToLongBiFunction<K, V> weigher;
     private RemovalListener<K, V> removalListener;
 
@@ -44,19 +46,35 @@ public CacheBuilder<K, V> setMaximumWeight(long maximumWeight) {
         return this;
     }
 
-    public CacheBuilder<K, V> setExpireAfterAccess(long expireAfterAccess) {
-        if (expireAfterAccess <= 0) {
+    /**
+     * Sets the amount of time before an entry in the cache expires after it was last accessed.
+     *
+     * @param expireAfterAccess The amount of time before an entry expires after it was last accessed. Must not be {@code null} and must
+     *                          be greater than 0.
+     */
+    public CacheBuilder<K, V> setExpireAfterAccess(TimeValue expireAfterAccess) {
+        Objects.requireNonNull(expireAfterAccess);
+        final long expireAfterAccessNanos = expireAfterAccess.getNanos();
+        if (expireAfterAccessNanos <= 0) {
             throw new IllegalArgumentException("expireAfterAccess <= 0");
         }
-        this.expireAfterAccess = expireAfterAccess;
+        this.expireAfterAccessNanos = expireAfterAccessNanos;
         return this;
     }
 
-    public CacheBuilder<K, V> setExpireAfterWrite(long expireAfterWrite) {
-        if (expireAfterWrite <= 0) {
+    /**
+     * Sets the amount of time before an entry in the cache expires after it was written.
+     *
+     * @param expireAfterWrite The amount of time before an entry expires after it was written. Must not be {@code null} and must be
+     *                         greater than 0.
+     */
+    public CacheBuilder<K, V> setExpireAfterWrite(TimeValue expireAfterWrite) {
+        Objects.requireNonNull(expireAfterWrite);
+        final long expireAfterWriteNanos = expireAfterWrite.getNanos();
+        if (expireAfterWriteNanos <= 0) {
             throw new IllegalArgumentException("expireAfterWrite <= 0");
         }
-        this.expireAfterWrite = expireAfterWrite;
+        this.expireAfterWriteNanos = expireAfterWriteNanos;
         return this;
     }
 
@@ -77,11 +95,11 @@ public Cache<K, V> build() {
         if (maximumWeight != -1) {
             cache.setMaximumWeight(maximumWeight);
         }
-        if (expireAfterAccess != -1) {
-            cache.setExpireAfterAccess(expireAfterAccess);
+        if (expireAfterAccessNanos != -1) {
+            cache.setExpireAfterAccessNanos(expireAfterAccessNanos);
         }
-        if (expireAfterWrite != -1) {
-            cache.setExpireAfterWrite(expireAfterWrite);
+        if (expireAfterWriteNanos != -1) {
+            cache.setExpireAfterWriteNanos(expireAfterWriteNanos);
         }
         if (weigher != null) {
             cache.setWeigher(weigher);
diff --git a/core/src/main/java/org/elasticsearch/common/collect/CopyOnWriteHashMap.java b/core/src/main/java/org/elasticsearch/common/collect/CopyOnWriteHashMap.java
index 5316768673660..85d7eda836395 100644
--- a/core/src/main/java/org/elasticsearch/common/collect/CopyOnWriteHashMap.java
+++ b/core/src/main/java/org/elasticsearch/common/collect/CopyOnWriteHashMap.java
@@ -433,7 +433,7 @@ private static class EntryIterator<K, V> implements Iterator<Map.Entry<K, V>> {
         private final Deque<Map.Entry<K, V>> entries;
         private final Deque<Node<K, V>> nodes;
 
-        public EntryIterator(Node<K, V> node) {
+        EntryIterator(Node<K, V> node) {
             entries = new ArrayDeque<>();
             nodes = new ArrayDeque<>();
             node.visit(entries, nodes);
diff --git a/core/src/main/java/org/elasticsearch/common/collect/Iterators.java b/core/src/main/java/org/elasticsearch/common/collect/Iterators.java
index d44bf7341c47b..a8c811a0d067d 100644
--- a/core/src/main/java/org/elasticsearch/common/collect/Iterators.java
+++ b/core/src/main/java/org/elasticsearch/common/collect/Iterators.java
@@ -36,7 +36,7 @@ static class ConcatenatedIterator<T> implements Iterator<T> {
         private final Iterator<? extends T>[] iterators;
         private int index = 0;
 
-        public ConcatenatedIterator(Iterator<? extends T>... iterators) {
+        ConcatenatedIterator(Iterator<? extends T>... iterators) {
             if (iterators == null) {
                 throw new NullPointerException("iterators");
             }
diff --git a/core/src/main/java/org/elasticsearch/common/component/AbstractLifecycleComponent.java b/core/src/main/java/org/elasticsearch/common/component/AbstractLifecycleComponent.java
index 6f1534b57d831..49e8b40b350c7 100644
--- a/core/src/main/java/org/elasticsearch/common/component/AbstractLifecycleComponent.java
+++ b/core/src/main/java/org/elasticsearch/common/component/AbstractLifecycleComponent.java
@@ -21,6 +21,7 @@
 
 import org.elasticsearch.common.settings.Settings;
 
+import java.io.IOException;
 import java.util.List;
 import java.util.concurrent.CopyOnWriteArrayList;
 
@@ -104,11 +105,17 @@ public void close() {
             listener.beforeClose();
         }
         lifecycle.moveToClosed();
-        doClose();
+        try {
+            doClose();
+        } catch (IOException e) {
+            // TODO: we need to separate out closing (ie shutting down) services, vs releasing runtime transient
+            // structures. Shutting down services should use IOUtils.close
+            logger.warn("failed to close " + getClass().getName(), e);
+        }
         for (LifecycleListener listener : listeners) {
             listener.afterClose();
         }
     }
 
-    protected abstract void doClose();
+    protected abstract void doClose() throws IOException;
 }
diff --git a/core/src/main/java/org/elasticsearch/common/compress/Compressor.java b/core/src/main/java/org/elasticsearch/common/compress/Compressor.java
index 883078dafe8ce..edd6f2fde7b03 100644
--- a/core/src/main/java/org/elasticsearch/common/compress/Compressor.java
+++ b/core/src/main/java/org/elasticsearch/common/compress/Compressor.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.common.compress;
 
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 
@@ -33,5 +34,9 @@ public interface Compressor {
 
     StreamInput streamInput(StreamInput in) throws IOException;
 
+    /**
+     * Creates a new stream output that compresses the contents and writes to the provided stream
+     * output. Closing the returned {@link StreamOutput} will close the provided stream output.
+     */
     StreamOutput streamOutput(StreamOutput out) throws IOException;
 }
diff --git a/core/src/main/java/org/elasticsearch/common/compress/CompressorFactory.java b/core/src/main/java/org/elasticsearch/common/compress/CompressorFactory.java
index 82e049704cc82..0bcc54d3d2992 100644
--- a/core/src/main/java/org/elasticsearch/common/compress/CompressorFactory.java
+++ b/core/src/main/java/org/elasticsearch/common/compress/CompressorFactory.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.common.xcontent.XContentType;
 
 import java.io.IOException;
+import java.util.Objects;
 
 /**
  */
@@ -70,9 +71,10 @@ private static boolean isAncient(BytesReference bytes) {
 
     /**
      * Uncompress the provided data, data can be detected as compressed using {@link #isCompressed(BytesReference)}.
+     * @throws NullPointerException a NullPointerException will be thrown when bytes is null
      */
     public static BytesReference uncompressIfNeeded(BytesReference bytes) throws IOException {
-        Compressor compressor = compressor(bytes);
+        Compressor compressor = compressor(Objects.requireNonNull(bytes, "the BytesReference must not be null"));
         BytesReference uncompressed;
         if (compressor != null) {
             uncompressed = uncompress(bytes, compressor);
diff --git a/core/src/main/java/org/elasticsearch/common/compress/DeflateCompressor.java b/core/src/main/java/org/elasticsearch/common/compress/DeflateCompressor.java
index 42e2efa358cfa..794a8db4960c6 100644
--- a/core/src/main/java/org/elasticsearch/common/compress/DeflateCompressor.java
+++ b/core/src/main/java/org/elasticsearch/common/compress/DeflateCompressor.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.common.compress;
 
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.compress.Compressor;
 import org.elasticsearch.common.io.stream.InputStreamStreamInput;
 import org.elasticsearch.common.io.stream.OutputStreamStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -47,7 +46,7 @@ public class DeflateCompressor implements Compressor {
     // It needs to be different from other compressors and to not be specific
     // enough so that no stream starting with these bytes could be detected as
     // a XContent
-    private static final byte[] HEADER = new byte[] { 'D', 'F', 'L', '\0' };
+    private static final byte[] HEADER = new byte[]{'D', 'F', 'L', '\0'};
     // 3 is a good trade-off between speed and compression ratio
     private static final int LEVEL = 3;
     // We use buffering on the input and output of in/def-laters in order to
@@ -88,6 +87,7 @@ public StreamInput streamInput(StreamInput in) throws IOException {
         decompressedIn = new BufferedInputStream(decompressedIn, BUFFER_SIZE);
         return new InputStreamStreamInput(decompressedIn) {
             final AtomicBoolean closed = new AtomicBoolean(false);
+
             public void close() throws IOException {
                 try {
                     super.close();
@@ -107,10 +107,11 @@ public StreamOutput streamOutput(StreamOutput out) throws IOException {
         final boolean nowrap = true;
         final Deflater deflater = new Deflater(LEVEL, nowrap);
         final boolean syncFlush = true;
-        OutputStream compressedOut = new DeflaterOutputStream(out, deflater, BUFFER_SIZE, syncFlush);
-        compressedOut = new BufferedOutputStream(compressedOut, BUFFER_SIZE);
+        DeflaterOutputStream deflaterOutputStream = new DeflaterOutputStream(out, deflater, BUFFER_SIZE, syncFlush);
+        OutputStream compressedOut = new BufferedOutputStream(deflaterOutputStream, BUFFER_SIZE);
         return new OutputStreamStreamOutput(compressedOut) {
             final AtomicBoolean closed = new AtomicBoolean(false);
+
             public void close() throws IOException {
                 try {
                     super.close();
diff --git a/core/src/main/java/org/elasticsearch/common/geo/GeoDistance.java b/core/src/main/java/org/elasticsearch/common/geo/GeoDistance.java
index f63636174f6ce..6b77d6dae6489 100644
--- a/core/src/main/java/org/elasticsearch/common/geo/GeoDistance.java
+++ b/core/src/main/java/org/elasticsearch/common/geo/GeoDistance.java
@@ -19,18 +19,13 @@
 
 package org.elasticsearch.common.geo;
 
-import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.SloppyMath;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.unit.DistanceUnit;
-import org.elasticsearch.index.fielddata.FieldData;
-import org.elasticsearch.index.fielddata.GeoPointValues;
-import org.elasticsearch.index.fielddata.MultiGeoPointValues;
-import org.elasticsearch.index.fielddata.NumericDoubleValues;
-import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
-import org.elasticsearch.index.fielddata.SortingNumericDoubleValues;
 
 import java.io.IOException;
 import java.util.Locale;
@@ -39,164 +34,57 @@
  * Geo distance calculation.
  */
 public enum GeoDistance implements Writeable {
-    /**
-     * Calculates distance as points on a plane. Faster, but less accurate than {@link #ARC}.
-     * @deprecated use {@link GeoUtils#planeDistance}
-     */
-    @Deprecated
-    PLANE {
-        @Override
-        public double calculate(double sourceLatitude, double sourceLongitude, double targetLatitude, double targetLongitude, DistanceUnit unit) {
-            double px = targetLongitude - sourceLongitude;
-            double py = targetLatitude - sourceLatitude;
-            return Math.sqrt(px * px + py * py) * unit.getDistancePerDegree();
-        }
-
-        @Override
-        public double normalize(double distance, DistanceUnit unit) {
-            return distance;
-        }
-
-        @Override
-        public FixedSourceDistance fixedSourceDistance(double sourceLatitude, double sourceLongitude, DistanceUnit unit) {
-            return new PlaneFixedSourceDistance(sourceLatitude, sourceLongitude, unit);
-        }
-    },
-
-    /**
-     * Calculates distance factor.
-     * Note: {@code calculate} is simply returning the RHS of the spherical law of cosines from 2 lat,lon points.
-     * {@code normalize} also returns the RHS of the spherical law of cosines for a given distance
-     * @deprecated use {@link SloppyMath#haversinMeters} to get distance in meters, law of cosines is being removed
-     */
-    @Deprecated
-    FACTOR {
-        @Override
-        public double calculate(double sourceLatitude, double sourceLongitude, double targetLatitude, double targetLongitude, DistanceUnit unit) {
-            double longitudeDifference = targetLongitude - sourceLongitude;
-            double a = Math.toRadians(90D - sourceLatitude);
-            double c = Math.toRadians(90D - targetLatitude);
-            return (Math.cos(a) * Math.cos(c)) + (Math.sin(a) * Math.sin(c) * Math.cos(Math.toRadians(longitudeDifference)));
-        }
-
-        @Override
-        public double normalize(double distance, DistanceUnit unit) {
-            return Math.cos(distance / unit.getEarthRadius());
-        }
-
-        @Override
-        public FixedSourceDistance fixedSourceDistance(double sourceLatitude, double sourceLongitude, DistanceUnit unit) {
-            return new FactorFixedSourceDistance(sourceLatitude, sourceLongitude);
-        }
-    },
-    /**
-     * Calculates distance as points on a globe.
-     * @deprecated use {@link GeoUtils#arcDistance}
-     */
-    @Deprecated
-    ARC {
-        @Override
-        public double calculate(double sourceLatitude, double sourceLongitude, double targetLatitude, double targetLongitude, DistanceUnit unit) {
-            double result = SloppyMath.haversinMeters(sourceLatitude, sourceLongitude, targetLatitude, targetLongitude);
-            return unit.fromMeters(result);
-        }
-
-        @Override
-        public double normalize(double distance, DistanceUnit unit) {
-            return distance;
-        }
-
-        @Override
-        public FixedSourceDistance fixedSourceDistance(double sourceLatitude, double sourceLongitude, DistanceUnit unit) {
-            return new ArcFixedSourceDistance(sourceLatitude, sourceLongitude, unit);
-        }
-    },
-    /**
-     * Calculates distance as points on a globe in a sloppy way. Close to the pole areas the accuracy
-     * of this function decreases.
-     */
-    @Deprecated
-    SLOPPY_ARC {
+    PLANE, ARC;
 
-        @Override
-        public double normalize(double distance, DistanceUnit unit) {
-            return distance;
-        }
-
-        @Override
-        public double calculate(double sourceLatitude, double sourceLongitude, double targetLatitude, double targetLongitude, DistanceUnit unit) {
-            return unit.fromMeters(SloppyMath.haversinMeters(sourceLatitude, sourceLongitude, targetLatitude, targetLongitude));
-        }
-
-        @Override
-        public FixedSourceDistance fixedSourceDistance(double sourceLatitude, double sourceLongitude, DistanceUnit unit) {
-            return new SloppyArcFixedSourceDistance(sourceLatitude, sourceLongitude, unit);
-        }
-    };
+    private static final DeprecationLogger DEPRECATION_LOGGER =
+        new DeprecationLogger(Loggers.getLogger(GeoDistance.class));
 
+    /** Creates a GeoDistance instance from an input stream */
     public static GeoDistance readFromStream(StreamInput in) throws IOException {
+        Version clientVersion = in.getVersion();
         int ord = in.readVInt();
+        // bwc client deprecation for FACTOR and SLOPPY_ARC
+        if (clientVersion.before(Version.V_5_3_3)) {
+            switch (ord) {
+                case 0: return PLANE;
+                case 1: // FACTOR uses PLANE
+                    // bwc client deprecation for FACTOR
+                    DEPRECATION_LOGGER.deprecated("[factor] is deprecated. Using [plane] instead.");
+                    return PLANE;
+                case 2: return ARC;
+                case 3: // SLOPPY_ARC uses ARC
+                    // bwc client deprecation for SLOPPY_ARC
+                    DEPRECATION_LOGGER.deprecated("[sloppy_arc] is deprecated. Using [arc] instead.");
+                    return ARC;
+                default:
+                    throw new IOException("Unknown GeoDistance ordinal [" + ord + "]");
+            }
+        }
+
         if (ord < 0 || ord >= values().length) {
             throw new IOException("Unknown GeoDistance ordinal [" + ord + "]");
         }
         return GeoDistance.values()[ord];
     }
 
+    /** Writes an instance of a GeoDistance object to an output stream */
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        out.writeVInt(this.ordinal());
-    }
-
-    /**
-     * Default {@link GeoDistance} function. This method should be used, If no specific function has been selected.
-     * This is an alias for <code>SLOPPY_ARC</code>
-     */
-    @Deprecated
-    public static final GeoDistance DEFAULT = SLOPPY_ARC;
-
-    public abstract double normalize(double distance, DistanceUnit unit);
-
-    public abstract double calculate(double sourceLatitude, double sourceLongitude, double targetLatitude, double targetLongitude, DistanceUnit unit);
-
-    public abstract FixedSourceDistance fixedSourceDistance(double sourceLatitude, double sourceLongitude, DistanceUnit unit);
-
-    private static final double MIN_LAT = Math.toRadians(-90d);  // -PI/2
-    private static final double MAX_LAT = Math.toRadians(90d);   //  PI/2
-    private static final double MIN_LON = Math.toRadians(-180d); // -PI
-    private static final double MAX_LON = Math.toRadians(180d);  //  PI
-
-    public static DistanceBoundingCheck distanceBoundingCheck(double sourceLatitude, double sourceLongitude, double distance, DistanceUnit unit) {
-        // angular distance in radians on a great circle
-        // assume worst-case: use the minor axis
-        double radDist = unit.toMeters(distance) / GeoUtils.EARTH_SEMI_MINOR_AXIS;
-
-        double radLat = Math.toRadians(sourceLatitude);
-        double radLon = Math.toRadians(sourceLongitude);
-
-        double minLat = radLat - radDist;
-        double maxLat = radLat + radDist;
-
-        double minLon, maxLon;
-        if (minLat > MIN_LAT && maxLat < MAX_LAT) {
-            double deltaLon = Math.asin(Math.sin(radDist) / Math.cos(radLat));
-            minLon = radLon - deltaLon;
-            if (minLon < MIN_LON) minLon += 2d * Math.PI;
-            maxLon = radLon + deltaLon;
-            if (maxLon > MAX_LON) maxLon -= 2d * Math.PI;
-        } else {
-            // a pole is within the distance
-            minLat = Math.max(minLat, MIN_LAT);
-            maxLat = Math.min(maxLat, MAX_LAT);
-            minLon = MIN_LON;
-            maxLon = MAX_LON;
+        Version clientVersion = out.getVersion();
+        int ord = this.ordinal();
+        if (clientVersion.before(Version.V_5_3_3)) {
+            switch (ord) {
+                case 0:
+                    out.write(0);  // write PLANE ordinal
+                    return;
+                case 1:
+                    out.write(2);  // write bwc ARC ordinal
+                    return;
+                default:
+                    throw new IOException("Unknown GeoDistance ordinal [" + ord + "]");
+            }
         }
-
-        GeoPoint topLeft = new GeoPoint(Math.toDegrees(maxLat), Math.toDegrees(minLon));
-        GeoPoint bottomRight = new GeoPoint(Math.toDegrees(minLat), Math.toDegrees(maxLon));
-        if (minLon > maxLon) {
-            return new Meridian180DistanceBoundingCheck(topLeft, bottomRight);
-        }
-        return new SimpleDistanceBoundingCheck(topLeft, bottomRight);
+        out.writeVInt(this.ordinal());
     }
 
     /**
@@ -204,8 +92,6 @@ public static DistanceBoundingCheck distanceBoundingCheck(double sourceLatitude,
      *
      * <ul>
      *     <li><b>plane</b> for <code>GeoDistance.PLANE</code></li>
-     *     <li><b>sloppy_arc</b> for <code>GeoDistance.SLOPPY_ARC</code></li>
-     *     <li><b>factor</b> for <code>GeoDistance.FACTOR</code></li>
      *     <li><b>arc</b> for <code>GeoDistance.ARC</code></li>
      * </ul>
      *
@@ -216,224 +102,21 @@ public static GeoDistance fromString(String name) {
         name = name.toLowerCase(Locale.ROOT);
         if ("plane".equals(name)) {
             return PLANE;
+        } else if ("sloppy_arc".equals(name)) {
+            DEPRECATION_LOGGER.deprecated("[sloppy_arc] is deprecated. Use [arc] instead.");
+            return ARC;
         } else if ("arc".equals(name)) {
             return ARC;
-        } else if ("sloppy_arc".equals(name)) {
-            return SLOPPY_ARC;
-        } else if ("factor".equals(name)) {
-            return FACTOR;
         }
         throw new IllegalArgumentException("No geo distance for [" + name + "]");
     }
 
-    public interface FixedSourceDistance {
-
-        double calculate(double targetLatitude, double targetLongitude);
-    }
-
-    public interface DistanceBoundingCheck {
-
-        boolean isWithin(double targetLatitude, double targetLongitude);
-
-        GeoPoint topLeft();
-
-        GeoPoint bottomRight();
-    }
-
-    public static final AlwaysDistanceBoundingCheck ALWAYS_INSTANCE = new AlwaysDistanceBoundingCheck();
-
-    private static class AlwaysDistanceBoundingCheck implements DistanceBoundingCheck {
-        @Override
-        public boolean isWithin(double targetLatitude, double targetLongitude) {
-            return true;
-        }
-
-        @Override
-        public GeoPoint topLeft() {
-            return null;
-        }
-
-        @Override
-        public GeoPoint bottomRight() {
-            return null;
-        }
-    }
-
-    public static class Meridian180DistanceBoundingCheck implements DistanceBoundingCheck {
-
-        private final GeoPoint topLeft;
-        private final GeoPoint bottomRight;
-
-        public Meridian180DistanceBoundingCheck(GeoPoint topLeft, GeoPoint bottomRight) {
-            this.topLeft = topLeft;
-            this.bottomRight = bottomRight;
-        }
-
-        @Override
-        public boolean isWithin(double targetLatitude, double targetLongitude) {
-            return (targetLatitude >= bottomRight.lat() && targetLatitude <= topLeft.lat()) &&
-                    (targetLongitude >= topLeft.lon() || targetLongitude <= bottomRight.lon());
-        }
-
-        @Override
-        public GeoPoint topLeft() {
-            return topLeft;
-        }
-
-        @Override
-        public GeoPoint bottomRight() {
-            return bottomRight;
-        }
-    }
-
-    public static class SimpleDistanceBoundingCheck implements DistanceBoundingCheck {
-        private final GeoPoint topLeft;
-        private final GeoPoint bottomRight;
-
-        public SimpleDistanceBoundingCheck(GeoPoint topLeft, GeoPoint bottomRight) {
-            this.topLeft = topLeft;
-            this.bottomRight = bottomRight;
-        }
-
-        @Override
-        public boolean isWithin(double targetLatitude, double targetLongitude) {
-            return (targetLatitude >= bottomRight.lat() && targetLatitude <= topLeft.lat()) &&
-                    (targetLongitude >= topLeft.lon() && targetLongitude <= bottomRight.lon());
-        }
-
-        @Override
-        public GeoPoint topLeft() {
-            return topLeft;
-        }
-
-        @Override
-        public GeoPoint bottomRight() {
-            return bottomRight;
-        }
-    }
-
-    public static class PlaneFixedSourceDistance implements FixedSourceDistance {
-
-        private final double sourceLatitude;
-        private final double sourceLongitude;
-        private final double distancePerDegree;
-
-        public PlaneFixedSourceDistance(double sourceLatitude, double sourceLongitude, DistanceUnit unit) {
-            this.sourceLatitude = sourceLatitude;
-            this.sourceLongitude = sourceLongitude;
-            this.distancePerDegree = unit.getDistancePerDegree();
-        }
-
-        @Override
-        public double calculate(double targetLatitude, double targetLongitude) {
-            double px = targetLongitude - sourceLongitude;
-            double py = targetLatitude - sourceLatitude;
-            return Math.sqrt(px * px + py * py) * distancePerDegree;
-        }
-    }
-
-    public static class FactorFixedSourceDistance implements FixedSourceDistance {
-
-        private final double sourceLongitude;
-
-        private final double a;
-        private final double sinA;
-        private final double cosA;
-
-        public FactorFixedSourceDistance(double sourceLatitude, double sourceLongitude) {
-            this.sourceLongitude = sourceLongitude;
-            this.a = Math.toRadians(90D - sourceLatitude);
-            this.sinA = Math.sin(a);
-            this.cosA = Math.cos(a);
-        }
-
-        @Override
-        public double calculate(double targetLatitude, double targetLongitude) {
-            double longitudeDifference = targetLongitude - sourceLongitude;
-            double c = Math.toRadians(90D - targetLatitude);
-            return (cosA * Math.cos(c)) + (sinA * Math.sin(c) * Math.cos(Math.toRadians(longitudeDifference)));
-        }
-    }
-
-    /**
-     * Basic implementation of {@link FixedSourceDistance}. This class keeps the basic parameters for a distance
-     * functions based on a fixed source. Namely latitude, longitude and unit.
-     */
-    public abstract static class FixedSourceDistanceBase implements FixedSourceDistance {
-        protected final double sourceLatitude;
-        protected final double sourceLongitude;
-        protected final DistanceUnit unit;
-
-        public FixedSourceDistanceBase(double sourceLatitude, double sourceLongitude, DistanceUnit unit) {
-            this.sourceLatitude = sourceLatitude;
-            this.sourceLongitude = sourceLongitude;
-            this.unit = unit;
-        }
-    }
-
-    public static class ArcFixedSourceDistance extends FixedSourceDistanceBase {
-
-        public ArcFixedSourceDistance(double sourceLatitude, double sourceLongitude, DistanceUnit unit) {
-            super(sourceLatitude, sourceLongitude, unit);
-        }
-
-        @Override
-        public double calculate(double targetLatitude, double targetLongitude) {
-            return ARC.calculate(sourceLatitude, sourceLongitude, targetLatitude, targetLongitude, unit);
-        }
-
-    }
-
-    public static class SloppyArcFixedSourceDistance extends FixedSourceDistanceBase {
-
-        public SloppyArcFixedSourceDistance(double sourceLatitude, double sourceLongitude, DistanceUnit unit) {
-            super(sourceLatitude, sourceLongitude, unit);
-        }
-
-        @Override
-        public double calculate(double targetLatitude, double targetLongitude) {
-            return SLOPPY_ARC.calculate(sourceLatitude, sourceLongitude, targetLatitude, targetLongitude, unit);
-        }
-    }
-
-
-    /**
-     * Return a {@link SortedNumericDoubleValues} instance that returns the distances to a list of geo-points for each document.
-     */
-    public static SortedNumericDoubleValues distanceValues(final MultiGeoPointValues geoPointValues, final FixedSourceDistance... distances) {
-        final GeoPointValues singleValues = FieldData.unwrapSingleton(geoPointValues);
-        if (singleValues != null && distances.length == 1) {
-            final Bits docsWithField = FieldData.unwrapSingletonBits(geoPointValues);
-            return FieldData.singleton(new NumericDoubleValues() {
-
-                @Override
-                public double get(int docID) {
-                    if (docsWithField != null && !docsWithField.get(docID)) {
-                        return 0d;
-                    }
-                    final GeoPoint point = singleValues.get(docID);
-                    return distances[0].calculate(point.lat(), point.lon());
-                }
-
-            }, docsWithField);
-        } else {
-            return new SortingNumericDoubleValues() {
-
-                @Override
-                public void setDocument(int doc) {
-                    geoPointValues.setDocument(doc);
-                    resize(geoPointValues.count() * distances.length);
-                    int valueCounter = 0;
-                    for (FixedSourceDistance distance : distances) {
-                        for (int i = 0; i < geoPointValues.count(); ++i) {
-                            final GeoPoint point = geoPointValues.valueAt(i);
-                            values[valueCounter] = distance.calculate(point.lat(), point.lon());
-                            valueCounter++;
-                        }
-                    }
-                    sort();
-                }
-            };
+    /** compute the distance between two points using the selected algorithm (PLANE, ARC) */
+    public double calculate(double srcLat, double srcLon, double dstLat, double dstLon, DistanceUnit unit) {
+        if (this == PLANE) {
+            return DistanceUnit.convert(GeoUtils.planeDistance(srcLat, srcLon, dstLat, dstLon),
+                DistanceUnit.METERS, unit);
         }
+        return DistanceUnit.convert(GeoUtils.arcDistance(srcLat, srcLon, dstLat, dstLon), DistanceUnit.METERS, unit);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/geo/GeoPoint.java b/core/src/main/java/org/elasticsearch/common/geo/GeoPoint.java
index 504fc41313702..15e2fb4fabbe5 100644
--- a/core/src/main/java/org/elasticsearch/common/geo/GeoPoint.java
+++ b/core/src/main/java/org/elasticsearch/common/geo/GeoPoint.java
@@ -19,8 +19,15 @@
 
 package org.elasticsearch.common.geo;
 
+import org.apache.lucene.document.LatLonDocValuesField;
+import org.apache.lucene.document.LatLonPoint;
+import org.apache.lucene.geo.GeoEncodingUtils;
+import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.spatial.geopoint.document.GeoPointField;
 import org.apache.lucene.util.BitUtil;
+import org.apache.lucene.util.BytesRef;
+
+import java.util.Arrays;
 
 import static org.elasticsearch.common.geo.GeoHashUtils.mortonEncode;
 import static org.elasticsearch.common.geo.GeoHashUtils.stringEncode;
@@ -88,6 +95,24 @@ public GeoPoint resetFromIndexHash(long hash) {
         return this;
     }
 
+    // todo this is a crutch because LatLonPoint doesn't have a helper for returning .stringValue()
+    // todo remove with next release of lucene
+    public GeoPoint resetFromIndexableField(IndexableField field) {
+        if (field instanceof LatLonPoint) {
+            BytesRef br = field.binaryValue();
+            byte[] bytes = Arrays.copyOfRange(br.bytes, br.offset, br.length);
+            return this.reset(
+                GeoEncodingUtils.decodeLatitude(bytes, 0),
+                GeoEncodingUtils.decodeLongitude(bytes, Integer.BYTES));
+        } else if (field instanceof LatLonDocValuesField) {
+            long encoded = (long)(field.numericValue());
+            return this.reset(
+                GeoEncodingUtils.decodeLatitude((int)(encoded >>> 32)),
+                GeoEncodingUtils.decodeLongitude((int)encoded));
+        }
+        return resetFromIndexHash(Long.parseLong(field.stringValue()));
+    }
+
     public GeoPoint resetFromGeoHash(String geohash) {
         final long hash = mortonEncode(geohash);
         return this.reset(GeoPointField.decodeLatitude(hash), GeoPointField.decodeLongitude(hash));
diff --git a/core/src/main/java/org/elasticsearch/common/geo/GeoUtils.java b/core/src/main/java/org/elasticsearch/common/geo/GeoUtils.java
index b81720057c655..0c223539418af 100644
--- a/core/src/main/java/org/elasticsearch/common/geo/GeoUtils.java
+++ b/core/src/main/java/org/elasticsearch/common/geo/GeoUtils.java
@@ -19,14 +19,23 @@
 
 package org.elasticsearch.common.geo;
 
+import org.apache.lucene.geo.Rectangle;
 import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree;
 import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree;
+import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.SloppyMath;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.unit.DistanceUnit;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentParser.Token;
 import org.elasticsearch.index.mapper.GeoPointFieldMapper;
+import org.elasticsearch.index.fielddata.FieldData;
+import org.elasticsearch.index.fielddata.GeoPointValues;
+import org.elasticsearch.index.fielddata.MultiGeoPointValues;
+import org.elasticsearch.index.fielddata.NumericDoubleValues;
+import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
+import org.elasticsearch.index.fielddata.SortingNumericDoubleValues;
+import org.elasticsearch.index.mapper.GeoPointFieldMapper;
 
 import java.io.IOException;
 
@@ -484,7 +493,8 @@ public static double arcDistance(double lat1, double lon1, double lat2, double l
 
     /**
      * Return the distance (in meters) between 2 lat,lon geo points using a simple tangential plane
-     * this provides a faster alternative to {@link GeoUtils#arcDistance} when points are within 5 km
+     * this provides a faster alternative to {@link GeoUtils#arcDistance} but is innaccurate for distances greater than
+     * 4 decimal degrees
      */
     public static double planeDistance(double lat1, double lon1, double lat2, double lon2) {
         double x = (lon2 - lon1) * SloppyMath.TO_RADIANS * Math.cos((lat2 + lat1) / 2.0 * SloppyMath.TO_RADIANS);
@@ -492,6 +502,62 @@ public static double planeDistance(double lat1, double lon1, double lat2, double
         return Math.sqrt(x * x + y * y) * EARTH_MEAN_RADIUS;
     }
 
+    /** check if point is within a rectangle
+     * todo: move this to lucene Rectangle class
+     */
+    public static boolean rectangleContainsPoint(Rectangle r, double lat, double lon) {
+        if (lat >= r.minLat && lat <= r.maxLat) {
+            // if rectangle crosses the dateline we only check if the lon is >= min or max
+            return r.crossesDateline() ? lon >= r.minLon || lon <= r.maxLon : lon >= r.minLon && lon <= r.maxLon;
+        }
+        return false;
+    }
+
+    /**
+     * Return a {@link SortedNumericDoubleValues} instance that returns the distances to a list of geo-points
+     * for each document.
+     */
+    public static SortedNumericDoubleValues distanceValues(final GeoDistance distance,
+                                                           final DistanceUnit unit,
+                                                           final MultiGeoPointValues geoPointValues,
+                                                           final GeoPoint... fromPoints) {
+        final GeoPointValues singleValues = FieldData.unwrapSingleton(geoPointValues);
+        if (singleValues != null && fromPoints.length == 1) {
+            final Bits docsWithField = FieldData.unwrapSingletonBits(geoPointValues);
+            return FieldData.singleton(new NumericDoubleValues() {
+
+                @Override
+                public double get(int docID) {
+                    if (docsWithField != null && !docsWithField.get(docID)) {
+                        return 0d;
+                    }
+                    final GeoPoint to = singleValues.get(docID);
+                    final GeoPoint from = fromPoints[0];
+                    return distance.calculate(from.lat(), from.lon(), to.lat(), to.lon(), unit);
+                }
+
+            }, docsWithField);
+        } else {
+            return new SortingNumericDoubleValues() {
+
+                @Override
+                public void setDocument(int doc) {
+                    geoPointValues.setDocument(doc);
+                    resize(geoPointValues.count() * fromPoints.length);
+                    int v = 0;
+                    for (GeoPoint from : fromPoints) {
+                        for (int i = 0; i < geoPointValues.count(); ++i) {
+                            final GeoPoint point = geoPointValues.valueAt(i);
+                            values[v] = distance.calculate(from.lat(), from.lon(), point.lat(), point.lon(), unit);
+                            v++;
+                        }
+                    }
+                    sort();
+                }
+            };
+        }
+    }
+
     private GeoUtils() {
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/geo/ShapeRelation.java b/core/src/main/java/org/elasticsearch/common/geo/ShapeRelation.java
index e9966834a0119..e83e18ce43255 100644
--- a/core/src/main/java/org/elasticsearch/common/geo/ShapeRelation.java
+++ b/core/src/main/java/org/elasticsearch/common/geo/ShapeRelation.java
@@ -44,16 +44,12 @@ public enum ShapeRelation implements Writeable {
     }
 
     public static ShapeRelation readFromStream(StreamInput in) throws IOException {
-        int ordinal = in.readVInt();
-        if (ordinal < 0 || ordinal >= values().length) {
-            throw new IOException("Unknown ShapeRelation ordinal [" + ordinal + "]");
-        }
-        return values()[ordinal];
+        return in.readEnum(ShapeRelation.class);
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        out.writeVInt(ordinal());
+        out.writeEnum(this);
     }
 
     public static ShapeRelation getRelationByName(String name) {
diff --git a/core/src/main/java/org/elasticsearch/common/geo/SpatialStrategy.java b/core/src/main/java/org/elasticsearch/common/geo/SpatialStrategy.java
index e1b0356b68636..d578dda7bcf65 100644
--- a/core/src/main/java/org/elasticsearch/common/geo/SpatialStrategy.java
+++ b/core/src/main/java/org/elasticsearch/common/geo/SpatialStrategy.java
@@ -34,7 +34,7 @@ public enum SpatialStrategy implements Writeable {
 
     private final String strategyName;
 
-    private SpatialStrategy(String strategyName) {
+    SpatialStrategy(String strategyName) {
         this.strategyName = strategyName;
     }
 
@@ -43,16 +43,12 @@ public String getStrategyName() {
     }
 
     public static SpatialStrategy readFromStream(StreamInput in) throws IOException {
-        int ordinal = in.readVInt();
-        if (ordinal < 0 || ordinal >= values().length) {
-            throw new IOException("Unknown SpatialStrategy ordinal [" + ordinal + "]");
-        }
-        return values()[ordinal];
+        return in.readEnum(SpatialStrategy.class);
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        out.writeVInt(ordinal());
+        out.writeEnum(this);
     }
 
     public static SpatialStrategy fromString(String strategyName) {
diff --git a/core/src/main/java/org/elasticsearch/common/geo/builders/ShapeBuilder.java b/core/src/main/java/org/elasticsearch/common/geo/builders/ShapeBuilder.java
index cb2f8bb4e78af..8f40d265acfc9 100644
--- a/core/src/main/java/org/elasticsearch/common/geo/builders/ShapeBuilder.java
+++ b/core/src/main/java/org/elasticsearch/common/geo/builders/ShapeBuilder.java
@@ -23,6 +23,7 @@
 import com.vividsolutions.jts.geom.Geometry;
 import com.vividsolutions.jts.geom.GeometryFactory;
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.Assertions;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.action.support.ToXContentToBytes;
 import org.elasticsearch.common.io.stream.NamedWriteable;
@@ -58,9 +59,7 @@ public abstract class ShapeBuilder extends ToXContentToBytes implements NamedWri
     static {
         // if asserts are enabled we run the debug statements even if they are not logged
         // to prevent exceptions only present if debug enabled
-        boolean debug = false;
-        assert debug = true;
-        DEBUG = debug;
+        DEBUG = Assertions.ENABLED;
     }
 
     public static final double DATELINE = 180;
@@ -79,21 +78,21 @@ public abstract class ShapeBuilder extends ToXContentToBytes implements NamedWri
     /** It's possible that some geometries in a MULTI* shape might overlap. With the possible exception of GeometryCollection,
      * this normally isn't allowed.
      */
-    protected final boolean multiPolygonMayOverlap = false;
+    protected static final boolean MULTI_POLYGON_MAY_OVERLAP = false;
     /** @see org.locationtech.spatial4j.shape.jts.JtsGeometry#validate() */
-    protected final boolean autoValidateJtsGeometry = true;
+    protected static final boolean AUTO_VALIDATE_JTS_GEOMETRY = true;
     /** @see org.locationtech.spatial4j.shape.jts.JtsGeometry#index() */
-    protected final boolean autoIndexJtsGeometry = true;//may want to turn off once SpatialStrategy impls do it.
+    protected static final boolean AUTO_INDEX_JTS_GEOMETRY = true;//may want to turn off once SpatialStrategy impls do it.
 
     protected ShapeBuilder() {
     }
 
     protected JtsGeometry jtsGeometry(Geometry geom) {
         //dateline180Check is false because ElasticSearch does it's own dateline wrapping
-        JtsGeometry jtsGeometry = new JtsGeometry(geom, SPATIAL_CONTEXT, false, multiPolygonMayOverlap);
-        if (autoValidateJtsGeometry)
+        JtsGeometry jtsGeometry = new JtsGeometry(geom, SPATIAL_CONTEXT, false, MULTI_POLYGON_MAY_OVERLAP);
+        if (AUTO_VALIDATE_JTS_GEOMETRY)
             jtsGeometry.validate();
-        if (autoIndexJtsGeometry)
+        if (AUTO_INDEX_JTS_GEOMETRY)
             jtsGeometry.index();
         return jtsGeometry;
     }
@@ -381,7 +380,7 @@ public int compare(Edge o1, Edge o2) {
         }
     }
 
-    public static enum Orientation {
+    public enum Orientation {
         LEFT,
         RIGHT;
 
@@ -427,7 +426,7 @@ protected static final boolean debugEnabled() {
     /**
      * Enumeration that lists all {@link GeoShapeType}s that can be handled
      */
-    public static enum GeoShapeType {
+    public enum GeoShapeType {
         POINT("point"),
         MULTIPOINT("multipoint"),
         LINESTRING("linestring"),
@@ -440,7 +439,7 @@ public static enum GeoShapeType {
 
         private final String shapename;
 
-        private GeoShapeType(String shapename) {
+        GeoShapeType(String shapename) {
             this.shapename = shapename;
         }
 
diff --git a/core/src/main/java/org/elasticsearch/common/inject/ConstantFactory.java b/core/src/main/java/org/elasticsearch/common/inject/ConstantFactory.java
index 87bf31e911e91..aa7029f2a9cae 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/ConstantFactory.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/ConstantFactory.java
@@ -30,7 +30,7 @@ class ConstantFactory<T> implements InternalFactory<T> {
 
     private final Initializable<T> initializable;
 
-    public ConstantFactory(Initializable<T> initializable) {
+    ConstantFactory(Initializable<T> initializable) {
         this.initializable = initializable;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/common/inject/DeferredLookups.java b/core/src/main/java/org/elasticsearch/common/inject/DeferredLookups.java
index 40d589f37dcb1..2bc8f770bf857 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/DeferredLookups.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/DeferredLookups.java
@@ -34,7 +34,7 @@ class DeferredLookups implements Lookups {
     private final InjectorImpl injector;
     private final List<Element> lookups = new ArrayList<>();
 
-    public DeferredLookups(InjectorImpl injector) {
+    DeferredLookups(InjectorImpl injector) {
         this.injector = injector;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/common/inject/EncounterImpl.java b/core/src/main/java/org/elasticsearch/common/inject/EncounterImpl.java
index 8b8b7b78218b8..ed49d28dea7c5 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/EncounterImpl.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/EncounterImpl.java
@@ -36,7 +36,7 @@ final class EncounterImpl<T> implements TypeEncounter<T> {
     private List<InjectionListener<? super T>> injectionListeners; // lazy
     private boolean valid = true;
 
-    public EncounterImpl(Errors errors, Lookups lookups) {
+    EncounterImpl(Errors errors, Lookups lookups) {
         this.errors = errors;
         this.lookups = lookups;
     }
diff --git a/core/src/main/java/org/elasticsearch/common/inject/ExposedKeyFactory.java b/core/src/main/java/org/elasticsearch/common/inject/ExposedKeyFactory.java
index efc10b27e498e..b3cf0a14b6be6 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/ExposedKeyFactory.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/ExposedKeyFactory.java
@@ -33,7 +33,7 @@ class ExposedKeyFactory<T> implements InternalFactory<T>, BindingProcessor.Creat
     private final PrivateElements privateElements;
     private BindingImpl<T> delegate;
 
-    public ExposedKeyFactory(Key<T> key, PrivateElements privateElements) {
+    ExposedKeyFactory(Key<T> key, PrivateElements privateElements) {
         this.key = key;
         this.privateElements = privateElements;
     }
diff --git a/core/src/main/java/org/elasticsearch/common/inject/Initializer.java b/core/src/main/java/org/elasticsearch/common/inject/Initializer.java
index 1d68f163bfe23..ce7d7765ce320 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/Initializer.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/Initializer.java
@@ -115,7 +115,7 @@ private class InjectableReference<T> implements Initializable<T> {
         private final Object source;
         private MembersInjectorImpl<T> membersInjector;
 
-        public InjectableReference(InjectorImpl injector, T instance, Object source) {
+        InjectableReference(InjectorImpl injector, T instance, Object source) {
             this.injector = injector;
             this.instance = Objects.requireNonNull(instance, "instance");
             this.source = Objects.requireNonNull(source, "source");
diff --git a/core/src/main/java/org/elasticsearch/common/inject/InjectionRequestProcessor.java b/core/src/main/java/org/elasticsearch/common/inject/InjectionRequestProcessor.java
index e8b38b5133089..909779cb442a6 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/InjectionRequestProcessor.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/InjectionRequestProcessor.java
@@ -86,7 +86,7 @@ private class StaticInjection {
         final StaticInjectionRequest request;
         List<SingleMemberInjector> memberInjectors;
 
-        public StaticInjection(InjectorImpl injector, StaticInjectionRequest request) {
+        StaticInjection(InjectorImpl injector, StaticInjectionRequest request) {
             this.injector = injector;
             this.source = request.getSource();
             this.request = request;
diff --git a/core/src/main/java/org/elasticsearch/common/inject/InternalFactoryToProviderAdapter.java b/core/src/main/java/org/elasticsearch/common/inject/InternalFactoryToProviderAdapter.java
index 8739d9182d8c8..54fecae9ba995 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/InternalFactoryToProviderAdapter.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/InternalFactoryToProviderAdapter.java
@@ -33,11 +33,11 @@ class InternalFactoryToProviderAdapter<T> implements InternalFactory<T> {
     private final Initializable<Provider<? extends T>> initializable;
     private final Object source;
 
-    public InternalFactoryToProviderAdapter(Initializable<Provider<? extends T>> initializable) {
+    InternalFactoryToProviderAdapter(Initializable<Provider<? extends T>> initializable) {
         this(initializable, SourceProvider.UNKNOWN_SOURCE);
     }
 
-    public InternalFactoryToProviderAdapter(
+    InternalFactoryToProviderAdapter(
             Initializable<Provider<? extends T>> initializable, Object source) {
         this.initializable = Objects.requireNonNull(initializable, "provider");
         this.source = Objects.requireNonNull(source, "source");
diff --git a/core/src/main/java/org/elasticsearch/common/inject/Key.java b/core/src/main/java/org/elasticsearch/common/inject/Key.java
index 83ab440d23c83..833aac2d3fc4f 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/Key.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/Key.java
@@ -380,7 +380,7 @@ private static void ensureIsBindingAnnotation(
         }
     }
 
-    static enum NullAnnotationStrategy implements AnnotationStrategy {
+    enum NullAnnotationStrategy implements AnnotationStrategy {
         INSTANCE;
 
         @Override
diff --git a/core/src/main/java/org/elasticsearch/common/inject/ModulesBuilder.java b/core/src/main/java/org/elasticsearch/common/inject/ModulesBuilder.java
index 3321b75f4e50b..6928033c69a1f 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/ModulesBuilder.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/ModulesBuilder.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.common.inject;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 
@@ -28,9 +29,7 @@ public class ModulesBuilder implements Iterable<Module> {
     private final List<Module> modules = new ArrayList<>();
 
     public ModulesBuilder add(Module... newModules) {
-        for (Module module : newModules) {
-            modules.add(module);
-        }
+        Collections.addAll(modules, newModules);
         return this;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/common/inject/ProviderToInternalFactoryAdapter.java b/core/src/main/java/org/elasticsearch/common/inject/ProviderToInternalFactoryAdapter.java
index d7b6afbe6daae..f443756908867 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/ProviderToInternalFactoryAdapter.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/ProviderToInternalFactoryAdapter.java
@@ -30,7 +30,7 @@ class ProviderToInternalFactoryAdapter<T> implements Provider<T> {
     private final InjectorImpl injector;
     private final InternalFactory<? extends T> internalFactory;
 
-    public ProviderToInternalFactoryAdapter(InjectorImpl injector,
+    ProviderToInternalFactoryAdapter(InjectorImpl injector,
                                             InternalFactory<? extends T> internalFactory) {
         this.injector = injector;
         this.internalFactory = internalFactory;
diff --git a/core/src/main/java/org/elasticsearch/common/inject/SingleFieldInjector.java b/core/src/main/java/org/elasticsearch/common/inject/SingleFieldInjector.java
index 10ba17d86cda6..ce02a26ffd0aa 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/SingleFieldInjector.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/SingleFieldInjector.java
@@ -34,7 +34,7 @@ class SingleFieldInjector implements SingleMemberInjector {
     final Dependency<?> dependency;
     final InternalFactory<?> factory;
 
-    public SingleFieldInjector(InjectorImpl injector, InjectionPoint injectionPoint, Errors errors)
+    SingleFieldInjector(InjectorImpl injector, InjectionPoint injectionPoint, Errors errors)
             throws ErrorsException {
         this.injectionPoint = injectionPoint;
         this.field = (Field) injectionPoint.getMember();
diff --git a/core/src/main/java/org/elasticsearch/common/inject/SingleMethodInjector.java b/core/src/main/java/org/elasticsearch/common/inject/SingleMethodInjector.java
index 9c407791160c4..7330d05df3b38 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/SingleMethodInjector.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/SingleMethodInjector.java
@@ -34,7 +34,7 @@ class SingleMethodInjector implements SingleMemberInjector {
     final SingleParameterInjector<?>[] parameterInjectors;
     final InjectionPoint injectionPoint;
 
-    public SingleMethodInjector(InjectorImpl injector, InjectionPoint injectionPoint, Errors errors)
+    SingleMethodInjector(InjectorImpl injector, InjectionPoint injectionPoint, Errors errors)
             throws ErrorsException {
         this.injectionPoint = injectionPoint;
         final Method method = (Method) injectionPoint.getMember();
diff --git a/core/src/main/java/org/elasticsearch/common/inject/assistedinject/AssistedConstructor.java b/core/src/main/java/org/elasticsearch/common/inject/assistedinject/AssistedConstructor.java
index 0716653708181..cb434a90369d3 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/assistedinject/AssistedConstructor.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/assistedinject/AssistedConstructor.java
@@ -43,13 +43,13 @@ class AssistedConstructor<T> {
     private final List<Parameter> allParameters;
 
     @SuppressWarnings("unchecked")
-    public AssistedConstructor(Constructor<T> constructor, List<TypeLiteral<?>> parameterTypes) {
+    AssistedConstructor(Constructor<T> constructor, List<TypeLiteral<?>> parameterTypes) {
         this.constructor = constructor;
 
         Annotation[][] annotations = constructor.getParameterAnnotations();
 
         List<Type> typeList = new ArrayList<>();
-        allParameters = new ArrayList<>();
+        allParameters = new ArrayList<>(parameterTypes.size());
 
         // categorize params as @Assisted or @Injected
         for (int i = 0; i < parameterTypes.size(); i++) {
diff --git a/core/src/main/java/org/elasticsearch/common/inject/assistedinject/Parameter.java b/core/src/main/java/org/elasticsearch/common/inject/assistedinject/Parameter.java
index 5ceb086db9f4d..a21dc3aa7f54d 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/assistedinject/Parameter.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/assistedinject/Parameter.java
@@ -39,7 +39,7 @@ class Parameter {
     private final Annotation bindingAnnotation;
     private final boolean isProvider;
 
-    public Parameter(Type type, Annotation[] annotations) {
+    Parameter(Type type, Annotation[] annotations) {
         this.type = type;
         this.bindingAnnotation = getBindingAnnotation(annotations);
         this.isAssisted = hasAssistedAnnotation(annotations);
diff --git a/core/src/main/java/org/elasticsearch/common/inject/assistedinject/ParameterListKey.java b/core/src/main/java/org/elasticsearch/common/inject/assistedinject/ParameterListKey.java
index fc2a96e19dfad..7967f47394879 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/assistedinject/ParameterListKey.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/assistedinject/ParameterListKey.java
@@ -34,11 +34,11 @@ class ParameterListKey {
 
     private final List<Type> paramList;
 
-    public ParameterListKey(List<Type> paramList) {
+    ParameterListKey(List<Type> paramList) {
         this.paramList = new ArrayList<>(paramList);
     }
 
-    public ParameterListKey(Type[] types) {
+    ParameterListKey(Type[] types) {
         this(Arrays.asList(types));
     }
 
diff --git a/core/src/main/java/org/elasticsearch/common/inject/internal/NullOutputException.java b/core/src/main/java/org/elasticsearch/common/inject/internal/NullOutputException.java
index cc5a23a786dcf..0fec6b5bac281 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/internal/NullOutputException.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/internal/NullOutputException.java
@@ -24,7 +24,7 @@
  * @author Bob Lee
  */
 class NullOutputException extends NullPointerException {
-    public NullOutputException(String s) {
+    NullOutputException(String s) {
         super(s);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/inject/matcher/AbstractMatcher.java b/core/src/main/java/org/elasticsearch/common/inject/matcher/AbstractMatcher.java
index 931d290fc19c3..76df334e4e315 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/matcher/AbstractMatcher.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/matcher/AbstractMatcher.java
@@ -36,7 +36,7 @@ public Matcher<T> or(Matcher<? super T> other) {
     private static class AndMatcher<T> extends AbstractMatcher<T> {
         private final Matcher<? super T> a, b;
 
-        public AndMatcher(Matcher<? super T> a, Matcher<? super T> b) {
+        AndMatcher(Matcher<? super T> a, Matcher<? super T> b) {
             this.a = a;
             this.b = b;
         }
@@ -67,7 +67,7 @@ public String toString() {
     private static class OrMatcher<T> extends AbstractMatcher<T> {
         private final Matcher<? super T> a, b;
 
-        public OrMatcher(Matcher<? super T> a, Matcher<? super T> b) {
+        OrMatcher(Matcher<? super T> a, Matcher<? super T> b) {
             this.a = a;
             this.b = b;
         }
diff --git a/core/src/main/java/org/elasticsearch/common/inject/matcher/Matchers.java b/core/src/main/java/org/elasticsearch/common/inject/matcher/Matchers.java
index e2ced98034e88..cc354145b11bb 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/matcher/Matchers.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/matcher/Matchers.java
@@ -113,7 +113,7 @@ public static Matcher<AnnotatedElement> annotatedWith(
     private static class AnnotatedWithType extends AbstractMatcher<AnnotatedElement> {
         private final Class<? extends Annotation> annotationType;
 
-        public AnnotatedWithType(Class<? extends Annotation> annotationType) {
+        AnnotatedWithType(Class<? extends Annotation> annotationType) {
             this.annotationType = Objects.requireNonNull(annotationType, "annotation type");
             checkForRuntimeRetention(annotationType);
         }
@@ -152,7 +152,7 @@ public static Matcher<AnnotatedElement> annotatedWith(
     private static class AnnotatedWith extends AbstractMatcher<AnnotatedElement> {
         private final Annotation annotation;
 
-        public AnnotatedWith(Annotation annotation) {
+        AnnotatedWith(Annotation annotation) {
             this.annotation = Objects.requireNonNull(annotation, "annotation");
             checkForRuntimeRetention(annotation.annotationType());
         }
@@ -191,7 +191,7 @@ public static Matcher<Class> subclassesOf(final Class<?> superclass) {
     private static class SubclassesOf extends AbstractMatcher<Class> {
         private final Class<?> superclass;
 
-        public SubclassesOf(Class<?> superclass) {
+        SubclassesOf(Class<?> superclass) {
             this.superclass = Objects.requireNonNull(superclass, "superclass");
         }
 
@@ -227,7 +227,7 @@ public static Matcher<Object> only(Object value) {
     private static class Only extends AbstractMatcher<Object> {
         private final Object value;
 
-        public Only(Object value) {
+        Only(Object value) {
             this.value = Objects.requireNonNull(value, "value");
         }
 
@@ -263,7 +263,7 @@ public static Matcher<Object> identicalTo(final Object value) {
     private static class IdenticalTo extends AbstractMatcher<Object> {
         private final Object value;
 
-        public IdenticalTo(Object value) {
+        IdenticalTo(Object value) {
             this.value = Objects.requireNonNull(value, "value");
         }
 
@@ -301,7 +301,7 @@ private static class InPackage extends AbstractMatcher<Class> {
         private final transient Package targetPackage;
         private final String packageName;
 
-        public InPackage(Package targetPackage) {
+        InPackage(Package targetPackage) {
             this.targetPackage = Objects.requireNonNull(targetPackage, "package");
             this.packageName = targetPackage.getName();
         }
@@ -345,7 +345,7 @@ public static Matcher<Class> inSubpackage(final String targetPackageName) {
     private static class InSubpackage extends AbstractMatcher<Class>  {
         private final String targetPackageName;
 
-        public InSubpackage(String targetPackageName) {
+        InSubpackage(String targetPackageName) {
             this.targetPackageName = targetPackageName;
         }
 
@@ -384,7 +384,7 @@ public static Matcher<Method> returns(
     private static class Returns extends AbstractMatcher<Method> {
         private final Matcher<? super Class<?>> returnType;
 
-        public Returns(Matcher<? super Class<?>> returnType) {
+        Returns(Matcher<? super Class<?>> returnType) {
             this.returnType = Objects.requireNonNull(returnType, "return type matcher");
         }
 
diff --git a/core/src/main/java/org/elasticsearch/common/inject/name/NamedImpl.java b/core/src/main/java/org/elasticsearch/common/inject/name/NamedImpl.java
index e4cc088e30af7..eb3fb00a51ad1 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/name/NamedImpl.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/name/NamedImpl.java
@@ -23,7 +23,7 @@ class NamedImpl implements Named {
 
     private final String value;
 
-    public NamedImpl(String value) {
+    NamedImpl(String value) {
         this.value = Objects.requireNonNull(value, "name");
     }
 
diff --git a/core/src/main/java/org/elasticsearch/common/io/ReleasableBytesStream.java b/core/src/main/java/org/elasticsearch/common/io/ReleasableBytesStream.java
deleted file mode 100644
index e31f206bcad9c..0000000000000
--- a/core/src/main/java/org/elasticsearch/common/io/ReleasableBytesStream.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.common.io;
-
-import org.elasticsearch.common.bytes.ReleasablePagedBytesReference;
-
-/**
- * A bytes stream that requires its bytes to be released once no longer used.
- */
-public interface ReleasableBytesStream extends BytesStream {
-
-    @Override
-    ReleasablePagedBytesReference bytes();
-
-}
diff --git a/core/src/main/java/org/elasticsearch/common/io/Streams.java b/core/src/main/java/org/elasticsearch/common/io/Streams.java
index f922fde3e753e..b6b9061ad7e49 100644
--- a/core/src/main/java/org/elasticsearch/common/io/Streams.java
+++ b/core/src/main/java/org/elasticsearch/common/io/Streams.java
@@ -20,7 +20,9 @@
 package org.elasticsearch.common.io;
 
 import org.apache.lucene.util.IOUtils;
-import org.elasticsearch.common.util.Callback;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.BytesStream;
+import org.elasticsearch.common.io.stream.StreamOutput;
 
 import java.io.BufferedReader;
 import java.io.IOException;
@@ -34,6 +36,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
+import java.util.function.Consumer;
 
 /**
  * Simple utility methods for file and stream copying.
@@ -219,21 +222,68 @@ public static int readFully(InputStream reader, byte[] dest, int offset, int len
 
     public static List<String> readAllLines(InputStream input) throws IOException {
         final List<String> lines = new ArrayList<>();
-        readAllLines(input, new Callback<String>() {
-            @Override
-            public void handle(String line) {
-                lines.add(line);
-            }
-        });
+        readAllLines(input, lines::add);
         return lines;
     }
 
-    public static void readAllLines(InputStream input, Callback<String> callback) throws IOException {
+    public static void readAllLines(InputStream input, Consumer<String> consumer) throws IOException {
         try (BufferedReader reader = new BufferedReader(new InputStreamReader(input, StandardCharsets.UTF_8))) {
             String line;
             while ((line = reader.readLine()) != null) {
-                callback.handle(line);
+                consumer.accept(line);
             }
         }
     }
+
+    /**
+     * Wraps the given {@link BytesStream} in a {@link StreamOutput} that simply flushes when
+     * close is called.
+     */
+    public static BytesStream flushOnCloseStream(BytesStream os) {
+        return new FlushOnCloseOutputStream(os);
+    }
+
+    /**
+     * A wrapper around a {@link BytesStream} that makes the close operation a flush. This is
+     * needed as sometimes a stream will be closed but the bytes that the stream holds still need
+     * to be used and the stream cannot be closed until the bytes have been consumed.
+     */
+    private static class FlushOnCloseOutputStream extends BytesStream {
+
+        private final BytesStream delegate;
+
+        private FlushOnCloseOutputStream(BytesStream bytesStreamOutput) {
+            this.delegate = bytesStreamOutput;
+        }
+
+        @Override
+        public void writeByte(byte b) throws IOException {
+            delegate.writeByte(b);
+        }
+
+        @Override
+        public void writeBytes(byte[] b, int offset, int length) throws IOException {
+            delegate.writeBytes(b, offset, length);
+        }
+
+        @Override
+        public void flush() throws IOException {
+            delegate.flush();
+        }
+
+        @Override
+        public void close() throws IOException {
+            flush();
+        }
+
+        @Override
+        public void reset() throws IOException {
+            delegate.reset();
+        }
+
+        @Override
+        public BytesReference bytes() {
+            return delegate.bytes();
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/ByteBufferStreamInput.java b/core/src/main/java/org/elasticsearch/common/io/stream/ByteBufferStreamInput.java
index d13f539a67006..ce43d74e02314 100644
--- a/core/src/main/java/org/elasticsearch/common/io/stream/ByteBufferStreamInput.java
+++ b/core/src/main/java/org/elasticsearch/common/io/stream/ByteBufferStreamInput.java
@@ -88,6 +88,13 @@ public int available() throws IOException {
         return buffer.remaining();
     }
 
+    @Override
+    protected void ensureCanReadBytes(int length) throws EOFException {
+        if (buffer.remaining() < length) {
+            throw new EOFException("tried to read: " + length + " bytes but only " + buffer.remaining() + " remaining");
+        }
+    }
+
     @Override
     public void mark(int readlimit) {
         buffer.mark();
diff --git a/core/src/main/java/org/elasticsearch/common/io/BytesStream.java b/core/src/main/java/org/elasticsearch/common/io/stream/BytesStream.java
similarity index 85%
rename from core/src/main/java/org/elasticsearch/common/io/BytesStream.java
rename to core/src/main/java/org/elasticsearch/common/io/stream/BytesStream.java
index 903c1dcb79962..c20dcf62c9bbd 100644
--- a/core/src/main/java/org/elasticsearch/common/io/BytesStream.java
+++ b/core/src/main/java/org/elasticsearch/common/io/stream/BytesStream.java
@@ -17,11 +17,11 @@
  * under the License.
  */
 
-package org.elasticsearch.common.io;
+package org.elasticsearch.common.io.stream;
 
 import org.elasticsearch.common.bytes.BytesReference;
 
-public interface BytesStream {
+public abstract class BytesStream extends StreamOutput {
 
-    BytesReference bytes();
-}
\ No newline at end of file
+    public abstract BytesReference bytes();
+}
diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/BytesStreamOutput.java b/core/src/main/java/org/elasticsearch/common/io/stream/BytesStreamOutput.java
index 3de5c757ae1bc..ab9a1896ef7cf 100644
--- a/core/src/main/java/org/elasticsearch/common/io/stream/BytesStreamOutput.java
+++ b/core/src/main/java/org/elasticsearch/common/io/stream/BytesStreamOutput.java
@@ -21,7 +21,6 @@
 
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.bytes.PagedBytesReference;
-import org.elasticsearch.common.io.BytesStream;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.ByteArray;
 
@@ -31,7 +30,7 @@
  * A @link {@link StreamOutput} that uses {@link BigArrays} to acquire pages of
  * bytes, which avoids frequent reallocation &amp; copying of the internal data.
  */
-public class BytesStreamOutput extends StreamOutput implements BytesStream {
+public class BytesStreamOutput extends BytesStream {
 
     protected final BigArrays bigArrays;
 
@@ -50,7 +49,7 @@ public BytesStreamOutput() {
     /**
      * Create a non recycling {@link BytesStreamOutput} with enough initial pages acquired
      * to satisfy the capacity given by expected size.
-     * 
+     *
      * @param expectedSize the expected maximum size of the stream in bytes.
      */
     public BytesStreamOutput(int expectedSize) {
@@ -69,13 +68,13 @@ public long position() throws IOException {
 
     @Override
     public void writeByte(byte b) throws IOException {
-        ensureCapacity(count+1);
+        ensureCapacity(count + 1L);
         bytes.set(count, b);
         count++;
     }
 
     @Override
-    public void writeBytes(byte[] b, int offset, int length) throws IOException {
+    public void writeBytes(byte[] b, int offset, int length) {
         // nothing to copy
         if (length == 0) {
             return;
@@ -87,7 +86,7 @@ public void writeBytes(byte[] b, int offset, int length) throws IOException {
         }
 
         // get enough pages for new size
-        ensureCapacity(count+length);
+        ensureCapacity(((long) count) + length);
 
         // bulk copy
         bytes.set(count, b, offset, length);
@@ -113,28 +112,23 @@ public void flush() throws IOException {
     }
 
     @Override
-    public void seek(long position) throws IOException {
-        if (position > Integer.MAX_VALUE) {
-            throw new IllegalArgumentException("position " + position + " > Integer.MAX_VALUE");
-        }
-
-        count = (int)position;
-        ensureCapacity(count);
+    public void seek(long position) {
+        ensureCapacity(position);
+        count = (int) position;
     }
 
     public void skip(int length) {
-        count += length;
-        ensureCapacity(count);
+        seek(((long) count) + length);
     }
 
     @Override
-    public void close() throws IOException {
+    public void close() {
         // empty for now.
     }
 
     /**
      * Returns the current size of the buffer.
-     * 
+     *
      * @return the value of the <code>count</code> field, which is the number of valid
      *         bytes in this output stream.
      * @see java.io.ByteArrayOutputStream#count
@@ -156,7 +150,10 @@ public long ramBytesUsed() {
         return bytes.ramBytesUsed();
     }
 
-    private void ensureCapacity(int offset) {
+    void ensureCapacity(long offset) {
+        if (offset > Integer.MAX_VALUE) {
+            throw new IllegalArgumentException(getClass().getSimpleName() + " cannot hold more than 2GB of data");
+        }
         bytes = bigArrays.grow(bytes, offset);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/FilterStreamInput.java b/core/src/main/java/org/elasticsearch/common/io/stream/FilterStreamInput.java
index b8132b4e87033..1a3f9fe601da4 100644
--- a/core/src/main/java/org/elasticsearch/common/io/stream/FilterStreamInput.java
+++ b/core/src/main/java/org/elasticsearch/common/io/stream/FilterStreamInput.java
@@ -21,6 +21,7 @@
 
 import org.elasticsearch.Version;
 
+import java.io.EOFException;
 import java.io.IOException;
 
 /**
@@ -28,7 +29,7 @@
  */
 public abstract class FilterStreamInput extends StreamInput {
 
-    private final StreamInput delegate;
+    protected final StreamInput delegate;
 
     protected FilterStreamInput(StreamInput delegate) {
         this.delegate = delegate;
@@ -73,4 +74,9 @@ public Version getVersion() {
     public void setVersion(Version version) {
         delegate.setVersion(version);
     }
+
+    @Override
+    protected void ensureCanReadBytes(int length) throws EOFException {
+        delegate.ensureCanReadBytes(length);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/InputStreamStreamInput.java b/core/src/main/java/org/elasticsearch/common/io/stream/InputStreamStreamInput.java
index d786041af4908..f9611f3f78852 100644
--- a/core/src/main/java/org/elasticsearch/common/io/stream/InputStreamStreamInput.java
+++ b/core/src/main/java/org/elasticsearch/common/io/stream/InputStreamStreamInput.java
@@ -98,4 +98,9 @@ public int read(byte[] b, int off, int len) throws IOException {
     public long skip(long n) throws IOException {
         return is.skip(n);
     }
+
+    @Override
+    protected void ensureCanReadBytes(int length) throws EOFException {
+        // TODO what can we do here?
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/NamedWriteableAwareStreamInput.java b/core/src/main/java/org/elasticsearch/common/io/stream/NamedWriteableAwareStreamInput.java
index 55da169e6207f..5db80a711eeb9 100644
--- a/core/src/main/java/org/elasticsearch/common/io/stream/NamedWriteableAwareStreamInput.java
+++ b/core/src/main/java/org/elasticsearch/common/io/stream/NamedWriteableAwareStreamInput.java
@@ -36,14 +36,20 @@ public NamedWriteableAwareStreamInput(StreamInput delegate, NamedWriteableRegist
     @Override
     public <C extends NamedWriteable> C readNamedWriteable(Class<C> categoryClass) throws IOException {
         String name = readString();
+        return readNamedWriteable(categoryClass, name);
+    }
+
+    @Override
+    public <C extends NamedWriteable> C readNamedWriteable(@SuppressWarnings("unused") Class<C> categoryClass,
+                                                           @SuppressWarnings("unused") String name) throws IOException {
         Writeable.Reader<? extends C> reader = namedWriteableRegistry.getReader(categoryClass, name);
         C c = reader.read(this);
         if (c == null) {
             throw new IOException(
-                    "Writeable.Reader [" + reader + "] returned null which is not allowed and probably means it screwed up the stream.");
+                "Writeable.Reader [" + reader + "] returned null which is not allowed and probably means it screwed up the stream.");
         }
         assert name.equals(c.getWriteableName()) : c + " claims to have a different name [" + c.getWriteableName()
-                + "] than it was read from [" + name + "].";
+            + "] than it was read from [" + name + "].";
         return c;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/NamedWriteableRegistry.java b/core/src/main/java/org/elasticsearch/common/io/stream/NamedWriteableRegistry.java
index 1a3f57052de45..8fde972e8e4e2 100644
--- a/core/src/main/java/org/elasticsearch/common/io/stream/NamedWriteableRegistry.java
+++ b/core/src/main/java/org/elasticsearch/common/io/stream/NamedWriteableRegistry.java
@@ -25,10 +25,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import org.elasticsearch.plugins.Plugin;
 
 /**
  * A registry for {@link org.elasticsearch.common.io.stream.Writeable.Reader} readers of {@link NamedWriteable}.
@@ -47,7 +43,7 @@ public static class Entry {
         /** A name for the writeable which is unique to the {@link #categoryClass}. */
         public final String name;
 
-        /** A reader captability of reading*/
+        /** A reader capability of reading*/
         public final Writeable.Reader<?> reader;
 
         /** Creates a new entry which can be stored by the registry. */
diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/NotSerializableExceptionWrapper.java b/core/src/main/java/org/elasticsearch/common/io/stream/NotSerializableExceptionWrapper.java
index b25d4bc9b72ff..fd4a215eabf2b 100644
--- a/core/src/main/java/org/elasticsearch/common/io/stream/NotSerializableExceptionWrapper.java
+++ b/core/src/main/java/org/elasticsearch/common/io/stream/NotSerializableExceptionWrapper.java
@@ -38,8 +38,7 @@ public final class NotSerializableExceptionWrapper extends ElasticsearchExceptio
     private final RestStatus status;
 
     public NotSerializableExceptionWrapper(Throwable other) {
-        super(ElasticsearchException.getExceptionName(other) +
-                        ": " + other.getMessage(), other.getCause());
+        super(ElasticsearchException.getExceptionName(other) + ": " + other.getMessage(), other.getCause());
         this.name = ElasticsearchException.getExceptionName(other);
         this.status = ExceptionsHelper.status(other);
         setStackTrace(other.getStackTrace());
@@ -51,6 +50,9 @@ public NotSerializableExceptionWrapper(Throwable other) {
             for (String key : ex.getHeaderKeys()) {
                 this.addHeader(key, ex.getHeader(key));
             }
+            for (String key : ex.getMetadataKeys()) {
+                this.addMetadata(key, ex.getMetadata(key));
+            }
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/ReleasableBytesStreamOutput.java b/core/src/main/java/org/elasticsearch/common/io/stream/ReleasableBytesStreamOutput.java
index 674ff18f0fc15..0bfe15d6dc2f5 100644
--- a/core/src/main/java/org/elasticsearch/common/io/stream/ReleasableBytesStreamOutput.java
+++ b/core/src/main/java/org/elasticsearch/common/io/stream/ReleasableBytesStreamOutput.java
@@ -20,29 +20,66 @@
 package org.elasticsearch.common.io.stream;
 
 import org.elasticsearch.common.bytes.ReleasablePagedBytesReference;
-import org.elasticsearch.common.io.ReleasableBytesStream;
+import org.elasticsearch.common.lease.Releasable;
+import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.ByteArray;
 
 /**
  * An bytes stream output that allows providing a {@link BigArrays} instance
  * expecting it to require releasing its content ({@link #bytes()}) once done.
  * <p>
- * Please note, its is the responsibility of the caller to make sure the bytes
- * reference do not "escape" and are released only once.
+ * Please note, closing this stream will release the bytes that are in use by any
+ * {@link ReleasablePagedBytesReference} returned from {@link #bytes()}, so this
+ * stream should only be closed after the bytes have been output or copied
+ * elsewhere.
  */
-public class ReleasableBytesStreamOutput extends BytesStreamOutput implements ReleasableBytesStream {
+public class ReleasableBytesStreamOutput extends BytesStreamOutput
+    implements Releasable {
+
+    private Releasable releasable;
 
     public ReleasableBytesStreamOutput(BigArrays bigarrays) {
-        super(BigArrays.PAGE_SIZE_IN_BYTES, bigarrays);
+        this(BigArrays.PAGE_SIZE_IN_BYTES, bigarrays);
     }
 
     public ReleasableBytesStreamOutput(int expectedSize, BigArrays bigArrays) {
         super(expectedSize, bigArrays);
+        this.releasable = Releasables.releaseOnce(this.bytes);
     }
 
+    /**
+     * Returns a {@link Releasable} implementation of a
+     * {@link org.elasticsearch.common.bytes.BytesReference} that represents the current state of
+     * the bytes in the stream.
+     */
     @Override
     public ReleasablePagedBytesReference bytes() {
-        return new ReleasablePagedBytesReference(bigArrays, bytes, count);
+        return new ReleasablePagedBytesReference(bigArrays, bytes, count, releasable);
     }
 
+    @Override
+    public void close() {
+        Releasables.close(releasable);
+    }
+
+    @Override
+    void ensureCapacity(long offset) {
+        final ByteArray prevBytes = this.bytes;
+        super.ensureCapacity(offset);
+        if (prevBytes != this.bytes) {
+            // re-create the releasable with the new reference
+            releasable = Releasables.releaseOnce(this.bytes);
+        }
+    }
+
+    @Override
+    public void reset() {
+        final ByteArray prevBytes = this.bytes;
+        super.reset();
+        if (prevBytes != this.bytes) {
+            // re-create the releasable with the new reference
+            releasable = Releasables.releaseOnce(this.bytes);
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java b/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
index 794ed6f36fac7..4681af3392ec5 100644
--- a/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
+++ b/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
@@ -24,9 +24,10 @@
 import org.apache.lucene.index.IndexFormatTooOldException;
 import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.LockObtainFailedException;
+import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.BitUtil;
 import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.CharsRefBuilder;
+import org.apache.lucene.util.CharsRef;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.Nullable;
@@ -58,6 +59,7 @@
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.function.IntFunction;
 import java.util.function.Supplier;
@@ -111,7 +113,7 @@ public void setVersion(Version version) {
      * bytes of the stream.
      */
     public BytesReference readBytesReference() throws IOException {
-        int length = readVInt();
+        int length = readArraySize();
         return readBytesReference(length);
     }
 
@@ -143,7 +145,7 @@ public BytesReference readBytesReference(int length) throws IOException {
     }
 
     public BytesRef readBytesRef() throws IOException {
-        int length = readVInt();
+        int length = readArraySize();
         return readBytesRef(length);
     }
 
@@ -200,7 +202,9 @@ public int readVInt() throws IOException {
             return i;
         }
         b = readByte();
-        assert (b & 0x80) == 0;
+        if ((b & 0x80) != 0) {
+            throw new IOException("Invalid vInt ((" + Integer.toHexString(b) + " & 0x7f) << 28) | " + Integer.toHexString(i));
+        }
         return i | ((b & 0x7F) << 28);
     }
 
@@ -212,9 +216,8 @@ public long readLong() throws IOException {
     }
 
     /**
-     * Reads a long stored in variable-length format.  Reads between one and
-     * nine bytes.  Smaller values take fewer bytes.  Negative numbers are not
-     * supported.
+     * Reads a long stored in variable-length format. Reads between one and ten bytes. Smaller values take fewer bytes. Negative numbers
+     * are encoded in ten bytes so prefer {@link #readLong()} or {@link #readZLong()} for negative numbers.
      */
     public long readVLong() throws IOException {
         byte b = readByte();
@@ -258,8 +261,16 @@ public long readVLong() throws IOException {
             return i;
         }
         b = readByte();
-        assert (b & 0x80) == 0;
-        return i | ((b & 0x7FL) << 56);
+        i |= ((b & 0x7FL) << 56);
+        if ((b & 0x80) == 0) {
+            return i;
+        }
+        b = readByte();
+        if (b != 0 && b != 1) {
+            throw new IOException("Invalid vlong (" + Integer.toHexString(b) + " << 63) | " + Long.toHexString(i));
+        }
+        i |= ((long) b) << 63;
+        return i;
     }
 
     public long readZLong() throws IOException {
@@ -323,15 +334,22 @@ public Integer readOptionalVInt() throws IOException {
         return null;
     }
 
-    private final CharsRefBuilder spare = new CharsRefBuilder();
+    // we don't use a CharsRefBuilder since we exactly know the size of the character array up front
+    // this prevents calling grow for every character since we don't need this
+    private final CharsRef spare = new CharsRef();
 
     public String readString() throws IOException {
-        final int charCount = readVInt();
-        spare.clear();
-        spare.grow(charCount);
-        int c;
-        while (spare.length() < charCount) {
-            c = readByte() & 0xff;
+        // TODO it would be nice to not call readByte() for every character but we don't know how much to read up-front
+        // we can make the loop much more complicated but that won't buy us much compared to the bounds checks in readByte()
+        final int charCount = readArraySize();
+        if (spare.chars.length < charCount) {
+            // we don't use ArrayUtils.grow since there is no need to copy the array
+            spare.chars = new char[ArrayUtil.oversize(charCount, Character.BYTES)];
+        }
+        spare.length = charCount;
+        final char[] buffer = spare.chars;
+        for (int i = 0; i < charCount; i++) {
+            final int c = readByte() & 0xff;
             switch (c >> 4) {
                 case 0:
                 case 1:
@@ -341,15 +359,17 @@ public String readString() throws IOException {
                 case 5:
                 case 6:
                 case 7:
-                    spare.append((char) c);
+                    buffer[i] = (char) c;
                     break;
                 case 12:
                 case 13:
-                    spare.append((char) ((c & 0x1F) << 6 | readByte() & 0x3F));
+                    buffer[i] = ((char) ((c & 0x1F) << 6 | readByte() & 0x3F));
                     break;
                 case 14:
-                    spare.append((char) ((c & 0x0F) << 12 | (readByte() & 0x3F) << 6 | (readByte() & 0x3F) << 0));
+                    buffer[i] = ((char) ((c & 0x0F) << 12 | (readByte() & 0x3F) << 6 | (readByte() & 0x3F) << 0));
                     break;
+                default:
+                    throw new IOException("Invalid string; unexpected character: " + c + " hex: " + Integer.toHexString(c));
             }
         }
         return spare.toString();
@@ -376,19 +396,28 @@ public final Double readOptionalDouble() throws IOException {
      * Reads a boolean.
      */
     public final boolean readBoolean() throws IOException {
-        return readByte() != 0;
+        return readBoolean(readByte());
+    }
+
+    private boolean readBoolean(final byte value) {
+        if (value == 0) {
+            return false;
+        } else if (value == 1) {
+            return true;
+        } else {
+            final String message = String.format(Locale.ROOT, "unexpected byte [0x%02x]", value);
+            throw new IllegalStateException(message);
+        }
     }
 
     @Nullable
     public final Boolean readOptionalBoolean() throws IOException {
-        byte val = readByte();
-        if (val == 2) {
+        final byte value = readByte();
+        if (value == 2) {
             return null;
+        } else {
+            return readBoolean(value);
         }
-        if (val == 1) {
-            return true;
-        }
-        return false;
     }
 
     /**
@@ -401,7 +430,7 @@ public final Boolean readOptionalBoolean() throws IOException {
     public abstract int available() throws IOException;
 
     public String[] readStringArray() throws IOException {
-        int size = readVInt();
+        int size = readArraySize();
         if (size == 0) {
             return Strings.EMPTY_ARRAY;
         }
@@ -421,7 +450,7 @@ public String[] readOptionalStringArray() throws IOException {
     }
 
     public <K, V> Map<K, V> readMap(Writeable.Reader<K> keyReader, Writeable.Reader<V> valueReader) throws IOException {
-        int size = readVInt();
+        int size = readArraySize();
         Map<K, V> map = new HashMap<>(size);
         for (int i = 0; i < size; i++) {
             K key = keyReader.read(this);
@@ -443,7 +472,7 @@ public <K, V> Map<K, V> readMap(Writeable.Reader<K> keyReader, Writeable.Reader<
      */
     public <K, V> Map<K, List<V>> readMapOfLists(final Writeable.Reader<K> keyReader, final Writeable.Reader<V> valueReader)
             throws IOException {
-        final int size = readVInt();
+        final int size = readArraySize();
         if (size == 0) {
             return Collections.emptyMap();
         }
@@ -520,7 +549,7 @@ public Object readGenericValue() throws IOException {
 
     @SuppressWarnings("unchecked")
     private List readArrayList() throws IOException {
-        int size = readVInt();
+        int size = readArraySize();
         List list = new ArrayList(size);
         for (int i = 0; i < size; i++) {
             list.add(readGenericValue());
@@ -534,7 +563,7 @@ private DateTime readDateTime() throws IOException {
     }
 
     private Object[] readArray() throws IOException {
-        int size8 = readVInt();
+        int size8 = readArraySize();
         Object[] list8 = new Object[size8];
         for (int i = 0; i < size8; i++) {
             list8[i] = readGenericValue();
@@ -543,7 +572,7 @@ private Object[] readArray() throws IOException {
     }
 
     private Map readLinkedHashMap() throws IOException {
-        int size9 = readVInt();
+        int size9 = readArraySize();
         Map map9 = new LinkedHashMap(size9);
         for (int i = 0; i < size9; i++) {
             map9.put(readString(), readGenericValue());
@@ -552,7 +581,7 @@ private Map readLinkedHashMap() throws IOException {
     }
 
     private Map readHashMap() throws IOException {
-        int size10 = readVInt();
+        int size10 = readArraySize();
         Map map10 = new HashMap(size10);
         for (int i = 0; i < size10; i++) {
             map10.put(readString(), readGenericValue());
@@ -589,7 +618,7 @@ public DateTimeZone readOptionalTimeZone() throws IOException {
     }
 
     public int[] readIntArray() throws IOException {
-        int length = readVInt();
+        int length = readArraySize();
         int[] values = new int[length];
         for (int i = 0; i < length; i++) {
             values[i] = readInt();
@@ -598,7 +627,7 @@ public int[] readIntArray() throws IOException {
     }
 
     public int[] readVIntArray() throws IOException {
-        int length = readVInt();
+        int length = readArraySize();
         int[] values = new int[length];
         for (int i = 0; i < length; i++) {
             values[i] = readVInt();
@@ -607,7 +636,7 @@ public int[] readVIntArray() throws IOException {
     }
 
     public long[] readLongArray() throws IOException {
-        int length = readVInt();
+        int length = readArraySize();
         long[] values = new long[length];
         for (int i = 0; i < length; i++) {
             values[i] = readLong();
@@ -616,7 +645,7 @@ public long[] readLongArray() throws IOException {
     }
 
     public long[] readVLongArray() throws IOException {
-        int length = readVInt();
+        int length = readArraySize();
         long[] values = new long[length];
         for (int i = 0; i < length; i++) {
             values[i] = readVLong();
@@ -625,7 +654,7 @@ public long[] readVLongArray() throws IOException {
     }
 
     public float[] readFloatArray() throws IOException {
-        int length = readVInt();
+        int length = readArraySize();
         float[] values = new float[length];
         for (int i = 0; i < length; i++) {
             values[i] = readFloat();
@@ -634,7 +663,7 @@ public float[] readFloatArray() throws IOException {
     }
 
     public double[] readDoubleArray() throws IOException {
-        int length = readVInt();
+        int length = readArraySize();
         double[] values = new double[length];
         for (int i = 0; i < length; i++) {
             values[i] = readDouble();
@@ -643,14 +672,14 @@ public double[] readDoubleArray() throws IOException {
     }
 
     public byte[] readByteArray() throws IOException {
-        final int length = readVInt();
+        final int length = readArraySize();
         final byte[] bytes = new byte[length];
         readBytes(bytes, 0, bytes.length);
         return bytes;
     }
 
     public <T> T[] readArray(Writeable.Reader<T> reader, IntFunction<T[]> arraySupplier) throws IOException {
-        int length = readVInt();
+        int length = readArraySize();
         T[] values = arraySupplier.apply(length);
         for (int i = 0; i < length; i++) {
             values[i] = reader.read(this);
@@ -781,7 +810,7 @@ public <T extends Exception> T readException() throws IOException {
                 case 17:
                     return (T) readStackTrace(new IOException(readOptionalString(), readException()), this);
                 default:
-                    assert false : "no such exception for id: " + key;
+                    throw new IOException("no such exception for id: " + key);
             }
         }
         return null;
@@ -798,6 +827,22 @@ public <C extends NamedWriteable> C readNamedWriteable(@SuppressWarnings("unused
         throw new UnsupportedOperationException("can't read named writeable from StreamInput");
     }
 
+    /**
+     * Reads a {@link NamedWriteable} from the current stream with the given name. It is assumed that the caller obtained the name
+     * from other source, so it's not read from the stream. The name is used for looking for
+     * the corresponding entry in the registry by name, so that the proper object can be read and returned.
+     * Default implementation throws {@link UnsupportedOperationException} as StreamInput doesn't hold a registry.
+     * Use {@link FilterInputStream} instead which wraps a stream and supports a {@link NamedWriteableRegistry} too.
+     *
+     * Prefer {@link StreamInput#readNamedWriteable(Class)} and {@link StreamOutput#writeNamedWriteable(NamedWriteable)} unless you
+     * have a compelling reason to use this method instead.
+     */
+    @Nullable
+    public <C extends NamedWriteable> C readNamedWriteable(@SuppressWarnings("unused") Class<C> categoryClass,
+                                                           @SuppressWarnings("unused") String name) throws IOException {
+        throw new UnsupportedOperationException("can't read named writeable from StreamInput");
+    }
+
     /**
      * Reads an optional {@link NamedWriteable}.
      */
@@ -822,7 +867,7 @@ public <C extends NamedWriteable> C readOptionalNamedWriteable(Class<C> category
      * @throws IOException if any step fails
      */
     public <T extends Streamable> List<T> readStreamableList(Supplier<T> constructor) throws IOException {
-        int count = readVInt();
+        int count = readArraySize();
         List<T> builder = new ArrayList<>(count);
         for (int i=0; i<count; i++) {
             T instance = constructor.get();
@@ -836,7 +881,7 @@ public <T extends Streamable> List<T> readStreamableList(Supplier<T> constructor
      * Reads a list of objects
      */
     public <T> List<T> readList(Writeable.Reader<T> reader) throws IOException {
-        int count = readVInt();
+        int count = readArraySize();
         List<T> builder = new ArrayList<>(count);
         for (int i=0; i<count; i++) {
             builder.add(reader.read(this));
@@ -848,7 +893,7 @@ public <T> List<T> readList(Writeable.Reader<T> reader) throws IOException {
      * Reads a list of {@link NamedWriteable}s.
      */
     public <T extends NamedWriteable> List<T> readNamedWriteableList(Class<T> categoryClass) throws IOException {
-        int count = readVInt();
+        int count = readArraySize();
         List<T> builder = new ArrayList<>(count);
         for (int i=0; i<count; i++) {
             builder.add(readNamedWriteable(categoryClass));
@@ -856,6 +901,18 @@ public <T extends NamedWriteable> List<T> readNamedWriteableList(Class<T> catego
         return builder;
     }
 
+    /**
+     * Reads an enum with type E that was serialized based on the value of it's ordinal
+     */
+    public <E extends Enum<E>> E readEnum(Class<E> enumClass) throws IOException {
+        int ordinal = readVInt();
+        E[] values = enumClass.getEnumConstants();
+        if (ordinal < 0 || ordinal >= values.length) {
+            throw new IOException("Unknown " + enumClass.getSimpleName() + " ordinal [" + ordinal + "]");
+        }
+        return values[ordinal];
+    }
+
     public static StreamInput wrap(byte[] bytes) {
         return wrap(bytes, 0, bytes.length);
     }
@@ -864,4 +921,28 @@ public static StreamInput wrap(byte[] bytes, int offset, int length) {
         return new InputStreamStreamInput(new ByteArrayInputStream(bytes, offset, length));
     }
 
+    /**
+     * Reads a vint via {@link #readVInt()} and applies basic checks to ensure the read array size is sane.
+     * This method uses {@link #ensureCanReadBytes(int)} to ensure this stream has enough bytes to read for the read array size.
+     */
+    private int readArraySize() throws IOException {
+        final int arraySize = readVInt();
+        if (arraySize > ArrayUtil.MAX_ARRAY_LENGTH) {
+            throw new IllegalStateException("array length must be <= to " + ArrayUtil.MAX_ARRAY_LENGTH  + " but was: " + arraySize);
+        }
+        if (arraySize < 0) {
+            throw new NegativeArraySizeException("array size must be positive but was: " + arraySize);
+        }
+        // lets do a sanity check that if we are reading an array size that is bigger that the remaining bytes we can safely
+        // throw an exception instead of allocating the array based on the size. A simple corrutpted byte can make a node go OOM
+        // if the size is large and for perf reasons we allocate arrays ahead of time
+        ensureCanReadBytes(arraySize);
+        return arraySize;
+    }
+
+    /**
+     * This method throws an {@link EOFException} if the given number of bytes can not be read from the this stream. This method might
+     * be a no-op depending on the underlying implementation if the information of the remaining bytes is not present.
+     */
+    protected abstract void ensureCanReadBytes(int length) throws EOFException;
 }
diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java b/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
index c1932fd4215ce..7b4df4b2d533a 100644
--- a/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
+++ b/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
@@ -24,6 +24,7 @@
 import org.apache.lucene.index.IndexFormatTooOldException;
 import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.LockObtainFailedException;
+import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.BitUtil;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefBuilder;
@@ -209,12 +210,22 @@ public void writeLong(long i) throws IOException {
     }
 
     /**
-     * Writes a non-negative long in a variable-length format.
-     * Writes between one and nine bytes. Smaller values take fewer bytes.
-     * Negative numbers are not supported.
+     * Writes a non-negative long in a variable-length format. Writes between one and ten bytes. Smaller values take fewer bytes. Negative
+     * numbers use ten bytes and trip assertions (if running in tests) so prefer {@link #writeLong(long)} or {@link #writeZLong(long)} for
+     * negative numbers.
      */
     public void writeVLong(long i) throws IOException {
-        assert i >= 0;
+        if (i < 0) {
+            throw new IllegalStateException("Negative longs unsupported, use writeLong or writeZLong for negative numbers [" + i + "]");
+        }
+        writeVLongNoCheck(i);
+    }
+
+    /**
+     * Writes a long in a variable-length format without first checking if it is negative. Package private for testing. Use
+     * {@link #writeVLong(long)} instead.
+     */
+    void writeVLongNoCheck(long i) throws IOException {
         while ((i & ~0x7F) != 0) {
             writeByte((byte) ((i & 0x7f) | 0x80));
             i >>>= 7;
@@ -298,23 +309,41 @@ public void writeText(Text text) throws IOException {
         }
     }
 
+    // we use a small buffer to convert strings to bytes since we want to prevent calling writeByte
+    // for every byte in the string (see #21660 for details).
+    // This buffer will never be the oversized limit of 1024 bytes and will not be shared across streams
+    private byte[] convertStringBuffer = BytesRef.EMPTY_BYTES; // TODO should we reduce it to 0 bytes once the stream is closed?
+
     public void writeString(String str) throws IOException {
-        int charCount = str.length();
+        final int charCount = str.length();
+        final int bufferSize = Math.min(3 * charCount, 1024); // at most 3 bytes per character is needed here
+        if (convertStringBuffer.length < bufferSize) { // we don't use ArrayUtils.grow since copying the bytes is unnecessary
+            convertStringBuffer = new byte[ArrayUtil.oversize(bufferSize, Byte.BYTES)];
+        }
+        byte[] buffer = convertStringBuffer;
+        int offset = 0;
         writeVInt(charCount);
-        int c;
         for (int i = 0; i < charCount; i++) {
-            c = str.charAt(i);
+            final int c = str.charAt(i);
             if (c <= 0x007F) {
-                writeByte((byte) c);
+                buffer[offset++] = ((byte) c);
             } else if (c > 0x07FF) {
-                writeByte((byte) (0xE0 | c >> 12 & 0x0F));
-                writeByte((byte) (0x80 | c >> 6 & 0x3F));
-                writeByte((byte) (0x80 | c >> 0 & 0x3F));
+                buffer[offset++] = ((byte) (0xE0 | c >> 12 & 0x0F));
+                buffer[offset++] = ((byte) (0x80 | c >> 6 & 0x3F));
+                buffer[offset++] = ((byte) (0x80 | c >> 0 & 0x3F));
             } else {
-                writeByte((byte) (0xC0 | c >> 6 & 0x1F));
-                writeByte((byte) (0x80 | c >> 0 & 0x3F));
+                buffer[offset++] = ((byte) (0xC0 | c >> 6 & 0x1F));
+                buffer[offset++] = ((byte) (0x80 | c >> 0 & 0x3F));
+            }
+            // make sure any possible char can fit into the buffer in any possible iteration
+            // we need at most 3 bytes so we flush the buffer once we have less than 3 bytes
+            // left before we start another iteration
+            if (offset > buffer.length-3) {
+                writeBytes(buffer, offset);
+                offset = 0;
             }
         }
+        writeBytes(buffer, offset);
     }
 
     public void writeFloat(float v) throws IOException {
@@ -349,7 +378,7 @@ public void writeOptionalBoolean(@Nullable Boolean b) throws IOException {
         if (b == null) {
             writeByte(TWO);
         } else {
-            writeByte(b ? ONE : ZERO);
+            writeBoolean(b);
         }
     }
 
@@ -448,16 +477,32 @@ public void writeMapWithConsistentOrder(@Nullable Map<String, ? extends Object>
      * @param keyWriter The key writer
      * @param valueWriter The value writer
      */
-    public <K, V> void writeMapOfLists(final Map<K, List<V>> map, final Writer<K> keyWriter, final Writer<V> valueWriter)
+    public final <K, V> void writeMapOfLists(final Map<K, List<V>> map, final Writer<K> keyWriter, final Writer<V> valueWriter)
             throws IOException {
-        writeVInt(map.size());
-
-        for (final Map.Entry<K, List<V>> entry : map.entrySet()) {
-            keyWriter.write(this, entry.getKey());
-            writeVInt(entry.getValue().size());
-            for (final V value : entry.getValue()) {
+        writeMap(map, keyWriter, (stream, list) -> {
+            writeVInt(list.size());
+            for (final V value : list) {
                 valueWriter.write(this, value);
             }
+        });
+    }
+
+    /**
+     * Write a {@link Map} of {@code K}-type keys to {@code V}-type.
+     * <pre><code>
+     * Map&lt;String, String&gt; map = ...;
+     * out.writeMap(map, StreamOutput::writeString, StreamOutput::writeString);
+     * </code></pre>
+     *
+     * @param keyWriter The key writer
+     * @param valueWriter The value writer
+     */
+    public final <K, V> void writeMap(final Map<K, V> map, final Writer<K> keyWriter, final Writer<V> valueWriter)
+        throws IOException {
+        writeVInt(map.size());
+        for (final Map.Entry<K, V> entry : map.entrySet()) {
+            keyWriter.write(this, entry.getKey());
+            valueWriter.write(this, entry.getValue());
         }
     }
 
@@ -783,7 +828,7 @@ public void writeException(Throwable throwable) throws IOException {
                 writeVInt(17);
             } else {
                 ElasticsearchException ex;
-                if (throwable instanceof ElasticsearchException && ElasticsearchException.isRegistered(throwable.getClass())) {
+                if (throwable instanceof ElasticsearchException && ElasticsearchException.isRegistered(throwable.getClass(), version)) {
                     ex = (ElasticsearchException) throwable;
                 } else {
                     ex = new NotSerializableExceptionWrapper(throwable);
@@ -871,6 +916,16 @@ public void writeList(List<? extends Writeable> list) throws IOException {
         }
     }
 
+    /**
+     * Writes a list of strings
+     */
+    public void writeStringList(List<String> list) throws IOException {
+        writeVInt(list.size());
+        for (String string: list) {
+            this.writeString(string);
+        }
+    }
+
     /**
      * Writes a list of {@link NamedWriteable} objects.
      */
@@ -880,4 +935,12 @@ public void writeNamedWriteableList(List<? extends NamedWriteable> list) throws
             writeNamedWriteable(obj);
         }
     }
+
+    /**
+     * Writes an enum with type E that by serialized it based on it's ordinal value
+     */
+    public <E extends Enum<E>> void writeEnum(E enumValue) throws IOException {
+        writeVInt(enumValue.ordinal());
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/Writeable.java b/core/src/main/java/org/elasticsearch/common/io/stream/Writeable.java
index 30607f3375909..9d645038d6528 100644
--- a/core/src/main/java/org/elasticsearch/common/io/stream/Writeable.java
+++ b/core/src/main/java/org/elasticsearch/common/io/stream/Writeable.java
@@ -34,7 +34,7 @@ public interface Writeable {
     /**
      * Write this into the {@linkplain StreamOutput}.
      */
-    void writeTo(final StreamOutput out) throws IOException;
+    void writeTo(StreamOutput out) throws IOException;
 
     /**
      * Reference to a method that can write some object to a {@link StreamOutput}.
@@ -60,7 +60,7 @@ interface Writer<V> {
          * @param out Output to write the {@code value} too
          * @param value The value to add
          */
-        void write(final StreamOutput out, final V value) throws IOException;
+        void write(StreamOutput out, V value) throws IOException;
 
     }
 
@@ -86,7 +86,7 @@ interface Reader<V> {
          *
          * @param in Input to read the value from
          */
-        V read(final StreamInput in) throws IOException;
+        V read(StreamInput in) throws IOException;
 
     }
 
diff --git a/core/src/main/java/org/elasticsearch/common/joda/DateMathParser.java b/core/src/main/java/org/elasticsearch/common/joda/DateMathParser.java
index 65ec7e7c2b445..ba5531c813c12 100644
--- a/core/src/main/java/org/elasticsearch/common/joda/DateMathParser.java
+++ b/core/src/main/java/org/elasticsearch/common/joda/DateMathParser.java
@@ -25,11 +25,11 @@
 import org.joda.time.format.DateTimeFormatter;
 
 import java.util.Objects;
-import java.util.concurrent.Callable;
+import java.util.function.LongSupplier;
 
 /**
  * A parser for date/time formatted text with optional date math.
- * 
+ *
  * The format of the datetime is configurable, and unix timestamps can also be used. Datemath
  * is appended to a datetime with the following syntax:
  * <code>||[+-/](\d+)?[yMwdhHms]</code>.
@@ -43,19 +43,19 @@ public DateMathParser(FormatDateTimeFormatter dateTimeFormatter) {
         this.dateTimeFormatter = dateTimeFormatter;
     }
 
-    public long parse(String text, Callable<Long> now) {
+    public long parse(String text, LongSupplier now) {
         return parse(text, now, false, null);
     }
 
     // Note: we take a callable here for the timestamp in order to be able to figure out
     // if it has been used. For instance, the request cache does not cache requests that make
     // use of `now`.
-    public long parse(String text, Callable<Long> now, boolean roundUp, DateTimeZone timeZone) {
+    public long parse(String text, LongSupplier now, boolean roundUp, DateTimeZone timeZone) {
         long time;
         String mathString;
         if (text.startsWith("now")) {
             try {
-                time = now.call();
+                time = now.getAsLong();
             } catch (Exception e) {
                 throw new ElasticsearchParseException("could not read the current timestamp", e);
             }
@@ -63,13 +63,10 @@ public long parse(String text, Callable<Long> now, boolean roundUp, DateTimeZone
         } else {
             int index = text.indexOf("||");
             if (index == -1) {
-                return parseDateTime(text, timeZone);
+                return parseDateTime(text, timeZone, roundUp);
             }
-            time = parseDateTime(text.substring(0, index), timeZone);
+            time = parseDateTime(text.substring(0, index), timeZone, false);
             mathString = text.substring(index + 2);
-            if (mathString.isEmpty()) {
-                return time;
-            }
         }
 
         return parseMath(mathString, time, roundUp, timeZone);
@@ -97,7 +94,7 @@ private long parseMath(String mathString, long time, boolean roundUp, DateTimeZo
                     throw new ElasticsearchParseException("operator not supported for date math [{}]", mathString);
                 }
             }
-                
+
             if (i >= mathString.length()) {
                 throw new ElasticsearchParseException("truncated date math [{}]", mathString);
             }
@@ -190,15 +187,29 @@ private long parseMath(String mathString, long time, boolean roundUp, DateTimeZo
         return dateTime.getMillis();
     }
 
-    private long parseDateTime(String value, DateTimeZone timeZone) {
+    private long parseDateTime(String value, DateTimeZone timeZone, boolean roundUpIfNoTime) {
         DateTimeFormatter parser = dateTimeFormatter.parser();
         if (timeZone != null) {
             parser = parser.withZone(timeZone);
         }
         try {
-            return parser.parseMillis(value);
+            MutableDateTime date;
+            // We use 01/01/1970 as a base date so that things keep working with date
+            // fields that are filled with times without dates
+            if (roundUpIfNoTime) {
+                date = new MutableDateTime(1970, 1, 1, 23, 59, 59, 999, DateTimeZone.UTC);
+            } else {
+                date = new MutableDateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeZone.UTC);
+            }
+            final int end = parser.parseInto(date, value, 0);
+            if (end < 0) {
+                int position = ~end;
+                throw new IllegalArgumentException("Parse failure at index [" + position + "] of [" + value + "]");
+            } else if (end != value.length()) {
+                throw new IllegalArgumentException("Unrecognized chars at the end of [" + value + "]: [" + value.substring(end) + "]");
+            }
+            return date.getMillis();
         } catch (IllegalArgumentException e) {
-            
             throw new ElasticsearchParseException("failed to parse date field [{}] with format [{}]", e, value, dateTimeFormatter.format());
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/common/joda/FormatDateTimeFormatter.java b/core/src/main/java/org/elasticsearch/common/joda/FormatDateTimeFormatter.java
index 428828bc0feb7..72a60e8678ca2 100644
--- a/core/src/main/java/org/elasticsearch/common/joda/FormatDateTimeFormatter.java
+++ b/core/src/main/java/org/elasticsearch/common/joda/FormatDateTimeFormatter.java
@@ -22,6 +22,7 @@
 import org.joda.time.format.DateTimeFormatter;
 
 import java.util.Locale;
+import java.util.Objects;
 
 /**
  * A simple wrapper around {@link DateTimeFormatter} that retains the
@@ -43,9 +44,9 @@ public FormatDateTimeFormatter(String format, DateTimeFormatter parser, Locale l
 
     public FormatDateTimeFormatter(String format, DateTimeFormatter parser, DateTimeFormatter printer, Locale locale) {
         this.format = format;
-        this.locale = locale;
-        this.printer = locale == null ? printer.withDefaultYear(1970) : printer.withLocale(locale).withDefaultYear(1970);
-        this.parser = locale == null ? parser.withDefaultYear(1970) : parser.withLocale(locale).withDefaultYear(1970);
+        this.locale = Objects.requireNonNull(locale, "A locale is required as JODA otherwise uses the default locale");
+        this.printer = printer.withLocale(locale).withDefaultYear(1970);
+        this.parser = parser.withLocale(locale).withDefaultYear(1970);
     }
 
     public String format() {
diff --git a/core/src/main/java/org/elasticsearch/common/joda/Joda.java b/core/src/main/java/org/elasticsearch/common/joda/Joda.java
index 34c882d0d8096..2da0e7d2b6865 100644
--- a/core/src/main/java/org/elasticsearch/common/joda/Joda.java
+++ b/core/src/main/java/org/elasticsearch/common/joda/Joda.java
@@ -336,9 +336,10 @@ public int parseInto(DateTimeParserBucket bucket, String text, int position) {
             boolean isPositive = text.startsWith("-") == false;
             boolean isTooLong = text.length() > estimateParsedLength();
 
-            if ((isPositive && isTooLong) ||
-                // timestamps have to have UTC timezone
-                bucket.getZone() != DateTimeZone.UTC) {
+            if (bucket.getZone() != DateTimeZone.UTC) {
+                String format = hasMilliSecondPrecision ? "epoch_millis" : "epoch_second";
+                throw new IllegalArgumentException("time_zone must be UTC for format [" + format + "]");
+            } else if (isPositive && isTooLong) {
                 return -1;
             }
 
diff --git a/core/src/main/java/org/elasticsearch/common/lease/Releasables.java b/core/src/main/java/org/elasticsearch/common/lease/Releasables.java
index bfabd20976d36..bd7b2a6e772a4 100644
--- a/core/src/main/java/org/elasticsearch/common/lease/Releasables.java
+++ b/core/src/main/java/org/elasticsearch/common/lease/Releasables.java
@@ -24,6 +24,7 @@
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.util.Arrays;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 /** Utility methods to work with {@link Releasable}s. */
 public enum Releasables {
@@ -93,4 +94,16 @@ public static Releasable wrap(final Iterable<Releasable> releasables) {
     public static Releasable wrap(final Releasable... releasables) {
         return () -> close(releasables);
     }
+
+    /**
+     * Equivalent to {@link #wrap(Releasable...)} but can be called multiple times without double releasing.
+     */
+    public static Releasable releaseOnce(final Releasable... releasables) {
+        final AtomicBoolean released = new AtomicBoolean(false);
+        return () -> {
+            if (released.compareAndSet(false, true)) {
+                close(releasables);
+            }
+        };
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/logging/DeprecationLogger.java b/core/src/main/java/org/elasticsearch/common/logging/DeprecationLogger.java
index d9b811585de0d..3ed1d9d30ac1a 100644
--- a/core/src/main/java/org/elasticsearch/common/logging/DeprecationLogger.java
+++ b/core/src/main/java/org/elasticsearch/common/logging/DeprecationLogger.java
@@ -21,13 +21,35 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.elasticsearch.Build;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.SuppressLoggerChecks;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.time.format.SignStyle;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.CopyOnWriteArraySet;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static java.time.temporal.ChronoField.DAY_OF_MONTH;
+import static java.time.temporal.ChronoField.DAY_OF_WEEK;
+import static java.time.temporal.ChronoField.HOUR_OF_DAY;
+import static java.time.temporal.ChronoField.MINUTE_OF_HOUR;
+import static java.time.temporal.ChronoField.MONTH_OF_YEAR;
+import static java.time.temporal.ChronoField.SECOND_OF_MINUTE;
+import static java.time.temporal.ChronoField.YEAR;
 
 /**
  * A logger that logs deprecation notices.
@@ -36,14 +58,6 @@ public class DeprecationLogger {
 
     private final Logger logger;
 
-    /**
-     * The "Warning" Header comes from RFC-7234. As the RFC describes, it's generally used for caching purposes, but it can be
-     * used for <em>any</em> warning.
-     *
-     * https://tools.ietf.org/html/rfc7234#section-5.5
-     */
-    public static final String DEPRECATION_HEADER = "Warning";
-
     /**
      * This is set once by the {@code Node} constructor, but it uses {@link CopyOnWriteArraySet} to ensure that tests can run in parallel.
      * <p>
@@ -64,7 +78,7 @@ public class DeprecationLogger {
      * @throws IllegalStateException if this {@code threadContext} has already been set
      */
     public static void setThreadContext(ThreadContext threadContext) {
-        assert threadContext != null;
+        Objects.requireNonNull(threadContext, "Cannot register a null ThreadContext");
 
         // add returning false means it _did_ have it already
         if (THREAD_CONTEXT.add(threadContext) == false) {
@@ -102,16 +116,175 @@ public DeprecationLogger(Logger parentLogger) {
         } else {
             name = "deprecation." + name;
         }
-        this.logger = LogManager.getLogger(name, parentLogger.getMessageFactory());
+        this.logger = LogManager.getLogger(name);
     }
 
     /**
-     * Logs a deprecated message.
+     * Logs a deprecation message, adding a formatted warning message as a response header on the thread context.
      */
     public void deprecated(String msg, Object... params) {
         deprecated(THREAD_CONTEXT, msg, params);
     }
 
+    // LRU set of keys used to determine if a deprecation message should be emitted to the deprecation logs
+    private Set<String> keys = Collections.newSetFromMap(Collections.synchronizedMap(new LinkedHashMap<String, Boolean>() {
+        @Override
+        protected boolean removeEldestEntry(final Map.Entry eldest) {
+            return size() > 128;
+        }
+    }));
+
+    /**
+     * Adds a formatted warning message as a response header on the thread context, and logs a deprecation message if the associated key has
+     * not recently been seen.
+     *
+     * @param key    the key used to determine if this deprecation should be logged
+     * @param msg    the message to log
+     * @param params parameters to the message
+     */
+    public void deprecatedAndMaybeLog(final String key, final String msg, final Object... params) {
+        deprecated(THREAD_CONTEXT, msg, keys.add(key), params);
+    }
+
+    /*
+     * RFC7234 specifies the warning format as warn-code <space> warn-agent <space> "warn-text" [<space> "warn-date"]. Here, warn-code is a
+     * three-digit number with various standard warn codes specified. The warn code 299 is apt for our purposes as it represents a
+     * miscellaneous persistent warning (can be presented to a human, or logged, and must not be removed by a cache). The warn-agent is an
+     * arbitrary token; here we use the Elasticsearch version and build hash. The warn text must be quoted. The warn-date is an optional
+     * quoted field that can be in a variety of specified date formats; here we use RFC 1123 format.
+     */
+    private static final String WARNING_FORMAT =
+            String.format(
+                    Locale.ROOT,
+                    "299 Elasticsearch-%s%s-%s ",
+                    Version.CURRENT.toString(),
+                    Build.CURRENT.isSnapshot() ? "-SNAPSHOT" : "",
+                    Build.CURRENT.shortHash()) +
+                    "\"%s\" \"%s\"";
+
+    /*
+     * RFC 7234 section 5.5 specifies that the warn-date is a quoted HTTP-date. HTTP-date is defined in RFC 7234 Appendix B as being from
+     * RFC 7231 section 7.1.1.1. RFC 7231 specifies an HTTP-date as an IMF-fixdate (or an obs-date referring to obsolete formats). The
+     * grammar for IMF-fixdate is specified as 'day-name "," SP date1 SP time-of-day SP GMT'. Here, day-name is
+     * (Mon|Tue|Wed|Thu|Fri|Sat|Sun). Then, date1 is 'day SP month SP year' where day is 2DIGIT, month is
+     * (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec), and year is 4DIGIT. Lastly, time-of-day is 'hour ":" minute ":" second' where
+     * hour is 2DIGIT, minute is 2DIGIT, and second is 2DIGIT. Finally, 2DIGIT and 4DIGIT have the obvious definitions.
+     */
+    private static final DateTimeFormatter RFC_7231_DATE_TIME;
+
+    static {
+        final Map<Long, String> dow = new HashMap<>();
+        dow.put(1L, "Mon");
+        dow.put(2L, "Tue");
+        dow.put(3L, "Wed");
+        dow.put(4L, "Thu");
+        dow.put(5L, "Fri");
+        dow.put(6L, "Sat");
+        dow.put(7L, "Sun");
+        final Map<Long, String> moy = new HashMap<>();
+        moy.put(1L, "Jan");
+        moy.put(2L, "Feb");
+        moy.put(3L, "Mar");
+        moy.put(4L, "Apr");
+        moy.put(5L, "May");
+        moy.put(6L, "Jun");
+        moy.put(7L, "Jul");
+        moy.put(8L, "Aug");
+        moy.put(9L, "Sep");
+        moy.put(10L, "Oct");
+        moy.put(11L, "Nov");
+        moy.put(12L, "Dec");
+        RFC_7231_DATE_TIME = new DateTimeFormatterBuilder()
+                .parseCaseInsensitive()
+                .parseLenient()
+                .optionalStart()
+                .appendText(DAY_OF_WEEK, dow)
+                .appendLiteral(", ")
+                .optionalEnd()
+                .appendValue(DAY_OF_MONTH, 2, 2, SignStyle.NOT_NEGATIVE)
+                .appendLiteral(' ')
+                .appendText(MONTH_OF_YEAR, moy)
+                .appendLiteral(' ')
+                .appendValue(YEAR, 4)
+                .appendLiteral(' ')
+                .appendValue(HOUR_OF_DAY, 2)
+                .appendLiteral(':')
+                .appendValue(MINUTE_OF_HOUR, 2)
+                .optionalStart()
+                .appendLiteral(':')
+                .appendValue(SECOND_OF_MINUTE, 2)
+                .optionalEnd()
+                .appendLiteral(' ')
+                .appendOffset("+HHMM", "GMT")
+                .toFormatter(Locale.getDefault(Locale.Category.FORMAT));
+    }
+
+    private static final ZoneId GMT = ZoneId.of("GMT");
+
+    /**
+     * Regular expression to test if a string matches the RFC7234 specification for warning headers. This pattern assumes that the warn code
+     * is always 299. Further, this pattern assumes that the warn agent represents a version of Elasticsearch including the build hash.
+     */
+    public static Pattern WARNING_HEADER_PATTERN = Pattern.compile(
+            "299 " + // warn code
+                    "Elasticsearch-\\d+\\.\\d+\\.\\d+(?:-(?:alpha|beta|rc)\\d+)?(?:-SNAPSHOT)?-(?:[a-f0-9]{7}|Unknown) " + // warn agent
+                    "\"((?:\t| |!|[\\x23-\\x5b]|[\\x5d-\\x7e]|[\\x80-\\xff]|\\\\|\\\\\")*)\" " + // quoted warning value, captured
+                    // quoted RFC 1123 date format
+                    "\"" + // opening quote
+                    "(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun), " + // weekday
+                    "\\d{2} " + // 2-digit day
+                    "(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) " + // month
+                    "\\d{4} " + // 4-digit year
+                    "\\d{2}:\\d{2}:\\d{2} " + // (two-digit hour):(two-digit minute):(two-digit second)
+                    "GMT" + // GMT
+                    "\""); // closing quote
+
+    /**
+     * Extracts the warning value from the value of a warning header that is formatted according to RFC 7234. That is, given a string
+     * {@code 299 Elasticsearch-6.0.0 "warning value" "Sat, 25 Feb 2017 10:27:43 GMT"}, the return value of this method would be {@code
+     * warning value}.
+     *
+     * @param s the value of a warning header formatted according to RFC 7234.
+     * @return the extracted warning value
+     */
+    public static String extractWarningValueFromWarningHeader(final String s) {
+        /*
+         * We know the exact format of the warning header, so to extract the warning value we can skip forward from the front to the first
+         * quote, and skip backwards from the end to the penultimate quote:
+         *
+         *   299 Elasticsearch-6.0.0 "warning value" "Sat, 25, Feb 2017 10:27:43 GMT"
+         *                           ^               ^                              ^
+         *                           firstQuote      penultimateQuote               lastQuote
+         *
+         * We do it this way rather than seeking forward after the first quote because there could be escaped quotes in the warning value
+         * but since there are none in the warning date, we can skip backwards to find the quote that closes the quoted warning value.
+         *
+         * We parse this manually rather than using the capturing regular expression because the regular expression involves a lot of
+         * backtracking and carries a performance penalty. However, when assertions are enabled, we still use the regular expression to
+         * verify that we are maintaining the warning header format.
+         */
+        final int firstQuote = s.indexOf('\"');
+        final int lastQuote = s.lastIndexOf('\"');
+        final int penultimateQuote = s.lastIndexOf('\"', lastQuote - 1);
+        final String warningValue = s.substring(firstQuote + 1, penultimateQuote - 2);
+        assert assertWarningValue(s, warningValue);
+        return warningValue;
+    }
+
+    /**
+     * Assert that the specified string has the warning value equal to the provided warning value.
+     *
+     * @param s            the string representing a full warning header
+     * @param warningValue the expected warning header
+     * @return {@code true} if the specified string has the expected warning value
+     */
+    private static boolean assertWarningValue(final String s, final String warningValue) {
+        final Matcher matcher = WARNING_HEADER_PATTERN.matcher(s);
+        final boolean matches = matcher.matches();
+        assert matches;
+        return matcher.group(1).equals(warningValue);
+    }
+
     /**
      * Logs a deprecated message to the deprecation log, as well as to the local {@link ThreadContext}.
      *
@@ -119,24 +292,53 @@ public void deprecated(String msg, Object... params) {
      * @param message The deprecation message.
      * @param params The parameters used to fill in the message, if any exist.
      */
+    void deprecated(final Set<ThreadContext> threadContexts, final String message, final Object... params) {
+        deprecated(threadContexts, message, true, params);
+    }
+
     @SuppressLoggerChecks(reason = "safely delegates to logger")
-    void deprecated(Set<ThreadContext> threadContexts, String message, Object... params) {
-        Iterator<ThreadContext> iterator = threadContexts.iterator();
+    void deprecated(final Set<ThreadContext> threadContexts, final String message, final boolean log, final Object... params) {
+        final Iterator<ThreadContext> iterator = threadContexts.iterator();
 
         if (iterator.hasNext()) {
             final String formattedMessage = LoggerMessageFormat.format(message, params);
-
+            final String warningHeaderValue = formatWarning(formattedMessage);
+            assert WARNING_HEADER_PATTERN.matcher(warningHeaderValue).matches();
+            assert extractWarningValueFromWarningHeader(warningHeaderValue).equals(escape(formattedMessage));
             while (iterator.hasNext()) {
                 try {
-                    iterator.next().addResponseHeader(DEPRECATION_HEADER, formattedMessage);
-                } catch (IllegalStateException e) {
+                    final ThreadContext next = iterator.next();
+                    next.addResponseHeader("Warning", warningHeaderValue, DeprecationLogger::extractWarningValueFromWarningHeader);
+                } catch (final IllegalStateException e) {
                     // ignored; it should be removed shortly
                 }
             }
-            logger.warn(formattedMessage);
-        } else {
+        }
+
+        if (log) {
             logger.warn(message, params);
         }
     }
 
+    /**
+     * Format a warning string in the proper warning format by prepending a warn code, warn agent, wrapping the warning string in quotes,
+     * and appending the RFC 7231 date.
+     *
+     * @param s the warning string to format
+     * @return a warning value formatted according to RFC 7234
+     */
+    public static String formatWarning(final String s) {
+        return String.format(Locale.ROOT, WARNING_FORMAT, escape(s), RFC_7231_DATE_TIME.format(ZonedDateTime.now(GMT)));
+    }
+
+    /**
+     * Escape backslashes and quotes in the specified string.
+     *
+     * @param s the string to escape
+     * @return the escaped string
+     */
+    public static String escape(String s) {
+        return s.replaceAll("([\"\\\\])", "\\\\$1");
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/common/logging/ESLoggerFactory.java b/core/src/main/java/org/elasticsearch/common/logging/ESLoggerFactory.java
index 853df3d31add0..1d6fdf9cd2df2 100644
--- a/core/src/main/java/org/elasticsearch/common/logging/ESLoggerFactory.java
+++ b/core/src/main/java/org/elasticsearch/common/logging/ESLoggerFactory.java
@@ -22,59 +22,52 @@
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.message.MessageFactory;
+import org.apache.logging.log4j.spi.ExtendedLogger;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 
-import java.util.Locale;
-import java.util.function.Function;
-
 /**
  * Factory to get {@link Logger}s
  */
-public abstract class ESLoggerFactory {
+public final class ESLoggerFactory {
+
+    private ESLoggerFactory() {
+
+    }
 
     public static final Setting<Level> LOG_DEFAULT_LEVEL_SETTING =
         new Setting<>("logger.level", Level.INFO.name(), Level::valueOf, Property.NodeScope);
     public static final Setting<Level> LOG_LEVEL_SETTING =
-        Setting.prefixKeySetting("logger.", Level.INFO.name(), Level::valueOf,
-            Property.Dynamic, Property.NodeScope);
+        Setting.prefixKeySetting("logger.", (key) -> new Setting<>(key, Level.INFO.name(), Level::valueOf, Property.Dynamic,
+            Property.NodeScope));
 
     public static Logger getLogger(String prefix, String name) {
-        name = name.intern();
-        final Logger logger = getLogger(new PrefixMessageFactory(), name);
-        final MessageFactory factory = logger.getMessageFactory();
-        // in some cases, we initialize the logger before we are ready to set the prefix
-        // we can not re-initialize the logger, so the above getLogger might return an existing
-        // instance without the prefix set; thus, we hack around this by resetting the prefix
-        if (prefix != null && factory instanceof PrefixMessageFactory) {
-            ((PrefixMessageFactory) factory).setPrefix(prefix.intern());
-        }
-        return logger;
+        return getLogger(prefix, LogManager.getLogger(name));
     }
 
-    public static Logger getLogger(MessageFactory messageFactory, String name) {
-        return LogManager.getLogger(name, messageFactory);
+    public static Logger getLogger(String prefix, Class<?> clazz) {
+        /*
+         * Do not use LogManager#getLogger(Class) as this now uses Class#getCanonicalName under the hood; as this returns null for local and
+         * anonymous classes, any place we create, for example, an abstract component defined as an anonymous class (e.g., in tests) will
+         * result in a logger with a null name which will blow up in a lookup inside of Log4j.
+         */
+        return getLogger(prefix, LogManager.getLogger(clazz.getName()));
     }
 
-    public static Logger getLogger(String name) {
-        return getLogger((String)null, name);
+    public static Logger getLogger(String prefix, Logger logger) {
+        return new PrefixLogger((ExtendedLogger)logger, logger.getName(), prefix);
     }
 
-    public static DeprecationLogger getDeprecationLogger(String name) {
-        return new DeprecationLogger(getLogger(name));
+    public static Logger getLogger(Class<?> clazz) {
+        return getLogger(null, clazz);
     }
 
-    public static DeprecationLogger getDeprecationLogger(String prefix, String name) {
-        return new DeprecationLogger(getLogger(prefix, name));
+    public static Logger getLogger(String name) {
+        return getLogger(null, name);
     }
 
     public static Logger getRootLogger() {
         return LogManager.getRootLogger();
     }
 
-    private ESLoggerFactory() {
-        // Utility class can't be built.
-    }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java b/core/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java
index 68e2886cf372e..4c12206859674 100644
--- a/core/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java
+++ b/core/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java
@@ -30,11 +30,18 @@
 import org.apache.logging.log4j.core.config.composite.CompositeConfiguration;
 import org.apache.logging.log4j.core.config.properties.PropertiesConfiguration;
 import org.apache.logging.log4j.core.config.properties.PropertiesConfigurationFactory;
+import org.apache.logging.log4j.status.StatusConsoleListener;
+import org.apache.logging.log4j.status.StatusData;
+import org.apache.logging.log4j.status.StatusListener;
+import org.apache.logging.log4j.status.StatusLogger;
 import org.elasticsearch.Version;
+import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.UserException;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.node.Node;
 
 import java.io.IOException;
 import java.nio.file.FileVisitOption;
@@ -48,56 +55,153 @@
 import java.util.EnumSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.StreamSupport;
 
 public class LogConfigurator {
 
-    public static void configure(final Environment environment, final boolean resolveConfig) throws IOException {
-        final Settings settings = environment.settings();
+    /*
+     * We want to detect situations where we touch logging before the configuration is loaded. If we do this, Log4j will status log an error
+     * message at the error level. With this error listener, we can capture if this happens. More broadly, we can detect any error-level
+     * status log message which likely indicates that something is broken. The listener is installed immediately on startup, and then when
+     * we get around to configuring logging we check that no error-level log messages have been logged by the status logger. If they have we
+     * fail startup and any such messages can be seen on the console.
+     */
+    private static final AtomicBoolean error = new AtomicBoolean();
+    private static final StatusListener ERROR_LISTENER = new StatusConsoleListener(Level.ERROR) {
+        @Override
+        public void log(StatusData data) {
+            error.set(true);
+            super.log(data);
+        }
+    };
 
-        setLogConfigurationSystemProperty(environment, settings);
+    /**
+     * Registers a listener for status logger errors. This listener should be registered as early as possible to ensure that no errors are
+     * logged by the status logger before logging is configured.
+     */
+    public static void registerErrorListener() {
+        error.set(false);
+        StatusLogger.getLogger().registerListener(ERROR_LISTENER);
+    }
 
+    /**
+     * Configure logging without reading a log4j2.properties file, effectively configuring the
+     * status logger and all loggers to the console.
+     *
+     * @param settings for configuring logger.level and individual loggers
+     */
+    public static void configureWithoutConfig(final Settings settings) {
+        Objects.requireNonNull(settings);
         // we initialize the status logger immediately otherwise Log4j will complain when we try to get the context
-        final ConfigurationBuilder<BuiltConfiguration> builder = ConfigurationBuilderFactory.newConfigurationBuilder();
-        builder.setStatusLevel(Level.ERROR);
-        Configurator.initialize(builder.build());
+        configureStatusLogger();
+        configureLoggerLevels(settings);
+    }
+
+    /**
+     * Configure logging reading from any log4j2.properties found in the config directory and its
+     * subdirectories from the specified environment. Will also configure logging to point the logs
+     * directory from the specified environment.
+     *
+     * @param environment the environment for reading configs and the logs path
+     * @throws IOException   if there is an issue readings any log4j2.properties in the config
+     *                       directory
+     * @throws UserException if there are no log4j2.properties in the specified configs path
+     */
+    public static void configure(final Environment environment) throws IOException, UserException {
+        Objects.requireNonNull(environment);
+        try {
+            // we are about to configure logging, check that the status logger did not log any error-level messages
+            checkErrorListener();
+        } finally {
+            // whether or not the error listener check failed we can remove the listener now
+            StatusLogger.getLogger().removeListener(ERROR_LISTENER);
+        }
+        configure(environment.settings(), environment.configFile(), environment.logsFile());
+    }
+
+    private static void checkErrorListener() {
+        assert errorListenerIsRegistered() : "expected error listener to be registered";
+        if (error.get()) {
+            throw new IllegalStateException("status logger logged an error before logging was configured");
+        }
+    }
+
+    private static boolean errorListenerIsRegistered() {
+        return StreamSupport.stream(StatusLogger.getLogger().getListeners().spliterator(), false).anyMatch(l -> l == ERROR_LISTENER);
+    }
+
+    private static void configure(final Settings settings, final Path configsPath, final Path logsPath) throws IOException, UserException {
+        Objects.requireNonNull(settings);
+        Objects.requireNonNull(configsPath);
+        Objects.requireNonNull(logsPath);
+
+        setLogConfigurationSystemProperty(logsPath, settings);
+        // we initialize the status logger immediately otherwise Log4j will complain when we try to get the context
+        configureStatusLogger();
 
         final LoggerContext context = (LoggerContext) LogManager.getContext(false);
 
-        if (resolveConfig) {
-            final List<AbstractConfiguration> configurations = new ArrayList<>();
-            final PropertiesConfigurationFactory factory = new PropertiesConfigurationFactory();
-            final Set<FileVisitOption> options = EnumSet.of(FileVisitOption.FOLLOW_LINKS);
-            Files.walkFileTree(environment.configFile(), options, Integer.MAX_VALUE, new SimpleFileVisitor<Path>() {
-                @Override
-                public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
-                    if (file.getFileName().toString().equals("log4j2.properties")) {
-                        configurations.add((PropertiesConfiguration) factory.getConfiguration(file.toString(), file.toUri()));
-                    }
-                    return FileVisitResult.CONTINUE;
+        final List<AbstractConfiguration> configurations = new ArrayList<>();
+        final PropertiesConfigurationFactory factory = new PropertiesConfigurationFactory();
+        final Set<FileVisitOption> options = EnumSet.of(FileVisitOption.FOLLOW_LINKS);
+        Files.walkFileTree(configsPath, options, Integer.MAX_VALUE, new SimpleFileVisitor<Path>() {
+            @Override
+            public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException {
+                if (file.getFileName().toString().equals("log4j2.properties")) {
+                    configurations.add((PropertiesConfiguration) factory.getConfiguration(context, file.toString(), file.toUri()));
                 }
-            });
-            context.start(new CompositeConfiguration(configurations));
-            warnIfOldConfigurationFilePresent(environment);
+                return FileVisitResult.CONTINUE;
+            }
+        });
+
+        if (configurations.isEmpty()) {
+            throw new UserException(
+                    ExitCodes.CONFIG,
+                    "no log4j2.properties found; tried [" + configsPath + "] and its subdirectories");
         }
 
+        context.start(new CompositeConfiguration(configurations));
+        warnIfOldConfigurationFilePresent(configsPath);
+
+        configureLoggerLevels(settings);
+    }
+
+    private static void configureStatusLogger() {
+        final ConfigurationBuilder<BuiltConfiguration> builder = ConfigurationBuilderFactory.newConfigurationBuilder();
+        builder.setStatusLevel(Level.ERROR);
+        Configurator.initialize(builder.build());
+    }
+
+    /**
+     * Configures the logging levels for loggers configured in the specified settings.
+     *
+     * @param settings the settings from which logger levels will be extracted
+     */
+    private static void configureLoggerLevels(final Settings settings) {
         if (ESLoggerFactory.LOG_DEFAULT_LEVEL_SETTING.exists(settings)) {
-            Loggers.setLevel(ESLoggerFactory.getRootLogger(), ESLoggerFactory.LOG_DEFAULT_LEVEL_SETTING.get(settings));
+            final Level level = ESLoggerFactory.LOG_DEFAULT_LEVEL_SETTING.get(settings);
+            Loggers.setLevel(ESLoggerFactory.getRootLogger(), level);
         }
 
         final Map<String, String> levels = settings.filter(ESLoggerFactory.LOG_LEVEL_SETTING::match).getAsMap();
-        for (String key : levels.keySet()) {
-            final Level level = ESLoggerFactory.LOG_LEVEL_SETTING.getConcreteSetting(key).get(settings);
-            Loggers.setLevel(Loggers.getLogger(key.substring("logger.".length())), level);
+        for (final String key : levels.keySet()) {
+            // do not set a log level for a logger named level (from the default log setting)
+            if (!key.equals(ESLoggerFactory.LOG_DEFAULT_LEVEL_SETTING.getKey())) {
+                final Level level = ESLoggerFactory.LOG_LEVEL_SETTING.getConcreteSetting(key).get(settings);
+                Loggers.setLevel(ESLoggerFactory.getLogger(key.substring("logger.".length())), level);
+            }
         }
     }
 
-    private static void warnIfOldConfigurationFilePresent(final Environment environment) throws IOException {
+    private static void warnIfOldConfigurationFilePresent(final Path configsPath) throws IOException {
         // TODO: the warning for unsupported logging configurations can be removed in 6.0.0
         assert Version.CURRENT.major < 6;
         final List<String> suffixes = Arrays.asList(".yml", ".yaml", ".json", ".properties");
         final Set<FileVisitOption> options = EnumSet.of(FileVisitOption.FOLLOW_LINKS);
-        Files.walkFileTree(environment.configFile(), options, Integer.MAX_VALUE, new SimpleFileVisitor<Path>() {
+        Files.walkFileTree(configsPath, options, Integer.MAX_VALUE, new SimpleFileVisitor<Path>() {
             @Override
             public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
                 final String fileName = file.getFileName().toString();
@@ -116,9 +220,34 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO
         });
     }
 
+    /**
+     * Set system properties that can be used in configuration files to specify paths and file patterns for log files. We expose three
+     * properties here:
+     * <ul>
+     * <li>
+     * {@code es.logs.base_path} the base path containing the log files
+     * </li>
+     * <li>
+     * {@code es.logs.cluster_name} the cluster name, used as the prefix of log filenames in the default configuration
+     * </li>
+     * <li>
+     * {@code es.logs.node_name} the node name, can be used as part of log filenames (only exposed if {@link Node#NODE_NAME_SETTING} is
+     * explicitly set)
+     * </li>
+     * </ul>
+     *
+     * @param logsPath the path to the log files
+     * @param settings the settings to extract the cluster and node names
+     */
     @SuppressForbidden(reason = "sets system property for logging configuration")
-    private static void setLogConfigurationSystemProperty(final Environment environment, final Settings settings) {
-        System.setProperty("es.logs", environment.logsFile().resolve(ClusterName.CLUSTER_NAME_SETTING.get(settings).value()).toString());
+    private static void setLogConfigurationSystemProperty(final Path logsPath, final Settings settings) {
+        // for backwards compatibility
+        System.setProperty("es.logs", logsPath.resolve(ClusterName.CLUSTER_NAME_SETTING.get(settings).value()).toString());
+        System.setProperty("es.logs.base_path", logsPath.toString());
+        System.setProperty("es.logs.cluster_name", ClusterName.CLUSTER_NAME_SETTING.get(settings).value());
+        if (Node.NODE_NAME_SETTING.exists(settings)) {
+            System.setProperty("es.logs.node_name", Node.NODE_NAME_SETTING.get(settings));
+        }
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/common/logging/Loggers.java b/core/src/main/java/org/elasticsearch/common/logging/Loggers.java
index 647ada099ba1e..812a0b70f2877 100644
--- a/core/src/main/java/org/elasticsearch/common/logging/Loggers.java
+++ b/core/src/main/java/org/elasticsearch/common/logging/Loggers.java
@@ -20,7 +20,9 @@
 package org.elasticsearch.common.logging;
 
 import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.core.Appender;
 import org.apache.logging.log4j.core.LoggerContext;
 import org.apache.logging.log4j.core.config.Configuration;
 import org.apache.logging.log4j.core.config.Configurator;
@@ -33,9 +35,12 @@
 import org.elasticsearch.node.Node;
 
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.List;
+import java.util.Map;
 
 import static java.util.Arrays.asList;
+import static javax.security.auth.login.Configuration.getConfiguration;
 import static org.elasticsearch.common.util.CollectionUtils.asArrayList;
 
 /**
@@ -43,24 +48,8 @@
  */
 public class Loggers {
 
-    static final String commonPrefix = System.getProperty("es.logger.prefix", "org.elasticsearch.");
-
     public static final String SPACE = " ";
 
-    private static boolean consoleLoggingEnabled = true;
-
-    public static void disableConsoleLogging() {
-        consoleLoggingEnabled = false;
-    }
-
-    public static void enableConsoleLogging() {
-        consoleLoggingEnabled = true;
-    }
-
-    public static boolean consoleLoggingEnabled() {
-        return consoleLoggingEnabled;
-    }
-
     public static Logger getLogger(Class<?> clazz, Settings settings, ShardId shardId, String... prefixes) {
         return getLogger(clazz, settings, shardId.getIndex(), asArrayList(Integer.toString(shardId.id()), prefixes).toArray(new String[0]));
     }
@@ -79,10 +68,16 @@ public static Logger getLogger(Class<?> clazz, Settings settings, Index index, S
     }
 
     public static Logger getLogger(Class<?> clazz, Settings settings, String... prefixes) {
-        return getLogger(buildClassLoggerName(clazz), settings, prefixes);
+        final List<String> prefixesList = prefixesList(settings, prefixes);
+        return getLogger(clazz, prefixesList.toArray(new String[prefixesList.size()]));
     }
 
     public static Logger getLogger(String loggerName, Settings settings, String... prefixes) {
+        final List<String> prefixesList = prefixesList(settings, prefixes);
+        return getLogger(loggerName, prefixesList.toArray(new String[prefixesList.size()]));
+    }
+
+    private static List<String> prefixesList(Settings settings, String... prefixes) {
         List<String> prefixesList = new ArrayList<>();
         if (Node.NODE_NAME_SETTING.exists(settings)) {
             prefixesList.add(Node.NODE_NAME_SETTING.get(settings));
@@ -90,26 +85,31 @@ public static Logger getLogger(String loggerName, Settings settings, String... p
         if (prefixes != null && prefixes.length > 0) {
             prefixesList.addAll(asList(prefixes));
         }
-        return getLogger(getLoggerName(loggerName), prefixesList.toArray(new String[prefixesList.size()]));
+        return prefixesList;
     }
 
     public static Logger getLogger(Logger parentLogger, String s) {
-        return ESLoggerFactory.getLogger(parentLogger.<MessageFactory>getMessageFactory(), getLoggerName(parentLogger.getName() + s));
+        assert parentLogger instanceof PrefixLogger;
+        return ESLoggerFactory.getLogger(((PrefixLogger)parentLogger).prefix(), parentLogger.getName() + s);
     }
 
     public static Logger getLogger(String s) {
-        return ESLoggerFactory.getLogger(getLoggerName(s));
+        return ESLoggerFactory.getLogger(s);
     }
 
     public static Logger getLogger(Class<?> clazz) {
-        return ESLoggerFactory.getLogger(getLoggerName(buildClassLoggerName(clazz)));
+        return ESLoggerFactory.getLogger(clazz);
     }
 
     public static Logger getLogger(Class<?> clazz, String... prefixes) {
-        return getLogger(buildClassLoggerName(clazz), prefixes);
+        return ESLoggerFactory.getLogger(formatPrefix(prefixes), clazz);
     }
 
     public static Logger getLogger(String name, String... prefixes) {
+        return ESLoggerFactory.getLogger(formatPrefix(prefixes), name);
+    }
+
+    private static String formatPrefix(String... prefixes) {
         String prefix = null;
         if (prefixes != null && prefixes.length > 0) {
             StringBuilder sb = new StringBuilder();
@@ -127,7 +127,7 @@ public static Logger getLogger(String name, String... prefixes) {
                 prefix = sb.toString();
             }
         }
-        return ESLoggerFactory.getLogger(prefix, getLoggerName(name));
+        return prefix;
     }
 
     /**
@@ -145,30 +145,60 @@ public static void setLevel(Logger logger, String level) {
     }
 
     public static void setLevel(Logger logger, Level level) {
-        if (!"".equals(logger.getName())) {
+        if (!LogManager.ROOT_LOGGER_NAME.equals(logger.getName())) {
             Configurator.setLevel(logger.getName(), level);
         } else {
-            LoggerContext ctx = LoggerContext.getContext(false);
-            Configuration config = ctx.getConfiguration();
-            LoggerConfig loggerConfig = config.getLoggerConfig(logger.getName());
+            final LoggerContext ctx = LoggerContext.getContext(false);
+            final Configuration config = ctx.getConfiguration();
+            final LoggerConfig loggerConfig = config.getLoggerConfig(logger.getName());
             loggerConfig.setLevel(level);
             ctx.updateLoggers();
         }
+
+        // we have to descend the hierarchy
+        final LoggerContext ctx = LoggerContext.getContext(false);
+        for (final LoggerConfig loggerConfig : ctx.getConfiguration().getLoggers().values()) {
+            if (LogManager.ROOT_LOGGER_NAME.equals(logger.getName()) || loggerConfig.getName().startsWith(logger.getName() + ".")) {
+                Configurator.setLevel(loggerConfig.getName(), level);
+            }
+        }
     }
 
-    private static String buildClassLoggerName(Class<?> clazz) {
-        String name = clazz.getName();
-        if (name.startsWith("org.elasticsearch.")) {
-            name = Classes.getPackageName(clazz);
+    public static void addAppender(final Logger logger, final Appender appender) {
+        final LoggerContext ctx = (LoggerContext) LogManager.getContext(false);
+        final Configuration config = ctx.getConfiguration();
+        config.addAppender(appender);
+        LoggerConfig loggerConfig = config.getLoggerConfig(logger.getName());
+        if (!logger.getName().equals(loggerConfig.getName())) {
+            loggerConfig = new LoggerConfig(logger.getName(), logger.getLevel(), true);
+            config.addLogger(logger.getName(), loggerConfig);
         }
-        return name;
+        loggerConfig.addAppender(appender, null, null);
+        ctx.updateLoggers();
     }
 
-    private static String getLoggerName(String name) {
-        if (name.startsWith("org.elasticsearch.")) {
-            name = name.substring("org.elasticsearch.".length());
+    public static void removeAppender(final Logger logger, final Appender appender) {
+        final LoggerContext ctx = (LoggerContext) LogManager.getContext(false);
+        final Configuration config = ctx.getConfiguration();
+        LoggerConfig loggerConfig = config.getLoggerConfig(logger.getName());
+        if (!logger.getName().equals(loggerConfig.getName())) {
+            loggerConfig = new LoggerConfig(logger.getName(), logger.getLevel(), true);
+            config.addLogger(logger.getName(), loggerConfig);
+        }
+        loggerConfig.removeAppender(appender.getName());
+        ctx.updateLoggers();
+    }
+
+    public static Appender findAppender(final Logger logger, final Class<? extends Appender> clazz) {
+        final LoggerContext ctx = (LoggerContext) LogManager.getContext(false);
+        final Configuration config = ctx.getConfiguration();
+        final LoggerConfig loggerConfig = config.getLoggerConfig(logger.getName());
+        for (final Map.Entry<String, Appender> entry : loggerConfig.getAppenders().entrySet()) {
+            if (entry.getValue().getClass().equals(clazz)) {
+                return entry.getValue();
+            }
         }
-        return commonPrefix + name;
+        return null;
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/common/logging/PrefixLogger.java b/core/src/main/java/org/elasticsearch/common/logging/PrefixLogger.java
new file mode 100644
index 0000000000000..a78330c3e8564
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/logging/PrefixLogger.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.logging;
+
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.Marker;
+import org.apache.logging.log4j.MarkerManager;
+import org.apache.logging.log4j.message.Message;
+import org.apache.logging.log4j.spi.ExtendedLogger;
+import org.apache.logging.log4j.spi.ExtendedLoggerWrapper;
+
+import java.util.WeakHashMap;
+
+/**
+ * A logger that prefixes all messages with a fixed prefix specified during construction. The prefix mechanism uses the marker construct, so
+ * for the prefixes to appear, the logging layout pattern must include the marker in its pattern.
+ */
+class PrefixLogger extends ExtendedLoggerWrapper {
+
+    /*
+     * We can not use the built-in Marker tracking (MarkerManager) because the MarkerManager holds a permanent reference to the marker;
+     * however, we have transient markers from index-level and shard-level components so this would effectively be a memory leak. Since we
+     * can not tie into the lifecycle of these components, we have to use a mechanism that enables garbage collection of such markers when
+     * they are no longer in use.
+     */
+    private static final WeakHashMap<String, Marker> markers = new WeakHashMap<>();
+
+    /**
+     * Return the size of the cached markers. This size can vary as markers are cached but collected during GC activity when a given prefix
+     * is no longer in use.
+     *
+     * @return the size of the cached markers
+     */
+    static int markersSize() {
+        return markers.size();
+    }
+
+    /**
+     * The marker for this prefix logger.
+     */
+    private final Marker marker;
+
+    /**
+     * Obtain the prefix for this prefix logger. This can be used to create a logger with the same prefix as this one.
+     *
+     * @return the prefix
+     */
+    public String prefix() {
+        return marker.getName();
+    }
+
+    /**
+     * Construct a prefix logger with the specified name and prefix.
+     *
+     * @param logger the extended logger to wrap
+     * @param name   the name of this prefix logger
+     * @param prefix the prefix for this prefix logger
+     */
+    PrefixLogger(final ExtendedLogger logger, final String name, final String prefix) {
+        super(logger, name, null);
+
+        final String actualPrefix = (prefix == null ? "" : prefix).intern();
+        final Marker actualMarker;
+        // markers is not thread-safe, so we synchronize access
+        synchronized (markers) {
+            final Marker maybeMarker = markers.get(actualPrefix);
+            if (maybeMarker == null) {
+                actualMarker = new MarkerManager.Log4jMarker(actualPrefix);
+                /*
+                 * We must create a new instance here as otherwise the marker will hold a reference to the key in the weak hash map; as
+                 * those references are held strongly, this would give a strong reference back to the key preventing them from ever being
+                 * collected. This also guarantees that no other strong reference can be held to the prefix anywhere.
+                 */
+                markers.put(new String(actualPrefix), actualMarker);
+            } else {
+                actualMarker = maybeMarker;
+            }
+        }
+        this.marker = actualMarker;
+    }
+
+    @Override
+    public void logMessage(final String fqcn, final Level level, final Marker marker, final Message message, final Throwable t) {
+        assert marker == null;
+        super.logMessage(fqcn, level, this.marker, message, t);
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/common/logging/PrefixMessageFactory.java b/core/src/main/java/org/elasticsearch/common/logging/PrefixMessageFactory.java
deleted file mode 100644
index a141ceb75aae6..0000000000000
--- a/core/src/main/java/org/elasticsearch/common/logging/PrefixMessageFactory.java
+++ /dev/null
@@ -1,221 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.common.logging;
-
-import org.apache.logging.log4j.message.Message;
-import org.apache.logging.log4j.message.MessageFactory2;
-import org.apache.logging.log4j.message.ObjectMessage;
-import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.apache.logging.log4j.message.SimpleMessage;
-
-public class PrefixMessageFactory implements MessageFactory2 {
-
-    private String prefix = "";
-
-    public String getPrefix() {
-        return prefix;
-    }
-
-    public void setPrefix(String prefix) {
-        this.prefix = prefix;
-    }
-
-    @Override
-    public Message newMessage(Object message) {
-        return new PrefixObjectMessage(prefix, message);
-    }
-
-    private static class PrefixObjectMessage extends ObjectMessage {
-
-        private final String prefix;
-        private final Object object;
-        private String prefixObjectString;
-
-        private PrefixObjectMessage(String prefix, Object object) {
-            super(object);
-            this.prefix = prefix;
-            this.object = object;
-        }
-
-        @Override
-        public String getFormattedMessage() {
-            if (prefixObjectString == null) {
-                prefixObjectString = prefix + super.getFormattedMessage();
-            }
-            return prefixObjectString;
-        }
-
-        @Override
-        public void formatTo(StringBuilder buffer) {
-            buffer.append(prefix);
-            super.formatTo(buffer);
-        }
-
-        @Override
-        public Object[] getParameters() {
-            return new Object[]{prefix, object};
-        }
-
-    }
-
-    @Override
-    public Message newMessage(String message) {
-        return new PrefixSimpleMessage(prefix, message);
-    }
-
-    private static class PrefixSimpleMessage extends SimpleMessage {
-
-        private final String prefix;
-        private String prefixMessage;
-
-        PrefixSimpleMessage(String prefix, String message) {
-            super(message);
-            this.prefix = prefix;
-        }
-
-        PrefixSimpleMessage(String prefix, CharSequence charSequence) {
-            super(charSequence);
-            this.prefix = prefix;
-        }
-
-        @Override
-        public String getFormattedMessage() {
-            if (prefixMessage == null) {
-                prefixMessage = prefix + super.getFormattedMessage();
-            }
-            return prefixMessage;
-        }
-
-        @Override
-        public void formatTo(StringBuilder buffer) {
-            buffer.append(prefix);
-            super.formatTo(buffer);
-        }
-
-        @Override
-        public int length() {
-            return prefixMessage.length();
-        }
-
-        @Override
-        public char charAt(int index) {
-            return prefixMessage.charAt(index);
-        }
-
-        @Override
-        public CharSequence subSequence(int start, int end) {
-            return prefixMessage.subSequence(start, end);
-        }
-
-    }
-
-    @Override
-    public Message newMessage(String message, Object... params) {
-        return new PrefixParameterizedMessage(prefix, message, params);
-    }
-
-    private static class PrefixParameterizedMessage extends ParameterizedMessage {
-
-        private static ThreadLocal<StringBuilder> threadLocalStringBuilder = ThreadLocal.withInitial(StringBuilder::new);
-
-        private final String prefix;
-        private String formattedMessage;
-
-        private PrefixParameterizedMessage(String prefix, String messagePattern, Object... arguments) {
-            super(messagePattern, arguments);
-            this.prefix = prefix;
-        }
-
-        @Override
-        public String getFormattedMessage() {
-            if (formattedMessage == null) {
-                final StringBuilder buffer = threadLocalStringBuilder.get();
-                buffer.setLength(0);
-                formatTo(buffer);
-                formattedMessage = buffer.toString();
-            }
-            return formattedMessage;
-        }
-
-        @Override
-        public void formatTo(StringBuilder buffer) {
-            buffer.append(prefix);
-            super.formatTo(buffer);
-        }
-
-    }
-
-    @Override
-    public Message newMessage(CharSequence charSequence) {
-        return new PrefixSimpleMessage(prefix, charSequence);
-    }
-
-    @Override
-    public Message newMessage(String message, Object p0) {
-        return new PrefixParameterizedMessage(prefix, message, p0);
-    }
-
-    @Override
-    public Message newMessage(String message, Object p0, Object p1) {
-        return new PrefixParameterizedMessage(prefix, message, p0, p1);
-    }
-
-    @Override
-    public Message newMessage(String message, Object p0, Object p1, Object p2) {
-        return new PrefixParameterizedMessage(prefix, message, p0, p1, p2);
-    }
-
-    @Override
-    public Message newMessage(String message, Object p0, Object p1, Object p2, Object p3) {
-        return new PrefixParameterizedMessage(prefix, message, p0, p1, p2, p3);
-    }
-
-    @Override
-    public Message newMessage(String message, Object p0, Object p1, Object p2, Object p3, Object p4) {
-        return new PrefixParameterizedMessage(prefix, message, p0, p1, p2, p3, p4);
-    }
-
-    @Override
-    public Message newMessage(String message, Object p0, Object p1, Object p2, Object p3, Object p4, Object p5) {
-        return new PrefixParameterizedMessage(prefix, message, p0, p1, p2, p3, p4, p5);
-    }
-
-    @Override
-    public Message newMessage(String message, Object p0, Object p1, Object p2, Object p3, Object p4, Object p5, Object p6) {
-        return new PrefixParameterizedMessage(prefix, message, p0, p1, p2, p3, p4, p5, p6);
-    }
-
-    @Override
-    public Message newMessage(String message, Object p0, Object p1, Object p2, Object p3, Object p4, Object p5, Object p6, Object p7) {
-        return new PrefixParameterizedMessage(prefix, message, p0, p1, p2, p3, p4, p5, p6, p7);
-    }
-
-    @Override
-    public Message newMessage(
-        String message, Object p0, Object p1, Object p2, Object p3, Object p4, Object p5, Object p6, Object p7, Object p8) {
-        return new PrefixParameterizedMessage(prefix, message, p0, p1, p2, p3, p4, p5, p6, p7, p8);
-    }
-
-    @Override
-    public Message newMessage(
-        String message, Object p0, Object p1, Object p2, Object p3, Object p4, Object p5, Object p6, Object p7, Object p8, Object p9) {
-        return new PrefixParameterizedMessage(prefix, message, p0, p1, p2, p3, p4, p5, p6, p7, p8, p9);
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/LoggerInfoStream.java b/core/src/main/java/org/elasticsearch/common/lucene/LoggerInfoStream.java
index c4ef2ef8c703f..680760444b2a6 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/LoggerInfoStream.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/LoggerInfoStream.java
@@ -23,21 +23,19 @@
 import org.apache.lucene.util.InfoStream;
 import org.elasticsearch.common.logging.Loggers;
 
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
 /** An InfoStream (for Lucene's IndexWriter) that redirects
  *  messages to "lucene.iw.ifd" and "lucene.iw" Logger.trace. */
-
 public final class LoggerInfoStream extends InfoStream {
-    /** Used for component-specific logging: */
 
-    /** Logger for everything */
-    private final Logger logger;
+    private final Logger parentLogger;
 
-    /** Logger for IndexFileDeleter */
-    private final Logger ifdLogger;
+    private final Map<String, Logger> loggers = new ConcurrentHashMap<>();
 
-    public LoggerInfoStream(Logger parentLogger) {
-        logger = Loggers.getLogger(parentLogger, ".lucene.iw");
-        ifdLogger = Loggers.getLogger(parentLogger, ".lucene.iw.ifd");
+    public LoggerInfoStream(final Logger parentLogger) {
+        this.parentLogger = parentLogger;
     }
 
     @Override
@@ -53,14 +51,12 @@ public boolean isEnabled(String component) {
     }
 
     private Logger getLogger(String component) {
-        if (component.equals("IFD")) {
-            return ifdLogger;
-        } else {
-            return logger;
-        }
+        return loggers.computeIfAbsent(component, c -> Loggers.getLogger(parentLogger, "." + c));
     }
 
     @Override
     public void close() {
+
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java b/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java
index 94e1f05e46b86..ed4fedf64ae81 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java
@@ -27,6 +27,7 @@
 import org.apache.lucene.codecs.CodecUtil;
 import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.PostingsFormat;
+import org.apache.lucene.document.LatLonDocValuesField;
 import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexCommit;
@@ -48,6 +49,7 @@
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.ScorerSupplier;
 import org.apache.lucene.search.SimpleCollector;
 import org.apache.lucene.search.SortField;
 import org.apache.lucene.search.TimeLimitingCollector;
@@ -55,6 +57,9 @@
 import org.apache.lucene.search.TopFieldDocs;
 import org.apache.lucene.search.TwoPhaseIterator;
 import org.apache.lucene.search.Weight;
+import org.apache.lucene.search.grouping.CollapseTopFieldDocs;
+import org.apache.lucene.search.SortedNumericSortField;
+import org.apache.lucene.search.SortedSetSortField;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
@@ -282,23 +287,23 @@ public static boolean exists(IndexSearcher searcher, Query query) throws IOExcep
     }
 
     public static TopDocs readTopDocs(StreamInput in) throws IOException {
-        if (in.readBoolean()) {
+        byte type = in.readByte();
+        if (type == 0) {
+            int totalHits = in.readVInt();
+            float maxScore = in.readFloat();
+
+            ScoreDoc[] scoreDocs = new ScoreDoc[in.readVInt()];
+            for (int i = 0; i < scoreDocs.length; i++) {
+                scoreDocs[i] = new ScoreDoc(in.readVInt(), in.readFloat());
+            }
+            return new TopDocs(totalHits, scoreDocs, maxScore);
+        } else if (type == 1) {
             int totalHits = in.readVInt();
             float maxScore = in.readFloat();
 
             SortField[] fields = new SortField[in.readVInt()];
             for (int i = 0; i < fields.length; i++) {
-                String field = null;
-                if (in.readBoolean()) {
-                    field = in.readString();
-                }
-                SortField.Type sortType = readSortType(in);
-                Object missingValue = readMissingValue(in);
-                boolean reverse = in.readBoolean();
-                fields[i] = new SortField(field, sortType, reverse);
-                if (missingValue != null) {
-                    fields[i].setMissingValue(missingValue);
-                }
+                fields[i] = readSortField(in);
             }
 
             FieldDoc[] fieldDocs = new FieldDoc[in.readVInt()];
@@ -306,15 +311,25 @@ public static TopDocs readTopDocs(StreamInput in) throws IOException {
                 fieldDocs[i] = readFieldDoc(in);
             }
             return new TopFieldDocs(totalHits, fieldDocs, fields, maxScore);
-        } else {
+        } else if (type == 2) {
             int totalHits = in.readVInt();
             float maxScore = in.readFloat();
 
-            ScoreDoc[] scoreDocs = new ScoreDoc[in.readVInt()];
-            for (int i = 0; i < scoreDocs.length; i++) {
-                scoreDocs[i] = new ScoreDoc(in.readVInt(), in.readFloat());
+            String field = in.readString();
+            SortField[] fields = new SortField[in.readVInt()];
+            for (int i = 0; i < fields.length; i++) {
+               fields[i] = readSortField(in);
             }
-            return new TopDocs(totalHits, scoreDocs, maxScore);
+            int size = in.readVInt();
+            Object[] collapseValues = new Object[size];
+            FieldDoc[] fieldDocs = new FieldDoc[size];
+            for (int i = 0; i < fieldDocs.length; i++) {
+                fieldDocs[i] = readFieldDoc(in);
+                collapseValues[i] = readSortValue(in);
+            }
+            return new CollapseTopFieldDocs(field, totalHits, fieldDocs, fields, collapseValues, maxScore);
+        } else {
+            throw new IllegalStateException("Unknown type " + type);
         }
     }
 
@@ -349,13 +364,62 @@ public static FieldDoc readFieldDoc(StreamInput in) throws IOException {
         return new FieldDoc(in.readVInt(), in.readFloat(), cFields);
     }
 
+    private static Comparable readSortValue(StreamInput in) throws IOException {
+        byte type = in.readByte();
+        if (type == 0) {
+            return null;
+        } else if (type == 1) {
+            return in.readString();
+        } else if (type == 2) {
+            return in.readInt();
+        } else if (type == 3) {
+            return in.readLong();
+        } else if (type == 4) {
+            return in.readFloat();
+        } else if (type == 5) {
+            return in.readDouble();
+        } else if (type == 6) {
+            return in.readByte();
+        } else if (type == 7) {
+            return in.readShort();
+        } else if (type == 8) {
+            return in.readBoolean();
+        } else if (type == 9) {
+            return in.readBytesRef();
+        } else {
+            throw new IOException("Can't match type [" + type + "]");
+        }
+    }
+
     public static ScoreDoc readScoreDoc(StreamInput in) throws IOException {
         return new ScoreDoc(in.readVInt(), in.readFloat());
     }
 
+    private static final Class<?> GEO_DISTANCE_SORT_TYPE_CLASS = LatLonDocValuesField.newDistanceSort("some_geo_field", 0, 0).getClass();
+
     public static void writeTopDocs(StreamOutput out, TopDocs topDocs) throws IOException {
-        if (topDocs instanceof TopFieldDocs) {
-            out.writeBoolean(true);
+        if (topDocs instanceof CollapseTopFieldDocs) {
+            out.writeByte((byte) 2);
+            CollapseTopFieldDocs collapseDocs = (CollapseTopFieldDocs) topDocs;
+
+            out.writeVInt(topDocs.totalHits);
+            out.writeFloat(topDocs.getMaxScore());
+
+            out.writeString(collapseDocs.field);
+
+            out.writeVInt(collapseDocs.fields.length);
+            for (SortField sortField : collapseDocs.fields) {
+               writeSortField(out, sortField);
+            }
+
+            out.writeVInt(topDocs.scoreDocs.length);
+            for (int i = 0; i < topDocs.scoreDocs.length; i++) {
+                ScoreDoc doc = collapseDocs.scoreDocs[i];
+                writeFieldDoc(out, (FieldDoc) doc);
+                writeSortValue(out, collapseDocs.collapseValues[i]);
+            }
+        } else if (topDocs instanceof TopFieldDocs) {
+            out.writeByte((byte) 1);
             TopFieldDocs topFieldDocs = (TopFieldDocs) topDocs;
 
             out.writeVInt(topDocs.totalHits);
@@ -363,21 +427,7 @@ public static void writeTopDocs(StreamOutput out, TopDocs topDocs) throws IOExce
 
             out.writeVInt(topFieldDocs.fields.length);
             for (SortField sortField : topFieldDocs.fields) {
-                if (sortField.getField() == null) {
-                    out.writeBoolean(false);
-                } else {
-                    out.writeBoolean(true);
-                    out.writeString(sortField.getField());
-                }
-                if (sortField.getComparatorSource() != null) {
-                    IndexFieldData.XFieldComparatorSource comparatorSource = (IndexFieldData.XFieldComparatorSource) sortField.getComparatorSource();
-                    writeSortType(out, comparatorSource.reducedType());
-                    writeMissingValue(out, comparatorSource.missingValue(sortField.getReverse()));
-                } else {
-                    writeSortType(out, sortField.getType());
-                    writeMissingValue(out, sortField.getMissingValue());
-                }
-                out.writeBoolean(sortField.getReverse());
+              writeSortField(out, sortField);
             }
 
             out.writeVInt(topDocs.scoreDocs.length);
@@ -385,7 +435,7 @@ public static void writeTopDocs(StreamOutput out, TopDocs topDocs) throws IOExce
                 writeFieldDoc(out, (FieldDoc) doc);
             }
         } else {
-            out.writeBoolean(false);
+            out.writeByte((byte) 0);
             out.writeVInt(topDocs.totalHits);
             out.writeFloat(topDocs.getMaxScore());
 
@@ -421,44 +471,49 @@ private static Object readMissingValue(StreamInput in) throws IOException {
         }
     }
 
+
+    private static void writeSortValue(StreamOutput out, Object field) throws IOException {
+        if (field == null) {
+            out.writeByte((byte) 0);
+        } else {
+            Class type = field.getClass();
+            if (type == String.class) {
+                out.writeByte((byte) 1);
+                out.writeString((String) field);
+            } else if (type == Integer.class) {
+                out.writeByte((byte) 2);
+                out.writeInt((Integer) field);
+            } else if (type == Long.class) {
+                out.writeByte((byte) 3);
+                out.writeLong((Long) field);
+            } else if (type == Float.class) {
+                out.writeByte((byte) 4);
+                out.writeFloat((Float) field);
+            } else if (type == Double.class) {
+                out.writeByte((byte) 5);
+                out.writeDouble((Double) field);
+            } else if (type == Byte.class) {
+                out.writeByte((byte) 6);
+                out.writeByte((Byte) field);
+            } else if (type == Short.class) {
+                out.writeByte((byte) 7);
+                out.writeShort((Short) field);
+            } else if (type == Boolean.class) {
+                out.writeByte((byte) 8);
+                out.writeBoolean((Boolean) field);
+            } else if (type == BytesRef.class) {
+                out.writeByte((byte) 9);
+                out.writeBytesRef((BytesRef) field);
+            } else {
+                throw new IOException("Can't handle sort field value of type [" + type + "]");
+            }
+        }
+    }
+
     public static void writeFieldDoc(StreamOutput out, FieldDoc fieldDoc) throws IOException {
         out.writeVInt(fieldDoc.fields.length);
         for (Object field : fieldDoc.fields) {
-            if (field == null) {
-                out.writeByte((byte) 0);
-            } else {
-                Class type = field.getClass();
-                if (type == String.class) {
-                    out.writeByte((byte) 1);
-                    out.writeString((String) field);
-                } else if (type == Integer.class) {
-                    out.writeByte((byte) 2);
-                    out.writeInt((Integer) field);
-                } else if (type == Long.class) {
-                    out.writeByte((byte) 3);
-                    out.writeLong((Long) field);
-                } else if (type == Float.class) {
-                    out.writeByte((byte) 4);
-                    out.writeFloat((Float) field);
-                } else if (type == Double.class) {
-                    out.writeByte((byte) 5);
-                    out.writeDouble((Double) field);
-                } else if (type == Byte.class) {
-                    out.writeByte((byte) 6);
-                    out.writeByte((Byte) field);
-                } else if (type == Short.class) {
-                    out.writeByte((byte) 7);
-                    out.writeShort((Short) field);
-                } else if (type == Boolean.class) {
-                    out.writeByte((byte) 8);
-                    out.writeBoolean((Boolean) field);
-                } else if (type == BytesRef.class) {
-                    out.writeByte((byte) 9);
-                    out.writeBytesRef((BytesRef) field);
-                } else {
-                    throw new IOException("Can't handle sort field value of type [" + type + "]");
-                }
-            }
+            writeSortValue(out, field);
         }
         out.writeVInt(fieldDoc.doc);
         out.writeFloat(fieldDoc.score);
@@ -477,10 +532,68 @@ public static SortField.Type readSortType(StreamInput in) throws IOException {
         return SortField.Type.values()[in.readVInt()];
     }
 
+    public static SortField readSortField(StreamInput in) throws IOException {
+        String field = null;
+        if (in.readBoolean()) {
+            field = in.readString();
+        }
+        SortField.Type sortType = readSortType(in);
+        Object missingValue = readMissingValue(in);
+        boolean reverse = in.readBoolean();
+        SortField sortField = new SortField(field, sortType, reverse);
+        if (missingValue != null) {
+            sortField.setMissingValue(missingValue);
+        }
+        return sortField;
+    }
+
     public static void writeSortType(StreamOutput out, SortField.Type sortType) throws IOException {
         out.writeVInt(sortType.ordinal());
     }
 
+    public static void writeSortField(StreamOutput out, SortField sortField) throws IOException {
+        if (sortField.getClass() == GEO_DISTANCE_SORT_TYPE_CLASS) {
+            // for geo sorting, we replace the SortField with a SortField that assumes a double field.
+            // this works since the SortField is only used for merging top docs
+            SortField newSortField = new SortField(sortField.getField(), SortField.Type.DOUBLE);
+            newSortField.setMissingValue(sortField.getMissingValue());
+            sortField = newSortField;
+        } else if (sortField.getClass() == SortedSetSortField.class) {
+            // for multi-valued sort field, we replace the SortedSetSortField with a simple SortField.
+            // It works because the sort field is only used to merge results from different shards.
+            SortField newSortField = new SortField(sortField.getField(), SortField.Type.STRING, sortField.getReverse());
+            newSortField.setMissingValue(sortField.getMissingValue());
+            sortField = newSortField;
+        } else if (sortField.getClass() == SortedNumericSortField.class) {
+            // for multi-valued sort field, we replace the SortedSetSortField with a simple SortField.
+            // It works because the sort field is only used to merge results from different shards.
+            SortField newSortField = new SortField(sortField.getField(),
+                ((SortedNumericSortField) sortField).getNumericType(),
+                sortField.getReverse());
+            newSortField.setMissingValue(sortField.getMissingValue());
+            sortField = newSortField;
+        }
+
+        if (sortField.getClass() != SortField.class) {
+            throw new IllegalArgumentException("Cannot serialize SortField impl [" + sortField + "]");
+        }
+        if (sortField.getField() == null) {
+            out.writeBoolean(false);
+        } else {
+            out.writeBoolean(true);
+            out.writeString(sortField.getField());
+        }
+        if (sortField.getComparatorSource() != null) {
+            IndexFieldData.XFieldComparatorSource comparatorSource = (IndexFieldData.XFieldComparatorSource) sortField.getComparatorSource();
+            writeSortType(out, comparatorSource.reducedType());
+            writeMissingValue(out, comparatorSource.missingValue(sortField.getReverse()));
+        } else {
+            writeSortType(out, sortField.getType());
+            writeMissingValue(out, sortField.getMissingValue());
+        }
+        out.writeBoolean(sortField.getReverse());
+    }
+
     public static Explanation readExplanation(StreamInput in) throws IOException {
         boolean match = in.readBoolean();
         String description = in.readString();
@@ -729,14 +842,16 @@ public void delete() {
     }
 
     /**
-     * Given a {@link Scorer}, return a {@link Bits} instance that will match
+     * Given a {@link ScorerSupplier}, return a {@link Bits} instance that will match
      * all documents contained in the set. Note that the returned {@link Bits}
      * instance MUST be consumed in order.
      */
-    public static Bits asSequentialAccessBits(final int maxDoc, @Nullable Scorer scorer) throws IOException {
-        if (scorer == null) {
+    public static Bits asSequentialAccessBits(final int maxDoc, @Nullable ScorerSupplier scorerSupplier) throws IOException {
+        if (scorerSupplier == null) {
             return new Bits.MatchNoBits(maxDoc);
         }
+        // Since we want bits, we need random-access
+        final Scorer scorer = scorerSupplier.get(true); // this never returns null
         final TwoPhaseIterator twoPhase = scorer.twoPhaseIterator();
         final DocIdSetIterator iterator;
         if (twoPhase == null) {
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/ShardCoreKeyMap.java b/core/src/main/java/org/elasticsearch/common/lucene/ShardCoreKeyMap.java
index 146fb7ba05ec4..b237a746bf043 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/ShardCoreKeyMap.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/ShardCoreKeyMap.java
@@ -21,6 +21,7 @@
 
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.LeafReader.CoreClosedListener;
+import org.elasticsearch.Assertions;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.ShardUtils;
 
@@ -29,9 +30,9 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.IdentityHashMap;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
 
 /**
  * A map between segment core cache keys and the shard that these segments
@@ -50,7 +51,7 @@ public final class ShardCoreKeyMap {
     private final Map<String, Set<Object>> indexToCoreKey;
 
     public ShardCoreKeyMap() {
-        coreKeyToShard = new IdentityHashMap<>();
+        coreKeyToShard = new ConcurrentHashMap<>();
         indexToCoreKey = new HashMap<>();
     }
 
@@ -64,9 +65,17 @@ public void add(LeafReader reader) {
             throw new IllegalArgumentException("Could not extract shard id from " + reader);
         }
         final Object coreKey = reader.getCoreCacheKey();
+
+        if (coreKeyToShard.containsKey(coreKey)) {
+            // Do this check before entering the synchronized block in order to
+            // avoid taking the mutex if possible (which should happen most of
+            // the time).
+            return;
+        }
+
         final String index = shardId.getIndexName();
         synchronized (this) {
-            if (coreKeyToShard.put(coreKey, shardId) == null) {
+            if (coreKeyToShard.containsKey(coreKey) == false) {
                 Set<Object> objects = indexToCoreKey.get(index);
                 if (objects == null) {
                     objects = new HashSet<>();
@@ -90,6 +99,14 @@ public void add(LeafReader reader) {
                 try {
                     reader.addCoreClosedListener(listener);
                     addedListener = true;
+
+                    // Only add the core key to the map as a last operation so that
+                    // if another thread sees that the core key is already in the
+                    // map (like the check just before this synchronized block),
+                    // then it means that the closed listener has already been
+                    // registered.
+                    ShardId previous = coreKeyToShard.put(coreKey, shardId);
+                    assert previous == null;
                 } finally {
                     if (false == addedListener) {
                         try {
@@ -132,10 +149,7 @@ public synchronized int size() {
     }
 
     private synchronized boolean assertSize() {
-        // this is heavy and should only used in assertions
-        boolean assertionsEnabled = false;
-        assert assertionsEnabled = true;
-        if (assertionsEnabled == false) {
+        if (!Assertions.ENABLED) {
             throw new AssertionError("only run this if assertions are enabled");
         }
         Collection<Set<Object>> values = indexToCoreKey.values();
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/all/AllTermQuery.java b/core/src/main/java/org/elasticsearch/common/lucene/all/AllTermQuery.java
index 500fa206c9602..5307a417e107d 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/all/AllTermQuery.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/all/AllTermQuery.java
@@ -32,7 +32,6 @@
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.Explanation;
 import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.TermQuery;
@@ -64,6 +63,10 @@ public AllTermQuery(Term term) {
         this.term = term;
     }
 
+    public Term getTerm() {
+        return term;
+    }
+
     @Override
     public boolean equals(Object obj) {
         if (sameClassAs(obj) == false) {
@@ -83,21 +86,18 @@ public Query rewrite(IndexReader reader) throws IOException {
         if (rewritten != this) {
             return rewritten;
         }
-        boolean fieldExists = false;
         boolean hasPayloads = false;
         for (LeafReaderContext context : reader.leaves()) {
             final Terms terms = context.reader().terms(term.field());
             if (terms != null) {
-                fieldExists = true;
                 if (terms.hasPayloads()) {
                     hasPayloads = true;
                     break;
                 }
             }
         }
-        if (fieldExists == false) {
-            return new MatchNoDocsQuery();
-        }
+        // if the terms does not exist we could return a MatchNoDocsQuery but this would break the unified highlighter
+        // which rewrites query with an empty reader.
         if (hasPayloads == false) {
             return new TermQuery(term);
         }
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/FilteredCollector.java b/core/src/main/java/org/elasticsearch/common/lucene/search/FilteredCollector.java
index 5bb9223504488..4c95d19f14367 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/search/FilteredCollector.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/search/FilteredCollector.java
@@ -22,7 +22,7 @@
 import org.apache.lucene.search.Collector;
 import org.apache.lucene.search.FilterLeafCollector;
 import org.apache.lucene.search.LeafCollector;
-import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.ScorerSupplier;
 import org.apache.lucene.search.Weight;
 import org.apache.lucene.util.Bits;
 import org.elasticsearch.common.lucene.Lucene;
@@ -44,9 +44,9 @@ public FilteredCollector(Collector collector, Weight filter) {
 
     @Override
     public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException {
-        final Scorer filterScorer = filter.scorer(context);
+        final ScorerSupplier filterScorerSupplier = filter.scorerSupplier(context);
         final LeafCollector in = collector.getLeafCollector(context);
-        final Bits bits = Lucene.asSequentialAccessBits(context.reader().maxDoc(), filterScorer);
+        final Bits bits = Lucene.asSequentialAccessBits(context.reader().maxDoc(), filterScorerSupplier);
 
         return new FilterLeafCollector(in) {
             @Override
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/MatchNoDocsQuery.java b/core/src/main/java/org/elasticsearch/common/lucene/search/MatchNoDocsQuery.java
deleted file mode 100644
index 9caf350926c56..0000000000000
--- a/core/src/main/java/org/elasticsearch/common/lucene/search/MatchNoDocsQuery.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.common.lucene.search;
-
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.Weight;
-import org.apache.lucene.search.ConstantScoreWeight;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Explanation;
-import org.apache.lucene.search.Scorer;
-
-import java.io.IOException;
-import java.util.Set;
-
-/**
- * A query that matches no documents and prints the reason why in the toString method.
- */
-public class MatchNoDocsQuery extends Query {
-    /**
-     * The reason why the query does not match any document.
-     */
-    private final String reason;
-
-    public MatchNoDocsQuery(String reason) {
-        this.reason = reason;
-    }
-
-    public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException {
-        return new ConstantScoreWeight(this) {
-            @Override
-            public void extractTerms(Set<Term> terms) {
-            }
-
-            @Override
-            public Explanation explain(LeafReaderContext context, int doc) throws IOException {
-                return Explanation.noMatch(reason);
-            }
-
-            @Override
-            public Scorer scorer(LeafReaderContext context) throws IOException {
-                return null;
-            }
-        };
-    }
-
-    @Override
-    public String toString(String field) {
-        return "MatchNoDocsQuery[\"" + reason + "\"]";
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-        return sameClassAs(obj);
-    }
-
-    @Override
-    public int hashCode() {
-        return classHash();
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/MoreLikeThisQuery.java b/core/src/main/java/org/elasticsearch/common/lucene/search/MoreLikeThisQuery.java
index 06ab2b4a53012..73a290463574c 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/search/MoreLikeThisQuery.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/search/MoreLikeThisQuery.java
@@ -227,10 +227,6 @@ public String[] getLikeTexts() {
         return likeText;
     }
 
-    public void setLikeText(String likeText) {
-        setLikeText(new String[]{likeText});
-    }
-
     public void setLikeText(String... likeText) {
         this.likeText = likeText;
     }
@@ -239,7 +235,7 @@ public Fields[] getLikeFields() {
         return likeFields;
     }
 
-    public void setLikeText(Fields... likeFields) {
+    public void setLikeFields(Fields... likeFields) {
         this.likeFields = likeFields;
     }
 
@@ -247,7 +243,7 @@ public void setLikeText(List<String> likeText) {
         setLikeText(likeText.toArray(Strings.EMPTY_ARRAY));
     }
 
-    public void setUnlikeText(Fields... unlikeFields) {
+    public void setUnlikeFields(Fields... unlikeFields) {
         this.unlikeFields = unlikeFields;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQuery.java b/core/src/main/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQuery.java
index 87bfdacb1c760..b8e1039b2df1d 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQuery.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQuery.java
@@ -25,6 +25,8 @@
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.MultiPhraseQuery;
 import org.apache.lucene.search.Query;
@@ -115,6 +117,18 @@ public void add(Term[] terms, int position) {
         positions.add(position);
     }
 
+    /**
+     * Returns the terms for each position in this phrase
+     */
+    public Term[][] getTerms() {
+        Term[][] terms = new Term[termArrays.size()][];
+        for (int i = 0; i < termArrays.size(); i++) {
+            terms[i] = new Term[termArrays.get(i).length];
+            System.arraycopy(termArrays.get(i), 0, terms[i], 0, termArrays.get(i).length);
+        }
+        return terms;
+    }
+
     /**
      * Returns the relative positions of terms in this phrase.
      */
@@ -150,7 +164,17 @@ public Query rewrite(IndexReader reader) throws IOException {
             }
         }
         if (terms.isEmpty()) {
-            return Queries.newMatchNoDocsQuery("No terms supplied for " + MultiPhrasePrefixQuery.class.getName());
+            if (sizeMinus1 == 0) {
+                // no prefix and the phrase query is empty
+                return Queries.newMatchNoDocsQuery("No terms supplied for " + MultiPhrasePrefixQuery.class.getName());
+            }
+
+            // if the terms does not exist we could return a MatchNoDocsQuery but this would break the unified highlighter
+            // which rewrites query with an empty reader.
+            return new BooleanQuery.Builder()
+                .add(query.build(), BooleanClause.Occur.MUST)
+                .add(Queries.newMatchNoDocsQuery("No terms supplied for " + MultiPhrasePrefixQuery.class.getName()),
+                    BooleanClause.Occur.MUST).build();
         }
         query.add(terms.toArray(Term.class), position);
         return query.build();
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/Queries.java b/core/src/main/java/org/elasticsearch/common/lucene/search/Queries.java
index 2a3fd94e914d3..11d181076aeca 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/search/Queries.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/search/Queries.java
@@ -20,11 +20,13 @@
 package org.elasticsearch.common.lucene.search;
 
 import org.apache.lucene.index.Term;
+import org.apache.lucene.queries.ExtendedCommonTermsQuery;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.PrefixQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
@@ -53,7 +55,11 @@ public static Query newNestedFilter() {
     }
 
     public static Query newNonNestedFilter() {
-        return not(newNestedFilter());
+        // TODO: this is slow, make it a positive query
+        return new BooleanQuery.Builder()
+                .add(new MatchAllDocsQuery(), Occur.FILTER)
+                .add(newNestedFilter(), Occur.MUST_NOT)
+                .build();
     }
 
     public static BooleanQuery filtered(@Nullable Query query, @Nullable Query filter) {
@@ -137,6 +143,22 @@ public static Query applyMinimumShouldMatch(BooleanQuery query, @Nullable String
         }
     }
 
+    /**
+     * Potentially apply minimum should match value if we have a query that it can be applied to,
+     * otherwise return the original query.
+     */
+    public static Query maybeApplyMinimumShouldMatch(Query query, @Nullable String minimumShouldMatch) {
+        // If the coordination factor is disabled on a boolean query we don't apply the minimum should match.
+        // This is done to make sure that the minimum_should_match doesn't get applied when there is only one word
+        // and multiple variations of the same word in the query (synonyms for instance).
+        if (query instanceof BooleanQuery && !((BooleanQuery) query).isCoordDisabled()) {
+            return applyMinimumShouldMatch((BooleanQuery) query, minimumShouldMatch);
+        } else if (query instanceof ExtendedCommonTermsQuery) {
+            ((ExtendedCommonTermsQuery)query).setLowFreqMinimumNumberShouldMatch(minimumShouldMatch);
+        }
+        return query;
+    }
+
     private static Pattern spaceAroundLessThanPattern = Pattern.compile("(\\s+<\\s*)|(\\s*<\\s+)");
     private static Pattern spacePattern = Pattern.compile(" ");
     private static Pattern lessThanPattern = Pattern.compile("<");
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/function/CombineFunction.java b/core/src/main/java/org/elasticsearch/common/lucene/search/function/CombineFunction.java
index e8bd288799810..399f3d7a2e613 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/search/function/CombineFunction.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/search/function/CombineFunction.java
@@ -31,7 +31,7 @@ public enum CombineFunction implements Writeable {
     MULTIPLY {
         @Override
         public float combine(double queryScore, double funcScore, double maxBoost) {
-            return toFloat(queryScore * Math.min(funcScore, maxBoost));
+            return (float) (queryScore * Math.min(funcScore, maxBoost));
         }
 
         @Override
@@ -48,7 +48,7 @@ public Explanation explain(Explanation queryExpl, Explanation funcExpl, float ma
     REPLACE {
         @Override
         public float combine(double queryScore, double funcScore, double maxBoost) {
-            return toFloat(Math.min(funcScore, maxBoost));
+            return (float) (Math.min(funcScore, maxBoost));
         }
 
         @Override
@@ -64,7 +64,7 @@ public Explanation explain(Explanation queryExpl, Explanation funcExpl, float ma
     SUM {
         @Override
         public float combine(double queryScore, double funcScore, double maxBoost) {
-            return toFloat(queryScore + Math.min(funcScore, maxBoost));
+            return (float) (queryScore + Math.min(funcScore, maxBoost));
         }
 
         @Override
@@ -79,7 +79,7 @@ public Explanation explain(Explanation queryExpl, Explanation funcExpl, float ma
     AVG {
         @Override
         public float combine(double queryScore, double funcScore, double maxBoost) {
-            return toFloat((Math.min(funcScore, maxBoost) + queryScore) / 2.0);
+            return (float) ((Math.min(funcScore, maxBoost) + queryScore) / 2.0);
         }
 
         @Override
@@ -87,7 +87,7 @@ public Explanation explain(Explanation queryExpl, Explanation funcExpl, float ma
             Explanation minExpl = Explanation.match(Math.min(funcExpl.getValue(), maxBoost), "min of:",
                     funcExpl, Explanation.match(maxBoost, "maxBoost"));
             return Explanation.match(
-                    toFloat((Math.min(funcExpl.getValue(), maxBoost) + queryExpl.getValue()) / 2.0), "avg of",
+                    (float) ((Math.min(funcExpl.getValue(), maxBoost) + queryExpl.getValue()) / 2.0), "avg of",
                     queryExpl, minExpl);
         }
 
@@ -95,7 +95,7 @@ public Explanation explain(Explanation queryExpl, Explanation funcExpl, float ma
     MIN {
         @Override
         public float combine(double queryScore, double funcScore, double maxBoost) {
-            return toFloat(Math.min(queryScore, Math.min(funcScore, maxBoost)));
+            return (float) (Math.min(queryScore, Math.min(funcScore, maxBoost)));
         }
 
         @Override
@@ -112,7 +112,7 @@ public Explanation explain(Explanation queryExpl, Explanation funcExpl, float ma
     MAX {
         @Override
         public float combine(double queryScore, double funcScore, double maxBoost) {
-            return toFloat(Math.max(queryScore, Math.min(funcScore, maxBoost)));
+            return (float) (Math.max(queryScore, Math.min(funcScore, maxBoost)));
         }
 
         @Override
@@ -129,29 +129,15 @@ public Explanation explain(Explanation queryExpl, Explanation funcExpl, float ma
 
     public abstract float combine(double queryScore, double funcScore, double maxBoost);
 
-    public static float toFloat(double input) {
-        assert deviation(input) <= 0.001 : "input " + input + " out of float scope for function score deviation: " + deviation(input);
-        return (float) input;
-    }
-
-    private static double deviation(double input) { // only with assert!
-        float floatVersion = (float) input;
-        return Double.compare(floatVersion, input) == 0 || input == 0.0d ? 0 : 1.d - (floatVersion) / input;
-    }
-
     public abstract Explanation explain(Explanation queryExpl, Explanation funcExpl, float maxBoost);
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        out.writeVInt(this.ordinal());
+        out.writeEnum(this);
     }
 
     public static CombineFunction readFromStream(StreamInput in) throws IOException {
-        int ordinal = in.readVInt();
-        if (ordinal < 0 || ordinal >= values().length) {
-            throw new IOException("Unknown CombineFunction ordinal [" + ordinal + "]");
-        }
-        return values()[ordinal];
+        return in.readEnum(CombineFunction.class);
     }
 
     public static CombineFunction fromString(String combineFunction) {
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FieldValueFactorFunction.java b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FieldValueFactorFunction.java
index 3bc5542c2aa13..42047d664f3b6 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FieldValueFactorFunction.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FieldValueFactorFunction.java
@@ -96,7 +96,7 @@ public Explanation explainScore(int docId, Explanation subQueryScore) {
                 String defaultStr = missing != null ? "?:" + missing : "";
                 double score = score(docId, subQueryScore.getValue());
                 return Explanation.match(
-                        CombineFunction.toFloat(score),
+                        (float) score,
                         String.format(Locale.ROOT,
                                 "field value function: %s(doc['%s'].value%s * factor=%s)", modifierStr, field, defaultStr, boostFactor));
             }
@@ -191,15 +191,11 @@ public double apply(double n) {
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
-            out.writeVInt(this.ordinal());
+            out.writeEnum(this);
         }
 
         public static Modifier readFromStream(StreamInput in) throws IOException {
-            int ordinal = in.readVInt();
-            if (ordinal < 0 || ordinal >= values().length) {
-                throw new IOException("Unknown Modifier ordinal [" + ordinal + "]");
-            }
-            return values()[ordinal];
+            return in.readEnum(Modifier.class);
         }
 
         @Override
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java
index fd7c8f6c49df1..cf12d574fd625 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java
@@ -27,6 +27,7 @@
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.ScorerSupplier;
 import org.apache.lucene.search.Weight;
 import org.apache.lucene.util.Bits;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -37,7 +38,6 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
@@ -82,15 +82,11 @@ public enum ScoreMode implements Writeable {
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
-            out.writeVInt(this.ordinal());
+            out.writeEnum(this);
         }
 
         public static ScoreMode readFromStream(StreamInput in) throws IOException {
-            int ordinal = in.readVInt();
-            if (ordinal < 0 || ordinal >= values().length) {
-                throw new IOException("Unknown ScoreMode ordinal [" + ordinal + "]");
-            }
-            return values()[ordinal];
+            return in.readEnum(ScoreMode.class);
         }
 
         public static ScoreMode fromString(String scoreMode) {
@@ -157,7 +153,7 @@ class CustomBoostFactorWeight extends Weight {
         final Weight[] filterWeights;
         final boolean needsScores;
 
-        public CustomBoostFactorWeight(Query parent, Weight subQueryWeight, Weight[] filterWeights, boolean needsScores) throws IOException {
+        CustomBoostFactorWeight(Query parent, Weight subQueryWeight, Weight[] filterWeights, boolean needsScores) throws IOException {
             super(parent);
             this.subQueryWeight = subQueryWeight;
             this.filterWeights = filterWeights;
@@ -189,8 +185,8 @@ private FiltersFunctionFactorScorer functionScorer(LeafReaderContext context) th
             for (int i = 0; i < filterFunctions.length; i++) {
                 FilterFunction filterFunction = filterFunctions[i];
                 functions[i] = filterFunction.function.getLeafScoreFunction(context);
-                Scorer filterScorer = filterWeights[i].scorer(context);
-                docSets[i] = Lucene.asSequentialAccessBits(context.reader().maxDoc(), filterScorer);
+                ScorerSupplier filterScorerSupplier = filterWeights[i].scorerSupplier(context);
+                docSets[i] = Lucene.asSequentialAccessBits(context.reader().maxDoc(), filterScorerSupplier);
             }
             return new FiltersFunctionFactorScorer(this, subQueryScorer, scoreMode, filterFunctions, maxBoost, functions, docSets, combineFunction, needsScores);
         }
@@ -215,12 +211,12 @@ public Explanation explain(LeafReaderContext context, int doc) throws IOExceptio
             List<Explanation> filterExplanations = new ArrayList<>();
             for (int i = 0; i < filterFunctions.length; ++i) {
                 Bits docSet = Lucene.asSequentialAccessBits(context.reader().maxDoc(),
-                        filterWeights[i].scorer(context));
+                        filterWeights[i].scorerSupplier(context));
                 if (docSet.get(doc)) {
                     FilterFunction filterFunction = filterFunctions[i];
                     Explanation functionExplanation = filterFunction.function.getLeafScoreFunction(context).explainScore(doc, expl);
                     double factor = functionExplanation.getValue();
-                    float sc = CombineFunction.toFloat(factor);
+                    float sc = (float) factor;
                     Explanation filterExplanation = Explanation.match(sc, "function score, product of:",
                             Explanation.match(1.0f, "match filter: " + filterFunction.filter.toString()), functionExplanation);
                     filterExplanations.add(filterExplanation);
@@ -233,7 +229,7 @@ public Explanation explain(LeafReaderContext context, int doc) throws IOExceptio
             Explanation factorExplanation;
             if (filterExplanations.size() > 0) {
                 factorExplanation = Explanation.match(
-                        CombineFunction.toFloat(score),
+                        (float) score,
                         "function score, score mode [" + scoreMode.toString().toLowerCase(Locale.ROOT) + "]",
                         filterExplanations);
 
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java
index 5e94e82021f81..61de1ab303f24 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java
@@ -109,7 +109,7 @@ class CustomBoostFactorWeight extends Weight {
         final Weight subQueryWeight;
         final boolean needsScores;
 
-        public CustomBoostFactorWeight(Query parent, Weight subQueryWeight, boolean needsScores) throws IOException {
+        CustomBoostFactorWeight(Query parent, Weight subQueryWeight, boolean needsScores) throws IOException {
             super(parent);
             this.subQueryWeight = subQueryWeight;
             this.needsScores = needsScores;
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/function/RandomScoreFunction.java b/core/src/main/java/org/elasticsearch/common/lucene/search/function/RandomScoreFunction.java
index 3810c16bc0ed4..5ad7232d6ad1b 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/search/function/RandomScoreFunction.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/search/function/RandomScoreFunction.java
@@ -77,7 +77,7 @@ public double score(int docId, float subQueryScore) {
             @Override
             public Explanation explainScore(int docId, Explanation subQueryScore) {
                 return Explanation.match(
-                        CombineFunction.toFloat(score(docId, subQueryScore.getValue())),
+                        (float) score(docId, subQueryScore.getValue()),
                         "random score function (seed: " + originalSeed + ")");
             }
         };
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunction.java b/core/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunction.java
index 47b8525073551..5f1901c368714 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunction.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunction.java
@@ -38,7 +38,7 @@ static final class CannedScorer extends Scorer {
         protected int docid;
         protected float score;
 
-        public CannedScorer() {
+        CannedScorer() {
             super(null);
         }
 
@@ -110,7 +110,7 @@ public Explanation explainScore(int docId, Explanation subQueryScore) throws IOE
                             subQueryScore.getValue(), "_score: ",
                             subQueryScore);
                     return Explanation.match(
-                            CombineFunction.toFloat(score), explanation,
+                            (float) score, explanation,
                             scoreExp);
                 }
                 return exp;
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/function/WeightFactorFunction.java b/core/src/main/java/org/elasticsearch/common/lucene/search/function/WeightFactorFunction.java
index cadb3f55df493..2ed20c9598703 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/search/function/WeightFactorFunction.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/search/function/WeightFactorFunction.java
@@ -71,7 +71,7 @@ public Explanation explainScore(int docId, Explanation subQueryScore) throws IOE
 
     @Override
     public boolean needsScores() {
-        return false;
+        return scoreFunction.needsScores();
     }
 
     public Explanation explainWeight() {
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDAndVersionLookup.java b/core/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDAndVersionLookup.java
index 67f06c4f8d05b..fd61829f15311 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDAndVersionLookup.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDAndVersionLookup.java
@@ -29,8 +29,7 @@
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.lucene.uid.Versions.DocIdAndVersion;
-import org.elasticsearch.index.mapper.UidFieldMapper;
+import org.elasticsearch.common.lucene.uid.VersionsResolver.DocIdAndVersion;
 import org.elasticsearch.index.mapper.VersionFieldMapper;
 
 import java.io.IOException;
@@ -48,52 +47,71 @@ final class PerThreadIDAndVersionLookup {
     // we keep it around for now, to reduce the amount of e.g. hash lookups by field and stuff
 
     /** terms enum for uid field */
+    final String uidField;
     private final TermsEnum termsEnum;
     /** _version data */
     private final NumericDocValues versions;
+
     /** Reused for iteration (when the term exists) */
     private PostingsEnum docsEnum;
 
+    /** used for assertions to make sure class usage meets assumptions */
+    private final Object readerKey;
+
     /**
      * Initialize lookup for the provided segment
      */
-    public PerThreadIDAndVersionLookup(LeafReader reader) throws IOException {
-        TermsEnum termsEnum = null;
-        NumericDocValues versions = null;
-
+    PerThreadIDAndVersionLookup(LeafReader reader, String uidField) throws IOException {
+        this.uidField = uidField;
         Fields fields = reader.fields();
-        if (fields != null) {
-            Terms terms = fields.terms(UidFieldMapper.NAME);
-            if (terms != null) {
-                termsEnum = terms.iterator();
-                assert termsEnum != null;
-                versions = reader.getNumericDocValues(VersionFieldMapper.NAME);
-                assert versions != null;
-            }
+        Terms terms = fields.terms(uidField);
+        if (terms == null) {
+            throw new IllegalArgumentException("reader misses the [" + uidField +
+                "] field");
         }
-
-        this.versions = versions;
-        this.termsEnum = termsEnum;
+        termsEnum = terms.iterator();
+        versions = reader.getNumericDocValues(VersionFieldMapper.NAME);
+        if (versions == null) {
+            throw new IllegalArgumentException("reader misses the [" + VersionFieldMapper.NAME +
+                "] field");
+        }
+        Object readerKey = null;
+        assert (readerKey = reader.getCoreCacheKey()) != null;
+        this.readerKey = readerKey;
     }
 
     /** Return null if id is not found. */
-    public DocIdAndVersion lookup(BytesRef id, Bits liveDocs, LeafReaderContext context) throws IOException {
+    public DocIdAndVersion lookupVersion(BytesRef id, Bits liveDocs, LeafReaderContext context)
+        throws IOException {
+        assert context.reader().getCoreCacheKey().equals(readerKey) :
+            "context's reader is not the same as the reader class was initialized on.";
+        int docID = getDocID(id, liveDocs);
+
+        if (docID != DocIdSetIterator.NO_MORE_DOCS) {
+            return new DocIdAndVersion(docID, versions.get(docID), context);
+        } else {
+            return null;
+        }
+    }
+
+    /**
+     * returns the internal lucene doc id for the given id bytes.
+     * {@link DocIdSetIterator#NO_MORE_DOCS} is returned if not found
+     * */
+    private int getDocID(BytesRef id, Bits liveDocs) throws IOException {
         if (termsEnum.seekExact(id)) {
+            int docID = DocIdSetIterator.NO_MORE_DOCS;
             // there may be more than one matching docID, in the case of nested docs, so we want the last one:
             docsEnum = termsEnum.postings(docsEnum, 0);
-            int docID = DocIdSetIterator.NO_MORE_DOCS;
             for (int d = docsEnum.nextDoc(); d != DocIdSetIterator.NO_MORE_DOCS; d = docsEnum.nextDoc()) {
                 if (liveDocs != null && liveDocs.get(d) == false) {
                     continue;
                 }
                 docID = d;
             }
-
-            if (docID != DocIdSetIterator.NO_MORE_DOCS) {
-                return new DocIdAndVersion(docID, versions.get(docID), context);
-            }
+            return docID;
+        } else {
+            return DocIdSetIterator.NO_MORE_DOCS;
         }
-
-        return null;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/uid/Versions.java b/core/src/main/java/org/elasticsearch/common/lucene/uid/Versions.java
index 72dc9c8937317..e0997954d7932 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/uid/Versions.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/uid/Versions.java
@@ -19,21 +19,8 @@
 
 package org.elasticsearch.common.lucene.uid;
 
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.LeafReader.CoreClosedListener;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.util.CloseableThreadLocal;
-import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
-import org.elasticsearch.index.mapper.UidFieldMapper;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.concurrent.ConcurrentMap;
-
 /** Utility class to resolve the Lucene doc ID and version for a given uid. */
-public class Versions {
+public final class Versions {
 
     /** used to indicate the write operation should succeed regardless of current version **/
     public static final long MATCH_ANY = -3L;
@@ -49,98 +36,4 @@ public class Versions {
      */
     public static final long MATCH_DELETED = -4L;
 
-    // TODO: is there somewhere else we can store these?
-    static final ConcurrentMap<Object, CloseableThreadLocal<PerThreadIDAndVersionLookup>> lookupStates = ConcurrentCollections.newConcurrentMapWithAggressiveConcurrency();
-
-    // Evict this reader from lookupStates once it's closed:
-    private static final CoreClosedListener removeLookupState = new CoreClosedListener() {
-        @Override
-        public void onClose(Object key) {
-            CloseableThreadLocal<PerThreadIDAndVersionLookup> ctl = lookupStates.remove(key);
-            if (ctl != null) {
-                ctl.close();
-            }
-        }
-    };
-
-    private static PerThreadIDAndVersionLookup getLookupState(LeafReader reader) throws IOException {
-        Object key = reader.getCoreCacheKey();
-        CloseableThreadLocal<PerThreadIDAndVersionLookup> ctl = lookupStates.get(key);
-        if (ctl == null) {
-            // First time we are seeing this reader's core; make a
-            // new CTL:
-            ctl = new CloseableThreadLocal<>();
-            CloseableThreadLocal<PerThreadIDAndVersionLookup> other = lookupStates.putIfAbsent(key, ctl);
-            if (other == null) {
-                // Our CTL won, we must remove it when the
-                // core is closed:
-                reader.addCoreClosedListener(removeLookupState);
-            } else {
-                // Another thread beat us to it: just use
-                // their CTL:
-                ctl = other;
-            }
-        }
-
-        PerThreadIDAndVersionLookup lookupState = ctl.get();
-        if (lookupState == null) {
-            lookupState = new PerThreadIDAndVersionLookup(reader);
-            ctl.set(lookupState);
-        }
-
-        return lookupState;
-    }
-
-    private Versions() {
-    }
-
-    /** Wraps an {@link LeafReaderContext}, a doc ID <b>relative to the context doc base</b> and a version. */
-    public static class DocIdAndVersion {
-        public final int docId;
-        public final long version;
-        public final LeafReaderContext context;
-
-        public DocIdAndVersion(int docId, long version, LeafReaderContext context) {
-            this.docId = docId;
-            this.version = version;
-            this.context = context;
-        }
-    }
-
-    /**
-     * Load the internal doc ID and version for the uid from the reader, returning<ul>
-     * <li>null if the uid wasn't found,
-     * <li>a doc ID and a version otherwise
-     * </ul>
-     */
-    public static DocIdAndVersion loadDocIdAndVersion(IndexReader reader, Term term) throws IOException {
-        assert term.field().equals(UidFieldMapper.NAME);
-        List<LeafReaderContext> leaves = reader.leaves();
-        if (leaves.isEmpty()) {
-            return null;
-        }
-        // iterate backwards to optimize for the frequently updated documents
-        // which are likely to be in the last segments
-        for (int i = leaves.size() - 1; i >= 0; i--) {
-            LeafReaderContext context = leaves.get(i);
-            LeafReader leaf = context.reader();
-            PerThreadIDAndVersionLookup lookup = getLookupState(leaf);
-            DocIdAndVersion result = lookup.lookup(term.bytes(), leaf.getLiveDocs(), context);
-            if (result != null) {
-                return result;
-            }
-        }
-        return null;
-    }
-
-    /**
-     * Load the version for the uid from the reader, returning<ul>
-     * <li>{@link #NOT_FOUND} if no matching doc exists,
-     * <li>the version associated with the provided uid otherwise
-     * </ul>
-     */
-    public static long loadVersion(IndexReader reader, Term term) throws IOException {
-        final DocIdAndVersion docIdAndVersion = loadDocIdAndVersion(reader, term);
-        return docIdAndVersion == null ? NOT_FOUND : docIdAndVersion.version;
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/uid/VersionsResolver.java b/core/src/main/java/org/elasticsearch/common/lucene/uid/VersionsResolver.java
new file mode 100644
index 0000000000000..bb82881ecd0e0
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/lucene/uid/VersionsResolver.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.lucene.uid;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.LeafReader.CoreClosedListener;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.util.CloseableThreadLocal;
+import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.ConcurrentMap;
+
+import static org.elasticsearch.common.lucene.uid.Versions.NOT_FOUND;
+
+/** Utility class to resolve the Lucene doc ID and version for a given uid. */
+public class VersionsResolver {
+
+    static final ConcurrentMap<Object, CloseableThreadLocal<PerThreadIDAndVersionLookup>>
+        lookupStates = ConcurrentCollections.newConcurrentMapWithAggressiveConcurrency();
+
+    // Evict this reader from lookupStates once it's closed:
+    private static final CoreClosedListener removeLookupState = key -> {
+        CloseableThreadLocal<PerThreadIDAndVersionLookup> ctl = lookupStates.remove(key);
+        if (ctl != null) {
+            ctl.close();
+        }
+    };
+
+    private static PerThreadIDAndVersionLookup getLookupState(LeafReader reader, String uidField)
+        throws IOException {
+        Object key = reader.getCoreCacheKey();
+        CloseableThreadLocal<PerThreadIDAndVersionLookup> ctl = lookupStates.get(key);
+        if (ctl == null) {
+            // First time we are seeing this reader's core; make a
+            // new CTL:
+            ctl = new CloseableThreadLocal<>();
+            CloseableThreadLocal<PerThreadIDAndVersionLookup> other =
+                lookupStates.putIfAbsent(key, ctl);
+            if (other == null) {
+                // Our CTL won, we must remove it when the
+                // core is closed:
+                reader.addCoreClosedListener(removeLookupState);
+            } else {
+                // Another thread beat us to it: just use
+                // their CTL:
+                ctl = other;
+            }
+        }
+
+        PerThreadIDAndVersionLookup lookupState = ctl.get();
+        if (lookupState == null) {
+            lookupState = new PerThreadIDAndVersionLookup(reader, uidField);
+            ctl.set(lookupState);
+        } else if (Objects.equals(lookupState.uidField, uidField) == false) {
+            throw new AssertionError("Index does not consistently use the same uid field: ["
+                    + uidField + "] != [" + lookupState.uidField + "]");
+        }
+
+        return lookupState;
+    }
+
+    private VersionsResolver() {
+    }
+
+    /**
+     * Wraps an {@link LeafReaderContext}, a doc ID <b>relative to the context doc base</b> and
+     * a version.
+     **/
+    public static class DocIdAndVersion {
+        public final int docId;
+        public final long version;
+        public final LeafReaderContext context;
+
+        public DocIdAndVersion(int docId, long version, LeafReaderContext context) {
+            this.docId = docId;
+            this.version = version;
+            this.context = context;
+        }
+    }
+
+    /**
+     * Load the internal doc ID and version for the uid from the reader, returning<ul>
+     * <li>null if the uid wasn't found,
+     * <li>a doc ID and a version otherwise
+     * </ul>
+     */
+    public static DocIdAndVersion loadDocIdAndVersion(IndexReader reader, Term term)
+        throws IOException {
+        List<LeafReaderContext> leaves = reader.leaves();
+        if (leaves.isEmpty()) {
+            return null;
+        }
+        // iterate backwards to optimize for the frequently updated documents
+        // which are likely to be in the last segments
+        for (int i = leaves.size() - 1; i >= 0; i--) {
+            LeafReaderContext context = leaves.get(i);
+            LeafReader leaf = context.reader();
+            PerThreadIDAndVersionLookup lookup = getLookupState(leaf, term.field());
+            DocIdAndVersion result =
+                lookup.lookupVersion(term.bytes(), leaf.getLiveDocs(), context);
+            if (result != null) {
+                return result;
+            }
+        }
+        return null;
+    }
+
+    /**
+     * Load the version for the uid from the reader, returning<ul>
+     * <li>{@link Versions#NOT_FOUND} if no matching doc exists,
+     * <li>the version associated with the provided uid otherwise
+     * </ul>
+     */
+    public static long loadVersion(IndexReader reader, Term term) throws IOException {
+        final DocIdAndVersion docIdAndVersion = loadDocIdAndVersion(reader, term);
+        return docIdAndVersion == null ? NOT_FOUND : docIdAndVersion.version;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java b/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java
index bb4a4bd3b3093..1c7e862e76dea 100644
--- a/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java
+++ b/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java
@@ -24,37 +24,46 @@
 import org.elasticsearch.cluster.routing.allocation.command.AllocateReplicaAllocationCommand;
 import org.elasticsearch.cluster.routing.allocation.command.AllocateStalePrimaryAllocationCommand;
 import org.elasticsearch.cluster.routing.allocation.command.AllocationCommand;
-import org.elasticsearch.cluster.routing.allocation.command.AllocationCommandRegistry;
 import org.elasticsearch.cluster.routing.allocation.command.CancelAllocationCommand;
 import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand;
+import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.inject.AbstractModule;
-import org.elasticsearch.common.inject.util.Providers;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.ExtensionPoint;
-import org.elasticsearch.http.HttpServer;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.http.HttpServerTransport;
+import org.elasticsearch.indices.breaker.CircuitBreakerService;
+import org.elasticsearch.plugins.NetworkPlugin;
 import org.elasticsearch.tasks.RawTaskStatus;
 import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.Transport;
-import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.transport.TransportInterceptor;
+import org.elasticsearch.transport.TransportRequest;
+import org.elasticsearch.transport.TransportRequestHandler;
 import org.elasticsearch.transport.local.LocalTransport;
 
+import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Supplier;
 
 /**
  * A module to handle registering and binding all network related classes.
  */
-public class NetworkModule extends AbstractModule {
+public final class NetworkModule {
 
     public static final String TRANSPORT_TYPE_KEY = "transport.type";
-    public static final String TRANSPORT_SERVICE_TYPE_KEY = "transport.service.type";
     public static final String HTTP_TYPE_KEY = "http.type";
     public static final String LOCAL_TRANSPORT = "local";
     public static final String HTTP_TYPE_DEFAULT_KEY = "http.type.default";
@@ -65,58 +74,89 @@ public class NetworkModule extends AbstractModule {
     public static final Setting<String> HTTP_DEFAULT_TYPE_SETTING = Setting.simpleString(HTTP_TYPE_DEFAULT_KEY, Property.NodeScope);
     public static final Setting<String> HTTP_TYPE_SETTING = Setting.simpleString(HTTP_TYPE_KEY, Property.NodeScope);
     public static final Setting<Boolean> HTTP_ENABLED = Setting.boolSetting("http.enabled", true, Property.NodeScope);
-    public static final Setting<String> TRANSPORT_SERVICE_TYPE_SETTING =
-        Setting.simpleString(TRANSPORT_SERVICE_TYPE_KEY, Property.NodeScope);
     public static final Setting<String> TRANSPORT_TYPE_SETTING = Setting.simpleString(TRANSPORT_TYPE_KEY, Property.NodeScope);
 
-    private final NetworkService networkService;
     private final Settings settings;
     private final boolean transportClient;
 
-    private final AllocationCommandRegistry allocationCommandRegistry = new AllocationCommandRegistry();
-    private final ExtensionPoint.SelectedType<TransportService> transportServiceTypes = new ExtensionPoint.SelectedType<>("transport_service", TransportService.class);
-    private final ExtensionPoint.SelectedType<Transport> transportTypes = new ExtensionPoint.SelectedType<>("transport", Transport.class);
-    private final ExtensionPoint.SelectedType<HttpServerTransport> httpTransportTypes = new ExtensionPoint.SelectedType<>("http_transport", HttpServerTransport.class);
-    private final List<NamedWriteableRegistry.Entry> namedWriteables = new ArrayList<>();
+    private static final List<NamedWriteableRegistry.Entry> namedWriteables = new ArrayList<>();
+    private static final List<NamedXContentRegistry.Entry> namedXContents = new ArrayList<>();
+
+    static {
+        registerAllocationCommand(CancelAllocationCommand::new, CancelAllocationCommand::fromXContent,
+            CancelAllocationCommand.COMMAND_NAME_FIELD);
+        registerAllocationCommand(MoveAllocationCommand::new, MoveAllocationCommand::fromXContent,
+            MoveAllocationCommand.COMMAND_NAME_FIELD);
+        registerAllocationCommand(AllocateReplicaAllocationCommand::new, AllocateReplicaAllocationCommand::fromXContent,
+            AllocateReplicaAllocationCommand.COMMAND_NAME_FIELD);
+        registerAllocationCommand(AllocateEmptyPrimaryAllocationCommand::new, AllocateEmptyPrimaryAllocationCommand::fromXContent,
+            AllocateEmptyPrimaryAllocationCommand.COMMAND_NAME_FIELD);
+        registerAllocationCommand(AllocateStalePrimaryAllocationCommand::new, AllocateStalePrimaryAllocationCommand::fromXContent,
+            AllocateStalePrimaryAllocationCommand.COMMAND_NAME_FIELD);
+        namedWriteables.add(
+            new NamedWriteableRegistry.Entry(Task.Status.class, ReplicationTask.Status.NAME, ReplicationTask.Status::new));
+        namedWriteables.add(
+            new NamedWriteableRegistry.Entry(Task.Status.class, RawTaskStatus.NAME, RawTaskStatus::new));
+    }
+
+    private final Map<String, Supplier<Transport>> transportFactories = new HashMap<>();
+    private final Map<String, Supplier<HttpServerTransport>> transportHttpFactories = new HashMap<>();
+    private final List<TransportInterceptor> transportIntercetors = new ArrayList<>();
 
     /**
      * Creates a network module that custom networking classes can be plugged into.
-     * @param networkService A constructed network service object to bind.
      * @param settings The settings for the node
      * @param transportClient True if only transport classes should be allowed to be registered, false otherwise.
      */
-    public NetworkModule(NetworkService networkService, Settings settings, boolean transportClient) {
-        this.networkService = networkService;
+    public NetworkModule(Settings settings, boolean transportClient, List<NetworkPlugin> plugins, ThreadPool threadPool,
+                         BigArrays bigArrays,
+                         CircuitBreakerService circuitBreakerService,
+                         NamedWriteableRegistry namedWriteableRegistry,
+                         NamedXContentRegistry xContentRegistry,
+                         NetworkService networkService, HttpServerTransport.Dispatcher dispatcher) {
         this.settings = settings;
         this.transportClient = transportClient;
-        registerTransportService("default", TransportService.class);
-        registerTransport(LOCAL_TRANSPORT, LocalTransport.class);
-        namedWriteables.add(new NamedWriteableRegistry.Entry(Task.Status.class, ReplicationTask.Status.NAME, ReplicationTask.Status::new));
-        namedWriteables.add(new NamedWriteableRegistry.Entry(Task.Status.class, RawTaskStatus.NAME, RawTaskStatus::new));
-        registerBuiltinAllocationCommands();
+        registerTransport(LOCAL_TRANSPORT, () -> new LocalTransport(settings, threadPool, namedWriteableRegistry, circuitBreakerService));
+        for (NetworkPlugin plugin : plugins) {
+            if (transportClient == false && HTTP_ENABLED.get(settings)) {
+                Map<String, Supplier<HttpServerTransport>> httpTransportFactory = plugin.getHttpTransports(settings, threadPool, bigArrays,
+                    circuitBreakerService, namedWriteableRegistry, xContentRegistry, networkService, dispatcher);
+                for (Map.Entry<String, Supplier<HttpServerTransport>> entry : httpTransportFactory.entrySet()) {
+                    registerHttpTransport(entry.getKey(), entry.getValue());
+                }
+            }
+            Map<String, Supplier<Transport>> httpTransportFactory = plugin.getTransports(settings, threadPool, bigArrays,
+                circuitBreakerService, namedWriteableRegistry, networkService);
+            for (Map.Entry<String, Supplier<Transport>> entry : httpTransportFactory.entrySet()) {
+                registerTransport(entry.getKey(), entry.getValue());
+            }
+            List<TransportInterceptor> transportInterceptors = plugin.getTransportInterceptors(threadPool.getThreadContext());
+            for (TransportInterceptor interceptor : transportInterceptors) {
+                registerTransportInterceptor(interceptor);
+            }
+        }
     }
 
     public boolean isTransportClient() {
         return transportClient;
     }
 
-    /** Adds a transport service implementation that can be selected by setting {@link #TRANSPORT_SERVICE_TYPE_KEY}. */
-    public void registerTransportService(String name, Class<? extends TransportService> clazz) {
-        transportServiceTypes.registerExtension(name, clazz);
-    }
-
     /** Adds a transport implementation that can be selected by setting {@link #TRANSPORT_TYPE_KEY}. */
-    public void registerTransport(String name, Class<? extends Transport> clazz) {
-        transportTypes.registerExtension(name, clazz);
+    private void registerTransport(String key, Supplier<Transport> factory) {
+        if (transportFactories.putIfAbsent(key, factory) != null) {
+            throw new IllegalArgumentException("transport for name: " + key + " is already registered");
+        }
     }
 
     /** Adds an http transport implementation that can be selected by setting {@link #HTTP_TYPE_KEY}. */
     // TODO: we need another name than "http transport"....so confusing with transportClient...
-    public void registerHttpTransport(String name, Class<? extends HttpServerTransport> clazz) {
+    private void registerHttpTransport(String key, Supplier<HttpServerTransport> factory) {
         if (transportClient) {
-            throw new IllegalArgumentException("Cannot register http transport " + clazz.getName() + " for transport client");
+            throw new IllegalArgumentException("Cannot register http transport " + key + " for transport client");
+        }
+        if (transportHttpFactories.putIfAbsent(key, factory) != null) {
+            throw new IllegalArgumentException("transport for name: " + key + " is already registered");
         }
-        httpTransportTypes.registerExtension(name, clazz);
     }
 
     /**
@@ -129,56 +169,91 @@ public void registerHttpTransport(String name, Class<? extends HttpServerTranspo
      * @param commandName the names under which the command should be parsed. The {@link ParseField#getPreferredName()} is special because
      *        it is the name under which the command's reader is registered.
      */
-    private <T extends AllocationCommand> void registerAllocationCommand(Writeable.Reader<T> reader, AllocationCommand.Parser<T> parser,
-            ParseField commandName) {
-        allocationCommandRegistry.register(parser, commandName);
-        namedWriteables.add(new Entry(AllocationCommand.class, commandName.getPreferredName(), reader));
+    private static <T extends AllocationCommand> void registerAllocationCommand(Writeable.Reader<T> reader,
+            CheckedFunction<XContentParser, T, IOException> parser, ParseField commandName) {
+        namedXContents.add(new NamedXContentRegistry.Entry(AllocationCommand.class, commandName, parser));
+        namedWriteables.add(new NamedWriteableRegistry.Entry(AllocationCommand.class, commandName.getPreferredName(), reader));
     }
 
-    /**
-     * The registry of allocation command parsers.
-     */
-    public AllocationCommandRegistry getAllocationCommandRegistry() {
-        return allocationCommandRegistry;
+    public static List<NamedWriteableRegistry.Entry> getNamedWriteables() {
+        return Collections.unmodifiableList(namedWriteables);
     }
 
-    public List<Entry> getNamedWriteables() {
-        return namedWriteables;
+    public static List<NamedXContentRegistry.Entry> getNamedXContents() {
+        return Collections.unmodifiableList(namedXContents);
     }
 
-    @Override
-    protected void configure() {
-        bind(NetworkService.class).toInstance(networkService);
-        transportServiceTypes.bindType(binder(), settings, TRANSPORT_SERVICE_TYPE_KEY, "default");
-        transportTypes.bindType(binder(), settings, TRANSPORT_TYPE_KEY, TRANSPORT_DEFAULT_TYPE_SETTING.get(settings));
-
-        if (transportClient == false) {
-            if (HTTP_ENABLED.get(settings)) {
-                bind(HttpServer.class).asEagerSingleton();
-                httpTransportTypes.bindType(binder(), settings, HTTP_TYPE_SETTING.getKey(), HTTP_DEFAULT_TYPE_SETTING.get(settings));
-            } else {
-                bind(HttpServer.class).toProvider(Providers.of(null));
-            }
-            // Bind the AllocationCommandRegistry so RestClusterRerouteAction can get it.
-            bind(AllocationCommandRegistry.class).toInstance(allocationCommandRegistry);
+    public Supplier<HttpServerTransport> getHttpServerTransportSupplier() {
+        final String name;
+        if (HTTP_TYPE_SETTING.exists(settings)) {
+            name = HTTP_TYPE_SETTING.get(settings);
+        } else {
+            name = HTTP_DEFAULT_TYPE_SETTING.get(settings);
+        }
+        final Supplier<HttpServerTransport> factory = transportHttpFactories.get(name);
+        if (factory == null) {
+            throw new IllegalStateException("Unsupported http.type [" + name + "]");
         }
+        return factory;
     }
 
-    private void registerBuiltinAllocationCommands() {
-        registerAllocationCommand(CancelAllocationCommand::new, CancelAllocationCommand::fromXContent,
-                CancelAllocationCommand.COMMAND_NAME_FIELD);
-        registerAllocationCommand(MoveAllocationCommand::new, MoveAllocationCommand::fromXContent,
-                MoveAllocationCommand.COMMAND_NAME_FIELD);
-        registerAllocationCommand(AllocateReplicaAllocationCommand::new, AllocateReplicaAllocationCommand::fromXContent,
-                AllocateReplicaAllocationCommand.COMMAND_NAME_FIELD);
-        registerAllocationCommand(AllocateEmptyPrimaryAllocationCommand::new, AllocateEmptyPrimaryAllocationCommand::fromXContent,
-                AllocateEmptyPrimaryAllocationCommand.COMMAND_NAME_FIELD);
-        registerAllocationCommand(AllocateStalePrimaryAllocationCommand::new, AllocateStalePrimaryAllocationCommand::fromXContent,
-                AllocateStalePrimaryAllocationCommand.COMMAND_NAME_FIELD);
+    public boolean isHttpEnabled() {
+        return transportClient == false && HTTP_ENABLED.get(settings);
+    }
+
+    public Supplier<Transport> getTransportSupplier() {
+        final String name;
+        if (TRANSPORT_TYPE_SETTING.exists(settings)) {
+            name = TRANSPORT_TYPE_SETTING.get(settings);
+        } else {
+            name = TRANSPORT_DEFAULT_TYPE_SETTING.get(settings);
+        }
+        final Supplier<Transport> factory = transportFactories.get(name);
+        if (factory == null) {
+            throw new IllegalStateException("Unsupported transport.type [" + name + "]");
+        }
+        return factory;
+    }
+
+    /**
+     * Registers a new {@link TransportInterceptor}
+     */
+    private void registerTransportInterceptor(TransportInterceptor interceptor) {
+        this.transportIntercetors.add(Objects.requireNonNull(interceptor, "interceptor must not be null"));
+    }
 
+    /**
+     * Returns a composite {@link TransportInterceptor} containing all registered interceptors
+     * @see #registerTransportInterceptor(TransportInterceptor)
+     */
+    public TransportInterceptor getTransportInterceptor() {
+        return new CompositeTransportInterceptor(this.transportIntercetors);
     }
 
-    public boolean canRegisterHttpExtensions() {
-        return transportClient == false;
+    static final class CompositeTransportInterceptor implements TransportInterceptor {
+        final List<TransportInterceptor> transportInterceptors;
+
+        private CompositeTransportInterceptor(List<TransportInterceptor> transportInterceptors) {
+            this.transportInterceptors = new ArrayList<>(transportInterceptors);
+        }
+
+        @Override
+        public <T extends TransportRequest> TransportRequestHandler<T> interceptHandler(String action, String executor,
+                                                                                        boolean forceExecution,
+                                                                                        TransportRequestHandler<T> actualHandler) {
+            for (TransportInterceptor interceptor : this.transportInterceptors) {
+                actualHandler = interceptor.interceptHandler(action, executor, forceExecution, actualHandler);
+            }
+            return actualHandler;
+        }
+
+        @Override
+        public AsyncSender interceptSender(AsyncSender sender) {
+            for (TransportInterceptor interceptor : this.transportInterceptors) {
+                sender = interceptor.interceptSender(sender);
+            }
+            return sender;
+        }
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/common/network/NetworkUtils.java b/core/src/main/java/org/elasticsearch/common/network/NetworkUtils.java
index 8652d4c5c0521..9e06c39b83e03 100644
--- a/core/src/main/java/org/elasticsearch/common/network/NetworkUtils.java
+++ b/core/src/main/java/org/elasticsearch/common/network/NetworkUtils.java
@@ -32,6 +32,7 @@
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
+import java.util.Optional;
 
 /**
  * Utilities for network interfaces / addresses binding and publishing.
@@ -227,14 +228,15 @@ static InetAddress[] getAllAddresses() throws SocketException {
     
     /** Returns addresses for the given interface (it must be marked up) */
     static InetAddress[] getAddressesForInterface(String name) throws SocketException {
-        NetworkInterface intf = NetworkInterface.getByName(name);
-        if (intf == null) {
+        Optional<NetworkInterface> networkInterface = getInterfaces().stream().filter((netIf) -> name.equals(netIf.getName())).findFirst();
+
+        if (networkInterface.isPresent() == false) {
             throw new IllegalArgumentException("No interface named '" + name + "' found, got " + getInterfaces());
         }
-        if (!intf.isUp()) {
+        if (!networkInterface.get().isUp()) {
             throw new IllegalArgumentException("Interface '" + name + "' is not up and running");
         }
-        List<InetAddress> list = Collections.list(intf.getInetAddresses());
+        List<InetAddress> list = Collections.list(networkInterface.get().getInetAddresses());
         if (list.isEmpty()) {
             throw new IllegalArgumentException("Interface '" + name + "' has no internet addresses");
         }
diff --git a/core/src/main/java/org/elasticsearch/common/path/PathTrie.java b/core/src/main/java/org/elasticsearch/common/path/PathTrie.java
index c711bfa2a61b1..44accbf190aa2 100644
--- a/core/src/main/java/org/elasticsearch/common/path/PathTrie.java
+++ b/core/src/main/java/org/elasticsearch/common/path/PathTrie.java
@@ -111,7 +111,10 @@ public synchronized void insert(String[] path, int index, T value) {
                 // in case the target(last) node already exist but without a value
                 // than the value should be updated.
                 if (index == (path.length - 1)) {
-                    assert (node.value == null || node.value == value);
+                    if (node.value != null) {
+                        throw new IllegalArgumentException("Path [" + String.join("/", path)+ "] already has a value ["
+                                + node.value + "]");
+                    }
                     if (node.value == null) {
                         node.value = value;
                     }
@@ -190,6 +193,9 @@ public String toString() {
     public void insert(String path, T value) {
         String[] strings = path.split(SEPARATOR);
         if (strings.length == 0) {
+            if (rootValue != null) {
+                throw new IllegalArgumentException("Path [/] already has a value [" + rootValue + "]");
+            }
             rootValue = value;
             return;
         }
diff --git a/core/src/main/java/org/elasticsearch/common/recycler/Recyclers.java b/core/src/main/java/org/elasticsearch/common/recycler/Recyclers.java
index 5bac8f7bcfd2b..f84441fbce436 100644
--- a/core/src/main/java/org/elasticsearch/common/recycler/Recyclers.java
+++ b/core/src/main/java/org/elasticsearch/common/recycler/Recyclers.java
@@ -170,7 +170,7 @@ public static <T> Recycler<T> concurrent(final Recycler.Factory<T> factory, fina
                 }
             }
 
-            final int slot() {
+            int slot() {
                 final long id = Thread.currentThread().getId();
                 // don't trust Thread.hashCode to have equiprobable low bits
                 int slot = (int) BitMixer.mix64(id);
diff --git a/core/src/main/java/org/elasticsearch/common/regex/Regex.java b/core/src/main/java/org/elasticsearch/common/regex/Regex.java
index 061ad6c26c0e0..f1f945288e43d 100644
--- a/core/src/main/java/org/elasticsearch/common/regex/Regex.java
+++ b/core/src/main/java/org/elasticsearch/common/regex/Regex.java
@@ -19,8 +19,13 @@
 
 package org.elasticsearch.common.regex;
 
+import org.apache.lucene.util.automaton.Automata;
+import org.apache.lucene.util.automaton.Automaton;
+import org.apache.lucene.util.automaton.Operations;
 import org.elasticsearch.common.Strings;
 
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Locale;
 import java.util.regex.Pattern;
 
@@ -46,6 +51,33 @@ public static boolean isMatchAllPattern(String str) {
         return str.equals("*");
     }
 
+    /** Return an {@link Automaton} that matches the given pattern. */
+    public static Automaton simpleMatchToAutomaton(String pattern) {
+        List<Automaton> automata = new ArrayList<>();
+        int previous = 0;
+        for (int i = pattern.indexOf('*'); i != -1; i = pattern.indexOf('*', i + 1)) {
+            automata.add(Automata.makeString(pattern.substring(previous, i)));
+            automata.add(Automata.makeAnyString());
+            previous = i + 1;
+        }
+        automata.add(Automata.makeString(pattern.substring(previous)));
+        return Operations.concatenate(automata);
+    }
+
+    /**
+     * Return an Automaton that matches the union of the provided patterns.
+     */
+    public static Automaton simpleMatchToAutomaton(String... patterns) {
+        if (patterns.length < 1) {
+            throw new IllegalArgumentException("There must be at least one pattern, zero given");
+        }
+        List<Automaton> automata = new ArrayList<>();
+        for (String pattern : patterns) {
+            automata.add(simpleMatchToAutomaton(pattern));
+        }
+        return Operations.union(automata);
+    }
+
     /**
      * Match a String against the given pattern, supporting the following simple
      * pattern styles: "xxx*", "*xxx", "*xxx*" and "xxx*yyy" matches (with an
diff --git a/core/src/main/java/org/elasticsearch/common/rounding/DateTimeUnit.java b/core/src/main/java/org/elasticsearch/common/rounding/DateTimeUnit.java
index 375c1a27212a3..07997c8af3138 100644
--- a/core/src/main/java/org/elasticsearch/common/rounding/DateTimeUnit.java
+++ b/core/src/main/java/org/elasticsearch/common/rounding/DateTimeUnit.java
@@ -43,7 +43,7 @@ public enum DateTimeUnit {
     private final byte id;
     private final Function<DateTimeZone, DateTimeField> fieldFunction;
 
-    private DateTimeUnit(byte id, Function<DateTimeZone, DateTimeField> fieldFunction) {
+    DateTimeUnit(byte id, Function<DateTimeZone, DateTimeField> fieldFunction) {
         this.id = id;
         this.fieldFunction = fieldFunction;
     }
diff --git a/core/src/main/java/org/elasticsearch/common/rounding/Rounding.java b/core/src/main/java/org/elasticsearch/common/rounding/Rounding.java
index ad9f926e881f7..50461d6184ae7 100644
--- a/core/src/main/java/org/elasticsearch/common/rounding/Rounding.java
+++ b/core/src/main/java/org/elasticsearch/common/rounding/Rounding.java
@@ -128,15 +128,38 @@ public byte id() {
         @Override
         public long round(long utcMillis) {
             long rounded = field.roundFloor(utcMillis);
-            if (timeZone.isFixed() == false && timeZone.getOffset(utcMillis) != timeZone.getOffset(rounded)) {
-                // in this case, we crossed a time zone transition. In some edge
-                // cases this will
-                // result in a value that is not a rounded value itself. We need
-                // to round again
-                // to make sure. This will have no affect in cases where
-                // 'rounded' was already a proper
-                // rounded value
-                rounded = field.roundFloor(rounded);
+            if (timeZone.isFixed() == false) {
+                // special cases for non-fixed time zones with dst transitions
+                if (timeZone.getOffset(utcMillis) != timeZone.getOffset(rounded)) {
+                    /*
+                     * the offset change indicates a dst transition. In some
+                     * edge cases this will result in a value that is not a
+                     * rounded value before the transition. We round again to
+                     * make sure we really return a rounded value. This will
+                     * have no effect in cases where we already had a valid
+                     * rounded value
+                     */
+                    rounded = field.roundFloor(rounded);
+                } else {
+                    /*
+                     * check if the current time instant is at a start of a DST
+                     * overlap by comparing the offset of the instant and the
+                     * previous millisecond. We want to detect negative offset
+                     * changes that result in an overlap
+                     */
+                    if (timeZone.getOffset(rounded) < timeZone.getOffset(rounded - 1)) {
+                        /*
+                         * we are rounding a date just after a DST overlap. if
+                         * the overlap is smaller than the time unit we are
+                         * rounding to, we want to add the overlapping part to
+                         * the following rounding interval
+                         */
+                        long previousRounded = field.roundFloor(rounded - 1);
+                        if (rounded - previousRounded < field.getDurationField().getUnitMillis()) {
+                            rounded = previousRounded;
+                        }
+                    }
+                }
             }
             assert rounded == field.roundFloor(rounded);
             return rounded;
diff --git a/core/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java b/core/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java
index 787fa950bea63..05b7d96c8f6db 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java
@@ -35,11 +35,10 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.SortedMap;
-import java.util.TreeMap;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
+import java.util.function.Predicate;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
@@ -56,6 +55,7 @@ public abstract class AbstractScopedSettings extends AbstractComponent {
     private final Setting.Property scope;
     private static final Pattern KEY_PATTERN = Pattern.compile("^(?:[-\\w]+[.])*[-\\w]+$");
     private static final Pattern GROUP_KEY_PATTERN = Pattern.compile("^(?:[-\\w]+[.])+$");
+    private static final Pattern AFFIX_KEY_PATTERN = Pattern.compile("^(?:[-\\w]+[.])+[*](?:[.][-\\w]+)+$");
 
     protected AbstractScopedSettings(Settings settings, Set<Setting<?>> settingsSet, Setting.Property scope) {
         super(settings);
@@ -85,7 +85,8 @@ protected AbstractScopedSettings(Settings settings, Set<Setting<?>> settingsSet,
     }
 
     protected void validateSettingKey(Setting setting) {
-        if (isValidKey(setting.getKey()) == false && (setting.isGroupSetting() && isValidGroupKey(setting.getKey())) == false) {
+        if (isValidKey(setting.getKey()) == false && (setting.isGroupSetting() && isValidGroupKey(setting.getKey())
+            || isValidAffixKey(setting.getKey())) == false) {
             throw new IllegalArgumentException("illegal settings key: [" + setting.getKey() + "]");
         }
     }
@@ -110,6 +111,11 @@ private static boolean isValidGroupKey(String key) {
         return GROUP_KEY_PATTERN.matcher(key).matches();
     }
 
+    // pkg private for tests
+    static boolean isValidAffixKey(String key) {
+        return AFFIX_KEY_PATTERN.matcher(key).matches();
+    }
+
     public Setting.Property getScope() {
         return this.scope;
     }
@@ -188,6 +194,19 @@ public synchronized <T> void addSettingsUpdateConsumer(Setting<T> setting, Consu
         addSettingsUpdater(setting.newUpdater(consumer, logger, validator));
     }
 
+    /**
+     * Adds a settings consumer for affix settings. Affix settings have a namespace associated to it that needs to be available to the
+     * consumer in order to be processed correctly.
+     */
+    public synchronized <T> void addAffixUpdateConsumer(Setting.AffixSetting<T> setting,  BiConsumer<String, T> consumer,
+                                                        BiConsumer<String, T> validator) {
+        final Setting<?> registeredSetting = this.complexMatchers.get(setting.getKey());
+        if (setting != registeredSetting) {
+            throw new IllegalArgumentException("Setting is not registered for key [" + setting.getKey() + "]");
+        }
+        addSettingsUpdater(setting.newAffixUpdater(consumer, logger, validator));
+    }
+
     synchronized void addSettingsUpdater(SettingUpdater<?> updater) {
         this.settingUpdaters.add(updater);
     }
@@ -232,11 +251,9 @@ public final void validate(Settings.Builder settingsBuilder) {
      */
     public final void validate(Settings settings) {
         List<RuntimeException> exceptions = new ArrayList<>();
-        // we want them sorted for deterministic error messages
-        SortedMap<String, String> sortedSettings = new TreeMap<>(settings.getAsMap());
-        for (Map.Entry<String, String> entry : sortedSettings.entrySet()) {
+        for (String key : settings.keySet()) { // settings iterate in deterministic fashion
             try {
-                validate(entry.getKey(), settings);
+                validate(key, settings);
             } catch (RuntimeException ex) {
                 exceptions.add(ex);
             }
@@ -260,7 +277,12 @@ public final void validate(String key, Settings settings) {
                 }
             }
             CollectionUtil.timSort(scoredKeys, (a,b) -> b.v1().compareTo(a.v1()));
-            String msg = "unknown setting [" + key + "]";
+            String msgPrefix = "unknown setting";
+            SecureSettings secureSettings = settings.getSecureSettings();
+            if (secureSettings != null && settings.getSecureSettings().getSettingNames().contains(key)) {
+                msgPrefix = "unknown secure setting";
+            }
+            String msg = msgPrefix + " [" + key + "]";
             List<String> keys = scoredKeys.stream().map((a) -> a.v2()).collect(Collectors.toList());
             if (keys.isEmpty() == false) {
                 msg += " did you mean " + (keys.size() == 1 ? "[" + keys.get(0) + "]": "any of " + keys.toString()) + "?";
@@ -358,11 +380,19 @@ private boolean assertMatcher(String key, int numComplexMatchers) {
     /**
      * Returns <code>true</code> if the setting for the given key is dynamically updateable. Otherwise <code>false</code>.
      */
-    public boolean hasDynamicSetting(String key) {
+    public boolean isDynamicSetting(String key) {
         final Setting<?> setting = get(key);
         return setting != null && setting.isDynamic();
     }
 
+    /**
+     * Returns <code>true</code> if the setting for the given key is final. Otherwise <code>false</code>.
+     */
+    public boolean isFinalSetting(String key) {
+        final Setting<?> setting = get(key);
+        return setting != null && setting.isFinal();
+    }
+
     /**
      * Returns a settings object that contains all settings that are not
      * already set in the given source. The diff contains either the default value for each
@@ -371,9 +401,10 @@ public boolean hasDynamicSetting(String key) {
     public Settings diff(Settings source, Settings defaultSettings) {
         Settings.Builder builder = Settings.builder();
         for (Setting<?> setting : keySettings.values()) {
-            if (setting.exists(source) == false) {
-                builder.put(setting.getKey(), setting.getRaw(defaultSettings));
-            }
+            setting.diff(builder, source, defaultSettings);
+        }
+        for (Setting<?> setting : complexMatchers.values()) {
+            setting.diff(builder, source, defaultSettings);
         }
         return builder.build();
     }
@@ -440,31 +471,48 @@ private boolean updateSettings(Settings toApply, Settings.Builder target, Settin
         boolean changed = false;
         final Set<String> toRemove = new HashSet<>();
         Settings.Builder settingsBuilder = Settings.builder();
+        final Predicate<String> canUpdate = (key) -> (
+            isFinalSetting(key) == false && // it's not a final setting
+                ((onlyDynamic == false && get(key) != null) || isDynamicSetting(key)));
+        final Predicate<String> canRemove = (key) ->(// we can delete if
+            isFinalSetting(key) == false && // it's not a final setting
+                (onlyDynamic && isDynamicSetting(key)  // it's a dynamicSetting and we only do dynamic settings
+                || get(key) == null && key.startsWith(ARCHIVED_SETTINGS_PREFIX) // the setting is not registered AND it's been archived
+                || (onlyDynamic == false && get(key) != null))); // if it's not dynamic AND we have a key
         for (Map.Entry<String, String> entry : toApply.getAsMap().entrySet()) {
-            if (entry.getValue() == null) {
+            if (entry.getValue() == null && (canRemove.test(entry.getKey()) || entry.getKey().endsWith("*"))) {
+                // this either accepts null values that suffice the canUpdate test OR wildcard expressions (key ends with *)
+                // we don't validate if there is any dynamic setting with that prefix yet we could do in the future
                 toRemove.add(entry.getKey());
-            } else if ((onlyDynamic == false && get(entry.getKey()) != null) || hasDynamicSetting(entry.getKey())) {
+                // we don't set changed here it's set after we apply deletes below if something actually changed
+            } else if (entry.getValue() != null && canUpdate.test(entry.getKey())) {
                 validate(entry.getKey(), toApply);
                 settingsBuilder.put(entry.getKey(), entry.getValue());
                 updates.put(entry.getKey(), entry.getValue());
                 changed = true;
             } else {
-                throw new IllegalArgumentException(type + " setting [" + entry.getKey() + "], not dynamically updateable");
+                if (isFinalSetting(entry.getKey())) {
+                    throw new IllegalArgumentException("final " + type + " setting [" + entry.getKey() + "], not updateable");
+                } else {
+                    throw new IllegalArgumentException(type + " setting [" + entry.getKey() + "], not dynamically updateable");
+                }
             }
-
         }
-        changed |= applyDeletes(toRemove, target);
+        changed |= applyDeletes(toRemove, target, canRemove);
         target.put(settingsBuilder.build());
         return changed;
     }
 
-    private static boolean applyDeletes(Set<String> deletes, Settings.Builder builder) {
+    private static boolean applyDeletes(Set<String> deletes, Settings.Builder builder, Predicate<String> canRemove) {
         boolean changed = false;
         for (String entry : deletes) {
             Set<String> keysToRemove = new HashSet<>();
             Set<String> keySet = builder.internalMap().keySet();
             for (String key : keySet) {
-                if (Regex.simpleMatch(entry, key)) {
+                if (Regex.simpleMatch(entry, key) && canRemove.test(key)) {
+                    // we have to re-check with canRemove here since we might have a wildcard expression foo.* that matches
+                    // dynamic as well as static settings if that is the case we might remove static settings since we resolve the
+                    // wildcards late
                     keysToRemove.add(key);
                 }
             }
@@ -493,11 +541,21 @@ private static Setting<?> findOverlappingSetting(Setting<?> newSetting, Map<Stri
     }
 
     /**
-     * Archives broken or unknown settings. Any setting that is not recognized or fails
-     * validation will be archived. This means the setting is prefixed with {@value ARCHIVED_SETTINGS_PREFIX}
-     * and remains in the settings object. This can be used to detect broken settings via APIs.
+     * Archives invalid or unknown settings. Any setting that is not recognized or fails validation
+     * will be archived. This means the setting is prefixed with {@value ARCHIVED_SETTINGS_PREFIX}
+     * and remains in the settings object. This can be used to detect invalid settings via APIs.
+     *
+     * @param settings        the {@link Settings} instance to scan for unknown or invalid settings
+     * @param unknownConsumer callback on unknown settings (consumer receives unknown key and its
+     *                        associated value)
+     * @param invalidConsumer callback on invalid settings (consumer receives invalid key, its
+     *                        associated value and an exception)
+     * @return a {@link Settings} instance with the unknown or invalid settings archived
      */
-    public Settings archiveUnknownOrBrokenSettings(Settings settings) {
+    public Settings archiveUnknownOrInvalidSettings(
+        final Settings settings,
+        final Consumer<Map.Entry<String, String>> unknownConsumer,
+        final BiConsumer<Map.Entry<String, String>, IllegalArgumentException> invalidConsumer) {
         Settings.Builder builder = Settings.builder();
         boolean changed = false;
         for (Map.Entry<String, String> entry : settings.getAsMap().entrySet()) {
@@ -511,10 +569,10 @@ public Settings archiveUnknownOrBrokenSettings(Settings settings) {
                         builder.put(entry.getKey(), entry.getValue());
                     } else {
                         changed = true;
-                        logger.warn("found unknown setting: {} value: {} - archiving", entry.getKey(), entry.getValue());
+                        unknownConsumer.accept(entry);
                         /*
-                         * We put them back in here such that tools can check from the outside if there are any indices with broken
-                         * settings. The setting can remain there but we want users to be aware that some of their setting are broken and
+                         * We put them back in here such that tools can check from the outside if there are any indices with invalid
+                         * settings. The setting can remain there but we want users to be aware that some of their setting are invalid and
                          * they can research why and what they need to do to replace them.
                          */
                         builder.put(ARCHIVED_SETTINGS_PREFIX + entry.getKey(), entry.getValue());
@@ -522,12 +580,10 @@ public Settings archiveUnknownOrBrokenSettings(Settings settings) {
                 }
             } catch (IllegalArgumentException ex) {
                 changed = true;
-                logger.warn(
-                    (Supplier<?>) () -> new ParameterizedMessage(
-                        "found invalid setting: {} value: {} - archiving", entry.getKey(), entry.getValue()), ex);
+                invalidConsumer.accept(entry, ex);
                 /*
-                 * We put them back in here such that tools can check from the outside if there are any indices with broken settings. The
-                 * setting can remain there but we want users to be aware that some of their setting are broken and they can research why
+                 * We put them back in here such that tools can check from the outside if there are any indices with invalid settings. The
+                 * setting can remain there but we want users to be aware that some of their setting are invalid and they can research why
                  * and what they need to do to replace them.
                  */
                 builder.put(ARCHIVED_SETTINGS_PREFIX + entry.getKey(), entry.getValue());
diff --git a/core/src/main/java/org/elasticsearch/common/settings/AddFileKeyStoreCommand.java b/core/src/main/java/org/elasticsearch/common/settings/AddFileKeyStoreCommand.java
new file mode 100644
index 0000000000000..5ccac9a2ac3fa
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/settings/AddFileKeyStoreCommand.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.List;
+
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+import org.elasticsearch.cli.EnvironmentAwareCommand;
+import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cli.UserException;
+import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.common.io.PathUtils;
+import org.elasticsearch.env.Environment;
+
+/**
+ * A subcommand for the keystore cli which adds a file setting.
+ */
+class AddFileKeyStoreCommand extends EnvironmentAwareCommand {
+
+    private final OptionSpec<Void> forceOption;
+    private final OptionSpec<String> arguments;
+
+    AddFileKeyStoreCommand() {
+        super("Add a file setting to the keystore");
+        this.forceOption = parser.acceptsAll(Arrays.asList("f", "force"), "Overwrite existing setting without prompting");
+        // jopt simple has issue with multiple non options, so we just get one set of them here
+        // and convert to File when necessary
+        // see https://github.com/jopt-simple/jopt-simple/issues/103
+        this.arguments = parser.nonOptions("setting [filepath]");
+    }
+
+    @Override
+    protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
+        KeyStoreWrapper keystore = KeyStoreWrapper.load(env.configFile());
+        if (keystore == null) {
+            throw new UserException(ExitCodes.DATA_ERROR, "Elasticsearch keystore not found. Use 'create' command to create one.");
+        }
+
+        keystore.decrypt(new char[0] /* TODO: prompt for password when they are supported */);
+
+        List<String> argumentValues = arguments.values(options);
+        if (argumentValues.size() == 0) {
+            throw new UserException(ExitCodes.USAGE, "Missing setting name");
+        }
+        String setting = argumentValues.get(0);
+        if (keystore.getSettingNames().contains(setting) && options.has(forceOption) == false) {
+            if (terminal.promptYesNo("Setting " + setting + " already exists. Overwrite?", false) == false) {
+                terminal.println("Exiting without modifying keystore.");
+                return;
+            }
+        }
+
+        if (argumentValues.size() == 1) {
+            throw new UserException(ExitCodes.USAGE, "Missing file name");
+        }
+        Path file = getPath(argumentValues.get(1));
+        if (Files.exists(file) == false) {
+            throw new UserException(ExitCodes.IO_ERROR, "File [" + file.toString() + "] does not exist");
+        }
+        if (argumentValues.size() > 2) {
+            throw new UserException(ExitCodes.USAGE, "Unrecognized extra arguments [" +
+                String.join(", ", argumentValues.subList(2, argumentValues.size())) + "] after filepath");
+        }
+        keystore.setFile(setting, Files.readAllBytes(file));
+        keystore.save(env.configFile());
+    }
+
+    @SuppressForbidden(reason="file arg for cli")
+    private Path getPath(String file) {
+        return PathUtils.get(file);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/common/settings/AddStringKeyStoreCommand.java b/core/src/main/java/org/elasticsearch/common/settings/AddStringKeyStoreCommand.java
new file mode 100644
index 0000000000000..599fac8c376f0
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/settings/AddStringKeyStoreCommand.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+import org.elasticsearch.cli.EnvironmentAwareCommand;
+import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cli.UserException;
+import org.elasticsearch.env.Environment;
+
+/**
+ * A subcommand for the keystore cli which adds a string setting.
+ */
+class AddStringKeyStoreCommand extends EnvironmentAwareCommand {
+
+    private final OptionSpec<Void> stdinOption;
+    private final OptionSpec<Void> forceOption;
+    private final OptionSpec<String> arguments;
+
+    AddStringKeyStoreCommand() {
+        super("Add a string setting to the keystore");
+        this.stdinOption = parser.acceptsAll(Arrays.asList("x", "stdin"), "Read setting value from stdin");
+        this.forceOption = parser.acceptsAll(Arrays.asList("f", "force"), "Overwrite existing setting without prompting");
+        this.arguments = parser.nonOptions("setting name");
+    }
+
+    // pkg private so tests can manipulate
+    InputStream getStdin() {
+        return System.in;
+    }
+
+    @Override
+    protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
+        KeyStoreWrapper keystore = KeyStoreWrapper.load(env.configFile());
+        if (keystore == null) {
+            throw new UserException(ExitCodes.DATA_ERROR, "Elasticsearch keystore not found. Use 'create' command to create one.");
+        }
+
+        keystore.decrypt(new char[0] /* TODO: prompt for password when they are supported */);
+
+        String setting = arguments.value(options);
+        if (setting == null) {
+            throw new UserException(ExitCodes.USAGE, "The setting name can not be null");
+        }
+        if (keystore.getSettingNames().contains(setting) && options.has(forceOption) == false) {
+            if (terminal.promptYesNo("Setting " + setting + " already exists. Overwrite?", false) == false) {
+                terminal.println("Exiting without modifying keystore.");
+                return;
+            }
+        }
+
+        final char[] value;
+        if (options.has(stdinOption)) {
+            BufferedReader stdinReader = new BufferedReader(new InputStreamReader(getStdin(), StandardCharsets.UTF_8));
+            value = stdinReader.readLine().toCharArray();
+        } else {
+            value = terminal.readSecret("Enter value for " + setting + ": ");
+        }
+
+        try {
+            keystore.setString(setting, value);
+        } catch (IllegalArgumentException e) {
+            throw new UserException(ExitCodes.DATA_ERROR, "String value must contain only ASCII");
+        }
+        keystore.save(env.configFile());
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
index 1ce156b853682..7c5f236b636e3 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
@@ -19,18 +19,22 @@
 package org.elasticsearch.common.settings;
 
 import org.elasticsearch.action.admin.indices.close.TransportCloseIndexAction;
+import org.elasticsearch.script.ScriptModes;
+import org.elasticsearch.transport.RemoteClusterService;
+import org.elasticsearch.transport.RemoteClusterAware;
 import org.elasticsearch.action.search.TransportSearchAction;
 import org.elasticsearch.action.support.AutoCreateIndex;
 import org.elasticsearch.action.support.DestructiveOperations;
 import org.elasticsearch.action.support.master.TransportMasterNodeReadAction;
 import org.elasticsearch.bootstrap.BootstrapSettings;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.client.transport.TransportClientNodesService;
+import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.InternalClusterInfoService;
 import org.elasticsearch.cluster.NodeConnectionsService;
 import org.elasticsearch.cluster.action.index.MappingUpdatedAction;
+import org.elasticsearch.cluster.metadata.IndexGraveyard;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings;
 import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
@@ -54,10 +58,10 @@
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.discovery.DiscoveryModule;
 import org.elasticsearch.discovery.DiscoverySettings;
+import org.elasticsearch.discovery.zen.ElectMasterService;
+import org.elasticsearch.discovery.zen.FaultDetection;
+import org.elasticsearch.discovery.zen.UnicastZenPing;
 import org.elasticsearch.discovery.zen.ZenDiscovery;
-import org.elasticsearch.discovery.zen.elect.ElectMasterService;
-import org.elasticsearch.discovery.zen.fd.FaultDetection;
-import org.elasticsearch.discovery.zen.ping.unicast.UnicastZenPing;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.gateway.GatewayService;
@@ -75,6 +79,7 @@
 import org.elasticsearch.indices.recovery.RecoverySettings;
 import org.elasticsearch.indices.store.IndicesStore;
 import org.elasticsearch.indices.ttl.IndicesTTLService;
+import org.elasticsearch.ingest.IngestService;
 import org.elasticsearch.monitor.fs.FsService;
 import org.elasticsearch.monitor.jvm.JvmGcMonitorService;
 import org.elasticsearch.monitor.jvm.JvmService;
@@ -88,6 +93,7 @@
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.search.SearchService;
+import org.elasticsearch.search.fetch.subphase.highlight.FastVectorHighlighter;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TcpTransport;
 import org.elasticsearch.transport.Transport;
@@ -164,11 +170,11 @@ public void apply(Settings value, Settings current, Settings previous) {
 
     public static Set<Setting<?>> BUILT_IN_CLUSTER_SETTINGS = Collections.unmodifiableSet(new HashSet<>(
             Arrays.asList(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING,
-                    TransportClientNodesService.CLIENT_TRANSPORT_NODES_SAMPLER_INTERVAL, // TODO these transport client settings are kind
+                    TransportClient.CLIENT_TRANSPORT_NODES_SAMPLER_INTERVAL, // TODO these transport client settings are kind
                     // of odd here and should only be valid if we are a transport client
-                    TransportClientNodesService.CLIENT_TRANSPORT_PING_TIMEOUT,
-                    TransportClientNodesService.CLIENT_TRANSPORT_IGNORE_CLUSTER_NAME,
-                    TransportClientNodesService.CLIENT_TRANSPORT_SNIFF,
+                    TransportClient.CLIENT_TRANSPORT_PING_TIMEOUT,
+                    TransportClient.CLIENT_TRANSPORT_IGNORE_CLUSTER_NAME,
+                    TransportClient.CLIENT_TRANSPORT_SNIFF,
                     AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING,
                     BalancedShardsAllocator.INDEX_BALANCE_FACTOR_SETTING,
                     BalancedShardsAllocator.SHARD_BALANCE_FACTOR_SETTING,
@@ -191,6 +197,7 @@ public void apply(Settings value, Settings current, Settings previous) {
                     IndicesTTLService.INDICES_TTL_INTERVAL_SETTING,
                     MappingUpdatedAction.INDICES_MAPPING_DYNAMIC_TIMEOUT_SETTING,
                     MetaData.SETTING_READ_ONLY_SETTING,
+                    MetaData.SETTING_READ_ONLY_ALLOW_DELETE_SETTING,
                     RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING,
                     RecoverySettings.INDICES_RECOVERY_RETRY_DELAY_STATE_SYNC_SETTING,
                     RecoverySettings.INDICES_RECOVERY_RETRY_DELAY_NETWORK_SETTING,
@@ -226,7 +233,6 @@ public void apply(Settings value, Settings current, Settings previous) {
                     NetworkModule.HTTP_DEFAULT_TYPE_SETTING,
                     NetworkModule.TRANSPORT_DEFAULT_TYPE_SETTING,
                     NetworkModule.HTTP_TYPE_SETTING,
-                    NetworkModule.TRANSPORT_SERVICE_TYPE_SETTING,
                     NetworkModule.TRANSPORT_TYPE_SETTING,
                     HttpTransportSettings.SETTING_CORS_ALLOW_CREDENTIALS,
                     HttpTransportSettings.SETTING_CORS_ENABLED,
@@ -245,6 +251,7 @@ public void apply(Settings value, Settings current, Settings previous) {
                     HttpTransportSettings.SETTING_CORS_ALLOW_METHODS,
                     HttpTransportSettings.SETTING_CORS_ALLOW_HEADERS,
                     HttpTransportSettings.SETTING_HTTP_DETAILED_ERRORS_ENABLED,
+                    HttpTransportSettings.SETTING_HTTP_CONTENT_TYPE_REQUIRED,
                     HttpTransportSettings.SETTING_HTTP_MAX_CONTENT_LENGTH,
                     HttpTransportSettings.SETTING_HTTP_MAX_CHUNK_SIZE,
                     HttpTransportSettings.SETTING_HTTP_MAX_HEADER_SIZE,
@@ -261,6 +268,11 @@ public void apply(Settings value, Settings current, Settings previous) {
                     SearchService.DEFAULT_SEARCH_TIMEOUT_SETTING,
                     ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING,
                     TransportSearchAction.SHARD_COUNT_LIMIT_SETTING,
+                    RemoteClusterAware.REMOTE_CLUSTERS_SEEDS,
+                    RemoteClusterService.REMOTE_CONNECTIONS_PER_CLUSTER,
+                    RemoteClusterService.REMOTE_INITIAL_CONNECTION_TIMEOUT_SETTING,
+                    RemoteClusterService.REMOTE_NODE_ATTRIBUTE,
+                    RemoteClusterService.ENABLE_REMOTE_CLUSTERS,
                     TransportService.TRACE_LOG_EXCLUDE_SETTING,
                     TransportService.TRACE_LOG_INCLUDE_SETTING,
                     TransportCloseIndexAction.CLUSTER_INDICES_CLOSE_ENABLE_SETTING,
@@ -305,6 +317,8 @@ public void apply(Settings value, Settings current, Settings previous) {
                     IndexSettings.QUERY_STRING_ANALYZE_WILDCARD,
                     IndexSettings.QUERY_STRING_ALLOW_LEADING_WILDCARD,
                     PrimaryShardAllocator.NODE_INITIAL_SHARDS_SETTING,
+                    ScriptModes.TYPES_ALLOWED_SETTING,
+                    ScriptModes.CONTEXTS_ALLOWED_SETTING,
                     ScriptService.SCRIPT_CACHE_SIZE_SETTING,
                     ScriptService.SCRIPT_CACHE_EXPIRE_SETTING,
                     ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING,
@@ -318,9 +332,12 @@ public void apply(Settings value, Settings current, Settings previous) {
                     HunspellService.HUNSPELL_IGNORE_CASE,
                     HunspellService.HUNSPELL_DICTIONARY_OPTIONS,
                     IndicesStore.INDICES_STORE_DELETE_SHARD_TIMEOUT,
+                    Environment.DEFAULT_PATH_CONF_SETTING,
                     Environment.PATH_CONF_SETTING,
+                    Environment.DEFAULT_PATH_DATA_SETTING,
                     Environment.PATH_DATA_SETTING,
                     Environment.PATH_HOME_SETTING,
+                    Environment.DEFAULT_PATH_LOGS_SETTING,
                     Environment.PATH_LOGS_SETTING,
                     Environment.PATH_REPO_SETTING,
                     Environment.PATH_SCRIPTS_SETTING,
@@ -329,7 +346,7 @@ public void apply(Settings value, Settings current, Settings previous) {
                     NodeEnvironment.NODE_ID_SEED_SETTING,
                     DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING,
                     DiscoveryModule.DISCOVERY_TYPE_SETTING,
-                    DiscoveryModule.ZEN_MASTER_SERVICE_TYPE_SETTING,
+                    DiscoveryModule.DISCOVERY_HOSTS_PROVIDER_SETTING,
                     FaultDetection.PING_RETRIES_SETTING,
                     FaultDetection.PING_TIMEOUT_SETTING,
                     FaultDetection.REGISTER_CONNECTION_LISTENER_SETTING,
@@ -345,9 +362,11 @@ public void apply(Settings value, Settings current, Settings previous) {
                     ZenDiscovery.MASTER_ELECTION_IGNORE_NON_MASTER_PINGS_SETTING,
                     UnicastZenPing.DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING,
                     UnicastZenPing.DISCOVERY_ZEN_PING_UNICAST_CONCURRENT_CONNECTS_SETTING,
+                    UnicastZenPing.DISCOVERY_ZEN_PING_UNICAST_HOSTS_RESOLVE_TIMEOUT,
                     SearchService.DEFAULT_KEEPALIVE_SETTING,
                     SearchService.KEEPALIVE_INTERVAL_SETTING,
-                    Node.WRITE_PORTS_FIELD_SETTING,
+                    SearchService.LOW_LEVEL_CANCELLATION_SETTING,
+                    Node.WRITE_PORTS_FILE_SETTING,
                     Node.NODE_NAME_SETTING,
                     Node.NODE_DATA_SETTING,
                     Node.NODE_MASTER_SETTING,
@@ -396,9 +415,10 @@ public void apply(Settings value, Settings current, Settings previous) {
                     PluginsService.MANDATORY_SETTING,
                     BootstrapSettings.SECURITY_FILTER_BAD_DEFAULTS_SETTING,
                     BootstrapSettings.MEMORY_LOCK_SETTING,
+                    BootstrapSettings.SYSTEM_CALL_FILTER_SETTING,
+                    // TODO: remove in 6.0.0
                     BootstrapSettings.SECCOMP_SETTING,
                     BootstrapSettings.CTRLHANDLER_SETTING,
-                    BootstrapSettings.IGNORE_SYSTEM_BOOTSTRAP_CHECKS,
                     IndexingMemoryController.INDEX_BUFFER_SIZE_SETTING,
                     IndexingMemoryController.MIN_INDEX_BUFFER_SIZE_SETTING,
                     IndexingMemoryController.MAX_INDEX_BUFFER_SIZE_SETTING,
@@ -410,6 +430,9 @@ public void apply(Settings value, Settings current, Settings previous) {
                     ResourceWatcherService.RELOAD_INTERVAL_LOW,
                     SearchModule.INDICES_MAX_CLAUSE_COUNT_SETTING,
                     ThreadPool.ESTIMATED_TIME_INTERVAL_SETTING,
-                    Node.BREAKER_TYPE_KEY
+                    FastVectorHighlighter.SETTING_TV_HIGHLIGHT_MULTI_VALUE,
+                    Node.BREAKER_TYPE_KEY,
+                    IngestService.NEW_INGEST_DATE_FORMAT,
+                    IndexGraveyard.SETTING_MAX_TOMBSTONES
             )));
 }
diff --git a/core/src/main/java/org/elasticsearch/common/settings/CreateKeyStoreCommand.java b/core/src/main/java/org/elasticsearch/common/settings/CreateKeyStoreCommand.java
new file mode 100644
index 0000000000000..08860cb5ea992
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/settings/CreateKeyStoreCommand.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import joptsimple.OptionSet;
+import org.elasticsearch.cli.EnvironmentAwareCommand;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.env.Environment;
+
+/**
+ * A subcommand for the keystore cli to create a new keystore.
+ */
+class CreateKeyStoreCommand extends EnvironmentAwareCommand {
+
+    CreateKeyStoreCommand() {
+        super("Creates a new elasticsearch keystore");
+    }
+
+    @Override
+    protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
+        Path keystoreFile = KeyStoreWrapper.keystorePath(env.configFile());
+        if (Files.exists(keystoreFile)) {
+            if (terminal.promptYesNo("An elasticsearch keystore already exists. Overwrite?", false) == false) {
+                terminal.println("Exiting without creating keystore.");
+                return;
+            }
+        }
+
+
+        char[] password = new char[0];// terminal.readSecret("Enter passphrase (empty for no passphrase): ");
+        /* TODO: uncomment when entering passwords on startup is supported
+        char[] passwordRepeat = terminal.readSecret("Enter same passphrase again: ");
+        if (Arrays.equals(password, passwordRepeat) == false) {
+            throw new UserException(ExitCodes.DATA_ERROR, "Passphrases are not equal, exiting.");
+        }*/
+
+        KeyStoreWrapper keystore = KeyStoreWrapper.create(password);
+        keystore.save(env.configFile());
+        terminal.println("Created elasticsearch keystore in " + env.configFile());
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
index fc0d5f4df9fbc..8762d4cb8e8c2 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
@@ -71,6 +71,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
         IndexMetaData.INDEX_AUTO_EXPAND_REPLICAS_SETTING,
         IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING,
         IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING,
+        IndexMetaData.INDEX_ROUTING_PARTITION_SIZE_SETTING,
         IndexMetaData.INDEX_SHADOW_REPLICAS_SETTING,
         IndexMetaData.INDEX_SHARED_FILESYSTEM_SETTING,
         IndexMetaData.INDEX_READ_ONLY_SETTING,
@@ -78,8 +79,10 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
         IndexMetaData.INDEX_BLOCKS_WRITE_SETTING,
         IndexMetaData.INDEX_BLOCKS_METADATA_SETTING,
         IndexMetaData.INDEX_SHARED_FS_ALLOW_RECOVERY_ON_ANY_NODE_SETTING,
+        IndexMetaData.INDEX_BLOCKS_READ_ONLY_ALLOW_DELETE_SETTING,
         IndexMetaData.INDEX_PRIORITY_SETTING,
         IndexMetaData.INDEX_DATA_PATH_SETTING,
+        IndexMetaData.INDEX_FORMAT_SETTING,
         SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_DEBUG_SETTING,
         SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_WARN_SETTING,
         SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_INFO_SETTING,
@@ -89,7 +92,6 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
         SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_INFO_SETTING,
         SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_TRACE_SETTING,
         SearchSlowLog.INDEX_SEARCH_SLOWLOG_LEVEL,
-        SearchSlowLog.INDEX_SEARCH_SLOWLOG_REFORMAT,
         IndexingSlowLog.INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_WARN_SETTING,
         IndexingSlowLog.INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_DEBUG_SETTING,
         IndexingSlowLog.INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_INFO_SETTING,
@@ -110,6 +112,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
         IndexSettings.INDEX_REFRESH_INTERVAL_SETTING,
         IndexSettings.MAX_RESULT_WINDOW_SETTING,
         IndexSettings.MAX_RESCORE_WINDOW_SETTING,
+        IndexSettings.MAX_ADJACENCY_MATRIX_FILTERS_SETTING,
         IndexSettings.INDEX_TRANSLOG_SYNC_INTERVAL_SETTING,
         IndexSettings.DEFAULT_FIELD_SETTING,
         IndexSettings.QUERY_STRING_LENIENT_SETTING,
@@ -132,11 +135,13 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
         MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING,
         MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING,
         MapperService.INDEX_MAPPING_DEPTH_LIMIT_SETTING,
+        MapperService.INDEX_MAPPING_SINGLE_TYPE_SETTING,
         BitsetFilterCache.INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING,
         IndexModule.INDEX_STORE_TYPE_SETTING,
         IndexModule.INDEX_STORE_PRE_LOAD_SETTING,
         IndexModule.INDEX_QUERY_CACHE_ENABLED_SETTING,
         IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING,
+        IndexModule.INDEX_QUERY_CACHE_TERM_QUERIES_SETTING,
         PrimaryShardAllocator.INDEX_RECOVERY_INITIAL_SHARDS_SETTING,
         FsDirectoryService.INDEX_LOCK_FACTOR_SETTING,
         EngineConfig.INDEX_CODEC_SETTING,
@@ -186,10 +191,13 @@ protected boolean isPrivateSetting(String key) {
             case IndexMetaData.SETTING_INDEX_UUID:
             case IndexMetaData.SETTING_VERSION_CREATED:
             case IndexMetaData.SETTING_VERSION_UPGRADED:
+            case IndexMetaData.SETTING_INDEX_PROVIDED_NAME:
             case MergePolicyConfig.INDEX_MERGE_ENABLED:
+            case IndexMetaData.INDEX_SHRINK_SOURCE_UUID_KEY:
+            case IndexMetaData.INDEX_SHRINK_SOURCE_NAME_KEY:
                 return true;
             default:
-                return false;
+                return IndexMetaData.INDEX_ROUTING_INITIAL_RECOVERY_GROUP_SETTING.getRawKey().match(key);
         }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/settings/KeyStoreCli.java b/core/src/main/java/org/elasticsearch/common/settings/KeyStoreCli.java
new file mode 100644
index 0000000000000..16818341cbd0c
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/settings/KeyStoreCli.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import org.elasticsearch.cli.MultiCommand;
+import org.elasticsearch.cli.Terminal;
+
+/**
+ * A cli tool for managing secrets in the elasticsearch keystore.
+ */
+public class KeyStoreCli extends MultiCommand {
+
+    private KeyStoreCli() {
+        super("A tool for managing settings stored in the elasticsearch keystore");
+        subcommands.put("create", new CreateKeyStoreCommand());
+        subcommands.put("list", new ListKeyStoreCommand());
+        subcommands.put("add", new AddStringKeyStoreCommand());
+        subcommands.put("add-file", new AddFileKeyStoreCommand());
+        subcommands.put("remove", new RemoveSettingKeyStoreCommand());
+    }
+
+    public static void main(String[] args) throws Exception {
+        exit(new KeyStoreCli().main(args, Terminal.DEFAULT));
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java b/core/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java
new file mode 100644
index 0000000000000..a96f1cf841951
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java
@@ -0,0 +1,385 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import javax.crypto.SecretKey;
+import javax.crypto.SecretKeyFactory;
+import javax.crypto.spec.PBEKeySpec;
+import javax.security.auth.DestroyFailedException;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.CharBuffer;
+import java.nio.charset.CharsetEncoder;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.nio.file.attribute.PosixFileAttributeView;
+import java.nio.file.attribute.PosixFilePermissions;
+import java.security.GeneralSecurityException;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.store.BufferedChecksumIndexInput;
+import org.apache.lucene.store.ChecksumIndexInput;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.SimpleFSDirectory;
+import org.apache.lucene.util.SetOnce;
+
+/**
+ * A wrapper around a Java KeyStore which provides supplements the keystore with extra metadata.
+ *
+ * Loading a keystore has 2 phases. First, call {@link #load(Path)}. Then call
+ * {@link #decrypt(char[])} with the keystore password, or an empty char array if
+ * {@link #hasPassword()} is {@code false}.  Loading and decrypting should happen
+ * in a single thread. Once decrypted, keys may be read with the wrapper in
+ * multiple threads.
+ */
+public class KeyStoreWrapper implements SecureSettings {
+
+    /** An identifier for the type of data that may be stored in a keystore entry. */
+    private enum KeyType {
+        STRING,
+        FILE
+    }
+
+    /** The name of the keystore file to read and write. */
+    private static final String KEYSTORE_FILENAME = "elasticsearch.keystore";
+
+    /** The version of the metadata written before the keystore data. */
+    private static final int FORMAT_VERSION = 2;
+
+    /** The oldest metadata format version that can be read. */
+    private static final int MIN_FORMAT_VERSION = 1;
+
+    /** The keystore type for a newly created keystore. */
+    private static final String NEW_KEYSTORE_TYPE = "PKCS12";
+
+    /** The algorithm used to store string setting contents. */
+    private static final String NEW_KEYSTORE_STRING_KEY_ALGO = "PBE";
+
+    /** The algorithm used to store file setting contents. */
+    private static final String NEW_KEYSTORE_FILE_KEY_ALGO = "PBE";
+
+    /** An encoder to check whether string values are ascii. */
+    private static final CharsetEncoder ASCII_ENCODER = StandardCharsets.US_ASCII.newEncoder();
+
+    /** The metadata format version used to read the current keystore wrapper. */
+    private final int formatVersion;
+
+    /** True iff the keystore has a password needed to read. */
+    private final boolean hasPassword;
+
+    /** The type of the keystore, as passed to {@link java.security.KeyStore#getInstance(String)} */
+    private final String type;
+
+    /** A factory necessary for constructing instances of string secrets in a {@link KeyStore}. */
+    private final SecretKeyFactory stringFactory;
+
+    /** A factory necessary for constructing instances of file secrets in a {@link KeyStore}. */
+    private final SecretKeyFactory fileFactory;
+
+    /**
+     * The settings that exist in the keystore, mapped to their type of data.
+     */
+    private final Map<String, KeyType> settingTypes;
+
+    /** The raw bytes of the encrypted keystore. */
+    private final byte[] keystoreBytes;
+
+    /** The loaded keystore. See {@link #decrypt(char[])}. */
+    private final SetOnce<KeyStore> keystore = new SetOnce<>();
+
+    /** The password for the keystore. See {@link #decrypt(char[])}. */
+    private final SetOnce<KeyStore.PasswordProtection> keystorePassword = new SetOnce<>();
+
+    private KeyStoreWrapper(int formatVersion, boolean hasPassword, String type,
+                            String stringKeyAlgo, String fileKeyAlgo,
+                            Map<String, KeyType> settingTypes, byte[] keystoreBytes) {
+        this.formatVersion = formatVersion;
+        this.hasPassword = hasPassword;
+        this.type = type;
+        try {
+            stringFactory = SecretKeyFactory.getInstance(stringKeyAlgo);
+            fileFactory = SecretKeyFactory.getInstance(fileKeyAlgo);
+        } catch (NoSuchAlgorithmException e) {
+            throw new RuntimeException(e);
+        }
+        this.settingTypes = settingTypes;
+        this.keystoreBytes = keystoreBytes;
+    }
+
+    /** Returns a path representing the ES keystore in the given config dir. */
+    static Path keystorePath(Path configDir) {
+        return configDir.resolve(KEYSTORE_FILENAME);
+    }
+
+    /** Constructs a new keystore with the given password. */
+    static KeyStoreWrapper create(char[] password) throws Exception {
+        KeyStoreWrapper wrapper = new KeyStoreWrapper(FORMAT_VERSION, password.length != 0, NEW_KEYSTORE_TYPE,
+            NEW_KEYSTORE_STRING_KEY_ALGO, NEW_KEYSTORE_FILE_KEY_ALGO, new HashMap<>(), null);
+        KeyStore keyStore = KeyStore.getInstance(NEW_KEYSTORE_TYPE);
+        keyStore.load(null, null);
+        wrapper.keystore.set(keyStore);
+        wrapper.keystorePassword.set(new KeyStore.PasswordProtection(password));
+        return wrapper;
+    }
+
+    /**
+     * Loads information about the Elasticsearch keystore from the provided config directory.
+     *
+     * {@link #decrypt(char[])} must be called before reading or writing any entries.
+     * Returns {@code null} if no keystore exists.
+     */
+    public static KeyStoreWrapper load(Path configDir) throws IOException {
+        Path keystoreFile = keystorePath(configDir);
+        if (Files.exists(keystoreFile) == false) {
+            return null;
+        }
+
+        SimpleFSDirectory directory = new SimpleFSDirectory(configDir);
+        try (IndexInput indexInput = directory.openInput(KEYSTORE_FILENAME, IOContext.READONCE)) {
+            ChecksumIndexInput input = new BufferedChecksumIndexInput(indexInput);
+            int formatVersion = CodecUtil.checkHeader(input, KEYSTORE_FILENAME, MIN_FORMAT_VERSION, FORMAT_VERSION);
+            byte hasPasswordByte = input.readByte();
+            boolean hasPassword = hasPasswordByte == 1;
+            if (hasPassword == false && hasPasswordByte != 0) {
+                throw new IllegalStateException("hasPassword boolean is corrupt: "
+                    + String.format(Locale.ROOT, "%02x", hasPasswordByte));
+            }
+            String type = input.readString();
+            String stringKeyAlgo = input.readString();
+            final String fileKeyAlgo;
+            if (formatVersion >= 2) {
+                fileKeyAlgo = input.readString();
+            } else {
+                fileKeyAlgo = NEW_KEYSTORE_FILE_KEY_ALGO;
+            }
+            final Map<String, KeyType> settingTypes;
+            if (formatVersion >= 2) {
+                settingTypes = input.readMapOfStrings().entrySet().stream().collect(Collectors.toMap(
+                    Map.Entry::getKey,
+                    e -> KeyType.valueOf(e.getValue())));
+            } else {
+                settingTypes = new HashMap<>();
+            }
+            byte[] keystoreBytes = new byte[input.readInt()];
+            input.readBytes(keystoreBytes, 0, keystoreBytes.length);
+            CodecUtil.checkFooter(input);
+            return new KeyStoreWrapper(formatVersion, hasPassword, type, stringKeyAlgo, fileKeyAlgo, settingTypes, keystoreBytes);
+        }
+    }
+
+    @Override
+    public boolean isLoaded() {
+        return keystore.get() != null;
+    }
+
+    /** Return true iff calling {@link #decrypt(char[])} requires a non-empty password. */
+    public boolean hasPassword() {
+        return hasPassword;
+    }
+
+    /**
+     * Decrypts the underlying java keystore.
+     *
+     * This may only be called once. The provided password will be zeroed out.
+     */
+    public void decrypt(char[] password) throws GeneralSecurityException, IOException {
+        if (keystore.get() != null) {
+            throw new IllegalStateException("Keystore has already been decrypted");
+        }
+        keystore.set(KeyStore.getInstance(type));
+        try (InputStream in = new ByteArrayInputStream(keystoreBytes)) {
+            keystore.get().load(in, password);
+        } finally {
+            Arrays.fill(keystoreBytes, (byte)0);
+        }
+
+        keystorePassword.set(new KeyStore.PasswordProtection(password));
+        Arrays.fill(password, '\0');
+
+
+        Enumeration<String> aliases = keystore.get().aliases();
+        if (formatVersion == 1) {
+            while (aliases.hasMoreElements()) {
+                settingTypes.put(aliases.nextElement(), KeyType.STRING);
+            }
+        } else {
+            // verify integrity: keys in keystore match what the metadata thinks exist
+            Set<String> expectedSettings = new HashSet<>(settingTypes.keySet());
+            while (aliases.hasMoreElements()) {
+                String settingName = aliases.nextElement();
+                if (expectedSettings.remove(settingName) == false) {
+                    throw new SecurityException("Keystore has been corrupted or tampered with");
+                }
+            }
+            if (expectedSettings.isEmpty() == false) {
+                throw new SecurityException("Keystore has been corrupted or tampered with");
+            }
+        }
+    }
+
+    /** Write the keystore to the given config directory. */
+    void save(Path configDir) throws Exception {
+        char[] password = this.keystorePassword.get().getPassword();
+
+        SimpleFSDirectory directory = new SimpleFSDirectory(configDir);
+        // write to tmp file first, then overwrite
+        String tmpFile = KEYSTORE_FILENAME + ".tmp";
+        try (IndexOutput output = directory.createOutput(tmpFile, IOContext.DEFAULT)) {
+            CodecUtil.writeHeader(output, KEYSTORE_FILENAME, FORMAT_VERSION);
+            output.writeByte(password.length == 0 ? (byte)0 : (byte)1);
+            output.writeString(NEW_KEYSTORE_TYPE);
+            output.writeString(NEW_KEYSTORE_STRING_KEY_ALGO);
+            output.writeString(NEW_KEYSTORE_FILE_KEY_ALGO);
+            output.writeMapOfStrings(settingTypes.entrySet().stream().collect(Collectors.toMap(
+                Map.Entry::getKey,
+                e -> e.getValue().name())));
+
+            // TODO: in the future if we ever change any algorithms used above, we need
+            // to create a new KeyStore here instead of using the existing one, so that
+            // the encoded material inside the keystore is updated
+            assert type.equals(NEW_KEYSTORE_TYPE) : "keystore type changed";
+            assert stringFactory.getAlgorithm().equals(NEW_KEYSTORE_STRING_KEY_ALGO) : "string pbe algo changed";
+            assert fileFactory.getAlgorithm().equals(NEW_KEYSTORE_FILE_KEY_ALGO) : "file pbe algo changed";
+
+            ByteArrayOutputStream keystoreBytesStream = new ByteArrayOutputStream();
+            keystore.get().store(keystoreBytesStream, password);
+            byte[] keystoreBytes = keystoreBytesStream.toByteArray();
+            output.writeInt(keystoreBytes.length);
+            output.writeBytes(keystoreBytes, keystoreBytes.length);
+            CodecUtil.writeFooter(output);
+        }
+
+        Path keystoreFile = keystorePath(configDir);
+        Files.move(configDir.resolve(tmpFile), keystoreFile, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
+        PosixFileAttributeView attrs = Files.getFileAttributeView(keystoreFile, PosixFileAttributeView.class);
+        if (attrs != null) {
+            // don't rely on umask: ensure the keystore has minimal permissions
+            attrs.setPermissions(PosixFilePermissions.fromString("rw-rw----"));
+        }
+    }
+
+    @Override
+    public Set<String> getSettingNames() {
+        return settingTypes.keySet();
+    }
+
+    // TODO: make settings accessible only to code that registered the setting
+    @Override
+    public SecureString getString(String setting) throws GeneralSecurityException {
+        KeyStore.Entry entry = keystore.get().getEntry(setting, keystorePassword.get());
+        if (settingTypes.get(setting) != KeyType.STRING ||
+            entry instanceof KeyStore.SecretKeyEntry == false) {
+            throw new IllegalStateException("Secret setting " + setting + " is not a string");
+        }
+        // TODO: only allow getting a setting once?
+        KeyStore.SecretKeyEntry secretKeyEntry = (KeyStore.SecretKeyEntry) entry;
+        PBEKeySpec keySpec = (PBEKeySpec) stringFactory.getKeySpec(secretKeyEntry.getSecretKey(), PBEKeySpec.class);
+        SecureString value = new SecureString(keySpec.getPassword());
+        keySpec.clearPassword();
+        return value;
+    }
+
+    @Override
+    public InputStream getFile(String setting) throws GeneralSecurityException {
+        KeyStore.Entry entry = keystore.get().getEntry(setting, keystorePassword.get());
+        if (settingTypes.get(setting) != KeyType.FILE ||
+            entry instanceof KeyStore.SecretKeyEntry == false) {
+            throw new IllegalStateException("Secret setting " + setting + " is not a file");
+        }
+        KeyStore.SecretKeyEntry secretKeyEntry = (KeyStore.SecretKeyEntry) entry;
+        PBEKeySpec keySpec = (PBEKeySpec) fileFactory.getKeySpec(secretKeyEntry.getSecretKey(), PBEKeySpec.class);
+        // The PBE keyspec gives us chars, we first convert to bytes, then decode base64 inline.
+        char[] chars = keySpec.getPassword();
+        byte[] bytes = new byte[chars.length];
+        for (int i = 0; i < bytes.length; ++i) {
+            bytes[i] = (byte)chars[i]; // PBE only stores the lower 8 bits, so this narrowing is ok
+        }
+        keySpec.clearPassword(); // wipe the original copy
+        InputStream bytesStream = new ByteArrayInputStream(bytes) {
+            @Override
+            public void close() throws IOException {
+                super.close();
+                Arrays.fill(bytes, (byte)0); // wipe our second copy when the stream is exhausted
+            }
+        };
+        return Base64.getDecoder().wrap(bytesStream);
+    }
+
+    /**
+     * Set a string setting.
+     *
+     * @throws IllegalArgumentException if the value is not ASCII
+     */
+    void setString(String setting, char[] value) throws GeneralSecurityException {
+        if (ASCII_ENCODER.canEncode(CharBuffer.wrap(value)) == false) {
+            throw new IllegalArgumentException("Value must be ascii");
+        }
+        SecretKey secretKey = stringFactory.generateSecret(new PBEKeySpec(value));
+        keystore.get().setEntry(setting, new KeyStore.SecretKeyEntry(secretKey), keystorePassword.get());
+        settingTypes.put(setting, KeyType.STRING);
+    }
+
+    /** Set a file setting. */
+    void setFile(String setting, byte[] bytes) throws GeneralSecurityException {
+        bytes = Base64.getEncoder().encode(bytes);
+        char[] chars = new char[bytes.length];
+        for (int i = 0; i < chars.length; ++i) {
+            chars[i] = (char)bytes[i]; // PBE only stores the lower 8 bits, so this narrowing is ok
+        }
+        SecretKey secretKey = stringFactory.generateSecret(new PBEKeySpec(chars));
+        keystore.get().setEntry(setting, new KeyStore.SecretKeyEntry(secretKey), keystorePassword.get());
+        settingTypes.put(setting, KeyType.FILE);
+    }
+
+    /** Remove the given setting from the keystore. */
+    void remove(String setting) throws KeyStoreException {
+        keystore.get().deleteEntry(setting);
+        settingTypes.remove(setting);
+    }
+
+    @Override
+    public void close() throws IOException {
+        try {
+            if (keystorePassword.get() != null) {
+                keystorePassword.get().destroy();
+            }
+        } catch (DestroyFailedException e) {
+            throw new IOException(e);
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/common/settings/ListKeyStoreCommand.java b/core/src/main/java/org/elasticsearch/common/settings/ListKeyStoreCommand.java
new file mode 100644
index 0000000000000..8eef02f213189
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/settings/ListKeyStoreCommand.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import joptsimple.OptionSet;
+import org.elasticsearch.cli.EnvironmentAwareCommand;
+import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cli.UserException;
+import org.elasticsearch.env.Environment;
+
+/**
+ * A subcommand for the keystore cli to list all settings in the keystore.
+ */
+class ListKeyStoreCommand extends EnvironmentAwareCommand {
+
+    ListKeyStoreCommand() {
+        super("List entries in the keystore");
+    }
+
+    @Override
+    protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
+        KeyStoreWrapper keystore = KeyStoreWrapper.load(env.configFile());
+        if (keystore == null) {
+            throw new UserException(ExitCodes.DATA_ERROR, "Elasticsearch keystore not found. Use 'create' command to create one.");
+        }
+
+        keystore.decrypt(new char[0] /* TODO: prompt for password when they are supported */);
+
+        List<String> sortedEntries = new ArrayList<>(keystore.getSettingNames());
+        Collections.sort(sortedEntries);
+        for (String entry : sortedEntries) {
+            terminal.println(entry);
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/common/settings/RemoveSettingKeyStoreCommand.java b/core/src/main/java/org/elasticsearch/common/settings/RemoveSettingKeyStoreCommand.java
new file mode 100644
index 0000000000000..c766dd769e22b
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/settings/RemoveSettingKeyStoreCommand.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import java.util.List;
+
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+import org.elasticsearch.cli.EnvironmentAwareCommand;
+import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cli.UserException;
+import org.elasticsearch.env.Environment;
+
+/**
+ * A subcommand for the keystore cli to remove a setting.
+ */
+class RemoveSettingKeyStoreCommand extends EnvironmentAwareCommand {
+
+    private final OptionSpec<String> arguments;
+
+    RemoveSettingKeyStoreCommand() {
+        super("Remove a setting from the keystore");
+        arguments = parser.nonOptions("setting names");
+    }
+
+    @Override
+    protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
+        List<String> settings = arguments.values(options);
+        if (settings.isEmpty()) {
+            throw new UserException(ExitCodes.USAGE, "Must supply at least one setting to remove");
+        }
+
+        KeyStoreWrapper keystore = KeyStoreWrapper.load(env.configFile());
+        if (keystore == null) {
+            throw new UserException(ExitCodes.DATA_ERROR, "Elasticsearch keystore not found. Use 'create' command to create one.");
+        }
+
+        keystore.decrypt(new char[0] /* TODO: prompt for password when they are supported */);
+
+        for (String setting : arguments.values(options)) {
+            if (keystore.getSettingNames().contains(setting) == false) {
+                throw new UserException(ExitCodes.CONFIG, "Setting [" + setting + "] does not exist in the keystore.");
+            }
+            keystore.remove(setting);
+        }
+        keystore.save(env.configFile());
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/common/settings/SecureSetting.java b/core/src/main/java/org/elasticsearch/common/settings/SecureSetting.java
new file mode 100644
index 0000000000000..07ff3e03f4de8
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/settings/SecureSetting.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import java.io.InputStream;
+import java.security.GeneralSecurityException;
+import java.util.EnumSet;
+import java.util.Set;
+
+import org.elasticsearch.common.util.ArrayUtils;
+
+
+/**
+ * A secure setting.
+ *
+ * This class allows access to settings from the Elasticsearch keystore.
+ */
+public abstract class SecureSetting<T> extends Setting<T> {
+    private static final Set<Property> ALLOWED_PROPERTIES = EnumSet.of(Property.Deprecated, Property.Shared);
+
+    private static final Property[] FIXED_PROPERTIES = {
+        Property.NodeScope
+    };
+
+    private static final Property[] LEGACY_PROPERTIES = {
+        Property.NodeScope, Property.Deprecated, Property.Filtered
+    };
+
+    private SecureSetting(String key, Property... properties) {
+        super(key, (String)null, null, ArrayUtils.concat(properties, FIXED_PROPERTIES, Property.class));
+        assert assertAllowedProperties(properties);
+    }
+
+    private boolean assertAllowedProperties(Setting.Property... properties) {
+        for (Setting.Property property : properties) {
+            if (ALLOWED_PROPERTIES.contains(property) == false) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    @Override
+    public String getDefaultRaw(Settings settings) {
+        throw new UnsupportedOperationException("secure settings are not strings");
+    }
+
+    @Override
+    public T getDefault(Settings settings) {
+        throw new UnsupportedOperationException("secure settings are not strings");
+    }
+
+    @Override
+    public String getRaw(Settings settings) {
+        throw new UnsupportedOperationException("secure settings are not strings");
+    }
+
+    @Override
+    public boolean exists(Settings settings) {
+        final SecureSettings secureSettings = settings.getSecureSettings();
+        return secureSettings != null && secureSettings.getSettingNames().contains(getKey());
+    }
+
+    @Override
+    public T get(Settings settings) {
+        checkDeprecation(settings);
+        final SecureSettings secureSettings = settings.getSecureSettings();
+        if (secureSettings == null || secureSettings.getSettingNames().contains(getKey()) == false) {
+            if (super.exists(settings)) {
+                throw new IllegalArgumentException("Setting [" + getKey() + "] is a secure setting" +
+                    " and must be stored inside the Elasticsearch keystore, but was found inside elasticsearch.yml");
+            }
+            return getFallback(settings);
+        }
+        try {
+            return getSecret(secureSettings);
+        } catch (GeneralSecurityException e) {
+            throw new RuntimeException("failed to read secure setting " + getKey(), e);
+        }
+    }
+
+    /** Returns the secret setting from the keyStoreReader store. */
+    abstract T getSecret(SecureSettings secureSettings) throws GeneralSecurityException;
+
+    /** Returns the value from a fallback setting. Returns null if no fallback exists. */
+    abstract T getFallback(Settings settings);
+
+    // TODO: override toXContent
+
+    /**
+     * Overrides the diff operation to make this a no-op for secure settings as they shouldn't be returned in a diff
+     */
+    @Override
+    public void diff(Settings.Builder builder, Settings source, Settings defaultSettings) {
+    }
+
+    /**
+     * A setting which contains a sensitive string.
+     *
+     * This may be any sensitive string, e.g. a username, a password, an auth token, etc.
+     */
+    public static Setting<SecureString> secureString(String name, Setting<SecureString> fallback,
+                                                     Property... properties) {
+        return new SecureStringSetting(name, fallback, properties);
+    }
+
+    /**
+     * A setting which contains a file. Reading the setting opens an input stream to the file.
+     *
+     * This may be any sensitive file, e.g. a set of credentials normally in plaintext.
+     */
+    public static Setting<InputStream> secureFile(String name, Setting<InputStream> fallback,
+                                                  Property... properties) {
+        return new SecureFileSetting(name, fallback, properties);
+    }
+
+    private static class SecureStringSetting extends SecureSetting<SecureString> {
+        private final Setting<SecureString> fallback;
+
+        private SecureStringSetting(String name, Setting<SecureString> fallback, Property... properties) {
+            super(name, properties);
+            this.fallback = fallback;
+        }
+
+        @Override
+        protected SecureString getSecret(SecureSettings secureSettings) throws GeneralSecurityException {
+            return secureSettings.getString(getKey());
+        }
+
+        @Override
+        SecureString getFallback(Settings settings) {
+            if (fallback != null) {
+                return fallback.get(settings);
+            }
+            return new SecureString(new char[0]); // this means "setting does not exist"
+        }
+    }
+
+    private static class SecureFileSetting extends SecureSetting<InputStream> {
+        private final Setting<InputStream> fallback;
+
+        private SecureFileSetting(String name, Setting<InputStream> fallback, Property... properties) {
+            super(name, properties);
+            this.fallback = fallback;
+        }
+
+        @Override
+        protected InputStream getSecret(SecureSettings secureSettings) throws GeneralSecurityException {
+            return secureSettings.getFile(getKey());
+        }
+
+        @Override
+        InputStream getFallback(Settings settings) {
+            if (fallback != null) {
+                return fallback.get(settings);
+            }
+            return null;
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/common/settings/SecureSettings.java b/core/src/main/java/org/elasticsearch/common/settings/SecureSettings.java
new file mode 100644
index 0000000000000..98f980c1ec6c8
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/settings/SecureSettings.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.InputStream;
+import java.security.GeneralSecurityException;
+import java.util.Set;
+
+/**
+ * An accessor for settings which are securely stored. See {@link SecureSetting}.
+ */
+public interface SecureSettings extends Closeable {
+
+    /** Returns true iff the settings are loaded and retrievable. */
+    boolean isLoaded();
+
+    /** Returns the names of all secure settings available. */
+    Set<String> getSettingNames();
+
+    /** Return a string setting. The {@link SecureString} should be closed once it is used. */
+    SecureString getString(String setting) throws GeneralSecurityException;
+
+    /** Return a file setting. The {@link InputStream} should be closed once it is used. */
+    InputStream getFile(String setting) throws GeneralSecurityException;
+
+    @Override
+    void close() throws IOException;
+}
diff --git a/core/src/main/java/org/elasticsearch/common/settings/SecureString.java b/core/src/main/java/org/elasticsearch/common/settings/SecureString.java
new file mode 100644
index 0000000000000..85c4c566db149
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/settings/SecureString.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import java.io.Closeable;
+import java.util.Arrays;
+import java.util.Objects;
+
+/**
+ * A String implementations which allows clearing the underlying char array.
+ */
+public final class SecureString implements CharSequence, Closeable {
+
+    private char[] chars;
+
+    /**
+     * Constructs a new SecureString which controls the passed in char array.
+     *
+     * Note: When this instance is closed, the array will be zeroed out.
+     */
+    public SecureString(char[] chars) {
+        this.chars = Objects.requireNonNull(chars);
+    }
+
+    /**
+     * Constructs a new SecureString from an existing String.
+     *
+     * NOTE: This is not actually secure, since the provided String cannot be deallocated, but
+     * this constructor allows for easy compatibility between new and old apis.
+     *
+     * @deprecated Only use for compatibility between deprecated string settings and new secure strings
+     */
+    @Deprecated
+    public SecureString(String s) {
+        this(s.toCharArray());
+    }
+
+    /** Constant time equality to avoid potential timing attacks. */
+    @Override
+    public synchronized boolean equals(Object o) {
+        ensureNotClosed();
+        if (this == o) return true;
+        if (o == null || o instanceof CharSequence == false) return false;
+        CharSequence that = (CharSequence) o;
+        if (chars.length != that.length()) {
+            return false;
+        }
+
+        int equals = 0;
+        for (int i = 0; i < chars.length; i++) {
+            equals |= chars[i] ^ that.charAt(i);
+        }
+
+        return equals == 0;
+    }
+
+    @Override
+    public synchronized int hashCode() {
+        return Arrays.hashCode(chars);
+    }
+
+    @Override
+    public synchronized int length() {
+        ensureNotClosed();
+        return chars.length;
+    }
+
+    @Override
+    public synchronized char charAt(int index) {
+        ensureNotClosed();
+        return chars[index];
+    }
+
+    @Override
+    public SecureString subSequence(int start, int end) {
+        throw new UnsupportedOperationException("Cannot get subsequence of SecureString");
+    }
+
+    /**
+     * Convert to a {@link String}. This should only be used with APIs that do not take {@link CharSequence}.
+     */
+    @Override
+    public synchronized String toString() {
+        return new String(chars);
+    }
+
+    /**
+     * Closes the string by clearing the underlying char array.
+     */
+    @Override
+    public synchronized void close() {
+        if (chars != null) {
+            Arrays.fill(chars, '\0');
+            chars = null;
+        }
+    }
+
+    /**
+     * Returns a new copy of this object that is backed by its own char array. Closing the new instance has no effect on the instance it
+     * was created from. This is useful for APIs which accept a char array and you want to be safe about the API potentially modifying the
+     * char array. For example:
+     *
+     * <pre>
+     *     try (SecureString copy = secureString.clone()) {
+     *         // pass thee char[] to a external API
+     *         PasswordAuthentication auth = new PasswordAuthentication(username, copy.getChars());
+     *         ...
+     *     }
+     * </pre>
+     */
+    @Override
+    public synchronized SecureString clone() {
+        ensureNotClosed();
+        return new SecureString(Arrays.copyOf(chars, chars.length));
+    }
+
+    /**
+     * Returns the underlying char[]. This is a dangerous operation as the array may be modified while it is being used by other threads
+     * or a consumer may modify the values in the array. For safety, it is preferable to use {@link #clone()} and pass its chars to the
+     * consumer when the chars are needed multiple times.
+     */
+    public synchronized char[] getChars() {
+        ensureNotClosed();
+        return chars;
+    }
+
+    /** Throw an exception if this string has been closed, indicating something is trying to access the data after being closed. */
+    private void ensureNotClosed() {
+        if (chars == null) {
+            throw new IllegalStateException("SecureString has already been closed");
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/common/settings/Setting.java b/core/src/main/java/org/elasticsearch/common/settings/Setting.java
index c22e12b3ce899..89a312092eb09 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/Setting.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/Setting.java
@@ -32,6 +32,7 @@
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.MemorySizeValue;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -41,14 +42,17 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.EnumSet;
+import java.util.IdentityHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 import java.util.function.Function;
+import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 /**
  * A setting. Encapsulates typical stuff like default value, parsing, and scope.
@@ -91,6 +95,12 @@ public enum Property {
          */
         Dynamic,
 
+        /**
+         * mark this setting as final, not updateable even when the context is not dynamic
+         * ie. Setting this property on an index scoped setting will fail update when the index is closed
+         */
+        Final,
+
         /**
          * mark this setting as deprecated
          */
@@ -118,7 +128,8 @@ public enum Property {
 
     private Setting(Key key, @Nullable Setting<T> fallbackSetting, Function<Settings, String> defaultValue, Function<String, T> parser,
             Property... properties) {
-        assert parser.apply(defaultValue.apply(Settings.EMPTY)) != null || this.isGroupSetting(): "parser returned null";
+        assert this instanceof SecureSetting || this.isGroupSetting() || parser.apply(defaultValue.apply(Settings.EMPTY)) != null
+               : "parser returned null";
         this.key = key;
         this.fallbackSetting = fallbackSetting;
         this.defaultValue = defaultValue;
@@ -130,6 +141,9 @@ private Setting(Key key, @Nullable Setting<T> fallbackSetting, Function<Settings
             this.properties = EMPTY_PROPERTIES;
         } else {
             this.properties = EnumSet.copyOf(Arrays.asList(properties));
+            if (isDynamic() && isFinal()) {
+                throw new IllegalArgumentException("final setting [" + key + "] cannot be dynamic");
+            }
         }
     }
 
@@ -213,6 +227,13 @@ public final boolean isDynamic() {
         return properties.contains(Property.Dynamic);
     }
 
+    /**
+     * Returns <code>true</code> if this setting is final, otherwise <code>false</code>
+     */
+    public final boolean isFinal() {
+        return properties.contains(Property.Final);
+    }
+
     /**
      * Returns the setting properties
      * @see Property
@@ -269,11 +290,18 @@ boolean hasComplexMatcher() {
         return isGroupSetting();
     }
 
+    /**
+     * Returns <code>true</code> iff this setting is a list setting.
+     */
+    boolean isListSetting() {
+        return false;
+    }
+
     /**
      * Returns the default value string representation for this setting.
      * @param settings a settings object for settings that has a default value depending on another setting if available
      */
-    public final String getDefaultRaw(Settings settings) {
+    public String getDefaultRaw(Settings settings) {
         return defaultValue.apply(settings);
     }
 
@@ -281,7 +309,7 @@ public final String getDefaultRaw(Settings settings) {
      * Returns the default value for this setting.
      * @param settings a settings object for settings that has a default value depending on another setting if available
      */
-    public final T getDefault(Settings settings) {
+    public T getDefault(Settings settings) {
         return parser.apply(getDefaultRaw(settings));
     }
 
@@ -289,7 +317,7 @@ public final T getDefault(Settings settings) {
      * Returns <code>true</code> iff this setting is present in the given settings object. Otherwise <code>false</code>
      */
     public boolean exists(Settings settings) {
-        return settings.get(getKey()) != null;
+        return settings.getAsMap().containsKey(getKey());
     }
 
     /**
@@ -311,19 +339,40 @@ public T get(Settings settings) {
         }
     }
 
+    /**
+     * Add this setting to the builder if it doesn't exists in the source settings.
+     * The value added to the builder is taken from the given default settings object.
+     * @param builder the settings builder to fill the diff into
+     * @param source the source settings object to diff
+     * @param defaultSettings the default settings object to diff against
+     */
+    public void diff(Settings.Builder builder, Settings source, Settings defaultSettings) {
+        if (exists(source) == false) {
+            builder.put(getKey(), getRaw(defaultSettings));
+        }
+    }
+
     /**
      * Returns the raw (string) settings value. If the setting is not present in the given settings object the default value is returned
      * instead. This is useful if the value can't be parsed due to an invalid value to access the actual value.
      */
     public String getRaw(Settings settings) {
+        checkDeprecation(settings);
+        return settings.get(getKey(), defaultValue.apply(settings));
+    }
+
+    /** Logs a deprecation warning if the setting is deprecated and used. */
+    void checkDeprecation(Settings settings) {
         // They're using the setting, so we need to tell them to stop
         if (this.isDeprecated() && this.exists(settings)) {
             // It would be convenient to show its replacement key, but replacement is often not so simple
-            final DeprecationLogger deprecationLogger = new DeprecationLogger(Loggers.getLogger(getClass()));
-            deprecationLogger.deprecated("[{}] setting was deprecated in Elasticsearch and it will be removed in a future release! " +
-                    "See the breaking changes lists in the documentation for details", getKey());
+            final String key = getKey();
+            Settings.DeprecationLoggerHolder.deprecationLogger.deprecatedAndMaybeLog(
+                    key,
+                    "[{}] setting was deprecated in Elasticsearch and will be removed in a future release! "
+                            + "See the breaking changes documentation for the next major version.",
+                    key);
         }
-        return settings.get(getKey(), defaultValue.apply(settings));
     }
 
     /**
@@ -353,12 +402,12 @@ public final T get(Settings primary, Settings secondary) {
         if (exists(primary)) {
             return get(primary);
         }
-        if (fallbackSetting == null) {
-            return get(secondary);
-        }
         if (exists(secondary)) {
             return get(secondary);
         }
+        if (fallbackSetting == null) {
+            return get(primary);
+        }
         if (fallbackSetting.exists(primary)) {
             return fallbackSetting.get(primary);
         }
@@ -391,8 +440,8 @@ AbstractScopedSettings.SettingUpdater<T> newUpdater(Consumer<T> consumer, Logger
     }
 
     /**
-     * Updates settings that depend on eachother. See {@link AbstractScopedSettings#addSettingsUpdateConsumer(Setting, Setting, BiConsumer)}
-     * and its usage for details.
+     * Updates settings that depend on each other.
+     * See {@link AbstractScopedSettings#addSettingsUpdateConsumer(Setting, Setting, BiConsumer)} and its usage for details.
      */
     static <A, B> AbstractScopedSettings.SettingUpdater<Tuple<A, B>> compoundUpdater(final BiConsumer<A, B> consumer,
             final Setting<A> aSetting, final Setting<B> bSetting, Logger logger) {
@@ -411,6 +460,12 @@ public Tuple<A, B> getValue(Settings current, Settings previous) {
 
             @Override
             public void apply(Tuple<A, B> value, Settings current, Settings previous) {
+                if (aSettingUpdater.hasChanged(current, previous)) {
+                    logger.info("updating [{}] from [{}] to [{}]", aSetting.key, aSetting.getRaw(previous), aSetting.getRaw(current));
+                }
+                if (bSettingUpdater.hasChanged(current, previous)) {
+                    logger.info("updating [{}] from [{}] to [{}]", bSetting.key, bSetting.getRaw(previous), bSetting.getRaw(current));
+                }
                 consumer.accept(value.v1(), value.v2());
             }
 
@@ -421,13 +476,266 @@ public String toString() {
         };
     }
 
+    public static class AffixSetting<T> extends Setting<T> {
+        private final AffixKey key;
+        private final Function<String, Setting<T>> delegateFactory;
+
+        public AffixSetting(AffixKey key, Setting<T> delegate, Function<String, Setting<T>> delegateFactory) {
+            super(key, delegate.defaultValue, delegate.parser, delegate.properties.toArray(new Property[0]));
+            this.key = key;
+            this.delegateFactory = delegateFactory;
+        }
+
+        boolean isGroupSetting() {
+            return true;
+        }
+
+        private Stream<String> matchStream(Settings settings) {
+            return settings.getAsMap().keySet().stream().filter((key) -> match(key)).map(settingKey -> key.getConcreteString(settingKey));
+        }
+
+        AbstractScopedSettings.SettingUpdater<Map<AbstractScopedSettings.SettingUpdater<T>, T>> newAffixUpdater(
+            BiConsumer<String, T> consumer, Logger logger, BiConsumer<String, T> validator) {
+            return new AbstractScopedSettings.SettingUpdater<Map<AbstractScopedSettings.SettingUpdater<T>, T>>() {
+
+                @Override
+                public boolean hasChanged(Settings current, Settings previous) {
+                    return  Stream.concat(matchStream(current), matchStream(previous)).findAny().isPresent();
+                }
+
+                @Override
+                public Map<AbstractScopedSettings.SettingUpdater<T>, T> getValue(Settings current, Settings previous) {
+                    // we collect all concrete keys and then delegate to the actual setting for validation and settings extraction
+                    final Map<AbstractScopedSettings.SettingUpdater<T>, T> result = new IdentityHashMap<>();
+                    Stream.concat(matchStream(current), matchStream(previous)).distinct().forEach(aKey -> {
+                        String namespace = key.getNamespace(aKey);
+                        AbstractScopedSettings.SettingUpdater<T> updater =
+                            getConcreteSetting(aKey).newUpdater((v) -> consumer.accept(namespace, v), logger,
+                                (v) -> validator.accept(namespace, v));
+                        if (updater.hasChanged(current, previous)) {
+                            // only the ones that have changed otherwise we might get too many updates
+                            // the hasChanged above checks only if there are any changes
+                            T value = updater.getValue(current, previous);
+                            result.put(updater, value);
+                        }
+                    });
+                    return result;
+                }
+
+                @Override
+                public void apply(Map<AbstractScopedSettings.SettingUpdater<T>, T> value, Settings current, Settings previous) {
+                    for (Map.Entry<AbstractScopedSettings.SettingUpdater<T>, T> entry : value.entrySet()) {
+                        entry.getKey().apply(entry.getValue(), current, previous);
+                    }
+                }
+            };
+        }
+
+        @Override
+        public T get(Settings settings) {
+            throw new UnsupportedOperationException("affix settings can't return values" +
+                " use #getConcreteSetting to obtain a concrete setting");
+        }
+
+        @Override
+        public String getRaw(Settings settings) {
+            throw new UnsupportedOperationException("affix settings can't return values" +
+                " use #getConcreteSetting to obtain a concrete setting");
+        }
+
+        @Override
+        public Setting<T> getConcreteSetting(String key) {
+            if (match(key)) {
+                return delegateFactory.apply(key);
+            } else {
+                throw new IllegalArgumentException("key [" + key + "] must match [" + getKey() + "] but didn't.");
+            }
+        }
+
+        /**
+         * Get a setting with the given namespace filled in for prefix and suffix.
+         */
+        public Setting<T> getConcreteSettingForNamespace(String namespace) {
+            String fullKey = key.toConcreteKey(namespace).toString();
+            return getConcreteSetting(fullKey);
+        }
+
+        @Override
+        public void diff(Settings.Builder builder, Settings source, Settings defaultSettings) {
+            matchStream(defaultSettings).forEach((key) -> getConcreteSetting(key).diff(builder, source, defaultSettings));
+        }
+
+        /**
+         * Returns the namespace for a concrete settting. Ie. an affix setting with prefix: <tt>search.</tt> and suffix: <tt>username</tt>
+         * will return <tt>remote</tt> as a namespace for the setting <tt>search.remote.username</tt>
+         */
+        public String getNamespace(Setting<T> concreteSetting) {
+            return key.getNamespace(concreteSetting.getKey());
+        }
+
+        /**
+         * Returns a stream of all concrete setting instances for the given settings. AffixSetting is only a specification, concrete
+         * settings depend on an actual set of setting keys.
+         */
+        public Stream<Setting<T>> getAllConcreteSettings(Settings settings) {
+            return matchStream(settings).distinct().map(this::getConcreteSetting);
+        }
+    }
+
+
+    private static class GroupSetting extends Setting<Settings> {
+        private final String key;
+        private final Consumer<Settings> validator;
+
+        private GroupSetting(String key, Consumer<Settings> validator, Property... properties) {
+            super(new GroupKey(key), (s) -> "", (s) -> null, properties);
+            this.key = key;
+            this.validator = validator;
+        }
+
+        @Override
+        public boolean isGroupSetting() {
+            return true;
+        }
+
+        @Override
+        public String getRaw(Settings settings) {
+            Settings subSettings = get(settings);
+            try {
+                XContentBuilder builder = XContentFactory.jsonBuilder();
+                builder.startObject();
+                subSettings.toXContent(builder, EMPTY_PARAMS);
+                builder.endObject();
+                return builder.string();
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        }
+
+        @Override
+        public Settings get(Settings settings) {
+            Settings byPrefix = settings.getByPrefix(getKey());
+            validator.accept(byPrefix);
+            return byPrefix;
+        }
+
+        @Override
+        public boolean exists(Settings settings) {
+            for (Map.Entry<String, String> entry : settings.getAsMap().entrySet()) {
+                if (entry.getKey().startsWith(key)) {
+                    return true;
+                }
+            }
+            return false;
+        }
+
+        @Override
+        public void diff(Settings.Builder builder, Settings source, Settings defaultSettings) {
+            Map<String, String> leftGroup = get(source).getAsMap();
+            Settings defaultGroup = get(defaultSettings);
+            for (Map.Entry<String, String> entry : defaultGroup.getAsMap().entrySet()) {
+                if (leftGroup.containsKey(entry.getKey()) == false) {
+                    builder.put(getKey() + entry.getKey(), entry.getValue());
+                }
+            }
+        }
+
+        @Override
+        public AbstractScopedSettings.SettingUpdater<Settings> newUpdater(Consumer<Settings> consumer, Logger logger,
+                                                                          Consumer<Settings> validator) {
+            if (isDynamic() == false) {
+                throw new IllegalStateException("setting [" + getKey() + "] is not dynamic");
+            }
+            final Setting<?> setting = this;
+            return new AbstractScopedSettings.SettingUpdater<Settings>() {
+
+                @Override
+                public boolean hasChanged(Settings current, Settings previous) {
+                    Settings currentSettings = get(current);
+                    Settings previousSettings = get(previous);
+                    return currentSettings.equals(previousSettings) == false;
+                }
+
+                @Override
+                public Settings getValue(Settings current, Settings previous) {
+                    Settings currentSettings = get(current);
+                    Settings previousSettings = get(previous);
+                    try {
+                        validator.accept(currentSettings);
+                    } catch (Exception | AssertionError e) {
+                        throw new IllegalArgumentException("illegal value can't update [" + key + "] from ["
+                                + previousSettings.getAsMap() + "] to [" + currentSettings.getAsMap() + "]", e);
+                    }
+                    return currentSettings;
+                }
+
+                @Override
+                public void apply(Settings value, Settings current, Settings previous) {
+                    if (logger.isInfoEnabled()) { // getRaw can create quite some objects
+                        logger.info("updating [{}] from [{}] to [{}]", key, getRaw(previous), getRaw(current));
+                    }
+                    consumer.accept(value);
+                }
+
+                @Override
+                public String toString() {
+                    return "Updater for: " + setting.toString();
+                }
+            };
+        }
+    }
+
+    private static class ListSetting<T> extends Setting<List<T>> {
+        private final Function<Settings, List<String>> defaultStringValue;
+
+        private ListSetting(String key, Function<Settings, List<String>> defaultStringValue, Function<String, List<T>> parser,
+                            Property... properties) {
+            super(new ListKey(key), (s) -> Setting.arrayToParsableString(defaultStringValue.apply(s).toArray(Strings.EMPTY_ARRAY)), parser,
+                properties);
+            this.defaultStringValue = defaultStringValue;
+        }
+
+        @Override
+        public String getRaw(Settings settings) {
+            String[] array = settings.getAsArray(getKey(), null);
+            return array == null ? defaultValue.apply(settings) : arrayToParsableString(array);
+        }
+
+        @Override
+        boolean hasComplexMatcher() {
+            return true;
+        }
+
+        @Override
+        boolean isListSetting() {
+            return true;
+        }
+
+        @Override
+        public boolean exists(Settings settings) {
+            boolean exists = super.exists(settings);
+            return exists || settings.get(getKey() + ".0") != null;
+        }
+
+        @Override
+        public void diff(Settings.Builder builder, Settings source, Settings defaultSettings) {
+            if (exists(source) == false) {
+                String[] asArray = defaultSettings.getAsArray(getKey(), null);
+                if (asArray == null) {
+                    builder.putArray(getKey(), defaultStringValue.apply(defaultSettings));
+                } else {
+                    builder.putArray(getKey(), asArray);
+                }
+            }
+        }
+    }
 
     private final class Updater implements AbstractScopedSettings.SettingUpdater<T> {
         private final Consumer<T> consumer;
         private final Logger logger;
         private final Consumer<T> accept;
 
-        public Updater(Consumer<T> consumer, Logger logger, Consumer<T> accept) {
+        Updater(Consumer<T> consumer, Logger logger, Consumer<T> accept) {
             this.consumer = consumer;
             this.logger = logger;
             this.accept = accept;
@@ -540,15 +848,25 @@ public static Setting<Integer> intSetting(String key, int defaultValue, Property
     }
 
     public static Setting<Boolean> boolSetting(String key, boolean defaultValue, Property... properties) {
-        return new Setting<>(key, (s) -> Boolean.toString(defaultValue), Booleans::parseBooleanExact, properties);
+        return new Setting<>(key, (s) -> Boolean.toString(defaultValue), (value) -> parseBoolean(key, value), properties);
     }
 
     public static Setting<Boolean> boolSetting(String key, Setting<Boolean> fallbackSetting, Property... properties) {
-        return new Setting<>(key, fallbackSetting, Booleans::parseBooleanExact, properties);
+        return new Setting<>(key, fallbackSetting, (value) -> parseBoolean(key, value), properties);
     }
 
     public static Setting<Boolean> boolSetting(String key, Function<Settings, String> defaultValueFn, Property... properties) {
-        return new Setting<>(key, defaultValueFn, Booleans::parseBooleanExact, properties);
+        return new Setting<>(key, defaultValueFn, (value) -> parseBoolean(key, value), properties);
+    }
+
+    private static Boolean parseBoolean(String key, String value) {
+        // let the parser handle all cases for non-proper booleans without a deprecation warning by throwing IAE
+        boolean booleanValue = Booleans.parseBooleanExact(value);
+        if (Booleans.isStrictlyBoolean(value) == false) {
+            DeprecationLogger deprecationLogger = new DeprecationLogger(Loggers.getLogger(Setting.class));
+            deprecationLogger.deprecated("Expected a boolean [true/false] for setting [{}] but got [{}]", key, value);
+        }
+        return booleanValue;
     }
 
     public static Setting<ByteSizeValue> byteSizeSetting(String key, ByteSizeValue value, Property... properties) {
@@ -578,10 +896,10 @@ public static Setting<ByteSizeValue> byteSizeSetting(String key, Function<Settin
 
     public static ByteSizeValue parseByteSize(String s, ByteSizeValue minValue, ByteSizeValue maxValue, String key) {
         ByteSizeValue value = ByteSizeValue.parseBytesSizeValue(s, key);
-        if (value.bytes() < minValue.bytes()) {
+        if (value.getBytes() < minValue.getBytes()) {
             throw new IllegalArgumentException("Failed to parse value [" + s + "] for setting [" + key + "] must be >= " + minValue);
         }
-        if (value.bytes() > maxValue.bytes()) {
+        if (value.getBytes() > maxValue.getBytes()) {
             throw new IllegalArgumentException("Failed to parse value [" + s + "] for setting [" + key + "] must be <= " + maxValue);
         }
         return value;
@@ -591,9 +909,9 @@ public static ByteSizeValue parseByteSize(String s, ByteSizeValue minValue, Byte
      * Creates a setting which specifies a memory size. This can either be
      * specified as an absolute bytes value or as a percentage of the heap
      * memory.
-     * 
+     *
      * @param key the key for the setting
-     * @param defaultValue the default value for this setting 
+     * @param defaultValue the default value for this setting
      * @param properties properties properties for this setting like scope, filtering...
      * @return the setting object
      */
@@ -606,9 +924,9 @@ public static Setting<ByteSizeValue> memorySizeSetting(String key, ByteSizeValue
      * Creates a setting which specifies a memory size. This can either be
      * specified as an absolute bytes value or as a percentage of the heap
      * memory.
-     * 
+     *
      * @param key the key for the setting
-     * @param defaultValue a function that supplies the default value for this setting 
+     * @param defaultValue a function that supplies the default value for this setting
      * @param properties properties properties for this setting like scope, filtering...
      * @return the setting object
      */
@@ -620,7 +938,7 @@ public static Setting<ByteSizeValue> memorySizeSetting(String key, Function<Sett
      * Creates a setting which specifies a memory size. This can either be
      * specified as an absolute bytes value or as a percentage of the heap
      * memory.
-     * 
+     *
      * @param key the key for the setting
      * @param defaultPercentage the default value of this setting as a percentage of the heap memory
      * @param properties properties properties for this setting like scope, filtering...
@@ -630,10 +948,6 @@ public static Setting<ByteSizeValue> memorySizeSetting(String key, String defaul
         return new Setting<>(key, (s) -> defaultPercentage, (s) -> MemorySizeValue.parseBytesSizeValueOrHeapRatio(s, key), properties);
     }
 
-    public static Setting<TimeValue> positiveTimeSetting(String key, TimeValue defaultValue, Property... properties) {
-        return timeSetting(key, defaultValue, TimeValue.timeValueMillis(0), properties);
-    }
-
     public static <T> Setting<List<T>> listSetting(String key, List<String> defaultStringValue, Function<String, T> singleValueParser,
                                                    Property... properties) {
         return listSetting(key, (s) -> defaultStringValue, singleValueParser, properties);
@@ -647,32 +961,18 @@ public static <T> Setting<List<T>> listSetting(String key, Setting<List<T>> fall
 
     public static <T> Setting<List<T>> listSetting(String key, Function<Settings, List<String>> defaultStringValue,
                                                    Function<String, T> singleValueParser, Property... properties) {
+        if (defaultStringValue.apply(Settings.EMPTY) == null) {
+            throw new IllegalArgumentException("default value function must not return null");
+        }
         Function<String, List<T>> parser = (s) ->
                 parseableStringToList(s).stream().map(singleValueParser).collect(Collectors.toList());
 
-        return new Setting<List<T>>(new ListKey(key),
-            (s) -> arrayToParsableString(defaultStringValue.apply(s).toArray(Strings.EMPTY_ARRAY)), parser, properties) {
-            @Override
-            public String getRaw(Settings settings) {
-                String[] array = settings.getAsArray(getKey(), null);
-                return array == null ? defaultValue.apply(settings) : arrayToParsableString(array);
-            }
-
-            @Override
-            boolean hasComplexMatcher() {
-                return true;
-            }
-
-            @Override
-            public boolean exists(Settings settings) {
-                boolean exists = super.exists(settings);
-                return exists || settings.get(getKey() + ".0") != null;
-            }
-        };
+        return new ListSetting<>(key, defaultStringValue, parser, properties);
     }
 
     private static List<String> parseableStringToList(String parsableString) {
-        try (XContentParser xContentParser = XContentType.JSON.xContent().createParser(parsableString)) {
+        // EMPTY is safe here because we never call namedObject
+        try (XContentParser xContentParser = XContentType.JSON.xContent().createParser(NamedXContentRegistry.EMPTY, parsableString)) {
             XContentParser.Token token = xContentParser.nextToken();
             if (token != XContentParser.Token.START_ARRAY) {
                 throw new IllegalArgumentException("expected START_ARRAY but got " + token);
@@ -690,7 +990,6 @@ private static List<String> parseableStringToList(String parsableString) {
         }
     }
 
-
     private static String arrayToParsableString(String[] array) {
         try {
             XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent());
@@ -704,94 +1003,18 @@ private static String arrayToParsableString(String[] array) {
             throw new ElasticsearchException(ex);
         }
     }
+
     public static Setting<Settings> groupSetting(String key, Property... properties) {
         return groupSetting(key, (s) -> {}, properties);
     }
-    public static Setting<Settings> groupSetting(String key, Consumer<Settings> validator, Property... properties) {
-        return new Setting<Settings>(new GroupKey(key), (s) -> "", (s) -> null, properties) {
-            @Override
-            public boolean isGroupSetting() {
-                return true;
-            }
-
-            @Override
-            public String getRaw(Settings settings) {
-                Settings subSettings = get(settings);
-                try {
-                    XContentBuilder builder = XContentFactory.jsonBuilder();
-                    builder.startObject();
-                    subSettings.toXContent(builder, EMPTY_PARAMS);
-                    builder.endObject();
-                    return builder.string();
-                } catch (IOException e) {
-                    throw new RuntimeException(e);
-                }
-            }
-
-            @Override
-            public Settings get(Settings settings) {
-                Settings byPrefix = settings.getByPrefix(getKey());
-                validator.accept(byPrefix);
-                return byPrefix;
-            }
-
-            @Override
-            public boolean exists(Settings settings) {
-                for (Map.Entry<String, String> entry : settings.getAsMap().entrySet()) {
-                    if (entry.getKey().startsWith(key)) {
-                        return true;
-                    }
-                }
-                return false;
-            }
-
-            @Override
-            public AbstractScopedSettings.SettingUpdater<Settings> newUpdater(Consumer<Settings> consumer, Logger logger,
-                    Consumer<Settings> validator) {
-                if (isDynamic() == false) {
-                    throw new IllegalStateException("setting [" + getKey() + "] is not dynamic");
-                }
-                final Setting<?> setting = this;
-                return new AbstractScopedSettings.SettingUpdater<Settings>() {
-
-                    @Override
-                    public boolean hasChanged(Settings current, Settings previous) {
-                        Settings currentSettings = get(current);
-                        Settings previousSettings = get(previous);
-                        return currentSettings.equals(previousSettings) == false;
-                    }
-
-                    @Override
-                    public Settings getValue(Settings current, Settings previous) {
-                        Settings currentSettings = get(current);
-                        Settings previousSettings = get(previous);
-                        try {
-                            validator.accept(currentSettings);
-                        } catch (Exception | AssertionError e) {
-                            throw new IllegalArgumentException("illegal value can't update [" + key + "] from ["
-                                    + previousSettings.getAsMap() + "] to [" + currentSettings.getAsMap() + "]", e);
-                        }
-                        return currentSettings;
-                    }
 
-                    @Override
-                    public void apply(Settings value, Settings current, Settings previous) {
-                        logger.info("updating [{}] from [{}] to [{}]", key, getRaw(previous), getRaw(current));
-                        consumer.accept(value);
-                    }
-
-                    @Override
-                    public String toString() {
-                        return "Updater for: " + setting.toString();
-                    }
-                };
-            }
-        };
+    public static Setting<Settings> groupSetting(String key, Consumer<Settings> validator, Property... properties) {
+        return new GroupSetting(key, validator, properties);
     }
 
-    public static Setting<TimeValue> timeSetting(String key, Function<Settings, String> defaultValue, TimeValue minValue,
+    public static Setting<TimeValue> timeSetting(String key, Function<Settings, TimeValue> defaultValue, TimeValue minValue,
                                                  Property... properties) {
-        return new Setting<>(key, defaultValue, (s) -> {
+        return new Setting<>(key, (s) -> defaultValue.apply(s).getStringRep(), (s) -> {
             TimeValue timeValue = TimeValue.parseTimeValue(s, null, key);
             if (timeValue.millis() < minValue.millis()) {
                 throw new IllegalArgumentException("Failed to parse value [" + s + "] for setting [" + key + "] must be >= " + minValue);
@@ -801,17 +1024,21 @@ public static Setting<TimeValue> timeSetting(String key, Function<Settings, Stri
     }
 
     public static Setting<TimeValue> timeSetting(String key, TimeValue defaultValue, TimeValue minValue, Property... properties) {
-        return timeSetting(key, (s) -> defaultValue.getStringRep(), minValue, properties);
+        return timeSetting(key, (s) -> defaultValue, minValue, properties);
     }
 
     public static Setting<TimeValue> timeSetting(String key, TimeValue defaultValue, Property... properties) {
-        return new Setting<>(key, (s) -> defaultValue.toString(), (s) -> TimeValue.parseTimeValue(s, key), properties);
+        return new Setting<>(key, (s) -> defaultValue.getStringRep(), (s) -> TimeValue.parseTimeValue(s, key), properties);
     }
 
     public static Setting<TimeValue> timeSetting(String key, Setting<TimeValue> fallbackSetting, Property... properties) {
         return new Setting<>(key, fallbackSetting, (s) -> TimeValue.parseTimeValue(s, key), properties);
     }
 
+    public static Setting<TimeValue> positiveTimeSetting(String key, TimeValue defaultValue, Property... properties) {
+        return timeSetting(key, defaultValue, TimeValue.timeValueMillis(0), properties);
+    }
+
     public static Setting<Double> doubleSetting(String key, double defaultValue, double minValue, Property... properties) {
         return new Setting<>(key, (s) -> Double.toString(defaultValue), (s) -> {
             final double d = Double.parseDouble(s);
@@ -840,50 +1067,24 @@ public int hashCode() {
      * can easily be added with this setting. Yet, prefix key settings don't support updaters out of the box unless
      * {@link #getConcreteSetting(String)} is used to pull the updater.
      */
-    public static <T> Setting<T> prefixKeySetting(String prefix, String defaultValue, Function<String, T> parser,
-                                                  Property... properties) {
-        return affixKeySetting(AffixKey.withPrefix(prefix), (s) -> defaultValue, parser, properties);
+    public static <T> AffixSetting<T> prefixKeySetting(String prefix, Function<String, Setting<T>> delegateFactory) {
+        return affixKeySetting(new AffixKey(prefix), delegateFactory);
     }
 
     /**
      * This setting type allows to validate settings that have the same type and a common prefix and suffix. For instance
-     * storage.${backend}.enable=[true|false] can easily be added with this setting. Yet, adfix key settings don't support updaters
+     * storage.${backend}.enable=[true|false] can easily be added with this setting. Yet, affix key settings don't support updaters
      * out of the box unless {@link #getConcreteSetting(String)} is used to pull the updater.
      */
-    public static <T> Setting<T> adfixKeySetting(String prefix, String suffix, Function<Settings, String> defaultValue,
-                                                 Function<String, T> parser, Property... properties) {
-        return affixKeySetting(AffixKey.withAdfix(prefix, suffix), defaultValue, parser, properties);
-    }
-
-    public static <T> Setting<T> adfixKeySetting(String prefix, String suffix, String defaultValue, Function<String, T> parser,
-                                                 Property... properties) {
-        return adfixKeySetting(prefix, suffix, (s) -> defaultValue, parser, properties);
+    public static <T> AffixSetting<T> affixKeySetting(String prefix, String suffix, Function<String, Setting<T>> delegateFactory) {
+        return affixKeySetting(new AffixKey(prefix, suffix), delegateFactory);
     }
 
-    public static <T> Setting<T> affixKeySetting(AffixKey key, Function<Settings, String> defaultValue, Function<String, T> parser,
-                                                 Property... properties) {
-        return new Setting<T>(key, defaultValue, parser, properties) {
+    private static <T> AffixSetting<T> affixKeySetting(AffixKey key, Function<String, Setting<T>> delegateFactory) {
+        Setting<T> delegate = delegateFactory.apply("_na_");
+        return new AffixSetting<>(key, delegate, delegateFactory);
+    };
 
-            @Override
-            boolean isGroupSetting() {
-                return true;
-            }
-
-            @Override
-            AbstractScopedSettings.SettingUpdater<T> newUpdater(Consumer<T> consumer, Logger logger, Consumer<T> validator) {
-                throw new UnsupportedOperationException("Affix settings can't be updated. Use #getConcreteSetting for updating.");
-            }
-
-            @Override
-            public Setting<T> getConcreteSetting(String key) {
-                if (match(key)) {
-                    return new Setting<>(key, defaultValue, parser, properties);
-                } else {
-                    throw new IllegalArgumentException("key [" + key + "] must match [" + getKey() + "] but didn't.");
-                }
-            }
-        };
-    }
 
 
     public interface Key {
@@ -949,34 +1150,60 @@ public boolean match(String toTest) {
         }
     }
 
+    /**
+     * A key that allows for static pre and suffix. This is used for settings
+     * that have dynamic namespaces like for different accounts etc.
+     */
     public static final class AffixKey implements Key {
-        public static AffixKey withPrefix(String prefix) {
-            return new AffixKey(prefix, null);
-        }
-
-        public static AffixKey withAdfix(String prefix, String suffix) {
-            return new AffixKey(prefix, suffix);
-        }
-
+        private final Pattern pattern;
         private final String prefix;
         private final String suffix;
 
-        public AffixKey(String prefix, String suffix) {
+        AffixKey(String prefix) {
+            this(prefix, null);
+        }
+
+        AffixKey(String prefix, String suffix) {
             assert prefix != null || suffix != null: "Either prefix or suffix must be non-null";
+
             this.prefix = prefix;
+            if (prefix.endsWith(".") == false) {
+                throw new IllegalArgumentException("prefix must end with a '.'");
+            }
             this.suffix = suffix;
+            if (suffix == null) {
+                pattern = Pattern.compile("(" + Pattern.quote(prefix) + "((?:[-\\w]+[.])*[-\\w]+$))");
+            } else {
+                // the last part of this regexp is for lists since they are represented as x.${namespace}.y.1, x.${namespace}.y.2
+                pattern = Pattern.compile("(" + Pattern.quote(prefix) + "([-\\w]+)\\." + Pattern.quote(suffix) + ")(?:\\.\\d+)?");
+            }
         }
 
         @Override
         public boolean match(String key) {
-            boolean match = true;
-            if (prefix != null) {
-                match = key.startsWith(prefix);
+            return pattern.matcher(key).matches();
+        }
+
+        /**
+         * Returns a string representation of the concrete setting key
+         */
+        String getConcreteString(String key) {
+            Matcher matcher = pattern.matcher(key);
+            if (matcher.matches() == false) {
+                throw new IllegalStateException("can't get concrete string for key " + key + " key doesn't match");
             }
-            if (suffix != null) {
-                match = match && key.endsWith(suffix);
+            return matcher.group(1);
+        }
+
+        /**
+         * Returns a string representation of the concrete setting key
+         */
+        String getNamespace(String key) {
+            Matcher matcher = pattern.matcher(key);
+            if (matcher.matches() == false) {
+                throw new IllegalStateException("can't get concrete string for key " + key + " key doesn't match");
             }
-            return match;
+            return matcher.group(2);
         }
 
         public SimpleKey toConcreteKey(String missingPart) {
@@ -999,9 +1226,9 @@ public String toString() {
                 sb.append(prefix);
             }
             if (suffix != null) {
-                sb.append("*");
+                sb.append('*');
+                sb.append('.');
                 sb.append(suffix);
-                sb.append(".");
             }
             return sb.toString();
         }
diff --git a/core/src/main/java/org/elasticsearch/common/settings/Settings.java b/core/src/main/java/org/elasticsearch/common/settings/Settings.java
index 9e5dd0efbe28e..908d5d11fbbd4 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/Settings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/Settings.java
@@ -19,12 +19,16 @@
 
 package org.elasticsearch.common.settings;
 
+import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.LogConfigurator;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.loader.SettingsLoader;
 import org.elasticsearch.common.settings.loader.SettingsLoaderFactory;
 import org.elasticsearch.common.unit.ByteSizeUnit;
@@ -35,6 +39,7 @@
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentType;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -42,6 +47,9 @@
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.security.GeneralSecurityException;
+import java.util.AbstractMap;
+import java.util.AbstractSet;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -49,19 +57,21 @@
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
-import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
+import java.util.NoSuchElementException;
 import java.util.Objects;
 import java.util.Set;
-import java.util.SortedMap;
 import java.util.TreeMap;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
 import java.util.function.Predicate;
+import java.util.function.UnaryOperator;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import static org.elasticsearch.common.unit.ByteSizeValue.parseBytesSizeValue;
 import static org.elasticsearch.common.unit.SizeValue.parseSizeValue;
@@ -71,15 +81,36 @@
  * An immutable settings implementation.
  */
 public final class Settings implements ToXContent {
-
     public static final Settings EMPTY = new Builder().build();
     private static final Pattern ARRAY_PATTERN = Pattern.compile("(.*)\\.\\d+$");
 
-    private SortedMap<String, String> settings;
+    /** The raw settings from the full key to raw string value. */
+    private final Map<String, String> settings;
+
+    /** The secure settings storage associated with these settings. */
+    private final SecureSettings secureSettings;
 
-    Settings(Map<String, String> settings) {
+    /** The first level of setting names. This is constructed lazily in {@link #names()}. */
+    private final SetOnce<Set<String>> firstLevelNames = new SetOnce<>();
+
+    /**
+     * Setting names found in this Settings for both string and secure settings.
+     * This is constructed lazily in {@link #keySet()}.
+     */
+    private final SetOnce<Set<String>> keys = new SetOnce<>();
+
+    Settings(Map<String, String> settings, SecureSettings secureSettings) {
         // we use a sorted map for consistent serialization when using getAsMap()
         this.settings = Collections.unmodifiableSortedMap(new TreeMap<>(settings));
+        this.secureSettings = secureSettings;
+    }
+
+    /**
+     * Retrieve the secure settings in these settings.
+     */
+    SecureSettings getSecureSettings() {
+        // pkg private so it can only be accessed by local subclasses of SecureSetting
+        return secureSettings;
     }
 
     /**
@@ -87,7 +118,8 @@ public final class Settings implements ToXContent {
      * @return an unmodifiable map of settings
      */
     public Map<String, String> getAsMap() {
-        return Collections.unmodifiableMap(this.settings);
+        // settings is always unmodifiable
+        return this.settings;
     }
 
     /**
@@ -186,30 +218,16 @@ private Object convertMapsToArrays(Map<String, Object> map) {
      * A settings that are filtered (and key is removed) with the specified prefix.
      */
     public Settings getByPrefix(String prefix) {
-        Builder builder = new Builder();
-        for (Map.Entry<String, String> entry : getAsMap().entrySet()) {
-            if (entry.getKey().startsWith(prefix)) {
-                if (entry.getKey().length() < prefix.length()) {
-                    // ignore this. one
-                    continue;
-                }
-                builder.put(entry.getKey().substring(prefix.length()), entry.getValue());
-            }
-        }
-        return builder.build();
+        return new Settings(new FilteredMap(this.settings, (k) -> k.startsWith(prefix), prefix), secureSettings == null ? null :
+            new PrefixedSecureSettings(secureSettings, prefix, s -> s.startsWith(prefix)));
     }
 
     /**
      * Returns a new settings object that contains all setting of the current one filtered by the given settings key predicate.
      */
     public Settings filter(Predicate<String> predicate) {
-        Builder builder = new Builder();
-        for (Map.Entry<String, String> entry : getAsMap().entrySet()) {
-            if (predicate.test(entry.getKey())) {
-                builder.put(entry.getKey(), entry.getValue());
-            }
-        }
-        return builder.build();
+        return new Settings(new FilteredMap(this.settings, predicate, null), secureSettings == null ? null :
+            new PrefixedSecureSettings(secureSettings, "", predicate));
     }
 
     /**
@@ -302,12 +320,27 @@ public Long getAsLong(String setting, Long defaultValue) {
         }
     }
 
+    /**
+     * We have to lazy initialize the deprecation logger as otherwise a static logger here would be constructed before logging is configured
+     * leading to a runtime failure (see {@link LogConfigurator#checkErrorListener()} ). The premature construction would come from any
+     * {@link Setting} object constructed in, for example, {@link org.elasticsearch.env.Environment}.
+     */
+    static class DeprecationLoggerHolder {
+        static DeprecationLogger deprecationLogger = new DeprecationLogger(Loggers.getLogger(Settings.class));
+    }
+
     /**
      * Returns the setting value (as boolean) associated with the setting key. If it does not exists,
      * returns the default value provided.
      */
     public Boolean getAsBoolean(String setting, Boolean defaultValue) {
-        return Booleans.parseBoolean(get(setting), defaultValue);
+        String rawValue = get(setting);
+        Boolean booleanValue = Booleans.parseBooleanExact(rawValue, defaultValue);
+        if (rawValue != null && Booleans.isStrictlyBoolean(rawValue) == false) {
+            final DeprecationLogger deprecationLogger = DeprecationLoggerHolder.deprecationLogger;
+            deprecationLogger.deprecated("Expected a boolean [true/false] for setting [{}] but got [{}]", setting, rawValue);
+        }
+        return booleanValue;
     }
 
     /**
@@ -395,6 +428,20 @@ public String[] getAsArray(String settingPrefix, String[] defaultArray) throws S
     public String[] getAsArray(String settingPrefix, String[] defaultArray, Boolean commaDelimited) throws SettingsException {
         List<String> result = new ArrayList<>();
 
+        final String valueFromPrefix = get(settingPrefix);
+        final String valueFromPreifx0 = get(settingPrefix + ".0");
+
+        if (valueFromPrefix != null && valueFromPreifx0 != null) {
+            final String message = String.format(
+                    Locale.ROOT,
+                    "settings object contains values for [%s=%s] and [%s=%s]",
+                    settingPrefix,
+                    valueFromPrefix,
+                    settingPrefix + ".0",
+                    valueFromPreifx0);
+            throw new IllegalStateException(message);
+        }
+
         if (get(settingPrefix) != null) {
             if (commaDelimited) {
                 String[] strings = Strings.splitStringByCommaToArray(get(settingPrefix));
@@ -443,36 +490,23 @@ public Map<String, Settings> getGroups(String settingPrefix, boolean ignoreNonGr
         }
         return getGroupsInternal(settingPrefix, ignoreNonGrouped);
     }
+
     private Map<String, Settings> getGroupsInternal(String settingPrefix, boolean ignoreNonGrouped) throws SettingsException {
-        // we don't really care that it might happen twice
-        Map<String, Map<String, String>> map = new LinkedHashMap<>();
-        for (Object o : settings.keySet()) {
-            String setting = (String) o;
-            if (setting.startsWith(settingPrefix)) {
-                String nameValue = setting.substring(settingPrefix.length());
-                int dotIndex = nameValue.indexOf('.');
-                if (dotIndex == -1) {
-                    if (ignoreNonGrouped) {
-                        continue;
-                    }
-                    throw new SettingsException("Failed to get setting group for [" + settingPrefix + "] setting prefix and setting ["
-                            + setting + "] because of a missing '.'");
-                }
-                String name = nameValue.substring(0, dotIndex);
-                String value = nameValue.substring(dotIndex + 1);
-                Map<String, String> groupSettings = map.get(name);
-                if (groupSettings == null) {
-                    groupSettings = new LinkedHashMap<>();
-                    map.put(name, groupSettings);
+        Settings prefixSettings = getByPrefix(settingPrefix);
+        Map<String, Settings> groups = new HashMap<>();
+        for (String groupName : prefixSettings.names()) {
+            Settings groupSettings = prefixSettings.getByPrefix(groupName + ".");
+            if (groupSettings.isEmpty()) {
+                if (ignoreNonGrouped) {
+                    continue;
                 }
-                groupSettings.put(value, get(setting));
+                throw new SettingsException("Failed to get setting group for [" + settingPrefix + "] setting prefix and setting ["
+                    + settingPrefix + groupName + "] because of a missing '.'");
             }
+            groups.put(groupName, groupSettings);
         }
-        Map<String, Settings> retVal = new LinkedHashMap<>();
-        for (Map.Entry<String, Map<String, String>> entry : map.entrySet()) {
-            retVal.put(entry.getKey(), new Settings(Collections.unmodifiableMap(entry.getValue())));
-        }
-        return Collections.unmodifiableMap(retVal);
+
+        return Collections.unmodifiableMap(groups);
     }
     /**
      * Returns group settings for the given setting prefix.
@@ -504,16 +538,24 @@ public Version getAsVersion(String setting, Version defaultVersion) throws Setti
      * @return  The direct keys of this settings
      */
     public Set<String> names() {
-        Set<String> names = new HashSet<>();
-        for (String key : settings.keySet()) {
-            int i = key.indexOf(".");
-            if (i < 0) {
-                names.add(key);
-            } else {
-                names.add(key.substring(0, i));
+        synchronized (firstLevelNames) {
+            if (firstLevelNames.get() == null) {
+                Stream<String> stream = settings.keySet().stream();
+                if (secureSettings != null) {
+                    stream = Stream.concat(stream, secureSettings.getSettingNames().stream());
+                }
+                Set<String> names = stream.map(k -> {
+                    int i = k.indexOf('.');
+                    if (i < 0) {
+                        return k;
+                    } else {
+                        return k.substring(0, i);
+                    }
+                }).collect(Collectors.toSet());
+                firstLevelNames.set(Collections.unmodifiableSet(names));
             }
         }
-        return names;
+        return firstLevelNames.get();
     }
 
     /**
@@ -553,8 +595,10 @@ public static Settings readSettingsFromStream(StreamInput in) throws IOException
     }
 
     public static void writeSettingsToStream(Settings settings, StreamOutput out) throws IOException {
-        out.writeVInt(settings.getAsMap().size());
-        for (Map.Entry<String, String> entry : settings.getAsMap().entrySet()) {
+        // pull getAsMap() to exclude secure settings in size()
+        Set<Map.Entry<String, String>> entries = settings.getAsMap().entrySet();
+        out.writeVInt(entries.size());
+        for (Map.Entry<String, String> entry : entries) {
             out.writeString(entry.getKey());
             out.writeOptionalString(entry.getValue());
         }
@@ -582,12 +626,36 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         return builder;
     }
 
+    public static final Set<String> FORMAT_PARAMS =
+        Collections.unmodifiableSet(new HashSet<>(Arrays.asList("settings_filter", "flat_settings")));
+
     /**
      * Returns <tt>true</tt> if this settings object contains no settings
      * @return <tt>true</tt> if this settings object contains no settings
      */
     public boolean isEmpty() {
-        return this.settings.isEmpty();
+        return this.settings.isEmpty() && (secureSettings == null || secureSettings.getSettingNames().isEmpty());
+    }
+
+    /** Returns the number of settings in this settings object. */
+    public int size() {
+        return keySet().size();
+    }
+
+    /** Returns the fully qualified setting names contained in this settings object. */
+    public Set<String> keySet() {
+        synchronized (keys) {
+            if (keys.get() == null) {
+                if (secureSettings == null) {
+                    keys.set(settings.keySet());
+                } else {
+                    Stream<String> stream = Stream.concat(settings.keySet().stream(), secureSettings.getSettingNames().stream());
+                    // uniquify, since for legacy reasons the same setting name may exist in both
+                    keys.set(Collections.unmodifiableSet(stream.collect(Collectors.toSet())));
+                }
+            }
+        }
+        return keys.get();
     }
 
     /**
@@ -599,7 +667,10 @@ public static class Builder {
 
         public static final Settings EMPTY_SETTINGS = new Builder().build();
 
-        private final Map<String, String> map = new LinkedHashMap<>();
+        // we use a sorted map for consistent serialization when using getAsMap()
+        private final Map<String, String> map = new TreeMap<>();
+
+        private SetOnce<SecureSettings> secureSettings = new SetOnce<>();
 
         private Builder() {
 
@@ -623,6 +694,23 @@ public String get(String key) {
             return map.get(key);
         }
 
+        /** Return the current secure settings, or {@code null} if none have been set. */
+        public SecureSettings getSecureSettings() {
+            return secureSettings.get();
+        }
+
+        public Builder setSecureSettings(SecureSettings secureSettings) {
+            if (secureSettings.isLoaded() == false) {
+                throw new IllegalStateException("Secure settings must already be loaded");
+            }
+            if (this.secureSettings.get() != null) {
+                throw new IllegalArgumentException("Secure settings already set. Existing settings: " +
+                    this.secureSettings.get().getSettingNames() + ", new settings: " + secureSettings.getSettingNames());
+            }
+            this.secureSettings.set(secureSettings);
+            return this;
+        }
+
         /**
          * Puts tuples of key value pairs of settings. Simplified version instead of repeating calling
          * put for each one.
@@ -847,6 +935,9 @@ public Builder put(String settingPrefix, String groupName, String[] settings, St
         public Builder put(Settings settings) {
             removeNonArraysFieldsIfNewSettingsContainsFieldAsArray(settings.getAsMap());
             map.putAll(settings.getAsMap());
+            if (settings.getSecureSettings() != null) {
+                setSecureSettings(settings.getSecureSettings());
+            }
             return this;
         }
 
@@ -903,7 +994,9 @@ public Builder put(Dictionary<Object,Object> properties) {
         /**
          * Loads settings from the actual string content that represents them using the
          * {@link SettingsLoaderFactory#loaderFromSource(String)}.
+         * @deprecated use {@link #loadFromSource(String, XContentType)} to avoid content type detection
          */
+        @Deprecated
         public Builder loadFromSource(String source) {
             SettingsLoader settingsLoader = SettingsLoaderFactory.loaderFromSource(source);
             try {
@@ -915,9 +1008,24 @@ public Builder loadFromSource(String source) {
             return this;
         }
 
+        /**
+         * Loads settings from the actual string content that represents them using the
+         * {@link SettingsLoaderFactory#loaderFromXContentType(XContentType)} method to obtain a loader
+         */
+        public Builder loadFromSource(String source, XContentType xContentType) {
+            SettingsLoader settingsLoader = SettingsLoaderFactory.loaderFromXContentType(xContentType);
+            try {
+                Map<String, String> loadedSettings = settingsLoader.load(source);
+                put(loadedSettings);
+            } catch (Exception e) {
+                throw new SettingsException("Failed to load settings from [" + source + "]", e);
+            }
+            return this;
+        }
+
         /**
          * Loads settings from a url that represents them using the
-         * {@link SettingsLoaderFactory#loaderFromSource(String)}.
+         * {@link SettingsLoaderFactory#loaderFromResource(String)}.
          */
         public Builder loadFromPath(Path path) throws IOException {
             // NOTE: loadFromStream will close the input stream
@@ -926,7 +1034,7 @@ public Builder loadFromPath(Path path) throws IOException {
 
         /**
          * Loads settings from a stream that represents them using the
-         * {@link SettingsLoaderFactory#loaderFromSource(String)}.
+         * {@link SettingsLoaderFactory#loaderFromResource(String)}.
          */
         public Builder loadFromStream(String resourceName, InputStream is) throws IOException {
             SettingsLoader settingsLoader = SettingsLoaderFactory.loaderFromResource(resourceName);
@@ -937,12 +1045,10 @@ public Builder loadFromStream(String resourceName, InputStream is) throws IOExce
             return this;
         }
 
-        public Builder putProperties(Map<String, String> esSettings, Predicate<String> keyPredicate, Function<String, String> keyFunction) {
+        public Builder putProperties(final Map<String, String> esSettings, final Function<String, String> keyFunction) {
             for (final Map.Entry<String, String> esSetting : esSettings.entrySet()) {
                 final String key = esSetting.getKey();
-                if (keyPredicate.test(key)) {
-                    map.put(keyFunction.apply(key), esSetting.getValue());
-                }
+                map.put(keyFunction.apply(key), esSetting.getValue());
             }
             return this;
         }
@@ -1029,7 +1135,171 @@ public Builder normalizePrefix(String prefix) {
          * set on this builder.
          */
         public Settings build() {
-            return new Settings(Collections.unmodifiableMap(map));
+            return new Settings(map, secureSettings.get());
+        }
+    }
+
+    // TODO We could use an FST internally to make things even faster and more compact
+    private static final class FilteredMap extends AbstractMap<String, String> {
+        private final Map<String, String> delegate;
+        private final Predicate<String> filter;
+        private final String prefix;
+        // we cache that size since we have to iterate the entire set
+        // this is safe to do since this map is only used with unmodifiable maps
+        private int size = -1;
+        @Override
+        public Set<Entry<String, String>> entrySet() {
+            Set<Entry<String, String>> delegateSet = delegate.entrySet();
+            AbstractSet<Entry<String, String>> filterSet = new AbstractSet<Entry<String, String>>() {
+
+                @Override
+                public Iterator<Entry<String, String>> iterator() {
+                    Iterator<Entry<String, String>> iter = delegateSet.iterator();
+
+                    return new Iterator<Entry<String, String>>() {
+                        private int numIterated;
+                        private Entry<String, String> currentElement;
+                        @Override
+                        public boolean hasNext() {
+                            if (currentElement != null) {
+                                return true; // protect against calling hasNext twice
+                            } else {
+                                if (numIterated == size) { // early terminate
+                                    assert size != -1 : "size was never set: " + numIterated + " vs. " + size;
+                                    return false;
+                                }
+                                while (iter.hasNext()) {
+                                    if (filter.test((currentElement = iter.next()).getKey())) {
+                                        numIterated++;
+                                        return true;
+                                    }
+                                }
+                                // we didn't find anything
+                                currentElement = null;
+                                return false;
+                            }
+                        }
+
+                        @Override
+                        public Entry<String, String> next() {
+                            if (currentElement == null && hasNext() == false) { // protect against no #hasNext call or not respecting it
+
+                                throw new NoSuchElementException("make sure to call hasNext first");
+                            }
+                            final Entry<String, String> current = this.currentElement;
+                            this.currentElement = null;
+                            if (prefix == null) {
+                                return current;
+                            }
+                            return new Entry<String, String>() {
+                                @Override
+                                public String getKey() {
+                                    return current.getKey().substring(prefix.length());
+                                }
+
+                                @Override
+                                public String getValue() {
+                                    return current.getValue();
+                                }
+
+                                @Override
+                                public String setValue(String value) {
+                                    throw new UnsupportedOperationException();
+                                }
+                            };
+                        }
+                    };
+                }
+
+                @Override
+                public int size() {
+                    return FilteredMap.this.size();
+                }
+            };
+            return filterSet;
+        }
+
+        private FilteredMap(Map<String, String> delegate, Predicate<String> filter, String prefix) {
+            this.delegate = delegate;
+            this.filter = filter;
+            this.prefix = prefix;
+        }
+
+        @Override
+        public String get(Object key) {
+            if (key instanceof String) {
+                final String theKey = prefix == null ? (String)key : prefix + key;
+                if (filter.test(theKey)) {
+                    return delegate.get(theKey);
+                }
+            }
+            return null;
+        }
+
+        @Override
+        public boolean containsKey(Object key) {
+            if (key instanceof String) {
+                final String theKey = prefix == null ? (String) key : prefix + key;
+                if (filter.test(theKey)) {
+                    return delegate.containsKey(theKey);
+                }
+            }
+            return false;
+        }
+
+        @Override
+        public int size() {
+            if (size == -1) {
+                size = Math.toIntExact(delegate.keySet().stream().filter((e) -> filter.test(e)).count());
+            }
+            return size;
+        }
+    }
+
+    private static class PrefixedSecureSettings implements SecureSettings {
+        private final SecureSettings delegate;
+        private final UnaryOperator<String> addPrefix;
+        private final UnaryOperator<String> removePrefix;
+        private final Predicate<String> keyPredicate;
+        private final SetOnce<Set<String>> settingNames = new SetOnce<>();
+
+        PrefixedSecureSettings(SecureSettings delegate, String prefix, Predicate<String> keyPredicate) {
+            this.delegate = delegate;
+            this.addPrefix = s -> prefix + s;
+            this.removePrefix = s -> s.substring(prefix.length());
+            this.keyPredicate = keyPredicate;
+        }
+
+        @Override
+        public boolean isLoaded() {
+            return delegate.isLoaded();
+        }
+
+        @Override
+        public Set<String> getSettingNames() {
+            synchronized (settingNames) {
+                if (settingNames.get() == null) {
+                    Set<String> names = delegate.getSettingNames().stream()
+                        .filter(keyPredicate).map(removePrefix).collect(Collectors.toSet());
+                    settingNames.set(Collections.unmodifiableSet(names));
+                }
+            }
+            return settingNames.get();
+        }
+
+        @Override
+        public SecureString getString(String setting) throws GeneralSecurityException{
+            return delegate.getString(addPrefix.apply(setting));
+        }
+
+        @Override
+        public InputStream getFile(String setting) throws GeneralSecurityException{
+            return delegate.getFile(addPrefix.apply(setting));
+        }
+
+        @Override
+        public void close() throws IOException {
+            delegate.close();
         }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/settings/SettingsModule.java b/core/src/main/java/org/elasticsearch/common/settings/SettingsModule.java
index 60276ce14f7a1..acad70275a6f5 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/SettingsModule.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/SettingsModule.java
@@ -54,6 +54,7 @@ public class SettingsModule implements Module {
     private final Logger logger;
     private final IndexScopedSettings indexScopedSettings;
     private final ClusterSettings clusterSettings;
+    private final SettingsFilter settingsFilter;
 
     public SettingsModule(Settings settings, Setting<?>... additionalSettings) {
         this(settings, Arrays.asList(additionalSettings), Collections.emptyList());
@@ -137,12 +138,13 @@ public SettingsModule(Settings settings, List<Setting<?>> additionalSettings, Li
         final Predicate<String> acceptOnlyClusterSettings = TRIBE_CLIENT_NODE_SETTINGS_PREDICATE.negate();
         clusterSettings.validate(settings.filter(acceptOnlyClusterSettings));
         validateTribeSettings(settings, clusterSettings);
+        this.settingsFilter = new SettingsFilter(settings, settingsFilterPattern);
      }
 
     @Override
     public void configure(Binder binder) {
         binder.bind(Settings.class).toInstance(settings);
-        binder.bind(SettingsFilter.class).toInstance(new SettingsFilter(settings, settingsFilterPattern));
+        binder.bind(SettingsFilter.class).toInstance(settingsFilter);
         binder.bind(ClusterSettings.class).toInstance(clusterSettings);
         binder.bind(IndexScopedSettings.class).toInstance(indexScopedSettings);
     }
@@ -155,8 +157,12 @@ public void configure(Binder binder) {
      */
     private void registerSetting(Setting<?> setting) {
         if (setting.isFiltered()) {
-            if (settingsFilterPattern.contains(setting.getKey()) == false) {
-                registerSettingsFilter(setting.getKey());
+            final String key = setting.getKey();
+            if (settingsFilterPattern.contains(key) == false) {
+                registerSettingsFilter(key);
+                if (setting.isListSetting()) {
+                    registerSettingsFilter(key + ".*");
+                }
             }
         }
         if (setting.hasNodeScope() || setting.hasIndexScope()) {
@@ -218,4 +224,8 @@ public IndexScopedSettings getIndexScopedSettings() {
     public ClusterSettings getClusterSettings() {
         return clusterSettings;
     }
+
+    public SettingsFilter getSettingsFilter() {
+        return settingsFilter;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/settings/loader/SettingsLoaderFactory.java b/core/src/main/java/org/elasticsearch/common/settings/loader/SettingsLoaderFactory.java
index 5f2da22c5f2dd..5d8cb4918b2ea 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/loader/SettingsLoaderFactory.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/loader/SettingsLoaderFactory.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.common.settings.loader;
 
+import org.elasticsearch.common.xcontent.XContentType;
+
 /**
  * A class holding factory methods for settings loaders that attempts
  * to infer the type of the underlying settings content.
@@ -33,9 +35,7 @@ private SettingsLoaderFactory() {
      * name. This factory method assumes that if the resource name ends
      * with ".json" then the content should be parsed as JSON, else if
      * the resource name ends with ".yml" or ".yaml" then the content
-     * should be parsed as YAML, else if the resource name ends with
-     * ".properties" then the content should be parsed as properties,
-     * otherwise default to attempting to parse as JSON. Note that the
+     * should be parsed as YAML, otherwise throws an exception. Note that the
      * parsers returned by this method will not accept null-valued
      * keys.
      *
@@ -59,13 +59,15 @@ public static SettingsLoader loaderFromResource(String resourceName) {
      * contains an opening and closing brace ('{' and '}') then the
      * content should be parsed as JSON, else if the underlying content
      * fails this condition but contains a ':' then the content should
-     * be parsed as YAML, and otherwise should be parsed as properties.
+     * be parsed as YAML, and otherwise throws an exception.
      * Note that the JSON and YAML parsers returned by this method will
      * accept null-valued keys.
      *
      * @param source The underlying settings content.
      * @return A settings loader.
+     * @deprecated use {@link #loaderFromXContentType(XContentType)} instead
      */
+    @Deprecated
     public static SettingsLoader loaderFromSource(String source) {
         if (source.indexOf('{') != -1 && source.indexOf('}') != -1) {
             return new JsonSettingsLoader(true);
@@ -76,4 +78,20 @@ public static SettingsLoader loaderFromSource(String source) {
         }
     }
 
+    /**
+     * Returns a {@link SettingsLoader} based on the {@link XContentType}. Note only {@link XContentType#JSON} and
+     * {@link XContentType#YAML} are supported
+     *
+     * @param xContentType The content type
+     * @return A settings loader.
+     */
+    public static SettingsLoader loaderFromXContentType(XContentType xContentType) {
+        if (xContentType == XContentType.JSON) {
+            return new JsonSettingsLoader(true);
+        } else if (xContentType == XContentType.YAML) {
+            return new YamlSettingsLoader(true);
+        } else {
+            throw new IllegalArgumentException("unsupported content type [" + xContentType + "]");
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/settings/loader/XContentSettingsLoader.java b/core/src/main/java/org/elasticsearch/common/settings/loader/XContentSettingsLoader.java
index 30c62b91c79bd..d7eaa627a2868 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/loader/XContentSettingsLoader.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/loader/XContentSettingsLoader.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.common.settings.loader;
 
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
@@ -46,14 +47,16 @@ public abstract class XContentSettingsLoader implements SettingsLoader {
 
     @Override
     public Map<String, String> load(String source) throws IOException {
-        try (XContentParser parser = XContentFactory.xContent(contentType()).createParser(source)) {
+        // It is safe to use EMPTY here because this never uses namedObject
+        try (XContentParser parser = XContentFactory.xContent(contentType()).createParser(NamedXContentRegistry.EMPTY, source)) {
             return load(parser);
         }
     }
 
     @Override
     public Map<String, String> load(byte[] source) throws IOException {
-        try (XContentParser parser = XContentFactory.xContent(contentType()).createParser(source)) {
+        // It is safe to use EMPTY here because this never uses namedObject
+        try (XContentParser parser = XContentFactory.xContent(contentType()).createParser(NamedXContentRegistry.EMPTY, source)) {
             return load(parser);
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/common/text/Text.java b/core/src/main/java/org/elasticsearch/common/text/Text.java
index 39eb817fe3c3d..d895b7c11b02d 100644
--- a/core/src/main/java/org/elasticsearch/common/text/Text.java
+++ b/core/src/main/java/org/elasticsearch/common/text/Text.java
@@ -100,7 +100,10 @@ public int hashCode() {
 
     @Override
     public boolean equals(Object obj) {
-        if (obj == null) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj == null || getClass() != obj.getClass()) {
             return false;
         }
         return bytes().equals(((Text) obj).bytes());
diff --git a/core/src/main/java/org/elasticsearch/common/transport/InetSocketTransportAddress.java b/core/src/main/java/org/elasticsearch/common/transport/InetSocketTransportAddress.java
index 94c1a2390ac2d..027d7344a29aa 100644
--- a/core/src/main/java/org/elasticsearch/common/transport/InetSocketTransportAddress.java
+++ b/core/src/main/java/org/elasticsearch/common/transport/InetSocketTransportAddress.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.common.transport;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.network.NetworkAddress;
@@ -53,19 +54,38 @@ public InetSocketTransportAddress(InetSocketAddress address) {
      * Read from a stream.
      */
     public InetSocketTransportAddress(StreamInput in) throws IOException {
+        this(in, null);
+    }
+
+    /**
+     * Read from a stream and use the {@code hostString} when creating the InetAddress if the input comes from a version on or prior
+     * {@link Version#V_5_0_2} as the hostString was not serialized
+     */
+    public InetSocketTransportAddress(StreamInput in, String hostString) throws IOException {
         final int len = in.readByte();
         final byte[] a = new byte[len]; // 4 bytes (IPv4) or 16 bytes (IPv6)
         in.readFully(a);
-        InetAddress inetAddress = InetAddress.getByAddress(a);
+        final InetAddress inetAddress;
+        if (in.getVersion().after(Version.V_5_0_2)) {
+            String host = in.readString();
+            inetAddress = InetAddress.getByAddress(host, a); // the host string was serialized so we can ignore the passed in value
+        } else {
+            // prior to this version, we did not serialize the host string so we used the passed in value
+            inetAddress = InetAddress.getByAddress(hostString, a);
+        }
         int port = in.readInt();
         this.address = new InetSocketAddress(inetAddress, port);
     }
 
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         byte[] bytes = address().getAddress().getAddress();  // 4 bytes (IPv4) or 16 bytes (IPv6)
         out.writeByte((byte) bytes.length); // 1 byte
         out.write(bytes, 0, bytes.length);
+        if (out.getVersion().after(Version.V_5_0_2)) {
+            out.writeString(address.getHostString());
+        }
         // don't serialize scope ids over the network!!!!
         // these only make sense with respect to the local machine, and will only formulate
         // the address incorrectly remotely.
@@ -90,7 +110,7 @@ public boolean isLoopbackOrLinkLocalAddress() {
 
     @Override
     public String getHost() {
-       return getAddress(); // just delegate no resolving
+       return address.getHostString(); // just delegate no resolving done by getHostString
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/common/transport/LocalTransportAddress.java b/core/src/main/java/org/elasticsearch/common/transport/LocalTransportAddress.java
index 48becc832daab..a6af5ed1bfcf3 100644
--- a/core/src/main/java/org/elasticsearch/common/transport/LocalTransportAddress.java
+++ b/core/src/main/java/org/elasticsearch/common/transport/LocalTransportAddress.java
@@ -53,6 +53,13 @@ public LocalTransportAddress(StreamInput in) throws IOException {
         id = in.readString();
     }
 
+    /**
+     * Same as {@link LocalTransportAddress(StreamInput)} but accepts the second argument
+     */
+    public LocalTransportAddress(StreamInput in, String hostString) throws IOException {
+        this(in);
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(id);
diff --git a/core/src/main/java/org/elasticsearch/common/transport/PortsRange.java b/core/src/main/java/org/elasticsearch/common/transport/PortsRange.java
index f88f1de8fe0d5..ffc87dab783fe 100644
--- a/core/src/main/java/org/elasticsearch/common/transport/PortsRange.java
+++ b/core/src/main/java/org/elasticsearch/common/transport/PortsRange.java
@@ -83,4 +83,11 @@ public boolean iterate(PortCallback callback) throws NumberFormatException {
     public interface PortCallback {
         boolean onPortNumber(int portNumber);
     }
+
+    @Override
+    public String toString() {
+        return "PortsRange{" +
+            "portRange='" + portRange + '\'' +
+            '}';
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/transport/TransportAddress.java b/core/src/main/java/org/elasticsearch/common/transport/TransportAddress.java
index 08e8af2bffe57..1467caf19eb67 100644
--- a/core/src/main/java/org/elasticsearch/common/transport/TransportAddress.java
+++ b/core/src/main/java/org/elasticsearch/common/transport/TransportAddress.java
@@ -21,7 +21,6 @@
 
 import org.elasticsearch.common.io.stream.Writeable;
 
-
 /**
  *
  */
@@ -49,4 +48,5 @@ public interface TransportAddress extends Writeable {
     boolean isLoopbackOrLinkLocalAddress();
 
     String toString();
+
 }
diff --git a/core/src/main/java/org/elasticsearch/common/transport/TransportAddressSerializers.java b/core/src/main/java/org/elasticsearch/common/transport/TransportAddressSerializers.java
index 784bee52d63e3..5d5cf62720b8a 100644
--- a/core/src/main/java/org/elasticsearch/common/transport/TransportAddressSerializers.java
+++ b/core/src/main/java/org/elasticsearch/common/transport/TransportAddressSerializers.java
@@ -21,7 +21,6 @@
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Writeable;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -33,17 +32,17 @@
  * A global registry of all supported types of {@link TransportAddress}s. This registry is not open for modification by plugins.
  */
 public abstract class TransportAddressSerializers {
-    private static final Map<Short, Writeable.Reader<TransportAddress>> ADDRESS_REGISTRY;
+    private static final Map<Short, CheckedBiFunction<StreamInput, String, TransportAddress>> ADDRESS_REGISTRY;
 
     static {
-        Map<Short, Writeable.Reader<TransportAddress>> registry = new HashMap<>();
+        Map<Short, CheckedBiFunction<StreamInput, String, TransportAddress>> registry = new HashMap<>();
         addAddressType(registry, InetSocketTransportAddress.TYPE_ID, InetSocketTransportAddress::new);
         addAddressType(registry, LocalTransportAddress.TYPE_ID, LocalTransportAddress::new);
         ADDRESS_REGISTRY = unmodifiableMap(registry);
     }
 
-    private static void addAddressType(Map<Short, Writeable.Reader<TransportAddress>> registry, short uniqueAddressTypeId,
-            Writeable.Reader<TransportAddress> address) {
+    private static void addAddressType(Map<Short, CheckedBiFunction<StreamInput, String, TransportAddress>> registry,
+                                       short uniqueAddressTypeId, CheckedBiFunction<StreamInput, String, TransportAddress> address) {
         if (registry.containsKey(uniqueAddressTypeId)) {
             throw new IllegalStateException("Address [" + uniqueAddressTypeId + "] already bound");
         }
@@ -51,17 +50,35 @@ private static void addAddressType(Map<Short, Writeable.Reader<TransportAddress>
     }
 
     public static TransportAddress addressFromStream(StreamInput input) throws IOException {
+        return addressFromStream(input, null);
+    }
+
+    public static TransportAddress addressFromStream(StreamInput input, String hostString) throws IOException {
         // TODO why don't we just use named writeables here?
         short addressUniqueId = input.readShort();
-        Writeable.Reader<TransportAddress> addressType = ADDRESS_REGISTRY.get(addressUniqueId);
+        CheckedBiFunction<StreamInput, String, TransportAddress> addressType = ADDRESS_REGISTRY.get(addressUniqueId);
         if (addressType == null) {
             throw new IOException("No transport address mapped to [" + addressUniqueId + "]");
         }
-        return addressType.read(input);
+        return addressType.apply(input, hostString);
     }
 
     public static void addressToStream(StreamOutput out, TransportAddress address) throws IOException {
         out.writeShort(address.uniqueAddressTypeId());
         address.writeTo(out);
     }
+
+    /** A BiFuntion that can throw an IOException */
+    @FunctionalInterface
+    interface CheckedBiFunction<T, U, R> {
+
+        /**
+         * Applies this function to the given arguments.
+         *
+         * @param t the first function argument
+         * @param u the second function argument
+         * @return the function result
+         */
+        R apply(T t, U u) throws IOException;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/unit/ByteSizeUnit.java b/core/src/main/java/org/elasticsearch/common/unit/ByteSizeUnit.java
index 4a159957d5e36..e7e43b6d78ae5 100644
--- a/core/src/main/java/org/elasticsearch/common/unit/ByteSizeUnit.java
+++ b/core/src/main/java/org/elasticsearch/common/unit/ByteSizeUnit.java
@@ -19,16 +19,20 @@
 
 package org.elasticsearch.common.unit;
 
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+
+import java.io.IOException;
+
 /**
  * A <tt>SizeUnit</tt> represents size at a given unit of
  * granularity and provides utility methods to convert across units.
  * A <tt>SizeUnit</tt> does not maintain size information, but only
  * helps organize and use size representations that may be maintained
  * separately across various contexts.
- *
- *
  */
-public enum ByteSizeUnit {
+public enum ByteSizeUnit implements Writeable {
     BYTES {
         @Override
         public long toBytes(long size) {
@@ -225,6 +229,13 @@ public long toPB(long size) {
 
     static final long MAX = Long.MAX_VALUE;
 
+    public static ByteSizeUnit fromId(int id) {
+        if (id < 0 || id >= values().length) {
+            throw new IllegalArgumentException("No byte size unit found for id [" + id + "]");
+        }
+        return values()[id];
+    }
+
     /**
      * Scale d by m, checking for overflow.
      * This has a short name to make above code more readable.
@@ -235,7 +246,6 @@ static long x(long d, long m, long over) {
         return d * m;
     }
 
-
     public abstract long toBytes(long size);
 
     public abstract long toKB(long size);
@@ -247,4 +257,16 @@ static long x(long d, long m, long over) {
     public abstract long toTB(long size);
 
     public abstract long toPB(long size);
-}
\ No newline at end of file
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeVInt(this.ordinal());
+    }
+
+    /**
+     * Reads a {@link ByteSizeUnit} from a given {@link StreamInput}
+     */
+    public static ByteSizeUnit readFrom(StreamInput in) throws IOException {
+        return ByteSizeUnit.fromId(in.readVInt());
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/common/unit/ByteSizeValue.java b/core/src/main/java/org/elasticsearch/common/unit/ByteSizeValue.java
index 5944a8e06e6f6..e0782e32cae4e 100644
--- a/core/src/main/java/org/elasticsearch/common/unit/ByteSizeValue.java
+++ b/core/src/main/java/org/elasticsearch/common/unit/ByteSizeValue.java
@@ -29,145 +29,101 @@
 import java.util.Locale;
 import java.util.Objects;
 
-public class ByteSizeValue implements Writeable {
+public class ByteSizeValue implements Writeable, Comparable<ByteSizeValue> {
 
     private final long size;
-    private final ByteSizeUnit sizeUnit;
+    private final ByteSizeUnit unit;
 
     public ByteSizeValue(StreamInput in) throws IOException {
         size = in.readVLong();
-        sizeUnit = ByteSizeUnit.BYTES;
+        unit = ByteSizeUnit.BYTES;
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        out.writeVLong(bytes());
+        out.writeVLong(getBytes());
     }
 
     public ByteSizeValue(long bytes) {
         this(bytes, ByteSizeUnit.BYTES);
     }
 
-    public ByteSizeValue(long size, ByteSizeUnit sizeUnit) {
+    public ByteSizeValue(long size, ByteSizeUnit unit) {
         this.size = size;
-        this.sizeUnit = sizeUnit;
+        this.unit = unit;
     }
 
     public int bytesAsInt() {
-        long bytes = bytes();
+        long bytes = getBytes();
         if (bytes > Integer.MAX_VALUE) {
             throw new IllegalArgumentException("size [" + toString() + "] is bigger than max int");
         }
         return (int) bytes;
     }
 
-    public long bytes() {
-        return sizeUnit.toBytes(size);
-    }
-
     public long getBytes() {
-        return bytes();
-    }
-
-    public long kb() {
-        return sizeUnit.toKB(size);
+        return unit.toBytes(size);
     }
 
     public long getKb() {
-        return kb();
-    }
-
-    public long mb() {
-        return sizeUnit.toMB(size);
+        return unit.toKB(size);
     }
 
     public long getMb() {
-        return mb();
-    }
-
-    public long gb() {
-        return sizeUnit.toGB(size);
+        return unit.toMB(size);
     }
 
     public long getGb() {
-        return gb();
-    }
-
-    public long tb() {
-        return sizeUnit.toTB(size);
+        return unit.toGB(size);
     }
 
     public long getTb() {
-        return tb();
-    }
-
-    public long pb() {
-        return sizeUnit.toPB(size);
+        return unit.toTB(size);
     }
 
     public long getPb() {
-        return pb();
-    }
-
-    public double kbFrac() {
-        return ((double) bytes()) / ByteSizeUnit.C1;
+        return unit.toPB(size);
     }
 
     public double getKbFrac() {
-        return kbFrac();
-    }
-
-    public double mbFrac() {
-        return ((double) bytes()) / ByteSizeUnit.C2;
+        return ((double) getBytes()) / ByteSizeUnit.C1;
     }
 
     public double getMbFrac() {
-        return mbFrac();
-    }
-
-    public double gbFrac() {
-        return ((double) bytes()) / ByteSizeUnit.C3;
+        return ((double) getBytes()) / ByteSizeUnit.C2;
     }
 
     public double getGbFrac() {
-        return gbFrac();
-    }
-
-    public double tbFrac() {
-        return ((double) bytes()) / ByteSizeUnit.C4;
+        return ((double) getBytes()) / ByteSizeUnit.C3;
     }
 
     public double getTbFrac() {
-        return tbFrac();
-    }
-
-    public double pbFrac() {
-        return ((double) bytes()) / ByteSizeUnit.C5;
+        return ((double) getBytes()) / ByteSizeUnit.C4;
     }
 
     public double getPbFrac() {
-        return pbFrac();
+        return ((double) getBytes()) / ByteSizeUnit.C5;
     }
 
     @Override
     public String toString() {
-        long bytes = bytes();
+        long bytes = getBytes();
         double value = bytes;
         String suffix = "b";
         if (bytes >= ByteSizeUnit.C5) {
-            value = pbFrac();
+            value = getPbFrac();
             suffix = "pb";
         } else if (bytes >= ByteSizeUnit.C4) {
-            value = tbFrac();
+            value = getTbFrac();
             suffix = "tb";
         } else if (bytes >= ByteSizeUnit.C3) {
-            value = gbFrac();
+            value = getGbFrac();
             suffix = "gb";
         } else if (bytes >= ByteSizeUnit.C2) {
-            value = mbFrac();
+            value = getMbFrac();
             suffix = "mb";
         } else if (bytes >= ByteSizeUnit.C1) {
-            value = kbFrac();
+            value = getKbFrac();
             suffix = "kb";
         }
         return Strings.format1Decimals(value, suffix);
@@ -235,15 +191,18 @@ public boolean equals(Object o) {
             return false;
         }
 
-        ByteSizeValue sizeValue = (ByteSizeValue) o;
-
-        return bytes() == sizeValue.bytes();
+        return compareTo((ByteSizeValue) o) == 0;
     }
 
     @Override
     public int hashCode() {
-        int result = Long.hashCode(size);
-        result = 31 * result + (sizeUnit != null ? sizeUnit.hashCode() : 0);
-        return result;
+        return Double.hashCode(((double) size) * unit.toBytes(1));
+    }
+
+    @Override
+    public int compareTo(ByteSizeValue other) {
+        double thisValue = ((double) size) * unit.toBytes(1);
+        double otherValue = ((double) other.size) * other.unit.toBytes(1);
+        return Double.compare(thisValue, otherValue);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/unit/MemorySizeValue.java b/core/src/main/java/org/elasticsearch/common/unit/MemorySizeValue.java
index 6a99e06ac01ae..2830d8318a343 100644
--- a/core/src/main/java/org/elasticsearch/common/unit/MemorySizeValue.java
+++ b/core/src/main/java/org/elasticsearch/common/unit/MemorySizeValue.java
@@ -42,7 +42,7 @@ public static ByteSizeValue parseBytesSizeValueOrHeapRatio(String sValue, String
                 if (percent < 0 || percent > 100) {
                     throw new ElasticsearchParseException("percentage should be in [0-100], got [{}]", percentAsString);
                 }
-                return new ByteSizeValue((long) ((percent / 100) * JvmInfo.jvmInfo().getMem().getHeapMax().bytes()), ByteSizeUnit.BYTES);
+                return new ByteSizeValue((long) ((percent / 100) * JvmInfo.jvmInfo().getMem().getHeapMax().getBytes()), ByteSizeUnit.BYTES);
             } catch (NumberFormatException e) {
                 throw new ElasticsearchParseException("failed to parse [{}] as a double", e, percentAsString);
             }
diff --git a/core/src/main/java/org/elasticsearch/common/unit/SizeValue.java b/core/src/main/java/org/elasticsearch/common/unit/SizeValue.java
index cba51f29eeb4a..0f90582007bfd 100644
--- a/core/src/main/java/org/elasticsearch/common/unit/SizeValue.java
+++ b/core/src/main/java/org/elasticsearch/common/unit/SizeValue.java
@@ -27,7 +27,7 @@
 
 import java.io.IOException;
 
-public class SizeValue implements Writeable {
+public class SizeValue implements Writeable, Comparable<SizeValue> {
 
     private final long size;
     private final SizeUnit sizeUnit;
@@ -201,18 +201,18 @@ public boolean equals(Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
 
-        SizeValue sizeValue = (SizeValue) o;
-
-        if (size != sizeValue.size) return false;
-        if (sizeUnit != sizeValue.sizeUnit) return false;
-
-        return true;
+        return compareTo((SizeValue) o) == 0;
     }
 
     @Override
     public int hashCode() {
-        int result = Long.hashCode(size);
-        result = 31 * result + (sizeUnit != null ? sizeUnit.hashCode() : 0);
-        return result;
+        return Double.hashCode(((double) size) * sizeUnit.toSingles(1));
+    }
+
+    @Override
+    public int compareTo(SizeValue other) {
+        double thisValue = ((double) size) * sizeUnit.toSingles(1);
+        double otherValue = ((double) other.size) * other.sizeUnit.toSingles(1);
+        return Double.compare(thisValue, otherValue);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/unit/TimeValue.java b/core/src/main/java/org/elasticsearch/common/unit/TimeValue.java
index ed67019c10399..0f6eabed1e3de 100644
--- a/core/src/main/java/org/elasticsearch/common/unit/TimeValue.java
+++ b/core/src/main/java/org/elasticsearch/common/unit/TimeValue.java
@@ -31,6 +31,7 @@
 
 import java.io.IOException;
 import java.util.Collections;
+import java.util.EnumMap;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Locale;
@@ -39,7 +40,7 @@
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
-public class TimeValue implements Writeable {
+public class TimeValue implements Writeable, Comparable<TimeValue> {
 
     /** How many nano-seconds in one milli-second */
     public static final long NSEC_PER_MSEC = TimeUnit.NANOSECONDS.convert(1, TimeUnit.MILLISECONDS);
@@ -48,7 +49,7 @@ public class TimeValue implements Writeable {
     private static Map<Byte, TimeUnit> BYTE_TIME_UNIT_MAP;
 
     static {
-        final Map<TimeUnit, Byte> timeUnitByteMap = new HashMap<>();
+        final Map<TimeUnit, Byte> timeUnitByteMap = new EnumMap<>(TimeUnit.class);
         timeUnitByteMap.put(TimeUnit.NANOSECONDS, (byte)0);
         timeUnitByteMap.put(TimeUnit.MICROSECONDS, (byte)1);
         timeUnitByteMap.put(TimeUnit.MILLISECONDS, (byte)2);
@@ -249,6 +250,12 @@ public String format(PeriodType type) {
         return PeriodFormat.getDefault().withParseType(type).print(period);
     }
 
+    /**
+     * Returns a {@link String} representation of the current {@link TimeValue}.
+     *
+     * Note that this method might produce fractional time values (ex 1.6m) which cannot be
+     * parsed by method like {@link TimeValue#parse(String, String, String)}.
+     */
     @Override
     public String toString() {
         if (duration < 0) {
@@ -319,22 +326,20 @@ public static TimeValue parseTimeValue(String sValue, TimeValue defaultValue, St
         }
         final String normalized = sValue.toLowerCase(Locale.ROOT).trim();
         if (normalized.endsWith("nanos")) {
-            return new TimeValue(parse(sValue, normalized, 5), TimeUnit.NANOSECONDS);
+            return new TimeValue(parse(sValue, normalized, "nanos"), TimeUnit.NANOSECONDS);
         } else if (normalized.endsWith("micros")) {
-            return new TimeValue(parse(sValue, normalized, 6), TimeUnit.MICROSECONDS);
+            return new TimeValue(parse(sValue, normalized, "micros"), TimeUnit.MICROSECONDS);
         } else if (normalized.endsWith("ms")) {
-            return new TimeValue(parse(sValue, normalized, 2), TimeUnit.MILLISECONDS);
+            return new TimeValue(parse(sValue, normalized, "ms"), TimeUnit.MILLISECONDS);
         } else if (normalized.endsWith("s")) {
-            return new TimeValue(parse(sValue, normalized, 1), TimeUnit.SECONDS);
+            return new TimeValue(parse(sValue, normalized, "s"), TimeUnit.SECONDS);
         } else if (sValue.endsWith("m")) {
-            // parsing minutes should be case sensitive as `M` is generally
-            // accepted to mean months not minutes. This is the only case where
-            // the upper and lower case forms indicate different time units
-            return new TimeValue(parse(sValue, normalized, 1), TimeUnit.MINUTES);
+            // parsing minutes should be case-sensitive as 'M' means "months", not "minutes"; this is the only special case.
+            return new TimeValue(parse(sValue, normalized, "m"), TimeUnit.MINUTES);
         } else if (normalized.endsWith("h")) {
-            return new TimeValue(parse(sValue, normalized, 1), TimeUnit.HOURS);
+            return new TimeValue(parse(sValue, normalized, "h"), TimeUnit.HOURS);
         } else if (normalized.endsWith("d")) {
-            return new TimeValue(parse(sValue, normalized, 1), TimeUnit.DAYS);
+            return new TimeValue(parse(sValue, normalized, "d"), TimeUnit.DAYS);
         } else if (normalized.matches("-0*1")) {
             return TimeValue.MINUS_ONE;
         } else if (normalized.matches("0+")) {
@@ -348,8 +353,8 @@ public static TimeValue parseTimeValue(String sValue, TimeValue defaultValue, St
         }
     }
 
-    private static long parse(final String initialInput, final String normalized, final int suffixLength) {
-        final String s = normalized.substring(0, normalized.length() - suffixLength).trim();
+    private static long parse(final String initialInput, final String normalized, final String suffix) {
+        final String s = normalized.substring(0, normalized.length() - suffix.length()).trim();
         try {
             return Long.parseLong(s);
         } catch (final NumberFormatException e) {
@@ -375,17 +380,22 @@ public boolean equals(Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
 
-        TimeValue timeValue = (TimeValue) o;
-        return timeUnit.toNanos(duration) == timeValue.timeUnit.toNanos(timeValue.duration);
+        return this.compareTo(((TimeValue) o)) == 0;
     }
 
     @Override
     public int hashCode() {
-        long normalized = timeUnit.toNanos(duration);
-        return Long.hashCode(normalized);
+        return Double.hashCode(((double) duration) * timeUnit.toNanos(1));
     }
 
     public static long nsecToMSec(long ns) {
         return ns / NSEC_PER_MSEC;
     }
+
+    @Override
+    public int compareTo(TimeValue timeValue) {
+        double thisValue = ((double) duration) * timeUnit.toNanos(1);
+        double otherValue = ((double) timeValue.duration) * timeValue.timeUnit.toNanos(1);
+        return Double.compare(thisValue, otherValue);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/util/AbstractArray.java b/core/src/main/java/org/elasticsearch/common/util/AbstractArray.java
index 1187dfef7e5a0..6a4895c7950ad 100644
--- a/core/src/main/java/org/elasticsearch/common/util/AbstractArray.java
+++ b/core/src/main/java/org/elasticsearch/common/util/AbstractArray.java
@@ -23,13 +23,14 @@
 
 import java.util.Collection;
 import java.util.Collections;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 
 abstract class AbstractArray implements BigArray {
 
     private final BigArrays bigArrays;
     public final boolean clearOnResize;
-    private boolean released = false;
+    private final AtomicBoolean closed = new AtomicBoolean(false);
 
     AbstractArray(BigArrays bigArrays, boolean clearOnResize) {
         this.bigArrays = bigArrays;
@@ -38,10 +39,13 @@ abstract class AbstractArray implements BigArray {
 
     @Override
     public final void close() {
-        bigArrays.adjustBreaker(-ramBytesUsed());
-        assert !released : "double release";
-        released = true;
-        doClose();
+        if (closed.compareAndSet(false, true)) {
+            try {
+                bigArrays.adjustBreaker(-ramBytesUsed(), true);
+            } finally {
+                doClose();
+            }
+        }
     }
 
     protected abstract void doClose();
diff --git a/core/src/main/java/org/elasticsearch/common/util/AbstractBigArray.java b/core/src/main/java/org/elasticsearch/common/util/AbstractBigArray.java
index f26dad1fdb5af..73a05f7f2cfd6 100644
--- a/core/src/main/java/org/elasticsearch/common/util/AbstractBigArray.java
+++ b/core/src/main/java/org/elasticsearch/common/util/AbstractBigArray.java
@@ -87,6 +87,11 @@ public final long size() {
 
     @Override
     public final long ramBytesUsed() {
+        return ramBytesEstimated(size);
+    }
+
+    /** Given the size of the array, estimate the number of bytes it will use. */
+    public final long ramBytesEstimated(final long size) {
         // rough approximate, we only take into account the size of the values, not the overhead of the array objects
         return ((long) pageIndex(size - 1) + 1) * pageSize() * numBytesPerElement();
     }
diff --git a/core/src/main/java/org/elasticsearch/common/util/ArrayUtils.java b/core/src/main/java/org/elasticsearch/common/util/ArrayUtils.java
index bb8442efa08c0..de1d665f7c6d8 100644
--- a/core/src/main/java/org/elasticsearch/common/util/ArrayUtils.java
+++ b/core/src/main/java/org/elasticsearch/common/util/ArrayUtils.java
@@ -87,5 +87,4 @@ public static <T> T[] concat(T[] one, T[] other, Class<T> clazz) {
         System.arraycopy(other, 0, target, one.length, other.length);
         return target;
     }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/common/util/BigArrays.java b/core/src/main/java/org/elasticsearch/common/util/BigArrays.java
index 6a15a3d90009b..5c539a791cf6b 100644
--- a/core/src/main/java/org/elasticsearch/common/util/BigArrays.java
+++ b/core/src/main/java/org/elasticsearch/common/util/BigArrays.java
@@ -25,7 +25,6 @@
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.breaker.CircuitBreakingException;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.recycler.Recycler;
@@ -91,7 +90,7 @@ public void close() {
 
     private abstract static class AbstractArrayWrapper extends AbstractArray implements BigArray {
 
-        protected static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(ByteArrayWrapper.class);
+        static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(ByteArrayWrapper.class);
 
         private final Releasable releasable;
         private final long size;
@@ -377,6 +376,7 @@ public BigArrays(Settings settings, @Nullable final CircuitBreakerService breake
         // Checking the breaker is disabled if not specified
         this(new PageCacheRecycler(settings), breakerService, false);
     }
+
     // public for tests
     public BigArrays(PageCacheRecycler recycler, @Nullable final CircuitBreakerService breakerService, boolean checkBreaker) {
         this.checkBreaker = checkBreaker;
@@ -392,21 +392,26 @@ public BigArrays(PageCacheRecycler recycler, @Nullable final CircuitBreakerServi
     /**
      * Adjust the circuit breaker with the given delta, if the delta is
      * negative, or checkBreaker is false, the breaker will be adjusted
-     * without tripping
+     * without tripping.  If the data was already created before calling
+     * this method, and the breaker trips, we add the delta without breaking
+     * to account for the created data.  If the data has not been created yet,
+     * we do not add the delta to the breaker if it trips.
      */
-    void adjustBreaker(long delta) {
+    void adjustBreaker(final long delta, final boolean isDataAlreadyCreated) {
         if (this.breakerService != null) {
             CircuitBreaker breaker = this.breakerService.getBreaker(CircuitBreaker.REQUEST);
-            if (this.checkBreaker == true) {
+            if (this.checkBreaker) {
                 // checking breaker means potentially tripping, but it doesn't
                 // have to if the delta is negative
                 if (delta > 0) {
                     try {
                         breaker.addEstimateBytesAndMaybeBreak(delta, "<reused_arrays>");
                     } catch (CircuitBreakingException e) {
-                        // since we've already created the data, we need to
-                        // add it so closing the stream re-adjusts properly
-                        breaker.addWithoutBreaking(delta);
+                        if (isDataAlreadyCreated) {
+                            // since we've already created the data, we need to
+                            // add it so closing the stream re-adjusts properly
+                            breaker.addWithoutBreaking(delta);
+                        }
                         // re-throw the original exception
                         throw e;
                     }
@@ -435,15 +440,20 @@ public CircuitBreakerService breakerService() {
 
     private <T extends AbstractBigArray> T resizeInPlace(T array, long newSize) {
         final long oldMemSize = array.ramBytesUsed();
+        final long oldSize = array.size();
+        assert oldMemSize == array.ramBytesEstimated(oldSize) :
+            "ram bytes used should equal that which was previously estimated: ramBytesUsed=" +
+            oldMemSize + ", ramBytesEstimated=" + array.ramBytesEstimated(oldSize);
+        final long estimatedIncreaseInBytes = array.ramBytesEstimated(newSize) - oldMemSize;
+        adjustBreaker(estimatedIncreaseInBytes, false);
         array.resize(newSize);
-        adjustBreaker(array.ramBytesUsed() - oldMemSize);
         return array;
     }
 
     private <T extends BigArray> T validate(T array) {
         boolean success = false;
         try {
-            adjustBreaker(array.ramBytesUsed());
+            adjustBreaker(array.ramBytesUsed(), true);
             success = true;
         } finally {
             if (!success) {
@@ -459,16 +469,17 @@ private <T extends BigArray> T validate(T array) {
      * @param clearOnResize whether values should be set to 0 on initialization and resize
      */
     public ByteArray newByteArray(long size, boolean clearOnResize) {
-        final ByteArray array;
         if (size > BYTE_PAGE_SIZE) {
-            array = new BigByteArray(size, this, clearOnResize);
+            // when allocating big arrays, we want to first ensure we have the capacity by
+            // checking with the circuit breaker before attempting to allocate
+            adjustBreaker(BigByteArray.estimateRamBytes(size), false);
+            return new BigByteArray(size, this, clearOnResize);
         } else if (size >= BYTE_PAGE_SIZE / 2 && recycler != null) {
             final Recycler.V<byte[]> page = recycler.bytePage(clearOnResize);
-            array = new ByteArrayWrapper(this, page.v(), size, page, clearOnResize);
+            return validate(new ByteArrayWrapper(this, page.v(), size, page, clearOnResize));
         } else {
-            array = new ByteArrayWrapper(this, new byte[(int) size], size, null, clearOnResize);
+            return validate(new ByteArrayWrapper(this, new byte[(int) size], size, null, clearOnResize));
         }
-        return validate(array);
     }
 
     /**
@@ -541,16 +552,17 @@ public boolean equals(ByteArray array, ByteArray other) {
      * @param clearOnResize whether values should be set to 0 on initialization and resize
      */
     public IntArray newIntArray(long size, boolean clearOnResize) {
-        final IntArray array;
         if (size > INT_PAGE_SIZE) {
-            array = new BigIntArray(size, this, clearOnResize);
+            // when allocating big arrays, we want to first ensure we have the capacity by
+            // checking with the circuit breaker before attempting to allocate
+            adjustBreaker(BigIntArray.estimateRamBytes(size), false);
+            return new BigIntArray(size, this, clearOnResize);
         } else if (size >= INT_PAGE_SIZE / 2 && recycler != null) {
             final Recycler.V<int[]> page = recycler.intPage(clearOnResize);
-            array = new IntArrayWrapper(this, page.v(), size, page, clearOnResize);
+            return validate(new IntArrayWrapper(this, page.v(), size, page, clearOnResize));
         } else {
-            array = new IntArrayWrapper(this, new int[(int) size], size, null, clearOnResize);
+            return validate(new IntArrayWrapper(this, new int[(int) size], size, null, clearOnResize));
         }
-        return validate(array);
     }
 
     /**
@@ -591,16 +603,17 @@ public IntArray grow(IntArray array, long minSize) {
      * @param clearOnResize whether values should be set to 0 on initialization and resize
      */
     public LongArray newLongArray(long size, boolean clearOnResize) {
-        final LongArray array;
         if (size > LONG_PAGE_SIZE) {
-            array = new BigLongArray(size, this, clearOnResize);
+            // when allocating big arrays, we want to first ensure we have the capacity by
+            // checking with the circuit breaker before attempting to allocate
+            adjustBreaker(BigLongArray.estimateRamBytes(size), false);
+            return new BigLongArray(size, this, clearOnResize);
         } else if (size >= LONG_PAGE_SIZE / 2 && recycler != null) {
             final Recycler.V<long[]> page = recycler.longPage(clearOnResize);
-            array = new LongArrayWrapper(this, page.v(), size, page, clearOnResize);
+            return validate(new LongArrayWrapper(this, page.v(), size, page, clearOnResize));
         } else {
-            array = new LongArrayWrapper(this, new long[(int) size], size, null, clearOnResize);
+            return validate(new LongArrayWrapper(this, new long[(int) size], size, null, clearOnResize));
         }
-        return validate(array);
     }
 
     /**
@@ -641,16 +654,17 @@ public LongArray grow(LongArray array, long minSize) {
      * @param clearOnResize whether values should be set to 0 on initialization and resize
      */
     public DoubleArray newDoubleArray(long size, boolean clearOnResize) {
-        final DoubleArray arr;
         if (size > LONG_PAGE_SIZE) {
-            arr = new BigDoubleArray(size, this, clearOnResize);
+            // when allocating big arrays, we want to first ensure we have the capacity by
+            // checking with the circuit breaker before attempting to allocate
+            adjustBreaker(BigDoubleArray.estimateRamBytes(size), false);
+            return new BigDoubleArray(size, this, clearOnResize);
         } else if (size >= LONG_PAGE_SIZE / 2 && recycler != null) {
             final Recycler.V<long[]> page = recycler.longPage(clearOnResize);
-            arr = new DoubleArrayWrapper(this, page.v(), size, page, clearOnResize);
+            return validate(new DoubleArrayWrapper(this, page.v(), size, page, clearOnResize));
         } else {
-            arr = new DoubleArrayWrapper(this, new long[(int) size], size, null, clearOnResize);
+            return validate(new DoubleArrayWrapper(this, new long[(int) size], size, null, clearOnResize));
         }
-        return validate(arr);
     }
 
     /** Allocate a new {@link DoubleArray} of the given capacity. */
@@ -688,16 +702,17 @@ public DoubleArray grow(DoubleArray array, long minSize) {
      * @param clearOnResize whether values should be set to 0 on initialization and resize
      */
     public FloatArray newFloatArray(long size, boolean clearOnResize) {
-        final FloatArray array;
         if (size > INT_PAGE_SIZE) {
-            array = new BigFloatArray(size, this, clearOnResize);
+            // when allocating big arrays, we want to first ensure we have the capacity by
+            // checking with the circuit breaker before attempting to allocate
+            adjustBreaker(BigFloatArray.estimateRamBytes(size), false);
+            return new BigFloatArray(size, this, clearOnResize);
         } else if (size >= INT_PAGE_SIZE / 2 && recycler != null) {
             final Recycler.V<int[]> page = recycler.intPage(clearOnResize);
-            array = new FloatArrayWrapper(this, page.v(), size, page, clearOnResize);
+            return validate(new FloatArrayWrapper(this, page.v(), size, page, clearOnResize));
         } else {
-            array = new FloatArrayWrapper(this, new int[(int) size], size, null, clearOnResize);
+            return validate(new FloatArrayWrapper(this, new int[(int) size], size, null, clearOnResize));
         }
-        return validate(array);
     }
 
     /** Allocate a new {@link FloatArray} of the given capacity. */
@@ -736,14 +751,16 @@ public FloatArray grow(FloatArray array, long minSize) {
     public <T> ObjectArray<T> newObjectArray(long size) {
         final ObjectArray<T> array;
         if (size > OBJECT_PAGE_SIZE) {
-            array = new BigObjectArray<>(size, this);
+            // when allocating big arrays, we want to first ensure we have the capacity by
+            // checking with the circuit breaker before attempting to allocate
+            adjustBreaker(BigObjectArray.estimateRamBytes(size), false);
+            return new BigObjectArray<>(size, this);
         } else if (size >= OBJECT_PAGE_SIZE / 2 && recycler != null) {
             final Recycler.V<Object[]> page = recycler.objectPage();
-            array = new ObjectArrayWrapper<>(this, page.v(), size, page);
+            return validate(new ObjectArrayWrapper<>(this, page.v(), size, page));
         } else {
-            array = new ObjectArrayWrapper<>(this, new Object[(int) size], size, null);
+            return validate(new ObjectArrayWrapper<>(this, new Object[(int) size], size, null));
         }
-        return validate(array);
     }
 
     /** Resize the array to the exact provided size. */
diff --git a/core/src/main/java/org/elasticsearch/common/util/BigByteArray.java b/core/src/main/java/org/elasticsearch/common/util/BigByteArray.java
index cac3132385f6c..789e6dc6bbaf6 100644
--- a/core/src/main/java/org/elasticsearch/common/util/BigByteArray.java
+++ b/core/src/main/java/org/elasticsearch/common/util/BigByteArray.java
@@ -33,10 +33,12 @@
  */
 final class BigByteArray extends AbstractBigArray implements ByteArray {
 
+    private static final BigByteArray ESTIMATOR = new BigByteArray(0, BigArrays.NON_RECYCLING_INSTANCE, false);
+
     private byte[][] pages;
 
     /** Constructor. */
-    public BigByteArray(long size, BigArrays bigArrays, boolean clearOnResize) {
+    BigByteArray(long size, BigArrays bigArrays, boolean clearOnResize) {
         super(BYTE_PAGE_SIZE, bigArrays, clearOnResize);
         this.size = size;
         pages = new byte[numPages(size)][];
@@ -44,7 +46,7 @@ public BigByteArray(long size, BigArrays bigArrays, boolean clearOnResize) {
             pages[i] = newBytePage(i);
         }
     }
-    
+
     @Override
     public byte get(long index) {
         final int pageIndex = pageIndex(index);
@@ -147,4 +149,9 @@ public void resize(long newSize) {
         this.size = newSize;
     }
 
+    /** Estimates the number of bytes that would be consumed by an array of the given size. */
+    public static long estimateRamBytes(final long size) {
+        return ESTIMATOR.ramBytesEstimated(size);
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/common/util/BigDoubleArray.java b/core/src/main/java/org/elasticsearch/common/util/BigDoubleArray.java
index 4aab593affe6d..a2c770ee9958d 100644
--- a/core/src/main/java/org/elasticsearch/common/util/BigDoubleArray.java
+++ b/core/src/main/java/org/elasticsearch/common/util/BigDoubleArray.java
@@ -32,10 +32,12 @@
  */
 final class BigDoubleArray extends AbstractBigArray implements DoubleArray {
 
+    private static final BigDoubleArray ESTIMATOR = new BigDoubleArray(0, BigArrays.NON_RECYCLING_INSTANCE, false);
+
     private long[][] pages;
 
     /** Constructor. */
-    public BigDoubleArray(long size, BigArrays bigArrays, boolean clearOnResize) {
+    BigDoubleArray(long size, BigArrays bigArrays, boolean clearOnResize) {
         super(LONG_PAGE_SIZE, bigArrays, clearOnResize);
         this.size = size;
         pages = new long[numPages(size)][];
@@ -110,4 +112,9 @@ public void fill(long fromIndex, long toIndex, double value) {
         }
     }
 
+    /** Estimates the number of bytes that would be consumed by an array of the given size. */
+    public static long estimateRamBytes(final long size) {
+        return ESTIMATOR.ramBytesEstimated(size);
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/common/util/BigFloatArray.java b/core/src/main/java/org/elasticsearch/common/util/BigFloatArray.java
index 1fa79a9f3dbe0..b67db2e84de31 100644
--- a/core/src/main/java/org/elasticsearch/common/util/BigFloatArray.java
+++ b/core/src/main/java/org/elasticsearch/common/util/BigFloatArray.java
@@ -32,10 +32,12 @@
  */
 final class BigFloatArray extends AbstractBigArray implements FloatArray {
 
+    private static final BigFloatArray ESTIMATOR = new BigFloatArray(0, BigArrays.NON_RECYCLING_INSTANCE, false);
+
     private int[][] pages;
 
     /** Constructor. */
-    public BigFloatArray(long size, BigArrays bigArrays, boolean clearOnResize) {
+    BigFloatArray(long size, BigArrays bigArrays, boolean clearOnResize) {
         super(INT_PAGE_SIZE, bigArrays, clearOnResize);
         this.size = size;
         pages = new int[numPages(size)][];
@@ -110,4 +112,9 @@ public void fill(long fromIndex, long toIndex, float value) {
         }
     }
 
+    /** Estimates the number of bytes that would be consumed by an array of the given size. */
+    public static long estimateRamBytes(final long size) {
+        return ESTIMATOR.ramBytesEstimated(size);
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/common/util/BigIntArray.java b/core/src/main/java/org/elasticsearch/common/util/BigIntArray.java
index 4ce5fc7aceeaf..d2a1ca3f49c61 100644
--- a/core/src/main/java/org/elasticsearch/common/util/BigIntArray.java
+++ b/core/src/main/java/org/elasticsearch/common/util/BigIntArray.java
@@ -32,10 +32,12 @@
  */
 final class BigIntArray extends AbstractBigArray implements IntArray {
 
+    private static final BigIntArray ESTIMATOR = new BigIntArray(0, BigArrays.NON_RECYCLING_INSTANCE, false);
+
     private int[][] pages;
 
     /** Constructor. */
-    public BigIntArray(long size, BigArrays bigArrays, boolean clearOnResize) {
+    BigIntArray(long size, BigArrays bigArrays, boolean clearOnResize) {
         super(INT_PAGE_SIZE, bigArrays, clearOnResize);
         this.size = size;
         pages = new int[numPages(size)][];
@@ -108,4 +110,9 @@ public void resize(long newSize) {
         this.size = newSize;
     }
 
+    /** Estimates the number of bytes that would be consumed by an array of the given size. */
+    public static long estimateRamBytes(final long size) {
+        return ESTIMATOR.ramBytesEstimated(size);
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/common/util/BigLongArray.java b/core/src/main/java/org/elasticsearch/common/util/BigLongArray.java
index 2e3248143b4e8..69f919382f8e4 100644
--- a/core/src/main/java/org/elasticsearch/common/util/BigLongArray.java
+++ b/core/src/main/java/org/elasticsearch/common/util/BigLongArray.java
@@ -32,10 +32,12 @@
  */
 final class BigLongArray extends AbstractBigArray implements LongArray {
 
+    private static final BigLongArray ESTIMATOR = new BigLongArray(0, BigArrays.NON_RECYCLING_INSTANCE, false);
+
     private long[][] pages;
 
     /** Constructor. */
-    public BigLongArray(long size, BigArrays bigArrays, boolean clearOnResize) {
+    BigLongArray(long size, BigArrays bigArrays, boolean clearOnResize) {
         super(LONG_PAGE_SIZE, bigArrays, clearOnResize);
         this.size = size;
         pages = new long[numPages(size)][];
@@ -111,4 +113,9 @@ public void fill(long fromIndex, long toIndex, long value) {
         }
     }
 
+    /** Estimates the number of bytes that would be consumed by an array of the given size. */
+    public static long estimateRamBytes(final long size) {
+        return ESTIMATOR.ramBytesEstimated(size);
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/common/util/BigObjectArray.java b/core/src/main/java/org/elasticsearch/common/util/BigObjectArray.java
index 19a41d3096da9..1ed012e2bb393 100644
--- a/core/src/main/java/org/elasticsearch/common/util/BigObjectArray.java
+++ b/core/src/main/java/org/elasticsearch/common/util/BigObjectArray.java
@@ -32,10 +32,12 @@
  */
 final class BigObjectArray<T> extends AbstractBigArray implements ObjectArray<T> {
 
+    private static final BigObjectArray ESTIMATOR = new BigObjectArray(0, BigArrays.NON_RECYCLING_INSTANCE);
+
     private Object[][] pages;
 
     /** Constructor. */
-    public BigObjectArray(long size, BigArrays bigArrays) {
+    BigObjectArray(long size, BigArrays bigArrays) {
         super(OBJECT_PAGE_SIZE, bigArrays, true);
         this.size = size;
         pages = new Object[numPages(size)][];
@@ -85,4 +87,9 @@ public void resize(long newSize) {
         this.size = newSize;
     }
 
-}
\ No newline at end of file
+    /** Estimates the number of bytes that would be consumed by an array of the given size. */
+    public static long estimateRamBytes(final long size) {
+        return ESTIMATOR.ramBytesEstimated(size);
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/common/util/Callback.java b/core/src/main/java/org/elasticsearch/common/util/Callback.java
deleted file mode 100644
index d4e3c94f70048..0000000000000
--- a/core/src/main/java/org/elasticsearch/common/util/Callback.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.common.util;
-
-/**
- *
- */
-public interface Callback<T> {
-
-    void handle(T t);
-
-}
diff --git a/core/src/main/java/org/elasticsearch/common/util/CollectionUtils.java b/core/src/main/java/org/elasticsearch/common/util/CollectionUtils.java
index 5171be0ee91a3..54a49f7e4f254 100644
--- a/core/src/main/java/org/elasticsearch/common/util/CollectionUtils.java
+++ b/core/src/main/java/org/elasticsearch/common/util/CollectionUtils.java
@@ -226,7 +226,7 @@ private static class RotatedList<T> extends AbstractList<T> implements RandomAcc
         private final List<T> in;
         private final int distance;
 
-        public RotatedList(List<T> list, int distance) {
+        RotatedList(List<T> list, int distance) {
             if (distance < 0 || distance >= list.size()) {
                 throw new IllegalArgumentException();
             }
diff --git a/core/src/main/java/org/elasticsearch/common/util/ExtensionPoint.java b/core/src/main/java/org/elasticsearch/common/util/ExtensionPoint.java
deleted file mode 100644
index fcdfaafb1d5b9..0000000000000
--- a/core/src/main/java/org/elasticsearch/common/util/ExtensionPoint.java
+++ /dev/null
@@ -1,246 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.common.util;
-
-import org.elasticsearch.common.inject.Binder;
-import org.elasticsearch.common.inject.multibindings.MapBinder;
-import org.elasticsearch.common.inject.multibindings.Multibinder;
-import org.elasticsearch.common.settings.Settings;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * This class defines an official elasticsearch extension point. It registers
- * all extensions by a single name and ensures that extensions are not registered
- * more than once.
- */
-public abstract class ExtensionPoint {
-    protected final String name;
-    protected final Class<?>[] singletons;
-
-    /**
-     * Creates a new extension point
-     *
-     * @param name           the human readable underscore case name of the extension point. This is used in error messages etc.
-     * @param singletons     a list of singletons to bind with this extension point - these are bound in {@link #bind(Binder)}
-     */
-    public ExtensionPoint(String name, Class<?>... singletons) {
-        this.name = name;
-        this.singletons = singletons;
-    }
-
-    /**
-     * Binds the extension as well as the singletons to the given guice binder.
-     *
-     * @param binder the binder to use
-     */
-    public final void bind(Binder binder) {
-        for (Class<?> c : singletons) {
-            binder.bind(c).asEagerSingleton();
-        }
-        bindExtensions(binder);
-    }
-
-    /**
-     * Subclasses can bind their type, map or set extensions here.
-     */
-    protected abstract void bindExtensions(Binder binder);
-
-    /**
-     * A map based extension point which allows to register keyed implementations ie. parsers or some kind of strategies.
-     */
-    public static class ClassMap<T> extends ExtensionPoint {
-        protected final Class<T> extensionClass;
-        protected final Map<String, Class<? extends T>> extensions = new HashMap<>();
-        private final Set<String> reservedKeys;
-
-        /**
-         * Creates a new {@link ClassMap}
-         *
-         * @param name           the human readable underscore case name of the extension point. This is used in error messages etc.
-         * @param extensionClass the base class that should be extended
-         * @param singletons     a list of singletons to bind with this extension point - these are bound in {@link #bind(Binder)}
-         * @param reservedKeys   a set of reserved keys by internal implementations
-         */
-        public ClassMap(String name, Class<T> extensionClass, Set<String> reservedKeys, Class<?>... singletons) {
-            super(name, singletons);
-            this.extensionClass = extensionClass;
-            this.reservedKeys = reservedKeys;
-        }
-
-        /**
-         * Returns the extension for the given key or <code>null</code>
-         */
-        public Class<? extends T> getExtension(String type) {
-            return extensions.get(type);
-        }
-
-        /**
-         * Registers an extension class for a given key. This method will thr
-         *
-         * @param key       the extensions key
-         * @param extension the extension
-         * @throws IllegalArgumentException iff the key is already registered or if the key is a reserved key for an internal implementation
-         */
-        public final void registerExtension(String key, Class<? extends T> extension) {
-            if (extensions.containsKey(key) || reservedKeys.contains(key)) {
-                throw new IllegalArgumentException("Can't register the same [" + this.name + "] more than once for [" + key + "]");
-            }
-            extensions.put(key, extension);
-        }
-
-        @Override
-        protected final void bindExtensions(Binder binder) {
-            MapBinder<String, T> parserMapBinder = MapBinder.newMapBinder(binder, String.class, extensionClass);
-            for (Map.Entry<String, Class<? extends T>> clazz : extensions.entrySet()) {
-                parserMapBinder.addBinding(clazz.getKey()).to(clazz.getValue());
-            }
-        }
-    }
-
-    /**
-     * A Type extension point which basically allows to registered keyed extensions like {@link ClassMap}
-     * but doesn't instantiate and bind all the registered key value pairs but instead replace a singleton based on a given setting via {@link #bindType(Binder, Settings, String, String)}
-     * Note: {@link #bind(Binder)} is not supported by this class
-     */
-    public static final class SelectedType<T> extends ClassMap<T> {
-
-        public SelectedType(String name, Class<T> extensionClass) {
-            super(name, extensionClass, Collections.emptySet());
-        }
-
-        /**
-         * Binds the extension class to the class that is registered for the give configured for the settings key in
-         * the settings object.
-         *
-         * @param binder       the binder to use
-         * @param settings     the settings to look up the key to find the implementation to bind
-         * @param settingsKey  the key to use with the settings
-         * @param defaultValue the default value if the settings do not contain the key, or null if there is no default
-         * @return the actual bound type key
-         */
-        public String bindType(Binder binder, Settings settings, String settingsKey, String defaultValue) {
-            final String type = settings.get(settingsKey, defaultValue);
-            if (type == null) {
-                throw new IllegalArgumentException("Missing setting [" + settingsKey + "]");
-            }
-            final Class<? extends T> instance = getExtension(type);
-            if (instance == null) {
-                throw new IllegalArgumentException("Unknown [" + this.name + "] type [" + type + "] possible values: "
-                    + extensions.keySet());
-            }
-            if (extensionClass == instance) {
-                binder.bind(extensionClass).asEagerSingleton();
-            } else {
-                binder.bind(extensionClass).to(instance).asEagerSingleton();
-            }
-            return type;
-        }
-
-    }
-
-    /**
-     * A set based extension point which allows to register extended classes that might be used to chain additional functionality etc.
-     */
-    public static final class ClassSet<T> extends ExtensionPoint {
-        protected final Class<T> extensionClass;
-        private final Set<Class<? extends T>> extensions = new HashSet<>();
-
-        /**
-         * Creates a new {@link ClassSet}
-         *
-         * @param name           the human readable underscore case name of the extension point. This is used in error messages etc.
-         * @param extensionClass the base class that should be extended
-         * @param singletons     a list of singletons to bind with this extension point - these are bound in {@link #bind(Binder)}
-         */
-        public ClassSet(String name, Class<T> extensionClass, Class<?>... singletons) {
-            super(name, singletons);
-            this.extensionClass = extensionClass;
-        }
-
-        /**
-         * Registers a new extension
-         *
-         * @param extension the extension to register
-         * @throws IllegalArgumentException iff the class is already registered
-         */
-        public void registerExtension(Class<? extends T> extension) {
-            if (extensions.contains(extension)) {
-                throw new IllegalArgumentException("Can't register the same [" + this.name + "] more than once for [" + extension.getName() + "]");
-            }
-            extensions.add(extension);
-        }
-
-        @Override
-        protected void bindExtensions(Binder binder) {
-            Multibinder<T> allocationMultibinder = Multibinder.newSetBinder(binder, extensionClass);
-            for (Class<? extends T> clazz : extensions) {
-                binder.bind(clazz).asEagerSingleton();
-                allocationMultibinder.addBinding().to(clazz);
-            }
-        }
-    }
-
-    /**
-     * A an instance of a map, mapping one instance value to another. Both key and value are instances, not classes
-     * like with other extension points.
-     */
-    public static final class InstanceMap<K, V> extends ExtensionPoint {
-        private final Map<K, V> map = new HashMap<>();
-        private final Class<K> keyType;
-        private final Class<V> valueType;
-
-        /**
-         * Creates a new {@link ClassSet}
-         *
-         * @param name           the human readable underscore case name of the extension point. This is used in error messages.
-         * @param singletons     a list of singletons to bind with this extension point - these are bound in {@link #bind(Binder)}
-         */
-        public InstanceMap(String name, Class<K> keyType, Class<V> valueType, Class<?>... singletons) {
-            super(name, singletons);
-            this.keyType = keyType;
-            this.valueType = valueType;
-        }
-
-        /**
-         * Registers a mapping from {@code key} to {@code value}
-         *
-         * @throws IllegalArgumentException iff the key is already registered
-         */
-        public void registerExtension(K key, V value) {
-            V old = map.put(key, value);
-            if (old != null) {
-                throw new IllegalArgumentException("Cannot register [" + this.name + "] with key [" + key + "] to [" + value + "], already registered to [" + old + "]");
-            }
-        }
-
-        @Override
-        protected void bindExtensions(Binder binder) {
-            MapBinder<K, V> mapBinder = MapBinder.newMapBinder(binder, keyType, valueType);
-            for (Map.Entry<K, V> entry : map.entrySet()) {
-                mapBinder.addBinding(entry.getKey()).toInstance(entry.getValue());
-            }
-        }
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/common/util/IndexFolderUpgrader.java b/core/src/main/java/org/elasticsearch/common/util/IndexFolderUpgrader.java
index 7f550bc1c26ea..528982385ac54 100644
--- a/core/src/main/java/org/elasticsearch/common/util/IndexFolderUpgrader.java
+++ b/core/src/main/java/org/elasticsearch/common/util/IndexFolderUpgrader.java
@@ -20,10 +20,13 @@
 package org.elasticsearch.common.util;
 
 import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.apache.logging.log4j.util.Supplier;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
@@ -64,8 +67,8 @@ void upgrade(final Index index, final Path source, final Path target) throws IOE
         } catch (NoSuchFileException | FileNotFoundException exception) {
             // thrown when the source is non-existent because the folder was renamed
             // by another node (shared FS) after we checked if the target exists
-            logger.error("multiple nodes trying to upgrade [{}] in parallel, retry upgrading with single node",
-                exception, target);
+            logger.error((Supplier<?>) () -> new ParameterizedMessage("multiple nodes trying to upgrade [{}] in parallel, retry " +
+                "upgrading with single node", target), exception);
             throw exception;
         } finally {
             if (success) {
@@ -84,7 +87,7 @@ void upgrade(final Index index, final Path source, final Path target) throws IOE
     void upgrade(final String indexFolderName) throws IOException {
         for (NodeEnvironment.NodePath nodePath : nodeEnv.nodePaths()) {
             final Path indexFolderPath = nodePath.indicesPath.resolve(indexFolderName);
-            final IndexMetaData indexMetaData = IndexMetaData.FORMAT.loadLatestState(logger, indexFolderPath);
+            final IndexMetaData indexMetaData = IndexMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, indexFolderPath);
             if (indexMetaData != null) {
                 final Index index = indexMetaData.getIndex();
                 if (needsUpgrade(index, indexFolderName)) {
diff --git a/core/src/main/java/org/elasticsearch/common/util/LongObjectPagedHashMap.java b/core/src/main/java/org/elasticsearch/common/util/LongObjectPagedHashMap.java
index 4095f5d7014a5..a79e8d88be68a 100644
--- a/core/src/main/java/org/elasticsearch/common/util/LongObjectPagedHashMap.java
+++ b/core/src/main/java/org/elasticsearch/common/util/LongObjectPagedHashMap.java
@@ -161,7 +161,7 @@ public Cursor<T> next() {
             }
 
             @Override
-            public final void remove() {
+            public void remove() {
                 throw new UnsupportedOperationException();
             }
 
diff --git a/core/src/main/java/org/elasticsearch/common/util/PageCacheRecycler.java b/core/src/main/java/org/elasticsearch/common/util/PageCacheRecycler.java
index 6a68dda0272a1..947aad487374e 100644
--- a/core/src/main/java/org/elasticsearch/common/util/PageCacheRecycler.java
+++ b/core/src/main/java/org/elasticsearch/common/util/PageCacheRecycler.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.common.util;
 
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.recycler.AbstractRecyclerC;
@@ -29,7 +28,6 @@
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 
 import java.util.Arrays;
@@ -70,7 +68,7 @@ public void close() {
     protected PageCacheRecycler(Settings settings) {
         super(settings);
         final Type type = TYPE_SETTING .get(settings);
-        final long limit = LIMIT_HEAP_SETTING .get(settings).bytes();
+        final long limit = LIMIT_HEAP_SETTING .get(settings).getBytes();
         final int availableProcessors = EsExecutors.boundedNumberOfProcessors(settings);
 
         // We have a global amount of memory that we need to divide across data types.
diff --git a/core/src/main/java/org/elasticsearch/common/util/concurrent/AtomicArray.java b/core/src/main/java/org/elasticsearch/common/util/concurrent/AtomicArray.java
index 2278220d9dd3d..fa82aa0ac634a 100644
--- a/core/src/main/java/org/elasticsearch/common/util/concurrent/AtomicArray.java
+++ b/core/src/main/java/org/elasticsearch/common/util/concurrent/AtomicArray.java
@@ -31,16 +31,8 @@
  * to get the concrete values as a list using {@link #asList()}.
  */
 public class AtomicArray<E> {
-
-    private static final AtomicArray EMPTY = new AtomicArray(0);
-
-    @SuppressWarnings("unchecked")
-    public static <E> E empty() {
-        return (E) EMPTY;
-    }
-
     private final AtomicReferenceArray<E> array;
-    private volatile List<Entry<E>> nonNullList;
+    private volatile List<E> nonNullList;
 
     public AtomicArray(int size) {
         array = new AtomicReferenceArray<>(size);
@@ -53,7 +45,6 @@ public int length() {
         return array.length();
     }
 
-
     /**
      * Sets the element at position {@code i} to the given value.
      *
@@ -87,19 +78,18 @@ public E get(int i) {
     }
 
     /**
-     * Returns the it as a non null list, with an Entry wrapping each value allowing to
-     * retain its index.
+     * Returns the it as a non null list.
      */
-    public List<Entry<E>> asList() {
+    public List<E> asList() {
         if (nonNullList == null) {
             if (array == null || array.length() == 0) {
                 nonNullList = Collections.emptyList();
             } else {
-                List<Entry<E>> list = new ArrayList<>(array.length());
+                List<E> list = new ArrayList<>(array.length());
                 for (int i = 0; i < array.length(); i++) {
                     E e = array.get(i);
                     if (e != null) {
-                        list.add(new Entry<>(i, e));
+                        list.add(e);
                     }
                 }
                 nonNullList = list;
@@ -120,23 +110,4 @@ public E[] toArray(E[] a) {
         }
         return a;
     }
-
-    /**
-     * An entry within the array.
-     */
-    public static class Entry<E> {
-        /**
-         * The original index of the value within the array.
-         */
-        public final int index;
-        /**
-         * The value.
-         */
-        public final E value;
-
-        public Entry(int index, E value) {
-            this.index = index;
-            this.value = value;
-        }
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/util/concurrent/BaseFuture.java b/core/src/main/java/org/elasticsearch/common/util/concurrent/BaseFuture.java
index c3e60ec5be3f7..ee9aea9ed70c3 100644
--- a/core/src/main/java/org/elasticsearch/common/util/concurrent/BaseFuture.java
+++ b/core/src/main/java/org/elasticsearch/common/util/concurrent/BaseFuture.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.common.util.concurrent;
 
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.Transports;
@@ -60,7 +61,9 @@ public abstract class BaseFuture<V> implements Future<V> {
     public V get(long timeout, TimeUnit unit) throws InterruptedException,
             TimeoutException, ExecutionException {
         assert timeout <= 0 ||
-            (Transports.assertNotTransportThread(BLOCKING_OP_REASON) && ThreadPool.assertNotScheduleThread(BLOCKING_OP_REASON));
+            (Transports.assertNotTransportThread(BLOCKING_OP_REASON) &&
+                ThreadPool.assertNotScheduleThread(BLOCKING_OP_REASON) &&
+                ClusterService.assertNotClusterStateUpdateThread(BLOCKING_OP_REASON));
         return sync.get(unit.toNanos(timeout));
     }
 
@@ -82,7 +85,9 @@ public V get(long timeout, TimeUnit unit) throws InterruptedException,
      */
     @Override
     public V get() throws InterruptedException, ExecutionException {
-        assert Transports.assertNotTransportThread(BLOCKING_OP_REASON) && ThreadPool.assertNotScheduleThread(BLOCKING_OP_REASON);
+        assert Transports.assertNotTransportThread(BLOCKING_OP_REASON) &&
+            ThreadPool.assertNotScheduleThread(BLOCKING_OP_REASON) &&
+            ClusterService.assertNotClusterStateUpdateThread(BLOCKING_OP_REASON);
         return sync.get();
     }
 
diff --git a/core/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java b/core/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java
index 2d682648ca40d..7ec587cf72718 100644
--- a/core/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java
+++ b/core/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java
@@ -25,8 +25,12 @@
 import org.elasticsearch.node.Node;
 
 import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.AbstractExecutorService;
 import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ExecutorService;
 import java.util.concurrent.LinkedTransferQueue;
+import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
@@ -53,8 +57,8 @@ public static int boundedNumberOfProcessors(Settings settings) {
         return PROCESSORS_SETTING.get(settings);
     }
 
-    public static PrioritizedEsThreadPoolExecutor newSinglePrioritizing(String name, ThreadFactory threadFactory, ThreadContext contextHolder) {
-        return new PrioritizedEsThreadPoolExecutor(name, 1, 1, 0L, TimeUnit.MILLISECONDS, threadFactory, contextHolder);
+    public static PrioritizedEsThreadPoolExecutor newSinglePrioritizing(String name, ThreadFactory threadFactory, ThreadContext contextHolder, ScheduledExecutorService timer) {
+        return new PrioritizedEsThreadPoolExecutor(name, 1, 1, 0L, TimeUnit.MILLISECONDS, threadFactory, contextHolder, timer);
     }
 
     public static EsThreadPoolExecutor newScaling(String name, int min, int max, long keepAliveTime, TimeUnit unit, ThreadFactory threadFactory, ThreadContext contextHolder) {
@@ -74,6 +78,50 @@ public static EsThreadPoolExecutor newFixed(String name, int size, int queueCapa
         return new EsThreadPoolExecutor(name, size, size, 0, TimeUnit.MILLISECONDS, queue, threadFactory, new EsAbortPolicy(), contextHolder);
     }
 
+    private static final ExecutorService DIRECT_EXECUTOR_SERVICE = new AbstractExecutorService() {
+
+        @Override
+        public void shutdown() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public List<Runnable> shutdownNow() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public boolean isShutdown() {
+            return false;
+        }
+
+        @Override
+        public boolean isTerminated() {
+            return false;
+        }
+
+        @Override
+        public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public void execute(Runnable command) {
+            command.run();
+        }
+
+    };
+
+    /**
+     * Returns an {@link ExecutorService} that executes submitted tasks on the current thread. This executor service does not support being
+     * shutdown.
+     *
+     * @return an {@link ExecutorService} that executes submitted tasks on the current thread
+     */
+    public static ExecutorService newDirectExecutorService() {
+        return DIRECT_EXECUTOR_SERVICE;
+    }
+
     public static String threadName(Settings settings, String ... names) {
         String namePrefix =
                 Arrays
@@ -113,7 +161,7 @@ static class EsThreadFactory implements ThreadFactory {
         final AtomicInteger threadNumber = new AtomicInteger(1);
         final String namePrefix;
 
-        public EsThreadFactory(String namePrefix) {
+        EsThreadFactory(String namePrefix) {
             this.namePrefix = namePrefix;
             SecurityManager s = System.getSecurityManager();
             group = (s != null) ? s.getThreadGroup() :
@@ -141,7 +189,7 @@ static class ExecutorScalingQueue<E> extends LinkedTransferQueue<E> {
 
         ThreadPoolExecutor executor;
 
-        public ExecutorScalingQueue() {
+        ExecutorScalingQueue() {
         }
 
         @Override
diff --git a/core/src/main/java/org/elasticsearch/common/util/concurrent/EsThreadPoolExecutor.java b/core/src/main/java/org/elasticsearch/common/util/concurrent/EsThreadPoolExecutor.java
index 2f664679bb444..9662292cf69f6 100644
--- a/core/src/main/java/org/elasticsearch/common/util/concurrent/EsThreadPoolExecutor.java
+++ b/core/src/main/java/org/elasticsearch/common/util/concurrent/EsThreadPoolExecutor.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.common.util.concurrent;
 
-
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.ThreadPoolExecutor;
@@ -109,6 +108,27 @@ protected void doExecute(final Runnable command) {
         }
     }
 
+    @Override
+    protected void afterExecute(Runnable r, Throwable t) {
+        super.afterExecute(r, t);
+        assert assertDefaultContext(r);
+    }
+
+    private boolean assertDefaultContext(Runnable r) {
+        try {
+            assert contextHolder.isDefaultContext() : "the thread context is not the default context and the thread [" +
+                Thread.currentThread().getName() + "] is being returned to the pool after executing [" + r + "]";
+        } catch (IllegalStateException ex) {
+            // sometimes we execute on a closed context and isDefaultContext doen't bypass the ensureOpen checks
+            // this must not trigger an exception here since we only assert if the default is restored and
+            // we don't really care if we are closed
+            if (contextHolder.isClosed() == false) {
+                throw ex;
+            }
+        }
+        return true;
+    }
+
     /**
      * Returns a stream of all pending tasks. This is similar to {@link #getQueue()} but will expose the originally submitted
      * {@link Runnable} instances rather than potentially wrapped ones.
diff --git a/core/src/main/java/org/elasticsearch/common/util/concurrent/PrioritizedEsThreadPoolExecutor.java b/core/src/main/java/org/elasticsearch/common/util/concurrent/PrioritizedEsThreadPoolExecutor.java
index f55c84e943aad..ee38637b04c0c 100644
--- a/core/src/main/java/org/elasticsearch/common/util/concurrent/PrioritizedEsThreadPoolExecutor.java
+++ b/core/src/main/java/org/elasticsearch/common/util/concurrent/PrioritizedEsThreadPoolExecutor.java
@@ -44,11 +44,14 @@
 public class PrioritizedEsThreadPoolExecutor extends EsThreadPoolExecutor {
 
     private static final TimeValue NO_WAIT_TIME_VALUE = TimeValue.timeValueMillis(0);
-    private AtomicLong insertionOrder = new AtomicLong();
-    private Queue<Runnable> current = ConcurrentCollections.newQueue();
+    private final AtomicLong insertionOrder = new AtomicLong();
+    private final Queue<Runnable> current = ConcurrentCollections.newQueue();
+    private final ScheduledExecutorService timer;
 
-    PrioritizedEsThreadPoolExecutor(String name, int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit, ThreadFactory threadFactory, ThreadContext contextHolder) {
+    PrioritizedEsThreadPoolExecutor(String name, int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit,
+                                    ThreadFactory threadFactory, ThreadContext contextHolder, ScheduledExecutorService timer) {
         super(name, corePoolSize, maximumPoolSize, keepAliveTime, unit, new PriorityBlockingQueue<>(), threadFactory, contextHolder);
+        this.timer = timer;
     }
 
     public Pending[] getPending() {
@@ -88,7 +91,13 @@ private void addPending(List<Runnable> runnables, List<Pending> pending, boolean
         for (Runnable runnable : runnables) {
             if (runnable instanceof TieBreakingPrioritizedRunnable) {
                 TieBreakingPrioritizedRunnable t = (TieBreakingPrioritizedRunnable) runnable;
-                pending.add(new Pending(unwrap(t.runnable), t.priority(), t.insertionOrder, executing));
+                Runnable innerRunnable = t.runnable;
+                if (innerRunnable != null) {
+                    /** innerRunnable can be null if task is finished but not removed from executor yet,
+                     * see {@link TieBreakingPrioritizedRunnable#run} and {@link TieBreakingPrioritizedRunnable#runAndClean}
+                     */
+                    pending.add(new Pending(unwrap(innerRunnable), t.priority(), t.insertionOrder, executing));
+                }
             } else if (runnable instanceof PrioritizedFutureTask) {
                 PrioritizedFutureTask t = (PrioritizedFutureTask) runnable;
                 Object task = t.task;
@@ -107,10 +116,11 @@ protected void beforeExecute(Thread t, Runnable r) {
 
     @Override
     protected void afterExecute(Runnable r, Throwable t) {
+        super.afterExecute(r, t);
         current.remove(r);
     }
 
-    public void execute(Runnable command, final ScheduledExecutorService timer, final TimeValue timeout, final Runnable timeoutCallback) {
+    public void execute(Runnable command, final TimeValue timeout, final Runnable timeoutCallback) {
         command = wrapRunnable(command);
         doExecute(command);
         if (timeout.nanos() >= 0) {
@@ -249,14 +259,14 @@ private final class PrioritizedFutureTask<T> extends FutureTask<T> implements Co
         final Priority priority;
         final long insertionOrder;
 
-        public PrioritizedFutureTask(Runnable runnable, Priority priority, T value, long insertionOrder) {
+        PrioritizedFutureTask(Runnable runnable, Priority priority, T value, long insertionOrder) {
             super(runnable, value);
             this.task = runnable;
             this.priority = priority;
             this.insertionOrder = insertionOrder;
         }
 
-        public PrioritizedFutureTask(PrioritizedCallable<T> callable, long insertionOrder) {
+        PrioritizedFutureTask(PrioritizedCallable<T> callable, long insertionOrder) {
             super(callable);
             this.task = callable;
             this.priority = callable.priority();
diff --git a/core/src/main/java/org/elasticsearch/common/util/concurrent/ReleasableLock.java b/core/src/main/java/org/elasticsearch/common/util/concurrent/ReleasableLock.java
index 1a90c6992fc43..46d6abff17632 100644
--- a/core/src/main/java/org/elasticsearch/common/util/concurrent/ReleasableLock.java
+++ b/core/src/main/java/org/elasticsearch/common/util/concurrent/ReleasableLock.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.common.util.concurrent;
 
+import org.elasticsearch.Assertions;
 import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.index.engine.EngineException;
 
@@ -35,9 +36,7 @@ public class ReleasableLock implements Releasable {
 
     public ReleasableLock(Lock lock) {
         this.lock = lock;
-        boolean useHoldingThreads = false;
-        assert (useHoldingThreads = true);
-        if (useHoldingThreads) {
+        if (Assertions.ENABLED) {
             holdingThreads = new ThreadLocal<>();
         } else {
             holdingThreads = null;
diff --git a/core/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java b/core/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java
index 8c04c24ec5b6e..1ce119636f734 100644
--- a/core/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java
+++ b/core/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java
@@ -33,7 +33,12 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 /**
  * A ThreadContext is a map of string headers and a transient map of keyed objects that are associated with
@@ -69,6 +74,7 @@ public final class ThreadContext implements Closeable, Writeable {
     private static final ThreadContextStruct DEFAULT_CONTEXT = new ThreadContextStruct();
     private final Map<String, String> defaultHeader;
     private final ContextThreadLocal threadLocal;
+    private boolean isSystemContext;
 
     /**
      * Creates a new ThreadContext instance
@@ -115,12 +121,57 @@ public StoredContext stashAndMergeHeaders(Map<String, String> headers) {
         return () -> threadLocal.set(context);
     }
 
+
     /**
      * Just like {@link #stashContext()} but no default context is set.
+     * @param preserveResponseHeaders if set to <code>true</code> the response headers of the restore thread will be preserved.
      */
-    public StoredContext newStoredContext() {
+    public StoredContext newStoredContext(boolean preserveResponseHeaders) {
         final ThreadContextStruct context = threadLocal.get();
-        return () -> threadLocal.set(context);
+        return ()  -> {
+            if (preserveResponseHeaders && threadLocal.get() != context) {
+                threadLocal.set(context.putResponseHeaders(threadLocal.get().responseHeaders));
+            } else {
+                threadLocal.set(context);
+            }
+        };
+    }
+
+    /**
+     * Returns a supplier that gathers a {@link #newStoredContext(boolean)} and restores it once the
+     * returned supplier is invoked. The context returned from the supplier is a stored version of the
+     * suppliers callers context that should be restored once the originally gathered context is not needed anymore.
+     * For instance this method should be used like this:
+     *
+     * <pre>
+     *     Supplier&lt;ThreadContext.StoredContext&gt; restorable = context.newRestorableContext(true);
+     *     new Thread() {
+     *         public void run() {
+     *             try (ThreadContext.StoredContext ctx = restorable.get()) {
+     *                 // execute with the parents context and restore the threads context afterwards
+     *             }
+     *         }
+     *
+     *     }.start();
+     * </pre>
+     *
+     * @param preserveResponseHeaders if set to <code>true</code> the response headers of the restore thread will be preserved.
+     * @return a restorable context supplier
+     */
+    public Supplier<StoredContext> newRestorableContext(boolean preserveResponseHeaders) {
+        return wrapRestorable(newStoredContext(preserveResponseHeaders));
+    }
+
+    /**
+     * Same as {@link #newRestorableContext(boolean)} but wraps an existing context to restore.
+     * @param storedContext the context to restore
+     */
+    public Supplier<StoredContext> wrapRestorable(StoredContext storedContext) {
+        return () -> {
+            StoredContext context = newStoredContext(false);
+            storedContext.restore();
+            return context;
+        };
     }
 
     @Override
@@ -208,12 +259,25 @@ public <T> T getTransient(String key) {
     }
 
     /**
-     * Add the <em>unique</em> response {@code value} for the specified {@code key}.
-     * <p>
-     * Any duplicate {@code value} is ignored.
+     * Add the {@code value} for the specified {@code key} Any duplicate {@code value} is ignored.
+     *
+     * @param key         the header name
+     * @param value       the header value
      */
-    public void addResponseHeader(String key, String value) {
-        threadLocal.set(threadLocal.get().putResponse(key, value));
+    public void addResponseHeader(final String key, final String value) {
+        addResponseHeader(key, value, v -> v);
+    }
+
+    /**
+     * Add the {@code value} for the specified {@code key} with the specified {@code uniqueValue} used for de-duplication. Any duplicate
+     * {@code value} after applying {@code uniqueValue} is ignored.
+     *
+     * @param key         the header name
+     * @param value       the header value
+     * @param uniqueValue the function that produces de-duplication values
+     */
+    public void addResponseHeader(final String key, final String value, final Function<String, String> uniqueValue) {
+        threadLocal.set(threadLocal.get().putResponse(key, value, uniqueValue));
     }
 
     /**
@@ -246,6 +310,35 @@ public Runnable unwrap(Runnable command) {
         return command;
     }
 
+    /**
+     * Returns true if the current context is the default context.
+     */
+    boolean isDefaultContext() {
+        return threadLocal.get() == DEFAULT_CONTEXT;
+    }
+
+    /**
+     * Marks this thread context as an internal system context. This signals that actions in this context are issued
+     * by the system itself rather than by a user action.
+     */
+    public void markAsSystemContext() {
+        threadLocal.set(threadLocal.get().setSystemContext());
+    }
+
+    /**
+     * Returns <code>true</code> iff this context is a system context
+     */
+    public boolean isSystemContext() {
+        return threadLocal.get().isSystemContext;
+    }
+
+    /**
+     * Returns <code>true</code> if the context is closed, otherwise <code>true</code>
+     */
+    boolean isClosed() {
+        return threadLocal.closed.get();
+    }
+
     @FunctionalInterface
     public interface StoredContext extends AutoCloseable {
         @Override
@@ -260,6 +353,7 @@ private static final class ThreadContextStruct {
         private final Map<String, String> requestHeaders;
         private final Map<String, Object> transientHeaders;
         private final Map<String, List<String>> responseHeaders;
+        private final boolean isSystemContext;
 
         private ThreadContextStruct(StreamInput in) throws IOException {
             final int numRequest = in.readVInt();
@@ -271,27 +365,36 @@ private ThreadContextStruct(StreamInput in) throws IOException {
             this.requestHeaders = requestHeaders;
             this.responseHeaders = in.readMapOfLists(StreamInput::readString, StreamInput::readString);
             this.transientHeaders = Collections.emptyMap();
+            isSystemContext = false; // we never serialize this it's a transient flag
+        }
+
+        private ThreadContextStruct setSystemContext() {
+            if (isSystemContext) {
+                return this;
+            }
+            return new ThreadContextStruct(requestHeaders, responseHeaders, transientHeaders, true);
         }
 
         private ThreadContextStruct(Map<String, String> requestHeaders,
                                     Map<String, List<String>> responseHeaders,
-                                    Map<String, Object> transientHeaders) {
+                                    Map<String, Object> transientHeaders, boolean isSystemContext) {
             this.requestHeaders = requestHeaders;
             this.responseHeaders = responseHeaders;
             this.transientHeaders = transientHeaders;
+            this.isSystemContext = isSystemContext;
         }
 
         /**
          * This represents the default context and it should only ever be called by {@link #DEFAULT_CONTEXT}.
          */
         private ThreadContextStruct() {
-            this(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
+            this(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), false);
         }
 
         private ThreadContextStruct putRequest(String key, String value) {
             Map<String, String> newRequestHeaders = new HashMap<>(this.requestHeaders);
             putSingleHeader(key, value, newRequestHeaders);
-            return new ThreadContextStruct(newRequestHeaders, responseHeaders, transientHeaders);
+            return new ThreadContextStruct(newRequestHeaders, responseHeaders, transientHeaders, isSystemContext);
         }
 
         private void putSingleHeader(String key, String value, Map<String, String> newHeaders) {
@@ -309,18 +412,40 @@ private ThreadContextStruct putHeaders(Map<String, String> headers) {
                     putSingleHeader(entry.getKey(), entry.getValue(), newHeaders);
                 }
                 newHeaders.putAll(this.requestHeaders);
-                return new ThreadContextStruct(newHeaders, responseHeaders, transientHeaders);
+                return new ThreadContextStruct(newHeaders, responseHeaders, transientHeaders, isSystemContext);
             }
         }
 
-        private ThreadContextStruct putResponse(String key, String value) {
+        private ThreadContextStruct putResponseHeaders(Map<String, List<String>> headers) {
+            assert headers != null;
+            if (headers.isEmpty()) {
+                return this;
+            }
+            final Map<String, List<String>> newResponseHeaders = new HashMap<>(this.responseHeaders);
+            for (Map.Entry<String, List<String>> entry : headers.entrySet()) {
+                String key = entry.getKey();
+                final List<String> existingValues = newResponseHeaders.get(key);
+                if (existingValues != null) {
+                    List<String> newValues = Stream.concat(entry.getValue().stream(),
+                        existingValues.stream()).distinct().collect(Collectors.toList());
+                    newResponseHeaders.put(key, Collections.unmodifiableList(newValues));
+                } else {
+                    newResponseHeaders.put(key, entry.getValue());
+                }
+            }
+            return new ThreadContextStruct(requestHeaders, newResponseHeaders, transientHeaders, isSystemContext);
+        }
+
+        private ThreadContextStruct putResponse(final String key, final String value, final Function<String, String> uniqueValue) {
             assert value != null;
 
             final Map<String, List<String>> newResponseHeaders = new HashMap<>(this.responseHeaders);
             final List<String> existingValues = newResponseHeaders.get(key);
 
             if (existingValues != null) {
-                if (existingValues.contains(value)) {
+                final Set<String> existingUniqueValues = existingValues.stream().map(uniqueValue).collect(Collectors.toSet());
+                assert existingValues.size() == existingUniqueValues.size();
+                if (existingUniqueValues.contains(uniqueValue.apply(value))) {
                     return this;
                 }
 
@@ -332,7 +457,7 @@ private ThreadContextStruct putResponse(String key, String value) {
                 newResponseHeaders.put(key, Collections.singletonList(value));
             }
 
-            return new ThreadContextStruct(requestHeaders, newResponseHeaders, transientHeaders);
+            return new ThreadContextStruct(requestHeaders, newResponseHeaders, transientHeaders, isSystemContext);
         }
 
         private ThreadContextStruct putTransient(String key, Object value) {
@@ -340,7 +465,7 @@ private ThreadContextStruct putTransient(String key, Object value) {
             if (newTransient.putIfAbsent(key, value) != null) {
                 throw new IllegalArgumentException("value for key [" + key + "] already present");
             }
-            return new ThreadContextStruct(requestHeaders, responseHeaders, newTransient);
+            return new ThreadContextStruct(requestHeaders, responseHeaders, newTransient, isSystemContext);
         }
 
         boolean isEmpty() {
@@ -431,7 +556,7 @@ private class ContextPreservingRunnable implements Runnable {
         private final ThreadContext.StoredContext ctx;
 
         private ContextPreservingRunnable(Runnable in) {
-            ctx = newStoredContext();
+            ctx = newStoredContext(false);
             this.in = in;
         }
 
@@ -468,10 +593,12 @@ public Runnable unwrap() {
      */
     private class ContextPreservingAbstractRunnable extends AbstractRunnable {
         private final AbstractRunnable in;
-        private final ThreadContext.StoredContext ctx;
+        private final ThreadContext.StoredContext creatorsContext;
+
+        private ThreadContext.StoredContext threadsOriginalContext = null;
 
         private ContextPreservingAbstractRunnable(AbstractRunnable in) {
-            ctx = newStoredContext();
+            creatorsContext = newStoredContext(false);
             this.in = in;
         }
 
@@ -482,7 +609,13 @@ public boolean isForceExecution() {
 
         @Override
         public void onAfter() {
-            in.onAfter();
+            try {
+                in.onAfter();
+            } finally {
+                if (threadsOriginalContext != null) {
+                    threadsOriginalContext.restore();
+                }
+            }
         }
 
         @Override
@@ -498,8 +631,9 @@ public void onRejection(Exception e) {
         @Override
         protected void doRun() throws Exception {
             boolean whileRunning = false;
-            try (ThreadContext.StoredContext ignore = stashContext()){
-                ctx.restore();
+            threadsOriginalContext = stashContext();
+            try {
+                creatorsContext.restore();
                 whileRunning = true;
                 in.doRun();
                 whileRunning = false;
diff --git a/core/src/main/java/org/elasticsearch/common/util/set/Sets.java b/core/src/main/java/org/elasticsearch/common/util/set/Sets.java
index 4b323c42a371a..0f1fe22c02010 100644
--- a/core/src/main/java/org/elasticsearch/common/util/set/Sets.java
+++ b/core/src/main/java/org/elasticsearch/common/util/set/Sets.java
@@ -21,11 +21,19 @@
 
 import java.util.Collection;
 import java.util.Collections;
+import java.util.EnumSet;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Objects;
 import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.BiConsumer;
+import java.util.function.BinaryOperator;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.stream.Collector;
 import java.util.stream.Collectors;
 
 public final class Sets {
@@ -63,12 +71,72 @@ public static <T> boolean haveEmptyIntersection(Set<T> left, Set<T> right) {
         return !left.stream().anyMatch(k -> right.contains(k));
     }
 
+    /**
+     * The relative complement, or difference, of the specified left and right set. Namely, the resulting set contains all the elements that
+     * are in the left set but not in the right set. Neither input is mutated by this operation, an entirely new set is returned.
+     *
+     * @param left  the left set
+     * @param right the right set
+     * @param <T>   the type of the elements of the sets
+     * @return the relative complement of the left set with respect to the right set
+     */
     public static <T> Set<T> difference(Set<T> left, Set<T> right) {
         Objects.requireNonNull(left);
         Objects.requireNonNull(right);
         return left.stream().filter(k -> !right.contains(k)).collect(Collectors.toSet());
     }
 
+    /**
+     * The relative complement, or difference, of the specified left and right set, returned as a sorted set. Namely, the resulting set
+     * contains all the elements that are in the left set but not in the right set, and the set is sorted using the natural ordering of
+     * element type. Neither input is mutated by this operation, an entirely new set is returned.
+     *
+     * @param left  the left set
+     * @param right the right set
+     * @param <T>   the type of the elements of the sets
+     * @return the sorted relative complement of the left set with respect to the right set
+     */
+    public static <T> SortedSet<T> sortedDifference(Set<T> left, Set<T> right) {
+        Objects.requireNonNull(left);
+        Objects.requireNonNull(right);
+        return left.stream().filter(k -> !right.contains(k)).collect(new SortedSetCollector<>());
+    }
+
+    private static class SortedSetCollector<T> implements Collector<T, SortedSet<T>, SortedSet<T>> {
+
+        @Override
+        public Supplier<SortedSet<T>> supplier() {
+            return TreeSet::new;
+        }
+
+        @Override
+        public BiConsumer<SortedSet<T>, T> accumulator() {
+            return (s, e) -> s.add(e);
+        }
+
+        @Override
+        public BinaryOperator<SortedSet<T>> combiner() {
+            return (s, t) -> {
+                s.addAll(t);
+                return s;
+            };
+        }
+
+        @Override
+        public Function<SortedSet<T>, SortedSet<T>> finisher() {
+            return Function.identity();
+        }
+
+        static final Set<Characteristics> CHARACTERISTICS =
+                Collections.unmodifiableSet(EnumSet.of(Collector.Characteristics.IDENTITY_FINISH));
+
+        @Override
+        public Set<Characteristics> characteristics() {
+            return CHARACTERISTICS;
+        }
+
+    }
+
     public static <T> Set<T> union(Set<T> left, Set<T> right) {
         Objects.requireNonNull(left);
         Objects.requireNonNull(right);
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/AbstractObjectParser.java b/core/src/main/java/org/elasticsearch/common/xcontent/AbstractObjectParser.java
index 6f8a606d9a644..91acb267056b0 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/AbstractObjectParser.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/AbstractObjectParser.java
@@ -19,8 +19,8 @@
 
 package org.elasticsearch.common.xcontent;
 
+import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
@@ -34,37 +34,26 @@
 /**
  * Superclass for {@link ObjectParser} and {@link ConstructingObjectParser}. Defines most of the "declare" methods so they can be shared.
  */
-public abstract class AbstractObjectParser<Value, Context extends ParseFieldMatcherSupplier>
-        implements BiFunction<XContentParser, Context, Value> {
-    /**
-     * Reads an object from a parser using some context.
-     */
-    @FunctionalInterface
-    public interface ContextParser<Context, T> {
-        T parse(XContentParser p, Context c) throws IOException;
-    }
-
-    /**
-     * Reads an object right from the parser without any context.
-     */
-    @FunctionalInterface
-    public interface NoContextParser<T> {
-        T parse(XContentParser p) throws IOException;
-    }
+public abstract class AbstractObjectParser<Value, Context>
+        implements BiFunction<XContentParser, Context, Value>, ContextParser<Context, Value> {
 
     /**
      * Declare some field. Usually it is easier to use {@link #declareString(BiConsumer, ParseField)} or
-     * {@link #declareObject(BiConsumer, BiFunction, ParseField)} rather than call this directly.
+     * {@link #declareObject(BiConsumer, ContextParser, ParseField)} rather than call this directly.
      */
     public abstract <T> void declareField(BiConsumer<Value, T> consumer, ContextParser<Context, T> parser, ParseField parseField,
             ValueType type);
 
-    public <T> void declareField(BiConsumer<Value, T> consumer, NoContextParser<T> parser, ParseField parseField, ValueType type) {
-        declareField(consumer, (p, c) -> parser.parse(p), parseField, type);
+    public <T> void declareField(BiConsumer<Value, T> consumer, CheckedFunction<XContentParser, T, IOException> parser,
+            ParseField parseField, ValueType type) {
+        if (parser == null) {
+            throw new IllegalArgumentException("[parser] is required");
+        }
+        declareField(consumer, (p, c) -> parser.apply(p), parseField, type);
     }
 
-    public <T> void declareObject(BiConsumer<Value, T> consumer, BiFunction<XContentParser, Context, T> objectParser, ParseField field) {
-        declareField(consumer, (p, c) -> objectParser.apply(p, c), field, ValueType.OBJECT);
+    public <T> void declareObject(BiConsumer<Value, T> consumer, ContextParser<Context, T> objectParser, ParseField field) {
+        declareField(consumer, (p, c) -> objectParser.parse(p, c), field, ValueType.OBJECT);
     }
 
     public void declareFloat(BiConsumer<Value, Float> consumer, ParseField field) {
@@ -100,9 +89,9 @@ public void declareBoolean(BiConsumer<Value, Boolean> consumer, ParseField field
         declareField(consumer, XContentParser::booleanValue, field, ValueType.BOOLEAN);
     }
 
-    public <T> void declareObjectArray(BiConsumer<Value, List<T>> consumer, BiFunction<XContentParser, Context, T> objectParser,
+    public <T> void declareObjectArray(BiConsumer<Value, List<T>> consumer, ContextParser<Context, T> objectParser,
             ParseField field) {
-        declareField(consumer, (p, c) -> parseArray(p, () -> objectParser.apply(p, c)), field, ValueType.OBJECT_ARRAY);
+        declareField(consumer, (p, c) -> parseArray(p, () -> objectParser.parse(p, c)), field, ValueType.OBJECT_ARRAY);
     }
 
     public void declareStringArray(BiConsumer<Value, List<String>> consumer, ParseField field) {
@@ -126,7 +115,7 @@ public void declareIntArray(BiConsumer<Value, List<Integer>> consumer, ParseFiel
     }
 
     public void declareRawObject(BiConsumer<Value, BytesReference> consumer, ParseField field) {
-        NoContextParser<BytesReference> bytesParser = p -> {
+        CheckedFunction<XContentParser, BytesReference, IOException> bytesParser = p -> {
             try (XContentBuilder builder = JsonXContent.contentBuilder()) {
                 builder.prettyPrint();
                 builder.copyCurrentStructure(p);
@@ -141,7 +130,7 @@ private interface IOSupplier<T> {
     }
     private static <T> List<T> parseArray(XContentParser parser, IOSupplier<T> supplier) throws IOException {
         List<T> list = new ArrayList<>();
-        if (parser.currentToken().isValue()) {
+        if (parser.currentToken().isValue() || parser.currentToken() == XContentParser.Token.START_OBJECT) {
             list.add(supplier.get()); // single value
         } else {
             while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/ConstructingObjectParser.java b/core/src/main/java/org/elasticsearch/common/xcontent/ConstructingObjectParser.java
index e1400463a7226..02ccf0d67ba94 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/ConstructingObjectParser.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/ConstructingObjectParser.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.common.xcontent;
 
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
 
@@ -74,7 +73,7 @@
  * Note: if optional constructor arguments aren't specified then the number of allocations is always the worst case.
  * </p>
  */
-public final class ConstructingObjectParser<Value, Context extends ParseFieldMatcherSupplier> extends AbstractObjectParser<Value, Context> {
+public final class ConstructingObjectParser<Value, Context> extends AbstractObjectParser<Value, Context> {
     /**
      * Consumer that marks a field as a required constructor argument instead of a real object field.
      */
@@ -94,7 +93,7 @@ public final class ConstructingObjectParser<Value, Context extends ParseFieldMat
      */
     private final List<ConstructorArgInfo> constructorArgInfos = new ArrayList<>();
     private final ObjectParser<Target, Context> objectParser;
-    private final Function<Object[], Value> builder;
+    private final BiFunction<Object[], Context, Value> builder;
     /**
      * The number of fields on the targetObject. This doesn't include any constructor arguments and is the size used for the array backing
      * the field queue.
@@ -103,7 +102,7 @@ public final class ConstructingObjectParser<Value, Context extends ParseFieldMat
 
     /**
      * Build the parser.
-     * 
+     *
      * @param name The name given to the delegate ObjectParser for error identification. Use what you'd use if the object worked with
      *        ObjectParser.
      * @param builder A function that builds the object from an array of Objects. Declare this inline with the parser, casting the elements
@@ -113,8 +112,44 @@ public final class ConstructingObjectParser<Value, Context extends ParseFieldMat
      *        allocations.
      */
     public ConstructingObjectParser(String name, Function<Object[], Value> builder) {
-        objectParser = new ObjectParser<>(name);
+        this(name, false, builder);
+    }
+
+    /**
+     * Build the parser.
+     *
+     * @param name The name given to the delegate ObjectParser for error identification. Use what you'd use if the object worked with
+     *        ObjectParser.
+     * @param ignoreUnknownFields Should this parser ignore unknown fields? This should generally be set to true only when parsing responses
+     *        from external systems, never when parsing requests from users.
+     * @param builder A function that builds the object from an array of Objects. Declare this inline with the parser, casting the elements
+     *        of the array to the arguments so they work with your favorite constructor. The objects in the array will be in the same order
+     *        that you declared the {{@link #constructorArg()}s and none will be null. If any of the constructor arguments aren't defined in
+     *        the XContent then parsing will throw an error. We use an array here rather than a {@code Map<String, Object>} to save on
+     *        allocations.
+     */
+    public ConstructingObjectParser(String name, boolean ignoreUnknownFields, Function<Object[], Value> builder) {
+        this(name, ignoreUnknownFields, (args, context) -> builder.apply(args));
+    }
+
+    /**
+     * Build the parser.
+     *
+     * @param name The name given to the delegate ObjectParser for error identification. Use what you'd use if the object worked with
+     *        ObjectParser.
+     * @param ignoreUnknownFields Should this parser ignore unknown fields? This should generally be set to true only when parsing responses
+     *        from external systems, never when parsing requests from users.
+     * @param builder A binary function that builds the object from an array of Objects and the parser context.  Declare this inline with
+     *        the parser, casting the elements of the array to the arguments so they work with your favorite constructor. The objects in
+     *        the array will be in the same order that you declared the {{@link #constructorArg()}s and none will be null. The second
+     *        argument is the value of the context provided to the {@link #parse(XContentParser, Object) parse function}. If any of the
+     *        constructor arguments aren't defined in the XContent then parsing will throw an error. We use an array here rather than a
+     *        {@code Map<String, Object>} to save on allocations.
+     */
+    public ConstructingObjectParser(String name, boolean ignoreUnknownFields, BiFunction<Object[], Context, Value> builder) {
+        objectParser = new ObjectParser<>(name, ignoreUnknownFields, null);
         this.builder = builder;
+
     }
 
     /**
@@ -123,12 +158,17 @@ public ConstructingObjectParser(String name, Function<Object[], Value> builder)
     @Override
     public Value apply(XContentParser parser, Context context) {
         try {
-            return objectParser.parse(parser, new Target(parser), context).finish();
+            return parse(parser, context);
         } catch (IOException e) {
             throw new ParsingException(parser.getTokenLocation(), "[" + objectParser.getName()  + "] failed to parse object", e);
         }
     }
 
+    @Override
+    public Value parse(XContentParser parser, Context context) throws IOException {
+        return objectParser.parse(parser, new Target(parser, context), context).finish();
+    }
+
     /**
      * Pass the {@linkplain BiConsumer} this returns the declare methods to declare a required constructor argument. See this class's
      * javadoc for an example. The order in which these are declared matters: it is the order that they come in the array passed to
@@ -153,6 +193,19 @@ public static <Value, FieldT> BiConsumer<Value, FieldT> optionalConstructorArg()
 
     @Override
     public <T> void declareField(BiConsumer<Value, T> consumer, ContextParser<Context, T> parser, ParseField parseField, ValueType type) {
+        if (consumer == null) {
+            throw new IllegalArgumentException("[consumer] is required");
+        }
+        if (parser == null) {
+            throw new IllegalArgumentException("[parser] is required");
+        }
+        if (parseField == null) {
+            throw new IllegalArgumentException("[parseField] is required");
+        }
+        if (type == null) {
+            throw new IllegalArgumentException("[type] is required");
+        }
+
         if (consumer == REQUIRED_CONSTRUCTOR_ARG_MARKER || consumer == OPTIONAL_CONSTRUCTOR_ARG_MARKER) {
             /*
              * Constructor arguments are detected by this "marker" consumer. It keeps the API looking clean even if it is a bit sleezy. We
@@ -201,7 +254,7 @@ private <T> BiConsumer<Target, T> queueingConsumer(BiConsumer<Value, T> consumer
 
     /**
      * The target of the {@linkplain ConstructingObjectParser}. One of these is built every time you call
-     * {@linkplain ConstructingObjectParser#apply(XContentParser, ParseFieldMatcherSupplier)} Note that it is not static so it inherits
+     * {@linkplain ConstructingObjectParser#apply(XContentParser, Object)} Note that it is not static so it inherits
      * {@linkplain ConstructingObjectParser}'s type parameters.
      */
     private class Target {
@@ -214,6 +267,12 @@ private class Target {
          * location of each field so that we can give a useful error message when replaying the queue.
          */
         private final XContentParser parser;
+
+        /**
+         * The parse context that is used for this invocation. Stored here so that it can be passed to the {@link #builder}.
+         */
+        private Context context;
+
         /**
          * How many of the constructor parameters have we collected? We keep track of this so we don't have to count the
          * {@link #constructorArgs} array looking for nulls when we receive another constructor parameter. When this is equal to the size of
@@ -233,8 +292,9 @@ private class Target {
          */
         private Value targetObject;
 
-        public Target(XContentParser parser) {
+        Target(XContentParser parser, Context context) {
             this.parser = parser;
+            this.context = context;
         }
 
         /**
@@ -267,7 +327,7 @@ private void queue(Consumer<Value> queueMe) {
         }
 
         /**
-         * Finish parsing the object. 
+         * Finish parsing the object.
          */
         private Value finish() {
             if (targetObject != null) {
@@ -308,7 +368,7 @@ private Value finish() {
 
         private void buildTarget() {
             try {
-                targetObject = builder.apply(constructorArgs);
+                targetObject = builder.apply(constructorArgs, context);
                 while (queuedFieldsCount > 0) {
                     queuedFieldsCount -= 1;
                     queuedFields[queuedFieldsCount].accept(targetObject);
@@ -326,7 +386,7 @@ private static class ConstructorArgInfo {
         final ParseField field;
         final boolean required;
 
-        public ConstructorArgInfo(ParseField field, boolean required) {
+        ConstructorArgInfo(ParseField field, boolean required) {
             this.field = field;
             this.required = required;
         }
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/ContextParser.java b/core/src/main/java/org/elasticsearch/common/xcontent/ContextParser.java
new file mode 100644
index 0000000000000..5862ea76aef80
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/ContextParser.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.xcontent;
+
+import java.io.IOException;
+
+/**
+ * Reads an object from a parser using some context.
+ */
+@FunctionalInterface
+public interface ContextParser<Context, T> {
+    T parse(XContentParser p, Context c) throws IOException;
+}
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/FromXContentBuilder.java b/core/src/main/java/org/elasticsearch/common/xcontent/FromXContentBuilder.java
deleted file mode 100644
index 0b0370b490e69..0000000000000
--- a/core/src/main/java/org/elasticsearch/common/xcontent/FromXContentBuilder.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.common.xcontent;
-
-import org.elasticsearch.common.ParseFieldMatcher;
-
-import java.io.IOException;
-
-/**
- * Indicates that the class supports XContent deserialization.
- */
-public interface FromXContentBuilder<T> {
-    /**
-     * Parses an object with the type T from parser
-     */
-    T fromXContent(XContentParser parser, ParseFieldMatcher parseFieldMatcher) throws IOException;
-}
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/NamedXContentRegistry.java b/core/src/main/java/org/elasticsearch/common/xcontent/NamedXContentRegistry.java
new file mode 100644
index 0000000000000..b991c90b51b1d
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/NamedXContentRegistry.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.xcontent;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.common.CheckedFunction;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import static java.util.Collections.emptyList;
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.unmodifiableMap;
+import static java.util.Objects.requireNonNull;
+
+public class NamedXContentRegistry {
+    /**
+     * The empty {@link NamedXContentRegistry} for use when you are sure that you aren't going to call
+     * {@link XContentParser#namedObject(Class, String, Object)}. Be *very* careful with this singleton because a parser using it will fail
+     * every call to {@linkplain XContentParser#namedObject(Class, String, Object)}. Every non-test usage really should be checked thorowly
+     * and marked with a comment about how it was checked. That way anyone that sees code that uses it knows that it is potentially
+     * dangerous.
+     */
+    public static final NamedXContentRegistry EMPTY = new NamedXContentRegistry(emptyList());
+
+    /**
+     * An entry in the {@linkplain NamedXContentRegistry} containing the name of the object and the parser that can parse it.
+     */
+    public static class Entry {
+        /** The class that this entry can read. */
+        public final Class<?> categoryClass;
+
+        /** A name for the entry which is unique within the {@link #categoryClass}. */
+        public final ParseField name;
+
+        /** A parser capability of parser the entry's class. */
+        private final ContextParser<Object, ?> parser;
+
+        /** Creates a new entry which can be stored by the registry. */
+        public <T> Entry(Class<T> categoryClass, ParseField name, CheckedFunction<XContentParser, ? extends T, IOException> parser) {
+            this.categoryClass = Objects.requireNonNull(categoryClass);
+            this.name = Objects.requireNonNull(name);
+            this.parser = Objects.requireNonNull((p, c) -> parser.apply(p));
+        }
+        /**
+         * Creates a new entry which can be stored by the registry.
+         * Prefer {@link Entry#Entry(Class, ParseField, CheckedFunction)} unless you need a context to carry around while parsing.
+         */
+        public <T> Entry(Class<T> categoryClass, ParseField name, ContextParser<Object, ? extends T> parser) {
+            this.categoryClass = Objects.requireNonNull(categoryClass);
+            this.name = Objects.requireNonNull(name);
+            this.parser = Objects.requireNonNull(parser);
+        }
+    }
+
+    private final Map<Class<?>, Map<String, Entry>> registry;
+
+    public NamedXContentRegistry(List<Entry> entries) {
+        if (entries.isEmpty()) {
+            registry = emptyMap();
+            return;
+        }
+        entries = new ArrayList<>(entries);
+        entries.sort((e1, e2) -> e1.categoryClass.getName().compareTo(e2.categoryClass.getName()));
+
+        Map<Class<?>, Map<String, Entry>> registry = new HashMap<>();
+        Map<String, Entry> parsers = null;
+        Class<?> currentCategory = null;
+        for (Entry entry : entries) {
+            if (currentCategory != entry.categoryClass) {
+                if (currentCategory != null) {
+                    // we've seen the last of this category, put it into the big map
+                    registry.put(currentCategory, unmodifiableMap(parsers));
+                }
+                parsers = new HashMap<>();
+                currentCategory = entry.categoryClass;
+            }
+
+            for (String name : entry.name.getAllNamesIncludedDeprecated()) {
+                Object old = parsers.put(name, entry);
+                if (old != null) {
+                    throw new IllegalArgumentException("NamedXContent [" + currentCategory.getName() + "][" + entry.name + "]" +
+                        " is already registered for [" + old.getClass().getName() + "]," +
+                        " cannot register [" + entry.parser.getClass().getName() + "]");
+                }
+            }
+        }
+        // handle the last category
+        registry.put(currentCategory, unmodifiableMap(parsers));
+
+        this.registry = unmodifiableMap(registry);
+    }
+
+    /**
+     * Parse a named object, throwing an exception if the parser isn't found. Throws an {@link ElasticsearchException} if the
+     * {@code categoryClass} isn't registered because this is almost always a bug. Throws a {@link UnknownNamedObjectException} if the
+     * {@code categoryClass} is registered but the {@code name} isn't.
+     */
+    public <T, C> T parseNamedObject(Class<T> categoryClass, String name, XContentParser parser, C context) throws IOException {
+        Map<String, Entry> parsers = registry.get(categoryClass);
+        if (parsers == null) {
+            if (registry.isEmpty()) {
+                // The "empty" registry will never work so we throw a better exception as a hint.
+                throw new ElasticsearchException("namedObject is not supported for this parser");
+            }
+            throw new ElasticsearchException("Unknown namedObject category [" + categoryClass.getName() + "]");
+        }
+        Entry entry = parsers.get(name);
+        if (entry == null) {
+            throw new UnknownNamedObjectException(parser.getTokenLocation(), categoryClass, name);
+        }
+        if (false == entry.name.match(name)) {
+            /* Note that this shouldn't happen because we already looked up the entry using the names but we need to call `match` anyway
+             * because it is responsible for logging deprecation warnings. */
+            throw new ParsingException(parser.getTokenLocation(),
+                    "Unknown " + categoryClass.getSimpleName() + " [" + name + "]: Parser didn't match");
+        }
+        return categoryClass.cast(entry.parser.parse(parser, context));
+    }
+
+    /**
+     * Thrown when {@link NamedXContentRegistry#parseNamedObject(Class, String, XContentParser, Object)} is called with an unregistered
+     * name. When this bubbles up to the rest layer it is converted into a response with {@code 400 BAD REQUEST} status.
+     */
+    public static class UnknownNamedObjectException extends ParsingException {
+        private final String categoryClass;
+        private final String name;
+
+        public UnknownNamedObjectException(XContentLocation contentLocation, Class<?> categoryClass,
+                String name) {
+            super(contentLocation, "Unknown " + categoryClass.getSimpleName() + " [" + name + "]");
+            this.categoryClass = requireNonNull(categoryClass, "categoryClass is required").getName();
+            this.name = requireNonNull(name, "name is required");
+        }
+
+        /**
+         * Read from a stream.
+         */
+        public UnknownNamedObjectException(StreamInput in) throws IOException {
+            super(in);
+            categoryClass = in.readString();
+            name = in.readString();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeString(categoryClass);
+            out.writeString(name);
+        }
+
+        /**
+         * Category class that was missing a parser. This is a String instead of a class because the class might not be on the classpath
+         * of all nodes or it might be exclusive to a plugin or something.
+         */
+        public String getCategoryClass() {
+            return categoryClass;
+        }
+
+        /**
+         * Name of the missing parser.
+         */
+        public String getName() {
+            return name;
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java b/core/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java
index 44d9e6e1993b1..2e9c62abe3bdd 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java
@@ -18,9 +18,8 @@
  */
 package org.elasticsearch.common.xcontent;
 
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.ParsingException;
 
 import java.io.IOException;
@@ -46,8 +45,8 @@
 
 /**
  * A declarative, stateless parser that turns XContent into setter calls. A single parser should be defined for each object being parsed,
- * nested elements can be added via {@link #declareObject(BiConsumer, BiFunction, ParseField)} which should be satisfied where possible by
- * passing another instance of {@link ObjectParser}, this one customized for that Object.
+ * nested elements can be added via {@link #declareObject(BiConsumer, ContextParser, ParseField)} which should be satisfied where possible
+ * by passing another instance of {@link ObjectParser}, this one customized for that Object.
  * <p>
  * This class works well for object that do have a constructor argument or that can be built using information available from earlier in the
  * XContent. For objects that have constructors with required arguments that are specified on the same level as other fields see
@@ -67,7 +66,7 @@
  * It's highly recommended to use the high level declare methods like {@link #declareString(BiConsumer, ParseField)} instead of
  * {@link #declareField} which can be used to implement exceptional parsing operations not covered by the high level methods.
  */
-public final class ObjectParser<Value, Context extends ParseFieldMatcherSupplier> extends AbstractObjectParser<Value, Context> {
+public final class ObjectParser<Value, Context> extends AbstractObjectParser<Value, Context> {
     /**
      * Adapts an array (or varags) setter into a list setter.
      */
@@ -83,6 +82,11 @@ public static <Value, ElementValue> BiConsumer<Value, List<ElementValue>> fromLi
     private final Map<String, FieldParser> fieldParserMap = new HashMap<>();
     private final String name;
     private final Supplier<Value> valueSupplier;
+    /**
+     * Should this parser ignore unknown fields? This should generally be set to true only when parsing responses from external systems,
+     * never when parsing requests from users.
+     */
+    private final boolean ignoreUnknownFields;
 
     /**
      * Creates a new ObjectParser instance with a name. This name is used to reference the parser in exceptions and messages.
@@ -96,18 +100,31 @@ public ObjectParser(String name) {
      * @param name the parsers name, used to reference the parser in exceptions and messages.
      * @param valueSupplier a supplier that creates a new Value instance used when the parser is used as an inner object parser.
      */
-    public ObjectParser(String name, Supplier<Value> valueSupplier) {
+    public ObjectParser(String name, @Nullable Supplier<Value> valueSupplier) {
+        this(name, false, valueSupplier);
+    }
+
+    /**
+     * Creates a new ObjectParser instance which a name.
+     * @param name the parsers name, used to reference the parser in exceptions and messages.
+     * @param ignoreUnknownFields Should this parser ignore unknown fields? This should generally be set to true only when parsing
+     *      responses from external systems, never when parsing requests from users.
+     * @param valueSupplier a supplier that creates a new Value instance used when the parser is used as an inner object parser.
+     */
+    public ObjectParser(String name, boolean ignoreUnknownFields, @Nullable Supplier<Value> valueSupplier) {
         this.name = name;
         this.valueSupplier = valueSupplier;
+        this.ignoreUnknownFields = ignoreUnknownFields;
     }
 
     /**
      * Parses a Value from the given {@link XContentParser}
      * @param parser the parser to build a value from
-     * @param context must at least provide a {@link ParseFieldMatcher}
+     * @param context context needed for parsing
      * @return a new value instance drawn from the provided value supplier on {@link #ObjectParser(String, Supplier)}
      * @throws IOException if an IOException occurs.
      */
+    @Override
     public Value parse(XContentParser parser, Context context) throws IOException {
         if (valueSupplier == null) {
             throw new NullPointerException("valueSupplier is not set");
@@ -134,7 +151,7 @@ public Value parse(XContentParser parser, Value value, Context context) throws I
             }
         }
 
-        FieldParser<Value> fieldParser = null;
+        FieldParser fieldParser = null;
         String currentFieldName = null;
         while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
             if (token == XContentParser.Token.FIELD_NAME) {
@@ -144,9 +161,13 @@ public Value parse(XContentParser parser, Value value, Context context) throws I
                 if (currentFieldName == null) {
                     throw new IllegalStateException("[" + name  + "] no field found");
                 }
-                assert fieldParser != null;
-                fieldParser.assertSupports(name, token, currentFieldName, context.getParseFieldMatcher());
-                parseSub(parser, fieldParser, currentFieldName, value, context);
+                if (fieldParser == null) {
+                    assert ignoreUnknownFields : "this should only be possible if configured to ignore known fields";
+                    parser.skipChildren(); // noop if parser points to a value, skips children if parser is start object or start array
+                } else {
+                    fieldParser.assertSupports(name, token, currentFieldName);
+                    parseSub(parser, fieldParser, currentFieldName, value, context);
+                }
                 fieldParser = null;
             }
         }
@@ -169,6 +190,12 @@ public interface Parser<Value, Context> {
         void parse(XContentParser parser, Value value, Context context) throws IOException;
     }
     public void declareField(Parser<Value, Context> p, ParseField parseField, ValueType type) {
+        if (parseField == null) {
+            throw new IllegalArgumentException("[parseField] is required");
+        }
+        if (type == null) {
+            throw new IllegalArgumentException("[type] is required");
+        }
         FieldParser fieldParser = new FieldParser(p, type.supportedTokens(), parseField, type);
         for (String fieldValue : parseField.getAllNamesIncludedDeprecated()) {
             fieldParserMap.putIfAbsent(fieldValue, fieldParser);
@@ -178,6 +205,12 @@ public void declareField(Parser<Value, Context> p, ParseField parseField, ValueT
     @Override
     public <T> void declareField(BiConsumer<Value, T> consumer, ContextParser<Context, T> parser, ParseField parseField,
             ValueType type) {
+        if (consumer == null) {
+            throw new IllegalArgumentException("[consumer] is required");
+        }
+        if (parser == null) {
+            throw new IllegalArgumentException("[parser] is required");
+        }
         declareField((p, v, c) -> consumer.accept(v, parser.parse(p, c)), parseField, type);
     }
 
@@ -322,13 +355,13 @@ public String getName() {
         return name;
     }
 
-    private void parseArray(XContentParser parser, FieldParser<Value> fieldParser, String currentFieldName, Value value, Context context)
+    private void parseArray(XContentParser parser, FieldParser fieldParser, String currentFieldName, Value value, Context context)
             throws IOException {
         assert parser.currentToken() == XContentParser.Token.START_ARRAY : "Token was: " + parser.currentToken();
         parseValue(parser, fieldParser, currentFieldName, value, context);
     }
 
-    private void parseValue(XContentParser parser, FieldParser<Value> fieldParser, String currentFieldName, Value value, Context context)
+    private void parseValue(XContentParser parser, FieldParser fieldParser, String currentFieldName, Value value, Context context)
             throws IOException {
         try {
             fieldParser.parser.parse(parser, value, context);
@@ -337,7 +370,7 @@ private void parseValue(XContentParser parser, FieldParser<Value> fieldParser, S
         }
     }
 
-    private void parseSub(XContentParser parser, FieldParser<Value> fieldParser, String currentFieldName, Value value, Context context)
+    private void parseSub(XContentParser parser, FieldParser fieldParser, String currentFieldName, Value value, Context context)
             throws IOException {
         final XContentParser.Token token = parser.currentToken();
         switch (token) {
@@ -361,28 +394,28 @@ private void parseSub(XContentParser parser, FieldParser<Value> fieldParser, Str
     }
 
     private FieldParser getParser(String fieldName) {
-        FieldParser<Value> parser = fieldParserMap.get(fieldName);
-        if (parser == null) {
+        FieldParser parser = fieldParserMap.get(fieldName);
+        if (parser == null && false == ignoreUnknownFields) {
             throw new IllegalArgumentException("[" + name  + "] unknown field [" + fieldName + "], parser not found");
         }
         return parser;
     }
 
-    public static class FieldParser<T> {
-        private final Parser parser;
+    private class FieldParser {
+        private final Parser<Value, Context> parser;
         private final EnumSet<XContentParser.Token> supportedTokens;
         private final ParseField parseField;
         private final ValueType type;
 
-        public FieldParser(Parser parser, EnumSet<XContentParser.Token> supportedTokens, ParseField parseField, ValueType type) {
+        FieldParser(Parser<Value, Context> parser, EnumSet<XContentParser.Token> supportedTokens, ParseField parseField, ValueType type) {
             this.parser = parser;
             this.supportedTokens = supportedTokens;
             this.parseField = parseField;
             this.type = type;
         }
 
-        public void assertSupports(String parserName, XContentParser.Token token, String currentFieldName, ParseFieldMatcher matcher) {
-            if (matcher.match(currentFieldName, parseField) == false) {
+        void assertSupports(String parserName, XContentParser.Token token, String currentFieldName) {
+            if (parseField.match(currentFieldName) == false) {
                 throw new IllegalStateException("[" + parserName  + "] parsefield doesn't accept: " + currentFieldName);
             }
             if (supportedTokens.contains(token) == false) {
@@ -415,10 +448,11 @@ public enum ValueType {
         FLOAT(VALUE_NUMBER, VALUE_STRING),
         FLOAT_OR_NULL(VALUE_NUMBER, VALUE_STRING, VALUE_NULL),
         DOUBLE(VALUE_NUMBER, VALUE_STRING),
+        DOUBLE_OR_NULL(VALUE_NUMBER, VALUE_STRING, VALUE_NULL),
         LONG(VALUE_NUMBER, VALUE_STRING),
         LONG_OR_NULL(VALUE_NUMBER, VALUE_STRING, VALUE_NULL),
         INT(VALUE_NUMBER, VALUE_STRING),
-        BOOLEAN(VALUE_BOOLEAN),
+        BOOLEAN(VALUE_BOOLEAN, VALUE_STRING),
         STRING_ARRAY(START_ARRAY, VALUE_STRING),
         FLOAT_ARRAY(START_ARRAY, VALUE_NUMBER, VALUE_STRING),
         DOUBLE_ARRAY(START_ARRAY, VALUE_NUMBER, VALUE_STRING),
@@ -429,7 +463,10 @@ public enum ValueType {
         OBJECT_ARRAY(START_OBJECT, START_ARRAY),
         OBJECT_OR_BOOLEAN(START_OBJECT, VALUE_BOOLEAN),
         OBJECT_OR_STRING(START_OBJECT, VALUE_STRING),
-        VALUE(VALUE_BOOLEAN, VALUE_NULL, VALUE_EMBEDDED_OBJECT, VALUE_NUMBER, VALUE_STRING);
+        OBJECT_ARRAY_BOOLEAN_OR_STRING(START_OBJECT, START_ARRAY, VALUE_BOOLEAN, VALUE_STRING),
+        OBJECT_ARRAY_OR_STRING(START_OBJECT, START_ARRAY, VALUE_STRING),
+        VALUE(VALUE_BOOLEAN, VALUE_NULL, VALUE_EMBEDDED_OBJECT, VALUE_NUMBER, VALUE_STRING),
+        VALUE_OBJECT_ARRAY(VALUE_BOOLEAN, VALUE_NULL, VALUE_EMBEDDED_OBJECT, VALUE_NUMBER, VALUE_STRING, START_OBJECT, START_ARRAY);
 
         private final EnumSet<XContentParser.Token> tokens;
 
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/ParseFieldRegistry.java b/core/src/main/java/org/elasticsearch/common/xcontent/ParseFieldRegistry.java
index 81f5b995c1815..0282fba764621 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/ParseFieldRegistry.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/ParseFieldRegistry.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.common.xcontent;
 
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.collect.Tuple;
 
@@ -75,12 +74,11 @@ public void register(T value, ParseField parseField) {
      * Lookup a value from the registry by name while checking that the name matches the ParseField.
      *
      * @param name The name of the thing to look up.
-     * @param parseFieldMatcher to build nice error messages.
      * @return The value being looked up. Never null.
      * @throws ParsingException if the named thing isn't in the registry or the name was deprecated and deprecated names aren't supported.
      */
-    public T lookup(String name, ParseFieldMatcher parseFieldMatcher, XContentLocation xContentLocation) {
-        T value = lookupReturningNullIfNotFound(name, parseFieldMatcher);
+    public T lookup(String name, XContentLocation xContentLocation) {
+        T value = lookupReturningNullIfNotFound(name);
         if (value == null) {
             throw new ParsingException(xContentLocation, "no [" + registryName + "] registered for [" + name + "]");
         }
@@ -91,19 +89,17 @@ public T lookup(String name, ParseFieldMatcher parseFieldMatcher, XContentLocati
      * Lookup a value from the registry by name while checking that the name matches the ParseField.
      *
      * @param name The name of the thing to look up.
-     * @param parseFieldMatcher The parseFieldMatcher. This is used to resolve the {@link ParseFieldMatcher} and to build nice
-     *        error messages.
      * @return The value being looked up or null if it wasn't found.
      * @throws ParsingException if the named thing isn't in the registry or the name was deprecated and deprecated names aren't supported.
      */
-    public T lookupReturningNullIfNotFound(String name, ParseFieldMatcher parseFieldMatcher) {
+    public T lookupReturningNullIfNotFound(String name) {
         Tuple<ParseField, T> parseFieldAndValue = registry.get(name);
         if (parseFieldAndValue == null) {
             return null;
         }
         ParseField parseField = parseFieldAndValue.v1();
         T value = parseFieldAndValue.v2();
-        boolean match = parseFieldMatcher.match(name, parseField);
+        boolean match = parseField.match(name);
         //this is always expected to match, ParseField is useful for deprecation warnings etc. here
         assert match : "ParseField did not match registered name [" + name + "][" + registryName + "]";
         return value;
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/StatusToXContent.java b/core/src/main/java/org/elasticsearch/common/xcontent/StatusToXContent.java
deleted file mode 100644
index f22aa39613fad..0000000000000
--- a/core/src/main/java/org/elasticsearch/common/xcontent/StatusToXContent.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.common.xcontent;
-
-import org.elasticsearch.rest.RestStatus;
-
-/**
- * Objects that can both render themselves in as json/yaml/etc and can provide a {@link RestStatus} for their response. Usually should be
- * implemented by top level responses sent back to users from REST endpoints.
- */
-public interface StatusToXContent extends ToXContent {
-
-    /**
-     * Returns the REST status to make sure it is returned correctly
-     */
-    RestStatus status();
-}
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/StatusToXContentObject.java b/core/src/main/java/org/elasticsearch/common/xcontent/StatusToXContentObject.java
new file mode 100644
index 0000000000000..ba6ccdfffad08
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/StatusToXContentObject.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.common.xcontent;
+
+import org.elasticsearch.rest.RestStatus;
+
+/**
+ * Objects that can both render themselves in as json/yaml/etc and can provide a {@link RestStatus} for their response. Usually should be
+ * implemented by top level responses sent back to users from REST endpoints.
+ */
+public interface StatusToXContentObject extends ToXContentObject {
+
+    /**
+     * Returns the REST status to make sure it is returned correctly
+     */
+    RestStatus status();
+}
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/ToXContent.java b/core/src/main/java/org/elasticsearch/common/xcontent/ToXContent.java
index 01111fa940a48..92fc675807b85 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/ToXContent.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/ToXContent.java
@@ -20,15 +20,18 @@
 package org.elasticsearch.common.xcontent;
 
 import org.elasticsearch.common.Booleans;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.Loggers;
 
 import java.io.IOException;
 import java.util.Map;
 
 /**
  * An interface allowing to transfer an object to "XContent" using an {@link XContentBuilder}.
+ * The output may or may not be a value object. Objects implementing {@link ToXContentObject} output a valid value
+ * but those that don't may or may not require emitting a startObject and an endObject.
  */
 public interface ToXContent {
-
     interface Params {
         String param(String key);
 
@@ -63,6 +66,7 @@ public Boolean paramAsBoolean(String key, Boolean defaultValue) {
     };
 
     class MapParams implements Params {
+        private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(MapParams.class));
 
         private final Map<String, String> params;
 
@@ -86,12 +90,16 @@ public String param(String key, String defaultValue) {
 
         @Override
         public boolean paramAsBoolean(String key, boolean defaultValue) {
-            return Booleans.parseBoolean(param(key), defaultValue);
+            return paramAsBoolean(key, (Boolean) defaultValue);
         }
 
         @Override
         public Boolean paramAsBoolean(String key, Boolean defaultValue) {
-            return Booleans.parseBoolean(param(key), defaultValue);
+            String rawParam = param(key);
+            if (rawParam != null && Booleans.isStrictlyBoolean(rawParam) == false) {
+                DEPRECATION_LOGGER.deprecated("Expected a boolean [true/false] for [{}] but got [{}]", key, rawParam);
+            }
+            return Booleans.parseBoolean(rawParam, defaultValue);
         }
     }
 
@@ -126,4 +134,8 @@ public Boolean paramAsBoolean(String key, Boolean defaultValue) {
     }
 
     XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException;
+
+    default boolean isFragment() {
+        return true;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/ToXContentObject.java b/core/src/main/java/org/elasticsearch/common/xcontent/ToXContentObject.java
new file mode 100644
index 0000000000000..ed9aa3047193a
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/ToXContentObject.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.xcontent;
+
+/**
+ * An interface allowing to transfer an object to "XContent" using an {@link XContentBuilder}.
+ * The difference between {@link ToXContent} and {@link ToXContentObject} is that the former may output a fragment that
+ * requires to start and end a new anonymous object externally, while the latter guarantees that what gets printed
+ * out is fully valid syntax without any external addition.
+ */
+public interface ToXContentObject extends ToXContent {
+
+    @Override
+    default boolean isFragment() {
+        return false;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContent.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContent.java
index 83facb00f004d..8dad910d1fc84 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/XContent.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContent.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.common.xcontent;
 
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 
 import java.io.IOException;
@@ -62,31 +61,31 @@ default XContentGenerator createGenerator(OutputStream os) throws IOException {
     /**
      * Creates a parser over the provided string content.
      */
-    XContentParser createParser(String content) throws IOException;
+    XContentParser createParser(NamedXContentRegistry xContentRegistry, String content) throws IOException;
 
     /**
      * Creates a parser over the provided input stream.
      */
-    XContentParser createParser(InputStream is) throws IOException;
+    XContentParser createParser(NamedXContentRegistry xContentRegistry, InputStream is) throws IOException;
 
     /**
      * Creates a parser over the provided bytes.
      */
-    XContentParser createParser(byte[] data) throws IOException;
+    XContentParser createParser(NamedXContentRegistry xContentRegistry, byte[] data) throws IOException;
 
     /**
      * Creates a parser over the provided bytes.
      */
-    XContentParser createParser(byte[] data, int offset, int length) throws IOException;
+    XContentParser createParser(NamedXContentRegistry xContentRegistry, byte[] data, int offset, int length) throws IOException;
 
     /**
      * Creates a parser over the provided bytes.
      */
-    XContentParser createParser(BytesReference bytes) throws IOException;
+    XContentParser createParser(NamedXContentRegistry xContentRegistry, BytesReference bytes) throws IOException;
 
     /**
      * Creates a parser over the provided reader.
      */
-    XContentParser createParser(Reader reader) throws IOException;
+    XContentParser createParser(NamedXContentRegistry xContentRegistry, Reader reader) throws IOException;
 
 }
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java
index 5274773b99457..f0427ce246669 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java
@@ -20,10 +20,9 @@
 package org.elasticsearch.common.xcontent;
 
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.geo.GeoPoint;
-import org.elasticsearch.common.io.BytesStream;
+import org.elasticsearch.common.io.stream.BytesStream;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.text.Text;
@@ -34,44 +33,109 @@
 import org.joda.time.format.DateTimeFormatter;
 import org.joda.time.format.ISODateTimeFormat;
 
+import java.io.Flushable;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
-import java.math.BigDecimal;
-import java.math.RoundingMode;
 import java.nio.file.Path;
+import java.util.Arrays;
 import java.util.Calendar;
 import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
+import java.util.IdentityHashMap;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
 /**
  * A utility to build XContent (ie json).
  */
-public final class XContentBuilder implements BytesStream, Releasable {
-
-    public static final DateTimeFormatter defaultDatePrinter = ISODateTimeFormat.dateTime().withZone(DateTimeZone.UTC);
+public final class XContentBuilder implements Releasable, Flushable {
 
+    /**
+     * Create a new {@link XContentBuilder} using the given {@link XContent} content.
+     * <p>
+     * The builder uses an internal {@link BytesStreamOutput} output stream to build the content.
+     * </p>
+     *
+     * @param xContent the {@link XContent}
+     * @return a new {@link XContentBuilder}
+     * @throws IOException if an {@link IOException} occurs while building the content
+     */
     public static XContentBuilder builder(XContent xContent) throws IOException {
         return new XContentBuilder(xContent, new BytesStreamOutput());
     }
 
+    /**
+     * Create a new {@link XContentBuilder} using the given {@link XContent} content and some inclusive and/or exclusive filters.
+     * <p>
+     * The builder uses an internal {@link BytesStreamOutput} output stream to build the content. When both exclusive and
+     * inclusive filters are provided, the underlying builder will first use exclusion filters to remove fields and then will check the
+     * remaining fields against the inclusive filters.
+     * <p>
+     *
+     * @param xContent the {@link XContent}
+     * @param includes the inclusive filters: only fields and objects that match the inclusive filters will be written to the output.
+     * @param excludes the exclusive filters: only fields and objects that don't match the exclusive filters will be written to the output.
+     * @throws IOException if an {@link IOException} occurs while building the content
+     */
     public static XContentBuilder builder(XContent xContent, Set<String> includes, Set<String> excludes) throws IOException {
         return new XContentBuilder(xContent, new BytesStreamOutput(), includes, excludes);
     }
 
-    private XContentGenerator generator;
+    public static final DateTimeFormatter DEFAULT_DATE_PRINTER = ISODateTimeFormat.dateTime().withZone(DateTimeZone.UTC);
+
+    private static final Map<Class<?>, Writer> WRITERS;
+    static {
+        Map<Class<?>, Writer> writers = new HashMap<>();
+        writers.put(Boolean.class, (b, v) -> b.value((Boolean) v));
+        writers.put(Byte.class, (b, v) -> b.value((Byte) v));
+        writers.put(byte[].class, (b, v) -> b.value((byte[]) v));
+        writers.put(BytesRef.class, (b, v) -> b.binaryValue((BytesRef) v));
+        writers.put(Date.class, (b, v) -> b.value((Date) v));
+        writers.put(Double.class, (b, v) -> b.value((Double) v));
+        writers.put(double[].class, (b, v) -> b.values((double[]) v));
+        writers.put(Float.class, (b, v) -> b.value((Float) v));
+        writers.put(float[].class, (b, v) -> b.values((float[]) v));
+        writers.put(GeoPoint.class, (b, v) -> b.value((GeoPoint) v));
+        writers.put(Integer.class, (b, v) -> b.value((Integer) v));
+        writers.put(int[].class, (b, v) -> b.values((int[]) v));
+        writers.put(Long.class, (b, v) -> b.value((Long) v));
+        writers.put(long[].class, (b, v) -> b.values((long[]) v));
+        writers.put(Short.class, (b, v) -> b.value((Short) v));
+        writers.put(short[].class, (b, v) -> b.values((short[]) v));
+        writers.put(String.class, (b, v) -> b.value((String) v));
+        writers.put(String[].class, (b, v) -> b.values((String[]) v));
+        writers.put(Text.class, (b, v) -> b.value((Text) v));
+
+        WRITERS = Collections.unmodifiableMap(writers);
+    }
+
+    @FunctionalInterface
+    private interface Writer {
+        void write(XContentBuilder builder, Object value) throws IOException;
+    }
+
+    /**
+     * XContentGenerator used to build the XContent object
+     */
+    private final XContentGenerator generator;
 
+    /**
+     * Output stream to which the built object is written
+     */
     private final OutputStream bos;
 
+    /**
+     * When this flag is set to true, some types of values are written in a format easier to read for a human.
+     */
     private boolean humanReadable = false;
 
     /**
-     * Constructs a new builder using the provided xcontent and an OutputStream. Make sure
+     * Constructs a new builder using the provided XContent and an OutputStream. Make sure
      * to call {@link #close()} when the builder is done with.
      */
     public XContentBuilder(XContent xContent, OutputStream bos) throws IOException {
@@ -79,7 +143,7 @@ public XContentBuilder(XContent xContent, OutputStream bos) throws IOException {
     }
 
     /**
-     * Constructs a new builder using the provided xcontent, an OutputStream and
+     * Constructs a new builder using the provided XContent, an OutputStream and
      * some filters. If filters are specified, only those values matching a
      * filter will be written to the output stream. Make sure to call
      * {@link #close()} when the builder is done with.
@@ -117,41 +181,45 @@ public boolean isPrettyPrint() {
         return generator.isPrettyPrint();
     }
 
+    /**
+     * Indicate that the current {@link XContentBuilder} must write a line feed ("\n")
+     * at the end of the built object.
+     * <p>
+     * This only applies for JSON XContent type. It has no effect for other types.
+     */
     public XContentBuilder lfAtEnd() {
         generator.usePrintLineFeedAtEnd();
         return this;
     }
 
+    /**
+     * Set the "human readable" flag. Once set, some types of values are written in a
+     * format easier to read for a human.
+     */
     public XContentBuilder humanReadable(boolean humanReadable) {
         this.humanReadable = humanReadable;
         return this;
     }
 
+    /**
+     * @return the value of the "human readable" flag. When the value is equal to true,
+     * some types of values are written in a format easier to read for a human.
+     */
     public boolean humanReadable() {
         return this.humanReadable;
     }
 
-    public XContentBuilder field(String name, ToXContent xContent) throws IOException {
-        field(name);
-        xContent.toXContent(this, ToXContent.EMPTY_PARAMS);
-        return this;
-    }
+    ////////////////////////////////////////////////////////////////////////////
+    // Structure (object, array, field, null values...)
+    //////////////////////////////////
 
-    public XContentBuilder field(String name, ToXContent xContent, ToXContent.Params params) throws IOException {
-        field(name);
-        xContent.toXContent(this, params);
+    public XContentBuilder startObject() throws IOException {
+        generator.writeStartObject();
         return this;
     }
 
     public XContentBuilder startObject(String name) throws IOException {
-        field(name);
-        startObject();
-        return this;
-    }
-
-    public XContentBuilder startObject() throws IOException {
-        generator.writeStartObject();
-        return this;
+        return field(name).startObject();
     }
 
     public XContentBuilder endObject() throws IOException {
@@ -159,33 +227,13 @@ public XContentBuilder endObject() throws IOException {
         return this;
     }
 
-    public XContentBuilder array(String name, String... values) throws IOException {
-        startArray(name);
-        for (String value : values) {
-            value(value);
-        }
-        endArray();
-        return this;
-    }
-
-    public XContentBuilder array(String name, Object... values) throws IOException {
-        startArray(name);
-        for (Object value : values) {
-            value(value);
-        }
-        endArray();
+    public XContentBuilder startArray() throws IOException {
+        generator.writeStartArray();
         return this;
     }
 
     public XContentBuilder startArray(String name) throws IOException {
-        field(name);
-        startArray();
-        return this;
-    }
-
-    public XContentBuilder startArray() throws IOException {
-        generator.writeStartArray();
-        return this;
+        return field(name).startArray();
     }
 
     public XContentBuilder endArray() throws IOException {
@@ -194,563 +242,780 @@ public XContentBuilder endArray() throws IOException {
     }
 
     public XContentBuilder field(String name) throws IOException {
-        if (name == null) {
-            throw new IllegalArgumentException("field name cannot be null");
-        }
+        ensureNameNotNull(name);
         generator.writeFieldName(name);
         return this;
     }
 
-    public XContentBuilder field(String name, char[] value, int offset, int length) throws IOException {
-        field(name);
-        if (value == null) {
-            generator.writeNull();
-        } else {
-            generator.writeString(value, offset, length);
-        }
+    public XContentBuilder nullField(String name) throws IOException {
+        ensureNameNotNull(name);
+        generator.writeNullField(name);
         return this;
     }
 
-    public XContentBuilder field(String name, String value) throws IOException {
-        field(name);
-        if (value == null) {
-            generator.writeNull();
-        } else {
-            generator.writeString(value);
-        }
+    public XContentBuilder nullValue() throws IOException {
+        generator.writeNull();
         return this;
     }
 
-    public XContentBuilder field(String name, Integer value) throws IOException {
-        field(name);
-        if (value == null) {
-            generator.writeNull();
-        } else {
-            generator.writeNumber(value.intValue());
-        }
-        return this;
+    ////////////////////////////////////////////////////////////////////////////
+    // Boolean
+    //////////////////////////////////
+
+    public XContentBuilder field(String name, Boolean value) throws IOException {
+        return (value == null) ? nullField(name) : field(name, value.booleanValue());
     }
 
-    public XContentBuilder field(String name, int value) throws IOException {
-        field(name);
-        generator.writeNumber(value);
+    public XContentBuilder field(String name, boolean value) throws IOException {
+        ensureNameNotNull(name);
+        generator.writeBooleanField(name, value);
         return this;
     }
 
-    public XContentBuilder field(String name, Long value) throws IOException {
-        field(name);
-        if (value == null) {
-            generator.writeNull();
-        } else {
-            generator.writeNumber(value.longValue());
+    public XContentBuilder array(String name, boolean[] values) throws IOException {
+        return field(name).values(values);
+    }
+
+    private XContentBuilder values(boolean[] values) throws IOException {
+        if (values == null) {
+            return nullValue();
+        }
+        startArray();
+        for (boolean b : values) {
+            value(b);
         }
+        endArray();
         return this;
     }
 
-    public XContentBuilder field(String name, long value) throws IOException {
-        field(name);
-        generator.writeNumber(value);
-        return this;
+    public XContentBuilder value(Boolean value) throws IOException {
+        return (value == null) ? nullValue() : value(value.booleanValue());
     }
 
-    public XContentBuilder field(String name, Float value) throws IOException {
-        field(name);
-        if (value == null) {
-            generator.writeNull();
-        } else {
-            generator.writeNumber(value.floatValue());
-        }
+    public XContentBuilder value(boolean value) throws IOException {
+        generator.writeBoolean(value);
         return this;
     }
 
-    public XContentBuilder field(String name, float value) throws IOException {
-        field(name);
+    ////////////////////////////////////////////////////////////////////////////
+    // Byte
+    //////////////////////////////////
+
+    public XContentBuilder field(String name, Byte value) throws IOException {
+        return (value == null) ? nullField(name) : field(name, value.byteValue());
+    }
+
+    public XContentBuilder field(String name, byte value) throws IOException {
+        return field(name).value(value);
+    }
+
+    public XContentBuilder value(Byte value) throws IOException {
+        return (value == null) ? nullValue() : value(value.byteValue());
+    }
+
+    public XContentBuilder value(byte value) throws IOException {
         generator.writeNumber(value);
         return this;
     }
 
+    ////////////////////////////////////////////////////////////////////////////
+    // Double
+    //////////////////////////////////
+
     public XContentBuilder field(String name, Double value) throws IOException {
-        field(name);
-        if (value == null) {
-            generator.writeNull();
-        } else {
-            generator.writeNumber(value);
-        }
-        return this;
+        return (value == null) ? nullField(name) : field(name, value.doubleValue());
     }
 
     public XContentBuilder field(String name, double value) throws IOException {
-        field(name);
-        generator.writeNumber(value);
+        ensureNameNotNull(name);
+        generator.writeNumberField(name, value);
         return this;
     }
 
-    public XContentBuilder field(String name, BigDecimal value) throws IOException {
-        return field(name, value, value.scale(), RoundingMode.HALF_UP, true);
+    public XContentBuilder array(String name, double[] values) throws IOException {
+        return field(name).values(values);
     }
 
-    public XContentBuilder field(String name, BigDecimal value, int scale, RoundingMode rounding, boolean toDouble) throws IOException {
-        field(name);
-        if (value == null) {
-            generator.writeNull();
-        } else {
-            if (toDouble) {
-                try {
-                    generator.writeNumber(value.setScale(scale, rounding).doubleValue());
-                } catch (ArithmeticException e) {
-                    generator.writeString(value.toEngineeringString());
-                }
-            } else {
-                generator.writeString(value.toEngineeringString());
-            }
+    private XContentBuilder values(double[] values) throws IOException {
+        if (values == null) {
+            return nullValue();
+        }
+        startArray();
+        for (double b : values) {
+            value(b);
         }
+        endArray();
         return this;
     }
 
-    /**
-     * Writes the binary content of the given BytesRef
-     * Use {@link org.elasticsearch.common.xcontent.XContentParser#binaryValue()} to read the value back
-     */
-    public XContentBuilder field(String name, BytesRef value) throws IOException {
-        field(name);
-        generator.writeBinary(value.bytes, value.offset, value.length);
-        return this;
+    public XContentBuilder value(Double value) throws IOException {
+        return (value == null) ? nullValue() : value(value.doubleValue());
     }
 
-    /**
-     * Writes the binary content of the given BytesReference
-     * Use {@link org.elasticsearch.common.xcontent.XContentParser#binaryValue()} to read the value back
-     */
-    public XContentBuilder field(String name, BytesReference value) throws IOException {
-        field(name);
-        final BytesRef ref = value.toBytesRef();
-        generator.writeBinary(ref.bytes, ref.offset, ref.length);
+    public XContentBuilder value(double value) throws IOException {
+        generator.writeNumber(value);
         return this;
     }
 
-    /**
-     * Writes the binary content of the given BytesRef as UTF-8 bytes
-     * Use {@link XContentParser#utf8Bytes()} to read the value back
-     */
-    public XContentBuilder utf8Field(String name, BytesRef value) throws IOException {
-        field(name);
-        generator.writeUTF8String(value.bytes, value.offset, value.length);
-        return this;
-    }
+    ////////////////////////////////////////////////////////////////////////////
+    // Float
+    //////////////////////////////////
 
-    public XContentBuilder field(String name, Text value) throws IOException {
-        field(name);
-        if (value.hasString()) {
-            generator.writeString(value.string());
-        } else {
-            // TODO: TextBytesOptimization we can use a buffer here to convert it? maybe add a request to jackson to support InputStream as well?
-            final BytesRef ref = value.bytes().toBytesRef();
-            generator.writeUTF8String(ref.bytes, ref.offset, ref.length);
-        }
-        return this;
+    public XContentBuilder field(String name, Float value) throws IOException {
+        return (value == null) ? nullField(name) : field(name, value.floatValue());
     }
 
-    public XContentBuilder field(String name, byte[] value, int offset, int length) throws IOException {
-        field(name);
-        generator.writeBinary(value, offset, length);
+    public XContentBuilder field(String name, float value) throws IOException {
+        ensureNameNotNull(name);
+        generator.writeNumberField(name, value);
         return this;
     }
 
-    public XContentBuilder field(String name, Map<String, Object> value) throws IOException {
-        field(name);
-        value(value);
-        return this;
+    public XContentBuilder array(String name, float[] values) throws IOException {
+        return field(name).values(values);
     }
 
-    public XContentBuilder field(String name, Iterable<?> value) throws IOException {
-        if (value instanceof Path) {
-            //treat Paths as single value
-            field(name);
-            value(value);
-        } else {
-            startArray(name);
-            for (Object o : value) {
-                value(o);
-            }
-            endArray();
+    private XContentBuilder values(float[] values) throws IOException {
+        if (values == null) {
+            return nullValue();
         }
-        return this;
-    }
-
-    public XContentBuilder field(String name, boolean... value) throws IOException {
-        startArray(name);
-        for (boolean o : value) {
-            value(o);
+        startArray();
+        for (float f : values) {
+            value(f);
         }
         endArray();
         return this;
     }
 
-    public XContentBuilder field(String name, String... value) throws IOException {
-        startArray(name);
-        for (String o : value) {
-            value(o);
-        }
-        endArray();
-        return this;
+    public XContentBuilder value(Float value) throws IOException {
+        return (value == null) ? nullValue() : value(value.floatValue());
     }
 
-    public XContentBuilder field(String name, Object... value) throws IOException {
-        startArray(name);
-        for (Object o : value) {
-            value(o);
-        }
-        endArray();
+    public XContentBuilder value(float value) throws IOException {
+        generator.writeNumber(value);
         return this;
     }
 
-    public XContentBuilder field(String name, int... value) throws IOException {
-        startArray(name);
-        for (Object o : value) {
-            value(o);
-        }
-        endArray();
-        return this;
+    ////////////////////////////////////////////////////////////////////////////
+    // Integer
+    //////////////////////////////////
+
+    public XContentBuilder field(String name, Integer value) throws IOException {
+        return (value == null) ? nullField(name) : field(name, value.intValue());
     }
 
-    public XContentBuilder field(String name, long... value) throws IOException {
-        startArray(name);
-        for (Object o : value) {
-            value(o);
-        }
-        endArray();
+    public XContentBuilder field(String name, int value) throws IOException {
+        ensureNameNotNull(name);
+        generator.writeNumberField(name, value);
         return this;
     }
 
-    public XContentBuilder field(String name, float... value) throws IOException {
-        startArray(name);
-        for (Object o : value) {
-            value(o);
-        }
-        endArray();
-        return this;
+    public XContentBuilder array(String name, int[] values) throws IOException {
+        return field(name).values(values);
     }
 
-    public XContentBuilder field(String name, double... value) throws IOException {
-        startArray(name);
-        for (Object o : value) {
-            value(o);
+    private XContentBuilder values(int[] values) throws IOException {
+        if (values == null) {
+            return nullValue();
+        }
+        startArray();
+        for (int i : values) {
+            value(i);
         }
         endArray();
         return this;
     }
 
-    public XContentBuilder field(String name, Object value) throws IOException {
-        field(name);
-        writeValue(value);
-        return this;
+    public XContentBuilder value(Integer value) throws IOException {
+        return (value == null) ? nullValue() : value(value.intValue());
     }
 
-    public XContentBuilder value(Object value) throws IOException {
-        writeValue(value);
+    public XContentBuilder value(int value) throws IOException {
+        generator.writeNumber(value);
         return this;
     }
 
-    public XContentBuilder field(String name, boolean value) throws IOException {
-        field(name);
-        generator.writeBoolean(value);
+    ////////////////////////////////////////////////////////////////////////////
+    // Long
+    //////////////////////////////////
+
+    public XContentBuilder field(String name, Long value) throws IOException {
+        return (value == null) ? nullField(name) : field(name, value.longValue());
+    }
+
+    public XContentBuilder field(String name, long value) throws IOException {
+        ensureNameNotNull(name);
+        generator.writeNumberField(name, value);
         return this;
     }
 
-    public XContentBuilder field(String name, byte[] value) throws IOException {
-        field(name);
-        if (value == null) {
-            generator.writeNull();
-        } else {
-            generator.writeBinary(value);
+    public XContentBuilder array(String name, long[] values) throws IOException {
+        return field(name).values(values);
+    }
+
+    private XContentBuilder values(long[] values) throws IOException {
+        if (values == null) {
+            return nullValue();
+        }
+        startArray();
+        for (long l : values) {
+            value(l);
         }
+        endArray();
         return this;
     }
 
-    public XContentBuilder field(String name, ReadableInstant date) throws IOException {
-        field(name);
-        return value(date);
+    public XContentBuilder value(Long value) throws IOException {
+        return (value == null) ? nullValue() : value(value.longValue());
     }
 
-    public XContentBuilder field(String name, ReadableInstant date, DateTimeFormatter formatter) throws IOException {
-        field(name);
-        return value(date, formatter);
+    public XContentBuilder value(long value) throws IOException {
+        generator.writeNumber(value);
+        return this;
     }
 
-    public XContentBuilder field(String name, Date date) throws IOException {
-        field(name);
-        return value(date);
-    }
+    ////////////////////////////////////////////////////////////////////////////
+    // Short
+    //////////////////////////////////
 
-    public XContentBuilder field(String name, Date date, DateTimeFormatter formatter) throws IOException {
-        field(name);
-        return value(date, formatter);
+    public XContentBuilder field(String name, Short value) throws IOException {
+        return (value == null) ? nullField(name) : field(name, value.shortValue());
     }
 
-    public XContentBuilder nullField(String name) throws IOException {
-        generator.writeNullField(name);
-        return this;
+    public XContentBuilder field(String name, short value) throws IOException {
+        return field(name).value(value);
     }
 
-    public XContentBuilder nullValue() throws IOException {
-        generator.writeNull();
-        return this;
+    public XContentBuilder array(String name, short[] values) throws IOException {
+        return field(name).values(values);
     }
 
-    public XContentBuilder rawField(String fieldName, InputStream content) throws IOException {
-        generator.writeRawField(fieldName, content);
+    private XContentBuilder values(short[] values) throws IOException {
+        if (values == null) {
+            return nullValue();
+        }
+        startArray();
+        for (short s : values) {
+            value(s);
+        }
+        endArray();
         return this;
     }
 
-    public XContentBuilder rawField(String fieldName, BytesReference content) throws IOException {
-        generator.writeRawField(fieldName, content);
-        return this;
+    public XContentBuilder value(Short value) throws IOException {
+        return (value == null) ? nullValue() : value(value.shortValue());
     }
 
-    public XContentBuilder rawValue(BytesReference content) throws IOException {
-        generator.writeRawValue(content);
+    public XContentBuilder value(short value) throws IOException {
+        generator.writeNumber(value);
         return this;
     }
 
-    public XContentBuilder timeValueField(String rawFieldName, String readableFieldName, TimeValue timeValue) throws IOException {
-        if (humanReadable) {
-            field(readableFieldName, timeValue.toString());
+    ////////////////////////////////////////////////////////////////////////////
+    // String
+    //////////////////////////////////
+
+    public XContentBuilder field(String name, String value) throws IOException {
+        if (value == null) {
+            return nullField(name);
         }
-        field(rawFieldName, timeValue.millis());
+        ensureNameNotNull(name);
+        generator.writeStringField(name, value);
         return this;
     }
 
-    public XContentBuilder timeValueField(String rawFieldName, String readableFieldName, long rawTime) throws IOException {
-        if (humanReadable) {
-            field(readableFieldName, new TimeValue(rawTime).toString());
+    public XContentBuilder array(String name, String... values) throws IOException {
+        return field(name).values(values);
+    }
+
+    private XContentBuilder values(String[] values) throws IOException {
+        if (values == null) {
+            return nullValue();
         }
-        field(rawFieldName, rawTime);
+        startArray();
+        for (String s : values) {
+            value(s);
+        }
+        endArray();
         return this;
     }
 
-    public XContentBuilder timeValueField(String rawFieldName, String readableFieldName, long rawTime, TimeUnit timeUnit) throws
-        IOException {
-        if (humanReadable) {
-            field(readableFieldName, new TimeValue(rawTime, timeUnit).toString());
+    public XContentBuilder value(String value) throws IOException {
+        if (value == null) {
+            return nullValue();
         }
-        field(rawFieldName, rawTime);
+        generator.writeString(value);
         return this;
     }
 
-    public XContentBuilder dateValueField(String rawFieldName, String readableFieldName, long rawTimestamp) throws IOException {
-        if (humanReadable) {
-            field(readableFieldName, defaultDatePrinter.print(rawTimestamp));
+    ////////////////////////////////////////////////////////////////////////////
+    // Binary
+    //////////////////////////////////
+
+    public XContentBuilder field(String name, byte[] value) throws IOException {
+        if (value == null) {
+            return nullField(name);
         }
-        field(rawFieldName, rawTimestamp);
+        ensureNameNotNull(name);
+        generator.writeBinaryField(name, value);
         return this;
     }
 
-    public XContentBuilder byteSizeField(String rawFieldName, String readableFieldName, ByteSizeValue byteSizeValue) throws IOException {
-        if (humanReadable) {
-            field(readableFieldName, byteSizeValue.toString());
+    public XContentBuilder value(byte[] value) throws IOException {
+        if (value == null) {
+            return nullValue();
         }
-        field(rawFieldName, byteSizeValue.bytes());
+        generator.writeBinary(value);
         return this;
     }
 
-    public XContentBuilder byteSizeField(String rawFieldName, String readableFieldName, long rawSize) throws IOException {
-        if (humanReadable) {
-            field(readableFieldName, new ByteSizeValue(rawSize).toString());
+    public XContentBuilder field(String name, byte[] value, int offset, int length) throws IOException {
+        return field(name).value(value, offset, length);
+    }
+
+    public XContentBuilder value(byte[] value, int offset, int length) throws IOException {
+        if (value == null) {
+            return nullValue();
         }
-        field(rawFieldName, rawSize);
+        generator.writeBinary(value, offset, length);
         return this;
     }
 
-    public XContentBuilder percentageField(String rawFieldName, String readableFieldName, double percentage) throws IOException {
-        if (humanReadable) {
-            field(readableFieldName, String.format(Locale.ROOT, "%1.1f%%", percentage));
+    /**
+     * Writes the binary content of the given {@link BytesRef}.
+     *
+     * Use {@link org.elasticsearch.common.xcontent.XContentParser#binaryValue()} to read the value back
+     */
+    public XContentBuilder field(String name, BytesRef value) throws IOException {
+        return field(name).binaryValue(value);
+    }
+
+    /**
+     * Writes the binary content of the given {@link BytesRef} as UTF-8 bytes.
+     *
+     * Use {@link XContentParser#utf8Bytes()} to read the value back
+     */
+    public XContentBuilder utf8Field(String name, BytesRef value) throws IOException {
+        return field(name).utf8Value(value);
+    }
+
+    /**
+     * Writes the binary content of the given {@link BytesRef}.
+     *
+     * Use {@link org.elasticsearch.common.xcontent.XContentParser#binaryValue()} to read the value back
+     */
+    public XContentBuilder binaryValue(BytesRef value) throws IOException {
+        if (value == null) {
+            return nullValue();
         }
-        field(rawFieldName, percentage);
+        value(value.bytes, value.offset, value.length);
         return this;
     }
 
-    public XContentBuilder value(Boolean value) throws IOException {
+    /**
+     * Writes the binary content of the given {@link BytesRef} as UTF-8 bytes.
+     *
+     * Use {@link XContentParser#utf8Bytes()} to read the value back
+     */
+    public XContentBuilder utf8Value(BytesRef value) throws IOException {
         if (value == null) {
             return nullValue();
         }
-        return value(value.booleanValue());
+        generator.writeUTF8String(value.bytes, value.offset, value.length);
+        return this;
     }
 
-    public XContentBuilder value(boolean value) throws IOException {
-        generator.writeBoolean(value);
-        return this;
+    /**
+     * Writes the binary content of the given {@link BytesReference}.
+     *
+     * Use {@link org.elasticsearch.common.xcontent.XContentParser#binaryValue()} to read the value back
+     */
+    public XContentBuilder field(String name, BytesReference value) throws IOException {
+        return field(name).value(value);
     }
 
-    public XContentBuilder value(ReadableInstant date) throws IOException {
-        return value(date, defaultDatePrinter);
+    /**
+     * Writes the binary content of the given {@link BytesReference}.
+     *
+     * Use {@link org.elasticsearch.common.xcontent.XContentParser#binaryValue()} to read the value back
+     */
+    public XContentBuilder value(BytesReference value) throws IOException {
+        return (value == null) ? nullValue() : binaryValue(value.toBytesRef());
     }
 
-    public XContentBuilder value(ReadableInstant date, DateTimeFormatter dateTimeFormatter) throws IOException {
-        if (date == null) {
+    ////////////////////////////////////////////////////////////////////////////
+    // Text
+    //////////////////////////////////
+
+    public XContentBuilder field(String name, Text value) throws IOException {
+        return field(name).value(value);
+    }
+
+    public XContentBuilder value(Text value) throws IOException {
+        if (value == null) {
             return nullValue();
+        } else if (value.hasString()) {
+            return value(value.string());
+        } else {
+            // TODO: TextBytesOptimization we can use a buffer here to convert it? maybe add a
+            // request to jackson to support InputStream as well?
+            return utf8Value(value.bytes().toBytesRef());
         }
-        return value(dateTimeFormatter.print(date));
     }
 
-    public XContentBuilder value(Date date) throws IOException {
-        return value(date, defaultDatePrinter);
+    ////////////////////////////////////////////////////////////////////////////
+    // Date
+    //////////////////////////////////
+
+    public XContentBuilder field(String name, ReadableInstant value) throws IOException {
+        return field(name).value(value);
     }
 
-    public XContentBuilder value(Date date, DateTimeFormatter dateTimeFormatter) throws IOException {
-        if (date == null) {
-            return nullValue();
-        }
-        return value(dateTimeFormatter.print(date.getTime()));
+    public XContentBuilder field(String name, ReadableInstant value, DateTimeFormatter formatter) throws IOException {
+        return field(name).value(value, formatter);
     }
 
-    public XContentBuilder value(Integer value) throws IOException {
+    public XContentBuilder value(ReadableInstant value) throws IOException {
+        return value(value, DEFAULT_DATE_PRINTER);
+    }
+
+    public XContentBuilder value(ReadableInstant value, DateTimeFormatter formatter) throws IOException {
         if (value == null) {
             return nullValue();
         }
-        return value(value.intValue());
+        ensureFormatterNotNull(formatter);
+        return value(formatter.print(value));
     }
 
-    public XContentBuilder value(int value) throws IOException {
-        generator.writeNumber(value);
-        return this;
+    public XContentBuilder field(String name, Date value) throws IOException {
+        return field(name).value(value);
     }
 
-    public XContentBuilder value(Long value) throws IOException {
+    public XContentBuilder field(String name, Date value, DateTimeFormatter formatter) throws IOException {
+        return field(name).value(value, formatter);
+    }
+
+    public XContentBuilder value(Date value) throws IOException {
+        return value(value, DEFAULT_DATE_PRINTER);
+    }
+
+    public XContentBuilder value(Date value, DateTimeFormatter formatter) throws IOException {
         if (value == null) {
             return nullValue();
         }
-        return value(value.longValue());
+        return value(formatter, value.getTime());
     }
 
-    public XContentBuilder value(long value) throws IOException {
-        generator.writeNumber(value);
+    public XContentBuilder dateField(String name, String readableName, long value) throws IOException {
+        if (humanReadable) {
+            field(readableName).value(DEFAULT_DATE_PRINTER, value);
+        }
+        field(name, value);
         return this;
     }
 
-    public XContentBuilder value(Float value) throws IOException {
+    XContentBuilder value(Calendar value) throws IOException {
         if (value == null) {
             return nullValue();
         }
-        return value(value.floatValue());
+        return value(DEFAULT_DATE_PRINTER, value.getTimeInMillis());
     }
 
-    public XContentBuilder value(float value) throws IOException {
-        generator.writeNumber(value);
-        return this;
+    XContentBuilder value(DateTimeFormatter formatter, long value) throws IOException {
+        ensureFormatterNotNull(formatter);
+        return value(formatter.print(value));
     }
 
-    public XContentBuilder value(Double value) throws IOException {
+    ////////////////////////////////////////////////////////////////////////////
+    // GeoPoint & LatLon
+    //////////////////////////////////
+
+    public XContentBuilder field(String name, GeoPoint value) throws IOException {
+        return field(name).value(value);
+    }
+
+    public XContentBuilder value(GeoPoint value) throws IOException {
         if (value == null) {
             return nullValue();
         }
-        return value(value.doubleValue());
+        return latlon(value.getLat(), value.getLon());
     }
 
-    public XContentBuilder value(double value) throws IOException {
-        generator.writeNumber(value);
-        return this;
+    public XContentBuilder latlon(String name, double lat, double lon) throws IOException {
+        return field(name).latlon(lat, lon);
     }
 
-    public XContentBuilder value(String value) throws IOException {
+    public XContentBuilder latlon(double lat, double lon) throws IOException {
+        return startObject().field("lat", lat).field("lon", lon).endObject();
+    }
+
+    ////////////////////////////////////////////////////////////////////////////
+    // Path
+    //////////////////////////////////
+
+    public XContentBuilder value(Path value) throws IOException {
         if (value == null) {
             return nullValue();
         }
-        generator.writeString(value);
-        return this;
+        return value(value.toString());
     }
 
-    public XContentBuilder value(byte[] value) throws IOException {
-        if (value == null) {
+    ////////////////////////////////////////////////////////////////////////////
+    // Objects
+    //
+    // These methods are used when the type of value is unknown. It tries to fallback
+    // on typed methods and use Object.toString() as a last resort. Always prefer using
+    // typed methods over this.
+    //////////////////////////////////
+
+    public XContentBuilder field(String name, Object value) throws IOException {
+        return field(name).value(value);
+    }
+
+    public XContentBuilder array(String name, Object... values) throws IOException {
+        return field(name).values(values);
+    }
+
+    XContentBuilder values(Object[] values) throws IOException {
+        if (values == null) {
             return nullValue();
         }
-        generator.writeBinary(value);
+
+        // checks that the array of object does not contain references to itself because
+        // iterating over entries will cause a stackoverflow error
+        ensureNoSelfReferences(values);
+
+        startArray();
+        for (Object o : values) {
+            value(o);
+        }
+        endArray();
         return this;
     }
 
-    public XContentBuilder value(byte[] value, int offset, int length) throws IOException {
+    public XContentBuilder value(Object value) throws IOException {
+        unknownValue(value);
+        return this;
+    }
+
+    private void unknownValue(Object value) throws IOException {
         if (value == null) {
-            return nullValue();
+            nullValue();
+            return;
+        }
+        Writer writer = WRITERS.get(value.getClass());
+        if (writer != null) {
+            writer.write(this, value);
+        } else if (value instanceof Path) {
+            //Path implements Iterable<Path> and causes endless recursion and a StackOverFlow if treated as an Iterable here
+            value((Path) value);
+        } else if (value instanceof Map) {
+            map((Map) value);
+        } else if (value instanceof Iterable) {
+            value((Iterable<?>) value);
+        } else if (value instanceof Object[]) {
+            values((Object[]) value);
+        } else if (value instanceof Calendar) {
+            value((Calendar) value);
+        } else if (value instanceof ReadableInstant) {
+            value((ReadableInstant) value);
+        } else if (value instanceof BytesReference) {
+            value((BytesReference) value);
+        } else if (value instanceof ToXContent) {
+            value((ToXContent) value);
+        } else {
+            // This is a "value" object (like enum, DistanceUnit, etc) just toString() it
+            // (yes, it can be misleading when toString a Java class, but really, jackson should be used in that case)
+            value(Objects.toString(value));
         }
-        generator.writeBinary(value, offset, length);
-        return this;
     }
 
-    /**
-     * Writes the binary content of the given BytesRef
-     * Use {@link org.elasticsearch.common.xcontent.XContentParser#binaryValue()} to read the value back
-     */
-    public XContentBuilder value(BytesRef value) throws IOException {
+    ////////////////////////////////////////////////////////////////////////////
+    // ToXContent
+    //////////////////////////////////
+
+    public XContentBuilder field(String name, ToXContent value) throws IOException {
+        return field(name).value(value);
+    }
+
+    public XContentBuilder field(String name, ToXContent value, ToXContent.Params params) throws IOException {
+        return field(name).value(value, params);
+    }
+
+    private XContentBuilder value(ToXContent value) throws IOException {
+        return value(value, ToXContent.EMPTY_PARAMS);
+    }
+
+    private XContentBuilder value(ToXContent value, ToXContent.Params params) throws IOException {
         if (value == null) {
             return nullValue();
         }
-        generator.writeBinary(value.bytes, value.offset, value.length);
+        value.toXContent(this, params);
         return this;
     }
 
-    /**
-     * Writes the binary content of the given BytesReference
-     * Use {@link org.elasticsearch.common.xcontent.XContentParser#binaryValue()} to read the value back
-     */
-    public XContentBuilder value(BytesReference value) throws IOException {
-        if (value == null) {
+    ////////////////////////////////////////////////////////////////////////////
+    // Maps & Iterable
+    //////////////////////////////////
+
+    public XContentBuilder field(String name, Map<String, Object> values) throws IOException {
+        return field(name).map(values);
+    }
+
+    public XContentBuilder map(Map<String, ?> values) throws IOException {
+        if (values == null) {
             return nullValue();
         }
-        BytesRef ref = value.toBytesRef();
-        generator.writeBinary(ref.bytes, ref.offset, ref.length);
+
+        // checks that the map does not contain references to itself because
+        // iterating over map entries will cause a stackoverflow error
+        ensureNoSelfReferences(values);
+
+        startObject();
+        for (Map.Entry<String, ?> value : values.entrySet()) {
+            field(value.getKey());
+            unknownValue(value.getValue());
+        }
+        endObject();
         return this;
     }
 
-    public XContentBuilder value(Text value) throws IOException {
-        if (value == null) {
+    public XContentBuilder field(String name, Iterable<?> values) throws IOException {
+        return field(name).value(values);
+    }
+
+    private XContentBuilder value(Iterable<?> values) throws IOException {
+        if (values == null) {
             return nullValue();
-        } else if (value.hasString()) {
-            generator.writeString(value.string());
+        }
+
+        if (values instanceof Path) {
+            //treat as single value
+            value((Path) values);
         } else {
-            BytesRef bytesRef = value.bytes().toBytesRef();
-            generator.writeUTF8String(bytesRef.bytes, bytesRef.offset, bytesRef.length);
+            // checks that the iterable does not contain references to itself because
+            // iterating over entries will cause a stackoverflow error
+            ensureNoSelfReferences(values);
+
+            startArray();
+            for (Object value : values) {
+                unknownValue(value);
+            }
+            endArray();
         }
         return this;
     }
 
-    public XContentBuilder map(Map<String, ?> map) throws IOException {
-        if (map == null) {
-            return nullValue();
+    ////////////////////////////////////////////////////////////////////////////
+    // Misc.
+    //////////////////////////////////
+
+    public XContentBuilder timeValueField(String rawFieldName, String readableFieldName, TimeValue timeValue) throws IOException {
+        if (humanReadable) {
+            field(readableFieldName, timeValue.toString());
         }
-        writeMap(map);
+        field(rawFieldName, timeValue.millis());
         return this;
     }
 
-    public XContentBuilder value(Map<String, Object> map) throws IOException {
-        if (map == null) {
-            return nullValue();
+    public XContentBuilder timeValueField(String rawFieldName, String readableFieldName, long rawTime) throws IOException {
+        if (humanReadable) {
+            field(readableFieldName, new TimeValue(rawTime).toString());
         }
-        writeMap(map);
+        field(rawFieldName, rawTime);
         return this;
     }
 
-    public XContentBuilder value(Iterable<?> value) throws IOException {
-        if (value == null) {
-            return nullValue();
+    public XContentBuilder timeValueField(String rawFieldName, String readableFieldName, long rawTime, TimeUnit timeUnit) throws
+            IOException {
+        if (humanReadable) {
+            field(readableFieldName, new TimeValue(rawTime, timeUnit).toString());
         }
-        if (value instanceof Path) {
-            //treat as single value
-            writeValue(value);
-        } else {
-            startArray();
-            for (Object o : value) {
-                value(o);
-            }
-            endArray();
+        field(rawFieldName, rawTime);
+        return this;
+    }
+
+
+    public XContentBuilder percentageField(String rawFieldName, String readableFieldName, double percentage) throws IOException {
+        if (humanReadable) {
+            field(readableFieldName, String.format(Locale.ROOT, "%1.1f%%", percentage));
         }
+        field(rawFieldName, percentage);
         return this;
     }
 
-    public XContentBuilder latlon(String name, double lat, double lon) throws IOException {
-        return startObject(name).field("lat", lat).field("lon", lon).endObject();
+    public XContentBuilder byteSizeField(String rawFieldName, String readableFieldName, ByteSizeValue byteSizeValue) throws IOException {
+        if (humanReadable) {
+            field(readableFieldName, byteSizeValue.toString());
+        }
+        field(rawFieldName, byteSizeValue.getBytes());
+        return this;
     }
 
-    public XContentBuilder latlon(double lat, double lon) throws IOException {
-        return startObject().field("lat", lat).field("lon", lon).endObject();
+    public XContentBuilder byteSizeField(String rawFieldName, String readableFieldName, long rawSize) throws IOException {
+        if (humanReadable) {
+            field(readableFieldName, new ByteSizeValue(rawSize).toString());
+        }
+        field(rawFieldName, rawSize);
+        return this;
+    }
+
+    ////////////////////////////////////////////////////////////////////////////
+    // Raw fields
+    //////////////////////////////////
+
+    /**
+     * Writes a raw field with the value taken from the bytes in the stream
+     * @deprecated use {@link #rawField(String, InputStream, XContentType)} to avoid content type auto-detection
+     */
+    @Deprecated
+    public XContentBuilder rawField(String name, InputStream value) throws IOException {
+        generator.writeRawField(name, value);
+        return this;
+    }
+
+    /**
+     * Writes a raw field with the value taken from the bytes in the stream
+     */
+    public XContentBuilder rawField(String name, InputStream value, XContentType contentType) throws IOException {
+        generator.writeRawField(name, value, contentType);
+        return this;
+    }
+
+    /**
+     * Writes a raw field with the given bytes as the value
+     * @deprecated use {@link #rawField(String name, BytesReference, XContentType)} to avoid content type auto-detection
+     */
+    @Deprecated
+    public XContentBuilder rawField(String name, BytesReference value) throws IOException {
+        generator.writeRawField(name, value);
+        return this;
+    }
+
+    /**
+     * Writes a raw field with the given bytes as the value
+     */
+    public XContentBuilder rawField(String name, BytesReference value, XContentType contentType) throws IOException {
+        generator.writeRawField(name, value, contentType);
+        return this;
+    }
+
+    /**
+     * Writes a value with the source coming directly from the bytes
+     * @deprecated use {@link #rawValue(BytesReference, XContentType)} to avoid content type auto-detection
+     */
+    @Deprecated
+    public XContentBuilder rawValue(BytesReference value) throws IOException {
+        generator.writeRawValue(value);
+        return this;
+    }
+
+    /**
+     * Writes a value with the source coming directly from the bytes
+     */
+    public XContentBuilder rawValue(BytesReference value, XContentType contentType) throws IOException {
+        generator.writeRawValue(value, contentType);
+        return this;
     }
 
     public XContentBuilder copyCurrentStructure(XContentParser parser) throws IOException {
@@ -758,9 +1023,9 @@ public XContentBuilder copyCurrentStructure(XContentParser parser) throws IOExce
         return this;
     }
 
-    public XContentBuilder flush() throws IOException {
+    @Override
+    public void flush() throws IOException {
         generator.flush();
-        return this;
     }
 
     @Override
@@ -768,7 +1033,7 @@ public void close() {
         try {
             generator.close();
         } catch (IOException e) {
-            throw new IllegalStateException("failed to close the XContentBuilder", e);
+            throw new IllegalStateException("Failed to close the XContentBuilder", e);
         }
     }
 
@@ -776,7 +1041,6 @@ public XContentGenerator generator() {
         return this.generator;
     }
 
-    @Override
     public BytesReference bytes() {
         close();
         return ((BytesStream) bos).bytes();
@@ -786,156 +1050,48 @@ public BytesReference bytes() {
      * Returns a string representation of the builder (only applicable for text based xcontent).
      */
     public String string() throws IOException {
-        close();
         return bytes().utf8ToString();
     }
 
-
-    private void writeMap(Map<String, ?> map) throws IOException {
-        generator.writeStartObject();
-
-        for (Map.Entry<String, ?> entry : map.entrySet()) {
-            field(entry.getKey());
-            Object value = entry.getValue();
-            if (value == null) {
-                generator.writeNull();
-            } else {
-                writeValue(value);
-            }
-        }
-        generator.writeEndObject();
-    }
-
-    @FunctionalInterface
-    interface Writer {
-        void write(XContentGenerator g, Object v) throws IOException;
+    static void ensureNameNotNull(String name) {
+        ensureNotNull(name, "Field name cannot be null");
     }
 
-    private static final Map<Class<?>, Writer> MAP;
-
-    static {
-        Map<Class<?>, Writer> map = new HashMap<>();
-        map.put(String.class, (g, v) -> g.writeString((String) v));
-        map.put(Integer.class, (g, v) -> g.writeNumber((Integer) v));
-        map.put(Long.class, (g, v) -> g.writeNumber((Long) v));
-        map.put(Float.class, (g, v) -> g.writeNumber((Float) v));
-        map.put(Double.class, (g, v) -> g.writeNumber((Double) v));
-        map.put(Byte.class, (g, v) -> g.writeNumber((Byte) v));
-        map.put(Short.class, (g, v) -> g.writeNumber((Short) v));
-        map.put(Boolean.class, (g, v) -> g.writeBoolean((Boolean) v));
-        map.put(GeoPoint.class, (g, v) -> {
-            g.writeStartObject();
-            g.writeNumberField("lat", ((GeoPoint) v).lat());
-            g.writeNumberField("lon", ((GeoPoint) v).lon());
-            g.writeEndObject();
-        });
-        map.put(int[].class, (g, v) -> {
-            g.writeStartArray();
-            for (int item : (int[]) v) {
-                g.writeNumber(item);
-            }
-            g.writeEndArray();
-        });
-        map.put(long[].class, (g, v) -> {
-            g.writeStartArray();
-            for (long item : (long[]) v) {
-                g.writeNumber(item);
-            }
-            g.writeEndArray();
-        });
-        map.put(float[].class, (g, v) -> {
-            g.writeStartArray();
-            for (float item : (float[]) v) {
-                g.writeNumber(item);
-            }
-            g.writeEndArray();
-        });
-        map.put(double[].class, (g, v) -> {
-            g.writeStartArray();
-            for (double item : (double[])v) {
-                g.writeNumber(item);
-            }
-            g.writeEndArray();
-        });
-        map.put(byte[].class, (g, v) -> g.writeBinary((byte[]) v));
-        map.put(short[].class, (g, v) -> {
-            g.writeStartArray();
-            for (short item : (short[])v) {
-                g.writeNumber(item);
-            }
-            g.writeEndArray();
-        });
-        map.put(BytesRef.class, (g, v) -> {
-            BytesRef bytes = (BytesRef) v;
-            g.writeBinary(bytes.bytes, bytes.offset, bytes.length);
-        });
-        map.put(Text.class, (g, v) -> {
-            Text text = (Text) v;
-            if (text.hasString()) {
-                    g.writeString(text.string());
-            } else {
-                BytesRef ref = text.bytes().toBytesRef();
-                g.writeUTF8String(ref.bytes, ref.offset, ref.length);
-            }
-        });
-        MAP = Collections.unmodifiableMap(map);
+    static void ensureFormatterNotNull(DateTimeFormatter formatter) {
+        ensureNotNull(formatter, "DateTimeFormatter cannot be null");
     }
 
-    private void writeValue(Object value) throws IOException {
+    static void ensureNotNull(Object value, String message) {
         if (value == null) {
-            generator.writeNull();
-            return;
-        }
-        Class<?> type = value.getClass();
-        Writer writer = MAP.get(type);
-        if (writer != null) {
-            writer.write(generator, value);
-        } else if (value instanceof Map) {
-            writeMap((Map) value);
-        } else if (value instanceof Path) {
-            //Path implements Iterable<Path> and causes endless recursion and a StackOverFlow if treated as an Iterable here
-            generator.writeString(value.toString());
-        } else if (value instanceof Iterable) {
-            writeIterable((Iterable<?>) value);
-        } else if (value instanceof Object[]) {
-            writeObjectArray((Object[]) value);
-        } else if (value instanceof Date) {
-            generator.writeString(XContentBuilder.defaultDatePrinter.print(((Date) value).getTime()));
-        } else if (value instanceof Calendar) {
-            generator.writeString(XContentBuilder.defaultDatePrinter.print((((Calendar) value)).getTimeInMillis()));
-        } else if (value instanceof ReadableInstant) {
-            generator.writeString(XContentBuilder.defaultDatePrinter.print((((ReadableInstant) value)).getMillis()));
-        } else if (value instanceof BytesReference) {
-            writeBytesReference((BytesReference) value);
-        } else if (value instanceof ToXContent) {
-            ((ToXContent) value).toXContent(this, ToXContent.EMPTY_PARAMS);
-        } else {
-            // if this is a "value" object, like enum, DistanceUnit, ..., just toString it
-            // yea, it can be misleading when toString a Java class, but really, jackson should be used in that case
-            generator.writeString(value.toString());
-            //throw new ElasticsearchIllegalArgumentException("type not supported for generic value conversion: " + type);
+            throw new IllegalArgumentException(message);
         }
     }
 
-    private void writeBytesReference(BytesReference value) throws IOException {
-        BytesRef ref = value.toBytesRef();
-        generator.writeBinary(ref.bytes, ref.offset, ref.length);
+    static void ensureNoSelfReferences(Object value) {
+        ensureNoSelfReferences(value, Collections.newSetFromMap(new IdentityHashMap<>()));
     }
 
-    private void writeIterable(Iterable<?> value) throws IOException {
-        generator.writeStartArray();
-        for (Object v : value) {
-            writeValue(v);
-        }
-        generator.writeEndArray();
-    }
+    private static void ensureNoSelfReferences(final Object value, final Set<Object> ancestors) {
+        if (value != null) {
 
-    private void writeObjectArray(Object[] value) throws IOException {
-        generator.writeStartArray();
-        for (Object v : value) {
-            writeValue(v);
+            Iterable<?> it;
+            if (value instanceof Map) {
+                it = ((Map) value).values();
+            } else if ((value instanceof Iterable) && (value instanceof Path == false)) {
+                it = (Iterable) value;
+            } else if (value instanceof Object[]) {
+                it = Arrays.asList((Object[]) value);
+            } else {
+                return;
+            }
+
+            if (ancestors.add(value) == false) {
+                throw new IllegalArgumentException("Object has already been built and is self-referencing itself");
+            }
+            for (Object o : it) {
+                ensureNoSelfReferences(o, ancestors);
+            }
+            ancestors.remove(value);
         }
-        generator.writeEndArray();
     }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentFactory.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentFactory.java
index 47dd1249d4adb..a5350e3c66212 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentFactory.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentFactory.java
@@ -39,7 +39,7 @@
  */
 public class XContentFactory {
 
-    private static int GUESS_HEADER_LENGTH = 20;
+    private static final int GUESS_HEADER_LENGTH = 20;
 
     /**
      * Returns a content builder using JSON format ({@link org.elasticsearch.common.xcontent.XContentType#JSON}.
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java
index a2cceae836724..60a188ca6ce58 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java
@@ -20,14 +20,13 @@
 package org.elasticsearch.common.xcontent;
 
 import org.elasticsearch.common.bytes.BytesReference;
+
 import java.io.Closeable;
+import java.io.Flushable;
 import java.io.IOException;
 import java.io.InputStream;
 
-/**
- *
- */
-public interface XContentGenerator extends Closeable {
+public interface XContentGenerator extends Closeable, Flushable {
 
     XContentType contentType();
 
@@ -37,68 +36,97 @@ public interface XContentGenerator extends Closeable {
 
     void usePrintLineFeedAtEnd();
 
+    void writeStartObject() throws IOException;
+
+    void writeEndObject() throws IOException;
+
     void writeStartArray() throws IOException;
 
     void writeEndArray() throws IOException;
 
-    void writeStartObject() throws IOException;
+    void writeFieldName(String name) throws IOException;
 
-    void writeEndObject() throws IOException;
+    void writeNull() throws IOException;
 
-    void writeFieldName(String name) throws IOException;
+    void writeNullField(String name) throws IOException;
 
-    void writeString(String text) throws IOException;
+    void writeBooleanField(String name, boolean value) throws IOException;
 
-    void writeString(char[] text, int offset, int len) throws IOException;
+    void writeBoolean(boolean value) throws IOException;
 
-    void writeUTF8String(byte[] text, int offset, int length) throws IOException;
+    void writeNumberField(String name, double value) throws IOException;
 
-    void writeBinary(byte[] data, int offset, int len) throws IOException;
+    void writeNumber(double value) throws IOException;
 
-    void writeBinary(byte[] data) throws IOException;
+    void writeNumberField(String name, float value) throws IOException;
 
-    void writeNumber(int v) throws IOException;
+    void writeNumber(float value) throws IOException;
 
-    void writeNumber(long v) throws IOException;
+    void writeNumberField(String name, int value) throws IOException;
 
-    void writeNumber(double d) throws IOException;
+    void writeNumber(int value) throws IOException;
 
-    void writeNumber(float f) throws IOException;
+    void writeNumberField(String name, long value) throws IOException;
 
-    void writeBoolean(boolean state) throws IOException;
+    void writeNumber(long value) throws IOException;
 
-    void writeNull() throws IOException;
+    void writeNumber(short value) throws IOException;
 
-    void writeStringField(String fieldName, String value) throws IOException;
+    void writeStringField(String name, String value) throws IOException;
 
-    void writeBooleanField(String fieldName, boolean value) throws IOException;
+    void writeString(String value) throws IOException;
 
-    void writeNullField(String fieldName) throws IOException;
+    void writeString(char[] text, int offset, int len) throws IOException;
 
-    void writeNumberField(String fieldName, int value) throws IOException;
+    void writeUTF8String(byte[] value, int offset, int length) throws IOException;
 
-    void writeNumberField(String fieldName, long value) throws IOException;
+    void writeBinaryField(String name, byte[] value) throws IOException;
 
-    void writeNumberField(String fieldName, double value) throws IOException;
+    void writeBinary(byte[] value) throws IOException;
 
-    void writeNumberField(String fieldName, float value) throws IOException;
+    void writeBinary(byte[] value, int offset, int length) throws IOException;
 
-    void writeBinaryField(String fieldName, byte[] data) throws IOException;
+    /**
+     * Writes a raw field with the value taken from the bytes in the stream
+     * @deprecated use {@link #writeRawField(String, InputStream, XContentType)} to avoid content type auto-detection
+     */
+    @Deprecated
+    void writeRawField(String name, InputStream value) throws IOException;
 
-    void writeArrayFieldStart(String fieldName) throws IOException;
+    /**
+     * Writes a raw field with the value taken from the bytes in the stream
+     */
+    void writeRawField(String name, InputStream value, XContentType xContentType) throws IOException;
 
-    void writeObjectFieldStart(String fieldName) throws IOException;
+    /**
+     * Writes a raw field with the given bytes as the value
+     * @deprecated use {@link #writeRawField(String, BytesReference, XContentType)} to avoid content type auto-detection
+     */
+    @Deprecated
+    void writeRawField(String name, BytesReference value) throws IOException;
 
-    void writeRawField(String fieldName, InputStream content) throws IOException;
+    /**
+     * Writes a raw field with the given bytes as the value
+     */
+    void writeRawField(String name, BytesReference value, XContentType xContentType) throws IOException;
 
-    void writeRawField(String fieldName, BytesReference content) throws IOException;
+    /**
+     * Writes a value with the source coming directly from the bytes
+     * @deprecated use {@link #writeRawValue(BytesReference, XContentType)} to avoid content type auto-detection
+     */
+    @Deprecated
+    void writeRawValue(BytesReference value) throws IOException;
 
-    void writeRawValue(BytesReference content) throws IOException;
+    /**
+     * Writes a value with the source coming directly from the bytes
+     */
+    void writeRawValue(BytesReference value, XContentType xContentType) throws IOException;
 
     void copyCurrentStructure(XContentParser parser) throws IOException;
 
-    void flush() throws IOException;
+    /**
+     * Returns {@code true} if this XContentGenerator has been closed. A closed generator can not do any more output.
+     */
+    boolean isClosed();
 
-    @Override
-    void close() throws IOException;
 }
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java
index 2832527a5834c..c3c5c8829ad66 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java
@@ -44,23 +44,61 @@
 @SuppressWarnings("unchecked")
 public class XContentHelper {
 
-    public static XContentParser createParser(BytesReference bytes) throws IOException {
+    /**
+     * Creates a parser based on the bytes provided
+     * @deprecated use {@link #createParser(NamedXContentRegistry, BytesReference, XContentType)} to avoid content type auto-detection
+     */
+    @Deprecated
+    public static XContentParser createParser(NamedXContentRegistry xContentRegistry, BytesReference bytes) throws IOException {
+        Compressor compressor = CompressorFactory.compressor(bytes);
+        if (compressor != null) {
+            InputStream compressedInput = compressor.streamInput(bytes.streamInput());
+            if (compressedInput.markSupported() == false) {
+                compressedInput = new BufferedInputStream(compressedInput);
+            }
+            final XContentType contentType = XContentFactory.xContentType(compressedInput);
+            return XContentFactory.xContent(contentType).createParser(xContentRegistry, compressedInput);
+        } else {
+            return XContentFactory.xContent(bytes).createParser(xContentRegistry, bytes.streamInput());
+        }
+    }
+
+    /**
+     * Creates a parser for the bytes using the supplied content-type
+     */
+    public static XContentParser createParser(NamedXContentRegistry xContentRegistry, BytesReference bytes,
+                                              XContentType xContentType) throws IOException {
+        Objects.requireNonNull(xContentType);
         Compressor compressor = CompressorFactory.compressor(bytes);
         if (compressor != null) {
             InputStream compressedInput = compressor.streamInput(bytes.streamInput());
             if (compressedInput.markSupported() == false) {
                 compressedInput = new BufferedInputStream(compressedInput);
             }
-            XContentType contentType = XContentFactory.xContentType(compressedInput);
-            return XContentFactory.xContent(contentType).createParser(compressedInput);
+            return XContentFactory.xContent(xContentType).createParser(xContentRegistry, compressedInput);
         } else {
-            return XContentFactory.xContent(bytes).createParser(bytes.streamInput());
+            return xContentType.xContent().createParser(xContentRegistry, bytes.streamInput());
         }
     }
 
-    public static Tuple<XContentType, Map<String, Object>> convertToMap(BytesReference bytes, boolean ordered) throws ElasticsearchParseException {
+    /**
+     * Converts the given bytes into a map that is optionally ordered.
+     * @deprecated this method relies on auto-detection of content type. Use {@link #convertToMap(BytesReference, boolean, XContentType)}
+     *             instead with the proper {@link XContentType}
+     */
+    @Deprecated
+    public static Tuple<XContentType, Map<String, Object>> convertToMap(BytesReference bytes, boolean ordered)
+            throws ElasticsearchParseException {
+        return convertToMap(bytes, ordered, null);
+    }
+
+    /**
+     * Converts the given bytes into a map that is optionally ordered. The provided {@link XContentType} must be non-null.
+     */
+    public static Tuple<XContentType, Map<String, Object>> convertToMap(BytesReference bytes, boolean ordered, XContentType xContentType)
+        throws ElasticsearchParseException {
         try {
-            XContentType contentType;
+            final XContentType contentType;
             InputStream input;
             Compressor compressor = CompressorFactory.compressor(bytes);
             if (compressor != null) {
@@ -68,34 +106,68 @@ public static Tuple<XContentType, Map<String, Object>> convertToMap(BytesReferen
                 if (compressedStreamInput.markSupported() == false) {
                     compressedStreamInput = new BufferedInputStream(compressedStreamInput);
                 }
-                contentType = XContentFactory.xContentType(compressedStreamInput);
                 input = compressedStreamInput;
             } else {
-                contentType = XContentFactory.xContentType(bytes);
                 input = bytes.streamInput();
             }
-            try (XContentParser parser = XContentFactory.xContent(contentType).createParser(input)) {
-                if (ordered) {
-                    return Tuple.tuple(contentType, parser.mapOrdered());
-                } else {
-                    return Tuple.tuple(contentType, parser.map());
-                }
-            }
+            contentType = xContentType != null ? xContentType : XContentFactory.xContentType(input);
+            return new Tuple<>(Objects.requireNonNull(contentType), convertToMap(XContentFactory.xContent(contentType), input, ordered));
+        } catch (IOException e) {
+            throw new ElasticsearchParseException("Failed to parse content to map", e);
+        }
+    }
+
+    /**
+     * Convert a string in some {@link XContent} format to a {@link Map}. Throws an {@link ElasticsearchParseException} if there is any
+     * error.
+     */
+    public static Map<String, Object> convertToMap(XContent xContent, String string, boolean ordered) throws ElasticsearchParseException {
+        // It is safe to use EMPTY here because this never uses namedObject
+        try (XContentParser parser = xContent.createParser(NamedXContentRegistry.EMPTY, string)) {
+            return ordered ? parser.mapOrdered() : parser.map();
+        } catch (IOException e) {
+            throw new ElasticsearchParseException("Failed to parse content to map", e);
+        }
+    }
+
+    /**
+     * Convert a string in some {@link XContent} format to a {@link Map}. Throws an {@link ElasticsearchParseException} if there is any
+     * error. Note that unlike {@link #convertToMap(BytesReference, boolean)}, this doesn't automatically uncompress the input.
+     */
+    public static Map<String, Object> convertToMap(XContent xContent, InputStream input, boolean ordered)
+            throws ElasticsearchParseException {
+        // It is safe to use EMPTY here because this never uses namedObject
+        try (XContentParser parser = xContent.createParser(NamedXContentRegistry.EMPTY, input)) {
+            return ordered ? parser.mapOrdered() : parser.map();
         } catch (IOException e) {
             throw new ElasticsearchParseException("Failed to parse content to map", e);
         }
     }
 
+    @Deprecated
     public static String convertToJson(BytesReference bytes, boolean reformatJson) throws IOException {
         return convertToJson(bytes, reformatJson, false);
     }
 
+    @Deprecated
     public static String convertToJson(BytesReference bytes, boolean reformatJson, boolean prettyPrint) throws IOException {
-        XContentType xContentType = XContentFactory.xContentType(bytes);
+        return convertToJson(bytes, reformatJson, prettyPrint, XContentFactory.xContentType(bytes));
+    }
+
+    public static String convertToJson(BytesReference bytes, boolean reformatJson, XContentType xContentType) throws IOException {
+        return convertToJson(bytes, reformatJson, false, xContentType);
+    }
+
+    public static String convertToJson(BytesReference bytes, boolean reformatJson, boolean prettyPrint, XContentType xContentType)
+        throws IOException {
+        Objects.requireNonNull(xContentType);
         if (xContentType == XContentType.JSON && !reformatJson) {
             return bytes.utf8ToString();
         }
-        try (XContentParser parser = XContentFactory.xContent(xContentType).createParser(bytes.streamInput())) {
+
+        // It is safe to use EMPTY here because this never uses namedObject
+        try (XContentParser parser = XContentFactory.xContent(xContentType).createParser(NamedXContentRegistry.EMPTY,
+                bytes.streamInput())) {
             parser.nextToken();
             XContentBuilder builder = XContentFactory.jsonBuilder();
             if (prettyPrint) {
@@ -194,7 +266,6 @@ public static boolean update(Map<String, Object> source, Map<String, Object> cha
      * Merges the defaults provided as the second parameter into the content of the first. Only does recursive merge
      * for inner maps.
      */
-    @SuppressWarnings({"unchecked"})
     public static void mergeDefaults(Map<String, Object> content, Map<String, Object> defaults) {
         for (Map.Entry<String, Object> defaultEntry : defaults.entrySet()) {
             if (!content.containsKey(defaultEntry.getKey())) {
@@ -258,33 +329,36 @@ private static boolean allListValuesAreMapsOfOne(List list) {
         return true;
     }
 
-    public static void copyCurrentStructure(XContentGenerator generator, XContentParser parser) throws IOException {
+    /**
+     * Low level implementation detail of {@link XContentGenerator#copyCurrentStructure(XContentParser)}.
+     */
+    public static void copyCurrentStructure(XContentGenerator destination, XContentParser parser) throws IOException {
         XContentParser.Token token = parser.currentToken();
 
         // Let's handle field-name separately first
         if (token == XContentParser.Token.FIELD_NAME) {
-            generator.writeFieldName(parser.currentName());
+            destination.writeFieldName(parser.currentName());
             token = parser.nextToken();
             // fall-through to copy the associated value
         }
 
         switch (token) {
             case START_ARRAY:
-                generator.writeStartArray();
+                destination.writeStartArray();
                 while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
-                    copyCurrentStructure(generator, parser);
+                    copyCurrentStructure(destination, parser);
                 }
-                generator.writeEndArray();
+                destination.writeEndArray();
                 break;
             case START_OBJECT:
-                generator.writeStartObject();
+                destination.writeStartObject();
                 while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
-                    copyCurrentStructure(generator, parser);
+                    copyCurrentStructure(destination, parser);
                 }
-                generator.writeEndObject();
+                destination.writeEndObject();
                 break;
             default: // others are simple:
-                copyCurrentEvent(generator, parser);
+                copyCurrentEvent(destination, parser);
         }
     }
 
@@ -342,7 +416,10 @@ public static void copyCurrentEvent(XContentGenerator generator, XContentParser
     /**
      * Writes a "raw" (bytes) field, handling cases where the bytes are compressed, and tries to optimize writing using
      * {@link XContentBuilder#rawField(String, org.elasticsearch.common.bytes.BytesReference)}.
+     * @deprecated use {@link #writeRawField(String, BytesReference, XContentType, XContentBuilder, Params)} to avoid content type
+     * auto-detection
      */
+    @Deprecated
     public static void writeRawField(String field, BytesReference source, XContentBuilder builder, ToXContent.Params params) throws IOException {
         Compressor compressor = CompressorFactory.compressor(source);
         if (compressor != null) {
@@ -352,4 +429,48 @@ public static void writeRawField(String field, BytesReference source, XContentBu
             builder.rawField(field, source);
         }
     }
+
+    /**
+     * Writes a "raw" (bytes) field, handling cases where the bytes are compressed, and tries to optimize writing using
+     * {@link XContentBuilder#rawField(String, org.elasticsearch.common.bytes.BytesReference, XContentType)}.
+     */
+    public static void writeRawField(String field, BytesReference source, XContentType xContentType, XContentBuilder builder,
+                                     ToXContent.Params params) throws IOException {
+        Objects.requireNonNull(xContentType);
+        Compressor compressor = CompressorFactory.compressor(source);
+        if (compressor != null) {
+            InputStream compressedStreamInput = compressor.streamInput(source.streamInput());
+            builder.rawField(field, compressedStreamInput, xContentType);
+        } else {
+            builder.rawField(field, source, xContentType);
+        }
+    }
+
+    /**
+     * Returns the bytes that represent the XContent output of the provided {@link ToXContent} object, using the provided
+     * {@link XContentType}. Wraps the output into a new anonymous object according to the value returned
+     * by the {@link ToXContent#isFragment()} method returns.
+     */
+    public static BytesReference toXContent(ToXContent toXContent, XContentType xContentType, boolean humanReadable) throws IOException {
+        return toXContent(toXContent, xContentType, ToXContent.EMPTY_PARAMS, humanReadable);
+    }
+
+    /**
+     * Returns the bytes that represent the XContent output of the provided {@link ToXContent} object, using the provided
+     * {@link XContentType}. Wraps the output into a new anonymous object according to the value returned
+     * by the {@link ToXContent#isFragment()} method returns.
+     */
+    public static BytesReference toXContent(ToXContent toXContent, XContentType xContentType, Params params, boolean humanReadable) throws IOException {
+        try (XContentBuilder builder = XContentBuilder.builder(xContentType.xContent())) {
+            builder.humanReadable(humanReadable);
+            if (toXContent.isFragment()) {
+                builder.startObject();
+            }
+            toXContent.toXContent(builder, params);
+            if (toXContent.isFragment()) {
+                builder.endObject();
+            }
+            return builder.bytes();
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java
index f8513828636d1..4947e1d5c775c 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java
@@ -33,7 +33,7 @@
  *
  * <pre>
  *     XContentType xContentType = XContentType.JSON;
- *     XContentParser parser = xContentType.xContent().createParser("{\"key\" : \"value\"}");
+ *     XContentParser parser = xContentType.xContent().createParser(NamedXContentRegistry.EMPTY, "{\"key\" : \"value\"}");
  * </pre>
  */
 public interface XContentParser extends Releasable {
@@ -131,6 +131,10 @@ enum NumberType {
 
     Map<String, Object> mapOrdered() throws IOException;
 
+    Map<String, String> mapStrings() throws IOException;
+
+    Map<String, String> mapStringsOrdered() throws IOException;
+
     List<Object> list() throws IOException;
 
     List<Object> listOrderedMap() throws IOException;
@@ -245,5 +249,16 @@ enum NumberType {
      */
     XContentLocation getTokenLocation();
 
+    // TODO remove context entirely when it isn't needed
+    /**
+     * Parse an object by name.
+     */
+    <T> T namedObject(Class<T> categoryClass, String name, Object context) throws IOException;
+
+    /**
+     * The registry used to resolve {@link #namedObject(Class, String, Object)}. Use this when building a sub-parser from this parser.
+     */
+    NamedXContentRegistry getXContentRegistry();
+
     boolean isClosed();
 }
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentParserUtils.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentParserUtils.java
new file mode 100644
index 0000000000000..e28b44b42c5a2
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentParserUtils.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.xcontent;
+
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.xcontent.XContentParser.Token;
+
+import java.io.IOException;
+import java.util.Locale;
+import java.util.function.Consumer;
+import java.util.function.Supplier;
+
+/**
+ * A set of static methods to get {@link Token} from {@link XContentParser}
+ * while checking for their types and throw {@link ParsingException} if needed.
+ */
+public final class XContentParserUtils {
+
+    private XContentParserUtils() {
+    }
+
+    /**
+     * Makes sure that current token is of type {@link XContentParser.Token#FIELD_NAME} and the the field name is equal to the provided one
+     * @throws ParsingException if the token is not of type {@link XContentParser.Token#FIELD_NAME} or is not equal to the given field name
+     */
+    public static void ensureFieldName(XContentParser parser, Token token, String fieldName) throws IOException {
+        ensureExpectedToken(Token.FIELD_NAME, token, parser::getTokenLocation);
+        String currentName = parser.currentName();
+        if (currentName.equals(fieldName) == false) {
+            String message = "Failed to parse object: expecting field with name [%s] but found [%s]";
+            throw new ParsingException(parser.getTokenLocation(), String.format(Locale.ROOT, message, fieldName, currentName));
+        }
+    }
+
+    /**
+     * @throws ParsingException with a "unknown field found" reason
+     */
+    public static void throwUnknownField(String field, XContentLocation location) {
+        String message = "Failed to parse object: unknown field [%s] found";
+        throw new ParsingException(location, String.format(Locale.ROOT, message, field));
+    }
+
+    /**
+     * @throws ParsingException with a "unknown token found" reason
+     */
+    public static void throwUnknownToken(XContentParser.Token token, XContentLocation location) {
+        String message = "Failed to parse object: unexpected token [%s] found";
+        throw new ParsingException(location, String.format(Locale.ROOT, message, token));
+    }
+
+    /**
+     * Makes sure that provided token is of the expected type
+     *
+     * @throws ParsingException if the token is not equal to the expected type
+     */
+    public static void ensureExpectedToken(Token expected, Token actual, Supplier<XContentLocation> location) {
+        if (actual != expected) {
+            String message = "Failed to parse object: expecting token of type [%s] but found [%s]";
+            throw new ParsingException(location.get(), String.format(Locale.ROOT, message, expected, actual));
+        }
+    }
+
+    /**
+     * Parse the current token depending on its token type. The following token types will be
+     * parsed by the corresponding parser methods:
+     * <ul>
+     *    <li>XContentParser.Token.VALUE_STRING: parser.text()</li>
+     *    <li>XContentParser.Token.VALUE_NUMBER: parser.numberValue()</li>
+     *    <li>XContentParser.Token.VALUE_BOOLEAN: parser.booleanValue()</li>
+     *    <li>XContentParser.Token.VALUE_EMBEDDED_OBJECT: parser.binaryValue()</li>
+     * </ul>
+     *
+     * @throws ParsingException if the token none of the allowed values
+     */
+    public static Object parseStoredFieldsValue(XContentParser parser) throws IOException {
+        XContentParser.Token token = parser.currentToken();
+        Object value = null;
+        if (token == XContentParser.Token.VALUE_STRING) {
+            //binary values will be parsed back and returned as base64 strings when reading from json and yaml
+            value = parser.text();
+        } else if (token == XContentParser.Token.VALUE_NUMBER) {
+            value = parser.numberValue();
+        } else if (token == XContentParser.Token.VALUE_BOOLEAN) {
+            value = parser.booleanValue();
+        } else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
+            //binary values will be parsed back and returned as BytesArray when reading from cbor and smile
+            value = new BytesArray(parser.binaryValue());
+        } else {
+            throwUnknownToken(token, parser.getTokenLocation());
+        }
+        return value;
+    }
+
+    /**
+     * This method expects that the current field name is the concatenation of a type, a delimiter and a name
+     * (ex: terms#foo where "terms" refers to the type of a registered {@link NamedXContentRegistry.Entry},
+     * "#" is the delimiter and "foo" the name of the object to parse).
+     *
+     * It also expected that following this field name is either an Object or an array xContent structure and
+     * the cursor points to the start token of this structure.
+     *
+     * The method splits the field's name to extract the type and name and then parses the object
+     * using the {@link XContentParser#namedObject(Class, String, Object)} method.
+     *
+     * @param parser      the current {@link XContentParser}
+     * @param delimiter   the delimiter to use to splits the field's name
+     * @param objectClass the object class of the object to parse
+     * @param consumer    something to consume the parsed object
+     * @param <T>         the type of the object to parse
+     * @throws IOException if anything went wrong during parsing or if the type or name cannot be derived
+     *                     from the field's name
+     * @throws ParsingException if the parser isn't positioned on either START_OBJECT or START_ARRAY at the beginning
+     */
+    public static <T> void parseTypedKeysObject(XContentParser parser, String delimiter, Class<T> objectClass, Consumer<T> consumer)
+            throws IOException {
+        if (parser.currentToken() != XContentParser.Token.START_OBJECT && parser.currentToken() != XContentParser.Token.START_ARRAY) {
+            throwUnknownToken(parser.currentToken(), parser.getTokenLocation());
+        }
+        String currentFieldName = parser.currentName();
+        if (Strings.hasLength(currentFieldName)) {
+            int position = currentFieldName.indexOf(delimiter);
+            if (position > 0) {
+                String type = currentFieldName.substring(0, position);
+                String name = currentFieldName.substring(position + 1);
+                consumer.accept(parser.namedObject(objectClass, type, name));
+                return;
+            }
+            // if we didn't find a delimiter we ignore the object or array for forward compatibility instead of throwing an error
+            parser.skipChildren();
+        } else {
+            throw new ParsingException(parser.getTokenLocation(), "Failed to parse object: empty key");
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentType.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentType.java
index 296f9d2aedde7..40caa6e911087 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentType.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentType.java
@@ -21,6 +21,7 @@
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.cbor.CborXContent;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.common.xcontent.smile.SmileXContent;
@@ -28,18 +29,19 @@
 
 import java.io.IOException;
 import java.util.Locale;
+import java.util.Objects;
 
 /**
  * The content type of {@link org.elasticsearch.common.xcontent.XContent}.
  */
-public enum XContentType {
+public enum XContentType implements Writeable {
 
     /**
      * A JSON based content type.
      */
     JSON(0) {
         @Override
-        protected String mediaTypeWithoutParameters() {
+        public String mediaTypeWithoutParameters() {
             return "application/json";
         }
 
@@ -63,7 +65,7 @@ public XContent xContent() {
      */
     SMILE(1) {
         @Override
-        protected String mediaTypeWithoutParameters() {
+        public String mediaTypeWithoutParameters() {
             return "application/smile";
         }
 
@@ -82,7 +84,7 @@ public XContent xContent() {
      */
     YAML(2) {
         @Override
-        protected String mediaTypeWithoutParameters() {
+        public String mediaTypeWithoutParameters() {
             return "application/yaml";
         }
 
@@ -101,7 +103,7 @@ public XContent xContent() {
      */
     CBOR(3) {
         @Override
-        protected String mediaTypeWithoutParameters() {
+        public String mediaTypeWithoutParameters() {
             return "application/cbor";
         }
 
@@ -116,23 +118,46 @@ public XContent xContent() {
         }
     };
 
+    /**
+     * Accepts either a format string, which is equivalent to {@link XContentType#shortName()} or a media type that optionally has
+     * parameters and attempts to match the value to an {@link XContentType}. The comparisons are done in lower case format and this method
+     * also supports a wildcard accept for {@code application/*}. This method can be used to parse the {@code Accept} HTTP header or a
+     * format query string parameter. This method will return {@code null} if no match is found
+     */
     public static XContentType fromMediaTypeOrFormat(String mediaType) {
         if (mediaType == null) {
             return null;
         }
         for (XContentType type : values()) {
-            if (isSameMediaTypeAs(mediaType, type)) {
+            if (isSameMediaTypeOrFormatAs(mediaType, type)) {
                 return type;
             }
         }
-        if(mediaType.toLowerCase(Locale.ROOT).startsWith("application/*")) {
+        final String lowercaseMediaType = mediaType.toLowerCase(Locale.ROOT);
+        if (lowercaseMediaType.startsWith("application/*")) {
             return JSON;
         }
 
         return null;
     }
 
-    private static boolean isSameMediaTypeAs(String stringType, XContentType type) {
+    /**
+     * Attempts to match the given media type with the known {@link XContentType} values. This match is done in a case-insensitive manner.
+     * The provided media type should not include any parameters. This method is suitable for parsing part of the {@code Content-Type}
+     * HTTP header. This method will return {@code null} if no match is found
+     */
+    public static XContentType fromMediaType(String mediaType) {
+        final String lowercaseMediaType = Objects.requireNonNull(mediaType, "mediaType cannot be null").toLowerCase(Locale.ROOT);
+        for (XContentType type : values()) {
+            if (type.mediaTypeWithoutParameters().equals(lowercaseMediaType)) {
+                return type;
+            }
+        }
+
+        return null;
+    }
+
+    private static boolean isSameMediaTypeOrFormatAs(String stringType, XContentType type) {
         return type.mediaTypeWithoutParameters().equalsIgnoreCase(stringType) ||
                 stringType.toLowerCase(Locale.ROOT).startsWith(type.mediaTypeWithoutParameters().toLowerCase(Locale.ROOT) + ";") ||
                 type.shortName().equalsIgnoreCase(stringType);
@@ -156,7 +181,7 @@ public String mediaType() {
 
     public abstract XContent xContent();
 
-    protected abstract String mediaTypeWithoutParameters();
+    public abstract String mediaTypeWithoutParameters();
 
     public static XContentType readFrom(StreamInput in) throws IOException {
         int index = in.readVInt();
@@ -168,7 +193,8 @@ public static XContentType readFrom(StreamInput in) throws IOException {
         throw new IllegalStateException("Unknown XContentType with index [" + index + "]");
     }
 
-    public static void writeTo(XContentType contentType, StreamOutput out) throws IOException {
-        out.writeVInt(contentType.index);
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeVInt(index);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java b/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java
index 4224b5328a712..e9518cf11305a 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.FastStringReader;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentGenerator;
@@ -76,33 +77,33 @@ public XContentGenerator createGenerator(OutputStream os, Set<String> includes,
     }
 
     @Override
-    public XContentParser createParser(String content) throws IOException {
-        return new CborXContentParser(cborFactory.createParser(new FastStringReader(content)));
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, String content) throws IOException {
+        return new CborXContentParser(xContentRegistry, cborFactory.createParser(new FastStringReader(content)));
     }
 
     @Override
-    public XContentParser createParser(InputStream is) throws IOException {
-        return new CborXContentParser(cborFactory.createParser(is));
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, InputStream is) throws IOException {
+        return new CborXContentParser(xContentRegistry, cborFactory.createParser(is));
     }
 
     @Override
-    public XContentParser createParser(byte[] data) throws IOException {
-        return new CborXContentParser(cborFactory.createParser(data));
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, byte[] data) throws IOException {
+        return new CborXContentParser(xContentRegistry, cborFactory.createParser(data));
     }
 
     @Override
-    public XContentParser createParser(byte[] data, int offset, int length) throws IOException {
-        return new CborXContentParser(cborFactory.createParser(data, offset, length));
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, byte[] data, int offset, int length) throws IOException {
+        return new CborXContentParser(xContentRegistry, cborFactory.createParser(data, offset, length));
     }
 
     @Override
-    public XContentParser createParser(BytesReference bytes) throws IOException {
-        return createParser(bytes.streamInput());
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, BytesReference bytes) throws IOException {
+        return createParser(xContentRegistry, bytes.streamInput());
     }
 
     @Override
-    public XContentParser createParser(Reader reader) throws IOException {
-        return new CborXContentParser(cborFactory.createParser(reader));
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, Reader reader) throws IOException {
+        return new CborXContentParser(xContentRegistry, cborFactory.createParser(reader));
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentGenerator.java
index e63a928109d6e..119cb5c98c6c4 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentGenerator.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentGenerator.java
@@ -20,20 +20,14 @@
 package org.elasticsearch.common.xcontent.cbor;
 
 import com.fasterxml.jackson.core.JsonGenerator;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContentGenerator;
 
 import java.io.OutputStream;
-import java.util.Collections;
 import java.util.Set;
 
 public class CborXContentGenerator extends JsonXContentGenerator {
 
-    public CborXContentGenerator(JsonGenerator jsonGenerator, OutputStream os) {
-        this(jsonGenerator, os, Collections.emptySet(), Collections.emptySet());
-    }
-
     public CborXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, Set<String> includes, Set<String> excludes) {
         super(jsonGenerator, os, includes, excludes);
     }
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentParser.java b/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentParser.java
index ed10ea47c0e87..b6610b5f3adea 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentParser.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentParser.java
@@ -20,6 +20,8 @@
 package org.elasticsearch.common.xcontent.cbor;
 
 import com.fasterxml.jackson.core.JsonParser;
+
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContentParser;
 
@@ -28,8 +30,8 @@
  */
 public class CborXContentParser extends JsonXContentParser {
 
-    public CborXContentParser(JsonParser parser) {
-        super(parser);
+    public CborXContentParser(NamedXContentRegistry xContentRegistry, JsonParser parser) {
+        super(xContentRegistry, parser);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java
index c8afb94f7815c..816cf5e2659eb 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java
@@ -25,6 +25,7 @@
 import com.fasterxml.jackson.core.JsonParser;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.FastStringReader;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentGenerator;
@@ -98,32 +99,32 @@ public XContentGenerator createGenerator(OutputStream os, Set<String> includes,
     }
 
     @Override
-    public XContentParser createParser(String content) throws IOException {
-        return new JsonXContentParser(jsonFactory.createParser(new FastStringReader(content)));
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, String content) throws IOException {
+        return new JsonXContentParser(xContentRegistry, jsonFactory.createParser(new FastStringReader(content)));
     }
 
     @Override
-    public XContentParser createParser(InputStream is) throws IOException {
-        return new JsonXContentParser(jsonFactory.createParser(is));
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, InputStream is) throws IOException {
+        return new JsonXContentParser(xContentRegistry, jsonFactory.createParser(is));
     }
 
     @Override
-    public XContentParser createParser(byte[] data) throws IOException {
-        return new JsonXContentParser(jsonFactory.createParser(data));
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, byte[] data) throws IOException {
+        return new JsonXContentParser(xContentRegistry, jsonFactory.createParser(data));
     }
 
     @Override
-    public XContentParser createParser(byte[] data, int offset, int length) throws IOException {
-        return new JsonXContentParser(jsonFactory.createParser(data, offset, length));
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, byte[] data, int offset, int length) throws IOException {
+        return new JsonXContentParser(xContentRegistry, jsonFactory.createParser(data, offset, length));
     }
 
     @Override
-    public XContentParser createParser(BytesReference bytes) throws IOException {
-        return createParser(bytes.streamInput());
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, BytesReference bytes) throws IOException {
+        return createParser(xContentRegistry, bytes.streamInput());
     }
 
     @Override
-    public XContentParser createParser(Reader reader) throws IOException {
-        return new JsonXContentParser(jsonFactory.createParser(reader));
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, Reader reader) throws IOException {
+        return new JsonXContentParser(xContentRegistry, jsonFactory.createParser(reader));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java
index 4a393b9dd1040..1e09f8334f772 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java
@@ -31,6 +31,7 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentGenerator;
@@ -43,13 +44,9 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
-import java.util.Collections;
 import java.util.Objects;
 import java.util.Set;
 
-/**
- *
- */
 public class JsonXContentGenerator implements XContentGenerator {
 
     /** Generator used to write content **/
@@ -75,10 +72,6 @@ public class JsonXContentGenerator implements XContentGenerator {
     private static final DefaultPrettyPrinter.Indenter INDENTER = new DefaultIndenter("  ", LF.getValue());
     private boolean prettyPrint = false;
 
-    public JsonXContentGenerator(JsonGenerator jsonGenerator, OutputStream os) {
-        this(jsonGenerator, os, Collections.emptySet(), Collections.emptySet());
-    }
-
     public JsonXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, Set<String> includes, Set<String> excludes) {
         Objects.requireNonNull(includes, "Including filters must not be null");
         Objects.requireNonNull(excludes, "Excluding filters must not be null");
@@ -130,16 +123,6 @@ public void usePrintLineFeedAtEnd() {
         writeLineFeedAtEnd = true;
     }
 
-    @Override
-    public void writeStartArray() throws IOException {
-        generator.writeStartArray();
-    }
-
-    @Override
-    public void writeEndArray() throws IOException {
-        generator.writeEndArray();
-    }
-
     private boolean isFiltered() {
         return filter != null;
     }
@@ -184,118 +167,124 @@ public void writeEndObject() throws IOException {
         generator.writeEndObject();
     }
 
+
     @Override
-    public void writeFieldName(String name) throws IOException {
-        generator.writeFieldName(name);
+    public void writeStartArray() throws IOException {
+        generator.writeStartArray();
     }
 
     @Override
-    public void writeString(String text) throws IOException {
-        generator.writeString(text);
+    public void writeEndArray() throws IOException {
+        generator.writeEndArray();
     }
 
     @Override
-    public void writeString(char[] text, int offset, int len) throws IOException {
-        generator.writeString(text, offset, len);
+    public void writeFieldName(String name) throws IOException {
+        generator.writeFieldName(name);
     }
 
     @Override
-    public void writeUTF8String(byte[] text, int offset, int length) throws IOException {
-        generator.writeUTF8String(text, offset, length);
+    public void writeNull() throws IOException {
+        generator.writeNull();
     }
 
     @Override
-    public void writeBinary(byte[] data, int offset, int len) throws IOException {
-        generator.writeBinary(data, offset, len);
+    public void writeNullField(String name) throws IOException {
+        generator.writeNullField(name);
     }
 
     @Override
-    public void writeBinary(byte[] data) throws IOException {
-        generator.writeBinary(data);
+    public void writeBooleanField(String name, boolean value) throws IOException {
+        generator.writeBooleanField(name, value);
     }
 
     @Override
-    public void writeNumber(int v) throws IOException {
-        generator.writeNumber(v);
+    public void writeBoolean(boolean value) throws IOException {
+        generator.writeBoolean(value);
     }
 
     @Override
-    public void writeNumber(long v) throws IOException {
-        generator.writeNumber(v);
+    public void writeNumberField(String name, double value) throws IOException {
+        generator.writeNumberField(name, value);
     }
 
     @Override
-    public void writeNumber(double d) throws IOException {
-        generator.writeNumber(d);
+    public void writeNumber(double value) throws IOException {
+        generator.writeNumber(value);
     }
 
     @Override
-    public void writeNumber(float f) throws IOException {
-        generator.writeNumber(f);
+    public void writeNumberField(String name, float value) throws IOException {
+        generator.writeNumberField(name, value);
     }
 
     @Override
-    public void writeBoolean(boolean state) throws IOException {
-        generator.writeBoolean(state);
+    public void writeNumber(float value) throws IOException {
+        generator.writeNumber(value);
     }
 
     @Override
-    public void writeNull() throws IOException {
-        generator.writeNull();
+    public void writeNumberField(String name, int value) throws IOException {
+        generator.writeNumberField(name, value);
     }
 
     @Override
-    public void writeStringField(String fieldName, String value) throws IOException {
-        generator.writeStringField(fieldName, value);
+    public void writeNumber(int value) throws IOException {
+        generator.writeNumber(value);
     }
 
     @Override
-    public void writeBooleanField(String fieldName, boolean value) throws IOException {
-        generator.writeBooleanField(fieldName, value);
+    public void writeNumberField(String name, long value) throws IOException {
+        generator.writeNumberField(name, value);
     }
 
     @Override
-    public void writeNullField(String fieldName) throws IOException {
-        generator.writeNullField(fieldName);
+    public void writeNumber(long value) throws IOException {
+        generator.writeNumber(value);
     }
 
     @Override
-    public void writeNumberField(String fieldName, int value) throws IOException {
-        generator.writeNumberField(fieldName, value);
+    public void writeNumber(short value) throws IOException {
+        generator.writeNumber(value);
     }
 
     @Override
-    public void writeNumberField(String fieldName, long value) throws IOException {
-        generator.writeNumberField(fieldName, value);
+    public void writeStringField(String name, String value) throws IOException {
+        generator.writeStringField(name, value);
     }
 
     @Override
-    public void writeNumberField(String fieldName, double value) throws IOException {
-        generator.writeNumberField(fieldName, value);
+    public void writeString(String value) throws IOException {
+        generator.writeString(value);
     }
 
     @Override
-    public void writeNumberField(String fieldName, float value) throws IOException {
-        generator.writeNumberField(fieldName, value);
+    public void writeString(char[] value, int offset, int len) throws IOException {
+        generator.writeString(value, offset, len);
     }
 
     @Override
-    public void writeBinaryField(String fieldName, byte[] data) throws IOException {
-        generator.writeBinaryField(fieldName, data);
+    public void writeUTF8String(byte[] value, int offset, int length) throws IOException {
+        generator.writeUTF8String(value, offset, length);
     }
 
     @Override
-    public void writeArrayFieldStart(String fieldName) throws IOException {
-        generator.writeArrayFieldStart(fieldName);
+    public void writeBinaryField(String name, byte[] value) throws IOException {
+        generator.writeBinaryField(name, value);
     }
 
     @Override
-    public void writeObjectFieldStart(String fieldName) throws IOException {
-        generator.writeObjectFieldStart(fieldName);
+    public void writeBinary(byte[] value) throws IOException {
+        generator.writeBinary(value);
     }
 
-    private void writeStartRaw(String fieldName) throws IOException {
-        writeFieldName(fieldName);
+    @Override
+    public void writeBinary(byte[] value, int offset, int len) throws IOException {
+        generator.writeBinary(value, offset, len);
+    }
+
+    private void writeStartRaw(String name) throws IOException {
+        writeFieldName(name);
         generator.writeRaw(':');
     }
 
@@ -309,7 +298,7 @@ public void writeEndRaw() {
     }
 
     @Override
-    public void writeRawField(String fieldName, InputStream content) throws IOException {
+    public void writeRawField(String name, InputStream content) throws IOException {
         if (content.markSupported() == false) {
             // needed for the XContentFactory.xContentType call
             content = new BufferedInputStream(content);
@@ -318,14 +307,20 @@ public void writeRawField(String fieldName, InputStream content) throws IOExcept
         if (contentType == null) {
             throw new IllegalArgumentException("Can't write raw bytes whose xcontent-type can't be guessed");
         }
+        writeRawField(name, content, contentType);
+    }
+
+    @Override
+    public void writeRawField(String name, InputStream content, XContentType contentType) throws IOException {
         if (mayWriteRawData(contentType) == false) {
-            try (XContentParser parser = XContentFactory.xContent(contentType).createParser(content)) {
+            // EMPTY is safe here because we never call namedObject when writing raw data
+            try (XContentParser parser = XContentFactory.xContent(contentType).createParser(NamedXContentRegistry.EMPTY, content)) {
                 parser.nextToken();
-                writeFieldName(fieldName);
+                writeFieldName(name);
                 copyCurrentStructure(parser);
             }
         } else {
-            writeStartRaw(fieldName);
+            writeStartRaw(name);
             flush();
             Streams.copy(content, os);
             writeEndRaw();
@@ -333,16 +328,21 @@ public void writeRawField(String fieldName, InputStream content) throws IOExcept
     }
 
     @Override
-    public final void writeRawField(String fieldName, BytesReference content) throws IOException {
+    public final void writeRawField(String name, BytesReference content) throws IOException {
         XContentType contentType = XContentFactory.xContentType(content);
         if (contentType == null) {
             throw new IllegalArgumentException("Can't write raw bytes whose xcontent-type can't be guessed");
         }
+        writeRawField(name, content, contentType);
+    }
+
+    @Override
+    public final void writeRawField(String name, BytesReference content, XContentType contentType) throws IOException {
         if (mayWriteRawData(contentType) == false) {
-            writeFieldName(fieldName);
+            writeFieldName(name);
             copyRawValue(content, contentType.xContent());
         } else {
-            writeStartRaw(fieldName);
+            writeStartRaw(name);
             flush();
             content.writeTo(os);
             writeEndRaw();
@@ -355,6 +355,11 @@ public final void writeRawValue(BytesReference content) throws IOException {
         if (contentType == null) {
             throw new IllegalArgumentException("Can't write raw bytes whose xcontent-type can't be guessed");
         }
+        writeRawValue(content, contentType);
+    }
+
+    @Override
+    public final void writeRawValue(BytesReference content, XContentType contentType) throws IOException {
         if (mayWriteRawData(contentType) == false) {
             copyRawValue(content, contentType.xContent());
         } else {
@@ -385,8 +390,9 @@ protected boolean supportsRawWrites() {
     }
 
     protected void copyRawValue(BytesReference content, XContent xContent) throws IOException {
+        // EMPTY is safe here because we never call namedObject
         try (StreamInput input = content.streamInput();
-             XContentParser parser = xContent.createParser(input)) {
+             XContentParser parser = xContent.createParser(NamedXContentRegistry.EMPTY, input)) {
             copyCurrentStructure(parser);
         }
     }
@@ -416,7 +422,7 @@ public void close() throws IOException {
         }
         JsonStreamContext context = generator.getOutputContext();
         if ((context != null) && (context.inRoot() ==  false)) {
-            throw new IOException("unclosed object or array found");
+            throw new IOException("Unclosed object or array found");
         }
         if (writeLineFeedAtEnd) {
             flush();
@@ -426,4 +432,8 @@ public void close() throws IOException {
         generator.close();
     }
 
+    @Override
+    public boolean isClosed() {
+        return generator.isClosed();
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentParser.java b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentParser.java
index 5728e6035e6c2..0a570b7685e39 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentParser.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentParser.java
@@ -22,8 +22,10 @@
 import com.fasterxml.jackson.core.JsonLocation;
 import com.fasterxml.jackson.core.JsonParser;
 import com.fasterxml.jackson.core.JsonToken;
+
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentLocation;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.support.AbstractXContentParser;
@@ -38,7 +40,8 @@ public class JsonXContentParser extends AbstractXContentParser {
 
     final JsonParser parser;
 
-    public JsonXContentParser(JsonParser parser) {
+    public JsonXContentParser(NamedXContentRegistry xContentRegistry, JsonParser parser) {
+        super(xContentRegistry);
         this.parser = parser;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java b/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java
index 94ac9b9435626..a11bdda931cfb 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java
@@ -25,6 +25,7 @@
 import com.fasterxml.jackson.dataformat.smile.SmileGenerator;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.FastStringReader;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentGenerator;
@@ -77,32 +78,32 @@ public XContentGenerator createGenerator(OutputStream os, Set<String> includes,
     }
 
     @Override
-    public XContentParser createParser(String content) throws IOException {
-        return new SmileXContentParser(smileFactory.createParser(new FastStringReader(content)));
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, String content) throws IOException {
+        return new SmileXContentParser(xContentRegistry, smileFactory.createParser(new FastStringReader(content)));
     }
 
     @Override
-    public XContentParser createParser(InputStream is) throws IOException {
-        return new SmileXContentParser(smileFactory.createParser(is));
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, InputStream is) throws IOException {
+        return new SmileXContentParser(xContentRegistry, smileFactory.createParser(is));
     }
 
     @Override
-    public XContentParser createParser(byte[] data) throws IOException {
-        return new SmileXContentParser(smileFactory.createParser(data));
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, byte[] data) throws IOException {
+        return new SmileXContentParser(xContentRegistry, smileFactory.createParser(data));
     }
 
     @Override
-    public XContentParser createParser(byte[] data, int offset, int length) throws IOException {
-        return new SmileXContentParser(smileFactory.createParser(data, offset, length));
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, byte[] data, int offset, int length) throws IOException {
+        return new SmileXContentParser(xContentRegistry, smileFactory.createParser(data, offset, length));
     }
 
     @Override
-    public XContentParser createParser(BytesReference bytes) throws IOException {
-        return createParser(bytes.streamInput());
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, BytesReference bytes) throws IOException {
+        return createParser(xContentRegistry, bytes.streamInput());
     }
 
     @Override
-    public XContentParser createParser(Reader reader) throws IOException {
-        return new SmileXContentParser(smileFactory.createParser(reader));
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, Reader reader) throws IOException {
+        return new SmileXContentParser(xContentRegistry, smileFactory.createParser(reader));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContentGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContentGenerator.java
index afa420805f743..f368c0e383f31 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContentGenerator.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContentGenerator.java
@@ -20,20 +20,14 @@
 package org.elasticsearch.common.xcontent.smile;
 
 import com.fasterxml.jackson.core.JsonGenerator;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContentGenerator;
 
 import java.io.OutputStream;
-import java.util.Collections;
 import java.util.Set;
 
 public class SmileXContentGenerator extends JsonXContentGenerator {
 
-    public SmileXContentGenerator(JsonGenerator jsonGenerator, OutputStream os) {
-        this(jsonGenerator, os, Collections.emptySet(), Collections.emptySet());
-    }
-
     public SmileXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, Set<String> includes, Set<String> excludes) {
         super(jsonGenerator, os, includes, excludes);
     }
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContentParser.java b/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContentParser.java
index 2bbf99db27de8..0ed4a27850598 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContentParser.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContentParser.java
@@ -20,6 +20,8 @@
 package org.elasticsearch.common.xcontent.smile;
 
 import com.fasterxml.jackson.core.JsonParser;
+
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContentParser;
 
@@ -28,8 +30,8 @@
  */
 public class SmileXContentParser extends JsonXContentParser {
 
-    public SmileXContentParser(JsonParser parser) {
-        super(parser);
+    public SmileXContentParser(NamedXContentRegistry xContentRegistry, JsonParser parser) {
+        super(xContentRegistry, parser);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java b/core/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java
index 9f313a59b90fc..1c23ddf006de0 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java
@@ -22,6 +22,11 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.Booleans;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.LogConfigurator;
+import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
@@ -35,7 +40,6 @@
  *
  */
 public abstract class AbstractXContentParser implements XContentParser {
-
     // Currently this is not a setting that can be changed and is a policy
     // that relates to how parsing of things like "boost" are done across
     // the whole of Elasticsearch (eg if String "1.0" is a valid float).
@@ -52,7 +56,21 @@ private static void checkCoerceString(boolean coerce, Class<? extends Number> cl
         }
     }
 
+    /**
+     * We have to lazy initialize the deprecation logger as otherwise a static logger here would be constructed before logging is configured
+     * leading to a runtime failure (see {@link LogConfigurator#checkErrorListener()} ). The premature construction would come from any
+     * {@link Setting} object constructed in, for example, {@link org.elasticsearch.env.Environment}.
+     */
+    private static class DeprecationLoggerHolder {
+        static DeprecationLogger deprecationLogger = new DeprecationLogger(Loggers.getLogger(AbstractXContentParser.class));
+    }
+
 
+    private final NamedXContentRegistry xContentRegistry;
+
+    public AbstractXContentParser(NamedXContentRegistry xContentRegistry) {
+        this.xContentRegistry = xContentRegistry;
+    }
 
     // The 3rd party parsers we rely on are known to silently truncate fractions: see
     //   http://fasterxml.github.io/jackson-core/javadoc/2.3.0/com/fasterxml/jackson/core/JsonParser.html#getShortValue()
@@ -87,13 +105,28 @@ public boolean isBooleanValue() throws IOException {
 
     @Override
     public boolean booleanValue() throws IOException {
+        boolean interpretedAsLenient = false;
+        boolean booleanValue;
+        String rawValue = null;
+
         Token token = currentToken();
         if (token == Token.VALUE_NUMBER) {
-            return intValue() != 0;
+            interpretedAsLenient = true;
+            booleanValue = intValue() != 0;
+            rawValue = String.valueOf(intValue());
         } else if (token == Token.VALUE_STRING) {
-            return Booleans.parseBoolean(textCharacters(), textOffset(), textLength(), false /* irrelevant */);
+            rawValue = new String(textCharacters(), textOffset(), textLength());
+            interpretedAsLenient = Booleans.isStrictlyBoolean(rawValue) == false;
+            booleanValue = Booleans.parseBoolean(rawValue, false /* irrelevant */);
+        } else {
+            booleanValue = doBooleanValue();
+        }
+        if (interpretedAsLenient) {
+            final DeprecationLogger deprecationLogger = DeprecationLoggerHolder.deprecationLogger;
+            deprecationLogger.deprecated("Expected a boolean [true/false] for property [{}] but got [{}]", currentName(), rawValue);
         }
-        return doBooleanValue();
+        return booleanValue;
+
     }
 
     protected abstract boolean doBooleanValue() throws IOException;
@@ -108,7 +141,7 @@ public short shortValue(boolean coerce) throws IOException {
         Token token = currentToken();
         if (token == Token.VALUE_STRING) {
             checkCoerceString(coerce, Short.class);
-            return Short.parseShort(text());
+            return (short) Double.parseDouble(text());
         }
         short result = doShortValue();
         ensureNumberConversion(coerce, result, Short.class);
@@ -122,13 +155,12 @@ public int intValue() throws IOException {
         return intValue(DEFAULT_NUMBER_COERCE_POLICY);
     }
 
-
     @Override
     public int intValue(boolean coerce) throws IOException {
         Token token = currentToken();
         if (token == Token.VALUE_STRING) {
             checkCoerceString(coerce, Integer.class);
-            return Integer.parseInt(text());
+            return (int) Double.parseDouble(text());
         }
         int result = doIntValue();
         ensureNumberConversion(coerce, result, Integer.class);
@@ -147,7 +179,13 @@ public long longValue(boolean coerce) throws IOException {
         Token token = currentToken();
         if (token == Token.VALUE_STRING) {
             checkCoerceString(coerce, Long.class);
-            return Long.parseLong(text());
+            // longs need special handling so we don't lose precision while parsing
+            String stringValue = text();
+            try {
+                return Long.parseLong(stringValue);
+            } catch (NumberFormatException e) {
+                return (long) Double.parseDouble(stringValue);
+            }
         }
         long result = doLongValue();
         ensureNumberConversion(coerce, result, Long.class);
@@ -218,6 +256,16 @@ public Map<String, Object> mapOrdered() throws IOException {
         return readOrderedMap(this);
     }
 
+    @Override
+    public Map<String, String> mapStrings() throws IOException {
+        return readMapStrings(this);
+    }
+
+    @Override
+    public Map<String, String> mapStringsOrdered() throws IOException {
+        return readOrderedMapStrings(this);
+    }
+
     @Override
     public List<Object> list() throws IOException {
         return readList(this);
@@ -232,10 +280,18 @@ interface MapFactory {
         Map<String, Object> newMap();
     }
 
+    interface MapStringsFactory {
+        Map<String, String> newMap();
+    }
+
     static final MapFactory SIMPLE_MAP_FACTORY = HashMap::new;
 
     static final MapFactory ORDERED_MAP_FACTORY = LinkedHashMap::new;
 
+    static final MapStringsFactory SIMPLE_MAP_STRINGS_FACTORY = HashMap::new;
+
+    static final MapStringsFactory ORDERED_MAP_STRINGS_FACTORY = LinkedHashMap::new;
+
     static Map<String, Object> readMap(XContentParser parser) throws IOException {
         return readMap(parser, SIMPLE_MAP_FACTORY);
     }
@@ -244,6 +300,14 @@ static Map<String, Object> readOrderedMap(XContentParser parser) throws IOExcept
         return readMap(parser, ORDERED_MAP_FACTORY);
     }
 
+    static Map<String, String> readMapStrings(XContentParser parser) throws IOException {
+        return readMapStrings(parser, SIMPLE_MAP_STRINGS_FACTORY);
+    }
+
+    static Map<String, String> readOrderedMapStrings(XContentParser parser) throws IOException {
+        return readMapStrings(parser, ORDERED_MAP_STRINGS_FACTORY);
+    }
+
     static List<Object> readList(XContentParser parser) throws IOException {
         return readList(parser, SIMPLE_MAP_FACTORY);
     }
@@ -272,6 +336,26 @@ static Map<String, Object> readMap(XContentParser parser, MapFactory mapFactory)
         return map;
     }
 
+    static Map<String, String> readMapStrings(XContentParser parser, MapStringsFactory mapStringsFactory) throws IOException {
+        Map<String, String> map = mapStringsFactory.newMap();
+        XContentParser.Token token = parser.currentToken();
+        if (token == null) {
+            token = parser.nextToken();
+        }
+        if (token == XContentParser.Token.START_OBJECT) {
+            token = parser.nextToken();
+        }
+        for (; token == XContentParser.Token.FIELD_NAME; token = parser.nextToken()) {
+            // Must point to field name
+            String fieldName = parser.currentName();
+            // And then the value...
+            parser.nextToken();
+            String value = parser.text();
+            map.put(fieldName, value);
+        }
+        return map;
+    }
+
     static List<Object> readList(XContentParser parser, MapFactory mapFactory) throws IOException {
         XContentParser.Token token = parser.currentToken();
         if (token == null) {
@@ -313,6 +397,16 @@ static Object readValue(XContentParser parser, MapFactory mapFactory, XContentPa
         return null;
     }
 
+    @Override
+    public <T> T namedObject(Class<T> categoryClass, String name, Object context) throws IOException {
+        return xContentRegistry.parseNamedObject(categoryClass, name, this, context);
+    }
+
+    @Override
+    public NamedXContentRegistry getXContentRegistry() {
+        return xContentRegistry;
+    }
+
     @Override
     public abstract boolean isClosed();
 }
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java b/core/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java
index a8c120f424b5c..c8d0b8c615c48 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java
@@ -19,20 +19,31 @@
 
 package org.elasticsearch.common.xcontent.support;
 
+import org.apache.lucene.util.automaton.Automata;
+import org.apache.lucene.util.automaton.Automaton;
+import org.apache.lucene.util.automaton.CharacterRunAutomaton;
+import org.apache.lucene.util.automaton.Operations;
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.common.Booleans;
+import org.elasticsearch.common.Numbers;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.unit.TimeValue;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Function;
 
 /**
  *
  */
 public class XContentMapValues {
+    private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(XContentMapValues.class));
 
     /**
      * Extracts raw values (string, int, and so on) based on the path provided returning all of them
@@ -134,115 +145,179 @@ private static Object extractValue(String[] pathElements, int index, Object curr
         return null;
     }
 
-    public static Map<String, Object> filter(Map<String, Object> map, String[] includes, String[] excludes) {
-        Map<String, Object> result = new HashMap<>();
-        filter(map, result, includes == null ? Strings.EMPTY_ARRAY : includes, excludes == null ? Strings.EMPTY_ARRAY : excludes, new StringBuilder());
-        return result;
+    /**
+     * Only keep properties in {@code map} that match the {@code includes} but
+     * not the {@code excludes}. An empty list of includes is interpreted as a
+     * wildcard while an empty list of excludes does not match anything.
+     *
+     * If a property matches both an include and an exclude, then the exclude
+     * wins.
+     *
+     * If an object matches, then any of its sub properties are automatically
+     * considered as matching as well, both for includes and excludes.
+     *
+     * Dots in field names are treated as sub objects. So for instance if a
+     * document contains {@code a.b} as a property and {@code a} is an include,
+     * then {@code a.b} will be kept in the filtered map.
+     */
+    public static Map<String, Object> filter(Map<String, ?> map, String[] includes, String[] excludes) {
+        return filter(includes, excludes).apply(map);
     }
 
-    private static void filter(Map<String, Object> map, Map<String, Object> into, String[] includes, String[] excludes, StringBuilder sb) {
-        if (includes.length == 0 && excludes.length == 0) {
-            into.putAll(map);
-            return;
+    /**
+     * Returns a function that filters a document map based on the given include and exclude rules.
+     * @see #filter(Map, String[], String[]) for details
+     */
+    public static Function<Map<String, ?>, Map<String, Object>> filter(String[] includes, String[] excludes) {
+        CharacterRunAutomaton matchAllAutomaton = new CharacterRunAutomaton(Automata.makeAnyString());
+
+        CharacterRunAutomaton include;
+        if (includes == null || includes.length == 0) {
+            include = matchAllAutomaton;
+        } else {
+            Automaton includeA = Regex.simpleMatchToAutomaton(includes);
+            includeA = makeMatchDotsInFieldNames(includeA);
+            include = new CharacterRunAutomaton(includeA);
+        }
+
+        Automaton excludeA;
+        if (excludes == null || excludes.length == 0) {
+            excludeA = Automata.makeEmpty();
+        } else {
+            excludeA = Regex.simpleMatchToAutomaton(excludes);
+            excludeA = makeMatchDotsInFieldNames(excludeA);
         }
-        for (Map.Entry<String, Object> entry : map.entrySet()) {
+        CharacterRunAutomaton exclude = new CharacterRunAutomaton(excludeA);
+
+        // NOTE: We cannot use Operations.minus because of the special case that
+        // we want all sub properties to match as soon as an object matches
+
+        return (map) -> filter(map,
+            include, 0,
+            exclude, 0,
+            matchAllAutomaton);
+    }
+
+    /** Make matches on objects also match dots in field names.
+     *  For instance, if the original simple regex is `foo`, this will translate
+     *  it into `foo` OR `foo.*`. */
+    private static Automaton makeMatchDotsInFieldNames(Automaton automaton) {
+        return Operations.union(
+                automaton,
+                Operations.concatenate(Arrays.asList(automaton, Automata.makeChar('.'), Automata.makeAnyString())));
+    }
+
+    private static int step(CharacterRunAutomaton automaton, String key, int state) {
+        for (int i = 0; state != -1 && i < key.length(); ++i) {
+            state = automaton.step(state, key.charAt(i));
+        }
+        return state;
+    }
+
+    private static Map<String, Object> filter(Map<String, ?> map,
+            CharacterRunAutomaton includeAutomaton, int initialIncludeState,
+            CharacterRunAutomaton excludeAutomaton, int initialExcludeState,
+            CharacterRunAutomaton matchAllAutomaton) {
+        Map<String, Object> filtered = new HashMap<>();
+        for (Map.Entry<String, ?> entry : map.entrySet()) {
             String key = entry.getKey();
-            int mark = sb.length();
-            if (sb.length() > 0) {
-                sb.append('.');
+
+            int includeState = step(includeAutomaton, key, initialIncludeState);
+            if (includeState == -1) {
+                continue;
             }
-            sb.append(key);
-            String path = sb.toString();
 
-            if (Regex.simpleMatch(excludes, path)) {
-                sb.setLength(mark);
+            int excludeState = step(excludeAutomaton, key, initialExcludeState);
+            if (excludeState != -1 && excludeAutomaton.isAccept(excludeState)) {
                 continue;
             }
 
-            boolean exactIncludeMatch = false; // true if the current position was specifically mentioned
-            boolean pathIsPrefixOfAnInclude = false; // true if potentially a sub scope can be included
-            if (includes.length == 0) {
-                // implied match anything
-                exactIncludeMatch = true;
-            } else {
-                for (String include : includes) {
-                    // check for prefix matches as well to see if we need to zero in, something like: obj1.arr1.* or *.field
-                    // note, this does not work well with middle matches, like obj1.*.obj3
-                    if (include.charAt(0) == '*') {
-                        if (Regex.simpleMatch(include, path)) {
-                            exactIncludeMatch = true;
-                            break;
-                        }
-                        pathIsPrefixOfAnInclude = true;
-                        continue;
-                    }
-                    if (include.startsWith(path)) {
-                        if (include.length() == path.length()) {
-                            exactIncludeMatch = true;
-                            break;
-                        } else if (include.length() > path.length() && include.charAt(path.length()) == '.') {
-                            // include might may match deeper paths. Dive deeper.
-                            pathIsPrefixOfAnInclude = true;
-                            continue;
-                        }
-                    }
-                    if (Regex.simpleMatch(include, path)) {
-                        exactIncludeMatch = true;
-                        break;
-                    }
+            Object value = entry.getValue();
+
+            CharacterRunAutomaton subIncludeAutomaton = includeAutomaton;
+            int subIncludeState = includeState;
+            if (includeAutomaton.isAccept(includeState)) {
+                if (excludeState == -1 || excludeAutomaton.step(excludeState, '.') == -1) {
+                    // the exclude has no chances to match inner properties
+                    filtered.put(key, value);
+                    continue;
+                } else {
+                    // the object matched, so consider that the include matches every inner property
+                    // we only care about excludes now
+                    subIncludeAutomaton = matchAllAutomaton;
+                    subIncludeState = 0;
                 }
             }
 
-            if (!(pathIsPrefixOfAnInclude || exactIncludeMatch)) {
-                // skip subkeys, not interesting.
-                sb.setLength(mark);
-                continue;
-            }
+            if (value instanceof Map) {
+
+                subIncludeState = subIncludeAutomaton.step(subIncludeState, '.');
+                if (subIncludeState == -1) {
+                    continue;
+                }
+                if (excludeState != -1) {
+                    excludeState = excludeAutomaton.step(excludeState, '.');
+                }
+
+                Map<String, Object> valueAsMap = (Map<String, Object>) value;
+                Map<String, Object> filteredValue = filter(valueAsMap,
+                        subIncludeAutomaton, subIncludeState, excludeAutomaton, excludeState, matchAllAutomaton);
+                if (includeAutomaton.isAccept(includeState) || filteredValue.isEmpty() == false) {
+                    filtered.put(key, filteredValue);
+                }
 
+            } else if (value instanceof Iterable) {
 
-            if (entry.getValue() instanceof Map) {
-                Map<String, Object> innerInto = new HashMap<>();
-                // if we had an exact match, we want give deeper excludes their chance
-                filter((Map<String, Object>) entry.getValue(), innerInto, exactIncludeMatch ? Strings.EMPTY_ARRAY : includes, excludes, sb);
-                if (exactIncludeMatch || !innerInto.isEmpty()) {
-                    into.put(entry.getKey(), innerInto);
+                List<Object> filteredValue = filter((Iterable<?>) value,
+                        subIncludeAutomaton, subIncludeState, excludeAutomaton, excludeState, matchAllAutomaton);
+                if (filteredValue.isEmpty() == false) {
+                    filtered.put(key, filteredValue);
                 }
-            } else if (entry.getValue() instanceof List) {
-                List<Object> list = (List<Object>) entry.getValue();
-                List<Object> innerInto = new ArrayList<>(list.size());
-                // if we had an exact match, we want give deeper excludes their chance
-                filter(list, innerInto, exactIncludeMatch ? Strings.EMPTY_ARRAY : includes, excludes, sb);
-                into.put(entry.getKey(), innerInto);
-            } else if (exactIncludeMatch) {
-                into.put(entry.getKey(), entry.getValue());
+
+            } else {
+
+                // leaf property
+                if (includeAutomaton.isAccept(includeState)
+                        && (excludeState == -1 || excludeAutomaton.isAccept(excludeState) == false)) {
+                    filtered.put(key, value);
+                }
+
             }
-            sb.setLength(mark);
-        }
-    }
 
-    private static void filter(List<Object> from, List<Object> to, String[] includes, String[] excludes, StringBuilder sb) {
-        if (includes.length == 0 && excludes.length == 0) {
-            to.addAll(from);
-            return;
         }
+        return filtered;
+    }
 
-        for (Object o : from) {
-            if (o instanceof Map) {
-                Map<String, Object> innerInto = new HashMap<>();
-                filter((Map<String, Object>) o, innerInto, includes, excludes, sb);
-                if (!innerInto.isEmpty()) {
-                    to.add(innerInto);
+    private static List<Object> filter(Iterable<?> iterable,
+            CharacterRunAutomaton includeAutomaton, int initialIncludeState,
+            CharacterRunAutomaton excludeAutomaton, int initialExcludeState,
+            CharacterRunAutomaton matchAllAutomaton) {
+        List<Object> filtered = new ArrayList<>();
+        boolean isInclude = includeAutomaton.isAccept(initialIncludeState);
+        for (Object value : iterable) {
+            if (value instanceof Map) {
+                int includeState = includeAutomaton.step(initialIncludeState, '.');
+                int excludeState = initialExcludeState;
+                if (excludeState != -1) {
+                    excludeState = excludeAutomaton.step(excludeState, '.');
                 }
-            } else if (o instanceof List) {
-                List<Object> innerInto = new ArrayList<>();
-                filter((List<Object>) o, innerInto, includes, excludes, sb);
-                if (!innerInto.isEmpty()) {
-                    to.add(innerInto);
+                Map<String, Object> filteredValue = filter((Map<String, ?>)value,
+                        includeAutomaton, includeState, excludeAutomaton, excludeState, matchAllAutomaton);
+                if (filteredValue.isEmpty() == false) {
+                    filtered.add(filteredValue);
                 }
-            } else {
-                to.add(o);
+            } else if (value instanceof Iterable) {
+                List<Object> filteredValue = filter((Iterable<?>) value,
+                        includeAutomaton, initialIncludeState, excludeAutomaton, initialExcludeState, matchAllAutomaton);
+                if (filteredValue.isEmpty() == false) {
+                    filtered.add(filteredValue);
+                }
+            } else if (isInclude) {
+                // #22557: only accept this array value if the key we are on is accepted:
+                filtered.add(value);
             }
         }
+        return filtered;
     }
 
     public static boolean isObject(Object node) {
@@ -290,7 +365,7 @@ public static double nodeDoubleValue(Object node) {
 
     public static int nodeIntegerValue(Object node) {
         if (node instanceof Number) {
-            return ((Number) node).intValue();
+            return Numbers.toIntExact((Number) node);
         }
         return Integer.parseInt(node.toString());
     }
@@ -299,10 +374,7 @@ public static int nodeIntegerValue(Object node, int defaultValue) {
         if (node == null) {
             return defaultValue;
         }
-        if (node instanceof Number) {
-            return ((Number) node).intValue();
-        }
-        return Integer.parseInt(node.toString());
+        return nodeIntegerValue(node);
     }
 
     public static short nodeShortValue(Object node, short defaultValue) {
@@ -314,7 +386,7 @@ public static short nodeShortValue(Object node, short defaultValue) {
 
     public static short nodeShortValue(Object node) {
         if (node instanceof Number) {
-            return ((Number) node).shortValue();
+            return Numbers.toShortExact((Number) node);
         }
         return Short.parseShort(node.toString());
     }
@@ -328,7 +400,7 @@ public static byte nodeByteValue(Object node, byte defaultValue) {
 
     public static byte nodeByteValue(Object node) {
         if (node instanceof Number) {
-            return ((Number) node).byteValue();
+            return Numbers.toByteExact((Number) node);
         }
         return Byte.parseByte(node.toString());
     }
@@ -342,7 +414,7 @@ public static long nodeLongValue(Object node, long defaultValue) {
 
     public static long nodeLongValue(Object node) {
         if (node instanceof Number) {
-            return ((Number) node).longValue();
+            return Numbers.toLongExact((Number) node);
         }
         return Long.parseLong(node.toString());
     }
@@ -350,25 +422,35 @@ public static long nodeLongValue(Object node) {
     /**
      * This method is very lenient, use {@link #nodeBooleanValue} instead.
      */
-    public static boolean lenientNodeBooleanValue(Object node, boolean defaultValue) {
+    public static boolean lenientNodeBooleanValue(Object node, String name, boolean defaultValue) {
         if (node == null) {
             return defaultValue;
         }
-        return lenientNodeBooleanValue(node);
+        return lenientNodeBooleanValue(node, name);
     }
 
     /**
      * This method is very lenient, use {@link #nodeBooleanValue} instead.
      */
-    public static boolean lenientNodeBooleanValue(Object node) {
+    public static boolean lenientNodeBooleanValue(Object node, String name) {
+        boolean interpretedAsLenient = false;
+        boolean booleanValue;
+
         if (node instanceof Boolean) {
-            return (Boolean) node;
+            booleanValue = (Boolean) node;
+        } else if (node instanceof Number) {
+            interpretedAsLenient = true;
+            booleanValue = ((Number) node).intValue() != 0;
+        } else {
+            String value = node.toString();
+            booleanValue = ((value.equals("false") || value.equals("0") || value.equals("off"))) == false;
+            interpretedAsLenient = Booleans.isStrictlyBoolean(value) == false;
         }
-        if (node instanceof Number) {
-            return ((Number) node).intValue() != 0;
+
+        if (interpretedAsLenient) {
+            DEPRECATION_LOGGER.deprecated("Expected a boolean [true/false] for property [{}] but got [{}]", name, node.toString());
         }
-        String value = node.toString();
-        return !(value.equals("false") || value.equals("0") || value.equals("off"));
+        return booleanValue;
     }
 
     public static boolean nodeBooleanValue(Object node) {
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContent.java b/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContent.java
index 54da03118d70b..91447da1d924d 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContent.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContent.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.FastStringReader;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentGenerator;
@@ -72,32 +73,32 @@ public XContentGenerator createGenerator(OutputStream os, Set<String> includes,
     }
 
     @Override
-    public XContentParser createParser(String content) throws IOException {
-        return new YamlXContentParser(yamlFactory.createParser(new FastStringReader(content)));
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, String content) throws IOException {
+        return new YamlXContentParser(xContentRegistry, yamlFactory.createParser(new FastStringReader(content)));
     }
 
     @Override
-    public XContentParser createParser(InputStream is) throws IOException {
-        return new YamlXContentParser(yamlFactory.createParser(is));
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, InputStream is) throws IOException {
+        return new YamlXContentParser(xContentRegistry, yamlFactory.createParser(is));
     }
 
     @Override
-    public XContentParser createParser(byte[] data) throws IOException {
-        return new YamlXContentParser(yamlFactory.createParser(data));
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, byte[] data) throws IOException {
+        return new YamlXContentParser(xContentRegistry, yamlFactory.createParser(data));
     }
 
     @Override
-    public XContentParser createParser(byte[] data, int offset, int length) throws IOException {
-        return new YamlXContentParser(yamlFactory.createParser(data, offset, length));
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, byte[] data, int offset, int length) throws IOException {
+        return new YamlXContentParser(xContentRegistry, yamlFactory.createParser(data, offset, length));
     }
 
     @Override
-    public XContentParser createParser(BytesReference bytes) throws IOException {
-        return createParser(bytes.streamInput());
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, BytesReference bytes) throws IOException {
+        return createParser(xContentRegistry, bytes.streamInput());
     }
 
     @Override
-    public XContentParser createParser(Reader reader) throws IOException {
-        return new YamlXContentParser(yamlFactory.createParser(reader));
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, Reader reader) throws IOException {
+        return new YamlXContentParser(xContentRegistry, yamlFactory.createParser(reader));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContentGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContentGenerator.java
index d2c53c8a02009..0d969c21a0f9f 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContentGenerator.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContentGenerator.java
@@ -20,20 +20,14 @@
 package org.elasticsearch.common.xcontent.yaml;
 
 import com.fasterxml.jackson.core.JsonGenerator;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContentGenerator;
 
 import java.io.OutputStream;
-import java.util.Collections;
 import java.util.Set;
 
 public class YamlXContentGenerator extends JsonXContentGenerator {
 
-    public YamlXContentGenerator(JsonGenerator jsonGenerator, OutputStream os) {
-        this(jsonGenerator, os, Collections.emptySet(), Collections.emptySet());
-    }
-
     public YamlXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, Set<String> includes, Set<String> excludes) {
         super(jsonGenerator, os, includes, excludes);
     }
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContentParser.java b/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContentParser.java
index 3b674c054dc12..ddc2c19d993b8 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContentParser.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContentParser.java
@@ -20,6 +20,8 @@
 package org.elasticsearch.common.xcontent.yaml;
 
 import com.fasterxml.jackson.core.JsonParser;
+
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContentParser;
 
@@ -28,8 +30,8 @@
  */
 public class YamlXContentParser extends JsonXContentParser {
 
-    public YamlXContentParser(JsonParser parser) {
-        super(parser);
+    public YamlXContentParser(NamedXContentRegistry xContentRegistry, JsonParser parser) {
+        super(xContentRegistry, parser);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/discovery/DiscoveryModule.java b/core/src/main/java/org/elasticsearch/discovery/DiscoveryModule.java
index 040066adeb6b1..2328b5a861675 100644
--- a/core/src/main/java/org/elasticsearch/discovery/DiscoveryModule.java
+++ b/core/src/main/java/org/elasticsearch/discovery/DiscoveryModule.java
@@ -19,120 +19,89 @@
 
 package org.elasticsearch.discovery;
 
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.common.inject.AbstractModule;
-import org.elasticsearch.common.inject.multibindings.Multibinder;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.ExtensionPoint;
-import org.elasticsearch.discovery.local.LocalDiscovery;
+import org.elasticsearch.discovery.single.SingleNodeDiscovery;
+import org.elasticsearch.discovery.zen.UnicastHostsProvider;
 import org.elasticsearch.discovery.zen.ZenDiscovery;
-import org.elasticsearch.discovery.zen.elect.ElectMasterService;
-import org.elasticsearch.discovery.zen.ping.ZenPing;
-import org.elasticsearch.discovery.zen.ping.ZenPingService;
-import org.elasticsearch.discovery.zen.ping.unicast.UnicastHostsProvider;
-import org.elasticsearch.discovery.zen.ping.unicast.UnicastZenPing;
+import org.elasticsearch.plugins.DiscoveryPlugin;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
 
-import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
 import java.util.function.Function;
+import java.util.function.Supplier;
 
 /**
  * A module for loading classes for node discovery.
  */
-public class DiscoveryModule extends AbstractModule {
+public class DiscoveryModule {
 
     public static final Setting<String> DISCOVERY_TYPE_SETTING =
-        new Setting<>("discovery.type", "zen", Function.identity(),
-            Property.NodeScope);
-    public static final Setting<String> ZEN_MASTER_SERVICE_TYPE_SETTING =
-        new Setting<>("discovery.zen.masterservice.type", "zen", Function.identity(), Property.NodeScope);
+        new Setting<>("discovery.type", "zen", Function.identity(), Property.NodeScope);
+    public static final Setting<Optional<String>> DISCOVERY_HOSTS_PROVIDER_SETTING =
+        new Setting<>("discovery.zen.hosts_provider", (String)null, Optional::ofNullable, Property.NodeScope);
 
-    private final Settings settings;
-    private final Map<String, List<Class<? extends UnicastHostsProvider>>> unicastHostProviders = new HashMap<>();
-    private final ExtensionPoint.ClassSet<ZenPing> zenPings = new ExtensionPoint.ClassSet<>("zen_ping", ZenPing.class);
-    private final Map<String, Class<? extends Discovery>> discoveryTypes = new HashMap<>();
-    private final Map<String, Class<? extends ElectMasterService>> masterServiceType = new HashMap<>();
+    private final Discovery discovery;
 
-    public DiscoveryModule(Settings settings) {
-        this.settings = settings;
-        addDiscoveryType("local", LocalDiscovery.class);
-        addDiscoveryType("zen", ZenDiscovery.class);
-        addElectMasterService("zen", ElectMasterService.class);
-        // always add the unicast hosts, or things get angry!
-        addZenPing(UnicastZenPing.class);
-    }
+    public DiscoveryModule(Settings settings, ThreadPool threadPool, TransportService transportService,
+                           NamedWriteableRegistry namedWriteableRegistry, NetworkService networkService, ClusterService clusterService,
+                           List<DiscoveryPlugin> plugins) {
+        final UnicastHostsProvider hostsProvider;
 
-    /**
-     * Adds a custom unicast hosts provider to build a dynamic list of unicast hosts list when doing unicast discovery.
-     *
-     * @param type discovery for which this provider is relevant
-     * @param unicastHostProvider the host provider
-     */
-    public void addUnicastHostProvider(String type, Class<? extends UnicastHostsProvider> unicastHostProvider) {
-        List<Class<? extends UnicastHostsProvider>> providerList = unicastHostProviders.get(type);
-        if (providerList == null) {
-            providerList = new ArrayList<>();
-            unicastHostProviders.put(type, providerList);
+        Map<String, Supplier<UnicastHostsProvider>> hostProviders = new HashMap<>();
+        for (DiscoveryPlugin plugin : plugins) {
+            plugin.getZenHostsProviders(transportService, networkService).entrySet().forEach(entry -> {
+                if (hostProviders.put(entry.getKey(), entry.getValue()) != null) {
+                    throw new IllegalArgumentException("Cannot register zen hosts provider [" + entry.getKey() + "] twice");
+                }
+            });
         }
-        providerList.add(unicastHostProvider);
-    }
-
-    /**
-     * Adds a custom Discovery type.
-     */
-    public void addDiscoveryType(String type, Class<? extends Discovery> clazz) {
-        if (discoveryTypes.containsKey(type)) {
-            throw new IllegalArgumentException("discovery type [" + type + "] is already registered");
+        Optional<String> hostsProviderName = DISCOVERY_HOSTS_PROVIDER_SETTING.get(settings);
+        if (hostsProviderName.isPresent()) {
+            Supplier<UnicastHostsProvider> hostsProviderSupplier = hostProviders.get(hostsProviderName.get());
+            if (hostsProviderSupplier == null) {
+                throw new IllegalArgumentException("Unknown zen hosts provider [" + hostsProviderName.get() + "]");
+            }
+            hostsProvider = Objects.requireNonNull(hostsProviderSupplier.get());
+        } else {
+            hostsProvider = Collections::emptyList;
         }
-        discoveryTypes.put(type, clazz);
-    }
 
-    /**
-     * Adds a custom zen master service type.
-     */
-    public void addElectMasterService(String type, Class<? extends ElectMasterService> masterService) {
-        if (masterServiceType.containsKey(type)) {
-            throw new IllegalArgumentException("master service type [" + type + "] is already registered");
+        Map<String, Supplier<Discovery>> discoveryTypes = new HashMap<>();
+        discoveryTypes.put("zen",
+            () -> new ZenDiscovery(settings, threadPool, transportService, namedWriteableRegistry, clusterService, hostsProvider));
+        discoveryTypes.put("none", () -> new NoneDiscovery(settings, clusterService, clusterService.getClusterSettings()));
+        discoveryTypes.put("single-node", () -> new SingleNodeDiscovery(settings, clusterService));
+        for (DiscoveryPlugin plugin : plugins) {
+            plugin.getDiscoveryTypes(threadPool, transportService, namedWriteableRegistry,
+                clusterService, hostsProvider).entrySet().forEach(entry -> {
+                if (discoveryTypes.put(entry.getKey(), entry.getValue()) != null) {
+                    throw new IllegalArgumentException("Cannot register discovery type [" + entry.getKey() + "] twice");
+                }
+            });
         }
-        this.masterServiceType.put(type, masterService);
-    }
-
-    public void addZenPing(Class<? extends ZenPing> clazz) {
-        zenPings.registerExtension(clazz);
-    }
-
-    @Override
-    protected void configure() {
         String discoveryType = DISCOVERY_TYPE_SETTING.get(settings);
-        Class<? extends Discovery> discoveryClass = discoveryTypes.get(discoveryType);
-        if (discoveryClass == null) {
-            throw new IllegalArgumentException("Unknown Discovery type [" + discoveryType + "]");
+        Supplier<Discovery> discoverySupplier = discoveryTypes.get(discoveryType);
+        if (discoverySupplier == null) {
+            throw new IllegalArgumentException("Unknown discovery type [" + discoveryType + "]");
         }
+        Loggers.getLogger(getClass(), settings).info("using discovery type [{}]", discoveryType);
+        discovery = Objects.requireNonNull(discoverySupplier.get());
+    }
 
-        if (discoveryType.equals("local") == false) {
-            String masterServiceTypeKey = ZEN_MASTER_SERVICE_TYPE_SETTING.get(settings);
-            final Class<? extends ElectMasterService> masterService = masterServiceType.get(masterServiceTypeKey);
-            if (masterService == null) {
-                throw new IllegalArgumentException("Unknown master service type [" + masterServiceTypeKey + "]");
-            }
-            if (masterService == ElectMasterService.class) {
-                bind(ElectMasterService.class).asEagerSingleton();
-            } else {
-                bind(ElectMasterService.class).to(masterService).asEagerSingleton();
-            }
-            bind(ZenPingService.class).asEagerSingleton();
-            Multibinder<UnicastHostsProvider> unicastHostsProviderMultibinder = Multibinder.newSetBinder(binder(), UnicastHostsProvider.class);
-            for (Class<? extends UnicastHostsProvider> unicastHostProvider :
-                    unicastHostProviders.getOrDefault(discoveryType, Collections.emptyList())) {
-                unicastHostsProviderMultibinder.addBinding().to(unicastHostProvider);
-            }
-            zenPings.bind(binder());
-        }
-        bind(Discovery.class).to(discoveryClass).asEagerSingleton();
+    public Discovery getDiscovery() {
+        return discovery;
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/discovery/DiscoverySettings.java b/core/src/main/java/org/elasticsearch/discovery/DiscoverySettings.java
index 6f5a6c9a745a8..e9a83678f8a2c 100644
--- a/core/src/main/java/org/elasticsearch/discovery/DiscoverySettings.java
+++ b/core/src/main/java/org/elasticsearch/discovery/DiscoverySettings.java
@@ -37,8 +37,8 @@
 public class DiscoverySettings extends AbstractComponent {
 
     public static final int NO_MASTER_BLOCK_ID = 2;
-    public static final ClusterBlock NO_MASTER_BLOCK_ALL = new ClusterBlock(NO_MASTER_BLOCK_ID, "no master", true, true, RestStatus.SERVICE_UNAVAILABLE, ClusterBlockLevel.ALL);
-    public static final ClusterBlock NO_MASTER_BLOCK_WRITES = new ClusterBlock(NO_MASTER_BLOCK_ID, "no master", true, false, RestStatus.SERVICE_UNAVAILABLE, EnumSet.of(ClusterBlockLevel.WRITE, ClusterBlockLevel.METADATA_WRITE));
+    public static final ClusterBlock NO_MASTER_BLOCK_ALL = new ClusterBlock(NO_MASTER_BLOCK_ID, "no master", true, true, false, RestStatus.SERVICE_UNAVAILABLE, ClusterBlockLevel.ALL);
+    public static final ClusterBlock NO_MASTER_BLOCK_WRITES = new ClusterBlock(NO_MASTER_BLOCK_ID, "no master", true, false, false, RestStatus.SERVICE_UNAVAILABLE, EnumSet.of(ClusterBlockLevel.WRITE, ClusterBlockLevel.METADATA_WRITE));
     /**
      * sets the timeout for a complete publishing cycle, including both sending and committing. the master
      * will continue to process the next cluster state update after this time has elapsed
diff --git a/core/src/main/java/org/elasticsearch/discovery/DiscoveryStats.java b/core/src/main/java/org/elasticsearch/discovery/DiscoveryStats.java
index fc419ff06a672..9542b14e5699c 100644
--- a/core/src/main/java/org/elasticsearch/discovery/DiscoveryStats.java
+++ b/core/src/main/java/org/elasticsearch/discovery/DiscoveryStats.java
@@ -25,7 +25,7 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.discovery.zen.publish.PendingClusterStateStats;
+import org.elasticsearch.discovery.zen.PendingClusterStateStats;
 
 import java.io.IOException;
 
diff --git a/core/src/main/java/org/elasticsearch/discovery/InitialStateDiscoveryListener.java b/core/src/main/java/org/elasticsearch/discovery/InitialStateDiscoveryListener.java
deleted file mode 100644
index 1ec55c874b4c6..0000000000000
--- a/core/src/main/java/org/elasticsearch/discovery/InitialStateDiscoveryListener.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.discovery;
-
-/**
- * A listener that should be called by the {@link org.elasticsearch.discovery.Discovery} component
- * when the first valid initial cluster state has been submitted and processed by the cluster service.
- * <p>
- * Note, this listener should be registered with the discovery service before it has started.
- *
- *
- */
-public interface InitialStateDiscoveryListener {
-
-    void initialStateProcessed();
-}
diff --git a/core/src/main/java/org/elasticsearch/discovery/NoneDiscovery.java b/core/src/main/java/org/elasticsearch/discovery/NoneDiscovery.java
new file mode 100644
index 0000000000000..91b04ce396ba6
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/discovery/NoneDiscovery.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.discovery;
+
+import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.component.AbstractLifecycleComponent;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.discovery.zen.ElectMasterService;
+
+/**
+ * A {@link Discovery} implementation that is used by {@link org.elasticsearch.tribe.TribeService}. This implementation
+ * doesn't support any clustering features. Most notably {@link #startInitialJoin()} does nothing and
+ * {@link #publish(ClusterChangedEvent, AckListener)} is not supported.
+ */
+public class NoneDiscovery extends AbstractLifecycleComponent implements Discovery {
+
+    private final ClusterService clusterService;
+    private final DiscoverySettings discoverySettings;
+
+    @Inject
+    public NoneDiscovery(Settings settings, ClusterService clusterService, ClusterSettings clusterSettings) {
+        super(settings);
+        this.clusterService = clusterService;
+        this.discoverySettings = new DiscoverySettings(settings, clusterSettings);
+    }
+
+    @Override
+    public DiscoveryNode localNode() {
+        return clusterService.localNode();
+    }
+
+    @Override
+    public String nodeDescription() {
+        return clusterService.getClusterName().value() + "/" + clusterService.localNode().getId();
+    }
+
+    @Override
+    public void setAllocationService(AllocationService allocationService) {
+
+    }
+
+    @Override
+    public void publish(ClusterChangedEvent clusterChangedEvent, AckListener ackListener) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public DiscoveryStats stats() {
+        return null;
+    }
+
+    @Override
+    public DiscoverySettings getDiscoverySettings() {
+        return discoverySettings;
+    }
+
+    @Override
+    public void startInitialJoin() {
+
+    }
+
+    @Override
+    public int getMinimumMasterNodes() {
+        return ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.get(settings);
+    }
+
+    @Override
+    protected void doStart() {
+
+    }
+
+    @Override
+    protected void doStop() {
+
+    }
+
+    @Override
+    protected void doClose() {
+
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java b/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java
index d84f471cffeeb..6bab1a2ab128e 100644
--- a/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java
+++ b/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java
@@ -27,16 +27,17 @@
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.cluster.IncompatibleClusterStateVersionException;
+import org.elasticsearch.cluster.LocalClusterUpdateTask;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
-import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
@@ -47,7 +48,7 @@
 import org.elasticsearch.discovery.Discovery;
 import org.elasticsearch.discovery.DiscoverySettings;
 import org.elasticsearch.discovery.DiscoveryStats;
-import org.elasticsearch.discovery.zen.publish.PendingClusterStateStats;
+import org.elasticsearch.discovery.zen.PendingClusterStateStats;
 
 import java.util.HashSet;
 import java.util.Optional;
@@ -69,6 +70,7 @@ public class LocalDiscovery extends AbstractLifecycleComponent implements Discov
     private final ClusterName clusterName;
 
     private final DiscoverySettings discoverySettings;
+    private final NamedWriteableRegistry namedWriteableRegistry;
 
     private volatile boolean master = false;
 
@@ -77,11 +79,12 @@ public class LocalDiscovery extends AbstractLifecycleComponent implements Discov
     private volatile ClusterState lastProcessedClusterState;
 
     @Inject
-    public LocalDiscovery(Settings settings, ClusterService clusterService, ClusterSettings clusterSettings) {
+    public LocalDiscovery(Settings settings, ClusterService clusterService, ClusterSettings clusterSettings, NamedWriteableRegistry namedWriteableRegistry) {
         super(settings);
         this.clusterName = clusterService.getClusterName();
         this.clusterService = clusterService;
         this.discoverySettings = new DiscoverySettings(settings, clusterSettings);
+        this.namedWriteableRegistry = namedWriteableRegistry;
     }
 
     @Override
@@ -126,23 +129,16 @@ public void startInitialJoin() {
                 // we are the first master (and the master)
                 master = true;
                 final LocalDiscovery master = firstMaster;
-                clusterService.submitStateUpdateTask("local-disco-initial_connect(master)", new ClusterStateUpdateTask() {
+                clusterService.submitStateUpdateTask("local-disco-initial_connect(master)", new LocalClusterUpdateTask() {
 
                     @Override
-                    public boolean runOnlyOnMaster() {
-                        return false;
-                    }
-
-                    @Override
-                    public ClusterState execute(ClusterState currentState) {
+                    public ClusterTasksResult<LocalClusterUpdateTask> execute(ClusterState currentState) {
                         DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder();
                         for (LocalDiscovery discovery : clusterGroups.get(clusterName).members()) {
                             nodesBuilder.add(discovery.localNode());
                         }
                         nodesBuilder.localNodeId(master.localNode().getId()).masterNodeId(master.localNode().getId());
-                        // remove the NO_MASTER block in this case
-                        ClusterBlocks.Builder blocks = ClusterBlocks.builder().blocks(currentState.blocks()).removeGlobalBlock(discoverySettings.getNoMasterBlock());
-                        return ClusterState.builder(currentState).nodes(nodesBuilder).blocks(blocks).build();
+                        return newState(ClusterState.builder(currentState).nodes(nodesBuilder).build());
                     }
 
                     @Override
@@ -153,25 +149,16 @@ public void onFailure(String source, Exception e) {
             } else if (firstMaster != null) {
                 // tell the master to send the fact that we are here
                 final LocalDiscovery master = firstMaster;
-                firstMaster.clusterService.submitStateUpdateTask("local-disco-receive(from node[" + localNode() + "])", new ClusterStateUpdateTask() {
+                firstMaster.clusterService.submitStateUpdateTask("local-disco-receive(from node[" + localNode() + "])", new LocalClusterUpdateTask() {
                     @Override
-                    public boolean runOnlyOnMaster() {
-                        return false;
-                    }
-
-                    @Override
-                    public ClusterState execute(ClusterState currentState) {
+                    public ClusterTasksResult<LocalClusterUpdateTask> execute(ClusterState currentState) {
                         DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder();
                         for (LocalDiscovery discovery : clusterGroups.get(clusterName).members()) {
                             nodesBuilder.add(discovery.localNode());
                         }
                         nodesBuilder.localNodeId(master.localNode().getId()).masterNodeId(master.localNode().getId());
                         currentState = ClusterState.builder(currentState).nodes(nodesBuilder).build();
-                        RoutingAllocation.Result result =  master.allocationService.reroute(currentState, "node_add");
-                        if (result.changed()) {
-                            currentState = ClusterState.builder(currentState).routingResult(result).build();
-                        }
-                        return currentState;
+                        return newState(master.allocationService.reroute(currentState, "node_add"));
                     }
 
                     @Override
@@ -219,14 +206,9 @@ protected void doStop() {
                 }
 
                 final LocalDiscovery master = firstMaster;
-                master.clusterService.submitStateUpdateTask("local-disco-update", new ClusterStateUpdateTask() {
+                master.clusterService.submitStateUpdateTask("local-disco-update", new LocalClusterUpdateTask() {
                     @Override
-                    public boolean runOnlyOnMaster() {
-                        return false;
-                    }
-
-                    @Override
-                    public ClusterState execute(ClusterState currentState) {
+                    public ClusterTasksResult<LocalClusterUpdateTask> execute(ClusterState currentState) {
                         DiscoveryNodes newNodes = currentState.nodes().removeDeadMembers(newMembers, master.localNode().getId());
                         DiscoveryNodes.Delta delta = newNodes.delta(currentState.nodes());
                         if (delta.added()) {
@@ -234,9 +216,7 @@ public ClusterState execute(ClusterState currentState) {
                         }
                         // reroute here, so we eagerly remove dead nodes from the routing
                         ClusterState updatedState = ClusterState.builder(currentState).nodes(newNodes).build();
-                        RoutingAllocation.Result routingResult = master.allocationService.deassociateDeadNodes(
-                                ClusterState.builder(updatedState).build(), true, "node stopped");
-                        return ClusterState.builder(updatedState).routingResult(routingResult).build();
+                        return newState(master.allocationService.deassociateDeadNodes(updatedState, true, "node stopped"));
                     }
 
                     @Override
@@ -329,7 +309,7 @@ private void publish(LocalDiscovery[] members, ClusterChangedEvent clusterChange
                             clusterStateDiffBytes = BytesReference.toBytes(os.bytes());
                         }
                         try {
-                            newNodeSpecificClusterState = discovery.lastProcessedClusterState.readDiffFrom(StreamInput.wrap(clusterStateDiffBytes)).apply(discovery.lastProcessedClusterState);
+                            newNodeSpecificClusterState = ClusterState.readDiffFrom(StreamInput.wrap(clusterStateDiffBytes), discovery.localNode()).apply(discovery.lastProcessedClusterState);
                             logger.trace("sending diff cluster state version [{}] with size {} to [{}]", clusterState.version(), clusterStateDiffBytes.length, discovery.localNode().getName());
                         } catch (IncompatibleClusterStateVersionException ex) {
                             logger.warn((Supplier<?>) () -> new ParameterizedMessage("incompatible cluster state version [{}] - resending complete cluster state", clusterState.version()), ex);
@@ -339,34 +319,28 @@ private void publish(LocalDiscovery[] members, ClusterChangedEvent clusterChange
                         if (clusterStateBytes == null) {
                             clusterStateBytes = Builder.toBytes(clusterState);
                         }
-                        newNodeSpecificClusterState = ClusterState.Builder.fromBytes(clusterStateBytes, discovery.localNode());
+                        newNodeSpecificClusterState = ClusterState.Builder.fromBytes(clusterStateBytes, discovery.localNode(), namedWriteableRegistry);
                     }
                     discovery.lastProcessedClusterState = newNodeSpecificClusterState;
                 }
                 final ClusterState nodeSpecificClusterState = newNodeSpecificClusterState;
 
-                nodeSpecificClusterState.status(ClusterState.ClusterStateStatus.RECEIVED);
                 // ignore cluster state messages that do not include "me", not in the game yet...
                 if (nodeSpecificClusterState.nodes().getLocalNode() != null) {
                     assert nodeSpecificClusterState.nodes().getMasterNode() != null : "received a cluster state without a master";
                     assert !nodeSpecificClusterState.blocks().hasGlobalBlock(discoverySettings.getNoMasterBlock()) : "received a cluster state with a master block";
 
-                    discovery.clusterService.submitStateUpdateTask("local-disco-receive(from master)", new ClusterStateUpdateTask() {
-                        @Override
-                        public boolean runOnlyOnMaster() {
-                            return false;
-                        }
-
+                    discovery.clusterService.submitStateUpdateTask("local-disco-receive(from master)", new LocalClusterUpdateTask() {
                         @Override
-                        public ClusterState execute(ClusterState currentState) {
+                        public ClusterTasksResult<LocalClusterUpdateTask> execute(ClusterState currentState) {
                             if (currentState.supersedes(nodeSpecificClusterState)) {
-                                return currentState;
+                                return unchanged();
                             }
 
                             if (currentState.blocks().hasGlobalBlock(discoverySettings.getNoMasterBlock())) {
                                 // its a fresh update from the master as we transition from a start of not having a master to having one
                                 logger.debug("got first state from fresh master [{}]", nodeSpecificClusterState.nodes().getMasterNodeId());
-                                return nodeSpecificClusterState;
+                                return newState(nodeSpecificClusterState);
                             }
 
                             ClusterState.Builder builder = ClusterState.builder(nodeSpecificClusterState);
@@ -378,7 +352,7 @@ public ClusterState execute(ClusterState currentState) {
                                 builder.metaData(currentState.metaData());
                             }
 
-                            return builder.build();
+                            return newState(builder.build());
                         }
 
                         @Override
diff --git a/core/src/main/java/org/elasticsearch/discovery/single/SingleNodeDiscovery.java b/core/src/main/java/org/elasticsearch/discovery/single/SingleNodeDiscovery.java
new file mode 100644
index 0000000000000..f4735c8bf3a0d
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/discovery/single/SingleNodeDiscovery.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.discovery.single;
+
+import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateTaskConfig;
+import org.elasticsearch.cluster.ClusterStateTaskExecutor;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Priority;
+import org.elasticsearch.common.component.AbstractLifecycleComponent;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.discovery.Discovery;
+import org.elasticsearch.discovery.DiscoverySettings;
+import org.elasticsearch.discovery.DiscoveryStats;
+import org.elasticsearch.discovery.zen.PendingClusterStateStats;
+import org.elasticsearch.discovery.zen.PendingClusterStatesQueue;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * A discovery implementation where the only member of the cluster is the local node.
+ */
+public class SingleNodeDiscovery extends AbstractLifecycleComponent implements Discovery {
+
+    private final ClusterService clusterService;
+    private final DiscoverySettings discoverySettings;
+
+    public SingleNodeDiscovery(final Settings settings, final ClusterService clusterService) {
+        super(Objects.requireNonNull(settings));
+        this.clusterService = Objects.requireNonNull(clusterService);
+        final ClusterSettings clusterSettings =
+                Objects.requireNonNull(clusterService.getClusterSettings());
+        this.discoverySettings = new DiscoverySettings(settings, clusterSettings);
+    }
+
+    @Override
+    public DiscoveryNode localNode() {
+        return clusterService.localNode();
+    }
+
+    @Override
+    public String nodeDescription() {
+        return clusterService.getClusterName().value() + "/" + clusterService.localNode().getId();
+    }
+
+    @Override
+    public void setAllocationService(final AllocationService allocationService) {
+
+    }
+
+    @Override
+    public void publish(final ClusterChangedEvent event, final AckListener listener) {
+
+    }
+
+    @Override
+    public DiscoveryStats stats() {
+        return new DiscoveryStats((PendingClusterStateStats) null);
+    }
+
+    @Override
+    public DiscoverySettings getDiscoverySettings() {
+        return discoverySettings;
+    }
+
+    @Override
+    public void startInitialJoin() {
+        final ClusterStateTaskExecutor<DiscoveryNode> executor =
+                new ClusterStateTaskExecutor<DiscoveryNode>() {
+
+                    @Override
+                    public ClusterTasksResult<DiscoveryNode> execute(
+                            final ClusterState current,
+                            final List<DiscoveryNode> tasks) throws Exception {
+                        assert tasks.size() == 1;
+                        final DiscoveryNodes.Builder nodes =
+                                DiscoveryNodes.builder(current.nodes());
+                        // always set the local node as master, there will not be other nodes
+                        nodes.masterNodeId(localNode().getId());
+                        final ClusterState next =
+                                ClusterState.builder(current).nodes(nodes).build();
+                        final ClusterTasksResult.Builder<DiscoveryNode> result =
+                                ClusterTasksResult.builder();
+                        return result.successes(tasks).build(next);
+                    }
+
+                    @Override
+                    public boolean runOnlyOnMaster() {
+                        return false;
+                    }
+
+                };
+        final ClusterStateTaskConfig config = ClusterStateTaskConfig.build(Priority.URGENT);
+        clusterService.submitStateUpdateTasks(
+                "single-node-start-initial-join",
+                Collections.singletonMap(localNode(), (s, e) -> {}), config, executor);
+    }
+
+    @Override
+    public int getMinimumMasterNodes() {
+        return 1;
+    }
+
+    @Override
+    protected void doStart() {
+
+    }
+
+    @Override
+    protected void doStop() {
+
+    }
+
+    @Override
+    protected void doClose() throws IOException {
+
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/DiscoveryNodesProvider.java b/core/src/main/java/org/elasticsearch/discovery/zen/DiscoveryNodesProvider.java
deleted file mode 100644
index b9ce79013696d..0000000000000
--- a/core/src/main/java/org/elasticsearch/discovery/zen/DiscoveryNodesProvider.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.discovery.zen;
-
-import org.elasticsearch.cluster.node.DiscoveryNodes;
-
-/**
- *
- */
-public interface DiscoveryNodesProvider {
-
-    DiscoveryNodes nodes();
-
-}
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/ElectMasterService.java b/core/src/main/java/org/elasticsearch/discovery/zen/ElectMasterService.java
new file mode 100644
index 0000000000000..248b0e35e3dd2
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/ElectMasterService.java
@@ -0,0 +1,230 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.discovery.zen;
+
+import com.carrotsearch.hppc.ObjectContainer;
+import org.apache.lucene.util.CollectionUtil;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.component.AbstractComponent;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.CollectionUtils;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+/**
+ *
+ */
+public class ElectMasterService extends AbstractComponent {
+
+    public static final Setting<Integer> DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING =
+        Setting.intSetting("discovery.zen.minimum_master_nodes", -1, Property.Dynamic, Property.NodeScope);
+
+    private volatile int minimumMasterNodes;
+
+    /**
+     * a class to encapsulate all the information about a candidate in a master election
+     * that is needed to decided which of the candidates should win
+     */
+    public static class MasterCandidate {
+
+        public static final long UNRECOVERED_CLUSTER_VERSION = -1;
+
+        final DiscoveryNode node;
+
+        final long clusterStateVersion;
+
+        public MasterCandidate(DiscoveryNode node, long clusterStateVersion) {
+            Objects.requireNonNull(node);
+            assert clusterStateVersion >= -1 : "got: " + clusterStateVersion;
+            assert node.isMasterNode();
+            this.node = node;
+            this.clusterStateVersion = clusterStateVersion;
+        }
+
+        public DiscoveryNode getNode() {
+            return node;
+        }
+
+        public long getClusterStateVersion() {
+            return clusterStateVersion;
+        }
+
+        @Override
+        public String toString() {
+            return "Candidate{" +
+                "node=" + node +
+                ", clusterStateVersion=" + clusterStateVersion +
+                '}';
+        }
+
+        /**
+         * compares two candidates to indicate which the a better master.
+         * A higher cluster state version is better
+         *
+         * @return -1 if c1 is a batter candidate, 1 if c2.
+         */
+        public static int compare(MasterCandidate c1, MasterCandidate c2) {
+            // we explicitly swap c1 and c2 here. the code expects "better" is lower in a sorted
+            // list, so if c2 has a higher cluster state version, it needs to come first.
+            int ret = Long.compare(c2.clusterStateVersion, c1.clusterStateVersion);
+            if (ret == 0) {
+                ret = compareNodes(c1.getNode(), c2.getNode());
+            }
+            return ret;
+        }
+    }
+
+    public ElectMasterService(Settings settings) {
+        super(settings);
+        this.minimumMasterNodes = DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.get(settings);
+        logger.debug("using minimum_master_nodes [{}]", minimumMasterNodes);
+    }
+
+    public void minimumMasterNodes(int minimumMasterNodes) {
+        this.minimumMasterNodes = minimumMasterNodes;
+    }
+
+    public int minimumMasterNodes() {
+        return minimumMasterNodes;
+    }
+
+    public int countMasterNodes(Iterable<DiscoveryNode> nodes) {
+        int count = 0;
+        for (DiscoveryNode node : nodes) {
+            if (node.isMasterNode()) {
+                count++;
+            }
+        }
+        return count;
+    }
+
+    public boolean hasEnoughCandidates(Collection<MasterCandidate> candidates) {
+        if (candidates.isEmpty()) {
+            return false;
+        }
+        if (minimumMasterNodes < 1) {
+            return true;
+        }
+        assert candidates.stream().map(MasterCandidate::getNode).collect(Collectors.toSet()).size() == candidates.size() :
+            "duplicates ahead: " + candidates;
+        return candidates.size() >= minimumMasterNodes;
+    }
+
+    /**
+     * Elects a new master out of the possible nodes, returning it. Returns <tt>null</tt>
+     * if no master has been elected.
+     */
+    public MasterCandidate electMaster(Collection<MasterCandidate> candidates) {
+        assert hasEnoughCandidates(candidates);
+        List<MasterCandidate> sortedCandidates = new ArrayList<>(candidates);
+        sortedCandidates.sort(MasterCandidate::compare);
+        return sortedCandidates.get(0);
+    }
+
+    /** selects the best active master to join, where multiple are discovered */
+    public DiscoveryNode tieBreakActiveMasters(Collection<DiscoveryNode> activeMasters) {
+        return activeMasters.stream().min(ElectMasterService::compareNodes).get();
+    }
+
+    public boolean hasEnoughMasterNodes(Iterable<DiscoveryNode> nodes) {
+        final int count = countMasterNodes(nodes);
+        return count > 0 && (minimumMasterNodes < 0 || count >= minimumMasterNodes);
+    }
+
+    public boolean hasTooManyMasterNodes(Iterable<DiscoveryNode> nodes) {
+        final int count = countMasterNodes(nodes);
+        return count > 1 && minimumMasterNodes <= count / 2;
+    }
+
+    public void logMinimumMasterNodesWarningIfNecessary(ClusterState oldState, ClusterState newState) {
+        // check if min_master_nodes setting is too low and log warning
+        if (hasTooManyMasterNodes(oldState.nodes()) == false && hasTooManyMasterNodes(newState.nodes())) {
+            logger.warn("value for setting \"{}\" is too low. This can result in data loss! Please set it to at least a quorum of master-" +
+                    "eligible nodes (current value: [{}], total number of master-eligible nodes used for publishing in this round: [{}])",
+                ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), minimumMasterNodes(),
+                newState.getNodes().getMasterNodes().size());
+        }
+    }
+
+    /**
+     * Returns the given nodes sorted by likelihood of being elected as master, most likely first.
+     * Non-master nodes are not removed but are rather put in the end
+     */
+    static List<DiscoveryNode> sortByMasterLikelihood(Iterable<DiscoveryNode> nodes) {
+        ArrayList<DiscoveryNode> sortedNodes = CollectionUtils.iterableAsArrayList(nodes);
+        CollectionUtil.introSort(sortedNodes, ElectMasterService::compareNodes);
+        return sortedNodes;
+    }
+
+    /**
+     * Returns a list of the next possible masters.
+     */
+    public DiscoveryNode[] nextPossibleMasters(ObjectContainer<DiscoveryNode> nodes, int numberOfPossibleMasters) {
+        List<DiscoveryNode> sortedNodes = sortedMasterNodes(Arrays.asList(nodes.toArray(DiscoveryNode.class)));
+        if (sortedNodes == null) {
+            return new DiscoveryNode[0];
+        }
+        List<DiscoveryNode> nextPossibleMasters = new ArrayList<>(numberOfPossibleMasters);
+        int counter = 0;
+        for (DiscoveryNode nextPossibleMaster : sortedNodes) {
+            if (++counter >= numberOfPossibleMasters) {
+                break;
+            }
+            nextPossibleMasters.add(nextPossibleMaster);
+        }
+        return nextPossibleMasters.toArray(new DiscoveryNode[nextPossibleMasters.size()]);
+    }
+
+    private List<DiscoveryNode> sortedMasterNodes(Iterable<DiscoveryNode> nodes) {
+        List<DiscoveryNode> possibleNodes = CollectionUtils.iterableAsArrayList(nodes);
+        if (possibleNodes.isEmpty()) {
+            return null;
+        }
+        // clean non master nodes
+        for (Iterator<DiscoveryNode> it = possibleNodes.iterator(); it.hasNext(); ) {
+            DiscoveryNode node = it.next();
+            if (!node.isMasterNode()) {
+                it.remove();
+            }
+        }
+        CollectionUtil.introSort(possibleNodes, ElectMasterService::compareNodes);
+        return possibleNodes;
+    }
+
+    /** master nodes go before other nodes, with a secondary sort by id **/
+     private static int compareNodes(DiscoveryNode o1, DiscoveryNode o2) {
+        if (o1.isMasterNode() && !o2.isMasterNode()) {
+            return -1;
+        }
+        if (!o1.isMasterNode() && o2.isMasterNode()) {
+            return 1;
+        }
+        return o1.getId().compareTo(o2.getId());
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/fd/FaultDetection.java b/core/src/main/java/org/elasticsearch/discovery/zen/FaultDetection.java
similarity index 94%
rename from core/src/main/java/org/elasticsearch/discovery/zen/fd/FaultDetection.java
rename to core/src/main/java/org/elasticsearch/discovery/zen/FaultDetection.java
index 1cfd46634a52f..715e8be03efb2 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/fd/FaultDetection.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/FaultDetection.java
@@ -16,7 +16,10 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.discovery.zen.fd;
+
+package org.elasticsearch.discovery.zen;
+
+import java.io.Closeable;
 
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -32,10 +35,10 @@
 import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
 
 /**
- * A base class for {@link org.elasticsearch.discovery.zen.fd.MasterFaultDetection} &amp; {@link org.elasticsearch.discovery.zen.fd.NodesFaultDetection},
+ * A base class for {@link MasterFaultDetection} &amp; {@link NodesFaultDetection},
  * making sure both use the same setting.
  */
-public abstract class FaultDetection extends AbstractComponent {
+public abstract class FaultDetection extends AbstractComponent implements Closeable {
 
     public static final Setting<Boolean> CONNECT_ON_NETWORK_DISCONNECT_SETTING =
         Setting.boolSetting("discovery.zen.fd.connect_on_network_disconnect", false, Property.NodeScope);
@@ -79,6 +82,7 @@ public FaultDetection(Settings settings, ThreadPool threadPool, TransportService
         }
     }
 
+    @Override
     public void close() {
         transportService.removeConnectionListener(connectionListener);
     }
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java b/core/src/main/java/org/elasticsearch/discovery/zen/MasterFaultDetection.java
similarity index 96%
rename from core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java
rename to core/src/main/java/org/elasticsearch/discovery/zen/MasterFaultDetection.java
index 6dc89998046c3..81d4c19d33ee2 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/MasterFaultDetection.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.discovery.zen.fd;
+package org.elasticsearch.discovery.zen;
 
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
@@ -111,28 +111,10 @@ public void restart(DiscoveryNode masterNode, String reason) {
         }
     }
 
-    public void start(final DiscoveryNode masterNode, String reason) {
-        synchronized (masterNodeMutex) {
-            if (logger.isDebugEnabled()) {
-                logger.debug("[master] starting fault detection against master [{}], reason [{}]", masterNode, reason);
-            }
-            innerStart(masterNode);
-        }
-    }
-
     private void innerStart(final DiscoveryNode masterNode) {
         this.masterNode = masterNode;
         this.retryCount = 0;
         this.notifiedMasterFailure.set(false);
-
-        // try and connect to make sure we are connected
-        try {
-            transportService.connectToNode(masterNode);
-        } catch (final Exception e) {
-            // notify master failure (which stops also) and bail..
-            notifyMasterFailure(masterNode, e, "failed to perform initial connect ");
-            return;
-        }
         if (masterPinger != null) {
             masterPinger.stop();
         }
@@ -168,7 +150,6 @@ public void close() {
         super.close();
         stop("closing");
         this.listeners.clear();
-        transportService.removeHandler(MASTER_PING_ACTION_NAME);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/MembershipAction.java b/core/src/main/java/org/elasticsearch/discovery/zen/MembershipAction.java
new file mode 100644
index 0000000000000..d83d870210410
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/MembershipAction.java
@@ -0,0 +1,267 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.discovery.zen;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.component.AbstractComponent;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.EmptyTransportResponseHandler;
+import org.elasticsearch.transport.TransportChannel;
+import org.elasticsearch.transport.TransportRequest;
+import org.elasticsearch.transport.TransportRequestHandler;
+import org.elasticsearch.transport.TransportResponse;
+import org.elasticsearch.transport.TransportService;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+/**
+ *
+ */
+public class MembershipAction extends AbstractComponent {
+
+    public static final String DISCOVERY_JOIN_ACTION_NAME = "internal:discovery/zen/join";
+    public static final String DISCOVERY_JOIN_VALIDATE_ACTION_NAME = "internal:discovery/zen/join/validate";
+    public static final String DISCOVERY_LEAVE_ACTION_NAME = "internal:discovery/zen/leave";
+
+    public interface JoinCallback {
+        void onSuccess();
+
+        void onFailure(Exception e);
+    }
+
+    public interface MembershipListener {
+        void onJoin(DiscoveryNode node, JoinCallback callback);
+
+        void onLeave(DiscoveryNode node);
+    }
+
+    private final TransportService transportService;
+
+    private final MembershipListener listener;
+
+    public MembershipAction(Settings settings, TransportService transportService, MembershipListener listener) {
+        super(settings);
+        this.transportService = transportService;
+        this.listener = listener;
+
+
+        transportService.registerRequestHandler(DISCOVERY_JOIN_ACTION_NAME, JoinRequest::new,
+            ThreadPool.Names.GENERIC, new JoinRequestRequestHandler());
+        transportService.registerRequestHandler(DISCOVERY_JOIN_VALIDATE_ACTION_NAME,
+            () -> new ValidateJoinRequest(), ThreadPool.Names.GENERIC,
+            new ValidateJoinRequestRequestHandler());
+        transportService.registerRequestHandler(DISCOVERY_LEAVE_ACTION_NAME, LeaveRequest::new,
+            ThreadPool.Names.GENERIC, new LeaveRequestRequestHandler());
+    }
+
+    public void sendLeaveRequest(DiscoveryNode masterNode, DiscoveryNode node) {
+        transportService.sendRequest(node, DISCOVERY_LEAVE_ACTION_NAME, new LeaveRequest(masterNode),
+            EmptyTransportResponseHandler.INSTANCE_SAME);
+    }
+
+    public void sendLeaveRequestBlocking(DiscoveryNode masterNode, DiscoveryNode node, TimeValue timeout) {
+        transportService.submitRequest(masterNode, DISCOVERY_LEAVE_ACTION_NAME, new LeaveRequest(node),
+            EmptyTransportResponseHandler.INSTANCE_SAME).txGet(timeout.millis(), TimeUnit.MILLISECONDS);
+    }
+
+    public void sendJoinRequestBlocking(DiscoveryNode masterNode, DiscoveryNode node, TimeValue timeout) {
+        transportService.submitRequest(masterNode, DISCOVERY_JOIN_ACTION_NAME, new JoinRequest(node),
+            EmptyTransportResponseHandler.INSTANCE_SAME).txGet(timeout.millis(), TimeUnit.MILLISECONDS);
+    }
+
+    /**
+     * Validates the join request, throwing a failure if it failed.
+     */
+    public void sendValidateJoinRequestBlocking(DiscoveryNode node, ClusterState state, TimeValue timeout) {
+        transportService.submitRequest(node, DISCOVERY_JOIN_VALIDATE_ACTION_NAME, new ValidateJoinRequest(state),
+            EmptyTransportResponseHandler.INSTANCE_SAME).txGet(timeout.millis(), TimeUnit.MILLISECONDS);
+    }
+
+    public static class JoinRequest extends TransportRequest {
+
+        DiscoveryNode node;
+
+        public JoinRequest() {
+        }
+
+        private JoinRequest(DiscoveryNode node) {
+            this.node = node;
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            super.readFrom(in);
+            node = new DiscoveryNode(in);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            node.writeTo(out);
+        }
+    }
+
+
+    private class JoinRequestRequestHandler implements TransportRequestHandler<JoinRequest> {
+
+        @Override
+        public void messageReceived(final JoinRequest request, final TransportChannel channel) throws Exception {
+            listener.onJoin(request.node, new JoinCallback() {
+                @Override
+                public void onSuccess() {
+                    try {
+                        channel.sendResponse(TransportResponse.Empty.INSTANCE);
+                    } catch (Exception e) {
+                        onFailure(e);
+                    }
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    try {
+                        channel.sendResponse(e);
+                    } catch (Exception inner) {
+                        inner.addSuppressed(e);
+                        logger.warn("failed to send back failure on join request", inner);
+                    }
+                }
+            });
+        }
+    }
+
+    static class ValidateJoinRequest extends TransportRequest {
+        private ClusterState state;
+
+        ValidateJoinRequest() {}
+
+        ValidateJoinRequest(ClusterState state) {
+            this.state = state;
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            super.readFrom(in);
+            this.state = ClusterState.readFrom(in, null);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            this.state.writeTo(out);
+        }
+    }
+
+    static class ValidateJoinRequestRequestHandler implements TransportRequestHandler<ValidateJoinRequest> {
+
+        @Override
+        public void messageReceived(ValidateJoinRequest request, TransportChannel channel) throws Exception {
+            ensureNodesCompatibility(Version.CURRENT, request.state.getNodes());
+            ensureIndexCompatibility(Version.CURRENT, request.state.getMetaData());
+            // for now, the mere fact that we can serialize the cluster state acts as validation....
+            channel.sendResponse(TransportResponse.Empty.INSTANCE);
+        }
+    }
+
+    /**
+     * Ensures that all indices are compatible with the given node version. This will ensure that all indices in the given metadata
+     * will not be created with a newer version of elasticsearch as well as that all indices are newer or equal to the minimum index
+     * compatibility version.
+     * @see Version#minimumIndexCompatibilityVersion()
+     * @throws IllegalStateException if any index is incompatible with the given version
+     */
+    static void ensureIndexCompatibility(final Version nodeVersion, MetaData metaData) {
+        Version supportedIndexVersion = nodeVersion.minimumIndexCompatibilityVersion();
+        // we ensure that all indices in the cluster we join are compatible with us no matter if they are
+        // closed or not we can't read mappings of these indices so we need to reject the join...
+        for (IndexMetaData idxMetaData : metaData) {
+            if (idxMetaData.getCreationVersion().after(nodeVersion)) {
+                throw new IllegalStateException("index " + idxMetaData.getIndex() + " version not supported: "
+                    + idxMetaData.getCreationVersion() + " the node version is: " + nodeVersion);
+            }
+            if (idxMetaData.getCreationVersion().before(supportedIndexVersion)) {
+                throw new IllegalStateException("index " + idxMetaData.getIndex() + " version not supported: "
+                    + idxMetaData.getCreationVersion() + " minimum compatible index version is: " + supportedIndexVersion);
+            }
+        }
+    }
+
+    /** ensures that the joining node has a version that's compatible with all current nodes*/
+    static void ensureNodesCompatibility(final Version joiningNodeVersion, DiscoveryNodes currentNodes) {
+        final Version minNodeVersion = currentNodes.getMinNodeVersion();
+        final Version maxNodeVersion = currentNodes.getMaxNodeVersion();
+        ensureNodesCompatibility(joiningNodeVersion, minNodeVersion, maxNodeVersion);
+    }
+
+    /** ensures that the joining node has a version that's compatible with a given version range */
+    static void ensureNodesCompatibility(Version joiningNodeVersion, Version minClusterNodeVersion, Version maxClusterNodeVersion) {
+        assert minClusterNodeVersion.onOrBefore(maxClusterNodeVersion) : minClusterNodeVersion + " > " + maxClusterNodeVersion;
+        if (joiningNodeVersion.isCompatible(maxClusterNodeVersion) == false) {
+            throw new IllegalStateException("node version [" + joiningNodeVersion + "] is not supported. " +
+                "The cluster contains nodes with version [" + maxClusterNodeVersion + "], which is incompatible.");
+        }
+        if (joiningNodeVersion.isCompatible(minClusterNodeVersion) == false) {
+            throw new IllegalStateException("node version [" + joiningNodeVersion + "] is not supported." +
+                "The cluster contains nodes with version [" + minClusterNodeVersion + "], which is incompatible.");
+        }
+    }
+
+    public static class LeaveRequest extends TransportRequest {
+
+        private DiscoveryNode node;
+
+        public LeaveRequest() {
+        }
+
+        private LeaveRequest(DiscoveryNode node) {
+            this.node = node;
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            super.readFrom(in);
+            node = new DiscoveryNode(in);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            node.writeTo(out);
+        }
+    }
+
+    private class LeaveRequestRequestHandler implements TransportRequestHandler<LeaveRequest> {
+
+        @Override
+        public void messageReceived(LeaveRequest request, TransportChannel channel) throws Exception {
+            listener.onLeave(request.node);
+            channel.sendResponse(TransportResponse.Empty.INSTANCE);
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java b/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java
index 6f0b8966d0916..7d5132b0a1fff 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java
@@ -29,20 +29,15 @@
 import org.elasticsearch.cluster.ClusterStateTaskExecutor;
 import org.elasticsearch.cluster.ClusterStateTaskListener;
 import org.elasticsearch.cluster.NotMasterException;
-import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
-import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.discovery.DiscoverySettings;
-import org.elasticsearch.discovery.zen.elect.ElectMasterService;
-import org.elasticsearch.discovery.zen.membership.MembershipAction;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -62,7 +57,6 @@ public class NodeJoinController extends AbstractComponent {
     private final ClusterService clusterService;
     private final AllocationService allocationService;
     private final ElectMasterService electMaster;
-    private final DiscoverySettings discoverySettings;
     private final JoinTaskExecutor joinTaskExecutor = new JoinTaskExecutor();
 
     // this is set while trying to become a master
@@ -71,12 +65,11 @@ public class NodeJoinController extends AbstractComponent {
 
 
     public NodeJoinController(ClusterService clusterService, AllocationService allocationService, ElectMasterService electMaster,
-                              DiscoverySettings discoverySettings, Settings settings) {
+                              Settings settings) {
         super(settings);
         this.clusterService = clusterService;
         this.allocationService = allocationService;
         this.electMaster = electMaster;
-        this.discoverySettings = discoverySettings;
     }
 
     /**
@@ -410,8 +403,8 @@ public String toString() {
     class JoinTaskExecutor implements ClusterStateTaskExecutor<DiscoveryNode> {
 
         @Override
-        public BatchResult<DiscoveryNode> execute(ClusterState currentState, List<DiscoveryNode> joiningNodes) throws Exception {
-            final BatchResult.Builder<DiscoveryNode> results = BatchResult.builder();
+        public ClusterTasksResult<DiscoveryNode> execute(ClusterState currentState, List<DiscoveryNode> joiningNodes) throws Exception {
+            final ClusterTasksResult.Builder<DiscoveryNode> results = ClusterTasksResult.builder();
 
             final DiscoveryNodes currentNodes = currentState.nodes();
             boolean nodesChanged = false;
@@ -437,6 +430,9 @@ public BatchResult<DiscoveryNode> execute(ClusterState currentState, List<Discov
 
             assert nodesBuilder.isLocalNodeElectedMaster();
 
+            Version minClusterNodeVersion = newState.nodes().getMinNodeVersion();
+            Version maxClusterNodeVersion = newState.nodes().getMaxNodeVersion();
+            // we only enforce major version transitions on a fully formed clusters
             // processing any joins
             for (final DiscoveryNode node : joiningNodes) {
                 if (node.equals(BECOME_MASTER_TASK) || node.equals(FINISH_ELECTION_TASK)) {
@@ -445,51 +441,59 @@ public BatchResult<DiscoveryNode> execute(ClusterState currentState, List<Discov
                     logger.debug("received a join request for an existing node [{}]", node);
                 } else {
                     try {
+                        MembershipAction.ensureNodesCompatibility(node.getVersion(), minClusterNodeVersion, maxClusterNodeVersion);
+                        // we do this validation quite late to prevent race conditions between nodes joining and importing dangling indices
+                        // we have to reject nodes that don't support all indices we have in this cluster
+                        MembershipAction.ensureIndexCompatibility(node.getVersion(), currentState.getMetaData());
                         nodesBuilder.add(node);
                         nodesChanged = true;
-                    } catch (IllegalArgumentException e) {
+                        minClusterNodeVersion = Version.min(minClusterNodeVersion, node.getVersion());
+                        maxClusterNodeVersion = Version.max(maxClusterNodeVersion, node.getVersion());
+                    } catch (IllegalArgumentException | IllegalStateException e) {
                         results.failure(node, e);
                         continue;
                     }
                 }
                 results.success(node);
             }
-
             if (nodesChanged) {
                 newState.nodes(nodesBuilder);
-                final ClusterState tmpState = newState.build();
-                RoutingAllocation.Result result = allocationService.reroute(tmpState, "node_join");
-                newState = ClusterState.builder(tmpState);
-                if (result.changed()) {
-                    newState.routingResult(result);
-                }
+                return results.build(allocationService.reroute(newState.build(), "node_join"));
+            } else {
+                // we must return a new cluster state instance to force publishing. This is important
+                // for the joining node to finalize its join and set us as a master
+                return results.build(newState.build());
             }
-
-            // we must return a new cluster state instance to force publishing. This is important
-            // for the joining node to finalize its join and set us as a master
-            return results.build(newState.build());
         }
 
         private ClusterState.Builder becomeMasterAndTrimConflictingNodes(ClusterState currentState, List<DiscoveryNode> joiningNodes) {
-            assert currentState.nodes().getMasterNodeId() == null : currentState.prettyPrint();
-            DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(currentState.nodes());
+            assert currentState.nodes().getMasterNodeId() == null : currentState;
+            DiscoveryNodes currentNodes = currentState.nodes();
+            DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(currentNodes);
             nodesBuilder.masterNodeId(currentState.nodes().getLocalNodeId());
-            ClusterBlocks clusterBlocks = ClusterBlocks.builder().blocks(currentState.blocks())
-                .removeGlobalBlock(discoverySettings.getNoMasterBlock()).build();
             for (final DiscoveryNode joiningNode : joiningNodes) {
-                final DiscoveryNode existingNode = nodesBuilder.get(joiningNode.getId());
-                if (existingNode != null && existingNode.equals(joiningNode) == false) {
-                    logger.debug("removing existing node [{}], which conflicts with incoming join from [{}]", existingNode, joiningNode);
-                    nodesBuilder.remove(existingNode.getId());
+                if (joiningNode == FINISH_ELECTION_TASK || joiningNode == BECOME_MASTER_TASK) {
+                    continue;
+                }
+                final DiscoveryNode nodeWithSameId = nodesBuilder.get(joiningNode.getId());
+                if (nodeWithSameId != null && nodeWithSameId.equals(joiningNode) == false) {
+                    logger.debug("removing existing node [{}], which conflicts with incoming join from [{}]", nodeWithSameId, joiningNode);
+                    nodesBuilder.remove(nodeWithSameId.getId());
+                }
+                final DiscoveryNode nodeWithSameAddress = currentNodes.findByAddress(joiningNode.getAddress());
+                if (nodeWithSameAddress != null && nodeWithSameAddress.equals(joiningNode) == false) {
+                    logger.debug("removing existing node [{}], which conflicts with incoming join from [{}]", nodeWithSameAddress,
+                        joiningNode);
+                    nodesBuilder.remove(nodeWithSameAddress.getId());
                 }
             }
 
+
             // now trim any left over dead nodes - either left there when the previous master stepped down
             // or removed by us above
-            ClusterState tmpState = ClusterState.builder(currentState).nodes(nodesBuilder).blocks(clusterBlocks).build();
-            RoutingAllocation.Result result = allocationService.deassociateDeadNodes(tmpState, false,
-                "removed dead nodes on election");
-            return ClusterState.builder(tmpState).routingResult(result);
+            ClusterState tmpState = ClusterState.builder(currentState).nodes(nodesBuilder).build();
+            return ClusterState.builder(allocationService.deassociateDeadNodes(tmpState, false,
+                "removed dead nodes on election"));
         }
 
         @Override
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/fd/NodesFaultDetection.java b/core/src/main/java/org/elasticsearch/discovery/zen/NodesFaultDetection.java
similarity index 97%
rename from core/src/main/java/org/elasticsearch/discovery/zen/fd/NodesFaultDetection.java
rename to core/src/main/java/org/elasticsearch/discovery/zen/NodesFaultDetection.java
index 40eb36cec1f15..5cd02a52504f5 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/fd/NodesFaultDetection.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/NodesFaultDetection.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.discovery.zen.fd;
+package org.elasticsearch.discovery.zen;
 
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
@@ -41,6 +41,8 @@
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
+import java.util.Collections;
+import java.util.Set;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.CopyOnWriteArrayList;
 
@@ -91,6 +93,14 @@ public void removeListener(Listener listener) {
         listeners.remove(listener);
     }
 
+    /**
+     * Gets the current set of nodes involved in node fault detection.
+     * NB: For testing purposes.
+     */
+    public Set<DiscoveryNode> getNodes() {
+        return Collections.unmodifiableSet(nodesFD.keySet());
+    }
+
     /**
      * make sure that nodes in clusterState are pinged. Any pinging to nodes which are not
      * part of the cluster will be stopped
@@ -129,7 +139,6 @@ public NodesFaultDetection stop() {
     public void close() {
         super.close();
         stop();
-        transportService.removeHandler(PING_ACTION_NAME);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/publish/PendingClusterStateStats.java b/core/src/main/java/org/elasticsearch/discovery/zen/PendingClusterStateStats.java
similarity index 98%
rename from core/src/main/java/org/elasticsearch/discovery/zen/publish/PendingClusterStateStats.java
rename to core/src/main/java/org/elasticsearch/discovery/zen/PendingClusterStateStats.java
index e060f68833801..8facf2f282cde 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/publish/PendingClusterStateStats.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/PendingClusterStateStats.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.discovery.zen.publish;
+package org.elasticsearch.discovery.zen;
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/publish/PendingClusterStatesQueue.java b/core/src/main/java/org/elasticsearch/discovery/zen/PendingClusterStatesQueue.java
similarity index 88%
rename from core/src/main/java/org/elasticsearch/discovery/zen/publish/PendingClusterStatesQueue.java
rename to core/src/main/java/org/elasticsearch/discovery/zen/PendingClusterStatesQueue.java
index 01fb96b71331c..018258066de8d 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/publish/PendingClusterStatesQueue.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/PendingClusterStatesQueue.java
@@ -16,7 +16,8 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.discovery.zen.publish;
+
+package org.elasticsearch.discovery.zen;
 
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
@@ -34,10 +35,12 @@
  * <p>
  * The queue is bound by {@link #maxQueueSize}. When the queue is at capacity and a new cluster state is inserted
  * the oldest cluster state will be dropped. This is safe because:
- * 1) Under normal operations, master will publish &amp; commit a cluster state before processing another change (i.e., the queue length is 1)
+ * 1) Under normal operations, master will publish &amp; commit a cluster state before processing
+ *    another change (i.e., the queue length is 1)
  * 2) If the master fails to commit a change, it will step down, causing a master election, which will flush the queue.
  * 3) In general it's safe to process the incoming cluster state as a replacement to the cluster state that's dropped.
- * a) If the dropped cluster is from the same master as the incoming one is, it is likely to be superseded by the incoming state (or another state in the queue).
+ * a) If the dropped cluster is from the same master as the incoming one is, it is likely to be superseded by the
+ *    incoming state (or another state in the queue).
  * This is only not true in very extreme cases of out of order delivery.
  * b) If the dropping cluster state is not from the same master, it means that:
  * i) we are no longer following the master of the dropped cluster state but follow the incoming one
@@ -70,7 +73,8 @@ public synchronized void addPending(ClusterState state) {
             ClusterStateContext context = pendingStates.remove(0);
             logger.warn("dropping pending state [{}]. more than [{}] pending states.", context, maxQueueSize);
             if (context.committed()) {
-                context.listener.onNewClusterStateFailed(new ElasticsearchException("too many pending states ([{}] pending)", maxQueueSize));
+                context.listener.onNewClusterStateFailed(new ElasticsearchException("too many pending states ([{}] pending)",
+                    maxQueueSize));
             }
         }
     }
@@ -82,11 +86,13 @@ public synchronized void addPending(ClusterState state) {
     public synchronized ClusterState markAsCommitted(String stateUUID, StateProcessedListener listener) {
         final ClusterStateContext context = findState(stateUUID);
         if (context == null) {
-            listener.onNewClusterStateFailed(new IllegalStateException("can't resolve cluster state with uuid [" + stateUUID + "] to commit"));
+            listener.onNewClusterStateFailed(new IllegalStateException("can't resolve cluster state with uuid" +
+                " [" + stateUUID + "] to commit"));
             return null;
         }
         if (context.committed()) {
-            listener.onNewClusterStateFailed(new IllegalStateException("cluster state with uuid [" + stateUUID + "] is already committed"));
+            listener.onNewClusterStateFailed(new IllegalStateException("cluster state with uuid" +
+                " [" + stateUUID + "] is already committed"));
             return null;
         }
         context.markAsCommitted(listener);
@@ -94,13 +100,14 @@ public synchronized ClusterState markAsCommitted(String stateUUID, StateProcesse
     }
 
     /**
-     * mark that the processing of the given state has failed. All committed states that are {@link ClusterState#supersedes(ClusterState)}-ed
-     * by this failed state, will be failed as well
+     * mark that the processing of the given state has failed. All committed states that are
+     * {@link ClusterState#supersedes(ClusterState)}-ed by this failed state, will be failed as well
      */
     public synchronized void markAsFailed(ClusterState state, Exception reason) {
         final ClusterStateContext failedContext = findState(state.stateUUID());
         if (failedContext == null) {
-            throw new IllegalArgumentException("can't resolve failed cluster state with uuid [" + state.stateUUID() + "], version [" + state.version() + "]");
+            throw new IllegalArgumentException("can't resolve failed cluster state with uuid [" + state.stateUUID()
+                + "], version [" + state.version() + "]");
         }
         if (failedContext.committed() == false) {
             throw new IllegalArgumentException("failed cluster state is not committed " + state);
@@ -128,15 +135,16 @@ public synchronized void markAsFailed(ClusterState state, Exception reason) {
     }
 
     /**
-     * indicates that a cluster state was successfully processed. Any committed state that is {@link ClusterState#supersedes(ClusterState)}-ed
-     * by the processed state will be marked as processed as well.
+     * indicates that a cluster state was successfully processed. Any committed state that is
+     * {@link ClusterState#supersedes(ClusterState)}-ed by the processed state will be marked as processed as well.
      * <p>
-     * NOTE: successfully processing a state indicates we are following the master it came from. Any committed state from another master will
-     * be failed by this method
+     * NOTE: successfully processing a state indicates we are following the master it came from. Any committed state
+     * from another master will be failed by this method
      */
     public synchronized void markAsProcessed(ClusterState state) {
         if (findState(state.stateUUID()) == null) {
-            throw new IllegalStateException("can't resolve processed cluster state with uuid [" + state.stateUUID() + "], version [" + state.version() + "]");
+            throw new IllegalStateException("can't resolve processed cluster state with uuid [" + state.stateUUID()
+                + "], version [" + state.version() + "]");
         }
         final DiscoveryNode currentMaster = state.nodes().getMasterNode();
         assert currentMaster != null : "processed cluster state mast have a master. " + state;
@@ -152,17 +160,16 @@ public synchronized void markAsProcessed(ClusterState state) {
                 contextsToRemove.add(pendingContext);
                 if (pendingContext.committed()) {
                     // this is a committed state , warn
-                    logger.warn("received a cluster state (uuid[{}]/v[{}]) from a different master than the current one, rejecting (received {}, current {})",
-                            pendingState.stateUUID(), pendingState.version(),
-                            pendingMasterNode, currentMaster);
+                    logger.warn("received a cluster state (uuid[{}]/v[{}]) from a different master than the current one,"
+                        + " rejecting (received {}, current {})",
+                            pendingState.stateUUID(), pendingState.version(), pendingMasterNode, currentMaster);
                     pendingContext.listener.onNewClusterStateFailed(
-                            new IllegalStateException("cluster state from a different master than the current one, rejecting (received " + pendingMasterNode + ", current " + currentMaster + ")")
-                    );
+                            new IllegalStateException("cluster state from a different master than the current one," +
+                                " rejecting (received " + pendingMasterNode + ", current " + currentMaster + ")"));
                 } else {
-                    logger.trace("removing non-committed state with uuid[{}]/v[{}] from [{}] - a state from [{}] was successfully processed",
-                            pendingState.stateUUID(), pendingState.version(), pendingMasterNode,
-                            currentMaster
-                    );
+                    logger.trace("removing non-committed state with uuid[{}]/v[{}] from [{}] - a state from" +
+                            " [{}] was successfully processed",
+                            pendingState.stateUUID(), pendingState.version(), pendingMasterNode, currentMaster);
                 }
             } else if (pendingState.stateUUID().equals(state.stateUUID())) {
                 assert pendingContext.committed() : "processed cluster state is not committed " + state;
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/PingContextProvider.java b/core/src/main/java/org/elasticsearch/discovery/zen/PingContextProvider.java
new file mode 100644
index 0000000000000..7567b69cfe459
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/PingContextProvider.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.discovery.zen;
+
+import org.elasticsearch.cluster.ClusterState;
+
+public interface PingContextProvider {
+
+    /** return the current cluster state of the node */
+    ClusterState clusterState();
+}
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java b/core/src/main/java/org/elasticsearch/discovery/zen/PublishClusterStateAction.java
similarity index 85%
rename from core/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java
rename to core/src/main/java/org/elasticsearch/discovery/zen/PublishClusterStateAction.java
index 06c25ebf81a66..009abe46949c2 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/PublishClusterStateAction.java
@@ -17,9 +17,10 @@
  * under the License.
  */
 
-package org.elasticsearch.discovery.zen.publish;
+package org.elasticsearch.discovery.zen;
 
 import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterChangedEvent;
@@ -34,6 +35,8 @@
 import org.elasticsearch.common.compress.Compressor;
 import org.elasticsearch.common.compress.CompressorFactory;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
@@ -42,7 +45,6 @@
 import org.elasticsearch.discovery.BlockingClusterStatePublishResponseHandler;
 import org.elasticsearch.discovery.Discovery;
 import org.elasticsearch.discovery.DiscoverySettings;
-import org.elasticsearch.discovery.zen.ZenDiscovery;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.BytesTransportRequest;
 import org.elasticsearch.transport.EmptyTransportResponseHandler;
@@ -83,6 +85,7 @@ public interface NewPendingClusterStateListener {
     }
 
     private final TransportService transportService;
+    private final NamedWriteableRegistry namedWriteableRegistry;
     private final Supplier<ClusterState> clusterStateSupplier;
     private final NewPendingClusterStateListener newPendingClusterStatelistener;
     private final DiscoverySettings discoverySettings;
@@ -92,24 +95,23 @@ public interface NewPendingClusterStateListener {
     public PublishClusterStateAction(
             Settings settings,
             TransportService transportService,
+            NamedWriteableRegistry namedWriteableRegistry,
             Supplier<ClusterState> clusterStateSupplier,
             NewPendingClusterStateListener listener,
             DiscoverySettings discoverySettings,
             ClusterName clusterName) {
         super(settings);
         this.transportService = transportService;
+        this.namedWriteableRegistry = namedWriteableRegistry;
         this.clusterStateSupplier = clusterStateSupplier;
         this.newPendingClusterStatelistener = listener;
         this.discoverySettings = discoverySettings;
         this.clusterName = clusterName;
         this.pendingStatesQueue = new PendingClusterStatesQueue(logger, settings.getAsInt(SETTINGS_MAX_PENDING_CLUSTER_STATES, 25));
-        transportService.registerRequestHandler(SEND_ACTION_NAME, BytesTransportRequest::new, ThreadPool.Names.SAME, new SendClusterStateRequestHandler());
-        transportService.registerRequestHandler(COMMIT_ACTION_NAME, CommitClusterStateRequest::new, ThreadPool.Names.SAME, new CommitClusterStateRequestHandler());
-    }
-
-    public void close() {
-        transportService.removeHandler(SEND_ACTION_NAME);
-        transportService.removeHandler(COMMIT_ACTION_NAME);
+        transportService.registerRequestHandler(SEND_ACTION_NAME, BytesTransportRequest::new, ThreadPool.Names.SAME, false, false,
+            new SendClusterStateRequestHandler());
+        transportService.registerRequestHandler(COMMIT_ACTION_NAME, CommitClusterStateRequest::new, ThreadPool.Names.SAME, false, false,
+            new CommitClusterStateRequestHandler());
     }
 
     public PendingClusterStatesQueue pendingStatesQueue() {
@@ -120,10 +122,12 @@ public PendingClusterStatesQueue pendingStatesQueue() {
      * publishes a cluster change event to other nodes. if at least minMasterNodes acknowledge the change it is committed and will
      * be processed by the master and the other nodes.
      * <p>
-     * The method is guaranteed to throw a {@link org.elasticsearch.discovery.Discovery.FailedToCommitClusterStateException} if the change is not committed and should be rejected.
+     * The method is guaranteed to throw a {@link org.elasticsearch.discovery.Discovery.FailedToCommitClusterStateException}
+     * if the change is not committed and should be rejected.
      * Any other exception signals the something wrong happened but the change is committed.
      */
-    public void publish(final ClusterChangedEvent clusterChangedEvent, final int minMasterNodes, final Discovery.AckListener ackListener) throws Discovery.FailedToCommitClusterStateException {
+    public void publish(final ClusterChangedEvent clusterChangedEvent, final int minMasterNodes,
+                        final Discovery.AckListener ackListener) throws Discovery.FailedToCommitClusterStateException {
         final DiscoveryNodes nodes;
         final SendingController sendingController;
         final Set<DiscoveryNode> nodesToPublishTo;
@@ -151,8 +155,10 @@ public void publish(final ClusterChangedEvent clusterChangedEvent, final int min
             buildDiffAndSerializeStates(clusterChangedEvent.state(), clusterChangedEvent.previousState(),
                     nodesToPublishTo, sendFullVersion, serializedStates, serializedDiffs);
 
-            final BlockingClusterStatePublishResponseHandler publishResponseHandler = new AckClusterStatePublishResponseHandler(nodesToPublishTo, ackListener);
-            sendingController = new SendingController(clusterChangedEvent.state(), minMasterNodes, totalMasterNodes, publishResponseHandler);
+            final BlockingClusterStatePublishResponseHandler publishResponseHandler =
+                new AckClusterStatePublishResponseHandler(nodesToPublishTo, ackListener);
+            sendingController = new SendingController(clusterChangedEvent.state(), minMasterNodes,
+                totalMasterNodes, publishResponseHandler);
         } catch (Exception e) {
             throw new Discovery.FailedToCommitClusterStateException("unexpected error while preparing to publish", e);
         }
@@ -203,7 +209,8 @@ private void innerPublish(final ClusterChangedEvent clusterChangedEvent, final S
                 DiscoveryNode[] pendingNodes = publishResponseHandler.pendingNodes();
                 // everyone may have just responded
                 if (pendingNodes.length > 0) {
-                    logger.warn("timed out waiting for all nodes to process published state [{}] (timeout [{}], pending nodes: {})", clusterState.version(), publishTimeout, pendingNodes);
+                    logger.warn("timed out waiting for all nodes to process published state [{}] (timeout [{}], pending nodes: {})",
+                        clusterState.version(), publishTimeout, pendingNodes);
                 }
             }
         } catch (InterruptedException e) {
@@ -213,7 +220,8 @@ private void innerPublish(final ClusterChangedEvent clusterChangedEvent, final S
     }
 
     private void buildDiffAndSerializeStates(ClusterState clusterState, ClusterState previousState, Set<DiscoveryNode> nodesToPublishTo,
-                                             boolean sendFullVersion, Map<Version, BytesReference> serializedStates, Map<Version, BytesReference> serializedDiffs) {
+                                             boolean sendFullVersion, Map<Version, BytesReference> serializedStates,
+                                             Map<Version, BytesReference> serializedDiffs) {
         Diff<ClusterState> diff = null;
         for (final DiscoveryNode node : nodesToPublishTo) {
             try {
@@ -246,7 +254,8 @@ private void sendFullClusterState(ClusterState clusterState, Map<Version, BytesR
                 serializedStates.put(node.getVersion(), bytes);
             } catch (Exception e) {
                 logger.warn(
-                    (org.apache.logging.log4j.util.Supplier<?>) () -> new ParameterizedMessage("failed to serialize cluster_state before publishing it to node {}", node), e);
+                    (org.apache.logging.log4j.util.Supplier<?>) () ->
+                        new ParameterizedMessage("failed to serialize cluster_state before publishing it to node {}", node), e);
                 sendingController.onNodeSendFailed(node, e);
                 return;
             }
@@ -272,7 +281,8 @@ private void sendClusterStateToNode(final ClusterState clusterState, BytesRefere
             // -> no need to put a timeout on the options here, because we want the response to eventually be received
             //  and not log an error if it arrives after the timeout
             // -> no need to compress, we already compressed the bytes
-            TransportRequestOptions options = TransportRequestOptions.builder().withType(TransportRequestOptions.Type.STATE).withCompress(false).build();
+            TransportRequestOptions options = TransportRequestOptions.builder()
+                .withType(TransportRequestOptions.Type.STATE).withCompress(false).build();
             transportService.sendRequest(node, SEND_ACTION_NAME,
                     new BytesTransportRequest(bytes, node.getVersion()),
                     options,
@@ -281,7 +291,8 @@ private void sendClusterStateToNode(final ClusterState clusterState, BytesRefere
                         @Override
                         public void handleResponse(TransportResponse.Empty response) {
                             if (sendingController.getPublishingTimedOut()) {
-                                logger.debug("node {} responded for cluster state [{}] (took longer than [{}])", node, clusterState.version(), publishTimeout);
+                                logger.debug("node {} responded for cluster state [{}] (took longer than [{}])", node,
+                                    clusterState.version(), publishTimeout);
                             }
                             sendingController.onNodeSendAck(node);
                         }
@@ -292,21 +303,24 @@ public void handleException(TransportException exp) {
                                 logger.debug("resending full cluster state to node {} reason {}", node, exp.getDetailedMessage());
                                 sendFullClusterState(clusterState, serializedStates, node, publishTimeout, sendingController);
                             } else {
-                                logger.debug((org.apache.logging.log4j.util.Supplier<?>) () -> new ParameterizedMessage("failed to send cluster state to {}", node), exp);
+                                logger.debug((org.apache.logging.log4j.util.Supplier<?>) () ->
+                                    new ParameterizedMessage("failed to send cluster state to {}", node), exp);
                                 sendingController.onNodeSendFailed(node, exp);
                             }
                         }
                     });
         } catch (Exception e) {
             logger.warn(
-                (org.apache.logging.log4j.util.Supplier<?>) () -> new ParameterizedMessage("error sending cluster state to {}", node), e);
+                (org.apache.logging.log4j.util.Supplier<?>) () ->
+                    new ParameterizedMessage("error sending cluster state to {}", node), e);
             sendingController.onNodeSendFailed(node, e);
         }
     }
 
     private void sendCommitToNode(final DiscoveryNode node, final ClusterState clusterState, final SendingController sendingController) {
         try {
-            logger.trace("sending commit for cluster state (uuid: [{}], version [{}]) to [{}]", clusterState.stateUUID(), clusterState.version(), node);
+            logger.trace("sending commit for cluster state (uuid: [{}], version [{}]) to [{}]",
+                clusterState.stateUUID(), clusterState.version(), node);
             TransportRequestOptions options = TransportRequestOptions.builder().withType(TransportRequestOptions.Type.STATE).build();
             // no need to put a timeout on the options here, because we want the response to eventually be received
             // and not log an error if it arrives after the timeout
@@ -325,12 +339,16 @@ public void handleResponse(TransportResponse.Empty response) {
 
                         @Override
                         public void handleException(TransportException exp) {
-                            logger.debug((org.apache.logging.log4j.util.Supplier<?>) () -> new ParameterizedMessage("failed to commit cluster state (uuid [{}], version [{}]) to {}", clusterState.stateUUID(), clusterState.version(), node), exp);
+                            logger.debug((org.apache.logging.log4j.util.Supplier<?>) () ->
+                                new ParameterizedMessage("failed to commit cluster state (uuid [{}], version [{}]) to {}",
+                                    clusterState.stateUUID(), clusterState.version(), node), exp);
                             sendingController.getPublishResponseHandler().onFailure(node, exp);
                         }
                     });
         } catch (Exception t) {
-            logger.warn((org.apache.logging.log4j.util.Supplier<?>) () -> new ParameterizedMessage("error sending cluster state commit (uuid [{}], version [{}]) to {}", clusterState.stateUUID(), clusterState.version(), node), t);
+            logger.warn((org.apache.logging.log4j.util.Supplier<?>) () ->
+                new ParameterizedMessage("error sending cluster state commit (uuid [{}], version [{}]) to {}",
+                    clusterState.stateUUID(), clusterState.version(), node), t);
             sendingController.getPublishResponseHandler().onFailure(node, t);
         }
     }
@@ -361,33 +379,37 @@ public static BytesReference serializeDiffClusterState(Diff diff, Version nodeVe
 
     protected void handleIncomingClusterStateRequest(BytesTransportRequest request, TransportChannel channel) throws IOException {
         Compressor compressor = CompressorFactory.compressor(request.bytes());
-        StreamInput in;
-        if (compressor != null) {
-            in = compressor.streamInput(request.bytes().streamInput());
-        } else {
-            in = request.bytes().streamInput();
-        }
-        in.setVersion(request.version());
-        synchronized (lastSeenClusterStateMutex) {
-            final ClusterState incomingState;
-            // If true we received full cluster state - otherwise diffs
-            if (in.readBoolean()) {
-                incomingState = ClusterState.Builder.readFrom(in, clusterStateSupplier.get().nodes().getLocalNode());
-                logger.debug("received full cluster state version [{}] with size [{}]", incomingState.version(), request.bytes().length());
-            } else if (lastSeenClusterState != null) {
-                Diff<ClusterState> diff = lastSeenClusterState.readDiffFrom(in);
-                incomingState = diff.apply(lastSeenClusterState);
-                logger.debug("received diff cluster state version [{}] with uuid [{}], diff size [{}]", incomingState.version(), incomingState.stateUUID(), request.bytes().length());
-            } else {
-                logger.debug("received diff for but don't have any local cluster state - requesting full state");
-                throw new IncompatibleClusterStateVersionException("have no local cluster state");
+        StreamInput in = request.bytes().streamInput();
+        try {
+            if (compressor != null) {
+                in = compressor.streamInput(in);
             }
-            // sanity check incoming state
-            validateIncomingState(incomingState, lastSeenClusterState);
+            in = new NamedWriteableAwareStreamInput(in, namedWriteableRegistry);
+            in.setVersion(request.version());
+            synchronized (lastSeenClusterStateMutex) {
+                final ClusterState incomingState;
+                // If true we received full cluster state - otherwise diffs
+                if (in.readBoolean()) {
+                    incomingState = ClusterState.readFrom(in, clusterStateSupplier.get().nodes().getLocalNode());
+                    logger.debug("received full cluster state version [{}] with size [{}]", incomingState.version(),
+                        request.bytes().length());
+                } else if (lastSeenClusterState != null) {
+                    Diff<ClusterState> diff = ClusterState.readDiffFrom(in, lastSeenClusterState.nodes().getLocalNode());
+                    incomingState = diff.apply(lastSeenClusterState);
+                    logger.debug("received diff cluster state version [{}] with uuid [{}], diff size [{}]",
+                        incomingState.version(), incomingState.stateUUID(), request.bytes().length());
+                } else {
+                    logger.debug("received diff for but don't have any local cluster state - requesting full state");
+                    throw new IncompatibleClusterStateVersionException("have no local cluster state");
+                }
+                // sanity check incoming state
+                validateIncomingState(incomingState, lastSeenClusterState);
 
-            pendingStatesQueue.addPending(incomingState);
-            lastSeenClusterState = incomingState;
-            lastSeenClusterState.status(ClusterState.ClusterStateStatus.RECEIVED);
+                pendingStatesQueue.addPending(incomingState);
+                lastSeenClusterState = incomingState;
+            }
+        } finally {
+            IOUtils.close(in);
         }
         channel.sendResponse(TransportResponse.Empty.INSTANCE);
     }
@@ -400,13 +422,15 @@ protected void handleIncomingClusterStateRequest(BytesTransportRequest request,
     void validateIncomingState(ClusterState incomingState, ClusterState lastSeenClusterState) {
         final ClusterName incomingClusterName = incomingState.getClusterName();
         if (!incomingClusterName.equals(this.clusterName)) {
-            logger.warn("received cluster state from [{}] which is also master but with a different cluster name [{}]", incomingState.nodes().getMasterNode(), incomingClusterName);
+            logger.warn("received cluster state from [{}] which is also master but with a different cluster name [{}]",
+                incomingState.nodes().getMasterNode(), incomingClusterName);
             throw new IllegalStateException("received state from a node that is not part of the cluster");
         }
         final ClusterState clusterState = clusterStateSupplier.get();
 
         if (clusterState.nodes().getLocalNode().equals(incomingState.nodes().getLocalNode()) == false) {
-            logger.warn("received a cluster state from [{}] and not part of the cluster, should not happen", incomingState.nodes().getMasterNode());
+            logger.warn("received a cluster state from [{}] and not part of the cluster, should not happen",
+                incomingState.nodes().getMasterNode());
             throw new IllegalStateException("received state with a local node that does not match the current local node");
         }
 
@@ -425,7 +449,8 @@ void validateIncomingState(ClusterState incomingState, ClusterState lastSeenClus
     }
 
     protected void handleCommitRequest(CommitClusterStateRequest request, final TransportChannel channel) {
-        final ClusterState state = pendingStatesQueue.markAsCommitted(request.stateUUID, new PendingClusterStatesQueue.StateProcessedListener() {
+        final ClusterState state = pendingStatesQueue.markAsCommitted(request.stateUUID,
+            new PendingClusterStatesQueue.StateProcessedListener() {
             @Override
             public void onNewClusterStateProcessed() {
                 try {
@@ -448,7 +473,8 @@ public void onNewClusterStateFailed(Exception e) {
             }
         });
         if (state != null) {
-            newPendingClusterStatelistener.onNewClusterState("master " + state.nodes().getMasterNode() + " committed version [" + state.version() + "]");
+            newPendingClusterStatelistener.onNewClusterState("master " + state.nodes().getMasterNode() +
+                " committed version [" + state.version() + "]");
         }
     }
 
@@ -518,13 +544,15 @@ public BlockingClusterStatePublishResponseHandler getPublishResponseHandler() {
         // an external marker to note that the publishing process is timed out. This is useful for proper logging.
         final AtomicBoolean publishingTimedOut = new AtomicBoolean();
 
-        private SendingController(ClusterState clusterState, int minMasterNodes, int totalMasterNodes, BlockingClusterStatePublishResponseHandler publishResponseHandler) {
+        private SendingController(ClusterState clusterState, int minMasterNodes, int totalMasterNodes,
+                                  BlockingClusterStatePublishResponseHandler publishResponseHandler) {
             this.clusterState = clusterState;
             this.publishResponseHandler = publishResponseHandler;
             this.neededMastersToCommit = Math.max(0, minMasterNodes - 1); // we are one of the master nodes
             this.pendingMasterNodes = totalMasterNodes - 1;
             if (this.neededMastersToCommit > this.pendingMasterNodes) {
-                throw new Discovery.FailedToCommitClusterStateException("not enough masters to ack sent cluster state. [{}] needed , have [{}]", neededMastersToCommit, pendingMasterNodes);
+                throw new Discovery.FailedToCommitClusterStateException("not enough masters to ack sent cluster state." +
+                    "[{}] needed , have [{}]", neededMastersToCommit, pendingMasterNodes);
             }
             this.committed = neededMastersToCommit == 0;
             this.committedOrFailedLatch = new CountDownLatch(committed ? 0 : 1);
@@ -598,7 +626,8 @@ private synchronized void decrementPendingMasterAcksAndChangeForFailure() {
 
         public synchronized void onNodeSendFailed(DiscoveryNode node, Exception e) {
             if (node.isMasterNode()) {
-                logger.trace("master node {} failed to ack cluster state version [{}]. processing ... (current pending [{}], needed [{}])",
+                logger.trace("master node {} failed to ack cluster state version [{}]. " +
+                        "processing ... (current pending [{}], needed [{}])",
                         node, clusterState.version(), pendingMasterNodes, neededMastersToCommit);
                 decrementPendingMasterAcksAndChangeForFailure();
             }
@@ -629,7 +658,8 @@ private synchronized boolean markAsFailed(String details, Exception reason) {
             if (committedOrFailed()) {
                 return committed == false;
             }
-            logger.trace((org.apache.logging.log4j.util.Supplier<?>) () -> new ParameterizedMessage("failed to commit version [{}]. {}", clusterState.version(), details), reason);
+            logger.trace((org.apache.logging.log4j.util.Supplier<?>) () -> new ParameterizedMessage("failed to commit version [{}]. {}",
+                clusterState.version(), details), reason);
             committed = false;
             committedOrFailedLatch.countDown();
             return true;
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastHostsProvider.java b/core/src/main/java/org/elasticsearch/discovery/zen/UnicastHostsProvider.java
similarity index 95%
rename from core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastHostsProvider.java
rename to core/src/main/java/org/elasticsearch/discovery/zen/UnicastHostsProvider.java
index dbfaed572b1e6..9ff3215cd6480 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastHostsProvider.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/UnicastHostsProvider.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.discovery.zen.ping.unicast;
+package org.elasticsearch.discovery.zen;
 
 import org.elasticsearch.cluster.node.DiscoveryNode;
 
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/UnicastZenPing.java b/core/src/main/java/org/elasticsearch/discovery/zen/UnicastZenPing.java
new file mode 100644
index 0000000000000..3a68b2b4cd8a3
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/UnicastZenPing.java
@@ -0,0 +1,689 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.discovery.zen;
+
+import com.carrotsearch.hppc.cursors.ObjectCursor;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.apache.logging.log4j.util.Supplier;
+import org.apache.lucene.store.AlreadyClosedException;
+import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.component.AbstractComponent;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.lease.Releasable;
+import org.elasticsearch.common.lease.Releasables;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.CollectionUtils;
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
+import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor;
+import org.elasticsearch.common.util.concurrent.KeyedLock;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.ConnectTransportException;
+import org.elasticsearch.transport.ConnectionProfile;
+import org.elasticsearch.transport.NodeNotConnectedException;
+import org.elasticsearch.transport.RemoteTransportException;
+import org.elasticsearch.transport.Transport.Connection;
+import org.elasticsearch.transport.TransportChannel;
+import org.elasticsearch.transport.TransportException;
+import org.elasticsearch.transport.TransportRequest;
+import org.elasticsearch.transport.TransportRequestHandler;
+import org.elasticsearch.transport.TransportRequestOptions;
+import org.elasticsearch.transport.TransportResponse;
+import org.elasticsearch.transport.TransportResponseHandler;
+import org.elasticsearch.transport.TransportService;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Queue;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Consumer;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static java.util.Collections.emptyList;
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.emptySet;
+import static org.elasticsearch.common.util.concurrent.ConcurrentCollections.newConcurrentMap;
+import static org.elasticsearch.discovery.zen.ZenPing.PingResponse.readPingResponse;
+
+public class UnicastZenPing extends AbstractComponent implements ZenPing {
+
+    public static final String ACTION_NAME = "internal:discovery/zen/unicast";
+    public static final Setting<List<String>> DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING =
+        Setting.listSetting("discovery.zen.ping.unicast.hosts", emptyList(), Function.identity(),
+            Property.NodeScope);
+    public static final Setting<Integer> DISCOVERY_ZEN_PING_UNICAST_CONCURRENT_CONNECTS_SETTING =
+        Setting.intSetting("discovery.zen.ping.unicast.concurrent_connects", 10, 0, Property.NodeScope);
+    public static final Setting<TimeValue> DISCOVERY_ZEN_PING_UNICAST_HOSTS_RESOLVE_TIMEOUT =
+        Setting.positiveTimeSetting("discovery.zen.ping.unicast.hosts.resolve_timeout", TimeValue.timeValueSeconds(5), Property.NodeScope);
+
+    // these limits are per-address
+    public static final int LIMIT_FOREIGN_PORTS_COUNT = 1;
+    public static final int LIMIT_LOCAL_PORTS_COUNT = 5;
+
+    private final ThreadPool threadPool;
+    private final TransportService transportService;
+    private final ClusterName clusterName;
+
+    private final List<String> configuredHosts;
+
+    private final int limitPortCounts;
+
+    private volatile PingContextProvider contextProvider;
+
+    private final AtomicInteger pingingRoundIdGenerator = new AtomicInteger();
+
+    // used as a node id prefix for configured unicast host nodes/address
+    private static final String UNICAST_NODE_PREFIX = "#zen_unicast_";
+
+    private final Map<Integer, PingingRound> activePingingRounds = newConcurrentMap();
+
+    // a list of temporal responses a node will return for a request (holds responses from other nodes)
+    private final Queue<PingResponse> temporalResponses = ConcurrentCollections.newQueue();
+
+    private final UnicastHostsProvider hostsProvider;
+
+    protected final EsThreadPoolExecutor unicastZenPingExecutorService;
+
+    private final TimeValue resolveTimeout;
+
+    private volatile boolean closed = false;
+
+    public UnicastZenPing(Settings settings, ThreadPool threadPool, TransportService transportService,
+                          UnicastHostsProvider unicastHostsProvider) {
+        super(settings);
+        this.threadPool = threadPool;
+        this.transportService = transportService;
+        this.clusterName = ClusterName.CLUSTER_NAME_SETTING.get(settings);
+        this.hostsProvider = unicastHostsProvider;
+
+        final int concurrentConnects = DISCOVERY_ZEN_PING_UNICAST_CONCURRENT_CONNECTS_SETTING.get(settings);
+        if (DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING.exists(settings)) {
+            configuredHosts = DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING.get(settings);
+            // we only limit to 1 addresses, makes no sense to ping 100 ports
+            limitPortCounts = LIMIT_FOREIGN_PORTS_COUNT;
+        } else {
+            // if unicast hosts are not specified, fill with simple defaults on the local machine
+            configuredHosts = transportService.getLocalAddresses();
+            limitPortCounts = LIMIT_LOCAL_PORTS_COUNT;
+        }
+        resolveTimeout = DISCOVERY_ZEN_PING_UNICAST_HOSTS_RESOLVE_TIMEOUT.get(settings);
+        logger.debug(
+            "using initial hosts {}, with concurrent_connects [{}], resolve_timeout [{}]",
+            configuredHosts,
+            concurrentConnects,
+            resolveTimeout);
+
+        transportService.registerRequestHandler(ACTION_NAME, UnicastPingRequest::new, ThreadPool.Names.SAME,
+            new UnicastPingRequestHandler());
+
+        final ThreadFactory threadFactory = EsExecutors.daemonThreadFactory(settings, "[unicast_connect]");
+        unicastZenPingExecutorService = EsExecutors.newScaling(
+            "unicast_connect",
+            0, concurrentConnects,
+            60,
+            TimeUnit.SECONDS,
+            threadFactory,
+            threadPool.getThreadContext());
+    }
+
+    /**
+     * Resolves a list of hosts to a list of discovery nodes. Each host is resolved into a transport address (or a collection of addresses
+     * if the number of ports is greater than one) and the transport addresses are used to created discovery nodes. Host lookups are done
+     * in parallel using specified executor service up to the specified resolve timeout.
+     *
+     * @param executorService  the executor service used to parallelize hostname lookups
+     * @param logger           logger used for logging messages regarding hostname lookups
+     * @param hosts            the hosts to resolve
+     * @param limitPortCounts  the number of ports to resolve (should be 1 for non-local transport)
+     * @param transportService the transport service
+     * @param nodeId_prefix    a prefix to use for node ids
+     * @param resolveTimeout   the timeout before returning from hostname lookups
+     * @return a list of discovery nodes with resolved transport addresses
+     */
+    public static List<DiscoveryNode> resolveHostsLists(
+        final ExecutorService executorService,
+        final Logger logger,
+        final List<String> hosts,
+        final int limitPortCounts,
+        final TransportService transportService,
+        final String nodeId_prefix,
+        final TimeValue resolveTimeout) throws InterruptedException {
+        Objects.requireNonNull(executorService);
+        Objects.requireNonNull(logger);
+        Objects.requireNonNull(hosts);
+        Objects.requireNonNull(transportService);
+        Objects.requireNonNull(nodeId_prefix);
+        Objects.requireNonNull(resolveTimeout);
+        if (resolveTimeout.nanos() < 0) {
+            throw new IllegalArgumentException("resolve timeout must be non-negative but was [" + resolveTimeout + "]");
+        }
+        // create tasks to submit to the executor service; we will wait up to resolveTimeout for these tasks to complete
+        final List<Callable<TransportAddress[]>> callables =
+            hosts
+                .stream()
+                .map(hn -> (Callable<TransportAddress[]>) () -> transportService.addressesFromString(hn, limitPortCounts))
+                .collect(Collectors.toList());
+        final List<Future<TransportAddress[]>> futures =
+            executorService.invokeAll(callables, resolveTimeout.nanos(), TimeUnit.NANOSECONDS);
+        final List<DiscoveryNode> discoveryNodes = new ArrayList<>();
+        final Set<TransportAddress> localAddresses = new HashSet<>();
+        localAddresses.add(transportService.boundAddress().publishAddress());
+        localAddresses.addAll(Arrays.asList(transportService.boundAddress().boundAddresses()));
+        // ExecutorService#invokeAll guarantees that the futures are returned in the iteration order of the tasks so we can associate the
+        // hostname with the corresponding task by iterating together
+        final Iterator<String> it = hosts.iterator();
+        for (final Future<TransportAddress[]> future : futures) {
+            final String hostname = it.next();
+            if (!future.isCancelled()) {
+                assert future.isDone();
+                try {
+                    final TransportAddress[] addresses = future.get();
+                    logger.trace("resolved host [{}] to {}", hostname, addresses);
+                    for (int addressId = 0; addressId < addresses.length; addressId++) {
+                        final TransportAddress address = addresses[addressId];
+                        // no point in pinging ourselves
+                        if (localAddresses.contains(address) == false) {
+                            discoveryNodes.add(
+                                new DiscoveryNode(
+                                    nodeId_prefix + hostname + "_" + addressId + "#",
+                                    address,
+                                    emptyMap(),
+                                    emptySet(),
+                                    Version.CURRENT.minimumCompatibilityVersion()));
+                        }
+                    }
+                } catch (final ExecutionException e) {
+                    assert e.getCause() != null;
+                    final String message = "failed to resolve host [" + hostname + "]";
+                    logger.warn(message, e.getCause());
+                }
+            } else {
+                logger.warn("timed out after [{}] resolving host [{}]", resolveTimeout, hostname);
+            }
+        }
+        return discoveryNodes;
+    }
+
+    @Override
+    public void close() {
+        ThreadPool.terminate(unicastZenPingExecutorService, 10, TimeUnit.SECONDS);
+        Releasables.close(activePingingRounds.values());
+        closed = true;
+    }
+
+    @Override
+    public void start(PingContextProvider contextProvider) {
+        this.contextProvider = contextProvider;
+    }
+
+    /**
+     * Clears the list of cached ping responses.
+     */
+    public void clearTemporalResponses() {
+        temporalResponses.clear();
+    }
+
+    /**
+     * Sends three rounds of pings notifying the specified {@link Consumer} when pinging is complete. Pings are sent after resolving
+     * configured unicast hosts to their IP address (subject to DNS caching within the JVM). A batch of pings is sent, then another batch
+     * of pings is sent at half the specified {@link TimeValue}, and then another batch of pings is sent at the specified {@link TimeValue}.
+     * The pings that are sent carry a timeout of 1.25 times the specified {@link TimeValue}. When pinging each node, a connection and
+     * handshake is performed, with a connection timeout of the specified {@link TimeValue}.
+     *
+     * @param resultsConsumer the callback when pinging is complete
+     * @param duration        the timeout for various components of the pings
+     */
+    @Override
+    public void ping(final Consumer<PingCollection> resultsConsumer, final TimeValue duration) {
+        ping(resultsConsumer, duration, duration);
+    }
+
+    /**
+     * a variant of {@link #ping(Consumer, TimeValue)}, but allows separating the scheduling duration
+     * from the duration used for request level time outs. This is useful for testing
+     */
+    protected void ping(final Consumer<PingCollection> resultsConsumer,
+                        final TimeValue scheduleDuration,
+                        final TimeValue requestDuration) {
+        final List<DiscoveryNode> seedNodes;
+        try {
+            seedNodes = resolveHostsLists(
+                unicastZenPingExecutorService,
+                logger,
+                configuredHosts,
+                limitPortCounts,
+                transportService,
+                UNICAST_NODE_PREFIX,
+                resolveTimeout);
+        } catch (InterruptedException e) {
+            throw new RuntimeException(e);
+        }
+        seedNodes.addAll(hostsProvider.buildDynamicNodes());
+        final DiscoveryNodes nodes = contextProvider.clusterState().nodes();
+        // add all possible master nodes that were active in the last known cluster configuration
+        for (ObjectCursor<DiscoveryNode> masterNode : nodes.getMasterNodes().values()) {
+            seedNodes.add(masterNode.value);
+        }
+
+        final ConnectionProfile connectionProfile =
+            ConnectionProfile.buildSingleChannelProfile(TransportRequestOptions.Type.REG, requestDuration, requestDuration);
+        final PingingRound pingingRound = new PingingRound(pingingRoundIdGenerator.incrementAndGet(), seedNodes, resultsConsumer,
+            nodes.getLocalNode(), connectionProfile);
+        activePingingRounds.put(pingingRound.id(), pingingRound);
+        final AbstractRunnable pingSender = new AbstractRunnable() {
+            @Override
+            public void onFailure(Exception e) {
+                if (e instanceof AlreadyClosedException == false) {
+                    logger.warn("unexpected error while pinging", e);
+                }
+            }
+
+            @Override
+            protected void doRun() throws Exception {
+                sendPings(requestDuration, pingingRound);
+            }
+        };
+        threadPool.generic().execute(pingSender);
+        threadPool.schedule(TimeValue.timeValueMillis(scheduleDuration.millis() / 3), ThreadPool.Names.GENERIC, pingSender);
+        threadPool.schedule(TimeValue.timeValueMillis(scheduleDuration.millis() / 3 * 2), ThreadPool.Names.GENERIC, pingSender);
+        threadPool.schedule(scheduleDuration, ThreadPool.Names.GENERIC, new AbstractRunnable() {
+            @Override
+            protected void doRun() throws Exception {
+                finishPingingRound(pingingRound);
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                logger.warn("unexpected error while finishing pinging round", e);
+            }
+        });
+    }
+
+    // for testing
+    protected void finishPingingRound(PingingRound pingingRound) {
+        pingingRound.close();
+    }
+
+    protected class PingingRound implements Releasable {
+        private final int id;
+        private final Map<TransportAddress, Connection> tempConnections = new HashMap<>();
+        private final KeyedLock<TransportAddress> connectionLock = new KeyedLock<>(true);
+        private final PingCollection pingCollection;
+        private final List<DiscoveryNode> seedNodes;
+        private final Consumer<PingCollection> pingListener;
+        private final DiscoveryNode localNode;
+        private final ConnectionProfile connectionProfile;
+
+        private AtomicBoolean closed = new AtomicBoolean(false);
+
+        PingingRound(int id, List<DiscoveryNode> seedNodes, Consumer<PingCollection> resultsConsumer, DiscoveryNode localNode,
+                     ConnectionProfile connectionProfile) {
+            this.id = id;
+            this.seedNodes = Collections.unmodifiableList(new ArrayList<>(seedNodes));
+            this.pingListener = resultsConsumer;
+            this.localNode = localNode;
+            this.connectionProfile = connectionProfile;
+            this.pingCollection = new PingCollection();
+        }
+
+        public int id() {
+            return this.id;
+        }
+
+        public boolean isClosed() {
+            return this.closed.get();
+        }
+
+        public List<DiscoveryNode> getSeedNodes() {
+            ensureOpen();
+            return seedNodes;
+        }
+
+        public Connection getOrConnect(DiscoveryNode node) throws IOException {
+            Connection result;
+            try (Releasable ignore = connectionLock.acquire(node.getAddress())) {
+                result = tempConnections.get(node.getAddress());
+                if (result == null) {
+                    ensureOpen();
+                    boolean success = false;
+                    logger.trace("[{}] opening connection to [{}]", id(), node);
+                    result = transportService.openConnection(node, connectionProfile);
+                    try {
+                        transportService.handshake(result, connectionProfile.getHandshakeTimeout().millis());
+                        synchronized (this) {
+                            // acquire lock and check if closed, to prevent leaving an open connection after closing
+                            ensureOpen();
+                            Connection existing = tempConnections.put(node.getAddress(), result);
+                            assert existing == null;
+                            success = true;
+                        }
+                    } finally {
+                        if (success == false) {
+                            logger.trace("[{}] closing connection to [{}] due to failure", id(), node);
+                            IOUtils.closeWhileHandlingException(result);
+                        }
+                    }
+                }
+            }
+            return result;
+        }
+
+        private void ensureOpen() {
+            if (isClosed()) {
+                throw new AlreadyClosedException("pinging round [" + id + "] is finished");
+            }
+        }
+
+        public void addPingResponseToCollection(PingResponse pingResponse) {
+            if (localNode.equals(pingResponse.node()) == false) {
+                pingCollection.addPing(pingResponse);
+            }
+        }
+
+        @Override
+        public void close() {
+            List<Connection> toClose = null;
+            synchronized (this) {
+                if (closed.compareAndSet(false, true)) {
+                    activePingingRounds.remove(id);
+                    toClose = new ArrayList<>(tempConnections.values());
+                    tempConnections.clear();
+                }
+            }
+            if (toClose != null) {
+                // we actually closed
+                try {
+                    pingListener.accept(pingCollection);
+                } finally {
+                    IOUtils.closeWhileHandlingException(toClose);
+                }
+            }
+        }
+
+        public ConnectionProfile getConnectionProfile() {
+            return connectionProfile;
+        }
+    }
+
+
+    protected void sendPings(final TimeValue timeout, final PingingRound pingingRound) {
+        final UnicastPingRequest pingRequest = new UnicastPingRequest();
+        pingRequest.id = pingingRound.id();
+        pingRequest.timeout = timeout;
+        ClusterState lastState = contextProvider.clusterState();
+
+        pingRequest.pingResponse = createPingResponse(lastState);
+
+        Set<DiscoveryNode> nodesFromResponses = temporalResponses.stream().map(pingResponse -> {
+            assert clusterName.equals(pingResponse.clusterName()) :
+                "got a ping request from a different cluster. expected " + clusterName + " got " + pingResponse.clusterName();
+            return pingResponse.node();
+        }).collect(Collectors.toSet());
+
+        // dedup by address
+        final Map<TransportAddress, DiscoveryNode> uniqueNodesByAddress =
+            Stream.concat(pingingRound.getSeedNodes().stream(), nodesFromResponses.stream())
+                .collect(Collectors.toMap(DiscoveryNode::getAddress, Function.identity(), (n1, n2) -> n1));
+
+
+        // resolve what we can via the latest cluster state
+        final Set<DiscoveryNode> nodesToPing = uniqueNodesByAddress.values().stream()
+            .map(node -> {
+                DiscoveryNode foundNode = lastState.nodes().findByAddress(node.getAddress());
+                if (foundNode == null) {
+                    return node;
+                } else {
+                    return foundNode;
+                }
+            }).collect(Collectors.toSet());
+
+        nodesToPing.forEach(node -> sendPingRequestToNode(node, timeout, pingingRound, pingRequest));
+    }
+
+    private void sendPingRequestToNode(final DiscoveryNode node, TimeValue timeout, final PingingRound pingingRound,
+                                       final UnicastPingRequest pingRequest) {
+        submitToExecutor(new AbstractRunnable() {
+            @Override
+            protected void doRun() throws Exception {
+                Connection connection = null;
+                if (transportService.nodeConnected(node)) {
+                    try {
+                        // concurrency can still cause disconnects
+                        connection = transportService.getConnection(node);
+                    } catch (NodeNotConnectedException e) {
+                        logger.trace("[{}] node [{}] just disconnected, will create a temp connection", pingingRound.id(), node);
+                    }
+                }
+
+                if (connection == null) {
+                    connection = pingingRound.getOrConnect(node);
+                }
+
+                logger.trace("[{}] sending to {}", pingingRound.id(), node);
+                transportService.sendRequest(connection, ACTION_NAME, pingRequest,
+                    TransportRequestOptions.builder().withTimeout((long) (timeout.millis() * 1.25)).build(),
+                    getPingResponseHandler(pingingRound, node));
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                if (e instanceof ConnectTransportException || e instanceof AlreadyClosedException) {
+                    // can't connect to the node - this is more common path!
+                    logger.trace(
+                        (Supplier<?>) () -> new ParameterizedMessage(
+                            "[{}] failed to ping {}", pingingRound.id(), node), e);
+                } else if (e instanceof RemoteTransportException) {
+                    // something went wrong on the other side
+                    logger.debug(
+                        (Supplier<?>) () -> new ParameterizedMessage(
+                            "[{}] received a remote error as a response to ping {}", pingingRound.id(), node), e);
+                } else {
+                    logger.warn(
+                        (Supplier<?>) () -> new ParameterizedMessage(
+                            "[{}] failed send ping to {}", pingingRound.id(), node), e);
+                }
+            }
+
+            @Override
+            public void onRejection(Exception e) {
+                // The RejectedExecutionException can come from the fact unicastZenPingExecutorService is at its max down in sendPings
+                // But don't bail here, we can retry later on after the send ping has been scheduled.
+                logger.debug("Ping execution rejected", e);
+            }
+        });
+    }
+
+    // for testing
+    protected void submitToExecutor(AbstractRunnable abstractRunnable) {
+        unicastZenPingExecutorService.execute(abstractRunnable);
+    }
+
+    // for testing
+    protected TransportResponseHandler<UnicastPingResponse> getPingResponseHandler(final PingingRound pingingRound,
+                                                                                   final DiscoveryNode node) {
+        return new TransportResponseHandler<UnicastPingResponse>() {
+
+            @Override
+            public UnicastPingResponse newInstance() {
+                return new UnicastPingResponse();
+            }
+
+            @Override
+            public String executor() {
+                return ThreadPool.Names.SAME;
+            }
+
+            @Override
+            public void handleResponse(UnicastPingResponse response) {
+                logger.trace("[{}] received response from {}: {}", pingingRound.id(), node, Arrays.toString(response.pingResponses));
+                if (pingingRound.isClosed()) {
+                    if (logger.isTraceEnabled()) {
+                        logger.trace("[{}] skipping received response from {}. already closed", pingingRound.id(), node);
+                    }
+                } else {
+                    Stream.of(response.pingResponses).forEach(pingingRound::addPingResponseToCollection);
+                }
+            }
+
+            @Override
+            public void handleException(TransportException exp) {
+                if (exp instanceof ConnectTransportException || exp.getCause() instanceof ConnectTransportException) {
+                    // ok, not connected...
+                    logger.trace((Supplier<?>) () -> new ParameterizedMessage("failed to connect to {}", node), exp);
+                } else if (closed == false) {
+                    logger.warn((Supplier<?>) () -> new ParameterizedMessage("failed to send ping to [{}]", node), exp);
+                }
+            }
+        };
+    }
+
+    private UnicastPingResponse handlePingRequest(final UnicastPingRequest request) {
+        assert clusterName.equals(request.pingResponse.clusterName()) :
+            "got a ping request from a different cluster. expected " + clusterName + " got " + request.pingResponse.clusterName();
+        temporalResponses.add(request.pingResponse);
+        // add to any ongoing pinging
+        activePingingRounds.values().forEach(p -> p.addPingResponseToCollection(request.pingResponse));
+        threadPool.schedule(TimeValue.timeValueMillis(request.timeout.millis() * 2), ThreadPool.Names.SAME,
+            () -> temporalResponses.remove(request.pingResponse));
+
+        List<PingResponse> pingResponses = CollectionUtils.iterableAsArrayList(temporalResponses);
+        pingResponses.add(createPingResponse(contextProvider.clusterState()));
+
+        UnicastPingResponse unicastPingResponse = new UnicastPingResponse();
+        unicastPingResponse.id = request.id;
+        unicastPingResponse.pingResponses = pingResponses.toArray(new PingResponse[pingResponses.size()]);
+
+        return unicastPingResponse;
+    }
+
+    class UnicastPingRequestHandler implements TransportRequestHandler<UnicastPingRequest> {
+
+        @Override
+        public void messageReceived(UnicastPingRequest request, TransportChannel channel) throws Exception {
+            if (request.pingResponse.clusterName().equals(clusterName)) {
+                channel.sendResponse(handlePingRequest(request));
+            } else {
+                throw new IllegalStateException(
+                    String.format(
+                        Locale.ROOT,
+                        "mismatched cluster names; request: [%s], local: [%s]",
+                        request.pingResponse.clusterName().value(),
+                        clusterName.value()));
+            }
+        }
+
+    }
+
+    public static class UnicastPingRequest extends TransportRequest {
+
+        int id;
+        TimeValue timeout;
+        PingResponse pingResponse;
+
+        public UnicastPingRequest() {
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            super.readFrom(in);
+            id = in.readInt();
+            timeout = new TimeValue(in);
+            pingResponse = readPingResponse(in);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeInt(id);
+            timeout.writeTo(out);
+            pingResponse.writeTo(out);
+        }
+    }
+
+    private PingResponse createPingResponse(ClusterState clusterState) {
+        DiscoveryNodes discoNodes = clusterState.nodes();
+        return new PingResponse(discoNodes.getLocalNode(), discoNodes.getMasterNode(), clusterState);
+    }
+
+    static class UnicastPingResponse extends TransportResponse {
+
+        int id;
+
+        PingResponse[] pingResponses;
+
+        UnicastPingResponse() {
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            super.readFrom(in);
+            id = in.readInt();
+            pingResponses = new PingResponse[in.readVInt()];
+            for (int i = 0; i < pingResponses.length; i++) {
+                pingResponses[i] = readPingResponse(in);
+            }
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeInt(id);
+            out.writeVInt(pingResponses.length);
+            for (PingResponse pingResponse : pingResponses) {
+                pingResponse.writeTo(out);
+            }
+        }
+    }
+
+    protected Version getVersion() {
+        return Version.CURRENT; // for tests
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java b/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
index c4fc4f15f4067..8ea0c2e42c629 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
@@ -22,49 +22,40 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
+import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateTaskConfig;
 import org.elasticsearch.cluster.ClusterStateTaskExecutor;
 import org.elasticsearch.cluster.ClusterStateTaskListener;
-import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.LocalClusterUpdateTask;
 import org.elasticsearch.cluster.NotMasterException;
-import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
-import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.component.Lifecycle;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.inject.internal.Nullable;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.lease.Releasables;
+import org.elasticsearch.common.logging.LoggerMessageFormat;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.discovery.Discovery;
 import org.elasticsearch.discovery.DiscoverySettings;
 import org.elasticsearch.discovery.DiscoveryStats;
-import org.elasticsearch.discovery.zen.elect.ElectMasterService;
-import org.elasticsearch.discovery.zen.fd.MasterFaultDetection;
-import org.elasticsearch.discovery.zen.fd.NodesFaultDetection;
-import org.elasticsearch.discovery.zen.membership.MembershipAction;
-import org.elasticsearch.discovery.zen.ping.PingContextProvider;
-import org.elasticsearch.discovery.zen.ping.ZenPing;
-import org.elasticsearch.discovery.zen.ping.ZenPingService;
-import org.elasticsearch.discovery.zen.publish.PendingClusterStateStats;
-import org.elasticsearch.discovery.zen.publish.PublishClusterStateAction;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.EmptyTransportResponseHandler;
 import org.elasticsearch.transport.TransportChannel;
@@ -76,15 +67,14 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
-import java.util.function.BiFunction;
+import java.util.function.Consumer;
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
@@ -95,7 +85,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
         Setting.positiveTimeSetting("discovery.zen.ping_timeout", timeValueSeconds(3), Property.NodeScope);
     public static final Setting<TimeValue> JOIN_TIMEOUT_SETTING =
         Setting.timeSetting("discovery.zen.join_timeout",
-            settings -> TimeValue.timeValueMillis(PING_TIMEOUT_SETTING.get(settings).millis() * 20).toString(),
+            settings -> TimeValue.timeValueMillis(PING_TIMEOUT_SETTING.get(settings).millis() * 20),
             TimeValue.timeValueMillis(0), Property.NodeScope);
     public static final Setting<Integer> JOIN_RETRY_ATTEMPTS_SETTING =
         Setting.intSetting("discovery.zen.join_retry_attempts", 3, 1, Property.NodeScope);
@@ -107,7 +97,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
         Setting.boolSetting("discovery.zen.send_leave_request", true, Property.NodeScope);
     public static final Setting<TimeValue> MASTER_ELECTION_WAIT_FOR_JOINS_TIMEOUT_SETTING =
         Setting.timeSetting("discovery.zen.master_election.wait_for_joins_timeout",
-            settings -> TimeValue.timeValueMillis(JOIN_TIMEOUT_SETTING.get(settings).millis() / 2).toString(), TimeValue.timeValueMillis(0),
+            settings -> TimeValue.timeValueMillis(JOIN_TIMEOUT_SETTING.get(settings).millis() / 2), TimeValue.timeValueMillis(0),
             Property.NodeScope);
     public static final Setting<Boolean> MASTER_ELECTION_IGNORE_NON_MASTER_PINGS_SETTING =
             Setting.boolSetting("discovery.zen.master_election.ignore_non_master_pings", false, Property.NodeScope);
@@ -115,15 +105,17 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
     public static final String DISCOVERY_REJOIN_ACTION_NAME = "internal:discovery/zen/rejoin";
 
     private final TransportService transportService;
+    private final NamedWriteableRegistry namedWriteableRegistry;
     private final ClusterService clusterService;
     private AllocationService allocationService;
     private final ClusterName clusterName;
     private final DiscoverySettings discoverySettings;
-    private final ZenPingService pingService;
+    protected final ZenPing zenPing; // protected to allow tests access
     private final MasterFaultDetection masterFD;
     private final NodesFaultDetection nodesFD;
     private final PublishClusterStateAction publishClusterState;
     private final MembershipAction membership;
+    private final ThreadPool threadPool;
 
     private final TimeValue pingTimeout;
     private final TimeValue joinTimeout;
@@ -146,31 +138,28 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
 
     private final JoinThreadControl joinThreadControl;
 
-    /** counts the time this node has joined the cluster or have elected it self as master */
-    private final AtomicLong clusterJoinsCounter = new AtomicLong();
-
     // must initialized in doStart(), when we have the allocationService set
     private volatile NodeJoinController nodeJoinController;
     private volatile NodeRemovalClusterStateTaskExecutor nodeRemovalExecutor;
 
-    @Inject
-    public ZenDiscovery(Settings settings, ThreadPool threadPool,
-                        TransportService transportService, final ClusterService clusterService, ClusterSettings clusterSettings,
-                        ZenPingService pingService, ElectMasterService electMasterService) {
+    public ZenDiscovery(Settings settings, ThreadPool threadPool, TransportService transportService,
+                        NamedWriteableRegistry namedWriteableRegistry,
+                        ClusterService clusterService, UnicastHostsProvider hostsProvider) {
         super(settings);
         this.clusterService = clusterService;
         this.clusterName = clusterService.getClusterName();
         this.transportService = transportService;
-        this.discoverySettings = new DiscoverySettings(settings, clusterSettings);
-        this.pingService = pingService;
-        this.electMaster = electMasterService;
+        this.namedWriteableRegistry = namedWriteableRegistry;
+        this.discoverySettings = new DiscoverySettings(settings, clusterService.getClusterSettings());
+        this.zenPing = newZenPing(settings, threadPool, transportService, hostsProvider);
+        this.electMaster = new ElectMasterService(settings);
         this.pingTimeout = PING_TIMEOUT_SETTING.get(settings);
-
         this.joinTimeout = JOIN_TIMEOUT_SETTING.get(settings);
         this.joinRetryAttempts = JOIN_RETRY_ATTEMPTS_SETTING.get(settings);
         this.joinRetryDelay = JOIN_RETRY_DELAY_SETTING.get(settings);
         this.maxPingsFromAnotherMaster = MAX_PINGS_FROM_ANOTHER_MASTER_SETTING.get(settings);
         this.sendLeaveRequest = SEND_LEAVE_REQUEST_SETTING.get(settings);
+        this.threadPool = threadPool;
 
         this.masterElectionIgnoreNonMasters = MASTER_ELECTION_IGNORE_NON_MASTER_PINGS_SETTING.get(settings);
         this.masterElectionWaitForJoinsTimeout = MASTER_ELECTION_WAIT_FOR_JOINS_TIMEOUT_SETTING.get(settings);
@@ -178,17 +167,29 @@ public ZenDiscovery(Settings settings, ThreadPool threadPool,
         logger.debug("using ping_timeout [{}], join.timeout [{}], master_election.ignore_non_master [{}]",
                 this.pingTimeout, joinTimeout, masterElectionIgnoreNonMasters);
 
-        clusterSettings.addSettingsUpdateConsumer(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING, this::handleMinimumMasterNodesChanged, (value) -> {
-            final ClusterState clusterState = clusterService.state();
-            int masterNodes = clusterState.nodes().getMasterNodes().size();
-            if (value > masterNodes) {
-                throw new IllegalArgumentException("cannot set " + ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey() + " to more than the current master nodes count [" + masterNodes + "]");
-            }
+        clusterService.getClusterSettings().addSettingsUpdateConsumer(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING,
+            this::handleMinimumMasterNodesChanged, (value) -> {
+                final ClusterState clusterState = clusterService.state();
+                int masterNodes = clusterState.nodes().getMasterNodes().size();
+                // the purpose of this validation is to make sure that the master doesn't step down
+                // due to a change in master nodes, which also means that there is no way to revert
+                // an accidental change. Since we validate using the current cluster state (and
+                // not the one from which the settings come from) we have to be careful and only
+                // validate if the local node is already a master. Doing so all the time causes
+                // subtle issues. For example, a node that joins a cluster has no nodes in its
+                // current cluster state. When it receives a cluster state from the master with
+                // a dynamic minimum master nodes setting int it, we must make sure we don't reject
+                // it.
+
+                if (clusterState.nodes().isLocalNodeElectedMaster() && value > masterNodes) {
+                    throw new IllegalArgumentException("cannot set "
+                        + ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey() + " to more than the current" +
+                        " master nodes count [" + masterNodes + "]");
+                }
         });
 
         this.masterFD = new MasterFaultDetection(settings, threadPool, transportService, clusterService);
         this.masterFD.addListener(new MasterNodeFailureListener());
-
         this.nodesFD = new NodesFaultDetection(settings, threadPool, transportService, clusterService.getClusterName());
         this.nodesFD.addListener(new NodeFaultDetectionListener());
 
@@ -196,19 +197,24 @@ public ZenDiscovery(Settings settings, ThreadPool threadPool,
                 new PublishClusterStateAction(
                         settings,
                         transportService,
+                        namedWriteableRegistry,
                         clusterService::state,
                         new NewPendingClusterStateListener(),
                         discoverySettings,
                         clusterService.getClusterName());
-        this.pingService.setPingContextProvider(this);
-        this.membership = new MembershipAction(settings, transportService, this, new MembershipListener());
-
-        this.joinThreadControl = new JoinThreadControl(threadPool);
+        this.membership = new MembershipAction(settings, transportService, new MembershipListener());
+        this.joinThreadControl = new JoinThreadControl();
 
         transportService.registerRequestHandler(
             DISCOVERY_REJOIN_ACTION_NAME, RejoinClusterRequest::new, ThreadPool.Names.SAME, new RejoinClusterRequestHandler());
     }
 
+    // protected to allow overriding in tests
+    protected ZenPing newZenPing(Settings settings, ThreadPool threadPool, TransportService transportService,
+                                 UnicastHostsProvider hostsProvider) {
+        return new UnicastZenPing(settings, threadPool, transportService, hostsProvider);
+    }
+
     @Override
     public void setAllocationService(AllocationService allocationService) {
         this.allocationService = allocationService;
@@ -218,26 +224,21 @@ public void setAllocationService(AllocationService allocationService) {
     protected void doStart() {
         nodesFD.setLocalNode(clusterService.localNode());
         joinThreadControl.start();
-        pingService.start();
-        this.nodeJoinController = new NodeJoinController(clusterService, allocationService, electMaster, discoverySettings, settings);
-        this.nodeRemovalExecutor = new NodeRemovalClusterStateTaskExecutor(allocationService, electMaster, this::rejoin, logger);
+        zenPing.start(this);
+        this.nodeJoinController = new NodeJoinController(clusterService, allocationService, electMaster, settings);
+        this.nodeRemovalExecutor = new NodeRemovalClusterStateTaskExecutor(allocationService, electMaster, this::submitRejoin, logger);
     }
 
     @Override
     public void startInitialJoin() {
         // start the join thread from a cluster state update. See {@link JoinThreadControl} for details.
-        clusterService.submitStateUpdateTask("initial_join", new ClusterStateUpdateTask() {
-
-            @Override
-            public boolean runOnlyOnMaster() {
-                return false;
-            }
+        clusterService.submitStateUpdateTask("initial_join", new LocalClusterUpdateTask() {
 
             @Override
-            public ClusterState execute(ClusterState currentState) throws Exception {
+            public ClusterTasksResult<LocalClusterUpdateTask> execute(ClusterState currentState) throws Exception {
                 // do the join on a different thread, the DiscoveryService waits for 30s anyhow till it is discovered
                 joinThreadControl.startNewThreadIfNotRunning();
-                return currentState;
+                return unchanged();
             }
 
             @Override
@@ -250,10 +251,10 @@ public void onFailure(String source, @org.elasticsearch.common.Nullable Exceptio
     @Override
     protected void doStop() {
         joinThreadControl.stop();
-        pingService.stop();
         masterFD.stop("zen disco stop");
         nodesFD.stop();
-        DiscoveryNodes nodes = nodes();
+        Releasables.close(zenPing); // stop any ongoing pinging
+        DiscoveryNodes nodes = clusterState().nodes();
         if (sendLeaveRequest) {
             if (nodes.getMasterNode() == null) {
                 // if we don't know who the master is, nothing to do here
@@ -281,12 +282,8 @@ protected void doStop() {
     }
 
     @Override
-    protected void doClose() {
-        masterFD.close();
-        nodesFD.close();
-        publishClusterState.close();
-        membership.close();
-        pingService.close();
+    protected void doClose() throws IOException {
+        IOUtils.close(masterFD, nodesFD);
     }
 
     @Override
@@ -299,45 +296,40 @@ public String nodeDescription() {
         return clusterName.value() + "/" + clusterService.localNode().getId();
     }
 
-    /** start of {@link org.elasticsearch.discovery.zen.ping.PingContextProvider } implementation */
-    @Override
-    public DiscoveryNodes nodes() {
-        return clusterService.state().nodes();
-    }
-
     @Override
-    public boolean nodeHasJoinedClusterOnce() {
-        return clusterJoinsCounter.get() > 0;
+    public ClusterState clusterState() {
+        return clusterService.state();
     }
 
-    /** end of {@link org.elasticsearch.discovery.zen.ping.PingContextProvider } implementation */
-
-
     @Override
     public void publish(ClusterChangedEvent clusterChangedEvent, AckListener ackListener) {
         if (!clusterChangedEvent.state().getNodes().isLocalNodeElectedMaster()) {
             throw new IllegalStateException("Shouldn't publish state when not master");
         }
-        nodesFD.updateNodesAndPing(clusterChangedEvent.state());
         try {
             publishClusterState.publish(clusterChangedEvent, electMaster.minimumMasterNodes(), ackListener);
         } catch (FailedToCommitClusterStateException t) {
             // cluster service logs a WARN message
             logger.debug("failed to publish cluster state version [{}] (not enough nodes acknowledged, min master nodes [{}])", clusterChangedEvent.state().version(), electMaster.minimumMasterNodes());
-            clusterService.submitStateUpdateTask("zen-disco-failed-to-publish", new ClusterStateUpdateTask(Priority.IMMEDIATE) {
-                @Override
-                public ClusterState execute(ClusterState currentState) {
-                    return rejoin(currentState, "failed to publish to min_master_nodes");
-                }
-
-                @Override
-                public void onFailure(String source, Exception e) {
-                    logger.error((Supplier<?>) () -> new ParameterizedMessage("unexpected failure during [{}]", source), e);
-                }
-
-            });
+            submitRejoin("zen-disco-failed-to-publish");
             throw t;
         }
+
+        // update the set of nodes to ping after the new cluster state has been published
+        nodesFD.updateNodesAndPing(clusterChangedEvent.state());
+
+        // clean the pending cluster queue - we are currently master, so any pending cluster state should be failed
+        // note that we also clean the queue on master failure (see handleMasterGone) but a delayed cluster state publish
+        // from a stale master can still make it in the queue during the election (but not be committed)
+        publishClusterState.pendingStatesQueue().failAllStatesAndClear(new ElasticsearchException("elected as master"));
+    }
+
+    /**
+     * Gets the current set of nodes involved in the node fault detection.
+     * NB: for testing purposes
+     */
+    public Set<DiscoveryNode> getFaultDetectionNodes() {
+        return nodesFD.getNodes();
     }
 
     @Override
@@ -364,12 +356,15 @@ public boolean joiningCluster() {
         return joinThreadControl.joinThreadActive();
     }
 
-
     // used for testing
     public ClusterState[] pendingClusterStates() {
         return publishClusterState.pendingStatesQueue().pendingClusterStates();
     }
 
+    PendingClusterStatesQueue pendingClusterStatesQueue() {
+        return publishClusterState.pendingStatesQueue();
+    }
+
     /**
      * the main function of a join thread. This function is guaranteed to join the cluster
      * or spawn a new join thread upon failure to do so.
@@ -397,8 +392,6 @@ public void onElectedAsMaster(ClusterState state) {
                             joinThreadControl.markThreadAsDone(currentThread);
                             // we only starts nodesFD if we are master (it may be that we received a cluster state while pinging)
                             nodesFD.updateNodesAndPing(state); // start the nodes FD
-                            long count = clusterJoinsCounter.incrementAndGet();
-                            logger.trace("cluster joins counter set to [{}] (elected as master)", count);
                         }
 
                         @Override
@@ -418,18 +411,13 @@ public void onFailure(Throwable t) {
 
             // finalize join through the cluster state update thread
             final DiscoveryNode finalMasterNode = masterNode;
-            clusterService.submitStateUpdateTask("finalize_join (" + masterNode + ")", new ClusterStateUpdateTask() {
+            clusterService.submitStateUpdateTask("finalize_join (" + masterNode + ")", new LocalClusterUpdateTask() {
                 @Override
-                public boolean runOnlyOnMaster() {
-                    return false;
-                }
-
-                @Override
-                public ClusterState execute(ClusterState currentState) throws Exception {
+                public ClusterTasksResult<LocalClusterUpdateTask> execute(ClusterState currentState) throws Exception {
                     if (!success) {
                         // failed to join. Try again...
                         joinThreadControl.markThreadAsDoneAndStartNew(currentThread);
-                        return currentState;
+                        return unchanged();
                     }
 
                     if (currentState.getNodes().getMasterNode() == null) {
@@ -437,7 +425,7 @@ public ClusterState execute(ClusterState currentState) throws Exception {
                         // a valid master.
                         logger.debug("no master node is set, despite of join request completing. retrying pings.");
                         joinThreadControl.markThreadAsDoneAndStartNew(currentThread);
-                        return currentState;
+                        return unchanged();
                     }
 
                     if (!currentState.getNodes().getMasterNode().equals(finalMasterNode)) {
@@ -447,7 +435,7 @@ public ClusterState execute(ClusterState currentState) throws Exception {
                     // Note: we do not have to start master fault detection here because it's set at {@link #processNextPendingClusterState }
                     // when the first cluster state arrives.
                     joinThreadControl.markThreadAsDone(currentThread);
-                    return currentState;
+                    return unchanged();
                 }
 
                 @Override
@@ -505,12 +493,27 @@ private boolean joinElectedMaster(DiscoveryNode masterNode) {
         }
     }
 
+    private void submitRejoin(String source) {
+        clusterService.submitStateUpdateTask(source, new LocalClusterUpdateTask(Priority.IMMEDIATE) {
+            @Override
+            public ClusterTasksResult<LocalClusterUpdateTask> execute(ClusterState currentState) {
+                return rejoin(currentState, source);
+            }
+
+            @Override
+            public void onFailure(String source, Exception e) {
+                logger.error((Supplier<?>) () -> new ParameterizedMessage("unexpected failure during [{}]", source), e);
+            }
+
+        });
+    }
+
     // visible for testing
     static class NodeRemovalClusterStateTaskExecutor implements ClusterStateTaskExecutor<NodeRemovalClusterStateTaskExecutor.Task>, ClusterStateTaskListener {
 
         private final AllocationService allocationService;
         private final ElectMasterService electMasterService;
-        private final BiFunction<ClusterState, String, ClusterState> rejoin;
+        private final Consumer<String> rejoin;
         private final Logger logger;
 
         static class Task {
@@ -518,7 +521,7 @@ static class Task {
             private final DiscoveryNode node;
             private final String reason;
 
-            public Task(final DiscoveryNode node, final String reason) {
+            Task(final DiscoveryNode node, final String reason) {
                 this.node = node;
                 this.reason = reason;
             }
@@ -540,7 +543,7 @@ public String toString() {
         NodeRemovalClusterStateTaskExecutor(
                 final AllocationService allocationService,
                 final ElectMasterService electMasterService,
-                final BiFunction<ClusterState, String, ClusterState> rejoin,
+                final Consumer<String> rejoin,
                 final Logger logger) {
             this.allocationService = allocationService;
             this.electMasterService = electMasterService;
@@ -549,7 +552,7 @@ public String toString() {
         }
 
         @Override
-        public BatchResult<Task> execute(final ClusterState currentState, final List<Task> tasks) throws Exception {
+        public ClusterTasksResult<Task> execute(final ClusterState currentState, final List<Task> tasks) throws Exception {
             final DiscoveryNodes.Builder remainingNodesBuilder = DiscoveryNodes.builder(currentState.nodes());
             boolean removed = false;
             for (final Task task : tasks) {
@@ -563,18 +566,19 @@ public BatchResult<Task> execute(final ClusterState currentState, final List<Tas
 
             if (!removed) {
                 // no nodes to remove, keep the current cluster state
-                return BatchResult.<Task>builder().successes(tasks).build(currentState);
+                return ClusterTasksResult.<Task>builder().successes(tasks).build(currentState);
             }
 
             final ClusterState remainingNodesClusterState = remainingNodesClusterState(currentState, remainingNodesBuilder);
 
-            final BatchResult.Builder<Task> resultBuilder = BatchResult.<Task>builder().successes(tasks);
-            if (!electMasterService.hasEnoughMasterNodes(remainingNodesClusterState.nodes())) {
-                return resultBuilder.build(rejoin.apply(remainingNodesClusterState, "not enough master nodes"));
+            final ClusterTasksResult.Builder<Task> resultBuilder = ClusterTasksResult.<Task>builder().successes(tasks);
+            if (electMasterService.hasEnoughMasterNodes(remainingNodesClusterState.nodes()) == false) {
+                final int masterNodes = electMasterService.countMasterNodes(remainingNodesClusterState.nodes());
+                rejoin.accept(LoggerMessageFormat.format("not enough master nodes (has [{}], but needed [{}])",
+                                                         masterNodes, electMasterService.minimumMasterNodes()));
+                return resultBuilder.build(currentState);
             } else {
-                final RoutingAllocation.Result routingResult =
-                    allocationService.deassociateDeadNodes(remainingNodesClusterState, true, describeTasks(tasks));
-                return resultBuilder.build(ClusterState.builder(remainingNodesClusterState).routingResult(routingResult).build());
+                return resultBuilder.build(allocationService.deassociateDeadNodes(remainingNodesClusterState, true, describeTasks(tasks)));
             }
         }
 
@@ -613,7 +617,7 @@ private void handleLeaveRequest(final DiscoveryNode node) {
         }
         if (localNodeMaster()) {
             removeNode(node, "zen-disco-node-left", "left");
-        } else if (node.equals(nodes().getMasterNode())) {
+        } else if (node.equals(clusterState().nodes().getMasterNode())) {
             handleMasterGone(node, null, "shut_down");
         }
     }
@@ -641,14 +645,14 @@ private void handleMinimumMasterNodesChanged(final int minimumMasterNodes) {
             // We only set the new value. If the master doesn't see enough nodes it will revoke it's mastership.
             return;
         }
-        clusterService.submitStateUpdateTask("zen-disco-mini-master-nodes-changed", new ClusterStateUpdateTask(Priority.IMMEDIATE) {
+        clusterService.submitStateUpdateTask("zen-disco-min-master-nodes-changed", new LocalClusterUpdateTask(Priority.IMMEDIATE) {
             @Override
-            public ClusterState execute(ClusterState currentState) {
+            public ClusterTasksResult<LocalClusterUpdateTask> execute(ClusterState currentState) {
                 // check if we have enough master nodes, if not, we need to move into joining the cluster again
                 if (!electMaster.hasEnoughMasterNodes(currentState.nodes())) {
                     return rejoin(currentState, "not enough master nodes on change of minimum_master_nodes from [" + prevMinimumMasterNode + "] to [" + minimumMasterNodes + "]");
                 }
-                return currentState;
+                return unchanged();
             }
 
 
@@ -681,29 +685,19 @@ private void handleMasterGone(final DiscoveryNode masterNode, final Throwable ca
 
         logger.info((Supplier<?>) () -> new ParameterizedMessage("master_left [{}], reason [{}]", masterNode, reason), cause);
 
-        clusterService.submitStateUpdateTask("master_failed (" + masterNode + ")", new ClusterStateUpdateTask(Priority.IMMEDIATE) {
+        clusterService.submitStateUpdateTask("master_failed (" + masterNode + ")", new LocalClusterUpdateTask(Priority.IMMEDIATE) {
 
             @Override
-            public boolean runOnlyOnMaster() {
-                return false;
-            }
-
-            @Override
-            public ClusterState execute(ClusterState currentState) {
+            public ClusterTasksResult<LocalClusterUpdateTask> execute(ClusterState currentState) {
                 if (!masterNode.equals(currentState.nodes().getMasterNode())) {
                     // master got switched on us, no need to send anything
-                    return currentState;
+                    return unchanged();
                 }
 
-                DiscoveryNodes discoveryNodes = DiscoveryNodes.builder(currentState.nodes())
-                        // make sure the old master node, which has failed, is not part of the nodes we publish
-                        .remove(masterNode)
-                        .masterNodeId(null).build();
-
                 // flush any pending cluster states from old master, so it will not be set as master again
                 publishClusterState.pendingStatesQueue().failAllStatesAndClear(new ElasticsearchException("master left [{}]", reason));
 
-                return rejoin(ClusterState.builder(currentState).nodes(discoveryNodes).build(), "master left (reason = " + reason + ")");
+                return rejoin(currentState, "master left (reason = " + reason + ")");
             }
 
             @Override
@@ -711,29 +705,20 @@ public void onFailure(String source, Exception e) {
                 logger.error((Supplier<?>) () -> new ParameterizedMessage("unexpected failure during [{}]", source), e);
             }
 
-            @Override
-            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
-            }
-
         });
     }
 
     void processNextPendingClusterState(String reason) {
-        clusterService.submitStateUpdateTask("zen-disco-receive(from master [" + reason + "])", new ClusterStateUpdateTask(Priority.URGENT) {
-            @Override
-            public boolean runOnlyOnMaster() {
-                return false;
-            }
-
+        clusterService.submitStateUpdateTask("zen-disco-receive(from master [" + reason + "])", new LocalClusterUpdateTask(Priority.URGENT) {
             ClusterState newClusterState = null;
 
             @Override
-            public ClusterState execute(ClusterState currentState) {
+            public ClusterTasksResult<LocalClusterUpdateTask> execute(ClusterState currentState) {
                 newClusterState = publishClusterState.pendingStatesQueue().getNextClusterStateToProcess();
 
                 // all pending states have been processed
                 if (newClusterState == null) {
-                    return currentState;
+                    return unchanged();
                 }
 
                 assert newClusterState.nodes().getMasterNode() != null : "received a cluster state without a master";
@@ -744,21 +729,12 @@ public ClusterState execute(ClusterState currentState) {
                 }
 
                 if (shouldIgnoreOrRejectNewClusterState(logger, currentState, newClusterState)) {
-                    return currentState;
-                }
-
-                // check to see that we monitor the correct master of the cluster
-                if (masterFD.masterNode() == null || !masterFD.masterNode().equals(newClusterState.nodes().getMasterNode())) {
-                    masterFD.restart(newClusterState.nodes().getMasterNode(), "new cluster state received and we are monitoring the wrong master [" + masterFD.masterNode() + "]");
+                    return unchanged();
                 }
-
                 if (currentState.blocks().hasGlobalBlock(discoverySettings.getNoMasterBlock())) {
                     // its a fresh update from the master as we transition from a start of not having a master to having one
                     logger.debug("got first state from fresh master [{}]", newClusterState.nodes().getMasterNodeId());
-                    long count = clusterJoinsCounter.incrementAndGet();
-                    logger.trace("updated cluster join cluster to [{}]", count);
-
-                    return newClusterState;
+                    return newState(newClusterState);
                 }
 
 
@@ -788,7 +764,7 @@ public ClusterState execute(ClusterState currentState) {
                     builder.metaData(metaDataBuilder);
                 }
 
-                return builder.build();
+                return newState(builder.build());
             }
 
             @Override
@@ -808,6 +784,10 @@ public void onFailure(String source, Exception e) {
             public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
                 try {
                     if (newClusterState != null) {
+                        // check to see that we monitor the correct master of the cluster
+                        if (masterFD.masterNode() == null || !masterFD.masterNode().equals(newClusterState.nodes().getMasterNode())) {
+                            masterFD.restart(newClusterState.nodes().getMasterNode(), "new cluster state received and we are monitoring the wrong master [" + masterFD.masterNode() + "]");
+                        }
                         publishClusterState.pendingStatesQueue().markAsProcessed(newClusterState);
                     }
                 } catch (Exception e) {
@@ -864,16 +844,10 @@ void handleJoinRequest(final DiscoveryNode node, final ClusterState state, final
         } else if (nodeJoinController == null) {
             throw new IllegalStateException("discovery module is not yet started");
         } else {
-            // The minimum supported version for a node joining a master:
-            Version minimumNodeJoinVersion = localNode().getVersion().minimumCompatibilityVersion();
-            // Sanity check: maybe we don't end up here, because serialization may have failed.
-            if (node.getVersion().before(minimumNodeJoinVersion)) {
-                callback.onFailure(
-                        new IllegalStateException("Can't handle join request from a node with a version [" + node.getVersion() + "] that is lower than the minimum compatible version [" + minimumNodeJoinVersion.minimumCompatibilityVersion() + "]")
-                );
-                return;
-            }
-
+            // we do this in a couple of places including the cluster update thread. This one here is really just best effort
+            // to ensure we fail as fast as possible.
+            MembershipAction.ensureNodesCompatibility(node.getVersion(), state.getNodes());
+            MembershipAction.ensureIndexCompatibility(node.getVersion(), state.getMetaData());
             // try and connect to the node, if it fails, we can raise an exception back to the client...
             transportService.connectToNode(node);
 
@@ -892,14 +866,14 @@ void handleJoinRequest(final DiscoveryNode node, final ClusterState state, final
 
     private DiscoveryNode findMaster() {
         logger.trace("starting to ping");
-        ZenPing.PingResponse[] fullPingResponses = pingService.pingAndWait(pingTimeout);
+        List<ZenPing.PingResponse> fullPingResponses = pingAndWait(pingTimeout).toList();
         if (fullPingResponses == null) {
             logger.trace("No full ping responses");
             return null;
         }
         if (logger.isTraceEnabled()) {
             StringBuilder sb = new StringBuilder();
-            if (fullPingResponses.length == 0) {
+            if (fullPingResponses.size() == 0) {
                 sb.append(" {none}");
             } else {
                 for (ZenPing.PingResponse pingResponse : fullPingResponses) {
@@ -909,69 +883,58 @@ private DiscoveryNode findMaster() {
             logger.trace("full ping responses:{}", sb);
         }
 
+        final DiscoveryNode localNode = clusterService.localNode();
+
+        // add our selves
+        assert fullPingResponses.stream().map(ZenPing.PingResponse::node)
+            .filter(n -> n.equals(localNode)).findAny().isPresent() == false;
+
+        fullPingResponses.add(new ZenPing.PingResponse(localNode, null, clusterService.state()));
+
         // filter responses
         final List<ZenPing.PingResponse> pingResponses = filterPingResponses(fullPingResponses, masterElectionIgnoreNonMasters, logger);
 
-        final DiscoveryNode localNode = clusterService.localNode();
-        List<DiscoveryNode> pingMasters = new ArrayList<>();
+        List<DiscoveryNode> activeMasters = new ArrayList<>();
         for (ZenPing.PingResponse pingResponse : pingResponses) {
-            if (pingResponse.master() != null) {
-                // We can't include the local node in pingMasters list, otherwise we may up electing ourselves without
-                // any check / verifications from other nodes in ZenDiscover#innerJoinCluster()
-                if (!localNode.equals(pingResponse.master())) {
-                    pingMasters.add(pingResponse.master());
-                }
+            // We can't include the local node in pingMasters list, otherwise we may up electing ourselves without
+            // any check / verifications from other nodes in ZenDiscover#innerJoinCluster()
+            if (pingResponse.master() != null && !localNode.equals(pingResponse.master())) {
+                activeMasters.add(pingResponse.master());
             }
         }
 
         // nodes discovered during pinging
-        Set<DiscoveryNode> activeNodes = new HashSet<>();
-        // nodes discovered who has previously been part of the cluster and do not ping for the very first time
-        Set<DiscoveryNode> joinedOnceActiveNodes = new HashSet<>();
-        if (localNode.isMasterNode()) {
-            activeNodes.add(localNode);
-            long joinsCounter = clusterJoinsCounter.get();
-            if (joinsCounter > 0) {
-                logger.trace("adding local node to the list of active nodes that have previously joined the cluster (joins counter is [{}])", joinsCounter);
-                joinedOnceActiveNodes.add(localNode);
-            }
-        }
+        List<ElectMasterService.MasterCandidate> masterCandidates = new ArrayList<>();
         for (ZenPing.PingResponse pingResponse : pingResponses) {
-            activeNodes.add(pingResponse.node());
-            if (pingResponse.hasJoinedOnce()) {
-                joinedOnceActiveNodes.add(pingResponse.node());
+            if (pingResponse.node().isMasterNode()) {
+                masterCandidates.add(new ElectMasterService.MasterCandidate(pingResponse.node(), pingResponse.getClusterStateVersion()));
             }
         }
 
-        if (pingMasters.isEmpty()) {
-            if (electMaster.hasEnoughMasterNodes(activeNodes)) {
-                // we give preference to nodes who have previously already joined the cluster. Those will
-                // have a cluster state in memory, including an up to date routing table (which is not persistent to disk
-                // by the gateway)
-                DiscoveryNode master = electMaster.electMaster(joinedOnceActiveNodes);
-                if (master != null) {
-                    return master;
-                }
-                return electMaster.electMaster(activeNodes);
+        if (activeMasters.isEmpty()) {
+            if (electMaster.hasEnoughCandidates(masterCandidates)) {
+                final ElectMasterService.MasterCandidate winner = electMaster.electMaster(masterCandidates);
+                logger.trace("candidate {} won election", winner);
+                return winner.getNode();
             } else {
                 // if we don't have enough master nodes, we bail, because there are not enough master to elect from
-                logger.trace("not enough master nodes [{}]", activeNodes);
+                logger.warn("not enough master nodes discovered during pinging (found [{}], but needed [{}]), pinging again",
+                            masterCandidates, electMaster.minimumMasterNodes());
                 return null;
             }
         } else {
-
-            assert !pingMasters.contains(localNode) : "local node should never be elected as master when other nodes indicate an active master";
+            assert !activeMasters.contains(localNode) : "local node should never be elected as master when other nodes indicate an active master";
             // lets tie break between discovered nodes
-            return electMaster.electMaster(pingMasters);
+            return electMaster.tieBreakActiveMasters(activeMasters);
         }
     }
 
-    static List<ZenPing.PingResponse> filterPingResponses(ZenPing.PingResponse[] fullPingResponses, boolean masterElectionIgnoreNonMasters, Logger logger) {
+    static List<ZenPing.PingResponse> filterPingResponses(List<ZenPing.PingResponse> fullPingResponses, boolean masterElectionIgnoreNonMasters, Logger logger) {
         List<ZenPing.PingResponse> pingResponses;
         if (masterElectionIgnoreNonMasters) {
-            pingResponses = Arrays.stream(fullPingResponses).filter(ping -> ping.node().isMasterNode()).collect(Collectors.toList());
+            pingResponses = fullPingResponses.stream().filter(ping -> ping.node().isMasterNode()).collect(Collectors.toList());
         } else {
-            pingResponses = Arrays.asList(fullPingResponses);
+            pingResponses = fullPingResponses;
         }
 
         if (logger.isDebugEnabled()) {
@@ -988,7 +951,7 @@ static List<ZenPing.PingResponse> filterPingResponses(ZenPing.PingResponse[] ful
         return pingResponses;
     }
 
-    protected ClusterState rejoin(ClusterState clusterState, String reason) {
+    protected ClusterStateTaskExecutor.ClusterTasksResult<LocalClusterUpdateTask> rejoin(ClusterState clusterState, String reason) {
 
         // *** called from within an cluster state update task *** //
         assert Thread.currentThread().getName().contains(ClusterService.UPDATE_THREAD_NAME);
@@ -997,29 +960,17 @@ protected ClusterState rejoin(ClusterState clusterState, String reason) {
         nodesFD.stop();
         masterFD.stop(reason);
 
-
-        ClusterBlocks clusterBlocks = ClusterBlocks.builder().blocks(clusterState.blocks())
-                .addGlobalBlock(discoverySettings.getNoMasterBlock())
-                .build();
-
-        // clean the nodes, we are now not connected to anybody, since we try and reform the cluster
-        DiscoveryNodes discoveryNodes = new DiscoveryNodes.Builder(clusterState.nodes()).masterNodeId(null).build();
-
         // TODO: do we want to force a new thread if we actively removed the master? this is to give a full pinging cycle
         // before a decision is made.
         joinThreadControl.startNewThreadIfNotRunning();
-
-        return ClusterState.builder(clusterState)
-                .blocks(clusterBlocks)
-                .nodes(discoveryNodes)
-                .build();
+        return LocalClusterUpdateTask.noMaster();
     }
 
     private boolean localNodeMaster() {
-        return nodes().isLocalNodeElectedMaster();
+        return clusterState().nodes().isLocalNodeElectedMaster();
     }
 
-    private ClusterState handleAnotherMaster(ClusterState localClusterState, final DiscoveryNode otherMaster, long otherClusterStateVersion, String reason) {
+    private ClusterStateTaskExecutor.ClusterTasksResult handleAnotherMaster(ClusterState localClusterState, final DiscoveryNode otherMaster, long otherClusterStateVersion, String reason) {
         assert localClusterState.nodes().isLocalNodeElectedMaster() : "handleAnotherMaster called but current node is not a master";
         assert Thread.currentThread().getName().contains(ClusterService.UPDATE_THREAD_NAME) : "not called from the cluster state update thread";
 
@@ -1027,22 +978,49 @@ private ClusterState handleAnotherMaster(ClusterState localClusterState, final D
             return rejoin(localClusterState, "zen-disco-discovered another master with a new cluster_state [" + otherMaster + "][" + reason + "]");
         } else {
             logger.warn("discovered [{}] which is also master but with an older cluster_state, telling [{}] to rejoin the cluster ([{}])", otherMaster, otherMaster, reason);
-            try {
-                // make sure we're connected to this node (connect to node does nothing if we're already connected)
-                // since the network connections are asymmetric, it may be that we received a state but have disconnected from the node
-                // in the past (after a master failure, for example)
-                transportService.connectToNode(otherMaster);
-                transportService.sendRequest(otherMaster, DISCOVERY_REJOIN_ACTION_NAME, new RejoinClusterRequest(localClusterState.nodes().getLocalNodeId()), new EmptyTransportResponseHandler(ThreadPool.Names.SAME) {
-
-                    @Override
-                    public void handleException(TransportException exp) {
-                        logger.warn((Supplier<?>) () -> new ParameterizedMessage("failed to send rejoin request to [{}]", otherMaster), exp);
-                    }
-                });
-            } catch (Exception e) {
-                logger.warn((Supplier<?>) () -> new ParameterizedMessage("failed to send rejoin request to [{}]", otherMaster), e);
-            }
-            return localClusterState;
+            // spawn to a background thread to not do blocking operations on the cluster state thread
+            threadPool.generic().execute(new AbstractRunnable() {
+                @Override
+                public void onFailure(Exception e) {
+                    logger.warn((Supplier<?>) () -> new ParameterizedMessage("failed to send rejoin request to [{}]", otherMaster), e);
+                }
+
+                @Override
+                protected void doRun() throws Exception {
+                    // make sure we're connected to this node (connect to node does nothing if we're already connected)
+                    // since the network connections are asymmetric, it may be that we received a state but have disconnected from the node
+                    // in the past (after a master failure, for example)
+                    transportService.connectToNode(otherMaster);
+                    transportService.sendRequest(otherMaster, DISCOVERY_REJOIN_ACTION_NAME, new RejoinClusterRequest(localNode().getId()), new EmptyTransportResponseHandler(ThreadPool.Names.SAME) {
+
+                        @Override
+                        public void handleException(TransportException exp) {
+                            logger.warn((Supplier<?>) () -> new ParameterizedMessage("failed to send rejoin request to [{}]", otherMaster), exp);
+                        }
+                    });
+                }
+            });
+            return LocalClusterUpdateTask.unchanged();
+        }
+    }
+
+    private ZenPing.PingCollection pingAndWait(TimeValue timeout) {
+        final CompletableFuture<ZenPing.PingCollection> response = new CompletableFuture<>();
+        try {
+            zenPing.ping(response::complete, timeout);
+        } catch (Exception ex) {
+            // logged later
+            response.completeExceptionally(ex);
+        }
+
+        try {
+            return response.get();
+        } catch (InterruptedException e) {
+            logger.trace("pingAndWait interrupted");
+            return new ZenPing.PingCollection();
+        } catch (ExecutionException e) {
+            logger.warn("Ping execution failed", e);
+            return new ZenPing.PingCollection();
         }
     }
 
@@ -1089,12 +1067,16 @@ public void onPingReceived(final NodesFaultDetection.PingRequest pingRequest) {
                 return;
             }
             logger.debug("got a ping from another master {}. resolving who should rejoin. current ping count: [{}]", pingRequest.masterNode(), pingsWhileMaster.get());
-            clusterService.submitStateUpdateTask("ping from another master", new ClusterStateUpdateTask(Priority.IMMEDIATE) {
+            clusterService.submitStateUpdateTask("ping from another master", new LocalClusterUpdateTask(Priority.IMMEDIATE) {
 
                 @Override
-                public ClusterState execute(ClusterState currentState) throws Exception {
-                    pingsWhileMaster.set(0);
-                    return handleAnotherMaster(currentState, pingRequest.masterNode(), pingRequest.clusterStateVersion(), "node fd ping");
+                public ClusterTasksResult<LocalClusterUpdateTask> execute(ClusterState currentState) throws Exception {
+                    if (currentState.nodes().isLocalNodeElectedMaster()) {
+                        pingsWhileMaster.set(0);
+                        return handleAnotherMaster(currentState, pingRequest.masterNode(), pingRequest.clusterStateVersion(), "node fd ping");
+                    } else {
+                        return unchanged();
+                    }
                 }
 
                 @Override
@@ -1140,15 +1122,10 @@ public void writeTo(StreamOutput out) throws IOException {
     class RejoinClusterRequestHandler implements TransportRequestHandler<RejoinClusterRequest> {
         @Override
         public void messageReceived(final RejoinClusterRequest request, final TransportChannel channel) throws Exception {
-            clusterService.submitStateUpdateTask("received a request to rejoin the cluster from [" + request.fromNodeId + "]", new ClusterStateUpdateTask(Priority.IMMEDIATE) {
+            clusterService.submitStateUpdateTask("received a request to rejoin the cluster from [" + request.fromNodeId + "]", new LocalClusterUpdateTask(Priority.IMMEDIATE) {
 
                 @Override
-                public boolean runOnlyOnMaster() {
-                    return false;
-                }
-
-                @Override
-                public ClusterState execute(ClusterState currentState) {
+                public ClusterTasksResult<LocalClusterUpdateTask> execute(ClusterState currentState) {
                     try {
                         channel.sendResponse(TransportResponse.Empty.INSTANCE);
                     } catch (Exception e) {
@@ -1172,14 +1149,9 @@ public void onFailure(String source, Exception e) {
      */
     private class JoinThreadControl {
 
-        private final ThreadPool threadPool;
         private final AtomicBoolean running = new AtomicBoolean(false);
         private final AtomicReference<Thread> currentJoinThread = new AtomicReference<>();
 
-        public JoinThreadControl(ThreadPool threadPool) {
-            this.threadPool = threadPool;
-        }
-
         /** returns true if join thread control is started and there is currently an active join thread */
         public boolean joinThreadActive() {
             Thread currentThread = currentJoinThread.get();
@@ -1192,7 +1164,7 @@ public boolean joinThreadActive(Thread joinThread) {
         }
 
         /** cleans any running joining thread and calls {@link #rejoin} */
-        public ClusterState stopRunningThreadAndRejoin(ClusterState clusterState, String reason) {
+        public ClusterStateTaskExecutor.ClusterTasksResult<LocalClusterUpdateTask> stopRunningThreadAndRejoin(ClusterState clusterState, String reason) {
             ClusterService.assertClusterStateThread();
             currentJoinThread.set(null);
             return rejoin(clusterState, reason);
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/ZenPing.java b/core/src/main/java/org/elasticsearch/discovery/zen/ZenPing.java
new file mode 100644
index 0000000000000..016d2a5423ce1
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/ZenPing.java
@@ -0,0 +1,194 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.discovery.zen;
+
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Streamable;
+import org.elasticsearch.common.lease.Releasable;
+import org.elasticsearch.common.unit.TimeValue;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Consumer;
+
+import static org.elasticsearch.gateway.GatewayService.STATE_NOT_RECOVERED_BLOCK;
+
+public interface ZenPing extends Releasable {
+
+    void start(PingContextProvider contextProvider);
+
+    void ping(Consumer<PingCollection> resultsConsumer, TimeValue timeout);
+
+    class PingResponse implements Streamable {
+
+        public static final PingResponse[] EMPTY = new PingResponse[0];
+
+        private static final AtomicLong idGenerator = new AtomicLong();
+
+        // an always increasing unique identifier for this ping response.
+        // lower values means older pings.
+        private long id;
+
+        private ClusterName clusterName;
+
+        private DiscoveryNode node;
+
+        private DiscoveryNode master;
+
+        private long clusterStateVersion;
+
+        private PingResponse() {
+        }
+
+        /**
+         * @param node                the node which this ping describes
+         * @param master              the current master of the node
+         * @param clusterName         the cluster name of the node
+         * @param clusterStateVersion the current cluster state version of that node
+         *                            ({@link ElectMasterService.MasterCandidate#UNRECOVERED_CLUSTER_VERSION} for not recovered)
+         */
+        PingResponse(DiscoveryNode node, DiscoveryNode master, ClusterName clusterName, long clusterStateVersion) {
+            this.id = idGenerator.incrementAndGet();
+            this.node = node;
+            this.master = master;
+            this.clusterName = clusterName;
+            this.clusterStateVersion = clusterStateVersion;
+        }
+
+        public PingResponse(DiscoveryNode node, DiscoveryNode master, ClusterState state) {
+            this(node, master, state.getClusterName(),
+                state.blocks().hasGlobalBlock(STATE_NOT_RECOVERED_BLOCK) ?
+                    ElectMasterService.MasterCandidate.UNRECOVERED_CLUSTER_VERSION : state.version());
+        }
+
+            /**
+             * an always increasing unique identifier for this ping response.
+             * lower values means older pings.
+             */
+        public long id() {
+            return this.id;
+        }
+
+        public ClusterName clusterName() {
+            return this.clusterName;
+        }
+
+        /** the node which this ping describes */
+        public DiscoveryNode node() {
+            return node;
+        }
+
+        /** the current master of the node */
+        public DiscoveryNode master() {
+            return master;
+        }
+
+        /**
+         * the current cluster state version of that node ({@link ElectMasterService.MasterCandidate#UNRECOVERED_CLUSTER_VERSION}
+         * for not recovered) */
+        public long getClusterStateVersion() {
+            return clusterStateVersion;
+        }
+
+        public static PingResponse readPingResponse(StreamInput in) throws IOException {
+            PingResponse response = new PingResponse();
+            response.readFrom(in);
+            return response;
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            clusterName = new ClusterName(in);
+            node = new DiscoveryNode(in);
+            if (in.readBoolean()) {
+                master = new DiscoveryNode(in);
+            }
+            this.clusterStateVersion = in.readLong();
+            this.id = in.readLong();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            clusterName.writeTo(out);
+            node.writeTo(out);
+            if (master == null) {
+                out.writeBoolean(false);
+            } else {
+                out.writeBoolean(true);
+                master.writeTo(out);
+            }
+            out.writeLong(clusterStateVersion);
+            out.writeLong(id);
+        }
+
+        @Override
+        public String toString() {
+            return "ping_response{node [" + node + "], id[" + id + "], master [" + master + "]," +
+                   "cluster_state_version [" + clusterStateVersion + "], cluster_name[" + clusterName.value() + "]}";
+        }
+    }
+
+
+    /**
+     * a utility collection of pings where only the most recent ping is stored per node
+     */
+    class PingCollection {
+
+        Map<DiscoveryNode, PingResponse> pings;
+
+        public PingCollection() {
+            pings = new HashMap<>();
+        }
+
+        /**
+         * adds a ping if newer than previous pings from the same node
+         *
+         * @return true if added, false o.w.
+         */
+        public synchronized boolean addPing(PingResponse ping) {
+            PingResponse existingResponse = pings.get(ping.node());
+            // in case both existing and new ping have the same id (probably because they come
+            // from nodes from version <1.4.0) we prefer to use the last added one.
+            if (existingResponse == null || existingResponse.id() <= ping.id()) {
+                pings.put(ping.node(), ping);
+                return true;
+            }
+            return false;
+        }
+
+        /** serialize current pings to a list. It is guaranteed that the list contains one ping response per node */
+        public synchronized List<PingResponse> toList() {
+            return new ArrayList<>(pings.values());
+        }
+
+        /** the number of nodes for which there are known pings */
+        public synchronized int size() {
+            return pings.size();
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/elect/ElectMasterService.java b/core/src/main/java/org/elasticsearch/discovery/zen/elect/ElectMasterService.java
deleted file mode 100644
index 3ef9138f933b9..0000000000000
--- a/core/src/main/java/org/elasticsearch/discovery/zen/elect/ElectMasterService.java
+++ /dev/null
@@ -1,181 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.discovery.zen.elect;
-
-import com.carrotsearch.hppc.ObjectContainer;
-import org.apache.lucene.util.CollectionUtil;
-import org.elasticsearch.Version;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Setting.Property;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.CollectionUtils;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.Iterator;
-import java.util.List;
-
-/**
- *
- */
-public class ElectMasterService extends AbstractComponent {
-
-    public static final Setting<Integer> DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING =
-        Setting.intSetting("discovery.zen.minimum_master_nodes", -1, Property.Dynamic, Property.NodeScope);
-
-    // This is the minimum version a master needs to be on, otherwise it gets ignored
-    // This is based on the minimum compatible version of the current version this node is on
-    private final Version minMasterVersion;
-    private final NodeComparator nodeComparator = new NodeComparator();
-
-    private volatile int minimumMasterNodes;
-
-    @Inject
-    public ElectMasterService(Settings settings) {
-        super(settings);
-        this.minMasterVersion = Version.CURRENT.minimumCompatibilityVersion();
-        this.minimumMasterNodes = DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.get(settings);
-        logger.debug("using minimum_master_nodes [{}]", minimumMasterNodes);
-    }
-
-    public void minimumMasterNodes(int minimumMasterNodes) {
-        this.minimumMasterNodes = minimumMasterNodes;
-    }
-
-    public int minimumMasterNodes() {
-        return minimumMasterNodes;
-    }
-
-    public boolean hasEnoughMasterNodes(Iterable<DiscoveryNode> nodes) {
-        if (minimumMasterNodes < 1) {
-            return true;
-        }
-        int count = 0;
-        for (DiscoveryNode node : nodes) {
-            if (node.isMasterNode()) {
-                count++;
-            }
-        }
-        return count >= minimumMasterNodes;
-    }
-
-    public boolean hasTooManyMasterNodes(Iterable<DiscoveryNode> nodes) {
-        int count = 0;
-        for (DiscoveryNode node : nodes) {
-            if (node.isMasterNode()) {
-                count++;
-            }
-        }
-        return count > 1 && minimumMasterNodes <= count / 2;
-    }
-
-    public void logMinimumMasterNodesWarningIfNecessary(ClusterState oldState, ClusterState newState) {
-        // check if min_master_nodes setting is too low and log warning
-        if (hasTooManyMasterNodes(oldState.nodes()) == false && hasTooManyMasterNodes(newState.nodes())) {
-            logger.warn("value for setting \"{}\" is too low. This can result in data loss! Please set it to at least a quorum of master-" +
-                    "eligible nodes (current value: [{}], total number of master-eligible nodes used for publishing in this round: [{}])",
-                ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), minimumMasterNodes(),
-                newState.getNodes().getMasterNodes().size());
-        }
-    }
-
-    /**
-     * Returns the given nodes sorted by likelihood of being elected as master, most likely first.
-     * Non-master nodes are not removed but are rather put in the end
-     */
-    public List<DiscoveryNode> sortByMasterLikelihood(Iterable<DiscoveryNode> nodes) {
-        ArrayList<DiscoveryNode> sortedNodes = CollectionUtils.iterableAsArrayList(nodes);
-        CollectionUtil.introSort(sortedNodes, nodeComparator);
-        return sortedNodes;
-    }
-
-    /**
-     * Returns a list of the next possible masters.
-     */
-    public DiscoveryNode[] nextPossibleMasters(ObjectContainer<DiscoveryNode> nodes, int numberOfPossibleMasters) {
-        List<DiscoveryNode> sortedNodes = sortedMasterNodes(Arrays.asList(nodes.toArray(DiscoveryNode.class)));
-        if (sortedNodes == null) {
-            return new DiscoveryNode[0];
-        }
-        List<DiscoveryNode> nextPossibleMasters = new ArrayList<>(numberOfPossibleMasters);
-        int counter = 0;
-        for (DiscoveryNode nextPossibleMaster : sortedNodes) {
-            if (++counter >= numberOfPossibleMasters) {
-                break;
-            }
-            nextPossibleMasters.add(nextPossibleMaster);
-        }
-        return nextPossibleMasters.toArray(new DiscoveryNode[nextPossibleMasters.size()]);
-    }
-
-    /**
-     * Elects a new master out of the possible nodes, returning it. Returns <tt>null</tt>
-     * if no master has been elected.
-     */
-    public DiscoveryNode electMaster(Iterable<DiscoveryNode> nodes) {
-        List<DiscoveryNode> sortedNodes = sortedMasterNodes(nodes);
-        if (sortedNodes == null || sortedNodes.isEmpty()) {
-            return null;
-        }
-        DiscoveryNode masterNode = sortedNodes.get(0);
-        // Sanity check: maybe we don't end up here, because serialization may have failed.
-        if (masterNode.getVersion().before(minMasterVersion)) {
-            logger.warn("ignoring master [{}], because the version [{}] is lower than the minimum compatible version [{}]", masterNode, masterNode.getVersion(), minMasterVersion);
-            return null;
-        } else {
-            return masterNode;
-        }
-    }
-
-    private List<DiscoveryNode> sortedMasterNodes(Iterable<DiscoveryNode> nodes) {
-        List<DiscoveryNode> possibleNodes = CollectionUtils.iterableAsArrayList(nodes);
-        if (possibleNodes.isEmpty()) {
-            return null;
-        }
-        // clean non master nodes
-        for (Iterator<DiscoveryNode> it = possibleNodes.iterator(); it.hasNext(); ) {
-            DiscoveryNode node = it.next();
-            if (!node.isMasterNode()) {
-                it.remove();
-            }
-        }
-        CollectionUtil.introSort(possibleNodes, nodeComparator);
-        return possibleNodes;
-    }
-
-    private static class NodeComparator implements Comparator<DiscoveryNode> {
-
-        @Override
-        public int compare(DiscoveryNode o1, DiscoveryNode o2) {
-            if (o1.isMasterNode() && !o2.isMasterNode()) {
-                return -1;
-            }
-            if (!o1.isMasterNode() && o2.isMasterNode()) {
-                return 1;
-            }
-            return o1.getId().compareTo(o2.getId());
-        }
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/membership/MembershipAction.java b/core/src/main/java/org/elasticsearch/discovery/zen/membership/MembershipAction.java
deleted file mode 100644
index 961b8d79728ea..0000000000000
--- a/core/src/main/java/org/elasticsearch/discovery/zen/membership/MembershipAction.java
+++ /dev/null
@@ -1,222 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.discovery.zen.membership;
-
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.discovery.zen.DiscoveryNodesProvider;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.EmptyTransportResponseHandler;
-import org.elasticsearch.transport.TransportChannel;
-import org.elasticsearch.transport.TransportRequest;
-import org.elasticsearch.transport.TransportRequestHandler;
-import org.elasticsearch.transport.TransportResponse;
-import org.elasticsearch.transport.TransportService;
-
-import java.io.IOException;
-import java.util.concurrent.TimeUnit;
-
-/**
- *
- */
-public class MembershipAction extends AbstractComponent {
-
-    public static final String DISCOVERY_JOIN_ACTION_NAME = "internal:discovery/zen/join";
-    public static final String DISCOVERY_JOIN_VALIDATE_ACTION_NAME = "internal:discovery/zen/join/validate";
-    public static final String DISCOVERY_LEAVE_ACTION_NAME = "internal:discovery/zen/leave";
-
-    public interface JoinCallback {
-        void onSuccess();
-
-        void onFailure(Exception e);
-    }
-
-    public interface MembershipListener {
-        void onJoin(DiscoveryNode node, JoinCallback callback);
-
-        void onLeave(DiscoveryNode node);
-    }
-
-    private final TransportService transportService;
-
-    private final DiscoveryNodesProvider nodesProvider;
-
-    private final MembershipListener listener;
-
-    public MembershipAction(Settings settings, TransportService transportService, DiscoveryNodesProvider nodesProvider, MembershipListener listener) {
-        super(settings);
-        this.transportService = transportService;
-        this.nodesProvider = nodesProvider;
-        this.listener = listener;
-
-        transportService.registerRequestHandler(DISCOVERY_JOIN_ACTION_NAME, JoinRequest::new, ThreadPool.Names.GENERIC, new JoinRequestRequestHandler());
-        transportService.registerRequestHandler(DISCOVERY_JOIN_VALIDATE_ACTION_NAME, ValidateJoinRequest::new, ThreadPool.Names.GENERIC, new ValidateJoinRequestRequestHandler());
-        transportService.registerRequestHandler(DISCOVERY_LEAVE_ACTION_NAME, LeaveRequest::new, ThreadPool.Names.GENERIC, new LeaveRequestRequestHandler());
-    }
-
-    public void close() {
-        transportService.removeHandler(DISCOVERY_JOIN_ACTION_NAME);
-        transportService.removeHandler(DISCOVERY_JOIN_VALIDATE_ACTION_NAME);
-        transportService.removeHandler(DISCOVERY_LEAVE_ACTION_NAME);
-    }
-
-    public void sendLeaveRequest(DiscoveryNode masterNode, DiscoveryNode node) {
-        transportService.sendRequest(node, DISCOVERY_LEAVE_ACTION_NAME, new LeaveRequest(masterNode), EmptyTransportResponseHandler.INSTANCE_SAME);
-    }
-
-    public void sendLeaveRequestBlocking(DiscoveryNode masterNode, DiscoveryNode node, TimeValue timeout) {
-        transportService.submitRequest(masterNode, DISCOVERY_LEAVE_ACTION_NAME, new LeaveRequest(node), EmptyTransportResponseHandler.INSTANCE_SAME).txGet(timeout.millis(), TimeUnit.MILLISECONDS);
-    }
-
-    public void sendJoinRequestBlocking(DiscoveryNode masterNode, DiscoveryNode node, TimeValue timeout) {
-        transportService.submitRequest(masterNode, DISCOVERY_JOIN_ACTION_NAME, new JoinRequest(node), EmptyTransportResponseHandler.INSTANCE_SAME)
-                .txGet(timeout.millis(), TimeUnit.MILLISECONDS);
-    }
-
-    /**
-     * Validates the join request, throwing a failure if it failed.
-     */
-    public void sendValidateJoinRequestBlocking(DiscoveryNode node, ClusterState state, TimeValue timeout) {
-        transportService.submitRequest(node, DISCOVERY_JOIN_VALIDATE_ACTION_NAME, new ValidateJoinRequest(state), EmptyTransportResponseHandler.INSTANCE_SAME)
-                .txGet(timeout.millis(), TimeUnit.MILLISECONDS);
-    }
-
-    public static class JoinRequest extends TransportRequest {
-
-        DiscoveryNode node;
-
-        public JoinRequest() {
-        }
-
-        private JoinRequest(DiscoveryNode node) {
-            this.node = node;
-        }
-
-        @Override
-        public void readFrom(StreamInput in) throws IOException {
-            super.readFrom(in);
-            node = new DiscoveryNode(in);
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            super.writeTo(out);
-            node.writeTo(out);
-        }
-    }
-
-
-    private class JoinRequestRequestHandler implements TransportRequestHandler<JoinRequest> {
-
-        @Override
-        public void messageReceived(final JoinRequest request, final TransportChannel channel) throws Exception {
-            listener.onJoin(request.node, new JoinCallback() {
-                @Override
-                public void onSuccess() {
-                    try {
-                        channel.sendResponse(TransportResponse.Empty.INSTANCE);
-                    } catch (Exception e) {
-                        onFailure(e);
-                    }
-                }
-
-                @Override
-                public void onFailure(Exception e) {
-                    try {
-                        channel.sendResponse(e);
-                    } catch (Exception inner) {
-                        inner.addSuppressed(e);
-                        logger.warn("failed to send back failure on join request", inner);
-                    }
-                }
-            });
-        }
-    }
-
-    class ValidateJoinRequest extends TransportRequest {
-        private ClusterState state;
-
-        ValidateJoinRequest() {
-        }
-
-        ValidateJoinRequest(ClusterState state) {
-            this.state = state;
-        }
-
-        @Override
-        public void readFrom(StreamInput in) throws IOException {
-            super.readFrom(in);
-            this.state = ClusterState.Builder.readFrom(in, nodesProvider.nodes().getLocalNode());
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            super.writeTo(out);
-            this.state.writeTo(out);
-        }
-    }
-
-    class ValidateJoinRequestRequestHandler implements TransportRequestHandler<ValidateJoinRequest> {
-
-        @Override
-        public void messageReceived(ValidateJoinRequest request, TransportChannel channel) throws Exception {
-            // for now, the mere fact that we can serialize the cluster state acts as validation....
-            channel.sendResponse(TransportResponse.Empty.INSTANCE);
-        }
-    }
-
-    public static class LeaveRequest extends TransportRequest {
-
-        private DiscoveryNode node;
-
-        public LeaveRequest() {
-        }
-
-        private LeaveRequest(DiscoveryNode node) {
-            this.node = node;
-        }
-
-        @Override
-        public void readFrom(StreamInput in) throws IOException {
-            super.readFrom(in);
-            node = new DiscoveryNode(in);
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            super.writeTo(out);
-            node.writeTo(out);
-        }
-    }
-
-    private class LeaveRequestRequestHandler implements TransportRequestHandler<LeaveRequest> {
-
-        @Override
-        public void messageReceived(LeaveRequest request, TransportChannel channel) throws Exception {
-            listener.onLeave(request.node);
-            channel.sendResponse(TransportResponse.Empty.INSTANCE);
-        }
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/ping/PingContextProvider.java b/core/src/main/java/org/elasticsearch/discovery/zen/ping/PingContextProvider.java
deleted file mode 100644
index 568bc3ec16d75..0000000000000
--- a/core/src/main/java/org/elasticsearch/discovery/zen/ping/PingContextProvider.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.discovery.zen.ping;
-
-import org.elasticsearch.discovery.zen.DiscoveryNodesProvider;
-
-/**
- *
- */
-public interface PingContextProvider extends DiscoveryNodesProvider {
-
-    /** return true if this node has previously joined the cluster at least once. False if this is first join */
-    boolean nodeHasJoinedClusterOnce();
-
-}
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/ping/ZenPing.java b/core/src/main/java/org/elasticsearch/discovery/zen/ping/ZenPing.java
deleted file mode 100644
index 5a9f5f463e236..0000000000000
--- a/core/src/main/java/org/elasticsearch/discovery/zen/ping/ZenPing.java
+++ /dev/null
@@ -1,190 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.discovery.zen.ping;
-
-import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.common.component.LifecycleComponent;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Streamable;
-import org.elasticsearch.common.unit.TimeValue;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicLong;
-
-public interface ZenPing extends LifecycleComponent {
-
-    void setPingContextProvider(PingContextProvider contextProvider);
-
-    void ping(PingListener listener, TimeValue timeout);
-
-    public interface PingListener {
-
-        void onPing(PingResponse[] pings);
-    }
-
-    public static class PingResponse implements Streamable {
-
-        public static final PingResponse[] EMPTY = new PingResponse[0];
-
-        private static final AtomicLong idGenerator = new AtomicLong();
-
-        // an always increasing unique identifier for this ping response.
-        // lower values means older pings.
-        private long id;
-
-        private ClusterName clusterName;
-
-        private DiscoveryNode node;
-
-        private DiscoveryNode master;
-
-        private boolean hasJoinedOnce;
-
-        private PingResponse() {
-        }
-
-        /**
-         * @param node          the node which this ping describes
-         * @param master        the current master of the node
-         * @param clusterName   the cluster name of the node
-         * @param hasJoinedOnce true if the joined has successfully joined the cluster before
-         */
-        public PingResponse(DiscoveryNode node, DiscoveryNode master, ClusterName clusterName, boolean hasJoinedOnce) {
-            this.id = idGenerator.incrementAndGet();
-            this.node = node;
-            this.master = master;
-            this.clusterName = clusterName;
-            this.hasJoinedOnce = hasJoinedOnce;
-        }
-
-        /**
-         * an always increasing unique identifier for this ping response.
-         * lower values means older pings.
-         */
-        public long id() {
-            return this.id;
-        }
-
-        public ClusterName clusterName() {
-            return this.clusterName;
-        }
-
-        /** the node which this ping describes */
-        public DiscoveryNode node() {
-            return node;
-        }
-
-        /** the current master of the node */
-        public DiscoveryNode master() {
-            return master;
-        }
-
-        /** true if the joined has successfully joined the cluster before */
-        public boolean hasJoinedOnce() {
-            return hasJoinedOnce;
-        }
-
-        public static PingResponse readPingResponse(StreamInput in) throws IOException {
-            PingResponse response = new PingResponse();
-            response.readFrom(in);
-            return response;
-        }
-
-        @Override
-        public void readFrom(StreamInput in) throws IOException {
-            clusterName = new ClusterName(in);
-            node = new DiscoveryNode(in);
-            if (in.readBoolean()) {
-                master = new DiscoveryNode(in);
-            }
-            this.hasJoinedOnce = in.readBoolean();
-            this.id = in.readLong();
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            clusterName.writeTo(out);
-            node.writeTo(out);
-            if (master == null) {
-                out.writeBoolean(false);
-            } else {
-                out.writeBoolean(true);
-                master.writeTo(out);
-            }
-            out.writeBoolean(hasJoinedOnce);
-            out.writeLong(id);
-        }
-
-        @Override
-        public String toString() {
-            return "ping_response{node [" + node + "], id[" + id + "], master [" + master + "], hasJoinedOnce [" + hasJoinedOnce + "], cluster_name[" + clusterName.value() + "]}";
-        }
-    }
-
-
-    /**
-     * a utility collection of pings where only the most recent ping is stored per node
-     */
-    public static class PingCollection {
-
-        Map<DiscoveryNode, PingResponse> pings;
-
-        public PingCollection() {
-            pings = new HashMap<>();
-        }
-
-        /**
-         * adds a ping if newer than previous pings from the same node
-         *
-         * @return true if added, false o.w.
-         */
-        public synchronized boolean addPing(PingResponse ping) {
-            PingResponse existingResponse = pings.get(ping.node());
-            // in case both existing and new ping have the same id (probably because they come
-            // from nodes from version <1.4.0) we prefer to use the last added one.
-            if (existingResponse == null || existingResponse.id() <= ping.id()) {
-                pings.put(ping.node(), ping);
-                return true;
-            }
-            return false;
-        }
-
-        /** adds multiple pings if newer than previous pings from the same node */
-        public synchronized void addPings(PingResponse[] pings) {
-            for (PingResponse ping : pings) {
-                addPing(ping);
-            }
-        }
-
-        /** serialize current pings to an array */
-        public synchronized PingResponse[] toArray() {
-            return pings.values().toArray(new PingResponse[pings.size()]);
-        }
-
-        /** the number of nodes for which there are known pings */
-        public synchronized int size() {
-            return pings.size();
-        }
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/ping/ZenPingService.java b/core/src/main/java/org/elasticsearch/discovery/zen/ping/ZenPingService.java
deleted file mode 100644
index bd5855666aca2..0000000000000
--- a/core/src/main/java/org/elasticsearch/discovery/zen/ping/ZenPingService.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.discovery.zen.ping;
-
-import org.elasticsearch.common.component.AbstractLifecycleComponent;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
-
-public class ZenPingService extends AbstractLifecycleComponent implements ZenPing {
-
-    private List<ZenPing> zenPings = Collections.emptyList();
-
-    @Inject
-    public ZenPingService(Settings settings, Set<ZenPing> zenPings) {
-        super(settings);
-        this.zenPings = Collections.unmodifiableList(new ArrayList<>(zenPings));
-    }
-
-    public List<ZenPing> zenPings() {
-        return this.zenPings;
-    }
-
-    @Override
-    public void setPingContextProvider(PingContextProvider contextProvider) {
-        if (lifecycle.started()) {
-            throw new IllegalStateException("Can't set nodes provider when started");
-        }
-        for (ZenPing zenPing : zenPings) {
-            zenPing.setPingContextProvider(contextProvider);
-        }
-    }
-
-    @Override
-    protected void doStart() {
-        for (ZenPing zenPing : zenPings) {
-            zenPing.start();
-        }
-    }
-
-    @Override
-    protected void doStop() {
-        for (ZenPing zenPing : zenPings) {
-            zenPing.stop();
-        }
-    }
-
-    @Override
-    protected void doClose() {
-        for (ZenPing zenPing : zenPings) {
-            zenPing.close();
-        }
-    }
-
-    public PingResponse[] pingAndWait(TimeValue timeout) {
-        final AtomicReference<PingResponse[]> response = new AtomicReference<>();
-        final CountDownLatch latch = new CountDownLatch(1);
-        ping(new PingListener() {
-            @Override
-            public void onPing(PingResponse[] pings) {
-                response.set(pings);
-                latch.countDown();
-            }
-        }, timeout);
-        try {
-            latch.await();
-            return response.get();
-        } catch (InterruptedException e) {
-            logger.trace("pingAndWait interrupted");
-            return null;
-        }
-    }
-
-    @Override
-    public void ping(PingListener listener, TimeValue timeout) {
-        List<? extends ZenPing> zenPings = this.zenPings;
-        CompoundPingListener compoundPingListener = new CompoundPingListener(listener, zenPings);
-        for (ZenPing zenPing : zenPings) {
-            try {
-                zenPing.ping(compoundPingListener, timeout);
-            } catch (EsRejectedExecutionException ex) {
-                logger.debug("Ping execution rejected", ex);
-                compoundPingListener.onPing(null);
-            }
-        }
-    }
-
-    private static class CompoundPingListener implements PingListener {
-
-        private final PingListener listener;
-
-        private final AtomicInteger counter;
-
-        private PingCollection responses = new PingCollection();
-
-        private CompoundPingListener(PingListener listener, List<? extends ZenPing> zenPings) {
-            this.listener = listener;
-            this.counter = new AtomicInteger(zenPings.size());
-        }
-
-        @Override
-        public void onPing(PingResponse[] pings) {
-            if (pings != null) {
-                responses.addPings(pings);
-            }
-            if (counter.decrementAndGet() == 0) {
-                listener.onPing(responses.toArray());
-            }
-        }
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java b/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java
deleted file mode 100644
index 176ac5763e353..0000000000000
--- a/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java
+++ /dev/null
@@ -1,599 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.discovery.zen.ping.unicast;
-
-import com.carrotsearch.hppc.cursors.ObjectCursor;
-import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.apache.logging.log4j.util.Supplier;
-import org.apache.lucene.util.IOUtils;
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.Version;
-import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.UUIDs;
-import org.elasticsearch.common.component.AbstractLifecycleComponent;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Setting.Property;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.TransportAddress;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.util.CollectionUtils;
-import org.elasticsearch.common.util.concurrent.AbstractRunnable;
-import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
-import org.elasticsearch.common.util.concurrent.EsExecutors;
-import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
-import org.elasticsearch.discovery.zen.elect.ElectMasterService;
-import org.elasticsearch.discovery.zen.ping.PingContextProvider;
-import org.elasticsearch.discovery.zen.ping.ZenPing;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.ConnectTransportException;
-import org.elasticsearch.transport.RemoteTransportException;
-import org.elasticsearch.transport.TransportChannel;
-import org.elasticsearch.transport.TransportException;
-import org.elasticsearch.transport.TransportRequest;
-import org.elasticsearch.transport.TransportRequestHandler;
-import org.elasticsearch.transport.TransportRequestOptions;
-import org.elasticsearch.transport.TransportResponse;
-import org.elasticsearch.transport.TransportResponseHandler;
-import org.elasticsearch.transport.TransportService;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Queue;
-import java.util.Set;
-import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.RejectedExecutionException;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.function.Function;
-
-import static java.util.Collections.emptyList;
-import static java.util.Collections.emptyMap;
-import static java.util.Collections.emptySet;
-import static org.elasticsearch.common.util.concurrent.ConcurrentCollections.newConcurrentMap;
-import static org.elasticsearch.discovery.zen.ping.ZenPing.PingResponse.readPingResponse;
-
-/**
- *
- */
-public class UnicastZenPing extends AbstractLifecycleComponent implements ZenPing {
-
-    public static final String ACTION_NAME = "internal:discovery/zen/unicast";
-    public static final Setting<List<String>> DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING =
-        Setting.listSetting("discovery.zen.ping.unicast.hosts", emptyList(), Function.identity(),
-            Property.NodeScope);
-    public static final Setting<Integer> DISCOVERY_ZEN_PING_UNICAST_CONCURRENT_CONNECTS_SETTING =
-        Setting.intSetting("discovery.zen.ping.unicast.concurrent_connects", 10, 0, Property.NodeScope);
-
-    // these limits are per-address
-    public static final int LIMIT_FOREIGN_PORTS_COUNT = 1;
-    public static final int LIMIT_LOCAL_PORTS_COUNT = 5;
-
-
-    private final ThreadPool threadPool;
-    private final TransportService transportService;
-    private final ClusterName clusterName;
-    private final ElectMasterService electMasterService;
-
-    private final int concurrentConnects;
-
-    private final DiscoveryNode[] configuredTargetNodes;
-
-    private volatile PingContextProvider contextProvider;
-
-    private final AtomicInteger pingHandlerIdGenerator = new AtomicInteger();
-
-    // used to generate unique ids for nodes/address we temporarily connect to
-    private final AtomicInteger unicastNodeIdGenerator = new AtomicInteger();
-
-    // used as a node id prefix for nodes/address we temporarily connect to
-    private static final String UNICAST_NODE_PREFIX = "#zen_unicast_";
-
-    private final Map<Integer, SendPingsHandler> receivedResponses = newConcurrentMap();
-
-    // a list of temporal responses a node will return for a request (holds requests from other configuredTargetNodes)
-    private final Queue<PingResponse> temporalResponses = ConcurrentCollections.newQueue();
-
-    private final CopyOnWriteArrayList<UnicastHostsProvider> hostsProviders = new CopyOnWriteArrayList<>();
-
-    private final ExecutorService unicastConnectExecutor;
-
-    private volatile boolean closed = false;
-
-    @Inject
-    public UnicastZenPing(Settings settings, ThreadPool threadPool, TransportService transportService,
-                          ElectMasterService electMasterService, @Nullable Set<UnicastHostsProvider> unicastHostsProviders) {
-        super(settings);
-        this.threadPool = threadPool;
-        this.transportService = transportService;
-        this.clusterName = ClusterName.CLUSTER_NAME_SETTING.get(settings);
-        this.electMasterService = electMasterService;
-
-        if (unicastHostsProviders != null) {
-            for (UnicastHostsProvider unicastHostsProvider : unicastHostsProviders) {
-                addHostsProvider(unicastHostsProvider);
-            }
-        }
-
-        this.concurrentConnects = DISCOVERY_ZEN_PING_UNICAST_CONCURRENT_CONNECTS_SETTING.get(settings);
-        List<String> hosts = DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING.get(settings);
-        final int limitPortCounts;
-        if (hosts.isEmpty()) {
-            // if unicast hosts are not specified, fill with simple defaults on the local machine
-            limitPortCounts = LIMIT_LOCAL_PORTS_COUNT;
-            hosts.addAll(transportService.getLocalAddresses());
-        } else {
-            // we only limit to 1 addresses, makes no sense to ping 100 ports
-            limitPortCounts = LIMIT_FOREIGN_PORTS_COUNT;
-        }
-
-        logger.debug("using initial hosts {}, with concurrent_connects [{}]", hosts, concurrentConnects);
-
-        List<DiscoveryNode> configuredTargetNodes = new ArrayList<>();
-        for (String host : hosts) {
-            try {
-                TransportAddress[] addresses = transportService.addressesFromString(host, limitPortCounts);
-                for (TransportAddress address : addresses) {
-                    configuredTargetNodes.add(new DiscoveryNode(UNICAST_NODE_PREFIX + unicastNodeIdGenerator.incrementAndGet() + "#",
-                            address, emptyMap(), emptySet(), getVersion().minimumCompatibilityVersion()));
-                }
-            } catch (Exception e) {
-                throw new IllegalArgumentException("Failed to resolve address for [" + host + "]", e);
-            }
-        }
-        this.configuredTargetNodes = configuredTargetNodes.toArray(new DiscoveryNode[configuredTargetNodes.size()]);
-
-        transportService.registerRequestHandler(ACTION_NAME, UnicastPingRequest::new, ThreadPool.Names.SAME,
-                new UnicastPingRequestHandler());
-
-        ThreadFactory threadFactory = EsExecutors.daemonThreadFactory(settings, "[unicast_connect]");
-        unicastConnectExecutor = EsExecutors.newScaling("unicast_connect", 0, concurrentConnects, 60, TimeUnit.SECONDS,
-                threadFactory, threadPool.getThreadContext());
-    }
-
-    @Override
-    protected void doStart() {
-    }
-
-    @Override
-    protected void doStop() {
-    }
-
-    @Override
-    protected void doClose() {
-        transportService.removeHandler(ACTION_NAME);
-        ThreadPool.terminate(unicastConnectExecutor, 0, TimeUnit.SECONDS);
-        try {
-            IOUtils.close(receivedResponses.values());
-        } catch (IOException e) {
-            throw new ElasticsearchException("Error wile closing send ping handlers", e);
-        }
-        closed = true;
-    }
-
-    public void addHostsProvider(UnicastHostsProvider provider) {
-        hostsProviders.add(provider);
-    }
-
-    @Override
-    public void setPingContextProvider(PingContextProvider contextProvider) {
-        this.contextProvider = contextProvider;
-    }
-
-    /**
-     * Clears the list of cached ping responses.
-     */
-    public void clearTemporalResponses() {
-        temporalResponses.clear();
-    }
-
-    public PingResponse[] pingAndWait(TimeValue duration) {
-        final AtomicReference<PingResponse[]> response = new AtomicReference<>();
-        final CountDownLatch latch = new CountDownLatch(1);
-        ping(pings -> {
-            response.set(pings);
-            latch.countDown();
-        }, duration);
-        try {
-            latch.await();
-            return response.get();
-        } catch (InterruptedException e) {
-            return null;
-        }
-    }
-
-    @Override
-    public void ping(final PingListener listener, final TimeValue duration) {
-        final SendPingsHandler sendPingsHandler = new SendPingsHandler(pingHandlerIdGenerator.incrementAndGet());
-        try {
-            receivedResponses.put(sendPingsHandler.id(), sendPingsHandler);
-            try {
-                sendPings(duration, null, sendPingsHandler);
-            } catch (RejectedExecutionException e) {
-                logger.debug("Ping execution rejected", e);
-                // The RejectedExecutionException can come from the fact unicastConnectExecutor is at its max down in sendPings
-                // But don't bail here, we can retry later on after the send ping has been scheduled.
-            }
-
-            threadPool.schedule(TimeValue.timeValueMillis(duration.millis() / 2), ThreadPool.Names.GENERIC, new AbstractRunnable() {
-                @Override
-                protected void doRun() {
-                    sendPings(duration, null, sendPingsHandler);
-                    threadPool.schedule(TimeValue.timeValueMillis(duration.millis() / 2), ThreadPool.Names.GENERIC, new AbstractRunnable() {
-                        @Override
-                        protected void doRun() throws Exception {
-                            sendPings(duration, TimeValue.timeValueMillis(duration.millis() / 2), sendPingsHandler);
-                            sendPingsHandler.close();
-                            listener.onPing(sendPingsHandler.pingCollection().toArray());
-                            for (DiscoveryNode node : sendPingsHandler.nodeToDisconnect) {
-                                logger.trace("[{}] disconnecting from {}", sendPingsHandler.id(), node);
-                                transportService.disconnectFromNode(node);
-                            }
-                        }
-
-                        @Override
-                        public void onFailure(Exception e) {
-                            logger.debug("Ping execution failed", e);
-                            sendPingsHandler.close();
-                        }
-                    });
-                }
-
-                @Override
-                public void onFailure(Exception e) {
-                    logger.debug("Ping execution failed", e);
-                    sendPingsHandler.close();
-                }
-            });
-        } catch (EsRejectedExecutionException ex) { // TODO: remove this once ScheduledExecutor has support for AbstractRunnable
-            sendPingsHandler.close();
-            // we are shutting down
-        } catch (Exception e) {
-            sendPingsHandler.close();
-            throw new ElasticsearchException("Ping execution failed", e);
-        }
-    }
-
-    class SendPingsHandler implements Closeable {
-        private final int id;
-        private final Set<DiscoveryNode> nodeToDisconnect = ConcurrentCollections.newConcurrentSet();
-        private final PingCollection pingCollection;
-
-        private AtomicBoolean closed = new AtomicBoolean(false);
-
-        SendPingsHandler(int id) {
-            this.id = id;
-            this.pingCollection = new PingCollection();
-        }
-
-        public int id() {
-            return this.id;
-        }
-
-        public boolean isClosed() {
-            return this.closed.get();
-        }
-
-        public PingCollection pingCollection() {
-            return pingCollection;
-        }
-
-        @Override
-        public void close() {
-            if (closed.compareAndSet(false, true)) {
-                receivedResponses.remove(id);
-            }
-        }
-    }
-
-
-    void sendPings(final TimeValue timeout, @Nullable TimeValue waitTime, final SendPingsHandler sendPingsHandler) {
-        final UnicastPingRequest pingRequest = new UnicastPingRequest();
-        pingRequest.id = sendPingsHandler.id();
-        pingRequest.timeout = timeout;
-        DiscoveryNodes discoNodes = contextProvider.nodes();
-
-        pingRequest.pingResponse = createPingResponse(discoNodes);
-
-        HashSet<DiscoveryNode> nodesToPingSet = new HashSet<>();
-        for (PingResponse temporalResponse : temporalResponses) {
-            // Only send pings to nodes that have the same cluster name.
-            if (clusterName.equals(temporalResponse.clusterName())) {
-                nodesToPingSet.add(temporalResponse.node());
-            }
-        }
-
-        for (UnicastHostsProvider provider : hostsProviders) {
-            nodesToPingSet.addAll(provider.buildDynamicNodes());
-        }
-
-        // add all possible master nodes that were active in the last known cluster configuration
-        for (ObjectCursor<DiscoveryNode> masterNode : discoNodes.getMasterNodes().values()) {
-            nodesToPingSet.add(masterNode.value);
-        }
-
-        // sort the nodes by likelihood of being an active master
-        List<DiscoveryNode> sortedNodesToPing = electMasterService.sortByMasterLikelihood(nodesToPingSet);
-
-        // new add the unicast targets first
-        List<DiscoveryNode> nodesToPing = CollectionUtils.arrayAsArrayList(configuredTargetNodes);
-        nodesToPing.addAll(sortedNodesToPing);
-
-        final CountDownLatch latch = new CountDownLatch(nodesToPing.size());
-        for (final DiscoveryNode node : nodesToPing) {
-            // make sure we are connected
-            final boolean nodeFoundByAddress;
-            DiscoveryNode nodeToSend = discoNodes.findByAddress(node.getAddress());
-            if (nodeToSend != null) {
-                nodeFoundByAddress = true;
-            } else {
-                nodeToSend = node;
-                nodeFoundByAddress = false;
-            }
-
-            if (!transportService.nodeConnected(nodeToSend)) {
-                if (sendPingsHandler.isClosed()) {
-                    return;
-                }
-                // if we find on the disco nodes a matching node by address, we are going to restore the connection
-                // anyhow down the line if its not connected...
-                // if we can't resolve the node, we don't know and we have to clean up after pinging. We do have
-                // to make sure we don't disconnect a true node which was temporarily removed from the DiscoveryNodes
-                // but will be added again during the pinging. We therefore create a new temporary node
-                if (!nodeFoundByAddress) {
-                    if (!nodeToSend.getId().startsWith(UNICAST_NODE_PREFIX)) {
-                        DiscoveryNode tempNode = new DiscoveryNode("",
-                            UNICAST_NODE_PREFIX + unicastNodeIdGenerator.incrementAndGet() + "_" + nodeToSend.getId() + "#",
-                            UUIDs.randomBase64UUID(), nodeToSend.getHostName(), nodeToSend.getHostAddress(), nodeToSend.getAddress(),
-                            nodeToSend.getAttributes(), nodeToSend.getRoles(), nodeToSend.getVersion());
-
-                        logger.trace("replacing {} with temp node {}", nodeToSend, tempNode);
-                        nodeToSend = tempNode;
-                    }
-                    sendPingsHandler.nodeToDisconnect.add(nodeToSend);
-                }
-                // fork the connection to another thread
-                final DiscoveryNode finalNodeToSend = nodeToSend;
-                unicastConnectExecutor.execute(new Runnable() {
-                    @Override
-                    public void run() {
-                        if (sendPingsHandler.isClosed()) {
-                            return;
-                        }
-                        boolean success = false;
-                        try {
-                            // connect to the node, see if we manage to do it, if not, bail
-                            if (!nodeFoundByAddress) {
-                                logger.trace("[{}] connecting (light) to {}", sendPingsHandler.id(), finalNodeToSend);
-                                transportService.connectToNodeLightAndHandshake(finalNodeToSend, timeout.getMillis());
-                            } else {
-                                logger.trace("[{}] connecting to {}", sendPingsHandler.id(), finalNodeToSend);
-                                transportService.connectToNode(finalNodeToSend);
-                            }
-                            logger.trace("[{}] connected to {}", sendPingsHandler.id(), node);
-                            if (receivedResponses.containsKey(sendPingsHandler.id())) {
-                                // we are connected and still in progress, send the ping request
-                                sendPingRequestToNode(sendPingsHandler.id(), timeout, pingRequest, latch, node, finalNodeToSend);
-                            } else {
-                                // connect took too long, just log it and bail
-                                latch.countDown();
-                                logger.trace("[{}] connect to {} was too long outside of ping window, bailing",
-                                        sendPingsHandler.id(), node);
-                            }
-                            success = true;
-                        } catch (ConnectTransportException e) {
-                            // can't connect to the node - this is a more common path!
-                            logger.trace(
-                                (Supplier<?>) () -> new ParameterizedMessage(
-                                    "[{}] failed to connect to {}", sendPingsHandler.id(), finalNodeToSend), e);
-                        } catch (RemoteTransportException e) {
-                            // something went wrong on the other side
-                            logger.debug(
-                                (Supplier<?>) () -> new ParameterizedMessage(
-                                    "[{}] received a remote error as a response to ping {}", sendPingsHandler.id(), finalNodeToSend), e);
-                        } catch (Exception e) {
-                            logger.warn(
-                                (Supplier<?>) () -> new ParameterizedMessage(
-                                    "[{}] failed send ping to {}", sendPingsHandler.id(), finalNodeToSend), e);
-                        } finally {
-                            if (!success) {
-                                latch.countDown();
-                            }
-                        }
-                    }
-                });
-            } else {
-                sendPingRequestToNode(sendPingsHandler.id(), timeout, pingRequest, latch, node, nodeToSend);
-            }
-        }
-        if (waitTime != null) {
-            try {
-                latch.await(waitTime.millis(), TimeUnit.MILLISECONDS);
-            } catch (InterruptedException e) {
-                // ignore
-            }
-        }
-    }
-
-    private void sendPingRequestToNode(final int id, final TimeValue timeout, final UnicastPingRequest pingRequest,
-                                       final CountDownLatch latch, final DiscoveryNode node, final DiscoveryNode nodeToSend) {
-        logger.trace("[{}] sending to {}", id, nodeToSend);
-        transportService.sendRequest(nodeToSend, ACTION_NAME, pingRequest, TransportRequestOptions.builder()
-                .withTimeout((long) (timeout.millis() * 1.25)).build(), new TransportResponseHandler<UnicastPingResponse>() {
-
-            @Override
-            public UnicastPingResponse newInstance() {
-                return new UnicastPingResponse();
-            }
-
-            @Override
-            public String executor() {
-                return ThreadPool.Names.SAME;
-            }
-
-            @Override
-            public void handleResponse(UnicastPingResponse response) {
-                logger.trace("[{}] received response from {}: {}", id, nodeToSend, Arrays.toString(response.pingResponses));
-                try {
-                    DiscoveryNodes discoveryNodes = contextProvider.nodes();
-                    for (PingResponse pingResponse : response.pingResponses) {
-                        if (pingResponse.node().equals(discoveryNodes.getLocalNode())) {
-                            // that's us, ignore
-                            continue;
-                        }
-                        SendPingsHandler sendPingsHandler = receivedResponses.get(response.id);
-                        if (sendPingsHandler == null) {
-                            if (!closed) {
-                                // Only log when we're not closing the node. Having no send ping handler is then expected
-                                logger.warn("received ping response {} with no matching handler id [{}]", pingResponse, response.id);
-                            }
-                        } else {
-                            sendPingsHandler.pingCollection().addPing(pingResponse);
-                        }
-                    }
-                } finally {
-                    latch.countDown();
-                }
-            }
-
-            @Override
-            public void handleException(TransportException exp) {
-                latch.countDown();
-                if (exp instanceof ConnectTransportException) {
-                    // ok, not connected...
-                    logger.trace((Supplier<?>) () -> new ParameterizedMessage("failed to connect to {}", nodeToSend), exp);
-                } else {
-                    logger.warn((Supplier<?>) () -> new ParameterizedMessage("failed to send ping to [{}]", node), exp);
-                }
-            }
-        });
-    }
-
-    private UnicastPingResponse handlePingRequest(final UnicastPingRequest request) {
-        if (!lifecycle.started()) {
-            throw new IllegalStateException("received ping request while not started");
-        }
-        temporalResponses.add(request.pingResponse);
-        threadPool.schedule(TimeValue.timeValueMillis(request.timeout.millis() * 2), ThreadPool.Names.SAME, new Runnable() {
-            @Override
-            public void run() {
-                temporalResponses.remove(request.pingResponse);
-            }
-        });
-
-        List<PingResponse> pingResponses = CollectionUtils.iterableAsArrayList(temporalResponses);
-        pingResponses.add(createPingResponse(contextProvider.nodes()));
-
-
-        UnicastPingResponse unicastPingResponse = new UnicastPingResponse();
-        unicastPingResponse.id = request.id;
-        unicastPingResponse.pingResponses = pingResponses.toArray(new PingResponse[pingResponses.size()]);
-
-        return unicastPingResponse;
-    }
-
-    class UnicastPingRequestHandler implements TransportRequestHandler<UnicastPingRequest> {
-
-        @Override
-        public void messageReceived(UnicastPingRequest request, TransportChannel channel) throws Exception {
-            channel.sendResponse(handlePingRequest(request));
-        }
-    }
-
-    public static class UnicastPingRequest extends TransportRequest {
-
-        int id;
-        TimeValue timeout;
-        PingResponse pingResponse;
-
-        public UnicastPingRequest() {
-        }
-
-        @Override
-        public void readFrom(StreamInput in) throws IOException {
-            super.readFrom(in);
-            id = in.readInt();
-            timeout = new TimeValue(in);
-            pingResponse = readPingResponse(in);
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            super.writeTo(out);
-            out.writeInt(id);
-            timeout.writeTo(out);
-            pingResponse.writeTo(out);
-        }
-    }
-
-    private PingResponse createPingResponse(DiscoveryNodes discoNodes) {
-        return new PingResponse(discoNodes.getLocalNode(), discoNodes.getMasterNode(), clusterName,
-                contextProvider.nodeHasJoinedClusterOnce());
-    }
-
-    static class UnicastPingResponse extends TransportResponse {
-
-        int id;
-
-        PingResponse[] pingResponses;
-
-        UnicastPingResponse() {
-        }
-
-        @Override
-        public void readFrom(StreamInput in) throws IOException {
-            super.readFrom(in);
-            id = in.readInt();
-            pingResponses = new PingResponse[in.readVInt()];
-            for (int i = 0; i < pingResponses.length; i++) {
-                pingResponses[i] = readPingResponse(in);
-            }
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            super.writeTo(out);
-            out.writeInt(id);
-            out.writeVInt(pingResponses.length);
-            for (PingResponse pingResponse : pingResponses) {
-                pingResponse.writeTo(out);
-            }
-        }
-    }
-
-    protected Version getVersion() {
-        return Version.CURRENT; // for tests
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/env/Environment.java b/core/src/main/java/org/elasticsearch/env/Environment.java
index 4b544aa38820c..8b039f79da4a7 100644
--- a/core/src/main/java/org/elasticsearch/env/Environment.java
+++ b/core/src/main/java/org/elasticsearch/env/Environment.java
@@ -49,11 +49,18 @@
 // public+forbidden api!
 public class Environment {
     public static final Setting<String> PATH_HOME_SETTING = Setting.simpleString("path.home", Property.NodeScope);
-    public static final Setting<String> PATH_CONF_SETTING = Setting.simpleString("path.conf", Property.NodeScope);
-    public static final Setting<String> PATH_SCRIPTS_SETTING = Setting.simpleString("path.scripts", Property.NodeScope);
+    public static final Setting<String> DEFAULT_PATH_CONF_SETTING = Setting.simpleString("default.path.conf", Property.NodeScope);
+    public static final Setting<String> PATH_CONF_SETTING =
+            new Setting<>("path.conf", DEFAULT_PATH_CONF_SETTING, Function.identity(), Property.NodeScope);
+    public static final Setting<String> PATH_SCRIPTS_SETTING =
+        Setting.simpleString("path.scripts", Property.NodeScope, Property.Deprecated);
+    public static final Setting<List<String>> DEFAULT_PATH_DATA_SETTING =
+            Setting.listSetting("default.path.data", Collections.emptyList(), Function.identity(), Property.NodeScope);
     public static final Setting<List<String>> PATH_DATA_SETTING =
-        Setting.listSetting("path.data", Collections.emptyList(), Function.identity(), Property.NodeScope);
-    public static final Setting<String> PATH_LOGS_SETTING = Setting.simpleString("path.logs", Property.NodeScope);
+            Setting.listSetting("path.data", DEFAULT_PATH_DATA_SETTING, Function.identity(), Property.NodeScope);
+    public static final Setting<String> DEFAULT_PATH_LOGS_SETTING = Setting.simpleString("default.path.logs", Property.NodeScope);
+    public static final Setting<String> PATH_LOGS_SETTING =
+            new Setting<>("path.logs", DEFAULT_PATH_LOGS_SETTING, Function.identity(), Property.NodeScope);
     public static final Setting<List<String>> PATH_REPO_SETTING =
         Setting.listSetting("path.repo", Collections.emptyList(), Function.identity(), Property.NodeScope);
     public static final Setting<String> PATH_SHARED_DATA_SETTING = Setting.simpleString("path.shared_data", Property.NodeScope);
@@ -61,6 +68,8 @@ public class Environment {
 
     private final Settings settings;
 
+    private final String configExtension;
+
     private final Path[] dataFiles;
 
     private final Path[] dataWithClusterFiles;
@@ -69,8 +78,6 @@ public class Environment {
 
     private final Path configFile;
 
-    private final Path scriptsFile;
-
     private final Path pluginsFile;
 
     private final Path modulesFile;
@@ -108,6 +115,12 @@ public class Environment {
     }
 
     public Environment(Settings settings) {
+        this(settings, null);
+    }
+
+    // Note: Do not use this ctor, it is for correct deprecation logging in 5.5 and will be removed
+    public Environment(Settings settings, String configExtension) {
+        this.configExtension = configExtension;
         final Path homeFile;
         if (PATH_HOME_SETTING.exists(settings)) {
             homeFile = PathUtils.get(cleanPath(PATH_HOME_SETTING.get(settings)));
@@ -115,18 +128,13 @@ public Environment(Settings settings) {
             throw new IllegalStateException(PATH_HOME_SETTING.getKey() + " is not configured");
         }
 
-        if (PATH_CONF_SETTING.exists(settings)) {
+        // this is trappy, Setting#get(Settings) will get a fallback setting yet return false for Settings#exists(Settings)
+        if (PATH_CONF_SETTING.exists(settings) || DEFAULT_PATH_CONF_SETTING.exists(settings)) {
             configFile = PathUtils.get(cleanPath(PATH_CONF_SETTING.get(settings)));
         } else {
             configFile = homeFile.resolve("config");
         }
 
-        if (PATH_SCRIPTS_SETTING.exists(settings)) {
-            scriptsFile = PathUtils.get(cleanPath(PATH_SCRIPTS_SETTING.get(settings)));
-        } else {
-            scriptsFile = configFile.resolve("scripts");
-        }
-
         pluginsFile = homeFile.resolve("plugins");
 
         List<String> dataPaths = PATH_DATA_SETTING.get(settings);
@@ -156,7 +164,9 @@ public Environment(Settings settings) {
         } else {
             repoFiles = new Path[0];
         }
-        if (PATH_LOGS_SETTING.exists(settings)) {
+
+        // this is trappy, Setting#get(Settings) will get a fallback setting yet return false for Settings#exists(Settings)
+        if (PATH_LOGS_SETTING.exists(settings) || DEFAULT_PATH_LOGS_SETTING.exists(settings)) {
             logsFile = PathUtils.get(cleanPath(PATH_LOGS_SETTING.get(settings)));
         } else {
             logsFile = homeFile.resolve("logs");
@@ -174,7 +184,9 @@ public Environment(Settings settings) {
 
         Settings.Builder finalSettings = Settings.builder().put(settings);
         finalSettings.put(PATH_HOME_SETTING.getKey(), homeFile);
-        finalSettings.putArray(PATH_DATA_SETTING.getKey(), dataPaths);
+        if (PATH_DATA_SETTING.exists(settings)) {
+            finalSettings.putArray(PATH_DATA_SETTING.getKey(), dataPaths);
+        }
         finalSettings.put(PATH_LOGS_SETTING.getKey(), logsFile);
         this.settings = finalSettings.build();
 
@@ -274,8 +286,14 @@ public URL resolveRepoURL(URL url) {
         }
     }
 
+    /** Return then extension of the config file that was loaded, or*/
+    public String configExtension() {
+        return configExtension;
+    }
+
+    // TODO: rename all these "file" methods to "dir"
     /**
-     * The config location.
+     * The config directory.
      */
     public Path configFile() {
         return configFile;
@@ -285,7 +303,11 @@ public Path configFile() {
      * Location of on-disk scripts
      */
     public Path scriptsFile() {
-        return scriptsFile;
+        if (PATH_SCRIPTS_SETTING.exists(settings)) {
+            return PathUtils.get(cleanPath(PATH_SCRIPTS_SETTING.get(settings)));
+        } else {
+            return configFile.resolve("scripts");
+        }
     }
 
     public Path pluginsFile() {
diff --git a/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java b/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java
index df9514cdf8830..469cd99cb72a4 100644
--- a/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java
+++ b/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java
@@ -45,6 +45,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.gateway.MetaDataStateFormat;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
@@ -216,7 +217,7 @@ public NodeEnvironment(Settings settings, Environment environment) throws IOExce
                                         "Elasticsearch 6.0 will not allow the cluster name as a folder within the data path", dataDir);
                         dataDir = dataDirWithClusterName;
                     }
-                    Path dir = dataDir.resolve(NODES_FOLDER).resolve(Integer.toString(possibleLockId));
+                    Path dir = resolveNodePath(dataDir, possibleLockId);
                     Files.createDirectories(dir);
 
                     try (Directory luceneDir = FSDirectory.open(dir, NativeFSLockFactory.INSTANCE)) {
@@ -226,7 +227,8 @@ public NodeEnvironment(Settings settings, Environment environment) throws IOExce
                             nodePaths[dirIndex] = new NodePath(dir);
                             nodeLockId = possibleLockId;
                         } catch (LockObtainFailedException ex) {
-                            startupTraceLogger.trace("failed to obtain node lock on {}", dir.toAbsolutePath());
+                            startupTraceLogger.trace(
+                                    new ParameterizedMessage("failed to obtain node lock on {}", dir.toAbsolutePath()), ex);
                             // release all the ones that were obtained up until now
                             releaseAndNullLocks(locks);
                             break;
@@ -282,6 +284,17 @@ public NodeEnvironment(Settings settings, Environment environment) throws IOExce
         }
     }
 
+    /**
+     * Resolve a specific nodes/{node.id} path for the specified path and node lock id.
+     *
+     * @param path       the path
+     * @param nodeLockId the node lock id
+     * @return the resolved path
+     */
+    public static Path resolveNodePath(final Path path, final int nodeLockId) {
+        return path.resolve(NODES_FOLDER).resolve(Integer.toString(nodeLockId));
+    }
+
     /** Returns true if the directory is empty */
     private static boolean dirEmpty(final Path path) throws IOException {
         try (DirectoryStream<Path> stream = Files.newDirectoryStream(path)) {
@@ -398,7 +411,7 @@ private void maybeLogHeapDetails() {
     private static NodeMetaData loadOrCreateNodeMetaData(Settings settings, Logger logger,
                                                          NodePath... nodePaths) throws IOException {
         final Path[] paths = Arrays.stream(nodePaths).map(np -> np.path).toArray(Path[]::new);
-        NodeMetaData metaData = NodeMetaData.FORMAT.loadLatestState(logger, paths);
+        NodeMetaData metaData = NodeMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, paths);
         if (metaData == null) {
             metaData = new NodeMetaData(generateNodeId(settings));
         }
@@ -757,6 +770,14 @@ public NodePath[] nodePaths() {
         return nodePaths;
     }
 
+    public int getNodeLockId() {
+        assertEnvIsLocked();
+        if (nodePaths == null || locks == null) {
+            throw new IllegalStateException("node is not configured to store local location");
+        }
+        return nodeLockId;
+    }
+
     /**
      * Returns all index paths.
      */
@@ -769,6 +790,8 @@ public Path[] indexPaths(Index index) {
         return indexPaths;
     }
 
+
+
     /**
      * Returns all shard paths excluding custom shard path. Note: Shards are only allocated on one of the
      * returned paths. The returned array may contain paths to non-existing directories.
@@ -797,19 +820,36 @@ public Set<String> availableIndexFolders() throws IOException {
         assertEnvIsLocked();
         Set<String> indexFolders = new HashSet<>();
         for (NodePath nodePath : nodePaths) {
-            Path indicesLocation = nodePath.indicesPath;
-            if (Files.isDirectory(indicesLocation)) {
-                try (DirectoryStream<Path> stream = Files.newDirectoryStream(indicesLocation)) {
-                    for (Path index : stream) {
-                        if (Files.isDirectory(index)) {
-                            indexFolders.add(index.getFileName().toString());
-                        }
+            indexFolders.addAll(availableIndexFoldersForPath(nodePath));
+        }
+        return indexFolders;
+
+    }
+
+    /**
+     * Return all directory names in the nodes/{node.id}/indices directory for the given node path.
+     *
+     * @param nodePath the path
+     * @return all directories that could be indices for the given node path.
+     * @throws IOException if an I/O exception occurs traversing the filesystem
+     */
+    public Set<String> availableIndexFoldersForPath(final NodePath nodePath) throws IOException {
+        if (nodePaths == null || locks == null) {
+            throw new IllegalStateException("node is not configured to store local location");
+        }
+        assertEnvIsLocked();
+        final Set<String> indexFolders = new HashSet<>();
+        Path indicesLocation = nodePath.indicesPath;
+        if (Files.isDirectory(indicesLocation)) {
+            try (DirectoryStream<Path> stream = Files.newDirectoryStream(indicesLocation)) {
+                for (Path index : stream) {
+                    if (Files.isDirectory(index)) {
+                        indexFolders.add(index.getFileName().toString());
                     }
                 }
             }
         }
         return indexFolders;
-
     }
 
     /**
@@ -917,11 +957,12 @@ public void ensureAtomicMoveSupported() throws IOException {
         final NodePath[] nodePaths = nodePaths();
         for (NodePath nodePath : nodePaths) {
             assert Files.isDirectory(nodePath.path) : nodePath.path + " is not a directory";
-            final Path src = nodePath.path.resolve("__es__.tmp");
-            final Path target = nodePath.path.resolve("__es__.final");
+            final Path src = nodePath.path.resolve(TEMP_FILE_NAME + ".tmp");
+            final Path target = nodePath.path.resolve(TEMP_FILE_NAME + ".final");
             try {
+                Files.deleteIfExists(src);
                 Files.createFile(src);
-                Files.move(src, target, StandardCopyOption.ATOMIC_MOVE);
+                Files.move(src, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
             } catch (AtomicMoveNotSupportedException ex) {
                 throw new IllegalStateException("atomic_move is not supported by the filesystem on path ["
                         + nodePath.path
@@ -1025,14 +1066,19 @@ private void assertCanWrite() throws IOException {
         }
     }
 
+    // package private for testing
+    static final String TEMP_FILE_NAME = ".es_temp_file";
+
     private static void tryWriteTempFile(Path path) throws IOException {
         if (Files.exists(path)) {
-            Path resolve = path.resolve(".es_temp_file");
+            Path resolve = path.resolve(TEMP_FILE_NAME);
             try {
-                Files.createFile(resolve);
+                // delete any lingering file from a previous failure
                 Files.deleteIfExists(resolve);
+                Files.createFile(resolve);
+                Files.delete(resolve);
             } catch (IOException ex) {
-                throw new IOException("failed to write in data directory [" + path + "] write permission is required", ex);
+                throw new IOException("failed to test writes in data directory [" + path + "] write permission is required", ex);
             }
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/env/NodeMetaData.java b/core/src/main/java/org/elasticsearch/env/NodeMetaData.java
index 60625b1852dd6..38a4fce9cdc3d 100644
--- a/core/src/main/java/org/elasticsearch/env/NodeMetaData.java
+++ b/core/src/main/java/org/elasticsearch/env/NodeMetaData.java
@@ -20,8 +20,6 @@
 package org.elasticsearch.env;
 
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -70,8 +68,7 @@ public String toString() {
         return "node_id [" + nodeId + "]";
     }
 
-    private static ObjectParser<Builder, ParseFieldMatcherSupplier> PARSER = new ObjectParser<>("node_meta_data",
-        Builder::new);
+    private static ObjectParser<Builder, Void> PARSER = new ObjectParser<>("node_meta_data", Builder::new);
 
     static {
         PARSER.declareString(Builder::setNodeId, new ParseField(NODE_ID_KEY));
@@ -110,7 +107,7 @@ public void toXContent(XContentBuilder builder, NodeMetaData nodeMetaData) throw
 
         @Override
         public NodeMetaData fromXContent(XContentParser parser) throws IOException {
-            return PARSER.apply(parser, () -> ParseFieldMatcher.STRICT).build();
+            return PARSER.apply(parser, null).build();
         }
     };
 }
diff --git a/core/src/main/java/org/elasticsearch/env/ShardLockObtainFailedException.java b/core/src/main/java/org/elasticsearch/env/ShardLockObtainFailedException.java
index d1a8ce3b6d480..21bba81adbb1f 100644
--- a/core/src/main/java/org/elasticsearch/env/ShardLockObtainFailedException.java
+++ b/core/src/main/java/org/elasticsearch/env/ShardLockObtainFailedException.java
@@ -19,30 +19,36 @@
 
 package org.elasticsearch.env;
 
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.index.shard.ShardId;
 
+import java.io.IOException;
+
 /**
  * Exception used when the in-memory lock for a shard cannot be obtained
  */
-public class ShardLockObtainFailedException extends Exception {
-    private final ShardId shardId;
+public class ShardLockObtainFailedException extends ElasticsearchException {
 
     public ShardLockObtainFailedException(ShardId shardId, String message) {
-        super(message);
-        this.shardId = shardId;
+        super(buildMessage(shardId, message));
+        this.setShard(shardId);
     }
 
     public ShardLockObtainFailedException(ShardId shardId, String message, Throwable cause) {
-        super(message, cause);
-        this.shardId = shardId;
+        super(buildMessage(shardId, message), cause);
+        this.setShard(shardId);
+    }
+
+    public ShardLockObtainFailedException(StreamInput in) throws IOException {
+        super(in);
     }
 
-    @Override
-    public String getMessage() {
+    private static String buildMessage(ShardId shardId, String message) {
         StringBuilder sb = new StringBuilder();
         sb.append(shardId.toString());
         sb.append(": ");
-        sb.append(super.getMessage());
+        sb.append(message);
         return sb.toString();
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/gateway/AsyncShardFetch.java b/core/src/main/java/org/elasticsearch/gateway/AsyncShardFetch.java
index 42c40034b1056..e2bbae775e5d7 100644
--- a/core/src/main/java/org/elasticsearch/gateway/AsyncShardFetch.java
+++ b/core/src/main/java/org/elasticsearch/gateway/AsyncShardFetch.java
@@ -44,6 +44,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicLong;
 
 import static java.util.Collections.emptySet;
 import static java.util.Collections.unmodifiableSet;
@@ -67,10 +68,11 @@ public interface Lister<NodesResponse extends BaseNodesResponse<NodeResponse>, N
 
     protected final Logger logger;
     protected final String type;
-    private final ShardId shardId;
+    protected final ShardId shardId;
     private final Lister<BaseNodesResponse<T>, T> action;
     private final Map<String, NodeEntry<T>> cache = new HashMap<>();
     private final Set<String> nodesToIgnore = new HashSet<>();
+    private final AtomicLong round = new AtomicLong();
     private boolean closed;
 
     @SuppressWarnings("unchecked")
@@ -112,20 +114,22 @@ public synchronized FetchResult<T> fetchData(DiscoveryNodes nodes, Set<String> i
         }
         nodesToIgnore.addAll(ignoreNodes);
         fillShardCacheWithDataNodes(cache, nodes);
-        Set<NodeEntry<T>> nodesToFetch = findNodesToFetch(cache);
+        List<NodeEntry<T>> nodesToFetch = findNodesToFetch(cache);
         if (nodesToFetch.isEmpty() == false) {
             // mark all node as fetching and go ahead and async fetch them
+            // use a unique round id to detect stale responses in processAsyncFetch
+            final long fetchingRound = round.incrementAndGet();
             for (NodeEntry<T> nodeEntry : nodesToFetch) {
-                nodeEntry.markAsFetching();
+                nodeEntry.markAsFetching(fetchingRound);
             }
             DiscoveryNode[] discoNodesToFetch = nodesToFetch.stream().map(NodeEntry::getNodeId).map(nodes::get)
                 .toArray(DiscoveryNode[]::new);
-            asyncFetch(shardId, discoNodesToFetch);
+            asyncFetch(discoNodesToFetch, fetchingRound);
         }
 
         // if we are still fetching, return null to indicate it
-        if (hasAnyNodeFetching(cache) == true) {
-            return new FetchResult<>(shardId, null, emptySet(), emptySet());
+        if (hasAnyNodeFetching(cache)) {
+            return new FetchResult<>(shardId, null, emptySet());
         } else {
             // nothing to fetch, yay, build the return value
             Map<DiscoveryNode, T> fetchData = new HashMap<>();
@@ -137,7 +141,7 @@ public synchronized FetchResult<T> fetchData(DiscoveryNodes nodes, Set<String> i
 
                 DiscoveryNode node = nodes.get(nodeId);
                 if (node != null) {
-                    if (nodeEntry.isFailed() == true) {
+                    if (nodeEntry.isFailed()) {
                         // if its failed, remove it from the list of nodes, so if this run doesn't work
                         // we try again next round to fetch it again
                         it.remove();
@@ -158,7 +162,7 @@ public synchronized FetchResult<T> fetchData(DiscoveryNodes nodes, Set<String> i
             if (failedNodes.isEmpty() == false || allIgnoreNodes.isEmpty() == false) {
                 reroute(shardId, "nodes failed [" + failedNodes.size() + "], ignored [" + allIgnoreNodes.size() + "]");
             }
-            return new FetchResult<>(shardId, fetchData, failedNodes, allIgnoreNodes);
+            return new FetchResult<>(shardId, fetchData, allIgnoreNodes);
         }
     }
 
@@ -168,7 +172,7 @@ public synchronized FetchResult<T> fetchData(DiscoveryNodes nodes, Set<String> i
      * the shard (response + failures), issuing a reroute at the end of it to make sure there will be another round
      * of allocations taking this new data into account.
      */
-    protected synchronized void processAsyncFetch(ShardId shardId, List<T> responses, List<FailedNodeException> failures) {
+    protected synchronized void processAsyncFetch(List<T> responses, List<FailedNodeException> failures, long fetchingRound) {
         if (closed) {
             // we are closed, no need to process this async fetch at all
             logger.trace("{} ignoring fetched [{}] results, already closed", shardId, type);
@@ -179,15 +183,19 @@ protected synchronized void processAsyncFetch(ShardId shardId, List<T> responses
         if (responses != null) {
             for (T response : responses) {
                 NodeEntry<T> nodeEntry = cache.get(response.getNode().getId());
-                // if the entry is there, and not marked as failed already, process it
-                if (nodeEntry == null) {
-                    continue;
-                }
-                if (nodeEntry.isFailed()) {
-                    logger.trace("{} node {} has failed for [{}] (failure [{}])", shardId, nodeEntry.getNodeId(), type, nodeEntry.getFailure());
-                } else {
-                    logger.trace("{} marking {} as done for [{}], result is [{}]", shardId, nodeEntry.getNodeId(), type, response);
-                    nodeEntry.doneFetching(response);
+                if (nodeEntry != null) {
+                    if (nodeEntry.getFetchingRound() != fetchingRound) {
+                        assert nodeEntry.getFetchingRound() > fetchingRound : "node entries only replaced by newer rounds";
+                        logger.trace("{} received response for [{}] from node {} for an older fetching round (expected: {} but was: {})",
+                            shardId, nodeEntry.getNodeId(), type, nodeEntry.getFetchingRound(), fetchingRound);
+                    } else if (nodeEntry.isFailed()) {
+                        logger.trace("{} node {} has failed for [{}] (failure [{}])", shardId, nodeEntry.getNodeId(), type,
+                            nodeEntry.getFailure());
+                    } else {
+                        // if the entry is there, for the right fetching round and not marked as failed already, process it
+                        logger.trace("{} marking {} as done for [{}], result is [{}]", shardId, nodeEntry.getNodeId(), type, response);
+                        nodeEntry.doneFetching(response);
+                    }
                 }
             }
         }
@@ -195,15 +203,24 @@ protected synchronized void processAsyncFetch(ShardId shardId, List<T> responses
             for (FailedNodeException failure : failures) {
                 logger.trace("{} processing failure {} for [{}]", shardId, failure, type);
                 NodeEntry<T> nodeEntry = cache.get(failure.nodeId());
-                // if the entry is there, and not marked as failed already, process it
-                if (nodeEntry != null && nodeEntry.isFailed() == false) {
-                    Throwable unwrappedCause = ExceptionsHelper.unwrapCause(failure.getCause());
-                    // if the request got rejected or timed out, we need to try it again next time...
-                    if (unwrappedCause instanceof EsRejectedExecutionException || unwrappedCause instanceof ReceiveTimeoutTransportException || unwrappedCause instanceof ElasticsearchTimeoutException) {
-                        nodeEntry.restartFetching();
-                    } else {
-                        logger.warn((Supplier<?>) () -> new ParameterizedMessage("{}: failed to list shard for {} on node [{}]", shardId, type, failure.nodeId()), failure);
-                        nodeEntry.doneFetching(failure.getCause());
+                if (nodeEntry != null) {
+                    if (nodeEntry.getFetchingRound() != fetchingRound) {
+                        assert nodeEntry.getFetchingRound() > fetchingRound : "node entries only replaced by newer rounds";
+                        logger.trace("{} received failure for [{}] from node {} for an older fetching round (expected: {} but was: {})",
+                            shardId, nodeEntry.getNodeId(), type, nodeEntry.getFetchingRound(), fetchingRound);
+                    } else if (nodeEntry.isFailed() == false) {
+                        // if the entry is there, for the right fetching round and not marked as failed already, process it
+                        Throwable unwrappedCause = ExceptionsHelper.unwrapCause(failure.getCause());
+                        // if the request got rejected or timed out, we need to try it again next time...
+                        if (unwrappedCause instanceof EsRejectedExecutionException ||
+                            unwrappedCause instanceof ReceiveTimeoutTransportException ||
+                            unwrappedCause instanceof ElasticsearchTimeoutException) {
+                            nodeEntry.restartFetching();
+                        } else {
+                            logger.warn((Supplier<?>) () -> new ParameterizedMessage("{}: failed to list shard for {} on node [{}]",
+                                shardId, type, failure.nodeId()), failure);
+                            nodeEntry.doneFetching(failure.getCause());
+                        }
                     }
                 }
             }
@@ -241,8 +258,8 @@ private void fillShardCacheWithDataNodes(Map<String, NodeEntry<T>> shardCache, D
      * Finds all the nodes that need to be fetched. Those are nodes that have no
      * data, and are not in fetch mode.
      */
-    private Set<NodeEntry<T>> findNodesToFetch(Map<String, NodeEntry<T>> shardCache) {
-        Set<NodeEntry<T>> nodesToFetch = new HashSet<>();
+    private List<NodeEntry<T>> findNodesToFetch(Map<String, NodeEntry<T>> shardCache) {
+        List<NodeEntry<T>> nodesToFetch = new ArrayList<>();
         for (NodeEntry<T> nodeEntry : shardCache.values()) {
             if (nodeEntry.hasData() == false && nodeEntry.isFetching() == false) {
                 nodesToFetch.add(nodeEntry);
@@ -267,12 +284,12 @@ private boolean hasAnyNodeFetching(Map<String, NodeEntry<T>> shardCache) {
      * Async fetches data for the provided shard with the set of nodes that need to be fetched from.
      */
     // visible for testing
-    void asyncFetch(final ShardId shardId, final DiscoveryNode[] nodes) {
+    void asyncFetch(final DiscoveryNode[] nodes, long fetchingRound) {
         logger.trace("{} fetching [{}] from {}", shardId, type, nodes);
         action.list(shardId, nodes, new ActionListener<BaseNodesResponse<T>>() {
             @Override
             public void onResponse(BaseNodesResponse<T> response) {
-                processAsyncFetch(shardId, response.getNodes(), response.failures());
+                processAsyncFetch(response.getNodes(), response.failures(), fetchingRound);
             }
 
             @Override
@@ -281,7 +298,7 @@ public void onFailure(Exception e) {
                 for (final DiscoveryNode node: nodes) {
                     failures.add(new FailedNodeException(node.getId(), "total failure in fetching", e));
                 }
-                processAsyncFetch(shardId, null, failures);
+                processAsyncFetch(null, failures, fetchingRound);
             }
         });
     }
@@ -294,13 +311,11 @@ public static class FetchResult<T extends BaseNodeResponse> {
 
         private final ShardId shardId;
         private final Map<DiscoveryNode, T> data;
-        private final Set<String> failedNodes;
         private final Set<String> ignoreNodes;
 
-        public FetchResult(ShardId shardId, Map<DiscoveryNode, T> data, Set<String> failedNodes, Set<String> ignoreNodes) {
+        public FetchResult(ShardId shardId, Map<DiscoveryNode, T> data, Set<String> ignoreNodes) {
             this.shardId = shardId;
             this.data = data;
-            this.failedNodes = failedNodes;
             this.ignoreNodes = ignoreNodes;
         }
 
@@ -342,8 +357,9 @@ static class NodeEntry<T> {
         private T value;
         private boolean valueSet;
         private Throwable failure;
+        private long fetchingRound;
 
-        public NodeEntry(String nodeId) {
+        NodeEntry(String nodeId) {
             this.nodeId = nodeId;
         }
 
@@ -355,13 +371,14 @@ boolean isFetching() {
             return fetching;
         }
 
-        void markAsFetching() {
+        void markAsFetching(long fetchingRound) {
             assert fetching == false : "double marking a node as fetching";
-            fetching = true;
+            this.fetching = true;
+            this.fetchingRound = fetchingRound;
         }
 
         void doneFetching(T value) {
-            assert fetching == true : "setting value but not in fetching mode";
+            assert fetching : "setting value but not in fetching mode";
             assert failure == null : "setting value when failure already set";
             this.valueSet = true;
             this.value = value;
@@ -369,7 +386,7 @@ void doneFetching(T value) {
         }
 
         void doneFetching(Throwable failure) {
-            assert fetching == true : "setting value but not in fetching mode";
+            assert fetching : "setting value but not in fetching mode";
             assert valueSet == false : "setting failure when already set value";
             assert failure != null : "setting failure can't be null";
             this.failure = failure;
@@ -377,7 +394,7 @@ void doneFetching(Throwable failure) {
         }
 
         void restartFetching() {
-            assert fetching == true : "restarting fetching, but not in fetching mode";
+            assert fetching : "restarting fetching, but not in fetching mode";
             assert valueSet == false : "value can't be set when restarting fetching";
             assert failure == null : "failure can't be set when restarting fetching";
             this.fetching = false;
@@ -388,7 +405,7 @@ boolean isFailed() {
         }
 
         boolean hasData() {
-            return valueSet == true || failure != null;
+            return valueSet || failure != null;
         }
 
         Throwable getFailure() {
@@ -399,8 +416,12 @@ Throwable getFailure() {
         @Nullable
         T getValue() {
             assert failure == null : "trying to fetch value, but its marked as failed, check isFailed";
-            assert valueSet == true : "value is not set, hasn't been fetched yet";
+            assert valueSet : "value is not set, hasn't been fetched yet";
             return value;
         }
+
+        long getFetchingRound() {
+            return fetchingRound;
+        }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/gateway/BaseGatewayShardAllocator.java b/core/src/main/java/org/elasticsearch/gateway/BaseGatewayShardAllocator.java
new file mode 100644
index 0000000000000..275311cf6a8f5
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/gateway/BaseGatewayShardAllocator.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gateway;
+
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.cluster.routing.RoutingNode;
+import org.elasticsearch.cluster.routing.RoutingNodes;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.allocation.AllocateUnassignedDecision;
+import org.elasticsearch.cluster.routing.allocation.AllocationDecision;
+import org.elasticsearch.cluster.routing.allocation.NodeAllocationResult;
+import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
+import org.elasticsearch.cluster.routing.allocation.decider.Decision;
+import org.elasticsearch.common.component.AbstractComponent;
+import org.elasticsearch.common.settings.Settings;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * An abstract class that implements basic functionality for allocating
+ * shards to nodes based on shard copies that already exist in the cluster.
+ *
+ * Individual implementations of this class are responsible for providing
+ * the logic to determine to which nodes (if any) those shards are allocated.
+ */
+public abstract class BaseGatewayShardAllocator extends AbstractComponent {
+
+    public BaseGatewayShardAllocator(Settings settings) {
+        super(settings);
+    }
+
+    /**
+     * Allocate unassigned shards to nodes (if any) where valid copies of the shard already exist.
+     * It is up to the individual implementations of {@link #makeAllocationDecision(ShardRouting, RoutingAllocation, Logger)}
+     * to make decisions on assigning shards to nodes.
+     *
+     * @param allocation the allocation state container object
+     */
+    public void allocateUnassigned(RoutingAllocation allocation) {
+        final RoutingNodes routingNodes = allocation.routingNodes();
+        final RoutingNodes.UnassignedShards.UnassignedIterator unassignedIterator = routingNodes.unassigned().iterator();
+        while (unassignedIterator.hasNext()) {
+            final ShardRouting shard = unassignedIterator.next();
+            final AllocateUnassignedDecision allocateUnassignedDecision = makeAllocationDecision(shard, allocation, logger);
+
+            if (allocateUnassignedDecision.isDecisionTaken() == false) {
+                // no decision was taken by this allocator
+                continue;
+            }
+
+            if (allocateUnassignedDecision.getAllocationDecision() == AllocationDecision.YES) {
+                unassignedIterator.initialize(allocateUnassignedDecision.getTargetNode().getId(),
+                    allocateUnassignedDecision.getAllocationId(),
+                    shard.primary() ? ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE :
+                                      allocation.clusterInfo().getShardSize(shard, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE),
+                    allocation.changes());
+            } else {
+                unassignedIterator.removeAndIgnore(allocateUnassignedDecision.getAllocationStatus(), allocation.changes());
+            }
+        }
+    }
+
+    /**
+     * Make a decision on the allocation of an unassigned shard.  This method is used by
+     * {@link #allocateUnassigned(RoutingAllocation)} to make decisions about whether or not
+     * the shard can be allocated by this allocator and if so, to which node it will be allocated.
+     *
+     * @param unassignedShard  the unassigned shard to allocate
+     * @param allocation       the current routing state
+     * @param logger           the logger
+     * @return an {@link AllocateUnassignedDecision} with the final decision of whether to allocate and details of the decision
+     */
+    public abstract AllocateUnassignedDecision makeAllocationDecision(ShardRouting unassignedShard,
+                                                                      RoutingAllocation allocation,
+                                                                      Logger logger);
+
+    /**
+     * Builds decisions for all nodes in the cluster, so that the explain API can provide information on
+     * allocation decisions for each node, while still waiting to allocate the shard (e.g. due to fetching shard data).
+     */
+    protected List<NodeAllocationResult> buildDecisionsForAllNodes(ShardRouting shard, RoutingAllocation allocation) {
+        List<NodeAllocationResult> results = new ArrayList<>();
+        for (RoutingNode node : allocation.routingNodes()) {
+            Decision decision = allocation.deciders().canAllocate(shard, node, allocation);
+            results.add(new NodeAllocationResult(node.node(), null, decision));
+        }
+        return results;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/gateway/DanglingIndicesState.java b/core/src/main/java/org/elasticsearch/gateway/DanglingIndicesState.java
index 370778898fc2c..acfcadb2f51b5 100644
--- a/core/src/main/java/org/elasticsearch/gateway/DanglingIndicesState.java
+++ b/core/src/main/java/org/elasticsearch/gateway/DanglingIndicesState.java
@@ -20,9 +20,12 @@
 package org.elasticsearch.gateway;
 
 import com.carrotsearch.hppc.cursors.ObjectCursor;
+import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.cluster.metadata.IndexGraveyard;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -48,7 +51,7 @@
  * their state written on disk, but don't exists in the metadata of the cluster), and importing
  * them into the cluster.
  */
-public class DanglingIndicesState extends AbstractComponent {
+public class DanglingIndicesState extends AbstractComponent implements ClusterStateListener {
 
     private final NodeEnvironment nodeEnv;
     private final MetaStateService metaStateService;
@@ -58,11 +61,12 @@ public class DanglingIndicesState extends AbstractComponent {
 
     @Inject
     public DanglingIndicesState(Settings settings, NodeEnvironment nodeEnv, MetaStateService metaStateService,
-                                LocalAllocateDangledIndices allocateDangledIndices) {
+                                LocalAllocateDangledIndices allocateDangledIndices, ClusterService clusterService) {
         super(settings);
         this.nodeEnv = nodeEnv;
         this.metaStateService = metaStateService;
         this.allocateDangledIndices = allocateDangledIndices;
+        clusterService.addListener(this);
     }
 
     /**
@@ -153,7 +157,7 @@ Map<Index, IndexMetaData> findNewDanglingIndices(final MetaData metaData) {
      * for allocation.
      */
     private void allocateDanglingIndices() {
-        if (danglingIndices.isEmpty() == true) {
+        if (danglingIndices.isEmpty()) {
             return;
         }
         try {
@@ -174,4 +178,11 @@ public void onFailure(Throwable e) {
             logger.warn("failed to send allocate dangled", e);
         }
     }
+
+    @Override
+    public void clusterChanged(ClusterChangedEvent event) {
+        if (event.state().blocks().disableStatePersistence() == false) {
+            processDanglingIndices(event.state().metaData());
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/gateway/Gateway.java b/core/src/main/java/org/elasticsearch/gateway/Gateway.java
index 3030632a769c4..94284f66e223d 100644
--- a/core/src/main/java/org/elasticsearch/gateway/Gateway.java
+++ b/core/src/main/java/org/elasticsearch/gateway/Gateway.java
@@ -21,11 +21,12 @@
 
 import com.carrotsearch.hppc.ObjectFloatHashMap;
 import com.carrotsearch.hppc.cursors.ObjectCursor;
+
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.ClusterStateListener;
+import org.elasticsearch.cluster.ClusterStateApplier;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -34,13 +35,13 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.discovery.Discovery;
 import org.elasticsearch.index.Index;
-import org.elasticsearch.index.NodeServicesProvider;
 import org.elasticsearch.indices.IndicesService;
 
 import java.util.Arrays;
+import java.util.Map;
 import java.util.function.Supplier;
 
-public class Gateway extends AbstractComponent implements ClusterStateListener {
+public class Gateway extends AbstractComponent implements ClusterStateApplier {
 
     private final ClusterService clusterService;
 
@@ -50,19 +51,17 @@ public class Gateway extends AbstractComponent implements ClusterStateListener {
 
     private final Supplier<Integer> minimumMasterNodesProvider;
     private final IndicesService indicesService;
-    private final NodeServicesProvider nodeServicesProvider;
 
     public Gateway(Settings settings, ClusterService clusterService, GatewayMetaState metaState,
                    TransportNodesListGatewayMetaState listGatewayMetaState, Discovery discovery,
-                   NodeServicesProvider nodeServicesProvider, IndicesService indicesService) {
+                   IndicesService indicesService) {
         super(settings);
-        this.nodeServicesProvider = nodeServicesProvider;
         this.indicesService = indicesService;
         this.clusterService = clusterService;
         this.metaState = metaState;
         this.listGatewayMetaState = listGatewayMetaState;
         this.minimumMasterNodesProvider = discovery::getMinimumMasterNodes;
-        clusterService.addLast(this);
+        clusterService.addLowPriorityApplier(this);
     }
 
     public void performStateRecovery(final GatewayStateRecoveredListener listener) throws GatewayException {
@@ -133,7 +132,7 @@ public void performStateRecovery(final GatewayStateRecoveredListener listener) t
                     try {
                         if (electedIndexMetaData.getState() == IndexMetaData.State.OPEN) {
                             // verify that we can actually create this index - if not we recover it as closed with lots of warn logs
-                            indicesService.verifyIndexMetadata(nodeServicesProvider, electedIndexMetaData, electedIndexMetaData);
+                            indicesService.verifyIndexMetadata(electedIndexMetaData, electedIndexMetaData);
                         }
                     } catch (Exception e) {
                         final Index electedIndex = electedIndexMetaData.getIndex();
@@ -148,18 +147,40 @@ public void performStateRecovery(final GatewayStateRecoveredListener listener) t
             }
         }
         final ClusterSettings clusterSettings = clusterService.getClusterSettings();
-        metaDataBuilder.persistentSettings(clusterSettings.archiveUnknownOrBrokenSettings(metaDataBuilder.persistentSettings()));
-        metaDataBuilder.transientSettings(clusterSettings.archiveUnknownOrBrokenSettings(metaDataBuilder.transientSettings()));
+        metaDataBuilder.persistentSettings(
+            clusterSettings.archiveUnknownOrInvalidSettings(
+                metaDataBuilder.persistentSettings(),
+                e -> logUnknownSetting("persistent", e),
+                (e, ex) -> logInvalidSetting("persistent", e, ex)));
+        metaDataBuilder.transientSettings(
+            clusterSettings.archiveUnknownOrInvalidSettings(
+                metaDataBuilder.transientSettings(),
+                e -> logUnknownSetting("transient", e),
+                (e, ex) -> logInvalidSetting("transient", e, ex)));
         ClusterState.Builder builder = ClusterState.builder(clusterService.getClusterName());
         builder.metaData(metaDataBuilder);
         listener.onSuccess(builder.build());
     }
 
+    private void logUnknownSetting(String settingType, Map.Entry<String, String> e) {
+        logger.warn("ignoring unknown {} setting: [{}] with value [{}]; archiving", settingType, e.getKey(), e.getValue());
+    }
+
+    private void logInvalidSetting(String settingType, Map.Entry<String, String> e, IllegalArgumentException ex) {
+        logger.warn(
+            (org.apache.logging.log4j.util.Supplier<?>)
+                () -> new ParameterizedMessage("ignoring invalid {} setting: [{}] with value [{}]; archiving",
+                    settingType,
+                    e.getKey(),
+                    e.getValue()),
+            ex);
+    }
+
     @Override
-    public void clusterChanged(final ClusterChangedEvent event) {
+    public void applyClusterState(final ClusterChangedEvent event) {
         // order is important, first metaState, and then shardsState
         // so dangling indices will be recorded
-        metaState.clusterChanged(event);
+        metaState.applyClusterState(event);
     }
 
     public interface GatewayStateRecoveredListener {
diff --git a/core/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java b/core/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java
index c84a9c3378ae2..371753f9085b4 100644
--- a/core/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java
+++ b/core/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java
@@ -22,15 +22,13 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.support.nodes.BaseNodeResponse;
 import org.elasticsearch.action.support.nodes.BaseNodesResponse;
-import org.elasticsearch.cluster.ClusterChangedEvent;
-import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.RoutingService;
 import org.elasticsearch.cluster.routing.ShardRouting;
-import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation;
+import org.elasticsearch.cluster.routing.allocation.AllocateUnassignedDecision;
+import org.elasticsearch.cluster.routing.allocation.FailedShard;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
-import org.elasticsearch.cluster.routing.allocation.StartedRerouteAllocation;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
@@ -40,6 +38,7 @@
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData;
 
+import java.util.List;
 import java.util.concurrent.ConcurrentMap;
 
 /**
@@ -62,44 +61,23 @@ public GatewayAllocator(Settings settings, final TransportNodesListGatewayStarte
         this.replicaShardAllocator = new InternalReplicaShardAllocator(settings, storeAction);
     }
 
-    /**
-     * Returns true if the given shard has an async fetch pending
-     */
-    public boolean hasFetchPending(ShardId shardId, boolean primary) {
-        if (primary) {
-            AsyncShardFetch<TransportNodesListGatewayStartedShards.NodeGatewayStartedShards> fetch = asyncFetchStarted.get(shardId);
-            if (fetch != null) {
-                return fetch.getNumberOfInFlightFetches() > 0;
-            }
-        } else {
-            AsyncShardFetch<TransportNodesListShardStoreMetaData.NodeStoreFilesMetaData> fetch = asyncFetchStore.get(shardId);
-            if (fetch != null) {
-                return fetch.getNumberOfInFlightFetches() > 0;
-            }
-        }
-        return false;
-    }
-
     public void setReallocation(final ClusterService clusterService, final RoutingService routingService) {
         this.routingService = routingService;
-        clusterService.add(new ClusterStateListener() {
-            @Override
-            public void clusterChanged(ClusterChangedEvent event) {
-                boolean cleanCache = false;
-                DiscoveryNode localNode = event.state().nodes().getLocalNode();
-                if (localNode != null) {
-                    if (localNode.isMasterNode() == true && event.localNodeMaster() == false) {
-                        cleanCache = true;
-                    }
-                } else {
+        clusterService.addStateApplier(event -> {
+            boolean cleanCache = false;
+            DiscoveryNode localNode = event.state().nodes().getLocalNode();
+            if (localNode != null) {
+                if (localNode.isMasterNode() && event.localNodeMaster() == false) {
                     cleanCache = true;
                 }
-                if (cleanCache) {
-                    Releasables.close(asyncFetchStarted.values());
-                    asyncFetchStarted.clear();
-                    Releasables.close(asyncFetchStore.values());
-                    asyncFetchStore.clear();
-                }
+            } else {
+                cleanCache = true;
+            }
+            if (cleanCache) {
+                Releasables.close(asyncFetchStarted.values());
+                asyncFetchStarted.clear();
+                Releasables.close(asyncFetchStore.values());
+                asyncFetchStore.clear();
             }
         });
     }
@@ -115,21 +93,28 @@ public int getNumberOfInFlightFetch() {
         return count;
     }
 
-    public void applyStartedShards(StartedRerouteAllocation allocation) {
-        for (ShardRouting shard : allocation.startedShards()) {
-            Releasables.close(asyncFetchStarted.remove(shard.shardId()));
-            Releasables.close(asyncFetchStore.remove(shard.shardId()));
+    public void applyStartedShards(final RoutingAllocation allocation, final List<ShardRouting> startedShards) {
+        for (ShardRouting startedShard : startedShards) {
+            Releasables.close(asyncFetchStarted.remove(startedShard.shardId()));
+            Releasables.close(asyncFetchStore.remove(startedShard.shardId()));
         }
     }
 
-    public void applyFailedShards(FailedRerouteAllocation allocation) {
-        for (FailedRerouteAllocation.FailedShard shard : allocation.failedShards()) {
-            Releasables.close(asyncFetchStarted.remove(shard.routingEntry.shardId()));
-            Releasables.close(asyncFetchStore.remove(shard.routingEntry.shardId()));
+    public void applyFailedShards(final RoutingAllocation allocation, final List<FailedShard> failedShards) {
+        for (FailedShard failedShard : failedShards) {
+            Releasables.close(asyncFetchStarted.remove(failedShard.getRoutingEntry().shardId()));
+            Releasables.close(asyncFetchStore.remove(failedShard.getRoutingEntry().shardId()));
         }
     }
 
     public void allocateUnassigned(final RoutingAllocation allocation) {
+        innerAllocatedUnassigned(allocation, primaryShardAllocator, replicaShardAllocator);
+    }
+
+    // allow for testing infra to change shard allocators implementation
+    protected static void innerAllocatedUnassigned(RoutingAllocation allocation,
+                                                   PrimaryShardAllocator primaryShardAllocator,
+                                                   ReplicaShardAllocator replicaShardAllocator) {
         RoutingNodes.UnassignedShards unassigned = allocation.routingNodes().unassigned();
         unassigned.sort(PriorityComparator.getAllocationComparator(allocation)); // sort for priority ordering
 
@@ -138,9 +123,21 @@ public void allocateUnassigned(final RoutingAllocation allocation) {
         replicaShardAllocator.allocateUnassigned(allocation);
     }
 
+    /**
+     * Computes and returns the design for allocating a single unassigned shard.  If called on an assigned shard,
+     * {@link AllocateUnassignedDecision#NOT_TAKEN} is returned.
+     */
+    public AllocateUnassignedDecision decideUnassignedShardAllocation(ShardRouting unassignedShard, RoutingAllocation routingAllocation) {
+        if (unassignedShard.primary()) {
+            return primaryShardAllocator.makeAllocationDecision(unassignedShard, routingAllocation, logger);
+        } else {
+            return replicaShardAllocator.makeAllocationDecision(unassignedShard, routingAllocation, logger);
+        }
+    }
+
     class InternalAsyncFetch<T extends BaseNodeResponse> extends AsyncShardFetch<T> {
 
-        public InternalAsyncFetch(Logger logger, String type, ShardId shardId, Lister<? extends BaseNodesResponse<T>, T> action) {
+        InternalAsyncFetch(Logger logger, String type, ShardId shardId, Lister<? extends BaseNodesResponse<T>, T> action) {
             super(logger, type, shardId, action);
         }
 
@@ -155,22 +152,19 @@ class InternalPrimaryShardAllocator extends PrimaryShardAllocator {
 
         private final TransportNodesListGatewayStartedShards startedAction;
 
-        public InternalPrimaryShardAllocator(Settings settings, TransportNodesListGatewayStartedShards startedAction) {
+        InternalPrimaryShardAllocator(Settings settings, TransportNodesListGatewayStartedShards startedAction) {
             super(settings);
             this.startedAction = startedAction;
         }
 
         @Override
         protected AsyncShardFetch.FetchResult<TransportNodesListGatewayStartedShards.NodeGatewayStartedShards> fetchData(ShardRouting shard, RoutingAllocation allocation) {
-            AsyncShardFetch<TransportNodesListGatewayStartedShards.NodeGatewayStartedShards> fetch = asyncFetchStarted.get(shard.shardId());
-            if (fetch == null) {
-                fetch = new InternalAsyncFetch<>(logger, "shard_started", shard.shardId(), startedAction);
-                asyncFetchStarted.put(shard.shardId(), fetch);
-            }
+            AsyncShardFetch<TransportNodesListGatewayStartedShards.NodeGatewayStartedShards> fetch =
+                asyncFetchStarted.computeIfAbsent(shard.shardId(), shardId -> new InternalAsyncFetch<>(logger, "shard_started", shardId, startedAction));
             AsyncShardFetch.FetchResult<TransportNodesListGatewayStartedShards.NodeGatewayStartedShards> shardState =
                     fetch.fetchData(allocation.nodes(), allocation.getIgnoreNodes(shard.shardId()));
 
-            if (shardState.hasData() == true) {
+            if (shardState.hasData()) {
                 shardState.processAllocation(allocation);
             }
             return shardState;
@@ -181,24 +175,26 @@ class InternalReplicaShardAllocator extends ReplicaShardAllocator {
 
         private final TransportNodesListShardStoreMetaData storeAction;
 
-        public InternalReplicaShardAllocator(Settings settings, TransportNodesListShardStoreMetaData storeAction) {
+        InternalReplicaShardAllocator(Settings settings, TransportNodesListShardStoreMetaData storeAction) {
             super(settings);
             this.storeAction = storeAction;
         }
 
         @Override
         protected AsyncShardFetch.FetchResult<TransportNodesListShardStoreMetaData.NodeStoreFilesMetaData> fetchData(ShardRouting shard, RoutingAllocation allocation) {
-            AsyncShardFetch<TransportNodesListShardStoreMetaData.NodeStoreFilesMetaData> fetch = asyncFetchStore.get(shard.shardId());
-            if (fetch == null) {
-                fetch = new InternalAsyncFetch<>(logger, "shard_store", shard.shardId(), storeAction);
-                asyncFetchStore.put(shard.shardId(), fetch);
-            }
+            AsyncShardFetch<TransportNodesListShardStoreMetaData.NodeStoreFilesMetaData> fetch =
+                asyncFetchStore.computeIfAbsent(shard.shardId(), shardId -> new InternalAsyncFetch<>(logger, "shard_store", shard.shardId(), storeAction));
             AsyncShardFetch.FetchResult<TransportNodesListShardStoreMetaData.NodeStoreFilesMetaData> shardStores =
                     fetch.fetchData(allocation.nodes(), allocation.getIgnoreNodes(shard.shardId()));
-            if (shardStores.hasData() == true) {
+            if (shardStores.hasData()) {
                 shardStores.processAllocation(allocation);
             }
             return shardStores;
         }
+
+        @Override
+        protected boolean hasInitiatedFetching(ShardRouting shard) {
+            return asyncFetchStore.get(shard.shardId()) != null;
+        }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java b/core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java
index a05e85299a816..d798f8b7745b1 100644
--- a/core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java
+++ b/core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java
@@ -23,7 +23,7 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.ClusterStateListener;
+import org.elasticsearch.cluster.ClusterStateApplier;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.metadata.MetaDataIndexUpgradeService;
@@ -31,6 +31,7 @@
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -50,15 +51,17 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.function.BiConsumer;
+import java.util.function.Consumer;
+import java.util.function.UnaryOperator;
 
 import static java.util.Collections.emptySet;
 import static java.util.Collections.unmodifiableSet;
 
-public class GatewayMetaState extends AbstractComponent implements ClusterStateListener {
+public class GatewayMetaState extends AbstractComponent implements ClusterStateApplier {
 
     private final NodeEnvironment nodeEnv;
     private final MetaStateService metaStateService;
-    private final DanglingIndicesState danglingIndicesState;
 
     @Nullable
     private volatile MetaData previousMetaData;
@@ -67,13 +70,12 @@ public class GatewayMetaState extends AbstractComponent implements ClusterStateL
 
     @Inject
     public GatewayMetaState(Settings settings, NodeEnvironment nodeEnv, MetaStateService metaStateService,
-                            DanglingIndicesState danglingIndicesState, TransportNodesListGatewayMetaState nodesListGatewayMetaState,
+                            TransportNodesListGatewayMetaState nodesListGatewayMetaState,
                             MetaDataIndexUpgradeService metaDataIndexUpgradeService, MetaDataUpgrader metaDataUpgrader)
         throws Exception {
         super(settings);
         this.nodeEnv = nodeEnv;
         this.metaStateService = metaStateService;
-        this.danglingIndicesState = danglingIndicesState;
         nodesListGatewayMetaState.init(this);
 
         if (DiscoveryNode.isDataNode(settings)) {
@@ -117,7 +119,7 @@ public MetaData loadMetaState() throws Exception {
     }
 
     @Override
-    public void clusterChanged(ClusterChangedEvent event) {
+    public void applyClusterState(ClusterChangedEvent event) {
 
         final ClusterState state = event.state();
         if (state.blocks().disableStatePersistence()) {
@@ -181,7 +183,6 @@ public void clusterChanged(ClusterChangedEvent event) {
             }
         }
 
-        danglingIndicesState.processDanglingIndices(newMetaData);
         if (success) {
             previousMetaData = newMetaData;
             previouslyWrittenIndices = unmodifiableSet(relevantIndices);
@@ -192,7 +193,7 @@ public static Set<Index> getRelevantIndices(ClusterState state, ClusterState pre
         Set<Index> relevantIndices;
         if (isDataOnlyNode(state)) {
             relevantIndices = getRelevantIndicesOnDataOnlyNode(state, previousState, previouslyWrittenIndices);
-        } else if (state.nodes().getLocalNode().isMasterNode() == true) {
+        } else if (state.nodes().getLocalNode().isMasterNode()) {
             relevantIndices = getRelevantIndicesForMasterEligibleNode(state);
         } else {
             relevantIndices = Collections.emptySet();
@@ -222,8 +223,8 @@ private void ensureNoPre019State() throws Exception {
                     final String name = stateFile.getFileName().toString();
                     if (name.startsWith("metadata-")) {
                         throw new IllegalStateException("Detected pre 0.19 metadata file please upgrade to a version before "
-                                + Version.CURRENT.minimumCompatibilityVersion()
-                                + " first to upgrade state structures - metadata found: [" + stateFile.getParent().toAbsolutePath());
+                            + Version.CURRENT.minimumCompatibilityVersion()
+                            + " first to upgrade state structures - metadata found: [" + stateFile.getParent().toAbsolutePath());
                     }
                 }
             }
@@ -245,27 +246,46 @@ static MetaData upgradeMetaData(MetaData metaData,
         boolean changed = false;
         final MetaData.Builder upgradedMetaData = MetaData.builder(metaData);
         for (IndexMetaData indexMetaData : metaData) {
-            IndexMetaData newMetaData = metaDataIndexUpgradeService.upgradeIndexMetaData(indexMetaData);
+            IndexMetaData newMetaData = metaDataIndexUpgradeService.upgradeIndexMetaData(indexMetaData,
+                Version.CURRENT.minimumIndexCompatibilityVersion());
             changed |= indexMetaData != newMetaData;
             upgradedMetaData.put(newMetaData, false);
         }
-        // collect current customs
-        Map<String, MetaData.Custom> existingCustoms = new HashMap<>();
-        for (ObjectObjectCursor<String, MetaData.Custom> customCursor : metaData.customs()) {
-            existingCustoms.put(customCursor.key, customCursor.value);
-        }
         // upgrade global custom meta data
-        Map<String, MetaData.Custom> upgradedCustoms = metaDataUpgrader.customMetaDataUpgraders.apply(existingCustoms);
-        if (upgradedCustoms.equals(existingCustoms) == false) {
-            existingCustoms.keySet().forEach(upgradedMetaData::removeCustom);
-            for (Map.Entry<String, MetaData.Custom> upgradedCustomEntry : upgradedCustoms.entrySet()) {
-                upgradedMetaData.putCustom(upgradedCustomEntry.getKey(), upgradedCustomEntry.getValue());
-            }
+        if (applyPluginUpgraders(metaData.getCustoms(), metaDataUpgrader.customMetaDataUpgraders,
+            upgradedMetaData::removeCustom,upgradedMetaData::putCustom)) {
+            changed = true;
+        }
+        // upgrade current templates
+        if (applyPluginUpgraders(metaData.getTemplates(), metaDataUpgrader.indexTemplateMetaDataUpgraders,
+            upgradedMetaData::removeTemplate, (s, indexTemplateMetaData) -> upgradedMetaData.put(indexTemplateMetaData))) {
             changed = true;
         }
         return changed ? upgradedMetaData.build() : metaData;
     }
 
+    private static <Data> boolean applyPluginUpgraders(ImmutableOpenMap<String, Data> existingData,
+                                                       UnaryOperator<Map<String, Data>> upgrader,
+                                                       Consumer<String> removeData,
+                                                       BiConsumer<String, Data> putData) {
+        // collect current data
+        Map<String, Data> existingMap = new HashMap<>();
+        for (ObjectObjectCursor<String, Data> customCursor : existingData) {
+            existingMap.put(customCursor.key, customCursor.value);
+        }
+        // upgrade global custom meta data
+        Map<String, Data> upgradedCustoms = upgrader.apply(existingMap);
+        if (upgradedCustoms.equals(existingMap) == false) {
+            // remove all data first so a plugin can remove custom metadata or templates if needed
+            existingMap.keySet().forEach(removeData);
+            for (Map.Entry<String, Data> upgradedCustomEntry : upgradedCustoms.entrySet()) {
+                putData.accept(upgradedCustomEntry.getKey(), upgradedCustomEntry.getValue());
+            }
+            return true;
+        }
+        return false;
+    }
+
     // shard state BWC
     private void ensureNoPre019ShardState(NodeEnvironment nodeEnv) throws Exception {
         for (Path dataLocation : nodeEnv.nodeDataPaths()) {
diff --git a/core/src/main/java/org/elasticsearch/gateway/GatewayService.java b/core/src/main/java/org/elasticsearch/gateway/GatewayService.java
index b953173c6899a..a771f6cc265b2 100644
--- a/core/src/main/java/org/elasticsearch/gateway/GatewayService.java
+++ b/core/src/main/java/org/elasticsearch/gateway/GatewayService.java
@@ -34,7 +34,6 @@
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
-import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.inject.Inject;
@@ -44,7 +43,6 @@
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.discovery.Discovery;
-import org.elasticsearch.index.NodeServicesProvider;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -71,7 +69,7 @@ public class GatewayService extends AbstractLifecycleComponent implements Cluste
     public static final Setting<Integer> RECOVER_AFTER_MASTER_NODES_SETTING =
         Setting.intSetting("gateway.recover_after_master_nodes", 0, 0, Property.NodeScope);
 
-    public static final ClusterBlock STATE_NOT_RECOVERED_BLOCK = new ClusterBlock(1, "state not recovered / initialized", true, true, RestStatus.SERVICE_UNAVAILABLE, ClusterBlockLevel.ALL);
+    public static final ClusterBlock STATE_NOT_RECOVERED_BLOCK = new ClusterBlock(1, "state not recovered / initialized", true, true, false, RestStatus.SERVICE_UNAVAILABLE, ClusterBlockLevel.ALL);
 
     public static final TimeValue DEFAULT_RECOVER_AFTER_TIME_IF_EXPECTED_NODES_IS_SET = TimeValue.timeValueMinutes(5);
 
@@ -99,10 +97,10 @@ public class GatewayService extends AbstractLifecycleComponent implements Cluste
     public GatewayService(Settings settings, AllocationService allocationService, ClusterService clusterService,
                           ThreadPool threadPool, GatewayMetaState metaState,
                           TransportNodesListGatewayMetaState listGatewayMetaState, Discovery discovery,
-                          NodeServicesProvider nodeServicesProvider, IndicesService indicesService) {
+                          IndicesService indicesService) {
         super(settings);
         this.gateway = new Gateway(settings, clusterService, metaState, listGatewayMetaState, discovery,
-            nodeServicesProvider, indicesService);
+            indicesService);
         this.allocationService = allocationService;
         this.clusterService = clusterService;
         this.threadPool = threadPool;
@@ -134,12 +132,13 @@ public GatewayService(Settings settings, AllocationService allocationService, Cl
 
     @Override
     protected void doStart() {
-        clusterService.addLast(this);
+        // use post applied so that the state will be visible to the background recovery thread we spawn in performStateRecovery
+        clusterService.addListener(this);
     }
 
     @Override
     protected void doStop() {
-        clusterService.remove(this);
+        clusterService.removeListener(this);
     }
 
     @Override
@@ -258,9 +257,14 @@ public ClusterState execute(ClusterState currentState) {
                     // automatically generate a UID for the metadata if we need to
                     metaDataBuilder.generateClusterUuidIfNeeded();
 
-                    if (MetaData.SETTING_READ_ONLY_SETTING.get(recoveredState.metaData().settings()) || MetaData.SETTING_READ_ONLY_SETTING.get(currentState.metaData().settings())) {
+                    if (MetaData.SETTING_READ_ONLY_SETTING.get(recoveredState.metaData().settings())
+                        || MetaData.SETTING_READ_ONLY_SETTING.get(currentState.metaData().settings())) {
                         blocks.addGlobalBlock(MetaData.CLUSTER_READ_ONLY_BLOCK);
                     }
+                    if (MetaData.SETTING_READ_ONLY_ALLOW_DELETE_SETTING.get(recoveredState.metaData().settings())
+                        || MetaData.SETTING_READ_ONLY_ALLOW_DELETE_SETTING.get(currentState.metaData().settings())) {
+                        blocks.addGlobalBlock(MetaData.CLUSTER_READ_ONLY_ALLOW_DELETE_BLOCK);
+                    }
 
                     for (IndexMetaData indexMetaData : recoveredState.metaData()) {
                         metaDataBuilder.put(indexMetaData, false);
@@ -282,11 +286,8 @@ public ClusterState execute(ClusterState currentState) {
                     routingTableBuilder.version(0);
 
                     // now, reroute
-                    RoutingAllocation.Result routingResult = allocationService.reroute(
-                            ClusterState.builder(updatedState).routingTable(routingTableBuilder.build()).build(),
-                            "state recovered");
-
-                    return ClusterState.builder(updatedState).routingResult(routingResult).build();
+                    updatedState = ClusterState.builder(updatedState).routingTable(routingTableBuilder.build()).build();
+                    return allocationService.reroute(updatedState, "state recovered");
                 }
 
                 @Override
diff --git a/core/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java b/core/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java
index 707cc89704f81..5a2f635a8cdd3 100644
--- a/core/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java
+++ b/core/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java
@@ -21,6 +21,7 @@
 
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
+import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.block.ClusterBlocks;
@@ -28,9 +29,9 @@
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.metadata.MetaDataIndexUpgradeService;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
-import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
@@ -129,10 +130,18 @@ public ClusterState execute(ClusterState currentState) {
                     MetaData.Builder metaData = MetaData.builder(currentState.metaData());
                     ClusterBlocks.Builder blocks = ClusterBlocks.builder().blocks(currentState.blocks());
                     RoutingTable.Builder routingTableBuilder = RoutingTable.builder(currentState.routingTable());
-
+                    final Version minIndexCompatibilityVersion = currentState.getNodes().getMaxNodeVersion()
+                        .minimumIndexCompatibilityVersion();
                     boolean importNeeded = false;
                     StringBuilder sb = new StringBuilder();
                     for (IndexMetaData indexMetaData : request.indices) {
+                        if (indexMetaData.getCreationVersion().before(minIndexCompatibilityVersion)) {
+                            logger.warn("ignoring dangled index [{}] on node [{}]" +
+                                " since it's created version [{}] is not supported by at least one node in the cluster minVersion [{}]",
+                                indexMetaData.getIndex(), request.fromNode, indexMetaData.getCreationVersion(),
+                                minIndexCompatibilityVersion);
+                            continue;
+                        }
                         if (currentState.metaData().hasIndex(indexMetaData.getIndex().getName())) {
                             continue;
                         }
@@ -147,7 +156,8 @@ public ClusterState execute(ClusterState currentState) {
                         try {
                             // The dangled index might be from an older version, we need to make sure it's compatible
                             // with the current version and upgrade it if needed.
-                            upgradedIndexMetaData = metaDataIndexUpgradeService.upgradeIndexMetaData(indexMetaData);
+                            upgradedIndexMetaData = metaDataIndexUpgradeService.upgradeIndexMetaData(indexMetaData,
+                                minIndexCompatibilityVersion);
                         } catch (Exception ex) {
                             // upgrade failed - adding index as closed
                             logger.warn((Supplier<?>) () -> new ParameterizedMessage("found dangled index [{}] on node [{}]. This index cannot be upgraded to the latest version, adding as closed", indexMetaData.getIndex(), request.fromNode), ex);
@@ -169,10 +179,8 @@ public ClusterState execute(ClusterState currentState) {
                     ClusterState updatedState = ClusterState.builder(currentState).metaData(metaData).blocks(blocks).routingTable(routingTable).build();
 
                     // now, reroute
-                    RoutingAllocation.Result routingResult = allocationService.reroute(
+                    return allocationService.reroute(
                             ClusterState.builder(updatedState).routingTable(routingTable).build(), "dangling indices allocated");
-
-                    return ClusterState.builder(updatedState).routingResult(routingResult).build();
                 }
 
                 @Override
@@ -217,7 +225,7 @@ public void readFrom(StreamInput in) throws IOException {
             fromNode = new DiscoveryNode(in);
             indices = new IndexMetaData[in.readVInt()];
             for (int i = 0; i < indices.length; i++) {
-                indices[i] = IndexMetaData.Builder.readFrom(in);
+                indices[i] = IndexMetaData.readFrom(in);
             }
         }
 
diff --git a/core/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java b/core/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java
index 71c3190e2ee1a..fb48405b72538 100644
--- a/core/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java
+++ b/core/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java
@@ -35,6 +35,7 @@
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.lucene.store.IndexOutputOutputStream;
 import org.elasticsearch.common.lucene.store.InputStreamIndexInput;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentHelper;
@@ -181,9 +182,9 @@ protected XContentBuilder newXContentBuilder(XContentType type, OutputStream str
      * Reads the state from a given file and compares the expected version against the actual version of
      * the state.
      */
-    public final T read(Path file) throws IOException {
+    public final T read(NamedXContentRegistry namedXContentRegistry, Path file) throws IOException {
         try (Directory dir = newDirectory(file.getParent())) {
-            try (final IndexInput indexInput = dir.openInput(file.getFileName().toString(), IOContext.DEFAULT)) {
+            try (IndexInput indexInput = dir.openInput(file.getFileName().toString(), IOContext.DEFAULT)) {
                  // We checksum the entire file before we even go and parse it. If it's corrupted we barf right here.
                 CodecUtil.checksumEntireFile(indexInput);
                 final int fileVersion = CodecUtil.checkHeader(indexInput, STATE_FILE_CODEC, MIN_COMPATIBLE_STATE_FILE_VERSION,
@@ -196,8 +197,8 @@ public final T read(Path file) throws IOException {
                 long filePointer = indexInput.getFilePointer();
                 long contentSize = indexInput.length() - CodecUtil.footerLength() - filePointer;
                 try (IndexInput slice = indexInput.slice("state_xcontent", filePointer, contentSize)) {
-                    try (XContentParser parser = XContentFactory.xContent(xContentType).createParser(new InputStreamIndexInput(slice,
-                        contentSize))) {
+                    try (XContentParser parser = XContentFactory.xContent(xContentType).createParser(namedXContentRegistry,
+                            new InputStreamIndexInput(slice, contentSize))) {
                         return fromXContent(parser);
                     }
                 }
@@ -260,7 +261,7 @@ long findMaxStateId(final String prefix, Path... locations) throws IOException {
      * @param dataLocations the data-locations to try.
      * @return the latest state or <code>null</code> if no state was found.
      */
-    public  T loadLatestState(Logger logger, Path... dataLocations) throws IOException {
+    public  T loadLatestState(Logger logger, NamedXContentRegistry namedXContentRegistry, Path... dataLocations) throws IOException {
         List<PathAndStateId> files = new ArrayList<>();
         long maxStateId = -1;
         boolean maxStateIdIsLegacy = true;
@@ -311,14 +312,14 @@ public  T loadLatestState(Logger logger, Path... dataLocations) throws IOExcepti
                         logger.debug("{}: no data for [{}], ignoring...", prefix, stateFile.toAbsolutePath());
                         continue;
                     }
-                    try (final XContentParser parser = XContentHelper.createParser(new BytesArray(data))) {
+                    try (XContentParser parser = XContentHelper.createParser(namedXContentRegistry, new BytesArray(data))) {
                         state = fromXContent(parser);
                     }
                     if (state == null) {
                         logger.debug("{}: no data for [{}], ignoring...", prefix, stateFile.toAbsolutePath());
                     }
                 } else {
-                    state = read(stateFile);
+                    state = read(namedXContentRegistry, stateFile);
                     logger.trace("state id [{}] read from [{}]", id, stateFile.getFileName());
                 }
                 return state;
diff --git a/core/src/main/java/org/elasticsearch/gateway/MetaStateService.java b/core/src/main/java/org/elasticsearch/gateway/MetaStateService.java
index e58a48d41b4f1..344a50ffa4472 100644
--- a/core/src/main/java/org/elasticsearch/gateway/MetaStateService.java
+++ b/core/src/main/java/org/elasticsearch/gateway/MetaStateService.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.Index;
 
@@ -40,10 +41,12 @@
 public class MetaStateService extends AbstractComponent {
 
     private final NodeEnvironment nodeEnv;
+    private final NamedXContentRegistry namedXContentRegistry;
 
-    public MetaStateService(Settings settings, NodeEnvironment nodeEnv) {
+    public MetaStateService(Settings settings, NodeEnvironment nodeEnv, NamedXContentRegistry namedXContentRegistry) {
         super(settings);
         this.nodeEnv = nodeEnv;
+        this.namedXContentRegistry = namedXContentRegistry;
     }
 
     /**
@@ -59,7 +62,8 @@ MetaData loadFullState() throws Exception {
             metaDataBuilder = MetaData.builder();
         }
         for (String indexFolderName : nodeEnv.availableIndexFolders()) {
-            IndexMetaData indexMetaData = IndexMetaData.FORMAT.loadLatestState(logger, nodeEnv.resolveIndexFolder(indexFolderName));
+            IndexMetaData indexMetaData = IndexMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry,
+                nodeEnv.resolveIndexFolder(indexFolderName));
             if (indexMetaData != null) {
                 metaDataBuilder.put(indexMetaData, false);
             } else {
@@ -74,7 +78,7 @@ MetaData loadFullState() throws Exception {
      */
     @Nullable
     public IndexMetaData loadIndexState(Index index) throws IOException {
-        return IndexMetaData.FORMAT.loadLatestState(logger, nodeEnv.indexPaths(index));
+        return IndexMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry, nodeEnv.indexPaths(index));
     }
 
     /**
@@ -86,7 +90,7 @@ List<IndexMetaData> loadIndicesStates(Predicate<String> excludeIndexPathIdsPredi
             if (excludeIndexPathIdsPredicate.test(indexFolderName)) {
                 continue;
             }
-            IndexMetaData indexMetaData = IndexMetaData.FORMAT.loadLatestState(logger,
+            IndexMetaData indexMetaData = IndexMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry,
                 nodeEnv.resolveIndexFolder(indexFolderName));
             if (indexMetaData != null) {
                 final String indexPathId = indexMetaData.getIndex().getUUID();
@@ -106,7 +110,7 @@ List<IndexMetaData> loadIndicesStates(Predicate<String> excludeIndexPathIdsPredi
      * Loads the global state, *without* index state, see {@link #loadFullState()} for that.
      */
     MetaData loadGlobalState() throws IOException {
-        MetaData globalState = MetaData.FORMAT.loadLatestState(logger, nodeEnv.nodeDataPaths());
+        MetaData globalState = MetaData.FORMAT.loadLatestState(logger, namedXContentRegistry, nodeEnv.nodeDataPaths());
         // ES 2.0 now requires units for all time and byte-sized settings, so we add the default unit if it's missing
         // TODO: can we somehow only do this for pre-2.0 cluster state?
         if (globalState != null) {
diff --git a/core/src/main/java/org/elasticsearch/gateway/PrimaryShardAllocator.java b/core/src/main/java/org/elasticsearch/gateway/PrimaryShardAllocator.java
index a11300ac496ef..3be22b3f69d83 100644
--- a/core/src/main/java/org/elasticsearch/gateway/PrimaryShardAllocator.java
+++ b/core/src/main/java/org/elasticsearch/gateway/PrimaryShardAllocator.java
@@ -19,35 +19,42 @@
 
 package org.elasticsearch.gateway;
 
+import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
 import org.apache.lucene.util.CollectionUtil;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo.AllocationStatus;
+import org.elasticsearch.cluster.routing.allocation.AllocateUnassignedDecision;
+import org.elasticsearch.cluster.routing.allocation.NodeAllocationResult;
+import org.elasticsearch.cluster.routing.allocation.NodeAllocationResult.ShardStoreInfo;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
-import org.elasticsearch.common.component.AbstractComponent;
+import org.elasticsearch.cluster.routing.allocation.decider.Decision.Type;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.ShardLockObtainFailedException;
+import org.elasticsearch.gateway.AsyncShardFetch.FetchResult;
 import org.elasticsearch.gateway.TransportNodesListGatewayStartedShards.NodeGatewayStartedShards;
 import org.elasticsearch.index.shard.ShardStateMetaData;
 
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
-import java.util.LinkedList;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 /**
  * The primary shard allocator allocates unassigned primary shards to nodes that hold
@@ -62,7 +69,7 @@
  * nor does it allocate primaries when a primary shard failed and there is a valid replica
  * copy that can immediately be promoted to primary, as this takes place in {@link RoutingNodes#failShard}.
  */
-public abstract class PrimaryShardAllocator extends AbstractComponent {
+public abstract class PrimaryShardAllocator extends BaseGatewayShardAllocator {
 
     private static final Function<String, String> INITIAL_SHARDS_PARSER = (value) -> {
         switch (value) {
@@ -94,121 +101,203 @@ public PrimaryShardAllocator(Settings settings) {
         logger.debug("using initial_shards [{}]", NODE_INITIAL_SHARDS_SETTING.get(settings));
     }
 
-    public void allocateUnassigned(RoutingAllocation allocation) {
-        final RoutingNodes routingNodes = allocation.routingNodes();
-        final MetaData metaData = allocation.metaData();
-
-        final RoutingNodes.UnassignedShards.UnassignedIterator unassignedIterator = routingNodes.unassigned().iterator();
-        while (unassignedIterator.hasNext()) {
-            final ShardRouting shard = unassignedIterator.next();
-
-            if (shard.primary() == false) {
-                continue;
-            }
+    /**
+     * Is the allocator responsible for allocating the given {@link ShardRouting}?
+     */
+    private static boolean isResponsibleFor(final ShardRouting shard) {
+        return shard.primary() // must be primary
+                && shard.unassigned() // must be unassigned
+                // only handle either an existing store or a snapshot recovery
+                && (shard.recoverySource().getType() == RecoverySource.Type.EXISTING_STORE
+                    || shard.recoverySource().getType() == RecoverySource.Type.SNAPSHOT);
+    }
 
-            if (shard.recoverySource().getType() != RecoverySource.Type.EXISTING_STORE &&
-                shard.recoverySource().getType() != RecoverySource.Type.SNAPSHOT) {
-                continue;
-            }
+    @Override
+    public AllocateUnassignedDecision makeAllocationDecision(final ShardRouting unassignedShard,
+                                                             final RoutingAllocation allocation,
+                                                             final Logger logger) {
+        if (isResponsibleFor(unassignedShard) == false) {
+            // this allocator is not responsible for allocating this shard
+            return AllocateUnassignedDecision.NOT_TAKEN;
+        }
 
-            final AsyncShardFetch.FetchResult<NodeGatewayStartedShards> shardState = fetchData(shard, allocation);
-            if (shardState.hasData() == false) {
-                logger.trace("{}: ignoring allocation, still fetching shard started state", shard);
-                allocation.setHasPendingAsyncFetch();
-                unassignedIterator.removeAndIgnore(AllocationStatus.FETCHING_SHARD_DATA, allocation.changes());
-                continue;
+        final boolean explain = allocation.debugDecision();
+        final FetchResult<NodeGatewayStartedShards> shardState = fetchData(unassignedShard, allocation);
+        if (shardState.hasData() == false) {
+            allocation.setHasPendingAsyncFetch();
+            List<NodeAllocationResult> nodeDecisions = null;
+            if (explain) {
+                nodeDecisions = buildDecisionsForAllNodes(unassignedShard, allocation);
             }
+            return AllocateUnassignedDecision.no(AllocationStatus.FETCHING_SHARD_DATA, nodeDecisions);
+        }
 
-            // don't create a new IndexSetting object for every shard as this could cause a lot of garbage
-            // on cluster restart if we allocate a boat load of shards
-            final IndexMetaData indexMetaData = metaData.getIndexSafe(shard.index());
-            final Set<String> inSyncAllocationIds = indexMetaData.inSyncAllocationIds(shard.id());
-            final boolean snapshotRestore = shard.recoverySource().getType() == RecoverySource.Type.SNAPSHOT;
-            final boolean recoverOnAnyNode = recoverOnAnyNode(indexMetaData);
-
-            final NodeShardsResult nodeShardsResult;
-            final boolean enoughAllocationsFound;
-
-            if (inSyncAllocationIds.isEmpty()) {
-                assert Version.indexCreated(indexMetaData.getSettings()).before(Version.V_5_0_0_alpha1) : "trying to allocated a primary with an empty allocation id set, but index is new";
-                // when we load an old index (after upgrading cluster) or restore a snapshot of an old index
-                // fall back to old version-based allocation mode
-                // Note that once the shard has been active, lastActiveAllocationIds will be non-empty
-                nodeShardsResult = buildVersionBasedNodeShardsResult(shard, snapshotRestore || recoverOnAnyNode, allocation.getIgnoreNodes(shard.shardId()), shardState);
-                if (snapshotRestore || recoverOnAnyNode) {
-                    enoughAllocationsFound = nodeShardsResult.allocationsFound > 0;
-                } else {
-                    enoughAllocationsFound = isEnoughVersionBasedAllocationsFound(indexMetaData, nodeShardsResult);
-                }
-                logger.debug("[{}][{}]: version-based allocation for pre-{} index found {} allocations of {}", shard.index(), shard.id(), Version.V_5_0_0_alpha1, nodeShardsResult.allocationsFound, shard);
+        // don't create a new IndexSetting object for every shard as this could cause a lot of garbage
+        // on cluster restart if we allocate a boat load of shards
+        final IndexMetaData indexMetaData = allocation.metaData().getIndexSafe(unassignedShard.index());
+        final Set<String> inSyncAllocationIds = indexMetaData.inSyncAllocationIds(unassignedShard.id());
+        final boolean snapshotRestore = unassignedShard.recoverySource().getType() == RecoverySource.Type.SNAPSHOT;
+        final boolean recoverOnAnyNode = recoverOnAnyNode(indexMetaData);
+
+        final NodeShardsResult nodeShardsResult;
+        final boolean enoughAllocationsFound;
+
+        if (inSyncAllocationIds.isEmpty()) {
+            assert Version.indexCreated(indexMetaData.getSettings()).before(Version.V_5_0_0_alpha1) :
+                "trying to allocate a primary with an empty in sync allocation id set, but index is new. index: "
+                    + indexMetaData.getIndex();
+            // when we load an old index (after upgrading cluster) or restore a snapshot of an old index
+            // fall back to old version-based allocation mode
+            // Note that once the shard has been active, lastActiveAllocationIds will be non-empty
+            nodeShardsResult = buildVersionBasedNodeShardsResult(unassignedShard, snapshotRestore || recoverOnAnyNode,
+                                                                 allocation.getIgnoreNodes(unassignedShard.shardId()), shardState, logger);
+            if (snapshotRestore || recoverOnAnyNode) {
+                enoughAllocationsFound = nodeShardsResult.allocationsFound > 0;
             } else {
-                assert inSyncAllocationIds.isEmpty() == false;
-                // use allocation ids to select nodes
-                nodeShardsResult = buildAllocationIdBasedNodeShardsResult(shard, snapshotRestore || recoverOnAnyNode,
-                        allocation.getIgnoreNodes(shard.shardId()), inSyncAllocationIds, shardState);
-                enoughAllocationsFound = nodeShardsResult.orderedAllocationCandidates.size() > 0;
-                logger.debug("[{}][{}]: found {} allocation candidates of {} based on allocation ids: [{}]", shard.index(), shard.id(), nodeShardsResult.orderedAllocationCandidates.size(), shard, inSyncAllocationIds);
+                enoughAllocationsFound = isEnoughVersionBasedAllocationsFound(indexMetaData, nodeShardsResult);
             }
+            logger.debug("[{}][{}]: version-based allocation for pre-{} index found {} allocations of {}", unassignedShard.index(),
+                         unassignedShard.id(), Version.V_5_0_0_alpha1, nodeShardsResult.allocationsFound, unassignedShard);
+        } else {
+            assert inSyncAllocationIds.isEmpty() == false;
+            // use allocation ids to select nodes
+            nodeShardsResult = buildAllocationIdBasedNodeShardsResult(unassignedShard, snapshotRestore || recoverOnAnyNode,
+                allocation.getIgnoreNodes(unassignedShard.shardId()), inSyncAllocationIds, shardState, logger);
+            enoughAllocationsFound = nodeShardsResult.orderedAllocationCandidates.size() > 0;
+            logger.debug("[{}][{}]: found {} allocation candidates of {} based on allocation ids: [{}]", unassignedShard.index(),
+                         unassignedShard.id(), nodeShardsResult.orderedAllocationCandidates.size(), unassignedShard, inSyncAllocationIds);
+        }
 
-            if (enoughAllocationsFound == false){
-                if (snapshotRestore) {
-                    // let BalancedShardsAllocator take care of allocating this shard
-                    logger.debug("[{}][{}]: missing local data, will restore from [{}]", shard.index(), shard.id(), shard.recoverySource());
-                } else if (recoverOnAnyNode) {
-                    // let BalancedShardsAllocator take care of allocating this shard
-                    logger.debug("[{}][{}]: missing local data, recover from any node", shard.index(), shard.id());
-                } else {
-                    // we can't really allocate, so ignore it and continue
-                    unassignedIterator.removeAndIgnore(AllocationStatus.NO_VALID_SHARD_COPY, allocation.changes());
-                    logger.debug("[{}][{}]: not allocating, number_of_allocated_shards_found [{}]", shard.index(), shard.id(), nodeShardsResult.allocationsFound);
-                }
-                continue;
+        if (enoughAllocationsFound == false) {
+            if (snapshotRestore) {
+                // let BalancedShardsAllocator take care of allocating this shard
+                logger.debug("[{}][{}]: missing local data, will restore from [{}]",
+                             unassignedShard.index(), unassignedShard.id(), unassignedShard.recoverySource());
+                return AllocateUnassignedDecision.NOT_TAKEN;
+            } else if (recoverOnAnyNode) {
+                // let BalancedShardsAllocator take care of allocating this shard
+                logger.debug("[{}][{}]: missing local data, recover from any node", unassignedShard.index(), unassignedShard.id());
+                return AllocateUnassignedDecision.NOT_TAKEN;
+            } else {
+                // We have a shard that was previously allocated, but we could not find a valid shard copy to allocate the primary.
+                // We could just be waiting for the node that holds the primary to start back up, in which case the allocation for
+                // this shard will be picked up when the node joins and we do another allocation reroute
+                logger.debug("[{}][{}]: not allocating, number_of_allocated_shards_found [{}]",
+                             unassignedShard.index(), unassignedShard.id(), nodeShardsResult.allocationsFound);
+                return AllocateUnassignedDecision.no(AllocationStatus.NO_VALID_SHARD_COPY,
+                    explain ? buildNodeDecisions(null, shardState, inSyncAllocationIds) : null);
             }
+        }
 
-            final NodesToAllocate nodesToAllocate = buildNodesToAllocate(
-                allocation, nodeShardsResult.orderedAllocationCandidates, shard, false
-            );
+        NodesToAllocate nodesToAllocate = buildNodesToAllocate(
+            allocation, nodeShardsResult.orderedAllocationCandidates, unassignedShard, false
+        );
+        DiscoveryNode node = null;
+        String allocationId = null;
+        boolean throttled = false;
+        if (nodesToAllocate.yesNodeShards.isEmpty() == false) {
+            DecidedNode decidedNode = nodesToAllocate.yesNodeShards.get(0);
+            logger.debug("[{}][{}]: allocating [{}] to [{}] on primary allocation",
+                         unassignedShard.index(), unassignedShard.id(), unassignedShard, decidedNode.nodeShardState.getNode());
+            node = decidedNode.nodeShardState.getNode();
+            allocationId = decidedNode.nodeShardState.allocationId();
+        } else if (nodesToAllocate.throttleNodeShards.isEmpty() && !nodesToAllocate.noNodeShards.isEmpty()) {
+            // The deciders returned a NO decision for all nodes with shard copies, so we check if primary shard
+            // can be force-allocated to one of the nodes.
+            nodesToAllocate = buildNodesToAllocate(allocation, nodeShardsResult.orderedAllocationCandidates, unassignedShard, true);
             if (nodesToAllocate.yesNodeShards.isEmpty() == false) {
-                NodeGatewayStartedShards nodeShardState = nodesToAllocate.yesNodeShards.get(0);
-                logger.debug("[{}][{}]: allocating [{}] to [{}] on primary allocation", shard.index(), shard.id(), shard, nodeShardState.getNode());
-                unassignedIterator.initialize(nodeShardState.getNode().getId(), nodeShardState.allocationId(), ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE, allocation.changes());
-            } else if (nodesToAllocate.throttleNodeShards.isEmpty() == true && nodesToAllocate.noNodeShards.isEmpty() == false) {
-                // The deciders returned a NO decision for all nodes with shard copies, so we check if primary shard
-                // can be force-allocated to one of the nodes.
-                final NodesToAllocate nodesToForceAllocate = buildNodesToAllocate(
-                    allocation, nodeShardsResult.orderedAllocationCandidates, shard, true
-                );
-                if (nodesToForceAllocate.yesNodeShards.isEmpty() == false) {
-                    NodeGatewayStartedShards nodeShardState = nodesToForceAllocate.yesNodeShards.get(0);
-                    logger.debug("[{}][{}]: allocating [{}] to [{}] on forced primary allocation",
-                                 shard.index(), shard.id(), shard, nodeShardState.getNode());
-                    unassignedIterator.initialize(nodeShardState.getNode().getId(), nodeShardState.allocationId(),
-                                                  ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE, allocation.changes());
-                } else if (nodesToForceAllocate.throttleNodeShards.isEmpty() == false) {
-                    logger.debug("[{}][{}]: throttling allocation [{}] to [{}] on forced primary allocation",
-                                 shard.index(), shard.id(), shard, nodesToForceAllocate.throttleNodeShards);
-                    unassignedIterator.removeAndIgnore(AllocationStatus.DECIDERS_THROTTLED, allocation.changes());
-                } else {
-                    logger.debug("[{}][{}]: forced primary allocation denied [{}]", shard.index(), shard.id(), shard);
-                    unassignedIterator.removeAndIgnore(AllocationStatus.DECIDERS_NO, allocation.changes());
-                }
+                final DecidedNode decidedNode = nodesToAllocate.yesNodeShards.get(0);
+                final NodeGatewayStartedShards nodeShardState = decidedNode.nodeShardState;
+                logger.debug("[{}][{}]: allocating [{}] to [{}] on forced primary allocation",
+                             unassignedShard.index(), unassignedShard.id(), unassignedShard, nodeShardState.getNode());
+                node = nodeShardState.getNode();
+                allocationId = nodeShardState.allocationId();
+            } else if (nodesToAllocate.throttleNodeShards.isEmpty() == false) {
+                logger.debug("[{}][{}]: throttling allocation [{}] to [{}] on forced primary allocation",
+                             unassignedShard.index(), unassignedShard.id(), unassignedShard, nodesToAllocate.throttleNodeShards);
+                throttled = true;
             } else {
-                // we are throttling this, but we have enough to allocate to this node, ignore it for now
-                logger.debug("[{}][{}]: throttling allocation [{}] to [{}] on primary allocation", shard.index(), shard.id(), shard, nodesToAllocate.throttleNodeShards);
-                unassignedIterator.removeAndIgnore(AllocationStatus.DECIDERS_THROTTLED, allocation.changes());
+                logger.debug("[{}][{}]: forced primary allocation denied [{}]",
+                             unassignedShard.index(), unassignedShard.id(), unassignedShard);
             }
+        } else {
+            // we are throttling this, since we are allowed to allocate to this node but there are enough allocations
+            // taking place on the node currently, ignore it for now
+            logger.debug("[{}][{}]: throttling allocation [{}] to [{}] on primary allocation",
+                         unassignedShard.index(), unassignedShard.id(), unassignedShard, nodesToAllocate.throttleNodeShards);
+            throttled = true;
+        }
+
+        List<NodeAllocationResult> nodeResults = null;
+        if (explain) {
+            nodeResults = buildNodeDecisions(nodesToAllocate, shardState, inSyncAllocationIds);
+        }
+        if (allocation.hasPendingAsyncFetch()) {
+            return AllocateUnassignedDecision.no(AllocationStatus.FETCHING_SHARD_DATA, nodeResults);
+        } else if (node != null) {
+            return AllocateUnassignedDecision.yes(node, allocationId, nodeResults, false);
+        } else if (throttled) {
+            return AllocateUnassignedDecision.throttle(nodeResults);
+        } else {
+            return AllocateUnassignedDecision.no(AllocationStatus.DECIDERS_NO, nodeResults, true);
+        }
+    }
+
+    /**
+     * Builds a map of nodes to the corresponding allocation decisions for those nodes.
+     */
+    private static List<NodeAllocationResult> buildNodeDecisions(NodesToAllocate nodesToAllocate,
+                                                                 FetchResult<NodeGatewayStartedShards> fetchedShardData,
+                                                                 Set<String> inSyncAllocationIds) {
+        List<NodeAllocationResult> nodeResults = new ArrayList<>();
+        Collection<NodeGatewayStartedShards> ineligibleShards;
+        if (nodesToAllocate != null) {
+            final Set<DiscoveryNode> discoNodes = new HashSet<>();
+            nodeResults.addAll(Stream.of(nodesToAllocate.yesNodeShards, nodesToAllocate.throttleNodeShards, nodesToAllocate.noNodeShards)
+                                .flatMap(Collection::stream)
+                                .map(dnode -> {
+                                    discoNodes.add(dnode.nodeShardState.getNode());
+                                    return new NodeAllocationResult(dnode.nodeShardState.getNode(),
+                                                                       shardStoreInfo(dnode.nodeShardState, inSyncAllocationIds),
+                                                                       dnode.decision);
+                                }).collect(Collectors.toList()));
+            ineligibleShards = fetchedShardData.getData().values().stream().filter(shardData ->
+                discoNodes.contains(shardData.getNode()) == false
+            ).collect(Collectors.toList());
+        } else {
+            // there were no shard copies that were eligible for being assigned the allocation,
+            // so all fetched shard data are ineligible shards
+            ineligibleShards = fetchedShardData.getData().values();
         }
+
+        nodeResults.addAll(ineligibleShards.stream().map(shardData ->
+            new NodeAllocationResult(shardData.getNode(), shardStoreInfo(shardData, inSyncAllocationIds), null)
+        ).collect(Collectors.toList()));
+
+        return nodeResults;
+    }
+
+    private static ShardStoreInfo shardStoreInfo(NodeGatewayStartedShards nodeShardState, Set<String> inSyncAllocationIds) {
+        final Exception storeErr = nodeShardState.storeException();
+        final boolean inSync = nodeShardState.allocationId() != null && inSyncAllocationIds.contains(nodeShardState.allocationId());
+        return new ShardStoreInfo(nodeShardState.allocationId(), inSync, storeErr);
     }
 
+    private static final Comparator<NodeGatewayStartedShards> NO_STORE_EXCEPTION_FIRST_COMPARATOR =
+        Comparator.comparing((NodeGatewayStartedShards state) -> state.storeException() == null).reversed();
+    private static final Comparator<NodeGatewayStartedShards> PRIMARY_FIRST_COMPARATOR =
+        Comparator.comparing(NodeGatewayStartedShards::primary).reversed();
+
     /**
      * Builds a list of nodes. If matchAnyShard is set to false, only nodes that have an allocation id matching
-     * lastActiveAllocationIds are added to the list. Otherwise, any node that has a shard is added to the list, but
+     * inSyncAllocationIds are added to the list. Otherwise, any node that has a shard is added to the list, but
      * entries with matching allocation id are always at the front of the list.
      */
-    protected NodeShardsResult buildAllocationIdBasedNodeShardsResult(ShardRouting shard, boolean matchAnyShard, Set<String> ignoreNodes,
-                                                                      Set<String> lastActiveAllocationIds, AsyncShardFetch.FetchResult<NodeGatewayStartedShards> shardState) {
-        LinkedList<NodeGatewayStartedShards> matchingNodeShardStates = new LinkedList<>();
-        LinkedList<NodeGatewayStartedShards> nonMatchingNodeShardStates = new LinkedList<>();
+    protected static NodeShardsResult buildAllocationIdBasedNodeShardsResult(ShardRouting shard, boolean matchAnyShard,
+                                                                             Set<String> ignoreNodes, Set<String> inSyncAllocationIds,
+                                                                             FetchResult<NodeGatewayStartedShards> shardState,
+                                                                             Logger logger) {
+        List<NodeGatewayStartedShards> nodeShardStates = new ArrayList<>();
         int numberOfAllocationsFound = 0;
         for (NodeGatewayStartedShards nodeShardState : shardState.getData().values()) {
             DiscoveryNode node = nodeShardState.getNode();
@@ -229,31 +318,36 @@ protected NodeShardsResult buildAllocationIdBasedNodeShardsResult(ShardRouting s
                 }
             } else {
                 final String finalAllocationId = allocationId;
-                logger.trace((Supplier<?>) () -> new ParameterizedMessage("[{}] on node [{}] has allocation id [{}] but the store can not be opened, treating as no allocation id", shard, nodeShardState.getNode(), finalAllocationId), nodeShardState.storeException());
-                allocationId = null;
+                if (nodeShardState.storeException() instanceof ShardLockObtainFailedException) {
+                    logger.trace((Supplier<?>) () -> new ParameterizedMessage("[{}] on node [{}] has allocation id [{}] but the store can not be opened as it's locked, treating as valid shard", shard, nodeShardState.getNode(), finalAllocationId), nodeShardState.storeException());
+                } else {
+                    logger.trace((Supplier<?>) () -> new ParameterizedMessage("[{}] on node [{}] has allocation id [{}] but the store can not be opened, treating as no allocation id", shard, nodeShardState.getNode(), finalAllocationId), nodeShardState.storeException());
+                    allocationId = null;
+                }
             }
 
             if (allocationId != null) {
+                assert nodeShardState.storeException() == null ||
+                    nodeShardState.storeException() instanceof ShardLockObtainFailedException :
+                    "only allow store that can be opened or that throws a ShardLockObtainFailedException while being opened but got a store throwing " + nodeShardState.storeException();
                 numberOfAllocationsFound++;
-                if (lastActiveAllocationIds.contains(allocationId)) {
-                    if (nodeShardState.primary()) {
-                        matchingNodeShardStates.addFirst(nodeShardState);
-                    } else {
-                        matchingNodeShardStates.addLast(nodeShardState);
-                    }
-                } else if (matchAnyShard) {
-                    if (nodeShardState.primary()) {
-                        nonMatchingNodeShardStates.addFirst(nodeShardState);
-                    } else {
-                        nonMatchingNodeShardStates.addLast(nodeShardState);
-                    }
+                if (matchAnyShard || inSyncAllocationIds.contains(nodeShardState.allocationId())) {
+                    nodeShardStates.add(nodeShardState);
                 }
             }
         }
 
-        List<NodeGatewayStartedShards> nodeShardStates = new ArrayList<>();
-        nodeShardStates.addAll(matchingNodeShardStates);
-        nodeShardStates.addAll(nonMatchingNodeShardStates);
+        final Comparator<NodeGatewayStartedShards> comparator; // allocation preference
+        if (matchAnyShard) {
+            // prefer shards with matching allocation ids
+            Comparator<NodeGatewayStartedShards> matchingAllocationsFirst = Comparator.comparing(
+                (NodeGatewayStartedShards state) -> inSyncAllocationIds.contains(state.allocationId())).reversed();
+            comparator = matchingAllocationsFirst.thenComparing(NO_STORE_EXCEPTION_FIRST_COMPARATOR).thenComparing(PRIMARY_FIRST_COMPARATOR);
+        } else {
+            comparator = NO_STORE_EXCEPTION_FIRST_COMPARATOR.thenComparing(PRIMARY_FIRST_COMPARATOR);
+        }
+
+        nodeShardStates.sort(comparator);
 
         if (logger.isTraceEnabled()) {
             logger.trace("{} candidates for allocation: {}", shard, nodeShardStates.stream().map(s -> s.getNode().getName()).collect(Collectors.joining(", ")));
@@ -299,9 +393,9 @@ private NodesToAllocate buildNodesToAllocate(RoutingAllocation allocation,
                                                  List<NodeGatewayStartedShards> nodeShardStates,
                                                  ShardRouting shardRouting,
                                                  boolean forceAllocate) {
-        List<NodeGatewayStartedShards> yesNodeShards = new ArrayList<>();
-        List<NodeGatewayStartedShards> throttledNodeShards = new ArrayList<>();
-        List<NodeGatewayStartedShards> noNodeShards = new ArrayList<>();
+        List<DecidedNode> yesNodeShards = new ArrayList<>();
+        List<DecidedNode> throttledNodeShards = new ArrayList<>();
+        List<DecidedNode> noNodeShards = new ArrayList<>();
         for (NodeGatewayStartedShards nodeShardState : nodeShardStates) {
             RoutingNode node = allocation.routingNodes().node(nodeShardState.getNode().getId());
             if (node == null) {
@@ -310,23 +404,25 @@ private NodesToAllocate buildNodesToAllocate(RoutingAllocation allocation,
 
             Decision decision = forceAllocate ? allocation.deciders().canForceAllocatePrimary(shardRouting, node, allocation) :
                                                 allocation.deciders().canAllocate(shardRouting, node, allocation);
-            if (decision.type() == Decision.Type.THROTTLE) {
-                throttledNodeShards.add(nodeShardState);
-            } else if (decision.type() == Decision.Type.NO) {
-                noNodeShards.add(nodeShardState);
+            DecidedNode decidedNode = new DecidedNode(nodeShardState, decision);
+            if (decision.type() == Type.THROTTLE) {
+                throttledNodeShards.add(decidedNode);
+            } else if (decision.type() == Type.NO) {
+                noNodeShards.add(decidedNode);
             } else {
-                yesNodeShards.add(nodeShardState);
+                yesNodeShards.add(decidedNode);
             }
         }
-        return new NodesToAllocate(Collections.unmodifiableList(yesNodeShards), Collections.unmodifiableList(throttledNodeShards), Collections.unmodifiableList(noNodeShards));
+        return new NodesToAllocate(Collections.unmodifiableList(yesNodeShards), Collections.unmodifiableList(throttledNodeShards),
+                                      Collections.unmodifiableList(noNodeShards));
     }
 
     /**
      * Builds a list of previously started shards. If matchAnyShard is set to false, only shards with the highest shard version are added to
      * the list. Otherwise, any existing shard is added to the list, but entries with highest version are always at the front of the list.
      */
-    NodeShardsResult buildVersionBasedNodeShardsResult(ShardRouting shard, boolean matchAnyShard, Set<String> ignoreNodes,
-                                                       AsyncShardFetch.FetchResult<NodeGatewayStartedShards> shardState) {
+    static NodeShardsResult buildVersionBasedNodeShardsResult(ShardRouting shard, boolean matchAnyShard, Set<String> ignoreNodes,
+                                                              FetchResult<NodeGatewayStartedShards> shardState, Logger logger) {
         final List<NodeGatewayStartedShards> allocationCandidates = new ArrayList<>();
         int numberOfAllocationsFound = 0;
         long highestVersion = ShardStateMetaData.NO_VERSION;
@@ -353,10 +449,19 @@ NodeShardsResult buildVersionBasedNodeShardsResult(ShardRouting shard, boolean m
                     logger.trace("[{}] on node [{}] has allocation id [{}]", shard, nodeShardState.getNode(), nodeShardState.allocationId());
                 }
             } else {
-                final long finalVerison = version;
-                // when there is an store exception, we disregard the reported version and assign it as no version (same as shard does not exist)
-                logger.trace((Supplier<?>) () -> new ParameterizedMessage("[{}] on node [{}] has version [{}] but the store can not be opened, treating no version", shard, nodeShardState.getNode(), finalVerison), nodeShardState.storeException());
-                version = ShardStateMetaData.NO_VERSION;
+                final long finalVersion = version;
+                if (nodeShardState.storeException() instanceof ShardLockObtainFailedException) {
+                    logger.trace((Supplier<?>) () -> new ParameterizedMessage("[{}] on node [{}] has version [{}] but the store can not be opened as it's locked, treating as valid shard", shard, nodeShardState.getNode(), finalVersion), nodeShardState.storeException());
+                    if (nodeShardState.allocationId() != null) {
+                        version = Long.MAX_VALUE; // shard was already selected in a 5.x cluster as primary, prefer this shard copy again.
+                    } else {
+                        version = 0L; // treat as lowest version so that this shard is the least likely to be selected as primary
+                    }
+                } else {
+                    // disregard the reported version and assign it as no version (same as shard does not exist)
+                    logger.trace((Supplier<?>) () -> new ParameterizedMessage("[{}] on node [{}] has version [{}] but the store can not be opened, treating no version", shard, nodeShardState.getNode(), finalVersion), nodeShardState.storeException());
+                    version = ShardStateMetaData.NO_VERSION;
+                }
             }
 
             if (version != ShardStateMetaData.NO_VERSION) {
@@ -396,33 +501,47 @@ NodeShardsResult buildVersionBasedNodeShardsResult(ShardRouting shard, boolean m
      * recovered on any node
      */
     private boolean recoverOnAnyNode(IndexMetaData metaData) {
+        // don't use the setting directly, not to trigger verbose deprecation logging
         return (IndexMetaData.isOnSharedFilesystem(metaData.getSettings()) || IndexMetaData.isOnSharedFilesystem(this.settings))
-            && IndexMetaData.INDEX_SHARED_FS_ALLOW_RECOVERY_ON_ANY_NODE_SETTING.get(metaData.getSettings(), this.settings);
+            && (metaData.getSettings().getAsBoolean(IndexMetaData.SETTING_SHARED_FS_ALLOW_RECOVERY_ON_ANY_NODE, false) ||
+                this.settings.getAsBoolean(IndexMetaData.SETTING_SHARED_FS_ALLOW_RECOVERY_ON_ANY_NODE, false));
     }
 
-    protected abstract AsyncShardFetch.FetchResult<NodeGatewayStartedShards> fetchData(ShardRouting shard, RoutingAllocation allocation);
+    protected abstract FetchResult<NodeGatewayStartedShards> fetchData(ShardRouting shard, RoutingAllocation allocation);
 
-    static class NodeShardsResult {
-        public final List<NodeGatewayStartedShards> orderedAllocationCandidates;
-        public final int allocationsFound;
+    private static class NodeShardsResult {
+        final List<NodeGatewayStartedShards> orderedAllocationCandidates;
+        final int allocationsFound;
 
-        public NodeShardsResult(List<NodeGatewayStartedShards> orderedAllocationCandidates, int allocationsFound) {
+        NodeShardsResult(List<NodeGatewayStartedShards> orderedAllocationCandidates, int allocationsFound) {
             this.orderedAllocationCandidates = orderedAllocationCandidates;
             this.allocationsFound = allocationsFound;
         }
     }
 
     static class NodesToAllocate {
-        final List<NodeGatewayStartedShards> yesNodeShards;
-        final List<NodeGatewayStartedShards> throttleNodeShards;
-        final List<NodeGatewayStartedShards> noNodeShards;
+        final List<DecidedNode> yesNodeShards;
+        final List<DecidedNode> throttleNodeShards;
+        final List<DecidedNode> noNodeShards;
 
-        public NodesToAllocate(List<NodeGatewayStartedShards> yesNodeShards,
-                               List<NodeGatewayStartedShards> throttleNodeShards,
-                               List<NodeGatewayStartedShards> noNodeShards) {
+        NodesToAllocate(List<DecidedNode> yesNodeShards, List<DecidedNode> throttleNodeShards, List<DecidedNode> noNodeShards) {
             this.yesNodeShards = yesNodeShards;
             this.throttleNodeShards = throttleNodeShards;
             this.noNodeShards = noNodeShards;
         }
     }
+
+    /**
+     * This class encapsulates the shard state retrieved from a node and the decision that was made
+     * by the allocator for allocating to the node that holds the shard copy.
+     */
+    private static class DecidedNode {
+        final NodeGatewayStartedShards nodeShardState;
+        final Decision decision;
+
+        private DecidedNode(NodeGatewayStartedShards nodeShardState, Decision decision) {
+            this.nodeShardState = nodeShardState;
+            this.decision = decision;
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java b/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java
index d2fbeee577675..0ea705717e7f5 100644
--- a/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java
+++ b/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java
@@ -23,6 +23,7 @@
 import com.carrotsearch.hppc.ObjectLongMap;
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectLongCursor;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -31,24 +32,31 @@
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.routing.UnassignedInfo.AllocationStatus;
-import org.elasticsearch.cluster.routing.RoutingChangesObserver;
+import org.elasticsearch.cluster.routing.allocation.AllocateUnassignedDecision;
+import org.elasticsearch.cluster.routing.allocation.NodeAllocationResult;
+import org.elasticsearch.cluster.routing.allocation.NodeAllocationResult.ShardStoreInfo;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.component.AbstractComponent;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.store.StoreFileMetaData;
 import org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData;
+import org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.NodeStoreFilesMetaData;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 
+import static org.elasticsearch.cluster.routing.UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING;
+
 /**
  */
-public abstract class ReplicaShardAllocator extends AbstractComponent {
+public abstract class ReplicaShardAllocator extends BaseGatewayShardAllocator {
 
     public ReplicaShardAllocator(Settings settings) {
         super(settings);
@@ -65,7 +73,7 @@ public void processExistingRecoveries(RoutingAllocation allocation) {
         List<Runnable> shardCancellationActions = new ArrayList<>();
         for (RoutingNode routingNode : routingNodes) {
             for (ShardRouting shard : routingNode) {
-                if (shard.primary() == true) {
+                if (shard.primary()) {
                     continue;
                 }
                 if (shard.initializing() == false) {
@@ -80,7 +88,7 @@ public void processExistingRecoveries(RoutingAllocation allocation) {
                     continue;
                 }
 
-                AsyncShardFetch.FetchResult<TransportNodesListShardStoreMetaData.NodeStoreFilesMetaData> shardStores = fetchData(shard, allocation);
+                AsyncShardFetch.FetchResult<NodeStoreFilesMetaData> shardStores = fetchData(shard, allocation);
                 if (shardStores.hasData() == false) {
                     logger.trace("{}: fetching new stores for initializing shard", shard);
                     continue; // still fetching
@@ -96,7 +104,7 @@ public void processExistingRecoveries(RoutingAllocation allocation) {
                     continue;
                 }
 
-                MatchingNodes matchingNodes = findMatchingNodes(shard, allocation, primaryStore, shardStores);
+                MatchingNodes matchingNodes = findMatchingNodes(shard, allocation, primaryStore, shardStores, false);
                 if (matchingNodes.getNodeWithHighestMatch() != null) {
                     DiscoveryNode currentNode = allocation.nodes().get(shard.currentNodeId());
                     DiscoveryNode nodeWithHighestMatch = matchingNodes.getNodeWithHighestMatch();
@@ -109,7 +117,7 @@ public void processExistingRecoveries(RoutingAllocation allocation) {
                     }
                     if (currentNode.equals(nodeWithHighestMatch) == false
                             && Objects.equals(currentSyncId, primaryStore.syncId()) == false
-                            && matchingNodes.isNodeMatchBySyncID(nodeWithHighestMatch) == true) {
+                            && matchingNodes.isNodeMatchBySyncID(nodeWithHighestMatch)) {
                         // we found a better match that has a full sync id match, the existing allocation is not fully synced
                         // so we found a better one, cancel this one
                         logger.debug("cancelling allocation of replica on [{}], sync id match found on node [{}]",
@@ -128,86 +136,108 @@ public void processExistingRecoveries(RoutingAllocation allocation) {
         }
     }
 
-    public void allocateUnassigned(RoutingAllocation allocation) {
-        final RoutingNodes routingNodes = allocation.routingNodes();
-        final RoutingNodes.UnassignedShards.UnassignedIterator unassignedIterator = routingNodes.unassigned().iterator();
-        while (unassignedIterator.hasNext()) {
-            ShardRouting shard = unassignedIterator.next();
-            if (shard.primary()) {
-                continue;
-            }
+    /**
+     * Is the allocator responsible for allocating the given {@link ShardRouting}?
+     */
+    private static boolean isResponsibleFor(final ShardRouting shard) {
+        return shard.primary() == false // must be a replica
+                   && shard.unassigned() // must be unassigned
+                   // if we are allocating a replica because of index creation, no need to go and find a copy, there isn't one...
+                   && shard.unassignedInfo().getReason() != UnassignedInfo.Reason.INDEX_CREATED;
+    }
 
-            // if we are allocating a replica because of index creation, no need to go and find a copy, there isn't one...
-            if (shard.unassignedInfo().getReason() == UnassignedInfo.Reason.INDEX_CREATED) {
-                continue;
-            }
+    @Override
+    public AllocateUnassignedDecision makeAllocationDecision(final ShardRouting unassignedShard,
+                                                             final RoutingAllocation allocation,
+                                                             final Logger logger) {
+        if (isResponsibleFor(unassignedShard) == false) {
+            // this allocator is not responsible for deciding on this shard
+            return AllocateUnassignedDecision.NOT_TAKEN;
+        }
 
-            // pre-check if it can be allocated to any node that currently exists, so we won't list the store for it for nothing
-            Decision decision = canBeAllocatedToAtLeastOneNode(shard, allocation);
-            if (decision.type() != Decision.Type.YES) {
-                logger.trace("{}: ignoring allocation, can't be allocated on any node", shard);
-                unassignedIterator.removeAndIgnore(UnassignedInfo.AllocationStatus.fromDecision(decision), allocation.changes());
-                continue;
-            }
+        final RoutingNodes routingNodes = allocation.routingNodes();
+        final boolean explain = allocation.debugDecision();
+        // pre-check if it can be allocated to any node that currently exists, so we won't list the store for it for nothing
+        Tuple<Decision, Map<String, NodeAllocationResult>> result = canBeAllocatedToAtLeastOneNode(unassignedShard, allocation);
+        Decision allocateDecision = result.v1();
+        if (allocateDecision.type() != Decision.Type.YES
+                && (explain == false || hasInitiatedFetching(unassignedShard) == false)) {
+            // only return early if we are not in explain mode, or we are in explain mode but we have not
+            // yet attempted to fetch any shard data
+            logger.trace("{}: ignoring allocation, can't be allocated on any node", unassignedShard);
+            return AllocateUnassignedDecision.no(UnassignedInfo.AllocationStatus.fromDecision(allocateDecision.type()),
+                result.v2() != null ? new ArrayList<>(result.v2().values()) : null);
+        }
 
-            AsyncShardFetch.FetchResult<TransportNodesListShardStoreMetaData.NodeStoreFilesMetaData> shardStores = fetchData(shard, allocation);
-            if (shardStores.hasData() == false) {
-                logger.trace("{}: ignoring allocation, still fetching shard stores", shard);
-                allocation.setHasPendingAsyncFetch();
-                unassignedIterator.removeAndIgnore(AllocationStatus.FETCHING_SHARD_DATA, allocation.changes());
-                continue; // still fetching
+        AsyncShardFetch.FetchResult<NodeStoreFilesMetaData> shardStores = fetchData(unassignedShard, allocation);
+        if (shardStores.hasData() == false) {
+            logger.trace("{}: ignoring allocation, still fetching shard stores", unassignedShard);
+            allocation.setHasPendingAsyncFetch();
+            List<NodeAllocationResult> nodeDecisions = null;
+            if (explain) {
+                nodeDecisions = buildDecisionsForAllNodes(unassignedShard, allocation);
             }
+            return AllocateUnassignedDecision.no(AllocationStatus.FETCHING_SHARD_DATA, nodeDecisions);
+        }
 
-            ShardRouting primaryShard = routingNodes.activePrimary(shard.shardId());
-            assert primaryShard != null : "the replica shard can be allocated on at least one node, so there must be an active primary";
-            TransportNodesListShardStoreMetaData.StoreFilesMetaData primaryStore = findStore(primaryShard, allocation, shardStores);
-            if (primaryStore == null) {
-                // if we can't find the primary data, it is probably because the primary shard is corrupted (and listing failed)
-                // we want to let the replica be allocated in order to expose the actual problem with the primary that the replica
-                // will try and recover from
-                // Note, this is the existing behavior, as exposed in running CorruptFileTest#testNoPrimaryData
-                logger.trace("{}: no primary shard store found or allocated, letting actual allocation figure it out", shard);
-                continue;
-            }
+        ShardRouting primaryShard = routingNodes.activePrimary(unassignedShard.shardId());
+        if (primaryShard == null) {
+            assert explain : "primary should only be null here if we are in explain mode, so we didn't " +
+                                 "exit early when canBeAllocatedToAtLeastOneNode didn't return a YES decision";
+            return AllocateUnassignedDecision.no(UnassignedInfo.AllocationStatus.fromDecision(allocateDecision.type()),
+                new ArrayList<>(result.v2().values()));
+        }
 
-            MatchingNodes matchingNodes = findMatchingNodes(shard, allocation, primaryStore, shardStores);
+        TransportNodesListShardStoreMetaData.StoreFilesMetaData primaryStore = findStore(primaryShard, allocation, shardStores);
+        if (primaryStore == null) {
+            // if we can't find the primary data, it is probably because the primary shard is corrupted (and listing failed)
+            // we want to let the replica be allocated in order to expose the actual problem with the primary that the replica
+            // will try and recover from
+            // Note, this is the existing behavior, as exposed in running CorruptFileTest#testNoPrimaryData
+            logger.trace("{}: no primary shard store found or allocated, letting actual allocation figure it out", unassignedShard);
+            return AllocateUnassignedDecision.NOT_TAKEN;
+        }
 
-            if (matchingNodes.getNodeWithHighestMatch() != null) {
-                RoutingNode nodeWithHighestMatch = allocation.routingNodes().node(matchingNodes.getNodeWithHighestMatch().getId());
-                // we only check on THROTTLE since we checked before before on NO
-                decision = allocation.deciders().canAllocate(shard, nodeWithHighestMatch, allocation);
-                if (decision.type() == Decision.Type.THROTTLE) {
-                    logger.debug("[{}][{}]: throttling allocation [{}] to [{}] in order to reuse its unallocated persistent store", shard.index(), shard.id(), shard, nodeWithHighestMatch.node());
-                    // we are throttling this, but we have enough to allocate to this node, ignore it for now
-                    unassignedIterator.removeAndIgnore(UnassignedInfo.AllocationStatus.fromDecision(decision), allocation.changes());
-                } else {
-                    logger.debug("[{}][{}]: allocating [{}] to [{}] in order to reuse its unallocated persistent store", shard.index(), shard.id(), shard, nodeWithHighestMatch.node());
-                    // we found a match
-                    unassignedIterator.initialize(nodeWithHighestMatch.nodeId(), null, allocation.clusterInfo().getShardSize(shard, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE), allocation.changes());
-                }
-            } else if (matchingNodes.hasAnyData() == false) {
-                // if we didn't manage to find *any* data (regardless of matching sizes), check if the allocation of the replica shard needs to be delayed
-                ignoreUnassignedIfDelayed(unassignedIterator, shard, allocation.changes());
+        MatchingNodes matchingNodes = findMatchingNodes(unassignedShard, allocation, primaryStore, shardStores, explain);
+        assert explain == false || matchingNodes.nodeDecisions != null : "in explain mode, we must have individual node decisions";
+
+        List<NodeAllocationResult> nodeDecisions = augmentExplanationsWithStoreInfo(result.v2(), matchingNodes.nodeDecisions);
+        if (allocateDecision.type() != Decision.Type.YES) {
+            return AllocateUnassignedDecision.no(UnassignedInfo.AllocationStatus.fromDecision(allocateDecision.type()), nodeDecisions);
+        } else if (matchingNodes.getNodeWithHighestMatch() != null) {
+            RoutingNode nodeWithHighestMatch = allocation.routingNodes().node(matchingNodes.getNodeWithHighestMatch().getId());
+            // we only check on THROTTLE since we checked before before on NO
+            Decision decision = allocation.deciders().canAllocate(unassignedShard, nodeWithHighestMatch, allocation);
+            if (decision.type() == Decision.Type.THROTTLE) {
+                logger.debug("[{}][{}]: throttling allocation [{}] to [{}] in order to reuse its unallocated persistent store",
+                    unassignedShard.index(), unassignedShard.id(), unassignedShard, nodeWithHighestMatch.node());
+                // we are throttling this, as we have enough other shards to allocate to this node, so ignore it for now
+                return AllocateUnassignedDecision.throttle(nodeDecisions);
+            } else {
+                logger.debug("[{}][{}]: allocating [{}] to [{}] in order to reuse its unallocated persistent store",
+                    unassignedShard.index(), unassignedShard.id(), unassignedShard, nodeWithHighestMatch.node());
+                // we found a match
+                return AllocateUnassignedDecision.yes(nodeWithHighestMatch.node(), null, nodeDecisions, true);
             }
+        } else if (matchingNodes.hasAnyData() == false && unassignedShard.unassignedInfo().isDelayed()) {
+            // if we didn't manage to find *any* data (regardless of matching sizes), and the replica is
+            // unassigned due to a node leaving, so we delay allocation of this replica to see if the
+            // node with the shard copy will rejoin so we can re-use the copy it has
+            logger.debug("{}: allocation of [{}] is delayed", unassignedShard.shardId(), unassignedShard);
+            long remainingDelayMillis = 0L;
+            long totalDelayMillis = 0L;
+            if (explain) {
+                UnassignedInfo unassignedInfo = unassignedShard.unassignedInfo();
+                MetaData metadata = allocation.metaData();
+                IndexMetaData indexMetaData = metadata.index(unassignedShard.index());
+                totalDelayMillis = INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.get(indexMetaData.getSettings()).getMillis();
+                long remainingDelayNanos = unassignedInfo.getRemainingDelay(System.nanoTime(), indexMetaData.getSettings());
+                remainingDelayMillis = TimeValue.timeValueNanos(remainingDelayNanos).millis();
+            }
+            return AllocateUnassignedDecision.delayed(remainingDelayMillis, totalDelayMillis, nodeDecisions);
         }
-    }
 
-    /**
-     * Check if the allocation of the replica is to be delayed. Compute the delay and if it is delayed, add it to the ignore unassigned list
-     * Note: we only care about replica in delayed allocation, since if we have an unassigned primary it
-     *       will anyhow wait to find an existing copy of the shard to be allocated
-     * Note: the other side of the equation is scheduling a reroute in a timely manner, which happens in the RoutingService
-     *
-     * PUBLIC FOR TESTS!
-     *
-     * @param unassignedIterator iterator over unassigned shards
-     * @param shard the shard which might be delayed
-     */
-    public void ignoreUnassignedIfDelayed(RoutingNodes.UnassignedShards.UnassignedIterator unassignedIterator, ShardRouting shard, RoutingChangesObserver changes) {
-        if (shard.unassignedInfo().isDelayed()) {
-            logger.debug("{}: allocation of [{}] is delayed", shard.shardId(), shard);
-            unassignedIterator.removeAndIgnore(AllocationStatus.DELAYED_ALLOCATION, changes);
-        }
+        return AllocateUnassignedDecision.NOT_TAKEN;
     }
 
     /**
@@ -215,10 +245,14 @@ public void ignoreUnassignedIfDelayed(RoutingNodes.UnassignedShards.UnassignedIt
      *
      * Returns the best allocation decision for allocating the shard on any node (i.e. YES if at least one
      * node decided YES, THROTTLE if at least one node decided THROTTLE, and NO if none of the nodes decided
-     * YES or THROTTLE.
+     * YES or THROTTLE).  If in explain mode, also returns the node-level explanations as the second element
+     * in the returned tuple.
      */
-    private Decision canBeAllocatedToAtLeastOneNode(ShardRouting shard, RoutingAllocation allocation) {
+    private Tuple<Decision, Map<String, NodeAllocationResult>> canBeAllocatedToAtLeastOneNode(ShardRouting shard,
+                                                                                              RoutingAllocation allocation) {
         Decision madeDecision = Decision.NO;
+        final boolean explain = allocation.debugDecision();
+        Map<String, NodeAllocationResult> nodeDecisions = explain ? new HashMap<>() : null;
         for (ObjectCursor<DiscoveryNode> cursor : allocation.nodes().getDataNodes().values()) {
             RoutingNode node = allocation.routingNodes().node(cursor.value.getId());
             if (node == null) {
@@ -227,25 +261,52 @@ private Decision canBeAllocatedToAtLeastOneNode(ShardRouting shard, RoutingAlloc
             // if we can't allocate it on a node, ignore it, for example, this handles
             // cases for only allocating a replica after a primary
             Decision decision = allocation.deciders().canAllocate(shard, node, allocation);
-            if (decision.type() == Decision.Type.YES) {
-                return decision;
+            if (decision.type() == Decision.Type.YES && madeDecision.type() != Decision.Type.YES) {
+                if (explain) {
+                    madeDecision = decision;
+                } else {
+                    return Tuple.tuple(decision, nodeDecisions);
+                }
             } else if (madeDecision.type() == Decision.Type.NO && decision.type() == Decision.Type.THROTTLE) {
                 madeDecision = decision;
             }
+            if (explain) {
+                nodeDecisions.put(node.nodeId(), new NodeAllocationResult(node.node(), null, decision));
+            }
+        }
+        return Tuple.tuple(madeDecision, nodeDecisions);
+    }
+
+    /**
+     * Takes the store info for nodes that have a shard store and adds them to the node decisions,
+     * leaving the node explanations untouched for those nodes that do not have any store information.
+     */
+    private List<NodeAllocationResult> augmentExplanationsWithStoreInfo(Map<String, NodeAllocationResult> nodeDecisions,
+                                                                        Map<String, NodeAllocationResult> withShardStores) {
+        if (nodeDecisions == null || withShardStores == null) {
+            return null;
+        }
+        List<NodeAllocationResult> augmented = new ArrayList<>();
+        for (Map.Entry<String, NodeAllocationResult> entry : nodeDecisions.entrySet()) {
+            if (withShardStores.containsKey(entry.getKey())) {
+                augmented.add(withShardStores.get(entry.getKey()));
+            } else {
+                augmented.add(entry.getValue());
+            }
         }
-        return madeDecision;
+        return augmented;
     }
 
     /**
      * Finds the store for the assigned shard in the fetched data, returns null if none is found.
      */
-    private TransportNodesListShardStoreMetaData.StoreFilesMetaData findStore(ShardRouting shard, RoutingAllocation allocation, AsyncShardFetch.FetchResult<TransportNodesListShardStoreMetaData.NodeStoreFilesMetaData> data) {
+    private TransportNodesListShardStoreMetaData.StoreFilesMetaData findStore(ShardRouting shard, RoutingAllocation allocation, AsyncShardFetch.FetchResult<NodeStoreFilesMetaData> data) {
         assert shard.currentNodeId() != null;
         DiscoveryNode primaryNode = allocation.nodes().get(shard.currentNodeId());
         if (primaryNode == null) {
             return null;
         }
-        TransportNodesListShardStoreMetaData.NodeStoreFilesMetaData primaryNodeFilesStore = data.getData().get(primaryNode);
+        NodeStoreFilesMetaData primaryNodeFilesStore = data.getData().get(primaryNode);
         if (primaryNodeFilesStore == null) {
             return null;
         }
@@ -254,9 +315,11 @@ private TransportNodesListShardStoreMetaData.StoreFilesMetaData findStore(ShardR
 
     private MatchingNodes findMatchingNodes(ShardRouting shard, RoutingAllocation allocation,
                                             TransportNodesListShardStoreMetaData.StoreFilesMetaData primaryStore,
-                                            AsyncShardFetch.FetchResult<TransportNodesListShardStoreMetaData.NodeStoreFilesMetaData> data) {
+                                            AsyncShardFetch.FetchResult<NodeStoreFilesMetaData> data,
+                                            boolean explain) {
         ObjectLongMap<DiscoveryNode> nodesToSize = new ObjectLongHashMap<>();
-        for (Map.Entry<DiscoveryNode, TransportNodesListShardStoreMetaData.NodeStoreFilesMetaData> nodeStoreEntry : data.getData().entrySet()) {
+        Map<String, NodeAllocationResult> nodeDecisions = explain ? new HashMap<>() : null;
+        for (Map.Entry<DiscoveryNode, NodeStoreFilesMetaData> nodeStoreEntry : data.getData().entrySet()) {
             DiscoveryNode discoNode = nodeStoreEntry.getKey();
             TransportNodesListShardStoreMetaData.StoreFilesMetaData storeFilesMetaData = nodeStoreEntry.getValue().storeFilesMetaData();
             // we don't have any files at all, it is an empty index
@@ -273,41 +336,70 @@ private MatchingNodes findMatchingNodes(ShardRouting shard, RoutingAllocation al
             // we only check for NO, since if this node is THROTTLING and it has enough "same data"
             // then we will try and assign it next time
             Decision decision = allocation.deciders().canAllocate(shard, node, allocation);
+
+            long matchingBytes = -1;
+            if (explain) {
+                matchingBytes = computeMatchingBytes(primaryStore, storeFilesMetaData);
+                ShardStoreInfo shardStoreInfo = new ShardStoreInfo(matchingBytes);
+                nodeDecisions.put(node.nodeId(), new NodeAllocationResult(discoNode, shardStoreInfo, decision));
+            }
+
             if (decision.type() == Decision.Type.NO) {
                 continue;
             }
 
-            String primarySyncId = primaryStore.syncId();
-            String replicaSyncId = storeFilesMetaData.syncId();
-            // see if we have a sync id we can make use of
-            if (replicaSyncId != null && replicaSyncId.equals(primarySyncId)) {
-                logger.trace("{}: node [{}] has same sync id {} as primary", shard, discoNode.getName(), replicaSyncId);
-                nodesToSize.put(discoNode, Long.MAX_VALUE);
-            } else {
-                long sizeMatched = 0;
-                for (StoreFileMetaData storeFileMetaData : storeFilesMetaData) {
-                    String metaDataFileName = storeFileMetaData.name();
-                    if (primaryStore.fileExists(metaDataFileName) && primaryStore.file(metaDataFileName).isSame(storeFileMetaData)) {
-                        sizeMatched += storeFileMetaData.length();
-                    }
+            if (matchingBytes < 0) {
+                matchingBytes = computeMatchingBytes(primaryStore, storeFilesMetaData);
+            }
+            nodesToSize.put(discoNode, matchingBytes);
+            if (logger.isTraceEnabled()) {
+                if (matchingBytes == Long.MAX_VALUE) {
+                    logger.trace("{}: node [{}] has same sync id {} as primary", shard, discoNode.getName(), storeFilesMetaData.syncId());
+                } else {
+                    logger.trace("{}: node [{}] has [{}/{}] bytes of re-usable data",
+                        shard, discoNode.getName(), new ByteSizeValue(matchingBytes), matchingBytes);
                 }
-                logger.trace("{}: node [{}] has [{}/{}] bytes of re-usable data",
-                        shard, discoNode.getName(), new ByteSizeValue(sizeMatched), sizeMatched);
-                nodesToSize.put(discoNode, sizeMatched);
             }
         }
 
-        return new MatchingNodes(nodesToSize);
+        return new MatchingNodes(nodesToSize, nodeDecisions);
+    }
+
+    private static long computeMatchingBytes(TransportNodesListShardStoreMetaData.StoreFilesMetaData primaryStore,
+                                             TransportNodesListShardStoreMetaData.StoreFilesMetaData storeFilesMetaData) {
+        String primarySyncId = primaryStore.syncId();
+        String replicaSyncId = storeFilesMetaData.syncId();
+        // see if we have a sync id we can make use of
+        if (replicaSyncId != null && replicaSyncId.equals(primarySyncId)) {
+            return Long.MAX_VALUE;
+        } else {
+            long sizeMatched = 0;
+            for (StoreFileMetaData storeFileMetaData : storeFilesMetaData) {
+                String metaDataFileName = storeFileMetaData.name();
+                if (primaryStore.fileExists(metaDataFileName) && primaryStore.file(metaDataFileName).isSame(storeFileMetaData)) {
+                    sizeMatched += storeFileMetaData.length();
+                }
+            }
+            return sizeMatched;
+        }
     }
 
-    protected abstract AsyncShardFetch.FetchResult<TransportNodesListShardStoreMetaData.NodeStoreFilesMetaData> fetchData(ShardRouting shard, RoutingAllocation allocation);
+    protected abstract AsyncShardFetch.FetchResult<NodeStoreFilesMetaData> fetchData(ShardRouting shard, RoutingAllocation allocation);
+
+    /**
+     * Returns a boolean indicating whether fetching shard data has been triggered at any point for the given shard.
+     */
+    protected abstract boolean hasInitiatedFetching(ShardRouting shard);
 
     static class MatchingNodes {
         private final ObjectLongMap<DiscoveryNode> nodesToSize;
         private final DiscoveryNode nodeWithHighestMatch;
+        @Nullable
+        private final Map<String, NodeAllocationResult> nodeDecisions;
 
-        public MatchingNodes(ObjectLongMap<DiscoveryNode> nodesToSize) {
+        MatchingNodes(ObjectLongMap<DiscoveryNode> nodesToSize, @Nullable Map<String, NodeAllocationResult> nodeDecisions) {
             this.nodesToSize = nodesToSize;
+            this.nodeDecisions = nodeDecisions;
 
             long highestMatchSize = 0;
             DiscoveryNode highestMatchNode = null;
diff --git a/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayMetaState.java b/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayMetaState.java
index 24886dc72d39c..a67750e7b6a12 100644
--- a/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayMetaState.java
+++ b/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayMetaState.java
@@ -103,11 +103,6 @@ protected NodeGatewayMetaState nodeOperation(NodeRequest request) {
         }
     }
 
-    @Override
-    protected boolean accumulateExceptions() {
-        return true;
-    }
-
     public static class Request extends BaseNodesRequest<Request> {
 
         public Request() {
@@ -188,7 +183,7 @@ public MetaData metaData() {
         public void readFrom(StreamInput in) throws IOException {
             super.readFrom(in);
             if (in.readBoolean()) {
-                metaData = MetaData.Builder.readFrom(in);
+                metaData = MetaData.readFrom(in);
             }
         }
 
diff --git a/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java b/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java
index 31fc290c10cb8..f9f98296a26f7 100644
--- a/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java
+++ b/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java
@@ -39,6 +39,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.shard.ShardId;
@@ -115,7 +116,7 @@ protected NodeGatewayStartedShards nodeOperation(NodeRequest request) {
         try {
             final ShardId shardId = request.getShardId();
             logger.trace("{} loading local shard state info", shardId);
-            ShardStateMetaData shardStateMetaData = ShardStateMetaData.FORMAT.loadLatestState(logger,
+            ShardStateMetaData shardStateMetaData = ShardStateMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY,
                 nodeEnv.availableShardPaths(request.shardId));
             if (shardStateMetaData != null) {
                 IndexMetaData metaData = clusterService.state().metaData().index(shardId.getIndex());
@@ -123,7 +124,8 @@ protected NodeGatewayStartedShards nodeOperation(NodeRequest request) {
                     // we may send this requests while processing the cluster state that recovered the index
                     // sometimes the request comes in before the local node processed that cluster state
                     // in such cases we can load it from disk
-                    metaData = IndexMetaData.FORMAT.loadLatestState(logger, nodeEnv.indexPaths(shardId.getIndex()));
+                    metaData = IndexMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY,
+                        nodeEnv.indexPaths(shardId.getIndex()));
                 }
                 if (metaData == null) {
                     ElasticsearchException e = new ElasticsearchException("failed to find local IndexMetaData");
@@ -170,11 +172,6 @@ protected NodeGatewayStartedShards nodeOperation(NodeRequest request) {
         }
     }
 
-    @Override
-    protected boolean accumulateExceptions() {
-        return true;
-    }
-
     public static class Request extends BaseNodesRequest<Request> {
 
         private ShardId shardId;
diff --git a/core/src/main/java/org/elasticsearch/http/HttpServer.java b/core/src/main/java/org/elasticsearch/http/HttpServer.java
deleted file mode 100644
index ccf2d764c052a..0000000000000
--- a/core/src/main/java/org/elasticsearch/http/HttpServer.java
+++ /dev/null
@@ -1,201 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.http;
-
-import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.breaker.CircuitBreaker;
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.component.AbstractLifecycleComponent;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.io.Streams;
-import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.indices.breaker.CircuitBreakerService;
-import org.elasticsearch.node.service.NodeService;
-import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.RestChannel;
-import org.elasticsearch.rest.RestController;
-import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.RestResponse;
-import org.elasticsearch.rest.RestStatus;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import static org.elasticsearch.rest.RestStatus.FORBIDDEN;
-import static org.elasticsearch.rest.RestStatus.INTERNAL_SERVER_ERROR;
-
-/**
- * A component to serve http requests, backed by rest handlers.
- */
-public class HttpServer extends AbstractLifecycleComponent implements HttpServerAdapter {
-    private final HttpServerTransport transport;
-
-    private final RestController restController;
-
-    private final NodeClient client;
-
-    private final CircuitBreakerService circuitBreakerService;
-
-    @Inject
-    public HttpServer(Settings settings, HttpServerTransport transport, RestController restController,
-                      NodeClient client, CircuitBreakerService circuitBreakerService) {
-        super(settings);
-        this.transport = transport;
-        this.restController = restController;
-        this.client = client;
-        this.circuitBreakerService = circuitBreakerService;
-        transport.httpServerAdapter(this);
-    }
-
-
-    @Override
-    protected void doStart() {
-        transport.start();
-        if (logger.isInfoEnabled()) {
-            logger.info("{}", transport.boundAddress());
-        }
-    }
-
-    @Override
-    protected void doStop() {
-        transport.stop();
-    }
-
-    @Override
-    protected void doClose() {
-        transport.close();
-    }
-
-    public HttpInfo info() {
-        return transport.info();
-    }
-
-    public HttpStats stats() {
-        return transport.stats();
-    }
-
-    public void dispatchRequest(RestRequest request, RestChannel channel, ThreadContext threadContext) {
-        if (request.rawPath().equals("/favicon.ico")) {
-            handleFavicon(request, channel);
-            return;
-        }
-        RestChannel responseChannel = channel;
-        try {
-            int contentLength = request.content().length();
-            if (restController.canTripCircuitBreaker(request)) {
-                inFlightRequestsBreaker(circuitBreakerService).addEstimateBytesAndMaybeBreak(contentLength, "<http_request>");
-            } else {
-                inFlightRequestsBreaker(circuitBreakerService).addWithoutBreaking(contentLength);
-            }
-            // iff we could reserve bytes for the request we need to send the response also over this channel
-            responseChannel = new ResourceHandlingHttpChannel(channel, circuitBreakerService, contentLength);
-            restController.dispatchRequest(request, responseChannel, client, threadContext);
-        } catch (Exception e) {
-            restController.sendErrorResponse(request, responseChannel, e);
-        }
-    }
-
-    void handleFavicon(RestRequest request, RestChannel channel) {
-        if (request.method() == RestRequest.Method.GET) {
-            try {
-                try (InputStream stream = getClass().getResourceAsStream("/config/favicon.ico")) {
-                    ByteArrayOutputStream out = new ByteArrayOutputStream();
-                    Streams.copy(stream, out);
-                    BytesRestResponse restResponse = new BytesRestResponse(RestStatus.OK, "image/x-icon", out.toByteArray());
-                    channel.sendResponse(restResponse);
-                }
-            } catch (IOException e) {
-                channel.sendResponse(new BytesRestResponse(INTERNAL_SERVER_ERROR, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY));
-            }
-        } else {
-            channel.sendResponse(new BytesRestResponse(FORBIDDEN, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY));
-        }
-    }
-
-    private static final class ResourceHandlingHttpChannel implements RestChannel {
-        private final RestChannel delegate;
-        private final CircuitBreakerService circuitBreakerService;
-        private final int contentLength;
-        private final AtomicBoolean closed = new AtomicBoolean();
-
-        public ResourceHandlingHttpChannel(RestChannel delegate, CircuitBreakerService circuitBreakerService, int contentLength) {
-            this.delegate = delegate;
-            this.circuitBreakerService = circuitBreakerService;
-            this.contentLength = contentLength;
-        }
-
-        @Override
-        public XContentBuilder newBuilder() throws IOException {
-            return delegate.newBuilder();
-        }
-
-        @Override
-        public XContentBuilder newErrorBuilder() throws IOException {
-            return delegate.newErrorBuilder();
-        }
-
-        @Override
-        public XContentBuilder newBuilder(@Nullable BytesReference autoDetectSource, boolean useFiltering) throws IOException {
-            return delegate.newBuilder(autoDetectSource, useFiltering);
-        }
-
-        @Override
-        public BytesStreamOutput bytesOutput() {
-            return delegate.bytesOutput();
-        }
-
-        @Override
-        public RestRequest request() {
-            return delegate.request();
-        }
-
-        @Override
-        public boolean detailedErrorsEnabled() {
-            return delegate.detailedErrorsEnabled();
-        }
-
-        @Override
-        public void sendResponse(RestResponse response) {
-            close();
-            delegate.sendResponse(response);
-        }
-
-        private void close() {
-            // attempt to close once atomically
-            if (closed.compareAndSet(false, true) == false) {
-                throw new IllegalStateException("Channel is already closed");
-            }
-            inFlightRequestsBreaker(circuitBreakerService).addWithoutBreaking(-contentLength);
-        }
-
-    }
-
-    private static CircuitBreaker inFlightRequestsBreaker(CircuitBreakerService circuitBreakerService) {
-        // We always obtain a fresh breaker to reflect changes to the breaker configuration.
-        return circuitBreakerService.getBreaker(CircuitBreaker.IN_FLIGHT_REQUESTS);
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/http/HttpServerAdapter.java b/core/src/main/java/org/elasticsearch/http/HttpServerAdapter.java
deleted file mode 100644
index 16b0bd0044306..0000000000000
--- a/core/src/main/java/org/elasticsearch/http/HttpServerAdapter.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.http;
-
-import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.rest.RestChannel;
-import org.elasticsearch.rest.RestRequest;
-
-/**
- *
- */
-public interface HttpServerAdapter {
-
-    void dispatchRequest(RestRequest request, RestChannel channel, ThreadContext context);
-
-}
diff --git a/core/src/main/java/org/elasticsearch/http/HttpServerTransport.java b/core/src/main/java/org/elasticsearch/http/HttpServerTransport.java
index 4dc4a888d8af4..40f08f3998c3d 100644
--- a/core/src/main/java/org/elasticsearch/http/HttpServerTransport.java
+++ b/core/src/main/java/org/elasticsearch/http/HttpServerTransport.java
@@ -21,6 +21,9 @@
 
 import org.elasticsearch.common.component.LifecycleComponent;
 import org.elasticsearch.common.transport.BoundTransportAddress;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.rest.RestChannel;
+import org.elasticsearch.rest.RestRequest;
 
 public interface HttpServerTransport extends LifecycleComponent {
 
@@ -33,6 +36,32 @@ public interface HttpServerTransport extends LifecycleComponent {
 
     HttpStats stats();
 
-    void httpServerAdapter(HttpServerAdapter httpServerAdapter);
+    /**
+     * Dispatches HTTP requests.
+     */
+    interface Dispatcher {
+
+        /**
+         * Dispatches the {@link RestRequest} to the relevant request handler or responds to the given rest channel directly if
+         * the request can't be handled by any request handler.
+         *
+         * @param request       the request to dispatch
+         * @param channel       the response channel of this request
+         * @param threadContext the thread context
+         */
+        void dispatchRequest(RestRequest request, RestChannel channel, ThreadContext threadContext);
+
+        /**
+         * Dispatches a bad request. For example, if a request is malformed it will be dispatched via this method with the cause of the bad
+         * request.
+         *
+         * @param request       the request to dispatch
+         * @param channel       the response channel of this request
+         * @param threadContext the thread context
+         * @param cause         the cause of the bad request
+         */
+        void dispatchBadRequest(RestRequest request, RestChannel channel, ThreadContext threadContext, Throwable cause);
+
+    }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/http/HttpTransportSettings.java b/core/src/main/java/org/elasticsearch/http/HttpTransportSettings.java
index 60bc3449d0be5..b5e254aa4c2ef 100644
--- a/core/src/main/java/org/elasticsearch/http/HttpTransportSettings.java
+++ b/core/src/main/java/org/elasticsearch/http/HttpTransportSettings.java
@@ -68,6 +68,8 @@ public final class HttpTransportSettings {
         Setting.intSetting("http.publish_port", -1, -1, Property.NodeScope);
     public static final Setting<Boolean> SETTING_HTTP_DETAILED_ERRORS_ENABLED =
         Setting.boolSetting("http.detailed_errors.enabled", true, Property.NodeScope);
+    public static final Setting<Boolean> SETTING_HTTP_CONTENT_TYPE_REQUIRED =
+        Setting.boolSetting("http.content_type.required", false, Property.NodeScope);
     public static final Setting<ByteSizeValue> SETTING_HTTP_MAX_CONTENT_LENGTH =
         Setting.byteSizeSetting("http.max_content_length", new ByteSizeValue(100, ByteSizeUnit.MB), Property.NodeScope);
     public static final Setting<ByteSizeValue> SETTING_HTTP_MAX_CHUNK_SIZE =
diff --git a/core/src/main/java/org/elasticsearch/index/CompositeIndexEventListener.java b/core/src/main/java/org/elasticsearch/index/CompositeIndexEventListener.java
index 3b2cf5cbd0749..90d8a205e8b57 100644
--- a/core/src/main/java/org/elasticsearch/index/CompositeIndexEventListener.java
+++ b/core/src/main/java/org/elasticsearch/index/CompositeIndexEventListener.java
@@ -30,6 +30,7 @@
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.IndexShardState;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason;
 
 import java.util.ArrayList;
 import java.util.Collection;
@@ -176,48 +177,24 @@ public void beforeIndexShardCreated(ShardId shardId, Settings indexSettings) {
     }
 
     @Override
-    public void beforeIndexClosed(IndexService indexService) {
+    public void beforeIndexRemoved(IndexService indexService, IndexRemovalReason reason) {
         for (IndexEventListener listener : listeners) {
             try {
-                listener.beforeIndexClosed(indexService);
+                listener.beforeIndexRemoved(indexService, reason);
             } catch (Exception e) {
-                logger.warn("failed to invoke before index closed callback", e);
+                logger.warn("failed to invoke before index removed callback", e);
                 throw e;
             }
         }
     }
 
     @Override
-    public void beforeIndexDeleted(IndexService indexService) {
+    public void afterIndexRemoved(Index index, IndexSettings indexSettings, IndexRemovalReason reason) {
         for (IndexEventListener listener : listeners) {
             try {
-                listener.beforeIndexDeleted(indexService);
+                listener.afterIndexRemoved(index, indexSettings, reason);
             } catch (Exception e) {
-                logger.warn("failed to invoke before index deleted callback", e);
-                throw e;
-            }
-        }
-    }
-
-    @Override
-    public void afterIndexDeleted(Index index, Settings indexSettings) {
-        for (IndexEventListener listener : listeners) {
-            try {
-                listener.afterIndexDeleted(index, indexSettings);
-            } catch (Exception e) {
-                logger.warn("failed to invoke after index deleted callback", e);
-                throw e;
-            }
-        }
-    }
-
-    @Override
-    public void afterIndexClosed(Index index, Settings indexSettings) {
-        for (IndexEventListener listener : listeners) {
-            try {
-                listener.afterIndexClosed(index, indexSettings);
-            } catch (Exception e) {
-                logger.warn("failed to invoke after index closed callback", e);
+                logger.warn("failed to invoke after index removed callback", e);
                 throw e;
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/index/Index.java b/core/src/main/java/org/elasticsearch/index/Index.java
index 25b293ad3879e..da94ad2ec7250 100644
--- a/core/src/main/java/org/elasticsearch/index/Index.java
+++ b/core/src/main/java/org/elasticsearch/index/Index.java
@@ -21,8 +21,6 @@
 
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -42,7 +40,7 @@ public class Index implements Writeable, ToXContent {
     public static final Index[] EMPTY_ARRAY = new Index[0];
     private static final String INDEX_UUID_KEY = "index_uuid";
     private static final String INDEX_NAME_KEY = "index_name";
-    private static final ObjectParser<Builder, ParseFieldMatcherSupplier> INDEX_PARSER = new ObjectParser<>("index", Builder::new);
+    private static final ObjectParser<Builder, Void> INDEX_PARSER = new ObjectParser<>("index", Builder::new);
     static {
         INDEX_PARSER.declareString(Builder::name, new ParseField(INDEX_NAME_KEY));
         INDEX_PARSER.declareString(Builder::uuid, new ParseField(INDEX_UUID_KEY));
@@ -118,11 +116,7 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa
     }
 
     public static Index fromXContent(final XContentParser parser) throws IOException {
-        return INDEX_PARSER.parse(parser, () -> ParseFieldMatcher.STRICT).build();
-    }
-
-    public static final Index parseIndex(final XContentParser parser, final ParseFieldMatcherSupplier supplier) {
-        return INDEX_PARSER.apply(parser, supplier).build();
+        return INDEX_PARSER.parse(parser, null).build();
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/index/IndexModule.java b/core/src/main/java/org/elasticsearch/index/IndexModule.java
index f6227ca3276fc..22c58f308eea9 100644
--- a/core/src/main/java/org/elasticsearch/index/IndexModule.java
+++ b/core/src/main/java/org/elasticsearch/index/IndexModule.java
@@ -20,16 +20,21 @@
 package org.elasticsearch.index;
 
 import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.analysis.AnalysisRegistry;
-import org.elasticsearch.index.cache.query.QueryCache;
-import org.elasticsearch.index.cache.query.IndexQueryCache;
 import org.elasticsearch.index.cache.query.DisabledQueryCache;
+import org.elasticsearch.index.cache.query.IndexQueryCache;
+import org.elasticsearch.index.cache.query.QueryCache;
 import org.elasticsearch.index.engine.EngineFactory;
+import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.shard.IndexEventListener;
 import org.elasticsearch.index.shard.IndexSearcherWrapper;
 import org.elasticsearch.index.shard.IndexingOperationListener;
@@ -40,8 +45,11 @@
 import org.elasticsearch.index.store.IndexStore;
 import org.elasticsearch.index.store.IndexStoreConfig;
 import org.elasticsearch.indices.IndicesQueryCache;
+import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
 import org.elasticsearch.indices.mapper.MapperRegistry;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -93,6 +101,11 @@ public final class IndexModule {
     public static final Setting<Boolean> INDEX_QUERY_CACHE_EVERYTHING_SETTING =
         Setting.boolSetting("index.queries.cache.everything", false, Property.IndexScope);
 
+    // This setting is an escape hatch in case not caching term queries would slow some users down
+    // Do not document.
+    public static final Setting<Boolean> INDEX_QUERY_CACHE_TERM_QUERIES_SETTING =
+        Setting.boolSetting("index.queries.cache.term_queries", false, Property.IndexScope);
+
     private final IndexSettings indexSettings;
     private final IndexStoreConfig indexStoreConfig;
     private final AnalysisRegistry analysisRegistry;
@@ -306,12 +319,14 @@ public interface IndexSearcherWrapperFactory {
         /**
          * Returns a new IndexSearcherWrapper. This method is called once per index per node
          */
-        IndexSearcherWrapper newWrapper(final IndexService indexService);
+        IndexSearcherWrapper newWrapper(IndexService indexService);
     }
 
-    public IndexService newIndexService(NodeEnvironment environment, IndexService.ShardStoreDeleter shardStoreDeleter,
-                                        NodeServicesProvider servicesProvider, IndicesQueryCache indicesQueryCache,
-                                        MapperRegistry mapperRegistry, IndicesFieldDataCache indicesFieldDataCache) throws IOException {
+    public IndexService newIndexService(NodeEnvironment environment, NamedXContentRegistry xContentRegistry,
+            IndexService.ShardStoreDeleter shardStoreDeleter, CircuitBreakerService circuitBreakerService, BigArrays bigArrays,
+            ThreadPool threadPool, ScriptService scriptService,
+            ClusterService clusterService, Client client, IndicesQueryCache indicesQueryCache, MapperRegistry mapperRegistry,
+            IndicesFieldDataCache indicesFieldDataCache) throws IOException {
         final IndexEventListener eventListener = freeze();
         IndexSearcherWrapperFactory searcherWrapperFactory = indexSearcherWrapper.get() == null
             ? (shard) -> null : indexSearcherWrapper.get();
@@ -344,9 +359,20 @@ public IndexService newIndexService(NodeEnvironment environment, IndexService.Sh
         } else {
             queryCache = new DisabledQueryCache(indexSettings);
         }
-        return new IndexService(indexSettings, environment, new SimilarityService(indexSettings, similarities), shardStoreDeleter,
-            analysisRegistry, engineFactory.get(), servicesProvider, queryCache, store, eventListener, searcherWrapperFactory,
-            mapperRegistry, indicesFieldDataCache, searchOperationListeners, indexOperationListeners);
+        return new IndexService(indexSettings, environment, xContentRegistry, new SimilarityService(indexSettings, similarities),
+                shardStoreDeleter, analysisRegistry, engineFactory.get(), circuitBreakerService, bigArrays, threadPool, scriptService,
+                clusterService, client, queryCache, store, eventListener, searcherWrapperFactory, mapperRegistry,
+                indicesFieldDataCache, searchOperationListeners, indexOperationListeners);
+    }
+
+    /**
+     * creates a new mapper service to do administrative work like mapping updates. This *should not* be used for document parsing.
+     * doing so will result in an exception.
+     */
+    public MapperService newIndexMapperService(NamedXContentRegistry xContentRegistry, MapperRegistry mapperRegistry) throws IOException {
+        return new MapperService(indexSettings, analysisRegistry.build(indexSettings), xContentRegistry,
+            new SimilarityService(indexSettings, similarities), mapperRegistry,
+            () -> { throw new UnsupportedOperationException("no index query shard context available"); });
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/index/IndexNotFoundException.java b/core/src/main/java/org/elasticsearch/index/IndexNotFoundException.java
index 035b90dd25e15..4442ee276c9cf 100644
--- a/core/src/main/java/org/elasticsearch/index/IndexNotFoundException.java
+++ b/core/src/main/java/org/elasticsearch/index/IndexNotFoundException.java
@@ -24,9 +24,16 @@
 import java.io.IOException;
 
 public final class IndexNotFoundException extends ResourceNotFoundException {
+    /**
+     * Construct with a custom message.
+     */
+    public IndexNotFoundException(String message, String index) {
+        super(message);
+        setIndex(index);
+    }
 
     public IndexNotFoundException(String index) {
-        this(index, null);
+        this(index, (Throwable) null);
     }
 
     public IndexNotFoundException(String index, Throwable cause) {
diff --git a/core/src/main/java/org/elasticsearch/index/IndexService.java b/core/src/main/java/org/elasticsearch/index/IndexService.java
index e662e46c79de6..a4b16efe1f641 100644
--- a/core/src/main/java/org/elasticsearch/index/IndexService.java
+++ b/core/src/main/java/org/elasticsearch/index/IndexService.java
@@ -22,42 +22,37 @@
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
 import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.Query;
 import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.util.Accountable;
 import org.apache.lucene.util.IOUtils;
-import org.elasticsearch.cluster.metadata.AliasMetaData;
+import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.FutureUtils;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.env.ShardLock;
 import org.elasticsearch.env.ShardLockObtainFailedException;
 import org.elasticsearch.index.analysis.AnalysisRegistry;
-import org.elasticsearch.index.analysis.AnalysisService;
+import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.cache.IndexCache;
 import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
 import org.elasticsearch.index.cache.query.QueryCache;
 import org.elasticsearch.index.engine.Engine;
-import org.elasticsearch.index.engine.EngineClosedException;
 import org.elasticsearch.index.engine.EngineFactory;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.shard.IndexEventListener;
 import org.elasticsearch.index.shard.IndexSearcherWrapper;
 import org.elasticsearch.index.shard.IndexShard;
+import org.elasticsearch.index.shard.IndexShardClosedException;
 import org.elasticsearch.index.shard.IndexingOperationListener;
 import org.elasticsearch.index.shard.SearchOperationListener;
 import org.elasticsearch.index.shard.ShadowIndexShard;
@@ -68,11 +63,11 @@
 import org.elasticsearch.index.store.IndexStore;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.index.translog.Translog;
-import org.elasticsearch.indices.AliasFilterParsingException;
-import org.elasticsearch.indices.InvalidAliasNameException;
+import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.cluster.IndicesClusterStateService;
 import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
 import org.elasticsearch.indices.mapper.MapperRegistry;
+import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.Closeable;
@@ -84,11 +79,11 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.LongSupplier;
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.unmodifiableMap;
@@ -97,16 +92,15 @@
 public class IndexService extends AbstractIndexComponent implements IndicesClusterStateService.AllocatedIndex<IndexShard> {
 
     private final IndexEventListener eventListener;
-    private final AnalysisService analysisService;
     private final IndexFieldDataService indexFieldData;
     private final BitsetFilterCache bitsetFilterCache;
     private final NodeEnvironment nodeEnv;
     private final ShardStoreDeleter shardStoreDeleter;
-    private final NodeServicesProvider nodeServicesProvider;
     private final IndexStore indexStore;
     private final IndexSearcherWrapper searcherWrapper;
     private final IndexCache indexCache;
     private final MapperService mapperService;
+    private final NamedXContentRegistry xContentRegistry;
     private final SimilarityService similarityService;
     private final EngineFactory engineFactory;
     private final IndexWarmer warmer;
@@ -120,13 +114,22 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
     private volatile AsyncTranslogFSync fsyncTask;
     private final ThreadPool threadPool;
     private final BigArrays bigArrays;
+    private final ScriptService scriptService;
+    private final ClusterService clusterService;
+    private final Client client;
 
     public IndexService(IndexSettings indexSettings, NodeEnvironment nodeEnv,
+                        NamedXContentRegistry xContentRegistry,
                         SimilarityService similarityService,
                         ShardStoreDeleter shardStoreDeleter,
                         AnalysisRegistry registry,
                         @Nullable EngineFactory engineFactory,
-                        NodeServicesProvider nodeServicesProvider,
+                        CircuitBreakerService circuitBreakerService,
+                        BigArrays bigArrays,
+                        ThreadPool threadPool,
+                        ScriptService scriptService,
+                        ClusterService clusterService,
+                        Client client,
                         QueryCache queryCache,
                         IndexStore indexStore,
                         IndexEventListener eventListener,
@@ -137,18 +140,21 @@ public IndexService(IndexSettings indexSettings, NodeEnvironment nodeEnv,
                         List<IndexingOperationListener> indexingOperationListeners) throws IOException {
         super(indexSettings);
         this.indexSettings = indexSettings;
-        this.analysisService = registry.build(indexSettings);
+        this.xContentRegistry = xContentRegistry;
         this.similarityService = similarityService;
-        this.mapperService = new MapperService(indexSettings, analysisService, similarityService, mapperRegistry,
-            IndexService.this::newQueryShardContext);
-        this.indexFieldData = new IndexFieldDataService(indexSettings, indicesFieldDataCache,
-            nodeServicesProvider.getCircuitBreakerService(), mapperService);
+        this.mapperService = new MapperService(indexSettings, registry.build(indexSettings), xContentRegistry, similarityService,
+            mapperRegistry,
+            // we parse all percolator queries as they would be parsed on shard 0
+            () -> newQueryShardContext(0, null, System::currentTimeMillis));
+        this.indexFieldData = new IndexFieldDataService(indexSettings, indicesFieldDataCache, circuitBreakerService, mapperService);
         this.shardStoreDeleter = shardStoreDeleter;
-        this.bigArrays = nodeServicesProvider.getBigArrays();
-        this.threadPool = nodeServicesProvider.getThreadPool();
+        this.bigArrays = bigArrays;
+        this.threadPool = threadPool;
+        this.scriptService = scriptService;
+        this.clusterService = clusterService;
+        this.client = client;
         this.eventListener = eventListener;
         this.nodeEnv = nodeEnv;
-        this.nodeServicesProvider = nodeServicesProvider;
         this.indexStore = indexStore;
         indexFieldData.setListener(new FieldDataCacheListener(this));
         this.bitsetFilterCache = new BitsetFilterCache(indexSettings, new BitsetCacheListener(this));
@@ -214,14 +220,18 @@ public IndexFieldDataService fieldData() {
         return indexFieldData;
     }
 
-    public AnalysisService analysisService() {
-        return this.analysisService;
+    public IndexAnalyzers getIndexAnalyzers() {
+        return this.mapperService.getIndexAnalyzers();
     }
 
     public MapperService mapperService() {
         return mapperService;
     }
 
+    public NamedXContentRegistry xContentRegistry() {
+        return xContentRegistry;
+    }
+
     public SimilarityService similarityService() {
         return similarityService;
     }
@@ -239,7 +249,7 @@ public synchronized void close(final String reason, boolean delete) throws IOExc
                     }
                 }
             } finally {
-                IOUtils.close(bitsetFilterCache, indexCache, indexFieldData, analysisService, refreshTask, fsyncTask);
+                IOUtils.close(bitsetFilterCache, indexCache, indexFieldData, mapperService, refreshTask, fsyncTask);
             }
         }
     }
@@ -320,13 +330,13 @@ public synchronized IndexShard createShard(ShardRouting routing) throws IOExcept
             }
 
             if (shards.containsKey(shardId.id())) {
-                throw new IndexShardAlreadyExistsException(shardId + " already exists");
+                throw new IllegalStateException(shardId + " already exists");
             }
 
             logger.debug("creating shard_id {}", shardId);
             // if we are on a shared FS we only own the shard (ie. we can safely delete it) if we are the primary.
-            final boolean canDeleteShardContent = IndexMetaData.isOnSharedFilesystem(indexSettings) == false ||
-                (primary && IndexMetaData.isOnSharedFilesystem(indexSettings));
+            final boolean canDeleteShardContent = this.indexSettings.isOnSharedFilesystem() == false ||
+                (primary && this.indexSettings.isOnSharedFilesystem());
             final Engine.Warmer engineWarmer = (searcher) -> {
                 IndexShard shard =  getShardOrNull(shardId.getId());
                 if (shard != null) {
@@ -335,7 +345,7 @@ public synchronized IndexShard createShard(ShardRouting routing) throws IOExcept
             };
             store = new Store(shardId, this.indexSettings, indexStore.newDirectoryService(path), lock,
                 new StoreCloseListener(shardId, canDeleteShardContent, () -> eventListener.onStoreClosed(shardId)));
-            if (useShadowEngine(primary, indexSettings)) {
+            if (useShadowEngine(primary, this.indexSettings)) {
                 indexShard = new ShadowIndexShard(routing, this.indexSettings, path, store, indexCache, mapperService, similarityService,
                     indexFieldData, engineFactory, eventListener, searcherWrapper, threadPool, bigArrays, engineWarmer,
                     searchOperationListeners);
@@ -362,8 +372,8 @@ public synchronized IndexShard createShard(ShardRouting routing) throws IOExcept
         }
     }
 
-    static boolean useShadowEngine(boolean primary, Settings indexSettings) {
-        return primary == false && IndexMetaData.isIndexUsingShadowReplicas(indexSettings);
+    static boolean useShadowEngine(boolean primary, IndexSettings indexSettings) {
+        return primary == false && indexSettings.isShadowReplicaIndex();
     }
 
     @Override
@@ -405,7 +415,11 @@ private void closeShard(String reason, ShardId sId, IndexShard indexShard, Store
             }
         } finally {
             try {
-                store.close();
+                if (store != null) {
+                    store.close();
+                } else {
+                    logger.trace("[{}] store not initialized prior to closing shard, nothing to close", shardId);
+                }
             } catch (Exception e) {
                 logger.warn(
                     (Supplier<?>) () -> new ParameterizedMessage(
@@ -435,10 +449,6 @@ private void onShardClose(ShardLock lock, boolean ownsShard) {
         }
     }
 
-    public NodeServicesProvider getIndexServices() {
-        return nodeServicesProvider;
-    }
-
     @Override
     public IndexSettings getIndexSettings() {
         return indexSettings;
@@ -446,34 +456,40 @@ public IndexSettings getIndexSettings() {
 
     /**
      * Creates a new QueryShardContext. The context has not types set yet, if types are required set them via
-     * {@link QueryShardContext#setTypes(String...)}
+     * {@link QueryShardContext#setTypes(String...)}.
+     *
+     * Passing a {@code null} {@link IndexReader} will return a valid context, however it won't be able to make
+     * {@link IndexReader}-specific optimizations, such as rewriting containing range queries.
      */
-    public QueryShardContext newQueryShardContext(IndexReader indexReader) {
+    public QueryShardContext newQueryShardContext(int shardId, IndexReader indexReader, LongSupplier nowInMillis) {
         return new QueryShardContext(
-                indexSettings, indexCache.bitsetFilterCache(), indexFieldData, mapperService(),
-                similarityService(), nodeServicesProvider.getScriptService(), nodeServicesProvider.getIndicesQueriesRegistry(),
-                nodeServicesProvider.getClient(), indexReader,
-                nodeServicesProvider.getClusterService().state()
-        );
+            shardId, indexSettings, indexCache.bitsetFilterCache(), indexFieldData, mapperService(),
+                similarityService(), scriptService, xContentRegistry,
+                client, indexReader,
+            nowInMillis);
     }
 
     /**
-     * Creates a new QueryShardContext. The context has not types set yet, if types are required set them via
-     * {@link QueryShardContext#setTypes(String...)}. This context may be used for query parsing but cannot be
-     * used for rewriting since it does not know about the current {@link IndexReader}.
+     * The {@link ThreadPool} to use for this index.
      */
-    public QueryShardContext newQueryShardContext() {
-        return newQueryShardContext(null);
-    }
-
     public ThreadPool getThreadPool() {
         return threadPool;
     }
 
+    /**
+     * The {@link BigArrays} to use for this index.
+     */
     public BigArrays getBigArrays() {
         return bigArrays;
     }
 
+    /**
+     * The {@link ScriptService} to use for this index.
+     */
+    public ScriptService getScriptService() {
+        return scriptService;
+    }
+
     List<IndexingOperationListener> getIndexOperationListeners() { // pkg private for testing
         return indexingOperationListeners;
     }
@@ -492,14 +508,14 @@ private class StoreCloseListener implements Store.OnClose {
         private final boolean ownsShard;
         private final Closeable[] toClose;
 
-        public StoreCloseListener(ShardId shardId, boolean ownsShard, Closeable... toClose) {
+        StoreCloseListener(ShardId shardId, boolean ownsShard, Closeable... toClose) {
             this.shardId = shardId;
             this.ownsShard = ownsShard;
             this.toClose = toClose;
         }
 
         @Override
-        public void handle(ShardLock lock) {
+        public void accept(ShardLock lock) {
             try {
                 assert lock.getShardId().equals(shardId) : "shard id mismatch, expected: " + shardId + " but got: " + lock.getShardId();
                 onShardClose(lock, ownsShard);
@@ -547,7 +563,7 @@ public void onRemoval(ShardId shardId, Accountable accountable) {
     private final class FieldDataCacheListener implements IndexFieldDataCache.Listener {
         final IndexService indexService;
 
-        public FieldDataCacheListener(IndexService indexService) {
+        FieldDataCacheListener(IndexService indexService) {
             this.indexService = indexService;
         }
 
@@ -572,64 +588,6 @@ public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, lon
         }
     }
 
-    /**
-     * Returns the filter associated with listed filtering aliases.
-     * <p>
-     * The list of filtering aliases should be obtained by calling MetaData.filteringAliases.
-     * Returns <tt>null</tt> if no filtering is required.</p>
-     */
-    public Query aliasFilter(QueryShardContext context, String... aliasNames) {
-        if (aliasNames == null || aliasNames.length == 0) {
-            return null;
-        }
-        final ImmutableOpenMap<String, AliasMetaData> aliases = indexSettings.getIndexMetaData().getAliases();
-        if (aliasNames.length == 1) {
-            AliasMetaData alias = aliases.get(aliasNames[0]);
-            if (alias == null) {
-                // This shouldn't happen unless alias disappeared after filteringAliases was called.
-                throw new InvalidAliasNameException(index(), aliasNames[0], "Unknown alias name was passed to alias Filter");
-            }
-            return parse(alias, context);
-        } else {
-            // we need to bench here a bit, to see maybe it makes sense to use OrFilter
-            BooleanQuery.Builder combined = new BooleanQuery.Builder();
-            for (String aliasName : aliasNames) {
-                AliasMetaData alias = aliases.get(aliasName);
-                if (alias == null) {
-                    // This shouldn't happen unless alias disappeared after filteringAliases was called.
-                    throw new InvalidAliasNameException(indexSettings.getIndex(), aliasNames[0],
-                        "Unknown alias name was passed to alias Filter");
-                }
-                Query parsedFilter = parse(alias, context);
-                if (parsedFilter != null) {
-                    combined.add(parsedFilter, BooleanClause.Occur.SHOULD);
-                } else {
-                    // The filter might be null only if filter was removed after filteringAliases was called
-                    return null;
-                }
-            }
-            return combined.build();
-        }
-    }
-
-    private Query parse(AliasMetaData alias, QueryShardContext shardContext) {
-        if (alias.filter() == null) {
-            return null;
-        }
-        try {
-            byte[] filterSource = alias.filter().uncompressed();
-            try (XContentParser parser = XContentFactory.xContent(filterSource).createParser(filterSource)) {
-                Optional<QueryBuilder> innerQueryBuilder = shardContext.newParseContext(parser).parseInnerQueryBuilder();
-                if (innerQueryBuilder.isPresent()) {
-                    return shardContext.toFilter(innerQueryBuilder.get()).query();
-                }
-                return null;
-            }
-        } catch (IOException ex) {
-            throw new AliasFilterParsingException(shardContext.index(), alias.getAlias(), "Invalid alias filter", ex);
-        }
-    }
-
     public IndexMetaData getMetaData() {
         return indexSettings.getIndexMetaData();
     }
@@ -707,7 +665,7 @@ private void maybeFSyncTranslogs() {
                     if (translog.syncNeeded()) {
                         translog.sync();
                     }
-                } catch (EngineClosedException | AlreadyClosedException ex) {
+                } catch (AlreadyClosedException ex) {
                     // fine - continue;
                 } catch (IOException e) {
                     logger.warn("failed to sync translog", e);
@@ -731,7 +689,7 @@ private void maybeRefreshEngine() {
                             if (shard.isRefreshNeeded()) {
                                 shard.refresh("schedule");
                             }
-                        } catch (EngineClosedException | AlreadyClosedException ex) {
+                        } catch (IndexShardClosedException | AlreadyClosedException ex) {
                             // fine - continue;
                         }
                         continue;
diff --git a/core/src/main/java/org/elasticsearch/index/IndexSettings.java b/core/src/main/java/org/elasticsearch/index/IndexSettings.java
index 5666fb416f0c5..ea484dff3ae65 100644
--- a/core/src/main/java/org/elasticsearch/index/IndexSettings.java
+++ b/core/src/main/java/org/elasticsearch/index/IndexSettings.java
@@ -22,7 +22,6 @@
 import org.apache.lucene.index.MergePolicy;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.IndexScopedSettings;
@@ -33,12 +32,12 @@
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.mapper.AllFieldMapper;
+import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.node.Node;
 
 import java.util.Locale;
 import java.util.concurrent.TimeUnit;
-import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.function.Predicate;
 
@@ -99,6 +98,13 @@ public final class IndexSettings {
      */
     public static final Setting<Integer> MAX_RESCORE_WINDOW_SETTING =
             Setting.intSetting("index.max_rescore_window", MAX_RESULT_WINDOW_SETTING, 1, Property.Dynamic, Property.IndexScope);
+    /**
+     * Index setting describing the maximum number of filters clauses that can be used
+     * in an adjacency_matrix aggregation. The max number of buckets produced by  
+     * N filters is (N*N)/2 so a limit of 100 filters is imposed by default.
+     */
+    public static final Setting<Integer> MAX_ADJACENCY_MATRIX_FILTERS_SETTING =
+        Setting.intSetting("index.max_adjacency_matrix_filters", 100, 2, Property.Dynamic, Property.IndexScope);    
     public static final TimeValue DEFAULT_REFRESH_INTERVAL = new TimeValue(1, TimeUnit.SECONDS);
     public static final Setting<TimeValue> INDEX_REFRESH_INTERVAL_SETTING =
         Setting.timeSetting("index.refresh_interval", DEFAULT_REFRESH_INTERVAL, new TimeValue(-1, TimeUnit.MILLISECONDS),
@@ -135,7 +141,6 @@ public final class IndexSettings {
     private final Settings nodeSettings;
     private final int numberOfShards;
     private final boolean isShadowReplicaIndex;
-    private final ParseFieldMatcher parseFieldMatcher;
     // volatile fields are updated via #updateIndexMetaData(IndexMetaData) under lock
     private volatile Settings settings;
     private volatile IndexMetaData indexMetaData;
@@ -155,6 +160,7 @@ public final class IndexSettings {
     private long gcDeletesInMillis = DEFAULT_GC_DELETES.millis();
     private volatile boolean warmerEnabled;
     private volatile int maxResultWindow;
+    private volatile int maxAdjacencyMatrixFilters;
     private volatile int maxRescoreWindow;
     private volatile boolean TTLPurgeDisabled;
     /**
@@ -165,7 +171,10 @@ public final class IndexSettings {
      * The maximum number of slices allowed in a scroll request.
      */
     private volatile int maxSlicesPerScroll;
-
+    /**
+     * Whether the index is required to have at most one type.
+     */
+    private final boolean singleType;
 
     /**
      * Returns the default search field for this index.
@@ -237,7 +246,6 @@ public IndexSettings(final IndexMetaData indexMetaData, final Settings nodeSetti
         this.queryStringLenient = QUERY_STRING_LENIENT_SETTING.get(settings);
         this.queryStringAnalyzeWildcard = QUERY_STRING_ANALYZE_WILDCARD.get(nodeSettings);
         this.queryStringAllowLeadingWildcard = QUERY_STRING_ALLOW_LEADING_WILDCARD.get(nodeSettings);
-        this.parseFieldMatcher = new ParseFieldMatcher(settings);
         this.defaultAllowUnmappedFields = scopedSettings.get(ALLOW_UNMAPPED);
         this.indexNameMatcher = indexNameMatcher;
         this.durability = scopedSettings.get(INDEX_TRANSLOG_DURABILITY_SETTING);
@@ -248,12 +256,13 @@ public IndexSettings(final IndexMetaData indexMetaData, final Settings nodeSetti
         gcDeletesInMillis = scopedSettings.get(INDEX_GC_DELETES_SETTING).getMillis();
         warmerEnabled = scopedSettings.get(INDEX_WARMER_ENABLED_SETTING);
         maxResultWindow = scopedSettings.get(MAX_RESULT_WINDOW_SETTING);
+        maxAdjacencyMatrixFilters = scopedSettings.get(MAX_ADJACENCY_MATRIX_FILTERS_SETTING);
         maxRescoreWindow = scopedSettings.get(MAX_RESCORE_WINDOW_SETTING);
         TTLPurgeDisabled = scopedSettings.get(INDEX_TTL_DISABLE_PURGE_SETTING);
         maxRefreshListeners = scopedSettings.get(MAX_REFRESH_LISTENERS_PER_SHARD);
         maxSlicesPerScroll = scopedSettings.get(MAX_SLICES_PER_SCROLL);
         this.mergePolicyConfig = new MergePolicyConfig(logger, this);
-        assert indexNameMatcher.test(indexMetaData.getIndex().getName());
+        singleType = scopedSettings.get(MapperService.INDEX_MAPPING_SINGLE_TYPE_SETTING);
 
         scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING, mergePolicyConfig::setNoCFSRatio);
         scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_MERGE_POLICY_EXPUNGE_DELETES_ALLOWED_SETTING, mergePolicyConfig::setExpungeDeletesAllowed);
@@ -263,12 +272,14 @@ public IndexSettings(final IndexMetaData indexMetaData, final Settings nodeSetti
         scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGED_SEGMENT_SETTING, mergePolicyConfig::setMaxMergedSegment);
         scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING, mergePolicyConfig::setSegmentsPerTier);
         scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_MERGE_POLICY_RECLAIM_DELETES_WEIGHT_SETTING, mergePolicyConfig::setReclaimDeletesWeight);
-        scopedSettings.addSettingsUpdateConsumer(MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING, mergeSchedulerConfig::setMaxThreadCount);
-        scopedSettings.addSettingsUpdateConsumer(MergeSchedulerConfig.MAX_MERGE_COUNT_SETTING, mergeSchedulerConfig::setMaxMergeCount);
+
+        scopedSettings.addSettingsUpdateConsumer(MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING, MergeSchedulerConfig.MAX_MERGE_COUNT_SETTING,
+            mergeSchedulerConfig::setMaxThreadAndMergeCount);
         scopedSettings.addSettingsUpdateConsumer(MergeSchedulerConfig.AUTO_THROTTLE_SETTING, mergeSchedulerConfig::setAutoThrottle);
         scopedSettings.addSettingsUpdateConsumer(INDEX_TRANSLOG_DURABILITY_SETTING, this::setTranslogDurability);
         scopedSettings.addSettingsUpdateConsumer(INDEX_TTL_DISABLE_PURGE_SETTING, this::setTTLPurgeDisabled);
         scopedSettings.addSettingsUpdateConsumer(MAX_RESULT_WINDOW_SETTING, this::setMaxResultWindow);
+        scopedSettings.addSettingsUpdateConsumer(MAX_ADJACENCY_MATRIX_FILTERS_SETTING, this::setMaxAdjacencyMatrixFilters);
         scopedSettings.addSettingsUpdateConsumer(MAX_RESCORE_WINDOW_SETTING, this::setMaxRescoreWindow);
         scopedSettings.addSettingsUpdateConsumer(INDEX_WARMER_ENABLED_SETTING, this::setEnableWarmer);
         scopedSettings.addSettingsUpdateConsumer(INDEX_GC_DELETES_SETTING, this::setGCDeletes);
@@ -334,15 +345,6 @@ public boolean isOnSharedFilesystem() {
         return IndexMetaData.isOnSharedFilesystem(getSettings());
     }
 
-    /**
-     * Returns <code>true</code> iff the given settings indicate that the index associated
-     * with these settings uses shadow replicas. Otherwise <code>false</code>. The default
-     * setting for this is <code>false</code>.
-     */
-    public boolean isIndexUsingShadowReplicas() {
-        return IndexMetaData.isOnSharedFilesystem(getSettings());
-    }
-
     /**
      * Returns the version the index was created on.
      * @see Version#indexCreated(Settings)
@@ -381,6 +383,11 @@ public IndexMetaData getIndexMetaData() {
      */
     public boolean isShadowReplicaIndex() { return isShadowReplicaIndex; }
 
+    /**
+     * Returns whether the index enforces at most one type.
+     */
+    public boolean isSingleType() { return singleType; }
+
     /**
      * Returns the node settings. The settings returned from {@link #getSettings()} are a merged version of the
      * index settings and the node settings where node settings are overwritten by index settings.
@@ -389,11 +396,6 @@ public Settings getNodeSettings() {
         return nodeSettings;
     }
 
-    /**
-     * Returns a {@link ParseFieldMatcher} for this index.
-     */
-    public ParseFieldMatcher getParseFieldMatcher() { return parseFieldMatcher; }
-
     /**
      * Returns <code>true</code> if the given expression matches the index name or one of it's aliases
      */
@@ -483,6 +485,17 @@ public int getMaxResultWindow() {
     private void setMaxResultWindow(int maxResultWindow) {
         this.maxResultWindow = maxResultWindow;
     }
+    
+    /**
+     * Returns the max number of filters in adjacency_matrix aggregation search requests
+     */
+    public int getMaxAdjacencyMatrixFilters() {
+        return this.maxAdjacencyMatrixFilters;
+    }
+
+    private void setMaxAdjacencyMatrixFilters(int maxAdjacencyFilters) {
+        this.maxAdjacencyMatrixFilters = maxAdjacencyFilters;
+    }    
 
     /**
      * Returns the maximum rescore window for search requests.
diff --git a/core/src/main/java/org/elasticsearch/index/IndexShardAlreadyExistsException.java b/core/src/main/java/org/elasticsearch/index/IndexShardAlreadyExistsException.java
index 564988d05946f..d83b56f8b32a9 100644
--- a/core/src/main/java/org/elasticsearch/index/IndexShardAlreadyExistsException.java
+++ b/core/src/main/java/org/elasticsearch/index/IndexShardAlreadyExistsException.java
@@ -26,7 +26,9 @@
 
 /**
  *
+ * @deprecated use {@link IllegalStateException} instead
  */
+@Deprecated
 public class IndexShardAlreadyExistsException extends ElasticsearchException {
 
     public IndexShardAlreadyExistsException(String message) {
@@ -36,4 +38,4 @@ public IndexShardAlreadyExistsException(String message) {
     public IndexShardAlreadyExistsException(StreamInput in) throws IOException {
         super(in);
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/index/IndexWarmer.java b/core/src/main/java/org/elasticsearch/index/IndexWarmer.java
index 439acb239a359..a2e57e83ed5af 100644
--- a/core/src/main/java/org/elasticsearch/index/IndexWarmer.java
+++ b/core/src/main/java/org/elasticsearch/index/IndexWarmer.java
@@ -56,9 +56,8 @@ public final class IndexWarmer extends AbstractComponent {
         ArrayList<Listener> list = new ArrayList<>();
         final Executor executor = threadPool.executor(ThreadPool.Names.WARMER);
         list.add(new FieldDataWarmer(executor));
-        for (Listener listener : listeners) {
-            list.add(listener);
-        }
+
+        Collections.addAll(list, listeners);
         this.listeners = Collections.unmodifiableList(list);
     }
 
@@ -113,7 +112,7 @@ public interface Listener {
     private static class FieldDataWarmer implements IndexWarmer.Listener {
 
         private final Executor executor;
-        public FieldDataWarmer(Executor executor) {
+        FieldDataWarmer(Executor executor) {
             this.executor = executor;
         }
 
diff --git a/core/src/main/java/org/elasticsearch/index/IndexingSlowLog.java b/core/src/main/java/org/elasticsearch/index/IndexingSlowLog.java
index 513e87878d6b8..f7f7334f6b946 100644
--- a/core/src/main/java/org/elasticsearch/index/IndexingSlowLog.java
+++ b/core/src/main/java/org/elasticsearch/index/IndexingSlowLog.java
@@ -22,6 +22,7 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
@@ -30,11 +31,14 @@
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.shard.IndexingOperationListener;
+import org.elasticsearch.index.shard.ShardId;
 
 import java.io.IOException;
 import java.util.concurrent.TimeUnit;
 
 public final class IndexingSlowLog implements IndexingOperationListener {
+    private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(IndexingSlowLog.class));
+
     private final Index index;
     private boolean reformat;
     private long indexWarnThreshold;
@@ -76,13 +80,19 @@ public final class IndexingSlowLog implements IndexingOperationListener {
      * and everything else is interpreted as Elasticsearch interprets booleans
      * which is then converted to 0 for false and Integer.MAX_VALUE for true.
      */
-    public static final Setting<Integer> INDEX_INDEXING_SLOWLOG_MAX_SOURCE_CHARS_TO_LOG_SETTING = new Setting<>(INDEX_INDEXING_SLOWLOG_PREFIX + ".source", "1000", (value) -> {
-        try {
-            return Integer.parseInt(value, 10);
-        } catch (NumberFormatException e) {
-            return Booleans.parseBoolean(value, true) ? Integer.MAX_VALUE : 0;
-        }
-    }, Property.Dynamic, Property.IndexScope);
+    public static final Setting<Integer> INDEX_INDEXING_SLOWLOG_MAX_SOURCE_CHARS_TO_LOG_SETTING =
+            new Setting<>(INDEX_INDEXING_SLOWLOG_PREFIX + ".source", "1000", (value) -> {
+                try {
+                    return Integer.parseInt(value, 10);
+                } catch (NumberFormatException e) {
+                    boolean booleanValue = Booleans.parseBoolean(value, true);
+                    if (value != null && Booleans.isStrictlyBoolean(value) == false) {
+                        DEPRECATION_LOGGER.deprecated("Expected a boolean [true/false] or a number for setting [{}] but got [{}]",
+                            INDEX_INDEXING_SLOWLOG_PREFIX + ".source", value);
+                    }
+                    return booleanValue ? Integer.MAX_VALUE : 0;
+                }
+            }, Property.Dynamic, Property.IndexScope);
 
     IndexingSlowLog(IndexSettings indexSettings) {
         this.indexLogger = Loggers.getLogger(INDEX_INDEXING_SLOWLOG_PREFIX + ".index", indexSettings.getSettings());
@@ -90,17 +100,22 @@ public final class IndexingSlowLog implements IndexingOperationListener {
 
         indexSettings.getScopedSettings().addSettingsUpdateConsumer(INDEX_INDEXING_SLOWLOG_REFORMAT_SETTING, this::setReformat);
         this.reformat = indexSettings.getValue(INDEX_INDEXING_SLOWLOG_REFORMAT_SETTING);
-        indexSettings.getScopedSettings().addSettingsUpdateConsumer(INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_WARN_SETTING, this::setWarnThreshold);
+        indexSettings.getScopedSettings()
+                .addSettingsUpdateConsumer(INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_WARN_SETTING, this::setWarnThreshold);
         this.indexWarnThreshold = indexSettings.getValue(INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_WARN_SETTING).nanos();
-        indexSettings.getScopedSettings().addSettingsUpdateConsumer(INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_INFO_SETTING, this::setInfoThreshold);
+        indexSettings.getScopedSettings()
+                .addSettingsUpdateConsumer(INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_INFO_SETTING, this::setInfoThreshold);
         this.indexInfoThreshold = indexSettings.getValue(INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_INFO_SETTING).nanos();
-        indexSettings.getScopedSettings().addSettingsUpdateConsumer(INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_DEBUG_SETTING, this::setDebugThreshold);
+        indexSettings.getScopedSettings()
+                .addSettingsUpdateConsumer(INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_DEBUG_SETTING, this::setDebugThreshold);
         this.indexDebugThreshold = indexSettings.getValue(INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_DEBUG_SETTING).nanos();
-        indexSettings.getScopedSettings().addSettingsUpdateConsumer(INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_TRACE_SETTING, this::setTraceThreshold);
+        indexSettings.getScopedSettings()
+                .addSettingsUpdateConsumer(INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_TRACE_SETTING, this::setTraceThreshold);
         this.indexTraceThreshold = indexSettings.getValue(INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_TRACE_SETTING).nanos();
         indexSettings.getScopedSettings().addSettingsUpdateConsumer(INDEX_INDEXING_SLOWLOG_LEVEL_SETTING, this::setLevel);
         setLevel(indexSettings.getValue(INDEX_INDEXING_SLOWLOG_LEVEL_SETTING));
-        indexSettings.getScopedSettings().addSettingsUpdateConsumer(INDEX_INDEXING_SLOWLOG_MAX_SOURCE_CHARS_TO_LOG_SETTING, this::setMaxSourceCharsToLog);
+        indexSettings.getScopedSettings().addSettingsUpdateConsumer(INDEX_INDEXING_SLOWLOG_MAX_SOURCE_CHARS_TO_LOG_SETTING,
+                this::setMaxSourceCharsToLog);
         this.maxSourceCharsToLog = indexSettings.getValue(INDEX_INDEXING_SLOWLOG_MAX_SOURCE_CHARS_TO_LOG_SETTING);
     }
 
@@ -133,23 +148,20 @@ private void setReformat(boolean reformat) {
         this.reformat = reformat;
     }
 
-
     @Override
-    public void postIndex(Engine.Index index, boolean created) {
-        final long took = index.endTime() - index.startTime();
-        postIndexing(index.parsedDoc(), took);
-    }
-
-
-    private void postIndexing(ParsedDocument doc, long tookInNanos) {
-        if (indexWarnThreshold >= 0 && tookInNanos > indexWarnThreshold) {
-            indexLogger.warn("{}", new SlowLogParsedDocumentPrinter(index, doc, tookInNanos, reformat, maxSourceCharsToLog));
-        } else if (indexInfoThreshold >= 0 && tookInNanos > indexInfoThreshold) {
-            indexLogger.info("{}", new SlowLogParsedDocumentPrinter(index, doc, tookInNanos, reformat, maxSourceCharsToLog));
-        } else if (indexDebugThreshold >= 0 && tookInNanos > indexDebugThreshold) {
-            indexLogger.debug("{}", new SlowLogParsedDocumentPrinter(index, doc, tookInNanos, reformat, maxSourceCharsToLog));
-        } else if (indexTraceThreshold >= 0 && tookInNanos > indexTraceThreshold) {
-            indexLogger.trace("{}", new SlowLogParsedDocumentPrinter(index, doc, tookInNanos, reformat, maxSourceCharsToLog));
+    public void postIndex(ShardId shardId, Engine.Index indexOperation, Engine.IndexResult result) {
+        if (result.hasFailure() == false) {
+            final ParsedDocument doc = indexOperation.parsedDoc();
+            final long tookInNanos = result.getTook();
+            if (indexWarnThreshold >= 0 && tookInNanos > indexWarnThreshold) {
+                indexLogger.warn("{}", new SlowLogParsedDocumentPrinter(index, doc, tookInNanos, reformat, maxSourceCharsToLog));
+            } else if (indexInfoThreshold >= 0 && tookInNanos > indexInfoThreshold) {
+                indexLogger.info("{}", new SlowLogParsedDocumentPrinter(index, doc, tookInNanos, reformat, maxSourceCharsToLog));
+            } else if (indexDebugThreshold >= 0 && tookInNanos > indexDebugThreshold) {
+                indexLogger.debug("{}", new SlowLogParsedDocumentPrinter(index, doc, tookInNanos, reformat, maxSourceCharsToLog));
+            } else if (indexTraceThreshold >= 0 && tookInNanos > indexTraceThreshold) {
+                indexLogger.trace("{}", new SlowLogParsedDocumentPrinter(index, doc, tookInNanos, reformat, maxSourceCharsToLog));
+            }
         }
     }
 
@@ -172,7 +184,8 @@ static final class SlowLogParsedDocumentPrinter {
         public String toString() {
             StringBuilder sb = new StringBuilder();
             sb.append(index).append(" ");
-            sb.append("took[").append(TimeValue.timeValueNanos(tookInNanos)).append("], took_millis[").append(TimeUnit.NANOSECONDS.toMillis(tookInNanos)).append("], ");
+            sb.append("took[").append(TimeValue.timeValueNanos(tookInNanos)).append("], ");
+            sb.append("took_millis[").append(TimeUnit.NANOSECONDS.toMillis(tookInNanos)).append("], ");
             sb.append("type[").append(doc.type()).append("], ");
             sb.append("id[").append(doc.id()).append("], ");
             if (doc.routing() == null) {
@@ -185,7 +198,7 @@ public String toString() {
                 return sb.toString();
             }
             try {
-                String source = XContentHelper.convertToJson(doc.source(), reformat);
+                String source = XContentHelper.convertToJson(doc.source(), reformat, doc.getXContentType());
                 sb.append(", source[").append(Strings.cleanTruncate(source, maxSourceCharsToLog)).append("]");
             } catch (IOException e) {
                 sb.append(", source[_failed_to_convert_]");
diff --git a/core/src/main/java/org/elasticsearch/index/MergePolicyConfig.java b/core/src/main/java/org/elasticsearch/index/MergePolicyConfig.java
index 52b98e2bd01fa..bbc2f9d099ee4 100644
--- a/core/src/main/java/org/elasticsearch/index/MergePolicyConfig.java
+++ b/core/src/main/java/org/elasticsearch/index/MergePolicyConfig.java
@@ -171,10 +171,10 @@ public final class MergePolicyConfig {
         maxMergeAtOnce = adjustMaxMergeAtOnceIfNeeded(maxMergeAtOnce, segmentsPerTier);
         mergePolicy.setNoCFSRatio(indexSettings.getValue(INDEX_COMPOUND_FORMAT_SETTING));
         mergePolicy.setForceMergeDeletesPctAllowed(forceMergeDeletesPctAllowed);
-        mergePolicy.setFloorSegmentMB(floorSegment.mbFrac());
+        mergePolicy.setFloorSegmentMB(floorSegment.getMbFrac());
         mergePolicy.setMaxMergeAtOnce(maxMergeAtOnce);
         mergePolicy.setMaxMergeAtOnceExplicit(maxMergeAtOnceExplicit);
-        mergePolicy.setMaxMergedSegmentMB(maxMergedSegment.mbFrac());
+        mergePolicy.setMaxMergedSegmentMB(maxMergedSegment.getMbFrac());
         mergePolicy.setSegmentsPerTier(segmentsPerTier);
         mergePolicy.setReclaimDeletesWeight(reclaimDeletesWeight);
         if (logger.isTraceEnabled()) {
@@ -192,7 +192,7 @@ void setSegmentsPerTier(Double segmentsPerTier) {
     }
 
     void setMaxMergedSegment(ByteSizeValue maxMergedSegment) {
-        mergePolicy.setMaxMergedSegmentMB(maxMergedSegment.mbFrac());
+        mergePolicy.setMaxMergedSegmentMB(maxMergedSegment.getMbFrac());
     }
 
     void setMaxMergesAtOnceExplicit(Integer maxMergeAtOnceExplicit) {
@@ -204,7 +204,7 @@ void setMaxMergesAtOnce(Integer maxMergeAtOnce) {
     }
 
     void setFloorSegmentSetting(ByteSizeValue floorSegementSetting) {
-        mergePolicy.setFloorSegmentMB(floorSegementSetting.mbFrac());
+        mergePolicy.setFloorSegmentMB(floorSegementSetting.getMbFrac());
     }
 
     void setExpungeDeletesAllowed(Double value) {
@@ -247,7 +247,7 @@ private static double parseNoCFSRatio(String noCFSRatio) {
                 }
                 return value;
             } catch (NumberFormatException ex) {
-                throw new IllegalArgumentException("Expected a boolean or a value in the interval [0..1] but was: [" + noCFSRatio + "]", ex);
+                throw new IllegalArgumentException("Expected a boolean [true/false] or a value in the interval [0..1] but was: [" + noCFSRatio + "]", ex);
             }
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/index/MergeSchedulerConfig.java b/core/src/main/java/org/elasticsearch/index/MergeSchedulerConfig.java
index 2eb43a50ee47b..0487a802fd932 100644
--- a/core/src/main/java/org/elasticsearch/index/MergeSchedulerConfig.java
+++ b/core/src/main/java/org/elasticsearch/index/MergeSchedulerConfig.java
@@ -69,13 +69,15 @@ public final class MergeSchedulerConfig {
     private volatile int maxMergeCount;
 
     MergeSchedulerConfig(IndexSettings indexSettings) {
-        maxThreadCount = indexSettings.getValue(MAX_THREAD_COUNT_SETTING);
-        maxMergeCount = indexSettings.getValue(MAX_MERGE_COUNT_SETTING);
+        int maxThread = indexSettings.getValue(MAX_THREAD_COUNT_SETTING);
+        int maxMerge = indexSettings.getValue(MAX_MERGE_COUNT_SETTING);
+        setMaxThreadAndMergeCount(maxThread, maxMerge);
         this.autoThrottle = indexSettings.getValue(AUTO_THROTTLE_SETTING);
     }
 
     /**
      * Returns <code>true</code> iff auto throttle is enabled.
+     *
      * @see ConcurrentMergeScheduler#enableAutoIOThrottle()
      */
     public boolean isAutoThrottle() {
@@ -100,8 +102,19 @@ public int getMaxThreadCount() {
      * Expert: directly set the maximum number of merge threads and
      * simultaneous merges allowed.
      */
-    void setMaxThreadCount(int maxThreadCount) {
+    void setMaxThreadAndMergeCount(int maxThreadCount, int maxMergeCount) {
+        if (maxThreadCount < 1) {
+            throw new IllegalArgumentException("maxThreadCount should be at least 1");
+        }
+        if (maxMergeCount < 1) {
+            throw new IllegalArgumentException("maxMergeCount should be at least 1");
+        }
+        if (maxThreadCount > maxMergeCount) {
+            throw new IllegalArgumentException("maxThreadCount (= " + maxThreadCount +
+                ") should be <= maxMergeCount (= " + maxMergeCount + ")");
+        }
         this.maxThreadCount = maxThreadCount;
+        this.maxMergeCount = maxMergeCount;
     }
 
     /**
@@ -110,12 +123,4 @@ void setMaxThreadCount(int maxThreadCount) {
     public int getMaxMergeCount() {
         return maxMergeCount;
     }
-
-    /**
-     *
-     * Expert: set the maximum number of simultaneous merges allowed.
-     */
-    void setMaxMergeCount(int maxMergeCount) {
-        this.maxMergeCount = maxMergeCount;
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/NodeServicesProvider.java b/core/src/main/java/org/elasticsearch/index/NodeServicesProvider.java
deleted file mode 100644
index 866c938c0f5c1..0000000000000
--- a/core/src/main/java/org/elasticsearch/index/NodeServicesProvider.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index;
-
-import org.elasticsearch.client.Client;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.util.BigArrays;
-import org.elasticsearch.indices.breaker.CircuitBreakerService;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
-import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.threadpool.ThreadPool;
-
-/**
- * Simple provider class that holds the Index and Node level services used by
- * a shard.
- * This is just a temporary solution until we cleaned up index creation and removed injectors on that level as well.
- */
-public final class NodeServicesProvider {
-
-    private final ThreadPool threadPool;
-    private final BigArrays bigArrays;
-    private final Client client;
-    private final IndicesQueriesRegistry indicesQueriesRegistry;
-    private final ScriptService scriptService;
-    private final CircuitBreakerService circuitBreakerService;
-    private final ClusterService clusterService;
-
-    @Inject
-    public NodeServicesProvider(ThreadPool threadPool, BigArrays bigArrays, Client client, ScriptService scriptService,
-                                IndicesQueriesRegistry indicesQueriesRegistry, CircuitBreakerService circuitBreakerService,
-                                ClusterService clusterService) {
-        this.threadPool = threadPool;
-        this.bigArrays = bigArrays;
-        this.client = client;
-        this.indicesQueriesRegistry = indicesQueriesRegistry;
-        this.scriptService = scriptService;
-        this.circuitBreakerService = circuitBreakerService;
-        this.clusterService = clusterService;
-    }
-
-    public ThreadPool getThreadPool() {
-        return threadPool;
-    }
-
-    public BigArrays getBigArrays() { return bigArrays; }
-
-    public Client getClient() {
-        return client;
-    }
-
-    public IndicesQueriesRegistry getIndicesQueriesRegistry() {
-        return indicesQueriesRegistry;
-    }
-
-    public ScriptService getScriptService() {
-        return scriptService;
-    }
-
-    public CircuitBreakerService getCircuitBreakerService() {
-        return circuitBreakerService;
-    }
-
-    public ClusterService getClusterService() {
-        return clusterService;
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/index/SearchSlowLog.java b/core/src/main/java/org/elasticsearch/index/SearchSlowLog.java
index 19086416b805e..a48e3d7bd72c5 100644
--- a/core/src/main/java/org/elasticsearch/index/SearchSlowLog.java
+++ b/core/src/main/java/org/elasticsearch/index/SearchSlowLog.java
@@ -25,14 +25,14 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.index.shard.SearchOperationListener;
 import org.elasticsearch.search.internal.SearchContext;
 
+import java.util.Collections;
 import java.util.concurrent.TimeUnit;
 
 public final class SearchSlowLog implements SearchOperationListener {
-    private boolean reformat;
-
     private long queryWarnThreshold;
     private long queryInfoThreshold;
     private long queryDebugThreshold;
@@ -73,20 +73,17 @@ public final class SearchSlowLog implements SearchOperationListener {
     public static final Setting<TimeValue> INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_TRACE_SETTING =
         Setting.timeSetting(INDEX_SEARCH_SLOWLOG_PREFIX + ".threshold.fetch.trace", TimeValue.timeValueNanos(-1),
             TimeValue.timeValueMillis(-1), Property.Dynamic, Property.IndexScope);
-    public static final Setting<Boolean> INDEX_SEARCH_SLOWLOG_REFORMAT =
-        Setting.boolSetting(INDEX_SEARCH_SLOWLOG_PREFIX + ".reformat", true, Property.Dynamic, Property.IndexScope);
     public static final Setting<SlowLogLevel> INDEX_SEARCH_SLOWLOG_LEVEL =
         new Setting<>(INDEX_SEARCH_SLOWLOG_PREFIX + ".level", SlowLogLevel.TRACE.name(), SlowLogLevel::parse, Property.Dynamic,
             Property.IndexScope);
 
+    private static final ToXContent.Params FORMAT_PARAMS = new ToXContent.MapParams(Collections.singletonMap("pretty", "false"));
+
     public SearchSlowLog(IndexSettings indexSettings) {
 
         this.queryLogger = Loggers.getLogger(INDEX_SEARCH_SLOWLOG_PREFIX + ".query", indexSettings.getSettings());
         this.fetchLogger = Loggers.getLogger(INDEX_SEARCH_SLOWLOG_PREFIX + ".fetch", indexSettings.getSettings());
 
-        indexSettings.getScopedSettings().addSettingsUpdateConsumer(INDEX_SEARCH_SLOWLOG_REFORMAT, this::setReformat);
-        this.reformat = indexSettings.getValue(INDEX_SEARCH_SLOWLOG_REFORMAT);
-
         indexSettings.getScopedSettings().addSettingsUpdateConsumer(INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_WARN_SETTING, this::setQueryWarnThreshold);
         this.queryWarnThreshold = indexSettings.getValue(INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_WARN_SETTING).nanos();
         indexSettings.getScopedSettings().addSettingsUpdateConsumer(INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_INFO_SETTING, this::setQueryInfoThreshold);
@@ -117,38 +114,36 @@ private void setLevel(SlowLogLevel level) {
     @Override
     public void onQueryPhase(SearchContext context, long tookInNanos) {
         if (queryWarnThreshold >= 0 && tookInNanos > queryWarnThreshold) {
-            queryLogger.warn("{}", new SlowLogSearchContextPrinter(context, tookInNanos, reformat));
+            queryLogger.warn("{}", new SlowLogSearchContextPrinter(context, tookInNanos));
         } else if (queryInfoThreshold >= 0 && tookInNanos > queryInfoThreshold) {
-            queryLogger.info("{}", new SlowLogSearchContextPrinter(context, tookInNanos, reformat));
+            queryLogger.info("{}", new SlowLogSearchContextPrinter(context, tookInNanos));
         } else if (queryDebugThreshold >= 0 && tookInNanos > queryDebugThreshold) {
-            queryLogger.debug("{}", new SlowLogSearchContextPrinter(context, tookInNanos, reformat));
+            queryLogger.debug("{}", new SlowLogSearchContextPrinter(context, tookInNanos));
         } else if (queryTraceThreshold >= 0 && tookInNanos > queryTraceThreshold) {
-            queryLogger.trace("{}", new SlowLogSearchContextPrinter(context, tookInNanos, reformat));
+            queryLogger.trace("{}", new SlowLogSearchContextPrinter(context, tookInNanos));
         }
     }
 
     @Override
     public void onFetchPhase(SearchContext context, long tookInNanos) {
         if (fetchWarnThreshold >= 0 && tookInNanos > fetchWarnThreshold) {
-            fetchLogger.warn("{}", new SlowLogSearchContextPrinter(context, tookInNanos, reformat));
+            fetchLogger.warn("{}", new SlowLogSearchContextPrinter(context, tookInNanos));
         } else if (fetchInfoThreshold >= 0 && tookInNanos > fetchInfoThreshold) {
-            fetchLogger.info("{}", new SlowLogSearchContextPrinter(context, tookInNanos, reformat));
+            fetchLogger.info("{}", new SlowLogSearchContextPrinter(context, tookInNanos));
         } else if (fetchDebugThreshold >= 0 && tookInNanos > fetchDebugThreshold) {
-            fetchLogger.debug("{}", new SlowLogSearchContextPrinter(context, tookInNanos, reformat));
+            fetchLogger.debug("{}", new SlowLogSearchContextPrinter(context, tookInNanos));
         } else if (fetchTraceThreshold >= 0 && tookInNanos > fetchTraceThreshold) {
-            fetchLogger.trace("{}", new SlowLogSearchContextPrinter(context, tookInNanos, reformat));
+            fetchLogger.trace("{}", new SlowLogSearchContextPrinter(context, tookInNanos));
         }
     }
 
     static final class SlowLogSearchContextPrinter {
         private final SearchContext context;
         private final long tookInNanos;
-        private final boolean reformat;
 
-        public SlowLogSearchContextPrinter(SearchContext context, long tookInNanos, boolean reformat) {
+        SlowLogSearchContextPrinter(SearchContext context, long tookInNanos) {
             this.context = context;
             this.tookInNanos = tookInNanos;
-            this.reformat = reformat;
         }
 
         @Override
@@ -172,7 +167,7 @@ public String toString() {
             }
             sb.append("search_type[").append(context.searchType()).append("], total_shards[").append(context.numberOfShards()).append("], ");
             if (context.request().source() != null) {
-                sb.append("source[").append(context.request().source()).append("], ");
+                sb.append("source[").append(context.request().source().toString(FORMAT_PARAMS)).append("], ");
             } else {
                 sb.append("source[], ");
             }
@@ -180,10 +175,6 @@ public String toString() {
         }
     }
 
-    private void setReformat(boolean reformat) {
-        this.reformat = reformat;
-    }
-
     private void setQueryWarnThreshold(TimeValue warnThreshold) {
         this.queryWarnThreshold = warnThreshold.nanos();
     }
@@ -216,10 +207,6 @@ private void setFetchTraceThreshold(TimeValue traceThreshold) {
         this.fetchTraceThreshold = traceThreshold.nanos();
     }
 
-    boolean isReformat() {
-        return reformat;
-    }
-
     long getQueryWarnThreshold() {
         return queryWarnThreshold;
     }
diff --git a/core/src/main/java/org/elasticsearch/index/VersionType.java b/core/src/main/java/org/elasticsearch/index/VersionType.java
index 062fbce10dea2..352056499f61a 100644
--- a/core/src/main/java/org/elasticsearch/index/VersionType.java
+++ b/core/src/main/java/org/elasticsearch/index/VersionType.java
@@ -198,6 +198,55 @@ public boolean validateVersionForReads(long version) {
             return version >= 0L || version == Versions.MATCH_ANY;
         }
 
+    },
+    /**
+     * Warning: this version type should be used with care. Concurrent indexing may result in loss of data on replicas
+     *
+     * @deprecated this version type will be removed in the next major version
+     */
+    @Deprecated
+    FORCE((byte) 3) {
+        @Override
+        public boolean isVersionConflictForWrites(long currentVersion, long expectedVersion, boolean deleted) {
+            if (currentVersion == Versions.NOT_FOUND) {
+                return false;
+            }
+            if (expectedVersion == Versions.MATCH_ANY) {
+                throw new IllegalStateException("you must specify a version when use VersionType.FORCE");
+            }
+            return false;
+        }
+
+        @Override
+        public String explainConflictForWrites(long currentVersion, long expectedVersion, boolean deleted) {
+            throw new AssertionError("VersionType.FORCE should never result in a write conflict");
+        }
+
+        @Override
+        public boolean isVersionConflictForReads(long currentVersion, long expectedVersion) {
+            return false;
+        }
+
+        @Override
+        public String explainConflictForReads(long currentVersion, long expectedVersion) {
+            throw new AssertionError("VersionType.FORCE should never result in a read conflict");
+        }
+
+        @Override
+        public long updateVersion(long currentVersion, long expectedVersion) {
+            return expectedVersion;
+        }
+
+        @Override
+        public boolean validateVersionForWrites(long version) {
+            return version >= 0L;
+        }
+
+        @Override
+        public boolean validateVersionForReads(long version) {
+            return version >= 0L || version == Versions.MATCH_ANY;
+        }
+
     };
 
     private final byte value;
@@ -291,6 +340,8 @@ public static VersionType fromString(String versionType) {
             return EXTERNAL;
         } else if ("external_gte".equals(versionType)) {
             return EXTERNAL_GTE;
+        } else if ("force".equals(versionType)) {
+            return FORCE;
         }
         throw new IllegalArgumentException("No version type match [" + versionType + "]");
     }
@@ -309,18 +360,18 @@ public static VersionType fromValue(byte value) {
             return EXTERNAL;
         } else if (value == 2) {
             return EXTERNAL_GTE;
+        } else if (value == 3) {
+            return FORCE;
         }
         throw new IllegalArgumentException("No version type match [" + value + "]");
     }
 
     public static VersionType readFromStream(StreamInput in) throws IOException {
-        int ordinal = in.readVInt();
-        assert (ordinal == 0 || ordinal == 1 || ordinal == 2 || ordinal == 3);
-        return VersionType.values()[ordinal];
+        return in.readEnum(VersionType.class);
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        out.writeVInt(ordinal());
+        out.writeEnum(this);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/ASCIIFoldingTokenFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/ASCIIFoldingTokenFilterFactory.java
index b7417b2637481..4318ef273dca9 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/ASCIIFoldingTokenFilterFactory.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/ASCIIFoldingTokenFilterFactory.java
@@ -47,6 +47,20 @@ public TokenStream create(TokenStream tokenStream) {
 
     @Override
     public Object getMultiTermComponent() {
-        return this;
+        if (preserveOriginal == false) {
+            return this;
+        } else {
+            // See https://issues.apache.org/jira/browse/LUCENE-7536 for the reasoning
+            return new TokenFilterFactory() {
+                @Override
+                public String name() {
+                    return ASCIIFoldingTokenFilterFactory.this.name();
+                }
+                @Override
+                public TokenStream create(TokenStream tokenStream) {
+                    return new ASCIIFoldingFilter(tokenStream, false);
+                }
+            };
+        }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java b/core/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java
index 119e0c16ea0dc..36357afe678b5 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java
@@ -18,14 +18,20 @@
  */
 package org.elasticsearch.index.analysis;
 
+import org.apache.logging.log4j.Logger;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.mapper.TextFieldMapper;
 import org.elasticsearch.indices.analysis.AnalysisModule;
 import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider;
 import org.elasticsearch.indices.analysis.PreBuiltAnalyzers;
@@ -39,6 +45,7 @@
 import java.util.HashMap;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Collectors;
 
@@ -46,7 +53,7 @@
 
 /**
  * An internal registry for tokenizer, token filter, char filter and analyzer.
- * This class exists per node and allows to create per-index {@link AnalysisService} via {@link #build(IndexSettings)}
+ * This class exists per node and allows to create per-index {@link IndexAnalyzers} via {@link #build(IndexSettings)}
  */
 public final class AnalysisRegistry implements Closeable {
     public static final String INDEX_ANALYSIS_CHAR_FILTER = "index.analysis.char_filter";
@@ -60,17 +67,20 @@ public final class AnalysisRegistry implements Closeable {
     private final Map<String, AnalysisProvider<TokenFilterFactory>> tokenFilters;
     private final Map<String, AnalysisProvider<TokenizerFactory>> tokenizers;
     private final Map<String, AnalysisProvider<AnalyzerProvider<?>>> analyzers;
+    private final Map<String, AnalysisProvider<AnalyzerProvider<?>>> normalizers;
 
     public AnalysisRegistry(Environment environment,
                             Map<String, AnalysisProvider<CharFilterFactory>> charFilters,
                             Map<String, AnalysisProvider<TokenFilterFactory>> tokenFilters,
                             Map<String, AnalysisProvider<TokenizerFactory>> tokenizers,
-                            Map<String, AnalysisProvider<AnalyzerProvider<?>>> analyzers) {
+                            Map<String, AnalysisProvider<AnalyzerProvider<?>>> analyzers,
+                            Map<String, AnalysisProvider<AnalyzerProvider<?>>> normalizers) {
         this.environment = environment;
         this.charFilters = unmodifiableMap(charFilters);
         this.tokenFilters = unmodifiableMap(tokenFilters);
         this.tokenizers = unmodifiableMap(tokenizers);
         this.analyzers = unmodifiableMap(analyzers);
+        this.normalizers = unmodifiableMap(normalizers);
     }
 
     /**
@@ -136,28 +146,50 @@ public void close() throws IOException {
     }
 
     /**
-     * Creates an index-level {@link AnalysisService} from this registry using the given index settings
+     * Creates an index-level {@link IndexAnalyzers} from this registry using the given index settings
      */
-    public AnalysisService build(IndexSettings indexSettings) throws IOException {
-        final Map<String, Settings> charFiltersSettings = indexSettings.getSettings().getGroups(INDEX_ANALYSIS_CHAR_FILTER);
-        final Map<String, Settings> tokenFiltersSettings = indexSettings.getSettings().getGroups(INDEX_ANALYSIS_FILTER);
-        final Map<String, Settings> tokenizersSettings = indexSettings.getSettings().getGroups(INDEX_ANALYSIS_TOKENIZER);
-        final Map<String, Settings> analyzersSettings = indexSettings.getSettings().getGroups("index.analysis.analyzer");
-
-        final Map<String, CharFilterFactory> charFilterFactories = buildMapping(false, "charfilter", indexSettings, charFiltersSettings, charFilters, prebuiltAnalysis.charFilterFactories);
-        final Map<String, TokenizerFactory> tokenizerFactories = buildMapping(false, "tokenizer", indexSettings, tokenizersSettings, tokenizers, prebuiltAnalysis.tokenizerFactories);
+    public IndexAnalyzers build(IndexSettings indexSettings) throws IOException {
+
+        final Map<String, CharFilterFactory> charFilterFactories = buildCharFilterFactories(indexSettings);
+        final Map<String, TokenizerFactory> tokenizerFactories = buildTokenizerFactories(indexSettings);
+        final Map<String, TokenFilterFactory> tokenFilterFactories = buildTokenFilterFactories(indexSettings);
+        final Map<String, AnalyzerProvider<?>> analyzierFactories = buildAnalyzerFactories(indexSettings);
+        final Map<String, AnalyzerProvider<?>> normalizerFactories = buildNormalizerFactories(indexSettings);
+        return build(indexSettings, analyzierFactories, normalizerFactories, tokenizerFactories, charFilterFactories, tokenFilterFactories);
+    }
 
+    public Map<String, TokenFilterFactory> buildTokenFilterFactories(IndexSettings indexSettings) throws IOException {
+        final Map<String, Settings> tokenFiltersSettings = indexSettings.getSettings().getGroups(INDEX_ANALYSIS_FILTER);
         Map<String, AnalysisModule.AnalysisProvider<TokenFilterFactory>> tokenFilters = new HashMap<>(this.tokenFilters);
         /*
-         * synonym is different than everything else since it needs access to the tokenizer factories for this index.
+         * synonym and synonym_graph are different than everything else since they need access to the tokenizer factories for the index.
          * instead of building the infrastructure for plugins we rather make it a real exception to not pollute the general interface and
          * hide internal data-structures as much as possible.
          */
-        tokenFilters.put("synonym", requriesAnalysisSettings((is, env, name, settings) -> new SynonymTokenFilterFactory(is, env, this, name, settings)));
-        final Map<String, TokenFilterFactory> tokenFilterFactories = buildMapping(false, "tokenfilter", indexSettings, tokenFiltersSettings, Collections.unmodifiableMap(tokenFilters), prebuiltAnalysis.tokenFilterFactories);
-        final Map<String, AnalyzerProvider<?>> analyzierFactories = buildMapping(true, "analyzer", indexSettings, analyzersSettings,
-                analyzers, prebuiltAnalysis.analyzerProviderFactories);
-        return new AnalysisService(indexSettings, analyzierFactories, tokenizerFactories, charFilterFactories, tokenFilterFactories);
+        tokenFilters.put("synonym", requiresAnalysisSettings((is, env, name, settings) -> new SynonymTokenFilterFactory(is, env, this, name, settings)));
+        tokenFilters.put("synonym_graph", requiresAnalysisSettings((is, env, name, settings) -> new SynonymGraphTokenFilterFactory(is, env, this, name, settings)));
+        return buildMapping(Component.FILTER, indexSettings, tokenFiltersSettings, Collections.unmodifiableMap(tokenFilters), prebuiltAnalysis.tokenFilterFactories);
+    }
+
+    public Map<String, TokenizerFactory> buildTokenizerFactories(IndexSettings indexSettings) throws IOException {
+        final Map<String, Settings> tokenizersSettings = indexSettings.getSettings().getGroups(INDEX_ANALYSIS_TOKENIZER);
+        return buildMapping(Component.TOKENIZER, indexSettings, tokenizersSettings, tokenizers, prebuiltAnalysis.tokenizerFactories);
+    }
+
+    public Map<String, CharFilterFactory> buildCharFilterFactories(IndexSettings indexSettings) throws IOException {
+        final Map<String, Settings> charFiltersSettings = indexSettings.getSettings().getGroups(INDEX_ANALYSIS_CHAR_FILTER);
+        return buildMapping(Component.CHAR_FILTER, indexSettings, charFiltersSettings, charFilters, prebuiltAnalysis.charFilterFactories);
+    }
+
+    public Map<String, AnalyzerProvider<?>> buildAnalyzerFactories(IndexSettings indexSettings) throws IOException {
+        final Map<String, Settings> analyzersSettings = indexSettings.getSettings().getGroups("index.analysis.analyzer");
+        return buildMapping(Component.ANALYZER, indexSettings, analyzersSettings, analyzers, prebuiltAnalysis.analyzerProviderFactories);
+    }
+
+    public Map<String, AnalyzerProvider<?>> buildNormalizerFactories(IndexSettings indexSettings) throws IOException {
+        final Map<String, Settings> noralizersSettings = indexSettings.getSettings().getGroups("index.analysis.normalizer");
+        // TODO: Have pre-built normalizers
+        return buildMapping(Component.NORMALIZER, indexSettings, noralizersSettings, normalizers, Collections.emptyMap());
     }
 
     /**
@@ -172,9 +204,9 @@ public AnalysisProvider<TokenizerFactory> getTokenizerProvider(String tokenizer,
         final Map<String, Settings> tokenizerSettings = indexSettings.getSettings().getGroups("index.analysis.tokenizer");
         if (tokenizerSettings.containsKey(tokenizer)) {
             Settings currentSettings = tokenizerSettings.get(tokenizer);
-            return getAnalysisProvider("tokenizer", tokenizers, tokenizer, currentSettings.get("type"));
+            return getAnalysisProvider(Component.TOKENIZER, tokenizers, tokenizer, currentSettings.get("type"));
         } else {
-            return prebuiltAnalysis.tokenizerFactories.get(tokenizer);
+            return getTokenizerProvider(tokenizer);
         }
     }
 
@@ -192,17 +224,19 @@ public AnalysisProvider<TokenFilterFactory> getTokenFilterProvider(String tokenF
             Settings currentSettings = tokenFilterSettings.get(tokenFilter);
             String typeName = currentSettings.get("type");
             /*
-             * synonym is different than everything else since it needs access to the tokenizer factories for this index.
+             * synonym and synonym_graph are different than everything else since they need access to the tokenizer factories for the index.
              * instead of building the infrastructure for plugins we rather make it a real exception to not pollute the general interface and
              * hide internal data-structures as much as possible.
              */
             if ("synonym".equals(typeName)) {
-                return requriesAnalysisSettings((is, env, name, settings) -> new SynonymTokenFilterFactory(is, env, this, name, settings));
+                return requiresAnalysisSettings((is, env, name, settings) -> new SynonymTokenFilterFactory(is, env, this, name, settings));
+            } else if ("synonym_graph".equals(typeName)) {
+                return requiresAnalysisSettings((is, env, name, settings) -> new SynonymGraphTokenFilterFactory(is, env, this, name, settings));
             } else {
-                return getAnalysisProvider("tokenfilter", tokenFilters, tokenFilter, typeName);
+                return getAnalysisProvider(Component.FILTER, tokenFilters, tokenFilter, typeName);
             }
         } else {
-            return prebuiltAnalysis.tokenFilterFactories.get(tokenFilter);
+            return getTokenFilterProvider(tokenFilter);
         }
     }
 
@@ -218,13 +252,13 @@ public AnalysisProvider<CharFilterFactory> getCharFilterProvider(String charFilt
         final Map<String, Settings> tokenFilterSettings = indexSettings.getSettings().getGroups("index.analysis.char_filter");
         if (tokenFilterSettings.containsKey(charFilter)) {
             Settings currentSettings = tokenFilterSettings.get(charFilter);
-            return getAnalysisProvider("charfilter", charFilters, charFilter, currentSettings.get("type"));
+            return getAnalysisProvider(Component.CHAR_FILTER, charFilters, charFilter, currentSettings.get("type"));
         } else {
-            return prebuiltAnalysis.charFilterFactories.get(charFilter);
+            return getCharFilterProvider(charFilter);
         }
     }
 
-    private static <T> AnalysisModule.AnalysisProvider<T> requriesAnalysisSettings(AnalysisModule.AnalysisProvider<T> provider) {
+    private static <T> AnalysisModule.AnalysisProvider<T> requiresAnalysisSettings(AnalysisModule.AnalysisProvider<T> provider) {
         return new AnalysisModule.AnalysisProvider<T>() {
             @Override
             public T get(IndexSettings indexSettings, Environment environment, String name, Settings settings) throws IOException {
@@ -237,7 +271,40 @@ public boolean requiresAnalysisSettings() {
         };
     }
 
-    private <T> Map<String, T> buildMapping(boolean analyzer, String toBuild, IndexSettings settings, Map<String, Settings> settingsMap,
+    enum Component {
+        ANALYZER {
+            @Override
+            public String toString() {
+                return "analyzer";
+            }
+        },
+        NORMALIZER {
+            @Override
+            public String toString() {
+                return "normalizer";
+            }
+        },
+        CHAR_FILTER {
+            @Override
+            public String toString() {
+                return "char_filter";
+            }
+        },
+        TOKENIZER {
+            @Override
+            public String toString() {
+                return "tokenizer";
+            }
+        },
+        FILTER {
+            @Override
+            public String toString() {
+                return "filter";
+            }
+        };
+    }
+
+    private <T> Map<String, T> buildMapping(Component component, IndexSettings settings, Map<String, Settings> settingsMap,
             Map<String, AnalysisModule.AnalysisProvider<T>> providerMap, Map<String, AnalysisModule.AnalysisProvider<T>> defaultInstance)
             throws IOException {
         Settings defaultSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, settings.getIndexVersionCreated()).build();
@@ -246,29 +313,34 @@ private <T> Map<String, T> buildMapping(boolean analyzer, String toBuild, IndexS
             String name = entry.getKey();
             Settings currentSettings = entry.getValue();
             String typeName = currentSettings.get("type");
-            if (analyzer) {
-                T factory;
+            if (component == Component.ANALYZER) {
+                T factory = null;
                 if (typeName == null) {
                     if (currentSettings.get("tokenizer") != null) {
                         factory = (T) new CustomAnalyzerProvider(settings, name, currentSettings);
                     } else {
-                        throw new IllegalArgumentException(toBuild + " [" + name + "] must specify either an analyzer type, or a tokenizer");
+                        throw new IllegalArgumentException(component + " [" + name + "] must specify either an analyzer type, or a tokenizer");
                     }
                 } else if (typeName.equals("custom")) {
                     factory = (T) new CustomAnalyzerProvider(settings, name, currentSettings);
-                } else {
-                    AnalysisModule.AnalysisProvider<T> type = providerMap.get(typeName);
-                    if (type == null) {
-                        throw new IllegalArgumentException("Unknown " + toBuild + " type [" + typeName + "] for [" + name + "]");
-                    }
-                    factory = type.get(settings, environment, name, currentSettings);
                 }
-                factories.put(name, factory);
-            }  else {
-                AnalysisProvider<T> type = getAnalysisProvider(toBuild, providerMap, name, typeName);
-                final T factory = type.get(settings, environment, name, currentSettings);
-                factories.put(name, factory);
+                if (factory != null) {
+                    factories.put(name, factory);
+                    continue;
+                }
+            } else if (component == Component.NORMALIZER) {
+                if (typeName == null || typeName.equals("custom")) {
+                    T factory = (T) new CustomNormalizerProvider(settings, name, currentSettings);
+                    factories.put(name, factory);
+                    continue;
+                }
+            }
+            AnalysisProvider<T> type = getAnalysisProvider(component, providerMap, name, typeName);
+            if (type == null) {
+                throw new IllegalArgumentException("Unknown " + component + " type [" + typeName + "] for [" + name + "]");
             }
+            final T factory = type.get(settings, environment, name, currentSettings);
+            factories.put(name, factory);
 
         }
         // go over the char filters in the bindings and register the ones that are not configured
@@ -306,13 +378,13 @@ private <T> Map<String, T> buildMapping(boolean analyzer, String toBuild, IndexS
         return factories;
     }
 
-    private <T> AnalysisProvider<T> getAnalysisProvider(String toBuild, Map<String, AnalysisProvider<T>> providerMap, String name, String typeName) {
+    private <T> AnalysisProvider<T> getAnalysisProvider(Component component, Map<String, AnalysisProvider<T>> providerMap, String name, String typeName) {
         if (typeName == null) {
-            throw new IllegalArgumentException(toBuild + " [" + name + "] must specify either an analyzer type, or a tokenizer");
+            throw new IllegalArgumentException(component + " [" + name + "] must specify either an analyzer type, or a tokenizer");
         }
         AnalysisProvider<T> type = providerMap.get(typeName);
         if (type == null) {
-            throw new IllegalArgumentException("Unknown " + toBuild + " type [" + typeName + "] for [" + name + "]");
+            throw new IllegalArgumentException("Unknown " + component + " type [" + typeName + "] for [" + name + "]");
         }
         return type;
     }
@@ -399,4 +471,159 @@ public void close() throws IOException {
             IOUtils.close(analyzerProviderFactories.values().stream().map((a) -> ((PreBuiltAnalyzerProviderFactory)a).analyzer()).collect(Collectors.toList()));
         }
     }
+
+    public IndexAnalyzers build(IndexSettings indexSettings,
+                                Map<String, AnalyzerProvider<?>> analyzerProviders,
+                                Map<String, AnalyzerProvider<?>> normalizerProviders,
+                                Map<String, TokenizerFactory> tokenizerFactoryFactories,
+                                Map<String, CharFilterFactory> charFilterFactoryFactories,
+                                Map<String, TokenFilterFactory> tokenFilterFactoryFactories) {
+
+        Index index = indexSettings.getIndex();
+        analyzerProviders = new HashMap<>(analyzerProviders);
+        Logger logger = Loggers.getLogger(getClass(), indexSettings.getSettings());
+        DeprecationLogger deprecationLogger = new DeprecationLogger(logger);
+        Map<String, NamedAnalyzer> analyzerAliases = new HashMap<>();
+        Map<String, NamedAnalyzer> analyzers = new HashMap<>();
+        Map<String, NamedAnalyzer> normalizers = new HashMap<>();
+        for (Map.Entry<String, AnalyzerProvider<?>> entry : analyzerProviders.entrySet()) {
+            processAnalyzerFactory(deprecationLogger, indexSettings, entry.getKey(), entry.getValue(), analyzerAliases, analyzers,
+                tokenFilterFactoryFactories, charFilterFactoryFactories, tokenizerFactoryFactories);
+        }
+        for (Map.Entry<String, AnalyzerProvider<?>> entry : normalizerProviders.entrySet()) {
+            processNormalizerFactory(deprecationLogger, indexSettings, entry.getKey(), entry.getValue(), normalizers,
+                    tokenFilterFactoryFactories, charFilterFactoryFactories);
+        }
+        for (Map.Entry<String, NamedAnalyzer> entry : analyzerAliases.entrySet()) {
+            String key = entry.getKey();
+            if (analyzers.containsKey(key) &&
+                ("default".equals(key) || "default_search".equals(key) || "default_search_quoted".equals(key)) == false) {
+                throw new IllegalStateException("already registered analyzer with name: " + key);
+            } else {
+                NamedAnalyzer configured = entry.getValue();
+                analyzers.put(key, configured);
+            }
+        }
+
+        if (!analyzers.containsKey("default")) {
+            processAnalyzerFactory(deprecationLogger, indexSettings, "default", new StandardAnalyzerProvider(indexSettings, null, "default", Settings.Builder.EMPTY_SETTINGS),
+                analyzerAliases, analyzers, tokenFilterFactoryFactories, charFilterFactoryFactories, tokenizerFactoryFactories);
+        }
+        if (!analyzers.containsKey("default_search")) {
+            analyzers.put("default_search", analyzers.get("default"));
+        }
+        if (!analyzers.containsKey("default_search_quoted")) {
+            analyzers.put("default_search_quoted", analyzers.get("default_search"));
+        }
+
+
+        NamedAnalyzer defaultAnalyzer = analyzers.get("default");
+        if (defaultAnalyzer == null) {
+            throw new IllegalArgumentException("no default analyzer configured");
+        }
+        if (analyzers.containsKey("default_index")) {
+            final Version createdVersion = indexSettings.getIndexVersionCreated();
+            if (createdVersion.onOrAfter(Version.V_5_0_0_alpha1)) {
+                throw new IllegalArgumentException("setting [index.analysis.analyzer.default_index] is not supported anymore, use [index.analysis.analyzer.default] instead for index [" + index.getName() + "]");
+            } else {
+                deprecationLogger.deprecated("setting [index.analysis.analyzer.default_index] is deprecated, use [index.analysis.analyzer.default] instead for index [{}]", index.getName());
+            }
+        }
+        NamedAnalyzer defaultIndexAnalyzer = analyzers.containsKey("default_index") ? analyzers.get("default_index") : defaultAnalyzer;
+        NamedAnalyzer defaultSearchAnalyzer = analyzers.containsKey("default_search") ? analyzers.get("default_search") : defaultAnalyzer;
+        NamedAnalyzer defaultSearchQuoteAnalyzer = analyzers.containsKey("default_search_quote") ? analyzers.get("default_search_quote") : defaultSearchAnalyzer;
+
+        for (Map.Entry<String, NamedAnalyzer> analyzer : analyzers.entrySet()) {
+            if (analyzer.getKey().startsWith("_")) {
+                throw new IllegalArgumentException("analyzer name must not start with '_'. got \"" + analyzer.getKey() + "\"");
+            }
+        }
+        return new IndexAnalyzers(indexSettings, defaultIndexAnalyzer, defaultSearchAnalyzer, defaultSearchQuoteAnalyzer,
+            unmodifiableMap(analyzers), unmodifiableMap(normalizers));
+    }
+
+    private void processAnalyzerFactory(DeprecationLogger deprecationLogger,
+                                        IndexSettings indexSettings,
+                                        String name,
+                                        AnalyzerProvider<?> analyzerFactory,
+                                        Map<String, NamedAnalyzer> analyzerAliases,
+                                        Map<String, NamedAnalyzer> analyzers, Map<String, TokenFilterFactory> tokenFilters,
+                                        Map<String, CharFilterFactory> charFilters, Map<String, TokenizerFactory> tokenizers) {
+        /*
+         * Lucene defaults positionIncrementGap to 0 in all analyzers but
+         * Elasticsearch defaults them to 0 only before version 2.0
+         * and 100 afterwards so we override the positionIncrementGap if it
+         * doesn't match here.
+         */
+        int overridePositionIncrementGap = TextFieldMapper.Defaults.POSITION_INCREMENT_GAP;
+        if (analyzerFactory instanceof CustomAnalyzerProvider) {
+            ((CustomAnalyzerProvider) analyzerFactory).build(tokenizers, charFilters, tokenFilters);
+            /*
+             * Custom analyzers already default to the correct, version
+             * dependent positionIncrementGap and the user is be able to
+             * configure the positionIncrementGap directly on the analyzer so
+             * we disable overriding the positionIncrementGap to preserve the
+             * user's setting.
+             */
+            overridePositionIncrementGap = Integer.MIN_VALUE;
+        }
+        Analyzer analyzerF = analyzerFactory.get();
+        if (analyzerF == null) {
+            throw new IllegalArgumentException("analyzer [" + analyzerFactory.name() + "] created null analyzer");
+        }
+        NamedAnalyzer analyzer;
+        if (analyzerF instanceof NamedAnalyzer) {
+            // if we got a named analyzer back, use it...
+            analyzer = (NamedAnalyzer) analyzerF;
+            if (overridePositionIncrementGap >= 0 && analyzer.getPositionIncrementGap(analyzer.name()) != overridePositionIncrementGap) {
+                // unless the positionIncrementGap needs to be overridden
+                analyzer = new NamedAnalyzer(analyzer, overridePositionIncrementGap);
+            }
+        } else {
+            analyzer = new NamedAnalyzer(name, analyzerFactory.scope(), analyzerF, overridePositionIncrementGap);
+        }
+        if (analyzers.containsKey(name)) {
+            throw new IllegalStateException("already registered analyzer with name: " + name);
+        }
+        analyzers.put(name, analyzer);
+        // TODO: remove alias support completely when we no longer support pre 5.0 indices
+        final String analyzerAliasKey = "index.analysis.analyzer." + analyzerFactory.name() + ".alias";
+        if (indexSettings.getSettings().get(analyzerAliasKey) != null) {
+            if (indexSettings.getIndexVersionCreated().onOrAfter(Version.V_5_0_0_beta1)) {
+                // do not allow alias creation if the index was created on or after v5.0 alpha6
+                throw new IllegalArgumentException("setting [" + analyzerAliasKey + "] is not supported");
+            }
+
+            // the setting is now removed but we only support it for loading indices created before v5.0
+            deprecationLogger.deprecated("setting [{}] is only allowed on index [{}] because it was created before 5.x; " +
+                "analyzer aliases can no longer be created on new indices.", analyzerAliasKey, indexSettings.getIndex().getName());
+            Set<String> aliases = Sets.newHashSet(indexSettings.getSettings().getAsArray(analyzerAliasKey));
+            for (String alias : aliases) {
+                if (analyzerAliases.putIfAbsent(alias, analyzer) != null) {
+                    throw new IllegalStateException("alias [" + alias + "] is already used by [" + analyzerAliases.get(alias).name() + "]");
+                }
+            }
+        }
+    }
+
+    private void processNormalizerFactory(DeprecationLogger deprecationLogger,
+            IndexSettings indexSettings,
+            String name,
+            AnalyzerProvider<?> normalizerFactory,
+            Map<String, NamedAnalyzer> normalizers,
+            Map<String, TokenFilterFactory> tokenFilters,
+            Map<String, CharFilterFactory> charFilters) {
+        if (normalizerFactory instanceof CustomNormalizerProvider) {
+            ((CustomNormalizerProvider) normalizerFactory).build(charFilters, tokenFilters);
+        }
+        Analyzer normalizerF = normalizerFactory.get();
+        if (normalizerF == null) {
+            throw new IllegalArgumentException("normalizer [" + normalizerFactory.name() + "] created null normalizer");
+        }
+        NamedAnalyzer normalizer = new NamedAnalyzer(name, normalizerFactory.scope(), normalizerF);
+        if (normalizers.containsKey(name)) {
+            throw new IllegalStateException("already registered analyzer with name: " + name);
+        }
+        normalizers.put(name, normalizer);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/AnalysisService.java b/core/src/main/java/org/elasticsearch/index/analysis/AnalysisService.java
deleted file mode 100644
index cb84e6c6d0a34..0000000000000
--- a/core/src/main/java/org/elasticsearch/index/analysis/AnalysisService.java
+++ /dev/null
@@ -1,218 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.analysis;
-
-import org.apache.lucene.analysis.Analyzer;
-import org.elasticsearch.Version;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.set.Sets;
-import org.elasticsearch.index.AbstractIndexComponent;
-import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.mapper.TextFieldMapper;
-
-import java.io.Closeable;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-
-import static java.util.Collections.unmodifiableMap;
-
-/**
- *
- */
-public class AnalysisService extends AbstractIndexComponent implements Closeable {
-
-    private final Map<String, NamedAnalyzer> analyzers;
-    private final Map<String, TokenizerFactory> tokenizers;
-    private final Map<String, CharFilterFactory> charFilters;
-    private final Map<String, TokenFilterFactory> tokenFilters;
-
-    private final NamedAnalyzer defaultIndexAnalyzer;
-    private final NamedAnalyzer defaultSearchAnalyzer;
-    private final NamedAnalyzer defaultSearchQuoteAnalyzer;
-
-    public AnalysisService(IndexSettings indexSettings,
-                           Map<String, AnalyzerProvider<?>> analyzerProviders,
-                           Map<String, TokenizerFactory> tokenizerFactoryFactories,
-                           Map<String, CharFilterFactory> charFilterFactoryFactories,
-                           Map<String, TokenFilterFactory> tokenFilterFactoryFactories) {
-        super(indexSettings);
-        this.tokenizers = unmodifiableMap(tokenizerFactoryFactories);
-        this.charFilters = unmodifiableMap(charFilterFactoryFactories);
-        this.tokenFilters = unmodifiableMap(tokenFilterFactoryFactories);
-        analyzerProviders = new HashMap<>(analyzerProviders);
-
-        Map<String, NamedAnalyzer> analyzerAliases = new HashMap<>();
-        Map<String, NamedAnalyzer> analyzers = new HashMap<>();
-        for (Map.Entry<String, AnalyzerProvider<?>> entry : analyzerProviders.entrySet()) {
-            processAnalyzerFactory(entry.getKey(), entry.getValue(), analyzerAliases, analyzers);
-        }
-        for (Map.Entry<String, NamedAnalyzer> entry : analyzerAliases.entrySet()) {
-            String key = entry.getKey();
-            if (analyzers.containsKey(key) &&
-                ("default".equals(key) || "default_search".equals(key) || "default_search_quoted".equals(key)) == false) {
-                throw new IllegalStateException("already registered analyzer with name: " + key);
-            } else {
-                NamedAnalyzer configured = entry.getValue();
-                analyzers.put(key, configured);
-            }
-        }
-
-        if (!analyzers.containsKey("default")) {
-            processAnalyzerFactory("default", new StandardAnalyzerProvider(indexSettings, null, "default", Settings.Builder.EMPTY_SETTINGS),
-                analyzerAliases, analyzers);
-        }
-        if (!analyzers.containsKey("default_search")) {
-            analyzers.put("default_search", analyzers.get("default"));
-        }
-        if (!analyzers.containsKey("default_search_quoted")) {
-            analyzers.put("default_search_quoted", analyzers.get("default_search"));
-        }
-
-
-        NamedAnalyzer defaultAnalyzer = analyzers.get("default");
-        if (defaultAnalyzer == null) {
-            throw new IllegalArgumentException("no default analyzer configured");
-        }
-        if (analyzers.containsKey("default_index")) {
-            final Version createdVersion = indexSettings.getIndexVersionCreated();
-            if (createdVersion.onOrAfter(Version.V_5_0_0_alpha1)) {
-                throw new IllegalArgumentException("setting [index.analysis.analyzer.default_index] is not supported anymore, use [index.analysis.analyzer.default] instead for index [" + index().getName() + "]");
-            } else {
-                deprecationLogger.deprecated("setting [index.analysis.analyzer.default_index] is deprecated, use [index.analysis.analyzer.default] instead for index [{}]", index().getName());
-            }
-        }
-        defaultIndexAnalyzer = analyzers.containsKey("default_index") ? analyzers.get("default_index") : defaultAnalyzer;
-        defaultSearchAnalyzer = analyzers.containsKey("default_search") ? analyzers.get("default_search") : defaultAnalyzer;
-        defaultSearchQuoteAnalyzer = analyzers.containsKey("default_search_quote") ? analyzers.get("default_search_quote") : defaultSearchAnalyzer;
-
-        for (Map.Entry<String, NamedAnalyzer> analyzer : analyzers.entrySet()) {
-            if (analyzer.getKey().startsWith("_")) {
-                throw new IllegalArgumentException("analyzer name must not start with '_'. got \"" + analyzer.getKey() + "\"");
-            }
-        }
-        this.analyzers = unmodifiableMap(analyzers);
-    }
-
-    private void processAnalyzerFactory(String name, AnalyzerProvider<?> analyzerFactory, Map<String, NamedAnalyzer> analyzerAliases, Map<String, NamedAnalyzer> analyzers) {
-        /*
-         * Lucene defaults positionIncrementGap to 0 in all analyzers but
-         * Elasticsearch defaults them to 0 only before version 2.0
-         * and 100 afterwards so we override the positionIncrementGap if it
-         * doesn't match here.
-         */
-        int overridePositionIncrementGap = TextFieldMapper.Defaults.POSITION_INCREMENT_GAP;
-        if (analyzerFactory instanceof CustomAnalyzerProvider) {
-            ((CustomAnalyzerProvider) analyzerFactory).build(this);
-            /*
-             * Custom analyzers already default to the correct, version
-             * dependent positionIncrementGap and the user is be able to
-             * configure the positionIncrementGap directly on the analyzer so
-             * we disable overriding the positionIncrementGap to preserve the
-             * user's setting.
-             */
-            overridePositionIncrementGap = Integer.MIN_VALUE;
-        }
-        Analyzer analyzerF = analyzerFactory.get();
-        if (analyzerF == null) {
-            throw new IllegalArgumentException("analyzer [" + analyzerFactory.name() + "] created null analyzer");
-        }
-        NamedAnalyzer analyzer;
-        if (analyzerF instanceof NamedAnalyzer) {
-            // if we got a named analyzer back, use it...
-            analyzer = (NamedAnalyzer) analyzerF;
-            if (overridePositionIncrementGap >= 0 && analyzer.getPositionIncrementGap(analyzer.name()) != overridePositionIncrementGap) {
-                // unless the positionIncrementGap needs to be overridden
-                analyzer = new NamedAnalyzer(analyzer, overridePositionIncrementGap);
-            }
-        } else {
-            analyzer = new NamedAnalyzer(name, analyzerFactory.scope(), analyzerF, overridePositionIncrementGap);
-        }
-        if (analyzers.containsKey(name)) {
-            throw new IllegalStateException("already registered analyzer with name: " + name);
-        }
-        analyzers.put(name, analyzer);
-        // TODO: remove alias support completely when we no longer support pre 5.0 indices
-        final String analyzerAliasKey = "index.analysis.analyzer." + analyzerFactory.name() + ".alias";
-        if (indexSettings.getSettings().get(analyzerAliasKey) != null) {
-            if (indexSettings.getIndexVersionCreated().onOrAfter(Version.V_5_0_0_alpha6)) {
-                // do not allow alias creation if the index was created on or after v5.0 alpha6
-                throw new IllegalArgumentException("setting [" + analyzerAliasKey + "] is not supported");
-            }
-
-            // the setting is now removed but we only support it for loading indices created before v5.0
-            deprecationLogger.deprecated("setting [{}] is only allowed on index [{}] because it was created before 5.x; " +
-                                         "analyzer aliases can no longer be created on new indices.", analyzerAliasKey, index().getName());
-            Set<String> aliases = Sets.newHashSet(indexSettings.getSettings().getAsArray(analyzerAliasKey));
-            for (String alias : aliases) {
-                if (analyzerAliases.putIfAbsent(alias, analyzer) != null) {
-                    throw new IllegalStateException("alias [" + alias + "] is already used by [" + analyzerAliases.get(alias).name() + "]");
-                }
-            }
-        }
-    }
-
-    @Override
-    public void close() {
-        for (NamedAnalyzer analyzer : analyzers.values()) {
-            if (analyzer.scope() == AnalyzerScope.INDEX) {
-                try {
-                    analyzer.close();
-                } catch (NullPointerException e) {
-                    // because analyzers are aliased, they might be closed several times
-                    // an NPE is thrown in this case, so ignore....
-                    // TODO: Analyzer's can no longer have aliases in indices created in 5.x and beyond,
-                    // so we only allow the aliases for analyzers on indices created pre 5.x for backwards
-                    // compatibility.  Once pre 5.0 indices are no longer supported, this check should be removed.
-                } catch (Exception e) {
-                    logger.debug("failed to close analyzer {}", analyzer);
-                }
-            }
-        }
-    }
-
-    public NamedAnalyzer analyzer(String name) {
-        return analyzers.get(name);
-    }
-
-    public NamedAnalyzer defaultIndexAnalyzer() {
-        return defaultIndexAnalyzer;
-    }
-
-    public NamedAnalyzer defaultSearchAnalyzer() {
-        return defaultSearchAnalyzer;
-    }
-
-    public NamedAnalyzer defaultSearchQuoteAnalyzer() {
-        return defaultSearchQuoteAnalyzer;
-    }
-
-    public TokenizerFactory tokenizer(String name) {
-        return tokenizers.get(name);
-    }
-
-    public CharFilterFactory charFilter(String name) {
-        return charFilters.get(name);
-    }
-
-    public TokenFilterFactory tokenFilter(String name) {
-        return tokenFilters.get(name);
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/CJKBigramFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/CJKBigramFilterFactory.java
index d7d5139bed5cd..4c709b14730d3 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/CJKBigramFilterFactory.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/CJKBigramFilterFactory.java
@@ -21,6 +21,7 @@
 
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.cjk.CJKBigramFilter;
+import org.apache.lucene.analysis.miscellaneous.DisableGraphAttribute;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
@@ -73,7 +74,17 @@ public CJKBigramFilterFactory(IndexSettings indexSettings, Environment environme
 
     @Override
     public TokenStream create(TokenStream tokenStream) {
-        return new CJKBigramFilter(tokenStream, flags, outputUnigrams);
+        CJKBigramFilter filter = new CJKBigramFilter(tokenStream, flags, outputUnigrams);
+        if (outputUnigrams) {
+            /**
+             * We disable the graph analysis on this token stream
+             * because it produces bigrams AND unigrams.
+             * Graph analysis on such token stream is useless and dangerous as it may create too many paths
+             * since shingles of different size are not aligned in terms of positions.
+             */
+            filter.addAttribute(DisableGraphAttribute.class);
+        }
+        return filter;
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/CharMatcher.java b/core/src/main/java/org/elasticsearch/index/analysis/CharMatcher.java
index 238fde5e6b3f5..b9e70d05bb77b 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/CharMatcher.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/CharMatcher.java
@@ -45,7 +45,7 @@ public boolean isTokenChar(int c) {
         }
     }
 
-    public enum Basic implements CharMatcher {
+    enum Basic implements CharMatcher {
         LETTER {
             @Override
             public boolean isTokenChar(int c) {
@@ -97,7 +97,7 @@ public boolean isTokenChar(int c) {
         }
     }
 
-    public final class Builder {
+    final class Builder {
         private final Set<CharMatcher> matchers;
         Builder() {
             matchers = new HashSet<>();
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzer.java b/core/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzer.java
index b68a321359ef2..d15e4e2e2fd61 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzer.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzer.java
@@ -97,4 +97,27 @@ protected Reader initReader(String fieldName, Reader reader) {
         }
         return reader;
     }
+
+    @Override
+    protected Reader initReaderForNormalization(String fieldName, Reader reader) {
+      for (CharFilterFactory charFilter : charFilters) {
+        if (charFilter instanceof MultiTermAwareComponent) {
+          charFilter = (CharFilterFactory) ((MultiTermAwareComponent) charFilter).getMultiTermComponent();
+          reader = charFilter.create(reader);
+        }
+      }
+      return reader;
+    }
+
+    @Override
+    protected TokenStream normalize(String fieldName, TokenStream in) {
+      TokenStream result = in;
+      for (TokenFilterFactory filter : tokenFilters) {
+        if (filter instanceof MultiTermAwareComponent) {
+          filter = (TokenFilterFactory) ((MultiTermAwareComponent) filter).getMultiTermComponent();
+          result = filter.create(result);
+        }
+      }
+      return result;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzerProvider.java
index 144cbe817430b..5aa38584bbae3 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzerProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzerProvider.java
@@ -26,6 +26,7 @@
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 
 /**
  * A custom analyzer that is built out of a single {@link org.apache.lucene.analysis.Tokenizer} and a list
@@ -43,35 +44,36 @@ public CustomAnalyzerProvider(IndexSettings indexSettings,
         this.analyzerSettings = settings;
     }
 
-    public void build(AnalysisService analysisService) {
+    public void build(final Map<String, TokenizerFactory> tokenizers, final Map<String, CharFilterFactory> charFilters,
+                      final Map<String, TokenFilterFactory> tokenFilters) {
         String tokenizerName = analyzerSettings.get("tokenizer");
         if (tokenizerName == null) {
             throw new IllegalArgumentException("Custom Analyzer [" + name() + "] must be configured with a tokenizer");
         }
 
-        TokenizerFactory tokenizer = analysisService.tokenizer(tokenizerName);
+        TokenizerFactory tokenizer = tokenizers.get(tokenizerName);
         if (tokenizer == null) {
             throw new IllegalArgumentException("Custom Analyzer [" + name() + "] failed to find tokenizer under name [" + tokenizerName + "]");
         }
 
-        List<CharFilterFactory> charFilters = new ArrayList<>();
         String[] charFilterNames = analyzerSettings.getAsArray("char_filter");
+        List<CharFilterFactory> charFiltersList = new ArrayList<>(charFilterNames.length);
         for (String charFilterName : charFilterNames) {
-            CharFilterFactory charFilter = analysisService.charFilter(charFilterName);
+            CharFilterFactory charFilter = charFilters.get(charFilterName);
             if (charFilter == null) {
                 throw new IllegalArgumentException("Custom Analyzer [" + name() + "] failed to find char_filter under name [" + charFilterName + "]");
             }
-            charFilters.add(charFilter);
+            charFiltersList.add(charFilter);
         }
 
-        List<TokenFilterFactory> tokenFilters = new ArrayList<>();
         String[] tokenFilterNames = analyzerSettings.getAsArray("filter");
+        List<TokenFilterFactory> tokenFilterList = new ArrayList<>(tokenFilterNames.length);
         for (String tokenFilterName : tokenFilterNames) {
-            TokenFilterFactory tokenFilter = analysisService.tokenFilter(tokenFilterName);
+            TokenFilterFactory tokenFilter = tokenFilters.get(tokenFilterName);
             if (tokenFilter == null) {
                 throw new IllegalArgumentException("Custom Analyzer [" + name() + "] failed to find filter under name [" + tokenFilterName + "]");
             }
-            tokenFilters.add(tokenFilter);
+            tokenFilterList.add(tokenFilter);
         }
 
         int positionIncrementGap = TextFieldMapper.Defaults.POSITION_INCREMENT_GAP;
@@ -93,8 +95,8 @@ public void build(AnalysisService analysisService) {
 
         int offsetGap = analyzerSettings.getAsInt("offset_gap", -1);;
         this.customAnalyzer = new CustomAnalyzer(tokenizer,
-                charFilters.toArray(new CharFilterFactory[charFilters.size()]),
-                tokenFilters.toArray(new TokenFilterFactory[tokenFilters.size()]),
+                charFiltersList.toArray(new CharFilterFactory[charFiltersList.size()]),
+                tokenFilterList.toArray(new TokenFilterFactory[tokenFilterList.size()]),
                 positionIncrementGap,
                 offsetGap
         );
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/CustomNormalizerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/CustomNormalizerProvider.java
new file mode 100644
index 0000000000000..2fcc987df6aa3
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/analysis/CustomNormalizerProvider.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.analysis;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.indices.analysis.PreBuiltTokenizers;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A custom normalizer that is built out of a char and token filters. On the
+ * contrary to analyzers, it does not support tokenizers and only supports a
+ * subset of char and token filters.
+ */
+public final class CustomNormalizerProvider extends AbstractIndexAnalyzerProvider<CustomAnalyzer> {
+
+    private final Settings analyzerSettings;
+
+    private CustomAnalyzer customAnalyzer;
+
+    public CustomNormalizerProvider(IndexSettings indexSettings,
+                                  String name, Settings settings) {
+        super(indexSettings, name, settings);
+        this.analyzerSettings = settings;
+    }
+
+    public void build(final Map<String, CharFilterFactory> charFilters, final Map<String, TokenFilterFactory> tokenFilters) {
+        String tokenizerName = analyzerSettings.get("tokenizer");
+        if (tokenizerName != null) {
+            throw new IllegalArgumentException("Custom normalizer [" + name() + "] cannot configure a tokenizer");
+        }
+
+        String[] charFilterNames = analyzerSettings.getAsArray("char_filter");
+        List<CharFilterFactory> charFiltersList = new ArrayList<>(charFilterNames.length);
+        for (String charFilterName : charFilterNames) {
+            CharFilterFactory charFilter = charFilters.get(charFilterName);
+            if (charFilter == null) {
+                throw new IllegalArgumentException("Custom normalizer [" + name() + "] failed to find char_filter under name ["
+                        + charFilterName + "]");
+            }
+            if (charFilter instanceof MultiTermAwareComponent == false) {
+                throw new IllegalArgumentException("Custom normalizer [" + name() + "] may not use char filter ["
+                        + charFilterName + "]");
+            }
+            charFilter = (CharFilterFactory) ((MultiTermAwareComponent) charFilter).getMultiTermComponent();
+            charFiltersList.add(charFilter);
+        }
+
+        String[] tokenFilterNames = analyzerSettings.getAsArray("filter");
+        List<TokenFilterFactory> tokenFilterList = new ArrayList<>(tokenFilterNames.length);
+        for (String tokenFilterName : tokenFilterNames) {
+            TokenFilterFactory tokenFilter = tokenFilters.get(tokenFilterName);
+            if (tokenFilter == null) {
+                throw new IllegalArgumentException("Custom Analyzer [" + name() + "] failed to find filter under name ["
+                        + tokenFilterName + "]");
+            }
+            if (tokenFilter instanceof MultiTermAwareComponent == false) {
+                throw new IllegalArgumentException("Custom normalizer [" + name() + "] may not use filter [" + tokenFilterName + "]");
+            }
+            tokenFilter = (TokenFilterFactory) ((MultiTermAwareComponent) tokenFilter).getMultiTermComponent();
+            tokenFilterList.add(tokenFilter);
+        }
+
+        this.customAnalyzer = new CustomAnalyzer(
+                PreBuiltTokenizers.KEYWORD.getTokenizerFactory(indexSettings.getIndexVersionCreated()),
+                charFiltersList.toArray(new CharFilterFactory[charFiltersList.size()]),
+                tokenFilterList.toArray(new TokenFilterFactory[tokenFilterList.size()])
+        );
+    }
+
+    @Override
+    public CustomAnalyzer get() {
+        return this.customAnalyzer;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/FlattenGraphTokenFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/FlattenGraphTokenFilterFactory.java
new file mode 100644
index 0000000000000..6c9487a2cb32b
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/analysis/FlattenGraphTokenFilterFactory.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.analysis;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.core.FlattenGraphFilter;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.IndexSettings;
+
+public class FlattenGraphTokenFilterFactory extends AbstractTokenFilterFactory {
+
+    public FlattenGraphTokenFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
+        super(indexSettings, name, settings);
+    }
+
+    @Override
+    public TokenStream create(TokenStream tokenStream) {
+        return new FlattenGraphFilter(tokenStream);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/IndexAnalyzers.java b/core/src/main/java/org/elasticsearch/index/analysis/IndexAnalyzers.java
new file mode 100644
index 0000000000000..f3200d606fb45
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/analysis/IndexAnalyzers.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.index.analysis;
+
+import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.index.AbstractIndexComponent;
+import org.elasticsearch.index.IndexSettings;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Map;
+import java.util.stream.Stream;
+
+/**
+ * IndexAnalyzers contains a name to analyzer mapping for a specific index.
+ * This class only holds analyzers that are explicitly configured for an index and doesn't allow
+ * access to individual tokenizers, char or token filter.
+ *
+ * @see AnalysisRegistry
+ */
+public final class IndexAnalyzers extends AbstractIndexComponent implements Closeable {
+    private final NamedAnalyzer defaultIndexAnalyzer;
+    private final NamedAnalyzer defaultSearchAnalyzer;
+    private final NamedAnalyzer defaultSearchQuoteAnalyzer;
+    private final Map<String, NamedAnalyzer> analyzers;
+    private final Map<String, NamedAnalyzer> normalizers;
+    private final IndexSettings indexSettings;
+
+    public IndexAnalyzers(IndexSettings indexSettings, NamedAnalyzer defaultIndexAnalyzer, NamedAnalyzer defaultSearchAnalyzer,
+                          NamedAnalyzer defaultSearchQuoteAnalyzer, Map<String, NamedAnalyzer> analyzers,
+                          Map<String, NamedAnalyzer> normalizers) {
+        super(indexSettings);
+        this.defaultIndexAnalyzer = defaultIndexAnalyzer;
+        this.defaultSearchAnalyzer = defaultSearchAnalyzer;
+        this.defaultSearchQuoteAnalyzer = defaultSearchQuoteAnalyzer;
+        this.analyzers = analyzers;
+        this.normalizers = normalizers;
+        this.indexSettings = indexSettings;
+    }
+
+    /**
+     * Returns an analyzer mapped to the given name or <code>null</code> if not present
+     */
+    public NamedAnalyzer get(String name) {
+        return analyzers.get(name);
+    }
+
+    /**
+     * Returns a normalizer mapped to the given name or <code>null</code> if not present
+     */
+    public NamedAnalyzer getNormalizer(String name) {
+        return normalizers.get(name);
+    }
+
+    /**
+     * Returns the default index analyzer for this index
+     */
+    public NamedAnalyzer getDefaultIndexAnalyzer() {
+        return defaultIndexAnalyzer;
+    }
+
+    /**
+     * Returns the default search analyzer for this index
+     */
+    public NamedAnalyzer getDefaultSearchAnalyzer() {
+        return defaultSearchAnalyzer;
+    }
+
+    /**
+     * Returns the default search quote analyzer for this index
+     */
+    public NamedAnalyzer getDefaultSearchQuoteAnalyzer() {
+        return defaultSearchQuoteAnalyzer;
+    }
+
+    @Override
+    public void close() throws IOException {
+       IOUtils.close(() -> Stream.concat(analyzers.values().stream(), normalizers.values().stream())
+           .filter(a -> a.scope() == AnalyzerScope.INDEX)
+           .iterator());
+    }
+
+    /**
+     * Returns the indices settings
+     */
+    public IndexSettings getIndexSettings() {
+        return indexSettings;
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/KeywordMarkerTokenFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/KeywordMarkerTokenFilterFactory.java
index 0805a3bdf8609..7869e3495812d 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/KeywordMarkerTokenFilterFactory.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/KeywordMarkerTokenFilterFactory.java
@@ -21,30 +21,68 @@
 
 import org.apache.lucene.analysis.CharArraySet;
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.miscellaneous.PatternKeywordMarkerFilter;
 import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
 
 import java.util.Set;
+import java.util.regex.Pattern;
 
+/**
+ * A factory for creating keyword marker token filters that prevent tokens from
+ * being modified by stemmers.  Two types of keyword marker filters are available:
+ * the {@link SetKeywordMarkerFilter} and the {@link PatternKeywordMarkerFilter}.
+ *
+ * The {@link SetKeywordMarkerFilter} uses a set of keywords to denote which tokens
+ * should be excluded from stemming.  This filter is created if the settings include
+ * {@code keywords}, which contains the list of keywords, or {@code `keywords_path`},
+ * which contains a path to a file in the config directory with the keywords.
+ *
+ * The {@link PatternKeywordMarkerFilter} uses a regular expression pattern to match
+ * against tokens that should be excluded from stemming.  This filter is created if
+ * the settings include {@code keywords_pattern}, which contains the regular expression
+ * to match against.
+ */
 public class KeywordMarkerTokenFilterFactory extends AbstractTokenFilterFactory {
 
     private final CharArraySet keywordLookup;
+    private final Pattern keywordPattern;
 
     public KeywordMarkerTokenFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
 
         boolean ignoreCase = settings.getAsBoolean("ignore_case", false);
-        Set<?> rules = Analysis.getWordSet(env, settings, "keywords");
-        if (rules == null) {
-            throw new IllegalArgumentException("keyword filter requires either `keywords` or `keywords_path` to be configured");
+        String patternString = settings.get("keywords_pattern");
+        if (patternString != null) {
+            // a pattern for matching keywords is specified, as opposed to a
+            // set of keyword strings to match against
+            if (settings.get("keywords") != null || settings.get("keywords_path") != null) {
+                throw new IllegalArgumentException(
+                    "cannot specify both `keywords_pattern` and `keywords` or `keywords_path`");
+            }
+            keywordPattern = Pattern.compile(patternString);
+            keywordLookup = null;
+        } else {
+            Set<?> rules = Analysis.getWordSet(env, settings, "keywords");
+            if (rules == null) {
+                throw new IllegalArgumentException(
+                    "keyword filter requires either `keywords`, `keywords_path`, " +
+                    "or `keywords_pattern` to be configured");
+            }
+            // a set of keywords (or a path to them) is specified
+            keywordLookup = new CharArraySet(rules, ignoreCase);
+            keywordPattern = null;
         }
-        keywordLookup = new CharArraySet(rules, ignoreCase);
     }
 
     @Override
     public TokenStream create(TokenStream tokenStream) {
-        return new SetKeywordMarkerFilter(tokenStream, keywordLookup);
+        if (keywordPattern != null) {
+            return new PatternKeywordMarkerFilter(tokenStream, keywordPattern);
+        } else {
+            return new SetKeywordMarkerFilter(tokenStream, keywordLookup);
+        }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/NamedAnalyzer.java b/core/src/main/java/org/elasticsearch/index/analysis/NamedAnalyzer.java
index 1dd562c4bb144..416967e94f5f0 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/NamedAnalyzer.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/NamedAnalyzer.java
@@ -39,10 +39,6 @@ public NamedAnalyzer(NamedAnalyzer analyzer, int positionIncrementGap) {
         this(analyzer.name(), analyzer.scope(), analyzer.analyzer(), positionIncrementGap);
     }
 
-    public NamedAnalyzer(String name, Analyzer analyzer) {
-        this(name, AnalyzerScope.INDEX, analyzer);
-    }
-
     public NamedAnalyzer(String name, AnalyzerScope scope, Analyzer analyzer) {
         this(name, scope, analyzer, Integer.MIN_VALUE);
     }
@@ -119,4 +115,12 @@ public boolean equals(Object o) {
     public int hashCode() {
         return Objects.hash(name);
     }
+
+    @Override
+    public void close() {
+        super.close();
+        if (scope == AnalyzerScope.INDEX) {
+            analyzer.close();
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/PatternAnalyzer.java b/core/src/main/java/org/elasticsearch/index/analysis/PatternAnalyzer.java
index 7554f459bfaf4..5d4d9f2df3f4f 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/PatternAnalyzer.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/PatternAnalyzer.java
@@ -53,4 +53,13 @@ protected TokenStreamComponents createComponents(String s) {
         }
         return new TokenStreamComponents(tokenizer, stream);
     }
+
+    @Override
+    protected TokenStream normalize(String fieldName, TokenStream in) {
+        TokenStream stream = in;
+        if (lowercase) {
+            stream = new LowerCaseFilter(stream);
+        }
+        return stream;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/PatternReplaceCharFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/PatternReplaceCharFilterFactory.java
index db428db153d49..2562f20373bef 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/PatternReplaceCharFilterFactory.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/PatternReplaceCharFilterFactory.java
@@ -18,6 +18,9 @@
  */
 package org.elasticsearch.index.analysis;
 
+import java.io.Reader;
+import java.util.regex.Pattern;
+
 import org.apache.lucene.analysis.pattern.PatternReplaceCharFilter;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.regex.Regex;
@@ -25,10 +28,7 @@
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
 
-import java.io.Reader;
-import java.util.regex.Pattern;
-
-public class PatternReplaceCharFilterFactory extends AbstractCharFilterFactory {
+public class PatternReplaceCharFilterFactory extends AbstractCharFilterFactory implements MultiTermAwareComponent {
 
     private final Pattern pattern;
     private final String replacement;
@@ -56,4 +56,9 @@ public String getReplacement() {
     public Reader create(Reader tokenStream) {
         return new PatternReplaceCharFilter(pattern, replacement, tokenStream);
     }
+
+    @Override
+    public Object getMultiTermComponent() {
+        return this;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/ShingleTokenFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/ShingleTokenFilterFactory.java
index 0c6ae7b965212..1f7fc301cbac7 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/ShingleTokenFilterFactory.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/ShingleTokenFilterFactory.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.index.analysis;
 
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.miscellaneous.DisableGraphAttribute;
 import org.apache.lucene.analysis.shingle.ShingleFilter;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
@@ -89,6 +90,15 @@ public TokenStream create(TokenStream tokenStream) {
             filter.setOutputUnigramsIfNoShingles(outputUnigramsIfNoShingles);
             filter.setTokenSeparator(tokenSeparator);
             filter.setFillerToken(fillerToken);
+            if (outputUnigrams || (minShingleSize != maxShingleSize)) {
+                /**
+                 * We disable the graph analysis on this token stream
+                 * because it produces shingles of different size.
+                 * Graph analysis on such token stream is useless and dangerous as it may create too many paths
+                 * since shingles of different size are not aligned in terms of positions.
+                 */
+                filter.addAttribute(DisableGraphAttribute.class);
+            }
             return filter;
         }
 
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/SynonymGraphTokenFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/SynonymGraphTokenFilterFactory.java
new file mode 100644
index 0000000000000..cfb37f0b075c7
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/analysis/SynonymGraphTokenFilterFactory.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.analysis;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.synonym.SynonymGraphFilter;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.IndexSettings;
+
+import java.io.IOException;
+
+public class SynonymGraphTokenFilterFactory extends SynonymTokenFilterFactory {
+    public SynonymGraphTokenFilterFactory(IndexSettings indexSettings, Environment env, AnalysisRegistry analysisRegistry,
+                                     String name, Settings settings) throws IOException {
+        super(indexSettings, env, analysisRegistry, name, settings);
+    }
+
+    @Override
+    public TokenStream create(TokenStream tokenStream) {
+        // fst is null means no synonyms
+        return synonymMap.fst == null ? tokenStream : new SynonymGraphFilter(tokenStream, synonymMap, ignoreCase);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/SynonymTokenFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/SynonymTokenFilterFactory.java
index 11f1303328c7a..d32c66e0dfe8e 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/SynonymTokenFilterFactory.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/SynonymTokenFilterFactory.java
@@ -40,8 +40,8 @@
 
 public class SynonymTokenFilterFactory extends AbstractTokenFilterFactory {
 
-    private final SynonymMap synonymMap;
-    private final boolean ignoreCase;
+    protected final SynonymMap synonymMap;
+    protected final boolean ignoreCase;
 
     public SynonymTokenFilterFactory(IndexSettings indexSettings, Environment env, AnalysisRegistry analysisRegistry,
                                       String name, Settings settings) throws IOException {
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/WordDelimiterGraphTokenFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/WordDelimiterGraphTokenFilterFactory.java
new file mode 100644
index 0000000000000..aa88a1647a78e
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/analysis/WordDelimiterGraphTokenFilterFactory.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.analysis;
+
+import org.apache.lucene.analysis.CharArraySet;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.miscellaneous.WordDelimiterGraphFilter;
+import org.apache.lucene.analysis.miscellaneous.WordDelimiterIterator;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.IndexSettings;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.apache.lucene.analysis.miscellaneous.WordDelimiterFilter.CATENATE_ALL;
+import static org.apache.lucene.analysis.miscellaneous.WordDelimiterFilter.CATENATE_NUMBERS;
+import static org.apache.lucene.analysis.miscellaneous.WordDelimiterFilter.CATENATE_WORDS;
+import static org.apache.lucene.analysis.miscellaneous.WordDelimiterFilter.GENERATE_NUMBER_PARTS;
+import static org.apache.lucene.analysis.miscellaneous.WordDelimiterFilter.GENERATE_WORD_PARTS;
+import static org.apache.lucene.analysis.miscellaneous.WordDelimiterFilter.PRESERVE_ORIGINAL;
+import static org.apache.lucene.analysis.miscellaneous.WordDelimiterFilter.SPLIT_ON_CASE_CHANGE;
+import static org.apache.lucene.analysis.miscellaneous.WordDelimiterFilter.SPLIT_ON_NUMERICS;
+import static org.apache.lucene.analysis.miscellaneous.WordDelimiterFilter.STEM_ENGLISH_POSSESSIVE;
+import static org.elasticsearch.index.analysis.WordDelimiterTokenFilterFactory.parseTypes;
+
+public class WordDelimiterGraphTokenFilterFactory extends AbstractTokenFilterFactory {
+
+    private final byte[] charTypeTable;
+    private final int flags;
+    private final CharArraySet protoWords;
+
+    public WordDelimiterGraphTokenFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) {
+        super(indexSettings, name, settings);
+
+        // Sample Format for the type table:
+        // $ => DIGIT
+        // % => DIGIT
+        // . => DIGIT
+        // \u002C => DIGIT
+        // \u200D => ALPHANUM
+        List<String> charTypeTableValues = Analysis.getWordList(env, settings, "type_table");
+        if (charTypeTableValues == null) {
+            this.charTypeTable = WordDelimiterIterator.DEFAULT_WORD_DELIM_TABLE;
+        } else {
+            this.charTypeTable = parseTypes(charTypeTableValues);
+        }
+        int flags = 0;
+        // If set, causes parts of words to be generated: "PowerShot" => "Power" "Shot"
+        flags |= getFlag(GENERATE_WORD_PARTS, settings, "generate_word_parts", true);
+        // If set, causes number subwords to be generated: "500-42" => "500" "42"
+        flags |= getFlag(GENERATE_NUMBER_PARTS, settings, "generate_number_parts", true);
+        // 1, causes maximum runs of word parts to be catenated: "wi-fi" => "wifi"
+        flags |= getFlag(CATENATE_WORDS, settings, "catenate_words", false);
+        // If set, causes maximum runs of number parts to be catenated: "500-42" => "50042"
+        flags |= getFlag(CATENATE_NUMBERS, settings, "catenate_numbers", false);
+        // If set, causes all subword parts to be catenated: "wi-fi-4000" => "wifi4000"
+        flags |= getFlag(CATENATE_ALL, settings, "catenate_all", false);
+        // 1, causes "PowerShot" to be two tokens; ("Power-Shot" remains two parts regards)
+        flags |= getFlag(SPLIT_ON_CASE_CHANGE, settings, "split_on_case_change", true);
+        // If set, includes original words in subwords: "500-42" => "500" "42" "500-42"
+        flags |= getFlag(PRESERVE_ORIGINAL, settings, "preserve_original", false);
+        // 1, causes "j2se" to be three tokens; "j" "2" "se"
+        flags |= getFlag(SPLIT_ON_NUMERICS, settings, "split_on_numerics", true);
+        // If set, causes trailing "'s" to be removed for each subword: "O'Neil's" => "O", "Neil"
+        flags |= getFlag(STEM_ENGLISH_POSSESSIVE, settings, "stem_english_possessive", true);
+        // If not null is the set of tokens to protect from being delimited
+        Set<?> protectedWords = Analysis.getWordSet(env, settings, "protected_words");
+        this.protoWords = protectedWords == null ? null : CharArraySet.copy(protectedWords);
+        this.flags = flags;
+    }
+
+    @Override
+    public TokenStream create(TokenStream tokenStream) {
+        return new WordDelimiterGraphFilter(tokenStream, charTypeTable, flags, protoWords);
+    }
+
+    private int getFlag(int flag, Settings settings, String key, boolean defaultValue) {
+        if (settings.getAsBoolean(key, defaultValue)) {
+            return flag;
+        }
+        return 0;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/WordDelimiterTokenFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/WordDelimiterTokenFilterFactory.java
index ccc60f4ce7027..74d10553e4afe 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/WordDelimiterTokenFilterFactory.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/WordDelimiterTokenFilterFactory.java
@@ -113,7 +113,7 @@ public int getFlag(int flag, Settings settings, String key, boolean defaultValue
     /**
      * parses a list of MappingCharFilter style rules into a custom byte[] type table
      */
-    private byte[] parseTypes(Collection<String> rules) {
+    static byte[] parseTypes(Collection<String> rules) {
         SortedMap<Character, Byte> typeMap = new TreeMap<>();
         for (String rule : rules) {
             Matcher m = typePattern.matcher(rule);
@@ -137,7 +137,7 @@ private byte[] parseTypes(Collection<String> rules) {
         return types;
     }
 
-    private Byte parseType(String s) {
+    private static Byte parseType(String s) {
         if (s.equals("LOWER"))
             return WordDelimiterFilter.LOWER;
         else if (s.equals("UPPER"))
@@ -154,9 +154,8 @@ else if (s.equals("SUBWORD_DELIM"))
             return null;
     }
 
-    char[] out = new char[256];
-
-    private String parseString(String s) {
+    private static String parseString(String s) {
+        char[] out = new char[256];
         int readPos = 0;
         int len = s.length();
         int writePos = 0;
diff --git a/core/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java b/core/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java
index 0e4c54e7a7d34..fe11a30fad6f8 100644
--- a/core/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java
+++ b/core/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java
@@ -117,8 +117,7 @@ public void clear(String reason) {
     private BitSet getAndLoadIfNotPresent(final Query query, final LeafReaderContext context) throws IOException, ExecutionException {
         final Object coreCacheReader = context.reader().getCoreCacheKey();
         final ShardId shardId = ShardUtils.extractShardId(context.reader());
-        if (shardId != null // can't require it because of the percolator
-                && indexSettings.getIndex().equals(shardId.getIndex()) == false) {
+        if (indexSettings.getIndex().equals(shardId.getIndex()) == false) {
             // insanity
             throw new IllegalStateException("Trying to load bit set for index " + shardId.getIndex()
                     + " with cache of index " + indexSettings.getIndex());
@@ -236,7 +235,7 @@ public IndexWarmer.TerminationHandle warmReader(final IndexShard indexShard, fin
                     hasNested = true;
                     for (ObjectMapper objectMapper : docMapper.objectMappers().values()) {
                         if (objectMapper.nested().isNested()) {
-                            ObjectMapper parentObjectMapper = docMapper.findParentObjectMapper(objectMapper);
+                            ObjectMapper parentObjectMapper = objectMapper.getParentObjectMapper(mapperService);
                             if (parentObjectMapper != null && parentObjectMapper.nested().isNested()) {
                                 warmUp.add(parentObjectMapper.nestedTypeFilter());
                             }
diff --git a/core/src/main/java/org/elasticsearch/index/cache/query/QueryCache.java b/core/src/main/java/org/elasticsearch/index/cache/query/QueryCache.java
index bee947b54f0d1..d1747fc30ee6f 100644
--- a/core/src/main/java/org/elasticsearch/index/cache/query/QueryCache.java
+++ b/core/src/main/java/org/elasticsearch/index/cache/query/QueryCache.java
@@ -28,7 +28,7 @@
  */
 public interface QueryCache extends IndexComponent, Closeable, org.apache.lucene.search.QueryCache {
 
-    static class EntriesStats {
+    class EntriesStats {
         public final long sizeInBytes;
         public final long count;
 
diff --git a/core/src/main/java/org/elasticsearch/index/cache/query/QueryCacheStats.java b/core/src/main/java/org/elasticsearch/index/cache/query/QueryCacheStats.java
index fcbb619004886..99673afac6f35 100644
--- a/core/src/main/java/org/elasticsearch/index/cache/query/QueryCacheStats.java
+++ b/core/src/main/java/org/elasticsearch/index/cache/query/QueryCacheStats.java
@@ -108,13 +108,6 @@ public long getEvictions() {
         return cacheCount - cacheSize;
     }
 
-    public static QueryCacheStats readQueryCacheStats(StreamInput in) throws IOException {
-        QueryCacheStats stats = new QueryCacheStats();
-        stats.readFrom(in);
-        return stats;
-    }
-
-
     @Override
     public void readFrom(StreamInput in) throws IOException {
         ramBytesUsed = in.readLong();
diff --git a/core/src/main/java/org/elasticsearch/index/engine/CommitStats.java b/core/src/main/java/org/elasticsearch/index/engine/CommitStats.java
index 48fb8a80eeb84..eb2e35a5a23fb 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/CommitStats.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/CommitStats.java
@@ -49,13 +49,6 @@ public CommitStats(SegmentInfos segmentInfos) {
     }
 
     private CommitStats() {
-
-    }
-
-    public static CommitStats readCommitStatsFrom(StreamInput in) throws IOException {
-        CommitStats commitStats = new CommitStats();
-        commitStats.readFrom(in);
-        return commitStats;
     }
 
     public static CommitStats readOptionalCommitStatsFrom(StreamInput in) throws IOException {
diff --git a/core/src/main/java/org/elasticsearch/index/engine/DeleteFailedEngineException.java b/core/src/main/java/org/elasticsearch/index/engine/DeleteFailedEngineException.java
index 068df25e2fdc6..16716ccdcb03c 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/DeleteFailedEngineException.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/DeleteFailedEngineException.java
@@ -25,15 +25,13 @@
 import java.io.IOException;
 
 /**
- *
+ * Deprecated as not used in 6.0, should be removed in 7.0
+ * Still exists for bwc in serializing/deserializing from
+ * 5.x nodes
  */
+@Deprecated
 public class DeleteFailedEngineException extends EngineException {
-
-    public DeleteFailedEngineException(ShardId shardId, Engine.Delete delete, Throwable cause) {
-        super(shardId, "Delete failed for [" + delete.uid().text() + "]", cause);
-    }
-
     public DeleteFailedEngineException(StreamInput in) throws IOException{
         super(in);
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/index/engine/DeleteVersionValue.java b/core/src/main/java/org/elasticsearch/index/engine/DeleteVersionValue.java
index baacc4b240d6b..45b28d96ba283 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/DeleteVersionValue.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/DeleteVersionValue.java
@@ -29,18 +29,18 @@ class DeleteVersionValue extends VersionValue {
 
     private final long time;
 
-    public DeleteVersionValue(long version, long time) {
+    DeleteVersionValue(long version, long time) {
         super(version);
         this.time = time;
     }
 
     @Override
-    public long time() {
+    public long getTime() {
         return this.time;
     }
 
     @Override
-    public boolean delete() {
+    public boolean isDelete() {
         return true;
     }
 
@@ -48,4 +48,12 @@ public boolean delete() {
     public long ramBytesUsed() {
         return BASE_RAM_BYTES_USED;
     }
+
+    @Override
+    public String toString() {
+        return "DeleteVersionValue{" +
+            "version=" + getVersion() +
+            ",time=" + time +
+            '}';
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/engine/ElasticsearchConcurrentMergeScheduler.java b/core/src/main/java/org/elasticsearch/index/engine/ElasticsearchConcurrentMergeScheduler.java
index 466da06dec4fa..f4876149cac13 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/ElasticsearchConcurrentMergeScheduler.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/ElasticsearchConcurrentMergeScheduler.java
@@ -67,7 +67,7 @@ class ElasticsearchConcurrentMergeScheduler extends ConcurrentMergeScheduler {
     private final Set<OnGoingMerge> readOnlyOnGoingMerges = Collections.unmodifiableSet(onGoingMerges);
     private final MergeSchedulerConfig config;
 
-    public ElasticsearchConcurrentMergeScheduler(ShardId shardId, IndexSettings indexSettings) {
+    ElasticsearchConcurrentMergeScheduler(ShardId shardId, IndexSettings indexSettings) {
         this.config = indexSettings.getMergeSchedulerConfig();
         this.shardId = shardId;
         this.indexSettings = indexSettings.getSettings();
@@ -110,10 +110,15 @@ protected void doMerge(IndexWriter writer, MergePolicy.OneMerge merge) throws IO
             totalMergesNumDocs.inc(totalNumDocs);
             totalMergesSizeInBytes.inc(totalSizeInBytes);
             totalMerges.inc(tookMS);
-
-            long stoppedMS = TimeValue.nsecToMSec(merge.rateLimiter.getTotalStoppedNS());
-            long throttledMS = TimeValue.nsecToMSec(merge.rateLimiter.getTotalPausedNS());
-
+            long stoppedMS = TimeValue.nsecToMSec(
+                merge.getMergeProgress().getPauseTimes().get(MergePolicy.OneMergeProgress.PauseReason.STOPPED)
+            );
+            long throttledMS = TimeValue.nsecToMSec(
+                merge.getMergeProgress().getPauseTimes().get(MergePolicy.OneMergeProgress.PauseReason.PAUSED)
+            );
+            final Thread thread = Thread.currentThread();
+            long totalBytesWritten = OneMergeHelper.getTotalBytesWritten(thread, merge);
+            double mbPerSec = OneMergeHelper.getMbPerSec(thread, merge);
             totalMergeStoppedTime.inc(stoppedMS);
             totalMergeThrottledTime.inc(throttledMS);
 
@@ -125,8 +130,8 @@ protected void doMerge(IndexWriter writer, MergePolicy.OneMerge merge) throws IO
                                            totalNumDocs,
                                            TimeValue.timeValueMillis(stoppedMS),
                                            TimeValue.timeValueMillis(throttledMS),
-                                           merge.rateLimiter.getTotalBytesWritten()/1024f/1024f,
-                                           merge.rateLimiter.getMBPerSec());
+                                           totalBytesWritten/1024f/1024f,
+                                           mbPerSec);
 
             if (tookMS > 20000) { // if more than 20 seconds, DEBUG log it
                 logger.debug("{}", message);
diff --git a/core/src/main/java/org/elasticsearch/index/engine/Engine.java b/core/src/main/java/org/elasticsearch/index/engine/Engine.java
index 9df03beb1abce..e1ccf5efcc629 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/Engine.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/Engine.java
@@ -42,6 +42,7 @@
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.util.Accountable;
 import org.apache.lucene.util.Accountables;
+import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -54,15 +55,15 @@
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.lucene.uid.Versions;
+import org.elasticsearch.common.lucene.uid.VersionsResolver;
+import org.elasticsearch.common.lucene.uid.VersionsResolver.DocIdAndVersion;
 import org.elasticsearch.common.metrics.CounterMetric;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.ReleasableLock;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.mapper.ParseContext.Document;
 import org.elasticsearch.index.mapper.ParsedDocument;
-import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.index.merge.MergeStats;
-import org.elasticsearch.index.shard.DocsStats;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.index.translog.Translog;
@@ -76,8 +77,10 @@
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.locks.Condition;
@@ -95,13 +98,14 @@ public abstract class Engine implements Closeable {
     protected final EngineConfig engineConfig;
     protected final Store store;
     protected final AtomicBoolean isClosed = new AtomicBoolean(false);
+    private final CountDownLatch closedLatch = new CountDownLatch(1);
     protected final EventListener eventListener;
     protected final SnapshotDeletionPolicy deletionPolicy;
     protected final ReentrantLock failEngineLock = new ReentrantLock();
     protected final ReentrantReadWriteLock rwl = new ReentrantReadWriteLock();
     protected final ReleasableLock readLock = new ReleasableLock(rwl.readLock());
     protected final ReleasableLock writeLock = new ReleasableLock(rwl.writeLock());
-    protected volatile Exception failedEngine = null;
+    protected final SetOnce<Exception> failedEngine = new SetOnce<>();
     /*
      * on <tt>lastWriteNanos</tt> we use System.nanoTime() to initialize this since:
      *  - we use the value for figuring out if the shard / engine is active so if we startup and no write has happened yet we still consider it active
@@ -233,17 +237,13 @@ boolean isThrottled() {
     /**
      * Returns the number of milliseconds this engine was under index throttling.
      */
-    public long getIndexThrottleTimeInMillis() {
-        return 0;
-    }
+    public abstract long getIndexThrottleTimeInMillis();
 
     /**
      * Returns the <code>true</code> iff this engine is currently under index throttling.
      * @see #getIndexThrottleTimeInMillis()
      */
-    public boolean isThrottled() {
-        return false;
-    }
+    public abstract boolean isThrottled();
 
     /** A Lock implementation that always allows the lock to be acquired */
     protected static final class NoOpLock implements Lock {
@@ -276,9 +276,135 @@ public Condition newCondition() {
         }
     }
 
-    public abstract void index(Index operation) throws EngineException;
+    /**
+     * Perform document index operation on the engine
+     * @param index operation to perform
+     * @return {@link IndexResult} containing updated translog location, version and
+     * document specific failures
+     *
+     * Note: engine level failures (i.e. persistent engine failures) are thrown
+     */
+    public abstract IndexResult index(Index index) throws IOException;
+
+    /**
+     * Perform document delete operation on the engine
+     * @param delete operation to perform
+     * @return {@link DeleteResult} containing updated translog location, version and
+     * document specific failures
+     *
+     * Note: engine level failures (i.e. persistent engine failures) are thrown
+     */
+    public abstract DeleteResult delete(Delete delete) throws IOException;
+
+    /**
+     * Base class for index and delete operation results
+     * Holds result meta data (e.g. translog location, updated version)
+     * for an executed write {@link Operation}
+     **/
+    public abstract static class Result {
+        private final Operation.TYPE operationType;
+        private final long version;
+        private final Exception failure;
+        private final SetOnce<Boolean> freeze = new SetOnce<>();
+        private Translog.Location translogLocation;
+        private long took;
+
+        protected Result(Operation.TYPE operationType, Exception failure, long version) {
+            this.operationType = operationType;
+            this.failure = failure;
+            this.version = version;
+        }
+
+        protected Result(Operation.TYPE operationType, long version) {
+            this(operationType, null, version);
+        }
+
+        /** whether the operation had failure */
+        public boolean hasFailure() {
+            return failure != null;
+        }
+
+        /** get the updated document version */
+        public long getVersion() {
+            return version;
+        }
+
+        /** get the translog location after executing the operation */
+        public Translog.Location getTranslogLocation() {
+            return translogLocation;
+        }
+
+        /** get document failure while executing the operation {@code null} in case of no failure */
+        public Exception getFailure() {
+            return failure;
+        }
+
+        /** get total time in nanoseconds */
+        public long getTook() {
+            return took;
+        }
+
+        public Operation.TYPE getOperationType() {
+            return operationType;
+        }
+
+        void setTranslogLocation(Translog.Location translogLocation) {
+            if (freeze.get() == null) {
+                assert failure == null : "failure has to be null to set translog location";
+                this.translogLocation = translogLocation;
+            } else {
+                throw new IllegalStateException("result is already frozen");
+            }
+        }
+
+        void setTook(long took) {
+            if (freeze.get() == null) {
+                this.took = took;
+            } else {
+                throw new IllegalStateException("result is already frozen");
+            }
+        }
+
+        void freeze() {
+            freeze.set(true);
+        }
+    }
+
+    public static class IndexResult extends Result {
+        private final boolean created;
+
+        public IndexResult(long version, boolean created) {
+            super(Operation.TYPE.INDEX, version);
+            this.created = created;
+        }
+
+        public IndexResult(Exception failure, long version) {
+            super(Operation.TYPE.INDEX, failure, version);
+            this.created = false;
+        }
+
+        public boolean isCreated() {
+            return created;
+        }
+    }
+
+    public static class DeleteResult extends Result {
+        private final boolean found;
 
-    public abstract void delete(Delete delete) throws EngineException;
+        public DeleteResult(long version, boolean found) {
+            super(Operation.TYPE.DELETE, version);
+            this.found = found;
+        }
+
+        public DeleteResult(Exception failure, long version, boolean found) {
+            super(Operation.TYPE.DELETE, failure, version);
+            this.found = found;
+        }
+
+        public boolean isFound() {
+            return found;
+        }
+    }
 
     /**
      * Attempts to do a special commit where the given syncID is put into the commit data. The attempt
@@ -298,9 +424,9 @@ public enum SyncedFlushResult {
 
     protected final GetResult getFromSearcher(Get get, Function<String, Searcher> searcherFactory) throws EngineException {
         final Searcher searcher = searcherFactory.apply("get");
-        final Versions.DocIdAndVersion docIdAndVersion;
+        final DocIdAndVersion docIdAndVersion;
         try {
-            docIdAndVersion = Versions.loadDocIdAndVersion(searcher.reader(), get.uid());
+            docIdAndVersion = VersionsResolver.loadDocIdAndVersion(searcher.reader(), get.uid());
         } catch (Exception e) {
             Releasables.closeWhileHandlingException(searcher);
             //TODO: A better exception goes here
@@ -310,8 +436,7 @@ protected final GetResult getFromSearcher(Get get, Function<String, Searcher> se
         if (docIdAndVersion != null) {
             if (get.versionType().isVersionConflictForReads(docIdAndVersion.version, get.version())) {
                 Releasables.close(searcher);
-                Uid uid = Uid.createUid(get.uid().text());
-                throw new VersionConflictEngineException(shardId, uid.type(), uid.id(),
+                throw new VersionConflictEngineException(shardId, get.type(), get.id(),
                         get.versionType().explainConflictForReads(docIdAndVersion.version, get.version()));
             }
         }
@@ -326,10 +451,6 @@ protected final GetResult getFromSearcher(Get get, Function<String, Searcher> se
         }
     }
 
-    public final GetResult get(Get get) throws EngineException {
-        return get(get, this::acquireSearcher);
-    }
-
     public abstract GetResult get(Get get, Function<String, Searcher> searcherFactory) throws EngineException;
 
     /**
@@ -359,7 +480,7 @@ public final Searcher acquireSearcher(String source) throws EngineException {
                     manager.release(searcher);
                 }
             }
-        } catch (EngineClosedException ex) {
+        } catch (AlreadyClosedException ex) {
             throw ex;
         } catch (Exception ex) {
             ensureOpen(); // throw EngineCloseException here if we are already closed
@@ -377,7 +498,7 @@ public final Searcher acquireSearcher(String source) throws EngineException {
 
     protected void ensureOpen() {
         if (isClosed.get()) {
-            throw new EngineClosedException(shardId, failedEngine);
+            throw new AlreadyClosedException(shardId + " engine is closed", failedEngine.get());
         }
     }
 
@@ -412,7 +533,7 @@ protected static SegmentInfos readLastCommittedSegmentInfos(final SearcherManage
      */
     public final SegmentsStats segmentsStats(boolean includeSegmentFileSizes) {
         ensureOpen();
-        try (final Searcher searcher = acquireSearcher("segments_stats")) {
+        try (Searcher searcher = acquireSearcher("segments_stats")) {
             SegmentsStats stats = new SegmentsStats();
             for (LeafReaderContext reader : searcher.reader().leaves()) {
                 final SegmentReader segmentReader = segmentReader(reader.reader());
@@ -478,7 +599,9 @@ private ImmutableOpenMap<String, Long> getSegmentFileSizes(SegmentReader segment
             try {
                 length = directory.fileLength(file);
             } catch (NoSuchFileException | FileNotFoundException e) {
-                logger.warn("Tried to query fileLength but file is gone [{}] [{}]", e, directory, file);
+                final Directory finalDirectory = directory;
+                logger.warn((Supplier<?>)
+                    () -> new ParameterizedMessage("Tried to query fileLength but file is gone [{}] [{}]", finalDirectory, file), e);
             } catch (IOException e) {
                 final Directory finalDirectory = directory;
                 logger.warn(
@@ -670,17 +793,19 @@ public void failEngine(String reason, @Nullable Exception failure) {
         if (failEngineLock.tryLock()) {
             store.incRef();
             try {
+                if (failedEngine.get() != null) {
+                    logger.warn((Supplier<?>) () -> new ParameterizedMessage("tried to fail engine but engine is already failed. ignoring. [{}]", reason), failure);
+                    return;
+                }
+                // this must happen before we close IW or Translog such that we can check this state to opt out of failing the engine
+                // again on any caught AlreadyClosedException
+                failedEngine.set((failure != null) ? failure : new IllegalStateException(reason));
                 try {
                     // we just go and close this engine - no way to recover
-                    closeNoLock("engine failed on: [" + reason + "]");
+                    closeNoLock("engine failed on: [" + reason + "]", closedLatch);
                 } finally {
-                    if (failedEngine != null) {
-                        logger.debug((Supplier<?>) () -> new ParameterizedMessage("tried to fail engine but engine is already failed. ignoring. [{}]", reason), failure);
-                        return;
-                    }
                     logger.warn((Supplier<?>) () -> new ParameterizedMessage("failed engine [{}]", reason), failure);
                     // we must set a failure exception, generate one if not supplied
-                    failedEngine = (failure != null) ? failure : new IllegalStateException(reason);
                     // we first mark the store as corrupted before we notify any listeners
                     // this must happen first otherwise we might try to reallocate so quickly
                     // on the same node that we don't see the corrupted marker file when
@@ -762,13 +887,27 @@ public void close() {
     }
 
     public abstract static class Operation {
+
+        /** type of operation (index, delete), subclasses use static types */
+        public enum TYPE {
+            INDEX, DELETE;
+
+            private final String lowercase;
+
+            TYPE() {
+                this.lowercase = this.toString().toLowerCase(Locale.ROOT);
+            }
+
+            public String getLowercase() {
+                return lowercase;
+            }
+        }
+
         private final Term uid;
-        private long version;
+        private final long version;
         private final VersionType versionType;
         private final Origin origin;
-        private Translog.Location location;
         private final long startTime;
-        private long endTime;
 
         public Operation(Term uid, long version, VersionType versionType, Origin origin, long startTime) {
             this.uid = uid;
@@ -801,27 +940,7 @@ public long version() {
             return this.version;
         }
 
-        public void updateVersion(long version) {
-            this.version = version;
-        }
-
-        public void setTranslogLocation(Translog.Location location) {
-            this.location = location;
-        }
-
-        public Translog.Location getTranslogLocation() {
-            return this.location;
-        }
-
-        public int sizeInBytes() {
-            if (location != null) {
-                return location.size;
-            } else {
-                return estimatedSizeInBytes();
-            }
-        }
-
-        protected abstract int estimatedSizeInBytes();
+        public abstract int estimatedSizeInBytes();
 
         public VersionType versionType() {
             return this.versionType;
@@ -834,20 +953,11 @@ public long startTime() {
             return this.startTime;
         }
 
-        public void endTime(long endTime) {
-            this.endTime = endTime;
-        }
-
-        /**
-         * Returns operation end time in nanoseconds.
-         */
-        public long endTime() {
-            return this.endTime;
-        }
-
-        abstract String type();
+        public abstract String type();
 
         abstract String id();
+
+        abstract TYPE operationType();
     }
 
     public static class Index extends Operation {
@@ -855,7 +965,6 @@ public static class Index extends Operation {
         private final ParsedDocument doc;
         private final long autoGeneratedIdTimestamp;
         private final boolean isRetry;
-        private boolean created;
 
         public Index(Term uid, ParsedDocument doc, long version, VersionType versionType, Origin origin, long startTime,
                      long autoGeneratedIdTimestamp, boolean isRetry) {
@@ -887,6 +996,11 @@ public String id() {
             return this.doc.id();
         }
 
+        @Override
+        TYPE operationType() {
+            return TYPE.INDEX;
+        }
+
         public String routing() {
             return this.doc.routing();
         }
@@ -899,12 +1013,6 @@ public long ttl() {
             return this.doc.ttl();
         }
 
-        @Override
-        public void updateVersion(long version) {
-            super.updateVersion(version);
-            this.doc.version().setLongValue(version);
-        }
-
         public String parent() {
             return this.doc.parent();
         }
@@ -917,16 +1025,8 @@ public BytesReference source() {
             return this.doc.source();
         }
 
-        public boolean isCreated() {
-            return created;
-        }
-
-        public void setCreated(boolean created) {
-            this.created = created;
-        }
-
         @Override
-        protected int estimatedSizeInBytes() {
+        public int estimatedSizeInBytes() {
             return (id().length() + type().length()) * 2 + source().length() + 12;
         }
 
@@ -953,21 +1053,19 @@ public static class Delete extends Operation {
 
         private final String type;
         private final String id;
-        private boolean found;
 
-        public Delete(String type, String id, Term uid, long version, VersionType versionType, Origin origin, long startTime, boolean found) {
+        public Delete(String type, String id, Term uid, long version, VersionType versionType, Origin origin, long startTime) {
             super(uid, version, versionType, origin, startTime);
             this.type = type;
             this.id = id;
-            this.found = found;
         }
 
         public Delete(String type, String id, Term uid) {
-            this(type, id, uid, Versions.MATCH_ANY, VersionType.INTERNAL, Origin.PRIMARY, System.nanoTime(), false);
+            this(type, id, uid, Versions.MATCH_ANY, VersionType.INTERNAL, Origin.PRIMARY, System.nanoTime());
         }
 
         public Delete(Delete template, VersionType versionType) {
-            this(template.type(), template.id(), template.uid(), template.version(), versionType, template.origin(), template.startTime(), template.found());
+            this(template.type(), template.id(), template.uid(), template.version(), versionType, template.origin(), template.startTime());
         }
 
         @Override
@@ -980,30 +1078,28 @@ public String id() {
             return this.id;
         }
 
-        public void updateVersion(long version, boolean found) {
-            updateVersion(version);
-            this.found = found;
-        }
-
-        public boolean found() {
-            return this.found;
+        @Override
+        TYPE operationType() {
+            return TYPE.DELETE;
         }
 
         @Override
-        protected int estimatedSizeInBytes() {
+        public int estimatedSizeInBytes() {
             return (uid().field().length() + uid().text().length()) * 2 + 20;
         }
-
     }
 
     public static class Get {
         private final boolean realtime;
         private final Term uid;
+        private final String type, id;
         private long version = Versions.MATCH_ANY;
         private VersionType versionType = VersionType.INTERNAL;
 
-        public Get(boolean realtime, Term uid) {
+        public Get(boolean realtime, String type, String id, Term uid) {
             this.realtime = realtime;
+            this.type = type;
+            this.id = id;
             this.uid = uid;
         }
 
@@ -1011,6 +1107,14 @@ public boolean realtime() {
             return this.realtime;
         }
 
+        public String type() {
+            return type;
+        }
+
+        public String id() {
+            return id;
+        }
+
         public Term uid() {
             return uid;
         }
@@ -1037,12 +1141,12 @@ public Get versionType(VersionType versionType) {
     public static class GetResult implements Releasable {
         private final boolean exists;
         private final long version;
-        private final Versions.DocIdAndVersion docIdAndVersion;
+        private final DocIdAndVersion docIdAndVersion;
         private final Searcher searcher;
 
         public static final GetResult NOT_EXISTS = new GetResult(false, Versions.NOT_FOUND, null, null);
 
-        private GetResult(boolean exists, long version, Versions.DocIdAndVersion docIdAndVersion, Searcher searcher) {
+        private GetResult(boolean exists, long version, DocIdAndVersion docIdAndVersion, Searcher searcher) {
             this.exists = exists;
             this.version = version;
             this.docIdAndVersion = docIdAndVersion;
@@ -1052,7 +1156,7 @@ private GetResult(boolean exists, long version, Versions.DocIdAndVersion docIdAn
         /**
          * Build a non-realtime get result from the searcher.
          */
-        public GetResult(Searcher searcher, Versions.DocIdAndVersion docIdAndVersion) {
+        public GetResult(Searcher searcher, DocIdAndVersion docIdAndVersion) {
             this(true, docIdAndVersion.version, docIdAndVersion, searcher);
         }
 
@@ -1068,7 +1172,7 @@ public Searcher searcher() {
             return this.searcher;
         }
 
-        public Versions.DocIdAndVersion docIdAndVersion() {
+        public DocIdAndVersion docIdAndVersion() {
             return docIdAndVersion;
         }
 
@@ -1086,8 +1190,10 @@ public void release() {
 
     /**
      * Method to close the engine while the write lock is held.
+     * Must decrement the supplied when closing work is done and resources are
+     * freed.
      */
-    protected abstract void closeNoLock(String reason);
+    protected abstract void closeNoLock(String reason, CountDownLatch closedLatch);
 
     /**
      * Flush the engine (committing segments to disk and truncating the
@@ -1102,9 +1208,7 @@ public void flushAndClose() throws IOException {
                     logger.debug("flushing shard on close - this might take some time to sync files to disk");
                     try {
                         flush(); // TODO we might force a flush in the future since we have the write lock already even though recoveries are running.
-                    } catch (FlushNotAllowedEngineException ex) {
-                        logger.debug("flush not allowed during flushAndClose - skipping");
-                    } catch (EngineClosedException ex) {
+                    } catch (AlreadyClosedException ex) {
                         logger.debug("engine already closed - skipping flushAndClose");
                     }
                 } finally {
@@ -1112,6 +1216,7 @@ public void flushAndClose() throws IOException {
                 }
             }
         }
+        awaitPendingClose();
     }
 
     @Override
@@ -1120,9 +1225,18 @@ public void close() throws IOException {
             logger.debug("close now acquiring writeLock");
             try (ReleasableLock lock = writeLock.acquire()) {
                 logger.debug("close acquired writeLock");
-                closeNoLock("api");
+                closeNoLock("api", closedLatch);
             }
         }
+        awaitPendingClose();
+    }
+
+    private void awaitPendingClose() {
+        try {
+            closedLatch.await();
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+        }
     }
 
     public static class CommitId implements Writeable {
@@ -1194,16 +1308,6 @@ public long getLastWriteNanos() {
         return this.lastWriteNanos;
     }
 
-    /**
-     * Returns the engines current document statistics
-     */
-    public DocsStats getDocStats() {
-        try (Engine.Searcher searcher = acquireSearcher("doc_stats")) {
-            IndexReader reader = searcher.reader();
-            return new DocsStats(reader.numDocs(), reader.numDeletedDocs());
-        }
-    }
-
     /**
      * Called for each new opened engine searcher to warm new segments
      * @see EngineConfig#getWarmer()
@@ -1230,4 +1334,11 @@ public interface Warmer {
      * This operation will close the engine if the recovery fails.
      */
     public abstract Engine recoverFromTranslog() throws IOException;
+
+    /**
+     * Returns <code>true</code> iff this engine is currently recovering from translog.
+     */
+    public boolean isRecovering() {
+        return false;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/engine/EngineClosedException.java b/core/src/main/java/org/elasticsearch/index/engine/EngineClosedException.java
index 3ecae762a2adb..cc3ffcac6b0b9 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/EngineClosedException.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/EngineClosedException.java
@@ -33,6 +33,7 @@
  *
  *
  */
+@Deprecated
 public class EngineClosedException extends IndexShardClosedException {
 
     public EngineClosedException(ShardId shardId) {
diff --git a/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java b/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java
index 9f9d2186a835c..02967514ae55e 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java
@@ -24,9 +24,9 @@
 import org.apache.lucene.index.SnapshotDeletionPolicy;
 import org.apache.lucene.search.QueryCache;
 import org.apache.lucene.search.QueryCachingPolicy;
+import org.apache.lucene.search.ReferenceManager;
 import org.apache.lucene.search.similarities.Similarity;
 import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.unit.ByteSizeUnit;
@@ -34,7 +34,6 @@
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.codec.CodecService;
-import org.elasticsearch.index.shard.RefreshListeners;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.TranslogRecoveryPerformer;
 import org.elasticsearch.index.store.Store;
@@ -42,6 +41,8 @@
 import org.elasticsearch.indices.IndexingMemoryController;
 import org.elasticsearch.threadpool.ThreadPool;
 
+import java.util.List;
+
 /*
  * Holds all the configuration that is used to create an {@link Engine}.
  * Once {@link Engine} has been created with this object, changes to this
@@ -67,8 +68,7 @@ public final class EngineConfig {
     private final QueryCache queryCache;
     private final QueryCachingPolicy queryCachingPolicy;
     private final long maxUnsafeAutoIdTimestamp;
-    @Nullable
-    private final RefreshListeners refreshListeners;
+    private final List<ReferenceManager.RefreshListener> refreshListeners;
 
     /**
      * Index setting to change the low level lucene codec used for writing new segments.
@@ -112,7 +112,7 @@ public EngineConfig(OpenMode openMode, ShardId shardId, ThreadPool threadPool,
                         MergePolicy mergePolicy, Analyzer analyzer,
                         Similarity similarity, CodecService codecService, Engine.EventListener eventListener,
                         TranslogRecoveryPerformer translogRecoveryPerformer, QueryCache queryCache, QueryCachingPolicy queryCachingPolicy,
-                        TranslogConfig translogConfig, TimeValue flushMergesAfter, RefreshListeners refreshListeners,
+                        TranslogConfig translogConfig, TimeValue flushMergesAfter, List<ReferenceManager.RefreshListener> refreshListeners,
                         long maxUnsafeAutoIdTimestamp) {
         if (openMode == null) {
             throw new IllegalArgumentException("openMode must not be null");
@@ -188,7 +188,7 @@ public Codec getCodec() {
 
     /**
      * Returns a thread-pool mainly used to get estimated time stamps from
-     * {@link org.elasticsearch.threadpool.ThreadPool#estimatedTimeInMillis()} and to schedule
+     * {@link org.elasticsearch.threadpool.ThreadPool#relativeTimeInMillis()} and to schedule
      * async force merge calls on the {@link org.elasticsearch.threadpool.ThreadPool.Names#FORCE_MERGE} thread-pool
      */
     public ThreadPool getThreadPool() {
@@ -322,9 +322,9 @@ public enum OpenMode {
     }
 
     /**
-     * {@linkplain RefreshListeners} instance to configure.
+     * The refresh listeners to add to Lucene
      */
-    public RefreshListeners getRefreshListeners() {
+    public List<ReferenceManager.RefreshListener> getRefreshListeners() {
         return refreshListeners;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/engine/FlushNotAllowedEngineException.java b/core/src/main/java/org/elasticsearch/index/engine/FlushNotAllowedEngineException.java
deleted file mode 100644
index d9371707e3bd1..0000000000000
--- a/core/src/main/java/org/elasticsearch/index/engine/FlushNotAllowedEngineException.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.engine;
-
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.rest.RestStatus;
-
-import java.io.IOException;
-
-/**
- *
- */
-public class FlushNotAllowedEngineException extends EngineException {
-
-    public FlushNotAllowedEngineException(ShardId shardId, String msg) {
-        super(shardId, msg);
-    }
-
-    public FlushNotAllowedEngineException(StreamInput in) throws IOException{
-        super(in);
-    }
-
-    @Override
-    public RestStatus status() {
-        return RestStatus.SERVICE_UNAVAILABLE;
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/index/engine/IndexFailedEngineException.java b/core/src/main/java/org/elasticsearch/index/engine/IndexFailedEngineException.java
index 4728b7f899a5a..2eab6bbd17d27 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/IndexFailedEngineException.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/IndexFailedEngineException.java
@@ -27,22 +27,17 @@
 import java.util.Objects;
 
 /**
- *
+ * Deprecated as not used in 6.0, should be removed in 7.0
+ * Still exists for bwc in serializing/deserializing from
+ * 5.x nodes
  */
+@Deprecated
 public class IndexFailedEngineException extends EngineException {
 
     private final String type;
 
     private final String id;
 
-    public IndexFailedEngineException(ShardId shardId, String type, String id, Throwable cause) {
-        super(shardId, "Index failed for [" + type + "#" + id + "]", cause);
-        Objects.requireNonNull(type, "type must not be null");
-        Objects.requireNonNull(id, "id must not be null");
-        this.type = type;
-        this.id = id;
-    }
-
     public IndexFailedEngineException(StreamInput in) throws IOException{
         super(in);
         type = in.readString();
@@ -63,4 +58,4 @@ public String type() {
     public String id() {
         return this.id;
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
index 774465fb71a4c..8523c7b872ad6 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
@@ -33,8 +33,10 @@
 import org.apache.lucene.index.SegmentInfos;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.ReferenceManager;
 import org.apache.lucene.search.SearcherFactory;
 import org.apache.lucene.search.SearcherManager;
+import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.LockObtainFailedException;
@@ -50,6 +52,7 @@
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
 import org.elasticsearch.common.lucene.uid.Versions;
+import org.elasticsearch.common.lucene.uid.VersionsResolver;
 import org.elasticsearch.common.metrics.CounterMetric;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
@@ -57,10 +60,11 @@
 import org.elasticsearch.common.util.concurrent.ReleasableLock;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.VersionType;
-import org.elasticsearch.index.mapper.Uid;
+import org.elasticsearch.index.mapper.IdFieldMapper;
+import org.elasticsearch.index.mapper.ParseContext;
+import org.elasticsearch.index.mapper.UidFieldMapper;
 import org.elasticsearch.index.merge.MergeStats;
 import org.elasticsearch.index.merge.OnGoingMerge;
-import org.elasticsearch.index.shard.DocsStats;
 import org.elasticsearch.index.shard.ElasticsearchMergePolicy;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.TranslogRecoveryPerformer;
@@ -74,7 +78,10 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
@@ -82,9 +89,6 @@
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.function.Function;
 
-/**
- *
- */
 public class InternalEngine extends Engine {
     /**
      * When we last pruned expired tombstones from versionMap.deletes:
@@ -114,12 +118,15 @@ public class InternalEngine extends Engine {
 
     private final IndexThrottle throttle;
 
+    private final String uidField;
+
     // How many callers are currently requesting index throttling.  Currently there are only two situations where we do this: when merges
     // are falling behind and when writing indexing buffer to disk is too slow.  When this is 0, there is no throttling, else we throttling
     // incoming indexing ops to a single thread:
     private final AtomicInteger throttleRequestCount = new AtomicInteger();
     private final EngineConfig.OpenMode openMode;
-    private final AtomicBoolean allowCommits = new AtomicBoolean(true);
+    private final AtomicBoolean pendingTranslogRecovery = new AtomicBoolean(false);
+    private static final String MAX_UNSAFE_AUTO_ID_TIMESTAMP_COMMIT_ID = "max_unsafe_auto_id_timestamp";
     private final AtomicLong maxUnsafeAutoIdTimestamp = new AtomicLong(-1);
     private final CounterMetric numVersionLookups = new CounterMetric();
     private final CounterMetric numIndexVersionsLookups = new CounterMetric();
@@ -127,12 +134,13 @@ public class InternalEngine extends Engine {
     public InternalEngine(EngineConfig engineConfig) throws EngineException {
         super(engineConfig);
         openMode = engineConfig.getOpenMode();
-        if (engineConfig.getIndexSettings().getIndexVersionCreated().before(Version.V_5_0_0_alpha6)) {
+        if (engineConfig.getIndexSettings().getIndexVersionCreated().before(Version.V_5_0_0_beta1)) {
             // no optimization for pre 5.0.0.alpha6 since translog might not have all information needed
             maxUnsafeAutoIdTimestamp.set(Long.MAX_VALUE);
         } else {
             maxUnsafeAutoIdTimestamp.set(engineConfig.getMaxUnsafeAutoIdTimestamp());
         }
+        this.uidField = engineConfig.getIndexSettings().isSingleType() ? IdFieldMapper.NAME : UidFieldMapper.NAME;
         this.versionMap = new LiveVersionMap();
         store.incRef();
         IndexWriter writer = null;
@@ -141,12 +149,13 @@ public InternalEngine(EngineConfig engineConfig) throws EngineException {
         EngineMergeScheduler scheduler = null;
         boolean success = false;
         try {
-            this.lastDeleteVersionPruneTimeMSec = engineConfig.getThreadPool().estimatedTimeInMillis();
+            this.lastDeleteVersionPruneTimeMSec = engineConfig.getThreadPool().relativeTimeInMillis();
             mergeScheduler = scheduler = new EngineMergeScheduler(engineConfig.getShardId(), engineConfig.getIndexSettings());
             throttle = new IndexThrottle();
             this.searcherFactory = new SearchFactory(logger, isClosed, engineConfig);
             try {
                 writer = createWriter(openMode == EngineConfig.OpenMode.CREATE_INDEX_AND_TRANSLOG);
+                updateMaxUnsafeAutoIdTimestampFromWriter(writer);
                 indexWriter = writer;
                 translog = openTranslog(engineConfig, writer);
                 assert translog.getGeneration() != null;
@@ -166,11 +175,11 @@ public InternalEngine(EngineConfig engineConfig) throws EngineException {
             manager = createSearcherManager();
             this.searcherManager = manager;
             this.versionMap.setManager(searcherManager);
+            assert pendingTranslogRecovery.get() == false : "translog recovery can't be pending before we set it";
             // don't allow commits until we are done with recovering
-            allowCommits.compareAndSet(true, openMode != EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG);
-            if (engineConfig.getRefreshListeners() != null) {
-                searcherManager.addListener(engineConfig.getRefreshListeners());
-                engineConfig.getRefreshListeners().setTranslog(translog);
+            pendingTranslogRecovery.set(openMode == EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG);
+            for (ReferenceManager.RefreshListener listener: engineConfig.getRefreshListeners()) {
+                searcherManager.addListener(listener);
             }
             success = true;
         } finally {
@@ -186,6 +195,17 @@ public InternalEngine(EngineConfig engineConfig) throws EngineException {
         logger.trace("created new InternalEngine");
     }
 
+    private void updateMaxUnsafeAutoIdTimestampFromWriter(IndexWriter writer) {
+        long commitMaxUnsafeAutoIdTimestamp = Long.MIN_VALUE;
+        for (Map.Entry<String, String> entry : writer.getLiveCommitData()) {
+            if (entry.getKey().equals(MAX_UNSAFE_AUTO_ID_TIMESTAMP_COMMIT_ID)) {
+                commitMaxUnsafeAutoIdTimestamp = Long.parseLong(entry.getValue());
+                break;
+            }
+        }
+        maxUnsafeAutoIdTimestamp.set(Math.max(maxUnsafeAutoIdTimestamp.get(), commitMaxUnsafeAutoIdTimestamp));
+    }
+
     @Override
     public InternalEngine recoverFromTranslog() throws IOException {
         flushLock.lock();
@@ -194,14 +214,14 @@ public InternalEngine recoverFromTranslog() throws IOException {
             if (openMode != EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG) {
                 throw new IllegalStateException("Can't recover from translog with open mode: " + openMode);
             }
-            if (allowCommits.get()) {
+            if (pendingTranslogRecovery.get() == false) {
                 throw new IllegalStateException("Engine has already been recovered");
             }
             try {
                 recoverFromTranslog(engineConfig.getTranslogRecoveryPerformer());
             } catch (Exception e) {
                 try {
-                    allowCommits.set(false); // just play safe and never allow commits on this
+                    pendingTranslogRecovery.set(true); // just play safe and never allow commits on this see #ensureCanFlush
                     failEngine("failed to recover from translog", e);
                 } catch (Exception inner) {
                     e.addSuppressed(inner);
@@ -225,8 +245,8 @@ private void recoverFromTranslog(TranslogRecoveryPerformer handler) throws IOExc
         }
         // flush if we recovered something or if we have references to older translogs
         // note: if opsRecovered == 0 and we have older translogs it means they are corrupted or 0 length.
-        assert allowCommits.get() == false : "commits are allowed but shouldn't";
-        allowCommits.set(true); // we are good - now we can commit
+        assert pendingTranslogRecovery.get() : "translogRecovery is not pending but should be";
+        pendingTranslogRecovery.set(false); // we are good - now we can commit
         if (opsRecovered > 0) {
             logger.trace("flushing post recovery from translog. ops recovered [{}]. committed translog id [{}]. current id [{}]",
                 opsRecovered, translogGeneration == null ? null : translogGeneration.translogFileGeneration, translog.currentFileGeneration());
@@ -329,18 +349,18 @@ private SearcherManager createSearcherManager() throws EngineException {
 
     @Override
     public GetResult get(Get get, Function<String, Searcher> searcherFactory) throws EngineException {
+        assert Objects.equals(get.uid().field(), uidField) : get.uid().field();
         try (ReleasableLock lock = readLock.acquire()) {
             ensureOpen();
             if (get.realtime()) {
                 VersionValue versionValue = versionMap.getUnderLock(get.uid());
                 if (versionValue != null) {
-                    if (versionValue.delete()) {
+                    if (versionValue.isDelete()) {
                         return GetResult.NOT_EXISTS;
                     }
-                    if (get.versionType().isVersionConflictForReads(versionValue.version(), get.version())) {
-                        Uid uid = Uid.createUid(get.uid().text());
-                        throw new VersionConflictEngineException(shardId, uid.type(), uid.id(),
-                                get.versionType().explainConflictForReads(versionValue.version(), get.version()));
+                    if (get.versionType().isVersionConflictForReads(versionValue.getVersion(), get.version())) {
+                        throw new VersionConflictEngineException(shardId, get.type(), get.id(),
+                            get.versionType().explainConflictForReads(versionValue.getVersion(), get.version()));
                     }
                     refresh("realtime_get");
                 }
@@ -351,73 +371,46 @@ public GetResult get(Get get, Function<String, Searcher> searcherFactory) throws
         }
     }
 
-    private boolean checkVersionConflict(
-            final Operation op,
-            final long currentVersion,
-            final long expectedVersion,
-            final boolean deleted) {
-        if (op.versionType().isVersionConflictForWrites(currentVersion, expectedVersion, deleted)) {
-            if (op.origin().isRecovery()) {
-                // version conflict, but okay
-                return true;
-            } else {
-                // fatal version conflict
-                throw new VersionConflictEngineException(shardId, op.type(), op.id(),
-                        op.versionType().explainConflictForWrites(currentVersion, expectedVersion, deleted));
-            }
-        }
-        return false;
-    }
-
-    private long checkDeletedAndGCed(VersionValue versionValue) {
-        long currentVersion;
-        if (engineConfig.isEnableGcDeletes() && versionValue.delete() && (engineConfig.getThreadPool().estimatedTimeInMillis() - versionValue.time()) > getGcDeletesInMillis()) {
-            currentVersion = Versions.NOT_FOUND; // deleted, and GC
-        } else {
-            currentVersion = versionValue.version();
-        }
-        return currentVersion;
-    }
-
-    private static VersionValueSupplier NEW_VERSION_VALUE = (u, t) -> new VersionValue(u);
-
-    @FunctionalInterface
-    private interface VersionValueSupplier {
-        VersionValue apply(long updatedVersion, long time);
+    /**
+     * the status of the current doc version in lucene, compared to the version in an incoming
+     * operation
+     */
+    enum OpVsLuceneDocStatus {
+        /** the op is more recent than the one that last modified the doc found in lucene */
+        OP_NEWER,
+        /** the op is older or the same as the one that last modified the doc found in lucene */
+        OP_STALE_OR_EQUAL,
+        /** no doc was found in lucene */
+        LUCENE_DOC_NOT_FOUND
     }
 
-    private <T extends Engine.Operation> void maybeAddToTranslog(
-            final T op,
-            final long updatedVersion,
-            final Function<T, Translog.Operation> toTranslogOp,
-            final VersionValueSupplier toVersionValue) throws IOException {
-        if (op.origin() != Operation.Origin.LOCAL_TRANSLOG_RECOVERY) {
-            final Translog.Location translogLocation = translog.add(toTranslogOp.apply(op));
-            op.setTranslogLocation(translogLocation);
+    /** resolves the current version of the document, returning null if not found */
+    private VersionValue resolveDocVersion(final Operation op) throws IOException {
+        assert incrementVersionLookup(); // used for asserting in tests
+        VersionValue versionValue = versionMap.getUnderLock(op.uid());
+        if (versionValue == null) {
+            assert incrementIndexVersionLookup(); // used for asserting in tests
+            final long currentVersion = loadCurrentVersionFromIndex(op.uid());
+            if (currentVersion != Versions.NOT_FOUND) {
+                versionValue = new VersionValue(currentVersion);
+            }
+        } else if (engineConfig.isEnableGcDeletes() && versionValue.isDelete() &&
+            (engineConfig.getThreadPool().relativeTimeInMillis() - versionValue.getTime()) >
+                getGcDeletesInMillis()) {
+            versionValue = null;
         }
-        versionMap.putUnderLock(op.uid().bytes(), toVersionValue.apply(updatedVersion, engineConfig.getThreadPool().estimatedTimeInMillis()));
-
+        return versionValue;
     }
 
-    @Override
-    public void index(Index index) {
-        try (ReleasableLock lock = readLock.acquire()) {
-            ensureOpen();
-            if (index.origin().isRecovery()) {
-                // Don't throttle recovery operations
-                innerIndex(index);
-            } else {
-                try (Releasable r = throttle.acquireThrottle()) {
-                    innerIndex(index);
-                }
-            }
-        } catch (IllegalStateException | IOException e) {
-            try {
-                maybeFailEngine("index", e);
-            } catch (Exception inner) {
-                e.addSuppressed(inner);
-            }
-            throw new IndexFailedEngineException(shardId, index.type(), index.id(), e);
+    private OpVsLuceneDocStatus compareOpToLuceneDocBasedOnVersions(final Operation op)
+        throws IOException {
+        assert op.version() >= 0 : "versions should be non-negative. got " + op.version();
+        final VersionValue versionValue = resolveDocVersion(op);
+        if (versionValue == null) {
+            return OpVsLuceneDocStatus.LUCENE_DOC_NOT_FOUND;
+        } else {
+            return op.versionType().isVersionConflictForWrites(versionValue.getVersion(), op.version(), versionValue.isDelete()) ?
+                OpVsLuceneDocStatus.OP_STALE_OR_EQUAL : OpVsLuceneDocStatus.OP_NEWER;
         }
     }
 
@@ -433,11 +426,11 @@ private boolean canOptimizeAddDocument(Index index) {
                 case PEER_RECOVERY:
                 case REPLICA:
                     assert index.version() == 1 && index.versionType() == VersionType.EXTERNAL
-                    : "version: " + index.version() + " type: " + index.versionType();
+                        : "version: " + index.version() + " type: " + index.versionType();
                     return true;
                 case LOCAL_TRANSLOG_RECOVERY:
                     assert index.isRetry();
-                    return false; // even if retry is set we never optimize local recovery
+                    return true; // allow to optimize in order to update the max safe time stamp
                 default:
                     throw new IllegalArgumentException("unknown origin " + index.origin());
             }
@@ -445,176 +438,466 @@ private boolean canOptimizeAddDocument(Index index) {
         return false;
     }
 
-    private void innerIndex(Index index) throws IOException {
-        try (Releasable ignored = acquireLock(index.uid())) {
-            lastWriteNanos = index.startTime();
-            /* if we have an autoGeneratedID that comes into the engine we can potentially optimize
-             * and just use addDocument instead of updateDocument and skip the entire version and index lookup across the board.
-             * Yet, we have to deal with multiple document delivery, for this we use a property of the document that is added
-             * to detect if it has potentially been added before. We use the documents timestamp for this since it's something
-             * that:
-             *  - doesn't change per document
-             *  - is preserved in the transaction log
-             *  - and is assigned before we start to index / replicate
-             * NOTE: it's not important for this timestamp to be consistent across nodes etc. it's just a number that is in the common
-             * case increasing and can be used in the failure case when we retry and resent documents to establish a happens before relationship.
-             * for instance:
-             *  - doc A has autoGeneratedIdTimestamp = 10, isRetry = false
-             *  - doc B has autoGeneratedIdTimestamp = 9, isRetry = false
-             *
-             *  while both docs are in in flight, we disconnect on one node, reconnect and send doc A again
-             *  - now doc A' has autoGeneratedIdTimestamp = 10, isRetry = true
-             *
-             *  if A' arrives on the shard first we update maxUnsafeAutoIdTimestamp to 10 and use update document. All subsequent
-             *  documents that arrive (A and B) will also use updateDocument since their timestamps are less than maxUnsafeAutoIdTimestamp.
-             *  While this is not strictly needed for doc B it is just much simpler to implement since it will just de-optimize some doc in the worst case.
-             *
-             *  if A arrives on the shard first we use addDocument since maxUnsafeAutoIdTimestamp is < 10. A` will then just be skipped or calls
-             *  updateDocument.
-             */
-            long currentVersion;
-            final boolean deleted;
-            // if anything is fishy here ie. there is a retry we go and force updateDocument below so we are updating the document in the
-            // lucene index without checking the version map but we still do the version check
-            final boolean forceUpdateDocument;
-            if (canOptimizeAddDocument(index)) {
-                long deOptimizeTimestamp = maxUnsafeAutoIdTimestamp.get();
-                if (index.isRetry()) {
-                    forceUpdateDocument = true;
-                    do {
-                        deOptimizeTimestamp = maxUnsafeAutoIdTimestamp.get();
-                        if (deOptimizeTimestamp >= index.getAutoGeneratedIdTimestamp()) {
-                            break;
-                        }
-                    } while(maxUnsafeAutoIdTimestamp.compareAndSet(deOptimizeTimestamp,
-                        index.getAutoGeneratedIdTimestamp()) == false);
-                    assert maxUnsafeAutoIdTimestamp.get() >= index.getAutoGeneratedIdTimestamp();
+    private boolean assertVersionType(final Engine.Operation operation) {
+        if (operation.origin() == Operation.Origin.REPLICA ||
+            operation.origin() == Operation.Origin.PEER_RECOVERY ||
+            operation.origin() == Operation.Origin.LOCAL_TRANSLOG_RECOVERY) {
+            // ensure that replica operation has expected version type for replication
+            // ensure that versionTypeForReplicationAndRecovery is idempotent
+            assert operation.versionType() == operation.versionType().versionTypeForReplicationAndRecovery()
+                : "unexpected version type in request from [" + operation.origin().name() + "] " +
+                "found [" + operation.versionType().name() + "] " +
+                "expected [" + operation.versionType().versionTypeForReplicationAndRecovery().name() + "]";
+        }
+        return true;
+    }
+
+    @Override
+    public IndexResult index(Index index) throws IOException {
+        assert Objects.equals(index.uid().field(), uidField) : index.uid().field();
+        final boolean doThrottle = index.origin().isRecovery() == false;
+        try (ReleasableLock releasableLock = readLock.acquire()) {
+            ensureOpen();
+            assert assertVersionType(index);
+            try (Releasable ignored = acquireLock(index.uid());
+                 Releasable indexThrottle = doThrottle ? () -> {} : throttle.acquireThrottle()) {
+                lastWriteNanos = index.startTime();
+                /* A NOTE ABOUT APPEND ONLY OPTIMIZATIONS:
+                 * if we have an autoGeneratedID that comes into the engine we can potentially optimize
+                 * and just use addDocument instead of updateDocument and skip the entire version and index lookupVersion across the board.
+                 * Yet, we have to deal with multiple document delivery, for this we use a property of the document that is added
+                 * to detect if it has potentially been added before. We use the documents timestamp for this since it's something
+                 * that:
+                 *  - doesn't change per document
+                 *  - is preserved in the transaction log
+                 *  - and is assigned before we start to index / replicate
+                 * NOTE: it's not important for this timestamp to be consistent across nodes etc. it's just a number that is in the common
+                 * case increasing and can be used in the failure case when we retry and resent documents to establish a happens before relationship.
+                 * for instance:
+                 *  - doc A has autoGeneratedIdTimestamp = 10, isRetry = false
+                 *  - doc B has autoGeneratedIdTimestamp = 9, isRetry = false
+                 *
+                 *  while both docs are in in flight, we disconnect on one node, reconnect and send doc A again
+                 *  - now doc A' has autoGeneratedIdTimestamp = 10, isRetry = true
+                 *
+                 *  if A' arrives on the shard first we update maxUnsafeAutoIdTimestamp to 10 and use update document. All subsequent
+                 *  documents that arrive (A and B) will also use updateDocument since their timestamps are less than maxUnsafeAutoIdTimestamp.
+                 *  While this is not strictly needed for doc B it is just much simpler to implement since it will just de-optimize some doc in the worst case.
+                 *
+                 *  if A arrives on the shard first we use addDocument since maxUnsafeAutoIdTimestamp is < 10. A` will then just be skipped or calls
+                 *  updateDocument.
+                 */
+                final IndexingStrategy plan;
+
+                if (index.origin() == Operation.Origin.PRIMARY) {
+                    plan = planIndexingAsPrimary(index);
                 } else {
-                    // in this case we force
-                    forceUpdateDocument = deOptimizeTimestamp >= index.getAutoGeneratedIdTimestamp();
+                    // non-primary mode (i.e., replica or recovery)
+                    plan = planIndexingAsNonPrimary(index);
                 }
-                currentVersion = Versions.NOT_FOUND;
-                deleted = true;
-            } else {
-                // update the document
-                forceUpdateDocument = false; // we don't force it - it depends on the version
-                final VersionValue versionValue = versionMap.getUnderLock(index.uid());
-                assert incrementVersionLookup();
-                if (versionValue == null) {
-                    currentVersion = loadCurrentVersionFromIndex(index.uid());
-                    deleted = currentVersion == Versions.NOT_FOUND;
+
+                final IndexResult indexResult;
+                if (plan.earlyResultOnPreFlightError.isPresent()) {
+                    indexResult = plan.earlyResultOnPreFlightError.get();
+                    assert indexResult.hasFailure();
+                } else if (plan.indexIntoLucene) {
+                    indexResult = indexIntoLucene(index, plan);
                 } else {
-                    currentVersion = checkDeletedAndGCed(versionValue);
-                    deleted = versionValue.delete();
+                    assert index.origin() != Operation.Origin.PRIMARY;
+                    indexResult = new IndexResult(plan.versionForIndexing, plan.currentNotFoundOrDeleted);
                 }
+                if (indexResult.hasFailure() == false &&
+                    plan.indexIntoLucene && // if we didn't store it in lucene, there is no need to store it in the translog
+                    index.origin() != Operation.Origin.LOCAL_TRANSLOG_RECOVERY) {
+                    Translog.Location location =
+                        translog.add(new Translog.Index(index, indexResult));
+                    indexResult.setTranslogLocation(location);
+                }
+                indexResult.setTook(System.nanoTime() - index.startTime());
+                indexResult.freeze();
+                return indexResult;
+            }
+        } catch (RuntimeException | IOException e) {
+            try {
+                maybeFailEngine("index", e);
+            } catch (Exception inner) {
+                e.addSuppressed(inner);
+            }
+            throw e;
+        }
+    }
+
+    private IndexingStrategy planIndexingAsNonPrimary(Index index) throws IOException {
+        final IndexingStrategy plan;
+        if (canOptimizeAddDocument(index) && mayHaveBeenIndexedBefore(index) == false) {
+            // no need to deal with out of order delivery - we never saw this one
+            assert index.version() == 1L :
+                "can optimize on replicas but incoming version is [" + index.version() + "]";
+            plan = IndexingStrategy.optimizedAppendOnly();
+        } else {
+            // drop out of order operations
+            assert index.versionType().versionTypeForReplicationAndRecovery() == index.versionType() :
+                "resolving out of order delivery based on versioning but version type isn't fit for it. got ["
+                    + index.versionType() + "]";
+            // unlike the primary, replicas don't really care to about creation status of documents
+            // this allows to ignore the case where a document was found in the live version maps in
+            // a delete state and return false for the created flag in favor of code simplicity
+            final OpVsLuceneDocStatus opVsLucene = compareOpToLuceneDocBasedOnVersions(index);
+            if (opVsLucene == OpVsLuceneDocStatus.OP_STALE_OR_EQUAL) {
+                plan = IndexingStrategy.skipAsStale(false, index.version());
+            } else {
+                plan = IndexingStrategy.processNormally(
+                    opVsLucene == OpVsLuceneDocStatus.LUCENE_DOC_NOT_FOUND, index.version()
+                );
+            }
+        }
+        return plan;
+    }
+
+    private IndexingStrategy planIndexingAsPrimary(Index index) throws IOException {
+        assert index.origin() == Operation.Origin.PRIMARY :
+            "planing as primary but origin isn't. got " + index.origin();
+        final IndexingStrategy plan;
+        // resolve an external operation into an internal one which is safe to replay
+        if (canOptimizeAddDocument(index)) {
+            if (mayHaveBeenIndexedBefore(index)) {
+                plan = IndexingStrategy.overrideExistingAsIfNotThere(1L);
+            } else {
+                plan = IndexingStrategy.optimizedAppendOnly();
             }
-            final long expectedVersion = index.version();
-            if (checkVersionConflict(index, currentVersion, expectedVersion, deleted)) {
-                index.setCreated(false);
-                return;
+        } else {
+            // resolves incoming version
+            final VersionValue versionValue = resolveDocVersion(index);
+            final long currentVersion;
+            final boolean currentNotFoundOrDeleted;
+            if (versionValue == null) {
+                currentVersion = Versions.NOT_FOUND;
+                currentNotFoundOrDeleted = true;
+            } else {
+                currentVersion = versionValue.getVersion();
+                currentNotFoundOrDeleted = versionValue.isDelete();
             }
-            final long updatedVersion = updateVersion(index, currentVersion, expectedVersion);
-            index.setCreated(deleted);
-            if (currentVersion == Versions.NOT_FOUND && forceUpdateDocument == false) {
-                // document does not exists, we can optimize for create
-                index(index, indexWriter);
+            if (index.versionType().isVersionConflictForWrites(
+                currentVersion, index.version(), currentNotFoundOrDeleted)) {
+                plan = IndexingStrategy.skipDueToVersionConflict(
+                    new VersionConflictEngineException(shardId, index, currentVersion,
+                        currentNotFoundOrDeleted),
+                    currentNotFoundOrDeleted, currentVersion);
             } else {
-                update(index, indexWriter);
+                plan = IndexingStrategy.processNormally(currentNotFoundOrDeleted,
+                    index.versionType().updateVersion(currentVersion, index.version())
+                );
             }
-            maybeAddToTranslog(index, updatedVersion, Translog.Index::new, NEW_VERSION_VALUE);
         }
+        return plan;
     }
 
-    private long updateVersion(Engine.Operation op, long currentVersion, long expectedVersion) {
-        final long updatedVersion = op.versionType().updateVersion(currentVersion, expectedVersion);
-        op.updateVersion(updatedVersion);
-        return updatedVersion;
+    private IndexResult indexIntoLucene(Index index, IndexingStrategy plan)
+        throws IOException {
+        assert plan.versionForIndexing >= 0 : "version must be set. got " + plan.versionForIndexing;
+        assert plan.indexIntoLucene;
+        index.parsedDoc().version().setLongValue(plan.versionForIndexing);
+        try {
+            if (plan.useLuceneUpdateDocument) {
+                update(index.uid(), index.docs(), indexWriter);
+            } else {
+                // document does not exists, we can optimize for create, but double check if assertions are running
+                assert assertDocDoesNotExist(index, canOptimizeAddDocument(index) == false);
+                index(index.docs(), indexWriter);
+            }
+            versionMap.putUnderLock(index.uid().bytes(), new VersionValue(plan.versionForIndexing));
+            return new IndexResult(plan.versionForIndexing, plan.currentNotFoundOrDeleted);
+        } catch (Exception ex) {
+            if (indexWriter.getTragicException() == null) {
+                /* There is no tragic event recorded so this must be a document failure.
+                 *
+                 * The handling inside IW doesn't guarantee that an tragic / aborting exception
+                 * will be used as THE tragicEventException since if there are multiple exceptions causing an abort in IW
+                 * only one wins. Yet, only the one that wins will also close the IW and in turn fail the engine such that
+                 * we can potentially handle the exception before the engine is failed.
+                 * Bottom line is that we can only rely on the fact that if it's a document failure then
+                 * `indexWriter.getTragicException()` will be null otherwise we have to rethrow and treat it as fatal or rather
+                 * non-document failure
+                 *
+                 * we return a `MATCH_ANY` version to indicate no document was index. The value is
+                 * not used anyway
+                 */
+                return new IndexResult(ex, Versions.MATCH_ANY);
+            } else {
+                throw ex;
+            }
+        }
+    }
+
+    /**
+     * returns true if the indexing operation may have already be processed by this engine.
+     * Note that it is OK to rarely return true even if this is not the case. However a `false`
+     * return value must always be correct.
+     */
+    private boolean mayHaveBeenIndexedBefore(Index index) {
+        assert canOptimizeAddDocument(index);
+        boolean mayHaveBeenIndexBefore;
+        long deOptimizeTimestamp = maxUnsafeAutoIdTimestamp.get();
+        if (index.isRetry()) {
+            mayHaveBeenIndexBefore = true;
+            do {
+                deOptimizeTimestamp = maxUnsafeAutoIdTimestamp.get();
+                if (deOptimizeTimestamp >= index.getAutoGeneratedIdTimestamp()) {
+                    break;
+                }
+            } while (maxUnsafeAutoIdTimestamp.compareAndSet(deOptimizeTimestamp,
+                index.getAutoGeneratedIdTimestamp()) == false);
+            assert maxUnsafeAutoIdTimestamp.get() >= index.getAutoGeneratedIdTimestamp();
+        } else {
+            // in this case we force
+            mayHaveBeenIndexBefore = deOptimizeTimestamp >= index.getAutoGeneratedIdTimestamp();
+        }
+        return mayHaveBeenIndexBefore;
+    }
+
+    private static void index(final List<ParseContext.Document> docs, final IndexWriter indexWriter) throws IOException {
+        if (docs.size() > 1) {
+            indexWriter.addDocuments(docs);
+        } else {
+            indexWriter.addDocument(docs.get(0));
+        }
+    }
+
+    private static final class IndexingStrategy {
+        final boolean currentNotFoundOrDeleted;
+        final boolean useLuceneUpdateDocument;
+        final long versionForIndexing;
+        final boolean indexIntoLucene;
+        final Optional<IndexResult> earlyResultOnPreFlightError;
+
+        private IndexingStrategy(boolean currentNotFoundOrDeleted, boolean useLuceneUpdateDocument,
+                                 boolean indexIntoLucene, long versionForIndexing,
+                                 IndexResult earlyResultOnPreFlightError) {
+            assert useLuceneUpdateDocument == false || indexIntoLucene :
+                "use lucene update is set to true, but we're not indexing into lucene";
+            assert (indexIntoLucene && earlyResultOnPreFlightError != null) == false :
+                "can only index into lucene or have a preflight result but not both." +
+                    "indexIntoLucene: " + indexIntoLucene
+                    + "  earlyResultOnPreFlightError:" + earlyResultOnPreFlightError;
+            this.currentNotFoundOrDeleted = currentNotFoundOrDeleted;
+            this.useLuceneUpdateDocument = useLuceneUpdateDocument;
+            this.versionForIndexing = versionForIndexing;
+            this.indexIntoLucene = indexIntoLucene;
+            this.earlyResultOnPreFlightError =
+                earlyResultOnPreFlightError == null ? Optional.empty() :
+                    Optional.of(earlyResultOnPreFlightError);
+        }
+
+        static IndexingStrategy optimizedAppendOnly() {
+            return new IndexingStrategy(true, false, true, 1L, null);
+        }
+
+        static IndexingStrategy skipDueToVersionConflict(VersionConflictEngineException e,
+                                                         boolean currentNotFoundOrDeleted,
+                                                         long currentVersion) {
+            return new IndexingStrategy(currentNotFoundOrDeleted, false, false, Versions.NOT_FOUND,
+                new IndexResult(e, currentVersion));
+        }
+
+        static IndexingStrategy processNormally(boolean currentNotFoundOrDeleted, long versionForIndexing) {
+            return new IndexingStrategy(currentNotFoundOrDeleted, currentNotFoundOrDeleted == false, true, versionForIndexing, null);
+        }
+
+        static IndexingStrategy overrideExistingAsIfNotThere(long versionForIndexing) {
+            return new IndexingStrategy(true, true, true, versionForIndexing, null);
+        }
+
+        static IndexingStrategy skipAsStale(boolean currentNotFoundOrDeleted, long versionForIndexing) {
+            return new IndexingStrategy(currentNotFoundOrDeleted, false, false, versionForIndexing, null);
+        }
     }
 
-    private static void index(final Index index, final IndexWriter indexWriter) throws IOException {
-        if (index.docs().size() > 1) {
-            indexWriter.addDocuments(index.docs());
+    /**
+     * Asserts that the doc in the index operation really doesn't exist
+     */
+    private boolean assertDocDoesNotExist(final Index index, final boolean allowDeleted) throws IOException {
+        final VersionValue versionValue = versionMap.getUnderLock(index.uid());
+        if (versionValue != null) {
+            if (versionValue.isDelete() == false || allowDeleted == false) {
+                throw new AssertionError("doc [" + index.type() + "][" + index.id() + "] exists in version map (version " + versionValue + ")");
+            }
         } else {
-            indexWriter.addDocument(index.docs().get(0));
+            try (Searcher searcher = acquireSearcher("assert doc doesn't exist")) {
+                final long docsWithId = searcher.searcher().count(new TermQuery(index.uid()));
+                if (docsWithId > 0) {
+                    throw new AssertionError("doc [" + index.type() + "][" + index.id() + "] exists [" + docsWithId + "] times in index");
+                }
+            }
         }
+        return true;
     }
 
-    private static void update(final Index index, final IndexWriter indexWriter) throws IOException {
-        if (index.docs().size() > 1) {
-            indexWriter.updateDocuments(index.uid(), index.docs());
+    private static void update(final Term uid, final List<ParseContext.Document> docs, final IndexWriter indexWriter) throws IOException {
+        if (docs.size() > 1) {
+            indexWriter.updateDocuments(uid, docs);
         } else {
-            indexWriter.updateDocument(index.uid(), index.docs().get(0));
+            indexWriter.updateDocument(uid, docs.get(0));
         }
     }
 
     @Override
-    public void delete(Delete delete) throws EngineException {
-        try (ReleasableLock lock = readLock.acquire()) {
+    public DeleteResult delete(Delete delete) throws IOException {
+        assert Objects.equals(delete.uid().field(), uidField) : delete.uid().field();
+        assert assertVersionType(delete);
+        final DeleteResult deleteResult;
+        // NOTE: we don't throttle this when merges fall behind because delete-by-id does not create new segments:
+        try (ReleasableLock ignored = readLock.acquire(); Releasable ignored2 = acquireLock(delete.uid())) {
             ensureOpen();
-            // NOTE: we don't throttle this when merges fall behind because delete-by-id does not create new segments:
-            innerDelete(delete);
-        } catch (IllegalStateException | IOException e) {
+            lastWriteNanos = delete.startTime();
+            final DeletionStrategy plan;
+            if (delete.origin() == Operation.Origin.PRIMARY) {
+                plan = planDeletionAsPrimary(delete);
+            } else {
+                plan = planDeletionAsNonPrimary(delete);
+            }
+
+            if (plan.earlyResultOnPreflightError.isPresent()) {
+                deleteResult = plan.earlyResultOnPreflightError.get();
+            } else if (plan.deleteFromLucene) {
+                deleteResult = deleteInLucene(delete, plan);
+            } else {
+                assert delete.origin() != Operation.Origin.PRIMARY;
+                deleteResult = new DeleteResult(plan.versionOfDeletion, plan.currentlyDeleted == false);
+            }
+            if (!deleteResult.hasFailure() &&
+                plan.deleteFromLucene && // if it wasn't applied to lucene, we don't store it in the translog
+                delete.origin() != Operation.Origin.LOCAL_TRANSLOG_RECOVERY) {
+                Translog.Location location =
+                    translog.add(new Translog.Delete(delete, deleteResult));
+                deleteResult.setTranslogLocation(location);
+            }
+            deleteResult.setTook(System.nanoTime() - delete.startTime());
+            deleteResult.freeze();
+        } catch (RuntimeException | IOException e) {
             try {
-                maybeFailEngine("delete", e);
+                maybeFailEngine("index", e);
             } catch (Exception inner) {
                 e.addSuppressed(inner);
             }
-            throw new DeleteFailedEngineException(shardId, delete, e);
+            throw e;
         }
-
         maybePruneDeletedTombstones();
+        return deleteResult;
     }
 
-    private void maybePruneDeletedTombstones() {
-        // It's expensive to prune because we walk the deletes map acquiring dirtyLock for each uid so we only do it
-        // every 1/4 of gcDeletesInMillis:
-        if (engineConfig.isEnableGcDeletes() && engineConfig.getThreadPool().estimatedTimeInMillis() - lastDeleteVersionPruneTimeMSec > getGcDeletesInMillis() * 0.25) {
-            pruneDeletedTombstones();
+    private DeletionStrategy planDeletionAsNonPrimary(Delete delete) throws IOException {
+        assert delete.origin() != Operation.Origin.PRIMARY : "planing as primary but got "
+            + delete.origin();
+        // drop out of order operations
+        assert delete.versionType().versionTypeForReplicationAndRecovery() == delete.versionType() :
+            "resolving out of order delivery based on versioning but version type isn't fit for it. got ["
+                + delete.versionType() + "]";
+        // unlike the primary, replicas don't really care to about found status of documents
+        // this allows to ignore the case where a document was found in the live version maps in
+        // a delete state and return true for the found flag in favor of code simplicity
+        final OpVsLuceneDocStatus opVsLucene = compareOpToLuceneDocBasedOnVersions(delete);
+
+        final DeletionStrategy plan;
+        if (opVsLucene == OpVsLuceneDocStatus.OP_STALE_OR_EQUAL) {
+            plan = DeletionStrategy.processButSkipLucene(false, delete.version());
+        } else {
+            plan = DeletionStrategy.processNormally(
+                opVsLucene == OpVsLuceneDocStatus.LUCENE_DOC_NOT_FOUND, delete.version());
         }
+        return plan;
     }
 
-    private void innerDelete(Delete delete) throws IOException {
-        try (Releasable ignored = acquireLock(delete.uid())) {
-            lastWriteNanos = delete.startTime();
-            final long currentVersion;
-            final boolean deleted;
-            final VersionValue versionValue = versionMap.getUnderLock(delete.uid());
-            assert incrementVersionLookup();
-            if (versionValue == null) {
-                currentVersion = loadCurrentVersionFromIndex(delete.uid());
-                deleted = currentVersion == Versions.NOT_FOUND;
+    private DeletionStrategy planDeletionAsPrimary(Delete delete) throws IOException {
+        assert delete.origin() == Operation.Origin.PRIMARY : "planing as primary but got "
+            + delete.origin();
+        // resolve operation from external to internal
+        final VersionValue versionValue = resolveDocVersion(delete);
+        assert incrementVersionLookup();
+        final long currentVersion;
+        final boolean currentlyDeleted;
+        if (versionValue == null) {
+            currentVersion = Versions.NOT_FOUND;
+            currentlyDeleted = true;
+        } else {
+            currentVersion = versionValue.getVersion();
+            currentlyDeleted = versionValue.isDelete();
+        }
+        final DeletionStrategy plan;
+        if (delete.versionType().isVersionConflictForWrites(currentVersion, delete.version(), currentlyDeleted)) {
+            plan = DeletionStrategy.skipDueToVersionConflict(
+                new VersionConflictEngineException(shardId, delete, currentVersion, currentlyDeleted),
+                currentVersion, currentlyDeleted);
+        } else {
+            plan = DeletionStrategy.processNormally(currentlyDeleted, delete.versionType().updateVersion(currentVersion, delete.version()));
+        }
+        return plan;
+    }
+
+    private DeleteResult deleteInLucene(Delete delete, DeletionStrategy plan)
+        throws IOException {
+        try {
+            if (plan.currentlyDeleted == false) {
+                // any exception that comes from this is a either an ACE or a fatal exception there
+                // can't be any document failures  coming from this
+                indexWriter.deleteDocuments(delete.uid());
+            }
+            versionMap.putUnderLock(delete.uid().bytes(),
+                new DeleteVersionValue(plan.versionOfDeletion,
+                    engineConfig.getThreadPool().relativeTimeInMillis()));
+            return new DeleteResult(plan.versionOfDeletion, plan.currentlyDeleted == false);
+        } catch (Exception ex) {
+            if (indexWriter.getTragicException() == null) {
+                // there is no tragic event and such it must be a document level failure
+                return new DeleteResult(ex, plan.versionOfDeletion, plan.currentlyDeleted == false);
             } else {
-                currentVersion = checkDeletedAndGCed(versionValue);
-                deleted = versionValue.delete();
+                throw ex;
             }
+        }
+    }
 
-            final long expectedVersion = delete.version();
-            if (checkVersionConflict(delete, currentVersion, expectedVersion, deleted)) return;
+    private static final class DeletionStrategy {
+        // of a rare double delete
+        final boolean deleteFromLucene;
+        final boolean currentlyDeleted;
+        final long versionOfDeletion;
+        final Optional<DeleteResult> earlyResultOnPreflightError;
+
+        private DeletionStrategy(boolean deleteFromLucene, boolean currentlyDeleted,
+                                 long versionOfDeletion, DeleteResult earlyResultOnPreflightError) {
+            assert (deleteFromLucene && earlyResultOnPreflightError != null) == false :
+                "can only delete from lucene or have a preflight result but not both." +
+                    "deleteFromLucene: " + deleteFromLucene
+                    + "  earlyResultOnPreFlightError:" + earlyResultOnPreflightError;
+            this.deleteFromLucene = deleteFromLucene;
+            this.currentlyDeleted = currentlyDeleted;
+            this.versionOfDeletion = versionOfDeletion;
+            this.earlyResultOnPreflightError = earlyResultOnPreflightError == null ?
+                Optional.empty() : Optional.of(earlyResultOnPreflightError);
+        }
 
-            final long updatedVersion = updateVersion(delete, currentVersion, expectedVersion);
+        static DeletionStrategy skipDueToVersionConflict(VersionConflictEngineException e,
+                                                         long currentVersion, boolean currentlyDeleted) {
+            return new DeletionStrategy(false, currentlyDeleted, Versions.NOT_FOUND,
+                new DeleteResult(e, currentVersion, currentlyDeleted == false));
+        }
 
-            final boolean found = deleteIfFound(delete, currentVersion, deleted, versionValue);
+        static DeletionStrategy processNormally(boolean currentlyDeleted, long versionOfDeletion) {
+            return new DeletionStrategy(true, currentlyDeleted, versionOfDeletion,
+                null);
 
-            delete.updateVersion(updatedVersion, found);
+        }
 
-            maybeAddToTranslog(delete, updatedVersion, Translog.Delete::new, DeleteVersionValue::new);
+        public static DeletionStrategy processButSkipLucene(boolean currentlyDeleted, long versionOfDeletion) {
+            return new DeletionStrategy(false, currentlyDeleted, versionOfDeletion, null);
         }
     }
 
-    private boolean deleteIfFound(Delete delete, long currentVersion, boolean deleted, VersionValue versionValue) throws IOException {
-        final boolean found;
-        if (currentVersion == Versions.NOT_FOUND) {
-            // doc does not exist and no prior deletes
-            found = false;
-        } else if (versionValue != null && deleted) {
-            // a "delete on delete", in this case, we still increment the version, log it, and return that version
-            found = false;
-        } else {
-            // we deleted a currently existing document
-            indexWriter.deleteDocuments(delete.uid());
-            found = true;
+    private void maybePruneDeletedTombstones() {
+        // It's expensive to prune because we walk the deletes map acquiring dirtyLock for each uid so we only do it
+        // every 1/4 of gcDeletesInMillis:
+        if (engineConfig.isEnableGcDeletes() && engineConfig.getThreadPool().relativeTimeInMillis() - lastDeleteVersionPruneTimeMSec > getGcDeletesInMillis() * 0.25) {
+            pruneDeletedTombstones();
         }
-        return found;
     }
 
     @Override
@@ -627,8 +910,6 @@ public void refresh(String source) throws EngineException {
         } catch (AlreadyClosedException e) {
             failOnTragicEvent(e);
             throw e;
-        } catch (EngineClosedException e) {
-            throw e;
         } catch (Exception e) {
             try {
                 failEngine("refresh failed", e);
@@ -660,23 +941,21 @@ public void writeIndexingBuffer() throws EngineException {
             final long versionMapBytes = versionMap.ramBytesUsedForRefresh();
             final long indexingBufferBytes = indexWriter.ramBytesUsed();
 
-            final boolean useRefresh = versionMapRefreshPending.get() || (indexingBufferBytes/4 < versionMapBytes);
+            final boolean useRefresh = versionMapRefreshPending.get() || (indexingBufferBytes / 4 < versionMapBytes);
             if (useRefresh) {
                 // The version map is using > 25% of the indexing buffer, so we do a refresh so the version map also clears
                 logger.debug("use refresh to write indexing buffer (heap size=[{}]), to also clear version map (heap size=[{}])",
-                             new ByteSizeValue(indexingBufferBytes), new ByteSizeValue(versionMapBytes));
+                    new ByteSizeValue(indexingBufferBytes), new ByteSizeValue(versionMapBytes));
                 refresh("write indexing buffer");
             } else {
                 // Most of our heap is used by the indexing buffer, so we do a cheaper (just writes segments, doesn't open a new searcher) IW.flush:
                 logger.debug("use IndexWriter.flush to write indexing buffer (heap size=[{}]) since version map is small (heap size=[{}])",
-                             new ByteSizeValue(indexingBufferBytes), new ByteSizeValue(versionMapBytes));
+                    new ByteSizeValue(indexingBufferBytes), new ByteSizeValue(versionMapBytes));
                 indexWriter.flush();
             }
         } catch (AlreadyClosedException e) {
             failOnTragicEvent(e);
             throw e;
-        } catch (EngineClosedException e) {
-            throw e;
         } catch (Exception e) {
             try {
                 failEngine("writeIndexingBuffer failed", e);
@@ -769,7 +1048,7 @@ public CommitId flush(boolean force, boolean waitIfOngoing) throws EngineExcepti
                     flushLock.lock();
                     logger.trace("acquired flush lock after blocking");
                 } else {
-                    throw new FlushNotAllowedEngineException(shardId, "already flushing...");
+                    return new CommitId(lastCommittedSegmentInfos.getId());
                 }
             } else {
                 logger.trace("acquired flush lock immediately");
@@ -789,30 +1068,30 @@ public CommitId flush(boolean force, boolean waitIfOngoing) throws EngineExcepti
                     } catch (Exception e) {
                         throw new FlushFailedEngineException(shardId, e);
                     }
-                }
-                /*
-                 * we have to inc-ref the store here since if the engine is closed by a tragic event
-                 * we don't acquire the write lock and wait until we have exclusive access. This might also
-                 * dec the store reference which can essentially close the store and unless we can inc the reference
-                 * we can't use it.
-                 */
-                store.incRef();
-                try {
-                    // reread the last committed segment infos
-                    lastCommittedSegmentInfos = store.readLastCommittedSegmentsInfo();
-                } catch (Exception e) {
-                    if (isClosed.get() == false) {
-                        try {
-                            logger.warn("failed to read latest segment infos on flush", e);
-                        } catch (Exception inner) {
-                            e.addSuppressed(inner);
-                        }
-                        if (Lucene.isCorruptionException(e)) {
-                            throw new FlushFailedEngineException(shardId, e);
+                    /*
+                     * we have to inc-ref the store here since if the engine is closed by a tragic event
+                     * we don't acquire the write lock and wait until we have exclusive access. This might also
+                     * dec the store reference which can essentially close the store and unless we can inc the reference
+                     * we can't use it.
+                     */
+                    store.incRef();
+                    try {
+                        // reread the last committed segment infos
+                        lastCommittedSegmentInfos = store.readLastCommittedSegmentsInfo();
+                    } catch (Exception e) {
+                        if (isClosed.get() == false) {
+                            try {
+                                logger.warn("failed to read latest segment infos on flush", e);
+                            } catch (Exception inner) {
+                                e.addSuppressed(inner);
+                            }
+                            if (Lucene.isCorruptionException(e)) {
+                                throw new FlushFailedEngineException(shardId, e);
+                            }
                         }
+                    } finally {
+                        store.decRef();
                     }
-                } finally {
-                    store.decRef();
                 }
                 newCommitId = lastCommittedSegmentInfos.getId();
             } catch (FlushFailedEngineException ex) {
@@ -831,7 +1110,7 @@ public CommitId flush(boolean force, boolean waitIfOngoing) throws EngineExcepti
     }
 
     private void pruneDeletedTombstones() {
-        long timeMSec = engineConfig.getThreadPool().estimatedTimeInMillis();
+        long timeMSec = engineConfig.getThreadPool().relativeTimeInMillis();
 
         // TODO: not good that we reach into LiveVersionMap here; can we move this inside VersionMap instead?  problem is the dirtyLock...
 
@@ -843,7 +1122,7 @@ private void pruneDeletedTombstones() {
                 // Must re-get it here, vs using entry.getValue(), in case the uid was indexed/deleted since we pulled the iterator:
                 VersionValue versionValue = versionMap.getTombstoneUnderLock(uid);
                 if (versionValue != null) {
-                    if (timeMSec - versionValue.time() > getGcDeletesInMillis()) {
+                    if (timeMSec - versionValue.getTime() > getGcDeletesInMillis()) {
                         versionMap.removeTombstoneUnderLock(uid);
                     }
                 }
@@ -853,9 +1132,14 @@ private void pruneDeletedTombstones() {
         lastDeleteVersionPruneTimeMSec = timeMSec;
     }
 
+    // testing
+    void clearDeletedTombstones() {
+        versionMap.clearTombstones();
+    }
+
     @Override
     public void forceMerge(final boolean flush, int maxNumSegments, boolean onlyExpungeDeletes,
-                           final boolean upgrade, final boolean upgradeOnlyAncientSegments) throws EngineException, EngineClosedException, IOException {
+                           final boolean upgrade, final boolean upgradeOnlyAncientSegments) throws EngineException, IOException {
         /*
          * We do NOT acquire the readlock here since we are waiting on the merges to finish
          * that's fine since the IW.rollback should stop all the threads and trigger an IOException
@@ -940,22 +1224,34 @@ public IndexCommit acquireIndexCommit(final boolean flushFirst) throws EngineExc
         }
     }
 
-    private void failOnTragicEvent(AlreadyClosedException ex) {
+    @SuppressWarnings("finally")
+    private boolean failOnTragicEvent(AlreadyClosedException ex) {
+        final boolean engineFailed;
         // if we are already closed due to some tragic exception
         // we need to fail the engine. it might have already been failed before
         // but we are double-checking it's failed and closed
         if (indexWriter.isOpen() == false && indexWriter.getTragicException() != null) {
-            final Exception tragedy = indexWriter.getTragicException() instanceof Exception ?
-                (Exception) indexWriter.getTragicException() :
-                new Exception(indexWriter.getTragicException());
-            failEngine("already closed by tragic event on the index writer", tragedy);
+            if (indexWriter.getTragicException() instanceof Error) {
+                try {
+                    logger.error("tragic event in index writer", ex);
+                } finally {
+                    throw (Error) indexWriter.getTragicException();
+                }
+            } else {
+                failEngine("already closed by tragic event on the index writer", (Exception) indexWriter.getTragicException());
+                engineFailed = true;
+            }
         } else if (translog.isOpen() == false && translog.getTragicException() != null) {
             failEngine("already closed by tragic event on the translog", translog.getTragicException());
-        } else {
+            engineFailed = true;
+        } else if (failedEngine.get() == null && isClosed.get() == false) { // we are closed but the engine is not failed yet?
             // this smells like a bug - we only expect ACE if we are in a fatal case ie. either translog or IW is closed by
             // a tragic event or has closed itself. if that is not the case we are in a buggy state and raise an assertion error
             throw new AssertionError("Unexpected AlreadyClosedException", ex);
+        } else {
+            engineFailed = false;
         }
+        return engineFailed;
     }
 
     @Override
@@ -968,8 +1264,7 @@ protected boolean maybeFailEngine(String source, Exception e) {
         // exception that should only be thrown in a tragic event. we pass on the checks to failOnTragicEvent which will
         // throw and AssertionError if the tragic event condition is not met.
         if (e instanceof AlreadyClosedException) {
-            failOnTragicEvent((AlreadyClosedException)e);
-            return true;
+            return failOnTragicEvent((AlreadyClosedException) e);
         } else if (e != null &&
             ((indexWriter.isOpen() == false && indexWriter.getTragicException() == e)
                 || (translog.isOpen() == false && translog.getTragicException() == e))) {
@@ -1026,7 +1321,7 @@ public List<Segment> segments(boolean verbose) {
      * is failed.
      */
     @Override
-    protected final void closeNoLock(String reason) {
+    protected final void closeNoLock(String reason, CountDownLatch closedLatch) {
         if (isClosed.compareAndSet(false, true)) {
             assert rwl.isWriteLockedByCurrentThread() || failEngineLock.isHeldByCurrentThread() : "Either the write lock must be held or the engine must be currently be failing itself";
             try {
@@ -1053,8 +1348,12 @@ protected final void closeNoLock(String reason) {
             } catch (Exception e) {
                 logger.warn("failed to rollback writer on close", e);
             } finally {
-                store.decRef();
-                logger.debug("engine closed [{}]", reason);
+                try {
+                    store.decRef();
+                    logger.debug("engine closed [{}]", reason);
+                } finally {
+                    closedLatch.countDown();
+                }
             }
         }
     }
@@ -1074,12 +1373,13 @@ private Releasable acquireLock(Term uid) {
 
     private long loadCurrentVersionFromIndex(Term uid) throws IOException {
         assert incrementIndexVersionLookup();
-        try (final Searcher searcher = acquireSearcher("load_version")) {
-            return Versions.loadVersion(searcher.reader(), uid);
+        try (Searcher searcher = acquireSearcher("load_version")) {
+            return VersionsResolver.loadVersion(searcher.reader(), uid);
         }
     }
 
-    private IndexWriter createWriter(boolean create) throws IOException {
+    // pkg-private for testing
+    IndexWriter createWriter(boolean create) throws IOException {
         try {
             final IndexWriterConfig iwc = new IndexWriterConfig(engineConfig.getAnalyzer());
             iwc.setCommitOnClose(false); // we by default don't commit on close
@@ -1099,7 +1399,7 @@ private IndexWriter createWriter(boolean create) throws IOException {
             mergePolicy = new ElasticsearchMergePolicy(mergePolicy);
             iwc.setMergePolicy(mergePolicy);
             iwc.setSimilarity(engineConfig.getSimilarity());
-            iwc.setRAMBufferSizeMB(engineConfig.getIndexingBufferSize().mbFrac());
+            iwc.setRAMBufferSizeMB(engineConfig.getIndexingBufferSize().getMbFrac());
             iwc.setCodec(engineConfig.getCodec());
             iwc.setUseCompoundFile(true); // always use compound on flush - reduces # of file-handles on refresh
             return new IndexWriter(store.directory(), iwc);
@@ -1125,7 +1425,7 @@ static final class SearchFactory extends EngineSearcherFactory {
         @Override
         public IndexSearcher newSearcher(IndexReader reader, IndexReader previousReader) throws IOException {
             IndexSearcher searcher = super.newSearcher(reader, previousReader);
-            if (reader instanceof LeafReader && isMergedSegment((LeafReader)reader)) {
+            if (reader instanceof LeafReader && isMergedSegment((LeafReader) reader)) {
                 // we call newSearcher from the IndexReaderWarmer which warms segments during merging
                 // in that case the reader is a LeafReader and all we need to do is to build a new Searcher
                 // and return it since it does it's own warming for that particular reader.
@@ -1148,7 +1448,7 @@ public IndexSearcher newSearcher(IndexReader reader, IndexReader previousReader)
     @Override
     public void activateThrottling() {
         int count = throttleRequestCount.incrementAndGet();
-        assert count >= 1: "invalid post-increment throttleRequestCount=" + count;
+        assert count >= 1 : "invalid post-increment throttleRequestCount=" + count;
         if (count == 1) {
             throttle.activate();
         }
@@ -1157,12 +1457,18 @@ public void activateThrottling() {
     @Override
     public void deactivateThrottling() {
         int count = throttleRequestCount.decrementAndGet();
-        assert count >= 0: "invalid post-decrement throttleRequestCount=" + count;
+        assert count >= 0 : "invalid post-decrement throttleRequestCount=" + count;
         if (count == 0) {
             throttle.deactivate();
         }
     }
 
+    @Override
+    public boolean isThrottled() {
+        return throttle.isThrottled();
+    }
+
+    @Override
     public long getIndexThrottleTimeInMillis() {
         return throttle.getThrottleTimeInMillis();
     }
@@ -1252,15 +1558,21 @@ protected void doRun() throws Exception {
     private void commitIndexWriter(IndexWriter writer, Translog translog, String syncId) throws IOException {
         ensureCanFlush();
         try {
-            Translog.TranslogGeneration translogGeneration = translog.getGeneration();
-            logger.trace("committing writer with translog id [{}]  and sync id [{}] ", translogGeneration.translogFileGeneration, syncId);
-            Map<String, String> commitData = new HashMap<>(2);
-            commitData.put(Translog.TRANSLOG_GENERATION_KEY, Long.toString(translogGeneration.translogFileGeneration));
-            commitData.put(Translog.TRANSLOG_UUID_KEY, translogGeneration.translogUUID);
-            if (syncId != null) {
-                commitData.put(Engine.SYNC_COMMIT_ID, syncId);
-            }
-            indexWriter.setCommitData(commitData);
+            final Translog.TranslogGeneration translogGeneration = translog.getGeneration();
+            final String translogFileGeneration = Long.toString(translogGeneration.translogFileGeneration);
+            final String translogUUID = translogGeneration.translogUUID;
+
+            writer.setLiveCommitData(() -> {
+                final Map<String, String> commitData = new HashMap<>(4);
+                commitData.put(Translog.TRANSLOG_GENERATION_KEY, translogFileGeneration);
+                commitData.put(Translog.TRANSLOG_UUID_KEY, translogUUID);
+                if (syncId != null) {
+                    commitData.put(Engine.SYNC_COMMIT_ID, syncId);
+                }
+                commitData.put(MAX_UNSAFE_AUTO_ID_TIMESTAMP_COMMIT_ID, Long.toString(maxUnsafeAutoIdTimestamp.get()));
+                logger.trace("committing writer with commit data [{}]", commitData);
+                return commitData.entrySet().iterator();
+            });
             writer.commit();
         } catch (Exception ex) {
             try {
@@ -1291,8 +1603,8 @@ private void ensureCanFlush() {
         // if we are in this stage we have to prevent flushes from this
         // engine otherwise we might loose documents if the flush succeeds
         // and the translog recover fails we we "commit" the translog on flush.
-        if (allowCommits.get() == false) {
-            throw new FlushNotAllowedEngineException(shardId, "flushes are disabled - pending translog recovery");
+        if (pendingTranslogRecovery.get()) {
+            throw new IllegalStateException(shardId.toString() + " flushes are disabled - pending translog recovery");
         }
     }
 
@@ -1312,14 +1624,6 @@ public MergeStats getMergeStats() {
         return mergeScheduler.stats();
     }
 
-    @Override
-    public DocsStats getDocStats() {
-        final int numDocs = indexWriter.numDocs();
-        final int maxDoc = indexWriter.maxDoc();
-        return new DocsStats(numDocs, maxDoc-numDocs);
-    }
-
-
     /**
      * Returns the number of times a version was looked up either from the index.
      * Note this is only available if assertions are enabled
@@ -1353,4 +1657,9 @@ private boolean incrementIndexVersionLookup() {
     boolean indexWriterHasDeletions() {
         return indexWriter.hasDeletions();
     }
+
+    @Override
+    public boolean isRecovering() {
+        return pendingTranslogRecovery.get();
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/engine/LiveVersionMap.java b/core/src/main/java/org/elasticsearch/index/engine/LiveVersionMap.java
index b489cec5768bb..7233420309c72 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/LiveVersionMap.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/LiveVersionMap.java
@@ -43,12 +43,12 @@ private static class Maps {
         // Used while refresh is running, and to hold adds/deletes until refresh finishes.  We read from both current and old on lookup:
         final Map<BytesRef,VersionValue> old;
 
-        public Maps(Map<BytesRef,VersionValue> current, Map<BytesRef,VersionValue> old) {
+        Maps(Map<BytesRef,VersionValue> current, Map<BytesRef,VersionValue> old) {
            this.current = current;
            this.old = old;
         }
 
-        public Maps() {
+        Maps() {
             this(ConcurrentCollections.<BytesRef,VersionValue>newConcurrentMapWithAggressiveConcurrency(),
                  ConcurrentCollections.<BytesRef,VersionValue>newConcurrentMapWithAggressiveConcurrency());
         }
@@ -164,7 +164,7 @@ void putUnderLock(BytesRef uid, VersionValue version) {
         if (prev != null) {
             // Deduct RAM for the version we just replaced:
             long prevBytes = BASE_BYTES_PER_CHM_ENTRY;
-            if (prev.delete() == false) {
+            if (prev.isDelete() == false) {
                 prevBytes += prev.ramBytesUsed() + uidRAMBytesUsed;
             }
             ramBytesUsedCurrent.addAndGet(-prevBytes);
@@ -172,13 +172,13 @@ void putUnderLock(BytesRef uid, VersionValue version) {
 
         // Add RAM for the new version:
         long newBytes = BASE_BYTES_PER_CHM_ENTRY;
-        if (version.delete() == false) {
+        if (version.isDelete() == false) {
             newBytes += version.ramBytesUsed() + uidRAMBytesUsed;
         }
         ramBytesUsedCurrent.addAndGet(newBytes);
 
         final VersionValue prevTombstone;
-        if (version.delete()) {
+        if (version.isDelete()) {
             // Also enroll the delete into tombstones, and account for its RAM too:
             prevTombstone = tombstones.put(uid, version);
 
@@ -187,7 +187,7 @@ void putUnderLock(BytesRef uid, VersionValue version) {
             // the accounting to current:
             ramBytesUsedTombstones.addAndGet(BASE_BYTES_PER_CHM_ENTRY + version.ramBytesUsed() + uidRAMBytesUsed);
 
-            if (prevTombstone == null && prev != null && prev.delete()) {
+            if (prevTombstone == null && prev != null && prev.isDelete()) {
                 // If prev was a delete that had already been removed from tombstones, then current was already accounting for the
                 // BytesRef/VersionValue RAM, so we now deduct that as well:
                 ramBytesUsedCurrent.addAndGet(-(prev.ramBytesUsed() + uidRAMBytesUsed));
@@ -211,12 +211,12 @@ void removeTombstoneUnderLock(BytesRef uid) {
 
         final VersionValue prev = tombstones.remove(uid);
         if (prev != null) {
-            assert prev.delete();
+            assert prev.isDelete();
             long v = ramBytesUsedTombstones.addAndGet(-(BASE_BYTES_PER_CHM_ENTRY + prev.ramBytesUsed() + uidRAMBytesUsed));
             assert v >= 0: "bytes=" + v;
         }
         final VersionValue curVersion = maps.current.get(uid);
-        if (curVersion != null && curVersion.delete()) {
+        if (curVersion != null && curVersion.isDelete()) {
             // We now shift accounting of the BytesRef from tombstones to current, because a refresh would clear this RAM.  This should be
             // uncommon, because with the default refresh=1s and gc_deletes=60s, deletes should be cleared from current long before we drop
             // them from tombstones:
@@ -234,6 +234,11 @@ Iterable<Map.Entry<BytesRef,VersionValue>> getAllTombstones() {
         return tombstones.entrySet();
     }
 
+    /** clears all tombstones ops */
+    void clearTombstones() {
+        tombstones.clear();
+    }
+
     /** Called when this index is closed. */
     synchronized void clear() {
         maps = new Maps();
diff --git a/core/src/main/java/org/elasticsearch/index/engine/SegmentsStats.java b/core/src/main/java/org/elasticsearch/index/engine/SegmentsStats.java
index 637beebfec8ed..ed8e150cd6c18 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/SegmentsStats.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/SegmentsStats.java
@@ -286,12 +286,6 @@ public long getMaxUnsafeAutoIdTimestamp() {
         return maxUnsafeAutoIdTimestamp;
     }
 
-    public static SegmentsStats readSegmentsStats(StreamInput in) throws IOException {
-        SegmentsStats stats = new SegmentsStats();
-        stats.readFrom(in);
-        return stats;
-    }
-
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject(Fields.SEGMENTS);
@@ -391,10 +385,9 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeLong(maxUnsafeAutoIdTimestamp);
 
         out.writeVInt(fileSizes.size());
-        for (Iterator<ObjectObjectCursor<String, Long>> it = fileSizes.iterator(); it.hasNext();) {
-            ObjectObjectCursor<String, Long> entry = it.next();
+        for (ObjectObjectCursor<String, Long> entry : fileSizes) {
             out.writeString(entry.key);
-            out.writeLong(entry.value);
+            out.writeLong(entry.value.longValue());
         }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/engine/ShadowEngine.java b/core/src/main/java/org/elasticsearch/index/engine/ShadowEngine.java
index 3aafcaff74859..fbd42b4031726 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/ShadowEngine.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/ShadowEngine.java
@@ -35,6 +35,7 @@
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.List;
+import java.util.concurrent.CountDownLatch;
 import java.util.function.Function;
 
 /**
@@ -52,8 +53,8 @@
  * regular primary (which uses {@link org.elasticsearch.index.engine.InternalEngine})
  *
  * Notice that since this Engine does not deal with the translog, any
- * {@link #get(Get get)} request goes directly to the searcher, meaning it is
- * non-realtime.
+ * {@link #get(Get, Function)} request goes directly to the searcher,
+ * meaning it is non-realtime.
  */
 public class ShadowEngine extends Engine {
 
@@ -67,9 +68,6 @@ public class ShadowEngine extends Engine {
 
     public ShadowEngine(EngineConfig engineConfig)  {
         super(engineConfig);
-        if (engineConfig.getRefreshListeners() != null) {
-            throw new IllegalArgumentException("ShadowEngine doesn't support RefreshListeners");
-        }
         SearcherFactory searcherFactory = new EngineSearcherFactory(engineConfig);
         final long nonexistentRetryTime = engineConfig.getIndexSettings().getSettings()
                 .getAsTime(NONEXISTENT_INDEX_RETRY_WAIT, DEFAULT_NONEXISTENT_INDEX_RETRY_WAIT)
@@ -106,12 +104,12 @@ public ShadowEngine(EngineConfig engineConfig)  {
 
 
     @Override
-    public void index(Index index) throws EngineException {
+    public IndexResult index(Index index) {
         throw new UnsupportedOperationException(shardId + " index operation not allowed on shadow engine");
     }
 
     @Override
-    public void delete(Delete delete) throws EngineException {
+    public DeleteResult delete(Delete delete) {
         throw new UnsupportedOperationException(shardId + " delete operation not allowed on shadow engine");
     }
 
@@ -162,6 +160,7 @@ public void forceMerge(boolean flush, int maxNumSegments, boolean onlyExpungeDel
     @Override
     public GetResult get(Get get, Function<String, Searcher> searcherFacotry) throws EngineException {
         // There is no translog, so we can get it directly from the searcher
+        // Since we never refresh we just drop the onRefresh parameter on the floor
         return getFromSearcher(get, searcherFacotry);
     }
 
@@ -191,9 +190,6 @@ public void refresh(String source) throws EngineException {
             ensureOpen();
             searcherManager.maybeRefreshBlocking();
         } catch (AlreadyClosedException e) {
-            // This means there's a bug somewhere: don't suppress it
-            throw new AssertionError(e);
-        } catch (EngineClosedException e) {
             throw e;
         } catch (Exception e) {
             try {
@@ -216,7 +212,7 @@ protected SearcherManager getSearcherManager() {
     }
 
     @Override
-    protected void closeNoLock(String reason) {
+    protected void closeNoLock(String reason, CountDownLatch closedLatch) {
         if (isClosed.compareAndSet(false, true)) {
             try {
                 logger.debug("shadow replica close searcher manager refCount: {}", store.refCount());
@@ -224,7 +220,11 @@ protected void closeNoLock(String reason) {
             } catch (Exception e) {
                 logger.warn("shadow replica failed to close searcher manager", e);
             } finally {
-                store.decRef();
+                try {
+                    store.decRef();
+                } finally {
+                    closedLatch.countDown();
+                }
             }
         }
     }
@@ -256,6 +256,16 @@ public void deactivateThrottling() {
         throw new UnsupportedOperationException("ShadowEngine has no IndexWriter");
     }
 
+    @Override
+    public boolean isThrottled() {
+        return false;
+    }
+
+    @Override
+    public long getIndexThrottleTimeInMillis() {
+        return 0L;
+    }
+
     @Override
     public Engine recoverFromTranslog() throws IOException {
         throw new UnsupportedOperationException("can't recover on a shadow engine");
diff --git a/core/src/main/java/org/elasticsearch/index/engine/VersionConflictEngineException.java b/core/src/main/java/org/elasticsearch/index/engine/VersionConflictEngineException.java
index 9b038c6e77c56..56c19827faba3 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/VersionConflictEngineException.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/VersionConflictEngineException.java
@@ -29,6 +29,10 @@
  */
 public class VersionConflictEngineException extends EngineException {
 
+    public VersionConflictEngineException(ShardId shardId, Engine.Operation op, long currentVersion, boolean deleted) {
+        this(shardId, op.type(), op.id(), op.versionType().explainConflictForWrites(currentVersion, op.version(), deleted));
+    }
+
     public VersionConflictEngineException(ShardId shardId, String type, String id, String explanation) {
         this(shardId, null, type, id, explanation);
     }
diff --git a/core/src/main/java/org/elasticsearch/index/engine/VersionValue.java b/core/src/main/java/org/elasticsearch/index/engine/VersionValue.java
index 662c88df5d9d3..53550578cc3a0 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/VersionValue.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/VersionValue.java
@@ -29,25 +29,25 @@ class VersionValue implements Accountable {
 
     private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(VersionValue.class);
 
+    /** the version of the document. used for versioned indexed operations and as a BWC layer, where no seq# are set yet */
     private final long version;
 
-    public VersionValue(long version) {
+    VersionValue(long version) {
         this.version = version;
     }
 
-    public long time() {
+    public long getTime() {
         throw new UnsupportedOperationException();
     }
 
-    public long version() {
+    public long getVersion() {
         return version;
     }
 
-    public boolean delete() {
+    public boolean isDelete() {
         return false;
     }
 
-
     @Override
     public long ramBytesUsed() {
         return BASE_RAM_BYTES_USED;
@@ -57,4 +57,10 @@ public long ramBytesUsed() {
     public Collection<Accountable> getChildResources() {
         return Collections.emptyList();
     }
+
+    @Override
+    public String toString() {
+        return "VersionValue{" +
+            "version=" + version + "}";
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/AtomicParentChildFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/AtomicParentChildFieldData.java
deleted file mode 100644
index f88d7c58774f1..0000000000000
--- a/core/src/main/java/org/elasticsearch/index/fielddata/AtomicParentChildFieldData.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.fielddata;
-
-import org.apache.lucene.index.SortedDocValues;
-
-import java.util.Set;
-
-/**
- * Specialization of {@link AtomicFieldData} for parent/child mappings.
- */
-public interface AtomicParentChildFieldData extends AtomicFieldData {
-
-    /**
-     * Return the set of types there is a mapping for.
-     */
-    Set<String> types();
-
-    /**
-     * Return the mapping for the given type. The returned
-     * {@link SortedDocValues} will map doc IDs to the identifier of their
-     * parent.
-     */
-    SortedDocValues getOrdinalsValues(String type);
-
-}
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/FieldDataStats.java b/core/src/main/java/org/elasticsearch/index/fielddata/FieldDataStats.java
index d8548f72476a0..70a41b96f7c46 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/FieldDataStats.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/FieldDataStats.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.index.fielddata;
 
-import com.carrotsearch.hppc.ObjectLongHashMap;
+import org.elasticsearch.common.FieldMemoryStats;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -29,21 +29,27 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
 import java.io.IOException;
+import java.util.Objects;
 
 /**
  */
 public class FieldDataStats implements Streamable, ToXContent {
 
+    private static final String FIELDDATA = "fielddata";
+    private static final String MEMORY_SIZE = "memory_size";
+    private static final String MEMORY_SIZE_IN_BYTES = "memory_size_in_bytes";
+    private static final String EVICTIONS = "evictions";
+    private static final String FIELDS = "fields";
     long memorySize;
     long evictions;
     @Nullable
-    ObjectLongHashMap<String> fields;
+    FieldMemoryStats fields;
 
     public FieldDataStats() {
 
     }
 
-    public FieldDataStats(long memorySize, long evictions, @Nullable ObjectLongHashMap<String> fields) {
+    public FieldDataStats(long memorySize, long evictions, @Nullable FieldMemoryStats fields) {
         this.memorySize = memorySize;
         this.evictions = evictions;
         this.fields = fields;
@@ -54,16 +60,9 @@ public void add(FieldDataStats stats) {
         this.evictions += stats.evictions;
         if (stats.fields != null) {
             if (fields == null) {
-                fields = stats.fields.clone();
+                fields = stats.fields.copy();
             } else {
-                assert !stats.fields.containsKey(null);
-                final Object[] keys = stats.fields.keys;
-                final long[] values = stats.fields.values;
-                for (int i = 0; i < keys.length; i++) {
-                    if (keys[i] != null) {
-                        fields.addTo((String) keys[i], values[i]);
-                    }
-                }
+                fields.add(stats.fields);
             }
         }
     }
@@ -81,78 +80,48 @@ public long getEvictions() {
     }
 
     @Nullable
-    public ObjectLongHashMap<String> getFields() {
+    public FieldMemoryStats getFields() {
         return fields;
     }
 
-    public static FieldDataStats readFieldDataStats(StreamInput in) throws IOException {
-        FieldDataStats stats = new FieldDataStats();
-        stats.readFrom(in);
-        return stats;
-    }
-
     @Override
     public void readFrom(StreamInput in) throws IOException {
         memorySize = in.readVLong();
         evictions = in.readVLong();
-        if (in.readBoolean()) {
-            int size = in.readVInt();
-            fields = new ObjectLongHashMap<>(size);
-            for (int i = 0; i < size; i++) {
-                fields.put(in.readString(), in.readVLong());
-            }
-        }
+        fields = in.readOptionalWriteable(FieldMemoryStats::new);
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeVLong(memorySize);
         out.writeVLong(evictions);
-        if (fields == null) {
-            out.writeBoolean(false);
-        } else {
-            out.writeBoolean(true);
-            out.writeVInt(fields.size());
-            assert !fields.containsKey(null);
-            final Object[] keys = fields.keys;
-            final long[] values = fields.values;
-            for (int i = 0; i < keys.length; i++) {
-                if (keys[i] != null) {
-                    out.writeString((String) keys[i]);
-                    out.writeVLong(values[i]);
-                }
-            }
-        }
+        out.writeOptionalWriteable(fields);
     }
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject(Fields.FIELDDATA);
-        builder.byteSizeField(Fields.MEMORY_SIZE_IN_BYTES, Fields.MEMORY_SIZE, memorySize);
-        builder.field(Fields.EVICTIONS, getEvictions());
+        builder.startObject(FIELDDATA);
+        builder.byteSizeField(MEMORY_SIZE_IN_BYTES, MEMORY_SIZE, memorySize);
+        builder.field(EVICTIONS, getEvictions());
         if (fields != null) {
-            builder.startObject(Fields.FIELDS);
-            assert !fields.containsKey(null);
-            final Object[] keys = fields.keys;
-            final long[] values = fields.values;
-            for (int i = 0; i < keys.length; i++) {
-                if (keys[i] != null) {
-                    builder.startObject((String) keys[i]);
-                    builder.byteSizeField(Fields.MEMORY_SIZE_IN_BYTES, Fields.MEMORY_SIZE, values[i]);
-                    builder.endObject();
-                }
-            }
-            builder.endObject();
+            fields.toXContent(builder, FIELDS, MEMORY_SIZE_IN_BYTES, MEMORY_SIZE);
         }
         builder.endObject();
         return builder;
     }
 
-    static final class Fields {
-        static final String FIELDDATA = "fielddata";
-        static final String MEMORY_SIZE = "memory_size";
-        static final String MEMORY_SIZE_IN_BYTES = "memory_size_in_bytes";
-        static final String EVICTIONS = "evictions";
-        static final String FIELDS = "fields";
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        FieldDataStats that = (FieldDataStats) o;
+        return memorySize == that.memorySize &&
+            evictions == that.evictions &&
+            Objects.equals(fields, that.fields);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(memorySize, evictions, fields);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java
index feacfe5999695..0b63dfb8df80a 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java
@@ -52,7 +52,7 @@
  */
 public interface IndexFieldData<FD extends AtomicFieldData> extends IndexComponent {
 
-    public static class CommonSettings {
+    class CommonSettings {
         public static final String SETTING_MEMORY_STORAGE_HINT = "memory_storage_hint";
 
         public enum MemoryStorageFormat {
@@ -85,9 +85,9 @@ public static MemoryStorageFormat fromString(String string) {
     FD loadDirect(LeafReaderContext context) throws Exception;
 
     /**
-     * Comparator used for sorting.
+     * Returns the {@link SortField} to used for sorting.
      */
-    XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested);
+    SortField sortField(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse);
 
     /**
      * Clears any resources associated with this field data.
@@ -136,17 +136,17 @@ public DocIdSetIterator innerDocs(LeafReaderContext ctx) throws IOException {
         }
 
         /** Whether missing values should be sorted first. */
-        protected final boolean sortMissingFirst(Object missingValue) {
+        public final boolean sortMissingFirst(Object missingValue) {
             return "_first".equals(missingValue);
         }
 
         /** Whether missing values should be sorted last, this is the default. */
-        protected final boolean sortMissingLast(Object missingValue) {
+        public final boolean sortMissingLast(Object missingValue) {
             return missingValue == null || "_last".equals(missingValue);
         }
 
         /** Return the missing object value according to the reduced type of the comparator. */
-        protected final Object missingObject(Object missingValue, boolean reversed) {
+        public final Object missingObject(Object missingValue, boolean reversed) {
             if (sortMissingFirst(missingValue) || sortMissingLast(missingValue)) {
                 final boolean min = sortMissingFirst(missingValue) ^ reversed;
                 switch (reducedType()) {
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataCache.java b/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataCache.java
index 948b19a8afbf5..5238f06a7909c 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataCache.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataCache.java
@@ -31,7 +31,7 @@ public interface IndexFieldDataCache {
 
     <FD extends AtomicFieldData, IFD extends IndexFieldData<FD>> FD load(LeafReaderContext context, IFD indexFieldData) throws Exception;
 
-    <FD extends AtomicFieldData, IFD extends IndexFieldData.Global<FD>> IFD load(final DirectoryReader indexReader, final IFD indexFieldData) throws Exception;
+    <FD extends AtomicFieldData, IFD extends IndexFieldData.Global<FD>> IFD load(DirectoryReader indexReader, IFD indexFieldData) throws Exception;
 
     /**
      * Clears all the field data stored cached in on this index.
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java
index abfe0d8e96af0..08a89eb82f3d8 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java
@@ -23,7 +23,7 @@
  */
 public interface IndexNumericFieldData extends IndexFieldData<AtomicNumericFieldData> {
 
-    public static enum NumericType {
+    enum NumericType {
         BOOLEAN(false),
         BYTE(false),
         SHORT(false),
@@ -35,7 +35,7 @@ public static enum NumericType {
 
         private final boolean floatingPoint;
 
-        private NumericType(boolean floatingPoint) {
+        NumericType(boolean floatingPoint) {
             this.floatingPoint = floatingPoint;
         }
 
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/IndexOrdinalsFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/IndexOrdinalsFieldData.java
index cb1471179c2d7..2e714fc80a12b 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/IndexOrdinalsFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/IndexOrdinalsFieldData.java
@@ -21,7 +21,7 @@
 
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexReader;
-
+import org.apache.lucene.index.MultiDocValues;
 
 
 /**
@@ -42,4 +42,9 @@ public interface IndexOrdinalsFieldData extends IndexFieldData.Global<AtomicOrdi
     @Override
     IndexOrdinalsFieldData localGlobalDirect(DirectoryReader indexReader) throws Exception;
 
+    /**
+     * Returns the underlying {@link MultiDocValues.OrdinalMap} for this fielddata
+     * or null if global ordinals are not needed (constant value or single segment).
+     */
+    MultiDocValues.OrdinalMap getOrdinalMap();
 }
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/IndexParentChildFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/IndexParentChildFieldData.java
deleted file mode 100644
index 0453b3f1a15a7..0000000000000
--- a/core/src/main/java/org/elasticsearch/index/fielddata/IndexParentChildFieldData.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.fielddata;
-
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexReader;
-
-
-
-
-/**
- * Soecialization of {@link IndexFieldData} for parent/child mappings.
- */
-public interface IndexParentChildFieldData extends IndexFieldData.Global<AtomicParentChildFieldData> {
-
-    /**
-     * Load a global view of the ordinals for the given {@link IndexReader},
-     * potentially from a cache.
-     */
-    @Override
-    IndexParentChildFieldData loadGlobal(DirectoryReader indexReader);
-
-    /**
-     * Load a global view of the ordinals for the given {@link IndexReader}.
-     */
-    @Override
-    IndexParentChildFieldData localGlobalDirect(DirectoryReader indexReader) throws Exception;
-
-}
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java
index 3991a37a8bf2c..a1e62979cae6e 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java
@@ -25,33 +25,62 @@
 import org.elasticsearch.common.geo.GeoHashUtils;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.geo.GeoUtils;
-import org.elasticsearch.common.unit.DistanceUnit;
+import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.joda.time.MutableDateTime;
 import org.joda.time.ReadableDateTime;
 
 import java.util.AbstractList;
-import java.util.Collections;
+import java.util.Comparator;
 import java.util.List;
+import java.util.function.UnaryOperator;
 
 
 /**
  * Script level doc values, the assumption is that any implementation will implement a <code>getValue</code>
  * and a <code>getValues</code> that return the relevant type that then can be used in scripts.
  */
-public interface ScriptDocValues<T> extends List<T> {
+public abstract class ScriptDocValues<T> extends AbstractList<T> {
 
     /**
      * Set the current doc ID.
      */
-    void setNextDocId(int docId);
+    public abstract void setNextDocId(int docId);
 
     /**
      * Return a copy of the list of the values for the current document.
      */
-    List<T> getValues();
+    public final List<T> getValues() {
+        return this;
+    }
+
+    // Throw meaningful exceptions if someone tries to modify the ScriptDocValues.
+    @Override
+    public final void add(int index, T element) {
+        throw new UnsupportedOperationException("doc values are unmodifiable");
+    }
+
+    @Override
+    public final boolean remove(Object o) {
+        throw new UnsupportedOperationException("doc values are unmodifiable");
+    }
+
+    @Override
+    public final void replaceAll(UnaryOperator<T> operator) {
+        throw new UnsupportedOperationException("doc values are unmodifiable");
+    }
 
-    public static final class Strings extends AbstractList<String> implements ScriptDocValues<String> {
+    @Override
+    public final T set(int index, T element) {
+        throw new UnsupportedOperationException("doc values are unmodifiable");
+    }
+
+    @Override
+    public final void sort(Comparator<? super T> c) {
+        throw new UnsupportedOperationException("doc values are unmodifiable");
+    }
+
+    public static final class Strings extends ScriptDocValues<String> {
 
         private final SortedBinaryDocValues values;
 
@@ -85,11 +114,6 @@ public String getValue() {
             }
         }
 
-        @Override
-        public List<String> getValues() {
-            return Collections.unmodifiableList(this);
-        }
-
         @Override
         public String get(int index) {
             return values.valueAt(index).utf8ToString();
@@ -102,10 +126,10 @@ public int size() {
 
     }
 
-    public static class Longs extends AbstractList<Long> implements ScriptDocValues<Long> {
+    public static final class Longs extends ScriptDocValues<Long> {
 
         private final SortedNumericDocValues values;
-        private final MutableDateTime date = new MutableDateTime(0, DateTimeZone.UTC);
+        private Dates dates;
 
         public Longs(SortedNumericDocValues values) {
             this.values = values;
@@ -114,6 +138,9 @@ public Longs(SortedNumericDocValues values) {
         @Override
         public void setNextDocId(int docId) {
             values.setDocument(docId);
+            if (dates != null) {
+                dates.refreshArray();
+            }
         }
 
         public SortedNumericDocValues getInternalValues() {
@@ -128,14 +155,20 @@ public long getValue() {
             return values.valueAt(0);
         }
 
-        @Override
-        public List<Long> getValues() {
-            return Collections.unmodifiableList(this);
+        public ReadableDateTime getDate() {
+            if (dates == null) {
+                dates = new Dates(values);
+                dates.refreshArray();
+            }
+            return dates.getValue();
         }
 
-        public ReadableDateTime getDate() {
-            date.setMillis(getValue());
-            return date;
+        public List<ReadableDateTime> getDates() {
+            if (dates == null) {
+                dates = new Dates(values);
+                dates.refreshArray();
+            }
+            return dates;
         }
 
         @Override
@@ -147,10 +180,87 @@ public Long get(int index) {
         public int size() {
             return values.count();
         }
+    }
+
+    public static final class Dates extends ScriptDocValues<ReadableDateTime> {
+        private static final ReadableDateTime EPOCH = new DateTime(0, DateTimeZone.UTC);
+
+        private final SortedNumericDocValues values;
+        /**
+         * Values wrapped in {@link MutableDateTime}. Null by default an allocated on first usage so we allocate a reasonably size. We keep
+         * this array so we don't have allocate new {@link MutableDateTime}s on every usage. Instead we reuse them for every document.
+         */
+        private MutableDateTime[] dates;
+
+        public Dates(SortedNumericDocValues values) {
+            this.values = values;
+        }
 
+        /**
+         * Fetch the first field value or 0 millis after epoch if there are no values.
+         */
+        public ReadableDateTime getValue() {
+            if (values.count() == 0) {
+                return EPOCH;
+            }
+            return get(0);
+        }
+
+        @Override
+        public ReadableDateTime get(int index) {
+            if (index >= values.count()) {
+                throw new IndexOutOfBoundsException(
+                        "attempted to fetch the [" + index + "] date when there are only [" + values.count() + "] dates.");
+            }
+            return dates[index];
+        }
+
+        @Override
+        public int size() {
+            return values.count();
+        }
+
+        @Override
+        public void setNextDocId(int docId) {
+            values.setDocument(docId);
+            refreshArray();
+        }
+
+        /**
+         * Refresh the backing array. Package private so it can be called when {@link Longs} loads dates.
+         */
+        void refreshArray() {
+            if (values.count() == 0) {
+                return;
+            }
+            if (dates == null) {
+                // Happens for the document. We delay allocating dates so we can allocate it with a reasonable size.
+                dates = new MutableDateTime[values.count()];
+                for (int i = 0; i < dates.length; i++) {
+                    dates[i] = new MutableDateTime(values.valueAt(i), DateTimeZone.UTC);
+                }
+                return;
+            }
+            if (values.count() > dates.length) {
+                // Happens when we move to a new document and it has more dates than any documents before it.
+                MutableDateTime[] backup = dates;
+                dates = new MutableDateTime[values.count()];
+                System.arraycopy(backup, 0, dates, 0, backup.length);
+                for (int i = 0; i < backup.length; i++) {
+                    dates[i].setMillis(values.valueAt(i));
+                }
+                for (int i = backup.length; i < dates.length; i++) {
+                    dates[i] = new MutableDateTime(values.valueAt(i), DateTimeZone.UTC);
+                }
+                return;
+            }
+            for (int i = 0; i < values.count(); i++) {
+                dates[i].setMillis(values.valueAt(i));
+            }
+        }
     }
 
-    public static class Doubles extends AbstractList<Double> implements ScriptDocValues<Double> {
+    public static final class Doubles extends ScriptDocValues<Double> {
 
         private final SortedNumericDoubleValues values;
 
@@ -175,11 +285,6 @@ public double getValue() {
             return values.valueAt(0);
         }
 
-        @Override
-        public List<Double> getValues() {
-            return Collections.unmodifiableList(this);
-        }
-
         @Override
         public Double get(int index) {
             return values.valueAt(index);
@@ -191,7 +296,7 @@ public int size() {
         }
     }
 
-    class GeoPoints extends AbstractList<GeoPoint> implements ScriptDocValues<GeoPoint> {
+    public static final class GeoPoints extends ScriptDocValues<GeoPoint> {
 
         private final MultiGeoPointValues values;
 
@@ -238,11 +343,6 @@ public double getLon() {
             return getValue().lon();
         }
 
-        @Override
-        public List<GeoPoint> getValues() {
-            return Collections.unmodifiableList(this);
-        }
-
         @Override
         public GeoPoint get(int index) {
             final GeoPoint point = values.valueAt(index);
@@ -291,4 +391,69 @@ public double geohashDistanceWithDefault(String geohash, double defaultValue) {
             return geohashDistance(geohash);
         }
     }
+
+    public static final class Booleans extends ScriptDocValues<Boolean> {
+
+        private final SortedNumericDocValues values;
+
+        public Booleans(SortedNumericDocValues values) {
+            this.values = values;
+        }
+
+        @Override
+        public void setNextDocId(int docId) {
+            values.setDocument(docId);
+        }
+
+        public boolean getValue() {
+            return values.count() != 0 && values.valueAt(0) == 1;
+        }
+
+        @Override
+        public Boolean get(int index) {
+            return values.valueAt(index) == 1;
+        }
+
+        @Override
+        public int size() {
+            return values.count();
+        }
+
+    }
+
+    public static final class BytesRefs extends ScriptDocValues<BytesRef> {
+
+        private final SortedBinaryDocValues values;
+
+        public BytesRefs(SortedBinaryDocValues values) {
+            this.values = values;
+        }
+
+        @Override
+        public void setNextDocId(int docId) {
+            values.setDocument(docId);
+        }
+
+        public SortedBinaryDocValues getInternalValues() {
+            return this.values;
+        }
+
+        public BytesRef getValue() {
+            int numValues = values.count();
+            if (numValues == 0) {
+                return new BytesRef();
+            }
+            return values.valueAt(0);
+        }
+
+        @Override
+        public BytesRef get(int index) {
+            return values.valueAt(index);
+        }
+
+        @Override
+        public int size() {
+            return values.count();
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ShardFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/ShardFieldData.java
index 9e21562e8c722..73c6cff81701a 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/ShardFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/ShardFieldData.java
@@ -21,6 +21,7 @@
 
 import com.carrotsearch.hppc.ObjectLongHashMap;
 import org.apache.lucene.util.Accountable;
+import org.elasticsearch.common.FieldMemoryStats;
 import org.elasticsearch.common.metrics.CounterMetric;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
@@ -47,7 +48,8 @@ public FieldDataStats stats(String... fields) {
                 }
             }
         }
-        return new FieldDataStats(totalMetric.count(), evictionsMetric.count(), fieldTotals);
+        return new FieldDataStats(totalMetric.count(), evictionsMetric.count(), fieldTotals == null ? null :
+            new FieldMemoryStats(fieldTotals));
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/SingletonSortedNumericDoubleValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/SingletonSortedNumericDoubleValues.java
index 6c3c5a4f1610c..4207ac73a1a0c 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/SingletonSortedNumericDoubleValues.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/SingletonSortedNumericDoubleValues.java
@@ -34,7 +34,7 @@ final class SingletonSortedNumericDoubleValues extends SortedNumericDoubleValues
   private double value;
   private int count;
 
-  public SingletonSortedNumericDoubleValues(NumericDoubleValues in, Bits docsWithField) {
+  SingletonSortedNumericDoubleValues(NumericDoubleValues in, Bits docsWithField) {
     this.in = in;
     this.docsWithField = docsWithField instanceof MatchAllBits ? null : docsWithField;
   }
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/UidIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/UidIndexFieldData.java
new file mode 100644
index 0000000000000..8673e9b21640e
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/UidIndexFieldData.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.RandomAccessOrds;
+import org.apache.lucene.index.MultiDocValues;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefBuilder;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
+import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
+import org.elasticsearch.index.fielddata.plain.AbstractAtomicOrdinalsFieldData;
+import org.elasticsearch.index.mapper.UidFieldMapper;
+import org.elasticsearch.search.MultiValueMode;
+
+/** Fielddata view of the _uid field on indices that do not index _uid but _id.
+ *  It gets fielddata on the {@code _id field}, which is in-memory since the _id
+ *  field does not have doc values, and prepends {@code ${type}#} to all values.
+ *  Note that it does not add memory compared to what fielddata on the _id is
+ *  already using: this is just a view.
+ *  TODO: Remove fielddata access on _uid and _id, or add doc values to _id.
+ */
+public final class UidIndexFieldData implements IndexOrdinalsFieldData {
+
+    private final Index index;
+    private final String type;
+    private final BytesRef prefix;
+    private final IndexOrdinalsFieldData idFieldData;
+
+    public UidIndexFieldData(Index index, String type, IndexOrdinalsFieldData idFieldData) {
+        this.index = index;
+        this.type = type;
+        BytesRefBuilder prefix = new BytesRefBuilder();
+        prefix.append(new BytesRef(type));
+        prefix.append((byte) '#');
+        this.prefix = prefix.toBytesRef();
+        this.idFieldData = idFieldData;
+    }
+
+    @Override
+    public Index index() {
+        return index;
+    }
+
+    @Override
+    public String getFieldName() {
+        return UidFieldMapper.NAME;
+    }
+
+    @Override
+    public SortField sortField(Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse) {
+        XFieldComparatorSource source = new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested);
+        return new SortField(getFieldName(), source, reverse);
+    }
+
+    @Override
+    public AtomicOrdinalsFieldData load(LeafReaderContext context) {
+        return new UidAtomicFieldData(prefix, idFieldData.load(context));
+    }
+
+    @Override
+    public AtomicOrdinalsFieldData loadDirect(LeafReaderContext context) throws Exception {
+        return new UidAtomicFieldData(prefix, idFieldData.loadDirect(context));
+    }
+
+    @Override
+    public void clear() {
+        idFieldData.clear();
+    }
+
+    @Override
+    public IndexOrdinalsFieldData loadGlobal(DirectoryReader indexReader) {
+        return new UidIndexFieldData(index, type, idFieldData.loadGlobal(indexReader));
+    }
+
+    @Override
+    public IndexOrdinalsFieldData localGlobalDirect(DirectoryReader indexReader) throws Exception {
+        return new UidIndexFieldData(index, type, idFieldData.localGlobalDirect(indexReader));
+    }
+
+    @Override
+    public MultiDocValues.OrdinalMap getOrdinalMap() {
+        return idFieldData.getOrdinalMap();
+    }
+
+    static final class UidAtomicFieldData implements AtomicOrdinalsFieldData {
+
+        private final BytesRef prefix;
+        private final AtomicOrdinalsFieldData idFieldData;
+
+        UidAtomicFieldData(BytesRef prefix, AtomicOrdinalsFieldData idFieldData) {
+            this.prefix = prefix;
+            this.idFieldData = idFieldData;
+        }
+
+        @Override
+        public ScriptDocValues<?> getScriptValues() {
+            return AbstractAtomicOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION.apply(getOrdinalsValues());
+        }
+
+        @Override
+        public SortedBinaryDocValues getBytesValues() {
+            return FieldData.toString(getOrdinalsValues());
+        }
+
+        @Override
+        public long ramBytesUsed() {
+            return 0; // simple wrapper
+        }
+
+        @Override
+        public void close() {
+            idFieldData.close();
+        }
+
+        @Override
+        public RandomAccessOrds getOrdinalsValues() {
+            RandomAccessOrds idValues = idFieldData.getOrdinalsValues();
+            return new AbstractRandomAccessOrds() {
+
+                private final BytesRefBuilder scratch = new BytesRefBuilder();
+
+                @Override
+                public BytesRef lookupOrd(long ord) {
+                    scratch.setLength(0);
+                    scratch.append(prefix);
+                    scratch.append(idValues.lookupOrd(ord));
+                    return scratch.get();
+                }
+
+                @Override
+                public long getValueCount() {
+                    return idValues.getValueCount();
+                }
+
+                @Override
+                protected void doSetDocument(int docID) {
+                    idValues.setDocument(docID);
+                }
+
+                @Override
+                public long ordAt(int index) {
+                    return idValues.ordAt(index);
+                }
+
+                @Override
+                public int cardinality() {
+                    return idValues.cardinality();
+                }
+            };
+        }
+
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java b/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java
index 51f8f2b42bd1f..48b6a1127a5e0 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java
@@ -79,7 +79,7 @@ protected SortedBinaryDocValues getValues(LeafReaderContext context) throws IOEx
     protected void setScorer(Scorer scorer) {}
 
     @Override
-    public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
+    public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) {
         assert indexFieldData == null || fieldname.equals(indexFieldData.getFieldName());
 
         final boolean sortMissingLast = sortMissingLast(missingValue) ^ reversed;
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java b/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java
index 4684399a23d68..390a5493e273d 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java
@@ -64,7 +64,7 @@ protected SortedNumericDoubleValues getValues(LeafReaderContext context) throws
     protected void setScorer(Scorer scorer) {}
 
     @Override
-    public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
+    public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) {
         assert indexFieldData == null || fieldname.equals(indexFieldData.getFieldName());
 
         final double dMissingValue = (Double) missingObject(missingValue, reversed);
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java b/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java
index ba9b031cede3e..0546a5e5e8ea7 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java
@@ -56,7 +56,7 @@ public SortField.Type reducedType() {
     }
 
     @Override
-    public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
+    public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) {
         assert indexFieldData == null || fieldname.equals(indexFieldData.getFieldName());
 
         final float dMissingValue = (Float) missingObject(missingValue, reversed);
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java b/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java
index b2fd25e54457b..d652673308594 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java
@@ -55,7 +55,7 @@ public SortField.Type reducedType() {
     }
 
     @Override
-    public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
+    public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) {
         assert indexFieldData == null || fieldname.equals(indexFieldData.getFieldName());
 
         final Long dMissingValue = (Long) missingObject(missingValue, reversed);
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java
index aaecf2fa89609..f4b79105aadb8 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java
@@ -31,6 +31,7 @@
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData;
 import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
+import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.index.fielddata.plain.AbstractAtomicOrdinalsFieldData;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 
@@ -38,6 +39,7 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
 
 /**
  * Utility class to build global ordinals.
@@ -48,7 +50,9 @@ public enum GlobalOrdinalsBuilder {
     /**
      * Build global ordinals for the provided {@link IndexReader}.
      */
-    public static IndexOrdinalsFieldData build(final IndexReader indexReader, IndexOrdinalsFieldData indexFieldData, IndexSettings indexSettings, CircuitBreakerService breakerService, Logger logger) throws IOException {
+    public static IndexOrdinalsFieldData build(final IndexReader indexReader, IndexOrdinalsFieldData indexFieldData,
+            IndexSettings indexSettings, CircuitBreakerService breakerService, Logger logger,
+            Function<RandomAccessOrds, ScriptDocValues<?>> scriptFunction) throws IOException {
         assert indexReader.leaves().size() > 1;
         long startTimeNS = System.nanoTime();
 
@@ -70,8 +74,8 @@ public static IndexOrdinalsFieldData build(final IndexReader indexReader, IndexO
                     new TimeValue(System.nanoTime() - startTimeNS, TimeUnit.NANOSECONDS)
             );
         }
-        return new InternalGlobalOrdinalsIndexFieldData(indexSettings, indexFieldData.getFieldName(),
-                atomicFD, ordinalMap, memorySizeInBytes
+        return new GlobalOrdinalsIndexFieldData(indexSettings, indexFieldData.getFieldName(),
+                atomicFD, ordinalMap, memorySizeInBytes, scriptFunction
         );
     }
 
@@ -81,7 +85,7 @@ public static IndexOrdinalsFieldData buildEmpty(IndexSettings indexSettings, fin
         final AtomicOrdinalsFieldData[] atomicFD = new AtomicOrdinalsFieldData[indexReader.leaves().size()];
         final RandomAccessOrds[] subs = new RandomAccessOrds[indexReader.leaves().size()];
         for (int i = 0; i < indexReader.leaves().size(); ++i) {
-            atomicFD[i] = new AbstractAtomicOrdinalsFieldData() {
+            atomicFD[i] = new AbstractAtomicOrdinalsFieldData(AbstractAtomicOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION) {
                 @Override
                 public RandomAccessOrds getOrdinalsValues() {
                     return DocValues.emptySortedSet();
@@ -104,8 +108,8 @@ public void close() {
             subs[i] = atomicFD[i].getOrdinalsValues();
         }
         final OrdinalMap ordinalMap = OrdinalMap.build(null, subs, PackedInts.DEFAULT);
-        return new InternalGlobalOrdinalsIndexFieldData(indexSettings, indexFieldData.getFieldName(),
-                atomicFD, ordinalMap, 0
+        return new GlobalOrdinalsIndexFieldData(indexSettings, indexFieldData.getFieldName(),
+                atomicFD, ordinalMap, 0, AbstractAtomicOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION
         );
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsIndexFieldData.java
index 3e75620400247..b29ae47a14215 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsIndexFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsIndexFieldData.java
@@ -20,6 +20,9 @@
 
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.MultiDocValues;
+import org.apache.lucene.index.RandomAccessOrds;
+import org.apache.lucene.search.SortField;
 import org.apache.lucene.util.Accountable;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.index.AbstractIndexComponent;
@@ -28,23 +31,39 @@
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
+import org.elasticsearch.index.fielddata.ScriptDocValues;
+import org.elasticsearch.index.fielddata.plain.AbstractAtomicOrdinalsFieldData;
 import org.elasticsearch.search.MultiValueMode;
 
 import java.util.Collection;
 import java.util.Collections;
+import java.util.function.Function;
 
 /**
  * {@link IndexFieldData} base class for concrete global ordinals implementations.
  */
-public abstract class GlobalOrdinalsIndexFieldData extends AbstractIndexComponent implements IndexOrdinalsFieldData, Accountable {
+public class GlobalOrdinalsIndexFieldData extends AbstractIndexComponent implements IndexOrdinalsFieldData, Accountable {
 
     private final String fieldName;
     private final long memorySizeInBytes;
 
-    protected GlobalOrdinalsIndexFieldData(IndexSettings indexSettings, String fieldName, long memorySizeInBytes) {
+    private final MultiDocValues.OrdinalMap ordinalMap;
+    private final Atomic[] atomicReaders;
+    private final Function<RandomAccessOrds, ScriptDocValues<?>> scriptFunction;
+
+
+    protected GlobalOrdinalsIndexFieldData(IndexSettings indexSettings, String fieldName, AtomicOrdinalsFieldData[] segmentAfd,
+                                           MultiDocValues.OrdinalMap ordinalMap, long memorySizeInBytes, Function<RandomAccessOrds,
+                                           ScriptDocValues<?>> scriptFunction) {
         super(indexSettings);
         this.fieldName = fieldName;
         this.memorySizeInBytes = memorySizeInBytes;
+        this.ordinalMap = ordinalMap;
+        this.atomicReaders = new Atomic[segmentAfd.length];
+        for (int i = 0; i < segmentAfd.length; i++) {
+            atomicReaders[i] = new Atomic(segmentAfd[i], ordinalMap, i);
+        }
+        this.scriptFunction = scriptFunction;
     }
 
     @Override
@@ -68,7 +87,7 @@ public String getFieldName() {
     }
 
     @Override
-    public XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested) {
+    public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse) {
         throw new UnsupportedOperationException("no global ordinals sorting yet");
     }
 
@@ -87,4 +106,57 @@ public Collection<Accountable> getChildResources() {
         // TODO: break down ram usage?
         return Collections.emptyList();
     }
+
+    @Override
+    public AtomicOrdinalsFieldData load(LeafReaderContext context) {
+        return atomicReaders[context.ord];
+    }
+
+    @Override
+    public MultiDocValues.OrdinalMap getOrdinalMap() {
+        return ordinalMap;
+    }
+
+    private final class Atomic extends AbstractAtomicOrdinalsFieldData {
+
+        private final AtomicOrdinalsFieldData afd;
+        private final MultiDocValues.OrdinalMap ordinalMap;
+        private final int segmentIndex;
+
+        private Atomic(AtomicOrdinalsFieldData afd, MultiDocValues.OrdinalMap ordinalMap, int segmentIndex) {
+            super(scriptFunction);
+            this.afd = afd;
+            this.ordinalMap = ordinalMap;
+            this.segmentIndex = segmentIndex;
+        }
+
+        @Override
+        public RandomAccessOrds getOrdinalsValues() {
+            final RandomAccessOrds values = afd.getOrdinalsValues();
+            if (values.getValueCount() == ordinalMap.getValueCount()) {
+                // segment ordinals match global ordinals
+                return values;
+            }
+            final RandomAccessOrds[] bytesValues = new RandomAccessOrds[atomicReaders.length];
+            for (int i = 0; i < bytesValues.length; i++) {
+                bytesValues[i] = atomicReaders[i].afd.getOrdinalsValues();
+            }
+            return new GlobalOrdinalMapping(ordinalMap, bytesValues, segmentIndex);
+        }
+
+        @Override
+        public long ramBytesUsed() {
+            return afd.ramBytesUsed();
+        }
+
+        @Override
+        public Collection<Accountable> getChildResources() {
+            return afd.getChildResources();
+        }
+
+        @Override
+        public void close() {
+        }
+
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/InternalGlobalOrdinalsIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/InternalGlobalOrdinalsIndexFieldData.java
deleted file mode 100644
index 5b8ef83b10e03..0000000000000
--- a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/InternalGlobalOrdinalsIndexFieldData.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.index.fielddata.ordinals;
-
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.MultiDocValues.OrdinalMap;
-import org.apache.lucene.index.RandomAccessOrds;
-import org.apache.lucene.util.Accountable;
-import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData;
-import org.elasticsearch.index.fielddata.plain.AbstractAtomicOrdinalsFieldData;
-
-import java.util.Collection;
-
-/**
- * {@link org.elasticsearch.index.fielddata.IndexFieldData} impl based on global ordinals.
- */
-final class InternalGlobalOrdinalsIndexFieldData extends GlobalOrdinalsIndexFieldData {
-
-    private final Atomic[] atomicReaders;
-
-    InternalGlobalOrdinalsIndexFieldData(IndexSettings indexSettings, String fieldName, AtomicOrdinalsFieldData[] segmentAfd, OrdinalMap ordinalMap, long memorySizeInBytes) {
-        super(indexSettings, fieldName, memorySizeInBytes);
-        this.atomicReaders = new Atomic[segmentAfd.length];
-        for (int i = 0; i < segmentAfd.length; i++) {
-            atomicReaders[i] = new Atomic(segmentAfd[i], ordinalMap, i);
-        }
-    }
-
-    @Override
-    public AtomicOrdinalsFieldData load(LeafReaderContext context) {
-        return atomicReaders[context.ord];
-    }
-
-    private final class Atomic extends AbstractAtomicOrdinalsFieldData {
-
-        private final AtomicOrdinalsFieldData afd;
-        private final OrdinalMap ordinalMap;
-        private final int segmentIndex;
-
-        private Atomic(AtomicOrdinalsFieldData afd, OrdinalMap ordinalMap, int segmentIndex) {
-            this.afd = afd;
-            this.ordinalMap = ordinalMap;
-            this.segmentIndex = segmentIndex;
-        }
-
-        @Override
-        public RandomAccessOrds getOrdinalsValues() {
-            final RandomAccessOrds values = afd.getOrdinalsValues();
-            if (values.getValueCount() == ordinalMap.getValueCount()) {
-                // segment ordinals match global ordinals
-                return values;
-            }
-            final RandomAccessOrds[] bytesValues = new RandomAccessOrds[atomicReaders.length];
-            for (int i = 0; i < bytesValues.length; i++) {
-                bytesValues[i] = atomicReaders[i].afd.getOrdinalsValues();
-            }
-            return new GlobalOrdinalMapping(ordinalMap, bytesValues, segmentIndex);
-        }
-
-        @Override
-        public long ramBytesUsed() {
-            return afd.ramBytesUsed();
-        }
-
-        @Override
-        public Collection<Accountable> getChildResources() {
-            return afd.getChildResources();
-        }
-
-        @Override
-        public void close() {
-        }
-
-    }
-
-}
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/SinglePackedOrdinals.java b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/SinglePackedOrdinals.java
index aa09bac4dcf82..747a33900b18d 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/SinglePackedOrdinals.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/SinglePackedOrdinals.java
@@ -69,7 +69,7 @@ private static class Docs extends SortedDocValues {
         private final PackedInts.Reader reader;
         private final ValuesHolder values;
 
-        public Docs(SinglePackedOrdinals parent, ValuesHolder values) {
+        Docs(SinglePackedOrdinals parent, ValuesHolder values) {
             this.maxOrd = parent.valueCount;
             this.reader = parent.reader;
             this.values = values;
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicOrdinalsFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicOrdinalsFieldData.java
index 51de4c1be53dc..1a6e5d0886cfe 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicOrdinalsFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicOrdinalsFieldData.java
@@ -29,15 +29,26 @@
 
 import java.util.Collection;
 import java.util.Collections;
+import java.util.function.Function;
 
 
 /**
  */
 public abstract class AbstractAtomicOrdinalsFieldData implements AtomicOrdinalsFieldData {
 
+    public static final Function<RandomAccessOrds, ScriptDocValues<?>> DEFAULT_SCRIPT_FUNCTION =
+            ((Function<RandomAccessOrds, SortedBinaryDocValues>) FieldData::toString)
+            .andThen(ScriptDocValues.Strings::new);
+
+    private final Function<RandomAccessOrds, ScriptDocValues<?>> scriptFunction;
+
+    protected AbstractAtomicOrdinalsFieldData(Function<RandomAccessOrds, ScriptDocValues<?>> scriptFunction) {
+        this.scriptFunction = scriptFunction;
+    }
+
     @Override
-    public final ScriptDocValues getScriptValues() {
-        return new ScriptDocValues.Strings(getBytesValues());
+    public final ScriptDocValues<?> getScriptValues() {
+        return scriptFunction.apply(getOrdinalsValues());
     }
 
     @Override
@@ -46,7 +57,7 @@ public final SortedBinaryDocValues getBytesValues() {
     }
 
     public static AtomicOrdinalsFieldData empty() {
-        return new AbstractAtomicOrdinalsFieldData() {
+        return new AbstractAtomicOrdinalsFieldData(DEFAULT_SCRIPT_FUNCTION) {
 
             @Override
             public long ramBytesUsed() {
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicParentChildFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicParentChildFieldData.java
deleted file mode 100644
index 1a801d75411fb..0000000000000
--- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicParentChildFieldData.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.fielddata.plain;
-
-import org.apache.lucene.index.DocValues;
-import org.apache.lucene.index.SortedDocValues;
-import org.apache.lucene.util.Accountable;
-import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.index.fielddata.AtomicParentChildFieldData;
-import org.elasticsearch.index.fielddata.ScriptDocValues;
-import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Set;
-
-import static java.util.Collections.emptySet;
-
-
-/**
- */
-abstract class AbstractAtomicParentChildFieldData implements AtomicParentChildFieldData {
-
-    @Override
-    public final ScriptDocValues getScriptValues() {
-        return new ScriptDocValues.Strings(getBytesValues());
-    }
-
-    @Override
-    public final SortedBinaryDocValues getBytesValues() {
-        return new SortedBinaryDocValues() {
-
-            private final BytesRef[] terms = new BytesRef[2];
-            private int count;
-
-            @Override
-            public void setDocument(int docId) {
-                count = 0;
-                for (String type : types()) {
-                    final SortedDocValues values = getOrdinalsValues(type);
-                    final int ord = values.getOrd(docId);
-                    if (ord >= 0) {
-                        terms[count++] = values.lookupOrd(ord);
-                    }
-                }
-                assert count <= 2 : "A single doc can potentially be both parent and child, so the maximum allowed values is 2";
-                if (count > 1) {
-                    int cmp = terms[0].compareTo(terms[1]);
-                    if (cmp > 0) {
-                        ArrayUtil.swap(terms, 0, 1);
-                    } else if (cmp == 0) {
-                        // If the id is the same between types the only omit one. For example: a doc has parent#1 in _uid field and has grand_parent#1 in _parent field.
-                        count = 1;
-                    }
-                }
-            }
-
-            @Override
-            public int count() {
-                return count;
-            }
-
-            @Override
-            public BytesRef valueAt(int index) {
-                return terms[index];
-            }
-        };
-    }
-
-    public static AtomicParentChildFieldData empty() {
-        return new AbstractAtomicParentChildFieldData() {
-
-            @Override
-            public long ramBytesUsed() {
-                return 0;
-            }
-
-            @Override
-            public Collection<Accountable> getChildResources() {
-                return Collections.emptyList();
-            }
-
-            @Override
-            public void close() {
-            }
-
-            @Override
-            public SortedDocValues getOrdinalsValues(String type) {
-                return DocValues.emptySorted();
-            }
-
-            @Override
-            public Set<String> types() {
-                return emptySet();
-            }
-        };
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractGeoPointDVIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractGeoPointDVIndexFieldData.java
index 23e770121a729..a9c7e7c79b4ea 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractGeoPointDVIndexFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractGeoPointDVIndexFieldData.java
@@ -22,6 +22,7 @@
 import org.apache.lucene.index.DocValues;
 import org.apache.lucene.index.LeafReaderContext;
 import org.elasticsearch.Version;
+import org.apache.lucene.search.SortField;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
@@ -44,7 +45,7 @@ public abstract class AbstractGeoPointDVIndexFieldData extends DocValuesIndexFie
     }
 
     @Override
-    public final XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested) {
+    public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse) {
         throw new IllegalArgumentException("can't sort on geo_point field without using specific sorting feature, like geo_distance");
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexGeoPointFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexGeoPointFieldData.java
index 90554bd130846..bdf1bbac33235 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexGeoPointFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexGeoPointFieldData.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.index.fielddata.plain;
 
+import org.apache.lucene.search.SortField;
 import org.apache.lucene.spatial.geopoint.document.GeoPointField;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefIterator;
@@ -28,6 +29,7 @@
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.fielddata.AtomicGeoPointFieldData;
+import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.fielddata.IndexGeoPointFieldData;
@@ -99,12 +101,12 @@ public GeoPoint next() throws IOException {
         }
     }
 
-    public AbstractIndexGeoPointFieldData(IndexSettings indexSettings, String fieldName, IndexFieldDataCache cache) {
+    AbstractIndexGeoPointFieldData(IndexSettings indexSettings, String fieldName, IndexFieldDataCache cache) {
         super(indexSettings, fieldName, cache);
     }
 
     @Override
-    public final XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested) {
+    public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse) {
         throw new IllegalArgumentException("can't sort on geo_point field without using specific sorting feature, like geo_distance");
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java
index 0223c1a0b63b3..1dbd082f93bc8 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java
@@ -22,20 +22,17 @@
 import org.apache.lucene.index.FilteredTermsEnum;
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.MultiDocValues;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.common.Nullable;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData;
-import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
-import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
 import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
-import org.elasticsearch.search.MultiValueMode;
 
 import java.io.IOException;
 
@@ -56,8 +53,8 @@ protected AbstractIndexOrdinalsFieldData(IndexSettings indexSettings, String fie
     }
 
     @Override
-    public XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested) {
-        return new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested);
+    public MultiDocValues.OrdinalMap getOrdinalMap() {
+        return null;
     }
 
     @Override
@@ -97,7 +94,8 @@ public IndexOrdinalsFieldData loadGlobal(DirectoryReader indexReader) {
 
     @Override
     public IndexOrdinalsFieldData localGlobalDirect(DirectoryReader indexReader) throws Exception {
-        return GlobalOrdinalsBuilder.build(indexReader, this, indexSettings, breakerService, logger);
+        return GlobalOrdinalsBuilder.build(indexReader, this, indexSettings, breakerService, logger,
+                AbstractAtomicOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION);
     }
 
     @Override
@@ -131,7 +129,7 @@ private static final class FrequencyFilter extends FilteredTermsEnum {
 
         private int minFreq;
         private int maxFreq;
-        public FrequencyFilter(TermsEnum delegate, int minFreq, int maxFreq) {
+        FrequencyFilter(TermsEnum delegate, int minFreq, int maxFreq) {
             super(delegate, false);
             this.minFreq = minFreq;
             this.maxFreq = maxFreq;
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLatLonPointDVIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLatLonPointDVIndexFieldData.java
new file mode 100644
index 0000000000000..3b4ac58e0e81e
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLatLonPointDVIndexFieldData.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.index.fielddata.plain;
+
+import org.apache.lucene.document.LatLonDocValuesField;
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.DocValuesType;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.SortField;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.fielddata.AtomicGeoPointFieldData;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexFieldDataCache;
+import org.elasticsearch.index.fielddata.IndexGeoPointFieldData;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.indices.breaker.CircuitBreakerService;
+import org.elasticsearch.search.MultiValueMode;
+
+import java.io.IOException;
+
+public abstract class AbstractLatLonPointDVIndexFieldData extends DocValuesIndexFieldData
+    implements IndexGeoPointFieldData {
+    AbstractLatLonPointDVIndexFieldData(Index index, String fieldName) {
+        super(index, fieldName);
+    }
+
+    @Override
+    public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMode, XFieldComparatorSource.Nested nested, boolean reverse) {
+        throw new IllegalArgumentException("can't sort on geo_point field without using specific sorting feature, like geo_distance");
+    }
+
+    public static class LatLonPointDVIndexFieldData extends AbstractLatLonPointDVIndexFieldData {
+        public LatLonPointDVIndexFieldData(Index index, String fieldName) {
+            super(index, fieldName);
+        }
+
+        @Override
+        public AtomicGeoPointFieldData load(LeafReaderContext context) {
+            try {
+                LeafReader reader = context.reader();
+                FieldInfo info = reader.getFieldInfos().fieldInfo(fieldName);
+                if (info != null) {
+                    checkCompatible(info);
+                }
+                return new LatLonPointDVAtomicFieldData(DocValues.getSortedNumeric(reader, fieldName));
+            } catch (IOException e) {
+                throw new IllegalStateException("Cannot load doc values", e);
+            }
+        }
+
+        @Override
+        public AtomicGeoPointFieldData loadDirect(LeafReaderContext context) throws Exception {
+            return load(context);
+        }
+
+        /** helper: checks a fieldinfo and throws exception if its definitely not a LatLonDocValuesField */
+        static void checkCompatible(FieldInfo fieldInfo) {
+            // dv properties could be "unset", if you e.g. used only StoredField with this same name in the segment.
+            if (fieldInfo.getDocValuesType() != DocValuesType.NONE
+                && fieldInfo.getDocValuesType() != LatLonDocValuesField.TYPE.docValuesType()) {
+                throw new IllegalArgumentException("field=\"" + fieldInfo.name + "\" was indexed with docValuesType="
+                    + fieldInfo.getDocValuesType() + " but this type has docValuesType="
+                    + LatLonDocValuesField.TYPE.docValuesType() + ", is the field really a LatLonDocValuesField?");
+            }
+        }
+    }
+
+    public static class Builder implements IndexFieldData.Builder {
+        @Override
+        public IndexFieldData<?> build(IndexSettings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache,
+                                       CircuitBreakerService breakerService, MapperService mapperService) {
+            // ignore breaker
+            return new LatLonPointDVIndexFieldData(indexSettings.getIndex(), fieldType.name());
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AtomicLongFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AtomicLongFieldData.java
index b3b0604e9e21c..c52ccb90bed8a 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AtomicLongFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AtomicLongFieldData.java
@@ -19,28 +19,24 @@
 
 package org.elasticsearch.index.fielddata.plain;
 
-import org.apache.lucene.index.DocValues;
-import org.apache.lucene.index.SortedNumericDocValues;
-import org.apache.lucene.util.Accountable;
 import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
 import org.elasticsearch.index.fielddata.FieldData;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
 import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
 
-import java.util.Collection;
-import java.util.Collections;
-
-
 /**
  * Specialization of {@link AtomicNumericFieldData} for integers.
  */
 abstract class AtomicLongFieldData implements AtomicNumericFieldData {
 
     private final long ramBytesUsed;
+    /** True if this numeric data is for a boolean field, and so only has values 0 and 1. */
+    private final boolean isBoolean;
 
-    AtomicLongFieldData(long ramBytesUsed) {
+    AtomicLongFieldData(long ramBytesUsed, boolean isBoolean) {
         this.ramBytesUsed = ramBytesUsed;
+        this.isBoolean = isBoolean;
     }
 
     @Override
@@ -50,7 +46,11 @@ public long ramBytesUsed() {
 
     @Override
     public final ScriptDocValues getScriptValues() {
-        return new ScriptDocValues.Longs(getLongValues());
+        if (isBoolean) {
+            return new ScriptDocValues.Booleans(getLongValues());
+        } else {
+            return new ScriptDocValues.Longs(getLongValues());
+        }
     }
 
     @Override
@@ -63,24 +63,6 @@ public final SortedNumericDoubleValues getDoubleValues() {
         return FieldData.castToDouble(getLongValues());
     }
 
-    public static AtomicNumericFieldData empty(final int maxDoc) {
-        return new AtomicLongFieldData(0) {
-
-            @Override
-            public SortedNumericDocValues getLongValues() {
-                return DocValues.emptySortedNumeric(maxDoc);
-            }
-
-            @Override
-            public Collection<Accountable> getChildResources() {
-                return Collections.emptyList();
-            }
-
-        };
-    }
-
     @Override
-    public void close() {
-    }
-
+    public void close() {}
 }
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVIndexFieldData.java
index 586ad1f0d4869..a7e1981766704 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVIndexFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVIndexFieldData.java
@@ -20,9 +20,12 @@
 package org.elasticsearch.index.fielddata.plain;
 
 import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.SortedSetSortField;
+import org.apache.lucene.search.SortedSetSelector;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.fielddata.IndexFieldData;
-import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
 import org.elasticsearch.search.MultiValueMode;
 
@@ -43,7 +46,21 @@ public BinaryDVAtomicFieldData loadDirect(LeafReaderContext context) throws Exce
     }
 
     @Override
-    public org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource comparatorSource(Object missingValue, MultiValueMode sortMode, Nested nested) {
-        return new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested);
+    public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMode, XFieldComparatorSource.Nested nested, boolean reverse) {
+        XFieldComparatorSource source = new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested);
+        /**
+         * Check if we can use a simple {@link SortedSetSortField} compatible with index sorting and
+         * returns a custom sort field otherwise.
+         */
+        if (nested != null ||
+                (sortMode != MultiValueMode.MAX && sortMode != MultiValueMode.MIN) ||
+                (source.sortMissingFirst(missingValue) == false && source.sortMissingLast(missingValue) == false)) {
+            return new SortField(getFieldName(), source, reverse);
+        }
+        SortField sortField = new SortedSetSortField(fieldName, reverse,
+            sortMode == MultiValueMode.MAX ? SortedSetSelector.Type.MAX : SortedSetSelector.Type.MIN);
+        sortField.setMissingValue(source.sortMissingLast(missingValue) ^ reverse ?
+            SortedSetSortField.STRING_LAST : SortedSetSortField.STRING_FIRST);
+        return sortField;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVAtomicFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVAtomicFieldData.java
index 9963f7d51a85e..8d43241ba75e0 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVAtomicFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVAtomicFieldData.java
@@ -101,7 +101,7 @@ public BytesRef valueAt(int index) {
 
     @Override
     public ScriptDocValues getScriptValues() {
-        throw new UnsupportedOperationException();
+        return new ScriptDocValues.BytesRefs(getBytesValues());
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVIndexFieldData.java
index bd3cdd71184c5..398093c034b79 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVIndexFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVIndexFieldData.java
@@ -21,6 +21,7 @@
 
 import org.apache.lucene.index.DocValues;
 import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.SortField;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
@@ -41,7 +42,7 @@ public BytesBinaryDVIndexFieldData(Index index, String fieldName) {
     }
 
     @Override
-    public final XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested) {
+    public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse) {
         throw new IllegalArgumentException("can't sort on binary field");
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/ConstantIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/ConstantIndexFieldData.java
new file mode 100644
index 0000000000000..19367d06679ff
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/ConstantIndexFieldData.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.plain;
+
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.RandomAccessOrds;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexFieldDataCache;
+import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
+import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.TextFieldMapper;
+import org.elasticsearch.indices.breaker.CircuitBreakerService;
+import org.elasticsearch.search.MultiValueMode;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.function.Function;
+
+public class ConstantIndexFieldData extends AbstractIndexOrdinalsFieldData {
+
+    public static class Builder implements IndexFieldData.Builder {
+
+        private final Function<MapperService, String> valueFunction;
+
+        public Builder(Function<MapperService, String> valueFunction) {
+            this.valueFunction = valueFunction;
+        }
+
+        @Override
+        public IndexFieldData<?> build(IndexSettings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache,
+                CircuitBreakerService breakerService, MapperService mapperService) {
+            return new ConstantIndexFieldData(indexSettings, fieldType.name(), valueFunction.apply(mapperService));
+        }
+
+    }
+
+    private static class ConstantAtomicFieldData extends AbstractAtomicOrdinalsFieldData {
+
+        private final String value;
+
+        ConstantAtomicFieldData(String value) {
+            super(DEFAULT_SCRIPT_FUNCTION);
+            this.value = value;
+        }
+
+        @Override
+        public long ramBytesUsed() {
+            return 0;
+        }
+
+        @Override
+        public Collection<Accountable> getChildResources() {
+            return Collections.emptyList();
+        }
+
+        @Override
+        public RandomAccessOrds getOrdinalsValues() {
+            final BytesRef term = new BytesRef(value);
+            final SortedDocValues sortedValues = new SortedDocValues() {
+
+                @Override
+                public BytesRef lookupOrd(int ord) {
+                    return term;
+                }
+
+                @Override
+                public int getValueCount() {
+                    return 1;
+                }
+
+                @Override
+                public int getOrd(int docID) {
+                    return 0;
+                }
+            };
+            return (RandomAccessOrds) DocValues.singleton(sortedValues);
+        }
+
+        @Override
+        public void close() {
+        }
+
+    }
+
+    private final AtomicOrdinalsFieldData atomicFieldData;
+
+    private ConstantIndexFieldData(IndexSettings indexSettings, String name, String value) {
+        super(indexSettings, name, null, null,
+                TextFieldMapper.Defaults.FIELDDATA_MIN_FREQUENCY,
+                TextFieldMapper.Defaults.FIELDDATA_MAX_FREQUENCY,
+                TextFieldMapper.Defaults.FIELDDATA_MIN_SEGMENT_SIZE);
+        atomicFieldData = new ConstantAtomicFieldData(value);
+    }
+
+    @Override
+    public void clear() {
+    }
+
+    @Override
+    public final AtomicOrdinalsFieldData load(LeafReaderContext context) {
+        return atomicFieldData;
+    }
+
+    @Override
+    public AtomicOrdinalsFieldData loadDirect(LeafReaderContext context)
+            throws Exception {
+        return atomicFieldData;
+    }
+
+    @Override
+    public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMode, XFieldComparatorSource.Nested nested,
+            boolean reverse) {
+        final XFieldComparatorSource source = new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested);
+        return new SortField(getFieldName(), source, reverse);
+    }
+
+    @Override
+    public IndexOrdinalsFieldData loadGlobal(DirectoryReader indexReader) {
+        return this;
+    }
+
+    @Override
+    public IndexOrdinalsFieldData localGlobalDirect(DirectoryReader indexReader) throws Exception {
+        return loadGlobal(indexReader);
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/DocValuesIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/DocValuesIndexFieldData.java
index 4621876399663..c77ceb57457ea 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/DocValuesIndexFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/DocValuesIndexFieldData.java
@@ -21,12 +21,14 @@
 
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.RandomAccessOrds;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType;
+import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
@@ -34,6 +36,7 @@
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 
 import java.util.Set;
+import java.util.function.Function;
 
 import static java.util.Collections.unmodifiableSet;
 import static org.elasticsearch.common.util.set.Sets.newHashSet;
@@ -72,12 +75,18 @@ public static class Builder implements IndexFieldData.Builder {
         private static final Set<String> BINARY_INDEX_FIELD_NAMES = unmodifiableSet(newHashSet(UidFieldMapper.NAME, IdFieldMapper.NAME));
 
         private NumericType numericType;
+        private Function<RandomAccessOrds, ScriptDocValues<?>> scriptFunction = AbstractAtomicOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION;
 
         public Builder numericType(NumericType type) {
             this.numericType = type;
             return this;
         }
 
+        public Builder scriptFunction(Function<RandomAccessOrds, ScriptDocValues<?>> scriptFunction) {
+            this.scriptFunction = scriptFunction;
+            return this;
+        }
+
         @Override
         public IndexFieldData<?> build(IndexSettings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache,
                                        CircuitBreakerService breakerService, MapperService mapperService) {
@@ -89,7 +98,7 @@ public IndexFieldData<?> build(IndexSettings indexSettings, MappedFieldType fiel
             } else if (numericType != null) {
                 return new SortedNumericDVIndexFieldData(indexSettings.getIndex(), fieldName, numericType);
             } else {
-                return new SortedSetDVOrdinalsIndexFieldData(indexSettings, cache, fieldName, breakerService);
+                return new SortedSetDVOrdinalsIndexFieldData(indexSettings, cache, fieldName, breakerService, scriptFunction);
             }
         }
 
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointArrayAtomicFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointArrayAtomicFieldData.java
index b1a97a878ee45..0ce02713be444 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointArrayAtomicFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointArrayAtomicFieldData.java
@@ -49,7 +49,7 @@ static class WithOrdinals extends GeoPointArrayAtomicFieldData {
         private final Ordinals ordinals;
         private final int maxDoc;
 
-        public WithOrdinals(LongArray indexedPoints, Ordinals ordinals, int maxDoc) {
+        WithOrdinals(LongArray indexedPoints, Ordinals ordinals, int maxDoc) {
             super();
             this.indexedPoints = indexedPoints;
             this.ordinals = ordinals;
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointArrayIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointArrayIndexFieldData.java
index d484c503c2b54..18313f3274517 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointArrayIndexFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointArrayIndexFieldData.java
@@ -68,7 +68,7 @@ public AtomicGeoPointFieldData loadDirect(LeafReaderContext context) throws Exce
             estimator.afterLoad(null, data.ramBytesUsed());
             return data;
         }
-        return (indexSettings.getIndexVersionCreated().before(Version.V_2_2_0) == true) ?
+        return (indexSettings.getIndexVersionCreated().before(Version.V_2_2_0)) ?
             loadLegacyFieldData(reader, estimator, terms, data) : loadFieldData22(reader, estimator, terms, data);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointArrayLegacyAtomicFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointArrayLegacyAtomicFieldData.java
index 8d0953fb6e6b6..ec357633464b3 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointArrayLegacyAtomicFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointArrayLegacyAtomicFieldData.java
@@ -50,7 +50,7 @@ static class WithOrdinals extends GeoPointArrayLegacyAtomicFieldData {
         private final Ordinals ordinals;
         private final int maxDoc;
 
-        public WithOrdinals(DoubleArray lon, DoubleArray lat, Ordinals ordinals, int maxDoc) {
+        WithOrdinals(DoubleArray lon, DoubleArray lat, Ordinals ordinals, int maxDoc) {
             super();
             this.lon = lon;
             this.lat = lat;
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/IndexIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/IndexIndexFieldData.java
deleted file mode 100644
index d57c023371e58..0000000000000
--- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/IndexIndexFieldData.java
+++ /dev/null
@@ -1,136 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.fielddata.plain;
-
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.DocValues;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.RandomAccessOrds;
-import org.apache.lucene.index.SortedDocValues;
-import org.apache.lucene.util.Accountable;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData;
-import org.elasticsearch.index.fielddata.IndexFieldData;
-import org.elasticsearch.index.fielddata.IndexFieldDataCache;
-import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
-import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.TextFieldMapper;
-import org.elasticsearch.indices.breaker.CircuitBreakerService;
-
-import java.util.Collection;
-import java.util.Collections;
-
-public class IndexIndexFieldData extends AbstractIndexOrdinalsFieldData {
-
-    public static class Builder implements IndexFieldData.Builder {
-
-        @Override
-        public IndexFieldData<?> build(IndexSettings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache,
-                CircuitBreakerService breakerService, MapperService mapperService) {
-            return new IndexIndexFieldData(indexSettings, fieldType.name());
-        }
-
-    }
-
-    private static class IndexAtomicFieldData extends AbstractAtomicOrdinalsFieldData {
-
-        private final String index;
-
-        IndexAtomicFieldData(String index) {
-            this.index = index;
-        }
-
-        @Override
-        public long ramBytesUsed() {
-            return 0;
-        }
-
-        @Override
-        public Collection<Accountable> getChildResources() {
-            return Collections.emptyList();
-        }
-
-        @Override
-        public RandomAccessOrds getOrdinalsValues() {
-            final BytesRef term = new BytesRef(index);
-            final SortedDocValues sortedValues = new SortedDocValues() {
-
-                @Override
-                public BytesRef lookupOrd(int ord) {
-                    return term;
-                }
-
-                @Override
-                public int getValueCount() {
-                    return 1;
-                }
-
-                @Override
-                public int getOrd(int docID) {
-                    return 0;
-                }
-            };
-            return (RandomAccessOrds) DocValues.singleton(sortedValues);
-        }
-
-        @Override
-        public void close() {
-        }
-
-    }
-
-    private final AtomicOrdinalsFieldData atomicFieldData;
-
-    private IndexIndexFieldData(IndexSettings indexSettings, String name) {
-        super(indexSettings, name, null, null,
-                TextFieldMapper.Defaults.FIELDDATA_MIN_FREQUENCY,
-                TextFieldMapper.Defaults.FIELDDATA_MAX_FREQUENCY,
-                TextFieldMapper.Defaults.FIELDDATA_MIN_SEGMENT_SIZE);
-        atomicFieldData = new IndexAtomicFieldData(index().getName());
-    }
-
-    @Override
-    public void clear() {
-    }
-
-    @Override
-    public final AtomicOrdinalsFieldData load(LeafReaderContext context) {
-        return atomicFieldData;
-    }
-
-    @Override
-    public AtomicOrdinalsFieldData loadDirect(LeafReaderContext context)
-            throws Exception {
-        return atomicFieldData;
-    }
-
-    @Override
-    public IndexOrdinalsFieldData loadGlobal(DirectoryReader indexReader) {
-        return this;
-    }
-
-    @Override
-    public IndexOrdinalsFieldData localGlobalDirect(DirectoryReader indexReader) throws Exception {
-        return loadGlobal(indexReader);
-    }
-
-}
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointDVAtomicFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointDVAtomicFieldData.java
new file mode 100644
index 0000000000000..d11a79c255654
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointDVAtomicFieldData.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.index.fielddata.plain;
+
+import org.apache.lucene.geo.GeoEncodingUtils;
+import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.RamUsageEstimator;
+import org.elasticsearch.common.geo.GeoPoint;
+import org.elasticsearch.index.fielddata.MultiGeoPointValues;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+
+final class LatLonPointDVAtomicFieldData extends AbstractAtomicGeoPointFieldData {
+    private final SortedNumericDocValues values;
+
+    LatLonPointDVAtomicFieldData(SortedNumericDocValues values) {
+        super();
+        this.values = values;
+    }
+
+    @Override
+    public long ramBytesUsed() {
+        return 0; // not exposed by lucene
+    }
+
+    @Override
+    public Collection<Accountable> getChildResources() {
+        return Collections.emptyList();
+    }
+
+    @Override
+    public void close() {
+        // noop
+    }
+
+    @Override
+    public MultiGeoPointValues getGeoPointValues() {
+        return new MultiGeoPointValues() {
+            GeoPoint[] points = new GeoPoint[0];
+            private int count = 0;
+
+            @Override
+            public void setDocument(int docId) {
+                values.setDocument(docId);
+                count = values.count();
+                if (count > points.length) {
+                    final int previousLength = points.length;
+                    points = Arrays.copyOf(points, ArrayUtil.oversize(count, RamUsageEstimator.NUM_BYTES_OBJECT_REF));
+                    for (int i = previousLength; i < points.length; ++i) {
+                        points[i] = new GeoPoint(Double.NaN, Double.NaN);
+                    }
+                }
+                long encoded;
+                for (int i=0; i<count; ++i) {
+                    encoded = values.valueAt(i);
+                    points[i].reset(GeoEncodingUtils.decodeLatitude((int)(encoded >>> 32)), GeoEncodingUtils.decodeLongitude((int)encoded));
+                }
+            }
+
+            @Override
+            public int count() {
+                return count;
+            }
+
+            @Override
+            public GeoPoint valueAt(int index) {
+                return points[index];
+            }
+        };
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesAtomicFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesAtomicFieldData.java
index 93a981cab2de5..f94fde6955378 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesAtomicFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesAtomicFieldData.java
@@ -40,6 +40,7 @@ public class PagedBytesAtomicFieldData extends AbstractAtomicOrdinalsFieldData {
     protected final Ordinals ordinals;
 
     public PagedBytesAtomicFieldData(PagedBytes.Reader bytes, PackedLongValues termOrdToBytesOffset, Ordinals ordinals) {
+        super(DEFAULT_SCRIPT_FUNCTION);
         this.bytes = bytes;
         this.termOrdToBytesOffset = termOrdToBytesOffset;
         this.ordinals = ordinals;
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesIndexFieldData.java
index 5695f7ef15a9a..a56b0f1b8c18d 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesIndexFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesIndexFieldData.java
@@ -27,10 +27,12 @@
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.SortField;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.PagedBytes;
 import org.apache.lucene.util.packed.PackedInts;
 import org.apache.lucene.util.packed.PackedLongValues;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData;
@@ -38,11 +40,13 @@
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
 import org.elasticsearch.index.fielddata.RamAccountingTermsEnum;
+import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
 import org.elasticsearch.index.fielddata.ordinals.Ordinals;
 import org.elasticsearch.index.fielddata.ordinals.OrdinalsBuilder;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
+import org.elasticsearch.search.MultiValueMode;
 
 import java.io.IOException;
 
@@ -76,6 +80,12 @@ public PagedBytesIndexFieldData(IndexSettings indexSettings, String fieldName,
         super(indexSettings, fieldName, cache, breakerService, minFrequency, maxFrequency, minSegmentSize);
     }
 
+    @Override
+    public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMode, XFieldComparatorSource.Nested nested, boolean reverse) {
+        XFieldComparatorSource source = new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested);
+        return new SortField(getFieldName(), source, reverse);
+    }
+
     @Override
     public AtomicOrdinalsFieldData loadDirect(LeafReaderContext context) throws Exception {
         LeafReader reader = context.reader();
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java
deleted file mode 100644
index d483baa777e88..0000000000000
--- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java
+++ /dev/null
@@ -1,393 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.fielddata.plain;
-
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.DocValues;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.MultiDocValues;
-import org.apache.lucene.index.MultiDocValues.OrdinalMap;
-import org.apache.lucene.index.SortedDocValues;
-import org.apache.lucene.util.Accountable;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.LongValues;
-import org.apache.lucene.util.packed.PackedInts;
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.breaker.CircuitBreaker;
-import org.elasticsearch.common.lease.Releasable;
-import org.elasticsearch.common.lease.Releasables;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.fielddata.AtomicParentChildFieldData;
-import org.elasticsearch.index.fielddata.IndexFieldData;
-import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
-import org.elasticsearch.index.fielddata.IndexFieldDataCache;
-import org.elasticsearch.index.fielddata.IndexParentChildFieldData;
-import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.ParentFieldMapper;
-import org.elasticsearch.indices.breaker.CircuitBreakerService;
-import org.elasticsearch.search.MultiValueMode;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-
-/**
- * ParentChildIndexFieldData is responsible for loading the id cache mapping
- * needed for has_child and has_parent queries into memory.
- */
-public class ParentChildIndexFieldData extends AbstractIndexFieldData<AtomicParentChildFieldData> implements IndexParentChildFieldData {
-
-    private final Set<String> parentTypes;
-    private final CircuitBreakerService breakerService;
-
-    public ParentChildIndexFieldData(IndexSettings indexSettings, String fieldName,
-                                     IndexFieldDataCache cache, MapperService mapperService,
-                                     CircuitBreakerService breakerService) {
-        super(indexSettings, fieldName, cache);
-        this.breakerService = breakerService;
-        Set<String> parentTypes = new HashSet<>();
-        for (DocumentMapper mapper : mapperService.docMappers(false)) {
-            ParentFieldMapper parentFieldMapper = mapper.parentFieldMapper();
-            if (parentFieldMapper.active()) {
-                parentTypes.add(parentFieldMapper.type());
-            }
-        }
-        this.parentTypes = parentTypes;
-    }
-
-    @Override
-    public XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested) {
-        return new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested);
-    }
-
-    @Override
-    public AtomicParentChildFieldData load(LeafReaderContext context) {
-        final LeafReader reader = context.reader();
-        return new AbstractAtomicParentChildFieldData() {
-
-            public Set<String> types() {
-                return parentTypes;
-            }
-
-            @Override
-            public SortedDocValues getOrdinalsValues(String type) {
-                try {
-                    return DocValues.getSorted(reader, ParentFieldMapper.joinField(type));
-                } catch (IOException e) {
-                    throw new IllegalStateException("cannot load join doc values field for type [" + type + "]", e);
-                }
-            }
-
-            @Override
-            public long ramBytesUsed() {
-                // unknown
-                return 0;
-            }
-
-            @Override
-            public Collection<Accountable> getChildResources() {
-                return Collections.emptyList();
-            }
-
-            @Override
-            public void close() throws ElasticsearchException {
-            }
-        };
-    }
-
-    @Override
-    public AbstractAtomicParentChildFieldData loadDirect(LeafReaderContext context) throws Exception {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    protected AtomicParentChildFieldData empty(int maxDoc) {
-        return AbstractAtomicParentChildFieldData.empty();
-    }
-
-    public static class Builder implements IndexFieldData.Builder {
-
-        @Override
-        public IndexFieldData<?> build(IndexSettings indexSettings,
-                                       MappedFieldType fieldType,
-                                       IndexFieldDataCache cache, CircuitBreakerService breakerService,
-                                       MapperService mapperService) {
-            return new ParentChildIndexFieldData(indexSettings, fieldType.name(), cache,
-                    mapperService, breakerService);
-        }
-    }
-
-    @Override
-    public IndexParentChildFieldData loadGlobal(DirectoryReader indexReader) {
-        if (indexReader.leaves().size() <= 1) {
-            // ordinals are already global
-            return this;
-        }
-        try {
-            return cache.load(indexReader, this);
-        } catch (Exception e) {
-            if (e instanceof ElasticsearchException) {
-                throw (ElasticsearchException) e;
-            } else {
-                throw new ElasticsearchException(e);
-            }
-        }
-    }
-
-    private static OrdinalMap buildOrdinalMap(AtomicParentChildFieldData[] atomicFD, String parentType) throws IOException {
-        final SortedDocValues[] ordinals = new SortedDocValues[atomicFD.length];
-        for (int i = 0; i < ordinals.length; ++i) {
-            ordinals[i] = atomicFD[i].getOrdinalsValues(parentType);
-        }
-        return OrdinalMap.build(null, ordinals, PackedInts.DEFAULT);
-    }
-
-    private static class OrdinalMapAndAtomicFieldData {
-        final OrdinalMap ordMap;
-        final AtomicParentChildFieldData[] fieldData;
-
-        public OrdinalMapAndAtomicFieldData(OrdinalMap ordMap, AtomicParentChildFieldData[] fieldData) {
-            this.ordMap = ordMap;
-            this.fieldData = fieldData;
-        }
-    }
-
-    @Override
-    public IndexParentChildFieldData localGlobalDirect(DirectoryReader indexReader) throws Exception {
-        final long startTime = System.nanoTime();
-
-        long ramBytesUsed = 0;
-        final Map<String, OrdinalMapAndAtomicFieldData> perType = new HashMap<>();
-        for (String type : parentTypes) {
-            final AtomicParentChildFieldData[] fieldData = new AtomicParentChildFieldData[indexReader.leaves().size()];
-            for (LeafReaderContext context : indexReader.leaves()) {
-                fieldData[context.ord] = load(context);
-            }
-            final OrdinalMap ordMap = buildOrdinalMap(fieldData, type);
-            ramBytesUsed += ordMap.ramBytesUsed();
-            perType.put(type, new OrdinalMapAndAtomicFieldData(ordMap, fieldData));
-        }
-
-        final AtomicParentChildFieldData[] fielddata = new AtomicParentChildFieldData[indexReader.leaves().size()];
-        for (int i = 0; i < fielddata.length; ++i) {
-            fielddata[i] = new GlobalAtomicFieldData(parentTypes, perType, i);
-        }
-
-        breakerService.getBreaker(CircuitBreaker.FIELDDATA).addWithoutBreaking(ramBytesUsed);
-        if (logger.isDebugEnabled()) {
-            logger.debug(
-                    "global-ordinals [_parent] took [{}]",
-                    new TimeValue(System.nanoTime() - startTime, TimeUnit.NANOSECONDS)
-            );
-        }
-
-        return new GlobalFieldData(indexReader, fielddata, ramBytesUsed, perType);
-    }
-
-    private static class GlobalAtomicFieldData extends AbstractAtomicParentChildFieldData {
-
-        private final Set<String> types;
-        private final Map<String, OrdinalMapAndAtomicFieldData> atomicFD;
-        private final int segmentIndex;
-
-        public GlobalAtomicFieldData(Set<String> types, Map<String, OrdinalMapAndAtomicFieldData> atomicFD, int segmentIndex) {
-            this.types = types;
-            this.atomicFD = atomicFD;
-            this.segmentIndex = segmentIndex;
-        }
-
-        @Override
-        public Set<String> types() {
-            return types;
-        }
-
-        @Override
-        public SortedDocValues getOrdinalsValues(String type) {
-            final OrdinalMapAndAtomicFieldData atomicFD = this.atomicFD.get(type);
-            if (atomicFD == null) {
-                return DocValues.emptySorted();
-            }
-
-            final OrdinalMap ordMap = atomicFD.ordMap;
-            final SortedDocValues[] allSegmentValues = new SortedDocValues[atomicFD.fieldData.length];
-            for (int i = 0; i < allSegmentValues.length; ++i) {
-                allSegmentValues[i] = atomicFD.fieldData[i].getOrdinalsValues(type);
-            }
-            final SortedDocValues segmentValues = allSegmentValues[segmentIndex];
-            if (segmentValues.getValueCount() == ordMap.getValueCount()) {
-                // ords are already global
-                return segmentValues;
-            }
-            final LongValues globalOrds = ordMap.getGlobalOrds(segmentIndex);
-            return new SortedDocValues() {
-
-                @Override
-                public BytesRef lookupOrd(int ord) {
-                    final int segmentIndex = ordMap.getFirstSegmentNumber(ord);
-                    final int segmentOrd = (int) ordMap.getFirstSegmentOrd(ord);
-                    return allSegmentValues[segmentIndex].lookupOrd(segmentOrd);
-                }
-
-                @Override
-                public int getValueCount() {
-                    return (int) ordMap.getValueCount();
-                }
-
-                @Override
-                public int getOrd(int docID) {
-                    final int segmentOrd = segmentValues.getOrd(docID);
-                    // TODO: is there a way we can get rid of this branch?
-                    if (segmentOrd >= 0) {
-                        return (int) globalOrds.get(segmentOrd);
-                    } else {
-                        return segmentOrd;
-                    }
-                }
-            };
-        }
-
-        @Override
-        public long ramBytesUsed() {
-            // this class does not take memory on its own, the index-level field data does
-            // it through the use of ordinal maps
-            return 0;
-        }
-
-        @Override
-        public Collection<Accountable> getChildResources() {
-            return Collections.emptyList();
-        }
-
-        @Override
-        public void close() {
-            List<Releasable> closeables = new ArrayList<>();
-            for (OrdinalMapAndAtomicFieldData fds : atomicFD.values()) {
-                closeables.addAll(Arrays.asList(fds.fieldData));
-            }
-            Releasables.close(closeables);
-        }
-
-    }
-
-    public class GlobalFieldData implements IndexParentChildFieldData, Accountable {
-
-        private final Object coreCacheKey;
-        private final List<LeafReaderContext> leaves;
-        private final AtomicParentChildFieldData[] fielddata;
-        private final long ramBytesUsed;
-        private final Map<String, OrdinalMapAndAtomicFieldData> ordinalMapPerType;
-
-        GlobalFieldData(IndexReader reader, AtomicParentChildFieldData[] fielddata, long ramBytesUsed, Map<String, OrdinalMapAndAtomicFieldData> ordinalMapPerType) {
-            this.coreCacheKey = reader.getCoreCacheKey();
-            this.leaves = reader.leaves();
-            this.ramBytesUsed = ramBytesUsed;
-            this.fielddata = fielddata;
-            this.ordinalMapPerType = ordinalMapPerType;
-        }
-
-        @Override
-        public String getFieldName() {
-            return ParentChildIndexFieldData.this.getFieldName();
-        }
-
-        @Override
-        public AtomicParentChildFieldData load(LeafReaderContext context) {
-            assert context.reader().getCoreCacheKey() == leaves.get(context.ord).reader().getCoreCacheKey();
-            return fielddata[context.ord];
-        }
-
-        @Override
-        public AtomicParentChildFieldData loadDirect(LeafReaderContext context) throws Exception {
-            return load(context);
-        }
-
-        @Override
-        public XFieldComparatorSource comparatorSource(Object missingValue, MultiValueMode sortMode, Nested nested) {
-            throw new UnsupportedOperationException("No sorting on global ords");
-        }
-
-        @Override
-        public void clear() {
-            ParentChildIndexFieldData.this.clear();
-        }
-
-        @Override
-        public Index index() {
-            return ParentChildIndexFieldData.this.index();
-        }
-
-        @Override
-        public long ramBytesUsed() {
-            return ramBytesUsed;
-        }
-
-        @Override
-        public Collection<Accountable> getChildResources() {
-            return Collections.emptyList();
-        }
-
-        @Override
-        public IndexParentChildFieldData loadGlobal(DirectoryReader indexReader) {
-            if (indexReader.getCoreCacheKey() == coreCacheKey) {
-                return this;
-            }
-            throw new IllegalStateException();
-        }
-
-        @Override
-        public IndexParentChildFieldData localGlobalDirect(DirectoryReader indexReader) throws Exception {
-            return loadGlobal(indexReader);
-        }
-
-    }
-
-    /**
-     * Returns the global ordinal map for the specified type
-     */
-    // TODO: OrdinalMap isn't expose in the field data framework, because it is an implementation detail.
-    // However the JoinUtil works directly with OrdinalMap, so this is a hack to get access to OrdinalMap
-    // I don't think we should expose OrdinalMap in IndexFieldData, because only parent/child relies on it and for the
-    // rest of the code OrdinalMap is an implementation detail, but maybe we can expose it in IndexParentChildFieldData interface?
-    public static MultiDocValues.OrdinalMap getOrdinalMap(IndexParentChildFieldData indexParentChildFieldData, String type) {
-        if (indexParentChildFieldData instanceof ParentChildIndexFieldData.GlobalFieldData) {
-            return ((GlobalFieldData) indexParentChildFieldData).ordinalMapPerType.get(type).ordMap;
-        } else {
-            // one segment, local ordinals are global
-            return null;
-        }
-    }
-
-}
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericDVIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericDVIndexFieldData.java
index be877b9c68af3..9e8e6dba0b6d0 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericDVIndexFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericDVIndexFieldData.java
@@ -26,6 +26,9 @@
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.NumericDocValues;
 import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.SortedNumericSelector;
+import org.apache.lucene.search.SortedNumericSortField;
 import org.apache.lucene.util.Accountable;
 import org.apache.lucene.util.NumericUtils;
 import org.elasticsearch.index.Index;
@@ -60,17 +63,53 @@ public SortedNumericDVIndexFieldData(Index index, String fieldNames, NumericType
     }
 
     @Override
-    public org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource comparatorSource(Object missingValue, MultiValueMode sortMode, Nested nested) {
+    public SortField sortField(Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse) {
+        final XFieldComparatorSource source;
         switch (numericType) {
             case HALF_FLOAT:
             case FLOAT:
-                return new FloatValuesComparatorSource(this, missingValue, sortMode, nested);
+                source = new FloatValuesComparatorSource(this, missingValue, sortMode, nested);
+                break;
+
+            case DOUBLE:
+                source = new DoubleValuesComparatorSource(this, missingValue, sortMode, nested);
+                break;
+
+            default:
+                assert !numericType.isFloatingPoint();
+                source = new LongValuesComparatorSource(this, missingValue, sortMode, nested);
+                break;
+        }
+
+        /**
+         * Check if we can use a simple {@link SortedNumericSortField} compatible with index sorting and
+         * returns a custom sort field otherwise.
+         */
+        if (nested != null
+                || (sortMode != MultiValueMode.MAX && sortMode != MultiValueMode.MIN)
+                || numericType == NumericType.HALF_FLOAT) {
+            return new SortField(fieldName, source, reverse);
+        }
+
+        final SortField sortField;
+        final SortedNumericSelector.Type selectorType = sortMode == MultiValueMode.MAX ?
+            SortedNumericSelector.Type.MAX : SortedNumericSelector.Type.MIN;
+        switch (numericType) {
+            case FLOAT:
+                sortField = new SortedNumericSortField(fieldName, SortField.Type.FLOAT, reverse, selectorType);
+                break;
+
             case DOUBLE:
-                return new DoubleValuesComparatorSource(this, missingValue, sortMode, nested);
+                sortField = new SortedNumericSortField(fieldName, SortField.Type.DOUBLE, reverse, selectorType);
+                break;
+
             default:
                 assert !numericType.isFloatingPoint();
-                return new LongValuesComparatorSource(this, missingValue, sortMode, nested);
+                sortField = new SortedNumericSortField(fieldName, SortField.Type.LONG, reverse, selectorType);
+                break;
         }
+        sortField.setMissingValue(source.missingObject(missingValue, reverse));
+        return sortField;
     }
 
     @Override
@@ -96,7 +135,7 @@ public AtomicNumericFieldData load(LeafReaderContext context) {
             case DOUBLE:
                 return new SortedNumericDoubleFieldData(reader, field);
             default:
-                return new SortedNumericLongFieldData(reader, field);
+                return new SortedNumericLongFieldData(reader, field, numericType == NumericType.BOOLEAN);
         }
     }
 
@@ -117,8 +156,8 @@ static final class SortedNumericLongFieldData extends AtomicLongFieldData {
         final LeafReader reader;
         final String field;
 
-        SortedNumericLongFieldData(LeafReader reader, String field) {
-            super(0L);
+        SortedNumericLongFieldData(LeafReader reader, String field, boolean isBoolean) {
+            super(0L, isBoolean);
             this.reader = reader;
             this.field = field;
         }
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVBytesAtomicFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVBytesAtomicFieldData.java
index 0bcb8251b98c7..114031cd3553b 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVBytesAtomicFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVBytesAtomicFieldData.java
@@ -25,10 +25,12 @@
 import org.apache.lucene.util.Accountable;
 import org.elasticsearch.index.fielddata.AtomicFieldData;
 import org.elasticsearch.index.fielddata.FieldData;
+import org.elasticsearch.index.fielddata.ScriptDocValues;
 
 import java.io.IOException;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.function.Function;
 
 /**
  * An {@link AtomicFieldData} implementation that uses Lucene {@link org.apache.lucene.index.SortedSetDocValues}.
@@ -38,7 +40,9 @@ public final class SortedSetDVBytesAtomicFieldData extends AbstractAtomicOrdinal
     private final LeafReader reader;
     private final String field;
 
-    SortedSetDVBytesAtomicFieldData(LeafReader reader, String field) {
+    SortedSetDVBytesAtomicFieldData(LeafReader reader, String field, Function<RandomAccessOrds,
+            ScriptDocValues<?>> scriptFunction) {
+        super(scriptFunction);
         this.reader = reader;
         this.field = field;
     }
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVOrdinalsIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVOrdinalsIndexFieldData.java
index f0c8aa3d0767f..5e07faae8d878 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVOrdinalsIndexFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVOrdinalsIndexFieldData.java
@@ -21,41 +21,65 @@
 
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.RandomAccessOrds;
+import org.apache.lucene.index.MultiDocValues;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.SortedSetSortField;
+import org.apache.lucene.search.SortedSetSelector;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData;
-import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
+import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
 import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.search.MultiValueMode;
 
 import java.io.IOException;
+import java.util.function.Function;
 
 public class SortedSetDVOrdinalsIndexFieldData extends DocValuesIndexFieldData implements IndexOrdinalsFieldData {
 
     private final IndexSettings indexSettings;
     private final IndexFieldDataCache cache;
     private final CircuitBreakerService breakerService;
+    private final Function<RandomAccessOrds, ScriptDocValues<?>> scriptFunction;
 
-    public SortedSetDVOrdinalsIndexFieldData(IndexSettings indexSettings, IndexFieldDataCache cache, String fieldName, CircuitBreakerService breakerService) {
+    public SortedSetDVOrdinalsIndexFieldData(IndexSettings indexSettings, IndexFieldDataCache cache, String fieldName,
+            CircuitBreakerService breakerService, Function<RandomAccessOrds, ScriptDocValues<?>> scriptFunction) {
         super(indexSettings.getIndex(), fieldName);
         this.indexSettings = indexSettings;
         this.cache = cache;
         this.breakerService = breakerService;
+        this.scriptFunction = scriptFunction;
     }
 
     @Override
-    public org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource comparatorSource(Object missingValue, MultiValueMode sortMode, Nested nested) {
-        return new BytesRefFieldComparatorSource((IndexFieldData<?>) this, missingValue, sortMode, nested);
+    public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse) {
+        XFieldComparatorSource source = new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested);
+        /**
+         * Check if we can use a simple {@link SortedSetSortField} compatible with index sorting and
+         * returns a custom sort field otherwise.
+         */
+        if (nested != null ||
+                (sortMode != MultiValueMode.MAX && sortMode != MultiValueMode.MIN) ||
+                (source.sortMissingLast(missingValue) == false && source.sortMissingFirst(missingValue) == false)) {
+            return new SortField(getFieldName(), source, reverse);
+        }
+        SortField sortField = new SortedSetSortField(fieldName, reverse,
+            sortMode == MultiValueMode.MAX ? SortedSetSelector.Type.MAX : SortedSetSelector.Type.MIN);
+        sortField.setMissingValue(source.sortMissingLast(missingValue) ^ reverse ?
+            SortedSetSortField.STRING_LAST : SortedSetSortField.STRING_FIRST);
+        return sortField;
     }
 
     @Override
     public AtomicOrdinalsFieldData load(LeafReaderContext context) {
-        return new SortedSetDVBytesAtomicFieldData(context.reader(), fieldName);
+        return new SortedSetDVBytesAtomicFieldData(context.reader(), fieldName, scriptFunction);
     }
 
     @Override
@@ -100,6 +124,11 @@ public IndexOrdinalsFieldData loadGlobal(DirectoryReader indexReader) {
 
     @Override
     public IndexOrdinalsFieldData localGlobalDirect(DirectoryReader indexReader) throws Exception {
-        return GlobalOrdinalsBuilder.build(indexReader, this, indexSettings, breakerService, logger);
+        return GlobalOrdinalsBuilder.build(indexReader, this, indexSettings, breakerService, logger, scriptFunction);
+    }
+
+    @Override
+    public MultiDocValues.OrdinalMap getOrdinalMap() {
+        return null;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java b/core/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java
index ee9634f690c4a..415080de13498 100644
--- a/core/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java
+++ b/core/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java
@@ -23,6 +23,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ParentFieldMapper;
@@ -36,6 +37,7 @@
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -54,13 +56,14 @@ public class FieldsVisitor extends StoredFieldVisitor {
             UidFieldMapper.NAME,
             TimestampFieldMapper.NAME,
             TTLFieldMapper.NAME,
+            IdFieldMapper.NAME,
             RoutingFieldMapper.NAME,
             ParentFieldMapper.NAME));
 
     private final boolean loadSource;
     private final Set<String> requiredFields;
     protected BytesReference source;
-    protected Uid uid;
+    protected String type, id;
     protected Map<String, List<Object>> fieldsValues;
 
     public FieldsVisitor(boolean loadSource) {
@@ -82,6 +85,13 @@ public Status needsField(FieldInfo fieldInfo) throws IOException {
     }
 
     public void postProcess(MapperService mapperService) {
+        if (mapperService.getIndexSettings().isSingleType()) {
+            final Collection<String> types = mapperService.types();
+            assert types.size() <= 1 : types;
+            if (types.isEmpty() == false) {
+                type = types.iterator().next();
+            }
+        }
         for (Map.Entry<String, List<Object>> entry : fields().entrySet()) {
             MappedFieldType fieldType = mapperService.fullName(entry.getKey());
             if (fieldType == null) {
@@ -90,7 +100,7 @@ public void postProcess(MapperService mapperService) {
             }
             List<Object> fieldValues = entry.getValue();
             for (int i = 0; i < fieldValues.size(); i++) {
-                fieldValues.set(i, fieldType.valueForSearch(fieldValues.get(i)));
+                fieldValues.set(i, fieldType.valueForDisplay(fieldValues.get(i)));
             }
         }
     }
@@ -108,7 +118,11 @@ public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException {
     public void stringField(FieldInfo fieldInfo, byte[] bytes) throws IOException {
         final String value = new String(bytes, StandardCharsets.UTF_8);
         if (UidFieldMapper.NAME.equals(fieldInfo.name)) {
-            uid = Uid.createUid(value);
+            Uid uid = Uid.createUid(value);
+            type = uid.type();
+            id = uid.id();
+        } else if (IdFieldMapper.NAME.equals(fieldInfo.name)) {
+            id = value;
         } else {
             addValue(fieldInfo.name, value);
         }
@@ -139,7 +153,12 @@ public BytesReference source() {
     }
 
     public Uid uid() {
-        return uid;
+        if (id == null) {
+            return null;
+        } else if (type == null) {
+            throw new IllegalStateException("Call postProcess before getting the uid");
+        }
+        return new Uid(type, id);
     }
 
     public String routing() {
@@ -161,7 +180,8 @@ public Map<String, List<Object>> fields() {
     public void reset() {
         if (fieldsValues != null) fieldsValues.clear();
         source = null;
-        uid = null;
+        type = null;
+        id = null;
 
         requiredFields.addAll(BASE_REQUIRED_FIELDS);
         if (loadSource) {
diff --git a/core/src/main/java/org/elasticsearch/index/fieldvisitor/JustUidFieldsVisitor.java b/core/src/main/java/org/elasticsearch/index/fieldvisitor/JustUidFieldsVisitor.java
deleted file mode 100644
index 2a6c362274a4f..0000000000000
--- a/core/src/main/java/org/elasticsearch/index/fieldvisitor/JustUidFieldsVisitor.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.index.fieldvisitor;
-
-import org.apache.lucene.index.FieldInfo;
-import org.elasticsearch.index.mapper.UidFieldMapper;
-
-import java.io.IOException;
-
-/**
- */
-public class JustUidFieldsVisitor extends FieldsVisitor {
-
-    public JustUidFieldsVisitor() {
-        super(false);
-    }
-
-    @Override
-    public Status needsField(FieldInfo fieldInfo) throws IOException {
-        if (UidFieldMapper.NAME.equals(fieldInfo.name)) {
-            return Status.YES;
-        }
-        return uid != null ? Status.STOP : Status.NO;
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/index/fieldvisitor/SingleFieldsVisitor.java b/core/src/main/java/org/elasticsearch/index/fieldvisitor/SingleFieldsVisitor.java
index 2503286f710d9..0a72a573deb14 100644
--- a/core/src/main/java/org/elasticsearch/index/fieldvisitor/SingleFieldsVisitor.java
+++ b/core/src/main/java/org/elasticsearch/index/fieldvisitor/SingleFieldsVisitor.java
@@ -20,12 +20,12 @@
 
 import org.apache.lucene.index.FieldInfo;
 import org.elasticsearch.index.mapper.IdFieldMapper;
-import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.TypeFieldMapper;
+import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.index.mapper.UidFieldMapper;
 
 import java.io.IOException;
-import java.util.List;
 
 /**
  */
@@ -56,30 +56,17 @@ public void reset(String field) {
         super.reset();
     }
 
-    public void postProcess(MappedFieldType fieldType) {
-        if (uid != null) {
-            switch (field) {
-            case UidFieldMapper.NAME:
-                addValue(field, uid.toString());
-                break;
-            case IdFieldMapper.NAME:
-                addValue(field, uid.id());
-                break;
-            case TypeFieldMapper.NAME:
-                addValue(field, uid.type());
-                break;
-            }
-        }
-
-        if (fieldsValues == null) {
-            return;
+    @Override
+    public void postProcess(MapperService mapperService) {
+        super.postProcess(mapperService);
+        if (id != null) {
+            addValue(IdFieldMapper.NAME, id);
         }
-        List<Object> fieldValues = fieldsValues.get(fieldType.name());
-        if (fieldValues == null) {
-            return;
+        if (type != null) {
+            addValue(TypeFieldMapper.NAME, type);
         }
-        for (int i = 0; i < fieldValues.size(); i++) {
-            fieldValues.set(i, fieldType.valueForSearch(fieldValues.get(i)));
+        if (type != null && id != null) {
+            addValue(UidFieldMapper.NAME, Uid.createUid(type, id));
         }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/flush/FlushStats.java b/core/src/main/java/org/elasticsearch/index/flush/FlushStats.java
index 600651ad30662..ac9a4a5c9a10e 100644
--- a/core/src/main/java/org/elasticsearch/index/flush/FlushStats.java
+++ b/core/src/main/java/org/elasticsearch/index/flush/FlushStats.java
@@ -81,12 +81,6 @@ public TimeValue getTotalTime() {
         return new TimeValue(totalTimeInMillis);
     }
 
-    public static FlushStats readFlushStats(StreamInput in) throws IOException {
-        FlushStats flushStats = new FlushStats();
-        flushStats.readFrom(in);
-        return flushStats;
-    }
-
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject(Fields.FLUSH);
diff --git a/core/src/main/java/org/elasticsearch/index/get/GetField.java b/core/src/main/java/org/elasticsearch/index/get/GetField.java
index 0ebbf1f9ac178..9b6808517aa5d 100644
--- a/core/src/main/java/org/elasticsearch/index/get/GetField.java
+++ b/core/src/main/java/org/elasticsearch/index/get/GetField.java
@@ -22,17 +22,22 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.mapper.MapperService;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Objects;
+
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.parseStoredFieldsValue;
 
-/**
- *
- */
-public class GetField implements Streamable, Iterable<Object> {
+
+public class GetField implements Streamable, ToXContent, Iterable<Object> {
 
     private String name;
     private List<Object> values;
@@ -41,8 +46,8 @@ private GetField() {
     }
 
     public GetField(String name, List<Object> values) {
-        this.name = name;
-        this.values = values;
+        this.name = Objects.requireNonNull(name, "name must not be null");
+        this.values = Objects.requireNonNull(values, "values must not be null");
     }
 
     public String getName() {
@@ -93,4 +98,55 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeGenericValue(obj);
         }
     }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startArray(name);
+        for (Object value : values) {
+            //this call doesn't really need to support writing any kind of object.
+            //Stored fields values are converted using MappedFieldType#valueForDisplay.
+            //As a result they can either be Strings, Numbers, Booleans, or BytesReference, that's all.
+            builder.value(value);
+        }
+        builder.endArray();
+        return builder;
+    }
+
+    public static GetField fromXContent(XContentParser parser) throws IOException {
+        ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser::getTokenLocation);
+        String fieldName = parser.currentName();
+        XContentParser.Token token = parser.nextToken();
+        ensureExpectedToken(XContentParser.Token.START_ARRAY, token, parser::getTokenLocation);
+        List<Object> values = new ArrayList<>();
+        while((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+            values.add(parseStoredFieldsValue(parser));
+        }
+        return new GetField(fieldName, values);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        GetField objects = (GetField) o;
+        return Objects.equals(name, objects.name) &&
+                Objects.equals(values, objects.values);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(name, values);
+    }
+
+    @Override
+    public String toString() {
+        return "GetField{" +
+                "name='" + name + '\'' +
+                ", values=" + values +
+                '}';
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/get/GetResult.java b/core/src/main/java/org/elasticsearch/index/get/GetResult.java
index 0fa843adc47d4..6569902ceffc8 100644
--- a/core/src/main/java/org/elasticsearch/index/get/GetResult.java
+++ b/core/src/main/java/org/elasticsearch/index/get/GetResult.java
@@ -20,14 +20,17 @@
 package org.elasticsearch.index.get;
 
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressorFactory;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.search.lookup.SourceLookup;
 
 import java.io.IOException;
@@ -37,13 +40,20 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 
 import static java.util.Collections.emptyMap;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
 import static org.elasticsearch.index.get.GetField.readGetField;
 
-/**
- */
-public class GetResult implements Streamable, Iterable<GetField>, ToXContent {
+public class GetResult implements Streamable, Iterable<GetField>, ToXContentObject {
+
+    public static final String _INDEX = "_index";
+    public static final String _TYPE = "_type";
+    public static final String _ID = "_id";
+    private static final String _VERSION = "_version";
+    private static final String FOUND = "found";
+    private static final String FIELDS = "fields";
 
     private String index;
     private String type;
@@ -58,7 +68,8 @@ public class GetResult implements Streamable, Iterable<GetField>, ToXContent {
     GetResult() {
     }
 
-    public GetResult(String index, String type, String id, long version, boolean exists, BytesReference source, Map<String, GetField> fields) {
+    public GetResult(String index, String type, String id, long version, boolean exists, BytesReference source,
+                     Map<String, GetField> fields) {
         this.index = index;
         this.type = type;
         this.id = id;
@@ -124,6 +135,10 @@ public byte[] source() {
      * Returns bytes reference, also un compress the source if needed.
      */
     public BytesReference sourceRef() {
+        if (source == null) {
+            return null;
+        }
+
         try {
             this.source = CompressorFactory.uncompressIfNeeded(this.source);
             return this.source;
@@ -197,15 +212,6 @@ public Iterator<GetField> iterator() {
         return fields.values().iterator();
     }
 
-    static final class Fields {
-        static final String _INDEX = "_index";
-        static final String _TYPE = "_type";
-        static final String _ID = "_id";
-        static final String _VERSION = "_version";
-        static final String FOUND = "found";
-        static final String FIELDS = "fields";
-    }
-
     public XContentBuilder toXContentEmbedded(XContentBuilder builder, Params params) throws IOException {
         List<GetField> metaFields = new ArrayList<>();
         List<GetField> otherFields = new ArrayList<>();
@@ -226,20 +232,16 @@ public XContentBuilder toXContentEmbedded(XContentBuilder builder, Params params
             builder.field(field.getName(), field.getValue());
         }
 
-        builder.field(Fields.FOUND, exists);
+        builder.field(FOUND, exists);
 
         if (source != null) {
-            XContentHelper.writeRawField("_source", source, builder, params);
+            XContentHelper.writeRawField(SourceFieldMapper.NAME, source, builder, params);
         }
 
         if (!otherFields.isEmpty()) {
-            builder.startObject(Fields.FIELDS);
+            builder.startObject(FIELDS);
             for (GetField field : otherFields) {
-                builder.startArray(field.getName());
-                for (Object value : field.getValues()) {
-                    builder.value(value);
-                }
-                builder.endArray();
+                field.toXContent(builder, params);
             }
             builder.endObject();
         }
@@ -248,23 +250,79 @@ public XContentBuilder toXContentEmbedded(XContentBuilder builder, Params params
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        if (!isExists()) {
-            builder.field(Fields._INDEX, index);
-            builder.field(Fields._TYPE, type);
-            builder.field(Fields._ID, id);
-            builder.field(Fields.FOUND, false);
-        } else {
-            builder.field(Fields._INDEX, index);
-            builder.field(Fields._TYPE, type);
-            builder.field(Fields._ID, id);
+        builder.startObject();
+        builder.field(_INDEX, index);
+        builder.field(_TYPE, type);
+        builder.field(_ID, id);
+        if (isExists()) {
             if (version != -1) {
-                builder.field(Fields._VERSION, version);
+                builder.field(_VERSION, version);
             }
             toXContentEmbedded(builder, params);
+        } else {
+            builder.field(FOUND, false);
         }
+        builder.endObject();
         return builder;
     }
 
+    public static GetResult fromXContentEmbedded(XContentParser parser) throws IOException {
+        XContentParser.Token token = parser.nextToken();
+        ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser::getTokenLocation);
+
+        String currentFieldName = parser.currentName();
+        String index = null, type = null, id = null;
+        long version = -1;
+        Boolean found = null;
+        BytesReference source = null;
+        Map<String, GetField> fields = new HashMap<>();
+        while((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token.isValue()) {
+                if (_INDEX.equals(currentFieldName)) {
+                    index = parser.text();
+                } else if (_TYPE.equals(currentFieldName)) {
+                    type = parser.text();
+                } else if (_ID.equals(currentFieldName)) {
+                    id = parser.text();
+                }  else if (_VERSION.equals(currentFieldName)) {
+                    version = parser.longValue();
+                } else if (FOUND.equals(currentFieldName)) {
+                    found = parser.booleanValue();
+                } else {
+                    fields.put(currentFieldName, new GetField(currentFieldName, Collections.singletonList(parser.objectText())));
+                }
+            } else if (token == XContentParser.Token.START_OBJECT) {
+                if (SourceFieldMapper.NAME.equals(currentFieldName)) {
+                    try (XContentBuilder builder = XContentBuilder.builder(parser.contentType().xContent())) {
+                        //the original document gets slightly modified: whitespaces or pretty printing are not preserved,
+                        //it all depends on the current builder settings
+                        builder.copyCurrentStructure(parser);
+                        source = builder.bytes();
+                    }
+                } else if (FIELDS.equals(currentFieldName)) {
+                    while(parser.nextToken() != XContentParser.Token.END_OBJECT) {
+                        GetField getField = GetField.fromXContent(parser);
+                        fields.put(getField.getName(), getField);
+                    }
+                } else {
+                    parser.skipChildren(); // skip potential inner objects for forward compatibility
+                }
+            } else if (token == XContentParser.Token.START_ARRAY) {
+                parser.skipChildren(); // skip potential inner arrays for forward compatibility
+            }
+        }
+        return new GetResult(index, type, id, version, found, source, fields);
+    }
+
+    public static GetResult fromXContent(XContentParser parser) throws IOException {
+        XContentParser.Token token = parser.nextToken();
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser::getTokenLocation);
+
+        return fromXContentEmbedded(parser);
+    }
+
     public static GetResult readGetResult(StreamInput in) throws IOException {
         GetResult result = new GetResult();
         result.readFrom(in);
@@ -315,5 +373,28 @@ public void writeTo(StreamOutput out) throws IOException {
             }
         }
     }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        GetResult getResult = (GetResult) o;
+        return version == getResult.version &&
+                exists == getResult.exists &&
+                Objects.equals(index, getResult.index) &&
+                Objects.equals(type, getResult.type) &&
+                Objects.equals(id, getResult.id) &&
+                Objects.equals(fields, getResult.fields) &&
+                Objects.equals(sourceAsMap(), getResult.sourceAsMap());
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(index, type, id, version, exists, fields, sourceAsMap());
+    }
 }
 
diff --git a/core/src/main/java/org/elasticsearch/index/get/GetStats.java b/core/src/main/java/org/elasticsearch/index/get/GetStats.java
index 10b4f64c19ead..ad5c534264e8d 100644
--- a/core/src/main/java/org/elasticsearch/index/get/GetStats.java
+++ b/core/src/main/java/org/elasticsearch/index/get/GetStats.java
@@ -136,12 +136,6 @@ static final class Fields {
         static final String CURRENT = "current";
     }
 
-    public static GetStats readGetStats(StreamInput in) throws IOException {
-        GetStats stats = new GetStats();
-        stats.readFrom(in);
-        return stats;
-    }
-
     @Override
     public void readFrom(StreamInput in) throws IOException {
         existsCount = in.readVLong();
diff --git a/core/src/main/java/org/elasticsearch/index/get/ShardGetService.java b/core/src/main/java/org/elasticsearch/index/get/ShardGetService.java
index fb3fb5aa56d98..9324bf29d3915 100644
--- a/core/src/main/java/org/elasticsearch/index/get/ShardGetService.java
+++ b/core/src/main/java/org/elasticsearch/index/get/ShardGetService.java
@@ -24,7 +24,7 @@
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.lucene.uid.Versions;
+import org.elasticsearch.common.lucene.uid.VersionsResolver.DocIdAndVersion;
 import org.elasticsearch.common.metrics.CounterMetric;
 import org.elasticsearch.common.metrics.MeanMetric;
 import org.elasticsearch.common.util.set.Sets;
@@ -42,14 +42,13 @@
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ParentFieldMapper;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
-import org.elasticsearch.index.mapper.Uid;
-import org.elasticsearch.index.mapper.UidFieldMapper;
 import org.elasticsearch.index.shard.AbstractIndexShardComponent;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 import org.elasticsearch.search.fetch.subphase.ParentFieldSubFetchPhase;
 
 import java.io.IOException;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -76,11 +75,11 @@ public GetStats stats() {
         return new GetStats(existsMetric.count(), TimeUnit.NANOSECONDS.toMillis(existsMetric.sum()), missingMetric.count(), TimeUnit.NANOSECONDS.toMillis(missingMetric.sum()), currentMetric.count());
     }
 
-    public GetResult get(String type, String id, String[] gFields, boolean realtime, long version, VersionType versionType, FetchSourceContext fetchSourceContext, boolean ignoreErrorsOnGeneratedFields) {
+    public GetResult get(String type, String id, String[] gFields, boolean realtime, long version, VersionType versionType, FetchSourceContext fetchSourceContext) {
         currentMetric.inc();
         try {
             long now = System.nanoTime();
-            GetResult getResult = innerGet(type, id, gFields, realtime, version, versionType, fetchSourceContext, ignoreErrorsOnGeneratedFields);
+            GetResult getResult = innerGet(type, id, gFields, realtime, version, versionType, fetchSourceContext);
 
             if (getResult.isExists()) {
                 existsMetric.inc(System.nanoTime() - now);
@@ -139,13 +138,20 @@ private FetchSourceContext normalizeFetchSourceContent(@Nullable FetchSourceCont
         return FetchSourceContext.DO_NOT_FETCH_SOURCE;
     }
 
-    private GetResult innerGet(String type, String id, String[] gFields, boolean realtime, long version, VersionType versionType, FetchSourceContext fetchSourceContext, boolean ignoreErrorsOnGeneratedFields) {
+    private GetResult innerGet(String type, String id, String[] gFields, boolean realtime, long version, VersionType versionType, FetchSourceContext fetchSourceContext) {
         fetchSourceContext = normalizeFetchSourceContent(fetchSourceContext, gFields);
+        final Collection<String> types;
+        if (type == null || type.equals("_all")) {
+            types = mapperService.types();
+        } else {
+            types = Collections.singleton(type);
+        }
 
         Engine.GetResult get = null;
-        if (type == null || type.equals("_all")) {
-            for (String typeX : mapperService.types()) {
-                get = indexShard.get(new Engine.Get(realtime, new Term(UidFieldMapper.NAME, Uid.createUidAsBytes(typeX, id)))
+        for (String typeX : types) {
+            Term uidTerm = mapperService.createUidTerm(typeX, id);
+            if (uidTerm != null) {
+                get = indexShard.get(new Engine.Get(realtime, typeX, id, uidTerm)
                         .version(version).versionType(versionType));
                 if (get.exists()) {
                     type = typeX;
@@ -154,20 +160,10 @@ private GetResult innerGet(String type, String id, String[] gFields, boolean rea
                     get.release();
                 }
             }
-            if (get == null) {
-                return new GetResult(shardId.getIndexName(), type, id, -1, false, null, null);
-            }
-            if (!get.exists()) {
-                // no need to release here as well..., we release in the for loop for non exists
-                return new GetResult(shardId.getIndexName(), type, id, -1, false, null, null);
-            }
-        } else {
-            get = indexShard.get(new Engine.Get(realtime, new Term(UidFieldMapper.NAME, Uid.createUidAsBytes(type, id)))
-                    .version(version).versionType(versionType));
-            if (!get.exists()) {
-                get.release();
-                return new GetResult(shardId.getIndexName(), type, id, -1, false, null, null);
-            }
+        }
+
+        if (get == null || get.exists() == false) {
+            return new GetResult(shardId.getIndexName(), type, id, -1, false, null, null);
         }
 
         try {
@@ -181,7 +177,7 @@ private GetResult innerGet(String type, String id, String[] gFields, boolean rea
     private GetResult innerGetLoadFromStoredFields(String type, String id, String[] gFields, FetchSourceContext fetchSourceContext, Engine.GetResult get, MapperService mapperService) {
         Map<String, GetField> fields = null;
         BytesReference source = null;
-        Versions.DocIdAndVersion docIdAndVersion = get.docIdAndVersion();
+        DocIdAndVersion docIdAndVersion = get.docIdAndVersion();
         FieldsVisitor fieldVisitor = buildFieldsVisitors(gFields, fetchSourceContext);
         if (fieldVisitor != null) {
             try {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/AllFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/AllFieldMapper.java
index c418dd5e6e20f..2967ddce8fd8a 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/AllFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/AllFieldMapper.java
@@ -19,8 +19,8 @@
 
 package org.elasticsearch.index.mapper;
 
-import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
@@ -34,11 +34,11 @@
 import org.elasticsearch.index.similarity.SimilarityService;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
-import static org.elasticsearch.common.xcontent.support.XContentMapValues.lenientNodeBooleanValue;
 import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeMapValue;
 import static org.elasticsearch.index.mapper.TypeParsers.parseTextField;
 
@@ -103,18 +103,18 @@ public AllFieldMapper build(BuilderContext context) {
 
     public static class TypeParser implements MetadataFieldMapper.TypeParser {
         @Override
-        public MetadataFieldMapper.Builder parse(String name, Map<String, Object> node,
+        public MetadataFieldMapper.Builder<?,?> parse(String name, Map<String, Object> node,
                                                  ParserContext parserContext) throws MapperParsingException {
             Builder builder = new Builder(parserContext.mapperService().fullName(NAME));
-            builder.fieldType().setIndexAnalyzer(parserContext.analysisService().defaultIndexAnalyzer());
-            builder.fieldType().setSearchAnalyzer(parserContext.analysisService().defaultSearchAnalyzer());
-            builder.fieldType().setSearchQuoteAnalyzer(parserContext.analysisService().defaultSearchQuoteAnalyzer());
+            builder.fieldType().setIndexAnalyzer(parserContext.getIndexAnalyzers().getDefaultIndexAnalyzer());
+            builder.fieldType().setSearchAnalyzer(parserContext.getIndexAnalyzers().getDefaultSearchAnalyzer());
+            builder.fieldType().setSearchQuoteAnalyzer(parserContext.getIndexAnalyzers().getDefaultSearchQuoteAnalyzer());
 
             // parseField below will happily parse the doc_values setting, but it is then never passed to
             // the AllFieldMapper ctor in the builder since it is not valid. Here we validate
             // the doc values settings (old and new) are rejected
             Object docValues = node.get("doc_values");
-            if (docValues != null && lenientNodeBooleanValue(docValues)) {
+            if (docValues != null && TypeParsers.nodeBooleanValue(name, "doc_values", docValues)) {
                 throw new MapperParsingException("Field [" + name +
                     "] is always tokenized and cannot have doc values");
             }
@@ -135,8 +135,8 @@ public MetadataFieldMapper.Builder parse(String name, Map<String, Object> node,
                 String fieldName = entry.getKey();
                 Object fieldNode = entry.getValue();
                 if (fieldName.equals("enabled")) {
-                    builder.enabled(lenientNodeBooleanValue(fieldNode) ? EnabledAttributeMapper.ENABLED :
-                        EnabledAttributeMapper.DISABLED);
+                    boolean enabled = TypeParsers.nodeBooleanValue(name, "enabled", fieldNode);
+                    builder.enabled(enabled ? EnabledAttributeMapper.ENABLED : EnabledAttributeMapper.DISABLED);
                     iterator.remove();
                 }
             }
@@ -144,14 +144,20 @@ public MetadataFieldMapper.Builder parse(String name, Map<String, Object> node,
         }
 
         @Override
-        public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName) {
-            return new AllFieldMapper(indexSettings, fieldType);
+        public MetadataFieldMapper getDefault(MappedFieldType fieldType, ParserContext context) {
+            final Settings indexSettings = context.mapperService().getIndexSettings().getSettings();
+            if (fieldType != null) {
+                return new AllFieldMapper(indexSettings, fieldType);
+            } else {
+                return parse(NAME, Collections.emptyMap(), context)
+                        .build(new BuilderContext(indexSettings, new ContentPath(1)));
+            }
         }
     }
 
     static final class AllFieldType extends StringFieldType {
 
-        public AllFieldType() {
+        AllFieldType() {
         }
 
         protected AllFieldType(AllFieldType ref) {
@@ -182,7 +188,7 @@ public Query termQuery(Object value, QueryShardContext context) {
     private EnabledAttributeMapper enabledState;
 
     private AllFieldMapper(Settings indexSettings, MappedFieldType existing) {
-        this(existing == null ? Defaults.FIELD_TYPE.clone() : existing.clone(), Defaults.ENABLED, indexSettings);
+        this(existing.clone(), Defaults.ENABLED, indexSettings);
     }
 
     private AllFieldMapper(MappedFieldType fieldType, EnabledAttributeMapper enabled, Settings indexSettings) {
@@ -211,7 +217,7 @@ public Mapper parse(ParseContext context) throws IOException {
     }
 
     @Override
-    protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
+    protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
         if (!enabledState.enabled) {
             return;
         }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/BaseGeoPointFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/BaseGeoPointFieldMapper.java
index 2a24b6a94c6dc..738db9ccc381b 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/BaseGeoPointFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/BaseGeoPointFieldMapper.java
@@ -19,16 +19,22 @@
 
 package org.elasticsearch.index.mapper;
 
-import org.apache.lucene.document.Field;
+import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.Terms;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.common.geo.GeoHashUtils;
+import org.apache.lucene.spatial.util.MortonEncoder;
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.LegacyNumericUtils;
-import org.elasticsearch.Version;
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.fieldstats.FieldStats;
 import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.collect.Iterators;
+import org.elasticsearch.common.geo.GeoHashUtils;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.geo.GeoUtils;
 import org.elasticsearch.common.logging.DeprecationLogger;
@@ -89,7 +95,7 @@ public abstract static class Builder<T extends Builder, Y extends BaseGeoPointFi
 
         protected Boolean ignoreMalformed;
 
-        public Builder(String name, GeoPointFieldType fieldType) {
+        public Builder(String name, MappedFieldType fieldType) {
             super(name, fieldType, fieldType);
         }
 
@@ -143,7 +149,16 @@ public abstract Y build(BuilderContext context, String simpleName, MappedFieldTy
                                 FieldMapper geoHashMapper, MultiFields multiFields, Explicit<Boolean> ignoreMalformed, CopyTo copyTo);
 
         public Y build(Mapper.BuilderContext context) {
-            GeoPointFieldType geoPointFieldType = (GeoPointFieldType)fieldType;
+            // version 5.0 cuts over to LatLonPoint and no longer indexes geohash, or lat/lon separately
+            if (context.indexCreatedVersion().before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+                return buildLegacy(context);
+            }
+            return build(context, name, fieldType, defaultFieldType, context.indexSettings(),
+                null, null, null, multiFieldsBuilder.build(this, context), ignoreMalformed(context), copyTo);
+        }
+
+        private Y buildLegacy(Mapper.BuilderContext context) {
+            LegacyGeoPointFieldType geoPointFieldType = (LegacyGeoPointFieldType)fieldType;
 
             FieldMapper latMapper = null;
             FieldMapper lonMapper = null;
@@ -161,9 +176,9 @@ public Y build(Mapper.BuilderContext context) {
                     lonMapper = (LegacyDoubleFieldMapper) lonMapperBuilder.includeInAll(false).store(fieldType.stored()).docValues(false).build(context);
                 } else {
                     latMapper = new NumberFieldMapper.Builder(Names.LAT, NumberFieldMapper.NumberType.DOUBLE)
-                            .includeInAll(false).store(fieldType.stored()).docValues(false).build(context);
+                        .includeInAll(false).store(fieldType.stored()).docValues(false).build(context);
                     lonMapper = new NumberFieldMapper.Builder(Names.LON, NumberFieldMapper.NumberType.DOUBLE)
-                            .includeInAll(false).store(fieldType.stored()).docValues(false).build(context);
+                        .includeInAll(false).store(fieldType.stored()).docValues(false).build(context);
                 }
                 geoPointFieldType.setLatLonEnabled(latMapper.fieldType(), lonMapper.fieldType());
             }
@@ -183,7 +198,7 @@ public Y build(Mapper.BuilderContext context) {
             context.path().remove();
 
             return build(context, name, fieldType, defaultFieldType, context.indexSettings(),
-                    latMapper, lonMapper, geoHashMapper, multiFieldsBuilder.build(this, context), ignoreMalformed(context), copyTo);
+                latMapper, lonMapper, geoHashMapper, multiFieldsBuilder.build(this, context), ignoreMalformed(context), copyTo);
         }
     }
 
@@ -191,8 +206,11 @@ public abstract static class TypeParser implements Mapper.TypeParser {
         @Override
         public Mapper.Builder<?, ?> parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
             Builder builder;
-            if (parserContext.indexVersionCreated().before(Version.V_2_2_0)) {
+            Version indexVersionCreated = parserContext.indexVersionCreated();
+            if (indexVersionCreated.before(Version.V_2_2_0)) {
                 builder = new LegacyGeoPointFieldMapper.Builder(name);
+            } else if (indexVersionCreated.onOrAfter(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+                builder = new LatLonPointFieldMapper.Builder(name);
             } else {
                 builder = new GeoPointFieldMapper.Builder(name);
             }
@@ -202,82 +220,119 @@ public abstract static class TypeParser implements Mapper.TypeParser {
                 Map.Entry<String, Object> entry = iterator.next();
                 String propName = entry.getKey();
                 Object propNode = entry.getValue();
-                if (propName.equals("lat_lon")) {
-                    deprecationLogger.deprecated(CONTENT_TYPE + " lat_lon parameter is deprecated and will be removed "
-                        + "in the next major release");
-                    builder.enableLatLon(XContentMapValues.lenientNodeBooleanValue(propNode));
-                    iterator.remove();
-                } else if (propName.equals("precision_step")) {
-                    deprecationLogger.deprecated(CONTENT_TYPE + " precision_step parameter is deprecated and will be removed "
-                        + "in the next major release");
-                    builder.precisionStep(XContentMapValues.nodeIntegerValue(propNode));
-                    iterator.remove();
-                } else if (propName.equals("geohash")) {
-                    deprecationLogger.deprecated(CONTENT_TYPE + " geohash parameter is deprecated and will be removed "
-                        + "in the next major release");
-                    builder.enableGeoHash(XContentMapValues.lenientNodeBooleanValue(propNode));
-                    iterator.remove();
-                } else if (propName.equals("geohash_prefix")) {
-                    deprecationLogger.deprecated(CONTENT_TYPE + " geohash_prefix parameter is deprecated and will be removed "
-                        + "in the next major release");
-                    builder.geoHashPrefix(XContentMapValues.lenientNodeBooleanValue(propNode));
-                    if (XContentMapValues.lenientNodeBooleanValue(propNode)) {
-                        builder.enableGeoHash(true);
-                    }
-                    iterator.remove();
-                } else if (propName.equals("geohash_precision")) {
-                    deprecationLogger.deprecated(CONTENT_TYPE + " geohash_precision parameter is deprecated and will be removed "
-                        + "in the next major release");
-                    if (propNode instanceof Integer) {
-                        builder.geoHashPrecision(XContentMapValues.nodeIntegerValue(propNode));
-                    } else {
-                        builder.geoHashPrecision(GeoUtils.geoHashLevelsForPrecision(propNode.toString()));
+                if (indexVersionCreated.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+                    if (propName.equals("lat_lon")) {
+                        deprecationLogger.deprecated(CONTENT_TYPE + " lat_lon parameter is deprecated and will be removed "
+                            + "in the next major release");
+                        builder.enableLatLon(XContentMapValues.lenientNodeBooleanValue(propNode, propName));
+                        iterator.remove();
+                    } else if (propName.equals("precision_step")) {
+                        deprecationLogger.deprecated(CONTENT_TYPE + " precision_step parameter is deprecated and will be removed "
+                            + "in the next major release");
+                        builder.precisionStep(XContentMapValues.nodeIntegerValue(propNode));
+                        iterator.remove();
+                    } else if (propName.equals("geohash")) {
+                        deprecationLogger.deprecated(CONTENT_TYPE + " geohash parameter is deprecated and will be removed "
+                            + "in the next major release");
+                        builder.enableGeoHash(XContentMapValues.lenientNodeBooleanValue(propNode, propName));
+                        iterator.remove();
+                    } else if (propName.equals("geohash_prefix")) {
+                        deprecationLogger.deprecated(CONTENT_TYPE + " geohash_prefix parameter is deprecated and will be removed "
+                            + "in the next major release");
+                        builder.geoHashPrefix(XContentMapValues.lenientNodeBooleanValue(propNode, propName));
+                        if (XContentMapValues.lenientNodeBooleanValue(propNode, propName)) {
+                            builder.enableGeoHash(true);
+                        }
+                        iterator.remove();
+                    } else if (propName.equals("geohash_precision")) {
+                        deprecationLogger.deprecated(CONTENT_TYPE + " geohash_precision parameter is deprecated and will be removed "
+                            + "in the next major release");
+                        if (propNode instanceof Integer) {
+                            builder.geoHashPrecision(XContentMapValues.nodeIntegerValue(propNode));
+                        } else {
+                            builder.geoHashPrecision(GeoUtils.geoHashLevelsForPrecision(propNode.toString()));
+                        }
+                        iterator.remove();
                     }
-                    iterator.remove();
-                } else if (propName.equals(Names.IGNORE_MALFORMED)) {
-                    builder.ignoreMalformed(XContentMapValues.lenientNodeBooleanValue(propNode));
+                }
+
+                if (propName.equals(Names.IGNORE_MALFORMED)) {
+                    builder.ignoreMalformed(TypeParsers.nodeBooleanValue(name, Names.IGNORE_MALFORMED, propNode));
                     iterator.remove();
                 }
             }
 
             if (builder instanceof LegacyGeoPointFieldMapper.Builder) {
                 return LegacyGeoPointFieldMapper.parse((LegacyGeoPointFieldMapper.Builder) builder, node, parserContext);
+            } else if (builder instanceof LatLonPointFieldMapper.Builder) {
+                return (LatLonPointFieldMapper.Builder) builder;
             }
 
             return (GeoPointFieldMapper.Builder) builder;
         }
     }
 
-    public static class GeoPointFieldType extends MappedFieldType {
+    public abstract static class GeoPointFieldType extends MappedFieldType {
+        GeoPointFieldType() {
+        }
+
+        GeoPointFieldType(GeoPointFieldType ref) {
+            super(ref);
+        }
+
+        @Override
+        public String typeName() {
+            return CONTENT_TYPE;
+        }
+
+        @Override
+        public FieldStats stats(IndexReader reader) throws IOException {
+            int maxDoc = reader.maxDoc();
+            FieldInfo fi = org.apache.lucene.index.MultiFields.getMergedFieldInfos(reader).fieldInfo(name());
+            if (fi == null) {
+                return null;
+            }
+            /**
+             * we don't have a specific type for geo_point so we use an empty {@link FieldStats.Text}.
+             * TODO: we should maybe support a new type that knows how to (de)encode the min/max information
+             */
+            return new FieldStats.Text(maxDoc, -1, -1, -1, isSearchable(), isAggregatable());
+        }
+    }
+
+    public static class LegacyGeoPointFieldType extends GeoPointFieldType {
         protected MappedFieldType geoHashFieldType;
         protected int geoHashPrecision;
         protected boolean geoHashPrefixEnabled;
 
         protected MappedFieldType latFieldType;
         protected MappedFieldType lonFieldType;
+        protected boolean numericEncoded;
 
-        GeoPointFieldType() {}
+        LegacyGeoPointFieldType() {}
 
-        GeoPointFieldType(GeoPointFieldType ref) {
+        LegacyGeoPointFieldType(LegacyGeoPointFieldType ref) {
             super(ref);
             this.geoHashFieldType = ref.geoHashFieldType; // copying ref is ok, this can never be modified
             this.geoHashPrecision = ref.geoHashPrecision;
             this.geoHashPrefixEnabled = ref.geoHashPrefixEnabled;
             this.latFieldType = ref.latFieldType; // copying ref is ok, this can never be modified
             this.lonFieldType = ref.lonFieldType; // copying ref is ok, this can never be modified
+            this.numericEncoded = ref.numericEncoded;
         }
 
         @Override
         public MappedFieldType clone() {
-            return new GeoPointFieldType(this);
+            return new LegacyGeoPointFieldType(this);
         }
 
         @Override
         public boolean equals(Object o) {
             if (!super.equals(o)) return false;
-            GeoPointFieldType that = (GeoPointFieldType) o;
+            LegacyGeoPointFieldType that = (LegacyGeoPointFieldType) o;
             return  geoHashPrecision == that.geoHashPrecision &&
                     geoHashPrefixEnabled == that.geoHashPrefixEnabled &&
+                    numericEncoded == that.numericEncoded &&
                     java.util.Objects.equals(geoHashFieldType, that.geoHashFieldType) &&
                     java.util.Objects.equals(latFieldType, that.latFieldType) &&
                     java.util.Objects.equals(lonFieldType, that.lonFieldType);
@@ -285,19 +340,14 @@ public boolean equals(Object o) {
 
         @Override
         public int hashCode() {
-            return java.util.Objects.hash(super.hashCode(), geoHashFieldType, geoHashPrecision, geoHashPrefixEnabled, latFieldType,
-                    lonFieldType);
-        }
-
-        @Override
-        public String typeName() {
-            return CONTENT_TYPE;
+            return java.util.Objects.hash(super.hashCode(), geoHashFieldType, geoHashPrecision, geoHashPrefixEnabled,
+                    numericEncoded, latFieldType, lonFieldType);
         }
 
         @Override
         public void checkCompatibility(MappedFieldType fieldType, List<String> conflicts, boolean strict) {
             super.checkCompatibility(fieldType, conflicts, strict);
-            GeoPointFieldType other = (GeoPointFieldType)fieldType;
+            LegacyGeoPointFieldType other = (LegacyGeoPointFieldType)fieldType;
             if (isLatLonEnabled() != other.isLatLonEnabled()) {
                 conflicts.add("mapper [" + name() + "] has different [lat_lon]");
             }
@@ -378,6 +428,23 @@ public DocValueFormat docValueFormat(@Nullable String format, DateTimeZone timeZ
         public Query termQuery(Object value, QueryShardContext context) {
             throw new QueryShardException(context, "Geo fields do not support exact searching, use dedicated geo queries instead: [" + name() + "]");
         }
+
+        @Override
+        public FieldStats.GeoPoint stats(IndexReader reader) throws IOException {
+            String field = name();
+            FieldInfo fi = org.apache.lucene.index.MultiFields.getMergedFieldInfos(reader).fieldInfo(field);
+            if (fi == null) {
+                return null;
+            }
+
+            Terms terms = org.apache.lucene.index.MultiFields.getTerms(reader, field);
+            if (terms == null) {
+                return new FieldStats.GeoPoint(reader.maxDoc(), 0L, -1L, -1L, isSearchable(), isAggregatable());
+            }
+            return new FieldStats.GeoPoint(reader.maxDoc(), terms.getDocCount(), -1L, terms.getSumTotalTermFreq(), isSearchable(),
+                isAggregatable(), prefixCodedToGeoPoint(terms.getMin(), numericEncoded),
+                prefixCodedToGeoPoint(terms.getMax(), numericEncoded));
+        }
     }
 
     protected FieldMapper latMapper;
@@ -398,9 +465,10 @@ protected BaseGeoPointFieldMapper(String simpleName, MappedFieldType fieldType,
         this.ignoreMalformed = ignoreMalformed;
     }
 
-    @Override
-    public GeoPointFieldType fieldType() {
-        return (GeoPointFieldType) super.fieldType();
+
+
+    public LegacyGeoPointFieldType legacyFieldType() {
+        return (LegacyGeoPointFieldType) super.fieldType();
     }
 
     @Override
@@ -414,15 +482,22 @@ protected void doMerge(Mapper mergeWith, boolean updateAllTypes) {
 
     @Override
     public Iterator<Mapper> iterator() {
+        if (this instanceof LatLonPointFieldMapper == false) {
+            return Iterators.concat(super.iterator(), legacyIterator());
+        }
+        return super.iterator();
+    }
+
+    public Iterator<Mapper> legacyIterator() {
         List<Mapper> extras = new ArrayList<>();
-        if (fieldType().isGeoHashEnabled()) {
+        if (legacyFieldType().isGeoHashEnabled()) {
             extras.add(geoHashMapper);
         }
-        if (fieldType().isLatLonEnabled()) {
+        if (legacyFieldType().isLatLonEnabled()) {
             extras.add(latMapper);
             extras.add(lonMapper);
         }
-        return Iterators.concat(super.iterator(), extras.iterator());
+        return extras.iterator();
     }
 
     @Override
@@ -431,18 +506,18 @@ protected String contentType() {
     }
 
     @Override
-    protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
+    protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
         throw new UnsupportedOperationException("Parsing is implemented in parse(), this method should NEVER be called");
     }
 
     protected void parse(ParseContext context, GeoPoint point, String geoHash) throws IOException {
-        if (fieldType().isGeoHashEnabled()) {
+        if (legacyFieldType().isGeoHashEnabled()) {
             if (geoHash == null) {
                 geoHash = GeoHashUtils.stringEncode(point.lon(), point.lat());
             }
             addGeoHashField(context, geoHash);
         }
-        if (fieldType().isLatLonEnabled()) {
+        if (legacyFieldType().isLatLonEnabled()) {
             latMapper.parse(context.createExternalValueContext(point.lat()));
             lonMapper.parse(context.createExternalValueContext(point.lon()));
         }
@@ -517,8 +592,9 @@ public Mapper parse(ParseContext context) throws IOException {
     }
 
     private void addGeoHashField(ParseContext context, String geoHash) throws IOException {
-        int len = Math.min(fieldType().geoHashPrecision(), geoHash.length());
-        int min = fieldType().isGeoHashPrefixEnabled() ? 1 : len;
+        LegacyGeoPointFieldType ft = (LegacyGeoPointFieldType)fieldType;
+        int len = Math.min(ft.geoHashPrecision(), geoHash.length());
+        int min = ft.isGeoHashPrefixEnabled() ? 1 : len;
 
         for (int i = len; i >= min; i--) {
             // side effect of this call is adding the field
@@ -537,23 +613,30 @@ private void parsePointFromString(ParseContext context, GeoPoint sparse, String
     @Override
     protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException {
         super.doXContentBody(builder, includeDefaults, params);
-        if (includeDefaults || fieldType().isLatLonEnabled() != GeoPointFieldMapper.Defaults.ENABLE_LATLON) {
-            builder.field("lat_lon", fieldType().isLatLonEnabled());
+        if (this instanceof LatLonPointFieldMapper == false) {
+            legacyDoXContentBody(builder, includeDefaults, params);
         }
-        if (fieldType().isLatLonEnabled() && (includeDefaults || fieldType().latFieldType().numericPrecisionStep() != LegacyNumericUtils.PRECISION_STEP_DEFAULT)) {
-            builder.field("precision_step", fieldType().latFieldType().numericPrecisionStep());
+        if (includeDefaults || ignoreMalformed.explicit()) {
+            builder.field(Names.IGNORE_MALFORMED, ignoreMalformed.value());
         }
-        if (includeDefaults || fieldType().isGeoHashEnabled() != Defaults.ENABLE_GEOHASH) {
-            builder.field("geohash", fieldType().isGeoHashEnabled());
+    }
+
+    protected void legacyDoXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException {
+        LegacyGeoPointFieldType ft = (LegacyGeoPointFieldType) fieldType;
+        if (includeDefaults || ft.isLatLonEnabled() != GeoPointFieldMapper.Defaults.ENABLE_LATLON) {
+            builder.field("lat_lon", ft.isLatLonEnabled());
         }
-        if (includeDefaults || fieldType().isGeoHashPrefixEnabled() != Defaults.ENABLE_GEOHASH_PREFIX) {
-            builder.field("geohash_prefix", fieldType().isGeoHashPrefixEnabled());
+        if (ft.isLatLonEnabled() && (includeDefaults || ft.latFieldType().numericPrecisionStep() != LegacyNumericUtils.PRECISION_STEP_DEFAULT)) {
+            builder.field("precision_step", ft.latFieldType().numericPrecisionStep());
         }
-        if (fieldType().isGeoHashEnabled() && (includeDefaults || fieldType().geoHashPrecision() != Defaults.GEO_HASH_PRECISION)) {
-            builder.field("geohash_precision", fieldType().geoHashPrecision());
+        if (includeDefaults || ft.isGeoHashEnabled() != Defaults.ENABLE_GEOHASH) {
+            builder.field("geohash", ft.isGeoHashEnabled());
         }
-        if (includeDefaults || ignoreMalformed.explicit()) {
-            builder.field(Names.IGNORE_MALFORMED, ignoreMalformed.value());
+        if (includeDefaults || ft.isGeoHashPrefixEnabled() != Defaults.ENABLE_GEOHASH_PREFIX) {
+            builder.field("geohash_prefix", ft.isGeoHashPrefixEnabled());
+        }
+        if (ft.isGeoHashEnabled() && (includeDefaults || ft.geoHashPrecision() != Defaults.GEO_HASH_PRECISION)) {
+            builder.field("geohash_precision", ft.geoHashPrecision());
         }
     }
 
@@ -577,4 +660,19 @@ public FieldMapper updateFieldType(Map<String, MappedFieldType> fullNameToFieldT
         updated.lonMapper = lonUpdated;
         return updated;
     }
+
+    private static GeoPoint prefixCodedToGeoPoint(BytesRef val, boolean numericEncoded) {
+        final long encoded = numericEncoded ? LegacyNumericUtils.prefixCodedToLong(val) : prefixCodedToGeoCoded(val);
+        return new GeoPoint(MortonEncoder.decodeLatitude(encoded), MortonEncoder.decodeLongitude(encoded));
+    }
+
+    private static long prefixCodedToGeoCoded(BytesRef val) {
+        long result = fromBytes((byte)0, (byte)0, (byte)0, (byte)0, val.bytes[val.offset + 0], val.bytes[val.offset + 1],
+            val.bytes[val.offset + 2], val.bytes[val.offset + 3]);
+        return result << 32;
+    }
+
+    private static long fromBytes(byte b1, byte b2, byte b3, byte b4, byte b5, byte b6, byte b7, byte b8) {
+        return ((long)b1 & 255L) << 56 | ((long)b2 & 255L) << 48 | ((long)b3 & 255L) << 40 | ((long)b4 & 255L) << 32 | ((long)b5 & 255L) << 24 | ((long)b6 & 255L) << 16 | ((long)b7 & 255L) << 8 | (long)b8 & 255L;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java
index cb6fae8b59d50..fa50e13e19ae5 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java
@@ -22,6 +22,7 @@
 import com.carrotsearch.hppc.ObjectArrayList;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.store.ByteArrayDataOutput;
 import org.apache.lucene.util.BytesRef;
@@ -85,7 +86,7 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
 
     static final class BinaryFieldType extends MappedFieldType {
 
-        public BinaryFieldType() {}
+        BinaryFieldType() {}
 
         protected BinaryFieldType(BinaryFieldType ref) {
             super(ref);
@@ -104,7 +105,7 @@ public String typeName() {
 
 
         @Override
-        public BytesReference valueForSearch(Object value) {
+        public BytesReference valueForDisplay(Object value) {
             if (value == null) {
                 return null;
             }
@@ -140,7 +141,7 @@ protected BinaryFieldMapper(String simpleName, MappedFieldType fieldType, Mapped
     }
 
     @Override
-    protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
+    protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
         if (!fieldType().stored() && !fieldType().hasDocValues()) {
             return;
         }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java
index b27f564f2d7e8..03d1dc7d57187 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java
@@ -22,11 +22,14 @@
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermRangeQuery;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -34,6 +37,7 @@
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType;
 import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
+import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.DocValueFormat;
 import org.joda.time.DateTimeZone;
 
@@ -42,7 +46,6 @@
 import java.util.List;
 import java.util.Map;
 
-import static org.elasticsearch.common.xcontent.support.XContentMapValues.lenientNodeBooleanValue;
 import static org.elasticsearch.index.mapper.TypeParsers.parseField;
 
 /**
@@ -52,6 +55,8 @@ public class BooleanFieldMapper extends FieldMapper {
 
     public static final String CONTENT_TYPE = "boolean";
 
+    private static final DeprecationLogger deprecationLogger = new DeprecationLogger(Loggers.getLogger(BooleanFieldMapper.class));
+
     public static class Defaults {
         public static final MappedFieldType FIELD_TYPE = new BooleanFieldType();
 
@@ -106,7 +111,7 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
                     if (propNode == null) {
                         throw new MapperParsingException("Property [null_value] cannot be null.");
                     }
-                    builder.nullValue(lenientNodeBooleanValue(propNode));
+                    builder.nullValue(TypeParsers.nodeBooleanValue(name, "null_value", propNode));
                     iterator.remove();
                 }
             }
@@ -151,20 +156,29 @@ public BytesRef indexedValueForSearch(Object value) {
             } else {
                 sValue = value.toString();
             }
-            if (sValue.length() == 0) {
-                return Values.FALSE;
-            }
-            if (sValue.length() == 1 && sValue.charAt(0) == 'F') {
-                return Values.FALSE;
-            }
-            if (Booleans.parseBoolean(sValue, false)) {
-                return Values.TRUE;
+            switch (sValue) {
+                case "true":
+                    return Values.TRUE;
+                case "false":
+                    return Values.FALSE;
+                default:
+                    deprecationLogger.deprecated("searching using boolean value [" + sValue +
+                                    "] is deprecated, please use [true] or [false]");
+                    if (sValue.length() == 0) {
+                        return Values.FALSE;
+                    }
+                    if (sValue.length() == 1 && sValue.charAt(0) == 'F') {
+                        return Values.FALSE;
+                    }
+                    if (Booleans.parseBoolean(sValue, false)) {
+                        return Values.TRUE;
+                    }
+                    return Values.FALSE;
             }
-            return Values.FALSE;
         }
 
         @Override
-        public Boolean valueForSearch(Object value) {
+        public Boolean valueForDisplay(Object value) {
             if (value == null) {
                 return null;
             }
@@ -197,7 +211,7 @@ public DocValueFormat docValueFormat(@Nullable String format, DateTimeZone timeZ
         }
 
         @Override
-        public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper) {
+        public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) {
             failIfNotIndexed();
             return new TermRangeQuery(name(),
                 lowerTerm == null ? null : indexedValueForSearch(lowerTerm),
@@ -217,7 +231,7 @@ public BooleanFieldType fieldType() {
     }
 
     @Override
-    protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
+    protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
         if (fieldType().indexOptions() == IndexOptions.NONE && !fieldType().stored() && !fieldType().hasDocValues()) {
             return;
         }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java
index 13bb7d255a873..dfdb726ec57cc 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java
@@ -19,7 +19,7 @@
 package org.elasticsearch.index.mapper;
 
 import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.document.Field;
+import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.suggest.document.Completion50PostingsFormat;
 import org.apache.lucene.search.suggest.document.CompletionAnalyzer;
@@ -38,6 +38,7 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentParser.NumberType;
 import org.elasticsearch.common.xcontent.XContentParser.Token;
+import org.elasticsearch.index.analysis.AnalyzerScope;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.search.suggest.completion.CompletionSuggester;
 import org.elasticsearch.search.suggest.completion.context.ContextMapping;
@@ -126,22 +127,22 @@ public static class TypeParser implements Mapper.TypeParser {
                 if (fieldName.equals("type")) {
                     continue;
                 }
-                if (parserContext.parseFieldMatcher().match(fieldName, Fields.ANALYZER)) {
+                if (Fields.ANALYZER.match(fieldName)) {
                     indexAnalyzer = getNamedAnalyzer(parserContext, fieldNode.toString());
                     iterator.remove();
-                } else if (parserContext.parseFieldMatcher().match(fieldName, Fields.SEARCH_ANALYZER)) {
+                } else if (Fields.SEARCH_ANALYZER.match(fieldName)) {
                     searchAnalyzer = getNamedAnalyzer(parserContext, fieldNode.toString());
                     iterator.remove();
-                } else if (parserContext.parseFieldMatcher().match(fieldName, Fields.PRESERVE_SEPARATORS)) {
+                } else if (Fields.PRESERVE_SEPARATORS.match(fieldName)) {
                     builder.preserveSeparators(Boolean.parseBoolean(fieldNode.toString()));
                     iterator.remove();
-                } else if (parserContext.parseFieldMatcher().match(fieldName, Fields.PRESERVE_POSITION_INCREMENTS)) {
+                } else if (Fields.PRESERVE_POSITION_INCREMENTS.match(fieldName)) {
                     builder.preservePositionIncrements(Boolean.parseBoolean(fieldNode.toString()));
                     iterator.remove();
-                } else if (parserContext.parseFieldMatcher().match(fieldName, Fields.MAX_INPUT_LENGTH)) {
+                } else if (Fields.MAX_INPUT_LENGTH.match(fieldName)) {
                     builder.maxInputLength(Integer.parseInt(fieldNode.toString()));
                     iterator.remove();
-                } else if (parserContext.parseFieldMatcher().match(fieldName, Fields.CONTEXTS)) {
+                } else if (Fields.CONTEXTS.match(fieldName)) {
                     builder.contextMappings(ContextMappings.load(fieldNode, parserContext.indexVersionCreated()));
                     iterator.remove();
                 } else if (parseMultiField(builder, name, parserContext, fieldName, fieldNode)) {
@@ -153,7 +154,7 @@ public static class TypeParser implements Mapper.TypeParser {
                 if (searchAnalyzer != null) {
                     throw new MapperParsingException("analyzer on completion field [" + name + "] must be set when search_analyzer is set");
                 }
-                indexAnalyzer = searchAnalyzer = parserContext.analysisService().analyzer("simple");
+                indexAnalyzer = searchAnalyzer = parserContext.getIndexAnalyzers().get("simple");
             } else if (searchAnalyzer == null) {
                 searchAnalyzer = indexAnalyzer;
             }
@@ -164,7 +165,7 @@ public static class TypeParser implements Mapper.TypeParser {
         }
 
         private NamedAnalyzer getNamedAnalyzer(ParserContext parserContext, String name) {
-            NamedAnalyzer analyzer = parserContext.analysisService().analyzer(name);
+            NamedAnalyzer analyzer = parserContext.getIndexAnalyzers().get(name);
             if (analyzer == null) {
                 throw new IllegalArgumentException("Can't find default or mapped analyzer with name [" + name + "]");
             }
@@ -209,7 +210,7 @@ public void setContextMappings(ContextMappings contextMappings) {
         public NamedAnalyzer indexAnalyzer() {
             final NamedAnalyzer indexAnalyzer = super.indexAnalyzer();
             if (indexAnalyzer != null && !(indexAnalyzer.analyzer() instanceof CompletionAnalyzer)) {
-                return new NamedAnalyzer(indexAnalyzer.name(),
+                return new NamedAnalyzer(indexAnalyzer.name(), AnalyzerScope.INDEX,
                         new CompletionAnalyzer(indexAnalyzer, preserveSep, preservePositionIncrements));
 
             }
@@ -220,7 +221,7 @@ public NamedAnalyzer indexAnalyzer() {
         public NamedAnalyzer searchAnalyzer() {
             final NamedAnalyzer searchAnalyzer = super.searchAnalyzer();
             if (searchAnalyzer != null && !(searchAnalyzer.analyzer() instanceof CompletionAnalyzer)) {
-                return new NamedAnalyzer(searchAnalyzer.name(),
+                return new NamedAnalyzer(searchAnalyzer.name(), AnalyzerScope.INDEX,
                         new CompletionAnalyzer(searchAnalyzer, preserveSep, preservePositionIncrements));
             }
             return searchAnalyzer;
@@ -530,14 +531,10 @@ private void parse(ParseContext parseContext, Token token, XContentParser parser
                                 if (currentToken == XContentParser.Token.FIELD_NAME) {
                                     fieldName = parser.currentName();
                                     contextMapping = contextMappings.get(fieldName);
-                                } else if (currentToken == XContentParser.Token.VALUE_STRING
-                                        || currentToken == XContentParser.Token.START_ARRAY
-                                        || currentToken == XContentParser.Token.START_OBJECT) {
+                                } else {
                                     assert fieldName != null;
                                     assert !contextsMap.containsKey(fieldName);
                                     contextsMap.put(fieldName, contextMapping.parseContext(parseContext, parser));
-                                } else {
-                                    throw new IllegalArgumentException("contexts must be an object or an array , but was [" + currentToken + "]");
                                 }
                             }
                         } else {
@@ -589,7 +586,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     }
 
     @Override
-    protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
+    protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
         // no-op
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper2x.java b/core/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper2x.java
index 655af43710ffb..f1a9f1990cefd 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper2x.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper2x.java
@@ -22,6 +22,7 @@
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.search.suggest.analyzing.XAnalyzingSuggester;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.ElasticsearchParseException;
@@ -178,19 +179,19 @@ public static class TypeParser implements Mapper.TypeParser {
 
                     indexAnalyzer = getNamedAnalyzer(parserContext, fieldNode.toString());
                     iterator.remove();
-                } else if (parserContext.parseFieldMatcher().match(fieldName, Fields.SEARCH_ANALYZER)) {
+                } else if (Fields.SEARCH_ANALYZER.match(fieldName)) {
                     searchAnalyzer = getNamedAnalyzer(parserContext, fieldNode.toString());
                     iterator.remove();
                 } else if (fieldName.equals(Fields.PAYLOADS)) {
                     builder.payloads(Boolean.parseBoolean(fieldNode.toString()));
                     iterator.remove();
-                } else if (parserContext.parseFieldMatcher().match(fieldName, Fields.PRESERVE_SEPARATORS)) {
+                } else if (Fields.PRESERVE_SEPARATORS.match(fieldName)) {
                     builder.preserveSeparators(Boolean.parseBoolean(fieldNode.toString()));
                     iterator.remove();
-                } else if (parserContext.parseFieldMatcher().match(fieldName, Fields.PRESERVE_POSITION_INCREMENTS)) {
+                } else if (Fields.PRESERVE_POSITION_INCREMENTS.match(fieldName)) {
                     builder.preservePositionIncrements(Boolean.parseBoolean(fieldNode.toString()));
                     iterator.remove();
-                } else if (parserContext.parseFieldMatcher().match(fieldName, Fields.MAX_INPUT_LENGTH)) {
+                } else if (Fields.MAX_INPUT_LENGTH.match(fieldName)) {
                     builder.maxInputLength(Integer.parseInt(fieldNode.toString()));
                     iterator.remove();
                 } else if (parseMultiField(builder, name, parserContext, fieldName, fieldNode)) {
@@ -206,7 +207,7 @@ public static class TypeParser implements Mapper.TypeParser {
                     throw new MapperParsingException(
                         "analyzer on completion field [" + name + "] must be set when search_analyzer is set");
                 }
-                indexAnalyzer = searchAnalyzer = parserContext.analysisService().analyzer("simple");
+                indexAnalyzer = searchAnalyzer = parserContext.getIndexAnalyzers().get("simple");
             } else if (searchAnalyzer == null) {
                 searchAnalyzer = indexAnalyzer;
             }
@@ -217,7 +218,7 @@ public static class TypeParser implements Mapper.TypeParser {
         }
 
         private NamedAnalyzer getNamedAnalyzer(ParserContext parserContext, String name) {
-            NamedAnalyzer analyzer = parserContext.analysisService().analyzer(name);
+            NamedAnalyzer analyzer = parserContext.getIndexAnalyzers().get(name);
             if (analyzer == null) {
                 throw new IllegalArgumentException("Can't find default or mapped analyzer with name [" + name + "]");
             }
@@ -521,7 +522,7 @@ private static final class SuggestField extends Field {
         private final CompletionTokenStream.ToFiniteStrings toFiniteStrings;
         private final ContextMapping.Context ctx;
 
-        public SuggestField(String name, ContextMapping.Context ctx,
+        SuggestField(String name, ContextMapping.Context ctx,
                             String value, MappedFieldType type, BytesRef payload,
                             CompletionTokenStream.ToFiniteStrings toFiniteStrings) {
             super(name, value, type);
@@ -566,7 +567,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     }
 
     @Override
-    protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
+    protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/CustomDocValuesField.java b/core/src/main/java/org/elasticsearch/index/mapper/CustomDocValuesField.java
index a8b27d112453b..8e6cee222d4cd 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/CustomDocValuesField.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/CustomDocValuesField.java
@@ -39,7 +39,7 @@ abstract class CustomDocValuesField implements IndexableField {
 
     private final String name;
 
-    public CustomDocValuesField(String  name) {
+    CustomDocValuesField(String  name) {
         this.name = name;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java
index 717f036155254..f3ba596251f62 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java
@@ -19,14 +19,16 @@
 
 package org.elasticsearch.index.mapper;
 
-import org.apache.lucene.document.Field;
 import org.apache.lucene.document.StoredField;
 import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.document.LongPoint;
-import org.apache.lucene.index.XPointValues;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.PointValues;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.search.BoostQuery;
+import org.apache.lucene.search.IndexOrDocValuesQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.Version;
@@ -43,9 +45,9 @@
 import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType;
 import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
 import org.elasticsearch.index.mapper.LegacyNumberFieldMapper.Defaults;
+import org.elasticsearch.index.query.QueryRewriteContext;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.internal.SearchContext;
 import org.joda.time.DateTimeZone;
 
 import java.io.IOException;
@@ -54,8 +56,7 @@
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
-import java.util.concurrent.Callable;
-
+import java.util.ArrayList;
 import static org.elasticsearch.index.mapper.TypeParsers.parseDateTimeFormatter;
 
 /** A {@link FieldMapper} for ip addresses. */
@@ -69,6 +70,7 @@ public static class Builder extends FieldMapper.Builder<Builder, DateFieldMapper
 
         private Boolean ignoreMalformed;
         private Locale locale;
+        private boolean dateTimeFormatterSet = false;
 
         public Builder(String name) {
             super(name, new DateFieldType(), new DateFieldType());
@@ -96,8 +98,14 @@ protected Explicit<Boolean> ignoreMalformed(BuilderContext context) {
             return Defaults.IGNORE_MALFORMED;
         }
 
+        /** Whether an explicit format for this date field has been set already. */
+        public boolean isDateTimeFormatterSet() {
+            return dateTimeFormatterSet;
+        }
+
         public Builder dateTimeFormatter(FormatDateTimeFormatter dateTimeFormatter) {
             fieldType().setDateTimeFormatter(dateTimeFormatter);
+            dateTimeFormatterSet = true;
             return this;
         }
 
@@ -146,7 +154,7 @@ public Mapper.Builder<?,?> parse(String name, Map<String, Object> node, ParserCo
                     builder.nullValue(propNode.toString());
                     iterator.remove();
                 } else if (propName.equals("ignore_malformed")) {
-                    builder.ignoreMalformed(TypeParsers.nodeBooleanValue("ignore_malformed", propNode, parserContext));
+                    builder.ignoreMalformed(TypeParsers.nodeBooleanValue(name, "ignore_malformed", propNode));
                     iterator.remove();
                 } else if (propName.equals("locale")) {
                     builder.locale(LocaleUtils.parse(propNode.toString()));
@@ -163,69 +171,6 @@ public Mapper.Builder<?,?> parse(String name, Map<String, Object> node, ParserCo
     }
 
     public static final class DateFieldType extends MappedFieldType {
-
-        final class LateParsingQuery extends Query {
-
-            final Object lowerTerm;
-            final Object upperTerm;
-            final boolean includeLower;
-            final boolean includeUpper;
-            final DateTimeZone timeZone;
-            final DateMathParser forcedDateParser;
-
-            public LateParsingQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper,
-                    DateTimeZone timeZone, DateMathParser forcedDateParser) {
-                this.lowerTerm = lowerTerm;
-                this.upperTerm = upperTerm;
-                this.includeLower = includeLower;
-                this.includeUpper = includeUpper;
-                this.timeZone = timeZone;
-                this.forcedDateParser = forcedDateParser;
-            }
-
-            @Override
-            public Query rewrite(IndexReader reader) throws IOException {
-                Query rewritten = super.rewrite(reader);
-                if (rewritten != this) {
-                    return rewritten;
-                }
-                return innerRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, timeZone, forcedDateParser);
-            }
-
-            // Even though we only cache rewritten queries it is good to let all queries implement hashCode() and equals():
-            @Override
-            public boolean equals(Object o) {
-                if (this == o) return true;
-                if (sameClassAs(o) == false) return false;
-
-                LateParsingQuery that = (LateParsingQuery) o;
-                if (includeLower != that.includeLower) return false;
-                if (includeUpper != that.includeUpper) return false;
-                if (lowerTerm != null ? !lowerTerm.equals(that.lowerTerm) : that.lowerTerm != null) return false;
-                if (upperTerm != null ? !upperTerm.equals(that.upperTerm) : that.upperTerm != null) return false;
-                if (timeZone != null ? !timeZone.equals(that.timeZone) : that.timeZone != null) return false;
-
-                return true;
-            }
-
-            @Override
-            public int hashCode() {
-                return Objects.hash(classHash(), lowerTerm, upperTerm, includeLower, includeUpper, timeZone);
-            }
-
-            @Override
-            public String toString(String s) {
-                final StringBuilder sb = new StringBuilder();
-                return sb.append(name()).append(':')
-                    .append(includeLower ? '[' : '{')
-                    .append((lowerTerm == null) ? "*" : lowerTerm.toString())
-                    .append(" TO ")
-                    .append((upperTerm == null) ? "*" : upperTerm.toString())
-                    .append(includeUpper ? ']' : '}')
-                    .toString();
-            }
-        }
-
         protected FormatDateTimeFormatter dateTimeFormatter;
         protected DateMathParser dateMathParser;
 
@@ -268,16 +213,12 @@ public String typeName() {
         @Override
         public void checkCompatibility(MappedFieldType fieldType, List<String> conflicts, boolean strict) {
             super.checkCompatibility(fieldType, conflicts, strict);
-            if (strict) {
-                DateFieldType other = (DateFieldType)fieldType;
-                if (Objects.equals(dateTimeFormatter().format(), other.dateTimeFormatter().format()) == false) {
-                    conflicts.add("mapper [" + name()
-                        + "] is used by multiple types. Set update_all_types to true to update [format] across all types.");
-                }
-                if (Objects.equals(dateTimeFormatter().locale(), other.dateTimeFormatter().locale()) == false) {
-                    conflicts.add("mapper [" + name()
-                        + "] is used by multiple types. Set update_all_types to true to update [locale] across all types.");
-                }
+            DateFieldType other = (DateFieldType) fieldType;
+            if (Objects.equals(dateTimeFormatter().format(), other.dateTimeFormatter().format()) == false) {
+                conflicts.add("mapper [" + name() + "] has different [format] values");
+            }
+            if (Objects.equals(dateTimeFormatter().locale(), other.dateTimeFormatter().locale()) == false) {
+                conflicts.add("mapper [" + name() + "] has different [locale] values");
             }
         }
 
@@ -301,7 +242,7 @@ long parse(String value) {
 
         @Override
         public Query termQuery(Object value, @Nullable QueryShardContext context) {
-            Query query = innerRangeQuery(value, value, true, true, null, null);
+            Query query = innerRangeQuery(value, value, true, true, null, null, context);
             if (boost() != 1f) {
                 query = new BoostQuery(query, boost());
             }
@@ -309,19 +250,19 @@ public Query termQuery(Object value, @Nullable QueryShardContext context) {
         }
 
         @Override
-        public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper) {
+        public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) {
             failIfNotIndexed();
-            return rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, null, null);
+            return rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, null, null, context);
         }
 
         public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper,
-                @Nullable DateTimeZone timeZone, @Nullable DateMathParser forcedDateParser) {
+                @Nullable DateTimeZone timeZone, @Nullable DateMathParser forcedDateParser, QueryShardContext context) {
             failIfNotIndexed();
-            return new LateParsingQuery(lowerTerm, upperTerm, includeLower, includeUpper, timeZone, forcedDateParser);
+            return innerRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, timeZone, forcedDateParser, context);
         }
 
         Query innerRangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper,
-                @Nullable DateTimeZone timeZone, @Nullable DateMathParser forcedDateParser) {
+                @Nullable DateTimeZone timeZone, @Nullable DateMathParser forcedDateParser, QueryShardContext context) {
             failIfNotIndexed();
             DateMathParser parser = forcedDateParser == null
                     ? dateMathParser
@@ -330,7 +271,7 @@ Query innerRangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower,
             if (lowerTerm == null) {
                 l = Long.MIN_VALUE;
             } else {
-                l = parseToMilliseconds(lowerTerm, !includeLower, timeZone, parser);
+                l = parseToMilliseconds(lowerTerm, !includeLower, timeZone, parser, context);
                 if (includeLower == false) {
                     ++l;
                 }
@@ -338,16 +279,21 @@ Query innerRangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower,
             if (upperTerm == null) {
                 u = Long.MAX_VALUE;
             } else {
-                u = parseToMilliseconds(upperTerm, includeUpper, timeZone, parser);
+                u = parseToMilliseconds(upperTerm, includeUpper, timeZone, parser, context);
                 if (includeUpper == false) {
                     --u;
                 }
             }
-            return LongPoint.newRangeQuery(name(), l, u);
+            Query query = LongPoint.newRangeQuery(name(), l, u);
+            if (hasDocValues()) {
+                Query dvQuery = SortedNumericDocValuesField.newRangeQuery(name(), l, u);
+                query = new IndexOrDocValuesQuery(query, dvQuery);
+            }
+            return query;
         }
 
         public long parseToMilliseconds(Object value, boolean roundUp,
-                @Nullable DateTimeZone zone, @Nullable DateMathParser forcedDateParser) {
+                @Nullable DateTimeZone zone, @Nullable DateMathParser forcedDateParser, QueryRewriteContext context) {
             DateMathParser dateParser = dateMathParser();
             if (forcedDateParser != null) {
                 dateParser = forcedDateParser;
@@ -359,28 +305,23 @@ public long parseToMilliseconds(Object value, boolean roundUp,
             } else {
                 strValue = value.toString();
             }
-            return dateParser.parse(strValue, now(), roundUp, zone);
-        }
-
-        private static Callable<Long> now() {
-            return () -> {
-                final SearchContext context = SearchContext.current();
-                return context != null
-                        ? context.nowInMillis()
-                        : System.currentTimeMillis();
-            };
+            return dateParser.parse(strValue, context::nowInMillis, roundUp, zone);
         }
 
         @Override
         public FieldStats.Date stats(IndexReader reader) throws IOException {
             String field = name();
-            long size = XPointValues.size(reader, field);
-            if (size == 0) {
+            FieldInfo fi = org.apache.lucene.index.MultiFields.getMergedFieldInfos(reader).fieldInfo(name());
+            if (fi == null) {
                 return null;
             }
-            int docCount = XPointValues.getDocCount(reader, field);
-            byte[] min = XPointValues.getMinPackedValue(reader, field);
-            byte[] max = XPointValues.getMaxPackedValue(reader, field);
+            long size = PointValues.size(reader, field);
+            if (size == 0) {
+                return new FieldStats.Date(reader.maxDoc(), 0, -1, -1, isSearchable(), isAggregatable());
+            }
+            int docCount = PointValues.getDocCount(reader, field);
+            byte[] min = PointValues.getMinPackedValue(reader, field);
+            byte[] max = PointValues.getMaxPackedValue(reader, field);
             return new FieldStats.Date(reader.maxDoc(),docCount, -1L, size,
                 isSearchable(), isAggregatable(),
                 dateTimeFormatter(), LongPoint.decodeDimension(min, 0), LongPoint.decodeDimension(max, 0));
@@ -388,24 +329,15 @@ public FieldStats.Date stats(IndexReader reader) throws IOException {
 
         @Override
         public Relation isFieldWithinQuery(IndexReader reader,
-                Object from, Object to,
-                boolean includeLower, boolean includeUpper,
-                DateTimeZone timeZone, DateMathParser dateParser) throws IOException {
+                Object from, Object to, boolean includeLower, boolean includeUpper,
+                DateTimeZone timeZone, DateMathParser dateParser, QueryRewriteContext context) throws IOException {
             if (dateParser == null) {
                 dateParser = this.dateMathParser;
             }
 
-            if (XPointValues.size(reader, name()) == 0) {
-                // no points, so nothing matches
-                return Relation.DISJOINT;
-            }
-
-            long minValue = LongPoint.decodeDimension(XPointValues.getMinPackedValue(reader, name()), 0);
-            long maxValue = LongPoint.decodeDimension(XPointValues.getMaxPackedValue(reader, name()), 0);
-
             long fromInclusive = Long.MIN_VALUE;
             if (from != null) {
-                fromInclusive = parseToMilliseconds(from, !includeLower, timeZone, dateParser);
+                fromInclusive = parseToMilliseconds(from, !includeLower, timeZone, dateParser, context);
                 if (includeLower == false) {
                     if (fromInclusive == Long.MAX_VALUE) {
                         return Relation.DISJOINT;
@@ -416,7 +348,7 @@ public Relation isFieldWithinQuery(IndexReader reader,
 
             long toInclusive = Long.MAX_VALUE;
             if (to != null) {
-                toInclusive = parseToMilliseconds(to, includeUpper, timeZone, dateParser);
+                toInclusive = parseToMilliseconds(to, includeUpper, timeZone, dateParser, context);
                 if (includeUpper == false) {
                     if (toInclusive == Long.MIN_VALUE) {
                         return Relation.DISJOINT;
@@ -425,6 +357,17 @@ public Relation isFieldWithinQuery(IndexReader reader,
                 }
             }
 
+            // This check needs to be done after fromInclusive and toInclusive
+            // are resolved so we can throw an exception if they are invalid
+            // even if there are no points in the shard
+            if (PointValues.size(reader, name()) == 0) {
+                // no points, so nothing matches
+                return Relation.DISJOINT;
+            }
+
+            long minValue = LongPoint.decodeDimension(PointValues.getMinPackedValue(reader, name()), 0);
+            long maxValue = LongPoint.decodeDimension(PointValues.getMaxPackedValue(reader, name()), 0);
+
             if (minValue >= fromInclusive && maxValue <= toInclusive) {
                 return Relation.WITHIN;
             } else if (maxValue < fromInclusive || minValue > toInclusive) {
@@ -441,7 +384,7 @@ public IndexFieldData.Builder fielddataBuilder() {
         }
 
         @Override
-        public Object valueForSearch(Object value) {
+        public Object valueForDisplay(Object value) {
             Long val = (Long) value;
             if (val == null) {
                 return null;
@@ -496,7 +439,7 @@ protected DateFieldMapper clone() {
     }
 
     @Override
-    protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
+    protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
         String dateAsString;
         if (context.externalValueSet()) {
             Object dateAsObject = context.externalValue();
@@ -545,8 +488,8 @@ protected void parseCreateField(ParseContext context, List<Field> fields) throws
 
     @Override
     protected void doMerge(Mapper mergeWith, boolean updateAllTypes) {
+        final DateFieldMapper other = (DateFieldMapper) mergeWith;
         super.doMerge(mergeWith, updateAllTypes);
-        DateFieldMapper other = (DateFieldMapper) mergeWith;
         this.includeInAll = other.includeInAll;
         if (other.ignoreMalformed.explicit()) {
             this.ignoreMalformed = other.ignoreMalformed;
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentFieldMappers.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentFieldMappers.java
index 57f2ff40530e5..a3acaa9534e1b 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentFieldMappers.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentFieldMappers.java
@@ -78,8 +78,6 @@ public Collection<String> simpleMatchToFullName(String pattern) {
         for (FieldMapper fieldMapper : this) {
             if (Regex.simpleMatch(pattern, fieldMapper.fieldType().name())) {
                 fields.add(fieldMapper.fieldType().name());
-            } else if (Regex.simpleMatch(pattern, fieldMapper.fieldType().name())) {
-                fields.add(fieldMapper.fieldType().name());
             }
         }
         return fields;
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java
index a4d1a0c5e4b96..6d2a15762c88f 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java
@@ -32,8 +32,9 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.analysis.AnalysisService;
+import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.mapper.MetadataFieldMapper.TypeParser;
+import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
@@ -77,7 +78,8 @@ public Builder(RootObjectMapper.Builder builder, MapperService mapperService) {
                 final MetadataFieldMapper metadataMapper;
                 if (existingMetadataMapper == null) {
                     final TypeParser parser = entry.getValue();
-                    metadataMapper = parser.getDefault(indexSettings, mapperService.fullName(name), builder.name());
+                    metadataMapper = parser.getDefault(mapperService.fullName(name),
+                            mapperService.documentMapperParser().parserContext(builder.name()));
                 } else {
                     metadataMapper = existingMetadataMapper;
                 }
@@ -147,11 +149,11 @@ public DocumentMapper(MapperService mapperService, Mapping mapping) {
         }
         MapperUtils.collect(this.mapping.root, newObjectMappers, newFieldMappers);
 
-        final AnalysisService analysisService = mapperService.analysisService();
+        final IndexAnalyzers indexAnalyzers = mapperService.getIndexAnalyzers();
         this.fieldMappers = new DocumentFieldMappers(newFieldMappers,
-                analysisService.defaultIndexAnalyzer(),
-                analysisService.defaultSearchAnalyzer(),
-                analysisService.defaultSearchQuoteAnalyzer());
+                indexAnalyzers.getDefaultIndexAnalyzer(),
+                indexAnalyzers.getDefaultSearchAnalyzer(),
+                indexAnalyzers.getDefaultSearchQuoteAnalyzer());
 
         Map<String, ObjectMapper> builder = new HashMap<>();
         for (ObjectMapper objectMapper : newObjectMappers) {
@@ -250,8 +252,8 @@ public IndexFieldMapper IndexFieldMapper() {
         return metadataMapper(IndexFieldMapper.class);
     }
 
-    public Query typeFilter() {
-        return typeMapper().fieldType().termQuery(type, null);
+    public Query typeFilter(QueryShardContext context) {
+        return typeMapper().fieldType().termQuery(type, context);
     }
 
     public boolean hasNestedObjects() {
@@ -266,8 +268,9 @@ public Map<String, ObjectMapper> objectMappers() {
         return this.objectMappers;
     }
 
+    // TODO this method looks like it is only used in tests...
     public ParsedDocument parse(String index, String type, String id, BytesReference source) throws MapperParsingException {
-        return parse(SourceToParse.source(index, type, id, source));
+        return parse(SourceToParse.source(index, type, id, source, XContentType.JSON));
     }
 
     public ParsedDocument parse(SourceToParse source) throws MapperParsingException {
@@ -309,21 +312,6 @@ public ObjectMapper findNestedObjectMapper(int nestedDocId, SearchContext sc, Le
         return nestedObjectMapper;
     }
 
-    /**
-     * Returns the parent {@link ObjectMapper} instance of the specified object mapper or <code>null</code> if there
-     * isn't any.
-     */
-    // TODO: We should add: ObjectMapper#getParentObjectMapper()
-    public ObjectMapper findParentObjectMapper(ObjectMapper objectMapper) {
-        int indexOfLastDot = objectMapper.fullPath().lastIndexOf('.');
-        if (indexOfLastDot != -1) {
-            String parentNestObjectPath = objectMapper.fullPath().substring(0, indexOfLastDot);
-            return objectMappers().get(parentNestObjectPath);
-        } else {
-            return null;
-        }
-    }
-
     public boolean isParent(String type) {
         return mapperService.getParentTypes().contains(type);
     }
@@ -338,6 +326,11 @@ public DocumentMapper merge(Mapping mapping, boolean updateAllTypes) {
      */
     public DocumentMapper updateFieldType(Map<String, MappedFieldType> fullNameToFieldType) {
         Mapping updated = this.mapping.updateFieldType(fullNameToFieldType);
+        if (updated == this.mapping) {
+            // no change
+            return this;
+        }
+        assert updated == updated.updateFieldType(fullNameToFieldType) : "updateFieldType operation is not idempotent";
         return new DocumentMapper(mapperService, updated);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java
index f336fbb01ac30..be25775c1353e 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java
@@ -21,15 +21,14 @@
 
 import org.elasticsearch.Version;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.compress.CompressedXContent;
-import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.analysis.AnalysisService;
+import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.indices.mapper.MapperRegistry;
@@ -44,24 +43,24 @@
 public class DocumentMapperParser {
 
     final MapperService mapperService;
-    final AnalysisService analysisService;
+    final IndexAnalyzers indexAnalyzers;
+    private final NamedXContentRegistry xContentRegistry;
     private final SimilarityService similarityService;
     private final Supplier<QueryShardContext> queryShardContextSupplier;
 
     private final RootObjectMapper.TypeParser rootObjectTypeParser = new RootObjectMapper.TypeParser();
 
     private final Version indexVersionCreated;
-    private final ParseFieldMatcher parseFieldMatcher;
 
     private final Map<String, Mapper.TypeParser> typeParsers;
     private final Map<String, MetadataFieldMapper.TypeParser> rootTypeParsers;
 
-    public DocumentMapperParser(IndexSettings indexSettings, MapperService mapperService, AnalysisService analysisService,
-                                SimilarityService similarityService, MapperRegistry mapperRegistry,
+    public DocumentMapperParser(IndexSettings indexSettings, MapperService mapperService, IndexAnalyzers indexAnalyzers,
+                                NamedXContentRegistry xContentRegistry, SimilarityService similarityService, MapperRegistry mapperRegistry,
                                 Supplier<QueryShardContext> queryShardContextSupplier) {
-        this.parseFieldMatcher = new ParseFieldMatcher(indexSettings.getSettings());
         this.mapperService = mapperService;
-        this.analysisService = analysisService;
+        this.indexAnalyzers = indexAnalyzers;
+        this.xContentRegistry = xContentRegistry;
         this.similarityService = similarityService;
         this.queryShardContextSupplier = queryShardContextSupplier;
         this.typeParsers = mapperRegistry.getMapperParsers();
@@ -70,7 +69,8 @@ public DocumentMapperParser(IndexSettings indexSettings, MapperService mapperSer
     }
 
     public Mapper.TypeParser.ParserContext parserContext(String type) {
-        return new Mapper.TypeParser.ParserContext(type, analysisService, similarityService::getSimilarity, mapperService, typeParsers::get, indexVersionCreated, parseFieldMatcher, queryShardContextSupplier.get());
+        return new Mapper.TypeParser.ParserContext(type, indexAnalyzers, similarityService::getSimilarity, mapperService,
+                typeParsers::get, indexVersionCreated, queryShardContextSupplier);
     }
 
     public DocumentMapper parse(@Nullable String type, CompressedXContent source) throws MapperParsingException {
@@ -80,7 +80,7 @@ public DocumentMapper parse(@Nullable String type, CompressedXContent source) th
     public DocumentMapper parse(@Nullable String type, CompressedXContent source, String defaultSource) throws MapperParsingException {
         Map<String, Object> mapping = null;
         if (source != null) {
-            Map<String, Object> root = XContentHelper.convertToMap(source.compressedReference(), true).v2();
+            Map<String, Object> root = XContentHelper.convertToMap(source.compressedReference(), true, XContentType.JSON).v2();
             Tuple<String, Map<String, Object>> t = extractMapping(type, root);
             type = t.v1();
             mapping = t.v2();
@@ -107,7 +107,8 @@ private DocumentMapper parse(String type, Map<String, Object> mapping, String de
 
         Mapper.TypeParser.ParserContext parserContext = parserContext(type);
         // parse RootObjectMapper
-        DocumentMapper.Builder docBuilder = new DocumentMapper.Builder((RootObjectMapper.Builder) rootObjectTypeParser.parse(type, mapping, parserContext), mapperService);
+        DocumentMapper.Builder docBuilder = new DocumentMapper.Builder(
+                (RootObjectMapper.Builder) rootObjectTypeParser.parse(type, mapping, parserContext), mapperService);
         Iterator<Map.Entry<String, Object>> iterator = mapping.entrySet().iterator();
         // parse DocumentMapper
         while(iterator.hasNext()) {
@@ -118,6 +119,9 @@ private DocumentMapper parse(String type, Map<String, Object> mapping, String de
             MetadataFieldMapper.TypeParser typeParser = rootTypeParsers.get(fieldName);
             if (typeParser != null) {
                 iterator.remove();
+                if (false == fieldNode instanceof Map) {
+                    throw new IllegalArgumentException("[_parent] must be an object containing [type]");
+                }
                 Map<String, Object> fieldNodeMap = (Map<String, Object>) fieldNode;
                 docBuilder.put(typeParser.parse(fieldName, fieldNodeMap, parserContext));
                 fieldNodeMap.remove("type");
@@ -138,7 +142,8 @@ private DocumentMapper parse(String type, Map<String, Object> mapping, String de
     }
 
     public static void checkNoRemainingFields(String fieldName, Map<?, ?> fieldNodeMap, Version indexVersionCreated) {
-        checkNoRemainingFields(fieldNodeMap, indexVersionCreated, "Mapping definition for [" + fieldName + "] has unsupported parameters: ");
+        checkNoRemainingFields(fieldNodeMap, indexVersionCreated,
+                "Mapping definition for [" + fieldName + "] has unsupported parameters: ");
     }
 
     public static void checkNoRemainingFields(Map<?, ?> fieldNodeMap, Version indexVersionCreated, String message) {
@@ -157,7 +162,7 @@ private static String getRemainingFields(Map<?, ?> map) {
 
     private Tuple<String, Map<String, Object>> extractMapping(String type, String source) throws MapperParsingException {
         Map<String, Object> root;
-        try (XContentParser parser = XContentFactory.xContent(source).createParser(source)) {
+        try (XContentParser parser = XContentType.JSON.xContent().createParser(xContentRegistry, source)) {
             root = parser.mapOrdered();
         } catch (Exception e) {
             throw new MapperParsingException("failed to parse mapping definition", e);
@@ -180,4 +185,8 @@ private Tuple<String, Map<String, Object>> extractMapping(String type, Map<Strin
         }
         return mapping;
     }
+
+    NamedXContentRegistry getXContentRegistry() {
+        return xContentRegistry;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
index 60ca0c22cca6d..04f4993ddad78 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
@@ -48,7 +48,7 @@ final class DocumentParser {
     private final DocumentMapperParser docMapperParser;
     private final DocumentMapper docMapper;
 
-    public DocumentParser(IndexSettings indexSettings, DocumentMapperParser docMapperParser, DocumentMapper docMapper) {
+    DocumentParser(IndexSettings indexSettings, DocumentMapperParser docMapperParser, DocumentMapper docMapper) {
         this.indexSettings = indexSettings;
         this.docMapperParser = docMapperParser;
         this.docMapper = docMapper;
@@ -59,7 +59,7 @@ ParsedDocument parseDocument(SourceToParse source) throws MapperParsingException
 
         final Mapping mapping = docMapper.mapping();
         final ParseContext.InternalParseContext context;
-        try (XContentParser parser = XContentHelper.createParser(source.source())) {
+        try (XContentParser parser = XContentHelper.createParser(docMapperParser.getXContentRegistry(), source.source())) {
             context = new ParseContext.InternalParseContext(indexSettings.getSettings(),
                     docMapperParser, docMapper, source, parser);
             validateStart(parser);
@@ -156,6 +156,7 @@ private static ParsedDocument parsedDocument(SourceToParse source, ParseContext.
             source.ttl(),
             context.docs(),
             context.sourceToParse().source(),
+            context.sourceToParse().getXContentType(),
             update
         ).parent(source.parent());
     }
@@ -175,6 +176,17 @@ private static MapperParsingException wrapInMapperParsingException(SourceToParse
         return new MapperParsingException("failed to parse", e);
     }
 
+    private static String[] splitAndValidatePath(String fullFieldPath) {
+        String[] parts = fullFieldPath.split("\\.");
+        for (String part : parts) {
+            if (Strings.hasText(part) == false) {
+                throw new IllegalArgumentException(
+                    "object field starting or ending with a [.] makes object resolution ambiguous: [" + fullFieldPath + "]");
+            }
+        }
+        return parts;
+    }
+
     /** Creates a Mapping containing any dynamically added fields, or returns null if there were no dynamic mappings. */
     static Mapping createDynamicUpdate(Mapping mapping, DocumentMapper docMapper, List<Mapper> dynamicMappers) {
         if (dynamicMappers.isEmpty()) {
@@ -187,7 +199,7 @@ static Mapping createDynamicUpdate(Mapping mapping, DocumentMapper docMapper, Li
         Iterator<Mapper> dynamicMapperItr = dynamicMappers.iterator();
         List<ObjectMapper> parentMappers = new ArrayList<>();
         Mapper firstUpdate = dynamicMapperItr.next();
-        parentMappers.add(createUpdate(mapping.root(), firstUpdate.name().split("\\."), 0, firstUpdate));
+        parentMappers.add(createUpdate(mapping.root(), splitAndValidatePath(firstUpdate.name()), 0, firstUpdate));
         Mapper previousMapper = null;
         while (dynamicMapperItr.hasNext()) {
             Mapper newMapper = dynamicMapperItr.next();
@@ -199,7 +211,7 @@ static Mapping createDynamicUpdate(Mapping mapping, DocumentMapper docMapper, Li
                 continue;
             }
             previousMapper = newMapper;
-            String[] nameParts = newMapper.name().split("\\.");
+            String[] nameParts = splitAndValidatePath(newMapper.name());
 
             // We first need the stack to only contain mappers in common with the previously processed mapper
             // For example, if the first mapper processed was a.b.c, and we now have a.d, the stack will contain
@@ -414,15 +426,33 @@ private static ParseContext nestedContext(ParseContext context, ObjectMapper map
         context = context.createNestedContext(mapper.fullPath());
         ParseContext.Document nestedDoc = context.doc();
         ParseContext.Document parentDoc = nestedDoc.getParent();
-        // pre add the uid field if possible (id was already provided)
-        IndexableField uidField = parentDoc.getField(UidFieldMapper.NAME);
-        if (uidField != null) {
-            // we don't need to add it as a full uid field in nested docs, since we don't need versioning
-            // we also rely on this for UidField#loadVersion
 
-            // this is a deeply nested field
-            nestedDoc.add(new Field(UidFieldMapper.NAME, uidField.stringValue(), UidFieldMapper.Defaults.NESTED_FIELD_TYPE));
+        // We need to add the uid or id to this nested Lucene document too,
+        // If we do not do this then when a document gets deleted only the root Lucene document gets deleted and
+        // not the nested Lucene documents! Besides the fact that we would have zombie Lucene documents, the ordering of
+        // documents inside the Lucene index (document blocks) will be incorrect, as nested documents of different root
+        // documents are then aligned with other root documents. This will lead tothe nested query, sorting, aggregations
+        // and inner hits to fail or yield incorrect results.
+        if (context.mapperService().getIndexSettings().isSingleType()) {
+            IndexableField idField = parentDoc.getField(IdFieldMapper.NAME);
+            if (idField != null) {
+                // We just need to store the id as indexed field, so that IndexWriter#deleteDocuments(term) can then
+                // delete it when the root document is deleted too.
+                nestedDoc.add(new Field(IdFieldMapper.NAME, idField.stringValue(), IdFieldMapper.Defaults.NESTED_FIELD_TYPE));
+            } else {
+                throw new IllegalStateException("The root document of a nested document should have an id field");
+            }
+        } else {
+            IndexableField uidField = parentDoc.getField(UidFieldMapper.NAME);
+            if (uidField != null) {
+                /// We just need to store the uid as indexed field, so that IndexWriter#deleteDocuments(term) can then
+                // delete it when the root document is deleted too.
+                nestedDoc.add(new Field(UidFieldMapper.NAME, uidField.stringValue(), UidFieldMapper.Defaults.NESTED_FIELD_TYPE));
+            } else {
+                throw new IllegalStateException("The root document of a nested document should have an uid field");
+            }
         }
+
         // the type of the nested doc starts with __, so we can identify that its a nested one in filters
         // note, we don't prefix it with the type of the doc since it allows us to execute a nested query
         // across types (for example, with similar nested objects)
@@ -445,10 +475,9 @@ private static void parseObjectOrField(ParseContext context, Mapper mapper) thro
         }
     }
 
-    private static ObjectMapper parseObject(final ParseContext context, ObjectMapper mapper, String currentFieldName) throws IOException {
+    private static void parseObject(final ParseContext context, ObjectMapper mapper, String currentFieldName) throws IOException {
         assert currentFieldName != null;
 
-        ObjectMapper update = null;
         Mapper objectMapper = getMapper(mapper, currentFieldName);
         if (objectMapper != null) {
             context.path().add(currentFieldName);
@@ -456,7 +485,7 @@ private static ObjectMapper parseObject(final ParseContext context, ObjectMapper
             context.path().remove();
         } else {
 
-            final String[] paths = currentFieldName.split("\\.");
+            final String[] paths = splitAndValidatePath(currentFieldName);
             currentFieldName = paths[paths.length - 1];
             Tuple<Integer, ObjectMapper> parentMapperTuple = getDynamicParentMapper(context, paths, mapper);
             ObjectMapper parentMapper = parentMapperTuple.v2();
@@ -482,8 +511,6 @@ private static ObjectMapper parseObject(final ParseContext context, ObjectMapper
                 context.path().remove();
             }
         }
-
-        return update;
     }
 
     private static void parseArray(ParseContext context, ObjectMapper parentMapper, String lastFieldName) throws IOException {
@@ -500,7 +527,7 @@ private static void parseArray(ParseContext context, ObjectMapper parentMapper,
             }
         } else {
 
-            final String[] paths = arrayFieldName.split("\\.");
+            final String[] paths = splitAndValidatePath(arrayFieldName);
             arrayFieldName = paths[paths.length - 1];
             lastFieldName = arrayFieldName;
             Tuple<Integer, ObjectMapper> parentMapperTuple = getDynamicParentMapper(context, paths, parentMapper);
@@ -564,7 +591,7 @@ private static void parseValue(final ParseContext context, ObjectMapper parentMa
             parseObjectOrField(context, mapper);
         } else {
 
-            final String[] paths = currentFieldName.split("\\.");
+            final String[] paths = splitAndValidatePath(currentFieldName);
             currentFieldName = paths[paths.length - 1];
             Tuple<Integer, ObjectMapper> parentMapperTuple = getDynamicParentMapper(context, paths, parentMapper);
             parentMapper = parentMapperTuple.v2();
@@ -679,6 +706,12 @@ private static Mapper.Builder<?,?> createBuilderFromDynamicValue(final ParseCont
                             if (builder == null) {
                                 builder = newDateBuilder(currentFieldName, dateTimeFormatter, Version.indexCreated(context.indexSettings()));
                             }
+                            if (builder instanceof DateFieldMapper.Builder) {
+                                DateFieldMapper.Builder dateBuilder = (DateFieldMapper.Builder) builder;
+                                if (dateBuilder.isDateTimeFormatterSet() == false) {
+                                    dateBuilder.dateTimeFormatter(dateTimeFormatter);
+                                }
+                            }
                             return builder;
                         } catch (Exception e) {
                             // failure to parse this, continue
@@ -818,7 +851,7 @@ private static void parseCopy(String field, ParseContext context) throws IOExcep
             // The path of the dest field might be completely different from the current one so we need to reset it
             context = context.overridePath(new ContentPath(0));
 
-            final String[] paths = field.split("\\.");
+            final String[] paths = splitAndValidatePath(field);
             final String fieldName = paths[paths.length-1];
             Tuple<Integer, ObjectMapper> parentMapperTuple = getDynamicParentMapper(context, paths, null);
             ObjectMapper mapper = parentMapperTuple.v2();
@@ -858,7 +891,8 @@ private static Tuple<Integer, ObjectMapper> getDynamicParentMapper(ParseContext
                             Mapper.BuilderContext builderContext = new Mapper.BuilderContext(context.indexSettings(), context.path());
                             mapper = (ObjectMapper) builder.build(builderContext);
                             if (mapper.nested() != ObjectMapper.Nested.NO) {
-                                throw new MapperParsingException("It is forbidden to create dynamic nested objects ([" + context.path().pathAsText(paths[i]) + "]) through `copy_to`");
+                                throw new MapperParsingException("It is forbidden to create dynamic nested objects ([" + context.path().pathAsText(paths[i])
+                                        + "]) through `copy_to` or dots in field names");
                             }
                             context.addDynamicMapper(mapper);
                             break;
@@ -901,13 +935,18 @@ private static ObjectMapper.Dynamic dynamicOrDefault(ObjectMapper parentMapper,
 
     // looks up a child mapper, but takes into account field names that expand to objects
     static Mapper getMapper(ObjectMapper objectMapper, String fieldName) {
-        String[] subfields = fieldName.split("\\.");
+        String[] subfields = splitAndValidatePath(fieldName);
         for (int i = 0; i < subfields.length - 1; ++i) {
             Mapper mapper = objectMapper.getMapper(subfields[i]);
             if (mapper == null || (mapper instanceof ObjectMapper) == false) {
                 return null;
             }
             objectMapper = (ObjectMapper)mapper;
+            if (objectMapper.nested().isNested()) {
+                throw new MapperParsingException("Cannot add a value for field ["
+                        + fieldName + "] since one of the intermediate objects is mapped as a nested object: ["
+                        + mapper.name() + "]");
+            }
         }
         return objectMapper.getMapper(subfields[subfields.length - 1]);
     }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java b/core/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java
index 08620ed8c4535..bf7193848a070 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java
@@ -41,7 +41,7 @@ public class DynamicTemplate implements ToXContent {
 
     private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(DynamicTemplate.class));
 
-    public static enum MatchType {
+    public enum MatchType {
         SIMPLE {
             @Override
             public boolean matches(String pattern, String value) {
@@ -155,7 +155,7 @@ public static XContentFieldType fromString(String value) {
                     return v;
                 }
             }
-            throw new IllegalArgumentException("No xcontent type matched on [" + value + "], possible values are "
+            throw new IllegalArgumentException("No field type matched on [" + value + "], possible values are "
                     + Arrays.toString(values()));
         }
 
@@ -208,12 +208,8 @@ public static DynamicTemplate parse(String name, Map<String, Object> conf,
             try {
                 xcontentFieldType = XContentFieldType.fromString(matchMappingType);
             } catch (IllegalArgumentException e) {
-                // TODO: do this in 6.0
-                /*if (indexVersionCreated.onOrAfter(Version.V_6_0_0)) {
-                    throw e;
-                }*/
-
-                DEPRECATION_LOGGER.deprecated("Ignoring unrecognized match_mapping_type: [" + matchMappingType + "]");
+                DEPRECATION_LOGGER.deprecated("match_mapping_type [" + matchMappingType + "] is invalid and will be ignored: "
+                        + e.getMessage());
                 // this template is on an unknown type so it will never match anything
                 // null indicates that the template should be ignored
                 return null;
@@ -321,7 +317,7 @@ private Map<String, Object> processMap(Map<String, Object> map, String name, Str
     }
 
     private List processList(List list, String name, String dynamicType) {
-        List processedList = new ArrayList();
+        List processedList = new ArrayList(list.size());
         for (Object value : list) {
             if (value instanceof Map) {
                 value = processMap((Map<String, Object>) value, name, dynamicType);
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
index 4773d5da46847..0ed093b3a884f 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
@@ -24,6 +24,7 @@
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexableField;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.lucene.Lucene;
@@ -236,7 +237,7 @@ protected void setupFieldType(BuilderContext context) {
         }
     }
 
-    private final Version indexCreatedVersion;
+    protected final Version indexCreatedVersion;
     protected MappedFieldType fieldType;
     protected final MappedFieldType defaultFieldType;
     protected MultiFields multiFields;
@@ -246,7 +247,7 @@ protected FieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldT
         super(simpleName);
         assert indexSettings != null;
         this.indexCreatedVersion = Version.indexCreated(indexSettings);
-        if (indexCreatedVersion.onOrAfter(Version.V_5_0_0_alpha6)) {
+        if (indexCreatedVersion.onOrAfter(Version.V_5_0_0_beta1)) {
             if (simpleName.isEmpty()) {
                 throw new IllegalArgumentException("name cannot be empty string");
             }
@@ -281,15 +282,15 @@ public CopyTo copyTo() {
      * mappings were not modified.
      */
     public Mapper parse(ParseContext context) throws IOException {
-        final List<Field> fields = new ArrayList<>(2);
+        final List<IndexableField> fields = new ArrayList<>(2);
         try {
             parseCreateField(context, fields);
-            for (Field field : fields) {
+            for (IndexableField field : fields) {
                 if (!customBoost()
                         // don't set boosts eg. on dv fields
                         && field.fieldType().indexOptions() != IndexOptions.NONE
                         && indexCreatedVersion.before(Version.V_5_0_0_alpha1)) {
-                    field.setBoost(fieldType().boost());
+                    ((Field)(field)).setBoost(fieldType().boost());
                 }
                 context.doc().add(field);
             }
@@ -303,7 +304,7 @@ public Mapper parse(ParseContext context) throws IOException {
     /**
      * Parse the field value and populate <code>fields</code>.
      */
-    protected abstract void parseCreateField(ParseContext context, List<Field> fields) throws IOException;
+    protected abstract void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException;
 
     /**
      * Derived classes can override it to specify that boost value is set by derived classes.
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java
index 7343963f0996a..42c177b34d23a 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java
@@ -30,13 +30,12 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 
-import static org.elasticsearch.common.xcontent.support.XContentMapValues.lenientNodeBooleanValue;
-
 /**
  * A mapper that indexes the field names of a document under <code>_field_names</code>. This mapper is typically useful in order
  * to have fast <code>exists</code> and <code>missing</code> queries/filters.
@@ -98,7 +97,7 @@ public FieldNamesFieldMapper build(BuilderContext context) {
 
     public static class TypeParser implements MetadataFieldMapper.TypeParser {
         @Override
-        public MetadataFieldMapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
+        public MetadataFieldMapper.Builder<?,?> parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
             Builder builder = new Builder(parserContext.mapperService().fullName(NAME));
 
             for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
@@ -106,7 +105,7 @@ public MetadataFieldMapper.Builder parse(String name, Map<String, Object> node,
                 String fieldName = entry.getKey();
                 Object fieldNode = entry.getValue();
                 if (fieldName.equals("enabled")) {
-                    builder.enabled(lenientNodeBooleanValue(fieldNode));
+                    builder.enabled(TypeParsers.nodeBooleanValue(name, "enabled", fieldNode));
                     iterator.remove();
                 }
             }
@@ -114,8 +113,14 @@ public MetadataFieldMapper.Builder parse(String name, Map<String, Object> node,
         }
 
         @Override
-        public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName) {
-            return new FieldNamesFieldMapper(indexSettings, fieldType);
+        public MetadataFieldMapper getDefault(MappedFieldType fieldType, ParserContext context) {
+            final Settings indexSettings = context.mapperService().getIndexSettings().getSettings();
+            if (fieldType != null) {
+                return new FieldNamesFieldMapper(indexSettings, fieldType);
+            } else {
+                return parse(NAME, Collections.emptyMap(), context)
+                        .build(new BuilderContext(indexSettings, new ContentPath(1)));
+            }
         }
     }
 
@@ -183,7 +188,7 @@ public Query termQuery(Object value, QueryShardContext context) {
     }
 
     private FieldNamesFieldMapper(Settings indexSettings, MappedFieldType existing) {
-        this(existing == null ? Defaults.FIELD_TYPE.clone() : existing.clone(), indexSettings);
+        this(existing.clone(), indexSettings);
     }
 
     private FieldNamesFieldMapper(MappedFieldType fieldType, Settings indexSettings) {
@@ -238,7 +243,7 @@ public String next() {
                     }
 
                     @Override
-                    public final void remove() {
+                    public void remove() {
                         throw new UnsupportedOperationException();
                     }
 
@@ -248,7 +253,7 @@ public final void remove() {
     }
 
     @Override
-    protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
+    protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
         if (fieldType().isEnabled() == false) {
             return;
         }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java b/core/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java
index 5f6fddf09ef68..fee41e43f2a3c 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java
@@ -43,7 +43,7 @@ class FieldTypeLookup implements Iterable<MappedFieldType> {
     final CopyOnWriteHashMap<String, Set<String>> fullNameToTypes;
 
     /** Create a new empty instance. */
-    public FieldTypeLookup() {
+    FieldTypeLookup() {
         fullNameToFieldType = new CopyOnWriteHashMap<>();
         fullNameToTypes = new CopyOnWriteHashMap<>();
     }
@@ -93,7 +93,7 @@ public FieldTypeLookup copyAndAddAll(String type, Collection<FieldMapper> fieldM
             // is the update even legal?
             checkCompatibility(type, fieldMapper, updateAllTypes);
 
-            if (fieldType != fullNameFieldType) {
+            if (fieldType.equals(fullNameFieldType) == false) {
                 fullName = fullName.copyAndPut(fieldType.name(), fieldMapper.fieldType());
             }
 
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java
index c27ddc1811b64..8111e5e02a68f 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java
@@ -48,7 +48,7 @@ public class GeoPointFieldMapper extends BaseGeoPointFieldMapper  {
 
     public static class Defaults extends BaseGeoPointFieldMapper.Defaults {
 
-        public static final GeoPointFieldType FIELD_TYPE = new GeoPointFieldType();
+        public static final GeoPointFieldType FIELD_TYPE = new LegacyGeoPointFieldType();
 
         static {
             FIELD_TYPE.setIndexOptions(IndexOptions.DOCS);
@@ -79,6 +79,7 @@ public GeoPointFieldMapper build(BuilderContext context, String simpleName, Mapp
             if (context.indexCreatedVersion().before(Version.V_2_3_0)) {
                 fieldType.setNumericPrecisionStep(GeoPointField.PRECISION_STEP);
                 fieldType.setNumericType(FieldType.LegacyNumericType.LONG);
+                ((LegacyGeoPointFieldType)fieldType).numericEncoded = true;
             }
             setupFieldType(context);
             return new GeoPointFieldMapper(simpleName, fieldType, defaultFieldType, indexSettings, latMapper, lonMapper,
@@ -127,4 +128,9 @@ protected void parse(ParseContext context, GeoPoint point, String geoHash) throw
         }
         super.parse(context, point, geoHash);
     }
+
+    @Override
+    public LegacyGeoPointFieldType fieldType() {
+        return (LegacyGeoPointFieldType) super.fieldType();
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java
index 9c90dd44dbc1e..355c515bdd0e2 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java
@@ -18,11 +18,11 @@
  */
 package org.elasticsearch.index.mapper;
 
-import org.locationtech.spatial4j.shape.Point;
-import org.locationtech.spatial4j.shape.Shape;
-import org.locationtech.spatial4j.shape.jts.JtsGeometry;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.spatial.prefix.PrefixTreeStrategy;
 import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy;
@@ -32,8 +32,8 @@
 import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree;
 import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree;
 import org.elasticsearch.Version;
+import org.elasticsearch.action.fieldstats.FieldStats;
 import org.elasticsearch.common.Explicit;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.geo.GeoUtils;
 import org.elasticsearch.common.geo.SpatialStrategy;
 import org.elasticsearch.common.geo.builders.ShapeBuilder;
@@ -41,9 +41,11 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.DistanceUnit;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.query.QueryShardException;
+import org.locationtech.spatial4j.shape.Point;
+import org.locationtech.spatial4j.shape.Shape;
+import org.locationtech.spatial4j.shape.jts.JtsGeometry;
 
 import java.io.IOException;
 import java.util.Iterator;
@@ -51,9 +53,6 @@
 import java.util.Map;
 import java.util.Objects;
 
-import static org.elasticsearch.common.xcontent.support.XContentMapValues.lenientNodeBooleanValue;
-
-
 /**
  * FieldMapper for indexing {@link org.locationtech.spatial4j.shape.Shape}s.
  * <p>
@@ -182,11 +181,12 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
                     builder.fieldType().setStrategyName(fieldNode.toString());
                     iterator.remove();
                 } else if (Names.COERCE.equals(fieldName)) {
-                    builder.coerce(lenientNodeBooleanValue(fieldNode));
+                    builder.coerce(TypeParsers.nodeBooleanValue(fieldName, Names.COERCE, fieldNode));
                     iterator.remove();
                 } else if (Names.STRATEGY_POINTS_ONLY.equals(fieldName)
                     && builder.fieldType().strategyName.equals(SpatialStrategy.TERM.getStrategyName()) == false) {
-                    builder.fieldType().setPointsOnly(XContentMapValues.lenientNodeBooleanValue(fieldNode));
+                    boolean pointsOnly = TypeParsers.nodeBooleanValue(fieldName, Names.STRATEGY_POINTS_ONLY, fieldNode);
+                    builder.fieldType().setPointsOnly(pointsOnly);
                     iterator.remove();
                 }
             }
@@ -414,6 +414,20 @@ public PrefixTreeStrategy resolveStrategy(String strategyName) {
         public Query termQuery(Object value, QueryShardContext context) {
             throw new QueryShardException(context, "Geo fields do not support exact searching, use dedicated geo queries instead");
         }
+
+        @Override
+        public FieldStats stats(IndexReader reader) throws IOException {
+            int maxDoc = reader.maxDoc();
+            FieldInfo fi = org.apache.lucene.index.MultiFields.getMergedFieldInfos(reader).fieldInfo(name());
+            if (fi == null) {
+                return null;
+            }
+            /**
+             * we don't have a specific type for geo_shape so we use an empty {@link FieldStats.Text}.
+             * TODO: we should maybe support a new type that knows how to (de)encode the min/max information
+             */
+            return new FieldStats.Text(maxDoc, -1, -1, -1, isSearchable(), isAggregatable());
+        }
     }
 
     protected Explicit<Boolean> coerce;
@@ -462,7 +476,7 @@ public Mapper parse(ParseContext context) throws IOException {
     }
 
     @Override
-    protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
+    protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java
index d70a50eede99e..813a546aaed36 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java
@@ -21,25 +21,19 @@
 
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.queries.TermsQuery;
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.MultiTermQuery;
-import org.apache.lucene.search.PrefixQuery;
+import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.search.Query;
-import org.apache.lucene.search.RegexpQuery;
-import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.search.TermInSetQuery;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.common.lucene.Lucene;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.iterable.Iterables;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData;
 import org.elasticsearch.index.query.QueryShardContext;
 
 import java.io.IOException;
-import java.util.Collection;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 
@@ -58,16 +52,21 @@ public static class Defaults {
         public static final String NAME = IdFieldMapper.NAME;
 
         public static final MappedFieldType FIELD_TYPE = new IdFieldType();
+        public static final MappedFieldType NESTED_FIELD_TYPE;
 
         static {
             FIELD_TYPE.setTokenized(false);
-            FIELD_TYPE.setIndexOptions(IndexOptions.NONE);
-            FIELD_TYPE.setStored(false);
+            FIELD_TYPE.setIndexOptions(IndexOptions.DOCS);
+            FIELD_TYPE.setStored(true);
             FIELD_TYPE.setOmitNorms(true);
             FIELD_TYPE.setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
             FIELD_TYPE.setSearchAnalyzer(Lucene.KEYWORD_ANALYZER);
             FIELD_TYPE.setName(NAME);
             FIELD_TYPE.freeze();
+
+            NESTED_FIELD_TYPE = FIELD_TYPE.clone();
+            NESTED_FIELD_TYPE.setStored(false);
+            NESTED_FIELD_TYPE.freeze();
         }
     }
 
@@ -78,14 +77,15 @@ public MetadataFieldMapper.Builder parse(String name, Map<String, Object> node,
         }
 
         @Override
-        public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName) {
+        public MetadataFieldMapper getDefault(MappedFieldType fieldType, ParserContext context) {
+            final IndexSettings indexSettings = context.mapperService().getIndexSettings();
             return new IdFieldMapper(indexSettings, fieldType);
         }
     }
 
     static final class IdFieldType extends TermBasedFieldType {
 
-        public IdFieldType() {
+        IdFieldType() {
         }
 
         protected IdFieldType(IdFieldType ref) {
@@ -110,32 +110,66 @@ public boolean isSearchable() {
 
         @Override
         public Query termQuery(Object value, @Nullable QueryShardContext context) {
-            final BytesRef[] uids = Uid.createUidsForTypesAndId(context.queryTypes(), value);
-            return new TermsQuery(UidFieldMapper.NAME, uids);
+            return termsQuery(Arrays.asList(value), context);
         }
 
         @Override
-        public Query termsQuery(List values, @Nullable QueryShardContext context) {
-            return new TermsQuery(UidFieldMapper.NAME, Uid.createUidsForTypesAndIds(context.queryTypes(), values));
+        public Query termsQuery(List<?> values, @Nullable QueryShardContext context) {
+            if (indexOptions() != IndexOptions.NONE) {
+                // 6.x index, _id is indexed
+                return super.termsQuery(values, context);
+            }
+            // 5.x index, _uid is indexed
+            return new TermInSetQuery(UidFieldMapper.NAME, Uid.createUidsForTypesAndIds(context.queryTypes(), values));
+        }
+
+        @Override
+        public IndexFieldData.Builder fielddataBuilder() {
+            if (indexOptions() == IndexOptions.NONE) {
+                throw new IllegalArgumentException("Fielddata access on the _uid field is disallowed");
+            }
+            return new PagedBytesIndexFieldData.Builder(
+                    TextFieldMapper.Defaults.FIELDDATA_MIN_FREQUENCY,
+                    TextFieldMapper.Defaults.FIELDDATA_MAX_FREQUENCY,
+                    TextFieldMapper.Defaults.FIELDDATA_MIN_SEGMENT_SIZE);
+        }
+    }
+
+    static MappedFieldType defaultFieldType(IndexSettings indexSettings) {
+        MappedFieldType defaultFieldType = Defaults.FIELD_TYPE.clone();
+        if (indexSettings.isSingleType()) {
+            defaultFieldType.setIndexOptions(IndexOptions.DOCS);
+            defaultFieldType.setStored(true);
+        } else {
+            defaultFieldType.setIndexOptions(IndexOptions.NONE);
+            defaultFieldType.setStored(false);
         }
+        return defaultFieldType;
     }
 
-    private IdFieldMapper(Settings indexSettings, MappedFieldType existing) {
-        this(existing != null ? existing : Defaults.FIELD_TYPE, indexSettings);
+    private IdFieldMapper(IndexSettings indexSettings, MappedFieldType existing) {
+        this(existing == null ? defaultFieldType(indexSettings) : existing, indexSettings);
     }
 
-    private IdFieldMapper(MappedFieldType fieldType, Settings indexSettings) {
-        super(NAME, fieldType, Defaults.FIELD_TYPE, indexSettings);
+    private IdFieldMapper(MappedFieldType fieldType, IndexSettings indexSettings) {
+        super(NAME, fieldType, defaultFieldType(indexSettings), indexSettings.getSettings());
     }
 
     @Override
-    public void preParse(ParseContext context) throws IOException {}
+    public void preParse(ParseContext context) throws IOException {
+        super.parse(context);
+    }
 
     @Override
     public void postParse(ParseContext context) throws IOException {}
 
     @Override
-    protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {}
+    protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
+        if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) {
+            Field id = new Field(NAME, context.sourceToParse().id(), fieldType);
+            fields.add(id);
+        }
+    }
 
     @Override
     protected String contentType() {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java
index e1615add19ec9..51acef299104a 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java
@@ -19,8 +19,8 @@
 
 package org.elasticsearch.index.mapper;
 
-import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.Version;
@@ -30,7 +30,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.fielddata.IndexFieldData;
-import org.elasticsearch.index.fielddata.plain.IndexIndexFieldData;
+import org.elasticsearch.index.fielddata.plain.ConstantIndexFieldData;
 import org.elasticsearch.index.query.QueryShardContext;
 
 import java.io.IOException;
@@ -87,14 +87,15 @@ public MetadataFieldMapper.Builder<?,?> parse(String name, Map<String, Object> n
         }
 
         @Override
-        public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName) {
+        public MetadataFieldMapper getDefault(MappedFieldType fieldType, ParserContext context) {
+            final Settings indexSettings = context.mapperService().getIndexSettings().getSettings();
             return new IndexFieldMapper(indexSettings, fieldType);
         }
     }
 
     static final class IndexFieldType extends MappedFieldType {
 
-        public IndexFieldType() {}
+        IndexFieldType() {}
 
         protected IndexFieldType(IndexFieldType ref) {
             super(ref);
@@ -159,7 +160,7 @@ private boolean isSameIndex(Object value, String indexName) {
 
         @Override
         public IndexFieldData.Builder fielddataBuilder() {
-            return new IndexIndexFieldData.Builder();
+            return new ConstantIndexFieldData.Builder(mapperService -> mapperService.index().getName());
         }
     }
 
@@ -178,7 +179,7 @@ public void preParse(ParseContext context) throws IOException {}
     public void postParse(ParseContext context) throws IOException {}
 
     @Override
-    protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {}
+    protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {}
 
     @Override
     protected String contentType() {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java
index 69a8e06f859d7..19dcdf3fb6045 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java
@@ -19,13 +19,15 @@
 
 package org.elasticsearch.index.mapper;
 
-import org.apache.lucene.document.Field;
 import org.apache.lucene.document.InetAddressPoint;
 import org.apache.lucene.document.SortedSetDocValuesField;
 import org.apache.lucene.document.StoredField;
+import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.XPointValues;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.PointValues;
+import org.apache.lucene.index.RandomAccessOrds;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
@@ -37,6 +39,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
 import org.elasticsearch.index.mapper.LegacyNumberFieldMapper.Defaults;
 import org.elasticsearch.index.query.QueryShardContext;
@@ -45,6 +48,7 @@
 
 import java.io.IOException;
 import java.net.InetAddress;
+import java.util.Arrays;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -109,7 +113,7 @@ public Mapper.Builder<?,?> parse(String name, Map<String, Object> node, ParserCo
                     builder.nullValue(InetAddresses.forString(propNode.toString()));
                     iterator.remove();
                 } else if (propName.equals("ignore_malformed")) {
-                    builder.ignoreMalformed(TypeParsers.nodeBooleanValue("ignore_malformed", propNode, parserContext));
+                    builder.ignoreMalformed(TypeParsers.nodeBooleanValue(name, "ignore_malformed", propNode));
                     iterator.remove();
                 } else if (TypeParsers.parseMultiField(builder, name, parserContext, propName, propNode)) {
                     iterator.remove();
@@ -178,7 +182,31 @@ public Query termQuery(Object value, @Nullable QueryShardContext context) {
         }
 
         @Override
-        public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper) {
+        public Query termsQuery(List<?> values, QueryShardContext context) {
+            InetAddress[] addresses = new InetAddress[values.size()];
+            int i = 0;
+            for (Object value : values) {
+                InetAddress address;
+                if (value instanceof InetAddress) {
+                    address = (InetAddress) value;
+                } else {
+                    if (value instanceof BytesRef) {
+                        value = ((BytesRef) value).utf8ToString();
+                    }
+                    if (value.toString().contains("/")) {
+                        // the `terms` query contains some prefix queries, so we cannot create a set query
+                        // and need to fall back to a disjunction of `term` queries
+                        return super.termsQuery(values, context);
+                    }
+                    address = InetAddresses.forString(value.toString());
+                }
+                addresses[i++] = address;
+            }
+            return InetAddressPoint.newSetQuery(name(), addresses);
+        }
+
+        @Override
+        public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) {
             failIfNotIndexed();
             InetAddress lower;
             if (lowerTerm == null) {
@@ -212,26 +240,65 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower
         @Override
         public FieldStats.Ip stats(IndexReader reader) throws IOException {
             String field = name();
-            long size = XPointValues.size(reader, field);
-            if (size == 0) {
+            FieldInfo fi = org.apache.lucene.index.MultiFields.getMergedFieldInfos(reader).fieldInfo(name());
+            if (fi == null) {
                 return null;
             }
-            int docCount = XPointValues.getDocCount(reader, field);
-            byte[] min = XPointValues.getMinPackedValue(reader, field);
-            byte[] max = XPointValues.getMaxPackedValue(reader, field);
+            long size = PointValues.size(reader, field);
+            if (size == 0) {
+                return new FieldStats.Ip(reader.maxDoc(), 0, -1, -1, isSearchable(), isAggregatable());
+            }
+            int docCount = PointValues.getDocCount(reader, field);
+            byte[] min = PointValues.getMinPackedValue(reader, field);
+            byte[] max = PointValues.getMaxPackedValue(reader, field);
             return new FieldStats.Ip(reader.maxDoc(), docCount, -1L, size,
                 isSearchable(), isAggregatable(),
                 InetAddressPoint.decode(min), InetAddressPoint.decode(max));
         }
 
+        public static final class IpScriptDocValues extends ScriptDocValues<String> {
+
+            private final RandomAccessOrds values;
+
+            public IpScriptDocValues(RandomAccessOrds values) {
+                this.values = values;
+            }
+
+            @Override
+            public void setNextDocId(int docId) {
+                values.setDocument(docId);
+            }
+
+            public String getValue() {
+                if (isEmpty()) {
+                    return null;
+                } else {
+                    return get(0);
+                }
+            }
+
+            @Override
+            public String get(int index) {
+                BytesRef encoded = values.lookupOrd(values.ordAt(0));
+                InetAddress address = InetAddressPoint.decode(
+                        Arrays.copyOfRange(encoded.bytes, encoded.offset, encoded.offset + encoded.length));
+                return InetAddresses.toAddrString(address);
+            }
+
+            @Override
+            public int size() {
+                return values.cardinality();
+            }
+        }
+
         @Override
         public IndexFieldData.Builder fielddataBuilder() {
             failIfNoDocValues();
-            return new DocValuesIndexFieldData.Builder();
+            return new DocValuesIndexFieldData.Builder().scriptFunction(IpScriptDocValues::new);
         }
 
         @Override
-        public Object valueForSearch(Object value) {
+        public Object valueForDisplay(Object value) {
             if (value == null) {
                 return null;
             }
@@ -285,7 +352,7 @@ protected IpFieldMapper clone() {
     }
 
     @Override
-    protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
+    protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
         Object addressAsObject;
         if (context.externalValueSet()) {
             addressAsObject = context.externalValue();
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java
index 204e61aabe6b3..a80327a98864a 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java
@@ -19,16 +19,21 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.SortedSetDocValuesField;
 import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.Version;
+import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
 
@@ -38,6 +43,7 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 
 import static java.util.Collections.unmodifiableList;
@@ -80,6 +86,11 @@ public Builder(String name) {
             builder = this;
         }
 
+        @Override
+        public KeywordFieldType fieldType() {
+            return (KeywordFieldType) super.fieldType();
+        }
+
         public Builder ignoreAbove(int ignoreAbove) {
             if (ignoreAbove < 0) {
                 throw new IllegalArgumentException("[ignore_above] must be positive, got " + ignoreAbove);
@@ -102,6 +113,12 @@ public Builder eagerGlobalOrdinals(boolean eagerGlobalOrdinals) {
             return builder;
         }
 
+        public Builder normalizer(NamedAnalyzer normalizer) {
+            fieldType().setNormalizer(normalizer);
+            fieldType().setSearchAnalyzer(normalizer);
+            return builder;
+        }
+
         @Override
         public KeywordFieldMapper build(BuilderContext context) {
             setupFieldType(context);
@@ -113,7 +130,7 @@ public KeywordFieldMapper build(BuilderContext context) {
 
     public static class TypeParser implements Mapper.TypeParser {
         @Override
-        public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
+        public Mapper.Builder<?,?> parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
             if (parserContext.indexVersionCreated().before(Version.V_5_0_0_alpha1)) {
                 // Downgrade "keyword" to "string" in indexes created in 2.x so you can use modern syntax against old indexes
                 Set<String> unsupportedParameters = new HashSet<>(node.keySet());
@@ -134,7 +151,7 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
                     }
                     node.put("index", index);
                 }
-                
+
                 return new StringFieldMapper.TypeParser().parse(name, node, parserContext);
             }
             KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder(name);
@@ -158,6 +175,15 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
                 } else if (propName.equals("eager_global_ordinals")) {
                     builder.eagerGlobalOrdinals(XContentMapValues.nodeBooleanValue(propNode));
                     iterator.remove();
+                } else if (propName.equals("normalizer")) {
+                    if (propNode != null) {
+                        NamedAnalyzer normalizer = parserContext.getIndexAnalyzers().getNormalizer(propNode.toString());
+                        if (normalizer == null) {
+                            throw new MapperParsingException("normalizer [" + propNode.toString() + "] not found for field [" + name + "]");
+                        }
+                        builder.normalizer(normalizer);
+                    }
+                    iterator.remove();
                 }
             }
             return builder;
@@ -166,21 +192,58 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
 
     public static final class KeywordFieldType extends StringFieldType {
 
-        public KeywordFieldType() {}
+        private NamedAnalyzer normalizer = null;
+
+        public KeywordFieldType() {
+            setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
+            setSearchAnalyzer(Lucene.KEYWORD_ANALYZER);
+        }
 
         protected KeywordFieldType(KeywordFieldType ref) {
             super(ref);
+            this.normalizer = ref.normalizer;
         }
 
         public KeywordFieldType clone() {
             return new KeywordFieldType(this);
         }
 
+        @Override
+        public boolean equals(Object o) {
+            if (super.equals(o) == false) {
+                return false;
+            }
+            return Objects.equals(normalizer, ((KeywordFieldType) o).normalizer);
+        }
+
+        @Override
+        public void checkCompatibility(MappedFieldType otherFT, List<String> conflicts, boolean strict) {
+            super.checkCompatibility(otherFT, conflicts, strict);
+            KeywordFieldType other = (KeywordFieldType) otherFT;
+            if (Objects.equals(normalizer, other.normalizer) == false) {
+                conflicts.add("mapper [" + name() + "] has different [normalizer]");
+            }
+        }
+
+        @Override
+        public int hashCode() {
+            return 31 * super.hashCode() + Objects.hashCode(normalizer);
+        }
+
         @Override
         public String typeName() {
             return CONTENT_TYPE;
         }
 
+        public NamedAnalyzer normalizer() {
+            return normalizer;
+        }
+
+        public void setNormalizer(NamedAnalyzer normalizer) {
+            checkIfFrozen();
+            this.normalizer = normalizer;
+        }
+
         @Override
         public Query nullValueQuery() {
             if (nullValue() == null) {
@@ -196,7 +259,7 @@ public IndexFieldData.Builder fielddataBuilder() {
         }
 
         @Override
-        public Object valueForSearch(Object value) {
+        public Object valueForDisplay(Object value) {
             if (value == null) {
                 return null;
             }
@@ -204,13 +267,34 @@ public Object valueForSearch(Object value) {
             BytesRef binaryValue = (BytesRef) value;
             return binaryValue.utf8ToString();
         }
+
+        @Override
+        protected BytesRef indexedValueForSearch(Object value) {
+            if (searchAnalyzer() == Lucene.KEYWORD_ANALYZER) {
+                // keyword analyzer with the default attribute source which encodes terms using UTF8
+                // in that case we skip normalization, which may be slow if there many terms need to
+                // parse (eg. large terms query) since Analyzer.normalize involves things like creating
+                // attributes through reflection
+                // This if statement will be used whenever a normalizer is NOT configured
+                return super.indexedValueForSearch(value);
+            }
+
+            if (value == null) {
+                return null;
+            }
+            if (value instanceof BytesRef) {
+                value = ((BytesRef) value).utf8ToString();
+            }
+            return searchAnalyzer().normalize(name(), value.toString());
+        }
     }
 
     private Boolean includeInAll;
     private int ignoreAbove;
 
     protected KeywordFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
-                                int ignoreAbove, Boolean includeInAll, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) {
+                                int ignoreAbove, Boolean includeInAll,
+                                Settings indexSettings, MultiFields multiFields, CopyTo copyTo) {
         super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo);
         assert fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) <= 0;
         this.ignoreAbove = ignoreAbove;
@@ -229,14 +313,19 @@ protected KeywordFieldMapper clone() {
         return (KeywordFieldMapper) super.clone();
     }
 
+    @Override
+    public KeywordFieldType fieldType() {
+        return (KeywordFieldType) super.fieldType();
+    }
+
     // pkg-private for testing
     Boolean includeInAll() {
         return includeInAll;
     }
 
     @Override
-    protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
-        final String value;
+    protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
+        String value;
         if (context.externalValueSet()) {
             value = context.externalValue().toString();
         } else {
@@ -252,6 +341,27 @@ protected void parseCreateField(ParseContext context, List<Field> fields) throws
             return;
         }
 
+        final NamedAnalyzer normalizer = fieldType().normalizer();
+        if (normalizer != null) {
+            try (TokenStream ts = normalizer.tokenStream(name(), value)) {
+                final CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
+                ts.reset();
+                if (ts.incrementToken() == false) {
+                  throw new IllegalStateException("The normalization token stream is "
+                      + "expected to produce exactly 1 token, but got 0 for analyzer "
+                      + normalizer + " and input \"" + value + "\"");
+                }
+                final String newValue = termAtt.toString();
+                if (ts.incrementToken()) {
+                  throw new IllegalStateException("The normalization token stream is "
+                      + "expected to produce exactly 1 token, but got 2+ for analyzer "
+                      + normalizer + " and input \"" + value + "\"");
+                }
+                ts.end();
+                value = newValue;
+            }
+        }
+
         if (context.includeInAll(includeInAll, this)) {
             context.allEntries().addText(fieldType().name(), value, fieldType().boost());
         }
@@ -296,5 +406,11 @@ protected void doXContentBody(XContentBuilder builder, boolean includeDefaults,
         if (includeDefaults || ignoreAbove != Defaults.IGNORE_ABOVE) {
             builder.field("ignore_above", ignoreAbove);
         }
+
+        if (fieldType().normalizer() != null) {
+            builder.field("normalizer", fieldType().normalizer().name());
+        } else if (includeDefaults) {
+            builder.nullField("normalizer");
+        }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/LatLonPointFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/LatLonPointFieldMapper.java
new file mode 100644
index 0000000000000..df63992ba8886
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/mapper/LatLonPointFieldMapper.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.index.mapper;
+
+import org.apache.lucene.document.LatLonDocValuesField;
+import org.apache.lucene.document.LatLonPoint;
+import org.apache.lucene.document.StoredField;
+import org.apache.lucene.geo.GeoEncodingUtils;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.PointValues;
+import org.apache.lucene.search.Query;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.fieldstats.FieldStats;
+import org.elasticsearch.common.Explicit;
+import org.elasticsearch.common.geo.GeoPoint;
+import org.elasticsearch.common.geo.GeoUtils;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.plain.AbstractLatLonPointDVIndexFieldData;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.query.QueryShardException;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Map;
+
+/**
+ * Field Mapper for geo_point types.
+ *
+ * Uses lucene 6 LatLonPoint encoding
+ */
+public class LatLonPointFieldMapper extends BaseGeoPointFieldMapper {
+    public static final String CONTENT_TYPE = "geo_point";
+    // TODO replace this once 5.0.0-beta1 is released
+    public static final Version LAT_LON_FIELD_VERSION = Version.fromString("5.0.0-beta1");
+
+    public static class Defaults extends BaseGeoPointFieldMapper.Defaults {
+        public static final LatLonPointFieldType FIELD_TYPE = new LatLonPointFieldType();
+
+        static {
+            FIELD_TYPE.setTokenized(false);
+            FIELD_TYPE.setHasDocValues(true);
+            FIELD_TYPE.setDimensions(2, Integer.BYTES);
+            FIELD_TYPE.freeze();
+        }
+    }
+
+    public static class Builder extends BaseGeoPointFieldMapper.Builder<Builder, LatLonPointFieldMapper> {
+        public Builder(String name) {
+            super(name, Defaults.FIELD_TYPE);
+        }
+
+        @Override
+        public LatLonPointFieldMapper build(BuilderContext context, String simpleName, MappedFieldType fieldType,
+                                            MappedFieldType defaultFieldType, Settings indexSettings,
+                                            FieldMapper latMapper, FieldMapper lonMapper, FieldMapper geoHashMapper,
+                                            MultiFields multiFields, Explicit<Boolean> ignoreMalformed,
+                                            CopyTo copyTo) {
+            setupFieldType(context);
+            return new LatLonPointFieldMapper(simpleName, fieldType, defaultFieldType, indexSettings, multiFields,
+                ignoreMalformed, copyTo);
+        }
+
+        @Override
+        public LatLonPointFieldMapper build(BuilderContext context) {
+            return super.build(context);
+        }
+    }
+
+    public static class TypeParser extends BaseGeoPointFieldMapper.TypeParser {
+        @Override
+        public Mapper.Builder<?, ?> parse(String name, Map<String, Object> node, ParserContext parserContext)
+                throws MapperParsingException {
+            return super.parse(name, node, parserContext);
+        }
+    }
+
+    public LatLonPointFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
+                                  Settings indexSettings, MultiFields multiFields, Explicit<Boolean> ignoreMalformed,
+                                  CopyTo copyTo) {
+        super(simpleName, fieldType, defaultFieldType, indexSettings, null, null, null, multiFields, ignoreMalformed, copyTo);
+    }
+
+    public static class LatLonPointFieldType extends GeoPointFieldType {
+        LatLonPointFieldType() {
+        }
+
+        LatLonPointFieldType(LatLonPointFieldType ref) {
+            super(ref);
+        }
+
+        @Override
+        public String typeName() {
+            return CONTENT_TYPE;
+        }
+
+        @Override
+        public MappedFieldType clone() {
+            return new LatLonPointFieldType(this);
+        }
+
+        @Override
+        public IndexFieldData.Builder fielddataBuilder() {
+            failIfNoDocValues();
+            return new AbstractLatLonPointDVIndexFieldData.Builder();
+        }
+
+        @Override
+        public Query termQuery(Object value, QueryShardContext context) {
+            throw new QueryShardException(context, "Geo fields do not support exact searching, use dedicated geo queries instead: ["
+                + name() + "]");
+        }
+
+        @Override
+        public FieldStats.GeoPoint stats(IndexReader reader) throws IOException {
+            String field = name();
+            FieldInfo fi = org.apache.lucene.index.MultiFields.getMergedFieldInfos(reader).fieldInfo(name());
+            if (fi == null) {
+                return null;
+            }
+            final long size = PointValues.size(reader, field);
+            if (size == 0) {
+                return new FieldStats.GeoPoint(reader.maxDoc(), -1L, -1L, -1L, isSearchable(), isAggregatable());
+            }
+            final int docCount = PointValues.getDocCount(reader, field);
+            byte[] min = PointValues.getMinPackedValue(reader, field);
+            byte[] max = PointValues.getMaxPackedValue(reader, field);
+            GeoPoint minPt = new GeoPoint(GeoEncodingUtils.decodeLatitude(min, 0), GeoEncodingUtils.decodeLongitude(min, Integer.BYTES));
+            GeoPoint maxPt = new GeoPoint(GeoEncodingUtils.decodeLatitude(max, 0), GeoEncodingUtils.decodeLongitude(max, Integer.BYTES));
+            return new FieldStats.GeoPoint(reader.maxDoc(), docCount, -1L, size, isSearchable(), isAggregatable(),
+                minPt, maxPt);
+        }
+    }
+
+    @Override
+    protected void parse(ParseContext originalContext, GeoPoint point, String geoHash) throws IOException {
+        // Geopoint fields, by default, will not be included in _all
+        final ParseContext context = originalContext.setIncludeInAllDefault(false);
+
+        if (ignoreMalformed.value() == false) {
+            if (point.lat() > 90.0 || point.lat() < -90.0) {
+                throw new IllegalArgumentException("illegal latitude value [" + point.lat() + "] for " + name());
+            }
+            if (point.lon() > 180.0 || point.lon() < -180) {
+                throw new IllegalArgumentException("illegal longitude value [" + point.lon() + "] for " + name());
+            }
+        } else {
+            GeoUtils.normalizePoint(point);
+        }
+        if (fieldType().indexOptions() != IndexOptions.NONE) {
+            context.doc().add(new LatLonPoint(fieldType().name(), point.lat(), point.lon()));
+        }
+        if (fieldType().stored()) {
+            context.doc().add(new StoredField(fieldType().name(), point.toString()));
+        }
+        if (fieldType.hasDocValues()) {
+            context.doc().add(new LatLonDocValuesField(fieldType().name(), point.lat(), point.lon()));
+        }
+        // if the mapping contains multifields then use the geohash string
+        if (multiFields.iterator().hasNext()) {
+            multiFields.parse(this, context.createExternalValueContext(point.geohash()));
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/LegacyByteFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/LegacyByteFieldMapper.java
index 2c63806ebbe28..d69ae8a51a367 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/LegacyByteFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/LegacyByteFieldMapper.java
@@ -20,9 +20,10 @@
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.document.Field;
+import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.search.LegacyNumericRangeQuery;
 import org.apache.lucene.search.Query;
@@ -38,6 +39,7 @@
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType;
 import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
+import org.elasticsearch.index.query.QueryShardContext;
 
 import java.io.IOException;
 import java.util.Iterator;
@@ -107,7 +109,7 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
     }
 
     static final class ByteFieldType extends NumberFieldType {
-        public ByteFieldType() {
+        ByteFieldType() {
             super(LegacyNumericType.INT);
         }
 
@@ -131,7 +133,7 @@ public Byte nullValue() {
         }
 
         @Override
-        public Byte valueForSearch(Object value) {
+        public Byte valueForDisplay(Object value) {
             if (value == null) {
                 return null;
             }
@@ -146,7 +148,7 @@ public BytesRef indexedValueForSearch(Object value) {
         }
 
         @Override
-        public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper) {
+        public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) {
             return LegacyNumericRangeQuery.newIntRange(name(), numericPrecisionStep(),
                 lowerTerm == null ? null : (int)parseValue(lowerTerm),
                 upperTerm == null ? null : (int)parseValue(upperTerm),
@@ -156,9 +158,13 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower
         @Override
         public FieldStats.Long stats(IndexReader reader) throws IOException {
             int maxDoc = reader.maxDoc();
+            FieldInfo fi = org.apache.lucene.index.MultiFields.getMergedFieldInfos(reader).fieldInfo(name());
+            if (fi == null) {
+                return null;
+            }
             Terms terms = org.apache.lucene.index.MultiFields.getTerms(reader, name());
             if (terms == null) {
-                return null;
+                return new FieldStats.Long(maxDoc, 0, -1, -1, isSearchable(), isAggregatable());
             }
             long minValue = LegacyNumericUtils.getMinInt(terms);
             long maxValue = LegacyNumericUtils.getMaxInt(terms);
@@ -201,7 +207,7 @@ protected boolean customBoost() {
     }
 
     @Override
-    protected void innerParseCreateField(ParseContext context, List<Field> fields) throws IOException {
+    protected void innerParseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
         byte value;
         float boost = fieldType().boost();
         if (context.externalValueSet()) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/LegacyDateFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/LegacyDateFieldMapper.java
index 29689d06dff9a..42e588d963dc3 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/LegacyDateFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/LegacyDateFieldMapper.java
@@ -19,9 +19,10 @@
 
 package org.elasticsearch.index.mapper;
 
-import org.apache.lucene.document.Field;
+import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.search.LegacyNumericRangeQuery;
 import org.apache.lucene.search.Query;
@@ -43,8 +44,9 @@
 import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType;
 import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
 import org.elasticsearch.index.mapper.LegacyLongFieldMapper.CustomLongNumericField;
+import org.elasticsearch.index.query.QueryRewriteContext;
+import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.internal.SearchContext;
 import org.joda.time.DateTimeZone;
 
 import java.io.IOException;
@@ -53,7 +55,6 @@
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
-import java.util.concurrent.Callable;
 import java.util.concurrent.TimeUnit;
 
 import static org.elasticsearch.index.mapper.TypeParsers.parseDateTimeFormatter;
@@ -176,67 +177,6 @@ public static class TypeParser implements Mapper.TypeParser {
 
     public static class DateFieldType extends NumberFieldType {
 
-        final class LateParsingQuery extends Query {
-
-            final Object lowerTerm;
-            final Object upperTerm;
-            final boolean includeLower;
-            final boolean includeUpper;
-            final DateTimeZone timeZone;
-            final DateMathParser forcedDateParser;
-
-            public LateParsingQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, DateTimeZone timeZone, DateMathParser forcedDateParser) {
-                this.lowerTerm = lowerTerm;
-                this.upperTerm = upperTerm;
-                this.includeLower = includeLower;
-                this.includeUpper = includeUpper;
-                this.timeZone = timeZone;
-                this.forcedDateParser = forcedDateParser;
-            }
-
-            @Override
-            public Query rewrite(IndexReader reader) throws IOException {
-                Query rewritten = super.rewrite(reader);
-                if (rewritten != this) {
-                    return rewritten;
-                }
-                return innerRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, timeZone, forcedDateParser);
-            }
-
-            // Even though we only cache rewritten queries it is good to let all queries implement hashCode() and equals():
-            @Override
-            public boolean equals(Object o) {
-                if (this == o) return true;
-                if (sameClassAs(o) == false) return false;
-
-                LateParsingQuery that = (LateParsingQuery) o;
-                if (includeLower != that.includeLower) return false;
-                if (includeUpper != that.includeUpper) return false;
-                if (lowerTerm != null ? !lowerTerm.equals(that.lowerTerm) : that.lowerTerm != null) return false;
-                if (upperTerm != null ? !upperTerm.equals(that.upperTerm) : that.upperTerm != null) return false;
-                if (timeZone != null ? !timeZone.equals(that.timeZone) : that.timeZone != null) return false;
-
-                return true;
-            }
-
-            @Override
-            public int hashCode() {
-                return Objects.hash(classHash(), lowerTerm, upperTerm, includeLower, includeUpper, timeZone);
-            }
-
-            @Override
-            public String toString(String s) {
-                final StringBuilder sb = new StringBuilder();
-                return sb.append(name()).append(':')
-                    .append(includeLower ? '[' : '{')
-                    .append((lowerTerm == null) ? "*" : lowerTerm.toString())
-                    .append(" TO ")
-                    .append((upperTerm == null) ? "*" : upperTerm.toString())
-                    .append(includeUpper ? ']' : '}')
-                    .toString();
-            }
-        }
-
         protected FormatDateTimeFormatter dateTimeFormatter = Defaults.DATE_TIME_FORMATTER;
         protected TimeUnit timeUnit = Defaults.TIME_UNIT;
         protected DateMathParser dateMathParser = new DateMathParser(dateTimeFormatter);
@@ -339,7 +279,7 @@ public BytesRef indexedValueForSearch(Object value) {
         }
 
         @Override
-        public Object valueForSearch(Object value) {
+        public Object valueForDisplay(Object value) {
             Long val = (Long) value;
             if (val == null) {
                 return null;
@@ -348,16 +288,20 @@ public Object valueForSearch(Object value) {
         }
 
         @Override
-        public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper) {
-            return rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, null, null);
+        public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) {
+            return rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, null, null, context);
         }
 
         @Override
         public FieldStats.Date stats(IndexReader reader) throws IOException {
             int maxDoc = reader.maxDoc();
+            FieldInfo fi = org.apache.lucene.index.MultiFields.getMergedFieldInfos(reader).fieldInfo(name());
+            if (fi == null) {
+                return null;
+            }
             Terms terms = org.apache.lucene.index.MultiFields.getTerms(reader, name());
             if (terms == null) {
-                return null;
+                return new FieldStats.Date(maxDoc, 0, -1, -1, isSearchable(), isAggregatable());
             }
             long minValue = LegacyNumericUtils.getMinLong(terms);
             long maxValue = LegacyNumericUtils.getMaxLong(terms);
@@ -366,14 +310,20 @@ public FieldStats.Date stats(IndexReader reader) throws IOException {
                 dateTimeFormatter(), minValue, maxValue);
         }
 
-        public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable DateTimeZone timeZone, @Nullable DateMathParser forcedDateParser) {
-            return new LateParsingQuery(lowerTerm, upperTerm, includeLower, includeUpper, timeZone, forcedDateParser);
+        public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper,
+                @Nullable DateTimeZone timeZone, @Nullable DateMathParser forcedDateParser, QueryShardContext context) {
+            return  innerRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, timeZone, forcedDateParser, context);
         }
 
-        private Query innerRangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable DateTimeZone timeZone, @Nullable DateMathParser forcedDateParser) {
+        private Query innerRangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper,
+                @Nullable DateTimeZone timeZone, @Nullable DateMathParser forcedDateParser, QueryRewriteContext context) {
             return LegacyNumericRangeQuery.newLongRange(name(), numericPrecisionStep(),
-                lowerTerm == null ? null : parseToMilliseconds(lowerTerm, !includeLower, timeZone, forcedDateParser == null ? dateMathParser : forcedDateParser),
-                upperTerm == null ? null : parseToMilliseconds(upperTerm, includeUpper, timeZone, forcedDateParser == null ? dateMathParser : forcedDateParser),
+                    lowerTerm == null ? null
+                            : parseToMilliseconds(lowerTerm, !includeLower, timeZone,
+                                    forcedDateParser == null ? dateMathParser : forcedDateParser, context),
+                    upperTerm == null ? null
+                            : parseToMilliseconds(upperTerm, includeUpper, timeZone,
+                                    forcedDateParser == null ? dateMathParser : forcedDateParser, context),
                 includeLower, includeUpper);
         }
 
@@ -381,7 +331,7 @@ private Query innerRangeQuery(Object lowerTerm, Object upperTerm, boolean includ
         public Relation isFieldWithinQuery(IndexReader reader,
                 Object from, Object to,
                 boolean includeLower, boolean includeUpper,
-                DateTimeZone timeZone, DateMathParser dateParser) throws IOException {
+                DateTimeZone timeZone, DateMathParser dateParser, QueryRewriteContext context) throws IOException {
             if (dateParser == null) {
                 dateParser = this.dateMathParser;
             }
@@ -397,7 +347,7 @@ public Relation isFieldWithinQuery(IndexReader reader,
 
             long fromInclusive = Long.MIN_VALUE;
             if (from != null) {
-                fromInclusive = parseToMilliseconds(from, !includeLower, timeZone, dateParser);
+                fromInclusive = parseToMilliseconds(from, !includeLower, timeZone, dateParser, context);
                 if (includeLower == false) {
                     if (fromInclusive == Long.MAX_VALUE) {
                         return Relation.DISJOINT;
@@ -408,7 +358,7 @@ public Relation isFieldWithinQuery(IndexReader reader,
 
             long toInclusive = Long.MAX_VALUE;
             if (to != null) {
-                toInclusive = parseToMilliseconds(to, includeUpper, timeZone, dateParser);
+                toInclusive = parseToMilliseconds(to, includeUpper, timeZone, dateParser, context);
                 if (includeUpper == false) {
                     if (toInclusive == Long.MIN_VALUE) {
                         return Relation.DISJOINT;
@@ -426,7 +376,8 @@ public Relation isFieldWithinQuery(IndexReader reader,
             }
         }
 
-        public long parseToMilliseconds(Object value, boolean inclusive, @Nullable DateTimeZone zone, @Nullable DateMathParser forcedDateParser) {
+        public long parseToMilliseconds(Object value, boolean inclusive, @Nullable DateTimeZone zone,
+                @Nullable DateMathParser forcedDateParser, QueryRewriteContext context) {
             if (value instanceof Long) {
                 return ((Long) value).longValue();
             }
@@ -442,7 +393,7 @@ public long parseToMilliseconds(Object value, boolean inclusive, @Nullable DateT
             } else {
                 strValue = value.toString();
             }
-            return dateParser.parse(strValue, now(), inclusive, zone);
+            return dateParser.parse(strValue, context::nowInMillis, inclusive, zone);
         }
 
         @Override
@@ -474,25 +425,13 @@ public DateFieldType fieldType() {
         return (DateFieldType) super.fieldType();
     }
 
-    private static Callable<Long> now() {
-        return new Callable<Long>() {
-            @Override
-            public Long call() {
-                final SearchContext context = SearchContext.current();
-                return context != null
-                    ? context.nowInMillis()
-                    : System.currentTimeMillis();
-            }
-        };
-    }
-
     @Override
     protected boolean customBoost() {
         return true;
     }
 
     @Override
-    protected void innerParseCreateField(ParseContext context, List<Field> fields) throws IOException {
+    protected void innerParseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
         String dateAsString = null;
         float boost = fieldType().boost();
         if (context.externalValueSet()) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/LegacyDoubleFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/LegacyDoubleFieldMapper.java
index 07e459e8ea976..9895155d88ff2 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/LegacyDoubleFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/LegacyDoubleFieldMapper.java
@@ -21,9 +21,10 @@
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.document.Field;
+import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.search.LegacyNumericRangeQuery;
 import org.apache.lucene.search.Query;
@@ -41,6 +42,7 @@
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType;
 import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
+import org.elasticsearch.index.query.QueryShardContext;
 
 import java.io.IOException;
 import java.util.Iterator;
@@ -135,7 +137,7 @@ public java.lang.Double nullValue() {
         }
 
         @Override
-        public java.lang.Double valueForSearch(Object value) {
+        public java.lang.Double valueForDisplay(Object value) {
             if (value == null) {
                 return null;
             }
@@ -157,7 +159,7 @@ public BytesRef indexedValueForSearch(Object value) {
         }
 
         @Override
-        public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper) {
+        public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) {
             return LegacyNumericRangeQuery.newDoubleRange(name(), numericPrecisionStep(),
                 lowerTerm == null ? null : parseDoubleValue(lowerTerm),
                 upperTerm == null ? null : parseDoubleValue(upperTerm),
@@ -167,9 +169,13 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower
         @Override
         public FieldStats.Double stats(IndexReader reader) throws IOException {
             int maxDoc = reader.maxDoc();
+            FieldInfo fi = org.apache.lucene.index.MultiFields.getMergedFieldInfos(reader).fieldInfo(name());
+            if (fi == null) {
+                return null;
+            }
             Terms terms = org.apache.lucene.index.MultiFields.getTerms(reader, name());
             if (terms == null) {
-                return null;
+                return new FieldStats.Double(maxDoc, 0, -1, -1, isSearchable(), isAggregatable());
             }
             double minValue = NumericUtils.sortableLongToDouble(LegacyNumericUtils.getMinLong(terms));
             double maxValue = NumericUtils.sortableLongToDouble(LegacyNumericUtils.getMaxLong(terms));
@@ -201,7 +207,7 @@ protected boolean customBoost() {
     }
 
     @Override
-    protected void innerParseCreateField(ParseContext context, List<Field> fields) throws IOException {
+    protected void innerParseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
         double value;
         float boost = fieldType().boost();
         if (context.externalValueSet()) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/LegacyFloatFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/LegacyFloatFieldMapper.java
index 3fbc639ea67f8..0e473be04a455 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/LegacyFloatFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/LegacyFloatFieldMapper.java
@@ -21,9 +21,10 @@
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.document.Field;
+import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.search.LegacyNumericRangeQuery;
 import org.apache.lucene.search.Query;
@@ -40,6 +41,7 @@
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType;
 import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
+import org.elasticsearch.index.query.QueryShardContext;
 
 import java.io.IOException;
 import java.util.Iterator;
@@ -110,7 +112,7 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
 
     static final class FloatFieldType extends NumberFieldType {
 
-        public FloatFieldType() {
+        FloatFieldType() {
             super(LegacyNumericType.FLOAT);
         }
 
@@ -142,7 +144,7 @@ public BytesRef indexedValueForSearch(Object value) {
         }
 
         @Override
-        public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper) {
+        public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) {
             return LegacyNumericRangeQuery.newFloatRange(name(), numericPrecisionStep(),
                 lowerTerm == null ? null : parseValue(lowerTerm),
                 upperTerm == null ? null : parseValue(upperTerm),
@@ -152,9 +154,13 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower
         @Override
         public FieldStats.Double stats(IndexReader reader) throws IOException {
             int maxDoc = reader.maxDoc();
+            FieldInfo fi = org.apache.lucene.index.MultiFields.getMergedFieldInfos(reader).fieldInfo(name());
+            if (fi == null) {
+                return null;
+            }
             Terms terms = org.apache.lucene.index.MultiFields.getTerms(reader, name());
             if (terms == null) {
-                return null;
+                return new FieldStats.Double(maxDoc, 0, -1, -1, isSearchable(), isAggregatable());
             }
             float minValue = NumericUtils.sortableIntToFloat(LegacyNumericUtils.getMinInt(terms));
             float maxValue = NumericUtils.sortableIntToFloat(LegacyNumericUtils.getMaxInt(terms));
@@ -196,7 +202,7 @@ protected boolean customBoost() {
     }
 
     @Override
-    protected void innerParseCreateField(ParseContext context, List<Field> fields) throws IOException {
+    protected void innerParseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
         float value;
         float boost = fieldType().boost();
         if (context.externalValueSet()) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/LegacyGeoPointFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/LegacyGeoPointFieldMapper.java
index 3fe195c5d917b..619a0a824ef18 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/LegacyGeoPointFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/LegacyGeoPointFieldMapper.java
@@ -60,7 +60,7 @@ public static class Names extends BaseGeoPointFieldMapper.Names {
     public static class Defaults extends BaseGeoPointFieldMapper.Defaults{
         public static final Explicit<Boolean> COERCE = new Explicit<>(false, false);
 
-        public static final GeoPointFieldType FIELD_TYPE = new GeoPointFieldType();
+        public static final GeoPointFieldType FIELD_TYPE = new LegacyGeoPointFieldType();
 
         static {
             FIELD_TYPE.setIndexOptions(IndexOptions.DOCS);
@@ -123,7 +123,7 @@ public static Builder parse(Builder builder, Map<String, Object> node, Mapper.Ty
             String propName = entry.getKey();
             Object propNode = entry.getValue();
             if (propName.equals(Names.COERCE)) {
-                builder.coerce = XContentMapValues.lenientNodeBooleanValue(propNode);
+                builder.coerce = XContentMapValues.lenientNodeBooleanValue(propNode, propName);
                 iterator.remove();
             }
         }
@@ -297,7 +297,7 @@ protected void parse(ParseContext context, GeoPoint point, String geoHash) throw
             validPoint = true;
         }
 
-        if (coerce.value() == true && validPoint == false) {
+        if (coerce.value() && validPoint == false) {
             // by setting coerce to false we are assuming all geopoints are already in a valid coordinate system
             // thus this extra step can be skipped
             GeoUtils.normalizePoint(point, true, true);
@@ -331,6 +331,11 @@ protected void doXContentBody(XContentBuilder builder, boolean includeDefaults,
         }
     }
 
+    @Override
+    public LegacyGeoPointFieldType fieldType() {
+        return (LegacyGeoPointFieldType) super.fieldType();
+    }
+
     public static class CustomGeoPointDocValuesField extends CustomDocValuesField {
 
         private final ObjectHashSet<GeoPoint> points;
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/LegacyIntegerFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/LegacyIntegerFieldMapper.java
index 65b9b65eaf913..987f932e8cad3 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/LegacyIntegerFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/LegacyIntegerFieldMapper.java
@@ -21,9 +21,10 @@
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.document.Field;
+import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.search.LegacyNumericRangeQuery;
 import org.apache.lucene.search.Query;
@@ -39,6 +40,7 @@
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType;
 import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
+import org.elasticsearch.index.query.QueryShardContext;
 
 import java.io.IOException;
 import java.util.Iterator;
@@ -145,7 +147,7 @@ public BytesRef indexedValueForSearch(Object value) {
         }
 
         @Override
-        public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper) {
+        public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) {
             return LegacyNumericRangeQuery.newIntRange(name(), numericPrecisionStep(),
                 lowerTerm == null ? null : parseValue(lowerTerm),
                 upperTerm == null ? null : parseValue(upperTerm),
@@ -155,9 +157,13 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower
         @Override
         public FieldStats.Long stats(IndexReader reader) throws IOException {
             int maxDoc = reader.maxDoc();
+            FieldInfo fi = org.apache.lucene.index.MultiFields.getMergedFieldInfos(reader).fieldInfo(name());
+            if (fi == null) {
+                return null;
+            }
             Terms terms = org.apache.lucene.index.MultiFields.getTerms(reader, name());
             if (terms == null) {
-                return null;
+                return new FieldStats.Long(maxDoc, 0, -1, -1, isSearchable(), isAggregatable());
             }
             long minValue = LegacyNumericUtils.getMinInt(terms);
             long maxValue = LegacyNumericUtils.getMaxInt(terms);
@@ -200,7 +206,7 @@ protected boolean customBoost() {
     }
 
     @Override
-    protected void innerParseCreateField(ParseContext context, List<Field> fields) throws IOException {
+    protected void innerParseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
         int value;
         float boost = fieldType().boost();
         if (context.externalValueSet()) {
@@ -272,7 +278,7 @@ protected void innerParseCreateField(ParseContext context, List<Field> fields) t
         addIntegerFields(context, fields, value, boost);
     }
 
-    protected void addIntegerFields(ParseContext context, List<Field> fields, int value, float boost) {
+    protected void addIntegerFields(ParseContext context, List<IndexableField> fields, int value, float boost) {
         if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
             CustomIntegerNumericField field = new CustomIntegerNumericField(value, fieldType());
             if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/LegacyIpFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/LegacyIpFieldMapper.java
index 699124a4c05c2..c3704d34fb5c2 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/LegacyIpFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/LegacyIpFieldMapper.java
@@ -19,9 +19,10 @@
 
 package org.elasticsearch.index.mapper;
 
-import org.apache.lucene.document.Field;
+import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.search.LegacyNumericRangeQuery;
 import org.apache.lucene.search.Query;
@@ -80,7 +81,9 @@ public static long ipToLong(String ip) {
             }
             String[] octets = pattern.split(ip);
             if (octets.length != 4) {
-                throw new IllegalArgumentException("failed to parse ip [" + ip + "], not a valid ipv4 address (4 dots)");
+                // this must be ipv6, since isInetAddress returned true above
+                throw new IllegalArgumentException("ip [" + ip + "] is an IPv6 address, but this ip field is for an " +
+                                                   "index created before 5.0. Reindex into a new index to get IPv6 support.");
             }
             return (Long.parseLong(octets[0]) << 24) + (Integer.parseInt(octets[1]) << 16) +
                     (Integer.parseInt(octets[2]) << 8) + Integer.parseInt(octets[3]);
@@ -171,7 +174,7 @@ public String typeName() {
          * IPs should return as a string.
          */
         @Override
-        public Object valueForSearch(Object value) {
+        public Object valueForDisplay(Object value) {
             Long val = (Long) value;
             if (val == null) {
                 return null;
@@ -210,14 +213,14 @@ public Query termQuery(Object value, @Nullable QueryShardContext context) {
                 }
                 if (fromTo != null) {
                     return rangeQuery(fromTo[0] == 0 ? null : fromTo[0],
-                            fromTo[1] == MAX_IP ? null : fromTo[1], true, false);
+                            fromTo[1] == MAX_IP ? null : fromTo[1], true, false, context);
                 }
             }
             return super.termQuery(value, context);
         }
 
         @Override
-        public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper) {
+        public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) {
             return LegacyNumericRangeQuery.newLongRange(name(), numericPrecisionStep(),
                 lowerTerm == null ? null : parseValue(lowerTerm),
                 upperTerm == null ? null : parseValue(upperTerm),
@@ -227,9 +230,13 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower
         @Override
         public FieldStats stats(IndexReader reader) throws IOException {
             int maxDoc = reader.maxDoc();
+            FieldInfo fi = org.apache.lucene.index.MultiFields.getMergedFieldInfos(reader).fieldInfo(name());
+            if (fi == null) {
+                return null;
+            }
             Terms terms = org.apache.lucene.index.MultiFields.getTerms(reader, name());
             if (terms == null) {
-                return null;
+                return new FieldStats.Ip(maxDoc, 0, -1, -1, isSearchable(), isAggregatable());
             }
             long minValue = LegacyNumericUtils.getMinLong(terms);
             long maxValue = LegacyNumericUtils.getMaxLong(terms);
@@ -282,7 +289,7 @@ private static long parseValue(Object value) {
     }
 
     @Override
-    protected void innerParseCreateField(ParseContext context, List<Field> fields) throws IOException {
+    protected void innerParseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
         String ipAsString;
         if (context.externalValueSet()) {
             ipAsString = (String) context.externalValue();
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/LegacyIpIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/mapper/LegacyIpIndexFieldData.java
index feb3328227d0b..20326c2584167 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/LegacyIpIndexFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/LegacyIpIndexFieldData.java
@@ -24,7 +24,9 @@
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.search.SortField;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.fielddata.AtomicFieldData;
@@ -46,7 +48,7 @@ final class LegacyIpIndexFieldData implements IndexFieldData<AtomicFieldData> {
     protected final String fieldName;
     protected final Logger logger;
 
-    public LegacyIpIndexFieldData(Index index, String fieldName) {
+    LegacyIpIndexFieldData(Index index, String fieldName) {
         this.index = index;
         this.fieldName = fieldName;
         this.logger = Loggers.getLogger(getClass());
@@ -71,22 +73,22 @@ public Index index() {
     @Override
     public AtomicFieldData load(LeafReaderContext context) {
         return new AtomicFieldData() {
-            
+
             @Override
             public void close() {
                 // no-op
             }
-            
+
             @Override
             public long ramBytesUsed() {
                 return 0;
             }
-            
+
             @Override
             public ScriptDocValues<?> getScriptValues() {
                 throw new UnsupportedOperationException("Cannot run scripts on ip fields");
             }
-            
+
             @Override
             public SortedBinaryDocValues getBytesValues() {
                 SortedNumericDocValues values;
@@ -115,12 +117,12 @@ public BytesRef valueAt(int index) {
                         byte[] encoded = InetAddressPoint.encode(inet);
                         return new BytesRef(encoded);
                     }
-                    
+
                     @Override
                     public void setDocument(int docId) {
                         values.setDocument(docId);
                     }
-                    
+
                     @Override
                     public int count() {
                         return values.count();
@@ -137,9 +139,11 @@ public AtomicFieldData loadDirect(LeafReaderContext context)
     }
 
     @Override
-    public IndexFieldData.XFieldComparatorSource comparatorSource(
-            Object missingValue, MultiValueMode sortMode, Nested nested) {
-        return new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested);
+    public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMode,
+                               Nested nested, boolean reverse) {
+        final XFieldComparatorSource source =
+            new BytesRefFieldComparatorSource(this, missingValue,
+                sortMode, nested);
+        return new SortField(getFieldName(), source, reverse);
     }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/LegacyLongFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/LegacyLongFieldMapper.java
index 4661d1cd365aa..6d76401281997 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/LegacyLongFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/LegacyLongFieldMapper.java
@@ -21,9 +21,10 @@
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.document.Field;
+import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.search.LegacyNumericRangeQuery;
 import org.apache.lucene.search.Query;
@@ -39,6 +40,7 @@
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType;
 import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
+import org.elasticsearch.index.query.QueryShardContext;
 
 import java.io.IOException;
 import java.util.Iterator;
@@ -146,7 +148,7 @@ public BytesRef indexedValueForSearch(Object value) {
         }
 
         @Override
-        public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper) {
+        public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) {
             return LegacyNumericRangeQuery.newLongRange(name(), numericPrecisionStep(),
                 lowerTerm == null ? null : parseLongValue(lowerTerm),
                 upperTerm == null ? null : parseLongValue(upperTerm),
@@ -156,9 +158,13 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower
         @Override
         public FieldStats stats(IndexReader reader) throws IOException {
             int maxDoc = reader.maxDoc();
+            FieldInfo fi = org.apache.lucene.index.MultiFields.getMergedFieldInfos(reader).fieldInfo(name());
+            if (fi == null) {
+                return null;
+            }
             Terms terms = org.apache.lucene.index.MultiFields.getTerms(reader, name());
             if (terms == null) {
-                return null;
+                return new FieldStats.Long(maxDoc, 0, -1, -1, isSearchable(), isAggregatable());
             }
             long minValue = LegacyNumericUtils.getMinLong(terms);
             long maxValue = LegacyNumericUtils.getMaxLong(terms);
@@ -191,7 +197,7 @@ protected boolean customBoost() {
     }
 
     @Override
-    protected void innerParseCreateField(ParseContext context, List<Field> fields) throws IOException {
+    protected void innerParseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
         long value;
         float boost = fieldType().boost();
         if (context.externalValueSet()) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/LegacyNumberFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/LegacyNumberFieldMapper.java
index f377883aa24e5..99974c349423d 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/LegacyNumberFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/LegacyNumberFieldMapper.java
@@ -27,6 +27,7 @@
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.Nullable;
@@ -173,7 +174,7 @@ protected LegacyNumberFieldMapper clone() {
     }
 
     @Override
-    protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
+    protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
         RuntimeException e = null;
         try {
             innerParseCreateField(context, fields);
@@ -188,9 +189,9 @@ protected void parseCreateField(ParseContext context, List<Field> fields) throws
         }
     }
 
-    protected abstract void innerParseCreateField(ParseContext context, List<Field> fields) throws IOException;
+    protected abstract void innerParseCreateField(ParseContext context, List<IndexableField> fields) throws IOException;
 
-    protected final void addDocValue(ParseContext context, List<Field> fields, long value) {
+    protected final void addDocValue(ParseContext context, List<IndexableField> fields, long value) {
         fields.add(new SortedNumericDocValuesField(fieldType().name(), value));
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/LegacyShortFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/LegacyShortFieldMapper.java
index b42ec620aea43..d625b0a7a3aae 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/LegacyShortFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/LegacyShortFieldMapper.java
@@ -21,9 +21,10 @@
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.document.Field;
+import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.search.LegacyNumericRangeQuery;
 import org.apache.lucene.search.Query;
@@ -39,6 +40,7 @@
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType;
 import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
+import org.elasticsearch.index.query.QueryShardContext;
 
 import java.io.IOException;
 import java.util.Iterator;
@@ -111,7 +113,7 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
 
     static final class ShortFieldType extends NumberFieldType {
 
-        public ShortFieldType() {
+        ShortFieldType() {
             super(LegacyNumericType.INT);
         }
 
@@ -135,7 +137,7 @@ public Short nullValue() {
         }
 
         @Override
-        public Short valueForSearch(Object value) {
+        public Short valueForDisplay(Object value) {
             if (value == null) {
                 return null;
             }
@@ -150,7 +152,7 @@ public BytesRef indexedValueForSearch(Object value) {
         }
 
         @Override
-        public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper) {
+        public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) {
             return LegacyNumericRangeQuery.newIntRange(name(), numericPrecisionStep(),
                 lowerTerm == null ? null : (int)parseValue(lowerTerm),
                 upperTerm == null ? null : (int)parseValue(upperTerm),
@@ -160,9 +162,13 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower
         @Override
         public FieldStats.Long stats(IndexReader reader) throws IOException {
             int maxDoc = reader.maxDoc();
+            FieldInfo fi = org.apache.lucene.index.MultiFields.getMergedFieldInfos(reader).fieldInfo(name());
+            if (fi == null) {
+                return null;
+            }
             Terms terms = org.apache.lucene.index.MultiFields.getTerms(reader, name());
             if (terms == null) {
-                return null;
+                return new FieldStats.Long(maxDoc, 0, -1, -1, isSearchable(), isAggregatable());
             }
             long minValue = LegacyNumericUtils.getMinInt(terms);
             long maxValue = LegacyNumericUtils.getMaxInt(terms);
@@ -205,7 +211,7 @@ protected boolean customBoost() {
     }
 
     @Override
-    protected void innerParseCreateField(ParseContext context, List<Field> fields) throws IOException {
+    protected void innerParseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
         short value;
         float boost = fieldType().boost();
         if (context.externalValueSet()) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/LegacyTokenCountFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/LegacyTokenCountFieldMapper.java
index 2ed1b544a0292..5dfb93d4836f1 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/LegacyTokenCountFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/LegacyTokenCountFieldMapper.java
@@ -23,6 +23,7 @@
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.index.IndexableField;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.settings.Settings;
@@ -97,7 +98,7 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
                     builder.nullValue(nodeIntegerValue(propNode));
                     iterator.remove();
                 } else if (propName.equals("analyzer")) {
-                    NamedAnalyzer analyzer = parserContext.analysisService().analyzer(propNode.toString());
+                    NamedAnalyzer analyzer = parserContext.getIndexAnalyzers().get(propNode.toString());
                     if (analyzer == null) {
                         throw new MapperParsingException("Analyzer [" + propNode.toString() + "] not found for field [" + name + "]");
                     }
@@ -122,7 +123,7 @@ protected LegacyTokenCountFieldMapper(String simpleName, MappedFieldType fieldTy
     }
 
     @Override
-    protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
+    protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
         ValueAndBoost valueAndBoost = StringFieldMapper.parseCreateFieldForString(context, null /* Out null value is an int so we convert*/, fieldType().boost());
         if (valueAndBoost.value() == null && fieldType().nullValue() == null) {
             return;
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java
index 9a434cc8a368b..9e8b2d9018f25 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java
@@ -20,24 +20,31 @@
 package org.elasticsearch.index.mapper;
 
 import org.apache.lucene.document.FieldType;
+import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.index.PrefixCodedTerms;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.PrefixCodedTerms.TermIterator;
+import org.apache.lucene.search.BooleanClause.Occur;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.MultiTermQuery;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermInSetQuery;
 import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.BooleanClause.Occur;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.BoostQuery;
+import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.action.fieldstats.FieldStats;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.joda.DateMathParser;
+import org.elasticsearch.common.lucene.all.AllTermQuery;
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.query.QueryRewriteContext;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.query.QueryShardException;
 import org.elasticsearch.index.similarity.SimilarityProvider;
@@ -133,7 +140,7 @@ public int hashCode() {
             eagerGlobalOrdinals, similarity == null ? null : similarity.name(), nullValue, nullValueAsString);
     }
 
-    // norelease: we need to override freeze() and add safety checks that all settings are actually set
+    // TODO: we need to override freeze() and add safety checks that all settings are actually set
 
     /** Returns the name of this type, as would be specified in mapping properties */
     public abstract String typeName();
@@ -303,21 +310,21 @@ public void setNullValue(Object nullValue) {
     /** Given a value that comes from the stored fields API, convert it to the
      *  expected type. For instance a date field would store dates as longs and
      *  format it back to a string in this method. */
-    public Object valueForSearch(Object value) {
+    public Object valueForDisplay(Object value) {
         return value;
     }
 
     /** Returns true if the field is searchable.
      *
      */
-    protected boolean isSearchable() {
+    public boolean isSearchable() {
         return indexOptions() != IndexOptions.NONE;
     }
 
     /** Returns true if the field is aggregatable.
      *
      */
-    protected boolean isAggregatable() {
+    public boolean isAggregatable() {
         try {
             fielddataBuilder();
             return true;
@@ -343,7 +350,7 @@ public Query termsQuery(List<?> values, @Nullable QueryShardContext context) {
         return new ConstantScoreQuery(builder.build());
     }
 
-    public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper) {
+    public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) {
         throw new IllegalArgumentException("Field [" + name + "] of type [" + typeName() + "] does not support range queries");
     }
 
@@ -373,14 +380,16 @@ public Query nullValueQuery() {
      */
     public FieldStats stats(IndexReader reader) throws IOException {
         int maxDoc = reader.maxDoc();
+        FieldInfo fi = MultiFields.getMergedFieldInfos(reader).fieldInfo(name());
+        if (fi == null) {
+            return null;
+        }
         Terms terms = MultiFields.getTerms(reader, name());
         if (terms == null) {
-            return null;
+            return new FieldStats.Text(maxDoc, 0, -1, -1, isSearchable(), isAggregatable());
         }
         FieldStats stats = new FieldStats.Text(maxDoc, terms.getDocCount(),
-            terms.getSumDocFreq(), terms.getSumTotalTermFreq(),
-            isSearchable(), isAggregatable(),
-            terms.getMin(), terms.getMax());
+            terms.getSumDocFreq(), terms.getSumTotalTermFreq(), isSearchable(), isAggregatable(), terms.getMin(), terms.getMax());
         return stats;
     }
 
@@ -388,7 +397,7 @@ public FieldStats stats(IndexReader reader) throws IOException {
      * An enum used to describe the relation between the range of terms in a
      * shard when compared with a query range
      */
-    public static enum Relation {
+    public enum Relation {
         WITHIN,
         INTERSECTS,
         DISJOINT;
@@ -399,10 +408,10 @@ public static enum Relation {
      *  {@link Relation#INTERSECTS}, which is always fine to return when there is
      *  no way to check whether values are actually within bounds. */
     public Relation isFieldWithinQuery(
-            IndexReader reader,
-            Object from, Object to,
-            boolean includeLower, boolean includeUpper,
-            DateTimeZone timeZone, DateMathParser dateMathParser) throws IOException {
+        IndexReader reader,
+        Object from, Object to,
+        boolean includeLower, boolean includeUpper,
+        DateTimeZone timeZone, DateMathParser dateMathParser, QueryRewriteContext context) throws IOException {
         return Relation.INTERSECTS;
     }
 
@@ -462,6 +471,21 @@ public static Term extractTerm(Query termQuery) {
         while (termQuery instanceof BoostQuery) {
             termQuery = ((BoostQuery) termQuery).getQuery();
         }
+        if (termQuery instanceof AllTermQuery) {
+            return ((AllTermQuery) termQuery).getTerm();
+        } else if (termQuery instanceof TypeFieldMapper.TypesQuery) {
+            assert ((TypeFieldMapper.TypesQuery) termQuery).getTerms().length == 1;
+            return new Term(TypeFieldMapper.NAME, ((TypeFieldMapper.TypesQuery) termQuery).getTerms()[0]);
+        }
+        if (termQuery instanceof TermInSetQuery) {
+            TermInSetQuery tisQuery = (TermInSetQuery) termQuery;
+            PrefixCodedTerms terms = tisQuery.getTermData();
+            if (terms.size() == 1) {
+                TermIterator it = terms.iterator();
+                BytesRef term = it.next();
+                return new Term(it.field(), term);
+            }
+        }
         if (termQuery instanceof TermQuery == false) {
             throw new IllegalArgumentException("Cannot extract a term from a query of type "
                     + termQuery.getClass() + ": " + termQuery);
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/Mapper.java b/core/src/main/java/org/elasticsearch/index/mapper/Mapper.java
index d1341c8f7d46d..b7082ba81c4f1 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/Mapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/Mapper.java
@@ -21,16 +21,16 @@
 
 import org.elasticsearch.Version;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.index.analysis.AnalysisService;
+import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.similarity.SimilarityProvider;
 
 import java.util.Map;
 import java.util.Objects;
 import java.util.function.Function;
+import java.util.function.Supplier;
 
 public abstract class Mapper implements ToXContent, Iterable<Mapper> {
 
@@ -85,7 +85,7 @@ class ParserContext {
 
             private final String type;
 
-            private final AnalysisService analysisService;
+            private final IndexAnalyzers indexAnalyzers;
 
             private final Function<String, SimilarityProvider> similarityLookupService;
 
@@ -95,29 +95,26 @@ class ParserContext {
 
             private final Version indexVersionCreated;
 
-            private final ParseFieldMatcher parseFieldMatcher;
+            private final Supplier<QueryShardContext> queryShardContextSupplier;
 
-            private final QueryShardContext queryShardContext;
-
-            public ParserContext(String type, AnalysisService analysisService, Function<String, SimilarityProvider> similarityLookupService,
+            public ParserContext(String type, IndexAnalyzers indexAnalyzers, Function<String, SimilarityProvider> similarityLookupService,
                                  MapperService mapperService, Function<String, TypeParser> typeParsers,
-                                 Version indexVersionCreated, ParseFieldMatcher parseFieldMatcher, QueryShardContext queryShardContext) {
+                                 Version indexVersionCreated, Supplier<QueryShardContext> queryShardContextSupplier) {
                 this.type = type;
-                this.analysisService = analysisService;
+                this.indexAnalyzers = indexAnalyzers;
                 this.similarityLookupService = similarityLookupService;
                 this.mapperService = mapperService;
                 this.typeParsers = typeParsers;
                 this.indexVersionCreated = indexVersionCreated;
-                this.parseFieldMatcher = parseFieldMatcher;
-                this.queryShardContext = queryShardContext;
+                this.queryShardContextSupplier = queryShardContextSupplier;
             }
 
             public String type() {
                 return type;
             }
 
-            public AnalysisService analysisService() {
-                return analysisService;
+            public IndexAnalyzers getIndexAnalyzers() {
+                return indexAnalyzers;
             }
 
             public SimilarityProvider getSimilarity(String name) {
@@ -136,12 +133,8 @@ public Version indexVersionCreated() {
                 return indexVersionCreated;
             }
 
-            public ParseFieldMatcher parseFieldMatcher() {
-                return parseFieldMatcher;
-            }
-
-            public QueryShardContext queryShardContext() {
-                return queryShardContext;
+            public Supplier<QueryShardContext> queryShardContextSupplier() {
+                return queryShardContextSupplier;
             }
 
             public boolean isWithinMultiField() { return false; }
@@ -159,7 +152,8 @@ public ParserContext createMultiFieldContext(ParserContext in) {
 
             static class MultiFieldParserContext extends ParserContext {
                 MultiFieldParserContext(ParserContext in) {
-                    super(in.type(), in.analysisService, in.similarityLookupService(), in.mapperService(), in.typeParsers(), in.indexVersionCreated(), in.parseFieldMatcher(), in.queryShardContext());
+                    super(in.type(), in.indexAnalyzers, in.similarityLookupService(), in.mapperService(), in.typeParsers(),
+                            in.indexVersionCreated(), in.queryShardContextSupplier());
                 }
             }
 
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java
index 148fde7b64826..2b66d88a2d833 100755
--- a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java
@@ -21,22 +21,27 @@
 
 import com.carrotsearch.hppc.ObjectHashSet;
 import com.carrotsearch.hppc.cursors.ObjectCursor;
+
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.DelegatingAnalyzerWrapper;
+import org.apache.lucene.index.Term;
 import org.elasticsearch.ElasticsearchGenerationException;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.AbstractIndexComponent;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.analysis.AnalysisService;
+import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.mapper.Mapper.BuilderContext;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.similarity.SimilarityService;
@@ -44,12 +49,14 @@
 import org.elasticsearch.indices.TypeMissingException;
 import org.elasticsearch.indices.mapper.MapperRegistry;
 
+import java.io.Closeable;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -60,12 +67,8 @@
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
 import static java.util.Collections.unmodifiableMap;
-import static org.elasticsearch.common.collect.MapBuilder.newMapBuilder;
 
-/**
- *
- */
-public class MapperService extends AbstractIndexComponent {
+public class MapperService extends AbstractIndexComponent implements Closeable {
 
     /**
      * The reason why a mapping is being merged.
@@ -93,6 +96,8 @@ public enum MergeReason {
     public static final boolean INDEX_MAPPER_DYNAMIC_DEFAULT = true;
     public static final Setting<Boolean> INDEX_MAPPER_DYNAMIC_SETTING =
         Setting.boolSetting("index.mapper.dynamic", INDEX_MAPPER_DYNAMIC_DEFAULT, Property.Dynamic, Property.IndexScope);
+    public static final Setting<Boolean> INDEX_MAPPING_SINGLE_TYPE_SETTING =
+        Setting.boolSetting("index.mapping.single_type", false, Property.IndexScope, Property.Final);
     private static ObjectHashSet<String> META_FIELDS = ObjectHashSet.from(
             "_uid", "_id", "_type", "_all", "_parent", "_routing", "_index",
             "_size", "_timestamp", "_ttl"
@@ -100,7 +105,7 @@ public enum MergeReason {
     @Deprecated
     public static final String PERCOLATOR_LEGACY_TYPE_NAME = ".percolator";
 
-    private final AnalysisService analysisService;
+    private final IndexAnalyzers indexAnalyzers;
 
     /**
      * Will create types automatically if they do not exists in the mapping definition yet
@@ -112,8 +117,9 @@ public enum MergeReason {
     private volatile Map<String, DocumentMapper> mappers = emptyMap();
 
     private volatile FieldTypeLookup fieldTypes;
-    private volatile Map<String, ObjectMapper> fullPathObjectMappers = new HashMap<>();
+    private volatile Map<String, ObjectMapper> fullPathObjectMappers = emptyMap();
     private boolean hasNested = false; // updated dynamically to true when a nested object is added
+    private boolean allEnabled = false; // updated dynamically to true when _all is enabled
 
     private final DocumentMapperParser documentParser;
 
@@ -127,16 +133,17 @@ public enum MergeReason {
 
     final MapperRegistry mapperRegistry;
 
-    public MapperService(IndexSettings indexSettings, AnalysisService analysisService,
+    public MapperService(IndexSettings indexSettings, IndexAnalyzers indexAnalyzers, NamedXContentRegistry xContentRegistry,
                          SimilarityService similarityService, MapperRegistry mapperRegistry,
                          Supplier<QueryShardContext> queryShardContextSupplier) {
         super(indexSettings);
-        this.analysisService = analysisService;
+        this.indexAnalyzers = indexAnalyzers;
         this.fieldTypes = new FieldTypeLookup();
-        this.documentParser = new DocumentMapperParser(indexSettings, this, analysisService, similarityService, mapperRegistry, queryShardContextSupplier);
-        this.indexAnalyzer = new MapperAnalyzerWrapper(analysisService.defaultIndexAnalyzer(), p -> p.indexAnalyzer());
-        this.searchAnalyzer = new MapperAnalyzerWrapper(analysisService.defaultSearchAnalyzer(), p -> p.searchAnalyzer());
-        this.searchQuoteAnalyzer = new MapperAnalyzerWrapper(analysisService.defaultSearchQuoteAnalyzer(), p -> p.searchQuoteAnalyzer());
+        this.documentParser = new DocumentMapperParser(indexSettings, this, indexAnalyzers, xContentRegistry, similarityService,
+                mapperRegistry, queryShardContextSupplier);
+        this.indexAnalyzer = new MapperAnalyzerWrapper(indexAnalyzers.getDefaultIndexAnalyzer(), p -> p.indexAnalyzer());
+        this.searchAnalyzer = new MapperAnalyzerWrapper(indexAnalyzers.getDefaultSearchAnalyzer(), p -> p.searchAnalyzer());
+        this.searchQuoteAnalyzer = new MapperAnalyzerWrapper(indexAnalyzers.getDefaultSearchQuoteAnalyzer(), p -> p.searchQuoteAnalyzer());
         this.mapperRegistry = mapperRegistry;
 
         this.dynamic = this.indexSettings.getValue(INDEX_MAPPER_DYNAMIC_SETTING);
@@ -153,6 +160,13 @@ public boolean hasNested() {
         return this.hasNested;
     }
 
+    /**
+     * Returns true if the "_all" field is enabled on any type.
+     */
+    public boolean allEnabled() {
+        return this.allEnabled;
+    }
+
     /**
      * returns an immutable iterator over current document mappers.
      *
@@ -171,167 +185,256 @@ public Iterable<DocumentMapper> docMappers(final boolean includingDefaultMapping
         };
     }
 
-    public AnalysisService analysisService() {
-        return this.analysisService;
+    public IndexAnalyzers getIndexAnalyzers() {
+        return this.indexAnalyzers;
     }
 
     public DocumentMapperParser documentMapperParser() {
         return this.documentParser;
     }
 
-    public static Map<String, Object> parseMapping(String mappingSource) throws Exception {
-        try (XContentParser parser = XContentFactory.xContent(mappingSource).createParser(mappingSource)) {
+    /**
+     * Parses the mappings (formatted as JSON) into a map
+     */
+    public static Map<String, Object> parseMapping(NamedXContentRegistry xContentRegistry, String mappingSource) throws Exception {
+        try (XContentParser parser = XContentType.JSON.xContent().createParser(xContentRegistry, mappingSource)) {
             return parser.map();
         }
     }
 
+    /**
+     * Update mapping by only merging the metadata that is different between received and stored entries
+     */
     public boolean updateMapping(IndexMetaData indexMetaData) throws IOException {
         assert indexMetaData.getIndex().equals(index()) : "index mismatch: expected " + index() + " but was " + indexMetaData.getIndex();
         // go over and add the relevant mappings (or update them)
+        final Set<String> existingMappers = new HashSet<>(mappers.keySet());
+        final Map<String, DocumentMapper> updatedEntries;
+        try {
+            // only update entries if needed
+            updatedEntries = internalMerge(indexMetaData, MergeReason.MAPPING_RECOVERY, true, true);
+        } catch (Exception e) {
+            logger.warn((org.apache.logging.log4j.util.Supplier<?>) () -> new ParameterizedMessage("[{}] failed to apply mappings", index()), e);
+            throw e;
+        }
+
         boolean requireRefresh = false;
-        for (ObjectCursor<MappingMetaData> cursor : indexMetaData.getMappings().values()) {
-            MappingMetaData mappingMd = cursor.value;
-            String mappingType = mappingMd.type();
-            CompressedXContent mappingSource = mappingMd.source();
+
+        for (DocumentMapper documentMapper : updatedEntries.values()) {
+            String mappingType = documentMapper.type();
+            CompressedXContent incomingMappingSource = indexMetaData.mapping(mappingType).source();
+
+            String op = existingMappers.contains(mappingType) ? "updated" : "added";
+            if (logger.isDebugEnabled() && incomingMappingSource.compressed().length < 512) {
+                logger.debug("[{}] {} mapping [{}], source [{}]", index(), op, mappingType, incomingMappingSource.string());
+            } else if (logger.isTraceEnabled()) {
+                logger.trace("[{}] {} mapping [{}], source [{}]", index(), op, mappingType, incomingMappingSource.string());
+            } else {
+                logger.debug("[{}] {} mapping [{}] (source suppressed due to length, use TRACE level if needed)", index(), op, mappingType);
+            }
+
             // refresh mapping can happen when the parsing/merging of the mapping from the metadata doesn't result in the same
             // mapping, in this case, we send to the master to refresh its own version of the mappings (to conform with the
             // merge version of it, which it does when refreshing the mappings), and warn log it.
-            try {
-                DocumentMapper existingMapper = documentMapper(mappingType);
-
-                if (existingMapper == null || mappingSource.equals(existingMapper.mappingSource()) == false) {
-                    String op = existingMapper == null ? "adding" : "updating";
-                    if (logger.isDebugEnabled() && mappingSource.compressed().length < 512) {
-                        logger.debug("[{}] {} mapping [{}], source [{}]", index(), op, mappingType, mappingSource.string());
-                    } else if (logger.isTraceEnabled()) {
-                        logger.trace("[{}] {} mapping [{}], source [{}]", index(), op, mappingType, mappingSource.string());
-                    } else {
-                        logger.debug("[{}] {} mapping [{}] (source suppressed due to length, use TRACE level if needed)", index(), op,
-                            mappingType);
-                    }
-                    merge(mappingType, mappingSource, MergeReason.MAPPING_RECOVERY, true);
-                    if (!documentMapper(mappingType).mappingSource().equals(mappingSource)) {
-                        logger.debug("[{}] parsed mapping [{}], and got different sources\noriginal:\n{}\nparsed:\n{}", index(),
-                            mappingType, mappingSource, documentMapper(mappingType).mappingSource());
-                        requireRefresh = true;
-                    }
-                }
-            } catch (Exception e) {
-                logger.warn(
-                    (org.apache.logging.log4j.util.Supplier<?>)
-                        () -> new ParameterizedMessage("[{}] failed to add mapping [{}], source [{}]", index(), mappingType, mappingSource),
-                    e);
-                throw e;
+            if (documentMapper(mappingType).mappingSource().equals(incomingMappingSource) == false) {
+                logger.debug("[{}] parsed mapping [{}], and got different sources\noriginal:\n{}\nparsed:\n{}", index(), mappingType,
+                    incomingMappingSource, documentMapper(mappingType).mappingSource());
+
+                requireRefresh = true;
             }
         }
+
         return requireRefresh;
     }
 
-    //TODO: make this atomic
-    public void merge(Map<String, Map<String, Object>> mappings, boolean updateAllTypes) throws MapperParsingException {
-        // first, add the default mapping
-        if (mappings.containsKey(DEFAULT_MAPPING)) {
-            try {
-                this.merge(DEFAULT_MAPPING, new CompressedXContent(XContentFactory.jsonBuilder().map(mappings.get(DEFAULT_MAPPING)).string()), MergeReason.MAPPING_UPDATE, updateAllTypes);
-            } catch (Exception e) {
-                throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, DEFAULT_MAPPING, e.getMessage());
-            }
-        }
+    public void merge(Map<String, Map<String, Object>> mappings, MergeReason reason, boolean updateAllTypes) {
+        Map<String, CompressedXContent> mappingSourcesCompressed = new LinkedHashMap<>(mappings.size());
         for (Map.Entry<String, Map<String, Object>> entry : mappings.entrySet()) {
-            if (entry.getKey().equals(DEFAULT_MAPPING)) {
-                continue;
-            }
             try {
-                // apply the default here, its the first time we parse it
-                this.merge(entry.getKey(), new CompressedXContent(XContentFactory.jsonBuilder().map(entry.getValue()).string()), MergeReason.MAPPING_UPDATE, updateAllTypes);
+                mappingSourcesCompressed.put(entry.getKey(), new CompressedXContent(XContentFactory.jsonBuilder().map(entry.getValue()).string()));
             } catch (Exception e) {
                 throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, entry.getKey(), e.getMessage());
             }
         }
+
+        internalMerge(mappingSourcesCompressed, reason, updateAllTypes);
+    }
+
+    public void merge(IndexMetaData indexMetaData, MergeReason reason, boolean updateAllTypes) {
+        internalMerge(indexMetaData, reason, updateAllTypes, false);
     }
 
     public DocumentMapper merge(String type, CompressedXContent mappingSource, MergeReason reason, boolean updateAllTypes) {
-        if (DEFAULT_MAPPING.equals(type)) {
+        return internalMerge(Collections.singletonMap(type, mappingSource), reason, updateAllTypes).get(type);
+    }
+
+    private synchronized Map<String, DocumentMapper> internalMerge(IndexMetaData indexMetaData, MergeReason reason, boolean updateAllTypes,
+                                                                   boolean onlyUpdateIfNeeded) {
+        Map<String, CompressedXContent> map = new LinkedHashMap<>();
+        for (ObjectCursor<MappingMetaData> cursor : indexMetaData.getMappings().values()) {
+            MappingMetaData mappingMetaData = cursor.value;
+            if (onlyUpdateIfNeeded) {
+                DocumentMapper existingMapper = documentMapper(mappingMetaData.type());
+                if (existingMapper == null || mappingMetaData.source().equals(existingMapper.mappingSource()) == false) {
+                    map.put(mappingMetaData.type(), mappingMetaData.source());
+                }
+            } else {
+                map.put(mappingMetaData.type(), mappingMetaData.source());
+            }
+        }
+        return internalMerge(map, reason, updateAllTypes);
+    }
+
+    private synchronized Map<String, DocumentMapper> internalMerge(Map<String, CompressedXContent> mappings, MergeReason reason, boolean updateAllTypes) {
+        DocumentMapper defaultMapper = null;
+        String defaultMappingSource = null;
+
+        if (mappings.containsKey(DEFAULT_MAPPING)) {
             // verify we can parse it
             // NOTE: never apply the default here
-            DocumentMapper mapper = documentParser.parse(type, mappingSource);
-            // still add it as a document mapper so we have it registered and, for example, persisted back into
-            // the cluster meta data if needed, or checked for existence
-            synchronized (this) {
-                mappers = newMapBuilder(mappers).put(type, mapper).map();
+            try {
+                defaultMapper = documentParser.parse(DEFAULT_MAPPING, mappings.get(DEFAULT_MAPPING));
+            } catch (Exception e) {
+                throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, DEFAULT_MAPPING, e.getMessage());
             }
             try {
-                defaultMappingSource = mappingSource.string();
+                defaultMappingSource = mappings.get(DEFAULT_MAPPING).string();
             } catch (IOException e) {
                 throw new ElasticsearchGenerationException("failed to un-compress", e);
             }
-            return mapper;
+        }
+
+        final String defaultMappingSourceOrLastStored;
+        if (defaultMappingSource != null) {
+            defaultMappingSourceOrLastStored = defaultMappingSource;
         } else {
-            synchronized (this) {
-                final boolean applyDefault =
-                        // the default was already applied if we are recovering
-                        reason != MergeReason.MAPPING_RECOVERY
-                        // only apply the default mapping if we don't have the type yet
-                        && mappers.containsKey(type) == false;
-                DocumentMapper mergeWith = parse(type, mappingSource, applyDefault);
-                return merge(mergeWith, reason, updateAllTypes);
+            defaultMappingSourceOrLastStored = this.defaultMappingSource;
+        }
+
+        List<DocumentMapper> documentMappers = new ArrayList<>();
+        for (Map.Entry<String, CompressedXContent> entry : mappings.entrySet()) {
+            String type = entry.getKey();
+            if (type.equals(DEFAULT_MAPPING)) {
+                continue;
+            }
+
+            final boolean applyDefault =
+                // the default was already applied if we are recovering
+                reason != MergeReason.MAPPING_RECOVERY
+                    // only apply the default mapping if we don't have the type yet
+                    && mappers.containsKey(type) == false;
+
+            try {
+                DocumentMapper documentMapper =
+                    documentParser.parse(type, entry.getValue(), applyDefault ? defaultMappingSourceOrLastStored : null);
+                documentMappers.add(documentMapper);
+            } catch (Exception e) {
+                throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, entry.getKey(), e.getMessage());
             }
         }
+
+        return internalMerge(defaultMapper, defaultMappingSource, documentMappers, reason, updateAllTypes);
     }
 
-    private synchronized DocumentMapper merge(DocumentMapper mapper, MergeReason reason, boolean updateAllTypes) {
-        if (mapper.type().length() == 0) {
-            throw new InvalidTypeNameException("mapping type name is empty");
-        }
-        if (mapper.type().length() > 255) {
-            throw new InvalidTypeNameException("mapping type name [" + mapper.type() + "] is too long; limit is length 255 but was [" + mapper.type().length() + "]");
-        }
-        if (mapper.type().charAt(0) == '_') {
-            throw new InvalidTypeNameException("mapping type name [" + mapper.type() + "] can't start with '_'");
-        }
-        if (mapper.type().contains("#")) {
-            throw new InvalidTypeNameException("mapping type name [" + mapper.type() + "] should not include '#' in it");
-        }
-        if (mapper.type().contains(",")) {
-            throw new InvalidTypeNameException("mapping type name [" + mapper.type() + "] should not include ',' in it");
-        }
-        if (mapper.type().equals(mapper.parentFieldMapper().type())) {
-            throw new IllegalArgumentException("The [_parent.type] option can't point to the same type");
-        }
-        if (typeNameStartsWithIllegalDot(mapper)) {
-            throw new IllegalArgumentException("mapping type name [" + mapper.type() + "] must not start with a '.'");
-        }
+    private synchronized Map<String, DocumentMapper> internalMerge(@Nullable DocumentMapper defaultMapper, @Nullable String defaultMappingSource,
+                                                                   List<DocumentMapper> documentMappers, MergeReason reason, boolean updateAllTypes) {
+        boolean hasNested = this.hasNested;
+        boolean allEnabled = this.allEnabled;
+        Map<String, ObjectMapper> fullPathObjectMappers = this.fullPathObjectMappers;
+        FieldTypeLookup fieldTypes = this.fieldTypes;
+        Set<String> parentTypes = this.parentTypes;
+        Map<String, DocumentMapper> mappers = new HashMap<>(this.mappers);
 
-        // 1. compute the merged DocumentMapper
-        DocumentMapper oldMapper = mappers.get(mapper.type());
-        DocumentMapper newMapper;
-        if (oldMapper != null) {
-            newMapper = oldMapper.merge(mapper.mapping(), updateAllTypes);
-        } else {
-            newMapper = mapper;
+        Map<String, DocumentMapper> results = new LinkedHashMap<>(documentMappers.size() + 1);
+
+        if (defaultMapper != null) {
+            assert defaultMapper.type().equals(DEFAULT_MAPPING);
+            mappers.put(DEFAULT_MAPPING, defaultMapper);
+            results.put(DEFAULT_MAPPING, defaultMapper);
         }
 
-        // 2. check basic sanity of the new mapping
-        List<ObjectMapper> objectMappers = new ArrayList<>();
-        List<FieldMapper> fieldMappers = new ArrayList<>();
-        Collections.addAll(fieldMappers, newMapper.mapping().metadataMappers);
-        MapperUtils.collect(newMapper.mapping().root(), objectMappers, fieldMappers);
-        checkFieldUniqueness(newMapper.type(), objectMappers, fieldMappers);
-        checkObjectsCompatibility(objectMappers, updateAllTypes);
+        for (DocumentMapper mapper : documentMappers) {
+            // check naming
+            if (mapper.type().length() == 0) {
+                throw new InvalidTypeNameException("mapping type name is empty");
+            }
+            if (mapper.type().length() > 255) {
+                throw new InvalidTypeNameException("mapping type name [" + mapper.type() + "] is too long; limit is length 255 but was [" + mapper.type().length() + "]");
+            }
+            if (mapper.type().charAt(0) == '_') {
+                throw new InvalidTypeNameException("mapping type name [" + mapper.type() + "] can't start with '_'");
+            }
+            if (mapper.type().contains("#")) {
+                throw new InvalidTypeNameException("mapping type name [" + mapper.type() + "] should not include '#' in it");
+            }
+            if (mapper.type().contains(",")) {
+                throw new InvalidTypeNameException("mapping type name [" + mapper.type() + "] should not include ',' in it");
+            }
+            if (mapper.type().equals(mapper.parentFieldMapper().type())) {
+                throw new IllegalArgumentException("The [_parent.type] option can't point to the same type");
+            }
+            if (typeNameStartsWithIllegalDot(mapper)) {
+                throw new IllegalArgumentException("mapping type name [" + mapper.type() + "] must not start with a '.'");
+            }
+
+            // compute the merged DocumentMapper
+            DocumentMapper oldMapper = mappers.get(mapper.type());
+            DocumentMapper newMapper;
+            if (oldMapper != null) {
+                newMapper = oldMapper.merge(mapper.mapping(), updateAllTypes);
+            } else {
+                newMapper = mapper;
+            }
 
-        // 3. update lookup data-structures
-        // this will in particular make sure that the merged fields are compatible with other types
-        FieldTypeLookup fieldTypes = this.fieldTypes.copyAndAddAll(newMapper.type(), fieldMappers, updateAllTypes);
+            // check basic sanity of the new mapping
+            List<ObjectMapper> objectMappers = new ArrayList<>();
+            List<FieldMapper> fieldMappers = new ArrayList<>();
+            Collections.addAll(fieldMappers, newMapper.mapping().metadataMappers);
+            MapperUtils.collect(newMapper.mapping().root(), objectMappers, fieldMappers);
+            checkFieldUniqueness(newMapper.type(), objectMappers, fieldMappers, fullPathObjectMappers, fieldTypes);
+            checkObjectsCompatibility(objectMappers, updateAllTypes, fullPathObjectMappers);
+            checkPartitionedIndexConstraints(newMapper);
+
+            // update lookup data-structures
+            // this will in particular make sure that the merged fields are compatible with other types
+            fieldTypes = fieldTypes.copyAndAddAll(newMapper.type(), fieldMappers, updateAllTypes);
+
+            for (ObjectMapper objectMapper : objectMappers) {
+                if (fullPathObjectMappers == this.fullPathObjectMappers) {
+                    // first time through the loops
+                    fullPathObjectMappers = new HashMap<>(this.fullPathObjectMappers);
+                }
+                fullPathObjectMappers.put(objectMapper.fullPath(), objectMapper);
 
-        boolean hasNested = this.hasNested;
-        Map<String, ObjectMapper> fullPathObjectMappers = new HashMap<>(this.fullPathObjectMappers);
-        for (ObjectMapper objectMapper : objectMappers) {
-            fullPathObjectMappers.put(objectMapper.fullPath(), objectMapper);
-            if (objectMapper.nested().isNested()) {
-                hasNested = true;
+                if (objectMapper.nested().isNested()) {
+                    hasNested = true;
+                }
+            }
+
+            if (reason == MergeReason.MAPPING_UPDATE) {
+                // this check will only be performed on the master node when there is
+                // a call to the update mapping API. For all other cases like
+                // the master node restoring mappings from disk or data nodes
+                // deserializing cluster state that was sent by the master node,
+                // this check will be skipped.
+                checkTotalFieldsLimit(objectMappers.size() + fieldMappers.size());
             }
+
+            if (oldMapper == null && newMapper.parentFieldMapper().active()) {
+                if (parentTypes == this.parentTypes) {
+                    // first time through the loop
+                    parentTypes = new HashSet<>(this.parentTypes);
+                }
+                parentTypes.add(mapper.parentFieldMapper().type());
+            }
+
+            // this is only correct because types cannot be removed and we do not
+            // allow to disable an existing _all field
+            allEnabled |= mapper.allFieldMapper().enabled();
+
+            results.put(newMapper.type(), newMapper);
+            mappers.put(newMapper.type(), newMapper);
         }
-        fullPathObjectMappers = Collections.unmodifiableMap(fullPathObjectMappers);
 
         if (reason == MergeReason.MAPPING_UPDATE) {
             // this check will only be performed on the master node when there is
@@ -340,42 +443,62 @@ private synchronized DocumentMapper merge(DocumentMapper mapper, MergeReason rea
             // deserializing cluster state that was sent by the master node,
             // this check will be skipped.
             checkNestedFieldsLimit(fullPathObjectMappers);
-            checkTotalFieldsLimit(objectMappers.size() + fieldMappers.size());
             checkDepthLimit(fullPathObjectMappers.keySet());
         }
 
-        Set<String> parentTypes = this.parentTypes;
-        if (oldMapper == null && newMapper.parentFieldMapper().active()) {
-            parentTypes = new HashSet<>(parentTypes.size() + 1);
-            parentTypes.addAll(this.parentTypes);
-            parentTypes.add(mapper.parentFieldMapper().type());
-            parentTypes = Collections.unmodifiableSet(parentTypes);
-        }
-
-        Map<String, DocumentMapper> mappers = new HashMap<>(this.mappers);
-        mappers.put(newMapper.type(), newMapper);
         for (Map.Entry<String, DocumentMapper> entry : mappers.entrySet()) {
             if (entry.getKey().equals(DEFAULT_MAPPING)) {
                 continue;
             }
-            DocumentMapper m = entry.getValue();
+            DocumentMapper documentMapper = entry.getValue();
             // apply changes to the field types back
-            m = m.updateFieldType(fieldTypes.fullNameToFieldType);
-            entry.setValue(m);
+            DocumentMapper updatedDocumentMapper = documentMapper.updateFieldType(fieldTypes.fullNameToFieldType);
+            if (updatedDocumentMapper != documentMapper) {
+                // update both mappers and result
+                entry.setValue(updatedDocumentMapper);
+                if (results.containsKey(updatedDocumentMapper.type())) {
+                    results.put(updatedDocumentMapper.type(), updatedDocumentMapper);
+                }
+            }
         }
+
+        if (indexSettings.isSingleType()) {
+            Set<String> actualTypes = new HashSet<>(mappers.keySet());
+            actualTypes.remove(DEFAULT_MAPPING);
+            if (actualTypes.size() > 1) {
+                throw new IllegalArgumentException(
+                        "Rejecting mapping update to [" + index().getName() + "] as the final mapping would have more than 1 type: " + actualTypes);
+            }
+        }
+
+        // make structures immutable
         mappers = Collections.unmodifiableMap(mappers);
+        results = Collections.unmodifiableMap(results);
+
+        // only need to immutably rewrap these if the previous reference was changed.
+        // if not then they are already implicitly immutable.
+        if (fullPathObjectMappers != this.fullPathObjectMappers) {
+            fullPathObjectMappers = Collections.unmodifiableMap(fullPathObjectMappers);
+        }
+        if (parentTypes != this.parentTypes) {
+            parentTypes = Collections.unmodifiableSet(parentTypes);
+        }
 
-        // 4. commit the change
+        // commit the change
+        if (defaultMappingSource != null) {
+            this.defaultMappingSource = defaultMappingSource;
+        }
         this.mappers = mappers;
         this.fieldTypes = fieldTypes;
         this.hasNested = hasNested;
         this.fullPathObjectMappers = fullPathObjectMappers;
         this.parentTypes = parentTypes;
+        this.allEnabled = allEnabled;
 
-        assert assertSerialization(newMapper);
         assert assertMappersShareSameFieldType();
+        assert results.values().stream().allMatch(this::assertSerialization);
 
-        return newMapper;
+        return results;
     }
 
     private boolean assertMappersShareSameFieldType() {
@@ -412,8 +535,8 @@ private boolean assertSerialization(DocumentMapper mapper) {
         return true;
     }
 
-    private void checkFieldUniqueness(String type, Collection<ObjectMapper> objectMappers, Collection<FieldMapper> fieldMappers) {
-        assert Thread.holdsLock(this);
+    private static void checkFieldUniqueness(String type, Collection<ObjectMapper> objectMappers, Collection<FieldMapper> fieldMappers,
+                                             Map<String, ObjectMapper> fullPathObjectMappers, FieldTypeLookup fieldTypes) {
 
         // first check within mapping
         final Set<String> objectFullNames = new HashSet<>();
@@ -450,9 +573,8 @@ private void checkFieldUniqueness(String type, Collection<ObjectMapper> objectMa
         }
     }
 
-    private void checkObjectsCompatibility(Collection<ObjectMapper> objectMappers, boolean updateAllTypes) {
-        assert Thread.holdsLock(this);
-
+    private static void checkObjectsCompatibility(Collection<ObjectMapper> objectMappers, boolean updateAllTypes,
+                                                  Map<String, ObjectMapper> fullPathObjectMappers) {
         for (ObjectMapper newObjectMapper : objectMappers) {
             ObjectMapper existingObjectMapper = fullPathObjectMappers.get(newObjectMapper.fullPath());
             if (existingObjectMapper != null) {
@@ -504,6 +626,20 @@ private void checkDepthLimit(String objectPath, long maxDepth) {
         }
     }
 
+    private void checkPartitionedIndexConstraints(DocumentMapper newMapper) {
+        if (indexSettings.getIndexMetaData().isRoutingPartitionedIndex()) {
+            if (newMapper.parentFieldMapper().active()) {
+                throw new IllegalArgumentException("mapping type name [" + newMapper.type() + "] cannot have a "
+                        + "_parent field for the partitioned index [" + indexSettings.getIndex().getName() + "]");
+            }
+
+            if (!newMapper.routingFieldMapper().required()) {
+                throw new IllegalArgumentException("mapping type [" + newMapper.type() + "] must have routing "
+                        + "required for partitioned index [" + indexSettings.getIndex().getName() + "]");
+            }
+        }
+    }
+
     public DocumentMapper parse(String mappingType, CompressedXContent mappingSource, boolean applyDefault) throws MapperParsingException {
         return documentParser.parse(mappingType, mappingSource, applyDefault ? defaultMappingSource : null);
     }
@@ -588,14 +724,13 @@ public MappedFieldType unmappedFieldType(String type) {
             if (typeParser == null) {
                 throw new IllegalArgumentException("No mapper found for type [" + type + "]");
             }
-            final Mapper.Builder<?, ?> builder = typeParser.parse("__anonymous_" + type, emptyMap(), parserContext);
+            final Mapper.Builder<?, ?> builder = typeParser.parse("__anonymous_" + type, new HashMap<>(), parserContext);
             final BuilderContext builderContext = new BuilderContext(indexSettings.getSettings(), new ContentPath(1));
             fieldType = ((FieldMapper)builder.build(builderContext)).fieldType();
 
             // There is no need to synchronize writes here. In the case of concurrent access, we could just
             // compute some mappers several times, which is not a big deal
-            Map<String, MappedFieldType> newUnmappedFieldTypes = new HashMap<>();
-            newUnmappedFieldTypes.putAll(unmappedFieldTypes);
+            Map<String, MappedFieldType> newUnmappedFieldTypes = new HashMap<>(unmappedFieldTypes);
             newUnmappedFieldTypes.put(type, fieldType);
             unmappedFieldTypes = unmodifiableMap(newUnmappedFieldTypes);
         }
@@ -618,6 +753,11 @@ public Set<String> getParentTypes() {
         return parentTypes;
     }
 
+    @Override
+    public void close() throws IOException {
+        indexAnalyzers.close();
+    }
+
     /**
      * @return Whether a field is a metadata field.
      */
@@ -653,4 +793,16 @@ protected Analyzer getWrappedAnalyzer(String fieldName) {
             return defaultAnalyzer;
         }
     }
+
+    /** Return a term that uniquely identifies the document, or {@code null} if the type is not allowed. */
+    public Term createUidTerm(String type, String id) {
+        if (hasMapping(type) == false) {
+            return null;
+        }
+        if (indexSettings.isSingleType()) {
+            return new Term(IdFieldMapper.NAME, id);
+        } else {
+            return new Term(UidFieldMapper.NAME, Uid.createUidAsBytes(type, id));
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/Mapping.java b/core/src/main/java/org/elasticsearch/index/mapper/Mapping.java
index 0b92dbe45173a..9cb3ea7a57e48 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/Mapping.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/Mapping.java
@@ -104,12 +104,22 @@ public Mapping merge(Mapping mergeWith, boolean updateAllTypes) {
      * Recursively update sub field types.
      */
     public Mapping updateFieldType(Map<String, MappedFieldType> fullNameToFieldType) {
-        final MetadataFieldMapper[] updatedMeta = Arrays.copyOf(metadataMappers, metadataMappers.length);
-        for (int i = 0; i < updatedMeta.length; ++i) {
-            updatedMeta[i] = (MetadataFieldMapper) updatedMeta[i].updateFieldType(fullNameToFieldType);
+        MetadataFieldMapper[] updatedMeta = null;
+        for (int i = 0; i < metadataMappers.length; ++i) {
+            MetadataFieldMapper currentFieldMapper = metadataMappers[i];
+            MetadataFieldMapper updatedFieldMapper = (MetadataFieldMapper) currentFieldMapper.updateFieldType(fullNameToFieldType);
+            if (updatedFieldMapper != currentFieldMapper) {
+                if (updatedMeta == null) {
+                    updatedMeta = Arrays.copyOf(metadataMappers, metadataMappers.length);
+                }
+                updatedMeta[i] = updatedFieldMapper;
+            }
         }
         RootObjectMapper updatedRoot = root.updateFieldType(fullNameToFieldType);
-        return new Mapping(indexCreated, updatedRoot, updatedMeta, meta);
+        if (updatedMeta == null && updatedRoot == root) {
+            return this;
+        }
+        return new Mapping(indexCreated, updatedRoot, updatedMeta == null ? metadataMappers : updatedMeta, meta);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java
index 07a4b3b9a5142..ec84631e04161 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java
@@ -39,14 +39,13 @@ public interface TypeParser extends Mapper.TypeParser {
          * Get the default {@link MetadataFieldMapper} to use, if nothing had to be parsed.
          * @param fieldType null if this is the first root mapper on this index, the existing
          *                  fieldType for this index otherwise
-         * @param indexSettings  the index-level settings
          * @param fieldType      the existing field type for this meta mapper on the current index
          *                       or null if this is the first type being introduced
-         * @param typeName       the name of the type that this mapper will be used on
+         * @param parserContext context that may be useful to build the field like analyzers
          */
         // TODO: remove the fieldType parameter which is only used for bw compat with pre-2.0
         // since settings could be modified
-        MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName);
+        MetadataFieldMapper getDefault(MappedFieldType fieldType, ParserContext parserContext);
     }
 
     public abstract static class Builder<T extends Builder, Y extends MetadataFieldMapper> extends FieldMapper.Builder<T, Y> {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java
index 886b93fcf0e3a..fc17508fcd24d 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java
@@ -27,10 +27,13 @@
 import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.document.StoredField;
+import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.XPointValues;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.PointValues;
 import org.apache.lucene.search.BoostQuery;
+import org.apache.lucene.search.IndexOrDocValuesQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
@@ -38,6 +41,7 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.action.fieldstats.FieldStats;
 import org.elasticsearch.common.Explicit;
+import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
@@ -54,6 +58,7 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -158,13 +163,13 @@ public Mapper.Builder<?,?> parse(String name, Map<String, Object> node,
                     if (propNode == null) {
                         throw new MapperParsingException("Property [null_value] cannot be null.");
                     }
-                    builder.nullValue(type.parse(propNode));
+                    builder.nullValue(type.parse(propNode, false));
                     iterator.remove();
                 } else if (propName.equals("ignore_malformed")) {
-                    builder.ignoreMalformed(TypeParsers.nodeBooleanValue("ignore_malformed", propNode, parserContext));
+                    builder.ignoreMalformed(TypeParsers.nodeBooleanValue(name,"ignore_malformed", propNode));
                     iterator.remove();
                 } else if (propName.equals("coerce")) {
-                    builder.coerce(TypeParsers.nodeBooleanValue("coerce", propNode, parserContext));
+                    builder.coerce(TypeParsers.nodeBooleanValue(name, "coerce", propNode));
                     iterator.remove();
                 }
             }
@@ -175,8 +180,8 @@ public Mapper.Builder<?,?> parse(String name, Map<String, Object> node,
     public enum NumberType {
         HALF_FLOAT("half_float", NumericType.HALF_FLOAT) {
             @Override
-            Float parse(Object value) {
-                return (Float) FLOAT.parse(value);
+            Float parse(Object value, boolean coerce) {
+                return (Float) FLOAT.parse(value, false);
             }
 
             @Override
@@ -186,7 +191,7 @@ Float parse(XContentParser parser, boolean coerce) throws IOException {
 
             @Override
             Query termQuery(String field, Object value) {
-                float v = parse(value);
+                float v = parse(value, false);
                 return HalfFloatPoint.newExactQuery(field, v);
             }
 
@@ -194,31 +199,39 @@ Query termQuery(String field, Object value) {
             Query termsQuery(String field, List<Object> values) {
                 float[] v = new float[values.size()];
                 for (int i = 0; i < values.size(); ++i) {
-                    v[i] = parse(values.get(i));
+                    v[i] = parse(values.get(i), false);
                 }
                 return HalfFloatPoint.newSetQuery(field, v);
             }
 
             @Override
             Query rangeQuery(String field, Object lowerTerm, Object upperTerm,
-                             boolean includeLower, boolean includeUpper) {
+                             boolean includeLower, boolean includeUpper,
+                             boolean hasDocValues) {
                 float l = Float.NEGATIVE_INFINITY;
                 float u = Float.POSITIVE_INFINITY;
                 if (lowerTerm != null) {
-                    l = parse(lowerTerm);
+                    l = parse(lowerTerm, false);
                     if (includeLower) {
-                        l = Math.nextDown(l);
+                        l = HalfFloatPoint.nextDown(l);
                     }
                     l = HalfFloatPoint.nextUp(l);
                 }
                 if (upperTerm != null) {
-                    u = parse(upperTerm);
+                    u = parse(upperTerm, false);
                     if (includeUpper) {
-                        u = Math.nextUp(u);
+                        u = HalfFloatPoint.nextUp(u);
                     }
                     u = HalfFloatPoint.nextDown(u);
                 }
-                return HalfFloatPoint.newRangeQuery(field, l, u);
+                Query query = HalfFloatPoint.newRangeQuery(field, l, u);
+                if (hasDocValues) {
+                    Query dvQuery = SortedNumericDocValuesField.newRangeQuery(field,
+                            HalfFloatPoint.halfFloatToSortableShort(l),
+                            HalfFloatPoint.halfFloatToSortableShort(u));
+                    query = new IndexOrDocValuesQuery(query, dvQuery);
+                }
+                return query;
             }
 
             @Override
@@ -241,21 +254,25 @@ public List<Field> createFields(String name, Number value,
             @Override
             FieldStats.Double stats(IndexReader reader, String fieldName,
                                     boolean isSearchable, boolean isAggregatable) throws IOException {
-                long size = XPointValues.size(reader, fieldName);
-                if (size == 0) {
+                FieldInfo fi = org.apache.lucene.index.MultiFields.getMergedFieldInfos(reader).fieldInfo(fieldName);
+                if (fi == null) {
                     return null;
                 }
-                int docCount = XPointValues.getDocCount(reader, fieldName);
-                byte[] min = XPointValues.getMinPackedValue(reader, fieldName);
-                byte[] max = XPointValues.getMaxPackedValue(reader, fieldName);
-                return new FieldStats.Double(reader.maxDoc(),docCount, -1L, size,
+                long size = PointValues.size(reader, fieldName);
+                if (size == 0) {
+                    return new FieldStats.Double(reader.maxDoc(), 0, -1, -1, isSearchable, isAggregatable);
+                }
+                int docCount = PointValues.getDocCount(reader, fieldName);
+                byte[] min = PointValues.getMinPackedValue(reader, fieldName);
+                byte[] max = PointValues.getMaxPackedValue(reader, fieldName);
+                return new FieldStats.Double(reader.maxDoc(), docCount, -1L, size,
                     isSearchable, isAggregatable,
                     HalfFloatPoint.decodeDimension(min, 0), HalfFloatPoint.decodeDimension(max, 0));
             }
         },
         FLOAT("float", NumericType.FLOAT) {
             @Override
-            Float parse(Object value) {
+            Float parse(Object value, boolean coerce) {
                 if (value instanceof Number) {
                     return ((Number) value).floatValue();
                 }
@@ -272,7 +289,7 @@ Float parse(XContentParser parser, boolean coerce) throws IOException {
 
             @Override
             Query termQuery(String field, Object value) {
-                float v = parse(value);
+                float v = parse(value, false);
                 return FloatPoint.newExactQuery(field, v);
             }
 
@@ -280,29 +297,37 @@ Query termQuery(String field, Object value) {
             Query termsQuery(String field, List<Object> values) {
                 float[] v = new float[values.size()];
                 for (int i = 0; i < values.size(); ++i) {
-                    v[i] = parse(values.get(i));
+                    v[i] = parse(values.get(i), false);
                 }
                 return FloatPoint.newSetQuery(field, v);
             }
 
             @Override
             Query rangeQuery(String field, Object lowerTerm, Object upperTerm,
-                             boolean includeLower, boolean includeUpper) {
+                             boolean includeLower, boolean includeUpper,
+                             boolean hasDocValues) {
                 float l = Float.NEGATIVE_INFINITY;
                 float u = Float.POSITIVE_INFINITY;
                 if (lowerTerm != null) {
-                    l = parse(lowerTerm);
+                    l = parse(lowerTerm, false);
                     if (includeLower == false) {
-                        l = Math.nextUp(l);
+                        l = FloatPoint.nextUp(l);
                     }
                 }
                 if (upperTerm != null) {
-                    u = parse(upperTerm);
+                    u = parse(upperTerm, false);
                     if (includeUpper == false) {
-                        u = Math.nextDown(u);
+                        u = FloatPoint.nextDown(u);
                     }
                 }
-                return FloatPoint.newRangeQuery(field, l, u);
+                Query query = FloatPoint.newRangeQuery(field, l, u);
+                if (hasDocValues) {
+                    Query dvQuery = SortedNumericDocValuesField.newRangeQuery(field,
+                            NumericUtils.floatToSortableInt(l),
+                            NumericUtils.floatToSortableInt(u));
+                    query = new IndexOrDocValuesQuery(query, dvQuery);
+                }
+                return query;
             }
 
             @Override
@@ -325,13 +350,17 @@ public List<Field> createFields(String name, Number value,
             @Override
             FieldStats.Double stats(IndexReader reader, String fieldName,
                                     boolean isSearchable, boolean isAggregatable) throws IOException {
-                long size = XPointValues.size(reader, fieldName);
-                if (size == 0) {
+                FieldInfo fi = org.apache.lucene.index.MultiFields.getMergedFieldInfos(reader).fieldInfo(fieldName);
+                if (fi == null) {
                     return null;
                 }
-                int docCount = XPointValues.getDocCount(reader, fieldName);
-                byte[] min = XPointValues.getMinPackedValue(reader, fieldName);
-                byte[] max = XPointValues.getMaxPackedValue(reader, fieldName);
+                long size = PointValues.size(reader, fieldName);
+                if (size == 0) {
+                    return new FieldStats.Double(reader.maxDoc(), 0, -1, -1, isSearchable, isAggregatable);
+                }
+                int docCount = PointValues.getDocCount(reader, fieldName);
+                byte[] min = PointValues.getMinPackedValue(reader, fieldName);
+                byte[] max = PointValues.getMaxPackedValue(reader, fieldName);
                 return new FieldStats.Double(reader.maxDoc(),docCount, -1L, size,
                     isSearchable, isAggregatable,
                     FloatPoint.decodeDimension(min, 0), FloatPoint.decodeDimension(max, 0));
@@ -339,14 +368,8 @@ FieldStats.Double stats(IndexReader reader, String fieldName,
         },
         DOUBLE("double", NumericType.DOUBLE) {
             @Override
-            Double parse(Object value) {
-                if (value instanceof Number) {
-                    return ((Number) value).doubleValue();
-                }
-                if (value instanceof BytesRef) {
-                    value = ((BytesRef) value).utf8ToString();
-                }
-                return Double.parseDouble(value.toString());
+            Double parse(Object value, boolean coerce) {
+                return objectToDouble(value);
             }
 
             @Override
@@ -356,7 +379,7 @@ Double parse(XContentParser parser, boolean coerce) throws IOException {
 
             @Override
             Query termQuery(String field, Object value) {
-                double v = parse(value);
+                double v = parse(value, false);
                 return DoublePoint.newExactQuery(field, v);
             }
 
@@ -364,29 +387,37 @@ Query termQuery(String field, Object value) {
             Query termsQuery(String field, List<Object> values) {
                 double[] v = new double[values.size()];
                 for (int i = 0; i < values.size(); ++i) {
-                    v[i] = parse(values.get(i));
+                    v[i] = parse(values.get(i), false);
                 }
                 return DoublePoint.newSetQuery(field, v);
             }
 
             @Override
             Query rangeQuery(String field, Object lowerTerm, Object upperTerm,
-                             boolean includeLower, boolean includeUpper) {
+                             boolean includeLower, boolean includeUpper,
+                             boolean hasDocValues) {
                 double l = Double.NEGATIVE_INFINITY;
                 double u = Double.POSITIVE_INFINITY;
                 if (lowerTerm != null) {
-                    l = parse(lowerTerm);
+                    l = parse(lowerTerm, false);
                     if (includeLower == false) {
-                        l = Math.nextUp(l);
+                        l = DoublePoint.nextUp(l);
                     }
                 }
                 if (upperTerm != null) {
-                    u = parse(upperTerm);
+                    u = parse(upperTerm, false);
                     if (includeUpper == false) {
-                        u = Math.nextDown(u);
+                        u = DoublePoint.nextDown(u);
                     }
                 }
-                return DoublePoint.newRangeQuery(field, l, u);
+                Query query = DoublePoint.newRangeQuery(field, l, u);
+                if (hasDocValues) {
+                    Query dvQuery = SortedNumericDocValuesField.newRangeQuery(field,
+                            NumericUtils.doubleToSortableLong(l),
+                            NumericUtils.doubleToSortableLong(u));
+                    query = new IndexOrDocValuesQuery(query, dvQuery);
+                }
+                return query;
             }
 
             @Override
@@ -409,13 +440,17 @@ public List<Field> createFields(String name, Number value,
             @Override
             FieldStats.Double stats(IndexReader reader, String fieldName,
                                     boolean isSearchable, boolean isAggregatable) throws IOException {
-                long size = XPointValues.size(reader, fieldName);
-                if (size == 0) {
+                FieldInfo fi = org.apache.lucene.index.MultiFields.getMergedFieldInfos(reader).fieldInfo(fieldName);
+                if (fi == null) {
                     return null;
                 }
-                int docCount = XPointValues.getDocCount(reader, fieldName);
-                byte[] min = XPointValues.getMinPackedValue(reader, fieldName);
-                byte[] max = XPointValues.getMaxPackedValue(reader, fieldName);
+                long size = PointValues.size(reader, fieldName);
+                if (size == 0) {
+                    return new FieldStats.Double(reader.maxDoc(),0, -1, -1, isSearchable, isAggregatable);
+                }
+                int docCount = PointValues.getDocCount(reader, fieldName);
+                byte[] min = PointValues.getMinPackedValue(reader, fieldName);
+                byte[] max = PointValues.getMaxPackedValue(reader, fieldName);
                 return new FieldStats.Double(reader.maxDoc(),docCount, -1L, size,
                     isSearchable, isAggregatable,
                     DoublePoint.decodeDimension(min, 0), DoublePoint.decodeDimension(max, 0));
@@ -423,21 +458,21 @@ FieldStats.Double stats(IndexReader reader, String fieldName,
         },
         BYTE("byte", NumericType.BYTE) {
             @Override
-            Byte parse(Object value) {
+            Byte parse(Object value, boolean coerce) {
+                double doubleValue = objectToDouble(value);
+
+                if (doubleValue < Byte.MIN_VALUE || doubleValue > Byte.MAX_VALUE) {
+                    throw new IllegalArgumentException("Value [" + value + "] is out of range for a byte");
+                }
+                if (!coerce && doubleValue % 1 != 0) {
+                    throw new IllegalArgumentException("Value [" + value + "] has a decimal part");
+                }
+
                 if (value instanceof Number) {
-                    double doubleValue = ((Number) value).doubleValue();
-                    if (doubleValue < Byte.MIN_VALUE || doubleValue > Byte.MAX_VALUE) {
-                        throw new IllegalArgumentException("Value [" + value + "] is out of range for a byte");
-                    }
-                    if (doubleValue % 1 != 0) {
-                        throw new IllegalArgumentException("Value [" + value + "] has a decimal part");
-                    }
                     return ((Number) value).byteValue();
                 }
-                if (value instanceof BytesRef) {
-                    value = ((BytesRef) value).utf8ToString();
-                }
-                return Byte.parseByte(value.toString());
+
+                return (byte) doubleValue;
             }
 
             @Override
@@ -461,8 +496,9 @@ Query termsQuery(String field, List<Object> values) {
 
             @Override
             Query rangeQuery(String field, Object lowerTerm, Object upperTerm,
-                             boolean includeLower, boolean includeUpper) {
-                return INTEGER.rangeQuery(field, lowerTerm, upperTerm, includeLower, includeUpper);
+                             boolean includeLower, boolean includeUpper,
+                             boolean hasDocValues) {
+                return INTEGER.rangeQuery(field, lowerTerm, upperTerm, includeLower, includeUpper, hasDocValues);
             }
 
             @Override
@@ -484,30 +520,26 @@ Number valueForSearch(Number value) {
         },
         SHORT("short", NumericType.SHORT) {
             @Override
-            Short parse(Object value) {
+            Short parse(Object value, boolean coerce) {
+                double doubleValue = objectToDouble(value);
+
+                if (doubleValue < Short.MIN_VALUE || doubleValue > Short.MAX_VALUE) {
+                    throw new IllegalArgumentException("Value [" + value + "] is out of range for a short");
+                }
+                if (!coerce && doubleValue % 1 != 0) {
+                    throw new IllegalArgumentException("Value [" + value + "] has a decimal part");
+                }
+
                 if (value instanceof Number) {
-                    double doubleValue = ((Number) value).doubleValue();
-                    if (doubleValue < Short.MIN_VALUE || doubleValue > Short.MAX_VALUE) {
-                        throw new IllegalArgumentException("Value [" + value + "] is out of range for a short");
-                    }
-                    if (doubleValue % 1 != 0) {
-                        throw new IllegalArgumentException("Value [" + value + "] has a decimal part");
-                    }
                     return ((Number) value).shortValue();
                 }
-                if (value instanceof BytesRef) {
-                    value = ((BytesRef) value).utf8ToString();
-                }
-                return Short.parseShort(value.toString());
+
+                return (short) doubleValue;
             }
 
             @Override
             Short parse(XContentParser parser, boolean coerce) throws IOException {
-                int value = parser.intValue(coerce);
-                if (value < Short.MIN_VALUE || value > Short.MAX_VALUE) {
-                    throw new IllegalArgumentException("Value [" + value + "] is out of range for a short");
-                }
-                return (short) value;
+                return parser.shortValue(coerce);
             }
 
             @Override
@@ -522,8 +554,9 @@ Query termsQuery(String field, List<Object> values) {
 
             @Override
             Query rangeQuery(String field, Object lowerTerm, Object upperTerm,
-                             boolean includeLower, boolean includeUpper) {
-                return INTEGER.rangeQuery(field, lowerTerm, upperTerm, includeLower, includeUpper);
+                             boolean includeLower, boolean includeUpper,
+                             boolean hasDocValues) {
+                return INTEGER.rangeQuery(field, lowerTerm, upperTerm, includeLower, includeUpper, hasDocValues);
             }
 
             @Override
@@ -545,21 +578,21 @@ Number valueForSearch(Number value) {
         },
         INTEGER("integer", NumericType.INT) {
             @Override
-            Integer parse(Object value) {
+            Integer parse(Object value, boolean coerce) {
+                double doubleValue = objectToDouble(value);
+
+                if (doubleValue < Integer.MIN_VALUE || doubleValue > Integer.MAX_VALUE) {
+                    throw new IllegalArgumentException("Value [" + value + "] is out of range for an integer");
+                }
+                if (!coerce && doubleValue % 1 != 0) {
+                    throw new IllegalArgumentException("Value [" + value + "] has a decimal part");
+                }
+
                 if (value instanceof Number) {
-                    double doubleValue = ((Number) value).doubleValue();
-                    if (doubleValue < Integer.MIN_VALUE || doubleValue > Integer.MAX_VALUE) {
-                        throw new IllegalArgumentException("Value [" + value + "] is out of range for an integer");
-                    }
-                    if (doubleValue % 1 != 0) {
-                        throw new IllegalArgumentException("Value [" + value + "] has a decimal part");
-                    }
                     return ((Number) value).intValue();
                 }
-                if (value instanceof BytesRef) {
-                    value = ((BytesRef) value).utf8ToString();
-                }
-                return Integer.parseInt(value.toString());
+
+                return (int) doubleValue;
             }
 
             @Override
@@ -569,27 +602,50 @@ Integer parse(XContentParser parser, boolean coerce) throws IOException {
 
             @Override
             Query termQuery(String field, Object value) {
-                int v = parse(value);
+                if (hasDecimalPart(value)) {
+                    return Queries.newMatchNoDocsQuery("Value [" + value + "] has a decimal part");
+                }
+                int v = parse(value, true);
                 return IntPoint.newExactQuery(field, v);
             }
 
             @Override
             Query termsQuery(String field, List<Object> values) {
                 int[] v = new int[values.size()];
-                for (int i = 0; i < values.size(); ++i) {
-                    v[i] = parse(values.get(i));
+                int upTo = 0;
+
+                for (int i = 0; i < values.size(); i++) {
+                    Object value = values.get(i);
+                    if (!hasDecimalPart(value)) {
+                        v[upTo++] = parse(value, true);
+                    }
+                }
+
+                if (upTo == 0) {
+                    return Queries.newMatchNoDocsQuery("All values have a decimal part");
+                }
+                if (upTo != v.length) {
+                    v = Arrays.copyOf(v, upTo);
                 }
                 return IntPoint.newSetQuery(field, v);
             }
 
             @Override
             Query rangeQuery(String field, Object lowerTerm, Object upperTerm,
-                             boolean includeLower, boolean includeUpper) {
+                             boolean includeLower, boolean includeUpper,
+                             boolean hasDocValues) {
                 int l = Integer.MIN_VALUE;
                 int u = Integer.MAX_VALUE;
                 if (lowerTerm != null) {
-                    l = parse(lowerTerm);
-                    if (includeLower == false) {
+                    l = parse(lowerTerm, true);
+                    // if the lower bound is decimal:
+                    // - if the bound is positive then we increment it:
+                    //      if lowerTerm=1.5 then the (inclusive) bound becomes 2
+                    // - if the bound is negative then we leave it as is:
+                    //      if lowerTerm=-1.5 then the (inclusive) bound becomes -1 due to the call to longValue
+                    boolean lowerTermHasDecimalPart = hasDecimalPart(lowerTerm);
+                    if ((lowerTermHasDecimalPart == false && includeLower == false) ||
+                            (lowerTermHasDecimalPart && signum(lowerTerm) > 0)) {
                         if (l == Integer.MAX_VALUE) {
                             return new MatchNoDocsQuery();
                         }
@@ -597,15 +653,22 @@ Query rangeQuery(String field, Object lowerTerm, Object upperTerm,
                     }
                 }
                 if (upperTerm != null) {
-                    u = parse(upperTerm);
-                    if (includeUpper == false) {
+                    u = parse(upperTerm, true);
+                    boolean upperTermHasDecimalPart = hasDecimalPart(upperTerm);
+                    if ((upperTermHasDecimalPart == false && includeUpper == false) ||
+                            (upperTermHasDecimalPart && signum(upperTerm) < 0)) {
                         if (u == Integer.MIN_VALUE) {
                             return new MatchNoDocsQuery();
                         }
                         --u;
                     }
                 }
-                return IntPoint.newRangeQuery(field, l, u);
+                Query query = IntPoint.newRangeQuery(field, l, u);
+                if (hasDocValues) {
+                    Query dvQuery = SortedNumericDocValuesField.newRangeQuery(field, l, u);
+                    query = new IndexOrDocValuesQuery(query, dvQuery);
+                }
+                return query;
             }
 
             @Override
@@ -627,13 +690,17 @@ public List<Field> createFields(String name, Number value,
             @Override
             FieldStats.Long stats(IndexReader reader, String fieldName,
                                   boolean isSearchable, boolean isAggregatable) throws IOException {
-                long size = XPointValues.size(reader, fieldName);
-                if (size == 0) {
+                FieldInfo fi = org.apache.lucene.index.MultiFields.getMergedFieldInfos(reader).fieldInfo(fieldName);
+                if (fi == null) {
                     return null;
                 }
-                int docCount = XPointValues.getDocCount(reader, fieldName);
-                byte[] min = XPointValues.getMinPackedValue(reader, fieldName);
-                byte[] max = XPointValues.getMaxPackedValue(reader, fieldName);
+                long size = PointValues.size(reader, fieldName);
+                if (size == 0) {
+                    return new FieldStats.Long(reader.maxDoc(), 0, -1, -1, isSearchable, isAggregatable);
+                }
+                int docCount = PointValues.getDocCount(reader, fieldName);
+                byte[] min = PointValues.getMinPackedValue(reader, fieldName);
+                byte[] max = PointValues.getMaxPackedValue(reader, fieldName);
                 return new FieldStats.Long(reader.maxDoc(),docCount, -1L, size,
                     isSearchable, isAggregatable,
                     IntPoint.decodeDimension(min, 0), IntPoint.decodeDimension(max, 0));
@@ -641,21 +708,28 @@ FieldStats.Long stats(IndexReader reader, String fieldName,
         },
         LONG("long", NumericType.LONG) {
             @Override
-            Long parse(Object value) {
+            Long parse(Object value, boolean coerce) {
+                double doubleValue = objectToDouble(value);
+
+                if (doubleValue < Long.MIN_VALUE || doubleValue > Long.MAX_VALUE) {
+                    throw new IllegalArgumentException("Value [" + value + "] is out of range for a long");
+                }
+                if (!coerce && doubleValue % 1 != 0) {
+                    throw new IllegalArgumentException("Value [" + value + "] has a decimal part");
+                }
+
                 if (value instanceof Number) {
-                    double doubleValue = ((Number) value).doubleValue();
-                    if (doubleValue < Long.MIN_VALUE || doubleValue > Long.MAX_VALUE) {
-                        throw new IllegalArgumentException("Value [" + value + "] is out of range for a long");
-                    }
-                    if (doubleValue % 1 != 0) {
-                        throw new IllegalArgumentException("Value [" + value + "] has a decimal part");
-                    }
                     return ((Number) value).longValue();
                 }
-                if (value instanceof BytesRef) {
-                    value = ((BytesRef) value).utf8ToString();
+
+                // longs need special handling so we don't lose precision while parsing
+                String stringValue = (value instanceof BytesRef) ? ((BytesRef) value).utf8ToString() : value.toString();
+
+                try {
+                    return Long.parseLong(stringValue);
+                } catch (NumberFormatException e) {
+                    return (long) Double.parseDouble(stringValue);
                 }
-                return Long.parseLong(value.toString());
             }
 
             @Override
@@ -665,27 +739,50 @@ Long parse(XContentParser parser, boolean coerce) throws IOException {
 
             @Override
             Query termQuery(String field, Object value) {
-                long v = parse(value);
+                if (hasDecimalPart(value)) {
+                    return Queries.newMatchNoDocsQuery("Value [" + value + "] has a decimal part");
+                }
+                long v = parse(value, true);
                 return LongPoint.newExactQuery(field, v);
             }
 
             @Override
             Query termsQuery(String field, List<Object> values) {
                 long[] v = new long[values.size()];
-                for (int i = 0; i < values.size(); ++i) {
-                    v[i] = parse(values.get(i));
+                int upTo = 0;
+
+                for (int i = 0; i < values.size(); i++) {
+                    Object value = values.get(i);
+                    if (!hasDecimalPart(value)) {
+                        v[upTo++] = parse(value, true);
+                    }
+                }
+
+                if (upTo == 0) {
+                    return Queries.newMatchNoDocsQuery("All values have a decimal part");
+                }
+                if (upTo != v.length) {
+                    v = Arrays.copyOf(v, upTo);
                 }
                 return LongPoint.newSetQuery(field, v);
             }
 
             @Override
             Query rangeQuery(String field, Object lowerTerm, Object upperTerm,
-                             boolean includeLower, boolean includeUpper) {
+                             boolean includeLower, boolean includeUpper,
+                             boolean hasDocValues) {
                 long l = Long.MIN_VALUE;
                 long u = Long.MAX_VALUE;
                 if (lowerTerm != null) {
-                    l = parse(lowerTerm);
-                    if (includeLower == false) {
+                    l = parse(lowerTerm, true);
+                    // if the lower bound is decimal:
+                    // - if the bound is positive then we increment it:
+                    //      if lowerTerm=1.5 then the (inclusive) bound becomes 2
+                    // - if the bound is negative then we leave it as is:
+                    //      if lowerTerm=-1.5 then the (inclusive) bound becomes -1 due to the call to longValue
+                    boolean lowerTermHasDecimalPart = hasDecimalPart(lowerTerm);
+                    if ((lowerTermHasDecimalPart == false && includeLower == false) ||
+                            (lowerTermHasDecimalPart && signum(lowerTerm) > 0)) {
                         if (l == Long.MAX_VALUE) {
                             return new MatchNoDocsQuery();
                         }
@@ -693,15 +790,22 @@ Query rangeQuery(String field, Object lowerTerm, Object upperTerm,
                     }
                 }
                 if (upperTerm != null) {
-                    u = parse(upperTerm);
-                    if (includeUpper == false) {
+                    u = parse(upperTerm, true);
+                    boolean upperTermHasDecimalPart = hasDecimalPart(upperTerm);
+                    if ((upperTermHasDecimalPart == false && includeUpper == false) ||
+                            (upperTermHasDecimalPart && signum(upperTerm) < 0)) {
                         if (u == Long.MIN_VALUE) {
                             return new MatchNoDocsQuery();
                         }
                         --u;
                     }
                 }
-                return LongPoint.newRangeQuery(field, l, u);
+                Query query = LongPoint.newRangeQuery(field, l, u);
+                if (hasDocValues) {
+                    Query dvQuery = SortedNumericDocValuesField.newRangeQuery(field, l, u);
+                    query = new IndexOrDocValuesQuery(query, dvQuery);
+                }
+                return query;
             }
 
             @Override
@@ -723,13 +827,17 @@ public List<Field> createFields(String name, Number value,
             @Override
             FieldStats.Long stats(IndexReader reader, String fieldName,
                                   boolean isSearchable, boolean isAggregatable) throws IOException {
-                long size = XPointValues.size(reader, fieldName);
-                if (size == 0) {
+                FieldInfo fi = org.apache.lucene.index.MultiFields.getMergedFieldInfos(reader).fieldInfo(fieldName);
+                if (fi == null) {
                     return null;
                 }
-                int docCount = XPointValues.getDocCount(reader, fieldName);
-                byte[] min = XPointValues.getMinPackedValue(reader, fieldName);
-                byte[] max = XPointValues.getMaxPackedValue(reader, fieldName);
+                long size = PointValues.size(reader, fieldName);
+                if (size == 0) {
+                    return new FieldStats.Long(reader.maxDoc(), 0, -1, -1, isSearchable, isAggregatable);
+                }
+                int docCount = PointValues.getDocCount(reader, fieldName);
+                byte[] min = PointValues.getMinPackedValue(reader, fieldName);
+                byte[] max = PointValues.getMaxPackedValue(reader, fieldName);
                 return new FieldStats.Long(reader.maxDoc(),docCount, -1L, size,
                     isSearchable, isAggregatable,
                     LongPoint.decodeDimension(min, 0), LongPoint.decodeDimension(max, 0));
@@ -748,16 +856,17 @@ FieldStats.Long stats(IndexReader reader, String fieldName,
         public final String typeName() {
             return name;
         }
-        /** Get the associated numerit type */
+        /** Get the associated numeric type */
         final NumericType numericType() {
             return numericType;
         }
         abstract Query termQuery(String field, Object value);
         abstract Query termsQuery(String field, List<Object> values);
         abstract Query rangeQuery(String field, Object lowerTerm, Object upperTerm,
-                                  boolean includeLower, boolean includeUpper);
+                                  boolean includeLower, boolean includeUpper,
+                                  boolean hasDocValues);
         abstract Number parse(XContentParser parser, boolean coerce) throws IOException;
-        abstract Number parse(Object value);
+        abstract Number parse(Object value, boolean coerce);
         public abstract List<Field> createFields(String name, Number value, boolean indexed,
                                                  boolean docValued, boolean stored);
         abstract FieldStats<? extends Number> stats(IndexReader reader, String fieldName,
@@ -765,6 +874,55 @@ abstract FieldStats<? extends Number> stats(IndexReader reader, String fieldName
         Number valueForSearch(Number value) {
             return value;
         }
+
+        /**
+         * Returns true if the object is a number and has a decimal part
+         */
+        boolean hasDecimalPart(Object number) {
+            if (number instanceof Number) {
+                double doubleValue = ((Number) number).doubleValue();
+                return doubleValue % 1 != 0;
+            }
+            if (number instanceof BytesRef) {
+                number = ((BytesRef) number).utf8ToString();
+            }
+            if (number instanceof String) {
+                return Double.parseDouble((String) number) % 1 != 0;
+            }
+            return false;
+        }
+
+        /**
+         * Returns -1, 0, or 1 if the value is lower than, equal to, or greater than 0
+         */
+        double signum(Object value) {
+            if (value instanceof Number) {
+                double doubleValue = ((Number) value).doubleValue();
+                return Math.signum(doubleValue);
+            }
+            if (value instanceof BytesRef) {
+                value = ((BytesRef) value).utf8ToString();
+            }
+            return Math.signum(Double.parseDouble(value.toString()));
+        }
+
+        /**
+         * Converts an Object to a double by checking it against known types first
+         */
+        private static double objectToDouble(Object value) {
+            double doubleValue;
+
+            if (value instanceof Number) {
+                doubleValue = ((Number) value).doubleValue();
+            } else if (value instanceof BytesRef) {
+                doubleValue = Double.parseDouble(((BytesRef) value).utf8ToString());
+            } else {
+                doubleValue = Double.parseDouble(value.toString());
+            }
+
+            return doubleValue;
+        }
+
     }
 
     public static final class NumberFieldType extends MappedFieldType {
@@ -815,9 +973,9 @@ public Query termsQuery(List values, QueryShardContext context) {
         }
 
         @Override
-        public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper) {
+        public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) {
             failIfNotIndexed();
-            Query query = type.rangeQuery(name(), lowerTerm, upperTerm, includeLower, includeUpper);
+            Query query = type.rangeQuery(name(), lowerTerm, upperTerm, includeLower, includeUpper, hasDocValues());
             if (boost() != 1f) {
                 query = new BoostQuery(query, boost());
             }
@@ -836,7 +994,7 @@ public IndexFieldData.Builder fielddataBuilder() {
         }
 
         @Override
-        public Object valueForSearch(Object value) {
+        public Object valueForDisplay(Object value) {
             if (value == null) {
                 return null;
             }
@@ -895,7 +1053,7 @@ protected NumberFieldMapper clone() {
     }
 
     @Override
-    protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
+    protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
         final boolean includeInAll = context.includeInAll(this.includeInAll, this);
 
         XContentParser parser = context.parser();
@@ -935,7 +1093,7 @@ protected void parseCreateField(ParseContext context, List<Field> fields) throws
         }
 
         if (numericValue == null) {
-            numericValue = fieldType().type.parse(value);
+            numericValue = fieldType().type.parse(value, coerce.value());
         }
 
         if (includeInAll) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java
index e063629bd6107..4072732b28565 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java
@@ -42,11 +42,6 @@
 import java.util.Locale;
 import java.util.Map;
 
-import static org.elasticsearch.common.xcontent.support.XContentMapValues.lenientNodeBooleanValue;
-
-/**
- *
- */
 public class ObjectMapper extends Mapper implements Cloneable {
 
     public static final String CONTENT_TYPE = "object";
@@ -58,7 +53,7 @@ public static class Defaults {
         public static final Dynamic DYNAMIC = null; // not set, inherited from root
     }
 
-    public static enum Dynamic {
+    public enum Dynamic {
         TRUE,
         FALSE,
         STRICT
@@ -188,11 +183,12 @@ protected static boolean parseObjectOrDocumentTypeProperties(String fieldName, O
                 if (value.equalsIgnoreCase("strict")) {
                     builder.dynamic(Dynamic.STRICT);
                 } else {
-                    builder.dynamic(lenientNodeBooleanValue(fieldNode) ? Dynamic.TRUE : Dynamic.FALSE);
+                    boolean dynamic = TypeParsers.nodeBooleanValue(fieldName, "dynamic", fieldNode);
+                    builder.dynamic(dynamic ? Dynamic.TRUE : Dynamic.FALSE);
                 }
                 return true;
             } else if (fieldName.equals("enabled")) {
-                builder.enabled(lenientNodeBooleanValue(fieldNode));
+                builder.enabled(TypeParsers.nodeBooleanValue(fieldName, "enabled", fieldNode));
                 return true;
             } else if (fieldName.equals("properties")) {
                 if (fieldNode instanceof Collection && ((Collection) fieldNode).isEmpty()) {
@@ -204,7 +200,7 @@ protected static boolean parseObjectOrDocumentTypeProperties(String fieldName, O
                 }
                 return true;
             } else if (fieldName.equals("include_in_all")) {
-                builder.includeInAll(lenientNodeBooleanValue(fieldNode));
+                builder.includeInAll(TypeParsers.nodeBooleanValue(fieldName, "include_in_all", fieldNode));
                 return true;
             }
             return false;
@@ -227,12 +223,12 @@ protected static void parseNested(String name, Map<String, Object> node, ObjectM
             }
             fieldNode = node.get("include_in_parent");
             if (fieldNode != null) {
-                nestedIncludeInParent = lenientNodeBooleanValue(fieldNode);
+                nestedIncludeInParent = TypeParsers.nodeBooleanValue(name, "include_in_parent", fieldNode);
                 node.remove("include_in_parent");
             }
             fieldNode = node.get("include_in_root");
             if (fieldNode != null) {
-                nestedIncludeInRoot = lenientNodeBooleanValue(fieldNode);
+                nestedIncludeInRoot = TypeParsers.nodeBooleanValue(name, "include_in_root", fieldNode);
                 node.remove("include_in_root");
             }
             if (nested) {
@@ -325,7 +321,7 @@ protected static void parseProperties(ObjectMapper.Builder objBuilder, Map<Strin
         super(name);
         assert settings != null;
         Version indexCreatedVersion = Version.indexCreated(settings);
-        if (indexCreatedVersion.onOrAfter(Version.V_5_0_0_alpha6)) {
+        if (indexCreatedVersion.onOrAfter(Version.V_5_0_0_beta1)) {
             if (name.isEmpty()) {
                 throw new IllegalArgumentException("name cannot be empty string");
             }
@@ -413,6 +409,35 @@ public final Dynamic dynamic() {
         return dynamic;
     }
 
+    /**
+     * Returns the parent {@link ObjectMapper} instance of the specified object mapper or <code>null</code> if there
+     * isn't any.
+     */
+    public ObjectMapper getParentObjectMapper(MapperService mapperService) {
+        int indexOfLastDot = fullPath().lastIndexOf('.');
+        if (indexOfLastDot != -1) {
+            String parentNestObjectPath = fullPath().substring(0, indexOfLastDot);
+            return mapperService.getObjectMapper(parentNestObjectPath);
+        } else {
+            return null;
+        }
+    }
+
+    /**
+     * Returns whether all parent objects fields are nested too.
+     */
+    public boolean parentObjectMapperAreNested(MapperService mapperService) {
+        for (ObjectMapper parent = getParentObjectMapper(mapperService);
+             parent != null;
+             parent = parent.getParentObjectMapper(mapperService)) {
+
+            if (parent.nested().isNested() == false) {
+                return false;
+            }
+        }
+        return true;
+    }
+
     @Override
     public ObjectMapper merge(Mapper mergeWith, boolean updateAllTypes) {
         if (!(mergeWith instanceof ObjectMapper)) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/ParentFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/ParentFieldMapper.java
index 9caef2c7740d4..00508e0f2be69 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/ParentFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/ParentFieldMapper.java
@@ -18,10 +18,10 @@
  */
 package org.elasticsearch.index.mapper;
 
-import org.apache.lucene.document.Field;
 import org.apache.lucene.document.SortedDocValuesField;
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
@@ -37,7 +37,7 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.index.fielddata.IndexFieldData;
-import org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData;
+import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
 import org.elasticsearch.index.query.QueryShardContext;
 
 import java.io.IOException;
@@ -65,6 +65,7 @@ public static class Defaults {
             FIELD_TYPE.setIndexOptions(IndexOptions.NONE);
             FIELD_TYPE.setHasDocValues(true);
             FIELD_TYPE.setDocValuesType(DocValuesType.SORTED);
+            FIELD_TYPE.setEagerGlobalOrdinals(false);
             FIELD_TYPE.freeze();
         }
     }
@@ -77,6 +78,8 @@ public static class Builder extends MetadataFieldMapper.Builder<Builder, ParentF
 
         public Builder(String documentType) {
             super(Defaults.NAME, new ParentFieldType(Defaults.FIELD_TYPE, documentType), Defaults.FIELD_TYPE);
+            // Defaults to true
+            eagerGlobalOrdinals(true);
             this.documentType = documentType;
             builder = this;
         }
@@ -114,7 +117,7 @@ public MetadataFieldMapper.Builder parse(String name, Map<String, Object> node,
                 if (fieldName.equals("type")) {
                     builder.type(fieldNode.toString());
                     iterator.remove();
-                } else if (parserContext.parseFieldMatcher().match(fieldName, FIELDDATA)) {
+                } else if (FIELDDATA.match(fieldName)) {
                     // for bw compat only
                     Map<String, String> fieldDataSettings = SettingsLoader.Helper.loadNestedFromMap(nodeMapValue(fieldNode, "fielddata"));
                     if (fieldDataSettings.containsKey("loading")) {
@@ -130,7 +133,9 @@ public MetadataFieldMapper.Builder parse(String name, Map<String, Object> node,
         }
 
         @Override
-        public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName) {
+        public MetadataFieldMapper getDefault(MappedFieldType fieldType, ParserContext context) {
+            final Settings indexSettings = context.mapperService().getIndexSettings().getSettings();
+            final String typeName = context.type();
             KeywordFieldMapper parentJoinField = createParentJoinFieldMapper(typeName, new BuilderContext(indexSettings, new ContentPath(0)));
             MappedFieldType childJoinFieldType = new ParentFieldType(Defaults.FIELD_TYPE, typeName);
             childJoinFieldType.setName(ParentFieldMapper.NAME);
@@ -150,7 +155,7 @@ static final class ParentFieldType extends MappedFieldType {
 
         final String documentType;
 
-        public ParentFieldType() {
+        ParentFieldType() {
             documentType = null;
             setEagerGlobalOrdinals(true);
         }
@@ -194,7 +199,7 @@ public Query termsQuery(List values, @Nullable QueryShardContext context) {
 
         @Override
         public IndexFieldData.Builder fielddataBuilder() {
-            return new ParentChildIndexFieldData.Builder();
+            return new DocValuesIndexFieldData.Builder();
         }
     }
 
@@ -227,7 +232,7 @@ public void postParse(ParseContext context) throws IOException {
     }
 
     @Override
-    protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
+    protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
         boolean parent = context.docMapper().isParent(context.sourceToParse().type());
         if (parent) {
             fields.add(new SortedDocValuesField(parentJoinField.fieldType().name(), new BytesRef(context.sourceToParse().id())));
@@ -284,7 +289,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
 
         builder.startObject(CONTENT_TYPE);
         builder.field("type", parentType);
-        if (includeDefaults || fieldType().eagerGlobalOrdinals() != defaultFieldType.eagerGlobalOrdinals()) {
+        if (includeDefaults || fieldType().eagerGlobalOrdinals() == false) {
             builder.field("eager_global_ordinals", fieldType().eagerGlobalOrdinals());
         }
         builder.endObject();
@@ -293,20 +298,15 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
 
     @Override
     protected void doMerge(Mapper mergeWith, boolean updateAllTypes) {
-        super.doMerge(mergeWith, updateAllTypes);
         ParentFieldMapper fieldMergeWith = (ParentFieldMapper) mergeWith;
-        if (Objects.equals(parentType, fieldMergeWith.parentType) == false) {
+        ParentFieldType currentFieldType = (ParentFieldType) fieldType.clone();
+        super.doMerge(mergeWith, updateAllTypes);
+        if (fieldMergeWith.parentType != null && Objects.equals(parentType, fieldMergeWith.parentType) == false) {
             throw new IllegalArgumentException("The _parent field's type option can't be changed: [" + parentType + "]->[" + fieldMergeWith.parentType + "]");
         }
 
-        List<String> conflicts = new ArrayList<>();
-        fieldType().checkCompatibility(fieldMergeWith.fieldType, conflicts, true);
-        if (conflicts.isEmpty() == false) {
-            throw new IllegalArgumentException("Merge conflicts: " + conflicts);
-        }
-
         if (active()) {
-            fieldType = fieldMergeWith.fieldType.clone();
+            fieldType = currentFieldType;
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/ParseContext.java b/core/src/main/java/org/elasticsearch/index/mapper/ParseContext.java
index 8a2aca97e6843..7ff5c4a37fe5d 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/ParseContext.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/ParseContext.java
@@ -29,15 +29,11 @@
 import org.elasticsearch.common.lucene.all.AllEntries;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.analysis.AnalysisService;
 
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
 
-/**
- *
- */
 public abstract class ParseContext {
 
     /** Fork of {@link org.apache.lucene.document.Document} with additional functionality. */
@@ -242,11 +238,6 @@ public DocumentMapper docMapper() {
             return in.docMapper();
         }
 
-        @Override
-        public AnalysisService analysisService() {
-            return in.analysisService();
-        }
-
         @Override
         public MapperService mapperService() {
             return in.mapperService();
@@ -385,11 +376,6 @@ public DocumentMapper docMapper() {
             return this.docMapper;
         }
 
-        @Override
-        public AnalysisService analysisService() {
-            return docMapperParser.analysisService;
-        }
-
         @Override
         public MapperService mapperService() {
             return docMapperParser.mapperService;
@@ -525,8 +511,6 @@ public boolean isWithinMultiFields() {
 
     public abstract DocumentMapper docMapper();
 
-    public abstract AnalysisService analysisService();
-
     public abstract MapperService mapperService();
 
     public abstract Field version();
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java b/core/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java
index 85367e624d8a6..bf8c1e097236e 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java
@@ -22,6 +22,7 @@
 import org.apache.lucene.document.Field;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.mapper.ParseContext.Document;
 
 import java.util.List;
@@ -45,13 +46,14 @@ public class ParsedDocument {
     private final List<Document> documents;
 
     private BytesReference source;
+    private XContentType xContentType;
 
     private Mapping dynamicMappingsUpdate;
 
     private String parent;
 
     public ParsedDocument(Field version, String id, String type, String routing, long timestamp, long ttl, List<Document> documents,
-                          BytesReference source, Mapping dynamicMappingsUpdate) {
+                          BytesReference source, XContentType xContentType, Mapping dynamicMappingsUpdate) {
         this.version = version;
         this.id = id;
         this.type = type;
@@ -62,15 +64,12 @@ public ParsedDocument(Field version, String id, String type, String routing, lon
         this.documents = documents;
         this.source = source;
         this.dynamicMappingsUpdate = dynamicMappingsUpdate;
+        this.xContentType = xContentType;
     }
     public Field version() {
         return version;
     }
 
-    public BytesRef uid() {
-        return uid;
-    }
-
     public String id() {
         return this.id;
     }
@@ -103,8 +102,13 @@ public BytesReference source() {
         return this.source;
     }
 
-    public void setSource(BytesReference source) {
+    public XContentType getXContentType() {
+        return this.xContentType;
+    }
+
+    public void setSource(BytesReference source, XContentType xContentType) {
         this.source = source;
+        this.xContentType = xContentType;
     }
 
     public ParsedDocument parent(String parent) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java
new file mode 100644
index 0000000000000..9caa73e075382
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java
@@ -0,0 +1,815 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANYDa
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.index.mapper;
+
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.DoubleRange;
+import org.apache.lucene.document.FloatRange;
+import org.apache.lucene.document.IntRange;
+import org.apache.lucene.document.InetAddressPoint;
+import org.apache.lucene.document.InetAddressRange;
+import org.apache.lucene.document.LongRange;
+import org.apache.lucene.document.StoredField;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.search.BoostQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.Explicit;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.geo.ShapeRelation;
+import org.elasticsearch.common.joda.DateMathParser;
+import org.elasticsearch.common.joda.FormatDateTimeFormatter;
+import org.elasticsearch.common.network.InetAddresses;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.LocaleUtils;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.joda.time.DateTimeZone;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.elasticsearch.index.mapper.TypeParsers.parseDateTimeFormatter;
+import static org.elasticsearch.index.query.RangeQueryBuilder.GT_FIELD;
+import static org.elasticsearch.index.query.RangeQueryBuilder.GTE_FIELD;
+import static org.elasticsearch.index.query.RangeQueryBuilder.LT_FIELD;
+import static org.elasticsearch.index.query.RangeQueryBuilder.LTE_FIELD;
+
+/** A {@link FieldMapper} for indexing numeric and date ranges, and creating queries */
+public class RangeFieldMapper extends FieldMapper {
+    public static final boolean DEFAULT_INCLUDE_UPPER = true;
+    public static final boolean DEFAULT_INCLUDE_LOWER = true;
+
+    public static class Defaults {
+        public static final Explicit<Boolean> COERCE = new Explicit<>(true, false);
+    }
+
+    // this is private since it has a different default
+    static final Setting<Boolean> COERCE_SETTING =
+        Setting.boolSetting("index.mapping.coerce", true, Setting.Property.IndexScope);
+
+    public static class Builder extends FieldMapper.Builder<Builder, RangeFieldMapper> {
+        private Boolean coerce;
+        private Locale locale;
+
+        public Builder(String name, RangeType type) {
+            super(name, new RangeFieldType(type), new RangeFieldType(type));
+            builder = this;
+            locale = Locale.ROOT;
+        }
+
+        @Override
+        public RangeFieldType fieldType() {
+            return (RangeFieldType)fieldType;
+        }
+
+        @Override
+        public Builder docValues(boolean docValues) {
+            if (docValues == true) {
+                throw new IllegalArgumentException("field [" + name + "] does not currently support " + TypeParsers.DOC_VALUES);
+            }
+            return super.docValues(docValues);
+        }
+
+        public Builder coerce(boolean coerce) {
+            this.coerce = coerce;
+            return builder;
+        }
+
+        protected Explicit<Boolean> coerce(BuilderContext context) {
+            if (coerce != null) {
+                return new Explicit<>(coerce, true);
+            }
+            if (context.indexSettings() != null) {
+                return new Explicit<>(COERCE_SETTING.get(context.indexSettings()), false);
+            }
+            return Defaults.COERCE;
+        }
+
+        public Builder dateTimeFormatter(FormatDateTimeFormatter dateTimeFormatter) {
+            fieldType().setDateTimeFormatter(dateTimeFormatter);
+            return this;
+        }
+
+        @Override
+        public Builder nullValue(Object nullValue) {
+            throw new IllegalArgumentException("Field [" + name() + "] does not support null value.");
+        }
+
+        public void locale(Locale locale) {
+            this.locale = locale;
+        }
+
+        @Override
+        protected void setupFieldType(BuilderContext context) {
+            super.setupFieldType(context);
+            FormatDateTimeFormatter dateTimeFormatter = fieldType().dateTimeFormatter;
+            if (fieldType().rangeType == RangeType.DATE) {
+                if (!locale.equals(dateTimeFormatter.locale())) {
+                    fieldType().setDateTimeFormatter(new FormatDateTimeFormatter(dateTimeFormatter.format(),
+                        dateTimeFormatter.parser(), dateTimeFormatter.printer(), locale));
+                }
+            } else if (dateTimeFormatter != null) {
+                throw new IllegalArgumentException("field [" + name() + "] of type [" + fieldType().rangeType
+                    + "] should not define a dateTimeFormatter unless it is a " + RangeType.DATE + " type");
+            }
+        }
+
+        @Override
+        public RangeFieldMapper build(BuilderContext context) {
+            setupFieldType(context);
+            return new RangeFieldMapper(name, fieldType, defaultFieldType, coerce(context), includeInAll,
+                context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo);
+        }
+    }
+
+    public static class TypeParser implements Mapper.TypeParser {
+        final RangeType type;
+
+        public TypeParser(RangeType type) {
+            this.type = type;
+        }
+
+        @Override
+        public Mapper.Builder<?,?> parse(String name, Map<String, Object> node,
+                                         ParserContext parserContext) throws MapperParsingException {
+            Builder builder = new Builder(name, type);
+            TypeParsers.parseField(builder, name, node, parserContext);
+            for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
+                Map.Entry<String, Object> entry = iterator.next();
+                String propName = entry.getKey();
+                Object propNode = entry.getValue();
+                if (propName.equals("null_value")) {
+                    throw new MapperParsingException("Property [null_value] is not supported for [" + this.type.name
+                            + "] field types.");
+                } else if (propName.equals("coerce")) {
+                    builder.coerce(TypeParsers.nodeBooleanValue(name, "coerce", propNode));
+                    iterator.remove();
+                } else if (propName.equals("locale")) {
+                    builder.locale(LocaleUtils.parse(propNode.toString()));
+                    iterator.remove();
+                } else if (propName.equals("format")) {
+                    builder.dateTimeFormatter(parseDateTimeFormatter(propNode));
+                    iterator.remove();
+                } else if (TypeParsers.parseMultiField(builder, name, parserContext, propName, propNode)) {
+                    iterator.remove();
+                }
+            }
+            return builder;
+        }
+    }
+
+    public static final class RangeFieldType extends MappedFieldType {
+        protected RangeType rangeType;
+        protected FormatDateTimeFormatter dateTimeFormatter;
+        protected DateMathParser dateMathParser;
+
+        public RangeFieldType(RangeType type) {
+            super();
+            this.rangeType = Objects.requireNonNull(type);
+            setTokenized(false);
+            setHasDocValues(false);
+            setOmitNorms(true);
+            if (rangeType == RangeType.DATE) {
+                setDateTimeFormatter(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER);
+            }
+        }
+
+        public RangeFieldType(RangeFieldType other) {
+            super(other);
+            this.rangeType = other.rangeType;
+            if (other.dateTimeFormatter() != null) {
+                setDateTimeFormatter(other.dateTimeFormatter);
+            }
+        }
+
+        @Override
+        public MappedFieldType clone() {
+            return new RangeFieldType(this);
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (!super.equals(o)) return false;
+            RangeFieldType that = (RangeFieldType) o;
+            return Objects.equals(rangeType, that.rangeType) &&
+            (rangeType == RangeType.DATE) ?
+                Objects.equals(dateTimeFormatter.format(), that.dateTimeFormatter.format())
+                && Objects.equals(dateTimeFormatter.locale(), that.dateTimeFormatter.locale())
+                : dateTimeFormatter == null && that.dateTimeFormatter == null;
+        }
+
+        @Override
+        public int hashCode() {
+            return (dateTimeFormatter == null) ? Objects.hash(super.hashCode(), rangeType)
+                : Objects.hash(super.hashCode(), rangeType, dateTimeFormatter.format(), dateTimeFormatter.locale());
+        }
+
+        @Override
+        public String typeName() {
+            return rangeType.name;
+        }
+
+        @Override
+        public void checkCompatibility(MappedFieldType fieldType, List<String> conflicts, boolean strict) {
+            super.checkCompatibility(fieldType, conflicts, strict);
+            if (strict) {
+                RangeFieldType other = (RangeFieldType)fieldType;
+                if (this.rangeType != other.rangeType) {
+                    conflicts.add("mapper [" + name()
+                        + "] is attempting to update from type [" + rangeType.name
+                        + "] to incompatible type [" + other.rangeType.name + "].");
+                }
+                if (this.rangeType == RangeType.DATE) {
+                    if (Objects.equals(dateTimeFormatter().format(), other.dateTimeFormatter().format()) == false) {
+                        conflicts.add("mapper [" + name()
+                            + "] is used by multiple types. Set update_all_types to true to update [format] across all types.");
+                    }
+                    if (Objects.equals(dateTimeFormatter().locale(), other.dateTimeFormatter().locale()) == false) {
+                        conflicts.add("mapper [" + name()
+                            + "] is used by multiple types. Set update_all_types to true to update [locale] across all types.");
+                    }
+                }
+            }
+        }
+
+        public FormatDateTimeFormatter dateTimeFormatter() {
+            return dateTimeFormatter;
+        }
+
+        public void setDateTimeFormatter(FormatDateTimeFormatter dateTimeFormatter) {
+            checkIfFrozen();
+            this.dateTimeFormatter = dateTimeFormatter;
+            this.dateMathParser = new DateMathParser(dateTimeFormatter);
+        }
+
+        protected DateMathParser dateMathParser() {
+            return dateMathParser;
+        }
+
+        @Override
+        public Query termQuery(Object value, QueryShardContext context) {
+            Query query = rangeQuery(value, value, true, true, ShapeRelation.INTERSECTS, context);
+            if (boost() != 1f) {
+                query = new BoostQuery(query, boost());
+            }
+            return query;
+        }
+
+        public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper,
+                                ShapeRelation relation, QueryShardContext context) {
+            failIfNotIndexed();
+            return rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, relation, null, dateMathParser, context);
+        }
+
+        public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper,
+                                ShapeRelation relation, DateTimeZone timeZone, DateMathParser parser, QueryShardContext context) {
+            return rangeType.rangeQuery(name(), lowerTerm, upperTerm, includeLower, includeUpper, relation, timeZone, parser, context);
+        }
+    }
+
+    private Boolean includeInAll;
+    private Explicit<Boolean> coerce;
+
+    private RangeFieldMapper(
+        String simpleName,
+        MappedFieldType fieldType,
+        MappedFieldType defaultFieldType,
+        Explicit<Boolean> coerce,
+        Boolean includeInAll,
+        Settings indexSettings,
+        MultiFields multiFields,
+        CopyTo copyTo) {
+        super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo);
+        this.coerce = coerce;
+        this.includeInAll = includeInAll;
+    }
+
+    @Override
+    public RangeFieldType fieldType() {
+        return (RangeFieldType) super.fieldType();
+    }
+
+    @Override
+    protected String contentType() {
+        return fieldType.typeName();
+    }
+
+    @Override
+    protected RangeFieldMapper clone() {
+        return (RangeFieldMapper) super.clone();
+    }
+
+    @Override
+    protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
+        final boolean includeInAll = context.includeInAll(this.includeInAll, this);
+        Range range;
+        if (context.externalValueSet()) {
+            range = context.parseExternalValue(Range.class);
+        } else {
+            XContentParser parser = context.parser();
+            if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
+                RangeFieldType fieldType = fieldType();
+                RangeType rangeType = fieldType.rangeType;
+                String fieldName = null;
+                Object from = rangeType.minValue();
+                Object to = rangeType.maxValue();
+                boolean includeFrom = DEFAULT_INCLUDE_LOWER;
+                boolean includeTo = DEFAULT_INCLUDE_UPPER;
+                XContentParser.Token token;
+                while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                    if (token == XContentParser.Token.FIELD_NAME) {
+                        fieldName = parser.currentName();
+                    } else {
+                        if (fieldName.equals(GT_FIELD.getPreferredName())) {
+                            includeFrom = false;
+                            if (parser.currentToken() != XContentParser.Token.VALUE_NULL) {
+                                from = rangeType.parseFrom(fieldType, parser, coerce.value(), includeFrom);
+                            }
+                        } else if (fieldName.equals(GTE_FIELD.getPreferredName())) {
+                            includeFrom = true;
+                            if (parser.currentToken() != XContentParser.Token.VALUE_NULL) {
+                                from = rangeType.parseFrom(fieldType, parser, coerce.value(), includeFrom);
+                            }
+                        } else if (fieldName.equals(LT_FIELD.getPreferredName())) {
+                            includeTo = false;
+                            if (parser.currentToken() != XContentParser.Token.VALUE_NULL) {
+                                to = rangeType.parseTo(fieldType, parser, coerce.value(), includeTo);
+                            }
+                        } else if (fieldName.equals(LTE_FIELD.getPreferredName())) {
+                            includeTo = true;
+                            if (parser.currentToken() != XContentParser.Token.VALUE_NULL) {
+                                to = rangeType.parseTo(fieldType, parser, coerce.value(), includeTo);
+                            }
+                        } else {
+                            throw new MapperParsingException("error parsing field [" +
+                                name() + "], with unknown parameter [" + fieldName + "]");
+                        }
+                    }
+                }
+                range = new Range(rangeType, from, to, includeFrom, includeTo);
+            } else {
+                throw new MapperParsingException("error parsing field ["
+                    + name() + "], expected an object but got " + parser.currentName());
+            }
+        }
+        if (includeInAll) {
+            context.allEntries().addText(fieldType.name(), range.toString(), fieldType.boost());
+        }
+        boolean indexed = fieldType.indexOptions() != IndexOptions.NONE;
+        boolean docValued = fieldType.hasDocValues();
+        boolean stored = fieldType.stored();
+        fields.addAll(fieldType().rangeType.createFields(name(), range, indexed, docValued, stored));
+    }
+
+    @Override
+    protected void doMerge(Mapper mergeWith, boolean updateAllTypes) {
+        super.doMerge(mergeWith, updateAllTypes);
+        RangeFieldMapper other = (RangeFieldMapper) mergeWith;
+        this.includeInAll = other.includeInAll;
+        if (other.coerce.explicit()) {
+            this.coerce = other.coerce;
+        }
+    }
+
+    @Override
+    protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException {
+        super.doXContentBody(builder, includeDefaults, params);
+
+        if (fieldType().rangeType == RangeType.DATE
+                && (includeDefaults || (fieldType().dateTimeFormatter() != null
+                && fieldType().dateTimeFormatter().format().equals(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.format()) == false))) {
+            builder.field("format", fieldType().dateTimeFormatter().format());
+        }
+        if (fieldType().rangeType == RangeType.DATE
+                && (includeDefaults || (fieldType().dateTimeFormatter() != null
+                && fieldType().dateTimeFormatter().locale() != Locale.ROOT))) {
+            builder.field("locale", fieldType().dateTimeFormatter().locale());
+        }
+        if (includeDefaults || coerce.explicit()) {
+            builder.field("coerce", coerce.value());
+        }
+        if (includeInAll != null) {
+            builder.field("include_in_all", includeInAll);
+        } else if (includeDefaults) {
+            builder.field("include_in_all", false);
+        }
+    }
+
+    /** Enum defining the type of range */
+    public enum RangeType {
+        IP("ip_range") {
+            @Override
+            public Field getRangeField(String name, Range r) {
+                return new InetAddressRange(name, (InetAddress)r.from, (InetAddress)r.to);
+            }
+            @Override
+            public InetAddress parseFrom(RangeFieldType fieldType, XContentParser parser, boolean coerce, boolean included)
+                    throws IOException {
+                InetAddress address = InetAddresses.forString(parser.text());
+                return included ? address : nextUp(address);
+            }
+            @Override
+            public InetAddress parseTo(RangeFieldType fieldType, XContentParser parser, boolean coerce, boolean included)
+                    throws IOException {
+                InetAddress address = InetAddresses.forString(parser.text());
+                return included ? address : nextDown(address);
+            }
+            @Override
+            public InetAddress parse(Object value, boolean coerce) {
+                if (value instanceof InetAddress) {
+                    return (InetAddress) value;
+                } else {
+                    if (value instanceof BytesRef) {
+                        value = ((BytesRef) value).utf8ToString();
+                    }
+                    return InetAddresses.forString(value.toString());
+                }
+            }
+            @Override
+            public InetAddress minValue() {
+                return InetAddressPoint.MIN_VALUE;
+            }
+            @Override
+            public InetAddress maxValue() {
+                return InetAddressPoint.MAX_VALUE;
+            }
+            @Override
+            public InetAddress nextUp(Object value) {
+                return InetAddressPoint.nextUp((InetAddress)value);
+            }
+            @Override
+            public InetAddress nextDown(Object value) {
+                return InetAddressPoint.nextDown((InetAddress)value);
+            }
+            @Override
+            public Query withinQuery(String field, Object from, Object to, boolean includeLower, boolean includeUpper) {
+                InetAddress lower = (InetAddress)from;
+                InetAddress upper = (InetAddress)to;
+                return InetAddressRange.newWithinQuery(field,
+                    includeLower ? lower : nextUp(lower), includeUpper ? upper : nextDown(upper));
+            }
+            @Override
+            public Query containsQuery(String field, Object from, Object to, boolean includeLower, boolean includeUpper) {
+                InetAddress lower = (InetAddress)from;
+                InetAddress upper = (InetAddress)to;
+                return InetAddressRange.newContainsQuery(field,
+                    includeLower ? lower : nextUp(lower), includeUpper ? upper : nextDown(upper));
+            }
+            @Override
+            public Query intersectsQuery(String field, Object from, Object to, boolean includeLower, boolean includeUpper) {
+                InetAddress lower = (InetAddress)from;
+                InetAddress upper = (InetAddress)to;
+                return InetAddressRange.newIntersectsQuery(field,
+                    includeLower ? lower : nextUp(lower), includeUpper ? upper : nextDown(upper));
+            }
+            public String toString(InetAddress address) {
+                return InetAddresses.toAddrString(address);
+            }
+        },
+        DATE("date_range", NumberType.LONG) {
+            @Override
+            public Field getRangeField(String name, Range r) {
+                return new LongRange(name, new long[] {((Number)r.from).longValue()}, new long[] {((Number)r.to).longValue()});
+            }
+            private Number parse(DateMathParser dateMathParser, String dateStr) {
+                return dateMathParser.parse(dateStr, () -> {throw new IllegalArgumentException("now is not used at indexing time");});
+            }
+            @Override
+            public Number parseFrom(RangeFieldType fieldType, XContentParser parser, boolean coerce, boolean included)
+                    throws IOException {
+                Number value = parse(fieldType.dateMathParser, parser.text());
+                return included ? value : nextUp(value);
+            }
+            @Override
+            public Number parseTo(RangeFieldType fieldType, XContentParser parser, boolean coerce, boolean included)
+                    throws IOException{
+                Number value = parse(fieldType.dateMathParser, parser.text());
+                return included ? value : nextDown(value);
+            }
+            @Override
+            public Long minValue() {
+                return Long.MIN_VALUE;
+            }
+            @Override
+            public Long maxValue() {
+                return Long.MAX_VALUE;
+            }
+            @Override
+            public Long nextUp(Object value) {
+                return (long) LONG.nextUp(value);
+            }
+            @Override
+            public Long nextDown(Object value) {
+                return (long) LONG.nextDown(value);
+            }
+            @Override
+            public Query rangeQuery(String field, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper,
+                                    ShapeRelation relation, @Nullable DateTimeZone timeZone, @Nullable DateMathParser parser,
+                                    QueryShardContext context) {
+                DateTimeZone zone = (timeZone == null) ? DateTimeZone.UTC : timeZone;
+                DateMathParser dateMathParser = (parser == null) ?
+                    new DateMathParser(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER) : parser;
+                Long low = lowerTerm == null ? Long.MIN_VALUE :
+                    dateMathParser.parse(lowerTerm instanceof BytesRef ? ((BytesRef) lowerTerm).utf8ToString() : lowerTerm.toString(),
+                        context::nowInMillis, false, zone);
+                Long high = upperTerm == null ? Long.MAX_VALUE :
+                    dateMathParser.parse(upperTerm instanceof BytesRef ? ((BytesRef) upperTerm).utf8ToString() : upperTerm.toString(),
+                        context::nowInMillis, false, zone);
+
+                return super.rangeQuery(field, low, high, includeLower, includeUpper, relation, zone, dateMathParser, context);
+            }
+            @Override
+            public Query withinQuery(String field, Object from, Object to, boolean includeLower, boolean includeUpper) {
+                return LONG.withinQuery(field, from, to, includeLower, includeUpper);
+            }
+            @Override
+            public Query containsQuery(String field, Object from, Object to, boolean includeLower, boolean includeUpper) {
+                return LONG.containsQuery(field, from, to, includeLower, includeUpper);
+            }
+            @Override
+            public Query intersectsQuery(String field, Object from, Object to, boolean includeLower, boolean includeUpper) {
+                return LONG.intersectsQuery(field, from, to, includeLower, includeUpper);
+            }
+        },
+        // todo support half_float
+        FLOAT("float_range", NumberType.FLOAT) {
+            @Override
+            public Float minValue() {
+                return Float.NEGATIVE_INFINITY;
+            }
+            @Override
+            public Float maxValue() {
+                return Float.POSITIVE_INFINITY;
+            }
+            @Override
+            public Float nextUp(Object value) {
+                return Math.nextUp(((Number)value).floatValue());
+            }
+            @Override
+            public Float nextDown(Object value) {
+                return Math.nextDown(((Number)value).floatValue());
+            }
+            @Override
+            public Field getRangeField(String name, Range r) {
+                return new FloatRange(name, new float[] {((Number)r.from).floatValue()}, new float[] {((Number)r.to).floatValue()});
+            }
+            @Override
+            public Query withinQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo) {
+                return FloatRange.newWithinQuery(field,
+                    new float[] {includeFrom ? (Float)from : Math.nextUp((Float)from)},
+                    new float[] {includeTo ? (Float)to : Math.nextDown((Float)to)});
+            }
+            @Override
+            public Query containsQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo) {
+                return FloatRange.newContainsQuery(field,
+                    new float[] {includeFrom ? (Float)from : Math.nextUp((Float)from)},
+                    new float[] {includeTo ? (Float)to : Math.nextDown((Float)to)});
+            }
+            @Override
+            public Query intersectsQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo) {
+                return FloatRange.newIntersectsQuery(field,
+                    new float[] {includeFrom ? (Float)from : Math.nextUp((Float)from)},
+                    new float[] {includeTo ? (Float)to : Math.nextDown((Float)to)});
+            }
+        },
+        DOUBLE("double_range", NumberType.DOUBLE) {
+            @Override
+            public Double minValue() {
+                return Double.NEGATIVE_INFINITY;
+            }
+            @Override
+            public Double maxValue() {
+                return Double.POSITIVE_INFINITY;
+            }
+            @Override
+            public Double nextUp(Object value) {
+                return Math.nextUp(((Number)value).doubleValue());
+            }
+            @Override
+            public Double nextDown(Object value) {
+                return Math.nextDown(((Number)value).doubleValue());
+            }
+            @Override
+            public Field getRangeField(String name, Range r) {
+                return new DoubleRange(name, new double[] {((Number)r.from).doubleValue()}, new double[] {((Number)r.to).doubleValue()});
+            }
+            @Override
+            public Query withinQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo) {
+                return DoubleRange.newWithinQuery(field,
+                    new double[] {includeFrom ? (Double)from : Math.nextUp((Double)from)},
+                    new double[] {includeTo ? (Double)to : Math.nextDown((Double)to)});
+            }
+            @Override
+            public Query containsQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo) {
+                return DoubleRange.newContainsQuery(field,
+                    new double[] {includeFrom ? (Double)from : Math.nextUp((Double)from)},
+                    new double[] {includeTo ? (Double)to : Math.nextDown((Double)to)});
+            }
+            @Override
+            public Query intersectsQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo) {
+                return DoubleRange.newIntersectsQuery(field,
+                    new double[] {includeFrom ? (Double)from : Math.nextUp((Double)from)},
+                    new double[] {includeTo ? (Double)to : Math.nextDown((Double)to)});
+            }
+        },
+        // todo add BYTE support
+        // todo add SHORT support
+        INTEGER("integer_range", NumberType.INTEGER) {
+            @Override
+            public Integer minValue() {
+                return Integer.MIN_VALUE;
+            }
+            @Override
+            public Integer maxValue() {
+                return Integer.MAX_VALUE;
+            }
+            @Override
+            public Integer nextUp(Object value) {
+                return ((Number)value).intValue() + 1;
+            }
+            @Override
+            public Integer nextDown(Object value) {
+                return ((Number)value).intValue() - 1;
+            }
+            @Override
+            public Field getRangeField(String name, Range r) {
+                return new IntRange(name, new int[] {((Number)r.from).intValue()}, new int[] {((Number)r.to).intValue()});
+            }
+            @Override
+            public Query withinQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo) {
+                return IntRange.newWithinQuery(field, new int[] {(Integer)from + (includeFrom ? 0 : 1)},
+                    new int[] {(Integer)to - (includeTo ? 0 : 1)});
+            }
+            @Override
+            public Query containsQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo) {
+                return IntRange.newContainsQuery(field, new int[] {(Integer)from + (includeFrom ? 0 : 1)},
+                    new int[] {(Integer)to - (includeTo ? 0 : 1)});
+            }
+            @Override
+            public Query intersectsQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo) {
+                return IntRange.newIntersectsQuery(field, new int[] {(Integer)from + (includeFrom ? 0 : 1)},
+                    new int[] {(Integer)to - (includeTo ? 0 : 1)});
+            }
+        },
+        LONG("long_range", NumberType.LONG) {
+            @Override
+            public Long minValue() {
+                return Long.MIN_VALUE;
+            }
+            @Override
+            public Long maxValue() {
+                return Long.MAX_VALUE;
+            }
+            @Override
+            public Long nextUp(Object value) {
+                return ((Number)value).longValue() + 1;
+            }
+            @Override
+            public Long nextDown(Object value) {
+                return ((Number)value).longValue() - 1;
+            }
+            @Override
+            public Field getRangeField(String name, Range r) {
+                return new LongRange(name, new long[] {((Number)r.from).longValue()},
+                    new long[] {((Number)r.to).longValue()});
+            }
+            @Override
+            public Query withinQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo) {
+                return LongRange.newWithinQuery(field,  new long[] {(Long)from + (includeFrom ? 0 : 1)},
+                    new long[] {(Long)to - (includeTo ? 0 : 1)});
+            }
+            @Override
+            public Query containsQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo) {
+                return LongRange.newContainsQuery(field,  new long[] {(Long)from + (includeFrom ? 0 : 1)},
+                    new long[] {(Long)to - (includeTo ? 0 : 1)});
+            }
+            @Override
+            public Query intersectsQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo) {
+                return LongRange.newIntersectsQuery(field,  new long[] {(Long)from + (includeFrom ? 0 : 1)},
+                    new long[] {(Long)to - (includeTo ? 0 : 1)});
+            }
+        };
+
+        RangeType(String name) {
+            this.name = name;
+            this.numberType = null;
+        }
+
+        RangeType(String name, NumberType type) {
+            this.name = name;
+            this.numberType = type;
+        }
+
+        /** Get the associated type name. */
+        public final String typeName() {
+            return name;
+        }
+
+        public abstract Field getRangeField(String name, Range range);
+        public List<IndexableField> createFields(String name, Range range, boolean indexed, boolean docValued, boolean stored) {
+            assert range != null : "range cannot be null when creating fields";
+            List<IndexableField> fields = new ArrayList<>();
+            if (indexed) {
+                fields.add(getRangeField(name, range));
+            }
+            // todo add docValues ranges once aggregations are supported
+            if (stored) {
+                fields.add(new StoredField(name, range.toString()));
+            }
+            return fields;
+        }
+        /** parses from value. rounds according to included flag */
+        public Object parseFrom(RangeFieldType fieldType, XContentParser parser, boolean coerce, boolean included) throws IOException {
+            Number value = numberType.parse(parser, coerce);
+            return included ? value : (Number)nextUp(value);
+        }
+        /** parses to value. rounds according to included flag */
+        public Object parseTo(RangeFieldType fieldType, XContentParser parser, boolean coerce, boolean included) throws IOException {
+            Number value = numberType.parse(parser, coerce);
+            return included ? value : (Number)nextDown(value);
+        }
+
+        public abstract Object minValue();
+        public abstract Object maxValue();
+        public abstract Object nextUp(Object value);
+        public abstract Object nextDown(Object value);
+        public abstract Query withinQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo);
+        public abstract Query containsQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo);
+        public abstract Query intersectsQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo);
+        public Object parse(Object value, boolean coerce) {
+            return numberType.parse(value, coerce);
+        }
+        public Query rangeQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo,
+                ShapeRelation relation, @Nullable DateTimeZone timeZone, @Nullable DateMathParser dateMathParser,
+                QueryShardContext context) {
+            Object lower = from == null ? minValue() : parse(from, false);
+            Object upper = to == null ? maxValue() : parse(to, false);
+            if (relation == ShapeRelation.WITHIN) {
+                return withinQuery(field, lower, upper, includeFrom, includeTo);
+            } else if (relation == ShapeRelation.CONTAINS) {
+                return containsQuery(field, lower, upper, includeFrom, includeTo);
+            }
+            return intersectsQuery(field, lower, upper, includeFrom, includeTo);
+        }
+
+        public final String name;
+        private final NumberType numberType;
+    }
+
+    /** Class defining a range */
+    public static class Range {
+        RangeType type;
+        private Object from;
+        private Object to;
+        private boolean includeFrom;
+        private boolean includeTo;
+
+        public Range(RangeType type, Object from, Object to, boolean includeFrom, boolean includeTo) {
+            this.type = type;
+            this.from = from;
+            this.to = to;
+            this.includeFrom = includeFrom;
+            this.includeTo = includeTo;
+        }
+
+        @Override
+        public String toString() {
+            StringBuilder sb = new StringBuilder();
+            sb.append(includeFrom ? '[' : '(');
+            Object f = includeFrom || from.equals(type.minValue()) ? from : type.nextDown(from);
+            Object t = includeTo || to.equals(type.maxValue()) ? to : type.nextUp(to);
+            sb.append(type == RangeType.IP ? InetAddresses.toAddrString((InetAddress)f) : f.toString());
+            sb.append(" : ");
+            sb.append(type == RangeType.IP ? InetAddresses.toAddrString((InetAddress)t) : t.toString());
+            sb.append(includeTo ? ']' : ')');
+            return sb.toString();
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java
index aa3e78b8ee056..0eab67e57b267 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java
@@ -21,20 +21,17 @@
 
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexableField;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
-import static org.elasticsearch.common.xcontent.support.XContentMapValues.lenientNodeBooleanValue;
-
-/**
- *
- */
 public class RoutingFieldMapper extends MetadataFieldMapper {
 
     public static final String NAME = "_routing";
@@ -80,14 +77,14 @@ public RoutingFieldMapper build(BuilderContext context) {
 
     public static class TypeParser implements MetadataFieldMapper.TypeParser {
         @Override
-        public MetadataFieldMapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
+        public MetadataFieldMapper.Builder<?,?> parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
             Builder builder = new Builder(parserContext.mapperService().fullName(NAME));
             for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
                 Map.Entry<String, Object> entry = iterator.next();
                 String fieldName = entry.getKey();
                 Object fieldNode = entry.getValue();
                 if (fieldName.equals("required")) {
-                    builder.required(lenientNodeBooleanValue(fieldNode));
+                    builder.required(TypeParsers.nodeBooleanValue(name, "required", fieldNode));
                     iterator.remove();
                 }
             }
@@ -95,14 +92,20 @@ public MetadataFieldMapper.Builder parse(String name, Map<String, Object> node,
         }
 
         @Override
-        public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName) {
-            return new RoutingFieldMapper(indexSettings, fieldType);
+        public MetadataFieldMapper getDefault(MappedFieldType fieldType, ParserContext context) {
+            final Settings indexSettings = context.mapperService().getIndexSettings().getSettings();
+            if (fieldType != null) {
+                return new RoutingFieldMapper(indexSettings, fieldType);
+            } else {
+                return parse(NAME, Collections.emptyMap(), context)
+                        .build(new BuilderContext(indexSettings, new ContentPath(1)));
+            }
         }
     }
 
     static final class RoutingFieldType extends TermBasedFieldType {
 
-        public RoutingFieldType() {
+        RoutingFieldType() {
         }
 
         protected RoutingFieldType(RoutingFieldType ref) {
@@ -123,7 +126,7 @@ public String typeName() {
     private boolean required;
 
     private RoutingFieldMapper(Settings indexSettings, MappedFieldType existing) {
-        this(existing == null ? Defaults.FIELD_TYPE.clone() : existing.clone(), Defaults.REQUIRED, indexSettings);
+        this(existing.clone(), Defaults.REQUIRED, indexSettings);
     }
 
     private RoutingFieldMapper(MappedFieldType fieldType, boolean required, Settings indexSettings) {
@@ -157,7 +160,7 @@ public Mapper parse(ParseContext context) throws IOException {
     }
 
     @Override
-    protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
+    protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
         String routing = context.sourceToParse().routing();
         if (routing != null) {
             if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java
index 3608da30f7653..7ac5b8bf63b47 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java
@@ -19,8 +19,8 @@
 
 package org.elasticsearch.index.mapper;
 
-import org.apache.lucene.document.Field;
 import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.LeafReaderContext;
@@ -28,8 +28,10 @@
 import org.apache.lucene.index.SortedNumericDocValues;
 import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.SortField;
 import org.elasticsearch.action.fieldstats.FieldStats;
 import org.elasticsearch.common.Explicit;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -144,16 +146,16 @@ public Mapper.Builder<?,?> parse(String name, Map<String, Object> node,
                     if (propNode == null) {
                         throw new MapperParsingException("Property [null_value] cannot be null.");
                     }
-                    builder.nullValue(NumberFieldMapper.NumberType.DOUBLE.parse(propNode));
+                    builder.nullValue(NumberFieldMapper.NumberType.DOUBLE.parse(propNode, false));
                     iterator.remove();
                 } else if (propName.equals("ignore_malformed")) {
-                    builder.ignoreMalformed(TypeParsers.nodeBooleanValue("ignore_malformed", propNode, parserContext));
+                    builder.ignoreMalformed(TypeParsers.nodeBooleanValue(name, "ignore_malformed", propNode));
                     iterator.remove();
                 } else if (propName.equals("coerce")) {
-                    builder.coerce(TypeParsers.nodeBooleanValue("coerce", propNode, parserContext));
+                    builder.coerce(TypeParsers.nodeBooleanValue(name, "coerce", propNode));
                     iterator.remove();
                 } else if (propName.equals("scaling_factor")) {
-                    builder.scalingFactor(NumberFieldMapper.NumberType.DOUBLE.parse(propNode).doubleValue());
+                    builder.scalingFactor(NumberFieldMapper.NumberType.DOUBLE.parse(propNode, false).doubleValue());
                     iterator.remove();
                 }
             }
@@ -207,7 +209,7 @@ public void checkCompatibility(MappedFieldType other, List<String> conflicts, bo
         @Override
         public Query termQuery(Object value, QueryShardContext context) {
             failIfNotIndexed();
-            double queryValue = NumberFieldMapper.NumberType.DOUBLE.parse(value).doubleValue();
+            double queryValue = NumberFieldMapper.NumberType.DOUBLE.parse(value, false).doubleValue();
             long scaledValue = Math.round(queryValue * scalingFactor);
             Query query = NumberFieldMapper.NumberType.LONG.termQuery(name(), scaledValue);
             if (boost() != 1f) {
@@ -221,7 +223,7 @@ public Query termsQuery(List values, QueryShardContext context) {
             failIfNotIndexed();
             List<Long> scaledValues = new ArrayList<>(values.size());
             for (Object value : values) {
-                double queryValue = NumberFieldMapper.NumberType.DOUBLE.parse(value).doubleValue();
+                double queryValue = NumberFieldMapper.NumberType.DOUBLE.parse(value, false).doubleValue();
                 long scaledValue = Math.round(queryValue * scalingFactor);
                 scaledValues.add(scaledValue);
             }
@@ -233,11 +235,11 @@ public Query termsQuery(List values, QueryShardContext context) {
         }
 
         @Override
-        public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper) {
+        public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) {
             failIfNotIndexed();
             Long lo = null;
             if (lowerTerm != null) {
-                double dValue = NumberFieldMapper.NumberType.DOUBLE.parse(lowerTerm).doubleValue();
+                double dValue = NumberFieldMapper.NumberType.DOUBLE.parse(lowerTerm, false).doubleValue();
                 if (includeLower == false) {
                     dValue = Math.nextUp(dValue);
                 }
@@ -245,13 +247,13 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower
             }
             Long hi = null;
             if (upperTerm != null) {
-                double dValue = NumberFieldMapper.NumberType.DOUBLE.parse(upperTerm).doubleValue();
+                double dValue = NumberFieldMapper.NumberType.DOUBLE.parse(upperTerm, false).doubleValue();
                 if (includeUpper == false) {
                     dValue = Math.nextDown(dValue);
                 }
                 hi = Math.round(Math.floor(dValue * scalingFactor));
             }
-            Query query = NumberFieldMapper.NumberType.LONG.rangeQuery(name(), lo, hi, true, true);
+            Query query = NumberFieldMapper.NumberType.LONG.rangeQuery(name(), lo, hi, true, true, hasDocValues());
             if (boost() != 1f) {
                 query = new BoostQuery(query, boost());
             }
@@ -265,11 +267,16 @@ public FieldStats<?> stats(IndexReader reader) throws IOException {
             if (stats == null) {
                 return null;
             }
-            return new FieldStats.Double(stats.getMaxDoc(), stats.getDocCount(),
+            if (stats.hasMinMax()) {
+                return new FieldStats.Double(stats.getMaxDoc(), stats.getDocCount(),
                     stats.getSumDocFreq(), stats.getSumTotalTermFreq(),
                     stats.isSearchable(), stats.isAggregatable(),
-                    stats.getMinValue() == null ? null : stats.getMinValue() / scalingFactor,
-                    stats.getMaxValue() == null ? null : stats.getMaxValue() / scalingFactor);
+                    stats.getMinValue() / scalingFactor,
+                    stats.getMaxValue() / scalingFactor);
+            }
+            return new FieldStats.Double(stats.getMaxDoc(), stats.getDocCount(),
+                stats.getSumDocFreq(), stats.getSumTotalTermFreq(),
+                stats.isSearchable(), stats.isAggregatable());
         }
 
         @Override
@@ -288,7 +295,7 @@ public IndexFieldData<?> build(IndexSettings indexSettings, MappedFieldType fiel
         }
 
         @Override
-        public Object valueForSearch(Object value) {
+        public Object valueForDisplay(Object value) {
             if (value == null) {
                 return null;
             }
@@ -364,7 +371,7 @@ protected ScaledFloatFieldMapper clone() {
     }
 
     @Override
-    protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
+    protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
         final boolean includeInAll = context.includeInAll(this.includeInAll, this);
 
         XContentParser parser = context.parser();
@@ -404,7 +411,7 @@ protected void parseCreateField(ParseContext context, List<Field> fields) throws
         }
 
         if (numericValue == null) {
-            numericValue = NumberFieldMapper.NumberType.DOUBLE.parse(value);
+            numericValue = NumberFieldMapper.NumberType.DOUBLE.parse(value, false);
         }
 
         if (includeInAll) {
@@ -413,8 +420,12 @@ protected void parseCreateField(ParseContext context, List<Field> fields) throws
 
         double doubleValue = numericValue.doubleValue();
         if (Double.isFinite(doubleValue) == false) {
-            // since we encode to a long, we have no way to carry NaNs and infinities
-            throw new IllegalArgumentException("[scaled_float] only supports finite values, but got [" + doubleValue + "]");
+            if (ignoreMalformed.value()) {
+                return;
+            } else {
+                // since we encode to a long, we have no way to carry NaNs and infinities
+                throw new IllegalArgumentException("[scaled_float] only supports finite values, but got [" + doubleValue + "]");
+            }
         }
         long scaledValue = Math.round(doubleValue * fieldType().getScalingFactor());
 
@@ -487,9 +498,9 @@ public AtomicNumericFieldData loadDirect(LeafReaderContext context) throws Excep
         }
 
         @Override
-        public org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource comparatorSource(Object missingValue,
-                MultiValueMode sortMode, Nested nested) {
-            return new DoubleValuesComparatorSource(this, missingValue, sortMode, nested);
+        public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse) {
+            final XFieldComparatorSource source = new DoubleValuesComparatorSource(this, missingValue, sortMode, nested);
+            return new SortField(getFieldName(), source, reverse);
         }
 
         @Override
@@ -504,7 +515,10 @@ public Index index() {
 
         @Override
         public NumericType getNumericType() {
-            return scaledFieldData.getNumericType();
+            /**
+             * {@link ScaledFloatLeafFieldData#getDoubleValues()} transforms the raw long values in `scaled` floats.
+             */
+            return NumericType.DOUBLE;
         }
 
     }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java
index 4854eb5775259..2673bede0a8f9 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java
@@ -19,9 +19,9 @@
 
 package org.elasticsearch.index.mapper;
 
-import org.apache.lucene.document.Field;
 import org.apache.lucene.document.StoredField;
 import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.Version;
@@ -45,17 +45,14 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Function;
 
-import static org.elasticsearch.common.xcontent.support.XContentMapValues.lenientNodeBooleanValue;
-
-/**
- *
- */
 public class SourceFieldMapper extends MetadataFieldMapper {
 
     public static final String NAME = "_source";
 
     public static final String CONTENT_TYPE = "_source";
+    private final Function<Map<String, ?>, Map<String, Object>> filter;
 
     public static class Defaults {
         public static final String NAME = SourceFieldMapper.NAME;
@@ -109,7 +106,7 @@ public SourceFieldMapper build(BuilderContext context) {
 
     public static class TypeParser implements MetadataFieldMapper.TypeParser {
         @Override
-        public MetadataFieldMapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
+        public MetadataFieldMapper.Builder<?,?> parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
             Builder builder = new Builder();
 
             for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
@@ -117,7 +114,7 @@ public MetadataFieldMapper.Builder parse(String name, Map<String, Object> node,
                 String fieldName = entry.getKey();
                 Object fieldNode = entry.getValue();
                 if (fieldName.equals("enabled")) {
-                    builder.enabled(lenientNodeBooleanValue(fieldNode));
+                    builder.enabled(TypeParsers.nodeBooleanValue(name, "enabled", fieldNode));
                     iterator.remove();
                 } else if ("format".equals(fieldName) && parserContext.indexVersionCreated().before(Version.V_5_0_0_alpha1)) {
                     // ignore on old indices, reject on and after 5.0
@@ -144,14 +141,15 @@ public MetadataFieldMapper.Builder parse(String name, Map<String, Object> node,
         }
 
         @Override
-        public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName) {
+        public MetadataFieldMapper getDefault(MappedFieldType fieldType, ParserContext context) {
+            final Settings indexSettings = context.mapperService().getIndexSettings().getSettings();
             return new SourceFieldMapper(indexSettings);
         }
     }
 
     static final class SourceFieldType extends MappedFieldType {
 
-        public SourceFieldType() {}
+        SourceFieldType() {}
 
         protected SourceFieldType(SourceFieldType ref) {
             super(ref);
@@ -190,6 +188,8 @@ private SourceFieldMapper(boolean enabled, String[] includes, String[] excludes,
         this.enabled = enabled;
         this.includes = includes;
         this.excludes = excludes;
+        final boolean filtered = (includes != null && includes.length > 0) || (excludes != null && excludes.length > 0);
+        this.filter = enabled && filtered && fieldType().stored() ? XContentMapValues.filter(includes, excludes) : null;
         this.complete = enabled && includes == null && excludes == null;
     }
 
@@ -226,7 +226,7 @@ public Mapper parse(ParseContext context) throws IOException {
     }
 
     @Override
-    protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
+    protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
         if (!enabled) {
             return;
         }
@@ -239,12 +239,11 @@ protected void parseCreateField(ParseContext context, List<Field> fields) throws
             return;
         }
 
-        boolean filtered = (includes != null && includes.length > 0) || (excludes != null && excludes.length > 0);
-        if (filtered) {
+        if (filter != null) {
             // we don't update the context source if we filter, we want to keep it as is...
-
-            Tuple<XContentType, Map<String, Object>> mapTuple = XContentHelper.convertToMap(source, true);
-            Map<String, Object> filteredSource = XContentMapValues.filter(mapTuple.v2(), includes, excludes);
+            Tuple<XContentType, Map<String, Object>> mapTuple =
+                XContentHelper.convertToMap(source, true, context.sourceToParse().getXContentType());
+            Map<String, Object> filteredSource = filter.apply(mapTuple.v2());
             BytesStreamOutput bStream = new BytesStreamOutput();
             XContentType contentType = mapTuple.v1();
             XContentBuilder builder = XContentFactory.contentBuilder(contentType, bStream).map(filteredSource);
@@ -275,15 +274,15 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         }
 
         if (includes != null) {
-            builder.field("includes", includes);
+            builder.array("includes", includes);
         } else if (includeDefaults) {
-            builder.field("includes", Strings.EMPTY_ARRAY);
+            builder.array("includes", Strings.EMPTY_ARRAY);
         }
 
         if (excludes != null) {
-            builder.field("excludes", excludes);
+            builder.array("excludes", excludes);
         } else if (includeDefaults) {
-            builder.field("excludes", Strings.EMPTY_ARRAY);
+            builder.array("excludes", Strings.EMPTY_ARRAY);
         }
 
         builder.endObject();
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java b/core/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java
index 074ce8829e3a0..fc34e4a0094be 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java
@@ -24,18 +24,20 @@
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentType;
 
 /**
  *
  */
 public class SourceToParse {
 
-    public static SourceToParse source(String index, String type, String id, BytesReference source) {
-        return source(Origin.PRIMARY, index, type, id, source);
+    public static SourceToParse source(String index, String type, String id, BytesReference source, XContentType contentType) {
+        return source(Origin.PRIMARY, index, type, id, source, contentType);
     }
 
-    public static SourceToParse source(Origin origin, String index, String type, String id, BytesReference source) {
-        return new SourceToParse(origin, index, type, id, source);
+    public static SourceToParse source(Origin origin, String index, String type, String id, BytesReference source,
+                                       XContentType contentType) {
+        return new SourceToParse(origin, index, type, id, source, contentType);
     }
 
     private final Origin origin;
@@ -56,14 +58,17 @@ public static SourceToParse source(Origin origin, String index, String type, Str
 
     private long ttl;
 
-    private SourceToParse(Origin origin, String index, String type, String id, BytesReference source) {
+    private XContentType xContentType;
+
+    private SourceToParse(Origin origin, String index, String type, String id, BytesReference source, XContentType xContentType) {
         this.origin = Objects.requireNonNull(origin);
         this.index = Objects.requireNonNull(index);
         this.type = Objects.requireNonNull(type);
         this.id = Objects.requireNonNull(id);
         // we always convert back to byte array, since we store it and Field only supports bytes..
         // so, we might as well do it here, and improve the performance of working with direct byte arrays
-        this.source = new BytesArray(source.toBytesRef());
+        this.source = new BytesArray(Objects.requireNonNull(source).toBytesRef());
+        this.xContentType = Objects.requireNonNull(xContentType);
     }
 
     public Origin origin() {
@@ -99,6 +104,10 @@ public String routing() {
         return this.routing;
     }
 
+    public XContentType getXContentType() {
+        return this.xContentType;
+    }
+
     public SourceToParse routing(String routing) {
         this.routing = routing;
         return this;
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/StringFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/StringFieldMapper.java
index ec7a90148ad35..da11f79729563 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/StringFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/StringFieldMapper.java
@@ -23,6 +23,7 @@
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.SortedSetDocValuesField;
 import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.Version;
@@ -88,6 +89,8 @@ public static class Defaults {
 
     public static class Builder extends FieldMapper.Builder<Builder, StringFieldMapper> {
 
+        private final DeprecationLogger deprecationLogger;
+
         protected String nullValue = Defaults.NULL_VALUE;
 
         /**
@@ -102,6 +105,8 @@ public static class Builder extends FieldMapper.Builder<Builder, StringFieldMapp
         public Builder(String name) {
             super(name, Defaults.FIELD_TYPE, Defaults.FIELD_TYPE);
             builder = this;
+            Logger logger = Loggers.getLogger(getClass());
+            this.deprecationLogger = new DeprecationLogger(logger);
         }
 
         @Override
@@ -169,12 +174,18 @@ public StringFieldMapper build(BuilderContext context) {
             }
             if (positionIncrementGap != POSITION_INCREMENT_GAP_USE_ANALYZER) {
                 if (fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) {
-                    throw new IllegalArgumentException("Cannot set position_increment_gap on field ["
-                        + name + "] without positions enabled");
+                    if (context.indexCreatedVersion().onOrAfter(Version.V_5_0_0_alpha1)) {
+                        throw new IllegalArgumentException("Cannot set position_increment_gap on field ["
+                                + name + "] without positions enabled");
+                    } else {
+                        deprecationLogger.deprecated("setting position_increment_gap on field [{}] without positions enabled " +
+                                        "is deprecated and will be ignored", name);
+                    }
+                } else {
+                    fieldType.setIndexAnalyzer(new NamedAnalyzer(fieldType.indexAnalyzer(), positionIncrementGap));
+                    fieldType.setSearchAnalyzer(new NamedAnalyzer(fieldType.searchAnalyzer(), positionIncrementGap));
+                    fieldType.setSearchQuoteAnalyzer(new NamedAnalyzer(fieldType.searchQuoteAnalyzer(), positionIncrementGap));
                 }
-                fieldType.setIndexAnalyzer(new NamedAnalyzer(fieldType.indexAnalyzer(), positionIncrementGap));
-                fieldType.setSearchAnalyzer(new NamedAnalyzer(fieldType.searchAnalyzer(), positionIncrementGap));
-                fieldType.setSearchQuoteAnalyzer(new NamedAnalyzer(fieldType.searchQuoteAnalyzer(), positionIncrementGap));
             }
             setupFieldType(context);
             return new StringFieldMapper(
@@ -218,11 +229,11 @@ public Mapper.Builder parse(String fieldName, Map<String, Object> node, ParserCo
                             norms = ((Map<?,?>) norms).get("enabled");
                         }
                         if (norms != null) {
-                            node.put("norms", TypeParsers.nodeBooleanValue("norms", norms, parserContext));
+                            node.put("norms", TypeParsers.nodeBooleanValue(fieldName,"norms", norms));
                         }
                         Object omitNorms = node.remove("omit_norms");
                         if (omitNorms != null) {
-                            node.put("norms", TypeParsers.nodeBooleanValue("omit_norms", omitNorms, parserContext) == false);
+                            node.put("norms", TypeParsers.nodeBooleanValue(fieldName, "omit_norms", omitNorms) == false);
                         }
                     }
                     {
@@ -318,13 +329,13 @@ public Mapper.Builder parse(String fieldName, Map<String, Object> node, ParserCo
                     // we need to update to actual analyzers if they are not set in this case...
                     // so we can inject the position increment gap...
                     if (builder.fieldType().indexAnalyzer() == null) {
-                        builder.fieldType().setIndexAnalyzer(parserContext.analysisService().defaultIndexAnalyzer());
+                        builder.fieldType().setIndexAnalyzer(parserContext.getIndexAnalyzers().getDefaultIndexAnalyzer());
                     }
                     if (builder.fieldType().searchAnalyzer() == null) {
-                        builder.fieldType().setSearchAnalyzer(parserContext.analysisService().defaultSearchAnalyzer());
+                        builder.fieldType().setSearchAnalyzer(parserContext.getIndexAnalyzers().getDefaultSearchAnalyzer());
                     }
                     if (builder.fieldType().searchQuoteAnalyzer() == null) {
-                        builder.fieldType().setSearchQuoteAnalyzer(parserContext.analysisService().defaultSearchQuoteAnalyzer());
+                        builder.fieldType().setSearchQuoteAnalyzer(parserContext.getIndexAnalyzers().getDefaultSearchQuoteAnalyzer());
                     }
                     iterator.remove();
                 } else if (propName.equals("ignore_above")) {
@@ -527,7 +538,7 @@ public int getIgnoreAbove() {
     }
 
     @Override
-    protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
+    protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
         ValueAndBoost valueAndBoost = parseCreateFieldForString(context, fieldType().nullValueAsString(), fieldType().boost());
         if (valueAndBoost.value() == null) {
             return;
@@ -640,7 +651,10 @@ protected void doXContentBody(XContentBuilder builder, boolean includeDefaults,
         if (includeDefaults || ignoreAbove != Defaults.IGNORE_ABOVE) {
             builder.field("ignore_above", ignoreAbove);
         }
-        if (includeDefaults || fieldType().fielddata() != ((StringFieldType) defaultFieldType).fielddata()) {
+
+        if (includeDefaults || (fieldType.indexOptions() != IndexOptions.NONE
+                && fieldType().hasDocValues() == false
+                && fieldType().fielddata() == false)) {
             builder.field("fielddata", fieldType().fielddata());
         }
         if (fieldType().fielddata()) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java b/core/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java
index 9450c2a5b9749..37834b93a1e0f 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java
@@ -22,7 +22,7 @@
 import java.util.List;
 
 import org.apache.lucene.index.Term;
-import org.apache.lucene.queries.TermsQuery;
+import org.apache.lucene.search.TermInSetQuery;
 import org.apache.lucene.search.FuzzyQuery;
 import org.apache.lucene.search.MultiTermQuery;
 import org.apache.lucene.search.PrefixQuery;
@@ -46,17 +46,18 @@ protected StringFieldType(MappedFieldType ref) {
         super(ref);
     }
 
+    @Override
     public Query termsQuery(List<?> values, QueryShardContext context) {
         failIfNotIndexed();
         BytesRef[] bytesRefs = new BytesRef[values.size()];
         for (int i = 0; i < bytesRefs.length; i++) {
             bytesRefs[i] = indexedValueForSearch(values.get(i));
         }
-        return new TermsQuery(name(), bytesRefs);
+        return new TermInSetQuery(name(), bytesRefs);
     }
 
     @Override
-    public final Query fuzzyQuery(Object value, Fuzziness fuzziness, int prefixLength, int maxExpansions,
+    public Query fuzzyQuery(Object value, Fuzziness fuzziness, int prefixLength, int maxExpansions,
             boolean transpositions) {
         failIfNotIndexed();
         return new FuzzyQuery(new Term(name(), indexedValueForSearch(value)),
@@ -64,7 +65,7 @@ public final Query fuzzyQuery(Object value, Fuzziness fuzziness, int prefixLengt
     }
 
     @Override
-    public final Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, QueryShardContext context) {
+    public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, QueryShardContext context) {
         failIfNotIndexed();
         PrefixQuery query = new PrefixQuery(new Term(name(), indexedValueForSearch(value)));
         if (method != null) {
@@ -74,7 +75,7 @@ public final Query prefixQuery(String value, MultiTermQuery.RewriteMethod method
     }
 
     @Override
-    public final Query regexpQuery(String value, int flags, int maxDeterminizedStates,
+    public Query regexpQuery(String value, int flags, int maxDeterminizedStates,
             MultiTermQuery.RewriteMethod method, QueryShardContext context) {
         failIfNotIndexed();
         RegexpQuery query = new RegexpQuery(new Term(name(), indexedValueForSearch(value)), flags, maxDeterminizedStates);
@@ -85,7 +86,7 @@ public final Query regexpQuery(String value, int flags, int maxDeterminizedState
     }
 
     @Override
-    public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper) {
+    public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) {
         failIfNotIndexed();
         return new TermRangeQuery(name(),
             lowerTerm == null ? null : indexedValueForSearch(lowerTerm),
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/TTLFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/TTLFieldMapper.java
index f95f42156e1ca..8c3ccf3c765d1 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/TTLFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/TTLFieldMapper.java
@@ -19,8 +19,8 @@
 
 package org.elasticsearch.index.mapper;
 
-import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexableField;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Settings;
@@ -28,7 +28,6 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.AlreadyExpiredException;
-import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.Date;
@@ -103,7 +102,7 @@ public MetadataFieldMapper.Builder parse(String name, Map<String, Object> node,
                 String fieldName = entry.getKey();
                 Object fieldNode = entry.getValue();
                 if (fieldName.equals("enabled")) {
-                    EnabledAttributeMapper enabledState = lenientNodeBooleanValue(fieldNode) ? EnabledAttributeMapper.ENABLED : EnabledAttributeMapper.DISABLED;
+                    EnabledAttributeMapper enabledState = lenientNodeBooleanValue(fieldNode, fieldName) ? EnabledAttributeMapper.ENABLED : EnabledAttributeMapper.DISABLED;
                     builder.enabled(enabledState);
                     iterator.remove();
                 } else if (fieldName.equals("default")) {
@@ -118,7 +117,8 @@ public MetadataFieldMapper.Builder parse(String name, Map<String, Object> node,
         }
 
         @Override
-        public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName) {
+        public MetadataFieldMapper getDefault(MappedFieldType fieldType, ParserContext context) {
+            final Settings indexSettings = context.mapperService().getIndexSettings().getSettings();
             return new TTLFieldMapper(indexSettings);
         }
     }
@@ -139,15 +139,9 @@ public TTLFieldType clone() {
 
         // Overrides valueForSearch to display live value of remaining ttl
         @Override
-        public Object valueForSearch(Object value) {
-            long now;
-            SearchContext searchContext = SearchContext.current();
-            if (searchContext != null) {
-                now = searchContext.nowInMillis();
-            } else {
-                now = System.currentTimeMillis();
-            }
-            Long val = (Long) super.valueForSearch(value);
+        public Object valueForDisplay(Object value) {
+            final long now = System.currentTimeMillis();
+            Long val = (Long) super.valueForDisplay(value);
             return val - now;
         }
     }
@@ -177,11 +171,6 @@ public long defaultTTL() {
         return this.defaultTTL;
     }
 
-    // Other implementation for realtime get display
-    public Object valueForSearch(long expirationTime) {
-        return expirationTime - System.currentTimeMillis();
-    }
-
     @Override
     public void preParse(ParseContext context) throws IOException {
     }
@@ -209,7 +198,7 @@ public Mapper parse(ParseContext context) throws IOException, MapperParsingExcep
     }
 
     @Override
-    protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException, AlreadyExpiredException {
+    protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException, AlreadyExpiredException {
         if (enabledState.enabled) {
             long ttl = context.sourceToParse().ttl();
             if (ttl <= 0 && defaultTTL > 0) { // no ttl provided so we use the default value
@@ -273,4 +262,5 @@ protected void doMerge(Mapper mergeWith, boolean updateAllTypes) {
             }
         }
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/TermBasedFieldType.java b/core/src/main/java/org/elasticsearch/index/mapper/TermBasedFieldType.java
index 71d07aa385f51..89b09cc068e63 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/TermBasedFieldType.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/TermBasedFieldType.java
@@ -22,9 +22,9 @@
 import java.util.List;
 
 import org.apache.lucene.index.Term;
-import org.apache.lucene.queries.TermsQuery;
 import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermInSetQuery;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.Version;
@@ -35,7 +35,7 @@
  *  with the inverted index. */
 abstract class TermBasedFieldType extends MappedFieldType {
 
-    public TermBasedFieldType() {}
+    TermBasedFieldType() {}
 
     protected TermBasedFieldType(MappedFieldType ref) {
         super(ref);
@@ -66,7 +66,7 @@ public Query termsQuery(List<?> values, QueryShardContext context) {
         for (int i = 0; i < bytesRefs.length; i++) {
             bytesRefs[i] = indexedValueForSearch(values.get(i));
         }
-        return new TermsQuery(name(), bytesRefs);
+        return new TermInSetQuery(name(), bytesRefs);
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java
index 53b02717cc1da..f4bf643b1673e 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java
@@ -21,6 +21,7 @@
 
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.settings.Settings;
@@ -174,13 +175,13 @@ public Mapper.Builder parse(String fieldName, Map<String, Object> node, ParserCo
                     }
                     node.put("fielddata", fielddata);
                 }
-                
+
                 return new StringFieldMapper.TypeParser().parse(fieldName, node, parserContext);
             }
             TextFieldMapper.Builder builder = new TextFieldMapper.Builder(fieldName);
-            builder.fieldType().setIndexAnalyzer(parserContext.analysisService().defaultIndexAnalyzer());
-            builder.fieldType().setSearchAnalyzer(parserContext.analysisService().defaultSearchAnalyzer());
-            builder.fieldType().setSearchQuoteAnalyzer(parserContext.analysisService().defaultSearchQuoteAnalyzer());
+            builder.fieldType().setIndexAnalyzer(parserContext.getIndexAnalyzers().getDefaultIndexAnalyzer());
+            builder.fieldType().setSearchAnalyzer(parserContext.getIndexAnalyzers().getDefaultSearchAnalyzer());
+            builder.fieldType().setSearchQuoteAnalyzer(parserContext.getIndexAnalyzers().getDefaultSearchQuoteAnalyzer());
             parseTextField(builder, fieldName, node, parserContext);
             for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
                 Map.Entry<String, Object> entry = iterator.next();
@@ -334,7 +335,7 @@ public IndexFieldData.Builder fielddataBuilder() {
             if (fielddata == false) {
                 throw new IllegalArgumentException("Fielddata is disabled on text fields by default. Set fielddata=true on [" + name()
                         + "] in order to load fielddata in memory by uninverting the inverted index. Note that this can however "
-                        + "use significant memory.");
+                                + "use significant memory. Alternatively use a keyword field instead.");
             }
             return new PagedBytesIndexFieldData.Builder(fielddataMinFrequency, fielddataMaxFrequency, fielddataMinSegmentSize);
         }
@@ -371,7 +372,7 @@ public int getPositionIncrementGap() {
     }
 
     @Override
-    protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
+    protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
         final String value;
         if (context.externalValueSet()) {
             value = context.externalValue().toString();
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/TimestampFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/TimestampFieldMapper.java
index d57d2f89c6fd6..be315b4af8c87 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/TimestampFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/TimestampFieldMapper.java
@@ -19,9 +19,9 @@
 
 package org.elasticsearch.index.mapper;
 
-import org.apache.lucene.document.Field;
 import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexableField;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.TimestampParsingException;
 import org.elasticsearch.common.joda.FormatDateTimeFormatter;
@@ -130,7 +130,7 @@ public MetadataFieldMapper.Builder parse(String name, Map<String, Object> node,
                 String fieldName = entry.getKey();
                 Object fieldNode = entry.getValue();
                 if (fieldName.equals("enabled")) {
-                    EnabledAttributeMapper enabledState = lenientNodeBooleanValue(fieldNode) ? EnabledAttributeMapper.ENABLED : EnabledAttributeMapper.DISABLED;
+                    EnabledAttributeMapper enabledState = lenientNodeBooleanValue(fieldNode, fieldName) ? EnabledAttributeMapper.ENABLED : EnabledAttributeMapper.DISABLED;
                     builder.enabled(enabledState);
                     iterator.remove();
                 } else if (fieldName.equals("format")) {
@@ -145,7 +145,7 @@ public MetadataFieldMapper.Builder parse(String name, Map<String, Object> node,
                     }
                     iterator.remove();
                 } else if (fieldName.equals("ignore_missing")) {
-                    ignoreMissing = lenientNodeBooleanValue(fieldNode);
+                    ignoreMissing = lenientNodeBooleanValue(fieldNode, fieldName);
                     builder.ignoreMissing(ignoreMissing);
                     iterator.remove();
                 }
@@ -160,7 +160,8 @@ public MetadataFieldMapper.Builder parse(String name, Map<String, Object> node,
         }
 
         @Override
-        public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName) {
+        public MetadataFieldMapper getDefault(MappedFieldType fieldType, ParserContext context) {
+            final Settings indexSettings = context.mapperService().getIndexSettings().getSettings();
             return new TimestampFieldMapper(indexSettings, fieldType);
         }
     }
@@ -179,7 +180,7 @@ public TimestampFieldType clone() {
         }
 
         @Override
-        public Object valueForSearch(Object value) {
+        public Object valueForDisplay(Object value) {
             return value;
         }
     }
@@ -237,7 +238,7 @@ public Mapper parse(ParseContext context) throws IOException {
     }
 
     @Override
-    protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
+    protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
         if (enabledState.enabled) {
             long timestamp = context.sourceToParse().timestamp();
             if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java
index 9eeaf4012fad0..d4572837e0648 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java
@@ -22,8 +22,8 @@
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexableField;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -35,6 +35,7 @@
 import java.util.Map;
 
 import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeIntegerValue;
+import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue;
 import static org.elasticsearch.index.mapper.TypeParsers.parseField;
 
 /**
@@ -46,10 +47,12 @@ public class TokenCountFieldMapper extends FieldMapper {
 
     public static class Defaults {
         public static final MappedFieldType FIELD_TYPE = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.INTEGER);
+        public static final boolean DEFAULT_POSITION_INCREMENTS = true;
     }
 
     public static class Builder extends FieldMapper.Builder<Builder, TokenCountFieldMapper> {
         private NamedAnalyzer analyzer;
+        private boolean enablePositionIncrements = Defaults.DEFAULT_POSITION_INCREMENTS;
 
         public Builder(String name) {
             super(name, Defaults.FIELD_TYPE, Defaults.FIELD_TYPE);
@@ -65,18 +68,26 @@ public NamedAnalyzer analyzer() {
             return analyzer;
         }
 
+        public Builder enablePositionIncrements(boolean enablePositionIncrements) {
+            this.enablePositionIncrements = enablePositionIncrements;
+            return this;
+        }
+
+        public boolean enablePositionIncrements() {
+            return enablePositionIncrements;
+        }
+
         @Override
         public TokenCountFieldMapper build(BuilderContext context) {
             setupFieldType(context);
             return new TokenCountFieldMapper(name, fieldType, defaultFieldType,
-                    context.indexSettings(), analyzer, multiFieldsBuilder.build(this, context), copyTo);
+                    context.indexSettings(), analyzer, enablePositionIncrements, multiFieldsBuilder.build(this, context), copyTo);
         }
     }
 
     public static class TypeParser implements Mapper.TypeParser {
         @Override
-        @SuppressWarnings("unchecked")
-        public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
+        public Mapper.Builder<?,?> parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
             if (parserContext.indexVersionCreated().before(Version.V_5_0_0_alpha2)) {
                 return new LegacyTokenCountFieldMapper.TypeParser().parse(name, node, parserContext);
             }
@@ -89,12 +100,15 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
                     builder.nullValue(nodeIntegerValue(propNode));
                     iterator.remove();
                 } else if (propName.equals("analyzer")) {
-                    NamedAnalyzer analyzer = parserContext.analysisService().analyzer(propNode.toString());
+                    NamedAnalyzer analyzer = parserContext.getIndexAnalyzers().get(propNode.toString());
                     if (analyzer == null) {
                         throw new MapperParsingException("Analyzer [" + propNode.toString() + "] not found for field [" + name + "]");
                     }
                     builder.analyzer(analyzer);
                     iterator.remove();
+                } else if (propName.equals("enable_position_increments")) {
+                    builder.enablePositionIncrements(nodeBooleanValue(propNode));
+                    iterator.remove();
                 }
             }
             parseField(builder, name, node, parserContext);
@@ -106,15 +120,17 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
     }
 
     private NamedAnalyzer analyzer;
+    private boolean enablePositionIncrements;
 
     protected TokenCountFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
-            Settings indexSettings, NamedAnalyzer analyzer, MultiFields multiFields, CopyTo copyTo) {
+            Settings indexSettings, NamedAnalyzer analyzer, boolean enablePositionIncrements, MultiFields multiFields, CopyTo copyTo) {
         super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo);
         this.analyzer = analyzer;
+        this.enablePositionIncrements = enablePositionIncrements;
     }
 
     @Override
-    protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
+    protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
         final String value;
         if (context.externalValueSet()) {
             value = context.externalValue().toString();
@@ -122,11 +138,15 @@ protected void parseCreateField(ParseContext context, List<Field> fields) throws
             value = context.parser().textOrNull();
         }
 
+        if (value == null && fieldType().nullValue() == null) {
+            return;
+        }
+
         final int tokenCount;
         if (value == null) {
             tokenCount = (Integer) fieldType().nullValue();
         } else {
-            tokenCount = countPositions(analyzer, name(), value);
+            tokenCount = countPositions(analyzer, name(), value, enablePositionIncrements);
         }
 
         boolean indexed = fieldType().indexOptions() != IndexOptions.NONE;
@@ -140,19 +160,26 @@ protected void parseCreateField(ParseContext context, List<Field> fields) throws
      * @param analyzer analyzer to create token stream
      * @param fieldName field name to pass to analyzer
      * @param fieldValue field value to pass to analyzer
+     * @param enablePositionIncrements should we count position increments ?
      * @return number of position increments in a token stream
      * @throws IOException if tokenStream throws it
      */
-    static int countPositions(Analyzer analyzer, String fieldName, String fieldValue) throws IOException {
+    static int countPositions(Analyzer analyzer, String fieldName, String fieldValue, boolean enablePositionIncrements) throws IOException {
         try (TokenStream tokenStream = analyzer.tokenStream(fieldName, fieldValue)) {
             int count = 0;
             PositionIncrementAttribute position = tokenStream.addAttribute(PositionIncrementAttribute.class);
             tokenStream.reset();
             while (tokenStream.incrementToken()) {
-                count += position.getPositionIncrement();
+                if (enablePositionIncrements) {
+                    count += position.getPositionIncrement();
+                } else {
+                    count += Math.min(1, position.getPositionIncrement());
+                }
             }
             tokenStream.end();
-            count += position.getPositionIncrement();
+            if (enablePositionIncrements) {
+                count += position.getPositionIncrement();
+            }
             return count;
         }
     }
@@ -165,6 +192,14 @@ public String analyzer() {
         return analyzer.name();
     }
 
+    /**
+     * Indicates if position increments are counted.
+     * @return <code>true</code> if position increments are counted
+     */
+    public boolean enablePositionIncrements() {
+        return enablePositionIncrements;
+    }
+
     @Override
     protected String contentType() {
         return CONTENT_TYPE;
@@ -174,12 +209,16 @@ protected String contentType() {
     protected void doMerge(Mapper mergeWith, boolean updateAllTypes) {
         super.doMerge(mergeWith, updateAllTypes);
         this.analyzer = ((TokenCountFieldMapper) mergeWith).analyzer;
+        this.enablePositionIncrements = ((TokenCountFieldMapper) mergeWith).enablePositionIncrements;
     }
 
     @Override
     protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException {
         super.doXContentBody(builder, includeDefaults, params);
         builder.field("analyzer", analyzer());
+        if (includeDefaults || enablePositionIncrements() != Defaults.DEFAULT_POSITION_INCREMENTS) {
+            builder.field("enable_position_increments", enablePositionIncrements());
+        }
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java
index 0889fab663668..68d2ac9bcdf53 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java
@@ -21,29 +21,41 @@
 
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.SortedSetDocValuesField;
+import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.TermContext;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TermInSetQuery;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.Version;
-import org.elasticsearch.common.Nullable;
+import org.elasticsearch.action.fieldstats.FieldStats;
 import org.elasticsearch.common.lucene.Lucene;
+import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
 import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData;
+import org.elasticsearch.index.fielddata.plain.ConstantIndexFieldData;
 import org.elasticsearch.index.query.QueryShardContext;
 
 import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
+import java.util.function.Function;
 
 /**
  *
@@ -73,12 +85,13 @@ public static class Defaults {
 
     public static class TypeParser implements MetadataFieldMapper.TypeParser {
         @Override
-        public MetadataFieldMapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
+        public MetadataFieldMapper.Builder<?,?> parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
             throw new MapperParsingException(NAME + " is not configurable");
         }
 
         @Override
-        public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName) {
+        public MetadataFieldMapper getDefault(MappedFieldType fieldType, ParserContext context) {
+            final Settings indexSettings = context.mapperService().getIndexSettings().getSettings();
             return new TypeFieldMapper(indexSettings, fieldType);
         }
     }
@@ -86,7 +99,7 @@ public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fi
     static final class TypeFieldType extends StringFieldType {
         private boolean fielddata;
 
-        public TypeFieldType() {
+        TypeFieldType() {
             this.fielddata = false;
         }
 
@@ -127,63 +140,148 @@ public void setFielddata(boolean fielddata) {
             checkIfFrozen();
             this.fielddata = fielddata;
         }
-
+        
         @Override
         public IndexFieldData.Builder fielddataBuilder() {
             if (hasDocValues()) {
                 return new DocValuesIndexFieldData.Builder();
-            }
-            assert indexOptions() != IndexOptions.NONE;
-            if (fielddata) {
+            } else if (fielddata) {
                 return new PagedBytesIndexFieldData.Builder(TextFieldMapper.Defaults.FIELDDATA_MIN_FREQUENCY,
-                    TextFieldMapper.Defaults.FIELDDATA_MAX_FREQUENCY,
-                    TextFieldMapper.Defaults.FIELDDATA_MIN_SEGMENT_SIZE);
+                        TextFieldMapper.Defaults.FIELDDATA_MAX_FREQUENCY,
+                        TextFieldMapper.Defaults.FIELDDATA_MIN_SEGMENT_SIZE);
+            } else {
+                // means the index has a single type and the type field is implicit
+                Function<MapperService, String> typeFunction = mapperService -> {
+                    Collection<String> types = mapperService.types();
+                    if (types.size() > 1) {
+                        throw new AssertionError();
+                    }
+                    // If we reach here, there is necessarily one type since we were able to find a `_type` field
+                    String type = types.iterator().next();
+                    return type;
+                };
+                return new ConstantIndexFieldData.Builder(typeFunction);
+            }
+        }
+
+        @Override
+        public FieldStats<?> stats(IndexReader reader) throws IOException {
+            if (reader.maxDoc() == 0) {
+                return null;
             }
-            return super.fielddataBuilder();
+            return new FieldStats.Text(reader.maxDoc(), reader.numDocs(), reader.maxDoc(), reader.maxDoc(),
+                    isSearchable(), isAggregatable());
         }
 
         @Override
-        public Query termQuery(Object value, @Nullable QueryShardContext context) {
-            if (indexOptions() == IndexOptions.NONE) {
-                throw new AssertionError();
+        public boolean isSearchable() {
+            return true;
+        }
+
+        @Override
+        public Query termQuery(Object value, QueryShardContext context) {
+            return termsQuery(Arrays.asList(value), context);
+        }
+
+        @Override
+        public Query termsQuery(List<?> values, QueryShardContext context) {
+            if (context.getIndexSettings().isSingleType()) {
+                Collection<String> indexTypes = context.getMapperService().types();
+                if (indexTypes.isEmpty()) {
+                    return new MatchNoDocsQuery("No types");
+                }
+                assert indexTypes.size() == 1;
+                BytesRef indexType = indexedValueForSearch(indexTypes.iterator().next());
+                if (values.stream()
+                        .map(this::indexedValueForSearch)
+                        .anyMatch(indexType::equals)) {
+                    if (context.getMapperService().hasNested()) {
+                        // type filters are expected not to match nested docs
+                        return Queries.newNonNestedFilter();
+                    } else {
+                        return new MatchAllDocsQuery();
+                    }
+                } else {
+                    return new MatchNoDocsQuery("Type list does not contain the index type");
+                }
+            } else {
+                if (indexOptions() == IndexOptions.NONE) {
+                    throw new AssertionError();
+                }
+                final BytesRef[] types = values.stream()
+                        .map(this::indexedValueForSearch)
+                        .toArray(size -> new BytesRef[size]);
+                return new TypesQuery(types);
             }
-            return new TypeQuery(indexedValueForSearch(value));
         }
 
         @Override
         public void checkCompatibility(MappedFieldType other,
-                                       List<String> conflicts, boolean strict) {
+                List<String> conflicts, boolean strict) {
             super.checkCompatibility(other, conflicts, strict);
             TypeFieldType otherType = (TypeFieldType) other;
             if (strict) {
                 if (fielddata() != otherType.fielddata()) {
                     conflicts.add("mapper [" + name() + "] is used by multiple types. Set update_all_types to true to update [fielddata] "
-                        + "across all types.");
+                            + "across all types.");
                 }
             }
         }
     }
 
-    public static class TypeQuery extends Query {
+    /**
+     * Specialization for a disjunction over many _type
+     */
+    public static class TypesQuery extends Query {
+        // Same threshold as TermInSetQuery
+        private static final int BOOLEAN_REWRITE_TERM_COUNT_THRESHOLD = 16;
+
+        private final BytesRef[] types;
 
-        private final BytesRef type;
+        public TypesQuery(BytesRef... types) {
+            if (types == null) {
+                throw new NullPointerException("types cannot be null.");
+            }
+            if (types.length == 0) {
+                throw new IllegalArgumentException("types must contains at least one value.");
+            }
+            this.types = types;
+        }
 
-        public TypeQuery(BytesRef type) {
-            this.type = Objects.requireNonNull(type);
+        public BytesRef[] getTerms() {
+            return types;
         }
 
         @Override
         public Query rewrite(IndexReader reader) throws IOException {
-            Term term = new Term(CONTENT_TYPE, type);
-            TermContext context = TermContext.build(reader.getContext(), term);
-            if (context.docFreq() == reader.maxDoc()) {
-                // All docs have the same type.
-                // Using a match_all query will help Lucene perform some optimizations
-                // For instance, match_all queries as filter clauses are automatically removed
-                return new MatchAllDocsQuery();
-            } else {
-                return new ConstantScoreQuery(new TermQuery(term, context));
+            final int threshold = Math.min(BOOLEAN_REWRITE_TERM_COUNT_THRESHOLD, BooleanQuery.getMaxClauseCount());
+            if (types.length <= threshold) {
+                Set<BytesRef> uniqueTypes = new HashSet<>();
+                BooleanQuery.Builder bq = new BooleanQuery.Builder();
+                int totalDocFreq = 0;
+                for (BytesRef type : types) {
+                    if (uniqueTypes.add(type)) {
+                        Term term = new Term(CONTENT_TYPE, type);
+                        TermContext context = TermContext.build(reader.getContext(), term);
+                        if (context.docFreq() == 0) {
+                            // this _type is not present in the reader
+                            continue;
+                        }
+                        totalDocFreq += context.docFreq();
+                        // strict equality should be enough ?
+                        if (totalDocFreq >= reader.maxDoc()) {
+                            assert totalDocFreq == reader.maxDoc();
+                            // Matches all docs since _type is a single value field
+                            // Using a match_all query will help Lucene perform some optimizations
+                            // For instance, match_all queries as filter clauses are automatically removed
+                            return new MatchAllDocsQuery();
+                        }
+                        bq.add(new TermQuery(term, context), BooleanClause.Occur.SHOULD);
+                    }
+                }
+                return new ConstantScoreQuery(bq.build());
             }
+            return new TermInSetQuery(CONTENT_TYPE, types);
         }
 
         @Override
@@ -191,20 +289,26 @@ public boolean equals(Object obj) {
             if (sameClassAs(obj) == false) {
                 return false;
             }
-            TypeQuery that = (TypeQuery) obj;
-            return type.equals(that.type);
+            TypesQuery that = (TypesQuery) obj;
+            return Arrays.equals(types, that.types);
         }
 
         @Override
         public int hashCode() {
-            return 31 * classHash() + type.hashCode();
+            return 31 * classHash() + Arrays.hashCode(types);
         }
 
         @Override
         public String toString(String field) {
-            return "_type:" + type;
+            StringBuilder builder = new StringBuilder();
+            for (BytesRef type : types) {
+                if (builder.length() > 0) {
+                    builder.append(' ');
+                }
+                builder.append(new Term(CONTENT_TYPE, type).toString());
+            }
+            return builder.toString();
         }
-
     }
 
     private TypeFieldMapper(Settings indexSettings, MappedFieldType existing) {
@@ -217,12 +321,18 @@ private TypeFieldMapper(MappedFieldType fieldType, Settings indexSettings) {
     }
 
     private static MappedFieldType defaultFieldType(Settings indexSettings) {
-        MappedFieldType defaultFieldType = Defaults.FIELD_TYPE.clone();
         Version indexCreated = Version.indexCreated(indexSettings);
-        if (indexCreated.before(Version.V_2_1_0)) {
+        MappedFieldType defaultFieldType = Defaults.FIELD_TYPE.clone();
+        if (MapperService.INDEX_MAPPING_SINGLE_TYPE_SETTING.get(indexSettings)) {
+            defaultFieldType.setIndexOptions(IndexOptions.NONE);
+            defaultFieldType.setHasDocValues(false);
+        } else if (indexCreated.before(Version.V_2_1_0)) {
             // enables fielddata loading, doc values was disabled on _type between 2.0 and 2.1.
             ((TypeFieldType) defaultFieldType).setFielddata(true);
+            defaultFieldType.setIndexOptions(IndexOptions.DOCS);
+            defaultFieldType.setHasDocValues(false);
         } else {
+            defaultFieldType.setIndexOptions(IndexOptions.DOCS);
             defaultFieldType.setHasDocValues(true);
         }
         return defaultFieldType;
@@ -244,7 +354,7 @@ public Mapper parse(ParseContext context) throws IOException {
     }
 
     @Override
-    protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
+    protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
         if (fieldType().indexOptions() == IndexOptions.NONE && !fieldType().stored()) {
             return;
         }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java b/core/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java
index eaa97ac5100e0..957f05aced3a7 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java
@@ -31,17 +31,13 @@
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.similarity.SimilarityProvider;
 
-import java.util.Arrays;
 import java.util.Collections;
-import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
-import java.util.Set;
 
 import static org.elasticsearch.common.xcontent.support.XContentMapValues.isArray;
-import static org.elasticsearch.common.xcontent.support.XContentMapValues.lenientNodeBooleanValue;
 import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeFloatValue;
 import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeIntegerValue;
 import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeMapValue;
@@ -52,6 +48,8 @@
  */
 public class TypeParsers {
 
+    protected static final DeprecationLogger deprecationLogger = new DeprecationLogger(Loggers.getLogger(TypeParsers.class));
+
     public static final String DOC_VALUES = "doc_values";
     public static final String INDEX_OPTIONS_DOCS = "docs";
     public static final String INDEX_OPTIONS_FREQS = "freqs";
@@ -59,19 +57,9 @@ public class TypeParsers {
     public static final String INDEX_OPTIONS_OFFSETS = "offsets";
 
     private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(TypeParsers.class));
-    private static final Set<String> BOOLEAN_STRINGS = new HashSet<>(Arrays.asList("true", "false"));
 
-    public static boolean nodeBooleanValue(String name, Object node, Mapper.TypeParser.ParserContext parserContext) {
-        // Hook onto ParseFieldMatcher so that parsing becomes strict when setting index.query.parse.strict
-        if (parserContext.parseFieldMatcher().isStrict()) {
-            return XContentMapValues.nodeBooleanValue(node);
-        } else {
-            // TODO: remove this leniency in 6.0
-            if (BOOLEAN_STRINGS.contains(node.toString()) == false) {
-                DEPRECATION_LOGGER.deprecated("Expected a boolean for property [{}] but got [{}]", name, node);
-            }
-            return XContentMapValues.lenientNodeBooleanValue(node);
-        }
+    public static boolean nodeBooleanValue(String fieldName, String propertyName, Object node) {
+        return XContentMapValues.lenientNodeBooleanValue(node, fieldName + "." + propertyName);
     }
 
     @Deprecated // for legacy ints only
@@ -85,10 +73,10 @@ public static void parseNumberField(LegacyNumberFieldMapper.Builder builder, Str
                 builder.precisionStep(nodeIntegerValue(propNode));
                 iterator.remove();
             } else if (propName.equals("ignore_malformed")) {
-                builder.ignoreMalformed(nodeBooleanValue("ignore_malformed", propNode, parserContext));
+                builder.ignoreMalformed(nodeBooleanValue(name, "ignore_malformed", propNode));
                 iterator.remove();
             } else if (propName.equals("coerce")) {
-                builder.coerce(nodeBooleanValue("coerce", propNode, parserContext));
+                builder.coerce(nodeBooleanValue(name, "coerce", propNode));
                 iterator.remove();
             } else if (propName.equals("similarity")) {
                 SimilarityProvider similarityProvider = resolveSimilarity(parserContext, name, propNode.toString());
@@ -100,7 +88,8 @@ public static void parseNumberField(LegacyNumberFieldMapper.Builder builder, Str
         }
     }
 
-    private static void parseAnalyzersAndTermVectors(FieldMapper.Builder builder, String name, Map<String, Object> fieldNode, Mapper.TypeParser.ParserContext parserContext) {
+    private static void parseAnalyzersAndTermVectors(FieldMapper.Builder builder, String name, Map<String, Object> fieldNode,
+                                                     Mapper.TypeParser.ParserContext parserContext) {
         NamedAnalyzer indexAnalyzer = null;
         NamedAnalyzer searchAnalyzer = null;
         NamedAnalyzer searchQuoteAnalyzer = null;
@@ -113,33 +102,34 @@ private static void parseAnalyzersAndTermVectors(FieldMapper.Builder builder, St
                 parseTermVector(name, propNode.toString(), builder);
                 iterator.remove();
             } else if (propName.equals("store_term_vectors")) {
-                builder.storeTermVectors(nodeBooleanValue("store_term_vectors", propNode, parserContext));
+                builder.storeTermVectors(nodeBooleanValue(name, "store_term_vectors", propNode));
                 iterator.remove();
             } else if (propName.equals("store_term_vector_offsets")) {
-                builder.storeTermVectorOffsets(nodeBooleanValue("store_term_vector_offsets", propNode, parserContext));
+                builder.storeTermVectorOffsets(nodeBooleanValue(name, "store_term_vector_offsets", propNode));
                 iterator.remove();
             } else if (propName.equals("store_term_vector_positions")) {
-                builder.storeTermVectorPositions(nodeBooleanValue("store_term_vector_positions", propNode, parserContext));
+                builder.storeTermVectorPositions(
+                    nodeBooleanValue(name, "store_term_vector_positions", propNode));
                 iterator.remove();
             } else if (propName.equals("store_term_vector_payloads")) {
-                builder.storeTermVectorPayloads(nodeBooleanValue("store_term_vector_payloads", propNode, parserContext));
+                builder.storeTermVectorPayloads(nodeBooleanValue(name,"store_term_vector_payloads", propNode));
                 iterator.remove();
             } else if (propName.equals("analyzer")) {
-                NamedAnalyzer analyzer = parserContext.analysisService().analyzer(propNode.toString());
+                NamedAnalyzer analyzer = parserContext.getIndexAnalyzers().get(propNode.toString());
                 if (analyzer == null) {
                     throw new MapperParsingException("analyzer [" + propNode.toString() + "] not found for field [" + name + "]");
                 }
                 indexAnalyzer = analyzer;
                 iterator.remove();
             } else if (propName.equals("search_analyzer")) {
-                NamedAnalyzer analyzer = parserContext.analysisService().analyzer(propNode.toString());
+                NamedAnalyzer analyzer = parserContext.getIndexAnalyzers().get(propNode.toString());
                 if (analyzer == null) {
                     throw new MapperParsingException("analyzer [" + propNode.toString() + "] not found for field [" + name + "]");
                 }
                 searchAnalyzer = analyzer;
                 iterator.remove();
             } else if (propName.equals("search_quote_analyzer")) {
-                NamedAnalyzer analyzer = parserContext.analysisService().analyzer(propNode.toString());
+                NamedAnalyzer analyzer = parserContext.getIndexAnalyzers().get(propNode.toString());
                 if (analyzer == null) {
                     throw new MapperParsingException("analyzer [" + propNode.toString() + "] not found for field [" + name + "]");
                 }
@@ -153,7 +143,8 @@ private static void parseAnalyzersAndTermVectors(FieldMapper.Builder builder, St
         }
 
         if (searchAnalyzer == null && searchQuoteAnalyzer != null) {
-            throw new MapperParsingException("analyzer and search_analyzer on field [" + name + "] must be set when search_quote_analyzer is set");
+            throw new MapperParsingException("analyzer and search_analyzer on field [" + name +
+                "] must be set when search_quote_analyzer is set");
         }
 
         if (searchAnalyzer == null) {
@@ -175,16 +166,17 @@ private static void parseAnalyzersAndTermVectors(FieldMapper.Builder builder, St
         }
     }
 
-    public static boolean parseNorms(FieldMapper.Builder builder, String propName, Object propNode, Mapper.TypeParser.ParserContext parserContext) {
+    public static boolean parseNorms(FieldMapper.Builder builder, String fieldName, String propName, Object propNode,
+                                     Mapper.TypeParser.ParserContext parserContext) {
         if (propName.equals("norms")) {
             if (propNode instanceof Map) {
                 final Map<String, Object> properties = nodeMapValue(propNode, "norms");
-                for (Iterator<Entry<String, Object>> propsIterator = properties.entrySet().iterator(); propsIterator.hasNext();) {
+                for (Iterator<Entry<String, Object>> propsIterator = properties.entrySet().iterator(); propsIterator.hasNext(); ) {
                     Entry<String, Object> entry2 = propsIterator.next();
                     final String propName2 = entry2.getKey();
                     final Object propNode2 = entry2.getValue();
                     if (propName2.equals("enabled")) {
-                        builder.omitNorms(!lenientNodeBooleanValue(propNode2));
+                        builder.omitNorms(nodeBooleanValue(fieldName, "enabled", propNode2) == false);
                         propsIterator.remove();
                     } else if (propName2.equals("loading")) {
                         // ignore for bw compat
@@ -192,13 +184,14 @@ public static boolean parseNorms(FieldMapper.Builder builder, String propName, O
                     }
                 }
                 DocumentMapperParser.checkNoRemainingFields(propName, properties, parserContext.indexVersionCreated());
-                DEPRECATION_LOGGER.deprecated("The [norms{enabled:true/false}] way of specifying norms is deprecated, please use [norms:true/false] instead");
+                DEPRECATION_LOGGER.deprecated("The [norms{enabled:true/false}] way of specifying norms is deprecated, please use " +
+                    "[norms:true/false] instead");
             } else {
-                builder.omitNorms(nodeBooleanValue("norms", propNode, parserContext) == false);
+                builder.omitNorms(nodeBooleanValue(fieldName,"norms", propNode) == false);
             }
             return true;
         } else if (propName.equals("omit_norms")) {
-            builder.omitNorms(nodeBooleanValue("norms", propNode, parserContext));
+            builder.omitNorms(nodeBooleanValue(fieldName,"norms", propNode));
             DEPRECATION_LOGGER.deprecated("[omit_norms] is deprecated, please use [norms] instead with the opposite boolean value");
             return true;
         } else {
@@ -210,14 +203,15 @@ public static boolean parseNorms(FieldMapper.Builder builder, String propName, O
      * Parse text field attributes. In addition to {@link #parseField common attributes}
      * this will parse analysis and term-vectors related settings.
      */
-    public static void parseTextField(FieldMapper.Builder builder, String name, Map<String, Object> fieldNode, Mapper.TypeParser.ParserContext parserContext) {
+    public static void parseTextField(FieldMapper.Builder builder, String name, Map<String, Object> fieldNode,
+                                      Mapper.TypeParser.ParserContext parserContext) {
         parseField(builder, name, fieldNode, parserContext);
         parseAnalyzersAndTermVectors(builder, name, fieldNode, parserContext);
-        for (Iterator<Map.Entry<String, Object>> iterator = fieldNode.entrySet().iterator(); iterator.hasNext();) {
+        for (Iterator<Map.Entry<String, Object>> iterator = fieldNode.entrySet().iterator(); iterator.hasNext(); ) {
             Map.Entry<String, Object> entry = iterator.next();
             final String propName = entry.getKey();
             final Object propNode = entry.getValue();
-            if (parseNorms(builder, propName, propNode, parserContext)) {
+            if (parseNorms(builder, name, propName, propNode, parserContext)) {
                 iterator.remove();
             }
         }
@@ -240,25 +234,33 @@ public static void parseField(FieldMapper.Builder builder, String name, Map<Stri
                 throw new MapperParsingException("[" + propName + "] must not have a [null] value");
             }
             if (propName.equals("store")) {
-                builder.store(parseStore(name, propNode.toString(), parserContext));
+                builder.store(nodeBooleanValue(name, propName, propNode));
                 iterator.remove();
             } else if (propName.equals("index")) {
-                builder.index(parseIndex(name, propNode.toString(), parserContext));
+                builder.index(parseIndex(name, propNode.toString()));
                 iterator.remove();
             } else if (propName.equals(DOC_VALUES)) {
-                builder.docValues(nodeBooleanValue(DOC_VALUES, propNode, parserContext));
+                builder.docValues(nodeBooleanValue(name, DOC_VALUES, propNode));
                 iterator.remove();
             } else if (propName.equals("boost")) {
                 builder.boost(nodeFloatValue(propNode));
                 iterator.remove();
             } else if (parserContext.indexVersionCreated().before(Version.V_5_0_0_alpha1)
-                    && parseNorms(builder, propName, propNode, parserContext)) {
+                && parseNorms(builder, name, propName, propNode, parserContext)) {
                 iterator.remove();
             } else if (propName.equals("index_options")) {
                 builder.indexOptions(nodeIndexOptionValue(propNode));
                 iterator.remove();
             } else if (propName.equals("include_in_all")) {
-                builder.includeInAll(nodeBooleanValue("include_in_all", propNode, parserContext));
+                if (parserContext.isWithinMultiField()) {
+                    deprecationLogger.deprecated("include_in_all in multi fields is deprecated "
+                            + "because it doesn't do anything. Found the include_in_all in field "
+                            + "[{}] which is within a multi field.", name);
+                } else {
+                    deprecationLogger.deprecated("field [include_in_all] is deprecated, as [_all] is deprecated, " +
+                                    "and will be disallowed in 6.0, use [copy_to] instead.");
+                    builder.includeInAll(nodeBooleanValue(name, "include_in_all", propNode));
+                }
                 iterator.remove();
             } else if (propName.equals("similarity")) {
                 SimilarityProvider similarityProvider = resolveSimilarity(parserContext, name, propNode.toString());
@@ -292,7 +294,8 @@ && parseNorms(builder, propName, propNode, parserContext)) {
         }
     }
 
-    public static boolean parseMultiField(FieldMapper.Builder builder, String name, Mapper.TypeParser.ParserContext parserContext, String propName, Object propNode) {
+    public static boolean parseMultiField(FieldMapper.Builder builder, String name, Mapper.TypeParser.ParserContext parserContext,
+                                          String propName, Object propNode) {
         parserContext = parserContext.createMultiFieldContext(parserContext);
         if (propName.equals("fields")) {
 
@@ -304,13 +307,14 @@ public static boolean parseMultiField(FieldMapper.Builder builder, String name,
                 multiFieldsPropNodes = (Map<String, Object>) propNode;
             } else {
                 throw new MapperParsingException("expected map for property [fields] on field [" + propNode + "] or " +
-                        "[" + propName + "] but got a " + propNode.getClass());
+                    "[" + propName + "] but got a " + propNode.getClass());
             }
 
             for (Map.Entry<String, Object> multiFieldEntry : multiFieldsPropNodes.entrySet()) {
                 String multiFieldName = multiFieldEntry.getKey();
                 if (multiFieldName.contains(".")) {
-                    throw new MapperParsingException("Field name [" + multiFieldName + "] which is a multi field of [" + name + "] cannot contain '.'");
+                    throw new MapperParsingException("Field name [" + multiFieldName + "] which is a multi field of [" + name + "] cannot" +
+                        " contain '.'");
                 }
                 if (!(multiFieldEntry.getValue() instanceof Map)) {
                     throw new MapperParsingException("illegal field [" + multiFieldName + "], only fields can be specified inside fields");
@@ -385,7 +389,7 @@ public static void parseTermVector(String fieldName, String termVector, FieldMap
         }
     }
 
-    public static boolean parseIndex(String fieldName, String index, Mapper.TypeParser.ParserContext parserContext) throws MapperParsingException {
+    private static boolean parseIndex(String fieldName, String index) throws MapperParsingException {
         switch (index) {
         case "true":
             return true;
@@ -394,39 +398,18 @@ public static boolean parseIndex(String fieldName, String index, Mapper.TypePars
         case "not_analyzed":
         case "analyzed":
         case "no":
-            if (parserContext.parseFieldMatcher().isStrict() == false) {
-                DEPRECATION_LOGGER.deprecated("Expected a boolean for property [index] but got [{}]", index);
-                return "no".equals(index) == false;
-            } else {
-                throw new IllegalArgumentException("Can't parse [index] value [" + index + "] for field [" + fieldName + "], expected [true] or [false]");
-            }
+            DEPRECATION_LOGGER.deprecated("Expected a boolean [true/false] for property [index] but got [{}]", index);
+            return "no".equals(index) == false;
         default:
             throw new IllegalArgumentException("Can't parse [index] value [" + index + "] for field [" + fieldName + "], expected [true] or [false]");
         }
     }
 
-    public static boolean parseStore(String fieldName, String store, Mapper.TypeParser.ParserContext parserContext) throws MapperParsingException {
-        if (parserContext.parseFieldMatcher().isStrict()) {
-            return XContentMapValues.nodeBooleanValue(store);
-        } else {
-            if (BOOLEAN_STRINGS.contains(store) == false) {
-                DEPRECATION_LOGGER.deprecated("Expected a boolean for property [store] but got [{}]", store);
-            }
-            if ("no".equals(store)) {
-                return false;
-            } else if ("yes".equals(store)) {
-                return true;
-            } else {
-                return lenientNodeBooleanValue(store);
-            }
-        }
-    }
-
     @SuppressWarnings("unchecked")
     public static void parseCopyFields(Object propNode, FieldMapper.Builder builder) {
         FieldMapper.CopyTo.Builder copyToBuilder = new FieldMapper.CopyTo.Builder();
         if (isArray(propNode)) {
-            for(Object node : (List<Object>) propNode) {
+            for (Object node : (List<Object>) propNode) {
                 copyToBuilder.add(nodeStringValue(node, null));
             }
         } else {
@@ -436,8 +419,11 @@ public static void parseCopyFields(Object propNode, FieldMapper.Builder builder)
     }
 
     private static SimilarityProvider resolveSimilarity(Mapper.TypeParser.ParserContext parserContext, String name, String value) {
-        if (parserContext.indexVersionCreated().before(Version.V_5_0_0_alpha1) && "default".equals(value)) {
-            // "default" similarity has been renamed into "classic" in 3.x.
+        if (parserContext.indexVersionCreated().before(Version.V_5_0_0_alpha1) &&
+            "default".equals(value) &&
+            // check if "default" similarity is overridden
+            parserContext.getSimilarity("default") == null) {
+            // "default" similarity has been renamed into "classic" in 5.x.
             value = "classic";
         }
         SimilarityProvider similarityProvider = parserContext.getSimilarity(value);
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/Uid.java b/core/src/main/java/org/elasticsearch/index/mapper/Uid.java
index 2a8938b4ab7ff..344c8dc0cc091 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/Uid.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/Uid.java
@@ -21,12 +21,10 @@
 
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefBuilder;
-import org.elasticsearch.action.DocumentRequest;
 import org.elasticsearch.common.lucene.BytesRefs;
 
 import java.util.Collection;
 import java.util.Collections;
-import java.util.List;
 
 /**
  *
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/UidFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/UidFieldMapper.java
index f27fa30b91cf1..1e41b25ee42bb 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/UidFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/UidFieldMapper.java
@@ -19,17 +19,33 @@
 
 package org.elasticsearch.index.mapper;
 
-import org.apache.lucene.document.BinaryDocValuesField;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.MatchNoDocsQuery;
+import org.apache.lucene.search.TermInSetQuery;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefBuilder;
+import org.apache.lucene.util.StringHelper;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.lucene.Lucene;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexFieldDataCache;
+import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
+import org.elasticsearch.index.fielddata.UidIndexFieldData;
 import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.indices.breaker.CircuitBreakerService;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 
@@ -64,6 +80,8 @@ public static class Defaults {
         }
     }
 
+    private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(UidFieldMapper.class));
+
     public static class TypeParser implements MetadataFieldMapper.TypeParser {
         @Override
         public MetadataFieldMapper.Builder<?, ?> parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
@@ -71,14 +89,15 @@ public static class TypeParser implements MetadataFieldMapper.TypeParser {
         }
 
         @Override
-        public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName) {
+        public MetadataFieldMapper getDefault(MappedFieldType fieldType, ParserContext context) {
+            final IndexSettings indexSettings = context.mapperService().getIndexSettings();
             return new UidFieldMapper(indexSettings, fieldType);
         }
     }
 
     static final class UidFieldType extends TermBasedFieldType {
 
-        public UidFieldType() {
+        UidFieldType() {
         }
 
         protected UidFieldType(UidFieldType ref) {
@@ -97,20 +116,81 @@ public String typeName() {
 
         @Override
         public IndexFieldData.Builder fielddataBuilder() {
-            // TODO: add doc values support?
-            return new PagedBytesIndexFieldData.Builder(
-                    TextFieldMapper.Defaults.FIELDDATA_MIN_FREQUENCY,
-                    TextFieldMapper.Defaults.FIELDDATA_MAX_FREQUENCY,
-                    TextFieldMapper.Defaults.FIELDDATA_MIN_SEGMENT_SIZE);
+            if (indexOptions() == IndexOptions.NONE) {
+                DEPRECATION_LOGGER.deprecated("Fielddata access on the _uid field is deprecated, use _id instead");
+                return new IndexFieldData.Builder() {
+                    @Override
+                    public IndexFieldData<?> build(IndexSettings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache,
+                            CircuitBreakerService breakerService, MapperService mapperService) {
+                        MappedFieldType idFieldType = mapperService.fullName(IdFieldMapper.NAME);
+                        IndexOrdinalsFieldData idFieldData = (IndexOrdinalsFieldData) idFieldType.fielddataBuilder()
+                                .build(indexSettings, idFieldType, cache, breakerService, mapperService);
+                        final String type = mapperService.types().iterator().next();
+                        return new UidIndexFieldData(indexSettings.getIndex(), type, idFieldData);
+                    }
+                };
+            } else {
+                // Old index, _uid was indexed
+                return new PagedBytesIndexFieldData.Builder(
+                        TextFieldMapper.Defaults.FIELDDATA_MIN_FREQUENCY,
+                        TextFieldMapper.Defaults.FIELDDATA_MAX_FREQUENCY,
+                        TextFieldMapper.Defaults.FIELDDATA_MIN_SEGMENT_SIZE);
+            }
+        }
+
+        @Override
+        public Query termQuery(Object value, @Nullable QueryShardContext context) {
+            return termsQuery(Arrays.asList(value), context);
+        }
+
+        @Override
+        public Query termsQuery(List<?> values, @Nullable QueryShardContext context) {
+            if (indexOptions() != IndexOptions.NONE) {
+                return super.termsQuery(values, context);
+            }
+            Collection<String> indexTypes = context.getMapperService().types();
+            if (indexTypes.isEmpty()) {
+                return new MatchNoDocsQuery("No types");
+            }
+            assert indexTypes.size() == 1;
+            BytesRef indexType = indexedValueForSearch(indexTypes.iterator().next());
+            BytesRefBuilder prefixBuilder = new BytesRefBuilder();
+            prefixBuilder.append(indexType);
+            prefixBuilder.append((byte) '#');
+            BytesRef expectedPrefix = prefixBuilder.get();
+            List<BytesRef> ids = new ArrayList<>();
+            for (Object uid : values) {
+                BytesRef uidBytes = indexedValueForSearch(uid);
+                if (StringHelper.startsWith(uidBytes, expectedPrefix)) {
+                    BytesRef id = new BytesRef();
+                    id.bytes = uidBytes.bytes;
+                    id.offset = uidBytes.offset + expectedPrefix.length;
+                    id.length = uidBytes.length - expectedPrefix.length;
+                    ids.add(id);
+                }
+            }
+            return new TermInSetQuery(IdFieldMapper.NAME, ids);
+        }
+    }
+
+    static MappedFieldType defaultFieldType(IndexSettings indexSettings) {
+        MappedFieldType defaultFieldType = Defaults.FIELD_TYPE.clone();
+        if (indexSettings.isSingleType()) {
+            defaultFieldType.setIndexOptions(IndexOptions.NONE);
+            defaultFieldType.setStored(false);
+        } else {
+            defaultFieldType.setIndexOptions(IndexOptions.DOCS);
+            defaultFieldType.setStored(true);
         }
+        return defaultFieldType;
     }
 
-    private UidFieldMapper(Settings indexSettings, MappedFieldType existing) {
-        this(existing == null ? Defaults.FIELD_TYPE.clone() : existing, Defaults.FIELD_TYPE, indexSettings);
+    private UidFieldMapper(IndexSettings indexSettings, MappedFieldType existing) {
+        this(existing == null ? defaultFieldType(indexSettings) : existing, indexSettings);
     }
 
-    private UidFieldMapper(MappedFieldType fieldType, MappedFieldType defaultFieldType, Settings indexSettings) {
-        super(NAME, fieldType, defaultFieldType, indexSettings);
+    private UidFieldMapper(MappedFieldType fieldType, IndexSettings indexSettings) {
+        super(NAME, fieldType, defaultFieldType(indexSettings), indexSettings.getSettings());
     }
 
     @Override
@@ -128,11 +208,10 @@ public Mapper parse(ParseContext context) throws IOException {
     }
 
     @Override
-    protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
-        Field uid = new Field(NAME, Uid.createUid(context.sourceToParse().type(), context.sourceToParse().id()), Defaults.FIELD_TYPE);
-        fields.add(uid);
-        if (fieldType().hasDocValues()) {
-            fields.add(new BinaryDocValuesField(NAME, new BytesRef(uid.stringValue())));
+    protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
+        if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) {
+            Field uid = new Field(NAME, Uid.createUid(context.sourceToParse().type(), context.sourceToParse().id()), fieldType);
+            fields.add(uid);
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java
index eb46f7a21d662..1d2e997acba1d 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java
@@ -23,6 +23,7 @@
 import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -61,14 +62,15 @@ public static class TypeParser implements MetadataFieldMapper.TypeParser {
         }
 
         @Override
-        public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName) {
+        public MetadataFieldMapper getDefault(MappedFieldType fieldType, ParserContext context) {
+            final Settings indexSettings = context.mapperService().getIndexSettings().getSettings();
             return new VersionFieldMapper(indexSettings);
         }
     }
 
     static final class VersionFieldType extends MappedFieldType {
 
-        public VersionFieldType() {
+        VersionFieldType() {
         }
 
         protected VersionFieldType(VersionFieldType ref) {
@@ -101,7 +103,7 @@ public void preParse(ParseContext context) throws IOException {
     }
 
     @Override
-    protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
+    protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
         // see InternalEngine.updateVersion to see where the real version value is set
         final Field version = new NumericDocValuesField(NAME, -1L);
         context.version(version);
diff --git a/core/src/main/java/org/elasticsearch/index/merge/MergeStats.java b/core/src/main/java/org/elasticsearch/index/merge/MergeStats.java
index ee8c08f00d077..0274a7e233005 100644
--- a/core/src/main/java/org/elasticsearch/index/merge/MergeStats.java
+++ b/core/src/main/java/org/elasticsearch/index/merge/MergeStats.java
@@ -185,12 +185,6 @@ public ByteSizeValue getCurrentSize() {
         return new ByteSizeValue(currentSizeInBytes);
     }
 
-    public static MergeStats readMergeStats(StreamInput in) throws IOException {
-        MergeStats stats = new MergeStats();
-        stats.readFrom(in);
-        return stats;
-    }
-
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject(Fields.MERGES);
diff --git a/core/src/main/java/org/elasticsearch/index/query/AbstractQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/AbstractQueryBuilder.java
index 3ed3b80fb84ec..72af541476384 100644
--- a/core/src/main/java/org/elasticsearch/index/query/AbstractQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/AbstractQueryBuilder.java
@@ -30,6 +30,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lucene.BytesRefs;
+import org.elasticsearch.common.xcontent.AbstractObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentLocation;
 import org.elasticsearch.common.xcontent.XContentType;
@@ -249,8 +250,8 @@ protected static final void writeQueries(StreamOutput out, List<? extends QueryB
     }
 
     protected static final List<QueryBuilder> readQueries(StreamInput in) throws IOException {
-        List<QueryBuilder> queries = new ArrayList<>();
         int size = in.readVInt();
+        List<QueryBuilder> queries = new ArrayList<>(size);
         for (int i = 0; i < size; i++) {
             queries.add(in.readNamedWriteable(QueryBuilder.class));
         }
@@ -282,7 +283,7 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryShardContext) throws I
      * Extracts the inner hits from the query tree.
      * While it extracts inner hits, child inner hits are inlined into the inner hit builder they belong to.
      */
-    protected void extractInnerHitBuilders(Map<String, InnerHitBuilder> innerHits) {
+    protected void extractInnerHitBuilders(Map<String, InnerHitContextBuilder> innerHits) {
     }
 
     // Like Objects.requireNotNull(...) but instead throws a IllegalArgumentException
@@ -300,4 +301,15 @@ protected static void throwParsingExceptionOnMultipleFields(String queryName, XC
                     + processedFieldName + "] and [" + currentFieldName + "]");
         }
     }
+
+    /**
+     * Adds {@code boost} and {@code query_name} parsing to the
+     * {@link AbstractObjectParser} passed in. All query builders except
+     * {@link MatchAllQueryBuilder} and {@link MatchNoneQueryBuilder} support these fields so they
+     * should use this method.
+     */
+    protected static void declareStandardFields(AbstractObjectParser<? extends QueryBuilder, ?> parser) {
+        parser.declareFloat((builder, value) -> builder.boost(value), AbstractQueryBuilder.BOOST_FIELD);
+        parser.declareString((builder, value) -> builder.queryName(value), AbstractQueryBuilder.NAME_FIELD);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java
index 7b375f125c9b1..8f887e5704b6a 100644
--- a/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java
@@ -39,6 +39,8 @@
 import java.util.Objects;
 import java.util.Optional;
 import java.util.function.Consumer;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
 
 import static org.elasticsearch.common.lucene.search.Queries.fixNegativeQueryIfNeeded;
 
@@ -57,8 +59,7 @@ public class BoolQueryBuilder extends AbstractQueryBuilder<BoolQueryBuilder> {
     private static final String SHOULD = "should";
     private static final String MUST = "must";
     private static final ParseField DISABLE_COORD_FIELD = new ParseField("disable_coord");
-    private static final ParseField MINIMUM_SHOULD_MATCH = new ParseField("minimum_should_match");
-    private static final ParseField MINIMUM_NUMBER_SHOULD_MATCH = new ParseField("minimum_number_should_match");
+    private static final ParseField MINIMUM_SHOULD_MATCH = new ParseField("minimum_should_match", "minimum_number_should_match");
     private static final ParseField ADJUST_PURE_NEGATIVE = new ParseField("adjust_pure_negative");
 
     private final List<QueryBuilder> mustClauses = new ArrayList<>();
@@ -168,7 +169,7 @@ public List<QueryBuilder> mustNot() {
      * <tt>MUST</tt> clauses one or more <code>SHOULD</code> clauses must match a document
      * for the BooleanQuery to match. No <tt>null</tt> value allowed.
      *
-     * @see #minimumNumberShouldMatch(int)
+     * @see #minimumShouldMatch(int)
      */
     public BoolQueryBuilder should(QueryBuilder queryBuilder) {
         if (queryBuilder == null) {
@@ -182,7 +183,7 @@ public BoolQueryBuilder should(QueryBuilder queryBuilder) {
      * Gets the list of clauses that <b>should</b> be matched by the returned documents.
      *
      * @see #should(QueryBuilder)
-     *  @see #minimumNumberShouldMatch(int)
+     *  @see #minimumShouldMatch(int)
      */
     public List<QueryBuilder> should() {
         return this.shouldClauses;
@@ -204,18 +205,9 @@ public boolean disableCoord() {
     }
 
     /**
-     * Specifies a minimum number of the optional (should) boolean clauses which must be satisfied.
-     * <p>
-     * By default no optional clauses are necessary for a match
-     * (unless there are no required clauses).  If this method is used,
-     * then the specified number of clauses is required.
-     * <p>
-     * Use of this method is totally independent of specifying that
-     * any specific clauses are required (or prohibited).  This number will
-     * only be compared against the number of matching optional clauses.
-     *
-     * @param minimumNumberShouldMatch the number of optional clauses that must match
+     * @deprecated use {@link BoolQueryBuilder#minimumShouldMatch(int)} instead
      */
+    @Deprecated
     public BoolQueryBuilder minimumNumberShouldMatch(int minimumNumberShouldMatch) {
         this.minimumShouldMatch = Integer.toString(minimumNumberShouldMatch);
         return this;
@@ -223,9 +215,9 @@ public BoolQueryBuilder minimumNumberShouldMatch(int minimumNumberShouldMatch) {
 
 
     /**
-     * Specifies a minimum number of the optional (should) boolean clauses which must be satisfied.
-     * @see BoolQueryBuilder#minimumNumberShouldMatch(int)
+     * @deprecated use {@link BoolQueryBuilder#minimumShouldMatch(String)} instead
      */
+    @Deprecated
     public BoolQueryBuilder minimumNumberShouldMatch(String minimumNumberShouldMatch) {
         this.minimumShouldMatch = minimumNumberShouldMatch;
         return this;
@@ -239,13 +231,32 @@ public String minimumShouldMatch() {
     }
 
     /**
-     * Sets the minimum should match using the special syntax (for example, supporting percentage).
+     * Sets the minimum should match parameter using the special syntax (for example, supporting percentage).
+     * @see BoolQueryBuilder#minimumShouldMatch(int)
      */
     public BoolQueryBuilder minimumShouldMatch(String minimumShouldMatch) {
         this.minimumShouldMatch = minimumShouldMatch;
         return this;
     }
 
+    /**
+     * Specifies a minimum number of the optional (should) boolean clauses which must be satisfied.
+     * <p>
+     * By default no optional clauses are necessary for a match
+     * (unless there are no required clauses).  If this method is used,
+     * then the specified number of clauses is required.
+     * <p>
+     * Use of this method is totally independent of specifying that
+     * any specific clauses are required (or prohibited).  This number will
+     * only be compared against the number of matching optional clauses.
+     *
+     * @param minimumShouldMatch the number of optional clauses that must match
+     */
+    public BoolQueryBuilder minimumShouldMatch(int minimumShouldMatch) {
+        this.minimumShouldMatch = Integer.toString(minimumShouldMatch);
+        return this;
+    }
+
     /**
      * Returns <code>true</code> iff this query builder has at least one should, must, must not or filter clause.
      * Otherwise <code>false</code>.
@@ -338,10 +349,6 @@ public static Optional<BoolQueryBuilder> fromXContent(QueryParseContext parseCon
                 default:
                     throw new ParsingException(parser.getTokenLocation(), "[bool] query does not support [" + currentFieldName + "]");
                 }
-                if (parser.currentToken() != XContentParser.Token.END_OBJECT) {
-                    throw new ParsingException(parser.getTokenLocation(),
-                            "expected [END_OBJECT] but got [{}], possibly too many query clauses", parser.currentToken());
-                }
             } else if (token == XContentParser.Token.START_ARRAY) {
                 while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                     switch (currentFieldName) {
@@ -363,17 +370,15 @@ public static Optional<BoolQueryBuilder> fromXContent(QueryParseContext parseCon
                     }
                 }
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, DISABLE_COORD_FIELD)) {
+                if (DISABLE_COORD_FIELD.match(currentFieldName)) {
                     disableCoord = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, MINIMUM_SHOULD_MATCH)) {
+                } else if (MINIMUM_SHOULD_MATCH.match(currentFieldName)) {
                     minimumShouldMatch = parser.textOrNull();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, MINIMUM_NUMBER_SHOULD_MATCH)) {
-                    minimumShouldMatch = parser.textOrNull();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, ADJUST_PURE_NEGATIVE)) {
+                } else if (ADJUST_PURE_NEGATIVE.match(currentFieldName)) {
                     adjustPureNegative = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[bool] query does not support [" + currentFieldName + "]");
@@ -396,7 +401,7 @@ public static Optional<BoolQueryBuilder> fromXContent(QueryParseContext parseCon
         boolQuery.boost(boost);
         boolQuery.disableCoord(disableCoord);
         boolQuery.adjustPureNegative(adjustPureNegative);
-        boolQuery.minimumNumberShouldMatch(minimumShouldMatch);
+        boolQuery.minimumShouldMatch(minimumShouldMatch);
         boolQuery.queryName(queryName);
         return Optional.of(boolQuery);
     }
@@ -476,7 +481,12 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws
         changed |= rewriteClauses(queryRewriteContext, mustNotClauses, newBuilder::mustNot);
         changed |= rewriteClauses(queryRewriteContext, filterClauses, newBuilder::filter);
         changed |= rewriteClauses(queryRewriteContext, shouldClauses, newBuilder::should);
-
+        // lets do some early termination and prevent any kind of rewriting if we have a mandatory query that is a MatchNoneQueryBuilder
+        Optional<QueryBuilder> any = Stream.concat(newBuilder.mustClauses.stream(), newBuilder.filterClauses.stream())
+            .filter(b -> b instanceof MatchNoneQueryBuilder).findAny();
+        if (any.isPresent()) {
+            return any.get();
+        }
         if (changed) {
             newBuilder.adjustPureNegative = adjustPureNegative;
             newBuilder.disableCoord = disableCoord;
@@ -489,13 +499,13 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws
     }
 
     @Override
-    protected void extractInnerHitBuilders(Map<String, InnerHitBuilder> innerHits) {
+    protected void extractInnerHitBuilders(Map<String, InnerHitContextBuilder> innerHits) {
         List<QueryBuilder> clauses = new ArrayList<>(filter());
         clauses.addAll(must());
         clauses.addAll(should());
         // no need to include must_not (since there will be no hits for it)
         for (QueryBuilder clause : clauses) {
-            InnerHitBuilder.extractInnerHits(clause, innerHits);
+            InnerHitContextBuilder.extractInnerHits(clause, innerHits);
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/query/BoostingQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/BoostingQueryBuilder.java
index 4c86fce4a57cd..2523861e8b4d3 100644
--- a/core/src/main/java/org/elasticsearch/index/query/BoostingQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/BoostingQueryBuilder.java
@@ -154,21 +154,21 @@ public static Optional<BoostingQueryBuilder> fromXContent(QueryParseContext pars
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, POSITIVE_FIELD)) {
+                if (POSITIVE_FIELD.match(currentFieldName)) {
                     positiveQuery = parseContext.parseInnerQueryBuilder();
                     positiveQueryFound = true;
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, NEGATIVE_FIELD)) {
+                } else if (NEGATIVE_FIELD.match(currentFieldName)) {
                     negativeQuery = parseContext.parseInnerQueryBuilder();
                     negativeQueryFound = true;
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[boosting] query does not support [" + currentFieldName + "]");
                 }
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, NEGATIVE_BOOST_FIELD)) {
+                if (NEGATIVE_BOOST_FIELD.match(currentFieldName)) {
                     negativeBoost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[boosting] query does not support [" + currentFieldName + "]");
@@ -234,8 +234,8 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws
     }
 
     @Override
-    protected void extractInnerHitBuilders(Map<String, InnerHitBuilder> innerHits) {
-        InnerHitBuilder.extractInnerHits(positiveQuery, innerHits);
-        InnerHitBuilder.extractInnerHits(negativeQuery, innerHits);
+    protected void extractInnerHitBuilders(Map<String, InnerHitContextBuilder> innerHits) {
+        InnerHitContextBuilder.extractInnerHits(positiveQuery, innerHits);
+        InnerHitContextBuilder.extractInnerHits(negativeQuery, innerHits);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java
index 21328ff8fc4b7..97b499fee206a 100644
--- a/core/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java
@@ -291,15 +291,15 @@ public static Optional<CommonTermsQueryBuilder> fromXContent(QueryParseContext p
                     if (token == XContentParser.Token.FIELD_NAME) {
                         currentFieldName = parser.currentName();
                     } else if (token == XContentParser.Token.START_OBJECT) {
-                        if (parseContext.getParseFieldMatcher().match(currentFieldName, MINIMUM_SHOULD_MATCH_FIELD)) {
+                        if (MINIMUM_SHOULD_MATCH_FIELD.match(currentFieldName)) {
                             String innerFieldName = null;
                             while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
                                 if (token == XContentParser.Token.FIELD_NAME) {
                                     innerFieldName = parser.currentName();
                                 } else if (token.isValue()) {
-                                    if (parseContext.getParseFieldMatcher().match(innerFieldName, LOW_FREQ_FIELD)) {
+                                    if (LOW_FREQ_FIELD.match(innerFieldName)) {
                                         lowFreqMinimumShouldMatch = parser.text();
-                                    } else if (parseContext.getParseFieldMatcher().match(innerFieldName, HIGH_FREQ_FIELD)) {
+                                    } else if (HIGH_FREQ_FIELD.match(innerFieldName)) {
                                         highFreqMinimumShouldMatch = parser.text();
                                     } else {
                                         throw new ParsingException(parser.getTokenLocation(), "[" + CommonTermsQueryBuilder.NAME +
@@ -317,23 +317,23 @@ public static Optional<CommonTermsQueryBuilder> fromXContent(QueryParseContext p
                                     "] query does not support [" + currentFieldName + "]");
                         }
                     } else if (token.isValue()) {
-                        if (parseContext.getParseFieldMatcher().match(currentFieldName, QUERY_FIELD)) {
+                        if (QUERY_FIELD.match(currentFieldName)) {
                             text = parser.objectText();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, ANALYZER_FIELD)) {
+                        } else if (ANALYZER_FIELD.match(currentFieldName)) {
                             analyzer = parser.text();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, DISABLE_COORD_FIELD)) {
+                        } else if (DISABLE_COORD_FIELD.match(currentFieldName)) {
                             disableCoord = parser.booleanValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                        } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                             boost = parser.floatValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, HIGH_FREQ_OPERATOR_FIELD)) {
+                        } else if (HIGH_FREQ_OPERATOR_FIELD.match(currentFieldName)) {
                             highFreqOperator = Operator.fromString(parser.text());
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, LOW_FREQ_OPERATOR_FIELD)) {
+                        } else if (LOW_FREQ_OPERATOR_FIELD.match(currentFieldName)) {
                             lowFreqOperator = Operator.fromString(parser.text());
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, MINIMUM_SHOULD_MATCH_FIELD)) {
+                        } else if (MINIMUM_SHOULD_MATCH_FIELD.match(currentFieldName)) {
                             lowFreqMinimumShouldMatch = parser.text();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, CUTOFF_FREQUENCY_FIELD)) {
+                        } else if (CUTOFF_FREQUENCY_FIELD.match(currentFieldName)) {
                             cutoffFrequency = parser.floatValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                        } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                             queryName = parser.text();
                         } else {
                             throw new ParsingException(parser.getTokenLocation(), "[" + CommonTermsQueryBuilder.NAME +
@@ -383,7 +383,7 @@ protected Query doToQuery(QueryShardContext context) throws IOException {
                 analyzerObj = context.getMapperService().searchAnalyzer();
             }
         } else {
-            analyzerObj = context.getMapperService().analysisService().analyzer(analyzer);
+            analyzerObj = context.getMapperService().getIndexAnalyzers().get(analyzer);
             if (analyzerObj == null) {
                 throw new QueryShardException(context, "[common] analyzer [" + analyzer + "] not found");
             }
diff --git a/core/src/main/java/org/elasticsearch/index/query/ConstantScoreQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/ConstantScoreQueryBuilder.java
index 99b6ba731d23b..e9c54fee3273d 100644
--- a/core/src/main/java/org/elasticsearch/index/query/ConstantScoreQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/ConstantScoreQueryBuilder.java
@@ -102,7 +102,7 @@ public static Optional<ConstantScoreQueryBuilder> fromXContent(QueryParseContext
             } else if (parseContext.isDeprecatedSetting(currentFieldName)) {
                 // skip
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, INNER_QUERY_FIELD)) {
+                if (INNER_QUERY_FIELD.match(currentFieldName)) {
                     if (queryFound) {
                         throw new ParsingException(parser.getTokenLocation(), "[" + ConstantScoreQueryBuilder.NAME + "]"
                                 + " accepts only one 'filter' element.");
@@ -114,9 +114,9 @@ public static Optional<ConstantScoreQueryBuilder> fromXContent(QueryParseContext
                             "[constant_score] query does not support [" + currentFieldName + "]");
                 }
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
@@ -165,6 +165,9 @@ protected boolean doEquals(ConstantScoreQueryBuilder other) {
     @Override
     protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException {
         QueryBuilder rewrite = filterBuilder.rewrite(queryRewriteContext);
+        if (rewrite instanceof MatchNoneQueryBuilder) {
+            return rewrite; // we won't match anyway
+        }
         if (rewrite != filterBuilder) {
             return new ConstantScoreQueryBuilder(rewrite);
         }
@@ -172,7 +175,7 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws
     }
 
     @Override
-    protected void extractInnerHitBuilders(Map<String, InnerHitBuilder> innerHits) {
-        InnerHitBuilder.extractInnerHits(filterBuilder, innerHits);
+    protected void extractInnerHitBuilders(Map<String, InnerHitContextBuilder> innerHits) {
+        InnerHitContextBuilder.extractInnerHits(filterBuilder, innerHits);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/query/DisMaxQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/DisMaxQueryBuilder.java
index beeb5a05f45bf..3cb95007a0d84 100644
--- a/core/src/main/java/org/elasticsearch/index/query/DisMaxQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/DisMaxQueryBuilder.java
@@ -33,6 +33,7 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 
@@ -138,14 +139,14 @@ public static Optional<DisMaxQueryBuilder> fromXContent(QueryParseContext parseC
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, QUERIES_FIELD)) {
+                if (QUERIES_FIELD.match(currentFieldName)) {
                     queriesFound = true;
                     parseContext.parseInnerQueryBuilder().ifPresent(queries::add);
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[dis_max] query does not support [" + currentFieldName + "]");
                 }
             } else if (token == XContentParser.Token.START_ARRAY) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, QUERIES_FIELD)) {
+                if (QUERIES_FIELD.match(currentFieldName)) {
                     queriesFound = true;
                     while (token != XContentParser.Token.END_ARRAY) {
                         parseContext.parseInnerQueryBuilder().ifPresent(queries::add);
@@ -155,11 +156,11 @@ public static Optional<DisMaxQueryBuilder> fromXContent(QueryParseContext parseC
                     throw new ParsingException(parser.getTokenLocation(), "[dis_max] query does not support [" + currentFieldName + "]");
                 }
             } else {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, TIE_BREAKER_FIELD)) {
+                } else if (TIE_BREAKER_FIELD.match(currentFieldName)) {
                     tieBreaker = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[dis_max] query does not support [" + currentFieldName + "]");
@@ -207,4 +208,11 @@ protected boolean doEquals(DisMaxQueryBuilder other) {
     public String getWriteableName() {
         return NAME;
     }
+
+    @Override
+    protected void extractInnerHitBuilders(Map<String, InnerHitContextBuilder> innerHits) {
+        for (QueryBuilder query : queries) {
+            InnerHitContextBuilder.extractInnerHits(query, innerHits);
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/query/ExistsQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/ExistsQueryBuilder.java
index 93e491d626929..964edc7349246 100644
--- a/core/src/main/java/org/elasticsearch/index/query/ExistsQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/ExistsQueryBuilder.java
@@ -97,11 +97,11 @@ public static Optional<ExistsQueryBuilder> fromXContent(QueryParseContext parseC
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, FIELD_FIELD)) {
+                if (FIELD_FIELD.match(currentFieldName)) {
                     fieldPattern = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[" + ExistsQueryBuilder.NAME +
@@ -130,7 +130,7 @@ protected Query doToQuery(QueryShardContext context) throws IOException {
 
     public static Query newFilter(QueryShardContext context, String fieldPattern) {
         final FieldNamesFieldMapper.FieldNamesFieldType fieldNamesFieldType =
-                (FieldNamesFieldMapper.FieldNamesFieldType)context.getMapperService().fullName(FieldNamesFieldMapper.NAME);
+                (FieldNamesFieldMapper.FieldNamesFieldType) context.getMapperService().fullName(FieldNamesFieldMapper.NAME);
         if (fieldNamesFieldType == null) {
             // can only happen when no types exist, so no docs exist either
             return Queries.newMatchNoDocsQuery("Missing types in \"" + NAME + "\" query.");
@@ -145,6 +145,11 @@ public static Query newFilter(QueryShardContext context, String fieldPattern) {
             fields = context.simpleMatchToIndexNames(fieldPattern);
         }
 
+        if (fields.size() == 1) {
+            Query filter = fieldNamesFieldType.termQuery(fields.iterator().next(), context);
+            return new ConstantScoreQuery(filter);
+        }
+
         BooleanQuery.Builder boolFilterBuilder = new BooleanQuery.Builder();
         for (String field : fields) {
             Query filter = fieldNamesFieldType.termQuery(field, context);
diff --git a/core/src/main/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilder.java
index dc8bc786aed8b..cc3ad2e191f2e 100644
--- a/core/src/main/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilder.java
@@ -116,7 +116,7 @@ public static Optional<FieldMaskingSpanQueryBuilder> fromXContent(QueryParseCont
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, QUERY_FIELD)) {
+                if (QUERY_FIELD.match(currentFieldName)) {
                     Optional<QueryBuilder> query = parseContext.parseInnerQueryBuilder();
                     if (query.isPresent() == false || query.get() instanceof SpanQueryBuilder == false) {
                         throw new ParsingException(parser.getTokenLocation(), "[field_masking_span] query must be of type span query");
@@ -127,11 +127,11 @@ public static Optional<FieldMaskingSpanQueryBuilder> fromXContent(QueryParseCont
                             + currentFieldName + "]");
                 }
             } else {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, FIELD_FIELD)) {
+                } else if (FIELD_FIELD.match(currentFieldName)) {
                     field = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
diff --git a/core/src/main/java/org/elasticsearch/index/query/FuzzyQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/FuzzyQueryBuilder.java
index c75ba1fda99f4..92289f373c07f 100644
--- a/core/src/main/java/org/elasticsearch/index/query/FuzzyQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/FuzzyQueryBuilder.java
@@ -41,11 +41,7 @@
 
 /**
  * A Query that does fuzzy matching for a specific value.
- *
- * @deprecated Fuzzy queries are not useful enough. This class will be removed with Elasticsearch 4.0. In most cases you may want to use
- * a match query with the fuzziness parameter for strings or range queries for numeric and date fields.
  */
-@Deprecated
 public class FuzzyQueryBuilder extends AbstractQueryBuilder<FuzzyQueryBuilder> implements MultiTermQueryBuilder {
     public static final String NAME = "fuzzy";
 
@@ -90,6 +86,7 @@ public class FuzzyQueryBuilder extends AbstractQueryBuilder<FuzzyQueryBuilder> i
      * @param fieldName  The name of the field
      * @param value The value of the text
      */
+    @Deprecated
     public FuzzyQueryBuilder(String fieldName, String value) {
         this(fieldName, (Object) value);
     }
@@ -100,6 +97,7 @@ public FuzzyQueryBuilder(String fieldName, String value) {
      * @param fieldName  The name of the field
      * @param value The value of the text
      */
+    @Deprecated
     public FuzzyQueryBuilder(String fieldName, int value) {
         this(fieldName, (Object) value);
     }
@@ -110,6 +108,7 @@ public FuzzyQueryBuilder(String fieldName, int value) {
      * @param fieldName  The name of the field
      * @param value The value of the text
      */
+    @Deprecated
     public FuzzyQueryBuilder(String fieldName, long value) {
         this(fieldName, (Object) value);
     }
@@ -120,6 +119,7 @@ public FuzzyQueryBuilder(String fieldName, long value) {
      * @param fieldName  The name of the field
      * @param value The value of the text
      */
+    @Deprecated
     public FuzzyQueryBuilder(String fieldName, float value) {
         this(fieldName, (Object) value);
     }
@@ -130,6 +130,7 @@ public FuzzyQueryBuilder(String fieldName, float value) {
      * @param fieldName  The name of the field
      * @param value The value of the text
      */
+    @Deprecated
     public FuzzyQueryBuilder(String fieldName, double value) {
         this(fieldName, (Object) value);
     }
@@ -140,6 +141,7 @@ public FuzzyQueryBuilder(String fieldName, double value) {
      * @param fieldName  The name of the field
      * @param value The value of the text
      */
+    @Deprecated
     public FuzzyQueryBuilder(String fieldName, boolean value) {
         this(fieldName, (Object) value);
     }
@@ -150,6 +152,7 @@ public FuzzyQueryBuilder(String fieldName, boolean value) {
      * @param fieldName  The name of the field
      * @param value The value of the term
      */
+    @Deprecated
     public FuzzyQueryBuilder(String fieldName, Object value) {
         if (Strings.isEmpty(fieldName)) {
             throw new IllegalArgumentException("field name cannot be null or empty");
@@ -280,29 +283,32 @@ public static Optional<FuzzyQueryBuilder> fromXContent(QueryParseContext parseCo
                 while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
                     if (token == XContentParser.Token.FIELD_NAME) {
                         currentFieldName = parser.currentName();
-                    } else {
-                        if (parseContext.getParseFieldMatcher().match(currentFieldName, TERM_FIELD)) {
+                    } else if (token.isValue()) {
+                        if (TERM_FIELD.match(currentFieldName)) {
                             value = parser.objectBytes();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, VALUE_FIELD)) {
+                        } else if (VALUE_FIELD.match(currentFieldName)) {
                             value = parser.objectBytes();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                        } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                             boost = parser.floatValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, Fuzziness.FIELD)) {
+                        } else if (Fuzziness.FIELD.match(currentFieldName)) {
                             fuzziness = Fuzziness.parse(parser);
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, PREFIX_LENGTH_FIELD)) {
+                        } else if (PREFIX_LENGTH_FIELD.match(currentFieldName)) {
                             prefixLength = parser.intValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, MAX_EXPANSIONS_FIELD)) {
+                        } else if (MAX_EXPANSIONS_FIELD.match(currentFieldName)) {
                             maxExpansions = parser.intValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, TRANSPOSITIONS_FIELD)) {
+                        } else if (TRANSPOSITIONS_FIELD.match(currentFieldName)) {
                             transpositions = parser.booleanValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, REWRITE_FIELD)) {
+                        } else if (REWRITE_FIELD.match(currentFieldName)) {
                             rewrite = parser.textOrNull();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                        } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                             queryName = parser.text();
                         } else {
                             throw new ParsingException(parser.getTokenLocation(),
                                     "[fuzzy] query does not support [" + currentFieldName + "]");
                         }
+                    } else {
+                        throw new ParsingException(parser.getTokenLocation(),
+                                "[" + NAME + "] unexpected token [" + token + "] after [" + currentFieldName + "]");
                     }
                 }
             } else {
@@ -342,7 +348,7 @@ protected Query doToQuery(QueryShardContext context) throws IOException {
             query = new FuzzyQuery(new Term(fieldName, BytesRefs.toBytesRef(value)), maxEdits, prefixLength, maxExpansions, transpositions);
         }
         if (query instanceof MultiTermQuery) {
-            MultiTermQuery.RewriteMethod rewriteMethod = QueryParsers.parseRewriteMethod(context.getParseFieldMatcher(), rewrite, null);
+            MultiTermQuery.RewriteMethod rewriteMethod = QueryParsers.parseRewriteMethod(rewrite, null);
             QueryParsers.setRewriteMethod((MultiTermQuery) query, rewriteMethod);
         }
         return query;
diff --git a/core/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java
index ee7220111f378..a8819d5e76e38 100644
--- a/core/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java
@@ -19,7 +19,10 @@
 
 package org.elasticsearch.index.query;
 
+import org.apache.lucene.document.LatLonDocValuesField;
+import org.apache.lucene.document.LatLonPoint;
 import org.apache.lucene.geo.Rectangle;
+import org.apache.lucene.search.IndexOrDocValuesQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.spatial.geopoint.document.GeoPointField;
@@ -38,7 +41,8 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.fielddata.IndexGeoPointFieldData;
 import org.elasticsearch.index.mapper.BaseGeoPointFieldMapper;
-import org.elasticsearch.index.mapper.LegacyGeoPointFieldMapper;
+import org.elasticsearch.index.mapper.BaseGeoPointFieldMapper.LegacyGeoPointFieldType;
+import org.elasticsearch.index.mapper.LatLonPointFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.search.geo.LegacyInMemoryGeoBoundingBoxQuery;
 import org.elasticsearch.index.search.geo.LegacyIndexedGeoBoundingBoxQuery;
@@ -69,9 +73,9 @@ public class GeoBoundingBoxQueryBuilder extends AbstractQueryBuilder<GeoBounding
     private static final ParseField TYPE_FIELD = new ParseField("type");
     private static final ParseField VALIDATION_METHOD_FIELD = new ParseField("validation_method");
     private static final ParseField COERCE_FIELD =new ParseField("coerce", "normalize")
-            .withAllDeprecated("use field validation_method instead");
+            .withAllDeprecated("validation_method");
     private static final ParseField IGNORE_MALFORMED_FIELD = new ParseField("ignore_malformed")
-            .withAllDeprecated("use field validation_method instead");
+            .withAllDeprecated("validation_method");
     private static final ParseField FIELD_FIELD = new ParseField("field");
     private static final ParseField TOP_FIELD = new ParseField("top");
     private static final ParseField BOTTOM_FIELD = new ParseField("bottom");
@@ -297,7 +301,7 @@ public boolean ignoreUnmapped() {
 
     QueryValidationException checkLatLon(boolean indexCreatedBeforeV2_0) {
         // validation was not available prior to 2.x, so to support bwc percolation queries we only ignore_malformed on 2.x created indexes
-        if (GeoValidationMethod.isIgnoreMalformed(validationMethod) == true || indexCreatedBeforeV2_0) {
+        if (GeoValidationMethod.isIgnoreMalformed(validationMethod) || indexCreatedBeforeV2_0) {
             return null;
         }
 
@@ -359,7 +363,17 @@ public Query doToQuery(QueryShardContext context) {
             }
         }
 
-        if (indexVersionCreated.onOrAfter(Version.V_2_2_0)) {
+        if (indexVersionCreated.onOrAfter(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            Query query = LatLonPoint.newBoxQuery(fieldType.name(), luceneBottomRight.getLat(), luceneTopLeft.getLat(),
+                luceneTopLeft.getLon(), luceneBottomRight.getLon());
+            if (fieldType.hasDocValues()) {
+                Query dvQuery = LatLonDocValuesField.newBoxQuery(fieldType.name(),
+                        luceneBottomRight.getLat(), luceneTopLeft.getLat(),
+                        luceneTopLeft.getLon(), luceneBottomRight.getLon());
+                query = new IndexOrDocValuesQuery(query, dvQuery);
+            }
+            return query;
+        } else if (indexVersionCreated.onOrAfter(Version.V_2_2_0)) {
             // if index created V_2_2 use (soon to be legacy) numeric encoding postings format
             // if index created V_2_3 > use prefix encoded postings format
             final GeoPointField.TermEncoding encoding = (indexVersionCreated.before(Version.V_2_3_0)) ?
@@ -371,8 +385,8 @@ public Query doToQuery(QueryShardContext context) {
         Query query;
         switch(type) {
             case INDEXED:
-                LegacyGeoPointFieldMapper.GeoPointFieldType geoFieldType = ((LegacyGeoPointFieldMapper.GeoPointFieldType) fieldType);
-                query = LegacyIndexedGeoBoundingBoxQuery.create(luceneTopLeft, luceneBottomRight, geoFieldType);
+                LegacyGeoPointFieldType geoFieldType = ((LegacyGeoPointFieldType) fieldType);
+                query = LegacyIndexedGeoBoundingBoxQuery.create(luceneTopLeft, luceneBottomRight, geoFieldType, context);
                 break;
             case MEMORY:
                 IndexGeoPointFieldData indexFieldData = context.getForField(fieldType);
@@ -438,30 +452,30 @@ public static Optional<GeoBoundingBoxQueryBuilder> fromXContent(QueryParseContex
                         token = parser.nextToken();
                         if (parseContext.isDeprecatedSetting(currentFieldName)) {
                             // skip
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, FIELD_FIELD)) {
+                        } else if (FIELD_FIELD.match(currentFieldName)) {
                             fieldName = parser.text();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, TOP_FIELD)) {
+                        } else if (TOP_FIELD.match(currentFieldName)) {
                             top = parser.doubleValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, BOTTOM_FIELD)) {
+                        } else if (BOTTOM_FIELD.match(currentFieldName)) {
                             bottom = parser.doubleValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, LEFT_FIELD)) {
+                        } else if (LEFT_FIELD.match(currentFieldName)) {
                             left = parser.doubleValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, RIGHT_FIELD)) {
+                        } else if (RIGHT_FIELD.match(currentFieldName)) {
                             right = parser.doubleValue();
                         } else {
-                            if (parseContext.getParseFieldMatcher().match(currentFieldName, TOP_LEFT_FIELD)) {
+                            if (TOP_LEFT_FIELD.match(currentFieldName)) {
                                 GeoUtils.parseGeoPoint(parser, sparse);
                                 top = sparse.getLat();
                                 left = sparse.getLon();
-                            } else if (parseContext.getParseFieldMatcher().match(currentFieldName, BOTTOM_RIGHT_FIELD)) {
+                            } else if (BOTTOM_RIGHT_FIELD.match(currentFieldName)) {
                                 GeoUtils.parseGeoPoint(parser, sparse);
                                 bottom = sparse.getLat();
                                 right = sparse.getLon();
-                            } else if (parseContext.getParseFieldMatcher().match(currentFieldName, TOP_RIGHT_FIELD)) {
+                            } else if (TOP_RIGHT_FIELD.match(currentFieldName)) {
                                 GeoUtils.parseGeoPoint(parser, sparse);
                                 top = sparse.getLat();
                                 right = sparse.getLon();
-                            } else if (parseContext.getParseFieldMatcher().match(currentFieldName, BOTTOM_LEFT_FIELD)) {
+                            } else if (BOTTOM_LEFT_FIELD.match(currentFieldName)) {
                                 GeoUtils.parseGeoPoint(parser, sparse);
                                 bottom = sparse.getLat();
                                 left = sparse.getLon();
@@ -476,22 +490,22 @@ public static Optional<GeoBoundingBoxQueryBuilder> fromXContent(QueryParseContex
                     }
                 }
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, COERCE_FIELD)) {
+                } else if (COERCE_FIELD.match(currentFieldName)) {
                     coerce = parser.booleanValue();
                     if (coerce) {
                         ignoreMalformed = true;
                     }
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, VALIDATION_METHOD_FIELD)) {
+                } else if (VALIDATION_METHOD_FIELD.match(currentFieldName)) {
                     validationMethod = GeoValidationMethod.fromString(parser.text());
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, IGNORE_UNMAPPED_FIELD)) {
+                } else if (IGNORE_UNMAPPED_FIELD.match(currentFieldName)) {
                     ignoreUnmapped = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, TYPE_FIELD)) {
+                } else if (TYPE_FIELD.match(currentFieldName)) {
                     type = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, IGNORE_MALFORMED_FIELD)) {
+                } else if (IGNORE_MALFORMED_FIELD.match(currentFieldName)) {
                     ignoreMalformed = parser.booleanValue();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "failed to parse [{}] query. unexpected field [{}]",
diff --git a/core/src/main/java/org/elasticsearch/index/query/GeoDistanceQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/GeoDistanceQueryBuilder.java
index a0226f44f76ff..ea9c14f56ed31 100644
--- a/core/src/main/java/org/elasticsearch/index/query/GeoDistanceQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/GeoDistanceQueryBuilder.java
@@ -19,6 +19,9 @@
 
 package org.elasticsearch.index.query;
 
+import org.apache.lucene.document.LatLonDocValuesField;
+import org.apache.lucene.document.LatLonPoint;
+import org.apache.lucene.search.IndexOrDocValuesQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.spatial.geopoint.document.GeoPointField;
@@ -38,9 +41,10 @@
 import org.elasticsearch.index.fielddata.IndexGeoPointFieldData;
 import org.elasticsearch.index.mapper.BaseGeoPointFieldMapper;
 import org.elasticsearch.index.mapper.GeoPointFieldMapper;
+import org.elasticsearch.index.mapper.LatLonPointFieldMapper;
 import org.elasticsearch.index.mapper.LegacyGeoPointFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.search.geo.GeoDistanceRangeQuery;
+import org.elasticsearch.index.search.geo.LegacyGeoDistanceRangeQuery;
 
 import java.io.IOException;
 import java.util.Locale;
@@ -61,7 +65,7 @@ public class GeoDistanceQueryBuilder extends AbstractQueryBuilder<GeoDistanceQue
     /** Default for distance unit computation. */
     public static final DistanceUnit DEFAULT_DISTANCE_UNIT = DistanceUnit.DEFAULT;
     /** Default for geo distance computation. */
-    public static final GeoDistance DEFAULT_GEO_DISTANCE = GeoDistance.DEFAULT;
+    public static final GeoDistance DEFAULT_GEO_DISTANCE = GeoDistance.ARC;
     /** Default for optimising query through pre computed bounding box query. */
     @Deprecated
     public static final String DEFAULT_OPTIMIZE_BBOX = "memory";
@@ -72,10 +76,8 @@ public class GeoDistanceQueryBuilder extends AbstractQueryBuilder<GeoDistanceQue
     public static final boolean DEFAULT_IGNORE_UNMAPPED = false;
 
     private static final ParseField VALIDATION_METHOD_FIELD = new ParseField("validation_method");
-    private static final ParseField IGNORE_MALFORMED_FIELD = new ParseField("ignore_malformed")
-            .withAllDeprecated("use validation_method instead");
-    private static final ParseField COERCE_FIELD = new ParseField("coerce", "normalize")
-            .withAllDeprecated("use validation_method instead");
+    private static final ParseField IGNORE_MALFORMED_FIELD = new ParseField("ignore_malformed").withAllDeprecated("validation_method");
+    private static final ParseField COERCE_FIELD = new ParseField("coerce", "normalize").withAllDeprecated("validation_method");
     @Deprecated
     private static final ParseField OPTIMIZE_BBOX_FIELD = new ParseField("optimize_bbox")
             .withAllDeprecated("no replacement: `optimize_bbox` is no longer supported due to recent improvements");
@@ -90,7 +92,7 @@ public class GeoDistanceQueryBuilder extends AbstractQueryBuilder<GeoDistanceQue
     /** Point to use as center. */
     private GeoPoint center = new GeoPoint(Double.NaN, Double.NaN);
     /** Algorithm to use for distance computation. */
-    private GeoDistance geoDistance = DEFAULT_GEO_DISTANCE;
+    private GeoDistance geoDistance = GeoDistance.ARC;
     /** Whether or not to use a bbox for pre-filtering. TODO change to enum? */
     private String optimizeBbox = null;
     /** How strict should geo coordinate validation be? */
@@ -295,14 +297,19 @@ protected Query doToQuery(QueryShardContext shardContext) throws IOException {
             GeoUtils.normalizePoint(center, true, true);
         }
 
-        double normDistance = geoDistance.normalize(this.distance, DistanceUnit.DEFAULT);
-
-        if (indexVersionCreated.before(Version.V_2_2_0)) {
-            LegacyGeoPointFieldMapper.GeoPointFieldType geoFieldType = ((LegacyGeoPointFieldMapper.GeoPointFieldType) fieldType);
+        if (indexVersionCreated.onOrAfter(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            Query query = LatLonPoint.newDistanceQuery(fieldType.name(), center.lat(), center.lon(), this.distance);
+            if (fieldType.hasDocValues()) {
+                Query dvQuery = LatLonDocValuesField.newDistanceQuery(fieldType.name(), center.lat(), center.lon(), this.distance);
+                query = new IndexOrDocValuesQuery(query, dvQuery);
+            }
+            return query;
+        } else if (indexVersionCreated.before(Version.V_2_2_0)) {
+            LegacyGeoPointFieldMapper.LegacyGeoPointFieldType geoFieldType = (LegacyGeoPointFieldMapper.LegacyGeoPointFieldType) fieldType;
             IndexGeoPointFieldData indexFieldData = shardContext.getForField(fieldType);
             String bboxOptimization = Strings.isEmpty(optimizeBbox) ? DEFAULT_OPTIMIZE_BBOX : optimizeBbox;
-            return new GeoDistanceRangeQuery(center, null, normDistance, true, false, geoDistance,
-                    geoFieldType, indexFieldData, bboxOptimization);
+            return new LegacyGeoDistanceRangeQuery(center, null, this.distance, true, false, geoDistance,
+                    geoFieldType, indexFieldData, bboxOptimization, shardContext);
         }
 
         // if index created V_2_2 use (soon to be legacy) numeric encoding postings format
@@ -310,6 +317,7 @@ protected Query doToQuery(QueryShardContext shardContext) throws IOException {
         final GeoPointField.TermEncoding encoding = (indexVersionCreated.before(Version.V_2_3_0)) ?
             GeoPointField.TermEncoding.NUMERIC : GeoPointField.TermEncoding.PREFIX;
         // Lucene 6.0 and earlier requires a radial restriction
+        double normDistance = this.distance;
         if (indexVersionCreated.before(Version.V_5_0_0_alpha4)) {
             normDistance = GeoUtils.maxRadialDistance(center, normDistance);
         }
@@ -380,15 +388,15 @@ public static Optional<GeoDistanceQueryBuilder> fromXContent(QueryParseContext p
                     }
                 }
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, DISTANCE_FIELD)) {
+                if (DISTANCE_FIELD.match(currentFieldName)) {
                     if (token == XContentParser.Token.VALUE_STRING) {
                         vDistance = parser.text(); // a String
                     } else {
                         vDistance = parser.numberValue(); // a Number
                     }
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, UNIT_FIELD)) {
+                } else if (UNIT_FIELD.match(currentFieldName)) {
                     unit = DistanceUnit.fromString(parser.text());
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, DISTANCE_TYPE_FIELD)) {
+                } else if (DISTANCE_TYPE_FIELD.match(currentFieldName)) {
                     geoDistance = GeoDistance.fromString(parser.text());
                 } else if (currentFieldName.endsWith(GeoPointFieldMapper.Names.LAT_SUFFIX)) {
                     point.resetLat(parser.doubleValue());
@@ -396,22 +404,22 @@ public static Optional<GeoDistanceQueryBuilder> fromXContent(QueryParseContext p
                 } else if (currentFieldName.endsWith(GeoPointFieldMapper.Names.LON_SUFFIX)) {
                     point.resetLon(parser.doubleValue());
                     fieldName = currentFieldName.substring(0, currentFieldName.length() - GeoPointFieldMapper.Names.LON_SUFFIX.length());
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, OPTIMIZE_BBOX_FIELD)) {
+                } else if (OPTIMIZE_BBOX_FIELD.match(currentFieldName)) {
                     optimizeBbox = parser.textOrNull();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, COERCE_FIELD)) {
+                } else if (COERCE_FIELD.match(currentFieldName)) {
                     coerce = parser.booleanValue();
-                    if (coerce == true) {
+                    if (coerce) {
                         ignoreMalformed = true;
                     }
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, IGNORE_MALFORMED_FIELD)) {
+                } else if (IGNORE_MALFORMED_FIELD.match(currentFieldName)) {
                     ignoreMalformed = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, IGNORE_UNMAPPED_FIELD)) {
+                } else if (IGNORE_UNMAPPED_FIELD.match(currentFieldName)) {
                     ignoreUnmapped = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, VALIDATION_METHOD_FIELD)) {
+                } else if (VALIDATION_METHOD_FIELD.match(currentFieldName)) {
                     validationMethod = GeoValidationMethod.fromString(parser.text());
                 } else {
                     if (fieldName == null) {
diff --git a/core/src/main/java/org/elasticsearch/index/query/GeoDistanceRangeQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/GeoDistanceRangeQueryBuilder.java
index 0b06e07e02e23..1f950a4bc0db6 100644
--- a/core/src/main/java/org/elasticsearch/index/query/GeoDistanceRangeQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/GeoDistanceRangeQueryBuilder.java
@@ -32,15 +32,18 @@
 import org.elasticsearch.common.geo.GeoUtils;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.unit.DistanceUnit;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.fielddata.IndexGeoPointFieldData;
 import org.elasticsearch.index.mapper.BaseGeoPointFieldMapper;
+import org.elasticsearch.index.mapper.BaseGeoPointFieldMapper.LegacyGeoPointFieldType;
 import org.elasticsearch.index.mapper.GeoPointFieldMapper;
-import org.elasticsearch.index.mapper.LegacyGeoPointFieldMapper;
+import org.elasticsearch.index.mapper.LatLonPointFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.search.geo.GeoDistanceRangeQuery;
+import org.elasticsearch.index.search.geo.LegacyGeoDistanceRangeQuery;
 
 import java.io.IOException;
 import java.util.Locale;
@@ -50,9 +53,11 @@
 public class GeoDistanceRangeQueryBuilder extends AbstractQueryBuilder<GeoDistanceRangeQueryBuilder> {
     public static final String NAME = "geo_distance_range";
 
+    private static final DeprecationLogger deprecationLogger = new DeprecationLogger(Loggers.getLogger(GeoDistanceRangeQueryBuilder.class));
+
     public static final boolean DEFAULT_INCLUDE_LOWER = true;
     public static final boolean DEFAULT_INCLUDE_UPPER = true;
-    public static final GeoDistance DEFAULT_GEO_DISTANCE = GeoDistance.DEFAULT;
+    public static final GeoDistance DEFAULT_GEO_DISTANCE = GeoDistance.ARC;
     public static final DistanceUnit DEFAULT_UNIT = DistanceUnit.DEFAULT;
     @Deprecated
     public static final String DEFAULT_OPTIMIZE_BBOX = "memory";
@@ -326,9 +331,6 @@ protected Query doToQuery(QueryShardContext context) throws IOException {
             } else {
                 fromValue = DistanceUnit.parse((String) from, unit, DistanceUnit.DEFAULT);
             }
-            if (indexCreatedBeforeV2_2 == true) {
-                fromValue = geoDistance.normalize(fromValue, DistanceUnit.DEFAULT);
-            }
         } else {
             fromValue = 0.0;
         }
@@ -339,20 +341,22 @@ protected Query doToQuery(QueryShardContext context) throws IOException {
             } else {
                 toValue = DistanceUnit.parse((String) to, unit, DistanceUnit.DEFAULT);
             }
-            if (indexCreatedBeforeV2_2 == true) {
-                toValue = geoDistance.normalize(toValue, DistanceUnit.DEFAULT);
-            }
         } else {
             toValue = GeoUtils.maxRadialDistanceMeters(point.lat(), point.lon());
         }
 
         final Version indexVersionCreated = context.indexVersionCreated();
+        if (indexVersionCreated.onOrAfter(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            throw new QueryShardException(context, "[{}] queries are no longer supported for geo_point field types. "
+                + "Use geo_distance sort or aggregations", NAME);
+        }
+
+        deprecationLogger.deprecated("geo_distance_range search is deprecated. Use geo_distance aggregation or sort instead.");
+
         if (indexVersionCreated.before(Version.V_2_2_0)) {
-            LegacyGeoPointFieldMapper.GeoPointFieldType geoFieldType = ((LegacyGeoPointFieldMapper.GeoPointFieldType) fieldType);
             IndexGeoPointFieldData indexFieldData = context.getForField(fieldType);
-            String bboxOptimization = Strings.isEmpty(optimizeBbox) ? DEFAULT_OPTIMIZE_BBOX : optimizeBbox;
-            return new GeoDistanceRangeQuery(point, fromValue, toValue, includeLower, includeUpper, geoDistance, geoFieldType,
-                indexFieldData, bboxOptimization);
+            return new LegacyGeoDistanceRangeQuery(point, fromValue, toValue, includeLower, includeUpper, geoDistance,
+                ((LegacyGeoPointFieldType) fieldType), indexFieldData, optimizeBbox, context);
         }
 
         // if index created V_2_2 use (soon to be legacy) numeric encoding postings format
@@ -435,27 +439,27 @@ public static Optional<GeoDistanceRangeQueryBuilder> fromXContent(QueryParseCont
                             "] field name already set to [" + fieldName + "] but found [" + currentFieldName + "]");
                 }
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, FROM_FIELD)) {
+                if (FROM_FIELD.match(currentFieldName)) {
                     if (token == XContentParser.Token.VALUE_NULL) {
                     } else if (token == XContentParser.Token.VALUE_STRING) {
                         vFrom = parser.text(); // a String
                     } else {
                         vFrom = parser.numberValue(); // a Number
                     }
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, TO_FIELD)) {
+                } else if (TO_FIELD.match(currentFieldName)) {
                     if (token == XContentParser.Token.VALUE_NULL) {
                     } else if (token == XContentParser.Token.VALUE_STRING) {
                         vTo = parser.text(); // a String
                     } else {
                         vTo = parser.numberValue(); // a Number
                     }
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, INCLUDE_LOWER_FIELD)) {
+                } else if (INCLUDE_LOWER_FIELD.match(currentFieldName)) {
                     includeLower = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, INCLUDE_UPPER_FIELD)) {
+                } else if (INCLUDE_UPPER_FIELD.match(currentFieldName)) {
                     includeUpper = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, IGNORE_UNMAPPED_FIELD)) {
+                } else if (IGNORE_UNMAPPED_FIELD.match(currentFieldName)) {
                     ignoreUnmapped = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, GT_FIELD)) {
+                } else if (GT_FIELD.match(currentFieldName)) {
                     if (token == XContentParser.Token.VALUE_NULL) {
                     } else if (token == XContentParser.Token.VALUE_STRING) {
                         vFrom = parser.text(); // a String
@@ -463,7 +467,7 @@ public static Optional<GeoDistanceRangeQueryBuilder> fromXContent(QueryParseCont
                         vFrom = parser.numberValue(); // a Number
                     }
                     includeLower = false;
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, GTE_FIELD)) {
+                } else if (GTE_FIELD.match(currentFieldName)) {
                     if (token == XContentParser.Token.VALUE_NULL) {
                     } else if (token == XContentParser.Token.VALUE_STRING) {
                         vFrom = parser.text(); // a String
@@ -471,7 +475,7 @@ public static Optional<GeoDistanceRangeQueryBuilder> fromXContent(QueryParseCont
                         vFrom = parser.numberValue(); // a Number
                     }
                     includeLower = true;
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, LT_FIELD)) {
+                } else if (LT_FIELD.match(currentFieldName)) {
                     if (token == XContentParser.Token.VALUE_NULL) {
                     } else if (token == XContentParser.Token.VALUE_STRING) {
                         vTo = parser.text(); // a String
@@ -479,7 +483,7 @@ public static Optional<GeoDistanceRangeQueryBuilder> fromXContent(QueryParseCont
                         vTo = parser.numberValue(); // a Number
                     }
                     includeUpper = false;
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, LTE_FIELD)) {
+                } else if (LTE_FIELD.match(currentFieldName)) {
                     if (token == XContentParser.Token.VALUE_NULL) {
                     } else if (token == XContentParser.Token.VALUE_STRING) {
                         vTo = parser.text(); // a String
@@ -487,9 +491,9 @@ public static Optional<GeoDistanceRangeQueryBuilder> fromXContent(QueryParseCont
                         vTo = parser.numberValue(); // a Number
                     }
                     includeUpper = true;
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, UNIT_FIELD)) {
+                } else if (UNIT_FIELD.match(currentFieldName)) {
                     unit = DistanceUnit.fromString(parser.text());
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, DISTANCE_TYPE_FIELD)) {
+                } else if (DISTANCE_TYPE_FIELD.match(currentFieldName)) {
                     geoDistance = GeoDistance.fromString(parser.text());
                 } else if (currentFieldName.endsWith(GeoPointFieldMapper.Names.LAT_SUFFIX)) {
                     String maybeFieldName = currentFieldName.substring(0,
@@ -517,17 +521,17 @@ public static Optional<GeoDistanceRangeQueryBuilder> fromXContent(QueryParseCont
                         point = new GeoPoint();
                     }
                     point.resetLon(parser.doubleValue());
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, NAME_FIELD)) {
+                } else if (NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, BOOST_FIELD)) {
+                } else if (BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, OPTIMIZE_BBOX_FIELD)) {
+                } else if (OPTIMIZE_BBOX_FIELD.match(currentFieldName)) {
                     optimizeBbox = parser.textOrNull();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, COERCE_FIELD)) {
+                } else if (COERCE_FIELD.match(currentFieldName)) {
                     coerce = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, IGNORE_MALFORMED_FIELD)) {
+                } else if (IGNORE_MALFORMED_FIELD.match(currentFieldName)) {
                     ignoreMalformed = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, VALIDATION_METHOD)) {
+                } else if (VALIDATION_METHOD.match(currentFieldName)) {
                     validationMethod = GeoValidationMethod.fromString(parser.text());
                 } else {
                     if (fieldName == null) {
diff --git a/core/src/main/java/org/elasticsearch/index/query/GeoPolygonQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/GeoPolygonQueryBuilder.java
index 65ce33c1c988d..68b90340c563e 100644
--- a/core/src/main/java/org/elasticsearch/index/query/GeoPolygonQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/GeoPolygonQueryBuilder.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.index.query;
 
+import org.apache.lucene.document.LatLonPoint;
+import org.apache.lucene.geo.Polygon;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.spatial.geopoint.document.GeoPointField;
@@ -36,6 +38,7 @@
 import org.elasticsearch.common.xcontent.XContentParser.Token;
 import org.elasticsearch.index.fielddata.IndexGeoPointFieldData;
 import org.elasticsearch.index.mapper.BaseGeoPointFieldMapper;
+import org.elasticsearch.index.mapper.LatLonPointFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.search.geo.GeoPolygonQuery;
 
@@ -53,10 +56,8 @@ public class GeoPolygonQueryBuilder extends AbstractQueryBuilder<GeoPolygonQuery
      */
     public static final boolean DEFAULT_IGNORE_UNMAPPED = false;
 
-    private static final ParseField COERCE_FIELD = new ParseField("coerce", "normalize")
-            .withAllDeprecated("use validation_method instead");
-    private static final ParseField IGNORE_MALFORMED_FIELD = new ParseField("ignore_malformed")
-            .withAllDeprecated("use validation_method instead");
+    private static final ParseField COERCE_FIELD = new ParseField("coerce", "normalize").withAllDeprecated("validation_method");
+    private static final ParseField IGNORE_MALFORMED_FIELD = new ParseField("ignore_malformed").withAllDeprecated("validation_method");
     private static final ParseField VALIDATION_METHOD = new ParseField("validation_method");
     private static final ParseField POINTS_FIELD = new ParseField("points");
     private static final ParseField IGNORE_UNMAPPED_FIELD = new ParseField("ignore_unmapped");
@@ -172,7 +173,7 @@ protected Query doToQuery(QueryShardContext context) throws IOException {
             throw new QueryShardException(context, "field [" + fieldName + "] is not a geo_point field");
         }
 
-        List<GeoPoint> shell = new ArrayList<GeoPoint>();
+        List<GeoPoint> shell = new ArrayList<>(this.shell.size());
         for (GeoPoint geoPoint : this.shell) {
             shell.add(new GeoPoint(geoPoint));
         }
@@ -210,10 +211,14 @@ protected Query doToQuery(QueryShardContext context) throws IOException {
         double[] lons = new double[shellSize];
         GeoPoint p;
         for (int i=0; i<shellSize; ++i) {
-            p = new GeoPoint(shell.get(i));
+            p = shell.get(i);
             lats[i] = p.lat();
             lons[i] = p.lon();
         }
+
+        if (indexVersionCreated.onOrAfter(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            return LatLonPoint.newPolygonQuery(fieldType.name(), new Polygon(lats, lons));
+        }
         // if index created V_2_2 use (soon to be legacy) numeric encoding postings format
         // if index created V_2_3 > use prefix encoded postings format
         final GeoPointField.TermEncoding encoding = (indexVersionCreated.before(Version.V_2_3_0)) ?
@@ -268,7 +273,7 @@ public static Optional<GeoPolygonQueryBuilder> fromXContent(QueryParseContext pa
                     if (token == XContentParser.Token.FIELD_NAME) {
                         currentFieldName = parser.currentName();
                     } else if (token == XContentParser.Token.START_ARRAY) {
-                        if (parseContext.getParseFieldMatcher().match(currentFieldName, POINTS_FIELD)) {
+                        if (POINTS_FIELD.match(currentFieldName)) {
                             shell = new ArrayList<GeoPoint>();
                             while ((token = parser.nextToken()) != Token.END_ARRAY) {
                                 shell.add(GeoUtils.parseGeoPoint(parser));
@@ -287,16 +292,16 @@ public static Optional<GeoPolygonQueryBuilder> fromXContent(QueryParseContext pa
                     queryName = parser.text();
                 } else if ("boost".equals(currentFieldName)) {
                     boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, COERCE_FIELD)) {
+                } else if (COERCE_FIELD.match(currentFieldName)) {
                     coerce = parser.booleanValue();
                     if (coerce) {
                         ignoreMalformed = true;
                     }
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, IGNORE_UNMAPPED_FIELD)) {
+                } else if (IGNORE_UNMAPPED_FIELD.match(currentFieldName)) {
                     ignoreUnmapped = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, IGNORE_MALFORMED_FIELD)) {
+                } else if (IGNORE_MALFORMED_FIELD.match(currentFieldName)) {
                     ignoreMalformed = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, VALIDATION_METHOD)) {
+                } else if (VALIDATION_METHOD.match(currentFieldName)) {
                     validationMethod = GeoValidationMethod.fromString(parser.text());
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
diff --git a/core/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java
index 76e9bc0f9be4a..7833ba4597fe3 100644
--- a/core/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java
@@ -39,6 +39,7 @@
 import org.elasticsearch.common.geo.builders.ShapeBuilder;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -384,7 +385,8 @@ private ShapeBuilder fetch(Client client, GetRequest getRequest, String path) th
         String[] pathElements = path.split("\\.");
         int currentPathSlot = 0;
 
-        try (XContentParser parser = XContentHelper.createParser(response.getSourceAsBytesRef())) {
+        // It is safe to use EMPTY here because this never uses namedObject
+        try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, response.getSourceAsBytesRef())) {
             XContentParser.Token currentToken;
             while ((currentToken = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
                 if (currentToken == XContentParser.Token.FIELD_NAME) {
@@ -488,31 +490,31 @@ public static Optional<GeoShapeQueryBuilder> fromXContent(QueryParseContext pars
                     if (token == XContentParser.Token.FIELD_NAME) {
                         currentFieldName = parser.currentName();
                         token = parser.nextToken();
-                        if (parseContext.getParseFieldMatcher().match(currentFieldName, SHAPE_FIELD)) {
+                        if (SHAPE_FIELD.match(currentFieldName)) {
                             shape = ShapeBuilder.parse(parser);
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, STRATEGY_FIELD)) {
+                        } else if (STRATEGY_FIELD.match(currentFieldName)) {
                             String strategyName = parser.text();
                             strategy = SpatialStrategy.fromString(strategyName);
                             if (strategy == null) {
                                 throw new ParsingException(parser.getTokenLocation(), "Unknown strategy [" + strategyName + " ]");
                             }
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, RELATION_FIELD)) {
+                        } else if (RELATION_FIELD.match(currentFieldName)) {
                             shapeRelation = ShapeRelation.getRelationByName(parser.text());
                             if (shapeRelation == null) {
                                 throw new ParsingException(parser.getTokenLocation(), "Unknown shape operation [" + parser.text() + " ]");
                             }
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, INDEXED_SHAPE_FIELD)) {
+                        } else if (INDEXED_SHAPE_FIELD.match(currentFieldName)) {
                             while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
                                 if (token == XContentParser.Token.FIELD_NAME) {
                                     currentFieldName = parser.currentName();
                                 } else if (token.isValue()) {
-                                    if (parseContext.getParseFieldMatcher().match(currentFieldName, SHAPE_ID_FIELD)) {
+                                    if (SHAPE_ID_FIELD.match(currentFieldName)) {
                                         id = parser.text();
-                                    } else if (parseContext.getParseFieldMatcher().match(currentFieldName, SHAPE_TYPE_FIELD)) {
+                                    } else if (SHAPE_TYPE_FIELD.match(currentFieldName)) {
                                         type = parser.text();
-                                    } else if (parseContext.getParseFieldMatcher().match(currentFieldName, SHAPE_INDEX_FIELD)) {
+                                    } else if (SHAPE_INDEX_FIELD.match(currentFieldName)) {
                                         index = parser.text();
-                                    } else if (parseContext.getParseFieldMatcher().match(currentFieldName, SHAPE_PATH_FIELD)) {
+                                    } else if (SHAPE_PATH_FIELD.match(currentFieldName)) {
                                         shapePath = parser.text();
                                     }
                                 } else {
@@ -527,11 +529,11 @@ public static Optional<GeoShapeQueryBuilder> fromXContent(QueryParseContext pars
                     }
                 }
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, IGNORE_UNMAPPED_FIELD)) {
+                } else if (IGNORE_UNMAPPED_FIELD.match(currentFieldName)) {
                     ignoreUnmapped = parser.booleanValue();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[" + GeoShapeQueryBuilder.NAME +
diff --git a/core/src/main/java/org/elasticsearch/index/query/GeohashCellQuery.java b/core/src/main/java/org/elasticsearch/index/query/GeohashCellQuery.java
index 57a189b72f446..d306a97736ff5 100644
--- a/core/src/main/java/org/elasticsearch/index/query/GeohashCellQuery.java
+++ b/core/src/main/java/org/elasticsearch/index/query/GeohashCellQuery.java
@@ -36,6 +36,7 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentParser.Token;
 import org.elasticsearch.index.mapper.BaseGeoPointFieldMapper;
+import org.elasticsearch.index.mapper.LatLonPointFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 
 import java.io.IOException;
@@ -83,7 +84,7 @@ public class GeohashCellQuery {
      * @param geohashes   optional array of additional geohashes
      * @return a new GeoBoundinboxfilter
      */
-    public static Query create(QueryShardContext context, BaseGeoPointFieldMapper.GeoPointFieldType fieldType,
+    public static Query create(QueryShardContext context, BaseGeoPointFieldMapper.LegacyGeoPointFieldType fieldType,
                                String geohash, @Nullable List<CharSequence> geohashes) {
         MappedFieldType geoHashMapper = fieldType.geoHashFieldType();
         if (geoHashMapper == null) {
@@ -241,11 +242,14 @@ protected Query doToQuery(QueryShardContext context) throws IOException {
                 }
             }
 
-            if (!(fieldType instanceof BaseGeoPointFieldMapper.GeoPointFieldType)) {
+            if (fieldType instanceof LatLonPointFieldMapper.LatLonPointFieldType) {
+                throw new QueryShardException(context, "failed to parse [{}] query. "
+                    + "geo_point field no longer supports geohash_cell queries", NAME);
+            } else if (!(fieldType instanceof BaseGeoPointFieldMapper.LegacyGeoPointFieldType)) {
                 throw new QueryShardException(context, "failed to parse [{}] query. field [{}] is not a geo_point field", NAME, fieldName);
             }
 
-            BaseGeoPointFieldMapper.GeoPointFieldType geoFieldType = ((BaseGeoPointFieldMapper.GeoPointFieldType) fieldType);
+            BaseGeoPointFieldMapper.LegacyGeoPointFieldType geoFieldType = ((BaseGeoPointFieldMapper.LegacyGeoPointFieldType) fieldType);
             if (!geoFieldType.isGeoHashPrefixEnabled()) {
                 throw new QueryShardException(context, "failed to parse [{}] query. [geohash_prefix] is not enabled for field [{}]", NAME,
                         fieldName);
@@ -301,7 +305,7 @@ public static Optional<Builder> fromXContent(QueryParseContext parseContext) thr
 
                     if (parseContext.isDeprecatedSetting(field)) {
                         // skip
-                    } else if (parseContext.getParseFieldMatcher().match(field, PRECISION_FIELD)) {
+                    } else if (PRECISION_FIELD.match(field)) {
                         token = parser.nextToken();
                         if (token == Token.VALUE_NUMBER) {
                             levels = parser.intValue();
@@ -309,16 +313,16 @@ public static Optional<Builder> fromXContent(QueryParseContext parseContext) thr
                             double meters = DistanceUnit.parse(parser.text(), DistanceUnit.DEFAULT, DistanceUnit.METERS);
                             levels = GeoUtils.geoHashLevelsForPrecision(meters);
                         }
-                    } else if (parseContext.getParseFieldMatcher().match(field, NEIGHBORS_FIELD)) {
+                    } else if (NEIGHBORS_FIELD.match(field)) {
                         parser.nextToken();
                         neighbors = parser.booleanValue();
-                    } else if (parseContext.getParseFieldMatcher().match(field, AbstractQueryBuilder.NAME_FIELD)) {
+                    } else if (AbstractQueryBuilder.NAME_FIELD.match(field)) {
                         parser.nextToken();
                         queryName = parser.text();
-                    } else if (parseContext.getParseFieldMatcher().match(field, IGNORE_UNMAPPED_FIELD)) {
+                    } else if (IGNORE_UNMAPPED_FIELD.match(field)) {
                         parser.nextToken();
                         ignoreUnmapped = parser.booleanValue();
-                    } else if (parseContext.getParseFieldMatcher().match(field, AbstractQueryBuilder.BOOST_FIELD)) {
+                    } else if (AbstractQueryBuilder.BOOST_FIELD.match(field)) {
                         parser.nextToken();
                         boost = parser.floatValue();
                     } else {
diff --git a/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java
deleted file mode 100644
index 81fe106833d11..0000000000000
--- a/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java
+++ /dev/null
@@ -1,494 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.index.query;
-
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.MultiDocValues;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.MatchNoDocsQuery;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.join.JoinUtil;
-import org.apache.lucene.search.join.ScoreMode;
-import org.apache.lucene.search.similarities.Similarity;
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.lucene.search.Queries;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.fielddata.IndexParentChildFieldData;
-import org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.ParentFieldMapper;
-
-import java.io.IOException;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Optional;
-
-/**
- * A query builder for <tt>has_child</tt> query.
- */
-public class HasChildQueryBuilder extends AbstractQueryBuilder<HasChildQueryBuilder> {
-    public static final String NAME = "has_child";
-
-    /**
-     * The default maximum number of children that are required to match for the parent to be considered a match.
-     */
-    public static final int DEFAULT_MAX_CHILDREN = Integer.MAX_VALUE;
-    /**
-     * The default minimum number of children that are required to match for the parent to be considered a match.
-     */
-    public static final int DEFAULT_MIN_CHILDREN = 0;
-
-    /**
-     * The default value for ignore_unmapped.
-     */
-    public static final boolean DEFAULT_IGNORE_UNMAPPED = false;
-
-    private static final ParseField QUERY_FIELD = new ParseField("query", "filter");
-    private static final ParseField TYPE_FIELD = new ParseField("type", "child_type");
-    private static final ParseField MAX_CHILDREN_FIELD = new ParseField("max_children");
-    private static final ParseField MIN_CHILDREN_FIELD = new ParseField("min_children");
-    private static final ParseField SCORE_MODE_FIELD = new ParseField("score_mode");
-    private static final ParseField INNER_HITS_FIELD = new ParseField("inner_hits");
-    private static final ParseField IGNORE_UNMAPPED_FIELD = new ParseField("ignore_unmapped");
-
-    private final QueryBuilder query;
-    private final String type;
-    private final ScoreMode scoreMode;
-    private InnerHitBuilder innerHitBuilder;
-    private int minChildren = DEFAULT_MIN_CHILDREN;
-    private int maxChildren = DEFAULT_MAX_CHILDREN;
-    private boolean ignoreUnmapped = false;
-
-    public HasChildQueryBuilder(String type, QueryBuilder query, ScoreMode scoreMode) {
-        this(type, query, DEFAULT_MIN_CHILDREN, DEFAULT_MAX_CHILDREN, scoreMode, null);
-    }
-
-    private HasChildQueryBuilder(String type, QueryBuilder query, int minChildren, int maxChildren, ScoreMode scoreMode,
-                                InnerHitBuilder innerHitBuilder) {
-        this.type = requireValue(type, "[" + NAME + "] requires 'type' field");
-        this.query = requireValue(query, "[" + NAME + "] requires 'query' field");
-        this.scoreMode = requireValue(scoreMode, "[" + NAME + "] requires 'score_mode' field");
-        this.innerHitBuilder = innerHitBuilder;
-        this.minChildren = minChildren;
-        this.maxChildren = maxChildren;
-    }
-
-    /**
-     * Read from a stream.
-     */
-    public HasChildQueryBuilder(StreamInput in) throws IOException {
-        super(in);
-        type = in.readString();
-        minChildren = in.readInt();
-        maxChildren = in.readInt();
-        scoreMode = ScoreMode.values()[in.readVInt()];
-        query = in.readNamedWriteable(QueryBuilder.class);
-        innerHitBuilder = in.readOptionalWriteable(InnerHitBuilder::new);
-        ignoreUnmapped = in.readBoolean();
-    }
-
-    @Override
-    protected void doWriteTo(StreamOutput out) throws IOException {
-        out.writeString(type);
-        out.writeInt(minChildren);
-        out.writeInt(maxChildren);
-        out.writeVInt(scoreMode.ordinal());
-        out.writeNamedWriteable(query);
-        out.writeOptionalWriteable(innerHitBuilder);
-        out.writeBoolean(ignoreUnmapped);
-    }
-
-    /**
-     * Defines the minimum number of children that are required to match for the parent to be considered a match and
-     * the maximum number of children that are required to match for the parent to be considered a match.
-     */
-    public HasChildQueryBuilder minMaxChildren(int minChildren, int maxChildren) {
-        if (minChildren < 0) {
-            throw new IllegalArgumentException("[" + NAME + "] requires non-negative 'min_children' field");
-        }
-        if (maxChildren < 0) {
-            throw new IllegalArgumentException("[" + NAME + "] requires non-negative 'max_children' field");
-        }
-        if (maxChildren < minChildren) {
-            throw new IllegalArgumentException("[" + NAME + "] 'max_children' is less than 'min_children'");
-        }
-        this.minChildren = minChildren;
-        this.maxChildren = maxChildren;
-        return this;
-    }
-
-    /**
-     * Returns inner hit definition in the scope of this query and reusing the defined type and query.
-     */
-    public InnerHitBuilder innerHit() {
-        return innerHitBuilder;
-    }
-
-    public HasChildQueryBuilder innerHit(InnerHitBuilder innerHit) {
-        this.innerHitBuilder = new InnerHitBuilder(Objects.requireNonNull(innerHit), query, type);
-        return this;
-    }
-
-    /**
-     * Returns the children query to execute.
-     */
-    public QueryBuilder query() {
-        return query;
-    }
-
-    /**
-     * Returns the child type
-     */
-    public String childType() {
-        return type;
-    }
-
-    /**
-     * Returns how the scores from the matching child documents are mapped into the parent document.
-     */
-    public ScoreMode scoreMode() {
-        return scoreMode;
-    }
-
-    /**
-     * Returns the minimum number of children that are required to match for the parent to be considered a match.
-     * The default is {@value #DEFAULT_MAX_CHILDREN}
-     */
-    public int minChildren() {
-        return minChildren;
-    }
-
-    /**
-     * Returns the maximum number of children that are required to match for the parent to be considered a match.
-     * The default is {@value #DEFAULT_MIN_CHILDREN}
-     */
-    public int maxChildren() { return maxChildren; }
-
-    /**
-     * Sets whether the query builder should ignore unmapped types (and run a
-     * {@link MatchNoDocsQuery} in place of this query) or throw an exception if
-     * the type is unmapped.
-     */
-    public HasChildQueryBuilder ignoreUnmapped(boolean ignoreUnmapped) {
-        this.ignoreUnmapped = ignoreUnmapped;
-        return this;
-    }
-
-    /**
-     * Gets whether the query builder will ignore unmapped types (and run a
-     * {@link MatchNoDocsQuery} in place of this query) or throw an exception if
-     * the type is unmapped.
-     */
-    public boolean ignoreUnmapped() {
-        return ignoreUnmapped;
-    }
-
-    @Override
-    protected void doXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject(NAME);
-        builder.field(QUERY_FIELD.getPreferredName());
-        query.toXContent(builder, params);
-        builder.field(TYPE_FIELD.getPreferredName(), type);
-        builder.field(SCORE_MODE_FIELD.getPreferredName(), scoreModeAsString(scoreMode));
-        builder.field(MIN_CHILDREN_FIELD.getPreferredName(), minChildren);
-        builder.field(MAX_CHILDREN_FIELD.getPreferredName(), maxChildren);
-        builder.field(IGNORE_UNMAPPED_FIELD.getPreferredName(), ignoreUnmapped);
-        printBoostAndQueryName(builder);
-        if (innerHitBuilder != null) {
-            builder.field(INNER_HITS_FIELD.getPreferredName(), innerHitBuilder, params);
-        }
-        builder.endObject();
-    }
-
-    public static Optional<HasChildQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
-        XContentParser parser = parseContext.parser();
-        float boost = AbstractQueryBuilder.DEFAULT_BOOST;
-        String childType = null;
-        ScoreMode scoreMode = ScoreMode.None;
-        int minChildren = HasChildQueryBuilder.DEFAULT_MIN_CHILDREN;
-        int maxChildren = HasChildQueryBuilder.DEFAULT_MAX_CHILDREN;
-        boolean ignoreUnmapped = DEFAULT_IGNORE_UNMAPPED;
-        String queryName = null;
-        InnerHitBuilder innerHitBuilder = null;
-        String currentFieldName = null;
-        XContentParser.Token token;
-        Optional<QueryBuilder> iqb = Optional.empty();
-        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-            if (token == XContentParser.Token.FIELD_NAME) {
-                currentFieldName = parser.currentName();
-            } else if (parseContext.isDeprecatedSetting(currentFieldName)) {
-                // skip
-            } else if (token == XContentParser.Token.START_OBJECT) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, QUERY_FIELD)) {
-                    iqb = parseContext.parseInnerQueryBuilder();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, INNER_HITS_FIELD)) {
-                    innerHitBuilder = InnerHitBuilder.fromXContent(parseContext);
-                } else {
-                    throw new ParsingException(parser.getTokenLocation(), "[has_child] query does not support [" + currentFieldName + "]");
-                }
-            } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, TYPE_FIELD)) {
-                    childType = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, SCORE_MODE_FIELD)) {
-                    scoreMode = parseScoreMode(parser.text());
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
-                    boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, MIN_CHILDREN_FIELD)) {
-                    minChildren = parser.intValue(true);
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, MAX_CHILDREN_FIELD)) {
-                    maxChildren = parser.intValue(true);
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, IGNORE_UNMAPPED_FIELD)) {
-                    ignoreUnmapped = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
-                    queryName = parser.text();
-                } else {
-                    throw new ParsingException(parser.getTokenLocation(), "[has_child] query does not support [" + currentFieldName + "]");
-                }
-            }
-        }
-
-        if (iqb.isPresent() == false) {
-            // if inner query is empty, bubble this up to caller so they can decide how to deal with it
-            return Optional.empty();
-        }
-
-        HasChildQueryBuilder hasChildQueryBuilder = new HasChildQueryBuilder(childType, iqb.get(), scoreMode);
-        if (innerHitBuilder != null) {
-            hasChildQueryBuilder.innerHit(innerHitBuilder);
-        }
-        hasChildQueryBuilder.minMaxChildren(minChildren, maxChildren);
-        hasChildQueryBuilder.queryName(queryName);
-        hasChildQueryBuilder.boost(boost);
-        hasChildQueryBuilder.ignoreUnmapped(ignoreUnmapped);
-        return Optional.of(hasChildQueryBuilder);
-    }
-
-    public static ScoreMode parseScoreMode(String scoreModeString) {
-        if ("none".equals(scoreModeString)) {
-            return ScoreMode.None;
-        } else if ("min".equals(scoreModeString)) {
-            return ScoreMode.Min;
-        } else if ("max".equals(scoreModeString)) {
-            return ScoreMode.Max;
-        } else if ("avg".equals(scoreModeString)) {
-            return ScoreMode.Avg;
-        } else if ("sum".equals(scoreModeString)) {
-            return ScoreMode.Total;
-        }
-        throw new IllegalArgumentException("No score mode for child query [" + scoreModeString + "] found");
-    }
-
-    public static String scoreModeAsString(ScoreMode scoreMode) {
-        if (scoreMode == ScoreMode.Total) {
-            // Lucene uses 'total' but 'sum' is more consistent with other elasticsearch APIs
-            return "sum";
-        } else {
-            return scoreMode.name().toLowerCase(Locale.ROOT);
-        }
-    }
-
-    @Override
-    public String getWriteableName() {
-        return NAME;
-    }
-
-    @Override
-    protected Query doToQuery(QueryShardContext context) throws IOException {
-        Query innerQuery;
-        final String[] previousTypes = context.getTypes();
-        context.setTypes(type);
-        try {
-            innerQuery = query.toQuery(context);
-        } finally {
-            context.setTypes(previousTypes);
-        }
-
-        DocumentMapper childDocMapper = context.getMapperService().documentMapper(type);
-        if (childDocMapper == null) {
-            if (ignoreUnmapped) {
-                return new MatchNoDocsQuery();
-            } else {
-                throw new QueryShardException(context, "[" + NAME + "] no mapping found for type [" + type + "]");
-            }
-        }
-        ParentFieldMapper parentFieldMapper = childDocMapper.parentFieldMapper();
-        if (parentFieldMapper.active() == false) {
-            throw new QueryShardException(context, "[" + NAME + "] _parent field has no parent type configured");
-        }
-        String parentType = parentFieldMapper.type();
-        DocumentMapper parentDocMapper = context.getMapperService().documentMapper(parentType);
-        if (parentDocMapper == null) {
-            throw new QueryShardException(context,
-                    "[" + NAME + "] Type [" + type + "] points to a non existent parent type [" + parentType + "]");
-        }
-
-        // wrap the query with type query
-        innerQuery = Queries.filtered(innerQuery, childDocMapper.typeFilter());
-
-        final ParentChildIndexFieldData parentChildIndexFieldData = context.getForField(parentFieldMapper.fieldType());
-        return new LateParsingQuery(parentDocMapper.typeFilter(), innerQuery, minChildren(), maxChildren(),
-                                    parentType, scoreMode, parentChildIndexFieldData, context.getSearchSimilarity());
-    }
-
-    /**
-     * A query that rewrites into another query using
-     * {@link JoinUtil#createJoinQuery(String, Query, Query, IndexSearcher, ScoreMode, MultiDocValues.OrdinalMap, int, int)}
-     * that executes the actual join.
-     *
-     * This query is exclusively used by the {@link HasChildQueryBuilder} and {@link HasParentQueryBuilder} to get access
-     * to the {@link DirectoryReader} used by the current search in order to retrieve the {@link MultiDocValues.OrdinalMap}.
-     * The {@link MultiDocValues.OrdinalMap} is required by {@link JoinUtil} to execute the join.
-     */
-    // TODO: Find a way to remove this query and let doToQuery(...) just return the query from JoinUtil.createJoinQuery(...)
-    public static final class LateParsingQuery extends Query {
-
-        private final Query toQuery;
-        private final Query innerQuery;
-        private final int minChildren;
-        private final int maxChildren;
-        private final String parentType;
-        private final ScoreMode scoreMode;
-        private final ParentChildIndexFieldData parentChildIndexFieldData;
-        private final Similarity similarity;
-
-        LateParsingQuery(Query toQuery, Query innerQuery, int minChildren, int maxChildren,
-                         String parentType, ScoreMode scoreMode, ParentChildIndexFieldData parentChildIndexFieldData,
-                         Similarity similarity) {
-            this.toQuery = toQuery;
-            this.innerQuery = innerQuery;
-            this.minChildren = minChildren;
-            this.maxChildren = maxChildren;
-            this.parentType = parentType;
-            this.scoreMode = scoreMode;
-            this.parentChildIndexFieldData = parentChildIndexFieldData;
-            this.similarity = similarity;
-        }
-
-        @Override
-        public Query rewrite(IndexReader reader) throws IOException {
-            Query rewritten = super.rewrite(reader);
-            if (rewritten != this) {
-                return rewritten;
-            }
-            if (reader instanceof DirectoryReader) {
-                String joinField = ParentFieldMapper.joinField(parentType);
-                IndexSearcher indexSearcher = new IndexSearcher(reader);
-                indexSearcher.setQueryCache(null);
-                indexSearcher.setSimilarity(similarity);
-                IndexParentChildFieldData indexParentChildFieldData = parentChildIndexFieldData.loadGlobal((DirectoryReader) reader);
-                MultiDocValues.OrdinalMap ordinalMap = ParentChildIndexFieldData.getOrdinalMap(indexParentChildFieldData, parentType);
-                return JoinUtil.createJoinQuery(joinField, innerQuery, toQuery, indexSearcher, scoreMode,
-                        ordinalMap, minChildren, maxChildren);
-            } else {
-                if (reader.leaves().isEmpty() && reader.numDocs() == 0) {
-                    // asserting reader passes down a MultiReader during rewrite which makes this
-                    // blow up since for this query to work we have to have a DirectoryReader otherwise
-                    // we can't load global ordinals - for this to work we simply check if the reader has no leaves
-                    // and rewrite to match nothing
-                    return new MatchNoDocsQuery();
-                }
-                throw new IllegalStateException("can't load global ordinals for reader of type: " +
-                        reader.getClass() + " must be a DirectoryReader");
-            }
-        }
-
-        @Override
-        public boolean equals(Object o) {
-            if (sameClassAs(o) == false) return false;
-
-            LateParsingQuery that = (LateParsingQuery) o;
-
-            if (minChildren != that.minChildren) return false;
-            if (maxChildren != that.maxChildren) return false;
-            if (!toQuery.equals(that.toQuery)) return false;
-            if (!innerQuery.equals(that.innerQuery)) return false;
-            if (!parentType.equals(that.parentType)) return false;
-            return scoreMode == that.scoreMode;
-        }
-
-        @Override
-        public int hashCode() {
-            return Objects.hash(classHash(), toQuery, innerQuery, minChildren, maxChildren, parentType, scoreMode);
-        }
-
-        @Override
-        public String toString(String s) {
-            return "LateParsingQuery {parentType=" + parentType + "}";
-        }
-
-        public int getMinChildren() {
-            return minChildren;
-        }
-
-        public int getMaxChildren() {
-            return maxChildren;
-        }
-
-        public ScoreMode getScoreMode() {
-            return scoreMode;
-        }
-
-        public Query getInnerQuery() {
-            return innerQuery;
-        }
-
-        public Similarity getSimilarity() {
-            return similarity;
-        }
-    }
-
-    @Override
-    protected boolean doEquals(HasChildQueryBuilder that) {
-        return Objects.equals(query, that.query)
-                && Objects.equals(type, that.type)
-                && Objects.equals(scoreMode, that.scoreMode)
-                && Objects.equals(minChildren, that.minChildren)
-                && Objects.equals(maxChildren, that.maxChildren)
-                && Objects.equals(innerHitBuilder, that.innerHitBuilder)
-                && Objects.equals(ignoreUnmapped, that.ignoreUnmapped);
-    }
-
-    @Override
-    protected int doHashCode() {
-        return Objects.hash(query, type, scoreMode, minChildren, maxChildren, innerHitBuilder, ignoreUnmapped);
-    }
-
-    @Override
-    protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException {
-        QueryBuilder rewrittenQuery = query.rewrite(queryRewriteContext);
-        if (rewrittenQuery != query) {
-            InnerHitBuilder rewrittenInnerHit = InnerHitBuilder.rewrite(innerHitBuilder, rewrittenQuery);
-            return new HasChildQueryBuilder(type, rewrittenQuery, minChildren, maxChildren, scoreMode, rewrittenInnerHit);
-        }
-        return this;
-    }
-
-    @Override
-    protected void extractInnerHitBuilders(Map<String, InnerHitBuilder> innerHits) {
-        if (innerHitBuilder != null) {
-            innerHitBuilder.inlineInnerHits(innerHits);
-        }
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/index/query/HasParentQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/HasParentQueryBuilder.java
deleted file mode 100644
index 5b89262ecc731..0000000000000
--- a/core/src/main/java/org/elasticsearch/index/query/HasParentQueryBuilder.java
+++ /dev/null
@@ -1,324 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.index.query;
-
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.MatchNoDocsQuery;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.join.ScoreMode;
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.lucene.search.Queries;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.ParentFieldMapper;
-
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.Set;
-
-/**
- * Builder for the 'has_parent' query.
- */
-public class HasParentQueryBuilder extends AbstractQueryBuilder<HasParentQueryBuilder> {
-    public static final String NAME = "has_parent";
-
-    /**
-     * The default value for ignore_unmapped.
-     */
-    public static final boolean DEFAULT_IGNORE_UNMAPPED = false;
-
-    private static final ParseField QUERY_FIELD = new ParseField("query", "filter");
-    private static final ParseField SCORE_MODE_FIELD = new ParseField("score_mode").withAllDeprecated("score");
-    private static final ParseField TYPE_FIELD = new ParseField("parent_type", "type");
-    private static final ParseField SCORE_FIELD = new ParseField("score");
-    private static final ParseField INNER_HITS_FIELD = new ParseField("inner_hits");
-    private static final ParseField IGNORE_UNMAPPED_FIELD = new ParseField("ignore_unmapped");
-
-    private final QueryBuilder query;
-    private final String type;
-    private final boolean score;
-    private InnerHitBuilder innerHit;
-    private boolean ignoreUnmapped = false;
-
-    public HasParentQueryBuilder(String type, QueryBuilder query, boolean score) {
-        this(type, query, score, null);
-    }
-
-    private HasParentQueryBuilder(String type, QueryBuilder query, boolean score, InnerHitBuilder innerHit) {
-        this.type = requireValue(type, "[" + NAME + "] requires 'type' field");
-        this.query = requireValue(query, "[" + NAME + "] requires 'query' field");
-        this.score = score;
-        this.innerHit = innerHit;
-    }
-
-    /**
-     * Read from a stream.
-     */
-    public HasParentQueryBuilder(StreamInput in) throws IOException {
-        super(in);
-        type = in.readString();
-        score = in.readBoolean();
-        query = in.readNamedWriteable(QueryBuilder.class);
-        innerHit = in.readOptionalWriteable(InnerHitBuilder::new);
-        ignoreUnmapped = in.readBoolean();
-    }
-
-    @Override
-    protected void doWriteTo(StreamOutput out) throws IOException {
-        out.writeString(type);
-        out.writeBoolean(score);
-        out.writeNamedWriteable(query);
-        out.writeOptionalWriteable(innerHit);
-        out.writeBoolean(ignoreUnmapped);
-    }
-
-    /**
-     * Returns the query to execute.
-     */
-    public QueryBuilder query() {
-        return query;
-    }
-
-    /**
-     * Returns <code>true</code> if the parent score is mapped into the child documents
-     */
-    public boolean score() {
-        return score;
-    }
-
-    /**
-     * Returns the parents type name
-     */
-    public String type() {
-        return type;
-    }
-
-    /**
-     *  Returns inner hit definition in the scope of this query and reusing the defined type and query.
-     */
-    public InnerHitBuilder innerHit() {
-        return innerHit;
-    }
-
-    public HasParentQueryBuilder innerHit(InnerHitBuilder innerHit) {
-        this.innerHit = new InnerHitBuilder(innerHit, query, type);
-        return this;
-    }
-
-    /**
-     * Sets whether the query builder should ignore unmapped types (and run a
-     * {@link MatchNoDocsQuery} in place of this query) or throw an exception if
-     * the type is unmapped.
-     */
-    public HasParentQueryBuilder ignoreUnmapped(boolean ignoreUnmapped) {
-        this.ignoreUnmapped = ignoreUnmapped;
-        return this;
-    }
-
-    /**
-     * Gets whether the query builder will ignore unmapped types (and run a
-     * {@link MatchNoDocsQuery} in place of this query) or throw an exception if
-     * the type is unmapped.
-     */
-    public boolean ignoreUnmapped() {
-        return ignoreUnmapped;
-    }
-
-    @Override
-    protected Query doToQuery(QueryShardContext context) throws IOException {
-        Query innerQuery;
-        String[] previousTypes = context.getTypes();
-        context.setTypes(type);
-        try {
-            innerQuery = query.toQuery(context);
-        } finally {
-            context.setTypes(previousTypes);
-        }
-
-        DocumentMapper parentDocMapper = context.getMapperService().documentMapper(type);
-        if (parentDocMapper == null) {
-            if (ignoreUnmapped) {
-                return new MatchNoDocsQuery();
-            } else {
-                throw new QueryShardException(context, "[" + NAME + "] query configured 'parent_type' [" + type + "] is not a valid type");
-            }
-        }
-
-        Set<String> childTypes = new HashSet<>();
-        ParentChildIndexFieldData parentChildIndexFieldData = null;
-        for (DocumentMapper documentMapper : context.getMapperService().docMappers(false)) {
-            ParentFieldMapper parentFieldMapper = documentMapper.parentFieldMapper();
-            if (parentFieldMapper.active() && type.equals(parentFieldMapper.type())) {
-                childTypes.add(documentMapper.type());
-                parentChildIndexFieldData = context.getForField(parentFieldMapper.fieldType());
-            }
-        }
-
-        if (childTypes.isEmpty()) {
-            throw new QueryShardException(context, "[" + NAME + "] no child types found for type [" + type + "]");
-        }
-
-        Query childrenQuery;
-        if (childTypes.size() == 1) {
-            DocumentMapper documentMapper = context.getMapperService().documentMapper(childTypes.iterator().next());
-            childrenQuery = documentMapper.typeFilter();
-        } else {
-            BooleanQuery.Builder childrenFilter = new BooleanQuery.Builder();
-            for (String childrenTypeStr : childTypes) {
-                DocumentMapper documentMapper = context.getMapperService().documentMapper(childrenTypeStr);
-                childrenFilter.add(documentMapper.typeFilter(), BooleanClause.Occur.SHOULD);
-            }
-            childrenQuery = childrenFilter.build();
-        }
-
-        // wrap the query with type query
-        innerQuery = Queries.filtered(innerQuery, parentDocMapper.typeFilter());
-        return new HasChildQueryBuilder.LateParsingQuery(childrenQuery,
-                                                         innerQuery,
-                                                         HasChildQueryBuilder.DEFAULT_MIN_CHILDREN,
-                                                         HasChildQueryBuilder.DEFAULT_MAX_CHILDREN,
-                                                         type,
-                                                         score ? ScoreMode.Max : ScoreMode.None,
-                                                         parentChildIndexFieldData,
-                                                         context.getSearchSimilarity());
-    }
-
-    @Override
-    protected void doXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject(NAME);
-        builder.field(QUERY_FIELD.getPreferredName());
-        query.toXContent(builder, params);
-        builder.field(TYPE_FIELD.getPreferredName(), type);
-        builder.field(SCORE_FIELD.getPreferredName(), score);
-        builder.field(IGNORE_UNMAPPED_FIELD.getPreferredName(), ignoreUnmapped);
-        printBoostAndQueryName(builder);
-        if (innerHit != null) {
-            builder.field(INNER_HITS_FIELD.getPreferredName(), innerHit, params);
-        }
-        builder.endObject();
-    }
-
-    public static Optional<HasParentQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
-        XContentParser parser = parseContext.parser();
-        float boost = AbstractQueryBuilder.DEFAULT_BOOST;
-        String parentType = null;
-        boolean score = false;
-        String queryName = null;
-        InnerHitBuilder innerHits = null;
-        boolean ignoreUnmapped = DEFAULT_IGNORE_UNMAPPED;
-
-        String currentFieldName = null;
-        XContentParser.Token token;
-        Optional<QueryBuilder> iqb = Optional.empty();
-        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-            if (token == XContentParser.Token.FIELD_NAME) {
-                currentFieldName = parser.currentName();
-            } else if (token == XContentParser.Token.START_OBJECT) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, QUERY_FIELD)) {
-                    iqb = parseContext.parseInnerQueryBuilder();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, INNER_HITS_FIELD)) {
-                    innerHits = InnerHitBuilder.fromXContent(parseContext);
-                } else {
-                    throw new ParsingException(parser.getTokenLocation(), "[has_parent] query does not support [" + currentFieldName + "]");
-                }
-            } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, TYPE_FIELD)) {
-                    parentType = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, SCORE_MODE_FIELD)) {
-                    String scoreModeValue = parser.text();
-                    if ("score".equals(scoreModeValue)) {
-                        score = true;
-                    } else if ("none".equals(scoreModeValue)) {
-                        score = false;
-                    } else {
-                        throw new ParsingException(parser.getTokenLocation(), "[has_parent] query does not support [" +
-                                scoreModeValue + "] as an option for score_mode");
-                    }
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, SCORE_FIELD)) {
-                    score = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, IGNORE_UNMAPPED_FIELD)) {
-                    ignoreUnmapped = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
-                    boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
-                    queryName = parser.text();
-                } else {
-                    throw new ParsingException(parser.getTokenLocation(), "[has_parent] query does not support [" + currentFieldName + "]");
-                }
-            }
-        }
-        if (iqb.isPresent() == false) {
-            // if inner query is empty, bubble this up to caller so they can decide how to deal with it
-            return Optional.empty();
-        }
-        HasParentQueryBuilder queryBuilder =  new HasParentQueryBuilder(parentType, iqb.get(), score)
-                .ignoreUnmapped(ignoreUnmapped)
-                .queryName(queryName)
-                .boost(boost);
-        if (innerHits != null) {
-            queryBuilder.innerHit(innerHits);
-        }
-        return Optional.of(queryBuilder);
-    }
-
-    @Override
-    public String getWriteableName() {
-        return NAME;
-    }
-
-    @Override
-    protected boolean doEquals(HasParentQueryBuilder that) {
-        return Objects.equals(query, that.query)
-                && Objects.equals(type, that.type)
-                && Objects.equals(score, that.score)
-                && Objects.equals(innerHit, that.innerHit)
-                && Objects.equals(ignoreUnmapped, that.ignoreUnmapped);
-    }
-
-    @Override
-    protected int doHashCode() {
-        return Objects.hash(query, type, score, innerHit, ignoreUnmapped);
-    }
-
-    @Override
-    protected QueryBuilder doRewrite(QueryRewriteContext queryShardContext) throws IOException {
-        QueryBuilder rewrittenQuery = query.rewrite(queryShardContext);
-        if (rewrittenQuery != query) {
-            InnerHitBuilder rewrittenInnerHit = InnerHitBuilder.rewrite(innerHit, rewrittenQuery);
-            return new HasParentQueryBuilder(type, rewrittenQuery, score, rewrittenInnerHit);
-        }
-        return this;
-    }
-
-    @Override
-    protected void extractInnerHitBuilders(Map<String, InnerHitBuilder> innerHits) {
-        if (innerHit!= null) {
-            innerHit.inlineInnerHits(innerHits);
-        }
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/index/query/IdsQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/IdsQueryBuilder.java
index c8f9f55f96e6f..81afd7c334edf 100644
--- a/core/src/main/java/org/elasticsearch/index/query/IdsQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/IdsQueryBuilder.java
@@ -19,30 +19,32 @@
 
 package org.elasticsearch.index.query;
 
-import org.apache.lucene.queries.TermsQuery;
+import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lucene.search.Queries;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.index.mapper.UidFieldMapper;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
-import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 
+import static org.elasticsearch.common.xcontent.ObjectParser.fromList;
+
 /**
  * A query that will return only documents matching specific ids (and a type).
  */
@@ -54,23 +56,22 @@ public class IdsQueryBuilder extends AbstractQueryBuilder<IdsQueryBuilder> {
 
     private final Set<String> ids = new HashSet<>();
 
-    private final String[] types;
+    private String[] types = Strings.EMPTY_ARRAY;
 
     /**
-     * Creates a new IdsQueryBuilder without providing the types of the documents to look for
+     * Creates a new IdsQueryBuilder with no types specified upfront
      */
     public IdsQueryBuilder() {
-        this.types = new String[0];
+        // nothing to do
     }
 
     /**
      * Creates a new IdsQueryBuilder by providing the types of the documents to look for
+     * @deprecated Replaced by {@link #types(String...)}
      */
+    @Deprecated
     public IdsQueryBuilder(String... types) {
-        if (types == null) {
-            throw new IllegalArgumentException("[ids] types cannot be null");
-        }
-        this.types = types;
+        types(types);
     }
 
     /**
@@ -88,6 +89,17 @@ protected void doWriteTo(StreamOutput out) throws IOException {
         out.writeStringArray(ids.toArray(new String[ids.size()]));
     }
 
+    /**
+     * Add types to query
+     */
+    public IdsQueryBuilder types(String... types) {
+        if (types == null) {
+            throw new IllegalArgumentException("[" + NAME + "] types cannot be null");
+        }
+        this.types = types;
+        return this;
+    }
+
     /**
      * Returns the types used in this query
      */
@@ -100,7 +112,7 @@ public String[] types() {
      */
     public IdsQueryBuilder addIds(String... ids) {
         if (ids == null) {
-            throw new IllegalArgumentException("[ids] ids cannot be null");
+            throw new IllegalArgumentException("[" + NAME + "] ids cannot be null");
         }
         Collections.addAll(this.ids, ids);
         return this;
@@ -126,71 +138,21 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep
         builder.endObject();
     }
 
-    public static Optional<IdsQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
-        XContentParser parser = parseContext.parser();
-        List<String> ids = new ArrayList<>();
-        List<String> types = new ArrayList<>();
-        float boost = AbstractQueryBuilder.DEFAULT_BOOST;
-        String queryName = null;
-
-        String currentFieldName = null;
-        XContentParser.Token token;
-        boolean idsProvided = false;
-        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-            if (token == XContentParser.Token.FIELD_NAME) {
-                currentFieldName = parser.currentName();
-            } else if (token == XContentParser.Token.START_ARRAY) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, VALUES_FIELD)) {
-                    idsProvided = true;
-                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
-                        if ((token == XContentParser.Token.VALUE_STRING) ||
-                                (token == XContentParser.Token.VALUE_NUMBER)) {
-                            String id = parser.textOrNull();
-                            if (id == null) {
-                                throw new ParsingException(parser.getTokenLocation(), "No value specified for term filter");
-                            }
-                            ids.add(id);
-                        } else {
-                            throw new ParsingException(parser.getTokenLocation(),
-                                    "Illegal value for id, expecting a string or number, got: " + token);
-                        }
-                    }
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, TYPE_FIELD)) {
-                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
-                        String value = parser.textOrNull();
-                        if (value == null) {
-                            throw new ParsingException(parser.getTokenLocation(), "No type specified for term filter");
-                        }
-                        types.add(value);
-                    }
-                } else {
-                    throw new ParsingException(parser.getTokenLocation(), "[" + IdsQueryBuilder.NAME +
-                            "] query does not support [" + currentFieldName + "]");
-                }
-            } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, TYPE_FIELD)) {
-                    types = Collections.singletonList(parser.text());
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
-                    boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
-                    queryName = parser.text();
-                } else {
-                    throw new ParsingException(parser.getTokenLocation(), "[" + IdsQueryBuilder.NAME +
-                            "] query does not support [" + currentFieldName + "]");
-                }
-            } else {
-                throw new ParsingException(parser.getTokenLocation(), "[" + IdsQueryBuilder.NAME +
-                        "] unknown token [" + token + "] after [" + currentFieldName + "]");
-            }
-        }
-        if (!idsProvided) {
-            throw new ParsingException(parser.getTokenLocation(), "[" + IdsQueryBuilder.NAME + "] query, no ids values provided");
-        }
+    private static ObjectParser<IdsQueryBuilder, QueryParseContext> PARSER = new ObjectParser<>(NAME,
+            () -> new IdsQueryBuilder());
 
-        IdsQueryBuilder query = new IdsQueryBuilder(types.toArray(new String[types.size()]));
-        query.addIds(ids.toArray(new String[ids.size()]));
-        query.boost(boost).queryName(queryName);
-        return Optional.of(query);
+    static {
+        PARSER.declareStringArray(fromList(String.class, IdsQueryBuilder::types), IdsQueryBuilder.TYPE_FIELD);
+        PARSER.declareStringArray(fromList(String.class, IdsQueryBuilder::addIds), IdsQueryBuilder.VALUES_FIELD);
+        declareStandardFields(PARSER);
+    }
+
+    public static Optional<IdsQueryBuilder> fromXContent(QueryParseContext context) {
+        try {
+            return Optional.of(PARSER.apply(context.parser(), context));
+        } catch (IllegalArgumentException e) {
+            throw new ParsingException(context.parser().getTokenLocation(), e.getMessage(), e);
+        }
     }
 
 
@@ -202,6 +164,10 @@ public String getWriteableName() {
     @Override
     protected Query doToQuery(QueryShardContext context) throws IOException {
         Query query;
+        MappedFieldType uidField = context.fieldMapper(UidFieldMapper.NAME);
+        if (uidField == null) {
+            return new MatchNoDocsQuery("No mappings");
+        }
         if (this.ids.isEmpty()) {
              query = Queries.newMatchNoDocsQuery("Missing ids in \"" + this.getName() + "\" query.");
         } else {
@@ -215,7 +181,7 @@ protected Query doToQuery(QueryShardContext context) throws IOException {
                 Collections.addAll(typesForQuery, types);
             }
 
-            query = new TermsQuery(UidFieldMapper.NAME, Uid.createUidsForTypesAndIds(typesForQuery, ids));
+            query = uidField.termsQuery(Arrays.asList(Uid.createUidsForTypesAndIds(typesForQuery, ids)), context);
         }
         return query;
     }
diff --git a/core/src/main/java/org/elasticsearch/index/query/IndicesQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/IndicesQueryBuilder.java
index 736387a0d24c7..f5ad441946985 100644
--- a/core/src/main/java/org/elasticsearch/index/query/IndicesQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/IndicesQueryBuilder.java
@@ -33,6 +33,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 
@@ -132,7 +133,7 @@ private static QueryBuilder defaultNoMatchQuery() {
     @Override
     protected void doXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject(NAME);
-        builder.field(INDICES_FIELD.getPreferredName(), indices);
+        builder.array(INDICES_FIELD.getPreferredName(), indices);
         builder.field(QUERY_FIELD.getPreferredName());
         innerQuery.toXContent(builder, params);
         builder.field(NO_MATCH_QUERY.getPreferredName());
@@ -157,16 +158,16 @@ public static Optional<IndicesQueryBuilder> fromXContent(QueryParseContext parse
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, QUERY_FIELD)) {
+                if (QUERY_FIELD.match(currentFieldName)) {
                     // the 2.0 behaviour when encountering "query" : {} is to return no docs for matching indices
                     innerQuery = parseContext.parseInnerQueryBuilder().orElse(new MatchNoneQueryBuilder());
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, NO_MATCH_QUERY)) {
+                } else if (NO_MATCH_QUERY.match(currentFieldName)) {
                     noMatchQuery = parseContext.parseInnerQueryBuilder().orElse(defaultNoMatchQuery());
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[indices] query does not support [" + currentFieldName + "]");
                 }
             } else if (token == XContentParser.Token.START_ARRAY) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, INDICES_FIELD)) {
+                if (INDICES_FIELD.match(currentFieldName)) {
                     if (indices.isEmpty() == false) {
                         throw new ParsingException(parser.getTokenLocation(), "[indices] indices or index already specified");
                     }
@@ -181,16 +182,16 @@ public static Optional<IndicesQueryBuilder> fromXContent(QueryParseContext parse
                     throw new ParsingException(parser.getTokenLocation(), "[indices] query does not support [" + currentFieldName + "]");
                 }
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, INDEX_FIELD)) {
+                if (INDEX_FIELD.match(currentFieldName)) {
                     if (indices.isEmpty() == false) {
                         throw new ParsingException(parser.getTokenLocation(), "[indices] indices or index already specified");
                     }
                     indices.add(parser.text());
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, NO_MATCH_QUERY)) {
+                } else if (NO_MATCH_QUERY.match(currentFieldName)) {
                     noMatchQuery = parseNoMatchQuery(parser.text());
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[indices] query does not support [" + currentFieldName + "]");
@@ -232,6 +233,12 @@ protected Query doToQuery(QueryShardContext context) throws IOException {
         return noMatchQuery.toQuery(context);
     }
 
+    @Override
+    protected void extractInnerHitBuilders(Map<String, InnerHitContextBuilder> innerHits) {
+        InnerHitContextBuilder.extractInnerHits(innerQuery, innerHits);
+        InnerHitContextBuilder.extractInnerHits(noMatchQuery, innerHits);
+    }
+
     @Override
     public int doHashCode() {
         return Objects.hash(innerQuery, noMatchQuery, Arrays.hashCode(indices));
@@ -246,10 +253,10 @@ protected boolean doEquals(IndicesQueryBuilder other) {
 
     @Override
     protected QueryBuilder doRewrite(QueryRewriteContext queryShardContext) throws IOException {
-        QueryBuilder newInnnerQuery = innerQuery.rewrite(queryShardContext);
+        QueryBuilder newInnerQuery = innerQuery.rewrite(queryShardContext);
         QueryBuilder newNoMatchQuery = noMatchQuery.rewrite(queryShardContext);
-        if (newInnnerQuery != innerQuery || newNoMatchQuery != noMatchQuery) {
-            return new IndicesQueryBuilder(innerQuery, indices).noMatchQuery(noMatchQuery);
+        if (newInnerQuery != innerQuery || newNoMatchQuery != noMatchQuery) {
+            return new IndicesQueryBuilder(newInnerQuery, indices).noMatchQuery(newNoMatchQuery);
         }
         return this;
     }
diff --git a/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java b/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
index 27b0138e1e5a0..7b331dd487b51 100644
--- a/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.index.query;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.support.ToXContentToBytes;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParsingException;
@@ -27,32 +28,21 @@
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.ObjectMapper;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptContext;
-import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder.ScriptField;
 import org.elasticsearch.search.fetch.StoredFieldsContext;
-import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext;
-import org.elasticsearch.search.fetch.subphase.DocValueFieldsFetchSubPhase;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
-import org.elasticsearch.search.fetch.subphase.InnerHitsContext;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
-import org.elasticsearch.search.internal.SearchContext;
-import org.elasticsearch.search.sort.SortAndFormats;
 import org.elasticsearch.search.sort.SortBuilder;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
+import java.util.Comparator;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
-import java.util.Map;
 import java.util.Objects;
-import java.util.Optional;
 import java.util.Set;
 
 import static org.elasticsearch.common.xcontent.XContentParser.Token.END_OBJECT;
@@ -60,13 +50,14 @@
 public final class InnerHitBuilder extends ToXContentToBytes implements Writeable {
 
     public static final ParseField NAME_FIELD = new ParseField("name");
-    public static final ParseField INNER_HITS_FIELD = new ParseField("inner_hits");
+    public static final ParseField IGNORE_UNMAPPED = new ParseField("ignore_unmapped");
     public static final QueryBuilder DEFAULT_INNER_HIT_QUERY = new MatchAllQueryBuilder();
 
     private static final ObjectParser<InnerHitBuilder, QueryParseContext> PARSER = new ObjectParser<>("inner_hits", InnerHitBuilder::new);
 
     static {
         PARSER.declareString(InnerHitBuilder::setName, NAME_FIELD);
+        PARSER.declareBoolean((innerHitBuilder, value) -> innerHitBuilder.ignoreUnmapped = value, IGNORE_UNMAPPED);
         PARSER.declareInt(InnerHitBuilder::setFrom, SearchSourceBuilder.FROM_FIELD);
         PARSER.declareInt(InnerHitBuilder::setSize, SearchSourceBuilder.SIZE_FIELD);
         PARSER.declareBoolean(InnerHitBuilder::setExplain, SearchSourceBuilder.EXPLAIN_FIELD);
@@ -95,42 +86,17 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
                 ObjectParser.ValueType.OBJECT_ARRAY);
         PARSER.declareField((p, i, c) -> {
             try {
-                i.setFetchSourceContext(FetchSourceContext.parse(c));
+                i.setFetchSourceContext(FetchSourceContext.fromXContent(c.parser()));
             } catch (IOException e) {
                 throw new ParsingException(p.getTokenLocation(), "Could not parse inner _source definition", e);
             }
-        }, SearchSourceBuilder._SOURCE_FIELD, ObjectParser.ValueType.OBJECT_OR_BOOLEAN);
+        }, SearchSourceBuilder._SOURCE_FIELD, ObjectParser.ValueType.OBJECT_ARRAY_BOOLEAN_OR_STRING);
         PARSER.declareObject(InnerHitBuilder::setHighlightBuilder, (p, c) -> HighlightBuilder.fromXContent(c),
                 SearchSourceBuilder.HIGHLIGHT_FIELD);
-        PARSER.declareObject(InnerHitBuilder::setChildInnerHits, (p, c) -> {
-            try {
-                Map<String, InnerHitBuilder> innerHitBuilders = new HashMap<>();
-                String innerHitName = null;
-                for (XContentParser.Token token = p.nextToken(); token != XContentParser.Token.END_OBJECT; token = p.nextToken()) {
-                    switch (token) {
-                        case START_OBJECT:
-                            InnerHitBuilder innerHitBuilder = InnerHitBuilder.fromXContent(c);
-                            innerHitBuilder.setName(innerHitName);
-                            innerHitBuilders.put(innerHitName, innerHitBuilder);
-                            break;
-                        case FIELD_NAME:
-                            innerHitName = p.currentName();
-                            break;
-                        default:
-                            throw new ParsingException(p.getTokenLocation(), "Expected [" + XContentParser.Token.START_OBJECT + "] in ["
-                                    + p.currentName() + "] but found [" + token + "]", p.getTokenLocation());
-                    }
-                }
-                return innerHitBuilders;
-            } catch (IOException e) {
-                throw new ParsingException(p.getTokenLocation(), "Could not parse inner query definition", e);
-            }
-        }, INNER_HITS_FIELD);
     }
 
     private String name;
-    private String nestedPath;
-    private String parentChildType;
+    private boolean ignoreUnmapped;
 
     private int from;
     private int size = 3;
@@ -145,67 +111,28 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
     private Set<ScriptField> scriptFields;
     private HighlightBuilder highlightBuilder;
     private FetchSourceContext fetchSourceContext;
-    private Map<String, InnerHitBuilder> childInnerHits;
 
     public InnerHitBuilder() {
+        this.name = null;
     }
 
-    private InnerHitBuilder(InnerHitBuilder other) {
-        name = other.name;
-        from = other.from;
-        size = other.size;
-        explain = other.explain;
-        version = other.version;
-        trackScores = other.trackScores;
-        if (other.storedFieldsContext != null) {
-            storedFieldsContext = new StoredFieldsContext(other.storedFieldsContext);
-        }
-        if (other.docValueFields != null) {
-            docValueFields = new ArrayList<> (other.docValueFields);
-        }
-        if (other.scriptFields != null) {
-            scriptFields = new HashSet<> (other.scriptFields);
-        }
-        if (other.fetchSourceContext != null) {
-            fetchSourceContext = new FetchSourceContext(
-                    other.fetchSourceContext.fetchSource(), other.fetchSourceContext.includes(), other.fetchSourceContext.excludes()
-            );
-        }
-        if (other.sorts != null) {
-            sorts = new ArrayList<>(other.sorts);
-        }
-        highlightBuilder = other.highlightBuilder;
-        if (other.childInnerHits != null) {
-            childInnerHits = new HashMap<>(other.childInnerHits);
-        }
-    }
-
-
-    InnerHitBuilder(InnerHitBuilder other, String nestedPath, QueryBuilder query) {
-        this(other);
-        this.query = query;
-        this.nestedPath = nestedPath;
-        if (name == null) {
-            this.name = nestedPath;
-        }
+    public InnerHitBuilder(String name) {
+        this.name = name;
     }
 
-    InnerHitBuilder(InnerHitBuilder other, QueryBuilder query, String parentChildType) {
-        this(other);
-        this.query = query;
-        this.parentChildType = parentChildType;
-        if (name == null) {
-            this.name = parentChildType;
-        }
-    }
 
     /**
      * Read from a stream.
      */
     public InnerHitBuilder(StreamInput in) throws IOException {
         name = in.readOptionalString();
-        nestedPath = in.readOptionalString();
-        parentChildType = in.readOptionalString();
+        if (in.getVersion().before(Version.V_5_5_0)) {
+            in.readOptionalString();
+            in.readOptionalString();
+        }
+        if (in.getVersion().onOrAfter(Version.V_5_2_0)) {
+            ignoreUnmapped = in.readBoolean();
+        }
         from = in.readVInt();
         size = in.readVInt();
         explain = in.readBoolean();
@@ -220,7 +147,7 @@ public InnerHitBuilder(StreamInput in) throws IOException {
                 scriptFields.add(new ScriptField(in));
             }
         }
-        fetchSourceContext = in.readOptionalStreamable(FetchSourceContext::new);
+        fetchSourceContext = in.readOptionalWriteable(FetchSourceContext::new);
         if (in.readBoolean()) {
             int size = in.readVInt();
             sorts = new ArrayList<>(size);
@@ -229,21 +156,23 @@ public InnerHitBuilder(StreamInput in) throws IOException {
             }
         }
         highlightBuilder = in.readOptionalWriteable(HighlightBuilder::new);
-        query = in.readNamedWriteable(QueryBuilder.class);
-        if (in.readBoolean()) {
-            int size = in.readVInt();
-            childInnerHits = new HashMap<>(size);
-            for (int i = 0; i < size; i++) {
-                childInnerHits.put(in.readString(), new InnerHitBuilder(in));
-            }
+        if (in.getVersion().before(Version.V_5_5_0)) {
+            /**
+             * this is needed for BWC with nodes pre 5.5
+             */
+            in.readNamedWriteable(QueryBuilder.class);
+            boolean hasChildren = in.readBoolean();
+            assert hasChildren == false;
         }
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
+        if (out.getVersion().before(Version.V_5_5_0)) {
+            throw new IOException("Invalid output version, must >= " + Version.V_5_5_0.toString());
+        }
         out.writeOptionalString(name);
-        out.writeOptionalString(nestedPath);
-        out.writeOptionalString(parentChildType);
+        out.writeBoolean(ignoreUnmapped);
         out.writeVInt(from);
         out.writeVInt(size);
         out.writeBoolean(explain);
@@ -255,11 +184,13 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeBoolean(hasScriptFields);
         if (hasScriptFields) {
             out.writeVInt(scriptFields.size());
-            for (ScriptField scriptField : scriptFields) {
-                scriptField.writeTo(out);
+            Iterator<ScriptField> iterator = scriptFields.stream()
+                    .sorted(Comparator.comparing(ScriptField::fieldName)).iterator();
+            while (iterator.hasNext()) {
+                iterator.next().writeTo(out);
             }
         }
-        out.writeOptionalStreamable(fetchSourceContext);
+        out.writeOptionalWriteable(fetchSourceContext);
         boolean hasSorts = sorts != null;
         out.writeBoolean(hasSorts);
         if (hasSorts) {
@@ -269,16 +200,82 @@ public void writeTo(StreamOutput out) throws IOException {
             }
         }
         out.writeOptionalWriteable(highlightBuilder);
-        out.writeNamedWriteable(query);
-        boolean hasChildInnerHits = childInnerHits != null;
-        out.writeBoolean(hasChildInnerHits);
-        if (hasChildInnerHits) {
-            out.writeVInt(childInnerHits.size());
-            for (Map.Entry<String, InnerHitBuilder> entry : childInnerHits.entrySet()) {
-                out.writeString(entry.getKey());
-                entry.getValue().writeTo(out);
+    }
+
+    /**
+     * BWC serialization for nested {@link InnerHitBuilder}.
+     * Should only be used to send nested inner hits to nodes pre 5.5.
+     */
+    protected void writeToNestedBWC(StreamOutput out, QueryBuilder query, String nestedPath) throws IOException {
+        assert out.getVersion().before(Version.V_5_5_0) :
+            "invalid output version, must be < " + Version.V_5_5_0.toString();
+        writeToBWC(out, query, nestedPath, null);
+    }
+
+    /**
+     * BWC serialization for collapsing {@link InnerHitBuilder}.
+     * Should only be used to send collapsing inner hits to nodes pre 5.5.
+     */
+    public void writeToCollapseBWC(StreamOutput out) throws IOException {
+        assert out.getVersion().before(Version.V_5_5_0) :
+            "invalid output version, must be < " + Version.V_5_5_0.toString();
+        writeToBWC(out, new MatchAllQueryBuilder(), null, null);
+    }
+
+    /**
+     * BWC serialization for parent/child {@link InnerHitBuilder}.
+     * Should only be used to send hasParent or hasChild inner hits to nodes pre 5.5.
+     */
+    public void writeToParentChildBWC(StreamOutput out, QueryBuilder query, String parentChildPath) throws IOException {
+        assert(out.getVersion().before(Version.V_5_5_0)) :
+            "invalid output version, must be < " + Version.V_5_5_0.toString();
+        writeToBWC(out, query, null, parentChildPath);
+    }
+
+    private void writeToBWC(StreamOutput out,
+                            QueryBuilder query,
+                            String nestedPath,
+                            String parentChildPath) throws IOException {
+        out.writeOptionalString(name);
+        if (nestedPath != null) {
+            out.writeOptionalString(nestedPath);
+            out.writeOptionalString(null);
+        } else {
+            out.writeOptionalString(null);
+            out.writeOptionalString(parentChildPath);
+        }
+        if (out.getVersion().onOrAfter(Version.V_5_2_0)) {
+            out.writeBoolean(ignoreUnmapped);
+        }
+        out.writeVInt(from);
+        out.writeVInt(size);
+        out.writeBoolean(explain);
+        out.writeBoolean(version);
+        out.writeBoolean(trackScores);
+        out.writeOptionalWriteable(storedFieldsContext);
+        out.writeGenericValue(docValueFields);
+        boolean hasScriptFields = scriptFields != null;
+        out.writeBoolean(hasScriptFields);
+        if (hasScriptFields) {
+            out.writeVInt(scriptFields.size());
+            Iterator<ScriptField> iterator = scriptFields.stream()
+                .sorted(Comparator.comparing(ScriptField::fieldName)).iterator();
+            while (iterator.hasNext()) {
+                iterator.next().writeTo(out);
+            }
+        }
+        out.writeOptionalWriteable(fetchSourceContext);
+        boolean hasSorts = sorts != null;
+        out.writeBoolean(hasSorts);
+        if (hasSorts) {
+            out.writeVInt(sorts.size());
+            for (SortBuilder<?> sort : sorts) {
+                out.writeNamedWriteable(sort);
             }
         }
+        out.writeOptionalWriteable(highlightBuilder);
+        out.writeNamedWriteable(query);
+        out.writeBoolean(false);
     }
 
     public String getName() {
@@ -290,6 +287,18 @@ public InnerHitBuilder setName(String name) {
         return this;
     }
 
+    public InnerHitBuilder setIgnoreUnmapped(boolean value) {
+        this.ignoreUnmapped = value;
+        return this;
+    }
+
+    /**
+     * Whether to include inner hits in the search response hits if required mappings is missing
+     */
+    public boolean isIgnoreUnmapped() {
+        return ignoreUnmapped;
+    }
+
     public int getFrom() {
         return from;
     }
@@ -500,129 +509,13 @@ QueryBuilder getQuery() {
         return query;
     }
 
-    void setChildInnerHits(Map<String, InnerHitBuilder> childInnerHits) {
-        this.childInnerHits = childInnerHits;
-    }
-
-    String getParentChildType() {
-        return parentChildType;
-    }
-
-    String getNestedPath() {
-        return nestedPath;
-    }
-
-    void addChildInnerHit(InnerHitBuilder innerHitBuilder) {
-        if (childInnerHits == null) {
-            childInnerHits = new HashMap<>();
-        }
-        this.childInnerHits.put(innerHitBuilder.getName(), innerHitBuilder);
-    }
-
-    public InnerHitsContext.BaseInnerHits build(SearchContext parentSearchContext,
-                                                InnerHitsContext innerHitsContext) throws IOException {
-        QueryShardContext queryShardContext = parentSearchContext.getQueryShardContext();
-        if (nestedPath != null) {
-            ObjectMapper nestedObjectMapper = queryShardContext.getObjectMapper(nestedPath);
-            ObjectMapper parentObjectMapper = queryShardContext.nestedScope().nextLevel(nestedObjectMapper);
-            InnerHitsContext.NestedInnerHits nestedInnerHits = new InnerHitsContext.NestedInnerHits(
-                    name, parentSearchContext, parentObjectMapper, nestedObjectMapper
-            );
-            setupInnerHitsContext(queryShardContext, nestedInnerHits);
-            if (childInnerHits != null) {
-                buildChildInnerHits(parentSearchContext, nestedInnerHits);
-            }
-            queryShardContext.nestedScope().previousLevel();
-            innerHitsContext.addInnerHitDefinition(nestedInnerHits);
-            return nestedInnerHits;
-        } else if (parentChildType != null) {
-            DocumentMapper documentMapper = queryShardContext.getMapperService().documentMapper(parentChildType);
-            InnerHitsContext.ParentChildInnerHits parentChildInnerHits = new InnerHitsContext.ParentChildInnerHits(
-                    name, parentSearchContext, queryShardContext.getMapperService(), documentMapper
-            );
-            setupInnerHitsContext(queryShardContext, parentChildInnerHits);
-            if (childInnerHits != null) {
-                buildChildInnerHits(parentSearchContext, parentChildInnerHits);
-            }
-            innerHitsContext.addInnerHitDefinition( parentChildInnerHits);
-            return parentChildInnerHits;
-        } else {
-            throw new IllegalStateException("Neither a nested or parent/child inner hit");
-        }
-    }
-
-    private void buildChildInnerHits(SearchContext parentSearchContext, InnerHitsContext.BaseInnerHits innerHits) throws IOException {
-        Map<String, InnerHitsContext.BaseInnerHits> childInnerHits = new HashMap<>();
-        for (Map.Entry<String, InnerHitBuilder> entry : this.childInnerHits.entrySet()) {
-            InnerHitsContext.BaseInnerHits childInnerHit = entry.getValue().build(
-                    parentSearchContext, new InnerHitsContext()
-            );
-            childInnerHits.put(entry.getKey(), childInnerHit);
-        }
-        innerHits.setChildInnerHits(childInnerHits);
-    }
-
-    private void setupInnerHitsContext(QueryShardContext context, InnerHitsContext.BaseInnerHits innerHitsContext) throws IOException {
-        innerHitsContext.from(from);
-        innerHitsContext.size(size);
-        innerHitsContext.explain(explain);
-        innerHitsContext.version(version);
-        innerHitsContext.trackScores(trackScores);
-        if (storedFieldsContext != null) {
-            innerHitsContext.storedFieldsContext(storedFieldsContext);
-        }
-        if (docValueFields != null) {
-            DocValueFieldsContext docValueFieldsContext = innerHitsContext
-                    .getFetchSubPhaseContext(DocValueFieldsFetchSubPhase.CONTEXT_FACTORY);
-            for (String field : docValueFields) {
-                docValueFieldsContext.add(new DocValueFieldsContext.DocValueField(field));
-            }
-            docValueFieldsContext.setHitExecutionNeeded(true);
-        }
-        if (scriptFields != null) {
-            for (ScriptField field : scriptFields) {
-                SearchScript searchScript = innerHitsContext.scriptService().search(innerHitsContext.lookup(), field.script(),
-                        ScriptContext.Standard.SEARCH, Collections.emptyMap());
-                innerHitsContext.scriptFields().add(new org.elasticsearch.search.fetch.subphase.ScriptFieldsContext.ScriptField(
-                        field.fieldName(), searchScript, field.ignoreFailure()));
-            }
-        }
-        if (fetchSourceContext != null) {
-            innerHitsContext.fetchSourceContext(fetchSourceContext);
-        }
-        if (sorts != null) {
-            Optional<SortAndFormats> optionalSort = SortBuilder.buildSort(sorts, context);
-            if (optionalSort.isPresent()) {
-                innerHitsContext.sort(optionalSort.get());
-            }
-        }
-        if (highlightBuilder != null) {
-            innerHitsContext.highlight(highlightBuilder.build(context));
-        }
-        ParsedQuery parsedQuery = new ParsedQuery(query.toQuery(context), context.copyNamedQueries());
-        innerHitsContext.parsedQuery(parsedQuery);
-    }
-
-    public void inlineInnerHits(Map<String, InnerHitBuilder> innerHits) {
-        InnerHitBuilder copy = new InnerHitBuilder(this);
-        copy.parentChildType = this.parentChildType;
-        copy.nestedPath = this.nestedPath;
-        copy.query = this.query;
-        innerHits.put(copy.getName(), copy);
-
-        Map<String, InnerHitBuilder> childInnerHits = new HashMap<>();
-        extractInnerHits(query, childInnerHits);
-        if (childInnerHits.size() > 0) {
-            copy.setChildInnerHits(childInnerHits);
-        }
-    }
-
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
         if (name != null) {
             builder.field(NAME_FIELD.getPreferredName(), name);
         }
+        builder.field(IGNORE_UNMAPPED.getPreferredName(), ignoreUnmapped);
         builder.field(SearchSourceBuilder.FROM_FIELD.getPreferredName(), from);
         builder.field(SearchSourceBuilder.SIZE_FIELD.getPreferredName(), size);
         builder.field(SearchSourceBuilder.VERSION_FIELD.getPreferredName(), version);
@@ -658,13 +551,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (highlightBuilder != null) {
             builder.field(SearchSourceBuilder.HIGHLIGHT_FIELD.getPreferredName(), highlightBuilder, params);
         }
-        if (childInnerHits != null) {
-            builder.startObject(INNER_HITS_FIELD.getPreferredName());
-            for (Map.Entry<String, InnerHitBuilder> entry : childInnerHits.entrySet()) {
-                builder.field(entry.getKey(), entry.getValue(), params);
-            }
-            builder.endObject();
-        }
         builder.endObject();
         return builder;
     }
@@ -676,8 +562,7 @@ public boolean equals(Object o) {
 
         InnerHitBuilder that = (InnerHitBuilder) o;
         return Objects.equals(name, that.name) &&
-                Objects.equals(nestedPath, that.nestedPath) &&
-                Objects.equals(parentChildType, that.parentChildType) &&
+                Objects.equals(ignoreUnmapped, that.ignoreUnmapped) &&
                 Objects.equals(from, that.from) &&
                 Objects.equals(size, that.size) &&
                 Objects.equals(explain, that.explain) &&
@@ -688,40 +573,16 @@ public boolean equals(Object o) {
                 Objects.equals(scriptFields, that.scriptFields) &&
                 Objects.equals(fetchSourceContext, that.fetchSourceContext) &&
                 Objects.equals(sorts, that.sorts) &&
-                Objects.equals(highlightBuilder, that.highlightBuilder) &&
-                Objects.equals(query, that.query) &&
-                Objects.equals(childInnerHits, that.childInnerHits);
+                Objects.equals(highlightBuilder, that.highlightBuilder);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(name, nestedPath, parentChildType, from, size, explain, version, trackScores, storedFieldsContext,
-            docValueFields, scriptFields, fetchSourceContext, sorts, highlightBuilder, query, childInnerHits);
+        return Objects.hash(name, ignoreUnmapped, from, size, explain, version, trackScores,
+                storedFieldsContext, docValueFields, scriptFields, fetchSourceContext, sorts, highlightBuilder);
     }
 
     public static InnerHitBuilder fromXContent(QueryParseContext context) throws IOException {
         return PARSER.parse(context.parser(), new InnerHitBuilder(), context);
     }
-
-    public static void extractInnerHits(QueryBuilder query, Map<String, InnerHitBuilder> innerHitBuilders) {
-        if (query instanceof AbstractQueryBuilder) {
-            ((AbstractQueryBuilder) query).extractInnerHitBuilders(innerHitBuilders);
-        } else {
-            throw new IllegalStateException("provided query builder [" + query.getClass() +
-                    "] class should inherit from AbstractQueryBuilder, but it doesn't");
-        }
-    }
-
-    static InnerHitBuilder rewrite(InnerHitBuilder original, QueryBuilder rewrittenQuery) {
-        if (original == null) {
-            return null;
-        }
-
-        InnerHitBuilder copy = new InnerHitBuilder(original);
-        copy.query = rewrittenQuery;
-        copy.parentChildType = original.parentChildType;
-        copy.nestedPath = original.nestedPath;
-        return copy;
-    }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/index/query/InnerHitContextBuilder.java b/core/src/main/java/org/elasticsearch/index/query/InnerHitContextBuilder.java
new file mode 100644
index 0000000000000..e68c5631844ea
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/query/InnerHitContextBuilder.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.query;
+
+import org.elasticsearch.script.ScriptContext;
+import org.elasticsearch.script.SearchScript;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext;
+import org.elasticsearch.search.fetch.subphase.InnerHitsContext;
+import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.search.sort.SortAndFormats;
+import org.elasticsearch.search.sort.SortBuilder;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * A builder for {@link InnerHitsContext.InnerHitSubContext}
+ */
+public abstract class InnerHitContextBuilder {
+    protected final QueryBuilder query;
+    protected final InnerHitBuilder innerHitBuilder;
+    protected final Map<String, InnerHitContextBuilder> children;
+
+    protected InnerHitContextBuilder(QueryBuilder query, InnerHitBuilder innerHitBuilder, Map<String, InnerHitContextBuilder> children) {
+        this.innerHitBuilder = innerHitBuilder;
+        this.children = children;
+        this.query = query;
+    }
+
+    public abstract void build(SearchContext parentSearchContext,
+                                InnerHitsContext innerHitsContext) throws IOException;
+
+    public static void extractInnerHits(QueryBuilder query, Map<String, InnerHitContextBuilder> innerHitBuilders) {
+        if (query instanceof AbstractQueryBuilder) {
+            ((AbstractQueryBuilder) query).extractInnerHitBuilders(innerHitBuilders);
+        } else {
+            throw new IllegalStateException("provided query builder [" + query.getClass() +
+                "] class should inherit from AbstractQueryBuilder, but it doesn't");
+        }
+    }
+
+    protected void setupInnerHitsContext(QueryShardContext queryShardContext,
+                                         InnerHitsContext.InnerHitSubContext innerHitsContext) throws IOException {
+        innerHitsContext.from(innerHitBuilder.getFrom());
+        innerHitsContext.size(innerHitBuilder.getSize());
+        innerHitsContext.explain(innerHitBuilder.isExplain());
+        innerHitsContext.version(innerHitBuilder.isVersion());
+        innerHitsContext.trackScores(innerHitBuilder.isTrackScores());
+        if (innerHitBuilder.getStoredFieldsContext() != null) {
+            innerHitsContext.storedFieldsContext(innerHitBuilder.getStoredFieldsContext());
+        }
+        if (innerHitBuilder.getDocValueFields() != null) {
+            innerHitsContext.docValueFieldsContext(new DocValueFieldsContext(innerHitBuilder.getDocValueFields()));
+        }
+        if (innerHitBuilder.getScriptFields() != null) {
+            for (SearchSourceBuilder.ScriptField field : innerHitBuilder.getScriptFields()) {
+                SearchScript searchScript = innerHitsContext.getQueryShardContext().getSearchScript(field.script(),
+                    ScriptContext.Standard.SEARCH);
+                innerHitsContext.scriptFields().add(new org.elasticsearch.search.fetch.subphase.ScriptFieldsContext.ScriptField(
+                    field.fieldName(), searchScript, field.ignoreFailure()));
+            }
+        }
+        if (innerHitBuilder.getFetchSourceContext() != null) {
+            innerHitsContext.fetchSourceContext(innerHitBuilder.getFetchSourceContext() );
+        }
+        if (innerHitBuilder.getSorts() != null) {
+            Optional<SortAndFormats> optionalSort = SortBuilder.buildSort(innerHitBuilder.getSorts(), queryShardContext);
+            if (optionalSort.isPresent()) {
+                innerHitsContext.sort(optionalSort.get());
+            }
+        }
+        if (innerHitBuilder.getHighlightBuilder() != null) {
+            innerHitsContext.highlight(innerHitBuilder.getHighlightBuilder().build(queryShardContext));
+        }
+        ParsedQuery parsedQuery = new ParsedQuery(query.toQuery(queryShardContext), queryShardContext.copyNamedQueries());
+        innerHitsContext.parsedQuery(parsedQuery);
+        Map<String, InnerHitsContext.InnerHitSubContext> baseChildren =
+            buildChildInnerHits(innerHitsContext.parentSearchContext(), children);
+        innerHitsContext.setChildInnerHits(baseChildren);
+    }
+
+    private static Map<String, InnerHitsContext.InnerHitSubContext> buildChildInnerHits(SearchContext parentSearchContext,
+                                Map<String, InnerHitContextBuilder> children) throws IOException {
+
+        Map<String, InnerHitsContext.InnerHitSubContext> childrenInnerHits = new HashMap<>();
+        for (Map.Entry<String, InnerHitContextBuilder> entry : children.entrySet()) {
+            InnerHitsContext childInnerHitsContext = new InnerHitsContext();
+            entry.getValue().build(
+                parentSearchContext, childInnerHitsContext);
+            if (childInnerHitsContext.getInnerHits() != null) {
+                childrenInnerHits.putAll(childInnerHitsContext.getInnerHits());
+            }
+        }
+        return childrenInnerHits;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/index/query/MatchAllQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/MatchAllQueryBuilder.java
index ba704809a4fd0..4e2deab6b93bf 100644
--- a/core/src/main/java/org/elasticsearch/index/query/MatchAllQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/MatchAllQueryBuilder.java
@@ -20,13 +20,12 @@
 package org.elasticsearch.index.query;
 
 import org.apache.lucene.search.Query;
-import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lucene.search.Queries;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.Optional;
@@ -48,7 +47,7 @@ public MatchAllQueryBuilder(StreamInput in) throws IOException {
     }
 
     @Override
-    protected void doWriteTo(StreamOutput out) throws IOException {
+    protected void doWriteTo(StreamOutput out) {
         // only superclass has state
     }
 
@@ -59,38 +58,22 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep
         builder.endObject();
     }
 
-    public static Optional<MatchAllQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
-        XContentParser parser = parseContext.parser();
+    private static final ObjectParser<MatchAllQueryBuilder, QueryParseContext> PARSER = new ObjectParser<>(NAME, MatchAllQueryBuilder::new);
 
-        String currentFieldName = null;
-        XContentParser.Token token;
-        String queryName = null;
-        float boost = AbstractQueryBuilder.DEFAULT_BOOST;
-        while (((token = parser.nextToken()) != XContentParser.Token.END_OBJECT && token != XContentParser.Token.END_ARRAY)) {
-            if (token == XContentParser.Token.FIELD_NAME) {
-                currentFieldName = parser.currentName();
-            } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
-                    queryName = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
-                    boost = parser.floatValue();
-                } else {
-                    throw new ParsingException(parser.getTokenLocation(), "[" + MatchAllQueryBuilder.NAME +
-                            "] query does not support [" + currentFieldName + "]");
-                }
-            } else {
-                throw new ParsingException(parser.getTokenLocation(), "[" + MatchAllQueryBuilder.NAME +
-                        "] unknown token [" + token + "] after [" + currentFieldName + "]");
-            }
+    static {
+        declareStandardFields(PARSER);
+    }
+
+    public static Optional<MatchAllQueryBuilder> fromXContent(QueryParseContext context) {
+        try {
+            return Optional.of(PARSER.apply(context.parser(), context));
+        } catch (IllegalArgumentException e) {
+            throw new ParsingException(context.parser().getTokenLocation(), e.getMessage(), e);
         }
-        MatchAllQueryBuilder queryBuilder = new MatchAllQueryBuilder();
-        queryBuilder.boost(boost);
-        queryBuilder.queryName(queryName);
-        return Optional.of(queryBuilder);
     }
 
     @Override
-    protected Query doToQuery(QueryShardContext context) throws IOException {
+    protected Query doToQuery(QueryShardContext context) {
         return Queries.newMatchAllQuery();
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/query/MatchNoneQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/MatchNoneQueryBuilder.java
index 55a6e544f495a..73ca411be32c6 100644
--- a/core/src/main/java/org/elasticsearch/index/query/MatchNoneQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/MatchNoneQueryBuilder.java
@@ -70,9 +70,9 @@ public static Optional<MatchNoneQueryBuilder> fromXContent(QueryParseContext par
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "["+MatchNoneQueryBuilder.NAME +
diff --git a/core/src/main/java/org/elasticsearch/index/query/MatchPhrasePrefixQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/MatchPhrasePrefixQueryBuilder.java
index bff28d0f5be0c..e4af7c8c98ebd 100644
--- a/core/src/main/java/org/elasticsearch/index/query/MatchPhrasePrefixQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/MatchPhrasePrefixQueryBuilder.java
@@ -164,7 +164,7 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep
     @Override
     protected Query doToQuery(QueryShardContext context) throws IOException {
         // validate context specific fields
-        if (analyzer != null && context.getAnalysisService().analyzer(analyzer) == null) {
+        if (analyzer != null && context.getIndexAnalyzers().get(analyzer) == null) {
             throw new QueryShardException(context, "[" + NAME + "] analyzer [" + analyzer + "] not found");
         }
 
@@ -213,17 +213,17 @@ public static Optional<MatchPhrasePrefixQueryBuilder> fromXContent(QueryParseCon
                     if (token == XContentParser.Token.FIELD_NAME) {
                         currentFieldName = parser.currentName();
                     } else if (token.isValue()) {
-                        if (parseContext.getParseFieldMatcher().match(currentFieldName, MatchQueryBuilder.QUERY_FIELD)) {
+                        if (MatchQueryBuilder.QUERY_FIELD.match(currentFieldName)) {
                             value = parser.objectText();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, MatchQueryBuilder.ANALYZER_FIELD)) {
+                        } else if (MatchQueryBuilder.ANALYZER_FIELD.match(currentFieldName)) {
                             analyzer = parser.text();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                        } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                             boost = parser.floatValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, MatchPhraseQueryBuilder.SLOP_FIELD)) {
+                        } else if (MatchPhraseQueryBuilder.SLOP_FIELD.match(currentFieldName)) {
                             slop = parser.intValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, MAX_EXPANSIONS_FIELD)) {
+                        } else if (MAX_EXPANSIONS_FIELD.match(currentFieldName)) {
                             maxExpansion = parser.intValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                        } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                             queryName = parser.text();
                         } else {
                             throw new ParsingException(parser.getTokenLocation(),
diff --git a/core/src/main/java/org/elasticsearch/index/query/MatchPhraseQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/MatchPhraseQueryBuilder.java
index 6fd6922d9cef5..6cf6d00678449 100644
--- a/core/src/main/java/org/elasticsearch/index/query/MatchPhraseQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/MatchPhraseQueryBuilder.java
@@ -140,7 +140,7 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep
     @Override
     protected Query doToQuery(QueryShardContext context) throws IOException {
         // validate context specific fields
-        if (analyzer != null && context.getAnalysisService().analyzer(analyzer) == null) {
+        if (analyzer != null && context.getIndexAnalyzers().get(analyzer) == null) {
             throw new QueryShardException(context, "[" + NAME + "] analyzer [" + analyzer + "] not found");
         }
 
@@ -184,15 +184,15 @@ public static Optional<MatchPhraseQueryBuilder> fromXContent(QueryParseContext p
                     if (token == XContentParser.Token.FIELD_NAME) {
                         currentFieldName = parser.currentName();
                     } else if (token.isValue()) {
-                        if (parseContext.getParseFieldMatcher().match(currentFieldName, MatchQueryBuilder.QUERY_FIELD)) {
+                        if (MatchQueryBuilder.QUERY_FIELD.match(currentFieldName)) {
                             value = parser.objectText();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, MatchQueryBuilder.ANALYZER_FIELD)) {
+                        } else if (MatchQueryBuilder.ANALYZER_FIELD.match(currentFieldName)) {
                             analyzer = parser.text();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                        } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                             boost = parser.floatValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, SLOP_FIELD)) {
+                        } else if (SLOP_FIELD.match(currentFieldName)) {
                             slop = parser.intValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                        } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                             queryName = parser.text();
                         } else {
                             throw new ParsingException(parser.getTokenLocation(),
diff --git a/core/src/main/java/org/elasticsearch/index/query/MatchQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/MatchQueryBuilder.java
index dc6ac994084a9..824829d505ba4 100644
--- a/core/src/main/java/org/elasticsearch/index/query/MatchQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/MatchQueryBuilder.java
@@ -19,8 +19,6 @@
 
 package org.elasticsearch.index.query;
 
-import org.apache.lucene.queries.ExtendedCommonTermsQuery;
-import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.FuzzyQuery;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.common.ParseField;
@@ -444,7 +442,7 @@ public void doXContent(XContentBuilder builder, Params params) throws IOExceptio
     @Override
     protected Query doToQuery(QueryShardContext context) throws IOException {
         // validate context specific fields
-        if (analyzer != null && context.getAnalysisService().analyzer(analyzer) == null) {
+        if (analyzer != null && context.getIndexAnalyzers().get(analyzer) == null) {
             throw new QueryShardException(context, "[" + NAME + "] analyzer [" + analyzer + "] not found");
         }
 
@@ -456,25 +454,13 @@ protected Query doToQuery(QueryShardContext context) throws IOException {
         matchQuery.setFuzzyPrefixLength(prefixLength);
         matchQuery.setMaxExpansions(maxExpansions);
         matchQuery.setTranspositions(fuzzyTranspositions);
-        matchQuery.setFuzzyRewriteMethod(QueryParsers.parseRewriteMethod(context.getParseFieldMatcher(), fuzzyRewrite, null));
+        matchQuery.setFuzzyRewriteMethod(QueryParsers.parseRewriteMethod(fuzzyRewrite, null));
         matchQuery.setLenient(lenient);
         matchQuery.setCommonTermsCutoff(cutoffFrequency);
         matchQuery.setZeroTermsQuery(zeroTermsQuery);
 
         Query query = matchQuery.parse(type, fieldName, value);
-        if (query == null) {
-            return null;
-        }
-
-        // If the coordination factor is disabled on a boolean query we don't apply the minimum should match.
-        // This is done to make sure that the minimum_should_match doesn't get applied when there is only one word
-        // and multiple variations of the same word in the query (synonyms for instance).
-        if (query instanceof BooleanQuery && !((BooleanQuery) query).isCoordDisabled()) {
-            query = Queries.applyMinimumShouldMatch((BooleanQuery) query, minimumShouldMatch);
-        } else if (query instanceof ExtendedCommonTermsQuery) {
-            ((ExtendedCommonTermsQuery)query).setLowFreqMinimumNumberShouldMatch(minimumShouldMatch);
-        }
-        return query;
+        return Queries.maybeApplyMinimumShouldMatch(query, minimumShouldMatch);
     }
 
     @Override
@@ -541,9 +527,9 @@ public static Optional<MatchQueryBuilder> fromXContent(QueryParseContext parseCo
                     if (token == XContentParser.Token.FIELD_NAME) {
                         currentFieldName = parser.currentName();
                     } else if (token.isValue()) {
-                        if (parseContext.getParseFieldMatcher().match(currentFieldName, QUERY_FIELD)) {
+                        if (QUERY_FIELD.match(currentFieldName)) {
                             value = parser.objectText();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, TYPE_FIELD)) {
+                        } else if (TYPE_FIELD.match(currentFieldName)) {
                             String tStr = parser.text();
                             if ("boolean".equals(tStr)) {
                                 type = MatchQuery.Type.BOOLEAN;
@@ -554,31 +540,31 @@ public static Optional<MatchQueryBuilder> fromXContent(QueryParseContext parseCo
                             } else {
                                 throw new ParsingException(parser.getTokenLocation(), "[" + NAME + "] query does not support type " + tStr);
                             }
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, ANALYZER_FIELD)) {
+                        } else if (ANALYZER_FIELD.match(currentFieldName)) {
                             analyzer = parser.text();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                        } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                             boost = parser.floatValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, SLOP_FIELD)) {
+                        } else if (SLOP_FIELD.match(currentFieldName)) {
                             slop = parser.intValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, Fuzziness.FIELD)) {
+                        } else if (Fuzziness.FIELD.match(currentFieldName)) {
                             fuzziness = Fuzziness.parse(parser);
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, PREFIX_LENGTH_FIELD)) {
+                        } else if (PREFIX_LENGTH_FIELD.match(currentFieldName)) {
                             prefixLength = parser.intValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, MAX_EXPANSIONS_FIELD)) {
+                        } else if (MAX_EXPANSIONS_FIELD.match(currentFieldName)) {
                             maxExpansion = parser.intValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, OPERATOR_FIELD)) {
+                        } else if (OPERATOR_FIELD.match(currentFieldName)) {
                             operator = Operator.fromString(parser.text());
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, MINIMUM_SHOULD_MATCH_FIELD)) {
+                        } else if (MINIMUM_SHOULD_MATCH_FIELD.match(currentFieldName)) {
                             minimumShouldMatch = parser.textOrNull();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, FUZZY_REWRITE_FIELD)) {
+                        } else if (FUZZY_REWRITE_FIELD.match(currentFieldName)) {
                             fuzzyRewrite = parser.textOrNull();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, FUZZY_TRANSPOSITIONS_FIELD)) {
+                        } else if (FUZZY_TRANSPOSITIONS_FIELD.match(currentFieldName)) {
                             fuzzyTranspositions = parser.booleanValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, LENIENT_FIELD)) {
+                        } else if (LENIENT_FIELD.match(currentFieldName)) {
                             lenient = parser.booleanValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, CUTOFF_FREQUENCY_FIELD)) {
+                        } else if (CUTOFF_FREQUENCY_FIELD.match(currentFieldName)) {
                             cutOffFrequency = parser.floatValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, ZERO_TERMS_QUERY_FIELD)) {
+                        } else if (ZERO_TERMS_QUERY_FIELD.match(currentFieldName)) {
                             String zeroTermsDocs = parser.text();
                             if ("none".equalsIgnoreCase(zeroTermsDocs)) {
                                 zeroTermsQuery = MatchQuery.ZeroTermsQuery.NONE;
@@ -588,7 +574,7 @@ public static Optional<MatchQueryBuilder> fromXContent(QueryParseContext parseCo
                                 throw new ParsingException(parser.getTokenLocation(),
                                         "Unsupported zero_terms_docs value [" + zeroTermsDocs + "]");
                             }
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                        } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                             queryName = parser.text();
                         } else {
                             throw new ParsingException(parser.getTokenLocation(),
diff --git a/core/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java
index 9fb1343b11899..e1ab8da5a31ae 100644
--- a/core/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java
@@ -21,13 +21,13 @@
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.index.Fields;
-import org.apache.lucene.queries.TermsQuery;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.termvectors.MultiTermVectorsItemResponse;
 import org.elasticsearch.action.termvectors.MultiTermVectorsRequest;
 import org.elasticsearch.action.termvectors.MultiTermVectorsResponse;
@@ -36,7 +36,6 @@
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -52,11 +51,11 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.VersionType;
-import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.mapper.UidFieldMapper;
 import org.elasticsearch.index.mapper.KeywordFieldMapper.KeywordFieldType;
+import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.StringFieldMapper.StringFieldType;
 import org.elasticsearch.index.mapper.TextFieldMapper.TextFieldType;
+import org.elasticsearch.index.mapper.UidFieldMapper;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -147,7 +146,7 @@ private interface Field {
      */
     public static final class Item implements ToXContent, Writeable {
         public static final Item[] EMPTY_ARRAY = new Item[0];
-        
+
         public interface Field {
             ParseField INDEX = new ParseField("_index");
             ParseField TYPE = new ParseField("_type");
@@ -164,6 +163,7 @@ public interface Field {
         private String type;
         private String id;
         private BytesReference doc;
+        private XContentType xContentType;
         private String[] fields;
         private Map<String, String> perFieldAnalyzer;
         private String routing;
@@ -180,7 +180,9 @@ public Item() {
             this.index = copy.index;
             this.type = copy.type;
             this.id = copy.id;
+            this.routing = copy.routing;
             this.doc = copy.doc;
+            this.xContentType = copy.xContentType;
             this.fields = copy.fields;
             this.perFieldAnalyzer = copy.perFieldAnalyzer;
             this.version = copy.version;
@@ -217,6 +219,7 @@ public Item(@Nullable String index, @Nullable String type, XContentBuilder doc)
             this.index = index;
             this.type = type;
             this.doc = doc.bytes();
+            this.xContentType = doc.contentType();
         }
 
         /**
@@ -227,6 +230,11 @@ public Item(@Nullable String index, @Nullable String type, XContentBuilder doc)
             type = in.readOptionalString();
             if (in.readBoolean()) {
                 doc = (BytesReference) in.readGenericValue();
+                if (in.getVersion().onOrAfter(Version.V_5_3_0)) {
+                    xContentType = XContentType.readFrom(in);
+                } else {
+                    xContentType = XContentFactory.xContentType(doc);
+                }
             } else {
                 id = in.readString();
             }
@@ -244,6 +252,9 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeBoolean(doc != null);
             if (doc != null) {
                 out.writeGenericValue(doc);
+                if (out.getVersion().onOrAfter(Version.V_5_3_0)) {
+                    xContentType.writeTo(out);
+                }
             } else {
                 out.writeString(id);
             }
@@ -328,10 +339,14 @@ public Item versionType(VersionType versionType) {
             return this;
         }
 
+        XContentType xContentType() {
+            return xContentType;
+        }
+
         /**
          * Convert this to a {@link TermVectorsRequest} for fetching the terms of the document.
          */
-        public TermVectorsRequest toTermVectorsRequest() {
+        TermVectorsRequest toTermVectorsRequest() {
             TermVectorsRequest termVectorsRequest = new TermVectorsRequest(index, type, id)
                     .selectedFields(fields)
                     .routing(routing)
@@ -345,7 +360,7 @@ public TermVectorsRequest toTermVectorsRequest() {
                     .termStatistics(false);
             // for artificial docs to make sure that the id has changed in the item too
             if (doc != null) {
-                termVectorsRequest.doc(doc, true);
+                termVectorsRequest.doc(doc, true, xContentType);
                 this.id = termVectorsRequest.id();
             }
             return termVectorsRequest;
@@ -354,22 +369,23 @@ public TermVectorsRequest toTermVectorsRequest() {
         /**
          * Parses and returns the given item.
          */
-        public static Item parse(XContentParser parser, ParseFieldMatcher parseFieldMatcher, Item item) throws IOException {
+        public static Item parse(XContentParser parser, Item item) throws IOException {
             XContentParser.Token token;
             String currentFieldName = null;
             while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
                 if (token == XContentParser.Token.FIELD_NAME) {
                     currentFieldName = parser.currentName();
                 } else if (currentFieldName != null) {
-                    if (parseFieldMatcher.match(currentFieldName, Field.INDEX)) {
+                    if (Field.INDEX.match(currentFieldName)) {
                         item.index = parser.text();
-                    } else if (parseFieldMatcher.match(currentFieldName, Field.TYPE)) {
+                    } else if (Field.TYPE.match(currentFieldName)) {
                         item.type = parser.text();
-                    } else if (parseFieldMatcher.match(currentFieldName, Field.ID)) {
+                    } else if (Field.ID.match(currentFieldName)) {
                         item.id = parser.text();
-                    } else if (parseFieldMatcher.match(currentFieldName, Field.DOC)) {
+                    } else if (Field.DOC.match(currentFieldName)) {
                         item.doc = jsonBuilder().copyCurrentStructure(parser).bytes();
-                    } else if (parseFieldMatcher.match(currentFieldName, Field.FIELDS)) {
+                        item.xContentType = XContentType.JSON;
+                    } else if (Field.FIELDS.match(currentFieldName)) {
                         if (token == XContentParser.Token.START_ARRAY) {
                             List<String> fields = new ArrayList<>();
                             while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
@@ -380,7 +396,7 @@ public static Item parse(XContentParser parser, ParseFieldMatcher parseFieldMatc
                             throw new ElasticsearchParseException(
                                     "failed to parse More Like This item. field [fields] must be an array");
                         }
-                    } else if (parseFieldMatcher.match(currentFieldName, Field.PER_FIELD_ANALYZER)) {
+                    } else if (Field.PER_FIELD_ANALYZER.match(currentFieldName)) {
                         item.perFieldAnalyzer(TermVectorsRequest.readPerFieldAnalyzer(parser.map()));
                     } else if ("_routing".equals(currentFieldName) || "routing".equals(currentFieldName)) {
                         item.routing = parser.text();
@@ -419,16 +435,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                 builder.field(Field.ID.getPreferredName(), this.id);
             }
             if (this.doc != null) {
-                XContentType contentType = XContentFactory.xContentType(this.doc);
-                if (contentType == builder.contentType()) {
-                    builder.rawField(Field.DOC.getPreferredName(), this.doc);
-                } else {
-                    try (XContentParser parser = XContentFactory.xContent(contentType).createParser(this.doc)) {
-                        parser.nextToken();
-                        builder.field(Field.DOC.getPreferredName());
-                        builder.copyCurrentStructure(parser);
-                    }
-                }
+                builder.rawField(Field.DOC.getPreferredName(), this.doc, xContentType);
             }
             if (this.fields != null) {
                 builder.array(Field.FIELDS.getPreferredName(), this.fields);
@@ -780,7 +787,7 @@ public static Item[] ids(String... ids) {
     protected void doXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject(NAME);
         if (fields != null) {
-            builder.field(Field.FIELDS.getPreferredName(), fields);
+            builder.array(Field.FIELDS.getPreferredName(), fields);
         }
         buildLikeField(builder, Field.LIKE.getPreferredName(), likeTexts, likeItems);
         buildLikeField(builder, Field.UNLIKE.getPreferredName(), unlikeTexts, unlikeItems);
@@ -791,7 +798,7 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep
         builder.field(Field.MIN_WORD_LENGTH.getPreferredName(), minWordLength);
         builder.field(Field.MAX_WORD_LENGTH.getPreferredName(), maxWordLength);
         if (stopWords != null) {
-            builder.field(Field.STOP_WORDS.getPreferredName(), stopWords);
+            builder.array(Field.STOP_WORDS.getPreferredName(), stopWords);
         }
         if (analyzer != null) {
             builder.field(Field.ANALYZER.getPreferredName(), analyzer);
@@ -840,33 +847,33 @@ public static Optional<MoreLikeThisQueryBuilder> fromXContent(QueryParseContext
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, Field.LIKE)) {
+                if (Field.LIKE.match(currentFieldName)) {
                     parseLikeField(parseContext, likeTexts, likeItems);
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, Field.UNLIKE)) {
+                } else if (Field.UNLIKE.match(currentFieldName)) {
                     parseLikeField(parseContext, unlikeTexts, unlikeItems);
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, Field.LIKE_TEXT)) {
+                } else if (Field.LIKE_TEXT.match(currentFieldName)) {
                     likeTexts.add(parser.text());
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, Field.MAX_QUERY_TERMS)) {
+                } else if (Field.MAX_QUERY_TERMS.match(currentFieldName)) {
                     maxQueryTerms = parser.intValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, Field.MIN_TERM_FREQ)) {
+                } else if (Field.MIN_TERM_FREQ.match(currentFieldName)) {
                     minTermFreq =parser.intValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, Field.MIN_DOC_FREQ)) {
+                } else if (Field.MIN_DOC_FREQ.match(currentFieldName)) {
                     minDocFreq = parser.intValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, Field.MAX_DOC_FREQ)) {
+                } else if (Field.MAX_DOC_FREQ.match(currentFieldName)) {
                     maxDocFreq = parser.intValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, Field.MIN_WORD_LENGTH)) {
+                } else if (Field.MIN_WORD_LENGTH.match(currentFieldName)) {
                     minWordLength = parser.intValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, Field.MAX_WORD_LENGTH)) {
+                } else if (Field.MAX_WORD_LENGTH.match(currentFieldName)) {
                     maxWordLength = parser.intValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, Field.ANALYZER)) {
+                } else if (Field.ANALYZER.match(currentFieldName)) {
                     analyzer = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, Field.MINIMUM_SHOULD_MATCH)) {
+                } else if (Field.MINIMUM_SHOULD_MATCH.match(currentFieldName)) {
                     minimumShouldMatch = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, Field.BOOST_TERMS)) {
+                } else if (Field.BOOST_TERMS.match(currentFieldName)) {
                     boostTerms = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, Field.INCLUDE)) {
+                } else if (Field.INCLUDE.match(currentFieldName)) {
                     include = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, Field.FAIL_ON_UNSUPPORTED_FIELD)) {
+                } else if (Field.FAIL_ON_UNSUPPORTED_FIELD.match(currentFieldName)) {
                     failOnUnsupportedField = parser.booleanValue();
                 } else if ("boost".equals(currentFieldName)) {
                     boost = parser.floatValue();
@@ -876,34 +883,34 @@ public static Optional<MoreLikeThisQueryBuilder> fromXContent(QueryParseContext
                     throw new ParsingException(parser.getTokenLocation(), "[mlt] query does not support [" + currentFieldName + "]");
                 }
             } else if (token == XContentParser.Token.START_ARRAY) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, Field.FIELDS)) {
+                if (Field.FIELDS.match(currentFieldName)) {
                     fields = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         fields.add(parser.text());
                     }
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, Field.LIKE)) {
+                } else if (Field.LIKE.match(currentFieldName)) {
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         parseLikeField(parseContext, likeTexts, likeItems);
                     }
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, Field.UNLIKE)) {
+                } else if (Field.UNLIKE.match(currentFieldName)) {
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         parseLikeField(parseContext, unlikeTexts, unlikeItems);
                     }
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, Field.IDS)) {
+                } else if (Field.IDS.match(currentFieldName)) {
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         if (!token.isValue()) {
                             throw new IllegalArgumentException("ids array element should only contain ids");
                         }
                         likeItems.add(new Item(null, null, parser.text()));
                     }
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, Field.DOCS)) {
+                } else if (Field.DOCS.match(currentFieldName)) {
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         if (token != XContentParser.Token.START_OBJECT) {
                             throw new IllegalArgumentException("docs array element should include an object");
                         }
-                        likeItems.add(Item.parse(parser, parseContext.getParseFieldMatcher(), new Item()));
+                        likeItems.add(Item.parse(parser, new Item()));
                     }
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, Field.STOP_WORDS)) {
+                } else if (Field.STOP_WORDS.match(currentFieldName)) {
                     stopWords = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         stopWords.add(parser.text());
@@ -912,9 +919,9 @@ public static Optional<MoreLikeThisQueryBuilder> fromXContent(QueryParseContext
                     throw new ParsingException(parser.getTokenLocation(), "[mlt] query does not support [" + currentFieldName + "]");
                 }
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, Field.LIKE)) {
+                if (Field.LIKE.match(currentFieldName)) {
                     parseLikeField(parseContext, likeTexts, likeItems);
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, Field.UNLIKE)) {
+                } else if (Field.UNLIKE.match(currentFieldName)) {
                     parseLikeField(parseContext, unlikeTexts, unlikeItems);
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[mlt] query does not support [" + currentFieldName + "]");
@@ -962,7 +969,7 @@ private static void parseLikeField(QueryParseContext parseContext, List<String>
         if (parser.currentToken().isValue()) {
             texts.add(parser.text());
         } else if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
-            items.add(Item.parse(parser, parseContext.getParseFieldMatcher(), new Item()));
+            items.add(Item.parse(parser, new Item()));
         } else {
             throw new IllegalArgumentException("Content of 'like' parameter should either be a string or an object");
         }
@@ -1021,7 +1028,7 @@ protected Query doToQuery(QueryShardContext context) throws IOException {
         }
 
         // set analyzer
-        Analyzer analyzerObj = context.getAnalysisService().analyzer(analyzer);
+        Analyzer analyzerObj = context.getIndexAnalyzers().get(analyzer);
         if (analyzerObj == null) {
             analyzerObj = context.getMapperService().searchAnalyzer();
         }
@@ -1081,14 +1088,14 @@ private Query handleItems(QueryShardContext context, MoreLikeThisQuery mltQuery,
         // fetching the items with multi-termvectors API
         MultiTermVectorsResponse likeItemsResponse = fetchResponse(context.getClient(), likeItems);
         // getting the Fields for liked items
-        mltQuery.setLikeText(getFieldsFor(likeItemsResponse));
+        mltQuery.setLikeFields(getFieldsFor(likeItemsResponse));
 
         // getting the Fields for unliked items
         if (unlikeItems.length > 0) {
             MultiTermVectorsResponse unlikeItemsResponse = fetchResponse(context.getClient(), unlikeItems);
             org.apache.lucene.index.Fields[] unlikeFields = getFieldsFor(unlikeItemsResponse);
             if (unlikeFields.length > 0) {
-                mltQuery.setUnlikeText(unlikeFields);
+                mltQuery.setUnlikeFields(unlikeFields);
             }
         }
 
@@ -1097,7 +1104,7 @@ private Query handleItems(QueryShardContext context, MoreLikeThisQuery mltQuery,
 
         // exclude the items from the search
         if (!include) {
-            handleExclude(boolQuery, likeItems);
+            handleExclude(boolQuery, likeItems, context);
         }
         return boolQuery.build();
     }
@@ -1150,7 +1157,12 @@ private static Fields[] getFieldsFor(MultiTermVectorsResponse responses) throws
         return likeFields.toArray(Fields.EMPTY_ARRAY);
     }
 
-    private static void handleExclude(BooleanQuery.Builder boolQuery, Item[] likeItems) {
+    private static void handleExclude(BooleanQuery.Builder boolQuery, Item[] likeItems, QueryShardContext context) {
+        MappedFieldType uidField = context.fieldMapper(UidFieldMapper.NAME);
+        if (uidField == null) {
+            // no mappings, nothing to exclude
+            return;
+        }
         // artificial docs get assigned a random id and should be disregarded
         List<BytesRef> uids = new ArrayList<>();
         for (Item item : likeItems) {
@@ -1160,7 +1172,7 @@ private static void handleExclude(BooleanQuery.Builder boolQuery, Item[] likeIte
             uids.add(createUidAsBytes(item.type(), item.id()));
         }
         if (!uids.isEmpty()) {
-            TermsQuery query = new TermsQuery(UidFieldMapper.NAME, uids.toArray(new BytesRef[uids.size()]));
+            Query query = uidField.termsQuery(uids, context);
             boolQuery.add(query, BooleanClause.Occur.MUST_NOT);
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java
index 6ffcb19ea3a9b..2729e8bcb2454 100644
--- a/core/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java
@@ -23,7 +23,6 @@
 import org.apache.lucene.search.Query;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -148,11 +147,11 @@ public ParseField parseField() {
             return parseField;
         }
 
-        public static Type parse(String value, ParseFieldMatcher parseFieldMatcher) {
+        public static Type parse(String value) {
             MultiMatchQueryBuilder.Type[] values = MultiMatchQueryBuilder.Type.values();
             Type type = null;
             for (MultiMatchQueryBuilder.Type t : values) {
-                if (parseFieldMatcher.match(value, t.parseField())) {
+                if (t.parseField().match(value)) {
                     type = t;
                     break;
                 }
@@ -304,7 +303,7 @@ public MultiMatchQueryBuilder type(Object type) {
         if (type == null) {
             throw new IllegalArgumentException("[" + NAME + "] requires type to be non-null");
         }
-        this.type = Type.parse(type.toString().toLowerCase(Locale.ROOT), ParseFieldMatcher.EMPTY);
+        this.type = Type.parse(type.toString().toLowerCase(Locale.ROOT));
         return this;
     }
 
@@ -584,7 +583,7 @@ public static Optional<MultiMatchQueryBuilder> fromXContent(QueryParseContext pa
         while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
-            } else if (parseContext.getParseFieldMatcher().match(currentFieldName, FIELDS_FIELD)) {
+            } else if (FIELDS_FIELD.match(currentFieldName)) {
                 if (token == XContentParser.Token.START_ARRAY) {
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         parseFieldAndBoost(parser, fieldsBoosts);
@@ -596,37 +595,37 @@ public static Optional<MultiMatchQueryBuilder> fromXContent(QueryParseContext pa
                             "[" + NAME + "] query does not support [" + currentFieldName + "]");
                 }
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, QUERY_FIELD)) {
+                if (QUERY_FIELD.match(currentFieldName)) {
                     value = parser.objectText();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, TYPE_FIELD)) {
-                    type = MultiMatchQueryBuilder.Type.parse(parser.text(), parseContext.getParseFieldMatcher());
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, ANALYZER_FIELD)) {
+                } else if (TYPE_FIELD.match(currentFieldName)) {
+                    type = MultiMatchQueryBuilder.Type.parse(parser.text());
+                } else if (ANALYZER_FIELD.match(currentFieldName)) {
                     analyzer = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, SLOP_FIELD)) {
+                } else if (SLOP_FIELD.match(currentFieldName)) {
                     slop = parser.intValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, Fuzziness.FIELD)) {
+                } else if (Fuzziness.FIELD.match(currentFieldName)) {
                     fuzziness = Fuzziness.parse(parser);
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, PREFIX_LENGTH_FIELD)) {
+                } else if (PREFIX_LENGTH_FIELD.match(currentFieldName)) {
                     prefixLength = parser.intValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, MAX_EXPANSIONS_FIELD)) {
+                } else if (MAX_EXPANSIONS_FIELD.match(currentFieldName)) {
                     maxExpansions = parser.intValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, OPERATOR_FIELD)) {
+                } else if (OPERATOR_FIELD.match(currentFieldName)) {
                     operator = Operator.fromString(parser.text());
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, MINIMUM_SHOULD_MATCH_FIELD)) {
+                } else if (MINIMUM_SHOULD_MATCH_FIELD.match(currentFieldName)) {
                     minimumShouldMatch = parser.textOrNull();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, FUZZY_REWRITE_FIELD)) {
+                } else if (FUZZY_REWRITE_FIELD.match(currentFieldName)) {
                     fuzzyRewrite = parser.textOrNull();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, USE_DIS_MAX_FIELD)) {
+                } else if (USE_DIS_MAX_FIELD.match(currentFieldName)) {
                     useDisMax = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, TIE_BREAKER_FIELD)) {
+                } else if (TIE_BREAKER_FIELD.match(currentFieldName)) {
                     tieBreaker = parser.floatValue();
-                }  else if (parseContext.getParseFieldMatcher().match(currentFieldName, CUTOFF_FREQUENCY_FIELD)) {
+                }  else if (CUTOFF_FREQUENCY_FIELD.match(currentFieldName)) {
                     cutoffFrequency = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, LENIENT_FIELD)) {
+                } else if (LENIENT_FIELD.match(currentFieldName)) {
                     lenient = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, ZERO_TERMS_QUERY_FIELD)) {
+                } else if (ZERO_TERMS_QUERY_FIELD.match(currentFieldName)) {
                     String zeroTermsDocs = parser.text();
                     if ("none".equalsIgnoreCase(zeroTermsDocs)) {
                         zeroTermsQuery = MatchQuery.ZeroTermsQuery.NONE;
@@ -635,7 +634,7 @@ public static Optional<MultiMatchQueryBuilder> fromXContent(QueryParseContext pa
                     } else {
                         throw new ParsingException(parser.getTokenLocation(), "Unsupported zero_terms_docs value [" + zeroTermsDocs + "]");
                     }
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
@@ -708,7 +707,7 @@ public String getWriteableName() {
     protected Query doToQuery(QueryShardContext context) throws IOException {
         MultiMatchQuery multiMatchQuery = new MultiMatchQuery(context);
         if (analyzer != null) {
-            if (context.getAnalysisService().analyzer(analyzer) == null) {
+            if (context.getIndexAnalyzers().get(analyzer) == null) {
                 throw new QueryShardException(context, "[" + NAME + "] analyzer [" + analyzer + "] not found");
             }
             multiMatchQuery.setAnalyzer(analyzer);
@@ -721,7 +720,7 @@ protected Query doToQuery(QueryShardContext context) throws IOException {
         multiMatchQuery.setMaxExpansions(maxExpansions);
         multiMatchQuery.setOccur(operator.toBooleanClauseOccur());
         if (fuzzyRewrite != null) {
-            multiMatchQuery.setFuzzyRewriteMethod(QueryParsers.parseRewriteMethod(context.getParseFieldMatcher(), fuzzyRewrite, null));
+            multiMatchQuery.setFuzzyRewriteMethod(QueryParsers.parseRewriteMethod(fuzzyRewrite, null));
         }
         if (tieBreaker != null) {
             multiMatchQuery.setTieBreaker(tieBreaker);
diff --git a/core/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java
index f3b0b7379dd8b..f91378c07233a 100644
--- a/core/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java
@@ -19,25 +19,44 @@
 
 package org.elasticsearch.index.query;
 
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.ReaderUtil;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.TopDocsCollector;
+import org.apache.lucene.search.TopFieldCollector;
+import org.apache.lucene.search.TopScoreDocCollector;
+import org.apache.lucene.search.TotalHitCountCollector;
+import org.apache.lucene.search.Weight;
 import org.apache.lucene.search.join.BitSetProducer;
+import org.apache.lucene.search.join.ParentChildrenBlockJoinQuery;
 import org.apache.lucene.search.join.ScoreMode;
-import org.apache.lucene.search.join.ToParentBlockJoinQuery;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.mapper.ObjectMapper;
+import org.elasticsearch.index.search.ESToParentBlockJoinQuery;
+import org.elasticsearch.index.search.NestedHelper;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.fetch.subphase.InnerHitsContext;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
+import java.util.HashMap;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 
+import static org.elasticsearch.search.fetch.subphase.InnerHitsContext.intersect;
+
 public class NestedQueryBuilder extends AbstractQueryBuilder<NestedQueryBuilder> {
     public static final String NAME = "nested";
     /**
@@ -85,7 +104,15 @@ protected void doWriteTo(StreamOutput out) throws IOException {
         out.writeString(path);
         out.writeVInt(scoreMode.ordinal());
         out.writeNamedWriteable(query);
-        out.writeOptionalWriteable(innerHitBuilder);
+        if (out.getVersion().before(Version.V_5_5_0)) {
+            final boolean hasInnerHit = innerHitBuilder != null;
+            out.writeBoolean(hasInnerHit);
+            if (hasInnerHit) {
+                innerHitBuilder.writeToNestedBWC(out, query, path);
+            }
+        } else {
+            out.writeOptionalWriteable(innerHitBuilder);
+        }
         out.writeBoolean(ignoreUnmapped);
     }
 
@@ -104,8 +131,8 @@ public InnerHitBuilder innerHit() {
         return innerHitBuilder;
     }
 
-    public NestedQueryBuilder innerHit(InnerHitBuilder innerHit) {
-        this.innerHitBuilder = new InnerHitBuilder(innerHit, path, query);
+    public NestedQueryBuilder innerHit(InnerHitBuilder innerHitBuilder) {
+        this.innerHitBuilder = innerHitBuilder;
         return this;
     }
 
@@ -143,7 +170,7 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep
         builder.field(PATH_FIELD.getPreferredName(), path);
         builder.field(IGNORE_UNMAPPED_FIELD.getPreferredName(), ignoreUnmapped);
         if (scoreMode != null) {
-            builder.field(SCORE_MODE_FIELD.getPreferredName(), HasChildQueryBuilder.scoreModeAsString(scoreMode));
+            builder.field(SCORE_MODE_FIELD.getPreferredName(), scoreModeAsString(scoreMode));
         }
         printBoostAndQueryName(builder);
         if (innerHitBuilder != null) {
@@ -167,44 +194,64 @@ public static Optional<NestedQueryBuilder> fromXContent(QueryParseContext parseC
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, QUERY_FIELD)) {
+                if (QUERY_FIELD.match(currentFieldName)) {
                     query = parseContext.parseInnerQueryBuilder();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, INNER_HITS_FIELD)) {
+                } else if (INNER_HITS_FIELD.match(currentFieldName)) {
                     innerHitBuilder = InnerHitBuilder.fromXContent(parseContext);
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[nested] query does not support [" + currentFieldName + "]");
                 }
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, PATH_FIELD)) {
+                if (PATH_FIELD.match(currentFieldName)) {
                     path = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, IGNORE_UNMAPPED_FIELD)) {
+                } else if (IGNORE_UNMAPPED_FIELD.match(currentFieldName)) {
                     ignoreUnmapped = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, SCORE_MODE_FIELD)) {
-                    scoreMode = HasChildQueryBuilder.parseScoreMode(parser.text());
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                } else if (SCORE_MODE_FIELD.match(currentFieldName)) {
+                    scoreMode = parseScoreMode(parser.text());
+                } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[nested] query does not support [" + currentFieldName + "]");
                 }
             }
         }
-
         if (query.isPresent() == false) {
             // if inner query is empty, bubble this up to caller so they can decide how to deal with it
             return Optional.empty();
         }
-        NestedQueryBuilder queryBuilder =  new NestedQueryBuilder(path, query.get(), scoreMode)
-                .ignoreUnmapped(ignoreUnmapped)
-                .queryName(queryName)
-                .boost(boost);
-        if (innerHitBuilder != null) {
-            queryBuilder.innerHit(innerHitBuilder);
-        }
+        NestedQueryBuilder queryBuilder =  new NestedQueryBuilder(path, query.get(), scoreMode, innerHitBuilder)
+            .ignoreUnmapped(ignoreUnmapped)
+            .queryName(queryName)
+            .boost(boost);
         return Optional.of(queryBuilder);
     }
 
+    public static ScoreMode parseScoreMode(String scoreModeString) {
+        if ("none".equals(scoreModeString)) {
+            return ScoreMode.None;
+        } else if ("min".equals(scoreModeString)) {
+            return ScoreMode.Min;
+        } else if ("max".equals(scoreModeString)) {
+            return ScoreMode.Max;
+        } else if ("avg".equals(scoreModeString)) {
+            return ScoreMode.Avg;
+        } else if ("sum".equals(scoreModeString)) {
+            return ScoreMode.Total;
+        }
+        throw new IllegalArgumentException("No score mode for child query [" + scoreModeString + "] found");
+    }
+
+    public static String scoreModeAsString(ScoreMode scoreMode) {
+        if (scoreMode == ScoreMode.Total) {
+            // Lucene uses 'total' but 'sum' is more consistent with other elasticsearch APIs
+            return "sum";
+        } else {
+            return scoreMode.name().toLowerCase(Locale.ROOT);
+        }
+    }
+
     @Override
     public final String getWriteableName() {
         return NAME;
@@ -238,38 +285,139 @@ protected Query doToQuery(QueryShardContext context) throws IOException {
             throw new IllegalStateException("[" + NAME + "] nested object under path [" + path + "] is not of nested type");
         }
         final BitSetProducer parentFilter;
-        final Query childFilter;
-        final Query innerQuery;
+        Query innerQuery;
         ObjectMapper objectMapper = context.nestedScope().getObjectMapper();
         if (objectMapper == null) {
             parentFilter = context.bitsetFilter(Queries.newNonNestedFilter());
         } else {
             parentFilter = context.bitsetFilter(objectMapper.nestedTypeFilter());
         }
-        childFilter = nestedObjectMapper.nestedTypeFilter();
+
         try {
             context.nestedScope().nextLevel(nestedObjectMapper);
             innerQuery = this.query.toQuery(context);
         } finally {
             context.nestedScope().previousLevel();
         }
-        return new ToParentBlockJoinQuery(Queries.filtered(innerQuery, childFilter), parentFilter, scoreMode);
+
+        // ToParentBlockJoinQuery requires that the inner query only matches documents
+        // in its child space
+        if (new NestedHelper(context.getMapperService()).mightMatchNonNestedDocs(innerQuery, path)) {
+            innerQuery = Queries.filtered(innerQuery, nestedObjectMapper.nestedTypeFilter());
+        }
+
+        return new ESToParentBlockJoinQuery(innerQuery, parentFilter, scoreMode,
+                objectMapper == null ? null : objectMapper.fullPath());
     }
 
     @Override
     protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException {
         QueryBuilder rewrittenQuery = query.rewrite(queryRewriteContext);
         if (rewrittenQuery != query) {
-            InnerHitBuilder rewrittenInnerHit = InnerHitBuilder.rewrite(innerHitBuilder, rewrittenQuery);
-            return new NestedQueryBuilder(path, rewrittenQuery, scoreMode, rewrittenInnerHit);
+            NestedQueryBuilder nestedQuery = new NestedQueryBuilder(path, rewrittenQuery, scoreMode, innerHitBuilder);
+            nestedQuery.ignoreUnmapped(ignoreUnmapped);
+            return nestedQuery;
         }
         return this;
     }
 
     @Override
-    protected void extractInnerHitBuilders(Map<String, InnerHitBuilder> innerHits) {
+    public void extractInnerHitBuilders(Map<String, InnerHitContextBuilder> innerHits) {
         if (innerHitBuilder != null) {
-            innerHitBuilder.inlineInnerHits(innerHits);
+            Map<String, InnerHitContextBuilder> children = new HashMap<>();
+            InnerHitContextBuilder.extractInnerHits(query, children);
+            InnerHitContextBuilder innerHitContextBuilder = new NestedInnerHitContextBuilder(path, query, innerHitBuilder, children);
+            String name = innerHitBuilder.getName() != null ? innerHitBuilder.getName() : path;
+            innerHits.put(name, innerHitContextBuilder);
+        }
+    }
+
+    static class NestedInnerHitContextBuilder extends InnerHitContextBuilder {
+        private final String path;
+
+        NestedInnerHitContextBuilder(String path, QueryBuilder query, InnerHitBuilder innerHitBuilder,
+                                     Map<String, InnerHitContextBuilder> children) {
+            super(query, innerHitBuilder, children);
+            this.path = path;
+        }
+
+        @Override
+        public void build(SearchContext parentSearchContext,
+                          InnerHitsContext innerHitsContext) throws IOException {
+            QueryShardContext queryShardContext = parentSearchContext.getQueryShardContext();
+            ObjectMapper nestedObjectMapper = queryShardContext.getObjectMapper(path);
+            if (nestedObjectMapper == null) {
+                if (innerHitBuilder.isIgnoreUnmapped() == false) {
+                    throw new IllegalStateException("[" + query.getName() + "] no mapping found for type [" + path + "]");
+                } else {
+                    return;
+                }
+            }
+            String name =  innerHitBuilder.getName() != null ? innerHitBuilder.getName() : nestedObjectMapper.fullPath();
+            ObjectMapper parentObjectMapper = queryShardContext.nestedScope().nextLevel(nestedObjectMapper);
+            NestedInnerHitSubContext nestedInnerHits = new NestedInnerHitSubContext(
+                name, parentSearchContext, parentObjectMapper, nestedObjectMapper
+            );
+            setupInnerHitsContext(queryShardContext, nestedInnerHits);
+            queryShardContext.nestedScope().previousLevel();
+            innerHitsContext.addInnerHitDefinition(nestedInnerHits);
+        }
+    }
+
+    static final class NestedInnerHitSubContext extends InnerHitsContext.InnerHitSubContext {
+
+        private final ObjectMapper parentObjectMapper;
+        private final ObjectMapper childObjectMapper;
+
+        NestedInnerHitSubContext(String name, SearchContext context, ObjectMapper parentObjectMapper, ObjectMapper childObjectMapper) {
+            super(name, context);
+            this.parentObjectMapper = parentObjectMapper;
+            this.childObjectMapper = childObjectMapper;
+        }
+
+        @Override
+        public TopDocs[] topDocs(SearchHit[] hits) throws IOException {
+            Weight innerHitQueryWeight = createInnerHitQueryWeight();
+            TopDocs[] result = new TopDocs[hits.length];
+            for (int i = 0; i < hits.length; i++) {
+                SearchHit hit = hits[i];
+                Query rawParentFilter;
+                if (parentObjectMapper == null) {
+                    rawParentFilter = Queries.newNonNestedFilter();
+                } else {
+                    rawParentFilter = parentObjectMapper.nestedTypeFilter();
+                }
+
+                int parentDocId = hit.docId();
+                final int readerIndex = ReaderUtil.subIndex(parentDocId, searcher().getIndexReader().leaves());
+                // With nested inner hits the nested docs are always in the same segement, so need to use the other segments
+                LeafReaderContext ctx = searcher().getIndexReader().leaves().get(readerIndex);
+
+                Query childFilter = childObjectMapper.nestedTypeFilter();
+                BitSetProducer parentFilter = context.bitsetFilterCache().getBitSetProducer(rawParentFilter);
+                Query q = new ParentChildrenBlockJoinQuery(parentFilter, childFilter, parentDocId);
+                Weight weight = context.searcher().createNormalizedWeight(q, false);
+                if (size() == 0) {
+                    TotalHitCountCollector totalHitCountCollector = new TotalHitCountCollector();
+                    intersect(weight, innerHitQueryWeight, totalHitCountCollector, ctx);
+                    result[i] = new TopDocs(totalHitCountCollector.getTotalHits(), Lucene.EMPTY_SCORE_DOCS, 0);
+                } else {
+                    int topN = Math.min(from() + size(), context.searcher().getIndexReader().maxDoc());
+                    TopDocsCollector<?> topDocsCollector;
+                    if (sort() != null) {
+                        topDocsCollector = TopFieldCollector.create(sort().sort, topN, true, trackScores(), trackScores());
+                    } else {
+                        topDocsCollector = TopScoreDocCollector.create(topN);
+                    }
+                    try {
+                        intersect(weight, innerHitQueryWeight, topDocsCollector, ctx);
+                    } finally {
+                        clearReleasables(Lifetime.COLLECTION);
+                    }
+                    result[i] = topDocsCollector.topDocs(from(), size());
+                }
+            }
+            return result;
         }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/query/Operator.java b/core/src/main/java/org/elasticsearch/index/query/Operator.java
index 7972dbb49ad81..de88abebad359 100644
--- a/core/src/main/java/org/elasticsearch/index/query/Operator.java
+++ b/core/src/main/java/org/elasticsearch/index/query/Operator.java
@@ -54,16 +54,12 @@ public QueryParser.Operator toQueryParserOperator() {
     }
 
     public static Operator readFromStream(StreamInput in) throws IOException {
-        int ordinal = in.readVInt();
-        if (ordinal < 0 || ordinal >= values().length) {
-            throw new IOException("Unknown Operator ordinal [" + ordinal + "]");
-        }
-        return values()[ordinal];
+        return in.readEnum(Operator.class);
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        out.writeVInt(this.ordinal());
+        out.writeEnum(this);
     }
 
     public static Operator fromString(String op) {
diff --git a/core/src/main/java/org/elasticsearch/index/query/ParentIdQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/ParentIdQueryBuilder.java
deleted file mode 100644
index 1b1a9508bc47d..0000000000000
--- a/core/src/main/java/org/elasticsearch/index/query/ParentIdQueryBuilder.java
+++ /dev/null
@@ -1,196 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.query;
-
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.DocValuesTermsQuery;
-import org.apache.lucene.search.MatchNoDocsQuery;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TermQuery;
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.ParentFieldMapper;
-import org.elasticsearch.index.mapper.TypeFieldMapper;
-
-import java.io.IOException;
-import java.util.Objects;
-import java.util.Optional;
-
-public final class ParentIdQueryBuilder extends AbstractQueryBuilder<ParentIdQueryBuilder> {
-    public static final String NAME = "parent_id";
-
-    /**
-     * The default value for ignore_unmapped.
-     */
-    public static final boolean DEFAULT_IGNORE_UNMAPPED = false;
-
-    private static final ParseField ID_FIELD = new ParseField("id");
-    private static final ParseField TYPE_FIELD = new ParseField("type", "child_type");
-    private static final ParseField IGNORE_UNMAPPED_FIELD = new ParseField("ignore_unmapped");
-
-    private final String type;
-    private final String id;
-
-    private boolean ignoreUnmapped = false;
-
-    public ParentIdQueryBuilder(String type, String id) {
-        this.type = type;
-        this.id = id;
-    }
-
-    /**
-     * Read from a stream.
-     */
-    public ParentIdQueryBuilder(StreamInput in) throws IOException {
-        super(in);
-        type = in.readString();
-        id = in.readString();
-        ignoreUnmapped = in.readBoolean();
-    }
-
-    @Override
-    protected void doWriteTo(StreamOutput out) throws IOException {
-        out.writeString(type);
-        out.writeString(id);
-        out.writeBoolean(ignoreUnmapped);
-    }
-
-    public String getType() {
-        return type;
-    }
-
-    public String getId() {
-        return id;
-    }
-
-    /**
-     * Sets whether the query builder should ignore unmapped types (and run a
-     * {@link MatchNoDocsQuery} in place of this query) or throw an exception if
-     * the type is unmapped.
-     */
-    public ParentIdQueryBuilder ignoreUnmapped(boolean ignoreUnmapped) {
-        this.ignoreUnmapped = ignoreUnmapped;
-        return this;
-    }
-
-    /**
-     * Gets whether the query builder will ignore unmapped types (and run a
-     * {@link MatchNoDocsQuery} in place of this query) or throw an exception if
-     * the type is unmapped.
-     */
-    public boolean ignoreUnmapped() {
-        return ignoreUnmapped;
-    }
-
-    @Override
-    protected void doXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject(NAME);
-        builder.field(TYPE_FIELD.getPreferredName(), type);
-        builder.field(ID_FIELD.getPreferredName(), id);
-        builder.field(IGNORE_UNMAPPED_FIELD.getPreferredName(), ignoreUnmapped);
-        printBoostAndQueryName(builder);
-        builder.endObject();
-    }
-
-    public static Optional<ParentIdQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
-        XContentParser parser = parseContext.parser();
-        float boost = AbstractQueryBuilder.DEFAULT_BOOST;
-        String type = null;
-        String id = null;
-        String queryName = null;
-        String currentFieldName = null;
-        boolean ignoreUnmapped = DEFAULT_IGNORE_UNMAPPED;
-        XContentParser.Token token;
-        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-            if (token == XContentParser.Token.FIELD_NAME) {
-                currentFieldName = parser.currentName();
-            } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, TYPE_FIELD)) {
-                    type = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, ID_FIELD)) {
-                    id = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, IGNORE_UNMAPPED_FIELD)) {
-                    ignoreUnmapped = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
-                    boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
-                    queryName = parser.text();
-                } else {
-                    throw new ParsingException(parser.getTokenLocation(), "[parent_id] query does not support [" + currentFieldName + "]");
-                }
-            } else {
-                throw new ParsingException(parser.getTokenLocation(), "[parent_id] query does not support [" + currentFieldName + "]");
-            }
-        }
-        ParentIdQueryBuilder queryBuilder = new ParentIdQueryBuilder(type, id);
-        queryBuilder.queryName(queryName);
-        queryBuilder.boost(boost);
-        queryBuilder.ignoreUnmapped(ignoreUnmapped);
-        return Optional.of(queryBuilder);
-    }
-
-
-    @Override
-    protected Query doToQuery(QueryShardContext context) throws IOException {
-        DocumentMapper childDocMapper = context.getMapperService().documentMapper(type);
-        if (childDocMapper == null) {
-            if (ignoreUnmapped) {
-                return new MatchNoDocsQuery();
-            } else {
-                throw new QueryShardException(context, "[" + NAME + "] no mapping found for type [" + type + "]");
-            }
-        }
-        ParentFieldMapper parentFieldMapper = childDocMapper.parentFieldMapper();
-        if (parentFieldMapper.active() == false) {
-            throw new QueryShardException(context, "[" + NAME + "] _parent field has no parent type configured");
-        }
-        String fieldName = ParentFieldMapper.joinField(parentFieldMapper.type());
-
-        BooleanQuery.Builder query = new BooleanQuery.Builder();
-        query.add(new DocValuesTermsQuery(fieldName, id), BooleanClause.Occur.MUST);
-        // Need to take child type into account, otherwise a child doc of different type with the same id could match
-        query.add(new TermQuery(new Term(TypeFieldMapper.NAME, type)), BooleanClause.Occur.FILTER);
-        return query.build();
-    }
-
-    @Override
-    protected boolean doEquals(ParentIdQueryBuilder that) {
-        return Objects.equals(type, that.type)
-                && Objects.equals(id, that.id)
-                && Objects.equals(ignoreUnmapped, that.ignoreUnmapped);
-    }
-
-    @Override
-    protected int doHashCode() {
-        return Objects.hash(type, id, ignoreUnmapped);
-    }
-
-    @Override
-    public String getWriteableName() {
-        return NAME;
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/index/query/PrefixQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/PrefixQueryBuilder.java
index 13cd9eca9d31c..7d76c1ebd7630 100644
--- a/core/src/main/java/org/elasticsearch/index/query/PrefixQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/PrefixQueryBuilder.java
@@ -140,13 +140,13 @@ public static Optional<PrefixQueryBuilder> fromXContent(QueryParseContext parseC
                     if (token == XContentParser.Token.FIELD_NAME) {
                         currentFieldName = parser.currentName();
                     } else {
-                        if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                        if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                             queryName = parser.text();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, PREFIX_FIELD)) {
+                        } else if (PREFIX_FIELD.match(currentFieldName)) {
                             value = parser.textOrNull();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                        } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                             boost = parser.floatValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, REWRITE_FIELD)) {
+                        } else if (REWRITE_FIELD.match(currentFieldName)) {
                             rewrite = parser.textOrNull();
                         } else {
                             throw new ParsingException(parser.getTokenLocation(),
@@ -174,7 +174,7 @@ public String getWriteableName() {
 
     @Override
     protected Query doToQuery(QueryShardContext context) throws IOException {
-        MultiTermQuery.RewriteMethod method = QueryParsers.parseRewriteMethod(context.getParseFieldMatcher(), rewrite, null);
+        MultiTermQuery.RewriteMethod method = QueryParsers.parseRewriteMethod(rewrite, null);
 
         Query query = null;
         MappedFieldType fieldType = context.fieldMapper(fieldName);
diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/QueryBuilder.java
index 197af655d54a2..7c6b332f4aab8 100644
--- a/core/src/main/java/org/elasticsearch/index/query/QueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/QueryBuilder.java
@@ -21,11 +21,11 @@
 
 import org.apache.lucene.search.Query;
 import org.elasticsearch.common.io.stream.NamedWriteable;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 
 import java.io.IOException;
 
-public interface QueryBuilder extends NamedWriteable, ToXContent {
+public interface QueryBuilder extends NamedWriteable, ToXContentObject {
 
     /**
      * Converts this QueryBuilder to a lucene {@link Query}.
diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryBuilders.java b/core/src/main/java/org/elasticsearch/index/query/QueryBuilders.java
index d63696ec4fbaa..e7eacfb995c52 100644
--- a/core/src/main/java/org/elasticsearch/index/query/QueryBuilders.java
+++ b/core/src/main/java/org/elasticsearch/index/query/QueryBuilders.java
@@ -29,12 +29,10 @@
 import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder;
 import org.elasticsearch.indices.TermsLookup;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService;
 
 import java.io.IOException;
 import java.util.Collection;
 import java.util.List;
-import java.util.Map;
 
 /**
  * A static factory for simple "import static" usage.
@@ -120,7 +118,7 @@ public static IdsQueryBuilder idsQuery() {
      * @param types The mapping/doc type
      */
     public static IdsQueryBuilder idsQuery(String... types) {
-        return new IdsQueryBuilder(types);
+        return new IdsQueryBuilder().types(types);
     }
 
     /**
@@ -199,13 +197,9 @@ public static TermQueryBuilder termQuery(String name, Object value) {
      * @param name  The name of the field
      * @param value The value of the term
      *
-     * @deprecated Fuzzy queries are not useful enough and will be removed with Elasticsearch 4.0. In most cases you may want to use
-     * a match query with the fuzziness parameter for strings or range queries for numeric and date fields.
-     *
      * @see #matchQuery(String, Object)
      * @see #rangeQuery(String)
      */
-    @Deprecated
     public static FuzzyQueryBuilder fuzzyQuery(String name, String value) {
         return new FuzzyQueryBuilder(name, value);
     }
@@ -216,13 +210,9 @@ public static FuzzyQueryBuilder fuzzyQuery(String name, String value) {
      * @param name  The name of the field
      * @param value The value of the term
      *
-     * @deprecated Fuzzy queries are not useful enough and will be removed with Elasticsearch 4.0. In most cases you may want to use
-     * a match query with the fuzziness parameter for strings or range queries for numeric and date fields.
-     *
      * @see #matchQuery(String, Object)
      * @see #rangeQuery(String)
      */
-    @Deprecated
     public static FuzzyQueryBuilder fuzzyQuery(String name, Object value) {
         return new FuzzyQueryBuilder(name, value);
     }
@@ -481,38 +471,6 @@ public static MoreLikeThisQueryBuilder moreLikeThisQuery(Item[] likeItems) {
         return moreLikeThisQuery(null, null, likeItems);
     }
 
-    /**
-     * Constructs a new has_child query, with the child type and the query to run on the child documents. The
-     * results of this query are the parent docs that those child docs matched.
-     *
-     * @param type      The child type.
-     * @param query     The query.
-     * @param scoreMode How the scores from the children hits should be aggregated into the parent hit.
-     */
-    public static HasChildQueryBuilder hasChildQuery(String type, QueryBuilder query, ScoreMode scoreMode) {
-        return new HasChildQueryBuilder(type, query, scoreMode);
-    }
-
-    /**
-     * Constructs a new parent query, with the parent type and the query to run on the parent documents. The
-     * results of this query are the children docs that those parent docs matched.
-     *
-     * @param type      The parent type.
-     * @param query     The query.
-     * @param score     Whether the score from the parent hit should propogate to the child hit
-     */
-    public static HasParentQueryBuilder hasParentQuery(String type, QueryBuilder query, boolean score) {
-        return new HasParentQueryBuilder(type, query, score);
-    }
-
-    /**
-     * Constructs a new parent id query that returns all child documents of the specified type that
-     * point to the specified id.
-     */
-    public static ParentIdQueryBuilder parentId(String type, String id) {
-        return new ParentIdQueryBuilder(type, id);
-    }
-
     public static NestedQueryBuilder nestedQuery(String path, QueryBuilder query, ScoreMode scoreMode) {
         return new NestedQueryBuilder(path, query, scoreMode);
     }
diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryParseContext.java b/core/src/main/java/org/elasticsearch/index/query/QueryParseContext.java
index 9ed374db212a8..6dde6ec3b9202 100644
--- a/core/src/main/java/org/elasticsearch/index/query/QueryParseContext.java
+++ b/core/src/main/java/org/elasticsearch/index/query/QueryParseContext.java
@@ -20,22 +20,19 @@
 package org.elasticsearch.index.query;
 
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.logging.Loggers;
-import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry.UnknownNamedObjectException;
+import org.elasticsearch.common.xcontent.XContentLocation;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptSettings;
 
 import java.io.IOException;
 import java.util.Objects;
 import java.util.Optional;
 
-public class QueryParseContext implements ParseFieldMatcherSupplier {
+public class QueryParseContext {
 
     private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(QueryParseContext.class));
 
@@ -43,19 +40,15 @@ public class QueryParseContext implements ParseFieldMatcherSupplier {
     private static final ParseField CACHE_KEY = new ParseField("_cache_key").withAllDeprecated("Filters are always used as cache keys");
 
     private final XContentParser parser;
-    private final IndicesQueriesRegistry indicesQueriesRegistry;
-    private final ParseFieldMatcher parseFieldMatcher;
     private final String defaultScriptLanguage;
 
-    public QueryParseContext(IndicesQueriesRegistry registry, XContentParser parser, ParseFieldMatcher parseFieldMatcher) {
-        this(Script.DEFAULT_SCRIPT_LANG, registry, parser, parseFieldMatcher);
+    public QueryParseContext(XContentParser parser) {
+        this(Script.DEFAULT_SCRIPT_LANG, parser);
     }
 
-    public QueryParseContext(String defaultScriptLanguage, IndicesQueriesRegistry registry, XContentParser parser,
-                             ParseFieldMatcher parseFieldMatcher) {
-        this.indicesQueriesRegistry = Objects.requireNonNull(registry, "indices queries registry cannot be null");
+    //TODO this constructor can be removed from master branch
+    public QueryParseContext(String defaultScriptLanguage, XContentParser parser) {
         this.parser = Objects.requireNonNull(parser, "parser cannot be null");
-        this.parseFieldMatcher = Objects.requireNonNull(parseFieldMatcher, "parse field matcher cannot be null");
         this.defaultScriptLanguage = defaultScriptLanguage;
     }
 
@@ -64,7 +57,7 @@ public XContentParser parser() {
     }
 
     public boolean isDeprecatedSetting(String setting) {
-        return this.parseFieldMatcher.match(setting, CACHE) || this.parseFieldMatcher.match(setting, CACHE_KEY);
+        return CACHE.match(setting) || CACHE_KEY.match(setting);
     }
 
     /**
@@ -73,6 +66,15 @@ public boolean isDeprecatedSetting(String setting) {
     public QueryBuilder parseTopLevelQueryBuilder() {
         try {
             QueryBuilder queryBuilder = null;
+            XContentParser.Token first = parser.nextToken();
+            if (first == null) {
+                return null;
+            } else if (first != XContentParser.Token.START_OBJECT) {
+                throw new ParsingException(
+                    parser.getTokenLocation(), "Expected [" + XContentParser.Token.START_OBJECT +
+                    "] but found [" + first + "]", parser.getTokenLocation()
+                );
+            }
             for (XContentParser.Token token = parser.nextToken(); token != XContentParser.Token.END_OBJECT; token = parser.nextToken()) {
                 if (token == XContentParser.Token.FIELD_NAME) {
                     String fieldName = parser.currentName();
@@ -95,49 +97,49 @@ public QueryBuilder parseTopLevelQueryBuilder() {
      * Parses a query excluding the query element that wraps it
      */
     public Optional<QueryBuilder> parseInnerQueryBuilder() throws IOException {
-        // move to START object
-        XContentParser.Token token;
         if (parser.currentToken() != XContentParser.Token.START_OBJECT) {
-            token = parser.nextToken();
-            if (token != XContentParser.Token.START_OBJECT) {
+            if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
                 throw new ParsingException(parser.getTokenLocation(), "[_na] query malformed, must start with start_object");
             }
         }
-        token = parser.nextToken();
-        if (token == XContentParser.Token.END_OBJECT) {
+        if (parser.nextToken() == XContentParser.Token.END_OBJECT) {
             // we encountered '{}' for a query clause
             String msg = "query malformed, empty clause found at [" + parser.getTokenLocation() +"]";
             DEPRECATION_LOGGER.deprecated(msg);
-            if (parseFieldMatcher.isStrict()) {
-                throw new IllegalArgumentException(msg);
-            }
             return Optional.empty();
         }
-        if (token != XContentParser.Token.FIELD_NAME) {
+        if (parser.currentToken() != XContentParser.Token.FIELD_NAME) {
             throw new ParsingException(parser.getTokenLocation(), "[_na] query malformed, no field after start_object");
         }
         String queryName = parser.currentName();
         // move to the next START_OBJECT
-        token = parser.nextToken();
-        if (token != XContentParser.Token.START_OBJECT) {
+        if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
             throw new ParsingException(parser.getTokenLocation(), "[" + queryName + "] query malformed, no start_object after query name");
         }
-        @SuppressWarnings("unchecked")
-        Optional<QueryBuilder> result = (Optional<QueryBuilder>) indicesQueriesRegistry.lookup(queryName, parseFieldMatcher,
-                parser.getTokenLocation()).fromXContent(this);
+        Optional<QueryBuilder> result;
+        try {
+            @SuppressWarnings("unchecked")
+            Optional<QueryBuilder> resultCast = (Optional<QueryBuilder>) parser.namedObject(Optional.class, queryName, this);
+            result = resultCast;
+        } catch (UnknownNamedObjectException e) {
+            // Preserve the error message from 5.0 until we have a compellingly better message so we don't break BWC.
+            // This intentionally doesn't include the causing exception because that'd change the "root_cause" of any unknown query errors
+            throw new ParsingException(new XContentLocation(e.getLineNumber(), e.getColumnNumber()),
+                    "no [query] registered for [" + e.getName() + "]");
+        }
+        //end_object of the specific query (e.g. match, multi_match etc.) element
         if (parser.currentToken() != XContentParser.Token.END_OBJECT) {
             throw new ParsingException(parser.getTokenLocation(),
                     "[" + queryName + "] malformed query, expected [END_OBJECT] but found [" + parser.currentToken() + "]");
         }
-        parser.nextToken();
+        //end_object of the query object
+        if (parser.nextToken() != XContentParser.Token.END_OBJECT) {
+            throw new ParsingException(parser.getTokenLocation(),
+                    "[" + queryName + "] malformed query, expected [END_OBJECT] but found [" + parser.currentToken() + "]");
+        }
         return result;
     }
 
-    @Override
-    public ParseFieldMatcher getParseFieldMatcher() {
-        return parseFieldMatcher;
-    }
-
     /**
      * Returns the default scripting language, that should be used if scripts don't specify the script language
      * explicitly.
diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java b/core/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java
index f12605088e6c5..b0dcd9da7acc5 100644
--- a/core/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java
+++ b/core/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java
@@ -20,44 +20,49 @@
 
 import org.apache.lucene.index.IndexReader;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
-import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
+import org.elasticsearch.script.CompiledScript;
+import org.elasticsearch.script.ExecutableScript;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.ScriptSettings;
+import org.elasticsearch.template.CompiledTemplate;
+
+import java.util.function.LongSupplier;
 
 /**
  * Context object used to rewrite {@link QueryBuilder} instances into simplified version.
  */
-public class QueryRewriteContext implements ParseFieldMatcherSupplier {
+public class QueryRewriteContext {
     protected final MapperService mapperService;
     protected final ScriptService scriptService;
     protected final IndexSettings indexSettings;
-    protected final IndicesQueriesRegistry indicesQueriesRegistry;
+    private final NamedXContentRegistry xContentRegistry;
     protected final Client client;
     protected final IndexReader reader;
-    protected final ClusterState clusterState;
+    protected final LongSupplier nowInMillis;
 
     public QueryRewriteContext(IndexSettings indexSettings, MapperService mapperService, ScriptService scriptService,
-                               IndicesQueriesRegistry indicesQueriesRegistry, Client client, IndexReader reader,
-                               ClusterState clusterState) {
+            NamedXContentRegistry xContentRegistry, Client client, IndexReader reader,
+            LongSupplier nowInMillis) {
         this.mapperService = mapperService;
         this.scriptService = scriptService;
         this.indexSettings = indexSettings;
-        this.indicesQueriesRegistry = indicesQueriesRegistry;
+        this.xContentRegistry = xContentRegistry;
         this.client = client;
         this.reader = reader;
-        this.clusterState = clusterState;
+        this.nowInMillis = nowInMillis;
     }
 
     /**
      * Returns a clients to fetch resources from local or remove nodes.
      */
-    public final Client getClient() {
+    public Client getClient() {
         return client;
     }
 
@@ -65,47 +70,36 @@ public final Client getClient() {
      * Returns the index settings for this context. This might return null if the
      * context has not index scope.
      */
-    public final IndexSettings getIndexSettings() {
+    public IndexSettings getIndexSettings() {
         return indexSettings;
     }
 
-    /**
-     * Returns a script service to fetch scripts.
-     */
-    public final ScriptService getScriptService() {
-        return scriptService;
-    }
-
     /**
      * Return the MapperService.
      */
-    public final MapperService getMapperService() {
+    public MapperService getMapperService() {
         return mapperService;
     }
 
-    /** Return the current {@link IndexReader}, or {@code null} if we are on the coordinating node. */
+    /** Return the current {@link IndexReader}, or {@code null} if no index reader is available, for
+     *  instance if we are on the coordinating node or if this rewrite context is used to index
+     *  queries (percolation). */
     public IndexReader getIndexReader() {
         return reader;
     }
 
-    @Override
-    public ParseFieldMatcher getParseFieldMatcher() {
-        return this.indexSettings.getParseFieldMatcher();
-    }
-
     /**
-     * Returns the cluster state as is when the operation started.
+     * The registry used to build new {@link XContentParser}s. Contains registered named parsers needed to parse the query.
      */
-    public ClusterState getClusterState() {
-        return clusterState;
+    public NamedXContentRegistry getXContentRegistry() {
+        return xContentRegistry;
     }
 
     /**
-     * Returns a new {@link QueryParseContext} that wraps the provided parser, using the ParseFieldMatcher settings that
-     * are configured in the index settings. The default script language will always default to Painless.
+     * Returns a new {@link QueryParseContext} that wraps the provided parser.
      */
     public QueryParseContext newParseContext(XContentParser parser) {
-        return new QueryParseContext(indicesQueriesRegistry, parser, indexSettings.getParseFieldMatcher());
+        return new QueryParseContext(parser);
     }
 
     /**
@@ -114,6 +108,15 @@ public QueryParseContext newParseContext(XContentParser parser) {
      */
     public QueryParseContext newParseContextWithLegacyScriptLanguage(XContentParser parser) {
         String defaultScriptLanguage = ScriptSettings.getLegacyDefaultLang(indexSettings.getNodeSettings());
-        return new QueryParseContext(defaultScriptLanguage, indicesQueriesRegistry, parser, indexSettings.getParseFieldMatcher());
+        return new QueryParseContext(defaultScriptLanguage, parser);
+    }
+
+    public long nowInMillis() {
+        return nowInMillis.getAsLong();
+    }
+
+    public BytesReference getTemplateBytes(Script template) {
+        CompiledTemplate compiledTemplate = scriptService.compileTemplate(template, ScriptContext.Standard.SEARCH);
+        return compiledTemplate.run(template.getParams());
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java
index 78869f5374a74..26d2e7ac39864 100644
--- a/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java
+++ b/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java
@@ -26,6 +26,9 @@
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.function.Function;
+import java.util.function.LongSupplier;
+
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.queryparser.classic.MapperQueryParser;
@@ -33,19 +36,23 @@
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.join.BitSetProducer;
 import org.apache.lucene.search.similarities.Similarity;
+import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.Version;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.lucene.search.Queries;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.analysis.AnalysisService;
+import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.mapper.ContentPath;
+import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperService;
@@ -53,9 +60,12 @@
 import org.elasticsearch.index.mapper.TextFieldMapper;
 import org.elasticsearch.index.query.support.NestedScope;
 import org.elasticsearch.index.similarity.SimilarityService;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
+import org.elasticsearch.script.CompiledScript;
+import org.elasticsearch.script.ExecutableScript;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.lookup.SearchLookup;
 
 /**
@@ -68,7 +78,10 @@ public class QueryShardContext extends QueryRewriteContext {
     private final BitsetFilterCache bitsetFilterCache;
     private final IndexFieldDataService indexFieldDataService;
     private final IndexSettings indexSettings;
+    private final int shardId;
     private String[] types = Strings.EMPTY_ARRAY;
+    private boolean cachable = true;
+    private final SetOnce<Boolean> frozen = new SetOnce<>();
 
     public void setTypes(String... types) {
         this.types = types;
@@ -80,31 +93,31 @@ public String[] getTypes() {
 
     private final Map<String, Query> namedQueries = new HashMap<>();
     private final MapperQueryParser queryParser = new MapperQueryParser(this);
-    private final IndicesQueriesRegistry indicesQueriesRegistry;
     private boolean allowUnmappedFields;
     private boolean mapUnmappedFieldAsString;
     private NestedScope nestedScope;
     private boolean isFilter;
 
-    public QueryShardContext(IndexSettings indexSettings, BitsetFilterCache bitsetFilterCache, IndexFieldDataService indexFieldDataService,
-                             MapperService mapperService, SimilarityService similarityService, ScriptService scriptService,
-                             final IndicesQueriesRegistry indicesQueriesRegistry, Client client,
-                             IndexReader reader, ClusterState clusterState) {
-        super(indexSettings, mapperService, scriptService, indicesQueriesRegistry, client, reader, clusterState);
+    public QueryShardContext(int shardId, IndexSettings indexSettings, BitsetFilterCache bitsetFilterCache,
+            IndexFieldDataService indexFieldDataService, MapperService mapperService, SimilarityService similarityService,
+            ScriptService scriptService, NamedXContentRegistry xContentRegistry,
+            Client client, IndexReader reader, LongSupplier nowInMillis) {
+        super(indexSettings, mapperService, scriptService, xContentRegistry, client, reader, nowInMillis);
+        this.shardId = shardId;
         this.indexSettings = indexSettings;
         this.similarityService = similarityService;
         this.mapperService = mapperService;
         this.bitsetFilterCache = bitsetFilterCache;
         this.indexFieldDataService = indexFieldDataService;
         this.allowUnmappedFields = indexSettings.isDefaultAllowUnmappedFields();
-        this.indicesQueriesRegistry = indicesQueriesRegistry;
         this.nestedScope = new NestedScope();
+
     }
 
     public QueryShardContext(QueryShardContext source) {
-        this(source.indexSettings, source.bitsetFilterCache, source.indexFieldDataService, source.mapperService,
-                source.similarityService, source.scriptService, source.indicesQueriesRegistry, source.client,
-                source.reader, source.clusterState);
+        this(source.shardId, source.indexSettings, source.bitsetFilterCache, source.indexFieldDataService, source.mapperService,
+                source.similarityService, source.scriptService, source.getXContentRegistry(), source.client,
+                source.reader, source.nowInMillis);
         this.types = source.getTypes();
     }
 
@@ -116,8 +129,8 @@ private void reset() {
         this.isFilter = false;
     }
 
-    public AnalysisService getAnalysisService() {
-        return mapperService.analysisService();
+    public IndexAnalyzers getIndexAnalyzers() {
+        return mapperService.getIndexAnalyzers();
     }
 
     public Similarity getSearchSimilarity() {
@@ -180,6 +193,10 @@ public void setIsFilter(boolean isFilter) {
         this.isFilter = isFilter;
     }
 
+    /**
+     * Returns all the fields that match a given pattern. If prefixed with a
+     * type then the fields will be returned with a type prefix.
+     */
     public Collection<String> simpleMatchToIndexNames(String pattern) {
         return mapperService.simpleMatchToIndexNames(pattern);
     }
@@ -192,6 +209,14 @@ public ObjectMapper getObjectMapper(String name) {
         return mapperService.getObjectMapper(name);
     }
 
+    /**
+     * Returns s {@link DocumentMapper} instance for the given type.
+     * Delegates to {@link MapperService#documentMapper(String)}
+     */
+    public DocumentMapper documentMapper(String type) {
+        return mapperService.documentMapper(type);
+    }
+
     /**
      * Gets the search analyzer for the given field, or the default if there is none present for the field
      * TODO: remove this by moving defaults into mappers themselves
@@ -250,24 +275,12 @@ public Collection<String> queryTypes() {
     private SearchLookup lookup = null;
 
     public SearchLookup lookup() {
-        SearchContext current = SearchContext.current();
-        if (current != null) {
-            return current.lookup();
-        }
         if (lookup == null) {
-            lookup = new SearchLookup(getMapperService(), indexFieldDataService, null);
+            lookup = new SearchLookup(getMapperService(), indexFieldDataService, types);
         }
         return lookup;
     }
 
-    public long nowInMillis() {
-        SearchContext current = SearchContext.current();
-        if (current != null) {
-            return current.nowInMillis();
-        }
-        return System.currentTimeMillis();
-    }
-
     public NestedScope nestedScope() {
         return nestedScope;
     }
@@ -305,12 +318,7 @@ public ParsedQuery toQuery(QueryBuilder queryBuilder) {
         });
     }
 
-    @FunctionalInterface
-    private interface CheckedFunction<T, R> {
-       R apply(T t) throws IOException;
-    }
-
-    private ParsedQuery toQuery(QueryBuilder queryBuilder, CheckedFunction<QueryBuilder, Query> filterOrQuery) {
+    private ParsedQuery toQuery(QueryBuilder queryBuilder, CheckedFunction<QueryBuilder, Query, IOException> filterOrQuery) {
         reset();
         try {
             QueryBuilder rewriteQuery = QueryBuilder.rewriteQuery(queryBuilder, this);
@@ -327,4 +335,100 @@ private ParsedQuery toQuery(QueryBuilder queryBuilder, CheckedFunction<QueryBuil
     public final Index index() {
         return indexSettings.getIndex();
     }
+
+    /**
+     * Compiles (or retrieves from cache) and binds the parameters to the
+     * provided script
+     */
+    public final SearchScript getSearchScript(Script script, ScriptContext context) {
+        failIfFrozen();
+        return scriptService.search(lookup(), script, context);
+    }
+    /**
+     * Returns a lazily created {@link SearchScript} that is compiled immediately but can be pulled later once all
+     * parameters are available.
+     */
+    public final Function<Map<String, Object>, SearchScript> getLazySearchScript(Script script, ScriptContext context) {
+        failIfFrozen();
+        CompiledScript compile = scriptService.compile(script, context);
+        return (p) -> scriptService.search(lookup(), compile, p);
+    }
+
+    /**
+     * Compiles (or retrieves from cache) and binds the parameters to the
+     * provided script
+     */
+    public final ExecutableScript getExecutableScript(Script script, ScriptContext context) {
+        failIfFrozen();
+        CompiledScript compiledScript = scriptService.compile(script, context);
+        return scriptService.executable(compiledScript, script.getParams());
+    }
+
+    /**
+     * Returns a lazily created {@link ExecutableScript} that is compiled immediately but can be pulled later once all
+     * parameters are available.
+     */
+    public final Function<Map<String, Object>, ExecutableScript> getLazyExecutableScript(Script script, ScriptContext context) {
+        failIfFrozen();
+        CompiledScript executable = scriptService.compile(script, context);
+        return (p) ->  scriptService.executable(executable, p);
+    }
+
+    /**
+     * if this method is called the query context will throw exception if methods are accessed
+     * that could yield different results across executions like {@link #getTemplateBytes(Script)}
+     */
+    public final void freezeContext() {
+        this.frozen.set(Boolean.TRUE);
+    }
+
+    /**
+     * This method fails if {@link #freezeContext()} is called before on this
+     * context. This is used to <i>seal</i>.
+     *
+     * This methods and all methods that call it should be final to ensure that
+     * setting the request as not cacheable and the freezing behaviour of this
+     * class cannot be bypassed. This is important so we can trust when this
+     * class says a request can be cached.
+     */
+    protected final void failIfFrozen() {
+        this.cachable = false;
+        if (frozen.get() == Boolean.TRUE) {
+            throw new IllegalArgumentException("features that prevent cachability are disabled on this context");
+        } else {
+            assert frozen.get() == null : frozen.get();
+        }
+    }
+
+    @Override
+    public final BytesReference getTemplateBytes(Script template) {
+        failIfFrozen();
+        return super.getTemplateBytes(template);
+    }
+
+    /**
+     * Returns <code>true</code> iff the result of the processed search request is cachable. Otherwise <code>false</code>
+     */
+    public final boolean isCachable() {
+        return cachable;
+    }
+
+    /**
+     * Returns the shard ID this context was created for.
+     */
+    public int getShardId() {
+        return shardId;
+    }
+
+    @Override
+    public final long nowInMillis() {
+        failIfFrozen();
+        return super.nowInMillis();
+    }
+
+    @Override
+    public Client getClient() {
+        failIfFrozen(); // we somebody uses a terms filter with lookup for instance can't be cached...
+        return super.getClient();
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryShardException.java b/core/src/main/java/org/elasticsearch/index/query/QueryShardException.java
index 1e31c7c50e160..9b6ce3a6e4b51 100644
--- a/core/src/main/java/org/elasticsearch/index/query/QueryShardException.java
+++ b/core/src/main/java/org/elasticsearch/index/query/QueryShardException.java
@@ -22,7 +22,6 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.rest.RestStatus;
 
@@ -60,11 +59,6 @@ public RestStatus status() {
         return RestStatus.BAD_REQUEST;
     }
 
-    @Override
-    protected void innerToXContent(XContentBuilder builder, Params params) throws IOException {
-        super.innerToXContent(builder, params);
-    }
-
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryStringQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/QueryStringQueryBuilder.java
index be6a170cc2285..6b37da0dd1821 100644
--- a/core/src/main/java/org/elasticsearch/index/query/QueryStringQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/QueryStringQueryBuilder.java
@@ -21,11 +21,12 @@
 
 import org.apache.lucene.queryparser.classic.MapperQueryParser;
 import org.apache.lucene.queryparser.classic.QueryParserSettings;
-import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.FuzzyQuery;
+import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.automaton.Operations;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -36,17 +37,30 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
+import org.elasticsearch.index.mapper.DateFieldMapper;
+import org.elasticsearch.index.mapper.IpFieldMapper;
+import org.elasticsearch.index.mapper.KeywordFieldMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.NumberFieldMapper;
+import org.elasticsearch.index.mapper.ScaledFloatFieldMapper;
+import org.elasticsearch.index.mapper.StringFieldMapper;
+import org.elasticsearch.index.mapper.TextFieldMapper;
+import org.elasticsearch.index.mapper.TimestampFieldMapper;
 import org.elasticsearch.index.query.support.QueryParsers;
 import org.joda.time.DateTimeZone;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.Set;
 import java.util.TreeMap;
 
 /**
@@ -57,11 +71,11 @@
  * them either using DisMax or a plain boolean query (see {@link #useDisMax(boolean)}).
  */
 public class QueryStringQueryBuilder extends AbstractQueryBuilder<QueryStringQueryBuilder> {
+
     public static final String NAME = "query_string";
 
     public static final boolean DEFAULT_AUTO_GENERATE_PHRASE_QUERIES = false;
     public static final int DEFAULT_MAX_DETERMINED_STATES = Operations.DEFAULT_MAX_DETERMINIZED_STATES;
-    public static final boolean DEFAULT_LOWERCASE_EXPANDED_TERMS = true;
     public static final boolean DEFAULT_ENABLE_POSITION_INCREMENTS = true;
     public static final boolean DEFAULT_ESCAPE = false;
     public static final boolean DEFAULT_USE_DIS_MAX = true;
@@ -71,7 +85,7 @@ public class QueryStringQueryBuilder extends AbstractQueryBuilder<QueryStringQue
     public static final float DEFAULT_TIE_BREAKER = 0.0f;
     public static final Fuzziness DEFAULT_FUZZINESS = Fuzziness.AUTO;
     public static final Operator DEFAULT_OPERATOR = Operator.OR;
-    public static final Locale DEFAULT_LOCALE = Locale.ROOT;
+    public static final boolean DEFAULT_SPLIT_ON_WHITESPACE = true;
 
     private static final ParseField QUERY_FIELD = new ParseField("query");
     private static final ParseField FIELDS_FIELD = new ParseField("fields");
@@ -81,9 +95,11 @@ public class QueryStringQueryBuilder extends AbstractQueryBuilder<QueryStringQue
     private static final ParseField QUOTE_ANALYZER_FIELD = new ParseField("quote_analyzer");
     private static final ParseField ALLOW_LEADING_WILDCARD_FIELD = new ParseField("allow_leading_wildcard");
     private static final ParseField AUTO_GENERATE_PHRASE_QUERIES_FIELD = new ParseField("auto_generate_phrase_queries");
-    private static final ParseField MAX_DETERMINED_STATES_FIELD = new ParseField("max_determined_states");
-    private static final ParseField LOWERCASE_EXPANDED_TERMS_FIELD = new ParseField("lowercase_expanded_terms");
-    private static final ParseField ENABLE_POSITION_INCREMENTS_FIELD = new ParseField("enable_position_increment");
+    private static final ParseField MAX_DETERMINIZED_STATES_FIELD = new ParseField("max_determinized_states", "max_determined_states");
+    private static final ParseField LOWERCASE_EXPANDED_TERMS_FIELD = new ParseField("lowercase_expanded_terms")
+            .withAllDeprecated("Decision is now made by the analyzer");
+    private static final ParseField ENABLE_POSITION_INCREMENTS_FIELD = new ParseField("enable_position_increments",
+            "enable_position_increment");
     private static final ParseField ESCAPE_FIELD = new ParseField("escape");
     private static final ParseField USE_DIS_MAX_FIELD = new ParseField("use_dis_max");
     private static final ParseField FUZZY_PREFIX_LENGTH_FIELD = new ParseField("fuzzy_prefix_length");
@@ -96,9 +112,28 @@ public class QueryStringQueryBuilder extends AbstractQueryBuilder<QueryStringQue
     private static final ParseField MINIMUM_SHOULD_MATCH_FIELD = new ParseField("minimum_should_match");
     private static final ParseField QUOTE_FIELD_SUFFIX_FIELD = new ParseField("quote_field_suffix");
     private static final ParseField LENIENT_FIELD = new ParseField("lenient");
-    private static final ParseField LOCALE_FIELD = new ParseField("locale");
+    private static final ParseField LOCALE_FIELD = new ParseField("locale")
+            .withAllDeprecated("Decision is now made by the analyzer");
     private static final ParseField TIME_ZONE_FIELD = new ParseField("time_zone");
-
+    private static final ParseField SPLIT_ON_WHITESPACE = new ParseField("split_on_whitespace");
+    private static final ParseField ALL_FIELDS_FIELD = new ParseField("all_fields");
+
+    // Mapping types the "all-ish" query can be executed against
+    public static final Set<String> ALLOWED_QUERY_MAPPER_TYPES;
+
+    static {
+        ALLOWED_QUERY_MAPPER_TYPES = new HashSet<>();
+        ALLOWED_QUERY_MAPPER_TYPES.add(DateFieldMapper.CONTENT_TYPE);
+        ALLOWED_QUERY_MAPPER_TYPES.add(IpFieldMapper.CONTENT_TYPE);
+        ALLOWED_QUERY_MAPPER_TYPES.add(KeywordFieldMapper.CONTENT_TYPE);
+        for (NumberFieldMapper.NumberType nt : NumberFieldMapper.NumberType.values()) {
+            ALLOWED_QUERY_MAPPER_TYPES.add(nt.typeName());
+        }
+        ALLOWED_QUERY_MAPPER_TYPES.add(ScaledFloatFieldMapper.CONTENT_TYPE);
+        ALLOWED_QUERY_MAPPER_TYPES.add(StringFieldMapper.CONTENT_TYPE);
+        ALLOWED_QUERY_MAPPER_TYPES.add(TextFieldMapper.CONTENT_TYPE);
+        ALLOWED_QUERY_MAPPER_TYPES.add(TimestampFieldMapper.CONTENT_TYPE);
+    }
 
     private final String queryString;
 
@@ -126,12 +161,8 @@ public class QueryStringQueryBuilder extends AbstractQueryBuilder<QueryStringQue
 
     private Boolean analyzeWildcard;
 
-    private boolean lowercaseExpandedTerms = DEFAULT_LOWERCASE_EXPANDED_TERMS;
-
     private boolean enablePositionIncrements = DEFAULT_ENABLE_POSITION_INCREMENTS;
 
-    private Locale locale = DEFAULT_LOCALE;
-
     private Fuzziness fuzziness = DEFAULT_FUZZINESS;
 
     private int fuzzyPrefixLength = DEFAULT_FUZZY_PREFIX_LENGTH;
@@ -156,9 +187,13 @@ public class QueryStringQueryBuilder extends AbstractQueryBuilder<QueryStringQue
 
     private DateTimeZone timeZone;
 
+    private Boolean useAllFields;
+
     /** To limit effort spent determinizing regexp queries. */
     private int maxDeterminizedStates = DEFAULT_MAX_DETERMINED_STATES;
 
+    private boolean splitOnWhitespace = DEFAULT_SPLIT_ON_WHITESPACE;
+
     public QueryStringQueryBuilder(String queryString) {
         if (queryString == null) {
             throw new IllegalArgumentException("query text missing");
@@ -184,9 +219,13 @@ public QueryStringQueryBuilder(StreamInput in) throws IOException {
         autoGeneratePhraseQueries = in.readBoolean();
         allowLeadingWildcard = in.readOptionalBoolean();
         analyzeWildcard = in.readOptionalBoolean();
-        lowercaseExpandedTerms = in.readBoolean();
+        if (in.getVersion().before(Version.V_5_1_1)) {
+            in.readBoolean(); // lowercase_expanded_terms
+        }
         enablePositionIncrements = in.readBoolean();
-        locale = Locale.forLanguageTag(in.readString());
+        if (in.getVersion().before(Version.V_5_1_1)) {
+            in.readString(); // locale
+        }
         fuzziness = new Fuzziness(in);
         fuzzyPrefixLength = in.readVInt();
         fuzzyMaxExpansions = in.readVInt();
@@ -200,6 +239,12 @@ public QueryStringQueryBuilder(StreamInput in) throws IOException {
         timeZone = in.readOptionalTimeZone();
         escape = in.readBoolean();
         maxDeterminizedStates = in.readVInt();
+        if (in.getVersion().onOrAfter(Version.V_5_1_1)) {
+            splitOnWhitespace = in.readBoolean();
+            useAllFields = in.readOptionalBoolean();
+        } else {
+            splitOnWhitespace = DEFAULT_SPLIT_ON_WHITESPACE;
+        }
     }
 
     @Override
@@ -218,9 +263,13 @@ protected void doWriteTo(StreamOutput out) throws IOException {
         out.writeBoolean(this.autoGeneratePhraseQueries);
         out.writeOptionalBoolean(this.allowLeadingWildcard);
         out.writeOptionalBoolean(this.analyzeWildcard);
-        out.writeBoolean(this.lowercaseExpandedTerms);
+        if (out.getVersion().before(Version.V_5_1_1)) {
+            out.writeBoolean(true); // lowercase_expanded_terms
+        }
         out.writeBoolean(this.enablePositionIncrements);
-        out.writeString(this.locale.toLanguageTag());
+        if (out.getVersion().before(Version.V_5_1_1)) {
+            out.writeString(Locale.ROOT.toLanguageTag()); // locale
+        }
         this.fuzziness.writeTo(out);
         out.writeVInt(this.fuzzyPrefixLength);
         out.writeVInt(this.fuzzyMaxExpansions);
@@ -234,6 +283,10 @@ protected void doWriteTo(StreamOutput out) throws IOException {
         out.writeOptionalTimeZone(timeZone);
         out.writeBoolean(this.escape);
         out.writeVInt(this.maxDeterminizedStates);
+        if (out.getVersion().onOrAfter(Version.V_5_1_1)) {
+            out.writeBoolean(this.splitOnWhitespace);
+            out.writeOptionalBoolean(this.useAllFields);
+        }
     }
 
     public String queryString() {
@@ -253,6 +306,20 @@ public String defaultField() {
         return this.defaultField;
     }
 
+    /**
+     * Tell the query_string query to use all fields explicitly, even if _all is
+     * enabled. If the "default_field" parameter or "fields" are specified, they
+     * will be ignored.
+     */
+    public QueryStringQueryBuilder useAllFields(Boolean useAllFields) {
+        this.useAllFields = useAllFields;
+        return this;
+    }
+
+    public Boolean useAllFields() {
+        return this.useAllFields;
+    }
+
     /**
      * Adds a field to run the query string against. The field will be associated with the
      * default boost of {@link AbstractQueryBuilder#DEFAULT_BOOST}.
@@ -389,19 +456,6 @@ public Boolean allowLeadingWildcard() {
         return this.allowLeadingWildcard;
     }
 
-    /**
-     * Whether terms of wildcard, prefix, fuzzy and range queries are to be automatically
-     * lower-cased or not.  Default is <tt>true</tt>.
-     */
-    public QueryStringQueryBuilder lowercaseExpandedTerms(boolean lowercaseExpandedTerms) {
-        this.lowercaseExpandedTerms = lowercaseExpandedTerms;
-        return this;
-    }
-
-    public boolean lowercaseExpandedTerms() {
-        return this.lowercaseExpandedTerms;
-    }
-
     /**
      * Set to <tt>true</tt> to enable position increments in result query. Defaults to
      * <tt>true</tt>.
@@ -473,6 +527,11 @@ public int phraseSlop() {
         return phraseSlop;
     }
 
+    public QueryStringQueryBuilder rewrite(String rewrite) {
+        this.rewrite = rewrite;
+        return this;
+    }
+
     /**
      * Set to <tt>true</tt> to enable analysis on wildcard and prefix queries.
      */
@@ -485,11 +544,6 @@ public Boolean analyzeWildcard() {
         return this.analyzeWildcard;
     }
 
-    public QueryStringQueryBuilder rewrite(String rewrite) {
-        this.rewrite = rewrite;
-        return this;
-    }
-
     public String rewrite() {
         return this.rewrite;
     }
@@ -528,15 +582,6 @@ public Boolean lenient() {
         return this.lenient;
     }
 
-    public QueryStringQueryBuilder locale(Locale locale) {
-        this.locale = locale == null ? DEFAULT_LOCALE : locale;
-        return this;
-    }
-
-    public Locale locale() {
-        return this.locale;
-    }
-
     /**
      * In case of date field, we can adjust the from/to fields using a timezone
      */
@@ -570,6 +615,19 @@ public boolean escape() {
         return this.escape;
     }
 
+    /**
+     * Whether query text should be split on whitespace prior to analysis.
+     * Default is <code>{@value #DEFAULT_SPLIT_ON_WHITESPACE}</code>.
+     */
+    public QueryStringQueryBuilder splitOnWhitespace(boolean value) {
+        this.splitOnWhitespace = value;
+        return this;
+    }
+
+    public boolean splitOnWhitespace() {
+        return splitOnWhitespace;
+    }
+
     @Override
     protected void doXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject(NAME);
@@ -593,11 +651,10 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep
             builder.field(QUOTE_ANALYZER_FIELD.getPreferredName(), this.quoteAnalyzer);
         }
         builder.field(AUTO_GENERATE_PHRASE_QUERIES_FIELD.getPreferredName(), this.autoGeneratePhraseQueries);
-        builder.field(MAX_DETERMINED_STATES_FIELD.getPreferredName(), this.maxDeterminizedStates);
+        builder.field(MAX_DETERMINIZED_STATES_FIELD.getPreferredName(), this.maxDeterminizedStates);
         if (this.allowLeadingWildcard != null) {
             builder.field(ALLOW_LEADING_WILDCARD_FIELD.getPreferredName(), this.allowLeadingWildcard);
         }
-        builder.field(LOWERCASE_EXPANDED_TERMS_FIELD.getPreferredName(), this.lowercaseExpandedTerms);
         builder.field(ENABLE_POSITION_INCREMENTS_FIELD.getPreferredName(), this.enablePositionIncrements);
         this.fuzziness.toXContent(builder, params);
         builder.field(FUZZY_PREFIX_LENGTH_FIELD.getPreferredName(), this.fuzzyPrefixLength);
@@ -621,11 +678,14 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep
         if (this.lenient != null) {
             builder.field(LENIENT_FIELD.getPreferredName(), this.lenient);
         }
-        builder.field(LOCALE_FIELD.getPreferredName(), this.locale.toLanguageTag());
         if (this.timeZone != null) {
             builder.field(TIME_ZONE_FIELD.getPreferredName(), this.timeZone.getID());
         }
         builder.field(ESCAPE_FIELD.getPreferredName(), this.escape);
+        builder.field(SPLIT_ON_WHITESPACE.getPreferredName(), this.splitOnWhitespace);
+        if (this.useAllFields != null) {
+            builder.field(ALL_FIELDS_FIELD.getPreferredName(), this.useAllFields);
+        }
         printBoostAndQueryName(builder);
         builder.endObject();
     }
@@ -642,7 +702,6 @@ public static Optional<QueryStringQueryBuilder> fromXContent(QueryParseContext p
         float boost = AbstractQueryBuilder.DEFAULT_BOOST;
         boolean autoGeneratePhraseQueries = QueryStringQueryBuilder.DEFAULT_AUTO_GENERATE_PHRASE_QUERIES;
         int maxDeterminizedStates = QueryStringQueryBuilder.DEFAULT_MAX_DETERMINED_STATES;
-        boolean lowercaseExpandedTerms = QueryStringQueryBuilder.DEFAULT_LOWERCASE_EXPANDED_TERMS;
         boolean enablePositionIncrements = QueryStringQueryBuilder.DEFAULT_ENABLE_POSITION_INCREMENTS;
         boolean escape = QueryStringQueryBuilder.DEFAULT_ESCAPE;
         boolean useDisMax = QueryStringQueryBuilder.DEFAULT_USE_DIS_MAX;
@@ -657,16 +716,17 @@ public static Optional<QueryStringQueryBuilder> fromXContent(QueryParseContext p
         Boolean lenient = null;
         Operator defaultOperator = QueryStringQueryBuilder.DEFAULT_OPERATOR;
         String timeZone = null;
-        Locale locale = QueryStringQueryBuilder.DEFAULT_LOCALE;
         Fuzziness fuzziness = QueryStringQueryBuilder.DEFAULT_FUZZINESS;
         String fuzzyRewrite = null;
         String rewrite = null;
+        boolean splitOnWhitespace = DEFAULT_SPLIT_ON_WHITESPACE;
+        Boolean useAllFields = null;
         Map<String, Float> fieldsAndWeights = new HashMap<>();
         while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.START_ARRAY) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, FIELDS_FIELD)) {
+                if (FIELDS_FIELD.match(currentFieldName)) {
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         String fField = null;
                         float fBoost = AbstractQueryBuilder.DEFAULT_BOOST;
@@ -690,66 +750,69 @@ public static Optional<QueryStringQueryBuilder> fromXContent(QueryParseContext p
                             "] query does not support [" + currentFieldName + "]");
                 }
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, QUERY_FIELD)) {
+                if (QUERY_FIELD.match(currentFieldName)) {
                     queryString = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, DEFAULT_FIELD_FIELD)) {
+                } else if (DEFAULT_FIELD_FIELD.match(currentFieldName)) {
                     defaultField = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, DEFAULT_OPERATOR_FIELD)) {
+                } else if (DEFAULT_OPERATOR_FIELD.match(currentFieldName)) {
                     defaultOperator = Operator.fromString(parser.text());
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, ANALYZER_FIELD)) {
+                } else if (ANALYZER_FIELD.match(currentFieldName)) {
                     analyzer = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, QUOTE_ANALYZER_FIELD)) {
+                } else if (QUOTE_ANALYZER_FIELD.match(currentFieldName)) {
                     quoteAnalyzer = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, ALLOW_LEADING_WILDCARD_FIELD)) {
+                } else if (ALLOW_LEADING_WILDCARD_FIELD.match(currentFieldName)) {
                     allowLeadingWildcard = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AUTO_GENERATE_PHRASE_QUERIES_FIELD)) {
+                } else if (AUTO_GENERATE_PHRASE_QUERIES_FIELD.match(currentFieldName)) {
                     autoGeneratePhraseQueries = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, MAX_DETERMINED_STATES_FIELD)) {
+                } else if (MAX_DETERMINIZED_STATES_FIELD.match(currentFieldName)) {
                     maxDeterminizedStates = parser.intValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, LOWERCASE_EXPANDED_TERMS_FIELD)) {
-                    lowercaseExpandedTerms = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, ENABLE_POSITION_INCREMENTS_FIELD)) {
+                } else if (LOWERCASE_EXPANDED_TERMS_FIELD.match(currentFieldName)) {
+                    // ignore, deprecated setting
+                } else if (ENABLE_POSITION_INCREMENTS_FIELD.match(currentFieldName)) {
                     enablePositionIncrements = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, ESCAPE_FIELD)) {
+                } else if (ESCAPE_FIELD.match(currentFieldName)) {
                     escape = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, USE_DIS_MAX_FIELD)) {
+                } else if (USE_DIS_MAX_FIELD.match(currentFieldName)) {
                     useDisMax = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, FUZZY_PREFIX_LENGTH_FIELD)) {
+                } else if (FUZZY_PREFIX_LENGTH_FIELD.match(currentFieldName)) {
                     fuzzyPrefixLength = parser.intValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, FUZZY_MAX_EXPANSIONS_FIELD)) {
+                } else if (FUZZY_MAX_EXPANSIONS_FIELD.match(currentFieldName)) {
                     fuzzyMaxExpansions = parser.intValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, FUZZY_REWRITE_FIELD)) {
+                } else if (FUZZY_REWRITE_FIELD.match(currentFieldName)) {
                     fuzzyRewrite = parser.textOrNull();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, PHRASE_SLOP_FIELD)) {
+                } else if (PHRASE_SLOP_FIELD.match(currentFieldName)) {
                     phraseSlop = parser.intValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, Fuzziness.FIELD)) {
+                } else if (Fuzziness.FIELD.match(currentFieldName)) {
                     fuzziness = Fuzziness.parse(parser);
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, TIE_BREAKER_FIELD)) {
+                } else if (TIE_BREAKER_FIELD.match(currentFieldName)) {
                     tieBreaker = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, ANALYZE_WILDCARD_FIELD)) {
+                } else if (ANALYZE_WILDCARD_FIELD.match(currentFieldName)) {
                     analyzeWildcard = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, REWRITE_FIELD)) {
+                } else if (REWRITE_FIELD.match(currentFieldName)) {
                     rewrite = parser.textOrNull();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, MINIMUM_SHOULD_MATCH_FIELD)) {
+                } else if (MINIMUM_SHOULD_MATCH_FIELD.match(currentFieldName)) {
                     minimumShouldMatch = parser.textOrNull();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, QUOTE_FIELD_SUFFIX_FIELD)) {
+                } else if (QUOTE_FIELD_SUFFIX_FIELD.match(currentFieldName)) {
                     quoteFieldSuffix = parser.textOrNull();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, LENIENT_FIELD)) {
+                } else if (LENIENT_FIELD.match(currentFieldName)) {
                     lenient = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, LOCALE_FIELD)) {
-                    String localeStr = parser.text();
-                    locale = Locale.forLanguageTag(localeStr);
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, TIME_ZONE_FIELD)) {
+                } else if (LOCALE_FIELD.match(currentFieldName)) {
+                    // ignore, deprecated setting
+                } else if (ALL_FIELDS_FIELD.match(currentFieldName)) {
+                    useAllFields = parser.booleanValue();
+                } else if (TIME_ZONE_FIELD.match(currentFieldName)) {
                     try {
                         timeZone = parser.text();
                     } catch (IllegalArgumentException e) {
                         throw new ParsingException(parser.getTokenLocation(), "[" + QueryStringQueryBuilder.NAME +
                                 "] time_zone [" + parser.text() + "] is unknown");
                     }
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
+                } else if (SPLIT_ON_WHITESPACE.match(currentFieldName)) {
+                    splitOnWhitespace = parser.booleanValue();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[" + QueryStringQueryBuilder.NAME +
                             "] query does not support [" + currentFieldName + "]");
@@ -763,6 +826,12 @@ public static Optional<QueryStringQueryBuilder> fromXContent(QueryParseContext p
             throw new ParsingException(parser.getTokenLocation(), "[" + QueryStringQueryBuilder.NAME + "] must be provided with a [query]");
         }
 
+        if ((useAllFields != null && useAllFields) &&
+                (defaultField != null || fieldsAndWeights.size() != 0)) {
+            throw new ParsingException(parser.getTokenLocation(),
+                    "cannot use [all_fields] parameter in conjunction with [default_field] or [fields]");
+        }
+
         QueryStringQueryBuilder queryStringQuery = new QueryStringQueryBuilder(queryString);
         queryStringQuery.fields(fieldsAndWeights);
         queryStringQuery.defaultField(defaultField);
@@ -772,7 +841,6 @@ public static Optional<QueryStringQueryBuilder> fromXContent(QueryParseContext p
         queryStringQuery.allowLeadingWildcard(allowLeadingWildcard);
         queryStringQuery.autoGeneratePhraseQueries(autoGeneratePhraseQueries);
         queryStringQuery.maxDeterminizedStates(maxDeterminizedStates);
-        queryStringQuery.lowercaseExpandedTerms(lowercaseExpandedTerms);
         queryStringQuery.enablePositionIncrements(enablePositionIncrements);
         queryStringQuery.escape(escape);
         queryStringQuery.useDisMax(useDisMax);
@@ -788,9 +856,10 @@ public static Optional<QueryStringQueryBuilder> fromXContent(QueryParseContext p
         queryStringQuery.quoteFieldSuffix(quoteFieldSuffix);
         queryStringQuery.lenient(lenient);
         queryStringQuery.timeZone(timeZone);
-        queryStringQuery.locale(locale);
         queryStringQuery.boost(boost);
         queryStringQuery.queryName(queryName);
+        queryStringQuery.splitOnWhitespace(splitOnWhitespace);
+        queryStringQuery.useAllFields(useAllFields);
         return Optional.of(queryStringQuery);
     }
 
@@ -810,10 +879,8 @@ protected boolean doEquals(QueryStringQueryBuilder other) {
                 Objects.equals(quoteFieldSuffix, other.quoteFieldSuffix) &&
                 Objects.equals(autoGeneratePhraseQueries, other.autoGeneratePhraseQueries) &&
                 Objects.equals(allowLeadingWildcard, other.allowLeadingWildcard) &&
-                Objects.equals(lowercaseExpandedTerms, other.lowercaseExpandedTerms) &&
                 Objects.equals(enablePositionIncrements, other.enablePositionIncrements) &&
                 Objects.equals(analyzeWildcard, other.analyzeWildcard) &&
-                Objects.equals(locale.toLanguageTag(), other.locale.toLanguageTag()) &&
                 Objects.equals(fuzziness, other.fuzziness) &&
                 Objects.equals(fuzzyPrefixLength, other.fuzzyPrefixLength) &&
                 Objects.equals(fuzzyMaxExpansions, other.fuzzyMaxExpansions) &&
@@ -827,55 +894,119 @@ protected boolean doEquals(QueryStringQueryBuilder other) {
                 timeZone == null ? other.timeZone == null : other.timeZone != null &&
                 Objects.equals(timeZone.getID(), other.timeZone.getID()) &&
                 Objects.equals(escape, other.escape) &&
-                Objects.equals(maxDeterminizedStates, other.maxDeterminizedStates);
+                Objects.equals(maxDeterminizedStates, other.maxDeterminizedStates) &&
+                Objects.equals(splitOnWhitespace, other.splitOnWhitespace) &&
+                Objects.equals(useAllFields, other.useAllFields);
     }
 
     @Override
     protected int doHashCode() {
         return Objects.hash(queryString, defaultField, fieldsAndWeights, defaultOperator, analyzer, quoteAnalyzer,
-                quoteFieldSuffix, autoGeneratePhraseQueries, allowLeadingWildcard, lowercaseExpandedTerms,
-                enablePositionIncrements, analyzeWildcard, locale.toLanguageTag(), fuzziness, fuzzyPrefixLength,
+                quoteFieldSuffix, autoGeneratePhraseQueries, allowLeadingWildcard, analyzeWildcard,
+                enablePositionIncrements, fuzziness, fuzzyPrefixLength,
                 fuzzyMaxExpansions, fuzzyRewrite, phraseSlop, useDisMax, tieBreaker, rewrite, minimumShouldMatch, lenient,
-                timeZone == null ? 0 : timeZone.getID(), escape, maxDeterminizedStates);
+                timeZone == null ? 0 : timeZone.getID(), escape, maxDeterminizedStates, splitOnWhitespace, useAllFields);
+    }
+
+    /**
+     * Given a shard context, return a map of all fields in the mappings that
+     * can be queried. The map will be field name to a float of 1.0f.
+     */
+    public static Map<String, Float> allQueryableDefaultFields(QueryShardContext context) {
+        Collection<String> allFields = context.simpleMatchToIndexNames("*");
+        Map<String, Float> fields = new HashMap<>();
+        for (String fieldName : allFields) {
+            if (MapperService.isMetadataField(fieldName)) {
+                // Ignore our metadata fields
+                continue;
+            }
+            MappedFieldType mft = context.fieldMapper(fieldName);
+            assert mft != null : "should never have a null mapper for an existing field";
+
+            // Ignore fields that are not in the allowed mapper types. Some
+            // types do not support term queries, and thus we cannot generate
+            // a special query for them.
+            String mappingType = mft.typeName();
+            if (ALLOWED_QUERY_MAPPER_TYPES.contains(mappingType)) {
+                fields.put(fieldName, 1.0f);
+            }
+        }
+        return fields;
     }
 
     @Override
     protected Query doToQuery(QueryShardContext context) throws IOException {
         //TODO would be nice to have all the settings in one place: some change though at query execution time
         //e.g. field names get expanded to concrete names, defaults get resolved sometimes to settings values etc.
+        if (splitOnWhitespace == false && autoGeneratePhraseQueries) {
+            throw new IllegalArgumentException("it is disallowed to disable [split_on_whitespace] " +
+                "if [auto_generate_phrase_queries] is activated");
+        }
         QueryParserSettings qpSettings;
         if (this.escape) {
             qpSettings = new QueryParserSettings(org.apache.lucene.queryparser.classic.QueryParser.escape(this.queryString));
         } else {
             qpSettings = new QueryParserSettings(this.queryString);
         }
-        qpSettings.defaultField(this.defaultField == null ? context.defaultField() : this.defaultField);
+
         Map<String, Float> resolvedFields = new TreeMap<>();
-        for (Map.Entry<String, Float> fieldsEntry : fieldsAndWeights.entrySet()) {
-            String fieldName = fieldsEntry.getKey();
-            Float weight = fieldsEntry.getValue();
-            if (Regex.isSimpleMatchPattern(fieldName)) {
-                for (String resolvedFieldName : context.getMapperService().simpleMatchToIndexNames(fieldName)) {
-                    resolvedFields.put(resolvedFieldName, weight);
+
+        if ((useAllFields != null && useAllFields) && (fieldsAndWeights.size() != 0 || this.defaultField != null)) {
+            throw addValidationError("cannot use [all_fields] parameter in conjunction with [default_field] or [fields]", null);
+        }
+
+        // If explicitly required to use all fields, use all fields, OR:
+        // Automatically determine the fields (to replace the _all field) if all of the following are true:
+        // - The _all field is disabled,
+        // - and the default_field has not been changed in the settings
+        // - and default_field is not specified in the request
+        // - and no fields are specified in the request
+        if ((this.useAllFields != null && this.useAllFields) ||
+                (context.getMapperService().allEnabled() == false &&
+                        "_all".equals(context.defaultField()) &&
+                        this.defaultField == null &&
+                        this.fieldsAndWeights.size() == 0)) {
+            // Use the automatically determined expansion of all queryable fields
+            resolvedFields = allQueryableDefaultFields(context);
+            // Automatically set leniency to "true" so mismatched fields don't cause exceptions
+            qpSettings.lenient(true);
+            if ("*".equals(queryString)) {
+                return new MatchAllDocsQuery();
+            }
+        } else {
+            qpSettings.defaultField(this.defaultField == null ? context.defaultField() : this.defaultField);
+
+            for (Map.Entry<String, Float> fieldsEntry : fieldsAndWeights.entrySet()) {
+                String fieldName = fieldsEntry.getKey();
+                Float weight = fieldsEntry.getValue();
+                if (Regex.isSimpleMatchPattern(fieldName)) {
+                    for (String resolvedFieldName : context.getMapperService().simpleMatchToIndexNames(fieldName)) {
+                        resolvedFields.put(resolvedFieldName, weight);
+                    }
+                } else {
+                    resolvedFields.put(fieldName, weight);
                 }
-            } else {
-                resolvedFields.put(fieldName, weight);
             }
+            qpSettings.lenient(lenient == null ? context.queryStringLenient() : lenient);
+        }
+        if (fieldsAndWeights.isEmpty() == false || resolvedFields.isEmpty() == false) {
+            // We set the fields and weight only if we have explicit fields to query
+            // Otherwise we set it to null and fallback to the default field.
+            qpSettings.fieldsAndWeights(resolvedFields);
         }
-        qpSettings.fieldsAndWeights(resolvedFields);
         qpSettings.defaultOperator(defaultOperator.toQueryParserOperator());
 
         if (analyzer == null) {
             qpSettings.defaultAnalyzer(context.getMapperService().searchAnalyzer());
         } else {
-            NamedAnalyzer namedAnalyzer = context.getAnalysisService().analyzer(analyzer);
+            NamedAnalyzer namedAnalyzer = context.getIndexAnalyzers().get(analyzer);
             if (namedAnalyzer == null) {
                 throw new QueryShardException(context, "[query_string] analyzer [" + analyzer + "] not found");
             }
             qpSettings.forceAnalyzer(namedAnalyzer);
         }
         if (quoteAnalyzer != null) {
-            NamedAnalyzer namedAnalyzer = context.getAnalysisService().analyzer(quoteAnalyzer);
+            NamedAnalyzer namedAnalyzer = context.getIndexAnalyzers().get(quoteAnalyzer);
             if (namedAnalyzer == null) {
                 throw new QueryShardException(context, "[query_string] quote_analyzer [" + quoteAnalyzer + "] not found");
             }
@@ -890,20 +1021,18 @@ protected Query doToQuery(QueryShardContext context) throws IOException {
         qpSettings.autoGeneratePhraseQueries(autoGeneratePhraseQueries);
         qpSettings.allowLeadingWildcard(allowLeadingWildcard == null ? context.queryStringAllowLeadingWildcard() : allowLeadingWildcard);
         qpSettings.analyzeWildcard(analyzeWildcard == null ? context.queryStringAnalyzeWildcard() : analyzeWildcard);
-        qpSettings.lowercaseExpandedTerms(lowercaseExpandedTerms);
         qpSettings.enablePositionIncrements(enablePositionIncrements);
-        qpSettings.locale(locale);
         qpSettings.fuzziness(fuzziness);
         qpSettings.fuzzyPrefixLength(fuzzyPrefixLength);
         qpSettings.fuzzyMaxExpansions(fuzzyMaxExpansions);
-        qpSettings.fuzzyRewriteMethod(QueryParsers.parseRewriteMethod(context.getParseFieldMatcher(), this.fuzzyRewrite));
+        qpSettings.fuzzyRewriteMethod(QueryParsers.parseRewriteMethod(this.fuzzyRewrite));
         qpSettings.phraseSlop(phraseSlop);
         qpSettings.useDisMax(useDisMax);
         qpSettings.tieBreaker(tieBreaker);
-        qpSettings.rewriteMethod(QueryParsers.parseRewriteMethod(context.getParseFieldMatcher(), this.rewrite));
-        qpSettings.lenient(lenient == null ? context.queryStringLenient() : lenient);
+        qpSettings.rewriteMethod(QueryParsers.parseRewriteMethod(this.rewrite));
         qpSettings.timeZone(timeZone);
         qpSettings.maxDeterminizedStates(maxDeterminizedStates);
+        qpSettings.splitOnWhitespace(splitOnWhitespace);
 
         MapperQueryParser queryParser = context.queryParser(qpSettings);
         Query query;
@@ -926,12 +1055,7 @@ protected Query doToQuery(QueryShardContext context) throws IOException {
         }
 
         query = Queries.fixNegativeQueryIfNeeded(query);
-        // If the coordination factor is disabled on a boolean query we don't apply the minimum should match.
-        // This is done to make sure that the minimum_should_match doesn't get applied when there is only one word
-        // and multiple variations of the same word in the query (synonyms for instance).
-        if (query instanceof BooleanQuery && !((BooleanQuery) query).isCoordDisabled()) {
-            query = Queries.applyMinimumShouldMatch((BooleanQuery) query, this.minimumShouldMatch());
-        }
+        query = Queries.maybeApplyMinimumShouldMatch(query, this.minimumShouldMatch);
 
         //restore the previous BoostQuery wrapping
         for (int i = boosts.size() - 1; i >= 0; i--) {
@@ -940,4 +1064,5 @@ protected Query doToQuery(QueryShardContext context) throws IOException {
 
         return query;
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java
index d19441e8cf7bf..23d37a2743520 100644
--- a/core/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java
@@ -23,9 +23,11 @@
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermRangeQuery;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.geo.ShapeRelation;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.joda.DateMathParser;
@@ -38,6 +40,7 @@
 import org.elasticsearch.index.mapper.LegacyDateFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.RangeFieldMapper;
 import org.joda.time.DateTimeZone;
 
 import java.io.IOException;
@@ -55,17 +58,18 @@ public class RangeQueryBuilder extends AbstractQueryBuilder<RangeQueryBuilder> i
 
     private static final ParseField FIELDDATA_FIELD = new ParseField("fielddata").withAllDeprecated("[no replacement]");
     private static final ParseField NAME_FIELD = new ParseField("_name")
-            .withAllDeprecated("query name is not supported in short version of range query");
-    private static final ParseField LTE_FIELD = new ParseField("lte", "le");
-    private static final ParseField GTE_FIELD = new ParseField("gte", "ge");
-    private static final ParseField FROM_FIELD = new ParseField("from");
-    private static final ParseField TO_FIELD = new ParseField("to");
+        .withAllDeprecated("query name is not supported in short version of range query");
+    public static final ParseField LTE_FIELD = new ParseField("lte", "le");
+    public static final ParseField GTE_FIELD = new ParseField("gte", "ge");
+    public static final ParseField FROM_FIELD = new ParseField("from");
+    public static final ParseField TO_FIELD = new ParseField("to");
     private static final ParseField INCLUDE_LOWER_FIELD = new ParseField("include_lower");
     private static final ParseField INCLUDE_UPPER_FIELD = new ParseField("include_upper");
-    private static final ParseField GT_FIELD = new ParseField("gt");
-    private static final ParseField LT_FIELD = new ParseField("lt");
+    public static final ParseField GT_FIELD = new ParseField("gt");
+    public static final ParseField LT_FIELD = new ParseField("lt");
     private static final ParseField TIME_ZONE_FIELD = new ParseField("time_zone");
     private static final ParseField FORMAT_FIELD = new ParseField("format");
+    private static final ParseField RELATION_FIELD = new ParseField("relation");
 
     private final String fieldName;
 
@@ -81,6 +85,8 @@ public class RangeQueryBuilder extends AbstractQueryBuilder<RangeQueryBuilder> i
 
     private FormatDateTimeFormatter format;
 
+    private ShapeRelation relation;
+
     /**
      * A Query that matches documents within an range of terms.
      *
@@ -108,6 +114,12 @@ public RangeQueryBuilder(StreamInput in) throws IOException {
         if (formatString != null) {
             format = Joda.forPattern(formatString);
         }
+        if (in.getVersion().onOrAfter(Version.V_5_2_0)) {
+            String relationString = in.readOptionalString();
+            if (relationString != null) {
+                relation = ShapeRelation.getRelationByName(relationString);
+            }
+        }
     }
 
     @Override
@@ -123,6 +135,13 @@ protected void doWriteTo(StreamOutput out) throws IOException {
             formatString = this.format.format();
         }
         out.writeOptionalString(formatString);
+        if (out.getVersion().onOrAfter(Version.V_5_2_0)) {
+            String relationString = null;
+            if (this.relation != null) {
+                relationString = this.relation.getRelationName();
+            }
+            out.writeOptionalString(relationString);
+        }
     }
 
     /**
@@ -260,6 +279,10 @@ public String timeZone() {
         return this.timeZone == null ? null : this.timeZone.getID();
     }
 
+    DateTimeZone getDateTimeZone() { // for testing
+        return timeZone;
+    }
+
     /**
      * In case of format field, we can parse the from/to fields using this time format
      */
@@ -278,6 +301,28 @@ public String format() {
         return this.format == null ? null : this.format.format();
     }
 
+    DateMathParser getForceDateParser() { // pkg private for testing
+        if (this.format != null) {
+            return new DateMathParser(this.format);
+        }
+        return null;
+    }
+
+    public ShapeRelation relation() {
+        return this.relation;
+    }
+
+    public RangeQueryBuilder relation(String relation) {
+        if (relation == null) {
+            throw new IllegalArgumentException("relation cannot be null");
+        }
+        this.relation = ShapeRelation.getRelationByName(relation);
+        if (this.relation == null) {
+            throw new IllegalArgumentException(relation + " is not a valid relation");
+        }
+        return this;
+    }
+
     @Override
     protected void doXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject(NAME);
@@ -292,6 +337,9 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep
         if (format != null) {
             builder.field(FORMAT_FIELD.getPreferredName(), format.format());
         }
+        if (relation != null) {
+            builder.field(RELATION_FIELD.getPreferredName(), relation.getRelationName());
+        }
         printBoostAndQueryName(builder);
         builder.endObject();
         builder.endObject();
@@ -309,6 +357,7 @@ public static Optional<RangeQueryBuilder> fromXContent(QueryParseContext parseCo
         float boost = AbstractQueryBuilder.DEFAULT_BOOST;
         String queryName = null;
         String format = null;
+        String relation = null;
 
         String currentFieldName = null;
         XContentParser.Token token;
@@ -324,33 +373,35 @@ public static Optional<RangeQueryBuilder> fromXContent(QueryParseContext parseCo
                     if (token == XContentParser.Token.FIELD_NAME) {
                         currentFieldName = parser.currentName();
                     } else {
-                        if (parseContext.getParseFieldMatcher().match(currentFieldName, FROM_FIELD)) {
+                        if (FROM_FIELD.match(currentFieldName)) {
                             from = parser.objectBytes();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, TO_FIELD)) {
+                        } else if (TO_FIELD.match(currentFieldName)) {
                             to = parser.objectBytes();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, INCLUDE_LOWER_FIELD)) {
+                        } else if (INCLUDE_LOWER_FIELD.match(currentFieldName)) {
                             includeLower = parser.booleanValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, INCLUDE_UPPER_FIELD)) {
+                        } else if (INCLUDE_UPPER_FIELD.match(currentFieldName)) {
                             includeUpper = parser.booleanValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                        } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                             boost = parser.floatValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, GT_FIELD)) {
+                        } else if (GT_FIELD.match(currentFieldName)) {
                             from = parser.objectBytes();
                             includeLower = false;
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, GTE_FIELD)) {
+                        } else if (GTE_FIELD.match(currentFieldName)) {
                             from = parser.objectBytes();
                             includeLower = true;
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, LT_FIELD)) {
+                        } else if (LT_FIELD.match(currentFieldName)) {
                             to = parser.objectBytes();
                             includeUpper = false;
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, LTE_FIELD)) {
+                        } else if (LTE_FIELD.match(currentFieldName)) {
                             to = parser.objectBytes();
                             includeUpper = true;
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, TIME_ZONE_FIELD)) {
+                        } else if (TIME_ZONE_FIELD.match(currentFieldName)) {
                             timeZone = parser.text();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, FORMAT_FIELD)) {
+                        } else if (FORMAT_FIELD.match(currentFieldName)) {
                             format = parser.text();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                        } else if (RELATION_FIELD.match(currentFieldName)) {
+                            relation = parser.text();
+                        } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                             queryName = parser.text();
                         } else {
                             throw new ParsingException(parser.getTokenLocation(),
@@ -359,9 +410,9 @@ public static Optional<RangeQueryBuilder> fromXContent(QueryParseContext parseCo
                     }
                 }
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, NAME_FIELD)) {
+                if (NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, FIELDDATA_FIELD)) {
+                } else if (FIELDDATA_FIELD.match(currentFieldName)) {
                     // ignore
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[range] query does not support [" + currentFieldName + "]");
@@ -382,6 +433,9 @@ public static Optional<RangeQueryBuilder> fromXContent(QueryParseContext parseCo
         if (format != null) {
             rangeQuery.format(format);
         }
+        if (relation != null) {
+            rangeQuery.relation(relation);
+        }
         return Optional.of(rangeQuery);
     }
 
@@ -406,7 +460,7 @@ protected MappedFieldType.Relation getRelation(QueryRewriteContext queryRewriteC
         } else {
             DateMathParser dateMathParser = format == null ? null : new DateMathParser(format);
             return fieldType.isFieldWithinQuery(queryRewriteContext.getIndexReader(), from, to, includeLower,
-                    includeUpper, timeZone, dateMathParser);
+                    includeUpper, timeZone, dateMathParser, queryRewriteContext);
         }
     }
 
@@ -417,12 +471,12 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws
         case DISJOINT:
             return new MatchNoneQueryBuilder();
         case WITHIN:
-            if (from != null || to != null) {
+            if (from != null || to != null || format != null || timeZone != null) {
                 RangeQueryBuilder newRangeQuery = new RangeQueryBuilder(fieldName);
                 newRangeQuery.from(null);
                 newRangeQuery.to(null);
-                newRangeQuery.format = format;
-                newRangeQuery.timeZone = timeZone;
+                newRangeQuery.format = null;
+                newRangeQuery.timeZone = null;
                 return newRangeQuery;
             } else {
                 return this;
@@ -440,26 +494,27 @@ protected Query doToQuery(QueryShardContext context) throws IOException {
         MappedFieldType mapper = context.fieldMapper(this.fieldName);
         if (mapper != null) {
             if (mapper instanceof LegacyDateFieldMapper.DateFieldType) {
-                DateMathParser forcedDateParser = null;
-                if (this.format  != null) {
-                    forcedDateParser = new DateMathParser(this.format);
-                }
+
                 query = ((LegacyDateFieldMapper.DateFieldType) mapper).rangeQuery(from, to, includeLower, includeUpper,
-                        timeZone, forcedDateParser);
+                        timeZone, getForceDateParser(), context);
             } else if (mapper instanceof DateFieldMapper.DateFieldType) {
+
+                query = ((DateFieldMapper.DateFieldType) mapper).rangeQuery(from, to, includeLower, includeUpper,
+                        timeZone, getForceDateParser(), context);
+            } else if (mapper instanceof RangeFieldMapper.RangeFieldType) {
                 DateMathParser forcedDateParser = null;
-                if (this.format  != null) {
+                if (mapper.typeName() == RangeFieldMapper.RangeType.DATE.name && this.format != null) {
                     forcedDateParser = new DateMathParser(this.format);
                 }
-                query = ((DateFieldMapper.DateFieldType) mapper).rangeQuery(from, to, includeLower, includeUpper,
-                        timeZone, forcedDateParser);
-            } else  {
+                query = ((RangeFieldMapper.RangeFieldType) mapper).rangeQuery(from, to, includeLower, includeUpper,
+                    relation, timeZone, forcedDateParser, context);
+            } else {
                 if (timeZone != null) {
                     throw new QueryShardException(context, "[range] time_zone can not be applied to non date field ["
                             + fieldName + "]");
                 }
                 //LUCENE 4 UPGRADE Mapper#rangeQuery should use bytesref as well?
-                query = mapper.rangeQuery(from, to, includeLower, includeUpper);
+                query = mapper.rangeQuery(from, to, includeLower, includeUpper, context);
             }
         } else {
             if (timeZone != null) {
diff --git a/core/src/main/java/org/elasticsearch/index/query/RegexpFlag.java b/core/src/main/java/org/elasticsearch/index/query/RegexpFlag.java
index c0fc8b80928e6..e00c19b68b587 100644
--- a/core/src/main/java/org/elasticsearch/index/query/RegexpFlag.java
+++ b/core/src/main/java/org/elasticsearch/index/query/RegexpFlag.java
@@ -78,7 +78,7 @@ public enum RegexpFlag {
 
     final int value;
 
-    private RegexpFlag(int value) {
+    RegexpFlag(int value) {
         this.value = value;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/query/RegexpQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/RegexpQueryBuilder.java
index d8481d1529c8f..df2f63665b3fa 100644
--- a/core/src/main/java/org/elasticsearch/index/query/RegexpQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/RegexpQueryBuilder.java
@@ -201,20 +201,20 @@ public static Optional<RegexpQueryBuilder> fromXContent(QueryParseContext parseC
                     if (token == XContentParser.Token.FIELD_NAME) {
                         currentFieldName = parser.currentName();
                     } else {
-                        if (parseContext.getParseFieldMatcher().match(currentFieldName, VALUE_FIELD)) {
+                        if (VALUE_FIELD.match(currentFieldName)) {
                             value = parser.textOrNull();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                        } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                             boost = parser.floatValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, REWRITE_FIELD)) {
+                        } else if (REWRITE_FIELD.match(currentFieldName)) {
                             rewrite = parser.textOrNull();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, FLAGS_FIELD)) {
+                        } else if (FLAGS_FIELD.match(currentFieldName)) {
                             String flags = parser.textOrNull();
                             flagsValue = RegexpFlag.resolveValue(flags);
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, MAX_DETERMINIZED_STATES_FIELD)) {
+                        } else if (MAX_DETERMINIZED_STATES_FIELD.match(currentFieldName)) {
                             maxDeterminizedStates = parser.intValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, FLAGS_VALUE_FIELD)) {
+                        } else if (FLAGS_VALUE_FIELD.match(currentFieldName)) {
                             flagsValue = parser.intValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                        } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                             queryName = parser.text();
                         } else {
                             throw new ParsingException(parser.getTokenLocation(),
@@ -223,7 +223,7 @@ public static Optional<RegexpQueryBuilder> fromXContent(QueryParseContext parseC
                     }
                 }
             } else {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, NAME_FIELD)) {
+                if (NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
                 } else {
                     throwParsingExceptionOnMultipleFields(NAME, parser.getTokenLocation(), fieldName, parser.currentName());
@@ -248,7 +248,7 @@ public String getWriteableName() {
 
     @Override
     protected Query doToQuery(QueryShardContext context) throws QueryShardException, IOException {
-        MultiTermQuery.RewriteMethod method = QueryParsers.parseRewriteMethod(context.getParseFieldMatcher(), rewrite, null);
+        MultiTermQuery.RewriteMethod method = QueryParsers.parseRewriteMethod(rewrite, null);
 
         Query query = null;
         MappedFieldType fieldType = context.fieldMapper(fieldName);
diff --git a/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java
index 3ff924b28db11..652d0d05a81b0 100644
--- a/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java
@@ -33,15 +33,10 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.script.LeafSearchScript;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.Script.ScriptField;
 import org.elasticsearch.script.ScriptContext;
-import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.SearchScript;
-import org.elasticsearch.search.lookup.SearchLookup;
 
 import java.io.IOException;
-import java.util.Collections;
-import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 
@@ -84,7 +79,7 @@ public String getWriteableName() {
     @Override
     protected void doXContent(XContentBuilder builder, Params builderParams) throws IOException {
         builder.startObject(NAME);
-        builder.field(ScriptField.SCRIPT.getPreferredName(), script);
+        builder.field(Script.SCRIPT_PARSE_FIELD.getPreferredName(), script);
         printBoostAndQueryName(builder);
         builder.endObject();
     }
@@ -105,18 +100,18 @@ public static Optional<ScriptQueryBuilder> fromXContent(QueryParseContext parseC
             } else if (parseContext.isDeprecatedSetting(currentFieldName)) {
                 // skip
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, ScriptField.SCRIPT)) {
-                    script = Script.parse(parser, parseContext.getParseFieldMatcher(), parseContext.getDefaultScriptLanguage());
+                if (Script.SCRIPT_PARSE_FIELD.match(currentFieldName)) {
+                    script = Script.parse(parser, parseContext.getDefaultScriptLanguage());
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[script] query does not support [" + currentFieldName + "]");
                 }
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, ScriptField.SCRIPT)) {
-                    script = Script.parse(parser, parseContext.getParseFieldMatcher(), parseContext.getDefaultScriptLanguage());
+                } else if (Script.SCRIPT_PARSE_FIELD.match(currentFieldName)) {
+                    script = Script.parse(parser, parseContext.getDefaultScriptLanguage());
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[script] query does not support [" + currentFieldName + "]");
                 }
@@ -134,18 +129,17 @@ public static Optional<ScriptQueryBuilder> fromXContent(QueryParseContext parseC
 
     @Override
     protected Query doToQuery(QueryShardContext context) throws IOException {
-        return new ScriptQuery(script, context.getScriptService(), context.lookup());
+        return new ScriptQuery(script, context.getSearchScript(script, ScriptContext.Standard.SEARCH));
     }
 
     static class ScriptQuery extends Query {
 
-        private final Script script;
+        final Script script;
+        final SearchScript searchScript;
 
-        private final SearchScript searchScript;
-
-        public ScriptQuery(Script script, ScriptService scriptService, SearchLookup searchLookup) {
+        ScriptQuery(Script script, SearchScript searchScript) {
             this.script = script;
-            this.searchScript = scriptService.search(searchLookup, script, ScriptContext.Standard.SEARCH, Collections.emptyMap());
+            this.searchScript = searchScript;
         }
 
         @Override
@@ -159,17 +153,23 @@ public String toString(String field) {
 
         @Override
         public boolean equals(Object obj) {
-            if (this == obj)
+            // TODO: Do this if/when we can assume scripts are pure functions
+            // and they have a reliable equals impl
+            /*if (this == obj)
                 return true;
             if (sameClassAs(obj) == false)
                 return false;
             ScriptQuery other = (ScriptQuery) obj;
-            return Objects.equals(script, other.script);
+            return Objects.equals(script, other.script);*/
+            return this == obj;
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(classHash(), script);
+            // TODO: Do this if/when we can assume scripts are pure functions
+            // and they have a reliable equals impl
+            // return Objects.hash(classHash(), script);
+            return System.identityHashCode(this);
         }
 
         @Override
@@ -216,4 +216,6 @@ protected int doHashCode() {
     protected boolean doEquals(ScriptQueryBuilder other) {
         return Objects.equals(script, other.script);
     }
+
+
 }
diff --git a/core/src/main/java/org/elasticsearch/index/query/SimpleQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/SimpleQueryParser.java
index 151e924ad163a..a147e496045e2 100644
--- a/core/src/main/java/org/elasticsearch/index/query/SimpleQueryParser.java
+++ b/core/src/main/java/org/elasticsearch/index/query/SimpleQueryParser.java
@@ -19,6 +19,7 @@
 package org.elasticsearch.index.query;
 
 import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.miscellaneous.DisableGraphAttribute;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
@@ -30,10 +31,11 @@
 import org.apache.lucene.search.PrefixQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.SynonymQuery;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.index.analysis.ShingleTokenFilterFactory;
 import org.elasticsearch.index.mapper.MappedFieldType;
 
 import java.io.IOException;
-import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 import java.util.List;
@@ -98,14 +100,13 @@ public Query newDefaultQuery(String text) {
      */
     @Override
     public Query newFuzzyQuery(String text, int fuzziness) {
-        if (settings.lowercaseExpandedTerms()) {
-            text = text.toLowerCase(settings.locale());
-        }
         BooleanQuery.Builder bq = new BooleanQuery.Builder();
         bq.setDisableCoord(true);
         for (Map.Entry<String,Float> entry : weights.entrySet()) {
+            final String fieldName = entry.getKey();
             try {
-                Query query = new FuzzyQuery(new Term(entry.getKey(), text), fuzziness);
+                final BytesRef term = getAnalyzer().normalize(fieldName, text);
+                Query query = new FuzzyQuery(new Term(fieldName, term), fuzziness);
                 bq.add(wrapWithBoost(query, entry.getValue()), BooleanClause.Occur.SHOULD);
             } catch (RuntimeException e) {
                 rethrowUnlessLenient(e);
@@ -120,9 +121,18 @@ public Query newPhraseQuery(String text, int slop) {
         bq.setDisableCoord(true);
         for (Map.Entry<String,Float> entry : weights.entrySet()) {
             try {
-                Query q = createPhraseQuery(entry.getKey(), text, slop);
+                String field = entry.getKey();
+                if (settings.quoteFieldSuffix() != null) {
+                    String quoteField = field + settings.quoteFieldSuffix();
+                    MappedFieldType quotedFieldType = context.fieldMapper(quoteField);
+                    if (quotedFieldType != null) {
+                        field = quoteField;
+                    }
+                }
+                Float boost = entry.getValue();
+                Query q = createPhraseQuery(field, text, slop);
                 if (q != null) {
-                    bq.add(wrapWithBoost(q, entry.getValue()), BooleanClause.Occur.SHOULD);
+                    bq.add(wrapWithBoost(q, boost), BooleanClause.Occur.SHOULD);
                 }
             } catch (RuntimeException e) {
                 rethrowUnlessLenient(e);
@@ -137,20 +147,19 @@ public Query newPhraseQuery(String text, int slop) {
      */
     @Override
     public Query newPrefixQuery(String text) {
-        if (settings.lowercaseExpandedTerms()) {
-            text = text.toLowerCase(settings.locale());
-        }
         BooleanQuery.Builder bq = new BooleanQuery.Builder();
         bq.setDisableCoord(true);
         for (Map.Entry<String,Float> entry : weights.entrySet()) {
+            final String fieldName = entry.getKey();
             try {
                 if (settings.analyzeWildcard()) {
-                    Query analyzedQuery = newPossiblyAnalyzedQuery(entry.getKey(), text);
+                    Query analyzedQuery = newPossiblyAnalyzedQuery(fieldName, text);
                     if (analyzedQuery != null) {
                         bq.add(wrapWithBoost(analyzedQuery, entry.getValue()), BooleanClause.Occur.SHOULD);
                     }
                 } else {
-                    Query query = new PrefixQuery(new Term(entry.getKey(), text));
+                    Term term = new Term(fieldName, getAnalyzer().normalize(fieldName, text));
+                    Query query = new PrefixQuery(term);
                     bq.add(wrapWithBoost(query, entry.getValue()), BooleanClause.Occur.SHOULD);
                 }
             } catch (RuntimeException e) {
@@ -160,6 +169,32 @@ public Query newPrefixQuery(String text) {
         return super.simplify(bq.build());
     }
 
+    /**
+     * Checks if graph analysis should be enabled for the field depending
+     * on the provided {@link Analyzer}
+     */
+    protected Query createFieldQuery(Analyzer analyzer, BooleanClause.Occur operator, String field,
+                                     String queryText, boolean quoted, int phraseSlop) {
+        assert operator == BooleanClause.Occur.SHOULD || operator == BooleanClause.Occur.MUST;
+
+        // Use the analyzer to get all the tokens, and then build an appropriate
+        // query based on the analysis chain.
+        try (TokenStream source = analyzer.tokenStream(field, queryText)) {
+            if (source.hasAttribute(DisableGraphAttribute.class)) {
+                /**
+                 * A {@link TokenFilter} in this {@link TokenStream} disabled the graph analysis to avoid
+                 * paths explosion. See {@link ShingleTokenFilterFactory} for details.
+                 */
+                setEnableGraphQueries(false);
+            }
+            Query query = super.createFieldQuery(source, operator, field, quoted, phraseSlop);
+            setEnableGraphQueries(true);
+            return query;
+        } catch (IOException e) {
+            throw new RuntimeException("Error analyzing query text", e);
+        }
+    }
+
     private static Query wrapWithBoost(Query query, float boost) {
         if (boost != AbstractQueryBuilder.DEFAULT_BOOST) {
             return new BoostQuery(query, boost);
@@ -173,11 +208,11 @@ private static Query wrapWithBoost(Query query, float boost) {
      * of {@code TermQuery}s and {@code PrefixQuery}s
      */
     private Query newPossiblyAnalyzedQuery(String field, String termStr) {
-        List<List<String>> tlist = new ArrayList<> ();
+        List<List<BytesRef>> tlist = new ArrayList<> ();
         // get Analyzer from superclass and tokenize the term
         try (TokenStream source = getAnalyzer().tokenStream(field, termStr)) {
             source.reset();
-            List<String> currentPos = new ArrayList<>();
+            List<BytesRef> currentPos = new ArrayList<>();
             CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class);
             PositionIncrementAttribute posAtt = source.addAttribute(PositionIncrementAttribute.class);
 
@@ -188,7 +223,8 @@ private Query newPossiblyAnalyzedQuery(String field, String termStr) {
                         tlist.add(currentPos);
                         currentPos = new ArrayList<>();
                     }
-                    currentPos.add(termAtt.toString());
+                    final BytesRef term = getAnalyzer().normalize(field, termAtt.toString());
+                    currentPos.add(term);
                     hasMoreTokens = source.incrementToken();
                 }
                 if (currentPos.isEmpty() == false) {
@@ -214,7 +250,7 @@ private Query newPossiblyAnalyzedQuery(String field, String termStr) {
         // build a boolean query with prefix on the last position only.
         BooleanQuery.Builder builder = new BooleanQuery.Builder();
         for (int pos = 0; pos < tlist.size(); pos++) {
-            List<String> plist = tlist.get(pos);
+            List<BytesRef> plist = tlist.get(pos);
             boolean isLastPos = (pos == tlist.size()-1);
             Query posQuery;
             if (plist.size() == 1) {
@@ -232,7 +268,7 @@ private Query newPossiblyAnalyzedQuery(String field, String termStr) {
                 posQuery = new SynonymQuery(terms);
             } else {
                 BooleanQuery.Builder innerBuilder = new BooleanQuery.Builder();
-                for (String token : plist) {
+                for (BytesRef token : plist) {
                     innerBuilder.add(new BooleanClause(new PrefixQuery(new Term(field, token)),
                         BooleanClause.Occur.SHOULD));
                 }
@@ -248,50 +284,24 @@ private Query newPossiblyAnalyzedQuery(String field, String termStr) {
      * their default values
      */
     static class Settings {
-        /** Locale to use for parsing. */
-        private Locale locale = SimpleQueryStringBuilder.DEFAULT_LOCALE;
-        /** Specifies whether parsed terms should be lowercased. */
-        private boolean lowercaseExpandedTerms = SimpleQueryStringBuilder.DEFAULT_LOWERCASE_EXPANDED_TERMS;
         /** Specifies whether lenient query parsing should be used. */
         private boolean lenient = SimpleQueryStringBuilder.DEFAULT_LENIENT;
         /** Specifies whether wildcards should be analyzed. */
         private boolean analyzeWildcard = SimpleQueryStringBuilder.DEFAULT_ANALYZE_WILDCARD;
+        /** Specifies a suffix, if any, to apply to field names for phrase matching. */
+        private String quoteFieldSuffix = null;
 
         /**
          * Generates default {@link Settings} object (uses ROOT locale, does
          * lowercase terms, no lenient parsing, no wildcard analysis).
          * */
-        public Settings() {
-        }
-
-        public Settings(Locale locale, Boolean lowercaseExpandedTerms, Boolean lenient, Boolean analyzeWildcard) {
-            this.locale = locale;
-            this.lowercaseExpandedTerms = lowercaseExpandedTerms;
-            this.lenient = lenient;
-            this.analyzeWildcard = analyzeWildcard;
+        Settings() {
         }
 
-        /** Specifies the locale to use for parsing, Locale.ROOT by default. */
-        public void locale(Locale locale) {
-            this.locale = (locale != null) ? locale : SimpleQueryStringBuilder.DEFAULT_LOCALE;
-        }
-
-        /** Returns the locale to use for parsing. */
-        public Locale locale() {
-            return this.locale;
-        }
-
-        /**
-         * Specifies whether to lowercase parse terms, defaults to true if
-         * unset.
-         */
-        public void lowercaseExpandedTerms(boolean lowercaseExpandedTerms) {
-            this.lowercaseExpandedTerms = lowercaseExpandedTerms;
-        }
-
-        /** Returns whether to lowercase parse terms. */
-        public boolean lowercaseExpandedTerms() {
-            return this.lowercaseExpandedTerms;
+        Settings(Settings other) {
+            this.lenient = other.lenient;
+            this.analyzeWildcard = other.analyzeWildcard;
+            this.quoteFieldSuffix = other.quoteFieldSuffix;
         }
 
         /** Specifies whether to use lenient parsing, defaults to false. */
@@ -314,12 +324,24 @@ public boolean analyzeWildcard() {
             return analyzeWildcard;
         }
 
+        /**
+         * Set the suffix to append to field names for phrase matching.
+         */
+        public void quoteFieldSuffix(String suffix) {
+            this.quoteFieldSuffix = suffix;
+        }
+
+        /**
+         * Return the suffix to append for phrase matching, or {@code null} if
+         * no suffix should be appended.
+         */
+        public String quoteFieldSuffix() {
+            return quoteFieldSuffix;
+        }
+
         @Override
         public int hashCode() {
-            // checking the return value of toLanguageTag() for locales only.
-            // For further reasoning see
-            // https://issues.apache.org/jira/browse/LUCENE-4021
-            return Objects.hash(locale.toLanguageTag(), lowercaseExpandedTerms, lenient, analyzeWildcard);
+            return Objects.hash(lenient, analyzeWildcard, quoteFieldSuffix);
         }
 
         @Override
@@ -331,14 +353,8 @@ public boolean equals(Object obj) {
                 return false;
             }
             Settings other = (Settings) obj;
-
-            // checking the return value of toLanguageTag() for locales only.
-            // For further reasoning see
-            // https://issues.apache.org/jira/browse/LUCENE-4021
-            return (Objects.equals(locale.toLanguageTag(), other.locale.toLanguageTag())
-                    && Objects.equals(lowercaseExpandedTerms, other.lowercaseExpandedTerms)
-                    && Objects.equals(lenient, other.lenient)
-                    && Objects.equals(analyzeWildcard, other.analyzeWildcard));
+            return Objects.equals(lenient, other.lenient) && Objects.equals(analyzeWildcard, other.analyzeWildcard)
+                    && Objects.equals(quoteFieldSuffix, other.quoteFieldSuffix);
         }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/query/SimpleQueryStringBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SimpleQueryStringBuilder.java
index f408c0f147317..52cf7ee971eeb 100644
--- a/core/src/main/java/org/elasticsearch/index/query/SimpleQueryStringBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/SimpleQueryStringBuilder.java
@@ -20,8 +20,8 @@
 package org.elasticsearch.index.query;
 
 import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.Query;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
@@ -78,10 +78,7 @@
  * > online documentation</a>.
  */
 public class SimpleQueryStringBuilder extends AbstractQueryBuilder<SimpleQueryStringBuilder> {
-    /** Default locale used for parsing.*/
-    public static final Locale DEFAULT_LOCALE = Locale.ROOT;
-    /** Default for lowercasing parsed terms.*/
-    public static final boolean DEFAULT_LOWERCASE_EXPANDED_TERMS = true;
+
     /** Default for using lenient query parsing.*/
     public static final boolean DEFAULT_LENIENT = false;
     /** Default for wildcard analysis.*/
@@ -97,13 +94,17 @@ public class SimpleQueryStringBuilder extends AbstractQueryBuilder<SimpleQuerySt
     private static final ParseField MINIMUM_SHOULD_MATCH_FIELD = new ParseField("minimum_should_match");
     private static final ParseField ANALYZE_WILDCARD_FIELD = new ParseField("analyze_wildcard");
     private static final ParseField LENIENT_FIELD = new ParseField("lenient");
-    private static final ParseField LOWERCASE_EXPANDED_TERMS_FIELD = new ParseField("lowercase_expanded_terms");
-    private static final ParseField LOCALE_FIELD = new ParseField("locale");
+    private static final ParseField LOWERCASE_EXPANDED_TERMS_FIELD = new ParseField("lowercase_expanded_terms")
+            .withAllDeprecated("Decision is now made by the analyzer");
+    private static final ParseField LOCALE_FIELD = new ParseField("locale")
+            .withAllDeprecated("Decision is now made by the analyzer");
     private static final ParseField FLAGS_FIELD = new ParseField("flags");
     private static final ParseField DEFAULT_OPERATOR_FIELD = new ParseField("default_operator");
     private static final ParseField ANALYZER_FIELD = new ParseField("analyzer");
     private static final ParseField QUERY_FIELD = new ParseField("query");
     private static final ParseField FIELDS_FIELD = new ParseField("fields");
+    private static final ParseField QUOTE_FIELD_SUFFIX_FIELD = new ParseField("quote_field_suffix");
+    private static final ParseField ALL_FIELDS_FIELD = new ParseField("all_fields");
 
     /** Query text to parse. */
     private final String queryText;
@@ -124,6 +125,8 @@ public class SimpleQueryStringBuilder extends AbstractQueryBuilder<SimpleQuerySt
     private String minimumShouldMatch;
     /** Any search flags to be used, ALL by default. */
     private int flags = DEFAULT_FLAGS;
+    /** Flag specifying whether query should be forced to expand to all searchable fields */
+    private Boolean useAllFields;
 
     /** Further search settings needed by the ES specific query string parser only. */
     private Settings settings = new Settings();
@@ -153,11 +156,19 @@ public SimpleQueryStringBuilder(StreamInput in) throws IOException {
         flags = in.readInt();
         analyzer = in.readOptionalString();
         defaultOperator = Operator.readFromStream(in);
-        settings.lowercaseExpandedTerms(in.readBoolean());
+        if (in.getVersion().before(Version.V_5_1_1)) {
+            in.readBoolean(); // lowercase_expanded_terms
+        }
         settings.lenient(in.readBoolean());
         settings.analyzeWildcard(in.readBoolean());
-        settings.locale(Locale.forLanguageTag(in.readString()));
+        if (in.getVersion().before(Version.V_5_1_1)) {
+            in.readString(); // locale
+        }
         minimumShouldMatch = in.readOptionalString();
+        if (in.getVersion().onOrAfter(Version.V_5_1_1)) {
+            settings.quoteFieldSuffix(in.readOptionalString());
+            useAllFields = in.readOptionalBoolean();
+        }
     }
 
     @Override
@@ -171,11 +182,19 @@ protected void doWriteTo(StreamOutput out) throws IOException {
         out.writeInt(flags);
         out.writeOptionalString(analyzer);
         defaultOperator.writeTo(out);
-        out.writeBoolean(settings.lowercaseExpandedTerms());
+        if (out.getVersion().before(Version.V_5_1_1)) {
+            out.writeBoolean(true); // lowercase_expanded_terms
+        }
         out.writeBoolean(settings.lenient());
         out.writeBoolean(settings.analyzeWildcard());
-        out.writeString(settings.locale().toLanguageTag());
+        if (out.getVersion().before(Version.V_5_1_1)) {
+            out.writeString(Locale.ROOT.toLanguageTag()); // locale
+        }
         out.writeOptionalString(minimumShouldMatch);
+        if (out.getVersion().onOrAfter(Version.V_5_1_1)) {
+            out.writeOptionalString(settings.quoteFieldSuffix());
+            out.writeOptionalBoolean(useAllFields);
+        }
     }
 
     /** Returns the text to parse the query from. */
@@ -224,6 +243,15 @@ public String analyzer() {
         return this.analyzer;
     }
 
+    public Boolean useAllFields() {
+        return useAllFields;
+    }
+
+    public SimpleQueryStringBuilder useAllFields(Boolean useAllFields) {
+        this.useAllFields = useAllFields;
+        return this;
+    }
+
     /**
      * Specify the default operator for the query. Defaults to "OR" if no
      * operator is specified.
@@ -268,28 +296,18 @@ int flags() {
     }
 
     /**
-     * Specifies whether parsed terms for this query should be lower-cased.
-     * Defaults to true if not set.
+     * Set the suffix to append to field names for phrase matching.
      */
-    public SimpleQueryStringBuilder lowercaseExpandedTerms(boolean lowercaseExpandedTerms) {
-        this.settings.lowercaseExpandedTerms(lowercaseExpandedTerms);
-        return this;
-    }
-
-    /** Returns whether parsed terms should be lower cased for this query. */
-    public boolean lowercaseExpandedTerms() {
-        return this.settings.lowercaseExpandedTerms();
-    }
-
-    /** Specifies the locale for parsing terms. Defaults to ROOT if none is set. */
-    public SimpleQueryStringBuilder locale(Locale locale) {
-        this.settings.locale(locale);
+    public SimpleQueryStringBuilder quoteFieldSuffix(String suffix) {
+        settings.quoteFieldSuffix(suffix);
         return this;
     }
 
-    /** Returns the locale for parsing terms for this query. */
-    public Locale locale() {
-        return this.settings.locale();
+    /**
+     * Return the suffix to append to field names for phrase matching.
+     */
+    public String quoteFieldSuffix() {
+        return settings.quoteFieldSuffix();
     }
 
     /** Specifies whether query parsing should be lenient. Defaults to false. */
@@ -335,17 +353,37 @@ public String minimumShouldMatch() {
     protected Query doToQuery(QueryShardContext context) throws IOException {
         // field names in builder can have wildcards etc, need to resolve them here
         Map<String, Float> resolvedFieldsAndWeights = new TreeMap<>();
-        // Use the default field if no fields specified
-        if (fieldsAndWeights.isEmpty()) {
-            resolvedFieldsAndWeights.put(resolveIndexName(context.defaultField(), context), AbstractQueryBuilder.DEFAULT_BOOST);
+
+        if ((useAllFields != null && useAllFields) && (fieldsAndWeights.size() != 0)) {
+            throw addValidationError("cannot use [all_fields] parameter in conjunction with [fields]", null);
+        }
+
+        // If explicitly required to use all fields, use all fields, OR:
+        // Automatically determine the fields (to replace the _all field) if all of the following are true:
+        // - The _all field is disabled,
+        // - and the default_field has not been changed in the settings
+        // - and no fields are specified in the request
+        Settings newSettings = new Settings(settings);
+        if ((this.useAllFields != null && this.useAllFields) ||
+                (context.getMapperService().allEnabled() == false &&
+                        "_all".equals(context.defaultField()) &&
+                        this.fieldsAndWeights.isEmpty())) {
+            resolvedFieldsAndWeights = QueryStringQueryBuilder.allQueryableDefaultFields(context);
+            // Need to use lenient mode when using "all-mode" so exceptions aren't thrown due to mismatched types
+            newSettings.lenient(true);
         } else {
-            for (Map.Entry<String, Float> fieldEntry : fieldsAndWeights.entrySet()) {
-                if (Regex.isSimpleMatchPattern(fieldEntry.getKey())) {
-                    for (String fieldName : context.getMapperService().simpleMatchToIndexNames(fieldEntry.getKey())) {
-                        resolvedFieldsAndWeights.put(fieldName, fieldEntry.getValue());
+            // Use the default field if no fields specified
+            if (fieldsAndWeights.isEmpty()) {
+                resolvedFieldsAndWeights.put(resolveIndexName(context.defaultField(), context), AbstractQueryBuilder.DEFAULT_BOOST);
+            } else {
+                for (Map.Entry<String, Float> fieldEntry : fieldsAndWeights.entrySet()) {
+                    if (Regex.isSimpleMatchPattern(fieldEntry.getKey())) {
+                        for (String fieldName : context.getMapperService().simpleMatchToIndexNames(fieldEntry.getKey())) {
+                            resolvedFieldsAndWeights.put(fieldName, fieldEntry.getValue());
+                        }
+                    } else {
+                        resolvedFieldsAndWeights.put(resolveIndexName(fieldEntry.getKey(), context), fieldEntry.getValue());
                     }
-                } else {
-                    resolvedFieldsAndWeights.put(resolveIndexName(fieldEntry.getKey(), context), fieldEntry.getValue());
                 }
             }
         }
@@ -355,7 +393,7 @@ protected Query doToQuery(QueryShardContext context) throws IOException {
         if (analyzer == null) {
             luceneAnalyzer = context.getMapperService().searchAnalyzer();
         } else {
-            luceneAnalyzer = context.getAnalysisService().analyzer(analyzer);
+            luceneAnalyzer = context.getIndexAnalyzers().get(analyzer);
             if (luceneAnalyzer == null) {
                 throw new QueryShardException(context, "[" + SimpleQueryStringBuilder.NAME + "] analyzer [" + analyzer
                         + "] not found");
@@ -363,17 +401,10 @@ protected Query doToQuery(QueryShardContext context) throws IOException {
 
         }
 
-        SimpleQueryParser sqp = new SimpleQueryParser(luceneAnalyzer, resolvedFieldsAndWeights, flags, settings, context);
+        SimpleQueryParser sqp = new SimpleQueryParser(luceneAnalyzer, resolvedFieldsAndWeights, flags, newSettings, context);
         sqp.setDefaultOperator(defaultOperator.toBooleanClauseOccur());
-
         Query query = sqp.parse(queryText);
-        // If the coordination factor is disabled on a boolean query we don't apply the minimum should match.
-        // This is done to make sure that the minimum_should_match doesn't get applied when there is only one word
-        // and multiple variations of the same word in the query (synonyms for instance).
-        if (minimumShouldMatch != null && query instanceof BooleanQuery && !((BooleanQuery) query).isCoordDisabled()) {
-            query = Queries.applyMinimumShouldMatch((BooleanQuery) query, minimumShouldMatch);
-        }
-        return query;
+        return Queries.maybeApplyMinimumShouldMatch(query, minimumShouldMatch);
     }
 
     private static String resolveIndexName(String fieldName, QueryShardContext context) {
@@ -404,14 +435,18 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep
 
         builder.field(FLAGS_FIELD.getPreferredName(), flags);
         builder.field(DEFAULT_OPERATOR_FIELD.getPreferredName(), defaultOperator.name().toLowerCase(Locale.ROOT));
-        builder.field(LOWERCASE_EXPANDED_TERMS_FIELD.getPreferredName(), settings.lowercaseExpandedTerms());
         builder.field(LENIENT_FIELD.getPreferredName(), settings.lenient());
         builder.field(ANALYZE_WILDCARD_FIELD.getPreferredName(), settings.analyzeWildcard());
-        builder.field(LOCALE_FIELD.getPreferredName(), (settings.locale().toLanguageTag()));
+        if (settings.quoteFieldSuffix() != null) {
+            builder.field(QUOTE_FIELD_SUFFIX_FIELD.getPreferredName(), settings.quoteFieldSuffix());
+        }
 
         if (minimumShouldMatch != null) {
             builder.field(MINIMUM_SHOULD_MATCH_FIELD.getPreferredName(), minimumShouldMatch);
         }
+        if (useAllFields != null) {
+            builder.field(ALL_FIELDS_FIELD.getPreferredName(), useAllFields);
+        }
 
         printBoostAndQueryName(builder);
         builder.endObject();
@@ -430,16 +465,16 @@ public static Optional<SimpleQueryStringBuilder> fromXContent(QueryParseContext
         String analyzerName = null;
         int flags = SimpleQueryStringFlag.ALL.value();
         boolean lenient = SimpleQueryStringBuilder.DEFAULT_LENIENT;
-        boolean lowercaseExpandedTerms = SimpleQueryStringBuilder.DEFAULT_LOWERCASE_EXPANDED_TERMS;
         boolean analyzeWildcard = SimpleQueryStringBuilder.DEFAULT_ANALYZE_WILDCARD;
-        Locale locale = null;
+        String quoteFieldSuffix = null;
+        Boolean useAllFields = null;
 
         XContentParser.Token token;
         while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.START_ARRAY) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, FIELDS_FIELD)) {
+                if (FIELDS_FIELD.match(currentFieldName)) {
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         String fField = null;
                         float fBoost = 1;
@@ -463,15 +498,15 @@ public static Optional<SimpleQueryStringBuilder> fromXContent(QueryParseContext
                             "] query does not support [" + currentFieldName + "]");
                 }
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, QUERY_FIELD)) {
+                if (QUERY_FIELD.match(currentFieldName)) {
                     queryBody = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, ANALYZER_FIELD)) {
+                } else if (ANALYZER_FIELD.match(currentFieldName)) {
                     analyzerName = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, DEFAULT_OPERATOR_FIELD)) {
+                } else if (DEFAULT_OPERATOR_FIELD.match(currentFieldName)) {
                     defaultOperator = Operator.fromString(parser.text());
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, FLAGS_FIELD)) {
+                } else if (FLAGS_FIELD.match(currentFieldName)) {
                     if (parser.currentToken() != XContentParser.Token.VALUE_NUMBER) {
                         // Possible options are:
                         // ALL, NONE, AND, OR, PREFIX, PHRASE, PRECEDENCE, ESCAPE, WHITESPACE, FUZZY, NEAR, SLOP
@@ -482,19 +517,22 @@ public static Optional<SimpleQueryStringBuilder> fromXContent(QueryParseContext
                             flags = SimpleQueryStringFlag.ALL.value();
                         }
                     }
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, LOCALE_FIELD)) {
-                    String localeStr = parser.text();
-                    locale = Locale.forLanguageTag(localeStr);
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, LOWERCASE_EXPANDED_TERMS_FIELD)) {
-                    lowercaseExpandedTerms = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, LENIENT_FIELD)) {
+                } else if (LOCALE_FIELD.match(currentFieldName)) {
+                    // ignore, deprecated setting
+                } else if (LOWERCASE_EXPANDED_TERMS_FIELD.match(currentFieldName)) {
+                    // ignore, deprecated setting
+                } else if (LENIENT_FIELD.match(currentFieldName)) {
                     lenient = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, ANALYZE_WILDCARD_FIELD)) {
+                } else if (ANALYZE_WILDCARD_FIELD.match(currentFieldName)) {
                     analyzeWildcard = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, MINIMUM_SHOULD_MATCH_FIELD)) {
+                } else if (MINIMUM_SHOULD_MATCH_FIELD.match(currentFieldName)) {
                     minimumShouldMatch = parser.textOrNull();
+                } else if (QUOTE_FIELD_SUFFIX_FIELD.match(currentFieldName)) {
+                    quoteFieldSuffix = parser.textOrNull();
+                } else if (ALL_FIELDS_FIELD.match(currentFieldName)) {
+                    useAllFields = parser.booleanValue();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[" + SimpleQueryStringBuilder.NAME +
                             "] unsupported field [" + parser.currentName() + "]");
@@ -510,10 +548,16 @@ public static Optional<SimpleQueryStringBuilder> fromXContent(QueryParseContext
             throw new ParsingException(parser.getTokenLocation(), "[" + SimpleQueryStringBuilder.NAME + "] query text missing");
         }
 
+        if ((useAllFields != null && useAllFields) && (fieldsAndWeights.size() != 0)) {
+            throw new ParsingException(parser.getTokenLocation(),
+                    "cannot use [all_fields] parameter in conjunction with [fields]");
+        }
+
         SimpleQueryStringBuilder qb = new SimpleQueryStringBuilder(queryBody);
         qb.boost(boost).fields(fieldsAndWeights).analyzer(analyzerName).queryName(queryName).minimumShouldMatch(minimumShouldMatch);
-        qb.flags(flags).defaultOperator(defaultOperator).locale(locale).lowercaseExpandedTerms(lowercaseExpandedTerms);
-        qb.lenient(lenient).analyzeWildcard(analyzeWildcard).boost(boost);
+        qb.flags(flags).defaultOperator(defaultOperator);
+        qb.lenient(lenient).analyzeWildcard(analyzeWildcard).boost(boost).quoteFieldSuffix(quoteFieldSuffix);
+        qb.useAllFields(useAllFields);
         return Optional.of(qb);
     }
 
@@ -524,7 +568,7 @@ public String getWriteableName() {
 
     @Override
     protected int doHashCode() {
-        return Objects.hash(fieldsAndWeights, analyzer, defaultOperator, queryText, minimumShouldMatch, settings, flags);
+        return Objects.hash(fieldsAndWeights, analyzer, defaultOperator, queryText, minimumShouldMatch, settings, flags, useAllFields);
     }
 
     @Override
@@ -532,7 +576,9 @@ protected boolean doEquals(SimpleQueryStringBuilder other) {
         return Objects.equals(fieldsAndWeights, other.fieldsAndWeights) && Objects.equals(analyzer, other.analyzer)
                 && Objects.equals(defaultOperator, other.defaultOperator) && Objects.equals(queryText, other.queryText)
                 && Objects.equals(minimumShouldMatch, other.minimumShouldMatch)
-                && Objects.equals(settings, other.settings) && (flags == other.flags);
+                && Objects.equals(settings, other.settings)
+                && (flags == other.flags)
+                && (useAllFields == other.useAllFields);
     }
-}
 
+}
diff --git a/core/src/main/java/org/elasticsearch/index/query/SimpleQueryStringFlag.java b/core/src/main/java/org/elasticsearch/index/query/SimpleQueryStringFlag.java
index 68d19db7cc69b..e8cbe035c90cd 100644
--- a/core/src/main/java/org/elasticsearch/index/query/SimpleQueryStringFlag.java
+++ b/core/src/main/java/org/elasticsearch/index/query/SimpleQueryStringFlag.java
@@ -43,7 +43,7 @@ public enum SimpleQueryStringFlag {
 
     final int value;
 
-    private SimpleQueryStringFlag(int value) {
+    SimpleQueryStringFlag(int value) {
         this.value = value;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanContainingQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SpanContainingQueryBuilder.java
index 42c64d05633a8..34311098ee640 100644
--- a/core/src/main/java/org/elasticsearch/index/query/SpanContainingQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/SpanContainingQueryBuilder.java
@@ -113,13 +113,13 @@ public static Optional<SpanContainingQueryBuilder> fromXContent(QueryParseContex
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, BIG_FIELD)) {
+                if (BIG_FIELD.match(currentFieldName)) {
                     Optional<QueryBuilder> query = parseContext.parseInnerQueryBuilder();
                     if (query.isPresent() == false || query.get() instanceof SpanQueryBuilder == false) {
                         throw new ParsingException(parser.getTokenLocation(), "span_containing [big] must be of type span query");
                     }
                     big = (SpanQueryBuilder) query.get();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, LITTLE_FIELD)) {
+                } else if (LITTLE_FIELD.match(currentFieldName)) {
                     Optional<QueryBuilder> query = parseContext.parseInnerQueryBuilder();
                     if (query.isPresent() == false || query.get() instanceof SpanQueryBuilder == false) {
                         throw new ParsingException(parser.getTokenLocation(), "span_containing [little] must be of type span query");
@@ -129,9 +129,9 @@ public static Optional<SpanContainingQueryBuilder> fromXContent(QueryParseContex
                     throw new ParsingException(parser.getTokenLocation(),
                             "[span_containing] query does not support [" + currentFieldName + "]");
                 }
-            } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+            } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                 boost = parser.floatValue();
-            } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+            } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                 queryName = parser.text();
             } else {
                 throw new ParsingException(parser.getTokenLocation(),
diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanFirstQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SpanFirstQueryBuilder.java
index 5cf5adc66a896..afac564238b9c 100644
--- a/core/src/main/java/org/elasticsearch/index/query/SpanFirstQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/SpanFirstQueryBuilder.java
@@ -115,7 +115,7 @@ public static Optional<SpanFirstQueryBuilder> fromXContent(QueryParseContext par
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, MATCH_FIELD)) {
+                if (MATCH_FIELD.match(currentFieldName)) {
                     Optional<QueryBuilder> query = parseContext.parseInnerQueryBuilder();
                     if (query.isPresent() == false || query.get() instanceof SpanQueryBuilder == false) {
                         throw new ParsingException(parser.getTokenLocation(), "spanFirst [match] must be of type span query");
@@ -125,11 +125,11 @@ public static Optional<SpanFirstQueryBuilder> fromXContent(QueryParseContext par
                     throw new ParsingException(parser.getTokenLocation(), "[span_first] query does not support [" + currentFieldName + "]");
                 }
             } else {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, END_FIELD)) {
+                } else if (END_FIELD.match(currentFieldName)) {
                     end = parser.intValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[span_first] query does not support [" + currentFieldName + "]");
diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java
index b58244f3ce1f0..1e1f44df86882 100644
--- a/core/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java
@@ -93,7 +93,7 @@ public static Optional<SpanMultiTermQueryBuilder> fromXContent(QueryParseContext
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, MATCH_FIELD)) {
+                if (MATCH_FIELD.match(currentFieldName)) {
                     Optional<QueryBuilder> query = parseContext.parseInnerQueryBuilder();
                     if (query.isPresent() == false || query.get() instanceof MultiTermQueryBuilder == false) {
                         throw new ParsingException(parser.getTokenLocation(),
@@ -104,9 +104,9 @@ public static Optional<SpanMultiTermQueryBuilder> fromXContent(QueryParseContext
                     throw new ParsingException(parser.getTokenLocation(), "[span_multi] query does not support [" + currentFieldName + "]");
                 }
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[span_multi] query does not support [" + currentFieldName + "]");
diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanNearQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SpanNearQueryBuilder.java
index c23597dbf1805..996a161e625a1 100644
--- a/core/src/main/java/org/elasticsearch/index/query/SpanNearQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/SpanNearQueryBuilder.java
@@ -46,6 +46,8 @@ public class SpanNearQueryBuilder extends AbstractQueryBuilder<SpanNearQueryBuil
 
     /** Default for flag controlling whether matches are required to be in-order */
     public static boolean DEFAULT_IN_ORDER = true;
+    /** Default slop value, this is the same that lucene {@link SpanNearQuery} uses if no slop is provided */
+    public static int DEFAULT_SLOP = 0;
 
     private static final ParseField SLOP_FIELD = new ParseField("slop");
     private static final ParseField COLLECT_PAYLOADS_FIELD = new ParseField("collect_payloads").withAllDeprecated("no longer supported");
@@ -149,8 +151,8 @@ public static Optional<SpanNearQueryBuilder> fromXContent(QueryParseContext pars
         XContentParser parser = parseContext.parser();
 
         float boost = AbstractQueryBuilder.DEFAULT_BOOST;
-        Integer slop = null;
-        boolean inOrder = SpanNearQueryBuilder.DEFAULT_IN_ORDER;
+        int slop = DEFAULT_SLOP;
+        boolean inOrder = DEFAULT_IN_ORDER;
         String queryName = null;
 
         List<SpanQueryBuilder> clauses = new ArrayList<>();
@@ -161,7 +163,7 @@ public static Optional<SpanNearQueryBuilder> fromXContent(QueryParseContext pars
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.START_ARRAY) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, CLAUSES_FIELD)) {
+                if (CLAUSES_FIELD.match(currentFieldName)) {
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         Optional<QueryBuilder> query = parseContext.parseInnerQueryBuilder();
                         if (query.isPresent() == false || query.get() instanceof SpanQueryBuilder == false) {
@@ -173,15 +175,15 @@ public static Optional<SpanNearQueryBuilder> fromXContent(QueryParseContext pars
                     throw new ParsingException(parser.getTokenLocation(), "[span_near] query does not support [" + currentFieldName + "]");
                 }
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, IN_ORDER_FIELD)) {
+                if (IN_ORDER_FIELD.match(currentFieldName)) {
                     inOrder = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, COLLECT_PAYLOADS_FIELD)) {
-                    // Deprecated in 3.0.0
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, SLOP_FIELD)) {
+                } else if (COLLECT_PAYLOADS_FIELD.match(currentFieldName)) {
+                    // Deprecated in 5.0.0
+                } else if (SLOP_FIELD.match(currentFieldName)) {
                     slop = parser.intValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[span_near] query does not support [" + currentFieldName + "]");
@@ -195,10 +197,6 @@ public static Optional<SpanNearQueryBuilder> fromXContent(QueryParseContext pars
             throw new ParsingException(parser.getTokenLocation(), "span_near must include [clauses]");
         }
 
-        if (slop == null) {
-            throw new ParsingException(parser.getTokenLocation(), "span_near must include [slop]");
-        }
-
         SpanNearQueryBuilder queryBuilder = new SpanNearQueryBuilder(clauses.get(0), slop);
         for (int i = 1; i < clauses.size(); i++) {
             queryBuilder.addClause(clauses.get(i));
@@ -211,6 +209,11 @@ public static Optional<SpanNearQueryBuilder> fromXContent(QueryParseContext pars
 
     @Override
     protected Query doToQuery(QueryShardContext context) throws IOException {
+        if (clauses.size() == 1) {
+            Query query = clauses.get(0).toQuery(context);
+            assert query instanceof SpanQuery;
+            return query;
+        }
         SpanQuery[] spanQueries = new SpanQuery[clauses.size()];
         for (int i = 0; i < clauses.size(); i++) {
             Query query = clauses.get(i).toQuery(context);
diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanNotQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SpanNotQueryBuilder.java
index ccf325c253f1d..d7dd6d6534b2d 100644
--- a/core/src/main/java/org/elasticsearch/index/query/SpanNotQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/SpanNotQueryBuilder.java
@@ -181,13 +181,13 @@ public static Optional<SpanNotQueryBuilder> fromXContent(QueryParseContext parse
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, INCLUDE_FIELD)) {
+                if (INCLUDE_FIELD.match(currentFieldName)) {
                     Optional<QueryBuilder> query = parseContext.parseInnerQueryBuilder();
                     if (query.isPresent() == false || query.get() instanceof SpanQueryBuilder == false) {
                         throw new ParsingException(parser.getTokenLocation(), "spanNot [include] must be of type span query");
                     }
                     include = (SpanQueryBuilder) query.get();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, EXCLUDE_FIELD)) {
+                } else if (EXCLUDE_FIELD.match(currentFieldName)) {
                     Optional<QueryBuilder> query = parseContext.parseInnerQueryBuilder();
                     if (query.isPresent() == false || query.get() instanceof SpanQueryBuilder == false) {
                         throw new ParsingException(parser.getTokenLocation(), "spanNot [exclude] must be of type span query");
@@ -197,15 +197,15 @@ public static Optional<SpanNotQueryBuilder> fromXContent(QueryParseContext parse
                     throw new ParsingException(parser.getTokenLocation(), "[span_not] query does not support [" + currentFieldName + "]");
                 }
             } else {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, DIST_FIELD)) {
+                if (DIST_FIELD.match(currentFieldName)) {
                     dist = parser.intValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, PRE_FIELD)) {
+                } else if (PRE_FIELD.match(currentFieldName)) {
                     pre = parser.intValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, POST_FIELD)) {
+                } else if (POST_FIELD.match(currentFieldName)) {
                     post = parser.intValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[span_not] query does not support [" + currentFieldName + "]");
diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanOrQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SpanOrQueryBuilder.java
index 336dd8bb0fe19..6676d168ee55d 100644
--- a/core/src/main/java/org/elasticsearch/index/query/SpanOrQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/SpanOrQueryBuilder.java
@@ -112,7 +112,7 @@ public static Optional<SpanOrQueryBuilder> fromXContent(QueryParseContext parseC
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.START_ARRAY) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, CLAUSES_FIELD)) {
+                if (CLAUSES_FIELD.match(currentFieldName)) {
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         Optional<QueryBuilder> query = parseContext.parseInnerQueryBuilder();
                         if (query.isPresent() == false || query.get() instanceof SpanQueryBuilder == false) {
@@ -124,9 +124,9 @@ public static Optional<SpanOrQueryBuilder> fromXContent(QueryParseContext parseC
                     throw new ParsingException(parser.getTokenLocation(), "[span_or] query does not support [" + currentFieldName + "]");
                 }
             } else {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[span_or] query does not support [" + currentFieldName + "]");
diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java
index b57206c33b55c..90073e3fec43d 100644
--- a/core/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java
@@ -110,13 +110,13 @@ public static Optional<SpanTermQueryBuilder> fromXContent(QueryParseContext pars
                     if (token == XContentParser.Token.FIELD_NAME) {
                         currentFieldName = parser.currentName();
                     } else {
-                        if (parseContext.getParseFieldMatcher().match(currentFieldName, TERM_FIELD)) {
+                        if (TERM_FIELD.match(currentFieldName)) {
                             value = parser.objectBytes();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, BaseTermQueryBuilder.VALUE_FIELD)) {
+                        } else if (BaseTermQueryBuilder.VALUE_FIELD.match(currentFieldName)) {
                             value = parser.objectBytes();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                        } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                             boost = parser.floatValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                        } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                             queryName = parser.text();
                         } else {
                             throw new ParsingException(parser.getTokenLocation(),
diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanWithinQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SpanWithinQueryBuilder.java
index 2c5cf1cda6917..eebc8b0eed0c8 100644
--- a/core/src/main/java/org/elasticsearch/index/query/SpanWithinQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/SpanWithinQueryBuilder.java
@@ -119,13 +119,13 @@ public static Optional<SpanWithinQueryBuilder> fromXContent(QueryParseContext pa
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, BIG_FIELD)) {
+                if (BIG_FIELD.match(currentFieldName)) {
                     Optional<QueryBuilder> query = parseContext.parseInnerQueryBuilder();
                     if (query.isPresent() == false || query.get() instanceof SpanQueryBuilder == false) {
                         throw new ParsingException(parser.getTokenLocation(), "span_within [big] must be of type span query");
                     }
                     big = (SpanQueryBuilder) query.get();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, LITTLE_FIELD)) {
+                } else if (LITTLE_FIELD.match(currentFieldName)) {
                     Optional<QueryBuilder> query = parseContext.parseInnerQueryBuilder();
                     if (query.isPresent() == false || query.get() instanceof SpanQueryBuilder == false) {
                         throw new ParsingException(parser.getTokenLocation(), "span_within [little] must be of type span query");
@@ -135,9 +135,9 @@ public static Optional<SpanWithinQueryBuilder> fromXContent(QueryParseContext pa
                     throw new ParsingException(parser.getTokenLocation(),
                             "[span_within] query does not support [" + currentFieldName + "]");
                 }
-            } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+            } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                 boost = parser.floatValue();
-            } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+            } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                 queryName = parser.text();
             } else {
                 throw new ParsingException(parser.getTokenLocation(), "[span_within] query does not support [" + currentFieldName + "]");
diff --git a/core/src/main/java/org/elasticsearch/index/query/TermQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/TermQueryBuilder.java
index 92ec60f402f4f..dd8019e5a1445 100644
--- a/core/src/main/java/org/elasticsearch/index/query/TermQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/TermQueryBuilder.java
@@ -104,13 +104,13 @@ public static Optional<TermQueryBuilder> fromXContent(QueryParseContext parseCon
                     if (token == XContentParser.Token.FIELD_NAME) {
                         currentFieldName = parser.currentName();
                     } else {
-                        if (parseContext.getParseFieldMatcher().match(currentFieldName, TERM_FIELD)) {
+                        if (TERM_FIELD.match(currentFieldName)) {
                             value = parser.objectBytes();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, VALUE_FIELD)) {
+                        } else if (VALUE_FIELD.match(currentFieldName)) {
                             value = parser.objectBytes();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                        } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                             queryName = parser.text();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                        } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                             boost = parser.floatValue();
                         } else {
                             throw new ParsingException(parser.getTokenLocation(),
diff --git a/core/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java
index 56e6f2a7a45a9..94435d29212d4 100644
--- a/core/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java
@@ -20,20 +20,25 @@
 package org.elasticsearch.index.query;
 
 import org.apache.lucene.index.Term;
-import org.apache.lucene.queries.TermsQuery;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermInSetQuery;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefBuilder;
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -43,11 +48,15 @@
 import org.elasticsearch.indices.TermsLookup;
 
 import java.io.IOException;
+import java.util.AbstractList;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
@@ -57,6 +66,7 @@
 public class TermsQueryBuilder extends AbstractQueryBuilder<TermsQueryBuilder> {
     public static final String NAME = "terms";
     public static final ParseField QUERY_NAME_FIELD = new ParseField(NAME, "in");
+    private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(TermsQueryBuilder.class));
 
     private final String fieldName;
     private final List<?> values;
@@ -80,7 +90,7 @@ public TermsQueryBuilder(String fieldName, TermsLookup termsLookup) {
             throw new IllegalArgumentException("Both values and termsLookup specified for terms query");
         }
         this.fieldName = fieldName;
-        this.values = values;
+        this.values = values == null ? null : convert(values);
         this.termsLookup = termsLookup;
     }
 
@@ -159,7 +169,7 @@ public TermsQueryBuilder(String fieldName, Iterable<?> values) {
             throw new IllegalArgumentException("No value specified for terms query");
         }
         this.fieldName = fieldName;
-        this.values = convertToBytesRefListIfStringList(values);
+        this.values = convert(values);
         this.termsLookup = null;
     }
 
@@ -185,43 +195,125 @@ public String fieldName() {
     }
 
     public List<Object> values() {
-        return convertToStringListIfBytesRefList(this.values);
+        return convertBack(this.values);
     }
 
     public TermsLookup termsLookup() {
         return this.termsLookup;
     }
 
+    private static final Set<Class<? extends Number>> INTEGER_TYPES = new HashSet<>(
+            Arrays.asList(Byte.class, Short.class, Integer.class, Long.class));
+    private static final Set<Class<?>> STRING_TYPES = new HashSet<>(
+            Arrays.asList(BytesRef.class, String.class));
+
     /**
-     * Same as {@link #convertToBytesRefIfString} but on Iterable.
-     * @param objs the Iterable of input object
-     * @return the same input or a list of {@link BytesRef} representation if input was a list of type string
+     * Same as {@link #convert(List)} but on an {@link Iterable}.
      */
-    private static List<Object> convertToBytesRefListIfStringList(Iterable<?> objs) {
-        if (objs == null) {
-            return null;
-        }
-        List<Object> newObjs = new ArrayList<>();
-        for (Object obj : objs) {
-            newObjs.add(convertToBytesRefIfString(obj));
+    private static List<?> convert(Iterable<?> values) {
+        List<?> list;
+        if (values instanceof List<?>) {
+            list = (List<?>) values;
+        } else {
+            ArrayList<Object> arrayList = new ArrayList<Object>();
+            for (Object o : values) {
+                arrayList.add(o);
+            }
+            list = arrayList;
         }
-        return newObjs;
+        return convert(list);
     }
 
     /**
-     * Same as {@link #convertToStringIfBytesRef} but on Iterable.
-     * @param objs the Iterable of input object
-     * @return the same input or a list of utf8 string if input was a list of type {@link BytesRef}
+     * Convert the list in a way that optimizes storage in the case that all
+     * elements are either integers or {@link String}s/{@link BytesRef}s. This
+     * is useful to help garbage collections for use-cases that involve sending
+     * very large terms queries to Elasticsearch. If the list does not only
+     * contain integers or {@link String}s, then a list is returned where all
+     * {@link String}s have been replaced with {@link BytesRef}s.
      */
-    private static List<Object> convertToStringListIfBytesRefList(Iterable<?> objs) {
-        if (objs == null) {
-            return null;
+    static List<?> convert(List<?> list) {
+        if (list.isEmpty()) {
+            return Collections.emptyList();
         }
-        List<Object> newObjs = new ArrayList<>();
-        for (Object obj : objs) {
-            newObjs.add(convertToStringIfBytesRef(obj));
+
+        final boolean allNumbers = list.stream().allMatch(o -> o != null && INTEGER_TYPES.contains(o.getClass()));
+        if (allNumbers) {
+            final long[] elements = list.stream().mapToLong(o -> ((Number) o).longValue()).toArray();
+            return new AbstractList<Object>() {
+                @Override
+                public Object get(int index) {
+                    return elements[index];
+                }
+                @Override
+                public int size() {
+                    return elements.length;
+                }
+            };
         }
-        return newObjs;
+
+        final boolean allStrings = list.stream().allMatch(o -> o != null && STRING_TYPES.contains(o.getClass()));
+        if (allStrings) {
+            final BytesRefBuilder builder = new BytesRefBuilder();
+            try (BytesStreamOutput bytesOut = new BytesStreamOutput()) {
+                final int[] endOffsets = new int[list.size()];
+                int i = 0;
+                for (Object o : list) {
+                    BytesRef b;
+                    if (o instanceof BytesRef) {
+                        b = (BytesRef) o;
+                    } else {
+                        builder.copyChars(o.toString());
+                        b = builder.get();
+                    }
+                    bytesOut.writeBytes(b.bytes, b.offset, b.length);
+                    if (i == 0) {
+                        endOffsets[0] = b.length;
+                    } else {
+                        endOffsets[i] = Math.addExact(endOffsets[i-1], b.length);
+                    }
+                    ++i;
+                }
+                final BytesReference bytes = bytesOut.bytes();
+                return new AbstractList<Object>() {
+                    public Object get(int i) {
+                        final int startOffset = i == 0 ? 0 : endOffsets[i-1];
+                        final int endOffset = endOffsets[i];
+                        return bytes.slice(startOffset, endOffset - startOffset).toBytesRef();
+                    }
+                    public int size() {
+                        return endOffsets.length;
+                    }
+                };
+            }
+        }
+
+        return list.stream().map(o -> o instanceof String ? new BytesRef(o.toString()) : o).collect(Collectors.toList());
+    }
+
+    /**
+     * Convert the internal {@link List} of values back to a user-friendly list.
+     * Integers are kept as-is since the terms query does not make any difference
+     * between {@link Integer}s and {@link Long}s, but {@link BytesRef}s are
+     * converted back to {@link String}s.
+     */
+    static List<Object> convertBack(List<?> list) {
+        return new AbstractList<Object>() {
+            @Override
+            public int size() {
+                return list.size();
+            }
+            @Override
+            public Object get(int index) {
+                Object o = list.get(index);
+                if (o instanceof BytesRef) {
+                    o = ((BytesRef) o).utf8ToString();
+                }
+                // we do not convert longs, all integer types are equivalent
+                // as far as this query is concerned
+                return o;
+            }
+        };
     }
 
     @Override
@@ -232,7 +324,7 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep
             termsLookup.toXContent(builder, params);
             builder.endObject();
         } else {
-            builder.field(fieldName, convertToStringListIfBytesRefList(values));
+            builder.field(fieldName, convertBack(values));
         }
         printBoostAndQueryName(builder);
         builder.endObject();
@@ -271,9 +363,9 @@ public static Optional<TermsQueryBuilder> fromXContent(QueryParseContext parseCo
                 fieldName = currentFieldName;
                 termsLookup = TermsLookup.parseTermsLookup(parser);
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
@@ -352,7 +444,7 @@ private static Query handleTermsQuery(List<?> terms, String fieldName, QueryShar
                 for (int i = 0; i < filterValues.length; i++) {
                     filterValues[i] = BytesRefs.toBytesRef(terms.get(i));
                 }
-                query = new TermsQuery(indexFieldName, filterValues);
+                query = new TermInSetQuery(indexFieldName, filterValues);
             }
         } else {
             BooleanQuery.Builder bq = new BooleanQuery.Builder();
@@ -385,6 +477,7 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws
         if (this.termsLookup != null) {
             TermsLookup termsLookup = new TermsLookup(this.termsLookup);
             if (termsLookup.index() == null) { // TODO this should go away?
+                DEPRECATION_LOGGER.deprecated("Omitting the index in terms lookup is deprecated");
                 if (queryRewriteContext.getIndexSettings() != null) {
                     termsLookup.index(queryRewriteContext.getIndexSettings().getIndex().getName());
                 } else {
diff --git a/core/src/main/java/org/elasticsearch/index/query/TypeQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/TypeQueryBuilder.java
index c5b071aa64efe..e3a9c5bebbdd2 100644
--- a/core/src/main/java/org/elasticsearch/index/query/TypeQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/TypeQueryBuilder.java
@@ -94,11 +94,11 @@ public static Optional<TypeQueryBuilder> fromXContent(QueryParseContext parseCon
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, VALUE_FIELD)) {
+                } else if (VALUE_FIELD.match(currentFieldName)) {
                     type = parser.utf8Bytes();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
@@ -133,7 +133,7 @@ protected Query doToQuery(QueryShardContext context) throws IOException {
             // no type means no documents
             return new MatchNoDocsQuery();
         } else {
-            return documentMapper.typeFilter();
+            return documentMapper.typeFilter(context);
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java
index 0cef251be07ed..728ade482c2ab 100644
--- a/core/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java
@@ -154,15 +154,15 @@ public static Optional<WildcardQueryBuilder> fromXContent(QueryParseContext pars
                     if (token == XContentParser.Token.FIELD_NAME) {
                         currentFieldName = parser.currentName();
                     } else {
-                        if (parseContext.getParseFieldMatcher().match(currentFieldName, WILDCARD_FIELD)) {
+                        if (WILDCARD_FIELD.match(currentFieldName)) {
                             value = parser.text();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, VALUE_FIELD)) {
+                        } else if (VALUE_FIELD.match(currentFieldName)) {
                             value = parser.text();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                        } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                             boost = parser.floatValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, REWRITE_FIELD)) {
+                        } else if (REWRITE_FIELD.match(currentFieldName)) {
                             rewrite = parser.textOrNull();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                        } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                             queryName = parser.text();
                         } else {
                             throw new ParsingException(parser.getTokenLocation(),
@@ -195,7 +195,7 @@ protected Query doToQuery(QueryShardContext context) throws IOException {
         }
 
         WildcardQuery query = new WildcardQuery(term);
-        MultiTermQuery.RewriteMethod rewriteMethod = QueryParsers.parseRewriteMethod(context.getParseFieldMatcher(), rewrite, null);
+        MultiTermQuery.RewriteMethod rewriteMethod = QueryParsers.parseRewriteMethod(rewrite, null);
         QueryParsers.setRewriteMethod(query, rewriteMethod);
         return query;
     }
diff --git a/core/src/main/java/org/elasticsearch/index/query/WrapperQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/WrapperQueryBuilder.java
index 922038ba43624..26c84eeb88b2c 100644
--- a/core/src/main/java/org/elasticsearch/index/query/WrapperQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/WrapperQueryBuilder.java
@@ -124,7 +124,7 @@ public static Optional<WrapperQueryBuilder> fromXContent(QueryParseContext parse
             throw new ParsingException(parser.getTokenLocation(), "[wrapper] query malformed");
         }
         String fieldName = parser.currentName();
-        if (! parseContext.getParseFieldMatcher().match(fieldName, QUERY_FIELD)) {
+        if (! QUERY_FIELD.match(fieldName)) {
             throw new ParsingException(parser.getTokenLocation(), "[wrapper] query malformed, expected `query` but was " + fieldName);
         }
         parser.nextToken();
@@ -161,11 +161,11 @@ protected boolean doEquals(WrapperQueryBuilder other) {
 
     @Override
     protected QueryBuilder doRewrite(QueryRewriteContext context) throws IOException {
-        try (XContentParser qSourceParser = XContentFactory.xContent(source).createParser(source)) {
+        try (XContentParser qSourceParser = XContentFactory.xContent(source).createParser(context.getXContentRegistry(), source)) {
             QueryParseContext parseContext = context.newParseContext(qSourceParser);
 
             final QueryBuilder queryBuilder = parseContext.parseInnerQueryBuilder().orElseThrow(
-                    () -> new ParsingException(qSourceParser.getTokenLocation(), "inner query cannot be empty"));
+                    () -> new ParsingException(qSourceParser.getTokenLocation(), "inner query cannot be empty")).rewrite(context);
             if (boost() != DEFAULT_BOOST || queryName() != null) {
                 final BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
                 boolQueryBuilder.must(queryBuilder);
diff --git a/core/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java b/core/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java
index c2d20587cf0c8..6a0ed7d3cac67 100644
--- a/core/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java
@@ -34,6 +34,7 @@
 import org.elasticsearch.common.lucene.search.function.ScoreFunction;
 import org.elasticsearch.common.unit.DistanceUnit;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -48,6 +49,7 @@
 import org.elasticsearch.index.mapper.LegacyNumberFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
+import org.elasticsearch.index.query.QueryRewriteContext;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.MultiValueMode;
 
@@ -147,10 +149,7 @@ public BytesReference getFunctionBytes() {
     @Override
     public void doXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject(getName());
-        builder.field(fieldName);
-        try (XContentParser parser = XContentFactory.xContent(functionBytes).createParser(functionBytes)) {
-            builder.copyCurrentStructure(parser);
-        }
+        builder.rawField(fieldName, functionBytes);
         builder.field(DecayFunctionParser.MULTI_VALUE_MODE.getPreferredName(), multiValueMode.name());
         builder.endObject();
     }
@@ -183,7 +182,8 @@ protected int doHashCode() {
     @Override
     protected ScoreFunction doToFunction(QueryShardContext context) throws IOException {
         AbstractDistanceScoreFunction scoreFunction;
-        try (XContentParser parser = XContentFactory.xContent(functionBytes).createParser(functionBytes)) {
+        // EMPTY is safe because parseVariable doesn't use namedObject
+        try (XContentParser parser = XContentFactory.xContent(functionBytes).createParser(NamedXContentRegistry.EMPTY, functionBytes)) {
             scoreFunction = parseVariable(fieldName, parser, context, multiValueMode);
         }
         return scoreFunction;
@@ -315,9 +315,10 @@ private AbstractDistanceScoreFunction parseDateVariable(XContentParser parser, Q
             origin = context.nowInMillis();
         } else {
             if (dateFieldType instanceof LegacyDateFieldMapper.DateFieldType) {
-                origin = ((LegacyDateFieldMapper.DateFieldType) dateFieldType).parseToMilliseconds(originString, false, null, null);
+                origin = ((LegacyDateFieldMapper.DateFieldType) dateFieldType).parseToMilliseconds(originString, false, null, null,
+                        context);
             } else {
-                origin = ((DateFieldMapper.DateFieldType) dateFieldType).parseToMilliseconds(originString, false, null, null);
+                origin = ((DateFieldMapper.DateFieldType) dateFieldType).parseToMilliseconds(originString, false, null, null, context);
             }
         }
 
@@ -338,9 +339,9 @@ static class GeoFieldDataScoreFunction extends AbstractDistanceScoreFunction {
         private final GeoPoint origin;
         private final IndexGeoPointFieldData fieldData;
 
-        private static final GeoDistance distFunction = GeoDistance.DEFAULT;
+        private static final GeoDistance distFunction = GeoDistance.ARC;
 
-        public GeoFieldDataScoreFunction(GeoPoint origin, double scale, double decay, double offset, DecayFunction func,
+        GeoFieldDataScoreFunction(GeoPoint origin, double scale, double decay, double offset, DecayFunction func,
                                          IndexGeoPointFieldData fieldData, MultiValueMode mode) {
             super(scale, decay, offset, func, mode);
             this.origin = origin;
@@ -422,7 +423,7 @@ static class NumericFieldDataScoreFunction extends AbstractDistanceScoreFunction
         private final IndexNumericFieldData fieldData;
         private final double origin;
 
-        public NumericFieldDataScoreFunction(double origin, double scale, double decay, double offset, DecayFunction func,
+        NumericFieldDataScoreFunction(double origin, double scale, double decay, double offset, DecayFunction func,
                                              IndexNumericFieldData fieldData, MultiValueMode mode) {
             super(scale, decay, offset, func, mode);
             this.fieldData = fieldData;
@@ -547,7 +548,7 @@ public double score(int docId, float subQueryScore) {
                 @Override
                 public Explanation explainScore(int docId, Explanation subQueryScore) throws IOException {
                     return Explanation.match(
-                            CombineFunction.toFloat(score(docId, subQueryScore.getValue())),
+                            (float) score(docId, subQueryScore.getValue()),
                             "Function for field " + getFieldName() + ":",
                             func.explainFunction(getDistanceString(ctx, docId), distance.get(docId), scale));
                 }
diff --git a/core/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionParser.java b/core/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionParser.java
index 2c8b9af28d6a8..5a767ecc3a0ba 100644
--- a/core/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionParser.java
+++ b/core/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionParser.java
@@ -112,7 +112,7 @@ public DFB fromXContent(QueryParseContext context) throws IOException, ParsingEx
                 XContentBuilder builder = XContentFactory.jsonBuilder();
                 builder.copyCurrentStructure(parser);
                 functionBytes = builder.bytes();
-            } else if (context.getParseFieldMatcher().match(currentFieldName, MULTI_VALUE_MODE)) {
+            } else if (MULTI_VALUE_MODE.match(currentFieldName)) {
                 multiValueMode = MultiValueMode.fromString(parser.text());
             } else {
                 throw new ParsingException(parser.getTokenLocation(), "malformed score function score parameters.");
diff --git a/core/src/main/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilder.java
index c9bb00270d416..d86e9fcc347e3 100644
--- a/core/src/main/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilder.java
@@ -31,13 +31,12 @@
 import org.elasticsearch.common.lucene.search.function.FiltersFunctionScoreQuery.FilterFunction;
 import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery;
 import org.elasticsearch.common.lucene.search.function.ScoreFunction;
-import org.elasticsearch.common.xcontent.ParseFieldRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentLocation;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.AbstractQueryBuilder;
-import org.elasticsearch.index.query.InnerHitBuilder;
+import org.elasticsearch.index.query.InnerHitContextBuilder;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
@@ -427,18 +426,18 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws
             newQueryBuilder.scoreMode = scoreMode;
             newQueryBuilder.minScore = minScore;
             newQueryBuilder.maxBoost = maxBoost;
+            newQueryBuilder.boostMode = boostMode;
             return newQueryBuilder;
         }
         return this;
     }
 
     @Override
-    protected void extractInnerHitBuilders(Map<String, InnerHitBuilder> innerHits) {
-        InnerHitBuilder.extractInnerHits(query(), innerHits);
+    protected void extractInnerHitBuilders(Map<String, InnerHitContextBuilder> innerHits) {
+        InnerHitContextBuilder.extractInnerHits(query(), innerHits);
     }
 
-    public static Optional<FunctionScoreQueryBuilder> fromXContent(ParseFieldRegistry<ScoreFunctionParser<?>> scoreFunctionsRegistry,
-            QueryParseContext parseContext) throws IOException {
+    public static Optional<FunctionScoreQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
 
         QueryBuilder query = null;
@@ -462,7 +461,7 @@ public static Optional<FunctionScoreQueryBuilder> fromXContent(ParseFieldRegistr
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, QUERY_FIELD)) {
+                if (QUERY_FIELD.match(currentFieldName)) {
                     if (query != null) {
                         throw new ParsingException(parser.getTokenLocation(), "failed to parse [{}] query. [query] is already defined.",
                                 NAME);
@@ -482,38 +481,35 @@ public static Optional<FunctionScoreQueryBuilder> fromXContent(ParseFieldRegistr
                     singleFunctionFound = true;
                     singleFunctionName = currentFieldName;
 
-                    // we try to parse a score function. If there is no score function for the current field name,
-                    // getScoreFunction will throw.
-                    ScoreFunctionBuilder<?> scoreFunction = scoreFunctionsRegistry
-                            .lookup(currentFieldName, parseContext.getParseFieldMatcher(), parser.getTokenLocation())
-                            .fromXContent(parseContext);
+                    ScoreFunctionBuilder<?> scoreFunction = parser.namedObject(ScoreFunctionBuilder.class, currentFieldName,
+                            parseContext);
                     filterFunctionBuilders.add(new FunctionScoreQueryBuilder.FilterFunctionBuilder(scoreFunction));
                 }
             } else if (token == XContentParser.Token.START_ARRAY) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, FUNCTIONS_FIELD)) {
+                if (FUNCTIONS_FIELD.match(currentFieldName)) {
                     if (singleFunctionFound) {
                         String errorString = "already found [" + singleFunctionName + "], now encountering [functions].";
                         handleMisplacedFunctionsDeclaration(parser.getTokenLocation(), errorString);
                     }
                     functionArrayFound = true;
-                    currentFieldName = parseFiltersAndFunctions(scoreFunctionsRegistry, parseContext, filterFunctionBuilders);
+                    currentFieldName = parseFiltersAndFunctions(parseContext, filterFunctionBuilders);
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "failed to parse [{}] query. array [{}] is not supported",
                             NAME, currentFieldName);
                 }
 
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, SCORE_MODE_FIELD)) {
+                if (SCORE_MODE_FIELD.match(currentFieldName)) {
                     scoreMode = FiltersFunctionScoreQuery.ScoreMode.fromString(parser.text());
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, BOOST_MODE_FIELD)) {
+                } else if (BOOST_MODE_FIELD.match(currentFieldName)) {
                     combineFunction = CombineFunction.fromString(parser.text());
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, MAX_BOOST_FIELD)) {
+                } else if (MAX_BOOST_FIELD.match(currentFieldName)) {
                     maxBoost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, MIN_SCORE_FIELD)) {
+                } else if (MIN_SCORE_FIELD.match(currentFieldName)) {
                     minScore = parser.floatValue();
                 } else {
                     if (singleFunctionFound) {
@@ -526,7 +522,7 @@ public static Optional<FunctionScoreQueryBuilder> fromXContent(ParseFieldRegistr
                         String errorString = "already found [functions] array, now encountering [" + currentFieldName + "].";
                         handleMisplacedFunctionsDeclaration(parser.getTokenLocation(), errorString);
                     }
-                    if (parseContext.getParseFieldMatcher().match(currentFieldName, WEIGHT_FIELD)) {
+                    if (WEIGHT_FIELD.match(currentFieldName)) {
                         filterFunctionBuilders.add(
                                 new FunctionScoreQueryBuilder.FilterFunctionBuilder(new WeightBuilder().setWeight(parser.floatValue())));
                         singleFunctionFound = true;
@@ -563,9 +559,8 @@ private static void handleMisplacedFunctionsDeclaration(XContentLocation content
                 MISPLACED_FUNCTION_MESSAGE_PREFIX + errorString);
     }
 
-    private static String parseFiltersAndFunctions(ParseFieldRegistry<ScoreFunctionParser<?>> scoreFunctionsRegistry,
-            QueryParseContext parseContext, List<FunctionScoreQueryBuilder.FilterFunctionBuilder> filterFunctionBuilders)
-                    throws IOException {
+    private static String parseFiltersAndFunctions(QueryParseContext parseContext,
+            List<FunctionScoreQueryBuilder.FilterFunctionBuilder> filterFunctionBuilders) throws IOException {
         String currentFieldName = null;
         XContentParser.Token token;
         XContentParser parser = parseContext.parser();
@@ -582,7 +577,7 @@ private static String parseFiltersAndFunctions(ParseFieldRegistry<ScoreFunctionP
                     if (token == XContentParser.Token.FIELD_NAME) {
                         currentFieldName = parser.currentName();
                     } else if (token == XContentParser.Token.START_OBJECT) {
-                        if (parseContext.getParseFieldMatcher().match(currentFieldName, FILTER_FIELD)) {
+                        if (FILTER_FIELD.match(currentFieldName)) {
                             filter = parseContext.parseInnerQueryBuilder().orElse(QueryBuilders.matchAllQuery());
                         } else {
                             if (scoreFunction != null) {
@@ -590,11 +585,10 @@ private static String parseFiltersAndFunctions(ParseFieldRegistry<ScoreFunctionP
                                         "failed to parse function_score functions. already found [{}], now encountering [{}].",
                                         scoreFunction.getName(), currentFieldName);
                             }
-                            scoreFunction = scoreFunctionsRegistry.lookup(currentFieldName, parseContext.getParseFieldMatcher(),
-                                    parser.getTokenLocation()).fromXContent(parseContext);
+                            scoreFunction = parser.namedObject(ScoreFunctionBuilder.class, currentFieldName, parseContext);
                         }
                     } else if (token.isValue()) {
-                        if (parseContext.getParseFieldMatcher().match(currentFieldName, WEIGHT_FIELD)) {
+                        if (WEIGHT_FIELD.match(currentFieldName)) {
                             functionWeight = parser.floatValue();
                         } else {
                             throw new ParsingException(parser.getTokenLocation(), "failed to parse [{}] query. field [{}] is not supported",
diff --git a/core/src/main/java/org/elasticsearch/index/query/functionscore/RandomScoreFunctionBuilder.java b/core/src/main/java/org/elasticsearch/index/query/functionscore/RandomScoreFunctionBuilder.java
index 4f4a3b54a2156..f2c92cc7d1ded 100644
--- a/core/src/main/java/org/elasticsearch/index/query/functionscore/RandomScoreFunctionBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/functionscore/RandomScoreFunctionBuilder.java
@@ -26,10 +26,12 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.UidFieldMapper;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -49,12 +51,19 @@ public RandomScoreFunctionBuilder() {
      */
     public RandomScoreFunctionBuilder(StreamInput in) throws IOException {
         super(in);
-        seed = in.readInt();
+        if (in.readBoolean()) {
+            seed = in.readInt();
+        }
     }
 
     @Override
     protected void doWriteTo(StreamOutput out) throws IOException {
-        out.writeInt(seed);
+        if (seed != null) {
+            out.writeBoolean(true);
+            out.writeInt(seed);
+        } else {
+            out.writeBoolean(false);
+        }
     }
 
     @Override
@@ -119,24 +128,21 @@ protected int doHashCode() {
 
     @Override
     protected ScoreFunction doToFunction(QueryShardContext context) {
-        final MappedFieldType fieldType = context.getMapperService().fullName("_uid");
+        final MappedFieldType fieldType;
+        if (context.getIndexSettings().isSingleType()) {
+            fieldType = context.getMapperService().fullName(IdFieldMapper.NAME);
+        } else {
+            fieldType = context.getMapperService().fullName(UidFieldMapper.NAME);
+        }
         if (fieldType == null) {
             // mapper could be null if we are on a shard with no docs yet, so this won't actually be used
             return new RandomScoreFunction();
         }
-        final int salt = (context.index().getName().hashCode() << 10) | getCurrentShardId();
+        final int salt = (context.index().getName().hashCode() << 10) | context.getShardId();
         final IndexFieldData<?> uidFieldData = context.getForField(fieldType);
         return new RandomScoreFunction(this.seed == null ? hash(context.nowInMillis()) : seed, salt, uidFieldData);
     }
 
-    /**
-     * Get the current shard's id for the seed. Protected because this method doesn't work during certain unit tests and needs to be
-     * replaced.
-     */
-    int getCurrentShardId() {
-        return SearchContext.current().indexShard().shardId().id();
-    }
-
     private static int hash(long value) {
         return Long.hashCode(value);
     }
diff --git a/core/src/main/java/org/elasticsearch/index/query/functionscore/ScoreFunctionBuilders.java b/core/src/main/java/org/elasticsearch/index/query/functionscore/ScoreFunctionBuilders.java
index e6fb632f5aa2a..100ff29dfeb69 100644
--- a/core/src/main/java/org/elasticsearch/index/query/functionscore/ScoreFunctionBuilders.java
+++ b/core/src/main/java/org/elasticsearch/index/query/functionscore/ScoreFunctionBuilders.java
@@ -20,6 +20,9 @@
 package org.elasticsearch.index.query.functionscore;
 
 import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptType;
+
+import static java.util.Collections.emptyMap;
 
 /**
  * Static method aliases for constructors of known {@link ScoreFunctionBuilder}s.
@@ -69,7 +72,7 @@ public static ScriptScoreFunctionBuilder scriptFunction(Script script) {
     }
 
     public static ScriptScoreFunctionBuilder scriptFunction(String script) {
-        return (new ScriptScoreFunctionBuilder(new Script(script)));
+        return (new ScriptScoreFunctionBuilder(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, script, emptyMap())));
     }
 
     public static RandomScoreFunctionBuilder randomFunction(int seed) {
diff --git a/core/src/main/java/org/elasticsearch/index/query/functionscore/ScoreFunctionParser.java b/core/src/main/java/org/elasticsearch/index/query/functionscore/ScoreFunctionParser.java
index 3c01c2d92f369..1a2fad90c464f 100644
--- a/core/src/main/java/org/elasticsearch/index/query/functionscore/ScoreFunctionParser.java
+++ b/core/src/main/java/org/elasticsearch/index/query/functionscore/ScoreFunctionParser.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.index.query.functionscore;
 
-import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.index.query.QueryParseContext;
 
 import java.io.IOException;
@@ -29,5 +28,5 @@
  */
 @FunctionalInterface
 public interface ScoreFunctionParser<FB extends ScoreFunctionBuilder<FB>> {
-    FB fromXContent(QueryParseContext context) throws IOException, ParsingException;
+    FB fromXContent(QueryParseContext context) throws IOException;
 }
diff --git a/core/src/main/java/org/elasticsearch/index/query/functionscore/ScriptScoreFunctionBuilder.java b/core/src/main/java/org/elasticsearch/index/query/functionscore/ScriptScoreFunctionBuilder.java
index e2fbc5955d7fb..68913cd9e2162 100644
--- a/core/src/main/java/org/elasticsearch/index/query/functionscore/ScriptScoreFunctionBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/functionscore/ScriptScoreFunctionBuilder.java
@@ -30,12 +30,10 @@
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.query.QueryShardException;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.Script.ScriptField;
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.SearchScript;
 
 import java.io.IOException;
-import java.util.Collections;
 import java.util.Objects;
 
 /**
@@ -74,7 +72,7 @@ public Script getScript() {
     @Override
     public void doXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject(getName());
-        builder.field(ScriptField.SCRIPT.getPreferredName(), script);
+        builder.field(Script.SCRIPT_PARSE_FIELD.getPreferredName(), script);
         builder.endObject();
     }
 
@@ -96,8 +94,7 @@ protected int doHashCode() {
     @Override
     protected ScoreFunction doToFunction(QueryShardContext context) {
         try {
-            SearchScript searchScript = context.getScriptService().search(context.lookup(), script, ScriptContext.Standard.SEARCH,
-                    Collections.emptyMap());
+            SearchScript searchScript = context.getSearchScript(script, ScriptContext.Standard.SEARCH);
             return new ScriptScoreFunction(script, searchScript);
         } catch (Exception e) {
             throw new QueryShardException(context, "script_score: the script could not be loaded", e);
@@ -114,8 +111,8 @@ public static ScriptScoreFunctionBuilder fromXContent(QueryParseContext parseCon
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, ScriptField.SCRIPT)) {
-                    script = Script.parse(parser, parseContext.getParseFieldMatcher(), parseContext.getDefaultScriptLanguage());
+                if (Script.SCRIPT_PARSE_FIELD.match(currentFieldName)) {
+                    script = Script.parse(parser, parseContext.getDefaultScriptLanguage());
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), NAME + " query does not support [" + currentFieldName + "]");
                 }
diff --git a/core/src/main/java/org/elasticsearch/index/query/support/QueryParsers.java b/core/src/main/java/org/elasticsearch/index/query/support/QueryParsers.java
index a500393c160da..ec92f174bb47f 100644
--- a/core/src/main/java/org/elasticsearch/index/query/support/QueryParsers.java
+++ b/core/src/main/java/org/elasticsearch/index/query/support/QueryParsers.java
@@ -22,7 +22,6 @@
 import org.apache.lucene.search.MultiTermQuery;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 
 /**
  *
@@ -47,28 +46,22 @@ public static void setRewriteMethod(MultiTermQuery query, @Nullable MultiTermQue
         query.setRewriteMethod(rewriteMethod);
     }
 
-    public static void setRewriteMethod(MultiTermQuery query, ParseFieldMatcher matcher, @Nullable String rewriteMethod) {
-        if (rewriteMethod == null) {
-            return;
-        }
-        query.setRewriteMethod(parseRewriteMethod(matcher, rewriteMethod));
+    public static MultiTermQuery.RewriteMethod parseRewriteMethod(@Nullable String rewriteMethod) {
+        return parseRewriteMethod(rewriteMethod, MultiTermQuery.CONSTANT_SCORE_REWRITE);
     }
 
-    public static MultiTermQuery.RewriteMethod parseRewriteMethod(ParseFieldMatcher matcher, @Nullable String rewriteMethod) {
-        return parseRewriteMethod(matcher, rewriteMethod, MultiTermQuery.CONSTANT_SCORE_REWRITE);
-    }
-
-    public static MultiTermQuery.RewriteMethod parseRewriteMethod(ParseFieldMatcher matcher, @Nullable String rewriteMethod, @Nullable MultiTermQuery.RewriteMethod defaultRewriteMethod) {
+    public static MultiTermQuery.RewriteMethod parseRewriteMethod(@Nullable String rewriteMethod,
+                                                                  @Nullable MultiTermQuery.RewriteMethod defaultRewriteMethod) {
         if (rewriteMethod == null) {
             return defaultRewriteMethod;
         }
-        if (matcher.match(rewriteMethod, CONSTANT_SCORE)) {
+        if (CONSTANT_SCORE.match(rewriteMethod)) {
             return MultiTermQuery.CONSTANT_SCORE_REWRITE;
         }
-        if (matcher.match(rewriteMethod, SCORING_BOOLEAN)) {
+        if (SCORING_BOOLEAN.match(rewriteMethod)) {
             return MultiTermQuery.SCORING_BOOLEAN_REWRITE;
         }
-        if (matcher.match(rewriteMethod, CONSTANT_SCORE_BOOLEAN)) {
+        if (CONSTANT_SCORE_BOOLEAN.match(rewriteMethod)) {
             return MultiTermQuery.CONSTANT_SCORE_BOOLEAN_REWRITE;
         }
 
@@ -84,18 +77,17 @@ public static MultiTermQuery.RewriteMethod parseRewriteMethod(ParseFieldMatcher
             final int size = Integer.parseInt(rewriteMethod.substring(firstDigit));
             String rewriteMethodName = rewriteMethod.substring(0, firstDigit);
 
-            if (matcher.match(rewriteMethodName, TOP_TERMS)) {
+            if (TOP_TERMS.match(rewriteMethodName)) {
                 return new MultiTermQuery.TopTermsScoringBooleanQueryRewrite(size);
             }
-            if (matcher.match(rewriteMethodName, TOP_TERMS_BOOST)) {
+            if (TOP_TERMS_BOOST.match(rewriteMethodName)) {
                 return new MultiTermQuery.TopTermsBoostOnlyBooleanQueryRewrite(size);
             }
-            if (matcher.match(rewriteMethodName, TOP_TERMS_BLENDED_FREQS)) {
+            if (TOP_TERMS_BLENDED_FREQS.match(rewriteMethodName)) {
                 return new MultiTermQuery.TopTermsBlendedFreqScoringRewrite(size);
             }
         }
 
         throw new IllegalArgumentException("Failed to parse rewrite_method [" + rewriteMethod + "]");
     }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/index/refresh/RefreshStats.java b/core/src/main/java/org/elasticsearch/index/refresh/RefreshStats.java
index 9b9b4673accd1..a8a8a0be41c42 100644
--- a/core/src/main/java/org/elasticsearch/index/refresh/RefreshStats.java
+++ b/core/src/main/java/org/elasticsearch/index/refresh/RefreshStats.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.index.refresh;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
@@ -27,6 +28,7 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
 import java.io.IOException;
+import java.util.Objects;
 
 public class RefreshStats implements Streamable, ToXContent {
 
@@ -34,18 +36,19 @@ public class RefreshStats implements Streamable, ToXContent {
 
     private long totalTimeInMillis;
 
+    /**
+     * Number of waiting refresh listeners.
+     */
+    private int listeners;
+
     public RefreshStats() {
 
     }
 
-    public RefreshStats(long total, long totalTimeInMillis) {
+    public RefreshStats(long total, long totalTimeInMillis, int listeners) {
         this.total = total;
         this.totalTimeInMillis = totalTimeInMillis;
-    }
-
-    public void add(long total, long totalTimeInMillis) {
-        this.total += total;
-        this.totalTimeInMillis += totalTimeInMillis;
+        this.listeners = listeners;
     }
 
     public void add(RefreshStats refreshStats) {
@@ -58,6 +61,7 @@ public void addTotals(RefreshStats refreshStats) {
         }
         this.total += refreshStats.total;
         this.totalTimeInMillis += refreshStats.totalTimeInMillis;
+        this.listeners += refreshStats.listeners;
     }
 
     /**
@@ -81,37 +85,56 @@ public TimeValue getTotalTime() {
         return new TimeValue(totalTimeInMillis);
     }
 
-    public static RefreshStats readRefreshStats(StreamInput in) throws IOException {
-        RefreshStats refreshStats = new RefreshStats();
-        refreshStats.readFrom(in);
-        return refreshStats;
+    /**
+     * The number of waiting refresh listeners.
+     */
+    public int getListeners() {
+        return listeners;
     }
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject(Fields.REFRESH);
-        builder.field(Fields.TOTAL, total);
-        builder.timeValueField(Fields.TOTAL_TIME_IN_MILLIS, Fields.TOTAL_TIME, totalTimeInMillis);
+        builder.startObject("refresh");
+        builder.field("total", total);
+        builder.timeValueField("total_time_in_millis", "total_time", totalTimeInMillis);
+        builder.field("listeners", listeners);
         builder.endObject();
         return builder;
     }
 
-    static final class Fields {
-        static final String REFRESH = "refresh";
-        static final String TOTAL = "total";
-        static final String TOTAL_TIME = "total_time";
-        static final String TOTAL_TIME_IN_MILLIS = "total_time_in_millis";
-    }
-
     @Override
     public void readFrom(StreamInput in) throws IOException {
         total = in.readVLong();
         totalTimeInMillis = in.readVLong();
+        if (in.getVersion().onOrAfter(Version.V_5_2_0)) {
+            listeners = in.readVInt();
+        } else {
+            listeners = 0;
+        }
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeVLong(total);
         out.writeVLong(totalTimeInMillis);
+        if (out.getVersion().onOrAfter(Version.V_5_2_0)) {
+            out.writeVInt(listeners);
+        }
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null || obj.getClass() != RefreshStats.class) {
+            return false;
+        }
+        RefreshStats rhs = (RefreshStats) obj;
+        return total == rhs.total
+                && totalTimeInMillis == rhs.totalTimeInMillis
+                && listeners == rhs.listeners;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(total, totalTimeInMillis, listeners);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java b/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java
new file mode 100644
index 0000000000000..5bfae5fde924a
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java
@@ -0,0 +1,453 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.support.ActiveShardCount;
+import org.elasticsearch.action.support.replication.ReplicationRequest;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskId;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
+import static org.elasticsearch.common.unit.TimeValue.timeValueMinutes;
+
+public abstract class AbstractBulkByScrollRequest<Self extends AbstractBulkByScrollRequest<Self>> extends ActionRequest {
+
+    public static final int SIZE_ALL_MATCHES = -1;
+    private static final TimeValue DEFAULT_SCROLL_TIMEOUT = timeValueMinutes(5);
+    private static final int DEFAULT_SCROLL_SIZE = 1000;
+
+    /**
+     * The search to be executed.
+     */
+    private SearchRequest searchRequest;
+
+    /**
+     * Maximum number of processed documents. Defaults to -1 meaning process all
+     * documents.
+     */
+    private int size = SIZE_ALL_MATCHES;
+
+    /**
+     * Should version conflicts cause aborts? Defaults to true.
+     */
+    private boolean abortOnVersionConflict = true;
+
+    /**
+     * Call refresh on the indexes we've written to after the request ends?
+     */
+    private boolean refresh = false;
+
+    /**
+     * Timeout to wait for the shards on to be available for each bulk request?
+     */
+    private TimeValue timeout = ReplicationRequest.DEFAULT_TIMEOUT;
+
+    /**
+     * The number of shard copies that must be active before proceeding with the write.
+     */
+    private ActiveShardCount activeShardCount = ActiveShardCount.DEFAULT;
+
+    /**
+     * Initial delay after a rejection before retrying a bulk request. With the default maxRetries the total backoff for retrying rejections
+     * is about one minute per bulk request. Once the entire bulk request is successful the retry counter resets.
+     */
+    private TimeValue retryBackoffInitialTime = timeValueMillis(500);
+
+    /**
+     * Total number of retries attempted for rejections. There is no way to ask for unlimited retries.
+     */
+    private int maxRetries = 11;
+
+    /**
+     * The throttle for this request in sub-requests per second. {@link Float#POSITIVE_INFINITY} means set no throttle and that is the
+     * default. Throttling is done between batches, as we start the next scroll requests. That way we can increase the scroll's timeout to
+     * make sure that it contains any time that we might wait.
+     */
+    private float requestsPerSecond = Float.POSITIVE_INFINITY;
+
+    /**
+     * Should this task store its result?
+     */
+    private boolean shouldStoreResult;
+
+    /**
+     * The number of slices this task should be divided into. Defaults to 1 meaning the task isn't sliced into subtasks.
+     */
+    private int slices = 1;
+
+    /**
+     * Constructor for deserialization.
+     */
+    public AbstractBulkByScrollRequest() {
+    }
+
+    /**
+     * Constructor for actual use.
+     *
+     * @param searchRequest the search request to execute to get the documents to process
+     * @param setDefaults should this request set the defaults on the search request? Usually set to true but leave it false to support
+     *        request slicing
+     */
+    public AbstractBulkByScrollRequest(SearchRequest searchRequest, boolean setDefaults) {
+        this.searchRequest = searchRequest;
+
+        // Set the defaults which differ from SearchRequest's defaults.
+        if (setDefaults) {
+            searchRequest.scroll(DEFAULT_SCROLL_TIMEOUT);
+            searchRequest.source(new SearchSourceBuilder());
+            searchRequest.source().size(DEFAULT_SCROLL_SIZE);
+        }
+    }
+
+    /**
+     * `this` cast to Self. Used for building fluent methods without cast
+     * warnings.
+     */
+    protected abstract Self self();
+
+    @Override
+    public ActionRequestValidationException validate() {
+        ActionRequestValidationException e = searchRequest.validate();
+        if (searchRequest.source().from() != -1) {
+            e = addValidationError("from is not supported in this context", e);
+        }
+        if (searchRequest.source().storedFields() != null) {
+            e = addValidationError("stored_fields is not supported in this context", e);
+        }
+        if (maxRetries < 0) {
+            e = addValidationError("retries cannnot be negative", e);
+        }
+        if (false == (size == -1 || size > 0)) {
+            e = addValidationError(
+                    "size should be greater than 0 if the request is limited to some number of documents or -1 if it isn't but it was ["
+                            + size + "]",
+                    e);
+        }
+        if (searchRequest.source().slice() != null && slices != 1) {
+            e = addValidationError("can't specify both slice and workers", e);
+        }
+        return e;
+    }
+
+    /**
+     * Maximum number of processed documents. Defaults to -1 meaning process all
+     * documents.
+     */
+    public int getSize() {
+        return size;
+    }
+
+    /**
+     * Maximum number of processed documents. Defaults to -1 meaning process all
+     * documents.
+     */
+    public Self setSize(int size) {
+        this.size = size;
+        return self();
+    }
+
+    /**
+     * Should version conflicts cause aborts? Defaults to false.
+     */
+    public boolean isAbortOnVersionConflict() {
+        return abortOnVersionConflict;
+    }
+
+    /**
+     * Should version conflicts cause aborts? Defaults to false.
+     */
+    public Self setAbortOnVersionConflict(boolean abortOnVersionConflict) {
+        this.abortOnVersionConflict = abortOnVersionConflict;
+        return self();
+    }
+
+    /**
+     * Sets abortOnVersionConflict based on REST-friendly names.
+     */
+    public void setConflicts(String conflicts) {
+        switch (conflicts) {
+        case "proceed":
+            setAbortOnVersionConflict(false);
+            return;
+        case "abort":
+            setAbortOnVersionConflict(true);
+            return;
+        default:
+            throw new IllegalArgumentException("conflicts may only be \"proceed\" or \"abort\" but was [" + conflicts + "]");
+        }
+    }
+
+    /**
+     * The search request that matches the documents to process.
+     */
+    public SearchRequest getSearchRequest() {
+        return searchRequest;
+    }
+
+    /**
+     * Call refresh on the indexes we've written to after the request ends?
+     */
+    public boolean isRefresh() {
+        return refresh;
+    }
+
+    /**
+     * Call refresh on the indexes we've written to after the request ends?
+     */
+    public Self setRefresh(boolean refresh) {
+        this.refresh = refresh;
+        return self();
+    }
+
+    /**
+     * Timeout to wait for the shards on to be available for each bulk request?
+     */
+    public TimeValue getTimeout() {
+        return timeout;
+    }
+
+    /**
+     * Timeout to wait for the shards on to be available for each bulk request?
+     */
+    public Self setTimeout(TimeValue timeout) {
+        this.timeout = timeout;
+        return self();
+    }
+
+    /**
+     * The number of shard copies that must be active before proceeding with the write.
+     */
+    public ActiveShardCount getWaitForActiveShards() {
+        return activeShardCount;
+    }
+
+    /**
+     * Sets the number of shard copies that must be active before proceeding with the write.
+     * See {@link ReplicationRequest#waitForActiveShards(ActiveShardCount)} for details.
+     */
+    public Self setWaitForActiveShards(ActiveShardCount activeShardCount) {
+        this.activeShardCount = activeShardCount;
+        return self();
+    }
+
+    /**
+     * A shortcut for {@link #setWaitForActiveShards(ActiveShardCount)} where the numerical
+     * shard count is passed in, instead of having to first call {@link ActiveShardCount#from(int)}
+     * to get the ActiveShardCount.
+     */
+    public Self setWaitForActiveShards(final int waitForActiveShards) {
+        return setWaitForActiveShards(ActiveShardCount.from(waitForActiveShards));
+    }
+
+    /**
+     * Initial delay after a rejection before retrying request.
+     */
+    public TimeValue getRetryBackoffInitialTime() {
+        return retryBackoffInitialTime;
+    }
+
+    /**
+     * Set the initial delay after a rejection before retrying request.
+     */
+    public Self setRetryBackoffInitialTime(TimeValue retryBackoffInitialTime) {
+        this.retryBackoffInitialTime = retryBackoffInitialTime;
+        return self();
+    }
+
+    /**
+     * Total number of retries attempted for rejections.
+     */
+    public int getMaxRetries() {
+        return maxRetries;
+    }
+
+    /**
+     * Set the total number of retries attempted for rejections. There is no way to ask for unlimited retries.
+     */
+    public Self setMaxRetries(int maxRetries) {
+        this.maxRetries = maxRetries;
+        return self();
+    }
+
+    /**
+     * The throttle for this request in sub-requests per second. {@link Float#POSITIVE_INFINITY} means set no throttle and that is the
+     * default. Throttling is done between batches, as we start the next scroll requests. That way we can increase the scroll's timeout to
+     * make sure that it contains any time that we might wait.
+     */
+    public float getRequestsPerSecond() {
+        return requestsPerSecond;
+    }
+
+    /**
+     * Set the throttle for this request in sub-requests per second. {@link Float#POSITIVE_INFINITY} means set no throttle and that is the
+     * default. Throttling is done between batches, as we start the next scroll requests. That way we can increase the scroll's timeout to
+     * make sure that it contains any time that we might wait.
+     */
+    public Self setRequestsPerSecond(float requestsPerSecond) {
+        if (requestsPerSecond <= 0) {
+            throw new IllegalArgumentException(
+                    "[requests_per_second] must be greater than 0. Use Float.POSITIVE_INFINITY to disable throttling.");
+        }
+        this.requestsPerSecond = requestsPerSecond;
+        return self();
+    }
+
+    /**
+     * Should this task store its result after it has finished?
+     */
+    public Self setShouldStoreResult(boolean shouldStoreResult) {
+        this.shouldStoreResult = shouldStoreResult;
+        return self();
+    }
+
+    @Override
+    public boolean getShouldStoreResult() {
+        return shouldStoreResult;
+    }
+
+    /**
+     * The number of slices this task should be divided into. Defaults to 1 meaning the task isn't sliced into subtasks.
+     */
+    public Self setSlices(int slices) {
+        if (slices < 1) {
+            throw new IllegalArgumentException("[slices] must be at least 1");
+        }
+        this.slices = slices;
+        return self();
+    }
+
+    /**
+     * The number of slices this task should be divided into. Defaults to 1 meaning the task isn't sliced into subtasks.
+     */
+    public int getSlices() {
+        return slices;
+    }
+
+    /**
+     * Build a new request for a slice of the parent request.
+     */
+    public abstract Self forSlice(TaskId slicingTask, SearchRequest slice);
+
+    /**
+     * Setup a clone of this request with the information needed to process a slice of it.
+     */
+    protected Self doForSlice(Self request, TaskId slicingTask) {
+        request.setAbortOnVersionConflict(abortOnVersionConflict).setRefresh(refresh).setTimeout(timeout)
+                .setWaitForActiveShards(activeShardCount).setRetryBackoffInitialTime(retryBackoffInitialTime).setMaxRetries(maxRetries)
+                // Parent task will store result
+                .setShouldStoreResult(false)
+                // Split requests per second between all slices
+                .setRequestsPerSecond(requestsPerSecond / slices)
+                // Size is split between workers. This means the size might round down!
+                .setSize(size == SIZE_ALL_MATCHES ? SIZE_ALL_MATCHES : size / slices)
+                // Sub requests don't have workers
+                .setSlices(1);
+        // Set the parent task so this task is cancelled if we cancel the parent
+        request.setParentTask(slicingTask);
+        // TODO It'd be nice not to refresh on every slice. Instead we should refresh after the sub requests finish.
+        return request;
+    }
+
+    @Override
+    public Task createTask(long id, String type, String action, TaskId parentTaskId) {
+        if (slices > 1) {
+            return new ParentBulkByScrollTask(id, type, action, getDescription(), parentTaskId, slices);
+        }
+        /* Extract the slice from the search request so it'll be available in the status. This is potentially useful for users that manually
+         * slice their search requests so they can keep track of it and **absolutely** useful for automatically sliced reindex requests so
+         * they can properly track the responses. */
+        Integer sliceId = searchRequest.source().slice() == null ? null : searchRequest.source().slice().getId();
+        return new WorkingBulkByScrollTask(id, type, action, getDescription(), parentTaskId, sliceId, requestsPerSecond);
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        searchRequest = new SearchRequest();
+        searchRequest.readFrom(in);
+        abortOnVersionConflict = in.readBoolean();
+        size = in.readVInt();
+        refresh = in.readBoolean();
+        timeout = new TimeValue(in);
+        activeShardCount = ActiveShardCount.readFrom(in);
+        retryBackoffInitialTime = new TimeValue(in);
+        maxRetries = in.readVInt();
+        requestsPerSecond = in.readFloat();
+        if (in.getVersion().onOrAfter(Version.V_5_1_1)) {
+            slices = in.readVInt();
+        } else {
+            slices = 1;
+        }
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        searchRequest.writeTo(out);
+        out.writeBoolean(abortOnVersionConflict);
+        out.writeVInt(size);
+        out.writeBoolean(refresh);
+        timeout.writeTo(out);
+        activeShardCount.writeTo(out);
+        retryBackoffInitialTime.writeTo(out);
+        out.writeVInt(maxRetries);
+        out.writeFloat(requestsPerSecond);
+        if (out.getVersion().onOrAfter(Version.V_5_1_1)) {
+            out.writeVInt(slices);
+        } else {
+            if (slices > 1) {
+                throw new IllegalArgumentException("Attempting to send sliced reindex-style request to a node that doesn't support "
+                        + "it. Version is [" + out.getVersion() + "] but must be [" + Version.V_5_1_1 + "]");
+            }
+        }
+    }
+
+    /**
+     * Append a short description of the search request to a StringBuilder. Used
+     * to make toString.
+     */
+    protected void searchToString(StringBuilder b) {
+        if (searchRequest.indices() != null && searchRequest.indices().length != 0) {
+            b.append(Arrays.toString(searchRequest.indices()));
+        } else {
+            b.append("[all indices]");
+        }
+        if (searchRequest.types() != null && searchRequest.types().length != 0) {
+            b.append(Arrays.toString(searchRequest.types()));
+        }
+    }
+
+    @Override
+    public String getDescription() {
+        return this.toString();
+    }
+}
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java b/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java
similarity index 91%
rename from modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java
rename to core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java
index 4d9c779f4875c..de3f22f0943a8 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java
@@ -31,11 +31,11 @@
 public abstract class AbstractBulkByScrollRequestBuilder<
                 Request extends AbstractBulkByScrollRequest<Request>,
                 Self extends AbstractBulkByScrollRequestBuilder<Request, Self>>
-        extends ActionRequestBuilder<Request, BulkIndexByScrollResponse, Self> {
+        extends ActionRequestBuilder<Request, BulkByScrollResponse, Self> {
     private final SearchRequestBuilder source;
 
     protected AbstractBulkByScrollRequestBuilder(ElasticsearchClient client,
-            Action<Request, BulkIndexByScrollResponse, Self> action, SearchRequestBuilder source, Request request) {
+            Action<Request, BulkByScrollResponse, Self> action, SearchRequestBuilder source, Request request) {
         super(client, action, request);
         this.source = source;
     }
@@ -141,4 +141,12 @@ public Self setShouldStoreResult(boolean shouldStoreResult) {
         request.setShouldStoreResult(shouldStoreResult);
         return self();
     }
+
+    /**
+     * The number of slices this task should be divided into. Defaults to 1 meaning the task isn't sliced into subtasks.
+     */
+    public Self setSlices(int workers) {
+        request.setSlices(workers);
+        return self();
+    }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequest.java b/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequest.java
similarity index 76%
rename from modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequest.java
rename to core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequest.java
index 10fb0bc676e11..62c2635b301c5 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequest.java
+++ b/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequest.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.script.Script;
+import org.elasticsearch.tasks.TaskId;
 
 import java.io.IOException;
 
@@ -34,11 +35,21 @@ public abstract class AbstractBulkIndexByScrollRequest<Self extends AbstractBulk
      */
     private Script script;
 
+    /**
+     * Constructor for deserialization.
+     */
     public AbstractBulkIndexByScrollRequest() {
     }
 
-    public AbstractBulkIndexByScrollRequest(SearchRequest source) {
-        super(source);
+    /**
+     * Constructor for actual use.
+     *
+     * @param searchRequest the search request to execute to get the documents to process
+     * @param setDefaults should this request set the defaults on the search request? Usually set to true but leave it false to support
+     *        request slicing
+     */
+    protected AbstractBulkIndexByScrollRequest(SearchRequest searchRequest, boolean setDefaults) {
+        super(searchRequest, setDefaults);
     }
 
     /**
@@ -56,6 +67,11 @@ public Self setScript(@Nullable Script script) {
         return self();
     }
 
+    @Override
+    protected Self doForSlice(Self request, TaskId slicingTask) {
+        return super.doForSlice(request, slicingTask).setScript(script);
+    }
+
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequestBuilder.java b/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequestBuilder.java
similarity index 93%
rename from modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequestBuilder.java
rename to core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequestBuilder.java
index e1b32baa4af0d..ca1c980995bba 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequestBuilder.java
@@ -30,7 +30,7 @@ public abstract class AbstractBulkIndexByScrollRequestBuilder<
         extends AbstractBulkByScrollRequestBuilder<Request, Self> {
 
     protected AbstractBulkIndexByScrollRequestBuilder(ElasticsearchClient client,
-            Action<Request, BulkIndexByScrollResponse, Self> action, SearchRequestBuilder search, Request request) {
+            Action<Request, BulkByScrollResponse, Self> action, SearchRequestBuilder search, Request request) {
         super(client, action, search, request);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/reindex/BulkByScrollResponse.java b/core/src/main/java/org/elasticsearch/index/reindex/BulkByScrollResponse.java
new file mode 100644
index 0000000000000..400baf7b9e2c6
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/reindex/BulkByScrollResponse.java
@@ -0,0 +1,201 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static java.lang.Math.max;
+import static java.lang.Math.min;
+import static java.util.Objects.requireNonNull;
+import static org.elasticsearch.common.unit.TimeValue.timeValueNanos;
+
+/**
+ * Response used for actions that index many documents using a scroll request.
+ */
+public class BulkByScrollResponse extends ActionResponse implements ToXContent {
+    private TimeValue took;
+    private BulkByScrollTask.Status status;
+    private List<Failure> bulkFailures;
+    private List<ScrollableHitSource.SearchFailure> searchFailures;
+    private boolean timedOut;
+
+    public BulkByScrollResponse() {
+    }
+
+    public BulkByScrollResponse(TimeValue took, BulkByScrollTask.Status status, List<Failure> bulkFailures,
+                                List<ScrollableHitSource.SearchFailure> searchFailures, boolean timedOut) {
+        this.took = took;
+        this.status = requireNonNull(status, "Null status not supported");
+        this.bulkFailures = bulkFailures;
+        this.searchFailures = searchFailures;
+        this.timedOut = timedOut;
+    }
+
+    public BulkByScrollResponse(Iterable<BulkByScrollResponse> toMerge, @Nullable String reasonCancelled) {
+        long mergedTook = 0;
+        List<BulkByScrollTask.StatusOrException> statuses = new ArrayList<>();
+        bulkFailures = new ArrayList<>();
+        searchFailures = new ArrayList<>();
+        for (BulkByScrollResponse response : toMerge) {
+            mergedTook = max(mergedTook, response.getTook().nanos());
+            statuses.add(new BulkByScrollTask.StatusOrException(response.status));
+            bulkFailures.addAll(response.getBulkFailures());
+            searchFailures.addAll(response.getSearchFailures());
+            timedOut |= response.isTimedOut();
+        }
+        took = timeValueNanos(mergedTook);
+        status = new BulkByScrollTask.Status(statuses, reasonCancelled);
+    }
+
+    public TimeValue getTook() {
+        return took;
+    }
+
+    public BulkByScrollTask.Status getStatus() {
+        return status;
+    }
+
+    public long getCreated() {
+        return status.getCreated();
+    }
+
+    public long getDeleted() {
+        return status.getDeleted();
+    }
+
+    public long getUpdated() {
+        return status.getUpdated();
+    }
+
+    public int getBatches() {
+        return status.getBatches();
+    }
+
+    public long getVersionConflicts() {
+        return status.getVersionConflicts();
+    }
+
+    public long getNoops() {
+        return status.getNoops();
+    }
+
+    /**
+     * The reason that the request was canceled or null if it hasn't been.
+     */
+    public String getReasonCancelled() {
+        return status.getReasonCancelled();
+    }
+
+    /**
+     * The number of times that the request had retry bulk actions.
+     */
+    public long getBulkRetries() {
+        return status.getBulkRetries();
+    }
+
+    /**
+     * The number of times that the request had retry search actions.
+     */
+    public long getSearchRetries() {
+        return status.getSearchRetries();
+    }
+
+    /**
+     * All of the bulk failures. Version conflicts are only included if the request sets abortOnVersionConflict to true (the default).
+     */
+    public List<Failure> getBulkFailures() {
+        return bulkFailures;
+    }
+
+    /**
+     * All search failures.
+     */
+    public List<ScrollableHitSource.SearchFailure> getSearchFailures() {
+        return searchFailures;
+    }
+
+    /**
+     * Did any of the sub-requests that were part of this request timeout?
+     */
+    public boolean isTimedOut() {
+        return timedOut;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        took.writeTo(out);
+        status.writeTo(out);
+        out.writeList(bulkFailures);
+        out.writeList(searchFailures);
+        out.writeBoolean(timedOut);
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        took = new TimeValue(in);
+        status = new BulkByScrollTask.Status(in);
+        bulkFailures = in.readList(Failure::new);
+        searchFailures = in.readList(ScrollableHitSource.SearchFailure::new);
+        timedOut = in.readBoolean();
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.field("took", took.millis());
+        builder.field("timed_out", timedOut);
+        status.innerXContent(builder, params);
+        builder.startArray("failures");
+        for (Failure failure: bulkFailures) {
+            builder.startObject();
+            failure.toXContent(builder, params);
+            builder.endObject();
+        }
+        for (ScrollableHitSource.SearchFailure failure: searchFailures) {
+            failure.toXContent(builder, params);
+        }
+        builder.endArray();
+        return builder;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder builder = new StringBuilder();
+        builder.append("BulkIndexByScrollResponse[");
+        builder.append("took=").append(took).append(',');
+        builder.append("timed_out=").append(timedOut).append(',');
+        status.innerToString(builder);
+        builder.append(",bulk_failures=").append(getBulkFailures().subList(0, min(3, getBulkFailures().size())));
+        builder.append(",search_failures=").append(getSearchFailures().subList(0, min(3, getSearchFailures().size())));
+        return builder.append(']').toString();
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java b/core/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java
new file mode 100644
index 0000000000000..284fea7a38bfc
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java
@@ -0,0 +1,518 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.tasks.CancellableTask;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskId;
+import org.elasticsearch.tasks.TaskInfo;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+
+import static java.lang.Math.min;
+import static java.util.Collections.emptyList;
+import static org.elasticsearch.common.unit.TimeValue.timeValueNanos;
+
+/**
+ * Task storing information about a currently running BulkByScroll request.
+ */
+public abstract class BulkByScrollTask extends CancellableTask {
+    public BulkByScrollTask(long id, String type, String action, String description, TaskId parentTaskId) {
+        super(id, type, action, description, parentTaskId);
+    }
+
+    /**
+     * The number of sub-slices that are still running. {@link WorkingBulkByScrollTask} will always have 0 and
+     * {@link ParentBulkByScrollTask} will return the number of waiting tasks. Used to decide how to perform rethrottling.
+     */
+    public abstract int runningSliceSubTasks();
+
+    /**
+     * Apply the {@code newRequestsPerSecond}.
+     */
+    public abstract void rethrottle(float newRequestsPerSecond);
+
+    /*
+     * Overridden to force children to return compatible status.
+     */
+    public abstract BulkByScrollTask.Status getStatus();
+
+    /**
+     * Build the status for this task given a snapshot of the information of running slices.
+     */
+    public abstract TaskInfo getInfoGivenSliceInfo(String localNodeId, List<TaskInfo> sliceInfo);
+
+    @Override
+    public boolean shouldCancelChildrenOnCancellation() {
+        return true;
+    }
+
+    public static class Status implements Task.Status, SuccessfullyProcessed {
+        public static final String NAME = "bulk-by-scroll";
+
+        /**
+         * XContent param name to indicate if "created" count must be included
+         * in the response.
+         */
+        public static final String INCLUDE_CREATED = "include_created";
+
+        /**
+         * XContent param name to indicate if "updated" count must be included
+         * in the response.
+         */
+        public static final String INCLUDE_UPDATED = "include_updated";
+
+        private final Integer sliceId;
+        private final long total;
+        private final long updated;
+        private final long created;
+        private final long deleted;
+        private final int batches;
+        private final long versionConflicts;
+        private final long noops;
+        private final long bulkRetries;
+        private final long searchRetries;
+        private final TimeValue throttled;
+        private final float requestsPerSecond;
+        private final String reasonCancelled;
+        private final TimeValue throttledUntil;
+        private final List<StatusOrException> sliceStatuses;
+
+        public Status(Integer sliceId, long total, long updated, long created, long deleted, int batches, long versionConflicts, long noops,
+                long bulkRetries, long searchRetries, TimeValue throttled, float requestsPerSecond, @Nullable String reasonCancelled,
+                TimeValue throttledUntil) {
+            this.sliceId = sliceId == null ? null : checkPositive(sliceId, "sliceId");
+            this.total = checkPositive(total, "total");
+            this.updated = checkPositive(updated, "updated");
+            this.created = checkPositive(created, "created");
+            this.deleted = checkPositive(deleted, "deleted");
+            this.batches = checkPositive(batches, "batches");
+            this.versionConflicts = checkPositive(versionConflicts, "versionConflicts");
+            this.noops = checkPositive(noops, "noops");
+            this.bulkRetries = checkPositive(bulkRetries, "bulkRetries");
+            this.searchRetries = checkPositive(searchRetries, "searchRetries");
+            this.throttled = throttled;
+            this.requestsPerSecond = requestsPerSecond;
+            this.reasonCancelled = reasonCancelled;
+            this.throttledUntil = throttledUntil;
+            this.sliceStatuses = emptyList();
+        }
+
+        /**
+         * Constructor merging many statuses.
+         *
+         * @param sliceStatuses Statuses of sub requests that this task was sliced into.
+         * @param reasonCancelled Reason that this *this* task was cancelled. Note that each entry in {@code sliceStatuses} can be cancelled
+         *        independently of this task but if this task is cancelled then the workers *should* be cancelled.
+         */
+        public Status(List<StatusOrException> sliceStatuses, @Nullable String reasonCancelled) {
+            sliceId = null;
+            this.reasonCancelled = reasonCancelled;
+
+            long mergedTotal = 0;
+            long mergedUpdated = 0;
+            long mergedCreated = 0;
+            long mergedDeleted = 0;
+            int mergedBatches = 0;
+            long mergedVersionConflicts = 0;
+            long mergedNoops = 0;
+            long mergedBulkRetries = 0;
+            long mergedSearchRetries = 0;
+            long mergedThrottled = 0;
+            float mergedRequestsPerSecond = 0;
+            long mergedThrottledUntil = Long.MAX_VALUE;
+
+            for (StatusOrException slice : sliceStatuses) {
+                if (slice == null) {
+                    // Hasn't returned yet.
+                    continue;
+                }
+                if (slice.status == null) {
+                    // This slice failed catastrophically so it doesn't count towards the status
+                    continue;
+                }
+                mergedTotal += slice.status.getTotal();
+                mergedUpdated += slice.status.getUpdated();
+                mergedCreated += slice.status.getCreated();
+                mergedDeleted += slice.status.getDeleted();
+                mergedBatches += slice.status.getBatches();
+                mergedVersionConflicts += slice.status.getVersionConflicts();
+                mergedNoops += slice.status.getNoops();
+                mergedBulkRetries += slice.status.getBulkRetries();
+                mergedSearchRetries += slice.status.getSearchRetries();
+                mergedThrottled += slice.status.getThrottled().nanos();
+                mergedRequestsPerSecond += slice.status.getRequestsPerSecond();
+                mergedThrottledUntil = min(mergedThrottledUntil, slice.status.getThrottledUntil().nanos());
+            }
+
+            total = mergedTotal;
+            updated = mergedUpdated;
+            created = mergedCreated;
+            deleted = mergedDeleted;
+            batches = mergedBatches;
+            versionConflicts = mergedVersionConflicts;
+            noops = mergedNoops;
+            bulkRetries = mergedBulkRetries;
+            searchRetries = mergedSearchRetries;
+            throttled = timeValueNanos(mergedThrottled);
+            requestsPerSecond = mergedRequestsPerSecond;
+            throttledUntil = timeValueNanos(mergedThrottledUntil == Long.MAX_VALUE ? 0 : mergedThrottledUntil);
+            this.sliceStatuses = sliceStatuses;
+        }
+
+        public Status(StreamInput in) throws IOException {
+            if (in.getVersion().onOrAfter(Version.V_5_1_1)) {
+                sliceId = in.readOptionalVInt();
+            } else {
+                sliceId = null;
+            }
+            total = in.readVLong();
+            updated = in.readVLong();
+            created = in.readVLong();
+            deleted = in.readVLong();
+            batches = in.readVInt();
+            versionConflicts = in.readVLong();
+            noops = in.readVLong();
+            bulkRetries = in.readVLong();
+            searchRetries = in.readVLong();
+            throttled = new TimeValue(in);
+            requestsPerSecond = in.readFloat();
+            reasonCancelled = in.readOptionalString();
+            throttledUntil = new TimeValue(in);
+            if (in.getVersion().onOrAfter(Version.V_5_1_1)) {
+                sliceStatuses = in.readList(stream -> stream.readOptionalWriteable(StatusOrException::new));
+            } else {
+                sliceStatuses = emptyList();
+            }
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            if (out.getVersion().onOrAfter(Version.V_5_1_1)) {
+                out.writeOptionalVInt(sliceId);
+            }
+            out.writeVLong(total);
+            out.writeVLong(updated);
+            out.writeVLong(created);
+            out.writeVLong(deleted);
+            out.writeVInt(batches);
+            out.writeVLong(versionConflicts);
+            out.writeVLong(noops);
+            out.writeVLong(bulkRetries);
+            out.writeVLong(searchRetries);
+            throttled.writeTo(out);
+            out.writeFloat(requestsPerSecond);
+            out.writeOptionalString(reasonCancelled);
+            throttledUntil.writeTo(out);
+            if (out.getVersion().onOrAfter(Version.V_5_1_1)) {
+                out.writeVInt(sliceStatuses.size());
+                for (StatusOrException sliceStatus : sliceStatuses) {
+                    out.writeOptionalWriteable(sliceStatus);
+                }
+            }
+        }
+
+        @Override
+        public String getWriteableName() {
+            return NAME;
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            innerXContent(builder, params);
+            return builder.endObject();
+        }
+
+        public XContentBuilder innerXContent(XContentBuilder builder, Params params)
+                throws IOException {
+            if (sliceId != null) {
+                builder.field("slice_id", sliceId);
+            }
+            builder.field("total", total);
+            if (params.paramAsBoolean(INCLUDE_UPDATED, true)) {
+                builder.field("updated", updated);
+            }
+            if (params.paramAsBoolean(INCLUDE_CREATED, true)) {
+                builder.field("created", created);
+            }
+            builder.field("deleted", deleted);
+            builder.field("batches", batches);
+            builder.field("version_conflicts", versionConflicts);
+            builder.field("noops", noops);
+            builder.startObject("retries"); {
+                builder.field("bulk", bulkRetries);
+                builder.field("search", searchRetries);
+            }
+            builder.endObject();
+            builder.timeValueField("throttled_millis", "throttled", throttled);
+            builder.field("requests_per_second", requestsPerSecond == Float.POSITIVE_INFINITY ? -1 : requestsPerSecond);
+            if (reasonCancelled != null) {
+                builder.field("canceled", reasonCancelled);
+            }
+            builder.timeValueField("throttled_until_millis", "throttled_until", throttledUntil);
+            if (false == sliceStatuses.isEmpty()) {
+                builder.startArray("slices");
+                for (StatusOrException slice : sliceStatuses) {
+                    if (slice == null) {
+                        builder.nullValue();
+                    } else {
+                        slice.toXContent(builder, params);
+                    }
+                }
+                builder.endArray();
+            }
+            return builder;
+        }
+
+        @Override
+        public String toString() {
+            StringBuilder builder = new StringBuilder();
+            builder.append("BulkIndexByScrollResponse[");
+            innerToString(builder);
+            return builder.append(']').toString();
+        }
+
+        public void innerToString(StringBuilder builder) {
+            builder.append("sliceId=").append(sliceId);
+            builder.append(",updated=").append(updated);
+            builder.append(",created=").append(created);
+            builder.append(",deleted=").append(deleted);
+            builder.append(",batches=").append(batches);
+            builder.append(",versionConflicts=").append(versionConflicts);
+            builder.append(",noops=").append(noops);
+            builder.append(",retries=").append(bulkRetries);
+            if (reasonCancelled != null) {
+                builder.append(",canceled=").append(reasonCancelled);
+            }
+            builder.append(",throttledUntil=").append(throttledUntil);
+            if (false == sliceStatuses.isEmpty()) {
+                builder.append(",workers=").append(sliceStatuses);
+            }
+        }
+
+        /**
+         * The id of the slice that this status is reporting or {@code null} if this isn't the status of a sub-slice.
+         */
+        Integer getSliceId() {
+            return sliceId;
+        }
+
+        /**
+         * The total number of documents this request will process. 0 means we don't yet know or, possibly, there are actually 0 documents
+         * to process. Its ok that these have the same meaning because any request with 0 actual documents should be quite short lived.
+         */
+        public long getTotal() {
+            return total;
+        }
+
+        @Override
+        public long getUpdated() {
+            return updated;
+        }
+
+        @Override
+        public long getCreated() {
+            return created;
+        }
+
+        @Override
+        public long getDeleted() {
+            return deleted;
+        }
+
+        /**
+         * Number of scan responses this request has processed.
+         */
+        public int getBatches() {
+            return batches;
+        }
+
+        /**
+         * Number of version conflicts this request has hit.
+         */
+        public long getVersionConflicts() {
+            return versionConflicts;
+        }
+
+        /**
+         * Number of noops (skipped bulk items) as part of this request.
+         */
+        public long getNoops() {
+            return noops;
+        }
+
+        /**
+         * Number of retries that had to be attempted due to bulk actions being rejected.
+         */
+        public long getBulkRetries() {
+            return bulkRetries;
+        }
+
+        /**
+         * Number of retries that had to be attempted due to search actions being rejected.
+         */
+        public long getSearchRetries() {
+            return searchRetries;
+        }
+
+        /**
+         * The total time this request has throttled itself not including the current throttle time if it is currently sleeping.
+         */
+        public TimeValue getThrottled() {
+            return throttled;
+        }
+
+        /**
+         * The number of requests per second to which to throttle the request. Float.POSITIVE_INFINITY means unlimited.
+         */
+        public float getRequestsPerSecond() {
+            return requestsPerSecond;
+        }
+
+        /**
+         * The reason that the request was canceled or null if it hasn't been.
+         */
+        public String getReasonCancelled() {
+            return reasonCancelled;
+        }
+
+        /**
+         * Remaining delay of any current throttle sleep or 0 if not sleeping.
+         */
+        public TimeValue getThrottledUntil() {
+            return throttledUntil;
+        }
+
+        /**
+         * Statuses of the sub requests into which this sub-request was sliced. Empty if this request wasn't sliced into sub-requests.
+         */
+        public List<StatusOrException> getSliceStatuses() {
+            return sliceStatuses;
+        }
+
+        private int checkPositive(int value, String name) {
+            if (value < 0) {
+                throw new IllegalArgumentException(name + " must be greater than 0 but was [" + value + "]");
+            }
+            return value;
+        }
+
+        private long checkPositive(long value, String name) {
+            if (value < 0) {
+                throw new IllegalArgumentException(name + " must be greater than 0 but was [" + value + "]");
+            }
+            return value;
+        }
+    }
+
+    /**
+     * The status of a slice of the request. Successful requests store the {@link StatusOrException#status} while failing requests store a
+     * {@link StatusOrException#exception}.
+     */
+    public static class StatusOrException implements Writeable, ToXContent {
+        private final Status status;
+        private final Exception exception;
+
+        public StatusOrException(Status status) {
+            this.status = status;
+            exception = null;
+        }
+
+        public StatusOrException(Exception exception) {
+            status = null;
+            this.exception = exception;
+        }
+
+        /**
+         * Read from a stream.
+         */
+        public StatusOrException(StreamInput in) throws IOException {
+            if (in.readBoolean()) {
+                status = new Status(in);
+                exception = null;
+            } else {
+                status = null;
+                exception = in.readException();
+            }
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            if (exception == null) {
+                out.writeBoolean(true);
+                status.writeTo(out);
+            } else {
+                out.writeBoolean(false);
+                out.writeException(exception);
+            }
+        }
+
+        public Status getStatus() {
+            return status;
+        }
+
+        public Exception getException() {
+            return exception;
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            if (exception == null) {
+                status.toXContent(builder, params);
+            } else {
+                builder.startObject();
+                ElasticsearchException.generateThrowableXContent(builder, params, exception);
+                builder.endObject();
+            }
+            return builder;
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null) {
+                return false;
+            }
+            if (obj.getClass() != BulkByScrollTask.StatusOrException.class) {
+                return false;
+            }
+            BulkByScrollTask.StatusOrException other = (StatusOrException) obj;
+            return Objects.equals(status, other.status)
+                    && Objects.equals(exception, other.exception);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(status, exception);
+        }
+    }
+
+}
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ClientScrollableHitSource.java b/core/src/main/java/org/elasticsearch/index/reindex/ClientScrollableHitSource.java
similarity index 85%
rename from modules/reindex/src/main/java/org/elasticsearch/index/reindex/ClientScrollableHitSource.java
rename to core/src/main/java/org/elasticsearch/index/reindex/ClientScrollableHitSource.java
index 4d5f762340014..53899757484a3 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ClientScrollableHitSource.java
+++ b/core/src/main/java/org/elasticsearch/index/reindex/ClientScrollableHitSource.java
@@ -37,6 +37,8 @@
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.mapper.ParentFieldMapper;
 import org.elasticsearch.index.mapper.RoutingFieldMapper;
 import org.elasticsearch.index.mapper.TTLFieldMapper;
@@ -81,18 +83,16 @@ public void doStart(Consumer<? super Response> onResponse) {
 
     @Override
     protected void doStartNextScroll(String scrollId, TimeValue extraKeepAlive, Consumer<? super Response> onResponse) {
-        SearchScrollRequest request = new SearchScrollRequest();
-        // Add the wait time into the scroll timeout so it won't timeout while we wait for throttling
-        request.scrollId(scrollId).scroll(timeValueNanos(firstSearchRequest.scroll().keepAlive().nanos() + extraKeepAlive.nanos()));
-        searchWithRetry(listener -> client.searchScroll(request, listener), r -> consume(r, onResponse));
+        searchWithRetry(listener -> {
+            SearchScrollRequest request = new SearchScrollRequest();
+            // Add the wait time into the scroll timeout so it won't timeout while we wait for throttling
+            request.scrollId(scrollId).scroll(timeValueNanos(firstSearchRequest.scroll().keepAlive().nanos() + extraKeepAlive.nanos()));
+            client.searchScroll(request, listener);
+        }, r -> consume(r, onResponse));
     }
 
     @Override
-    public void clearScroll(String scrollId) {
-        /*
-         * Fire off the clear scroll but don't wait for it it return before
-         * we send the use their response.
-         */
+    public void clearScroll(String scrollId, Runnable onCompletion) {
         ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
         clearScrollRequest.addScrollId(scrollId);
         /*
@@ -103,15 +103,22 @@ public void clearScroll(String scrollId) {
             @Override
             public void onResponse(ClearScrollResponse response) {
                 logger.debug("Freed [{}] contexts", response.getNumFreed());
+                onCompletion.run();
             }
 
             @Override
             public void onFailure(Exception e) {
                 logger.warn((Supplier<?>) () -> new ParameterizedMessage("Failed to clear scroll [{}]", scrollId), e);
+                onCompletion.run();
             }
         });
     }
 
+    @Override
+    protected void cleanup(Runnable onCompletion) {
+        onCompletion.run();
+    }
+
     /**
      * Run a search action and call onResponse when a the response comes in, retrying if the action fails with an exception caused by
      * rejected execution.
@@ -128,6 +135,10 @@ private void searchWithRetry(Consumer<ActionListener<SearchResponse>> action, Co
          */
         class RetryHelper extends AbstractRunnable implements ActionListener<SearchResponse> {
             private final Iterator<TimeValue> retries = backoffPolicy.iterator();
+            /**
+             * The runnable to run that retries in the same context as the original call.
+             */
+            private Runnable retryWithContext;
             private volatile int retryCount = 0;
 
             @Override
@@ -148,7 +159,7 @@ public void onFailure(Exception e) {
                         TimeValue delay = retries.next();
                         logger.trace((Supplier<?>) () -> new ParameterizedMessage("retrying rejected search after [{}]", delay), e);
                         countSearchRetry.run();
-                        threadPool.schedule(delay, ThreadPool.Names.SAME, this);
+                        threadPool.schedule(delay, ThreadPool.Names.SAME, retryWithContext);
                     } else {
                         logger.warn(
                             (Supplier<?>) () -> new ParameterizedMessage(
@@ -161,7 +172,10 @@ public void onFailure(Exception e) {
                 }
             }
         }
-        new RetryHelper().run();
+        RetryHelper helper = new RetryHelper();
+        // Wrap the helper in a runnable that preserves the current context so we keep it on retry.
+        helper.retryWithContext = threadPool.getThreadContext().preserveContext(helper);
+        helper.run();
     }
 
     private void consume(SearchResponse response, Consumer<? super Response> onResponse) {
@@ -175,7 +189,7 @@ private Response wrap(SearchResponse response) {
         } else {
             failures = new ArrayList<>(response.getShardFailures().length);
             for (ShardSearchFailure failure: response.getShardFailures()) {
-                String nodeId = failure.shard() == null ? null : failure.shard().nodeId();
+                String nodeId = failure.shard() == null ? null : failure.shard().getNodeId();
                 failures.add(new SearchFailure(failure.getCause(), failure.index(), failure.shardId(), nodeId));
             }
         }
@@ -197,9 +211,9 @@ private static class ClientHit implements Hit {
         private final SearchHit delegate;
         private final BytesReference source;
 
-        public ClientHit(SearchHit delegate) {
+        ClientHit(SearchHit delegate) {
             this.delegate = delegate;
-            source = delegate.hasSource() ? null : delegate.getSourceRef();
+            source = delegate.hasSource() ? delegate.getSourceRef() : null;
         }
 
         @Override
@@ -222,6 +236,10 @@ public BytesReference getSource() {
             return source;
         }
 
+        @Override
+        public XContentType getXContentType() {
+            return XContentFactory.xContentType(source);
+        }
         @Override
         public long getVersion() {
             return delegate.getVersion();
@@ -248,8 +266,8 @@ public Long getTTL() {
         }
 
         private <T> T fieldValue(String fieldName) {
-            SearchHitField field = delegate.field(fieldName);
-            return field == null ? null : field.value();
+            SearchHitField field = delegate.getField(fieldName);
+            return field == null ? null : field.getValue();
         }
     }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryAction.java b/core/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryAction.java
similarity index 89%
rename from modules/reindex/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryAction.java
rename to core/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryAction.java
index c789e9c77b4a6..c1abb16ca3977 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryAction.java
+++ b/core/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryAction.java
@@ -22,7 +22,7 @@
 import org.elasticsearch.action.Action;
 import org.elasticsearch.client.ElasticsearchClient;
 
-public class DeleteByQueryAction extends Action<DeleteByQueryRequest, BulkIndexByScrollResponse, DeleteByQueryRequestBuilder> {
+public class DeleteByQueryAction extends Action<DeleteByQueryRequest, BulkByScrollResponse, DeleteByQueryRequestBuilder> {
 
     public static final DeleteByQueryAction INSTANCE = new DeleteByQueryAction();
     public static final String NAME = "indices:data/write/delete/byquery";
@@ -37,7 +37,7 @@ public DeleteByQueryRequestBuilder newRequestBuilder(ElasticsearchClient client)
     }
 
     @Override
-    public BulkIndexByScrollResponse newResponse() {
-        return new BulkIndexByScrollResponse();
+    public BulkByScrollResponse newResponse() {
+        return new BulkByScrollResponse();
     }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequest.java b/core/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequest.java
similarity index 77%
rename from modules/reindex/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequest.java
rename to core/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequest.java
index df1f4d387abd7..cc010fd77bb44 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequest.java
+++ b/core/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequest.java
@@ -23,6 +23,9 @@
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.tasks.TaskId;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 
@@ -45,13 +48,21 @@
  */
 public class DeleteByQueryRequest extends AbstractBulkByScrollRequest<DeleteByQueryRequest> implements IndicesRequest.Replaceable {
 
+    private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(DeleteByQueryRequest.class));
+
     public DeleteByQueryRequest() {
     }
 
     public DeleteByQueryRequest(SearchRequest search) {
-        super(search);
+        this(search, true);
+    }
+
+    private DeleteByQueryRequest(SearchRequest search, boolean setDefaults) {
+        super(search, setDefaults);
         // Delete-By-Query does not require the source
-        search.source().fetchSource(false);
+        if (setDefaults) {
+            search.source().fetchSource(false);
+        }
     }
 
     @Override
@@ -67,10 +78,18 @@ public ActionRequestValidationException validate() {
         }
         if (getSearchRequest() == null || getSearchRequest().source() == null) {
             e = addValidationError("source is missing", e);
+        } else if (getSearchRequest().source().query() == null) {
+            DEPRECATION_LOGGER.deprecated("A request to _delete_by_query without an explicit " +
+                    "query is deprecated. Specify a query explicitly instead.");
         }
         return e;
     }
 
+    @Override
+    public DeleteByQueryRequest forSlice(TaskId slicingTask, SearchRequest slice) {
+        return doForSlice(new DeleteByQueryRequest(slice, false), slicingTask);
+    }
+
     @Override
     public String toString() {
         StringBuilder b = new StringBuilder();
@@ -99,4 +118,16 @@ public IndicesOptions indicesOptions() {
         assert getSearchRequest() != null;
         return getSearchRequest().indicesOptions();
     }
+
+    public String[] types() {
+        assert getSearchRequest() != null;
+        return getSearchRequest().types();
+    }
+
+    public DeleteByQueryRequest types(String... types) {
+        assert getSearchRequest() != null;
+        getSearchRequest().types(types);
+        return this;
+    }
+
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequestBuilder.java b/core/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequestBuilder.java
similarity index 93%
rename from modules/reindex/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequestBuilder.java
rename to core/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequestBuilder.java
index eb528793236ba..e94d1308a74be 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequestBuilder.java
@@ -28,12 +28,12 @@ public class DeleteByQueryRequestBuilder extends
         AbstractBulkByScrollRequestBuilder<DeleteByQueryRequest, DeleteByQueryRequestBuilder> {
 
     public DeleteByQueryRequestBuilder(ElasticsearchClient client,
-                                       Action<DeleteByQueryRequest, BulkIndexByScrollResponse, DeleteByQueryRequestBuilder> action) {
+                                       Action<DeleteByQueryRequest, BulkByScrollResponse, DeleteByQueryRequestBuilder> action) {
         this(client, action, new SearchRequestBuilder(client, SearchAction.INSTANCE));
     }
 
     private DeleteByQueryRequestBuilder(ElasticsearchClient client,
-                                        Action<DeleteByQueryRequest, BulkIndexByScrollResponse, DeleteByQueryRequestBuilder> action,
+                                        Action<DeleteByQueryRequest, BulkByScrollResponse, DeleteByQueryRequestBuilder> action,
                                         SearchRequestBuilder search) {
         super(client, action, search, new DeleteByQueryRequest(search.request()));
     }
diff --git a/core/src/main/java/org/elasticsearch/index/reindex/ParentBulkByScrollTask.java b/core/src/main/java/org/elasticsearch/index/reindex/ParentBulkByScrollTask.java
new file mode 100644
index 0000000000000..bea9bc203653d
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/reindex/ParentBulkByScrollTask.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.util.concurrent.AtomicArray;
+import org.elasticsearch.tasks.TaskId;
+import org.elasticsearch.tasks.TaskInfo;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static java.util.Collections.unmodifiableList;
+
+/**
+ * Task for parent bulk by scroll requests that have sub-workers.
+ */
+public class ParentBulkByScrollTask extends BulkByScrollTask {
+    /**
+     * Holds the responses as they come back. This uses {@link Tuple} as an "Either" style holder where only the response or the exception
+     * is set.
+     */
+    private final AtomicArray<Result> results;
+    private final AtomicInteger counter;
+
+    public ParentBulkByScrollTask(long id, String type, String action, String description, TaskId parentTaskId, int slices) {
+        super(id, type, action, description, parentTaskId);
+        this.results = new AtomicArray<>(slices);
+        this.counter  = new AtomicInteger(slices);
+    }
+
+    @Override
+    public void rethrottle(float newRequestsPerSecond) {
+        // Nothing to do because all rethrottling is done on slice sub tasks.
+    }
+
+    @Override
+    public Status getStatus() {
+        // We only have access to the statuses of requests that have finished so we return them
+        List<StatusOrException> statuses = Arrays.asList(new StatusOrException[results.length()]);
+        addResultsToList(statuses);
+        return new Status(unmodifiableList(statuses), getReasonCancelled());
+    }
+
+    @Override
+    public int runningSliceSubTasks() {
+        return counter.get();
+    }
+
+    @Override
+    public TaskInfo getInfoGivenSliceInfo(String localNodeId, List<TaskInfo> sliceInfo) {
+        /* Merge the list of finished sub requests with the provided info. If a slice is both finished and in the list then we prefer the
+         * finished status because we don't expect them to change after the task is finished. */
+        List<StatusOrException> sliceStatuses = Arrays.asList(new StatusOrException[results.length()]);
+        for (TaskInfo t : sliceInfo) {
+            Status status = (Status) t.getStatus();
+            sliceStatuses.set(status.getSliceId(), new StatusOrException(status));
+        }
+        addResultsToList(sliceStatuses);
+        Status status = new Status(sliceStatuses, getReasonCancelled());
+        return taskInfo(localNodeId, getDescription(), status);
+    }
+
+    private void addResultsToList(List<StatusOrException> sliceStatuses) {
+        for (Result t : results.asList()) {
+            if (t.response != null) {
+                sliceStatuses.set(t.sliceId, new StatusOrException(t.response.getStatus()));
+            } else {
+                sliceStatuses.set(t.sliceId, new StatusOrException(t.failure));
+            }
+        }
+    }
+
+    /**
+     * Record a response from a slice and respond to the listener if the request is finished.
+     */
+    public void onSliceResponse(ActionListener<BulkByScrollResponse> listener, int sliceId, BulkByScrollResponse response) {
+        results.setOnce(sliceId, new Result(sliceId, response));
+        /* If the request isn't finished we could automatically rethrottle the sub-requests here but we would only want to do that if we
+         * were fairly sure they had a while left to go. */
+        recordSliceCompletionAndRespondIfAllDone(listener);
+    }
+
+    /**
+     * Record a failure from a slice and respond to the listener if the request is finished.
+     */
+    public void onSliceFailure(ActionListener<BulkByScrollResponse> listener, int sliceId, Exception e) {
+        results.setOnce(sliceId, new Result(sliceId, e));
+        recordSliceCompletionAndRespondIfAllDone(listener);
+        // TODO cancel when a slice fails?
+    }
+
+    private void recordSliceCompletionAndRespondIfAllDone(ActionListener<BulkByScrollResponse> listener) {
+        if (counter.decrementAndGet() != 0) {
+            return;
+        }
+        List<BulkByScrollResponse> responses = new ArrayList<>(results.length());
+        Exception exception = null;
+        for (Result t : results.asList()) {
+            if (t.response == null) {
+                assert t.failure != null : "exception shouldn't be null if value is null";
+                if (exception == null) {
+                    exception = t.failure;
+                } else {
+                    exception.addSuppressed(t.failure);
+                }
+            } else {
+                assert t.failure == null : "exception should be null if response is not null";
+                responses.add(t.response);
+            }
+        }
+        if (exception == null) {
+            listener.onResponse(new BulkByScrollResponse(responses, getReasonCancelled()));
+        } else {
+            listener.onFailure(exception);
+        }
+    }
+
+    private static final class Result {
+        final BulkByScrollResponse response;
+        final int sliceId;
+        final Exception failure;
+
+        private Result(int sliceId, BulkByScrollResponse response) {
+            this.sliceId = sliceId;
+            this.response = response;
+            failure = null;
+        }
+
+        private Result(int sliceId, Exception failure) {
+            this.sliceId = sliceId;
+            this.failure = failure;
+            response = null;
+        }
+    }
+}
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexAction.java b/core/src/main/java/org/elasticsearch/index/reindex/ReindexAction.java
similarity index 86%
rename from modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexAction.java
rename to core/src/main/java/org/elasticsearch/index/reindex/ReindexAction.java
index aa16a859c1deb..1c53a925f0d71 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexAction.java
+++ b/core/src/main/java/org/elasticsearch/index/reindex/ReindexAction.java
@@ -22,7 +22,7 @@
 import org.elasticsearch.action.Action;
 import org.elasticsearch.client.ElasticsearchClient;
 
-public class ReindexAction extends Action<ReindexRequest, BulkIndexByScrollResponse, ReindexRequestBuilder> {
+public class ReindexAction extends Action<ReindexRequest, BulkByScrollResponse, ReindexRequestBuilder> {
     public static final ReindexAction INSTANCE = new ReindexAction();
     public static final String NAME = "indices:data/write/reindex";
 
@@ -36,7 +36,7 @@ public ReindexRequestBuilder newRequestBuilder(ElasticsearchClient client) {
     }
 
     @Override
-    public BulkIndexByScrollResponse newResponse() {
-        return new BulkIndexByScrollResponse();
+    public BulkByScrollResponse newResponse() {
+        return new BulkByScrollResponse();
     }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java b/core/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java
similarity index 77%
rename from modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java
rename to core/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java
index 8c11cd3430ff2..9fed5500d1664 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java
+++ b/core/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java
@@ -21,20 +21,15 @@
 
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.CompositeIndicesRequest;
-import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lucene.uid.Versions;
-import org.elasticsearch.index.reindex.remote.RemoteInfo;
+import org.elasticsearch.tasks.TaskId;
 
 import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
 
-import static java.util.Collections.singletonList;
-import static java.util.Collections.unmodifiableList;
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 import static org.elasticsearch.index.VersionType.INTERNAL;
 
@@ -56,7 +51,11 @@ public ReindexRequest() {
     }
 
     public ReindexRequest(SearchRequest search, IndexRequest destination) {
-        super(search);
+        this(search, destination, true);
+    }
+
+    private ReindexRequest(SearchRequest search, IndexRequest destination, boolean setDefaults) {
+        super(search, setDefaults);
         this.destination = destination;
     }
 
@@ -71,6 +70,9 @@ public ActionRequestValidationException validate() {
         if (getSearchRequest().indices() == null || getSearchRequest().indices().length == 0) {
             e = addValidationError("use _all if you really want to copy from all existing indexes", e);
         }
+        if (getSearchRequest().source().fetchSource() != null && getSearchRequest().source().fetchSource().fetchSource() == false) {
+            e = addValidationError("_source:false is not supported in this context", e);
+        }
         /*
          * Note that we don't call index's validator - it won't work because
          * we'll be filling in portions of it as we receive the docs. But we can
@@ -94,8 +96,13 @@ public ActionRequestValidationException validate() {
         if (destination.timestamp() != null) {
             e = addValidationError("setting timestamp on destination isn't supported. use scripts instead.", e);
         }
-        if (getRemoteInfo() != null && getSearchRequest().source().query() != null) {
-            e = addValidationError("reindex from remote sources should use RemoteInfo's query instead of source's query", e);
+        if (getRemoteInfo() != null) {
+            if (getSearchRequest().source().query() != null) {
+                e = addValidationError("reindex from remote sources should use RemoteInfo's query instead of source's query", e);
+            }
+            if (getSlices() != 1) {
+                e = addValidationError("reindex from remote sources doesn't support workers > 1 but was [" + getSlices() + "]", e);
+            }
         }
         return e;
     }
@@ -125,6 +132,13 @@ public RemoteInfo getRemoteInfo() {
         return remoteInfo;
     }
 
+    @Override
+    public ReindexRequest forSlice(TaskId slicingTask, SearchRequest slice) {
+        ReindexRequest sliced = doForSlice(new ReindexRequest(slice, destination, false), slicingTask);
+        sliced.setRemoteInfo(remoteInfo);
+        return sliced;
+    }
+
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
@@ -154,23 +168,4 @@ public String toString() {
         }
         return b.toString();
     }
-
-    // CompositeIndicesRequest implementation so plugins can reason about the request. This is really just a best effort thing.
-    /**
-     * Accessor to get the underlying {@link IndicesRequest}s that this request wraps. Note that this method is <strong>not
-     * accurate</strong> since it returns a prototype {@link IndexRequest} and not the actual requests that will be issued as part of the
-     * execution of this request. Additionally, scripts can modify the underlying {@link IndexRequest} and change values such as the index,
-     * type, {@link org.elasticsearch.action.support.IndicesOptions}. In short - only use this for very course reasoning about the request.
-     *
-     * @return a list comprising of the {@link SearchRequest} and the prototype {@link IndexRequest}
-     */
-    @Override
-    public List<? extends IndicesRequest> subRequests() {
-        assert getSearchRequest() != null;
-        assert getDestination() != null;
-        if (remoteInfo != null) {
-            return singletonList(getDestination());
-        }
-        return unmodifiableList(Arrays.asList(getSearchRequest(), getDestination()));
-    }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexRequestBuilder.java b/core/src/main/java/org/elasticsearch/index/reindex/ReindexRequestBuilder.java
similarity index 91%
rename from modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexRequestBuilder.java
rename to core/src/main/java/org/elasticsearch/index/reindex/ReindexRequestBuilder.java
index 1eadf2c15bc73..68bd3f4661828 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/reindex/ReindexRequestBuilder.java
@@ -25,20 +25,19 @@
 import org.elasticsearch.action.search.SearchAction;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
-import org.elasticsearch.index.reindex.remote.RemoteInfo;
 
 public class ReindexRequestBuilder extends
         AbstractBulkIndexByScrollRequestBuilder<ReindexRequest, ReindexRequestBuilder> {
     private final IndexRequestBuilder destination;
 
     public ReindexRequestBuilder(ElasticsearchClient client,
-            Action<ReindexRequest, BulkIndexByScrollResponse, ReindexRequestBuilder> action) {
+            Action<ReindexRequest, BulkByScrollResponse, ReindexRequestBuilder> action) {
         this(client, action, new SearchRequestBuilder(client, SearchAction.INSTANCE),
                 new IndexRequestBuilder(client, IndexAction.INSTANCE));
     }
 
     private ReindexRequestBuilder(ElasticsearchClient client,
-            Action<ReindexRequest, BulkIndexByScrollResponse, ReindexRequestBuilder> action,
+            Action<ReindexRequest, BulkByScrollResponse, ReindexRequestBuilder> action,
             SearchRequestBuilder search, IndexRequestBuilder destination) {
         super(client, action, search, new ReindexRequest(search.request(), destination.request()));
         this.destination = destination;
diff --git a/core/src/main/java/org/elasticsearch/index/reindex/RemoteInfo.java b/core/src/main/java/org/elasticsearch/index/reindex/RemoteInfo.java
new file mode 100644
index 0000000000000..105afcc95bc38
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/reindex/RemoteInfo.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.unit.TimeValue;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static java.util.Collections.unmodifiableMap;
+import static java.util.Objects.requireNonNull;
+import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
+
+public class RemoteInfo implements Writeable {
+    /**
+     * Default {@link #socketTimeout} for requests that don't have one set.
+     */
+    public static final TimeValue DEFAULT_SOCKET_TIMEOUT = timeValueSeconds(30);
+    /**
+     * Default {@link #connectTimeout} for requests that don't have one set.
+     */
+    public static final TimeValue DEFAULT_CONNECT_TIMEOUT = timeValueSeconds(30);
+
+    private final String scheme;
+    private final String host;
+    private final int port;
+    private final BytesReference query;
+    private final String username;
+    private final String password;
+    private final Map<String, String> headers;
+    /**
+     * Time to wait for a response from each request.
+     */
+    private final TimeValue socketTimeout;
+    /**
+     * Time to wait for a connecting to the remote cluster.
+     */
+    private final TimeValue connectTimeout;
+
+    public RemoteInfo(String scheme, String host, int port, BytesReference query, String username, String password,
+            Map<String, String> headers, TimeValue socketTimeout, TimeValue connectTimeout) {
+        this.scheme = requireNonNull(scheme, "[scheme] must be specified to reindex from a remote cluster");
+        this.host = requireNonNull(host, "[host] must be specified to reindex from a remote cluster");
+        this.port = port;
+        this.query = requireNonNull(query, "[query] must be specified to reindex from a remote cluster");
+        this.username = username;
+        this.password = password;
+        this.headers = unmodifiableMap(requireNonNull(headers, "[headers] is required"));
+        this.socketTimeout = requireNonNull(socketTimeout, "[socketTimeout] must be specified");
+        this.connectTimeout = requireNonNull(connectTimeout, "[connectTimeout] must be specified");
+    }
+
+    /**
+     * Read from a stream.
+     */
+    public RemoteInfo(StreamInput in) throws IOException {
+        scheme = in.readString();
+        host = in.readString();
+        port = in.readVInt();
+        query = in.readBytesReference();
+        username = in.readOptionalString();
+        password = in.readOptionalString();
+        int headersLength = in.readVInt();
+        Map<String, String> headers = new HashMap<>(headersLength);
+        for (int i = 0; i < headersLength; i++) {
+            headers.put(in.readString(), in.readString());
+        }
+        this.headers = unmodifiableMap(headers);
+        if (in.getVersion().onOrAfter(Version.V_5_2_0)) {
+            socketTimeout = new TimeValue(in);
+            connectTimeout = new TimeValue(in);
+        } else {
+            socketTimeout = DEFAULT_SOCKET_TIMEOUT;
+            connectTimeout = DEFAULT_CONNECT_TIMEOUT;
+        }
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(scheme);
+        out.writeString(host);
+        out.writeVInt(port);
+        out.writeBytesReference(query);
+        out.writeOptionalString(username);
+        out.writeOptionalString(password);
+        out.writeVInt(headers.size());
+        for (Map.Entry<String, String> header : headers.entrySet()) {
+            out.writeString(header.getKey());
+            out.writeString(header.getValue());
+        }
+        if (out.getVersion().onOrAfter(Version.V_5_2_0)) {
+            socketTimeout.writeTo(out);
+            connectTimeout.writeTo(out);
+        }
+    }
+
+    public String getScheme() {
+        return scheme;
+    }
+
+    public String getHost() {
+        return host;
+    }
+
+    public int getPort() {
+        return port;
+    }
+
+    public BytesReference getQuery() {
+        return query;
+    }
+
+    @Nullable
+    public String getUsername() {
+        return username;
+    }
+
+    @Nullable
+    public String getPassword() {
+        return password;
+    }
+
+    public Map<String, String> getHeaders() {
+        return headers;
+    }
+
+    /**
+     * Time to wait for a response from each request.
+     */
+    public TimeValue getSocketTimeout() {
+        return socketTimeout;
+    }
+
+    /**
+     * Time to wait to connect to the external cluster.
+     */
+    public TimeValue getConnectTimeout() {
+        return connectTimeout;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder b = new StringBuilder();
+        if (false == "http".equals(scheme)) {
+            // http is the default so it isn't worth taking up space if it is the scheme
+            b.append("scheme=").append(scheme).append(' ');
+        }
+        b.append("host=").append(host).append(" port=").append(port).append(" query=").append(query.utf8ToString());
+        if (username != null) {
+            b.append(" username=").append(username);
+        }
+        if (password != null) {
+            b.append(" password=<<>>");
+        }
+        return b.toString();
+    }
+}
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ScrollableHitSource.java b/core/src/main/java/org/elasticsearch/index/reindex/ScrollableHitSource.java
similarity index 87%
rename from modules/reindex/src/main/java/org/elasticsearch/index/reindex/ScrollableHitSource.java
rename to core/src/main/java/org/elasticsearch/index/reindex/ScrollableHitSource.java
index 0b4b66222bc4b..4dd32f56a1cba 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ScrollableHitSource.java
+++ b/core/src/main/java/org/elasticsearch/index/reindex/ScrollableHitSource.java
@@ -32,7 +32,7 @@
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.index.reindex.remote.RemoteScrollableHitSource;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.threadpool.ThreadPool;
 
@@ -47,7 +47,7 @@
 /**
  * A scrollable source of results.
  */
-public abstract class ScrollableHitSource implements Closeable {
+public abstract class ScrollableHitSource {
     private final AtomicReference<String> scrollId = new AtomicReference<>();
 
     protected final Logger logger;
@@ -81,20 +81,37 @@ public final void startNextScroll(TimeValue extraKeepAlive, Consumer<Response> o
         });
     }
     protected abstract void doStartNextScroll(String scrollId, TimeValue extraKeepAlive, Consumer<? super Response> onResponse);
-    
-    @Override
-    public void close() {
+
+    public final void close(Runnable onCompletion) {
         String scrollId = this.scrollId.get();
         if (Strings.hasLength(scrollId)) {
-            clearScroll(scrollId);
+            clearScroll(scrollId, () -> cleanup(onCompletion));
+        } else {
+            cleanup(onCompletion);
         }
     }
-    protected abstract void clearScroll(String scrollId);
+
+    /**
+     * Called to clear a scroll id.
+     *
+     * @param scrollId the id to clear
+     * @param onCompletion implementers must call this after completing the clear whether they are
+     *        successful or not
+     */
+    protected abstract void clearScroll(String scrollId, Runnable onCompletion);
+    /**
+     * Called after the process has been totally finished to clean up any resources the process
+     * needed like remote connections.
+     *
+     * @param onCompletion implementers must call this after completing the cleanup whether they are
+     *        successful or not
+     */
+    protected abstract void cleanup(Runnable onCompletion);
 
     /**
      * Set the id of the last scroll. Used for debugging.
      */
-    final void setScroll(String scrollId) {
+    public final void setScroll(String scrollId) {
         this.scrollId.set(scrollId);
     }
 
@@ -179,6 +196,10 @@ public interface Hit {
          * all.
          */
         @Nullable BytesReference getSource();
+        /**
+         * The content type of the hit source. Returns null if the source didn't come back from the search.
+         */
+        @Nullable XContentType getXContentType();
         /**
          * The document id of the parent of the hit if there is a parent or null if there isn't.
          */
@@ -198,8 +219,7 @@ public interface Hit {
     }
 
     /**
-     * An implementation of {@linkplain Hit} that uses getters and setters. Primarily used for testing and {@link RemoteScrollableHitSource}
-     * .
+     * An implementation of {@linkplain Hit} that uses getters and setters.
      */
     public static class BasicHit implements Hit {
         private final String index;
@@ -208,6 +228,7 @@ public static class BasicHit implements Hit {
         private final long version;
 
         private BytesReference source;
+        private XContentType xContentType;
         private String parent;
         private String routing;
         private Long timestamp;
@@ -245,8 +266,14 @@ public BytesReference getSource() {
             return source;
         }
 
-        public BasicHit setSource(BytesReference source) {
+        @Override
+        public XContentType getXContentType() {
+            return xContentType;
+        }
+
+        public BasicHit setSource(BytesReference source, XContentType xContentType) {
             this.source = source;
+            this.xContentType = xContentType;
             return this;
         }
 
@@ -367,7 +394,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.field("reason");
             {
                 builder.startObject();
-                ElasticsearchException.toXContent(builder, params, reason);
+                ElasticsearchException.generateThrowableXContent(builder, params, reason);
                 builder.endObject();
             }
             builder.endObject();
diff --git a/core/src/main/java/org/elasticsearch/index/reindex/SuccessfullyProcessed.java b/core/src/main/java/org/elasticsearch/index/reindex/SuccessfullyProcessed.java
new file mode 100644
index 0000000000000..6547984900e4b
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/reindex/SuccessfullyProcessed.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex;
+
+/**
+ * Implemented by {@link BulkByScrollTask} and {@link BulkByScrollTask.Status} to consistently implement
+ * {@link #getSuccessfullyProcessed()}.
+ */
+public interface SuccessfullyProcessed {
+    /**
+     * Total number of successfully processed documents.
+     */
+    default long getSuccessfullyProcessed() {
+        return getUpdated() + getCreated() + getDeleted();
+    }
+
+    /**
+     * Count of documents updated.
+     */
+    long getUpdated();
+    /**
+     * Count of documents created.
+     */
+    long getCreated();
+    /**
+     * Count of successful delete operations.
+     */
+    long getDeleted();
+}
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryAction.java b/core/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryAction.java
similarity index 87%
rename from modules/reindex/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryAction.java
rename to core/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryAction.java
index 0ff1b18bde057..1058f7f13078a 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryAction.java
+++ b/core/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryAction.java
@@ -23,7 +23,7 @@
 import org.elasticsearch.client.ElasticsearchClient;
 
 public class UpdateByQueryAction extends
-        Action<UpdateByQueryRequest, BulkIndexByScrollResponse, UpdateByQueryRequestBuilder> {
+        Action<UpdateByQueryRequest, BulkByScrollResponse, UpdateByQueryRequestBuilder> {
     public static final UpdateByQueryAction INSTANCE = new UpdateByQueryAction();
     public static final String NAME = "indices:data/write/update/byquery";
 
@@ -37,7 +37,7 @@ public UpdateByQueryRequestBuilder newRequestBuilder(ElasticsearchClient client)
     }
 
     @Override
-    public BulkIndexByScrollResponse newResponse() {
-        return new BulkIndexByScrollResponse();
+    public BulkByScrollResponse newResponse() {
+        return new BulkByScrollResponse();
     }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequest.java b/core/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequest.java
similarity index 88%
rename from modules/reindex/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequest.java
rename to core/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequest.java
index 3401ce4582b13..ad0123d76cedf 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequest.java
+++ b/core/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequest.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.tasks.TaskId;
 
 import java.io.IOException;
 
@@ -43,7 +44,11 @@ public UpdateByQueryRequest() {
     }
 
     public UpdateByQueryRequest(SearchRequest search) {
-        super(search);
+        this(search, true);
+    }
+
+    private UpdateByQueryRequest(SearchRequest search, boolean setDefaults) {
+        super(search, setDefaults);
     }
 
     /**
@@ -65,6 +70,13 @@ protected UpdateByQueryRequest self() {
         return this;
     }
 
+    @Override
+    public UpdateByQueryRequest forSlice(TaskId slicingTask, SearchRequest slice) {
+        UpdateByQueryRequest request = doForSlice(new UpdateByQueryRequest(slice, false), slicingTask);
+        request.setPipeline(pipeline);
+        return request;
+    }
+
     @Override
     public String toString() {
         StringBuilder b = new StringBuilder();
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequestBuilder.java b/core/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequestBuilder.java
similarity index 90%
rename from modules/reindex/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequestBuilder.java
rename to core/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequestBuilder.java
index 311e138fb1f78..06e0426864193 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequestBuilder.java
@@ -28,12 +28,12 @@ public class UpdateByQueryRequestBuilder extends
         AbstractBulkIndexByScrollRequestBuilder<UpdateByQueryRequest, UpdateByQueryRequestBuilder> {
 
     public UpdateByQueryRequestBuilder(ElasticsearchClient client,
-            Action<UpdateByQueryRequest, BulkIndexByScrollResponse, UpdateByQueryRequestBuilder> action) {
+            Action<UpdateByQueryRequest, BulkByScrollResponse, UpdateByQueryRequestBuilder> action) {
         this(client, action, new SearchRequestBuilder(client, SearchAction.INSTANCE));
     }
 
     private UpdateByQueryRequestBuilder(ElasticsearchClient client,
-            Action<UpdateByQueryRequest, BulkIndexByScrollResponse, UpdateByQueryRequestBuilder> action,
+            Action<UpdateByQueryRequest, BulkByScrollResponse, UpdateByQueryRequestBuilder> action,
             SearchRequestBuilder search) {
         super(client, action, search, new UpdateByQueryRequest(search.request()));
     }
diff --git a/core/src/main/java/org/elasticsearch/index/reindex/WorkingBulkByScrollTask.java b/core/src/main/java/org/elasticsearch/index/reindex/WorkingBulkByScrollTask.java
new file mode 100644
index 0000000000000..4e11b3c9595e0
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/reindex/WorkingBulkByScrollTask.java
@@ -0,0 +1,320 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex;
+
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.logging.ESLoggerFactory;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
+import org.elasticsearch.common.util.concurrent.FutureUtils;
+import org.elasticsearch.tasks.TaskId;
+import org.elasticsearch.tasks.TaskInfo;
+import org.elasticsearch.threadpool.ThreadPool;
+
+import java.util.List;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static java.lang.Math.max;
+import static java.lang.Math.round;
+import static org.elasticsearch.common.unit.TimeValue.timeValueNanos;
+
+/**
+ * {@link BulkByScrollTask} subclass for tasks that actually perform the work. Compare to {@link ParentBulkByScrollTask}.
+ */
+public class WorkingBulkByScrollTask extends BulkByScrollTask implements SuccessfullyProcessed {
+    private static final Logger logger = ESLoggerFactory.getLogger(BulkByScrollTask.class.getPackage().getName());
+
+    /**
+     * The id of the slice that this task is processing or {@code null} if this task isn't for a sliced request.
+     */
+    private final Integer sliceId;
+    /**
+     * The total number of documents this request will process. 0 means we don't yet know or, possibly, there are actually 0 documents
+     * to process. Its ok that these have the same meaning because any request with 0 actual documents should be quite short lived.
+     */
+    private final AtomicLong total = new AtomicLong(0);
+    private final AtomicLong updated = new AtomicLong(0);
+    private final AtomicLong created = new AtomicLong(0);
+    private final AtomicLong deleted = new AtomicLong(0);
+    private final AtomicLong noops = new AtomicLong(0);
+    private final AtomicInteger batch = new AtomicInteger(0);
+    private final AtomicLong versionConflicts = new AtomicLong(0);
+    private final AtomicLong bulkRetries = new AtomicLong(0);
+    private final AtomicLong searchRetries = new AtomicLong(0);
+    private final AtomicLong throttledNanos = new AtomicLong();
+    /**
+     * The number of requests per second to which to throttle the request that this task represents. The other variables are all AtomicXXX
+     * style variables but there isn't an AtomicFloat so we just use a volatile.
+     */
+    private volatile float requestsPerSecond;
+    /**
+     * Reference to any the last delayed prepareBulkRequest call. Used during rethrottling and canceling to reschedule the request.
+     */
+    private final AtomicReference<DelayedPrepareBulkRequest> delayedPrepareBulkRequestReference = new AtomicReference<>();
+
+    public WorkingBulkByScrollTask(long id, String type, String action, String description, TaskId parentTask, Integer sliceId,
+            float requestsPerSecond) {
+        super(id, type, action, description, parentTask);
+        this.sliceId = sliceId;
+        setRequestsPerSecond(requestsPerSecond);
+    }
+
+    @Override
+    public Status getStatus() {
+        return new Status(sliceId, total.get(), updated.get(), created.get(), deleted.get(), batch.get(), versionConflicts.get(),
+                noops.get(), bulkRetries.get(), searchRetries.get(), timeValueNanos(throttledNanos.get()), getRequestsPerSecond(),
+                getReasonCancelled(), throttledUntil());
+    }
+
+    @Override
+    protected void onCancelled() {
+        /* Drop the throttle to 0, immediately rescheduling any throttled
+         * operation so it will wake up and cancel itself. */
+        rethrottle(Float.POSITIVE_INFINITY);
+    }
+
+    @Override
+    public int runningSliceSubTasks() {
+        return 0;
+    }
+
+    @Override
+    public TaskInfo getInfoGivenSliceInfo(String localNodeId, List<TaskInfo> sliceInfo) {
+        throw new UnsupportedOperationException("This is only supported by " + ParentBulkByScrollTask.class.getName() + ".");
+    }
+
+    TimeValue throttledUntil() {
+        DelayedPrepareBulkRequest delayed = delayedPrepareBulkRequestReference.get();
+        if (delayed == null) {
+            return timeValueNanos(0);
+        }
+        if (delayed.future == null) {
+            return timeValueNanos(0);
+        }
+        return timeValueNanos(max(0, delayed.future.getDelay(TimeUnit.NANOSECONDS)));
+    }
+
+    public void setTotal(long totalHits) {
+        total.set(totalHits);
+    }
+
+    public void countBatch() {
+        batch.incrementAndGet();
+    }
+
+    public void countNoop() {
+        noops.incrementAndGet();
+    }
+
+    @Override
+    public long getCreated() {
+        return created.get();
+    }
+
+    public void countCreated() {
+        created.incrementAndGet();
+    }
+
+    @Override
+    public long getUpdated() {
+        return updated.get();
+    }
+
+    public void countUpdated() {
+        updated.incrementAndGet();
+    }
+
+    @Override
+    public long getDeleted() {
+        return deleted.get();
+    }
+
+    public void countDeleted() {
+        deleted.incrementAndGet();
+    }
+
+    public void countVersionConflict() {
+        versionConflicts.incrementAndGet();
+    }
+
+    public void countBulkRetry() {
+        bulkRetries.incrementAndGet();
+    }
+
+    public void countSearchRetry() {
+        searchRetries.incrementAndGet();
+    }
+
+    float getRequestsPerSecond() {
+        return requestsPerSecond;
+    }
+
+    /**
+     * Schedule prepareBulkRequestRunnable to run after some delay. This is where throttling plugs into reindexing so the request can be
+     * rescheduled over and over again.
+     */
+    public void delayPrepareBulkRequest(ThreadPool threadPool, TimeValue lastBatchStartTime, int lastBatchSize,
+                                        AbstractRunnable prepareBulkRequestRunnable) {
+        // Synchronize so we are less likely to schedule the same request twice.
+        synchronized (delayedPrepareBulkRequestReference) {
+            TimeValue delay = throttleWaitTime(lastBatchStartTime, timeValueNanos(System.nanoTime()), lastBatchSize);
+            logger.debug("[{}]: preparing bulk request for [{}]", getId(), delay);
+            delayedPrepareBulkRequestReference.set(new DelayedPrepareBulkRequest(threadPool, getRequestsPerSecond(),
+                    delay, new RunOnce(prepareBulkRequestRunnable)));
+        }
+    }
+
+    public TimeValue throttleWaitTime(TimeValue lastBatchStartTime, TimeValue now, int lastBatchSize) {
+        long earliestNextBatchStartTime = now.nanos() + (long) perfectlyThrottledBatchTime(lastBatchSize);
+        return timeValueNanos(max(0, earliestNextBatchStartTime - System.nanoTime()));
+    }
+
+    /**
+     * How many nanoseconds should a batch of lastBatchSize have taken if it were perfectly throttled? Package private for testing.
+     */
+    float perfectlyThrottledBatchTime(int lastBatchSize) {
+        if (requestsPerSecond == Float.POSITIVE_INFINITY) {
+            return 0;
+        }
+        //       requests
+        // ------------------- == seconds
+        // request per seconds
+        float targetBatchTimeInSeconds = lastBatchSize / requestsPerSecond;
+        // nanoseconds per seconds * seconds == nanoseconds
+        return TimeUnit.SECONDS.toNanos(1) * targetBatchTimeInSeconds;
+    }
+
+    private void setRequestsPerSecond(float requestsPerSecond) {
+        if (requestsPerSecond <= 0) {
+            throw new IllegalArgumentException("requests per second must be more than 0 but was [" + requestsPerSecond + "]");
+        }
+        this.requestsPerSecond = requestsPerSecond;
+    }
+
+    @Override
+    public void rethrottle(float newRequestsPerSecond) {
+        synchronized (delayedPrepareBulkRequestReference) {
+            logger.debug("[{}]: rethrottling to [{}] requests per second", getId(), newRequestsPerSecond);
+            setRequestsPerSecond(newRequestsPerSecond);
+
+            DelayedPrepareBulkRequest delayedPrepareBulkRequest = this.delayedPrepareBulkRequestReference.get();
+            if (delayedPrepareBulkRequest == null) {
+                // No request has been queued so nothing to reschedule.
+                logger.debug("[{}]: skipping rescheduling because there is no scheduled task", getId());
+                return;
+            }
+
+            this.delayedPrepareBulkRequestReference.set(delayedPrepareBulkRequest.rethrottle(newRequestsPerSecond));
+        }
+    }
+
+    class DelayedPrepareBulkRequest {
+        private final ThreadPool threadPool;
+        private final AbstractRunnable command;
+        private final float requestsPerSecond;
+        private final ScheduledFuture<?> future;
+
+        DelayedPrepareBulkRequest(ThreadPool threadPool, float requestsPerSecond, TimeValue delay, AbstractRunnable command) {
+            this.threadPool = threadPool;
+            this.requestsPerSecond = requestsPerSecond;
+            this.command = command;
+            this.future = threadPool.schedule(delay, ThreadPool.Names.GENERIC, new AbstractRunnable() {
+                @Override
+                protected void doRun() throws Exception {
+                    throttledNanos.addAndGet(delay.nanos());
+                    command.run();
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    command.onFailure(e);
+                }
+            });
+        }
+
+        DelayedPrepareBulkRequest rethrottle(float newRequestsPerSecond) {
+            if (newRequestsPerSecond < requestsPerSecond) {
+                /* The user is attempting to slow the request down. We'll let the
+                 * change in throttle take effect the next time we delay
+                 * prepareBulkRequest. We can't just reschedule the request further
+                 * out in the future because the bulk context might time out. */
+                logger.debug("[{}]: skipping rescheduling because the new throttle [{}] is slower than the old one [{}]", getId(),
+                        newRequestsPerSecond, requestsPerSecond);
+                return this;
+            }
+
+            long remainingDelay = future.getDelay(TimeUnit.NANOSECONDS);
+            // Actually reschedule the task
+            if (false == FutureUtils.cancel(future)) {
+                // Couldn't cancel, probably because the task has finished or been scheduled. Either way we have nothing to do here.
+                logger.debug("[{}]: skipping rescheduling because we couldn't cancel the task", getId());
+                return this;
+            }
+
+            /* Strangely enough getting here doesn't mean that you actually
+             * cancelled the request, just that you probably did. If you stress
+             * test it you'll find that requests sneak through. So each request
+             * is given a runOnce boolean to prevent that. */
+            TimeValue newDelay = newDelay(remainingDelay, newRequestsPerSecond);
+            logger.debug("[{}]: rescheduling for [{}] in the future", getId(), newDelay);
+            return new DelayedPrepareBulkRequest(threadPool, requestsPerSecond, newDelay, command);
+        }
+
+        /**
+         * Scale back remaining delay to fit the new delay.
+         */
+        TimeValue newDelay(long remainingDelay, float newRequestsPerSecond) {
+            if (remainingDelay < 0) {
+                return timeValueNanos(0);
+            }
+            return timeValueNanos(round(remainingDelay * requestsPerSecond / newRequestsPerSecond));
+        }
+    }
+
+    /**
+     * Runnable that can only be run one time. This is paranoia to prevent furiously rethrottling from running the command multiple times.
+     * Without it the command would be run multiple times.
+     */
+    private static class RunOnce extends AbstractRunnable {
+        private final AtomicBoolean hasRun = new AtomicBoolean(false);
+        private final AbstractRunnable delegate;
+
+        RunOnce(AbstractRunnable delegate) {
+            this.delegate = delegate;
+        }
+
+        @Override
+        protected void doRun() throws Exception {
+            if (hasRun.compareAndSet(false, true)) {
+                delegate.run();
+            }
+        }
+
+        @Override
+        public void onFailure(Exception e) {
+            delegate.onFailure(e);
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/index/reindex/package-info.java b/core/src/main/java/org/elasticsearch/index/reindex/package-info.java
new file mode 100644
index 0000000000000..00cb5106770d1
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/reindex/package-info.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Infrastructure for actions that modify documents based on the results of a scrolling query
+ * like reindex, update by query or delete by query.
+ */
+package org.elasticsearch.index.reindex;
diff --git a/core/src/main/java/org/elasticsearch/index/search/ESToParentBlockJoinQuery.java b/core/src/main/java/org/elasticsearch/index/search/ESToParentBlockJoinQuery.java
new file mode 100644
index 0000000000000..1ee427599cab2
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/search/ESToParentBlockJoinQuery.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.search;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Weight;
+import org.apache.lucene.search.join.BitSetProducer;
+import org.apache.lucene.search.join.ScoreMode;
+import org.apache.lucene.search.join.ToParentBlockJoinQuery;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/** A {@link ToParentBlockJoinQuery} that allows to retrieve its nested path. */
+public final class ESToParentBlockJoinQuery extends Query {
+
+    private final ToParentBlockJoinQuery query;
+    private final String path;
+
+    public ESToParentBlockJoinQuery(Query childQuery, BitSetProducer parentsFilter, ScoreMode scoreMode, String path) {
+        this(new ToParentBlockJoinQuery(childQuery, parentsFilter, scoreMode), path);
+    }
+
+    private ESToParentBlockJoinQuery(ToParentBlockJoinQuery query, String path) {
+        this.query = query;
+        this.path = path;
+    }
+
+    /** Return the child query. */
+    public Query getChildQuery() {
+        return query.getChildQuery();
+    }
+
+    /** Return the path of results of this query, or {@code null} if matches are at the root level. */
+    public String getPath() {
+        return path;
+    }
+
+    @Override
+    public Query rewrite(IndexReader reader) throws IOException {
+        Query innerRewrite = query.rewrite(reader);
+        if (innerRewrite != query) {
+            // Right now ToParentBlockJoinQuery always rewrites to a ToParentBlockJoinQuery
+            // so the else block will never be used. It is useful in the case that
+            // ToParentBlockJoinQuery one day starts to rewrite to a different query, eg.
+            // a MatchNoDocsQuery if it realizes that it cannot match any docs and rewrites
+            // to a MatchNoDocsQuery. In that case it would be fine to lose information
+            // about the nested path.
+            if (innerRewrite instanceof ToParentBlockJoinQuery) {
+                return new ESToParentBlockJoinQuery((ToParentBlockJoinQuery) innerRewrite, path);
+            } else {
+                return innerRewrite;
+            }
+        }
+        return super.rewrite(reader);
+    }
+
+    @Override
+    public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException {
+        return query.createWeight(searcher, needsScores);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (sameClassAs(obj) == false) {
+            return false;
+        }
+        ESToParentBlockJoinQuery that = (ESToParentBlockJoinQuery) obj;
+        return query.equals(that.query) && Objects.equals(path, that.path);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(getClass(), query, path);
+    }
+
+    @Override
+    public String toString(String field) {
+        return query.toString(field);
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/index/search/MatchQuery.java b/core/src/main/java/org/elasticsearch/index/search/MatchQuery.java
index 835ec8e143f5d..f3ea9447db74a 100644
--- a/core/src/main/java/org/elasticsearch/index/search/MatchQuery.java
+++ b/core/src/main/java/org/elasticsearch/index/search/MatchQuery.java
@@ -20,26 +20,38 @@
 package org.elasticsearch.index.search;
 
 import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.miscellaneous.DisableGraphAttribute;
+import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.queries.ExtendedCommonTermsQuery;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.FuzzyQuery;
 import org.apache.lucene.search.MultiPhraseQuery;
 import org.apache.lucene.search.MultiTermQuery;
 import org.apache.lucene.search.PhraseQuery;
+import org.apache.lucene.search.PrefixQuery;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.SynonymQuery;
 import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.spans.SpanMultiTermQueryWrapper;
+import org.apache.lucene.search.spans.SpanNearQuery;
+import org.apache.lucene.search.spans.SpanOrQuery;
+import org.apache.lucene.search.spans.SpanQuery;
+import org.apache.lucene.search.spans.SpanTermQuery;
 import org.apache.lucene.util.QueryBuilder;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.lucene.all.AllTermQuery;
 import org.elasticsearch.common.lucene.search.MultiPhrasePrefixQuery;
 import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.unit.Fuzziness;
+import org.elasticsearch.index.analysis.ShingleTokenFilterFactory;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.query.support.QueryParsers;
@@ -48,7 +60,7 @@
 
 public class MatchQuery {
 
-    public static enum Type implements Writeable {
+    public enum Type implements Writeable {
         /**
          * The text is analyzed and terms are added to a boolean query.
          */
@@ -64,7 +76,7 @@ public static enum Type implements Writeable {
 
         private final int ordinal;
 
-        private Type(int ordinal) {
+        Type(int ordinal) {
             this.ordinal = ordinal;
         }
 
@@ -84,13 +96,13 @@ public void writeTo(StreamOutput out) throws IOException {
         }
     }
 
-    public static enum ZeroTermsQuery implements Writeable {
+    public enum ZeroTermsQuery implements Writeable {
         NONE(0),
         ALL(1);
 
         private final int ordinal;
 
-        private ZeroTermsQuery(int ordinal) {
+        ZeroTermsQuery(int ordinal) {
             this.ordinal = ordinal;
         }
 
@@ -110,13 +122,19 @@ public void writeTo(StreamOutput out) throws IOException {
         }
     }
 
-    /** the default phrase slop */
+    /**
+     * the default phrase slop
+     */
     public static final int DEFAULT_PHRASE_SLOP = 0;
 
-    /** the default leniency setting */
+    /**
+     * the default leniency setting
+     */
     public static final boolean DEFAULT_LENIENCY = false;
 
-    /** the default zero terms query */
+    /**
+     * the default zero terms query
+     */
     public static final ZeroTermsQuery DEFAULT_ZERO_TERMS_QUERY = ZeroTermsQuery.NONE;
 
     protected final QueryShardContext context;
@@ -204,7 +222,7 @@ protected Analyzer getAnalyzer(MappedFieldType fieldType) {
             }
             return context.getMapperService().searchAnalyzer();
         } else {
-            Analyzer analyzer = context.getMapperService().analysisService().analyzer(this.analyzer);
+            Analyzer analyzer = context.getMapperService().getIndexAnalyzers().get(this.analyzer);
             if (analyzer == null) {
                 throw new IllegalArgumentException("No analyzer found for [" + this.analyzer + "]");
             }
@@ -290,7 +308,7 @@ private class MatchQueryBuilder extends QueryBuilder {
         /**
          * Creates a new QueryBuilder using the given analyzer.
          */
-        public MatchQueryBuilder(Analyzer analyzer, @Nullable MappedFieldType mapper) {
+        MatchQueryBuilder(Analyzer analyzer, @Nullable MappedFieldType mapper) {
             super(analyzer);
             this.mapper = mapper;
         }
@@ -300,52 +318,137 @@ protected Query newTermQuery(Term term) {
             return blendTermQuery(term, mapper);
         }
 
+        @Override
+        protected Query newSynonymQuery(Term[] terms) {
+            return blendTermsQuery(terms, mapper);
+        }
+
+        /**
+         * Checks if graph analysis should be enabled for the field depending
+         * on the provided {@link Analyzer}
+         */
+        protected Query createFieldQuery(Analyzer analyzer, BooleanClause.Occur operator, String field,
+                                         String queryText, boolean quoted, int phraseSlop) {
+            assert operator == BooleanClause.Occur.SHOULD || operator == BooleanClause.Occur.MUST;
+
+            // Use the analyzer to get all the tokens, and then build an appropriate
+            // query based on the analysis chain.
+            try (TokenStream source = analyzer.tokenStream(field, queryText)) {
+                if (source.hasAttribute(DisableGraphAttribute.class)) {
+                    /**
+                     * A {@link TokenFilter} in this {@link TokenStream} disabled the graph analysis to avoid
+                     * paths explosion. See {@link ShingleTokenFilterFactory} for details.
+                     */
+                    setEnableGraphQueries(false);
+                }
+                Query query = super.createFieldQuery(source, operator, field, quoted, phraseSlop);
+                setEnableGraphQueries(true);
+                return query;
+            } catch (IOException e) {
+                throw new RuntimeException("Error analyzing query text", e);
+            }
+        }
+
         public Query createPhrasePrefixQuery(String field, String queryText, int phraseSlop, int maxExpansions) {
             final Query query = createFieldQuery(getAnalyzer(), Occur.MUST, field, queryText, true, phraseSlop);
+            return toMultiPhrasePrefix(query, phraseSlop, maxExpansions);
+        }
+
+        private Query toMultiPhrasePrefix(final Query query, int phraseSlop, int maxExpansions) {
+            float boost = 1;
+            Query innerQuery = query;
+            while (innerQuery instanceof BoostQuery) {
+                BoostQuery bq = (BoostQuery) innerQuery;
+                boost *= bq.getBoost();
+                innerQuery = bq.getQuery();
+            }
+            if (query instanceof SpanQuery) {
+                return toSpanQueryPrefix((SpanQuery) query, boost);
+            }
             final MultiPhrasePrefixQuery prefixQuery = new MultiPhrasePrefixQuery();
             prefixQuery.setMaxExpansions(maxExpansions);
             prefixQuery.setSlop(phraseSlop);
-            if (query instanceof PhraseQuery) {
-                PhraseQuery pq = (PhraseQuery)query;
+            if (innerQuery instanceof PhraseQuery) {
+                PhraseQuery pq = (PhraseQuery) innerQuery;
                 Term[] terms = pq.getTerms();
                 int[] positions = pq.getPositions();
                 for (int i = 0; i < terms.length; i++) {
-                    prefixQuery.add(new Term[] {terms[i]}, positions[i]);
+                    prefixQuery.add(new Term[]{terms[i]}, positions[i]);
                 }
-                return prefixQuery;
-            } else if (query instanceof MultiPhraseQuery) {
-                MultiPhraseQuery pq = (MultiPhraseQuery)query;
+                return boost == 1 ? prefixQuery : new BoostQuery(prefixQuery, boost);
+            } else if (innerQuery instanceof MultiPhraseQuery) {
+                MultiPhraseQuery pq = (MultiPhraseQuery) innerQuery;
                 Term[][] terms = pq.getTermArrays();
                 int[] positions = pq.getPositions();
                 for (int i = 0; i < terms.length; i++) {
                     prefixQuery.add(terms[i], positions[i]);
                 }
-                return prefixQuery;
-            } else if (query instanceof TermQuery) {
-                prefixQuery.add(((TermQuery) query).getTerm());
-                return prefixQuery;
+                return boost == 1 ? prefixQuery : new BoostQuery(prefixQuery, boost);
+            } else if (innerQuery instanceof TermQuery) {
+                prefixQuery.add(((TermQuery) innerQuery).getTerm());
+                return boost == 1 ? prefixQuery : new BoostQuery(prefixQuery, boost);
+            } else if (innerQuery instanceof AllTermQuery) {
+                prefixQuery.add(((AllTermQuery) innerQuery).getTerm());
+                return boost == 1 ? prefixQuery : new BoostQuery(prefixQuery, boost);
             }
             return query;
         }
 
-        public Query createCommonTermsQuery(String field, String queryText, Occur highFreqOccur, Occur lowFreqOccur, float maxTermFrequency, MappedFieldType fieldType) {
+        private Query toSpanQueryPrefix(SpanQuery query, float boost) {
+            if (query instanceof SpanTermQuery) {
+                SpanMultiTermQueryWrapper<PrefixQuery> ret =
+                    new SpanMultiTermQueryWrapper<>(new PrefixQuery(((SpanTermQuery) query).getTerm()));
+                return boost == 1 ? ret : new BoostQuery(ret, boost);
+            } else if (query instanceof SpanNearQuery) {
+                SpanNearQuery spanNearQuery = (SpanNearQuery) query;
+                SpanQuery[] clauses = spanNearQuery.getClauses();
+                if (clauses[clauses.length-1] instanceof SpanTermQuery) {
+                    clauses[clauses.length-1] = new SpanMultiTermQueryWrapper<>(
+                        new PrefixQuery(((SpanTermQuery) clauses[clauses.length-1]).getTerm())
+                    );
+                }
+                SpanNearQuery newQuery = new SpanNearQuery(clauses, spanNearQuery.getSlop(), spanNearQuery.isInOrder());
+                return boost == 1 ? newQuery : new BoostQuery(newQuery, boost);
+            } else if (query instanceof SpanOrQuery) {
+                SpanOrQuery orQuery = (SpanOrQuery) query;
+                SpanQuery[] clauses = new SpanQuery[orQuery.getClauses().length];
+                for (int i = 0; i < clauses.length; i++) {
+                    clauses[i] = (SpanQuery) toSpanQueryPrefix(orQuery.getClauses()[i], 1);
+                }
+                return boost == 1 ? new SpanOrQuery(clauses) : new BoostQuery(new SpanOrQuery(clauses), boost);
+            } else {
+                return query;
+            }
+        }
+
+        public Query createCommonTermsQuery(String field, String queryText, Occur highFreqOccur, Occur lowFreqOccur, float
+            maxTermFrequency, MappedFieldType fieldType) {
             Query booleanQuery = createBooleanQuery(field, queryText, lowFreqOccur);
             if (booleanQuery != null && booleanQuery instanceof BooleanQuery) {
                 BooleanQuery bq = (BooleanQuery) booleanQuery;
-                ExtendedCommonTermsQuery query = new ExtendedCommonTermsQuery(highFreqOccur, lowFreqOccur, maxTermFrequency, ((BooleanQuery)booleanQuery).isCoordDisabled(), fieldType);
-                for (BooleanClause clause : bq.clauses()) {
-                    if (!(clause.getQuery() instanceof TermQuery)) {
-                        return booleanQuery;
-                    }
-                    query.add(((TermQuery) clause.getQuery()).getTerm());
-                }
-                return query;
+                return boolToExtendedCommonTermsQuery(bq, highFreqOccur, lowFreqOccur, maxTermFrequency, fieldType);
             }
             return booleanQuery;
+        }
 
+        private Query boolToExtendedCommonTermsQuery(BooleanQuery bq, Occur highFreqOccur, Occur lowFreqOccur, float
+            maxTermFrequency, MappedFieldType fieldType) {
+            ExtendedCommonTermsQuery query = new ExtendedCommonTermsQuery(highFreqOccur, lowFreqOccur, maxTermFrequency,
+                bq.isCoordDisabled(), fieldType);
+            for (BooleanClause clause : bq.clauses()) {
+                if (!(clause.getQuery() instanceof TermQuery)) {
+                    return bq;
+                }
+                query.add(((TermQuery) clause.getQuery()).getTerm());
+            }
+            return query;
         }
     }
 
+    protected Query blendTermsQuery(Term[] terms, MappedFieldType fieldType) {
+        return new SynonymQuery(terms);
+    }
+
     protected Query blendTermQuery(Term term, MappedFieldType fieldType) {
         if (fuzziness != null) {
             if (fieldType != null) {
diff --git a/core/src/main/java/org/elasticsearch/index/search/MultiMatchQuery.java b/core/src/main/java/org/elasticsearch/index/search/MultiMatchQuery.java
index 1a5b3c02e1057..4203cb76c548e 100644
--- a/core/src/main/java/org/elasticsearch/index/search/MultiMatchQuery.java
+++ b/core/src/main/java/org/elasticsearch/index/search/MultiMatchQuery.java
@@ -28,10 +28,11 @@
 import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.DisjunctionMaxQuery;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
 import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.query.AbstractQueryBuilder;
@@ -60,12 +61,7 @@ public MultiMatchQuery(QueryShardContext context) {
 
     private Query parseAndApply(Type type, String fieldName, Object value, String minimumShouldMatch, Float boostValue) throws IOException {
         Query query = parse(type, fieldName, value);
-        // If the coordination factor is disabled on a boolean query we don't apply the minimum should match.
-        // This is done to make sure that the minimum_should_match doesn't get applied when there is only one word
-        // and multiple variations of the same word in the query (synonyms for instance).
-        if (query instanceof BooleanQuery && !((BooleanQuery) query).isCoordDisabled()) {
-            query = Queries.applyMinimumShouldMatch((BooleanQuery) query, minimumShouldMatch);
-        }
+        query = Queries.maybeApplyMinimumShouldMatch(query, minimumShouldMatch);
         if (query != null && boostValue != null && boostValue != AbstractQueryBuilder.DEFAULT_BOOST) {
             query = new BoostQuery(query, boostValue);
         }
@@ -157,6 +153,10 @@ public Query blendTerm(Term term, MappedFieldType fieldType) {
             return MultiMatchQuery.super.blendTermQuery(term, fieldType);
         }
 
+        public Query blendTerms(Term[] terms, MappedFieldType fieldType) {
+            return MultiMatchQuery.super.blendTermsQuery(terms, fieldType);
+        }
+
         public Query termQuery(MappedFieldType fieldType, Object value) {
             return MultiMatchQuery.this.termQuery(fieldType, value, lenient);
         }
@@ -165,7 +165,7 @@ public Query termQuery(MappedFieldType fieldType, Object value) {
     final class CrossFieldsQueryBuilder extends QueryBuilder {
         private FieldAndFieldType[] blendedFields;
 
-        public CrossFieldsQueryBuilder(float tieBreaker) {
+        CrossFieldsQueryBuilder(float tieBreaker) {
             super(false, tieBreaker);
         }
 
@@ -222,12 +222,24 @@ public List<Query> buildGroupedQueries(MultiMatchQueryBuilder.Type type, Map<Str
             return queries.isEmpty() ? null : queries;
         }
 
+        @Override
+        public Query blendTerms(Term[] terms, MappedFieldType fieldType) {
+            if (blendedFields == null || blendedFields.length == 1) {
+                return super.blendTerms(terms, fieldType);
+            }
+            BytesRef[] values = new BytesRef[terms.length];
+            for (int i = 0; i < terms.length; i++) {
+                values[i] = terms[i].bytes();
+            }
+            return MultiMatchQuery.blendTerms(context, values, commonTermsCutoff, tieBreaker, blendedFields);
+        }
+
         @Override
         public Query blendTerm(Term term, MappedFieldType fieldType) {
             if (blendedFields == null) {
                 return super.blendTerm(term, fieldType);
             }
-            return MultiMatchQuery.blendTerm(term.bytes(), commonTermsCutoff, tieBreaker, blendedFields);
+            return MultiMatchQuery.blendTerm(context, term.bytes(), commonTermsCutoff, tieBreaker, blendedFields);
         }
 
         @Override
@@ -241,36 +253,52 @@ public Query termQuery(MappedFieldType fieldType, Object value) {
         }
     }
 
-    static Query blendTerm(BytesRef value, Float commonTermsCutoff, float tieBreaker, FieldAndFieldType... blendedFields) {
+    static Query blendTerm(QueryShardContext context, BytesRef value, Float commonTermsCutoff, float tieBreaker,
+                           FieldAndFieldType... blendedFields) {
+        return blendTerms(context, new BytesRef[] {value}, commonTermsCutoff, tieBreaker, blendedFields);
+    }
+
+    static Query blendTerms(QueryShardContext context, BytesRef[] values, Float commonTermsCutoff, float tieBreaker,
+            FieldAndFieldType... blendedFields) {
         List<Query> queries = new ArrayList<>();
-        Term[] terms = new Term[blendedFields.length];
-        float[] blendedBoost = new float[blendedFields.length];
+        Term[] terms = new Term[blendedFields.length * values.length];
+        float[] blendedBoost = new float[blendedFields.length * values.length];
         int i = 0;
         for (FieldAndFieldType ft : blendedFields) {
-            Query query;
-            try {
-                query = ft.fieldType.termQuery(value, null);
-            } catch (IllegalArgumentException e) {
-                // the query expects a certain class of values such as numbers
-                // of ip addresses and the value can't be parsed, so ignore this
-                // field
-                continue;
-            }
-            float boost = ft.boost;
-            while (query instanceof BoostQuery) {
-                BoostQuery bq = (BoostQuery) query;
-                query = bq.getQuery();
-                boost *= bq.getBoost();
-            }
-            if (query.getClass() == TermQuery.class) {
-                terms[i] = ((TermQuery) query).getTerm();
-                blendedBoost[i] = boost;
-                i++;
-            } else {
-                if (boost != 1f) {
-                    query = new BoostQuery(query, boost);
+            for (BytesRef term : values) {
+                Query query;
+                try {
+                    query = ft.fieldType.termQuery(term, context);
+                } catch (IllegalArgumentException e) {
+                    // the query expects a certain class of values such as numbers
+                    // of ip addresses and the value can't be parsed, so ignore this
+                    // field
+                    continue;
+                } catch (ElasticsearchParseException parseException) {
+                    // date fields throw an ElasticsearchParseException with the
+                    // underlying IAE as the cause, ignore this field if that is
+                    // the case
+                    if (parseException.getCause() instanceof IllegalArgumentException) {
+                        continue;
+                    }
+                    throw parseException;
+                }
+                float boost = ft.boost;
+                while (query instanceof BoostQuery) {
+                    BoostQuery bq = (BoostQuery) query;
+                    query = bq.getQuery();
+                    boost *= bq.getBoost();
+                }
+                if (query.getClass() == TermQuery.class) {
+                    terms[i] = ((TermQuery) query).getTerm();
+                    blendedBoost[i] = boost;
+                    i++;
+                } else {
+                    if (boost != 1f) {
+                        query = new BoostQuery(query, boost);
+                    }
+                    queries.add(query);
                 }
-                queries.add(query);
             }
         }
         if (i > 0) {
@@ -307,6 +335,14 @@ protected Query blendTermQuery(Term term, MappedFieldType fieldType) {
         return queryBuilder.blendTerm(term, fieldType);
     }
 
+    @Override
+    protected Query blendTermsQuery(Term[] terms, MappedFieldType fieldType) {
+        if (queryBuilder == null) {
+            return super.blendTermsQuery(terms, fieldType);
+        }
+        return queryBuilder.blendTerms(terms, fieldType);
+    }
+
     static final class FieldAndFieldType {
         final MappedFieldType fieldType;
         final float boost;
diff --git a/core/src/main/java/org/elasticsearch/index/search/NestedHelper.java b/core/src/main/java/org/elasticsearch/index/search/NestedHelper.java
new file mode 100644
index 0000000000000..f5a5c8143bcb0
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/search/NestedHelper.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.search;
+
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.BoostQuery;
+import org.apache.lucene.search.ConstantScoreQuery;
+import org.apache.lucene.search.IndexOrDocValuesQuery;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.MatchNoDocsQuery;
+import org.apache.lucene.search.PointRangeQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.BooleanClause.Occur;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.ObjectMapper;
+
+/** Utility class to filter parent and children clauses when building nested
+ * queries. */
+public final class NestedHelper {
+
+    private final MapperService mapperService;
+
+    public NestedHelper(MapperService mapperService) {
+        this.mapperService = mapperService;
+    }
+
+    /** Returns true if the given query might match nested documents. */
+    public boolean mightMatchNestedDocs(Query query) {
+        if (query instanceof ConstantScoreQuery) {
+            return mightMatchNestedDocs(((ConstantScoreQuery) query).getQuery());
+        } else if (query instanceof BoostQuery) {
+            return mightMatchNestedDocs(((BoostQuery) query).getQuery());
+        } else if (query instanceof MatchAllDocsQuery) {
+            return true;
+        } else if (query instanceof MatchNoDocsQuery) {
+            return false;
+        } else if (query instanceof TermQuery) {
+            // We only handle term queries and range queries, which should already
+            // cover a high majority of use-cases
+            return mightMatchNestedDocs(((TermQuery) query).getTerm().field());
+        } else if (query instanceof PointRangeQuery) {
+            return mightMatchNestedDocs(((PointRangeQuery) query).getField());
+        } else if (query instanceof IndexOrDocValuesQuery) {
+            return mightMatchNestedDocs(((IndexOrDocValuesQuery) query).getIndexQuery());
+        } else if (query instanceof BooleanQuery) {
+            final BooleanQuery bq = (BooleanQuery) query;
+            final boolean hasRequiredClauses = bq.clauses().stream().anyMatch(BooleanClause::isRequired);
+            if (hasRequiredClauses) {
+                return bq.clauses().stream()
+                        .filter(BooleanClause::isRequired)
+                        .map(BooleanClause::getQuery)
+                        .allMatch(this::mightMatchNestedDocs);
+            } else {
+                return bq.clauses().stream()
+                        .filter(c -> c.getOccur() == Occur.SHOULD)
+                        .map(BooleanClause::getQuery)
+                        .anyMatch(this::mightMatchNestedDocs);
+            }
+        } else if (query instanceof ESToParentBlockJoinQuery) {
+            return ((ESToParentBlockJoinQuery) query).getPath() != null;
+        } else {
+            return true;
+        }
+    }
+
+    /** Returns true if a query on the given field might match nested documents. */
+    boolean mightMatchNestedDocs(String field) {
+        if (field.startsWith("_")) {
+            // meta field. Every meta field behaves differently, eg. nested
+            // documents have the same _uid as their parent, put their path in
+            // the _type field but do not have _field_names. So we just ignore
+            // meta fields and return true, which is always safe, it just means
+            // we might add a nested filter when it is nor required.
+            return true;
+        }
+        if (mapperService.fullName(field) == null) {
+            // field does not exist
+            return false;
+        }
+        for (String parent = parentObject(field); parent != null; parent = parentObject(parent)) {
+            ObjectMapper mapper = mapperService.getObjectMapper(parent);
+            if (mapper != null && mapper.nested().isNested()) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    /** Returns true if the given query might match parent documents or documents
+     *  that are nested under a different path. */
+    public boolean mightMatchNonNestedDocs(Query query, String nestedPath) {
+        if (query instanceof ConstantScoreQuery) {
+            return mightMatchNonNestedDocs(((ConstantScoreQuery) query).getQuery(), nestedPath);
+        } else if (query instanceof BoostQuery) {
+            return mightMatchNonNestedDocs(((BoostQuery) query).getQuery(), nestedPath);
+        } else if (query instanceof MatchAllDocsQuery) {
+            return true;
+        } else if (query instanceof MatchNoDocsQuery) {
+            return false;
+        } else if (query instanceof TermQuery) {
+            return mightMatchNonNestedDocs(((TermQuery) query).getTerm().field(), nestedPath);
+        } else if (query instanceof PointRangeQuery) {
+            return mightMatchNonNestedDocs(((PointRangeQuery) query).getField(), nestedPath);
+        } else if (query instanceof IndexOrDocValuesQuery) {
+            return mightMatchNonNestedDocs(((IndexOrDocValuesQuery) query).getIndexQuery(), nestedPath);
+        } else if (query instanceof BooleanQuery) {
+            final BooleanQuery bq = (BooleanQuery) query;
+            final boolean hasRequiredClauses = bq.clauses().stream().anyMatch(BooleanClause::isRequired);
+            if (hasRequiredClauses) {
+                return bq.clauses().stream()
+                        .filter(BooleanClause::isRequired)
+                        .map(BooleanClause::getQuery)
+                        .allMatch(q -> mightMatchNonNestedDocs(q, nestedPath));
+            } else {
+                return bq.clauses().stream()
+                        .filter(c -> c.getOccur() == Occur.SHOULD)
+                        .map(BooleanClause::getQuery)
+                        .anyMatch(q -> mightMatchNonNestedDocs(q, nestedPath));
+            }
+        } else {
+            return true;
+        }
+    }
+
+    /** Returns true if a query on the given field might match parent documents
+     *  or documents that are nested under a different path. */
+    boolean mightMatchNonNestedDocs(String field, String nestedPath) {
+        if (field.startsWith("_")) {
+            // meta field. Every meta field behaves differently, eg. nested
+            // documents have the same _uid as their parent, put their path in
+            // the _type field but do not have _field_names. So we just ignore
+            // meta fields and return true, which is always safe, it just means
+            // we might add a nested filter when it is nor required.
+            return true;
+        }
+        if (mapperService.fullName(field) == null) {
+            return false;
+        }
+        for (String parent = parentObject(field); parent != null; parent = parentObject(parent)) {
+            ObjectMapper mapper = mapperService.getObjectMapper(parent);
+            if (mapper!= null && mapper.nested().isNested()) {
+                if (mapper.fullPath().equals(nestedPath)) {
+                    // If the mapper does not include in its parent or in the root object then
+                    // the query might only match nested documents with the given path
+                    return mapper.nested().isIncludeInParent() || mapper.nested().isIncludeInRoot();
+                } else {
+                    // the first parent nested mapper does not have the expected path
+                    // It might be misconfiguration or a sub nested mapper
+                    return true;
+                }
+            }
+        }
+        return true; // the field is not a sub field of the nested path
+    }
+
+    private static String parentObject(String field) {
+        int lastDot = field.lastIndexOf('.');
+        if (lastDot == -1) {
+            return null;
+        }
+        return field.substring(0, lastDot);
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/index/search/geo/GeoDistanceRangeQuery.java b/core/src/main/java/org/elasticsearch/index/search/geo/GeoDistanceRangeQuery.java
deleted file mode 100644
index 53cdf5861a318..0000000000000
--- a/core/src/main/java/org/elasticsearch/index/search/geo/GeoDistanceRangeQuery.java
+++ /dev/null
@@ -1,234 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.search.geo;
-
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.search.ConstantScoreScorer;
-import org.apache.lucene.search.ConstantScoreWeight;
-import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.Scorer;
-import org.apache.lucene.search.TwoPhaseIterator;
-import org.apache.lucene.search.Weight;
-import org.apache.lucene.util.NumericUtils;
-import org.elasticsearch.common.geo.GeoDistance;
-import org.elasticsearch.common.geo.GeoPoint;
-import org.elasticsearch.common.unit.DistanceUnit;
-import org.elasticsearch.index.fielddata.IndexGeoPointFieldData;
-import org.elasticsearch.index.fielddata.MultiGeoPointValues;
-import org.elasticsearch.index.mapper.LegacyGeoPointFieldMapper;
-
-import java.io.IOException;
-
-/**
- * Query geo_point fields by distance ranges. Used for indexes created prior to 2.2
- * @deprecated
- */
-@Deprecated
-public class GeoDistanceRangeQuery extends Query {
-
-    private final double lat;
-    private final double lon;
-
-    private final double inclusiveLowerPoint; // in meters
-    private final double inclusiveUpperPoint; // in meters
-
-    private final GeoDistance geoDistance;
-    private final GeoDistance.FixedSourceDistance fixedSourceDistance;
-    private GeoDistance.DistanceBoundingCheck distanceBoundingCheck;
-    private final Query boundingBoxFilter;
-
-    private final IndexGeoPointFieldData indexFieldData;
-
-    public GeoDistanceRangeQuery(GeoPoint point, Double lowerVal, Double upperVal, boolean includeLower,
-                                 boolean includeUpper, GeoDistance geoDistance, LegacyGeoPointFieldMapper.GeoPointFieldType fieldType,
-                                 IndexGeoPointFieldData indexFieldData, String optimizeBbox) {
-        this.lat = point.lat();
-        this.lon = point.lon();
-        this.geoDistance = geoDistance;
-        this.indexFieldData = indexFieldData;
-
-        this.fixedSourceDistance = geoDistance.fixedSourceDistance(lat, lon, DistanceUnit.DEFAULT);
-
-        if (lowerVal != null) {
-            double f = lowerVal.doubleValue();
-            long i = NumericUtils.doubleToSortableLong(f);
-            inclusiveLowerPoint = NumericUtils.sortableLongToDouble(includeLower ? i : (i + 1L));
-        } else {
-            inclusiveLowerPoint = Double.NEGATIVE_INFINITY;
-        }
-        if (upperVal != null) {
-            double f = upperVal.doubleValue();
-            long i = NumericUtils.doubleToSortableLong(f);
-            inclusiveUpperPoint = NumericUtils.sortableLongToDouble(includeUpper ? i : (i - 1L));
-        } else {
-            inclusiveUpperPoint = Double.POSITIVE_INFINITY;
-            // we disable bounding box in this case, since the upper point is all and we create bounding box up to the
-            // upper point it will effectively include all
-            // TODO we can create a bounding box up to from and "not" it
-            optimizeBbox = null;
-        }
-
-        if (optimizeBbox != null && !"none".equals(optimizeBbox)) {
-            distanceBoundingCheck = GeoDistance.distanceBoundingCheck(lat, lon, inclusiveUpperPoint, DistanceUnit.DEFAULT);
-            if ("memory".equals(optimizeBbox)) {
-                boundingBoxFilter = null;
-            } else if ("indexed".equals(optimizeBbox)) {
-                boundingBoxFilter = LegacyIndexedGeoBoundingBoxQuery.create(distanceBoundingCheck.topLeft(),
-                    distanceBoundingCheck.bottomRight(), fieldType);
-                distanceBoundingCheck = GeoDistance.ALWAYS_INSTANCE; // fine, we do the bounding box check using the filter
-            } else {
-                throw new IllegalArgumentException("type [" + optimizeBbox + "] for bounding box optimization not supported");
-            }
-        } else {
-            distanceBoundingCheck = GeoDistance.ALWAYS_INSTANCE;
-            boundingBoxFilter = null;
-        }
-    }
-
-    public double lat() {
-        return lat;
-    }
-
-    public double lon() {
-        return lon;
-    }
-
-    public GeoDistance geoDistance() {
-        return geoDistance;
-    }
-
-    public double minInclusiveDistance() {
-        return inclusiveLowerPoint;
-    }
-
-    public double maxInclusiveDistance() {
-        return inclusiveUpperPoint;
-    }
-
-    public String fieldName() {
-        return indexFieldData.getFieldName();
-    }
-
-    @Override
-    public Query rewrite(IndexReader reader) throws IOException {
-        return super.rewrite(reader);
-    }
-
-    @Override
-    public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException {
-        final Weight boundingBoxWeight;
-        if (boundingBoxFilter != null) {
-            boundingBoxWeight = searcher.createNormalizedWeight(boundingBoxFilter, false);
-        } else {
-            boundingBoxWeight = null;
-        }
-        return new ConstantScoreWeight(this) {
-            @Override
-            public Scorer scorer(LeafReaderContext context) throws IOException {
-                final DocIdSetIterator approximation;
-                if (boundingBoxWeight != null) {
-                    Scorer s = boundingBoxWeight.scorer(context);
-                    if (s == null) {
-                        // if the approximation does not match anything, we're done
-                        return null;
-                    }
-                    approximation = s.iterator();
-                } else {
-                    approximation = DocIdSetIterator.all(context.reader().maxDoc());
-                }
-                final MultiGeoPointValues values = indexFieldData.load(context).getGeoPointValues();
-                final TwoPhaseIterator twoPhaseIterator = new TwoPhaseIterator(approximation) {
-                    @Override
-                    public boolean matches() throws IOException {
-                        final int doc = approximation.docID();
-                        values.setDocument(doc);
-                        final int length = values.count();
-                        for (int i = 0; i < length; i++) {
-                            GeoPoint point = values.valueAt(i);
-                            if (distanceBoundingCheck.isWithin(point.lat(), point.lon())) {
-                                double d = fixedSourceDistance.calculate(point.lat(), point.lon());
-                                if (d >= inclusiveLowerPoint && d <= inclusiveUpperPoint) {
-                                    return true;
-                                }
-                            }
-                        }
-                        return false;
-                    }
-
-                    @Override
-                    public float matchCost() {
-                        if (distanceBoundingCheck == GeoDistance.ALWAYS_INSTANCE) {
-                            return 0.0f;
-                        } else {
-                            // TODO: is this right (up to 4 comparisons from GeoDistance.SimpleDistanceBoundingCheck)?
-                            return 4.0f;
-                        }
-                    }
-                };
-                return new ConstantScoreScorer(this, score(), twoPhaseIterator);
-            }
-        };
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) return true;
-        if (sameClassAs(o) == false) return false;
-
-        GeoDistanceRangeQuery filter = (GeoDistanceRangeQuery) o;
-
-        if (Double.compare(filter.inclusiveLowerPoint, inclusiveLowerPoint) != 0) return false;
-        if (Double.compare(filter.inclusiveUpperPoint, inclusiveUpperPoint) != 0) return false;
-        if (Double.compare(filter.lat, lat) != 0) return false;
-        if (Double.compare(filter.lon, lon) != 0) return false;
-        if (!indexFieldData.getFieldName().equals(filter.indexFieldData.getFieldName()))
-            return false;
-        if (geoDistance != filter.geoDistance) return false;
-
-        return true;
-    }
-
-    @Override
-    public String toString(String field) {
-        return "GeoDistanceRangeQuery(" + indexFieldData.getFieldName() + ", " + geoDistance + ", ["
-            + inclusiveLowerPoint + " - " + inclusiveUpperPoint + "], " + lat + ", " + lon + ")";
-    }
-
-    @Override
-    public int hashCode() {
-        int result = classHash();
-        long temp;
-        temp = lat != +0.0d ? Double.doubleToLongBits(lat) : 0L;
-        result = 31 * result + Long.hashCode(temp);
-        temp = lon != +0.0d ? Double.doubleToLongBits(lon) : 0L;
-        result = 31 * result + Long.hashCode(temp);
-        temp = inclusiveLowerPoint != +0.0d ? Double.doubleToLongBits(inclusiveLowerPoint) : 0L;
-        result = 31 * result + Long.hashCode(temp);
-        temp = inclusiveUpperPoint != +0.0d ? Double.doubleToLongBits(inclusiveUpperPoint) : 0L;
-        result = 31 * result + Long.hashCode(temp);
-        result = 31 * result + (geoDistance != null ? geoDistance.hashCode() : 0);
-        result = 31 * result + indexFieldData.getFieldName().hashCode();
-        return result;
-    }
-
-}
diff --git a/core/src/main/java/org/elasticsearch/index/search/geo/LegacyGeoDistanceRangeQuery.java b/core/src/main/java/org/elasticsearch/index/search/geo/LegacyGeoDistanceRangeQuery.java
new file mode 100644
index 0000000000000..716f67da6f8d4
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/search/geo/LegacyGeoDistanceRangeQuery.java
@@ -0,0 +1,241 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.search.geo;
+
+import org.apache.lucene.geo.Rectangle;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.ConstantScoreScorer;
+import org.apache.lucene.search.ConstantScoreWeight;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.TwoPhaseIterator;
+import org.apache.lucene.search.Weight;
+import org.apache.lucene.util.NumericUtils;
+import org.elasticsearch.common.geo.GeoDistance;
+import org.elasticsearch.common.geo.GeoPoint;
+import org.elasticsearch.common.geo.GeoUtils;
+import org.elasticsearch.common.unit.DistanceUnit;
+import org.elasticsearch.index.fielddata.IndexGeoPointFieldData;
+import org.elasticsearch.index.fielddata.MultiGeoPointValues;
+import org.elasticsearch.index.mapper.LegacyGeoPointFieldMapper;
+import org.elasticsearch.index.query.QueryShardContext;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * Query geo_point fields by distance ranges. Used for indexes created prior to 2.2
+ * @deprecated this is deprecated in favor of lucene's new GeoPointField queries
+ */
+@Deprecated
+public class LegacyGeoDistanceRangeQuery extends Query {
+    private final double lat;
+    private final double lon;
+
+    private final double inclusiveLowerPoint; // in meters
+    private final double inclusiveUpperPoint; // in meters
+
+    private final GeoDistance geoDistance;
+    private final Query boundingBoxFilter;
+    private final Rectangle bbox;
+
+    private final IndexGeoPointFieldData indexFieldData;
+
+    public LegacyGeoDistanceRangeQuery(GeoPoint point, Double lowerVal, Double upperVal, boolean includeLower,
+                                       boolean includeUpper, GeoDistance geoDistance,
+                                       LegacyGeoPointFieldMapper.LegacyGeoPointFieldType fieldType,
+                                       IndexGeoPointFieldData indexFieldData, String optimizeBbox,
+                                       QueryShardContext context) {
+        this.lat = point.lat();
+        this.lon = point.lon();
+        this.geoDistance = geoDistance;
+        this.indexFieldData = indexFieldData;
+
+        if (lowerVal != null) {
+            double f = lowerVal.doubleValue();
+            long i = NumericUtils.doubleToSortableLong(f);
+            inclusiveLowerPoint = NumericUtils.sortableLongToDouble(includeLower ? i : (i + 1L));
+        } else {
+            inclusiveLowerPoint = Double.NEGATIVE_INFINITY;
+        }
+        if (upperVal != null) {
+            double f = upperVal.doubleValue();
+            long i = NumericUtils.doubleToSortableLong(f);
+            inclusiveUpperPoint = NumericUtils.sortableLongToDouble(includeUpper ? i : (i - 1L));
+        } else {
+            inclusiveUpperPoint = Double.POSITIVE_INFINITY;
+            // we disable bounding box in this case, since the upper point is all and we create bounding box up to the
+            // upper point it will effectively include all
+            // TODO we can create a bounding box up to from and "not" it
+            optimizeBbox = null;
+        }
+
+        if (optimizeBbox != null && !"none".equals(optimizeBbox)) {
+            bbox = Rectangle.fromPointDistance(lat, lon, inclusiveUpperPoint);
+            if ("memory".equals(optimizeBbox)) {
+                boundingBoxFilter = null;
+            } else if ("indexed".equals(optimizeBbox)) {
+                boundingBoxFilter = LegacyIndexedGeoBoundingBoxQuery.create(bbox, fieldType, context);
+            } else {
+                throw new IllegalArgumentException("type [" + optimizeBbox + "] for bounding box optimization not supported");
+            }
+        } else {
+            bbox = null;
+            boundingBoxFilter = null;
+        }
+    }
+
+    public double lat() {
+        return lat;
+    }
+
+    public double lon() {
+        return lon;
+    }
+
+    public GeoDistance geoDistance() {
+        return geoDistance;
+    }
+
+    public double minInclusiveDistance() {
+        return inclusiveLowerPoint;
+    }
+
+    public double maxInclusiveDistance() {
+        return inclusiveUpperPoint;
+    }
+
+    public String fieldName() {
+        return indexFieldData.getFieldName();
+    }
+
+    @Override
+    public Query rewrite(IndexReader reader) throws IOException {
+        return super.rewrite(reader);
+    }
+
+    @Override
+    public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException {
+        final Weight boundingBoxWeight;
+        if (boundingBoxFilter != null) {
+            boundingBoxWeight = searcher.createNormalizedWeight(boundingBoxFilter, false);
+        } else {
+            boundingBoxWeight = null;
+        }
+        return new ConstantScoreWeight(this) {
+            @Override
+            public Scorer scorer(LeafReaderContext context) throws IOException {
+                final DocIdSetIterator approximation;
+                if (boundingBoxWeight != null) {
+                    Scorer s = boundingBoxWeight.scorer(context);
+                    if (s == null) {
+                        // if the approximation does not match anything, we're done
+                        return null;
+                    }
+                    approximation = s.iterator();
+                } else {
+                    approximation = DocIdSetIterator.all(context.reader().maxDoc());
+                }
+                final MultiGeoPointValues values = indexFieldData.load(context).getGeoPointValues();
+                final TwoPhaseIterator twoPhaseIterator = new TwoPhaseIterator(approximation) {
+                    @Override
+                    public boolean matches() throws IOException {
+                        final int doc = approximation.docID();
+                        values.setDocument(doc);
+                        final int length = values.count();
+                        for (int i = 0; i < length; i++) {
+                            GeoPoint point = values.valueAt(i);
+                            if (bbox == null
+                                || GeoUtils.rectangleContainsPoint(bbox, point.lat(), point.lon())) {
+                                double d = geoDistance.calculate(lat, lon, point.lat(), point.lon(), DistanceUnit.DEFAULT);
+                                if (d >= inclusiveLowerPoint && d <= inclusiveUpperPoint) {
+                                    return true;
+                                }
+                            }
+                        }
+                        return false;
+                    }
+
+                    @Override
+                    public float matchCost() {
+                        if (bbox != null) {
+                            // always within bounds so we're going to compute distance for every point
+                            return values.count();
+                        } else {
+                            // TODO: come up with better estimate of boundary points
+                            return 4.0f;
+                        }
+                    }
+                };
+                return new ConstantScoreScorer(this, score(), twoPhaseIterator);
+            }
+        };
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (sameClassAs(o) == false) return false;
+
+        LegacyGeoDistanceRangeQuery filter = (LegacyGeoDistanceRangeQuery) o;
+
+        if (Double.compare(filter.inclusiveLowerPoint, inclusiveLowerPoint) != 0) return false;
+        if (Double.compare(filter.inclusiveUpperPoint, inclusiveUpperPoint) != 0) return false;
+        if (Double.compare(filter.lat, lat) != 0) return false;
+        if (Double.compare(filter.lon, lon) != 0) return false;
+        if (!indexFieldData.getFieldName().equals(filter.indexFieldData.getFieldName()))
+            return false;
+        if (geoDistance != filter.geoDistance) return false;
+        if (boundingBoxFilter == null && filter.boundingBoxFilter != null) return false;
+        else if (boundingBoxFilter != null && boundingBoxFilter.equals(filter.boundingBoxFilter) == false) return false;
+        if (bbox == null && filter.bbox != null) return false;
+        else if (bbox != null && bbox.equals(filter.bbox) == false) return false;
+
+        return true;
+    }
+
+    @Override
+    public String toString(String field) {
+        return "GeoDistanceRangeQuery(" + indexFieldData.getFieldName() + ", " + geoDistance + ", ["
+            + inclusiveLowerPoint + " - " + inclusiveUpperPoint + "], " + lat + ", " + lon + ")";
+    }
+
+    @Override
+    public int hashCode() {
+        int result = classHash();
+        long temp;
+        temp = lat != +0.0d ? Double.doubleToLongBits(lat) : 0L;
+        result = 31 * result + Long.hashCode(temp);
+        temp = lon != +0.0d ? Double.doubleToLongBits(lon) : 0L;
+        result = 31 * result + Long.hashCode(temp);
+        temp = inclusiveLowerPoint != +0.0d ? Double.doubleToLongBits(inclusiveLowerPoint) : 0L;
+        result = 31 * result + Long.hashCode(temp);
+        temp = inclusiveUpperPoint != +0.0d ? Double.doubleToLongBits(inclusiveUpperPoint) : 0L;
+        result = 31 * result + Long.hashCode(temp);
+        result = 31 * result + (geoDistance != null ? geoDistance.hashCode() : 0);
+        result = 31 * result + indexFieldData.getFieldName().hashCode();
+        result = 31 * result + Objects.hash(boundingBoxFilter, bbox);
+        return result;
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/index/search/geo/LegacyInMemoryGeoBoundingBoxQuery.java b/core/src/main/java/org/elasticsearch/index/search/geo/LegacyInMemoryGeoBoundingBoxQuery.java
index 2d8ea7af49d05..78b80246a0438 100644
--- a/core/src/main/java/org/elasticsearch/index/search/geo/LegacyInMemoryGeoBoundingBoxQuery.java
+++ b/core/src/main/java/org/elasticsearch/index/search/geo/LegacyInMemoryGeoBoundingBoxQuery.java
@@ -106,7 +106,7 @@ private static class Meridian180GeoBoundingBoxBits implements Bits {
         private final GeoPoint topLeft;
         private final GeoPoint bottomRight;
 
-        public Meridian180GeoBoundingBoxBits(int maxDoc, MultiGeoPointValues values, GeoPoint topLeft, GeoPoint bottomRight) {
+        Meridian180GeoBoundingBoxBits(int maxDoc, MultiGeoPointValues values, GeoPoint topLeft, GeoPoint bottomRight) {
             this.maxDoc = maxDoc;
             this.values = values;
             this.topLeft = topLeft;
@@ -139,7 +139,7 @@ private static class GeoBoundingBoxBits implements Bits {
         private final GeoPoint topLeft;
         private final GeoPoint bottomRight;
 
-        public GeoBoundingBoxBits(int maxDoc, MultiGeoPointValues values, GeoPoint topLeft, GeoPoint bottomRight) {
+        GeoBoundingBoxBits(int maxDoc, MultiGeoPointValues values, GeoPoint topLeft, GeoPoint bottomRight) {
             this.maxDoc = maxDoc;
             this.values = values;
             this.topLeft = topLeft;
diff --git a/core/src/main/java/org/elasticsearch/index/search/geo/LegacyIndexedGeoBoundingBoxQuery.java b/core/src/main/java/org/elasticsearch/index/search/geo/LegacyIndexedGeoBoundingBoxQuery.java
index f60a12f023dd7..f04fe7c9767d0 100644
--- a/core/src/main/java/org/elasticsearch/index/search/geo/LegacyIndexedGeoBoundingBoxQuery.java
+++ b/core/src/main/java/org/elasticsearch/index/search/geo/LegacyIndexedGeoBoundingBoxQuery.java
@@ -19,48 +19,59 @@
 
 package org.elasticsearch.index.search.geo;
 
+import org.apache.lucene.geo.Rectangle;
 import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.common.geo.GeoPoint;
-import org.elasticsearch.index.mapper.LegacyGeoPointFieldMapper;
+import org.elasticsearch.index.mapper.BaseGeoPointFieldMapper.LegacyGeoPointFieldType;
+import org.elasticsearch.index.query.QueryShardContext;
 
 /**
+ * Bounding Box filter used for indexes created before 2.2
  *
- * @deprecated This query is no longer used for geo_point indexes created after version 2.1
+ * @deprecated this class is deprecated in favor of lucene's GeoPointField
  */
 @Deprecated
 public class LegacyIndexedGeoBoundingBoxQuery {
+    public static Query create(final Rectangle bbox, final LegacyGeoPointFieldType fieldType, QueryShardContext context) {
+        return create(bbox.minLat, bbox.minLon, bbox.maxLat, bbox.maxLon, fieldType, context);
+    }
+
+    public static Query create(final GeoPoint topLeft, final GeoPoint bottomRight, final LegacyGeoPointFieldType fieldType,
+                               QueryShardContext context) {
+        return create(bottomRight.getLat(), topLeft.getLon(), topLeft.getLat(), bottomRight.getLon(), fieldType, context);
+    }
 
-    public static Query create(GeoPoint topLeft, GeoPoint bottomRight, LegacyGeoPointFieldMapper.GeoPointFieldType fieldType) {
+    private static Query create(double minLat, double minLon, double maxLat, double maxLon,
+                                LegacyGeoPointFieldType fieldType, QueryShardContext context) {
         if (!fieldType.isLatLonEnabled()) {
             throw new IllegalArgumentException("lat/lon is not enabled (indexed) for field [" + fieldType.name()
                 + "], can't use indexed filter on it");
         }
         //checks to see if bounding box crosses 180 degrees
-        if (topLeft.lon() > bottomRight.lon()) {
-            return westGeoBoundingBoxFilter(topLeft, bottomRight, fieldType);
-        } else {
-            return eastGeoBoundingBoxFilter(topLeft, bottomRight, fieldType);
+        if (minLon > maxLon) {
+            return westGeoBoundingBoxFilter(minLat, minLon, maxLat, maxLon, fieldType, context);
         }
+        return eastGeoBoundingBoxFilter(minLat, minLon, maxLat, maxLon, fieldType, context);
     }
 
-    private static Query westGeoBoundingBoxFilter(GeoPoint topLeft, GeoPoint bottomRight,
-                                                  LegacyGeoPointFieldMapper.GeoPointFieldType fieldType) {
+    private static Query westGeoBoundingBoxFilter(double minLat, double minLon, double maxLat, double maxLon,
+                                                  LegacyGeoPointFieldType fieldType, QueryShardContext context) {
         BooleanQuery.Builder filter = new BooleanQuery.Builder();
         filter.setMinimumNumberShouldMatch(1);
-        filter.add(fieldType.lonFieldType().rangeQuery(null, bottomRight.lon(), true, true), Occur.SHOULD);
-        filter.add(fieldType.lonFieldType().rangeQuery(topLeft.lon(), null, true, true), Occur.SHOULD);
-        filter.add(fieldType.latFieldType().rangeQuery(bottomRight.lat(), topLeft.lat(), true, true), Occur.MUST);
+        filter.add(fieldType.lonFieldType().rangeQuery(null, maxLon, true, true, context), Occur.SHOULD);
+        filter.add(fieldType.lonFieldType().rangeQuery(minLon, null, true, true, context), Occur.SHOULD);
+        filter.add(fieldType.latFieldType().rangeQuery(minLat, maxLat, true, true, context), Occur.MUST);
         return new ConstantScoreQuery(filter.build());
     }
 
-    private static Query eastGeoBoundingBoxFilter(GeoPoint topLeft, GeoPoint bottomRight,
-                                                  LegacyGeoPointFieldMapper.GeoPointFieldType fieldType) {
+    private static Query eastGeoBoundingBoxFilter(double minLat, double minLon, double maxLat, double maxLon,
+                                                  LegacyGeoPointFieldType fieldType, QueryShardContext context) {
         BooleanQuery.Builder filter = new BooleanQuery.Builder();
-        filter.add(fieldType.lonFieldType().rangeQuery(topLeft.lon(), bottomRight.lon(), true, true), Occur.MUST);
-        filter.add(fieldType.latFieldType().rangeQuery(bottomRight.lat(), topLeft.lat(), true, true), Occur.MUST);
+        filter.add(fieldType.lonFieldType().rangeQuery(minLon, maxLon, true, true, context), Occur.MUST);
+        filter.add(fieldType.latFieldType().rangeQuery(minLat, maxLat, true, true, context), Occur.MUST);
         return new ConstantScoreQuery(filter.build());
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/search/stats/SearchStats.java b/core/src/main/java/org/elasticsearch/index/search/stats/SearchStats.java
index d514571d0e200..824ca598ae2bb 100644
--- a/core/src/main/java/org/elasticsearch/index/search/stats/SearchStats.java
+++ b/core/src/main/java/org/elasticsearch/index/search/stats/SearchStats.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.index.search.stats;
 
+import org.elasticsearch.action.support.ToXContentToBytes;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -32,9 +33,7 @@
 import java.util.HashMap;
 import java.util.Map;
 
-/**
- */
-public class SearchStats implements Streamable, ToXContent {
+public class SearchStats extends ToXContentToBytes implements Streamable {
 
     public static class Stats implements Streamable, ToXContent {
 
@@ -340,22 +339,12 @@ static final class Fields {
         static final String SUGGEST_CURRENT = "suggest_current";
     }
 
-    public static SearchStats readSearchStats(StreamInput in) throws IOException {
-        SearchStats searchStats = new SearchStats();
-        searchStats.readFrom(in);
-        return searchStats;
-    }
-
     @Override
     public void readFrom(StreamInput in) throws IOException {
         totalStats = Stats.readStats(in);
         openContexts = in.readVLong();
         if (in.readBoolean()) {
-            int size = in.readVInt();
-            groupStats = new HashMap<>(size);
-            for (int i = 0; i < size; i++) {
-                groupStats.put(in.readString(), Stats.readStats(in));
-            }
+            groupStats = in.readMap(StreamInput::readString, Stats::readStats);
         }
     }
 
@@ -367,24 +356,7 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeBoolean(false);
         } else {
             out.writeBoolean(true);
-            out.writeVInt(groupStats.size());
-            for (Map.Entry<String, Stats> entry : groupStats.entrySet()) {
-                out.writeString(entry.getKey());
-                entry.getValue().writeTo(out);
-            }
-        }
-    }
-
-    @Override
-    public String toString() {
-        try {
-            XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint();
-            builder.startObject();
-            toXContent(builder, EMPTY_PARAMS);
-            builder.endObject();
-            return builder.string();
-        } catch (IOException e) {
-            return "{ \"error\" : \"" + e.getMessage() + "\"}";
+            out.writeMap(groupStats, StreamOutput::writeString, (stream, stats) -> stats.writeTo(stream));
         }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/search/stats/ShardSearchStats.java b/core/src/main/java/org/elasticsearch/index/search/stats/ShardSearchStats.java
index 68a2a30a2e974..8a6353e4cab32 100644
--- a/core/src/main/java/org/elasticsearch/index/search/stats/ShardSearchStats.java
+++ b/core/src/main/java/org/elasticsearch/index/search/stats/ShardSearchStats.java
@@ -82,8 +82,10 @@ public void onFailedQueryPhase(SearchContext searchContext) {
         computeStats(searchContext, statsHolder -> {
             if (searchContext.hasOnlySuggest()) {
                 statsHolder.suggestCurrent.dec();
+                assert statsHolder.suggestCurrent.count() >= 0;
             } else {
                 statsHolder.queryCurrent.dec();
+                assert statsHolder.queryCurrent.count() >= 0;
             }
         });
     }
@@ -94,9 +96,11 @@ public void onQueryPhase(SearchContext searchContext, long tookInNanos) {
             if (searchContext.hasOnlySuggest()) {
                 statsHolder.suggestMetric.inc(tookInNanos);
                 statsHolder.suggestCurrent.dec();
+                assert statsHolder.suggestCurrent.count() >= 0;
             } else {
                 statsHolder.queryMetric.inc(tookInNanos);
                 statsHolder.queryCurrent.dec();
+                assert statsHolder.queryCurrent.count() >= 0;
             }
         });
     }
@@ -116,6 +120,7 @@ public void onFetchPhase(SearchContext searchContext, long tookInNanos) {
         computeStats(searchContext, statsHolder -> {
             statsHolder.fetchMetric.inc(tookInNanos);
             statsHolder.fetchCurrent.dec();
+            assert statsHolder.fetchCurrent.count() >= 0;
         });
     }
 
@@ -176,6 +181,7 @@ public void onNewScrollContext(SearchContext context) {
     @Override
     public void onFreeScrollContext(SearchContext context) {
         totalStats.scrollCurrent.dec();
+        assert totalStats.scrollCurrent.count() >= 0;
         totalStats.scrollMetric.inc(System.nanoTime() - context.getOriginNanoTime());
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/shard/CommitPoint.java b/core/src/main/java/org/elasticsearch/index/shard/CommitPoint.java
index 9082fc072dadb..acd2f9f84559f 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/CommitPoint.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/CommitPoint.java
@@ -64,7 +64,7 @@ public String checksum() {
         }
     }
 
-    public static enum Type {
+    public enum Type {
         GENERATED,
         SAVED
     }
diff --git a/core/src/main/java/org/elasticsearch/index/shard/CommitPoints.java b/core/src/main/java/org/elasticsearch/index/shard/CommitPoints.java
index 2a5b350602889..dde6a82ea79c9 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/CommitPoints.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/CommitPoints.java
@@ -21,6 +21,7 @@
 
 import org.apache.lucene.util.CollectionUtil;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -121,7 +122,8 @@ public static byte[] toXContent(CommitPoint commitPoint) throws Exception {
     }
 
     public static CommitPoint fromXContent(byte[] data) throws Exception {
-        try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(data)) {
+        // EMPTY is safe here because we never call namedObject
+        try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(NamedXContentRegistry.EMPTY, data)) {
             String currentFieldName = null;
             XContentParser.Token token = parser.nextToken();
             if (token == null) {
diff --git a/core/src/main/java/org/elasticsearch/index/shard/DocsStats.java b/core/src/main/java/org/elasticsearch/index/shard/DocsStats.java
index bfb95b426b110..7fb2d09436fe2 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/DocsStats.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/DocsStats.java
@@ -59,12 +59,6 @@ public long getDeleted() {
         return this.deleted;
     }
 
-    public static DocsStats readDocStats(StreamInput in) throws IOException {
-        DocsStats docsStats = new DocsStats();
-        docsStats.readFrom(in);
-        return docsStats;
-    }
-
     @Override
     public void readFrom(StreamInput in) throws IOException {
         count = in.readVLong();
diff --git a/core/src/main/java/org/elasticsearch/index/shard/ElasticsearchQueryCachingPolicy.java b/core/src/main/java/org/elasticsearch/index/shard/ElasticsearchQueryCachingPolicy.java
new file mode 100644
index 0000000000000..3ea3955a1f416
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/shard/ElasticsearchQueryCachingPolicy.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.shard;
+
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.QueryCachingPolicy;
+import org.apache.lucene.search.TermQuery;
+
+import java.io.IOException;
+
+/**
+ * A {@link QueryCachingPolicy} that does not cache {@link TermQuery}s.
+ */
+final class ElasticsearchQueryCachingPolicy implements QueryCachingPolicy {
+
+    private final QueryCachingPolicy in;
+
+    ElasticsearchQueryCachingPolicy(QueryCachingPolicy in) {
+        this.in = in;
+    }
+
+    @Override
+    public void onUse(Query query) {
+        if (query.getClass() != TermQuery.class) {
+            // Do not waste space in the history for term queries. The assumption
+            // is that these queries are very fast so not worth caching
+            in.onUse(query);
+        }
+    }
+
+    @Override
+    public boolean shouldCache(Query query) throws IOException {
+        if (query.getClass() == TermQuery.class) {
+            return false;
+        }
+        return in.shouldCache(query);
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexEventListener.java b/core/src/main/java/org/elasticsearch/index/shard/IndexEventListener.java
index f5c6dca7d2fc7..24ed98e9affe3 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/IndexEventListener.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/IndexEventListener.java
@@ -23,6 +23,8 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason;
 
 /**
  * An index event listener is the primary extension point for plugins and build-in services
@@ -103,31 +105,32 @@ default void afterIndexCreated(IndexService indexService) {
 
     }
 
-    /**
-     * Called before the index shard gets created.
-     */
-    default void beforeIndexShardCreated(ShardId shardId, Settings indexSettings) {
-    }
-
-
     /**
      * Called before the index get closed.
      *
      * @param indexService The index service
+     * @param reason       the reason for index removal
      */
-    default void beforeIndexClosed(IndexService indexService) {
+    default void beforeIndexRemoved(IndexService indexService, IndexRemovalReason reason) {
 
     }
 
     /**
-     * Called after the index has been closed.
+     * Called after the index has been removed.
      *
      * @param index The index
+     * @param reason       the reason for index removal
      */
-    default void afterIndexClosed(Index index, Settings indexSettings) {
+    default void afterIndexRemoved(Index index, IndexSettings indexSettings, IndexRemovalReason reason) {
 
     }
 
+    /**
+     * Called before the index shard gets created.
+     */
+    default void beforeIndexShardCreated(ShardId shardId, Settings indexSettings) {
+    }
+
     /**
      * Called before the index shard gets deleted from disk
      * Note: this method is only executed on the first attempt of deleting the shard. Retries are will not invoke
@@ -149,28 +152,6 @@ default void beforeIndexShardDeleted(ShardId shardId, Settings indexSettings) {
     default void afterIndexShardDeleted(ShardId shardId, Settings indexSettings) {
     }
 
-    /**
-     * Called after the index has been deleted.
-     * This listener method is invoked after {@link #afterIndexClosed(org.elasticsearch.index.Index, org.elasticsearch.common.settings.Settings)}
-     * when an index is deleted
-     *
-     * @param index The index
-     */
-    default void afterIndexDeleted(Index index, Settings indexSettings) {
-
-    }
-
-    /**
-     * Called before the index gets deleted.
-     * This listener method is invoked after
-     * {@link #beforeIndexClosed(org.elasticsearch.index.IndexService)} when an index is deleted
-     *
-     * @param indexService The index service
-     */
-    default void beforeIndexDeleted(IndexService indexService) {
-
-    }
-
     /**
      * Called on the Master node only before the {@link IndexService} instances is created to simulate an index creation.
      * This happens right before the index and it's metadata is registered in the cluster state
diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index a38585cf46849..bee40f801e741 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -22,21 +22,18 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.index.CheckIndex;
-import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.index.IndexCommit;
-import org.apache.lucene.index.IndexFormatTooNewException;
-import org.apache.lucene.index.IndexFormatTooOldException;
-import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.KeepOnlyLastCommitDeletionPolicy;
 import org.apache.lucene.index.SnapshotDeletionPolicy;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.search.Query;
 import org.apache.lucene.search.QueryCachingPolicy;
+import org.apache.lucene.search.ReferenceManager;
 import org.apache.lucene.search.UsageTrackingQueryCachingPolicy;
 import org.apache.lucene.store.AlreadyClosedException;
-import org.apache.lucene.store.Lock;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.ThreadInterruptedException;
+import org.elasticsearch.Assertions;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.flush.FlushRequest;
@@ -48,7 +45,6 @@
 import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.RecoverySource.SnapshotRecoverySource;
 import org.elasticsearch.cluster.routing.ShardRouting;
-import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.collect.Tuple;
@@ -62,7 +58,6 @@
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
-import org.elasticsearch.common.util.Callback;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.AsyncIOProcessor;
 import org.elasticsearch.index.Index;
@@ -77,7 +72,6 @@
 import org.elasticsearch.index.codec.CodecService;
 import org.elasticsearch.index.engine.CommitStats;
 import org.elasticsearch.index.engine.Engine;
-import org.elasticsearch.index.engine.EngineClosedException;
 import org.elasticsearch.index.engine.EngineConfig;
 import org.elasticsearch.index.engine.EngineException;
 import org.elasticsearch.index.engine.EngineFactory;
@@ -91,13 +85,13 @@
 import org.elasticsearch.index.flush.FlushStats;
 import org.elasticsearch.index.get.GetStats;
 import org.elasticsearch.index.get.ShardGetService;
-import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.DocumentMapperForType;
-import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.SourceToParse;
 import org.elasticsearch.index.mapper.Uid;
+import org.elasticsearch.index.mapper.UidFieldMapper;
 import org.elasticsearch.index.merge.MergeStats;
 import org.elasticsearch.index.recovery.RecoveryStats;
 import org.elasticsearch.index.refresh.RefreshStats;
@@ -116,9 +110,9 @@
 import org.elasticsearch.indices.IndexingMemoryController;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.cluster.IndicesClusterStateService;
+import org.elasticsearch.indices.recovery.PeerRecoveryTargetService;
 import org.elasticsearch.indices.recovery.RecoveryFailedException;
 import org.elasticsearch.indices.recovery.RecoveryState;
-import org.elasticsearch.indices.recovery.PeerRecoveryTargetService;
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.repositories.Repository;
 import org.elasticsearch.search.suggest.completion.CompletionFieldStats;
@@ -126,16 +120,13 @@
 import org.elasticsearch.search.suggest.completion2x.Completion090PostingsFormat;
 import org.elasticsearch.threadpool.ThreadPool;
 
-import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.PrintStream;
 import java.nio.channels.ClosedByInterruptException;
 import java.nio.charset.StandardCharsets;
-import java.nio.file.NoSuchFileException;
 import java.util.ArrayList;
 import java.util.EnumSet;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
@@ -272,7 +263,11 @@ public IndexShard(ShardRouting shardRouting, IndexSettings indexSettings, ShardP
         if (IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING.get(settings)) {
             cachingPolicy = QueryCachingPolicy.ALWAYS_CACHE;
         } else {
-            cachingPolicy = new UsageTrackingQueryCachingPolicy();
+            QueryCachingPolicy cachingPolicy = new UsageTrackingQueryCachingPolicy();
+            if (IndexModule.INDEX_QUERY_CACHE_TERM_QUERIES_SETTING.get(settings) == false) {
+                cachingPolicy = new ElasticsearchQueryCachingPolicy(cachingPolicy);
+            }
+            this.cachingPolicy = cachingPolicy;
         }
         indexShardOperationsLock = new IndexShardOperationsLock(shardId, logger, threadPool);
         searcherWrapper = indexSearcherWrapper;
@@ -338,8 +333,21 @@ public long getPrimaryTerm() {
     public void updatePrimaryTerm(final long newTerm) {
         synchronized (mutex) {
             if (newTerm != primaryTerm) {
-                assert shardRouting.primary() == false : "a primary shard should never update it's term. shard: " + shardRouting
-                    + " current term [" + primaryTerm + "] new term [" + newTerm + "]";
+                // Note that due to cluster state batching an initializing primary shard term can failed and re-assigned
+                // in one state causing it's term to be incremented. Note that if both current shard state and new
+                // shard state are initializing, we could replace the current shard and reinitialize it. It is however
+                // possible that this shard is being started. This can happen if:
+                // 1) Shard is post recovery and sends shard started to the master
+                // 2) Node gets disconnected and rejoins
+                // 3) Master assigns the shard back to the node
+                // 4) Master processes the shard started and starts the shard
+                // 5) The node process the cluster state where the shard is both started and primary term is incremented.
+                //
+                // We could fail the shard in that case, but this will cause it to be removed from the insync allocations list
+                // potentially preventing re-allocation.
+                assert shardRouting.primary() == false || shardRouting.initializing() == false :
+                    "a started primary shard should never update it's term. shard: " + shardRouting
+                        + " current term [" + primaryTerm + "] new term [" + newTerm + "]";
                 assert newTerm > primaryTerm : "primary terms can only go up. current [" + primaryTerm + "], new [" + newTerm + "]";
                 primaryTerm = newTerm;
             }
@@ -367,61 +375,50 @@ public QueryCachingPolicy getQueryCachingPolicy() {
      * @throws IndexShardRelocatedException if shard is marked as relocated and relocation aborted
      * @throws IOException                  if shard state could not be persisted
      */
-    public void updateRoutingEntry(final ShardRouting newRouting) throws IOException {
-        final ShardRouting currentRouting = this.shardRouting;
-        if (!newRouting.shardId().equals(shardId())) {
-            throw new IllegalArgumentException("Trying to set a routing entry with shardId " + newRouting.shardId() + " on a shard with shardId " + shardId() + "");
-        }
-        if ((currentRouting == null || newRouting.isSameAllocation(currentRouting)) == false) {
-            throw new IllegalArgumentException("Trying to set a routing entry with a different allocation. Current " + currentRouting + ", new " + newRouting);
-        }
-        if (currentRouting != null) {
-            if (!newRouting.primary() && currentRouting.primary()) {
-                logger.warn("suspect illegal state: trying to move shard from primary mode to replica mode");
+    public void updateRoutingEntry(ShardRouting newRouting) throws IOException {
+        final ShardRouting currentRouting;
+        synchronized (mutex) {
+            currentRouting = this.shardRouting;
+
+            if (!newRouting.shardId().equals(shardId())) {
+                throw new IllegalArgumentException("Trying to set a routing entry with shardId " + newRouting.shardId() + " on a shard with shardId " + shardId());
             }
-            // if its the same routing, return
-            if (currentRouting.equals(newRouting)) {
-                return;
+            if ((currentRouting == null || newRouting.isSameAllocation(currentRouting)) == false) {
+                throw new IllegalArgumentException("Trying to set a routing entry with a different allocation. Current " + currentRouting + ", new " + newRouting);
+            }
+            if (currentRouting != null && currentRouting.primary() && newRouting.primary() == false) {
+                throw new IllegalArgumentException("illegal state: trying to move shard from primary mode to replica mode. Current "
+                    + currentRouting + ", new " + newRouting);
             }
-        }
 
-        if (state == IndexShardState.POST_RECOVERY) {
-            // if the state is started or relocating (cause it might move right away from started to relocating)
-            // then move to STARTED
-            if (newRouting.state() == ShardRoutingState.STARTED || newRouting.state() == ShardRoutingState.RELOCATING) {
+            if (state == IndexShardState.POST_RECOVERY && newRouting.active()) {
+                assert currentRouting.active() == false : "we are in POST_RECOVERY, but our shard routing is active " + currentRouting;
                 // we want to refresh *before* we move to internal STARTED state
                 try {
                     getEngine().refresh("cluster_state_started");
                 } catch (Exception e) {
                     logger.debug("failed to refresh due to move to cluster wide started", e);
                 }
-
-                boolean movedToStarted = false;
-                synchronized (mutex) {
-                    // do the check under a mutex, so we make sure to only change to STARTED if in POST_RECOVERY
-                    if (state == IndexShardState.POST_RECOVERY) {
-                        changeState(IndexShardState.STARTED, "global state is [" + newRouting.state() + "]");
-                        movedToStarted = true;
-                    } else {
-                        logger.debug("state [{}] not changed, not in POST_RECOVERY, global state is [{}]", state, newRouting.state());
-                    }
-                }
-                if (movedToStarted) {
-                    indexEventListener.afterIndexShardStarted(this);
-                }
+                changeState(IndexShardState.STARTED, "global state is [" + newRouting.state() + "]");
+            } else if (state == IndexShardState.RELOCATED &&
+                (newRouting.relocating() == false || newRouting.equalsIgnoringMetaData(currentRouting) == false)) {
+                // if the shard is marked as RELOCATED we have to fail when any changes in shard routing occur (e.g. due to recovery
+                // failure / cancellation). The reason is that at the moment we cannot safely move back to STARTED without risking two
+                // active primaries.
+                throw new IndexShardRelocatedException(shardId(), "Shard is marked as relocated, cannot safely move to state " + newRouting.state());
             }
+            assert newRouting.active() == false || state == IndexShardState.STARTED || state == IndexShardState.RELOCATED ||
+                state == IndexShardState.CLOSED :
+                "routing is active, but local shard state isn't. routing: " + newRouting + ", local state: " + state;
+            this.shardRouting = newRouting;
+            persistMetadata(newRouting, currentRouting);
         }
-
-        if (state == IndexShardState.RELOCATED &&
-            (newRouting.relocating() == false || newRouting.equalsIgnoringMetaData(currentRouting) == false)) {
-            // if the shard is marked as RELOCATED we have to fail when any changes in shard routing occur (e.g. due to recovery
-            // failure / cancellation). The reason is that at the moment we cannot safely move back to STARTED without risking two
-            // active primaries.
-            throw new IndexShardRelocatedException(shardId(), "Shard is marked as relocated, cannot safely move to state " + newRouting.state());
+        if (currentRouting != null && currentRouting.active() == false && newRouting.active()) {
+            indexEventListener.afterIndexShardStarted(this);
+        }
+        if (newRouting.equals(currentRouting) == false) {
+            indexEventListener.shardRoutingChanged(this, currentRouting, newRouting);
         }
-        this.shardRouting = newRouting;
-        indexEventListener.shardRoutingChanged(this, currentRouting, newRouting);
-        persistMetadata(newRouting, currentRouting);
     }
 
     /**
@@ -451,6 +448,7 @@ public IndexShardState markAsRecovering(String reason, RecoveryState recoverySta
     }
 
     public void relocated(String reason) throws IllegalIndexShardStateException, InterruptedException {
+        assert shardRouting.primary() : "only primaries can be marked as relocated: " + shardRouting;
         try {
             indexShardOperationsLock.blockOperations(30, TimeUnit.MINUTES, () -> {
                 // no shard operation locks are being held here, move state from started to relocated
@@ -460,6 +458,15 @@ public void relocated(String reason) throws IllegalIndexShardStateException, Int
                     if (state != IndexShardState.STARTED) {
                         throw new IndexShardNotStartedException(shardId, state);
                     }
+                    // if the master cancelled the recovery, the target will be removed
+                    // and the recovery will stopped.
+                    // However, it is still possible that we concurrently end up here
+                    // and therefore have to protect we don't mark the shard as relocated when
+                    // its shard routing says otherwise.
+                    if (shardRouting.relocating() == false) {
+                        throw new IllegalIndexShardStateException(shardId, IndexShardState.STARTED,
+                            ": shard is no longer relocating " + shardRouting);
+                    }
                     changeState(IndexShardState.RELOCATED, reason);
                 }
             });
@@ -485,6 +492,7 @@ public IndexShardState state() {
      * @return the previous shard state
      */
     private IndexShardState changeState(IndexShardState newState, String reason) {
+        assert Thread.holdsLock(mutex);
         logger.debug("state: [{}]->[{}], reason [{}]", state, newState, reason);
         IndexShardState previousState = state;
         state = newState;
@@ -523,86 +531,93 @@ static Engine.Index prepareIndex(DocumentMapperForType docMapper, SourceToParse
         if (docMapper.getMapping() != null) {
             doc.addDynamicMappingsUpdate(docMapper.getMapping());
         }
-        MappedFieldType uidFieldType = docMapper.getDocumentMapper().uidMapper().fieldType();
-        Query uidQuery = uidFieldType.termQuery(doc.uid(), null);
-        Term uid = MappedFieldType.extractTerm(uidQuery);
+        Term uid;
+        if (docMapper.getDocumentMapper().idFieldMapper().fieldType().indexOptions() != IndexOptions.NONE) {
+            uid = new Term(IdFieldMapper.NAME, doc.id());
+        } else {
+            uid = new Term(UidFieldMapper.NAME, Uid.createUidAsBytes(doc.type(), doc.id()));
+        }
         return new Engine.Index(uid, doc, version, versionType, origin, startTime, autoGeneratedIdTimestamp, isRetry);
     }
 
-    public void index(Engine.Index index) {
+    public Engine.IndexResult index(Engine.Index index) throws IOException {
         ensureWriteAllowed(index);
         Engine engine = getEngine();
-        index(engine, index);
+        return index(engine, index);
     }
 
-    private void index(Engine engine, Engine.Index index) {
+    private Engine.IndexResult index(Engine engine, Engine.Index index) throws IOException {
         active.set(true);
-        index = indexingOperationListeners.preIndex(index);
+        final Engine.IndexResult result;
+        index = indexingOperationListeners.preIndex(shardId, index);
         try {
             if (logger.isTraceEnabled()) {
-                logger.trace("index [{}][{}]{}", index.type(), index.id(), index.docs());
+                logger.trace("index [{}][{}] (v# [{}])",  index.type(), index.id(), index.version());
             }
-            engine.index(index);
-            index.endTime(System.nanoTime());
+            result = engine.index(index);
         } catch (Exception e) {
-            indexingOperationListeners.postIndex(index, e);
+            indexingOperationListeners.postIndex(shardId, index, e);
             throw e;
         }
-        indexingOperationListeners.postIndex(index, index.isCreated());
+        indexingOperationListeners.postIndex(shardId, index, result);
+        return result;
     }
 
     public Engine.Delete prepareDeleteOnPrimary(String type, String id, long version, VersionType versionType) {
         verifyPrimary();
-        final DocumentMapper documentMapper = docMapper(type).getDocumentMapper();
-        final MappedFieldType uidFieldType = documentMapper.uidMapper().fieldType();
-        final Query uidQuery = uidFieldType.termQuery(Uid.createUid(type, id), null);
-        final Term uid = MappedFieldType.extractTerm(uidQuery);
+        final Term uid = extractUidForDelete(type, id);
         return prepareDelete(type, id, uid, version, versionType, Engine.Operation.Origin.PRIMARY);
     }
 
     public Engine.Delete prepareDeleteOnReplica(String type, String id, long version, VersionType versionType) {
-        final DocumentMapper documentMapper = docMapper(type).getDocumentMapper();
-        final MappedFieldType uidFieldType = documentMapper.uidMapper().fieldType();
-        final Query uidQuery = uidFieldType.termQuery(Uid.createUid(type, id), null);
-        final Term uid = MappedFieldType.extractTerm(uidQuery);
+        final Term uid = extractUidForDelete(type, id);
         return prepareDelete(type, id, uid, version, versionType, Engine.Operation.Origin.REPLICA);
     }
 
     static Engine.Delete prepareDelete(String type, String id, Term uid, long version, VersionType versionType, Engine.Operation.Origin origin) {
         long startTime = System.nanoTime();
-        return new Engine.Delete(type, id, uid, version, versionType, origin, startTime, false);
+        return new Engine.Delete(type, id, uid, version, versionType, origin, startTime);
     }
 
-    public void delete(Engine.Delete delete) {
+    public Engine.DeleteResult delete(Engine.Delete delete) throws IOException {
         ensureWriteAllowed(delete);
         Engine engine = getEngine();
-        delete(engine, delete);
+        return delete(engine, delete);
+    }
+
+    private Term extractUidForDelete(String type, String id) {
+        if (indexSettings.isSingleType()) {
+            // This is only correct because we create types dynamically on delete operations
+            // otherwise this could match the same _id from a different type
+            return new Term(IdFieldMapper.NAME, id);
+        } else {
+            return new Term(UidFieldMapper.NAME, Uid.createUidAsBytes(type, id));
+        }
     }
 
-    private void delete(Engine engine, Engine.Delete delete) {
+    private Engine.DeleteResult delete(Engine engine, Engine.Delete delete) throws IOException {
         active.set(true);
-        delete = indexingOperationListeners.preDelete(delete);
+        final Engine.DeleteResult result;
+        delete = indexingOperationListeners.preDelete(shardId, delete);
         try {
             if (logger.isTraceEnabled()) {
                 logger.trace("delete [{}]", delete.uid().text());
             }
-            engine.delete(delete);
-            delete.endTime(System.nanoTime());
+            result = engine.delete(delete);
         } catch (Exception e) {
-            indexingOperationListeners.postDelete(delete, e);
+            indexingOperationListeners.postDelete(shardId, delete, e);
             throw e;
         }
-
-        indexingOperationListeners.postDelete(delete);
+        indexingOperationListeners.postDelete(shardId, delete, result);
+        return result;
     }
-
     public Engine.GetResult get(Engine.Get get) {
         readAllowed();
         return getEngine().get(get, this::acquireSearcher);
     }
 
     /**
-     * Writes all indexing changes to disk and opens a new searcher reflecting all changes.  This can throw {@link EngineClosedException}.
+     * Writes all indexing changes to disk and opens a new searcher reflecting all changes.  This can throw {@link AlreadyClosedException}.
      */
     public void refresh(String source) {
         verifyNotClosed();
@@ -614,9 +629,7 @@ public void refresh(String source) {
                 if (logger.isTraceEnabled()) {
                     logger.trace("refresh with source [{}] indexBufferRAMBytesUsed [{}]", source, new ByteSizeValue(bytes));
                 }
-                long time = System.nanoTime();
                 getEngine().refresh(source);
-                refreshMetric.inc(System.nanoTime() - time);
             } finally {
                 if (logger.isTraceEnabled()) {
                     logger.trace("remove [{}] writing bytes for shard [{}]", new ByteSizeValue(bytes), shardId());
@@ -627,9 +640,7 @@ public void refresh(String source) {
             if (logger.isTraceEnabled()) {
                 logger.trace("refresh with source [{}]", source);
             }
-            long time = System.nanoTime();
             getEngine().refresh(source);
-            refreshMetric.inc(System.nanoTime() - time);
         }
     }
 
@@ -641,7 +652,9 @@ public long getWritingBytes() {
     }
 
     public RefreshStats refreshStats() {
-        return new RefreshStats(refreshMetric.count(), TimeUnit.NANOSECONDS.toMillis(refreshMetric.sum()));
+        // Null refreshListeners means this shard doesn't support them so there can't be any.
+        int listeners = refreshListeners == null ? 0 : refreshListeners.pendingCount();
+        return new RefreshStats(refreshMetric.count(), TimeUnit.NANOSECONDS.toMillis(refreshMetric.sum()), listeners);
     }
 
     public FlushStats flushStats() {
@@ -649,9 +662,9 @@ public FlushStats flushStats() {
     }
 
     public DocsStats docStats() {
-        readAllowed();
-        final Engine engine = getEngine();
-        return engine.getDocStats();
+        try (Engine.Searcher searcher = acquireSearcher("doc_stats")) {
+            return new DocsStats(searcher.reader().numDocs(), searcher.reader().numDeletedDocs());
+        }
     }
 
     /**
@@ -723,7 +736,7 @@ public TranslogStats translogStats() {
 
     public CompletionStats completionStats(String... fields) {
         CompletionStats completionStats = new CompletionStats();
-        try (final Engine.Searcher currentSearcher = acquireSearcher("completion_stats")) {
+        try (Engine.Searcher currentSearcher = acquireSearcher("completion_stats")) {
             completionStats.add(CompletionFieldStats.completionStats(currentSearcher.reader(), fields));
             // Necessary for 2.x shards:
             Completion090PostingsFormat postingsFormat = ((Completion090PostingsFormat)
@@ -734,9 +747,14 @@ public CompletionStats completionStats(String... fields) {
     }
 
     public Engine.SyncedFlushResult syncFlush(String syncId, Engine.CommitId expectedCommitId) {
-        verifyStartedOrRecovering();
+        verifyNotClosed();
         logger.trace("trying to sync flush. sync id [{}]. expected commit id [{}]]", syncId, expectedCommitId);
-        return getEngine().syncFlush(syncId, expectedCommitId);
+        Engine engine = getEngine();
+        if (engine.isRecovering()) {
+            throw new IllegalIndexShardStateException(shardId(), state, "syncFlush is only allowed if the engine is not recovery" +
+                " from translog");
+        }
+        return engine.syncFlush(syncId, expectedCommitId);
     }
 
     public Engine.CommitId flush(FlushRequest request) throws ElasticsearchException {
@@ -747,18 +765,23 @@ public Engine.CommitId flush(FlushRequest request) throws ElasticsearchException
         }
         // we allows flush while recovering, since we allow for operations to happen
         // while recovering, and we want to keep the translog at bay (up to deletes, which
-        // we don't gc).
-        verifyStartedOrRecovering();
-
+        // we don't gc). Yet, we don't use flush internally to clear deletes and flush the indexwriter since
+        // we use #writeIndexingBuffer for this now.
+        verifyNotClosed();
+        Engine engine = getEngine();
+        if (engine.isRecovering()) {
+            throw new IllegalIndexShardStateException(shardId(), state, "flush is only allowed if the engine is not recovery" +
+                " from translog");
+        }
         long time = System.nanoTime();
-        Engine.CommitId commitId = getEngine().flush(force, waitIfOngoing);
+        Engine.CommitId commitId = engine.flush(force, waitIfOngoing);
         flushMetric.inc(System.nanoTime() - time);
         return commitId;
 
     }
 
     public void forceMerge(ForceMergeRequest forceMerge) throws IOException {
-        verifyStarted();
+        verifyActive();
         if (logger.isTraceEnabled()) {
             logger.trace("force merge with {}", forceMerge);
         }
@@ -770,7 +793,7 @@ public void forceMerge(ForceMergeRequest forceMerge) throws IOException {
      * Upgrades the shard to the current version of Lucene and returns the minimum segment version
      */
     public org.apache.lucene.util.Version upgrade(UpgradeRequest upgrade) throws IOException {
-        verifyStarted();
+        verifyActive();
         if (logger.isTraceEnabled()) {
             logger.trace("upgrade with {}", upgrade);
         }
@@ -827,12 +850,13 @@ public void releaseIndexCommit(IndexCommit snapshot) throws IOException {
      * without having to worry about the current state of the engine and concurrent flushes.
      *
      * @throws org.apache.lucene.index.IndexNotFoundException     if no index is found in the current directory
-     * @throws CorruptIndexException      if the lucene index is corrupted. This can be caused by a checksum mismatch or an
-     *                                    unexpected exception when opening the index reading the segments file.
-     * @throws IndexFormatTooOldException if the lucene index is too old to be opened.
-     * @throws IndexFormatTooNewException if the lucene index is too new to be opened.
-     * @throws FileNotFoundException      if one or more files referenced by a commit are not present.
-     * @throws NoSuchFileException        if one or more files referenced by a commit are not present.
+     * @throws org.apache.lucene.index.CorruptIndexException      if the lucene index is corrupted. This can be caused by a checksum
+     *                                                            mismatch or an unexpected exception when opening the index reading the
+     *                                                            segments file.
+     * @throws org.apache.lucene.index.IndexFormatTooOldException if the lucene index is too old to be opened.
+     * @throws org.apache.lucene.index.IndexFormatTooNewException if the lucene index is too new to be opened.
+     * @throws java.io.FileNotFoundException                      if one or more files referenced by a commit are not present.
+     * @throws java.nio.file.NoSuchFileException                  if one or more files referenced by a commit are not present.
      */
     public Store.MetadataSnapshot snapshotStoreMetadata() throws IOException {
         IndexCommit indexCommit = null;
@@ -844,9 +868,7 @@ public Store.MetadataSnapshot snapshotStoreMetadata() throws IOException {
                 // That can be done out of mutex, since the engine can be closed half way.
                 Engine engine = getEngineOrNull();
                 if (engine == null) {
-                    try (Lock ignored = store.directory().obtainLock(IndexWriter.WRITE_LOCK_NAME)) {
-                        return store.getMetadata(null);
-                    }
+                    return store.getMetadata(null, true);
                 }
             }
             indexCommit = deletionPolicy.snapshot();
@@ -897,8 +919,10 @@ public void close(String reason, boolean flushEngine) throws IOException {
                     if (engine != null && flushEngine) {
                         engine.flushAndClose();
                     }
-                } finally { // playing safe here and close the engine even if the above succeeds - close can be called multiple times
-                    IOUtils.close(engine);
+                } finally {
+                    // playing safe here and close the engine even if the above succeeds - close can be called multiple times
+                    // Also closing refreshListeners to prevent us from accumulating any more listeners
+                    IOUtils.close(engine, refreshListeners);
                     indexShardOperationsLock.close();
                 }
             }
@@ -971,6 +995,9 @@ private void internalPerformTranslogRecovery(boolean skipTranslogRecovery, boole
         recoveryState.setStage(RecoveryState.Stage.VERIFY_INDEX);
         // also check here, before we apply the translog
         if (Booleans.parseBoolean(checkIndexOnStartup, false)) {
+            if (Booleans.isBoolean(checkIndexOnStartup) && Booleans.isStrictlyBoolean(checkIndexOnStartup) == false) {
+                deprecationLogger.deprecated("Expected [true/false/fix/checksum] for setting [{}] but got [{}]", IndexSettings.INDEX_CHECK_ON_STARTUP.getKey(), checkIndexOnStartup);
+            }
             try {
                 checkIndex();
             } catch (IOException ex) {
@@ -1005,7 +1032,10 @@ private void internalPerformTranslogRecovery(boolean skipTranslogRecovery, boole
             active.set(true);
             newEngine.recoverFromTranslog();
         }
+    }
 
+    protected void onNewEngine(Engine newEngine) {
+        refreshListeners.setTranslog(newEngine.getTranslog());
     }
 
     /**
@@ -1110,13 +1140,6 @@ private void verifyReplicationTarget() {
         }
     }
 
-    protected final void verifyStartedOrRecovering() throws IllegalIndexShardStateException {
-        IndexShardState state = this.state; // one time volatile read
-        if (state != IndexShardState.STARTED && state != IndexShardState.RECOVERING && state != IndexShardState.POST_RECOVERY) {
-            throw new IllegalIndexShardStateException(shardId, state, "operation only allowed when started/recovering");
-        }
-    }
-
     private void verifyNotClosed() throws IllegalIndexShardStateException {
         verifyNotClosed(null);
     }
@@ -1132,10 +1155,10 @@ private void verifyNotClosed(Exception suppressed) throws IllegalIndexShardState
         }
     }
 
-    protected final void verifyStarted() throws IllegalIndexShardStateException {
+    protected final void verifyActive() throws IllegalIndexShardStateException {
         IndexShardState state = this.state; // one time volatile read
-        if (state != IndexShardState.STARTED) {
-            throw new IndexShardNotStartedException(shardId, state);
+        if (state != IndexShardState.STARTED && state != IndexShardState.RELOCATED) {
+            throw new IllegalIndexShardStateException(shardId, state, "operation only allowed when shard is active");
         }
     }
 
@@ -1154,7 +1177,7 @@ public long getIndexBufferRAMBytesUsed() {
         }
     }
 
-    public void addShardFailureCallback(Callback<ShardFailure> onShardFailure) {
+    public void addShardFailureCallback(Consumer<ShardFailure> onShardFailure) {
         this.shardEventListener.delegates.add(onShardFailure);
     }
 
@@ -1168,7 +1191,11 @@ public void checkIdle(long inactiveTimeNS) {
             boolean wasActive = active.getAndSet(false);
             if (wasActive) {
                 logger.debug("shard is now inactive");
-                indexEventListener.onShardInactive(this);
+                try {
+                    indexEventListener.onShardInactive(this);
+                } catch (Exception e) {
+                    logger.warn("failed to notify index event listener", e);
+                }
             }
         }
     }
@@ -1225,8 +1252,8 @@ boolean shouldFlush() {
         if (engine != null) {
             try {
                 Translog translog = engine.getTranslog();
-                return translog.sizeInBytes() > indexSettings.getFlushThresholdSize().bytes();
-            } catch (AlreadyClosedException | EngineClosedException ex) {
+                return translog.sizeInBytes() > indexSettings.getFlushThresholdSize().getBytes();
+            } catch (AlreadyClosedException ex) {
                 // that's fine we are already close - no need to flush
             }
         }
@@ -1265,7 +1292,7 @@ public IndexEventListener getIndexEventListener() {
     public void activateThrottling() {
         try {
             getEngine().activateThrottling();
-        } catch (EngineClosedException ex) {
+        } catch (AlreadyClosedException ex) {
             // ignore
         }
     }
@@ -1273,13 +1300,13 @@ public void activateThrottling() {
     public void deactivateThrottling() {
         try {
             getEngine().deactivateThrottling();
-        } catch (EngineClosedException ex) {
+        } catch (AlreadyClosedException ex) {
             // ignore
         }
     }
 
     private void handleRefreshException(Exception e) {
-        if (e instanceof EngineClosedException) {
+        if (e instanceof AlreadyClosedException) {
             // ignore
         } else if (e instanceof RefreshFailedEngineException) {
             RefreshFailedEngineException rfee = (RefreshFailedEngineException) e;
@@ -1414,7 +1441,7 @@ private void doCheckIndex() throws IOException {
     Engine getEngine() {
         Engine engine = getEngineOrNull();
         if (engine == null) {
-            throw new EngineClosedException(shardId);
+            throw new AlreadyClosedException("engine is closed");
         }
         return engine;
     }
@@ -1514,7 +1541,7 @@ public void startRecovery(RecoveryState recoveryState, PeerRecoveryTargetService
                         }
                     });
                 } else {
-                    final Exception e;
+                    final RuntimeException e;
                     if (numShards == -1) {
                         e = new IndexNotFoundException(mergeSourceIndex);
                     } else {
@@ -1522,7 +1549,7 @@ public void startRecovery(RecoveryState recoveryState, PeerRecoveryTargetService
                             + " are started yet, expected " + numShards + " found " + startedShards.size() + " can't recover shard "
                             + shardId());
                     }
-                    recoveryListener.onRecoveryFailure(recoveryState, new RecoveryFailedException(recoveryState, null, e), true);
+                    throw e;
                 }
                 break;
             default:
@@ -1531,15 +1558,15 @@ public void startRecovery(RecoveryState recoveryState, PeerRecoveryTargetService
     }
 
     class ShardEventListener implements Engine.EventListener {
-        private final CopyOnWriteArrayList<Callback<ShardFailure>> delegates = new CopyOnWriteArrayList<>();
+        private final CopyOnWriteArrayList<Consumer<ShardFailure>> delegates = new CopyOnWriteArrayList<>();
 
         // called by the current engine
         @Override
         public void onFailedEngine(String reason, @Nullable Exception failure) {
             final ShardFailure shardFailure = new ShardFailure(shardRouting, reason, failure);
-            for (Callback<ShardFailure> listener : delegates) {
+            for (Consumer<ShardFailure> listener : delegates) {
                 try {
-                    listener.handle(shardFailure);
+                    listener.accept(shardFailure);
                 } catch (Exception inner) {
                     inner.addSuppressed(failure);
                     logger.warn("exception while notifying engine failure", inner);
@@ -1551,11 +1578,13 @@ public void onFailedEngine(String reason, @Nullable Exception failure) {
     private Engine createNewEngine(EngineConfig config) {
         synchronized (mutex) {
             if (state == IndexShardState.CLOSED) {
-                throw new EngineClosedException(shardId);
+                throw new AlreadyClosedException(shardId + " can't create engine - shard is closed");
             }
             assert this.currentEngineReference.get() == null;
-            this.currentEngineReference.set(newEngine(config));
-
+            Engine engine = newEngine(config);
+            onNewEngine(engine); // call this before we pass the memory barrier otherwise actions that happen
+            // inside the callback are not visible. This one enforces happens-before
+            this.currentEngineReference.set(engine);
         }
 
         // time elapses after the engine is created above (pulling the config settings) until we set the engine reference, during which
@@ -1606,11 +1635,16 @@ private DocumentMapperForType docMapper(String type) {
 
     private EngineConfig newEngineConfig(EngineConfig.OpenMode openMode, long maxUnsafeAutoIdTimestamp) {
         final IndexShardRecoveryPerformer translogRecoveryPerformer = new IndexShardRecoveryPerformer(shardId, mapperService, logger);
+        final List<ReferenceManager.RefreshListener> refreshListenerList = new ArrayList<>();
+        refreshListenerList.add(new RefreshMetricUpdater(refreshMetric));
+        if (refreshListeners != null) {
+            refreshListenerList.add(refreshListeners);
+        }
         return new EngineConfig(openMode, shardId,
             threadPool, indexSettings, warmer, store, deletionPolicy, indexSettings.getMergePolicy(),
             mapperService.indexAnalyzer(), similarityService.similarity(mapperService), codecService, shardEventListener, translogRecoveryPerformer, indexCache.query(), cachingPolicy, translogConfig,
-            IndexingMemoryController.SHARD_INACTIVE_TIME_SETTING.get(indexSettings.getSettings()), refreshListeners,
-            maxUnsafeAutoIdTimestamp);
+            IndexingMemoryController.SHARD_INACTIVE_TIME_SETTING.get(indexSettings.getSettings()),
+            refreshListenerList, maxUnsafeAutoIdTimestamp);
     }
 
     /**
@@ -1651,7 +1685,7 @@ protected void write(List<Tuple<Translog.Location, Consumer<Exception>>> candida
                 try {
                     final Engine engine = getEngine();
                     engine.getTranslog().ensureSynced(candidates.stream().map(Tuple::v1));
-                } catch (EngineClosedException ex) {
+                } catch (AlreadyClosedException ex) {
                     // that's fine since we already synced everything on engine close - this also is conform with the methods
                     // documentation
                 } catch (IOException ex) { // if this fails we are in deep shit - fail the request
@@ -1742,7 +1776,7 @@ protected RefreshListeners buildRefreshListeners() {
     /**
      * Simple struct encapsulating a shard failure
      *
-     * @see IndexShard#addShardFailureCallback(Callback)
+     * @see IndexShard#addShardFailureCallback(Consumer)
      */
     public static final class ShardFailure {
         public final ShardRouting routing;
@@ -1766,8 +1800,7 @@ EngineFactory getEngineFactory() {
      * refresh listeners.
      * Otherwise <code>false</code>.
      *
-     * @throws EngineClosedException  if the engine is already closed
-     * @throws AlreadyClosedException if the internal indexwriter in the engine is already closed
+     * @throws AlreadyClosedException if the engine or internal indexwriter in the engine is already closed
      */
     public boolean isRefreshNeeded() {
         return getEngine().refreshNeeded() || (refreshListeners != null && refreshListeners.refreshNeeded());
@@ -1806,13 +1839,45 @@ public int recoveryFromSnapshot(Engine engine, Translog.Snapshot snapshot) throw
         }
 
         @Override
-        protected void index(Engine engine, Engine.Index engineIndex) {
+        protected void index(Engine engine, Engine.Index engineIndex) throws IOException {
             IndexShard.this.index(engine, engineIndex);
         }
 
         @Override
-        protected void delete(Engine engine, Engine.Delete engineDelete) {
+        protected void delete(Engine engine, Engine.Delete engineDelete) throws IOException {
             IndexShard.this.delete(engine, engineDelete);
         }
     }
+
+    private static class RefreshMetricUpdater implements ReferenceManager.RefreshListener {
+
+        private final MeanMetric refreshMetric;
+        private long currentRefreshStartTime;
+        private Thread callingThread = null;
+
+        private RefreshMetricUpdater(MeanMetric refreshMetric) {
+            this.refreshMetric = refreshMetric;
+        }
+
+        @Override
+        public void beforeRefresh() throws IOException {
+            if (Assertions.ENABLED) {
+                assert callingThread == null : "beforeRefresh was called by " + callingThread.getName() +
+                    " without a corresponding call to afterRefresh";
+                callingThread = Thread.currentThread();
+            }
+            currentRefreshStartTime = System.nanoTime();
+        }
+
+        @Override
+        public void afterRefresh(boolean didRefresh) throws IOException {
+            if (Assertions.ENABLED) {
+                assert callingThread != null : "afterRefresh called but not beforeRefresh";
+                assert callingThread == Thread.currentThread() : "beforeRefreshed called by a different thread. current ["
+                    + Thread.currentThread().getName() + "], thread that called beforeRefresh [" + callingThread.getName() + "]";
+                callingThread = null;
+            }
+            refreshMetric.inc(System.nanoTime() - currentRefreshStartTime);
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShardOperationsLock.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShardOperationsLock.java
index cde14dec17307..3f48b60d45a3b 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/IndexShardOperationsLock.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShardOperationsLock.java
@@ -19,10 +19,14 @@
 package org.elasticsearch.index.shard;
 
 import org.apache.logging.log4j.Logger;
+import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ContextPreservingActionListener;
 import org.elasticsearch.action.support.ThreadedActionListener;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.lease.Releasable;
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
+import org.elasticsearch.common.util.concurrent.ThreadContext.StoredContext;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.Closeable;
@@ -32,6 +36,7 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Supplier;
 
 public class IndexShardOperationsLock implements Closeable {
     private final ShardId shardId;
@@ -103,14 +108,17 @@ public void blockOperations(long timeout, TimeUnit timeUnit, Runnable onBlocked)
     }
 
     /**
-     * Acquires a lock whenever lock acquisition is not blocked. If the lock is directly available, the provided
-     * ActionListener will be called on the calling thread. During calls of {@link #blockOperations(long, TimeUnit, Runnable)}, lock
-     * acquisition can be delayed. The provided ActionListener will then be called using the provided executor once blockOperations
-     * terminates.
+     * Acquires a permit whenever permit acquisition is not blocked. If the permit is directly available, the provided
+     * {@link ActionListener} will be called on the calling thread. During calls of
+     * {@link #blockOperations(long, TimeUnit, Runnable)}, permit acquisition can be delayed.
+     * The {@link ActionListener#onResponse(Object)} method will then be called using the provided executor once operations are no
+     * longer blocked. Note that the executor will not be used for {@link ActionListener#onFailure(Exception)} calls. Those will run
+     * directly on the calling thread, which in case of delays, will be a generic thread. Callers should thus make sure
+     * that the {@link ActionListener#onFailure(Exception)} method provided here only contains lightweight operations.
      *
-     * @param onAcquired ActionListener that is invoked once acquisition is successful or failed
-     * @param executorOnDelay executor to use for delayed call
-     * @param forceExecution whether the runnable should force its execution in case it gets rejected
+     * @param onAcquired      {@link ActionListener} that is invoked once acquisition is successful or failed
+     * @param executorOnDelay executor to use for the possibly delayed {@link ActionListener#onResponse(Object)} call
+     * @param forceExecution  whether the runnable should force its execution in case it gets rejected
      */
     public void acquire(ActionListener<Releasable> onAcquired, String executorOnDelay, boolean forceExecution) {
         if (closed) {
@@ -126,11 +134,12 @@ public void acquire(ActionListener<Releasable> onAcquired, String executorOnDela
                     if (delayedOperations == null) {
                         delayedOperations = new ArrayList<>();
                     }
+                    final Supplier<StoredContext> contextSupplier = threadPool.getThreadContext().newRestorableContext(false);
                     if (executorOnDelay != null) {
-                        delayedOperations.add(
-                            new ThreadedActionListener<>(logger, threadPool, executorOnDelay, onAcquired, forceExecution));
+                        delayedOperations.add(new PermitAwareThreadedActionListener(threadPool, executorOnDelay,
+                                new ContextPreservingActionListener<>(contextSupplier, onAcquired), forceExecution));
                     } else {
-                        delayedOperations.add(onAcquired);
+                        delayedOperations.add(new ContextPreservingActionListener<>(contextSupplier, onAcquired));
                     }
                     return;
                 }
@@ -163,4 +172,56 @@ public int getActiveOperationsCount() {
             return TOTAL_PERMITS - availablePermits;
         }
     }
+
+    /**
+     * A permit-aware action listener wrapper that spawns onResponse listener invocations off on a configurable thread-pool.
+     * Being permit-aware, it also releases the permit when hitting thread-pool rejections and falls back to the
+     * invoker's thread to communicate failures.
+     */
+    private static class PermitAwareThreadedActionListener implements ActionListener<Releasable> {
+
+        private final ThreadPool threadPool;
+        private final String executor;
+        private final ActionListener<Releasable> listener;
+        private final boolean forceExecution;
+
+        private PermitAwareThreadedActionListener(ThreadPool threadPool, String executor, ActionListener<Releasable> listener,
+                                                  boolean forceExecution) {
+            this.threadPool = threadPool;
+            this.executor = executor;
+            this.listener = listener;
+            this.forceExecution = forceExecution;
+        }
+
+        @Override
+        public void onResponse(final Releasable releasable) {
+            threadPool.executor(executor).execute(new AbstractRunnable() {
+                @Override
+                public boolean isForceExecution() {
+                    return forceExecution;
+                }
+
+                @Override
+                protected void doRun() throws Exception {
+                    listener.onResponse(releasable);
+                }
+
+                @Override
+                public void onRejection(Exception e) {
+                    IOUtils.closeWhileHandlingException(releasable);
+                    super.onRejection(e);
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    listener.onFailure(e); // will possibly execute on the caller thread
+                }
+            });
+        }
+
+        @Override
+        public void onFailure(final Exception e) {
+            listener.onFailure(e);
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexingOperationListener.java b/core/src/main/java/org/elasticsearch/index/shard/IndexingOperationListener.java
index 042ddec924e3c..335196fe68198 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/IndexingOperationListener.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/IndexingOperationListener.java
@@ -33,37 +33,47 @@ public interface IndexingOperationListener {
     /**
      * Called before the indexing occurs.
      */
-    default Engine.Index preIndex(Engine.Index operation) {
+    default Engine.Index preIndex(ShardId shardId, Engine.Index operation) {
         return operation;
     }
 
     /**
-     * Called after the indexing operation occurred.
+     * Called after the indexing operation occurred. Note that this is
+     * also called when indexing a document did not succeed due to document
+     * related failures. See {@link #postIndex(ShardId, Engine.Index, Exception)}
+     * for engine level failures
      */
-    default void postIndex(Engine.Index index, boolean created) {}
+    default void postIndex(ShardId shardId, Engine.Index index, Engine.IndexResult result) {}
 
     /**
-     * Called after the indexing operation occurred with exception.
+     * Called after the indexing operation occurred with engine level exception.
+     * See {@link #postIndex(ShardId, Engine.Index, Engine.IndexResult)} for document
+     * related failures
      */
-    default void postIndex(Engine.Index index, Exception ex) {}
+    default void postIndex(ShardId shardId, Engine.Index index, Exception ex) {}
 
     /**
      * Called before the delete occurs.
      */
-    default Engine.Delete preDelete(Engine.Delete delete) {
+    default Engine.Delete preDelete(ShardId shardId, Engine.Delete delete) {
         return delete;
     }
 
 
     /**
-     * Called after the delete operation occurred.
+     * Called after the delete operation occurred. Note that this is
+     * also called when deleting a document did not succeed due to document
+     * related failures. See {@link #postDelete(ShardId, Engine.Delete, Exception)}
+     * for engine level failures
      */
-    default void postDelete(Engine.Delete delete) {}
+    default void postDelete(ShardId shardId, Engine.Delete delete, Engine.DeleteResult result) {}
 
     /**
-     * Called after the delete operation occurred with exception.
+     * Called after the delete operation occurred with engine level exception.
+     * See {@link #postDelete(ShardId, Engine.Delete, Engine.DeleteResult)} for document
+     * related failures
      */
-    default void postDelete(Engine.Delete delete, Exception ex) {}
+    default void postDelete(ShardId shardId, Engine.Delete delete, Exception ex) {}
 
     /**
      * A Composite listener that multiplexes calls to each of the listeners methods.
@@ -78,11 +88,11 @@ public CompositeListener(List<IndexingOperationListener> listeners, Logger logge
         }
 
         @Override
-        public Engine.Index preIndex(Engine.Index operation) {
+        public Engine.Index preIndex(ShardId shardId, Engine.Index operation) {
             assert operation != null;
             for (IndexingOperationListener listener : listeners) {
                 try {
-                    listener.preIndex(operation);
+                    listener.preIndex(shardId, operation);
                 } catch (Exception e) {
                     logger.warn((Supplier<?>) () -> new ParameterizedMessage("preIndex listener [{}] failed", listener), e);
                 }
@@ -91,11 +101,11 @@ public Engine.Index preIndex(Engine.Index operation) {
         }
 
         @Override
-        public void postIndex(Engine.Index index, boolean created) {
+        public void postIndex(ShardId shardId, Engine.Index index, Engine.IndexResult result) {
             assert index != null;
             for (IndexingOperationListener listener : listeners) {
                 try {
-                    listener.postIndex(index, created);
+                    listener.postIndex(shardId, index, result);
                 } catch (Exception e) {
                     logger.warn((Supplier<?>) () -> new ParameterizedMessage("postIndex listener [{}] failed", listener), e);
                 }
@@ -103,11 +113,11 @@ public void postIndex(Engine.Index index, boolean created) {
         }
 
         @Override
-        public void postIndex(Engine.Index index, Exception ex) {
+        public void postIndex(ShardId shardId, Engine.Index index, Exception ex) {
             assert index != null && ex != null;
             for (IndexingOperationListener listener : listeners) {
                 try {
-                    listener.postIndex(index, ex);
+                    listener.postIndex(shardId, index, ex);
                 } catch (Exception inner) {
                     inner.addSuppressed(ex);
                     logger.warn((Supplier<?>) () -> new ParameterizedMessage("postIndex listener [{}] failed", listener), inner);
@@ -116,11 +126,11 @@ public void postIndex(Engine.Index index, Exception ex) {
         }
 
         @Override
-        public Engine.Delete preDelete(Engine.Delete delete) {
+        public Engine.Delete preDelete(ShardId shardId, Engine.Delete delete) {
             assert delete != null;
             for (IndexingOperationListener listener : listeners) {
                 try {
-                    listener.preDelete(delete);
+                    listener.preDelete(shardId, delete);
                 } catch (Exception e) {
                     logger.warn((Supplier<?>) () -> new ParameterizedMessage("preDelete listener [{}] failed", listener), e);
                 }
@@ -129,11 +139,11 @@ public Engine.Delete preDelete(Engine.Delete delete) {
         }
 
         @Override
-        public void postDelete(Engine.Delete delete) {
+        public void postDelete(ShardId shardId, Engine.Delete delete, Engine.DeleteResult result) {
             assert delete != null;
             for (IndexingOperationListener listener : listeners) {
                 try {
-                    listener.postDelete(delete);
+                    listener.postDelete(shardId, delete, result);
                 } catch (Exception e) {
                     logger.warn((Supplier<?>) () -> new ParameterizedMessage("postDelete listener [{}] failed", listener), e);
                 }
@@ -141,11 +151,11 @@ public void postDelete(Engine.Delete delete) {
         }
 
         @Override
-        public void postDelete(Engine.Delete delete, Exception ex) {
+        public void postDelete(ShardId shardId, Engine.Delete delete, Exception ex) {
             assert delete != null && ex != null;
             for (IndexingOperationListener listener : listeners) {
                 try {
-                    listener.postDelete(delete, ex);
+                    listener.postDelete(shardId, delete, ex);
                 } catch (Exception inner) {
                     inner.addSuppressed(ex);
                     logger.warn((Supplier<?>) () -> new ParameterizedMessage("postDelete listener [{}] failed", listener), inner);
diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexingStats.java b/core/src/main/java/org/elasticsearch/index/shard/IndexingStats.java
index 97f9dd2b92fe9..cc657746f10d5 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/IndexingStats.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/IndexingStats.java
@@ -145,11 +145,7 @@ public void readFrom(StreamInput in) throws IOException {
             indexCount = in.readVLong();
             indexTimeInMillis = in.readVLong();
             indexCurrent = in.readVLong();
-
-            if(in.getVersion().onOrAfter(Version.V_2_1_0)){
-                indexFailedCount = in.readVLong();
-            }
-
+            indexFailedCount = in.readVLong();
             deleteCount = in.readVLong();
             deleteTimeInMillis = in.readVLong();
             deleteCurrent = in.readVLong();
@@ -163,11 +159,7 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeVLong(indexCount);
             out.writeVLong(indexTimeInMillis);
             out.writeVLong(indexCurrent);
-
-            if(out.getVersion().onOrAfter(Version.V_2_1_0)) {
-                out.writeVLong(indexFailedCount);
-            }
-
+            out.writeVLong(indexFailedCount);
             out.writeVLong(deleteCount);
             out.writeVLong(deleteTimeInMillis);
             out.writeVLong(deleteCurrent);
@@ -285,21 +277,11 @@ static final class Fields {
         static final String THROTTLED_TIME = "throttle_time";
     }
 
-    public static IndexingStats readIndexingStats(StreamInput in) throws IOException {
-        IndexingStats indexingStats = new IndexingStats();
-        indexingStats.readFrom(in);
-        return indexingStats;
-    }
-
     @Override
     public void readFrom(StreamInput in) throws IOException {
         totalStats = Stats.readStats(in);
         if (in.readBoolean()) {
-            int size = in.readVInt();
-            typeStats = new HashMap<>(size);
-            for (int i = 0; i < size; i++) {
-                typeStats.put(in.readString(), Stats.readStats(in));
-            }
+            typeStats = in.readMap(StreamInput::readString, Stats::readStats);
         }
     }
 
@@ -310,11 +292,7 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeBoolean(false);
         } else {
             out.writeBoolean(true);
-            out.writeVInt(typeStats.size());
-            for (Map.Entry<String, Stats> entry : typeStats.entrySet()) {
-                out.writeString(entry.getKey());
-                entry.getValue().writeTo(out);
-            }
+            out.writeMap(typeStats, StreamOutput::writeString, (stream, stats) -> stats.writeTo(stream));
         }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/shard/InternalIndexingStats.java b/core/src/main/java/org/elasticsearch/index/shard/InternalIndexingStats.java
index f62b8f7fe3c25..ada869a1d9c0b 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/InternalIndexingStats.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/InternalIndexingStats.java
@@ -65,7 +65,7 @@ IndexingStats stats(boolean isThrottled, long currentThrottleInMillis, String...
     }
 
     @Override
-    public Engine.Index preIndex(Engine.Index operation) {
+    public Engine.Index preIndex(ShardId shardId, Engine.Index operation) {
         if (!operation.origin().isRecovery()) {
             totalStats.indexCurrent.inc();
             typeStats(operation.type()).indexCurrent.inc();
@@ -74,19 +74,23 @@ public Engine.Index preIndex(Engine.Index operation) {
     }
 
     @Override
-    public void postIndex(Engine.Index index, boolean created) {
-        if (!index.origin().isRecovery()) {
-            long took = index.endTime() - index.startTime();
-            totalStats.indexMetric.inc(took);
-            totalStats.indexCurrent.dec();
-            StatsHolder typeStats = typeStats(index.type());
-            typeStats.indexMetric.inc(took);
-            typeStats.indexCurrent.dec();
+    public void postIndex(ShardId shardId, Engine.Index index, Engine.IndexResult result) {
+        if (result.hasFailure() == false) {
+            if (!index.origin().isRecovery()) {
+                long took = result.getTook();
+                totalStats.indexMetric.inc(took);
+                totalStats.indexCurrent.dec();
+                StatsHolder typeStats = typeStats(index.type());
+                typeStats.indexMetric.inc(took);
+                typeStats.indexCurrent.dec();
+            }
+        } else {
+            postIndex(shardId, index, result.getFailure());
         }
     }
 
     @Override
-    public void postIndex(Engine.Index index, Exception ex) {
+    public void postIndex(ShardId shardId, Engine.Index index, Exception ex) {
         if (!index.origin().isRecovery()) {
             totalStats.indexCurrent.dec();
             typeStats(index.type()).indexCurrent.dec();
@@ -96,7 +100,7 @@ public void postIndex(Engine.Index index, Exception ex) {
     }
 
     @Override
-    public Engine.Delete preDelete(Engine.Delete delete) {
+    public Engine.Delete preDelete(ShardId shardId, Engine.Delete delete) {
         if (!delete.origin().isRecovery()) {
             totalStats.deleteCurrent.inc();
             typeStats(delete.type()).deleteCurrent.inc();
@@ -106,19 +110,23 @@ public Engine.Delete preDelete(Engine.Delete delete) {
     }
 
     @Override
-    public void postDelete(Engine.Delete delete) {
-        if (!delete.origin().isRecovery()) {
-            long took = delete.endTime() - delete.startTime();
-            totalStats.deleteMetric.inc(took);
-            totalStats.deleteCurrent.dec();
-            StatsHolder typeStats = typeStats(delete.type());
-            typeStats.deleteMetric.inc(took);
-            typeStats.deleteCurrent.dec();
+    public void postDelete(ShardId shardId, Engine.Delete delete, Engine.DeleteResult result) {
+        if (result.hasFailure() == false) {
+            if (!delete.origin().isRecovery()) {
+                long took = result.getTook();
+                totalStats.deleteMetric.inc(took);
+                totalStats.deleteCurrent.dec();
+                StatsHolder typeStats = typeStats(delete.type());
+                typeStats.deleteMetric.inc(took);
+                typeStats.deleteCurrent.dec();
+            }
+        } else {
+            postDelete(shardId, delete, result.getFailure());
         }
     }
 
     @Override
-    public void postDelete(Engine.Delete delete, Exception ex) {
+    public void postDelete(ShardId shardId, Engine.Delete delete, Exception ex) {
         if (!delete.origin().isRecovery()) {
             totalStats.deleteCurrent.dec();
             typeStats(delete.type()).deleteCurrent.dec();
diff --git a/core/src/main/java/org/elasticsearch/index/shard/LocalShardSnapshot.java b/core/src/main/java/org/elasticsearch/index/shard/LocalShardSnapshot.java
index 7b79f785ff338..9fd5113b2ad15 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/LocalShardSnapshot.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/LocalShardSnapshot.java
@@ -26,8 +26,7 @@
 import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.store.Lock;
 import org.apache.lucene.store.NoLockFactory;
-import org.elasticsearch.cluster.metadata.MappingMetaData;
-import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.store.Store;
 
@@ -44,7 +43,7 @@ final class LocalShardSnapshot implements Closeable {
     private final IndexCommit indexCommit;
     private final AtomicBoolean closed = new AtomicBoolean(false);
 
-    public LocalShardSnapshot(IndexShard shard) {
+    LocalShardSnapshot(IndexShard shard) {
         this.shard = shard;
         store = shard.store();
         store.incRef();
@@ -125,8 +124,8 @@ public void close() throws IOException {
         }
     }
 
-    ImmutableOpenMap<String, MappingMetaData> getMappings() {
-        return shard.indexSettings.getIndexMetaData().getMappings();
+    IndexMetaData getIndexMetaData() {
+        return shard.indexSettings.getIndexMetaData();
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java b/core/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java
index ca94f1ea96131..f0df6e12b8cec 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.index.translog.Translog;
 
+import java.io.Closeable;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
@@ -35,18 +36,26 @@
 
 /**
  * Allows for the registration of listeners that are called when a change becomes visible for search. This functionality is exposed from
- * {@link IndexShard} but kept here so it can be tested without standing up the entire thing. 
+ * {@link IndexShard} but kept here so it can be tested without standing up the entire thing.
+ *
+ * When {@link Closeable#close()}d it will no longer accept listeners and flush any existing listeners.
  */
-public final class RefreshListeners implements ReferenceManager.RefreshListener {
+public final class RefreshListeners implements ReferenceManager.RefreshListener, Closeable {
     private final IntSupplier getMaxRefreshListeners;
     private final Runnable forceRefresh;
     private final Executor listenerExecutor;
     private final Logger logger;
 
+    /**
+     * Is this closed? If true then we won't add more listeners and have flushed all pending listeners.
+     */
+    private volatile boolean closed = false;
     /**
      * List of refresh listeners. Defaults to null and built on demand because most refresh cycles won't need it. Entries are never removed
      * from it, rather, it is nulled and rebuilt when needed again. The (hopefully) rare entries that didn't make the current refresh cycle
      * are just added back to the new list. Both the reference and the contents are always modified while synchronized on {@code this}.
+     *
+     * We never set this to non-null while closed it {@code true}.
      */
     private volatile List<Tuple<Translog.Location, Consumer<Boolean>>> refreshListeners = null;
     /**
@@ -80,12 +89,17 @@ public boolean addOrNotify(Translog.Location location, Consumer<Boolean> listene
             return true;
         }
         synchronized (this) {
-            if (refreshListeners == null) {
-                refreshListeners = new ArrayList<>();
+            List<Tuple<Translog.Location, Consumer<Boolean>>> listeners = refreshListeners;
+            if (listeners == null) {
+                if (closed) {
+                    throw new IllegalStateException("can't wait for refresh on a closed index");
+                }
+                listeners = new ArrayList<>();
+                refreshListeners = listeners;
             }
-            if (refreshListeners.size() < getMaxRefreshListeners.getAsInt()) {
+            if (listeners.size() < getMaxRefreshListeners.getAsInt()) {
                 // We have a free slot so register the listener
-                refreshListeners.add(new Tuple<>(location, listener));
+                listeners.add(new Tuple<>(location, listener));
                 return false;
             }
         }
@@ -95,12 +109,34 @@ public boolean addOrNotify(Translog.Location location, Consumer<Boolean> listene
         return true;
     }
 
+    @Override
+    public void close() throws IOException {
+        List<Tuple<Translog.Location, Consumer<Boolean>>> oldListeners;
+        synchronized (this) {
+            oldListeners = refreshListeners;
+            refreshListeners = null;
+            closed = true;
+        }
+        // Fire any listeners we might have had
+        fireListeners(oldListeners);
+    }
+
     /**
      * Returns true if there are pending listeners.
      */
     public boolean refreshNeeded() {
+        // A null list doesn't need a refresh. If we're closed we don't need a refresh either.
+        return refreshListeners != null && false == closed;
+    }
+
+    /**
+     * The number of pending listeners.
+     */
+    public int pendingCount() {
         // No need to synchronize here because we're doing a single volatile read
-        return refreshListeners != null;
+        List<Tuple<Translog.Location, Consumer<Boolean>>> listeners = refreshListeners;
+        // A null list means we haven't accumulated any listeners. Otherwise we need the size.
+        return listeners == null ? 0 : listeners.size();
     }
 
     /**
@@ -125,33 +161,25 @@ public void beforeRefresh() throws IOException {
 
     @Override
     public void afterRefresh(boolean didRefresh) throws IOException {
-        /*
-         * We intentionally ignore didRefresh here because our timing is a little off. It'd be a useful flag if we knew everything that made
+        /* We intentionally ignore didRefresh here because our timing is a little off. It'd be a useful flag if we knew everything that made
          * it into the refresh, but the way we snapshot the translog position before the refresh, things can sneak into the refresh that we
-         * don't know about.
-         */
+         * don't know about. */
         if (null == currentRefreshLocation) {
-            /*
-             * The translog had an empty last write location at the start of the refresh so we can't alert anyone to anything. This
-             * usually happens during recovery. The next refresh cycle out to pick up this refresh.
-             */
+            /* The translog had an empty last write location at the start of the refresh so we can't alert anyone to anything. This
+             * usually happens during recovery. The next refresh cycle out to pick up this refresh. */
             return;
         }
-        /*
-         * Set the lastRefreshedLocation so listeners that come in for locations before that will just execute inline without messing
+        /* Set the lastRefreshedLocation so listeners that come in for locations before that will just execute inline without messing
          * around with refreshListeners or synchronizing at all. Note that it is not safe for us to abort early if we haven't advanced the
          * position here because we set and read lastRefreshedLocation outside of a synchronized block. We do that so that waiting for a
          * refresh that has already passed is just a volatile read but the cost is that any check whether or not we've advanced the
          * position will introduce a race between adding the listener and the position check. We could work around this by moving this
          * assignment into the synchronized block below and double checking lastRefreshedLocation in addOrNotify's synchronized block but
-         * that doesn't seem worth it given that we already skip this process early if there aren't any listeners to iterate.
-         */
+         * that doesn't seem worth it given that we already skip this process early if there aren't any listeners to iterate. */
         lastRefreshedLocation = currentRefreshLocation;
-        /*
-         * Grab the current refresh listeners and replace them with null while synchronized. Any listeners that come in after this won't be
+        /* Grab the current refresh listeners and replace them with null while synchronized. Any listeners that come in after this won't be
          * in the list we iterate over and very likely won't be candidates for refresh anyway because we've already moved the
-         * lastRefreshedLocation.
-         */
+         * lastRefreshedLocation. */
         List<Tuple<Translog.Location, Consumer<Boolean>>> candidates;
         synchronized (this) {
             candidates = refreshListeners;
@@ -162,16 +190,15 @@ public void afterRefresh(boolean didRefresh) throws IOException {
             refreshListeners = null;
         }
         // Iterate the list of listeners, copying the listeners to fire to one list and those to preserve to another list.
-        List<Consumer<Boolean>> listenersToFire = null;
+        List<Tuple<Translog.Location, Consumer<Boolean>>> listenersToFire = null;
         List<Tuple<Translog.Location, Consumer<Boolean>>> preservedListeners = null;
         for (Tuple<Translog.Location, Consumer<Boolean>> tuple : candidates) {
             Translog.Location location = tuple.v1();
-            Consumer<Boolean> listener = tuple.v2();
             if (location.compareTo(currentRefreshLocation) <= 0) {
                 if (listenersToFire == null) {
                     listenersToFire = new ArrayList<>();
                 }
-                listenersToFire.add(listener);
+                listenersToFire.add(tuple);
             } else {
                 if (preservedListeners == null) {
                     preservedListeners = new ArrayList<>();
@@ -179,27 +206,36 @@ public void afterRefresh(boolean didRefresh) throws IOException {
                 preservedListeners.add(tuple);
             }
         }
-        /*
-         * Now add any preserved listeners back to the running list of refresh listeners while under lock. We'll try them next time. While
-         * we were iterating the list of listeners new listeners could have come in. That means that adding all of our preserved listeners
-         * might push our list of listeners above the maximum number of slots allowed. This seems unlikely because we expect few listeners
-         * to be preserved. And the next listener while we're full will trigger a refresh anyway.
-         */
+        /* Now deal with the listeners that it isn't time yet to fire. We need to do this under lock so we don't miss a concurrent close or
+         * newly registered listener. If we're not closed we just add the listeners to the list of listeners we check next time. If we are
+         * closed we fire the listeners even though it isn't time for them. */
         if (preservedListeners != null) {
             synchronized (this) {
                 if (refreshListeners == null) {
-                    refreshListeners = new ArrayList<>();
+                    if (closed) {
+                        listenersToFire.addAll(preservedListeners);
+                    } else {
+                        refreshListeners = preservedListeners;
+                    }
+                } else {
+                    assert closed == false : "Can't be closed and have non-null refreshListeners";
+                    refreshListeners.addAll(preservedListeners);
                 }
-                refreshListeners.addAll(preservedListeners);
             }
         }
         // Lastly, fire the listeners that are ready on the listener thread pool
+        fireListeners(listenersToFire);
+    }
+
+    /**
+     * Fire some listeners. Does nothing if the list of listeners is null.
+     */
+    private void fireListeners(List<Tuple<Translog.Location, Consumer<Boolean>>> listenersToFire) {
         if (listenersToFire != null) {
-            final List<Consumer<Boolean>> finalListenersToFire = listenersToFire;
             listenerExecutor.execute(() -> {
-                for (Consumer<Boolean> listener : finalListenersToFire) {
+                for (Tuple<Translog.Location, Consumer<Boolean>> listener : listenersToFire) {
                     try {
-                        listener.accept(false);
+                        listener.v2().accept(false);
                     } catch (Exception e) {
                         logger.warn("Error firing refresh listener", e);
                     }
diff --git a/core/src/main/java/org/elasticsearch/index/shard/SearchOperationListener.java b/core/src/main/java/org/elasticsearch/index/shard/SearchOperationListener.java
index 11723c3d50a01..153a985ab0892 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/SearchOperationListener.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/SearchOperationListener.java
@@ -21,7 +21,9 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
+import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.transport.TransportRequest;
 
 import java.util.List;
 
@@ -104,6 +106,15 @@ public interface SearchOperationListener {
      */
     default void onFreeScrollContext(SearchContext context) {};
 
+    /**
+     * Executed prior to using a {@link SearchContext} that has been retrieved
+     * from the active contexts. If the context is deemed invalid a runtime
+     * exception can be thrown, which will prevent the context from being used.
+     * @param context the context retrieved from the active contexts
+     * @param transportRequest the request that is going to use the search context
+     */
+    default void validateSearchContext(SearchContext context, TransportRequest transportRequest) {}
+
     /**
      * A Composite listener that multiplexes calls to each of the listeners methods.
      */
@@ -225,5 +236,18 @@ public void onFreeScrollContext(SearchContext context) {
                 }
             }
         }
+
+        @Override
+        public void validateSearchContext(SearchContext context, TransportRequest request) {
+            Exception exception = null;
+            for (SearchOperationListener listener : listeners) {
+                try {
+                    listener.validateSearchContext(context, request);
+                } catch (Exception e) {
+                    exception = ExceptionsHelper.useOrSuppress(exception, e);
+                }
+            }
+            ExceptionsHelper.reThrowIfNotNull(exception);
+        }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java
index 45a471e1aa9fb..fcab0cf3fc752 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java
@@ -65,7 +65,7 @@ public ShadowIndexShard(ShardRouting shardRouting, IndexSettings indexSettings,
      */
     @Override
     public void updateRoutingEntry(ShardRouting newRouting) throws IOException {
-        if (newRouting.primary() == true) {// becoming a primary
+        if (newRouting.primary()) {// becoming a primary
             throw new IllegalStateException("can't promote shard to primary");
         }
         super.updateRoutingEntry(newRouting);
@@ -114,4 +114,9 @@ public void addRefreshListener(Translog.Location location, Consumer<Boolean> lis
     public Store.MetadataSnapshot snapshotStoreMetadata() throws IOException {
         throw new UnsupportedOperationException("can't snapshot the directory as the primary may change it underneath us");
     }
+
+    @Override
+    protected void onNewEngine(Engine newEngine) {
+        // nothing to do here - the superclass sets the translog on some listeners but we don't have such a thing
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/shard/ShardId.java b/core/src/main/java/org/elasticsearch/index/shard/ShardId.java
index a9bc63ae44f63..a806c414e9aea 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/ShardId.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/ShardId.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.index.shard;
 
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
@@ -71,6 +72,22 @@ public String toString() {
         return "[" + index.getName() + "][" + shardId + "]";
     }
 
+    /**
+     * Parse the string representation of this shardId back to an object.
+     * We lose index uuid information here, but since we use toString in
+     * rest responses, this is the best we can do to reconstruct the object
+     * on the client side.
+     */
+    public static ShardId fromString(String shardIdString) {
+        int splitPosition = shardIdString.indexOf("][");
+        if (splitPosition <= 0 || shardIdString.charAt(0) != '[' || shardIdString.charAt(shardIdString.length() - 1) != ']') {
+            throw new IllegalArgumentException("Unexpected shardId string format, expected [indexName][shardId] but got " + shardIdString);
+        }
+        String indexName = shardIdString.substring(1, splitPosition);
+        int shardId = Integer.parseInt(shardIdString.substring(splitPosition + 2, shardIdString.length() - 1));
+        return new ShardId(new Index(indexName, IndexMetaData.INDEX_UUID_NA_VALUE), shardId);
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
diff --git a/core/src/main/java/org/elasticsearch/index/shard/ShardPath.java b/core/src/main/java/org/elasticsearch/index/shard/ShardPath.java
index 23b17c290f12e..eb2d530fd99e0 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/ShardPath.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/ShardPath.java
@@ -21,6 +21,7 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.env.ShardLock;
 import org.elasticsearch.index.IndexSettings;
@@ -113,7 +114,8 @@ public static ShardPath loadShardPath(Logger logger, NodeEnvironment env, ShardI
         final Path[] paths = env.availableShardPaths(shardId);
         Path loadedPath = null;
         for (Path path : paths) {
-            ShardStateMetaData load = ShardStateMetaData.FORMAT.loadLatestState(logger, path);
+            // EMPTY is safe here because we never call namedObject
+            ShardStateMetaData load = ShardStateMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, path);
             if (load != null) {
                 if (load.indexUUID.equals(indexUUID) == false && IndexMetaData.INDEX_UUID_NA_VALUE.equals(load.indexUUID) == false) {
                     logger.warn("{} found shard on path: [{}] with a different index UUID - this shard seems to be leftover from a different index with the same name. Remove the leftover shard in order to reuse the path with the current index", shardId, path);
@@ -150,7 +152,8 @@ public static void deleteLeftoverShardDirectory(Logger logger, NodeEnvironment e
         final String indexUUID = indexSettings.getUUID();
         final Path[] paths = env.availableShardPaths(lock.getShardId());
         for (Path path : paths) {
-            ShardStateMetaData load = ShardStateMetaData.FORMAT.loadLatestState(logger, path);
+            // EMPTY is safe here because we never call namedObject
+            ShardStateMetaData load = ShardStateMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, path);
             if (load != null) {
                 if (load.indexUUID.equals(indexUUID) == false && IndexMetaData.INDEX_UUID_NA_VALUE.equals(load.indexUUID) == false) {
                     logger.warn("{} deleting leftover shard on path: [{}] with a different index UUID", lock.getShardId(), path);
diff --git a/core/src/main/java/org/elasticsearch/index/shard/SnapshotStatus.java b/core/src/main/java/org/elasticsearch/index/shard/SnapshotStatus.java
index a72cfc48b6506..2014f81ec56fa 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/SnapshotStatus.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/SnapshotStatus.java
@@ -24,7 +24,7 @@
  */
 public class SnapshotStatus {
 
-    public static enum Stage {
+    public enum Stage {
         NONE,
         INDEX,
         TRANSLOG,
diff --git a/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java b/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
index 44b4ed933f7cd..04c2113dea34b 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
@@ -104,12 +104,11 @@ boolean recoverFromLocalShards(BiConsumer<String, MappingMetaData> mappingUpdate
             if (indices.size() > 1) {
                 throw new IllegalArgumentException("can't add shards from more than one index");
             }
-            for (ObjectObjectCursor<String, MappingMetaData> mapping : shards.get(0).getMappings()) {
+            IndexMetaData indexMetaData = shards.get(0).getIndexMetaData();
+            for (ObjectObjectCursor<String, MappingMetaData> mapping : indexMetaData.getMappings()) {
                 mappingUpdateConsumer.accept(mapping.key, mapping.value);
             }
-            for (ObjectObjectCursor<String, MappingMetaData> mapping : shards.get(0).getMappings()) {
-                indexShard.mapperService().merge(mapping.key,mapping.value.source(), MapperService.MergeReason.MAPPING_RECOVERY, true);
-            }
+            indexShard.mapperService().merge(indexMetaData, MapperService.MergeReason.MAPPING_RECOVERY, true);
             return executeRecovery(indexShard, () -> {
                 logger.debug("starting recovery from local shards {}", shards);
                 try {
@@ -363,7 +362,7 @@ private void internalRecoverFromStore(IndexShard indexShard) throws IndexShardRe
             indexShard.finalizeRecovery();
             indexShard.postRecovery("post recovery from shard_store");
         } catch (EngineException | IOException e) {
-            throw new IndexShardRecoveryException(shardId, "failed to recovery from gateway", e);
+            throw new IndexShardRecoveryException(shardId, "failed to recover from gateway", e);
         } finally {
             store.decRef();
         }
diff --git a/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java b/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java
index 64ae0c7700752..6aaff1b5cd93b 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java
@@ -22,13 +22,13 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.engine.IgnoreOnRecoveryEngineException;
 import org.elasticsearch.index.mapper.DocumentMapperForType;
 import org.elasticsearch.index.mapper.MapperException;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.Mapping;
-import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.rest.RestStatus;
 
@@ -146,7 +146,7 @@ private void maybeAddMappingUpdate(String type, Mapping update, String docId, bo
      *                            cause a {@link MapperException} to be thrown if an update
      *                            is encountered.
      */
-    private void performRecoveryOperation(Engine engine, Translog.Operation operation, boolean allowMappingUpdates, Engine.Operation.Origin origin) {
+    private void performRecoveryOperation(Engine engine, Translog.Operation operation, boolean allowMappingUpdates, Engine.Operation.Origin origin) throws IOException {
 
         try {
             switch (operation.opType()) {
@@ -154,7 +154,8 @@ private void performRecoveryOperation(Engine engine, Translog.Operation operatio
                     Translog.Index index = (Translog.Index) operation;
                     // we set canHaveDuplicates to true all the time such that we de-optimze the translog case and ensure that all
                     // autoGeneratedID docs that are coming from the primary are updated correctly.
-                    Engine.Index engineIndex = IndexShard.prepareIndex(docMapper(index.type()), source(shardId.getIndexName(), index.type(), index.id(), index.source())
+                    Engine.Index engineIndex = IndexShard.prepareIndex(docMapper(index.type()),
+                        source(shardId.getIndexName(), index.type(), index.id(), index.source(), XContentFactory.xContentType(index.source()))
                             .routing(index.routing()).parent(index.parent()).timestamp(index.timestamp()).ttl(index.ttl()),
                         index.version(), index.versionType().versionTypeForReplicationAndRecovery(), origin, index.getAutoGeneratedIdTimestamp(), true);
                     maybeAddMappingUpdate(engineIndex.type(), engineIndex.parsedDoc().dynamicMappingsUpdate(), engineIndex.id(), allowMappingUpdates);
@@ -165,12 +166,11 @@ private void performRecoveryOperation(Engine engine, Translog.Operation operatio
                     break;
                 case DELETE:
                     Translog.Delete delete = (Translog.Delete) operation;
-                    Uid uid = Uid.createUid(delete.uid().text());
                     if (logger.isTraceEnabled()) {
-                        logger.trace("[translog] recover [delete] op of [{}][{}]", uid.type(), uid.id());
+                        logger.trace("[translog] recover [delete] op of [{}][{}]", delete.type(), delete.id());
                     }
-                    final Engine.Delete engineDelete = new Engine.Delete(uid.type(), uid.id(), delete.uid(), delete.version(),
-                        delete.versionType().versionTypeForReplicationAndRecovery(), origin, System.nanoTime(), false);
+                    final Engine.Delete engineDelete = new Engine.Delete(delete.type(), delete.id(), delete.uid(), delete.version(),
+                        delete.versionType().versionTypeForReplicationAndRecovery(), origin, System.nanoTime());
                     delete(engine, engineDelete);
                     break;
                 default:
@@ -197,11 +197,11 @@ private void performRecoveryOperation(Engine engine, Translog.Operation operatio
         operationProcessed();
     }
 
-    protected void index(Engine engine, Engine.Index engineIndex) {
+    protected void index(Engine engine, Engine.Index engineIndex) throws IOException {
         engine.index(engineIndex);
     }
 
-    protected void delete(Engine engine, Engine.Delete engineDelete) {
+    protected void delete(Engine engine, Engine.Delete engineDelete) throws IOException {
         engine.delete(engineDelete);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/similarity/BooleanSimilarityProvider.java b/core/src/main/java/org/elasticsearch/index/similarity/BooleanSimilarityProvider.java
new file mode 100644
index 0000000000000..c4903eda96162
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/similarity/BooleanSimilarityProvider.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.similarity;
+
+import org.apache.lucene.search.similarities.BooleanSimilarity;
+import org.elasticsearch.common.settings.Settings;
+
+/**
+ * {@link SimilarityProvider} for the {@link BooleanSimilarity},
+ * which is a simple similarity that gives terms a score equal
+ * to their query boost only.  This is useful in situations where
+ * a field does not need to be scored by a full-text ranking
+ * algorithm, but rather all that matters is whether the query
+ * terms matched or not.
+ */
+public class BooleanSimilarityProvider extends AbstractSimilarityProvider {
+
+    private final BooleanSimilarity similarity = new BooleanSimilarity();
+
+    public BooleanSimilarityProvider(String name, Settings settings) {
+        super(name);
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public BooleanSimilarity get() {
+        return similarity;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java b/core/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java
index a8b7fafb9804e..838f9764fa9ea 100644
--- a/core/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java
+++ b/core/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java
@@ -44,18 +44,19 @@ public final class SimilarityService extends AbstractIndexComponent {
     public static final Map<String, BiFunction<String, Settings, SimilarityProvider>> BUILT_IN;
     static {
         Map<String, BiFunction<String, Settings, SimilarityProvider>> defaults = new HashMap<>();
-        Map<String, BiFunction<String, Settings, SimilarityProvider>> buildIn = new HashMap<>();
         defaults.put("classic", ClassicSimilarityProvider::new);
         defaults.put("BM25", BM25SimilarityProvider::new);
-        buildIn.put("classic", ClassicSimilarityProvider::new);
-        buildIn.put("BM25", BM25SimilarityProvider::new);
-        buildIn.put("DFR", DFRSimilarityProvider::new);
-        buildIn.put("IB", IBSimilarityProvider::new);
-        buildIn.put("LMDirichlet", LMDirichletSimilarityProvider::new);
-        buildIn.put("LMJelinekMercer", LMJelinekMercerSimilarityProvider::new);
-        buildIn.put("DFI", DFISimilarityProvider::new);
+        defaults.put("boolean", BooleanSimilarityProvider::new);
+
+        Map<String, BiFunction<String, Settings, SimilarityProvider>> builtIn = new HashMap<>(defaults);
+        builtIn.put("DFR", DFRSimilarityProvider::new);
+        builtIn.put("IB", IBSimilarityProvider::new);
+        builtIn.put("LMDirichlet", LMDirichletSimilarityProvider::new);
+        builtIn.put("LMJelinekMercer", LMJelinekMercerSimilarityProvider::new);
+        builtIn.put("DFI", DFISimilarityProvider::new);
+
         DEFAULTS = Collections.unmodifiableMap(defaults);
-        BUILT_IN = Collections.unmodifiableMap(buildIn);
+        BUILT_IN = Collections.unmodifiableMap(builtIn);
     }
 
     public SimilarityService(IndexSettings indexSettings, Map<String, BiFunction<String, Settings, SimilarityProvider>> similarities) {
diff --git a/core/src/main/java/org/elasticsearch/index/snapshots/IndexShardSnapshotStatus.java b/core/src/main/java/org/elasticsearch/index/snapshots/IndexShardSnapshotStatus.java
index 51c8bcf5d7e27..644caa7520be5 100644
--- a/core/src/main/java/org/elasticsearch/index/snapshots/IndexShardSnapshotStatus.java
+++ b/core/src/main/java/org/elasticsearch/index/snapshots/IndexShardSnapshotStatus.java
@@ -27,7 +27,7 @@ public class IndexShardSnapshotStatus {
     /**
      * Snapshot stage
      */
-    public static enum Stage {
+    public enum Stage {
         /**
          * Snapshot hasn't started yet
          */
@@ -66,7 +66,7 @@ public static enum Stage {
 
     private long indexVersion;
 
-    private boolean aborted;
+    private volatile boolean aborted;
 
     private String failure;
 
diff --git a/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java b/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java
index 5bb0f728bc14e..37b728d43d622 100644
--- a/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java
+++ b/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java
@@ -23,11 +23,9 @@
 import org.apache.lucene.util.Version;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.common.xcontent.FromXContentBuilder;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -41,9 +39,7 @@
 /**
  * Shard snapshot metadata
  */
-public class BlobStoreIndexShardSnapshot implements ToXContent, FromXContentBuilder<BlobStoreIndexShardSnapshot> {
-
-    public static final BlobStoreIndexShardSnapshot PROTO = new BlobStoreIndexShardSnapshot();
+public class BlobStoreIndexShardSnapshot implements ToXContent {
 
     /**
      * Information about snapshotted file
@@ -70,7 +66,7 @@ public FileInfo(String name, StoreFileMetaData metaData, ByteSizeValue partSize)
 
             long partBytes = Long.MAX_VALUE;
             if (partSize != null) {
-                partBytes = partSize.bytes();
+                partBytes = partSize.getBytes();
             }
 
             long totalLength = metaData.length();
@@ -261,7 +257,7 @@ public static void toXContent(FileInfo file, XContentBuilder builder, ToXContent
                 builder.field(CHECKSUM, file.metadata.checksum());
             }
             if (file.partSize != null) {
-                builder.field(PART_SIZE, file.partSize.bytes());
+                builder.field(PART_SIZE, file.partSize.getBytes());
             }
 
             if (file.metadata.writtenBy() != null) {
@@ -507,8 +503,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
      * @param parser parser
      * @return shard snapshot metadata
      */
-    public BlobStoreIndexShardSnapshot fromXContent(XContentParser parser, ParseFieldMatcher parseFieldMatcher) throws IOException {
-
+    public static BlobStoreIndexShardSnapshot fromXContent(XContentParser parser) throws IOException {
         String snapshot = null;
         long indexVersion = -1;
         long startTime = 0;
@@ -527,24 +522,24 @@ public BlobStoreIndexShardSnapshot fromXContent(XContentParser parser, ParseFiel
                     String currentFieldName = parser.currentName();
                     token = parser.nextToken();
                     if (token.isValue()) {
-                        if (parseFieldMatcher.match(currentFieldName, PARSE_NAME)) {
+                        if (PARSE_NAME.match(currentFieldName)) {
                             snapshot = parser.text();
-                        } else if (parseFieldMatcher.match(currentFieldName, PARSE_INDEX_VERSION)) {
+                        } else if (PARSE_INDEX_VERSION.match(currentFieldName)) {
                             // The index-version is needed for backward compatibility with v 1.0
                             indexVersion = parser.longValue();
-                        } else if (parseFieldMatcher.match(currentFieldName, PARSE_START_TIME)) {
+                        } else if (PARSE_START_TIME.match(currentFieldName)) {
                             startTime = parser.longValue();
-                        } else if (parseFieldMatcher.match(currentFieldName, PARSE_TIME)) {
+                        } else if (PARSE_TIME.match(currentFieldName)) {
                             time = parser.longValue();
-                        } else if (parseFieldMatcher.match(currentFieldName, PARSE_NUMBER_OF_FILES)) {
+                        } else if (PARSE_NUMBER_OF_FILES.match(currentFieldName)) {
                             numberOfFiles = parser.intValue();
-                        } else if (parseFieldMatcher.match(currentFieldName, PARSE_TOTAL_SIZE)) {
+                        } else if (PARSE_TOTAL_SIZE.match(currentFieldName)) {
                             totalSize = parser.longValue();
                         } else {
                             throw new ElasticsearchParseException("unknown parameter [{}]", currentFieldName);
                         }
                     } else if (token == XContentParser.Token.START_ARRAY) {
-                        if (parseFieldMatcher.match(currentFieldName, PARSE_FILES)) {
+                        if (PARSE_FILES.match(currentFieldName)) {
                             while ((parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                                 indexFiles.add(FileInfo.fromXContent(parser));
                             }
@@ -562,5 +557,4 @@ public BlobStoreIndexShardSnapshot fromXContent(XContentParser parser, ParseFiel
         return new BlobStoreIndexShardSnapshot(snapshot, indexVersion, Collections.unmodifiableList(indexFiles),
                                                startTime, time, numberOfFiles, totalSize);
     }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java b/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java
index 5b66d9b6f6ff8..56c343a5ae9b5 100644
--- a/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java
+++ b/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java
@@ -21,8 +21,6 @@
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.xcontent.FromXContentBuilder;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -44,9 +42,7 @@
  * This class is used to find files that were already snapshotted and clear out files that no longer referenced by any
  * snapshots
  */
-public class BlobStoreIndexShardSnapshots implements Iterable<SnapshotFiles>, ToXContent, FromXContentBuilder<BlobStoreIndexShardSnapshots> {
-
-    public static final BlobStoreIndexShardSnapshots PROTO = new BlobStoreIndexShardSnapshots();
+public class BlobStoreIndexShardSnapshots implements Iterable<SnapshotFiles>, ToXContent {
 
     private final List<SnapshotFiles> shardSnapshots;
     private final Map<String, FileInfo> files;
@@ -232,8 +228,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         return builder;
     }
 
-    @Override
-    public BlobStoreIndexShardSnapshots fromXContent(XContentParser parser, ParseFieldMatcher parseFieldMatcher) throws IOException {
+    public static BlobStoreIndexShardSnapshots fromXContent(XContentParser parser) throws IOException {
         XContentParser.Token token = parser.currentToken();
         if (token == null) { // New parser
             token = parser.nextToken();
@@ -248,7 +243,7 @@ public BlobStoreIndexShardSnapshots fromXContent(XContentParser parser, ParseFie
                 String currentFieldName = parser.currentName();
                 token = parser.nextToken();
                 if (token == XContentParser.Token.START_ARRAY) {
-                    if (parseFieldMatcher.match(currentFieldName, ParseFields.FILES) == false) {
+                    if (ParseFields.FILES.match(currentFieldName) == false) {
                         throw new ElasticsearchParseException("unknown array [{}]", currentFieldName);
                     }
                     while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
@@ -256,7 +251,7 @@ public BlobStoreIndexShardSnapshots fromXContent(XContentParser parser, ParseFie
                         files.put(fileInfo.name(), fileInfo);
                     }
                 } else if (token == XContentParser.Token.START_OBJECT) {
-                    if (parseFieldMatcher.match(currentFieldName, ParseFields.SNAPSHOTS) == false) {
+                    if (ParseFields.SNAPSHOTS.match(currentFieldName) == false) {
                         throw new ElasticsearchParseException("unknown object [{}]", currentFieldName);
                     }
                     while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
@@ -271,7 +266,7 @@ public BlobStoreIndexShardSnapshots fromXContent(XContentParser parser, ParseFie
                             if (token == XContentParser.Token.FIELD_NAME) {
                                 currentFieldName = parser.currentName();
                                 if (parser.nextToken() == XContentParser.Token.START_ARRAY) {
-                                    if (parseFieldMatcher.match(currentFieldName, ParseFields.FILES) == false) {
+                                    if (ParseFields.FILES.match(currentFieldName) == false) {
                                         throw new ElasticsearchParseException("unknown array [{}]", currentFieldName);
                                     }
                                     List<String> fileNames = new ArrayList<>();
@@ -289,7 +284,7 @@ public BlobStoreIndexShardSnapshots fromXContent(XContentParser parser, ParseFie
             }
         }
 
-        List<SnapshotFiles> snapshots = new ArrayList<>();
+        List<SnapshotFiles> snapshots = new ArrayList<>(snapshotsMap.size());
         for (Map.Entry<String, List<String>> entry : snapshotsMap.entrySet()) {
             List<FileInfo> fileInfosBuilder = new ArrayList<>();
             for (String file : entry.getValue()) {
diff --git a/core/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java b/core/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java
index 69eedd7ef190c..c758d43778352 100644
--- a/core/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java
+++ b/core/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java
@@ -91,7 +91,7 @@ public Directory newDirectory() throws IOException {
         Set<String> preLoadExtensions = new HashSet<>(
                 indexSettings.getValue(IndexModule.INDEX_STORE_PRE_LOAD_SETTING));
         wrapped = setPreload(wrapped, location, lockFactory, preLoadExtensions);
-        if (IndexMetaData.isOnSharedFilesystem(indexSettings.getSettings())) {
+        if (indexSettings.isOnSharedFilesystem()) {
             wrapped = new SleepingLockWrapper(wrapped, 5000);
         }
         return new RateLimitedFSDirectory(wrapped, this, this) ;
@@ -105,7 +105,10 @@ public void onPause(long nanos) {
     protected Directory newFSDirectory(Path location, LockFactory lockFactory) throws IOException {
         final String storeType = indexSettings.getSettings().get(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(),
             IndexModule.Type.FS.getSettingsKey());
-        if (IndexModule.Type.FS.match(storeType) || IndexModule.Type.DEFAULT.match(storeType)) {
+        if (IndexModule.Type.FS.match(storeType)) {
+            return FSDirectory.open(location, lockFactory); // use lucene defaults
+        } else if (IndexModule.Type.DEFAULT.match(storeType)) {
+            deprecationLogger.deprecated("[default] store type is deprecated, use [fs] instead");
             return FSDirectory.open(location, lockFactory); // use lucene defaults
         } else if (IndexModule.Type.SIMPLEFS.match(storeType)) {
             return new SimpleFSDirectory(location, lockFactory);
diff --git a/core/src/main/java/org/elasticsearch/index/store/Store.java b/core/src/main/java/org/elasticsearch/index/store/Store.java
index abcb4a44c8086..88b1ba3fb8696 100644
--- a/core/src/main/java/org/elasticsearch/index/store/Store.java
+++ b/core/src/main/java/org/elasticsearch/index/store/Store.java
@@ -65,7 +65,6 @@
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.util.Callback;
 import org.elasticsearch.common.util.SingleObjectCache;
 import org.elasticsearch.common.util.concurrent.AbstractRefCounted;
 import org.elasticsearch.common.util.concurrent.RefCounted;
@@ -84,6 +83,7 @@
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
+import java.nio.file.AccessDeniedException;
 import java.nio.file.NoSuchFileException;
 import java.nio.file.Path;
 import java.util.ArrayList;
@@ -96,6 +96,7 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.function.Consumer;
 import java.util.zip.CRC32;
 import java.util.zip.Checksum;
 
@@ -222,7 +223,8 @@ final void ensureOpen() {
      * {@link #readMetadataSnapshot(Path, ShardId, NodeEnvironment.ShardLocker, Logger)} to read a meta data while locking
      * {@link IndexShard#snapshotStoreMetadata()} to safely read from an existing shard
      * {@link IndexShard#acquireIndexCommit(boolean)} to get an {@link IndexCommit} which is safe to use but has to be freed
-     *
+     * @param commit the index commit to read the snapshot from or <code>null</code> if the latest snapshot should be read from the
+     *               directory
      * @throws CorruptIndexException      if the lucene index is corrupted. This can be caused by a checksum mismatch or an
      *                                    unexpected exception when opening the index reading the segments file.
      * @throws IndexFormatTooOldException if the lucene index is too old to be opened.
@@ -232,20 +234,50 @@ final void ensureOpen() {
      * @throws IndexNotFoundException     if the commit point can't be found in this store
      */
     public MetadataSnapshot getMetadata(IndexCommit commit) throws IOException {
+        return getMetadata(commit, false);
+    }
+
+    /**
+     * Returns a new MetadataSnapshot for the given commit. If the given commit is <code>null</code>
+     * the latest commit point is used.
+     *
+     * Note that this method requires the caller verify it has the right to access the store and
+     * no concurrent file changes are happening. If in doubt, you probably want to use one of the following:
+     *
+     * {@link #readMetadataSnapshot(Path, ShardId, NodeEnvironment.ShardLocker, Logger)} to read a meta data while locking
+     * {@link IndexShard#snapshotStoreMetadata()} to safely read from an existing shard
+     * {@link IndexShard#acquireIndexCommit(boolean)} to get an {@link IndexCommit} which is safe to use but has to be freed
+     *
+     * @param commit the index commit to read the snapshot from or <code>null</code> if the latest snapshot should be read from the
+     *               directory
+     * @param lockDirectory if <code>true</code> the index writer lock will be obtained before reading the snapshot. This should
+     *                      only be used if there is no started shard using this store.
+     * @throws CorruptIndexException      if the lucene index is corrupted. This can be caused by a checksum mismatch or an
+     *                                    unexpected exception when opening the index reading the segments file.
+     * @throws IndexFormatTooOldException if the lucene index is too old to be opened.
+     * @throws IndexFormatTooNewException if the lucene index is too new to be opened.
+     * @throws FileNotFoundException      if one or more files referenced by a commit are not present.
+     * @throws NoSuchFileException        if one or more files referenced by a commit are not present.
+     * @throws IndexNotFoundException     if the commit point can't be found in this store
+     */
+    public MetadataSnapshot getMetadata(IndexCommit commit, boolean lockDirectory) throws IOException {
         ensureOpen();
         failIfCorrupted();
-        metadataLock.readLock().lock();
-        try {
+        assert lockDirectory ? commit == null : true : "IW lock should not be obtained if there is a commit point available";
+        // if we lock the directory we also acquire the write lock since that makes sure that nobody else tries to lock the IW
+        // on this store at the same time.
+        java.util.concurrent.locks.Lock lock = lockDirectory ? metadataLock.writeLock() : metadataLock.readLock();
+        lock.lock();
+        try (Closeable ignored = lockDirectory ? directory.obtainLock(IndexWriter.WRITE_LOCK_NAME) : () -> {} ) {
             return new MetadataSnapshot(commit, directory, logger);
         } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
             markStoreCorrupted(ex);
             throw ex;
         } finally {
-            metadataLock.readLock().unlock();
+            lock.unlock();
         }
     }
 
-
     /**
      * Renames all the given files from the key of the map to the
      * value of the map. All successfully renamed files are removed from the map in-place.
@@ -362,7 +394,7 @@ private void closeInternal() {
             try {
                 directory.innerClose(); // this closes the distributorDirectory as well
             } finally {
-                onClose.handle(shardLock);
+                onClose.accept(shardLock);
             }
         } catch (IOException e) {
             logger.debug("failed to close directory", e);
@@ -371,7 +403,6 @@ private void closeInternal() {
         }
     }
 
-
     /**
      * Reads a MetadataSnapshot from the given index locations or returns an empty snapshot if it can't be read.
      *
@@ -388,7 +419,7 @@ public static MetadataSnapshot readMetadataSnapshot(Path indexLocation, ShardId
         } catch (FileNotFoundException | NoSuchFileException ex) {
             logger.info("Failed to open / find files while reading metadata snapshot");
         } catch (ShardLockObtainFailedException ex) {
-            logger.info("{}: failed to obtain shard lock", ex, shardId);
+            logger.info((Supplier<?>) () -> new ParameterizedMessage("{}: failed to obtain shard lock", shardId), ex);
         }
         return MetadataSnapshot.EMPTY;
     }
@@ -413,15 +444,12 @@ public static boolean canOpenIndex(Logger logger, Path indexLocation, ShardId sh
      * segment infos and possible corruption markers. If the index can not
      * be opened, an exception is thrown
      */
-    public static void tryOpenIndex(Path indexLocation, ShardId shardId, NodeEnvironment.ShardLocker shardLocker, Logger logger) throws IOException {
+    public static void tryOpenIndex(Path indexLocation, ShardId shardId, NodeEnvironment.ShardLocker shardLocker, Logger logger) throws IOException, ShardLockObtainFailedException {
         try (ShardLock lock = shardLocker.lock(shardId, TimeUnit.SECONDS.toMillis(5));
              Directory dir = new SimpleFSDirectory(indexLocation)) {
             failIfCorrupted(dir, shardId);
             SegmentInfos segInfo = Lucene.readSegmentInfos(dir);
             logger.trace("{} loaded segment info [{}]", shardId, segInfo);
-        } catch (ShardLockObtainFailedException ex) {
-            logger.error("{} unable to acquire shard lock", ex, shardId);
-            throw new IOException(ex);
         }
     }
 
@@ -582,7 +610,7 @@ private static void failIfCorrupted(Directory directory, ShardId shardId) throws
     /**
      * This method deletes every file in this store that is not contained in the given source meta data or is a
      * legacy checksum file. After the delete it pulls the latest metadata snapshot from the store and compares it
-     * to the given snapshot. If the snapshots are inconsistent an illegal state exception is thrown
+     * to the given snapshot. If the snapshots are inconsistent an illegal state exception is thrown.
      *
      * @param reason         the reason for this cleanup operation logged for each deleted file
      * @param sourceMetaData the metadata used for cleanup. all files in this metadata should be kept around.
@@ -626,9 +654,9 @@ final void verifyAfterCleanup(MetadataSnapshot sourceMetaData, MetadataSnapshot
                 for (StoreFileMetaData meta : recoveryDiff.different) {
                     StoreFileMetaData local = targetMetaData.get(meta.name());
                     StoreFileMetaData remote = sourceMetaData.get(meta.name());
-                    // if we have different files the they must have no checksums otherwise something went wrong during recovery.
-                    // we have that problem when we have an empty index is only a segments_1 file then we can't tell if it's a Lucene 4.8 file
-                    // and therefore no checksum. That isn't much of a problem since we simply copy it over anyway but those files come out as
+                    // if we have different files then they must have no checksums; otherwise something went wrong during recovery.
+                    // we have that problem when we have an empty index is only a segments_1 file so we can't tell if it's a Lucene 4.8 file
+                    // and therefore no checksum is included. That isn't a problem since we simply copy it over anyway but those files come out as
                     // different in the diff. That's why we have to double check here again if the rest of it matches.
 
                     // all is fine this file is just part of a commit or a segment that is different
@@ -661,7 +689,6 @@ static final class StoreDirectory extends FilterDirectory {
             this.deletesLogger = deletesLogger;
         }
 
-
         @Override
         public void close() throws IOException {
             assert false : "Nobody should close this directory except of the Store itself";
@@ -843,7 +870,7 @@ private static void checksumFromLuceneFile(Directory directory, String file, Map
                 Logger logger, Version version, boolean readFileAsHash) throws IOException {
             final String checksum;
             final BytesRefBuilder fileHash = new BytesRefBuilder();
-            try (final IndexInput in = directory.openInput(file, IOContext.READONCE)) {
+            try (IndexInput in = directory.openInput(file, IOContext.READONCE)) {
                 final long length;
                 try {
                     length = in.length();
@@ -1179,11 +1206,11 @@ static class VerifyingIndexInput extends ChecksumIndexInput {
         private final byte[] checksum = new byte[8];
         private long verifiedPosition = 0;
 
-        public VerifyingIndexInput(IndexInput input) {
+        VerifyingIndexInput(IndexInput input) {
             this(input, new BufferedChecksum(new CRC32()));
         }
 
-        public VerifyingIndexInput(IndexInput input, Checksum digest) {
+        VerifyingIndexInput(IndexInput input, Checksum digest) {
             super("VerifyingIndexInput(" + input + ")");
             this.input = input;
             this.digest = digest;
@@ -1336,14 +1363,14 @@ public void markStoreCorrupted(IOException exception) throws IOException {
     /**
      * A listener that is executed once the store is closed and all references to it are released
      */
-    public interface OnClose extends Callback<ShardLock> {
+    public interface OnClose extends Consumer<ShardLock> {
         OnClose EMPTY = new OnClose() {
             /**
              * This method is called while the provided {@link org.elasticsearch.env.ShardLock} is held.
              * This method is only called once after all resources for a store are released.
              */
             @Override
-            public void handle(ShardLock Lock) {
+            public void accept(ShardLock Lock) {
             }
         };
     }
@@ -1352,7 +1379,7 @@ private static class StoreStatsCache extends SingleObjectCache<StoreStats> {
         private final Directory directory;
         private final DirectoryService directoryService;
 
-        public StoreStatsCache(TimeValue refreshInterval, Directory directory, DirectoryService directoryService) throws IOException {
+        StoreStatsCache(TimeValue refreshInterval, Directory directory, DirectoryService directoryService) throws IOException {
             super(refreshInterval, new StoreStats(estimateSize(directory), directoryService.throttleTimeInNanos()));
             this.directory = directory;
             this.directoryService = directoryService;
@@ -1373,8 +1400,9 @@ private static long estimateSize(Directory directory) throws IOException {
             for (String file : files) {
                 try {
                     estimatedSize += directory.fileLength(file);
-                } catch (NoSuchFileException | FileNotFoundException e) {
-                    // ignore, the file is not there no more
+                } catch (NoSuchFileException | FileNotFoundException | AccessDeniedException e) {
+                    // ignore, the file is not there no more; on Windows, if one thread concurrently deletes a file while
+                    // calling Files.size, you can also sometimes hit AccessDeniedException
                 }
             }
             return estimatedSize;
diff --git a/core/src/main/java/org/elasticsearch/index/store/StoreStats.java b/core/src/main/java/org/elasticsearch/index/store/StoreStats.java
index d777d7b7830c2..d3406c44891fa 100644
--- a/core/src/main/java/org/elasticsearch/index/store/StoreStats.java
+++ b/core/src/main/java/org/elasticsearch/index/store/StoreStats.java
@@ -79,12 +79,6 @@ public TimeValue getThrottleTime() {
         return throttleTime();
     }
 
-    public static StoreStats readStoreStats(StreamInput in) throws IOException {
-        StoreStats store = new StoreStats();
-        store.readFrom(in);
-        return store;
-    }
-
     @Override
     public void readFrom(StreamInput in) throws IOException {
         sizeInBytes = in.readVLong();
diff --git a/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java b/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java
index 50583a148d757..b3fa7b29991b0 100644
--- a/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java
+++ b/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java
@@ -34,8 +34,9 @@
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.lucene.uid.Versions;
+import org.elasticsearch.common.lucene.uid.VersionsResolver.DocIdAndVersion;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.get.GetField;
@@ -49,8 +50,6 @@
 import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.index.mapper.StringFieldMapper;
 import org.elasticsearch.index.mapper.TextFieldMapper;
-import org.elasticsearch.index.mapper.Uid;
-import org.elasticsearch.index.mapper.UidFieldMapper;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.search.dfs.AggregatedDfs;
 
@@ -82,9 +81,13 @@ public static TermVectorsResponse getTermVectors(IndexShard indexShard, TermVect
     static TermVectorsResponse getTermVectors(IndexShard indexShard, TermVectorsRequest request, LongSupplier nanoTimeSupplier) {
         final long startTime = nanoTimeSupplier.getAsLong();
         final TermVectorsResponse termVectorsResponse = new TermVectorsResponse(indexShard.shardId().getIndex().getName(), request.type(), request.id());
-        final Term uidTerm = new Term(UidFieldMapper.NAME, Uid.createUidAsBytes(request.type(), request.id()));
-
-        Engine.GetResult get = indexShard.get(new Engine.Get(request.realtime(), uidTerm).version(request.version()).versionType(request.versionType()));
+        final Term uidTerm = indexShard.mapperService().createUidTerm(request.type(), request.id());
+        if (uidTerm == null) {
+            termVectorsResponse.setExists(false);
+            return termVectorsResponse;
+        }
+        Engine.GetResult get = indexShard.get(new Engine.Get(request.realtime(), request.type(), request.id(), uidTerm)
+                .version(request.version()).versionType(request.versionType()));
 
         Fields termVectorsByField = null;
         AggregatedDfs dfs = null;
@@ -98,7 +101,7 @@ static TermVectorsResponse getTermVectors(IndexShard indexShard, TermVectorsRequ
         final Engine.Searcher searcher = indexShard.acquireSearcher("term_vector");
         try {
             Fields topLevelFields = MultiFields.getFields(get.searcher() != null ? get.searcher().reader() : searcher.reader());
-            Versions.DocIdAndVersion docIdAndVersion = get.docIdAndVersion();
+            DocIdAndVersion docIdAndVersion = get.docIdAndVersion();
             /* from an artificial document */
             if (request.doc() != null) {
                 termVectorsByField = generateTermVectorsFromDoc(indexShard, request);
@@ -214,12 +217,12 @@ private static Analyzer getAnalyzerAtField(IndexShard indexShard, String field,
         MapperService mapperService = indexShard.mapperService();
         Analyzer analyzer;
         if (perFieldAnalyzer != null && perFieldAnalyzer.containsKey(field)) {
-            analyzer = mapperService.analysisService().analyzer(perFieldAnalyzer.get(field).toString());
+            analyzer = mapperService.getIndexAnalyzers().get(perFieldAnalyzer.get(field).toString());
         } else {
             analyzer = mapperService.fullName(field).indexAnalyzer();
         }
         if (analyzer == null) {
-            analyzer = mapperService.analysisService().defaultIndexAnalyzer();
+            analyzer = mapperService.getIndexAnalyzers().getDefaultIndexAnalyzer();
         }
         return analyzer;
     }
@@ -258,8 +261,12 @@ private static Fields generateTermVectors(IndexShard indexShard, Map<String, Obj
         for (Map.Entry<String, Collection<Object>> entry : values.entrySet()) {
             String field = entry.getKey();
             Analyzer analyzer = getAnalyzerAtField(indexShard, field, perFieldAnalyzer);
-            for (Object text : entry.getValue()) {
-                index.addField(field, text.toString(), analyzer);
+            if (entry.getValue() instanceof List) {
+                for (Object text : entry.getValue()) {
+                    index.addField(field, text.toString(), analyzer);
+                }
+            } else {
+                index.addField(field, entry.getValue().toString(), analyzer);
             }
         }
         /* and read vectors from it */
@@ -268,7 +275,8 @@ private static Fields generateTermVectors(IndexShard indexShard, Map<String, Obj
 
     private static Fields generateTermVectorsFromDoc(IndexShard indexShard, TermVectorsRequest request) throws IOException {
         // parse the document, at the moment we do update the mapping, just like percolate
-        ParsedDocument parsedDocument = parseDocument(indexShard, indexShard.shardId().getIndexName(), request.type(), request.doc());
+        ParsedDocument parsedDocument =
+            parseDocument(indexShard, indexShard.shardId().getIndexName(), request.type(), request.doc(), request.xContentType());
 
         // select the right fields and generate term vectors
         ParseContext.Document doc = parsedDocument.rootDoc();
@@ -291,13 +299,15 @@ private static Fields generateTermVectorsFromDoc(IndexShard indexShard, TermVect
             String[] values = doc.getValues(field.name());
             getFields.add(new GetField(field.name(), Arrays.asList((Object[]) values)));
         }
-        return generateTermVectors(indexShard, XContentHelper.convertToMap(parsedDocument.source(), true).v2(), getFields, request.offsets(), request.perFieldAnalyzer(), seenFields);
+        return generateTermVectors(indexShard, XContentHelper.convertToMap(parsedDocument.source(), true, request.xContentType()).v2(),
+            getFields, request.offsets(), request.perFieldAnalyzer(), seenFields);
     }
 
-    private static ParsedDocument parseDocument(IndexShard indexShard, String index, String type, BytesReference doc) {
+    private static ParsedDocument parseDocument(IndexShard indexShard, String index, String type, BytesReference doc,
+                                                XContentType xContentType) {
         MapperService mapperService = indexShard.mapperService();
         DocumentMapperForType docMapper = mapperService.documentMapperWithAutoCreate(type);
-        ParsedDocument parsedDocument = docMapper.getDocumentMapper().parse(source(index, type, "_id_for_tv_api", doc));
+        ParsedDocument parsedDocument = docMapper.getDocumentMapper().parse(source(index, type, "_id_for_tv_api", doc, xContentType));
         if (docMapper.getMapping() != null) {
             parsedDocument.addDynamicMappingsUpdate(docMapper.getMapping());
         }
diff --git a/core/src/main/java/org/elasticsearch/index/translog/BaseTranslogReader.java b/core/src/main/java/org/elasticsearch/index/translog/BaseTranslogReader.java
index 49e8249e9d370..bf61febb741eb 100644
--- a/core/src/main/java/org/elasticsearch/index/translog/BaseTranslogReader.java
+++ b/core/src/main/java/org/elasticsearch/index/translog/BaseTranslogReader.java
@@ -58,34 +58,21 @@ public final long getFirstOperationOffset() {
         return firstOperationOffset;
     }
 
-    public Translog.Operation read(Translog.Location location) throws IOException {
-        assert location.generation == generation : "read location's translog generation [" + location.generation + "] is not [" + generation + "]";
-        ByteBuffer buffer = ByteBuffer.allocate(location.size);
-        try (BufferedChecksumStreamInput checksumStreamInput = checksummedStream(buffer, location.translogLocation, location.size, null)) {
-            return read(checksumStreamInput);
-        }
-    }
-
     /** read the size of the op (i.e., number of bytes, including the op size) written at the given position */
-    protected final int readSize(ByteBuffer reusableBuffer, long position) {
+    protected final int readSize(ByteBuffer reusableBuffer, long position) throws IOException {
         // read op size from disk
         assert reusableBuffer.capacity() >= 4 : "reusable buffer must have capacity >=4 when reading opSize. got [" + reusableBuffer.capacity() + "]";
-        try {
-            reusableBuffer.clear();
-            reusableBuffer.limit(4);
-            readBytes(reusableBuffer, position);
-            reusableBuffer.flip();
-            // Add an extra 4 to account for the operation size integer itself
-            final int size = reusableBuffer.getInt() + 4;
-            final long maxSize = sizeInBytes() - position;
-            if (size < 0 || size > maxSize) {
-                throw new TranslogCorruptedException("operation size is corrupted must be [0.." + maxSize + "] but was: " + size);
-            }
-
-            return size;
-        } catch (IOException e) {
-            throw new ElasticsearchException("unexpected exception reading from translog snapshot of " + this.path, e);
+        reusableBuffer.clear();
+        reusableBuffer.limit(4);
+        readBytes(reusableBuffer, position);
+        reusableBuffer.flip();
+        // Add an extra 4 to account for the operation size integer itself
+        final int size = reusableBuffer.getInt() + 4;
+        final long maxSize = sizeInBytes() - position;
+        if (size < 0 || size > maxSize) {
+            throw new TranslogCorruptedException("operation size is corrupted must be [0.." + maxSize + "] but was: " + size);
         }
+        return size;
     }
 
     public Translog.Snapshot newSnapshot() {
diff --git a/core/src/main/java/org/elasticsearch/index/translog/BufferedChecksumStreamInput.java b/core/src/main/java/org/elasticsearch/index/translog/BufferedChecksumStreamInput.java
index 58aa60a23c857..ba6da4ba522bb 100644
--- a/core/src/main/java/org/elasticsearch/index/translog/BufferedChecksumStreamInput.java
+++ b/core/src/main/java/org/elasticsearch/index/translog/BufferedChecksumStreamInput.java
@@ -20,8 +20,10 @@
 package org.elasticsearch.index.translog;
 
 import org.apache.lucene.store.BufferedChecksum;
+import org.elasticsearch.common.io.stream.FilterStreamInput;
 import org.elasticsearch.common.io.stream.StreamInput;
 
+import java.io.EOFException;
 import java.io.IOException;
 import java.util.zip.CRC32;
 import java.util.zip.Checksum;
@@ -30,19 +32,18 @@
  * Similar to Lucene's BufferedChecksumIndexInput, however this wraps a
  * {@link StreamInput} so anything read will update the checksum
  */
-public final class BufferedChecksumStreamInput extends StreamInput {
+public final class BufferedChecksumStreamInput extends FilterStreamInput {
     private static final int SKIP_BUFFER_SIZE = 1024;
     private byte[] skipBuffer;
-    private final StreamInput in;
     private final Checksum digest;
 
     public BufferedChecksumStreamInput(StreamInput in) {
-        this.in = in;
+        super(in);
         this.digest = new BufferedChecksum(new CRC32());
     }
 
     public BufferedChecksumStreamInput(StreamInput in, BufferedChecksumStreamInput reuse) {
-        this.in = in;
+        super(in);
         if (reuse == null ) {
             this.digest = new BufferedChecksum(new CRC32());
         } else {
@@ -58,20 +59,20 @@ public long getChecksum() {
 
     @Override
     public byte readByte() throws IOException {
-        final byte b = in.readByte();
+        final byte b = delegate.readByte();
         digest.update(b);
         return b;
     }
 
     @Override
     public void readBytes(byte[] b, int offset, int len) throws IOException {
-        in.readBytes(b, offset, len);
+        delegate.readBytes(b, offset, len);
         digest.update(b, offset, len);
     }
 
     @Override
     public void reset() throws IOException {
-        in.reset();
+        delegate.reset();
         digest.reset();
     }
 
@@ -80,14 +81,9 @@ public int read() throws IOException {
         return readByte() & 0xFF;
     }
 
-    @Override
-    public void close() throws IOException {
-        in.close();
-    }
-
     @Override
     public boolean markSupported() {
-        return in.markSupported();
+        return delegate.markSupported();
     }
 
 
@@ -109,17 +105,14 @@ public long skip(long numBytes) throws IOException {
         return skipped;
     }
 
-    @Override
-    public int available() throws IOException {
-        return in.available();
-    }
 
     @Override
     public synchronized void mark(int readlimit) {
-        in.mark(readlimit);
+        delegate.mark(readlimit);
     }
 
     public void resetDigest() {
         digest.reset();
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/index/translog/Checkpoint.java b/core/src/main/java/org/elasticsearch/index/translog/Checkpoint.java
index d4eb980782718..1e2b25b1d1a84 100644
--- a/core/src/main/java/org/elasticsearch/index/translog/Checkpoint.java
+++ b/core/src/main/java/org/elasticsearch/index/translog/Checkpoint.java
@@ -89,7 +89,7 @@ public String toString() {
 
     public static Checkpoint read(Path path) throws IOException {
         try (Directory dir = new SimpleFSDirectory(path.getParent())) {
-            try (final IndexInput indexInput = dir.openInput(path.getFileName().toString(), IOContext.DEFAULT)) {
+            try (IndexInput indexInput = dir.openInput(path.getFileName().toString(), IOContext.DEFAULT)) {
                 if (indexInput.length() == LEGACY_NON_CHECKSUMMED_FILE_LENGTH) {
                     // OLD unchecksummed file that was written < ES 5.0.0
                     return Checkpoint.readNonChecksummed(indexInput);
@@ -111,7 +111,7 @@ public synchronized byte[] toByteArray() {
             }
         };
         final String resourceDesc = "checkpoint(path=\"" + checkpointFile + "\", gen=" + checkpoint + ")";
-        try (final OutputStreamIndexOutput indexOutput =
+        try (OutputStreamIndexOutput indexOutput =
                  new OutputStreamIndexOutput(resourceDesc, checkpointFile.toString(), byteOutputStream, FILE_SIZE)) {
             CodecUtil.writeHeader(indexOutput, CHECKPOINT_CODEC, INITIAL_VERSION);
             checkpoint.write(indexOutput);
diff --git a/core/src/main/java/org/elasticsearch/index/translog/Translog.java b/core/src/main/java/org/elasticsearch/index/translog/Translog.java
index 0082b7a033685..ef0c0ecb654b7 100644
--- a/core/src/main/java/org/elasticsearch/index/translog/Translog.java
+++ b/core/src/main/java/org/elasticsearch/index/translog/Translog.java
@@ -25,7 +25,6 @@
 import org.apache.lucene.index.TwoPhaseCommit;
 import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.util.IOUtils;
-import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.bytes.BytesArray;
@@ -39,17 +38,16 @@
 import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
-import org.elasticsearch.common.util.concurrent.FutureUtils;
 import org.elasticsearch.common.util.concurrent.ReleasableLock;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.index.shard.AbstractIndexShardComponent;
 import org.elasticsearch.index.shard.IndexShardComponent;
 
 import java.io.Closeable;
 import java.io.EOFException;
 import java.io.IOException;
-import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -57,9 +55,9 @@
 import java.nio.file.StandardOpenOption;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
-import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -112,7 +110,6 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
 
     // the list of translog readers is guaranteed to be in order of translog generation
     private final List<TranslogReader> readers = new ArrayList<>();
-    private volatile ScheduledFuture<?> syncScheduler;
     // this is a concurrent set and is not protected by any of the locks. The main reason
     // is that is being accessed by two separate classes (additions & reading are done by Translog, remove by View when closed)
     private final Set<View> outstandingViews = ConcurrentCollections.newConcurrentSet();
@@ -312,7 +309,6 @@ public void close() throws IOException {
                     closeFilesIfNoPendingViews();
                 }
             } finally {
-                FutureUtils.cancel(syncScheduler);
                 logger.debug("translog closed");
             }
         }
@@ -387,31 +383,6 @@ TranslogWriter createWriter(long fileGeneration) throws IOException {
         return newFile;
     }
 
-
-    /**
-     * Read the Operation object from the given location. This method will try to read the given location from
-     * the current or from the currently committing translog file. If the location is in a file that has already
-     * been closed or even removed the method will return <code>null</code> instead.
-     */
-    Translog.Operation read(Location location) { // TODO this is only here for testing - we can remove it?
-        try (ReleasableLock lock = readLock.acquire()) {
-            final BaseTranslogReader reader;
-            final long currentGeneration = current.getGeneration();
-            if (currentGeneration == location.generation) {
-                reader = current;
-            } else if (readers.isEmpty() == false && readers.get(readers.size() - 1).getGeneration() == location.generation) {
-                reader = readers.get(readers.size() - 1);
-            } else if (currentGeneration < location.generation) {
-                throw new IllegalStateException("location generation [" + location.generation + "] is greater than the current generation [" + currentGeneration + "]");
-            } else {
-                return null;
-            }
-            return reader.read(location);
-        } catch (IOException e) {
-            throw new ElasticsearchException("failed to read source from translog location " + location, e);
-        }
-    }
-
     /**
      * Adds a delete / index operations to the transaction log.
      *
@@ -435,7 +406,6 @@ public Location add(Operation operation) throws IOException {
             try (ReleasableLock lock = readLock.acquire()) {
                 ensureOpen();
                 Location location = current.add(bytes);
-                assert assertBytesAtLocation(location, bytes);
                 return location;
             }
         } catch (AlreadyClosedException | IOException ex) {
@@ -453,7 +423,7 @@ public Location add(Operation operation) throws IOException {
             }
             throw new TranslogException(shardId, "Failed to write operation [" + operation + "]", e);
         } finally {
-            Releasables.close(out.bytes());
+            Releasables.close(out);
         }
     }
 
@@ -472,13 +442,6 @@ public Location getLastWriteLocation() {
         }
     }
 
-    boolean assertBytesAtLocation(Translog.Location location, BytesReference expectedBytes) throws IOException {
-        // tests can override this
-        ByteBuffer buffer = ByteBuffer.allocate(location.size);
-        current.readBytes(buffer, location.translogLocation);
-        return new BytesArray(buffer.array()).equals(expectedBytes);
-    }
-
     /**
      * Snapshots the current transaction log allowing to safely iterate over the snapshot.
      * Snapshots are fixed in time and will not be updated with future operations.
@@ -865,13 +828,13 @@ public Index(StreamInput in) throws IOException {
             }
         }
 
-        public Index(Engine.Index index) {
+        public Index(Engine.Index index, Engine.IndexResult indexResult) {
             this.id = index.id();
             this.type = index.type();
             this.source = index.source();
             this.routing = index.routing();
             this.parent = index.parent();
-            this.version = index.version();
+            this.version = indexResult.getVersion();
             this.timestamp = index.timestamp();
             this.ttl = index.ttl();
             this.versionType = index.versionType();
@@ -1014,32 +977,45 @@ public long getAutoGeneratedIdTimestamp() {
     }
 
     public static class Delete implements Operation {
-        public static final int SERIALIZATION_FORMAT = 2; // since 2.0-beta1 and 1.1
+        public static final int FORMAT_5_0 = 2; // 5.0 - 5.5
+        private static final int FORMAT_SINGLE_TYPE = FORMAT_5_0 + 1; // 5.5 - 6.0
 
+        private final String type, id;
         private final Term uid;
         private final long version;
         private final VersionType versionType;
 
         public Delete(StreamInput in) throws IOException {
             final int format = in.readVInt();// SERIALIZATION_FORMAT
-            assert format == SERIALIZATION_FORMAT : "format was: " + format;
-            uid = new Term(in.readString(), in.readString());
+            assert format >= FORMAT_5_0 && format <= FORMAT_SINGLE_TYPE : "format was: " + format;
+            if (format >= FORMAT_SINGLE_TYPE) {
+                type = in.readString();
+                id = in.readString();
+                uid = new Term(in.readString(), in.readString());
+            } else {
+                uid = new Term(in.readString(), in.readString());
+                // the uid was constructed from the type and id so we can
+                // extract them back
+                Uid uidObject = Uid.createUid(uid.text());
+                type = uidObject.type();
+                id = uidObject.id();
+            }
             this.version = in.readLong();
             this.versionType = VersionType.fromValue(in.readByte());
             assert versionType.validateVersionForWrites(this.version);
         }
 
-        public Delete(Engine.Delete delete) {
-            this.uid = delete.uid();
-            this.version = delete.version();
-            this.versionType = delete.versionType();
+        public Delete(Engine.Delete delete, Engine.DeleteResult deleteResult) {
+            this(delete.type(), delete.id(), delete.uid(), deleteResult.getVersion(), delete.versionType());
         }
 
-        public Delete(Term uid) {
-            this(uid, Versions.MATCH_ANY, VersionType.INTERNAL);
+        public Delete(String type, String id, Term uid) {
+            this(type, id, uid, Versions.MATCH_ANY, VersionType.INTERNAL);
         }
 
-        public Delete(Term uid, long version, VersionType versionType) {
+        public Delete(String type, String id, Term uid, long version, VersionType versionType) {
+            this.type = Objects.requireNonNull(type);
+            this.id = Objects.requireNonNull(id);
             this.uid = uid;
             this.version = version;
             this.versionType = versionType;
@@ -1055,6 +1031,14 @@ public long estimateSize() {
             return ((uid.field().length() + uid.text().length()) * 2) + 20;
         }
 
+        public String type() {
+            return type;
+        }
+
+        public String id() {
+            return id;
+        }
+
         public Term uid() {
             return this.uid;
         }
@@ -1074,7 +1058,9 @@ public Source getSource() {
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
-            out.writeVInt(SERIALIZATION_FORMAT);
+            out.writeVInt(FORMAT_SINGLE_TYPE);
+            out.writeString(type);
+            out.writeString(id);
             out.writeString(uid.field());
             out.writeString(uid.text());
             out.writeLong(version);
@@ -1200,7 +1186,7 @@ public static void writeOperations(StreamOutput outStream, List<Operation> toWri
                 bytes.writeTo(outStream);
             }
         } finally {
-            Releasables.close(out.bytes());
+            Releasables.close(out);
         }
 
     }
diff --git a/core/src/main/java/org/elasticsearch/index/translog/TranslogSnapshot.java b/core/src/main/java/org/elasticsearch/index/translog/TranslogSnapshot.java
index f33ec1bd60702..ffbe1002eb146 100644
--- a/core/src/main/java/org/elasticsearch/index/translog/TranslogSnapshot.java
+++ b/core/src/main/java/org/elasticsearch/index/translog/TranslogSnapshot.java
@@ -26,7 +26,7 @@
 import java.nio.channels.FileChannel;
 import java.nio.file.Path;
 
-public class TranslogSnapshot extends BaseTranslogReader implements Translog.Snapshot {
+final class TranslogSnapshot extends BaseTranslogReader implements Translog.Snapshot {
 
     private final int totalOperations;
     protected final long length;
@@ -40,7 +40,7 @@ public class TranslogSnapshot extends BaseTranslogReader implements Translog.Sna
      * Create a snapshot of translog file channel. The length parameter should be consistent with totalOperations and point
      * at the end of the last operation in this snapshot.
      */
-    public TranslogSnapshot(long generation, FileChannel channel, Path path, long firstOperationOffset, long length, int totalOperations) {
+    TranslogSnapshot(long generation, FileChannel channel, Path path, long firstOperationOffset, long length, int totalOperations) {
         super(generation, channel, path, firstOperationOffset);
         this.length = length;
         this.totalOperations = totalOperations;
@@ -51,7 +51,7 @@ public TranslogSnapshot(long generation, FileChannel channel, Path path, long fi
     }
 
     @Override
-    public final int totalOperations() {
+    public int totalOperations() {
         return totalOperations;
     }
 
@@ -64,7 +64,7 @@ public Translog.Operation next() throws IOException {
         }
     }
 
-    protected final Translog.Operation readOperation() throws IOException {
+    protected Translog.Operation readOperation() throws IOException {
         final int opSize = readSize(reusableBuffer, position);
         reuse = checksummedStream(reusableBuffer, position, opSize, reuse);
         Translog.Operation op = read(reuse);
diff --git a/core/src/main/java/org/elasticsearch/index/translog/TranslogToolCli.java b/core/src/main/java/org/elasticsearch/index/translog/TranslogToolCli.java
index d565074a50c13..944296d6813e7 100644
--- a/core/src/main/java/org/elasticsearch/index/translog/TranslogToolCli.java
+++ b/core/src/main/java/org/elasticsearch/index/translog/TranslogToolCli.java
@@ -21,10 +21,6 @@
 
 import org.elasticsearch.cli.MultiCommand;
 import org.elasticsearch.cli.Terminal;
-import org.elasticsearch.common.logging.LogConfigurator;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.node.internal.InternalSettingsPreparer;
 
 /**
  * Class encapsulating and dispatching commands from the {@code elasticsearch-translog} command line tool
@@ -37,17 +33,7 @@ private TranslogToolCli() {
     }
 
     public static void main(String[] args) throws Exception {
-        // initialize default for es.logger.level because we will not read the log4j2.properties
-        String loggerLevel = System.getProperty("es.logger.level", "INFO");
-        String pathHome = System.getProperty("es.path.home");
-        // Set the appender for all potential log files to terminal so that other components that use the logger print out the
-        // same terminal.
-        Environment loggingEnvironment = InternalSettingsPreparer.prepareEnvironment(Settings.builder()
-                .put("path.home", pathHome)
-                .put("logger.level", loggerLevel)
-                .build(), Terminal.DEFAULT);
-        LogConfigurator.configure(loggingEnvironment, false);
-
         exit(new TranslogToolCli().main(args, Terminal.DEFAULT));
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java b/core/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java
index bb4a84651c561..241729c4acb35 100644
--- a/core/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java
+++ b/core/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java
@@ -332,7 +332,7 @@ protected final boolean isClosed() {
 
     private final class BufferedChannelOutputStream extends BufferedOutputStream {
 
-        public BufferedChannelOutputStream(OutputStream out, int size) throws IOException {
+        BufferedChannelOutputStream(OutputStream out, int size) throws IOException {
             super(out, size);
         }
 
diff --git a/core/src/main/java/org/elasticsearch/index/translog/TruncateTranslogCommand.java b/core/src/main/java/org/elasticsearch/index/translog/TruncateTranslogCommand.java
index 6514cd42709d2..ef339b31972e8 100644
--- a/core/src/main/java/org/elasticsearch/index/translog/TruncateTranslogCommand.java
+++ b/core/src/main/java/org/elasticsearch/index/translog/TruncateTranslogCommand.java
@@ -34,10 +34,11 @@
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.cli.SettingCommand;
+import org.elasticsearch.cli.EnvironmentAwareCommand;
 import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.io.PathUtils;
+import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexNotFoundException;
 
 import java.io.IOException;
@@ -54,7 +55,7 @@
 import java.util.Map;
 import java.util.Set;
 
-public class TruncateTranslogCommand extends SettingCommand {
+public class TruncateTranslogCommand extends EnvironmentAwareCommand {
 
     private final OptionSpec<String> translogFolder;
     private final OptionSpec<Void> batchMode;
@@ -86,7 +87,7 @@ private Path getTranslogPath(OptionSet options) {
     }
 
     @Override
-    protected void execute(Terminal terminal, OptionSet options, Map<String, String> settings) throws Exception {
+    protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
         boolean batch = options.has(batchMode);
 
         Path translogPath = getTranslogPath(options);
diff --git a/core/src/main/java/org/elasticsearch/index/warmer/WarmerStats.java b/core/src/main/java/org/elasticsearch/index/warmer/WarmerStats.java
index 233dbf4f5fe8a..21dec0f62a043 100644
--- a/core/src/main/java/org/elasticsearch/index/warmer/WarmerStats.java
+++ b/core/src/main/java/org/elasticsearch/index/warmer/WarmerStats.java
@@ -86,12 +86,6 @@ public TimeValue totalTime() {
         return new TimeValue(totalTimeInMillis);
     }
 
-    public static WarmerStats readWarmerStats(StreamInput in) throws IOException {
-        WarmerStats refreshStats = new WarmerStats();
-        refreshStats.readFrom(in);
-        return refreshStats;
-    }
-
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject(Fields.WARMER);
diff --git a/core/src/main/java/org/elasticsearch/indices/AbstractIndexShardCacheEntity.java b/core/src/main/java/org/elasticsearch/indices/AbstractIndexShardCacheEntity.java
index c0d929d82f5eb..98afd8781b4f8 100644
--- a/core/src/main/java/org/elasticsearch/indices/AbstractIndexShardCacheEntity.java
+++ b/core/src/main/java/org/elasticsearch/indices/AbstractIndexShardCacheEntity.java
@@ -19,40 +19,15 @@
 
 package org.elasticsearch.indices;
 
-import org.apache.lucene.index.DirectoryReader;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.cache.RemovalNotification;
-import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.index.cache.request.ShardRequestCache;
 import org.elasticsearch.index.shard.IndexShard;
 
-import java.io.IOException;
-
 /**
  * Abstract base class for the an {@link IndexShard} level {@linkplain IndicesRequestCache.CacheEntity}.
  */
 abstract class AbstractIndexShardCacheEntity implements IndicesRequestCache.CacheEntity {
-    @FunctionalInterface
-    public interface Loader {
-        void load(StreamOutput out) throws IOException;
-    }
-
-    private final Loader loader;
-    private boolean loadedFromCache = true;
-
-    protected AbstractIndexShardCacheEntity(Loader loader) {
-        this.loader = loader;
-    }
-
-    /**
-     * When called after passing this through
-     * {@link IndicesRequestCache#getOrCompute(IndicesRequestCache.CacheEntity, DirectoryReader, BytesReference)} this will return whether
-     * or not the result was loaded from the cache.
-     */
-    public final boolean loadedFromCache() {
-        return loadedFromCache;
-    }
 
     /**
      * Get the {@linkplain ShardRequestCache} used to track cache statistics.
@@ -60,27 +35,7 @@ public final boolean loadedFromCache() {
     protected abstract ShardRequestCache stats();
 
     @Override
-    public final IndicesRequestCache.Value loadValue() throws IOException {
-        /* BytesStreamOutput allows to pass the expected size but by default uses
-         * BigArrays.PAGE_SIZE_IN_BYTES which is 16k. A common cached result ie.
-         * a date histogram with 3 buckets is ~100byte so 16k might be very wasteful
-         * since we don't shrink to the actual size once we are done serializing.
-         * By passing 512 as the expected size we will resize the byte array in the stream
-         * slowly until we hit the page size and don't waste too much memory for small query
-         * results.*/
-        final int expectedSizeInBytes = 512;
-        try (BytesStreamOutput out = new BytesStreamOutput(expectedSizeInBytes)) {
-            loader.load(out);
-            // for now, keep the paged data structure, which might have unused bytes to fill a page, but better to keep
-            // the memory properly paged instead of having varied sized bytes
-            final BytesReference reference = out.bytes();
-            loadedFromCache = false;
-            return new IndicesRequestCache.Value(reference, out.ramBytesUsed());
-        }
-    }
-
-    @Override
-    public final void onCached(IndicesRequestCache.Key key, IndicesRequestCache.Value value) {
+    public final void onCached(IndicesRequestCache.Key key, BytesReference value) {
         stats().onCached(key, value);
     }
 
@@ -95,7 +50,7 @@ public final void onMiss() {
     }
 
     @Override
-    public final void onRemoval(RemovalNotification<IndicesRequestCache.Key, IndicesRequestCache.Value> notification) {
+    public final void onRemoval(RemovalNotification<IndicesRequestCache.Key, BytesReference> notification) {
         stats().onRemoval(notification.getKey(), notification.getValue(),
                 notification.getRemovalReason() == RemovalNotification.RemovalReason.EVICTED);
     }
diff --git a/core/src/main/java/org/elasticsearch/indices/IndexAlreadyExistsException.java b/core/src/main/java/org/elasticsearch/indices/IndexAlreadyExistsException.java
deleted file mode 100644
index f64addfd2f1c1..0000000000000
--- a/core/src/main/java/org/elasticsearch/indices/IndexAlreadyExistsException.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.indices;
-
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.rest.RestStatus;
-
-import java.io.IOException;
-
-/**
- *
- */
-public class IndexAlreadyExistsException extends ElasticsearchException {
-
-    public IndexAlreadyExistsException(Index index) {
-        this(index, "index " + index.toString() + " already exists");
-    }
-
-    public IndexAlreadyExistsException(Index index, String message) {
-        super(message);
-        setIndex(index);
-    }
-
-    public IndexAlreadyExistsException(StreamInput in) throws IOException{
-        super(in);
-    }
-
-    @Override
-    public RestStatus status() {
-        return RestStatus.BAD_REQUEST;
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/indices/IndexTemplateAlreadyExistsException.java b/core/src/main/java/org/elasticsearch/indices/IndexTemplateAlreadyExistsException.java
index 3b665e051e049..947e3eab90b74 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndexTemplateAlreadyExistsException.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndexTemplateAlreadyExistsException.java
@@ -26,8 +26,9 @@
 import java.io.IOException;
 
 /**
- *
+ * @deprecated use {@link IllegalArgumentException} instead
  */
+@Deprecated
 public class IndexTemplateAlreadyExistsException extends ElasticsearchException {
 
     private final String name;
diff --git a/core/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java b/core/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java
index 75c3f06062fbb..1b960bb15992b 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java
@@ -21,6 +21,7 @@
 
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
+import org.apache.lucene.store.AlreadyClosedException;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
@@ -30,11 +31,10 @@
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.index.engine.Engine;
-import org.elasticsearch.index.engine.EngineClosedException;
-import org.elasticsearch.index.engine.FlushNotAllowedEngineException;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.IndexShardState;
 import org.elasticsearch.index.shard.IndexingOperationListener;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.threadpool.ThreadPool.Cancellable;
 import org.elasticsearch.threadpool.ThreadPool.Names;
@@ -52,7 +52,7 @@
 public class IndexingMemoryController extends AbstractComponent implements IndexingOperationListener, Closeable {
 
     /** How much heap (% or bytes) we will share across all actively indexing shards on this node (default: 10%). */
-    public static final Setting<ByteSizeValue> INDEX_BUFFER_SIZE_SETTING = 
+    public static final Setting<ByteSizeValue> INDEX_BUFFER_SIZE_SETTING =
             Setting.memorySizeSetting("indices.memory.index_buffer_size", "10%", Property.NodeScope);
 
     /** Only applies when <code>indices.memory.index_buffer_size</code> is a %, to set a floor on the actual size in bytes (default: 48 MB). */
@@ -106,10 +106,10 @@ public class IndexingMemoryController extends AbstractComponent implements Index
             // We only apply the min/max when % value was used for the index buffer:
             ByteSizeValue minIndexingBuffer = MIN_INDEX_BUFFER_SIZE_SETTING.get(this.settings);
             ByteSizeValue maxIndexingBuffer = MAX_INDEX_BUFFER_SIZE_SETTING.get(this.settings);
-            if (indexingBuffer.bytes() < minIndexingBuffer.bytes()) {
+            if (indexingBuffer.getBytes() < minIndexingBuffer.getBytes()) {
                 indexingBuffer = minIndexingBuffer;
             }
-            if (maxIndexingBuffer.bytes() != -1 && indexingBuffer.bytes() > maxIndexingBuffer.bytes()) {
+            if (maxIndexingBuffer.getBytes() != -1 && indexingBuffer.getBytes() > maxIndexingBuffer.getBytes()) {
                 indexingBuffer = maxIndexingBuffer;
             }
         }
@@ -190,11 +190,6 @@ void forceCheck() {
         statusChecker.run();
     }
 
-    /** called by IndexShard to record that this many bytes were written to translog */
-    public void bytesWritten(int bytes) {
-        statusChecker.bytesWritten(bytes);
-    }
-
     /** Asks this shard to throttle indexing to one thread */
     protected void activateThrottling(IndexShard shard) {
         shard.activateThrottling();
@@ -206,24 +201,27 @@ protected void deactivateThrottling(IndexShard shard) {
     }
 
     @Override
-    public void postIndex(Engine.Index index, boolean created) {
-        recordOperationBytes(index);
+    public void postIndex(ShardId shardId, Engine.Index index, Engine.IndexResult result) {
+        recordOperationBytes(index, result);
     }
 
     @Override
-    public void postDelete(Engine.Delete delete) {
-        recordOperationBytes(delete);
+    public void postDelete(ShardId shardId, Engine.Delete delete, Engine.DeleteResult result) {
+        recordOperationBytes(delete, result);
     }
 
-    private void recordOperationBytes(Engine.Operation op) {
-        bytesWritten(op.sizeInBytes());
+    /** called by IndexShard to record estimated bytes written to translog for the operation */
+    private void recordOperationBytes(Engine.Operation operation, Engine.Result result) {
+        if (result.hasFailure() == false) {
+            statusChecker.bytesWritten(operation.estimatedSizeInBytes());
+        }
     }
 
     private static final class ShardAndBytesUsed implements Comparable<ShardAndBytesUsed> {
         final long bytesUsed;
         final IndexShard shard;
 
-        public ShardAndBytesUsed(long bytesUsed, IndexShard shard) {
+        ShardAndBytesUsed(long bytesUsed, IndexShard shard) {
             this.bytesUsed = bytesUsed;
             this.shard = shard;
         }
@@ -245,13 +243,13 @@ final class ShardsIndicesStatusChecker implements Runnable {
         public void bytesWritten(int bytes) {
             long totalBytes = bytesWrittenSinceCheck.addAndGet(bytes);
             assert totalBytes >= 0;
-            while (totalBytes > indexingBuffer.bytes()/30) {
+            while (totalBytes > indexingBuffer.getBytes()/30) {
 
                 if (runLock.tryLock()) {
                     try {
                         // Must pull this again because it may have changed since we first checked:
                         totalBytes = bytesWrittenSinceCheck.get();
-                        if (totalBytes > indexingBuffer.bytes()/30) {
+                        if (totalBytes > indexingBuffer.getBytes()/30) {
                             bytesWrittenSinceCheck.addAndGet(-totalBytes);
                             // NOTE: this is only an approximate check, because bytes written is to the translog, vs indexing memory buffer which is
                             // typically smaller but can be larger in extreme cases (many unique terms).  This logic is here only as a safety against
@@ -320,9 +318,9 @@ private void runUnlocked() {
 
             // If we are using more than 50% of our budget across both indexing buffer and bytes we are still moving to disk, then we now
             // throttle the top shards to send back-pressure to ongoing indexing:
-            boolean doThrottle = (totalBytesWriting + totalBytesUsed) > 1.5 * indexingBuffer.bytes();
+            boolean doThrottle = (totalBytesWriting + totalBytesUsed) > 1.5 * indexingBuffer.getBytes();
 
-            if (totalBytesUsed > indexingBuffer.bytes()) {
+            if (totalBytesUsed > indexingBuffer.getBytes()) {
                 // OK we are now over-budget; fill the priority queue and ask largest shard(s) to refresh:
                 PriorityQueue<ShardAndBytesUsed> queue = new PriorityQueue<>();
 
@@ -357,7 +355,7 @@ private void runUnlocked() {
                 logger.debug("now write some indexing buffers: total indexing heap bytes used [{}] vs {} [{}], currently writing bytes [{}], [{}] shards with non-zero indexing buffer",
                              new ByteSizeValue(totalBytesUsed), INDEX_BUFFER_SIZE_SETTING.getKey(), indexingBuffer, new ByteSizeValue(totalBytesWriting), queue.size());
 
-                while (totalBytesUsed > indexingBuffer.bytes() && queue.isEmpty() == false) {
+                while (totalBytesUsed > indexingBuffer.getBytes() && queue.isEmpty() == false) {
                     ShardAndBytesUsed largest = queue.poll();
                     logger.debug("write indexing buffer to disk for shard [{}] to free up its [{}] indexing buffer", largest.shard.shardId(), new ByteSizeValue(largest.bytesUsed));
                     writeIndexingBufferAsync(largest.shard);
@@ -386,8 +384,8 @@ private void runUnlocked() {
     protected void checkIdle(IndexShard shard, long inactiveTimeNS) {
         try {
             shard.checkIdle(inactiveTimeNS);
-        } catch (EngineClosedException | FlushNotAllowedEngineException e) {
-            logger.trace("ignore exception while checking if shard {} is inactive", e, shard.shardId());
+        } catch (AlreadyClosedException e) {
+            logger.trace((Supplier<?>) () -> new ParameterizedMessage("ignore exception while checking if shard {} is inactive", shard.shardId()), e);
         }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesModule.java b/core/src/main/java/org/elasticsearch/indices/IndicesModule.java
index fed710244fbb7..82a511ea3d1be 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndicesModule.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndicesModule.java
@@ -22,12 +22,9 @@
 import org.elasticsearch.action.admin.indices.rollover.Condition;
 import org.elasticsearch.action.admin.indices.rollover.MaxAgeCondition;
 import org.elasticsearch.action.admin.indices.rollover.MaxDocsCondition;
-import org.elasticsearch.action.update.UpdateHelper;
-import org.elasticsearch.cluster.metadata.MetaDataIndexUpgradeService;
 import org.elasticsearch.common.geo.ShapesAvailability;
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry;
-import org.elasticsearch.index.NodeServicesProvider;
 import org.elasticsearch.index.mapper.AllFieldMapper;
 import org.elasticsearch.index.mapper.BinaryFieldMapper;
 import org.elasticsearch.index.mapper.BooleanFieldMapper;
@@ -40,11 +37,13 @@
 import org.elasticsearch.index.mapper.IndexFieldMapper;
 import org.elasticsearch.index.mapper.IpFieldMapper;
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
+import org.elasticsearch.index.mapper.LatLonPointFieldMapper;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MetadataFieldMapper;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
 import org.elasticsearch.index.mapper.ObjectMapper;
 import org.elasticsearch.index.mapper.ParentFieldMapper;
+import org.elasticsearch.index.mapper.RangeFieldMapper;
 import org.elasticsearch.index.mapper.RoutingFieldMapper;
 import org.elasticsearch.index.mapper.ScaledFloatFieldMapper;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
@@ -59,9 +58,6 @@
 import org.elasticsearch.indices.cluster.IndicesClusterStateService;
 import org.elasticsearch.indices.flush.SyncedFlushService;
 import org.elasticsearch.indices.mapper.MapperRegistry;
-import org.elasticsearch.indices.recovery.RecoverySettings;
-import org.elasticsearch.indices.recovery.PeerRecoverySourceService;
-import org.elasticsearch.indices.recovery.PeerRecoveryTargetService;
 import org.elasticsearch.indices.store.IndicesStore;
 import org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData;
 import org.elasticsearch.indices.ttl.IndicesTTLService;
@@ -77,16 +73,11 @@
  * Configures classes and services that are shared by indices on each node.
  */
 public class IndicesModule extends AbstractModule {
-
-    private final Map<String, Mapper.TypeParser> mapperParsers;
-    private final Map<String, MetadataFieldMapper.TypeParser> metadataMapperParsers;
-    private final MapperRegistry mapperRegistry;
     private final List<Entry> namedWritables = new ArrayList<>();
+    private final MapperRegistry mapperRegistry;
 
     public IndicesModule(List<MapperPlugin> mapperPlugins) {
-        this.mapperParsers = getMappers(mapperPlugins);
-        this.metadataMapperParsers = getMetadataMappers(mapperPlugins);
-        this.mapperRegistry = new MapperRegistry(mapperParsers, metadataMapperParsers);
+        this.mapperRegistry = new MapperRegistry(getMappers(mapperPlugins), getMetadataMappers(mapperPlugins));
         registerBuiltinWritables();
     }
 
@@ -106,6 +97,9 @@ private Map<String, Mapper.TypeParser> getMappers(List<MapperPlugin> mapperPlugi
         for (NumberFieldMapper.NumberType type : NumberFieldMapper.NumberType.values()) {
             mappers.put(type.typeName(), new NumberFieldMapper.TypeParser(type));
         }
+        for (RangeFieldMapper.RangeType type : RangeFieldMapper.RangeType.values()) {
+            mappers.put(type.typeName(), new RangeFieldMapper.TypeParser(type));
+        }
         mappers.put(BooleanFieldMapper.CONTENT_TYPE, new BooleanFieldMapper.TypeParser());
         mappers.put(BinaryFieldMapper.CONTENT_TYPE, new BinaryFieldMapper.TypeParser());
         mappers.put(DateFieldMapper.CONTENT_TYPE, new DateFieldMapper.TypeParser());
@@ -119,6 +113,7 @@ private Map<String, Mapper.TypeParser> getMappers(List<MapperPlugin> mapperPlugi
         mappers.put(ObjectMapper.NESTED_CONTENT_TYPE, new ObjectMapper.TypeParser());
         mappers.put(CompletionFieldMapper.CONTENT_TYPE, new CompletionFieldMapper.TypeParser());
         mappers.put(GeoPointFieldMapper.CONTENT_TYPE, new GeoPointFieldMapper.TypeParser());
+        mappers.put(LatLonPointFieldMapper.CONTENT_TYPE, new LatLonPointFieldMapper.TypeParser());
         if (ShapesAvailability.JTS_AVAILABLE && ShapesAvailability.SPATIAL4J_AVAILABLE) {
             mappers.put(GeoShapeFieldMapper.CONTENT_TYPE, new GeoShapeFieldMapper.TypeParser());
         }
@@ -170,27 +165,17 @@ private Map<String, MetadataFieldMapper.TypeParser> getMetadataMappers(List<Mapp
 
     @Override
     protected void configure() {
-        bindMapperExtension();
-
-        bind(RecoverySettings.class).asEagerSingleton();
-        bind(PeerRecoveryTargetService.class).asEagerSingleton();
-        bind(PeerRecoverySourceService.class).asEagerSingleton();
         bind(IndicesStore.class).asEagerSingleton();
         bind(IndicesClusterStateService.class).asEagerSingleton();
         bind(SyncedFlushService.class).asEagerSingleton();
         bind(TransportNodesListShardStoreMetaData.class).asEagerSingleton();
         bind(IndicesTTLService.class).asEagerSingleton();
-        bind(UpdateHelper.class).asEagerSingleton();
-        bind(MetaDataIndexUpgradeService.class).asEagerSingleton();
-        bind(NodeServicesProvider.class).asEagerSingleton();
     }
 
-    // public for testing
+    /**
+     * A registry for all field mappers.
+     */
     public MapperRegistry getMapperRegistry() {
         return mapperRegistry;
     }
-
-    protected void bindMapperExtension() {
-        bind(MapperRegistry.class).toInstance(getMapperRegistry());
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java b/core/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java
index 08dcd48a56f7d..db52dca993626 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java
@@ -23,12 +23,13 @@
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.BulkScorer;
 import org.apache.lucene.search.Explanation;
-import org.apache.lucene.search.LRUQueryCache;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.QueryCache;
 import org.apache.lucene.search.QueryCachingPolicy;
 import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.ScorerSupplier;
 import org.apache.lucene.search.Weight;
+import org.apache.lucene.search.LRUQueryCache;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.lucene.ShardCoreKeyMap;
 import org.elasticsearch.common.settings.Setting;
@@ -49,12 +50,12 @@
 
 public class IndicesQueryCache extends AbstractComponent implements QueryCache, Closeable {
 
-    public static final Setting<ByteSizeValue> INDICES_CACHE_QUERY_SIZE_SETTING = 
+    public static final Setting<ByteSizeValue> INDICES_CACHE_QUERY_SIZE_SETTING =
             Setting.memorySizeSetting("indices.queries.cache.size", "10%", Property.NodeScope);
-    public static final Setting<Integer> INDICES_CACHE_QUERY_COUNT_SETTING = 
+    public static final Setting<Integer> INDICES_CACHE_QUERY_COUNT_SETTING =
             Setting.intSetting("indices.queries.cache.count", 10000, 1, Property.NodeScope);
     // enables caching on all segments instead of only the larger ones, for testing only
-    public static final Setting<Boolean> INDICES_QUERIES_CACHE_ALL_SEGMENTS_SETTING = 
+    public static final Setting<Boolean> INDICES_QUERIES_CACHE_ALL_SEGMENTS_SETTING =
             Setting.boolSetting("indices.queries.cache.all_segments", false, Property.NodeScope);
 
     private final LRUQueryCache cache;
@@ -74,9 +75,9 @@ public IndicesQueryCache(Settings settings) {
         logger.debug("using [node] query cache with size [{}] max filter count [{}]",
                 size, count);
         if (INDICES_QUERIES_CACHE_ALL_SEGMENTS_SETTING.get(settings)) {
-            cache = new ElasticsearchLRUQueryCache(count, size.bytes(), context -> true);
+            cache = new ElasticsearchLRUQueryCache(count, size.getBytes(), context -> true);
         } else {
-            cache = new ElasticsearchLRUQueryCache(count, size.bytes());
+            cache = new ElasticsearchLRUQueryCache(count, size.getBytes());
         }
         sharedRamBytesUsed = 0;
     }
@@ -102,7 +103,7 @@ public QueryCacheStats getStats(ShardId shard) {
         }
         final double weight = totalSize == 0
                 ? 1d / stats.size()
-                : shardStats.getCacheSize() / totalSize;
+                : ((double) shardStats.getCacheSize()) / totalSize;
         final long additionalRamBytesUsed = Math.round(weight * sharedRamBytesUsed);
         shardStats.add(new QueryCacheStats(additionalRamBytesUsed, 0, 0, 0, 0));
         return shardStats;
@@ -155,6 +156,12 @@ public Scorer scorer(LeafReaderContext context) throws IOException {
             return in.scorer(context);
         }
 
+        @Override
+        public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
+            shardKeyMap.add(context.reader());
+            return in.scorerSupplier(context);
+        }
+
         @Override
         public BulkScorer bulkScorer(LeafReaderContext context) throws IOException {
             shardKeyMap.add(context.reader());
diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java b/core/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java
index f78ccb22c9df1..e7bd76fb34d09 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java
@@ -41,13 +41,12 @@
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 
 import java.io.Closeable;
-import java.io.IOException;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.Set;
 import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
 
 /**
  * The indices request cache allows to cache a shard level request stage responses, helping with improving
@@ -63,7 +62,7 @@
  * is functional.
  */
 public final class IndicesRequestCache extends AbstractComponent implements RemovalListener<IndicesRequestCache.Key,
-    IndicesRequestCache.Value>, Closeable {
+    BytesReference>, Closeable {
 
     /**
      * A setting to enable or disable request caching on an index level. Its dynamic by default
@@ -80,17 +79,17 @@ public final class IndicesRequestCache extends AbstractComponent implements Remo
     private final Set<CleanupKey> keysToClean = ConcurrentCollections.newConcurrentSet();
     private final ByteSizeValue size;
     private final TimeValue expire;
-    private final Cache<Key, Value> cache;
+    private final Cache<Key, BytesReference> cache;
 
     IndicesRequestCache(Settings settings) {
         super(settings);
         this.size = INDICES_CACHE_QUERY_SIZE.get(settings);
         this.expire = INDICES_CACHE_QUERY_EXPIRE.exists(settings) ? INDICES_CACHE_QUERY_EXPIRE.get(settings) : null;
-        long sizeInBytes = size.bytes();
-        CacheBuilder<Key, Value> cacheBuilder = CacheBuilder.<Key, Value>builder()
+        long sizeInBytes = size.getBytes();
+        CacheBuilder<Key, BytesReference> cacheBuilder = CacheBuilder.<Key, BytesReference>builder()
             .setMaximumWeight(sizeInBytes).weigher((k, v) -> k.ramBytesUsed() + v.ramBytesUsed()).removalListener(this);
         if (expire != null) {
-            cacheBuilder.setExpireAfterAccess(TimeUnit.MILLISECONDS.toNanos(expire.millis()));
+            cacheBuilder.setExpireAfterAccess(expire);
         }
         cache = cacheBuilder.build();
     }
@@ -106,15 +105,16 @@ void clear(CacheEntity entity) {
     }
 
     @Override
-    public void onRemoval(RemovalNotification<Key, Value> notification) {
+    public void onRemoval(RemovalNotification<Key, BytesReference> notification) {
         notification.getKey().entity.onRemoval(notification);
     }
 
-    BytesReference getOrCompute(CacheEntity cacheEntity, DirectoryReader reader, BytesReference cacheKey) throws Exception {
+    BytesReference getOrCompute(CacheEntity cacheEntity, Supplier<BytesReference> loader,
+            DirectoryReader reader, BytesReference cacheKey) throws Exception {
         final Key key =  new Key(cacheEntity, reader.getVersion(), cacheKey);
-        Loader loader = new Loader(cacheEntity);
-        Value value = cache.computeIfAbsent(key, loader);
-        if (loader.isLoaded()) {
+        Loader cacheLoader = new Loader(cacheEntity, loader);
+        BytesReference value = cache.computeIfAbsent(key, cacheLoader);
+        if (cacheLoader.isLoaded()) {
             key.entity.onMiss();
             // see if its the first time we see this reader, and make sure to register a cleanup key
             CleanupKey cleanupKey = new CleanupKey(cacheEntity, reader.getVersion());
@@ -127,16 +127,28 @@ BytesReference getOrCompute(CacheEntity cacheEntity, DirectoryReader reader, Byt
         } else {
             key.entity.onHit();
         }
-        return value.reference;
+        return value;
     }
 
-    private static class Loader implements CacheLoader<Key, Value> {
+    /**
+     * Invalidates the given the cache entry for the given key and it's context
+     * @param cacheEntity the cache entity to invalidate for
+     * @param reader the reader to invalidate the cache entry for
+     * @param cacheKey the cache key to invalidate
+     */
+    void invalidate(CacheEntity cacheEntity, DirectoryReader reader, BytesReference cacheKey) {
+        cache.invalidate(new Key(cacheEntity, reader.getVersion(), cacheKey));
+    }
+
+    private static class Loader implements CacheLoader<Key, BytesReference> {
 
         private final CacheEntity entity;
+        private final Supplier<BytesReference> loader;
         private boolean loaded;
 
-        Loader(CacheEntity entity) {
+        Loader(CacheEntity entity, Supplier<BytesReference> loader) {
             this.entity = entity;
+            this.loader = loader;
         }
 
         public boolean isLoaded() {
@@ -144,8 +156,8 @@ public boolean isLoaded() {
         }
 
         @Override
-        public Value load(Key key) throws Exception {
-            Value value = entity.loadValue();
+        public BytesReference load(Key key) throws Exception {
+            BytesReference value = loader.get();
             entity.onCached(key, value);
             loaded = true;
             return value;
@@ -155,16 +167,12 @@ public Value load(Key key) throws Exception {
     /**
      * Basic interface to make this cache testable.
      */
-    interface CacheEntity {
-        /**
-         * Loads the actual cache value. this is the heavy lifting part.
-         */
-        Value loadValue() throws IOException;
+    interface CacheEntity extends Accountable {
 
         /**
-         * Called after the value was loaded via {@link #loadValue()}
+         * Called after the value was loaded.
          */
-        void onCached(Key key, Value value);
+        void onCached(Key key, BytesReference value);
 
         /**
          * Returns <code>true</code> iff the resource behind this entity is still open ie.
@@ -191,32 +199,12 @@ interface CacheEntity {
         /**
          * Called when this entity instance is removed
          */
-        void onRemoval(RemovalNotification<Key, Value> notification);
-    }
-
-
-
-    static class Value implements Accountable {
-        final BytesReference reference;
-        final long ramBytesUsed;
-
-        Value(BytesReference reference, long ramBytesUsed) {
-            this.reference = reference;
-            this.ramBytesUsed = ramBytesUsed;
-        }
-
-        @Override
-        public long ramBytesUsed() {
-            return ramBytesUsed;
-        }
-
-        @Override
-        public Collection<Accountable> getChildResources() {
-            return Collections.emptyList();
-        }
+        void onRemoval(RemovalNotification<Key, BytesReference> notification);
     }
 
     static class Key implements Accountable {
+        private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(Key.class);
+
         public final CacheEntity entity; // use as identity equality
         public final long readerVersion; // use the reader version to now keep a reference to a "short" lived reader until its reaped
         public final BytesReference value;
@@ -229,7 +217,7 @@ static class Key implements Accountable {
 
         @Override
         public long ramBytesUsed() {
-            return RamUsageEstimator.NUM_BYTES_OBJECT_REF + Long.BYTES + value.length();
+            return BASE_RAM_BYTES_USED + entity.ramBytesUsed() + value.length();
         }
 
         @Override
diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesService.java b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
index abc9873efaf88..9468723ef93d6 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndicesService.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
@@ -19,15 +19,16 @@
 
 package org.elasticsearch.indices;
 
-import com.carrotsearch.hppc.cursors.ObjectCursor;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.apache.logging.log4j.util.Supplier;
 import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.LockObtainFailedException;
 import org.apache.lucene.util.CollectionUtil;
 import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.RamUsageEstimator;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ResourceAlreadyExistsException;
 import org.elasticsearch.action.admin.indices.stats.CommonStats;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags.Flag;
@@ -35,22 +36,25 @@
 import org.elasticsearch.action.admin.indices.stats.ShardStats;
 import org.elasticsearch.action.fieldstats.FieldStats;
 import org.elasticsearch.action.search.SearchType;
+import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.io.FileSystemUtils;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.IndexScopedSettings;
@@ -59,9 +63,12 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.util.Callback;
+import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.iterable.Iterables;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.env.ShardLock;
 import org.elasticsearch.env.ShardLockObtainFailedException;
@@ -72,7 +79,6 @@
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.NodeServicesProvider;
 import org.elasticsearch.index.analysis.AnalysisRegistry;
 import org.elasticsearch.index.cache.request.ShardRequestCache;
 import org.elasticsearch.index.engine.Engine;
@@ -82,6 +88,8 @@
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.merge.MergeStats;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.recovery.RecoveryStats;
 import org.elasticsearch.index.refresh.RefreshStats;
 import org.elasticsearch.index.search.stats.SearchStats;
@@ -93,16 +101,16 @@
 import org.elasticsearch.index.shard.IndexingStats;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.store.IndexStoreConfig;
-import org.elasticsearch.indices.AbstractIndexShardCacheEntity.Loader;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.cluster.IndicesClusterStateService;
 import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
 import org.elasticsearch.indices.mapper.MapperRegistry;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.indices.recovery.PeerRecoveryTargetService;
 import org.elasticsearch.indices.recovery.RecoveryState;
 import org.elasticsearch.plugins.PluginsService;
 import org.elasticsearch.repositories.RepositoriesService;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.search.query.QueryPhase;
@@ -113,12 +121,11 @@
 import java.io.IOException;
 import java.nio.file.Files;
 import java.util.ArrayList;
-import java.util.Collections;
-import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
@@ -126,9 +133,12 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Consumer;
 import java.util.function.Predicate;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
+import static java.util.Collections.emptyList;
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.unmodifiableMap;
 import static org.elasticsearch.common.collect.MapBuilder.newMapBuilder;
@@ -142,15 +152,19 @@ public class IndicesService extends AbstractLifecycleComponent
         Setting.positiveTimeSetting("indices.cache.cleanup_interval", TimeValue.timeValueMinutes(1), Property.NodeScope);
     private final PluginsService pluginsService;
     private final NodeEnvironment nodeEnv;
+    private final NamedXContentRegistry xContentRegistry;
     private final TimeValue shardsClosedTimeout;
     private final AnalysisRegistry analysisRegistry;
-    private final IndicesQueriesRegistry indicesQueriesRegistry;
     private final IndexNameExpressionResolver indexNameExpressionResolver;
     private final IndexScopedSettings indexScopeSetting;
     private final IndicesFieldDataCache indicesFieldDataCache;
     private final CacheCleaner cacheCleaner;
     private final ThreadPool threadPool;
     private final CircuitBreakerService circuitBreakerService;
+    private final BigArrays bigArrays;
+    private final ScriptService scriptService;
+    private final ClusterService clusterService;
+    private final Client client;
     private volatile Map<String, IndexService> indices = emptyMap();
     private final Map<Index, List<PendingDelete>> pendingDeletes = new HashMap<>();
     private final AtomicInteger numUncompletedDeletes = new AtomicInteger();
@@ -170,20 +184,21 @@ protected void doStart() {
         threadPool.schedule(this.cleanInterval, ThreadPool.Names.SAME, this.cacheCleaner);
     }
 
-    public IndicesService(Settings settings, PluginsService pluginsService, NodeEnvironment nodeEnv,
+    public IndicesService(Settings settings, PluginsService pluginsService, NodeEnvironment nodeEnv, NamedXContentRegistry xContentRegistry,
                           ClusterSettings clusterSettings, AnalysisRegistry analysisRegistry,
-                          IndicesQueriesRegistry indicesQueriesRegistry, IndexNameExpressionResolver indexNameExpressionResolver,
+                          IndexNameExpressionResolver indexNameExpressionResolver,
                           MapperRegistry mapperRegistry, NamedWriteableRegistry namedWriteableRegistry,
                           ThreadPool threadPool, IndexScopedSettings indexScopedSettings, CircuitBreakerService circuitBreakerService,
+                          BigArrays bigArrays, ScriptService scriptService, ClusterService clusterService, Client client,
                           MetaStateService metaStateService) {
         super(settings);
         this.threadPool = threadPool;
         this.pluginsService = pluginsService;
         this.nodeEnv = nodeEnv;
+        this.xContentRegistry = xContentRegistry;
         this.shardsClosedTimeout = settings.getAsTime(INDICES_SHARDS_CLOSED_TIMEOUT, new TimeValue(1, TimeUnit.DAYS));
         this.indexStoreConfig = new IndexStoreConfig(settings);
         this.analysisRegistry = analysisRegistry;
-        this.indicesQueriesRegistry = indicesQueriesRegistry;
         this.indexNameExpressionResolver = indexNameExpressionResolver;
         this.indicesRequestCache = new IndicesRequestCache(settings);
         this.indicesQueryCache = new IndicesQueryCache(settings);
@@ -196,6 +211,10 @@ public IndicesService(Settings settings, PluginsService pluginsService, NodeEnvi
                                                                 () -> Iterables.flatten(this).iterator());
         this.indexScopeSetting = indexScopedSettings;
         this.circuitBreakerService = circuitBreakerService;
+        this.bigArrays = bigArrays;
+        this.scriptService = scriptService;
+        this.clusterService = clusterService;
+        this.client = client;
         this.indicesFieldDataCache = new IndicesFieldDataCache(settings, new IndexFieldDataCache.Listener() {
             @Override
             public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, long sizeInBytes) {
@@ -218,9 +237,7 @@ protected void doStop() {
         for (final Index index : indices) {
             indicesStopExecutor.execute(() -> {
                 try {
-                    removeIndex(index, "shutdown", false);
-                } catch (Exception e) {
-                    logger.warn((Supplier<?>) () -> new ParameterizedMessage("failed to remove index on stop [{}]", index), e);
+                    removeIndex(index, IndexRemovalReason.NO_LONGER_ASSIGNED, "shutdown");
                 } finally {
                     latch.countDown();
                 }
@@ -283,26 +300,48 @@ public NodeIndicesStats stats(boolean includePrevious, CommonStatsFlags flags) {
             }
         }
 
-        Map<Index, List<IndexShardStats>> statsByShard = new HashMap<>();
-        for (IndexService indexService : this) {
-            for (IndexShard indexShard : indexService) {
+        return new NodeIndicesStats(oldStats, statsByShard(this, flags));
+    }
+
+    Map<Index, List<IndexShardStats>> statsByShard(final IndicesService indicesService, final CommonStatsFlags flags) {
+        final Map<Index, List<IndexShardStats>> statsByShard = new HashMap<>();
+
+        for (final IndexService indexService : indicesService) {
+            for (final IndexShard indexShard : indexService) {
                 try {
-                    if (indexShard.routingEntry() == null) {
+                    final IndexShardStats indexShardStats = indicesService.indexShardStats(indicesService, indexShard, flags);
+
+                    if (indexShardStats == null) {
                         continue;
                     }
-                    IndexShardStats indexShardStats = new IndexShardStats(indexShard.shardId(), new ShardStats[] { new ShardStats(indexShard.routingEntry(), indexShard.shardPath(), new CommonStats(indicesQueryCache, indexShard, flags), indexShard.commitStats()) });
-                    if (!statsByShard.containsKey(indexService.index())) {
+
+                    if (statsByShard.containsKey(indexService.index()) == false) {
                         statsByShard.put(indexService.index(), arrayAsArrayList(indexShardStats));
                     } else {
                         statsByShard.get(indexService.index()).add(indexShardStats);
                     }
-                } catch (IllegalIndexShardStateException e) {
+                } catch (IllegalIndexShardStateException | AlreadyClosedException e) {
                     // we can safely ignore illegal state on ones that are closing for example
                     logger.trace((Supplier<?>) () -> new ParameterizedMessage("{} ignoring shard stats", indexShard.shardId()), e);
                 }
             }
         }
-        return new NodeIndicesStats(oldStats, statsByShard);
+
+        return statsByShard;
+    }
+
+    IndexShardStats indexShardStats(final IndicesService indicesService, final IndexShard indexShard, final CommonStatsFlags flags) {
+        if (indexShard.routingEntry() == null) {
+            return null;
+        }
+
+        return new IndexShardStats(indexShard.shardId(),
+                                   new ShardStats[] {
+                                       new ShardStats(indexShard.routingEntry(),
+                                                      indexShard.shardPath(),
+                                                      new CommonStats(indicesService.getIndicesQueryCache(), indexShard, flags),
+                                                      indexShard.commitStats())
+                                   });
     }
 
     /**
@@ -350,17 +389,17 @@ public IndexService indexServiceSafe(Index index) {
      * Creates a new {@link IndexService} for the given metadata.
      * @param indexMetaData the index metadata to create the index for
      * @param builtInListeners a list of built-in lifecycle {@link IndexEventListener} that should should be used along side with the per-index listeners
-     * @throws IndexAlreadyExistsException if the index already exists.
+     * @throws ResourceAlreadyExistsException if the index already exists.
      */
     @Override
-    public synchronized IndexService createIndex(final NodeServicesProvider nodeServicesProvider, IndexMetaData indexMetaData, List<IndexEventListener> builtInListeners) throws IOException {
+    public synchronized IndexService createIndex(IndexMetaData indexMetaData, List<IndexEventListener> builtInListeners) throws IOException {
         ensureChangesAllowed();
         if (indexMetaData.getIndexUUID().equals(IndexMetaData.INDEX_UUID_NA_VALUE)) {
             throw new IllegalArgumentException("index must have a real UUID found value: [" + indexMetaData.getIndexUUID() + "]");
         }
         final Index index = indexMetaData.getIndex();
         if (hasIndex(index)) {
-            throw new IndexAlreadyExistsException(index);
+            throw new ResourceAlreadyExistsException(index);
         }
         List<IndexEventListener> finalListeners = new ArrayList<>(builtInListeners);
         final IndexEventListener onStoreClose = new IndexEventListener() {
@@ -371,7 +410,7 @@ public void onStoreClosed(ShardId shardId) {
         };
         finalListeners.add(onStoreClose);
         finalListeners.add(oldShardsStats);
-        final IndexService indexService = createIndexService("create index", nodeServicesProvider, indexMetaData, indicesQueryCache, indicesFieldDataCache, finalListeners, indexingMemoryController);
+        final IndexService indexService = createIndexService("create index", indexMetaData, indicesQueryCache, indicesFieldDataCache, finalListeners, indexingMemoryController);
         boolean success = false;
         try {
             indexService.getIndexEventListener().afterIndexCreated(indexService);
@@ -388,9 +427,8 @@ public void onStoreClosed(ShardId shardId) {
     /**
      * This creates a new IndexService without registering it
      */
-    private synchronized IndexService createIndexService(final String reason, final NodeServicesProvider nodeServicesProvider, IndexMetaData indexMetaData, IndicesQueryCache indicesQueryCache, IndicesFieldDataCache indicesFieldDataCache, List<IndexEventListener> builtInListeners, IndexingOperationListener... indexingOperationListeners) throws IOException {
+    private synchronized IndexService createIndexService(final String reason, IndexMetaData indexMetaData, IndicesQueryCache indicesQueryCache, IndicesFieldDataCache indicesFieldDataCache, List<IndexEventListener> builtInListeners, IndexingOperationListener... indexingOperationListeners) throws IOException {
         final Index index = indexMetaData.getIndex();
-        final ClusterService clusterService = nodeServicesProvider.getClusterService();
         final Predicate<String> indexNameMatcher = (indexExpression) -> indexNameExpressionResolver.matchesIndex(index.getName(), indexExpression, clusterService.state());
         final IndexSettings idxSettings = new IndexSettings(indexMetaData, this.settings, indexNameMatcher, indexScopeSetting);
         logger.debug("creating Index [{}], shards [{}]/[{}{}] - reason [{}]",
@@ -407,7 +445,23 @@ private synchronized IndexService createIndexService(final String reason, final
         for (IndexEventListener listener : builtInListeners) {
             indexModule.addIndexEventListener(listener);
         }
-        return indexModule.newIndexService(nodeEnv, this, nodeServicesProvider, indicesQueryCache, mapperRegistry, indicesFieldDataCache);
+        return indexModule.newIndexService(nodeEnv, xContentRegistry, this, circuitBreakerService, bigArrays, threadPool, scriptService,
+                clusterService, client, indicesQueryCache, mapperRegistry, indicesFieldDataCache);
+    }
+
+    /**
+     * creates a new mapper service for the given index, in order to do administrative work like mapping updates.
+     * This *should not* be used for document parsing. Doing so will result in an exception.
+     *
+     * Note: the returned {@link MapperService} should be closed when unneeded.
+     */
+    public synchronized MapperService createIndexMapperService(IndexMetaData indexMetaData) throws IOException {
+        final Index index = indexMetaData.getIndex();
+        final Predicate<String> indexNameMatcher = (indexExpression) -> indexNameExpressionResolver.matchesIndex(index.getName(), indexExpression, clusterService.state());
+        final IndexSettings idxSettings = new IndexSettings(indexMetaData, this.settings, indexNameMatcher, indexScopeSetting);
+        final IndexModule indexModule = new IndexModule(idxSettings, indexStoreConfig, analysisRegistry);
+        pluginsService.onIndexModule(indexModule);
+        return indexModule.newIndexMapperService(xContentRegistry, mapperRegistry);
     }
 
     /**
@@ -416,7 +470,7 @@ private synchronized IndexService createIndexService(final String reason, final
      * This method will throw an exception if the creation or the update fails.
      * The created {@link IndexService} will not be registered and will be closed immediately.
      */
-    public synchronized void verifyIndexMetadata(final NodeServicesProvider nodeServicesProvider, IndexMetaData metaData, IndexMetaData metaDataUpdate) throws IOException {
+    public synchronized void verifyIndexMetadata(IndexMetaData metaData, IndexMetaData metaDataUpdate) throws IOException {
         final List<Closeable> closeables = new ArrayList<>();
         try {
             IndicesFieldDataCache indicesFieldDataCache = new IndicesFieldDataCache(settings, new IndexFieldDataCache.Listener() {});
@@ -424,14 +478,10 @@ public synchronized void verifyIndexMetadata(final NodeServicesProvider nodeServ
             IndicesQueryCache indicesQueryCache = new IndicesQueryCache(settings);
             closeables.add(indicesQueryCache);
             // this will also fail if some plugin fails etc. which is nice since we can verify that early
-            final IndexService service = createIndexService("metadata verification", nodeServicesProvider,
-                metaData, indicesQueryCache, indicesFieldDataCache, Collections.emptyList());
+            final IndexService service = createIndexService("metadata verification", metaData, indicesQueryCache, indicesFieldDataCache,
+                    emptyList());
             closeables.add(() -> service.close("metadata verification", false));
-            for (ObjectCursor<MappingMetaData> typeMapping : metaData.getMappings().values()) {
-                // don't apply the default mapping, it has been applied when the mapping was created
-                service.mapperService().merge(typeMapping.value.type(), typeMapping.value.source(),
-                    MapperService.MergeReason.MAPPING_RECOVERY, true);
-            }
+            service.mapperService().merge(metaData, MapperService.MergeReason.MAPPING_RECOVERY, true);
             if (metaData.equals(metaDataUpdate) == false) {
                 service.updateMetaData(metaDataUpdate);
             }
@@ -443,7 +493,7 @@ public synchronized void verifyIndexMetadata(final NodeServicesProvider nodeServ
     @Override
     public IndexShard createShard(ShardRouting shardRouting, RecoveryState recoveryState, PeerRecoveryTargetService recoveryTargetService,
                                   PeerRecoveryTargetService.RecoveryListener recoveryListener, RepositoriesService repositoriesService,
-                                  NodeServicesProvider nodeServicesProvider, Callback<IndexShard.ShardFailure> onShardFailure) throws IOException {
+                                  Consumer<IndexShard.ShardFailure> onShardFailure) throws IOException {
         ensureChangesAllowed();
         IndexService indexService = indexService(shardRouting.index());
         IndexShard indexShard = indexService.createShard(shardRouting);
@@ -453,7 +503,7 @@ public IndexShard createShard(ShardRouting shardRouting, RecoveryState recoveryS
                 assert recoveryState.getRecoverySource().getType() == RecoverySource.Type.LOCAL_SHARDS:
                     "mapping update consumer only required by local shards recovery";
                 try {
-                    nodeServicesProvider.getClient().admin().indices().preparePutMapping()
+                    client.admin().indices().preparePutMapping()
                         .setConcreteIndex(shardRouting.index()) // concrete index - no name clash, it uses uuid
                         .setType(type)
                         .setSource(mapping.source().string())
@@ -465,22 +515,8 @@ public IndexShard createShard(ShardRouting shardRouting, RecoveryState recoveryS
         return indexShard;
     }
 
-    /**
-     * Removes the given index from this service and releases all associated resources. Persistent parts of the index
-     * like the shards files, state and transaction logs are kept around in the case of a disaster recovery.
-     * @param index the index to remove
-     * @param reason  the high level reason causing this removal
-     */
     @Override
-    public void removeIndex(Index index, String reason) {
-        try {
-            removeIndex(index, reason, false);
-        } catch (Exception e) {
-            logger.warn((Supplier<?>) () -> new ParameterizedMessage("failed to remove index ({})", reason), e);
-        }
-    }
-
-    private void removeIndex(Index index, String reason, boolean delete) {
+    public void removeIndex(final Index index, final IndexRemovalReason reason, final String extraInfo) {
         final String indexName = index.getName();
         try {
             final IndexService indexService;
@@ -498,22 +534,18 @@ private void removeIndex(Index index, String reason, boolean delete) {
                 listener = indexService.getIndexEventListener();
             }
 
-            listener.beforeIndexClosed(indexService);
-            if (delete) {
-                listener.beforeIndexDeleted(indexService);
-            }
-            logger.debug("{} closing index service (reason [{}])", index, reason);
-            indexService.close(reason, delete);
-            logger.debug("{} closed... (reason [{}])", index, reason);
-            listener.afterIndexClosed(indexService.index(), indexService.getIndexSettings().getSettings());
-            if (delete) {
-                final IndexSettings indexSettings = indexService.getIndexSettings();
-                listener.afterIndexDeleted(indexService.index(), indexSettings.getSettings());
+            listener.beforeIndexRemoved(indexService, reason);
+            logger.debug("{} closing index service (reason [{}][{}])", index, reason, extraInfo);
+            indexService.close(extraInfo, reason == IndexRemovalReason.DELETED);
+            logger.debug("{} closed... (reason [{}][{}])", index, reason, extraInfo);
+            final IndexSettings indexSettings = indexService.getIndexSettings();
+            listener.afterIndexRemoved(indexService.index(), indexSettings, reason);
+            if (reason == IndexRemovalReason.DELETED) {
                 // now we are done - try to wipe data on disk if possible
-                deleteIndexStore(reason, indexService.index(), indexSettings);
+                deleteIndexStore(extraInfo, indexService.index(), indexSettings);
             }
-        } catch (IOException ex) {
-            throw new ElasticsearchException("failed to remove index " + index, ex);
+        } catch (Exception e) {
+            logger.warn((Supplier<?>) () -> new ParameterizedMessage("failed to remove index {} ([{}][{}])", index, reason, extraInfo), e);
         }
     }
 
@@ -553,27 +585,9 @@ public synchronized void beforeIndexShardClosed(ShardId shardId, @Nullable Index
         }
     }
 
-    /**
-     * Deletes the given index. Persistent parts of the index
-     * like the shards files, state and transaction logs are removed once all resources are released.
-     *
-     * Equivalent to {@link #removeIndex(Index, String)} but fires
-     * different lifecycle events to ensure pending resources of this index are immediately removed.
-     * @param index the index to delete
-     * @param reason the high level reason causing this delete
-     */
-    @Override
-    public void deleteIndex(Index index, String reason) {
-        try {
-            removeIndex(index, reason, true);
-        } catch (Exception e) {
-            logger.warn((Supplier<?>) () -> new ParameterizedMessage("failed to delete index ({})", reason), e);
-        }
-    }
-
     /**
      * Deletes an index that is not assigned to this node. This method cleans up all disk folders relating to the index
-     * but does not deal with in-memory structures. For those call {@link #deleteIndex(Index, String)}
+     * but does not deal with in-memory structures. For those call {@link #removeIndex(Index, IndexRemovalReason, String)}
      */
     @Override
     public void deleteUnassignedIndex(String reason, IndexMetaData metaData, ClusterState clusterState) {
@@ -586,7 +600,7 @@ public void deleteUnassignedIndex(String reason, IndexMetaData metaData, Cluster
                                                     "the cluster state [" + index.getIndexUUID() + "] [" + metaData.getIndexUUID() + "]");
                 }
                 deleteIndexStore(reason, metaData, clusterState);
-            } catch (IOException e) {
+            } catch (Exception e) {
                 logger.warn((Supplier<?>) () -> new ParameterizedMessage("[{}] failed to delete unassigned index (reason [{}])", metaData.getIndex(), reason), e);
             }
         }
@@ -683,8 +697,9 @@ public void deleteShardStore(String reason, ShardId shardId, ClusterState cluste
         final IndexMetaData metaData = clusterState.getMetaData().indices().get(shardId.getIndexName());
 
         final IndexSettings indexSettings = buildIndexSettings(metaData);
-        if (canDeleteShardContent(shardId, indexSettings) == false) {
-            throw new IllegalStateException("Can't delete shard " + shardId);
+        ShardDeletionCheckResult shardDeletionCheckResult = canDeleteShardContent(shardId, indexSettings);
+        if (shardDeletionCheckResult != ShardDeletionCheckResult.FOLDER_FOUND_CAN_DELETE) {
+            throw new IllegalStateException("Can't delete shard " + shardId + " (cause: " + shardDeletionCheckResult + ")");
         }
         nodeEnv.deleteShardDirectorySafe(shardId, indexSettings);
         logger.debug("{} deleted shard reason [{}]", shardId, reason);
@@ -747,14 +762,14 @@ public IndexMetaData verifyIndexIsDeleted(final Index index, final ClusterState
             final IndexMetaData metaData;
             try {
                 metaData = metaStateService.loadIndexState(index);
-            } catch (IOException e) {
+            } catch (Exception e) {
                 logger.warn((Supplier<?>) () -> new ParameterizedMessage("[{}] failed to load state file from a stale deleted index, folders will be left on disk", index), e);
                 return null;
             }
             final IndexSettings indexSettings = buildIndexSettings(metaData);
             try {
                 deleteIndexStoreIfDeletionAllowed("stale deleted index", index, indexSettings, ALWAYS_TRUE);
-            } catch (IOException e) {
+            } catch (Exception e) {
                 // we just warn about the exception here because if deleteIndexStoreIfDeletionAllowed
                 // throws an exception, it gets added to the list of pending deletes to be tried again
                 logger.warn((Supplier<?>) () -> new ParameterizedMessage("[{}] failed to delete index on disk", metaData.getIndex()), e);
@@ -765,39 +780,50 @@ public IndexMetaData verifyIndexIsDeleted(final Index index, final ClusterState
     }
 
     /**
-     * Returns <code>true</code> iff the shards content for the given shard can be deleted.
-     * This method will return <code>false</code> if:
-     * <ul>
-     *     <li>if the shard is still allocated / active on this node</li>
-     *     <li>if for instance if the shard is located on shared and should not be deleted</li>
-     *     <li>if the shards data locations do not exists</li>
-     * </ul>
+     * result type returned by {@link #canDeleteShardContent signaling different reasons why a shard can / cannot be deleted}
+     */
+    public enum ShardDeletionCheckResult {
+        FOLDER_FOUND_CAN_DELETE, // shard data exists and can be deleted
+        STILL_ALLOCATED, // the shard is still allocated / active on this node
+        NO_FOLDER_FOUND, // the shards data locations do not exist
+        SHARED_FILE_SYSTEM, // the shard is located on shared and should not be deleted
+        NO_LOCAL_STORAGE // node does not have local storage (see DiscoveryNode.nodeRequiresLocalStorage)
+    }
+
+    /**
+     * Returns <code>ShardDeletionCheckResult</code> signaling whether the shards content for the given shard can be deleted.
      *
      * @param shardId the shard to delete.
      * @param indexSettings the shards's relevant {@link IndexSettings}. This is required to access the indexes settings etc.
      */
-    public boolean canDeleteShardContent(ShardId shardId, IndexSettings indexSettings) {
+    public ShardDeletionCheckResult canDeleteShardContent(ShardId shardId, IndexSettings indexSettings) {
         assert shardId.getIndex().equals(indexSettings.getIndex());
         final IndexService indexService = indexService(shardId.getIndex());
         if (indexSettings.isOnSharedFilesystem() == false) {
            if (nodeEnv.hasNodeFile()) {
                 final boolean isAllocated = indexService != null && indexService.hasShard(shardId.id());
                 if (isAllocated) {
-                    return false; // we are allocated - can't delete the shard
+                    return ShardDeletionCheckResult.STILL_ALLOCATED; // we are allocated - can't delete the shard
                 } else if (indexSettings.hasCustomDataPath()) {
                     // lets see if it's on a custom path (return false if the shared doesn't exist)
                     // we don't need to delete anything that is not there
-                    return Files.exists(nodeEnv.resolveCustomLocation(indexSettings, shardId));
+                    return Files.exists(nodeEnv.resolveCustomLocation(indexSettings, shardId)) ?
+                        ShardDeletionCheckResult.FOLDER_FOUND_CAN_DELETE :
+                        ShardDeletionCheckResult.NO_FOLDER_FOUND;
                 } else {
                     // lets see if it's path is available (return false if the shared doesn't exist)
                     // we don't need to delete anything that is not there
-                    return FileSystemUtils.exists(nodeEnv.availableShardPaths(shardId));
+                    return FileSystemUtils.exists(nodeEnv.availableShardPaths(shardId)) ?
+                        ShardDeletionCheckResult.FOLDER_FOUND_CAN_DELETE :
+                        ShardDeletionCheckResult.NO_FOLDER_FOUND;
                 }
-            }
+            } else {
+               return ShardDeletionCheckResult.NO_LOCAL_STORAGE;
+           }
         } else {
             logger.trace("{} skipping shard directory deletion due to shadow replicas", shardId);
+            return ShardDeletionCheckResult.SHARED_FILE_SYSTEM;
         }
-        return false;
     }
 
     private IndexSettings buildIndexSettings(IndexMetaData metaData) {
@@ -851,7 +877,7 @@ private static final class PendingDelete implements Comparable<PendingDelete> {
         /**
          * Creates a new pending delete of an index
          */
-        public PendingDelete(ShardId shardId, IndexSettings settings) {
+        PendingDelete(ShardId shardId, IndexSettings settings) {
             this.index = shardId.getIndex();
             this.shardId = shardId.getId();
             this.settings = settings;
@@ -861,7 +887,7 @@ public PendingDelete(ShardId shardId, IndexSettings settings) {
         /**
          * Creates a new pending delete of a shard
          */
-        public PendingDelete(Index index, IndexSettings settings) {
+        PendingDelete(Index index, IndexSettings settings) {
             this.index = index;
             this.shardId = -1;
             this.settings = settings;
@@ -982,13 +1008,6 @@ public boolean hasUncompletedPendingDeletes() {
         return numUncompletedDeletes.get() > 0;
     }
 
-    /**
-     * Returns this nodes {@link IndicesQueriesRegistry}
-     */
-    public IndicesQueriesRegistry getIndicesQueryRegistry() {
-        return indicesQueriesRegistry;
-    }
-
     public AnalysisRegistry getAnalysis() {
         return analysisRegistry;
     }
@@ -1008,7 +1027,7 @@ private static final class CacheCleaner implements Runnable, Releasable {
         private final AtomicBoolean closed = new AtomicBoolean(false);
         private final IndicesRequestCache requestCache;
 
-        public CacheCleaner(IndicesFieldDataCache cache, IndicesRequestCache requestCache, Logger logger, ThreadPool threadPool, TimeValue interval) {
+        CacheCleaner(IndicesFieldDataCache cache, IndicesRequestCache requestCache, Logger logger, ThreadPool threadPool, TimeValue interval) {
             this.cache = cache;
             this.requestCache = requestCache;
             this.logger = logger;
@@ -1049,8 +1068,6 @@ public void close() {
     }
 
 
-    private static final Set<SearchType> CACHEABLE_SEARCH_TYPES = EnumSet.of(SearchType.QUERY_THEN_FETCH, SearchType.QUERY_AND_FETCH);
-
     /**
      * Can the shard request be cached at all?
      */
@@ -1060,7 +1077,7 @@ public boolean canCache(ShardSearchRequest request, SearchContext context) {
         // on the overridden statistics. So if you ran two queries on the same index with different stats
         // (because an other shard was updated) you would get wrong results because of the scores
         // (think about top_hits aggs or scripts using the score)
-        if (!CACHEABLE_SEARCH_TYPES.contains(context.searchType())) {
+        if (SearchType.QUERY_THEN_FETCH != context.searchType()) {
             return false;
         }
         IndexSettings settings = context.indexShard().indexSettings();
@@ -1082,7 +1099,7 @@ public boolean canCache(ShardSearchRequest request, SearchContext context) {
         }
         // if now in millis is used (or in the future, a more generic "isDeterministic" flag
         // then we can't cache based on "now" key within the search request, as it is not deterministic
-        if (context.nowInMillisUsed()) {
+        if (context.getQueryShardContext().isCachable() == false) {
             return false;
         }
         return true;
@@ -1093,7 +1110,7 @@ public void clearRequestCache(IndexShard shard) {
         if (shard == null) {
             return;
         }
-        indicesRequestCache.clear(new IndexShardCacheEntity(shard, null));
+        indicesRequestCache.clear(new IndexShardCacheEntity(shard));
         logger.trace("{} explicit cache clear", shard.shardId());
     }
 
@@ -1105,18 +1122,35 @@ public void clearRequestCache(IndexShard shard) {
      */
     public void loadIntoContext(ShardSearchRequest request, SearchContext context, QueryPhase queryPhase) throws Exception {
         assert canCache(request, context);
-        final IndexShardCacheEntity entity = new IndexShardCacheEntity(context.indexShard(), out -> {
+        final DirectoryReader directoryReader = context.searcher().getDirectoryReader();
+
+        boolean[] loadedFromCache = new boolean[] { true };
+        BytesReference bytesReference = cacheShardLevelResult(context.indexShard(), directoryReader, request.cacheKey(), out -> {
             queryPhase.execute(context);
-            context.queryResult().writeToNoId(out);
+            try {
+                context.queryResult().writeToNoId(out);
+
+            } catch (IOException e) {
+                throw new AssertionError("Could not serialize response", e);
+            }
+            loadedFromCache[0] = false;
         });
-        final DirectoryReader directoryReader = context.searcher().getDirectoryReader();
-        final BytesReference bytesReference = indicesRequestCache.getOrCompute(entity, directoryReader, request.cacheKey());
-        if (entity.loadedFromCache()) {
+
+        if (loadedFromCache[0]) {
             // restore the cached query result into the context
             final QuerySearchResult result = context.queryResult();
             StreamInput in = new NamedWriteableAwareStreamInput(bytesReference.streamInput(), namedWriteableRegistry);
             result.readFromWithId(context.id(), in);
-            result.shardTarget(context.shardTarget());
+            result.setSearchShardTarget(context.shardTarget());
+        } else if (context.queryResult().searchTimedOut()) {
+            // we have to invalidate the cache entry if we cached a query result form a request that timed out.
+            // we can't really throw exceptions in the loading part to signal a timed out search to the outside world since if there are
+            // multiple requests that wait for the cache entry to be calculated they'd fail all with the same exception.
+            // instead we all caching such a result for the time being, return the timed out result for all other searches with that cache
+            // key invalidate the result in the thread that caused the timeout. This will end up to be simpler and eventually correct since
+            // running a search that times out concurrently will likely timeout again if it's run while we have this `stale` result in the
+            // cache. One other option is to not cache requests with a timeout at all...
+            indicesRequestCache.invalidate(new IndexShardCacheEntity(context.indexShard()), directoryReader, request.cacheKey());
         }
     }
 
@@ -1137,7 +1171,11 @@ public FieldStats<?> getFieldStats(IndexShard shard, Engine.Searcher searcher, S
         }
         BytesReference cacheKey = new BytesArray("fieldstats:" + field);
         BytesReference statsRef = cacheShardLevelResult(shard, searcher.getDirectoryReader(), cacheKey, out -> {
-            out.writeOptionalWriteable(fieldType.stats(searcher.reader()));
+            try {
+                out.writeOptionalWriteable(fieldType.stats(searcher.reader()));
+            } catch (IOException e) {
+                throw new IllegalStateException("Failed to write field stats output", e);
+            }
         });
         try (StreamInput in = statsRef.streamInput()) {
             return in.readOptionalWriteable(FieldStats::readFrom);
@@ -1156,17 +1194,33 @@ public ByteSizeValue getTotalIndexingBufferBytes() {
      * @param loader loads the data into the cache if needed
      * @return the contents of the cache or the result of calling the loader
      */
-    private BytesReference cacheShardLevelResult(IndexShard shard, DirectoryReader reader, BytesReference cacheKey, Loader loader)
+    private BytesReference cacheShardLevelResult(IndexShard shard, DirectoryReader reader, BytesReference cacheKey, Consumer<StreamOutput> loader)
             throws Exception {
-        IndexShardCacheEntity cacheEntity = new IndexShardCacheEntity(shard, loader);
-        return indicesRequestCache.getOrCompute(cacheEntity, reader, cacheKey);
+        IndexShardCacheEntity cacheEntity = new IndexShardCacheEntity(shard);
+        Supplier<BytesReference> supplier = () -> {
+            /* BytesStreamOutput allows to pass the expected size but by default uses
+             * BigArrays.PAGE_SIZE_IN_BYTES which is 16k. A common cached result ie.
+             * a date histogram with 3 buckets is ~100byte so 16k might be very wasteful
+             * since we don't shrink to the actual size once we are done serializing.
+             * By passing 512 as the expected size we will resize the byte array in the stream
+             * slowly until we hit the page size and don't waste too much memory for small query
+             * results.*/
+            final int expectedSizeInBytes = 512;
+            try (BytesStreamOutput out = new BytesStreamOutput(expectedSizeInBytes)) {
+                loader.accept(out);
+                // for now, keep the paged data structure, which might have unused bytes to fill a page, but better to keep
+                // the memory properly paged instead of having varied sized bytes
+                return out.bytes();
+            }
+        };
+        return indicesRequestCache.getOrCompute(cacheEntity, supplier, reader, cacheKey);
     }
 
     static final class IndexShardCacheEntity extends AbstractIndexShardCacheEntity {
+        private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(IndexShardCacheEntity.class);
         private final IndexShard indexShard;
 
-        protected IndexShardCacheEntity(IndexShard indexShard, Loader loader) {
-            super(loader);
+        protected IndexShardCacheEntity(IndexShard indexShard) {
             this.indexShard = indexShard;
         }
 
@@ -1184,6 +1238,13 @@ public boolean isOpen() {
         public Object getCacheIdentity() {
             return indexShard;
         }
+
+        @Override
+        public long ramBytesUsed() {
+            // No need to take the IndexShard into account since it is shared
+            // across many entities
+            return BASE_RAM_BYTES_USED;
+        }
     }
 
     @FunctionalInterface
@@ -1195,4 +1256,17 @@ interface IndexDeletionAllowedPredicate {
         (Index index, IndexSettings indexSettings) -> canDeleteIndexContents(index, indexSettings);
     private final IndexDeletionAllowedPredicate ALWAYS_TRUE = (Index index, IndexSettings indexSettings) -> true;
 
+    public AliasFilter buildAliasFilter(ClusterState state, String index, String... expressions) {
+        /* Being static, parseAliasFilter doesn't have access to whatever guts it needs to parse a query. Instead of passing in a bunch
+         * of dependencies we pass in a function that can perform the parsing. */
+        CheckedFunction<byte[], Optional<QueryBuilder>, IOException> filterParser = bytes -> {
+            try (XContentParser parser = XContentFactory.xContent(bytes).createParser(xContentRegistry, bytes)) {
+                return new QueryParseContext(parser).parseInnerQueryBuilder();
+            }
+        };
+        String[] aliases = indexNameExpressionResolver.filteringAliases(state, index, expressions);
+        IndexMetaData indexMetaData = state.metaData().index(index);
+        return new AliasFilter(ShardSearchRequest.parseAliasFilter(filterParser, indexMetaData, aliases), aliases);
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/indices/InvalidAliasNameException.java b/core/src/main/java/org/elasticsearch/indices/InvalidAliasNameException.java
index 4e2c443ff4a4e..84215d45f6895 100644
--- a/core/src/main/java/org/elasticsearch/indices/InvalidAliasNameException.java
+++ b/core/src/main/java/org/elasticsearch/indices/InvalidAliasNameException.java
@@ -36,6 +36,10 @@ public InvalidAliasNameException(Index index, String name, String desc) {
         setIndex(index);
     }
 
+    public InvalidAliasNameException(String name, String description) {
+        super("Invalid alias name [{}]: {}", name, description);
+    }
+
     public InvalidAliasNameException(StreamInput in) throws IOException{
         super(in);
     }
diff --git a/core/src/main/java/org/elasticsearch/indices/NodeIndicesStats.java b/core/src/main/java/org/elasticsearch/indices/NodeIndicesStats.java
index 6c251d3bf1ce0..9133ca81e2837 100644
--- a/core/src/main/java/org/elasticsearch/indices/NodeIndicesStats.java
+++ b/core/src/main/java/org/elasticsearch/indices/NodeIndicesStats.java
@@ -188,10 +188,11 @@ public void writeTo(StreamOutput out) throws IOException {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        String level = params.param("level", "node");
-        boolean isLevelValid = "node".equalsIgnoreCase(level) || "indices".equalsIgnoreCase(level) || "shards".equalsIgnoreCase(level);
+        final String level = params.param("level", "node");
+        final boolean isLevelValid =
+            "indices".equalsIgnoreCase(level) || "node".equalsIgnoreCase(level) || "shards".equalsIgnoreCase(level);
         if (!isLevelValid) {
-            return builder;
+            throw new IllegalArgumentException("level parameter must be one of [indices] or [node] or [shards] but was [" + level + "]");
         }
 
         // "node" level
diff --git a/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java b/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java
index 5dd0203d61720..61950942e6076 100644
--- a/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java
+++ b/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java
@@ -60,6 +60,7 @@
 import org.elasticsearch.index.analysis.FingerprintAnalyzerProvider;
 import org.elasticsearch.index.analysis.FingerprintTokenFilterFactory;
 import org.elasticsearch.index.analysis.FinnishAnalyzerProvider;
+import org.elasticsearch.index.analysis.FlattenGraphTokenFilterFactory;
 import org.elasticsearch.index.analysis.FrenchAnalyzerProvider;
 import org.elasticsearch.index.analysis.FrenchStemTokenFilterFactory;
 import org.elasticsearch.index.analysis.GalicianAnalyzerProvider;
@@ -139,6 +140,7 @@
 import org.elasticsearch.index.analysis.UpperCaseTokenFilterFactory;
 import org.elasticsearch.index.analysis.WhitespaceAnalyzerProvider;
 import org.elasticsearch.index.analysis.WhitespaceTokenizerFactory;
+import org.elasticsearch.index.analysis.WordDelimiterGraphTokenFilterFactory;
 import org.elasticsearch.index.analysis.WordDelimiterTokenFilterFactory;
 import org.elasticsearch.index.analysis.compound.DictionaryCompoundWordTokenFilterFactory;
 import org.elasticsearch.index.analysis.compound.HyphenationCompoundWordTokenFilterFactory;
@@ -152,13 +154,12 @@
  */
 public final class AnalysisModule {
     static {
-        Settings build = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
-                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
-                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
-                .build();
+        Settings build = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).put(IndexMetaData
+            .SETTING_NUMBER_OF_REPLICAS, 1).put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).build();
         IndexMetaData metaData = IndexMetaData.builder("_na_").settings(build).build();
         NA_INDEX_SETTINGS = new IndexSettings(metaData, Settings.EMPTY);
     }
+
     private static final IndexSettings NA_INDEX_SETTINGS;
 
     private final HunspellService hunspellService;
@@ -171,8 +172,9 @@ public AnalysisModule(Environment environment, List<AnalysisPlugin> plugins) thr
         NamedRegistry<AnalysisProvider<TokenFilterFactory>> tokenFilters = setupTokenFilters(plugins, hunspellService);
         NamedRegistry<AnalysisProvider<TokenizerFactory>> tokenizers = setupTokenizers(plugins);
         NamedRegistry<AnalysisProvider<AnalyzerProvider<?>>> analyzers = setupAnalyzers(plugins);
-        analysisRegistry = new AnalysisRegistry(environment, charFilters.getRegistry(), tokenFilters.getRegistry(),
-                tokenizers.getRegistry(), analyzers.getRegistry());
+        NamedRegistry<AnalysisProvider<AnalyzerProvider<?>>> normalizers = setupNormalizers(plugins);
+        analysisRegistry = new AnalysisRegistry(environment, charFilters.getRegistry(), tokenFilters.getRegistry(), tokenizers
+            .getRegistry(), analyzers.getRegistry(), normalizers.getRegistry());
     }
 
     HunspellService getHunspellService() {
@@ -198,8 +200,8 @@ public NamedRegistry<org.apache.lucene.analysis.hunspell.Dictionary> setupHunspe
         return hunspellDictionaries;
     }
 
-    private NamedRegistry<AnalysisProvider<TokenFilterFactory>> setupTokenFilters(List<AnalysisPlugin> plugins,
-            HunspellService hunspellService) {
+    private NamedRegistry<AnalysisProvider<TokenFilterFactory>> setupTokenFilters(List<AnalysisPlugin> plugins, HunspellService
+        hunspellService) {
         NamedRegistry<AnalysisProvider<TokenFilterFactory>> tokenFilters = new NamedRegistry<>("token_filter");
         tokenFilters.register("stop", StopTokenFilterFactory::new);
         tokenFilters.register("reverse", ReverseTokenFilterFactory::new);
@@ -224,8 +226,10 @@ private NamedRegistry<AnalysisProvider<TokenFilterFactory>> setupTokenFilters(Li
         tokenFilters.register("snowball", SnowballTokenFilterFactory::new);
         tokenFilters.register("stemmer", StemmerTokenFilterFactory::new);
         tokenFilters.register("word_delimiter", WordDelimiterTokenFilterFactory::new);
+        tokenFilters.register("word_delimiter_graph", WordDelimiterGraphTokenFilterFactory::new);
         tokenFilters.register("delimited_payload_filter", DelimitedPayloadTokenFilterFactory::new);
         tokenFilters.register("elision", ElisionTokenFilterFactory::new);
+        tokenFilters.register("flatten_graph", FlattenGraphTokenFilterFactory::new);
         tokenFilters.register("keep", requriesAnalysisSettings(KeepWordFilterFactory::new));
         tokenFilters.register("keep_types", requriesAnalysisSettings(KeepTypesFilterFactory::new));
         tokenFilters.register("pattern_capture", requriesAnalysisSettings(PatternCaptureGroupTokenFilterFactory::new));
@@ -251,8 +255,8 @@ private NamedRegistry<AnalysisProvider<TokenFilterFactory>> setupTokenFilters(Li
         tokenFilters.register("scandinavian_folding", ScandinavianFoldingFilterFactory::new);
         tokenFilters.register("serbian_normalization", SerbianNormalizationFilterFactory::new);
 
-        tokenFilters.register("hunspell", requriesAnalysisSettings(
-                (indexSettings, env, name, settings) -> new HunspellTokenFilterFactory(indexSettings, name, settings, hunspellService)));
+        tokenFilters.register("hunspell", requriesAnalysisSettings((indexSettings, env, name, settings) -> new HunspellTokenFilterFactory
+            (indexSettings, name, settings, hunspellService)));
         tokenFilters.register("cjk_bigram", CJKBigramFilterFactory::new);
         tokenFilters.register("cjk_width", CJKWidthFilterFactory::new);
 
@@ -335,12 +339,20 @@ private NamedRegistry<AnalysisProvider<AnalyzerProvider<?>>> setupAnalyzers(List
         return analyzers;
     }
 
+    private NamedRegistry<AnalysisProvider<AnalyzerProvider<?>>> setupNormalizers(List<AnalysisPlugin> plugins) {
+        NamedRegistry<AnalysisProvider<AnalyzerProvider<?>>> normalizers = new NamedRegistry<>("normalizer");
+        // TODO: provide built-in normalizer providers?
+        // TODO: pluggability?
+        return normalizers;
+    }
+
     private static <T> AnalysisModule.AnalysisProvider<T> requriesAnalysisSettings(AnalysisModule.AnalysisProvider<T> provider) {
         return new AnalysisModule.AnalysisProvider<T>() {
             @Override
             public T get(IndexSettings indexSettings, Environment environment, String name, Settings settings) throws IOException {
                 return provider.get(indexSettings, environment, name, settings);
             }
+
             @Override
             public boolean requiresAnalysisSettings() {
                 return true;
@@ -355,10 +367,11 @@ public interface AnalysisProvider<T> {
 
         /**
          * Creates a new analysis provider.
+         *
          * @param indexSettings the index settings for the index this provider is created for
-         * @param environment the nodes environment to load resources from persistent storage
-         * @param name the name of the analysis component
-         * @param settings the component specific settings without context prefixes
+         * @param environment   the nodes environment to load resources from persistent storage
+         * @param name          the name of the analysis component
+         * @param settings      the component specific settings without context prefixes
          * @return a new provider instance
          * @throws IOException if an {@link IOException} occurs
          */
@@ -369,11 +382,11 @@ public interface AnalysisProvider<T> {
          * This can be used to get a default instance of an analysis factory without binding to an index.
          *
          * @param environment the nodes environment to load resources from persistent storage
-         * @param name the name of the analysis component
+         * @param name        the name of the analysis component
          * @return a new provider instance
-         * @throws IOException if an {@link IOException} occurs
+         * @throws IOException              if an {@link IOException} occurs
          * @throws IllegalArgumentException if the provider requires analysis settings ie. if {@link #requiresAnalysisSettings()} returns
-         *  <code>true</code>
+         *                                  <code>true</code>
          */
         default T get(Environment environment, String name) throws IOException {
             if (requiresAnalysisSettings()) {
diff --git a/core/src/main/java/org/elasticsearch/indices/analysis/PreBuiltTokenFilters.java b/core/src/main/java/org/elasticsearch/indices/analysis/PreBuiltTokenFilters.java
index a31f60fc5bdba..cf048f61baacc 100644
--- a/core/src/main/java/org/elasticsearch/indices/analysis/PreBuiltTokenFilters.java
+++ b/core/src/main/java/org/elasticsearch/indices/analysis/PreBuiltTokenFilters.java
@@ -42,6 +42,7 @@
 import org.apache.lucene.analysis.hi.HindiNormalizationFilter;
 import org.apache.lucene.analysis.in.IndicNormalizationFilter;
 import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter;
+import org.apache.lucene.analysis.miscellaneous.DisableGraphAttribute;
 import org.apache.lucene.analysis.miscellaneous.KeywordRepeatFilter;
 import org.apache.lucene.analysis.miscellaneous.LengthFilter;
 import org.apache.lucene.analysis.miscellaneous.LimitTokenCountFilter;
@@ -51,6 +52,7 @@
 import org.apache.lucene.analysis.miscellaneous.TruncateTokenFilter;
 import org.apache.lucene.analysis.miscellaneous.UniqueTokenFilter;
 import org.apache.lucene.analysis.miscellaneous.WordDelimiterFilter;
+import org.apache.lucene.analysis.miscellaneous.WordDelimiterGraphFilter;
 import org.apache.lucene.analysis.ngram.EdgeNGramTokenFilter;
 import org.apache.lucene.analysis.ngram.NGramTokenFilter;
 import org.apache.lucene.analysis.payloads.DelimitedPayloadTokenFilter;
@@ -65,6 +67,7 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.index.analysis.DelimitedPayloadTokenFilterFactory;
 import org.elasticsearch.index.analysis.LimitTokenCountFilterFactory;
+import org.elasticsearch.index.analysis.MultiTermAwareComponent;
 import org.elasticsearch.index.analysis.TokenFilterFactory;
 import org.elasticsearch.indices.analysis.PreBuiltCacheFactory.CachingStrategy;
 import org.tartarus.snowball.ext.DutchStemmer;
@@ -89,6 +92,18 @@ public TokenStream create(TokenStream tokenStream, Version version) {
         }
     },
 
+    WORD_DELIMITER_GRAPH(CachingStrategy.ONE) {
+        @Override
+        public TokenStream create(TokenStream tokenStream, Version version) {
+            return new WordDelimiterGraphFilter(tokenStream,
+                WordDelimiterGraphFilter.GENERATE_WORD_PARTS |
+                    WordDelimiterGraphFilter.GENERATE_NUMBER_PARTS |
+                    WordDelimiterGraphFilter.SPLIT_ON_CASE_CHANGE |
+                    WordDelimiterGraphFilter.SPLIT_ON_NUMERICS |
+                    WordDelimiterGraphFilter.STEM_ENGLISH_POSSESSIVE, null);
+        }
+    },
+
     STOP(CachingStrategy.LUCENE) {
         @Override
         public TokenStream create(TokenStream tokenStream, Version version) {
@@ -115,6 +130,10 @@ public TokenStream create(TokenStream tokenStream, Version version) {
         public TokenStream create(TokenStream tokenStream, Version version) {
             return new ASCIIFoldingFilter(tokenStream);
         }
+        @Override
+        protected boolean isMultiTermAware() {
+            return true;
+        }
     },
 
     LENGTH(CachingStrategy.LUCENE) {
@@ -136,6 +155,10 @@ public TokenStream create(TokenStream tokenStream, Version version) {
         public TokenStream create(TokenStream tokenStream, Version version) {
             return new LowerCaseFilter(tokenStream);
         }
+        @Override
+        protected boolean isMultiTermAware() {
+            return true;
+        }
     },
 
     UPPERCASE(CachingStrategy.LUCENE) {
@@ -143,6 +166,10 @@ public TokenStream create(TokenStream tokenStream, Version version) {
         public TokenStream create(TokenStream tokenStream, Version version) {
             return new UpperCaseFilter(tokenStream);
         }
+        @Override
+        protected boolean isMultiTermAware() {
+            return true;
+        }
     },
 
     KSTEM(CachingStrategy.ONE) {
@@ -221,6 +248,10 @@ public TokenStream create(TokenStream tokenStream, Version version) {
         public TokenStream create(TokenStream tokenStream, Version version) {
             return new ElisionFilter(tokenStream, FrenchAnalyzer.DEFAULT_ARTICLES);
         }
+        @Override
+        protected boolean isMultiTermAware() {
+            return true;
+        }
     },
 
     ARABIC_STEM(CachingStrategy.ONE) {
@@ -284,6 +315,10 @@ public TokenStream create(TokenStream tokenStream, Version version) {
         public TokenStream create(TokenStream tokenStream, Version version) {
             return new ArabicNormalizationFilter(tokenStream);
         }
+        @Override
+        protected boolean isMultiTermAware() {
+            return true;
+        }
     },
 
     PERSIAN_NORMALIZATION(CachingStrategy.ONE) {
@@ -291,6 +326,10 @@ public TokenStream create(TokenStream tokenStream, Version version) {
         public TokenStream create(TokenStream tokenStream, Version version) {
             return new PersianNormalizationFilter(tokenStream);
         }
+        @Override
+        protected boolean isMultiTermAware() {
+            return true;
+        }
     },
 
     TYPE_AS_PAYLOAD(CachingStrategy.ONE) {
@@ -303,7 +342,15 @@ public TokenStream create(TokenStream tokenStream, Version version) {
     SHINGLE(CachingStrategy.ONE) {
         @Override
         public TokenStream create(TokenStream tokenStream, Version version) {
-            return new ShingleFilter(tokenStream);
+            final TokenStream filter = new ShingleFilter(tokenStream);
+            /**
+             * We disable the graph analysis on this token stream
+             * because it produces shingles of different size.
+             * Graph analysis on such token stream is useless and dangerous as it may create too many paths
+             * since shingles of different size are not aligned in terms of positions.
+             */
+            filter.addAttribute(DisableGraphAttribute.class);
+            return filter;
         }
     },
 
@@ -312,6 +359,10 @@ public TokenStream create(TokenStream tokenStream, Version version) {
         public TokenStream create(TokenStream tokenStream, Version version) {
             return new GermanNormalizationFilter(tokenStream);
         }
+        @Override
+        protected boolean isMultiTermAware() {
+            return true;
+        }
     },
 
     HINDI_NORMALIZATION(CachingStrategy.ONE) {
@@ -319,6 +370,10 @@ public TokenStream create(TokenStream tokenStream, Version version) {
         public TokenStream create(TokenStream tokenStream, Version version) {
             return new HindiNormalizationFilter(tokenStream);
         }
+        @Override
+        protected boolean isMultiTermAware() {
+            return true;
+        }
     },
 
     INDIC_NORMALIZATION(CachingStrategy.ONE) {
@@ -326,6 +381,10 @@ public TokenStream create(TokenStream tokenStream, Version version) {
         public TokenStream create(TokenStream tokenStream, Version version) {
             return new IndicNormalizationFilter(tokenStream);
         }
+        @Override
+        protected boolean isMultiTermAware() {
+            return true;
+        }
     },
 
     SORANI_NORMALIZATION(CachingStrategy.ONE) {
@@ -333,6 +392,10 @@ public TokenStream create(TokenStream tokenStream, Version version) {
         public TokenStream create(TokenStream tokenStream, Version version) {
             return new SoraniNormalizationFilter(tokenStream);
         }
+        @Override
+        protected boolean isMultiTermAware() {
+            return true;
+        }
     },
 
     SCANDINAVIAN_NORMALIZATION(CachingStrategy.ONE) {
@@ -340,6 +403,10 @@ public TokenStream create(TokenStream tokenStream, Version version) {
         public TokenStream create(TokenStream tokenStream, Version version) {
             return new ScandinavianNormalizationFilter(tokenStream);
         }
+        @Override
+        protected boolean isMultiTermAware() {
+            return true;
+        }
     },
 
     SCANDINAVIAN_FOLDING(CachingStrategy.ONE) {
@@ -347,6 +414,10 @@ public TokenStream create(TokenStream tokenStream, Version version) {
         public TokenStream create(TokenStream tokenStream, Version version) {
             return new ScandinavianFoldingFilter(tokenStream);
         }
+        @Override
+        protected boolean isMultiTermAware() {
+            return true;
+        }
     },
 
     APOSTROPHE(CachingStrategy.ONE) {
@@ -361,6 +432,10 @@ public TokenStream create(TokenStream tokenStream, Version version) {
         public TokenStream create(TokenStream tokenStream, Version version) {
             return new CJKWidthFilter(tokenStream);
         }
+        @Override
+        protected boolean isMultiTermAware() {
+            return true;
+        }
     },
 
     DECIMAL_DIGIT(CachingStrategy.ONE) {
@@ -368,6 +443,10 @@ public TokenStream create(TokenStream tokenStream, Version version) {
         public TokenStream create(TokenStream tokenStream, Version version) {
             return new DecimalDigitFilter(tokenStream);
         }
+        @Override
+        protected boolean isMultiTermAware() {
+            return true;
+        }
     },
 
     CJK_BIGRAM(CachingStrategy.ONE) {
@@ -389,11 +468,15 @@ public TokenStream create(TokenStream tokenStream, Version version) {
         public TokenStream create(TokenStream tokenStream, Version version) {
             return new LimitTokenCountFilter(tokenStream, LimitTokenCountFilterFactory.DEFAULT_MAX_TOKEN_COUNT, LimitTokenCountFilterFactory.DEFAULT_CONSUME_ALL_TOKENS);
         }
-    }
+    },
 
     ;
 
-    public abstract  TokenStream create(TokenStream tokenStream, Version version);
+    protected boolean isMultiTermAware() {
+        return false;
+    }
+
+    public abstract TokenStream create(TokenStream tokenStream, Version version);
 
     protected final PreBuiltCacheFactory.PreBuiltCache<TokenFilterFactory> cache;
 
@@ -402,21 +485,42 @@ public TokenStream create(TokenStream tokenStream, Version version) {
         cache = PreBuiltCacheFactory.getCache(cachingStrategy);
     }
 
+    private interface MultiTermAwareTokenFilterFactory extends TokenFilterFactory, MultiTermAwareComponent {}
+
     public synchronized TokenFilterFactory getTokenFilterFactory(final Version version) {
         TokenFilterFactory factory = cache.get(version);
         if (factory == null) {
-            final String finalName = name();
-            factory = new TokenFilterFactory() {
-                @Override
-                public String name() {
-                    return finalName.toLowerCase(Locale.ROOT);
-                }
-
-                @Override
-                public TokenStream create(TokenStream tokenStream) {
-                    return valueOf(finalName).create(tokenStream, version);
-                }
-            };
+            final String finalName = name().toLowerCase(Locale.ROOT);
+            if (isMultiTermAware()) {
+                factory = new MultiTermAwareTokenFilterFactory() {
+                    @Override
+                    public String name() {
+                        return finalName;
+                    }
+
+                    @Override
+                    public TokenStream create(TokenStream tokenStream) {
+                        return PreBuiltTokenFilters.this.create(tokenStream, version);
+                    }
+
+                    @Override
+                    public Object getMultiTermComponent() {
+                        return this;
+                    }
+                };
+            } else {
+                factory = new TokenFilterFactory() {
+                    @Override
+                    public String name() {
+                        return finalName;
+                    }
+
+                    @Override
+                    public TokenStream create(TokenStream tokenStream) {
+                        return PreBuiltTokenFilters.this.create(tokenStream, version);
+                    }
+                };
+            }
             cache.put(version, factory);
         }
 
diff --git a/core/src/main/java/org/elasticsearch/indices/analysis/PreBuiltTokenizers.java b/core/src/main/java/org/elasticsearch/indices/analysis/PreBuiltTokenizers.java
index 424b5e4534f29..a979e8f1eb8a6 100644
--- a/core/src/main/java/org/elasticsearch/indices/analysis/PreBuiltTokenizers.java
+++ b/core/src/main/java/org/elasticsearch/indices/analysis/PreBuiltTokenizers.java
@@ -33,6 +33,8 @@
 import org.apache.lucene.analysis.th.ThaiTokenizer;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.regex.Regex;
+import org.elasticsearch.index.analysis.MultiTermAwareComponent;
+import org.elasticsearch.index.analysis.TokenFilterFactory;
 import org.elasticsearch.index.analysis.TokenizerFactory;
 import org.elasticsearch.indices.analysis.PreBuiltCacheFactory.CachingStrategy;
 
@@ -90,6 +92,10 @@ protected Tokenizer create(Version version) {
         protected Tokenizer create(Version version) {
             return new LowerCaseTokenizer();
         }
+        @Override
+        protected TokenFilterFactory getMultiTermComponent(Version version) {
+            return PreBuiltTokenFilters.LOWERCASE.getTokenFilterFactory(version);
+        }
     },
 
     WHITESPACE(CachingStrategy.LUCENE) {
@@ -131,6 +137,10 @@ protected Tokenizer create(Version version) {
 
     protected abstract  Tokenizer create(Version version);
 
+    protected TokenFilterFactory getMultiTermComponent(Version version) {
+        return null;
+    }
+
     protected final PreBuiltCacheFactory.PreBuiltCache<TokenizerFactory> cache;
 
 
@@ -138,22 +148,42 @@ protected Tokenizer create(Version version) {
         cache = PreBuiltCacheFactory.getCache(cachingStrategy);
     }
 
+    private interface MultiTermAwareTokenizerFactory extends TokenizerFactory, MultiTermAwareComponent {}
+
     public synchronized TokenizerFactory getTokenizerFactory(final Version version) {
         TokenizerFactory tokenizerFactory = cache.get(version);
         if (tokenizerFactory == null) {
-            final String finalName = name();
-
-            tokenizerFactory = new TokenizerFactory() {
-                @Override
-                public String name() {
-                    return finalName.toLowerCase(Locale.ROOT);
-                }
-
-                @Override
-                public Tokenizer create() {
-                    return valueOf(finalName).create(version);
-                }
-            };
+            final String finalName = name().toLowerCase(Locale.ROOT);
+            if (getMultiTermComponent(version) != null) {
+                tokenizerFactory = new MultiTermAwareTokenizerFactory() {
+                    @Override
+                    public String name() {
+                        return finalName;
+                    }
+
+                    @Override
+                    public Tokenizer create() {
+                        return PreBuiltTokenizers.this.create(version);
+                    }
+
+                    @Override
+                    public Object getMultiTermComponent() {
+                        return PreBuiltTokenizers.this.getMultiTermComponent(version);
+                    }
+                };
+            } else {
+                tokenizerFactory = new TokenizerFactory() {
+                    @Override
+                    public String name() {
+                        return finalName;
+                    }
+
+                    @Override
+                    public Tokenizer create() {
+                        return PreBuiltTokenizers.this.create(version);
+                    }
+                };
+            }
             cache.put(version, tokenizerFactory);
         }
 
diff --git a/core/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java b/core/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java
index 715cf47a6efb4..b8ec92ba15374 100644
--- a/core/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java
+++ b/core/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java
@@ -81,25 +81,25 @@ public class HierarchyCircuitBreakerService extends CircuitBreakerService {
     public HierarchyCircuitBreakerService(Settings settings, ClusterSettings clusterSettings) {
         super(settings);
         this.fielddataSettings = new BreakerSettings(CircuitBreaker.FIELDDATA,
-                FIELDDATA_CIRCUIT_BREAKER_LIMIT_SETTING.get(settings).bytes(),
+                FIELDDATA_CIRCUIT_BREAKER_LIMIT_SETTING.get(settings).getBytes(),
                 FIELDDATA_CIRCUIT_BREAKER_OVERHEAD_SETTING.get(settings),
                 FIELDDATA_CIRCUIT_BREAKER_TYPE_SETTING.get(settings)
         );
 
         this.inFlightRequestsSettings = new BreakerSettings(CircuitBreaker.IN_FLIGHT_REQUESTS,
-                IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_LIMIT_SETTING.get(settings).bytes(),
+                IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_LIMIT_SETTING.get(settings).getBytes(),
                 IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_OVERHEAD_SETTING.get(settings),
                 IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_TYPE_SETTING.get(settings)
         );
 
         this.requestSettings = new BreakerSettings(CircuitBreaker.REQUEST,
-                REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING.get(settings).bytes(),
+                REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING.get(settings).getBytes(),
                 REQUEST_CIRCUIT_BREAKER_OVERHEAD_SETTING.get(settings),
                 REQUEST_CIRCUIT_BREAKER_TYPE_SETTING.get(settings)
         );
 
         this.parentSettings = new BreakerSettings(CircuitBreaker.PARENT,
-                TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING.get(settings).bytes(), 1.0,
+                TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING.get(settings).getBytes(), 1.0,
                 CircuitBreaker.Type.PARENT);
 
         if (logger.isTraceEnabled()) {
@@ -117,7 +117,7 @@ public HierarchyCircuitBreakerService(Settings settings, ClusterSettings cluster
     }
 
     private void setRequestBreakerLimit(ByteSizeValue newRequestMax, Double newRequestOverhead) {
-        BreakerSettings newRequestSettings = new BreakerSettings(CircuitBreaker.REQUEST, newRequestMax.bytes(), newRequestOverhead,
+        BreakerSettings newRequestSettings = new BreakerSettings(CircuitBreaker.REQUEST, newRequestMax.getBytes(), newRequestOverhead,
                 HierarchyCircuitBreakerService.this.requestSettings.getType());
         registerBreaker(newRequestSettings);
         HierarchyCircuitBreakerService.this.requestSettings = newRequestSettings;
@@ -125,7 +125,7 @@ private void setRequestBreakerLimit(ByteSizeValue newRequestMax, Double newReque
     }
 
     private void setInFlightRequestsBreakerLimit(ByteSizeValue newInFlightRequestsMax, Double newInFlightRequestsOverhead) {
-        BreakerSettings newInFlightRequestsSettings = new BreakerSettings(CircuitBreaker.IN_FLIGHT_REQUESTS, newInFlightRequestsMax.bytes(),
+        BreakerSettings newInFlightRequestsSettings = new BreakerSettings(CircuitBreaker.IN_FLIGHT_REQUESTS, newInFlightRequestsMax.getBytes(),
             newInFlightRequestsOverhead, HierarchyCircuitBreakerService.this.inFlightRequestsSettings.getType());
         registerBreaker(newInFlightRequestsSettings);
         HierarchyCircuitBreakerService.this.inFlightRequestsSettings = newInFlightRequestsSettings;
@@ -133,7 +133,7 @@ private void setInFlightRequestsBreakerLimit(ByteSizeValue newInFlightRequestsMa
     }
 
     private void setFieldDataBreakerLimit(ByteSizeValue newFielddataMax, Double newFielddataOverhead) {
-        long newFielddataLimitBytes = newFielddataMax == null ? HierarchyCircuitBreakerService.this.fielddataSettings.getLimit() : newFielddataMax.bytes();
+        long newFielddataLimitBytes = newFielddataMax == null ? HierarchyCircuitBreakerService.this.fielddataSettings.getLimit() : newFielddataMax.getBytes();
         newFielddataOverhead = newFielddataOverhead == null ? HierarchyCircuitBreakerService.this.fielddataSettings.getOverhead() : newFielddataOverhead;
         BreakerSettings newFielddataSettings = new BreakerSettings(CircuitBreaker.FIELDDATA, newFielddataLimitBytes, newFielddataOverhead,
                 HierarchyCircuitBreakerService.this.fielddataSettings.getType());
@@ -143,13 +143,13 @@ private void setFieldDataBreakerLimit(ByteSizeValue newFielddataMax, Double newF
     }
 
     private boolean validateTotalCircuitBreakerLimit(ByteSizeValue byteSizeValue) {
-        BreakerSettings newParentSettings = new BreakerSettings(CircuitBreaker.PARENT, byteSizeValue.bytes(), 1.0, CircuitBreaker.Type.PARENT);
+        BreakerSettings newParentSettings = new BreakerSettings(CircuitBreaker.PARENT, byteSizeValue.getBytes(), 1.0, CircuitBreaker.Type.PARENT);
         validateSettings(new BreakerSettings[]{newParentSettings});
         return true;
     }
 
     private void setTotalCircuitBreakerLimit(ByteSizeValue byteSizeValue) {
-        BreakerSettings newParentSettings = new BreakerSettings(CircuitBreaker.PARENT, byteSizeValue.bytes(), 1.0, CircuitBreaker.Type.PARENT);
+        BreakerSettings newParentSettings = new BreakerSettings(CircuitBreaker.PARENT, byteSizeValue.getBytes(), 1.0, CircuitBreaker.Type.PARENT);
         this.parentSettings = newParentSettings;
     }
 
@@ -177,7 +177,7 @@ public CircuitBreaker getBreaker(String name) {
     @Override
     public AllCircuitBreakerStats stats() {
         long parentEstimated = 0;
-        List<CircuitBreakerStats> allStats = new ArrayList<>();
+        List<CircuitBreakerStats> allStats = new ArrayList<>(this.breakers.size());
         // Gather the "estimated" count for the parent breaker by adding the
         // estimations for each individual breaker
         for (CircuitBreaker breaker : this.breakers.values()) {
@@ -208,10 +208,11 @@ public void checkParentLimit(String label) throws CircuitBreakingException {
         long parentLimit = this.parentSettings.getLimit();
         if (totalUsed > parentLimit) {
             this.parentTripCount.incrementAndGet();
-            throw new CircuitBreakingException("[parent] Data too large, data for [" +
-                            label + "] would be larger than limit of [" +
-                            parentLimit + "/" + new ByteSizeValue(parentLimit) + "]",
-                    totalUsed, parentLimit);
+            final String message = "[parent] Data too large, data for [" + label + "]" +
+                    " would be [" + totalUsed + "/" + new ByteSizeValue(totalUsed) + "]" +
+                    ", which is larger than the limit of [" +
+                    parentLimit + "/" + new ByteSizeValue(parentLimit) + "]";
+            throw new CircuitBreakingException(message, totalUsed, parentLimit);
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java b/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
index e766f6ecefb64..76aff15a34e64 100644
--- a/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
+++ b/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
@@ -23,9 +23,10 @@
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
 import org.apache.lucene.store.LockObtainFailedException;
+import org.elasticsearch.ResourceAlreadyExistsException;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.ClusterStateListener;
+import org.elasticsearch.cluster.ClusterStateApplier;
 import org.elasticsearch.cluster.action.index.NodeMappingRefreshAction;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
@@ -43,7 +44,6 @@
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.util.Callback;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.env.ShardLockObtainFailedException;
@@ -52,15 +52,12 @@
 import org.elasticsearch.index.IndexComponent;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.IndexShardAlreadyExistsException;
-import org.elasticsearch.index.NodeServicesProvider;
 import org.elasticsearch.index.shard.IndexEventListener;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.IndexShardRelocatedException;
 import org.elasticsearch.index.shard.IndexShardState;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.ShardNotFoundException;
-import org.elasticsearch.indices.IndexAlreadyExistsException;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.flush.SyncedFlushService;
 import org.elasticsearch.indices.recovery.PeerRecoverySourceService;
@@ -70,6 +67,7 @@
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.search.SearchService;
 import org.elasticsearch.snapshots.RestoreService;
+import org.elasticsearch.snapshots.SnapshotShardsService;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.IOException;
@@ -83,8 +81,14 @@
 import java.util.Set;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
 
-public class IndicesClusterStateService extends AbstractLifecycleComponent implements ClusterStateListener {
+import static org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.CLOSED;
+import static org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.DELETED;
+import static org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.FAILURE;
+import static org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.NO_LONGER_ASSIGNED;
+
+public class IndicesClusterStateService extends AbstractLifecycleComponent implements ClusterStateApplier {
 
     final AllocatedIndices<? extends Shard, ? extends AllocatedIndex<? extends Shard>> indicesService;
     private final ClusterService clusterService;
@@ -92,7 +96,6 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent imple
     private final PeerRecoveryTargetService recoveryTargetService;
     private final ShardStateAction shardStateAction;
     private final NodeMappingRefreshAction nodeMappingRefreshAction;
-    private final NodeServicesProvider nodeServicesProvider;
 
     private static final ShardStateAction.Listener SHARD_STATE_ACTION_LISTENER = new ShardStateAction.Listener() {
     };
@@ -115,11 +118,11 @@ public IndicesClusterStateService(Settings settings, IndicesService indicesServi
                                       NodeMappingRefreshAction nodeMappingRefreshAction,
                                       RepositoriesService repositoriesService, RestoreService restoreService,
                                       SearchService searchService, SyncedFlushService syncedFlushService,
-                                      PeerRecoverySourceService peerRecoverySourceService, NodeServicesProvider nodeServicesProvider) {
+                                      PeerRecoverySourceService peerRecoverySourceService, SnapshotShardsService snapshotShardsService) {
         this(settings, (AllocatedIndices<? extends Shard, ? extends AllocatedIndex<? extends Shard>>) indicesService,
             clusterService, threadPool, recoveryTargetService, shardStateAction,
             nodeMappingRefreshAction, repositoriesService, restoreService, searchService, syncedFlushService, peerRecoverySourceService,
-            nodeServicesProvider);
+            snapshotShardsService);
     }
 
     // for tests
@@ -131,9 +134,10 @@ public IndicesClusterStateService(Settings settings, IndicesService indicesServi
                                NodeMappingRefreshAction nodeMappingRefreshAction,
                                RepositoriesService repositoriesService, RestoreService restoreService,
                                SearchService searchService, SyncedFlushService syncedFlushService,
-                               PeerRecoverySourceService peerRecoverySourceService, NodeServicesProvider nodeServicesProvider) {
+                               PeerRecoverySourceService peerRecoverySourceService, SnapshotShardsService snapshotShardsService) {
         super(settings);
-        this.buildInIndexListener = Arrays.asList(peerRecoverySourceService, recoveryTargetService, searchService, syncedFlushService);
+        this.buildInIndexListener = Arrays.asList(peerRecoverySourceService, recoveryTargetService, searchService, syncedFlushService,
+                                                  snapshotShardsService);
         this.indicesService = indicesService;
         this.clusterService = clusterService;
         this.threadPool = threadPool;
@@ -143,17 +147,21 @@ public IndicesClusterStateService(Settings settings, IndicesService indicesServi
         this.restoreService = restoreService;
         this.repositoriesService = repositoriesService;
         this.sendRefreshMapping = this.settings.getAsBoolean("indices.cluster.send_refresh_mapping", true);
-        this.nodeServicesProvider = nodeServicesProvider;
     }
 
     @Override
     protected void doStart() {
-        clusterService.addFirst(this);
+        // Doesn't make sense to manage shards on non-master and non-data nodes
+        if (DiscoveryNode.isDataNode(settings) || DiscoveryNode.isMasterNode(settings)) {
+            clusterService.addHighPriorityApplier(this);
+        }
     }
 
     @Override
     protected void doStop() {
-        clusterService.remove(this);
+        if (DiscoveryNode.isDataNode(settings) || DiscoveryNode.isMasterNode(settings)) {
+            clusterService.removeApplier(this);
+        }
     }
 
     @Override
@@ -161,7 +169,7 @@ protected void doClose() {
     }
 
     @Override
-    public synchronized void clusterChanged(final ClusterChangedEvent event) {
+    public synchronized void applyClusterState(final ClusterChangedEvent event) {
         if (!lifecycle.started()) {
             return;
         }
@@ -173,7 +181,8 @@ public synchronized void clusterChanged(final ClusterChangedEvent event) {
         // TODO: feels hacky, a block disables state persistence, and then we clean the allocated shards, maybe another flag in blocks?
         if (state.blocks().disableStatePersistence()) {
             for (AllocatedIndex<? extends Shard> indexService : indicesService) {
-                indicesService.removeIndex(indexService.index(), "cleaning index (disabled block persistence)"); // also cleans shards
+                indicesService.removeIndex(indexService.index(), NO_LONGER_ASSIGNED,
+                    "cleaning index (disabled block persistence)"); // also cleans shards
             }
             return;
         }
@@ -186,7 +195,7 @@ public synchronized void clusterChanged(final ClusterChangedEvent event) {
 
         failMissingShards(state);
 
-        removeShards(state);
+        removeShards(state);   // removes any local shards that doesn't match what the master expects
 
         updateIndices(event); // can also fail shards, but these are then guaranteed to be in failedShardsCache
 
@@ -221,7 +230,7 @@ private void updateFailedShardsCache(final ClusterState state) {
                 if (masterNode != null) { // TODO: can we remove this? Is resending shard failures the responsibility of shardStateAction?
                     String message = "master " + masterNode + " has not removed previously failed shard. resending shard failure";
                     logger.trace("[{}] re-sending failed shard [{}], reason [{}]", matchedRouting.shardId(), matchedRouting, message);
-                    shardStateAction.localShardFailed(matchedRouting, message, null, SHARD_STATE_ACTION_LISTENER);
+                    shardStateAction.localShardFailed(matchedRouting, message, null, SHARD_STATE_ACTION_LISTENER, state);
                 }
             }
         }
@@ -246,7 +255,7 @@ private void deleteIndices(final ClusterChangedEvent event) {
             final IndexSettings indexSettings;
             if (indexService != null) {
                 indexSettings = indexService.getIndexSettings();
-                indicesService.deleteIndex(index, "index no longer part of the metadata");
+                indicesService.removeIndex(index, DELETED, "index no longer part of the metadata");
             } else if (previousState.metaData().hasIndex(index.getName())) {
                 // The deleted index was part of the previous cluster state, but not loaded on the local node
                 final IndexMetaData metaData = previousState.metaData().index(index);
@@ -320,11 +329,14 @@ private void removeUnallocatedIndices(final ClusterChangedEvent event) {
                 // to remove the in-memory structures for the index and not delete the
                 // contents on disk because the index will later be re-imported as a
                 // dangling index
-                assert state.metaData().index(index) != null || event.isNewCluster() :
+                final IndexMetaData indexMetaData = state.metaData().index(index);
+                assert indexMetaData != null || event.isNewCluster() :
                     "index " + index + " does not exist in the cluster state, it should either " +
-                    "have been deleted or the cluster must be new";
-                logger.debug("{} removing index, no shards allocated", index);
-                indicesService.removeIndex(index, "removing index (no shards allocated)");
+                        "have been deleted or the cluster must be new";
+                final AllocatedIndices.IndexRemovalReason reason =
+                    indexMetaData != null && indexMetaData.getState() == IndexMetaData.State.CLOSE ? CLOSED : NO_LONGER_ASSIGNED;
+                logger.debug("{} removing index, [{}]", index, reason);
+                indicesService.removeIndex(index, reason, "removing index (no shards allocated)");
             }
         }
     }
@@ -345,7 +357,8 @@ private void failMissingShards(final ClusterState state) {
                 failedShardsCache.containsKey(shardId) == false &&
                 indicesService.getShardOrNull(shardId) == null) {
                 // the master thinks we are active, but we don't have this shard at all, mark it as failed
-                sendFailShard(shardRouting, "master marked shard as active, but shard has not been created, mark shard as failed", null);
+                sendFailShard(shardRouting, "master marked shard as active, but shard has not been created, mark shard as failed", null,
+                    state);
             }
         }
     }
@@ -370,11 +383,21 @@ private void removeShards(final ClusterState state) {
                 ShardRouting currentRoutingEntry = shard.routingEntry();
                 ShardId shardId = currentRoutingEntry.shardId();
                 ShardRouting newShardRouting = localRoutingNode == null ? null : localRoutingNode.getByShardId(shardId);
-                if (newShardRouting == null || newShardRouting.isSameAllocation(currentRoutingEntry) == false) {
+                if (newShardRouting == null) {
                     // we can just remove the shard without cleaning it locally, since we will clean it in IndicesStore
                     // once all shards are allocated
                     logger.debug("{} removing shard (not allocated)", shardId);
                     indexService.removeShard(shardId.id(), "removing shard (not allocated)");
+                } else if (newShardRouting.isSameAllocation(currentRoutingEntry) == false) {
+                    logger.debug("{} removing shard (stale allocation id, stale {}, new {})", shardId,
+                        currentRoutingEntry, newShardRouting);
+                    indexService.removeShard(shardId.id(), "removing shard (stale copy)");
+                } else if (newShardRouting.initializing() && currentRoutingEntry.active()) {
+                    // this can happen if the node was isolated/gc-ed, rejoins the cluster and a new shard with the same allocation id
+                    // is assigned to it. Batch cluster state processing or if shard fetching completes before the node gets a new cluster
+                    // state may result in a new shard being initialized while having the same allocation id as the currently started shard.
+                    logger.debug("{} removing shard (not active, current {}, new {})", shardId, currentRoutingEntry, newShardRouting);
+                    indexService.removeShard(shardId.id(), "removing shard (stale copy)");
                 } else {
                     // remove shards where recovery source has changed. This re-initializes shards later in createOrUpdateShards
                     if (newShardRouting.recoverySource() != null && newShardRouting.recoverySource().getType() == Type.PEER) {
@@ -418,7 +441,7 @@ private void createIndices(final ClusterState state) {
 
             AllocatedIndex<? extends Shard> indexService = null;
             try {
-                indexService = indicesService.createIndex(nodeServicesProvider, indexMetaData, buildInIndexListener);
+                indexService = indicesService.createIndex(indexMetaData, buildInIndexListener);
                 if (indexService.updateMapping(indexMetaData) && sendRefreshMapping) {
                     nodeMappingRefreshAction.nodeMappingRefresh(state.nodes().getMasterNode(),
                         new NodeMappingRefreshAction.NodeMappingRefreshRequest(indexMetaData.getIndex().getName(),
@@ -431,10 +454,10 @@ private void createIndices(final ClusterState state) {
                     failShardReason = "failed to create index";
                 } else {
                     failShardReason = "failed to update mapping for index";
-                    indicesService.removeIndex(index, "removing index (mapping update failed)");
+                    indicesService.removeIndex(index, FAILURE, "removing index (mapping update failed)");
                 }
                 for (ShardRouting shardRouting : entry.getValue()) {
-                    sendFailShard(shardRouting, failShardReason, e);
+                    sendFailShard(shardRouting, failShardReason, e, state);
                 }
             }
         }
@@ -460,14 +483,14 @@ private void updateIndices(ClusterChangedEvent event) {
                         );
                     }
                 } catch (Exception e) {
-                    indicesService.removeIndex(indexService.index(), "removing index (mapping update failed)");
+                    indicesService.removeIndex(indexService.index(), FAILURE, "removing index (mapping update failed)");
 
                     // fail shards that would be created or updated by createOrUpdateShards
                     RoutingNode localRoutingNode = state.getRoutingNodes().node(state.nodes().getLocalNodeId());
                     if (localRoutingNode != null) {
                         for (final ShardRouting shardRouting : localRoutingNode) {
                             if (shardRouting.index().equals(index) && failedShardsCache.containsKey(shardRouting.shardId()) == false) {
-                                sendFailShard(shardRouting, "failed to update mapping for index", e);
+                                sendFailShard(shardRouting, "failed to update mapping for index", e, state);
                             }
                         }
                     }
@@ -493,15 +516,15 @@ private void createOrUpdateShards(final ClusterState state) {
                 Shard shard = indexService.getShardOrNull(shardId.id());
                 if (shard == null) {
                     assert shardRouting.initializing() : shardRouting + " should have been removed by failMissingShards";
-                    createShard(nodes, routingTable, shardRouting);
+                    createShard(nodes, routingTable, shardRouting, state);
                 } else {
-                    updateShard(nodes, shardRouting, shard);
+                    updateShard(nodes, shardRouting, shard, routingTable, state);
                 }
             }
         }
     }
 
-    private void createShard(DiscoveryNodes nodes, RoutingTable routingTable, ShardRouting shardRouting) {
+    private void createShard(DiscoveryNodes nodes, RoutingTable routingTable, ShardRouting shardRouting, ClusterState state) {
         assert shardRouting.initializing() : "only allow shard creation for initializing shard but was " + shardRouting;
 
         DiscoveryNode sourceNode = null;
@@ -517,17 +540,14 @@ private void createShard(DiscoveryNodes nodes, RoutingTable routingTable, ShardR
             logger.debug("{} creating shard", shardRouting.shardId());
             RecoveryState recoveryState = new RecoveryState(shardRouting, nodes.getLocalNode(), sourceNode);
             indicesService.createShard(shardRouting, recoveryState, recoveryTargetService, new RecoveryListener(shardRouting),
-                repositoriesService, nodeServicesProvider, failedShardHandler);
-        } catch (IndexShardAlreadyExistsException e) {
-            // ignore this, the method call can happen several times
-            logger.debug("Trying to create shard that already exists", e);
-            assert false;
+                repositoriesService, failedShardHandler);
         } catch (Exception e) {
-            failAndRemoveShard(shardRouting, true, "failed to create shard", e);
+            failAndRemoveShard(shardRouting, true, "failed to create shard", e, state);
         }
     }
 
-    private void updateShard(DiscoveryNodes nodes, ShardRouting shardRouting, Shard shard) {
+    private void updateShard(DiscoveryNodes nodes, ShardRouting shardRouting, Shard shard, RoutingTable routingTable,
+                             ClusterState clusterState) {
         final ShardRouting currentRoutingEntry = shard.routingEntry();
         assert currentRoutingEntry.isSameAllocation(shardRouting) :
             "local shard has a different allocation id but wasn't cleaning by removeShards. "
@@ -536,7 +556,7 @@ private void updateShard(DiscoveryNodes nodes, ShardRouting shardRouting, Shard
         try {
             shard.updateRoutingEntry(shardRouting);
         } catch (Exception e) {
-            failAndRemoveShard(shardRouting, true, "failed updating shard routing entry", e);
+            failAndRemoveShard(shardRouting, true, "failed updating shard routing entry", e, clusterState);
             return;
         }
 
@@ -551,16 +571,15 @@ private void updateShard(DiscoveryNodes nodes, ShardRouting shardRouting, Shard
             }
             if (nodes.getMasterNode() != null) {
                 shardStateAction.shardStarted(shardRouting, "master " + nodes.getMasterNode() +
-                    " marked shard as initializing, but shard state is [" + state + "], mark shard as started",
-                    SHARD_STATE_ACTION_LISTENER);
+                        " marked shard as initializing, but shard state is [" + state + "], mark shard as started",
+                    SHARD_STATE_ACTION_LISTENER, clusterState);
             }
         }
     }
 
     /**
      * Finds the routing source node for peer recovery, return null if its not found. Note, this method expects the shard
-     * routing to *require* peer recovery, use {@link ShardRouting#recoverySource()} to
-     * check if its needed or not.
+     * routing to *require* peer recovery, use {@link ShardRouting#recoverySource()} to check if its needed or not.
      */
     private static DiscoveryNode findSourceNodeForPeerRecovery(Logger logger, RoutingTable routingTable, DiscoveryNodes nodes,
                                                                ShardRouting shardRouting) {
@@ -626,10 +645,11 @@ public void onRecoveryFailure(RecoveryState state, RecoveryFailedException e, bo
     }
 
     private synchronized void handleRecoveryFailure(ShardRouting shardRouting, boolean sendShardFailure, Exception failure) {
-        failAndRemoveShard(shardRouting, sendShardFailure, "failed recovery", failure);
+        failAndRemoveShard(shardRouting, sendShardFailure, "failed recovery", failure, clusterService.state());
     }
 
-    private void failAndRemoveShard(ShardRouting shardRouting, boolean sendShardFailure, String message, @Nullable Exception failure) {
+    private void failAndRemoveShard(ShardRouting shardRouting, boolean sendShardFailure, String message, @Nullable Exception failure,
+                                    ClusterState state) {
         try {
             AllocatedIndex<? extends Shard> indexService = indicesService.indexService(shardRouting.shardId().getIndex());
             if (indexService != null) {
@@ -648,17 +668,17 @@ private void failAndRemoveShard(ShardRouting shardRouting, boolean sendShardFail
                 inner);
         }
         if (sendShardFailure) {
-            sendFailShard(shardRouting, message, failure);
+            sendFailShard(shardRouting, message, failure, state);
         }
     }
 
-    private void sendFailShard(ShardRouting shardRouting, String message, @Nullable Exception failure) {
+    private void sendFailShard(ShardRouting shardRouting, String message, @Nullable Exception failure, ClusterState state) {
         try {
             logger.warn(
                 (Supplier<?>) () -> new ParameterizedMessage(
                     "[{}] marking and sending shard failed due to [{}]", shardRouting.shardId(), message), failure);
             failedShardsCache.put(shardRouting.shardId(), shardRouting);
-            shardStateAction.localShardFailed(shardRouting, message, failure, SHARD_STATE_ACTION_LISTENER);
+            shardStateAction.localShardFailed(shardRouting, message, failure, SHARD_STATE_ACTION_LISTENER, state);
         } catch (Exception inner) {
             if (failure != null) inner.addSuppressed(failure);
             logger.warn(
@@ -671,13 +691,14 @@ private void sendFailShard(ShardRouting shardRouting, String message, @Nullable
         }
     }
 
-    private class FailedShardHandler implements Callback<IndexShard.ShardFailure> {
+    private class FailedShardHandler implements Consumer<IndexShard.ShardFailure> {
         @Override
-        public void handle(final IndexShard.ShardFailure shardFailure) {
+        public void accept(final IndexShard.ShardFailure shardFailure) {
             final ShardRouting shardRouting = shardFailure.routing;
             threadPool.generic().execute(() -> {
                 synchronized (IndicesClusterStateService.this) {
-                    failAndRemoveShard(shardRouting, true, "shard failure, reason [" + shardFailure.reason + "]", shardFailure.cause);
+                    failAndRemoveShard(shardRouting, true, "shard failure, reason [" + shardFailure.reason + "]", shardFailure.cause,
+                        clusterService.state());
                 }
             });
         }
@@ -750,10 +771,9 @@ public interface AllocatedIndices<T extends Shard, U extends AllocatedIndex<T>>
          * @param indexMetaData the index metadata to create the index for
          * @param builtInIndexListener a list of built-in lifecycle {@link IndexEventListener} that should should be used along side with
          *                             the per-index listeners
-         * @throws IndexAlreadyExistsException if the index already exists.
+         * @throws ResourceAlreadyExistsException if the index already exists.
          */
-        U createIndex(NodeServicesProvider nodeServicesProvider, IndexMetaData indexMetaData,
-                         List<IndexEventListener> builtInIndexListener) throws IOException;
+        U createIndex(IndexMetaData indexMetaData, List<IndexEventListener> builtInIndexListener) throws IOException;
 
         /**
          * Verify that the contents on disk for the given index is deleted; if not, delete the contents.
@@ -765,20 +785,10 @@ U createIndex(NodeServicesProvider nodeServicesProvider, IndexMetaData indexMeta
          */
         IndexMetaData verifyIndexIsDeleted(Index index, ClusterState clusterState);
 
-        /**
-         * Deletes the given index. Persistent parts of the index
-         * like the shards files, state and transaction logs are removed once all resources are released.
-         *
-         * Equivalent to {@link #removeIndex(Index, String)} but fires
-         * different lifecycle events to ensure pending resources of this index are immediately removed.
-         * @param index the index to delete
-         * @param reason the high level reason causing this delete
-         */
-        void deleteIndex(Index index, String reason);
 
         /**
          * Deletes an index that is not assigned to this node. This method cleans up all disk folders relating to the index
-         * but does not deal with in-memory structures. For those call {@link #deleteIndex(Index, String)}
+         * but does not deal with in-memory structures. For those call {@link #removeIndex(Index, IndexRemovalReason, String)}
          */
         void deleteUnassignedIndex(String reason, IndexMetaData metaData, ClusterState clusterState);
 
@@ -786,9 +796,10 @@ U createIndex(NodeServicesProvider nodeServicesProvider, IndexMetaData indexMeta
          * Removes the given index from this service and releases all associated resources. Persistent parts of the index
          * like the shards files, state and transaction logs are kept around in the case of a disaster recovery.
          * @param index the index to remove
-         * @param reason  the high level reason causing this removal
+         * @param reason the reason to remove the index
+         * @param extraInfo extra information that will be used for logging and reporting
          */
-        void removeIndex(Index index, String reason);
+        void removeIndex(Index index, IndexRemovalReason reason, String extraInfo);
 
         /**
          * Returns an IndexService for the specified index if exists otherwise returns <code>null</code>.
@@ -800,7 +811,7 @@ U createIndex(NodeServicesProvider nodeServicesProvider, IndexMetaData indexMeta
          */
         T createShard(ShardRouting shardRouting, RecoveryState recoveryState, PeerRecoveryTargetService recoveryTargetService,
                       PeerRecoveryTargetService.RecoveryListener recoveryListener, RepositoriesService repositoriesService,
-                      NodeServicesProvider nodeServicesProvider, Callback<IndexShard.ShardFailure> onShardFailure) throws IOException;
+                      Consumer<IndexShard.ShardFailure> onShardFailure) throws IOException;
 
         /**
          * Returns shard for the specified id if it exists otherwise returns <code>null</code>.
@@ -814,6 +825,33 @@ default T getShardOrNull(ShardId shardId) {
         }
 
         void processPendingDeletes(Index index, IndexSettings indexSettings, TimeValue timeValue)
-                throws IOException, InterruptedException, ShardLockObtainFailedException;
+            throws IOException, InterruptedException, ShardLockObtainFailedException;
+
+        enum IndexRemovalReason {
+            /**
+             * Shard of this index were previously assigned to this node but all shards have been relocated.
+             * The index should be removed and all associated resources released. Persistent parts of the index
+             * like the shards files, state and transaction logs are kept around in the case of a disaster recovery.
+             */
+            NO_LONGER_ASSIGNED,
+            /**
+             * The index is deleted. Persistent parts of the index  like the shards files, state and transaction logs are removed once
+             * all resources are released.
+             */
+            DELETED,
+
+            /**
+             * The index have been closed. The index should be removed and all associated resources released. Persistent parts of the index
+             * like the shards files, state and transaction logs are kept around in the case of a disaster recovery.
+             */
+            CLOSED,
+
+            /**
+             * Something around index management has failed and the index should be removed.
+             * Persistent parts of the index like the shards files, state and transaction logs are kept around in the
+             * case of a disaster recovery.
+             */
+            FAILURE
+        }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCache.java b/core/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCache.java
index c5a8be9b11441..3d4676f39cea0 100644
--- a/core/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCache.java
+++ b/core/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCache.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.indices.fielddata.cache;
 
+import java.util.Collections;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexReader;
@@ -58,7 +59,7 @@ public class IndicesFieldDataCache extends AbstractComponent implements RemovalL
     public IndicesFieldDataCache(Settings settings, IndexFieldDataCache.Listener indicesFieldDataCacheListener) {
         super(settings);
         this.indicesFieldDataCacheListener = indicesFieldDataCacheListener;
-        final long sizeInBytes = INDICES_FIELDDATA_CACHE_SIZE_KEY.get(settings).bytes();
+        final long sizeInBytes = INDICES_FIELDDATA_CACHE_SIZE_KEY.get(settings).getBytes();
         CacheBuilder<Key, Accountable> cacheBuilder = CacheBuilder.<Key, Accountable>builder()
                 .removalListener(this);
         if (sizeInBytes > 0) {
@@ -129,9 +130,8 @@ public <FD extends AtomicFieldData, IFD extends IndexFieldData<FD>> FD load(fina
             //noinspection unchecked
             final Accountable accountable = cache.computeIfAbsent(key, k -> {
                 context.reader().addCoreClosedListener(IndexFieldCache.this);
-                for (Listener listener : this.listeners) {
-                    k.listeners.add(listener);
-                }
+                Collections.addAll(k.listeners, this.listeners);
+                
                 final AtomicFieldData fieldData = indexFieldData.loadDirect(context);
                 for (Listener listener : k.listeners) {
                     try {
@@ -153,9 +153,7 @@ public <FD extends AtomicFieldData, IFD extends IndexFieldData.Global<FD>> IFD l
             //noinspection unchecked
             final Accountable accountable = cache.computeIfAbsent(key, k -> {
                 ElasticsearchDirectoryReader.addReaderCloseListener(indexReader, IndexFieldCache.this);
-                for (Listener listener : this.listeners) {
-                    k.listeners.add(listener);
-                }
+                Collections.addAll(k.listeners, this.listeners);
                 final Accountable ifd = (Accountable) indexFieldData.localGlobalDirect(indexReader);
                 for (Listener listener : k.listeners) {
                     try {
diff --git a/core/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java b/core/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java
index 273682db3125b..3dce0ecdfd493 100644
--- a/core/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java
+++ b/core/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java
@@ -658,10 +658,10 @@ static final class InFlightOpsResponse extends TransportResponse {
 
         int opCount;
 
-        public InFlightOpsResponse() {
+        InFlightOpsResponse() {
         }
 
-        public InFlightOpsResponse(int opCount) {
+        InFlightOpsResponse(int opCount) {
             this.opCount = opCount;
         }
 
diff --git a/core/src/main/java/org/elasticsearch/indices/query/IndicesQueriesRegistry.java b/core/src/main/java/org/elasticsearch/indices/query/IndicesQueriesRegistry.java
deleted file mode 100644
index f1d45b55495fb..0000000000000
--- a/core/src/main/java/org/elasticsearch/indices/query/IndicesQueriesRegistry.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.indices.query;
-
-import org.elasticsearch.common.xcontent.ParseFieldRegistry;
-import org.elasticsearch.index.query.QueryParser;
-
-/**
- * Extensions to ParseFieldRegistry to make Guice happy.
- */
-public class IndicesQueriesRegistry extends ParseFieldRegistry<QueryParser<?>> {
-    public IndicesQueriesRegistry() {
-        super("query");
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/PeerRecoverySourceService.java b/core/src/main/java/org/elasticsearch/indices/recovery/PeerRecoverySourceService.java
index 3fd12a6993e30..dc40c76236fe2 100644
--- a/core/src/main/java/org/elasticsearch/indices/recovery/PeerRecoverySourceService.java
+++ b/core/src/main/java/org/elasticsearch/indices/recovery/PeerRecoverySourceService.java
@@ -84,7 +84,7 @@ public void beforeIndexShardClosed(ShardId shardId, @Nullable IndexShard indexSh
         }
     }
 
-    private RecoveryResponse recover(final StartRecoveryRequest request) throws IOException {
+    private RecoveryResponse recover(StartRecoveryRequest request) throws IOException {
         final IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex());
         final IndexShard shard = indexService.getShard(request.shardId().id());
 
@@ -113,6 +113,19 @@ private RecoveryResponse recover(final StartRecoveryRequest request) throws IOEx
             throw new DelayRecoveryException("source node has the state of the target shard to be [" + targetShardRouting.state() + "], expecting to be [initializing]");
         }
 
+        if (request.targetAllocationId() == null) {
+            // ES versions < 5.4.0 do not send targetAllocationId as part of recovery request, just assume that we have the correct id
+            request = new StartRecoveryRequest(request.shardId(), targetShardRouting.allocationId().getId(), request.sourceNode(),
+                request.targetNode(), request.metadataSnapshot(), request.isPrimaryRelocation(), request.recoveryId());
+        }
+
+        if (request.targetAllocationId().equals(targetShardRouting.allocationId().getId()) == false) {
+            logger.debug("delaying recovery of {} due to target allocation id mismatch (expected: [{}], but was: [{}])",
+                request.shardId(), request.targetAllocationId(), targetShardRouting.allocationId().getId());
+            throw new DelayRecoveryException("source node has the state of the target shard to have allocation id [" +
+                targetShardRouting.allocationId().getId() + "], expecting to be [" + request.targetAllocationId() + "]");
+        }
+
         RecoverySourceHandler handler = ongoingRecoveries.addNewRecovery(request, shard);
         logger.trace("[{}][{}] starting recovery to {}", request.shardId().getIndex().getName(), request.shardId().id(), request.targetNode());
         try {
diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java b/core/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java
index f26d0787f4178..edc627e6f5243 100644
--- a/core/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java
+++ b/core/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java
@@ -33,7 +33,6 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
@@ -49,6 +48,7 @@
 import org.elasticsearch.index.shard.ShardNotFoundException;
 import org.elasticsearch.index.shard.TranslogRecoveryPerformer;
 import org.elasticsearch.index.store.Store;
+import org.elasticsearch.indices.recovery.RecoveriesCollection.RecoveryRef;
 import org.elasticsearch.node.NodeClosedException;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.ConnectTransportException;
@@ -91,7 +91,6 @@ public static class Actions {
 
     private final RecoveriesCollection onGoingRecoveries;
 
-    @Inject
     public PeerRecoveryTargetService(Settings settings, ThreadPool threadPool, TransportService transportService, RecoverySettings
             recoverySettings, ClusterService clusterService) {
         super(settings);
@@ -141,65 +140,86 @@ public void startRecovery(final IndexShard indexShard, final DiscoveryNode sourc
         threadPool.generic().execute(new RecoveryRunner(recoveryId));
     }
 
-    protected void retryRecovery(final RecoveryTarget recoveryTarget, final Throwable reason, TimeValue retryAfter, final
-    StartRecoveryRequest currentRequest) {
+    protected void retryRecovery(final long recoveryId, final Throwable reason, TimeValue retryAfter, TimeValue activityTimeout) {
         logger.trace(
             (Supplier<?>) () -> new ParameterizedMessage(
-                "will retry recovery with id [{}] in [{}]", recoveryTarget.recoveryId(), retryAfter), reason);
-        retryRecovery(recoveryTarget, retryAfter, currentRequest);
+                "will retry recovery with id [{}] in [{}]", recoveryId, retryAfter), reason);
+        retryRecovery(recoveryId, retryAfter, activityTimeout);
     }
 
-    protected void retryRecovery(final RecoveryTarget recoveryTarget, final String reason, TimeValue retryAfter, final
-    StartRecoveryRequest currentRequest) {
-        logger.trace("will retry recovery with id [{}] in [{}] (reason [{}])", recoveryTarget.recoveryId(), retryAfter, reason);
-        retryRecovery(recoveryTarget, retryAfter, currentRequest);
+    protected void retryRecovery(final long recoveryId, final String reason, TimeValue retryAfter, TimeValue activityTimeout) {
+        logger.trace("will retry recovery with id [{}] in [{}] (reason [{}])", recoveryId, retryAfter, reason);
+        retryRecovery(recoveryId, retryAfter, activityTimeout);
     }
 
-    private void retryRecovery(final RecoveryTarget recoveryTarget, TimeValue retryAfter, final StartRecoveryRequest currentRequest) {
-        try {
-            onGoingRecoveries.resetRecovery(recoveryTarget.recoveryId(), recoveryTarget.shardId());
-        } catch (Exception e) {
-            onGoingRecoveries.failRecovery(recoveryTarget.recoveryId(), new RecoveryFailedException(currentRequest, e), true);
+    private void retryRecovery(final long recoveryId, TimeValue retryAfter, TimeValue activityTimeout) {
+        RecoveryTarget newTarget = onGoingRecoveries.resetRecovery(recoveryId, activityTimeout);
+        if (newTarget != null) {
+            threadPool.schedule(retryAfter, ThreadPool.Names.GENERIC, new RecoveryRunner(newTarget.recoveryId()));
         }
-        threadPool.schedule(retryAfter, ThreadPool.Names.GENERIC, new RecoveryRunner(recoveryTarget.recoveryId()));
     }
 
-    private void doRecovery(final RecoveryTarget recoveryTarget) {
-        assert recoveryTarget.sourceNode() != null : "can't do a recovery without a source node";
+    private void doRecovery(final long recoveryId) {
+        final StartRecoveryRequest request;
+        final CancellableThreads cancellableThreads;
+        final RecoveryState.Timer timer;
 
-        logger.trace("collecting local files for {}", recoveryTarget);
-        Store.MetadataSnapshot metadataSnapshot = null;
-        try {
-            if (recoveryTarget.indexShard().indexSettings().isOnSharedFilesystem()) {
-                // we are not going to copy any files, so don't bother listing files, potentially running
-                // into concurrency issues with the primary changing files underneath us.
-                metadataSnapshot = Store.MetadataSnapshot.EMPTY;
-            } else {
-                metadataSnapshot = recoveryTarget.indexShard().snapshotStoreMetadata();
+        try (RecoveryRef recoveryRef = onGoingRecoveries.getRecovery(recoveryId)) {
+            if (recoveryRef == null) {
+                logger.trace("not running recovery with id [{}] - can't find it (probably finished)", recoveryId);
+                return;
             }
-        } catch (org.apache.lucene.index.IndexNotFoundException e) {
-            // happens on an empty folder. no need to log
-            metadataSnapshot = Store.MetadataSnapshot.EMPTY;
-        } catch (IOException e) {
-            logger.warn("error while listing local files, recover as if there are none", e);
-            metadataSnapshot = Store.MetadataSnapshot.EMPTY;
-        } catch (Exception e) {
-            // this will be logged as warning later on...
-            logger.trace("unexpected error while listing local files, failing recovery", e);
-            onGoingRecoveries.failRecovery(recoveryTarget.recoveryId(),
+            RecoveryTarget recoveryTarget = recoveryRef.target();
+            assert recoveryTarget.sourceNode() != null : "can't do a recovery without a source node";
+
+            logger.trace("collecting local files for {}", recoveryTarget.sourceNode());
+            Store.MetadataSnapshot metadataSnapshot;
+            try {
+                if (recoveryTarget.indexShard().indexSettings().isOnSharedFilesystem()) {
+                    // we are not going to copy any files, so don't bother listing files, potentially running
+                    // into concurrency issues with the primary changing files underneath us.
+                    metadataSnapshot = Store.MetadataSnapshot.EMPTY;
+                } else {
+                    metadataSnapshot = recoveryTarget.indexShard().snapshotStoreMetadata();
+                }
+                logger.trace("{} local file count: [{}]", recoveryTarget, metadataSnapshot.size());
+            } catch (org.apache.lucene.index.IndexNotFoundException e) {
+                // happens on an empty folder. no need to log
+                logger.trace("{} shard folder empty, recover all files", recoveryTarget);
+                metadataSnapshot = Store.MetadataSnapshot.EMPTY;
+            } catch (IOException e) {
+                logger.warn("error while listing local files, recover as if there are none", e);
+                metadataSnapshot = Store.MetadataSnapshot.EMPTY;
+            } catch (Exception e) {
+                // this will be logged as warning later on...
+                logger.trace("unexpected error while listing local files, failing recovery", e);
+                onGoingRecoveries.failRecovery(recoveryTarget.recoveryId(),
                     new RecoveryFailedException(recoveryTarget.state(), "failed to list local files", e), true);
-            return;
+                return;
+            }
+
+            try {
+                logger.trace("{} preparing shard for peer recovery", recoveryTarget.shardId());
+                recoveryTarget.indexShard().prepareForIndexRecovery();
+
+                request = new StartRecoveryRequest(recoveryTarget.shardId(),
+                    recoveryTarget.indexShard().routingEntry().allocationId().getId(), recoveryTarget.sourceNode(),
+                    clusterService.localNode(), metadataSnapshot, recoveryTarget.state().getPrimary(), recoveryTarget.recoveryId());
+                cancellableThreads = recoveryTarget.CancellableThreads();
+                timer = recoveryTarget.state().getTimer();
+            } catch (Exception e) {
+                // this will be logged as warning later on...
+                logger.trace("unexpected error while preparing shard for peer recovery, failing recovery", e);
+                onGoingRecoveries.failRecovery(recoveryTarget.recoveryId(),
+                    new RecoveryFailedException(recoveryTarget.state(), "failed to prepare shard for recovery", e), true);
+                return;
+            }
         }
-        logger.trace("{} local file count: [{}]", recoveryTarget, metadataSnapshot.size());
-        final StartRecoveryRequest request = new StartRecoveryRequest(recoveryTarget.shardId(), recoveryTarget.sourceNode(),
-                clusterService.localNode(), metadataSnapshot, recoveryTarget.state().getPrimary(), recoveryTarget.recoveryId());
 
-        final AtomicReference<RecoveryResponse> responseHolder = new AtomicReference<>();
         try {
-            logger.trace("[{}][{}] starting recovery from {}", request.shardId().getIndex().getName(), request.shardId().id(), request
-                    .sourceNode());
-            recoveryTarget.indexShard().prepareForIndexRecovery();
-            recoveryTarget.CancellableThreads().execute(() -> responseHolder.set(
+            logger.trace("{} starting recovery from {}", request.shardId(), request.sourceNode());
+            final AtomicReference<RecoveryResponse> responseHolder = new AtomicReference<>();
+            cancellableThreads.execute(() -> responseHolder.set(
                     transportService.submitRequest(request.sourceNode(), PeerRecoverySourceService.Actions.START_RECOVERY, request,
                             new FutureTransportResponseHandler<RecoveryResponse>() {
                                 @Override
@@ -209,9 +229,9 @@ public RecoveryResponse newInstance() {
                             }).txGet()));
             final RecoveryResponse recoveryResponse = responseHolder.get();
             assert responseHolder != null;
-            final TimeValue recoveryTime = new TimeValue(recoveryTarget.state().getTimer().time());
+            final TimeValue recoveryTime = new TimeValue(timer.time());
             // do this through ongoing recoveries to remove it from the collection
-            onGoingRecoveries.markRecoveryAsDone(recoveryTarget.recoveryId());
+            onGoingRecoveries.markRecoveryAsDone(recoveryId);
             if (logger.isTraceEnabled()) {
                 StringBuilder sb = new StringBuilder();
                 sb.append('[').append(request.shardId().getIndex().getName()).append(']').append('[').append(request.shardId().id())
@@ -231,7 +251,7 @@ public RecoveryResponse newInstance() {
                         .append("\n");
                 logger.trace("{}", sb);
             } else {
-                logger.debug("{} recovery done from [{}], took [{}]", request.shardId(), recoveryTarget.sourceNode(), recoveryTime);
+                logger.debug("{} recovery done from [{}], took [{}]", request.shardId(), request.sourceNode(), recoveryTime);
             }
         } catch (CancellableThreads.ExecutionCancelledException e) {
             logger.trace("recovery cancelled", e);
@@ -247,8 +267,8 @@ public RecoveryResponse newInstance() {
             Throwable cause = ExceptionsHelper.unwrapCause(e);
             if (cause instanceof CancellableThreads.ExecutionCancelledException) {
                 // this can also come from the source wrapped in a RemoteTransportException
-                onGoingRecoveries.failRecovery(recoveryTarget.recoveryId(), new RecoveryFailedException(request, "source has canceled the" +
-                        " recovery", cause), false);
+                onGoingRecoveries.failRecovery(recoveryId, new RecoveryFailedException(request,
+                    "source has canceled the recovery", cause), false);
                 return;
             }
             if (cause instanceof RecoveryEngineException) {
@@ -264,31 +284,34 @@ public RecoveryResponse newInstance() {
 
             // here, we would add checks against exception that need to be retried (and not removeAndClean in this case)
 
-            if (cause instanceof IllegalIndexShardStateException || cause instanceof IndexNotFoundException || cause instanceof
-                    ShardNotFoundException) {
+            if (cause instanceof IllegalIndexShardStateException || cause instanceof IndexNotFoundException ||
+                cause instanceof ShardNotFoundException) {
                 // if the target is not ready yet, retry
-                retryRecovery(recoveryTarget, "remote shard not ready", recoverySettings.retryDelayStateSync(), request);
+                retryRecovery(recoveryId, "remote shard not ready", recoverySettings.retryDelayStateSync(),
+                    recoverySettings.activityTimeout());
                 return;
             }
 
             if (cause instanceof DelayRecoveryException) {
-                retryRecovery(recoveryTarget, cause, recoverySettings.retryDelayStateSync(), request);
+                retryRecovery(recoveryId, cause, recoverySettings.retryDelayStateSync(),
+                    recoverySettings.activityTimeout());
                 return;
             }
 
             if (cause instanceof ConnectTransportException) {
-                logger.debug("delaying recovery of {} for [{}] due to networking error [{}]", recoveryTarget.shardId(), recoverySettings
-                        .retryDelayNetwork(), cause.getMessage());
-                retryRecovery(recoveryTarget, cause.getMessage(), recoverySettings.retryDelayNetwork(), request);
+                logger.debug("delaying recovery of {} for [{}] due to networking error [{}]", request.shardId(),
+                    recoverySettings.retryDelayNetwork(), cause.getMessage());
+                retryRecovery(recoveryId, cause.getMessage(), recoverySettings.retryDelayNetwork(),
+                    recoverySettings.activityTimeout());
                 return;
             }
 
             if (cause instanceof AlreadyClosedException) {
-                onGoingRecoveries.failRecovery(recoveryTarget.recoveryId(), new RecoveryFailedException(request, "source shard is " +
-                        "closed", cause), false);
+                onGoingRecoveries.failRecovery(recoveryId,
+                    new RecoveryFailedException(request, "source shard is closed", cause), false);
                 return;
             }
-            onGoingRecoveries.failRecovery(recoveryTarget.recoveryId(), new RecoveryFailedException(request, e), true);
+            onGoingRecoveries.failRecovery(recoveryId, new RecoveryFailedException(request, e), true);
         }
     }
 
@@ -302,9 +325,9 @@ class PrepareForTranslogOperationsRequestHandler implements TransportRequestHand
 
         @Override
         public void messageReceived(RecoveryPrepareForTranslogOperationsRequest request, TransportChannel channel) throws Exception {
-            try (RecoveriesCollection.RecoveryRef recoveryRef = onGoingRecoveries.getRecoverySafe(request.recoveryId(), request.shardId()
+            try (RecoveryRef recoveryRef = onGoingRecoveries.getRecoverySafe(request.recoveryId(), request.shardId()
             )) {
-                recoveryRef.status().prepareForTranslogOperations(request.totalTranslogOps(), request.getMaxUnsafeAutoIdTimestamp());
+                recoveryRef.target().prepareForTranslogOperations(request.totalTranslogOps(), request.getMaxUnsafeAutoIdTimestamp());
             }
             channel.sendResponse(TransportResponse.Empty.INSTANCE);
         }
@@ -314,9 +337,9 @@ class FinalizeRecoveryRequestHandler implements TransportRequestHandler<Recovery
 
         @Override
         public void messageReceived(RecoveryFinalizeRecoveryRequest request, TransportChannel channel) throws Exception {
-            try (RecoveriesCollection.RecoveryRef recoveryRef = onGoingRecoveries.getRecoverySafe(request.recoveryId(), request.shardId()
+            try (RecoveryRef recoveryRef = onGoingRecoveries.getRecoverySafe(request.recoveryId(), request.shardId()
             )) {
-                recoveryRef.status().finalizeRecovery();
+                recoveryRef.target().finalizeRecovery();
             }
             channel.sendResponse(TransportResponse.Empty.INSTANCE);
         }
@@ -326,9 +349,9 @@ class WaitForClusterStateRequestHandler implements TransportRequestHandler<Recov
 
         @Override
         public void messageReceived(RecoveryWaitForClusterStateRequest request, TransportChannel channel) throws Exception {
-            try (RecoveriesCollection.RecoveryRef recoveryRef = onGoingRecoveries.getRecoverySafe(request.recoveryId(), request.shardId()
+            try (RecoveryRef recoveryRef = onGoingRecoveries.getRecoverySafe(request.recoveryId(), request.shardId()
             )) {
-                recoveryRef.status().ensureClusterStateVersion(request.clusterStateVersion());
+                recoveryRef.target().ensureClusterStateVersion(request.clusterStateVersion());
             }
             channel.sendResponse(TransportResponse.Empty.INSTANCE);
         }
@@ -338,10 +361,10 @@ class TranslogOperationsRequestHandler implements TransportRequestHandler<Recove
 
         @Override
         public void messageReceived(final RecoveryTranslogOperationsRequest request, final TransportChannel channel) throws IOException {
-            try (RecoveriesCollection.RecoveryRef recoveryRef =
+            try (RecoveryRef recoveryRef =
                          onGoingRecoveries.getRecoverySafe(request.recoveryId(), request.shardId())) {
                 final ClusterStateObserver observer = new ClusterStateObserver(clusterService, null, logger, threadPool.getThreadContext());
-                final RecoveryTarget recoveryTarget = recoveryRef.status();
+                final RecoveryTarget recoveryTarget = recoveryRef.target();
                 try {
                     recoveryTarget.indexTranslogOperations(request.operations(), request.totalTranslogOps());
                     channel.sendResponse(TransportResponse.Empty.INSTANCE);
@@ -399,21 +422,21 @@ public void onTimeout(TimeValue timeout) {
     }
 
     private void waitForClusterState(long clusterStateVersion) {
-        ClusterStateObserver observer = new ClusterStateObserver(clusterService, TimeValue.timeValueMinutes(5), logger,
+        final ClusterState clusterState = clusterService.state();
+        ClusterStateObserver observer = new ClusterStateObserver(clusterState, clusterService, TimeValue.timeValueMinutes(5), logger,
             threadPool.getThreadContext());
-        final ClusterState clusterState = observer.observedState();
         if (clusterState.getVersion() >= clusterStateVersion) {
             logger.trace("node has cluster state with version higher than {} (current: {})", clusterStateVersion,
                 clusterState.getVersion());
             return;
         } else {
             logger.trace("waiting for cluster state version {} (current: {})", clusterStateVersion, clusterState.getVersion());
-            final PlainActionFuture<Void> future = new PlainActionFuture<>();
+            final PlainActionFuture<Long> future = new PlainActionFuture<>();
             observer.waitForNextChange(new ClusterStateObserver.Listener() {
 
                 @Override
                 public void onNewClusterState(ClusterState state) {
-                    future.onResponse(null);
+                    future.onResponse(state.getVersion());
                 }
 
                 @Override
@@ -425,23 +448,16 @@ public void onClusterServiceClose() {
                 public void onTimeout(TimeValue timeout) {
                     future.onFailure(new IllegalStateException("cluster state never updated to version " + clusterStateVersion));
                 }
-            }, new ClusterStateObserver.ValidationPredicate() {
-
-                @Override
-                protected boolean validate(ClusterState newState) {
-                    return newState.getVersion() >= clusterStateVersion;
-                }
-            });
+            }, newState -> newState.getVersion() >= clusterStateVersion);
             try {
-                future.get();
-                logger.trace("successfully waited for cluster state with version {} (current: {})", clusterStateVersion,
-                    observer.observedState().getVersion());
+                long currentVersion = future.get();
+                logger.trace("successfully waited for cluster state with version {} (current: {})", clusterStateVersion, currentVersion);
             } catch (Exception e) {
                 logger.debug(
                     (Supplier<?>) () -> new ParameterizedMessage(
                         "failed waiting for cluster state with version {} (current: {})",
                         clusterStateVersion,
-                        observer.observedState()),
+                        clusterService.state().getVersion()),
                     e);
                 throw ExceptionsHelper.convertToRuntime(e);
             }
@@ -452,9 +468,9 @@ class FilesInfoRequestHandler implements TransportRequestHandler<RecoveryFilesIn
 
         @Override
         public void messageReceived(RecoveryFilesInfoRequest request, TransportChannel channel) throws Exception {
-            try (RecoveriesCollection.RecoveryRef recoveryRef = onGoingRecoveries.getRecoverySafe(request.recoveryId(), request.shardId()
+            try (RecoveryRef recoveryRef = onGoingRecoveries.getRecoverySafe(request.recoveryId(), request.shardId()
             )) {
-                recoveryRef.status().receiveFileInfo(request.phase1FileNames, request.phase1FileSizes, request.phase1ExistingFileNames,
+                recoveryRef.target().receiveFileInfo(request.phase1FileNames, request.phase1FileSizes, request.phase1ExistingFileNames,
                         request.phase1ExistingFileSizes, request.totalTranslogOps);
                 channel.sendResponse(TransportResponse.Empty.INSTANCE);
             }
@@ -465,9 +481,9 @@ class CleanFilesRequestHandler implements TransportRequestHandler<RecoveryCleanF
 
         @Override
         public void messageReceived(RecoveryCleanFilesRequest request, TransportChannel channel) throws Exception {
-            try (RecoveriesCollection.RecoveryRef recoveryRef = onGoingRecoveries.getRecoverySafe(request.recoveryId(), request.shardId()
+            try (RecoveryRef recoveryRef = onGoingRecoveries.getRecoverySafe(request.recoveryId(), request.shardId()
             )) {
-                recoveryRef.status().cleanFiles(request.totalTranslogOps(), request.sourceMetaSnapshot());
+                recoveryRef.target().cleanFiles(request.totalTranslogOps(), request.sourceMetaSnapshot());
                 channel.sendResponse(TransportResponse.Empty.INSTANCE);
             }
         }
@@ -480,10 +496,10 @@ class FileChunkTransportRequestHandler implements TransportRequestHandler<Recove
 
         @Override
         public void messageReceived(final RecoveryFileChunkRequest request, TransportChannel channel) throws Exception {
-            try (RecoveriesCollection.RecoveryRef recoveryRef = onGoingRecoveries.getRecoverySafe(request.recoveryId(), request.shardId()
+            try (RecoveryRef recoveryRef = onGoingRecoveries.getRecoverySafe(request.recoveryId(), request.shardId()
             )) {
-                final RecoveryTarget status = recoveryRef.status();
-                final RecoveryState.Index indexState = status.state().getIndex();
+                final RecoveryTarget recoveryTarget = recoveryRef.target();
+                final RecoveryState.Index indexState = recoveryTarget.state().getIndex();
                 if (request.sourceThrottleTimeInNanos() != RecoveryState.Index.UNKNOWN) {
                     indexState.addSourceThrottling(request.sourceThrottleTimeInNanos());
                 }
@@ -496,11 +512,11 @@ public void messageReceived(final RecoveryFileChunkRequest request, TransportCha
                         bytesSinceLastPause.addAndGet(-bytes);
                         long throttleTimeInNanos = rateLimiter.pause(bytes);
                         indexState.addTargetThrottling(throttleTimeInNanos);
-                        status.indexShard().recoveryStats().addThrottleTime(throttleTimeInNanos);
+                        recoveryTarget.indexShard().recoveryStats().addThrottleTime(throttleTimeInNanos);
                     }
                 }
 
-                status.writeFileChunk(request.metadata(), request.position(), request.content(),
+                recoveryTarget.writeFileChunk(request.metadata(), request.position(), request.content(),
                         request.lastChunk(), request.totalTranslogOps()
                 );
             }
@@ -518,13 +534,13 @@ class RecoveryRunner extends AbstractRunnable {
 
         @Override
         public void onFailure(Exception e) {
-            try (RecoveriesCollection.RecoveryRef recoveryRef = onGoingRecoveries.getRecovery(recoveryId)) {
+            try (RecoveryRef recoveryRef = onGoingRecoveries.getRecovery(recoveryId)) {
                 if (recoveryRef != null) {
                     logger.error(
                         (Supplier<?>) () -> new ParameterizedMessage(
                             "unexpected error during recovery [{}], failing shard", recoveryId), e);
                     onGoingRecoveries.failRecovery(recoveryId,
-                            new RecoveryFailedException(recoveryRef.status().state(), "unexpected error", e),
+                            new RecoveryFailedException(recoveryRef.target().state(), "unexpected error", e),
                             true // be safe
                     );
                 } else {
@@ -537,16 +553,7 @@ public void onFailure(Exception e) {
 
         @Override
         public void doRun() {
-            RecoveriesCollection.RecoveryRef recoveryRef = onGoingRecoveries.getRecovery(recoveryId);
-            if (recoveryRef == null) {
-                logger.trace("not running recovery with id [{}] - can't find it (probably finished)", recoveryId);
-                return;
-            }
-            try {
-                doRecovery(recoveryRef.status());
-            } finally {
-                recoveryRef.close();
-            }
+            doRecovery(recoveryId);
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveriesCollection.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveriesCollection.java
index 65a48b18e22a3..b5d9c1d34bd3f 100644
--- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveriesCollection.java
+++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveriesCollection.java
@@ -25,7 +25,6 @@
 import org.elasticsearch.ElasticsearchTimeoutException;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.util.Callback;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.index.shard.IndexShard;
@@ -33,9 +32,12 @@
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.threadpool.ThreadPool;
 
-import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.LongConsumer;
 
 /**
  * This class holds a collection of all on going recoveries on the current node (i.e., the node is the target node
@@ -50,9 +52,9 @@ public class RecoveriesCollection {
 
     private final Logger logger;
     private final ThreadPool threadPool;
-    private final Callback<Long> ensureClusterStateVersionCallback;
+    private final LongConsumer ensureClusterStateVersionCallback;
 
-    public RecoveriesCollection(Logger logger, ThreadPool threadPool, Callback<Long> ensureClusterStateVersionCallback) {
+    public RecoveriesCollection(Logger logger, ThreadPool threadPool, LongConsumer ensureClusterStateVersionCallback) {
         this.logger = logger;
         this.threadPool = threadPool;
         this.ensureClusterStateVersionCallback = ensureClusterStateVersionCallback;
@@ -65,13 +67,18 @@ public RecoveriesCollection(Logger logger, ThreadPool threadPool, Callback<Long>
      */
     public long startRecovery(IndexShard indexShard, DiscoveryNode sourceNode,
                               PeerRecoveryTargetService.RecoveryListener listener, TimeValue activityTimeout) {
-        RecoveryTarget status = new RecoveryTarget(indexShard, sourceNode, listener, ensureClusterStateVersionCallback);
-        RecoveryTarget existingStatus = onGoingRecoveries.putIfAbsent(status.recoveryId(), status);
-        assert existingStatus == null : "found two RecoveryStatus instances with the same id";
-        logger.trace("{} started recovery from {}, id [{}]", indexShard.shardId(), sourceNode, status.recoveryId());
+        RecoveryTarget recoveryTarget = new RecoveryTarget(indexShard, sourceNode, listener, ensureClusterStateVersionCallback);
+        startRecoveryInternal(recoveryTarget, activityTimeout);
+        return recoveryTarget.recoveryId();
+    }
+
+    private void startRecoveryInternal(RecoveryTarget recoveryTarget, TimeValue activityTimeout) {
+        RecoveryTarget existingTarget = onGoingRecoveries.putIfAbsent(recoveryTarget.recoveryId(), recoveryTarget);
+        assert existingTarget == null : "found two RecoveryStatus instances with the same id";
+        logger.trace("{} started recovery from {}, id [{}]", recoveryTarget.shardId(), recoveryTarget.sourceNode(),
+            recoveryTarget.recoveryId());
         threadPool.schedule(activityTimeout, ThreadPool.Names.GENERIC,
-                new RecoveryMonitor(status.recoveryId(), status.lastAccessTime(), activityTimeout));
-        return status.recoveryId();
+                new RecoveryMonitor(recoveryTarget.recoveryId(), recoveryTarget.lastAccessTime(), activityTimeout));
     }
 
 
@@ -79,22 +86,49 @@ public long startRecovery(IndexShard indexShard, DiscoveryNode sourceNode,
      * Resets the recovery and performs a recovery restart on the currently recovering index shard
      *
      * @see IndexShard#performRecoveryRestart()
+     * @return newly created RecoveryTarget
      */
-    public void resetRecovery(long id, ShardId shardId) throws IOException {
-        try (RecoveryRef ref = getRecoverySafe(id, shardId)) {
-            // instead of adding complicated state to RecoveryTarget we just flip the
-            // target instance when we reset a recovery, that way we have only one cleanup
-            // path on the RecoveryTarget and are always within the bounds of ref-counting
-            // which is important since we verify files are on disk etc. after we have written them etc.
-            RecoveryTarget status = ref.status();
-            RecoveryTarget resetRecovery = status.resetRecovery();
-            if (onGoingRecoveries.replace(id, status, resetRecovery) == false) {
-                resetRecovery.cancel("replace failed"); // this is important otherwise we leak a reference to the store
-                throw new IllegalStateException("failed to replace recovery target");
+    public RecoveryTarget resetRecovery(final long recoveryId, TimeValue activityTimeout) {
+        RecoveryTarget oldRecoveryTarget = null;
+        final RecoveryTarget newRecoveryTarget;
+
+        try {
+            synchronized (onGoingRecoveries) {
+                // swap recovery targets in a synchronized block to ensure that the newly added recovery target is picked up by
+                // cancelRecoveriesForShard whenever the old recovery target is picked up
+                oldRecoveryTarget = onGoingRecoveries.remove(recoveryId);
+                if (oldRecoveryTarget == null) {
+                    return null;
+                }
+
+                newRecoveryTarget = oldRecoveryTarget.retryCopy();
+                startRecoveryInternal(newRecoveryTarget, activityTimeout);
+            }
+
+            // Closes the current recovery target
+            boolean successfulReset = oldRecoveryTarget.resetRecovery(newRecoveryTarget.CancellableThreads());
+            if (successfulReset) {
+                logger.trace("{} restarted recovery from {}, id [{}], previous id [{}]", newRecoveryTarget.shardId(),
+                    newRecoveryTarget.sourceNode(), newRecoveryTarget.recoveryId(), oldRecoveryTarget.recoveryId());
+                return newRecoveryTarget;
+            } else {
+                logger.trace("{} recovery could not be reset as it is already cancelled, recovery from {}, id [{}], previous id [{}]",
+                    newRecoveryTarget.shardId(), newRecoveryTarget.sourceNode(), newRecoveryTarget.recoveryId(),
+                    oldRecoveryTarget.recoveryId());
+                cancelRecovery(newRecoveryTarget.recoveryId(), "recovery cancelled during reset");
+                return null;
             }
+        } catch (Exception e) {
+            // fail shard to be safe
+            oldRecoveryTarget.notifyListener(new RecoveryFailedException(oldRecoveryTarget.state(), "failed to retry recovery", e), true);
+            return null;
         }
     }
 
+    public RecoveryTarget getRecoveryTarget(long id) {
+        return onGoingRecoveries.get(id);
+    }
+
     /**
      * gets the {@link RecoveryTarget } for a given id. The RecoveryStatus returned has it's ref count already incremented
      * to make sure it's safe to use. However, you must call {@link RecoveryTarget#decRef()} when you are done with it, typically
@@ -116,7 +150,7 @@ public RecoveryRef getRecoverySafe(long id, ShardId shardId) {
         if (recoveryRef == null) {
             throw new IndexShardClosedException(shardId);
         }
-        assert recoveryRef.status().shardId().equals(shardId);
+        assert recoveryRef.target().shardId().equals(shardId);
         return recoveryRef;
     }
 
@@ -143,7 +177,8 @@ public boolean cancelRecovery(long id, String reason) {
     public void failRecovery(long id, RecoveryFailedException e, boolean sendShardFailure) {
         RecoveryTarget removed = onGoingRecoveries.remove(id);
         if (removed != null) {
-            logger.trace("{} failing recovery from {}, id [{}]. Send shard failure: [{}]", removed.shardId(), removed.sourceNode(), removed.recoveryId(), sendShardFailure);
+            logger.trace("{} failing recovery from {}, id [{}]. Send shard failure: [{}]", removed.shardId(), removed.sourceNode(),
+                removed.recoveryId(), sendShardFailure);
             removed.fail(e, sendShardFailure);
         }
     }
@@ -171,11 +206,22 @@ public int size() {
      */
     public boolean cancelRecoveriesForShard(ShardId shardId, String reason) {
         boolean cancelled = false;
-        for (RecoveryTarget status : onGoingRecoveries.values()) {
-            if (status.shardId().equals(shardId)) {
-                cancelled |= cancelRecovery(status.recoveryId(), reason);
+        List<RecoveryTarget> matchedRecoveries = new ArrayList<>();
+        synchronized (onGoingRecoveries) {
+            for (Iterator<RecoveryTarget> it = onGoingRecoveries.values().iterator(); it.hasNext(); ) {
+                RecoveryTarget status = it.next();
+                if (status.shardId().equals(shardId)) {
+                    matchedRecoveries.add(status);
+                    it.remove();
+                }
             }
         }
+        for (RecoveryTarget removed : matchedRecoveries) {
+            logger.trace("{} canceled recovery from {}, id [{}] (reason [{}])",
+                removed.shardId(), removed.sourceNode(), removed.recoveryId(), reason);
+            removed.cancel(reason);
+            cancelled = true;
+        }
         return cancelled;
     }
 
@@ -205,7 +251,7 @@ public void close() {
             }
         }
 
-        public RecoveryTarget status() {
+        public RecoveryTarget target() {
             return status;
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryFailedException.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryFailedException.java
index 3c3d96a4f9baa..283de68f82b5b 100644
--- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryFailedException.java
+++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryFailedException.java
@@ -49,7 +49,8 @@ public RecoveryFailedException(ShardId shardId, DiscoveryNode sourceNode, Discov
     }
 
     public RecoveryFailedException(ShardId shardId, DiscoveryNode sourceNode, DiscoveryNode targetNode, @Nullable String extraInfo, Throwable cause) {
-        super(shardId + ": Recovery failed from " + sourceNode + " into " + targetNode + (extraInfo == null ? "" : " (" + extraInfo + ")"), cause);
+        super(shardId + ": Recovery failed " + (sourceNode != null ? "from " + sourceNode + " into " : "on ") +
+                  targetNode + (extraInfo == null ? "" : " (" + extraInfo + ")"), cause);
     }
 
     public RecoveryFailedException(StreamInput in) throws IOException {
diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySettings.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySettings.java
index 82595458479b3..e238277b698d0 100644
--- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySettings.java
+++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySettings.java
@@ -22,7 +22,6 @@
 import org.apache.lucene.store.RateLimiter;
 import org.apache.lucene.store.RateLimiter.SimpleRateLimiter;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
@@ -61,7 +60,7 @@ public class RecoverySettings extends AbstractComponent {
      */
     public static final Setting<TimeValue> INDICES_RECOVERY_INTERNAL_LONG_ACTION_TIMEOUT_SETTING =
         Setting.timeSetting("indices.recovery.internal_action_long_timeout",
-            (s) -> TimeValue.timeValueMillis(INDICES_RECOVERY_INTERNAL_ACTION_TIMEOUT_SETTING.get(s).millis() * 2).toString(),
+            (s) -> TimeValue.timeValueMillis(INDICES_RECOVERY_INTERNAL_ACTION_TIMEOUT_SETTING.get(s).millis() * 2),
             TimeValue.timeValueSeconds(0), Property.Dynamic,  Property.NodeScope);
 
     /**
@@ -70,7 +69,7 @@ public class RecoverySettings extends AbstractComponent {
      */
     public static final Setting<TimeValue> INDICES_RECOVERY_ACTIVITY_TIMEOUT_SETTING =
         Setting.timeSetting("indices.recovery.recovery_activity_timeout",
-            (s) -> INDICES_RECOVERY_INTERNAL_LONG_ACTION_TIMEOUT_SETTING.getRaw(s) , TimeValue.timeValueSeconds(0),
+            INDICES_RECOVERY_INTERNAL_LONG_ACTION_TIMEOUT_SETTING::get, TimeValue.timeValueSeconds(0),
             Property.Dynamic, Property.NodeScope);
 
     public static final ByteSizeValue DEFAULT_CHUNK_SIZE = new ByteSizeValue(512, ByteSizeUnit.KB);
@@ -85,7 +84,6 @@ public class RecoverySettings extends AbstractComponent {
 
     private volatile ByteSizeValue chunkSize = DEFAULT_CHUNK_SIZE;
 
-    @Inject
     public RecoverySettings(Settings settings, ClusterSettings clusterSettings) {
         super(settings);
 
@@ -99,10 +97,10 @@ public RecoverySettings(Settings settings, ClusterSettings clusterSettings) {
 
         this.activityTimeout = INDICES_RECOVERY_ACTIVITY_TIMEOUT_SETTING.get(settings);
         this.maxBytesPerSec = INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.get(settings);
-        if (maxBytesPerSec.bytes() <= 0) {
+        if (maxBytesPerSec.getBytes() <= 0) {
             rateLimiter = null;
         } else {
-            rateLimiter = new SimpleRateLimiter(maxBytesPerSec.mbFrac());
+            rateLimiter = new SimpleRateLimiter(maxBytesPerSec.getMbFrac());
         }
 
 
@@ -142,7 +140,7 @@ public TimeValue internalActionLongTimeout() {
 
     public ByteSizeValue getChunkSize() { return chunkSize; }
 
-    void setChunkSize(ByteSizeValue chunkSize) { // only settable for tests
+    public void setChunkSize(ByteSizeValue chunkSize) { // only settable for tests
         if (chunkSize.bytesAsInt() <= 0) {
             throw new IllegalArgumentException("chunkSize must be > 0");
         }
@@ -172,12 +170,12 @@ public void setInternalActionLongTimeout(TimeValue internalActionLongTimeout) {
 
     private void setMaxBytesPerSec(ByteSizeValue maxBytesPerSec) {
         this.maxBytesPerSec = maxBytesPerSec;
-        if (maxBytesPerSec.bytes() <= 0) {
+        if (maxBytesPerSec.getBytes() <= 0) {
             rateLimiter = null;
         } else if (rateLimiter != null) {
-            rateLimiter.setMBPerSec(maxBytesPerSec.mbFrac());
+            rateLimiter.setMBPerSec(maxBytesPerSec.getMbFrac());
         } else {
-            rateLimiter = new SimpleRateLimiter(maxBytesPerSec.mbFrac());
+            rateLimiter = new SimpleRateLimiter(maxBytesPerSec.getMbFrac());
         }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
index 790376ba78ebd..5f66ffd4315ee 100644
--- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
+++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
@@ -148,6 +148,8 @@ public RecoveryResponse recoverToTarget() throws IOException {
 
             // engine was just started at the end of phase 1
             if (shard.state() == IndexShardState.RELOCATED) {
+                assert request.isPrimaryRelocation() == false :
+                    "recovery target should not retry primary relocation if previous attempt made it past finalization step";
                 /**
                  * The primary shard has been relocated while we copied files. This means that we can't guarantee any more that all
                  * operations that were replicated during the file copy (when the target engine was not yet opened) will be present in the
@@ -220,7 +222,7 @@ public void phase1(final IndexCommit snapshot, final Translog.View translogView)
                 final long numDocsSource = recoverySourceMetadata.getNumDocs();
                 if (numDocsTarget != numDocsSource) {
                     throw new IllegalStateException("try to recover " + request.shardId() + " from primary shard with sync id but number " +
-                            "of docs differ: " + numDocsTarget + " (" + request.sourceNode().getName() + ", primary) vs " + numDocsSource
+                            "of docs differ: " + numDocsSource + " (" + request.sourceNode().getName() + ", primary) vs " + numDocsTarget
                             + "(" + request.targetNode().getName() + ")");
                 }
                 // we shortcut recovery here because we have nothing to copy. but we must still start the engine on the target.
@@ -327,7 +329,7 @@ public void phase1(final IndexCommit snapshot, final Translog.View translogView)
                 }
             }
 
-            prepareTargetForTranslog(translogView.totalOperations());
+            prepareTargetForTranslog(translogView.totalOperations(), shard.segmentStats(false).getMaxUnsafeAutoIdTimestamp());
 
             logger.trace("[{}][{}] recovery [phase1] to {}: took [{}]", indexName, shardId, request.targetNode(), stopWatch.totalTime());
             response.phase1Time = stopWatch.totalTime().millis();
@@ -339,15 +341,14 @@ public void phase1(final IndexCommit snapshot, final Translog.View translogView)
     }
 
 
-    protected void prepareTargetForTranslog(final int totalTranslogOps) throws IOException {
+    protected void prepareTargetForTranslog(final int totalTranslogOps, long maxUnsafeAutoIdTimestamp) throws IOException {
         StopWatch stopWatch = new StopWatch().start();
         logger.trace("{} recovery [phase1] to {}: prepare remote engine for translog", request.shardId(), request.targetNode());
         final long startEngineStart = stopWatch.totalTime().millis();
         // Send a request preparing the new shard's translog to receive
         // operations. This ensures the shard engine is started and disables
         // garbage collection (not the JVM's GC!) of tombstone deletes
-        cancellableThreads.executeIO(() -> recoveryTarget.prepareForTranslogOperations(totalTranslogOps,
-            shard.segmentStats(false).getMaxUnsafeAutoIdTimestamp()));
+        cancellableThreads.executeIO(() -> recoveryTarget.prepareForTranslogOperations(totalTranslogOps, maxUnsafeAutoIdTimestamp));
         stopWatch.stop();
 
         response.startTime = stopWatch.totalTime().millis() - startEngineStart;
@@ -547,7 +548,7 @@ void sendFiles(Store store, StoreFileMetaData[] files, Function<StoreFileMetaDat
             ArrayUtil.timSort(files, (a, b) -> Long.compare(a.length(), b.length())); // send smallest first
             for (int i = 0; i < files.length; i++) {
                 final StoreFileMetaData md = files[i];
-                try (final IndexInput indexInput = store.directory().openInput(md.name(), IOContext.READONCE)) {
+                try (IndexInput indexInput = store.directory().openInput(md.name(), IOContext.READONCE)) {
                     // it's fine that we are only having the indexInput in the try/with block. The copy methods handles
                     // exceptions during close correctly and doesn't hide the original exception.
                     Streams.copy(new InputStreamIndexInput(indexInput, md.length()), outputStreamFactory.apply(md));
diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryState.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryState.java
index ef4f5072d7c79..77d8b4d707739 100644
--- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryState.java
+++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryState.java
@@ -45,7 +45,7 @@
  */
 public class RecoveryState implements ToXContent, Streamable {
 
-    public static enum Stage {
+    public enum Stage {
         INIT((byte) 0),
 
         /**
@@ -260,9 +260,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.field(Fields.TYPE, recoverySource.getType());
         builder.field(Fields.STAGE, stage.toString());
         builder.field(Fields.PRIMARY, primary);
-        builder.dateValueField(Fields.START_TIME_IN_MILLIS, Fields.START_TIME, timer.startTime);
+        builder.dateField(Fields.START_TIME_IN_MILLIS, Fields.START_TIME, timer.startTime);
         if (timer.stopTime > 0) {
-            builder.dateValueField(Fields.STOP_TIME_IN_MILLIS, Fields.STOP_TIME, timer.stopTime);
+            builder.dateField(Fields.STOP_TIME_IN_MILLIS, Fields.STOP_TIME, timer.stopTime);
         }
         builder.timeValueField(Fields.TOTAL_TIME_IN_MILLIS, Fields.TOTAL_TIME, timer.time());
 
diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java
index d608dc50e2326..98e2cc85bc9ec 100644
--- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java
+++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java
@@ -36,7 +36,6 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.lucene.Lucene;
-import org.elasticsearch.common.util.Callback;
 import org.elasticsearch.common.util.CancellableThreads;
 import org.elasticsearch.common.util.concurrent.AbstractRefCounted;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
@@ -55,8 +54,10 @@
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.LongConsumer;
 
 /**
  * Represents a recovery where the current node is the target node of the recovery. To track recoveries in a central place, instances of
@@ -68,7 +69,7 @@ public class RecoveryTarget extends AbstractRefCounted implements RecoveryTarget
 
     private static final AtomicLong idGenerator = new AtomicLong();
 
-    private final String RECOVERY_PREFIX = "recovery.";
+    private static final String RECOVERY_PREFIX = "recovery.";
 
     private final ShardId shardId;
     private final long recoveryId;
@@ -77,7 +78,7 @@ public class RecoveryTarget extends AbstractRefCounted implements RecoveryTarget
     private final String tempFilePrefix;
     private final Store store;
     private final PeerRecoveryTargetService.RecoveryListener listener;
-    private final Callback<Long> ensureClusterStateVersionCallback;
+    private final LongConsumer ensureClusterStateVersionCallback;
 
     private final AtomicBoolean finished = new AtomicBoolean();
 
@@ -87,17 +88,11 @@ public class RecoveryTarget extends AbstractRefCounted implements RecoveryTarget
     // last time this status was accessed
     private volatile long lastAccessTime = System.nanoTime();
 
-    private final Map<String, String> tempFileNames = ConcurrentCollections.newConcurrentMap();
+    // latch that can be used to blockingly wait for RecoveryTarget to be closed
+    private final CountDownLatch closedLatch = new CountDownLatch(1);
 
-    private RecoveryTarget(RecoveryTarget copyFrom) { // copy constructor
-        this(copyFrom.indexShard, copyFrom.sourceNode, copyFrom.listener, copyFrom.cancellableThreads, copyFrom.recoveryId,
-            copyFrom.ensureClusterStateVersionCallback);
-    }
+    private final Map<String, String> tempFileNames = ConcurrentCollections.newConcurrentMap();
 
-    public RecoveryTarget(IndexShard indexShard, DiscoveryNode sourceNode, PeerRecoveryTargetService.RecoveryListener listener,
-                          Callback<Long> ensureClusterStateVersionCallback) {
-        this(indexShard, sourceNode, listener, new CancellableThreads(), idGenerator.incrementAndGet(), ensureClusterStateVersionCallback);
-    }
     /**
      * creates a new recovery target object that represents a recovery to the provided indexShard
      *
@@ -108,11 +103,14 @@ public RecoveryTarget(IndexShard indexShard, DiscoveryNode sourceNode, PeerRecov
      *                                          version. Necessary for primary relocation so that new primary knows about all other ongoing
      *                                          replica recoveries when replicating documents (see {@link RecoverySourceHandler}).
      */
-    private RecoveryTarget(IndexShard indexShard, DiscoveryNode sourceNode, PeerRecoveryTargetService.RecoveryListener listener,
-                           CancellableThreads cancellableThreads, long recoveryId, Callback<Long> ensureClusterStateVersionCallback) {
+    public RecoveryTarget(
+            final IndexShard indexShard,
+            final DiscoveryNode sourceNode,
+            final PeerRecoveryTargetService.RecoveryListener listener,
+            final LongConsumer ensureClusterStateVersionCallback) {
         super("recovery_status");
-        this.cancellableThreads = cancellableThreads;
-        this.recoveryId = recoveryId;
+        this.cancellableThreads = new CancellableThreads();
+        this.recoveryId = idGenerator.incrementAndGet();
         this.listener = listener;
         this.logger = Loggers.getLogger(getClass(), indexShard.indexSettings().getSettings(), indexShard.shardId());
         this.indexShard = indexShard;
@@ -126,6 +124,13 @@ private RecoveryTarget(IndexShard indexShard, DiscoveryNode sourceNode, PeerReco
         indexShard.recoveryStats().incCurrentAsTarget();
     }
 
+    /**
+     * returns a fresh RecoveryTarget to retry recovery from the same source node onto the same IndexShard and using the same listener
+     */
+    public RecoveryTarget retryCopy() {
+        return new RecoveryTarget(this.indexShard, this.sourceNode, this.listener, this.ensureClusterStateVersionCallback);
+    }
+
     public long recoveryId() {
         return recoveryId;
     }
@@ -177,19 +182,37 @@ public void renameAllTempFiles() throws IOException {
     }
 
     /**
-     * Closes the current recovery target and returns a
-     * clone to reset the ongoing recovery.
-     * Note: the returned target must be canceled, failed or finished
-     * in order to release all it's reference.
+     * Closes the current recovery target and waits up to a certain timeout for resources to be freed.
+     * Returns true if resetting the recovery was successful, false if the recovery target is already cancelled / failed or marked as done.
      */
-    RecoveryTarget resetRecovery() throws IOException {
-        ensureRefCount();
+    boolean resetRecovery(CancellableThreads newTargetCancellableThreads) throws IOException {
         if (finished.compareAndSet(false, true)) {
-            // release the initial reference. recovery files will be cleaned as soon as ref count goes to zero, potentially now
-            decRef();
+            try {
+                logger.debug("reset of recovery with shard {} and id [{}]", shardId, recoveryId);
+            } finally {
+                // release the initial reference. recovery files will be cleaned as soon as ref count goes to zero, potentially now.
+                decRef();
+            }
+            try {
+                newTargetCancellableThreads.execute(closedLatch::await);
+            } catch (CancellableThreads.ExecutionCancelledException e) {
+                logger.trace("new recovery target cancelled for shard {} while waiting on old recovery target with id [{}] to close",
+                    shardId, recoveryId);
+                return false;
+            }
+            RecoveryState.Stage stage = indexShard.recoveryState().getStage();
+            if (indexShard.recoveryState().getPrimary() && (stage == RecoveryState.Stage.FINALIZE || stage == RecoveryState.Stage.DONE)) {
+                // once primary relocation has moved past the finalization step, the relocation source can be moved to RELOCATED state
+                // and start indexing as primary into the target shard (see TransportReplicationAction). Resetting the target shard in this
+                // state could mean that indexing is halted until the recovery retry attempt is completed and could also destroy existing
+                // documents indexed and acknowledged before the reset.
+                assert stage != RecoveryState.Stage.DONE : "recovery should not have completed when it's being reset";
+                throw new IllegalStateException("cannot reset recovery as previous attempt made it past finalization step");
+            }
+            indexShard.performRecoveryRestart();
+            return true;
         }
-        indexShard.performRecoveryRestart();
-        return new RecoveryTarget(this);
+        return false;
     }
 
     /**
@@ -220,7 +243,7 @@ public void cancel(String reason) {
     public void fail(RecoveryFailedException e, boolean sendShardFailure) {
         if (finished.compareAndSet(false, true)) {
             try {
-                listener.onRecoveryFailure(state(), e, sendShardFailure);
+                notifyListener(e, sendShardFailure);
             } finally {
                 try {
                     cancellableThreads.cancel("failed recovery [" + ExceptionsHelper.stackTrace(e) + "]");
@@ -232,6 +255,10 @@ public void fail(RecoveryFailedException e, boolean sendShardFailure) {
         }
     }
 
+    public void notifyListener(RecoveryFailedException e, boolean sendShardFailure) {
+        listener.onRecoveryFailure(state(), e, sendShardFailure);
+    }
+
     /** mark the current recovery as done */
     public void markAsDone() {
         if (finished.compareAndSet(false, true)) {
@@ -309,6 +336,7 @@ protected void closeInternal() {
             // free store. increment happens in constructor
             store.decRef();
             indexShard.recoveryStats().decCurrentAsTarget();
+            closedLatch.countDown();
         }
     }
 
@@ -339,7 +367,7 @@ public void finalizeRecovery() {
 
     @Override
     public void ensureClusterStateVersion(long clusterStateVersion) {
-        ensureClusterStateVersionCallback.handle(clusterStateVersion);
+        ensureClusterStateVersionCallback.accept(clusterStateVersion);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RemoteRecoveryTargetHandler.java b/core/src/main/java/org/elasticsearch/indices/recovery/RemoteRecoveryTargetHandler.java
index 327eb3b8eca7e..19d70ef264a2c 100644
--- a/core/src/main/java/org/elasticsearch/indices/recovery/RemoteRecoveryTargetHandler.java
+++ b/core/src/main/java/org/elasticsearch/indices/recovery/RemoteRecoveryTargetHandler.java
@@ -52,8 +52,6 @@ public class RemoteRecoveryTargetHandler implements RecoveryTargetHandler {
     public RemoteRecoveryTargetHandler(long recoveryId, ShardId shardId, TransportService transportService, DiscoveryNode targetNode,
                                        RecoverySettings recoverySettings, Consumer<Long> onSourceThrottle) {
         this.transportService = transportService;
-
-
         this.recoveryId = recoveryId;
         this.shardId = shardId;
         this.targetNode = targetNode;
diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/SharedFSRecoverySourceHandler.java b/core/src/main/java/org/elasticsearch/indices/recovery/SharedFSRecoverySourceHandler.java
index 591176f047a2f..509dd996d19a9 100644
--- a/core/src/main/java/org/elasticsearch/indices/recovery/SharedFSRecoverySourceHandler.java
+++ b/core/src/main/java/org/elasticsearch/indices/recovery/SharedFSRecoverySourceHandler.java
@@ -50,6 +50,7 @@ public RecoveryResponse recoverToTarget() throws IOException {
         boolean engineClosed = false;
         try {
             logger.trace("{} recovery [phase1] to {}: skipping phase 1 for shared filesystem", request.shardId(), request.targetNode());
+            long maxUnsafeAutoIdTimestamp = shard.segmentStats(false).getMaxUnsafeAutoIdTimestamp();
             if (request.isPrimaryRelocation()) {
                 logger.debug("[phase1] closing engine on primary for shared filesystem recovery");
                 try {
@@ -62,7 +63,7 @@ public RecoveryResponse recoverToTarget() throws IOException {
                     shard.failShard("failed to close engine (phase1)", e);
                 }
             }
-            prepareTargetForTranslog(0);
+            prepareTargetForTranslog(0, maxUnsafeAutoIdTimestamp);
             finalizeRecovery();
             return response;
         } catch (Exception e) {
diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/StartRecoveryRequest.java b/core/src/main/java/org/elasticsearch/indices/recovery/StartRecoveryRequest.java
index 9aa56fd8cb039..8e71491e211e8 100644
--- a/core/src/main/java/org/elasticsearch/indices/recovery/StartRecoveryRequest.java
+++ b/core/src/main/java/org/elasticsearch/indices/recovery/StartRecoveryRequest.java
@@ -19,8 +19,9 @@
 
 package org.elasticsearch.indices.recovery;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.routing.RecoverySource;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.index.shard.ShardId;
@@ -38,6 +39,8 @@ public class StartRecoveryRequest extends TransportRequest {
 
     private ShardId shardId;
 
+    private String targetAllocationId;
+
     private DiscoveryNode sourceNode;
 
     private DiscoveryNode targetNode;
@@ -55,9 +58,11 @@ public StartRecoveryRequest() {
      * @param sourceNode       The node to recover from
      * @param targetNode       The node to recover to
      */
-    public StartRecoveryRequest(ShardId shardId, DiscoveryNode sourceNode, DiscoveryNode targetNode, Store.MetadataSnapshot metadataSnapshot, boolean primaryRelocation, long recoveryId) {
+    public StartRecoveryRequest(ShardId shardId, String targetAllocationId, DiscoveryNode sourceNode, DiscoveryNode targetNode,
+                                Store.MetadataSnapshot metadataSnapshot, boolean primaryRelocation, long recoveryId) {
         this.recoveryId = recoveryId;
         this.shardId = shardId;
+        this.targetAllocationId = targetAllocationId;
         this.sourceNode = sourceNode;
         this.targetNode = targetNode;
         this.metadataSnapshot = metadataSnapshot;
@@ -72,6 +77,11 @@ public ShardId shardId() {
         return shardId;
     }
 
+    @Nullable
+    public String targetAllocationId() {
+        return targetAllocationId;
+    }
+
     public DiscoveryNode sourceNode() {
         return sourceNode;
     }
@@ -93,6 +103,11 @@ public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
         recoveryId = in.readLong();
         shardId = ShardId.readShardId(in);
+        if (in.getVersion().onOrAfter(Version.V_5_4_0)) {
+            targetAllocationId = in.readString();
+        } else {
+            targetAllocationId = null;
+        }
         sourceNode = new DiscoveryNode(in);
         targetNode = new DiscoveryNode(in);
         metadataSnapshot = new Store.MetadataSnapshot(in);
@@ -104,6 +119,9 @@ public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         out.writeLong(recoveryId);
         shardId.writeTo(out);
+        if (out.getVersion().onOrAfter(Version.V_5_4_0)) {
+            out.writeString(targetAllocationId);
+        }
         sourceNode.writeTo(out);
         targetNode.writeTo(out);
         metadataSnapshot.writeTo(out);
diff --git a/core/src/main/java/org/elasticsearch/indices/store/IndicesStore.java b/core/src/main/java/org/elasticsearch/indices/store/IndicesStore.java
index 439806b454b4c..120deb2f25f45 100644
--- a/core/src/main/java/org/elasticsearch/indices/store/IndicesStore.java
+++ b/core/src/main/java/org/elasticsearch/indices/store/IndicesStore.java
@@ -26,11 +26,13 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.cluster.ClusterStateObserver;
-import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.LocalClusterUpdateTask;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
+import org.elasticsearch.cluster.routing.RoutingNode;
+import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.collect.Tuple;
@@ -62,7 +64,10 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 
@@ -82,6 +87,9 @@ public class IndicesStore extends AbstractComponent implements ClusterStateListe
     private final TransportService transportService;
     private final ThreadPool threadPool;
 
+    // Cache successful shard deletion checks to prevent unnecessary file system lookups
+    private final Set<ShardId> folderNotFoundCache = new HashSet<>();
+
     private TimeValue deleteShardTimeout;
 
     @Inject
@@ -94,12 +102,20 @@ public IndicesStore(Settings settings, IndicesService indicesService,
         this.threadPool = threadPool;
         transportService.registerRequestHandler(ACTION_SHARD_EXISTS, ShardActiveRequest::new, ThreadPool.Names.SAME, new ShardActiveRequestHandler());
         this.deleteShardTimeout = INDICES_STORE_DELETE_SHARD_TIMEOUT.get(settings);
-        clusterService.addLast(this);
+        // Doesn't make sense to delete shards on non-data nodes
+        if (DiscoveryNode.isDataNode(settings)) {
+            // we double check nothing has changed when responses come back from other nodes.
+            // it's easier to do that check when the current cluster state is visible.
+            // also it's good in general to let things settle down
+            clusterService.addListener(this);
+        }
     }
 
     @Override
     public void close() {
-        clusterService.remove(this);
+        if (DiscoveryNode.isDataNode(settings)) {
+            clusterService.removeListener(this);
+        }
     }
 
     @Override
@@ -112,11 +128,31 @@ public void clusterChanged(ClusterChangedEvent event) {
             return;
         }
 
-        for (IndexRoutingTable indexRoutingTable : event.state().routingTable()) {
+        RoutingTable routingTable = event.state().routingTable();
+
+        // remove entries from cache that don't exist in the routing table anymore (either closed or deleted indices)
+        // - removing shard data of deleted indices is handled by IndicesClusterStateService
+        // - closed indices don't need to be removed from the cache but we do it anyway for code simplicity
+        for (Iterator<ShardId> it = folderNotFoundCache.iterator(); it.hasNext(); ) {
+            ShardId shardId = it.next();
+            if (routingTable.hasIndex(shardId.getIndex()) == false) {
+                it.remove();
+            }
+        }
+        // remove entries from cache which are allocated to this node
+        final String localNodeId = event.state().nodes().getLocalNodeId();
+        RoutingNode localRoutingNode = event.state().getRoutingNodes().node(localNodeId);
+        if (localRoutingNode != null) {
+            for (ShardRouting routing : localRoutingNode) {
+                folderNotFoundCache.remove(routing.shardId());
+            }
+        }
+
+        for (IndexRoutingTable indexRoutingTable : routingTable) {
             // Note, closed indices will not have any routing information, so won't be deleted
             for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) {
-                if (shardCanBeDeleted(event.state(), indexShardRoutingTable)) {
-                    ShardId shardId = indexShardRoutingTable.shardId();
+                ShardId shardId = indexShardRoutingTable.shardId();
+                if (folderNotFoundCache.contains(shardId) == false && shardCanBeDeleted(localNodeId, indexShardRoutingTable)) {
                     IndexService indexService = indicesService.indexService(indexRoutingTable.getIndex());
                     final IndexSettings indexSettings;
                     if (indexService == null) {
@@ -125,15 +161,33 @@ public void clusterChanged(ClusterChangedEvent event) {
                     } else {
                         indexSettings = indexService.getIndexSettings();
                     }
-                    if (indicesService.canDeleteShardContent(shardId, indexSettings)) {
-                        deleteShardIfExistElseWhere(event.state(), indexShardRoutingTable);
+                    IndicesService.ShardDeletionCheckResult shardDeletionCheckResult = indicesService.canDeleteShardContent(shardId, indexSettings);
+                    switch (shardDeletionCheckResult) {
+                        case FOLDER_FOUND_CAN_DELETE:
+                            deleteShardIfExistElseWhere(event.state(), indexShardRoutingTable);
+                            break;
+                        case NO_FOLDER_FOUND:
+                            folderNotFoundCache.add(shardId);
+                            break;
+                        case NO_LOCAL_STORAGE:
+                            assert false : "shard deletion only runs on data nodes which always have local storage";
+                            // nothing to do
+                            break;
+                        case STILL_ALLOCATED:
+                            // nothing to do
+                            break;
+                        case SHARED_FILE_SYSTEM:
+                            // nothing to do
+                            break;
+                        default:
+                            assert false : "unknown shard deletion check result: " + shardDeletionCheckResult;
                     }
                 }
             }
         }
     }
 
-    boolean shardCanBeDeleted(ClusterState state, IndexShardRoutingTable indexShardRoutingTable) {
+    static boolean shardCanBeDeleted(String localNodeId, IndexShardRoutingTable indexShardRoutingTable) {
         // a shard can be deleted if all its copies are active, and its not allocated on this node
         if (indexShardRoutingTable.size() == 0) {
             // should not really happen, there should always be at least 1 (primary) shard in a
@@ -143,27 +197,12 @@ boolean shardCanBeDeleted(ClusterState state, IndexShardRoutingTable indexShardR
 
         for (ShardRouting shardRouting : indexShardRoutingTable) {
             // be conservative here, check on started, not even active
-            if (!shardRouting.started()) {
+            if (shardRouting.started() == false) {
                 return false;
             }
 
-            // if the allocated or relocation node id doesn't exists in the cluster state  it may be a stale node,
-            // make sure we don't do anything with this until the routing table has properly been rerouted to reflect
-            // the fact that the node does not exists
-            DiscoveryNode node = state.nodes().get(shardRouting.currentNodeId());
-            if (node == null) {
-                return false;
-            }
-            if (shardRouting.relocatingNodeId() != null) {
-                node = state.nodes().get(shardRouting.relocatingNodeId());
-                if (node == null) {
-                    return false;
-                }
-            }
-
-            // check if shard is active on the current node or is getting relocated to the our node
-            String localNodeId = state.getNodes().getLocalNode().getId();
-            if (localNodeId.equals(shardRouting.currentNodeId()) || localNodeId.equals(shardRouting.relocatingNodeId())) {
+            // check if shard is active on the current node
+            if (localNodeId.equals(shardRouting.currentNodeId())) {
                 return false;
             }
         }
@@ -176,19 +215,13 @@ private void deleteShardIfExistElseWhere(ClusterState state, IndexShardRoutingTa
         String indexUUID = indexShardRoutingTable.shardId().getIndex().getUUID();
         ClusterName clusterName = state.getClusterName();
         for (ShardRouting shardRouting : indexShardRoutingTable) {
-            // Node can't be null, because otherwise shardCanBeDeleted() would have returned false
+            assert shardRouting.started() : "expected started shard but was " + shardRouting;
             DiscoveryNode currentNode = state.nodes().get(shardRouting.currentNodeId());
-            assert currentNode != null;
-
             requests.add(new Tuple<>(currentNode, new ShardActiveRequest(clusterName, indexUUID, shardRouting.shardId(), deleteShardTimeout)));
-            if (shardRouting.relocatingNodeId() != null) {
-                DiscoveryNode relocatingNode = state.nodes().get(shardRouting.relocatingNodeId());
-                assert relocatingNode != null;
-                requests.add(new Tuple<>(relocatingNode, new ShardActiveRequest(clusterName, indexUUID, shardRouting.shardId(), deleteShardTimeout)));
-            }
         }
 
-        ShardActiveResponseHandler responseHandler = new ShardActiveResponseHandler(indexShardRoutingTable.shardId(), state, requests.size());
+        ShardActiveResponseHandler responseHandler = new ShardActiveResponseHandler(indexShardRoutingTable.shardId(), state.getVersion(),
+            requests.size());
         for (Tuple<DiscoveryNode, ShardActiveRequest> request : requests) {
             logger.trace("{} sending shard active check to {}", request.v2().shardId, request.v1());
             transportService.sendRequest(request.v1(), ACTION_SHARD_EXISTS, request.v2(), responseHandler);
@@ -199,14 +232,14 @@ private class ShardActiveResponseHandler implements TransportResponseHandler<Sha
 
         private final ShardId shardId;
         private final int expectedActiveCopies;
-        private final ClusterState clusterState;
+        private final long clusterStateVersion;
         private final AtomicInteger awaitingResponses;
         private final AtomicInteger activeCopies;
 
-        public ShardActiveResponseHandler(ShardId shardId, ClusterState clusterState, int expectedActiveCopies) {
+        ShardActiveResponseHandler(ShardId shardId, long clusterStateVersion, int expectedActiveCopies) {
             this.shardId = shardId;
             this.expectedActiveCopies = expectedActiveCopies;
-            this.clusterState = clusterState;
+            this.clusterStateVersion = clusterStateVersion;
             this.awaitingResponses = new AtomicInteger(expectedActiveCopies);
             this.activeCopies = new AtomicInteger();
         }
@@ -248,29 +281,24 @@ private void allNodesResponded() {
             }
 
             ClusterState latestClusterState = clusterService.state();
-            if (clusterState.getVersion() != latestClusterState.getVersion()) {
-                logger.trace("not deleting shard {}, the latest cluster state version[{}] is not equal to cluster state before shard active api call [{}]", shardId, latestClusterState.getVersion(), clusterState.getVersion());
+            if (clusterStateVersion != latestClusterState.getVersion()) {
+                logger.trace("not deleting shard {}, the latest cluster state version[{}] is not equal to cluster state before shard active api call [{}]", shardId, latestClusterState.getVersion(), clusterStateVersion);
                 return;
             }
 
-            clusterService.submitStateUpdateTask("indices_store ([" + shardId + "] active fully on other nodes)", new ClusterStateUpdateTask() {
+            clusterService.submitStateUpdateTask("indices_store ([" + shardId + "] active fully on other nodes)", new LocalClusterUpdateTask() {
                 @Override
-                public boolean runOnlyOnMaster() {
-                    return false;
-                }
-
-                @Override
-                public ClusterState execute(ClusterState currentState) throws Exception {
-                    if (clusterState.getVersion() != currentState.getVersion()) {
-                        logger.trace("not deleting shard {}, the update task state version[{}] is not equal to cluster state before shard active api call [{}]", shardId, currentState.getVersion(), clusterState.getVersion());
-                        return currentState;
+                public ClusterTasksResult<LocalClusterUpdateTask> execute(ClusterState currentState) throws Exception {
+                    if (clusterStateVersion != currentState.getVersion()) {
+                        logger.trace("not deleting shard {}, the update task state version[{}] is not equal to cluster state before shard active api call [{}]", shardId, currentState.getVersion(), clusterStateVersion);
+                        return unchanged();
                     }
                     try {
                         indicesService.deleteShardStore("no longer used", shardId, currentState);
                     } catch (Exception ex) {
                         logger.debug((Supplier<?>) () -> new ParameterizedMessage("{} failed to delete unallocated shard, ignoring", shardId), ex);
                     }
-                    return currentState;
+                    return unchanged();
                 }
 
                 @Override
@@ -330,16 +358,13 @@ public void sendResult(boolean shardActive) {
                                 logger.error((Supplier<?>) () -> new ParameterizedMessage("failed send response for shard active while trying to delete shard {} - shard will probably not be removed", request.shardId), e);
                             }
                         }
-                    }, new ClusterStateObserver.ValidationPredicate() {
-                        @Override
-                        protected boolean validate(ClusterState newState) {
-                            // the shard is not there in which case we want to send back a false (shard is not active), so the cluster state listener must be notified
-                            // or the shard is active in which case we want to send back that the shard is active
-                            // here we could also evaluate the cluster state and get the information from there. we
-                            // don't do it because we would have to write another method for this that would have the same effect
-                            IndexShard indexShard = getShard(request);
-                            return indexShard == null || shardActive(indexShard);
-                        }
+                    }, newState -> {
+                        // the shard is not there in which case we want to send back a false (shard is not active), so the cluster state listener must be notified
+                        // or the shard is active in which case we want to send back that the shard is active
+                        // here we could also evaluate the cluster state and get the information from there. we
+                        // don't do it because we would have to write another method for this that would have the same effect
+                        IndexShard currentShard = getShard(request);
+                        return currentShard == null || shardActive(currentShard);
                     });
                 }
             }
@@ -374,7 +399,7 @@ private static class ShardActiveRequest extends TransportRequest {
         private String indexUUID;
         private ShardId shardId;
 
-        public ShardActiveRequest() {
+        ShardActiveRequest() {
         }
 
         ShardActiveRequest(ClusterName clusterName, String indexUUID, ShardId shardId, TimeValue timeout) {
diff --git a/core/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java b/core/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java
index 341b0e57858b0..acfd3e1233af6 100644
--- a/core/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java
+++ b/core/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java
@@ -39,6 +39,7 @@
 import org.elasticsearch.common.io.stream.Streamable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.gateway.AsyncShardFetch;
 import org.elasticsearch.index.IndexService;
@@ -133,7 +134,8 @@ private StoreFilesMetaData listStoreMetaData(ShardId shardId) throws IOException
                 // we may send this requests while processing the cluster state that recovered the index
                 // sometimes the request comes in before the local node processed that cluster state
                 // in such cases we can load it from disk
-                metaData = IndexMetaData.FORMAT.loadLatestState(logger, nodeEnv.indexPaths(shardId.getIndex()));
+                metaData = IndexMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY,
+                    nodeEnv.indexPaths(shardId.getIndex()));
             }
             if (metaData == null) {
                 logger.trace("{} node doesn't have meta data for the requests index, responding with empty", shardId);
@@ -160,11 +162,6 @@ private StoreFilesMetaData listStoreMetaData(ShardId shardId) throws IOException
         }
     }
 
-    @Override
-    protected boolean accumulateExceptions() {
-        return true;
-    }
-
     public static class StoreFilesMetaData implements Iterable<StoreFileMetaData>, Streamable {
         private ShardId shardId;
         Store.MetadataSnapshot metadataSnapshot;
diff --git a/core/src/main/java/org/elasticsearch/indices/ttl/IndicesTTLService.java b/core/src/main/java/org/elasticsearch/indices/ttl/IndicesTTLService.java
index 3eea62273ade4..d7f1ef6993b14 100644
--- a/core/src/main/java/org/elasticsearch/indices/ttl/IndicesTTLService.java
+++ b/core/src/main/java/org/elasticsearch/indices/ttl/IndicesTTLService.java
@@ -115,7 +115,7 @@ private class PurgerThread extends Thread {
         private final CountDownLatch shutdownLatch = new CountDownLatch(1);
 
 
-        public PurgerThread(String name, TimeValue interval) {
+        PurgerThread(String name, TimeValue interval) {
             super(name);
             setDaemon(true);
             this.notifier = new Notifier(interval);
@@ -197,7 +197,8 @@ public TimeValue getInterval() {
 
     private void purgeShards(List<IndexShard> shardsToPurge) {
         for (IndexShard shardToPurge : shardsToPurge) {
-            Query query = shardToPurge.mapperService().fullName(TTLFieldMapper.NAME).rangeQuery(null, System.currentTimeMillis(), false, true);
+            Query query = shardToPurge.mapperService().fullName(TTLFieldMapper.NAME).rangeQuery(null, System.currentTimeMillis(), false,
+                    true, null);
             Engine.Searcher searcher = shardToPurge.acquireSearcher("indices_ttl");
             try {
                 logger.debug("[{}][{}] purging shard", shardToPurge.routingEntry().index(), shardToPurge.routingEntry().id());
@@ -226,7 +227,7 @@ private static class DocToPurge {
         public final long version;
         public final String routing;
 
-        public DocToPurge(String type, String id, long version, String routing) {
+        DocToPurge(String type, String id, long version, String routing) {
             this.type = type;
             this.id = id;
             this.version = version;
@@ -239,7 +240,7 @@ private class ExpiredDocsCollector extends SimpleCollector {
         private List<DocToPurge> docsToPurge = new ArrayList<>();
         private NumericDocValues versions;
 
-        public ExpiredDocsCollector() {
+        ExpiredDocsCollector() {
         }
 
         @Override
@@ -319,7 +320,7 @@ private static final class Notifier {
         private final Condition condition = lock.newCondition();
         private volatile TimeValue timeout;
 
-        public Notifier(TimeValue timeout) {
+        Notifier(TimeValue timeout) {
             assert timeout != null;
             this.timeout = timeout;
         }
diff --git a/core/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java b/core/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java
index 908e344698066..08669188a9fdb 100644
--- a/core/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java
+++ b/core/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java
@@ -224,7 +224,13 @@ public static Object readObject(String processorType, String processorTag, Map<S
 
     public static ElasticsearchException newConfigurationException(String processorType, String processorTag,
                                                                         String propertyName, String reason) {
-        ElasticsearchParseException exception = new ElasticsearchParseException("[" + propertyName + "] " + reason);
+        String msg;
+        if (propertyName == null) {
+           msg = reason;
+        } else {
+            msg = "[" + propertyName + "] " + reason;
+        }
+        ElasticsearchParseException exception = new ElasticsearchParseException(msg);
         addHeadersToException(exception, processorType, processorTag, propertyName);
         return exception;
     }
@@ -238,15 +244,24 @@ public static ElasticsearchException newConfigurationException(String processorT
 
     public static List<Processor> readProcessorConfigs(List<Map<String, Map<String, Object>>> processorConfigs,
                                                        Map<String, Processor.Factory> processorFactories) throws Exception {
+        Exception exception = null;
         List<Processor> processors = new ArrayList<>();
         if (processorConfigs != null) {
             for (Map<String, Map<String, Object>> processorConfigWithKey : processorConfigs) {
                 for (Map.Entry<String, Map<String, Object>> entry : processorConfigWithKey.entrySet()) {
-                    processors.add(readProcessor(processorFactories, entry.getKey(), entry.getValue()));
+                    try {
+                        processors.add(readProcessor(processorFactories, entry.getKey(), entry.getValue()));
+                    } catch (Exception e) {
+                        exception = ExceptionsHelper.useOrSuppress(exception, e);
+                    }
                 }
             }
         }
 
+        if (exception != null) {
+            throw exception;
+        }
+
         return processors;
     }
 
@@ -274,6 +289,7 @@ private static void addHeadersToException(ElasticsearchException exception, Stri
 
     public static Processor readProcessor(Map<String, Processor.Factory> processorFactories,
                                            String type, Map<String, Object> config) throws Exception {
+        String tag = ConfigurationUtils.readOptionalStringProperty(null, null, config, TAG_KEY);
         Processor.Factory factory = processorFactories.get(type);
         if (factory != null) {
             boolean ignoreFailure = ConfigurationUtils.readBooleanProperty(null, null, config, "ignore_failure", false);
@@ -281,7 +297,6 @@ public static Processor readProcessor(Map<String, Processor.Factory> processorFa
                 ConfigurationUtils.readOptionalList(null, null, config, Pipeline.ON_FAILURE_KEY);
 
             List<Processor> onFailureProcessors = readProcessorConfigs(onFailureProcessorConfigs, processorFactories);
-            String tag = ConfigurationUtils.readOptionalStringProperty(null, null, config, TAG_KEY);
 
             if (onFailureProcessorConfigs != null && onFailureProcessors.isEmpty()) {
                 throw newConfigurationException(type, tag, Pipeline.ON_FAILURE_KEY,
@@ -303,6 +318,6 @@ public static Processor readProcessor(Map<String, Processor.Factory> processorFa
                 throw newConfigurationException(type, tag, null, e);
             }
         }
-        throw new ElasticsearchParseException("No processor type exists with name [" + type + "]");
+        throw newConfigurationException(type, tag, null, "No processor type exists with name [" + type + "]");
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/ingest/IngestDocument.java b/core/src/main/java/org/elasticsearch/ingest/IngestDocument.java
index 8010c6e6c3f05..937a1d5e23a69 100644
--- a/core/src/main/java/org/elasticsearch/ingest/IngestDocument.java
+++ b/core/src/main/java/org/elasticsearch/ingest/IngestDocument.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.ingest;
 
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.IndexFieldMapper;
 import org.elasticsearch.index.mapper.ParentFieldMapper;
@@ -29,18 +30,17 @@
 import org.elasticsearch.index.mapper.TimestampFieldMapper;
 import org.elasticsearch.index.mapper.TypeFieldMapper;
 
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Base64;
 import java.util.Date;
+import java.util.EnumMap;
 import java.util.HashMap;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
-import java.util.TimeZone;
 
 /**
  * Represents a single document being captured before indexing and holds the source and metadata (like id, type and index).
@@ -58,6 +58,11 @@ public final class IngestDocument {
 
     public IngestDocument(String index, String type, String id, String routing, String parent, String timestamp,
                           String ttl, Map<String, Object> source) {
+        this(index, type, id, routing, parent, timestamp, ttl, source, false);
+    }
+
+    public IngestDocument(String index, String type, String id, String routing, String parent, String timestamp,
+                          String ttl, Map<String, Object> source, boolean newDateFormat) {
         this.sourceAndMetadata = new HashMap<>();
         this.sourceAndMetadata.putAll(source);
         this.sourceAndMetadata.put(MetaData.INDEX.getFieldName(), index);
@@ -77,9 +82,11 @@ public IngestDocument(String index, String type, String id, String routing, Stri
         }
 
         this.ingestMetadata = new HashMap<>();
-        DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZZ", Locale.ROOT);
-        df.setTimeZone(TimeZone.getTimeZone("UTC"));
-        this.ingestMetadata.put(TIMESTAMP, df.format(new Date()));
+        if (newDateFormat) {
+            this.ingestMetadata.put(TIMESTAMP, ZonedDateTime.now(ZoneOffset.UTC));
+        } else {
+            this.ingestMetadata.put(TIMESTAMP, new Date());
+        }
     }
 
     /**
@@ -116,6 +123,28 @@ public <T> T getFieldValue(String path, Class<T> clazz) {
         return cast(path, context, clazz);
     }
 
+    /**
+     * Returns the value contained in the document for the provided path
+     *
+     * @param path The path within the document in dot-notation
+     * @param clazz The expected class of the field value
+     * @param ignoreMissing The flag to determine whether to throw an exception when `path` is not found in the document.
+     * @return the value for the provided path if existing, null otherwise.
+     * @throws IllegalArgumentException only if ignoreMissing is false and the path is null, empty, invalid, if the field doesn't exist
+     * or if the field that is found at the provided path is not of the expected type.
+     */
+    public <T> T getFieldValue(String path, Class<T> clazz, boolean ignoreMissing) {
+        try {
+            return getFieldValue(path, clazz);
+        } catch (IllegalArgumentException e) {
+            if (ignoreMissing && hasField(path) != true) {
+                return null;
+            } else {
+                throw e;
+            }
+        }
+    }
+
     /**
      * Returns the value contained in the document with the provided templated path
      * @param pathTemplate The path within the document in dot-notation
@@ -138,8 +167,24 @@ public <T> T getFieldValue(TemplateService.Template pathTemplate, Class<T> clazz
      * or if the field that is found at the provided path is not of the expected type.
      */
     public byte[] getFieldValueAsBytes(String path) {
-        Object object = getFieldValue(path, Object.class);
-        if (object instanceof byte[]) {
+        return getFieldValueAsBytes(path, false);
+    }
+
+    /**
+     * Returns the value contained in the document for the provided path as a byte array.
+     * If the path value is a string, a base64 decode operation will happen.
+     * If the path value is a byte array, it is just returned
+     * @param path The path within the document in dot-notation
+     * @param ignoreMissing The flag to determine whether to throw an exception when `path` is not found in the document.
+     * @return the byte array for the provided path if existing
+     * @throws IllegalArgumentException if the path is null, empty, invalid, if the field doesn't exist
+     * or if the field that is found at the provided path is not of the expected type.
+     */
+    public byte[] getFieldValueAsBytes(String path, boolean ignoreMissing) {
+        Object object = getFieldValue(path, Object.class, ignoreMissing);
+        if (object == null) {
+            return null;
+        } else if (object instanceof byte[]) {
             return (byte[]) object;
         } else if (object instanceof String) {
             return Base64.getDecoder().decode(object.toString());
@@ -531,7 +576,7 @@ private Map<String, Object> createTemplateModel() {
      * Metadata fields that used to be accessible as ordinary top level fields will be removed as part of this call.
      */
     public Map<MetaData, String> extractMetadata() {
-        Map<MetaData, String> metadataMap = new HashMap<>();
+        Map<MetaData, String> metadataMap = new EnumMap<>(MetaData.class);
         for (MetaData metaData : MetaData.values()) {
             metadataMap.put(metaData, cast(metaData.getFieldName(), sourceAndMetadata.remove(metaData.getFieldName()), String.class));
         }
@@ -582,6 +627,11 @@ private static Object deepCopy(Object value) {
             value instanceof Long || value instanceof Float ||
             value instanceof Double || value instanceof Boolean) {
             return value;
+        } else if (value instanceof Date) {
+            return ((Date) value).clone();
+        } else if (value instanceof ZonedDateTime) {
+            ZonedDateTime zonedDateTime = (ZonedDateTime) value;
+            return ZonedDateTime.of(zonedDateTime.toLocalDate(), zonedDateTime.toLocalTime(), zonedDateTime.getZone());
         } else {
             throw new IllegalArgumentException("unexpected value type [" + value.getClass() + "]");
         }
diff --git a/core/src/main/java/org/elasticsearch/ingest/IngestMetadata.java b/core/src/main/java/org/elasticsearch/ingest/IngestMetadata.java
index 9ad369e22d421..ca8a5df845014 100644
--- a/core/src/main/java/org/elasticsearch/ingest/IngestMetadata.java
+++ b/core/src/main/java/org/elasticsearch/ingest/IngestMetadata.java
@@ -21,10 +21,9 @@
 
 import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.cluster.DiffableUtils;
+import org.elasticsearch.cluster.NamedDiff;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ObjectParser;
@@ -45,9 +44,8 @@
 public final class IngestMetadata implements MetaData.Custom {
 
     public static final String TYPE = "ingest";
-    public static final IngestMetadata PROTO = new IngestMetadata();
     private static final ParseField PIPELINES_FIELD = new ParseField("pipeline");
-    private static final ObjectParser<List<PipelineConfiguration>, ParseFieldMatcherSupplier> INGEST_METADATA_PARSER = new ObjectParser<>(
+    private static final ObjectParser<List<PipelineConfiguration>, Void> INGEST_METADATA_PARSER = new ObjectParser<>(
             "ingest_metadata", ArrayList::new);
 
     static {
@@ -67,7 +65,7 @@ public IngestMetadata(Map<String, PipelineConfiguration> pipelines) {
     }
 
     @Override
-    public String type() {
+    public String getWriteableName() {
         return TYPE;
     }
 
@@ -75,15 +73,14 @@ public Map<String, PipelineConfiguration> getPipelines() {
         return pipelines;
     }
 
-    @Override
-    public IngestMetadata readFrom(StreamInput in) throws IOException {
+    public IngestMetadata(StreamInput in) throws IOException {
         int size = in.readVInt();
         Map<String, PipelineConfiguration> pipelines = new HashMap<>(size);
         for (int i = 0; i < size; i++) {
-            PipelineConfiguration pipeline = PipelineConfiguration.readPipelineConfiguration(in);
+            PipelineConfiguration pipeline = PipelineConfiguration.readFrom(in);
             pipelines.put(pipeline.getId(), pipeline);
         }
-        return new IngestMetadata(pipelines);
+        this.pipelines = Collections.unmodifiableMap(pipelines);
     }
 
     @Override
@@ -94,10 +91,9 @@ public void writeTo(StreamOutput out) throws IOException {
         }
     }
 
-    @Override
-    public IngestMetadata fromXContent(XContentParser parser) throws IOException {
+    public static IngestMetadata fromXContent(XContentParser parser) throws IOException {
         Map<String, PipelineConfiguration> pipelines = new HashMap<>();
-        List<PipelineConfiguration> configs = INGEST_METADATA_PARSER.parse(parser, () -> ParseFieldMatcher.STRICT);
+        List<PipelineConfiguration> configs = INGEST_METADATA_PARSER.parse(parser, null);
         for (PipelineConfiguration pipeline : configs) {
             pipelines.put(pipeline.getId(), pipeline);
         }
@@ -116,7 +112,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
 
     @Override
     public EnumSet<MetaData.XContentContext> context() {
-        return MetaData.API_AND_GATEWAY;
+        return MetaData.ALL_CONTEXTS;
     }
 
     @Override
@@ -124,12 +120,11 @@ public Diff<MetaData.Custom> diff(MetaData.Custom before) {
         return new IngestMetadataDiff((IngestMetadata) before, this);
     }
 
-    @Override
-    public Diff<MetaData.Custom> readDiffFrom(StreamInput in) throws IOException {
+    public static NamedDiff<MetaData.Custom> readDiffFrom(StreamInput in) throws IOException {
         return new IngestMetadataDiff(in);
     }
 
-    static class IngestMetadataDiff implements Diff<MetaData.Custom> {
+    static class IngestMetadataDiff implements NamedDiff<MetaData.Custom> {
 
         final Diff<Map<String, PipelineConfiguration>> pipelines;
 
@@ -137,8 +132,9 @@ static class IngestMetadataDiff implements Diff<MetaData.Custom> {
             this.pipelines = DiffableUtils.diff(before.pipelines, after.pipelines, DiffableUtils.getStringKeySerializer());
         }
 
-        public IngestMetadataDiff(StreamInput in) throws IOException {
-            pipelines = DiffableUtils.readJdkMapDiff(in, DiffableUtils.getStringKeySerializer(), PipelineConfiguration.PROTOTYPE);
+        IngestMetadataDiff(StreamInput in) throws IOException {
+            pipelines = DiffableUtils.readJdkMapDiff(in, DiffableUtils.getStringKeySerializer(), PipelineConfiguration::readFrom,
+                PipelineConfiguration::readDiffFrom);
         }
 
         @Override
@@ -150,6 +146,11 @@ public MetaData.Custom apply(MetaData.Custom part) {
         public void writeTo(StreamOutput out) throws IOException {
             pipelines.writeTo(out);
         }
+
+        @Override
+        public String getWriteableName() {
+            return TYPE;
+        }
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/ingest/IngestService.java b/core/src/main/java/org/elasticsearch/ingest/IngestService.java
index 5249ed7a7dc84..1455e37588a8c 100644
--- a/core/src/main/java/org/elasticsearch/ingest/IngestService.java
+++ b/core/src/main/java/org/elasticsearch/ingest/IngestService.java
@@ -25,6 +25,8 @@
 import java.util.List;
 import java.util.Map;
 
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.analysis.AnalysisRegistry;
@@ -32,15 +34,19 @@
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.threadpool.ThreadPool;
 
+import static org.elasticsearch.common.settings.Setting.Property;
+
 /**
  * Holder class for several ingest related services.
  */
 public class IngestService {
+    public static final Setting<Boolean> NEW_INGEST_DATE_FORMAT =
+        Setting.boolSetting("ingest.new_date_format", false, Property.NodeScope, Property.Dynamic, Property.Deprecated);
 
     private final PipelineStore pipelineStore;
     private final PipelineExecutionService pipelineExecutionService;
 
-    public IngestService(Settings settings, ThreadPool threadPool,
+    public IngestService(ClusterSettings clusterSettings, Settings settings, ThreadPool threadPool,
                          Environment env, ScriptService scriptService, AnalysisRegistry analysisRegistry,
                          List<IngestPlugin> ingestPlugins) {
 
@@ -56,7 +62,7 @@ public IngestService(Settings settings, ThreadPool threadPool,
                 }
             }
         }
-        this.pipelineStore = new PipelineStore(settings, Collections.unmodifiableMap(processorFactories));
+        this.pipelineStore = new PipelineStore(clusterSettings, settings, Collections.unmodifiableMap(processorFactories));
         this.pipelineExecutionService = new PipelineExecutionService(pipelineStore, threadPool);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/ingest/IngestStats.java b/core/src/main/java/org/elasticsearch/ingest/IngestStats.java
index dee806e0230e7..add02a5da9023 100644
--- a/core/src/main/java/org/elasticsearch/ingest/IngestStats.java
+++ b/core/src/main/java/org/elasticsearch/ingest/IngestStats.java
@@ -54,7 +54,7 @@ public IngestStats(StreamInput in) throws IOException {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         totalStats.writeTo(out);
-        out.writeVLong(statsPerPipeline.size());
+        out.writeVInt(statsPerPipeline.size());
         for (Map.Entry<String, Stats> entry : statsPerPipeline.entrySet()) {
             out.writeString(entry.getKey());
             entry.getValue().writeTo(out);
diff --git a/core/src/main/java/org/elasticsearch/ingest/InternalTemplateService.java b/core/src/main/java/org/elasticsearch/ingest/InternalTemplateService.java
index 677be3b4b0cc6..b5aa2dbc51a8a 100644
--- a/core/src/main/java/org/elasticsearch/ingest/InternalTemplateService.java
+++ b/core/src/main/java/org/elasticsearch/ingest/InternalTemplateService.java
@@ -19,15 +19,16 @@
 
 package org.elasticsearch.ingest;
 
+import java.util.Collections;
+import java.util.Map;
+
+import org.apache.logging.log4j.util.Supplier;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.script.CompiledScript;
-import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptService;
-
-import java.util.Collections;
-import java.util.Map;
+import org.elasticsearch.script.ScriptType;
+import org.elasticsearch.template.CompiledTemplate;
 
 public class InternalTemplateService implements TemplateService {
 
@@ -42,20 +43,12 @@ public Template compile(String template) {
         int mustacheStart = template.indexOf("{{");
         int mustacheEnd = template.indexOf("}}");
         if (mustacheStart != -1 && mustacheEnd != -1 && mustacheStart < mustacheEnd) {
-            Script script = new Script(template, ScriptService.ScriptType.INLINE, "mustache", Collections.emptyMap());
-            CompiledScript compiledScript = scriptService.compile(
-                script,
-                ScriptContext.Standard.INGEST,
-                Collections.emptyMap());
+            Script script = new Script(ScriptType.INLINE, "mustache", template, Collections.emptyMap());
+            CompiledTemplate compiledTemplate = scriptService.compileTemplate(script, ScriptContext.Standard.INGEST);
             return new Template() {
                 @Override
                 public String execute(Map<String, Object> model) {
-                    ExecutableScript executableScript = scriptService.executable(compiledScript, model);
-                    Object result = executableScript.run();
-                    if (result instanceof BytesReference) {
-                        return ((BytesReference) result).utf8ToString();
-                    }
-                    return String.valueOf(result);
+                    return compiledTemplate.run(model).utf8ToString();
                 }
 
                 @Override
@@ -72,7 +65,7 @@ class StringTemplate implements Template {
 
         private final String value;
 
-        public StringTemplate(String value) {
+        StringTemplate(String value) {
             this.value = value;
         }
 
diff --git a/core/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java b/core/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java
index 56f4e2c6a79bc..1d7ba958f1498 100644
--- a/core/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java
+++ b/core/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java
@@ -19,60 +19,61 @@
 
 package org.elasticsearch.ingest;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.cluster.AbstractDiffable;
+import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ContextParser;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
 
 import java.io.IOException;
 import java.util.Map;
-import java.util.function.BiFunction;
+import java.util.Objects;
 
 /**
  * Encapsulates a pipeline's id and configuration as a blob
  */
 public final class PipelineConfiguration extends AbstractDiffable<PipelineConfiguration> implements ToXContent {
 
-    static final PipelineConfiguration PROTOTYPE = new PipelineConfiguration(null, null);
-
-    public static PipelineConfiguration readPipelineConfiguration(StreamInput in) throws IOException {
-        return PROTOTYPE.readFrom(in);
-    }
-    private static final ObjectParser<Builder, ParseFieldMatcherSupplier> PARSER = new ObjectParser<>("pipeline_config", Builder::new);
+    private static final ObjectParser<Builder, Void> PARSER = new ObjectParser<>("pipeline_config", Builder::new);
     static {
         PARSER.declareString(Builder::setId, new ParseField("id"));
         PARSER.declareField((parser, builder, aVoid) -> {
             XContentBuilder contentBuilder = XContentBuilder.builder(parser.contentType().xContent());
             XContentHelper.copyCurrentStructure(contentBuilder.generator(), parser);
-            builder.setConfig(contentBuilder.bytes());
+            builder.setConfig(contentBuilder.bytes(), contentBuilder.contentType());
         }, new ParseField("config"), ObjectParser.ValueType.OBJECT);
+
     }
 
-    public static BiFunction<XContentParser, ParseFieldMatcherSupplier, PipelineConfiguration> getParser() {
-        return (p, c) -> PARSER.apply(p ,c).build();
+    public static ContextParser<Void, PipelineConfiguration> getParser() {
+        return (parser, context) -> PARSER.apply(parser, null).build();
     }
     private static class Builder {
 
         private String id;
         private BytesReference config;
+        private XContentType xContentType;
 
         void setId(String id) {
             this.id = id;
         }
 
-        void setConfig(BytesReference config) {
+        void setConfig(BytesReference config, XContentType xContentType) {
             this.config = config;
+            this.xContentType = xContentType;
         }
 
         PipelineConfiguration build() {
-            return new PipelineConfiguration(id, config);
+            return new PipelineConfiguration(id, config, xContentType);
         }
     }
 
@@ -81,10 +82,12 @@ PipelineConfiguration build() {
     // and the way the map of maps config is read requires a deep copy (it removes instead of gets entries to check for unused options)
     // also the get pipeline api just directly returns this to the caller
     private final BytesReference config;
+    private final XContentType xContentType;
 
-    public PipelineConfiguration(String id, BytesReference config) {
-        this.id = id;
-        this.config = config;
+    public PipelineConfiguration(String id, BytesReference config, XContentType xContentType) {
+        this.id = Objects.requireNonNull(id);
+        this.config = Objects.requireNonNull(config);
+        this.xContentType = Objects.requireNonNull(xContentType);
     }
 
     public String getId() {
@@ -92,7 +95,17 @@ public String getId() {
     }
 
     public Map<String, Object> getConfigAsMap() {
-        return XContentHelper.convertToMap(config, true).v2();
+        return XContentHelper.convertToMap(config, true, xContentType).v2();
+    }
+
+    // pkg-private for tests
+    XContentType getXContentType() {
+        return xContentType;
+    }
+
+    // pkg-private for tests
+    BytesReference getConfig() {
+        return config;
     }
 
     @Override
@@ -104,15 +117,27 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         return builder;
     }
 
-    @Override
-    public PipelineConfiguration readFrom(StreamInput in) throws IOException {
-        return new PipelineConfiguration(in.readString(), in.readBytesReference());
+    public static PipelineConfiguration readFrom(StreamInput in) throws IOException {
+        if (in.getVersion().onOrAfter(Version.V_5_3_0)) {
+            return new PipelineConfiguration(in.readString(), in.readBytesReference(), XContentType.readFrom(in));
+        } else {
+            final String id = in.readString();
+            final BytesReference config = in.readBytesReference();
+            return new PipelineConfiguration(id, config, XContentFactory.xContentType(config));
+        }
+    }
+
+    public static Diff<PipelineConfiguration> readDiffFrom(StreamInput in) throws IOException {
+        return readDiffFrom(PipelineConfiguration::readFrom, in);
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(id);
         out.writeBytesReference(config);
+        if (out.getVersion().onOrAfter(Version.V_5_3_0)) {
+            xContentType.writeTo(out);
+        }
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/ingest/PipelineExecutionService.java b/core/src/main/java/org/elasticsearch/ingest/PipelineExecutionService.java
index e714663653422..86fe0af75d2b0 100644
--- a/core/src/main/java/org/elasticsearch/ingest/PipelineExecutionService.java
+++ b/core/src/main/java/org/elasticsearch/ingest/PipelineExecutionService.java
@@ -19,10 +19,10 @@
 
 package org.elasticsearch.ingest;
 
-import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.cluster.ClusterChangedEvent;
-import org.elasticsearch.cluster.ClusterStateListener;
+import org.elasticsearch.cluster.ClusterStateApplier;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.metrics.CounterMetric;
 import org.elasticsearch.common.metrics.MeanMetric;
@@ -38,7 +38,7 @@
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 
-public class PipelineExecutionService implements ClusterStateListener {
+public class PipelineExecutionService implements ClusterStateApplier {
 
     private final PipelineStore store;
     private final ThreadPool threadPool;
@@ -68,7 +68,7 @@ protected void doRun() throws Exception {
         });
     }
 
-    public void executeBulkRequest(Iterable<ActionRequest<?>> actionRequests,
+    public void executeBulkRequest(Iterable<DocWriteRequest> actionRequests,
                                    BiConsumer<IndexRequest, Exception> itemFailureHandler,
                                    Consumer<Exception> completionHandler) {
         threadPool.executor(ThreadPool.Names.BULK).execute(new AbstractRunnable() {
@@ -80,7 +80,7 @@ public void onFailure(Exception e) {
 
             @Override
             protected void doRun() throws Exception {
-                for (ActionRequest actionRequest : actionRequests) {
+                for (DocWriteRequest actionRequest : actionRequests) {
                     if ((actionRequest instanceof IndexRequest)) {
                         IndexRequest indexRequest = (IndexRequest) actionRequest;
                         if (Strings.hasText(indexRequest.getPipeline())) {
@@ -112,7 +112,7 @@ public IngestStats stats() {
     }
 
     @Override
-    public void clusterChanged(ClusterChangedEvent event) {
+    public void applyClusterState(ClusterChangedEvent event) {
         IngestMetadata ingestMetadata = event.state().getMetaData().custom(IngestMetadata.TYPE);
         if (ingestMetadata != null) {
             updatePipelineStats(ingestMetadata);
@@ -162,7 +162,8 @@ private void innerExecute(IndexRequest indexRequest, Pipeline pipeline) throws E
             String timestamp = indexRequest.timestamp();
             String ttl = indexRequest.ttl() == null ? null : indexRequest.ttl().toString();
             Map<String, Object> sourceAsMap = indexRequest.sourceAsMap();
-            IngestDocument ingestDocument = new IngestDocument(index, type, id, routing, parent, timestamp, ttl, sourceAsMap);
+            IngestDocument ingestDocument = new IngestDocument(index, type, id, routing, parent, timestamp, ttl,
+                sourceAsMap, store.isNewIngestDateFormat());
             pipeline.execute(ingestDocument);
 
             Map<IngestDocument.MetaData, String> metadataMap = ingestDocument.extractMetadata();
diff --git a/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java b/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java
index 94850674e755c..327ab18907c56 100644
--- a/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java
+++ b/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java
@@ -19,13 +19,6 @@
 
 package org.elasticsearch.ingest;
 
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.ResourceNotFoundException;
@@ -36,19 +29,30 @@
 import org.elasticsearch.cluster.AckedClusterStateUpdateTask;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.ClusterStateListener;
+import org.elasticsearch.cluster.ClusterStateApplier;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.regex.Regex;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentHelper;
 
-public class PipelineStore extends AbstractComponent implements ClusterStateListener {
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
+public class PipelineStore extends AbstractComponent implements ClusterStateApplier {
 
     private final Pipeline.Factory factory = new Pipeline.Factory();
     private final Map<String, Processor.Factory> processorFactories;
+    private volatile boolean newIngestDateFormat;
 
     // Ideally this should be in IngestMetadata class, but we don't have the processor factories around there.
     // We know of all the processor factories when a node with all its plugin have been initialized. Also some
@@ -56,13 +60,19 @@ public class PipelineStore extends AbstractComponent implements ClusterStateList
     // are loaded, so in the cluster state we just save the pipeline config and here we keep the actual pipelines around.
     volatile Map<String, Pipeline> pipelines = new HashMap<>();
 
-    public PipelineStore(Settings settings, Map<String, Processor.Factory> processorFactories) {
+    public PipelineStore(ClusterSettings clusterSettings, Settings settings, Map<String, Processor.Factory> processorFactories) {
         super(settings);
         this.processorFactories = processorFactories;
+        this.newIngestDateFormat = IngestService.NEW_INGEST_DATE_FORMAT.get(settings);
+        clusterSettings.addSettingsUpdateConsumer(IngestService.NEW_INGEST_DATE_FORMAT, this::setNewIngestDateFormat);
+    }
+
+    private void setNewIngestDateFormat(Boolean newIngestDateFormat) {
+        this.newIngestDateFormat = newIngestDateFormat;
     }
 
     @Override
-    public void clusterChanged(ClusterChangedEvent event) {
+    public void applyClusterState(ClusterChangedEvent event) {
         innerUpdatePipelines(event.previousState(), event.state());
     }
 
@@ -111,17 +121,26 @@ ClusterState innerDelete(DeletePipelineRequest request, ClusterState currentStat
             return currentState;
         }
         Map<String, PipelineConfiguration> pipelines = currentIngestMetadata.getPipelines();
-        if (pipelines.containsKey(request.getId()) == false) {
+        Set<String> toRemove = new HashSet<>();
+        for (String pipelineKey : pipelines.keySet()) {
+            if (Regex.simpleMatch(request.getId(), pipelineKey)) {
+                toRemove.add(pipelineKey);
+            }
+        }
+        if (toRemove.isEmpty() && Regex.isMatchAllPattern(request.getId()) == false) {
             throw new ResourceNotFoundException("pipeline [{}] is missing", request.getId());
-        } else {
-            pipelines = new HashMap<>(pipelines);
-            pipelines.remove(request.getId());
-            ClusterState.Builder newState = ClusterState.builder(currentState);
-            newState.metaData(MetaData.builder(currentState.getMetaData())
-                    .putCustom(IngestMetadata.TYPE, new IngestMetadata(pipelines))
-                    .build());
-            return newState.build();
+        } else if (toRemove.isEmpty()) {
+            return currentState;
         }
+        final Map<String, PipelineConfiguration> pipelinesCopy = new HashMap<>(pipelines);
+        for (String key : toRemove) {
+            pipelinesCopy.remove(key);
+        }
+        ClusterState.Builder newState = ClusterState.builder(currentState);
+        newState.metaData(MetaData.builder(currentState.getMetaData())
+                .putCustom(IngestMetadata.TYPE, new IngestMetadata(pipelinesCopy))
+                .build());
+        return newState.build();
     }
 
     /**
@@ -151,14 +170,14 @@ void validatePipeline(Map<DiscoveryNode, IngestInfo> ingestInfos, PutPipelineReq
             throw new IllegalStateException("Ingest info is empty");
         }
 
-        Map<String, Object> pipelineConfig = XContentHelper.convertToMap(request.getSource(), false).v2();
+        Map<String, Object> pipelineConfig = XContentHelper.convertToMap(request.getSource(), false, request.getXContentType()).v2();
         Pipeline pipeline = factory.create(request.getId(), pipelineConfig, processorFactories);
-        List<IllegalArgumentException> exceptions = new ArrayList<>();
+        List<Exception> exceptions = new ArrayList<>();
         for (Processor processor : pipeline.flattenAllProcessors()) {
             for (Map.Entry<DiscoveryNode, IngestInfo> entry : ingestInfos.entrySet()) {
                 if (entry.getValue().containsProcessor(processor.getType()) == false) {
                     String message = "Processor type [" + processor.getType() + "] is not installed on node [" + entry.getKey() + "]";
-                    exceptions.add(new IllegalArgumentException(message));
+                    exceptions.add(ConfigurationUtils.newConfigurationException(processor.getType(), processor.getTag(), null, message));
                 }
             }
         }
@@ -174,7 +193,7 @@ ClusterState innerPut(PutPipelineRequest request, ClusterState currentState) {
             pipelines = new HashMap<>();
         }
 
-        pipelines.put(request.getId(), new PipelineConfiguration(request.getId(), request.getSource()));
+        pipelines.put(request.getId(), new PipelineConfiguration(request.getId(), request.getSource(), request.getXContentType()));
         ClusterState.Builder newState = ClusterState.builder(currentState);
         newState.metaData(MetaData.builder(currentState.getMetaData())
             .putCustom(IngestMetadata.TYPE, new IngestMetadata(pipelines))
@@ -193,6 +212,10 @@ public Map<String, Processor.Factory> getProcessorFactories() {
         return processorFactories;
     }
 
+    public boolean isNewIngestDateFormat() {
+        return newIngestDateFormat;
+    }
+
     /**
      * @return pipeline configuration specified by id. If multiple ids or wildcards are specified multiple pipelines
      * may be returned
diff --git a/core/src/main/java/org/elasticsearch/ingest/Processor.java b/core/src/main/java/org/elasticsearch/ingest/Processor.java
index af4ea954dd546..228ca5f4930e4 100644
--- a/core/src/main/java/org/elasticsearch/ingest/Processor.java
+++ b/core/src/main/java/org/elasticsearch/ingest/Processor.java
@@ -29,6 +29,8 @@
 /**
  * A processor implementation may modify the data belonging to a document.
  * Whether changes are made and what exactly is modified is up to the implementation.
+ *
+ * Processors may get called concurrently and thus need to be thread-safe.
  */
 public interface Processor {
 
diff --git a/core/src/main/java/org/elasticsearch/monitor/fs/FsInfo.java b/core/src/main/java/org/elasticsearch/monitor/fs/FsInfo.java
index bf7dce9c0d5ec..98655b90787f7 100644
--- a/core/src/main/java/org/elasticsearch/monitor/fs/FsInfo.java
+++ b/core/src/main/java/org/elasticsearch/monitor/fs/FsInfo.java
@@ -135,9 +135,9 @@ private double addDouble(double current, double other) {
         }
 
         public void add(Path path) {
-            total = addLong(total, path.total);
-            free = addLong(free, path.free);
-            available = addLong(available, path.available);
+            total = FsProbe.adjustForHugeFilesystems(addLong(total, path.total));
+            free = FsProbe.adjustForHugeFilesystems(addLong(free, path.free));
+            available = FsProbe.adjustForHugeFilesystems(addLong(available, path.available));
             if (path.spins != null && path.spins.booleanValue()) {
                 // Spinning is contagious!
                 spins = Boolean.TRUE;
diff --git a/core/src/main/java/org/elasticsearch/monitor/fs/FsProbe.java b/core/src/main/java/org/elasticsearch/monitor/fs/FsProbe.java
index 4cdbed367c987..d825100352670 100644
--- a/core/src/main/java/org/elasticsearch/monitor/fs/FsProbe.java
+++ b/core/src/main/java/org/elasticsearch/monitor/fs/FsProbe.java
@@ -126,6 +126,18 @@ List<String> readProcDiskStats() throws IOException {
         return Files.readAllLines(PathUtils.get("/proc/diskstats"));
     }
 
+    /* See: https://bugs.openjdk.java.net/browse/JDK-8162520 */
+    /**
+     * Take a large value intended to be positive, and if it has overflowed,
+     * return {@code Long.MAX_VALUE} instead of a negative number.
+     */
+    static long adjustForHugeFilesystems(long bytes) {
+        if (bytes < 0) {
+            return Long.MAX_VALUE;
+        }
+        return bytes;
+    }
+
     public static FsInfo.Path getFSInfo(NodePath nodePath) throws IOException {
         FsInfo.Path fsPath = new FsInfo.Path();
         fsPath.path = nodePath.path.toAbsolutePath().toString();
@@ -133,7 +145,7 @@ public static FsInfo.Path getFSInfo(NodePath nodePath) throws IOException {
         // NOTE: we use already cached (on node startup) FileStore and spins
         // since recomputing these once per second (default) could be costly,
         // and they should not change:
-        fsPath.total = nodePath.fileStore.getTotalSpace();
+        fsPath.total = adjustForHugeFilesystems(nodePath.fileStore.getTotalSpace());
         fsPath.free = nodePath.fileStore.getUnallocatedSpace();
         fsPath.available = nodePath.fileStore.getUsableSpace();
         fsPath.type = nodePath.fileStore.type();
diff --git a/core/src/main/java/org/elasticsearch/monitor/fs/FsService.java b/core/src/main/java/org/elasticsearch/monitor/fs/FsService.java
index 96467b4d407ca..58557cdc305f1 100644
--- a/core/src/main/java/org/elasticsearch/monitor/fs/FsService.java
+++ b/core/src/main/java/org/elasticsearch/monitor/fs/FsService.java
@@ -68,7 +68,7 @@ private class FsInfoCache extends SingleObjectCache<FsInfo> {
 
         private final FsInfo initialValue;
 
-        public FsInfoCache(TimeValue interval, FsInfo initialValue) {
+        FsInfoCache(TimeValue interval, FsInfo initialValue) {
             super(interval, initialValue);
             this.initialValue = initialValue;
         }
diff --git a/core/src/main/java/org/elasticsearch/monitor/jvm/JvmGcMonitorService.java b/core/src/main/java/org/elasticsearch/monitor/jvm/JvmGcMonitorService.java
index 3a19fe5bd0056..f260d7430e2e5 100644
--- a/core/src/main/java/org/elasticsearch/monitor/jvm/JvmGcMonitorService.java
+++ b/core/src/main/java/org/elasticsearch/monitor/jvm/JvmGcMonitorService.java
@@ -71,7 +71,7 @@ static class GcOverheadThreshold {
         final int infoThreshold;
         final int debugThreshold;
 
-        public GcOverheadThreshold(final int warnThreshold, final int infoThreshold, final int debugThreshold) {
+        GcOverheadThreshold(final int warnThreshold, final int infoThreshold, final int debugThreshold) {
             this.warnThreshold = warnThreshold;
             this.infoThreshold = infoThreshold;
             this.debugThreshold = debugThreshold;
@@ -355,7 +355,7 @@ static class SlowGcEvent {
             final JvmStats currentJvmStats;
             final ByteSizeValue maxHeapUsed;
 
-            public SlowGcEvent(
+            SlowGcEvent(
                 final GarbageCollector currentGc,
                 final long collectionCount,
                 final TimeValue collectionTime,
@@ -380,7 +380,7 @@ public SlowGcEvent(
         private final Map<String, JvmGcMonitorService.GcThreshold> gcThresholds;
         final GcOverheadThreshold gcOverheadThreshold;
 
-        public JvmMonitor(final Map<String, GcThreshold> gcThresholds, final GcOverheadThreshold gcOverheadThreshold) {
+        JvmMonitor(final Map<String, GcThreshold> gcThresholds, final GcOverheadThreshold gcOverheadThreshold) {
             this.gcThresholds = Objects.requireNonNull(gcThresholds);
             this.gcOverheadThreshold = Objects.requireNonNull(gcOverheadThreshold);
         }
@@ -486,9 +486,9 @@ long now() {
             return System.nanoTime();
         }
 
-        abstract void onSlowGc(final Threshold threshold, final long seq, final SlowGcEvent slowGcEvent);
+        abstract void onSlowGc(Threshold threshold, long seq, SlowGcEvent slowGcEvent);
 
-        abstract void onGcOverhead(final Threshold threshold, final long total, final long elapsed, final long seq);
+        abstract void onGcOverhead(Threshold threshold, long total, long elapsed, long seq);
 
     }
 
diff --git a/core/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java b/core/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java
index ca0bb4f3e8010..c18a1c5e3fdd9 100644
--- a/core/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java
+++ b/core/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java
@@ -103,6 +103,7 @@ public class JvmInfo implements Writeable, ToXContent {
         String onOutOfMemoryError = null;
         String useCompressedOops = "unknown";
         String useG1GC = "unknown";
+        String useSerialGC = "unknown";
         long configuredInitialHeapSize = -1;
         long configuredMaxHeapSize = -1;
         try {
@@ -148,6 +149,13 @@ public class JvmInfo implements Writeable, ToXContent {
                 configuredMaxHeapSize = Long.parseLong((String) valueMethod.invoke(maxHeapSizeVmOptionObject));
             } catch (Exception ignored) {
             }
+
+            try {
+                Object useSerialGCVmOptionObject = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "UseSerialGC");
+                useSerialGC = (String) valueMethod.invoke(useSerialGCVmOptionObject);
+            } catch (Exception ignored) {
+            }
+
         } catch (Exception ignored) {
 
         }
@@ -155,7 +163,7 @@ public class JvmInfo implements Writeable, ToXContent {
         INSTANCE = new JvmInfo(pid, System.getProperty("java.version"), runtimeMXBean.getVmName(), runtimeMXBean.getVmVersion(),
                 runtimeMXBean.getVmVendor(), runtimeMXBean.getStartTime(), configuredInitialHeapSize, configuredMaxHeapSize,
                 mem, inputArguments, bootClassPath, classPath, systemProperties, gcCollectors, memoryPools, onError, onOutOfMemoryError,
-                useCompressedOops, useG1GC);
+                useCompressedOops, useG1GC, useSerialGC);
     }
 
     public static JvmInfo jvmInfo() {
@@ -186,11 +194,12 @@ public static JvmInfo jvmInfo() {
     private final String onOutOfMemoryError;
     private final String useCompressedOops;
     private final String useG1GC;
+    private final String useSerialGC;
 
     private JvmInfo(long pid, String version, String vmName, String vmVersion, String vmVendor, long startTime,
                    long configuredInitialHeapSize, long configuredMaxHeapSize, Mem mem, String[] inputArguments, String bootClassPath,
                    String classPath, Map<String, String> systemProperties, String[] gcCollectors, String[] memoryPools, String onError,
-                   String onOutOfMemoryError, String useCompressedOops, String useG1GC) {
+                   String onOutOfMemoryError, String useCompressedOops, String useG1GC, String useSerialGC) {
         this.pid = pid;
         this.version = version;
         this.vmName = vmName;
@@ -210,6 +219,7 @@ private JvmInfo(long pid, String version, String vmName, String vmVersion, Strin
         this.onOutOfMemoryError = onOutOfMemoryError;
         this.useCompressedOops = useCompressedOops;
         this.useG1GC = useG1GC;
+        this.useSerialGC = useSerialGC;
     }
 
     public JvmInfo(StreamInput in) throws IOException {
@@ -230,12 +240,13 @@ public JvmInfo(StreamInput in) throws IOException {
         gcCollectors = in.readStringArray();
         memoryPools = in.readStringArray();
         useCompressedOops = in.readString();
-        //the following members are only used locally for boostrap checks, never serialized nor printed out
+        //the following members are only used locally for bootstrap checks, never serialized nor printed out
         this.configuredMaxHeapSize = -1;
         this.configuredInitialHeapSize = -1;
         this.onError = null;
         this.onOutOfMemoryError = null;
         this.useG1GC = "unknown";
+        this.useSerialGC = "unknown";
     }
 
     @Override
@@ -415,6 +426,10 @@ public String useG1GC() {
         return this.useG1GC;
     }
 
+    public String useSerialGC() {
+        return this.useSerialGC;
+    }
+
     public String[] getGcCollectors() {
         return gcCollectors;
     }
@@ -431,7 +446,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.field(Fields.VM_NAME, vmName);
         builder.field(Fields.VM_VERSION, vmVersion);
         builder.field(Fields.VM_VENDOR, vmVendor);
-        builder.dateValueField(Fields.START_TIME_IN_MILLIS, Fields.START_TIME, startTime);
+        builder.dateField(Fields.START_TIME_IN_MILLIS, Fields.START_TIME, startTime);
 
         builder.startObject(Fields.MEM);
         builder.byteSizeField(Fields.HEAP_INIT_IN_BYTES, Fields.HEAP_INIT, mem.heapInit);
@@ -441,11 +456,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.byteSizeField(Fields.DIRECT_MAX_IN_BYTES, Fields.DIRECT_MAX, mem.directMemoryMax);
         builder.endObject();
 
-        builder.field(Fields.GC_COLLECTORS, gcCollectors);
-        builder.field(Fields.MEMORY_POOLS, memoryPools);
+        builder.array(Fields.GC_COLLECTORS, gcCollectors);
+        builder.array(Fields.MEMORY_POOLS, memoryPools);
 
         builder.field(Fields.USING_COMPRESSED_OOPS, useCompressedOops);
 
+        builder.field(Fields.INPUT_ARGUMENTS, inputArguments);
+
         builder.endObject();
         return builder;
     }
@@ -474,6 +491,7 @@ static final class Fields {
         static final String GC_COLLECTORS = "gc_collectors";
         static final String MEMORY_POOLS = "memory_pools";
         static final String USING_COMPRESSED_OOPS = "using_compressed_ordinary_object_pointers";
+        static final String INPUT_ARGUMENTS = "input_arguments";
     }
 
     public static class Mem implements Writeable {
diff --git a/core/src/main/java/org/elasticsearch/monitor/os/OsProbe.java b/core/src/main/java/org/elasticsearch/monitor/os/OsProbe.java
index 08abfc05f1d6c..43ef51658b726 100644
--- a/core/src/main/java/org/elasticsearch/monitor/os/OsProbe.java
+++ b/core/src/main/java/org/elasticsearch/monitor/os/OsProbe.java
@@ -19,17 +19,25 @@
 
 package org.elasticsearch.monitor.os;
 
+import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.Constants;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.io.PathUtils;
+import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.monitor.Probes;
 
 import java.io.IOException;
 import java.lang.management.ManagementFactory;
 import java.lang.management.OperatingSystemMXBean;
+import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 public class OsProbe {
 
@@ -108,48 +116,326 @@ public long getTotalSwapSpaceSize() {
     }
 
     /**
-     * Returns the system load averages
+     * The system load averages as an array.
+     *
+     * On Windows, this method returns {@code null}.
+     *
+     * On Linux, this method returns the 1, 5, and 15-minute load averages.
+     *
+     * On macOS, this method should return the 1-minute load average.
+     *
+     * @return the available system load averages or {@code null}
      */
-    public double[] getSystemLoadAverage() {
-        if (Constants.LINUX || Constants.FREE_BSD) {
-            final String procLoadAvg = Constants.LINUX ? "/proc/loadavg" : "/compat/linux/proc/loadavg";
-            double[] loadAverage = readProcLoadavg(procLoadAvg);
-            if (loadAverage != null) {
-                return loadAverage;
-            }
-            // fallback
-        }
+    final double[] getSystemLoadAverage() {
         if (Constants.WINDOWS) {
             return null;
+        } else if (Constants.LINUX) {
+            try {
+                final String procLoadAvg = readProcLoadavg();
+                assert procLoadAvg.matches("(\\d+\\.\\d+\\s+){3}\\d+/\\d+\\s+\\d+");
+                final String[] fields = procLoadAvg.split("\\s+");
+                return new double[]{Double.parseDouble(fields[0]), Double.parseDouble(fields[1]), Double.parseDouble(fields[2])};
+            } catch (final IOException e) {
+                if (logger.isDebugEnabled()) {
+                    logger.debug("error reading /proc/loadavg", e);
+                }
+                return null;
+            }
+        } else {
+            assert Constants.MAC_OS_X;
+            if (getSystemLoadAverage == null) {
+                return null;
+            }
+            try {
+                final double oneMinuteLoadAverage = (double) getSystemLoadAverage.invoke(osMxBean);
+                return new double[]{oneMinuteLoadAverage >= 0 ? oneMinuteLoadAverage : -1, -1, -1};
+            } catch (IllegalAccessException | InvocationTargetException e) {
+                if (logger.isDebugEnabled()) {
+                    logger.debug("error reading one minute load average from operating system", e);
+                }
+                return null;
+            }
         }
-        if (getSystemLoadAverage == null) {
-            return null;
-        }
-        try {
-            double oneMinuteLoadAverage = (double) getSystemLoadAverage.invoke(osMxBean);
-            return new double[] { oneMinuteLoadAverage >= 0 ? oneMinuteLoadAverage : -1, -1, -1 };
-        } catch (Exception e) {
-            return null;
+    }
+
+    /**
+     * The line from {@code /proc/loadavg}. The first three fields are the load averages averaged over 1, 5, and 15 minutes. The fourth
+     * field is two numbers separated by a slash, the first is the number of currently runnable scheduling entities, the second is the
+     * number of scheduling entities on the system. The fifth field is the PID of the most recently created process.
+     *
+     * @return the line from {@code /proc/loadavg} or {@code null}
+     */
+    @SuppressForbidden(reason = "access /proc/loadavg")
+    String readProcLoadavg() throws IOException {
+        return readSingleLine(PathUtils.get("/proc/loadavg"));
+    }
+
+    public short getSystemCpuPercent() {
+        return Probes.getLoadAndScaleToPercent(getSystemCpuLoad, osMxBean);
+    }
+
+    /**
+     * Reads a file containing a single line.
+     *
+     * @param path path to the file to read
+     * @return the single line
+     * @throws IOException if an I/O exception occurs reading the file
+     */
+    private String readSingleLine(final Path path) throws IOException {
+        final List<String> lines = Files.readAllLines(path);
+        assert lines != null && lines.size() == 1;
+        return lines.get(0);
+    }
+
+    // this property is to support a hack to workaround an issue with Docker containers mounting the cgroups hierarchy inconsistently with
+    // respect to /proc/self/cgroup; for Docker containers this should be set to "/"
+    private static final String CONTROL_GROUPS_HIERARCHY_OVERRIDE = System.getProperty("es.cgroups.hierarchy.override");
+
+    /**
+     * A map of the control groups to which the Elasticsearch process belongs. Note that this is a map because the control groups can vary
+     * from subsystem to subsystem. Additionally, this map can not be cached because a running process can be reclassified.
+     *
+     * @return a map from subsystems to the control group for the Elasticsearch process.
+     * @throws IOException if an I/O exception occurs reading {@code /proc/self/cgroup}
+     */
+    private Map<String, String> getControlGroups() throws IOException {
+        final List<String> lines = readProcSelfCgroup();
+        final Map<String, String> controllerMap = new HashMap<>();
+        for (final String line : lines) {
+            /*
+             * The virtual file /proc/self/cgroup lists the control groups that the Elasticsearch process is a member of. Each line contains
+             * three colon-separated fields of the form hierarchy-ID:subsystem-list:cgroup-path. For cgroups version 1 hierarchies, the
+             * subsystem-list is a comma-separated list of subsystems. The subsystem-list can be empty if the hierarchy represents a cgroups
+             * version 2 hierarchy. For cgroups version 1
+             */
+            final String[] fields = line.split(":");
+            assert fields.length == 3;
+            final String[] controllers = fields[1].split(",");
+            for (final String controller : controllers) {
+                final String controlGroupPath;
+                if (CONTROL_GROUPS_HIERARCHY_OVERRIDE != null) {
+                    /*
+                     * Docker violates the relationship between /proc/self/cgroup and the /sys/fs/cgroup hierarchy. It's possible that this
+                     * will be fixed in future versions of Docker with cgroup namespaces, but this requires modern kernels. Thus, we provide
+                     * an undocumented hack for overriding the control group path. Do not rely on this hack, it will be removed.
+                     */
+                    controlGroupPath = CONTROL_GROUPS_HIERARCHY_OVERRIDE;
+                } else {
+                    controlGroupPath = fields[2];
+                }
+                final String previous = controllerMap.put(controller, controlGroupPath);
+                assert previous == null;
+            }
         }
+        return controllerMap;
     }
 
-    @SuppressForbidden(reason = "access /proc")
-    private static double[] readProcLoadavg(String procLoadavg) {
-        try {
-            List<String> lines = Files.readAllLines(PathUtils.get(procLoadavg));
-            if (!lines.isEmpty()) {
-                String[] fields = lines.get(0).split("\\s+");
-                return new double[] { Double.parseDouble(fields[0]), Double.parseDouble(fields[1]), Double.parseDouble(fields[2]) };
+    /**
+     * The lines from {@code /proc/self/cgroup}. This file represents the control groups to which the Elasticsearch process belongs. Each
+     * line in this file represents a control group hierarchy of the form
+     * <p>
+     * {@code \d+:([^:,]+(?:,[^:,]+)?):(/.*)}
+     * <p>
+     * with the first field representing the hierarchy ID, the second field representing a comma-separated list of the subsystems bound to
+     * the hierarchy, and the last field representing the control group.
+     *
+     * @return the lines from {@code /proc/self/cgroup}
+     * @throws IOException if an I/O exception occurs reading {@code /proc/self/cgroup}
+     */
+    @SuppressForbidden(reason = "access /proc/self/cgroup")
+    List<String> readProcSelfCgroup() throws IOException {
+        final List<String> lines = Files.readAllLines(PathUtils.get("/proc/self/cgroup"));
+        assert lines != null && !lines.isEmpty();
+        return lines;
+    }
+
+    /**
+     * The total CPU time in nanoseconds consumed by all tasks in the cgroup to which the Elasticsearch process belongs for the {@code
+     * cpuacct} subsystem.
+     *
+     * @param controlGroup the control group for the Elasticsearch process for the {@code cpuacct} subsystem
+     * @return the total CPU time in nanoseconds
+     * @throws IOException if an I/O exception occurs reading {@code cpuacct.usage} for the control group
+     */
+    private long getCgroupCpuAcctUsageNanos(final String controlGroup) throws IOException {
+        return Long.parseLong(readSysFsCgroupCpuAcctCpuAcctUsage(controlGroup));
+    }
+
+    /**
+     * Returns the line from {@code cpuacct.usage} for the control group to which the Elasticsearch process belongs for the {@code cpuacct}
+     * subsystem. This line represents the total CPU time in nanoseconds consumed by all tasks in the same control group.
+     *
+     * @param controlGroup the control group to which the Elasticsearch process belongs for the {@code cpuacct} subsystem
+     * @return the line from {@code cpuacct.usage}
+     * @throws IOException if an I/O exception occurs reading {@code cpuacct.usage} for the control group
+     */
+    @SuppressForbidden(reason = "access /sys/fs/cgroup/cpuacct")
+    String readSysFsCgroupCpuAcctCpuAcctUsage(final String controlGroup) throws IOException {
+        return readSingleLine(PathUtils.get("/sys/fs/cgroup/cpuacct", controlGroup, "cpuacct.usage"));
+    }
+
+    /**
+     * The total period of time in microseconds for how frequently the Elasticsearch control group's access to CPU resources will be
+     * reallocated.
+     *
+     * @param controlGroup the control group for the Elasticsearch process for the {@code cpuacct} subsystem
+     * @return the CFS quota period in microseconds
+     * @throws IOException if an I/O exception occurs reading {@code cpu.cfs_period_us} for the control group
+     */
+    private long getCgroupCpuAcctCpuCfsPeriodMicros(final String controlGroup) throws IOException {
+        return Long.parseLong(readSysFsCgroupCpuAcctCpuCfsPeriod(controlGroup));
+    }
+
+    /**
+     * Returns the line from {@code cpu.cfs_period_us} for the control group to which the Elasticsearch process belongs for the {@code cpu}
+     * subsystem. This line represents the period of time in microseconds for how frequently the control group's access to CPU resources
+     * will be reallocated.
+     *
+     * @param controlGroup the control group to which the Elasticsearch process belongs for the {@code cpu} subsystem
+     * @return the line from {@code cpu.cfs_period_us}
+     * @throws IOException if an I/O exception occurs reading {@code cpu.cfs_period_us} for the control group
+     */
+    @SuppressForbidden(reason = "access /sys/fs/cgroup/cpu")
+    String readSysFsCgroupCpuAcctCpuCfsPeriod(final String controlGroup) throws IOException {
+        return readSingleLine(PathUtils.get("/sys/fs/cgroup/cpu", controlGroup, "cpu.cfs_period_us"));
+    }
+
+    /**
+     * The total time in microseconds that all tasks in the Elasticsearch control group can run during one period as specified by {@code
+     * cpu.cfs_period_us}.
+     *
+     * @param controlGroup the control group for the Elasticsearch process for the {@code cpuacct} subsystem
+     * @return the CFS quota in microseconds
+     * @throws IOException if an I/O exception occurs reading {@code cpu.cfs_quota_us} for the control group
+     */
+    private long getCgroupCpuAcctCpuCfsQuotaMicros(final String controlGroup) throws IOException {
+        return Long.parseLong(readSysFsCgroupCpuAcctCpuAcctCfsQuota(controlGroup));
+    }
+
+    /**
+     * Returns the line from {@code cpu.cfs_quota_us} for the control group to which the Elasticsearch process belongs for the {@code cpu}
+     * subsystem. This line represents the total time in microseconds that all tasks in the control group can run during one period as
+     * specified by {@code cpu.cfs_period_us}.
+     *
+     * @param controlGroup the control group to which the Elasticsearch process belongs for the {@code cpu} subsystem
+     * @return the line from {@code cpu.cfs_quota_us}
+     * @throws IOException if an I/O exception occurs reading {@code cpu.cfs_quota_us} for the control group
+     */
+    @SuppressForbidden(reason = "access /sys/fs/cgroup/cpu")
+    String readSysFsCgroupCpuAcctCpuAcctCfsQuota(final String controlGroup) throws IOException {
+        return readSingleLine(PathUtils.get("/sys/fs/cgroup/cpu", controlGroup, "cpu.cfs_quota_us"));
+    }
+
+    /**
+     * The CPU time statistics for all tasks in the Elasticsearch control group.
+     *
+     * @param controlGroup the control group for the Elasticsearch process for the {@code cpuacct} subsystem
+     * @return the CPU time statistics
+     * @throws IOException if an I/O exception occurs reading {@code cpu.stat} for the control group
+     */
+    private OsStats.Cgroup.CpuStat getCgroupCpuAcctCpuStat(final String controlGroup) throws IOException {
+        final List<String> lines = readSysFsCgroupCpuAcctCpuStat(controlGroup);
+        long numberOfPeriods = -1;
+        long numberOfTimesThrottled = -1;
+        long timeThrottledNanos = -1;
+        for (final String line : lines) {
+            final String[] fields = line.split("\\s+");
+            switch (fields[0]) {
+                case "nr_periods":
+                    numberOfPeriods = Long.parseLong(fields[1]);
+                    break;
+                case "nr_throttled":
+                    numberOfTimesThrottled = Long.parseLong(fields[1]);
+                    break;
+                case "throttled_time":
+                    timeThrottledNanos = Long.parseLong(fields[1]);
+                    break;
             }
-        } catch (IOException e) {
-            // do not fail Elasticsearch if something unexpected
-            // happens here
         }
-        return null;
+        assert numberOfPeriods != -1;
+        assert numberOfTimesThrottled != -1;
+        assert timeThrottledNanos != -1;
+        return new OsStats.Cgroup.CpuStat(numberOfPeriods, numberOfTimesThrottled, timeThrottledNanos);
     }
 
-    public short getSystemCpuPercent() {
-        return Probes.getLoadAndScaleToPercent(getSystemCpuLoad, osMxBean);
+    /**
+     * Returns the lines from {@code cpu.stat} for the control group to which the Elasticsearch process belongs for the {@code cpu}
+     * subsystem. These lines represent the CPU time statistics and have the form
+     * <blockquote><pre>
+     * nr_periods \d+
+     * nr_throttled \d+
+     * throttled_time \d+
+     * </pre></blockquote>
+     * where {@code nr_periods} is the number of period intervals as specified by {@code cpu.cfs_period_us} that have elapsed, {@code
+     * nr_throttled} is the number of times tasks in the given control group have been throttled, and {@code throttled_time} is the total
+     * time in nanoseconds for which tasks in the given control group have been throttled.
+     *
+     * @param controlGroup the control group to which the Elasticsearch process belongs for the {@code cpu} subsystem
+     * @return the lines from {@code cpu.stat}
+     * @throws IOException if an I/O exception occurs reading {@code cpu.stat} for the control group
+     */
+    @SuppressForbidden(reason = "access /sys/fs/cgroup/cpu")
+    List<String> readSysFsCgroupCpuAcctCpuStat(final String controlGroup) throws IOException {
+        final List<String> lines = Files.readAllLines(PathUtils.get("/sys/fs/cgroup/cpu", controlGroup, "cpu.stat"));
+        assert lines != null && lines.size() == 3;
+        return lines;
+    }
+
+    /**
+     * Checks if cgroup stats are available by checking for the existence of {@code /proc/self/cgroup}, {@code /sys/fs/cgroup/cpu}, and
+     * {@code /sys/fs/cgroup/cpuacct}.
+     *
+     * @return {@code true} if the stats are available, otherwise {@code false}
+     */
+    @SuppressForbidden(reason = "access /proc/self/cgroup, /sys/fs/cgroup/cpu, and /sys/fs/cgroup/cpuacct")
+    boolean areCgroupStatsAvailable() {
+        if (!Files.exists(PathUtils.get("/proc/self/cgroup"))) {
+            return false;
+        }
+        if (!Files.exists(PathUtils.get("/sys/fs/cgroup/cpu"))) {
+            return false;
+        }
+        if (!Files.exists(PathUtils.get("/sys/fs/cgroup/cpuacct"))) {
+            return false;
+        }
+        return true;
+    }
+
+    /**
+     * Basic cgroup stats.
+     *
+     * @return basic cgroup stats, or {@code null} if an I/O exception occurred reading the cgroup stats
+     */
+    private OsStats.Cgroup getCgroup() {
+        try {
+            if (!areCgroupStatsAvailable()) {
+                return null;
+            } else {
+                final Map<String, String> controllerMap = getControlGroups();
+                assert !controllerMap.isEmpty();
+
+                final String cpuAcctControlGroup = controllerMap.get("cpuacct");
+                assert cpuAcctControlGroup != null;
+                final long cgroupCpuAcctUsageNanos = getCgroupCpuAcctUsageNanos(cpuAcctControlGroup);
+
+                final String cpuControlGroup = controllerMap.get("cpu");
+                assert cpuControlGroup != null;
+                final long cgroupCpuAcctCpuCfsPeriodMicros = getCgroupCpuAcctCpuCfsPeriodMicros(cpuControlGroup);
+                final long cgroupCpuAcctCpuCfsQuotaMicros = getCgroupCpuAcctCpuCfsQuotaMicros(cpuControlGroup);
+                final OsStats.Cgroup.CpuStat cpuStat = getCgroupCpuAcctCpuStat(cpuControlGroup);
+
+                return new OsStats.Cgroup(
+                    cpuAcctControlGroup,
+                    cgroupCpuAcctUsageNanos,
+                    cpuControlGroup,
+                    cgroupCpuAcctCpuCfsPeriodMicros,
+                    cgroupCpuAcctCpuCfsQuotaMicros,
+                    cpuStat);
+            }
+        } catch (final IOException e) {
+            logger.debug("error reading control group stats", e);
+            return null;
+        }
     }
 
     private static class OsProbeHolder {
@@ -160,24 +446,27 @@ public static OsProbe getInstance() {
         return OsProbeHolder.INSTANCE;
     }
 
-    private OsProbe() {
+    OsProbe() {
+
     }
 
+    private final Logger logger = ESLoggerFactory.getLogger(getClass());
+
     public OsInfo osInfo(long refreshInterval, int allocatedProcessors) {
         return new OsInfo(refreshInterval, Runtime.getRuntime().availableProcessors(),
                 allocatedProcessors, Constants.OS_NAME, Constants.OS_ARCH, Constants.OS_VERSION);
     }
 
     public OsStats osStats() {
-        OsStats.Cpu cpu = new OsStats.Cpu(getSystemCpuPercent(), getSystemLoadAverage());
-        OsStats.Mem mem = new OsStats.Mem(getTotalPhysicalMemorySize(), getFreePhysicalMemorySize());
-        OsStats.Swap swap = new OsStats.Swap(getTotalSwapSpaceSize(), getFreeSwapSpaceSize());
-        return new OsStats(System.currentTimeMillis(), cpu, mem , swap);
+        final OsStats.Cpu cpu = new OsStats.Cpu(getSystemCpuPercent(), getSystemLoadAverage());
+        final OsStats.Mem mem = new OsStats.Mem(getTotalPhysicalMemorySize(), getFreePhysicalMemorySize());
+        final OsStats.Swap swap = new OsStats.Swap(getTotalSwapSpaceSize(), getFreeSwapSpaceSize());
+        final OsStats.Cgroup cgroup = Constants.LINUX ? getCgroup() : null;
+        return new OsStats(System.currentTimeMillis(), cpu, mem, swap, cgroup);
     }
 
     /**
-     * Returns a given method of the OperatingSystemMXBean,
-     * or null if the method is not found or unavailable.
+     * Returns a given method of the OperatingSystemMXBean, or null if the method is not found or unavailable.
      */
     private static Method getMethod(String methodName) {
         try {
@@ -187,4 +476,5 @@ private static Method getMethod(String methodName) {
             return null;
         }
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/monitor/os/OsService.java b/core/src/main/java/org/elasticsearch/monitor/os/OsService.java
index cb67eef852c45..1eb0781373573 100644
--- a/core/src/main/java/org/elasticsearch/monitor/os/OsService.java
+++ b/core/src/main/java/org/elasticsearch/monitor/os/OsService.java
@@ -55,7 +55,7 @@ public synchronized OsStats stats() {
     }
 
     private class OsStatsCache extends SingleObjectCache<OsStats> {
-        public OsStatsCache(TimeValue interval, OsStats initValue) {
+        OsStatsCache(TimeValue interval, OsStats initValue) {
             super(interval, initValue);
         }
 
diff --git a/core/src/main/java/org/elasticsearch/monitor/os/OsStats.java b/core/src/main/java/org/elasticsearch/monitor/os/OsStats.java
index 06d864f31a480..c3783c600ea4c 100644
--- a/core/src/main/java/org/elasticsearch/monitor/os/OsStats.java
+++ b/core/src/main/java/org/elasticsearch/monitor/os/OsStats.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.monitor.os;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -36,12 +37,14 @@ public class OsStats implements Writeable, ToXContent {
     private final Cpu cpu;
     private final Mem mem;
     private final Swap swap;
+    private final Cgroup cgroup;
 
-    public OsStats(long timestamp, Cpu cpu, Mem mem, Swap swap) {
+    public OsStats(final long timestamp, final Cpu cpu, final Mem mem, final Swap swap, final Cgroup cgroup) {
         this.timestamp = timestamp;
-        this.cpu = Objects.requireNonNull(cpu, "cpu must not be null");
-        this.mem = Objects.requireNonNull(mem, "mem must not be null");;
-        this.swap = Objects.requireNonNull(swap, "swap must not be null");;
+        this.cpu = Objects.requireNonNull(cpu);
+        this.mem = Objects.requireNonNull(mem);
+        this.swap = Objects.requireNonNull(swap);
+        this.cgroup = cgroup;
     }
 
     public OsStats(StreamInput in) throws IOException {
@@ -49,6 +52,11 @@ public OsStats(StreamInput in) throws IOException {
         this.cpu = new Cpu(in);
         this.mem = new Mem(in);
         this.swap = new Swap(in);
+        if (in.getVersion().onOrAfter(Version.V_5_1_1)) {
+            this.cgroup = in.readOptionalWriteable(Cgroup::new);
+        } else {
+            this.cgroup = null;
+        }
     }
 
     @Override
@@ -57,6 +65,9 @@ public void writeTo(StreamOutput out) throws IOException {
         cpu.writeTo(out);
         mem.writeTo(out);
         swap.writeTo(out);
+        if (out.getVersion().onOrAfter(Version.V_5_1_1)) {
+            out.writeOptionalWriteable(cgroup);
+        }
     }
 
     public long getTimestamp() {
@@ -73,6 +84,10 @@ public Swap getSwap() {
         return swap;
     }
 
+    public Cgroup getCgroup() {
+        return cgroup;
+    }
+
     static final class Fields {
         static final String OS = "os";
         static final String TIMESTAMP = "timestamp";
@@ -103,6 +118,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         cpu.toXContent(builder, params);
         mem.toXContent(builder, params);
         swap.toXContent(builder, params);
+        if (cgroup != null) {
+            cgroup.toXContent(builder, params);
+        }
         builder.endObject();
         return builder;
     }
@@ -241,7 +259,7 @@ public ByteSizeValue getUsed() {
         }
 
         public short getUsedPercent() {
-            return calculatePercentage(getUsed().bytes(), total);
+            return calculatePercentage(getUsed().getBytes(), total);
         }
 
         public ByteSizeValue getFree() {
@@ -265,7 +283,211 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         }
     }
 
+    /**
+     * Encapsulates basic cgroup statistics.
+     */
+    public static class Cgroup implements Writeable, ToXContent {
+
+        private final String cpuAcctControlGroup;
+        private final long cpuAcctUsageNanos;
+        private final String cpuControlGroup;
+        private final long cpuCfsPeriodMicros;
+        private final long cpuCfsQuotaMicros;
+        private final CpuStat cpuStat;
+
+        /**
+         * The control group for the {@code cpuacct} subsystem.
+         *
+         * @return the control group
+         */
+        public String getCpuAcctControlGroup() {
+            return cpuAcctControlGroup;
+        }
+
+        /**
+         * The total CPU time consumed by all tasks in the
+         * {@code cpuacct} control group from
+         * {@link Cgroup#cpuAcctControlGroup}.
+         *
+         * @return the total CPU time in nanoseconds
+         */
+        public long getCpuAcctUsageNanos() {
+            return cpuAcctUsageNanos;
+        }
+
+        /**
+         * The control group for the {@code cpu} subsystem.
+         *
+         * @return the control group
+         */
+        public String getCpuControlGroup() {
+            return cpuControlGroup;
+        }
+
+        /**
+         * The period of time for how frequently the control group from
+         * {@link Cgroup#cpuControlGroup} has its access to CPU
+         * resources reallocated.
+         *
+         * @return the period of time in microseconds
+         */
+        public long getCpuCfsPeriodMicros() {
+            return cpuCfsPeriodMicros;
+        }
+
+        /**
+         * The total amount of time for which all tasks in the control
+         * group from {@link Cgroup#cpuControlGroup} can run in one
+         * period as represented by {@link Cgroup#cpuCfsPeriodMicros}.
+         *
+         * @return the total amount of time in microseconds
+         */
+        public long getCpuCfsQuotaMicros() {
+            return cpuCfsQuotaMicros;
+        }
+
+        /**
+         * The CPU time statistics. See {@link CpuStat}.
+         *
+         * @return the CPU time statistics.
+         */
+        public CpuStat getCpuStat() {
+            return cpuStat;
+        }
+
+        public Cgroup(
+            final String cpuAcctControlGroup,
+            final long cpuAcctUsageNanos,
+            final String cpuControlGroup,
+            final long cpuCfsPeriodMicros,
+            final long cpuCfsQuotaMicros,
+            final CpuStat cpuStat) {
+            this.cpuAcctControlGroup = Objects.requireNonNull(cpuAcctControlGroup);
+            this.cpuAcctUsageNanos = cpuAcctUsageNanos;
+            this.cpuControlGroup = Objects.requireNonNull(cpuControlGroup);
+            this.cpuCfsPeriodMicros = cpuCfsPeriodMicros;
+            this.cpuCfsQuotaMicros = cpuCfsQuotaMicros;
+            this.cpuStat = Objects.requireNonNull(cpuStat);
+        }
+
+        Cgroup(final StreamInput in) throws IOException {
+            cpuAcctControlGroup = in.readString();
+            cpuAcctUsageNanos = in.readLong();
+            cpuControlGroup = in.readString();
+            cpuCfsPeriodMicros = in.readLong();
+            cpuCfsQuotaMicros = in.readLong();
+            cpuStat = new CpuStat(in);
+        }
+
+        @Override
+        public void writeTo(final StreamOutput out) throws IOException {
+            out.writeString(cpuAcctControlGroup);
+            out.writeLong(cpuAcctUsageNanos);
+            out.writeString(cpuControlGroup);
+            out.writeLong(cpuCfsPeriodMicros);
+            out.writeLong(cpuCfsQuotaMicros);
+            cpuStat.writeTo(out);
+        }
+
+        @Override
+        public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException {
+            builder.startObject("cgroup");
+            {
+                builder.startObject("cpuacct");
+                {
+                    builder.field("control_group", cpuAcctControlGroup);
+                    builder.field("usage_nanos", cpuAcctUsageNanos);
+                }
+                builder.endObject();
+                builder.startObject("cpu");
+                {
+                    builder.field("control_group", cpuControlGroup);
+                    builder.field("cfs_period_micros", cpuCfsPeriodMicros);
+                    builder.field("cfs_quota_micros", cpuCfsQuotaMicros);
+                    cpuStat.toXContent(builder, params);
+                }
+                builder.endObject();
+            }
+            builder.endObject();
+            return builder;
+        }
+
+        /**
+         * Encapsulates CPU time statistics.
+         */
+        public static class CpuStat implements Writeable, ToXContent {
+
+            private final long numberOfElapsedPeriods;
+            private final long numberOfTimesThrottled;
+            private final long timeThrottledNanos;
+
+            /**
+             * The number of elapsed periods.
+             *
+             * @return the number of elapsed periods as measured by
+             * {@code cpu.cfs_period_us}
+             */
+            public long getNumberOfElapsedPeriods() {
+                return numberOfElapsedPeriods;
+            }
+
+            /**
+             * The number of times tasks in the control group have been
+             * throttled.
+             *
+             * @return the number of times
+             */
+            public long getNumberOfTimesThrottled() {
+                return numberOfTimesThrottled;
+            }
+
+            /**
+             * The total time duration for which tasks in the control
+             * group have been throttled.
+             *
+             * @return the total time in nanoseconds
+             */
+            public long getTimeThrottledNanos() {
+                return timeThrottledNanos;
+            }
+
+            public CpuStat(final long numberOfElapsedPeriods, final long numberOfTimesThrottled, final long timeThrottledNanos) {
+                this.numberOfElapsedPeriods = numberOfElapsedPeriods;
+                this.numberOfTimesThrottled = numberOfTimesThrottled;
+                this.timeThrottledNanos = timeThrottledNanos;
+            }
+
+            CpuStat(final StreamInput in) throws IOException {
+                numberOfElapsedPeriods = in.readLong();
+                numberOfTimesThrottled = in.readLong();
+                timeThrottledNanos = in.readLong();
+            }
+
+            @Override
+            public void writeTo(final StreamOutput out) throws IOException {
+                out.writeLong(numberOfElapsedPeriods);
+                out.writeLong(numberOfTimesThrottled);
+                out.writeLong(timeThrottledNanos);
+            }
+
+            @Override
+            public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+                builder.startObject("stat");
+                {
+                    builder.field("number_of_elapsed_periods", numberOfElapsedPeriods);
+                    builder.field("number_of_times_throttled", numberOfTimesThrottled);
+                    builder.field("time_throttled_nanos", timeThrottledNanos);
+                }
+                builder.endObject();
+                return builder;
+            }
+
+        }
+
+    }
+
     public static short calculatePercentage(long used, long max) {
         return max <= 0 ? 0 : (short) (Math.round((100d * used) / max));
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/monitor/process/ProcessService.java b/core/src/main/java/org/elasticsearch/monitor/process/ProcessService.java
index 99593003b340a..1a1273af87c80 100644
--- a/core/src/main/java/org/elasticsearch/monitor/process/ProcessService.java
+++ b/core/src/main/java/org/elasticsearch/monitor/process/ProcessService.java
@@ -57,7 +57,7 @@ public ProcessStats stats() {
     }
 
     private class ProcessStatsCache extends SingleObjectCache<ProcessStats> {
-        public ProcessStatsCache(TimeValue interval, ProcessStats initValue) {
+        ProcessStatsCache(TimeValue interval, ProcessStats initValue) {
             super(interval, initValue);
         }
 
diff --git a/core/src/main/java/org/elasticsearch/node/internal/InternalSettingsPreparer.java b/core/src/main/java/org/elasticsearch/node/InternalSettingsPreparer.java
similarity index 85%
rename from core/src/main/java/org/elasticsearch/node/internal/InternalSettingsPreparer.java
rename to core/src/main/java/org/elasticsearch/node/InternalSettingsPreparer.java
index dba7f303130c6..08fd1874fd38c 100644
--- a/core/src/main/java/org/elasticsearch/node/internal/InternalSettingsPreparer.java
+++ b/core/src/main/java/org/elasticsearch/node/InternalSettingsPreparer.java
@@ -17,15 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.node.internal;
-
-import org.elasticsearch.cli.Terminal;
-import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsException;
-import org.elasticsearch.env.Environment;
+package org.elasticsearch.node;
 
 import java.io.IOException;
 import java.nio.file.Files;
@@ -37,7 +29,14 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.function.Function;
-import java.util.function.Predicate;
+
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsException;
+import org.elasticsearch.env.Environment;
 
 import static org.elasticsearch.common.Strings.cleanPath;
 
@@ -47,8 +46,6 @@
 public class InternalSettingsPreparer {
 
     private static final String[] ALLOWED_SUFFIXES = {".yml", ".yaml", ".json"};
-    static final String PROPERTY_DEFAULTS_PREFIX = "default.";
-    static final Predicate<String> PROPERTY_DEFAULTS_PREDICATE = key -> key.startsWith(PROPERTY_DEFAULTS_PREFIX);
 
     public static final String SECRET_PROMPT_VALUE = "${prompt.secret}";
     public static final String TEXT_PROMPT_VALUE = "${prompt.text}";
@@ -58,7 +55,7 @@ public class InternalSettingsPreparer {
      */
     public static Settings prepareSettings(Settings input) {
         Settings.Builder output = Settings.builder();
-        initializeSettings(output, input, true, Collections.emptyMap());
+        initializeSettings(output, input, Collections.emptyMap());
         finalizeSettings(output, null);
         return output.build();
     }
@@ -89,9 +86,10 @@ public static Environment prepareEnvironment(Settings input, Terminal terminal)
     public static Environment prepareEnvironment(Settings input, Terminal terminal, Map<String, String> properties) {
         // just create enough settings to build the environment, to get the config dir
         Settings.Builder output = Settings.builder();
-        initializeSettings(output, input, true, properties);
+        initializeSettings(output, input, properties);
         Environment environment = new Environment(output.build());
 
+        output = Settings.builder(); // start with a fresh output
         boolean settingsFileFound = false;
         Set<String> foundSuffixes = new HashSet<>();
         for (String allowedSuffix : ALLOWED_SUFFIXES) {
@@ -109,31 +107,33 @@ public static Environment prepareEnvironment(Settings input, Terminal terminal,
             }
         }
         if (foundSuffixes.size() > 1) {
-            throw new SettingsException("multiple settings files found with suffixes: " + Strings.collectionToDelimitedString(foundSuffixes, ","));
+            throw new SettingsException("multiple settings files found with suffixes: "
+                + Strings.collectionToDelimitedString(foundSuffixes, ","));
         }
 
         // re-initialize settings now that the config file has been loaded
-        // TODO: only re-initialize if a config file was actually loaded
-        initializeSettings(output, input, false, properties);
+        initializeSettings(output, input, properties);
         finalizeSettings(output, terminal);
 
         environment = new Environment(output.build());
 
         // we put back the path.logs so we can use it in the logging configuration file
         output.put(Environment.PATH_LOGS_SETTING.getKey(), cleanPath(environment.logsFile().toAbsolutePath().toString()));
-        return new Environment(output.build());
+        String configExtension = foundSuffixes.isEmpty() ? null : foundSuffixes.iterator().next();
+        return new Environment(output.build(), configExtension);
     }
 
     /**
-     * Initializes the builder with the given input settings, and loads system properties settings if allowed.
-     * If loadDefaults is true, system property default settings are loaded.
+     * Initializes the builder with the given input settings, and applies settings from the specified map (these settings typically come
+     * from the command line).
+     *
+     * @param output the settings builder to apply the input and default settings to
+     * @param input the input settings
+     * @param esSettings a map from which to apply settings
      */
-    private static void initializeSettings(Settings.Builder output, Settings input, boolean loadDefaults, Map<String, String> esSettings) {
+    static void initializeSettings(final Settings.Builder output, final Settings input, final Map<String, String> esSettings) {
         output.put(input);
-        if (loadDefaults) {
-            output.putProperties(esSettings, PROPERTY_DEFAULTS_PREDICATE, key -> key.substring(PROPERTY_DEFAULTS_PREFIX.length()));
-        }
-        output.putProperties(esSettings, PROPERTY_DEFAULTS_PREDICATE.negate(), Function.identity());
+        output.putProperties(esSettings, Function.identity());
         output.replacePropertyPlaceholders();
     }
 
@@ -198,7 +198,9 @@ private static void replacePromptPlaceholders(Settings.Builder settings, Termina
 
     private static String promptForValue(String key, Terminal terminal, boolean secret) {
         if (terminal == null) {
-            throw new UnsupportedOperationException("found property [" + key + "] with value [" + (secret ? SECRET_PROMPT_VALUE : TEXT_PROMPT_VALUE) +"]. prompting for property values is only supported when running elasticsearch in the foreground");
+            throw new UnsupportedOperationException("found property [" + key + "] with value ["
+                + (secret ? SECRET_PROMPT_VALUE : TEXT_PROMPT_VALUE)
+                + "]. prompting for property values is only supported when running elasticsearch in the foreground");
         }
 
         if (secret) {
diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java
index 402064e88d299..9638f77bd0c00 100644
--- a/core/src/main/java/org/elasticsearch/node/Node.java
+++ b/core/src/main/java/org/elasticsearch/node/Node.java
@@ -19,39 +19,50 @@
 
 package org.elasticsearch.node;
 
-import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.core.LoggerContext;
-import org.apache.logging.log4j.core.config.Configurator;
 import org.apache.lucene.util.Constants;
 import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.Build;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchTimeoutException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionModule;
 import org.elasticsearch.action.GenericAction;
+import org.elasticsearch.action.search.SearchPhaseController;
+import org.elasticsearch.action.search.SearchTransportService;
 import org.elasticsearch.action.support.TransportAction;
+import org.elasticsearch.action.update.UpdateHelper;
+import org.elasticsearch.bootstrap.BootstrapCheck;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.cluster.ClusterInfoService;
 import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateObserver;
-import org.elasticsearch.cluster.MasterNodeChangePredicate;
+import org.elasticsearch.cluster.InternalClusterInfoService;
 import org.elasticsearch.cluster.NodeConnectionsService;
 import org.elasticsearch.cluster.action.index.MappingUpdatedAction;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.metadata.MetaDataIndexUpgradeService;
+import org.elasticsearch.cluster.metadata.TemplateUpgradeService;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.RoutingService;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.StopWatch;
+import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.component.Lifecycle;
 import org.elasticsearch.common.component.LifecycleComponent;
+import org.elasticsearch.common.inject.Binder;
 import org.elasticsearch.common.inject.Injector;
 import org.elasticsearch.common.inject.Key;
 import org.elasticsearch.common.inject.Module;
 import org.elasticsearch.common.inject.ModulesBuilder;
+import org.elasticsearch.common.inject.util.Providers;
+import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.logging.DeprecationLogger;
@@ -68,6 +79,7 @@
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.discovery.Discovery;
 import org.elasticsearch.discovery.DiscoveryModule;
@@ -78,7 +90,6 @@
 import org.elasticsearch.gateway.GatewayModule;
 import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.gateway.MetaStateService;
-import org.elasticsearch.http.HttpServer;
 import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.index.analysis.AnalysisRegistry;
 import org.elasticsearch.indices.IndicesModule;
@@ -88,21 +99,23 @@
 import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.indices.cluster.IndicesClusterStateService;
+import org.elasticsearch.indices.recovery.PeerRecoverySourceService;
+import org.elasticsearch.indices.recovery.PeerRecoveryTargetService;
+import org.elasticsearch.indices.recovery.RecoverySettings;
 import org.elasticsearch.indices.store.IndicesStore;
 import org.elasticsearch.indices.ttl.IndicesTTLService;
 import org.elasticsearch.ingest.IngestService;
 import org.elasticsearch.monitor.MonitorService;
 import org.elasticsearch.monitor.jvm.JvmInfo;
-import org.elasticsearch.node.internal.InternalSettingsPreparer;
-import org.elasticsearch.node.service.NodeService;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.AnalysisPlugin;
 import org.elasticsearch.plugins.ClusterPlugin;
 import org.elasticsearch.plugins.DiscoveryPlugin;
 import org.elasticsearch.plugins.IngestPlugin;
 import org.elasticsearch.plugins.MapperPlugin;
-import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.MetaDataUpgrader;
+import org.elasticsearch.plugins.NetworkPlugin;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.PluginsService;
 import org.elasticsearch.plugins.RepositoryPlugin;
 import org.elasticsearch.plugins.ScriptPlugin;
@@ -113,16 +126,18 @@
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.search.SearchService;
+import org.elasticsearch.search.fetch.FetchPhase;
 import org.elasticsearch.snapshots.SnapshotShardsService;
 import org.elasticsearch.snapshots.SnapshotsService;
 import org.elasticsearch.tasks.TaskResultsService;
 import org.elasticsearch.threadpool.ExecutorBuilder;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.Transport;
+import org.elasticsearch.transport.TransportInterceptor;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.tribe.TribeService;
 import org.elasticsearch.watcher.ResourceWatcherService;
 
-import javax.management.MBeanServerPermission;
 import java.io.BufferedWriter;
 import java.io.Closeable;
 import java.io.IOException;
@@ -133,7 +148,6 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
-import java.security.AccessControlException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -141,13 +155,17 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.function.UnaryOperator;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+import static java.util.stream.Collectors.toList;
+
 /**
  * A node represent a node within a cluster (<tt>cluster.name</tt>). The {@link #client()} can be used
  * in order to use a {@link Client} to perform actions/operations against the cluster.
@@ -155,7 +173,7 @@
 public class Node implements Closeable {
 
 
-    public static final Setting<Boolean> WRITE_PORTS_FIELD_SETTING =
+    public static final Setting<Boolean> WRITE_PORTS_FILE_SETTING =
         Setting.boolSetting("node.portsfile", false, Property.NodeScope);
     public static final Setting<Boolean> NODE_DATA_SETTING = Setting.boolSetting("node.data", true, Property.NodeScope);
     public static final Setting<Boolean> NODE_MASTER_SETTING =
@@ -172,7 +190,16 @@ public class Node implements Closeable {
     */
     public static final Setting<Boolean> NODE_LOCAL_STORAGE_SETTING = Setting.boolSetting("node.local_storage", true, Property.NodeScope);
     public static final Setting<String> NODE_NAME_SETTING = Setting.simpleString("node.name", Property.NodeScope);
-    public static final Setting<Settings> NODE_ATTRIBUTES = Setting.groupSetting("node.attr.", Property.NodeScope);
+    public static final Setting<Settings> NODE_ATTRIBUTES = Setting.groupSetting("node.attr.", (settings) -> {
+        Map<String, String> settingsMap = settings.getAsMap();
+        for (Map.Entry<String, String> entry : settingsMap.entrySet()) {
+            String value = entry.getValue();
+            if (Character.isWhitespace(value.charAt(0)) || Character.isWhitespace(value.charAt(value.length() - 1))) {
+                throw new IllegalArgumentException("node.attr." + entry.getKey() + " cannot have leading or trailing whitespace " +
+                                                       "[" + value + "]");
+            }
+        }
+    }, Property.NodeScope);
     public static final Setting<String> BREAKER_TYPE_KEY = new Setting<>("indices.breaker.type", "hierarchy", (s) -> {
         switch (s) {
             case "hierarchy":
@@ -203,6 +230,7 @@ public static final Settings addNodeNameIfNeeded(Settings settings, final String
     private final PluginsService pluginsService;
     private final NodeClient client;
     private final Collection<LifecycleComponent> pluginLifecycleComponents;
+    private final LocalNodeFactory localNodeFactory;
 
     /**
      * Constructs a node with the given settings.
@@ -237,22 +265,27 @@ protected Node(final Environment environment, Collection<Class<? extends Plugin>
                 nodeEnvironment = new NodeEnvironment(tmpSettings, environment);
                 resourcesToClose.add(nodeEnvironment);
             } catch (IOException ex) {
-                throw new IllegalStateException("Failed to created node environment", ex);
+                throw new IllegalStateException("Failed to create node environment", ex);
             }
-
             final boolean hadPredefinedNodeName = NODE_NAME_SETTING.exists(tmpSettings);
-                tmpSettings = addNodeNameIfNeeded(tmpSettings, nodeEnvironment.nodeId());
             Logger logger = Loggers.getLogger(Node.class, tmpSettings);
+            final String nodeId = nodeEnvironment.nodeId();
+            tmpSettings = addNodeNameIfNeeded(tmpSettings, nodeId);
+            if (DiscoveryNode.nodeRequiresLocalStorage(tmpSettings)) {
+                checkForIndexDataInDefaultPathData(tmpSettings, nodeEnvironment, logger);
+            }
+            // this must be captured after the node name is possibly added to the settings
+            final String nodeName = NODE_NAME_SETTING.get(tmpSettings);
             if (hadPredefinedNodeName == false) {
-                logger.info("node name [{}] derived from node ID; set [{}] to override",
-                    NODE_NAME_SETTING.get(tmpSettings), NODE_NAME_SETTING.getKey());
+                logger.info("node name [{}] derived from node ID [{}]; set [{}] to override", nodeName, nodeId, NODE_NAME_SETTING.getKey());
+            } else {
+                logger.info("node name [{}], node ID [{}]", nodeName, nodeId);
             }
 
-            final String displayVersion = Version.CURRENT + (Build.CURRENT.isSnapshot() ? "-SNAPSHOT" : "");
             final JvmInfo jvmInfo = JvmInfo.jvmInfo();
             logger.info(
                 "version[{}], pid[{}], build[{}/{}], OS[{}/{}/{}], JVM[{}/{}/{}/{}]",
-                displayVersion,
+                displayVersion(Version.CURRENT, Build.CURRENT.isSnapshot()),
                 jvmInfo.pid(),
                 Build.CURRENT.shortHash(),
                 Build.CURRENT.date(),
@@ -263,7 +296,8 @@ protected Node(final Environment environment, Collection<Class<? extends Plugin>
                 Constants.JVM_NAME,
                 Constants.JAVA_VERSION,
                 Constants.JVM_VERSION);
-
+            logger.info("JVM arguments {}", Arrays.toString(jvmInfo.getInputArguments()));
+            warnIfPreRelease(Version.CURRENT, Build.CURRENT.isSnapshot(), logger);
 
             if (logger.isDebugEnabled()) {
                 logger.debug("using config [{}], data [{}], logs [{}], plugins [{}]",
@@ -278,6 +312,8 @@ protected Node(final Environment environment, Collection<Class<? extends Plugin>
 
             this.pluginsService = new PluginsService(tmpSettings, environment.modulesFile(), environment.pluginsFile(), classpathPlugins);
             this.settings = pluginsService.updatedSettings();
+            localNodeFactory = new LocalNodeFactory(settings, nodeEnvironment.nodeId());
+
             // create the environment based on the finalized (processed) view of the settings
             // this is just to makes sure that people get the same settings, no matter where they ask them from
             this.environment = new Environment(this.settings);
@@ -292,13 +328,12 @@ protected Node(final Environment environment, Collection<Class<? extends Plugin>
             DeprecationLogger.setThreadContext(threadPool.getThreadContext());
             resourcesToClose.add(() -> DeprecationLogger.removeThreadContext(threadPool.getThreadContext()));
 
-            final List<Setting<?>> additionalSettings = new ArrayList<>();
-            final List<String> additionalSettingsFilter = new ArrayList<>();
-            additionalSettings.addAll(pluginsService.getPluginSettings());
-            additionalSettingsFilter.addAll(pluginsService.getPluginSettingsFilter());
+            final List<Setting<?>> additionalSettings = new ArrayList<>(pluginsService.getPluginSettings());
+            final List<String> additionalSettingsFilter = new ArrayList<>(pluginsService.getPluginSettingsFilter());
             for (final ExecutorBuilder<?> builder : threadPool.builders()) {
                 additionalSettings.addAll(builder.getRegisteredSettings());
             }
+            client = new NodeClient(settings, threadPool);
             final ResourceWatcherService resourceWatcherService = new ResourceWatcherService(settings, threadPool);
             final ScriptModule scriptModule = ScriptModule.create(settings, this.environment, resourceWatcherService,
                 pluginsService.filterPlugins(ScriptPlugin.class));
@@ -311,13 +346,13 @@ protected Node(final Environment environment, Collection<Class<? extends Plugin>
             resourcesToClose.add(resourceWatcherService);
             final NetworkService networkService = new NetworkService(settings,
                 getCustomNameResolvers(pluginsService.filterPlugins(DiscoveryPlugin.class)));
-            final ClusterService clusterService = new ClusterService(settings, settingsModule.getClusterSettings(), threadPool);
-            clusterService.add(scriptModule.getScriptService());
+            final ClusterService clusterService = new ClusterService(settings, settingsModule.getClusterSettings(), threadPool,
+                localNodeFactory::getNode);
+            clusterService.addListener(scriptModule.getScriptService());
             resourcesToClose.add(clusterService);
-            final TribeService tribeService = new TribeService(settings, clusterService, nodeEnvironment.nodeId());
-            resourcesToClose.add(tribeService);
-            final IngestService ingestService = new IngestService(settings, threadPool, this.environment,
+            final IngestService ingestService = new IngestService(clusterService.getClusterSettings(), settings, threadPool, this.environment,
                 scriptModule.getScriptService(), analysisModule.getAnalysisRegistry(), pluginsService.filterPlugins(IngestPlugin.class));
+            final ClusterInfoService clusterInfoService = newClusterInfoService(settings, clusterService, threadPool, client);
 
             ModulesBuilder modules = new ModulesBuilder();
             // plugin modules must be added here, before others or we can get crazy injection errors...
@@ -326,52 +361,99 @@ protected Node(final Environment environment, Collection<Class<? extends Plugin>
             }
             final MonitorService monitorService = new MonitorService(settings, nodeEnvironment, threadPool);
             modules.add(new NodeModule(this, monitorService));
-            NetworkModule networkModule = new NetworkModule(networkService, settings, false);
-            modules.add(networkModule);
-            modules.add(new DiscoveryModule(this.settings));
             ClusterModule clusterModule = new ClusterModule(settings, clusterService,
                 pluginsService.filterPlugins(ClusterPlugin.class));
             modules.add(clusterModule);
             IndicesModule indicesModule = new IndicesModule(pluginsService.filterPlugins(MapperPlugin.class));
             modules.add(indicesModule);
+
             SearchModule searchModule = new SearchModule(settings, false, pluginsService.filterPlugins(SearchPlugin.class));
-            modules.add(searchModule);
-            modules.add(new ActionModule(DiscoveryNode.isIngestNode(settings), false, settings,
-                clusterModule.getIndexNameExpressionResolver(), settingsModule.getClusterSettings(),
-                pluginsService.filterPlugins(ActionPlugin.class)));
-            modules.add(new GatewayModule());
-            modules.add(new RepositoriesModule(this.environment, pluginsService.filterPlugins(RepositoryPlugin.class)));
-            pluginsService.processModules(modules);
             CircuitBreakerService circuitBreakerService = createCircuitBreakerService(settingsModule.getSettings(),
                 settingsModule.getClusterSettings());
             resourcesToClose.add(circuitBreakerService);
+            ActionModule actionModule = new ActionModule(false, settings, clusterModule.getIndexNameExpressionResolver(),
+                    settingsModule.getIndexScopedSettings(), settingsModule.getClusterSettings(), settingsModule.getSettingsFilter(),
+                    threadPool, pluginsService.filterPlugins(ActionPlugin.class), client, circuitBreakerService);
+            modules.add(actionModule);
+            modules.add(new GatewayModule());
+
+
             BigArrays bigArrays = createBigArrays(settings, circuitBreakerService);
             resourcesToClose.add(bigArrays);
             modules.add(settingsModule);
             List<NamedWriteableRegistry.Entry> namedWriteables = Stream.of(
-                networkModule.getNamedWriteables().stream(),
+                NetworkModule.getNamedWriteables().stream(),
                 indicesModule.getNamedWriteables().stream(),
                 searchModule.getNamedWriteables().stream(),
                 pluginsService.filterPlugins(Plugin.class).stream()
-                    .flatMap(p -> p.getNamedWriteables().stream()))
+                    .flatMap(p -> p.getNamedWriteables().stream()),
+                ClusterModule.getNamedWriteables().stream())
                 .flatMap(Function.identity()).collect(Collectors.toList());
             final NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(namedWriteables);
-            final MetaStateService metaStateService = new MetaStateService(settings, nodeEnvironment);
-            final IndicesService indicesService = new IndicesService(settings, pluginsService, nodeEnvironment,
-                settingsModule.getClusterSettings(), analysisModule.getAnalysisRegistry(), searchModule.getQueryParserRegistry(),
+            NamedXContentRegistry xContentRegistry = new NamedXContentRegistry(Stream.of(
+                NetworkModule.getNamedXContents().stream(),
+                searchModule.getNamedXContents().stream(),
+                pluginsService.filterPlugins(Plugin.class).stream()
+                    .flatMap(p -> p.getNamedXContent().stream()),
+                ClusterModule.getNamedXWriteables().stream())
+                .flatMap(Function.identity()).collect(toList()));
+            final TribeService tribeService = new TribeService(settings, clusterService, nodeId, namedWriteableRegistry,
+                s -> newTribeClientNode(s, classpathPlugins));
+            resourcesToClose.add(tribeService);
+            modules.add(new RepositoriesModule(this.environment, pluginsService.filterPlugins(RepositoryPlugin.class), xContentRegistry));
+            final MetaStateService metaStateService = new MetaStateService(settings, nodeEnvironment, xContentRegistry);
+            final IndicesService indicesService = new IndicesService(settings, pluginsService, nodeEnvironment, xContentRegistry,
+                settingsModule.getClusterSettings(), analysisModule.getAnalysisRegistry(),
                 clusterModule.getIndexNameExpressionResolver(), indicesModule.getMapperRegistry(), namedWriteableRegistry,
-                threadPool, settingsModule.getIndexScopedSettings(), circuitBreakerService, metaStateService);
-            client = new NodeClient(settings, threadPool);
+                threadPool, settingsModule.getIndexScopedSettings(), circuitBreakerService, bigArrays, scriptModule.getScriptService(),
+                clusterService, client, metaStateService);
+
             Collection<Object> pluginComponents = pluginsService.filterPlugins(Plugin.class).stream()
                 .flatMap(p -> p.createComponents(client, clusterService, threadPool, resourceWatcherService,
-                                                 scriptModule.getScriptService(), searchModule.getSearchRequestParsers()).stream())
+                                                 scriptModule.getScriptService(), xContentRegistry).stream())
                 .collect(Collectors.toList());
+            final RestController restController = actionModule.getRestController();
+            final NetworkModule networkModule = new NetworkModule(settings, false, pluginsService.filterPlugins(NetworkPlugin.class),
+                    threadPool, bigArrays, circuitBreakerService, namedWriteableRegistry, xContentRegistry, networkService, restController);
             Collection<UnaryOperator<Map<String, MetaData.Custom>>> customMetaDataUpgraders =
                 pluginsService.filterPlugins(Plugin.class).stream()
-                .map(Plugin::getCustomMetaDataUpgrader)
-                .collect(Collectors.toList());
-            final MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(customMetaDataUpgraders);
+                    .map(Plugin::getCustomMetaDataUpgrader)
+                    .collect(Collectors.toList());
+            Collection<UnaryOperator<Map<String, IndexTemplateMetaData>>> indexTemplateMetaDataUpgraders =
+                pluginsService.filterPlugins(Plugin.class).stream()
+                    .map(Plugin::getIndexTemplateMetaDataUpgrader)
+                    .collect(Collectors.toList());
+            Collection<UnaryOperator<IndexMetaData>> indexMetaDataUpgraders = pluginsService.filterPlugins(Plugin.class).stream()
+                    .map(Plugin::getIndexMetaDataUpgrader).collect(Collectors.toList());
+            final MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(customMetaDataUpgraders, indexTemplateMetaDataUpgraders);
+            new TemplateUpgradeService(settings, client, clusterService, threadPool, indexTemplateMetaDataUpgraders);
+            final Transport transport = networkModule.getTransportSupplier().get();
+            final TransportService transportService = newTransportService(settings, transport, threadPool,
+                networkModule.getTransportInterceptor(), localNodeFactory, settingsModule.getClusterSettings());
+            final SearchTransportService searchTransportService =  new SearchTransportService(settings,
+                transportService);
+            final Consumer<Binder> httpBind;
+            final HttpServerTransport httpServerTransport;
+            if (networkModule.isHttpEnabled()) {
+                httpServerTransport = networkModule.getHttpServerTransportSupplier().get();
+                httpBind = b -> {
+                    b.bind(HttpServerTransport.class).toInstance(httpServerTransport);
+                };
+            } else {
+                httpBind = b -> {
+                    b.bind(HttpServerTransport.class).toProvider(Providers.of(null));
+                };
+                httpServerTransport = null;
+            }
+            final DiscoveryModule discoveryModule = new DiscoveryModule(this.settings, threadPool, transportService,
+                namedWriteableRegistry, networkService, clusterService, pluginsService.filterPlugins(DiscoveryPlugin.class));
+            NodeService nodeService = new NodeService(settings, threadPool, monitorService, discoveryModule.getDiscovery(),
+                transportService, indicesService, pluginsService, circuitBreakerService, scriptModule.getScriptService(),
+                httpServerTransport, ingestService, clusterService, settingsModule.getSettingsFilter());
+
             modules.add(b -> {
+                    b.bind(NodeService.class).toInstance(nodeService);
+                    b.bind(NamedXContentRegistry.class).toInstance(xContentRegistry);
                     b.bind(PluginsService.class).toInstance(pluginsService);
                     b.bind(Client.class).toInstance(client);
                     b.bind(NodeClient.class).toInstance(client);
@@ -389,14 +471,29 @@ protected Node(final Environment environment, Collection<Class<? extends Plugin>
                     b.bind(MetaDataUpgrader.class).toInstance(metaDataUpgrader);
                     b.bind(MetaStateService.class).toInstance(metaStateService);
                     b.bind(IndicesService.class).toInstance(indicesService);
-                    Class<? extends SearchService> searchServiceImpl = pickSearchServiceImplementation();
-                    if (searchServiceImpl == SearchService.class) {
-                        b.bind(SearchService.class).asEagerSingleton();
-                    } else {
-                        b.bind(SearchService.class).to(searchServiceImpl).asEagerSingleton();
+                    b.bind(SearchService.class).toInstance(newSearchService(clusterService, indicesService,
+                        threadPool, scriptModule.getScriptService(), bigArrays, searchModule.getFetchPhase()));
+                    b.bind(SearchTransportService.class).toInstance(searchTransportService);
+                    b.bind(SearchPhaseController.class).toInstance(new SearchPhaseController(settings, bigArrays,
+                            scriptModule.getScriptService()));
+                    b.bind(Transport.class).toInstance(transport);
+                    b.bind(TransportService.class).toInstance(transportService);
+                    b.bind(NetworkService.class).toInstance(networkService);
+                    b.bind(UpdateHelper.class).toInstance(new UpdateHelper(settings, scriptModule.getScriptService()));
+                    b.bind(MetaDataIndexUpgradeService.class).toInstance(new MetaDataIndexUpgradeService(settings, xContentRegistry,
+                        indicesModule.getMapperRegistry(), settingsModule.getIndexScopedSettings(), indexMetaDataUpgraders));
+                    b.bind(ClusterInfoService.class).toInstance(clusterInfoService);
+                    b.bind(Discovery.class).toInstance(discoveryModule.getDiscovery());
+                    {
+                        RecoverySettings recoverySettings = new RecoverySettings(settings, settingsModule.getClusterSettings());
+                        processRecoverySettings(settingsModule.getClusterSettings(), recoverySettings);
+                        b.bind(PeerRecoverySourceService.class).toInstance(new PeerRecoverySourceService(settings, transportService,
+                                indicesService, recoverySettings, clusterService));
+                        b.bind(PeerRecoveryTargetService.class).toInstance(new PeerRecoveryTargetService(settings, threadPool,
+                                transportService, recoverySettings, clusterService));
                     }
+                    httpBind.accept(b);
                     pluginComponents.stream().forEach(p -> b.bind((Class) p.getClass()).toInstance(p));
-
                 }
             );
             injector = modules.createInjector();
@@ -408,9 +505,13 @@ protected Node(final Environment environment, Collection<Class<? extends Plugin>
                 .map(injector::getInstance).collect(Collectors.toList()));
             resourcesToClose.addAll(pluginLifecycleComponents);
             this.pluginLifecycleComponents = Collections.unmodifiableList(pluginLifecycleComponents);
+            client.initialize(injector.getInstance(new Key<Map<GenericAction, TransportAction>>() {}),
+                    () -> clusterService.localNode().getId());
 
-            client.intialize(injector.getInstance(new Key<Map<GenericAction, TransportAction>>() {}));
-
+            if (NetworkModule.HTTP_ENABLED.get(settings)) {
+                logger.debug("initializing HTTP handlers ...");
+                actionModule.initRestHandlers(() -> clusterService.state().nodes());
+            }
             logger.info("initialized");
 
             success = true;
@@ -423,6 +524,97 @@ protected Node(final Environment environment, Collection<Class<? extends Plugin>
         }
     }
 
+    /**
+     * Checks for path.data and default.path.data being configured, and there being index data in any of the paths in default.path.data.
+     *
+     * @param settings the settings to check for path.data and default.path.data
+     * @param nodeEnv  the current node environment
+     * @param logger   a logger where messages regarding the detection will be logged
+     * @throws IOException if an I/O exception occurs reading the directory structure
+     */
+    static void checkForIndexDataInDefaultPathData(
+            final Settings settings, final NodeEnvironment nodeEnv, final Logger logger) throws IOException {
+        if (!Environment.PATH_DATA_SETTING.exists(settings) || !Environment.DEFAULT_PATH_DATA_SETTING.exists(settings)) {
+            return;
+        }
+
+        boolean clean = true;
+        for (final String defaultPathData : Environment.DEFAULT_PATH_DATA_SETTING.get(settings)) {
+            final Path defaultNodeDirectory = NodeEnvironment.resolveNodePath(getPath(defaultPathData), nodeEnv.getNodeLockId());
+            if (Files.exists(defaultNodeDirectory) == false) {
+                continue;
+            }
+
+            if (isDefaultPathDataInPathData(nodeEnv, defaultNodeDirectory)) {
+                continue;
+            }
+
+            final NodeEnvironment.NodePath nodePath = new NodeEnvironment.NodePath(defaultNodeDirectory);
+            final Set<String> availableIndexFolders = nodeEnv.availableIndexFoldersForPath(nodePath);
+            if (availableIndexFolders.isEmpty()) {
+                continue;
+            }
+
+            clean = false;
+            logger.error("detected index data in default.path.data [{}] where there should not be any", nodePath.indicesPath);
+            for (final String availableIndexFolder : availableIndexFolders) {
+                logger.info(
+                        "index folder [{}] in default.path.data [{}] must be moved to any of {}",
+                        availableIndexFolder,
+                        nodePath.indicesPath,
+                        Arrays.stream(nodeEnv.nodePaths()).map(np -> np.indicesPath).collect(Collectors.toList()));
+            }
+        }
+
+        if (clean) {
+            return;
+        }
+
+        final String message = String.format(
+                Locale.ROOT,
+                "detected index data in default.path.data %s where there should not be any; check the logs for details",
+                Environment.DEFAULT_PATH_DATA_SETTING.get(settings));
+        throw new IllegalStateException(message);
+    }
+
+    private static boolean isDefaultPathDataInPathData(final NodeEnvironment nodeEnv, final Path defaultNodeDirectory) throws IOException {
+        for (final NodeEnvironment.NodePath dataPath : nodeEnv.nodePaths()) {
+            if (Files.isSameFile(dataPath.path, defaultNodeDirectory)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    @SuppressForbidden(reason = "read path that is not configured in environment")
+    private static Path getPath(final String path) {
+        return PathUtils.get(path);
+    }
+
+    // visible for testing
+    static void warnIfPreRelease(final Version version, final boolean isSnapshot, final Logger logger) {
+        if (!version.isRelease() || isSnapshot) {
+            logger.warn(
+                "version [{}] is a pre-release version of Elasticsearch and is not suitable for production",
+                displayVersion(version, isSnapshot));
+        }
+    }
+
+    private static String displayVersion(final Version version, final boolean isSnapshot) {
+        return version + (isSnapshot ? "-SNAPSHOT" : "");
+    }
+
+    protected TransportService newTransportService(Settings settings, Transport transport, ThreadPool threadPool,
+                                                   TransportInterceptor interceptor,
+                                                   Function<BoundTransportAddress, DiscoveryNode> localNodeFactory,
+                                                   ClusterSettings clusterSettings) {
+        return new TransportService(settings, transport, threadPool, interceptor, localNodeFactory, clusterSettings);
+    }
+
+    protected void processRecoverySettings(ClusterSettings clusterSettings, RecoverySettings recoverySettings) {
+        // Noop in production, overridden by tests
+    }
+
     /**
      * The settings that were used to create the node.
      */
@@ -455,7 +647,7 @@ public NodeEnvironment getNodeEnvironment() {
     /**
      * Start the node. If the node is already started, this method is no-op.
      */
-    public Node start() {
+    public Node start() throws NodeValidationException {
         if (!lifecycle.moveToStarted()) {
             return this;
         }
@@ -475,7 +667,6 @@ public Node start() {
         injector.getInstance(RoutingService.class).start();
         injector.getInstance(SearchService.class).start();
         injector.getInstance(MonitorService.class).start();
-        injector.getInstance(RestController.class).start();
 
         final ClusterService clusterService = injector.getInstance(ClusterService.class);
 
@@ -489,6 +680,7 @@ public Node start() {
         injector.getInstance(ResourceWatcherService.class).start();
         injector.getInstance(GatewayService.class).start();
         Discovery discovery = injector.getInstance(Discovery.class);
+        clusterService.setDiscoverySettings(discovery.getDiscoverySettings());
         clusterService.addInitialStateBlock(discovery.getDiscoverySettings().getNoMasterBlock());
         clusterService.setClusterStatePublisher(discovery::publish);
 
@@ -500,29 +692,28 @@ public Node start() {
         TransportService transportService = injector.getInstance(TransportService.class);
         transportService.getTaskManager().setTaskResultsService(injector.getInstance(TaskResultsService.class));
         transportService.start();
+        validateNodeBeforeAcceptingRequests(settings, transportService.boundAddress(), pluginsService.filterPlugins(Plugin.class).stream()
+            .flatMap(p -> p.getBootstrapChecks().stream()).collect(Collectors.toList()));
 
-        validateNodeBeforeAcceptingRequests(settings, transportService.boundAddress());
-
-        DiscoveryNode localNode = DiscoveryNode.createLocal(settings,
-            transportService.boundAddress().publishAddress(), injector.getInstance(NodeEnvironment.class).nodeId());
-
-        // TODO: need to find a cleaner way to start/construct a service with some initial parameters,
-        // playing nice with the life cycle interfaces
-        clusterService.setLocalNode(localNode);
-        transportService.setLocalNode(localNode);
-        clusterService.add(transportService.getTaskManager());
-
+        clusterService.addStateApplier(transportService.getTaskManager());
         clusterService.start();
-
+        assert localNodeFactory.getNode() != null;
+        assert transportService.getLocalNode().equals(localNodeFactory.getNode())
+            : "transportService has a different local node than the factory provided";
+        assert clusterService.localNode().equals(localNodeFactory.getNode())
+            : "clusterService has a different local node than the factory provided";
         // start after cluster service so the local disco is known
         discovery.start();
         transportService.acceptIncomingRequests();
         discovery.startInitialJoin();
-        // tribe nodes don't have a master so we shouldn't register an observer
-        if (DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING.get(settings).millis() > 0) {
+        // tribe nodes don't have a master so we shouldn't register an observer         s
+        final TimeValue initialStateTimeout = DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING.get(settings);
+        if (initialStateTimeout.millis() > 0) {
             final ThreadPool thread = injector.getInstance(ThreadPool.class);
-            ClusterStateObserver observer = new ClusterStateObserver(clusterService, null, logger, thread.getThreadContext());
-            if (observer.observedState().nodes().getMasterNodeId() == null) {
+            ClusterState clusterState = clusterService.state();
+            ClusterStateObserver observer = new ClusterStateObserver(clusterState, clusterService, null, logger, thread.getThreadContext());
+            if (clusterState.nodes().getMasterNodeId() == null) {
+                logger.debug("waiting to join the cluster. timeout [{}]", initialStateTimeout);
                 final CountDownLatch latch = new CountDownLatch(1);
                 observer.waitForNextChange(new ClusterStateObserver.Listener() {
                     @Override
@@ -536,10 +727,10 @@ public void onClusterServiceClose() {
                     @Override
                     public void onTimeout(TimeValue timeout) {
                         logger.warn("timed out while waiting for initial discovery state - timeout: {}",
-                            DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING.get(settings));
+                            initialStateTimeout);
                         latch.countDown();
                     }
-                }, MasterNodeChangePredicate.INSTANCE, DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING.get(settings));
+                }, state -> state.nodes().getMasterNodeId() != null, initialStateTimeout);
 
                 try {
                     latch.await();
@@ -549,15 +740,12 @@ public void onTimeout(TimeValue timeout) {
             }
         }
 
+
         if (NetworkModule.HTTP_ENABLED.get(settings)) {
-            injector.getInstance(HttpServer.class).start();
+            injector.getInstance(HttpServerTransport.class).start();
         }
 
-        // start nodes now, after the http server, because it may take some time
-        tribeService.startNodes();
-
-
-        if (WRITE_PORTS_FIELD_SETTING.get(settings)) {
+        if (WRITE_PORTS_FILE_SETTING.get(settings)) {
             if (NetworkModule.HTTP_ENABLED.get(settings)) {
                 HttpServerTransport http = injector.getInstance(HttpServerTransport.class);
                 writePortsFile("http", http.boundAddress());
@@ -566,6 +754,8 @@ public void onTimeout(TimeValue timeout) {
             writePortsFile("transport", transport.boundAddress());
         }
 
+        // start nodes now, after the http server, because it may take some time
+        tribeService.startNodes();
         logger.info("started");
 
         return this;
@@ -581,23 +771,24 @@ private Node stop() {
         injector.getInstance(TribeService.class).stop();
         injector.getInstance(ResourceWatcherService.class).stop();
         if (NetworkModule.HTTP_ENABLED.get(settings)) {
-            injector.getInstance(HttpServer.class).stop();
+            injector.getInstance(HttpServerTransport.class).stop();
         }
 
         injector.getInstance(SnapshotsService.class).stop();
         injector.getInstance(SnapshotShardsService.class).stop();
         // stop any changes happening as a result of cluster state changes
         injector.getInstance(IndicesClusterStateService.class).stop();
+        // close discovery early to not react to pings anymore.
+        // This can confuse other nodes and delay things - mostly if we're the master and we're running tests.
+        injector.getInstance(Discovery.class).stop();
         // we close indices first, so operations won't be allowed on it
         injector.getInstance(IndicesTTLService.class).stop();
         injector.getInstance(RoutingService.class).stop();
         injector.getInstance(ClusterService.class).stop();
-        injector.getInstance(Discovery.class).stop();
         injector.getInstance(NodeConnectionsService.class).stop();
         injector.getInstance(MonitorService.class).stop();
         injector.getInstance(GatewayService.class).stop();
         injector.getInstance(SearchService.class).stop();
-        injector.getInstance(RestController.class).stop();
         injector.getInstance(TransportService.class).stop();
 
         pluginLifecycleComponents.forEach(LifecycleComponent::stop);
@@ -606,24 +797,6 @@ private Node stop() {
         injector.getInstance(IndicesService.class).stop();
         logger.info("stopped");
 
-        final String log4jShutdownEnabled = System.getProperty("es.log4j.shutdownEnabled", "true");
-        final boolean shutdownEnabled;
-        switch (log4jShutdownEnabled) {
-            case "true":
-                shutdownEnabled = true;
-                break;
-            case "false":
-                shutdownEnabled = false;
-                break;
-            default:
-                throw new IllegalArgumentException(
-                    "invalid value for [es.log4j.shutdownEnabled], was [" + log4jShutdownEnabled + "] but must be [true] or [false]");
-        }
-        if (shutdownEnabled) {
-            LoggerContext context = (LoggerContext) LogManager.getContext(false);
-            Configurator.shutdown(context);
-        }
-
         return this;
     }
 
@@ -649,7 +822,7 @@ public synchronized void close() throws IOException {
         toClose.add(injector.getInstance(NodeService.class));
         toClose.add(() -> stopWatch.stop().start("http"));
         if (NetworkModule.HTTP_ENABLED.get(settings)) {
-            toClose.add(injector.getInstance(HttpServer.class));
+            toClose.add(injector.getInstance(HttpServerTransport.class));
         }
         toClose.add(() -> stopWatch.stop().start("snapshot_service"));
         toClose.add(injector.getInstance(SnapshotsService.class));
@@ -677,8 +850,6 @@ public synchronized void close() throws IOException {
         toClose.add(injector.getInstance(GatewayService.class));
         toClose.add(() -> stopWatch.stop().start("search"));
         toClose.add(injector.getInstance(SearchService.class));
-        toClose.add(() -> stopWatch.stop().start("rest"));
-        toClose.add(injector.getInstance(RestController.class));
         toClose.add(() -> stopWatch.stop().start("transport"));
         toClose.add(injector.getInstance(TransportService.class));
 
@@ -686,7 +857,7 @@ public synchronized void close() throws IOException {
             toClose.add(() -> stopWatch.stop().start("plugin(" + plugin.getClass().getName() + ")"));
             toClose.add(plugin);
         }
-        toClose.addAll(pluginsService.filterPlugins(Closeable.class));
+        toClose.addAll(pluginsService.filterPlugins(Plugin.class));
 
         toClose.add(() -> stopWatch.stop().start("script"));
         toClose.add(injector.getInstance(ScriptService.class));
@@ -740,7 +911,9 @@ public Injector injector() {
      *                              bound and publishing to
      */
     @SuppressWarnings("unused")
-    protected void validateNodeBeforeAcceptingRequests(Settings settings, BoundTransportAddress boundTransportAddress) {
+    protected void validateNodeBeforeAcceptingRequests(
+        final Settings settings,
+        final BoundTransportAddress boundTransportAddress, List<BootstrapCheck> bootstrapChecks) throws NodeValidationException {
     }
 
     /** Writes a file to the logs dir containing the ports for the given transport type */
@@ -797,10 +970,12 @@ BigArrays createBigArrays(Settings settings, CircuitBreakerService circuitBreake
     }
 
     /**
-     * Select the search service implementation. Overrided by tests.
+     * Creates a new the SearchService. This method can be overwritten by tests to inject mock implementations.
      */
-    protected Class<? extends SearchService> pickSearchServiceImplementation() {
-        return SearchService.class;
+    protected SearchService newSearchService(ClusterService clusterService, IndicesService indicesService,
+                                             ThreadPool threadPool, ScriptService scriptService, BigArrays bigArrays,
+                                             FetchPhase fetchPhase) {
+        return new SearchService(clusterService, indicesService, threadPool, scriptService, bigArrays, fetchPhase);
     }
 
     /**
@@ -817,4 +992,37 @@ private List<NetworkService.CustomNameResolver> getCustomNameResolvers(List<Disc
         }
         return customNameResolvers;
     }
+
+    /** Constructs an internal node used as a client into a cluster fronted by this tribe node. */
+    protected Node newTribeClientNode(Settings settings, Collection<Class<? extends Plugin>> classpathPlugins) {
+        return new Node(new Environment(settings), classpathPlugins);
+    }
+
+    /** Constructs a ClusterInfoService which may be mocked for tests. */
+    protected ClusterInfoService newClusterInfoService(Settings settings, ClusterService clusterService,
+                                                       ThreadPool threadPool, NodeClient client) {
+        return new InternalClusterInfoService(settings, clusterService, threadPool, client);
+    }
+
+    private static class LocalNodeFactory implements Function<BoundTransportAddress, DiscoveryNode> {
+        private final SetOnce<DiscoveryNode> localNode = new SetOnce<>();
+        private final String persistentNodeId;
+        private final Settings settings;
+
+        private LocalNodeFactory(Settings settings, String persistentNodeId) {
+            this.persistentNodeId = persistentNodeId;
+            this.settings = settings;
+        }
+
+        @Override
+        public DiscoveryNode apply(BoundTransportAddress boundTransportAddress) {
+            localNode.set(DiscoveryNode.createLocal(settings, boundTransportAddress.publishAddress(), persistentNodeId));
+            return localNode.get();
+        }
+
+        DiscoveryNode getNode() {
+            assert localNode.get() != null;
+            return localNode.get();
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/node/NodeModule.java b/core/src/main/java/org/elasticsearch/node/NodeModule.java
index 6a8f8b90681bb..929e889503ea4 100644
--- a/core/src/main/java/org/elasticsearch/node/NodeModule.java
+++ b/core/src/main/java/org/elasticsearch/node/NodeModule.java
@@ -22,7 +22,6 @@
 import org.elasticsearch.cluster.routing.allocation.DiskThresholdMonitor;
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.monitor.MonitorService;
-import org.elasticsearch.node.service.NodeService;
 
 public class NodeModule extends AbstractModule {
 
@@ -38,7 +37,6 @@ public NodeModule(Node node, MonitorService monitorService) {
     protected void configure() {
         bind(Node.class).toInstance(node);
         bind(MonitorService.class).toInstance(monitorService);
-        bind(NodeService.class).asEagerSingleton();
         bind(DiskThresholdMonitor.class).asEagerSingleton();
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/node/service/NodeService.java b/core/src/main/java/org/elasticsearch/node/NodeService.java
similarity index 86%
rename from core/src/main/java/org/elasticsearch/node/service/NodeService.java
rename to core/src/main/java/org/elasticsearch/node/NodeService.java
index 39e151c886fcc..cb245487152e3 100644
--- a/core/src/main/java/org/elasticsearch/node/service/NodeService.java
+++ b/core/src/main/java/org/elasticsearch/node/NodeService.java
@@ -17,10 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.node.service;
-
-import java.io.Closeable;
-import java.io.IOException;
+package org.elasticsearch.node;
 
 import org.elasticsearch.Build;
 import org.elasticsearch.Version;
@@ -30,11 +27,10 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.discovery.Discovery;
-import org.elasticsearch.http.HttpServer;
+import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.ingest.IngestService;
@@ -44,8 +40,9 @@
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
-/**
- */
+import java.io.Closeable;
+import java.io.IOException;
+
 public class NodeService extends AbstractComponent implements Closeable {
 
     private final ThreadPool threadPool;
@@ -57,17 +54,16 @@ public class NodeService extends AbstractComponent implements Closeable {
     private final IngestService ingestService;
     private final SettingsFilter settingsFilter;
     private ScriptService scriptService;
+    private final HttpServerTransport httpServerTransport;
 
-    @Nullable
-    private final HttpServer httpServer;
 
     private final Discovery discovery;
 
-    @Inject
-    public NodeService(Settings settings, ThreadPool threadPool, MonitorService monitorService, Discovery discovery,
+    NodeService(Settings settings, ThreadPool threadPool, MonitorService monitorService, Discovery discovery,
                        TransportService transportService, IndicesService indicesService, PluginsService pluginService,
-                       CircuitBreakerService circuitBreakerService, ScriptService scriptService, @Nullable HttpServer httpServer,
-                       IngestService ingestService, ClusterService clusterService, SettingsFilter settingsFilter) {
+                       CircuitBreakerService circuitBreakerService, ScriptService scriptService,
+                       @Nullable HttpServerTransport httpServerTransport, IngestService ingestService, ClusterService clusterService,
+                       SettingsFilter settingsFilter) {
         super(settings);
         this.threadPool = threadPool;
         this.monitorService = monitorService;
@@ -76,12 +72,12 @@ public NodeService(Settings settings, ThreadPool threadPool, MonitorService moni
         this.discovery = discovery;
         this.pluginService = pluginService;
         this.circuitBreakerService = circuitBreakerService;
-        this.httpServer = httpServer;
+        this.httpServerTransport = httpServerTransport;
         this.ingestService = ingestService;
         this.settingsFilter = settingsFilter;
         this.scriptService = scriptService;
-        clusterService.add(ingestService.getPipelineStore());
-        clusterService.add(ingestService.getPipelineExecutionService());
+        clusterService.addStateApplier(ingestService.getPipelineStore());
+        clusterService.addStateApplier(ingestService.getPipelineExecutionService());
     }
 
     public NodeInfo info(boolean settings, boolean os, boolean process, boolean jvm, boolean threadPool,
@@ -93,7 +89,7 @@ public NodeInfo info(boolean settings, boolean os, boolean process, boolean jvm,
                 jvm ? monitorService.jvmService().info() : null,
                 threadPool ? this.threadPool.info() : null,
                 transport ? transportService.info() : null,
-                http ? (httpServer == null ? null : httpServer.info()) : null,
+                http ? (httpServerTransport == null ? null : httpServerTransport.info()) : null,
                 plugin ? (pluginService == null ? null : pluginService.info()) : null,
                 ingest ? (ingestService == null ? null : ingestService.info()) : null,
                 indices ? indicesService.getTotalIndexingBufferBytes() : null
@@ -113,7 +109,7 @@ public NodeStats stats(CommonStatsFlags indices, boolean os, boolean process, bo
                 threadPool ? this.threadPool.stats() : null,
                 fs ? monitorService.fsService().stats() : null,
                 transport ? transportService.stats() : null,
-                http ? httpServer.stats() : null,
+                http ? (httpServerTransport == null ? null : httpServerTransport.stats()) : null,
                 circuitBreaker ? circuitBreakerService.stats() : null,
                 script ? scriptService.stats() : null,
                 discoveryStats ? discovery.stats() : null,
@@ -129,4 +125,5 @@ public IngestService getIngestService() {
     public void close() throws IOException {
         indicesService.close();
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/node/NodeValidationException.java b/core/src/main/java/org/elasticsearch/node/NodeValidationException.java
new file mode 100644
index 0000000000000..884fc8e8b6ab1
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/node/NodeValidationException.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.node;
+
+import org.elasticsearch.bootstrap.BootstrapCheck;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.BoundTransportAddress;
+
+import java.util.List;
+
+/**
+ * An exception thrown during node validation. Node validation runs immediately before a node
+ * begins accepting network requests in
+ * {@link Node#validateNodeBeforeAcceptingRequests(Settings, BoundTransportAddress, List)}. This exception is a checked exception that is
+ * declared as thrown from this method for the purpose of bubbling up to the user.
+ */
+public class NodeValidationException extends Exception {
+
+    /**
+     * Creates a node validation exception with the specified validation message to be displayed to
+     * the user.
+     *
+     * @param message the message to display to the user
+     */
+    public NodeValidationException(final String message) {
+        super(message);
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/plugins/ActionPlugin.java b/core/src/main/java/org/elasticsearch/plugins/ActionPlugin.java
index 3d769d27a87da..346bf491d619b 100644
--- a/core/src/main/java/org/elasticsearch/plugins/ActionPlugin.java
+++ b/core/src/main/java/org/elasticsearch/plugins/ActionPlugin.java
@@ -25,13 +25,24 @@
 import org.elasticsearch.action.support.ActionFilter;
 import org.elasticsearch.action.support.TransportAction;
 import org.elasticsearch.action.support.TransportActions;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.IndexScopedSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsFilter;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestHandler;
 
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
+import java.util.function.Supplier;
+import java.util.function.UnaryOperator;
 
 /**
  * An additional extension point for {@link Plugin}s that extends Elasticsearch's scripting functionality. Implement it like this:
@@ -49,7 +60,7 @@ public interface ActionPlugin {
     /**
      * Actions added by this plugin.
      */
-    default List<ActionHandler<? extends ActionRequest<?>, ? extends ActionResponse>> getActions() {
+    default List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
         return Collections.emptyList();
     }
     /**
@@ -61,7 +72,9 @@ default List<Class<? extends ActionFilter>> getActionFilters() {
     /**
      * Rest handlers added by this plugin.
      */
-    default List<Class<? extends RestHandler>> getRestHandlers() {
+    default List<RestHandler> getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings,
+            IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter,
+            IndexNameExpressionResolver indexNameExpressionResolver, Supplier<DiscoveryNodes> nodesInCluster) {
         return Collections.emptyList();
     }
 
@@ -72,7 +85,16 @@ default Collection<String> getRestHeaders() {
         return Collections.emptyList();
     }
 
-    final class ActionHandler<Request extends ActionRequest<Request>, Response extends ActionResponse> {
+    /**
+     * Returns a function used to wrap each rest request before handling the request.
+     *
+     * Note: Only one installed plugin may implement a rest wrapper.
+     */
+    default UnaryOperator<RestHandler> getRestHandlerWrapper(ThreadContext threadContext) {
+        return null;
+    }
+
+    final class ActionHandler<Request extends ActionRequest, Response extends ActionResponse> {
         private final GenericAction<Request, Response> action;
         private final Class<? extends TransportAction<Request, Response>> transportAction;
         private final Class<?>[] supportTransportActions;
diff --git a/core/src/main/java/org/elasticsearch/plugins/DiscoveryPlugin.java b/core/src/main/java/org/elasticsearch/plugins/DiscoveryPlugin.java
index f6174c08d124f..61e87d83a183f 100644
--- a/core/src/main/java/org/elasticsearch/plugins/DiscoveryPlugin.java
+++ b/core/src/main/java/org/elasticsearch/plugins/DiscoveryPlugin.java
@@ -19,8 +19,19 @@
 
 package org.elasticsearch.plugins;
 
+import java.util.Collections;
+import java.util.Map;
+import java.util.function.Supplier;
+
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.discovery.Discovery;
+import org.elasticsearch.discovery.zen.UnicastHostsProvider;
+import org.elasticsearch.discovery.zen.ZenPing;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
 
 /**
  * An additional extension point for {@link Plugin}s that extends Elasticsearch's discovery functionality. To add an additional
@@ -36,6 +47,25 @@
  * }</pre>
  */
 public interface DiscoveryPlugin {
+
+    /**
+     * Returns custom discovery implementations added by this plugin.
+     *
+     * The key of the returned map is the name of the discovery implementation
+     * (see {@link org.elasticsearch.discovery.DiscoveryModule#DISCOVERY_TYPE_SETTING}, and
+     * the value is a supplier to construct the {@link Discovery}.
+     *
+     * @param threadPool Use to schedule ping actions
+     * @param transportService Use to communicate with other nodes
+     * @param clusterService Use to find current nodes in the cluster
+     * @param hostsProvider Use to find configured hosts which should be pinged for initial discovery
+     */
+    default Map<String, Supplier<Discovery>> getDiscoveryTypes(ThreadPool threadPool, TransportService transportService,
+                                                               NamedWriteableRegistry namedWriteableRegistry,
+                                                               ClusterService clusterService, UnicastHostsProvider hostsProvider) {
+        return Collections.emptyMap();
+    }
+
     /**
      * Override to add additional {@link NetworkService.CustomNameResolver}s.
      * This can be handy if you want to provide your own Network interface name like _mycard_
@@ -52,4 +82,20 @@ public interface DiscoveryPlugin {
     default NetworkService.CustomNameResolver getCustomNameResolver(Settings settings) {
         return null;
     }
+
+    /**
+     * Returns providers of unicast host lists for zen discovery.
+     *
+     * The key of the returned map is the name of the host provider
+     * (see {@link org.elasticsearch.discovery.DiscoveryModule#DISCOVERY_HOSTS_PROVIDER_SETTING}), and
+     * the value is a supplier to construct the host provider when it is selected for use.
+     *
+     * @param transportService Use to form the {@link org.elasticsearch.common.transport.TransportAddress} portion
+     *                         of a {@link org.elasticsearch.cluster.node.DiscoveryNode}
+     * @param networkService Use to find the publish host address of the current node
+     */
+    default Map<String, Supplier<UnicastHostsProvider>> getZenHostsProviders(TransportService transportService,
+                                                                             NetworkService networkService) {
+        return Collections.emptyMap();
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/plugins/DummyPluginInfo.java b/core/src/main/java/org/elasticsearch/plugins/DummyPluginInfo.java
index 90b1d32f4ae4a..73a3811f729f7 100644
--- a/core/src/main/java/org/elasticsearch/plugins/DummyPluginInfo.java
+++ b/core/src/main/java/org/elasticsearch/plugins/DummyPluginInfo.java
@@ -21,9 +21,13 @@
 public class DummyPluginInfo extends PluginInfo {
 
     private DummyPluginInfo(String name, String description, String version, String classname) {
-        super(name, description, version, classname);
+        super(name, description, version, classname, false);
     }
 
-    public static final DummyPluginInfo INSTANCE = new DummyPluginInfo(
-            "dummy_plugin_name", "dummy plugin description", "dummy_plugin_version", "DummyPluginName");
+    public static final DummyPluginInfo INSTANCE =
+            new DummyPluginInfo(
+                    "dummy_plugin_name",
+                    "dummy plugin description",
+                    "dummy_plugin_version",
+                    "DummyPluginName");
 }
diff --git a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
deleted file mode 100644
index 6579257b5a775..0000000000000
--- a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
+++ /dev/null
@@ -1,590 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.plugins;
-
-import joptsimple.OptionSet;
-import joptsimple.OptionSpec;
-import org.apache.lucene.search.spell.LevensteinDistance;
-import org.apache.lucene.util.CollectionUtil;
-import org.apache.lucene.util.IOUtils;
-import org.elasticsearch.Version;
-import org.elasticsearch.bootstrap.JarHell;
-import org.elasticsearch.cli.ExitCodes;
-import org.elasticsearch.cli.SettingCommand;
-import org.elasticsearch.cli.Terminal;
-import org.elasticsearch.cli.UserException;
-import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.hash.MessageDigests;
-import org.elasticsearch.common.io.FileSystemUtils;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.node.internal.InternalSettingsPreparer;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.OutputStream;
-import java.net.URL;
-import java.net.URLConnection;
-import java.net.URLDecoder;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.DirectoryStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.StandardCopyOption;
-import java.nio.file.attribute.PosixFileAttributeView;
-import java.nio.file.attribute.PosixFileAttributes;
-import java.nio.file.attribute.PosixFilePermission;
-import java.nio.file.attribute.PosixFilePermissions;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.TreeSet;
-import java.util.stream.Collectors;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipInputStream;
-
-import static org.elasticsearch.cli.Terminal.Verbosity.VERBOSE;
-
-/**
- * A command for the plugin cli to install a plugin into elasticsearch.
- *
- * The install command takes a plugin id, which may be any of the following:
- * <ul>
- * <li>An official elasticsearch plugin name</li>
- * <li>Maven coordinates to a plugin zip</li>
- * <li>A URL to a plugin zip</li>
- * </ul>
- *
- * Plugins are packaged as zip files. Each packaged plugin must contain a
- * plugin properties file. See {@link PluginInfo}.
- * <p>
- * The installation process first extracts the plugin files into a temporary
- * directory in order to verify the plugin satisfies the following requirements:
- * <ul>
- * <li>Jar hell does not exist, either between the plugin's own jars, or with elasticsearch</li>
- * <li>The plugin is not a module already provided with elasticsearch</li>
- * <li>If the plugin contains extra security permissions, the policy file is validated</li>
- * </ul>
- * <p>
- * A plugin may also contain an optional {@code bin} directory which contains scripts. The
- * scripts will be installed into a subdirectory of the elasticsearch bin directory, using
- * the name of the plugin, and the scripts will be marked executable.
- * <p>
- * A plugin may also contain an optional {@code config} directory which contains configuration
- * files specific to the plugin. The config files be installed into a subdirectory of the
- * elasticsearch config directory, using the name of the plugin. If any files to be installed
- * already exist, they will be skipped.
- */
-class InstallPluginCommand extends SettingCommand {
-
-    private static final String PROPERTY_STAGING_ID = "es.plugins.staging";
-
-    /** The builtin modules, which are plugins, but cannot be installed or removed. */
-    static final Set<String> MODULES;
-    static {
-        try (InputStream stream = InstallPluginCommand.class.getResourceAsStream("/modules.txt");
-            BufferedReader reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) {
-            Set<String> modules = new HashSet<>();
-            String line = reader.readLine();
-            while (line != null) {
-                modules.add(line.trim());
-                line = reader.readLine();
-            }
-            MODULES = Collections.unmodifiableSet(modules);
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    /** The official plugins that can be installed simply by name. */
-    static final Set<String> OFFICIAL_PLUGINS;
-    static {
-        try (InputStream stream = InstallPluginCommand.class.getResourceAsStream("/plugins.txt");
-            BufferedReader reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) {
-            Set<String> plugins = new TreeSet<>(); // use tree set to get sorting for help command
-            String line = reader.readLine();
-            while (line != null) {
-                plugins.add(line.trim());
-                line = reader.readLine();
-            }
-            plugins.add("x-pack");
-            OFFICIAL_PLUGINS = Collections.unmodifiableSet(plugins);
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    private final OptionSpec<Void> batchOption;
-    private final OptionSpec<String> arguments;
-
-
-    static final Set<PosixFilePermission> DIR_AND_EXECUTABLE_PERMS;
-    static final Set<PosixFilePermission> FILE_PERMS;
-
-    static {
-        Set<PosixFilePermission> dirAndExecutablePerms = new HashSet<>(7);
-        // Directories and executables get chmod 755
-        dirAndExecutablePerms.add(PosixFilePermission.OWNER_EXECUTE);
-        dirAndExecutablePerms.add(PosixFilePermission.OWNER_READ);
-        dirAndExecutablePerms.add(PosixFilePermission.OWNER_WRITE);
-        dirAndExecutablePerms.add(PosixFilePermission.GROUP_EXECUTE);
-        dirAndExecutablePerms.add(PosixFilePermission.GROUP_READ);
-        dirAndExecutablePerms.add(PosixFilePermission.OTHERS_READ);
-        dirAndExecutablePerms.add(PosixFilePermission.OTHERS_EXECUTE);
-        DIR_AND_EXECUTABLE_PERMS = Collections.unmodifiableSet(dirAndExecutablePerms);
-
-        Set<PosixFilePermission> filePerms = new HashSet<>(4);
-        // Files get chmod 644
-        filePerms.add(PosixFilePermission.OWNER_READ);
-        filePerms.add(PosixFilePermission.OWNER_WRITE);
-        filePerms.add(PosixFilePermission.GROUP_READ);
-        filePerms.add(PosixFilePermission.OTHERS_READ);
-        FILE_PERMS = Collections.unmodifiableSet(filePerms);
-    }
-
-    InstallPluginCommand() {
-        super("Install a plugin");
-        this.batchOption = parser.acceptsAll(Arrays.asList("b", "batch"),
-                "Enable batch mode explicitly, automatic confirmation of security permission");
-        this.arguments = parser.nonOptions("plugin id");
-    }
-
-    @Override
-    protected void printAdditionalHelp(Terminal terminal) {
-        terminal.println("The following official plugins may be installed by name:");
-        for (String plugin : OFFICIAL_PLUGINS) {
-            terminal.println("  " + plugin);
-        }
-        terminal.println("");
-    }
-
-    @Override
-    protected void execute(Terminal terminal, OptionSet options, Map<String, String> settings) throws Exception {
-        String pluginId = arguments.value(options);
-        boolean isBatch = options.has(batchOption) || System.console() == null;
-        execute(terminal, pluginId, isBatch, settings);
-    }
-
-    // pkg private for testing
-    void execute(Terminal terminal, String pluginId, boolean isBatch, Map<String, String> settings) throws Exception {
-        final Environment env = InternalSettingsPreparer.prepareEnvironment(Settings.EMPTY, terminal, settings);
-        // TODO: remove this leniency!! is it needed anymore?
-        if (Files.exists(env.pluginsFile()) == false) {
-            terminal.println("Plugins directory [" + env.pluginsFile() + "] does not exist. Creating...");
-            Files.createDirectory(env.pluginsFile());
-        }
-
-        Path pluginZip = download(terminal, pluginId, env.tmpFile());
-        Path extractedZip = unzip(pluginZip, env.pluginsFile());
-        install(terminal, isBatch, extractedZip, env);
-    }
-
-    /** Downloads the plugin and returns the file it was downloaded to. */
-    private Path download(Terminal terminal, String pluginId, Path tmpDir) throws Exception {
-        if (OFFICIAL_PLUGINS.contains(pluginId)) {
-            final String version = Version.CURRENT.toString();
-            final String url;
-            final String stagingHash = System.getProperty(PROPERTY_STAGING_ID);
-            if (stagingHash != null) {
-                url = String.format(Locale.ROOT,
-                    "https://staging.elastic.co/%3$s-%1$s/download/elasticsearch-plugins/%2$s/%2$s-%3$s.zip",
-                    stagingHash, pluginId, version);
-            } else {
-                url = String.format(Locale.ROOT,
-                    "https://artifacts.elastic.co/download/elasticsearch-plugins/%1$s/%1$s-%2$s.zip",
-                    pluginId, version);
-            }
-            terminal.println("-> Downloading " + pluginId + " from elastic");
-            return downloadZipAndChecksum(terminal, url, tmpDir);
-        }
-
-        // now try as maven coordinates, a valid URL would only have a colon and slash
-        String[] coordinates = pluginId.split(":");
-        if (coordinates.length == 3 && pluginId.contains("/") == false) {
-            String mavenUrl = String.format(Locale.ROOT, "https://repo1.maven.org/maven2/%1$s/%2$s/%3$s/%2$s-%3$s.zip",
-                    coordinates[0].replace(".", "/") /* groupId */, coordinates[1] /* artifactId */, coordinates[2] /* version */);
-            terminal.println("-> Downloading " + pluginId + " from maven central");
-            return downloadZipAndChecksum(terminal, mavenUrl, tmpDir);
-        }
-
-        // fall back to plain old URL
-        if (pluginId.contains(":/") == false) {
-            // definitely not a valid url, so assume it is a plugin name
-            List<String> plugins = checkMisspelledPlugin(pluginId);
-            String msg = "Unknown plugin " + pluginId;
-            if (plugins.isEmpty() == false) {
-                msg += ", did you mean " + (plugins.size() == 1 ? "[" + plugins.get(0) + "]": "any of " + plugins.toString()) + "?";
-            }
-            throw new UserException(ExitCodes.USAGE, msg);
-        }
-        terminal.println("-> Downloading " + URLDecoder.decode(pluginId, "UTF-8"));
-        return downloadZip(terminal, pluginId, tmpDir);
-    }
-
-    /** Returns all the official plugin names that look similar to pluginId. **/
-    private List<String> checkMisspelledPlugin(String pluginId) {
-        LevensteinDistance ld = new LevensteinDistance();
-        List<Tuple<Float, String>> scoredKeys = new ArrayList<>();
-        for (String officialPlugin : OFFICIAL_PLUGINS) {
-            float distance = ld.getDistance(pluginId, officialPlugin);
-            if (distance > 0.7f) {
-                scoredKeys.add(new Tuple<>(distance, officialPlugin));
-            }
-        }
-        CollectionUtil.timSort(scoredKeys, (a, b) -> b.v1().compareTo(a.v1()));
-        return scoredKeys.stream().map((a) -> a.v2()).collect(Collectors.toList());
-    }
-
-    /** Downloads a zip from the url, into a temp file under the given temp dir. */
-    private Path downloadZip(Terminal terminal, String urlString, Path tmpDir) throws IOException {
-        terminal.println(VERBOSE, "Retrieving zip from " + urlString);
-        URL url = new URL(urlString);
-        Path zip = Files.createTempFile(tmpDir, null, ".zip");
-        URLConnection urlConnection = url.openConnection();
-        int contentLength = urlConnection.getContentLength();
-        try (InputStream in = new TerminalProgressInputStream(urlConnection.getInputStream(), contentLength, terminal)) {
-            // must overwrite since creating the temp file above actually created the file
-            Files.copy(in, zip, StandardCopyOption.REPLACE_EXISTING);
-        }
-        return zip;
-    }
-
-    /**
-     * content length might be -1 for unknown and progress only makes sense if the content length is greater than 0
-     */
-    private class TerminalProgressInputStream extends ProgressInputStream {
-
-        private final Terminal terminal;
-        private int width = 50;
-        private final boolean enabled;
-
-        public TerminalProgressInputStream(InputStream is, int expectedTotalSize, Terminal terminal) {
-            super(is, expectedTotalSize);
-            this.terminal = terminal;
-            this.enabled = expectedTotalSize > 0;
-        }
-
-        @Override
-        public void onProgress(int percent) {
-            if (enabled) {
-                int currentPosition = percent * width / 100;
-                StringBuilder sb = new StringBuilder("\r[");
-                sb.append(String.join("=", Collections.nCopies(currentPosition, "")));
-                if (currentPosition > 0 && percent < 100) {
-                    sb.append(">");
-                }
-                sb.append(String.join(" ", Collections.nCopies(width - currentPosition, "")));
-                sb.append("] %s   ");
-                if (percent == 100) {
-                    sb.append("\n");
-                }
-                terminal.print(Terminal.Verbosity.NORMAL, String.format(Locale.ROOT, sb.toString(), percent + "%"));
-            }
-        }
-    }
-
-    /** Downloads a zip from the url, as well as a SHA1 checksum, and checks the checksum. */
-    private Path downloadZipAndChecksum(Terminal terminal, String urlString, Path tmpDir) throws Exception {
-        Path zip = downloadZip(terminal, urlString, tmpDir);
-
-        URL checksumUrl = new URL(urlString + ".sha1");
-        final String expectedChecksum;
-        try (InputStream in = checksumUrl.openStream()) {
-            BufferedReader checksumReader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8));
-            expectedChecksum = checksumReader.readLine();
-            if (checksumReader.readLine() != null) {
-                throw new UserException(ExitCodes.IO_ERROR, "Invalid checksum file at " + checksumUrl);
-            }
-        }
-
-        byte[] zipbytes = Files.readAllBytes(zip);
-        String gotChecksum = MessageDigests.toHexString(MessageDigests.sha1().digest(zipbytes));
-        if (expectedChecksum.equals(gotChecksum) == false) {
-            throw new UserException(ExitCodes.IO_ERROR, "SHA1 mismatch, expected " + expectedChecksum + " but got " + gotChecksum);
-        }
-
-        return zip;
-    }
-
-    private Path unzip(Path zip, Path pluginsDir) throws IOException, UserException {
-        // unzip plugin to a staging temp dir
-
-        final Path target = stagingDirectory(pluginsDir);
-
-        boolean hasEsDir = false;
-        // TODO: we should wrap this in a try/catch and try deleting the target dir on failure?
-        try (ZipInputStream zipInput = new ZipInputStream(Files.newInputStream(zip))) {
-            ZipEntry entry;
-            byte[] buffer = new byte[8192];
-            while ((entry = zipInput.getNextEntry()) != null) {
-                if (entry.getName().startsWith("elasticsearch/") == false) {
-                    // only extract the elasticsearch directory
-                    continue;
-                }
-                hasEsDir = true;
-                Path targetFile = target.resolve(entry.getName().substring("elasticsearch/".length()));
-
-                // Using the entry name as a path can result in an entry outside of the plugin dir, either if the
-                // name starts with the root of the filesystem, or it is a relative entry like ../whatever.
-                // This check attempts to identify both cases by first normalizing the path (which removes foo/..)
-                // and ensuring the normalized entry is still rooted with the target plugin directory.
-                if (targetFile.normalize().startsWith(target) == false) {
-                    throw new IOException("Zip contains entry name '" + entry.getName() + "' resolving outside of plugin directory");
-                }
-
-                // be on the safe side: do not rely on that directories are always extracted
-                // before their children (although this makes sense, but is it guaranteed?)
-                if (!Files.isSymbolicLink(targetFile.getParent())) {
-                    Files.createDirectories(targetFile.getParent());
-                }
-                if (entry.isDirectory() == false) {
-                    try (OutputStream out = Files.newOutputStream(targetFile)) {
-                        int len;
-                        while ((len = zipInput.read(buffer)) >= 0) {
-                            out.write(buffer, 0, len);
-                        }
-                    }
-                }
-                zipInput.closeEntry();
-            }
-        }
-        Files.delete(zip);
-        if (hasEsDir == false) {
-            IOUtils.rm(target);
-            throw new UserException(ExitCodes.DATA_ERROR, "`elasticsearch` directory is missing in the plugin zip");
-        }
-        return target;
-    }
-
-    private Path stagingDirectory(Path pluginsDir) throws IOException {
-        try {
-            return Files.createTempDirectory(pluginsDir, ".installing-", PosixFilePermissions.asFileAttribute(DIR_AND_EXECUTABLE_PERMS));
-        } catch (IllegalArgumentException e) {
-            // Jimfs throws an IAE where it should throw an UOE
-            // remove when google/jimfs#30 is integrated into Jimfs
-            // and the Jimfs test dependency is upgraded to include
-            // this pull request
-            final StackTraceElement[] elements = e.getStackTrace();
-            if (elements.length >= 1 &&
-                elements[0].getClassName().equals("com.google.common.jimfs.AttributeService") &&
-                elements[0].getMethodName().equals("setAttributeInternal")) {
-                return stagingDirectoryWithoutPosixPermissions(pluginsDir);
-            } else {
-                throw e;
-            }
-        } catch (UnsupportedOperationException e) {
-            return stagingDirectoryWithoutPosixPermissions(pluginsDir);
-        }
-    }
-
-    private Path stagingDirectoryWithoutPosixPermissions(Path pluginsDir) throws IOException {
-        return Files.createTempDirectory(pluginsDir, ".installing-");
-    }
-
-    /** Load information about the plugin, and verify it can be installed with no errors. */
-    private PluginInfo verify(Terminal terminal, Path pluginRoot, boolean isBatch, Environment env) throws Exception {
-        // read and validate the plugin descriptor
-        PluginInfo info = PluginInfo.readFromProperties(pluginRoot);
-        terminal.println(VERBOSE, info.toString());
-
-        // don't let luser install plugin as a module...
-        // they might be unavoidably in maven central and are packaged up the same way)
-        if (MODULES.contains(info.getName())) {
-            throw new UserException(
-                    ExitCodes.USAGE, "plugin '" + info.getName() + "' cannot be installed like this, it is a system module");
-        }
-
-        // check for jar hell before any copying
-        jarHellCheck(pluginRoot, env.pluginsFile());
-
-        // read optional security policy (extra permissions)
-        // if it exists, confirm or warn the user
-        Path policy = pluginRoot.resolve(PluginInfo.ES_PLUGIN_POLICY);
-        if (Files.exists(policy)) {
-            PluginSecurity.readPolicy(policy, terminal, env, isBatch);
-        }
-
-        return info;
-    }
-
-    /** check a candidate plugin for jar hell before installing it */
-    void jarHellCheck(Path candidate, Path pluginsDir) throws Exception {
-        // create list of current jars in classpath
-        final List<URL> jars = new ArrayList<>();
-        jars.addAll(Arrays.asList(JarHell.parseClassPath()));
-
-        // read existing bundles. this does some checks on the installation too.
-        PluginsService.getPluginBundles(pluginsDir);
-
-        // add plugin jars to the list
-        Path pluginJars[] = FileSystemUtils.files(candidate, "*.jar");
-        for (Path jar : pluginJars) {
-            jars.add(jar.toUri().toURL());
-        }
-        // TODO: no jars should be an error
-        // TODO: verify the classname exists in one of the jars!
-
-        // check combined (current classpath + new jars to-be-added)
-        JarHell.checkJarHell(jars.toArray(new URL[jars.size()]));
-    }
-
-    /**
-     * Installs the plugin from {@code tmpRoot} into the plugins dir.
-     * If the plugin has a bin dir and/or a config dir, those are copied.
-     */
-    private void install(Terminal terminal, boolean isBatch, Path tmpRoot, Environment env) throws Exception {
-        List<Path> deleteOnFailure = new ArrayList<>();
-        deleteOnFailure.add(tmpRoot);
-
-        try {
-            PluginInfo info = verify(terminal, tmpRoot, isBatch, env);
-
-            final Path destination = env.pluginsFile().resolve(info.getName());
-            if (Files.exists(destination)) {
-                throw new UserException(
-                    ExitCodes.USAGE,
-                    "plugin directory " + destination.toAbsolutePath() +
-                        " already exists. To update the plugin, uninstall it first using 'remove " + info.getName() + "' command");
-            }
-
-            Path tmpBinDir = tmpRoot.resolve("bin");
-            if (Files.exists(tmpBinDir)) {
-                Path destBinDir = env.binFile().resolve(info.getName());
-                deleteOnFailure.add(destBinDir);
-                installBin(info, tmpBinDir, destBinDir);
-            }
-
-            Path tmpConfigDir = tmpRoot.resolve("config");
-            if (Files.exists(tmpConfigDir)) {
-                // some files may already exist, and we don't remove plugin config files on plugin removal,
-                // so any installed config files are left on failure too
-                installConfig(info, tmpConfigDir, env.configFile().resolve(info.getName()));
-            }
-
-            Files.move(tmpRoot, destination, StandardCopyOption.ATOMIC_MOVE);
-            try (DirectoryStream<Path> stream = Files.newDirectoryStream(destination)) {
-                for (Path pluginFile : stream) {
-                    if (Files.isDirectory(pluginFile)) {
-                        setFileAttributes(pluginFile, DIR_AND_EXECUTABLE_PERMS);
-                    } else {
-                        setFileAttributes(pluginFile, FILE_PERMS);
-                    }
-                }
-            }
-            terminal.println("-> Installed " + info.getName());
-
-        } catch (Exception installProblem) {
-            try {
-                IOUtils.rm(deleteOnFailure.toArray(new Path[0]));
-            } catch (IOException exceptionWhileRemovingFiles) {
-                installProblem.addSuppressed(exceptionWhileRemovingFiles);
-            }
-            throw installProblem;
-        }
-    }
-
-    /** Copies the files from {@code tmpBinDir} into {@code destBinDir}, along with permissions from dest dirs parent. */
-    private void installBin(PluginInfo info, Path tmpBinDir, Path destBinDir) throws Exception {
-        if (Files.isDirectory(tmpBinDir) == false) {
-            throw new UserException(ExitCodes.IO_ERROR, "bin in plugin " + info.getName() + " is not a directory");
-        }
-        Files.createDirectory(destBinDir);
-        setFileAttributes(destBinDir, DIR_AND_EXECUTABLE_PERMS);
-
-        try (DirectoryStream<Path> stream = Files.newDirectoryStream(tmpBinDir)) {
-            for (Path srcFile : stream) {
-                if (Files.isDirectory(srcFile)) {
-                    throw new UserException(
-                        ExitCodes.DATA_ERROR,
-                        "Directories not allowed in bin dir for plugin " + info.getName() + ", found " + srcFile.getFileName());
-                }
-
-                Path destFile = destBinDir.resolve(tmpBinDir.relativize(srcFile));
-                Files.copy(srcFile, destFile);
-                setFileAttributes(destFile, DIR_AND_EXECUTABLE_PERMS);
-            }
-        }
-        IOUtils.rm(tmpBinDir); // clean up what we just copied
-    }
-
-    /**
-     * Copies the files from {@code tmpConfigDir} into {@code destConfigDir}.
-     * Any files existing in both the source and destination will be skipped.
-     */
-    private void installConfig(PluginInfo info, Path tmpConfigDir, Path destConfigDir) throws Exception {
-        if (Files.isDirectory(tmpConfigDir) == false) {
-            throw new UserException(ExitCodes.IO_ERROR, "config in plugin " + info.getName() + " is not a directory");
-        }
-
-        Files.createDirectories(destConfigDir);
-        setFileAttributes(destConfigDir, DIR_AND_EXECUTABLE_PERMS);
-        final PosixFileAttributeView destConfigDirAttributesView =
-            Files.getFileAttributeView(destConfigDir.getParent(), PosixFileAttributeView.class);
-        final PosixFileAttributes destConfigDirAttributes =
-            destConfigDirAttributesView != null ? destConfigDirAttributesView.readAttributes() : null;
-        if (destConfigDirAttributes != null) {
-            setOwnerGroup(destConfigDir, destConfigDirAttributes);
-        }
-
-        try (DirectoryStream<Path> stream = Files.newDirectoryStream(tmpConfigDir)) {
-            for (Path srcFile : stream) {
-                if (Files.isDirectory(srcFile)) {
-                    throw new UserException(ExitCodes.DATA_ERROR, "Directories not allowed in config dir for plugin " + info.getName());
-                }
-
-                Path destFile = destConfigDir.resolve(tmpConfigDir.relativize(srcFile));
-                if (Files.exists(destFile) == false) {
-                    Files.copy(srcFile, destFile);
-                    setFileAttributes(destFile, FILE_PERMS);
-                    if (destConfigDirAttributes != null) {
-                        setOwnerGroup(destFile, destConfigDirAttributes);
-                    }
-                }
-            }
-        }
-        IOUtils.rm(tmpConfigDir); // clean up what we just copied
-    }
-
-    private static void setOwnerGroup(final Path path, final PosixFileAttributes attributes) throws IOException {
-        Objects.requireNonNull(attributes);
-        PosixFileAttributeView fileAttributeView = Files.getFileAttributeView(path, PosixFileAttributeView.class);
-        assert fileAttributeView != null;
-        fileAttributeView.setOwner(attributes.owner());
-        fileAttributeView.setGroup(attributes.group());
-    }
-
-    /**
-     * Sets the attributes for a path iff posix attributes are supported
-     */
-    private static void setFileAttributes(final Path path, final Set<PosixFilePermission> permissions) throws IOException {
-        PosixFileAttributeView fileAttributeView = Files.getFileAttributeView(path, PosixFileAttributeView.class);
-        if (fileAttributeView != null) {
-            Files.setPosixFilePermissions(path, permissions);
-        }
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/plugins/ListPluginsCommand.java b/core/src/main/java/org/elasticsearch/plugins/ListPluginsCommand.java
deleted file mode 100644
index bd2f853bac022..0000000000000
--- a/core/src/main/java/org/elasticsearch/plugins/ListPluginsCommand.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.plugins;
-
-import joptsimple.OptionSet;
-import org.elasticsearch.cli.SettingCommand;
-import org.elasticsearch.cli.Terminal;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.node.internal.InternalSettingsPreparer;
-
-import java.io.IOException;
-import java.nio.file.DirectoryStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
-/**
- * A command for the plugin cli to list plugins installed in elasticsearch.
- */
-class ListPluginsCommand extends SettingCommand {
-
-    ListPluginsCommand() {
-        super("Lists installed elasticsearch plugins");
-    }
-
-    @Override
-    protected void execute(Terminal terminal, OptionSet options, Map<String, String> settings) throws Exception {
-        final Environment env = InternalSettingsPreparer.prepareEnvironment(Settings.EMPTY, terminal, settings);
-        if (Files.exists(env.pluginsFile()) == false) {
-            throw new IOException("Plugins directory missing: " + env.pluginsFile());
-        }
-
-        terminal.println(Terminal.Verbosity.VERBOSE, "Plugins directory: " + env.pluginsFile());
-        final List<Path> plugins = new ArrayList<>();
-        try (DirectoryStream<Path> paths = Files.newDirectoryStream(env.pluginsFile())) {
-            for (Path plugin : paths) {
-                plugins.add(plugin);
-            }
-        }
-        Collections.sort(plugins);
-        for (final Path plugin : plugins) {
-            terminal.println(plugin.getFileName().toString());
-            PluginInfo info = PluginInfo.readFromProperties(env.pluginsFile().resolve(plugin.toAbsolutePath()));
-            terminal.println(Terminal.Verbosity.VERBOSE, info.toString());
-        }
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/plugins/MetaDataUpgrader.java b/core/src/main/java/org/elasticsearch/plugins/MetaDataUpgrader.java
index aaeddbec11974..49dd91901d756 100644
--- a/core/src/main/java/org/elasticsearch/plugins/MetaDataUpgrader.java
+++ b/core/src/main/java/org/elasticsearch/plugins/MetaDataUpgrader.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.plugins;
 
+import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 
 import java.util.Collection;
@@ -32,7 +33,10 @@
 public class MetaDataUpgrader {
     public final UnaryOperator<Map<String, MetaData.Custom>> customMetaDataUpgraders;
 
-    public MetaDataUpgrader(Collection<UnaryOperator<Map<String, MetaData.Custom>>> customMetaDataUpgraders) {
+    public final UnaryOperator<Map<String, IndexTemplateMetaData>> indexTemplateMetaDataUpgraders;
+
+    public MetaDataUpgrader(Collection<UnaryOperator<Map<String, MetaData.Custom>>> customMetaDataUpgraders,
+                            Collection<UnaryOperator<Map<String, IndexTemplateMetaData>>> indexTemplateMetaDataUpgraders) {
         this.customMetaDataUpgraders = customs -> {
             Map<String, MetaData.Custom> upgradedCustoms = new HashMap<>(customs);
             for (UnaryOperator<Map<String, MetaData.Custom>> customMetaDataUpgrader : customMetaDataUpgraders) {
@@ -40,5 +44,13 @@ public MetaDataUpgrader(Collection<UnaryOperator<Map<String, MetaData.Custom>>>
             }
             return upgradedCustoms;
         };
+
+        this.indexTemplateMetaDataUpgraders = templates -> {
+            Map<String, IndexTemplateMetaData> upgradedTemplates = new HashMap<>(templates);
+            for (UnaryOperator<Map<String, IndexTemplateMetaData>> upgrader : indexTemplateMetaDataUpgraders) {
+                upgradedTemplates = upgrader.apply(upgradedTemplates);
+            }
+            return upgradedTemplates;
+        };
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/plugins/NetworkPlugin.java b/core/src/main/java/org/elasticsearch/plugins/NetworkPlugin.java
new file mode 100644
index 0000000000000..8f035e1621fd9
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/plugins/NetworkPlugin.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.plugins;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Supplier;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.network.NetworkService;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.http.HttpServerTransport;
+import org.elasticsearch.indices.breaker.CircuitBreakerService;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.Transport;
+import org.elasticsearch.transport.TransportInterceptor;
+
+/**
+ * Plugin for extending network and transport related classes
+ */
+public interface NetworkPlugin {
+
+    /**
+     * Returns a list of {@link TransportInterceptor} instances that are used to intercept incoming and outgoing
+     * transport (inter-node) requests. This must not return <code>null</code>
+     *
+     * @param threadContext a {@link ThreadContext} of the current nodes or clients {@link ThreadPool} that can be used to set additional
+     *                      headers in the interceptors
+     */
+    default List<TransportInterceptor> getTransportInterceptors(ThreadContext threadContext) {
+        return Collections.emptyList();
+    }
+
+    /**
+     * Returns a map of {@link Transport} suppliers.
+     * See {@link org.elasticsearch.common.network.NetworkModule#TRANSPORT_TYPE_KEY} to configure a specific implementation.
+     */
+    default Map<String, Supplier<Transport>> getTransports(Settings settings, ThreadPool threadPool, BigArrays bigArrays,
+                                                           CircuitBreakerService circuitBreakerService,
+                                                           NamedWriteableRegistry namedWriteableRegistry,
+                                                           NetworkService networkService) {
+        return Collections.emptyMap();
+    }
+
+    /**
+     * Returns a map of {@link HttpServerTransport} suppliers.
+     * See {@link org.elasticsearch.common.network.NetworkModule#HTTP_TYPE_SETTING} to configure a specific implementation.
+     */
+    default Map<String, Supplier<HttpServerTransport>> getHttpTransports(Settings settings, ThreadPool threadPool, BigArrays bigArrays,
+                                                                         CircuitBreakerService circuitBreakerService,
+                                                                         NamedWriteableRegistry namedWriteableRegistry,
+                                                                         NamedXContentRegistry xContentRegistry,
+                                                                         NetworkService networkService,
+                                                                         HttpServerTransport.Dispatcher dispatcher) {
+        return Collections.emptyMap();
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/plugins/Platforms.java b/core/src/main/java/org/elasticsearch/plugins/Platforms.java
new file mode 100644
index 0000000000000..91af58ebec46e
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/plugins/Platforms.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.plugins;
+
+import org.apache.lucene.util.Constants;
+
+import java.nio.file.Path;
+import java.util.Locale;
+
+/**
+ * Encapsulates platform-dependent methods for handling native components of plugins.
+ */
+public class Platforms {
+
+    private static final String PROGRAM_NAME = Constants.WINDOWS ? "controller.exe" : "controller";
+    public static final String PLATFORM_NAME = Platforms.platformName(Constants.OS_NAME, Constants.OS_ARCH);
+
+    private Platforms() {}
+
+    /**
+     * The path to the native controller for a plugin with native components.
+     */
+    public static Path nativeControllerPath(Path plugin) {
+        return plugin
+                .resolve("platform")
+                .resolve(PLATFORM_NAME)
+                .resolve("bin")
+                .resolve(PROGRAM_NAME);
+    }
+
+    /**
+     * Return the platform name based on the OS name and
+     * - darwin-x86_64
+     * - linux-x86-64
+     * - windows-x86_64
+     * For *nix platforms this is more-or-less `uname -s`-`uname -m` converted to lower case.
+     * However, for consistency between different operating systems on the same architecture
+     * "amd64" is replaced with "x86_64" and "i386" with "x86".
+     * For Windows it's "windows-" followed by either "x86" or "x86_64".
+     */
+    public static String platformName(final String osName, final String osArch) {
+        final String lowerCaseOs = osName.toLowerCase(Locale.ROOT);
+        final String normalizedOs;
+        if (lowerCaseOs.startsWith("windows")) {
+            normalizedOs = "windows";
+        } else if (lowerCaseOs.equals("mac os x")) {
+            normalizedOs = "darwin";
+        } else {
+            normalizedOs = lowerCaseOs;
+        }
+
+        final String lowerCaseArch = osArch.toLowerCase(Locale.ROOT);
+        final String normalizedArch;
+        if (lowerCaseArch.equals("amd64")) {
+            normalizedArch = "x86_64";
+        } else if (lowerCaseArch.equals("i386")) {
+            normalizedArch = "x86";
+        } else {
+            normalizedArch = lowerCaseArch;
+        }
+
+        return normalizedOs + "-" + normalizedArch;
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/plugins/Plugin.java b/core/src/main/java/org/elasticsearch/plugins/Plugin.java
index 1c79986e18f27..bf5bc49d279c0 100644
--- a/core/src/main/java/org/elasticsearch/plugins/Plugin.java
+++ b/core/src/main/java/org/elasticsearch/plugins/Plugin.java
@@ -19,47 +19,63 @@
 
 package org.elasticsearch.plugins;
 
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-
 import org.elasticsearch.action.ActionModule;
+import org.elasticsearch.bootstrap.BootstrapCheck;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.ClusterModule;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.component.LifecycleComponent;
 import org.elasticsearch.common.inject.Module;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.discovery.DiscoveryModule;
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.indices.analysis.AnalysisModule;
+import org.elasticsearch.repositories.RepositoriesModule;
 import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.SearchModule;
-import org.elasticsearch.search.SearchRequestParsers;
 import org.elasticsearch.threadpool.ExecutorBuilder;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.watcher.ResourceWatcherService;
 
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 import java.util.function.UnaryOperator;
 
 /**
- * An extension point allowing to plug in custom functionality.
- * <p>
- * Implement any of these interfaces to extend Elasticsearch:
+ * An extension point allowing to plug in custom functionality. This class has a number of extension points that are available to all
+ * plugins, in addition you can implement any of the following interfaces to further customize Elasticsearch:
  * <ul>
  * <li>{@link ActionPlugin}
  * <li>{@link AnalysisPlugin}
+ * <li>{@link ClusterPlugin}
+ * <li>{@link DiscoveryPlugin}
+ * <li>{@link IngestPlugin}
  * <li>{@link MapperPlugin}
+ * <li>{@link NetworkPlugin}
+ * <li>{@link RepositoryPlugin}
  * <li>{@link ScriptPlugin}
  * <li>{@link SearchPlugin}
  * </ul>
+ * <p>In addition to extension points this class also declares some {@code @Deprecated} {@code public final void onModule} methods. These
+ * methods should cause any extensions of {@linkplain Plugin} that used the pre-5.x style extension syntax to fail to build and point the
+ * plugin author at the new extension syntax. We hope that these make the process of upgrading a plugin from 2.x to 5.x only mildly painful.
  */
-public abstract class Plugin {
+public abstract class Plugin implements Closeable {
 
     /**
      * Node level guice modules.
@@ -88,11 +104,10 @@ public Collection<Class<? extends LifecycleComponent>> getGuiceServiceClasses()
      * @param threadPool A service to allow retrieving an executor to run an async action
      * @param resourceWatcherService A service to watch for changes to node local files
      * @param scriptService A service to allow running scripts on the local node
-     * @param searchRequestParsers Parsers for search requests which may be used to templatize search requests
      */
     public Collection<Object> createComponents(Client client, ClusterService clusterService, ThreadPool threadPool,
                                                ResourceWatcherService resourceWatcherService, ScriptService scriptService,
-                                               SearchRequestParsers searchRequestParsers) {
+                                               NamedXContentRegistry xContentRegistry) {
         return Collections.emptyList();
     }
 
@@ -112,6 +127,14 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
         return Collections.emptyList();
     }
 
+    /**
+     * Returns parsers for named objects this plugin will parse from {@link XContentParser#namedObject(Class, String, Object)}.
+     * @see NamedWriteableRegistry
+     */
+    public List<NamedXContentRegistry.Entry> getNamedXContent() {
+        return Collections.emptyList();
+    }
+
     /**
      * Called before a new index is created on a node. The given module can be used to register index-level
      * extensions.
@@ -132,16 +155,81 @@ public void onIndexModule(IndexModule indexModule) {}
      * Provides a function to modify global custom meta data on startup.
      * <p>
      * Plugins should return the input custom map via {@link UnaryOperator#identity()} if no upgrade is required.
+     * <p>
+     * The order of custom meta data upgraders calls is undefined and can change between runs so, it is expected that
+     * plugins will modify only data owned by them to avoid conflicts.
+     * <p>
      * @return Never {@code null}. The same or upgraded {@code MetaData.Custom} map.
      * @throws IllegalStateException if the node should not start because at least one {@code MetaData.Custom}
-     *         is unsupported
+     *                               is unsupported
      */
     public UnaryOperator<Map<String, MetaData.Custom>> getCustomMetaDataUpgrader() {
         return UnaryOperator.identity();
     }
 
     /**
-     * Old-style guice index level extension point.
+     * Provides a function to modify index template meta data on startup.
+     * <p>
+     * Plugins should return the input template map via {@link UnaryOperator#identity()} if no upgrade is required.
+     * <p>
+     * The order of the template upgrader calls is undefined and can change between runs so, it is expected that
+     * plugins will modify only templates owned by them to avoid conflicts.
+     * <p>
+     * @return Never {@code null}. The same or upgraded {@code IndexTemplateMetaData} map.
+     * @throws IllegalStateException if the node should not start because at least one {@code IndexTemplateMetaData}
+     *                               cannot be upgraded
+     */
+    public UnaryOperator<Map<String, IndexTemplateMetaData>> getIndexTemplateMetaDataUpgrader() {
+        return UnaryOperator.identity();
+    }
+
+    /**
+     * Provides a function to modify index meta data when an index is introduced into the cluster state for the first time.
+     * <p>
+     * Plugins should return the input index metadata via {@link UnaryOperator#identity()} if no upgrade is required.
+     * <p>
+     * The order of the index upgrader calls for the same index is undefined and can change between runs so, it is expected that
+     * plugins will modify only indices owned by them to avoid conflicts.
+     * <p>
+     * @return Never {@code null}. The same or upgraded {@code IndexMetaData}.
+     * @throws IllegalStateException if the node should not start because the index is unsupported
+     */
+    public UnaryOperator<IndexMetaData> getIndexMetaDataUpgrader() {
+        return UnaryOperator.identity();
+    }
+
+    /**
+     * Provides the list of this plugin's custom thread pools, empty if
+     * none.
+     *
+     * @param settings the current settings
+     * @return executors builders for this plugin's custom thread pools
+     */
+    public List<ExecutorBuilder<?>> getExecutorBuilders(Settings settings) {
+        return Collections.emptyList();
+    }
+
+    /**
+     * Returns a list of checks that are enforced when a node starts up once a node has the transport protocol bound to a non-loopback
+     * interface. In this case we assume the node is running in production and all bootstrap checks must pass. This allows plugins
+     * to provide a better out of the box experience by pre-configuring otherwise (in production) mandatory settings or to enforce certain
+     * configurations like OS settings or 3rd party resources.
+     */
+    public List<BootstrapCheck> getBootstrapChecks() { return Collections.emptyList(); }
+
+    /**
+     * Close the resources opened by this plugin.
+     *
+     * @throws IOException if the plugin failed to close its resources
+     */
+    @Override
+    public void close() throws IOException {
+
+    }
+
+    /**
+     * Old-style guice index level extension point. {@code @Deprecated} and {@code final} to act as a signpost for plugin authors upgrading
+     * from 2.x.
      *
      * @deprecated use #onIndexModule instead
      */
@@ -150,7 +238,8 @@ public final void onModule(IndexModule indexModule) {}
 
 
     /**
-     * Old-style guice settings extension point.
+     * Old-style guice settings extension point. {@code @Deprecated} and {@code final} to act as a signpost for plugin authors upgrading
+     * from 2.x.
      *
      * @deprecated use #getSettings and #getSettingsFilter instead
      */
@@ -158,7 +247,8 @@ public final void onModule(IndexModule indexModule) {}
     public final void onModule(SettingsModule settingsModule) {}
 
     /**
-     * Old-style guice scripting extension point.
+     * Old-style guice scripting extension point. {@code @Deprecated} and {@code final} to act as a signpost for plugin authors upgrading
+     * from 2.x.
      *
      * @deprecated implement {@link ScriptPlugin} instead
      */
@@ -166,7 +256,8 @@ public final void onModule(SettingsModule settingsModule) {}
     public final void onModule(ScriptModule module) {}
 
     /**
-     * Old-style analysis extension point.
+     * Old-style analysis extension point. {@code @Deprecated} and {@code final} to act as a signpost for plugin authors upgrading
+     * from 2.x.
      *
      * @deprecated implement {@link AnalysisPlugin} instead
      */
@@ -174,7 +265,8 @@ public final void onModule(ScriptModule module) {}
     public final void onModule(AnalysisModule module) {}
 
     /**
-     * Old-style action extension point.
+     * Old-style action extension point. {@code @Deprecated} and {@code final} to act as a signpost for plugin authors upgrading
+     * from 2.x.
      *
      * @deprecated implement {@link ActionPlugin} instead
      */
@@ -182,7 +274,8 @@ public final void onModule(AnalysisModule module) {}
     public final void onModule(ActionModule module) {}
 
     /**
-     * Old-style action extension point.
+     * Old-style search extension point. {@code @Deprecated} and {@code final} to act as a signpost for plugin authors upgrading
+     * from 2.x.
      *
      * @deprecated implement {@link SearchPlugin} instead
      */
@@ -190,13 +283,38 @@ public final void onModule(ActionModule module) {}
     public final void onModule(SearchModule module) {}
 
     /**
-     * Provides the list of this plugin's custom thread pools, empty if
-     * none.
+     * Old-style network extension point. {@code @Deprecated} and {@code final} to act as a signpost for plugin authors upgrading
+     * from 2.x.
      *
-     * @param settings the current settings
-     * @return executors builders for this plugin's custom thread pools
+     * @deprecated implement {@link NetworkPlugin} instead
      */
-    public List<ExecutorBuilder<?>> getExecutorBuilders(Settings settings) {
-        return Collections.emptyList();
-    }
+    @Deprecated
+    public final void onModule(NetworkModule module) {}
+
+    /**
+     * Old-style snapshot/restore extension point. {@code @Deprecated} and {@code final} to act as a signpost for plugin authors upgrading
+     * from 2.x.
+     *
+     * @deprecated implement {@link RepositoryPlugin} instead
+     */
+    @Deprecated
+    public final void onModule(RepositoriesModule module) {}
+
+    /**
+     * Old-style cluster extension point. {@code @Deprecated} and {@code final} to act as a signpost for plugin authors upgrading
+     * from 2.x.
+     *
+     * @deprecated implement {@link ClusterPlugin} instead
+     */
+    @Deprecated
+    public final void onModule(ClusterModule module) {}
+
+    /**
+     * Old-style discovery extension point. {@code @Deprecated} and {@code final} to act as a signpost for plugin authors upgrading
+     * from 2.x.
+     *
+     * @deprecated implement {@link DiscoveryPlugin} instead
+     */
+    @Deprecated
+    public final void onModule(DiscoveryModule module) {}
 }
diff --git a/core/src/main/java/org/elasticsearch/plugins/PluginCli.java b/core/src/main/java/org/elasticsearch/plugins/PluginCli.java
deleted file mode 100644
index ba401404f2e54..0000000000000
--- a/core/src/main/java/org/elasticsearch/plugins/PluginCli.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.plugins;
-
-import org.elasticsearch.cli.MultiCommand;
-import org.elasticsearch.cli.Terminal;
-import org.elasticsearch.common.logging.LogConfigurator;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.node.internal.InternalSettingsPreparer;
-
-/**
- * A cli tool for adding, removing and listing plugins for elasticsearch.
- */
-public class PluginCli extends MultiCommand {
-
-    private PluginCli() {
-        super("A tool for managing installed elasticsearch plugins");
-        subcommands.put("list", new ListPluginsCommand());
-        subcommands.put("install", new InstallPluginCommand());
-        subcommands.put("remove", new RemovePluginCommand());
-    }
-
-    public static void main(String[] args) throws Exception {
-        // initialize default for es.logger.level because we will not read the log4j2.properties
-        String loggerLevel = System.getProperty("es.logger.level", "INFO");
-        String pathHome = System.getProperty("es.path.home");
-        // Set the appender for all potential log files to terminal so that other components that use the logger print out the
-        // same terminal.
-        // The reason for this is that the plugin cli cannot be configured with a file appender because when the plugin command is
-        // executed there is no way of knowing where the logfiles should be placed. For example, if elasticsearch
-        // is run as service then the logs should be at /var/log/elasticsearch but when started from the tar they should be at es.home/logs.
-        // Therefore we print to Terminal.
-        Environment loggingEnvironment = InternalSettingsPreparer.prepareEnvironment(Settings.builder()
-                .put("path.home", pathHome)
-                .put("logger.level", loggerLevel)
-                .build(), Terminal.DEFAULT);
-        LogConfigurator.configure(loggingEnvironment, false);
-
-        exit(new PluginCli().main(args, Terminal.DEFAULT));
-    }
-
-}
diff --git a/core/src/main/java/org/elasticsearch/plugins/PluginInfo.java b/core/src/main/java/org/elasticsearch/plugins/PluginInfo.java
index 3e241eadd37ba..666cc22b92655 100644
--- a/core/src/main/java/org/elasticsearch/plugins/PluginInfo.java
+++ b/core/src/main/java/org/elasticsearch/plugins/PluginInfo.java
@@ -16,6 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+
 package org.elasticsearch.plugins;
 
 import org.elasticsearch.Version;
@@ -30,133 +31,215 @@
 import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Locale;
 import java.util.Properties;
 
+/**
+ * An in-memory representation of the plugin descriptor.
+ */
 public class PluginInfo implements Writeable, ToXContent {
 
     public static final String ES_PLUGIN_PROPERTIES = "plugin-descriptor.properties";
     public static final String ES_PLUGIN_POLICY = "plugin-security.policy";
 
-    static final class Fields {
-        static final String NAME = "name";
-        static final String DESCRIPTION = "description";
-        static final String URL = "url";
-        static final String VERSION = "version";
-        static final String CLASSNAME = "classname";
-    }
-
     private final String name;
     private final String description;
     private final String version;
     private final String classname;
+    private final boolean hasNativeController;
 
     /**
-     * Information about plugins
+     * Construct plugin info.
      *
-     * @param name        Its name
-     * @param description Its description
-     * @param version     Version number
+     * @param name                the name of the plugin
+     * @param description         a description of the plugin
+     * @param version             the version of Elasticsearch the plugin is built for
+     * @param classname           the entry point to the plugin
+     * @param hasNativeController whether or not the plugin has a native controller
      */
-    public PluginInfo(String name, String description, String version, String classname) {
+    public PluginInfo(
+            final String name,
+            final String description,
+            final String version,
+            final String classname,
+            final boolean hasNativeController) {
         this.name = name;
         this.description = description;
         this.version = version;
         this.classname = classname;
+        this.hasNativeController = hasNativeController;
     }
 
-    public PluginInfo(StreamInput in) throws IOException {
+    /**
+     * Construct plugin info from a stream.
+     *
+     * @param in the stream
+     * @throws IOException if an I/O exception occurred reading the plugin info from the stream
+     */
+    public PluginInfo(final StreamInput in) throws IOException {
         this.name = in.readString();
         this.description = in.readString();
         this.version = in.readString();
         this.classname = in.readString();
+        if (in.getVersion().onOrAfter(Version.V_5_4_0)) {
+            hasNativeController = in.readBoolean();
+        } else {
+            hasNativeController = false;
+        }
     }
 
     @Override
-    public void writeTo(StreamOutput out) throws IOException {
+    public void writeTo(final StreamOutput out) throws IOException {
         out.writeString(name);
         out.writeString(description);
         out.writeString(version);
         out.writeString(classname);
+        if (out.getVersion().onOrAfter(Version.V_5_4_0)) {
+            out.writeBoolean(hasNativeController);
+        }
     }
 
     /** reads (and validates) plugin metadata descriptor file */
-    public static PluginInfo readFromProperties(Path dir) throws IOException {
-        Path descriptor = dir.resolve(ES_PLUGIN_PROPERTIES);
-        Properties props = new Properties();
+
+    /**
+     * Reads and validates the plugin descriptor file.
+     *
+     * @param path the path to the root directory for the plugin
+     * @return the plugin info
+     * @throws IOException if an I/O exception occurred reading the plugin descriptor
+     */
+    public static PluginInfo readFromProperties(final Path path) throws IOException {
+        final Path descriptor = path.resolve(ES_PLUGIN_PROPERTIES);
+        final Properties props = new Properties();
         try (InputStream stream = Files.newInputStream(descriptor)) {
             props.load(stream);
         }
-        String name = props.getProperty("name");
+        final String name = props.getProperty("name");
         if (name == null || name.isEmpty()) {
-            throw new IllegalArgumentException("Property [name] is missing in [" + descriptor + "]");
+            throw new IllegalArgumentException(
+                    "property [name] is missing in [" + descriptor + "]");
         }
-        String description = props.getProperty("description");
+        final String description = props.getProperty("description");
         if (description == null) {
-            throw new IllegalArgumentException("Property [description] is missing for plugin [" + name + "]");
+            throw new IllegalArgumentException(
+                    "property [description] is missing for plugin [" + name + "]");
         }
-        String version = props.getProperty("version");
+        final String version = props.getProperty("version");
         if (version == null) {
-            throw new IllegalArgumentException("Property [version] is missing for plugin [" + name + "]");
+            throw new IllegalArgumentException(
+                    "property [version] is missing for plugin [" + name + "]");
         }
 
-        String esVersionString = props.getProperty("elasticsearch.version");
+        final String esVersionString = props.getProperty("elasticsearch.version");
         if (esVersionString == null) {
-            throw new IllegalArgumentException("Property [elasticsearch.version] is missing for plugin [" + name + "]");
+            throw new IllegalArgumentException(
+                    "property [elasticsearch.version] is missing for plugin [" + name + "]");
         }
-        Version esVersion = Version.fromString(esVersionString);
+        final Version esVersion = Version.fromString(esVersionString);
         if (esVersion.equals(Version.CURRENT) == false) {
-            throw new IllegalArgumentException("Plugin [" + name + "] is incompatible with Elasticsearch [" + Version.CURRENT.toString() +
-                    "]. Was designed for version [" + esVersionString + "]");
+            final String message = String.format(
+                    Locale.ROOT,
+                    "plugin [%s] is incompatible with version [%s]; was designed for version [%s]",
+                    name,
+                    Version.CURRENT.toString(),
+                    esVersionString);
+            throw new IllegalArgumentException(message);
         }
-        String javaVersionString = props.getProperty("java.version");
+        final String javaVersionString = props.getProperty("java.version");
         if (javaVersionString == null) {
-            throw new IllegalArgumentException("Property [java.version] is missing for plugin [" + name + "]");
+            throw new IllegalArgumentException(
+                    "property [java.version] is missing for plugin [" + name + "]");
         }
         JarHell.checkVersionFormat(javaVersionString);
         JarHell.checkJavaVersion(name, javaVersionString);
-        String classname = props.getProperty("classname");
+        final String classname = props.getProperty("classname");
         if (classname == null) {
-            throw new IllegalArgumentException("Property [classname] is missing for plugin [" + name + "]");
+            throw new IllegalArgumentException(
+                    "property [classname] is missing for plugin [" + name + "]");
         }
 
-        return new PluginInfo(name, description, version, classname);
+        final String hasNativeControllerValue = props.getProperty("has.native.controller");
+        final boolean hasNativeController;
+        if (hasNativeControllerValue == null) {
+            hasNativeController = false;
+        } else {
+            switch (hasNativeControllerValue) {
+                case "true":
+                    hasNativeController = true;
+                    break;
+                case "false":
+                    hasNativeController = false;
+                    break;
+                default:
+                    final String message = String.format(
+                            Locale.ROOT,
+                            "property [%s] must be [%s], [%s], or unspecified but was [%s]",
+                            "has_native_controller",
+                            "true",
+                            "false",
+                            hasNativeControllerValue);
+                    throw new IllegalArgumentException(message);
+            }
+        }
+
+        return new PluginInfo(name, description, version, classname, hasNativeController);
     }
 
     /**
-     * @return Plugin's name
+     * The name of the plugin.
+     *
+     * @return the plugin name
      */
     public String getName() {
         return name;
     }
 
     /**
-     * @return Plugin's description if any
+     * The description of the plugin.
+     *
+     * @return the plugin description
      */
     public String getDescription() {
         return description;
     }
 
     /**
-     * @return plugin's classname
+     * The entry point to the plugin.
+     *
+     * @return the entry point to the plugin
      */
     public String getClassname() {
         return classname;
     }
 
     /**
-     * @return Version number for the plugin
+     * The version of Elasticsearch the plugin was built for.
+     *
+     * @return the version
      */
     public String getVersion() {
         return version;
     }
 
+    /**
+     * Whether or not the plugin has a native controller.
+     *
+     * @return {@code true} if the plugin has a native controller
+     */
+    public boolean hasNativeController() {
+        return hasNativeController;
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
-        builder.field(Fields.NAME, name);
-        builder.field(Fields.VERSION, version);
-        builder.field(Fields.DESCRIPTION, description);
-        builder.field(Fields.CLASSNAME, classname);
+        {
+            builder.field("name", name);
+            builder.field("version", version);
+            builder.field("description", description);
+            builder.field("classname", classname);
+            builder.field("has_native_controller", hasNativeController);
+        }
         builder.endObject();
 
         return builder;
@@ -187,8 +270,9 @@ public String toString() {
                 .append("Name: ").append(name).append("\n")
                 .append("Description: ").append(description).append("\n")
                 .append("Version: ").append(version).append("\n")
+                .append("Native Controller: ").append(hasNativeController).append("\n")
                 .append(" * Classname: ").append(classname);
-
         return information.toString();
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/plugins/PluginSecurity.java b/core/src/main/java/org/elasticsearch/plugins/PluginSecurity.java
index f9c3d1826c992..d09acbe1f3c23 100644
--- a/core/src/main/java/org/elasticsearch/plugins/PluginSecurity.java
+++ b/core/src/main/java/org/elasticsearch/plugins/PluginSecurity.java
@@ -22,7 +22,6 @@
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.cli.Terminal.Verbosity;
-import org.elasticsearch.env.Environment;
 
 import java.io.IOException;
 import java.nio.file.Files;
@@ -37,60 +36,74 @@
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
+import java.util.function.Supplier;
 
 class PluginSecurity {
 
     /**
      * Reads plugin policy, prints/confirms exceptions
      */
-    static void readPolicy(Path file, Terminal terminal, Environment environment, boolean batch) throws IOException {
-        PermissionCollection permissions = parsePermissions(terminal, file, environment.tmpFile());
+    static void readPolicy(PluginInfo info, Path file, Terminal terminal, Supplier<Path> tmpFile, boolean batch) throws IOException {
+        PermissionCollection permissions = parsePermissions(terminal, file, tmpFile.get());
         List<Permission> requested = Collections.list(permissions.elements());
         if (requested.isEmpty()) {
             terminal.println(Verbosity.VERBOSE, "plugin has a policy file with no additional permissions");
-            return;
-        }
+        } else {
 
-        // sort permissions in a reasonable order
-        Collections.sort(requested, new Comparator<Permission>() {
-            @Override
-            public int compare(Permission o1, Permission o2) {
-                int cmp = o1.getClass().getName().compareTo(o2.getClass().getName());
-                if (cmp == 0) {
-                    String name1 = o1.getName();
-                    String name2 = o2.getName();
-                    if (name1 == null) {
-                        name1 = "";
-                    }
-                    if (name2 == null) {
-                        name2 = "";
-                    }
-                    cmp = name1.compareTo(name2);
+            // sort permissions in a reasonable order
+            Collections.sort(requested, new Comparator<Permission>() {
+                @Override
+                public int compare(Permission o1, Permission o2) {
+                    int cmp = o1.getClass().getName().compareTo(o2.getClass().getName());
                     if (cmp == 0) {
-                        String actions1 = o1.getActions();
-                        String actions2 = o2.getActions();
-                        if (actions1 == null) {
-                            actions1 = "";
+                        String name1 = o1.getName();
+                        String name2 = o2.getName();
+                        if (name1 == null) {
+                            name1 = "";
+                        }
+                        if (name2 == null) {
+                            name2 = "";
                         }
-                        if (actions2 == null) {
-                            actions2 = "";
+                        cmp = name1.compareTo(name2);
+                        if (cmp == 0) {
+                            String actions1 = o1.getActions();
+                            String actions2 = o2.getActions();
+                            if (actions1 == null) {
+                                actions1 = "";
+                            }
+                            if (actions2 == null) {
+                                actions2 = "";
+                            }
+                            cmp = actions1.compareTo(actions2);
                         }
-                        cmp = actions1.compareTo(actions2);
                     }
+                    return cmp;
                 }
-                return cmp;
+            });
+
+            terminal.println(Verbosity.NORMAL, "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@");
+            terminal.println(Verbosity.NORMAL, "@     WARNING: plugin requires additional permissions     @");
+            terminal.println(Verbosity.NORMAL, "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@");
+            // print all permissions:
+            for (Permission permission : requested) {
+                terminal.println(Verbosity.NORMAL, "* " + formatPermission(permission));
             }
-        });
-
-        terminal.println(Verbosity.NORMAL, "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@");
-        terminal.println(Verbosity.NORMAL, "@     WARNING: plugin requires additional permissions     @");
-        terminal.println(Verbosity.NORMAL, "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@");
-        // print all permissions:
-        for (Permission permission : requested) {
-            terminal.println(Verbosity.NORMAL, "* " + formatPermission(permission));
+            terminal.println(Verbosity.NORMAL, "See http://docs.oracle.com/javase/8/docs/technotes/guides/security/permissions.html");
+            terminal.println(Verbosity.NORMAL, "for descriptions of what these permissions allow and the associated risks.");
+            prompt(terminal, batch);
+        }
+
+        if (info.hasNativeController()) {
+            terminal.println(Verbosity.NORMAL, "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@");
+            terminal.println(Verbosity.NORMAL, "@        WARNING: plugin forks a native controller        @");
+            terminal.println(Verbosity.NORMAL, "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@");
+            terminal.println(Verbosity.NORMAL, "This plugin launches a native controller that is not subject to the Java");
+            terminal.println(Verbosity.NORMAL, "security manager nor to system call filters.");
+            prompt(terminal, batch);
         }
-        terminal.println(Verbosity.NORMAL, "See http://docs.oracle.com/javase/8/docs/technotes/guides/security/permissions.html");
-        terminal.println(Verbosity.NORMAL, "for descriptions of what these permissions allow and the associated risks.");
+    }
+
+    private static void prompt(final Terminal terminal, final boolean batch) {
         if (!batch) {
             terminal.println(Verbosity.NORMAL, "");
             String text = terminal.readText("Continue with installation? [y/N]");
diff --git a/core/src/main/java/org/elasticsearch/plugins/PluginsService.java b/core/src/main/java/org/elasticsearch/plugins/PluginsService.java
index 01704e9ed86e5..27aa9893fb5ad 100644
--- a/core/src/main/java/org/elasticsearch/plugins/PluginsService.java
+++ b/core/src/main/java/org/elasticsearch/plugins/PluginsService.java
@@ -20,8 +20,6 @@
 package org.elasticsearch.plugins;
 
 import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.apache.logging.log4j.util.Supplier;
 import org.apache.lucene.analysis.util.CharFilterFactory;
 import org.apache.lucene.analysis.util.TokenFilterFactory;
 import org.apache.lucene.analysis.util.TokenizerFactory;
@@ -36,7 +34,6 @@
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.component.LifecycleComponent;
 import org.elasticsearch.common.inject.Module;
-import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
@@ -45,24 +42,26 @@
 import org.elasticsearch.threadpool.ExecutorBuilder;
 
 import java.io.IOException;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
 import java.net.URL;
 import java.net.URLClassLoader;
 import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import static org.elasticsearch.common.io.FileSystemUtils.isAccessibleDirectory;
 
@@ -79,8 +78,6 @@ public class PluginsService extends AbstractComponent {
     public static final Setting<List<String>> MANDATORY_SETTING =
         Setting.listSetting("plugin.mandatory", Collections.emptyList(), Function.identity(), Property.NodeScope);
 
-    private final Map<Plugin, List<OnModuleReference>> onModuleReferences;
-
     public List<Setting<?>> getPluginSettings() {
         return plugins.stream().flatMap(p -> p.v2().getSettings().stream()).collect(Collectors.toList());
     }
@@ -89,16 +86,6 @@ public List<String> getPluginSettingsFilter() {
         return plugins.stream().flatMap(p -> p.v2().getSettingsFilter().stream()).collect(Collectors.toList());
     }
 
-    static class OnModuleReference {
-        public final Class<? extends Module> moduleClass;
-        public final Method onModuleMethod;
-
-        OnModuleReference(Class<? extends Module> moduleClass, Method onModuleMethod) {
-            this.moduleClass = moduleClass;
-            this.onModuleMethod = onModuleMethod;
-        }
-    }
-
     /**
      * Constructs a new PluginService
      * @param settings The settings of the system
@@ -114,7 +101,7 @@ public PluginsService(Settings settings, Path modulesDirectory, Path pluginsDire
         // first we load plugins that are on the classpath. this is for tests and transport clients
         for (Class<? extends Plugin> pluginClass : classpathPlugins) {
             Plugin plugin = loadPlugin(pluginClass, settings);
-            PluginInfo pluginInfo = new PluginInfo(pluginClass.getName(), "classpath plugin", "NA", pluginClass.getName());
+            PluginInfo pluginInfo = new PluginInfo(pluginClass.getName(), "classpath plugin", "NA", pluginClass.getName(), false);
             if (logger.isTraceEnabled()) {
                 logger.trace("plugin loaded from classpath [{}]", pluginInfo);
             }
@@ -122,16 +109,16 @@ public PluginsService(Settings settings, Path modulesDirectory, Path pluginsDire
             pluginsList.add(pluginInfo);
         }
 
+        Set<Bundle> seenBundles = new LinkedHashSet<>();
         List<PluginInfo> modulesList = new ArrayList<>();
         // load modules
         if (modulesDirectory != null) {
             try {
-                List<Bundle> bundles = getModuleBundles(modulesDirectory);
-                List<Tuple<PluginInfo, Plugin>> loaded = loadBundles(bundles);
-                pluginsLoaded.addAll(loaded);
-                for (Tuple<PluginInfo, Plugin> module : loaded) {
-                    modulesList.add(module.v1());
+                Set<Bundle> modules = getModuleBundles(modulesDirectory);
+                for (Bundle bundle : modules) {
+                    modulesList.add(bundle.plugin);
                 }
+                seenBundles.addAll(modules);
             } catch (IOException ex) {
                 throw new IllegalStateException("Unable to initialize modules", ex);
             }
@@ -140,17 +127,19 @@ public PluginsService(Settings settings, Path modulesDirectory, Path pluginsDire
         // now, find all the ones that are in plugins/
         if (pluginsDirectory != null) {
             try {
-                List<Bundle> bundles = getPluginBundles(pluginsDirectory);
-                List<Tuple<PluginInfo, Plugin>> loaded = loadBundles(bundles);
-                pluginsLoaded.addAll(loaded);
-                for (Tuple<PluginInfo, Plugin> plugin : loaded) {
-                    pluginsList.add(plugin.v1());
+                Set<Bundle> plugins = getPluginBundles(pluginsDirectory);
+                for (Bundle bundle : plugins) {
+                    pluginsList.add(bundle.plugin);
                 }
+                seenBundles.addAll(plugins);
             } catch (IOException ex) {
                 throw new IllegalStateException("Unable to initialize plugins", ex);
             }
         }
 
+        List<Tuple<PluginInfo, Plugin>> loaded = loadBundles(seenBundles);
+        pluginsLoaded.addAll(loaded);
+
         this.info = new PluginsAndModules(pluginsList, modulesList);
         this.plugins = Collections.unmodifiableList(pluginsLoaded);
 
@@ -178,40 +167,6 @@ public PluginsService(Settings settings, Path modulesDirectory, Path pluginsDire
         // but for now: just be transparent so we can debug any potential issues
         logPluginInfo(info.getModuleInfos(), "module", logger);
         logPluginInfo(info.getPluginInfos(), "plugin", logger);
-
-        Map<Plugin, List<OnModuleReference>> onModuleReferences = new HashMap<>();
-        for (Tuple<PluginInfo, Plugin> pluginEntry : this.plugins) {
-            Plugin plugin = pluginEntry.v2();
-            List<OnModuleReference> list = new ArrayList<>();
-            for (Method method : plugin.getClass().getMethods()) {
-                if (!method.getName().equals("onModule")) {
-                    continue;
-                }
-                // this is a deprecated final method, so all Plugin subclasses have it
-                if (method.getParameterTypes().length == 1 && method.getParameterTypes()[0].equals(IndexModule.class)) {
-                    continue;
-                }
-                if (method.getParameterTypes().length == 0 || method.getParameterTypes().length > 1) {
-                    logger.warn("Plugin: {} implementing onModule with no parameters or more than one parameter", pluginEntry.v1().getName());
-                    continue;
-                }
-                Class moduleClass = method.getParameterTypes()[0];
-                if (!Module.class.isAssignableFrom(moduleClass)) {
-                    if (method.getDeclaringClass() == Plugin.class) {
-                        // These are still part of the Plugin class to point the user to the new implementations
-                        continue;
-                    }
-                    throw new RuntimeException(
-                            "Plugin: [" + pluginEntry.v1().getName() + "] implements onModule taking a parameter that isn't a Module ["
-                                    + moduleClass.getSimpleName() + "]");
-                }
-                list.add(new OnModuleReference(moduleClass, method));
-            }
-            if (!list.isEmpty()) {
-                onModuleReferences.put(plugin, list);
-            }
-        }
-        this.onModuleReferences = Collections.unmodifiableMap(onModuleReferences);
     }
 
     private static void logPluginInfo(final List<PluginInfo> pluginInfos, final String type, final Logger logger) {
@@ -225,38 +180,6 @@ private static void logPluginInfo(final List<PluginInfo> pluginInfos, final Stri
         }
     }
 
-    private List<Tuple<PluginInfo, Plugin>> plugins() {
-        return plugins;
-    }
-
-    public void processModules(Iterable<Module> modules) {
-        for (Module module : modules) {
-            processModule(module);
-        }
-    }
-
-    public void processModule(Module module) {
-        for (Tuple<PluginInfo, Plugin> plugin : plugins()) {
-            // see if there are onModule references
-            List<OnModuleReference> references = onModuleReferences.get(plugin.v2());
-            if (references != null) {
-                for (OnModuleReference reference : references) {
-                    if (reference.moduleClass.isAssignableFrom(module.getClass())) {
-                        try {
-                            reference.onModuleMethod.invoke(plugin.v2(), module);
-                        } catch (IllegalAccessException | InvocationTargetException e) {
-                            logger.warn("plugin {}, failed to invoke custom onModule method", e, plugin.v1().getName());
-                            throw new ElasticsearchException("failed to invoke onModule", e);
-                        } catch (Exception e) {
-                            logger.warn((Supplier<?>) () -> new ParameterizedMessage("plugin {}, failed to invoke custom onModule method", plugin.v1().getName()), e);
-                            throw e;
-                        }
-                    }
-                }
-            }
-        }
-    }
-
     public Settings updatedSettings() {
         Map<String, String> foundSettings = new HashMap<>();
         final Settings.Builder builder = Settings.builder();
@@ -315,55 +238,93 @@ public PluginsAndModules info() {
     // a "bundle" is a group of plugins in a single classloader
     // really should be 1-1, but we are not so fortunate
     static class Bundle {
-        List<PluginInfo> plugins = new ArrayList<>();
-        List<URL> urls = new ArrayList<>();
+        final PluginInfo plugin;
+        final Set<URL> urls;
+
+        Bundle(PluginInfo plugin, Set<URL> urls) {
+            this.plugin = Objects.requireNonNull(plugin);
+            this.urls = Objects.requireNonNull(urls);
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Bundle bundle = (Bundle) o;
+            return Objects.equals(plugin, bundle.plugin);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(plugin);
+        }
     }
 
     // similar in impl to getPluginBundles, but DO NOT try to make them share code.
     // we don't need to inherit all the leniency, and things are different enough.
-    static List<Bundle> getModuleBundles(Path modulesDirectory) throws IOException {
+    static Set<Bundle> getModuleBundles(Path modulesDirectory) throws IOException {
         // damn leniency
         if (Files.notExists(modulesDirectory)) {
-            return Collections.emptyList();
+            return Collections.emptySet();
         }
-        List<Bundle> bundles = new ArrayList<>();
+        Set<Bundle> bundles = new LinkedHashSet<>();
         try (DirectoryStream<Path> stream = Files.newDirectoryStream(modulesDirectory)) {
             for (Path module : stream) {
-                if (FileSystemUtils.isHidden(module)) {
-                    continue; // skip over .DS_Store etc
-                }
                 PluginInfo info = PluginInfo.readFromProperties(module);
-                Bundle bundle = new Bundle();
-                bundle.plugins.add(info);
+                Set<URL> urls = new LinkedHashSet<>();
                 // gather urls for jar files
                 try (DirectoryStream<Path> jarStream = Files.newDirectoryStream(module, "*.jar")) {
                     for (Path jar : jarStream) {
                         // normalize with toRealPath to get symlinks out of our hair
-                        bundle.urls.add(jar.toRealPath().toUri().toURL());
+                        URL url = jar.toRealPath().toUri().toURL();
+                            if (urls.add(url) == false) {
+                                throw new IllegalStateException("duplicate codebase: " + url);
+                            }
                     }
                 }
-                bundles.add(bundle);
+                if (bundles.add(new Bundle(info, urls)) == false) {
+                    throw new IllegalStateException("duplicate module: " + info);
+                }
             }
         }
         return bundles;
     }
 
-    static List<Bundle> getPluginBundles(Path pluginsDirectory) throws IOException {
+    static void checkForFailedPluginRemovals(final Path pluginsDirectory) throws IOException {
+        /*
+         * Check for the existence of a marker file that indicates any plugins are in a garbage state from a failed attempt to remove the
+         * plugin.
+         */
+        try (DirectoryStream<Path> stream = Files.newDirectoryStream(pluginsDirectory, ".removing-*")) {
+            final Iterator<Path> iterator = stream.iterator();
+            if (iterator.hasNext()) {
+                final Path removing = iterator.next();
+                final String fileName = removing.getFileName().toString();
+                final String name = fileName.substring(1 + fileName.indexOf("-"));
+                final String message = String.format(
+                        Locale.ROOT,
+                        "found file [%s] from a failed attempt to remove the plugin [%s]; execute [elasticsearch-plugin remove %2$s]",
+                        removing,
+                        name);
+                throw new IllegalStateException(message);
+            }
+        }
+    }
+
+    static Set<Bundle> getPluginBundles(Path pluginsDirectory) throws IOException {
         Logger logger = Loggers.getLogger(PluginsService.class);
 
         // TODO: remove this leniency, but tests bogusly rely on it
         if (!isAccessibleDirectory(pluginsDirectory, logger)) {
-            return Collections.emptyList();
+            return Collections.emptySet();
         }
 
-        List<Bundle> bundles = new ArrayList<>();
+        Set<Bundle> bundles = new LinkedHashSet<>();
+
+        checkForFailedPluginRemovals(pluginsDirectory);
 
         try (DirectoryStream<Path> stream = Files.newDirectoryStream(pluginsDirectory)) {
             for (Path plugin : stream) {
-                if (FileSystemUtils.isHidden(plugin)) {
-                    logger.trace("--- skip hidden plugin file[{}]", plugin.toAbsolutePath());
-                    continue;
-                }
                 logger.trace("--- adding plugin [{}]", plugin.toAbsolutePath());
                 final PluginInfo info;
                 try {
@@ -373,47 +334,58 @@ static List<Bundle> getPluginBundles(Path pluginsDirectory) throws IOException {
                         + plugin.getFileName() + "]. Was the plugin built before 2.0?", e);
                 }
 
-                List<URL> urls = new ArrayList<>();
+                Set<URL> urls = new LinkedHashSet<>();
                 try (DirectoryStream<Path> jarStream = Files.newDirectoryStream(plugin, "*.jar")) {
                     for (Path jar : jarStream) {
                         // normalize with toRealPath to get symlinks out of our hair
-                        urls.add(jar.toRealPath().toUri().toURL());
+                        URL url = jar.toRealPath().toUri().toURL();
+                        if (urls.add(url) == false) {
+                            throw new IllegalStateException("duplicate codebase: " + url);
+                        }
                     }
                 }
-                final Bundle bundle = new Bundle();
-                bundles.add(bundle);
-                bundle.plugins.add(info);
-                bundle.urls.addAll(urls);
+                if (bundles.add(new Bundle(info, urls)) == false) {
+                    throw new IllegalStateException("duplicate plugin: " + info);
+                }
             }
         }
 
         return bundles;
     }
 
-    private List<Tuple<PluginInfo,Plugin>> loadBundles(List<Bundle> bundles) {
+    private List<Tuple<PluginInfo,Plugin>> loadBundles(Set<Bundle> bundles) {
         List<Tuple<PluginInfo, Plugin>> plugins = new ArrayList<>();
 
         for (Bundle bundle : bundles) {
             // jar-hell check the bundle against the parent classloader
             // pluginmanager does it, but we do it again, in case lusers mess with jar files manually
             try {
-                final List<URL> jars = new ArrayList<>();
-                jars.addAll(Arrays.asList(JarHell.parseClassPath()));
-                jars.addAll(bundle.urls);
-                JarHell.checkJarHell(jars.toArray(new URL[0]));
+                Set<URL> classpath = JarHell.parseClassPath();
+                // check we don't have conflicting codebases
+                Set<URL> intersection = new HashSet<>(classpath);
+                intersection.retainAll(bundle.urls);
+                if (intersection.isEmpty() == false) {
+                    throw new IllegalStateException("jar hell! duplicate codebases between" +
+                                                    " plugin and core: " + intersection);
+                }
+                // check we don't have conflicting classes
+                Set<URL> union = new HashSet<>(classpath);
+                union.addAll(bundle.urls);
+                JarHell.checkJarHell(union);
             } catch (Exception e) {
-                throw new IllegalStateException("failed to load bundle " + bundle.urls + " due to jar hell", e);
+                throw new IllegalStateException("failed to load plugin " + bundle.plugin +
+                                                " due to jar hell", e);
             }
 
-            // create a child to load the plugins in this bundle
-            ClassLoader loader = URLClassLoader.newInstance(bundle.urls.toArray(new URL[0]), getClass().getClassLoader());
-            for (PluginInfo pluginInfo : bundle.plugins) {
-                // reload lucene SPI with any new services from the plugin
-                reloadLuceneSPI(loader);
-                final Class<? extends Plugin> pluginClass = loadPluginClass(pluginInfo.getClassname(), loader);
-                final Plugin plugin = loadPlugin(pluginClass, settings);
-                plugins.add(new Tuple<>(pluginInfo, plugin));
-            }
+            // create a child to load the plugin in this bundle
+            ClassLoader loader = URLClassLoader.newInstance(bundle.urls.toArray(new URL[0]),
+                                                            getClass().getClassLoader());
+            // reload lucene SPI with any new services from the plugin
+            reloadLuceneSPI(loader);
+            final Class<? extends Plugin> pluginClass =
+                loadPluginClass(bundle.plugin.getClassname(), loader);
+            final Plugin plugin = loadPlugin(pluginClass, settings);
+            plugins.add(new Tuple<>(bundle.plugin, plugin));
         }
 
         return Collections.unmodifiableList(plugins);
diff --git a/core/src/main/java/org/elasticsearch/plugins/RemovePluginCommand.java b/core/src/main/java/org/elasticsearch/plugins/RemovePluginCommand.java
deleted file mode 100644
index 54cd34d674219..0000000000000
--- a/core/src/main/java/org/elasticsearch/plugins/RemovePluginCommand.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.plugins;
-
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.StandardCopyOption;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import joptsimple.OptionSet;
-import joptsimple.OptionSpec;
-import org.apache.lucene.util.IOUtils;
-import org.elasticsearch.cli.ExitCodes;
-import org.elasticsearch.cli.SettingCommand;
-import org.elasticsearch.cli.UserException;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.cli.Terminal;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.node.internal.InternalSettingsPreparer;
-
-import static org.elasticsearch.cli.Terminal.Verbosity.VERBOSE;
-
-/**
- * A command for the plugin cli to remove a plugin from elasticsearch.
- */
-final class RemovePluginCommand extends SettingCommand {
-
-    private final OptionSpec<String> arguments;
-
-    RemovePluginCommand() {
-        super("Removes a plugin from elasticsearch");
-        this.arguments = parser.nonOptions("plugin name");
-    }
-
-    @Override
-    protected void execute(Terminal terminal, OptionSet options, Map<String, String> settings) throws Exception {
-        String arg = arguments.value(options);
-        execute(terminal, arg, settings);
-    }
-
-    // pkg private for testing
-    void execute(Terminal terminal, String pluginName, Map<String, String> settings) throws Exception {
-        final Environment env = InternalSettingsPreparer.prepareEnvironment(Settings.EMPTY, terminal, settings);
-
-        terminal.println("-> Removing " + Strings.coalesceToEmpty(pluginName) + "...");
-
-        final Path pluginDir = env.pluginsFile().resolve(pluginName);
-        if (Files.exists(pluginDir) == false) {
-            throw new UserException(
-                    ExitCodes.USAGE,
-                    "plugin " + pluginName + " not found; run 'elasticsearch-plugin list' to get list of installed plugins");
-        }
-
-        final List<Path> pluginPaths = new ArrayList<>();
-
-        final Path pluginBinDir = env.binFile().resolve(pluginName);
-        if (Files.exists(pluginBinDir)) {
-            if (Files.isDirectory(pluginBinDir) == false) {
-                throw new UserException(ExitCodes.IO_ERROR, "Bin dir for " + pluginName + " is not a directory");
-            }
-            pluginPaths.add(pluginBinDir);
-            terminal.println(VERBOSE, "Removing: " + pluginBinDir);
-        }
-
-        terminal.println(VERBOSE, "Removing: " + pluginDir);
-        final Path tmpPluginDir = env.pluginsFile().resolve(".removing-" + pluginName);
-        Files.move(pluginDir, tmpPluginDir, StandardCopyOption.ATOMIC_MOVE);
-        pluginPaths.add(tmpPluginDir);
-
-        IOUtils.rm(pluginPaths.toArray(new Path[pluginPaths.size()]));
-
-        // we preserve the config files in case the user is upgrading the plugin, but we print
-        // a message so the user knows in case they want to remove manually
-        final Path pluginConfigDir = env.configFile().resolve(pluginName);
-        if (Files.exists(pluginConfigDir)) {
-            terminal.println(
-                    "-> Preserving plugin config files [" + pluginConfigDir + "] in case of upgrade, delete manually if not needed");
-        }
-    }
-
-}
diff --git a/core/src/main/java/org/elasticsearch/plugins/RepositoryPlugin.java b/core/src/main/java/org/elasticsearch/plugins/RepositoryPlugin.java
index 9306ee3707664..a3af52a9a4aca 100644
--- a/core/src/main/java/org/elasticsearch/plugins/RepositoryPlugin.java
+++ b/core/src/main/java/org/elasticsearch/plugins/RepositoryPlugin.java
@@ -22,6 +22,7 @@
 import java.util.Collections;
 import java.util.Map;
 
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.repositories.Repository;
 
@@ -38,7 +39,7 @@ public interface RepositoryPlugin {
      * The key of the returned {@link Map} is the type name of the repository and
      * the value is a factory to construct the {@link Repository} interface.
      */
-    default Map<String, Repository.Factory> getRepositories(Environment env) {
+    default Map<String, Repository.Factory> getRepositories(Environment env, NamedXContentRegistry namedXContentRegistry) {
         return Collections.emptyMap();
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/plugins/SearchPlugin.java b/core/src/main/java/org/elasticsearch/plugins/SearchPlugin.java
index 97ee715ef6a26..01685535a4e0e 100644
--- a/core/src/main/java/org/elasticsearch/plugins/SearchPlugin.java
+++ b/core/src/main/java/org/elasticsearch/plugins/SearchPlugin.java
@@ -20,16 +20,21 @@
 package org.elasticsearch.plugins;
 
 import org.apache.lucene.search.Query;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.lucene.search.function.ScoreFunction;
 import org.elasticsearch.common.xcontent.XContent;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParser;
 import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder;
 import org.elasticsearch.index.query.functionscore.ScoreFunctionParser;
+import org.elasticsearch.search.SearchExtBuilder;
 import org.elasticsearch.search.aggregations.Aggregation;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.Aggregator;
@@ -44,10 +49,13 @@
 import org.elasticsearch.search.fetch.FetchSubPhase;
 import org.elasticsearch.search.fetch.subphase.highlight.Highlighter;
 import org.elasticsearch.search.suggest.Suggester;
+import org.elasticsearch.search.suggest.SuggestionBuilder;
 
+import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
+import java.util.function.BiConsumer;
 
 import static java.util.Collections.emptyList;
 import static java.util.Collections.emptyMap;
@@ -82,6 +90,12 @@ default List<SearchExtensionSpec<MovAvgModel, MovAvgModel.AbstractModelParser>>
     default List<FetchSubPhase> getFetchSubPhases(FetchPhaseConstructionContext context) {
         return emptyList();
     }
+    /**
+     * The new {@link SearchExtBuilder}s defined by this plugin.
+     */
+    default List<SearchExtSpec<?>> getSearchExts() {
+        return emptyList();
+    }
     /**
      * Get the {@link Highlighter}s defined by this plugin.
      */
@@ -91,8 +105,8 @@ default Map<String, Highlighter> getHighlighters() {
     /**
      * The new {@link Suggester}s defined by this plugin.
      */
-    default Map<String, Suggester<?>> getSuggesters() {
-        return emptyMap();
+    default List<SuggesterSpec<?>> getSuggesters() {
+        return emptyList();
     }
     /**
      * The new {@link Query}s defined by this plugin.
@@ -112,6 +126,15 @@ default List<AggregationSpec> getAggregations() {
     default List<PipelineAggregationSpec> getPipelineAggregations() {
         return emptyList();
     }
+    /**
+     * The new search response listeners in the form of {@link BiConsumer}s added by this plugin.
+     * The listeners are invoked on the coordinating node, at the very end of the search request.
+     * This provides a convenient location if you wish to inspect/modify the final response (took time, etc).
+     * The BiConsumers are passed the original {@link SearchRequest} and the final {@link SearchResponse}
+     */
+    default List<BiConsumer<SearchRequest, SearchResponse>> getSearchResponseListeners() {
+        return emptyList();
+    }
 
     /**
      * Specification of custom {@link ScoreFunction}.
@@ -126,6 +149,38 @@ public ScoreFunctionSpec(String name, Writeable.Reader<T> reader, ScoreFunctionP
         }
     }
 
+    /**
+     * Specification for a {@link Suggester}.
+     */
+    class SuggesterSpec<T extends SuggestionBuilder<T>> extends SearchExtensionSpec<T, CheckedFunction<XContentParser, T, IOException>> {
+        /**
+         * Specification of custom {@link Suggester}.
+         *
+         * @param name holds the names by which this suggester might be parsed. The {@link ParseField#getPreferredName()} is special as it
+         *        is the name by under which the reader is registered. So it is the name that the query should use as its
+         *        {@link NamedWriteable#getWriteableName()} too.
+         * @param reader the reader registered for this suggester's builder. Typically a reference to a constructor that takes a
+         *        {@link StreamInput}
+         * @param parser the parser the reads the query suggester from xcontent
+         */
+        public SuggesterSpec(ParseField name, Writeable.Reader<T> reader, CheckedFunction<XContentParser, T, IOException> parser) {
+            super(name, reader, parser);
+        }
+
+        /**
+         * Specification of custom {@link Suggester}.
+         *
+         * @param name the name by which this suggester might be parsed or deserialized. Make sure that the query builder returns this name
+         *        for {@link NamedWriteable#getWriteableName()}.
+         * @param reader the reader registered for this suggester's builder. Typically a reference to a constructor that takes a
+         *        {@link StreamInput}
+         * @param parser the parser the reads the suggester builder from xcontent
+         */
+        public SuggesterSpec(String name, Writeable.Reader<T> reader, CheckedFunction<XContentParser, T, IOException> parser) {
+            super(name, reader, parser);
+        }
+    }
+
     /**
      * Specification of custom {@link Query}.
      */
@@ -160,7 +215,7 @@ public QuerySpec(String name, Writeable.Reader<T> reader, QueryParser<T> parser)
     /**
      * Specification for an {@link Aggregation}.
      */
-    public static class AggregationSpec extends SearchExtensionSpec<AggregationBuilder, Aggregator.Parser> {
+    class AggregationSpec extends SearchExtensionSpec<AggregationBuilder, Aggregator.Parser> {
         private final Map<String, Writeable.Reader<? extends InternalAggregation>> resultReaders = new TreeMap<>();
 
         /**
@@ -217,7 +272,7 @@ public Map<String, Writeable.Reader<? extends InternalAggregation>> getResultRea
     /**
      * Specification for a {@link PipelineAggregator}.
      */
-    public static class PipelineAggregationSpec extends SearchExtensionSpec<PipelineAggregationBuilder, PipelineAggregator.Parser> {
+    class PipelineAggregationSpec extends SearchExtensionSpec<PipelineAggregationBuilder, PipelineAggregator.Parser> {
         private final Map<String, Writeable.Reader<? extends InternalAggregation>> resultReaders = new TreeMap<>();
         private final Writeable.Reader<? extends PipelineAggregator> aggregatorReader;
 
@@ -290,6 +345,20 @@ public Map<String, Writeable.Reader<? extends InternalAggregation>> getResultRea
         }
     }
 
+    /**
+     * Specification for a {@link SearchExtBuilder} which represents an additional section that can be
+     * parsed in a search request (within the ext element).
+     */
+    class SearchExtSpec<T extends SearchExtBuilder> extends SearchExtensionSpec<T, CheckedFunction<XContentParser, T, IOException>> {
+        public SearchExtSpec(ParseField name, Writeable.Reader<? extends T> reader,
+                CheckedFunction<XContentParser, T, IOException> parser) {
+            super(name, reader, parser);
+        }
+
+        public SearchExtSpec(String name, Writeable.Reader<? extends T> reader, CheckedFunction<XContentParser, T, IOException> parser) {
+            super(name, reader, parser);
+        }
+    }
 
     /**
      * Specification of search time behavior extension like a custom {@link MovAvgModel} or {@link ScoreFunction}.
diff --git a/core/src/main/java/org/elasticsearch/plugins/spi/NamedXContentProvider.java b/core/src/main/java/org/elasticsearch/plugins/spi/NamedXContentProvider.java
new file mode 100644
index 0000000000000..ef511fcfeae35
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/plugins/spi/NamedXContentProvider.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.plugins.spi;
+
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+
+import java.util.List;
+
+/**
+ * Provides named XContent parsers.
+ */
+public interface NamedXContentProvider {
+
+    /**
+     * @return a list of {@link NamedXContentRegistry.Entry} that this plugin provides.
+     */
+    List<NamedXContentRegistry.Entry> getNamedXContentParsers();
+}
diff --git a/core/src/main/java/org/elasticsearch/plugins/spi/package-info.java b/core/src/main/java/org/elasticsearch/plugins/spi/package-info.java
new file mode 100644
index 0000000000000..7740e1424fb76
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/plugins/spi/package-info.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * This package contains interfaces for services provided by
+ * Elasticsearch plugins to external applications like the
+ * Java High Level Rest Client.
+ */
+package org.elasticsearch.plugins.spi;
diff --git a/core/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java b/core/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java
index 50ab90b4fe1b5..d03e2c1ac349f 100644
--- a/core/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java
+++ b/core/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.inject.binder.LinkedBindingBuilder;
 import org.elasticsearch.common.inject.multibindings.MapBinder;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.plugins.RepositoryPlugin;
 import org.elasticsearch.repositories.fs.FsRepository;
@@ -43,13 +44,13 @@ public class RepositoriesModule extends AbstractModule {
 
     private final Map<String, Repository.Factory> repositoryTypes;
 
-    public RepositoriesModule(Environment env, List<RepositoryPlugin> repoPlugins) {
+    public RepositoriesModule(Environment env, List<RepositoryPlugin> repoPlugins, NamedXContentRegistry namedXContentRegistry) {
         Map<String, Repository.Factory> factories = new HashMap<>();
-        factories.put(FsRepository.TYPE, (metadata) -> new FsRepository(metadata, env));
-        factories.put(URLRepository.TYPE, (metadata) -> new URLRepository(metadata, env));
+        factories.put(FsRepository.TYPE, (metadata) -> new FsRepository(metadata, env, namedXContentRegistry));
+        factories.put(URLRepository.TYPE, (metadata) -> new URLRepository(metadata, env, namedXContentRegistry));
 
         for (RepositoryPlugin repoPlugin : repoPlugins) {
-            Map<String, Repository.Factory> newRepoTypes = repoPlugin.getRepositories(env);
+            Map<String, Repository.Factory> newRepoTypes = repoPlugin.getRepositories(env, namedXContentRegistry);
             for (Map.Entry<String, Repository.Factory> entry : newRepoTypes.entrySet()) {
                 if (factories.put(entry.getKey(), entry.getValue()) != null) {
                     throw new IllegalArgumentException("Repository type [" + entry.getKey() + "] is already registered");
diff --git a/core/src/main/java/org/elasticsearch/repositories/RepositoriesService.java b/core/src/main/java/org/elasticsearch/repositories/RepositoriesService.java
index e5951d48a003c..7aea5d18afb43 100644
--- a/core/src/main/java/org/elasticsearch/repositories/RepositoriesService.java
+++ b/core/src/main/java/org/elasticsearch/repositories/RepositoriesService.java
@@ -25,7 +25,7 @@
 import org.elasticsearch.cluster.AckedClusterStateUpdateTask;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.ClusterStateListener;
+import org.elasticsearch.cluster.ClusterStateApplier;
 import org.elasticsearch.cluster.ack.ClusterStateUpdateRequest;
 import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse;
 import org.elasticsearch.cluster.metadata.MetaData;
@@ -53,7 +53,7 @@
 /**
  * Service responsible for maintaining and providing access to snapshot repositories on nodes.
  */
-public class RepositoriesService extends AbstractComponent implements ClusterStateListener {
+public class RepositoriesService extends AbstractComponent implements ClusterStateApplier {
 
     private final Map<String, Repository.Factory> typesRegistry;
 
@@ -72,7 +72,7 @@ public RepositoriesService(Settings settings, ClusterService clusterService, Tra
         // Doesn't make sense to maintain repositories on non-master and non-data nodes
         // Nothing happens there anyway
         if (DiscoveryNode.isDataNode(settings) || DiscoveryNode.isMasterNode(settings)) {
-            clusterService.add(this);
+            clusterService.addStateApplier(this);
         }
         this.verifyAction = new VerifyNodeRepositoryAction(settings, transportService, clusterService, this);
     }
@@ -253,7 +253,7 @@ public void onFailure(Exception e) {
      * @param event cluster changed event
      */
     @Override
-    public void clusterChanged(ClusterChangedEvent event) {
+    public void applyClusterState(ClusterChangedEvent event) {
         try {
             RepositoriesMetaData oldMetaData = event.previousState().getMetaData().custom(RepositoriesMetaData.TYPE);
             RepositoriesMetaData newMetaData = event.state().getMetaData().custom(RepositoriesMetaData.TYPE);
@@ -401,7 +401,7 @@ private class VerifyingRegisterRepositoryListener implements ActionListener<Clus
 
         private final ActionListener<ClusterStateUpdateResponse> listener;
 
-        public VerifyingRegisterRepositoryListener(String name, final ActionListener<ClusterStateUpdateResponse> listener) {
+        VerifyingRegisterRepositoryListener(String name, final ActionListener<ClusterStateUpdateResponse> listener) {
             this.name = name;
             this.listener = listener;
         }
diff --git a/core/src/main/java/org/elasticsearch/repositories/Repository.java b/core/src/main/java/org/elasticsearch/repositories/Repository.java
index b1f534bf684cd..462b7ea1dab12 100644
--- a/core/src/main/java/org/elasticsearch/repositories/Repository.java
+++ b/core/src/main/java/org/elasticsearch/repositories/Repository.java
@@ -115,16 +115,19 @@ interface Factory {
      * @param failure       global failure reason or null
      * @param totalShards   total number of shards
      * @param shardFailures list of shard failures
+     * @param repositoryStateId the unique id identifying the state of the repository when the snapshot began
      * @return snapshot description
      */
-    SnapshotInfo finalizeSnapshot(SnapshotId snapshotId, List<IndexId> indices, long startTime, String failure, int totalShards, List<SnapshotShardFailure> shardFailures);
+    SnapshotInfo finalizeSnapshot(SnapshotId snapshotId, List<IndexId> indices, long startTime, String failure, int totalShards,
+                                  List<SnapshotShardFailure> shardFailures, long repositoryStateId);
 
     /**
      * Deletes snapshot
      *
      * @param snapshotId snapshot id
+     * @param repositoryStateId the unique id identifying the state of the repository when the snapshot deletion began
      */
-    void deleteSnapshot(SnapshotId snapshotId);
+    void deleteSnapshot(SnapshotId snapshotId, long repositoryStateId);
 
     /**
      * Returns snapshot throttle time in nanoseconds
diff --git a/core/src/main/java/org/elasticsearch/repositories/RepositoryData.java b/core/src/main/java/org/elasticsearch/repositories/RepositoryData.java
index 4927e2b41b7f7..747aab8de8451 100644
--- a/core/src/main/java/org/elasticsearch/repositories/RepositoryData.java
+++ b/core/src/main/java/org/elasticsearch/repositories/RepositoryData.java
@@ -20,16 +20,21 @@
 package org.elasticsearch.repositories;
 
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.ResourceNotFoundException;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.snapshots.SnapshotId;
+import org.elasticsearch.snapshots.SnapshotState;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
@@ -42,14 +47,30 @@
  * A class that represents the data in a repository, as captured in the
  * repository's index blob.
  */
-public final class RepositoryData implements ToXContent {
+public final class RepositoryData {
 
-    public static final RepositoryData EMPTY = new RepositoryData(Collections.emptyList(), Collections.emptyMap());
+    /**
+     * The generation value indicating the repository has no index generational files.
+     */
+    public static final long EMPTY_REPO_GEN = -1L;
+    /**
+     * An instance initialized for an empty repository.
+     */
+    public static final RepositoryData EMPTY = new RepositoryData(EMPTY_REPO_GEN,
+        Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyList());
 
+    /**
+     * The generational id of the index file from which the repository data was read.
+     */
+    private final long genId;
     /**
      * The ids of the snapshots in the repository.
      */
-    private final List<SnapshotId> snapshotIds;
+    private final Map<String, SnapshotId> snapshotIds;
+    /**
+     * The states of each snapshot in the repository.
+     */
+    private final Map<String, SnapshotState> snapshotStates;
     /**
      * The indices found in the repository across all snapshots, as a name to {@link IndexId} mapping
      */
@@ -58,24 +79,69 @@ public final class RepositoryData implements ToXContent {
      * The snapshots that each index belongs to.
      */
     private final Map<IndexId, Set<SnapshotId>> indexSnapshots;
+    /**
+     * The snapshots that are no longer compatible with the current cluster ES version.
+     */
+    private final List<SnapshotId> incompatibleSnapshotIds;
 
-    public RepositoryData(List<SnapshotId> snapshotIds, Map<IndexId, Set<SnapshotId>> indexSnapshots) {
-        this.snapshotIds = Collections.unmodifiableList(snapshotIds);
-        this.indices = Collections.unmodifiableMap(indexSnapshots.keySet()
-                                                       .stream()
-                                                       .collect(Collectors.toMap(IndexId::getName, Function.identity())));
+    public RepositoryData(long genId,
+                          Map<String, SnapshotId> snapshotIds,
+                          Map<String, SnapshotState> snapshotStates,
+                          Map<IndexId, Set<SnapshotId>> indexSnapshots,
+                          List<SnapshotId> incompatibleSnapshotIds) {
+        this.genId = genId;
+        this.snapshotIds = Collections.unmodifiableMap(snapshotIds);
+        this.snapshotStates = Collections.unmodifiableMap(snapshotStates);
+        this.indices = Collections.unmodifiableMap(indexSnapshots.keySet().stream()
+            .collect(Collectors.toMap(IndexId::getName, Function.identity())));
         this.indexSnapshots = Collections.unmodifiableMap(indexSnapshots);
+        this.incompatibleSnapshotIds = Collections.unmodifiableList(incompatibleSnapshotIds);
     }
 
     protected RepositoryData copy() {
-        return new RepositoryData(snapshotIds, indexSnapshots);
+        return new RepositoryData(genId, snapshotIds, snapshotStates, indexSnapshots, incompatibleSnapshotIds);
     }
 
     /**
-     * Returns an unmodifiable list of the snapshot ids.
+     * Gets the generational index file id from which this instance was read.
      */
-    public List<SnapshotId> getSnapshotIds() {
-        return snapshotIds;
+    public long getGenId() {
+        return genId;
+    }
+
+    /**
+     * Returns an unmodifiable collection of the snapshot ids.
+     */
+    public Collection<SnapshotId> getSnapshotIds() {
+        return Collections.unmodifiableCollection(snapshotIds.values());
+    }
+
+    /**
+     * Returns an immutable collection of the snapshot ids in the repository that are incompatible with the
+     * current ES version.
+     */
+    public Collection<SnapshotId> getIncompatibleSnapshotIds() {
+        return incompatibleSnapshotIds;
+    }
+
+    /**
+     * Returns an immutable collection of all the snapshot ids in the repository, both active and
+     * incompatible snapshots.
+     */
+    public Collection<SnapshotId> getAllSnapshotIds() {
+        List<SnapshotId> allSnapshotIds = new ArrayList<>(snapshotIds.size() + incompatibleSnapshotIds.size());
+        allSnapshotIds.addAll(snapshotIds.values());
+        allSnapshotIds.addAll(incompatibleSnapshotIds);
+        return Collections.unmodifiableList(allSnapshotIds);
+    }
+
+    /**
+     * Returns the {@link SnapshotState} for the given snapshot.  Returns {@code null} if
+     * there is no state for the snapshot.
+     */
+    @Nullable
+    public SnapshotState getSnapshotState(final SnapshotId snapshotId) {
+        return snapshotStates.get(snapshotId.getUUID());
     }
 
     /**
@@ -89,12 +155,19 @@ public Map<String, IndexId> getIndices() {
      * Add a snapshot and its indices to the repository; returns a new instance.  If the snapshot
      * already exists in the repository data, this method throws an IllegalArgumentException.
      */
-    public RepositoryData addSnapshot(final SnapshotId snapshotId, final List<IndexId> snapshottedIndices) {
-        if (snapshotIds.contains(snapshotId)) {
-            throw new IllegalArgumentException("[" + snapshotId + "] already exists in the repository data");
+    public RepositoryData addSnapshot(final SnapshotId snapshotId,
+                                      final SnapshotState snapshotState,
+                                      final List<IndexId> snapshottedIndices) {
+        if (snapshotIds.containsKey(snapshotId.getUUID())) {
+            // if the snapshot id already exists in the repository data, it means an old master
+            // that is blocked from the cluster is trying to finalize a snapshot concurrently with
+            // the new master, so we make the operation idempotent
+            return this;
         }
-        List<SnapshotId> snapshots = new ArrayList<>(snapshotIds);
-        snapshots.add(snapshotId);
+        Map<String, SnapshotId> snapshots = new HashMap<>(snapshotIds);
+        snapshots.put(snapshotId.getUUID(), snapshotId);
+        Map<String, SnapshotState> newSnapshotStates = new HashMap<>(snapshotStates);
+        newSnapshotStates.put(snapshotId.getUUID(), snapshotState);
         Map<IndexId, Set<SnapshotId>> allIndexSnapshots = new HashMap<>(indexSnapshots);
         for (final IndexId indexId : snapshottedIndices) {
             if (allIndexSnapshots.containsKey(indexId)) {
@@ -110,24 +183,21 @@ public RepositoryData addSnapshot(final SnapshotId snapshotId, final List<IndexI
                 allIndexSnapshots.put(indexId, ids);
             }
         }
-        return new RepositoryData(snapshots, allIndexSnapshots);
-    }
-
-    /**
-     * Initializes the indices in the repository metadata; returns a new instance.
-     */
-    public RepositoryData initIndices(final Map<IndexId, Set<SnapshotId>> indexSnapshots) {
-        return new RepositoryData(snapshotIds, indexSnapshots);
+        return new RepositoryData(genId, snapshots, newSnapshotStates, allIndexSnapshots, incompatibleSnapshotIds);
     }
 
     /**
      * Remove a snapshot and remove any indices that no longer exist in the repository due to the deletion of the snapshot.
      */
     public RepositoryData removeSnapshot(final SnapshotId snapshotId) {
-        List<SnapshotId> newSnapshotIds = snapshotIds
-                                              .stream()
-                                              .filter(id -> snapshotId.equals(id) == false)
-                                              .collect(Collectors.toList());
+        Map<String, SnapshotId> newSnapshotIds = snapshotIds.values().stream()
+            .filter(id -> !snapshotId.equals(id))
+            .collect(Collectors.toMap(SnapshotId::getUUID, Function.identity()));
+        if (newSnapshotIds.size() == snapshotIds.size()) {
+            throw new ResourceNotFoundException("Attempting to remove non-existent snapshot [{}] from repository data", snapshotId);
+        }
+        Map<String, SnapshotState> newSnapshotStates = new HashMap<>(snapshotStates);
+        newSnapshotStates.remove(snapshotId.getUUID());
         Map<IndexId, Set<SnapshotId>> indexSnapshots = new HashMap<>();
         for (final IndexId indexId : indices.values()) {
             Set<SnapshotId> set;
@@ -135,7 +205,8 @@ public RepositoryData removeSnapshot(final SnapshotId snapshotId) {
             assert snapshotIds != null;
             if (snapshotIds.contains(snapshotId)) {
                 if (snapshotIds.size() == 1) {
-                    // removing the snapshot will mean no more snapshots have this index, so just skip over it
+                    // removing the snapshot will mean no more snapshots
+                    // have this index, so just skip over it
                     continue;
                 }
                 set = new LinkedHashSet<>(snapshotIds);
@@ -146,7 +217,7 @@ public RepositoryData removeSnapshot(final SnapshotId snapshotId) {
             indexSnapshots.put(indexId, set);
         }
 
-        return new RepositoryData(newSnapshotIds, indexSnapshots);
+        return new RepositoryData(genId, newSnapshotIds, newSnapshotStates, indexSnapshots, incompatibleSnapshotIds);
     }
 
     /**
@@ -155,11 +226,18 @@ public RepositoryData removeSnapshot(final SnapshotId snapshotId) {
     public Set<SnapshotId> getSnapshots(final IndexId indexId) {
         Set<SnapshotId> snapshotIds = indexSnapshots.get(indexId);
         if (snapshotIds == null) {
-            throw new IllegalArgumentException("unknown snapshot index " + indexId + "");
+            throw new IllegalArgumentException("unknown snapshot index " + indexId);
         }
         return snapshotIds;
     }
 
+    /**
+     * Initializes the indices in the repository metadata; returns a new instance.
+     */
+    public RepositoryData initIndices(final Map<IndexId, Set<SnapshotId>> indexSnapshots) {
+        return new RepositoryData(genId, snapshotIds, snapshotStates, indexSnapshots, incompatibleSnapshotIds);
+    }
+
     @Override
     public boolean equals(Object obj) {
         if (this == obj) {
@@ -170,13 +248,15 @@ public boolean equals(Object obj) {
         }
         @SuppressWarnings("unchecked") RepositoryData that = (RepositoryData) obj;
         return snapshotIds.equals(that.snapshotIds)
+                   && snapshotStates.equals(that.snapshotStates)
                    && indices.equals(that.indices)
-                   && indexSnapshots.equals(that.indexSnapshots);
+                   && indexSnapshots.equals(that.indexSnapshots)
+                   && incompatibleSnapshotIds.equals(that.incompatibleSnapshotIds);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(snapshotIds, indices, indexSnapshots);
+        return Objects.hash(snapshotIds, snapshotStates, indices, indexSnapshots, incompatibleSnapshotIds);
     }
 
     /**
@@ -224,17 +304,45 @@ public List<IndexId> resolveNewIndices(final List<String> indicesToResolve) {
         return snapshotIndices;
     }
 
+    /**
+     * Returns a new {@link RepositoryData} instance containing the same snapshot data as the
+     * invoking instance, with the given incompatible snapshots added to the new instance.
+     */
+    public RepositoryData addIncompatibleSnapshots(final List<SnapshotId> incompatibleSnapshotIds) {
+        List<SnapshotId> newSnapshotIds = new ArrayList<>(this.snapshotIds.values());
+        List<SnapshotId> newIncompatibleSnapshotIds = new ArrayList<>(this.incompatibleSnapshotIds);
+        for (SnapshotId snapshotId : incompatibleSnapshotIds) {
+            newSnapshotIds.remove(snapshotId);
+            newIncompatibleSnapshotIds.add(snapshotId);
+        }
+        Map<String, SnapshotId> snapshotMap = newSnapshotIds.stream().collect(Collectors.toMap(SnapshotId::getUUID, Function.identity()));
+        return new RepositoryData(this.genId, snapshotMap, this.snapshotStates, this.indexSnapshots, newIncompatibleSnapshotIds);
+    }
+
     private static final String SNAPSHOTS = "snapshots";
+    private static final String INCOMPATIBLE_SNAPSHOTS = "incompatible-snapshots";
     private static final String INDICES = "indices";
     private static final String INDEX_ID = "id";
+    private static final String NAME = "name";
+    private static final String UUID = "uuid";
+    private static final String STATE = "state";
 
-    @Override
-    public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException {
+    /**
+     * Writes the snapshots metadata and the related indices metadata to x-content, omitting the
+     * incompatible snapshots.
+     */
+    public XContentBuilder snapshotsToXContent(final XContentBuilder builder, final ToXContent.Params params) throws IOException {
         builder.startObject();
         // write the snapshots list
         builder.startArray(SNAPSHOTS);
         for (final SnapshotId snapshot : getSnapshotIds()) {
-            snapshot.toXContent(builder, params);
+            builder.startObject();
+            builder.field(NAME, snapshot.getName());
+            builder.field(UUID, snapshot.getUUID());
+            if (snapshotStates.containsKey(snapshot.getUUID())) {
+                builder.field(STATE, snapshotStates.get(snapshot.getUUID()).value());
+            }
+            builder.endObject();
         }
         builder.endArray();
         // write the indices map
@@ -246,7 +354,7 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa
             Set<SnapshotId> snapshotIds = indexSnapshots.get(indexId);
             assert snapshotIds != null;
             for (final SnapshotId snapshotId : snapshotIds) {
-                snapshotId.toXContent(builder, params);
+                builder.value(snapshotId.getUUID());
             }
             builder.endArray();
             builder.endObject();
@@ -256,21 +364,51 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa
         return builder;
     }
 
-    public static RepositoryData fromXContent(final XContentParser parser) throws IOException {
-        List<SnapshotId> snapshots = new ArrayList<>();
+    /**
+     * Reads an instance of {@link RepositoryData} from x-content, loading the snapshots and indices metadata.
+     */
+    public static RepositoryData snapshotsFromXContent(final XContentParser parser, long genId) throws IOException {
+        Map<String, SnapshotId> snapshots = new LinkedHashMap<>();
+        Map<String, SnapshotState> snapshotStates = new HashMap<>();
         Map<IndexId, Set<SnapshotId>> indexSnapshots = new HashMap<>();
         if (parser.nextToken() == XContentParser.Token.START_OBJECT) {
             while (parser.nextToken() == XContentParser.Token.FIELD_NAME) {
-                String currentFieldName = parser.currentName();
-                if (SNAPSHOTS.equals(currentFieldName)) {
+                String field = parser.currentName();
+                if (SNAPSHOTS.equals(field)) {
                     if (parser.nextToken() == XContentParser.Token.START_ARRAY) {
                         while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
-                            snapshots.add(SnapshotId.fromXContent(parser));
+                            final SnapshotId snapshotId;
+                            // the new format from 5.0 which contains the snapshot name and uuid
+                            if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
+                                String name = null;
+                                String uuid = null;
+                                SnapshotState state = null;
+                                while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
+                                    String currentFieldName = parser.currentName();
+                                    parser.nextToken();
+                                    if (NAME.equals(currentFieldName)) {
+                                        name = parser.text();
+                                    } else if (UUID.equals(currentFieldName)) {
+                                        uuid = parser.text();
+                                    } else if (STATE.equals(currentFieldName)) {
+                                        state = SnapshotState.fromValue(parser.numberValue().byteValue());
+                                    }
+                                }
+                                snapshotId = new SnapshotId(name, uuid);
+                                if (state != null) {
+                                    snapshotStates.put(uuid, state);
+                                }
+                            } else {
+                                // the old format pre 5.0 that only contains the snapshot name, use the name as the uuid too
+                                final String name = parser.text();
+                                snapshotId = new SnapshotId(name, name);
+                            }
+                            snapshots.put(snapshotId.getUUID(), snapshotId);
                         }
                     } else {
-                        throw new ElasticsearchParseException("expected array for [" + currentFieldName + "]");
+                        throw new ElasticsearchParseException("expected array for [" + field + "]");
                     }
-                } else if (INDICES.equals(currentFieldName)) {
+                } else if (INDICES.equals(field)) {
                     if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
                         throw new ElasticsearchParseException("start object expected [indices]");
                     }
@@ -291,13 +429,72 @@ public static RepositoryData fromXContent(final XContentParser parser) throws IO
                                     throw new ElasticsearchParseException("start array expected [snapshots]");
                                 }
                                 while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
-                                    snapshotIds.add(SnapshotId.fromXContent(parser));
+                                    String uuid = null;
+                                    // the old format pre 5.4.1 which contains the snapshot name and uuid
+                                    if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
+                                        while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
+                                            String currentFieldName = parser.currentName();
+                                            parser.nextToken();
+                                            if (UUID.equals(currentFieldName)) {
+                                                uuid = parser.text();
+                                            }
+                                        }
+                                    } else {
+                                        // the new format post 5.4.1 that only contains the snapshot uuid,
+                                        // since we already have the name/uuid combo in the snapshots array
+                                        uuid = parser.text();
+                                    }
+                                    snapshotIds.add(snapshots.get(uuid));
                                 }
                             }
                         }
                         assert indexId != null;
                         indexSnapshots.put(new IndexId(indexName, indexId), snapshotIds);
                     }
+                } else {
+                    throw new ElasticsearchParseException("unknown field name  [" + field + "]");
+                }
+            }
+        } else {
+            throw new ElasticsearchParseException("start object expected");
+        }
+        return new RepositoryData(genId, snapshots, snapshotStates, indexSnapshots, Collections.emptyList());
+    }
+
+    /**
+     * Writes the incompatible snapshot ids to x-content.
+     */
+    public XContentBuilder incompatibleSnapshotsToXContent(final XContentBuilder builder, final ToXContent.Params params)
+        throws IOException {
+
+        builder.startObject();
+        // write the incompatible snapshots list
+        builder.startArray(INCOMPATIBLE_SNAPSHOTS);
+        for (final SnapshotId snapshot : getIncompatibleSnapshotIds()) {
+            snapshot.toXContent(builder, params);
+        }
+        builder.endArray();
+        builder.endObject();
+        return builder;
+    }
+
+    /**
+     * Reads the incompatible snapshot ids from x-content, loading them into a new instance of {@link RepositoryData}
+     * that is created from the invoking instance, plus the incompatible snapshots that are read from x-content.
+     */
+    public RepositoryData incompatibleSnapshotsFromXContent(final XContentParser parser) throws IOException {
+        List<SnapshotId> incompatibleSnapshotIds = new ArrayList<>();
+        if (parser.nextToken() == XContentParser.Token.START_OBJECT) {
+            while (parser.nextToken() == XContentParser.Token.FIELD_NAME) {
+                String currentFieldName = parser.currentName();
+                if (INCOMPATIBLE_SNAPSHOTS.equals(currentFieldName)) {
+                    if (parser.nextToken() == XContentParser.Token.START_ARRAY) {
+                        while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
+                            incompatibleSnapshotIds.add(SnapshotId.fromXContent(parser));
+                        }
+                    } else {
+                        throw new ElasticsearchParseException("expected array for [" + currentFieldName + "]");
+                    }
                 } else {
                     throw new ElasticsearchParseException("unknown field name  [" + currentFieldName + "]");
                 }
@@ -305,7 +502,7 @@ public static RepositoryData fromXContent(final XContentParser parser) throws IO
         } else {
             throw new ElasticsearchParseException("start object expected");
         }
-        return new RepositoryData(snapshots, indexSnapshots);
+        return new RepositoryData(this.genId, this.snapshotIds, this.snapshotStates, this.indexSnapshots, incompatibleSnapshotIds);
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java b/core/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java
index 49edae3ce22ac..cc1170a4841a2 100644
--- a/core/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java
+++ b/core/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java
@@ -64,10 +64,6 @@ public VerifyNodeRepositoryAction(Settings settings, TransportService transportS
         transportService.registerRequestHandler(ACTION_NAME, VerifyNodeRepositoryRequest::new, ThreadPool.Names.SAME, new VerifyNodeRepositoryRequestHandler());
     }
 
-    public void close() {
-        transportService.removeHandler(ACTION_NAME);
-    }
-
     public void verify(String repository, String verificationToken, final ActionListener<VerifyResponse> listener) {
         final DiscoveryNodes discoNodes = clusterService.state().nodes();
         final DiscoveryNode localNode = discoNodes.getLocalNode();
diff --git a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreFormat.java b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreFormat.java
index 04900705e0ab2..2f25ec3b508cc 100644
--- a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreFormat.java
+++ b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreFormat.java
@@ -19,10 +19,10 @@
 package org.elasticsearch.repositories.blobstore;
 
 import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.blobstore.BlobContainer;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.xcontent.FromXContentBuilder;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -40,9 +40,9 @@ public abstract class BlobStoreFormat<T extends ToXContent> {
 
     protected final String blobNameFormat;
 
-    protected final FromXContentBuilder<T> reader;
+    protected final CheckedFunction<XContentParser, T, IOException> reader;
 
-    protected final ParseFieldMatcher parseFieldMatcher;
+    protected final NamedXContentRegistry namedXContentRegistry;
 
     // Serialization parameters to specify correct context for metadata serialization
     protected static final ToXContent.Params SNAPSHOT_ONLY_FORMAT_PARAMS;
@@ -60,12 +60,12 @@ public abstract class BlobStoreFormat<T extends ToXContent> {
     /**
      * @param blobNameFormat format of the blobname in {@link String#format(Locale, String, Object...)} format
      * @param reader the prototype object that can deserialize objects with type T
-     * @param parseFieldMatcher parse field matcher
      */
-    protected BlobStoreFormat(String blobNameFormat, FromXContentBuilder<T> reader, ParseFieldMatcher parseFieldMatcher) {
+    protected BlobStoreFormat(String blobNameFormat, CheckedFunction<XContentParser, T, IOException> reader,
+            NamedXContentRegistry namedXContentRegistry) {
         this.reader = reader;
         this.blobNameFormat = blobNameFormat;
-        this.parseFieldMatcher = parseFieldMatcher;
+        this.namedXContentRegistry = namedXContentRegistry;
     }
 
     /**
@@ -109,10 +109,9 @@ protected String blobName(String name) {
     }
 
     protected T read(BytesReference bytes) throws IOException {
-        try (XContentParser parser = XContentHelper.createParser(bytes)) {
-            T obj = reader.fromXContent(parser, parseFieldMatcher);
+        try (XContentParser parser = XContentHelper.createParser(namedXContentRegistry, bytes)) {
+            T obj = reader.apply(parser);
             return obj;
-
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
index 2969a6ee6e3c2..b0bacc553ce8f 100644
--- a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
+++ b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
@@ -38,13 +38,14 @@
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.Version;
+import org.elasticsearch.cluster.SnapshotsInProgress;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.metadata.RepositoryMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.Numbers;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.blobstore.BlobContainer;
@@ -68,6 +69,7 @@
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.iterable.Iterables;
 import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -93,6 +95,7 @@
 import org.elasticsearch.repositories.RepositoryData;
 import org.elasticsearch.repositories.RepositoryException;
 import org.elasticsearch.repositories.RepositoryVerificationException;
+import org.elasticsearch.snapshots.InvalidSnapshotNameException;
 import org.elasticsearch.snapshots.SnapshotCreationException;
 import org.elasticsearch.snapshots.SnapshotException;
 import org.elasticsearch.snapshots.SnapshotId;
@@ -100,11 +103,11 @@
 import org.elasticsearch.snapshots.SnapshotMissingException;
 import org.elasticsearch.snapshots.SnapshotShardFailure;
 
-import java.io.FileNotFoundException;
 import java.io.FilterInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.DirectoryNotEmptyException;
+import java.nio.file.FileAlreadyExistsException;
 import java.nio.file.NoSuchFileException;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -128,8 +131,9 @@
  * <pre>
  * {@code
  *   STORE_ROOT
- *   |- index-N           - list of all snapshot name as JSON array, N is the generation of the file
+ *   |- index-N           - list of all snapshot ids and the indices belonging to each snapshot, N is the generation of the file
  *   |- index.latest      - contains the numeric value of the latest generation of the index file (i.e. N from above)
+ *   |- incompatible-snapshots - list of all snapshot ids that are no longer compatible with the current version of the cluster
  *   |- snap-20131010 - JSON serialized Snapshot for snapshot "20131010"
  *   |- meta-20131010.dat - JSON serialized MetaData for snapshot "20131010" (includes only global metadata)
  *   |- snap-20131011 - JSON serialized Snapshot for snapshot "20131011"
@@ -167,6 +171,8 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
 
     protected final RepositoryMetaData metadata;
 
+    protected final NamedXContentRegistry namedXContentRegistry;
+
     private static final int BUFFER_SIZE = 4096;
 
     private static final String LEGACY_SNAPSHOT_PREFIX = "snapshot-";
@@ -181,6 +187,8 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
 
     private static final String INDEX_LATEST_BLOB = "index.latest";
 
+    private static final String INCOMPATIBLE_SNAPSHOTS_BLOB = "incompatible-snapshots";
+
     private static final String TESTS_FILE = "tests-";
 
     private static final String METADATA_NAME_FORMAT = "meta-%s.dat";
@@ -191,17 +199,17 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
 
     private static final String INDEX_METADATA_CODEC = "index-metadata";
 
-    protected static final String LEGACY_SNAPSHOT_NAME_FORMAT = LEGACY_SNAPSHOT_PREFIX + "%s";
+    private static final String LEGACY_SNAPSHOT_NAME_FORMAT = LEGACY_SNAPSHOT_PREFIX + "%s";
 
-    protected static final String SNAPSHOT_NAME_FORMAT = SNAPSHOT_PREFIX + "%s.dat";
+    private static final String SNAPSHOT_NAME_FORMAT = SNAPSHOT_PREFIX + "%s.dat";
 
-    protected static final String SNAPSHOT_INDEX_PREFIX = "index-";
+    private static final String SNAPSHOT_INDEX_PREFIX = "index-";
 
-    protected static final String SNAPSHOT_INDEX_NAME_FORMAT = SNAPSHOT_INDEX_PREFIX + "%s";
+    private static final String SNAPSHOT_INDEX_NAME_FORMAT = SNAPSHOT_INDEX_PREFIX + "%s";
 
-    protected static final String SNAPSHOT_INDEX_CODEC = "snapshots";
+    private static final String SNAPSHOT_INDEX_CODEC = "snapshots";
 
-    protected static final String DATA_BLOB_PREFIX = "__";
+    private static final String DATA_BLOB_PREFIX = "__";
 
     private final RateLimiter snapshotRateLimiter;
 
@@ -225,8 +233,6 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
 
     private final boolean readOnly;
 
-    private final ParseFieldMatcher parseFieldMatcher;
-
     private final ChecksumBlobStoreFormat<BlobStoreIndexShardSnapshot> indexShardSnapshotFormat;
 
     private final LegacyBlobStoreFormat<BlobStoreIndexShardSnapshot> indexShardSnapshotLegacyFormat;
@@ -239,32 +245,39 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
      * @param metadata       The metadata for this repository including name and settings
      * @param globalSettings Settings for the node this repository object is created on
      */
-    protected BlobStoreRepository(RepositoryMetaData metadata, Settings globalSettings) {
+    protected BlobStoreRepository(RepositoryMetaData metadata, Settings globalSettings, NamedXContentRegistry namedXContentRegistry) {
         super(globalSettings);
         this.metadata = metadata;
-        parseFieldMatcher = new ParseFieldMatcher(settings);
+        this.namedXContentRegistry = namedXContentRegistry;
         snapshotRateLimiter = getRateLimiter(metadata.settings(), "max_snapshot_bytes_per_sec", new ByteSizeValue(40, ByteSizeUnit.MB));
         restoreRateLimiter = getRateLimiter(metadata.settings(), "max_restore_bytes_per_sec", new ByteSizeValue(40, ByteSizeUnit.MB));
         readOnly = metadata.settings().getAsBoolean("readonly", false);
-        indexShardSnapshotFormat = new ChecksumBlobStoreFormat<>(SNAPSHOT_CODEC, SNAPSHOT_NAME_FORMAT, BlobStoreIndexShardSnapshot.PROTO, parseFieldMatcher, isCompress());
-        indexShardSnapshotLegacyFormat = new LegacyBlobStoreFormat<>(LEGACY_SNAPSHOT_NAME_FORMAT, BlobStoreIndexShardSnapshot.PROTO, parseFieldMatcher);
-        indexShardSnapshotsFormat = new ChecksumBlobStoreFormat<>(SNAPSHOT_INDEX_CODEC, SNAPSHOT_INDEX_NAME_FORMAT, BlobStoreIndexShardSnapshots.PROTO, parseFieldMatcher, isCompress());
 
+        indexShardSnapshotFormat = new ChecksumBlobStoreFormat<>(SNAPSHOT_CODEC, SNAPSHOT_NAME_FORMAT,
+            BlobStoreIndexShardSnapshot::fromXContent, namedXContentRegistry, isCompress());
+        indexShardSnapshotLegacyFormat = new LegacyBlobStoreFormat<>(LEGACY_SNAPSHOT_NAME_FORMAT,
+            BlobStoreIndexShardSnapshot::fromXContent, namedXContentRegistry);
+        indexShardSnapshotsFormat = new ChecksumBlobStoreFormat<>(SNAPSHOT_INDEX_CODEC, SNAPSHOT_INDEX_NAME_FORMAT,
+            BlobStoreIndexShardSnapshots::fromXContent, namedXContentRegistry, isCompress());
     }
 
     @Override
     protected void doStart() {
         this.snapshotsBlobContainer = blobStore().blobContainer(basePath());
-
-        ParseFieldMatcher parseFieldMatcher = new ParseFieldMatcher(settings);
-        globalMetaDataFormat = new ChecksumBlobStoreFormat<>(METADATA_CODEC, METADATA_NAME_FORMAT, MetaData.PROTO, parseFieldMatcher, isCompress());
-        globalMetaDataLegacyFormat = new LegacyBlobStoreFormat<>(LEGACY_METADATA_NAME_FORMAT, MetaData.PROTO, parseFieldMatcher);
-
-        indexMetaDataFormat = new ChecksumBlobStoreFormat<>(INDEX_METADATA_CODEC, METADATA_NAME_FORMAT, IndexMetaData.PROTO, parseFieldMatcher, isCompress());
-        indexMetaDataLegacyFormat = new LegacyBlobStoreFormat<>(LEGACY_SNAPSHOT_NAME_FORMAT, IndexMetaData.PROTO, parseFieldMatcher);
-
-        snapshotFormat = new ChecksumBlobStoreFormat<>(SNAPSHOT_CODEC, SNAPSHOT_NAME_FORMAT, SnapshotInfo.PROTO, parseFieldMatcher, isCompress());
-        snapshotLegacyFormat = new LegacyBlobStoreFormat<>(LEGACY_SNAPSHOT_NAME_FORMAT, SnapshotInfo.PROTO, parseFieldMatcher);
+        globalMetaDataFormat = new ChecksumBlobStoreFormat<>(METADATA_CODEC, METADATA_NAME_FORMAT,
+            MetaData::fromXContent, namedXContentRegistry, isCompress());
+        globalMetaDataLegacyFormat = new LegacyBlobStoreFormat<>(LEGACY_METADATA_NAME_FORMAT,
+            MetaData::fromXContent, namedXContentRegistry);
+
+        indexMetaDataFormat = new ChecksumBlobStoreFormat<>(INDEX_METADATA_CODEC, METADATA_NAME_FORMAT,
+            IndexMetaData::fromXContent, namedXContentRegistry, isCompress());
+        indexMetaDataLegacyFormat = new LegacyBlobStoreFormat<>(LEGACY_SNAPSHOT_NAME_FORMAT,
+            IndexMetaData::fromXContent, namedXContentRegistry);
+
+        snapshotFormat = new ChecksumBlobStoreFormat<>(SNAPSHOT_CODEC, SNAPSHOT_NAME_FORMAT,
+            SnapshotInfo::fromXContent, namedXContentRegistry, isCompress());
+        snapshotLegacyFormat = new LegacyBlobStoreFormat<>(LEGACY_SNAPSHOT_NAME_FORMAT,
+            SnapshotInfo::fromXContent, namedXContentRegistry);
     }
 
     @Override
@@ -322,12 +335,13 @@ public void initializeSnapshot(SnapshotId snapshotId, List<IndexId> indices, Met
         try {
             final String snapshotName = snapshotId.getName();
             // check if the snapshot name already exists in the repository
-            if (getSnapshots().stream().anyMatch(s -> s.getName().equals(snapshotName))) {
-                throw new SnapshotCreationException(metadata.name(), snapshotId, "snapshot with the same name already exists");
+            final RepositoryData repositoryData = getRepositoryData();
+            if (repositoryData.getAllSnapshotIds().stream().anyMatch(s -> s.getName().equals(snapshotName))) {
+                throw new InvalidSnapshotNameException(metadata.name(), snapshotId.getName(), "snapshot with the same name already exists");
             }
             if (snapshotFormat.exists(snapshotsBlobContainer, snapshotId.getUUID()) ||
                     snapshotLegacyFormat.exists(snapshotsBlobContainer, snapshotName)) {
-                throw new SnapshotCreationException(metadata.name(), snapshotId, "snapshot with such name already exists");
+                throw new InvalidSnapshotNameException(metadata.name(), snapshotId.getName(), "snapshot with the same name already exists");
             }
 
             // Write Global MetaData
@@ -348,8 +362,9 @@ public void initializeSnapshot(SnapshotId snapshotId, List<IndexId> indices, Met
     // Older repository index files (index-N) only contain snapshot info, not indices info,
     // so if the repository data is of the older format, populate it with the indices entries
     // so we know which indices of snapshots have blob ids in the older format.
-    private RepositoryData upgradeRepositoryData(final RepositoryData repositoryData) throws IOException {
+    private RepositoryData upgradeRepositoryData(RepositoryData repositoryData) throws IOException {
         final Map<IndexId, Set<SnapshotId>> indexToSnapshots = new HashMap<>();
+        final List<SnapshotId> incompatibleSnapshots = new ArrayList<>();
         for (final SnapshotId snapshotId : repositoryData.getSnapshotIds()) {
             final SnapshotInfo snapshotInfo;
             try {
@@ -358,7 +373,18 @@ private RepositoryData upgradeRepositoryData(final RepositoryData repositoryData
                 logger.warn((Supplier<?>) () -> new ParameterizedMessage("[{}] repository is on a pre-5.0 format with an index file that contains snapshot [{}] but " +
                         "the corresponding snap-{}.dat file cannot be read. The snapshot will no longer be included in " +
                         "the repository but its data directories will remain.", getMetadata().name(), snapshotId, snapshotId.getUUID()), e);
+                incompatibleSnapshots.add(snapshotId);
                 continue;
+            } catch (IllegalStateException e) {
+                if (e.getMessage().startsWith("unsupported compression")) {
+                    logger.warn((Supplier<?>) () -> new ParameterizedMessage("[{}] attempting to upgrade a pre-5.0 repository " +
+                        "with compression turned on, and snapshot [{}] was taken from a 1.x instance that used a no longer supported " +
+                        "compression format.", getMetadata().name(), snapshotId.getName()), e);
+                    incompatibleSnapshots.add(snapshotId);
+                    continue;
+                } else {
+                    throw e;
+                }
             }
             for (final String indexName : snapshotInfo.indices()) {
                 final IndexId indexId = new IndexId(indexName, indexName);
@@ -370,10 +396,12 @@ private RepositoryData upgradeRepositoryData(final RepositoryData repositoryData
             }
         }
         try {
-            final RepositoryData updatedRepoData = repositoryData.initIndices(indexToSnapshots);
+            final RepositoryData updatedRepoData = repositoryData.addIncompatibleSnapshots(incompatibleSnapshots)
+                                                       .initIndices(indexToSnapshots);
             if (isReadOnly() == false) {
                 // write the new index gen file with the indices included
-                writeIndexGen(updatedRepoData);
+                writeIndexGen(updatedRepoData, updatedRepoData.getGenId());
+                writeIncompatibleSnapshots(updatedRepoData);
             }
             return updatedRepoData;
         } catch (IOException e) {
@@ -382,7 +410,7 @@ private RepositoryData upgradeRepositoryData(final RepositoryData repositoryData
     }
 
     @Override
-    public void deleteSnapshot(SnapshotId snapshotId) {
+    public void deleteSnapshot(SnapshotId snapshotId, long repositoryStateId) {
         if (isReadOnly()) {
             throw new RepositoryException(metadata.name(), "cannot delete snapshot from a readonly repository");
         }
@@ -395,7 +423,7 @@ public void deleteSnapshot(SnapshotId snapshotId) {
         } catch (SnapshotMissingException ex) {
             throw ex;
         } catch (IllegalStateException | SnapshotException | ElasticsearchParseException ex) {
-            logger.warn("cannot read snapshot file [{}]", ex, snapshotId);
+            logger.warn((Supplier<?>) () -> new ParameterizedMessage("cannot read snapshot file [{}]", snapshotId), ex);
         }
         MetaData metaData = null;
         try {
@@ -405,12 +433,12 @@ public void deleteSnapshot(SnapshotId snapshotId) {
                 metaData = readSnapshotMetaData(snapshotId, null, repositoryData.resolveIndices(indices), true);
             }
         } catch (IOException | SnapshotException ex) {
-            logger.warn("cannot read metadata for snapshot [{}]", ex, snapshotId);
+            logger.warn((Supplier<?>) () -> new ParameterizedMessage("cannot read metadata for snapshot [{}]", snapshotId), ex);
         }
         try {
             // Delete snapshot from the index file, since it is the maintainer of truth of active snapshots
             final RepositoryData updatedRepositoryData = repositoryData.removeSnapshot(snapshotId);
-            writeIndexGen(updatedRepositoryData);
+            writeIndexGen(updatedRepositoryData, repositoryStateId);
 
             // delete the snapshot file
             safeSnapshotBlobDelete(snapshot, snapshotId.getUUID());
@@ -462,8 +490,8 @@ public void deleteSnapshot(SnapshotId snapshotId) {
                             "its index folder.", metadata.name(), indexId), ioe);
                 }
             }
-        } catch (IOException ex) {
-            throw new RepositoryException(metadata.name(), "failed to update snapshot in repository", ex);
+        } catch (IOException | ResourceNotFoundException ex) {
+            throw new RepositoryException(metadata.name(), "failed to delete snapshot [" + snapshotId + "]", ex);
         }
     }
 
@@ -524,29 +552,26 @@ public SnapshotInfo finalizeSnapshot(final SnapshotId snapshotId,
                                          final long startTime,
                                          final String failure,
                                          final int totalShards,
-                                         final List<SnapshotShardFailure> shardFailures) {
+                                         final List<SnapshotShardFailure> shardFailures,
+                                         final long repositoryStateId) {
+
+        SnapshotInfo blobStoreSnapshot = new SnapshotInfo(snapshotId,
+            indices.stream().map(IndexId::getName).collect(Collectors.toList()),
+            startTime, failure, System.currentTimeMillis(), totalShards, shardFailures);
         try {
-            SnapshotInfo blobStoreSnapshot = new SnapshotInfo(snapshotId,
-                                                              indices.stream().map(IndexId::getName).collect(Collectors.toList()),
-                                                              startTime,
-                                                              failure,
-                                                              System.currentTimeMillis(),
-                                                              totalShards,
-                                                              shardFailures);
             snapshotFormat.write(blobStoreSnapshot, snapshotsBlobContainer, snapshotId.getUUID());
             final RepositoryData repositoryData = getRepositoryData();
-            List<SnapshotId> snapshotIds = repositoryData.getSnapshotIds();
-            if (!snapshotIds.contains(snapshotId)) {
-                writeIndexGen(repositoryData.addSnapshot(snapshotId, indices));
-            }
-            return blobStoreSnapshot;
+            writeIndexGen(repositoryData.addSnapshot(snapshotId, blobStoreSnapshot.state(), indices), repositoryStateId);
+        } catch (FileAlreadyExistsException ex) {
+            // if another master was elected and took over finalizing the snapshot, it is possible
+            // that both nodes try to finalize the snapshot and write to the same blobs, so we just
+            // log a warning here and carry on
+            throw new RepositoryException(metadata.name(), "Blob already exists while " +
+                "finalizing snapshot, assume the snapshot has already been saved", ex);
         } catch (IOException ex) {
             throw new RepositoryException(metadata.name(), "failed to update snapshot in repository", ex);
         }
-    }
-
-    public List<SnapshotId> getSnapshots() {
-        return getRepositoryData().getSnapshotIds();
+        return blobStoreSnapshot;
     }
 
     @Override
@@ -558,11 +583,11 @@ public MetaData getSnapshotMetaData(SnapshotInfo snapshot, List<IndexId> indices
     public SnapshotInfo getSnapshotInfo(final SnapshotId snapshotId) {
         try {
             return snapshotFormat.read(snapshotsBlobContainer, snapshotId.getUUID());
-        } catch (FileNotFoundException | NoSuchFileException ex) {
+        } catch (NoSuchFileException ex) {
             // File is missing - let's try legacy format instead
             try {
                 return snapshotLegacyFormat.read(snapshotsBlobContainer, snapshotId.getName());
-            } catch (FileNotFoundException | NoSuchFileException ex1) {
+            } catch (NoSuchFileException ex1) {
                 throw new SnapshotMissingException(metadata.name(), snapshotId, ex);
             } catch (IOException | NotXContentException ex1) {
                 throw new SnapshotException(metadata.name(), snapshotId, "failed to get snapshots", ex1);
@@ -588,7 +613,7 @@ private MetaData readSnapshotMetaData(SnapshotId snapshotId, Version snapshotVer
         }
         try {
             metaData = globalMetaDataFormat(snapshotVersion).read(snapshotsBlobContainer, snapshotId.getUUID());
-        } catch (FileNotFoundException | NoSuchFileException ex) {
+        } catch (NoSuchFileException ex) {
             throw new SnapshotMissingException(metadata.name(), snapshotId, ex);
         } catch (IOException ex) {
             throw new SnapshotException(metadata.name(), snapshotId, "failed to get snapshots", ex);
@@ -601,7 +626,7 @@ private MetaData readSnapshotMetaData(SnapshotId snapshotId, Version snapshotVer
                 metaDataBuilder.put(indexMetaDataFormat(snapshotVersion).read(indexMetaDataBlobContainer, snapshotId.getUUID()), false);
             } catch (ElasticsearchParseException | IOException ex) {
                 if (ignoreIndexErrors) {
-                    logger.warn("[{}] [{}] failed to read metadata for index", ex, snapshotId, index.getName());
+                    logger.warn((Supplier<?>) () -> new ParameterizedMessage("[{}] [{}] failed to read metadata for index", snapshotId, index.getName()), ex);
                 } else {
                     throw ex;
                 }
@@ -621,10 +646,10 @@ private MetaData readSnapshotMetaData(SnapshotId snapshotId, Version snapshotVer
     private RateLimiter getRateLimiter(Settings repositorySettings, String setting, ByteSizeValue defaultRate) {
         ByteSizeValue maxSnapshotBytesPerSec = repositorySettings.getAsBytesSize(setting,
                 settings.getAsBytesSize(setting, defaultRate));
-        if (maxSnapshotBytesPerSec.bytes() <= 0) {
+        if (maxSnapshotBytesPerSec.getBytes() <= 0) {
             return null;
         } else {
-            return new RateLimiter.SimpleRateLimiter(maxSnapshotBytesPerSec.mbFrac());
+            return new RateLimiter.SimpleRateLimiter(maxSnapshotBytesPerSec.getMbFrac());
         }
     }
 
@@ -735,8 +760,33 @@ public RepositoryData getRepositoryData() {
             try (InputStream blob = snapshotsBlobContainer.readBlob(snapshotsIndexBlobName)) {
                 BytesStreamOutput out = new BytesStreamOutput();
                 Streams.copy(blob, out);
-                try (XContentParser parser = XContentHelper.createParser(out.bytes())) {
-                    repositoryData = RepositoryData.fromXContent(parser);
+                // EMPTY is safe here because RepositoryData#fromXContent calls namedObject
+                try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, out.bytes())) {
+                    repositoryData = RepositoryData.snapshotsFromXContent(parser, indexGen);
+                } catch (NotXContentException e) {
+                    logger.warn("[{}] index blob is not valid x-content [{} bytes]", snapshotsIndexBlobName, out.bytes().length());
+                    throw e;
+                }
+            }
+
+            // now load the incompatible snapshot ids, if they exist
+            try (InputStream blob = snapshotsBlobContainer.readBlob(INCOMPATIBLE_SNAPSHOTS_BLOB)) {
+                BytesStreamOutput out = new BytesStreamOutput();
+                Streams.copy(blob, out);
+                try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, out.bytes())) {
+                    repositoryData = repositoryData.incompatibleSnapshotsFromXContent(parser);
+                }
+            } catch (NoSuchFileException e) {
+                if (isReadOnly()) {
+                    logger.debug("[{}] Incompatible snapshots blob [{}] does not exist, the likely " +
+                                 "reason is that there are no incompatible snapshots in the repository",
+                                 metadata.name(), INCOMPATIBLE_SNAPSHOTS_BLOB);
+                } else {
+                    // write an empty incompatible-snapshots blob - we do this so that there
+                    // is a blob present, which helps speed up some cloud-based repositories
+                    // (e.g. S3), which retry if a blob is missing with exponential backoff,
+                    // delaying the read of repository data and sometimes causing a timeout
+                    writeIncompatibleSnapshots(RepositoryData.EMPTY);
                 }
             }
             if (legacyFormat) {
@@ -744,7 +794,7 @@ public RepositoryData getRepositoryData() {
                 repositoryData = upgradeRepositoryData(repositoryData);
             }
             return repositoryData;
-        } catch (NoSuchFileException nsfe) {
+        } catch (NoSuchFileException ex) {
             // repository doesn't have an index blob, its a new blank repo
             return RepositoryData.EMPTY;
         } catch (IOException ioe) {
@@ -766,44 +816,74 @@ BlobContainer blobContainer() {
         return snapshotsBlobContainer;
     }
 
-    protected void writeIndexGen(final RepositoryData repositoryData) throws IOException {
+    protected void writeIndexGen(final RepositoryData repositoryData, final long repositoryStateId) throws IOException {
         assert isReadOnly() == false; // can not write to a read only repository
+        final long currentGen = latestIndexBlobId();
+        if (repositoryStateId != SnapshotsInProgress.UNDEFINED_REPOSITORY_STATE_ID && currentGen != repositoryStateId) {
+            // the index file was updated by a concurrent operation, so we were operating on stale
+            // repository data
+            throw new RepositoryException(metadata.name(), "concurrent modification of the index-N file, expected current generation [" +
+                                              repositoryStateId + "], actual current generation [" + currentGen +
+                                              "] - possibly due to simultaneous snapshot deletion requests");
+        }
+        final long newGen = currentGen + 1;
         final BytesReference snapshotsBytes;
         try (BytesStreamOutput bStream = new BytesStreamOutput()) {
             try (StreamOutput stream = new OutputStreamStreamOutput(bStream)) {
                 XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON, stream);
-                repositoryData.toXContent(builder, ToXContent.EMPTY_PARAMS);
+                repositoryData.snapshotsToXContent(builder, ToXContent.EMPTY_PARAMS);
                 builder.close();
             }
             snapshotsBytes = bStream.bytes();
         }
-        final long gen = latestIndexBlobId() + 1;
         // write the index file
-        writeAtomic(INDEX_FILE_PREFIX + Long.toString(gen), snapshotsBytes);
+        final String indexBlob = INDEX_FILE_PREFIX + Long.toString(newGen);
+        logger.debug("Repository [{}] writing new index generational blob [{}]", metadata.name(), indexBlob);
+        writeAtomic(indexBlob, snapshotsBytes);
         // delete the N-2 index file if it exists, keep the previous one around as a backup
-        if (isReadOnly() == false && gen - 2 >= 0) {
-            final String oldSnapshotIndexFile = INDEX_FILE_PREFIX + Long.toString(gen - 2);
+        if (isReadOnly() == false && newGen - 2 >= 0) {
+            final String oldSnapshotIndexFile = INDEX_FILE_PREFIX + Long.toString(newGen - 2);
             if (snapshotsBlobContainer.blobExists(oldSnapshotIndexFile)) {
                 snapshotsBlobContainer.deleteBlob(oldSnapshotIndexFile);
             }
-            // delete the old index file (non-generational) if it exists
-            if (snapshotsBlobContainer.blobExists(SNAPSHOTS_FILE)) {
-                snapshotsBlobContainer.deleteBlob(SNAPSHOTS_FILE);
-            }
         }
 
         // write the current generation to the index-latest file
         final BytesReference genBytes;
         try (BytesStreamOutput bStream = new BytesStreamOutput()) {
-            bStream.writeLong(gen);
+            bStream.writeLong(newGen);
             genBytes = bStream.bytes();
         }
         if (snapshotsBlobContainer.blobExists(INDEX_LATEST_BLOB)) {
             snapshotsBlobContainer.deleteBlob(INDEX_LATEST_BLOB);
         }
+        logger.debug("Repository [{}] updating index.latest with generation [{}]", metadata.name(), newGen);
         writeAtomic(INDEX_LATEST_BLOB, genBytes);
     }
 
+    /**
+     * Writes the incompatible snapshot ids list to the `incompatible-snapshots` blob in the repository.
+     *
+     * Package private for testing.
+     */
+    void writeIncompatibleSnapshots(RepositoryData repositoryData) throws IOException {
+        assert isReadOnly() == false; // can not write to a read only repository
+        final BytesReference bytes;
+        try (BytesStreamOutput bStream = new BytesStreamOutput()) {
+            try (StreamOutput stream = new OutputStreamStreamOutput(bStream)) {
+                XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON, stream);
+                repositoryData.incompatibleSnapshotsToXContent(builder, ToXContent.EMPTY_PARAMS);
+                builder.close();
+            }
+            bytes = bStream.bytes();
+        }
+        if (snapshotsBlobContainer.blobExists(INCOMPATIBLE_SNAPSHOTS_BLOB)) {
+            snapshotsBlobContainer.deleteBlob(INCOMPATIBLE_SNAPSHOTS_BLOB);
+        }
+        // write the incompatible snapshots blob
+        writeAtomic(INCOMPATIBLE_SNAPSHOTS_BLOB, bytes);
+    }
+
     /**
      * Get the latest snapshot index blob id.  Snapshot index blobs are named index-N, where N is
      * the next version number from when the index blob was written.  Each individual index-N blob is
@@ -814,24 +894,25 @@ protected void writeIndexGen(final RepositoryData repositoryData) throws IOExcep
      */
     long latestIndexBlobId() throws IOException {
         try {
-            // first, try listing the blobs and determining which index blob is the latest
+            // First, try listing all index-N blobs (there should only be two index-N blobs at any given
+            // time in a repository if cleanup is happening properly) and pick the index-N blob with the
+            // highest N value - this will be the latest index blob for the repository.  Note, we do this
+            // instead of directly reading the index.latest blob to get the current index-N blob because
+            // index.latest is not written atomically and is not immutable - on every index-N change,
+            // we first delete the old index.latest and then write the new one.  If the repository is not
+            // read-only, it is possible that we try deleting the index.latest blob while it is being read
+            // by some other operation (such as the get snapshots operation).  In some file systems, it is
+            // illegal to delete a file while it is being read elsewhere (e.g. Windows).  For read-only
+            // repositories, we read for index.latest, both because listing blob prefixes is often unsupported
+            // and because the index.latest blob will never be deleted and re-written.
             return listBlobsToGetLatestIndexId();
         } catch (UnsupportedOperationException e) {
-            // could not list the blobs because the repository does not support the operation,
-            // try reading from the index-latest file
+            // If its a read-only repository, listing blobs by prefix may not be supported (e.g. a URL repository),
+            // in this case, try reading the latest index generation from the index.latest blob
             try {
                 return readSnapshotIndexLatestBlob();
-            } catch (IOException ioe) {
-                // we likely could not find the blob, this can happen in two scenarios:
-                //  (1) its an empty repository
-                //  (2) when writing the index-latest blob, if the blob already exists,
-                //      we first delete it, then atomically write the new blob.  there is
-                //      a small window in time when the blob is deleted and the new one
-                //      written - if the node crashes during that time, we won't have an
-                //      index-latest blob
-                // in a read-only repository, we can't know which of the two scenarios it is,
-                // but we will assume (1) because we can't do anything about (2) anyway
-                return -1;
+            } catch (NoSuchFileException nsfe) {
+                return RepositoryData.EMPTY_REPO_GEN;
             }
         }
     }
@@ -847,7 +928,7 @@ long readSnapshotIndexLatestBlob() throws IOException {
 
     private long listBlobsToGetLatestIndexId() throws IOException {
         Map<String, BlobMetaData> blobs = snapshotsBlobContainer.listBlobsByPrefix(INDEX_FILE_PREFIX);
-        long latest = -1;
+        long latest = RepositoryData.EMPTY_REPO_GEN;
         if (blobs.isEmpty()) {
             // no snapshot index blobs have been written yet
             return latest;
@@ -867,21 +948,21 @@ private long listBlobsToGetLatestIndexId() throws IOException {
     }
 
     private void writeAtomic(final String blobName, final BytesReference bytesRef) throws IOException {
-        final String tempBlobName = "pending-" + blobName;
+        final String tempBlobName = "pending-" + blobName + "-" + UUIDs.randomBase64UUID();
         try (InputStream stream = bytesRef.streamInput()) {
             snapshotsBlobContainer.writeBlob(tempBlobName, stream, bytesRef.length());
-        }
-        try {
             snapshotsBlobContainer.move(tempBlobName, blobName);
         } catch (IOException ex) {
-            // Move failed - try cleaning up
-            snapshotsBlobContainer.deleteBlob(tempBlobName);
+            // temporary blob creation or move failed - try cleaning up
+            try {
+                snapshotsBlobContainer.deleteBlob(tempBlobName);
+            } catch (IOException e) {
+                ex.addSuppressed(e);
+            }
             throw ex;
         }
     }
 
-
-
     @Override
     public void snapshotShard(IndexShard shard, SnapshotId snapshotId, IndexId indexId, IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus) {
         SnapshotContext snapshotContext = new SnapshotContext(shard, snapshotId, indexId, snapshotStatus);
@@ -986,11 +1067,11 @@ private class Context {
 
         protected final Version version;
 
-        public Context(SnapshotId snapshotId, Version version, IndexId indexId, ShardId shardId) {
+        Context(SnapshotId snapshotId, Version version, IndexId indexId, ShardId shardId) {
             this(snapshotId, version, indexId, shardId, shardId);
         }
 
-        public Context(SnapshotId snapshotId, Version version, IndexId indexId, ShardId shardId, ShardId snapshotShardId) {
+        Context(SnapshotId snapshotId, Version version, IndexId indexId, ShardId shardId, ShardId snapshotShardId) {
             this.snapshotId = snapshotId;
             this.version = version;
             this.shardId = shardId;
@@ -1036,7 +1117,7 @@ public BlobStoreIndexShardSnapshot loadSnapshot() {
             try {
                 return indexShardSnapshotFormat(version).read(blobContainer, snapshotId.getUUID());
             } catch (IOException ex) {
-                throw new IndexShardRestoreFailedException(shardId, "failed to read shard snapshot file", ex);
+                throw new SnapshotException(metadata.name(), snapshotId, "failed to read shard snapshot file for " + shardId, ex);
             }
         }
 
@@ -1135,7 +1216,8 @@ protected long findLatestFileNameGeneration(Map<String, BlobMetaData> blobs) {
          */
         protected Tuple<BlobStoreIndexShardSnapshots, Integer> buildBlobStoreIndexShardSnapshots(Map<String, BlobMetaData> blobs) {
             int latest = -1;
-            for (String name : blobs.keySet()) {
+            Set<String> blobKeys = blobs.keySet();
+            for (String name : blobKeys) {
                 if (name.startsWith(SNAPSHOT_INDEX_PREFIX)) {
                     try {
                         int gen = Integer.parseInt(name.substring(SNAPSHOT_INDEX_PREFIX.length()));
@@ -1156,15 +1238,17 @@ protected Tuple<BlobStoreIndexShardSnapshots, Integer> buildBlobStoreIndexShardS
                     final String file = SNAPSHOT_INDEX_PREFIX + latest;
                     logger.warn((Supplier<?>) () -> new ParameterizedMessage("failed to read index file [{}]", file), e);
                 }
+            } else if (blobKeys.isEmpty() == false) {
+                logger.debug("Could not find a readable index-N file in a non-empty shard snapshot directory [{}]", blobContainer.path());
             }
 
             // We couldn't load the index file - falling back to loading individual snapshots
             List<SnapshotFiles> snapshots = new ArrayList<>();
-            for (String name : blobs.keySet()) {
+            for (String name : blobKeys) {
                 try {
                     BlobStoreIndexShardSnapshot snapshot = null;
                     if (name.startsWith(SNAPSHOT_PREFIX)) {
-                        snapshot = indexShardSnapshotFormat.readBlob(blobContainer, snapshotId.getUUID());
+                        snapshot = indexShardSnapshotFormat.readBlob(blobContainer, name);
                     } else if (name.startsWith(LEGACY_SNAPSHOT_PREFIX)) {
                         snapshot = indexShardSnapshotLegacyFormat.readBlob(blobContainer, name);
                     }
@@ -1196,7 +1280,7 @@ private class SnapshotContext extends Context {
          * @param indexId        the id of the index being snapshotted
          * @param snapshotStatus snapshot status to report progress
          */
-        public SnapshotContext(IndexShard shard, SnapshotId snapshotId, IndexId indexId, IndexShardSnapshotStatus snapshotStatus) {
+        SnapshotContext(IndexShard shard, SnapshotId snapshotId, IndexId indexId, IndexShardSnapshotStatus snapshotStatus) {
             super(snapshotId, Version.CURRENT, indexId, shard.shardId());
             this.snapshotStatus = snapshotStatus;
             this.store = shard.store();
@@ -1399,7 +1483,7 @@ private boolean snapshotFileExistsInBlobs(BlobStoreIndexShardSnapshot.FileInfo f
         private class AbortableInputStream extends FilterInputStream {
             private final String fileName;
 
-            public AbortableInputStream(InputStream delegate, String fileName) {
+            AbortableInputStream(InputStream delegate, String fileName) {
                 super(delegate);
                 this.fileName = fileName;
             }
@@ -1437,7 +1521,7 @@ private static void maybeRecalculateMetadataHash(final BlobContainer blobContain
                 // we have a hash - check if our repo has a hash too otherwise we have
                 // to calculate it.
                 // we might have multiple parts even though the file is small... make sure we read all of it.
-                try (final InputStream stream = new PartSliceStream(blobContainer, fileInfo)) {
+                try (InputStream stream = new PartSliceStream(blobContainer, fileInfo)) {
                     BytesRefBuilder builder = new BytesRefBuilder();
                     Store.MetadataSnapshot.hashFile(builder, stream, fileInfo.length());
                     BytesRef hash = fileInfo.metadata().hash(); // reset the file infos metadata hash
@@ -1455,7 +1539,7 @@ private static final class PartSliceStream extends SlicedInputStream {
         private final BlobContainer container;
         private final BlobStoreIndexShardSnapshot.FileInfo info;
 
-        public PartSliceStream(BlobContainer container, BlobStoreIndexShardSnapshot.FileInfo info) {
+        PartSliceStream(BlobContainer container, BlobStoreIndexShardSnapshot.FileInfo info) {
             super(info.numberOfParts());
             this.info = info;
             this.container = container;
@@ -1485,7 +1569,7 @@ private class RestoreContext extends Context {
          * @param snapshotShardId shard in the snapshot that data should be restored from
          * @param recoveryState   recovery state to report progress
          */
-        public RestoreContext(IndexShard shard, SnapshotId snapshotId, Version version, IndexId indexId, ShardId snapshotShardId, RecoveryState recoveryState) {
+        RestoreContext(IndexShard shard, SnapshotId snapshotId, Version version, IndexId indexId, ShardId snapshotShardId, RecoveryState recoveryState) {
             super(snapshotId, version, indexId, shard.shardId(), snapshotShardId);
             this.recoveryState = recoveryState;
             this.targetShard = shard;
@@ -1647,7 +1731,7 @@ private void restoreFile(final BlobStoreIndexShardSnapshot.FileInfo fileInfo, fi
                     stream = new RateLimitingInputStream(partSliceStream, restoreRateLimiter, restoreRateLimitingTimeInNanos::inc);
                 }
 
-                try (final IndexOutput indexOutput = store.createVerifyingOutput(fileInfo.physicalName(), fileInfo.metadata(), IOContext.DEFAULT)) {
+                try (IndexOutput indexOutput = store.createVerifyingOutput(fileInfo.physicalName(), fileInfo.metadata(), IOContext.DEFAULT)) {
                     final byte[] buffer = new byte[BUFFER_SIZE];
                     int length;
                     while ((length = stream.read(buffer)) > 0) {
diff --git a/core/src/main/java/org/elasticsearch/repositories/blobstore/ChecksumBlobStoreFormat.java b/core/src/main/java/org/elasticsearch/repositories/blobstore/ChecksumBlobStoreFormat.java
index 17fad25e610eb..0cb38d9976d2d 100644
--- a/core/src/main/java/org/elasticsearch/repositories/blobstore/ChecksumBlobStoreFormat.java
+++ b/core/src/main/java/org/elasticsearch/repositories/blobstore/ChecksumBlobStoreFormat.java
@@ -23,7 +23,7 @@
 import org.apache.lucene.index.IndexFormatTooNewException;
 import org.apache.lucene.index.IndexFormatTooOldException;
 import org.apache.lucene.store.OutputStreamIndexOutput;
-import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.blobstore.BlobContainer;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -33,10 +33,11 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lucene.store.ByteArrayIndexInput;
 import org.elasticsearch.common.lucene.store.IndexOutputOutputStream;
-import org.elasticsearch.common.xcontent.FromXContentBuilder;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.gateway.CorruptStateException;
 
@@ -73,8 +74,9 @@ public class ChecksumBlobStoreFormat<T extends ToXContent> extends BlobStoreForm
      * @param compress       true if the content should be compressed
      * @param xContentType   content type that should be used for write operations
      */
-    public ChecksumBlobStoreFormat(String codec, String blobNameFormat, FromXContentBuilder<T> reader, ParseFieldMatcher parseFieldMatcher, boolean compress, XContentType xContentType) {
-        super(blobNameFormat, reader, parseFieldMatcher);
+    public ChecksumBlobStoreFormat(String codec, String blobNameFormat, CheckedFunction<XContentParser, T, IOException> reader,
+                                   NamedXContentRegistry namedXContentRegistry, boolean compress, XContentType xContentType) {
+        super(blobNameFormat, reader, namedXContentRegistry);
         this.xContentType = xContentType;
         this.compress = compress;
         this.codec = codec;
@@ -86,8 +88,9 @@ public ChecksumBlobStoreFormat(String codec, String blobNameFormat, FromXContent
      * @param reader         prototype object that can deserialize T from XContent
      * @param compress       true if the content should be compressed
      */
-    public ChecksumBlobStoreFormat(String codec, String blobNameFormat, FromXContentBuilder<T> reader, ParseFieldMatcher parseFieldMatcher, boolean compress) {
-        this(codec, blobNameFormat, reader, parseFieldMatcher, compress, DEFAULT_X_CONTENT_TYPE);
+    public ChecksumBlobStoreFormat(String codec, String blobNameFormat, CheckedFunction<XContentParser, T, IOException> reader,
+                                   NamedXContentRegistry namedXContentRegistry, boolean compress) {
+        this(codec, blobNameFormat, reader, namedXContentRegistry, compress, DEFAULT_X_CONTENT_TYPE);
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/repositories/blobstore/LegacyBlobStoreFormat.java b/core/src/main/java/org/elasticsearch/repositories/blobstore/LegacyBlobStoreFormat.java
index 206371261e7f3..7ea711d4df620 100644
--- a/core/src/main/java/org/elasticsearch/repositories/blobstore/LegacyBlobStoreFormat.java
+++ b/core/src/main/java/org/elasticsearch/repositories/blobstore/LegacyBlobStoreFormat.java
@@ -18,12 +18,13 @@
  */
 package org.elasticsearch.repositories.blobstore;
 
-import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.blobstore.BlobContainer;
 import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.xcontent.FromXContentBuilder;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -37,8 +38,9 @@ public class LegacyBlobStoreFormat<T extends ToXContent> extends BlobStoreFormat
      * @param blobNameFormat format of the blobname in {@link String#format} format
      * @param reader the prototype object that can deserialize objects with type T
      */
-    public LegacyBlobStoreFormat(String blobNameFormat, FromXContentBuilder<T> reader, ParseFieldMatcher parseFieldMatcher) {
-        super(blobNameFormat, reader, parseFieldMatcher);
+    public LegacyBlobStoreFormat(String blobNameFormat, CheckedFunction<XContentParser, T, IOException> reader,
+                                 NamedXContentRegistry namedXContentRegistry) {
+        super(blobNameFormat, reader, namedXContentRegistry);
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java b/core/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java
index c028913d343db..b490a2e784dc1 100644
--- a/core/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java
+++ b/core/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.repositories.RepositoryException;
 import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
@@ -72,8 +73,9 @@ public class FsRepository extends BlobStoreRepository {
     /**
      * Constructs a shared file system repository.
      */
-    public FsRepository(RepositoryMetaData metadata, Environment environment) throws IOException {
-        super(metadata, environment.settings());
+    public FsRepository(RepositoryMetaData metadata, Environment environment,
+                        NamedXContentRegistry namedXContentRegistry) throws IOException {
+        super(metadata, environment.settings(), namedXContentRegistry);
         String location = REPOSITORIES_LOCATION_SETTING.get(metadata.settings());
         if (location.isEmpty()) {
             logger.warn("the repository location is missing, it should point to a shared file system location that is available on all master and data nodes");
diff --git a/core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java b/core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java
index 5ca335c595364..fdeb27819bf08 100644
--- a/core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java
+++ b/core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.util.URIPattern;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.repositories.RepositoryException;
 import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
@@ -77,8 +78,9 @@ public class URLRepository extends BlobStoreRepository {
     /**
      * Constructs a read-only URL-based repository
      */
-    public URLRepository(RepositoryMetaData metadata, Environment environment) throws IOException {
-        super(metadata, environment.settings());
+    public URLRepository(RepositoryMetaData metadata, Environment environment,
+                         NamedXContentRegistry namedXContentRegistry) throws IOException {
+        super(metadata, environment.settings(), namedXContentRegistry);
 
         if (URL_SETTING.exists(metadata.settings()) == false && REPOSITORIES_URL_SETTING.exists(settings) ==  false) {
             throw new RepositoryException(metadata.name(), "missing url");
diff --git a/core/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java b/core/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java
index f146267c9be33..4db9aec6e93ef 100644
--- a/core/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java
+++ b/core/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java
@@ -20,13 +20,14 @@
 
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
 
 import java.io.IOException;
+import java.io.OutputStream;
 import java.util.Collections;
 import java.util.Set;
 import java.util.function.Predicate;
@@ -40,59 +41,79 @@ public abstract class AbstractRestChannel implements RestChannel {
 
     protected final RestRequest request;
     protected final boolean detailedErrorsEnabled;
+    private final String format;
+    private final String filterPath;
+    private final boolean pretty;
+    private final boolean human;
 
     private BytesStreamOutput bytesOut;
 
     protected AbstractRestChannel(RestRequest request, boolean detailedErrorsEnabled) {
         this.request = request;
         this.detailedErrorsEnabled = detailedErrorsEnabled;
+        this.format = request.param("format", request.header("Accept"));
+        this.filterPath = request.param("filter_path", null);
+        this.pretty = request.paramAsBoolean("pretty", false);
+        this.human = request.paramAsBoolean("human", false);
     }
 
     @Override
     public XContentBuilder newBuilder() throws IOException {
-        return newBuilder(request.hasContent() ? request.content() : null, true);
+        return newBuilder(request.getXContentType(), true);
     }
 
     @Override
     public XContentBuilder newErrorBuilder() throws IOException {
         // Disable filtering when building error responses
-        return newBuilder(request.hasContent() ? request.content() : null, false);
+        return newBuilder(request.getXContentType(), false);
     }
 
+    /**
+     * Creates a new {@link XContentBuilder} for a response to be sent using this channel. The builder's type is determined by the following
+     * logic. If the request has a format parameter that will be used to attempt to map to an {@link XContentType}. If there is no format
+     * parameter, the HTTP Accept header is checked to see if it can be matched to a {@link XContentType}. If this first attempt to map
+     * fails, the request content type will be used if the value is not {@code null}; if the value is {@code null} the output format falls
+     * back to JSON.
+     */
     @Override
-    public XContentBuilder newBuilder(@Nullable BytesReference autoDetectSource, boolean useFiltering) throws IOException {
-        XContentType contentType = XContentType.fromMediaTypeOrFormat(request.param("format", request.header("Accept")));
-        if (contentType == null) {
-            // try and guess it from the auto detect source
-            if (autoDetectSource != null) {
-                contentType = XContentFactory.xContentType(autoDetectSource);
+    public XContentBuilder newBuilder(@Nullable XContentType requestContentType, boolean useFiltering) throws IOException {
+        // try to determine the response content type from the media type or the format query string parameter, with the format parameter
+        // taking precedence over the Accept header
+        XContentType responseContentType = XContentType.fromMediaTypeOrFormat(format);
+        if (responseContentType == null) {
+            if (requestContentType != null) {
+                // if there was a parsed content-type for the incoming request use that since no format was specified using the query
+                // string parameter or the HTTP Accept header
+                responseContentType = requestContentType;
+            } else {
+                // default to JSON output when all else fails
+                responseContentType = XContentType.JSON;
             }
         }
-        if (contentType == null) {
-            // default to JSON
-            contentType = XContentType.JSON;
-        }
 
         Set<String> includes = Collections.emptySet();
         Set<String> excludes = Collections.emptySet();
         if (useFiltering) {
-            Set<String> filters = Strings.splitStringByCommaToSet(request.param("filter_path", null));
+            Set<String> filters = Strings.splitStringByCommaToSet(filterPath);
             includes = filters.stream().filter(INCLUDE_FILTER).collect(toSet());
             excludes = filters.stream().filter(EXCLUDE_FILTER).map(f -> f.substring(1)).collect(toSet());
         }
 
-        XContentBuilder builder = new XContentBuilder(XContentFactory.xContent(contentType), bytesOutput(), includes, excludes);
-        if (request.paramAsBoolean("pretty", false)) {
+        OutputStream unclosableOutputStream = Streams.flushOnCloseStream(bytesOutput());
+        XContentBuilder builder =
+            new XContentBuilder(XContentFactory.xContent(responseContentType), unclosableOutputStream, includes, excludes);
+        if (pretty) {
             builder.prettyPrint().lfAtEnd();
         }
 
-        builder.humanReadable(request.paramAsBoolean("human", builder.humanReadable()));
+        builder.humanReadable(human);
         return builder;
     }
 
     /**
-     * A channel level bytes output that can be reused. It gets reset on each call to this
-     * method.
+     * A channel level bytes output that can be reused. The bytes output is lazily instantiated
+     * by a call to {@link #newBytesOutput()}. Once the stream is created, it gets reset on each
+     * call to this method.
      */
     @Override
     public final BytesStreamOutput bytesOutput() {
@@ -104,6 +125,14 @@ public final BytesStreamOutput bytesOutput() {
         return bytesOut;
     }
 
+    /**
+     * An accessor to the raw value of the channel bytes output. This method will not instantiate
+     * a new stream if one does not exist and this method will not reset the stream.
+     */
+    protected final BytesStreamOutput bytesOutputOrNull() {
+        return bytesOut;
+    }
+
     protected BytesStreamOutput newBytesOutput() {
         return new BytesStreamOutput();
     }
@@ -117,5 +146,4 @@ public RestRequest request() {
     public boolean detailedErrorsEnabled() {
         return detailedErrorsEnabled;
     }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/BaseRestHandler.java b/core/src/main/java/org/elasticsearch/rest/BaseRestHandler.java
index 31e09d6706e04..81620ec8a7cf7 100644
--- a/core/src/main/java/org/elasticsearch/rest/BaseRestHandler.java
+++ b/core/src/main/java/org/elasticsearch/rest/BaseRestHandler.java
@@ -19,13 +19,28 @@
 
 package org.elasticsearch.rest;
 
-import org.elasticsearch.common.ParseFieldMatcher;
+import org.apache.lucene.search.spell.LevensteinDistance;
+import org.apache.lucene.util.CollectionUtil;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.CheckedConsumer;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.ActionPlugin;
 
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
 /**
  * Base handler for REST requests.
  * <p>
@@ -35,12 +50,110 @@
  * {@link ActionPlugin#getRestHeaders()}.
  */
 public abstract class BaseRestHandler extends AbstractComponent implements RestHandler {
+
     public static final Setting<Boolean> MULTI_ALLOW_EXPLICIT_INDEX =
         Setting.boolSetting("rest.action.multi.allow_explicit_index", true, Property.NodeScope);
-    protected final ParseFieldMatcher parseFieldMatcher;
 
     protected BaseRestHandler(Settings settings) {
         super(settings);
-        this.parseFieldMatcher = new ParseFieldMatcher(settings);
     }
+
+    @Override
+    public final void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
+        // prepare the request for execution; has the side effect of touching the request parameters
+        final RestChannelConsumer action = prepareRequest(request, client);
+
+        // validate unconsumed params, but we must exclude params used to format the response
+        // use a sorted set so the unconsumed parameters appear in a reliable sorted order
+        final SortedSet<String> unconsumedParams =
+            request.unconsumedParams().stream().filter(p -> !responseParams().contains(p)).collect(Collectors.toCollection(TreeSet::new));
+
+        // validate the non-response params
+        if (!unconsumedParams.isEmpty()) {
+            final Set<String> candidateParams = new HashSet<>();
+            candidateParams.addAll(request.consumedParams());
+            candidateParams.addAll(responseParams());
+            throw new IllegalArgumentException(unrecognized(request, unconsumedParams, candidateParams, "parameter"));
+        }
+
+        // execute the action
+        action.accept(channel);
+    }
+
+    protected final String unrecognized(
+        final RestRequest request,
+        final Set<String> invalids,
+        final Set<String> candidates,
+        final String detail) {
+        String message = String.format(
+            Locale.ROOT,
+            "request [%s] contains unrecognized %s%s: ",
+            request.path(),
+            detail,
+            invalids.size() > 1 ? "s" : "");
+        boolean first = true;
+        for (final String invalid : invalids) {
+            final LevensteinDistance ld = new LevensteinDistance();
+            final List<Tuple<Float, String>> scoredParams = new ArrayList<>();
+            for (final String candidate : candidates) {
+                final float distance = ld.getDistance(invalid, candidate);
+                if (distance > 0.5f) {
+                    scoredParams.add(new Tuple<>(distance, candidate));
+                }
+            }
+            CollectionUtil.timSort(scoredParams, (a, b) -> {
+                // sort by distance in reverse order, then parameter name for equal distances
+                int compare = a.v1().compareTo(b.v1());
+                if (compare != 0) return -compare;
+                else return a.v2().compareTo(b.v2());
+            });
+            if (first == false) {
+                message += ", ";
+            }
+            message += "[" + invalid + "]";
+            final List<String> keys = scoredParams.stream().map(Tuple::v2).collect(Collectors.toList());
+            if (keys.isEmpty() == false) {
+                message += " -> did you mean " + (keys.size() == 1 ? "[" + keys.get(0) + "]" : "any of " + keys.toString()) + "?";
+            }
+            first = false;
+        }
+
+        return message;
+    }
+
+    /**
+     * REST requests are handled by preparing a channel consumer that represents the execution of
+     * the request against a channel.
+     */
+    @FunctionalInterface
+    protected interface RestChannelConsumer extends CheckedConsumer<RestChannel, Exception> {
+    }
+
+    /**
+     * Prepare the request for execution. Implementations should consume all request params before
+     * returning the runnable for actual execution. Unconsumed params will immediately terminate
+     * execution of the request. However, some params are only used in processing the response;
+     * implementations can override {@link BaseRestHandler#responseParams()} to indicate such
+     * params.
+     *
+     * @param request the request to execute
+     * @param client  client for executing actions on the local node
+     * @return the action to execute
+     * @throws IOException if an I/O exception occurred parsing the request and preparing for
+     *                     execution
+     */
+    protected abstract RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException;
+
+    /**
+     * Parameters used for controlling the response and thus might not be consumed during
+     * preparation of the request execution in
+     * {@link BaseRestHandler#prepareRequest(RestRequest, NodeClient)}.
+     *
+     * @return a set of parameters used to control the response and thus should not trip strict
+     * URL parameter checks.
+     */
+    protected Set<String> responseParams() {
+        return Collections.emptySet();
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/BytesRestResponse.java b/core/src/main/java/org/elasticsearch/rest/BytesRestResponse.java
index 7af8249bf2ed0..11daaddd14720 100644
--- a/core/src/main/java/org/elasticsearch/rest/BytesRestResponse.java
+++ b/core/src/main/java/org/elasticsearch/rest/BytesRestResponse.java
@@ -23,21 +23,29 @@
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
-import java.util.Collections;
+
+import static java.util.Collections.singletonMap;
+import static org.elasticsearch.ElasticsearchException.REST_EXCEPTION_SKIP_STACK_TRACE;
+import static org.elasticsearch.ElasticsearchException.REST_EXCEPTION_SKIP_STACK_TRACE_DEFAULT;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
 
 
 public class BytesRestResponse extends RestResponse {
 
     public static final String TEXT_CONTENT_TYPE = "text/plain; charset=UTF-8";
 
+    private static final String STATUS = "status";
+
     private final RestStatus status;
     private final BytesReference content;
     private final String contentType;
@@ -85,11 +93,7 @@ public BytesRestResponse(RestChannel channel, Exception e) throws IOException {
 
     public BytesRestResponse(RestChannel channel, RestStatus status, Exception e) throws IOException {
         this.status = status;
-        if (channel.request().method() == RestRequest.Method.HEAD) {
-            this.content = BytesArray.EMPTY;
-            this.contentType = TEXT_CONTENT_TYPE;
-        } else {
-            XContentBuilder builder = convert(channel, status, e);
+        try (XContentBuilder builder = build(channel, status, e)) {
             this.content = builder.bytes();
             this.contentType = builder.contentType().mediaType();
         }
@@ -115,57 +119,68 @@ public RestStatus status() {
 
     private static final Logger SUPPRESSED_ERROR_LOGGER = ESLoggerFactory.getLogger("rest.suppressed");
 
-    private static XContentBuilder convert(RestChannel channel, RestStatus status, Exception e) throws IOException {
-        XContentBuilder builder = channel.newErrorBuilder().startObject();
-        if (e == null) {
-            builder.field("error", "unknown");
-        } else if (channel.detailedErrorsEnabled()) {
-            final ToXContent.Params params;
-            if (channel.request().paramAsBoolean("error_trace", !ElasticsearchException.REST_EXCEPTION_SKIP_STACK_TRACE_DEFAULT)) {
-                params =  new ToXContent.DelegatingMapParams(Collections.singletonMap(ElasticsearchException.REST_EXCEPTION_SKIP_STACK_TRACE, "false"), channel.request());
+    private static XContentBuilder build(RestChannel channel, RestStatus status, Exception e) throws IOException {
+        ToXContent.Params params = channel.request();
+        if (params.paramAsBoolean("error_trace", !REST_EXCEPTION_SKIP_STACK_TRACE_DEFAULT)) {
+            params =  new ToXContent.DelegatingMapParams(singletonMap(REST_EXCEPTION_SKIP_STACK_TRACE, "false"), params);
+        } else if (e != null) {
+            Supplier<?> messageSupplier = () -> new ParameterizedMessage("path: {}, params: {}",
+                    channel.request().rawPath(), channel.request().params());
+
+            if (status.getStatus() < 500) {
+                SUPPRESSED_ERROR_LOGGER.debug(messageSupplier, e);
             } else {
-                if (status.getStatus() < 500) {
-                    SUPPRESSED_ERROR_LOGGER.debug((Supplier<?>) () -> new ParameterizedMessage("path: {}, params: {}", channel.request().rawPath(), channel.request().params()), e);
-                } else {
-                    SUPPRESSED_ERROR_LOGGER.warn((Supplier<?>) () -> new ParameterizedMessage("path: {}, params: {}", channel.request().rawPath(), channel.request().params()), e);
-                }
-                params = channel.request();
-            }
-            builder.field("error");
-            builder.startObject();
-            final ElasticsearchException[] rootCauses = ElasticsearchException.guessRootCauses(e);
-            builder.field("root_cause");
-            builder.startArray();
-            for (ElasticsearchException rootCause : rootCauses){
-                builder.startObject();
-                rootCause.toXContent(builder, new ToXContent.DelegatingMapParams(Collections.singletonMap(ElasticsearchException.REST_EXCEPTION_SKIP_CAUSE, "true"), params));
-                builder.endObject();
+                SUPPRESSED_ERROR_LOGGER.warn(messageSupplier, e);
             }
-            builder.endArray();
-
-            ElasticsearchException.toXContent(builder, params, e);
-            builder.endObject();
-        } else {
-            builder.field("error", simpleMessage(e));
         }
-        builder.field("status", status.getStatus());
+
+        XContentBuilder builder = channel.newErrorBuilder().startObject();
+        ElasticsearchException.generateFailureXContent(builder, params, e, channel.detailedErrorsEnabled());
+        builder.field(STATUS, status.getStatus());
         builder.endObject();
         return builder;
     }
 
-    /*
-     * Builds a simple error string from the message of the first ElasticsearchException
-     */
-    private static String simpleMessage(Throwable t) throws IOException {
-        int counter = 0;
-        Throwable next = t;
-        while (next != null && counter++ < 10) {
-            if (t instanceof ElasticsearchException) {
-                return next.getClass().getSimpleName() + "[" + next.getMessage() + "]";
+    static BytesRestResponse createSimpleErrorResponse(RestChannel channel, RestStatus status, String errorMessage) throws IOException {
+        return new BytesRestResponse(status, channel.newErrorBuilder().startObject()
+            .field("error", errorMessage)
+            .field("status", status.getStatus())
+            .endObject());
+    }
+
+    public static ElasticsearchStatusException errorFromXContent(XContentParser parser) throws IOException {
+        XContentParser.Token token = parser.nextToken();
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser::getTokenLocation);
+
+        ElasticsearchException exception = null;
+        RestStatus status = null;
+
+        String currentFieldName = null;
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            }
+            if (STATUS.equals(currentFieldName)) {
+                if (token != XContentParser.Token.FIELD_NAME) {
+                    ensureExpectedToken(XContentParser.Token.VALUE_NUMBER, token, parser::getTokenLocation);
+                    status = RestStatus.fromCode(parser.intValue());
+                }
+            } else {
+                exception = ElasticsearchException.failureFromXContent(parser);
             }
-            next = next.getCause();
         }
 
-        return "No ElasticsearchException found";
+        if (exception == null) {
+            throw new IllegalStateException("Failed to parse elasticsearch status exception: no exception was found");
+        }
+
+        ElasticsearchStatusException result = new ElasticsearchStatusException(exception.getMessage(), status, exception.getCause());
+        for (String header : exception.getHeaderKeys()) {
+            result.addHeader(header, exception.getHeader(header));
+        }
+        for (String metadata : exception.getMetadataKeys()) {
+            result.addMetadata(metadata, exception.getMetadata(metadata));
+        }
+        return result;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/RestChannel.java b/core/src/main/java/org/elasticsearch/rest/RestChannel.java
index 2a56313fd8a79..8c8346f0ef4b2 100644
--- a/core/src/main/java/org/elasticsearch/rest/RestChannel.java
+++ b/core/src/main/java/org/elasticsearch/rest/RestChannel.java
@@ -20,9 +20,9 @@
 package org.elasticsearch.rest;
 
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentType;
 
 import java.io.IOException;
 
@@ -35,7 +35,7 @@ public interface RestChannel {
 
     XContentBuilder newErrorBuilder() throws IOException;
 
-    XContentBuilder newBuilder(@Nullable BytesReference autoDetectSource, boolean useFiltering) throws IOException;
+    XContentBuilder newBuilder(@Nullable XContentType xContentType, boolean useFiltering) throws IOException;
 
     BytesStreamOutput bytesOutput();
 
@@ -47,5 +47,4 @@ public interface RestChannel {
     boolean detailedErrorsEnabled();
 
     void sendResponse(RestResponse response);
-
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/RestController.java b/core/src/main/java/org/elasticsearch/rest/RestController.java
index e63f35884e88c..42db6e8adae03 100644
--- a/core/src/main/java/org/elasticsearch/rest/RestController.java
+++ b/core/src/main/java/org/elasticsearch/rest/RestController.java
@@ -20,29 +20,45 @@
 package org.elasticsearch.rest;
 
 import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.apache.logging.log4j.util.Supplier;
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.component.AbstractLifecycleComponent;
+import org.elasticsearch.common.component.AbstractComponent;
+import org.elasticsearch.common.io.Streams;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.path.PathTrie;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.http.HttpServerTransport;
+import org.elasticsearch.http.HttpTransportSettings;
+import org.elasticsearch.indices.breaker.CircuitBreakerService;
 
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
-import java.util.Arrays;
+import java.io.InputStream;
+import java.util.List;
+import java.util.Locale;
+import java.util.Objects;
 import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Supplier;
+import java.util.function.UnaryOperator;
 
 import static org.elasticsearch.rest.RestStatus.BAD_REQUEST;
+import static org.elasticsearch.rest.RestStatus.FORBIDDEN;
+import static org.elasticsearch.rest.RestStatus.INTERNAL_SERVER_ERROR;
+import static org.elasticsearch.rest.RestStatus.NOT_ACCEPTABLE;
 import static org.elasticsearch.rest.RestStatus.OK;
 
-/**
- *
- */
-public class RestController extends AbstractLifecycleComponent {
+public class RestController extends AbstractComponent implements HttpServerTransport.Dispatcher {
+
     private final PathTrie<RestHandler> getHandlers = new PathTrie<>(RestUtils.REST_DECODER);
     private final PathTrie<RestHandler> postHandlers = new PathTrie<>(RestUtils.REST_DECODER);
     private final PathTrie<RestHandler> putHandlers = new PathTrie<>(RestUtils.REST_DECODER);
@@ -50,43 +66,31 @@ public class RestController extends AbstractLifecycleComponent {
     private final PathTrie<RestHandler> headHandlers = new PathTrie<>(RestUtils.REST_DECODER);
     private final PathTrie<RestHandler> optionsHandlers = new PathTrie<>(RestUtils.REST_DECODER);
 
-    private final RestHandlerFilter handlerFilter = new RestHandlerFilter();
+    private final UnaryOperator<RestHandler> handlerWrapper;
+
+    private final NodeClient client;
+
+    private final CircuitBreakerService circuitBreakerService;
 
     /** Rest headers that are copied to internal requests made during a rest request. */
     private final Set<String> headersToCopy;
 
-    // non volatile since the assumption is that pre processors are registered on startup
-    private RestFilter[] filters = new RestFilter[0];
+    private final boolean isContentTypeRequired;
+
+    private final DeprecationLogger deprecationLogger;
 
-    public RestController(Settings settings, Set<String> headersToCopy) {
+    public RestController(Settings settings, Set<String> headersToCopy, UnaryOperator<RestHandler> handlerWrapper,
+                          NodeClient client, CircuitBreakerService circuitBreakerService) {
         super(settings);
         this.headersToCopy = headersToCopy;
-    }
-
-    @Override
-    protected void doStart() {
-    }
-
-    @Override
-    protected void doStop() {
-    }
-
-    @Override
-    protected void doClose() {
-        for (RestFilter filter : filters) {
-            filter.close();
+        if (handlerWrapper == null) {
+            handlerWrapper = h -> h; // passthrough if no wrapper set
         }
-    }
-
-    /**
-     * Registers a pre processor to be executed before the rest request is actually handled.
-     */
-    public synchronized void registerFilter(RestFilter preProcessor) {
-        RestFilter[] copy = new RestFilter[filters.length + 1];
-        System.arraycopy(filters, 0, copy, 0, filters.length);
-        copy[filters.length] = preProcessor;
-        Arrays.sort(copy, (o1, o2) -> Integer.compare(o1.order(), o2.order()));
-        filters = copy;
+        this.handlerWrapper = handlerWrapper;
+        this.client = client;
+        this.circuitBreakerService = circuitBreakerService;
+        this.isContentTypeRequired = HttpTransportSettings.SETTING_HTTP_CONTENT_TYPE_REQUIRED.get(settings);
+        this.deprecationLogger = new DeprecationLogger(logger);
     }
 
     /**
@@ -157,25 +161,6 @@ public void registerHandler(RestRequest.Method method, String path, RestHandler
         }
     }
 
-    /**
-     * Returns a filter chain (if needed) to execute. If this method returns null, simply execute
-     * as usual.
-     */
-    @Nullable
-    public RestFilterChain filterChainOrNull(RestFilter executionFilter) {
-        if (filters.length == 0) {
-            return null;
-        }
-        return new ControllerFilterChain(executionFilter);
-    }
-
-    /**
-     * Returns a filter chain with the final filter being the provided filter.
-     */
-    public RestFilterChain filterChain(RestFilter executionFilter) {
-        return new ControllerFilterChain(executionFilter);
-    }
-
     /**
      * @param request The current request. Must not be null.
      * @return true iff the circuit breaker limit must be enforced for processing this request.
@@ -185,33 +170,167 @@ public boolean canTripCircuitBreaker(RestRequest request) {
         return (handler != null) ? handler.canTripCircuitBreaker() : true;
     }
 
-    public void dispatchRequest(final RestRequest request, final RestChannel channel, final NodeClient client, ThreadContext threadContext) throws Exception {
-        if (!checkRequestParameters(request, channel)) {
+    @Override
+    public void dispatchRequest(RestRequest request, RestChannel channel, ThreadContext threadContext) {
+        if (request.rawPath().equals("/favicon.ico")) {
+            handleFavicon(request, channel);
             return;
         }
-        try (ThreadContext.StoredContext ignored = threadContext.stashContext()) {
+        RestChannel responseChannel = channel;
+        try {
+            final int contentLength = request.hasContent() ? request.content().length() : 0;
+            assert contentLength >= 0 : "content length was negative, how is that possible?";
+            final RestHandler handler = getHandler(request);
+
+            if (contentLength > 0 && hasContentTypeOrCanAutoDetect(request, handler) == false) {
+                sendContentTypeErrorMessage(request, responseChannel);
+            } else if (contentLength > 0 && handler != null && handler.supportsContentStream() &&
+                request.getXContentType() != XContentType.JSON && request.getXContentType() != XContentType.SMILE) {
+                responseChannel.sendResponse(BytesRestResponse.createSimpleErrorResponse(responseChannel,
+                    RestStatus.NOT_ACCEPTABLE, "Content-Type [" + request.getXContentType() +
+                        "] does not support stream parsing. Use JSON or SMILE instead"));
+            } else {
+                if (canTripCircuitBreaker(request)) {
+                    inFlightRequestsBreaker(circuitBreakerService).addEstimateBytesAndMaybeBreak(contentLength, "<http_request>");
+                } else {
+                    inFlightRequestsBreaker(circuitBreakerService).addWithoutBreaking(contentLength);
+                }
+                // iff we could reserve bytes for the request we need to send the response also over this channel
+                responseChannel = new ResourceHandlingHttpChannel(channel, circuitBreakerService, contentLength);
+                dispatchRequest(request, responseChannel, client, threadContext, handler);
+            }
+        } catch (Exception e) {
+            try {
+                responseChannel.sendResponse(new BytesRestResponse(channel, e));
+            } catch (Exception inner) {
+                inner.addSuppressed(e);
+                logger.error((Supplier<?>) () ->
+                    new ParameterizedMessage("failed to send failure response for uri [{}]", request.uri()), inner);
+            }
+        }
+    }
+
+    @Override
+    public void dispatchBadRequest(
+            final RestRequest request,
+            final RestChannel channel,
+            final ThreadContext threadContext,
+            final Throwable cause) {
+        try {
+            final Exception e;
+            if (cause == null) {
+                e = new ElasticsearchException("unknown cause");
+            } else if (cause instanceof Exception) {
+                e = (Exception) cause;
+            } else {
+                e = new ElasticsearchException(cause);
+            }
+            channel.sendResponse(new BytesRestResponse(channel, BAD_REQUEST, e));
+        } catch (final IOException e) {
+            if (cause != null) {
+                e.addSuppressed(cause);
+            }
+            logger.warn("failed to send bad request response", e);
+            channel.sendResponse(new BytesRestResponse(INTERNAL_SERVER_ERROR, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY));
+        }
+    }
+
+    void dispatchRequest(final RestRequest request, final RestChannel channel, final NodeClient client, ThreadContext threadContext,
+                         final RestHandler handler) throws Exception {
+        if (checkRequestParameters(request, channel) == false) {
+            channel
+                .sendResponse(BytesRestResponse.createSimpleErrorResponse(channel,BAD_REQUEST, "error traces in responses are disabled."));
+        } else {
             for (String key : headersToCopy) {
                 String httpHeader = request.header(key);
                 if (httpHeader != null) {
                     threadContext.putHeader(key, httpHeader);
                 }
             }
-            if (filters.length == 0) {
-                executeHandler(request, channel, client);
+
+            if (handler == null) {
+                if (request.method() == RestRequest.Method.OPTIONS) {
+                    // when we routinghave OPTIONS request, simply send OK by default (with the Access Control Origin header which gets automatically added)
+
+                    channel.sendResponse(new BytesRestResponse(OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY));
+                } else {
+                    final String msg = "No handler found for uri [" + request.uri() + "] and method [" + request.method() + "]";
+                    channel.sendResponse(new BytesRestResponse(BAD_REQUEST, msg));
+                }
             } else {
-                ControllerFilterChain filterChain = new ControllerFilterChain(handlerFilter);
-                filterChain.continueProcessing(request, channel, client);
+                final RestHandler wrappedHandler = Objects.requireNonNull(handlerWrapper.apply(handler));
+                wrappedHandler.handleRequest(request, channel, client);
             }
         }
     }
 
-    public void sendErrorResponse(RestRequest request, RestChannel channel, Exception e) {
-        try {
-            channel.sendResponse(new BytesRestResponse(channel, e));
-        } catch (Exception inner) {
-            inner.addSuppressed(e);
-            logger.error((Supplier<?>) () -> new ParameterizedMessage("failed to send failure response for uri [{}]", request.uri()), inner);
+    /**
+     * If a request contains content, this method will return {@code true} if the {@code Content-Type} header is present, matches an
+     * {@link XContentType} or the request is plain text, and content type is required. If content type is not required then this method
+     * returns true unless a content type could not be inferred from the body and the rest handler does not support plain text
+     */
+    private boolean hasContentTypeOrCanAutoDetect(final RestRequest restRequest, final RestHandler restHandler) {
+        if (restRequest.getXContentType() == null) {
+            if (restHandler != null && restHandler.supportsPlainText()) {
+                if (isContentTypeRequired) {
+                    // content type of null with a handler that supports plain text gets through for now. Once we remove plain text this can
+                    // be removed!
+                    deprecationLogger.deprecated("Plain text request bodies are deprecated. Use request parameters or body " +
+                        "in a supported format.");
+                } else {
+                    // attempt to autodetect since we do not know that it is truly plain-text
+                    final boolean detected = autoDetectXContentType(restRequest);
+                    if (detected == false) {
+                        deprecationLogger.deprecated("Plain text request bodies are deprecated. Use request parameters or body " +
+                            "in a supported format.");
+                    }
+                }
+            } else if (restHandler != null && restHandler.supportsContentStream() && restRequest.header("Content-Type") != null) {
+                final String lowercaseMediaType = restRequest.header("Content-Type").toLowerCase(Locale.ROOT);
+                // we also support newline delimited JSON: http://specs.okfnlabs.org/ndjson/
+                if (lowercaseMediaType.equals("application/x-ndjson")) {
+                    restRequest.setXContentType(XContentType.JSON);
+                } else if (lowercaseMediaType.equals("application/x-ldjson")) {
+                    restRequest.setXContentType(XContentType.JSON);
+                    deprecationLogger.deprecated("The Content-Type [application/x-ldjson] has been superseded by [application/x-ndjson] " +
+                        "in the specification and should be used instead.");
+                } else if (isContentTypeRequired) {
+                    return false;
+                } else {
+                    return autoDetectXContentType(restRequest);
+                }
+            } else if (isContentTypeRequired) {
+                return false;
+            } else {
+                return autoDetectXContentType(restRequest);
+            }
         }
+        return true;
+    }
+
+    private boolean autoDetectXContentType(RestRequest restRequest) {
+        deprecationLogger.deprecated("Content type detection for rest requests is deprecated. Specify the content type using " +
+            "the [Content-Type] header.");
+        XContentType xContentType = XContentFactory.xContentType(restRequest.content());
+        if (xContentType == null) {
+            return false;
+        } else {
+            restRequest.setXContentType(xContentType);
+        }
+        return true;
+    }
+
+    private void sendContentTypeErrorMessage(RestRequest restRequest, RestChannel channel) throws IOException {
+        final List<String> contentTypeHeader = restRequest.getAllHeaderValues("Content-Type");
+        final String errorMessage;
+        if (contentTypeHeader == null) {
+            errorMessage = "Content-Type header is missing";
+        } else {
+            errorMessage = "Content-Type header [" +
+                Strings.collectionToCommaDelimitedString(restRequest.getAllHeaderValues("Content-Type")) + "] is not supported";
+        }
+
+        channel.sendResponse(BytesRestResponse.createSimpleErrorResponse(channel, NOT_ACCEPTABLE, errorMessage));
     }
 
     /**
@@ -220,37 +339,14 @@ public void sendErrorResponse(RestRequest request, RestChannel channel, Exceptio
      */
     boolean checkRequestParameters(final RestRequest request, final RestChannel channel) {
         // error_trace cannot be used when we disable detailed errors
-        if (channel.detailedErrorsEnabled() == false && request.paramAsBoolean("error_trace", false)) {
-            try {
-                XContentBuilder builder = channel.newErrorBuilder();
-                builder.startObject().field("error","error traces in responses are disabled.").endObject().string();
-                RestResponse response = new BytesRestResponse(BAD_REQUEST, builder);
-                response.addHeader("Content-Type", "application/json");
-                channel.sendResponse(response);
-            } catch (IOException e) {
-                logger.warn("Failed to send response", e);
-            }
+        // we consume the error_trace parameter first to ensure that it is always consumed
+        if (request.paramAsBoolean("error_trace", false) && channel.detailedErrorsEnabled() == false) {
             return false;
         }
 
         return true;
     }
 
-    void executeHandler(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
-        final RestHandler handler = getHandler(request);
-        if (handler != null) {
-            handler.handleRequest(request, channel, client);
-        } else {
-            if (request.method() == RestRequest.Method.OPTIONS) {
-                // when we have OPTIONS request, simply send OK by default (with the Access Control Origin header which gets automatically added)
-                channel.sendResponse(new BytesRestResponse(OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY));
-            } else {
-                final String msg = "No handler found for uri [" + request.uri() + "] and method [" + request.method() + "]";
-                channel.sendResponse(new BytesRestResponse(BAD_REQUEST, msg));
-            }
-        }
-    }
-
     private RestHandler getHandler(RestRequest request) {
         String path = getPath(request);
         PathTrie<RestHandler> handlers = getHandlersForMethod(request.method());
@@ -286,43 +382,84 @@ private String getPath(RestRequest request) {
         return request.rawPath();
     }
 
-    class ControllerFilterChain implements RestFilterChain {
+    void handleFavicon(RestRequest request, RestChannel channel) {
+        if (request.method() == RestRequest.Method.GET) {
+            try {
+                try (InputStream stream = getClass().getResourceAsStream("/config/favicon.ico")) {
+                    ByteArrayOutputStream out = new ByteArrayOutputStream();
+                    Streams.copy(stream, out);
+                    BytesRestResponse restResponse = new BytesRestResponse(RestStatus.OK, "image/x-icon", out.toByteArray());
+                    channel.sendResponse(restResponse);
+                }
+            } catch (IOException e) {
+                channel.sendResponse(new BytesRestResponse(INTERNAL_SERVER_ERROR, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY));
+            }
+        } else {
+            channel.sendResponse(new BytesRestResponse(FORBIDDEN, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY));
+        }
+    }
+
+    private static final class ResourceHandlingHttpChannel implements RestChannel {
+        private final RestChannel delegate;
+        private final CircuitBreakerService circuitBreakerService;
+        private final int contentLength;
+        private final AtomicBoolean closed = new AtomicBoolean();
 
-        private final RestFilter executionFilter;
+        ResourceHandlingHttpChannel(RestChannel delegate, CircuitBreakerService circuitBreakerService, int contentLength) {
+            this.delegate = delegate;
+            this.circuitBreakerService = circuitBreakerService;
+            this.contentLength = contentLength;
+        }
 
-        private final AtomicInteger index = new AtomicInteger();
+        @Override
+        public XContentBuilder newBuilder() throws IOException {
+            return delegate.newBuilder();
+        }
 
-        ControllerFilterChain(RestFilter executionFilter) {
-            this.executionFilter = executionFilter;
+        @Override
+        public XContentBuilder newErrorBuilder() throws IOException {
+            return delegate.newErrorBuilder();
         }
 
         @Override
-        public void continueProcessing(RestRequest request, RestChannel channel, NodeClient client) {
-            try {
-                int loc = index.getAndIncrement();
-                if (loc > filters.length) {
-                    throw new IllegalStateException("filter continueProcessing was called more than expected");
-                } else if (loc == filters.length) {
-                    executionFilter.process(request, channel, client, this);
-                } else {
-                    RestFilter preProcessor = filters[loc];
-                    preProcessor.process(request, channel, client, this);
-                }
-            } catch (Exception e) {
-                try {
-                    channel.sendResponse(new BytesRestResponse(channel, e));
-                } catch (IOException e1) {
-                    logger.error((Supplier<?>) () -> new ParameterizedMessage("Failed to send failure response for uri [{}]", request.uri()), e1);
-                }
-            }
+        public XContentBuilder newBuilder(@Nullable XContentType xContentType, boolean useFiltering) throws IOException {
+            return delegate.newBuilder(xContentType, useFiltering);
         }
-    }
 
-    class RestHandlerFilter extends RestFilter {
+        @Override
+        public BytesStreamOutput bytesOutput() {
+            return delegate.bytesOutput();
+        }
+
+        @Override
+        public RestRequest request() {
+            return delegate.request();
+        }
+
+        @Override
+        public boolean detailedErrorsEnabled() {
+            return delegate.detailedErrorsEnabled();
+        }
 
         @Override
-        public void process(RestRequest request, RestChannel channel, NodeClient client, RestFilterChain filterChain) throws Exception {
-            executeHandler(request, channel, client);
+        public void sendResponse(RestResponse response) {
+            close();
+            delegate.sendResponse(response);
         }
+
+        private void close() {
+            // attempt to close once atomically
+            if (closed.compareAndSet(false, true) == false) {
+                throw new IllegalStateException("Channel is already closed");
+            }
+            inFlightRequestsBreaker(circuitBreakerService).addWithoutBreaking(-contentLength);
+        }
+
     }
+
+    private static CircuitBreaker inFlightRequestsBreaker(CircuitBreakerService circuitBreakerService) {
+        // We always obtain a fresh breaker to reflect changes to the breaker configuration.
+        return circuitBreakerService.getBreaker(CircuitBreaker.IN_FLIGHT_REQUESTS);
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/RestFilter.java b/core/src/main/java/org/elasticsearch/rest/RestFilter.java
deleted file mode 100644
index 276e99fc7e510..0000000000000
--- a/core/src/main/java/org/elasticsearch/rest/RestFilter.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.rest;
-
-import java.io.Closeable;
-
-import org.elasticsearch.client.node.NodeClient;
-
-/**
- * A filter allowing to filter rest operations.
- */
-public abstract class RestFilter implements Closeable {
-
-    /**
-     * Optionally, the order of the filter. Execution is done from lowest value to highest.
-     * It is a good practice to allow to configure this for the relevant filter.
-     */
-    public int order() {
-        return 0;
-    }
-
-    @Override
-    public void close() {
-        // a no op
-    }
-
-    /**
-     * Process the rest request. Using the channel to send a response, or the filter chain to continue
-     * processing the request.
-     */
-    public abstract void process(RestRequest request, RestChannel channel, NodeClient client, RestFilterChain filterChain) throws Exception;
-}
diff --git a/core/src/main/java/org/elasticsearch/rest/RestFilterChain.java b/core/src/main/java/org/elasticsearch/rest/RestFilterChain.java
deleted file mode 100644
index 239a6c6b1bbbc..0000000000000
--- a/core/src/main/java/org/elasticsearch/rest/RestFilterChain.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.rest;
-
-import org.elasticsearch.client.node.NodeClient;
-
-/**
- * A filter chain allowing to continue and process the rest request.
- */
-public interface RestFilterChain {
-
-    /**
-     * Continue processing the request. Should only be called if a response has not been sent
-     * through the channel.
-     */
-    void continueProcessing(RestRequest request, RestChannel channel, NodeClient client);
-}
diff --git a/core/src/main/java/org/elasticsearch/rest/RestHandler.java b/core/src/main/java/org/elasticsearch/rest/RestHandler.java
index 393e425baf937..215541b40e87f 100644
--- a/core/src/main/java/org/elasticsearch/rest/RestHandler.java
+++ b/core/src/main/java/org/elasticsearch/rest/RestHandler.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.rest;
 
 import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.xcontent.XContent;
 
 /**
  * Handler for REST requests
@@ -28,7 +29,6 @@ public interface RestHandler {
 
     /**
      * Handles a rest request.
-     *
      * @param request The request to handle
      * @param channel The channel to write the request response to
      * @param client A client to use to make internal requests on behalf of the original request
@@ -38,4 +38,22 @@ public interface RestHandler {
     default boolean canTripCircuitBreaker() {
         return true;
     }
+
+    /**
+     * Indicates if a RestHandler supports plain text bodies
+     * @deprecated use request parameters or bodies that can be parsed with XContent!
+     */
+    @Deprecated
+    default boolean supportsPlainText() {
+        return false;
+    }
+
+    /**
+     * Indicates if the RestHandler supports content as a stream. A stream would be multiple objects delineated by
+     * {@link XContent#streamSeparator()}. If a handler returns true this will affect the types of content that can be sent to
+     * this endpoint.
+     */
+    default boolean supportsContentStream() {
+        return false;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/RestRequest.java b/core/src/main/java/org/elasticsearch/rest/RestRequest.java
index 2db917dacf0bb..c8a7efa34d2af 100644
--- a/core/src/main/java/org/elasticsearch/rest/RestRequest.java
+++ b/core/src/main/java/org/elasticsearch/rest/RestRequest.java
@@ -19,28 +19,61 @@
 
 package org.elasticsearch.rest;
 
+import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.Booleans;
+import org.elasticsearch.common.CheckedConsumer;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
 
+import java.io.IOException;
 import java.net.SocketAddress;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
 import static org.elasticsearch.common.unit.ByteSizeValue.parseBytesSizeValue;
 import static org.elasticsearch.common.unit.TimeValue.parseTimeValue;
 
 public abstract class RestRequest implements ToXContent.Params {
 
+    private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(RestRequest.class));
+    // tchar pattern as defined by RFC7230 section 3.2.6
+    private static final Pattern TCHAR_PATTERN = Pattern.compile("[a-zA-z0-9!#$%&'*+\\-.\\^_`|~]+");
+
+    private final NamedXContentRegistry xContentRegistry;
     private final Map<String, String> params;
+    private final Map<String, List<String>> headers;
     private final String rawPath;
+    private final Set<String> consumedParams = new HashSet<>();
+    private final SetOnce<XContentType> xContentType = new SetOnce<>();
 
-    public RestRequest(String uri) {
+    /**
+     * Creates a new RestRequest
+     * @param xContentRegistry the xContentRegistry to use when parsing XContent
+     * @param uri the URI of the request that potentially contains request parameters
+     * @param headers a map of the headers. This map should implement a Case-Insensitive hashing for keys as HTTP header names are case
+     *                insensitive
+     */
+    public RestRequest(NamedXContentRegistry xContentRegistry, String uri, Map<String, List<String>> headers) {
+        this.xContentRegistry = xContentRegistry;
         final Map<String, String> params = new HashMap<>();
         int pathEndPos = uri.indexOf('?');
         if (pathEndPos < 0) {
@@ -50,11 +83,32 @@ public RestRequest(String uri) {
             RestUtils.decodeQueryString(uri, pathEndPos + 1, params);
         }
         this.params = params;
+        this.headers = Collections.unmodifiableMap(headers);
+        final List<String> contentType = getAllHeaderValues("Content-Type");
+        final XContentType xContentType = parseContentType(contentType);
+        if (xContentType != null) {
+            this.xContentType.set(xContentType);
+        }
     }
 
-    public RestRequest(Map<String, String> params, String path) {
+    /**
+     * Creates a new RestRequest
+     * @param xContentRegistry the xContentRegistry to use when parsing XContent
+     * @param params the parameters of the request
+     * @param path the path of the request. This should not contain request parameters
+     * @param headers a map of the headers. This map should implement a Case-Insensitive hashing for keys as HTTP header names are case
+     *                insensitive
+     */
+    public RestRequest(NamedXContentRegistry xContentRegistry, Map<String, String> params, String path, Map<String, List<String>> headers) {
+        this.xContentRegistry = xContentRegistry;
         this.params = params;
         this.rawPath = path;
+        this.headers = Collections.unmodifiableMap(headers);
+        final List<String> contentType = getAllHeaderValues("Content-Type");
+        final XContentType xContentType = parseContentType(contentType);
+        if (xContentType != null) {
+            this.xContentType.set(xContentType);
+        }
     }
 
     public enum Method {
@@ -86,9 +140,63 @@ public final String path() {
 
     public abstract BytesReference content();
 
-    public abstract String header(String name);
+    /**
+     * @return content of the request body or throw an exception if the body or content type is missing
+     */
+    public final BytesReference requiredContent() {
+        if (hasContent() == false) {
+            throw new ElasticsearchParseException("request body is required");
+        } else if (xContentType.get() == null) {
+            throw new IllegalStateException("unknown content type");
+        }
+        return content();
+    }
+
+    /**
+     * Get the value of the header or {@code null} if not found. This method only retrieves the first header value if multiple values are
+     * sent. Use of {@link #getAllHeaderValues(String)} should be preferred
+     */
+    public final String header(String name) {
+        List<String> values = headers.get(name);
+        if (values != null && values.isEmpty() == false) {
+            return values.get(0);
+        }
+        return null;
+    }
+
+    /**
+     * Get all values for the header or {@code null} if the header was not found
+     */
+    public final List<String> getAllHeaderValues(String name) {
+        List<String> values = headers.get(name);
+        if (values != null) {
+            return Collections.unmodifiableList(values);
+        }
+        return null;
+    }
+
+    /**
+     * Get all of the headers and values associated with the headers. Modifications of this map are not supported.
+     */
+    public final Map<String, List<String>> getHeaders() {
+        return headers;
+    }
+
+    /**
+     * The {@link XContentType} that was parsed from the {@code Content-Type} header. This value will be {@code null} in the case of
+     * a request without a valid {@code Content-Type} header, a request without content ({@link #hasContent()}, or a plain text request
+     */
+    @Nullable
+    public final XContentType getXContentType() {
+        return xContentType.get();
+    }
 
-    public abstract Iterable<Map.Entry<String, String>> headers();
+    /**
+     * Sets the {@link XContentType}
+     */
+    final void setXContentType(XContentType xContentType) {
+        this.xContentType.set(xContentType);
+    }
 
     @Nullable
     public SocketAddress getRemoteAddress() {
@@ -106,11 +214,13 @@ public final boolean hasParam(String key) {
 
     @Override
     public final String param(String key) {
+        consumedParams.add(key);
         return params.get(key);
     }
 
     @Override
     public final String param(String key, String defaultValue) {
+        consumedParams.add(key);
         String value = params.get(key);
         if (value == null) {
             return defaultValue;
@@ -122,6 +232,30 @@ public Map<String, String> params() {
         return params;
     }
 
+    /**
+     * Returns a list of parameters that have been consumed. This method returns a copy, callers
+     * are free to modify the returned list.
+     *
+     * @return the list of currently consumed parameters.
+     */
+    List<String> consumedParams() {
+        return consumedParams.stream().collect(Collectors.toList());
+    }
+
+    /**
+     * Returns a list of parameters that have not yet been consumed. This method returns a copy,
+     * callers are free to modify the returned list.
+     *
+     * @return the list of currently unconsumed parameters.
+     */
+    List<String> unconsumedParams() {
+        return params
+            .keySet()
+            .stream()
+            .filter(p -> !consumedParams.contains(p))
+            .collect(Collectors.toList());
+    }
+
     public float paramAsFloat(String key, float defaultValue) {
         String sValue = param(key);
         if (sValue == null) {
@@ -160,12 +294,21 @@ public long paramAsLong(String key, long defaultValue) {
 
     @Override
     public boolean paramAsBoolean(String key, boolean defaultValue) {
-        return Booleans.parseBoolean(param(key), defaultValue);
+        return paramAsBoolean(key, (Boolean) defaultValue);
     }
 
     @Override
     public Boolean paramAsBoolean(String key, Boolean defaultValue) {
-        return Booleans.parseBoolean(param(key), defaultValue);
+        String rawParam = param(key);
+        // Treat empty string as true because that allows the presence of the url parameter to mean "turn this on"
+        if (rawParam != null && rawParam.length() == 0) {
+            return true;
+        } else {
+            if (rawParam != null && Booleans.isStrictlyBoolean(rawParam) == false) {
+                DEPRECATION_LOGGER.deprecated("Expected a boolean [true/false] for request parameter [{}] but got [{}]", key, rawParam);
+            }
+            return Booleans.parseBoolean(rawParam, defaultValue);
+        }
     }
 
     public TimeValue paramAsTime(String key, TimeValue defaultValue) {
@@ -192,4 +335,164 @@ public String[] paramAsStringArrayOrEmptyIfAll(String key) {
         return params;
     }
 
+    /**
+     * Get the {@link NamedXContentRegistry} that should be used to create parsers from this request.
+     */
+    public NamedXContentRegistry getXContentRegistry() {
+        return xContentRegistry;
+    }
+
+    /**
+     * A parser for the contents of this request if there is a body, otherwise throws an {@link ElasticsearchParseException}. Use
+     * {@link #applyContentParser(CheckedConsumer)} if you want to gracefully handle when the request doesn't have any contents. Use
+     * {@link #contentOrSourceParamParser()} for requests that support specifying the request body in the {@code source} param.
+     */
+    public final XContentParser contentParser() throws IOException {
+        BytesReference content = requiredContent(); // will throw exception if body or content type missing
+        return xContentType.get().xContent().createParser(xContentRegistry, content);
+    }
+
+    /**
+     * If there is any content then call {@code applyParser} with the parser, otherwise do nothing.
+     */
+    public final void applyContentParser(CheckedConsumer<XContentParser, IOException> applyParser) throws IOException {
+        if (hasContent()) {
+            try (XContentParser parser = contentParser()) {
+                applyParser.accept(parser);
+            }
+        }
+    }
+
+    /**
+     * Does this request have content or a {@code source} parameter? Use this instead of {@link #hasContent()} if this
+     * {@linkplain RestHandler} treats the {@code source} parameter like the body content.
+     */
+    public final boolean hasContentOrSourceParam() {
+        return hasContent() || hasParam("source");
+    }
+
+    /**
+     * A parser for the contents of this request if it has contents, otherwise a parser for the {@code source} parameter if there is one,
+     * otherwise throws an {@link ElasticsearchParseException}. Use {@link #withContentOrSourceParamParserOrNull(CheckedConsumer)} instead
+     * if you need to handle the absence request content gracefully.
+     */
+    public final XContentParser contentOrSourceParamParser() throws IOException {
+        Tuple<XContentType, BytesReference> tuple = contentOrSourceParam();
+        return tuple.v1().xContent().createParser(xContentRegistry, tuple.v2());
+    }
+
+    /**
+     * Call a consumer with the parser for the contents of this request if it has contents, otherwise with a parser for the {@code source}
+     * parameter if there is one, otherwise with {@code null}. Use {@link #contentOrSourceParamParser()} if you should throw an exception
+     * back to the user when there isn't request content.
+     */
+    public final void withContentOrSourceParamParserOrNull(CheckedConsumer<XContentParser, IOException> withParser) throws IOException {
+        if (hasContentOrSourceParam()) {
+            Tuple<XContentType, BytesReference> tuple = contentOrSourceParam();
+            BytesReference content = tuple.v2();
+            XContentType xContentType = tuple.v1();
+            try (XContentParser parser = xContentType.xContent().createParser(xContentRegistry, content)) {
+                withParser.accept(parser);
+            }
+        } else {
+            withParser.accept(null);
+        }
+    }
+
+    /**
+     * Get the content of the request or the contents of the {@code source} param or throw an exception if both are missing.
+     * Prefer {@link #contentOrSourceParamParser()} or {@link #withContentOrSourceParamParserOrNull(CheckedConsumer)} if you need a parser.
+     */
+    public final Tuple<XContentType, BytesReference> contentOrSourceParam() {
+        if (hasContentOrSourceParam() == false) {
+            throw new ElasticsearchParseException("request body or source parameter is required");
+        } else if (hasContent()) {
+            return new Tuple<>(xContentType.get(), requiredContent());
+        }
+        String source = param("source");
+        String typeParam = param("source_content_type");
+        BytesArray bytes = new BytesArray(source);
+        final XContentType xContentType;
+        if (typeParam != null) {
+            xContentType = parseContentType(Collections.singletonList(typeParam));
+        } else {
+            DEPRECATION_LOGGER.deprecated("Deprecated use of the [source] parameter without the [source_content_type] parameter. Use " +
+                "the [source_content_type] parameter to specify the content type of the source such as [application/json]");
+            xContentType = XContentFactory.xContentType(bytes);
+        }
+
+        if (xContentType == null) {
+            throw new IllegalStateException("could not determine source content type");
+        }
+        return new Tuple<>(xContentType, bytes);
+    }
+
+    /**
+     * Call a consumer with the parser for the contents of this request if it has contents, otherwise with a parser for the {@code source}
+     * parameter if there is one, otherwise with {@code null}. Use {@link #contentOrSourceParamParser()} if you should throw an exception
+     * back to the user when there isn't request content. This version allows for plain text content
+     */
+    @Deprecated
+    public final void withContentOrSourceParamParserOrNullLenient(CheckedConsumer<XContentParser, IOException> withParser)
+        throws IOException {
+        if (hasContentOrSourceParam() == false) {
+            withParser.accept(null);
+        } else if (hasContent() && xContentType.get() == null) {
+            withParser.accept(null);
+        } else {
+            Tuple<XContentType, BytesReference> tuple = contentOrSourceParam();
+            BytesReference content = tuple.v2();
+            XContentType xContentType = tuple.v1();
+            if (content.length() > 0) {
+                try (XContentParser parser = xContentType.xContent().createParser(xContentRegistry, content)) {
+                    withParser.accept(parser);
+                }
+            } else {
+                withParser.accept(null);
+            }
+        }
+    }
+
+    /**
+     * Get the content of the request or the contents of the {@code source} param without the xcontent type. This is useful the request can
+     * accept non xcontent values.
+     * @deprecated we should only take xcontent
+     */
+    @Deprecated
+    public final BytesReference getContentOrSourceParamOnly() {
+        if (hasContent()) {
+            return content();
+        }
+        String source = param("source");
+        if (source != null) {
+            return new BytesArray(source);
+        }
+        return BytesArray.EMPTY;
+    }
+
+    /**
+     * Parses the given content type string for the media type. This method currently ignores parameters.
+     */
+    // TODO stop ignoring parameters such as charset...
+    private static XContentType parseContentType(List<String> header) {
+        if (header == null || header.isEmpty()) {
+            return null;
+        } else if (header.size() > 1) {
+            throw new IllegalArgumentException("only one Content-Type header should be provided");
+        }
+
+        String rawContentType = header.get(0);
+        final String[] elements = rawContentType.split("[ \t]*;");
+        if (elements.length > 0) {
+            final String[] splitMediaType = elements[0].split("/");
+            if (splitMediaType.length == 2 && TCHAR_PATTERN.matcher(splitMediaType[0]).matches()
+                && TCHAR_PATTERN.matcher(splitMediaType[1].trim()).matches()) {
+                return XContentType.fromMediaType(elements[0]);
+            } else {
+                throw new IllegalArgumentException("invalid Content-Type header [" + rawContentType + "]");
+            }
+        }
+        throw new IllegalArgumentException("empty Content-Type header");
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/RestStatus.java b/core/src/main/java/org/elasticsearch/rest/RestStatus.java
index d72eb2d11f4c8..e7c07f21147d0 100644
--- a/core/src/main/java/org/elasticsearch/rest/RestStatus.java
+++ b/core/src/main/java/org/elasticsearch/rest/RestStatus.java
@@ -479,7 +479,7 @@ public enum RestStatus {
      * is considered to be temporary. If the request that received this status code was the result of a user action,
      * the request MUST NOT be repeated until it is requested by a separate user action.
      */
-    INSUFFICIENT_STORAGE(506);
+    INSUFFICIENT_STORAGE(507);
 
     private static final Map<Integer, RestStatus> CODE_TO_STATUS;
     static {
diff --git a/core/src/main/java/org/elasticsearch/rest/action/RestActions.java b/core/src/main/java/org/elasticsearch/rest/action/RestActions.java
index bb72e3e2249b6..308879d47e8ee 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/RestActions.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/RestActions.java
@@ -19,29 +19,23 @@
 
 package org.elasticsearch.rest.action;
 
-import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.ShardOperationFailedException;
 import org.elasticsearch.action.support.broadcast.BroadcastResponse;
 import org.elasticsearch.action.support.nodes.BaseNodeResponse;
 import org.elasticsearch.action.support.nodes.BaseNodesResponse;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.ToXContent.Params;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.Operator;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryStringQueryBuilder;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestRequest;
@@ -56,6 +50,13 @@
  */
 public class RestActions {
 
+    public static final ParseField _SHARDS_FIELD = new ParseField("_shards");
+    public static final ParseField TOTAL_FIELD = new ParseField("total");
+    public static final ParseField SUCCESSFUL_FIELD = new ParseField("successful");
+    public static final ParseField SKIPPED_FIELD = new ParseField("skipped");
+    public static final ParseField FAILED_FIELD = new ParseField("failed");
+    public static final ParseField FAILURES_FIELD = new ParseField("failures");
+
     public static long parseVersion(RestRequest request) {
         if (request.hasParam("version")) {
             return request.paramAsLong("version", Versions.MATCH_ANY);
@@ -74,19 +75,22 @@ public static long parseVersion(RestRequest request, long defaultVersion) {
 
     public static void buildBroadcastShardsHeader(XContentBuilder builder, Params params, BroadcastResponse response) throws IOException {
         buildBroadcastShardsHeader(builder, params,
-                                   response.getTotalShards(), response.getSuccessfulShards(), response.getFailedShards(),
+                                   response.getTotalShards(), response.getSuccessfulShards(), -1, response.getFailedShards(),
                                    response.getShardFailures());
     }
 
     public static void buildBroadcastShardsHeader(XContentBuilder builder, Params params,
-                                                  int total, int successful, int failed,
+                                                  int total, int successful, int skipped, int failed,
                                                   ShardOperationFailedException[] shardFailures) throws IOException {
-        builder.startObject("_shards");
-        builder.field("total", total);
-        builder.field("successful", successful);
-        builder.field("failed", failed);
+        builder.startObject(_SHARDS_FIELD.getPreferredName());
+        builder.field(TOTAL_FIELD.getPreferredName(), total);
+        builder.field(SUCCESSFUL_FIELD.getPreferredName(), successful);
+        if (skipped >= 0) {
+            builder.field(SKIPPED_FIELD.getPreferredName(), skipped);
+        }
+        builder.field(FAILED_FIELD.getPreferredName(), failed);
         if (shardFailures != null && shardFailures.length > 0) {
-            builder.startArray("failures");
+            builder.startArray(FAILURES_FIELD.getPreferredName());
             final boolean group = params.paramAsBoolean("group_shard_failures", true); // we group by default
             for (ShardOperationFailedException shardFailure : group ? ExceptionsHelper.groupBy(shardFailures) : shardFailures) {
                 builder.startObject();
@@ -195,7 +199,6 @@ public static QueryBuilder urlParamsToQueryBuilder(RestRequest request) {
         queryBuilder.defaultField(request.param("df"));
         queryBuilder.analyzer(request.param("analyzer"));
         queryBuilder.analyzeWildcard(request.paramAsBoolean("analyze_wildcard", false));
-        queryBuilder.lowercaseExpandedTerms(request.paramAsBoolean("lowercase_expanded_terms", true));
         queryBuilder.lenient(request.paramAsBoolean("lenient", null));
         String defaultOperator = request.param("default_operator");
         if (defaultOperator != null) {
@@ -204,53 +207,9 @@ public static QueryBuilder urlParamsToQueryBuilder(RestRequest request) {
         return queryBuilder;
     }
 
-    /**
-     * Get Rest content from either payload or source parameter
-     * @param request Rest request
-     * @return rest content
-     */
-    public static BytesReference getRestContent(RestRequest request) {
-        assert request != null;
-
-        BytesReference content = request.content();
-        if (!request.hasContent()) {
-            String source = request.param("source");
-            if (source != null) {
-                content = new BytesArray(source);
-            }
-        }
-
-        return content;
-    }
-
-    public static QueryBuilder getQueryContent(BytesReference source, IndicesQueriesRegistry indicesQueriesRegistry,
-                                               ParseFieldMatcher parseFieldMatcher) {
-        try (XContentParser requestParser = XContentFactory.xContent(source).createParser(source)) {
-            QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, requestParser, parseFieldMatcher);
-            return context.parseTopLevelQueryBuilder();
-        } catch (IOException e) {
-            throw new ElasticsearchException("failed to parse source", e);
-        }
-    }
-
-    /**
-     * guesses the content type from either payload or source parameter
-     * @param request Rest request
-     * @return rest content type or <code>null</code> if not applicable.
-     */
-    public static XContentType guessBodyContentType(final RestRequest request) {
-        final BytesReference restContent = RestActions.getRestContent(request);
-        if (restContent == null) {
-            return null;
-        }
-        return XContentFactory.xContentType(restContent);
-    }
-
-    /**
-     * Returns <code>true</code> if either payload or source parameter is present. Otherwise <code>false</code>
-     */
-    public static boolean hasBodyContent(final RestRequest request) {
-        return request.hasContent() || request.hasParam("source");
+    public static QueryBuilder getQueryContent(XContentParser requestParser) {
+        QueryParseContext context = new QueryParseContext(requestParser);
+        return context.parseTopLevelQueryBuilder();
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/rest/action/RestBuilderListener.java b/core/src/main/java/org/elasticsearch/rest/action/RestBuilderListener.java
index cc93e72d80d2a..c460331afaa9d 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/RestBuilderListener.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/RestBuilderListener.java
@@ -34,11 +34,22 @@ public RestBuilderListener(RestChannel channel) {
 
     @Override
     public final RestResponse buildResponse(Response response) throws Exception {
-        return buildResponse(response, channel.newBuilder());
+        try (XContentBuilder builder = channel.newBuilder()) {
+            final RestResponse restResponse = buildResponse(response, builder);
+            assert assertBuilderClosed(builder);
+            return restResponse;
+        }
     }
 
     /**
-     * Builds a response to send back over the channel.
+     * Builds a response to send back over the channel. Implementors should ensure that they close the provided {@link XContentBuilder}
+     * using the {@link XContentBuilder#close()} method.
      */
     public abstract RestResponse buildResponse(Response response, XContentBuilder builder) throws Exception;
+
+    // pkg private method that we can override for testing
+    boolean assertBuilderClosed(XContentBuilder xContentBuilder) {
+        assert xContentBuilder.generator().isClosed() : "callers should ensure the XContentBuilder is closed themselves";
+        return true;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/RestFieldCapabilitiesAction.java b/core/src/main/java/org/elasticsearch/rest/action/RestFieldCapabilitiesAction.java
new file mode 100644
index 0000000000000..e983bdc182a01
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/rest/action/RestFieldCapabilitiesAction.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.rest.action;
+
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.BytesRestResponse;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestResponse;
+import org.elasticsearch.rest.RestStatus;
+
+import java.io.IOException;
+
+import static org.elasticsearch.rest.RestRequest.Method.GET;
+import static org.elasticsearch.rest.RestRequest.Method.POST;
+import static org.elasticsearch.rest.RestStatus.NOT_FOUND;
+import static org.elasticsearch.rest.RestStatus.OK;
+
+public class RestFieldCapabilitiesAction extends BaseRestHandler {
+    public RestFieldCapabilitiesAction(Settings settings, RestController controller) {
+        super(settings);
+        controller.registerHandler(GET, "/_field_caps", this);
+        controller.registerHandler(POST, "/_field_caps", this);
+        controller.registerHandler(GET, "/{index}/_field_caps", this);
+        controller.registerHandler(POST, "/{index}/_field_caps", this);
+    }
+
+    @Override
+    public RestChannelConsumer prepareRequest(final RestRequest request,
+                                              final NodeClient client) throws IOException {
+        if (request.hasContentOrSourceParam() && request.hasParam("fields")) {
+            throw new IllegalArgumentException("can't specify a request body and [fields]" +
+                " request parameter, either specify a request body or the" +
+                " [fields] request parameter");
+        }
+        final String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
+        final FieldCapabilitiesRequest fieldRequest;
+        if (request.hasContentOrSourceParam()) {
+            try (XContentParser parser = request.contentOrSourceParamParser()) {
+                fieldRequest = FieldCapabilitiesRequest.parseFields(parser);
+            }
+        } else {
+            fieldRequest = new FieldCapabilitiesRequest();
+            fieldRequest.fields(Strings.splitStringByCommaToArray(request.param("fields")));
+        }
+        fieldRequest.indices(indices);
+        fieldRequest.indicesOptions(
+            IndicesOptions.fromRequest(request, fieldRequest.indicesOptions())
+        );
+        return channel -> client.fieldCaps(fieldRequest,
+            new RestBuilderListener<FieldCapabilitiesResponse>(channel) {
+            @Override
+            public RestResponse buildResponse(FieldCapabilitiesResponse response,
+                                              XContentBuilder builder) throws Exception {
+                RestStatus status = OK;
+                builder.startObject();
+                response.toXContent(builder, request);
+                builder.endObject();
+                return new BytesRestResponse(status, builder);
+            }
+        });
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/rest/action/RestFieldStatsAction.java b/core/src/main/java/org/elasticsearch/rest/action/RestFieldStatsAction.java
index e6ef620db10d3..34e7b636aeb09 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/RestFieldStatsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/RestFieldStatsAction.java
@@ -25,40 +25,45 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.RestStatus;
 
+import java.io.IOException;
 import java.util.Map;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.rest.action.RestActions.buildBroadcastShardsHeader;
 
-/**
- */
 public class RestFieldStatsAction extends BaseRestHandler {
-
-    @Inject
     public RestFieldStatsAction(Settings settings, RestController controller) {
         super(settings);
-        controller.registerHandler(GET, "/_field_stats", this);
-        controller.registerHandler(POST, "/_field_stats", this);
-        controller.registerHandler(GET, "/{index}/_field_stats", this);
-        controller.registerHandler(POST, "/{index}/_field_stats", this);
+        controller.registerAsDeprecatedHandler(GET, "/_field_stats", this,
+             deprecationMessage(), deprecationLogger);
+        controller.registerAsDeprecatedHandler(POST, "/_field_stats", this,
+            deprecationMessage(), deprecationLogger);
+        controller.registerAsDeprecatedHandler(GET, "/{index}/_field_stats", this,
+            deprecationMessage(), deprecationLogger);
+        controller.registerAsDeprecatedHandler(POST, "/{index}/_field_stats", this,
+            deprecationMessage(), deprecationLogger);
+    }
+
+    static String deprecationMessage() {
+        return "[_field_stats] endpoint is deprecated! Use [_field_caps] instead or " +
+            "run a min/max aggregations on the desired fields.";
     }
 
     @Override
-    public void handleRequest(final RestRequest request,
-                              final RestChannel channel, final NodeClient client) throws Exception {
-        if (RestActions.hasBodyContent(request) && request.hasParam("fields")) {
+    public RestChannelConsumer prepareRequest(final RestRequest request,
+                                              final NodeClient client) throws IOException {
+        if (request.hasContentOrSourceParam() && request.hasParam("fields")) {
             throw new IllegalArgumentException("can't specify a request body and [fields] request parameter, " +
                 "either specify a request body or the [fields] request parameter");
         }
@@ -67,13 +72,15 @@ public void handleRequest(final RestRequest request,
         fieldStatsRequest.indices(Strings.splitStringByCommaToArray(request.param("index")));
         fieldStatsRequest.indicesOptions(IndicesOptions.fromRequest(request, fieldStatsRequest.indicesOptions()));
         fieldStatsRequest.level(request.param("level", FieldStatsRequest.DEFAULT_LEVEL));
-        if (RestActions.hasBodyContent(request)) {
-            fieldStatsRequest.source(RestActions.getRestContent(request));
+        if (request.hasContentOrSourceParam()) {
+            try (XContentParser parser = request.contentOrSourceParamParser()) {
+                fieldStatsRequest.source(parser);
+            }
         } else {
             fieldStatsRequest.setFields(Strings.splitStringByCommaToArray(request.param("fields")));
         }
 
-        client.fieldStats(fieldStatsRequest, new RestBuilderListener<FieldStatsResponse>(channel) {
+        return channel -> client.fieldStats(fieldStatsRequest, new RestBuilderListener<FieldStatsResponse>(channel) {
             @Override
             public RestResponse buildResponse(FieldStatsResponse response, XContentBuilder builder) throws Exception {
                 builder.startObject();
@@ -81,7 +88,7 @@ public RestResponse buildResponse(FieldStatsResponse response, XContentBuilder b
 
                 builder.startObject("indices");
                 for (Map.Entry<String, Map<String, FieldStats>> entry1 :
-                    response.getIndicesMergedFieldStats().entrySet()) {
+                        response.getIndicesMergedFieldStats().entrySet()) {
                     builder.startObject(entry1.getKey());
                     builder.startObject("fields");
                     for (Map.Entry<String, FieldStats> entry2 : entry1.getValue().entrySet()) {
diff --git a/core/src/main/java/org/elasticsearch/rest/action/RestMainAction.java b/core/src/main/java/org/elasticsearch/rest/action/RestMainAction.java
index 56053f414b069..006b6d71db4f3 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/RestMainAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/RestMainAction.java
@@ -23,12 +23,10 @@
 import org.elasticsearch.action.main.MainRequest;
 import org.elasticsearch.action.main.MainResponse;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -39,12 +37,7 @@
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.HEAD;
 
-/**
- *
- */
 public class RestMainAction extends BaseRestHandler {
-
-    @Inject
     public RestMainAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/", this);
@@ -52,8 +45,8 @@ public RestMainAction(Settings settings, RestController controller) {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) throws Exception {
-        client.execute(MainAction.INSTANCE, new MainRequest(), new RestBuilderListener<MainResponse>(channel) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
+        return channel -> client.execute(MainAction.INSTANCE, new MainRequest(), new RestBuilderListener<MainResponse>(channel) {
             @Override
             public RestResponse buildResponse(MainResponse mainResponse, XContentBuilder builder) throws Exception {
                 return convertMainResponse(mainResponse, request, builder);
@@ -63,9 +56,6 @@ public RestResponse buildResponse(MainResponse mainResponse, XContentBuilder bui
 
     static BytesRestResponse convertMainResponse(MainResponse response, RestRequest request, XContentBuilder builder) throws IOException {
         RestStatus status = response.isAvailable() ? RestStatus.OK : RestStatus.SERVICE_UNAVAILABLE;
-        if (request.method() == RestRequest.Method.HEAD) {
-            return new BytesRestResponse(status, builder);
-        }
 
         // Default to pretty printing, but allow ?pretty=false to disable
         if (request.hasParam("pretty") == false) {
diff --git a/core/src/main/java/org/elasticsearch/rest/action/RestStatusToXContentListener.java b/core/src/main/java/org/elasticsearch/rest/action/RestStatusToXContentListener.java
index f147d6bc00dc0..6abe61ea5edbe 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/RestStatusToXContentListener.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/RestStatusToXContentListener.java
@@ -18,7 +18,7 @@
  */
 package org.elasticsearch.rest.action;
 
-import org.elasticsearch.common.xcontent.StatusToXContent;
+import org.elasticsearch.common.xcontent.StatusToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestChannel;
@@ -30,7 +30,7 @@
 /**
  * Content listener that extracts that {@link RestStatus} from the response.
  */
-public class RestStatusToXContentListener<Response extends StatusToXContent> extends RestResponseListener<Response> {
+public class RestStatusToXContentListener<Response extends StatusToXContentObject> extends RestToXContentListener<Response> {
     private final Function<Response, String> extractLocation;
 
     /**
@@ -52,17 +52,12 @@ public RestStatusToXContentListener(RestChannel channel, Function<Response, Stri
     }
 
     @Override
-    public final RestResponse buildResponse(Response response) throws Exception {
-        return buildResponse(response, channel.newBuilder());
-    }
-
-    public final RestResponse buildResponse(Response response, XContentBuilder builder) throws Exception {
-        builder.startObject();
+    public RestResponse buildResponse(Response response, XContentBuilder builder) throws Exception {
+        assert response.isFragment() == false; //would be nice if we could make default methods final
         response.toXContent(builder, channel.request());
-        builder.endObject();
-        BytesRestResponse restResponse = new BytesRestResponse(response.status(), builder);
+        RestResponse restResponse = new BytesRestResponse(response.status(), builder);
         if (RestStatus.CREATED == restResponse.status()) {
-            String location = extractLocation.apply(response);
+            final String location = extractLocation.apply(response);
             if (location != null) {
                 restResponse.addHeader("Location", location);
             }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/RestToXContentListener.java b/core/src/main/java/org/elasticsearch/rest/action/RestToXContentListener.java
index 9e5c4f4065948..06fdd021a3261 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/RestToXContentListener.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/RestToXContentListener.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.rest.action;
 
 import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestChannel;
@@ -30,7 +31,7 @@
  * A REST based action listener that assumes the response is of type {@link ToXContent} and automatically
  * builds an XContent based response (wrapping the toXContent in startObject/endObject).
  */
-public class RestToXContentListener<Response extends ToXContent> extends RestResponseListener<Response> {
+public class RestToXContentListener<Response extends ToXContentObject> extends RestResponseListener<Response> {
 
     public RestToXContentListener(RestChannel channel) {
         super(channel);
@@ -41,10 +42,9 @@ public final RestResponse buildResponse(Response response) throws Exception {
         return buildResponse(response, channel.newBuilder());
     }
 
-    public final RestResponse buildResponse(Response response, XContentBuilder builder) throws Exception {
-        builder.startObject();
+    public RestResponse buildResponse(Response response, XContentBuilder builder) throws Exception {
+        assert response.isFragment() == false; //would be nice if we could make default methods final
         response.toXContent(builder, channel.request());
-        builder.endObject();
         return new BytesRestResponse(getStatus(response), builder);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestCancelTasksAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestCancelTasksAction.java
index 3c558fba937ee..28631501b7fd2 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestCancelTasksAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestCancelTasksAction.java
@@ -21,48 +21,51 @@
 
 import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.tasks.TaskId;
 
+import java.io.IOException;
+import java.util.function.Supplier;
+
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.rest.action.admin.cluster.RestListTasksAction.listTasksResponseListener;
 
 
 public class RestCancelTasksAction extends BaseRestHandler {
-    private final ClusterService clusterService;
+    private final Supplier<DiscoveryNodes> nodesInCluster;
 
-    @Inject
-    public RestCancelTasksAction(Settings settings, RestController controller, ClusterService clusterService) {
+    public RestCancelTasksAction(Settings settings, RestController controller, Supplier<DiscoveryNodes> nodesInCluster) {
         super(settings);
-        this.clusterService = clusterService;
+        this.nodesInCluster = nodesInCluster;
         controller.registerHandler(POST, "/_tasks/_cancel", this);
-        controller.registerHandler(POST, "/_tasks/{taskId}/_cancel", this);
+        controller.registerHandler(POST, "/_tasks/{task_id}/_cancel", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
-        String[] nodesIds = Strings.splitStringByCommaToArray(request.param("nodeId"));
-        TaskId taskId = new TaskId(request.param("taskId"));
-        String[] actions = Strings.splitStringByCommaToArray(request.param("actions"));
-        TaskId parentTaskId = new TaskId(request.param("parent_task_id"));
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
+        final String[] nodesIds = Strings.splitStringByCommaToArray(request.param("nodes"));
+        final TaskId taskId = new TaskId(request.param("task_id"));
+        final String[] actions = Strings.splitStringByCommaToArray(request.param("actions"));
+        final TaskId parentTaskId = new TaskId(request.param("parent_task_id"));
+        final String groupBy = request.param("group_by", "nodes");
 
         CancelTasksRequest cancelTasksRequest = new CancelTasksRequest();
         cancelTasksRequest.setTaskId(taskId);
-        cancelTasksRequest.setNodesIds(nodesIds);
+        cancelTasksRequest.setNodes(nodesIds);
         cancelTasksRequest.setActions(actions);
         cancelTasksRequest.setParentTaskId(parentTaskId);
-        client.admin().cluster().cancelTasks(cancelTasksRequest, listTasksResponseListener(clusterService, channel));
+        return channel ->
+            client.admin().cluster().cancelTasks(cancelTasksRequest, listTasksResponseListener(nodesInCluster, groupBy, channel));
     }
 
     @Override
     public boolean canTripCircuitBreaker() {
         return false;
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterAllocationExplainAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterAllocationExplainAction.java
index bae1d1b671491..8855e65f976aa 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterAllocationExplainAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterAllocationExplainAction.java
@@ -19,36 +19,27 @@
 
 package org.elasticsearch.rest.action.admin.cluster;
 
-import java.io.IOException;
-
-import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplainRequest;
 import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplainResponse;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.rest.action.RestActions;
 import org.elasticsearch.rest.action.RestBuilderListener;
 
+import java.io.IOException;
+
 /**
  * Class handling cluster allocation explanation at the REST level
  */
 public class RestClusterAllocationExplainAction extends BaseRestHandler {
-
-    @Inject
     public RestClusterAllocationExplainAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(RestRequest.Method.GET, "/_cluster/allocation/explain", this);
@@ -56,36 +47,26 @@ public RestClusterAllocationExplainAction(Settings settings, RestController cont
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         ClusterAllocationExplainRequest req;
-        if (RestActions.hasBodyContent(request) == false) {
+        if (request.hasContentOrSourceParam() == false) {
             // Empty request signals "explain the first unassigned shard you find"
             req = new ClusterAllocationExplainRequest();
         } else {
-            BytesReference content = RestActions.getRestContent(request);
-            try (XContentParser parser = XContentFactory.xContent(content).createParser(content)) {
+            try (XContentParser parser = request.contentOrSourceParamParser()) {
                 req = ClusterAllocationExplainRequest.parse(parser);
-            } catch (IOException e) {
-                logger.debug("failed to parse allocation explain request", e);
-                channel.sendResponse(
-                    new BytesRestResponse(ExceptionsHelper.status(e), BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY));
-                return;
             }
         }
 
-        try {
-            req.includeYesDecisions(request.paramAsBoolean("include_yes_decisions", false));
-            req.includeDiskInfo(request.paramAsBoolean("include_disk_info", false));
-            client.admin().cluster().allocationExplain(req, new RestBuilderListener<ClusterAllocationExplainResponse>(channel) {
+        req.includeYesDecisions(request.paramAsBoolean("include_yes_decisions", false));
+        req.includeDiskInfo(request.paramAsBoolean("include_disk_info", false));
+        return channel -> client.admin().cluster().allocationExplain(req,
+            new RestBuilderListener<ClusterAllocationExplainResponse>(channel) {
                 @Override
-                public RestResponse buildResponse(ClusterAllocationExplainResponse response, XContentBuilder builder) throws Exception {
+                public RestResponse buildResponse(ClusterAllocationExplainResponse response, XContentBuilder builder) throws IOException {
                     response.getExplanation().toXContent(builder, ToXContent.EMPTY_PARAMS);
                     return new BytesRestResponse(RestStatus.OK, builder);
                 }
             });
-        } catch (Exception e) {
-            logger.error("failed to explain allocation", e);
-            channel.sendResponse(new BytesRestResponse(ExceptionsHelper.status(e), BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY));
-        }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterGetSettingsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterGetSettingsAction.java
index ca2cbaf79face..5c3be8f4347f2 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterGetSettingsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterGetSettingsAction.java
@@ -24,7 +24,6 @@
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
@@ -32,7 +31,6 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -40,13 +38,13 @@
 import org.elasticsearch.rest.action.RestBuilderListener;
 
 import java.io.IOException;
+import java.util.Set;
 
 public class RestClusterGetSettingsAction extends BaseRestHandler {
 
     private final ClusterSettings clusterSettings;
     private final SettingsFilter settingsFilter;
 
-    @Inject
     public RestClusterGetSettingsAction(Settings settings, RestController controller, ClusterSettings clusterSettings,
             SettingsFilter settingsFilter) {
         super(settings);
@@ -56,13 +54,13 @@ public RestClusterGetSettingsAction(Settings settings, RestController controller
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         ClusterStateRequest clusterStateRequest = Requests.clusterStateRequest()
                 .routingTable(false)
                 .nodes(false);
         final boolean renderDefaults = request.paramAsBoolean("include_defaults", false);
         clusterStateRequest.local(request.paramAsBoolean("local", clusterStateRequest.local()));
-        client.admin().cluster().state(clusterStateRequest, new RestBuilderListener<ClusterStateResponse>(channel) {
+        return channel -> client.admin().cluster().state(clusterStateRequest, new RestBuilderListener<ClusterStateResponse>(channel) {
             @Override
             public RestResponse buildResponse(ClusterStateResponse response, XContentBuilder builder) throws Exception {
                 return new BytesRestResponse(RestStatus.OK, renderResponse(response.getState(), renderDefaults, builder, request));
@@ -70,6 +68,11 @@ public RestResponse buildResponse(ClusterStateResponse response, XContentBuilder
         });
     }
 
+    @Override
+    protected Set<String> responseParams() {
+        return Settings.FORMAT_PARAMS;
+    }
+
     @Override
     public boolean canTripCircuitBreaker() {
         return false;
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterHealthAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterHealthAction.java
index 5f64bcf8aa39f..d8dd34d4a2897 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterHealthAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterHealthAction.java
@@ -20,27 +20,25 @@
 package org.elasticsearch.rest.action.admin.cluster;
 
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
-import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestStatusToXContentListener;
 
+import java.io.IOException;
+import java.util.Collections;
 import java.util.Locale;
+import java.util.Set;
 
 import static org.elasticsearch.client.Requests.clusterHealthRequest;
 
 public class RestClusterHealthAction extends BaseRestHandler {
-
-    @Inject
     public RestClusterHealthAction(Settings settings, RestController controller) {
         super(settings);
 
@@ -49,7 +47,7 @@ public RestClusterHealthAction(Settings settings, RestController controller) {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         ClusterHealthRequest clusterHealthRequest = clusterHealthRequest(Strings.splitStringByCommaToArray(request.param("index")));
         clusterHealthRequest.local(request.paramAsBoolean("local", clusterHealthRequest.local()));
         clusterHealthRequest.masterNodeTimeout(request.paramAsTime("master_timeout", clusterHealthRequest.masterNodeTimeout()));
@@ -73,11 +71,19 @@ public void handleRequest(final RestRequest request, final RestChannel channel,
         if (request.param("wait_for_events") != null) {
             clusterHealthRequest.waitForEvents(Priority.valueOf(request.param("wait_for_events").toUpperCase(Locale.ROOT)));
         }
-        client.admin().cluster().health(clusterHealthRequest, new RestStatusToXContentListener<ClusterHealthResponse>(channel));
+        return channel -> client.admin().cluster().health(clusterHealthRequest, new RestStatusToXContentListener<>(channel));
+    }
+
+    private static final Set<String> RESPONSE_PARAMS = Collections.singleton("level");
+
+    @Override
+    protected Set<String> responseParams() {
+        return RESPONSE_PARAMS;
     }
 
     @Override
     public boolean canTripCircuitBreaker() {
         return false;
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterRerouteAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterRerouteAction.java
index 1a1e78b172040..c567fbb11f301 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterRerouteAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterRerouteAction.java
@@ -21,40 +21,36 @@
 
 import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteRequest;
 import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteResponse;
-import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.client.Requests;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.routing.allocation.command.AllocationCommandRegistry;
 import org.elasticsearch.cluster.routing.allocation.command.AllocationCommands;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.AcknowledgedRestListener;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.Set;
 
 /**
  */
 public class RestClusterRerouteAction extends BaseRestHandler {
-    private static final ObjectParser<ClusterRerouteRequest, ParseContext> PARSER = new ObjectParser<>("cluster_reroute");
+    private static final ObjectParser<ClusterRerouteRequest, Void> PARSER = new ObjectParser<>("cluster_reroute");
     static {
-        PARSER.declareField((p, v, c) -> v.commands(AllocationCommands.fromXContent(p, c.getParseFieldMatcher(), c.registry)),
-                new ParseField("commands"), ValueType.OBJECT_ARRAY);
+        PARSER.declareField((p, v, c) -> v.commands(AllocationCommands.fromXContent(p)), new ParseField("commands"),
+                ValueType.OBJECT_ARRAY);
         PARSER.declareBoolean(ClusterRerouteRequest::dryRun, new ParseField("dry_run"));
     }
 
@@ -62,67 +58,61 @@ public class RestClusterRerouteAction extends BaseRestHandler {
             .arrayToCommaDelimitedString(EnumSet.complementOf(EnumSet.of(ClusterState.Metric.METADATA)).toArray());
 
     private final SettingsFilter settingsFilter;
-    private final AllocationCommandRegistry registry;
 
-    @Inject
-    public RestClusterRerouteAction(Settings settings, RestController controller, SettingsFilter settingsFilter,
-            AllocationCommandRegistry registry) {
+    public RestClusterRerouteAction(Settings settings, RestController controller, SettingsFilter settingsFilter) {
         super(settings);
         this.settingsFilter = settingsFilter;
-        this.registry = registry;
         controller.registerHandler(RestRequest.Method.POST, "/_cluster/reroute", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) throws Exception {
-        ClusterRerouteRequest clusterRerouteRequest = createRequest(request, registry, parseFieldMatcher);
-        client.admin().cluster().reroute(clusterRerouteRequest, new AcknowledgedRestListener<ClusterRerouteResponse>(channel) {
-            @Override
-            protected void addCustomFields(XContentBuilder builder, ClusterRerouteResponse response) throws IOException {
-                builder.startObject("state");
-                // by default, return everything but metadata
-                if (request.param("metric") == null) {
-                    request.params().put("metric", DEFAULT_METRICS);
-                }
-                settingsFilter.addFilterSettingParams(request);
-                response.getState().toXContent(builder, request);
-                builder.endObject();
-                if (clusterRerouteRequest.explain()) {
-                    assert response.getExplanations() != null;
-                    response.getExplanations().toXContent(builder, ToXContent.EMPTY_PARAMS);
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
+        ClusterRerouteRequest clusterRerouteRequest = createRequest(request);
+
+        // by default, return everything but metadata
+        final String metric = request.param("metric");
+        if (metric == null) {
+            request.params().put("metric", DEFAULT_METRICS);
+        }
+
+        return channel ->
+            client.admin().cluster().reroute(clusterRerouteRequest, new AcknowledgedRestListener<ClusterRerouteResponse>(channel) {
+                @Override
+                protected void addCustomFields(XContentBuilder builder, ClusterRerouteResponse response) throws IOException {
+                    builder.startObject("state");
+                    settingsFilter.addFilterSettingParams(request);
+                    response.getState().toXContent(builder, request);
+                    builder.endObject();
+                    if (clusterRerouteRequest.explain()) {
+                        assert response.getExplanations() != null;
+                        response.getExplanations().toXContent(builder, ToXContent.EMPTY_PARAMS);
+                    }
                 }
-            }
         });
     }
 
-    public static ClusterRerouteRequest createRequest(RestRequest request, AllocationCommandRegistry registry,
-            ParseFieldMatcher parseFieldMatcher) throws IOException {
+    private static final Set<String> RESPONSE_PARAMS;
+
+    static {
+        final Set<String> responseParams = new HashSet<>();
+        responseParams.add("metric");
+        responseParams.addAll(Settings.FORMAT_PARAMS);
+        RESPONSE_PARAMS = Collections.unmodifiableSet(responseParams);
+    }
+
+    @Override
+    protected Set<String> responseParams() {
+        return RESPONSE_PARAMS;
+    }
+
+    public static ClusterRerouteRequest createRequest(RestRequest request) throws IOException {
         ClusterRerouteRequest clusterRerouteRequest = Requests.clusterRerouteRequest();
         clusterRerouteRequest.dryRun(request.paramAsBoolean("dry_run", clusterRerouteRequest.dryRun()));
         clusterRerouteRequest.explain(request.paramAsBoolean("explain", clusterRerouteRequest.explain()));
         clusterRerouteRequest.timeout(request.paramAsTime("timeout", clusterRerouteRequest.timeout()));
         clusterRerouteRequest.setRetryFailed(request.paramAsBoolean("retry_failed", clusterRerouteRequest.isRetryFailed()));
         clusterRerouteRequest.masterNodeTimeout(request.paramAsTime("master_timeout", clusterRerouteRequest.masterNodeTimeout()));
-        if (request.hasContent()) {
-            try (XContentParser parser = XContentHelper.createParser(request.content())) {
-                PARSER.parse(parser, clusterRerouteRequest, new ParseContext(registry, parseFieldMatcher));
-            }
-        }
+        request.applyContentParser(parser -> PARSER.parse(parser, clusterRerouteRequest, null));
         return clusterRerouteRequest;
     }
-
-    private static class ParseContext implements ParseFieldMatcherSupplier {
-        private final AllocationCommandRegistry registry;
-        private final ParseFieldMatcher parseFieldMatcher;
-
-        private ParseContext(AllocationCommandRegistry registry, ParseFieldMatcher parseFieldMatcher) {
-            this.registry = registry;
-            this.parseFieldMatcher = parseFieldMatcher;
-        }
-
-        @Override
-        public ParseFieldMatcher getParseFieldMatcher() {
-            return parseFieldMatcher;
-        }
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterSearchShardsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterSearchShardsAction.java
index 754b9b0d63342..a927cb8b4ac7a 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterSearchShardsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterSearchShardsAction.java
@@ -20,27 +20,22 @@
 package org.elasticsearch.rest.action.admin.cluster;
 
 import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsRequest;
-import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsResponse;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.client.Requests;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestToXContentListener;
 
+import java.io.IOException;
+
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
-/**
- */
 public class RestClusterSearchShardsAction extends BaseRestHandler {
-
-    @Inject
     public RestClusterSearchShardsAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_search_shards", this);
@@ -52,16 +47,19 @@ public RestClusterSearchShardsAction(Settings settings, RestController controlle
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
         final ClusterSearchShardsRequest clusterSearchShardsRequest = Requests.clusterSearchShardsRequest(indices);
         clusterSearchShardsRequest.local(request.paramAsBoolean("local", clusterSearchShardsRequest.local()));
 
-        clusterSearchShardsRequest.types(Strings.splitStringByCommaToArray(request.param("type")));
+        if (request.hasParam("type")) {
+            String type = request.param("type");
+            deprecationLogger.deprecated("type [" + type + "] doesn't have any effect in the search shards api, " +
+                    "it should be rather omitted");
+        }
         clusterSearchShardsRequest.routing(request.param("routing"));
         clusterSearchShardsRequest.preference(request.param("preference"));
         clusterSearchShardsRequest.indicesOptions(IndicesOptions.fromRequest(request, clusterSearchShardsRequest.indicesOptions()));
-
-        client.admin().cluster().searchShards(clusterSearchShardsRequest, new RestToXContentListener<ClusterSearchShardsResponse>(channel));
+        return channel -> client.admin().cluster().searchShards(clusterSearchShardsRequest, new RestToXContentListener<>(channel));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStateAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStateAction.java
index fab2ee0062f1a..fe99d544d46e3 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStateAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStateAction.java
@@ -22,30 +22,31 @@
 import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.client.Requests;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.RestBuilderListener;
 
+import java.io.IOException;
+import java.util.Collections;
 import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.Set;
 
 public class RestClusterStateAction extends BaseRestHandler {
 
     private final SettingsFilter settingsFilter;
 
-    @Inject
     public RestClusterStateAction(Settings settings, RestController controller, SettingsFilter settingsFilter) {
         super(settings);
         controller.registerHandler(RestRequest.Method.GET, "/_cluster/state", this);
@@ -56,7 +57,7 @@ public RestClusterStateAction(Settings settings, RestController controller, Sett
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         final ClusterStateRequest clusterStateRequest = Requests.clusterStateRequest();
         clusterStateRequest.indicesOptions(IndicesOptions.fromRequest(request, clusterStateRequest.indicesOptions()));
         clusterStateRequest.local(request.paramAsBoolean("local", clusterStateRequest.local()));
@@ -84,7 +85,7 @@ public void handleRequest(final RestRequest request, final RestChannel channel,
         }
         settingsFilter.addFilterSettingParams(request);
 
-        client.admin().cluster().state(clusterStateRequest, new RestBuilderListener<ClusterStateResponse>(channel) {
+        return channel -> client.admin().cluster().state(clusterStateRequest, new RestBuilderListener<ClusterStateResponse>(channel) {
             @Override
             public RestResponse buildResponse(ClusterStateResponse response, XContentBuilder builder) throws Exception {
                 builder.startObject();
@@ -96,6 +97,20 @@ public RestResponse buildResponse(ClusterStateResponse response, XContentBuilder
         });
     }
 
+    private static final Set<String> RESPONSE_PARAMS;
+
+    static {
+        final Set<String> responseParams = new HashSet<>();
+        responseParams.add("metric");
+        responseParams.addAll(Settings.FORMAT_PARAMS);
+        RESPONSE_PARAMS = Collections.unmodifiableSet(responseParams);
+    }
+
+    @Override
+    protected Set<String> responseParams() {
+        return RESPONSE_PARAMS;
+    }
+
     @Override
     public boolean canTripCircuitBreaker() {
         return false;
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStatsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStatsAction.java
index 7ef05d04553a2..e28c61776d17c 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStatsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStatsAction.java
@@ -21,20 +21,18 @@
 
 import org.elasticsearch.action.admin.cluster.stats.ClusterStatsRequest;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestActions.NodesResponseRestListener;
 
+import java.io.IOException;
+
 /**
  *
  */
 public class RestClusterStatsAction extends BaseRestHandler {
-
-    @Inject
     public RestClusterStatsAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(RestRequest.Method.GET, "/_cluster/stats", this);
@@ -42,10 +40,10 @@ public RestClusterStatsAction(Settings settings, RestController controller) {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         ClusterStatsRequest clusterStatsRequest = new ClusterStatsRequest().nodesIds(request.paramAsStringArray("nodeId", null));
         clusterStatsRequest.timeout(request.param("timeout"));
-        client.admin().cluster().clusterStats(clusterStatsRequest, new NodesResponseRestListener<>(channel));
+        return channel -> client.admin().cluster().clusterStats(clusterStatsRequest, new NodesResponseRestListener<>(channel));
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterUpdateSettingsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterUpdateSettingsAction.java
index 8de725dbe798c..02b62b8e4bf9e 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterUpdateSettingsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterUpdateSettingsAction.java
@@ -21,38 +21,34 @@
 
 import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
 import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse;
-import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.client.Requests;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.AcknowledgedRestListener;
 
 import java.io.IOException;
 import java.util.Map;
+import java.util.Set;
 
 public class RestClusterUpdateSettingsAction extends BaseRestHandler {
-
-    @Inject
     public RestClusterUpdateSettingsAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(RestRequest.Method.PUT, "/_cluster/settings", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) throws Exception {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         final ClusterUpdateSettingsRequest clusterUpdateSettingsRequest = Requests.clusterUpdateSettingsRequest();
         clusterUpdateSettingsRequest.timeout(request.paramAsTime("timeout", clusterUpdateSettingsRequest.timeout()));
         clusterUpdateSettingsRequest.masterNodeTimeout(
                 request.paramAsTime("master_timeout", clusterUpdateSettingsRequest.masterNodeTimeout()));
         Map<String, Object> source;
-        try (XContentParser parser = XContentFactory.xContent(request.content()).createParser(request.content())) {
+        try (XContentParser parser = request.contentParser()) {
             source = parser.map();
         }
         if (source.containsKey("transient")) {
@@ -62,7 +58,7 @@ public void handleRequest(final RestRequest request, final RestChannel channel,
             clusterUpdateSettingsRequest.persistentSettings((Map) source.get("persistent"));
         }
 
-        client.admin().cluster().updateSettings(clusterUpdateSettingsRequest,
+        return channel -> client.admin().cluster().updateSettings(clusterUpdateSettingsRequest,
                 new AcknowledgedRestListener<ClusterUpdateSettingsResponse>(channel) {
                     @Override
                     protected void addCustomFields(XContentBuilder builder, ClusterUpdateSettingsResponse response) throws IOException {
@@ -77,6 +73,11 @@ protected void addCustomFields(XContentBuilder builder, ClusterUpdateSettingsRes
                 });
     }
 
+    @Override
+    protected Set<String> responseParams() {
+        return Settings.FORMAT_PARAMS;
+    }
+
     @Override
     public boolean canTripCircuitBreaker() {
         return false;
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestCreateSnapshotAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestCreateSnapshotAction.java
index 96449131a61a7..bd0d7e2a9d73f 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestCreateSnapshotAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestCreateSnapshotAction.java
@@ -20,16 +20,15 @@
 package org.elasticsearch.rest.action.admin.cluster;
 
 import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest;
-import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestToXContentListener;
 
+import java.io.IOException;
+
 import static org.elasticsearch.client.Requests.createSnapshotRequest;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.rest.RestRequest.Method.PUT;
@@ -38,8 +37,6 @@
  * Creates a new snapshot
  */
 public class RestCreateSnapshotAction extends BaseRestHandler {
-
-    @Inject
     public RestCreateSnapshotAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(PUT, "/_snapshot/{repository}/{snapshot}", this);
@@ -47,11 +44,11 @@ public RestCreateSnapshotAction(Settings settings, RestController controller) {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         CreateSnapshotRequest createSnapshotRequest = createSnapshotRequest(request.param("repository"), request.param("snapshot"));
-        createSnapshotRequest.source(request.content().utf8ToString());
+        request.applyContentParser(p -> createSnapshotRequest.source(p.mapOrdered()));
         createSnapshotRequest.masterNodeTimeout(request.paramAsTime("master_timeout", createSnapshotRequest.masterNodeTimeout()));
         createSnapshotRequest.waitForCompletion(request.paramAsBoolean("wait_for_completion", false));
-        client.admin().cluster().createSnapshot(createSnapshotRequest, new RestToXContentListener<CreateSnapshotResponse>(channel));
+        return channel -> client.admin().cluster().createSnapshot(createSnapshotRequest, new RestToXContentListener<>(channel));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteRepositoryAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteRepositoryAction.java
index 78d063bae00c4..2019e04be5597 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteRepositoryAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteRepositoryAction.java
@@ -20,16 +20,15 @@
 package org.elasticsearch.rest.action.admin.cluster;
 
 import org.elasticsearch.action.admin.cluster.repositories.delete.DeleteRepositoryRequest;
-import org.elasticsearch.action.admin.cluster.repositories.delete.DeleteRepositoryResponse;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.AcknowledgedRestListener;
 
+import java.io.IOException;
+
 import static org.elasticsearch.client.Requests.deleteRepositoryRequest;
 import static org.elasticsearch.rest.RestRequest.Method.DELETE;
 
@@ -37,19 +36,17 @@
  * Unregisters a repository
  */
 public class RestDeleteRepositoryAction extends BaseRestHandler {
-
-    @Inject
     public RestDeleteRepositoryAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(DELETE, "/_snapshot/{repository}", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         DeleteRepositoryRequest deleteRepositoryRequest = deleteRepositoryRequest(request.param("repository"));
         deleteRepositoryRequest.masterNodeTimeout(request.paramAsTime("master_timeout", deleteRepositoryRequest.masterNodeTimeout()));
         deleteRepositoryRequest.timeout(request.paramAsTime("timeout", deleteRepositoryRequest.timeout()));
         deleteRepositoryRequest.masterNodeTimeout(request.paramAsTime("master_timeout", deleteRepositoryRequest.masterNodeTimeout()));
-        client.admin().cluster().deleteRepository(deleteRepositoryRequest, new AcknowledgedRestListener<DeleteRepositoryResponse>(channel));
+        return channel -> client.admin().cluster().deleteRepository(deleteRepositoryRequest, new AcknowledgedRestListener<>(channel));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteSnapshotAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteSnapshotAction.java
index d001a1e90e5c1..a11d47278a8e8 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteSnapshotAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteSnapshotAction.java
@@ -20,16 +20,15 @@
 package org.elasticsearch.rest.action.admin.cluster;
 
 import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest;
-import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotResponse;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.AcknowledgedRestListener;
 
+import java.io.IOException;
+
 import static org.elasticsearch.client.Requests.deleteSnapshotRequest;
 import static org.elasticsearch.rest.RestRequest.Method.DELETE;
 
@@ -37,17 +36,15 @@
  * Deletes a snapshot
  */
 public class RestDeleteSnapshotAction extends BaseRestHandler {
-
-    @Inject
     public RestDeleteSnapshotAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(DELETE, "/_snapshot/{repository}/{snapshot}", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         DeleteSnapshotRequest deleteSnapshotRequest = deleteSnapshotRequest(request.param("repository"), request.param("snapshot"));
         deleteSnapshotRequest.masterNodeTimeout(request.paramAsTime("master_timeout", deleteSnapshotRequest.masterNodeTimeout()));
-        client.admin().cluster().deleteSnapshot(deleteSnapshotRequest, new AcknowledgedRestListener<DeleteSnapshotResponse>(channel));
+        return channel -> client.admin().cluster().deleteSnapshot(deleteSnapshotRequest, new AcknowledgedRestListener<>(channel));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteStoredScriptAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteStoredScriptAction.java
index 212b42135e96c..a8f2567a64777 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteStoredScriptAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteStoredScriptAction.java
@@ -20,38 +20,49 @@
 
 import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.AcknowledgedRestListener;
 
+import java.io.IOException;
+
 import static org.elasticsearch.rest.RestRequest.Method.DELETE;
 
 public class RestDeleteStoredScriptAction extends BaseRestHandler {
 
-    @Inject
     public RestDeleteStoredScriptAction(Settings settings, RestController controller) {
-        this(settings, controller, true);
-    }
-
-    protected RestDeleteStoredScriptAction(Settings settings, RestController controller, boolean registerDefaultHandlers) {
         super(settings);
-        if (registerDefaultHandlers) {
-            controller.registerHandler(DELETE, "/_scripts/{lang}/{id}", this);
-        }
-    }
 
-    protected String getScriptLang(RestRequest request) {
-        return request.param("lang");
+        // Note {lang} is actually {id} in the first handler.  It appears
+        // parameters as part of the path must be of the same ordering relative
+        // to name or they will not work as expected.
+        controller.registerHandler(DELETE, "/_scripts/{lang}", this);
+        controller.registerHandler(DELETE, "/_scripts/{lang}/{id}", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, NodeClient client) {
-        DeleteStoredScriptRequest deleteStoredScriptRequest = new DeleteStoredScriptRequest(getScriptLang(request), request.param("id"));
-        client.admin().cluster().deleteStoredScript(deleteStoredScriptRequest, new AcknowledgedRestListener<>(channel));
-    }
+    public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+        String id = request.param("id");
+        String lang = request.param("lang");
 
+        // In the case where only {lang} is not null, we make it {id} because of
+        // name ordering issues in the handlers' paths.
+        if (id == null) {
+            id = lang;
+            lang = null;
+        }
+
+        if (lang != null) {
+            deprecationLogger.deprecated(
+                "specifying lang [" + lang + "] as part of the url path is deprecated");
+        }
+
+        DeleteStoredScriptRequest deleteStoredScriptRequest = new DeleteStoredScriptRequest(id, lang);
+        deleteStoredScriptRequest.timeout(request.paramAsTime("timeout", deleteStoredScriptRequest.timeout()));
+        deleteStoredScriptRequest.masterNodeTimeout(request.paramAsTime("master_timeout", deleteStoredScriptRequest.masterNodeTimeout()));
+
+        return channel -> client.admin().cluster().deleteStoredScript(deleteStoredScriptRequest, new AcknowledgedRestListener<>(channel));
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetRepositoriesAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetRepositoriesAction.java
index 802af3cb5b811..1af138400e310 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetRepositoriesAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetRepositoriesAction.java
@@ -25,18 +25,19 @@
 import org.elasticsearch.cluster.metadata.RepositoriesMetaData;
 import org.elasticsearch.cluster.metadata.RepositoryMetaData;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.action.RestBuilderListener;
 
+import java.io.IOException;
+import java.util.Set;
+
 import static org.elasticsearch.client.Requests.getRepositoryRequest;
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestStatus.OK;
@@ -48,7 +49,6 @@ public class RestGetRepositoriesAction extends BaseRestHandler {
 
     private final SettingsFilter settingsFilter;
 
-    @Inject
     public RestGetRepositoriesAction(Settings settings, RestController controller, SettingsFilter settingsFilter) {
         super(settings);
         controller.registerHandler(GET, "/_snapshot", this);
@@ -57,23 +57,30 @@ public RestGetRepositoriesAction(Settings settings, RestController controller, S
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         final String[] repositories = request.paramAsStringArray("repository", Strings.EMPTY_ARRAY);
         GetRepositoriesRequest getRepositoriesRequest = getRepositoryRequest(repositories);
         getRepositoriesRequest.masterNodeTimeout(request.paramAsTime("master_timeout", getRepositoriesRequest.masterNodeTimeout()));
         getRepositoriesRequest.local(request.paramAsBoolean("local", getRepositoriesRequest.local()));
         settingsFilter.addFilterSettingParams(request);
-        client.admin().cluster().getRepositories(getRepositoriesRequest, new RestBuilderListener<GetRepositoriesResponse>(channel) {
-            @Override
-            public RestResponse buildResponse(GetRepositoriesResponse response, XContentBuilder builder) throws Exception {
-                builder.startObject();
-                for (RepositoryMetaData repositoryMetaData : response.repositories()) {
-                    RepositoriesMetaData.toXContent(repositoryMetaData, builder, request);
-                }
-                builder.endObject();
+        return channel ->
+                client.admin().cluster().getRepositories(getRepositoriesRequest, new RestBuilderListener<GetRepositoriesResponse>(channel) {
+                    @Override
+                    public RestResponse buildResponse(GetRepositoriesResponse response, XContentBuilder builder) throws Exception {
+                        builder.startObject();
+                        for (RepositoryMetaData repositoryMetaData : response.repositories()) {
+                            RepositoriesMetaData.toXContent(repositoryMetaData, builder, request);
+                        }
+                        builder.endObject();
 
-                return new BytesRestResponse(OK, builder);
-            }
+                        return new BytesRestResponse(OK, builder);
+                    }
         });
     }
+
+    @Override
+    protected Set<String> responseParams() {
+        return Settings.FORMAT_PARAMS;
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetSnapshotsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetSnapshotsAction.java
index 9e10a87bc037e..7348cb5896cb4 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetSnapshotsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetSnapshotsAction.java
@@ -20,17 +20,16 @@
 package org.elasticsearch.rest.action.admin.cluster;
 
 import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest;
-import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestToXContentListener;
 
+import java.io.IOException;
+
 import static org.elasticsearch.client.Requests.getSnapshotsRequest;
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
@@ -38,23 +37,20 @@
  * Returns information about snapshot
  */
 public class RestGetSnapshotsAction extends BaseRestHandler {
-
-    @Inject
     public RestGetSnapshotsAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_snapshot/{repository}/{snapshot}", this);
     }
 
-
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         String repository = request.param("repository");
         String[] snapshots = request.paramAsStringArray("snapshot", Strings.EMPTY_ARRAY);
 
         GetSnapshotsRequest getSnapshotsRequest = getSnapshotsRequest(repository).snapshots(snapshots);
         getSnapshotsRequest.ignoreUnavailable(request.paramAsBoolean("ignore_unavailable", getSnapshotsRequest.ignoreUnavailable()));
-
+        getSnapshotsRequest.verbose(request.paramAsBoolean("verbose", getSnapshotsRequest.verbose()));
         getSnapshotsRequest.masterNodeTimeout(request.paramAsTime("master_timeout", getSnapshotsRequest.masterNodeTimeout()));
-        client.admin().cluster().getSnapshots(getSnapshotsRequest, new RestToXContentListener<GetSnapshotsResponse>(channel));
+        return channel -> client.admin().cluster().getSnapshots(getSnapshotsRequest, new RestToXContentListener<>(channel));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetStoredScriptAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetStoredScriptAction.java
index 1185685c49a54..7af21a9fb45be 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetStoredScriptAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetStoredScriptAction.java
@@ -21,68 +21,94 @@
 import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest;
 import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptResponse;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.RestBuilderListener;
+import org.elasticsearch.script.StoredScriptSource;
+
+import java.io.IOException;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
 public class RestGetStoredScriptAction extends BaseRestHandler {
 
-    @Inject
+    public static final ParseField _ID_PARSE_FIELD = new ParseField("_id");
+
+    public static final ParseField FOUND_PARSE_FIELD = new ParseField("found");
+
     public RestGetStoredScriptAction(Settings settings, RestController controller) {
-        this(settings, controller, true);
+        super(settings);
+
+        // Note {lang} is actually {id} in the first handler.  It appears
+        // parameters as part of the path must be of the same ordering relative
+        // to name or they will not work as expected.
+        controller.registerHandler(GET, "/_scripts/{lang}", this);
+        controller.registerHandler(GET, "/_scripts/{lang}/{id}", this);
     }
 
-    protected RestGetStoredScriptAction(Settings settings, RestController controller, boolean registerDefaultHandlers) {
-        super(settings);
-        if (registerDefaultHandlers) {
-            controller.registerHandler(GET, "/_scripts/{lang}/{id}", this);
+    @Override
+    public RestChannelConsumer prepareRequest(final RestRequest request, NodeClient client) throws IOException {
+        String id;
+        String lang;
+
+        // In the case where only {lang} is not null, we make it {id} because of
+        // name ordering issues in the handlers' paths.
+        if (request.param("id") == null) {
+            id = request.param("lang");;
+            lang = null;
+        } else {
+            id = request.param("id");
+            lang = request.param("lang");
         }
-    }
 
-    protected String getScriptFieldName() {
-        return Fields.SCRIPT;
-    }
+        if (lang != null) {
+            deprecationLogger.deprecated(
+                "specifying lang [" + lang + "] as part of the url path is deprecated");
+        }
 
-    protected String getScriptLang(RestRequest request) {
-        return request.param("lang");
-    }
+        GetStoredScriptRequest getRequest = new GetStoredScriptRequest(id, lang);
 
-    @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, NodeClient client) {
-        final GetStoredScriptRequest getRequest = new GetStoredScriptRequest(getScriptLang(request), request.param("id"));
-        client.admin().cluster().getStoredScript(getRequest, new RestBuilderListener<GetStoredScriptResponse>(channel) {
+        return channel -> client.admin().cluster().getStoredScript(getRequest, new RestBuilderListener<GetStoredScriptResponse>(channel) {
             @Override
             public RestResponse buildResponse(GetStoredScriptResponse response, XContentBuilder builder) throws Exception {
                 builder.startObject();
-                builder.field(Fields.LANG, getRequest.lang());
-                builder.field(Fields._ID, getRequest.id());
-                boolean found = response.getStoredScript() != null;
-                builder.field(Fields.FOUND, found);
-                RestStatus status = RestStatus.NOT_FOUND;
+                builder.field(_ID_PARSE_FIELD.getPreferredName(), id);
+
+                if (lang != null) {
+                    builder.field(StoredScriptSource.LANG_PARSE_FIELD.getPreferredName(), lang);
+                }
+
+                StoredScriptSource source = response.getSource();
+                boolean found = source != null;
+                builder.field(FOUND_PARSE_FIELD.getPreferredName(), found);
+
                 if (found) {
-                    builder.field(getScriptFieldName(), response.getStoredScript());
-                    status = RestStatus.OK;
+                    if (lang == null) {
+                        builder.startObject(StoredScriptSource.SCRIPT_PARSE_FIELD.getPreferredName());
+                        builder.field(StoredScriptSource.LANG_PARSE_FIELD.getPreferredName(), source.getLang());
+                        builder.field(StoredScriptSource.SOURCE_PARSE_FIELD.getPreferredName(), source.getSource());
+
+                        if (source.getOptions().isEmpty() == false) {
+                            builder.field(StoredScriptSource.OPTIONS_PARSE_FIELD.getPreferredName(), source.getOptions());
+                        }
+
+                        builder.endObject();
+                    } else {
+                        builder.field(StoredScriptSource.SCRIPT_PARSE_FIELD.getPreferredName(), source.getSource());
+                    }
                 }
+
                 builder.endObject();
-                return new BytesRestResponse(status, builder);
+
+                return new BytesRestResponse(found ? RestStatus.OK : RestStatus.NOT_FOUND, builder);
             }
         });
     }
-
-    private static final class Fields {
-        private static final String SCRIPT = "script";
-        private static final String LANG = "lang";
-        private static final String _ID = "_id";
-        private static final String FOUND = "found";
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetTaskAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetTaskAction.java
index f1edf672010c1..e013970553fd3 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetTaskAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetTaskAction.java
@@ -21,27 +21,26 @@
 
 import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskRequest;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestToXContentListener;
 import org.elasticsearch.tasks.TaskId;
 
+import java.io.IOException;
+
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
 public class RestGetTaskAction extends BaseRestHandler {
-    @Inject
     public RestGetTaskAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_tasks/{taskId}", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         TaskId taskId = new TaskId(request.param("taskId"));
         boolean waitForCompletion = request.paramAsBoolean("wait_for_completion", false);
         TimeValue timeout = request.paramAsTime("timeout", null);
@@ -50,6 +49,6 @@ public void handleRequest(final RestRequest request, final RestChannel channel,
         getTaskRequest.setTaskId(taskId);
         getTaskRequest.setWaitForCompletion(waitForCompletion);
         getTaskRequest.setTimeout(timeout);
-        client.admin().cluster().getTask(getTaskRequest, new RestToXContentListener<>(channel));
+        return channel -> client.admin().cluster().getTask(getTaskRequest, new RestToXContentListener<>(channel));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestListTasksAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestListTasksAction.java
index d5ff427e3d07d..4177386eff996 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestListTasksAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestListTasksAction.java
@@ -23,9 +23,8 @@
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -40,29 +39,40 @@
 import org.elasticsearch.rest.action.RestToXContentListener;
 import org.elasticsearch.tasks.TaskId;
 
+import java.io.IOException;
+import java.util.function.Supplier;
+
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
 
 public class RestListTasksAction extends BaseRestHandler {
-    private final ClusterService clusterService;
 
-    @Inject
-    public RestListTasksAction(Settings settings, RestController controller, ClusterService clusterService) {
+    private final Supplier<DiscoveryNodes> nodesInCluster;
+
+    public RestListTasksAction(Settings settings, RestController controller, Supplier<DiscoveryNodes> nodesInCluster) {
         super(settings);
-        this.clusterService = clusterService;
+        this.nodesInCluster = nodesInCluster;
         controller.registerHandler(GET, "/_tasks", this);
     }
 
+    @Override
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
+        final ListTasksRequest listTasksRequest = generateListTasksRequest(request);
+        final String groupBy = request.param("group_by", "nodes");
+        return channel -> client.admin().cluster().listTasks(listTasksRequest,
+                listTasksResponseListener(nodesInCluster, groupBy, channel));
+    }
+
     public static ListTasksRequest generateListTasksRequest(RestRequest request) {
         boolean detailed = request.paramAsBoolean("detailed", false);
-        String[] nodesIds = Strings.splitStringByCommaToArray(request.param("node_id"));
+        String[] nodes = Strings.splitStringByCommaToArray(request.param("nodes"));
         String[] actions = Strings.splitStringByCommaToArray(request.param("actions"));
         TaskId parentTaskId = new TaskId(request.param("parent_task_id"));
         boolean waitForCompletion = request.paramAsBoolean("wait_for_completion", false);
         TimeValue timeout = request.paramAsTime("timeout", null);
 
         ListTasksRequest listTasksRequest = new ListTasksRequest();
-        listTasksRequest.setNodesIds(nodesIds);
+        listTasksRequest.setNodes(nodes);
         listTasksRequest.setDetailed(detailed);
         listTasksRequest.setActions(actions);
         listTasksRequest.setParentTaskId(parentTaskId);
@@ -71,23 +81,19 @@ public static ListTasksRequest generateListTasksRequest(RestRequest request) {
         return listTasksRequest;
     }
 
-    @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
-        client.admin().cluster().listTasks(generateListTasksRequest(request), listTasksResponseListener(clusterService, channel));
-    }
-
     /**
      * Standard listener for extensions of {@link ListTasksResponse} that supports {@code group_by=nodes}.
      */
-    public static <T extends ListTasksResponse> ActionListener<T> listTasksResponseListener(ClusterService clusterService,
-            RestChannel channel) {
-        String groupBy = channel.request().param("group_by", "nodes");
+    public static <T extends ListTasksResponse> ActionListener<T> listTasksResponseListener(
+                Supplier<DiscoveryNodes> nodesInCluster,
+                String groupBy,
+                final RestChannel channel) {
         if ("nodes".equals(groupBy)) {
             return new RestBuilderListener<T>(channel) {
                 @Override
                 public RestResponse buildResponse(T response, XContentBuilder builder) throws Exception {
                     builder.startObject();
-                    response.toXContentGroupedByNode(builder, channel.request(), clusterService.state().nodes());
+                    response.toXContentGroupedByNode(builder, channel.request(), nodesInCluster.get());
                     builder.endObject();
                     return new BytesRestResponse(RestStatus.OK, builder);
                 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java
index 87af57276c2b0..50d1856c1973d 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java
@@ -24,24 +24,22 @@
 import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsResponse;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.RestResponseListener;
 
+import java.io.IOException;
+
 
 /**
  */
 public class RestNodesHotThreadsAction extends BaseRestHandler {
-
-    @Inject
     public RestNodesHotThreadsAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(RestRequest.Method.GET, "/_cluster/nodes/hotthreads", this);
@@ -56,7 +54,7 @@ public RestNodesHotThreadsAction(Settings settings, RestController controller) {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         String[] nodesIds = Strings.splitStringByCommaToArray(request.param("nodeId"));
         NodesHotThreadsRequest nodesHotThreadsRequest = new NodesHotThreadsRequest(nodesIds);
         nodesHotThreadsRequest.threads(request.paramAsInt("threads", nodesHotThreadsRequest.threads()));
@@ -65,18 +63,20 @@ public void handleRequest(final RestRequest request, final RestChannel channel,
         nodesHotThreadsRequest.interval(TimeValue.parseTimeValue(request.param("interval"), nodesHotThreadsRequest.interval(), "interval"));
         nodesHotThreadsRequest.snapshots(request.paramAsInt("snapshots", nodesHotThreadsRequest.snapshots()));
         nodesHotThreadsRequest.timeout(request.param("timeout"));
-        client.admin().cluster().nodesHotThreads(nodesHotThreadsRequest, new RestResponseListener<NodesHotThreadsResponse>(channel) {
-            @Override
-            public RestResponse buildResponse(NodesHotThreadsResponse response) throws Exception {
-                StringBuilder sb = new StringBuilder();
-                for (NodeHotThreads node : response.getNodes()) {
-                    sb.append("::: ").append(node.getNode().toString()).append("\n");
-                    Strings.spaceify(3, node.getHotThreads(), sb);
-                    sb.append('\n');
-                }
-                return new BytesRestResponse(RestStatus.OK, sb.toString());
-            }
-        });
+        return channel -> client.admin().cluster().nodesHotThreads(
+                nodesHotThreadsRequest,
+                new RestResponseListener<NodesHotThreadsResponse>(channel) {
+                    @Override
+                    public RestResponse buildResponse(NodesHotThreadsResponse response) throws Exception {
+                        StringBuilder sb = new StringBuilder();
+                        for (NodeHotThreads node : response.getNodes()) {
+                            sb.append("::: ").append(node.getNode().toString()).append("\n");
+                            Strings.spaceify(3, node.getHotThreads(), sb);
+                            sb.append('\n');
+                        }
+                        return new BytesRestResponse(RestStatus.OK, sb.toString());
+                    }
+                });
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesInfoAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesInfoAction.java
index c45709d07d5fa..dfe3b08697c97 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesInfoAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesInfoAction.java
@@ -22,16 +22,15 @@
 import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestActions.NodesResponseRestListener;
 
+import java.io.IOException;
 import java.util.Set;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
@@ -51,11 +50,10 @@ public class RestNodesInfoAction extends BaseRestHandler {
 
     private final SettingsFilter settingsFilter;
 
-    @Inject
     public RestNodesInfoAction(Settings settings, RestController controller, SettingsFilter settingsFilter) {
         super(settings);
         controller.registerHandler(GET, "/_nodes", this);
-        // this endpoint is used for metrics, not for nodeIds, like /_nodes/fs
+        // this endpoint is used for metrics, not for node IDs, like /_nodes/fs
         controller.registerHandler(GET, "/_nodes/{nodeId}", this);
         controller.registerHandler(GET, "/_nodes/{nodeId}/{metrics}", this);
         // added this endpoint to be aligned with stats
@@ -65,7 +63,7 @@ public RestNodesInfoAction(Settings settings, RestController controller, Setting
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         String[] nodeIds;
         Set<String> metrics;
 
@@ -108,7 +106,12 @@ public void handleRequest(final RestRequest request, final RestChannel channel,
 
         settingsFilter.addFilterSettingParams(request);
 
-        client.admin().cluster().nodesInfo(nodesInfoRequest, new NodesResponseRestListener<>(channel));
+        return channel -> client.admin().cluster().nodesInfo(nodesInfoRequest, new NodesResponseRestListener<>(channel));
+    }
+
+    @Override
+    protected Set<String> responseParams() {
+        return Settings.FORMAT_PARAMS;
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsAction.java
index be6847f1b5281..2ad431f1ea5d7 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsAction.java
@@ -24,21 +24,24 @@
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags.Flag;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestActions.NodesResponseRestListener;
 
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
 import java.util.Set;
+import java.util.TreeSet;
+import java.util.function.Consumer;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
 public class RestNodesStatsAction extends BaseRestHandler {
-
-    @Inject
     public RestNodesStatsAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_nodes/stats", this);
@@ -47,13 +50,42 @@ public RestNodesStatsAction(Settings settings, RestController controller) {
         controller.registerHandler(GET, "/_nodes/stats/{metric}", this);
         controller.registerHandler(GET, "/_nodes/{nodeId}/stats/{metric}", this);
 
-        controller.registerHandler(GET, "/_nodes/stats/{metric}/{indexMetric}", this);
+        controller.registerHandler(GET, "/_nodes/stats/{metric}/{index_metric}", this);
+
+        controller.registerHandler(GET, "/_nodes/{nodeId}/stats/{metric}/{index_metric}", this);
+    }
+
+    static final Map<String, Consumer<NodesStatsRequest>> METRICS;
+
+    static {
+        final Map<String, Consumer<NodesStatsRequest>> metrics = new HashMap<>();
+        metrics.put("os", r -> r.os(true));
+        metrics.put("jvm", r -> r.jvm(true));
+        metrics.put("thread_pool", r -> r.threadPool(true));
+        metrics.put("fs", r -> r.fs(true));
+        metrics.put("transport", r -> r.transport(true));
+        metrics.put("http", r -> r.http(true));
+        metrics.put("indices", r -> r.indices(true));
+        metrics.put("process", r -> r.process(true));
+        metrics.put("breaker", r -> r.breaker(true));
+        metrics.put("script", r -> r.script(true));
+        metrics.put("discovery", r -> r.discovery(true));
+        metrics.put("ingest", r -> r.ingest(true));
+        METRICS = Collections.unmodifiableMap(metrics);
+    }
+
+    static final Map<String, Consumer<CommonStatsFlags>> FLAGS;
 
-        controller.registerHandler(GET, "/_nodes/{nodeId}/stats/{metric}/{indexMetric}", this);
+    static {
+        final Map<String, Consumer<CommonStatsFlags>> flags = new HashMap<>();
+        for (final Flag flag : CommonStatsFlags.Flag.values()) {
+            flags.put(flag.getRestName(), f -> f.set(flag, true));
+        }
+        FLAGS = Collections.unmodifiableMap(flags);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         String[] nodesIds = Strings.splitStringByCommaToArray(request.param("nodeId"));
         Set<String> metrics = Strings.splitStringByCommaToSet(request.param("metric", "_all"));
 
@@ -61,35 +93,78 @@ public void handleRequest(final RestRequest request, final RestChannel channel,
         nodesStatsRequest.timeout(request.param("timeout"));
 
         if (metrics.size() == 1 && metrics.contains("_all")) {
+            if (request.hasParam("index_metric")) {
+                throw new IllegalArgumentException(
+                    String.format(
+                        Locale.ROOT,
+                        "request [%s] contains index metrics [%s] but all stats requested",
+                        request.path(),
+                        request.param("index_metric")));
+            }
             nodesStatsRequest.all();
             nodesStatsRequest.indices(CommonStatsFlags.ALL);
+        } else if (metrics.contains("_all")) {
+            throw new IllegalArgumentException(
+                String.format(Locale.ROOT,
+                    "request [%s] contains _all and individual metrics [%s]",
+                    request.path(),
+                    request.param("metric")));
         } else {
             nodesStatsRequest.clear();
-            nodesStatsRequest.os(metrics.contains("os"));
-            nodesStatsRequest.jvm(metrics.contains("jvm"));
-            nodesStatsRequest.threadPool(metrics.contains("thread_pool"));
-            nodesStatsRequest.fs(metrics.contains("fs"));
-            nodesStatsRequest.transport(metrics.contains("transport"));
-            nodesStatsRequest.http(metrics.contains("http"));
-            nodesStatsRequest.indices(metrics.contains("indices"));
-            nodesStatsRequest.process(metrics.contains("process"));
-            nodesStatsRequest.breaker(metrics.contains("breaker"));
-            nodesStatsRequest.script(metrics.contains("script"));
-            nodesStatsRequest.discovery(metrics.contains("discovery"));
-            nodesStatsRequest.ingest(metrics.contains("ingest"));
+
+            // use a sorted set so the unrecognized parameters appear in a reliable sorted order
+            final Set<String> invalidMetrics = new TreeSet<>();
+            for (final String metric : metrics) {
+                final Consumer<NodesStatsRequest> handler = METRICS.get(metric);
+                if (handler != null) {
+                    handler.accept(nodesStatsRequest);
+                } else {
+                    invalidMetrics.add(metric);
+                }
+            }
+
+            if (!invalidMetrics.isEmpty()) {
+                throw new IllegalArgumentException(unrecognized(request, invalidMetrics, METRICS.keySet(), "metric"));
+            }
 
             // check for index specific metrics
             if (metrics.contains("indices")) {
-                Set<String> indexMetrics = Strings.splitStringByCommaToSet(request.param("indexMetric", "_all"));
+                Set<String> indexMetrics = Strings.splitStringByCommaToSet(request.param("index_metric", "_all"));
                 if (indexMetrics.size() == 1 && indexMetrics.contains("_all")) {
                     nodesStatsRequest.indices(CommonStatsFlags.ALL);
                 } else {
                     CommonStatsFlags flags = new CommonStatsFlags();
-                    for (Flag flag : CommonStatsFlags.Flag.values()) {
-                        flags.set(flag, indexMetrics.contains(flag.getRestName()));
+                    flags.clear();
+                    // use a sorted set so the unrecognized parameters appear in a reliable sorted order
+                    final Set<String> invalidIndexMetrics = new TreeSet<>();
+                    for (final String indexMetric : indexMetrics) {
+                        final Consumer<CommonStatsFlags> handler = FLAGS.get(indexMetric);
+                        if (handler != null) {
+                            handler.accept(flags);
+                        } else {
+                            invalidIndexMetrics.add(indexMetric);
+                        }
+                    }
+
+                    if (invalidIndexMetrics.contains("percolate")) {
+                        deprecationLogger.deprecated(
+                            "percolate stats are no longer available and requests for percolate stats will fail starting in 6.0.0");
+                        invalidIndexMetrics.remove("percolate");
+                    }
+
+                    if (!invalidIndexMetrics.isEmpty()) {
+                        throw new IllegalArgumentException(unrecognized(request, invalidIndexMetrics, FLAGS.keySet(), "index metric"));
                     }
+
                     nodesStatsRequest.indices(flags);
                 }
+            } else if (request.hasParam("index_metric")) {
+                throw new IllegalArgumentException(
+                    String.format(
+                        Locale.ROOT,
+                        "request [%s] contains index metrics [%s] but indices stats not requested",
+                        request.path(),
+                        request.param("index_metric")));
             }
         }
 
@@ -107,15 +182,23 @@ public void handleRequest(final RestRequest request, final RestChannel channel,
         if (nodesStatsRequest.indices().isSet(Flag.Indexing) && (request.hasParam("types"))) {
             nodesStatsRequest.indices().types(request.paramAsStringArray("types", null));
         }
-        if (nodesStatsRequest.indices().isSet(Flag.Segments) && (request.hasParam("include_segment_file_sizes"))) {
-            nodesStatsRequest.indices().includeSegmentFileSizes(true);
+        if (nodesStatsRequest.indices().isSet(Flag.Segments)) {
+            nodesStatsRequest.indices().includeSegmentFileSizes(request.paramAsBoolean("include_segment_file_sizes", false));
         }
 
-        client.admin().cluster().nodesStats(nodesStatsRequest, new NodesResponseRestListener<>(channel));
+        return channel -> client.admin().cluster().nodesStats(nodesStatsRequest, new NodesResponseRestListener<>(channel));
+    }
+
+    private final Set<String> RESPONSE_PARAMS = Collections.singleton("level");
+
+    @Override
+    protected Set<String> responseParams() {
+        return RESPONSE_PARAMS;
     }
 
     @Override
     public boolean canTripCircuitBreaker() {
         return false;
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPendingClusterTasksAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPendingClusterTasksAction.java
index d1cb65092ce2d..29b2b72895bb9 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPendingClusterTasksAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPendingClusterTasksAction.java
@@ -21,27 +21,25 @@
 
 import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequest;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestToXContentListener;
 
-public class RestPendingClusterTasksAction extends BaseRestHandler {
+import java.io.IOException;
 
-    @Inject
+public class RestPendingClusterTasksAction extends BaseRestHandler {
     public RestPendingClusterTasksAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(RestRequest.Method.GET, "/_cluster/pending_tasks", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         PendingClusterTasksRequest pendingClusterTasksRequest = new PendingClusterTasksRequest();
         pendingClusterTasksRequest.masterNodeTimeout(request.paramAsTime("master_timeout", pendingClusterTasksRequest.masterNodeTimeout()));
         pendingClusterTasksRequest.local(request.paramAsBoolean("local", pendingClusterTasksRequest.local()));
-        client.admin().cluster().pendingClusterTasks(pendingClusterTasksRequest, new RestToXContentListener<>(channel));
+        return channel -> client.admin().cluster().pendingClusterTasks(pendingClusterTasksRequest, new RestToXContentListener<>(channel));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPutRepositoryAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPutRepositoryAction.java
index 002e1bfdc951b..afd2fd851aead 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPutRepositoryAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPutRepositoryAction.java
@@ -20,16 +20,16 @@
 package org.elasticsearch.rest.action.admin.cluster;
 
 import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest;
-import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryResponse;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.AcknowledgedRestListener;
 
+import java.io.IOException;
+
 import static org.elasticsearch.client.Requests.putRepositoryRequest;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.rest.RestRequest.Method.PUT;
@@ -38,8 +38,6 @@
  * Registers repositories
  */
 public class RestPutRepositoryAction extends BaseRestHandler {
-
-    @Inject
     public RestPutRepositoryAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(PUT, "/_snapshot/{repository}", this);
@@ -48,12 +46,14 @@ public RestPutRepositoryAction(Settings settings, RestController controller) {
 
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         PutRepositoryRequest putRepositoryRequest = putRepositoryRequest(request.param("repository"));
-        putRepositoryRequest.source(request.content().utf8ToString());
+        try (XContentParser parser = request.contentParser()) {
+            putRepositoryRequest.source(parser.mapOrdered());
+        }
         putRepositoryRequest.verify(request.paramAsBoolean("verify", true));
         putRepositoryRequest.masterNodeTimeout(request.paramAsTime("master_timeout", putRepositoryRequest.masterNodeTimeout()));
         putRepositoryRequest.timeout(request.paramAsTime("timeout", putRepositoryRequest.timeout()));
-        client.admin().cluster().putRepository(putRepositoryRequest, new AcknowledgedRestListener<PutRepositoryResponse>(channel));
+        return channel -> client.admin().cluster().putRepository(putRepositoryRequest, new AcknowledgedRestListener<>(channel));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPutStoredScriptAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPutStoredScriptAction.java
index c5156c4cd093b..9a9920dd4b064 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPutStoredScriptAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPutStoredScriptAction.java
@@ -20,40 +20,54 @@
 
 import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequest;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.AcknowledgedRestListener;
 
+import java.io.IOException;
+
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.rest.RestRequest.Method.PUT;
 
 public class RestPutStoredScriptAction extends BaseRestHandler {
 
-    @Inject
     public RestPutStoredScriptAction(Settings settings, RestController controller) {
-        this(settings, controller, true);
-    }
-
-    protected RestPutStoredScriptAction(Settings settings, RestController controller, boolean registerDefaultHandlers) {
         super(settings);
-        if (registerDefaultHandlers) {
-            controller.registerHandler(POST, "/_scripts/{lang}/{id}", this);
-            controller.registerHandler(PUT, "/_scripts/{lang}/{id}", this);
-        }
-    }
 
-    protected String getScriptLang(RestRequest request) {
-        return request.param("lang");
+        // Note {lang} is actually {id} in the first two handlers.  It appears
+        // parameters as part of the path must be of the same ordering relative
+        // to name or they will not work as expected.
+        controller.registerHandler(POST, "/_scripts/{lang}", this);
+        controller.registerHandler(PUT, "/_scripts/{lang}", this);
+        controller.registerHandler(POST, "/_scripts/{lang}/{id}", this);
+        controller.registerHandler(PUT, "/_scripts/{lang}/{id}", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, NodeClient client) {
-        PutStoredScriptRequest putRequest = new PutStoredScriptRequest(getScriptLang(request), request.param("id"));
-        putRequest.script(request.content());
-        client.admin().cluster().putStoredScript(putRequest, new AcknowledgedRestListener<>(channel));
+    public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+        String id = request.param("id");
+        String lang = request.param("lang");
+
+        // In the case where only {lang} is not null, we make it {id} because of
+        // name ordering issues in the handlers' paths.
+        if (id == null) {
+            id = lang;
+            lang = null;
+        }
+
+        BytesReference content = request.requiredContent();
+
+        if (lang != null) {
+            deprecationLogger.deprecated(
+                "specifying lang [" + lang + "] as part of the url path is deprecated, use request content instead");
+        }
+
+        PutStoredScriptRequest putRequest = new PutStoredScriptRequest(id, lang, content, request.getXContentType());
+        putRequest.masterNodeTimeout(request.paramAsTime("master_timeout", putRequest.masterNodeTimeout()));
+        putRequest.timeout(request.paramAsTime("timeout", putRequest.timeout()));
+        return channel -> client.admin().cluster().putStoredScript(putRequest, new AcknowledgedRestListener<>(channel));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestRemoteClusterInfoAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestRemoteClusterInfoAction.java
new file mode 100644
index 0000000000000..c15b2553e5de6
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestRemoteClusterInfoAction.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.rest.action.admin.cluster;
+
+import org.elasticsearch.action.admin.cluster.remote.RemoteInfoAction;
+import org.elasticsearch.action.admin.cluster.remote.RemoteInfoRequest;
+import org.elasticsearch.action.admin.cluster.remote.RemoteInfoResponse;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.BytesRestResponse;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestResponse;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.rest.action.RestBuilderListener;
+
+import java.io.IOException;
+
+import static org.elasticsearch.rest.RestRequest.Method.GET;
+
+public final class RestRemoteClusterInfoAction extends BaseRestHandler {
+
+    public RestRemoteClusterInfoAction(Settings settings, RestController controller) {
+        super(settings);
+        controller.registerHandler(GET, "_remote/info", this);
+    }
+
+    @Override
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client)
+        throws IOException {
+        return channel -> client.execute(RemoteInfoAction.INSTANCE, new RemoteInfoRequest(),
+            new RestBuilderListener<RemoteInfoResponse>(channel) {
+            @Override
+            public RestResponse buildResponse(RemoteInfoResponse response, XContentBuilder builder) throws Exception {
+                response.toXContent(builder, request);
+                return new BytesRestResponse(RestStatus.OK, builder);
+            }
+        });
+    }
+    @Override
+    public boolean canTripCircuitBreaker() {
+        return false;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestRestoreSnapshotAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestRestoreSnapshotAction.java
index 100866e02db8c..3948921c12da3 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestRestoreSnapshotAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestRestoreSnapshotAction.java
@@ -20,16 +20,15 @@
 package org.elasticsearch.rest.action.admin.cluster;
 
 import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest;
-import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestToXContentListener;
 
+import java.io.IOException;
+
 import static org.elasticsearch.client.Requests.restoreSnapshotRequest;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
@@ -37,19 +36,17 @@
  * Restores a snapshot
  */
 public class RestRestoreSnapshotAction extends BaseRestHandler {
-
-    @Inject
     public RestRestoreSnapshotAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(POST, "/_snapshot/{repository}/{snapshot}/_restore", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         RestoreSnapshotRequest restoreSnapshotRequest = restoreSnapshotRequest(request.param("repository"), request.param("snapshot"));
         restoreSnapshotRequest.masterNodeTimeout(request.paramAsTime("master_timeout", restoreSnapshotRequest.masterNodeTimeout()));
         restoreSnapshotRequest.waitForCompletion(request.paramAsBoolean("wait_for_completion", false));
-        restoreSnapshotRequest.source(request.content().utf8ToString());
-        client.admin().cluster().restoreSnapshot(restoreSnapshotRequest, new RestToXContentListener<RestoreSnapshotResponse>(channel));
+        request.applyContentParser(p -> restoreSnapshotRequest.source(p.mapOrdered()));
+        return channel -> client.admin().cluster().restoreSnapshot(restoreSnapshotRequest, new RestToXContentListener<>(channel));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestSnapshotsStatusAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestSnapshotsStatusAction.java
index 4333dfc02710a..c5122641c1a15 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestSnapshotsStatusAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestSnapshotsStatusAction.java
@@ -22,14 +22,14 @@
 import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusRequest;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestToXContentListener;
 
+import java.io.IOException;
+
 import static org.elasticsearch.client.Requests.snapshotsStatusRequest;
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
@@ -37,8 +37,6 @@
  * Returns status of currently running snapshot
  */
 public class RestSnapshotsStatusAction extends BaseRestHandler {
-
-    @Inject
     public RestSnapshotsStatusAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_snapshot/{repository}/{snapshot}/_status", this);
@@ -47,7 +45,7 @@ public RestSnapshotsStatusAction(Settings settings, RestController controller) {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         String repository = request.param("repository", "_all");
         String[] snapshots = request.paramAsStringArray("snapshot", Strings.EMPTY_ARRAY);
         if (snapshots.length == 1 && "_all".equalsIgnoreCase(snapshots[0])) {
@@ -57,6 +55,6 @@ public void handleRequest(final RestRequest request, final RestChannel channel,
         snapshotsStatusRequest.ignoreUnavailable(request.paramAsBoolean("ignore_unavailable", snapshotsStatusRequest.ignoreUnavailable()));
 
         snapshotsStatusRequest.masterNodeTimeout(request.paramAsTime("master_timeout", snapshotsStatusRequest.masterNodeTimeout()));
-        client.admin().cluster().snapshotsStatus(snapshotsStatusRequest, new RestToXContentListener<>(channel));
+        return channel -> client.admin().cluster().snapshotsStatus(snapshotsStatusRequest, new RestToXContentListener<>(channel));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestVerifyRepositoryAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestVerifyRepositoryAction.java
index 85aac84077736..a9fb44eae645b 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestVerifyRepositoryAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestVerifyRepositoryAction.java
@@ -21,30 +21,28 @@
 
 import org.elasticsearch.action.admin.cluster.repositories.verify.VerifyRepositoryRequest;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestToXContentListener;
 
+import java.io.IOException;
+
 import static org.elasticsearch.client.Requests.verifyRepositoryRequest;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
 public class RestVerifyRepositoryAction extends BaseRestHandler {
-
-    @Inject
     public RestVerifyRepositoryAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(POST, "/_snapshot/{repository}/_verify", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         VerifyRepositoryRequest verifyRepositoryRequest = verifyRepositoryRequest(request.param("repository"));
         verifyRepositoryRequest.masterNodeTimeout(request.paramAsTime("master_timeout", verifyRepositoryRequest.masterNodeTimeout()));
         verifyRepositoryRequest.timeout(request.paramAsTime("timeout", verifyRepositoryRequest.timeout()));
-        client.admin().cluster().verifyRepository(verifyRepositoryRequest, new RestToXContentListener<>(channel));
+        return channel -> client.admin().cluster().verifyRepository(verifyRepositoryRequest, new RestToXContentListener<>(channel));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestAliasesExistAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestAliasesExistAction.java
deleted file mode 100644
index dbb8ddde9d1ab..0000000000000
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestAliasesExistAction.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.rest.action.admin.indices;
-
-import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.admin.indices.alias.exists.AliasesExistResponse;
-import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
-import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.RestChannel;
-import org.elasticsearch.rest.RestController;
-import org.elasticsearch.rest.RestRequest;
-
-import static org.elasticsearch.rest.RestRequest.Method.HEAD;
-import static org.elasticsearch.rest.RestStatus.NOT_FOUND;
-import static org.elasticsearch.rest.RestStatus.OK;
-
-/**
- */
-public class RestAliasesExistAction extends BaseRestHandler {
-
-    @Inject
-    public RestAliasesExistAction(Settings settings, RestController controller) {
-        super(settings);
-        controller.registerHandler(HEAD, "/_alias/{name}", this);
-        controller.registerHandler(HEAD, "/{index}/_alias/{name}", this);
-        controller.registerHandler(HEAD, "/{index}/_alias", this);
-    }
-
-    @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
-        String[] aliases = request.paramAsStringArray("name", Strings.EMPTY_ARRAY);
-        final String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
-        GetAliasesRequest getAliasesRequest = new GetAliasesRequest(aliases);
-        getAliasesRequest.indices(indices);
-        getAliasesRequest.indicesOptions(IndicesOptions.fromRequest(request, getAliasesRequest.indicesOptions()));
-        getAliasesRequest.local(request.paramAsBoolean("local", getAliasesRequest.local()));
-
-        client.admin().indices().aliasesExist(getAliasesRequest, new ActionListener<AliasesExistResponse>() {
-
-            @Override
-            public void onResponse(AliasesExistResponse response) {
-                try {
-                    if (response.isExists()) {
-                        channel.sendResponse(new BytesRestResponse(OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY));
-                    } else {
-                        channel.sendResponse(new BytesRestResponse(NOT_FOUND, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY));
-                    }
-                } catch (Exception e) {
-                    onFailure(e);
-                }
-            }
-
-            @Override
-            public void onFailure(Exception e) {
-                try {
-                    channel.sendResponse(
-                        new BytesRestResponse(ExceptionsHelper.status(e), BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY));
-                } catch (Exception inner) {
-                    inner.addSuppressed(e);
-                    logger.error("Failed to send failure response", inner);
-                }
-            }
-        });
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestAnalyzeAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestAnalyzeAction.java
index 04d0bf57612e1..b1d9612936d1c 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestAnalyzeAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestAnalyzeAction.java
@@ -16,24 +16,19 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+
 package org.elasticsearch.rest.action.admin.indices;
 
 import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.action.RestActions;
 import org.elasticsearch.rest.action.RestToXContentListener;
 
 import java.io.IOException;
@@ -56,7 +51,6 @@ public static class Fields {
         public static final ParseField ATTRIBUTES = new ParseField("attributes");
     }
 
-    @Inject
     public RestAnalyzeAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_analyze", this);
@@ -65,123 +59,161 @@ public RestAnalyzeAction(Settings settings, RestController controller) {
         controller.registerHandler(POST, "/{index}/_analyze", this);
     }
 
+    private void deprecationLog(String key, RestRequest request) {
+        if (request.hasParam(key)) {
+            deprecationLogWithoutCheck(key);
+        }
+    }
+
+    private void deprecationLogWithoutCheck(String key) {
+        deprecationLogForText(key + " request parameter is deprecated and will be removed in the next major release." +
+            " Please use the JSON in the request body instead request param");
+    }
+
+    void deprecationLogForText(String msg) {
+        deprecationLogger.deprecated(msg);
+    }
+
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
 
         String[] texts = request.paramAsStringArrayOrEmptyIfAll("text");
+        deprecationLog("text", request);
 
         AnalyzeRequest analyzeRequest = new AnalyzeRequest(request.param("index"));
         analyzeRequest.text(texts);
         analyzeRequest.analyzer(request.param("analyzer"));
+        deprecationLog("analyzer", request);
         analyzeRequest.field(request.param("field"));
-        if (request.hasParam("tokenizer")) {
-            analyzeRequest.tokenizer(request.param("tokenizer"));
+        deprecationLog("field", request);
+        final String tokenizer = request.param("tokenizer");
+        if (tokenizer != null) {
+            analyzeRequest.tokenizer(tokenizer);
+            deprecationLogWithoutCheck("tokenizer");
         }
         for (String filter : request.paramAsStringArray("filter", Strings.EMPTY_ARRAY)) {
             analyzeRequest.addTokenFilter(filter);
+            deprecationLogWithoutCheck("filter");
         }
         for (String charFilter : request.paramAsStringArray("char_filter", Strings.EMPTY_ARRAY)) {
             analyzeRequest.addTokenFilter(charFilter);
+            deprecationLogWithoutCheck("char_filter");
         }
         analyzeRequest.explain(request.paramAsBoolean("explain", false));
+        deprecationLog("explain", request);
         analyzeRequest.attributes(request.paramAsStringArray("attributes", analyzeRequest.attributes()));
+        deprecationLog("attributes", request);
 
-        if (RestActions.hasBodyContent(request)) {
-            XContentType type = RestActions.guessBodyContentType(request);
-            if (type == null) {
-                if (texts == null || texts.length == 0) {
-                    texts = new String[]{ RestActions.getRestContent(request).utf8ToString() };
-                    analyzeRequest.text(texts);
+        try {
+            handleBodyContent(request, texts, analyzeRequest);
+        } catch (IOException e) {
+            throw new IllegalArgumentException("Failed to parse request body", e);
+        }
+
+        return channel -> client.admin().indices().analyze(analyzeRequest, new RestToXContentListener<>(channel));
+    }
+
+    void handleBodyContent(RestRequest request, final String[] texts, AnalyzeRequest analyzeRequest) throws IOException {
+        request.withContentOrSourceParamParserOrNullLenient(parser -> {
+            if (parser == null) {
+                if (request.hasContent()) {
+                    BytesReference body = request.getContentOrSourceParamOnly();
+                    if (texts == null || texts.length == 0) {
+                        final String[] localTexts = new String[]{ body.utf8ToString() };
+                        analyzeRequest.text(localTexts);
+                        deprecationLogForText(" plain text bodies is deprecated and " +
+                            "this feature will be removed in the next major release. Please use the text param in JSON");
+                    }
                 }
             } else {
-                // NOTE: if rest request with xcontent body has request parameters, the parameters does not override xcontent values
-                buildFromContent(RestActions.getRestContent(request), analyzeRequest, parseFieldMatcher);
+                // NOTE: if rest request with xcontent body has request parameters, the parameters do not override xcontent values
+                buildFromContent(parser, analyzeRequest);
             }
-        }
+        });
+    }
 
-        client.admin().indices().analyze(analyzeRequest, new RestToXContentListener<>(channel));
+
+    @Override
+    public boolean supportsPlainText() {
+        return true;
     }
 
-    public static void buildFromContent(BytesReference content, AnalyzeRequest analyzeRequest, ParseFieldMatcher parseFieldMatcher) {
-        try (XContentParser parser = XContentHelper.createParser(content)) {
-            if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
-                throw new IllegalArgumentException("Malformed content, must start with an object");
-            } else {
-                XContentParser.Token token;
-                String currentFieldName = null;
-                while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                    if (token == XContentParser.Token.FIELD_NAME) {
-                        currentFieldName = parser.currentName();
-                    } else if (parseFieldMatcher.match(currentFieldName, Fields.TEXT) && token == XContentParser.Token.VALUE_STRING) {
-                        analyzeRequest.text(parser.text());
-                    } else if (parseFieldMatcher.match(currentFieldName, Fields.TEXT) && token == XContentParser.Token.START_ARRAY) {
-                        List<String> texts = new ArrayList<>();
-                        while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
-                            if (token.isValue() == false) {
-                                throw new IllegalArgumentException(currentFieldName + " array element should only contain text");
-                            }
-                            texts.add(parser.text());
+    static void buildFromContent(XContentParser parser, AnalyzeRequest analyzeRequest) throws IOException {
+        if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
+            throw new IllegalArgumentException("Malformed content, must start with an object");
+        } else {
+            XContentParser.Token token;
+            String currentFieldName = null;
+            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                if (token == XContentParser.Token.FIELD_NAME) {
+                    currentFieldName = parser.currentName();
+                } else if (Fields.TEXT.match(currentFieldName) && token == XContentParser.Token.VALUE_STRING) {
+                    analyzeRequest.text(parser.text());
+                } else if (Fields.TEXT.match(currentFieldName) && token == XContentParser.Token.START_ARRAY) {
+                    List<String> texts = new ArrayList<>();
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                        if (token.isValue() == false) {
+                            throw new IllegalArgumentException(currentFieldName + " array element should only contain text");
                         }
-                        analyzeRequest.text(texts.toArray(new String[texts.size()]));
-                    } else if (parseFieldMatcher.match(currentFieldName, Fields.ANALYZER) && token == XContentParser.Token.VALUE_STRING) {
-                        analyzeRequest.analyzer(parser.text());
-                    } else if (parseFieldMatcher.match(currentFieldName, Fields.FIELD) && token == XContentParser.Token.VALUE_STRING) {
-                        analyzeRequest.field(parser.text());
-                    } else if (parseFieldMatcher.match(currentFieldName, Fields.TOKENIZER)) {
+                        texts.add(parser.text());
+                    }
+                    analyzeRequest.text(texts.toArray(new String[texts.size()]));
+                } else if (Fields.ANALYZER.match(currentFieldName) && token == XContentParser.Token.VALUE_STRING) {
+                    analyzeRequest.analyzer(parser.text());
+                } else if (Fields.FIELD.match(currentFieldName) && token == XContentParser.Token.VALUE_STRING) {
+                    analyzeRequest.field(parser.text());
+                } else if (Fields.TOKENIZER.match(currentFieldName)) {
+                    if (token == XContentParser.Token.VALUE_STRING) {
+                        analyzeRequest.tokenizer(parser.text());
+                    } else if (token == XContentParser.Token.START_OBJECT) {
+                        analyzeRequest.tokenizer(parser.map());
+                    } else {
+                        throw new IllegalArgumentException(currentFieldName + " should be tokenizer's name or setting");
+                    }
+                } else if (Fields.TOKEN_FILTERS.match(currentFieldName)
+                        && token == XContentParser.Token.START_ARRAY) {
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         if (token == XContentParser.Token.VALUE_STRING) {
-                            analyzeRequest.tokenizer(parser.text());
+                            analyzeRequest.addTokenFilter(parser.text());
                         } else if (token == XContentParser.Token.START_OBJECT) {
-                            analyzeRequest.tokenizer(parser.map());
+                            analyzeRequest.addTokenFilter(parser.map());
                         } else {
-                            throw new IllegalArgumentException(currentFieldName + " should be tokenizer's name or setting");
-                        }
-                    } else if (parseFieldMatcher.match(currentFieldName, Fields.TOKEN_FILTERS)
-                            && token == XContentParser.Token.START_ARRAY) {
-                        while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
-                            if (token == XContentParser.Token.VALUE_STRING) {
-                                analyzeRequest.addTokenFilter(parser.text());
-                            } else if (token == XContentParser.Token.START_OBJECT) {
-                                analyzeRequest.addTokenFilter(parser.map());
-                            } else {
-                                throw new IllegalArgumentException(currentFieldName
-                                        + " array element should contain filter's name or setting");
-                            }
+                            throw new IllegalArgumentException(currentFieldName
+                                    + " array element should contain filter's name or setting");
                         }
-                    } else if (parseFieldMatcher.match(currentFieldName, Fields.CHAR_FILTERS)
-                            && token == XContentParser.Token.START_ARRAY) {
-                        while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
-                            if (token == XContentParser.Token.VALUE_STRING) {
-                                analyzeRequest.addCharFilter(parser.text());
-                            } else if (token == XContentParser.Token.START_OBJECT) {
-                                analyzeRequest.addCharFilter(parser.map());
-                            } else {
-                                throw new IllegalArgumentException(currentFieldName
-                                        + " array element should contain char filter's name or setting");
-                            }
-                        }
-                    } else if (parseFieldMatcher.match(currentFieldName, Fields.EXPLAIN)) {
-                        if (parser.isBooleanValue()) {
-                            analyzeRequest.explain(parser.booleanValue());
+                    }
+                } else if (Fields.CHAR_FILTERS.match(currentFieldName)
+                        && token == XContentParser.Token.START_ARRAY) {
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                        if (token == XContentParser.Token.VALUE_STRING) {
+                            analyzeRequest.addCharFilter(parser.text());
+                        } else if (token == XContentParser.Token.START_OBJECT) {
+                            analyzeRequest.addCharFilter(parser.map());
                         } else {
-                            throw new IllegalArgumentException(currentFieldName + " must be either 'true' or 'false'");
+                            throw new IllegalArgumentException(currentFieldName
+                                    + " array element should contain char filter's name or setting");
                         }
-                    } else if (parseFieldMatcher.match(currentFieldName, Fields.ATTRIBUTES) && token == XContentParser.Token.START_ARRAY) {
-                        List<String> attributes = new ArrayList<>();
-                        while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
-                            if (token.isValue() == false) {
-                                throw new IllegalArgumentException(currentFieldName + " array element should only contain attribute name");
-                            }
-                            attributes.add(parser.text());
-                        }
-                        analyzeRequest.attributes(attributes.toArray(new String[attributes.size()]));
+                    }
+                } else if (Fields.EXPLAIN.match(currentFieldName)) {
+                    if (parser.isBooleanValue()) {
+                        analyzeRequest.explain(parser.booleanValue());
                     } else {
-                        throw new IllegalArgumentException("Unknown parameter ["
-                                + currentFieldName + "] in request body or parameter is of the wrong type[" + token + "] ");
+                        throw new IllegalArgumentException(currentFieldName + " must be either 'true' or 'false'");
                     }
+                } else if (Fields.ATTRIBUTES.match(currentFieldName) && token == XContentParser.Token.START_ARRAY) {
+                    List<String> attributes = new ArrayList<>();
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                        if (token.isValue() == false) {
+                            throw new IllegalArgumentException(currentFieldName + " array element should only contain attribute name");
+                        }
+                        attributes.add(parser.text());
+                    }
+                    analyzeRequest.attributes(attributes.toArray(new String[attributes.size()]));
+                } else {
+                    throw new IllegalArgumentException("Unknown parameter ["
+                            + currentFieldName + "] in request body or parameter is of the wrong type[" + token + "] ");
                 }
             }
-        } catch (IOException e) {
-            throw new IllegalArgumentException("Failed to parse request body", e);
         }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestClearIndicesCacheAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestClearIndicesCacheAction.java
index 391eaa64d5045..1544a01f9f09b 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestClearIndicesCacheAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestClearIndicesCacheAction.java
@@ -24,19 +24,17 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.action.RestBuilderListener;
 
+import java.io.IOException;
 import java.util.Map;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
@@ -45,8 +43,6 @@
 import static org.elasticsearch.rest.action.RestActions.buildBroadcastShardsHeader;
 
 public class RestClearIndicesCacheAction extends BaseRestHandler {
-
-    @Inject
     public RestClearIndicesCacheAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(POST, "/_cache/clear", this);
@@ -57,12 +53,13 @@ public RestClearIndicesCacheAction(Settings settings, RestController controller)
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         ClearIndicesCacheRequest clearIndicesCacheRequest = new ClearIndicesCacheRequest(
                 Strings.splitStringByCommaToArray(request.param("index")));
         clearIndicesCacheRequest.indicesOptions(IndicesOptions.fromRequest(request, clearIndicesCacheRequest.indicesOptions()));
-        fromRequest(request, clearIndicesCacheRequest, parseFieldMatcher);
-        client.admin().indices().clearCache(clearIndicesCacheRequest, new RestBuilderListener<ClearIndicesCacheResponse>(channel) {
+        fromRequest(request, clearIndicesCacheRequest);
+        return channel ->
+                client.admin().indices().clearCache(clearIndicesCacheRequest, new RestBuilderListener<ClearIndicesCacheResponse>(channel) {
             @Override
             public RestResponse buildResponse(ClearIndicesCacheResponse response, XContentBuilder builder) throws Exception {
                 builder.startObject();
@@ -78,20 +75,22 @@ public boolean canTripCircuitBreaker() {
         return false;
     }
 
-    public static ClearIndicesCacheRequest fromRequest(final RestRequest request, ClearIndicesCacheRequest clearIndicesCacheRequest,
-            ParseFieldMatcher parseFieldMatcher) {
+    public static ClearIndicesCacheRequest fromRequest(final RestRequest request, ClearIndicesCacheRequest clearIndicesCacheRequest) {
 
         for (Map.Entry<String, String> entry : request.params().entrySet()) {
-            if (parseFieldMatcher.match(entry.getKey(), Fields.QUERY)) {
+            if (Fields.QUERY.match(entry.getKey())) {
                 clearIndicesCacheRequest.queryCache(request.paramAsBoolean(entry.getKey(), clearIndicesCacheRequest.queryCache()));
             }
-            if (parseFieldMatcher.match(entry.getKey(), Fields.FIELD_DATA)) {
+            if (Fields.REQUEST.match(entry.getKey())) {
+                clearIndicesCacheRequest.requestCache(request.paramAsBoolean(entry.getKey(), clearIndicesCacheRequest.requestCache()));
+            }
+            if (Fields.FIELD_DATA.match(entry.getKey())) {
                 clearIndicesCacheRequest.fieldDataCache(request.paramAsBoolean(entry.getKey(), clearIndicesCacheRequest.fieldDataCache()));
             }
-            if (parseFieldMatcher.match(entry.getKey(), Fields.RECYCLER)) {
+            if (Fields.RECYCLER.match(entry.getKey())) {
                 clearIndicesCacheRequest.recycler(request.paramAsBoolean(entry.getKey(), clearIndicesCacheRequest.recycler()));
             }
-            if (parseFieldMatcher.match(entry.getKey(), Fields.FIELDS)) {
+            if (Fields.FIELDS.match(entry.getKey())) {
                 clearIndicesCacheRequest.fields(request.paramAsStringArray(entry.getKey(), clearIndicesCacheRequest.fields()));
             }
         }
@@ -101,6 +100,7 @@ public static ClearIndicesCacheRequest fromRequest(final RestRequest request, Cl
 
     public static class Fields {
         public static final ParseField QUERY = new ParseField("query", "filter", "filter_cache");
+        public static final ParseField REQUEST = new ParseField("request", "request_cache");
         public static final ParseField FIELD_DATA = new ParseField("field_data", "fielddata");
         public static final ParseField RECYCLER = new ParseField("recycler");
         public static final ParseField FIELDS = new ParseField("fields");
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCloseIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCloseIndexAction.java
index e5baa27f4eccb..2e0a46747ca77 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCloseIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCloseIndexAction.java
@@ -20,24 +20,18 @@
 package org.elasticsearch.rest.action.admin.indices;
 
 import org.elasticsearch.action.admin.indices.close.CloseIndexRequest;
-import org.elasticsearch.action.admin.indices.close.CloseIndexResponse;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.AcknowledgedRestListener;
 
-/**
- *
- */
-public class RestCloseIndexAction extends BaseRestHandler {
+import java.io.IOException;
 
-    @Inject
+public class RestCloseIndexAction extends BaseRestHandler {
     public RestCloseIndexAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(RestRequest.Method.POST, "/_close", this);
@@ -45,11 +39,12 @@ public RestCloseIndexAction(Settings settings, RestController controller) {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         CloseIndexRequest closeIndexRequest = new CloseIndexRequest(Strings.splitStringByCommaToArray(request.param("index")));
         closeIndexRequest.masterNodeTimeout(request.paramAsTime("master_timeout", closeIndexRequest.masterNodeTimeout()));
         closeIndexRequest.timeout(request.paramAsTime("timeout", closeIndexRequest.timeout()));
         closeIndexRequest.indicesOptions(IndicesOptions.fromRequest(request, closeIndexRequest.indicesOptions()));
-        client.admin().indices().close(closeIndexRequest, new AcknowledgedRestListener<CloseIndexResponse>(channel));
+        return channel -> client.admin().indices().close(closeIndexRequest, new AcknowledgedRestListener<>(channel));
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java
index 2a7f2a629a7f2..8c314efee634a 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java
@@ -21,42 +21,34 @@
 
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
-import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.action.support.ActiveShardCount;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.AcknowledgedRestListener;
 
 import java.io.IOException;
 
-/**
- *
- */
 public class RestCreateIndexAction extends BaseRestHandler {
-
-    @Inject
     public RestCreateIndexAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(RestRequest.Method.PUT, "/{index}", this);
     }
 
-    @SuppressWarnings({"unchecked"})
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         CreateIndexRequest createIndexRequest = new CreateIndexRequest(request.param("index"));
         if (request.hasContent()) {
-            createIndexRequest.source(request.content());
+            createIndexRequest.source(request.content(), request.getXContentType());
         }
         createIndexRequest.updateAllTypes(request.paramAsBoolean("update_all_types", false));
         createIndexRequest.timeout(request.paramAsTime("timeout", createIndexRequest.timeout()));
         createIndexRequest.masterNodeTimeout(request.paramAsTime("master_timeout", createIndexRequest.masterNodeTimeout()));
         createIndexRequest.waitForActiveShards(ActiveShardCount.parseString(request.param("wait_for_active_shards")));
-        client.admin().indices().create(createIndexRequest, new AcknowledgedRestListener<CreateIndexResponse>(channel) {
+        return channel -> client.admin().indices().create(createIndexRequest, new AcknowledgedRestListener<CreateIndexResponse>(channel) {
             @Override
             public void addCustomFields(XContentBuilder builder, CreateIndexResponse response) throws IOException {
                 response.addCustomFields(builder);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestDeleteIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestDeleteIndexAction.java
index d3e07effc1465..6ca806bcffe01 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestDeleteIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestDeleteIndexAction.java
@@ -20,24 +20,18 @@
 package org.elasticsearch.rest.action.admin.indices;
 
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
-import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.AcknowledgedRestListener;
 
-/**
- *
- */
-public class RestDeleteIndexAction extends BaseRestHandler {
+import java.io.IOException;
 
-    @Inject
+public class RestDeleteIndexAction extends BaseRestHandler {
     public RestDeleteIndexAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(RestRequest.Method.DELETE, "/", this);
@@ -45,11 +39,11 @@ public RestDeleteIndexAction(Settings settings, RestController controller) {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(Strings.splitStringByCommaToArray(request.param("index")));
         deleteIndexRequest.timeout(request.paramAsTime("timeout", deleteIndexRequest.timeout()));
         deleteIndexRequest.masterNodeTimeout(request.paramAsTime("master_timeout", deleteIndexRequest.masterNodeTimeout()));
         deleteIndexRequest.indicesOptions(IndicesOptions.fromRequest(request, deleteIndexRequest.indicesOptions()));
-        client.admin().indices().delete(deleteIndexRequest, new AcknowledgedRestListener<DeleteIndexResponse>(channel));
+        return channel -> client.admin().indices().delete(deleteIndexRequest, new AcknowledgedRestListener<>(channel));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestDeleteIndexTemplateAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestDeleteIndexTemplateAction.java
index 425581fe92795..e9f315e8aee0f 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestDeleteIndexTemplateAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestDeleteIndexTemplateAction.java
@@ -20,26 +20,24 @@
 
 import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.AcknowledgedRestListener;
 
-public class RestDeleteIndexTemplateAction extends BaseRestHandler {
+import java.io.IOException;
 
-    @Inject
+public class RestDeleteIndexTemplateAction extends BaseRestHandler {
     public RestDeleteIndexTemplateAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(RestRequest.Method.DELETE, "/_template/{name}", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         DeleteIndexTemplateRequest deleteIndexTemplateRequest = new DeleteIndexTemplateRequest(request.param("name"));
         deleteIndexTemplateRequest.masterNodeTimeout(request.paramAsTime("master_timeout", deleteIndexTemplateRequest.masterNodeTimeout()));
-        client.admin().indices().deleteTemplate(deleteIndexTemplateRequest, new AcknowledgedRestListener<>(channel));
+        return channel -> client.admin().indices().deleteTemplate(deleteIndexTemplateRequest, new AcknowledgedRestListener<>(channel));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestFlushAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestFlushAction.java
index b963a805934e4..00a91e36c5eb2 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestFlushAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestFlushAction.java
@@ -24,17 +24,17 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.action.RestBuilderListener;
 
+import java.io.IOException;
+
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.rest.RestStatus.OK;
@@ -44,8 +44,6 @@
  *
  */
 public class RestFlushAction extends BaseRestHandler {
-
-    @Inject
     public RestFlushAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(POST, "/_flush", this);
@@ -56,12 +54,12 @@ public RestFlushAction(Settings settings, RestController controller) {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         FlushRequest flushRequest = new FlushRequest(Strings.splitStringByCommaToArray(request.param("index")));
         flushRequest.indicesOptions(IndicesOptions.fromRequest(request, flushRequest.indicesOptions()));
         flushRequest.force(request.paramAsBoolean("force", flushRequest.force()));
         flushRequest.waitIfOngoing(request.paramAsBoolean("wait_if_ongoing", flushRequest.waitIfOngoing()));
-        client.admin().indices().flush(flushRequest, new RestBuilderListener<FlushResponse>(channel) {
+        return channel -> client.admin().indices().flush(flushRequest, new RestBuilderListener<FlushResponse>(channel) {
             @Override
             public RestResponse buildResponse(FlushResponse response, XContentBuilder builder) throws Exception {
                 builder.startObject();
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestForceMergeAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestForceMergeAction.java
index c376866ad1e46..e6fdfbee30a34 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestForceMergeAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestForceMergeAction.java
@@ -24,17 +24,17 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.action.RestBuilderListener;
 
+import java.io.IOException;
+
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.rest.RestStatus.OK;
 import static org.elasticsearch.rest.action.RestActions.buildBroadcastShardsHeader;
@@ -43,8 +43,6 @@
  *
  */
 public class RestForceMergeAction extends BaseRestHandler {
-
-    @Inject
     public RestForceMergeAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(POST, "/_forcemerge", this);
@@ -52,13 +50,13 @@ public RestForceMergeAction(Settings settings, RestController controller) {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         ForceMergeRequest mergeRequest = new ForceMergeRequest(Strings.splitStringByCommaToArray(request.param("index")));
         mergeRequest.indicesOptions(IndicesOptions.fromRequest(request, mergeRequest.indicesOptions()));
         mergeRequest.maxNumSegments(request.paramAsInt("max_num_segments", mergeRequest.maxNumSegments()));
         mergeRequest.onlyExpungeDeletes(request.paramAsBoolean("only_expunge_deletes", mergeRequest.onlyExpungeDeletes()));
         mergeRequest.flush(request.paramAsBoolean("flush", mergeRequest.flush()));
-        client.admin().indices().forceMerge(mergeRequest, new RestBuilderListener<ForceMergeResponse>(channel) {
+        return channel -> client.admin().indices().forceMerge(mergeRequest, new RestBuilderListener<ForceMergeResponse>(channel) {
             @Override
             public RestResponse buildResponse(ForceMergeResponse response, XContentBuilder builder) throws Exception {
                 builder.startObject();
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java
index 4b58fd0f16cb6..62c7fbaa30508 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.rest.action.admin.indices;
 
+import com.carrotsearch.hppc.cursors.ObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
@@ -26,93 +27,131 @@
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.metadata.AliasMetaData;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.RestBuilderListener;
 
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.stream.Collectors;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
-import static org.elasticsearch.rest.RestStatus.OK;
+import static org.elasticsearch.rest.RestRequest.Method.HEAD;
 
-/**
+/*
+ * The REST handler for get alias and head alias APIs.
  */
 public class RestGetAliasesAction extends BaseRestHandler {
 
-    @Inject
-    public RestGetAliasesAction(Settings settings, RestController controller) {
+    public RestGetAliasesAction(final Settings settings, final RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_alias/{name}", this);
+        controller.registerHandler(HEAD, "/_alias/{name}", this);
         controller.registerHandler(GET, "/{index}/_alias/{name}", this);
+        controller.registerHandler(HEAD, "/{index}/_alias/{name}", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         final String[] aliases = request.paramAsStringArrayOrEmptyIfAll("name");
-        final String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
         final GetAliasesRequest getAliasesRequest = new GetAliasesRequest(aliases);
+        final String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
         getAliasesRequest.indices(indices);
         getAliasesRequest.indicesOptions(IndicesOptions.fromRequest(request, getAliasesRequest.indicesOptions()));
         getAliasesRequest.local(request.paramAsBoolean("local", getAliasesRequest.local()));
 
-        client.admin().indices().getAliases(getAliasesRequest, new RestBuilderListener<GetAliasesResponse>(channel) {
+        return channel -> client.admin().indices().getAliases(getAliasesRequest, new RestBuilderListener<GetAliasesResponse>(channel) {
             @Override
             public RestResponse buildResponse(GetAliasesResponse response, XContentBuilder builder) throws Exception {
-                // empty body, if indices were specified but no aliases were
-                if (indices.length > 0 && response.getAliases().isEmpty()) {
-                    return new BytesRestResponse(OK, builder.startObject().endObject());
-                } else if (response.getAliases().isEmpty()) {
-                    String message = String.format(Locale.ROOT, "alias [%s] missing", toNamesString(getAliasesRequest.aliases()));
-                    builder.startObject()
-                            .field("error", message)
-                            .field("status", RestStatus.NOT_FOUND.getStatus())
-                            .endObject();
-                    return new BytesRestResponse(RestStatus.NOT_FOUND, builder);
+                final ImmutableOpenMap<String, List<AliasMetaData>> aliasMap = response.getAliases();
+
+                final Set<String> aliasNames = new HashSet<>();
+                for (final ObjectCursor<List<AliasMetaData>> cursor : aliasMap.values()) {
+                    for (final AliasMetaData aliasMetaData : cursor.value) {
+                        aliasNames.add(aliasMetaData.alias());
+                    }
+                }
+
+                // first remove requested aliases that are exact matches
+                final SortedSet<String> difference = Sets.sortedDifference(Arrays.stream(aliases).collect(Collectors.toSet()), aliasNames);
+
+                // now remove requested aliases that contain wildcards that are simple matches
+                final List<String> matches = new ArrayList<>();
+                outer:
+                for (final String pattern : difference) {
+                    if (pattern.contains("*")) {
+                        for (final String aliasName : aliasNames) {
+                            if (Regex.simpleMatch(pattern, aliasName)) {
+                                matches.add(pattern);
+                                continue outer;
+                            }
+                        }
+                    }
                 }
+                difference.removeAll(matches);
 
+                final RestStatus status;
                 builder.startObject();
-                for (ObjectObjectCursor<String, List<AliasMetaData>> entry : response.getAliases()) {
-                    builder.startObject(entry.key);
-                    builder.startObject(Fields.ALIASES);
-                    for (AliasMetaData alias : entry.value) {
-                        AliasMetaData.Builder.toXContent(alias, builder, ToXContent.EMPTY_PARAMS);
+                {
+                    if (difference.isEmpty()) {
+                        status = RestStatus.OK;
+                    } else {
+                        status = RestStatus.NOT_FOUND;
+                        final String message;
+                        if (difference.size() == 1) {
+                            message = String.format(Locale.ROOT, "alias [%s] missing", toNamesString(difference.iterator().next()));
+                        } else {
+                            message = String.format(Locale.ROOT, "aliases [%s] missing", toNamesString(difference.toArray(new String[0])));
+                        }
+                        builder.field("error", message);
+                        builder.field("status", status.getStatus());
+                    }
+
+                    for (final ObjectObjectCursor<String, List<AliasMetaData>> entry : response.getAliases()) {
+                        builder.startObject(entry.key);
+                        {
+                            builder.startObject("aliases");
+                            {
+                                for (final AliasMetaData alias : entry.value) {
+                                    AliasMetaData.Builder.toXContent(alias, builder, ToXContent.EMPTY_PARAMS);
+                                }
+                            }
+                            builder.endObject();
+                        }
+                        builder.endObject();
                     }
-                    builder.endObject();
-                    builder.endObject();
                 }
                 builder.endObject();
-                return new BytesRestResponse(OK, builder);
+                return new BytesRestResponse(status, builder);
             }
+
         });
     }
 
-    private static String toNamesString(String... names) {
+    private static String toNamesString(final String... names) {
         if (names == null || names.length == 0) {
             return "";
         } else if (names.length == 1) {
             return names[0];
         } else {
-            StringBuilder builder = new StringBuilder(names[0]);
-            for (int i = 1; i < names.length; i++) {
-                builder.append(',').append(names[i]);
-            }
-            return builder.toString();
+            return Arrays.stream(names).collect(Collectors.joining(","));
         }
     }
 
-    static class Fields {
-
-        static final String ALIASES = "aliases";
-
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java
index fba4192d5d4e6..fe6eb9a552f32 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java
@@ -25,18 +25,17 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.RestBuilderListener;
 
+import java.io.IOException;
 import java.util.Map;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
@@ -44,8 +43,6 @@
 import static org.elasticsearch.rest.RestStatus.OK;
 
 public class RestGetFieldMappingAction extends BaseRestHandler {
-
-    @Inject
     public RestGetFieldMappingAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_mapping/field/{fields}", this);
@@ -56,7 +53,7 @@ public RestGetFieldMappingAction(Settings settings, RestController controller) {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         final String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
         final String[] types = request.paramAsStringArrayOrEmptyIfAll("type");
         final String[] fields = Strings.splitStringByCommaToArray(request.param("fields"));
@@ -64,26 +61,27 @@ public void handleRequest(final RestRequest request, final RestChannel channel,
         getMappingsRequest.indices(indices).types(types).fields(fields).includeDefaults(request.paramAsBoolean("include_defaults", false));
         getMappingsRequest.indicesOptions(IndicesOptions.fromRequest(request, getMappingsRequest.indicesOptions()));
         getMappingsRequest.local(request.paramAsBoolean("local", getMappingsRequest.local()));
-        client.admin().indices().getFieldMappings(getMappingsRequest, new RestBuilderListener<GetFieldMappingsResponse>(channel) {
-            @Override
-            public RestResponse buildResponse(GetFieldMappingsResponse response, XContentBuilder builder) throws Exception {
-                Map<String, Map<String, Map<String, FieldMappingMetaData>>> mappingsByIndex = response.mappings();
+        return channel ->
+                client.admin().indices().getFieldMappings(getMappingsRequest, new RestBuilderListener<GetFieldMappingsResponse>(channel) {
+                    @Override
+                    public RestResponse buildResponse(GetFieldMappingsResponse response, XContentBuilder builder) throws Exception {
+                        Map<String, Map<String, Map<String, FieldMappingMetaData>>> mappingsByIndex = response.mappings();
 
-                boolean isPossibleSingleFieldRequest = indices.length == 1 && types.length == 1 && fields.length == 1;
-                if (isPossibleSingleFieldRequest && isFieldMappingMissingField(mappingsByIndex)) {
-                    return new BytesRestResponse(OK, builder.startObject().endObject());
-                }
+                        boolean isPossibleSingleFieldRequest = indices.length == 1 && types.length == 1 && fields.length == 1;
+                        if (isPossibleSingleFieldRequest && isFieldMappingMissingField(mappingsByIndex)) {
+                            return new BytesRestResponse(OK, builder.startObject().endObject());
+                        }
 
-                RestStatus status = OK;
-                if (mappingsByIndex.isEmpty() && fields.length > 0) {
-                    status = NOT_FOUND;
-                }
-                builder.startObject();
-                response.toXContent(builder, request);
-                builder.endObject();
-                return new BytesRestResponse(status, builder);
-            }
-        });
+                        RestStatus status = OK;
+                        if (mappingsByIndex.isEmpty() && fields.length > 0) {
+                            status = NOT_FOUND;
+                        }
+                        builder.startObject();
+                        response.toXContent(builder, request);
+                        builder.endObject();
+                        return new BytesRestResponse(status, builder);
+                    }
+                });
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java
index ead04532590fe..1814894636f77 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java
@@ -16,52 +16,65 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+
 package org.elasticsearch.rest.action.admin.indices;
 
 import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesRequest;
 import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.RestToXContentListener;
 
+import java.io.IOException;
+import java.util.Set;
+
 import static org.elasticsearch.rest.RestRequest.Method.GET;
+import static org.elasticsearch.rest.RestRequest.Method.HEAD;
 import static org.elasticsearch.rest.RestStatus.NOT_FOUND;
 import static org.elasticsearch.rest.RestStatus.OK;
 
+/**
+ * The REST handler for get template and head template APIs.
+ */
 public class RestGetIndexTemplateAction extends BaseRestHandler {
 
-    @Inject
-    public RestGetIndexTemplateAction(Settings settings, RestController controller) {
+    public RestGetIndexTemplateAction(final Settings settings, final RestController controller) {
         super(settings);
-
         controller.registerHandler(GET, "/_template", this);
         controller.registerHandler(GET, "/_template/{name}", this);
+        controller.registerHandler(HEAD, "/_template/{name}", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         final String[] names = Strings.splitStringByCommaToArray(request.param("name"));
 
-        GetIndexTemplatesRequest getIndexTemplatesRequest = new GetIndexTemplatesRequest(names);
+        final GetIndexTemplatesRequest getIndexTemplatesRequest = new GetIndexTemplatesRequest(names);
         getIndexTemplatesRequest.local(request.paramAsBoolean("local", getIndexTemplatesRequest.local()));
         getIndexTemplatesRequest.masterNodeTimeout(request.paramAsTime("master_timeout", getIndexTemplatesRequest.masterNodeTimeout()));
 
         final boolean implicitAll = getIndexTemplatesRequest.names().length == 0;
 
-        client.admin().indices().getTemplates(getIndexTemplatesRequest, new RestToXContentListener<GetIndexTemplatesResponse>(channel) {
-            @Override
-            protected RestStatus getStatus(GetIndexTemplatesResponse response) {
-                boolean templateExists = false == response.getIndexTemplates().isEmpty();
+        return channel ->
+                client.admin()
+                        .indices()
+                        .getTemplates(getIndexTemplatesRequest, new RestToXContentListener<GetIndexTemplatesResponse>(channel) {
+                            @Override
+                            protected RestStatus getStatus(final GetIndexTemplatesResponse response) {
+                                final boolean templateExists = response.getIndexTemplates().isEmpty() == false;
+                                return (templateExists || implicitAll) ? OK : NOT_FOUND;
+                            }
+                        });
+    }
 
-                return (templateExists || implicitAll) ? OK : NOT_FOUND;
-            }
-        });
+    @Override
+    protected Set<String> responseParams() {
+        return Settings.FORMAT_PARAMS;
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesAction.java
index 2ad6f245cf0ab..bbca5c402be35 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesAction.java
@@ -16,9 +16,11 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+
 package org.elasticsearch.rest.action.admin.indices;
 
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
+
 import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
 import org.elasticsearch.action.admin.indices.get.GetIndexRequest.Feature;
 import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
@@ -28,7 +30,6 @@
 import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
@@ -36,7 +37,6 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -44,33 +44,45 @@
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Set;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
+import static org.elasticsearch.rest.RestRequest.Method.HEAD;
 import static org.elasticsearch.rest.RestStatus.OK;
 
+/**
+ * The REST handler for get index and head index APIs.
+ */
 public class RestGetIndicesAction extends BaseRestHandler {
 
     private final IndexScopedSettings indexScopedSettings;
     private final SettingsFilter settingsFilter;
 
-    @Inject
-    public RestGetIndicesAction(Settings settings, RestController controller, IndexScopedSettings indexScopedSettings,
-            SettingsFilter settingsFilter) {
+    public RestGetIndicesAction(
+            final Settings settings,
+            final RestController controller,
+            final IndexScopedSettings indexScopedSettings,
+            final SettingsFilter settingsFilter) {
         super(settings);
         this.indexScopedSettings = indexScopedSettings;
         controller.registerHandler(GET, "/{index}", this);
+        controller.registerHandler(HEAD, "/{index}", this);
         controller.registerHandler(GET, "/{index}/{type}", this);
         this.settingsFilter = settingsFilter;
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
         String[] featureParams = request.paramAsStringArray("type", null);
+        if (featureParams != null && featureParams.length > 1) {
+            deprecationLogger.deprecated("Requesting comma-separated features is deprecated and " +
+                            "will be removed in 6.0+, retrieve all features instead.");
+        }
         // Work out if the indices is a list of features
         if (featureParams == null && indices.length > 0 && indices[0] != null && indices[0].startsWith("_") && !"_all".equals(indices[0])) {
             featureParams = indices;
-            indices = new String[] {"_all"};
+            indices = new String[]{"_all"};
         }
         final GetIndexRequest getIndexRequest = new GetIndexRequest();
         getIndexRequest.indices(indices);
@@ -81,70 +93,87 @@ public void handleRequest(final RestRequest request, final RestChannel channel,
         getIndexRequest.indicesOptions(IndicesOptions.fromRequest(request, getIndexRequest.indicesOptions()));
         getIndexRequest.local(request.paramAsBoolean("local", getIndexRequest.local()));
         getIndexRequest.humanReadable(request.paramAsBoolean("human", false));
-        client.admin().indices().getIndex(getIndexRequest, new RestBuilderListener<GetIndexResponse>(channel) {
+        final boolean defaults = request.paramAsBoolean("include_defaults", false);
+        return channel -> client.admin().indices().getIndex(getIndexRequest, new RestBuilderListener<GetIndexResponse>(channel) {
 
             @Override
-            public RestResponse buildResponse(GetIndexResponse response, XContentBuilder builder) throws Exception {
-                Feature[] features = getIndexRequest.features();
-                String[] indices = response.indices();
-
+            public RestResponse buildResponse(final GetIndexResponse response, final XContentBuilder builder) throws Exception {
                 builder.startObject();
-                for (String index : indices) {
-                    builder.startObject(index);
-                    for (Feature feature : features) {
-                        switch (feature) {
-                        case ALIASES:
-                            writeAliases(response.aliases().get(index), builder, request);
-                            break;
-                        case MAPPINGS:
-                            writeMappings(response.mappings().get(index), builder, request);
-                            break;
-                        case SETTINGS:
-                            writeSettings(response.settings().get(index), builder, request);
-                            break;
-                        default:
-                            throw new IllegalStateException("feature [" + feature + "] is not valid");
+                {
+                    for (final String index : response.indices()) {
+                        builder.startObject(index);
+                        {
+                            for (final Feature feature : getIndexRequest.features()) {
+                                switch (feature) {
+                                    case ALIASES:
+                                        writeAliases(response.aliases().get(index), builder, request);
+                                        break;
+                                    case MAPPINGS:
+                                        writeMappings(response.mappings().get(index), builder);
+                                        break;
+                                    case SETTINGS:
+                                        writeSettings(response.settings().get(index), builder, request, defaults);
+                                        break;
+                                    default:
+                                        throw new IllegalStateException("feature [" + feature + "] is not valid");
+                                }
+                            }
                         }
-                    }
-                    builder.endObject();
+                        builder.endObject();
 
+                    }
                 }
                 builder.endObject();
 
                 return new BytesRestResponse(OK, builder);
             }
 
-            private void writeAliases(List<AliasMetaData> aliases, XContentBuilder builder, Params params) throws IOException {
-                builder.startObject(Fields.ALIASES);
-                if (aliases != null) {
-                    for (AliasMetaData alias : aliases) {
-                        AliasMetaData.Builder.toXContent(alias, builder, params);
+            private void writeAliases(
+                    final List<AliasMetaData> aliases,
+                    final XContentBuilder builder,
+                    final Params params) throws IOException {
+                builder.startObject("aliases");
+                {
+                    if (aliases != null) {
+                        for (final AliasMetaData alias : aliases) {
+                            AliasMetaData.Builder.toXContent(alias, builder, params);
+                        }
                     }
                 }
                 builder.endObject();
             }
 
-            private void writeMappings(ImmutableOpenMap<String, MappingMetaData> mappings, XContentBuilder builder, Params params)
+            private void writeMappings(final ImmutableOpenMap<String, MappingMetaData> mappings, final XContentBuilder builder)
                     throws IOException {
-                builder.startObject(Fields.MAPPINGS);
-                if (mappings != null) {
-                    for (ObjectObjectCursor<String, MappingMetaData> typeEntry : mappings) {
-                        builder.field(typeEntry.key);
-                        builder.map(typeEntry.value.sourceAsMap());
+                builder.startObject("mappings");
+                {
+                    if (mappings != null) {
+                        for (final ObjectObjectCursor<String, MappingMetaData> typeEntry : mappings) {
+                            builder.field(typeEntry.key);
+                            builder.map(typeEntry.value.sourceAsMap());
+                        }
                     }
                 }
                 builder.endObject();
             }
 
-            private void writeSettings(Settings settings, XContentBuilder builder, Params params) throws IOException {
-                final boolean renderDefaults = request.paramAsBoolean("include_defaults", false);
-                builder.startObject(Fields.SETTINGS);
-                settings.toXContent(builder, params);
+            private void writeSettings(
+                    final Settings settings,
+                    final XContentBuilder builder,
+                    final Params params,
+                    final boolean defaults) throws IOException {
+                builder.startObject("settings");
+                {
+                    settings.toXContent(builder, params);
+                }
                 builder.endObject();
-                if (renderDefaults) {
+                if (defaults) {
                     builder.startObject("defaults");
-                    settingsFilter.filter(indexScopedSettings.diff(settings, RestGetIndicesAction.this.settings)).toXContent(builder,
-                            request);
+                    {
+                        settingsFilter
+                                .filter(indexScopedSettings.diff(settings, RestGetIndicesAction.this.settings))
+                                .toXContent(builder, request);
+                    }
                     builder.endObject();
                 }
             }
@@ -152,10 +181,9 @@ private void writeSettings(Settings settings, XContentBuilder builder, Params pa
         });
     }
 
-    static class Fields {
-        static final String ALIASES = "aliases";
-        static final String MAPPINGS = "mappings";
-        static final String SETTINGS = "settings";
+    @Override
+    protected Set<String> responseParams() {
+        return Settings.FORMAT_PARAMS;
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java
index b9c8472964074..aaf78b6cd20d6 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.rest.action.admin.indices;
 
+import com.carrotsearch.hppc.cursors.ObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
@@ -28,20 +29,32 @@
 import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.indices.TypeMissingException;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
+import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.RestBuilderListener;
 
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.stream.Collectors;
+
 import static org.elasticsearch.rest.RestRequest.Method.GET;
+import static org.elasticsearch.rest.RestRequest.Method.HEAD;
 import static org.elasticsearch.rest.RestStatus.OK;
 
 /**
@@ -49,61 +62,103 @@
  */
 public class RestGetMappingAction extends BaseRestHandler {
 
-    @Inject
-    public RestGetMappingAction(Settings settings, RestController controller) {
+    public RestGetMappingAction(final Settings settings, final RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/{index}/{type}/_mapping", this);
         controller.registerHandler(GET, "/{index}/_mappings/{type}", this);
         controller.registerHandler(GET, "/{index}/_mapping/{type}", this);
+        controller.registerHandler(HEAD, "/{index}/_mapping/{type}", this);
         controller.registerHandler(GET, "/_mapping/{type}", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         final String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
         final String[] types = request.paramAsStringArrayOrEmptyIfAll("type");
-        GetMappingsRequest getMappingsRequest = new GetMappingsRequest();
+        final GetMappingsRequest getMappingsRequest = new GetMappingsRequest();
         getMappingsRequest.indices(indices).types(types);
         getMappingsRequest.indicesOptions(IndicesOptions.fromRequest(request, getMappingsRequest.indicesOptions()));
         getMappingsRequest.local(request.paramAsBoolean("local", getMappingsRequest.local()));
-        client.admin().indices().getMappings(getMappingsRequest, new RestBuilderListener<GetMappingsResponse>(channel) {
+        return channel -> client.admin().indices().getMappings(getMappingsRequest, new RestBuilderListener<GetMappingsResponse>(channel) {
             @Override
-            public RestResponse buildResponse(GetMappingsResponse response, XContentBuilder builder) throws Exception {
-                builder.startObject();
-                ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappingsByIndex = response.getMappings();
-                if (mappingsByIndex.isEmpty()) {
-                    if (indices.length != 0 && types.length != 0) {
-                        return new BytesRestResponse(OK, builder.endObject());
-                    } else if (indices.length != 0) {
-                        return new BytesRestResponse(channel, new IndexNotFoundException(indices[0]));
-                    } else if (types.length != 0) {
-                        return new BytesRestResponse(channel, new TypeMissingException("_all", types[0]));
+            public RestResponse buildResponse(final GetMappingsResponse response, final XContentBuilder builder) throws Exception {
+                final ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappingsByIndex = response.getMappings();
+                if (mappingsByIndex.isEmpty() && (indices.length != 0 || types.length != 0)) {
+                    if (indices.length != 0 && types.length == 0) {
+                        builder.close();
+                        return new BytesRestResponse(channel, new IndexNotFoundException(String.join(",", indices)));
                     } else {
-                        return new BytesRestResponse(OK, builder.endObject());
+                        builder.close();
+                        return new BytesRestResponse(channel, new TypeMissingException("_all", String.join(",", types)));
                     }
                 }
 
-                for (ObjectObjectCursor<String, ImmutableOpenMap<String, MappingMetaData>> indexEntry : mappingsByIndex) {
-                    if (indexEntry.value.isEmpty()) {
-                        continue;
+                final Set<String> typeNames = new HashSet<>();
+                for (final ObjectCursor<ImmutableOpenMap<String, MappingMetaData>> cursor : mappingsByIndex.values()) {
+                    for (final ObjectCursor<String> inner : cursor.value.keys()) {
+                        typeNames.add(inner.value);
                     }
-                    builder.startObject(indexEntry.key);
-                    builder.startObject(Fields.MAPPINGS);
-                    for (ObjectObjectCursor<String, MappingMetaData> typeEntry : indexEntry.value) {
-                        builder.field(typeEntry.key);
-                        builder.map(typeEntry.value.sourceAsMap());
+                }
+
+                final SortedSet<String> difference = Sets.sortedDifference(Arrays.stream(types).collect(Collectors.toSet()), typeNames);
+
+                // now remove requested aliases that contain wildcards that are simple matches
+                final List<String> matches = new ArrayList<>();
+                outer:
+                for (final String pattern : difference) {
+                    if (pattern.contains("*")) {
+                        for (final String typeName : typeNames) {
+                            if (Regex.simpleMatch(pattern, typeName)) {
+                                matches.add(pattern);
+                                continue outer;
+                            }
+                        }
                     }
-                    builder.endObject();
-                    builder.endObject();
                 }
+                difference.removeAll(matches);
+
+                final RestStatus status;
+                builder.startObject();
+                {
+                    if (difference.isEmpty()) {
+                        status = RestStatus.OK;
+                    } else {
+                        status = RestStatus.NOT_FOUND;
+                        final String message;
+                        if (difference.size() == 1) {
+                            message = String.format(Locale.ROOT, "type [%s] missing", toNamesString(difference.iterator().next()));
+                        } else {
+                            message = String.format(Locale.ROOT, "types [%s] missing", toNamesString(difference.toArray(new String[0])));
+                        }
+                        builder.field("error", message);
+                        builder.field("status", status.getStatus());
+                    }
 
+                    for (final ObjectObjectCursor<String, ImmutableOpenMap<String, MappingMetaData>> indexEntry : mappingsByIndex) {
+                        if (indexEntry.value.isEmpty()) {
+                            continue;
+                        }
+                        builder.startObject(indexEntry.key);
+                        {
+                            builder.startObject("mappings");
+                            {
+                                for (final ObjectObjectCursor<String, MappingMetaData> typeEntry : indexEntry.value) {
+                                    builder.field(typeEntry.key, typeEntry.value.sourceAsMap());
+                                }
+                            }
+                            builder.endObject();
+                        }
+                        builder.endObject();
+                    }
+                }
                 builder.endObject();
-                return new BytesRestResponse(OK, builder);
+                return new BytesRestResponse(status, builder);
             }
         });
     }
 
-    static class Fields {
-        static final String MAPPINGS = "mappings";
+    private static String toNamesString(final String... names) {
+        return Arrays.stream(names).collect(Collectors.joining(","));
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetSettingsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetSettingsAction.java
index 936a96e035afb..fcad131a3599a 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetSettingsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetSettingsAction.java
@@ -20,24 +20,25 @@
 package org.elasticsearch.rest.action.admin.indices;
 
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
+
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.action.RestBuilderListener;
 
+import java.io.IOException;
+
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestStatus.OK;
 
@@ -46,7 +47,6 @@ public class RestGetSettingsAction extends BaseRestHandler {
     private final IndexScopedSettings indexScopedSettings;
     private final SettingsFilter settingsFilter;
 
-    @Inject
     public RestGetSettingsAction(Settings settings, RestController controller, IndexScopedSettings indexScopedSettings,
             final SettingsFilter settingsFilter) {
         super(settings);
@@ -58,7 +58,7 @@ public RestGetSettingsAction(Settings settings, RestController controller, Index
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         final String[] names = request.paramAsStringArrayOrEmptyIfAll("name");
         final boolean renderDefaults = request.paramAsBoolean("include_defaults", false);
         GetSettingsRequest getSettingsRequest = new GetSettingsRequest()
@@ -68,14 +68,14 @@ public void handleRequest(final RestRequest request, final RestChannel channel,
                 .names(names);
         getSettingsRequest.local(request.paramAsBoolean("local", getSettingsRequest.local()));
 
-        client.admin().indices().getSettings(getSettingsRequest, new RestBuilderListener<GetSettingsResponse>(channel) {
+        return channel -> client.admin().indices().getSettings(getSettingsRequest, new RestBuilderListener<GetSettingsResponse>(channel) {
 
             @Override
             public RestResponse buildResponse(GetSettingsResponse getSettingsResponse, XContentBuilder builder) throws Exception {
                 builder.startObject();
                 for (ObjectObjectCursor<String, Settings> cursor : getSettingsResponse.getIndexToSettings()) {
                     // no settings, jump over it to shorten the response data
-                    if (cursor.value.getAsMap().isEmpty()) {
+                    if (cursor.value.isEmpty()) {
                         continue;
                     }
                     builder.startObject(cursor.key);
@@ -94,4 +94,5 @@ public RestResponse buildResponse(GetSettingsResponse getSettingsResponse, XCont
             }
         });
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestHeadIndexTemplateAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestHeadIndexTemplateAction.java
deleted file mode 100644
index 3480fbb9afc00..0000000000000
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestHeadIndexTemplateAction.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.rest.action.admin.indices;
-
-import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesRequest;
-import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
-import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.RestChannel;
-import org.elasticsearch.rest.RestController;
-import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.RestResponse;
-import org.elasticsearch.rest.action.RestResponseListener;
-
-import static org.elasticsearch.rest.RestRequest.Method.HEAD;
-import static org.elasticsearch.rest.RestStatus.NOT_FOUND;
-import static org.elasticsearch.rest.RestStatus.OK;
-
-/**
- *
- */
-public class RestHeadIndexTemplateAction extends BaseRestHandler {
-
-    @Inject
-    public RestHeadIndexTemplateAction(Settings settings, RestController controller) {
-        super(settings);
-
-        controller.registerHandler(HEAD, "/_template/{name}", this);
-    }
-
-    @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
-        GetIndexTemplatesRequest getIndexTemplatesRequest = new GetIndexTemplatesRequest(request.param("name"));
-        getIndexTemplatesRequest.local(request.paramAsBoolean("local", getIndexTemplatesRequest.local()));
-        getIndexTemplatesRequest.masterNodeTimeout(request.paramAsTime("master_timeout", getIndexTemplatesRequest.masterNodeTimeout()));
-        client.admin().indices().getTemplates(getIndexTemplatesRequest, new RestResponseListener<GetIndexTemplatesResponse>(channel) {
-            @Override
-            public RestResponse buildResponse(GetIndexTemplatesResponse getIndexTemplatesResponse) {
-                boolean templateExists = getIndexTemplatesResponse.getIndexTemplates().size() > 0;
-                if (templateExists) {
-                    return new BytesRestResponse(OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY);
-                } else {
-                    return new BytesRestResponse(NOT_FOUND, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY);
-                }
-            }
-        });
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndexDeleteAliasesAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndexDeleteAliasesAction.java
index b027aeb8d6712..d2180a53c9630 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndexDeleteAliasesAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndexDeleteAliasesAction.java
@@ -19,25 +19,22 @@
 package org.elasticsearch.rest.action.admin.indices;
 
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
-import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.AcknowledgedRestListener;
 
+import java.io.IOException;
+
 import static org.elasticsearch.rest.RestRequest.Method.DELETE;
 
 /**
  */
 public class RestIndexDeleteAliasesAction extends BaseRestHandler {
-
-    @Inject
     public RestIndexDeleteAliasesAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(DELETE, "/{index}/_alias/{name}", this);
@@ -45,7 +42,7 @@ public RestIndexDeleteAliasesAction(Settings settings, RestController controller
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         final String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
         final String[] aliases = Strings.splitStringByCommaToArray(request.param("name"));
         IndicesAliasesRequest indicesAliasesRequest = new IndicesAliasesRequest();
@@ -53,6 +50,6 @@ public void handleRequest(final RestRequest request, final RestChannel channel,
         indicesAliasesRequest.addAliasAction(AliasActions.remove().indices(indices).aliases(aliases));
         indicesAliasesRequest.masterNodeTimeout(request.paramAsTime("master_timeout", indicesAliasesRequest.masterNodeTimeout()));
 
-        client.admin().indices().aliases(indicesAliasesRequest, new AcknowledgedRestListener<IndicesAliasesResponse>(channel));
+        return channel -> client.admin().indices().aliases(indicesAliasesRequest, new AcknowledgedRestListener<>(channel));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndexPutAliasAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndexPutAliasAction.java
index f7546bd57db05..2c68c4886103b 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndexPutAliasAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndexPutAliasAction.java
@@ -22,24 +22,20 @@
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.AcknowledgedRestListener;
 
+import java.io.IOException;
 import java.util.Map;
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.rest.RestRequest.Method.PUT;
 
 public class RestIndexPutAliasAction extends BaseRestHandler {
-
-    @Inject
     public RestIndexPutAliasAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(PUT, "/{index}/_alias/{name}", this);
@@ -58,7 +54,7 @@ public RestIndexPutAliasAction(Settings settings, RestController controller) {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) throws Exception {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
         String alias = request.param("name");
         Map<String, Object> filter = null;
@@ -67,7 +63,7 @@ public void handleRequest(final RestRequest request, final RestChannel channel,
         String searchRouting = null;
 
         if (request.hasContent()) {
-            try (XContentParser parser = XContentFactory.xContent(request.content()).createParser(request.content())) {
+            try (XContentParser parser = request.contentParser()) {
                 XContentParser.Token token = parser.nextToken();
                 if (token == null) {
                     throw new IllegalArgumentException("No index alias is specified");
@@ -117,6 +113,6 @@ public void handleRequest(final RestRequest request, final RestChannel channel,
             aliasAction.filter(filter);
         }
         indicesAliasesRequest.addAliasAction(aliasAction);
-        client.admin().indices().aliases(indicesAliasesRequest, new AcknowledgedRestListener<>(channel));
+        return channel -> client.admin().indices().aliases(indicesAliasesRequest, new AcknowledgedRestListener<>(channel));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesAliasesAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesAliasesAction.java
index fe8a6a1662818..58add2b4ea841 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesAliasesAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesAliasesAction.java
@@ -21,26 +21,22 @@
 
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions;
-import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ObjectParser;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.AcknowledgedRestListener;
 
+import java.io.IOException;
+
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
 public class RestIndicesAliasesAction extends BaseRestHandler {
-    static final ObjectParser<IndicesAliasesRequest, ParseFieldMatcherSupplier> PARSER = new ObjectParser<>("aliases");
+    static final ObjectParser<IndicesAliasesRequest, Void> PARSER = new ObjectParser<>("aliases");
     static {
         PARSER.declareObjectArray((request, actions) -> {
             for (AliasActions action: actions) {
@@ -49,23 +45,22 @@ public class RestIndicesAliasesAction extends BaseRestHandler {
         }, AliasActions.PARSER, new ParseField("actions"));
     }
 
-    @Inject
     public RestIndicesAliasesAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(POST, "/_aliases", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) throws Exception {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         IndicesAliasesRequest indicesAliasesRequest = new IndicesAliasesRequest();
         indicesAliasesRequest.masterNodeTimeout(request.paramAsTime("master_timeout", indicesAliasesRequest.masterNodeTimeout()));
         indicesAliasesRequest.timeout(request.paramAsTime("timeout", indicesAliasesRequest.timeout()));
-        try (XContentParser parser = XContentFactory.xContent(request.content()).createParser(request.content())) {
-            PARSER.parse(parser, indicesAliasesRequest, () -> ParseFieldMatcher.STRICT);
+        try (XContentParser parser = request.contentParser()) {
+            PARSER.parse(parser, indicesAliasesRequest, null);
         }
         if (indicesAliasesRequest.getAliasActions().isEmpty()) {
             throw new IllegalArgumentException("No action specified");
         }
-        client.admin().indices().aliases(indicesAliasesRequest, new AcknowledgedRestListener<IndicesAliasesResponse>(channel));
+        return channel -> client.admin().indices().aliases(indicesAliasesRequest, new AcknowledgedRestListener<>(channel));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesExistsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesExistsAction.java
deleted file mode 100644
index fa62a8443561c..0000000000000
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesExistsAction.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.rest.action.admin.indices;
-
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
-import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.RestChannel;
-import org.elasticsearch.rest.RestController;
-import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.RestResponse;
-import org.elasticsearch.rest.action.RestResponseListener;
-
-import static org.elasticsearch.rest.RestRequest.Method.HEAD;
-import static org.elasticsearch.rest.RestStatus.NOT_FOUND;
-import static org.elasticsearch.rest.RestStatus.OK;
-
-/**
- *
- */
-public class RestIndicesExistsAction extends BaseRestHandler {
-
-    @Inject
-    public RestIndicesExistsAction(Settings settings, RestController controller) {
-        super(settings);
-        controller.registerHandler(HEAD, "/{index}", this);
-    }
-
-    @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
-        IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest(Strings.splitStringByCommaToArray(request.param("index")));
-        indicesExistsRequest.indicesOptions(IndicesOptions.fromRequest(request, indicesExistsRequest.indicesOptions()));
-        indicesExistsRequest.local(request.paramAsBoolean("local", indicesExistsRequest.local()));
-        client.admin().indices().exists(indicesExistsRequest, new RestResponseListener<IndicesExistsResponse>(channel) {
-            @Override
-            public RestResponse buildResponse(IndicesExistsResponse response) {
-                if (response.isExists()) {
-                    return new BytesRestResponse(OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY);
-                } else {
-                    return new BytesRestResponse(NOT_FOUND, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY);
-                }
-            }
-
-        });
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesSegmentsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesSegmentsAction.java
index db9de980c52b8..6852b8527c754 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesSegmentsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesSegmentsAction.java
@@ -24,24 +24,22 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.action.RestBuilderListener;
 
+import java.io.IOException;
+
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestStatus.OK;
 import static org.elasticsearch.rest.action.RestActions.buildBroadcastShardsHeader;
 
 public class RestIndicesSegmentsAction extends BaseRestHandler {
-
-    @Inject
     public RestIndicesSegmentsAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_segments", this);
@@ -49,20 +47,21 @@ public RestIndicesSegmentsAction(Settings settings, RestController controller) {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         IndicesSegmentsRequest indicesSegmentsRequest = new IndicesSegmentsRequest(
                 Strings.splitStringByCommaToArray(request.param("index")));
         indicesSegmentsRequest.verbose(request.paramAsBoolean("verbose", false));
         indicesSegmentsRequest.indicesOptions(IndicesOptions.fromRequest(request, indicesSegmentsRequest.indicesOptions()));
-        client.admin().indices().segments(indicesSegmentsRequest, new RestBuilderListener<IndicesSegmentResponse>(channel) {
-            @Override
-            public RestResponse buildResponse(IndicesSegmentResponse response, XContentBuilder builder) throws Exception {
-                builder.startObject();
-                buildBroadcastShardsHeader(builder, request, response);
-                response.toXContent(builder, request);
-                builder.endObject();
-                return new BytesRestResponse(OK, builder);
-            }
-        });
+        return channel ->
+            client.admin().indices().segments(indicesSegmentsRequest, new RestBuilderListener<IndicesSegmentResponse>(channel) {
+                @Override
+                public RestResponse buildResponse(IndicesSegmentResponse response, XContentBuilder builder) throws Exception {
+                    builder.startObject();
+                    buildBroadcastShardsHeader(builder, request, response);
+                    response.toXContent(builder, request);
+                    builder.endObject();
+                    return new BytesRestResponse(OK, builder);
+                }
+            });
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesShardStoresAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesShardStoresAction.java
index 65c0dc8aa45bd..c00e6efffb032 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesShardStoresAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesShardStoresAction.java
@@ -25,17 +25,17 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.action.RestBuilderListener;
 
+import java.io.IOException;
+
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestStatus.OK;
 
@@ -43,8 +43,6 @@
  * Rest action for {@link IndicesShardStoresAction}
  */
 public class RestIndicesShardStoresAction extends BaseRestHandler {
-
-    @Inject
     public RestIndicesShardStoresAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_shard_stores", this);
@@ -52,21 +50,26 @@ public RestIndicesShardStoresAction(Settings settings, RestController controller
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         IndicesShardStoresRequest indicesShardStoresRequest = new IndicesShardStoresRequest(
                 Strings.splitStringByCommaToArray(request.param("index")));
         if (request.hasParam("status")) {
             indicesShardStoresRequest.shardStatuses(Strings.splitStringByCommaToArray(request.param("status")));
         }
         indicesShardStoresRequest.indicesOptions(IndicesOptions.fromRequest(request, indicesShardStoresRequest.indicesOptions()));
-        client.admin().indices().shardStores(indicesShardStoresRequest, new RestBuilderListener<IndicesShardStoresResponse>(channel) {
-            @Override
-            public RestResponse buildResponse(IndicesShardStoresResponse response, XContentBuilder builder) throws Exception {
-                builder.startObject();
-                response.toXContent(builder, request);
-                builder.endObject();
-                return new BytesRestResponse(OK, builder);
-            }
-        });
+        return channel ->
+            client.admin()
+                .indices()
+                .shardStores(indicesShardStoresRequest, new RestBuilderListener<IndicesShardStoresResponse>(channel) {
+                    @Override
+                    public RestResponse buildResponse(
+                        IndicesShardStoresResponse response,
+                        XContentBuilder builder) throws Exception {
+                        builder.startObject();
+                        response.toXContent(builder, request);
+                        builder.endObject();
+                        return new BytesRestResponse(OK, builder);
+                    }
+                });
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsAction.java
index c7dd62688fad7..c04b480527fec 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsAction.java
@@ -24,37 +24,63 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.action.RestBuilderListener;
 
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
 import java.util.Set;
+import java.util.TreeSet;
+import java.util.function.Consumer;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestStatus.OK;
 import static org.elasticsearch.rest.action.RestActions.buildBroadcastShardsHeader;
 
 public class RestIndicesStatsAction extends BaseRestHandler {
-
-    @Inject
     public RestIndicesStatsAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_stats", this);
         controller.registerHandler(GET, "/_stats/{metric}", this);
-        controller.registerHandler(GET, "/_stats/{metric}/{indexMetric}", this);
         controller.registerHandler(GET, "/{index}/_stats", this);
         controller.registerHandler(GET, "/{index}/_stats/{metric}", this);
     }
 
+    static final Map<String, Consumer<IndicesStatsRequest>> METRICS;
+
+    static {
+        final Map<String, Consumer<IndicesStatsRequest>> metrics = new HashMap<>();
+        metrics.put("docs", r -> r.docs(true));
+        metrics.put("store", r -> r.store(true));
+        metrics.put("indexing", r -> r.indexing(true));
+        metrics.put("search", r -> r.search(true));
+        metrics.put("suggest", r -> r.search(true));
+        metrics.put("get", r -> r.get(true));
+        metrics.put("merge", r -> r.merge(true));
+        metrics.put("refresh", r -> r.refresh(true));
+        metrics.put("flush", r -> r.flush(true));
+        metrics.put("warmer", r -> r.warmer(true));
+        metrics.put("query_cache", r -> r.queryCache(true));
+        metrics.put("segments", r -> r.segments(true));
+        metrics.put("fielddata", r -> r.fieldData(true));
+        metrics.put("completion", r -> r.completion(true));
+        metrics.put("request_cache", r -> r.requestCache(true));
+        metrics.put("recovery", r -> r.recovery(true));
+        metrics.put("translog", r -> r.translog(true));
+        METRICS = Collections.unmodifiableMap(metrics);
+    }
+
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         IndicesStatsRequest indicesStatsRequest = new IndicesStatsRequest();
         indicesStatsRequest.indicesOptions(IndicesOptions.fromRequest(request, indicesStatsRequest.indicesOptions()));
         indicesStatsRequest.indices(Strings.splitStringByCommaToArray(request.param("index")));
@@ -64,24 +90,34 @@ public void handleRequest(final RestRequest request, final RestChannel channel,
         // short cut, if no metrics have been specified in URI
         if (metrics.size() == 1 && metrics.contains("_all")) {
             indicesStatsRequest.all();
+        } else if (metrics.contains("_all")) {
+            throw new IllegalArgumentException(
+                String.format(Locale.ROOT,
+                    "request [%s] contains _all and individual metrics [%s]",
+                    request.path(),
+                    request.param("metric")));
         } else {
             indicesStatsRequest.clear();
-            indicesStatsRequest.docs(metrics.contains("docs"));
-            indicesStatsRequest.store(metrics.contains("store"));
-            indicesStatsRequest.indexing(metrics.contains("indexing"));
-            indicesStatsRequest.search(metrics.contains("search") || metrics.contains("suggest"));
-            indicesStatsRequest.get(metrics.contains("get"));
-            indicesStatsRequest.merge(metrics.contains("merge"));
-            indicesStatsRequest.refresh(metrics.contains("refresh"));
-            indicesStatsRequest.flush(metrics.contains("flush"));
-            indicesStatsRequest.warmer(metrics.contains("warmer"));
-            indicesStatsRequest.queryCache(metrics.contains("query_cache"));
-            indicesStatsRequest.segments(metrics.contains("segments"));
-            indicesStatsRequest.fieldData(metrics.contains("fielddata"));
-            indicesStatsRequest.completion(metrics.contains("completion"));
-            indicesStatsRequest.requestCache(metrics.contains("request_cache"));
-            indicesStatsRequest.recovery(metrics.contains("recovery"));
-            indicesStatsRequest.translog(metrics.contains("translog"));
+            // use a sorted set so the unrecognized parameters appear in a reliable sorted order
+            final Set<String> invalidMetrics = new TreeSet<>();
+            for (final String metric : metrics) {
+                final Consumer<IndicesStatsRequest> consumer = METRICS.get(metric);
+                if (consumer != null) {
+                    consumer.accept(indicesStatsRequest);
+                } else {
+                    invalidMetrics.add(metric);
+                }
+            }
+
+            if (invalidMetrics.contains("percolate")) {
+                deprecationLogger.deprecated(
+                    "percolate stats are no longer available and requests for percolate stats will fail starting in 6.0.0");
+                invalidMetrics.remove("percolate");
+            }
+
+            if (!invalidMetrics.isEmpty()) {
+                throw new IllegalArgumentException(unrecognized(request, invalidMetrics, METRICS.keySet(), "metric"));
+            }
         }
 
         if (request.hasParam("groups")) {
@@ -102,11 +138,11 @@ public void handleRequest(final RestRequest request, final RestChannel channel,
                     request.paramAsStringArray("fielddata_fields", request.paramAsStringArray("fields", Strings.EMPTY_ARRAY)));
         }
 
-        if (indicesStatsRequest.segments() && request.hasParam("include_segment_file_sizes")) {
-            indicesStatsRequest.includeSegmentFileSizes(true);
+        if (indicesStatsRequest.segments()) {
+            indicesStatsRequest.includeSegmentFileSizes(request.paramAsBoolean("include_segment_file_sizes", false));
         }
 
-        client.admin().indices().stats(indicesStatsRequest, new RestBuilderListener<IndicesStatsResponse>(channel) {
+        return channel -> client.admin().indices().stats(indicesStatsRequest, new RestBuilderListener<IndicesStatsResponse>(channel) {
             @Override
             public RestResponse buildResponse(IndicesStatsResponse response, XContentBuilder builder) throws Exception {
                 builder.startObject();
@@ -122,4 +158,12 @@ public RestResponse buildResponse(IndicesStatsResponse response, XContentBuilder
     public boolean canTripCircuitBreaker() {
         return false;
     }
+
+    private static final Set<String> RESPONSE_PARAMS = Collections.singleton("level");
+
+    @Override
+    protected Set<String> responseParams() {
+        return RESPONSE_PARAMS;
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestOpenIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestOpenIndexAction.java
index dd40705769fff..8b1050bbba7a1 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestOpenIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestOpenIndexAction.java
@@ -24,20 +24,18 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.AcknowledgedRestListener;
 
+import java.io.IOException;
+
 /**
  *
  */
 public class RestOpenIndexAction extends BaseRestHandler {
-
-    @Inject
     public RestOpenIndexAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(RestRequest.Method.POST, "/_open", this);
@@ -45,11 +43,11 @@ public RestOpenIndexAction(Settings settings, RestController controller) {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         OpenIndexRequest openIndexRequest = new OpenIndexRequest(Strings.splitStringByCommaToArray(request.param("index")));
         openIndexRequest.timeout(request.paramAsTime("timeout", openIndexRequest.timeout()));
         openIndexRequest.masterNodeTimeout(request.paramAsTime("master_timeout", openIndexRequest.masterNodeTimeout()));
         openIndexRequest.indicesOptions(IndicesOptions.fromRequest(request, openIndexRequest.indicesOptions()));
-        client.admin().indices().open(openIndexRequest, new AcknowledgedRestListener<OpenIndexResponse>(channel));
+        return channel -> client.admin().indices().open(openIndexRequest, new AcknowledgedRestListener<OpenIndexResponse>(channel));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java
index 398eb62c1f147..282d8e2440c3e 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java
@@ -16,41 +16,36 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+
 package org.elasticsearch.rest.action.admin.indices;
 
 import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
-import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.AcknowledgedRestListener;
 
-/**
- *
- */
-public class RestPutIndexTemplateAction extends BaseRestHandler {
+import java.io.IOException;
 
-    @Inject
+public class RestPutIndexTemplateAction extends BaseRestHandler {
     public RestPutIndexTemplateAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(RestRequest.Method.PUT, "/_template/{name}", this);
         controller.registerHandler(RestRequest.Method.POST, "/_template/{name}", this);
     }
 
-    @SuppressWarnings({"unchecked"})
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         PutIndexTemplateRequest putRequest = new PutIndexTemplateRequest(request.param("name"));
         putRequest.template(request.param("template", putRequest.template()));
         putRequest.order(request.paramAsInt("order", putRequest.order()));
         putRequest.masterNodeTimeout(request.paramAsTime("master_timeout", putRequest.masterNodeTimeout()));
         putRequest.create(request.paramAsBoolean("create", false));
         putRequest.cause(request.param("cause", ""));
-        putRequest.source(request.content());
-        client.admin().indices().putTemplate(putRequest, new AcknowledgedRestListener<PutIndexTemplateResponse>(channel));
+        putRequest.source(request.requiredContent(), request.getXContentType());
+        return channel -> client.admin().indices().putTemplate(putRequest, new AcknowledgedRestListener<>(channel));
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java
index 3a582d0b0a96b..06b48d0142b47 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java
@@ -20,18 +20,17 @@
 package org.elasticsearch.rest.action.admin.indices;
 
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
-import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.AcknowledgedRestListener;
 
+import java.io.IOException;
+
 import static org.elasticsearch.client.Requests.putMappingRequest;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.rest.RestRequest.Method.PUT;
@@ -40,9 +39,6 @@
  *
  */
 public class RestPutMappingAction extends BaseRestHandler {
-
-
-    @Inject
     public RestPutMappingAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(PUT, "/{index}/_mapping/", this);
@@ -68,14 +64,14 @@ public RestPutMappingAction(Settings settings, RestController controller) {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         PutMappingRequest putMappingRequest = putMappingRequest(Strings.splitStringByCommaToArray(request.param("index")));
         putMappingRequest.type(request.param("type"));
-        putMappingRequest.source(request.content().utf8ToString());
+        putMappingRequest.source(request.requiredContent(), request.getXContentType());
         putMappingRequest.updateAllTypes(request.paramAsBoolean("update_all_types", false));
         putMappingRequest.timeout(request.paramAsTime("timeout", putMappingRequest.timeout()));
         putMappingRequest.masterNodeTimeout(request.paramAsTime("master_timeout", putMappingRequest.masterNodeTimeout()));
         putMappingRequest.indicesOptions(IndicesOptions.fromRequest(request, putMappingRequest.indicesOptions()));
-        client.admin().indices().putMapping(putMappingRequest, new AcknowledgedRestListener<PutMappingResponse>(channel));
+        return channel -> client.admin().indices().putMapping(putMappingRequest, new AcknowledgedRestListener<>(channel));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRecoveryAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRecoveryAction.java
index 5dee73606fd72..ab1295e9c6015 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRecoveryAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRecoveryAction.java
@@ -24,17 +24,17 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.action.RestBuilderListener;
 
+import java.io.IOException;
+
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestStatus.OK;
 
@@ -42,8 +42,6 @@
  * REST handler to report on index recoveries.
  */
 public class RestRecoveryAction extends BaseRestHandler {
-
-    @Inject
     public RestRecoveryAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_recovery", this);
@@ -51,14 +49,14 @@ public RestRecoveryAction(Settings settings, RestController controller) {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
 
         final RecoveryRequest recoveryRequest = new RecoveryRequest(Strings.splitStringByCommaToArray(request.param("index")));
         recoveryRequest.detailed(request.paramAsBoolean("detailed", false));
         recoveryRequest.activeOnly(request.paramAsBoolean("active_only", false));
         recoveryRequest.indicesOptions(IndicesOptions.fromRequest(request, recoveryRequest.indicesOptions()));
 
-        client.admin().indices().recoveries(recoveryRequest, new RestBuilderListener<RecoveryResponse>(channel) {
+        return channel -> client.admin().indices().recoveries(recoveryRequest, new RestBuilderListener<RecoveryResponse>(channel) {
             @Override
             public RestResponse buildResponse(RecoveryResponse response, XContentBuilder builder) throws Exception {
                 response.detailed(recoveryRequest.detailed());
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRefreshAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRefreshAction.java
index 54088a7ddb244..c213f3be18b6b 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRefreshAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRefreshAction.java
@@ -24,28 +24,23 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.action.RestBuilderListener;
 
+import java.io.IOException;
+
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.rest.RestStatus.OK;
 import static org.elasticsearch.rest.action.RestActions.buildBroadcastShardsHeader;
 
-/**
- *
- */
 public class RestRefreshAction extends BaseRestHandler {
-
-    @Inject
     public RestRefreshAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(POST, "/_refresh", this);
@@ -56,10 +51,10 @@ public RestRefreshAction(Settings settings, RestController controller) {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         RefreshRequest refreshRequest = new RefreshRequest(Strings.splitStringByCommaToArray(request.param("index")));
         refreshRequest.indicesOptions(IndicesOptions.fromRequest(request, refreshRequest.indicesOptions()));
-        client.admin().indices().refresh(refreshRequest, new RestBuilderListener<RefreshResponse>(channel) {
+        return channel -> client.admin().indices().refresh(refreshRequest, new RestBuilderListener<RefreshResponse>(channel) {
             @Override
             public RestResponse buildResponse(RefreshResponse response, XContentBuilder builder) throws Exception {
                 builder.startObject();
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java
index 1433bc425715d..09ef33ecaf913 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java
@@ -20,39 +20,34 @@
 package org.elasticsearch.rest.action.admin.indices;
 
 import org.elasticsearch.action.admin.indices.rollover.RolloverRequest;
-import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.action.support.ActiveShardCount;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestToXContentListener;
 
+import java.io.IOException;
+
 /**
  *
  */
 public class RestRolloverIndexAction extends BaseRestHandler {
-
-    @Inject
     public RestRolloverIndexAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(RestRequest.Method.POST, "/{index}/_rollover", this);
         controller.registerHandler(RestRequest.Method.POST, "/{index}/_rollover/{new_index}", this);
     }
 
-    @SuppressWarnings({"unchecked"})
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         RolloverRequest rolloverIndexRequest = new RolloverRequest(request.param("index"), request.param("new_index"));
-        if (request.hasContent()) {
-            rolloverIndexRequest.source(request.content());
-        }
+        request.applyContentParser(parser -> RolloverRequest.PARSER.parse(parser, rolloverIndexRequest, null));
         rolloverIndexRequest.dryRun(request.paramAsBoolean("dry_run", false));
         rolloverIndexRequest.timeout(request.paramAsTime("timeout", rolloverIndexRequest.timeout()));
         rolloverIndexRequest.masterNodeTimeout(request.paramAsTime("master_timeout", rolloverIndexRequest.masterNodeTimeout()));
         rolloverIndexRequest.setWaitForActiveShards(ActiveShardCount.parseString(request.param("wait_for_active_shards")));
-        client.admin().indices().rolloverIndex(rolloverIndexRequest, new RestToXContentListener<>(channel));
+        return channel -> client.admin().indices().rolloverIndex(rolloverIndexRequest, new RestToXContentListener<>(channel));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestShrinkIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestShrinkIndexAction.java
index f04c9760a6397..1b63eb7f6d3ab 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestShrinkIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestShrinkIndexAction.java
@@ -20,35 +20,27 @@
 package org.elasticsearch.rest.action.admin.indices;
 
 import org.elasticsearch.action.admin.indices.shrink.ShrinkRequest;
-import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.action.admin.indices.shrink.ShrinkResponse;
 import org.elasticsearch.action.support.ActiveShardCount;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.AcknowledgedRestListener;
 
 import java.io.IOException;
 
-/**
- *
- */
 public class RestShrinkIndexAction extends BaseRestHandler {
-
-    @Inject
     public RestShrinkIndexAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(RestRequest.Method.PUT, "/{index}/_shrink/{target}", this);
         controller.registerHandler(RestRequest.Method.POST, "/{index}/_shrink/{target}", this);
     }
 
-    @SuppressWarnings({"unchecked"})
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         if (request.param("target") == null) {
             throw new IllegalArgumentException("no target index");
         }
@@ -56,13 +48,11 @@ public void handleRequest(final RestRequest request, final RestChannel channel,
             throw new IllegalArgumentException("no source index");
         }
         ShrinkRequest shrinkIndexRequest = new ShrinkRequest(request.param("target"), request.param("index"));
-        if (request.hasContent()) {
-            shrinkIndexRequest.source(request.content());
-        }
+        request.applyContentParser(parser -> ShrinkRequest.PARSER.parse(parser, shrinkIndexRequest, null));
         shrinkIndexRequest.timeout(request.paramAsTime("timeout", shrinkIndexRequest.timeout()));
         shrinkIndexRequest.masterNodeTimeout(request.paramAsTime("master_timeout", shrinkIndexRequest.masterNodeTimeout()));
         shrinkIndexRequest.setWaitForActiveShards(ActiveShardCount.parseString(request.param("wait_for_active_shards")));
-        client.admin().indices().shrinkIndex(shrinkIndexRequest, new AcknowledgedRestListener<ShrinkResponse>(channel) {
+        return channel -> client.admin().indices().shrinkIndex(shrinkIndexRequest, new AcknowledgedRestListener<ShrinkResponse>(channel) {
             @Override
             public void addCustomFields(XContentBuilder builder, ShrinkResponse response) throws IOException {
                 response.addCustomFields(builder);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSyncedFlushAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSyncedFlushAction.java
index 784a588db8927..903b0dee04f0b 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSyncedFlushAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSyncedFlushAction.java
@@ -24,17 +24,17 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.action.RestBuilderListener;
 
+import java.io.IOException;
+
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
@@ -42,8 +42,6 @@
  *
  */
 public class RestSyncedFlushAction extends BaseRestHandler {
-
-    @Inject
     public RestSyncedFlushAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(POST, "/_flush/synced", this);
@@ -54,11 +52,11 @@ public RestSyncedFlushAction(Settings settings, RestController controller) {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         IndicesOptions indicesOptions = IndicesOptions.fromRequest(request, IndicesOptions.lenientExpandOpen());
         SyncedFlushRequest syncedFlushRequest = new SyncedFlushRequest(Strings.splitStringByCommaToArray(request.param("index")));
         syncedFlushRequest.indicesOptions(indicesOptions);
-        client.admin().indices().syncedFlush(syncedFlushRequest, new RestBuilderListener<SyncedFlushResponse>(channel) {
+        return channel -> client.admin().indices().syncedFlush(syncedFlushRequest, new RestBuilderListener<SyncedFlushResponse>(channel) {
             @Override
             public RestResponse buildResponse(SyncedFlushResponse results, XContentBuilder builder) throws Exception {
                 builder.startObject();
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestTypesExistsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestTypesExistsAction.java
deleted file mode 100644
index 3877715395c34..0000000000000
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestTypesExistsAction.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.rest.action.admin.indices;
-
-import org.elasticsearch.action.admin.indices.exists.types.TypesExistsRequest;
-import org.elasticsearch.action.admin.indices.exists.types.TypesExistsResponse;
-import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.RestChannel;
-import org.elasticsearch.rest.RestController;
-import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.RestResponse;
-import org.elasticsearch.rest.action.RestResponseListener;
-
-import static org.elasticsearch.rest.RestRequest.Method.HEAD;
-import static org.elasticsearch.rest.RestStatus.NOT_FOUND;
-import static org.elasticsearch.rest.RestStatus.OK;
-
-/**
- * Rest api for checking if a type exists.
- */
-public class RestTypesExistsAction extends BaseRestHandler {
-
-    @Inject
-    public RestTypesExistsAction(Settings settings, RestController controller) {
-        super(settings);
-        controller.registerWithDeprecatedHandler(
-                HEAD, "/{index}/_mapping/{type}", this,
-                HEAD, "/{index}/{type}", deprecationLogger);
-    }
-
-    @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
-        TypesExistsRequest typesExistsRequest = new TypesExistsRequest(
-                Strings.splitStringByCommaToArray(request.param("index")), Strings.splitStringByCommaToArray(request.param("type"))
-        );
-        typesExistsRequest.local(request.paramAsBoolean("local", typesExistsRequest.local()));
-        typesExistsRequest.indicesOptions(IndicesOptions.fromRequest(request, typesExistsRequest.indicesOptions()));
-        client.admin().indices().typesExists(typesExistsRequest, new RestResponseListener<TypesExistsResponse>(channel) {
-            @Override
-            public RestResponse buildResponse(TypesExistsResponse response) throws Exception {
-                if (response.isExists()) {
-                    return new BytesRestResponse(OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY);
-                } else {
-                    return new BytesRestResponse(NOT_FOUND, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY);
-                }
-            }
-        });
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestUpdateSettingsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestUpdateSettingsAction.java
index 0c1b535901e4f..c0ee1edb10aac 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestUpdateSettingsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestUpdateSettingsAction.java
@@ -23,36 +23,25 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.AcknowledgedRestListener;
 
+import java.io.IOException;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Set;
 
-import static java.util.Collections.unmodifiableSet;
 import static org.elasticsearch.client.Requests.updateSettingsRequest;
-import static org.elasticsearch.common.util.set.Sets.newHashSet;
 
 /**
  *
  */
 public class RestUpdateSettingsAction extends BaseRestHandler {
-    private static final Set<String> VALUES_TO_EXCLUDE = unmodifiableSet(newHashSet(
-            "pretty",
-            "timeout",
-            "master_timeout",
-            "index",
-            "preserve_existing",
-            "expand_wildcards",
-            "ignore_unavailable",
-            "allow_no_indices"));
 
-    @Inject
     public RestUpdateSettingsAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(RestRequest.Method.PUT, "/{index}/_settings", this);
@@ -60,35 +49,33 @@ public RestUpdateSettingsAction(Settings settings, RestController controller) {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         UpdateSettingsRequest updateSettingsRequest = updateSettingsRequest(Strings.splitStringByCommaToArray(request.param("index")));
         updateSettingsRequest.timeout(request.paramAsTime("timeout", updateSettingsRequest.timeout()));
         updateSettingsRequest.setPreserveExisting(request.paramAsBoolean("preserve_existing", updateSettingsRequest.isPreserveExisting()));
         updateSettingsRequest.masterNodeTimeout(request.paramAsTime("master_timeout", updateSettingsRequest.masterNodeTimeout()));
         updateSettingsRequest.indicesOptions(IndicesOptions.fromRequest(request, updateSettingsRequest.indicesOptions()));
 
-        Settings.Builder updateSettings = Settings.builder();
-        String bodySettingsStr = request.content().utf8ToString();
-        if (Strings.hasText(bodySettingsStr)) {
-            Settings buildSettings = Settings.builder().loadFromSource(bodySettingsStr).build();
-            for (Map.Entry<String, String> entry : buildSettings.getAsMap().entrySet()) {
-                String key = entry.getKey();
-                String value = entry.getValue();
-                // clean up in case the body is wrapped with "settings" : { ... }
-                if (key.startsWith("settings.")) {
-                    key = key.substring("settings.".length());
-                }
-                updateSettings.put(key, value);
+        Map<String, Object> settings = new HashMap<>();
+        try (XContentParser parser = request.contentParser()) {
+            Map<String, Object> bodySettings = parser.map();
+            Object innerBodySettings = bodySettings.get("settings");
+            // clean up in case the body is wrapped with "settings" : { ... }
+            if (innerBodySettings instanceof Map) {
+                @SuppressWarnings("unchecked")
+                Map<String, Object> innerBodySettingsMap = (Map<String, Object>) innerBodySettings;
+                settings.putAll(innerBodySettingsMap);
+            } else {
+                settings.putAll(bodySettings);
             }
         }
-        for (Map.Entry<String, String> entry : request.params().entrySet()) {
-            if (VALUES_TO_EXCLUDE.contains(entry.getKey())) {
-                continue;
-            }
-            updateSettings.put(entry.getKey(), entry.getValue());
-        }
-        updateSettingsRequest.settings(updateSettings);
+        updateSettingsRequest.settings(settings);
 
-        client.admin().indices().updateSettings(updateSettingsRequest, new AcknowledgedRestListener<>(channel));
+        return channel -> client.admin().indices().updateSettings(updateSettingsRequest, new AcknowledgedRestListener<>(channel));
+    }
+
+    @Override
+    protected Set<String> responseParams() {
+        return Settings.FORMAT_PARAMS;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestUpgradeAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestUpgradeAction.java
index e0659e1cf525f..9437ad5eada93 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestUpgradeAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestUpgradeAction.java
@@ -20,23 +20,24 @@
 package org.elasticsearch.rest.action.admin.indices;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.action.admin.indices.upgrade.get.UpgradeStatusRequest;
 import org.elasticsearch.action.admin.indices.upgrade.get.UpgradeStatusResponse;
 import org.elasticsearch.action.admin.indices.upgrade.post.UpgradeRequest;
 import org.elasticsearch.action.admin.indices.upgrade.post.UpgradeResponse;
+import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.action.RestBuilderListener;
 
+import java.io.IOException;
 import java.util.Map;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
@@ -44,10 +45,7 @@
 import static org.elasticsearch.rest.RestStatus.OK;
 import static org.elasticsearch.rest.action.RestActions.buildBroadcastShardsHeader;
 
-
 public class RestUpgradeAction extends BaseRestHandler {
-
-    @Inject
     public RestUpgradeAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(POST, "/_upgrade", this);
@@ -58,31 +56,35 @@ public RestUpgradeAction(Settings settings, RestController controller) {
     }
 
     @Override
-    public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
+    public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
         if (request.method().equals(RestRequest.Method.GET)) {
-            handleGet(request, channel, client);
+            return handleGet(request, client);
         } else if (request.method().equals(RestRequest.Method.POST)) {
-            handlePost(request, channel, client);
+            return handlePost(request, client);
+        } else {
+            throw new IllegalArgumentException("illegal method [" + request.method() + "] for request [" + request.path() + "]");
         }
     }
 
-    void handleGet(final RestRequest request, RestChannel channel, NodeClient client) {
-        client.admin().indices().prepareUpgradeStatus(Strings.splitStringByCommaToArray(request.param("index")))
-                .execute(new RestBuilderListener<UpgradeStatusResponse>(channel) {
-                    @Override
-                    public RestResponse buildResponse(UpgradeStatusResponse response, XContentBuilder builder) throws Exception {
-                        builder.startObject();
-                        response.toXContent(builder, request);
-                        builder.endObject();
-                        return new BytesRestResponse(OK, builder);
-                    }
-                });
+    private RestChannelConsumer handleGet(final RestRequest request, NodeClient client) {
+        UpgradeStatusRequest statusRequest = new UpgradeStatusRequest(Strings.splitStringByCommaToArray(request.param("index")));
+        statusRequest.indicesOptions(IndicesOptions.fromRequest(request, statusRequest.indicesOptions()));
+        return channel -> client.admin().indices().upgradeStatus(statusRequest, new RestBuilderListener<UpgradeStatusResponse>(channel) {
+            @Override
+            public RestResponse buildResponse(UpgradeStatusResponse response, XContentBuilder builder) throws Exception {
+                builder.startObject();
+                response.toXContent(builder, request);
+                builder.endObject();
+                return new BytesRestResponse(OK, builder);
+            }
+        });
     }
 
-    void handlePost(final RestRequest request, RestChannel channel, NodeClient client) {
+    private RestChannelConsumer handlePost(final RestRequest request, NodeClient client) {
         UpgradeRequest upgradeReq = new UpgradeRequest(Strings.splitStringByCommaToArray(request.param("index")));
+        upgradeReq.indicesOptions(IndicesOptions.fromRequest(request, upgradeReq.indicesOptions()));
         upgradeReq.upgradeOnlyAncientSegments(request.paramAsBoolean("only_ancient_segments", false));
-        client.admin().indices().upgrade(upgradeReq, new RestBuilderListener<UpgradeResponse>(channel) {
+        return channel -> client.admin().indices().upgrade(upgradeReq, new RestBuilderListener<UpgradeResponse>(channel) {
             @Override
             public RestResponse buildResponse(UpgradeResponse response, XContentBuilder builder) throws Exception {
                 builder.startObject();
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryAction.java
index 7bf2a34ef632e..0c2374045dd9b 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryAction.java
@@ -26,11 +26,8 @@
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestChannel;
@@ -47,15 +44,8 @@
 import static org.elasticsearch.rest.RestStatus.OK;
 import static org.elasticsearch.rest.action.RestActions.buildBroadcastShardsHeader;
 
-/**
- *
- */
 public class RestValidateQueryAction extends BaseRestHandler {
-
-    private final IndicesQueriesRegistry indicesQueriesRegistry;
-
-    @Inject
-    public RestValidateQueryAction(Settings settings, RestController controller, IndicesQueriesRegistry indicesQueriesRegistry) {
+    public RestValidateQueryAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_validate/query", this);
         controller.registerHandler(POST, "/_validate/query", this);
@@ -63,62 +53,76 @@ public RestValidateQueryAction(Settings settings, RestController controller, Ind
         controller.registerHandler(POST, "/{index}/_validate/query", this);
         controller.registerHandler(GET, "/{index}/{type}/_validate/query", this);
         controller.registerHandler(POST, "/{index}/{type}/_validate/query", this);
-        this.indicesQueriesRegistry = indicesQueriesRegistry;
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) throws Exception {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         ValidateQueryRequest validateQueryRequest = new ValidateQueryRequest(Strings.splitStringByCommaToArray(request.param("index")));
         validateQueryRequest.indicesOptions(IndicesOptions.fromRequest(request, validateQueryRequest.indicesOptions()));
         validateQueryRequest.explain(request.paramAsBoolean("explain", false));
-        if (RestActions.hasBodyContent(request)) {
-            try {
-                validateQueryRequest
-                        .query(RestActions.getQueryContent(RestActions.getRestContent(request), indicesQueriesRegistry, parseFieldMatcher));
-            } catch(ParsingException e) {
-                channel.sendResponse(buildErrorResponse(channel.newBuilder(), e.getDetailedMessage(), validateQueryRequest.explain()));
-                return;
-            } catch(Exception e) {
-                channel.sendResponse(buildErrorResponse(channel.newBuilder(), e.getMessage(), validateQueryRequest.explain()));
-                return;
-            }
-        } else {
-            QueryBuilder queryBuilder = RestActions.urlParamsToQueryBuilder(request);
-            if (queryBuilder != null) {
-                validateQueryRequest.query(queryBuilder);
-            }
-        }
         validateQueryRequest.types(Strings.splitStringByCommaToArray(request.param("type")));
         validateQueryRequest.rewrite(request.paramAsBoolean("rewrite", false));
+        validateQueryRequest.allShards(request.paramAsBoolean("all_shards", false));
+
+        Exception bodyParsingException = null;
+        try {
+            request.withContentOrSourceParamParserOrNull(parser -> {
+                if (parser != null) {
+                    validateQueryRequest.query(RestActions.getQueryContent(parser));
+                } else if (request.hasParam("q")) {
+                    validateQueryRequest.query(RestActions.urlParamsToQueryBuilder(request));
+                }
+            });
+        } catch (Exception e) {
+            bodyParsingException = e;
+        }
 
-        client.admin().indices().validateQuery(validateQueryRequest, new RestBuilderListener<ValidateQueryResponse>(channel) {
-            @Override
-            public RestResponse buildResponse(ValidateQueryResponse response, XContentBuilder builder) throws Exception {
-                builder.startObject();
-                builder.field(VALID_FIELD, response.isValid());
-                buildBroadcastShardsHeader(builder, request, response);
-                if (response.getQueryExplanation() != null && !response.getQueryExplanation().isEmpty()) {
-                    builder.startArray(EXPLANATIONS_FIELD);
-                    for (QueryExplanation explanation : response.getQueryExplanation()) {
+        final Exception finalBodyParsingException = bodyParsingException;
+        return channel -> {
+            if (finalBodyParsingException != null) {
+                if (finalBodyParsingException instanceof ParsingException) {
+                    handleException(validateQueryRequest, ((ParsingException) finalBodyParsingException).getDetailedMessage(), channel);
+                } else {
+                    handleException(validateQueryRequest, finalBodyParsingException.getMessage(), channel);
+                }
+            } else {
+                client.admin().indices().validateQuery(validateQueryRequest, new RestBuilderListener<ValidateQueryResponse>(channel) {
+                    @Override
+                    public RestResponse buildResponse(ValidateQueryResponse response, XContentBuilder builder) throws Exception {
                         builder.startObject();
-                        if (explanation.getIndex() != null) {
-                            builder.field(INDEX_FIELD, explanation.getIndex());
-                        }
-                        builder.field(VALID_FIELD, explanation.isValid());
-                        if (explanation.getError() != null) {
-                            builder.field(ERROR_FIELD, explanation.getError());
-                        }
-                        if (explanation.getExplanation() != null) {
-                            builder.field(EXPLANATION_FIELD, explanation.getExplanation());
+                        builder.field(VALID_FIELD, response.isValid());
+                        buildBroadcastShardsHeader(builder, request, response);
+                        if (response.getQueryExplanation() != null && !response.getQueryExplanation().isEmpty()) {
+                            builder.startArray(EXPLANATIONS_FIELD);
+                            for (QueryExplanation explanation : response.getQueryExplanation()) {
+                                builder.startObject();
+                                if (explanation.getIndex() != null) {
+                                    builder.field(INDEX_FIELD, explanation.getIndex());
+                                }
+                                if(explanation.getShard() >= 0) {
+                                    builder.field(SHARD_FIELD, explanation.getShard());
+                                }
+                                builder.field(VALID_FIELD, explanation.isValid());
+                                if (explanation.getError() != null) {
+                                    builder.field(ERROR_FIELD, explanation.getError());
+                                }
+                                if (explanation.getExplanation() != null) {
+                                    builder.field(EXPLANATION_FIELD, explanation.getExplanation());
+                                }
+                                builder.endObject();
+                            }
+                            builder.endArray();
                         }
                         builder.endObject();
+                        return new BytesRestResponse(OK, builder);
                     }
-                    builder.endArray();
-                }
-                builder.endObject();
-                return new BytesRestResponse(OK, builder);
+                });
             }
-        });
+        };
+    }
+
+    private void handleException(final ValidateQueryRequest request, final String message, final RestChannel channel) throws IOException {
+        channel.sendResponse(buildErrorResponse(channel.newBuilder(), message, request.explain()));
     }
 
     private static BytesRestResponse buildErrorResponse(XContentBuilder builder, String error, boolean explain) throws IOException {
@@ -132,6 +136,7 @@ private static BytesRestResponse buildErrorResponse(XContentBuilder builder, Str
     }
 
     private static final String INDEX_FIELD = "index";
+    private static final String SHARD_FIELD = "shard";
     private static final String VALID_FIELD = "valid";
     private static final String EXPLANATIONS_FIELD = "explanations";
     private static final String ERROR_FIELD = "error";
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/AbstractCatAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/AbstractCatAction.java
index 8315e34d08e23..58dc861126bfe 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/AbstractCatAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/AbstractCatAction.java
@@ -20,54 +20,68 @@
 
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Table;
+import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.io.UTF8StreamWriter;
-import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.BytesStream;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestStatus;
 
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
 import static org.elasticsearch.rest.action.cat.RestTable.buildHelpWidths;
 import static org.elasticsearch.rest.action.cat.RestTable.pad;
 
-/**
- *
- */
 public abstract class AbstractCatAction extends BaseRestHandler {
 
     public AbstractCatAction(Settings settings) {
         super(settings);
     }
 
-    protected abstract void doRequest(final RestRequest request, final RestChannel channel, final NodeClient client);
+    protected abstract RestChannelConsumer doCatRequest(RestRequest request, NodeClient client);
 
     protected abstract void documentation(StringBuilder sb);
 
-    protected abstract Table getTableWithHeader(final RestRequest request);
+    protected abstract Table getTableWithHeader(RestRequest request);
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) throws Exception {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         boolean helpWanted = request.paramAsBoolean("help", false);
         if (helpWanted) {
-            Table table = getTableWithHeader(request);
-            int[] width = buildHelpWidths(table, request);
-            BytesStreamOutput bytesOutput = channel.bytesOutput();
-            UTF8StreamWriter out = new UTF8StreamWriter().setOutput(bytesOutput);
-            for (Table.Cell cell : table.getHeaders()) {
-                // need to do left-align always, so create new cells
-                pad(new Table.Cell(cell.value), width[0], request, out);
-                out.append(" | ");
-                pad(new Table.Cell(cell.attr.containsKey("alias") ? cell.attr.get("alias") : ""), width[1], request, out);
-                out.append(" | ");
-                pad(new Table.Cell(cell.attr.containsKey("desc") ? cell.attr.get("desc") : "not available"), width[2], request, out);
-                out.append("\n");
-            }
-            out.close();
-            channel.sendResponse(new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, bytesOutput.bytes()));
+            return channel -> {
+                Table table = getTableWithHeader(request);
+                int[] width = buildHelpWidths(table, request);
+                BytesStream bytesOutput = Streams.flushOnCloseStream(channel.bytesOutput());
+                UTF8StreamWriter out = new UTF8StreamWriter().setOutput(bytesOutput);
+                for (Table.Cell cell : table.getHeaders()) {
+                    // need to do left-align always, so create new cells
+                    pad(new Table.Cell(cell.value), width[0], request, out);
+                    out.append(" | ");
+                    pad(new Table.Cell(cell.attr.containsKey("alias") ? cell.attr.get("alias") : ""), width[1], request, out);
+                    out.append(" | ");
+                    pad(new Table.Cell(cell.attr.containsKey("desc") ? cell.attr.get("desc") : "not available"), width[2], request, out);
+                    out.append("\n");
+                }
+                out.close();
+                channel.sendResponse(new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, bytesOutput.bytes()));
+            };
         } else {
-            doRequest(request, channel, client);
+            return doCatRequest(request, client);
         }
     }
+
+    static Set<String> RESPONSE_PARAMS =
+            Collections.unmodifiableSet(new HashSet<>(Arrays.asList("format", "h", "v", "ts", "pri", "bytes", "size", "time", "s")));
+
+    @Override
+    protected Set<String> responseParams() {
+        return RESPONSE_PARAMS;
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestAliasAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestAliasAction.java
index 82d59784fc950..a783a9c2a8215 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestAliasAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestAliasAction.java
@@ -19,15 +19,14 @@
 package org.elasticsearch.rest.action.cat;
 
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
+
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.metadata.AliasMetaData;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.Table;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -37,27 +36,21 @@
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
-/**
- *
- */
 public class RestAliasAction extends AbstractCatAction {
-
-    @Inject
     public RestAliasAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_cat/aliases", this);
         controller.registerHandler(GET, "/_cat/aliases/{alias}", this);
     }
 
-
     @Override
-    protected void doRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    protected RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client) {
         final GetAliasesRequest getAliasesRequest = request.hasParam("alias") ?
-                new GetAliasesRequest(request.param("alias")) :
+                new GetAliasesRequest(Strings.commaDelimitedListToStringArray(request.param("alias"))) :
                 new GetAliasesRequest();
         getAliasesRequest.local(request.paramAsBoolean("local", getAliasesRequest.local()));
 
-        client.admin().indices().getAliases(getAliasesRequest, new RestResponseListener<GetAliasesResponse>(channel) {
+        return channel -> client.admin().indices().getAliases(getAliasesRequest, new RestResponseListener<GetAliasesResponse>(channel) {
             @Override
             public RestResponse buildResponse(GetAliasesResponse response) throws Exception {
                 Table tab = buildTable(request, response);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestAllocationAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestAllocationAction.java
index 7649d59af9923..0077297cbf3c9 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestAllocationAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestAllocationAction.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.rest.action.cat;
 
 import com.carrotsearch.hppc.ObjectIntScatterMap;
+
 import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequest;
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
@@ -31,10 +32,8 @@
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.Table;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -45,8 +44,6 @@
 
 
 public class RestAllocationAction extends AbstractCatAction {
-
-    @Inject
     public RestAllocationAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_cat/allocation", this);
@@ -59,14 +56,14 @@ protected void documentation(StringBuilder sb) {
     }
 
     @Override
-    public void doRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client) {
         final String[] nodes = Strings.splitStringByCommaToArray(request.param("nodes", "data:true"));
         final ClusterStateRequest clusterStateRequest = new ClusterStateRequest();
         clusterStateRequest.clear().routingTable(true);
         clusterStateRequest.local(request.paramAsBoolean("local", clusterStateRequest.local()));
         clusterStateRequest.masterNodeTimeout(request.paramAsTime("master_timeout", clusterStateRequest.masterNodeTimeout()));
 
-        client.admin().cluster().state(clusterStateRequest, new RestActionListener<ClusterStateResponse>(channel) {
+        return channel -> client.admin().cluster().state(clusterStateRequest, new RestActionListener<ClusterStateResponse>(channel) {
             @Override
             public void processResponse(final ClusterStateResponse state) {
                 NodesStatsRequest statsRequest = new NodesStatsRequest(nodes);
@@ -126,10 +123,10 @@ private Table buildTable(RestRequest request, final ClusterStateResponse state,
             //if we don't know how much we use (non data nodes), it means 0
             long used = 0;
             short diskPercent = -1;
-            if (total.bytes() > 0) {
-                used = total.bytes() - avail.bytes();
-                if (used >= 0 && avail.bytes() >= 0) {
-                    diskPercent = (short) (used * 100 / (used + avail.bytes()));
+            if (total.getBytes() > 0) {
+                used = total.getBytes() - avail.getBytes();
+                if (used >= 0 && avail.getBytes() >= 0) {
+                    diskPercent = (short) (used * 100 / (used + avail.getBytes()));
                 }
             }
 
@@ -137,8 +134,8 @@ private Table buildTable(RestRequest request, final ClusterStateResponse state,
             table.addCell(shardCount);
             table.addCell(nodeStats.getIndices().getStore().getSize());
             table.addCell(used < 0 ? null : new ByteSizeValue(used));
-            table.addCell(avail.bytes() < 0 ? null : avail);
-            table.addCell(total.bytes() < 0 ? null : total);
+            table.addCell(avail.getBytes() < 0 ? null : avail);
+            table.addCell(total.getBytes() < 0 ? null : total);
             table.addCell(diskPercent < 0 ? null : diskPercent);
             table.addCell(node.getHostName());
             table.addCell(node.getHostAddress());
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestCatAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestCatAction.java
index b9cc5011a810c..7442a7d85ee6b 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestCatAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestCatAction.java
@@ -24,12 +24,12 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestStatus;
 
-import java.util.Set;
+import java.io.IOException;
+import java.util.List;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
@@ -40,7 +40,7 @@ public class RestCatAction extends BaseRestHandler {
     private final String HELP;
 
     @Inject
-    public RestCatAction(Settings settings, RestController controller, Set<AbstractCatAction> catActions) {
+    public RestCatAction(Settings settings, RestController controller, List<AbstractCatAction> catActions) {
         super(settings);
         controller.registerHandler(GET, "/_cat", this);
         StringBuilder sb = new StringBuilder();
@@ -52,7 +52,8 @@ public RestCatAction(Settings settings, RestController controller, Set<AbstractC
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
-        channel.sendResponse(new BytesRestResponse(RestStatus.OK, HELP));
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
+        return channel -> channel.sendResponse(new BytesRestResponse(RestStatus.OK, HELP));
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestCountAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestCountAction.java
index 4faddc3168cc5..00eb89a9c0242 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestCountAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestCountAction.java
@@ -19,17 +19,14 @@
 
 package org.elasticsearch.rest.action.cat;
 
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.Table;
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -37,18 +34,15 @@
 import org.elasticsearch.rest.action.RestResponseListener;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 
+import java.io.IOException;
+
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
 public class RestCountAction extends AbstractCatAction {
-
-    private final IndicesQueriesRegistry indicesQueriesRegistry;
-
-    @Inject
-    public RestCountAction(Settings settings, RestController restController, RestController controller, IndicesQueriesRegistry indicesQueriesRegistry) {
+    public RestCountAction(Settings settings, RestController restController) {
         super(settings);
         restController.registerHandler(GET, "/_cat/count", this);
         restController.registerHandler(GET, "/_cat/count/{index}", this);
-        this.indicesQueriesRegistry = indicesQueriesRegistry;
     }
 
     @Override
@@ -58,21 +52,26 @@ protected void documentation(StringBuilder sb) {
     }
 
     @Override
-    public void doRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client) {
         String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
         SearchRequest countRequest = new SearchRequest(indices);
-        String source = request.param("source");
         SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().size(0);
         countRequest.source(searchSourceBuilder);
-        if (source != null) {
-            searchSourceBuilder.query(RestActions.getQueryContent(new BytesArray(source), indicesQueriesRegistry, parseFieldMatcher));
-        } else {
-            QueryBuilder queryBuilder = RestActions.urlParamsToQueryBuilder(request);
-            if (queryBuilder != null) {
-                searchSourceBuilder.query(queryBuilder);
-            }
+        try {
+            request.withContentOrSourceParamParserOrNull(parser -> {
+                if (parser == null) {
+                    QueryBuilder queryBuilder = RestActions.urlParamsToQueryBuilder(request);
+                    if (queryBuilder != null) {
+                        searchSourceBuilder.query(queryBuilder);
+                    }
+                } else {
+                    searchSourceBuilder.query(RestActions.getQueryContent(parser));
+                }
+            });
+        } catch (IOException e) {
+            throw new ElasticsearchException("Couldn't parse query", e);
         }
-        client.search(countRequest, new RestResponseListener<SearchResponse>(channel) {
+        return channel -> client.search(countRequest, new RestResponseListener<SearchResponse>(channel) {
             @Override
             public RestResponse buildResponse(SearchResponse countResponse) throws Exception {
                 return RestTable.buildResponse(buildTable(request, countResponse), channel);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestFielddataAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestFielddataAction.java
index fcdad0c3f7eea..4156ea4619273 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestFielddataAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestFielddataAction.java
@@ -20,15 +20,14 @@
 package org.elasticsearch.rest.action.cat;
 
 import com.carrotsearch.hppc.cursors.ObjectLongCursor;
+
 import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequest;
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Table;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -40,8 +39,6 @@
  * Cat API class to display information about the size of fielddata fields per node
  */
 public class RestFielddataAction extends AbstractCatAction {
-
-    @Inject
     public RestFielddataAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_cat/fielddata", this);
@@ -49,14 +46,14 @@ public RestFielddataAction(Settings settings, RestController controller) {
     }
 
     @Override
-    protected void doRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    protected RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client) {
         final NodesStatsRequest nodesStatsRequest = new NodesStatsRequest("data:true");
         nodesStatsRequest.clear();
         nodesStatsRequest.indices(true);
         String[] fields = request.paramAsStringArray("fields", null);
         nodesStatsRequest.indices().fieldDataFields(fields == null ? new String[] {"*"} : fields);
 
-        client.admin().cluster().nodesStats(nodesStatsRequest, new RestResponseListener<NodesStatsResponse>(channel) {
+        return channel -> client.admin().cluster().nodesStats(nodesStatsRequest, new RestResponseListener<NodesStatsResponse>(channel) {
             @Override
             public RestResponse buildResponse(NodesStatsResponse nodeStatses) throws Exception {
                 return RestTable.buildResponse(buildTable(request, nodeStatses), channel);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestHealthAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestHealthAction.java
index cd226e28b562c..2bc1c11b9a1d0 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestHealthAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestHealthAction.java
@@ -23,9 +23,7 @@
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Table;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -36,8 +34,6 @@
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
 public class RestHealthAction extends AbstractCatAction {
-
-    @Inject
     public RestHealthAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_cat/health", this);
@@ -49,10 +45,10 @@ protected void documentation(StringBuilder sb) {
     }
 
     @Override
-    public void doRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client) {
         ClusterHealthRequest clusterHealthRequest = new ClusterHealthRequest();
 
-        client.admin().cluster().health(clusterHealthRequest, new RestResponseListener<ClusterHealthResponse>(channel) {
+        return channel -> client.admin().cluster().health(clusterHealthRequest, new RestResponseListener<ClusterHealthResponse>(channel) {
             @Override
             public RestResponse buildResponse(final ClusterHealthResponse health) throws Exception {
                 return RestTable.buildResponse(buildTable(health, request), channel);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
index 3c65e32c746db..26bbc865fb276 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
@@ -23,23 +23,23 @@
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
+import org.elasticsearch.action.admin.indices.stats.CommonStats;
 import org.elasticsearch.action.admin.indices.stats.IndexStats;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.client.Requests;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.health.ClusterIndexHealth;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.Table;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -48,7 +48,11 @@
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
 import java.util.Locale;
+import java.util.Set;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
@@ -56,7 +60,6 @@ public class RestIndicesAction extends AbstractCatAction {
 
     private final IndexNameExpressionResolver indexNameExpressionResolver;
 
-    @Inject
     public RestIndicesAction(Settings settings, RestController controller, IndexNameExpressionResolver indexNameExpressionResolver) {
         super(settings);
         this.indexNameExpressionResolver = indexNameExpressionResolver;
@@ -71,7 +74,7 @@ protected void documentation(StringBuilder sb) {
     }
 
     @Override
-    public void doRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client) {
         final String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
         final ClusterStateRequest clusterStateRequest = new ClusterStateRequest();
         clusterStateRequest.clear().indices(indices).metaData(true);
@@ -80,7 +83,7 @@ public void doRequest(final RestRequest request, final RestChannel channel, fina
         final IndicesOptions strictExpandIndicesOptions = IndicesOptions.strictExpand();
         clusterStateRequest.indicesOptions(strictExpandIndicesOptions);
 
-        client.admin().cluster().state(clusterStateRequest, new RestActionListener<ClusterStateResponse>(channel) {
+        return channel -> client.admin().cluster().state(clusterStateRequest, new RestActionListener<ClusterStateResponse>(channel) {
             @Override
             public void processResponse(final ClusterStateResponse clusterStateResponse) {
                 final ClusterState state = clusterStateResponse.getState();
@@ -122,6 +125,19 @@ public RestResponse buildResponse(IndicesStatsResponse indicesStatsResponse) thr
         });
     }
 
+    private static final Set<String> RESPONSE_PARAMS;
+
+    static {
+        final Set<String> responseParams = new HashSet<>(Arrays.asList("local", "health"));
+        responseParams.addAll(AbstractCatAction.RESPONSE_PARAMS);
+        RESPONSE_PARAMS = Collections.unmodifiableSet(responseParams);
+    }
+
+    @Override
+    protected Set<String> responseParams() {
+        return RESPONSE_PARAMS;
+    }
+
     @Override
     protected Table getTableWithHeader(final RestRequest request) {
         Table table = new Table();
@@ -243,6 +259,9 @@ protected Table getTableWithHeader(final RestRequest request) {
         table.addCell("refresh.time", "sibling:pri;alias:rti,refreshTime;default:false;text-align:right;desc:time spent in refreshes");
         table.addCell("pri.refresh.time", "default:false;text-align:right;desc:time spent in refreshes");
 
+        table.addCell("refresh.listeners", "sibling:pri;alias:rli,refreshListeners;default:false;text-align:right;desc:number of pending refresh listeners");
+        table.addCell("pri.refresh.listeners", "default:false;text-align:right;desc:number of pending refresh listeners");
+
         table.addCell("search.fetch_current", "sibling:pri;alias:sfc,searchFetchCurrent;default:false;text-align:right;desc:current fetch phase ops");
         table.addCell("pri.search.fetch_current", "default:false;text-align:right;desc:current fetch phase ops");
 
@@ -314,16 +333,35 @@ protected Table getTableWithHeader(final RestRequest request) {
     }
 
     // package private for testing
-    Table buildTable(RestRequest request, Index[] indices, ClusterHealthResponse health, IndicesStatsResponse stats, MetaData indexMetaDatas) {
+    Table buildTable(RestRequest request, Index[] indices, ClusterHealthResponse response, IndicesStatsResponse stats, MetaData indexMetaDatas) {
+        final String healthParam = request.param("health");
+        final ClusterHealthStatus status;
+        if (healthParam != null) {
+            status = ClusterHealthStatus.fromString(healthParam);
+        } else {
+            status = null;
+        }
+
         Table table = getTableWithHeader(request);
 
         for (final Index index : indices) {
             final String indexName = index.getName();
-            ClusterIndexHealth indexHealth = health.getIndices().get(indexName);
+            ClusterIndexHealth indexHealth = response.getIndices().get(indexName);
             IndexStats indexStats = stats.getIndices().get(indexName);
             IndexMetaData indexMetaData = indexMetaDatas.getIndices().get(indexName);
             IndexMetaData.State state = indexMetaData.getState();
 
+            if (status != null) {
+                if (state == IndexMetaData.State.CLOSE ||
+                        (indexHealth == null && !ClusterHealthStatus.RED.equals(status)) ||
+                        !indexHealth.getStatus().equals(status)) {
+                    continue;
+                }
+            }
+
+            final CommonStats primaryStats = indexStats == null ? new CommonStats() : indexStats.getPrimaries();
+            final CommonStats totalStats = indexStats == null ? new CommonStats() : indexStats.getTotal();
+
             table.startRow();
             table.addCell(state == IndexMetaData.State.OPEN ? (indexHealth == null ? "red*" : indexHealth.getStatus().toString().toLowerCase(Locale.ROOT)) : null);
             table.addCell(state.toString().toLowerCase(Locale.ROOT));
@@ -331,179 +369,183 @@ Table buildTable(RestRequest request, Index[] indices, ClusterHealthResponse hea
             table.addCell(index.getUUID());
             table.addCell(indexHealth == null ? null : indexHealth.getNumberOfShards());
             table.addCell(indexHealth == null ? null : indexHealth.getNumberOfReplicas());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getDocs().getCount());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getDocs().getDeleted());
+
+            table.addCell(primaryStats.getDocs() == null ? null : primaryStats.getDocs().getCount());
+            table.addCell(primaryStats.getDocs() == null ? null : primaryStats.getDocs().getDeleted());
 
             table.addCell(indexMetaData.getCreationDate());
             table.addCell(new DateTime(indexMetaData.getCreationDate(), DateTimeZone.UTC));
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getStore().size());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getStore().size());
+            table.addCell(totalStats.getStore() == null ? null : totalStats.getStore().size());
+            table.addCell(primaryStats.getStore() == null ? null : primaryStats.getStore().size());
+
+            table.addCell(totalStats.getCompletion() == null ? null : totalStats.getCompletion().getSize());
+            table.addCell(primaryStats.getCompletion() == null ? null : primaryStats.getCompletion().getSize());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getCompletion().getSize());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getCompletion().getSize());
+            table.addCell(totalStats.getFieldData() == null ? null : totalStats.getFieldData().getMemorySize());
+            table.addCell(primaryStats.getFieldData() == null ? null : primaryStats.getFieldData().getMemorySize());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getFieldData().getMemorySize());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getFieldData().getMemorySize());
+            table.addCell(totalStats.getFieldData() == null ? null : totalStats.getFieldData().getEvictions());
+            table.addCell(primaryStats.getFieldData() == null ? null : primaryStats.getFieldData().getEvictions());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getFieldData().getEvictions());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getFieldData().getEvictions());
+            table.addCell(totalStats.getQueryCache() == null ? null : totalStats.getQueryCache().getMemorySize());
+            table.addCell(primaryStats.getQueryCache() == null ? null : primaryStats.getQueryCache().getMemorySize());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getQueryCache().getMemorySize());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getQueryCache().getMemorySize());
+            table.addCell(totalStats.getQueryCache() == null ? null : totalStats.getQueryCache().getEvictions());
+            table.addCell(primaryStats.getQueryCache() == null ? null : primaryStats.getQueryCache().getEvictions());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getQueryCache().getEvictions());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getQueryCache().getEvictions());
+            table.addCell(totalStats.getRequestCache() == null ? null : totalStats.getRequestCache().getMemorySize());
+            table.addCell(primaryStats.getRequestCache() == null ? null : primaryStats.getRequestCache().getMemorySize());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getRequestCache().getMemorySize());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getRequestCache().getMemorySize());
+            table.addCell(totalStats.getRequestCache() == null ? null : totalStats.getRequestCache().getEvictions());
+            table.addCell(primaryStats.getRequestCache() == null ? null : primaryStats.getRequestCache().getEvictions());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getRequestCache().getEvictions());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getRequestCache().getEvictions());
+            table.addCell(totalStats.getRequestCache() == null ? null : totalStats.getRequestCache().getHitCount());
+            table.addCell(primaryStats.getRequestCache() == null ? null : primaryStats.getRequestCache().getHitCount());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getRequestCache().getHitCount());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getRequestCache().getHitCount());
+            table.addCell(totalStats.getRequestCache() == null ? null : totalStats.getRequestCache().getMissCount());
+            table.addCell(primaryStats.getRequestCache() == null ? null : primaryStats.getRequestCache().getMissCount());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getRequestCache().getMissCount());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getRequestCache().getMissCount());
+            table.addCell(totalStats.getFlush() == null ? null : totalStats.getFlush().getTotal());
+            table.addCell(primaryStats.getFlush() == null ? null : primaryStats.getFlush().getTotal());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getFlush().getTotal());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getFlush().getTotal());
+            table.addCell(totalStats.getFlush() == null ? null : totalStats.getFlush().getTotalTime());
+            table.addCell(primaryStats.getFlush() == null ? null : primaryStats.getFlush().getTotalTime());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getFlush().getTotalTime());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getFlush().getTotalTime());
+            table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().current());
+            table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().current());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getGet().current());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getGet().current());
+            table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getTime());
+            table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getTime());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getGet().getTime());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getGet().getTime());
+            table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getCount());
+            table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getCount());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getGet().getCount());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getGet().getCount());
+            table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getExistsTime());
+            table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getExistsTime());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getGet().getExistsTime());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getGet().getExistsTime());
+            table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getExistsCount());
+            table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getExistsCount());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getGet().getExistsCount());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getGet().getExistsCount());
+            table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getMissingTime());
+            table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getMissingTime());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getGet().getMissingTime());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getGet().getMissingTime());
+            table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getMissingCount());
+            table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getMissingCount());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getGet().getMissingCount());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getGet().getMissingCount());
+            table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getDeleteCurrent());
+            table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getDeleteCurrent());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getIndexing().getTotal().getDeleteCurrent());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getIndexing().getTotal().getDeleteCurrent());
+            table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getDeleteTime());
+            table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getDeleteTime());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getIndexing().getTotal().getDeleteTime());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getIndexing().getTotal().getDeleteTime());
+            table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getDeleteCount());
+            table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getDeleteCount());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getIndexing().getTotal().getDeleteCount());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getIndexing().getTotal().getDeleteCount());
+            table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getIndexCurrent());
+            table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getIndexCurrent());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getIndexing().getTotal().getIndexCurrent());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getIndexing().getTotal().getIndexCurrent());
+            table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getIndexTime());
+            table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getIndexTime());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getIndexing().getTotal().getIndexTime());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getIndexing().getTotal().getIndexTime());
+            table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getIndexCount());
+            table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getIndexCount());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getIndexing().getTotal().getIndexCount());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getIndexing().getTotal().getIndexCount());
+            table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getIndexFailedCount());
+            table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getIndexFailedCount());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getIndexing().getTotal().getIndexFailedCount());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getIndexing().getTotal().getIndexFailedCount());
+            table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getCurrent());
+            table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getCurrent());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getMerge().getCurrent());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getMerge().getCurrent());
+            table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getCurrentNumDocs());
+            table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getCurrentNumDocs());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getMerge().getCurrentNumDocs());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getMerge().getCurrentNumDocs());
+            table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getCurrentSize());
+            table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getCurrentSize());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getMerge().getCurrentSize());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getMerge().getCurrentSize());
+            table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getTotal());
+            table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getTotal());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getMerge().getTotal());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getMerge().getTotal());
+            table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getTotalNumDocs());
+            table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getTotalNumDocs());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getMerge().getTotalNumDocs());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getMerge().getTotalNumDocs());
+            table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getTotalSize());
+            table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getTotalSize());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getMerge().getTotalSize());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getMerge().getTotalSize());
+            table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getTotalTime());
+            table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getTotalTime());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getMerge().getTotalTime());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getMerge().getTotalTime());
+            table.addCell(totalStats.getRefresh() == null ? null : totalStats.getRefresh().getTotal());
+            table.addCell(primaryStats.getRefresh() == null ? null : primaryStats.getRefresh().getTotal());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getRefresh().getTotal());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getRefresh().getTotal());
+            table.addCell(totalStats.getRefresh() == null ? null : totalStats.getRefresh().getTotalTime());
+            table.addCell(primaryStats.getRefresh() == null ? null : primaryStats.getRefresh().getTotalTime());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getRefresh().getTotalTime());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getRefresh().getTotalTime());
+            table.addCell(totalStats.getRefresh() == null ? null : totalStats.getRefresh().getListeners());
+            table.addCell(primaryStats.getRefresh() == null ? null : primaryStats.getRefresh().getListeners());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getSearch().getTotal().getFetchCurrent());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSearch().getTotal().getFetchCurrent());
+            table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getFetchCurrent());
+            table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getFetchCurrent());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getSearch().getTotal().getFetchTime());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSearch().getTotal().getFetchTime());
+            table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getFetchTime());
+            table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getFetchTime());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getSearch().getTotal().getFetchCount());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSearch().getTotal().getFetchCount());
+            table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getFetchCount());
+            table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getFetchCount());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getSearch().getOpenContexts());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSearch().getOpenContexts());
+            table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getOpenContexts());
+            table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getOpenContexts());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getSearch().getTotal().getQueryCurrent());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSearch().getTotal().getQueryCurrent());
+            table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getQueryCurrent());
+            table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getQueryCurrent());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getSearch().getTotal().getQueryTime());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSearch().getTotal().getQueryTime());
+            table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getQueryTime());
+            table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getQueryTime());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getSearch().getTotal().getQueryCount());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSearch().getTotal().getQueryCount());
+            table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getQueryCount());
+            table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getQueryCount());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getSearch().getTotal().getScrollCurrent());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSearch().getTotal().getScrollCurrent());
+            table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getScrollCurrent());
+            table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getScrollCurrent());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getSearch().getTotal().getScrollTime());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSearch().getTotal().getScrollTime());
+            table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getScrollTime());
+            table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getScrollTime());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getSearch().getTotal().getScrollCount());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSearch().getTotal().getScrollCount());
+            table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getScrollCount());
+            table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getScrollCount());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getSegments().getCount());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSegments().getCount());
+            table.addCell(totalStats.getSegments() == null ? null : totalStats.getSegments().getCount());
+            table.addCell(primaryStats.getSegments() == null ? null : primaryStats.getSegments().getCount());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getSegments().getMemory());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSegments().getMemory());
+            table.addCell(totalStats.getSegments() == null ? null : totalStats.getSegments().getMemory());
+            table.addCell(primaryStats.getSegments() == null ? null : primaryStats.getSegments().getMemory());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getSegments().getIndexWriterMemory());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSegments().getIndexWriterMemory());
+            table.addCell(totalStats.getSegments() == null ? null : totalStats.getSegments().getIndexWriterMemory());
+            table.addCell(primaryStats.getSegments() == null ? null : primaryStats.getSegments().getIndexWriterMemory());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getSegments().getVersionMapMemory());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSegments().getVersionMapMemory());
+            table.addCell(totalStats.getSegments() == null ? null : totalStats.getSegments().getVersionMapMemory());
+            table.addCell(primaryStats.getSegments() == null ? null : primaryStats.getSegments().getVersionMapMemory());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getSegments().getBitsetMemory());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSegments().getBitsetMemory());
+            table.addCell(totalStats.getSegments() == null ? null : totalStats.getSegments().getBitsetMemory());
+            table.addCell(primaryStats.getSegments() == null ? null : primaryStats.getSegments().getBitsetMemory());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getWarmer().current());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getWarmer().current());
+            table.addCell(totalStats.getWarmer() == null ? null : totalStats.getWarmer().current());
+            table.addCell(primaryStats.getWarmer() == null ? null : primaryStats.getWarmer().current());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getWarmer().total());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getWarmer().total());
+            table.addCell(totalStats.getWarmer() == null ? null : totalStats.getWarmer().total());
+            table.addCell(primaryStats.getWarmer() == null ? null : primaryStats.getWarmer().total());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getWarmer().totalTime());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getWarmer().totalTime());
+            table.addCell(totalStats.getWarmer() == null ? null : totalStats.getWarmer().totalTime());
+            table.addCell(primaryStats.getWarmer() == null ? null : primaryStats.getWarmer().totalTime());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getSearch().getTotal().getSuggestCurrent());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSearch().getTotal().getSuggestCurrent());
+            table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getSuggestCurrent());
+            table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getSuggestCurrent());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getSearch().getTotal().getSuggestTime());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSearch().getTotal().getSuggestTime());
+            table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getSuggestTime());
+            table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getSuggestTime());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getSearch().getTotal().getSuggestCount());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSearch().getTotal().getSuggestCount());
+            table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getSuggestCount());
+            table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getSuggestCount());
 
             table.addCell(indexStats == null ? null : indexStats.getTotal().getTotalMemory());
             table.addCell(indexStats == null ? null : indexStats.getPrimaries().getTotalMemory());
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestMasterAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestMasterAction.java
index 5902ba60e5727..db5c1149cc0ad 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestMasterAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestMasterAction.java
@@ -25,9 +25,7 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Table;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -36,8 +34,6 @@
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
 public class RestMasterAction extends AbstractCatAction {
-
-    @Inject
     public RestMasterAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_cat/master", this);
@@ -49,13 +45,13 @@ protected void documentation(StringBuilder sb) {
     }
 
     @Override
-    public void doRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client) {
         final ClusterStateRequest clusterStateRequest = new ClusterStateRequest();
         clusterStateRequest.clear().nodes(true);
         clusterStateRequest.local(request.paramAsBoolean("local", clusterStateRequest.local()));
         clusterStateRequest.masterNodeTimeout(request.paramAsTime("master_timeout", clusterStateRequest.masterNodeTimeout()));
 
-        client.admin().cluster().state(clusterStateRequest, new RestResponseListener<ClusterStateResponse>(channel) {
+        return channel -> client.admin().cluster().state(clusterStateRequest, new RestResponseListener<ClusterStateResponse>(channel) {
             @Override
             public RestResponse buildResponse(final ClusterStateResponse clusterStateResponse) throws Exception {
                 return RestTable.buildResponse(buildTable(request, clusterStateResponse), channel);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodeAttrsAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodeAttrsAction.java
index 5ab98316c7cf5..6819d2421de2a 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodeAttrsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodeAttrsAction.java
@@ -29,10 +29,8 @@
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.Table;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.InetSocketTransportAddress;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -44,8 +42,6 @@
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
 public class RestNodeAttrsAction extends AbstractCatAction {
-
-    @Inject
     public RestNodeAttrsAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_cat/nodeattrs", this);
@@ -57,13 +53,13 @@ protected void documentation(StringBuilder sb) {
     }
 
     @Override
-    public void doRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client) {
         final ClusterStateRequest clusterStateRequest = new ClusterStateRequest();
         clusterStateRequest.clear().nodes(true);
         clusterStateRequest.local(request.paramAsBoolean("local", clusterStateRequest.local()));
         clusterStateRequest.masterNodeTimeout(request.paramAsTime("master_timeout", clusterStateRequest.masterNodeTimeout()));
 
-        client.admin().cluster().state(clusterStateRequest, new RestActionListener<ClusterStateResponse>(channel) {
+        return channel -> client.admin().cluster().state(clusterStateRequest, new RestActionListener<ClusterStateResponse>(channel) {
             @Override
             public void processResponse(final ClusterStateResponse clusterStateResponse) {
                 NodesInfoRequest nodesInfoRequest = new NodesInfoRequest();
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java
index 2c1900feefa91..5f64bb188533f 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java
@@ -32,11 +32,11 @@
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.Table;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.InetSocketTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.http.HttpInfo;
 import org.elasticsearch.index.cache.query.QueryCacheStats;
 import org.elasticsearch.index.cache.request.RequestCacheStats;
@@ -54,7 +54,6 @@
 import org.elasticsearch.monitor.jvm.JvmStats;
 import org.elasticsearch.monitor.os.OsStats;
 import org.elasticsearch.monitor.process.ProcessStats;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -69,8 +68,6 @@
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
 public class RestNodesAction extends AbstractCatAction {
-
-    @Inject
     public RestNodesAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_cat/nodes", this);
@@ -82,13 +79,13 @@ protected void documentation(StringBuilder sb) {
     }
 
     @Override
-    public void doRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client) {
         final ClusterStateRequest clusterStateRequest = new ClusterStateRequest();
         clusterStateRequest.clear().nodes(true);
         clusterStateRequest.local(request.paramAsBoolean("local", clusterStateRequest.local()));
         clusterStateRequest.masterNodeTimeout(request.paramAsTime("master_timeout", clusterStateRequest.masterNodeTimeout()));
-
-        client.admin().cluster().state(clusterStateRequest, new RestActionListener<ClusterStateResponse>(channel) {
+        final boolean fullId = request.paramAsBoolean("full_id", false);
+        return channel -> client.admin().cluster().state(clusterStateRequest, new RestActionListener<ClusterStateResponse>(channel) {
             @Override
             public void processResponse(final ClusterStateResponse clusterStateResponse) {
                 NodesInfoRequest nodesInfoRequest = new NodesInfoRequest();
@@ -101,7 +98,8 @@ public void processResponse(final NodesInfoResponse nodesInfoResponse) {
                         client.admin().cluster().nodesStats(nodesStatsRequest, new RestResponseListener<NodesStatsResponse>(channel) {
                             @Override
                             public RestResponse buildResponse(NodesStatsResponse nodesStatsResponse) throws Exception {
-                                return RestTable.buildResponse(buildTable(request, clusterStateResponse, nodesInfoResponse, nodesStatsResponse), channel);
+                                return RestTable.buildResponse(buildTable(fullId, request, clusterStateResponse, nodesInfoResponse,
+                                    nodesStatsResponse), channel);
                             }
                         });
                     }
@@ -123,7 +121,10 @@ protected Table getTableWithHeader(final RestRequest request) {
         table.addCell("version", "default:false;alias:v;desc:es version");
         table.addCell("build", "default:false;alias:b;desc:es build hash");
         table.addCell("jdk", "default:false;alias:j;desc:jdk version");
-        table.addCell("disk.avail", "default:false;alias:d,disk,diskAvail;text-align:right;desc:available disk space");
+        table.addCell("disk.total", "default:false;alias:dt,diskTotal;text-align:right;desc:total disk space");
+        table.addCell("disk.used", "default:false;alias:du,diskUsed;text-align:right;desc:used disk space");
+        table.addCell("disk.avail", "default:false;alias:d,da,disk,diskAvail;text-align:right;desc:available disk space");
+        table.addCell("disk.used_percent", "default:false;alias:dup,diskUsedPercent;text-align:right;desc:used disk space percentage");
         table.addCell("heap.current", "default:false;alias:hc,heapCurrent;text-align:right;desc:used heap");
         table.addCell("heap.percent", "alias:hp,heapPercent;text-align:right;desc:used heap ratio");
         table.addCell("heap.max", "default:false;alias:hm,heapMax;text-align:right;desc:max configured heap");
@@ -131,7 +132,8 @@ protected Table getTableWithHeader(final RestRequest request) {
         table.addCell("ram.percent", "alias:rp,ramPercent;text-align:right;desc:used machine memory ratio");
         table.addCell("ram.max", "default:false;alias:rm,ramMax;text-align:right;desc:total machine memory");
         table.addCell("file_desc.current", "default:false;alias:fdc,fileDescriptorCurrent;text-align:right;desc:used file descriptors");
-        table.addCell("file_desc.percent", "default:false;alias:fdp,fileDescriptorPercent;text-align:right;desc:used file descriptor ratio");
+        table.addCell("file_desc.percent",
+            "default:false;alias:fdp,fileDescriptorPercent;text-align:right;desc:used file descriptor ratio");
         table.addCell("file_desc.max", "default:false;alias:fdm,fileDescriptorMax;text-align:right;desc:max file descriptors");
 
         table.addCell("cpu", "alias:cpu;text-align:right;desc:recent cpu usage");
@@ -139,7 +141,8 @@ protected Table getTableWithHeader(final RestRequest request) {
         table.addCell("load_5m", "alias:l;text-align:right;desc:5m load avg");
         table.addCell("load_15m", "alias:l;text-align:right;desc:15m load avg");
         table.addCell("uptime", "default:false;alias:u;text-align:right;desc:node uptime");
-        table.addCell("node.role", "alias:r,role,nodeRole;desc:m:master eligible node, d:data node, i:ingest node, -:coordinating node only");
+        table.addCell("node.role",
+            "alias:r,role,nodeRole;desc:m:master eligible node, d:data node, i:ingest node, -:coordinating node only");
         table.addCell("master", "alias:m;desc:*:current master");
         table.addCell("name", "alias:n;desc:node name");
 
@@ -152,9 +155,12 @@ protected Table getTableWithHeader(final RestRequest request) {
         table.addCell("query_cache.evictions", "alias:qce,queryCacheEvictions;default:false;text-align:right;desc:query cache evictions");
 
         table.addCell("request_cache.memory_size", "alias:rcm,requestCacheMemory;default:false;text-align:right;desc:used request cache");
-        table.addCell("request_cache.evictions", "alias:rce,requestCacheEvictions;default:false;text-align:right;desc:request cache evictions");
-        table.addCell("request_cache.hit_count", "alias:rchc,requestCacheHitCount;default:false;text-align:right;desc:request cache hit counts");
-        table.addCell("request_cache.miss_count", "alias:rcmc,requestCacheMissCount;default:false;text-align:right;desc:request cache miss counts");
+        table.addCell("request_cache.evictions",
+            "alias:rce,requestCacheEvictions;default:false;text-align:right;desc:request cache evictions");
+        table.addCell("request_cache.hit_count",
+            "alias:rchc,requestCacheHitCount;default:false;text-align:right;desc:request cache hit counts");
+        table.addCell("request_cache.miss_count",
+            "alias:rcmc,requestCacheMissCount;default:false;text-align:right;desc:request cache miss counts");
 
         table.addCell("flush.total", "alias:ft,flushTotal;default:false;text-align:right;desc:number of flushes");
         table.addCell("flush.total_time", "alias:ftt,flushTotalTime;default:false;text-align:right;desc:time spent in flush");
@@ -167,16 +173,20 @@ protected Table getTableWithHeader(final RestRequest request) {
         table.addCell("get.missing_time", "alias:gmti,getMissingTime;default:false;text-align:right;desc:time spent in failed gets");
         table.addCell("get.missing_total", "alias:gmto,getMissingTotal;default:false;text-align:right;desc:number of failed gets");
 
-        table.addCell("indexing.delete_current", "alias:idc,indexingDeleteCurrent;default:false;text-align:right;desc:number of current deletions");
+        table.addCell("indexing.delete_current",
+            "alias:idc,indexingDeleteCurrent;default:false;text-align:right;desc:number of current deletions");
         table.addCell("indexing.delete_time", "alias:idti,indexingDeleteTime;default:false;text-align:right;desc:time spent in deletions");
         table.addCell("indexing.delete_total", "alias:idto,indexingDeleteTotal;default:false;text-align:right;desc:number of delete ops");
-        table.addCell("indexing.index_current", "alias:iic,indexingIndexCurrent;default:false;text-align:right;desc:number of current indexing ops");
+        table.addCell("indexing.index_current",
+            "alias:iic,indexingIndexCurrent;default:false;text-align:right;desc:number of current indexing ops");
         table.addCell("indexing.index_time", "alias:iiti,indexingIndexTime;default:false;text-align:right;desc:time spent in indexing");
         table.addCell("indexing.index_total", "alias:iito,indexingIndexTotal;default:false;text-align:right;desc:number of indexing ops");
-        table.addCell("indexing.index_failed", "alias:iif,indexingIndexFailed;default:false;text-align:right;desc:number of failed indexing ops");
+        table.addCell("indexing.index_failed",
+            "alias:iif,indexingIndexFailed;default:false;text-align:right;desc:number of failed indexing ops");
 
         table.addCell("merges.current", "alias:mc,mergesCurrent;default:false;text-align:right;desc:number of current merges");
-        table.addCell("merges.current_docs", "alias:mcd,mergesCurrentDocs;default:false;text-align:right;desc:number of current merging docs");
+        table.addCell("merges.current_docs",
+            "alias:mcd,mergesCurrentDocs;default:false;text-align:right;desc:number of current merging docs");
         table.addCell("merges.current_size", "alias:mcs,mergesCurrentSize;default:false;text-align:right;desc:size of current merges");
         table.addCell("merges.total", "alias:mt,mergesTotal;default:false;text-align:right;desc:number of completed merge ops");
         table.addCell("merges.total_docs", "alias:mtd,mergesTotalDocs;default:false;text-align:right;desc:docs merged");
@@ -185,9 +195,12 @@ protected Table getTableWithHeader(final RestRequest request) {
 
         table.addCell("refresh.total", "alias:rto,refreshTotal;default:false;text-align:right;desc:total refreshes");
         table.addCell("refresh.time", "alias:rti,refreshTime;default:false;text-align:right;desc:time spent in refreshes");
+        table.addCell("refresh.listeners", "alias:rli,refreshListeners;default:false;text-align:right;"
+                + "desc:number of pending refresh listeners");
 
         table.addCell("script.compilations", "alias:scrcc,scriptCompilations;default:false;text-align:right;desc:script compilations");
-        table.addCell("script.cache_evictions", "alias:scrce,scriptCacheEvictions;default:false;text-align:right;desc:script cache evictions");
+        table.addCell("script.cache_evictions",
+            "alias:scrce,scriptCacheEvictions;default:false;text-align:right;desc:script cache evictions");
 
         table.addCell("search.fetch_current", "alias:sfc,searchFetchCurrent;default:false;text-align:right;desc:current fetch phase ops");
         table.addCell("search.fetch_time", "alias:sfti,searchFetchTime;default:false;text-align:right;desc:time spent in fetch phase");
@@ -197,14 +210,19 @@ protected Table getTableWithHeader(final RestRequest request) {
         table.addCell("search.query_time", "alias:sqti,searchQueryTime;default:false;text-align:right;desc:time spent in query phase");
         table.addCell("search.query_total", "alias:sqto,searchQueryTotal;default:false;text-align:right;desc:total query phase ops");
         table.addCell("search.scroll_current", "alias:scc,searchScrollCurrent;default:false;text-align:right;desc:open scroll contexts");
-        table.addCell("search.scroll_time", "alias:scti,searchScrollTime;default:false;text-align:right;desc:time scroll contexts held open");
+        table.addCell("search.scroll_time",
+            "alias:scti,searchScrollTime;default:false;text-align:right;desc:time scroll contexts held open");
         table.addCell("search.scroll_total", "alias:scto,searchScrollTotal;default:false;text-align:right;desc:completed scroll contexts");
 
         table.addCell("segments.count", "alias:sc,segmentsCount;default:false;text-align:right;desc:number of segments");
         table.addCell("segments.memory", "alias:sm,segmentsMemory;default:false;text-align:right;desc:memory used by segments");
-        table.addCell("segments.index_writer_memory", "alias:siwm,segmentsIndexWriterMemory;default:false;text-align:right;desc:memory used by index writer");
-        table.addCell("segments.version_map_memory", "alias:svmm,segmentsVersionMapMemory;default:false;text-align:right;desc:memory used by version map");
-        table.addCell("segments.fixed_bitset_memory", "alias:sfbm,fixedBitsetMemory;default:false;text-align:right;desc:memory used by fixed bit sets for nested object field types and type filters for types referred in _parent fields");
+        table.addCell("segments.index_writer_memory",
+            "alias:siwm,segmentsIndexWriterMemory;default:false;text-align:right;desc:memory used by index writer");
+        table.addCell("segments.version_map_memory",
+            "alias:svmm,segmentsVersionMapMemory;default:false;text-align:right;desc:memory used by version map");
+        table.addCell("segments.fixed_bitset_memory",
+            "alias:sfbm,fixedBitsetMemory;default:false;text-align:right;desc:memory used by fixed bit sets for nested object field types" +
+                " and type filters for types referred in _parent fields");
 
         table.addCell("suggest.current", "alias:suc,suggestCurrent;default:false;text-align:right;desc:number of current suggest ops");
         table.addCell("suggest.time", "alias:suti,suggestTime;default:false;text-align:right;desc:time spend in suggest");
@@ -214,8 +232,8 @@ protected Table getTableWithHeader(final RestRequest request) {
         return table;
     }
 
-    private Table buildTable(RestRequest req, ClusterStateResponse state, NodesInfoResponse nodesInfo, NodesStatsResponse nodesStats) {
-        boolean fullId = req.paramAsBoolean("full_id", false);
+    private Table buildTable(boolean fullId, RestRequest req, ClusterStateResponse state, NodesInfoResponse nodesInfo,
+                             NodesStatsResponse nodesStats) {
 
         DiscoveryNodes nodes = state.getState().nodes();
         String masterId = nodes.getMasterNodeId();
@@ -257,7 +275,15 @@ private Table buildTable(RestRequest req, ClusterStateResponse state, NodesInfoR
             table.addCell(node.getVersion().toString());
             table.addCell(info == null ? null : info.getBuild().shortHash());
             table.addCell(jvmInfo == null ? null : jvmInfo.version());
+            
+            long diskTotal = fsInfo.getTotal().getTotal().getBytes();
+            long diskUsed = diskTotal - fsInfo.getTotal().getAvailable().getBytes();
+            double diskUsedRatio = diskTotal == 0 ? 1.0 : (double) diskUsed / diskTotal;
+            table.addCell(fsInfo == null ? null : fsInfo.getTotal().getTotal());
+            table.addCell(fsInfo == null ? null : new ByteSizeValue(diskUsed));
             table.addCell(fsInfo == null ? null : fsInfo.getTotal().getAvailable());
+            table.addCell(fsInfo == null ? null : String.format(Locale.ROOT, "%.2f", 100.0 * diskUsedRatio));
+            
             table.addCell(jvmStats == null ? null : jvmStats.getMem().getHeapUsed());
             table.addCell(jvmStats == null ? null : jvmStats.getMem().getHeapUsedPercent());
             table.addCell(jvmInfo == null ? null : jvmInfo.getMem().getHeapMax());
@@ -265,14 +291,18 @@ private Table buildTable(RestRequest req, ClusterStateResponse state, NodesInfoR
             table.addCell(osStats == null ? null : osStats.getMem() == null ? null : osStats.getMem().getUsedPercent());
             table.addCell(osStats == null ? null : osStats.getMem() == null ? null : osStats.getMem().getTotal());
             table.addCell(processStats == null ? null : processStats.getOpenFileDescriptors());
-            table.addCell(processStats == null ? null : calculatePercentage(processStats.getOpenFileDescriptors(), processStats.getMaxFileDescriptors()));
+            table.addCell(processStats == null ? null : calculatePercentage(processStats.getOpenFileDescriptors(),
+                processStats.getMaxFileDescriptors()));
             table.addCell(processStats == null ? null : processStats.getMaxFileDescriptors());
 
             table.addCell(osStats == null ? null : Short.toString(osStats.getCpu().getPercent()));
             boolean hasLoadAverage = osStats != null && osStats.getCpu().getLoadAverage() != null;
-            table.addCell(!hasLoadAverage || osStats.getCpu().getLoadAverage()[0] == -1 ? null : String.format(Locale.ROOT, "%.2f", osStats.getCpu().getLoadAverage()[0]));
-            table.addCell(!hasLoadAverage || osStats.getCpu().getLoadAverage()[1] == -1 ? null : String.format(Locale.ROOT, "%.2f", osStats.getCpu().getLoadAverage()[1]));
-            table.addCell(!hasLoadAverage || osStats.getCpu().getLoadAverage()[2] == -1 ? null : String.format(Locale.ROOT, "%.2f", osStats.getCpu().getLoadAverage()[2]));
+            table.addCell(!hasLoadAverage || osStats.getCpu().getLoadAverage()[0] == -1 ? null :
+                String.format(Locale.ROOT, "%.2f", osStats.getCpu().getLoadAverage()[0]));
+            table.addCell(!hasLoadAverage || osStats.getCpu().getLoadAverage()[1] == -1 ? null :
+                String.format(Locale.ROOT, "%.2f", osStats.getCpu().getLoadAverage()[1]));
+            table.addCell(!hasLoadAverage || osStats.getCpu().getLoadAverage()[2] == -1 ? null :
+                String.format(Locale.ROOT, "%.2f", osStats.getCpu().getLoadAverage()[2]));
             table.addCell(jvmStats == null ? null : jvmStats.getUptime());
 
             final String roles;
@@ -336,6 +366,7 @@ private Table buildTable(RestRequest req, ClusterStateResponse state, NodesInfoR
             RefreshStats refreshStats = indicesStats == null ? null : indicesStats.getRefresh();
             table.addCell(refreshStats == null ? null : refreshStats.getTotal());
             table.addCell(refreshStats == null ? null : refreshStats.getTotalTime());
+            table.addCell(refreshStats == null ? null : refreshStats.getListeners());
 
             ScriptStats scriptStats = stats == null ? null : stats.getScriptStats();
             table.addCell(scriptStats == null ? null : scriptStats.getCompilations());
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestPendingClusterTasksAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestPendingClusterTasksAction.java
index 773c6d292b5fc..a9f044c144642 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestPendingClusterTasksAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestPendingClusterTasksAction.java
@@ -24,9 +24,7 @@
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.service.PendingClusterTask;
 import org.elasticsearch.common.Table;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -35,7 +33,6 @@
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
 public class RestPendingClusterTasksAction extends AbstractCatAction {
-    @Inject
     public RestPendingClusterTasksAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_cat/pending_tasks", this);
@@ -47,17 +44,20 @@ protected void documentation(StringBuilder sb) {
     }
 
     @Override
-    public void doRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client) {
         PendingClusterTasksRequest pendingClusterTasksRequest = new PendingClusterTasksRequest();
         pendingClusterTasksRequest.masterNodeTimeout(request.paramAsTime("master_timeout", pendingClusterTasksRequest.masterNodeTimeout()));
         pendingClusterTasksRequest.local(request.paramAsBoolean("local", pendingClusterTasksRequest.local()));
-        client.admin().cluster().pendingClusterTasks(pendingClusterTasksRequest, new RestResponseListener<PendingClusterTasksResponse>(channel) {
-            @Override
-            public RestResponse buildResponse(PendingClusterTasksResponse pendingClusterTasks) throws Exception {
-                Table tab = buildTable(request, pendingClusterTasks);
-                return RestTable.buildResponse(tab, channel);
-            }
-        });
+        return channel ->
+                client.admin()
+                        .cluster()
+                        .pendingClusterTasks(pendingClusterTasksRequest, new RestResponseListener<PendingClusterTasksResponse>(channel) {
+                            @Override
+                            public RestResponse buildResponse(PendingClusterTasksResponse pendingClusterTasks) throws Exception {
+                                Table tab = buildTable(request, pendingClusterTasks);
+                                return RestTable.buildResponse(tab, channel);
+                            }
+                        });
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestPluginsAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestPluginsAction.java
index ef8385653f166..7851c15b32f76 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestPluginsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestPluginsAction.java
@@ -28,10 +28,8 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Table;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.PluginInfo;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -41,8 +39,6 @@
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
 public class RestPluginsAction extends AbstractCatAction {
-
-    @Inject
     public RestPluginsAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_cat/plugins", this);
@@ -54,13 +50,13 @@ protected void documentation(StringBuilder sb) {
     }
 
     @Override
-    public void doRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client) {
         final ClusterStateRequest clusterStateRequest = new ClusterStateRequest();
         clusterStateRequest.clear().nodes(true);
         clusterStateRequest.local(request.paramAsBoolean("local", clusterStateRequest.local()));
         clusterStateRequest.masterNodeTimeout(request.paramAsTime("master_timeout", clusterStateRequest.masterNodeTimeout()));
 
-        client.admin().cluster().state(clusterStateRequest, new RestActionListener<ClusterStateResponse>(channel) {
+        return channel -> client.admin().cluster().state(clusterStateRequest, new RestActionListener<ClusterStateResponse>(channel) {
             @Override
             public void processResponse(final ClusterStateResponse clusterStateResponse) throws Exception {
                 NodesInfoRequest nodesInfoRequest = new NodesInfoRequest();
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestRecoveryAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestRecoveryAction.java
index b0ab8db8b2973..e2e831f890d99 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestRecoveryAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestRecoveryAction.java
@@ -28,11 +28,9 @@
 import org.elasticsearch.cluster.routing.RecoverySource.SnapshotRecoverySource;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.Table;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.indices.recovery.RecoveryState;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -50,9 +48,7 @@
  * be specified to limit output to a particular index or indices.
  */
 public class RestRecoveryAction extends AbstractCatAction {
-
-    @Inject
-    public RestRecoveryAction(Settings settings, RestController restController, RestController controller) {
+    public RestRecoveryAction(Settings settings, RestController restController) {
         super(settings);
         restController.registerHandler(GET, "/_cat/recovery", this);
         restController.registerHandler(GET, "/_cat/recovery/{index}", this);
@@ -65,13 +61,13 @@ protected void documentation(StringBuilder sb) {
     }
 
     @Override
-    public void doRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client) {
         final RecoveryRequest recoveryRequest = new RecoveryRequest(Strings.splitStringByCommaToArray(request.param("index")));
         recoveryRequest.detailed(request.paramAsBoolean("detailed", false));
         recoveryRequest.activeOnly(request.paramAsBoolean("active_only", false));
         recoveryRequest.indicesOptions(IndicesOptions.fromRequest(request, recoveryRequest.indicesOptions()));
 
-        client.admin().indices().recoveries(recoveryRequest, new RestResponseListener<RecoveryResponse>(channel) {
+        return channel -> client.admin().indices().recoveries(recoveryRequest, new RestResponseListener<RecoveryResponse>(channel) {
             @Override
             public RestResponse buildResponse(final RecoveryResponse response) throws Exception {
                 return RestTable.buildResponse(buildRecoveryTable(request, response), channel);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestRepositoriesAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestRepositoriesAction.java
index 05130504e5091..631c03050492d 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestRepositoriesAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestRepositoriesAction.java
@@ -24,9 +24,7 @@
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.metadata.RepositoryMetaData;
 import org.elasticsearch.common.Table;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -38,24 +36,26 @@
  * Cat API class to display information about snapshot repositories
  */
 public class RestRepositoriesAction extends AbstractCatAction {
-    @Inject
     public RestRepositoriesAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_cat/repositories", this);
     }
 
     @Override
-    protected void doRequest(RestRequest request, RestChannel channel, NodeClient client) {
+    protected RestChannelConsumer doCatRequest(RestRequest request, NodeClient client) {
         GetRepositoriesRequest getRepositoriesRequest = new GetRepositoriesRequest();
         getRepositoriesRequest.local(request.paramAsBoolean("local", getRepositoriesRequest.local()));
         getRepositoriesRequest.masterNodeTimeout(request.paramAsTime("master_timeout", getRepositoriesRequest.masterNodeTimeout()));
 
-        client.admin().cluster().getRepositories(getRepositoriesRequest, new RestResponseListener<GetRepositoriesResponse>(channel) {
-            @Override
-            public RestResponse buildResponse(GetRepositoriesResponse getRepositoriesResponse) throws Exception {
-                return RestTable.buildResponse(buildTable(request, getRepositoriesResponse), channel);
-            }
-        });
+        return channel ->
+                client.admin()
+                        .cluster()
+                        .getRepositories(getRepositoriesRequest, new RestResponseListener<GetRepositoriesResponse>(channel) {
+                            @Override
+                            public RestResponse buildResponse(GetRepositoriesResponse getRepositoriesResponse) throws Exception {
+                                return RestTable.buildResponse(buildTable(request, getRepositoriesResponse), channel);
+                            }
+                        });
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestSegmentsAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestSegmentsAction.java
index d2b30c49ca9d6..48983ab836b7a 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestSegmentsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestSegmentsAction.java
@@ -30,10 +30,8 @@
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.Table;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.engine.Segment;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -46,8 +44,6 @@
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
 public class RestSegmentsAction extends AbstractCatAction {
-
-    @Inject
     public RestSegmentsAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_cat/segments", this);
@@ -55,7 +51,7 @@ public RestSegmentsAction(Settings settings, RestController controller) {
     }
 
     @Override
-    protected void doRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    protected RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client) {
         final String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
 
         final ClusterStateRequest clusterStateRequest = new ClusterStateRequest();
@@ -63,7 +59,7 @@ protected void doRequest(final RestRequest request, final RestChannel channel, f
         clusterStateRequest.masterNodeTimeout(request.paramAsTime("master_timeout", clusterStateRequest.masterNodeTimeout()));
         clusterStateRequest.clear().nodes(true).routingTable(true).indices(indices);
 
-        client.admin().cluster().state(clusterStateRequest, new RestActionListener<ClusterStateResponse>(channel) {
+        return channel -> client.admin().cluster().state(clusterStateRequest, new RestActionListener<ClusterStateResponse>(channel) {
             @Override
             public void processResponse(final ClusterStateResponse clusterStateResponse) {
                 final IndicesSegmentsRequest indicesSegmentsRequest = new IndicesSegmentsRequest();
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java
index 2dd4b6a10d137..3582fe60e0040 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java
@@ -31,12 +31,10 @@
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.Table;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.engine.CommitStats;
 import org.elasticsearch.index.engine.Engine;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -48,8 +46,6 @@
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
 public class RestShardsAction extends AbstractCatAction {
-
-    @Inject
     public RestShardsAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_cat/shards", this);
@@ -63,13 +59,13 @@ protected void documentation(StringBuilder sb) {
     }
 
     @Override
-    public void doRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client) {
         final String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
         final ClusterStateRequest clusterStateRequest = new ClusterStateRequest();
         clusterStateRequest.local(request.paramAsBoolean("local", clusterStateRequest.local()));
         clusterStateRequest.masterNodeTimeout(request.paramAsTime("master_timeout", clusterStateRequest.masterNodeTimeout()));
         clusterStateRequest.clear().nodes(true).metaData(true).routingTable(true).indices(indices);
-        client.admin().cluster().state(clusterStateRequest, new RestActionListener<ClusterStateResponse>(channel) {
+        return channel -> client.admin().cluster().state(clusterStateRequest, new RestActionListener<ClusterStateResponse>(channel) {
             @Override
             public void processResponse(final ClusterStateResponse clusterStateResponse) {
                 IndicesStatsRequest indicesStatsRequest = new IndicesStatsRequest();
@@ -145,6 +141,7 @@ protected Table getTableWithHeader(final RestRequest request) {
 
         table.addCell("refresh.total", "alias:rto,refreshTotal;default:false;text-align:right;desc:total refreshes");
         table.addCell("refresh.time", "alias:rti,refreshTime;default:false;text-align:right;desc:time spent in refreshes");
+        table.addCell("refresh.listeners", "alias:rli,refreshListeners;default:false;text-align:right;desc:number of pending refresh listeners");
 
         table.addCell("search.fetch_current", "alias:sfc,searchFetchCurrent;default:false;text-align:right;desc:current fetch phase ops");
         table.addCell("search.fetch_time", "alias:sfti,searchFetchTime;default:false;text-align:right;desc:time spent in fetch phase");
@@ -287,6 +284,7 @@ private Table buildTable(RestRequest request, ClusterStateResponse state, Indice
 
             table.addCell(commonStats == null ? null : commonStats.getRefresh().getTotal());
             table.addCell(commonStats == null ? null : commonStats.getRefresh().getTotalTime());
+            table.addCell(commonStats == null ? null : commonStats.getRefresh().getListeners());
 
             table.addCell(commonStats == null ? null : commonStats.getSearch().getTotal().getFetchCurrent());
             table.addCell(commonStats == null ? null : commonStats.getSearch().getTotal().getFetchTime());
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestSnapshotAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestSnapshotAction.java
index 021b00be24e1c..54337e5e14369 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestSnapshotAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestSnapshotAction.java
@@ -24,10 +24,8 @@
 import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Table;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -45,14 +43,14 @@
  * Cat API class to display information about snapshots
  */
 public class RestSnapshotAction extends AbstractCatAction {
-    @Inject
     public RestSnapshotAction(Settings settings, RestController controller) {
         super(settings);
+        controller.registerHandler(GET, "/_cat/snapshots", this);
         controller.registerHandler(GET, "/_cat/snapshots/{repository}", this);
     }
 
     @Override
-    protected void doRequest(final RestRequest request, RestChannel channel, NodeClient client) {
+    protected RestChannelConsumer doCatRequest(final RestRequest request, NodeClient client) {
         GetSnapshotsRequest getSnapshotsRequest = new GetSnapshotsRequest()
                 .repository(request.param("repository"))
                 .snapshots(new String[]{GetSnapshotsRequest.ALL_SNAPSHOTS});
@@ -61,12 +59,13 @@ protected void doRequest(final RestRequest request, RestChannel channel, NodeCli
 
         getSnapshotsRequest.masterNodeTimeout(request.paramAsTime("master_timeout", getSnapshotsRequest.masterNodeTimeout()));
 
-        client.admin().cluster().getSnapshots(getSnapshotsRequest, new RestResponseListener<GetSnapshotsResponse>(channel) {
-            @Override
-            public RestResponse buildResponse(GetSnapshotsResponse getSnapshotsResponse) throws Exception {
-                return RestTable.buildResponse(buildTable(request, getSnapshotsResponse), channel);
-            }
-        });
+        return channel ->
+            client.admin().cluster().getSnapshots(getSnapshotsRequest, new RestResponseListener<GetSnapshotsResponse>(channel) {
+                @Override
+                public RestResponse buildResponse(GetSnapshotsResponse getSnapshotsResponse) throws Exception {
+                    return RestTable.buildResponse(buildTable(request, getSnapshotsResponse), channel);
+                }
+            });
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestTable.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestTable.java
index 78d30407ffc51..ac8237471cc2f 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestTable.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestTable.java
@@ -22,8 +22,11 @@
 import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.Table;
+import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.io.UTF8StreamWriter;
-import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.BytesStream;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.SizeValue;
@@ -38,13 +41,16 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
 import java.util.LinkedHashSet;
 import java.util.List;
+import java.util.Locale;
+import java.util.Map;
 import java.util.Set;
 
-/**
- */
 public class RestTable {
+    private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(RestRequest.class));
 
     public static RestResponse buildResponse(Table table, RestChannel channel) throws Exception {
         RestRequest request = channel.request();
@@ -61,13 +67,13 @@ public static RestResponse buildXContentBuilder(Table table, RestChannel channel
         List<DisplayHeader> displayHeaders = buildDisplayHeaders(table, request);
 
         builder.startArray();
-        for (int row = 0; row < table.getRows().size(); row++) {
+        List<Integer> rowOrder = getRowOrder(table, request);
+        for (Integer row : rowOrder) {
             builder.startObject();
             for (DisplayHeader header : displayHeaders) {
                 builder.field(header.display, renderValue(request, table.getAsMap().get(header.name).get(row).value));
             }
             builder.endObject();
-
         }
         builder.endArray();
         return new BytesRestResponse(RestStatus.OK, builder);
@@ -80,7 +86,7 @@ public static RestResponse buildTextPlainResponse(Table table, RestChannel chann
         List<DisplayHeader> headers = buildDisplayHeaders(table, request);
         int[] width = buildWidths(table, request, verbose, headers);
 
-        BytesStreamOutput bytesOut = channel.bytesOutput();
+        BytesStream bytesOut = Streams.flushOnCloseStream(channel.bytesOutput());
         UTF8StreamWriter out = new UTF8StreamWriter().setOutput(bytesOut);
         int lastHeader = headers.size() - 1;
         if (verbose) {
@@ -94,7 +100,10 @@ public static RestResponse buildTextPlainResponse(Table table, RestChannel chann
             }
             out.append("\n");
         }
-        for (int row = 0; row < table.getRows().size(); row++) {
+
+        List<Integer> rowOrder = getRowOrder(table, request);
+
+        for (Integer row: rowOrder) {
             for (int col = 0; col < headers.size(); col++) {
                 DisplayHeader header = headers.get(col);
                 boolean isLastColumn = col == lastHeader;
@@ -109,6 +118,38 @@ public static RestResponse buildTextPlainResponse(Table table, RestChannel chann
         return new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, bytesOut.bytes());
     }
 
+    static List<Integer> getRowOrder(Table table, RestRequest request) {
+        String[] columnOrdering = request.paramAsStringArray("s", null);
+
+        List<Integer> rowOrder = new ArrayList<>();
+        for (int i = 0; i < table.getRows().size(); i++) {
+            rowOrder.add(i);
+        }
+
+        if (columnOrdering != null) {
+            Map<String, String> headerAliasMap = table.getAliasMap();
+            List<ColumnOrderElement> ordering = new ArrayList<>();
+            for (int i = 0; i < columnOrdering.length; i++) {
+                String columnHeader = columnOrdering[i];
+                boolean reverse = false;
+                if (columnHeader.endsWith(":desc")) {
+                    columnHeader = columnHeader.substring(0, columnHeader.length() - ":desc".length());
+                    reverse = true;
+                } else if (columnHeader.endsWith(":asc")) {
+                    columnHeader = columnHeader.substring(0, columnHeader.length() - ":asc".length());
+                }
+                if (headerAliasMap.containsKey(columnHeader)) {
+                        ordering.add(new ColumnOrderElement(headerAliasMap.get(columnHeader), reverse));
+                } else {
+                    throw new UnsupportedOperationException(
+                        String.format(Locale.ROOT, "Unable to sort by unknown sort key `%s`", columnHeader));
+                }
+            }
+            Collections.sort(rowOrder, new TableIndexComparator(table, ordering));
+        }
+        return rowOrder;
+    }
+
     static List<DisplayHeader> buildDisplayHeaders(Table table, RestRequest request) {
         List<DisplayHeader> display = new ArrayList<>();
         if (request.hasParam("h")) {
@@ -153,7 +194,13 @@ static List<DisplayHeader> buildDisplayHeaders(Table table, RestRequest request)
         } else {
             for (Table.Cell cell : table.getHeaders()) {
                 String d = cell.attr.get("default");
-                if (Booleans.parseBoolean(d, true) && checkOutputTimestamp(cell.value.toString(), request)) {
+                boolean defaultValue = Booleans.parseBoolean(d, true);
+                if (d != null && Booleans.isStrictlyBoolean(d) == false) {
+                    DEPRECATION_LOGGER.deprecated(
+                        "Expected a boolean [true/false] for attribute [default] of table header [{}] but got [{}]",
+                        cell.value.toString(), d);
+                }
+                if (defaultValue && checkOutputTimestamp(cell.value.toString(), request)) {
                     display.add(new DisplayHeader(cell.value.toString(), cell.value.toString()));
                 }
             }
@@ -302,17 +349,17 @@ private static String renderValue(RestRequest request, Object value) {
             ByteSizeValue v = (ByteSizeValue) value;
             String resolution = request.param("bytes");
             if ("b".equals(resolution)) {
-                return Long.toString(v.bytes());
+                return Long.toString(v.getBytes());
             } else if ("k".equals(resolution) || "kb".equals(resolution)) {
-                return Long.toString(v.kb());
+                return Long.toString(v.getKb());
             } else if ("m".equals(resolution) || "mb".equals(resolution)) {
-                return Long.toString(v.mb());
+                return Long.toString(v.getMb());
             } else if ("g".equals(resolution) || "gb".equals(resolution)) {
-                return Long.toString(v.gb());
+                return Long.toString(v.getGb());
             } else if ("t".equals(resolution) || "tb".equals(resolution)) {
-                return Long.toString(v.tb());
+                return Long.toString(v.getTb());
             } else if ("p".equals(resolution) || "pb".equals(resolution)) {
-                return Long.toString(v.pb());
+                return Long.toString(v.getPb());
             } else {
                 return v.toString();
             }
@@ -370,4 +417,71 @@ static class DisplayHeader {
             this.display = display;
         }
     }
+
+    static class TableIndexComparator implements Comparator<Integer> {
+        private final Table table;
+        private final int maxIndex;
+        private final List<ColumnOrderElement> ordering;
+
+        TableIndexComparator(Table table, List<ColumnOrderElement> ordering) {
+            this.table = table;
+            this.maxIndex = table.getRows().size();
+            this.ordering = ordering;
+        }
+
+        private int compareCell(Object o1, Object o2) {
+            if (o1 == null && o2 == null) {
+                return 0;
+            } else if (o1 == null) {
+                return -1;
+            } else if (o2 == null) {
+                return 1;
+            } else {
+                if (o1 instanceof Comparable && o1.getClass().equals(o2.getClass())) {
+                    return ((Comparable) o1).compareTo(o2);
+                } else {
+                    return o1.toString().compareTo(o2.toString());
+                }
+            }
+        }
+
+        @Override
+        public int compare(Integer rowIndex1, Integer rowIndex2) {
+            if (rowIndex1 < maxIndex && rowIndex1 >= 0 && rowIndex2 < maxIndex && rowIndex2 >= 0) {
+                Map<String, List<Table.Cell>> tableMap = table.getAsMap();
+                for (ColumnOrderElement orderingElement : ordering) {
+                    String column = orderingElement.getColumn();
+                    if (tableMap.containsKey(column)) {
+                        int comparison = compareCell(tableMap.get(column).get(rowIndex1).value,
+                            tableMap.get(column).get(rowIndex2).value);
+                        if (comparison != 0) {
+                            return orderingElement.isReversed() ? -1 * comparison : comparison;
+                        }
+                    }
+                }
+                return 0;
+            } else {
+                throw new AssertionError(String.format(Locale.ENGLISH, "Invalid comparison of indices (%s, %s): Table has %s rows.",
+                    rowIndex1, rowIndex2, table.getRows().size()));
+            }
+        }
+    }
+
+    static class ColumnOrderElement {
+        private final String column;
+        private final boolean reverse;
+
+        ColumnOrderElement(String column, boolean reverse) {
+            this.column = column;
+            this.reverse = reverse;
+        }
+
+        public String getColumn() {
+            return column;
+        }
+
+        public boolean isReversed() {
+            return reverse;
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestTasksAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestTasksAction.java
index 99eee4e735ad0..a1b48b7115be3 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestTasksAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestTasksAction.java
@@ -24,14 +24,11 @@
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.Table;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.InetSocketTransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -41,19 +38,22 @@
 import org.joda.time.format.DateTimeFormatter;
 
 import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
+import java.util.function.Supplier;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.action.admin.cluster.RestListTasksAction.generateListTasksRequest;
 
 public class RestTasksAction extends AbstractCatAction {
-    private final ClusterService clusterService;
+    private final Supplier<DiscoveryNodes> nodesInCluster;
 
-    @Inject
-    public RestTasksAction(Settings settings, RestController controller, ClusterService clusterService) {
+    public RestTasksAction(Settings settings, RestController controller, Supplier<DiscoveryNodes> nodesInCluster) {
         super(settings);
         controller.registerHandler(GET, "/_cat/tasks", this);
-        this.clusterService = clusterService;
+        this.nodesInCluster = nodesInCluster;
     }
 
     @Override
@@ -62,8 +62,9 @@ protected void documentation(StringBuilder sb) {
     }
 
     @Override
-    public void doRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
-        client.admin().cluster().listTasks(generateListTasksRequest(request), new RestResponseListener<ListTasksResponse>(channel) {
+    public RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client) {
+        return channel ->
+                client.admin().cluster().listTasks(generateListTasksRequest(request), new RestResponseListener<ListTasksResponse>(channel) {
             @Override
             public RestResponse buildResponse(ListTasksResponse listTasksResponse) throws Exception {
                 return RestTable.buildResponse(buildTable(request, listTasksResponse), channel);
@@ -71,6 +72,20 @@ public RestResponse buildResponse(ListTasksResponse listTasksResponse) throws Ex
         });
     }
 
+    private static final Set<String> RESPONSE_PARAMS;
+
+    static {
+        final Set<String> responseParams = new HashSet<>();
+        responseParams.add("detailed");
+        responseParams.addAll(AbstractCatAction.RESPONSE_PARAMS);
+        RESPONSE_PARAMS = Collections.unmodifiableSet(responseParams);
+    }
+
+    @Override
+    protected Set<String> responseParams() {
+        return RESPONSE_PARAMS;
+    }
+
     @Override
     protected Table getTableWithHeader(final RestRequest request) {
         boolean detailed = request.paramAsBoolean("detailed", false);
@@ -142,7 +157,7 @@ private void buildRow(Table table, boolean fullId, boolean detailed, DiscoveryNo
     }
 
     private void buildGroups(Table table, boolean fullId, boolean detailed, List<TaskGroup> taskGroups) {
-        DiscoveryNodes discoveryNodes = clusterService.state().nodes();
+        DiscoveryNodes discoveryNodes = nodesInCluster.get();
         List<TaskGroup> sortedGroups = new ArrayList<>(taskGroups);
         sortedGroups.sort((o1, o2) -> Long.compare(o1.getTaskInfo().getStartTime(), o2.getTaskInfo().getStartTime()));
         for (TaskGroup taskGroup : sortedGroups) {
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestTemplatesAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestTemplatesAction.java
new file mode 100644
index 0000000000000..8416a346299d2
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestTemplatesAction.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.rest.action.cat;
+
+import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
+
+import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.common.Table;
+import org.elasticsearch.common.regex.Regex;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestResponse;
+import org.elasticsearch.rest.action.RestResponseListener;
+
+import static org.elasticsearch.rest.RestRequest.Method.GET;
+
+public class RestTemplatesAction extends AbstractCatAction {
+    public RestTemplatesAction(Settings settings, RestController controller) {
+        super(settings);
+        controller.registerHandler(GET, "/_cat/templates", this);
+        controller.registerHandler(GET, "/_cat/templates/{name}", this);
+    }
+
+    @Override
+    protected void documentation(StringBuilder sb) {
+        sb.append("/_cat/templates\n");
+    }
+
+    @Override
+    protected RestChannelConsumer doCatRequest(final RestRequest request, NodeClient client) {
+        final String matchPattern = request.hasParam("name") ? request.param("name") : null;
+        final ClusterStateRequest clusterStateRequest = new ClusterStateRequest();
+        clusterStateRequest.clear().metaData(true);
+        clusterStateRequest.local(request.paramAsBoolean("local", clusterStateRequest.local()));
+        clusterStateRequest.masterNodeTimeout(request.paramAsTime("master_timeout", clusterStateRequest.masterNodeTimeout()));
+
+        return channel -> client.admin().cluster().state(clusterStateRequest, new RestResponseListener<ClusterStateResponse>(channel) {
+            @Override
+            public RestResponse buildResponse(ClusterStateResponse clusterStateResponse) throws Exception {
+                return RestTable.buildResponse(buildTable(request, clusterStateResponse, matchPattern), channel);
+            }
+        });
+    }
+
+    @Override
+    protected Table getTableWithHeader(RestRequest request) {
+        Table table = new Table();
+        table.startHeaders();
+        table.addCell("name", "alias:n;desc:template name");
+        table.addCell("template", "alias:t;desc:template pattern string");
+        table.addCell("order", "alias:o;desc:template application order number");
+        table.addCell("version", "alias:v;desc:version");
+        table.endHeaders();
+        return table;
+    }
+
+    private Table buildTable(RestRequest request, ClusterStateResponse clusterStateResponse, String patternString) {
+        Table table = getTableWithHeader(request);
+        MetaData metadata = clusterStateResponse.getState().metaData();
+        for (ObjectObjectCursor<String, IndexTemplateMetaData> entry : metadata.templates()) {
+            IndexTemplateMetaData indexData = entry.value;
+            if (patternString == null || Regex.simpleMatch(patternString, indexData.name())) {
+                table.startRow();
+                table.addCell(indexData.name());
+                table.addCell(indexData.getTemplate());
+                table.addCell(indexData.getOrder());
+                table.addCell(indexData.getVersion());
+                table.endRow();
+            }
+        }
+        return table;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestThreadPoolAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestThreadPoolAction.java
index 6f3c5c11ce01e..f841a3a77e8da 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestThreadPoolAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestThreadPoolAction.java
@@ -31,11 +31,9 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Table;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.InetSocketTransportAddress;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -54,8 +52,6 @@
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
 public class RestThreadPoolAction extends AbstractCatAction {
-
-    @Inject
     public RestThreadPoolAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_cat/thread_pool", this);
@@ -65,17 +61,17 @@ public RestThreadPoolAction(Settings settings, RestController controller) {
     @Override
     protected void documentation(StringBuilder sb) {
         sb.append("/_cat/thread_pool\n");
-        sb.append("/_cat/thread_pool/{thread_pools}");
+        sb.append("/_cat/thread_pool/{thread_pools}\n");
     }
 
     @Override
-    public void doRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client) {
         final ClusterStateRequest clusterStateRequest = new ClusterStateRequest();
         clusterStateRequest.clear().nodes(true);
         clusterStateRequest.local(request.paramAsBoolean("local", clusterStateRequest.local()));
         clusterStateRequest.masterNodeTimeout(request.paramAsTime("master_timeout", clusterStateRequest.masterNodeTimeout()));
 
-        client.admin().cluster().state(clusterStateRequest, new RestActionListener<ClusterStateResponse>(channel) {
+        return channel -> client.admin().cluster().state(clusterStateRequest, new RestActionListener<ClusterStateResponse>(channel) {
             @Override
             public void processResponse(final ClusterStateResponse clusterStateResponse) {
                 NodesInfoRequest nodesInfoRequest = new NodesInfoRequest();
@@ -97,6 +93,19 @@ public RestResponse buildResponse(NodesStatsResponse nodesStatsResponse) throws
         });
     }
 
+    private static final Set<String> RESPONSE_PARAMS;
+
+    static {
+        final Set<String> responseParams = new HashSet<>(AbstractCatAction.RESPONSE_PARAMS);
+        responseParams.add("thread_pool_patterns");
+        RESPONSE_PARAMS = Collections.unmodifiableSet(responseParams);
+    }
+
+    @Override
+    protected Set<String> responseParams() {
+        return RESPONSE_PARAMS;
+    }
+
     @Override
     protected Table getTableWithHeader(final RestRequest request) {
         final Table table = new Table();
diff --git a/core/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java b/core/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java
index f5dca3f22c9f8..8745c4bfe2d98 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java
@@ -19,28 +19,25 @@
 
 package org.elasticsearch.rest.action.document;
 
-import org.elasticsearch.action.bulk.BulkItemResponse;
 import org.elasticsearch.action.bulk.BulkRequest;
-import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.bulk.BulkShardRequest;
 import org.elasticsearch.action.support.ActiveShardCount;
-import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.client.Requests;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.RestResponse;
-import org.elasticsearch.rest.action.RestBuilderListener;
+import org.elasticsearch.rest.action.RestStatusToXContentListener;
+import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
+
+import java.io.IOException;
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.rest.RestRequest.Method.PUT;
-import static org.elasticsearch.rest.RestStatus.OK;
 
 /**
  * <pre>
@@ -52,10 +49,11 @@
  * </pre>
  */
 public class RestBulkAction extends BaseRestHandler {
+    private static final DeprecationLogger DEPRECATION_LOGGER =
+        new DeprecationLogger(Loggers.getLogger(RestBulkAction.class));
 
     private final boolean allowExplicitIndex;
 
-    @Inject
     public RestBulkAction(Settings settings, RestController controller) {
         super(settings);
 
@@ -70,52 +68,32 @@ public RestBulkAction(Settings settings, RestController controller) {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) throws Exception {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         BulkRequest bulkRequest = Requests.bulkRequest();
         String defaultIndex = request.param("index");
         String defaultType = request.param("type");
         String defaultRouting = request.param("routing");
+        FetchSourceContext defaultFetchSourceContext = FetchSourceContext.parseFromRestRequest(request);
         String fieldsParam = request.param("fields");
-        String defaultPipeline = request.param("pipeline");
+        if (fieldsParam != null) {
+            DEPRECATION_LOGGER.deprecated("Deprecated field [fields] used, expected [_source] instead");
+        }
         String[] defaultFields = fieldsParam != null ? Strings.commaDelimitedListToStringArray(fieldsParam) : null;
-
+        String defaultPipeline = request.param("pipeline");
         String waitForActiveShards = request.param("wait_for_active_shards");
         if (waitForActiveShards != null) {
             bulkRequest.waitForActiveShards(ActiveShardCount.parseString(waitForActiveShards));
         }
         bulkRequest.timeout(request.paramAsTime("timeout", BulkShardRequest.DEFAULT_TIMEOUT));
         bulkRequest.setRefreshPolicy(request.param("refresh"));
-        bulkRequest.add(request.content(), defaultIndex, defaultType, defaultRouting, defaultFields, defaultPipeline,
-                null, allowExplicitIndex);
-
-        client.bulk(bulkRequest, new RestBuilderListener<BulkResponse>(channel) {
-            @Override
-            public RestResponse buildResponse(BulkResponse response, XContentBuilder builder) throws Exception {
-                builder.startObject();
-                builder.field(Fields.TOOK, response.getTookInMillis());
-                if (response.getIngestTookInMillis() != BulkResponse.NO_INGEST_TOOK) {
-                    builder.field(Fields.INGEST_TOOK, response.getIngestTookInMillis());
-                }
-                builder.field(Fields.ERRORS, response.hasFailures());
-                builder.startArray(Fields.ITEMS);
-                for (BulkItemResponse itemResponse : response) {
-                    builder.startObject();
-                    itemResponse.toXContent(builder, request);
-                    builder.endObject();
-                }
-                builder.endArray();
+        bulkRequest.add(request.requiredContent(), defaultIndex, defaultType, defaultRouting, defaultFields,
+            defaultFetchSourceContext, defaultPipeline, null, allowExplicitIndex, request.getXContentType());
 
-                builder.endObject();
-                return new BytesRestResponse(OK, builder);
-            }
-        });
+        return channel -> client.bulk(bulkRequest, new RestStatusToXContentListener<>(channel));
     }
 
-    static final class Fields {
-        static final String ITEMS = "items";
-        static final String ERRORS = "errors";
-        static final String TOOK = "took";
-        static final String INGEST_TOOK = "ingest_took";
+    @Override
+    public boolean supportsContentStream() {
+        return true;
     }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/document/RestCountAction.java b/core/src/main/java/org/elasticsearch/rest/action/document/RestCountAction.java
index e5ca6f2cadd02..4d469ea3e0ce6 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/document/RestCountAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/document/RestCountAction.java
@@ -24,15 +24,11 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -40,20 +36,15 @@
 import org.elasticsearch.rest.action.RestBuilderListener;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 
+import java.io.IOException;
+
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.rest.action.RestActions.buildBroadcastShardsHeader;
 import static org.elasticsearch.search.internal.SearchContext.DEFAULT_TERMINATE_AFTER;
 
-/**
- *
- */
 public class RestCountAction extends BaseRestHandler {
-
-    private final IndicesQueriesRegistry indicesQueriesRegistry;
-
-    @Inject
-    public RestCountAction(Settings settings, RestController controller, IndicesQueriesRegistry indicesQueriesRegistry) {
+    public RestCountAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(POST, "/_count", this);
         controller.registerHandler(GET, "/_count", this);
@@ -61,24 +52,24 @@ public RestCountAction(Settings settings, RestController controller, IndicesQuer
         controller.registerHandler(GET, "/{index}/_count", this);
         controller.registerHandler(POST, "/{index}/{type}/_count", this);
         controller.registerHandler(GET, "/{index}/{type}/_count", this);
-        this.indicesQueriesRegistry = indicesQueriesRegistry;
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         SearchRequest countRequest = new SearchRequest(Strings.splitStringByCommaToArray(request.param("index")));
         countRequest.indicesOptions(IndicesOptions.fromRequest(request, countRequest.indicesOptions()));
         SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().size(0);
         countRequest.source(searchSourceBuilder);
-        if (RestActions.hasBodyContent(request)) {
-            BytesReference restContent = RestActions.getRestContent(request);
-            searchSourceBuilder.query(RestActions.getQueryContent(restContent, indicesQueriesRegistry, parseFieldMatcher));
-        } else {
-            QueryBuilder queryBuilder = RestActions.urlParamsToQueryBuilder(request);
-            if (queryBuilder != null) {
-                searchSourceBuilder.query(queryBuilder);
+        request.withContentOrSourceParamParserOrNull(parser -> {
+            if (parser == null) {
+                QueryBuilder queryBuilder = RestActions.urlParamsToQueryBuilder(request);
+                if (queryBuilder != null) {
+                    searchSourceBuilder.query(queryBuilder);
+                }
+            } else {
+                searchSourceBuilder.query(RestActions.getQueryContent(parser));
             }
-        }
+        });
         countRequest.routing(request.param("routing"));
         float minScore = request.paramAsFloat("min_score", -1f);
         if (minScore != -1f) {
@@ -93,7 +84,7 @@ public void handleRequest(final RestRequest request, final RestChannel channel,
         } else if (terminateAfter > 0) {
             searchSourceBuilder.terminateAfter(terminateAfter);
         }
-        client.search(countRequest, new RestBuilderListener<SearchResponse>(channel) {
+        return channel -> client.search(countRequest, new RestBuilderListener<SearchResponse>(channel) {
             @Override
             public RestResponse buildResponse(SearchResponse response, XContentBuilder builder) throws Exception {
                 builder.startObject();
@@ -102,11 +93,12 @@ public RestResponse buildResponse(SearchResponse response, XContentBuilder build
                 }
                 builder.field("count", response.getHits().totalHits());
                 buildBroadcastShardsHeader(builder, request, response.getTotalShards(), response.getSuccessfulShards(),
-                        response.getFailedShards(), response.getShardFailures());
+                    0, response.getFailedShards(), response.getShardFailures());
 
                 builder.endObject();
                 return new BytesRestResponse(response.status(), builder);
             }
         });
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/document/RestDeleteAction.java b/core/src/main/java/org/elasticsearch/rest/action/document/RestDeleteAction.java
index 392cff7ffbae6..a39ec5e747b59 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/document/RestDeleteAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/document/RestDeleteAction.java
@@ -22,31 +22,29 @@
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestActions;
 import org.elasticsearch.rest.action.RestStatusToXContentListener;
 
+import java.io.IOException;
+
 import static org.elasticsearch.rest.RestRequest.Method.DELETE;
 
 /**
  *
  */
 public class RestDeleteAction extends BaseRestHandler {
-
-    @Inject
     public RestDeleteAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(DELETE, "/{index}/{type}/{id}", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         DeleteRequest deleteRequest = new DeleteRequest(request.param("index"), request.param("type"), request.param("id"));
         deleteRequest.routing(request.param("routing"));
         deleteRequest.parent(request.param("parent")); // order is important, set it after routing, so it will set the routing
@@ -60,6 +58,6 @@ public void handleRequest(final RestRequest request, final RestChannel channel,
             deleteRequest.waitForActiveShards(ActiveShardCount.parseString(waitForActiveShards));
         }
 
-        client.delete(deleteRequest, new RestStatusToXContentListener<>(channel));
+        return channel -> client.delete(deleteRequest, new RestStatusToXContentListener<>(channel));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java b/core/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java
index 8c782a8d128d3..c1a7c5868094b 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java
@@ -23,34 +23,33 @@
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.RestResponse;
+import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.RestActions;
-import org.elasticsearch.rest.action.RestBuilderListener;
+import org.elasticsearch.rest.action.RestToXContentListener;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 
+import java.io.IOException;
+
 import static org.elasticsearch.rest.RestRequest.Method.GET;
+import static org.elasticsearch.rest.RestRequest.Method.HEAD;
 import static org.elasticsearch.rest.RestStatus.NOT_FOUND;
 import static org.elasticsearch.rest.RestStatus.OK;
 
 public class RestGetAction extends BaseRestHandler {
 
-    @Inject
-    public RestGetAction(Settings settings, RestController controller) {
+    public RestGetAction(final Settings settings, final RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/{index}/{type}/{id}", this);
+        controller.registerHandler(HEAD, "/{index}/{type}/{id}", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         final GetRequest getRequest = new GetRequest(request.param("index"), request.param("type"), request.param("id"));
         getRequest.operationThreaded(true);
         getRequest.refresh(request.paramAsBoolean("refresh", getRequest.refresh()));
@@ -58,13 +57,15 @@ public void handleRequest(final RestRequest request, final RestChannel channel,
         getRequest.parent(request.param("parent"));
         getRequest.preference(request.param("preference"));
         getRequest.realtime(request.paramAsBoolean("realtime", getRequest.realtime()));
-        getRequest.ignoreErrorsOnGeneratedFields(request.paramAsBoolean("ignore_errors_on_generated_fields", false));
-
-        String sField = request.param("fields");
-        if (sField != null) {
-            String[] sFields = Strings.splitStringByCommaToArray(sField);
-            if (sFields != null) {
-                getRequest.fields(sFields);
+        if (request.param("fields") != null) {
+            throw new IllegalArgumentException("the parameter [fields] is no longer supported, " +
+                "please use [stored_fields] to retrieve stored fields or [_source] to load the field from _source");
+        }
+        final String fieldsParam = request.param("stored_fields");
+        if (fieldsParam != null) {
+            final String[] fields = Strings.splitStringByCommaToArray(fieldsParam);
+            if (fields != null) {
+                getRequest.storedFields(fields);
             }
         }
 
@@ -73,18 +74,12 @@ public void handleRequest(final RestRequest request, final RestChannel channel,
 
         getRequest.fetchSourceContext(FetchSourceContext.parseFromRestRequest(request));
 
-        client.get(getRequest, new RestBuilderListener<GetResponse>(channel) {
+        return channel -> client.get(getRequest, new RestToXContentListener<GetResponse>(channel) {
             @Override
-            public RestResponse buildResponse(GetResponse response, XContentBuilder builder) throws Exception {
-                builder.startObject();
-                response.toXContent(builder, request);
-                builder.endObject();
-                if (!response.isExists()) {
-                    return new BytesRestResponse(NOT_FOUND, builder);
-                } else {
-                    return new BytesRestResponse(OK, builder);
-                }
+            protected RestStatus getStatus(final GetResponse response) {
+                return response.isExists() ? OK : NOT_FOUND;
             }
         });
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java b/core/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java
index 1ecfe317f4eac..341c1ddc91753 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java
@@ -23,12 +23,10 @@
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -38,50 +36,54 @@
 import java.io.IOException;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
+import static org.elasticsearch.rest.RestRequest.Method.HEAD;
 import static org.elasticsearch.rest.RestStatus.NOT_FOUND;
 import static org.elasticsearch.rest.RestStatus.OK;
 
+/**
+ * The REST handler for get source and head source APIs.
+ */
 public class RestGetSourceAction extends BaseRestHandler {
 
-    @Inject
-    public RestGetSourceAction(Settings settings, RestController controller) {
+    public RestGetSourceAction(final Settings settings, final RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/{index}/{type}/{id}/_source", this);
+        controller.registerHandler(HEAD, "/{index}/{type}/{id}/_source", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         final GetRequest getRequest = new GetRequest(request.param("index"), request.param("type"), request.param("id"));
         getRequest.operationThreaded(true);
         getRequest.refresh(request.paramAsBoolean("refresh", getRequest.refresh()));
-        getRequest.routing(request.param("routing"));  // order is important, set it after routing, so it will set the routing
+        getRequest.routing(request.param("routing"));
         getRequest.parent(request.param("parent"));
         getRequest.preference(request.param("preference"));
         getRequest.realtime(request.paramAsBoolean("realtime", getRequest.realtime()));
 
         getRequest.fetchSourceContext(FetchSourceContext.parseFromRestRequest(request));
 
-        if (getRequest.fetchSourceContext() != null && !getRequest.fetchSourceContext().fetchSource()) {
-            try {
-                ActionRequestValidationException validationError = new ActionRequestValidationException();
+        return channel -> {
+            if (getRequest.fetchSourceContext() != null && !getRequest.fetchSourceContext().fetchSource()) {
+                final ActionRequestValidationException validationError = new ActionRequestValidationException();
                 validationError.addValidationError("fetching source can not be disabled");
                 channel.sendResponse(new BytesRestResponse(channel, validationError));
-            } catch (IOException e) {
-                logger.error("Failed to send failure response", e);
-            }
-        }
-
-        client.get(getRequest, new RestResponseListener<GetResponse>(channel) {
-            @Override
-            public RestResponse buildResponse(GetResponse response) throws Exception {
-                XContentBuilder builder = channel.newBuilder(response.getSourceInternal(), false);
-                if (response.isSourceEmpty()) { // check if doc source (or doc itself) is missing
-                    return new BytesRestResponse(NOT_FOUND, builder);
-                } else {
-                    builder.rawValue(response.getSourceInternal());
-                    return new BytesRestResponse(OK, builder);
-                }
+            } else {
+                client.get(getRequest, new RestResponseListener<GetResponse>(channel) {
+                    @Override
+                    public RestResponse buildResponse(final GetResponse response) throws Exception {
+                        final XContentBuilder builder = channel.newBuilder(request.getXContentType(), false);
+                        // check if doc source (or doc itself) is missing
+                        if (response.isSourceEmpty()) {
+                            return new BytesRestResponse(NOT_FOUND, builder);
+                        } else {
+                            builder.rawValue(response.getSourceInternal());
+                            return new BytesRestResponse(OK, builder);
+                        }
+                    }
+                });
             }
-        });
+        };
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/document/RestHeadAction.java b/core/src/main/java/org/elasticsearch/rest/action/document/RestHeadAction.java
deleted file mode 100644
index 9fb706bd8e6d1..0000000000000
--- a/core/src/main/java/org/elasticsearch/rest/action/document/RestHeadAction.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.rest.action.document;
-
-import org.elasticsearch.action.get.GetRequest;
-import org.elasticsearch.action.get.GetResponse;
-import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.RestChannel;
-import org.elasticsearch.rest.RestController;
-import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.RestResponse;
-import org.elasticsearch.rest.action.RestResponseListener;
-
-import static org.elasticsearch.rest.RestRequest.Method.HEAD;
-import static org.elasticsearch.rest.RestStatus.NOT_FOUND;
-import static org.elasticsearch.rest.RestStatus.OK;
-
-/**
- * Base class for {@code HEAD} request handlers for a single document.
- */
-public abstract class RestHeadAction extends BaseRestHandler {
-
-    /**
-     * Handler to check for document existence.
-     */
-    public static class Document extends RestHeadAction {
-
-        @Inject
-        public Document(Settings settings, RestController controller) {
-            super(settings, false);
-            controller.registerHandler(HEAD, "/{index}/{type}/{id}", this);
-        }
-    }
-
-    /**
-     * Handler to check for document source existence (may be disabled in the mapping).
-     */
-    public static class Source extends RestHeadAction {
-
-        @Inject
-        public Source(Settings settings, RestController controller) {
-            super(settings, true);
-            controller.registerHandler(HEAD, "/{index}/{type}/{id}/_source", this);
-        }
-    }
-
-    private final boolean source;
-
-    /**
-     * All subclasses must be registered in {@link org.elasticsearch.common.network.NetworkModule}.
-     *
-     * @param settings injected settings
-     * @param source   {@code false} to check for {@link GetResponse#isExists()}.
-     *                 {@code true} to also check for {@link GetResponse#isSourceEmpty()}.
-     */
-    public RestHeadAction(Settings settings, boolean source) {
-        super(settings);
-        this.source = source;
-    }
-
-    @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
-        final GetRequest getRequest = new GetRequest(request.param("index"), request.param("type"), request.param("id"));
-        getRequest.operationThreaded(true);
-        getRequest.refresh(request.paramAsBoolean("refresh", getRequest.refresh()));
-        getRequest.routing(request.param("routing"));  // order is important, set it after routing, so it will set the routing
-        getRequest.parent(request.param("parent"));
-        getRequest.preference(request.param("preference"));
-        getRequest.realtime(request.paramAsBoolean("realtime", getRequest.realtime()));
-        // don't get any fields back...
-        getRequest.fields(Strings.EMPTY_ARRAY);
-        // TODO we can also just return the document size as Content-Length
-
-        client.get(getRequest, new RestResponseListener<GetResponse>(channel) {
-            @Override
-            public RestResponse buildResponse(GetResponse response) {
-                if (!response.isExists()) {
-                    return new BytesRestResponse(NOT_FOUND, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY);
-                } else if (source && response.isSourceEmpty()) { // doc exists, but source might not (disabled in the mapping)
-                    return new BytesRestResponse(NOT_FOUND, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY);
-                } else {
-                    return new BytesRestResponse(OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY);
-                }
-            }
-        });
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java
index 6c9723b5b9346..8de694c4f2bdd 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java
@@ -22,13 +22,9 @@
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestActions;
@@ -38,14 +34,8 @@
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.rest.RestRequest.Method.PUT;
-import static org.elasticsearch.rest.RestStatus.BAD_REQUEST;
 
-/**
- *
- */
 public class RestIndexAction extends BaseRestHandler {
-
-    @Inject
     public RestIndexAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(POST, "/{index}/{type}", this); // auto id creation
@@ -62,46 +52,42 @@ protected CreateHandler(Settings settings) {
         }
 
         @Override
-        public void handleRequest(RestRequest request, RestChannel channel, final NodeClient client) {
+        public RestChannelConsumer prepareRequest(RestRequest request, final NodeClient client) throws IOException {
             request.params().put("op_type", "create");
-            RestIndexAction.this.handleRequest(request, channel, client);
+            return RestIndexAction.this.prepareRequest(request, client);
         }
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         IndexRequest indexRequest = new IndexRequest(request.param("index"), request.param("type"), request.param("id"));
         indexRequest.routing(request.param("routing"));
         indexRequest.parent(request.param("parent")); // order is important, set it after routing, so it will set the routing
+        if (request.hasParam("timestamp")) {
+            deprecationLogger.deprecated("The [timestamp] parameter of index requests is deprecated");
+        }
         indexRequest.timestamp(request.param("timestamp"));
         if (request.hasParam("ttl")) {
+            deprecationLogger.deprecated("The [ttl] parameter of index requests is deprecated");
             indexRequest.ttl(request.param("ttl"));
         }
         indexRequest.setPipeline(request.param("pipeline"));
-        indexRequest.source(request.content());
+        indexRequest.source(request.requiredContent(), request.getXContentType());
         indexRequest.timeout(request.paramAsTime("timeout", IndexRequest.DEFAULT_TIMEOUT));
         indexRequest.setRefreshPolicy(request.param("refresh"));
         indexRequest.version(RestActions.parseVersion(request));
         indexRequest.versionType(VersionType.fromString(request.param("version_type"), indexRequest.versionType()));
         String sOpType = request.param("op_type");
-        if (sOpType != null) {
-            try {
-                indexRequest.opType(IndexRequest.OpType.fromString(sOpType));
-            } catch (IllegalArgumentException eia){
-                try {
-                    XContentBuilder builder = channel.newErrorBuilder();
-                    channel.sendResponse(
-                            new BytesRestResponse(BAD_REQUEST, builder.startObject().field("error", eia.getMessage()).endObject()));
-                } catch (IOException e1) {
-                    logger.warn("Failed to send response", e1);
-                    return;
-                }
-            }
-        }
         String waitForActiveShards = request.param("wait_for_active_shards");
         if (waitForActiveShards != null) {
             indexRequest.waitForActiveShards(ActiveShardCount.parseString(waitForActiveShards));
         }
-        client.index(indexRequest, new RestStatusToXContentListener<>(channel, r -> r.getLocation(indexRequest.routing())));
+        if (sOpType != null) {
+            indexRequest.opType(sOpType);
+        }
+
+        return channel ->
+                client.index(indexRequest, new RestStatusToXContentListener<>(channel, r -> r.getLocation(indexRequest.routing())));
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/document/RestMultiGetAction.java b/core/src/main/java/org/elasticsearch/rest/action/document/RestMultiGetAction.java
index 995c43059dac7..cba8fafbda39e 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/document/RestMultiGetAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/document/RestMultiGetAction.java
@@ -20,19 +20,18 @@
 package org.elasticsearch.rest.action.document;
 
 import org.elasticsearch.action.get.MultiGetRequest;
-import org.elasticsearch.action.get.MultiGetResponse;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.action.RestActions;
 import org.elasticsearch.rest.action.RestToXContentListener;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 
+import java.io.IOException;
+
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
@@ -40,7 +39,6 @@ public class RestMultiGetAction extends BaseRestHandler {
 
     private final boolean allowExplicitIndex;
 
-    @Inject
     public RestMultiGetAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_mget", this);
@@ -54,23 +52,27 @@ public RestMultiGetAction(Settings settings, RestController controller) {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) throws Exception {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         MultiGetRequest multiGetRequest = new MultiGetRequest();
         multiGetRequest.refresh(request.paramAsBoolean("refresh", multiGetRequest.refresh()));
         multiGetRequest.preference(request.param("preference"));
         multiGetRequest.realtime(request.paramAsBoolean("realtime", multiGetRequest.realtime()));
-        multiGetRequest.ignoreErrorsOnGeneratedFields(request.paramAsBoolean("ignore_errors_on_generated_fields", false));
-
+        if (request.param("fields") != null) {
+            throw new IllegalArgumentException("The parameter [fields] is no longer supported, " +
+                "please use [stored_fields] to retrieve stored fields or _source filtering if the field is not stored");
+        }
         String[] sFields = null;
-        String sField = request.param("fields");
+        String sField = request.param("stored_fields");
         if (sField != null) {
             sFields = Strings.splitStringByCommaToArray(sField);
         }
 
         FetchSourceContext defaultFetchSource = FetchSourceContext.parseFromRestRequest(request);
-        multiGetRequest.add(request.param("index"), request.param("type"), sFields, defaultFetchSource,
-                request.param("routing"), RestActions.getRestContent(request), allowExplicitIndex);
+        try (XContentParser parser = request.contentOrSourceParamParser()) {
+            multiGetRequest.add(request.param("index"), request.param("type"), sFields, defaultFetchSource,
+                request.param("routing"), parser, allowExplicitIndex);
+        }
 
-        client.multiGet(multiGetRequest, new RestToXContentListener<MultiGetResponse>(channel));
+        return channel -> client.multiGet(multiGetRequest, new RestToXContentListener<>(channel));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/document/RestMultiTermVectorsAction.java b/core/src/main/java/org/elasticsearch/rest/action/document/RestMultiTermVectorsAction.java
index dab23e8df352c..605c8654f500d 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/document/RestMultiTermVectorsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/document/RestMultiTermVectorsAction.java
@@ -24,21 +24,18 @@
 import org.elasticsearch.action.termvectors.TermVectorsRequest;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.action.RestActions;
 import org.elasticsearch.rest.action.RestToXContentListener;
 
+import java.io.IOException;
+
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
 public class RestMultiTermVectorsAction extends BaseRestHandler {
-
-    @Inject
     public RestMultiTermVectorsAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_mtermvectors", this);
@@ -50,15 +47,16 @@ public RestMultiTermVectorsAction(Settings settings, RestController controller)
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) throws Exception {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         MultiTermVectorsRequest multiTermVectorsRequest = new MultiTermVectorsRequest();
         TermVectorsRequest template = new TermVectorsRequest();
         template.index(request.param("index"));
         template.type(request.param("type"));
         RestTermVectorsAction.readURIParameters(template, request);
         multiTermVectorsRequest.ids(Strings.commaDelimitedListToStringArray(request.param("ids")));
-        multiTermVectorsRequest.add(template, RestActions.getRestContent(request));
+        request.withContentOrSourceParamParserOrNull(p -> multiTermVectorsRequest.add(template, p));
 
-        client.multiTermVectors(multiTermVectorsRequest, new RestToXContentListener<MultiTermVectorsResponse>(channel));
+        return channel -> client.multiTermVectors(multiTermVectorsRequest, new RestToXContentListener<MultiTermVectorsResponse>(channel));
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/document/RestTermVectorsAction.java b/core/src/main/java/org/elasticsearch/rest/action/document/RestTermVectorsAction.java
index b64219215a93b..a649e5eff4774 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/document/RestTermVectorsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/document/RestTermVectorsAction.java
@@ -20,21 +20,18 @@
 package org.elasticsearch.rest.action.document;
 
 import org.elasticsearch.action.termvectors.TermVectorsRequest;
-import org.elasticsearch.action.termvectors.TermVectorsResponse;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestActions;
 import org.elasticsearch.rest.action.RestToXContentListener;
 
+import java.io.IOException;
 import java.util.HashSet;
 import java.util.Set;
 
@@ -46,8 +43,6 @@
  * TermVectorsRequest.
  */
 public class RestTermVectorsAction extends BaseRestHandler {
-
-    @Inject
     public RestTermVectorsAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/{index}/{type}/_termvectors", this);
@@ -63,17 +58,16 @@ public RestTermVectorsAction(Settings settings, RestController controller) {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) throws Exception {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         TermVectorsRequest termVectorsRequest = new TermVectorsRequest(request.param("index"), request.param("type"), request.param("id"));
-        if (RestActions.hasBodyContent(request)) {
-            try (XContentParser parser = XContentFactory.xContent(RestActions.guessBodyContentType(request))
-                    .createParser(RestActions.getRestContent(request))){
+        if (request.hasContentOrSourceParam()) {
+            try (XContentParser parser = request.contentOrSourceParamParser()) {
                 TermVectorsRequest.parseRequest(termVectorsRequest, parser);
             }
         }
         readURIParameters(termVectorsRequest, request);
 
-        client.termVectors(termVectorsRequest, new RestToXContentListener<TermVectorsResponse>(channel));
+        return channel -> client.termVectors(termVectorsRequest, new RestToXContentListener<>(channel));
     }
 
     public static void readURIParameters(TermVectorsRequest termVectorsRequest, RestRequest request) {
diff --git a/core/src/main/java/org/elasticsearch/rest/action/document/RestUpdateAction.java b/core/src/main/java/org/elasticsearch/rest/action/document/RestUpdateAction.java
index d0d7916adfed1..26591c52a5d5e 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/document/RestUpdateAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/document/RestUpdateAction.java
@@ -24,30 +24,34 @@
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestActions;
 import org.elasticsearch.rest.action.RestStatusToXContentListener;
+import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
+
+import java.io.IOException;
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
 /**
  */
 public class RestUpdateAction extends BaseRestHandler {
+    private static final DeprecationLogger DEPRECATION_LOGGER =
+        new DeprecationLogger(Loggers.getLogger(RestUpdateAction.class));
 
-    @Inject
     public RestUpdateAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(POST, "/{index}/{type}/{id}/_update", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) throws Exception {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         UpdateRequest updateRequest = new UpdateRequest(request.param("index"), request.param("type"), request.param("id"));
         updateRequest.routing(request.param("routing"));
         updateRequest.parent(request.param("parent"));
@@ -58,21 +62,32 @@ public void handleRequest(final RestRequest request, final RestChannel channel,
             updateRequest.waitForActiveShards(ActiveShardCount.parseString(waitForActiveShards));
         }
         updateRequest.docAsUpsert(request.paramAsBoolean("doc_as_upsert", updateRequest.docAsUpsert()));
+        FetchSourceContext fetchSourceContext = FetchSourceContext.parseFromRestRequest(request);
         String sField = request.param("fields");
+        if (sField != null && fetchSourceContext != null) {
+            throw new IllegalArgumentException("[fields] and [_source] cannot be used in the same request");
+        }
         if (sField != null) {
+            DEPRECATION_LOGGER.deprecated("Deprecated field [fields] used, expected [_source] instead");
             String[] sFields = Strings.splitStringByCommaToArray(sField);
-            if (sFields != null) {
-                updateRequest.fields(sFields);
-            }
+            updateRequest.fields(sFields);
+        } else if (fetchSourceContext != null) {
+            updateRequest.fetchSource(fetchSourceContext);
         }
+
         updateRequest.retryOnConflict(request.paramAsInt("retry_on_conflict", updateRequest.retryOnConflict()));
         updateRequest.version(RestActions.parseVersion(request));
         updateRequest.versionType(VersionType.fromString(request.param("version_type"), updateRequest.versionType()));
 
+        if (request.hasParam("timestamp")) {
+            deprecationLogger.deprecated("The [timestamp] parameter of index requests is deprecated");
+        }
+        if (request.hasParam("ttl")) {
+            deprecationLogger.deprecated("The [ttl] parameter of index requests is deprecated");
+        }
 
-        // see if we have it in the body
-        if (request.hasContent()) {
-            updateRequest.source(request.content());
+        request.applyContentParser(parser -> {
+            updateRequest.fromXContent(parser);
             IndexRequest upsertRequest = updateRequest.upsertRequest();
             if (upsertRequest != null) {
                 upsertRequest.routing(request.param("routing"));
@@ -95,8 +110,10 @@ public void handleRequest(final RestRequest request, final RestChannel channel,
                 doc.version(RestActions.parseVersion(request));
                 doc.versionType(VersionType.fromString(request.param("version_type"), doc.versionType()));
             }
-        }
+        });
 
-        client.update(updateRequest, new RestStatusToXContentListener<>(channel, r -> r.getLocation(updateRequest.routing())));
+        return channel ->
+                client.update(updateRequest, new RestStatusToXContentListener<>(channel, r -> r.getLocation(updateRequest.routing())));
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/ingest/RestDeletePipelineAction.java b/core/src/main/java/org/elasticsearch/rest/action/ingest/RestDeletePipelineAction.java
index 593b55b8b754f..b776d2475cecb 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/ingest/RestDeletePipelineAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/ingest/RestDeletePipelineAction.java
@@ -21,27 +21,25 @@
 
 import org.elasticsearch.action.ingest.DeletePipelineRequest;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.AcknowledgedRestListener;
 
-public class RestDeletePipelineAction extends BaseRestHandler {
+import java.io.IOException;
 
-    @Inject
+public class RestDeletePipelineAction extends BaseRestHandler {
     public RestDeletePipelineAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(RestRequest.Method.DELETE, "/_ingest/pipeline/{id}", this);
     }
 
     @Override
-    public void handleRequest(RestRequest restRequest, RestChannel channel, NodeClient client) throws Exception {
+    public RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException {
         DeletePipelineRequest request = new DeletePipelineRequest(restRequest.param("id"));
         request.masterNodeTimeout(restRequest.paramAsTime("master_timeout", request.masterNodeTimeout()));
         request.timeout(restRequest.paramAsTime("timeout", request.timeout()));
-        client.admin().cluster().deletePipeline(request, new AcknowledgedRestListener<>(channel));
+        return channel -> client.admin().cluster().deletePipeline(request, new AcknowledgedRestListener<>(channel));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/ingest/RestGetPipelineAction.java b/core/src/main/java/org/elasticsearch/rest/action/ingest/RestGetPipelineAction.java
index 308fb146c3020..c8facf7b4cc15 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/ingest/RestGetPipelineAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/ingest/RestGetPipelineAction.java
@@ -22,17 +22,15 @@
 import org.elasticsearch.action.ingest.GetPipelineRequest;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestStatusToXContentListener;
 
-public class RestGetPipelineAction extends BaseRestHandler {
+import java.io.IOException;
 
-    @Inject
+public class RestGetPipelineAction extends BaseRestHandler {
     public RestGetPipelineAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(RestRequest.Method.GET, "/_ingest/pipeline", this);
@@ -40,9 +38,9 @@ public RestGetPipelineAction(Settings settings, RestController controller) {
     }
 
     @Override
-    public void handleRequest(RestRequest restRequest, RestChannel channel, NodeClient client) throws Exception {
+    public RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException {
         GetPipelineRequest request = new GetPipelineRequest(Strings.splitStringByCommaToArray(restRequest.param("id")));
         request.masterNodeTimeout(restRequest.paramAsTime("master_timeout", request.masterNodeTimeout()));
-        client.admin().cluster().getPipeline(request, new RestStatusToXContentListener<>(channel));
+        return channel -> client.admin().cluster().getPipeline(request, new RestStatusToXContentListener<>(channel));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/ingest/RestPutPipelineAction.java b/core/src/main/java/org/elasticsearch/rest/action/ingest/RestPutPipelineAction.java
index b6d34a6c25475..2496c9b4a2487 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/ingest/RestPutPipelineAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/ingest/RestPutPipelineAction.java
@@ -21,30 +21,31 @@
 
 import org.elasticsearch.action.ingest.PutPipelineRequest;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.AcknowledgedRestListener;
-import org.elasticsearch.rest.action.RestActions;
 
+import java.io.IOException;
 
-public class RestPutPipelineAction extends BaseRestHandler {
 
-    @Inject
+public class RestPutPipelineAction extends BaseRestHandler {
     public RestPutPipelineAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(RestRequest.Method.PUT, "/_ingest/pipeline/{id}", this);
     }
 
     @Override
-    public void handleRequest(RestRequest restRequest, RestChannel channel, NodeClient client) throws Exception {
-        PutPipelineRequest request = new PutPipelineRequest(restRequest.param("id"), RestActions.getRestContent(restRequest));
+    public RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException {
+        Tuple<XContentType, BytesReference> sourceTuple = restRequest.contentOrSourceParam();
+        PutPipelineRequest request = new PutPipelineRequest(restRequest.param("id"), sourceTuple.v2(), sourceTuple.v1());
         request.masterNodeTimeout(restRequest.paramAsTime("master_timeout", request.masterNodeTimeout()));
         request.timeout(restRequest.paramAsTime("timeout", request.timeout()));
-        client.admin().cluster().putPipeline(request, new AcknowledgedRestListener<>(channel));
+        return channel -> client.admin().cluster().putPipeline(request, new AcknowledgedRestListener<>(channel));
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulatePipelineAction.java b/core/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulatePipelineAction.java
index a51bdf5fef2fd..9dbe1808a8c59 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulatePipelineAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulatePipelineAction.java
@@ -21,19 +21,18 @@
 
 import org.elasticsearch.action.ingest.SimulatePipelineRequest;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.action.RestActions;
-import org.elasticsearch.rest.action.RestStatusToXContentListener;
 import org.elasticsearch.rest.action.RestToXContentListener;
 
-public class RestSimulatePipelineAction extends BaseRestHandler {
+import java.io.IOException;
 
-    @Inject
+public class RestSimulatePipelineAction extends BaseRestHandler {
     public RestSimulatePipelineAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(RestRequest.Method.POST, "/_ingest/pipeline/{id}/_simulate", this);
@@ -43,10 +42,11 @@ public RestSimulatePipelineAction(Settings settings, RestController controller)
     }
 
     @Override
-    public void handleRequest(RestRequest restRequest, RestChannel channel, NodeClient client) throws Exception {
-        SimulatePipelineRequest request = new SimulatePipelineRequest(RestActions.getRestContent(restRequest));
+    public RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException {
+        Tuple<XContentType, BytesReference> sourceTuple = restRequest.contentOrSourceParam();
+        SimulatePipelineRequest request = new SimulatePipelineRequest(sourceTuple.v2(), sourceTuple.v1());
         request.setId(restRequest.param("id"));
         request.setVerbose(restRequest.paramAsBoolean("verbose", false));
-        client.admin().cluster().simulatePipeline(request, new RestToXContentListener<>(channel));
+        return channel -> client.admin().cluster().simulatePipeline(request, new RestToXContentListener<>(channel));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/search/RestClearScrollAction.java b/core/src/main/java/org/elasticsearch/rest/action/search/RestClearScrollAction.java
index 4c8f84a222352..066711667c5f1 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/search/RestClearScrollAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/search/RestClearScrollAction.java
@@ -20,20 +20,13 @@
 package org.elasticsearch.rest.action.search;
 
 import org.elasticsearch.action.search.ClearScrollRequest;
-import org.elasticsearch.action.search.ClearScrollResponse;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.action.RestActions;
 import org.elasticsearch.rest.action.RestStatusToXContentListener;
 
 import java.io.IOException;
@@ -42,8 +35,6 @@
 import static org.elasticsearch.rest.RestRequest.Method.DELETE;
 
 public class RestClearScrollAction extends BaseRestHandler {
-
-    @Inject
     public RestClearScrollAction(Settings settings, RestController controller) {
         super(settings);
 
@@ -52,58 +43,40 @@ public RestClearScrollAction(Settings settings, RestController controller) {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         String scrollIds = request.param("scroll_id");
         ClearScrollRequest clearRequest = new ClearScrollRequest();
         clearRequest.setScrollIds(Arrays.asList(splitScrollIds(scrollIds)));
-        if (RestActions.hasBodyContent(request)) {
-            XContentType type = RestActions.guessBodyContentType(request);
-           if (type == null) {
-               scrollIds = RestActions.getRestContent(request).utf8ToString();
-               clearRequest.setScrollIds(Arrays.asList(splitScrollIds(scrollIds)));
-           } else {
-               // NOTE: if rest request with xcontent body has request parameters, these parameters does not override xcontent value
-               clearRequest.setScrollIds(null);
-               buildFromContent(RestActions.getRestContent(request), clearRequest);
-           }
-        }
+        request.withContentOrSourceParamParserOrNullLenient((xContentParser -> {
+            if (xContentParser == null) {
+                if (request.hasContent()) {
+                    // TODO: why do we accept this plain text value? maybe we can just use the scroll params?
+                    BytesReference body = request.content();
+                    String bodyScrollIds = body.utf8ToString();
+                    clearRequest.setScrollIds(Arrays.asList(splitScrollIds(bodyScrollIds)));
+                }
+            } else {
+                // NOTE: if rest request with xcontent body has request parameters, values parsed from request body have the precedence
+                try {
+                    clearRequest.fromXContent(xContentParser);
+                } catch (IOException e) {
+                    throw new IllegalArgumentException("Failed to parse request body", e);
+                }
+            }
+        }));
+
+        return channel -> client.clearScroll(clearRequest, new RestStatusToXContentListener<>(channel));
+    }
 
-        client.clearScroll(clearRequest, new RestStatusToXContentListener<ClearScrollResponse>(channel));
+    @Override
+    public boolean supportsPlainText() {
+        return true;
     }
 
-    public static String[] splitScrollIds(String scrollIds) {
+    private static String[] splitScrollIds(String scrollIds) {
         if (scrollIds == null) {
             return Strings.EMPTY_ARRAY;
         }
         return Strings.splitStringByCommaToArray(scrollIds);
     }
-
-    public static void buildFromContent(BytesReference content, ClearScrollRequest clearScrollRequest) {
-        try (XContentParser parser = XContentHelper.createParser(content)) {
-            if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
-                throw new IllegalArgumentException("Malformed content, must start with an object");
-            } else {
-                XContentParser.Token token;
-                String currentFieldName = null;
-                while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                    if (token == XContentParser.Token.FIELD_NAME) {
-                        currentFieldName = parser.currentName();
-                    } else if ("scroll_id".equals(currentFieldName) && token == XContentParser.Token.START_ARRAY) {
-                        while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
-                            if (token.isValue() == false) {
-                                throw new IllegalArgumentException("scroll_id array element should only contain scroll_id");
-                            }
-                            clearScrollRequest.addScrollId(parser.text());
-                        }
-                    } else {
-                        throw new IllegalArgumentException("Unknown parameter [" + currentFieldName
-                                + "] in request body or parameter is of the wrong type[" + token + "] ");
-                    }
-                }
-            }
-        } catch (IOException e) {
-            throw new IllegalArgumentException("Failed to parse request body", e);
-        }
-    }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/search/RestExplainAction.java b/core/src/main/java/org/elasticsearch/rest/action/search/RestExplainAction.java
index 7088b96c6de62..7339718c28b0b 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/search/RestExplainAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/search/RestExplainAction.java
@@ -24,16 +24,12 @@
 import org.elasticsearch.action.explain.ExplainResponse;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.get.GetResult;
 import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -52,43 +48,43 @@
  * Rest action for computing a score explanation for specific documents.
  */
 public class RestExplainAction extends BaseRestHandler {
-
-    private final IndicesQueriesRegistry indicesQueriesRegistry;
-
-    @Inject
-    public RestExplainAction(Settings settings, RestController controller, IndicesQueriesRegistry indicesQueriesRegistry) {
+    public RestExplainAction(Settings settings, RestController controller) {
         super(settings);
-        this.indicesQueriesRegistry = indicesQueriesRegistry;
         controller.registerHandler(GET, "/{index}/{type}/{id}/_explain", this);
         controller.registerHandler(POST, "/{index}/{type}/{id}/_explain", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         final ExplainRequest explainRequest = new ExplainRequest(request.param("index"), request.param("type"), request.param("id"));
         explainRequest.parent(request.param("parent"));
         explainRequest.routing(request.param("routing"));
         explainRequest.preference(request.param("preference"));
         String queryString = request.param("q");
-        if (RestActions.hasBodyContent(request)) {
-            BytesReference restContent = RestActions.getRestContent(request);
-            explainRequest.query(RestActions.getQueryContent(restContent, indicesQueriesRegistry, parseFieldMatcher));
-        } else if (queryString != null) {
-            QueryBuilder query = RestActions.urlParamsToQueryBuilder(request);
-            explainRequest.query(query);
-        }
+        request.withContentOrSourceParamParserOrNull(parser -> {
+            if (parser != null) {
+                explainRequest.query(RestActions.getQueryContent(parser));
+            } else if (queryString != null) {
+                QueryBuilder query = RestActions.urlParamsToQueryBuilder(request);
+                explainRequest.query(query);
+            }
+        });
 
-        String sField = request.param("fields");
+        if (request.param("fields") != null) {
+            throw new IllegalArgumentException("The parameter [fields] is no longer supported, " +
+                "please use [stored_fields] to retrieve stored fields");
+        }
+        String sField = request.param("stored_fields");
         if (sField != null) {
             String[] sFields = Strings.splitStringByCommaToArray(sField);
             if (sFields != null) {
-                explainRequest.fields(sFields);
+                explainRequest.storedFields(sFields);
             }
         }
 
         explainRequest.fetchSourceContext(FetchSourceContext.parseFromRestRequest(request));
 
-        client.explain(explainRequest, new RestBuilderListener<ExplainResponse>(channel) {
+        return channel -> client.explain(explainRequest, new RestBuilderListener<ExplainResponse>(channel) {
             @Override
             public RestResponse buildResponse(ExplainResponse response, XContentBuilder builder) throws Exception {
                 builder.startObject();
diff --git a/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java b/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java
index ae320bccac234..04e0d255641dd 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java
@@ -19,33 +19,32 @@
 
 package org.elasticsearch.rest.action.search;
 
-import java.io.IOException;
-import java.util.Map;
-import java.util.function.BiConsumer;
-
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.action.search.MultiSearchRequest;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContent;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
-import      org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.action.RestActions;
+import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestToXContentListener;
-import org.elasticsearch.search.SearchRequestParsers;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.BiConsumer;
+
 import static org.elasticsearch.common.xcontent.support.XContentMapValues.lenientNodeBooleanValue;
 import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeStringArrayValue;
 import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeStringValue;
@@ -56,13 +55,12 @@
  */
 public class RestMultiSearchAction extends BaseRestHandler {
 
+    private static final Set<String> RESPONSE_PARAMS = Collections.singleton(RestSearchAction.TYPED_KEYS_PARAM);
+
     private final boolean allowExplicitIndex;
-    private final SearchRequestParsers searchRequestParsers;
 
-    @Inject
-    public RestMultiSearchAction(Settings settings, RestController controller, SearchRequestParsers searchRequestParsers) {
+    public RestMultiSearchAction(Settings settings, RestController controller) {
         super(settings);
-        this.searchRequestParsers = searchRequestParsers;
 
         controller.registerHandler(GET, "/_msearch", this);
         controller.registerHandler(POST, "/_msearch", this);
@@ -75,52 +73,56 @@ public RestMultiSearchAction(Settings settings, RestController controller, Searc
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) throws Exception {
-        MultiSearchRequest multiSearchRequest = parseRequest(request, allowExplicitIndex, searchRequestParsers, parseFieldMatcher);
-        client.multiSearch(multiSearchRequest, new RestToXContentListener<>(channel));
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
+        MultiSearchRequest multiSearchRequest = parseRequest(request, allowExplicitIndex);
+        return channel -> client.multiSearch(multiSearchRequest, new RestToXContentListener<>(channel));
     }
 
     /**
      * Parses a {@link RestRequest} body and returns a {@link MultiSearchRequest}
      */
-    public static MultiSearchRequest parseRequest(RestRequest restRequest, boolean allowExplicitIndex,
-                                                  SearchRequestParsers searchRequestParsers,
-                                                  ParseFieldMatcher parseFieldMatcher) throws IOException {
-
+    public static MultiSearchRequest parseRequest(RestRequest restRequest, boolean allowExplicitIndex) throws IOException {
         MultiSearchRequest multiRequest = new MultiSearchRequest();
         if (restRequest.hasParam("max_concurrent_searches")) {
             multiRequest.maxConcurrentSearchRequests(restRequest.paramAsInt("max_concurrent_searches", 0));
         }
 
-        parseMultiLineRequest(restRequest, multiRequest.indicesOptions(), allowExplicitIndex, (searchRequest, bytes) -> {
-            try (XContentParser requestParser = XContentFactory.xContent(bytes).createParser(bytes)) {
-                final QueryParseContext queryParseContext = new QueryParseContext(searchRequestParsers.queryParsers,
-                    requestParser, parseFieldMatcher);
-                searchRequest.source(SearchSourceBuilder.fromXContent(queryParseContext,
-                    searchRequestParsers.aggParsers, searchRequestParsers.suggesters));
+        int preFilterShardSize = restRequest.paramAsInt("pre_filter_shard_size", SearchRequest.DEFAULT_PRE_FILTER_SHARD_SIZE);
+
+
+        parseMultiLineRequest(restRequest, multiRequest.indicesOptions(), allowExplicitIndex, (searchRequest, parser) -> {
+            try {
+                final QueryParseContext queryParseContext = new QueryParseContext(parser);
+                searchRequest.source(SearchSourceBuilder.fromXContent(queryParseContext));
                 multiRequest.add(searchRequest);
             } catch (IOException e) {
                 throw new ElasticsearchParseException("Exception when parsing search request", e);
             }
         });
-
+        List<SearchRequest> requests = multiRequest.requests();
+        preFilterShardSize = Math.max(1, preFilterShardSize / (requests.size()+1));
+        for (SearchRequest request : requests) {
+            // preserve if it's set on the request
+            request.setPreFilterShardSize(Math.min(preFilterShardSize, request.getPreFilterShardSize()));
+        }
         return multiRequest;
     }
 
     /**
-     * Parses a multi-line {@link RestRequest} body, instanciating a {@link SearchRequest} for each line and applying the given consumer.
+     * Parses a multi-line {@link RestRequest} body, instantiating a {@link SearchRequest} for each line and applying the given consumer.
      */
     public static void parseMultiLineRequest(RestRequest request, IndicesOptions indicesOptions, boolean allowExplicitIndex,
-                                             BiConsumer<SearchRequest, BytesReference> consumer) throws IOException {
+            BiConsumer<SearchRequest, XContentParser> consumer) throws IOException {
 
         String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
         String[] types = Strings.splitStringByCommaToArray(request.param("type"));
         String searchType = request.param("search_type");
         String routing = request.param("routing");
 
-        final BytesReference data = RestActions.getRestContent(request);
+        final Tuple<XContentType, BytesReference> sourceTuple = request.contentOrSourceParam();
+        final XContent xContent = sourceTuple.v1().xContent();
+        final BytesReference data = sourceTuple.v2();
 
-        XContent xContent = XContentFactory.xContent(data);
         int from = 0;
         int length = data.length();
         byte marker = xContent.streamSeparator();
@@ -157,7 +159,7 @@ public static void parseMultiLineRequest(RestRequest request, IndicesOptions ind
 
             // now parse the action
             if (nextMarker - from > 0) {
-                try (XContentParser parser = xContent.createParser(data.slice(from, nextMarker - from))) {
+                try (XContentParser parser = xContent.createParser(request.getXContentRegistry(), data.slice(from, nextMarker - from))) {
                     Map<String, Object> source = parser.map();
                     for (Map.Entry<String, Object> entry : source.entrySet()) {
                         Object value = entry.getValue();
@@ -171,7 +173,7 @@ public static void parseMultiLineRequest(RestRequest request, IndicesOptions ind
                         } else if ("search_type".equals(entry.getKey()) || "searchType".equals(entry.getKey())) {
                             searchRequest.searchType(nodeStringValue(value, null));
                         } else if ("request_cache".equals(entry.getKey()) || "requestCache".equals(entry.getKey())) {
-                            searchRequest.requestCache(lenientNodeBooleanValue(value));
+                            searchRequest.requestCache(lenientNodeBooleanValue(value, entry.getKey()));
                         } else if ("preference".equals(entry.getKey())) {
                             searchRequest.preference(nodeStringValue(value, null));
                         } else if ("routing".equals(entry.getKey())) {
@@ -190,12 +192,20 @@ public static void parseMultiLineRequest(RestRequest request, IndicesOptions ind
             if (nextMarker == -1) {
                 break;
             }
-            consumer.accept(searchRequest, data.slice(from, nextMarker - from));
+            BytesReference bytes = data.slice(from, nextMarker - from);
+            try (XContentParser parser = xContent.createParser(request.getXContentRegistry(), bytes)) {
+                consumer.accept(searchRequest, parser);
+            }
             // move pointers
             from = nextMarker + 1;
         }
     }
 
+    @Override
+    public boolean supportsContentStream() {
+        return true;
+    }
+
     private static int findNextMarker(byte marker, int from, BytesReference data, int length) {
         for (int i = from; i < length; i++) {
             if (data.get(i) == marker) {
@@ -204,4 +214,9 @@ private static int findNextMarker(byte marker, int from, BytesReference data, in
         }
         return -1;
     }
+
+    @Override
+    protected Set<String> responseParams() {
+        return RESPONSE_PARAMS;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java b/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
index 215165e6bbfc1..edbc6a226abd7 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
@@ -20,26 +20,19 @@
 package org.elasticsearch.rest.action.search;
 
 import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestActions;
 import org.elasticsearch.rest.action.RestStatusToXContentListener;
 import org.elasticsearch.search.Scroll;
-import org.elasticsearch.search.SearchRequestParsers;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.fetch.StoredFieldsContext;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
@@ -50,6 +43,8 @@
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Collections;
+import java.util.Set;
 
 import static org.elasticsearch.common.unit.TimeValue.parseTimeValue;
 import static org.elasticsearch.rest.RestRequest.Method.GET;
@@ -61,12 +56,11 @@
  */
 public class RestSearchAction extends BaseRestHandler {
 
-    private final SearchRequestParsers searchRequestParsers;
+    public static final String TYPED_KEYS_PARAM = "typed_keys";
+    private static final Set<String> RESPONSE_PARAMS = Collections.singleton(TYPED_KEYS_PARAM);
 
-    @Inject
-    public RestSearchAction(Settings settings, RestController controller, SearchRequestParsers searchRequestParsers) {
+    public RestSearchAction(Settings settings, RestController controller) {
         super(settings);
-        this.searchRequestParsers = searchRequestParsers;
         controller.registerHandler(GET, "/_search", this);
         controller.registerHandler(POST, "/_search", this);
         controller.registerHandler(GET, "/{index}/_search", this);
@@ -76,42 +70,50 @@ public RestSearchAction(Settings settings, RestController controller, SearchRequ
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) throws IOException {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         SearchRequest searchRequest = new SearchRequest();
-        BytesReference restContent = RestActions.hasBodyContent(request) ? RestActions.getRestContent(request) : null;
-        parseSearchRequest(searchRequest, request, searchRequestParsers, parseFieldMatcher, restContent);
-        client.search(searchRequest, new RestStatusToXContentListener<>(channel));
+        request.withContentOrSourceParamParserOrNull(parser ->
+            parseSearchRequest(searchRequest, request, parser));
+
+        return channel -> client.search(searchRequest, new RestStatusToXContentListener<>(channel));
     }
 
     /**
-     * Parses the rest request on top of the SearchRequest, preserving values
-     * that are not overridden by the rest request.
+     * Parses the rest request on top of the SearchRequest, preserving values that are not overridden by the rest request.
      *
-     * @param restContent
-     *            override body content to use for the request. If null body
-     *            content is read from the request using
-     *            RestAction.hasBodyContent.
+     * @param requestContentParser body of the request to read. This method does not attempt to read the body from the {@code request}
+     *        parameter
      */
-    public static void parseSearchRequest(SearchRequest searchRequest, RestRequest request, SearchRequestParsers searchRequestParsers,
-                                          ParseFieldMatcher parseFieldMatcher, BytesReference restContent) throws IOException {
+    public static void parseSearchRequest(SearchRequest searchRequest, RestRequest request,
+                                          XContentParser requestContentParser) throws IOException {
 
         if (searchRequest.source() == null) {
             searchRequest.source(new SearchSourceBuilder());
         }
         searchRequest.indices(Strings.splitStringByCommaToArray(request.param("index")));
-        if (restContent != null) {
-            try (XContentParser parser = XContentFactory.xContent(restContent).createParser(restContent)) {
-                QueryParseContext context = new QueryParseContext(searchRequestParsers.queryParsers, parser, parseFieldMatcher);
-                searchRequest.source().parseXContent(context, searchRequestParsers.aggParsers, searchRequestParsers.suggesters);
-            }
+        if (requestContentParser != null) {
+            QueryParseContext context = new QueryParseContext(requestContentParser);
+            searchRequest.source().parseXContent(context);
+        }
+
+        final int batchedReduceSize = request.paramAsInt("batched_reduce_size", searchRequest.getBatchedReduceSize());
+        searchRequest.setBatchedReduceSize(batchedReduceSize);
+        searchRequest.setPreFilterShardSize(request.paramAsInt("pre_filter_shard_size", searchRequest.getPreFilterShardSize()));
+
+        if (request.hasParam("max_concurrent_shard_requests")) {
+            // only set if we have the parameter since we auto adjust the max concurrency on the coordinator
+            // based on the number of nodes in the cluster
+            final int maxConcurrentShardRequests = request.paramAsInt("max_concurrent_shard_requests",
+                searchRequest.getMaxConcurrentShardRequests());
+            searchRequest.setMaxConcurrentShardRequests(maxConcurrentShardRequests);
         }
 
         // do not allow 'query_and_fetch' or 'dfs_query_and_fetch' search types
         // from the REST layer. these modes are an internal optimization and should
         // not be specified explicitly by the user.
         String searchType = request.param("search_type");
-        if (SearchType.fromString(searchType, parseFieldMatcher).equals(SearchType.QUERY_AND_FETCH) ||
-                SearchType.fromString(searchType, parseFieldMatcher).equals(SearchType.DFS_QUERY_AND_FETCH)) {
+        if ("query_and_fetch".equals(searchType) ||
+                "dfs_query_and_fetch".equals(searchType)) {
             throw new IllegalArgumentException("Unsupported search type [" + searchType + "]");
         } else {
             searchRequest.searchType(searchType);
@@ -237,4 +239,9 @@ private static void parseSearchSource(final SearchSourceBuilder searchSourceBuil
                         .suggestMode(SuggestMode.resolve(suggestMode))));
         }
     }
+
+    @Override
+    protected Set<String> responseParams() {
+        return RESPONSE_PARAMS;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java b/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java
index 9b9ddd3a93da7..8e535bfad7977 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java
@@ -19,22 +19,13 @@
 
 package org.elasticsearch.rest.action.search;
 
-import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchScrollRequest;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.action.RestActions;
 import org.elasticsearch.rest.action.RestStatusToXContentListener;
 import org.elasticsearch.search.Scroll;
 
@@ -45,8 +36,6 @@
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
 public class RestSearchScrollAction extends BaseRestHandler {
-
-    @Inject
     public RestSearchScrollAction(Settings settings, RestController controller) {
         super(settings);
 
@@ -57,7 +46,7 @@ public RestSearchScrollAction(Settings settings, RestController controller) {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         String scrollId = request.param("scroll_id");
         SearchScrollRequest searchScrollRequest = new SearchScrollRequest();
         searchScrollRequest.scrollId(scrollId);
@@ -65,44 +54,30 @@ public void handleRequest(final RestRequest request, final RestChannel channel,
         if (scroll != null) {
             searchScrollRequest.scroll(new Scroll(parseTimeValue(scroll, null, "scroll")));
         }
-
-        if (RestActions.hasBodyContent(request)) {
-            XContentType type = XContentFactory.xContentType(RestActions.getRestContent(request));
-            if (type == null) {
-                if (scrollId == null) {
-                    scrollId = RestActions.getRestContent(request).utf8ToString();
-                    searchScrollRequest.scrollId(scrollId);
+        request.withContentOrSourceParamParserOrNullLenient(xContentParser -> {
+            if (xContentParser == null) {
+                if (request.hasContent()) {
+                    // TODO: why do we accept this plain text value? maybe we can just use the scroll params?
+                    BytesReference body = request.getContentOrSourceParamOnly();
+                    if (scrollId == null) {
+                        String bodyScrollId = body.utf8ToString();
+                        searchScrollRequest.scrollId(bodyScrollId);
+                    }
                 }
             } else {
-                // NOTE: if rest request with xcontent body has request parameters, these parameters override xcontent values
-                buildFromContent(RestActions.getRestContent(request), searchScrollRequest);
+                // NOTE: if rest request with xcontent body has request parameters, values parsed from request body have the precedence
+                try {
+                    searchScrollRequest.fromXContent(xContentParser);
+                } catch (IOException e) {
+                    throw new IllegalArgumentException("Failed to parse request body", e);
+                }
             }
-        }
-        client.searchScroll(searchScrollRequest, new RestStatusToXContentListener<SearchResponse>(channel));
+        });
+        return channel -> client.searchScroll(searchScrollRequest, new RestStatusToXContentListener<>(channel));
     }
 
-    public static void buildFromContent(BytesReference content, SearchScrollRequest searchScrollRequest) {
-        try (XContentParser parser = XContentHelper.createParser(content)) {
-            if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
-                throw new IllegalArgumentException("Malformed content, must start with an object");
-            } else {
-                XContentParser.Token token;
-                String currentFieldName = null;
-                while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                    if (token == XContentParser.Token.FIELD_NAME) {
-                        currentFieldName = parser.currentName();
-                    } else if ("scroll_id".equals(currentFieldName) && token == XContentParser.Token.VALUE_STRING) {
-                        searchScrollRequest.scrollId(parser.text());
-                    } else if ("scroll".equals(currentFieldName) && token == XContentParser.Token.VALUE_STRING) {
-                        searchScrollRequest.scroll(new Scroll(TimeValue.parseTimeValue(parser.text(), null, "scroll")));
-                    } else {
-                        throw new IllegalArgumentException("Unknown parameter [" + currentFieldName
-                                + "] in request body or parameter is of the wrong type[" + token + "] ");
-                    }
-                }
-            }
-        } catch (IOException e) {
-            throw new IllegalArgumentException("Failed to parse request body", e);
-        }
+    @Override
+    public boolean supportsPlainText() {
+        return true;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/search/RestSuggestAction.java b/core/src/main/java/org/elasticsearch/rest/action/search/RestSuggestAction.java
index b55a1590f3d84..e414a67ab6440 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/search/RestSuggestAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/search/RestSuggestAction.java
@@ -19,77 +19,62 @@
 
 package org.elasticsearch.rest.action.search;
 
-import java.io.IOException;
-
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.rest.action.RestActions;
 import org.elasticsearch.rest.action.RestBuilderListener;
-import org.elasticsearch.search.SearchRequestParsers;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.suggest.Suggest;
 import org.elasticsearch.search.suggest.SuggestBuilder;
 
+import java.io.IOException;
+
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.rest.action.RestActions.buildBroadcastShardsHeader;
 
 public class RestSuggestAction extends BaseRestHandler {
-
-    private final SearchRequestParsers searchRequestParsers;
-
-    @Inject
-    public RestSuggestAction(Settings settings, RestController controller,
-                             SearchRequestParsers searchRequestParsers) {
+    public RestSuggestAction(Settings settings, RestController controller) {
         super(settings);
-        this.searchRequestParsers = searchRequestParsers;
-        controller.registerHandler(POST, "/_suggest", this);
-        controller.registerHandler(GET, "/_suggest", this);
-        controller.registerHandler(POST, "/{index}/_suggest", this);
-        controller.registerHandler(GET, "/{index}/_suggest", this);
+        controller.registerAsDeprecatedHandler(POST, "/_suggest", this,
+                "[POST /_suggest] is deprecated! Use [POST /_search] instead.", deprecationLogger);
+        controller.registerAsDeprecatedHandler(GET, "/_suggest", this,
+                "[GET /_suggest] is deprecated! Use [GET /_search] instead.", deprecationLogger);
+        controller.registerAsDeprecatedHandler(POST, "/{index}/_suggest", this,
+                "[POST /{index}/_suggest] is deprecated! Use [POST /{index}/_search] instead.", deprecationLogger);
+        controller.registerAsDeprecatedHandler(GET, "/{index}/_suggest", this,
+                "[GET /{index}/_suggest] is deprecated! Use [GET /{index}/_search] instead.", deprecationLogger);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) throws IOException {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         final SearchRequest searchRequest = new SearchRequest(
                 Strings.splitStringByCommaToArray(request.param("index")), new SearchSourceBuilder());
         searchRequest.indicesOptions(IndicesOptions.fromRequest(request, searchRequest.indicesOptions()));
-        if (RestActions.hasBodyContent(request)) {
-            final BytesReference sourceBytes = RestActions.getRestContent(request);
-            try (XContentParser parser = XContentFactory.xContent(sourceBytes).createParser(sourceBytes)) {
-                final QueryParseContext context = new QueryParseContext(searchRequestParsers.queryParsers, parser, parseFieldMatcher);
-                searchRequest.source().suggest(SuggestBuilder.fromXContent(context, searchRequestParsers.suggesters));
-            }
-        } else {
-            throw new IllegalArgumentException("no content or source provided to execute suggestion");
+        try (XContentParser parser = request.contentOrSourceParamParser()) {
+            searchRequest.source().suggest(SuggestBuilder.fromXContent(parser));
         }
         searchRequest.routing(request.param("routing"));
         searchRequest.preference(request.param("preference"));
-        client.search(searchRequest, new RestBuilderListener<SearchResponse>(channel) {
+        return channel -> client.search(searchRequest, new RestBuilderListener<SearchResponse>(channel) {
             @Override
             public RestResponse buildResponse(SearchResponse response, XContentBuilder builder) throws Exception {
                 RestStatus restStatus = RestStatus.status(response.getSuccessfulShards(),
                     response.getTotalShards(), response.getShardFailures());
                 builder.startObject();
                 buildBroadcastShardsHeader(builder, request, response.getTotalShards(),
-                    response.getSuccessfulShards(), response.getFailedShards(), response.getShardFailures());
+                    response.getSuccessfulShards(), response.getFailedShards(), response.getSkippedShards(), response.getShardFailures());
                 Suggest suggest = response.getSuggest();
                 if (suggest != null) {
                     suggest.toInnerXContent(builder, request);
diff --git a/core/src/main/java/org/elasticsearch/script/CompiledScript.java b/core/src/main/java/org/elasticsearch/script/CompiledScript.java
index ec2ad4192a2fc..818971f0f8990 100644
--- a/core/src/main/java/org/elasticsearch/script/CompiledScript.java
+++ b/core/src/main/java/org/elasticsearch/script/CompiledScript.java
@@ -24,7 +24,7 @@
  */
 public class CompiledScript {
 
-    private final ScriptService.ScriptType type;
+    private final ScriptType type;
     private final String name;
     private final String lang;
     private final Object compiled;
@@ -36,7 +36,7 @@ public class CompiledScript {
      * @param lang The language of the script to be executed.
      * @param compiled The compiled script Object that is executable.
      */
-    public CompiledScript(ScriptService.ScriptType type, String name, String lang, Object compiled) {
+    public CompiledScript(ScriptType type, String name, String lang, Object compiled) {
         this.type = type;
         this.name = name;
         this.lang = lang;
@@ -47,7 +47,7 @@ public CompiledScript(ScriptService.ScriptType type, String name, String lang, O
      * Method to get the type of language.
      * @return The type of language the script was compiled in.
      */
-    public ScriptService.ScriptType type() {
+    public ScriptType type() {
         return type;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/script/LeafSearchScript.java b/core/src/main/java/org/elasticsearch/script/LeafSearchScript.java
index 0bf9e0d50e072..762168d3c9086 100644
--- a/core/src/main/java/org/elasticsearch/script/LeafSearchScript.java
+++ b/core/src/main/java/org/elasticsearch/script/LeafSearchScript.java
@@ -19,18 +19,30 @@
 
 package org.elasticsearch.script;
 
+import org.apache.lucene.search.Scorer;
 import org.elasticsearch.common.lucene.ScorerAware;
 
 import java.util.Map;
 
 /**
  * A per-segment {@link SearchScript}.
+ *
+ * This is effectively a functional interface, requiring at least implementing {@link #runAsDouble()}.
  */
 public interface LeafSearchScript extends ScorerAware, ExecutableScript {
 
-    void setDocument(int doc);
+    /**
+     * Set the document this script will process next.
+     */
+    default void setDocument(int doc) {}
+
+    @Override
+    default void setScorer(Scorer scorer) {}
 
-    void setSource(Map<String, Object> source);
+    /**
+     * Set the source for the current document.
+     */
+    default void setSource(Map<String, Object> source) {}
     
     /**
      * Sets per-document aggregation {@code _value}.
@@ -44,8 +56,23 @@ default void setNextAggregationValue(Object value) {
         setNextVar("_value", value);
     }
 
-    long runAsLong();
+    @Override
+    default void setNextVar(String field, Object value) {}
 
-    double runAsDouble();
+    /**
+     * Return the result as a long. This is used by aggregation scripts over long fields.
+     */
+    default long runAsLong() {
+        throw new UnsupportedOperationException("runAsLong is not implemented");
+    }
+
+    @Override
+    default Object run() {
+        return runAsDouble();
+    }
 
+    /**
+     * Return the result as a double. This is the main use case of search script, used for document scoring.
+     */
+    double runAsDouble();
 }
diff --git a/core/src/main/java/org/elasticsearch/script/NativeScriptEngineService.java b/core/src/main/java/org/elasticsearch/script/NativeScriptEngineService.java
index 191c2b4bcf7a4..a8031d85ac696 100644
--- a/core/src/main/java/org/elasticsearch/script/NativeScriptEngineService.java
+++ b/core/src/main/java/org/elasticsearch/script/NativeScriptEngineService.java
@@ -19,9 +19,12 @@
 
 package org.elasticsearch.script;
 
+import org.apache.logging.log4j.Logger;
 import org.apache.lucene.index.LeafReaderContext;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.component.AbstractComponent;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.search.lookup.SearchLookup;
 
@@ -41,6 +44,11 @@ public class NativeScriptEngineService extends AbstractComponent implements Scri
 
     public NativeScriptEngineService(Settings settings, Map<String, NativeScriptFactory> scripts) {
         super(settings);
+        if (scripts.isEmpty() == false) {
+            Logger logger = Loggers.getLogger(ScriptModule.class);
+            DeprecationLogger deprecationLogger = new DeprecationLogger(logger);
+            deprecationLogger.deprecated("Native scripts are deprecated. Use a custom ScriptEngine to write scripts in java.");
+        }
         this.scripts = unmodifiableMap(scripts);
     }
 
@@ -72,10 +80,10 @@ public ExecutableScript executable(CompiledScript compiledScript, @Nullable Map<
     @Override
     public SearchScript search(CompiledScript compiledScript, final SearchLookup lookup, @Nullable final Map<String, Object> vars) {
         final NativeScriptFactory scriptFactory = (NativeScriptFactory) compiledScript.compiled();
+        final AbstractSearchScript script = (AbstractSearchScript) scriptFactory.newScript(vars);
         return new SearchScript() {
             @Override
             public LeafSearchScript getLeafSearchScript(LeafReaderContext context) throws IOException {
-                AbstractSearchScript script = (AbstractSearchScript) scriptFactory.newScript(vars);
                 script.setLookup(lookup.getLeafSearchLookup(context));
                 return script;
             }
diff --git a/core/src/main/java/org/elasticsearch/script/NativeScriptFactory.java b/core/src/main/java/org/elasticsearch/script/NativeScriptFactory.java
index 7fca2501903be..e889c1fe5663f 100644
--- a/core/src/main/java/org/elasticsearch/script/NativeScriptFactory.java
+++ b/core/src/main/java/org/elasticsearch/script/NativeScriptFactory.java
@@ -31,7 +31,9 @@
  * @see AbstractSearchScript
  * @see AbstractLongSearchScript
  * @see AbstractDoubleSearchScript
+ * @deprecated Create a {@link ScriptEngineService} instead of using native scripts
  */
+@Deprecated
 public interface NativeScriptFactory {
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/script/Script.java b/core/src/main/java/org/elasticsearch/script/Script.java
index 94abb43bc066c..4cce56186ed1f 100644
--- a/core/src/main/java/org/elasticsearch/script/Script.java
+++ b/core/src/main/java/org/elasticsearch/script/Script.java
@@ -19,273 +19,759 @@
 
 package org.elasticsearch.script;
 
-import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.common.Nullable;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.ESLoggerFactory;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParser.Token;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.script.ScriptService.ScriptType;
 
 import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
 
 /**
- * Script holds all the parameters necessary to compile or find in cache and then execute a script.
+ * {@link Script} represents used-defined input that can be used to
+ * compile and execute a script from the {@link ScriptService}
+ * based on the {@link ScriptType}.
+ *
+ * There are three types of scripts specified by {@link ScriptType}.
+ *
+ * The following describes the expected parameters for each type of script:
+ *
+ * <ul>
+ * <li> {@link ScriptType#INLINE}
+ * <ul>
+ * <li> {@link Script#lang}     - specifies the language, defaults to {@link Script#DEFAULT_SCRIPT_LANG}
+ * <li> {@link Script#idOrCode} - specifies the code to be compiled, must not be {@code null}
+ * <li> {@link Script#options}  - specifies the compiler options for this script; must not be {@code null},
+ *                                use an empty {@link Map} to specify no options
+ * <li> {@link Script#params}   - {@link Map} of user-defined parameters; must not be {@code null},
+ *                                use an empty {@link Map} to specify no params
+ * </ul>
+ * <li> {@link ScriptType#STORED}
+ * <ul>
+ * <li> {@link Script#lang}     - the language will be specified when storing the script, so this should
+ *                                be {@code null}; however, this can be specified to look up a stored
+ *                                script as part of the deprecated API
+ * <li> {@link Script#idOrCode} - specifies the id of the stored script to be looked up, must not be {@code null}
+ * <li> {@link Script#options}  - compiler options will be specified when a stored script is stored,
+ *                                so they have no meaning here and must be {@code null}
+ * <li> {@link Script#params}   - {@link Map} of user-defined parameters; must not be {@code null},
+ *                                use an empty {@link Map} to specify no params
+ * </ul>
+ * <li> {@link ScriptType#FILE}
+ * <ul>
+ * <li> {@link Script#lang}     - specifies the language for look up, defaults to {@link Script#DEFAULT_SCRIPT_LANG}
+ * <li> {@link Script#idOrCode} - specifies the id of the file script to be looked up, must not be {@code null}
+ * <li> {@link Script#options}  - compiler options will be specified when a file script is loaded,
+ *                                so they have no meaning here and must be {@code null}
+ * <li> {@link Script#params}   - {@link Map} of user-defined parameters; must not be {@code null},
+ *                                use an empty {@link Map} to specify no params
+ * </ul>
+ * </ul>
  */
-public final class Script implements ToXContent, Writeable {
+public final class Script implements ToXContentObject, Writeable {
+
+    public static final Version V_5_1_0_UNRELEASED = Version.fromId(5010099);
 
-    public static final ScriptType DEFAULT_TYPE = ScriptType.INLINE;
+    /**
+     * Standard logger necessary for allocation of the deprecation logger.
+     */
+    private static final Logger LOGGER = ESLoggerFactory.getLogger(ScriptMetaData.class);
+
+    /**
+     * Deprecation logger necessary for namespace changes related to stored scripts.
+     */
+    private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(LOGGER);
+
+    /**
+     * The name of the of the default scripting language.
+     */
     public static final String DEFAULT_SCRIPT_LANG = "painless";
 
-    private String script;
-    private ScriptType type;
-    @Nullable private String lang;
-    @Nullable private Map<String, Object> params;
-    @Nullable private XContentType contentType;
+    /**
+     * The name of the default template language.
+     */
+    public static final String DEFAULT_TEMPLATE_LANG = "mustache";
 
     /**
-     * Constructor for simple inline script. The script will have no lang or params set.
-     *
-     * @param script The inline script to execute.
+     * The default {@link ScriptType}.
      */
-    public Script(String script) {
-        this(script, ScriptType.INLINE, null, null);
-    }
+    public static final ScriptType DEFAULT_SCRIPT_TYPE = ScriptType.INLINE;
 
-    public Script(String script, ScriptType type, String lang, @Nullable Map<String, ?> params) {
-        this(script, type, lang, params, null);
-    }
+    /**
+     * Compiler option for {@link XContentType} used for templates.
+     */
+    public static final String CONTENT_TYPE_OPTION = "content_type";
 
     /**
-     * Constructor for Script.
-     *
-     * @param script        The cache key of the script to be compiled/executed. For inline scripts this is the actual
-     *                      script source code. For indexed scripts this is the id used in the request. For on file
-     *                      scripts this is the file name.
-     * @param type          The type of script -- dynamic, stored, or file.
-     * @param lang          The language of the script to be compiled/executed.
-     * @param params        The map of parameters the script will be executed with.
-     * @param contentType   The {@link XContentType} of the script. Only relevant for inline scripts that have not been
-     *                      defined as a plain string, but as json or yaml content. This class needs this information
-     *                      when serializing the script back to xcontent.
+     * Standard {@link ParseField} for outer level of script queries.
+     */
+    public static final ParseField SCRIPT_PARSE_FIELD = new ParseField("script");
+
+    /**
+     * Standard {@link ParseField} for lang on the inner level.
      */
-    @SuppressWarnings("unchecked")
-    public Script(String script, ScriptType type, String lang, @Nullable Map<String, ?> params,
-                  @Nullable XContentType  contentType) {
-        if (contentType != null && type != ScriptType.INLINE) {
-            throw new IllegalArgumentException("The parameter contentType only makes sense for inline scripts");
+    public static final ParseField LANG_PARSE_FIELD = new ParseField("lang");
+
+    /**
+     * Standard {@link ParseField} for options on the inner level.
+     */
+    public static final ParseField OPTIONS_PARSE_FIELD = new ParseField("options");
+
+    /**
+     * Standard {@link ParseField} for params on the inner level.
+     */
+    public static final ParseField PARAMS_PARSE_FIELD = new ParseField("params");
+
+    /**
+     * Helper class used by {@link ObjectParser} to store mutable {@link Script} variables and then
+     * construct an immutable {@link Script} object based on parsed XContent.
+     */
+    private static final class Builder {
+        private ScriptType type;
+        private String lang;
+        private String idOrCode;
+        private Map<String, String> options;
+        private Map<String, Object> params;
+
+        private Builder() {
+            // This cannot default to an empty map because options are potentially added at multiple points.
+            this.options = new HashMap<>();
+            this.params = Collections.emptyMap();
         }
-        this.script = Objects.requireNonNull(script);
-        this.type = Objects.requireNonNull(type);
-        this.lang = lang == null ? DEFAULT_SCRIPT_LANG : lang;
-        this.params = (Map<String, Object>) params;
-        this.contentType = contentType;
-    }
 
-    public Script(StreamInput in) throws IOException {
-        script = in.readString();
-        if (in.readBoolean()) {
-            type = ScriptType.readFrom(in);
+        /**
+         * Since inline scripts can accept code rather than just an id, they must also be able
+         * to handle template parsing, hence the need for custom parsing code.  Templates can
+         * consist of either an {@link String} or a JSON object.  If a JSON object is discovered
+         * then the content type option must also be saved as a compiler option.
+         */
+        private void setInline(XContentParser parser) {
+            try {
+                if (type != null) {
+                    throwOnlyOneOfType();
+                }
+
+                type = ScriptType.INLINE;
+
+                if (parser.currentToken() == Token.START_OBJECT) {
+                    //this is really for search templates, that need to be converted to json format
+                    XContentBuilder builder = XContentFactory.jsonBuilder();
+                    idOrCode = builder.copyCurrentStructure(parser).string();
+                    options.put(CONTENT_TYPE_OPTION, XContentType.JSON.mediaType());
+                } else {
+                    idOrCode = parser.text();
+                }
+            } catch (IOException exception) {
+                throw new UncheckedIOException(exception);
+            }
         }
-        lang = in.readOptionalString();
-        params = in.readMap();
-        if (in.readBoolean()) {
-            contentType = XContentType.readFrom(in);
+
+        /**
+         * Set both the id and the type of the stored script.
+         */
+        private void setStored(String idOrCode) {
+            if (type != null) {
+                throwOnlyOneOfType();
+            }
+
+            type = ScriptType.STORED;
+            this.idOrCode = idOrCode;
         }
-    }
 
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeString(script);
-        boolean hasType = type != null;
-        out.writeBoolean(hasType);
-        if (hasType) {
-            ScriptType.writeTo(type, out);
+        /**
+         * Set both the id and the type of the file script.
+         */
+        private void setFile(String idOrCode) {
+            if (type != null) {
+                throwOnlyOneOfType();
+            }
+
+            type = ScriptType.FILE;
+            this.idOrCode = idOrCode;
+        }
+
+        /**
+         * Helper method to throw an exception if more than one type of {@link Script} is specified.
+         */
+        private void throwOnlyOneOfType() {
+            throw new IllegalArgumentException("must only use one of [" +
+                ScriptType.INLINE.getParseField().getPreferredName() + " + , " +
+                ScriptType.STORED.getParseField().getPreferredName() + " + , " +
+                ScriptType.FILE.getParseField().getPreferredName() + "]" +
+                " when specifying a script");
+        }
+
+        private void setLang(String lang) {
+            this.lang = lang;
         }
-        out.writeOptionalString(lang);
-        out.writeMap(params);
-        boolean hasContentType = contentType != null;
-        out.writeBoolean(hasContentType);
-        if (hasContentType) {
-            XContentType.writeTo(contentType, out);
+
+        /**
+         * Options may have already been added if an inline template was specified.
+         * Appends the user-defined compiler options with the internal compiler options.
+         */
+        private void setOptions(Map<String, String> options) {
+            this.options.putAll(options);
         }
+
+        private void setParams(Map<String, Object> params) {
+            this.params = params;
+        }
+
+        /**
+         * Validates the parameters and creates an {@link Script}.
+         * @param defaultLang The default lang is not a compile-time constant and must be provided
+         *                    at run-time this way in case a legacy default language is used from
+         *                    previously stored queries.
+         */
+        private Script build(String defaultLang) {
+            if (type == null) {
+                throw new IllegalArgumentException("must specify either [source] for an inline script, [id] for a stored script, " +
+                    "or [" + ScriptType.FILE.getParseField().getPreferredName() + "] for a file script");
+            }
+
+            if (type == ScriptType.INLINE) {
+                if (lang == null) {
+                    lang = defaultLang;
+                }
+
+                if (idOrCode == null) {
+                    throw new IllegalArgumentException(
+                        "must specify <id> for an [" + ScriptType.INLINE.getParseField().getPreferredName() + "] script");
+                }
+
+                if (options.size() > 1 || options.size() == 1 && options.get(CONTENT_TYPE_OPTION) == null) {
+                    options.remove(CONTENT_TYPE_OPTION);
+
+                    throw new IllegalArgumentException("illegal compiler options [" + options + "] specified");
+                }
+            } else if (type == ScriptType.STORED) {
+                // Only issue this deprecation warning if we aren't using a template.  Templates during
+                // this deprecation phase must always specify the default template language or they would
+                // possibly pick up a script in a different language as defined by the user under the new
+                // namespace unintentionally.
+                if (lang != null && lang.equals(DEFAULT_TEMPLATE_LANG) == false) {
+                    DEPRECATION_LOGGER.deprecated("specifying the field [" + LANG_PARSE_FIELD.getPreferredName() + "] " +
+                        "for executing " + ScriptType.STORED + " scripts is deprecated; use only the field " +
+                        "[" + ScriptType.STORED.getParseField().getPreferredName() + "] to specify an <id>");
+                }
+
+                if (idOrCode == null) {
+                    throw new IllegalArgumentException(
+                        "must specify <code> for an [" + ScriptType.STORED.getParseField().getPreferredName() + "] script");
+                }
+
+                if (options.isEmpty()) {
+                    options = null;
+                } else {
+                    throw new IllegalArgumentException("field [" + OPTIONS_PARSE_FIELD.getPreferredName() + "] " +
+                        "cannot be specified using a [" + ScriptType.STORED.getParseField().getPreferredName() + "] script");
+                }
+            } else if (type == ScriptType.FILE) {
+                if (lang == null) {
+                    lang = defaultLang;
+                }
+
+                if (idOrCode == null) {
+                    throw new IllegalArgumentException(
+                        "must specify <code> for an [" + ScriptType.FILE.getParseField().getPreferredName() + "] script");
+                }
+
+                if (options.isEmpty()) {
+                    options = null;
+                } else {
+                    throw new IllegalArgumentException("field [" + OPTIONS_PARSE_FIELD.getPreferredName() + "] " +
+                        "cannot be specified using a [" + ScriptType.FILE.getParseField().getPreferredName() + "] script");
+                }
+            }
+
+            return new Script(type, lang, idOrCode, options, params);
+        }
+    }
+
+    private static final ObjectParser<Builder, Void> PARSER = new ObjectParser<>("script", Builder::new);
+
+    static {
+        // Defines the fields necessary to parse a Script as XContent using an ObjectParser.
+        PARSER.declareField(Builder::setInline, parser -> parser, ScriptType.INLINE.getParseField(), ValueType.OBJECT_OR_STRING);
+        PARSER.declareString(Builder::setStored, ScriptType.STORED.getParseField());
+        PARSER.declareString(Builder::setFile, ScriptType.FILE.getParseField());
+        PARSER.declareString(Builder::setLang, LANG_PARSE_FIELD);
+        PARSER.declareField(Builder::setOptions, XContentParser::mapStrings, OPTIONS_PARSE_FIELD, ValueType.OBJECT);
+        PARSER.declareField(Builder::setParams, XContentParser::map, PARAMS_PARSE_FIELD, ValueType.OBJECT);
     }
 
     /**
-     * Method for getting the script.
-     * @return The cache key of the script to be compiled/executed.  For dynamic scripts this is the actual
-     *         script source code.  For indexed scripts this is the id used in the request.  For on disk scripts
-     *         this is the file name.
+     * Convenience method to call {@link Script#parse(XContentParser, String)}
+     * using the default scripting language.
      */
-    public String getScript() {
-        return script;
+    public static Script parse(XContentParser parser) throws IOException {
+        return parse(parser, DEFAULT_SCRIPT_LANG);
     }
 
     /**
-     * Method for getting the type.
+     * This will parse XContent into a {@link Script}.  The following formats can be parsed:
+     *
+     * The simple format defaults to an {@link ScriptType#INLINE} with no compiler options or user-defined params:
+     *
+     * Example:
+     * {@code
+     * "return Math.log(doc.popularity) * 100;"
+     * }
+     *
+     * The complex format where {@link ScriptType} and idOrCode are required while lang, options and params are not required.
      *
-     * @return The type of script -- inline, stored, or file.
+     * {@code
+     * {
+     *     // Exactly one of "id" or "source" must be specified
+     *     "id" : "<id>",
+     *     // OR
+     *     "source": "<source>",
+     *     "lang" : "<lang>",
+     *     "options" : {
+     *         "option0" : "<option0>",
+     *         "option1" : "<option1>",
+     *         ...
+     *     },
+     *     "params" : {
+     *         "param0" : "<param0>",
+     *         "param1" : "<param1>",
+     *         ...
+     *     }
+     * }
+     * }
+     *
+     * Example:
+     * {@code
+     * {
+     *     "source" : "return Math.log(doc.popularity) * params.multiplier",
+     *     "lang" : "painless",
+     *     "params" : {
+     *         "multiplier" : 100.0
+     *     }
+     * }
+     * }
+     *
+     * This also handles templates in a special way.  If a complexly formatted query is specified as another complex
+     * JSON object the query is assumed to be a template, and the format will be preserved.
+     *
+     * {@code
+     * {
+     *     "source" : { "query" : ... },
+     *     "lang" : "<lang>",
+     *     "options" : {
+     *         "option0" : "<option0>",
+     *         "option1" : "<option1>",
+     *         ...
+     *     },
+     *     "params" : {
+     *         "param0" : "<param0>",
+     *         "param1" : "<param1>",
+     *         ...
+     *     }
+     * }
+     * }
+     *
+     * @param parser       The {@link XContentParser} to be used.
+     * @param defaultLang  The default language to use if no language is specified.  The default language isn't necessarily
+     *                     the one defined by {@link Script#DEFAULT_SCRIPT_LANG} due to backwards compatiblity requirements
+     *                     related to stored queries using previously default languauges.
+     *
+     * @return             The parsed {@link Script}.
      */
-    public ScriptType getType() {
-        return type;
+    public static Script parse(XContentParser parser, String defaultLang) throws IOException {
+        Objects.requireNonNull(defaultLang);
+
+        Token token = parser.currentToken();
+
+        if (token == null) {
+            token = parser.nextToken();
+        }
+
+        if (token == Token.VALUE_STRING) {
+            return new Script(ScriptType.INLINE, defaultLang, parser.text(), Collections.emptyMap());
+        }
+
+        return PARSER.apply(parser, null).build(defaultLang);
     }
 
+    private final ScriptType type;
+    private final String lang;
+    private final String idOrCode;
+    private final Map<String, String> options;
+    private final Map<String, Object> params;
+
     /**
-     * Method for getting language.
-     *
-     * @return The language of the script to be compiled/executed.
+     * Constructor for simple script using the default language and default type.
+     * @param idOrCode The id or code to use dependent on the default script type.
      */
-    public String getLang() {
-        return lang;
+    public Script(String idOrCode) {
+        this(DEFAULT_SCRIPT_TYPE, DEFAULT_SCRIPT_LANG, idOrCode, Collections.emptyMap(), Collections.emptyMap());
     }
 
     /**
-     * Method for getting the parameters.
-     *
-     * @return The map of parameters the script will be executed with.
+     * Constructor for a script that does not need to use compiler options.
+     * @param type     The {@link ScriptType}.
+     * @param lang     The language for this {@link Script} if the {@link ScriptType} is {@link ScriptType#INLINE} or
+     *                 {@link ScriptType#FILE}.  For {@link ScriptType#STORED} scripts this should be null, but can
+     *                 be specified to access scripts stored as part of the stored scripts deprecated API.
+     * @param idOrCode The id for this {@link Script} if the {@link ScriptType} is {@link ScriptType#FILE} or {@link ScriptType#STORED}.
+     *                 The code for this {@link Script} if the {@link ScriptType} is {@link ScriptType#INLINE}.
+     * @param params   The user-defined params to be bound for script execution.
      */
-    public Map<String, Object> getParams() {
-        return params;
+    public Script(ScriptType type, String lang, String idOrCode, Map<String, Object> params) {
+        this(type, lang, idOrCode, type == ScriptType.INLINE ? Collections.emptyMap() : null, params);
+    }
+
+    /**
+     * Constructor for a script that requires the use of compiler options.
+     * @param type     The {@link ScriptType}.
+     * @param lang     The language for this {@link Script} if the {@link ScriptType} is {@link ScriptType#INLINE} or
+     *                 {@link ScriptType#FILE}.  For {@link ScriptType#STORED} scripts this should be null, but can
+     *                 be specified to access scripts stored as part of the stored scripts deprecated API.
+     * @param idOrCode The id for this {@link Script} if the {@link ScriptType} is {@link ScriptType#FILE} or {@link ScriptType#STORED}.
+     *                 The code for this {@link Script} if the {@link ScriptType} is {@link ScriptType#INLINE}.
+     * @param options  The map of compiler options for this {@link Script} if the {@link ScriptType}
+     *                 is {@link ScriptType#INLINE}, {@code null} otherwise.
+     * @param params   The user-defined params to be bound for script execution.
+     */
+    public Script(ScriptType type, String lang, String idOrCode, Map<String, String> options, Map<String, Object> params) {
+        this.type = Objects.requireNonNull(type);
+        this.idOrCode = Objects.requireNonNull(idOrCode);
+        this.params = Collections.unmodifiableMap(Objects.requireNonNull(params));
+
+        if (type == ScriptType.INLINE) {
+            this.lang = Objects.requireNonNull(lang);
+            this.options = Collections.unmodifiableMap(Objects.requireNonNull(options));
+        } else if (type == ScriptType.STORED) {
+            this.lang = lang;
+
+            if (options != null) {
+                throw new IllegalStateException(
+                    "options must be null for [" + ScriptType.STORED.getParseField().getPreferredName() + "] scripts");
+            }
+
+            this.options = null;
+        } else if (type == ScriptType.FILE) {
+            this.lang = Objects.requireNonNull(lang);
+
+            if (options != null) {
+                throw new IllegalStateException(
+                    "options must be null for [" + ScriptType.FILE.getParseField().getPreferredName() + "] scripts");
+            }
+
+            this.options = null;
+        } else {
+            throw new IllegalStateException("unknown script type [" + type.getName() + "]");
+        }
     }
 
     /**
-     * @return The content type of the script if it is an inline script and the script has been defined as json
-     *         or yaml content instead of a plain string.
+     * Creates a {@link Script} read from an input stream.
      */
-    public XContentType getContentType() {
-        return contentType;
+    public Script(StreamInput in) throws IOException {
+        // Version 5.3 allows lang to be an optional parameter for stored scripts and expects
+        // options to be null for stored and file scripts.
+        if (in.getVersion().onOrAfter(Version.V_5_3_0)) {
+            this.type = ScriptType.readFrom(in);
+            this.lang = in.readOptionalString();
+            this.idOrCode = in.readString();
+            @SuppressWarnings("unchecked")
+            Map<String, String> options = (Map<String, String>)(Map)in.readMap();
+            this.options = options;
+            this.params = in.readMap();
+        // Version 5.1 to 5.3 (exclusive) requires all Script members to be non-null and supports the potential
+        // for more options than just XContentType.  Reorders the read in contents to be in
+        // same order as the constructor.
+        } else if (in.getVersion().onOrAfter(Version.V_5_1_1)) {
+            this.type = ScriptType.readFrom(in);
+            this.lang = in.readString();
+
+            this.idOrCode = in.readString();
+            @SuppressWarnings("unchecked")
+            Map<String, String> options = (Map<String, String>)(Map)in.readMap();
+
+            if (this.type != ScriptType.INLINE && options.isEmpty()) {
+                this.options = null;
+            } else {
+                this.options = options;
+            }
+
+            this.params = in.readMap();
+        // Prior to version 5.1 the script members are read in certain cases as optional and given
+        // default values when necessary.  Also the only option supported is for XContentType.
+        } else {
+            this.idOrCode = in.readString();
+
+            if (in.readBoolean()) {
+                this.type = ScriptType.readFrom(in);
+            } else {
+                this.type = DEFAULT_SCRIPT_TYPE;
+            }
+
+            String lang = in.readOptionalString();
+
+            if (lang == null) {
+                this.lang = DEFAULT_SCRIPT_LANG;
+            } else {
+                this.lang = lang;
+            }
+
+            Map<String, Object> params = in.readMap();
+
+            if (params == null) {
+                this.params = new HashMap<>();
+            } else {
+                this.params = params;
+            }
+
+            if (in.readBoolean()) {
+                this.options = new HashMap<>();
+                XContentType contentType = XContentType.readFrom(in);
+                this.options.put(CONTENT_TYPE_OPTION, contentType.mediaType());
+            } else if (type == ScriptType.INLINE) {
+                options = new HashMap<>();
+            } else {
+                this.options = null;
+            }
+        }
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params builderParams) throws IOException {
-        if (type == null) {
-            return builder.value(script);
+    public void writeTo(StreamOutput out) throws IOException {
+        // Version 5.3+ allows lang to be an optional parameter for stored scripts and expects
+        // options to be null for stored and file scripts.
+        if (out.getVersion().onOrAfter(Version.V_5_3_0)) {
+            type.writeTo(out);
+            out.writeOptionalString(lang);
+            out.writeString(idOrCode);
+            @SuppressWarnings("unchecked")
+            Map<String, Object> options = (Map<String, Object>)(Map)this.options;
+            out.writeMap(options);
+            out.writeMap(params);
+        // Version 5.1 to 5.3 (exclusive) requires all Script members to be non-null and supports the potential
+        // for more options than just XContentType.  Reorders the written out contents to be in
+        // same order as the constructor.
+        } else if (out.getVersion().onOrAfter(Version.V_5_1_1)) {
+            type.writeTo(out);
+
+            if (lang == null) {
+                out.writeString("");
+            } else {
+                out.writeString(lang);
+            }
+
+            out.writeString(idOrCode);
+            @SuppressWarnings("unchecked")
+            Map<String, Object> options = (Map<String, Object>)(Map)this.options;
+
+            if (options == null) {
+                out.writeMap(new HashMap<>());
+            } else {
+                out.writeMap(options);
+            }
+
+            out.writeMap(params);
+        // Prior to version 5.1 the Script members were possibly written as optional or null, though there is no case where a null
+        // value wasn't equivalent to it's default value when actually compiling/executing a script.  Meaning, there are no
+        // backwards compatibility issues, and now there's enforced consistency.  Also the only supported compiler
+        // option was XContentType.
+        } else {
+            out.writeString(idOrCode);
+            out.writeBoolean(true);
+            type.writeTo(out);
+            out.writeOptionalString(lang);
+
+            if (params.isEmpty()) {
+                out.writeMap(null);
+            } else {
+                out.writeMap(params);
+            }
+
+            if (options != null && options.containsKey(CONTENT_TYPE_OPTION)) {
+                XContentType contentType = XContentType.fromMediaTypeOrFormat(options.get(CONTENT_TYPE_OPTION));
+                out.writeBoolean(true);
+                contentType.writeTo(out);
+            } else {
+                out.writeBoolean(false);
+            }
         }
+    }
+
+    /**
+     * This will build scripts into the following XContent structure:
+     *
+     * {@code
+     * {
+     *     "<(id, source)>" : "<idOrCode>",
+     *     "lang" : "<lang>",
+     *     "options" : {
+     *         "option0" : "<option0>",
+     *         "option1" : "<option1>",
+     *         ...
+     *     },
+     *     "params" : {
+     *         "param0" : "<param0>",
+     *         "param1" : "<param1>",
+     *         ...
+     *     }
+     * }
+     * }
+     *
+     * Example:
+     * {@code
+     * {
+     *     "source" : "return Math.log(doc.popularity) * params.multiplier;",
+     *     "lang" : "painless",
+     *     "params" : {
+     *         "multiplier" : 100.0
+     *     }
+     * }
+     * }
+     *
+     * Note that lang, options, and params will only be included if there have been any specified.
+     *
+     * This also handles templates in a special way.  If the {@link Script#CONTENT_TYPE_OPTION} option
+     * is provided and the {@link ScriptType#INLINE} is specified then the template will be preserved as a raw field.
+     *
+     * {@code
+     * {
+     *     "source" : { "query" : ... },
+     *     "lang" : "<lang>",
+     *     "options" : {
+     *         "option0" : "<option0>",
+     *         "option1" : "<option1>",
+     *         ...
+     *     },
+     *     "params" : {
+     *         "param0" : "<param0>",
+     *         "param1" : "<param1>",
+     *         ...
+     *     }
+     * }
+     * }
+     */
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params builderParams) throws IOException {
         builder.startObject();
-        if (type == ScriptType.INLINE && contentType != null && builder.contentType() == contentType) {
-            builder.rawField(type.getParseField().getPreferredName(), new BytesArray(script));
+
+        String contentType = options == null ? null : options.get(CONTENT_TYPE_OPTION);
+
+        if (type == ScriptType.INLINE) {
+            if (contentType != null && builder.contentType().mediaType().equals(contentType)) {
+                builder.rawField(ScriptType.INLINE.getParseField().getPreferredName(), new BytesArray(idOrCode));
+            } else {
+                builder.field(ScriptType.INLINE.getParseField().getPreferredName(), idOrCode);
+            }
         } else {
-            builder.field(type.getParseField().getPreferredName(), script);
+            builder.field(type.getParseField().getPreferredName(), idOrCode);
         }
+
         if (lang != null) {
-            builder.field(ScriptField.LANG.getPreferredName(), lang);
+            builder.field(LANG_PARSE_FIELD.getPreferredName(), lang);
+        }
+
+        if (options != null && !options.isEmpty()) {
+            builder.field(OPTIONS_PARSE_FIELD.getPreferredName(), options);
         }
-        if (params != null) {
-            builder.field(ScriptField.PARAMS.getPreferredName(), params);
+
+        if (!params.isEmpty()) {
+            builder.field(PARAMS_PARSE_FIELD.getPreferredName(), params);
         }
+
         builder.endObject();
+
         return builder;
     }
 
-    public static Script parse(XContentParser parser, ParseFieldMatcher parseFieldMatcher) throws IOException {
-        return parse(parser, parseFieldMatcher, null);
+    /**
+     * @return The {@link ScriptType} for this {@link Script}.
+     */
+    public ScriptType getType() {
+        return type;
     }
 
-    public static Script parse(XContentParser parser, ParseFieldMatcher parseFieldMatcher, @Nullable String lang) throws IOException {
-        XContentParser.Token token = parser.currentToken();
-        // If the parser hasn't yet been pushed to the first token, do it now
-        if (token == null) {
-            token = parser.nextToken();
-        }
-        if (token == XContentParser.Token.VALUE_STRING) {
-            return new Script(parser.text(), ScriptType.INLINE, lang, null);
-        }
-        if (token != XContentParser.Token.START_OBJECT) {
-            throw new ElasticsearchParseException("expected a string value or an object, but found [{}] instead", token);
-        }
-        String script = null;
-        ScriptType type = null;
-        Map<String, Object> params = null;
-        XContentType contentType = null;
-        String cfn = null;
-        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-            if (token == XContentParser.Token.FIELD_NAME) {
-                cfn = parser.currentName();
-            } else if (parseFieldMatcher.match(cfn, ScriptType.INLINE.getParseField())) {
-                type = ScriptType.INLINE;
-                if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
-                    contentType = parser.contentType();
-                    XContentBuilder builder = XContentFactory.contentBuilder(contentType);
-                    script = builder.copyCurrentStructure(parser).bytes().utf8ToString();
-                } else {
-                    script = parser.text();
-                }
-            } else if (parseFieldMatcher.match(cfn, ScriptType.FILE.getParseField())) {
-                type = ScriptType.FILE;
-                if (token == XContentParser.Token.VALUE_STRING) {
-                    script = parser.text();
-                } else {
-                    throw new ElasticsearchParseException("expected a string value for field [{}], but found [{}]", cfn, token);
-                }
-            } else if (parseFieldMatcher.match(cfn, ScriptType.STORED.getParseField())) {
-                type = ScriptType.STORED;
-                if (token == XContentParser.Token.VALUE_STRING) {
-                    script = parser.text();
-                } else {
-                    throw new ElasticsearchParseException("expected a string value for field [{}], but found [{}]", cfn, token);
-                }
-            } else if (parseFieldMatcher.match(cfn, ScriptField.LANG)) {
-                if (token == XContentParser.Token.VALUE_STRING) {
-                    lang = parser.text();
-                } else {
-                    throw new ElasticsearchParseException("expected a string value for field [{}], but found [{}]", cfn, token);
-                }
-            } else if (parseFieldMatcher.match(cfn, ScriptField.PARAMS)) {
-                if (token == XContentParser.Token.START_OBJECT) {
-                    params = parser.map();
-                } else {
-                    throw new ElasticsearchParseException("expected an object for field [{}], but found [{}]", cfn, token);
-                }
-            } else {
-                throw new ElasticsearchParseException("unexpected field [{}]", cfn);
-            }
-        }
-        if (script == null) {
-            throw new ElasticsearchParseException("expected one of [{}], [{}] or [{}] fields, but found none",
-                    ScriptType.INLINE.getParseField() .getPreferredName(), ScriptType.FILE.getParseField().getPreferredName(),
-                    ScriptType.STORED.getParseField() .getPreferredName());
-        }
-        return new Script(script, type, lang, params, contentType);
+    /**
+     * @return The language for this {@link Script} if the {@link ScriptType} is {@link ScriptType#INLINE} or
+     *         {@link ScriptType#FILE}.  For {@link ScriptType#STORED} scripts this should be null, but can
+     *         be specified to access scripts stored as part of the stored scripts deprecated API.
+     */
+    public String getLang() {
+        return lang;
     }
 
-    @Override
-    public int hashCode() {
-        return Objects.hash(lang, params, script, type, contentType);
+    /**
+     * @return The id for this {@link Script} if the {@link ScriptType} is {@link ScriptType#FILE} or {@link ScriptType#STORED}.
+     *         The code for this {@link Script} if the {@link ScriptType} is {@link ScriptType#INLINE}.
+     */
+    public String getIdOrCode() {
+        return idOrCode;
     }
 
-    @Override
-    public boolean equals(Object obj) {
-        if (this == obj) return true;
-        if (obj == null) return false;
-        if (getClass() != obj.getClass()) return false;
-        Script other = (Script) obj;
-
-        return Objects.equals(lang, other.lang) &&
-                Objects.equals(params, other.params) &&
-                Objects.equals(script, other.script) &&
-                Objects.equals(type, other.type) &&
-                Objects.equals(contentType, other.contentType);
+    /**
+     * @return The map of compiler options for this {@link Script} if the {@link ScriptType}
+     *         is {@link ScriptType#INLINE}, {@code null} otherwise.
+     */
+    public Map<String, String> getOptions() {
+        return options;
+    }
+
+    /**
+     * @return The map of user-defined params for this {@link Script}.
+     */
+    public Map<String, Object> getParams() {
+        return params;
     }
 
     @Override
-    public String toString() {
-        return "[script: " + script + ", type: " + type.getParseField().getPreferredName() + ", lang: "
-                + lang + ", params: " + params + "]";
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        Script script = (Script)o;
+
+        if (type != script.type) return false;
+        if (lang != null ? !lang.equals(script.lang) : script.lang != null) return false;
+        if (!idOrCode.equals(script.idOrCode)) return false;
+        if (options != null ? !options.equals(script.options) : script.options != null) return false;
+        return params.equals(script.params);
+
     }
 
-    public interface ScriptField {
-        ParseField SCRIPT = new ParseField("script");
-        ParseField LANG = new ParseField("lang");
-        ParseField PARAMS = new ParseField("params");
+    @Override
+    public int hashCode() {
+        int result = type.hashCode();
+        result = 31 * result + (lang != null ? lang.hashCode() : 0);
+        result = 31 * result + idOrCode.hashCode();
+        result = 31 * result + (options != null ? options.hashCode() : 0);
+        result = 31 * result + params.hashCode();
+        return result;
     }
 
+    @Override
+    public String toString() {
+        return "Script{" +
+            "type=" + type +
+            ", lang='" + lang + '\'' +
+            ", idOrCode='" + idOrCode + '\'' +
+            ", options=" + options +
+            ", params=" + params +
+            '}';
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptContextRegistry.java b/core/src/main/java/org/elasticsearch/script/ScriptContextRegistry.java
index b4ed91faebaea..765be1d437818 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptContextRegistry.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptContextRegistry.java
@@ -19,8 +19,6 @@
 
 package org.elasticsearch.script;
 
-import org.elasticsearch.common.settings.Settings;
-
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -66,8 +64,8 @@ Collection<ScriptContext> scriptContexts() {
     /**
      * @return <tt>true</tt> if the provided {@link ScriptContext} is supported, <tt>false</tt> otherwise
      */
-    boolean isSupportedContext(ScriptContext scriptContext) {
-        return scriptContexts.containsKey(scriptContext.getKey());
+    boolean isSupportedContext(String scriptContext) {
+        return scriptContexts.containsKey(scriptContext);
     }
 
     //script contexts can be used in fine-grained settings, we need to be careful with what we allow here
@@ -81,8 +79,8 @@ private void validateScriptContext(ScriptContext.Plugin scriptContext) {
     }
 
     private static Set<String> reservedScriptContexts() {
-        Set<String> reserved = new HashSet<>(ScriptService.ScriptType.values().length + ScriptContext.Standard.values().length);
-        for (ScriptService.ScriptType scriptType : ScriptService.ScriptType.values()) {
+        Set<String> reserved = new HashSet<>(ScriptType.values().length + ScriptContext.Standard.values().length);
+        for (ScriptType scriptType : ScriptType.values()) {
             reserved.add(scriptType.toString());
         }
         for (ScriptContext.Standard scriptContext : ScriptContext.Standard.values()) {
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptEngineService.java b/core/src/main/java/org/elasticsearch/script/ScriptEngineService.java
index 55a931d8d57c3..4dc3870fac9a2 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptEngineService.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptEngineService.java
@@ -32,7 +32,12 @@ public interface ScriptEngineService extends Closeable {
 
     String getType();
 
-    String getExtension();
+    /**
+     * The extension for file scripts in this language.
+     */
+    default String getExtension() {
+        return getType();
+    }
 
     /**
      * Compiles a script.
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptException.java b/core/src/main/java/org/elasticsearch/script/ScriptException.java
index 475091f9f6d7a..91e6ad401fc88 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptException.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptException.java
@@ -87,8 +87,7 @@ public void writeTo(StreamOutput out) throws IOException {
     }
     
     @Override
-    protected void innerToXContent(XContentBuilder builder, Params params) throws IOException {
-        super.innerToXContent(builder, params);
+    protected void metadataToXContent(XContentBuilder builder, Params params) throws IOException {
         builder.field("script_stack", scriptStack);
         builder.field("script", script);
         builder.field("lang", lang);
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptMetaData.java b/core/src/main/java/org/elasticsearch/script/ScriptMetaData.java
index 979bffb4bccfc..63b5e2e46ab8d 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptMetaData.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptMetaData.java
@@ -18,22 +18,24 @@
  */
 package org.elasticsearch.script;
 
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ResourceNotFoundException;
-import org.elasticsearch.cluster.AbstractDiffable;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.cluster.DiffableUtils;
+import org.elasticsearch.cluster.NamedDiff;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.ESLoggerFactory;
+import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentParser.Token;
-import org.elasticsearch.common.xcontent.XContentType;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -41,255 +43,416 @@
 import java.util.HashMap;
 import java.util.Map;
 
-public final class ScriptMetaData implements MetaData.Custom {
+/**
+ * {@link ScriptMetaData} is used to store user-defined scripts
+ * as part of the {@link ClusterState}.  Currently scripts can
+ * be stored as part of the new namespace for a stored script where
+ * only an id is used or as part of the deprecated namespace where
+ * both a language and an id are used.
+ */
+public final class ScriptMetaData implements MetaData.Custom, Writeable, ToXContent {
+
+    /**
+     * A builder used to modify the currently stored scripts data held within
+     * the {@link ClusterState}.  Scripts can be added or deleted, then built
+     * to generate a new {@link Map} of scripts that will be used to update
+     * the current {@link ClusterState}.
+     */
+    public static final class Builder {
 
-    public static final String TYPE = "stored_scripts";
-    public static final ScriptMetaData PROTO = new ScriptMetaData(Collections.emptyMap());
+        private final Map<String, StoredScriptSource> scripts;
 
-    private final Map<String, ScriptAsBytes> scripts;
+        /**
+         * @param previous The current {@link ScriptMetaData} or {@code null} if there
+         *                 is no existing {@link ScriptMetaData}.
+         */
+        public Builder(ScriptMetaData previous) {
+            this.scripts = previous == null ? new HashMap<>() :new HashMap<>(previous.scripts);
+        }
 
-    ScriptMetaData(Map<String, ScriptAsBytes> scripts) {
-        this.scripts = scripts;
-    }
+        /**
+         * Add a new script to the existing stored scripts.  The script will be added under
+         * both the new namespace and the deprecated namespace, so that look ups under
+         * the deprecated namespace will continue to work.  Should a script already exist under
+         * the new namespace using a different language, it will be replaced and a deprecation
+         * warning will be issued.  The replaced script will still exist under the deprecated
+         * namespace and can continue to be looked up this way until it is deleted.
+         * <p>
+         * Take for example script 'A' with lang 'L0' and data 'D0'.  If we add script 'A' to the
+         * empty set, the scripts {@link Map} will be ["A" -- D0, "A#L0" -- D0].  If a script
+         * 'A' with lang 'L1' and data 'D1' is then added, the scripts {@link Map} will be
+         * ["A" -- D1, "A#L1" -- D1, "A#L0" -- D0].
+         * @param id The user-specified id to use for the look up.
+         * @param source The user-specified stored script data held in {@link StoredScriptSource}.
+         */
+        public Builder storeScript(String id, StoredScriptSource source) {
+            StoredScriptSource previous = scripts.put(id, source);
+            scripts.put(source.getLang() + "#" + id, source);
+
+            if (previous != null && previous.getLang().equals(source.getLang()) == false) {
+                DEPRECATION_LOGGER.deprecated("stored script [" + id + "] already exists using a different lang " +
+                    "[" + previous.getLang() + "], the new namespace for stored scripts will only use (id) instead of (lang, id)");
+            }
 
-    public BytesReference getScriptAsBytes(String language, String id) {
-        ScriptAsBytes scriptAsBytes = scripts.get(toKey(language, id));
-        if (scriptAsBytes != null) {
-            return scriptAsBytes.script;
-        } else {
-            return null;
+            return this;
         }
-    }
 
-    public String getScript(String language, String id) {
-        BytesReference scriptAsBytes = getScriptAsBytes(language, id);
-        if (scriptAsBytes == null) {
-            return null;
-        }
-        return scriptAsBytes.utf8ToString();
-    }
+        /**
+         * Delete a script from the existing stored scripts.  The script will be removed from the
+         * new namespace if the script language matches the current script under the same id or
+         * if the script language is {@code null}.  The script will be removed from the deprecated
+         * namespace on any delete either using using the specified lang parameter or the language
+         * found from looking up the script in the new namespace.
+         * <p>
+         * Take for example a scripts {@link Map} with {"A" -- D1, "A#L1" -- D1, "A#L0" -- D0}.
+         * If a script is removed specified by an id 'A' and lang {@code null} then the scripts
+         * {@link Map} will be {"A#L0" -- D0}.  To remove the final script, the deprecated
+         * namespace must be used, so an id 'A' and lang 'L0' would need to be specified.
+         * @param id The user-specified id to use for the look up.
+         * @param lang The user-specified language to use for the look up if using the deprecated
+         *             namespace, otherwise {@code null}.
+         */
+        public Builder deleteScript(String id, String lang) {
+            StoredScriptSource source = scripts.get(id);
+
+            if (lang == null) {
+                if (source == null) {
+                    throw new ResourceNotFoundException("stored script [" + id + "] does not exist and cannot be deleted");
+                }
+
+                lang = source.getLang();
+            }
 
-    public static String parseStoredScript(BytesReference scriptAsBytes) {
-        // Scripts can be stored via API in several ways:
-        // 1) wrapped into a 'script' json object or field
-        // 2) wrapped into a 'template' json object or field
-        // 3) just as is
-        // In order to fetch the actual script in consistent manner this parsing logic is needed:
-        try (XContentParser parser = XContentHelper.createParser(scriptAsBytes);
-             XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) {
-            parser.nextToken();
-            parser.nextToken();
-            if (parser.currentToken() == Token.END_OBJECT) {
-                throw new IllegalArgumentException("Empty script");
+            if (source != null) {
+                if (lang.equals(source.getLang())) {
+                    scripts.remove(id);
+                }
             }
-            switch (parser.currentName()) {
-                case "script":
-                case "template":
-                    if (parser.nextToken() == Token.VALUE_STRING) {
-                        return parser.text();
-                    } else {
-                        builder.copyCurrentStructure(parser);
-                    }
-                    break;
-                default:
-                    // There is no enclosing 'script' or 'template' object so we just need to return the script as is...
-                    // because the parsers current location is already beyond the beginning we need to add a START_OBJECT:
-                    builder.startObject();
-                    builder.copyCurrentStructure(parser);
-                    builder.endObject();
-                    break;
+
+            source = scripts.get(lang + "#" + id);
+
+            if (source == null) {
+                throw new ResourceNotFoundException(
+                    "stored script [" + id + "] using lang [" + lang + "] does not exist and cannot be deleted");
             }
-            return builder.string();
-        } catch (IOException e) {
-            throw new RuntimeException(e);
+
+            scripts.remove(lang + "#" + id);
+
+            return this;
         }
-    }
 
-    @Override
-    public String type() {
-        return TYPE;
+        /**
+         * @return A {@link ScriptMetaData} with the updated {@link Map} of scripts.
+         */
+        public ScriptMetaData build() {
+            return new ScriptMetaData(scripts);
+        }
     }
 
-    @Override
-    public ScriptMetaData fromXContent(XContentParser parser) throws IOException {
-        Map<String, ScriptAsBytes> scripts = new HashMap<>();
-        String key = null;
-        for (Token token = parser.nextToken(); token != Token.END_OBJECT; token = parser.nextToken()) {
-            switch (token) {
-                case FIELD_NAME:
-                    key = parser.currentName();
-                    break;
-                case VALUE_STRING:
-                    scripts.put(key, new ScriptAsBytes(new BytesArray(parser.text())));
-                    break;
-                default:
-                    throw new ParsingException(parser.getTokenLocation(), "Unexpected token [" + token + "]");
-            }
+    static final class ScriptMetadataDiff implements NamedDiff<MetaData.Custom> {
+
+        final Diff<Map<String, StoredScriptSource>> pipelines;
+
+        ScriptMetadataDiff(ScriptMetaData before, ScriptMetaData after) {
+            this.pipelines = DiffableUtils.diff(before.scripts, after.scripts, DiffableUtils.getStringKeySerializer());
         }
-        return new ScriptMetaData(scripts);
-    }
 
-    @Override
-    public EnumSet<MetaData.XContentContext> context() {
-        return MetaData.API_AND_GATEWAY;
-    }
+        ScriptMetadataDiff(StreamInput in) throws IOException {
+            pipelines = DiffableUtils.readJdkMapDiff(in, DiffableUtils.getStringKeySerializer(),
+                StoredScriptSource::new, StoredScriptSource::readDiffFrom);
+        }
 
-    @Override
-    public ScriptMetaData readFrom(StreamInput in) throws IOException {
-        int size = in.readVInt();
-        Map<String, ScriptAsBytes> scripts = new HashMap<>();
-        for (int i = 0; i < size; i++) {
-            String languageAndId = in.readString();
-            BytesReference script = in.readBytesReference();
-            scripts.put(languageAndId, new ScriptAsBytes(script));
+        @Override
+        public String getWriteableName() {
+            return TYPE;
         }
-        return new ScriptMetaData(scripts);
-    }
 
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        for (Map.Entry<String, ScriptAsBytes> entry : scripts.entrySet()) {
-            builder.field(entry.getKey(), entry.getValue().script.utf8ToString());
+        @Override
+        public MetaData.Custom apply(MetaData.Custom part) {
+            return new ScriptMetaData(pipelines.apply(((ScriptMetaData) part).scripts));
         }
-        return builder;
-    }
 
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeVInt(scripts.size());
-        for (Map.Entry<String, ScriptAsBytes> entry : scripts.entrySet()) {
-            out.writeString(entry.getKey());
-            entry.getValue().writeTo(out);
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            pipelines.writeTo(out);
         }
     }
 
-    @Override
-    public Diff<MetaData.Custom> diff(MetaData.Custom before) {
-        return new ScriptMetadataDiff((ScriptMetaData) before, this);
-    }
+    /**
+     * Convenience method to build and return a new
+     * {@link ScriptMetaData} adding the specified stored script.
+     */
+    static ScriptMetaData putStoredScript(ScriptMetaData previous, String id, StoredScriptSource source) {
+        Builder builder = new Builder(previous);
+        builder.storeScript(id, source);
 
-    @Override
-    public Diff<MetaData.Custom> readDiffFrom(StreamInput in) throws IOException {
-        return new ScriptMetadataDiff(in);
+        return builder.build();
     }
 
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) return true;
-        if (o == null || getClass() != o.getClass()) return false;
+    /**
+     * Convenience method to build and return a new
+     * {@link ScriptMetaData} deleting the specified stored script.
+     */
+    static ScriptMetaData deleteStoredScript(ScriptMetaData previous, String id, String lang) {
+        Builder builder = new ScriptMetaData.Builder(previous);
+        builder.deleteScript(id, lang);
 
-        ScriptMetaData other = (ScriptMetaData) o;
-        return scripts.equals(other.scripts);
+        return builder.build();
     }
 
-    @Override
-    public int hashCode() {
-        return scripts.hashCode();
-    }
+    /**
+     * Standard logger necessary for allocation of the deprecation logger.
+     */
+    private static final Logger LOGGER = ESLoggerFactory.getLogger(ScriptMetaData.class);
 
-    @Override
-    public String toString() {
-        return "ScriptMetaData{" +
-            "scripts=" + scripts +
-            '}';
-    }
+    /**
+     * Deprecation logger necessary for namespace changes related to stored scripts.
+     */
+    private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(LOGGER);
+
+    /**
+     * The type of {@link ClusterState} data.
+     */
+    public static final String TYPE = "stored_scripts";
 
-    static String toKey(String language, String id) {
-        if (id.contains("#")) {
-            throw new IllegalArgumentException("stored script id can't contain: '#'");
+    /**
+     * This will parse XContent into {@link ScriptMetaData}.
+     *
+     * The following format will be parsed for the new namespace:
+     *
+     * {@code
+     * {
+     *     "<id>" : "<{@link StoredScriptSource#fromXContent(XContentParser)}>",
+     *     "<id>" : "<{@link StoredScriptSource#fromXContent(XContentParser)}>",
+     *     ...
+     * }
+     * }
+     *
+     * The following format will be parsed for the deprecated namespace:
+     *
+     * {@code
+     * {
+     *     "<id>" : "<code>",
+     *     "<id>" : "<code>",
+     *     ...
+     * }
+     * }
+     *
+     * Note when using the deprecated namespace, the language will be pulled from
+     * the id and options will be set to an empty {@link Map}.
+     */
+    public static ScriptMetaData fromXContent(XContentParser parser) throws IOException {
+        Map<String, StoredScriptSource> scripts = new HashMap<>();
+        String id = null;
+        StoredScriptSource source;
+
+        Token token = parser.currentToken();
+
+        if (token == null) {
+            token = parser.nextToken();
         }
-        if (language.contains("#")) {
-            throw new IllegalArgumentException("stored script language can't contain: '#'");
+
+        if (token != Token.START_OBJECT) {
+            throw new ParsingException(parser.getTokenLocation(), "unexpected token [" + token + "], expected [{]");
         }
 
-        return language + "#" + id;
-    }
+        token = parser.nextToken();
 
-    public static final class Builder {
+        while (token != Token.END_OBJECT) {
+            switch (token) {
+                case FIELD_NAME:
+                    id = parser.currentName();
+                    break;
+                case VALUE_STRING:
+                    if (id == null) {
+                        throw new ParsingException(parser.getTokenLocation(),
+                            "unexpected token [" + token + "], expected [<id>, <code>, {]");
+                    }
 
-        private Map<String, ScriptAsBytes> scripts;
+                    int split = id.indexOf('#');
 
-        public Builder(ScriptMetaData previous) {
-            if (previous != null) {
-                this.scripts = new HashMap<>(previous.scripts);
-            } else {
-                this.scripts = new HashMap<>();
+                    if (split == -1) {
+                        throw new IllegalArgumentException("illegal stored script id [" + id + "], does not contain lang");
+                    } else {
+                        source = new StoredScriptSource(id.substring(0, split), parser.text(), Collections.emptyMap());
+                    }
+                    scripts.put(id, source);
+
+                    id = null;
+
+                    break;
+                case START_OBJECT:
+                    if (id == null) {
+                        throw new ParsingException(parser.getTokenLocation(),
+                            "unexpected token [" + token + "], expected [<id>, <code>, {]");
+                    }
+
+                    source = StoredScriptSource.fromXContent(parser);
+                    scripts.put(id, source);
+
+                    break;
+                default:
+                    throw new ParsingException(parser.getTokenLocation(), "unexpected token [" + token + "], expected [<id>, <code>, {]");
             }
-        }
 
-        public Builder storeScript(String lang, String id, BytesReference script) {
-            BytesReference scriptBytest = new BytesArray(parseStoredScript(script));
-            scripts.put(toKey(lang, id), new ScriptAsBytes(scriptBytest));
-            return this;
+            token = parser.nextToken();
         }
 
-        public Builder deleteScript(String lang, String id) {
-            if (scripts.remove(toKey(lang, id)) == null) {
-                throw new ResourceNotFoundException("Stored script with id [{}] for language [{}] does not exist", id, lang);
+        return new ScriptMetaData(scripts);
+    }
+
+    public static NamedDiff<MetaData.Custom> readDiffFrom(StreamInput in) throws IOException {
+        return new ScriptMetadataDiff(in);
+    }
+
+    private final Map<String, StoredScriptSource> scripts;
+
+    /**
+     * Standard constructor to create metadata to store scripts.
+     * @param scripts The currently stored scripts.  Must not be {@code null},
+     *                use and empty {@link Map} to specify there were no
+     *                previously stored scripts.
+     */
+    ScriptMetaData(Map<String, StoredScriptSource> scripts) {
+        this.scripts = Collections.unmodifiableMap(scripts);
+    }
+
+    public ScriptMetaData(StreamInput in) throws IOException {
+        Map<String, StoredScriptSource> scripts = new HashMap<>();
+        StoredScriptSource source;
+        int size = in.readVInt();
+
+        for (int i = 0; i < size; i++) {
+            String id = in.readString();
+
+            // Prior to version 5.3 all scripts were stored using the deprecated namespace.
+            // Split the id to find the language then use StoredScriptSource to parse the
+            // expected BytesReference after which a new StoredScriptSource is created
+            // with the appropriate language and options.
+            if (in.getVersion().before(Version.V_5_3_0)) {
+                int split = id.indexOf('#');
+
+                if (split == -1) {
+                    throw new IllegalArgumentException("illegal stored script id [" + id + "], does not contain lang");
+                } else {
+                    source = new StoredScriptSource(in);
+                    source = new StoredScriptSource(id.substring(0, split), source.getSource(), Collections.emptyMap());
+                }
+            // Version 5.3+ can just be parsed normally using StoredScriptSource.
+            } else {
+                source = new StoredScriptSource(in);
             }
-            return this;
-        }
 
-        public ScriptMetaData build() {
-            return new ScriptMetaData(Collections.unmodifiableMap(scripts));
+            scripts.put(id, source);
         }
+
+        this.scripts = Collections.unmodifiableMap(scripts);
     }
 
-    static final class ScriptMetadataDiff implements Diff<MetaData.Custom> {
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        // Version 5.3+ will output the contents of the scripts' Map using
+        // StoredScriptSource to stored the language, code, and options.
+        if (out.getVersion().onOrAfter(Version.V_5_3_0)) {
+            out.writeVInt(scripts.size());
+
+            for (Map.Entry<String, StoredScriptSource> entry : scripts.entrySet()) {
+                out.writeString(entry.getKey());
+                entry.getValue().writeTo(out);
+            }
+        // Prior to Version 5.3, stored scripts can only be read using the deprecated
+        // namespace.  Scripts using the deprecated namespace are first isolated in a
+        // temporary Map, then written out.  Since all scripts will be stored using the
+        // deprecated namespace, no scripts will be lost.
+        } else {
+            Map<String, StoredScriptSource> filtered = new HashMap<>();
 
-        final Diff<Map<String, ScriptAsBytes>> pipelines;
+            for (Map.Entry<String, StoredScriptSource> entry : scripts.entrySet()) {
+                if (entry.getKey().contains("#")) {
+                    filtered.put(entry.getKey(), entry.getValue());
+                }
+            }
 
-        ScriptMetadataDiff(ScriptMetaData before, ScriptMetaData after) {
-            this.pipelines = DiffableUtils.diff(before.scripts, after.scripts, DiffableUtils.getStringKeySerializer());
-        }
+            out.writeVInt(filtered.size());
 
-        public ScriptMetadataDiff(StreamInput in) throws IOException {
-            pipelines = DiffableUtils.readJdkMapDiff(in, DiffableUtils.getStringKeySerializer(), new ScriptAsBytes(null));
+            for (Map.Entry<String, StoredScriptSource> entry : filtered.entrySet()) {
+                out.writeString(entry.getKey());
+                entry.getValue().writeTo(out);
+            }
         }
+    }
 
-        @Override
-        public MetaData.Custom apply(MetaData.Custom part) {
-            return new ScriptMetaData(pipelines.apply(((ScriptMetaData) part).scripts));
-        }
 
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            pipelines.writeTo(out);
+
+    /**
+     * This will write XContent from {@link ScriptMetaData}.  The following format will be written:
+     *
+     * {@code
+     * {
+     *     "<id>" : "<{@link StoredScriptSource#toXContent(XContentBuilder, Params)}>",
+     *     "<id>" : "<{@link StoredScriptSource#toXContent(XContentBuilder, Params)}>",
+     *     ...
+     * }
+     * }
+     */
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        for (Map.Entry<String, StoredScriptSource> entry : scripts.entrySet()) {
+            builder.field(entry.getKey());
+            entry.getValue().toXContent(builder, params);
         }
+
+        return builder;
     }
 
-    static final class ScriptAsBytes extends AbstractDiffable<ScriptAsBytes> {
+    @Override
+    public Diff<MetaData.Custom> diff(MetaData.Custom before) {
+        return new ScriptMetadataDiff((ScriptMetaData)before, this);
+    }
 
-        public ScriptAsBytes(BytesReference script) {
-            this.script = script;
-        }
+    @Override
+    public String getWriteableName() {
+        return TYPE;
+    }
 
-        private final BytesReference script;
+    @Override
+    public EnumSet<MetaData.XContentContext> context() {
+        return MetaData.ALL_CONTEXTS;
+    }
 
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            out.writeBytesReference(script);
+    /**
+     * Retrieves a stored script from the new namespace if lang is {@code null}.
+     * Otherwise, returns a stored script from the deprecated namespace.  Either
+     * way an id is required.
+     */
+    StoredScriptSource getStoredScript(String id, String lang) {
+        if (lang == null) {
+            return scripts.get(id);
+        } else {
+            return scripts.get(lang + "#" + id);
         }
+    }
 
-        @Override
-        public ScriptAsBytes readFrom(StreamInput in) throws IOException {
-            return new ScriptAsBytes(in.readBytesReference());
-        }
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
 
-        @Override
-        public boolean equals(Object o) {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
+        ScriptMetaData that = (ScriptMetaData)o;
 
-            ScriptAsBytes that = (ScriptAsBytes) o;
+        return scripts.equals(that.scripts);
 
-            return script.equals(that.script);
+    }
 
-        }
+    @Override
+    public int hashCode() {
+        return scripts.hashCode();
+    }
 
-        @Override
-        public int hashCode() {
-            return script.hashCode();
-        }
+    @Override
+    public String toString() {
+        return "ScriptMetaData{" +
+            "scripts=" + scripts +
+            '}';
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptModes.java b/core/src/main/java/org/elasticsearch/script/ScriptModes.java
index 46ab2a44d21cd..f698e1a8b5427 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptModes.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptModes.java
@@ -21,12 +21,15 @@
 
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.script.ScriptService.ScriptType;
 
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.TreeMap;
+import java.util.function.Function;
 
 /**
  * Holds the boolean indicating the enabled mode for each of the different scripting languages available, each script source and each
@@ -39,12 +42,89 @@ public class ScriptModes {
 
     final Map<String, Boolean> scriptEnabled;
 
-    ScriptModes(ScriptSettings scriptSettings, Settings settings) {
+    private static final String NONE = "none";
+
+    public static final Setting<List<String>> TYPES_ALLOWED_SETTING =
+        Setting.listSetting("script.allowed_types", Collections.emptyList(), Function.identity(), Setting.Property.NodeScope);
+    public static final Setting<List<String>> CONTEXTS_ALLOWED_SETTING =
+        Setting.listSetting("script.allowed_contexts", Collections.emptyList(), Function.identity(), Setting.Property.NodeScope);
+
+    private final Set<String> typesAllowed;
+    private final Set<String> contextsAllowed;
+
+    ScriptModes(ScriptContextRegistry scriptContextRegistry, ScriptSettings scriptSettings, Settings settings) {
         HashMap<String, Boolean> scriptModes = new HashMap<>();
         for (Setting<Boolean> scriptModeSetting : scriptSettings.getScriptLanguageSettings()) {
             scriptModes.put(scriptModeSetting.getKey(), scriptModeSetting.get(settings));
         }
         this.scriptEnabled = Collections.unmodifiableMap(scriptModes);
+
+        this.typesAllowed = TYPES_ALLOWED_SETTING.exists(settings) ? new HashSet<>() : null;
+
+        if (this.typesAllowed != null) {
+            List<String> typesAllowedList = TYPES_ALLOWED_SETTING.get(settings);
+
+            if (typesAllowedList.isEmpty()) {
+                throw new IllegalArgumentException(
+                    "must specify at least one script type or none for setting [" + TYPES_ALLOWED_SETTING.getKey() + "].");
+            }
+
+            for (String settingType : typesAllowedList) {
+                if (NONE.equals(settingType)) {
+                    if (typesAllowedList.size() != 1) {
+                        throw new IllegalArgumentException("cannot specify both [" + NONE + "]" +
+                            " and other script types for setting [" + TYPES_ALLOWED_SETTING.getKey() + "].");
+                    } else {
+                        break;
+                    }
+                }
+
+                boolean found = false;
+
+                for (ScriptType scriptType : ScriptType.values()) {
+                    if (scriptType.getName().equals(settingType)) {
+                        found = true;
+                        this.typesAllowed.add(settingType);
+
+                        break;
+                    }
+                }
+
+                if (found == false) {
+                    throw new IllegalArgumentException(
+                        "unknown script type [" + settingType + "] found in setting [" + TYPES_ALLOWED_SETTING.getKey() + "].");
+                }
+            }
+        }
+
+        this.contextsAllowed = CONTEXTS_ALLOWED_SETTING.exists(settings) ? new HashSet<>() : null;
+
+        if (this.contextsAllowed != null) {
+            List<String> contextsAllowedList = CONTEXTS_ALLOWED_SETTING.get(settings);
+
+            if (contextsAllowedList.isEmpty()) {
+                throw new IllegalArgumentException(
+                    "must specify at least one script context or none for setting [" + CONTEXTS_ALLOWED_SETTING.getKey() + "].");
+            }
+
+            for (String settingContext : contextsAllowedList) {
+                if (NONE.equals(settingContext)) {
+                    if (contextsAllowedList.size() != 1) {
+                        throw new IllegalArgumentException("cannot specify both [" + NONE + "]" +
+                            " and other script contexts for setting [" + CONTEXTS_ALLOWED_SETTING.getKey() + "].");
+                    } else {
+                        break;
+                    }
+                }
+
+                if (scriptContextRegistry.isSupportedContext(settingContext)) {
+                    this.contextsAllowed.add(settingContext);
+                } else {
+                    throw new IllegalArgumentException(
+                        "unknown script context [" + settingContext + "] found in setting [" + CONTEXTS_ALLOWED_SETTING.getKey() + "].");
+                }
+            }
+        }
     }
 
     /**
@@ -61,6 +141,15 @@ public boolean getScriptEnabled(String lang, ScriptType scriptType, ScriptContex
         if (NativeScriptEngineService.NAME.equals(lang)) {
             return true;
         }
+
+        if (typesAllowed != null && typesAllowed.contains(scriptType.getName()) == false) {
+            throw new IllegalArgumentException("[" + scriptType.getName() + "] scripts cannot be executed");
+        }
+
+        if (contextsAllowed != null && contextsAllowed.contains(scriptContext.getKey()) == false) {
+            throw new IllegalArgumentException("[" + scriptContext.getKey() + "] scripts cannot be executed");
+        }
+
         Boolean scriptMode = scriptEnabled.get(getKey(lang, scriptType, scriptContext));
         if (scriptMode == null) {
             throw new IllegalArgumentException("script mode not found for lang [" + lang + "], script_type [" + scriptType + "], operation [" + scriptContext.getKey() + "]");
@@ -73,7 +162,7 @@ static String operationKey(ScriptContext scriptContext) {
     }
 
     static String sourceKey(ScriptType scriptType) {
-        return SCRIPT_SETTINGS_PREFIX + "." + scriptType.getScriptType();
+        return SCRIPT_SETTINGS_PREFIX + "." + scriptType.getName();
     }
 
     static String getGlobalKey(String lang, ScriptType scriptType) {
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptService.java b/core/src/main/java/org/elasticsearch/script/ScriptService.java
index 9e61f39378e4f..6ae0d206a9817 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptService.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptService.java
@@ -22,6 +22,7 @@
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
 import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest;
@@ -35,7 +36,6 @@
 import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.breaker.CircuitBreakingException;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -46,17 +46,18 @@
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.io.Streams;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.logging.LoggerMessageFormat;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.search.lookup.SearchLookup;
+import org.elasticsearch.template.CompiledTemplate;
 import org.elasticsearch.watcher.FileChangesListener;
 import org.elasticsearch.watcher.FileWatcher;
 import org.elasticsearch.watcher.ResourceWatcherService;
@@ -70,7 +71,6 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 import java.util.concurrent.ConcurrentMap;
@@ -86,11 +86,11 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust
     public static final Setting<TimeValue> SCRIPT_CACHE_EXPIRE_SETTING =
         Setting.positiveTimeSetting("script.cache.expire", TimeValue.timeValueMillis(0), Property.NodeScope);
     public static final Setting<Boolean> SCRIPT_AUTO_RELOAD_ENABLED_SETTING =
-        Setting.boolSetting("script.auto_reload_enabled", true, Property.NodeScope);
+        Setting.boolSetting("script.auto_reload_enabled", true, Property.NodeScope, Property.Deprecated);
     public static final Setting<Integer> SCRIPT_MAX_SIZE_IN_BYTES =
         Setting.intSetting("script.max_size_in_bytes", 65535, Property.NodeScope);
     public static final Setting<Integer> SCRIPT_MAX_COMPILATIONS_PER_MINUTE =
-        Setting.intSetting("script.max_compilations_per_minute", 15, 0, Property.Dynamic, Property.NodeScope);
+        Setting.intSetting("script.max_compilations_per_minute", 15, 0, Property.Dynamic, Property.NodeScope, Property.Deprecated);
 
     private final Collection<ScriptEngineService> scriptEngines;
     private final Map<String, ScriptEngineService> scriptEnginesByLang;
@@ -136,7 +136,7 @@ public ScriptService(Settings settings, Environment env,
 
         TimeValue cacheExpire = SCRIPT_CACHE_EXPIRE_SETTING.get(settings);
         if (cacheExpire.getNanos() != 0) {
-            cacheBuilder.setExpireAfterAccess(cacheExpire.nanos());
+            cacheBuilder.setExpireAfterAccess(cacheExpire);
         }
 
         logger.debug("using script cache with max_size [{}], expire [{}]", cacheMaxSize, cacheExpire);
@@ -152,7 +152,7 @@ public ScriptService(Settings settings, Environment env,
         this.scriptEnginesByLang = unmodifiableMap(enginesByLangBuilder);
         this.scriptEnginesByExt = unmodifiableMap(enginesByExtBuilder);
 
-        this.scriptModes = new ScriptModes(scriptSettings, settings);
+        this.scriptModes = new ScriptModes(scriptContextRegistry, scriptSettings, settings);
 
         // add file watcher for static scripts
         scriptsDirectory = env.scriptsFile();
@@ -162,7 +162,7 @@ public ScriptService(Settings settings, Environment env,
         FileWatcher fileWatcher = new FileWatcher(scriptsDirectory);
         fileWatcher.addListener(new ScriptChangesListener());
 
-        if (SCRIPT_AUTO_RELOAD_ENABLED_SETTING.get(settings)) {
+        if (SCRIPT_AUTO_RELOAD_ENABLED_SETTING.get(settings) && resourceWatcherService != null) {
             // automatic reload is enabled - register scripts
             resourceWatcherService.add(fileWatcher);
         } else {
@@ -209,18 +209,44 @@ void setMaxCompilationsPerMinute(Integer newMaxPerMinute) {
     /**
      * Checks if a script can be executed and compiles it if needed, or returns the previously compiled and cached script.
      */
-    public CompiledScript compile(Script script, ScriptContext scriptContext, Map<String, String> params) {
-        if (script == null) {
-            throw new IllegalArgumentException("The parameter script (Script) must not be null.");
-        }
-        if (scriptContext == null) {
-            throw new IllegalArgumentException("The parameter scriptContext (ScriptContext) must not be null.");
-        }
+    public CompiledScript compile(Script script, ScriptContext scriptContext) {
+        Objects.requireNonNull(script);
+        Objects.requireNonNull(scriptContext);
 
+        ScriptType type = script.getType();
         String lang = script.getLang();
-        ScriptEngineService scriptEngineService = getScriptEngineServiceForLang(lang);
-        if (canExecuteScript(lang, script.getType(), scriptContext) == false) {
-            throw new IllegalStateException("scripts of type [" + script.getType() + "], operation [" + scriptContext.getKey() + "] and lang [" + lang + "] are disabled");
+        String idOrCode = script.getIdOrCode();
+        Map<String, String> options = script.getOptions();
+
+        String id = idOrCode;
+
+        // lang may be null when looking up a stored script, so we must get the
+        // source to retrieve the lang before checking if the context is supported
+        if (type == ScriptType.STORED) {
+            // search template requests can possibly pass in the entire path instead
+            // of just an id for looking up a stored script, so we parse the path and
+            // check for appropriate errors
+            String[] path = id.split("/");
+
+            if (path.length == 3) {
+                if (lang != null && lang.equals(path[1]) == false) {
+                    throw new IllegalStateException("conflicting script languages, found [" + path[1] + "] but expected [" + lang + "]");
+                }
+
+                id = path[2];
+
+                deprecationLogger.deprecated("use of </lang/id> [" + idOrCode + "] for looking up" +
+                    " stored scripts/templates has been deprecated, use only <id> [" + id + "] instead");
+            } else if (path.length != 1) {
+                throw new IllegalArgumentException("illegal stored script format [" + id + "] use only <id>");
+            }
+
+            // a stored script must be pulled from the cluster state every time in case
+            // the script has been updated since the last compilation
+            StoredScriptSource source = getScriptFromClusterState(id, lang);
+            lang = source.getLang();
+            idOrCode = source.getSource();
+            options = source.getOptions();
         }
 
         // TODO: fix this through some API or something, that's wrong
@@ -229,89 +255,32 @@ public CompiledScript compile(Script script, ScriptContext scriptContext, Map<St
         boolean notSupported = scriptContext.getKey().equals(ScriptContext.Standard.UPDATE.getKey());
         if (expression && notSupported) {
             throw new UnsupportedOperationException("scripts of type [" + script.getType() + "]," +
-                    " operation [" + scriptContext.getKey() + "] and lang [" + lang + "] are not supported");
-        }
-
-        return compileInternal(script, params);
-    }
-
-    /**
-     * Check whether there have been too many compilations within the last minute, throwing a circuit breaking exception if so.
-     * This is a variant of the token bucket algorithm: https://en.wikipedia.org/wiki/Token_bucket
-     *
-     * It can be thought of as a bucket with water, every time the bucket is checked, water is added proportional to the amount of time that
-     * elapsed since the last time it was checked. If there is enough water, some is removed and the request is allowed. If there is not
-     * enough water the request is denied. Just like a normal bucket, if water is added that overflows the bucket, the extra water/capacity
-     * is discarded - there can never be more water in the bucket than the size of the bucket.
-     */
-    void checkCompilationLimit() {
-        long now = System.nanoTime();
-        long timePassed = now - lastInlineCompileTime;
-        lastInlineCompileTime = now;
-
-        scriptsPerMinCounter += ((double) timePassed) * compilesAllowedPerNano;
-
-        // It's been over the time limit anyway, readjust the bucket to be level
-        if (scriptsPerMinCounter > totalCompilesPerMinute) {
-            scriptsPerMinCounter = totalCompilesPerMinute;
+                " operation [" + scriptContext.getKey() + "] and lang [" + lang + "] are not supported");
         }
 
-        // If there is enough tokens in the bucket, allow the request and decrease the tokens by 1
-        if (scriptsPerMinCounter >= 1) {
-            scriptsPerMinCounter -= 1.0;
-        } else {
-            // Otherwise reject the request
-            throw new CircuitBreakingException("[script] Too many dynamic script compilations within one minute, max: [" +
-                            totalCompilesPerMinute + "/min]; please use on-disk, indexed, or scripts with parameters instead; " +
-                            "this limit can be changed by the [" + SCRIPT_MAX_COMPILATIONS_PER_MINUTE.getKey() + "] setting");
-        }
-    }
+        ScriptEngineService scriptEngineService = getScriptEngineServiceForLang(lang);
 
-    /**
-     * Compiles a script straight-away, or returns the previously compiled and cached script,
-     * without checking if it can be executed based on settings.
-     */
-    CompiledScript compileInternal(Script script, Map<String, String> params) {
-        if (script == null) {
-            throw new IllegalArgumentException("The parameter script (Script) must not be null.");
+        if (canExecuteScript(lang, type, scriptContext) == false) {
+            throw new IllegalStateException("scripts of type [" + script.getType() + "]," +
+                " operation [" + scriptContext.getKey() + "] and lang [" + lang + "] are disabled");
         }
 
-        String lang = script.getLang();
-        ScriptType type = script.getType();
-        //script.getScript() could return either a name or code for a script,
-        //but we check for a file script name first and an indexed script name second
-        String name = script.getScript();
-
         if (logger.isTraceEnabled()) {
-            logger.trace("Compiling lang: [{}] type: [{}] script: {}", lang, type, name);
+            logger.trace("compiling lang: [{}] type: [{}] script: {}", lang, type, idOrCode);
         }
 
-        ScriptEngineService scriptEngineService = getScriptEngineServiceForLang(lang);
-
         if (type == ScriptType.FILE) {
-            CacheKey cacheKey = new CacheKey(scriptEngineService, name, null, params);
-            //On disk scripts will be loaded into the staticCache by the listener
+            CacheKey cacheKey = new CacheKey(lang, idOrCode, options);
             CompiledScript compiledScript = staticCache.get(cacheKey);
 
             if (compiledScript == null) {
-                throw new IllegalArgumentException("Unable to find on disk file script [" + name + "] using lang [" + lang + "]");
+                throw new IllegalArgumentException("unable to find file script [" + idOrCode + "] using lang [" + lang + "]");
             }
 
             return compiledScript;
         }
 
-        //script.getScript() will be code if the script type is inline
-        String code = script.getScript();
-
-        if (type == ScriptType.STORED) {
-            //The look up for an indexed script must be done every time in case
-            //the script has been updated in the index since the last look up.
-            final IndexedScript indexedScript = new IndexedScript(lang, name);
-            name = indexedScript.id;
-            code = getScriptFromClusterState(indexedScript.lang, indexedScript.id);
-        }
-
-        CacheKey cacheKey = new CacheKey(scriptEngineService, type == ScriptType.INLINE ? null : name, code, params);
+        CacheKey cacheKey = new CacheKey(lang, idOrCode, options);
         CompiledScript compiledScript = cache.get(cacheKey);
 
         if (compiledScript != null) {
@@ -330,18 +299,17 @@ CompiledScript compileInternal(Script script, Map<String, String> params) {
 
                     // but give the script engine the chance to be better, give it separate name + source code
                     // for the inline case, then its anonymous: null.
-                    String actualName = (type == ScriptType.INLINE) ? null : name;
                     if (logger.isTraceEnabled()) {
-                        logger.trace("compiling script, type: [{}], lang: [{}], params: [{}]", type, lang, params);
+                        logger.trace("compiling script, type: [{}], lang: [{}], options: [{}]", type, lang, options);
                     }
                     // Check whether too many compilations have happened
                     checkCompilationLimit();
-                    compiledScript = new CompiledScript(type, name, lang, scriptEngineService.compile(actualName, code, params));
+                    compiledScript = new CompiledScript(type, id, lang, scriptEngineService.compile(id, idOrCode, options));
                 } catch (ScriptException good) {
                     // TODO: remove this try-catch completely, when all script engines have good exceptions!
                     throw good; // its already good
                 } catch (Exception exception) {
-                    throw new GeneralScriptException("Failed to compile " + type + " script [" + name + "] using lang [" + lang + "]", exception);
+                    throw new GeneralScriptException("Failed to compile " + type + " script [" + id + "] using lang [" + lang + "]", exception);
                 }
 
                 // Since the cache key is the script content itself we don't need to
@@ -354,65 +322,109 @@ CompiledScript compileInternal(Script script, Map<String, String> params) {
         }
     }
 
-    private String validateScriptLanguage(String scriptLang) {
-        Objects.requireNonNull(scriptLang);
-        if (scriptEnginesByLang.containsKey(scriptLang) == false) {
-            throw new IllegalArgumentException("script_lang not supported [" + scriptLang + "]");
+    /** Compiles a template. Note this will be moved to a separate TemplateService in the future. */
+    public CompiledTemplate compileTemplate(Script script, ScriptContext scriptContext) {
+        CompiledScript compiledScript = compile(script, scriptContext);
+        return params -> (BytesReference)executable(compiledScript, params).run();
+    }
+
+    /**
+     * Check whether there have been too many compilations within the last minute, throwing a circuit breaking exception if so.
+     * This is a variant of the token bucket algorithm: https://en.wikipedia.org/wiki/Token_bucket
+     *
+     * It can be thought of as a bucket with water, every time the bucket is checked, water is added proportional to the amount of time that
+     * elapsed since the last time it was checked. If there is enough water, some is removed and the request is allowed. If there is not
+     * enough water the request is denied. Just like a normal bucket, if water is added that overflows the bucket, the extra water/capacity
+     * is discarded - there can never be more water in the bucket than the size of the bucket.
+     */
+    void checkCompilationLimit() {
+        long now = System.nanoTime();
+        long timePassed = now - lastInlineCompileTime;
+        lastInlineCompileTime = now;
+
+        scriptsPerMinCounter += (timePassed) * compilesAllowedPerNano;
+
+        // It's been over the time limit anyway, readjust the bucket to be level
+        if (scriptsPerMinCounter > totalCompilesPerMinute) {
+            scriptsPerMinCounter = totalCompilesPerMinute;
         }
-        return scriptLang;
+
+        // If there is enough tokens in the bucket, allow the request and decrease the tokens by 1
+        if (scriptsPerMinCounter >= 1) {
+            scriptsPerMinCounter -= 1.0;
+        } else {
+            // Otherwise reject the request
+            throw new CircuitBreakingException("[script] Too many dynamic script compilations within one minute, max: [" +
+                            totalCompilesPerMinute + "/min]; please use on-disk, indexed, or scripts with parameters instead; " +
+                            "this limit can be changed by the [" + SCRIPT_MAX_COMPILATIONS_PER_MINUTE.getKey() + "] setting");
+        }
+    }
+
+    public boolean isLangSupported(String lang) {
+        Objects.requireNonNull(lang);
+
+        return scriptEnginesByLang.containsKey(lang);
     }
 
-    String getScriptFromClusterState(String scriptLang, String id) {
-        scriptLang = validateScriptLanguage(scriptLang);
+    StoredScriptSource getScriptFromClusterState(String id, String lang) {
+        if (lang != null && isLangSupported(lang) == false) {
+            throw new IllegalArgumentException("unable to get stored script with unsupported lang [" + lang + "]");
+        }
+
         ScriptMetaData scriptMetadata = clusterState.metaData().custom(ScriptMetaData.TYPE);
+
         if (scriptMetadata == null) {
-            throw new ResourceNotFoundException("Unable to find script [" + scriptLang + "/" + id + "] in cluster state");
+            throw new ResourceNotFoundException("unable to find script [" + id + "]" +
+                (lang == null ? "" : " using lang [" + lang + "]") + " in cluster state");
         }
 
-        String script = scriptMetadata.getScript(scriptLang, id);
-        if (script == null) {
-            throw new ResourceNotFoundException("Unable to find script [" + scriptLang + "/" + id + "] in cluster state");
+        StoredScriptSource source = scriptMetadata.getStoredScript(id, lang);
+
+        if (source == null) {
+            throw new ResourceNotFoundException("unable to find script [" + id + "]" +
+                (lang == null ? "" : " using lang [" + lang + "]") + " in cluster state");
         }
-        return script;
+
+        return source;
     }
 
-    void validateStoredScript(String id, String scriptLang, BytesReference scriptBytes) {
-        validateScriptSize(id, scriptBytes.length());
-        String script = ScriptMetaData.parseStoredScript(scriptBytes);
-        if (Strings.hasLength(scriptBytes)) {
-            //Just try and compile it
-            try {
-                ScriptEngineService scriptEngineService = getScriptEngineServiceForLang(scriptLang);
-                //we don't know yet what the script will be used for, but if all of the operations for this lang with
-                //indexed scripts are disabled, it makes no sense to even compile it.
-                if (isAnyScriptContextEnabled(scriptLang, ScriptType.STORED)) {
-                    Object compiled = scriptEngineService.compile(id, script, Collections.emptyMap());
-                    if (compiled == null) {
-                        throw new IllegalArgumentException("Unable to parse [" + script + "] lang [" + scriptLang +
-                                "] (ScriptService.compile returned null)");
-                    }
-                } else {
-                    logger.warn(
-                            "skipping compile of script [{}], lang [{}] as all scripted operations are disabled for indexed scripts",
-                            script, scriptLang);
+    public void putStoredScript(ClusterService clusterService, PutStoredScriptRequest request,
+                                ActionListener<PutStoredScriptResponse> listener) {
+        int max = SCRIPT_MAX_SIZE_IN_BYTES.get(settings);
+
+        if (request.content().length() > max) {
+            throw new IllegalArgumentException("exceeded max allowed stored script size in bytes [" + max + "] with size [" +
+                request.content().length() + "] for script [" + request.id() + "]");
+        }
+
+        StoredScriptSource source = StoredScriptSource.parse(request.lang(), request.content(), request.xContentType());
+
+        if (isLangSupported(source.getLang()) == false) {
+            throw new IllegalArgumentException("unable to put stored script with unsupported lang [" + source.getLang() + "]");
+        }
+
+        try {
+            ScriptEngineService scriptEngineService = getScriptEngineServiceForLang(source.getLang());
+
+            if (isAnyScriptContextEnabled(source.getLang(), ScriptType.STORED)) {
+                Object compiled = scriptEngineService.compile(request.id(), source.getSource(), Collections.emptyMap());
+
+                if (compiled == null) {
+                    throw new IllegalArgumentException("failed to parse/compile stored script [" + request.id() + "]" +
+                        (source.getSource() == null ? "" : " using code [" + source.getSource() + "]"));
                 }
-            } catch (ScriptException good) {
-                // TODO: remove this when all script engines have good exceptions!
-                throw good; // its already good!
-            } catch (Exception e) {
-                throw new IllegalArgumentException("Unable to parse [" + script +
-                        "] lang [" + scriptLang + "]", e);
+            } else {
+                throw new IllegalArgumentException(
+                    "cannot put stored script [" + request.id() + "], stored scripts cannot be run under any context");
             }
-        } else {
-            throw new IllegalArgumentException("Unable to find script in : " + scriptBytes.utf8ToString());
+        } catch (ScriptException good) {
+            throw good;
+        } catch (Exception exception) {
+            throw new IllegalArgumentException("failed to parse/compile stored script [" + request.id() + "]", exception);
         }
-    }
 
-    public void storeScript(ClusterService clusterService, PutStoredScriptRequest request, ActionListener<PutStoredScriptResponse> listener) {
-        String scriptLang = validateScriptLanguage(request.scriptLang());
-        //verify that the script compiles
-        validateStoredScript(request.id(), scriptLang, request.script());
-        clusterService.submitStateUpdateTask("put-script-" + request.id(), new AckedClusterStateUpdateTask<PutStoredScriptResponse>(request, listener) {
+        clusterService.submitStateUpdateTask("put-script-" + request.id(),
+            new AckedClusterStateUpdateTask<PutStoredScriptResponse>(request, listener) {
 
             @Override
             protected PutStoredScriptResponse newResponse(boolean acknowledged) {
@@ -421,23 +433,23 @@ protected PutStoredScriptResponse newResponse(boolean acknowledged) {
 
             @Override
             public ClusterState execute(ClusterState currentState) throws Exception {
-                return innerStoreScript(currentState, scriptLang, request);
+                ScriptMetaData smd = currentState.metaData().custom(ScriptMetaData.TYPE);
+                smd = ScriptMetaData.putStoredScript(smd, request.id(), source);
+                MetaData.Builder mdb = MetaData.builder(currentState.getMetaData()).putCustom(ScriptMetaData.TYPE, smd);
+
+                return ClusterState.builder(currentState).metaData(mdb).build();
             }
         });
     }
 
-    static ClusterState innerStoreScript(ClusterState currentState, String validatedScriptLang, PutStoredScriptRequest request) {
-        ScriptMetaData scriptMetadata = currentState.metaData().custom(ScriptMetaData.TYPE);
-        ScriptMetaData.Builder scriptMetadataBuilder = new ScriptMetaData.Builder(scriptMetadata);
-        scriptMetadataBuilder.storeScript(validatedScriptLang, request.id(), request.script());
-        MetaData.Builder metaDataBuilder = MetaData.builder(currentState.getMetaData())
-                .putCustom(ScriptMetaData.TYPE, scriptMetadataBuilder.build());
-        return ClusterState.builder(currentState).metaData(metaDataBuilder).build();
-    }
+    public void deleteStoredScript(ClusterService clusterService, DeleteStoredScriptRequest request,
+                                   ActionListener<DeleteStoredScriptResponse> listener) {
+        if (request.lang() != null && isLangSupported(request.lang()) == false) {
+            throw new IllegalArgumentException("unable to delete stored script with unsupported lang [" + request.lang() +"]");
+        }
 
-    public void deleteStoredScript(ClusterService clusterService, DeleteStoredScriptRequest request, ActionListener<DeleteStoredScriptResponse> listener) {
-        String scriptLang = validateScriptLanguage(request.scriptLang());
-        clusterService.submitStateUpdateTask("delete-script-" + request.id(), new AckedClusterStateUpdateTask<DeleteStoredScriptResponse>(request, listener) {
+        clusterService.submitStateUpdateTask("delete-script-" + request.id(),
+            new AckedClusterStateUpdateTask<DeleteStoredScriptResponse>(request, listener) {
 
             @Override
             protected DeleteStoredScriptResponse newResponse(boolean acknowledged) {
@@ -446,49 +458,46 @@ protected DeleteStoredScriptResponse newResponse(boolean acknowledged) {
 
             @Override
             public ClusterState execute(ClusterState currentState) throws Exception {
-                return innerDeleteScript(currentState, scriptLang, request);
+                ScriptMetaData smd = currentState.metaData().custom(ScriptMetaData.TYPE);
+                smd = ScriptMetaData.deleteStoredScript(smd, request.id(), request.lang());
+                MetaData.Builder mdb = MetaData.builder(currentState.getMetaData()).putCustom(ScriptMetaData.TYPE, smd);
+
+                return ClusterState.builder(currentState).metaData(mdb).build();
             }
         });
     }
 
-    static ClusterState innerDeleteScript(ClusterState currentState, String validatedLang, DeleteStoredScriptRequest request) {
-        ScriptMetaData scriptMetadata = currentState.metaData().custom(ScriptMetaData.TYPE);
-        ScriptMetaData.Builder scriptMetadataBuilder = new ScriptMetaData.Builder(scriptMetadata);
-        scriptMetadataBuilder.deleteScript(validatedLang, request.id());
-        MetaData.Builder metaDataBuilder = MetaData.builder(currentState.getMetaData())
-                .putCustom(ScriptMetaData.TYPE, scriptMetadataBuilder.build());
-        return ClusterState.builder(currentState).metaData(metaDataBuilder).build();
-    }
-
-    public String getStoredScript(ClusterState state, GetStoredScriptRequest request) {
+    public StoredScriptSource getStoredScript(ClusterState state, GetStoredScriptRequest request) {
         ScriptMetaData scriptMetadata = state.metaData().custom(ScriptMetaData.TYPE);
+
         if (scriptMetadata != null) {
-            return scriptMetadata.getScript(request.lang(), request.id());
+            return scriptMetadata.getStoredScript(request.id(), request.lang());
         } else {
             return null;
         }
     }
 
     /**
-     * Compiles (or retrieves from cache) and executes the provided script
+     * Executes a previously compiled script provided as an argument
      */
-    public ExecutableScript executable(Script script, ScriptContext scriptContext, Map<String, String> params) {
-        return executable(compile(script, scriptContext, params), script.getParams());
+    public ExecutableScript executable(CompiledScript compiledScript, Map<String, Object> params) {
+        return getScriptEngineServiceForLang(compiledScript.lang()).executable(compiledScript, params);
     }
 
     /**
-     * Executes a previously compiled script provided as an argument
+     * Compiles (or retrieves from cache) and executes the provided search script
      */
-    public ExecutableScript executable(CompiledScript compiledScript, Map<String, Object> vars) {
-        return getScriptEngineServiceForLang(compiledScript.lang()).executable(compiledScript, vars);
+    public SearchScript search(SearchLookup lookup, Script script, ScriptContext scriptContext) {
+        CompiledScript compiledScript = compile(script, scriptContext);
+        return search(lookup, compiledScript, script.getParams());
     }
 
     /**
-     * Compiles (or retrieves from cache) and executes the provided search script
+     * Binds provided parameters to a compiled script returning a
+     * {@link SearchScript} ready for execution
      */
-    public SearchScript search(SearchLookup lookup, Script script, ScriptContext scriptContext, Map<String, String> params) {
-        CompiledScript compiledScript = compile(script, scriptContext, params);
-        return getScriptEngineServiceForLang(compiledScript.lang()).search(compiledScript, lookup, script.getParams());
+    public SearchScript search(SearchLookup lookup, CompiledScript compiledScript,  Map<String, Object> params) {
+        return getScriptEngineServiceForLang(compiledScript.lang()).search(compiledScript, lookup, params);
     }
 
     private boolean isAnyScriptContextEnabled(String lang, ScriptType scriptType) {
@@ -502,7 +511,7 @@ private boolean isAnyScriptContextEnabled(String lang, ScriptType scriptType) {
 
     private boolean canExecuteScript(String lang, ScriptType scriptType, ScriptContext scriptContext) {
         assert lang != null;
-        if (scriptContextRegistry.isSupportedContext(scriptContext) == false) {
+        if (scriptContextRegistry.isSupportedContext(scriptContext.getKey()) == false) {
             throw new IllegalArgumentException("script context [" + scriptContext.getKey() + "] not supported");
         }
         return scriptModes.getScriptEnabled(lang, scriptType, scriptContext);
@@ -512,18 +521,6 @@ public ScriptStats stats() {
         return scriptMetrics.stats();
     }
 
-    private void validateScriptSize(String identifier, int scriptSizeInBytes) {
-        int allowedScriptSizeInBytes = SCRIPT_MAX_SIZE_IN_BYTES.get(settings);
-        if (scriptSizeInBytes > allowedScriptSizeInBytes) {
-            String message = LoggerMessageFormat.format(
-                    "Limit of script size in bytes [{}] has been exceeded for script [{}] with size [{}]",
-                    allowedScriptSizeInBytes,
-                    identifier,
-                    scriptSizeInBytes);
-            throw new IllegalArgumentException(message);
-        }
-    }
-
     @Override
     public void clusterChanged(ClusterChangedEvent event) {
         clusterState = event.state();
@@ -545,6 +542,7 @@ public void onRemoval(RemovalNotification<CacheKey, CompiledScript> notification
     }
 
     private class ScriptChangesListener implements FileChangesListener {
+        private boolean deprecationEmitted = false;
 
         private Tuple<String, String> getScriptNameExt(Path file) {
             Path scriptPath = scriptsDirectory.relativize(file);
@@ -577,6 +575,11 @@ public void onFileInit(Path file) {
             if (engineService == null) {
                 logger.warn("No script engine found for [{}]", scriptNameExt.v2());
             } else {
+                if (deprecationEmitted == false) {
+                    deprecationLogger.deprecated("File scripts are deprecated. Use stored or inline scripts instead.");
+                    deprecationEmitted = true;
+                }
+
                 try {
                     //we don't know yet what the script will be used for, but if all of the operations for this lang
                     // with file scripts are disabled, it makes no sense to even compile it and cache it.
@@ -584,17 +587,33 @@ public void onFileInit(Path file) {
                         logger.info("compiling script file [{}]", file.toAbsolutePath());
                         try (InputStreamReader reader = new InputStreamReader(Files.newInputStream(file), StandardCharsets.UTF_8)) {
                             String script = Streams.copyToString(reader);
-                            String name = scriptNameExt.v1();
-                            CacheKey cacheKey = new CacheKey(engineService, name, null, Collections.emptyMap());
+                            String id = scriptNameExt.v1();
+                            CacheKey cacheKey = new CacheKey(engineService.getType(), id, null);
                             // pass the actual file name to the compiler (for script engines that care about this)
                             Object executable = engineService.compile(file.getFileName().toString(), script, Collections.emptyMap());
-                            CompiledScript compiledScript = new CompiledScript(ScriptType.FILE, name, engineService.getType(), executable);
+                            CompiledScript compiledScript = new CompiledScript(ScriptType.FILE, id, engineService.getType(), executable);
                             staticCache.put(cacheKey, compiledScript);
                             scriptMetrics.onCompilation();
                         }
                     } else {
                         logger.warn("skipping compile of script file [{}] as all scripted operations are disabled for file scripts", file.toAbsolutePath());
                     }
+                } catch (ScriptException e) {
+                    try (XContentBuilder builder = JsonXContent.contentBuilder()) {
+                        builder.prettyPrint();
+                        builder.startObject();
+                        ElasticsearchException.generateThrowableXContent(builder, ToXContent.EMPTY_PARAMS, e);
+                        builder.endObject();
+                        logger.warn("failed to load/compile script [{}]: {}", scriptNameExt.v1(), builder.string());
+                    } catch (IOException ioe) {
+                        ioe.addSuppressed(e);
+                        logger.warn((Supplier<?>) () -> new ParameterizedMessage(
+                                "failed to log an appropriate warning after failing to load/compile script [{}]", scriptNameExt.v1()), ioe);
+                    }
+                    /* Log at the whole exception at the debug level as well just in case the stack trace is important. That way you can
+                     * turn on the stack trace if you need it. */
+                    logger.debug((Supplier<?>) () -> new ParameterizedMessage("failed to load/compile script [{}]. full exception:",
+                            scriptNameExt.v1()), e);
                 } catch (Exception e) {
                     logger.warn((Supplier<?>) () -> new ParameterizedMessage("failed to load/compile script [{}]", scriptNameExt.v1()), e);
                 }
@@ -613,7 +632,7 @@ public void onFileDeleted(Path file) {
                 ScriptEngineService engineService = getScriptEngineServiceForFileExt(scriptNameExt.v2());
                 assert engineService != null;
                 logger.info("removing script file [{}]", file.toAbsolutePath());
-                staticCache.remove(new CacheKey(engineService, scriptNameExt.v1(), null, Collections.emptyMap()));
+                staticCache.remove(new CacheKey(engineService.getType(), scriptNameExt.v1(), null));
             }
         }
 
@@ -624,79 +643,15 @@ public void onFileChanged(Path file) {
 
     }
 
-    /**
-     * The type of a script, more specifically where it gets loaded from:
-     * - provided dynamically at request time
-     * - loaded from an index
-     * - loaded from file
-     */
-    public enum ScriptType {
-
-        INLINE(0, "inline", "inline", false),
-        STORED(1, "id", "stored", false),
-        FILE(2, "file", "file", true);
-
-        private final int val;
-        private final ParseField parseField;
-        private final String scriptType;
-        private final boolean defaultScriptEnabled;
-
-        public static ScriptType readFrom(StreamInput in) throws IOException {
-            int scriptTypeVal = in.readVInt();
-            for (ScriptType type : values()) {
-                if (type.val == scriptTypeVal) {
-                    return type;
-                }
-            }
-            throw new IllegalArgumentException("Unexpected value read for ScriptType got [" + scriptTypeVal + "] expected one of ["
-                    + INLINE.val + "," + FILE.val + "," + STORED.val + "]");
-        }
-
-        public static void writeTo(ScriptType scriptType, StreamOutput out) throws IOException{
-            if (scriptType != null) {
-                out.writeVInt(scriptType.val);
-            } else {
-                out.writeVInt(INLINE.val); //Default to inline
-            }
-        }
-
-        ScriptType(int val, String name, String scriptType, boolean defaultScriptEnabled) {
-            this.val = val;
-            this.parseField = new ParseField(name);
-            this.scriptType = scriptType;
-            this.defaultScriptEnabled = defaultScriptEnabled;
-        }
-
-        public ParseField getParseField() {
-            return parseField;
-        }
-
-        public boolean getDefaultScriptEnabled() {
-            return defaultScriptEnabled;
-        }
-
-        public String getScriptType() {
-            return scriptType;
-        }
-
-        @Override
-        public String toString() {
-            return name().toLowerCase(Locale.ROOT);
-        }
-
-    }
-
     private static final class CacheKey {
         final String lang;
-        final String name;
-        final String code;
-        final Map<String, String> params;
+        final String idOrCode;
+        final Map<String, String> options;
 
-        private CacheKey(final ScriptEngineService service, final String name, final String code, final Map<String, String> params) {
-            this.lang = service.getType();
-            this.name = name;
-            this.code = code;
-            this.params = params;
+        private CacheKey(String lang, String idOrCode, Map<String, String> options) {
+            this.lang = lang;
+            this.idOrCode = idOrCode;
+            this.options = options;
         }
 
         @Override
@@ -706,44 +661,18 @@ public boolean equals(Object o) {
 
             CacheKey cacheKey = (CacheKey)o;
 
-            if (!lang.equals(cacheKey.lang)) return false;
-            if (name != null ? !name.equals(cacheKey.name) : cacheKey.name != null) return false;
-            if (code != null ? !code.equals(cacheKey.code) : cacheKey.code != null) return false;
-            return params.equals(cacheKey.params);
+            if (lang != null ? !lang.equals(cacheKey.lang) : cacheKey.lang != null) return false;
+            if (!idOrCode.equals(cacheKey.idOrCode)) return false;
+            return options != null ? options.equals(cacheKey.options) : cacheKey.options == null;
 
         }
 
         @Override
         public int hashCode() {
-            int result = lang.hashCode();
-            result = 31 * result + (name != null ? name.hashCode() : 0);
-            result = 31 * result + (code != null ? code.hashCode() : 0);
-            result = 31 * result + params.hashCode();
+            int result = lang != null ? lang.hashCode() : 0;
+            result = 31 * result + idOrCode.hashCode();
+            result = 31 * result + (options != null ? options.hashCode() : 0);
             return result;
         }
     }
-
-
-    private static class IndexedScript {
-        private final String lang;
-        private final String id;
-
-        IndexedScript(String lang, String script) {
-            this.lang = lang;
-            final String[] parts = script.split("/");
-            if (parts.length == 1) {
-                this.id = script;
-            } else {
-                if (parts.length != 3) {
-                    throw new IllegalArgumentException("Illegal index script format [" + script + "]" +
-                            " should be /lang/id");
-                } else {
-                    if (!parts[1].equals(this.lang)) {
-                        throw new IllegalStateException("Conflicting script language, found [" + parts[1] + "] expected + ["+ this.lang + "]");
-                    }
-                    this.id = parts[2];
-                }
-            }
-        }
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptSettings.java b/core/src/main/java/org/elasticsearch/script/ScriptSettings.java
index 1cb2b35624544..a897fdc7be86e 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptSettings.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptSettings.java
@@ -25,6 +25,7 @@
 
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.EnumMap;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -43,15 +44,16 @@ public class ScriptSettings {
     @Deprecated
     public static final String LEGACY_SCRIPT_SETTING = "script.legacy.default_lang";
 
-    private static final Map<ScriptService.ScriptType, Setting<Boolean>> SCRIPT_TYPE_SETTING_MAP;
+    private static final Map<ScriptType, Setting<Boolean>> SCRIPT_TYPE_SETTING_MAP;
 
     static {
-        Map<ScriptService.ScriptType, Setting<Boolean>> scriptTypeSettingMap = new HashMap<>();
-        for (ScriptService.ScriptType scriptType : ScriptService.ScriptType.values()) {
+        Map<ScriptType, Setting<Boolean>> scriptTypeSettingMap = new EnumMap<>(ScriptType.class);
+        for (ScriptType scriptType : ScriptType.values()) {
             scriptTypeSettingMap.put(scriptType, Setting.boolSetting(
                 ScriptModes.sourceKey(scriptType),
-                scriptType.getDefaultScriptEnabled(),
-                Property.NodeScope));
+                scriptType.isDefaultEnabled(),
+                Property.NodeScope,
+                Property.Deprecated));
         }
         SCRIPT_TYPE_SETTING_MAP = Collections.unmodifiableMap(scriptTypeSettingMap);
     }
@@ -79,12 +81,12 @@ private static Map<ScriptContext, Setting<Boolean>> contextSettings(ScriptContex
         Map<ScriptContext, Setting<Boolean>> scriptContextSettingMap = new HashMap<>();
         for (ScriptContext scriptContext : scriptContextRegistry.scriptContexts()) {
             scriptContextSettingMap.put(scriptContext,
-                    Setting.boolSetting(ScriptModes.operationKey(scriptContext), false, Property.NodeScope));
+                    Setting.boolSetting(ScriptModes.operationKey(scriptContext), false, Property.NodeScope, Property.Deprecated));
         }
         return scriptContextSettingMap;
     }
 
-    private static List<Setting<Boolean>> languageSettings(Map<ScriptService.ScriptType, Setting<Boolean>> scriptTypeSettingMap,
+    private static List<Setting<Boolean>> languageSettings(Map<ScriptType, Setting<Boolean>> scriptTypeSettingMap,
                                                               Map<ScriptContext, Setting<Boolean>> scriptContextSettingMap,
                                                               ScriptEngineRegistry scriptEngineRegistry,
                                                               ScriptContextRegistry scriptContextRegistry) {
@@ -96,20 +98,20 @@ private static List<Setting<Boolean>> languageSettings(Map<ScriptService.ScriptT
                 continue;
             }
             final String language = scriptEngineRegistry.getLanguage(scriptEngineService);
-            for (final ScriptService.ScriptType scriptType : ScriptService.ScriptType.values()) {
+            for (final ScriptType scriptType : ScriptType.values()) {
                 // Top level, like "script.engine.groovy.inline"
                 final boolean defaultNonFileScriptMode = scriptEngineRegistry.getDefaultInlineScriptEnableds().get(language);
                 boolean defaultLangAndType = defaultNonFileScriptMode;
                 // Files are treated differently because they are never default-deny
-                if (ScriptService.ScriptType.FILE == scriptType) {
-                    defaultLangAndType = ScriptService.ScriptType.FILE.getDefaultScriptEnabled();
+                if (ScriptType.FILE == scriptType) {
+                    defaultLangAndType = ScriptType.FILE.isDefaultEnabled();
                 }
                 final boolean defaultIfNothingSet = defaultLangAndType;
 
                 Function<Settings, String> defaultLangAndTypeFn = settings -> {
                     final Setting<Boolean> globalTypeSetting = scriptTypeSettingMap.get(scriptType);
                     final Setting<Boolean> langAndTypeSetting = Setting.boolSetting(ScriptModes.getGlobalKey(language, scriptType),
-                            defaultIfNothingSet, Property.NodeScope);
+                            defaultIfNothingSet, Property.NodeScope, Property.Deprecated);
 
                     if (langAndTypeSetting.exists(settings)) {
                         // fine-grained e.g. script.engine.groovy.inline
@@ -124,7 +126,7 @@ private static List<Setting<Boolean>> languageSettings(Map<ScriptService.ScriptT
 
                 // Setting for something like "script.engine.groovy.inline"
                 final Setting<Boolean> langAndTypeSetting = Setting.boolSetting(ScriptModes.getGlobalKey(language, scriptType),
-                        defaultLangAndTypeFn, Property.NodeScope);
+                        defaultLangAndTypeFn, Property.NodeScope, Property.Deprecated);
                 scriptModeSettings.add(langAndTypeSetting);
 
                 for (ScriptContext scriptContext : scriptContextRegistry.scriptContexts()) {
@@ -135,7 +137,7 @@ private static List<Setting<Boolean>> languageSettings(Map<ScriptService.ScriptT
                         final Setting<Boolean> globalOpSetting = scriptContextSettingMap.get(scriptContext);
                         final Setting<Boolean> globalTypeSetting = scriptTypeSettingMap.get(scriptType);
                         final Setting<Boolean> langAndTypeAndContextSetting = Setting.boolSetting(langAndTypeAndContextName,
-                                defaultIfNothingSet, Property.NodeScope);
+                                defaultIfNothingSet, Property.NodeScope, Property.Deprecated);
 
                         // fallback logic for script mode settings
                         if (langAndTypeAndContextSetting.exists(settings)) {
@@ -156,7 +158,8 @@ private static List<Setting<Boolean>> languageSettings(Map<ScriptService.ScriptT
                         }
                     };
                     // The actual setting for finest grained script settings
-                    Setting<Boolean> setting = Setting.boolSetting(langAndTypeAndContextName, defaultSettingFn, Property.NodeScope);
+                    Setting<Boolean> setting =
+                        Setting.boolSetting(langAndTypeAndContextName, defaultSettingFn, Property.NodeScope, Property.Deprecated);
                     scriptModeSettings.add(setting);
                 }
             }
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptType.java b/core/src/main/java/org/elasticsearch/script/ScriptType.java
new file mode 100644
index 0000000000000..119aec469b1aa
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/script/ScriptType.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.script;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+
+import java.io.IOException;
+import java.util.Locale;
+
+/**
+ * ScriptType represents the way a script is stored and retrieved from the {@link ScriptService}.
+ * It's also used to by {@link ScriptSettings} and {@link ScriptModes} to determine whether or not
+ * a {@link Script} is allowed to be executed based on both default and user-defined settings.
+ */
+public enum ScriptType implements Writeable {
+
+    /**
+     * INLINE scripts are specified in numerous queries and compiled on-the-fly.
+     * They will be cached based on the lang and code of the script.
+     * They are turned off by default because most languages are insecure
+     * (Groovy and others), but can be overriden by the specific {@link ScriptEngineService}
+     * if the language is naturally secure (Painless, Mustache, and Expressions).
+     */
+    INLINE ( 0 , new ParseField("source", "inline") , false ),
+
+    /**
+     * STORED scripts are saved as part of the {@link org.elasticsearch.cluster.ClusterState}
+     * based on user requests.  They will be cached when they are first used in a query.
+     * They are turned off by default because most languages are insecure
+     * (Groovy and others), but can be overriden by the specific {@link ScriptEngineService}
+     * if the language is naturally secure (Painless, Mustache, and Expressions).
+     */
+    STORED ( 1 , new ParseField("id", "stored") , false ),
+
+    /**
+     * FILE scripts are loaded from disk either on start-up or on-the-fly depending on
+     * user-defined settings.  They will be compiled and cached as soon as they are loaded
+     * from disk.  They are turned on by default as they should always be safe to execute.
+     */
+    FILE ( 2 , new ParseField("file") , true  );
+
+    /**
+     * Reads an int from the input stream and converts it to a {@link ScriptType}.
+     * @return The ScriptType read from the stream. Throws an {@link IllegalStateException}
+     * if no ScriptType is found based on the id.
+     */
+    public static ScriptType readFrom(StreamInput in) throws IOException {
+        int id = in.readVInt();
+
+        if (FILE.id == id) {
+            return FILE;
+        } else if (STORED.id == id) {
+            return STORED;
+        } else if (INLINE.id == id) {
+            return INLINE;
+        } else {
+            throw new IllegalStateException("Error reading ScriptType id [" + id + "] from stream, expected one of [" +
+                FILE.id + " [" + FILE.parseField.getPreferredName() + "], " +
+                STORED.id + " [" + STORED.parseField.getPreferredName() + "], " +
+                INLINE.id + " [" + INLINE.parseField.getPreferredName() + "]]");
+        }
+    }
+
+    private final int id;
+    private final ParseField parseField;
+    private final boolean defaultEnabled;
+
+    /**
+     * Standard constructor.
+     * @param id A unique identifier for a type that can be read/written to a stream.
+     * @param parseField Specifies the name used to parse input from queries.
+     * @param defaultEnabled Whether or not a {@link ScriptType} can be run by default.
+     */
+    ScriptType(int id, ParseField parseField, boolean defaultEnabled) {
+        this.id = id;
+        this.parseField = parseField;
+        this.defaultEnabled = defaultEnabled;
+    }
+
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeVInt(id);
+    }
+
+    /**
+     * @return The unique id for this {@link ScriptType}.
+     */
+    public int getId() {
+        return id;
+    }
+
+    /**
+     * @return The unique name for this {@link ScriptType} based on the {@link ParseField}.
+     */
+    public String getName() {
+        return name().toLowerCase(Locale.ROOT);
+    }
+
+    /**
+     * @return Specifies the name used to parse input from queries.
+     */
+    public ParseField getParseField() {
+        return parseField;
+    }
+
+    /**
+     * @return Whether or not a {@link ScriptType} can be run by default.  Note
+     * this can be potentially overriden by any {@link ScriptEngineService}.
+     */
+    public boolean isDefaultEnabled() {
+        return defaultEnabled;
+    }
+
+    /**
+     * @return The same as calling {@link #getName()}.
+     */
+    @Override
+    public String toString() {
+        return getName();
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/script/StoredScriptSource.java b/core/src/main/java/org/elasticsearch/script/StoredScriptSource.java
new file mode 100644
index 0000000000000..4c71b05a5082c
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/script/StoredScriptSource.java
@@ -0,0 +1,495 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.script;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptResponse;
+import org.elasticsearch.cluster.AbstractDiffable;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.Diff;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParser.Token;
+import org.elasticsearch.common.xcontent.XContentType;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * {@link StoredScriptSource} represents user-defined parameters for a script
+ * saved in the {@link ClusterState}.
+ */
+public class StoredScriptSource extends AbstractDiffable<StoredScriptSource> implements Writeable, ToXContentObject {
+
+    /**
+     * Standard {@link ParseField} for outer level of stored script source.
+     */
+    public static final ParseField SCRIPT_PARSE_FIELD = new ParseField("script");
+
+    /**
+     * Standard {@link ParseField} for outer level of stored script source.
+     */
+    public static final ParseField TEMPLATE_PARSE_FIELD = new ParseField("template");
+
+    /**
+     * Standard {@link ParseField} for lang on the inner level.
+     */
+    public static final ParseField LANG_PARSE_FIELD = new ParseField("lang");
+
+    /**
+     * Standard {@link ParseField} for source on the inner level.
+     */
+    public static final ParseField SOURCE_PARSE_FIELD = new ParseField("source", "code");
+
+    /**
+     * Standard {@link ParseField} for options on the inner level.
+     */
+    public static final ParseField OPTIONS_PARSE_FIELD = new ParseField("options");
+
+    /**
+     * Helper class used by {@link ObjectParser} to store mutable {@link StoredScriptSource} variables and then
+     * construct an immutable {@link StoredScriptSource} object based on parsed XContent.
+     */
+    private static final class Builder {
+        private String lang;
+        private String source;
+        private Map<String, String> options;
+
+        private Builder() {
+            // This cannot default to an empty map because options are potentially added at multiple points.
+            this.options = new HashMap<>();
+        }
+
+        private void setLang(String lang) {
+            this.lang = lang;
+        }
+
+        /**
+         * Since stored scripts can accept templates rather than just scripts, they must also be able
+         * to handle template parsing, hence the need for custom parsing source.  Templates can
+         * consist of either an {@link String} or a JSON object.  If a JSON object is discovered
+         * then the content type option must also be saved as a compiler option.
+         */
+        private void setSource(XContentParser parser) {
+            try {
+                if (parser.currentToken() == Token.START_OBJECT) {
+                    //this is really for search templates, that need to be converted to json format
+                    XContentBuilder builder = XContentFactory.jsonBuilder();
+                    source = builder.copyCurrentStructure(parser).string();
+                    options.put(Script.CONTENT_TYPE_OPTION, XContentType.JSON.mediaType());
+                } else {
+                    source = parser.text();
+                }
+            } catch (IOException exception) {
+                throw new UncheckedIOException(exception);
+            }
+        }
+
+        /**
+         * Options may have already been added if a template was specified.
+         * Appends the user-defined compiler options with the internal compiler options.
+         */
+        private void setOptions(Map<String, String> options) {
+            this.options.putAll(options);
+        }
+
+        /**
+         * Validates the parameters and creates an {@link StoredScriptSource}.
+         */
+        private StoredScriptSource build() {
+            if (lang == null) {
+                throw new IllegalArgumentException("must specify lang for stored script");
+            } else if (lang.isEmpty()) {
+                throw new IllegalArgumentException("lang cannot be empty");
+            }
+
+            if (source == null) {
+                throw new IllegalArgumentException("must specify source for stored script");
+            } else if (source.isEmpty()) {
+                throw new IllegalArgumentException("source cannot be empty");
+            }
+
+            if (options.size() > 1 || options.size() == 1 && options.get(Script.CONTENT_TYPE_OPTION) == null) {
+                throw new IllegalArgumentException("illegal compiler options [" + options + "] specified");
+            }
+
+            return new StoredScriptSource(lang, source, options);
+        }
+    }
+
+    private static final ObjectParser<Builder, Void> PARSER = new ObjectParser<>("stored script source", Builder::new);
+
+    static {
+        // Defines the fields necessary to parse a Script as XContent using an ObjectParser.
+        PARSER.declareString(Builder::setLang, LANG_PARSE_FIELD);
+        PARSER.declareField(Builder::setSource, parser -> parser, SOURCE_PARSE_FIELD, ValueType.OBJECT_OR_STRING);
+        PARSER.declareField(Builder::setOptions, XContentParser::mapStrings, OPTIONS_PARSE_FIELD, ValueType.OBJECT);
+    }
+
+    /**
+     * This will parse XContent into a {@link StoredScriptSource}.  The following formats can be parsed:
+     *
+     * The simple script format with no compiler options or user-defined params:
+     *
+     * Example:
+     * {@code
+     * {"script": "return Math.log(doc.popularity) * 100;"}
+     * }
+     *
+     * The above format requires the lang to be specified using the deprecated stored script namespace
+     * (as a url parameter during a put request).  See {@link ScriptMetaData} for more information about
+     * the stored script namespaces.
+     *
+     * The complex script format using the new stored script namespace
+     * where lang and source are required but options is optional:
+     *
+     * {@code
+     * {
+     *     "script" : {
+     *         "lang" : "<lang>",
+     *         "source" : "<source>",
+     *         "options" : {
+     *             "option0" : "<option0>",
+     *             "option1" : "<option1>",
+     *             ...
+     *         }
+     *     }
+     * }
+     * }
+     *
+     * Example:
+     * {@code
+     * {
+     *     "script": {
+     *         "lang" : "painless",
+     *         "source" : "return Math.log(doc.popularity) * params.multiplier"
+     *     }
+     * }
+     * }
+     *
+     * The use of "source" may also be substituted with "code" for backcompat with 5.3 to 5.5 format. For example:
+     *
+     * {@code
+     * {
+     *     "script" : {
+     *         "lang" : "<lang>",
+     *         "code" : "<source>",
+     *         "options" : {
+     *             "option0" : "<option0>",
+     *             "option1" : "<option1>",
+     *             ...
+     *         }
+     *     }
+     * }
+     * }
+     *
+     * The simple template format:
+     *
+     * {@code
+     * {
+     *     "query" : ...
+     * }
+     * }
+     *
+     * The complex template format:
+     *
+     * {@code
+     * {
+     *     "template": {
+     *         "query" : ...
+     *     }
+     * }
+     * }
+     *
+     * Note that templates can be handled as both strings and complex JSON objects.
+     * Also templates may be part of the 'source' parameter in a script.  The Parser
+     * can handle this case as well.
+     *
+     * @param lang    An optional parameter to allow for use of the deprecated stored
+     *                script namespace.  This will be used to specify the language
+     *                coming in as a url parameter from a request or for stored templates.
+     * @param content The content from the request to be parsed as described above.
+     * @return        The parsed {@link StoredScriptSource}.
+     */
+    public static StoredScriptSource parse(String lang, BytesReference content, XContentType xContentType) {
+        try (XContentParser parser = xContentType.xContent().createParser(NamedXContentRegistry.EMPTY, content)) {
+            Token token = parser.nextToken();
+
+            if (token != Token.START_OBJECT) {
+                throw new ParsingException(parser.getTokenLocation(), "unexpected token [" + token + "], expected [{]");
+            }
+
+            token = parser.nextToken();
+
+            if (token != Token.FIELD_NAME) {
+                throw new ParsingException(parser.getTokenLocation(), "unexpected token [" + token + ", expected [" +
+                    SCRIPT_PARSE_FIELD.getPreferredName() + ", " + TEMPLATE_PARSE_FIELD.getPreferredName());
+            }
+
+            String name = parser.currentName();
+
+            if (SCRIPT_PARSE_FIELD.getPreferredName().equals(name)) {
+                token = parser.nextToken();
+
+                if (token == Token.VALUE_STRING) {
+                    if (lang == null) {
+                        throw new IllegalArgumentException(
+                            "must specify lang as a url parameter when using the deprecated stored script namespace");
+                    }
+
+                    return new StoredScriptSource(lang, parser.text(), Collections.emptyMap());
+                } else if (token == Token.START_OBJECT) {
+                    if (lang == null) {
+                        return PARSER.apply(parser, null).build();
+                    } else {
+                        //this is really for search templates, that need to be converted to json format
+                        try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
+                            builder.copyCurrentStructure(parser);
+                            return new StoredScriptSource(lang, builder.string(), Collections.emptyMap());
+                        }
+                    }
+
+                } else {
+                    throw new ParsingException(parser.getTokenLocation(), "unexpected token [" + token + "], expected [{, <source>]");
+                }
+            } else {
+                if (lang == null) {
+                    throw new IllegalArgumentException("unexpected stored script format");
+                }
+
+                if (TEMPLATE_PARSE_FIELD.getPreferredName().equals(name)) {
+                    token = parser.nextToken();
+
+                    if (token == Token.VALUE_STRING) {
+                        return new StoredScriptSource(lang, parser.text(), Collections.emptyMap());
+                    }
+                }
+
+                try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
+                    if (token != Token.START_OBJECT) {
+                        builder.startObject();
+                        builder.copyCurrentStructure(parser);
+                        builder.endObject();
+                    } else {
+                        builder.copyCurrentStructure(parser);
+                    }
+
+                    return new StoredScriptSource(lang, builder.string(), Collections.emptyMap());
+                }
+            }
+        } catch (IOException ioe) {
+            throw new UncheckedIOException(ioe);
+        }
+    }
+
+    /**
+     * This will parse XContent into a {@link StoredScriptSource}. The following format is what will be parsed:
+     *
+     * {@code
+     * {
+     *     "script" : {
+     *         "lang" : "<lang>",
+     *         "source" : "<source>",
+     *         "options" : {
+     *             "option0" : "<option0>",
+     *             "option1" : "<option1>",
+     *             ...
+     *         }
+     *     }
+     * }
+     * }
+     *
+     * Note that the "source" parameter can also handle template parsing including from
+     * a complex JSON object.
+     */
+    public static StoredScriptSource fromXContent(XContentParser parser) throws IOException {
+        return PARSER.apply(parser, null).build();
+    }
+
+    /**
+     * Required for {@link ScriptMetaData.ScriptMetadataDiff}.  Uses
+     * the {@link StoredScriptSource#StoredScriptSource(StreamInput)}
+     * constructor.
+     */
+    public static Diff<StoredScriptSource> readDiffFrom(StreamInput in) throws IOException {
+        return readDiffFrom(StoredScriptSource::new, in);
+    }
+
+    private final String lang;
+    private final String source;
+    private final Map<String, String> options;
+
+    /**
+     * Constructor for use with {@link GetStoredScriptResponse}
+     * to support the deprecated stored script namespace.
+     */
+    public StoredScriptSource(String source) {
+        this.lang = null;
+        this.source = Objects.requireNonNull(source);
+        this.options = null;
+    }
+
+    /**
+     * Standard StoredScriptSource constructor.
+     * @param lang    The language to compile the script with.  Must not be {@code null}.
+     * @param source    The source source to compile with.  Must not be {@code null}.
+     * @param options Compiler options to be compiled with.  Must not be {@code null},
+     *                use an empty {@link Map} to represent no options.
+     */
+    public StoredScriptSource(String lang, String source, Map<String, String> options) {
+        this.lang = Objects.requireNonNull(lang);
+        this.source = Objects.requireNonNull(source);
+        this.options = Collections.unmodifiableMap(Objects.requireNonNull(options));
+    }
+
+    /**
+     * Reads a {@link StoredScriptSource} from a stream.  Version 5.3+ will read
+     * all of the lang, source, and options parameters.  For versions prior to 5.3,
+     * only the source parameter will be read in as a bytes reference.
+     */
+    public StoredScriptSource(StreamInput in) throws IOException {
+        if (in.getVersion().onOrAfter(Version.V_5_3_0)) {
+            this.lang = in.readString();
+            this.source = in.readString();
+            @SuppressWarnings("unchecked")
+            Map<String, String> options = (Map<String, String>)(Map)in.readMap();
+            this.options = options;
+        } else {
+            this.lang = null;
+            this.source = in.readBytesReference().utf8ToString();
+            this.options = null;
+        }
+    }
+
+    /**
+     * Writes a {@link StoredScriptSource} to a stream.  Version 5.3+ will write
+     * all of the lang, source, and options parameters.  For versions prior to 5.3,
+     * only the source parameter will be read in as a bytes reference.
+     */
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        if (out.getVersion().onOrAfter(Version.V_5_3_0)) {
+            out.writeString(lang);
+            out.writeString(source);
+            @SuppressWarnings("unchecked")
+            Map<String, Object> options = (Map<String, Object>)(Map)this.options;
+            out.writeMap(options);
+        } else {
+            out.writeBytesReference(new BytesArray(source));
+        }
+    }
+
+    /**
+     * This will write XContent from a {@link StoredScriptSource}. The following format will be written:
+     *
+     * {@code
+     * {
+     *     "script" : {
+     *         "lang" : "<lang>",
+     *         "source" : "<source>",
+     *         "options" : {
+     *             "option0" : "<option0>",
+     *             "option1" : "<option1>",
+     *             ...
+     *         }
+     *     }
+     * }
+     * }
+     *
+     * Note that the 'source' parameter can also handle templates written as complex JSON.
+     */
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(LANG_PARSE_FIELD.getPreferredName(), lang);
+        builder.field(SOURCE_PARSE_FIELD.getPreferredName(), source);
+        builder.field(OPTIONS_PARSE_FIELD.getPreferredName(), options);
+        builder.endObject();
+
+        return builder;
+    }
+
+    /**
+     * @return The language used for compiling this script.
+     */
+    public String getLang() {
+        return lang;
+    }
+
+    /**
+     * @return The source used for compiling this script.
+     */
+    public String getSource() {
+        return source;
+    }
+
+    /**
+     * @return The compiler options used for this script.
+     */
+    public Map<String, String> getOptions() {
+        return options;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        StoredScriptSource that = (StoredScriptSource)o;
+
+        if (lang != null ? !lang.equals(that.lang) : that.lang != null) return false;
+        if (source != null ? !source.equals(that.source) : that.source != null) return false;
+        return options != null ? options.equals(that.options) : that.options == null;
+
+    }
+
+    @Override
+    public int hashCode() {
+        int result = lang != null ? lang.hashCode() : 0;
+        result = 31 * result + (source != null ? source.hashCode() : 0);
+        result = 31 * result + (options != null ? options.hashCode() : 0);
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        return "StoredScriptSource{" +
+            "lang='" + lang + '\'' +
+            ", source='" + source + '\'' +
+            ", options=" + options +
+            '}';
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java b/core/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
similarity index 79%
rename from core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java
rename to core/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
index 780352508bfe5..45b8967545414 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java
+++ b/core/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
@@ -17,20 +17,18 @@
  * under the License.
  */
 
-package org.elasticsearch.search.internal;
+package org.elasticsearch.search;
 
-import org.apache.lucene.queries.TermsQuery;
 import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.Collector;
-import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.FieldDoc;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.Counter;
+import org.elasticsearch.action.search.SearchTask;
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery;
@@ -39,7 +37,6 @@
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.analysis.AnalysisService;
 import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.fielddata.IndexFieldDataService;
@@ -49,22 +46,25 @@
 import org.elasticsearch.index.mapper.TypeFieldMapper;
 import org.elasticsearch.index.query.AbstractQueryBuilder;
 import org.elasticsearch.index.query.ParsedQuery;
+import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.search.fetch.StoredFieldsContext;
+import org.elasticsearch.index.search.NestedHelper;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.similarity.SimilarityService;
-import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.aggregations.SearchContextAggregations;
+import org.elasticsearch.search.collapse.CollapseContext;
 import org.elasticsearch.search.dfs.DfsSearchResult;
 import org.elasticsearch.search.fetch.FetchPhase;
 import org.elasticsearch.search.fetch.FetchSearchResult;
-import org.elasticsearch.search.fetch.FetchSubPhase;
-import org.elasticsearch.search.fetch.FetchSubPhaseContext;
+import org.elasticsearch.search.fetch.StoredFieldsContext;
+import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 import org.elasticsearch.search.fetch.subphase.ScriptFieldsContext;
 import org.elasticsearch.search.fetch.subphase.highlight.SearchContextHighlight;
-import org.elasticsearch.search.lookup.SearchLookup;
+import org.elasticsearch.search.internal.ContextIndexSearcher;
+import org.elasticsearch.search.internal.ScrollContext;
+import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.search.profile.Profilers;
 import org.elasticsearch.search.query.QueryPhaseExecutionException;
 import org.elasticsearch.search.query.QuerySearchResult;
@@ -74,16 +74,15 @@
 import org.elasticsearch.search.suggest.SuggestionSearchContext;
 
 import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-/**
- *
- */
-public class DefaultSearchContext extends SearchContext {
+final class DefaultSearchContext extends SearchContext {
 
     private final long id;
     private final ShardSearchRequest request;
@@ -91,7 +90,6 @@ public class DefaultSearchContext extends SearchContext {
     private final Counter timeEstimateCounter;
     private SearchType searchType;
     private final Engine.Searcher engineSearcher;
-    private final ScriptService scriptService;
     private final BigArrays bigArrays;
     private final IndexShard indexShard;
     private final IndexService indexService;
@@ -99,7 +97,7 @@ public class DefaultSearchContext extends SearchContext {
     private final DfsSearchResult dfsResult;
     private final QuerySearchResult queryResult;
     private final FetchSearchResult fetchResult;
-    private float queryBoost = 1.0f;
+    private final float queryBoost;
     private TimeValue timeout;
     // terminate after count
     private int terminateAfter = DEFAULT_TERMINATE_AFTER;
@@ -110,14 +108,19 @@ public class DefaultSearchContext extends SearchContext {
     private StoredFieldsContext storedFields;
     private ScriptFieldsContext scriptFields;
     private FetchSourceContext fetchSourceContext;
+    private DocValueFieldsContext docValueFieldsContext;
     private int from = -1;
     private int size = -1;
     private SortAndFormats sort;
     private Float minimumScore;
     private boolean trackScores = false; // when sorting, track scores as well...
     private FieldDoc searchAfter;
+    private CollapseContext collapse;
+    private boolean lowLevelCancellation;
     // filter for sliced scroll
     private SliceBuilder sliceBuilder;
+    private SearchTask task;
+
 
     /**
      * The original query as sent by the user without the types and aliases
@@ -125,10 +128,7 @@ public class DefaultSearchContext extends SearchContext {
      * things like the type filter or alias filters.
      */
     private ParsedQuery originalQuery;
-    /**
-     * Just like originalQuery but with the filters from types, aliases and slice applied.
-     */
-    private ParsedQuery filteredQuery;
+
     /**
      * The query to actually execute.
      */
@@ -142,29 +142,25 @@ public class DefaultSearchContext extends SearchContext {
     private SearchContextHighlight highlight;
     private SuggestionSearchContext suggest;
     private List<RescoreSearchContext> rescore;
-    private SearchLookup searchLookup;
     private volatile long keepAlive;
     private final long originNanoTime = System.nanoTime();
     private volatile long lastAccessTime = -1;
     private Profilers profilers;
 
-    private final Map<String, FetchSubPhaseContext> subPhaseContexts = new HashMap<>();
+    private final Map<String, SearchExtBuilder> searchExtBuilders = new HashMap<>();
     private final Map<Class<?>, Collector> queryCollectors = new HashMap<>();
     private final QueryShardContext queryShardContext;
     private FetchPhase fetchPhase;
 
-    public DefaultSearchContext(long id, ShardSearchRequest request, SearchShardTarget shardTarget, Engine.Searcher engineSearcher,
-            IndexService indexService, IndexShard indexShard, ScriptService scriptService,
-            BigArrays bigArrays, Counter timeEstimateCounter, ParseFieldMatcher parseFieldMatcher, TimeValue timeout,
-            FetchPhase fetchPhase) {
-        super(parseFieldMatcher);
+    DefaultSearchContext(long id, ShardSearchRequest request, SearchShardTarget shardTarget, Engine.Searcher engineSearcher,
+                         IndexService indexService, IndexShard indexShard,
+                         BigArrays bigArrays, Counter timeEstimateCounter, TimeValue timeout, FetchPhase fetchPhase) {
         this.id = id;
         this.request = request;
         this.fetchPhase = fetchPhase;
         this.searchType = request.searchType();
         this.shardTarget = shardTarget;
         this.engineSearcher = engineSearcher;
-        this.scriptService = scriptService;
         // SearchContexts use a BigArrays that can circuit break
         this.bigArrays = bigArrays.withCircuitBreaking();
         this.dfsResult = new DfsSearchResult(id, shardTarget);
@@ -175,8 +171,9 @@ public DefaultSearchContext(long id, ShardSearchRequest request, SearchShardTarg
         this.searcher = new ContextIndexSearcher(engineSearcher, indexService.cache().query(), indexShard.getQueryCachingPolicy());
         this.timeEstimateCounter = timeEstimateCounter;
         this.timeout = timeout;
-        queryShardContext = indexService.newQueryShardContext(searcher.getIndexReader());
+        queryShardContext = indexService.newQueryShardContext(request.shardId().id(), searcher.getIndexReader(), request::nowInMillis);
         queryShardContext.setTypes(request.types());
+        queryBoost = request.indexBoost();
     }
 
     @Override
@@ -189,7 +186,7 @@ public void doClose() {
      * Should be called before executing the main query and after all other parameters have been set.
      */
     @Override
-    public void preProcess() {
+    public void preProcess(boolean rewrite) {
         if (hasOnlySuggest() ) {
             return;
         }
@@ -235,7 +232,12 @@ public void preProcess() {
         }
 
         // initialize the filtering alias based on the provided filters
-        aliasFilter = indexService.aliasFilter(queryShardContext, request.filteringAliases());
+        try {
+            final QueryBuilder queryBuilder = request.filteringAliases();
+            aliasFilter = queryBuilder == null ? null : queryBuilder.toFilter(queryShardContext);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
 
         if (query() == null) {
             parsedQuery(ParsedQuery.parsedMatchAllQuery());
@@ -243,75 +245,61 @@ public void preProcess() {
         if (queryBoost() != AbstractQueryBuilder.DEFAULT_BOOST) {
             parsedQuery(new ParsedQuery(new FunctionScoreQuery(query(), new WeightFactorFunction(queryBoost)), parsedQuery()));
         }
-        filteredQuery(buildFilteredQuery());
-        try {
-            this.query = searcher().rewrite(this.query);
-        } catch (IOException e) {
-            throw new QueryPhaseExecutionException(this, "Failed to rewrite main query", e);
-        }
-    }
-
-    private ParsedQuery buildFilteredQuery() {
-        Query searchFilter = searchFilter(queryShardContext.getTypes());
-        if (searchFilter == null) {
-            return originalQuery;
-        }
-        Query result;
-        if (Queries.isConstantMatchAllQuery(query())) {
-            result = new ConstantScoreQuery(searchFilter);
-        } else {
-            result = new BooleanQuery.Builder()
-                    .add(query, Occur.MUST)
-                    .add(searchFilter, Occur.FILTER)
-                    .build();
+        this.query = buildFilteredQuery(query);
+        if (rewrite) {
+            try {
+                this.query = searcher.rewrite(query);
+            } catch (IOException e) {
+                throw new QueryPhaseExecutionException(this, "Failed to rewrite main query", e);
+            }
         }
-        return new ParsedQuery(result, originalQuery);
     }
 
     @Override
-    @Nullable
-    public Query searchFilter(String[] types) {
-        Query typesFilter = createSearchFilter(types, aliasFilter, mapperService().hasNested());
-        if (sliceBuilder == null) {
-            return typesFilter;
+    public Query buildFilteredQuery(Query query) {
+        List<Query> filters = new ArrayList<>();
+        Query typeFilter = createTypeFilter(queryShardContext.getTypes());
+        if (typeFilter != null) {
+            filters.add(typeFilter);
         }
-         Query sliceFilter = sliceBuilder.toFilter(queryShardContext, shardTarget().getShardId().getId(),
-                queryShardContext.getIndexSettings().getNumberOfShards());
-        if (typesFilter == null) {
-            return sliceFilter;
-        }
-        return new BooleanQuery.Builder()
-            .add(typesFilter, Occur.FILTER)
-            .add(sliceFilter, Occur.FILTER)
-            .build();
-    }
 
-    // extracted to static helper method to make writing unit tests easier:
-    static Query createSearchFilter(String[] types, Query aliasFilter, boolean hasNestedFields) {
-        Query typesFilter = null;
-        if (types != null && types.length >= 1) {
-            BytesRef[] typesBytes = new BytesRef[types.length];
-            for (int i = 0; i < typesBytes.length; i++) {
-                typesBytes[i] = new BytesRef(types[i]);
-            }
-            typesFilter = new TermsQuery(TypeFieldMapper.NAME, typesBytes);
+        if (mapperService().hasNested()
+                && typeFilter == null // when a _type filter is set, it will automatically exclude nested docs
+                && new NestedHelper(mapperService()).mightMatchNestedDocs(query)
+                && (aliasFilter == null || new NestedHelper(mapperService()).mightMatchNestedDocs(aliasFilter))) {
+            filters.add(Queries.newNonNestedFilter());
         }
 
-        if (typesFilter == null && aliasFilter == null && hasNestedFields == false) {
-            return null;
+        if (aliasFilter != null) {
+            filters.add(aliasFilter);
         }
 
-        BooleanQuery.Builder bq = new BooleanQuery.Builder();
-        if (typesFilter != null) {
-            bq.add(typesFilter, Occur.FILTER);
-        } else if (hasNestedFields) {
-            bq.add(Queries.newNonNestedFilter(), Occur.FILTER);
+        if (sliceBuilder != null) {
+            filters.add(sliceBuilder.toFilter(queryShardContext, shardTarget().getShardId().getId(),
+                    queryShardContext.getIndexSettings().getNumberOfShards()));
         }
-        if (aliasFilter != null) {
-            bq.add(aliasFilter, Occur.FILTER);
+
+        if (filters.isEmpty()) {
+            return query;
+        } else {
+            BooleanQuery.Builder builder = new BooleanQuery.Builder();
+            builder.add(query, Occur.MUST);
+            for (Query filter : filters) {
+                builder.add(filter, Occur.FILTER);
+            }
+            return builder.build();
         }
+    }
 
-        return bq.build();
+    private Query createTypeFilter(String[] types) {
+        if (types != null && types.length >= 1) {
+            MappedFieldType ft = mapperService().fullName(TypeFieldMapper.NAME);
+            if (ft != null) {
+                // ft might be null if no documents have been indexed yet
+                return ft.termsQuery(Arrays.asList(types), queryShardContext);
+            }
+        }
+        return null;
     }
 
     @Override
@@ -349,22 +337,11 @@ public float queryBoost() {
         return queryBoost;
     }
 
-    @Override
-    public SearchContext queryBoost(float queryBoost) {
-        this.queryBoost = queryBoost;
-        return this;
-    }
-
     @Override
     public long getOriginNanoTime() {
         return originNanoTime;
     }
 
-    @Override
-    protected long nowInMillisImpl() {
-        return request.nowInMillis();
-    }
-
     @Override
     public ScrollContext scrollContext() {
         return this.scrollContext;
@@ -388,14 +365,16 @@ public SearchContext aggregations(SearchContextAggregations aggregations) {
     }
 
     @Override
-    public <SubPhaseContext extends FetchSubPhaseContext> SubPhaseContext getFetchSubPhaseContext(FetchSubPhase.ContextFactory<SubPhaseContext> contextFactory) {
-        String subPhaseName = contextFactory.getName();
-        if (subPhaseContexts.get(subPhaseName) == null) {
-            subPhaseContexts.put(subPhaseName, contextFactory.newContextInstance());
-        }
-        return (SubPhaseContext) subPhaseContexts.get(subPhaseName);
+    public void addSearchExt(SearchExtBuilder searchExtBuilder) {
+        //it's ok to use the writeable name here given that we enforce it to be the same as the name of the element that gets
+        //parsed by the corresponding parser. There is one single name and one single way to retrieve the parsed object from the context.
+        searchExtBuilders.put(searchExtBuilder.getWriteableName(), searchExtBuilder);
     }
 
+    @Override
+    public SearchExtBuilder getSearchExt(String name) {
+        return searchExtBuilders.get(name);
+    }
 
     @Override
     public SearchContextHighlight highlight() {
@@ -470,6 +449,17 @@ public SearchContext fetchSourceContext(FetchSourceContext fetchSourceContext) {
         return this;
     }
 
+    @Override
+    public DocValueFieldsContext docValueFieldsContext() {
+        return docValueFieldsContext;
+    }
+
+    @Override
+    public SearchContext docValueFieldsContext(DocValueFieldsContext docValueFieldsContext) {
+        this.docValueFieldsContext = docValueFieldsContext;
+        return this;
+    }
+
     @Override
     public ContextIndexSearcher searcher() {
         return this.searcher;
@@ -485,21 +475,11 @@ public MapperService mapperService() {
         return indexService.mapperService();
     }
 
-    @Override
-    public AnalysisService analysisService() {
-        return indexService.analysisService();
-    }
-
     @Override
     public SimilarityService similarityService() {
         return indexService.similarityService();
     }
 
-    @Override
-    public ScriptService scriptService() {
-        return scriptService;
-    }
-
     @Override
     public BigArrays bigArrays() {
         return bigArrays;
@@ -574,11 +554,31 @@ public SearchContext searchAfter(FieldDoc searchAfter) {
         return this;
     }
 
+    @Override
+    public boolean lowLevelCancellation() {
+        return lowLevelCancellation;
+    }
+
+    public void lowLevelCancellation(boolean lowLevelCancellation) {
+        this.lowLevelCancellation = lowLevelCancellation;
+    }
+
     @Override
     public FieldDoc searchAfter() {
         return searchAfter;
     }
 
+    @Override
+    public SearchContext collapse(CollapseContext collapse) {
+        this.collapse = collapse;
+        return this;
+    }
+
+    @Override
+    public CollapseContext collapse() {
+        return collapse;
+    }
+
     public SearchContext sliceBuilder(SliceBuilder sliceBuilder) {
         this.sliceBuilder = sliceBuilder;
         return this;
@@ -607,15 +607,6 @@ public SearchContext parsedQuery(ParsedQuery query) {
         return this;
     }
 
-    public ParsedQuery filteredQuery() {
-        return filteredQuery;
-    }
-
-    private void filteredQuery(ParsedQuery filteredQuery) {
-        this.filteredQuery = filteredQuery;
-        this.query = filteredQuery.query();
-    }
-
     @Override
     public ParsedQuery parsedQuery() {
         return this.originalQuery;
@@ -751,15 +742,6 @@ public void keepAlive(long keepAlive) {
         this.keepAlive = keepAlive;
     }
 
-    @Override
-    public SearchLookup lookup() {
-        // TODO: The types should take into account the parsing context in QueryParserContext...
-        if (searchLookup == null) {
-            searchLookup = new SearchLookup(mapperService(), fieldData(), request.types());
-        }
-        return searchLookup;
-    }
-
     @Override
     public DfsSearchResult dfsResult() {
         return dfsResult;
@@ -813,4 +795,19 @@ public Profilers getProfilers() {
     public void setProfilers(Profilers profilers) {
         this.profilers = profilers;
     }
+
+    @Override
+    public void setTask(SearchTask task) {
+        this.task = task;
+    }
+
+    @Override
+    public SearchTask getTask() {
+        return task;
+    }
+
+    @Override
+    public boolean isCancelled() {
+        return task.isCancelled();
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/DocValueFormat.java b/core/src/main/java/org/elasticsearch/search/DocValueFormat.java
index 4cbb8720d775b..eb76db3be687b 100644
--- a/core/src/main/java/org/elasticsearch/search/DocValueFormat.java
+++ b/core/src/main/java/org/elasticsearch/search/DocValueFormat.java
@@ -41,7 +41,7 @@
 import java.util.Arrays;
 import java.util.Locale;
 import java.util.Objects;
-import java.util.concurrent.Callable;
+import java.util.function.LongSupplier;
 
 /** A formatter for values as returned by the fielddata/doc-values APIs. */
 public interface DocValueFormat extends NamedWriteable {
@@ -63,11 +63,11 @@ public interface DocValueFormat extends NamedWriteable {
 
     /** Parse a value that was formatted with {@link #format(long)} back to the
      *  original long value. */
-    long parseLong(String value, boolean roundUp, Callable<Long> now);
+    long parseLong(String value, boolean roundUp, LongSupplier now);
 
     /** Parse a value that was formatted with {@link #format(double)} back to
      *  the original double value. */
-    double parseDouble(String value, boolean roundUp, Callable<Long> now);
+    double parseDouble(String value, boolean roundUp, LongSupplier now);
 
     /** Parse a value that was formatted with {@link #format(BytesRef)} back
      *  to the original BytesRef. */
@@ -100,7 +100,7 @@ public String format(BytesRef value) {
         }
 
         @Override
-        public long parseLong(String value, boolean roundUp, Callable<Long> now) {
+        public long parseLong(String value, boolean roundUp, LongSupplier now) {
             double d = Double.parseDouble(value);
             if (roundUp) {
                 d = Math.ceil(d);
@@ -111,7 +111,7 @@ public long parseLong(String value, boolean roundUp, Callable<Long> now) {
         }
 
         @Override
-        public double parseDouble(String value, boolean roundUp, Callable<Long> now) {
+        public double parseDouble(String value, boolean roundUp, LongSupplier now) {
             return Double.parseDouble(value);
         }
 
@@ -166,12 +166,12 @@ public String format(BytesRef value) {
         }
 
         @Override
-        public long parseLong(String value, boolean roundUp, Callable<Long> now) {
+        public long parseLong(String value, boolean roundUp, LongSupplier now) {
             return parser.parse(value, now, roundUp, timeZone);
         }
 
         @Override
-        public double parseDouble(String value, boolean roundUp, Callable<Long> now) {
+        public double parseDouble(String value, boolean roundUp, LongSupplier now) {
             return parseLong(value, roundUp, now);
         }
 
@@ -208,12 +208,12 @@ public String format(BytesRef value) {
         }
 
         @Override
-        public long parseLong(String value, boolean roundUp, Callable<Long> now) {
+        public long parseLong(String value, boolean roundUp, LongSupplier now) {
             throw new UnsupportedOperationException();
         }
 
         @Override
-        public double parseDouble(String value, boolean roundUp, Callable<Long> now) {
+        public double parseDouble(String value, boolean roundUp, LongSupplier now) {
             throw new UnsupportedOperationException();
         }
 
@@ -250,7 +250,7 @@ public String format(BytesRef value) {
         }
 
         @Override
-        public long parseLong(String value, boolean roundUp, Callable<Long> now) {
+        public long parseLong(String value, boolean roundUp, LongSupplier now) {
             switch (value) {
             case "false":
                 return 0;
@@ -261,8 +261,8 @@ public long parseLong(String value, boolean roundUp, Callable<Long> now) {
         }
 
         @Override
-        public double parseDouble(String value, boolean roundUp, Callable<Long> now) {
-            throw new UnsupportedOperationException();
+        public double parseDouble(String value, boolean roundUp, LongSupplier now) {
+            return parseLong(value, roundUp, now);
         }
 
         @Override
@@ -300,12 +300,12 @@ public String format(BytesRef value) {
         }
 
         @Override
-        public long parseLong(String value, boolean roundUp, Callable<Long> now) {
+        public long parseLong(String value, boolean roundUp, LongSupplier now) {
             throw new UnsupportedOperationException();
         }
 
         @Override
-        public double parseDouble(String value, boolean roundUp, Callable<Long> now) {
+        public double parseDouble(String value, boolean roundUp, LongSupplier now) {
             throw new UnsupportedOperationException();
         }
 
@@ -358,7 +358,7 @@ public String format(BytesRef value) {
         }
 
         @Override
-        public long parseLong(String value, boolean roundUp, Callable<Long> now) {
+        public long parseLong(String value, boolean roundUp, LongSupplier now) {
             Number n;
             try {
                 n = format.parse(value);
@@ -379,7 +379,7 @@ public long parseLong(String value, boolean roundUp, Callable<Long> now) {
         }
 
         @Override
-        public double parseDouble(String value, boolean roundUp, Callable<Long> now) {
+        public double parseDouble(String value, boolean roundUp, LongSupplier now) {
             Number n;
             try {
                 n = format.parse(value);
@@ -393,5 +393,22 @@ public double parseDouble(String value, boolean roundUp, Callable<Long> now) {
         public BytesRef parseBytesRef(String value) {
             throw new UnsupportedOperationException();
         }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) {
+                return true;
+            }
+            if (o == null || getClass() != o.getClass()) {
+                return false;
+            }
+            Decimal that = (Decimal) o;
+            return Objects.equals(pattern, that.pattern);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(pattern);
+        }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/MultiValueMode.java b/core/src/main/java/org/elasticsearch/search/MultiValueMode.java
index 90e3417ad1f11..826487d108813 100644
--- a/core/src/main/java/org/elasticsearch/search/MultiValueMode.java
+++ b/core/src/main/java/org/elasticsearch/search/MultiValueMode.java
@@ -935,14 +935,10 @@ public interface UnsortedNumericDoubleValues {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        out.writeVInt(this.ordinal());
+        out.writeEnum(this);
     }
 
     public static MultiValueMode readMultiValueModeFrom(StreamInput in) throws IOException {
-        int ordinal = in.readVInt();
-        if (ordinal < 0 || ordinal >= values().length) {
-            throw new IOException("Unknown MultiValueMode ordinal [" + ordinal + "]");
-        }
-        return values()[ordinal];
+        return in.readEnum(MultiValueMode.class);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/SearchExtBuilder.java b/core/src/main/java/org/elasticsearch/search/SearchExtBuilder.java
new file mode 100644
index 0000000000000..bf696fcc917ad
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/SearchExtBuilder.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search;
+
+import org.elasticsearch.common.CheckedFunction;
+import org.elasticsearch.common.io.stream.NamedWriteable;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.plugins.SearchPlugin;
+import org.elasticsearch.plugins.SearchPlugin.SearchExtSpec;
+
+/**
+ * Intermediate serializable representation of a search ext section. To be subclassed by plugins that support
+ * a custom section as part of a search request, which will be provided within the ext element.
+ * Any state needs to be serialized as part of the {@link Writeable#writeTo(StreamOutput)} method and
+ * read from the incoming stream, usually done adding a constructor that takes {@link StreamInput} as
+ * an argument.
+ *
+ * Registration happens through {@link SearchPlugin#getSearchExts()}, which also needs a {@link CheckedFunction} that's able to parse
+ * the incoming request from the REST layer into the proper {@link SearchExtBuilder} subclass.
+ *
+ * {@link #getWriteableName()} must return the same name as the one used for the registration
+ * of the {@link SearchExtSpec}.
+ *
+ * @see SearchExtSpec
+ */
+public abstract class SearchExtBuilder implements NamedWriteable, ToXContent {
+
+    public abstract int hashCode();
+
+    public abstract boolean equals(Object obj);
+}
diff --git a/core/src/main/java/org/elasticsearch/search/SearchHit.java b/core/src/main/java/org/elasticsearch/search/SearchHit.java
index c9ccddd05e648..71c97d734e1ad 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchHit.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchHit.java
@@ -21,210 +21,1071 @@
 
 import org.apache.lucene.search.Explanation;
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.action.OriginalIndices;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.compress.CompressorFactory;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.text.Text;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
 import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParser.Token;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.SourceFieldMapper;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
+import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.search.suggest.completion.CompletionSuggestion;
+import org.elasticsearch.transport.RemoteClusterAware;
 
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
+import java.util.Objects;
+
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.singletonMap;
+import static java.util.Collections.unmodifiableMap;
+import static org.elasticsearch.common.lucene.Lucene.readExplanation;
+import static org.elasticsearch.common.lucene.Lucene.writeExplanation;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureFieldName;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.parseStoredFieldsValue;
+import static org.elasticsearch.search.fetch.subphase.highlight.HighlightField.readHighlightField;
 
 /**
  * A single search hit.
  *
  * @see SearchHits
  */
-public interface SearchHit extends Streamable, ToXContent, Iterable<SearchHitField> {
+public class SearchHit implements Streamable, ToXContentObject, Iterable<SearchHitField> {
+
+    private transient int docId;
+
+    private static final float DEFAULT_SCORE = Float.NEGATIVE_INFINITY;
+    private float score = DEFAULT_SCORE;
+
+    private Text id;
+    private Text type;
+
+    private NestedIdentity nestedIdentity;
+
+    private long version = -1;
+
+    private BytesReference source;
+
+    private Map<String, SearchHitField> fields = emptyMap();
+
+    private Map<String, HighlightField> highlightFields = null;
+
+    private SearchSortValues sortValues = SearchSortValues.EMPTY;
+
+    private String[] matchedQueries = Strings.EMPTY_ARRAY;
+
+    private Explanation explanation;
+
+    @Nullable
+    private SearchShardTarget shard;
+
+    private transient String index;
+    private transient String clusterAlias;
+
+    private Map<String, Object> sourceAsMap;
+    private byte[] sourceAsBytes;
+
+    private Map<String, SearchHits> innerHits;
+
+    private SearchHit() {
+
+    }
+
+    public SearchHit(int docId) {
+        this(docId, null, null, null);
+    }
+
+    public SearchHit(int docId, String id, Text type, Map<String, SearchHitField> fields) {
+        this(docId, id, type, null, fields);
+    }
+
+    public SearchHit(int nestedTopDocId, String id, Text type, NestedIdentity nestedIdentity, Map<String, SearchHitField> fields) {
+        this.docId = nestedTopDocId;
+        if (id != null) {
+            this.id = new Text(id);
+        } else {
+            this.id = null;
+        }
+        this.type = type;
+        this.nestedIdentity = nestedIdentity;
+        this.fields = fields;
+    }
+
+    public int docId() {
+        return this.docId;
+    }
+
+    public void score(float score) {
+        this.score = score;
+    }
 
     /**
      * The score.
+     * @deprecated  use {@link #getScore()} instead
      */
-    float score();
+    @Deprecated
+    public float score() {
+        return this.score;
+    }
 
     /**
      * The score.
      */
-    float getScore();
+    public float getScore() {
+        return score();
+    }
+
+    public void version(long version) {
+        this.version = version;
+    }
+
+    /**
+     * The version of the hit.
+     * @deprecated use {@link #getVersion()} instead
+     */
+    @Deprecated
+    public long version() {
+        return this.version;
+    }
+
+    /**
+     * The version of the hit.
+     */
+    public long getVersion() {
+        return this.version;
+    }
 
     /**
      * The index of the hit.
+     * @deprecated use {@link #getIndex()} instead
      */
-    String index();
+    @Deprecated
+    public String index() {
+        return getIndex();
+    }
 
     /**
      * The index of the hit.
      */
-    String getIndex();
+    public String getIndex() {
+        return this.index;
+    }
 
     /**
      * The id of the document.
+     * @deprecated use {@link #getId()} instead
      */
-    String id();
+    @Deprecated
+    public String id() {
+        return getId();
+    }
 
     /**
      * The id of the document.
      */
-    String getId();
+    public String getId() {
+        return id != null ? id.string() : null;
+    }
 
     /**
      * The type of the document.
+     * @deprecated use {@link #getType()} instead
      */
-    String type();
+    @Deprecated
+    public String type() {
+        return getType();
+    }
 
     /**
      * The type of the document.
      */
-    String getType();
+    public String getType() {
+        return type != null ? type.string() : null;
+    }
 
     /**
      * If this is a nested hit then nested reference information is returned otherwise <code>null</code> is returned.
      */
-    NestedIdentity getNestedIdentity();
+    public NestedIdentity getNestedIdentity() {
+        return nestedIdentity;
+    }
 
     /**
-     * The version of the hit.
+     * Returns bytes reference, also un compress the source if needed.
+     * @deprecated use {@link #getSourceRef()} instead
      */
-    long version();
+    @Deprecated
+    public BytesReference sourceRef() {
+       return getSourceRef();
+    }
 
     /**
-     * The version of the hit.
+     * Sets representation, might be compressed....
      */
-    long getVersion();
+    public SearchHit sourceRef(BytesReference source) {
+        this.source = source;
+        this.sourceAsBytes = null;
+        this.sourceAsMap = null;
+        return this;
+    }
 
     /**
-     * Returns bytes reference, also un compress the source if needed.
+     * Returns bytes reference, also uncompress the source if needed.
      */
-    BytesReference sourceRef();
+    public BytesReference getSourceRef() {
+        if (this.source == null) {
+            return null;
+        }
 
-    /**
-     * Returns bytes reference, also un compress the source if needed.
-     */
-    BytesReference getSourceRef();
+        try {
+            this.source = CompressorFactory.uncompressIfNeeded(this.source);
+            return this.source;
+        } catch (IOException e) {
+            throw new ElasticsearchParseException("failed to decompress source", e);
+        }
+    }
 
     /**
      * The source of the document (can be <tt>null</tt>). Note, its a copy of the source
      * into a byte array, consider using {@link #sourceRef()} so there won't be a need to copy.
      */
-    byte[] source();
+    @Deprecated
+    public byte[] source() {
+        if (source == null) {
+            return null;
+        }
+        if (sourceAsBytes != null) {
+            return sourceAsBytes;
+        }
+        this.sourceAsBytes = BytesReference.toBytes(sourceRef());
+        return this.sourceAsBytes;
+    }
 
     /**
      * Is the source available or not. A source with no fields will return true. This will return false if {@code fields} doesn't contain
      * {@code _source} or if source is disabled in the mapping.
      */
-    boolean hasSource();
+    public boolean hasSource() {
+        return source != null;
+    }
 
     /**
      * The source of the document as a map (can be <tt>null</tt>).
      */
-    Map<String, Object> getSource();
+    public Map<String, Object> getSource() {
+        return sourceAsMap();
+    }
 
     /**
      * The source of the document as string (can be <tt>null</tt>).
+     * @deprecated use {@link #getSourceAsString()} instead
      */
-    String sourceAsString();
+    @Deprecated
+    public String sourceAsString() {
+        return getSourceAsString();
+    }
 
     /**
      * The source of the document as string (can be <tt>null</tt>).
      */
-    String getSourceAsString();
+    public String getSourceAsString() {
+        if (source == null) {
+            return null;
+        }
+        try {
+            return XContentHelper.convertToJson(sourceRef(), false);
+        } catch (IOException e) {
+            throw new ElasticsearchParseException("failed to convert source to a json string");
+        }
+    }
 
     /**
      * The source of the document as a map (can be <tt>null</tt>).
+     * @deprecated use {@link #getSourceAsMap()} instgead
      */
-    Map<String, Object> sourceAsMap() throws ElasticsearchParseException;
+    @Deprecated
+    public Map<String, Object> sourceAsMap() throws ElasticsearchParseException {
+        return getSourceAsMap();
+    }
 
     /**
-     * If enabled, the explanation of the search hit.
+     * The source of the document as a map (can be <tt>null</tt>).
      */
-    Explanation explanation();
+    public Map<String, Object> getSourceAsMap() throws ElasticsearchParseException {
+        if (source == null) {
+            return null;
+        }
+        if (sourceAsMap != null) {
+            return sourceAsMap;
+        }
+
+        sourceAsMap = SourceLookup.sourceAsMap(source);
+        return sourceAsMap;
+    }
+
+    @Override
+    public Iterator<SearchHitField> iterator() {
+        return fields.values().iterator();
+    }
 
     /**
-     * If enabled, the explanation of the search hit.
+     * The hit field matching the given field name.
+     * @deprecated use {@link #getField(String)} instead
      */
-    Explanation getExplanation();
+    @Deprecated
+    public SearchHitField field(String fieldName) {
+        return getField(fieldName);
+    }
 
     /**
      * The hit field matching the given field name.
      */
-    SearchHitField field(String fieldName);
+    public SearchHitField getField(String fieldName) {
+        return fields().get(fieldName);
+    }
 
     /**
      * A map of hit fields (from field name to hit fields) if additional fields
      * were required to be loaded.
+     * @deprecated use {@link #getFields()} instead
      */
-    Map<String, SearchHitField> fields();
+    @Deprecated
+    public Map<String, SearchHitField> fields() {
+        return getFields();
+    }
+
+    // returns the fields without handling null cases
+    public Map<String, SearchHitField> fieldsOrNull() {
+        return fields;
+    }
 
     /**
      * A map of hit fields (from field name to hit fields) if additional fields
      * were required to be loaded.
      */
-    Map<String, SearchHitField> getFields();
+    public Map<String, SearchHitField> getFields() {
+        return fields == null ? emptyMap() : fields;
+    }
+
+    public void fields(Map<String, SearchHitField> fields) {
+        this.fields = fields;
+    }
 
     /**
      * A map of highlighted fields.
+     * @deprecated use {@link #getHighlightFields()} instead
      */
-    Map<String, HighlightField> highlightFields();
+    @Deprecated
+    public Map<String, HighlightField> highlightFields() {
+        return highlightFields == null ? emptyMap() : highlightFields;
+    }
 
     /**
      * A map of highlighted fields.
      */
-    Map<String, HighlightField> getHighlightFields();
+    public Map<String, HighlightField> getHighlightFields() {
+        return highlightFields();
+    }
+
+    public void highlightFields(Map<String, HighlightField> highlightFields) {
+        this.highlightFields = highlightFields;
+    }
+
+    public void sortValues(Object[] sortValues, DocValueFormat[] sortValueFormats) {
+        sortValues(new SearchSortValues(sortValues, sortValueFormats));
+    }
+
+    public void sortValues(SearchSortValues sortValues) {
+        this.sortValues = sortValues;
+    }
 
     /**
      * An array of the sort values used.
+     * @deprecated use {@link #getSortValues()} instead
      */
-    Object[] sortValues();
+    @Deprecated
+    public Object[] sortValues() {
+        return sortValues.sortValues();
+    }
 
     /**
      * An array of the sort values used.
      */
-    Object[] getSortValues();
+    public Object[] getSortValues() {
+        return sortValues();
+    }
 
     /**
-     * The set of query and filter names the query matched with. Mainly makes sense for compound filters and queries.
+     * If enabled, the explanation of the search hit.
+     * @deprecated use {@link #getExplanation()} instead
      */
-    String[] matchedQueries();
+    @Deprecated
+    public Explanation explanation() {
+        return explanation;
+    }
 
     /**
-     * The set of query and filter names the query matched with. Mainly makes sense for compound filters and queries.
+     * If enabled, the explanation of the search hit.
      */
-    String[] getMatchedQueries();
+    public Explanation getExplanation() {
+        return explanation();
+    }
+
+    public void explanation(Explanation explanation) {
+        this.explanation = explanation;
+    }
 
     /**
      * The shard of the search hit.
+     * @deprecated use {@link #getShard()} instead
      */
-    SearchShardTarget shard();
+    @Deprecated
+    public SearchShardTarget shard() {
+        return shard;
+    }
 
     /**
      * The shard of the search hit.
      */
-    SearchShardTarget getShard();
+    public SearchShardTarget getShard() {
+        return shard();
+    }
+
+    public void shard(SearchShardTarget target) {
+        this.shard = target;
+        if (target != null) {
+            this.index = target.getIndex();
+            this.clusterAlias = target.getClusterAlias();
+        }
+    }
+
+    /**
+     * Returns the cluster alias this hit comes from or null if it comes from a local cluster
+     */
+    public String getClusterAlias() {
+        return clusterAlias;
+    }
+
+    public void matchedQueries(String[] matchedQueries) {
+        this.matchedQueries = matchedQueries;
+    }
+
+    /**
+     * The set of query and filter names the query matched with. Mainly makes sense for compound filters and queries.
+     */
+    @Deprecated
+    public String[] matchedQueries() {
+        return this.matchedQueries;
+    }
+
+    /**
+     * The set of query and filter names the query matched with. Mainly makes sense for compound filters and queries.
+     */
+    public String[] getMatchedQueries() {
+        return this.matchedQueries;
+    }
 
     /**
      * @return Inner hits or <code>null</code> if there are none
      */
-    Map<String, SearchHits> getInnerHits();
+    @SuppressWarnings("unchecked")
+    public Map<String, SearchHits> getInnerHits() {
+        return innerHits;
+    }
+
+    public void setInnerHits(Map<String, SearchHits> innerHits) {
+        this.innerHits = innerHits;
+    }
+
+    public static class Fields {
+        static final String _INDEX = "_index";
+        static final String _TYPE = "_type";
+        static final String _ID = "_id";
+        static final String _VERSION = "_version";
+        static final String _SCORE = "_score";
+        static final String FIELDS = "fields";
+        static final String HIGHLIGHT = "highlight";
+        static final String SORT = "sort";
+        static final String MATCHED_QUERIES = "matched_queries";
+        static final String _EXPLANATION = "_explanation";
+        static final String VALUE = "value";
+        static final String DESCRIPTION = "description";
+        static final String DETAILS = "details";
+        static final String INNER_HITS = "inner_hits";
+        static final String _SHARD = "_shard";
+        static final String _NODE = "_node";
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        toInnerXContent(builder, params);
+        builder.endObject();
+        return builder;
+    }
+
+    // public because we render hit as part of completion suggestion option
+    public XContentBuilder toInnerXContent(XContentBuilder builder, Params params) throws IOException {
+        List<SearchHitField> metaFields = new ArrayList<>();
+        List<SearchHitField> otherFields = new ArrayList<>();
+        if (fields != null && !fields.isEmpty()) {
+            for (SearchHitField field : fields.values()) {
+                if (field.values().isEmpty()) {
+                    continue;
+                }
+                if (field.isMetadataField()) {
+                    metaFields.add(field);
+                } else {
+                    otherFields.add(field);
+                }
+            }
+        }
+
+        // For inner_hit hits shard is null and that is ok, because the parent search hit has all this information.
+        // Even if this was included in the inner_hit hits this would be the same, so better leave it out.
+        if (explanation() != null && shard != null) {
+            builder.field(Fields._SHARD, shard.getShardId());
+            builder.field(Fields._NODE, shard.getNodeIdText());
+        }
+        if (nestedIdentity != null) {
+            nestedIdentity.toXContent(builder, params);
+        } else {
+            if (index != null) {
+                builder.field(Fields._INDEX, RemoteClusterAware.buildRemoteIndexName(clusterAlias, index));
+            }
+            if (type != null) {
+                builder.field(Fields._TYPE, type);
+            }
+            if (id != null) {
+                builder.field(Fields._ID, id);
+            }
+        }
+        if (version != -1) {
+            builder.field(Fields._VERSION, version);
+        }
+        if (Float.isNaN(score)) {
+            builder.nullField(Fields._SCORE);
+        } else {
+            builder.field(Fields._SCORE, score);
+        }
+        for (SearchHitField field : metaFields) {
+            Object value = field.value();
+            builder.field(field.name(), value);
+        }
+        if (source != null) {
+            XContentHelper.writeRawField(SourceFieldMapper.NAME, source, builder, params);
+        }
+        if (!otherFields.isEmpty()) {
+            builder.startObject(Fields.FIELDS);
+            for (SearchHitField field : otherFields) {
+                builder.startArray(field.name());
+                for (Object value : field.getValues()) {
+                    builder.value(value);
+                }
+                builder.endArray();
+            }
+            builder.endObject();
+        }
+        if (highlightFields != null && !highlightFields.isEmpty()) {
+            builder.startObject(Fields.HIGHLIGHT);
+            for (HighlightField field : highlightFields.values()) {
+                field.toXContent(builder, params);
+            }
+            builder.endObject();
+        }
+        sortValues.toXContent(builder, params);
+        if (matchedQueries.length > 0) {
+            builder.startArray(Fields.MATCHED_QUERIES);
+            for (String matchedFilter : matchedQueries) {
+                builder.value(matchedFilter);
+            }
+            builder.endArray();
+        }
+        if (explanation() != null) {
+            builder.field(Fields._EXPLANATION);
+            buildExplanation(builder, explanation());
+        }
+        if (innerHits != null) {
+            builder.startObject(Fields.INNER_HITS);
+            for (Map.Entry<String, SearchHits> entry : innerHits.entrySet()) {
+                builder.startObject(entry.getKey());
+                entry.getValue().toXContent(builder, params);
+                builder.endObject();
+            }
+            builder.endObject();
+        }
+        return builder;
+    }
+
+    /**
+     * This parser outputs a temporary map of the objects needed to create the
+     * SearchHit instead of directly creating the SearchHit. The reason for this
+     * is that this way we can reuse the parser when parsing xContent from
+     * {@link CompletionSuggestion.Entry.Option} which unfortunately inlines the
+     * output of
+     * {@link #toInnerXContent(XContentBuilder, org.elasticsearch.common.xcontent.ToXContent.Params)}
+     * of the included search hit. The output of the map is used to create the
+     * actual SearchHit instance via {@link #createFromMap(Map)}
+     */
+    private static ObjectParser<Map<String, Object>, Void> MAP_PARSER = new ObjectParser<>("innerHitParser", true, HashMap::new);
+
+    static {
+        declareInnerHitsParseFields(MAP_PARSER);
+    }
+
+    public static SearchHit fromXContent(XContentParser parser) {
+        return createFromMap(MAP_PARSER.apply(parser, null));
+    }
+
+    public static void declareInnerHitsParseFields(ObjectParser<Map<String, Object>, Void> parser) {
+        declareMetaDataFields(parser);
+        parser.declareString((map, value) -> map.put(Fields._TYPE, new Text(value)), new ParseField(Fields._TYPE));
+        parser.declareString((map, value) -> map.put(Fields._INDEX, value), new ParseField(Fields._INDEX));
+        parser.declareString((map, value) -> map.put(Fields._ID, value), new ParseField(Fields._ID));
+        parser.declareString((map, value) -> map.put(Fields._NODE, value), new ParseField(Fields._NODE));
+        parser.declareField((map, value) -> map.put(Fields._SCORE, value), SearchHit::parseScore, new ParseField(Fields._SCORE),
+                ValueType.FLOAT_OR_NULL);
+        parser.declareLong((map, value) -> map.put(Fields._VERSION, value), new ParseField(Fields._VERSION));
+        parser.declareField((map, value) -> map.put(Fields._SHARD, value), (p, c) -> ShardId.fromString(p.text()),
+                new ParseField(Fields._SHARD), ValueType.STRING);
+        parser.declareObject((map, value) -> map.put(SourceFieldMapper.NAME, value), (p, c) -> parseSourceBytes(p),
+                new ParseField(SourceFieldMapper.NAME));
+        parser.declareObject((map, value) -> map.put(Fields.HIGHLIGHT, value), (p, c) -> parseHighlightFields(p),
+                new ParseField(Fields.HIGHLIGHT));
+        parser.declareObject((map, value) -> {
+            Map<String, SearchHitField> fieldMap = get(Fields.FIELDS, map, new HashMap<String, SearchHitField>());
+            fieldMap.putAll(value);
+            map.put(Fields.FIELDS, fieldMap);
+        }, (p, c) -> parseFields(p), new ParseField(Fields.FIELDS));
+        parser.declareObject((map, value) -> map.put(Fields._EXPLANATION, value), (p, c) -> parseExplanation(p),
+                new ParseField(Fields._EXPLANATION));
+        parser.declareObject((map, value) -> map.put(NestedIdentity._NESTED, value), NestedIdentity::fromXContent,
+                new ParseField(NestedIdentity._NESTED));
+        parser.declareObject((map, value) -> map.put(Fields.INNER_HITS, value), (p,c) -> parseInnerHits(p),
+                new ParseField(Fields.INNER_HITS));
+        parser.declareStringArray((map, list) -> map.put(Fields.MATCHED_QUERIES, list), new ParseField(Fields.MATCHED_QUERIES));
+        parser.declareField((map, list) -> map.put(Fields.SORT, list), SearchSortValues::fromXContent, new ParseField(Fields.SORT),
+                ValueType.OBJECT_ARRAY);
+    }
+
+    public static SearchHit createFromMap(Map<String, Object> values) {
+        String id = get(Fields._ID, values, null);
+        Text type = get(Fields._TYPE, values, null);
+        NestedIdentity nestedIdentity = get(NestedIdentity._NESTED, values, null);
+        Map<String, SearchHitField> fields = get(Fields.FIELDS, values, null);
+
+        SearchHit searchHit = new SearchHit(-1, id, type, nestedIdentity, fields);
+        searchHit.index = get(Fields._INDEX, values, null);
+        searchHit.score(get(Fields._SCORE, values, DEFAULT_SCORE));
+        searchHit.version(get(Fields._VERSION, values, -1L));
+        searchHit.sortValues(get(Fields.SORT, values, SearchSortValues.EMPTY));
+        searchHit.highlightFields(get(Fields.HIGHLIGHT, values, null));
+        searchHit.sourceRef(get(SourceFieldMapper.NAME, values, null));
+        searchHit.explanation(get(Fields._EXPLANATION, values, null));
+        searchHit.setInnerHits(get(Fields.INNER_HITS, values, null));
+        List<String> matchedQueries = get(Fields.MATCHED_QUERIES, values, null);
+        if (matchedQueries != null) {
+            searchHit.matchedQueries(matchedQueries.toArray(new String[matchedQueries.size()]));
+        }
+        ShardId shardId = get(Fields._SHARD, values, null);
+        String nodeId = get(Fields._NODE, values, null);
+        if (shardId != null && nodeId != null) {
+            searchHit.shard(new SearchShardTarget(nodeId, shardId, null, OriginalIndices.NONE));
+        }
+        searchHit.fields(fields);
+        return searchHit;
+    }
+
+    @SuppressWarnings("unchecked")
+    private static <T> T get(String key, Map<String, Object> map, T defaultValue) {
+        return (T) map.getOrDefault(key, defaultValue);
+    }
+
+    private static float parseScore(XContentParser parser) throws IOException {
+        if (parser.currentToken() == XContentParser.Token.VALUE_NUMBER || parser.currentToken() == XContentParser.Token.VALUE_STRING) {
+            return parser.floatValue();
+        } else {
+            return Float.NaN;
+        }
+    }
+
+    private static BytesReference parseSourceBytes(XContentParser parser) throws IOException {
+        try (XContentBuilder builder = XContentBuilder.builder(parser.contentType().xContent())) {
+            // the original document gets slightly modified: whitespaces or
+            // pretty printing are not preserved,
+            // it all depends on the current builder settings
+            builder.copyCurrentStructure(parser);
+            return builder.bytes();
+        }
+    }
+
+    /**
+     * we need to declare parse fields for each metadata field, except for _ID, _INDEX and _TYPE which are
+     * handled individually. All other fields are parsed to an entry in the fields map
+     */
+    private static void declareMetaDataFields(ObjectParser<Map<String, Object>, Void> parser) {
+        for (String metadatafield : MapperService.getAllMetaFields()) {
+            if (metadatafield.equals(Fields._ID) == false && metadatafield.equals(Fields._INDEX) == false
+                    && metadatafield.equals(Fields._TYPE) == false) {
+                parser.declareField((map, field) -> {
+                    @SuppressWarnings("unchecked")
+                    Map<String, SearchHitField> fieldMap = (Map<String, SearchHitField>) map.computeIfAbsent(Fields.FIELDS,
+                            v -> new HashMap<String, SearchHitField>());
+                    fieldMap.put(field.getName(), field);
+                }, (p, c) -> {
+                    List<Object> values = new ArrayList<>();
+                    values.add(parseStoredFieldsValue(p));
+                    return new SearchHitField(metadatafield, values);
+                }, new ParseField(metadatafield), ValueType.VALUE);
+            }
+        }
+    }
+
+    private static Map<String, SearchHitField> parseFields(XContentParser parser) throws IOException {
+        Map<String, SearchHitField> fields = new HashMap<>();
+        while ((parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            String fieldName = parser.currentName();
+            ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.nextToken(), parser::getTokenLocation);
+            List<Object> values = new ArrayList<>();
+            while ((parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                values.add(parseStoredFieldsValue(parser));
+            }
+            fields.put(fieldName, new SearchHitField(fieldName, values));
+        }
+        return fields;
+    }
+
+    private static Map<String, SearchHits> parseInnerHits(XContentParser parser) throws IOException {
+        Map<String, SearchHits> innerHits = new HashMap<>();
+        while ((parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser::getTokenLocation);
+            String name = parser.currentName();
+            ensureExpectedToken(Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
+            ensureFieldName(parser, parser.nextToken(), SearchHits.Fields.HITS);
+            innerHits.put(name, SearchHits.fromXContent(parser));
+            ensureExpectedToken(XContentParser.Token.END_OBJECT, parser.nextToken(), parser::getTokenLocation);
+        }
+        return innerHits;
+    }
+
+    private static Map<String, HighlightField> parseHighlightFields(XContentParser parser) throws IOException {
+        Map<String, HighlightField> highlightFields = new HashMap<>();
+        while((parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            HighlightField highlightField = HighlightField.fromXContent(parser);
+            highlightFields.put(highlightField.getName(), highlightField);
+        }
+        return highlightFields;
+    }
+
+    private static Explanation parseExplanation(XContentParser parser) throws IOException {
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation);
+        XContentParser.Token token;
+        Float value = null;
+        String description = null;
+        List<Explanation> details = new ArrayList<>();
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, () -> parser.getTokenLocation());
+            String currentFieldName = parser.currentName();
+            token = parser.nextToken();
+            if (Fields.VALUE.equals(currentFieldName)) {
+                value = parser.floatValue();
+            } else if (Fields.DESCRIPTION.equals(currentFieldName)) {
+                description = parser.textOrNull();
+            } else if (Fields.DETAILS.equals(currentFieldName)) {
+                ensureExpectedToken(XContentParser.Token.START_ARRAY, token, () -> parser.getTokenLocation());
+                while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                    details.add(parseExplanation(parser));
+                }
+            } else {
+                parser.skipChildren();
+            }
+        }
+        if (value == null) {
+            throw new ParsingException(parser.getTokenLocation(), "missing explanation value");
+        }
+        if (description == null) {
+            throw new ParsingException(parser.getTokenLocation(), "missing explanation description");
+        }
+        return Explanation.match(value, description, details);
+    }
+
+    private void buildExplanation(XContentBuilder builder, Explanation explanation) throws IOException {
+        builder.startObject();
+        builder.field(Fields.VALUE, explanation.getValue());
+        builder.field(Fields.DESCRIPTION, explanation.getDescription());
+        Explanation[] innerExps = explanation.getDetails();
+        if (innerExps != null) {
+            builder.startArray(Fields.DETAILS);
+            for (Explanation exp : innerExps) {
+                buildExplanation(builder, exp);
+            }
+            builder.endArray();
+        }
+        builder.endObject();
+    }
+
+    public static SearchHit readSearchHit(StreamInput in) throws IOException {
+        SearchHit hit = new SearchHit();
+        hit.readFrom(in);
+        return hit;
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        score = in.readFloat();
+        id = in.readOptionalText();
+        type = in.readOptionalText();
+        nestedIdentity = in.readOptionalWriteable(NestedIdentity::new);
+        version = in.readLong();
+        source = in.readBytesReference();
+        if (source.length() == 0) {
+            source = null;
+        }
+        if (in.readBoolean()) {
+            explanation = readExplanation(in);
+        }
+        int size = in.readVInt();
+        if (size == 0) {
+            fields = emptyMap();
+        } else if (size == 1) {
+            SearchHitField hitField = SearchHitField.readSearchHitField(in);
+            fields = singletonMap(hitField.name(), hitField);
+        } else {
+            Map<String, SearchHitField> fields = new HashMap<>();
+            for (int i = 0; i < size; i++) {
+                SearchHitField hitField = SearchHitField.readSearchHitField(in);
+                fields.put(hitField.name(), hitField);
+            }
+            this.fields = unmodifiableMap(fields);
+        }
+
+        size = in.readVInt();
+        if (size == 0) {
+            highlightFields = emptyMap();
+        } else if (size == 1) {
+            HighlightField field = readHighlightField(in);
+            highlightFields = singletonMap(field.name(), field);
+        } else {
+            Map<String, HighlightField> highlightFields = new HashMap<>();
+            for (int i = 0; i < size; i++) {
+                HighlightField field = readHighlightField(in);
+                highlightFields.put(field.name(), field);
+            }
+            this.highlightFields = unmodifiableMap(highlightFields);
+        }
+
+        sortValues = new SearchSortValues(in);
+
+        size = in.readVInt();
+        if (size > 0) {
+            matchedQueries = new String[size];
+            for (int i = 0; i < size; i++) {
+                matchedQueries[i] = in.readString();
+            }
+        }
+        // we call the setter here because that also sets the local index parameter
+        shard(in.readOptionalWriteable(SearchShardTarget::new));
+        size = in.readVInt();
+        if (size > 0) {
+            innerHits = new HashMap<>(size);
+            for (int i = 0; i < size; i++) {
+                String key = in.readString();
+                SearchHits value = SearchHits.readSearchHits(in);
+                innerHits.put(key, value);
+            }
+        }
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeFloat(score);
+        out.writeOptionalText(id);
+        out.writeOptionalText(type);
+        out.writeOptionalWriteable(nestedIdentity);
+        out.writeLong(version);
+        out.writeBytesReference(source);
+        if (explanation == null) {
+            out.writeBoolean(false);
+        } else {
+            out.writeBoolean(true);
+            writeExplanation(out, explanation);
+        }
+        if (fields == null) {
+            out.writeVInt(0);
+        } else {
+            out.writeVInt(fields.size());
+            for (SearchHitField hitField : fields().values()) {
+                hitField.writeTo(out);
+            }
+        }
+        if (highlightFields == null) {
+            out.writeVInt(0);
+        } else {
+            out.writeVInt(highlightFields.size());
+            for (HighlightField highlightField : highlightFields.values()) {
+                highlightField.writeTo(out);
+            }
+        }
+        sortValues.writeTo(out);
+
+        if (matchedQueries.length == 0) {
+            out.writeVInt(0);
+        } else {
+            out.writeVInt(matchedQueries.length);
+            for (String matchedFilter : matchedQueries) {
+                out.writeString(matchedFilter);
+            }
+        }
+        out.writeOptionalWriteable(shard);
+        if (innerHits == null) {
+            out.writeVInt(0);
+        } else {
+            out.writeVInt(innerHits.size());
+            for (Map.Entry<String, SearchHits> entry : innerHits.entrySet()) {
+                out.writeString(entry.getKey());
+                entry.getValue().writeTo(out);
+            }
+        }
+    }
 
     /**
      * Encapsulates the nested identity of a hit.
      */
-    interface NestedIdentity {
+    public static final class NestedIdentity implements Writeable, ToXContent {
+
+        private static final String _NESTED = "_nested";
+        private static final String FIELD = "field";
+        private static final String OFFSET = "offset";
+
+        private Text field;
+        private int offset;
+        private NestedIdentity child;
+
+        public NestedIdentity(String field, int offset, NestedIdentity child) {
+            this.field = new Text(field);
+            this.offset = offset;
+            this.child = child;
+        }
+
+        NestedIdentity(StreamInput in) throws IOException {
+            field = in.readOptionalText();
+            offset = in.readInt();
+            child = in.readOptionalWriteable(NestedIdentity::new);
+        }
 
         /**
          * Returns the nested field in the source this hit originates from
          */
-        Text getField();
+        public Text getField() {
+            return field;
+        }
 
         /**
          * Returns the offset in the nested array of objects in the source this hit
          */
-        int getOffset();
+        public int getOffset() {
+            return offset;
+        }
 
         /**
          * Returns the next child nested level if there is any, otherwise <code>null</code> is returned.
          *
          * In the case of mappings with multiple levels of nested object fields
          */
-        NestedIdentity getChild();
+        public NestedIdentity getChild() {
+            return child;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeOptionalText(field);
+            out.writeInt(offset);
+            out.writeOptionalWriteable(child);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.field(_NESTED);
+            return innerToXContent(builder, params);
+        }
+
+        /**
+         * Rendering of the inner XContent object without the leading field name. This way the structure innerToXContent renders and
+         * fromXContent parses correspond to each other.
+         */
+        XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            if (field != null) {
+                builder.field(FIELD, field);
+            }
+            if (offset != -1) {
+                builder.field(OFFSET, offset);
+            }
+            if (child != null) {
+                builder = child.toXContent(builder, params);
+            }
+            builder.endObject();
+            return builder;
+        }
+
+        private static final ConstructingObjectParser<NestedIdentity, Void> PARSER = new ConstructingObjectParser<>("nested_identity", true,
+                ctorArgs -> new NestedIdentity((String) ctorArgs[0], (int) ctorArgs[1], (NestedIdentity) ctorArgs[2]));
+        static {
+            PARSER.declareString(constructorArg(), new ParseField(FIELD));
+            PARSER.declareInt(constructorArg(), new ParseField(OFFSET));
+            PARSER.declareObject(optionalConstructorArg(), PARSER, new ParseField(_NESTED));
+        }
+
+        static NestedIdentity fromXContent(XContentParser parser, Void context) {
+            return fromXContent(parser);
+        }
+
+        public static NestedIdentity fromXContent(XContentParser parser) {
+            return PARSER.apply(parser, null);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (this == obj) {
+                return true;
+            }
+            if (obj == null || getClass() != obj.getClass()) {
+                return false;
+            }
+            NestedIdentity other = (NestedIdentity) obj;
+            return Objects.equals(field, other.field) &&
+                    Objects.equals(offset, other.offset) &&
+                    Objects.equals(child, other.child);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(field, offset, child);
+        }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/SearchHitField.java b/core/src/main/java/org/elasticsearch/search/SearchHitField.java
index 5747bbebef8d1..51f075fdfa398 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchHitField.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchHitField.java
@@ -19,8 +19,14 @@
 
 package org.elasticsearch.search;
 
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
+import org.elasticsearch.index.mapper.MapperService;
 
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
 import java.util.List;
 
 /**
@@ -28,40 +34,104 @@
  *
  * @see SearchHit
  */
-public interface SearchHitField extends Streamable, Iterable<Object> {
+public class SearchHitField implements Streamable, Iterable<Object>  {
+
+    private String name;
+    private List<Object> values;
+
+    private SearchHitField() {
+    }
+
+    public SearchHitField(String name, List<Object> values) {
+        this.name = name;
+        this.values = values;
+    }
 
     /**
      * The name of the field.
+     * @deprecated use {@link #getName()} instead
      */
-    String name();
+    @Deprecated
+    public String name() {
+        return name;
+    }
 
     /**
      * The name of the field.
      */
-    String getName();
+    public String getName() {
+        return name();
+    }
 
     /**
      * The first value of the hit.
+     * @deprecated use {@link #getValue()} instead
      */
-    <V> V value();
+    @Deprecated
+    public <T> T value() {
+        return getValue();
+    }
 
     /**
      * The first value of the hit.
      */
-    <V> V getValue();
+    public <T> T getValue() {
+        if (values == null || values.isEmpty()) {
+            return null;
+        }
+        return (T) values.get(0);
+    }
 
     /**
      * The field values.
+     * @deprecated use {@link #getValues()} instead
      */
-    List<Object> values();
+    @Deprecated
+    public List<Object> values() {
+        return values;
+    }
 
     /**
      * The field values.
      */
-    List<Object> getValues();
+    public List<Object> getValues() {
+        return values();
+    }
 
     /**
      * @return The field is a metadata field
      */
-    boolean isMetadataField();
+    public boolean isMetadataField() {
+        return MapperService.isMetadataField(name);
+    }
+
+    @Override
+    public Iterator<Object> iterator() {
+        return values.iterator();
+    }
+
+    public static SearchHitField readSearchHitField(StreamInput in) throws IOException {
+        SearchHitField result = new SearchHitField();
+        result.readFrom(in);
+        return result;
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        name = in.readString();
+        int size = in.readVInt();
+        values = new ArrayList<>(size);
+        for (int i = 0; i < size; i++) {
+            values.add(in.readGenericValue());
+        }
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(name);
+        out.writeVInt(values.size());
+        for (Object value : values) {
+            out.writeGenericValue(value);
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/SearchHits.java b/core/src/main/java/org/elasticsearch/search/SearchHits.java
index 400e2ebc44fc5..58f4fc675cac0 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchHits.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchHits.java
@@ -19,48 +19,209 @@
 
 package org.elasticsearch.search;
 
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
 import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
 
 /**
  * The hits of a search request.
- *
- *
  */
-public interface SearchHits extends Streamable, ToXContent, Iterable<SearchHit> {
+public class SearchHits implements Streamable, ToXContent, Iterable<SearchHit> {
+
+    public static SearchHits empty() {
+        // We shouldn't use static final instance, since that could directly be returned by native transport clients
+        return new SearchHits(EMPTY, 0, 0);
+    }
+
+    public static final SearchHit[] EMPTY = new SearchHit[0];
+
+    private SearchHit[] hits;
+
+    public long totalHits;
+
+    private float maxScore;
+
+    SearchHits() {
+
+    }
+
+    public SearchHits(SearchHit[] hits, long totalHits, float maxScore) {
+        this.hits = hits;
+        this.totalHits = totalHits;
+        this.maxScore = maxScore;
+    }
 
     /**
      * The total number of hits that matches the search request.
+     * @deprecated use {@link #getTotalHits()} instead
      */
-    long totalHits();
+    @Deprecated
+    public long totalHits() {
+        return totalHits;
+    }
 
     /**
      * The total number of hits that matches the search request.
      */
-    long getTotalHits();
+    public long getTotalHits() {
+        return totalHits();
+    }
 
     /**
      * The maximum score of this query.
+     * @deprecated use {@link #getMaxScore()} instead
      */
-    float maxScore();
+    @Deprecated
+    public float maxScore() {
+        return this.maxScore;
+    }
 
     /**
      * The maximum score of this query.
      */
-    float getMaxScore();
+    public float getMaxScore() {
+        return maxScore();
+    }
 
     /**
      * The hits of the search request (based on the search type, and from / size provided).
+     * @deprecated use {@link #getHits()} instead
      */
-    SearchHit[] hits();
+    @Deprecated
+    public SearchHit[] hits() {
+        return this.hits;
+    }
 
     /**
      * Return the hit as the provided position.
      */
-    SearchHit getAt(int position);
+    public SearchHit getAt(int position) {
+        return hits[position];
+    }
 
     /**
      * The hits of the search request (based on the search type, and from / size provided).
      */
-    SearchHit[] getHits();
+    public SearchHit[] getHits() {
+        return hits();
+    }
+
+    @Override
+    public Iterator<SearchHit> iterator() {
+        return Arrays.stream(hits()).iterator();
+    }
+
+    public SearchHit[] internalHits() {
+        return this.hits;
+    }
+
+    public static final class Fields {
+        public static final String HITS = "hits";
+        public static final String TOTAL = "total";
+        public static final String MAX_SCORE = "max_score";
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject(Fields.HITS);
+        builder.field(Fields.TOTAL, totalHits);
+        if (Float.isNaN(maxScore)) {
+            builder.nullField(Fields.MAX_SCORE);
+        } else {
+            builder.field(Fields.MAX_SCORE, maxScore);
+        }
+        builder.field(Fields.HITS);
+        builder.startArray();
+        for (SearchHit hit : hits) {
+            hit.toXContent(builder, params);
+        }
+        builder.endArray();
+        builder.endObject();
+        return builder;
+    }
+
+    public static SearchHits fromXContent(XContentParser parser) throws IOException {
+        if (parser.currentToken() != XContentParser.Token.START_OBJECT) {
+            parser.nextToken();
+            ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation);
+        }
+        XContentParser.Token token = parser.currentToken();
+        String currentFieldName = null;
+        List<SearchHit> hits = new ArrayList<>();
+        long totalHits = 0;
+        float maxScore = 0f;
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token.isValue()) {
+                if (Fields.TOTAL.equals(currentFieldName)) {
+                    totalHits = parser.longValue();
+                } else if (Fields.MAX_SCORE.equals(currentFieldName)) {
+                    maxScore = parser.floatValue();
+                }
+            } else if (token == XContentParser.Token.VALUE_NULL) {
+                if (Fields.MAX_SCORE.equals(currentFieldName)) {
+                    maxScore = Float.NaN; // NaN gets rendered as null-field
+                }
+            } else if (token == XContentParser.Token.START_ARRAY) {
+                if (Fields.HITS.equals(currentFieldName)) {
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                        hits.add(SearchHit.fromXContent(parser));
+                    }
+                } else {
+                    parser.skipChildren();
+                }
+            } else if (token == XContentParser.Token.START_OBJECT) {
+                parser.skipChildren();
+            }
+        }
+        SearchHits searchHits = new SearchHits(hits.toArray(new SearchHit[hits.size()]), totalHits,
+                maxScore);
+        return searchHits;
+    }
+
+
+    public static SearchHits readSearchHits(StreamInput in) throws IOException {
+        SearchHits hits = new SearchHits();
+        hits.readFrom(in);
+        return hits;
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        totalHits = in.readVLong();
+        maxScore = in.readFloat();
+        int size = in.readVInt();
+        if (size == 0) {
+            hits = EMPTY;
+        } else {
+            hits = new SearchHit[size];
+            for (int i = 0; i < hits.length; i++) {
+                hits[i] = SearchHit.readSearchHit(in);
+            }
+        }
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeVLong(totalHits);
+        out.writeFloat(maxScore);
+        out.writeVInt(hits.length);
+        if (hits.length > 0) {
+            for (SearchHit hit : hits) {
+                hit.writeTo(out);
+            }
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/SearchModule.java b/core/src/main/java/org/elasticsearch/search/SearchModule.java
index ab36f03639cb0..a68a5b7fa17de 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchModule.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchModule.java
@@ -23,12 +23,14 @@
 import org.elasticsearch.common.NamedRegistry;
 import org.elasticsearch.common.geo.ShapesAvailability;
 import org.elasticsearch.common.geo.builders.ShapeBuilders;
-import org.elasticsearch.common.inject.AbstractModule;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ParseFieldRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.BoostingQueryBuilder;
 import org.elasticsearch.index.query.CommonTermsQueryBuilder;
@@ -43,8 +45,6 @@
 import org.elasticsearch.index.query.GeoPolygonQueryBuilder;
 import org.elasticsearch.index.query.GeoShapeQueryBuilder;
 import org.elasticsearch.index.query.GeohashCellQuery;
-import org.elasticsearch.index.query.HasChildQueryBuilder;
-import org.elasticsearch.index.query.HasParentQueryBuilder;
 import org.elasticsearch.index.query.IdsQueryBuilder;
 import org.elasticsearch.index.query.IndicesQueryBuilder;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
@@ -55,9 +55,9 @@
 import org.elasticsearch.index.query.MoreLikeThisQueryBuilder;
 import org.elasticsearch.index.query.MultiMatchQueryBuilder;
 import org.elasticsearch.index.query.NestedQueryBuilder;
-import org.elasticsearch.index.query.ParentIdQueryBuilder;
 import org.elasticsearch.index.query.PrefixQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryStringQueryBuilder;
 import org.elasticsearch.index.query.RangeQueryBuilder;
 import org.elasticsearch.index.query.RegexpQueryBuilder;
@@ -83,43 +83,38 @@
 import org.elasticsearch.index.query.functionscore.LinearDecayFunctionBuilder;
 import org.elasticsearch.index.query.functionscore.RandomScoreFunctionBuilder;
 import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder;
-import org.elasticsearch.index.query.functionscore.ScoreFunctionParser;
 import org.elasticsearch.index.query.functionscore.ScriptScoreFunctionBuilder;
 import org.elasticsearch.index.query.functionscore.WeightBuilder;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.plugins.SearchPlugin;
 import org.elasticsearch.plugins.SearchPlugin.AggregationSpec;
 import org.elasticsearch.plugins.SearchPlugin.FetchPhaseConstructionContext;
 import org.elasticsearch.plugins.SearchPlugin.PipelineAggregationSpec;
 import org.elasticsearch.plugins.SearchPlugin.QuerySpec;
 import org.elasticsearch.plugins.SearchPlugin.ScoreFunctionSpec;
+import org.elasticsearch.plugins.SearchPlugin.SearchExtSpec;
 import org.elasticsearch.plugins.SearchPlugin.SearchExtensionSpec;
-import org.elasticsearch.search.action.SearchTransportService;
+import org.elasticsearch.plugins.SearchPlugin.SuggesterSpec;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
-import org.elasticsearch.search.aggregations.Aggregator;
-import org.elasticsearch.search.aggregations.AggregatorParsers;
+import org.elasticsearch.search.aggregations.AggregatorFactories;
+import org.elasticsearch.search.aggregations.BaseAggregationBuilder;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.children.ChildrenAggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.children.InternalChildren;
+import org.elasticsearch.search.aggregations.bucket.adjacency.AdjacencyMatrixAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.adjacency.InternalAdjacencyMatrix;
 import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.filter.InternalFilter;
 import org.elasticsearch.search.aggregations.bucket.filters.FiltersAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.filters.InternalFilters;
 import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGridParser;
 import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoHashGrid;
 import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.global.InternalGlobal;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramParser;
 import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.histogram.HistogramParser;
 import org.elasticsearch.search.aggregations.bucket.histogram.InternalDateHistogram;
 import org.elasticsearch.search.aggregations.bucket.histogram.InternalHistogram;
 import org.elasticsearch.search.aggregations.bucket.missing.InternalMissing;
 import org.elasticsearch.search.aggregations.bucket.missing.MissingAggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.missing.MissingParser;
 import org.elasticsearch.search.aggregations.bucket.nested.InternalNested;
 import org.elasticsearch.search.aggregations.bucket.nested.InternalReverseNested;
 import org.elasticsearch.search.aggregations.bucket.nested.NestedAggregationBuilder;
@@ -127,24 +122,18 @@
 import org.elasticsearch.search.aggregations.bucket.range.InternalBinaryRange;
 import org.elasticsearch.search.aggregations.bucket.range.InternalRange;
 import org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.range.RangeParser;
 import org.elasticsearch.search.aggregations.bucket.range.date.DateRangeAggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.range.date.DateRangeParser;
 import org.elasticsearch.search.aggregations.bucket.range.date.InternalDateRange;
 import org.elasticsearch.search.aggregations.bucket.range.geodistance.GeoDistanceAggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.range.geodistance.GeoDistanceParser;
 import org.elasticsearch.search.aggregations.bucket.range.geodistance.InternalGeoDistance;
 import org.elasticsearch.search.aggregations.bucket.range.ip.IpRangeAggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.range.ip.IpRangeParser;
 import org.elasticsearch.search.aggregations.bucket.sampler.DiversifiedAggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.sampler.DiversifiedSamplerParser;
 import org.elasticsearch.search.aggregations.bucket.sampler.InternalSampler;
 import org.elasticsearch.search.aggregations.bucket.sampler.SamplerAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.sampler.UnmappedSampler;
 import org.elasticsearch.search.aggregations.bucket.significant.SignificantLongTerms;
 import org.elasticsearch.search.aggregations.bucket.significant.SignificantStringTerms;
 import org.elasticsearch.search.aggregations.bucket.significant.SignificantTermsAggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.significant.SignificantTermsParser;
 import org.elasticsearch.search.aggregations.bucket.significant.UnmappedSignificantTerms;
 import org.elasticsearch.search.aggregations.bucket.significant.heuristics.ChiSquare;
 import org.elasticsearch.search.aggregations.bucket.significant.heuristics.GND;
@@ -158,30 +147,21 @@
 import org.elasticsearch.search.aggregations.bucket.terms.LongTerms;
 import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.terms.TermsParser;
 import org.elasticsearch.search.aggregations.bucket.terms.UnmappedTerms;
 import org.elasticsearch.search.aggregations.metrics.avg.AvgAggregationBuilder;
-import org.elasticsearch.search.aggregations.metrics.avg.AvgParser;
 import org.elasticsearch.search.aggregations.metrics.avg.InternalAvg;
 import org.elasticsearch.search.aggregations.metrics.cardinality.CardinalityAggregationBuilder;
-import org.elasticsearch.search.aggregations.metrics.cardinality.CardinalityParser;
 import org.elasticsearch.search.aggregations.metrics.cardinality.InternalCardinality;
 import org.elasticsearch.search.aggregations.metrics.geobounds.GeoBoundsAggregationBuilder;
-import org.elasticsearch.search.aggregations.metrics.geobounds.GeoBoundsParser;
 import org.elasticsearch.search.aggregations.metrics.geobounds.InternalGeoBounds;
 import org.elasticsearch.search.aggregations.metrics.geocentroid.GeoCentroidAggregationBuilder;
-import org.elasticsearch.search.aggregations.metrics.geocentroid.GeoCentroidParser;
 import org.elasticsearch.search.aggregations.metrics.geocentroid.InternalGeoCentroid;
 import org.elasticsearch.search.aggregations.metrics.max.InternalMax;
 import org.elasticsearch.search.aggregations.metrics.max.MaxAggregationBuilder;
-import org.elasticsearch.search.aggregations.metrics.max.MaxParser;
 import org.elasticsearch.search.aggregations.metrics.min.InternalMin;
 import org.elasticsearch.search.aggregations.metrics.min.MinAggregationBuilder;
-import org.elasticsearch.search.aggregations.metrics.min.MinParser;
 import org.elasticsearch.search.aggregations.metrics.percentiles.PercentileRanksAggregationBuilder;
-import org.elasticsearch.search.aggregations.metrics.percentiles.PercentileRanksParser;
 import org.elasticsearch.search.aggregations.metrics.percentiles.PercentilesAggregationBuilder;
-import org.elasticsearch.search.aggregations.metrics.percentiles.PercentilesParser;
 import org.elasticsearch.search.aggregations.metrics.percentiles.hdr.InternalHDRPercentileRanks;
 import org.elasticsearch.search.aggregations.metrics.percentiles.hdr.InternalHDRPercentiles;
 import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.InternalTDigestPercentileRanks;
@@ -190,18 +170,14 @@
 import org.elasticsearch.search.aggregations.metrics.scripted.ScriptedMetricAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.stats.InternalStats;
 import org.elasticsearch.search.aggregations.metrics.stats.StatsAggregationBuilder;
-import org.elasticsearch.search.aggregations.metrics.stats.StatsParser;
 import org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStatsAggregationBuilder;
-import org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStatsParser;
 import org.elasticsearch.search.aggregations.metrics.stats.extended.InternalExtendedStats;
 import org.elasticsearch.search.aggregations.metrics.sum.InternalSum;
 import org.elasticsearch.search.aggregations.metrics.sum.SumAggregationBuilder;
-import org.elasticsearch.search.aggregations.metrics.sum.SumParser;
 import org.elasticsearch.search.aggregations.metrics.tophits.InternalTopHits;
 import org.elasticsearch.search.aggregations.metrics.tophits.TopHitsAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.valuecount.InternalValueCount;
 import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCountAggregationBuilder;
-import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCountParser;
 import org.elasticsearch.search.aggregations.pipeline.InternalSimpleValue;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.InternalBucketMetricValue;
@@ -242,7 +218,6 @@
 import org.elasticsearch.search.aggregations.pipeline.movavg.models.SimpleModel;
 import org.elasticsearch.search.aggregations.pipeline.serialdiff.SerialDiffPipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.serialdiff.SerialDiffPipelineAggregator;
-import org.elasticsearch.search.controller.SearchPhaseController;
 import org.elasticsearch.search.fetch.FetchPhase;
 import org.elasticsearch.search.fetch.FetchSubPhase;
 import org.elasticsearch.search.fetch.subphase.DocValueFieldsFetchSubPhase;
@@ -257,6 +232,7 @@
 import org.elasticsearch.search.fetch.subphase.highlight.Highlighter;
 import org.elasticsearch.search.fetch.subphase.highlight.PlainHighlighter;
 import org.elasticsearch.search.fetch.subphase.highlight.PostingsHighlighter;
+import org.elasticsearch.search.fetch.subphase.highlight.UnifiedHighlighter;
 import org.elasticsearch.search.rescore.QueryRescorerBuilder;
 import org.elasticsearch.search.rescore.RescoreBuilder;
 import org.elasticsearch.search.sort.FieldSortBuilder;
@@ -264,20 +240,19 @@
 import org.elasticsearch.search.sort.ScoreSortBuilder;
 import org.elasticsearch.search.sort.ScriptSortBuilder;
 import org.elasticsearch.search.sort.SortBuilder;
-import org.elasticsearch.search.suggest.Suggester;
-import org.elasticsearch.search.suggest.Suggesters;
 import org.elasticsearch.search.suggest.SuggestionBuilder;
-import org.elasticsearch.search.suggest.completion.CompletionSuggester;
+import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder;
 import org.elasticsearch.search.suggest.phrase.Laplace;
 import org.elasticsearch.search.suggest.phrase.LinearInterpolation;
-import org.elasticsearch.search.suggest.phrase.PhraseSuggester;
+import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder;
 import org.elasticsearch.search.suggest.phrase.SmoothingModel;
 import org.elasticsearch.search.suggest.phrase.StupidBackoff;
-import org.elasticsearch.search.suggest.term.TermSuggester;
+import org.elasticsearch.search.suggest.term.TermSuggestionBuilder;
 
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.function.Consumer;
 import java.util.function.Function;
 
@@ -287,19 +262,12 @@
 /**
  * Sets up things that can be done at search time like queries, aggregations, and suggesters.
  */
-public class SearchModule extends AbstractModule {
+public class SearchModule {
     public static final Setting<Integer> INDICES_MAX_CLAUSE_COUNT_SETTING = Setting.intSetting("indices.query.bool.max_clause_count",
             1024, 1, Integer.MAX_VALUE, Setting.Property.NodeScope);
 
     private final boolean transportClient;
     private final Map<String, Highlighter> highlighters;
-    private final Map<String, Suggester<?>> suggesters;
-    private final ParseFieldRegistry<ScoreFunctionParser<?>> scoreFunctionParserRegistry = new ParseFieldRegistry<>("score_function");
-    private final IndicesQueriesRegistry queryParserRegistry = new IndicesQueriesRegistry();
-    private final ParseFieldRegistry<Aggregator.Parser> aggregationParserRegistry = new ParseFieldRegistry<>("aggregation");
-    private final ParseFieldRegistry<PipelineAggregator.Parser> pipelineAggregationParserRegistry = new ParseFieldRegistry<>(
-            "pipline_aggregation");
-    private final AggregatorParsers aggregatorParsers = new AggregatorParsers(aggregationParserRegistry, pipelineAggregationParserRegistry);
     private final ParseFieldRegistry<SignificanceHeuristicParser> significanceHeuristicParserRegistry = new ParseFieldRegistry<>(
             "significance_heuristic");
     private final ParseFieldRegistry<MovAvgModel.AbstractModelParser> movingAverageModelParserRegistry = new ParseFieldRegistry<>(
@@ -308,13 +276,13 @@ public class SearchModule extends AbstractModule {
     private final List<FetchSubPhase> fetchSubPhases = new ArrayList<>();
 
     private final Settings settings;
-    private final List<Entry> namedWriteables = new ArrayList<>();
-    private final SearchRequestParsers searchRequestParsers;
+    private final List<NamedWriteableRegistry.Entry> namedWriteables = new ArrayList<>();
+    private final List<NamedXContentRegistry.Entry> namedXContents = new ArrayList<>();
 
     public SearchModule(Settings settings, boolean transportClient, List<SearchPlugin> plugins) {
         this.settings = settings;
         this.transportClient = transportClient;
-        suggesters = setupSuggesters(plugins);
+        registerSuggesters(plugins);
         highlighters = setupHighlighters(settings, plugins);
         registerScoreFunctions(plugins);
         registerQueryParsers(plugins);
@@ -326,24 +294,16 @@ public SearchModule(Settings settings, boolean transportClient, List<SearchPlugi
         registerAggregations(plugins);
         registerPipelineAggregations(plugins);
         registerFetchSubPhases(plugins);
+        registerSearchExts(plugins);
         registerShapes();
-        searchRequestParsers = new SearchRequestParsers(queryParserRegistry, aggregatorParsers, getSuggesters());
     }
 
-    public List<Entry> getNamedWriteables() {
+    public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
         return namedWriteables;
     }
 
-    public Suggesters getSuggesters() {
-        return new Suggesters(suggesters);
-    }
-
-    public IndicesQueriesRegistry getQueryParserRegistry() {
-        return queryParserRegistry;
-    }
-
-    public SearchRequestParsers getSearchRequestParsers() {
-        return searchRequestParsers;
+    public List<NamedXContentRegistry.Entry> getNamedXContents() {
+        return namedXContents;
     }
 
     /**
@@ -367,114 +327,101 @@ public ParseFieldRegistry<MovAvgModel.AbstractModelParser> getMovingAverageModel
         return movingAverageModelParserRegistry;
     }
 
-    /**
-     * Parsers for {@link AggregationBuilder}s and {@link PipelineAggregationBuilder}s.
-     */
-    public AggregatorParsers getAggregatorParsers() {
-        return aggregatorParsers;
-    }
-
-
-    @Override
-    protected void configure() {
-        if (false == transportClient) {
-            bind(IndicesQueriesRegistry.class).toInstance(queryParserRegistry);
-            bind(SearchRequestParsers.class).toInstance(searchRequestParsers);
-            configureSearch();
-        }
-    }
-
     private void registerAggregations(List<SearchPlugin> plugins) {
-        registerAggregation(new AggregationSpec(AvgAggregationBuilder.NAME, AvgAggregationBuilder::new, new AvgParser())
+        registerAggregation(new AggregationSpec(AvgAggregationBuilder.NAME, AvgAggregationBuilder::new, AvgAggregationBuilder::parse)
                 .addResultReader(InternalAvg::new));
-        registerAggregation(new AggregationSpec(SumAggregationBuilder.NAME, SumAggregationBuilder::new, new SumParser())
+        registerAggregation(new AggregationSpec(SumAggregationBuilder.NAME, SumAggregationBuilder::new, SumAggregationBuilder::parse)
                 .addResultReader(InternalSum::new));
-        registerAggregation(new AggregationSpec(MinAggregationBuilder.NAME, MinAggregationBuilder::new, new MinParser())
+        registerAggregation(new AggregationSpec(MinAggregationBuilder.NAME, MinAggregationBuilder::new, MinAggregationBuilder::parse)
                 .addResultReader(InternalMin::new));
-        registerAggregation(new AggregationSpec(MaxAggregationBuilder.NAME, MaxAggregationBuilder::new, new MaxParser())
+        registerAggregation(new AggregationSpec(MaxAggregationBuilder.NAME, MaxAggregationBuilder::new, MaxAggregationBuilder::parse)
                 .addResultReader(InternalMax::new));
-        registerAggregation(new AggregationSpec(StatsAggregationBuilder.NAME, StatsAggregationBuilder::new, new StatsParser())
+        registerAggregation(new AggregationSpec(StatsAggregationBuilder.NAME, StatsAggregationBuilder::new, StatsAggregationBuilder::parse)
                 .addResultReader(InternalStats::new));
         registerAggregation(new AggregationSpec(ExtendedStatsAggregationBuilder.NAME, ExtendedStatsAggregationBuilder::new,
-                new ExtendedStatsParser()).addResultReader(InternalExtendedStats::new));
+                ExtendedStatsAggregationBuilder::parse).addResultReader(InternalExtendedStats::new));
         registerAggregation(new AggregationSpec(ValueCountAggregationBuilder.NAME, ValueCountAggregationBuilder::new,
-                new ValueCountParser()).addResultReader(InternalValueCount::new));
+                ValueCountAggregationBuilder::parse).addResultReader(InternalValueCount::new));
         registerAggregation(new AggregationSpec(PercentilesAggregationBuilder.NAME, PercentilesAggregationBuilder::new,
-                new PercentilesParser())
+                PercentilesAggregationBuilder::parse)
                     .addResultReader(InternalTDigestPercentiles.NAME, InternalTDigestPercentiles::new)
                     .addResultReader(InternalHDRPercentiles.NAME, InternalHDRPercentiles::new));
         registerAggregation(new AggregationSpec(PercentileRanksAggregationBuilder.NAME, PercentileRanksAggregationBuilder::new,
-                new PercentileRanksParser())
+                PercentileRanksAggregationBuilder::parse)
                         .addResultReader(InternalTDigestPercentileRanks.NAME, InternalTDigestPercentileRanks::new)
                         .addResultReader(InternalHDRPercentileRanks.NAME, InternalHDRPercentileRanks::new));
         registerAggregation(new AggregationSpec(CardinalityAggregationBuilder.NAME, CardinalityAggregationBuilder::new,
-                new CardinalityParser()).addResultReader(InternalCardinality::new));
+                CardinalityAggregationBuilder::parse).addResultReader(InternalCardinality::new));
         registerAggregation(new AggregationSpec(GlobalAggregationBuilder.NAME, GlobalAggregationBuilder::new,
                 GlobalAggregationBuilder::parse).addResultReader(InternalGlobal::new));
-        registerAggregation(new AggregationSpec(MissingAggregationBuilder.NAME, MissingAggregationBuilder::new, new MissingParser())
-                .addResultReader(InternalMissing::new));
+        registerAggregation(new AggregationSpec(MissingAggregationBuilder.NAME, MissingAggregationBuilder::new,
+                MissingAggregationBuilder::parse).addResultReader(InternalMissing::new));
         registerAggregation(new AggregationSpec(FilterAggregationBuilder.NAME, FilterAggregationBuilder::new,
                 FilterAggregationBuilder::parse).addResultReader(InternalFilter::new));
         registerAggregation(new AggregationSpec(FiltersAggregationBuilder.NAME, FiltersAggregationBuilder::new,
                 FiltersAggregationBuilder::parse).addResultReader(InternalFilters::new));
+        registerAggregation(new AggregationSpec(AdjacencyMatrixAggregationBuilder.NAME, AdjacencyMatrixAggregationBuilder::new,
+                AdjacencyMatrixAggregationBuilder.getParser()).addResultReader(InternalAdjacencyMatrix::new));
         registerAggregation(new AggregationSpec(SamplerAggregationBuilder.NAME, SamplerAggregationBuilder::new,
                 SamplerAggregationBuilder::parse)
                     .addResultReader(InternalSampler.NAME, InternalSampler::new)
                     .addResultReader(UnmappedSampler.NAME, UnmappedSampler::new));
         registerAggregation(new AggregationSpec(DiversifiedAggregationBuilder.NAME, DiversifiedAggregationBuilder::new,
-                new DiversifiedSamplerParser())
+                DiversifiedAggregationBuilder::parse)
                     /* Reuses result readers from SamplerAggregator*/);
-        registerAggregation(new AggregationSpec(TermsAggregationBuilder.NAME, TermsAggregationBuilder::new, new TermsParser())
+        registerAggregation(new AggregationSpec(TermsAggregationBuilder.NAME, TermsAggregationBuilder::new,
+                TermsAggregationBuilder::parse)
                     .addResultReader(StringTerms.NAME, StringTerms::new)
                     .addResultReader(UnmappedTerms.NAME, UnmappedTerms::new)
                     .addResultReader(LongTerms.NAME, LongTerms::new)
                     .addResultReader(DoubleTerms.NAME, DoubleTerms::new));
         registerAggregation(new AggregationSpec(SignificantTermsAggregationBuilder.NAME, SignificantTermsAggregationBuilder::new,
-                new SignificantTermsParser(significanceHeuristicParserRegistry, queryParserRegistry))
+                SignificantTermsAggregationBuilder.getParser(significanceHeuristicParserRegistry))
                     .addResultReader(SignificantStringTerms.NAME, SignificantStringTerms::new)
                     .addResultReader(SignificantLongTerms.NAME, SignificantLongTerms::new)
                     .addResultReader(UnmappedSignificantTerms.NAME, UnmappedSignificantTerms::new));
         registerAggregation(new AggregationSpec(RangeAggregationBuilder.NAME, RangeAggregationBuilder::new,
-                new RangeParser()).addResultReader(InternalRange::new));
-        registerAggregation(new AggregationSpec(DateRangeAggregationBuilder.NAME, DateRangeAggregationBuilder::new, new DateRangeParser())
-                .addResultReader(InternalDateRange::new));
-        registerAggregation(new AggregationSpec(IpRangeAggregationBuilder.NAME, IpRangeAggregationBuilder::new, new IpRangeParser())
-                .addResultReader(InternalBinaryRange::new));
-        registerAggregation(new AggregationSpec(HistogramAggregationBuilder.NAME, HistogramAggregationBuilder::new, new HistogramParser())
-                .addResultReader(InternalHistogram::new));
+                RangeAggregationBuilder::parse).addResultReader(InternalRange::new));
+        registerAggregation(new AggregationSpec(DateRangeAggregationBuilder.NAME, DateRangeAggregationBuilder::new,
+                DateRangeAggregationBuilder::parse).addResultReader(InternalDateRange::new));
+        registerAggregation(new AggregationSpec(IpRangeAggregationBuilder.NAME, IpRangeAggregationBuilder::new,
+                IpRangeAggregationBuilder::parse).addResultReader(InternalBinaryRange::new));
+        registerAggregation(new AggregationSpec(HistogramAggregationBuilder.NAME, HistogramAggregationBuilder::new,
+                HistogramAggregationBuilder::parse).addResultReader(InternalHistogram::new));
         registerAggregation(new AggregationSpec(DateHistogramAggregationBuilder.NAME, DateHistogramAggregationBuilder::new,
-                new DateHistogramParser()).addResultReader(InternalDateHistogram::new));
+                DateHistogramAggregationBuilder::parse).addResultReader(InternalDateHistogram::new));
         registerAggregation(new AggregationSpec(GeoDistanceAggregationBuilder.NAME, GeoDistanceAggregationBuilder::new,
-                new GeoDistanceParser()).addResultReader(InternalGeoDistance::new));
-        registerAggregation(new AggregationSpec(GeoGridAggregationBuilder.NAME, GeoGridAggregationBuilder::new, new GeoHashGridParser())
-                .addResultReader(InternalGeoHashGrid::new));
+                GeoDistanceAggregationBuilder::parse).addResultReader(InternalGeoDistance::new));
+        registerAggregation(new AggregationSpec(GeoGridAggregationBuilder.NAME, GeoGridAggregationBuilder::new,
+                GeoGridAggregationBuilder::parse).addResultReader(InternalGeoHashGrid::new));
         registerAggregation(new AggregationSpec(NestedAggregationBuilder.NAME, NestedAggregationBuilder::new,
                 NestedAggregationBuilder::parse).addResultReader(InternalNested::new));
         registerAggregation(new AggregationSpec(ReverseNestedAggregationBuilder.NAME, ReverseNestedAggregationBuilder::new,
                 ReverseNestedAggregationBuilder::parse).addResultReader(InternalReverseNested::new));
         registerAggregation(new AggregationSpec(TopHitsAggregationBuilder.NAME, TopHitsAggregationBuilder::new,
                 TopHitsAggregationBuilder::parse).addResultReader(InternalTopHits::new));
-        registerAggregation(new AggregationSpec(GeoBoundsAggregationBuilder.NAME, GeoBoundsAggregationBuilder::new, new GeoBoundsParser())
-                .addResultReader(InternalGeoBounds::new));
+        registerAggregation(new AggregationSpec(GeoBoundsAggregationBuilder.NAME, GeoBoundsAggregationBuilder::new,
+                GeoBoundsAggregationBuilder::parse).addResultReader(InternalGeoBounds::new));
         registerAggregation(new AggregationSpec(GeoCentroidAggregationBuilder.NAME, GeoCentroidAggregationBuilder::new,
-                new GeoCentroidParser()).addResultReader(InternalGeoCentroid::new));
+                GeoCentroidAggregationBuilder::parse).addResultReader(InternalGeoCentroid::new));
         registerAggregation(new AggregationSpec(ScriptedMetricAggregationBuilder.NAME, ScriptedMetricAggregationBuilder::new,
                 ScriptedMetricAggregationBuilder::parse).addResultReader(InternalScriptedMetric::new));
-        registerAggregation(new AggregationSpec(ChildrenAggregationBuilder.NAME, ChildrenAggregationBuilder::new,
-                ChildrenAggregationBuilder::parse).addResultReader(InternalChildren::new));
-
         registerFromPlugin(plugins, SearchPlugin::getAggregations, this::registerAggregation);
     }
 
     private void registerAggregation(AggregationSpec spec) {
         if (false == transportClient) {
-            aggregationParserRegistry.register(spec.getParser(), spec.getName());
+            namedXContents.add(new NamedXContentRegistry.Entry(BaseAggregationBuilder.class, spec.getName(), (p, c) -> {
+                AggregatorFactories.AggParseContext context = (AggregatorFactories.AggParseContext) c;
+                return spec.getParser().parse(context.name, context.queryParseContext);
+            }));
         }
-        namedWriteables.add(new Entry(AggregationBuilder.class, spec.getName().getPreferredName(), spec.getReader()));
+        namedWriteables.add(
+                new NamedWriteableRegistry.Entry(AggregationBuilder.class, spec.getName().getPreferredName(), spec.getReader()));
         for (Map.Entry<String, Writeable.Reader<? extends InternalAggregation>> t : spec.getResultReaders().entrySet()) {
             String writeableName = t.getKey();
             Writeable.Reader<? extends InternalAggregation> internalReader = t.getValue();
-            namedWriteables.add(new Entry(InternalAggregation.class, writeableName, internalReader));
+            namedWriteables.add(new NamedWriteableRegistry.Entry(InternalAggregation.class, writeableName, internalReader));
         }
     }
 
@@ -561,22 +508,21 @@ private void registerPipelineAggregations(List<SearchPlugin> plugins) {
 
     private void registerPipelineAggregation(PipelineAggregationSpec spec) {
         if (false == transportClient) {
-            pipelineAggregationParserRegistry.register(spec.getParser(), spec.getName());
+            namedXContents.add(new NamedXContentRegistry.Entry(BaseAggregationBuilder.class, spec.getName(), (p, c) -> {
+                AggregatorFactories.AggParseContext context = (AggregatorFactories.AggParseContext) c;
+                return spec.getParser().parse(context.name, context.queryParseContext);
+            }));
         }
-        namedWriteables.add(new Entry(PipelineAggregationBuilder.class, spec.getName().getPreferredName(), spec.getReader()));
-        namedWriteables.add(new Entry(PipelineAggregator.class, spec.getName().getPreferredName(), spec.getAggregatorReader()));
+        namedWriteables.add(
+                new NamedWriteableRegistry.Entry(PipelineAggregationBuilder.class, spec.getName().getPreferredName(), spec.getReader()));
+        namedWriteables.add(
+                new NamedWriteableRegistry.Entry(PipelineAggregator.class, spec.getName().getPreferredName(), spec.getAggregatorReader()));
         for (Map.Entry<String, Writeable.Reader<? extends InternalAggregation>> resultReader : spec.getResultReaders().entrySet()) {
-            namedWriteables.add(new Entry(InternalAggregation.class, resultReader.getKey(), resultReader.getValue()));
+            namedWriteables
+                    .add(new NamedWriteableRegistry.Entry(InternalAggregation.class, resultReader.getKey(), resultReader.getValue()));
         }
     }
 
-    protected void configureSearch() {
-        // configure search private classes...
-        bind(SearchPhaseController.class).asEagerSingleton();
-        bind(FetchPhase.class).toInstance(new FetchPhase(fetchSubPhases));
-        bind(SearchTransportService.class).asEagerSingleton();
-    }
-
     private void registerShapes() {
         if (ShapesAvailability.JTS_AVAILABLE && ShapesAvailability.SPATIAL4J_AVAILABLE) {
             ShapeBuilders.register(namedWriteables);
@@ -584,14 +530,14 @@ private void registerShapes() {
     }
 
     private void registerRescorers() {
-        namedWriteables.add(new Entry(RescoreBuilder.class, QueryRescorerBuilder.NAME, QueryRescorerBuilder::new));
+        namedWriteables.add(new NamedWriteableRegistry.Entry(RescoreBuilder.class, QueryRescorerBuilder.NAME, QueryRescorerBuilder::new));
     }
 
     private void registerSorts() {
-        namedWriteables.add(new Entry(SortBuilder.class, GeoDistanceSortBuilder.NAME, GeoDistanceSortBuilder::new));
-        namedWriteables.add(new Entry(SortBuilder.class, ScoreSortBuilder.NAME, ScoreSortBuilder::new));
-        namedWriteables.add(new Entry(SortBuilder.class, ScriptSortBuilder.NAME, ScriptSortBuilder::new));
-        namedWriteables.add(new Entry(SortBuilder.class, FieldSortBuilder.NAME, FieldSortBuilder::new));
+        namedWriteables.add(new NamedWriteableRegistry.Entry(SortBuilder.class, GeoDistanceSortBuilder.NAME, GeoDistanceSortBuilder::new));
+        namedWriteables.add(new NamedWriteableRegistry.Entry(SortBuilder.class, ScoreSortBuilder.NAME, ScoreSortBuilder::new));
+        namedWriteables.add(new NamedWriteableRegistry.Entry(SortBuilder.class, ScriptSortBuilder.NAME, ScriptSortBuilder::new));
+        namedWriteables.add(new NamedWriteableRegistry.Entry(SortBuilder.class, FieldSortBuilder.NAME, FieldSortBuilder::new));
     }
 
     private <T> void registerFromPlugin(List<SearchPlugin> plugins, Function<SearchPlugin, List<T>> producer, Consumer<T> consumer) {
@@ -603,28 +549,26 @@ private <T> void registerFromPlugin(List<SearchPlugin> plugins, Function<SearchP
     }
 
     public static void registerSmoothingModels(List<Entry> namedWriteables) {
-        namedWriteables.add(new Entry(SmoothingModel.class, Laplace.NAME, Laplace::new));
-        namedWriteables.add(new Entry(SmoothingModel.class, LinearInterpolation.NAME, LinearInterpolation::new));
-        namedWriteables.add(new Entry(SmoothingModel.class, StupidBackoff.NAME, StupidBackoff::new));
+        namedWriteables.add(new NamedWriteableRegistry.Entry(SmoothingModel.class, Laplace.NAME, Laplace::new));
+        namedWriteables.add(new NamedWriteableRegistry.Entry(SmoothingModel.class, LinearInterpolation.NAME, LinearInterpolation::new));
+        namedWriteables.add(new NamedWriteableRegistry.Entry(SmoothingModel.class, StupidBackoff.NAME, StupidBackoff::new));
     }
 
-    private Map<String, Suggester<?>> setupSuggesters(List<SearchPlugin> plugins) {
+    private void registerSuggesters(List<SearchPlugin> plugins) {
         registerSmoothingModels(namedWriteables);
 
-        // Suggester<?> is weird - it is both a Parser and a reader....
-        NamedRegistry<Suggester<?>> suggesters = new NamedRegistry<Suggester<?>>("suggester") {
-            @Override
-            public void register(String name, Suggester<?> t) {
-                super.register(name, t);
-                namedWriteables.add(new Entry(SuggestionBuilder.class, name, t));
-            }
-        };
-        suggesters.register("phrase", PhraseSuggester.INSTANCE);
-        suggesters.register("term", TermSuggester.INSTANCE);
-        suggesters.register("completion", CompletionSuggester.INSTANCE);
+        registerSuggester(new SuggesterSpec<>("term", TermSuggestionBuilder::new, TermSuggestionBuilder::fromXContent));
+        registerSuggester(new SuggesterSpec<>("phrase", PhraseSuggestionBuilder::new, PhraseSuggestionBuilder::fromXContent));
+        registerSuggester(new SuggesterSpec<>("completion", CompletionSuggestionBuilder::new, CompletionSuggestionBuilder::fromXContent));
 
-        suggesters.extractAndRegister(plugins, SearchPlugin::getSuggesters);
-        return unmodifiableMap(suggesters.getRegistry());
+        registerFromPlugin(plugins, SearchPlugin::getSuggesters, this::registerSuggester);
+    }
+
+    private void registerSuggester(SuggesterSpec<?> suggester) {
+        namedWriteables.add(new NamedWriteableRegistry.Entry(
+                SuggestionBuilder.class, suggester.getName().getPreferredName(), suggester.getReader()));
+        namedXContents.add(new NamedXContentRegistry.Entry(SuggestionBuilder.class, suggester.getName(),
+                suggester.getParser()));
     }
 
     private Map<String, Highlighter> setupHighlighters(Settings settings, List<SearchPlugin> plugins) {
@@ -632,7 +576,7 @@ private Map<String, Highlighter> setupHighlighters(Settings settings, List<Searc
         highlighters.register("fvh",  new FastVectorHighlighter(settings));
         highlighters.register("plain", new PlainHighlighter());
         highlighters.register("postings", new PostingsHighlighter());
-
+        highlighters.register("unified", new UnifiedHighlighter());
         highlighters.extractAndRegister(plugins, SearchPlugin::getHighlighters);
 
         return unmodifiableMap(highlighters.getRegistry());
@@ -654,14 +598,18 @@ private void registerScoreFunctions(List<SearchPlugin> plugins) {
 
         //weight doesn't have its own parser, so every function supports it out of the box.
         //Can be a single function too when not associated to any other function, which is why it needs to be registered manually here.
-        namedWriteables.add(new Entry(ScoreFunctionBuilder.class, WeightBuilder.NAME, WeightBuilder::new));
+        namedWriteables.add(new NamedWriteableRegistry.Entry(ScoreFunctionBuilder.class, WeightBuilder.NAME, WeightBuilder::new));
 
         registerFromPlugin(plugins, SearchPlugin::getScoreFunctions, this::registerScoreFunction);
     }
 
     private void registerScoreFunction(ScoreFunctionSpec<?> scoreFunction) {
-        scoreFunctionParserRegistry.register(scoreFunction.getParser(), scoreFunction.getName());
-        namedWriteables.add(new Entry(ScoreFunctionBuilder.class, scoreFunction.getName().getPreferredName(), scoreFunction.getReader()));
+        namedWriteables.add(new NamedWriteableRegistry.Entry(
+                ScoreFunctionBuilder.class, scoreFunction.getName().getPreferredName(), scoreFunction.getReader()));
+        // TODO remove funky contexts
+        namedXContents.add(new NamedXContentRegistry.Entry(
+                ScoreFunctionBuilder.class, scoreFunction.getName(),
+                (XContentParser p, Object c) -> scoreFunction.getParser().fromXContent((QueryParseContext) c)));
     }
 
     private void registerValueFormats() {
@@ -677,7 +625,7 @@ private void registerValueFormats() {
      * Register a new ValueFormat.
      */
     private void registerValueFormat(String name, Writeable.Reader<? extends DocValueFormat> reader) {
-        namedWriteables.add(new Entry(DocValueFormat.class, name, reader));
+        namedWriteables.add(new NamedWriteableRegistry.Entry(DocValueFormat.class, name, reader));
     }
 
     private void registerSignificanceHeuristics(List<SearchPlugin> plugins) {
@@ -693,7 +641,8 @@ private void registerSignificanceHeuristics(List<SearchPlugin> plugins) {
 
     private void registerSignificanceHeuristic(SearchExtensionSpec<SignificanceHeuristic, SignificanceHeuristicParser> heuristic) {
         significanceHeuristicParserRegistry.register(heuristic.getParser(), heuristic.getName());
-        namedWriteables.add(new Entry(SignificanceHeuristic.class, heuristic.getName().getPreferredName(), heuristic.getReader()));
+        namedWriteables.add(new NamedWriteableRegistry.Entry(SignificanceHeuristic.class, heuristic.getName().getPreferredName(),
+                heuristic.getReader()));
     }
 
     private void registerMovingAverageModels(List<SearchPlugin> plugins) {
@@ -708,7 +657,8 @@ private void registerMovingAverageModels(List<SearchPlugin> plugins) {
 
     private void registerMovingAverageModel(SearchExtensionSpec<MovAvgModel, MovAvgModel.AbstractModelParser> movAvgModel) {
         movingAverageModelParserRegistry.register(movAvgModel.getParser(), movAvgModel.getName());
-        namedWriteables.add(new Entry(MovAvgModel.class, movAvgModel.getName().getPreferredName(), movAvgModel.getReader()));
+        namedWriteables.add(
+                new NamedWriteableRegistry.Entry(MovAvgModel.class, movAvgModel.getName().getPreferredName(), movAvgModel.getReader()));
     }
 
     private void registerFetchSubPhases(List<SearchPlugin> plugins) {
@@ -725,6 +675,15 @@ private void registerFetchSubPhases(List<SearchPlugin> plugins) {
         registerFromPlugin(plugins, p -> p.getFetchSubPhases(context), this::registerFetchSubPhase);
     }
 
+    private void registerSearchExts(List<SearchPlugin> plugins) {
+        registerFromPlugin(plugins, SearchPlugin::getSearchExts, this::registerSearchExt);
+    }
+
+    private void registerSearchExt(SearchExtSpec<?> spec) {
+        namedXContents.add(new NamedXContentRegistry.Entry(SearchExtBuilder.class, spec.getName(), spec.getParser()));
+        namedWriteables.add(new NamedWriteableRegistry.Entry(SearchExtBuilder.class, spec.getName().getPreferredName(), spec.getReader()));
+    }
+
     private void registerFetchSubPhase(FetchSubPhase subPhase) {
         Class<?> subPhaseClass = subPhase.getClass();
         if (fetchSubPhases.stream().anyMatch(p -> p.getClass().equals(subPhaseClass))) {
@@ -740,8 +699,6 @@ private void registerQueryParsers(List<SearchPlugin> plugins) {
                 MatchPhrasePrefixQueryBuilder::fromXContent));
         registerQuery(new QuerySpec<>(MultiMatchQueryBuilder.NAME, MultiMatchQueryBuilder::new, MultiMatchQueryBuilder::fromXContent));
         registerQuery(new QuerySpec<>(NestedQueryBuilder.NAME, NestedQueryBuilder::new, NestedQueryBuilder::fromXContent));
-        registerQuery(new QuerySpec<>(HasChildQueryBuilder.NAME, HasChildQueryBuilder::new, HasChildQueryBuilder::fromXContent));
-        registerQuery(new QuerySpec<>(HasParentQueryBuilder.NAME, HasParentQueryBuilder::new, HasParentQueryBuilder::fromXContent));
         registerQuery(new QuerySpec<>(DisMaxQueryBuilder.NAME, DisMaxQueryBuilder::new, DisMaxQueryBuilder::fromXContent));
         registerQuery(new QuerySpec<>(IdsQueryBuilder.NAME, IdsQueryBuilder::new, IdsQueryBuilder::fromXContent));
         registerQuery(new QuerySpec<>(MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new, MatchAllQueryBuilder::fromXContent));
@@ -777,7 +734,7 @@ private void registerQueryParsers(List<SearchPlugin> plugins) {
         registerQuery(
                 new QuerySpec<>(SpanMultiTermQueryBuilder.NAME, SpanMultiTermQueryBuilder::new, SpanMultiTermQueryBuilder::fromXContent));
         registerQuery(new QuerySpec<>(FunctionScoreQueryBuilder.NAME, FunctionScoreQueryBuilder::new,
-                c -> FunctionScoreQueryBuilder.fromXContent(scoreFunctionParserRegistry, c)));
+                FunctionScoreQueryBuilder::fromXContent));
         registerQuery(
                 new QuerySpec<>(SimpleQueryStringBuilder.NAME, SimpleQueryStringBuilder::new, SimpleQueryStringBuilder::fromXContent));
         registerQuery(new QuerySpec<>(TypeQueryBuilder.NAME, TypeQueryBuilder::new, TypeQueryBuilder::fromXContent));
@@ -791,7 +748,6 @@ private void registerQueryParsers(List<SearchPlugin> plugins) {
         registerQuery(new QuerySpec<>(GeoPolygonQueryBuilder.NAME, GeoPolygonQueryBuilder::new, GeoPolygonQueryBuilder::fromXContent));
         registerQuery(new QuerySpec<>(ExistsQueryBuilder.NAME, ExistsQueryBuilder::new, ExistsQueryBuilder::fromXContent));
         registerQuery(new QuerySpec<>(MatchNoneQueryBuilder.NAME, MatchNoneQueryBuilder::new, MatchNoneQueryBuilder::fromXContent));
-        registerQuery(new QuerySpec<>(ParentIdQueryBuilder.NAME, ParentIdQueryBuilder::new, ParentIdQueryBuilder::fromXContent));
 
         if (ShapesAvailability.JTS_AVAILABLE && ShapesAvailability.SPATIAL4J_AVAILABLE) {
             registerQuery(new QuerySpec<>(GeoShapeQueryBuilder.NAME, GeoShapeQueryBuilder::new, GeoShapeQueryBuilder::fromXContent));
@@ -801,7 +757,13 @@ private void registerQueryParsers(List<SearchPlugin> plugins) {
     }
 
     private void registerQuery(QuerySpec<?> spec) {
-        queryParserRegistry.register(spec.getParser(), spec.getName());
-        namedWriteables.add(new Entry(QueryBuilder.class, spec.getName().getPreferredName(), spec.getReader()));
+        namedWriteables.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, spec.getName().getPreferredName(), spec.getReader()));
+        // Using Optional here is fairly horrible, but in master we don't have to because the builders return QueryBuilder.
+        namedXContents.add(new NamedXContentRegistry.Entry(Optional.class, spec.getName(),
+                (XContentParser p, Object c) -> (Optional) spec.getParser().fromXContent((QueryParseContext) c)));
+    }
+
+    public FetchPhase getFetchPhase() {
+        return new FetchPhase(fetchSubPhases);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/SearchParseElement.java b/core/src/main/java/org/elasticsearch/search/SearchParseElement.java
deleted file mode 100644
index 9bf680deb5511..0000000000000
--- a/core/src/main/java/org/elasticsearch/search/SearchParseElement.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search;
-
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.search.internal.SearchContext;
-
-/**
- *
- */
-public interface SearchParseElement {
-
-    void parse(XContentParser parser, SearchContext context) throws Exception;
-}
diff --git a/core/src/main/java/org/elasticsearch/search/SearchParseException.java b/core/src/main/java/org/elasticsearch/search/SearchParseException.java
index c0a9a3702701d..b1c5d284a6439 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchParseException.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchParseException.java
@@ -75,12 +75,11 @@ public RestStatus status() {
     }
 
     @Override
-    protected void innerToXContent(XContentBuilder builder, Params params) throws IOException {
+    protected void metadataToXContent(XContentBuilder builder, Params params) throws IOException {
         if (lineNumber != UNKNOWN_POSITION) {
             builder.field("line", lineNumber);
             builder.field("col", columnNumber);
         }
-        super.innerToXContent(builder, params);
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/search/SearchPhase.java b/core/src/main/java/org/elasticsearch/search/SearchPhase.java
index 48c041f12f53f..33260706b3cb2 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchPhase.java
@@ -21,22 +21,18 @@
 
 import org.elasticsearch.search.internal.SearchContext;
 
-import java.util.Collections;
-import java.util.Map;
-
 /**
- *
+ * Represents a phase of a search request e.g. query, fetch etc.
  */
 public interface SearchPhase {
 
-    default Map<String, ? extends SearchParseElement> parseElements() {
-        return Collections.emptyMap();
-    }
-
     /**
      * Performs pre processing of the search context before the execute.
      */
     void preProcess(SearchContext context);
 
+    /**
+     * Executes the search phase
+     */
     void execute(SearchContext context);
 }
diff --git a/core/src/main/java/org/elasticsearch/search/SearchPhaseResult.java b/core/src/main/java/org/elasticsearch/search/SearchPhaseResult.java
index 067761b9b71fb..ede9f525a5a14 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchPhaseResult.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchPhaseResult.java
@@ -20,15 +20,63 @@
 package org.elasticsearch.search;
 
 import org.elasticsearch.common.io.stream.Streamable;
+import org.elasticsearch.search.fetch.FetchSearchResult;
+import org.elasticsearch.search.query.QuerySearchResult;
+import org.elasticsearch.transport.TransportResponse;
 
 /**
- *
+ * This class is a base class for all search releated results. It contains the shard target it
+ * was executed against, a shard index used to reference the result on the coordinating node
+ * and a request ID that is used to reference the request context on the executing node. The
+ * request ID is particularly important since it is used to reference and maintain a context
+ * across search phases to ensure the same point in time snapshot is used for querying and
+ * fetching etc.
  */
-public interface SearchPhaseResult extends Streamable {
+public abstract class SearchPhaseResult extends TransportResponse implements Streamable {
+
+    private SearchShardTarget searchShardTarget;
+    private int shardIndex = -1;
+    protected long requestId;
+
+    /**
+     * Returns the results request ID that is used to reference the search context on the executing
+     * node
+     */
+    public long getRequestId() {
+        return requestId;
+    }
+
+    /**
+     * Returns the shard index in the context of the currently executing search request that is
+     * used for accounting on the coordinating node
+     */
+    public int getShardIndex() {
+        assert shardIndex != -1 : "shardIndex is not set";
+        return shardIndex;
+    }
+
+    public SearchShardTarget getSearchShardTarget() {
+        return searchShardTarget;
+    }
+
+    public void setSearchShardTarget(SearchShardTarget shardTarget) {
+        this.searchShardTarget = shardTarget;
+    }
 
-    long id();
+    public void setShardIndex(int shardIndex) {
+        assert shardIndex >= 0 : "shardIndex must be >= 0 but was: " + shardIndex;
+        this.shardIndex = shardIndex;
+    }
 
-    SearchShardTarget shardTarget();
+    /**
+     * Returns the query result iff it's included in this response otherwise <code>null</code>
+     */
+    public QuerySearchResult queryResult() {
+        return null;
+    }
 
-    void shardTarget(SearchShardTarget shardTarget);
+    /**
+     * Returns the fetch result iff it's included in this response otherwise <code>null</code>
+     */
+    public FetchSearchResult fetchResult() { return null; }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/SearchRequestParsers.java b/core/src/main/java/org/elasticsearch/search/SearchRequestParsers.java
deleted file mode 100644
index 83eebd125d8e5..0000000000000
--- a/core/src/main/java/org/elasticsearch/search/SearchRequestParsers.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search;
-
-import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
-import org.elasticsearch.search.aggregations.AggregatorParsers;
-import org.elasticsearch.search.suggest.Suggesters;
-
-/**
- * A container for all parsers used to parse
- * {@link org.elasticsearch.action.search.SearchRequest} objects from a rest request.
- */
-public class SearchRequestParsers {
-    // TODO: this class should be renamed to SearchRequestParser, and all the parse
-    // methods split across RestSearchAction and SearchSourceBuilder should be moved here
-    // TODO: make all members private once parsing functions are moved here
-
-    // TODO: IndicesQueriesRegistry should be removed and just have the map of query parsers here
-    /**
-     * Query parsers that may be used in search requests.
-     * @see org.elasticsearch.index.query.QueryParseContext
-     * @see org.elasticsearch.search.builder.SearchSourceBuilder#fromXContent(QueryParseContext, AggregatorParsers, Suggesters)
-     */
-    public final IndicesQueriesRegistry queryParsers;
-
-    // TODO: AggregatorParsers should be removed and the underlying maps of agg
-    // and pipeline agg parsers should be here
-    /**
-     * Agg and pipeline agg parsers that may be used in search requests.
-     * @see org.elasticsearch.search.builder.SearchSourceBuilder#fromXContent(QueryParseContext, AggregatorParsers, Suggesters)
-     */
-    public final AggregatorParsers aggParsers;
-
-    // TODO: Suggesters should be removed and the underlying map moved here
-    /**
-     * Suggesters that may be used in search requests.
-     * @see org.elasticsearch.search.builder.SearchSourceBuilder#fromXContent(QueryParseContext, AggregatorParsers, Suggesters)
-     */
-    public final Suggesters suggesters;
-
-    public SearchRequestParsers(IndicesQueriesRegistry queryParsers, AggregatorParsers aggParsers, Suggesters suggesters) {
-        this.queryParsers = queryParsers;
-        this.aggParsers = aggParsers;
-        this.suggesters = suggesters;
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/SearchService.java b/core/src/main/java/org/elasticsearch/search/SearchService.java
index ba39c55a7a239..88f522e532806 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchService.java
@@ -19,46 +19,47 @@
 
 package org.elasticsearch.search;
 
-import com.carrotsearch.hppc.ObjectFloatHashMap;
 import org.apache.lucene.search.FieldDoc;
 import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.action.OriginalIndices;
+import org.elasticsearch.action.search.SearchTask;
+import org.elasticsearch.action.search.SearchType;
+import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.lucene.Lucene;
-import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.common.util.concurrent.ConcurrentMapLong;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentLocation;
-import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.engine.Engine;
-import org.elasticsearch.index.query.InnerHitBuilder;
+import org.elasticsearch.index.query.InnerHitContextBuilder;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
+import org.elasticsearch.index.query.MatchNoneQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.shard.IndexEventListener;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.SearchOperationListener;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason;
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.aggregations.AggregationInitializationException;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.SearchContextAggregations;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.collapse.CollapseContext;
 import org.elasticsearch.search.dfs.DfsPhase;
 import org.elasticsearch.search.dfs.DfsSearchResult;
 import org.elasticsearch.search.fetch.FetchPhase;
@@ -67,11 +68,9 @@
 import org.elasticsearch.search.fetch.ScrollQueryFetchSearchResult;
 import org.elasticsearch.search.fetch.ShardFetchRequest;
 import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext;
-import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext.DocValueField;
-import org.elasticsearch.search.fetch.subphase.DocValueFieldsFetchSubPhase;
 import org.elasticsearch.search.fetch.subphase.ScriptFieldsContext.ScriptField;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
-import org.elasticsearch.search.internal.DefaultSearchContext;
+import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.search.internal.InternalScrollSearchRequest;
 import org.elasticsearch.search.internal.ScrollContext;
 import org.elasticsearch.search.internal.SearchContext;
@@ -81,7 +80,6 @@
 import org.elasticsearch.search.query.QueryPhase;
 import org.elasticsearch.search.query.QuerySearchRequest;
 import org.elasticsearch.search.query.QuerySearchResult;
-import org.elasticsearch.search.query.QuerySearchResultProvider;
 import org.elasticsearch.search.query.ScrollQuerySearchResult;
 import org.elasticsearch.search.rescore.RescoreBuilder;
 import org.elasticsearch.search.searchafter.SearchAfterBuilder;
@@ -92,6 +90,7 @@
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.threadpool.ThreadPool.Cancellable;
 import org.elasticsearch.threadpool.ThreadPool.Names;
+import org.elasticsearch.transport.TransportRequest;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -102,7 +101,6 @@
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.atomic.AtomicLong;
 
-import static java.util.Collections.unmodifiableMap;
 import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
 import static org.elasticsearch.common.unit.TimeValue.timeValueMinutes;
 
@@ -113,6 +111,13 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
         Setting.positiveTimeSetting("search.default_keep_alive", timeValueMinutes(5), Property.NodeScope);
     public static final Setting<TimeValue> KEEPALIVE_INTERVAL_SETTING =
         Setting.positiveTimeSetting("search.keep_alive_interval", timeValueMinutes(1), Property.NodeScope);
+    /**
+     * Enables low-level, frequent search cancellation checks. Enabling low-level checks will make long running searches to react
+     * to the cancellation request faster. However, since it will produce more cancellation checks it might slow the search performance
+     * down.
+     */
+    public static final Setting<Boolean> LOW_LEVEL_CANCELLATION_SETTING =
+        Setting.boolSetting("search.low_level_cancellation", false, Property.Dynamic, Property.NodeScope);
 
     public static final TimeValue NO_TIMEOUT = timeValueMillis(-1);
     public static final Setting<TimeValue> DEFAULT_SEARCH_TIMEOUT_SETTING =
@@ -139,21 +144,17 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
 
     private volatile TimeValue defaultSearchTimeout;
 
+    private volatile boolean lowLevelCancellation;
+
     private final Cancellable keepAliveReaper;
 
     private final AtomicLong idGenerator = new AtomicLong();
 
     private final ConcurrentMapLong<SearchContext> activeContexts = ConcurrentCollections.newConcurrentMapLongWithAggressiveConcurrency();
 
-    private final Map<String, SearchParseElement> elementParsers;
-
-    private final ParseFieldMatcher parseFieldMatcher;
-
-    @Inject
-    public SearchService(Settings settings, ClusterSettings clusterSettings, ClusterService clusterService, IndicesService indicesService,
+    public SearchService(ClusterService clusterService, IndicesService indicesService,
                          ThreadPool threadPool, ScriptService scriptService, BigArrays bigArrays, FetchPhase fetchPhase) {
-        super(settings);
-        this.parseFieldMatcher = new ParseFieldMatcher(settings);
+        super(clusterService.getSettings());
         this.threadPool = threadPool;
         this.clusterService = clusterService;
         this.indicesService = indicesService;
@@ -165,40 +166,34 @@ public SearchService(Settings settings, ClusterSettings clusterSettings, Cluster
         TimeValue keepAliveInterval = KEEPALIVE_INTERVAL_SETTING.get(settings);
         this.defaultKeepAlive = DEFAULT_KEEPALIVE_SETTING.get(settings).millis();
 
-        Map<String, SearchParseElement> elementParsers = new HashMap<>();
-        elementParsers.putAll(dfsPhase.parseElements());
-        elementParsers.putAll(queryPhase.parseElements());
-        elementParsers.putAll(fetchPhase.parseElements());
-        this.elementParsers = unmodifiableMap(elementParsers);
-
         this.keepAliveReaper = threadPool.scheduleWithFixedDelay(new Reaper(), keepAliveInterval, Names.SAME);
 
         defaultSearchTimeout = DEFAULT_SEARCH_TIMEOUT_SETTING.get(settings);
-        clusterSettings.addSettingsUpdateConsumer(DEFAULT_SEARCH_TIMEOUT_SETTING, this::setDefaultSearchTimeout);
+        clusterService.getClusterSettings().addSettingsUpdateConsumer(DEFAULT_SEARCH_TIMEOUT_SETTING, this::setDefaultSearchTimeout);
+
+        lowLevelCancellation = LOW_LEVEL_CANCELLATION_SETTING.get(settings);
+        clusterService.getClusterSettings().addSettingsUpdateConsumer(LOW_LEVEL_CANCELLATION_SETTING, this::setLowLevelCancellation);
     }
 
     private void setDefaultSearchTimeout(TimeValue defaultSearchTimeout) {
         this.defaultSearchTimeout = defaultSearchTimeout;
     }
 
+    private void setLowLevelCancellation(Boolean lowLevelCancellation) {
+        this.lowLevelCancellation = lowLevelCancellation;
+    }
+
     @Override
-    public void afterIndexClosed(Index index, Settings indexSettings) {
-        // once an index is closed we can just clean up all the pending search context information
+    public void afterIndexRemoved(Index index, IndexSettings indexSettings, IndexRemovalReason reason) {
+        // once an index is removed due to deletion or closing, we can just clean up all the pending search context information
+        // if we then close all the contexts we can get some search failures along the way which are not expected.
+        // it's fine to keep the contexts open if the index is still "alive"
+        // unfortunately we don't have a clear way to signal today why an index is closed.
         // to release memory and let references to the filesystem go etc.
-        IndexMetaData idxMeta = SearchService.this.clusterService.state().metaData().index(index);
-        if (idxMeta != null && idxMeta.getState() == IndexMetaData.State.CLOSE) {
-            // we need to check if it's really closed
-            // since sometimes due to a relocation we already closed the shard and that causes the index to be closed
-            // if we then close all the contexts we can get some search failures along the way which are not expected.
-            // it's fine to keep the contexts open if the index is still "alive"
-            // unfortunately we don't have a clear way to signal today why an index is closed.
-            afterIndexDeleted(index, indexSettings);
+        if (reason == IndexRemovalReason.DELETED || reason == IndexRemovalReason.CLOSED) {
+            freeAllContextForIndex(index);
         }
-    }
 
-    @Override
-    public void afterIndexDeleted(Index index, Settings indexSettings) {
-        freeAllContextForIndex(index);
     }
 
     protected void putContext(SearchContext context) {
@@ -227,10 +222,11 @@ protected void doClose() {
         keepAliveReaper.cancel();
     }
 
-    public DfsSearchResult executeDfsPhase(ShardSearchRequest request) throws IOException {
+    public DfsSearchResult executeDfsPhase(ShardSearchRequest request, SearchTask task) throws IOException {
         final SearchContext context = createAndPutContext(request);
         context.incRef();
         try {
+            context.setTask(task);
             contextProcessing(context);
             dfsPhase.execute(context);
             contextProcessedSuccessfully(context);
@@ -249,6 +245,7 @@ public DfsSearchResult executeDfsPhase(ShardSearchRequest request) throws IOExce
      */
     private void loadOrExecuteQueryPhase(final ShardSearchRequest request, final SearchContext context) throws Exception {
         final boolean canCache = indicesService.canCache(request, context);
+        context.getQueryShardContext().freezeContext();
         if (canCache) {
             indicesService.loadIntoContext(request, context, queryPhase);
         } else {
@@ -256,32 +253,40 @@ private void loadOrExecuteQueryPhase(final ShardSearchRequest request, final Sea
         }
     }
 
-    public QuerySearchResultProvider executeQueryPhase(ShardSearchRequest request) throws IOException {
+    public SearchPhaseResult executeQueryPhase(ShardSearchRequest request, SearchTask task) throws IOException {
         final SearchContext context = createAndPutContext(request);
         final SearchOperationListener operationListener = context.indexShard().getSearchOperationListener();
         context.incRef();
+        boolean queryPhaseSuccess = false;
         try {
+            context.setTask(task);
             operationListener.onPreQueryPhase(context);
             long time = System.nanoTime();
             contextProcessing(context);
 
             loadOrExecuteQueryPhase(request, context);
 
-            if (context.queryResult().hasHits() == false && context.scrollContext() == null) {
+            if (context.queryResult().hasSearchContext() == false && context.scrollContext() == null) {
                 freeContext(context.id());
             } else {
                 contextProcessedSuccessfully(context);
             }
-            operationListener.onQueryPhase(context, System.nanoTime() - time);
-
+            final long afterQueryTime = System.nanoTime();
+            queryPhaseSuccess = true;
+            operationListener.onQueryPhase(context, afterQueryTime - time);
+            if (request.numberOfShards() == 1) {
+                return executeFetchPhase(context, operationListener, afterQueryTime);
+            }
             return context.queryResult();
         } catch (Exception e) {
             // execution exception can happen while loading the cache, strip it
             if (e instanceof ExecutionException) {
                 e = (e.getCause() == null || e.getCause() instanceof Exception) ?
-                        (Exception) e.getCause() : new ElasticsearchException(e.getCause());
+                    (Exception) e.getCause() : new ElasticsearchException(e.getCause());
+            }
+            if (!queryPhaseSuccess) {
+                operationListener.onFailedQueryPhase(context);
             }
-            operationListener.onFailedQueryPhase(context);
             logger.trace("Query phase failed", e);
             processFailure(context, e);
             throw ExceptionsHelper.convertToRuntime(e);
@@ -290,11 +295,31 @@ public QuerySearchResultProvider executeQueryPhase(ShardSearchRequest request) t
         }
     }
 
-    public ScrollQuerySearchResult executeQueryPhase(InternalScrollSearchRequest request) {
-        final SearchContext context = findContext(request.id());
+    private QueryFetchSearchResult executeFetchPhase(SearchContext context, SearchOperationListener operationListener,
+                                                        long afterQueryTime) {
+        operationListener.onPreFetchPhase(context);
+        try {
+            shortcutDocIdsToLoad(context);
+            fetchPhase.execute(context);
+            if (fetchPhaseShouldFreeContext(context)) {
+                freeContext(context.id());
+            } else {
+                contextProcessedSuccessfully(context);
+            }
+        } catch (Exception e) {
+            operationListener.onFailedFetchPhase(context);
+            throw ExceptionsHelper.convertToRuntime(e);
+        }
+        operationListener.onFetchPhase(context, System.nanoTime() - afterQueryTime);
+        return new QueryFetchSearchResult(context.queryResult(), context.fetchResult());
+    }
+
+    public ScrollQuerySearchResult executeQueryPhase(InternalScrollSearchRequest request, SearchTask task) {
+        final SearchContext context = findContext(request.id(), request);
         SearchOperationListener operationListener = context.indexShard().getSearchOperationListener();
         context.incRef();
         try {
+            context.setTask(task);
             operationListener.onPreQueryPhase(context);
             long time = System.nanoTime();
             contextProcessing(context);
@@ -313,8 +338,9 @@ public ScrollQuerySearchResult executeQueryPhase(InternalScrollSearchRequest req
         }
     }
 
-    public QuerySearchResult executeQueryPhase(QuerySearchRequest request) {
-        final SearchContext context = findContext(request.id());
+    public QuerySearchResult executeQueryPhase(QuerySearchRequest request, SearchTask task) {
+        final SearchContext context = findContext(request.id(), request);
+        context.setTask(task);
         IndexShard indexShard = context.indexShard();
         SearchOperationListener operationListener = indexShard.getSearchOperationListener();
         context.incRef();
@@ -325,7 +351,7 @@ public QuerySearchResult executeQueryPhase(QuerySearchRequest request) {
             operationListener.onPreQueryPhase(context);
             long time = System.nanoTime();
             queryPhase.execute(context);
-            if (context.queryResult().hasHits() == false && context.scrollContext() == null) {
+            if (context.queryResult().hasSearchContext() == false && context.scrollContext() == null) {
                 // no hits, we can release the context since there will be no fetch phase
                 freeContext(context.id());
             } else {
@@ -353,119 +379,28 @@ private boolean fetchPhaseShouldFreeContext(SearchContext context) {
         }
     }
 
-    public QueryFetchSearchResult executeFetchPhase(ShardSearchRequest request) throws IOException {
-        final SearchContext context = createAndPutContext(request);
-        context.incRef();
-        try {
-            contextProcessing(context);
-            SearchOperationListener operationListener = context.indexShard().getSearchOperationListener();
-            operationListener.onPreQueryPhase(context);
-            long time = System.nanoTime();
-            try {
-                loadOrExecuteQueryPhase(request, context);
-            } catch (Exception e) {
-                operationListener.onFailedQueryPhase(context);
-                throw ExceptionsHelper.convertToRuntime(e);
-            }
-            long time2 = System.nanoTime();
-            operationListener.onQueryPhase(context, time2 - time);
-            operationListener.onPreFetchPhase(context);
-            try {
-                shortcutDocIdsToLoad(context);
-                fetchPhase.execute(context);
-                if (fetchPhaseShouldFreeContext(context)) {
-                    freeContext(context.id());
-                } else {
-                    contextProcessedSuccessfully(context);
-                }
-            } catch (Exception e) {
-                operationListener.onFailedFetchPhase(context);
-                throw ExceptionsHelper.convertToRuntime(e);
-            }
-            operationListener.onFetchPhase(context, System.nanoTime() - time2);
-            return new QueryFetchSearchResult(context.queryResult(), context.fetchResult());
-        } catch (Exception e) {
-            logger.trace("Fetch phase failed", e);
-            processFailure(context, e);
-            throw ExceptionsHelper.convertToRuntime(e);
-        } finally {
-            cleanContext(context);
-        }
-    }
-
-    public QueryFetchSearchResult executeFetchPhase(QuerySearchRequest request) {
-        final SearchContext context = findContext(request.id());
-        context.incRef();
-        try {
-            contextProcessing(context);
-            context.searcher().setAggregatedDfs(request.dfs());
-            SearchOperationListener operationListener = context.indexShard().getSearchOperationListener();
-            operationListener.onPreQueryPhase(context);
-            long time = System.nanoTime();
-            try {
-                queryPhase.execute(context);
-            } catch (Exception e) {
-                operationListener.onFailedQueryPhase(context);
-                throw ExceptionsHelper.convertToRuntime(e);
-            }
-            long time2 = System.nanoTime();
-            operationListener.onQueryPhase(context, time2 - time);
-            operationListener.onPreFetchPhase(context);
-            try {
-                shortcutDocIdsToLoad(context);
-                fetchPhase.execute(context);
-                if (fetchPhaseShouldFreeContext(context)) {
-                    freeContext(request.id());
-                } else {
-                    contextProcessedSuccessfully(context);
-                }
-            } catch (Exception e) {
-                operationListener.onFailedFetchPhase(context);
-                throw ExceptionsHelper.convertToRuntime(e);
-            }
-            operationListener.onFetchPhase(context, System.nanoTime() - time2);
-            return new QueryFetchSearchResult(context.queryResult(), context.fetchResult());
-        } catch (Exception e) {
-            logger.trace("Fetch phase failed", e);
-            processFailure(context, e);
-            throw ExceptionsHelper.convertToRuntime(e);
-        } finally {
-            cleanContext(context);
-        }
-    }
-
-    public ScrollQueryFetchSearchResult executeFetchPhase(InternalScrollSearchRequest request) {
-        final SearchContext context = findContext(request.id());
+    public ScrollQueryFetchSearchResult executeFetchPhase(InternalScrollSearchRequest request, SearchTask task) {
+        final SearchContext context = findContext(request.id(), request);
         context.incRef();
         try {
+            context.setTask(task);
             contextProcessing(context);
             SearchOperationListener operationListener = context.indexShard().getSearchOperationListener();
             processScroll(request, context);
             operationListener.onPreQueryPhase(context);
-            long time = System.nanoTime();
+            final long time = System.nanoTime();
             try {
                 queryPhase.execute(context);
             } catch (Exception e) {
                 operationListener.onFailedQueryPhase(context);
                 throw ExceptionsHelper.convertToRuntime(e);
             }
-            long time2 = System.nanoTime();
-            operationListener.onQueryPhase(context, time2 - time);
-            operationListener.onPreFetchPhase(context);
-            try {
-                shortcutDocIdsToLoad(context);
-                fetchPhase.execute(context);
-                if (fetchPhaseShouldFreeContext(context)) {
-                    freeContext(request.id());
-                } else {
-                    contextProcessedSuccessfully(context);
-                }
-            } catch (Exception e) {
-                operationListener.onFailedFetchPhase(context);
-                throw ExceptionsHelper.convertToRuntime(e);
-            }
-            operationListener.onFetchPhase(context, System.nanoTime() - time2);
-            return new ScrollQueryFetchSearchResult(new QueryFetchSearchResult(context.queryResult(), context.fetchResult()), context.shardTarget());
+            long afterQueryTime = System.nanoTime();
+            operationListener.onQueryPhase(context, afterQueryTime - time);
+            QueryFetchSearchResult fetchSearchResult = executeFetchPhase(context, operationListener, afterQueryTime);
+
+            return new ScrollQueryFetchSearchResult(fetchSearchResult,
+                context.shardTarget());
         } catch (Exception e) {
             logger.trace("Fetch phase failed", e);
             processFailure(context, e);
@@ -475,11 +410,12 @@ public ScrollQueryFetchSearchResult executeFetchPhase(InternalScrollSearchReques
         }
     }
 
-    public FetchSearchResult executeFetchPhase(ShardFetchRequest request) {
-        final SearchContext context = findContext(request.id());
+    public FetchSearchResult executeFetchPhase(ShardFetchRequest request, SearchTask task) {
+        final SearchContext context = findContext(request.id(), request);
         final SearchOperationListener operationListener = context.indexShard().getSearchOperationListener();
         context.incRef();
         try {
+            context.setTask(task);
             contextProcessing(context);
             if (request.lastEmittedDoc() != null) {
                 context.scrollContext().lastEmittedDoc = request.lastEmittedDoc();
@@ -505,13 +441,20 @@ public FetchSearchResult executeFetchPhase(ShardFetchRequest request) {
         }
     }
 
-    private SearchContext findContext(long id) throws SearchContextMissingException {
+    private SearchContext findContext(long id, TransportRequest request) throws SearchContextMissingException {
         SearchContext context = activeContexts.get(id);
         if (context == null) {
             throw new SearchContextMissingException(id);
         }
-        SearchContext.setCurrent(context);
-        return context;
+
+        SearchOperationListener operationListener = context.indexShard().getSearchOperationListener();
+        try {
+            operationListener.validateSearchContext(context, request);
+            return context;
+        } catch (Exception e) {
+            processFailure(context, e);
+            throw e;
+        }
     }
 
     final SearchContext createAndPutContext(ShardSearchRequest request) throws IOException {
@@ -533,25 +476,8 @@ final SearchContext createAndPutContext(ShardSearchRequest request) throws IOExc
     }
 
     final SearchContext createContext(ShardSearchRequest request, @Nullable Engine.Searcher searcher) throws IOException {
-        IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex());
-        IndexShard indexShard = indexService.getShard(request.shardId().getId());
-        SearchShardTarget shardTarget = new SearchShardTarget(clusterService.localNode().getId(), indexShard.shardId());
-
-        Engine.Searcher engineSearcher = searcher == null ? indexShard.acquireSearcher("search") : searcher;
-
-        DefaultSearchContext context = new DefaultSearchContext(idGenerator.incrementAndGet(), request, shardTarget, engineSearcher,
-                indexService,
-                indexShard, scriptService, bigArrays, threadPool.estimatedTimeInMillisCounter(), parseFieldMatcher,
-                defaultSearchTimeout, fetchPhase);
-        SearchContext.setCurrent(context);
+        final DefaultSearchContext context = createSearchContext(request, defaultSearchTimeout, searcher);
         try {
-            request.rewrite(context.getQueryShardContext());
-            // reset that we have used nowInMillis from the context since it may
-            // have been rewritten so its no longer in the query and the request can
-            // be cached. If it is still present in the request (e.g. in a range
-            // aggregation) it will still be caught when the aggregation is
-            // evaluated.
-            context.resetNowInMillisUsed();
             if (request.scroll() != null) {
                 context.scrollContext(new ScrollContext());
                 context.scrollContext().scroll = request.scroll();
@@ -577,6 +503,7 @@ final SearchContext createContext(ShardSearchRequest request, @Nullable Engine.S
                 keepAlive = request.scroll().keepAlive().millis();
             }
             context.keepAlive(keepAlive);
+            context.lowLevelCancellation(lowLevelCancellation);
         } catch (Exception e) {
             context.close();
             throw ExceptionsHelper.convertToRuntime(e);
@@ -585,6 +512,32 @@ final SearchContext createContext(ShardSearchRequest request, @Nullable Engine.S
         return context;
     }
 
+    public DefaultSearchContext createSearchContext(ShardSearchRequest request, TimeValue timeout, @Nullable Engine.Searcher searcher)
+        throws IOException {
+        IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex());
+        IndexShard indexShard = indexService.getShard(request.shardId().getId());
+        SearchShardTarget shardTarget = new SearchShardTarget(clusterService.localNode().getId(),
+                indexShard.shardId(), request.getClusterAlias(), OriginalIndices.NONE);
+        Engine.Searcher engineSearcher = searcher == null ? indexShard.acquireSearcher("search") : searcher;
+
+        final DefaultSearchContext searchContext = new DefaultSearchContext(idGenerator.incrementAndGet(), request, shardTarget,
+            engineSearcher, indexService, indexShard, bigArrays, threadPool.estimatedTimeInMillisCounter(), timeout, fetchPhase);
+        boolean success = false;
+        try {
+            // we clone the query shard context here just for rewriting otherwise we
+            // might end up with incorrect state since we are using now() or script services
+            // during rewrite and normalized / evaluate templates etc.
+            request.rewrite(new QueryShardContext(searchContext.getQueryShardContext()));
+            assert searchContext.getQueryShardContext().isCachable();
+            success = true;
+        } finally {
+            if (success == false) {
+                IOUtils.closeWhileHandlingException(searchContext);
+            }
+        }
+        return searchContext;
+    }
+
     private void freeAllContextForIndex(Index index) {
         assert index != null;
         for (SearchContext ctx : activeContexts.values()) {
@@ -626,14 +579,13 @@ private void contextProcessing(SearchContext context) {
     }
 
     private void contextProcessedSuccessfully(SearchContext context) {
-        context.accessed(threadPool.estimatedTimeInMillis());
+        context.accessed(threadPool.relativeTimeInMillis());
     }
 
     private void cleanContext(SearchContext context) {
         try {
-            assert context == SearchContext.current();
             context.clearReleasables(Lifetime.PHASE);
-            SearchContext.removeCurrent();
+            context.setTask(null);
         } finally {
             context.decRef();
         }
@@ -659,24 +611,17 @@ private void parseSource(DefaultSearchContext context, SearchSourceBuilder sourc
         QueryShardContext queryShardContext = context.getQueryShardContext();
         context.from(source.from());
         context.size(source.size());
-        ObjectFloatHashMap<String> indexBoostMap = source.indexBoost();
-        if (indexBoostMap != null) {
-            Float indexBoost = indexBoostMap.get(context.shardTarget().index());
-            if (indexBoost != null) {
-                context.queryBoost(indexBoost);
-            }
-        }
-        Map<String, InnerHitBuilder> innerHitBuilders = new HashMap<>();
+        Map<String, InnerHitContextBuilder> innerHitBuilders = new HashMap<>();
         if (source.query() != null) {
-            InnerHitBuilder.extractInnerHits(source.query(), innerHitBuilders);
+            InnerHitContextBuilder.extractInnerHits(source.query(), innerHitBuilders);
             context.parsedQuery(queryShardContext.toQuery(source.query()));
         }
         if (source.postFilter() != null) {
-            InnerHitBuilder.extractInnerHits(source.postFilter(), innerHitBuilders);
+            InnerHitContextBuilder.extractInnerHits(source.postFilter(), innerHitBuilders);
             context.parsedPostFilter(queryShardContext.toQuery(source.postFilter()));
         }
         if (innerHitBuilders.size() > 0) {
-            for (Map.Entry<String, InnerHitBuilder> entry : innerHitBuilders.entrySet()) {
+            for (Map.Entry<String, InnerHitContextBuilder> entry : innerHitBuilders.entrySet()) {
                 try {
                     entry.getValue().build(context, context.innerHits());
                 } catch (IOException e) {
@@ -701,12 +646,13 @@ private void parseSource(DefaultSearchContext context, SearchSourceBuilder sourc
         if (source.profile()) {
             context.setProfilers(new Profilers(context.searcher()));
         }
-        context.timeout(source.timeout());
+        if (source.timeout() != null) {
+            context.timeout(source.timeout());
+        }
         context.terminateAfter(source.terminateAfter());
         if (source.aggregations() != null) {
             try {
-                AggregationContext aggContext = new AggregationContext(context);
-                AggregatorFactories factories = source.aggregations().build(aggContext, null);
+                AggregatorFactories factories = source.aggregations().build(context, null);
                 factories.validate();
                 context.aggregations(new SearchContextAggregations(factories));
             } catch (IOException e) {
@@ -736,11 +682,7 @@ private void parseSource(DefaultSearchContext context, SearchSourceBuilder sourc
             context.fetchSourceContext(source.fetchSource());
         }
         if (source.docValueFields() != null) {
-            DocValueFieldsContext docValuesFieldsContext = context.getFetchSubPhaseContext(DocValueFieldsFetchSubPhase.CONTEXT_FACTORY);
-            for (String field : source.docValueFields()) {
-                docValuesFieldsContext.add(new DocValueField(field));
-            }
-            docValuesFieldsContext.setHitExecutionNeeded(true);
+            context.docValueFieldsContext(new DocValueFieldsContext(source.docValueFields()));
         }
         if (source.highlighter() != null) {
             HighlightBuilder highlightBuilder = source.highlighter();
@@ -752,53 +694,13 @@ private void parseSource(DefaultSearchContext context, SearchSourceBuilder sourc
         }
         if (source.scriptFields() != null) {
             for (org.elasticsearch.search.builder.SearchSourceBuilder.ScriptField field : source.scriptFields()) {
-                SearchScript searchScript = context.scriptService().search(context.lookup(), field.script(), ScriptContext.Standard.SEARCH,
-                        Collections.emptyMap());
+                SearchScript searchScript = scriptService.search(context.lookup(), field.script(), ScriptContext.Standard.SEARCH);
                 context.scriptFields().add(new ScriptField(field.fieldName(), searchScript, field.ignoreFailure()));
             }
         }
         if (source.ext() != null) {
-            XContentParser extParser = null;
-            try {
-                extParser = XContentFactory.xContent(source.ext()).createParser(source.ext());
-                if (extParser.nextToken() != XContentParser.Token.START_OBJECT) {
-                    throw new SearchParseException(context, "expected start object, found [" + extParser.currentToken() + "] instead",
-                            extParser.getTokenLocation());
-                }
-                XContentParser.Token token;
-                String currentFieldName = null;
-                while ((token = extParser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                    if (token == XContentParser.Token.FIELD_NAME) {
-                        currentFieldName = extParser.currentName();
-                    } else {
-                        SearchParseElement parseElement = this.elementParsers.get(currentFieldName);
-                        if (parseElement == null) {
-                            if (currentFieldName != null && currentFieldName.equals("suggest")) {
-                                throw new SearchParseException(context,
-                                    "suggest is not supported in [ext], please use SearchSourceBuilder#suggest(SuggestBuilder) instead",
-                                    extParser.getTokenLocation());
-                            }
-                            throw new SearchParseException(context, "Unknown element [" + currentFieldName + "] in [ext]",
-                                    extParser.getTokenLocation());
-                        } else {
-                            parseElement.parse(extParser, context);
-                        }
-                    }
-                }
-            } catch (Exception e) {
-                String sSource = "_na_";
-                try {
-                    sSource = source.toString();
-                } catch (Exception inner) {
-                    e.addSuppressed(inner);
-                    // ignore
-                }
-                XContentLocation location = extParser != null ? extParser.getTokenLocation() : null;
-                throw new SearchParseException(context, "failed to parse ext source [" + sSource + "]", location, e);
-            } finally {
-                if (extParser != null) {
-                    extParser.close();
-                }
+            for (SearchExtBuilder searchExtBuilder : source.ext()) {
+                context.addSearchExt(searchExtBuilder);
             }
         }
         if (source.version() != null) {
@@ -836,6 +738,11 @@ private void parseSource(DefaultSearchContext context, SearchSourceBuilder sourc
             }
             context.storedFieldsContext(source.storedFields());
         }
+
+        if (source.collapse() != null) {
+            final CollapseContext collapseContext = source.collapse().build(context);
+            context.collapse(collapseContext);
+        }
     }
 
     /**
@@ -905,7 +812,7 @@ public int getActiveContexts() {
     class Reaper implements Runnable {
         @Override
         public void run() {
-            final long time = threadPool.estimatedTimeInMillis();
+            final long time = threadPool.relativeTimeInMillis();
             for (SearchContext context : activeContexts.values()) {
                 // Use the same value for both checks since lastAccessTime can
                 // be modified by another thread between checks!
@@ -914,10 +821,51 @@ public void run() {
                     continue;
                 }
                 if ((time - lastAccessTime > context.keepAlive())) {
-                    logger.debug("freeing search context [{}], time [{}], lastAccessTime [{}], keepAlive [{}]", context.id(), time, lastAccessTime, context.keepAlive());
+                    logger.debug("freeing search context [{}], time [{}], lastAccessTime [{}], keepAlive [{}]", context.id(), time,
+                        lastAccessTime, context.keepAlive());
                     freeContext(context.id());
                 }
             }
         }
     }
+
+    public AliasFilter buildAliasFilter(ClusterState state, String index, String... expressions) {
+        return indicesService.buildAliasFilter(state, index, expressions);
+    }
+
+    /**
+     * This method does a very quick rewrite of the query and returns true if the query can potentially match any documents.
+     * This method can have false positives while if it returns <code>false</code> the query won't match any documents on the current
+     * shard.
+     */
+    public boolean canMatch(ShardSearchRequest request) throws IOException {
+        assert request.searchType() == SearchType.QUERY_THEN_FETCH : "unexpected search type: " + request.searchType();
+        try (DefaultSearchContext context = createSearchContext(request, defaultSearchTimeout, null)) {
+            SearchSourceBuilder source = context.request().source();
+            if (canRewriteToMatchNone(source)) {
+                QueryBuilder queryBuilder = source.query();
+                return queryBuilder instanceof MatchNoneQueryBuilder == false;
+            }
+            return true; // null query means match_all
+        }
+    }
+
+    /**
+     * Returns true iff the given search source builder can be early terminated by rewriting to a match none query. Or in other words
+     * if the execution of a the search request can be early terminated without executing it. This is for instance not possible if
+     * a global aggregation is part of this request or if there is a suggest builder present.
+     */
+    public static boolean canRewriteToMatchNone(SearchSourceBuilder source) {
+        if (source == null || source.query() == null || source.query() instanceof MatchAllQueryBuilder || source.suggest() != null) {
+            return false;
+        } else {
+            AggregatorFactories.Builder aggregations = source.aggregations();
+            if (aggregations != null) {
+                if (aggregations.mustVisitAllDocs()) {
+                    return false;
+                }
+            }
+        }
+        return true;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/SearchShardTarget.java b/core/src/main/java/org/elasticsearch/search/SearchShardTarget.java
index 9fb8322718859..4ac742ff5d9fe 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchShardTarget.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchShardTarget.java
@@ -19,81 +19,86 @@
 
 package org.elasticsearch.search;
 
+import org.elasticsearch.Version;
+import org.elasticsearch.action.OriginalIndices;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Streamable;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.text.Text;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.transport.RemoteClusterAware;
 
 import java.io.IOException;
 
 /**
  * The target that the search request was executed on.
  */
-public class SearchShardTarget implements Writeable, Comparable<SearchShardTarget> {
+public final class SearchShardTarget implements Writeable, Comparable<SearchShardTarget> {
 
-    private Text nodeId;
-    private Text index;
-    private ShardId shardId;
+    private final Text nodeId;
+    private final ShardId shardId;
+    //original indices and cluster alias are only needed in the coordinating node throughout the search request execution.
+    //no need to serialize them as part of SearchShardTarget.
+    private final transient OriginalIndices originalIndices;
+    private final String clusterAlias;
 
     public SearchShardTarget(StreamInput in) throws IOException {
         if (in.readBoolean()) {
             nodeId = in.readText();
+        } else {
+            nodeId = null;
         }
         shardId = ShardId.readShardId(in);
-        index = new Text(shardId.getIndexName());
+        this.originalIndices = null;
+        if (in.getVersion().onOrAfter(Version.V_5_6_0)) {
+            clusterAlias = in.readOptionalString();
+        } else {
+            clusterAlias = null;
+        }
     }
 
-    public SearchShardTarget(String nodeId, ShardId shardId) {
+    public SearchShardTarget(String nodeId, ShardId shardId, String clusterAlias, OriginalIndices originalIndices) {
         this.nodeId = nodeId == null ? null : new Text(nodeId);
-        this.index = new Text(shardId.getIndexName());
         this.shardId = shardId;
+        this.originalIndices = originalIndices;
+        this.clusterAlias = clusterAlias;
     }
 
-    public SearchShardTarget(String nodeId, Index index, int shardId) {
-        this(nodeId,  new ShardId(index, shardId));
-    }
-
-    @Nullable
-    public String nodeId() {
-        return nodeId.string();
+    //this constructor is only used in tests
+    public SearchShardTarget(String nodeId, Index index, int shardId, String clusterAlias) {
+        this(nodeId,  new ShardId(index, shardId), clusterAlias, OriginalIndices.NONE);
     }
 
     @Nullable
     public String getNodeId() {
-        return nodeId();
+        return nodeId.string();
     }
 
-    public Text nodeIdText() {
+    public Text getNodeIdText() {
         return this.nodeId;
     }
 
-    public String index() {
-        return index.string();
-    }
-
     public String getIndex() {
-        return index();
+        return shardId.getIndexName();
     }
 
-    public Text indexText() {
-        return this.index;
+    public ShardId getShardId() {
+        return shardId;
     }
 
-    public ShardId shardId() {
-        return shardId;
+    public OriginalIndices getOriginalIndices() {
+        return originalIndices;
     }
 
-    public ShardId getShardId() {
-        return shardId;
+    public String getClusterAlias() {
+        return clusterAlias;
     }
 
     @Override
     public int compareTo(SearchShardTarget o) {
-        int i = index.string().compareTo(o.index());
+        int i = shardId.getIndexName().compareTo(o.getIndex());
         if (i == 0) {
             i = shardId.getId() - o.shardId.id();
         }
@@ -109,6 +114,9 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeText(nodeId);
         }
         shardId.writeTo(out);
+        if (out.getVersion().onOrAfter(Version.V_5_6_0)) {
+            out.writeOptionalString(clusterAlias);
+        }
     }
 
     @Override
@@ -118,23 +126,26 @@ public boolean equals(Object o) {
         SearchShardTarget that = (SearchShardTarget) o;
         if (shardId.equals(that.shardId) == false) return false;
         if (nodeId != null ? !nodeId.equals(that.nodeId) : that.nodeId != null) return false;
-
+        if (clusterAlias != null ? !clusterAlias.equals(that.clusterAlias) : that.clusterAlias != null) return false;
         return true;
     }
 
     @Override
     public int hashCode() {
         int result = nodeId != null ? nodeId.hashCode() : 0;
-        result = 31 * result + (index != null ? index.hashCode() : 0);
+        result = 31 * result + (shardId.getIndexName() != null ? shardId.getIndexName().hashCode() : 0);
         result = 31 * result + shardId.hashCode();
+        result = 31 * result + (clusterAlias != null ? clusterAlias.hashCode() : 0);
         return result;
     }
 
     @Override
     public String toString() {
+        String shardToString = "[" + RemoteClusterAware.buildRemoteIndexName(clusterAlias, shardId.getIndexName()) + "][" + shardId.getId()
+            + "]";
         if (nodeId == null) {
-            return "[_na_]" + shardId;
+            return "[_na_]" + shardToString;
         }
-        return "[" + nodeId + "]" + shardId;
+        return "[" + nodeId + "]" + shardToString;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/SearchSortValues.java b/core/src/main/java/org/elasticsearch/search/SearchSortValues.java
new file mode 100644
index 0000000000000..d3d55ff481afd
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/SearchSortValues.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParserUtils;
+import org.elasticsearch.search.SearchHit.Fields;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Objects;
+
+public class SearchSortValues implements ToXContent, Writeable {
+
+    static final SearchSortValues EMPTY = new SearchSortValues(new Object[0]);
+    private final Object[] sortValues;
+
+    SearchSortValues(Object[] sortValues) {
+        this.sortValues = Objects.requireNonNull(sortValues, "sort values must not be empty");
+    }
+
+    public SearchSortValues(Object[] sortValues, DocValueFormat[] sortValueFormats) {
+        Objects.requireNonNull(sortValues);
+        Objects.requireNonNull(sortValueFormats);
+        this.sortValues = Arrays.copyOf(sortValues, sortValues.length);
+        for (int i = 0; i < sortValues.length; ++i) {
+            if (this.sortValues[i] instanceof BytesRef) {
+                this.sortValues[i] = sortValueFormats[i].format((BytesRef) sortValues[i]);
+            }
+        }
+    }
+
+    public SearchSortValues(StreamInput in) throws IOException {
+        int size = in.readVInt();
+        if (size > 0) {
+            sortValues = new Object[size];
+            for (int i = 0; i < sortValues.length; i++) {
+                byte type = in.readByte();
+                if (type == 0) {
+                    sortValues[i] = null;
+                } else if (type == 1) {
+                    sortValues[i] = in.readString();
+                } else if (type == 2) {
+                    sortValues[i] = in.readInt();
+                } else if (type == 3) {
+                    sortValues[i] = in.readLong();
+                } else if (type == 4) {
+                    sortValues[i] = in.readFloat();
+                } else if (type == 5) {
+                    sortValues[i] = in.readDouble();
+                } else if (type == 6) {
+                    sortValues[i] = in.readByte();
+                } else if (type == 7) {
+                    sortValues[i] = in.readShort();
+                } else if (type == 8) {
+                    sortValues[i] = in.readBoolean();
+                } else {
+                    throw new IOException("Can't match type [" + type + "]");
+                }
+            }
+        } else {
+            sortValues = new Object[0];
+        }
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeVInt(sortValues.length);
+        for (Object sortValue : sortValues) {
+            if (sortValue == null) {
+                out.writeByte((byte) 0);
+            } else {
+                Class type = sortValue.getClass();
+                if (type == String.class) {
+                    out.writeByte((byte) 1);
+                    out.writeString((String) sortValue);
+                } else if (type == Integer.class) {
+                    out.writeByte((byte) 2);
+                    out.writeInt((Integer) sortValue);
+                } else if (type == Long.class) {
+                    out.writeByte((byte) 3);
+                    out.writeLong((Long) sortValue);
+                } else if (type == Float.class) {
+                    out.writeByte((byte) 4);
+                    out.writeFloat((Float) sortValue);
+                } else if (type == Double.class) {
+                    out.writeByte((byte) 5);
+                    out.writeDouble((Double) sortValue);
+                } else if (type == Byte.class) {
+                    out.writeByte((byte) 6);
+                    out.writeByte((Byte) sortValue);
+                } else if (type == Short.class) {
+                    out.writeByte((byte) 7);
+                    out.writeShort((Short) sortValue);
+                } else if (type == Boolean.class) {
+                    out.writeByte((byte) 8);
+                    out.writeBoolean((Boolean) sortValue);
+                } else {
+                    throw new IOException("Can't handle sort field value of type [" + type + "]");
+                }
+            }
+        }
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        if (sortValues.length > 0) {
+            builder.startArray(Fields.SORT);
+            for (Object sortValue : sortValues) {
+                builder.value(sortValue);
+            }
+            builder.endArray();
+        }
+        return builder;
+    }
+
+    public static SearchSortValues fromXContent(XContentParser parser) throws IOException {
+        XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser::getTokenLocation);
+        return new SearchSortValues(parser.list().toArray());
+    }
+
+    public Object[] sortValues() {
+        return sortValues;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+        SearchSortValues other = (SearchSortValues) obj;
+        return Arrays.equals(sortValues, other.sortValues);
+    }
+
+    @Override
+    public int hashCode() {
+        return Arrays.hashCode(sortValues);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/action/SearchTransportService.java b/core/src/main/java/org/elasticsearch/search/action/SearchTransportService.java
deleted file mode 100644
index 8552d21b5c335..0000000000000
--- a/core/src/main/java/org/elasticsearch/search/action/SearchTransportService.java
+++ /dev/null
@@ -1,367 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.action;
-
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ActionListenerResponseHandler;
-import org.elasticsearch.action.IndicesRequest;
-import org.elasticsearch.action.OriginalIndices;
-import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.search.SearchService;
-import org.elasticsearch.search.dfs.DfsSearchResult;
-import org.elasticsearch.search.fetch.FetchSearchResult;
-import org.elasticsearch.search.fetch.QueryFetchSearchResult;
-import org.elasticsearch.search.fetch.ScrollQueryFetchSearchResult;
-import org.elasticsearch.search.fetch.ShardFetchRequest;
-import org.elasticsearch.search.fetch.ShardFetchSearchRequest;
-import org.elasticsearch.search.internal.InternalScrollSearchRequest;
-import org.elasticsearch.search.internal.ShardSearchTransportRequest;
-import org.elasticsearch.search.query.QuerySearchRequest;
-import org.elasticsearch.search.query.QuerySearchResult;
-import org.elasticsearch.search.query.QuerySearchResultProvider;
-import org.elasticsearch.search.query.ScrollQuerySearchResult;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.TransportChannel;
-import org.elasticsearch.transport.TransportRequest;
-import org.elasticsearch.transport.TransportRequestHandler;
-import org.elasticsearch.transport.TransportResponse;
-import org.elasticsearch.transport.TransportService;
-
-import java.io.IOException;
-
-/**
- * An encapsulation of {@link org.elasticsearch.search.SearchService} operations exposed through
- * transport.
- */
-public class SearchTransportService extends AbstractComponent {
-
-    public static final String FREE_CONTEXT_SCROLL_ACTION_NAME = "indices:data/read/search[free_context/scroll]";
-    public static final String FREE_CONTEXT_ACTION_NAME = "indices:data/read/search[free_context]";
-    public static final String CLEAR_SCROLL_CONTEXTS_ACTION_NAME = "indices:data/read/search[clear_scroll_contexts]";
-    public static final String DFS_ACTION_NAME = "indices:data/read/search[phase/dfs]";
-    public static final String QUERY_ACTION_NAME = "indices:data/read/search[phase/query]";
-    public static final String QUERY_ID_ACTION_NAME = "indices:data/read/search[phase/query/id]";
-    public static final String QUERY_SCROLL_ACTION_NAME = "indices:data/read/search[phase/query/scroll]";
-    public static final String QUERY_FETCH_ACTION_NAME = "indices:data/read/search[phase/query+fetch]";
-    public static final String QUERY_QUERY_FETCH_ACTION_NAME = "indices:data/read/search[phase/query/query+fetch]";
-    public static final String QUERY_FETCH_SCROLL_ACTION_NAME = "indices:data/read/search[phase/query+fetch/scroll]";
-    public static final String FETCH_ID_SCROLL_ACTION_NAME = "indices:data/read/search[phase/fetch/id/scroll]";
-    public static final String FETCH_ID_ACTION_NAME = "indices:data/read/search[phase/fetch/id]";
-
-    private final TransportService transportService;
-    private final SearchService searchService;
-
-    @Inject
-    public SearchTransportService(Settings settings, TransportService transportService, SearchService searchService) {
-        super(settings);
-        this.transportService = transportService;
-        this.searchService = searchService;
-        transportService.registerRequestHandler(FREE_CONTEXT_SCROLL_ACTION_NAME, ScrollFreeContextRequest::new, ThreadPool.Names.SAME,
-                new FreeContextTransportHandler<>());
-        transportService.registerRequestHandler(FREE_CONTEXT_ACTION_NAME, SearchFreeContextRequest::new, ThreadPool.Names.SAME,
-                new FreeContextTransportHandler<>());
-        transportService.registerRequestHandler(CLEAR_SCROLL_CONTEXTS_ACTION_NAME, ClearScrollContextsRequest::new, ThreadPool.Names.SAME,
-                new ClearScrollContextsTransportHandler());
-        transportService.registerRequestHandler(DFS_ACTION_NAME, ShardSearchTransportRequest::new, ThreadPool.Names.SEARCH,
-                new SearchDfsTransportHandler());
-        transportService.registerRequestHandler(QUERY_ACTION_NAME, ShardSearchTransportRequest::new, ThreadPool.Names.SEARCH,
-                new SearchQueryTransportHandler());
-        transportService.registerRequestHandler(QUERY_ID_ACTION_NAME, QuerySearchRequest::new, ThreadPool.Names.SEARCH,
-                new SearchQueryByIdTransportHandler());
-        transportService.registerRequestHandler(QUERY_SCROLL_ACTION_NAME, InternalScrollSearchRequest::new, ThreadPool.Names.SEARCH,
-                new SearchQueryScrollTransportHandler());
-        transportService.registerRequestHandler(QUERY_FETCH_ACTION_NAME, ShardSearchTransportRequest::new, ThreadPool.Names.SEARCH,
-                new SearchQueryFetchTransportHandler());
-        transportService.registerRequestHandler(QUERY_QUERY_FETCH_ACTION_NAME, QuerySearchRequest::new, ThreadPool.Names.SEARCH,
-                new SearchQueryQueryFetchTransportHandler());
-        transportService.registerRequestHandler(QUERY_FETCH_SCROLL_ACTION_NAME, InternalScrollSearchRequest::new, ThreadPool.Names.SEARCH,
-                new SearchQueryFetchScrollTransportHandler());
-        transportService.registerRequestHandler(FETCH_ID_SCROLL_ACTION_NAME, ShardFetchRequest::new, ThreadPool.Names.SEARCH,
-                new FetchByIdTransportHandler<>());
-        transportService.registerRequestHandler(FETCH_ID_ACTION_NAME, ShardFetchSearchRequest::new, ThreadPool.Names.SEARCH,
-                new FetchByIdTransportHandler<>());
-    }
-
-    public void sendFreeContext(DiscoveryNode node, final long contextId, SearchRequest request) {
-        transportService.sendRequest(node, FREE_CONTEXT_ACTION_NAME, new SearchFreeContextRequest(request, contextId),
-                new ActionListenerResponseHandler<>(new ActionListener<SearchFreeContextResponse>() {
-            @Override
-            public void onResponse(SearchFreeContextResponse response) {
-                // no need to respond if it was freed or not
-            }
-
-            @Override
-            public void onFailure(Exception e) {
-
-            }
-        }, SearchFreeContextResponse::new));
-    }
-
-    public void sendFreeContext(DiscoveryNode node, long contextId, final ActionListener<SearchFreeContextResponse> listener) {
-        transportService.sendRequest(node, FREE_CONTEXT_SCROLL_ACTION_NAME, new ScrollFreeContextRequest(contextId),
-                new ActionListenerResponseHandler<>(listener, SearchFreeContextResponse::new));
-    }
-
-    public void sendClearAllScrollContexts(DiscoveryNode node, final ActionListener<TransportResponse> listener) {
-        transportService.sendRequest(node, CLEAR_SCROLL_CONTEXTS_ACTION_NAME, new ClearScrollContextsRequest(),
-                new ActionListenerResponseHandler<>(listener, () -> TransportResponse.Empty.INSTANCE));
-    }
-
-    public void sendExecuteDfs(DiscoveryNode node, final ShardSearchTransportRequest request,
-                               final ActionListener<DfsSearchResult> listener) {
-        transportService.sendRequest(node, DFS_ACTION_NAME, request, new ActionListenerResponseHandler<>(listener, DfsSearchResult::new));
-    }
-
-    public void sendExecuteQuery(DiscoveryNode node, final ShardSearchTransportRequest request,
-                                 final ActionListener<QuerySearchResultProvider> listener) {
-        transportService.sendRequest(node, QUERY_ACTION_NAME, request,
-                new ActionListenerResponseHandler<>(listener, QuerySearchResult::new));
-    }
-
-    public void sendExecuteQuery(DiscoveryNode node, final QuerySearchRequest request, final ActionListener<QuerySearchResult> listener) {
-        transportService.sendRequest(node, QUERY_ID_ACTION_NAME, request,
-            new ActionListenerResponseHandler<>(listener, QuerySearchResult::new));
-    }
-
-    public void sendExecuteQuery(DiscoveryNode node, final InternalScrollSearchRequest request,
-                                 final ActionListener<ScrollQuerySearchResult> listener) {
-        transportService.sendRequest(node, QUERY_SCROLL_ACTION_NAME, request,
-                new ActionListenerResponseHandler<>(listener, ScrollQuerySearchResult::new));
-    }
-
-    public void sendExecuteFetch(DiscoveryNode node, final ShardSearchTransportRequest request,
-                                 final ActionListener<QueryFetchSearchResult> listener) {
-        transportService.sendRequest(node, QUERY_FETCH_ACTION_NAME, request,
-                new ActionListenerResponseHandler<>(listener, QueryFetchSearchResult::new));
-    }
-
-    public void sendExecuteFetch(DiscoveryNode node, final QuerySearchRequest request,
-                                 final ActionListener<QueryFetchSearchResult> listener) {
-        transportService.sendRequest(node, QUERY_QUERY_FETCH_ACTION_NAME, request,
-                new ActionListenerResponseHandler<>(listener, QueryFetchSearchResult::new));
-    }
-
-    public void sendExecuteFetch(DiscoveryNode node, final InternalScrollSearchRequest request,
-                                 final ActionListener<ScrollQueryFetchSearchResult> listener) {
-        transportService.sendRequest(node, QUERY_FETCH_SCROLL_ACTION_NAME, request,
-                new ActionListenerResponseHandler<>(listener, ScrollQueryFetchSearchResult::new));
-    }
-
-    public void sendExecuteFetch(DiscoveryNode node, final ShardFetchSearchRequest request,
-                                 final ActionListener<FetchSearchResult> listener) {
-        sendExecuteFetch(node, FETCH_ID_ACTION_NAME, request, listener);
-    }
-
-    public void sendExecuteFetchScroll(DiscoveryNode node, final ShardFetchRequest request,
-                                       final ActionListener<FetchSearchResult> listener) {
-        sendExecuteFetch(node, FETCH_ID_SCROLL_ACTION_NAME, request, listener);
-    }
-
-    private void sendExecuteFetch(DiscoveryNode node, String action, final ShardFetchRequest request,
-                                  final ActionListener<FetchSearchResult> listener) {
-        transportService.sendRequest(node, action, request, new ActionListenerResponseHandler<>(listener, FetchSearchResult::new));
-    }
-
-    static class ScrollFreeContextRequest extends TransportRequest {
-        private long id;
-
-        ScrollFreeContextRequest() {
-        }
-
-        ScrollFreeContextRequest(long id) {
-            this.id = id;
-        }
-
-        public long id() {
-            return this.id;
-        }
-
-        @Override
-        public void readFrom(StreamInput in) throws IOException {
-            super.readFrom(in);
-            id = in.readLong();
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            super.writeTo(out);
-            out.writeLong(id);
-        }
-    }
-
-    static class SearchFreeContextRequest extends ScrollFreeContextRequest implements IndicesRequest {
-        private OriginalIndices originalIndices;
-
-        public SearchFreeContextRequest() {
-        }
-
-        SearchFreeContextRequest(SearchRequest request, long id) {
-            super(id);
-            this.originalIndices = new OriginalIndices(request);
-        }
-
-        @Override
-        public String[] indices() {
-            if (originalIndices == null) {
-                return null;
-            }
-            return originalIndices.indices();
-        }
-
-        @Override
-        public IndicesOptions indicesOptions() {
-            if (originalIndices == null) {
-                return null;
-            }
-            return originalIndices.indicesOptions();
-        }
-
-        @Override
-        public void readFrom(StreamInput in) throws IOException {
-            super.readFrom(in);
-            originalIndices = OriginalIndices.readOriginalIndices(in);
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            super.writeTo(out);
-            OriginalIndices.writeOriginalIndices(originalIndices, out);
-        }
-    }
-
-    public static class SearchFreeContextResponse extends TransportResponse {
-
-        private boolean freed;
-
-        SearchFreeContextResponse() {
-        }
-
-        SearchFreeContextResponse(boolean freed) {
-            this.freed = freed;
-        }
-
-        public boolean isFreed() {
-            return freed;
-        }
-
-        @Override
-        public void readFrom(StreamInput in) throws IOException {
-            super.readFrom(in);
-            freed = in.readBoolean();
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            super.writeTo(out);
-            out.writeBoolean(freed);
-        }
-    }
-
-    class FreeContextTransportHandler<FreeContextRequest extends ScrollFreeContextRequest>
-            implements TransportRequestHandler<FreeContextRequest> {
-        @Override
-        public void messageReceived(FreeContextRequest request, TransportChannel channel) throws Exception {
-            boolean freed = searchService.freeContext(request.id());
-            channel.sendResponse(new SearchFreeContextResponse(freed));
-        }
-    }
-
-    static class ClearScrollContextsRequest extends TransportRequest {
-    }
-
-    class ClearScrollContextsTransportHandler implements TransportRequestHandler<ClearScrollContextsRequest> {
-        @Override
-        public void messageReceived(ClearScrollContextsRequest request, TransportChannel channel) throws Exception {
-            searchService.freeAllScrollContexts();
-            channel.sendResponse(TransportResponse.Empty.INSTANCE);
-        }
-    }
-
-    class SearchDfsTransportHandler implements TransportRequestHandler<ShardSearchTransportRequest> {
-        @Override
-        public void messageReceived(ShardSearchTransportRequest request, TransportChannel channel) throws Exception {
-            DfsSearchResult result = searchService.executeDfsPhase(request);
-            channel.sendResponse(result);
-        }
-    }
-
-    class SearchQueryTransportHandler implements TransportRequestHandler<ShardSearchTransportRequest> {
-        @Override
-        public void messageReceived(ShardSearchTransportRequest request, TransportChannel channel) throws Exception {
-            QuerySearchResultProvider result = searchService.executeQueryPhase(request);
-            channel.sendResponse(result);
-        }
-    }
-
-    class SearchQueryByIdTransportHandler implements TransportRequestHandler<QuerySearchRequest> {
-        @Override
-        public void messageReceived(QuerySearchRequest request, TransportChannel channel) throws Exception {
-            QuerySearchResult result = searchService.executeQueryPhase(request);
-            channel.sendResponse(result);
-        }
-    }
-
-    class SearchQueryScrollTransportHandler implements TransportRequestHandler<InternalScrollSearchRequest> {
-        @Override
-        public void messageReceived(InternalScrollSearchRequest request, TransportChannel channel) throws Exception {
-            ScrollQuerySearchResult result = searchService.executeQueryPhase(request);
-            channel.sendResponse(result);
-        }
-    }
-
-    class SearchQueryFetchTransportHandler implements TransportRequestHandler<ShardSearchTransportRequest> {
-        @Override
-        public void messageReceived(ShardSearchTransportRequest request, TransportChannel channel) throws Exception {
-            QueryFetchSearchResult result = searchService.executeFetchPhase(request);
-            channel.sendResponse(result);
-        }
-    }
-
-    class SearchQueryQueryFetchTransportHandler implements TransportRequestHandler<QuerySearchRequest> {
-        @Override
-        public void messageReceived(QuerySearchRequest request, TransportChannel channel) throws Exception {
-            QueryFetchSearchResult result = searchService.executeFetchPhase(request);
-            channel.sendResponse(result);
-        }
-    }
-
-    class FetchByIdTransportHandler<Request extends ShardFetchRequest> implements TransportRequestHandler<Request> {
-        @Override
-        public void messageReceived(Request request, TransportChannel channel) throws Exception {
-            FetchSearchResult result = searchService.executeFetchPhase(request);
-            channel.sendResponse(result);
-        }
-    }
-
-    class SearchQueryFetchScrollTransportHandler implements TransportRequestHandler<InternalScrollSearchRequest> {
-        @Override
-        public void messageReceived(InternalScrollSearchRequest request, TransportChannel channel) throws Exception {
-            ScrollQueryFetchSearchResult result = searchService.executeFetchPhase(request);
-            channel.sendResponse(result);
-        }
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/AbstractAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/AbstractAggregationBuilder.java
index e84d76843043c..ef82c48b4e7a2 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/AbstractAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/AbstractAggregationBuilder.java
@@ -21,8 +21,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.Map;
@@ -40,17 +39,16 @@ public abstract class AbstractAggregationBuilder<AB extends AbstractAggregationB
      * Constructs a new aggregation builder.
      *
      * @param name  The aggregation name
-     * @param type  The aggregation type
      */
-    public AbstractAggregationBuilder(String name, Type type) {
-        super(name, type);
+    public AbstractAggregationBuilder(String name) {
+        super(name);
     }
 
     /**
      * Read from a stream.
      */
-    protected AbstractAggregationBuilder(StreamInput in, Type type) throws IOException {
-        super(in.readString(), type);
+    protected AbstractAggregationBuilder(StreamInput in) throws IOException {
+        super(in.readString());
         factoriesBuilder = new AggregatorFactories.Builder(in);
         metaData = in.readMap();
     }
@@ -117,17 +115,19 @@ public AB setMetaData(Map<String, Object> metaData) {
         return (AB) this;
     }
 
-    public String getType() {
-        return type.name();
+    @Override
+    public final String getWriteableName() {
+        // We always use the type of the aggregation as the writeable name
+        return getType();
     }
 
     @Override
-    public final AggregatorFactory<?> build(AggregationContext context, AggregatorFactory<?> parent) throws IOException {
+    public final AggregatorFactory<?> build(SearchContext context, AggregatorFactory<?> parent) throws IOException {
         AggregatorFactory<?> factory = doBuild(context, parent, factoriesBuilder);
         return factory;
     }
 
-    protected abstract AggregatorFactory<?> doBuild(AggregationContext context, AggregatorFactory<?> parent,
+    protected abstract AggregatorFactory<?> doBuild(SearchContext context, AggregatorFactory<?> parent,
             AggregatorFactories.Builder subfactoriesBuilder) throws IOException;
 
     @Override
@@ -137,7 +137,7 @@ public final XContentBuilder toXContent(XContentBuilder builder, Params params)
         if (this.metaData != null) {
             builder.field("meta", this.metaData);
         }
-        builder.field(type.name());
+        builder.field(getType());
         internalXContent(builder, params);
 
         if (factoriesBuilder != null && (factoriesBuilder.count()) > 0) {
@@ -153,7 +153,7 @@ public final XContentBuilder toXContent(XContentBuilder builder, Params params)
 
     @Override
     public int hashCode() {
-        return Objects.hash(factoriesBuilder, metaData, name, type, doHashCode());
+        return Objects.hash(factoriesBuilder, metaData, name, doHashCode());
     }
 
     protected abstract int doHashCode();
@@ -168,8 +168,6 @@ public boolean equals(Object obj) {
         AbstractAggregationBuilder<AB> other = (AbstractAggregationBuilder<AB>) obj;
         if (!Objects.equals(name, other.name))
             return false;
-        if (!Objects.equals(type, other.type))
-            return false;
         if (!Objects.equals(metaData, other.metaData))
             return false;
         if (!Objects.equals(factoriesBuilder, other.factoriesBuilder))
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/Aggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/Aggregation.java
index 83472081021f9..2d8a81b080829 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/Aggregation.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/Aggregation.java
@@ -18,12 +18,21 @@
  */
 package org.elasticsearch.search.aggregations;
 
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ToXContent;
+
 import java.util.Map;
 
 /**
- * An aggregation
+ * An aggregation. Extends {@link ToXContent} as it makes it easier to print out its content.
  */
-public interface Aggregation {
+public interface Aggregation extends ToXContent {
+
+    /**
+     * Delimiter used when prefixing aggregation names with their type
+     * using the typed_keys parameter
+     */
+    String TYPED_KEYS_DELIMITER = "#";
 
     /**
      * @return The name of this aggregation.
@@ -31,16 +40,32 @@ public interface Aggregation {
     String getName();
 
     /**
-     * Get the value of specified path in the aggregation.
-     * 
-     * @param path
-     *            the path to the property in the aggregation tree
-     * @return the value of the property
+     * @return a string representing the type of the aggregation. This type is added to
+     * the aggregation name in the response, so that it can later be used by clients
+     * to determine type of the aggregation and parse it into the proper object.
      */
-    Object getProperty(String path);
+    String getType();
 
     /**
      * Get the optional byte array metadata that was set on the aggregation
      */
     Map<String, Object> getMetaData();
+
+    /**
+     * Common xcontent fields that are shared among addAggregation
+     */
+    final class CommonFields extends ParseField.CommonFields {
+        public static final ParseField META = new ParseField("meta");
+        public static final ParseField BUCKETS = new ParseField("buckets");
+        public static final ParseField VALUE = new ParseField("value");
+        public static final ParseField VALUES = new ParseField("values");
+        public static final ParseField VALUE_AS_STRING = new ParseField("value_as_string");
+        public static final ParseField DOC_COUNT = new ParseField("doc_count");
+        public static final ParseField KEY = new ParseField("key");
+        public static final ParseField KEY_AS_STRING = new ParseField("key_as_string");
+        public static final ParseField FROM = new ParseField("from");
+        public static final ParseField FROM_AS_STRING = new ParseField("from_as_string");
+        public static final ParseField TO = new ParseField("to");
+        public static final ParseField TO_AS_STRING = new ParseField("to_as_string");
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilder.java
index 9417ec8a0e659..14875895b77e1 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilder.java
@@ -23,8 +23,8 @@
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.Map;
@@ -34,27 +34,21 @@
  */
 public abstract class AggregationBuilder
     extends ToXContentToBytes
-    implements NamedWriteable, ToXContent {
+    implements NamedWriteable, ToXContent, BaseAggregationBuilder {
 
     protected final String name;
-    protected final Type type;
     protected AggregatorFactories.Builder factoriesBuilder = AggregatorFactories.builder();
 
     /**
      * Constructs a new aggregation builder.
      *
      * @param name  The aggregation name
-     * @param type  The aggregation type
      */
-    protected AggregationBuilder(String name, Type type) {
+    protected AggregationBuilder(String name) {
         if (name == null) {
             throw new IllegalArgumentException("[name] must not be null: [" + name + "]");
         }
-        if (type == null) {
-            throw new IllegalArgumentException("[type] must not be null: [" + name + "]");
-        }
         this.name = name;
-        this.type = type;
     }
 
     /** Return this aggregation's name. */
@@ -63,9 +57,10 @@ public String getName() {
     }
 
     /** Internal: build an {@link AggregatorFactory} based on the configuration of this builder. */
-    protected abstract AggregatorFactory<?> build(AggregationContext context, AggregatorFactory<?> parent) throws IOException;
+    protected abstract AggregatorFactory<?> build(SearchContext context, AggregatorFactory<?> parent) throws IOException;
 
     /** Associate metadata with this {@link AggregationBuilder}. */
+    @Override
     public abstract AggregationBuilder setMetaData(Map<String, Object> metaData);
 
     /** Add a sub aggregation to this builder. */
@@ -77,13 +72,14 @@ public String getName() {
     /**
      * Internal: Registers sub-factories with this factory. The sub-factory will be
      * responsible for the creation of sub-aggregators under the aggregator
-     * created by this factory. This is only for use by {@link AggregatorParsers}.
+     * created by this factory. This is only for use by {@link AggregatorFactories#parseAggregators(QueryParseContext)}.
      *
      * @param subFactories
      *            The sub-factories
      * @return this factory (fluent interface)
      */
-    protected abstract AggregationBuilder subAggregations(AggregatorFactories.Builder subFactories);
+    @Override
+    public abstract AggregationBuilder subAggregations(AggregatorFactories.Builder subFactories);
 
     /** Common xcontent fields shared among aggregator builders */
     public static final class CommonFields extends ParseField.CommonFields {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java b/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java
index b1818971d6bb6..8b704ee8a69a2 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java
@@ -21,8 +21,8 @@
 import org.elasticsearch.common.geo.GeoDistance;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.search.aggregations.bucket.children.Children;
-import org.elasticsearch.search.aggregations.bucket.children.ChildrenAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.adjacency.AdjacencyMatrix;
+import org.elasticsearch.search.aggregations.bucket.adjacency.AdjacencyMatrixAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.filter.Filter;
 import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.filters.Filters;
@@ -82,6 +82,8 @@
 import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCount;
 import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCountAggregationBuilder;
 
+import java.util.Map;
+
 /**
  * Utility class to create aggregations.
  */
@@ -160,6 +162,20 @@ public static FiltersAggregationBuilder filters(String name, QueryBuilder... fil
         return new FiltersAggregationBuilder(name, filters);
     }
 
+    /**
+     * Create a new {@link AdjacencyMatrix} aggregation with the given name.
+     */
+    public static AdjacencyMatrixAggregationBuilder adjacencyMatrix(String name, Map<String, QueryBuilder> filters) {
+        return new AdjacencyMatrixAggregationBuilder(name, filters);
+    }
+
+    /**
+     * Create a new {@link AdjacencyMatrix} aggregation with the given name and separator
+     */
+    public static AdjacencyMatrixAggregationBuilder adjacencyMatrix(String name, String separator,  Map<String, QueryBuilder> filters) {
+        return new AdjacencyMatrixAggregationBuilder(name, separator, filters);
+    }
+
     /**
      * Create a new {@link Sampler} aggregation with the given name.
      */
@@ -202,13 +218,6 @@ public static ReverseNestedAggregationBuilder reverseNested(String name) {
         return new ReverseNestedAggregationBuilder(name);
     }
 
-    /**
-     * Create a new {@link Children} aggregation with the given name.
-     */
-    public static ChildrenAggregationBuilder children(String name, String childType) {
-        return new ChildrenAggregationBuilder(name, childType);
-    }
-
     /**
      * Create a new {@link GeoDistance} aggregation with the given name.
      */
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/AggregationPhase.java b/core/src/main/java/org/elasticsearch/search/aggregations/AggregationPhase.java
index 5dc29374310a3..9b4a02cd8163b 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/AggregationPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/AggregationPhase.java
@@ -18,8 +18,6 @@
  */
 package org.elasticsearch.search.aggregations;
 
-import org.apache.lucene.search.BooleanClause.Occur;
-import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.Collector;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.common.inject.Inject;
@@ -28,7 +26,6 @@
 import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregator;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.pipeline.SiblingPipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.profile.query.CollectorResult;
 import org.elasticsearch.search.profile.query.InternalProfileCollector;
@@ -40,7 +37,7 @@
 import java.util.List;
 
 /**
- *
+ * Aggregation phase of a search request, used to collect aggregations
  */
 public class AggregationPhase implements SearchPhase {
 
@@ -51,9 +48,6 @@ public AggregationPhase() {
     @Override
     public void preProcess(SearchContext context) {
         if (context.aggregations() != null) {
-            AggregationContext aggregationContext = new AggregationContext(context);
-            context.aggregations().aggregationContext(aggregationContext);
-
             List<Aggregator> collectors = new ArrayList<>();
             Aggregator[] aggregators;
             try {
@@ -88,7 +82,7 @@ public void execute(SearchContext context) {
             return;
         }
 
-        if (context.queryResult().aggregations() != null) {
+        if (context.queryResult().hasAggs()) {
             // no need to compute the aggs twice, they should be computed on a per context basis
             return;
         }
@@ -104,16 +98,8 @@ public void execute(SearchContext context) {
         // optimize the global collector based execution
         if (!globals.isEmpty()) {
             BucketCollector globalsCollector = BucketCollector.wrap(globals);
-            Query query = Queries.newMatchAllQuery();
-            Query searchFilter = context.searchFilter(context.getQueryShardContext().getTypes());
+            Query query = context.buildFilteredQuery(Queries.newMatchAllQuery());
 
-            if (searchFilter != null) {
-                BooleanQuery filtered = new BooleanQuery.Builder()
-                    .add(query, Occur.MUST)
-                    .add(searchFilter, Occur.FILTER)
-                    .build();
-                query = filtered;
-            }
             try {
                 final Collector collector;
                 if (context.getProfilers() == null) {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/Aggregations.java b/core/src/main/java/org/elasticsearch/search/aggregations/Aggregations.java
index d7a6668ed468c..ba51fc419fb0f 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/Aggregations.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/Aggregations.java
@@ -18,41 +18,135 @@
  */
 package org.elasticsearch.search.aggregations;
 
+import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
+import java.util.Objects;
+
+import static java.util.Collections.unmodifiableMap;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.parseTypedKeysObject;
 
 /**
- * Represents a set of computed addAggregation.
+ * Represents a set of {@link Aggregation}s
  */
-public interface Aggregations extends Iterable<Aggregation> {
+public class Aggregations implements Iterable<Aggregation>, ToXContent {
+
+    public static final String AGGREGATIONS_FIELD = "aggregations";
+
+    protected List<? extends Aggregation> aggregations = Collections.emptyList();
+    protected Map<String, Aggregation> aggregationsAsMap;
+
+    protected Aggregations() {
+    }
+
+    public Aggregations(List<? extends Aggregation> aggregations) {
+        this.aggregations = aggregations;
+    }
+
+    /**
+     * Iterates over the {@link Aggregation}s.
+     */
+    @Override
+    public final Iterator<Aggregation> iterator() {
+        return aggregations.stream().map((p) -> (Aggregation) p).iterator();
+    }
 
     /**
      * The list of {@link Aggregation}s.
      */
-    List<Aggregation> asList();
+    public final List<Aggregation> asList() {
+        return Collections.unmodifiableList(aggregations);
+    }
 
     /**
      * Returns the {@link Aggregation}s keyed by aggregation name.
      */
-    Map<String, Aggregation> asMap();
+    public final Map<String, Aggregation> asMap() {
+        return getAsMap();
+    }
 
     /**
      * Returns the {@link Aggregation}s keyed by aggregation name.
      */
-    Map<String, Aggregation> getAsMap();
+    public final Map<String, Aggregation> getAsMap() {
+        if (aggregationsAsMap == null) {
+            Map<String, Aggregation> newAggregationsAsMap = new HashMap<>(aggregations.size());
+            for (Aggregation aggregation : aggregations) {
+                newAggregationsAsMap.put(aggregation.getName(), aggregation);
+            }
+            this.aggregationsAsMap = unmodifiableMap(newAggregationsAsMap);
+        }
+        return aggregationsAsMap;
+    }
 
     /**
      * Returns the aggregation that is associated with the specified name.
      */
-    <A extends Aggregation> A get(String name);
+    @SuppressWarnings("unchecked")
+    public final <A extends Aggregation> A get(String name) {
+        return (A) asMap().get(name);
+    }
+
+    @Override
+    public final boolean equals(Object obj) {
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+        return aggregations.equals(((Aggregations) obj).aggregations);
+    }
+
+    @Override
+    public final int hashCode() {
+        return Objects.hash(getClass(), aggregations);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        if (aggregations.isEmpty()) {
+            return builder;
+        }
+        builder.startObject(AGGREGATIONS_FIELD);
+        toXContentInternal(builder, params);
+        return builder.endObject();
+    }
 
     /**
-     * Get the value of specified path in the aggregation.
-     * 
-     * @param path
-     *            the path to the property in the aggregation tree
-     * @return the value of the property
+     * Directly write all the aggregations without their bounding object. Used by sub-aggregations (non top level aggs)
      */
-    Object getProperty(String path);
+    public XContentBuilder toXContentInternal(XContentBuilder builder, Params params) throws IOException {
+        for (Aggregation aggregation : aggregations) {
+            aggregation.toXContent(builder, params);
+        }
+        return builder;
+    }
 
+    public static Aggregations fromXContent(XContentParser parser) throws IOException {
+        final List<Aggregation> aggregations = new ArrayList<>();
+        XContentParser.Token token;
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.START_OBJECT) {
+                SetOnce<Aggregation> typedAgg = new SetOnce<>();
+                String currentField = parser.currentName();
+                parseTypedKeysObject(parser, Aggregation.TYPED_KEYS_DELIMITER, Aggregation.class, typedAgg::set);
+                if (typedAgg.get() != null) {
+                    aggregations.add(typedAgg.get());
+                } else {
+                    throw new ParsingException(parser.getTokenLocation(),
+                            String.format(Locale.ROOT, "Could not parse aggregation keyed as [%s]", currentField));
+                }
+            }
+        }
+        return new Aggregations(aggregations);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/Aggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/Aggregator.java
index bd81a397f040d..41a0fa6dd30a7 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/Aggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/Aggregator.java
@@ -21,14 +21,13 @@
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.aggregations.bucket.BucketsAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 
@@ -77,9 +76,9 @@ public static boolean descendsFromBucketAggregator(Aggregator parent) {
     public abstract String name();
 
     /**
-     * Return the {@link AggregationContext} attached with this {@link Aggregator}.
+     * Return the {@link SearchContext} attached with this {@link Aggregator}.
      */
-    public abstract AggregationContext context();
+    public abstract SearchContext context();
 
     /**
      * Return the parent aggregator.
@@ -129,10 +128,10 @@ public ParseField parseField() {
             return parseField;
         }
 
-        public static SubAggCollectionMode parse(String value, ParseFieldMatcher parseFieldMatcher) {
+        public static SubAggCollectionMode parse(String value) {
             SubAggCollectionMode[] modes = SubAggCollectionMode.values();
             for (SubAggCollectionMode mode : modes) {
-                if (parseFieldMatcher.match(value, mode.parseField)) {
+                if (mode.parseField.match(value)) {
                     return mode;
                 }
             }
@@ -140,16 +139,12 @@ public static SubAggCollectionMode parse(String value, ParseFieldMatcher parseFi
         }
 
         public static SubAggCollectionMode readFromStream(StreamInput in) throws IOException {
-            int ordinal = in.readVInt();
-            if (ordinal < 0 || ordinal >= values().length) {
-                throw new IOException("Unknown SubAggCollectionMode ordinal [" + ordinal + "]");
-            }
-            return values()[ordinal];
+            return in.readEnum(SubAggCollectionMode.class);
         }
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
-            out.writeVInt(ordinal());
+            out.writeEnum(this);
         }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java b/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java
index c99da85f33130..2d2c2abf65ef9 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java
@@ -25,7 +25,7 @@
 import org.elasticsearch.search.aggregations.bucket.BestBucketsDeferringCollector;
 import org.elasticsearch.search.aggregations.bucket.DeferringBucketCollector;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.internal.SearchContext.Lifetime;
 import org.elasticsearch.search.query.QueryPhaseExecutionException;
 
@@ -45,7 +45,7 @@ public abstract class AggregatorBase extends Aggregator {
 
     protected final String name;
     protected final Aggregator parent;
-    protected final AggregationContext context;
+    protected final SearchContext context;
     private final Map<String, Object> metaData;
 
     protected final Aggregator[] subAggregators;
@@ -66,7 +66,7 @@ public abstract class AggregatorBase extends Aggregator {
      * @param parent                The parent aggregator (may be {@code null} for top level aggregators)
      * @param metaData              The metaData associated with this aggregator
      */
-    protected AggregatorBase(String name, AggregatorFactories factories, AggregationContext context, Aggregator parent,
+    protected AggregatorBase(String name, AggregatorFactories factories, SearchContext context, Aggregator parent,
             List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException {
         this.name = name;
         this.pipelineAggregators = pipelineAggregators;
@@ -76,11 +76,11 @@ protected AggregatorBase(String name, AggregatorFactories factories, Aggregation
         this.breakerService = context.bigArrays().breakerService();
         assert factories != null : "sub-factories provided to BucketAggregator must not be null, use AggragatorFactories.EMPTY instead";
         this.subAggregators = factories.createSubAggregators(this);
-        context.searchContext().addReleasable(this, Lifetime.PHASE);
+        context.addReleasable(this, Lifetime.PHASE);
         // Register a safeguard to highlight any invalid construction logic (call to this constructor without subsequent preCollection call)
         collectableSubAggregators = new BucketCollector() {
             void badState(){
-                throw new QueryPhaseExecutionException(AggregatorBase.this.context.searchContext(),
+                throw new QueryPhaseExecutionException(AggregatorBase.this.context,
                         "preCollection not called on new Aggregator before use", null);
             }
             @Override
@@ -245,7 +245,7 @@ public Aggregator subAggregator(String aggName) {
      * @return  The current aggregation context.
      */
     @Override
-    public AggregationContext context() {
+    public SearchContext context() {
         return context;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java b/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java
index 616b0091fe878..0015276f8dd7c 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java
@@ -19,19 +19,25 @@
 package org.elasticsearch.search.aggregations;
 
 import org.elasticsearch.action.support.ToXContentToBytes;
+import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.AggregationPath;
 import org.elasticsearch.search.aggregations.support.AggregationPath.PathElement;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.profile.Profilers;
 import org.elasticsearch.search.profile.aggregation.ProfilingAggregator;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedList;
@@ -39,11 +45,129 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 /**
  *
  */
 public class AggregatorFactories {
+    public static final Pattern VALID_AGG_NAME = Pattern.compile("[^\\[\\]>]+");
+
+    /**
+     * Parses the aggregation request recursively generating aggregator factories in turn.
+     *
+     * @param parseContext   The parse context.
+     *
+     * @return          The parsed aggregator factories.
+     *
+     * @throws IOException When parsing fails for unknown reasons.
+     */
+    public static AggregatorFactories.Builder parseAggregators(QueryParseContext parseContext) throws IOException {
+        return parseAggregators(parseContext, 0);
+    }
+
+    private static AggregatorFactories.Builder parseAggregators(QueryParseContext parseContext, int level) throws IOException {
+        Matcher validAggMatcher = VALID_AGG_NAME.matcher("");
+        AggregatorFactories.Builder factories = new AggregatorFactories.Builder();
+
+        XContentParser.Token token = null;
+        XContentParser parser = parseContext.parser();
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token != XContentParser.Token.FIELD_NAME) {
+                throw new ParsingException(parser.getTokenLocation(),
+                        "Unexpected token " + token + " in [aggs]: aggregations definitions must start with the name of the aggregation.");
+            }
+            final String aggregationName = parser.currentName();
+            if (!validAggMatcher.reset(aggregationName).matches()) {
+                throw new ParsingException(parser.getTokenLocation(), "Invalid aggregation name [" + aggregationName
+                        + "]. Aggregation names must be alpha-numeric and can only contain '_' and '-'");
+            }
+
+            token = parser.nextToken();
+            if (token != XContentParser.Token.START_OBJECT) {
+                throw new ParsingException(parser.getTokenLocation(), "Aggregation definition for [" + aggregationName + " starts with a ["
+                        + token + "], expected a [" + XContentParser.Token.START_OBJECT + "].");
+            }
+
+            BaseAggregationBuilder aggBuilder = null;
+            AggregatorFactories.Builder subFactories = null;
+
+            Map<String, Object> metaData = null;
+
+            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                if (token != XContentParser.Token.FIELD_NAME) {
+                    throw new ParsingException(
+                            parser.getTokenLocation(), "Expected [" + XContentParser.Token.FIELD_NAME + "] under a ["
+                            + XContentParser.Token.START_OBJECT + "], but got a [" + token + "] in [" + aggregationName + "]",
+                            parser.getTokenLocation());
+                }
+                final String fieldName = parser.currentName();
+
+                token = parser.nextToken();
+                if (token == XContentParser.Token.START_OBJECT) {
+                    switch (fieldName) {
+                    case "meta":
+                        metaData = parser.map();
+                        break;
+                    case "aggregations":
+                    case "aggs":
+                        if (subFactories != null) {
+                            throw new ParsingException(parser.getTokenLocation(),
+                                    "Found two sub aggregation definitions under [" + aggregationName + "]");
+                        }
+                        subFactories = parseAggregators(parseContext, level + 1);
+                        break;
+                    default:
+                        if (aggBuilder != null) {
+                            throw new ParsingException(parser.getTokenLocation(), "Found two aggregation type definitions in ["
+                                    + aggregationName + "]: [" + aggBuilder.getType() + "] and [" + fieldName + "]");
+                        }
+
+                        aggBuilder = parser.namedObject(BaseAggregationBuilder.class, fieldName,
+                                new AggParseContext(aggregationName, parseContext));
+                    }
+                } else {
+                    throw new ParsingException(parser.getTokenLocation(), "Expected [" + XContentParser.Token.START_OBJECT + "] under ["
+                            + fieldName + "], but got a [" + token + "] in [" + aggregationName + "]");
+                }
+            }
+
+            if (aggBuilder == null) {
+                throw new ParsingException(parser.getTokenLocation(), "Missing definition for aggregation [" + aggregationName + "]",
+                        parser.getTokenLocation());
+            } else {
+                if (metaData != null) {
+                    aggBuilder.setMetaData(metaData);
+                }
+
+                if (subFactories != null) {
+                    aggBuilder.subAggregations(subFactories);
+                }
+
+                if (aggBuilder instanceof AggregationBuilder) {
+                    factories.addAggregator((AggregationBuilder) aggBuilder);
+                } else {
+                    factories.addPipelineAggregator((PipelineAggregationBuilder) aggBuilder);
+                }
+            }
+        }
+
+        return factories;
+    }
+
+    /**
+     * Context to parse and aggregation. This should eventually be removed and replaced with a String.
+     */
+    public static final class AggParseContext {
+        public final String name;
+        public final QueryParseContext queryParseContext;
+
+        public AggParseContext(String name, QueryParseContext queryParseContext) {
+            this.name = name;
+            this.queryParseContext = queryParseContext;
+        }
+    }
 
     public static final AggregatorFactories EMPTY = new AggregatorFactories(null, new AggregatorFactory<?>[0],
             new ArrayList<PipelineAggregationBuilder>());
@@ -64,7 +188,7 @@ private AggregatorFactories(AggregatorFactory<?> parent, AggregatorFactory<?>[]
     }
 
     public List<PipelineAggregator> createPipelineAggregators() throws IOException {
-        List<PipelineAggregator> pipelineAggregators = new ArrayList<>();
+        List<PipelineAggregator> pipelineAggregators = new ArrayList<>(this.pipelineAggregatorFactories.size());
         for (PipelineAggregationBuilder factory : this.pipelineAggregatorFactories) {
             pipelineAggregators.add(factory.create());
         }
@@ -84,7 +208,7 @@ public Aggregator[] createSubAggregators(Aggregator parent) throws IOException {
             // aggs
             final boolean collectsFromSingleBucket = false;
             Aggregator factory = factories[i].create(parent, collectsFromSingleBucket);
-            Profilers profilers = factory.context().searchContext().getProfilers();
+            Profilers profilers = factory.context().getProfilers();
             if (profilers != null) {
                 factory = new ProfilingAggregator(factory, profilers.getAggregationProfiler());
             }
@@ -100,7 +224,7 @@ public Aggregator[] createTopLevelAggregators() throws IOException {
             // top-level aggs only get called with bucket 0
             final boolean collectsFromSingleBucket = true;
             Aggregator factory = factories[i].create(null, collectsFromSingleBucket);
-            Profilers profilers = factory.context().searchContext().getProfilers();
+            Profilers profilers = factory.context().getProfilers();
             if (profilers != null) {
                 factory = new ProfilingAggregator(factory, profilers.getAggregationProfiler());
             }
@@ -171,10 +295,22 @@ public void writeTo(StreamOutput out) throws IOException {
             }
         }
 
-        public Builder addAggregators(AggregatorFactories factories) {
-            throw new UnsupportedOperationException("This needs to be removed");
+        public boolean mustVisitAllDocs() {
+            for (AggregationBuilder builder : aggregationBuilders) {
+                if (builder instanceof GlobalAggregationBuilder) {
+                    return true;
+                } else if (builder instanceof TermsAggregationBuilder) {
+                    if (((TermsAggregationBuilder) builder).minDocCount() == 0) {
+                        return true;
+                    }
+                }
+
+            }
+            return false;
         }
 
+
+
         public Builder addAggregator(AggregationBuilder factory) {
             if (!names.add(factory.name)) {
                 throw new IllegalArgumentException("Two sibling aggregations cannot have the same name: [" + factory.name + "]");
@@ -196,7 +332,7 @@ Builder skipResolveOrder() {
             return this;
         }
 
-        public AggregatorFactories build(AggregationContext context, AggregatorFactory<?> parent) throws IOException {
+        public AggregatorFactories build(SearchContext context, AggregatorFactory<?> parent) throws IOException {
             if (aggregationBuilders.isEmpty() && pipelineAggregatorBuilders.isEmpty()) {
                 return EMPTY;
             }
@@ -258,7 +394,7 @@ private void resolvePipelineAggregatorOrder(Map<String, AggregationBuilder> aggB
                             } else {
                                 // Check the non-pipeline sub-aggregator
                                 // factories
-                                AggregationBuilder[] subBuilders = aggBuilder.factoriesBuilder.getAggregatorFactories();
+                                List<AggregationBuilder> subBuilders = aggBuilder.factoriesBuilder.aggregationBuilders;
                                 boolean foundSubBuilder = false;
                                 for (AggregationBuilder subBuilder : subBuilders) {
                                     if (aggName.equals(subBuilder.name)) {
@@ -300,12 +436,12 @@ private void resolvePipelineAggregatorOrder(Map<String, AggregationBuilder> aggB
             }
         }
 
-        AggregationBuilder[] getAggregatorFactories() {
-            return this.aggregationBuilders.toArray(new AggregationBuilder[this.aggregationBuilders.size()]);
+        public List<AggregationBuilder> getAggregatorFactories() {
+            return Collections.unmodifiableList(aggregationBuilders);
         }
 
-        List<PipelineAggregationBuilder> getPipelineAggregatorFactories() {
-            return this.pipelineAggregatorBuilders;
+        public List<PipelineAggregationBuilder> getPipelineAggregatorFactories() {
+            return Collections.unmodifiableList(pipelineAggregatorBuilders);
         }
 
         public int count() {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactory.java
index 44ecacd841773..7b6340996a928 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactory.java
@@ -24,10 +24,10 @@
 import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.ObjectArray;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.internal.SearchContext.Lifetime;
+
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
@@ -42,13 +42,13 @@ public static final class MultiBucketAggregatorWrapper extends Aggregator {
         ObjectArray<Aggregator> aggregators;
         ObjectArray<LeafBucketCollector> collectors;
 
-        MultiBucketAggregatorWrapper(BigArrays bigArrays, AggregationContext context, Aggregator parent, AggregatorFactory<?> factory,
+        MultiBucketAggregatorWrapper(BigArrays bigArrays, SearchContext context, Aggregator parent, AggregatorFactory<?> factory,
                 Aggregator first) {
             this.bigArrays = bigArrays;
             this.parent = parent;
             this.factory = factory;
             this.first = first;
-            context.searchContext().addReleasable(this, Lifetime.PHASE);
+            context.addReleasable(this, Lifetime.PHASE);
             aggregators = bigArrays.newObjectArray(1);
             aggregators.set(0, first);
             collectors = bigArrays.newObjectArray(1);
@@ -64,7 +64,7 @@ public String name() {
         }
 
         @Override
-        public AggregationContext context() {
+        public SearchContext context() {
             return first.context();
         }
 
@@ -130,7 +130,11 @@ public void collect(int doc, long bucket) throws IOException {
                             aggregators.set(bucket, aggregator);
                         }
                         collector = aggregator.getLeafCollector(ctx);
-                        collector.setScorer(scorer);
+                        if (scorer != null) {
+                            // Passing a null scorer can cause unexpected NPE at a later time,
+                            // which can't not be directly linked to the fact that a null scorer has been supplied.
+                            collector.setScorer(scorer);
+                        }
                         collectors.set(bucket, collector);
                     }
                     collector.collect(doc, 0);
@@ -162,26 +166,22 @@ public void close() {
     }
 
     protected final String name;
-    protected final Type type;
     protected final AggregatorFactory<?> parent;
     protected final AggregatorFactories factories;
     protected final Map<String, Object> metaData;
-    protected final AggregationContext context;
+    protected final SearchContext context;
 
     /**
      * Constructs a new aggregator factory.
      *
      * @param name
      *            The aggregation name
-     * @param type
-     *            The aggregation type
      * @throws IOException
      *             if an error occurs creating the factory
      */
-    public AggregatorFactory(String name, Type type, AggregationContext context, AggregatorFactory<?> parent,
+    public AggregatorFactory(String name, SearchContext context, AggregatorFactory<?> parent,
             AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
         this.name = name;
-        this.type = type;
         this.context = context;
         this.parent = parent;
         this.factories = subFactoriesBuilder.build(context, this);
@@ -225,10 +225,6 @@ public final Aggregator create(Aggregator parent, boolean collectsFromSingleBuck
         return createInternal(parent, collectsFromSingleBucket, this.factories.createPipelineAggregators(), this.metaData);
     }
 
-    public String getType() {
-        return type.name();
-    }
-
     public AggregatorFactory<?> getParent() {
         return parent;
     }
@@ -238,7 +234,7 @@ public AggregatorFactory<?> getParent() {
      * {@link Aggregator}s that only know how to collect bucket <tt>0</tt>, this
      * returns an aggregator that can collect any bucket.
      */
-    protected static Aggregator asMultiBucketAggregator(final AggregatorFactory<?> factory, final AggregationContext context,
+    protected static Aggregator asMultiBucketAggregator(final AggregatorFactory<?> factory, final SearchContext context,
             final Aggregator parent) throws IOException {
         final Aggregator first = factory.create(parent, true);
         final BigArrays bigArrays = context.bigArrays();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorParsers.java b/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorParsers.java
deleted file mode 100644
index df64d440dc0b7..0000000000000
--- a/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorParsers.java
+++ /dev/null
@@ -1,196 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.search.aggregations;
-
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.xcontent.ParseFieldRegistry;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-/**
- * A registry for all the aggregator parser, also servers as the main parser for the aggregations module
- */
-public class AggregatorParsers {
-    public static final Pattern VALID_AGG_NAME = Pattern.compile("[^\\[\\]>]+");
-
-    private final ParseFieldRegistry<Aggregator.Parser> aggregationParserRegistry;
-    private final ParseFieldRegistry<PipelineAggregator.Parser> pipelineAggregationParserRegistry;
-
-    public AggregatorParsers(ParseFieldRegistry<Aggregator.Parser> aggregationParserRegistry,
-            ParseFieldRegistry<PipelineAggregator.Parser> pipelineAggregationParserRegistry) {
-        this.aggregationParserRegistry = aggregationParserRegistry;
-        this.pipelineAggregationParserRegistry = pipelineAggregationParserRegistry;
-    }
-
-    /**
-     * Returns the parser that is registered under the given aggregation type.
-     *
-     * @param type The aggregation type
-     * @param parseFieldMatcher used for making error messages.
-     * @return The parser associated with the given aggregation type or null if it wasn't found.
-     */
-    public Aggregator.Parser parser(String type, ParseFieldMatcher parseFieldMatcher) {
-        return aggregationParserRegistry.lookupReturningNullIfNotFound(type, parseFieldMatcher);
-    }
-
-    /**
-     * Returns the parser that is registered under the given pipeline aggregator type.
-     *
-     * @param type The pipeline aggregator type
-     * @param parseFieldMatcher used for making error messages.
-     * @return The parser associated with the given pipeline aggregator type or null if it wasn't found.
-     */
-    public PipelineAggregator.Parser pipelineParser(String type, ParseFieldMatcher parseFieldMatcher) {
-        return pipelineAggregationParserRegistry.lookupReturningNullIfNotFound(type, parseFieldMatcher);
-    }
-
-    /**
-     * Parses the aggregation request recursively generating aggregator factories in turn.
-     *
-     * @param parseContext   The parse context.
-     *
-     * @return          The parsed aggregator factories.
-     *
-     * @throws IOException When parsing fails for unknown reasons.
-     */
-    public AggregatorFactories.Builder parseAggregators(QueryParseContext parseContext) throws IOException {
-        return parseAggregators(parseContext, 0);
-    }
-
-    private AggregatorFactories.Builder parseAggregators(QueryParseContext parseContext, int level) throws IOException {
-        Matcher validAggMatcher = VALID_AGG_NAME.matcher("");
-        AggregatorFactories.Builder factories = new AggregatorFactories.Builder();
-
-        XContentParser.Token token = null;
-        XContentParser parser = parseContext.parser();
-        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-            if (token != XContentParser.Token.FIELD_NAME) {
-                throw new ParsingException(parser.getTokenLocation(),
-                        "Unexpected token " + token + " in [aggs]: aggregations definitions must start with the name of the aggregation.");
-            }
-            final String aggregationName = parser.currentName();
-            if (!validAggMatcher.reset(aggregationName).matches()) {
-                throw new ParsingException(parser.getTokenLocation(), "Invalid aggregation name [" + aggregationName
-                        + "]. Aggregation names must be alpha-numeric and can only contain '_' and '-'");
-            }
-
-            token = parser.nextToken();
-            if (token != XContentParser.Token.START_OBJECT) {
-                throw new ParsingException(parser.getTokenLocation(), "Aggregation definition for [" + aggregationName + " starts with a ["
-                        + token + "], expected a [" + XContentParser.Token.START_OBJECT + "].");
-            }
-
-            AggregationBuilder aggFactory = null;
-            PipelineAggregationBuilder pipelineAggregatorFactory = null;
-            AggregatorFactories.Builder subFactories = null;
-
-            Map<String, Object> metaData = null;
-
-            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                if (token != XContentParser.Token.FIELD_NAME) {
-                    throw new ParsingException(
-                            parser.getTokenLocation(), "Expected [" + XContentParser.Token.FIELD_NAME + "] under a ["
-                            + XContentParser.Token.START_OBJECT + "], but got a [" + token + "] in [" + aggregationName + "]",
-                            parser.getTokenLocation());
-                }
-                final String fieldName = parser.currentName();
-
-                token = parser.nextToken();
-                if (token == XContentParser.Token.START_OBJECT) {
-                    switch (fieldName) {
-                    case "meta":
-                        metaData = parser.map();
-                        break;
-                    case "aggregations":
-                    case "aggs":
-                        if (subFactories != null) {
-                            throw new ParsingException(parser.getTokenLocation(),
-                                    "Found two sub aggregation definitions under [" + aggregationName + "]");
-                        }
-                        subFactories = parseAggregators(parseContext, level + 1);
-                        break;
-                    default:
-                        if (aggFactory != null) {
-                            throw new ParsingException(parser.getTokenLocation(), "Found two aggregation type definitions in ["
-                                    + aggregationName + "]: [" + aggFactory.type + "] and [" + fieldName + "]");
-                        }
-                        if (pipelineAggregatorFactory != null) {
-                            throw new ParsingException(parser.getTokenLocation(), "Found two aggregation type definitions in ["
-                                    + aggregationName + "]: [" + pipelineAggregatorFactory + "] and [" + fieldName + "]");
-                        }
-
-                        Aggregator.Parser aggregatorParser = parser(fieldName, parseContext.getParseFieldMatcher());
-                        if (aggregatorParser == null) {
-                            PipelineAggregator.Parser pipelineAggregatorParser = pipelineParser(fieldName,
-                                    parseContext.getParseFieldMatcher());
-                            if (pipelineAggregatorParser == null) {
-                                throw new ParsingException(parser.getTokenLocation(),
-                                        "Could not find aggregator type [" + fieldName + "] in [" + aggregationName + "]");
-                            } else {
-                                pipelineAggregatorFactory = pipelineAggregatorParser.parse(aggregationName, parseContext);
-                            }
-                        } else {
-                            aggFactory = aggregatorParser.parse(aggregationName, parseContext);
-                        }
-                    }
-                } else {
-                    throw new ParsingException(parser.getTokenLocation(), "Expected [" + XContentParser.Token.START_OBJECT + "] under ["
-                            + fieldName + "], but got a [" + token + "] in [" + aggregationName + "]");
-                }
-            }
-
-            if (aggFactory == null && pipelineAggregatorFactory == null) {
-                throw new ParsingException(parser.getTokenLocation(), "Missing definition for aggregation [" + aggregationName + "]",
-                        parser.getTokenLocation());
-            } else if (aggFactory != null) {
-                assert pipelineAggregatorFactory == null;
-            if (metaData != null) {
-                    aggFactory.setMetaData(metaData);
-            }
-
-            if (subFactories != null) {
-                    aggFactory.subAggregations(subFactories);
-            }
-
-                factories.addAggregator(aggFactory);
-            } else {
-                assert pipelineAggregatorFactory != null;
-                if (subFactories != null) {
-                    throw new ParsingException(parser.getTokenLocation(),
-                            "Aggregation [" + aggregationName + "] cannot define sub-aggregations",
-                            parser.getTokenLocation());
-                }
-                if (metaData != null) {
-                    pipelineAggregatorFactory.setMetaData(metaData);
-                }
-                factories.addPipelineAggregator(pipelineAggregatorFactory);
-            }
-        }
-
-        return factories;
-    }
-
-}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/BaseAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/BaseAggregationBuilder.java
new file mode 100644
index 0000000000000..8f076cfe4567f
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/BaseAggregationBuilder.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
+
+import java.util.Map;
+
+/**
+ * Interface shared by {@link AggregationBuilder} and {@link PipelineAggregationBuilder} so they can conveniently share the same namespace
+ * for {@link XContentParser#namedObject(Class, String, Object)}.
+ */
+public interface BaseAggregationBuilder {
+    /**
+     * The name of the type of aggregation built by this builder. 
+     */
+    String getType();
+
+    /**
+     * Set the aggregation's metadata. Returns {@code this} for chaining.
+     */
+    BaseAggregationBuilder setMetaData(Map<String, Object> metaData);
+
+    /**
+     * Set the sub aggregations if this aggregation supports sub aggregations. Returns {@code this} for chaining.
+     */
+    BaseAggregationBuilder subAggregations(Builder subFactories);
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/BucketCollector.java b/core/src/main/java/org/elasticsearch/search/aggregations/BucketCollector.java
index 2de6ae0cf933f..40e66bd964539 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/BucketCollector.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/BucketCollector.java
@@ -70,7 +70,7 @@ public static BucketCollector wrap(Iterable<? extends BucketCollector> collector
 
                     @Override
                     public LeafBucketCollector getLeafCollector(LeafReaderContext ctx) throws IOException {
-                        List<LeafBucketCollector> leafCollectors = new ArrayList<>();
+                        List<LeafBucketCollector> leafCollectors = new ArrayList<>(collectors.length);
                         for (BucketCollector c : collectors) {
                             leafCollectors.add(c.getLeafCollector(ctx));
                         }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java
index b7635d3dc320e..b34d3daa1a0e3 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java
@@ -18,14 +18,13 @@
  */
 package org.elasticsearch.search.aggregations;
 
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.rest.action.search.RestSearchAction;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.support.AggregationPath;
@@ -38,43 +37,26 @@
  * An internal implementation of {@link Aggregation}. Serves as a base class for all aggregation implementations.
  */
 public abstract class InternalAggregation implements Aggregation, ToXContent, NamedWriteable {
-    /**
-     * The aggregation type that holds all the string types that are associated with an aggregation:
-     * <ul>
-     *     <li>name - used as the parser type</li>
-     * </ul>
-     */
-    public static class Type {
-        private final String name;
-
-        public Type(String name) {
-            this.name = name;
-        }
-
-        /**
-         * @return The name of the type of aggregation.  This is the key for parsing the aggregation from XContent and is the name of the
-         * aggregation's builder when serialized.
-         */
-        public String name() {
-            return name;
-        }
-
-        @Override
-        public String toString() {
-            return name;
-        }
-    }
 
     public static class ReduceContext {
 
         private final BigArrays bigArrays;
         private final ScriptService scriptService;
-        private final ClusterState clusterState;
+        private final boolean isFinalReduce;
 
-        public ReduceContext(BigArrays bigArrays, ScriptService scriptService, ClusterState clusterState) {
+        public ReduceContext(BigArrays bigArrays, ScriptService scriptService, boolean isFinalReduce) {
             this.bigArrays = bigArrays;
             this.scriptService = scriptService;
-            this.clusterState = clusterState;
+            this.isFinalReduce = isFinalReduce;
+        }
+
+        /**
+         * Returns <code>true</code> iff the current reduce phase is the final reduce phase. This indicates if operations like
+         * pipeline aggregations should be applied or if specific features like <tt>minDocCount</tt> should be taken into account.
+         * Operations that are potentially loosing information can only be applied during the final reduce phase.
+         */
+        public boolean isFinalReduce() {
+            return isFinalReduce;
         }
 
         public BigArrays bigArrays() {
@@ -84,10 +66,6 @@ public BigArrays bigArrays() {
         public ScriptService scriptService() {
             return scriptService;
         }
-
-        public ClusterState clusterState() {
-            return clusterState;
-        }
     }
 
     protected final String name;
@@ -126,7 +104,6 @@ public final void writeTo(StreamOutput out) throws IOException {
 
     protected abstract void doWriteTo(StreamOutput out) throws IOException;
 
-
     @Override
     public String getName() {
         return name;
@@ -140,15 +117,23 @@ public String getName() {
      */
     public final InternalAggregation reduce(List<InternalAggregation> aggregations, ReduceContext reduceContext) {
         InternalAggregation aggResult = doReduce(aggregations, reduceContext);
-        for (PipelineAggregator pipelineAggregator : pipelineAggregators) {
-            aggResult = pipelineAggregator.reduce(aggResult, reduceContext);
+        if (reduceContext.isFinalReduce()) {
+            for (PipelineAggregator pipelineAggregator : pipelineAggregators) {
+                aggResult = pipelineAggregator.reduce(aggResult, reduceContext);
+            }
         }
         return aggResult;
     }
 
     public abstract InternalAggregation doReduce(List<InternalAggregation> aggregations, ReduceContext reduceContext);
 
-    @Override
+    /**
+     * Get the value of specified path in the aggregation.
+     *
+     * @param path
+     *            the path to the property in the aggregation tree
+     * @return the value of the property
+     */
     public Object getProperty(String path) {
         AggregationPath aggPath = AggregationPath.parse(path);
         return getProperty(aggPath.getPathElementsAsStringList());
@@ -183,11 +168,21 @@ public List<PipelineAggregator> pipelineAggregators() {
         return pipelineAggregators;
     }
 
+    @Override
+    public String getType() {
+        return getWriteableName();
+    }
+
     @Override
     public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject(name);
+        if (params.paramAsBoolean(RestSearchAction.TYPED_KEYS_PARAM, false)) {
+            // Concatenates the type and the name of the aggregation (ex: top_hits#foo)
+            builder.startObject(String.join(TYPED_KEYS_DELIMITER, getType(), getName()));
+        } else {
+            builder.startObject(getName());
+        }
         if (this.metaData != null) {
-            builder.field(CommonFields.META);
+            builder.field(CommonFields.META.getPreferredName());
             builder.map(this.metaData);
         }
         doXContentBody(builder, params);
@@ -196,24 +191,4 @@ public final XContentBuilder toXContent(XContentBuilder builder, Params params)
     }
 
     public abstract XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException;
-
-    /**
-     * Common xcontent fields that are shared among addAggregation
-     */
-    public static final class CommonFields extends ParseField.CommonFields {
-        // todo convert these to ParseField
-        public static final String META = "meta";
-        public static final String BUCKETS = "buckets";
-        public static final String VALUE = "value";
-        public static final String VALUES = "values";
-        public static final String VALUE_AS_STRING = "value_as_string";
-        public static final String DOC_COUNT = "doc_count";
-        public static final String KEY = "key";
-        public static final String KEY_AS_STRING = "key_as_string";
-        public static final String FROM = "from";
-        public static final String FROM_AS_STRING = "from_as_string";
-        public static final String TO = "to";
-        public static final String TO_AS_STRING = "to_as_string";
-    }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java b/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java
index 66e45156caf06..15ba641c711cb 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java
@@ -22,32 +22,22 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
 import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext;
-import org.elasticsearch.search.aggregations.support.AggregationPath;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
-import java.util.stream.Collectors;
 
-import static java.util.Collections.emptyMap;
-import static java.util.Collections.unmodifiableMap;
 /**
  * An internal implementation of {@link Aggregations}.
  */
-public class InternalAggregations implements Aggregations, ToXContent, Streamable {
+public final class InternalAggregations extends Aggregations implements ToXContent, Streamable {
 
     public static final InternalAggregations EMPTY = new InternalAggregations();
 
-    private List<InternalAggregation> aggregations = Collections.emptyList();
-
-    private Map<String, Aggregation> aggregationsAsMap;
-
     private InternalAggregations() {
     }
 
@@ -55,73 +45,7 @@ private InternalAggregations() {
      * Constructs a new addAggregation.
      */
     public InternalAggregations(List<InternalAggregation> aggregations) {
-        this.aggregations = aggregations;
-    }
-
-    /**
-     * Iterates over the {@link Aggregation}s.
-     */
-    @Override
-    public Iterator<Aggregation> iterator() {
-        return aggregations.stream().map((p) -> (Aggregation) p).iterator();
-    }
-
-    /**
-     * The list of {@link Aggregation}s.
-     */
-    @Override
-    public List<Aggregation> asList() {
-        return aggregations.stream().map((p) -> (Aggregation) p).collect(Collectors.toList());
-    }
-
-    /**
-     * Returns the {@link Aggregation}s keyed by map.
-     */
-    @Override
-    public Map<String, Aggregation> asMap() {
-        return getAsMap();
-    }
-
-    /**
-     * Returns the {@link Aggregation}s keyed by map.
-     */
-    @Override
-    public Map<String, Aggregation> getAsMap() {
-        if (aggregationsAsMap == null) {
-            Map<String, InternalAggregation> newAggregationsAsMap = new HashMap<>();
-            for (InternalAggregation aggregation : aggregations) {
-                newAggregationsAsMap.put(aggregation.getName(), aggregation);
-            }
-            this.aggregationsAsMap = unmodifiableMap(newAggregationsAsMap);
-        }
-        return aggregationsAsMap;
-    }
-
-    /**
-     * @return the aggregation of the specified name.
-     */
-    @SuppressWarnings("unchecked")
-    @Override
-    public <A extends Aggregation> A get(String name) {
-        return (A) asMap().get(name);
-    }
-
-    @Override
-    public Object getProperty(String path) {
-        AggregationPath aggPath = AggregationPath.parse(path);
-        return getProperty(aggPath.getPathElementsAsStringList());
-    }
-
-    public Object getProperty(List<String> path) {
-        if (path.isEmpty()) {
-            return this;
-        }
-        String aggName = path.get(0);
-        InternalAggregation aggregation = get(aggName);
-        if (aggregation == null) {
-            throw new IllegalArgumentException("Cannot find an aggregation named [" + aggName + "]");
-        }
-        return aggregation.getProperty(path.subList(1, path.size()));
+        super(aggregations);
     }
 
     /**
@@ -136,21 +60,16 @@ public static InternalAggregations reduce(List<InternalAggregations> aggregation
         }
 
         // first we collect all aggregations of the same type and list them together
-
         Map<String, List<InternalAggregation>> aggByName = new HashMap<>();
         for (InternalAggregations aggregations : aggregationsList) {
-            for (InternalAggregation aggregation : aggregations.aggregations) {
-                List<InternalAggregation> aggs = aggByName.get(aggregation.getName());
-                if (aggs == null) {
-                    aggs = new ArrayList<>(aggregationsList.size());
-                    aggByName.put(aggregation.getName(), aggs);
-                }
-                aggs.add(aggregation);
+            for (Aggregation aggregation : aggregations.aggregations) {
+                List<InternalAggregation> aggs = aggByName.computeIfAbsent(
+                        aggregation.getName(), k -> new ArrayList<>(aggregationsList.size()));
+                aggs.add((InternalAggregation)aggregation);
             }
         }
 
         // now we can use the first aggregation of each list to handle the reduce of its list
-
         List<InternalAggregation> reducedAggregations = new ArrayList<>();
         for (Map.Entry<String, List<InternalAggregation>> entry : aggByName.entrySet()) {
             List<InternalAggregation> aggregations = entry.getValue();
@@ -160,31 +79,6 @@ public static InternalAggregations reduce(List<InternalAggregations> aggregation
         return new InternalAggregations(reducedAggregations);
     }
 
-    /** The fields required to write this addAggregation to xcontent */
-    static class Fields {
-        public static final String AGGREGATIONS = "aggregations";
-    }
-
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        if (aggregations.isEmpty()) {
-            return builder;
-        }
-        builder.startObject(Fields.AGGREGATIONS);
-        toXContentInternal(builder, params);
-        return builder.endObject();
-    }
-
-    /**
-     * Directly write all the addAggregation without their bounding object. Used by sub-addAggregation (non top level addAggregation)
-     */
-    public XContentBuilder toXContentInternal(XContentBuilder builder, Params params) throws IOException {
-        for (Aggregation aggregation : aggregations) {
-            ((InternalAggregation) aggregation).toXContent(builder, params);
-        }
-        return builder;
-    }
-
     public static InternalAggregations readAggregations(StreamInput in) throws IOException {
         InternalAggregations result = new InternalAggregations();
         result.readFrom(in);
@@ -199,13 +93,13 @@ public static InternalAggregations readOptionalAggregations(StreamInput in) thro
     public void readFrom(StreamInput in) throws IOException {
         aggregations = in.readList(stream -> in.readNamedWriteable(InternalAggregation.class));
         if (aggregations.isEmpty()) {
-            aggregationsAsMap = emptyMap();
+            aggregationsAsMap = Collections.emptyMap();
         }
     }
 
     @Override
+    @SuppressWarnings("unchecked")
     public void writeTo(StreamOutput out) throws IOException {
-        out.writeNamedWriteableList(aggregations);
+        out.writeNamedWriteableList((List<InternalAggregation>)aggregations);
     }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java
index a1326aaed117c..8e8f4edcf3193 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.search.aggregations;
 
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 
@@ -62,6 +63,9 @@ protected InternalMultiBucketAggregation(StreamInput in) throws IOException {
      */
     public abstract B createBucket(InternalAggregations aggregations, B prototype);
 
+    @Override
+    public abstract List<? extends InternalBucket> getBuckets();
+
     @Override
     public Object getProperty(List<String> path) {
         if (path.isEmpty()) {
@@ -69,7 +73,7 @@ public Object getProperty(List<String> path) {
         } else if (path.get(0).equals("_bucket_count")) {
             return getBuckets().size();
         } else {
-            List<? extends Bucket> buckets = getBuckets();
+            List<? extends InternalBucket> buckets = getBuckets();
             Object[] propertyArray = new Object[buckets.size()];
             for (int i = 0; i < buckets.size(); i++) {
                 propertyArray[i] = buckets.get(i).getProperty(getName(), path);
@@ -78,8 +82,8 @@ public Object getProperty(List<String> path) {
         }
     }
 
-    public abstract static class InternalBucket implements Bucket {
-        @Override
+    public abstract static class InternalBucket implements Bucket, Writeable {
+
         public Object getProperty(String containingAggName, List<String> path) {
             if (path.isEmpty()) {
                 return this;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/NonCollectingAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/NonCollectingAggregator.java
index 7e0e3cc9f9fe4..d65afd2ba9125 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/NonCollectingAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/NonCollectingAggregator.java
@@ -21,7 +21,7 @@
 
 import org.apache.lucene.index.LeafReaderContext;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -33,12 +33,12 @@
  */
 public abstract class NonCollectingAggregator extends AggregatorBase {
 
-    protected NonCollectingAggregator(String name, AggregationContext context, Aggregator parent, AggregatorFactories subFactories,
+    protected NonCollectingAggregator(String name, SearchContext context, Aggregator parent, AggregatorFactories subFactories,
             List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException {
         super(name, subFactories, context, parent, pipelineAggregators, metaData);
     }
 
-    protected NonCollectingAggregator(String name, AggregationContext context, Aggregator parent,
+    protected NonCollectingAggregator(String name, SearchContext context, Aggregator parent,
             List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException {
         this(name, context, parent, AggregatorFactories.EMPTY, pipelineAggregators, metaData);
     }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/ParsedAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/ParsedAggregation.java
new file mode 100644
index 0000000000000..d79baac06b097
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/ParsedAggregation.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations;
+
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParser.Token;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+
+/**
+ * An implementation of {@link Aggregation} that is parsed from a REST response.
+ * Serves as a base class for all aggregation implementations that are parsed from REST.
+ */
+public abstract class ParsedAggregation implements Aggregation, ToXContent {
+
+    protected static void declareAggregationFields(ObjectParser<? extends ParsedAggregation, Void> objectParser) {
+        objectParser.declareObject((parsedAgg, metadata) -> parsedAgg.metadata = Collections.unmodifiableMap(metadata),
+                (parser, context) -> parser.map(), InternalAggregation.CommonFields.META);
+    }
+
+    private String name;
+    protected Map<String, Object> metadata;
+
+    @Override
+    public final String getName() {
+        return name;
+    }
+
+    protected void setName(String name) {
+        this.name = name;
+    }
+
+    @Override
+    public final Map<String, Object> getMetaData() {
+        return metadata;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
+        // Concatenates the type and the name of the aggregation (ex: top_hits#foo)
+        builder.startObject(String.join(InternalAggregation.TYPED_KEYS_DELIMITER, getType(), name));
+        if (this.metadata != null) {
+            builder.field(InternalAggregation.CommonFields.META.getPreferredName());
+            builder.map(this.metadata);
+        }
+        doXContentBody(builder, params);
+        builder.endObject();
+        return builder;
+    }
+
+    protected abstract XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException;
+
+    /**
+     * Parse a token of type XContentParser.Token.VALUE_NUMBER or XContentParser.Token.STRING to a double.
+     * In other cases the default value is returned instead.
+     */
+    protected static double parseDouble(XContentParser parser, double defaultNullValue) throws IOException {
+        Token currentToken = parser.currentToken();
+        if (currentToken == XContentParser.Token.VALUE_NUMBER || currentToken == XContentParser.Token.VALUE_STRING) {
+            return parser.doubleValue();
+        } else {
+            return defaultNullValue;
+        }
+    }
+}
\ No newline at end of file
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/ParsedMultiBucketAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/ParsedMultiBucketAggregation.java
new file mode 100644
index 0000000000000..1e601cb30fe75
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/ParsedMultiBucketAggregation.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations;
+
+import org.elasticsearch.common.CheckedBiConsumer;
+import org.elasticsearch.common.CheckedFunction;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParserUtils;
+import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Supplier;
+
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+
+public abstract class ParsedMultiBucketAggregation<B extends ParsedMultiBucketAggregation.Bucket>
+        extends ParsedAggregation implements MultiBucketsAggregation {
+
+    protected final List<B> buckets = new ArrayList<>();
+    protected boolean keyed = false;
+
+    @Override
+    protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+        if (keyed) {
+            builder.startObject(CommonFields.BUCKETS.getPreferredName());
+        } else {
+            builder.startArray(CommonFields.BUCKETS.getPreferredName());
+        }
+        for (B bucket : buckets) {
+            bucket.toXContent(builder, params);
+        }
+        if (keyed) {
+            builder.endObject();
+        } else {
+            builder.endArray();
+        }
+        return builder;
+    }
+
+    protected static void declareMultiBucketAggregationFields(final ObjectParser<? extends ParsedMultiBucketAggregation, Void> objectParser,
+                                                  final CheckedFunction<XContentParser, ParsedBucket, IOException> bucketParser,
+                                                  final CheckedFunction<XContentParser, ParsedBucket, IOException> keyedBucketParser) {
+        declareAggregationFields(objectParser);
+        objectParser.declareField((parser, aggregation, context) -> {
+            XContentParser.Token token = parser.currentToken();
+            if (token == XContentParser.Token.START_OBJECT) {
+                aggregation.keyed = true;
+                while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
+                    aggregation.buckets.add(keyedBucketParser.apply(parser));
+                }
+            } else if (token == XContentParser.Token.START_ARRAY) {
+                aggregation.keyed = false;
+                while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
+                    aggregation.buckets.add(bucketParser.apply(parser));
+                }
+            }
+        }, CommonFields.BUCKETS, ObjectParser.ValueType.OBJECT_ARRAY);
+    }
+
+    public abstract static class ParsedBucket implements MultiBucketsAggregation.Bucket {
+
+        private Aggregations aggregations;
+        private String keyAsString;
+        private long docCount;
+        private boolean keyed;
+
+        protected void setKeyAsString(String keyAsString) {
+            this.keyAsString = keyAsString;
+        }
+
+        @Override
+        public String getKeyAsString() {
+            return keyAsString;
+        }
+
+        protected void setDocCount(long docCount) {
+            this.docCount = docCount;
+        }
+
+        @Override
+        public long getDocCount() {
+            return docCount;
+        }
+
+        public void setKeyed(boolean keyed) {
+            this.keyed = keyed;
+        }
+
+        protected boolean isKeyed() {
+            return keyed;
+        }
+
+        protected void setAggregations(Aggregations aggregations) {
+            this.aggregations = aggregations;
+        }
+
+        @Override
+        public Aggregations getAggregations() {
+            return aggregations;
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            if (keyed) {
+                // Subclasses can override the getKeyAsString method to handle specific cases like
+                // keyed bucket with RAW doc value format where the key_as_string field is not printed
+                // out but we still need to have a string version of the key to use as the bucket's name.
+                builder.startObject(getKeyAsString());
+            } else {
+                builder.startObject();
+            }
+            if (keyAsString != null) {
+                builder.field(CommonFields.KEY_AS_STRING.getPreferredName(), getKeyAsString());
+            }
+            keyToXContent(builder);
+            builder.field(CommonFields.DOC_COUNT.getPreferredName(), docCount);
+            aggregations.toXContentInternal(builder, params);
+            builder.endObject();
+            return builder;
+        }
+
+        protected XContentBuilder keyToXContent(XContentBuilder builder) throws IOException {
+            return builder.field(CommonFields.KEY.getPreferredName(), getKey());
+        }
+
+        protected static <B extends ParsedBucket> B parseXContent(final XContentParser parser,
+                                                                  final boolean keyed,
+                                                                  final Supplier<B> bucketSupplier,
+                                                                  final CheckedBiConsumer<XContentParser, B, IOException> keyConsumer)
+                                                                        throws IOException {
+            final B bucket = bucketSupplier.get();
+            bucket.setKeyed(keyed);
+            XContentParser.Token token = parser.currentToken();
+            String currentFieldName = parser.currentName();
+            if (keyed) {
+                ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser::getTokenLocation);
+                ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
+            }
+
+            List<Aggregation> aggregations = new ArrayList<>();
+            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                if (token == XContentParser.Token.FIELD_NAME) {
+                    currentFieldName = parser.currentName();
+                } else if (token.isValue()) {
+                    if (CommonFields.KEY_AS_STRING.getPreferredName().equals(currentFieldName)) {
+                        bucket.setKeyAsString(parser.text());
+                    } else if (CommonFields.KEY.getPreferredName().equals(currentFieldName)) {
+                        keyConsumer.accept(parser, bucket);
+                    } else if (CommonFields.DOC_COUNT.getPreferredName().equals(currentFieldName)) {
+                        bucket.setDocCount(parser.longValue());
+                    }
+                } else if (token == XContentParser.Token.START_OBJECT) {
+                    XContentParserUtils.parseTypedKeysObject(parser, Aggregation.TYPED_KEYS_DELIMITER, Aggregation.class,
+                            aggregations::add);
+                }
+            }
+            bucket.setAggregations(new Aggregations(aggregations));
+            return bucket;
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/PipelineAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/PipelineAggregationBuilder.java
index f946232547152..8f965d1d87eb4 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/PipelineAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/PipelineAggregationBuilder.java
@@ -20,7 +20,7 @@
 
 import org.elasticsearch.action.support.ToXContentToBytes;
 import org.elasticsearch.common.io.stream.NamedWriteable;
-import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 
 import java.io.IOException;
@@ -32,7 +32,7 @@
  * specific type.
  */
 public abstract class PipelineAggregationBuilder extends ToXContentToBytes
-        implements NamedWriteable {
+        implements NamedWriteable, BaseAggregationBuilder {
 
     protected final String name;
     protected final String[] bucketsPaths;
@@ -79,6 +79,11 @@ protected abstract void validate(AggregatorFactory<?> parent, AggregatorFactory<
     protected abstract PipelineAggregator create() throws IOException;
 
     /** Associate metadata with this {@link PipelineAggregationBuilder}. */
+    @Override
     public abstract PipelineAggregationBuilder setMetaData(Map<String, Object> metaData);
 
+    @Override
+    public PipelineAggregationBuilder subAggregations(Builder subFactories) {
+        throw new IllegalArgumentException("Aggregation [" + name + "] cannot define sub-aggregations");
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/SearchContextAggregations.java b/core/src/main/java/org/elasticsearch/search/aggregations/SearchContextAggregations.java
index f003519350f87..9476af03846bf 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/SearchContextAggregations.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/SearchContextAggregations.java
@@ -18,8 +18,6 @@
  */
 package org.elasticsearch.search.aggregations;
 
-import org.elasticsearch.search.aggregations.support.AggregationContext;
-
 /**
  * The aggregation context that is part of the search context.
  */
@@ -27,7 +25,6 @@ public class SearchContextAggregations {
 
     private final AggregatorFactories factories;
     private Aggregator[] aggregators;
-    private AggregationContext aggregationContext;
 
     /**
      * Creates a new aggregation context with the parsed aggregator factories
@@ -44,14 +41,6 @@ public Aggregator[] aggregators() {
         return aggregators;
     }
 
-    public AggregationContext aggregationContext() {
-        return aggregationContext;
-    }
-
-    public void aggregationContext(AggregationContext aggregationContext) {
-        this.aggregationContext = aggregationContext;
-    }
-
     /**
      * Registers all the created aggregators (top level aggregators) for the search execution context.
      *
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BestBucketsDeferringCollector.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BestBucketsDeferringCollector.java
index 7c6ebae7403d3..61d85b80ad9e7 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BestBucketsDeferringCollector.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BestBucketsDeferringCollector.java
@@ -31,7 +31,7 @@
 import org.elasticsearch.search.aggregations.BucketCollector;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.LeafBucketCollector;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -49,7 +49,7 @@ private static class Entry {
         final PackedLongValues docDeltas;
         final PackedLongValues buckets;
 
-        public Entry(LeafReaderContext context, PackedLongValues docDeltas, PackedLongValues buckets) {
+        Entry(LeafReaderContext context, PackedLongValues docDeltas, PackedLongValues buckets) {
             this.context = context;
             this.docDeltas = docDeltas;
             this.buckets = buckets;
@@ -58,7 +58,7 @@ public Entry(LeafReaderContext context, PackedLongValues docDeltas, PackedLongVa
 
     final List<Entry> entries = new ArrayList<>();
     BucketCollector collector;
-    final AggregationContext aggContext;
+    final SearchContext searchContext;
     LeafReaderContext context;
     PackedLongValues.Builder docDeltas;
     PackedLongValues.Builder buckets;
@@ -67,8 +67,8 @@ public Entry(LeafReaderContext context, PackedLongValues docDeltas, PackedLongVa
     LongHash selectedBuckets;
 
     /** Sole constructor. */
-    public BestBucketsDeferringCollector(AggregationContext context) {
-        this.aggContext = context;
+    public BestBucketsDeferringCollector(SearchContext context) {
+        this.searchContext = context;
     }
 
     @Override
@@ -147,8 +147,8 @@ public void prepareSelectedBuckets(long... selectedBuckets) throws IOException {
         boolean needsScores = collector.needsScores();
         Weight weight = null;
         if (needsScores) {
-            weight = aggContext.searchContext().searcher()
-                        .createNormalizedWeight(aggContext.searchContext().query(), true);
+            weight = searchContext.searcher()
+                        .createNormalizedWeight(searchContext.query(), true);
         }
         for (Entry entry : entries) {
             final LeafBucketCollector leafCollector = collector.getLeafCollector(entry.context);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BestDocsDeferringCollector.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BestDocsDeferringCollector.java
index 90316c1a001d1..8d699b7c43852 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BestDocsDeferringCollector.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BestDocsDeferringCollector.java
@@ -163,7 +163,7 @@ class PerParentBucketSamples {
         private long parentBucket;
         private int matchedDocs;
 
-        public PerParentBucketSamples(long parentBucket, Scorer scorer, LeafReaderContext readerContext) {
+        PerParentBucketSamples(long parentBucket, Scorer scorer, LeafReaderContext readerContext) {
             try {
                 this.parentBucket = parentBucket;
                 tdc = createTopDocsCollector(shardSize);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketsAggregator.java
index ab655497c4c42..665bf9b79670c 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketsAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketsAggregator.java
@@ -28,7 +28,7 @@
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.aggregations.LeafBucketCollector;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -43,7 +43,7 @@ public abstract class BucketsAggregator extends AggregatorBase {
     private final BigArrays bigArrays;
     private IntArray docCounts;
 
-    public BucketsAggregator(String name, AggregatorFactories factories, AggregationContext context, Aggregator parent,
+    public BucketsAggregator(String name, AggregatorFactories factories, SearchContext context, Aggregator parent,
             List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException {
         super(name, factories, context, parent, pipelineAggregators, metaData);
         bigArrays = context.bigArrays();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/DeferringBucketCollector.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/DeferringBucketCollector.java
index 7ecf7672bfd77..3c63df2c06a76 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/DeferringBucketCollector.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/DeferringBucketCollector.java
@@ -24,7 +24,7 @@
 import org.elasticsearch.search.aggregations.BucketCollector;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.LeafBucketCollector;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 
@@ -82,7 +82,7 @@ public Aggregator parent() {
         }
 
         @Override
-        public AggregationContext context() {
+        public SearchContext context() {
             return in.context();
         }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/InternalSingleBucketAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/InternalSingleBucketAggregation.java
index e8b04680064e3..74eee9d6973a6 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/InternalSingleBucketAggregation.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/InternalSingleBucketAggregation.java
@@ -129,7 +129,7 @@ public Object getProperty(List<String> path) {
 
     @Override
     public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
-        builder.field(CommonFields.DOC_COUNT, docCount);
+        builder.field(CommonFields.DOC_COUNT.getPreferredName(), docCount);
         aggregations.toXContentInternal(builder, params);
         return builder;
     }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/MultiBucketsAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/MultiBucketsAggregation.java
index 2d8d26dd35c15..fc223916f72c1 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/MultiBucketsAggregation.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/MultiBucketsAggregation.java
@@ -19,9 +19,6 @@
 
 package org.elasticsearch.search.aggregations.bucket;
 
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.Streamable;
-import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.util.Comparators;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.search.aggregations.Aggregation;
@@ -29,7 +26,6 @@
 import org.elasticsearch.search.aggregations.HasAggregations;
 import org.elasticsearch.search.aggregations.support.AggregationPath;
 
-import java.io.IOException;
 import java.util.List;
 
 /**
@@ -40,7 +36,7 @@ public interface MultiBucketsAggregation extends Aggregation {
      * A bucket represents a criteria to which all documents that fall in it adhere to. It is also uniquely identified
      * by a key, and can potentially hold sub-aggregations computed over all documents in it.
      */
-    public interface Bucket extends HasAggregations, ToXContent, Writeable {
+    interface Bucket extends HasAggregations, ToXContent {
         /**
          * @return The key associated with the bucket
          */
@@ -62,9 +58,7 @@ public interface Bucket extends HasAggregations, ToXContent, Writeable {
         @Override
         Aggregations getAggregations();
 
-        Object getProperty(String containingAggName, List<String> path);
-
-        static class SubAggregationComparator<B extends Bucket> implements java.util.Comparator<B> {
+        class SubAggregationComparator<B extends Bucket> implements java.util.Comparator<B> {
 
             private final AggregationPath path;
             private final boolean asc;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/ParsedSingleBucketAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/ParsedSingleBucketAggregation.java
new file mode 100644
index 0000000000000..d6f6b2fe3ed0e
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/ParsedSingleBucketAggregation.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.bucket;
+
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParserUtils;
+import org.elasticsearch.search.aggregations.Aggregation;
+import org.elasticsearch.search.aggregations.Aggregations;
+import org.elasticsearch.search.aggregations.ParsedAggregation;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+
+/**
+ * A base class for all the single bucket aggregations.
+ */
+public abstract class ParsedSingleBucketAggregation extends ParsedAggregation implements SingleBucketAggregation {
+
+    private long docCount;
+    protected Aggregations aggregations = new Aggregations(Collections.emptyList());
+
+    @Override
+    public long getDocCount() {
+        return docCount;
+    }
+
+    protected void setDocCount(long docCount) {
+        this.docCount = docCount;
+    }
+
+    @Override
+    public Aggregations getAggregations() {
+        return aggregations;
+    }
+
+    @Override
+    public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+        builder.field(CommonFields.DOC_COUNT.getPreferredName(), docCount);
+        aggregations.toXContentInternal(builder, params);
+        return builder;
+    }
+
+    protected static <T extends ParsedSingleBucketAggregation> T parseXContent(final XContentParser parser, T aggregation, String name)
+            throws IOException {
+        aggregation.setName(name);
+        XContentParser.Token token = parser.currentToken();
+        String currentFieldName = parser.currentName();
+        if (token == XContentParser.Token.FIELD_NAME) {
+            token = parser.nextToken();
+        }
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser::getTokenLocation);
+
+        List<Aggregation> aggregations = new ArrayList<>();
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token.isValue()) {
+                if (CommonFields.DOC_COUNT.getPreferredName().equals(currentFieldName)) {
+                    aggregation.setDocCount(parser.longValue());
+                }
+            } else if (token == XContentParser.Token.START_OBJECT) {
+                if (CommonFields.META.getPreferredName().equals(currentFieldName)) {
+                    aggregation.metadata = parser.map();
+                } else {
+                    XContentParserUtils.parseTypedKeysObject(parser, Aggregation.TYPED_KEYS_DELIMITER, Aggregation.class,
+                            aggregations::add);
+                }
+            }
+        }
+        aggregation.aggregations = new Aggregations(aggregations);
+        return aggregation;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/SingleBucketAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/SingleBucketAggregator.java
index 44b1eea914662..f74df5d63b167 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/SingleBucketAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/SingleBucketAggregator.java
@@ -21,7 +21,7 @@
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -33,7 +33,7 @@
 public abstract class SingleBucketAggregator extends BucketsAggregator {
 
     protected SingleBucketAggregator(String name, AggregatorFactories factories,
-            AggregationContext aggregationContext, Aggregator parent,
+            SearchContext aggregationContext, Aggregator parent,
             List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException {
         super(name, factories, aggregationContext, parent, pipelineAggregators, metaData);
     }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrix.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrix.java
new file mode 100644
index 0000000000000..365fbb5547b33
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrix.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.adjacency;
+
+import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
+
+import java.util.List;
+
+/**
+ * A multi bucket aggregation where the buckets are defined by a set of filters
+ * (a bucket is produced per filter plus a bucket for each non-empty filter 
+ * intersection so A, B and A&amp;B).
+ */
+public interface AdjacencyMatrix extends MultiBucketsAggregation {
+
+    /**
+     * A bucket associated with a specific filter or pair (identified by its
+     * key)
+     */
+    interface Bucket extends MultiBucketsAggregation.Bucket {
+    }
+
+    /**
+     * The buckets created by this aggregation.
+     */
+    @Override
+    List<? extends Bucket> getBuckets();
+
+    Bucket getBucketByKey(String key);
+
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregationBuilder.java
new file mode 100644
index 0000000000000..eeb60d393e5c8
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregationBuilder.java
@@ -0,0 +1,237 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.adjacency;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.Aggregator;
+import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
+import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.bucket.adjacency.AdjacencyMatrixAggregator.KeyedFilter;
+import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.search.query.QueryPhaseExecutionException;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+
+public class AdjacencyMatrixAggregationBuilder extends AbstractAggregationBuilder<AdjacencyMatrixAggregationBuilder> {
+    public static final String NAME = "adjacency_matrix";
+
+    private static final String DEFAULT_SEPARATOR = "&";
+
+    private static final ParseField SEPARATOR_FIELD = new ParseField("separator");
+    private static final ParseField FILTERS_FIELD = new ParseField("filters");
+    private List<KeyedFilter> filters;
+    private String separator = DEFAULT_SEPARATOR;
+
+    public static Aggregator.Parser getParser() {
+        ObjectParser<AdjacencyMatrixAggregationBuilder, QueryParseContext> parser = new ObjectParser<>(
+                AdjacencyMatrixAggregationBuilder.NAME);
+        parser.declareString(AdjacencyMatrixAggregationBuilder::separator, SEPARATOR_FIELD);
+        parser.declareNamedObjects(AdjacencyMatrixAggregationBuilder::setFiltersAsList, KeyedFilter.PARSER, FILTERS_FIELD);
+        return new Aggregator.Parser() {
+            @Override
+            public AggregationBuilder parse(String aggregationName, QueryParseContext context) throws IOException {
+                AdjacencyMatrixAggregationBuilder result = parser.parse(context.parser(),
+                        new AdjacencyMatrixAggregationBuilder(aggregationName), context);
+                result.checkConsistency();
+                return result;
+            }
+        };
+    }
+
+    protected void checkConsistency() {
+        if ((filters == null) || (filters.size() == 0)) {
+            throw new IllegalStateException("[" + name  + "] is missing : " + FILTERS_FIELD.getPreferredName() + " parameter");
+        }        
+    }
+
+
+    protected void setFiltersAsMap(Map<String, QueryBuilder> filters) {
+        // Convert uniquely named objects into internal KeyedFilters
+        this.filters = new ArrayList<>(filters.size());
+        for (Entry<String, QueryBuilder> kv : filters.entrySet()) {
+            this.filters.add(new KeyedFilter(kv.getKey(), kv.getValue()));
+        }
+        // internally we want to have a fixed order of filters, regardless of
+        // the order of the filters in the request
+        Collections.sort(this.filters, Comparator.comparing(KeyedFilter::key));
+    }
+
+    protected void setFiltersAsList(List<KeyedFilter> filters) {
+        this.filters = new ArrayList<>(filters);
+        // internally we want to have a fixed order of filters, regardless of
+        // the order of the filters in the request
+        Collections.sort(this.filters, Comparator.comparing(KeyedFilter::key));
+    }
+    
+   
+    /**
+     * @param name
+     *            the name of this aggregation
+     */
+    protected AdjacencyMatrixAggregationBuilder(String name) {
+        super(name);
+    }    
+    
+    
+    /**
+     * @param name
+     *            the name of this aggregation
+     * @param filters
+     *            the filters and their keys to use with this aggregation.
+     */
+    public AdjacencyMatrixAggregationBuilder(String name, Map<String, QueryBuilder> filters) {
+        this(name, DEFAULT_SEPARATOR, filters);
+    }
+
+    /**
+     * @param name
+     *            the name of this aggregation
+     * @param separator
+     *            the string used to separate keys in intersections buckets e.g.
+     *            &amp; character for keyed filters A and B would return an
+     *            intersection bucket named A&amp;B
+     * @param filters
+     *            the filters and their key to use with this aggregation.
+     */
+    public AdjacencyMatrixAggregationBuilder(String name, String separator, Map<String, QueryBuilder> filters) {
+        super(name);
+        this.separator = separator;
+        setFiltersAsMap(filters);
+    }
+
+    /**
+     * Read from a stream.
+     */
+    public AdjacencyMatrixAggregationBuilder(StreamInput in) throws IOException {
+        super(in);
+        int filtersSize = in.readVInt();
+        separator = in.readString();
+        filters = new ArrayList<>(filtersSize);
+        for (int i = 0; i < filtersSize; i++) {
+            filters.add(new KeyedFilter(in));
+        }
+    }
+
+    @Override
+    protected void doWriteTo(StreamOutput out) throws IOException {
+        out.writeVInt(filters.size());
+        out.writeString(separator);
+        for (KeyedFilter keyedFilter : filters) {
+            keyedFilter.writeTo(out);
+        }
+    }
+
+    /**
+     * Set the separator used to join pairs of bucket keys
+     */
+    public AdjacencyMatrixAggregationBuilder separator(String separator) {
+        if (separator == null) {
+            throw new IllegalArgumentException("[separator] must not be null: [" + name + "]");
+        }
+        this.separator = separator;
+        return this;
+    }
+
+    /**
+     * Get the separator used to join pairs of bucket keys
+     */
+    public String separator() {
+        return separator;
+    }        
+    
+    /**
+     * Get the filters. This will be an unmodifiable map
+     */
+    public Map<String, QueryBuilder> filters() {
+        Map<String, QueryBuilder>result = new HashMap<>(this.filters.size());
+        for (KeyedFilter keyedFilter : this.filters) {
+            result.put(keyedFilter.key(), keyedFilter.filter());
+        }
+        return result;
+    }    
+    
+
+    @Override
+    protected AggregatorFactory<?> doBuild(SearchContext context, AggregatorFactory<?> parent, Builder subFactoriesBuilder)
+            throws IOException {
+        int maxFilters = context.indexShard().indexSettings().getMaxAdjacencyMatrixFilters();
+        if (filters.size() > maxFilters){
+            throw new QueryPhaseExecutionException(context,
+                    "Number of filters is too large, must be less than or equal to: [" + maxFilters + "] but was ["
+                            + filters.size() + "]."
+                            + "This limit can be set by changing the [" + IndexSettings.MAX_ADJACENCY_MATRIX_FILTERS_SETTING.getKey()
+                            + "] index level setting.");
+        }
+
+        List<KeyedFilter> rewrittenFilters = new ArrayList<>(filters.size());
+        for (KeyedFilter kf : filters) {
+            rewrittenFilters.add(new KeyedFilter(kf.key(), QueryBuilder.rewriteQuery(kf.filter(), context.getQueryShardContext())));
+        }
+
+        return new AdjacencyMatrixAggregatorFactory(name, rewrittenFilters, separator, context, parent,
+                subFactoriesBuilder, metaData);
+    }
+
+    @Override
+    protected XContentBuilder internalXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(SEPARATOR_FIELD.getPreferredName(), separator);
+        builder.startObject(AdjacencyMatrixAggregator.FILTERS_FIELD.getPreferredName());
+        for (KeyedFilter keyedFilter : filters) {
+            builder.field(keyedFilter.key(), keyedFilter.filter());
+        }
+        builder.endObject();
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    protected int doHashCode() {
+        return Objects.hash(filters, separator);
+    }
+
+    @Override
+    protected boolean doEquals(Object obj) {
+        AdjacencyMatrixAggregationBuilder other = (AdjacencyMatrixAggregationBuilder) obj;
+        return Objects.equals(filters, other.filters) && Objects.equals(separator, other.separator);
+    }
+
+    @Override
+    public String getType() {
+        return NAME;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregator.java
new file mode 100644
index 0000000000000..7c8e54bbfca95
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregator.java
@@ -0,0 +1,243 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.adjacency;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.Weight;
+import org.apache.lucene.util.Bits;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.lucene.Lucene;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.ObjectParser.NamedObjectParser;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.search.aggregations.Aggregator;
+import org.elasticsearch.search.aggregations.AggregatorFactories;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.LeafBucketCollector;
+import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
+import org.elasticsearch.search.aggregations.bucket.BucketsAggregator;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.search.internal.SearchContext;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Aggregation for adjacency matrices.
+ * 
+ * NOTE! This is an experimental class.
+ * 
+ * TODO the aggregation produces a sparse response but in the
+ * computation it uses a non-sparse structure (an array of Bits
+ * objects). This could be changed to a sparse structure in future.
+ * 
+ */
+public class AdjacencyMatrixAggregator extends BucketsAggregator {
+
+    public static final ParseField FILTERS_FIELD = new ParseField("filters");
+
+    protected static class KeyedFilter implements Writeable, ToXContent {
+        private final String key;
+        private final QueryBuilder filter;
+        
+        public static final NamedObjectParser<KeyedFilter, QueryParseContext> PARSER = 
+                (XContentParser p, QueryParseContext c, String name) -> 
+                     new KeyedFilter(name, c.parseInnerQueryBuilder().get());
+        
+
+        public KeyedFilter(String key, QueryBuilder filter) {
+            if (key == null) {
+                throw new IllegalArgumentException("[key] must not be null");
+            }
+            if (filter == null) {
+                throw new IllegalArgumentException("[filter] must not be null");
+            }
+            this.key = key;
+            this.filter = filter;
+        }
+
+        /**
+         * Read from a stream.
+         */
+        public KeyedFilter(StreamInput in) throws IOException {
+            key = in.readString();
+            filter = in.readNamedWriteable(QueryBuilder.class);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeString(key);
+            out.writeNamedWriteable(filter);
+        }
+
+        public String key() {
+            return key;
+        }
+
+        public QueryBuilder filter() {
+            return filter;
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.field(key, filter);
+            return builder;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(key, filter);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null) {
+                return false;
+            }
+            if (getClass() != obj.getClass()) {
+                return false;
+            }
+            KeyedFilter other = (KeyedFilter) obj;
+            return Objects.equals(key, other.key) && Objects.equals(filter, other.filter);
+        }
+    }
+
+    private final String[] keys;
+    private Weight[] filters;
+    private final int totalNumKeys;
+    private final int totalNumIntersections;
+    private final String separator;
+
+    public AdjacencyMatrixAggregator(String name, AggregatorFactories factories, String separator, String[] keys, 
+            Weight[] filters, SearchContext context, Aggregator parent, List<PipelineAggregator> pipelineAggregators, 
+            Map<String, Object> metaData) throws IOException {
+        super(name, factories, context, parent, pipelineAggregators, metaData);
+        this.separator = separator;
+        this.keys = keys;
+        this.filters = filters;
+        this.totalNumIntersections = ((keys.length * keys.length) - keys.length) / 2;
+        this.totalNumKeys = keys.length + totalNumIntersections;
+    }
+
+    private static class BitsIntersector implements Bits {
+        Bits a;
+        Bits b;
+
+        BitsIntersector(Bits a, Bits b) {
+            super();
+            this.a = a;
+            this.b = b;
+        }
+
+        @Override
+        public boolean get(int index) {
+            return a.get(index) && b.get(index);
+        }
+
+        @Override
+        public int length() {
+            return Math.min(a.length(), b.length());
+        }
+
+    }
+
+    @Override
+    public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, final LeafBucketCollector sub) throws IOException {
+        // no need to provide deleted docs to the filter
+        final Bits[] bits = new Bits[filters.length + totalNumIntersections];
+        for (int i = 0; i < filters.length; ++i) {
+            bits[i] = Lucene.asSequentialAccessBits(ctx.reader().maxDoc(), filters[i].scorerSupplier(ctx));
+        }
+        // Add extra Bits for intersections
+        int pos = filters.length;
+        for (int i = 0; i < filters.length; i++) {
+            for (int j = i + 1; j < filters.length; j++) {
+                bits[pos++] = new BitsIntersector(bits[i], bits[j]);
+            }
+        }
+        assert pos == bits.length;
+        return new LeafBucketCollectorBase(sub, null) {
+            @Override
+            public void collect(int doc, long bucket) throws IOException {
+                for (int i = 0; i < bits.length; i++) {
+                    if (bits[i].get(doc)) {
+                        collectBucket(sub, doc, bucketOrd(bucket, i));
+                    }
+                }
+            }
+        };
+    }
+
+    @Override
+    public InternalAggregation buildAggregation(long owningBucketOrdinal) throws IOException {
+
+        // Buckets are ordered into groups - [keyed filters] [key1&key2 intersects]
+
+        List<InternalAdjacencyMatrix.InternalBucket> buckets = new ArrayList<>(filters.length);
+        for (int i = 0; i < keys.length; i++) {
+            long bucketOrd = bucketOrd(owningBucketOrdinal, i);
+            int docCount = bucketDocCount(bucketOrd);
+            // Empty buckets are not returned because this aggregation will commonly be used under a
+            // a date-histogram where we will look for transactions over time and can expect many
+            // empty buckets.
+            if (docCount > 0) {
+                InternalAdjacencyMatrix.InternalBucket bucket = new InternalAdjacencyMatrix.InternalBucket(keys[i], 
+                        docCount, bucketAggregations(bucketOrd));
+                buckets.add(bucket);                
+            }
+        }
+        int pos = keys.length;
+        for (int i = 0; i < keys.length; i++) {
+            for (int j = i + 1; j < keys.length; j++) {
+                long bucketOrd = bucketOrd(owningBucketOrdinal, pos);
+                int docCount = bucketDocCount(bucketOrd);
+                // Empty buckets are not returned due to potential for very sparse matrices
+                if (docCount > 0) {
+                    String intersectKey = keys[i] + separator + keys[j];
+                    InternalAdjacencyMatrix.InternalBucket bucket = new InternalAdjacencyMatrix.InternalBucket(intersectKey, 
+                            docCount, bucketAggregations(bucketOrd));
+                    buckets.add(bucket);
+                }
+                pos++;
+            }
+        }
+        return new InternalAdjacencyMatrix(name, buckets, pipelineAggregators(), metaData());
+    }
+
+    @Override
+    public InternalAggregation buildEmptyAggregation() {
+        List<InternalAdjacencyMatrix.InternalBucket> buckets = new ArrayList<>(0);
+        return new InternalAdjacencyMatrix(name, buckets, pipelineAggregators(), metaData());
+    }
+
+    final long bucketOrd(long owningBucketOrdinal, int filterOrd) {
+        return owningBucketOrdinal * totalNumKeys + filterOrd;
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregatorFactory.java
new file mode 100644
index 0000000000000..6df88379d4eb0
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregatorFactory.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.adjacency;
+
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Weight;
+import org.elasticsearch.search.aggregations.Aggregator;
+import org.elasticsearch.search.aggregations.AggregatorFactories;
+import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.bucket.adjacency.AdjacencyMatrixAggregator.KeyedFilter;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.search.internal.SearchContext;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+public class AdjacencyMatrixAggregatorFactory extends AggregatorFactory<AdjacencyMatrixAggregatorFactory> {
+
+    private final String[] keys;
+    private final Weight[] weights;
+    private final String separator;
+
+    public AdjacencyMatrixAggregatorFactory(String name, List<KeyedFilter> filters, String separator, 
+            SearchContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactories, 
+            Map<String, Object> metaData) throws IOException {
+        super(name, context, parent, subFactories, metaData);
+        IndexSearcher contextSearcher = context.searcher();
+        this.separator = separator;
+        weights = new Weight[filters.size()];
+        keys = new String[filters.size()];
+        for (int i = 0; i < filters.size(); ++i) {
+            KeyedFilter keyedFilter = filters.get(i);
+            this.keys[i] = keyedFilter.key();
+            Query filter = keyedFilter.filter().toFilter(context.getQueryShardContext());
+            this.weights[i] = contextSearcher.createNormalizedWeight(filter, false);
+        }
+    }
+
+    @Override
+    public Aggregator createInternal(Aggregator parent, boolean collectsFromSingleBucket, List<PipelineAggregator> pipelineAggregators,
+            Map<String, Object> metaData) throws IOException {
+        return new AdjacencyMatrixAggregator(name, factories, separator, keys, weights, context, parent,
+                pipelineAggregators, metaData);
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/InternalAdjacencyMatrix.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/InternalAdjacencyMatrix.java
new file mode 100644
index 0000000000000..3d0839b7fb477
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/InternalAdjacencyMatrix.java
@@ -0,0 +1,218 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.adjacency;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.search.aggregations.Aggregations;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class InternalAdjacencyMatrix 
+    extends InternalMultiBucketAggregation<InternalAdjacencyMatrix,InternalAdjacencyMatrix.InternalBucket> 
+    implements AdjacencyMatrix {
+    public static class InternalBucket extends InternalMultiBucketAggregation.InternalBucket implements AdjacencyMatrix.Bucket {
+
+        private final String key;
+        private long docCount;
+        InternalAggregations aggregations;
+
+        public InternalBucket(String key, long docCount, InternalAggregations aggregations) {
+            this.key = key;
+            this.docCount = docCount;
+            this.aggregations = aggregations;
+        }
+
+        /**
+         * Read from a stream.
+         */
+        public InternalBucket(StreamInput in) throws IOException {
+            key = in.readOptionalString();
+            docCount = in.readVLong();
+            aggregations = InternalAggregations.readAggregations(in);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeOptionalString(key);
+            out.writeVLong(docCount);
+            aggregations.writeTo(out);
+        }
+
+        @Override
+        public String getKey() {
+            return key;
+        }
+
+        @Override
+        public String getKeyAsString() {
+            return key;
+        }
+
+        @Override
+        public long getDocCount() {
+            return docCount;
+        }
+
+        @Override
+        public Aggregations getAggregations() {
+            return aggregations;
+        }
+
+        InternalBucket reduce(List<InternalBucket> buckets, ReduceContext context) {
+            InternalBucket reduced = null;
+            List<InternalAggregations> aggregationsList = new ArrayList<>(buckets.size());
+            for (InternalBucket bucket : buckets) {
+                if (reduced == null) {
+                    reduced = new InternalBucket(bucket.key, bucket.docCount, bucket.aggregations);
+                } else {
+                    reduced.docCount += bucket.docCount;
+                }
+                aggregationsList.add(bucket.aggregations);
+            }
+            reduced.aggregations = InternalAggregations.reduce(aggregationsList, context);
+            return reduced;
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            builder.field(CommonFields.KEY.getPreferredName(), key);
+            builder.field(CommonFields.DOC_COUNT.getPreferredName(), docCount);
+            aggregations.toXContentInternal(builder, params);
+            builder.endObject();
+            return builder;
+        }
+    }
+
+    private final List<InternalBucket> buckets;
+    private Map<String, InternalBucket> bucketMap;
+
+    public InternalAdjacencyMatrix(String name, List<InternalBucket> buckets, 
+            List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
+        super(name, pipelineAggregators, metaData);
+        this.buckets = buckets;
+    }
+
+    /**
+     * Read from a stream.
+     */
+    public InternalAdjacencyMatrix(StreamInput in) throws IOException {
+        super(in);
+        int size = in.readVInt();
+        List<InternalBucket> buckets = new ArrayList<>(size);
+        for (int i = 0; i < size; i++) {
+            buckets.add(new InternalBucket(in));
+        }
+        this.buckets = buckets;
+        this.bucketMap = null;
+    }
+
+    @Override
+    protected void doWriteTo(StreamOutput out) throws IOException {
+        out.writeVInt(buckets.size());
+        for (InternalBucket bucket : buckets) {
+            bucket.writeTo(out);
+        }
+    }
+
+    @Override
+    public String getWriteableName() {
+        return AdjacencyMatrixAggregationBuilder.NAME;
+    }
+
+    @Override
+    public InternalAdjacencyMatrix create(List<InternalBucket> buckets) {
+        return new InternalAdjacencyMatrix(this.name, buckets, this.pipelineAggregators(), this.metaData);
+    }
+
+    @Override
+    public InternalBucket createBucket(InternalAggregations aggregations, InternalBucket prototype) {
+        return new InternalBucket(prototype.key, prototype.docCount, aggregations);
+    }
+
+    @Override
+    public List<InternalBucket> getBuckets() {
+        return buckets;
+    }
+
+    @Override
+    public InternalBucket getBucketByKey(String key) {
+        if (bucketMap == null) {
+            bucketMap = new HashMap<>(buckets.size());
+            for (InternalBucket bucket : buckets) {
+                bucketMap.put(bucket.getKey(), bucket);
+            }
+        }
+        return bucketMap.get(key);
+    }
+
+    @Override
+    public InternalAggregation doReduce(List<InternalAggregation> aggregations, ReduceContext reduceContext) {
+        Map<String, List<InternalBucket>> bucketsMap = new HashMap<>();
+        for (InternalAggregation aggregation : aggregations) {
+            InternalAdjacencyMatrix filters = (InternalAdjacencyMatrix) aggregation;
+            for (InternalBucket bucket : filters.buckets) {
+                List<InternalBucket> sameRangeList = bucketsMap.get(bucket.key);
+                if(sameRangeList == null){
+                    sameRangeList = new ArrayList<>(aggregations.size());
+                    bucketsMap.put(bucket.key, sameRangeList);
+                }
+                sameRangeList.add(bucket);
+            }
+        }
+
+        ArrayList<InternalBucket> reducedBuckets = new ArrayList<>(bucketsMap.size());
+        for (List<InternalBucket> sameRangeList : bucketsMap.values()) {
+            InternalBucket reducedBucket = sameRangeList.get(0).reduce(sameRangeList, reduceContext);
+            if(reducedBucket.docCount >= 1){                
+                reducedBuckets.add(reducedBucket);
+            }
+        }
+        Collections.sort(reducedBuckets, Comparator.comparing(InternalBucket::getKey));
+        
+        InternalAdjacencyMatrix reduced = new InternalAdjacencyMatrix(name, reducedBuckets, pipelineAggregators(), 
+                getMetaData());
+
+        return reduced;
+    }
+
+    @Override
+    public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+        builder.startArray(CommonFields.BUCKETS.getPreferredName());
+        for (InternalBucket bucket : buckets) {
+            bucket.toXContent(builder, params);
+        }
+        builder.endArray();
+        return builder;
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/ParsedAdjacencyMatrix.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/ParsedAdjacencyMatrix.java
new file mode 100644
index 0000000000000..1fb356d45c28c
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/ParsedAdjacencyMatrix.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.adjacency;
+
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class ParsedAdjacencyMatrix extends ParsedMultiBucketAggregation<ParsedAdjacencyMatrix.ParsedBucket> implements AdjacencyMatrix {
+
+    private Map<String, ParsedBucket> bucketMap;
+
+    @Override
+    public String getType() {
+        return AdjacencyMatrixAggregationBuilder.NAME;
+    }
+
+    @Override
+    public List<? extends AdjacencyMatrix.Bucket> getBuckets() {
+        return buckets;
+    }
+
+    @Override
+    public ParsedBucket getBucketByKey(String key) {
+        if (bucketMap == null) {
+            bucketMap = new HashMap<>(buckets.size());
+            for (ParsedBucket bucket : buckets) {
+                bucketMap.put(bucket.getKey(), bucket);
+            }
+        }
+        return bucketMap.get(key);
+    }
+
+    private static ObjectParser<ParsedAdjacencyMatrix, Void> PARSER =
+            new ObjectParser<>(ParsedAdjacencyMatrix.class.getSimpleName(), true, ParsedAdjacencyMatrix::new);
+    static {
+        declareMultiBucketAggregationFields(PARSER,
+                parser -> ParsedBucket.fromXContent(parser),
+                parser -> ParsedBucket.fromXContent(parser));
+    }
+
+    public static ParsedAdjacencyMatrix fromXContent(XContentParser parser, String name) throws IOException {
+        ParsedAdjacencyMatrix aggregation = PARSER.parse(parser, null);
+        aggregation.setName(name);
+        return aggregation;
+    }
+
+    public static class ParsedBucket extends ParsedMultiBucketAggregation.ParsedBucket implements AdjacencyMatrix.Bucket {
+
+        private String key;
+
+        @Override
+        public String getKey() {
+            return key;
+        }
+
+        @Override
+        public String getKeyAsString() {
+            return key;
+        }
+
+        static ParsedBucket fromXContent(XContentParser parser) throws IOException {
+            return parseXContent(parser, false, ParsedBucket::new, (p, bucket) -> bucket.key = p.text());
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/ChildrenAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/ChildrenAggregationBuilder.java
deleted file mode 100644
index a05dda207e381..0000000000000
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/ChildrenAggregationBuilder.java
+++ /dev/null
@@ -1,169 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.aggregations.bucket.children;
-
-import org.apache.lucene.search.Query;
-import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.ParentFieldMapper;
-import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
-import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
-import org.elasticsearch.search.aggregations.support.FieldContext;
-import org.elasticsearch.search.aggregations.support.ValueType;
-import org.elasticsearch.search.aggregations.support.ValuesSource.Bytes.ParentChild;
-import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
-import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
-import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
-import org.elasticsearch.search.aggregations.support.ValuesSourceType;
-
-import java.io.IOException;
-import java.util.Objects;
-
-public class ChildrenAggregationBuilder extends ValuesSourceAggregationBuilder<ParentChild, ChildrenAggregationBuilder> {
-    public static final String NAME = "children";
-    private static final Type TYPE = new Type(NAME);
-
-    private String parentType;
-    private final String childType;
-    private Query parentFilter;
-    private Query childFilter;
-
-    /**
-     * @param name
-     *            the name of this aggregation
-     * @param childType
-     *            the type of children documents
-     */
-    public ChildrenAggregationBuilder(String name, String childType) {
-        super(name, TYPE, ValuesSourceType.BYTES, ValueType.STRING);
-        if (childType == null) {
-            throw new IllegalArgumentException("[childType] must not be null: [" + name + "]");
-        }
-        this.childType = childType;
-    }
-
-    /**
-     * Read from a stream.
-     */
-    public ChildrenAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE, ValuesSourceType.BYTES, ValueType.STRING);
-        childType = in.readString();
-    }
-
-    @Override
-    protected void innerWriteTo(StreamOutput out) throws IOException {
-        out.writeString(childType);
-    }
-
-    @Override
-    protected ValuesSourceAggregatorFactory<ParentChild, ?> innerBuild(AggregationContext context,
-            ValuesSourceConfig<ParentChild> config, AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
-        return new ChildrenAggregatorFactory(name, type, config, parentType, childFilter, parentFilter, context, parent,
-                subFactoriesBuilder, metaData);
-    }
-
-    @Override
-    protected ValuesSourceConfig<ParentChild> resolveConfig(AggregationContext aggregationContext) {
-        ValuesSourceConfig<ParentChild> config = new ValuesSourceConfig<>(ValuesSourceType.BYTES);
-        DocumentMapper childDocMapper = aggregationContext.searchContext().mapperService().documentMapper(childType);
-
-        if (childDocMapper != null) {
-            ParentFieldMapper parentFieldMapper = childDocMapper.parentFieldMapper();
-            if (!parentFieldMapper.active()) {
-                throw new IllegalArgumentException("[children] no [_parent] field not configured that points to a parent type");
-            }
-            parentType = parentFieldMapper.type();
-            DocumentMapper parentDocMapper = aggregationContext.searchContext().mapperService().documentMapper(parentType);
-            if (parentDocMapper != null) {
-                parentFilter = parentDocMapper.typeFilter();
-                childFilter = childDocMapper.typeFilter();
-                ParentChildIndexFieldData parentChildIndexFieldData = aggregationContext.searchContext().fieldData()
-                        .getForField(parentFieldMapper.fieldType());
-                config.fieldContext(new FieldContext(parentFieldMapper.fieldType().name(), parentChildIndexFieldData,
-                        parentFieldMapper.fieldType()));
-            } else {
-                config.unmapped(true);
-            }
-        } else {
-            config.unmapped(true);
-        }
-        return config;
-    }
-
-    @Override
-    protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
-        builder.field(ParentToChildrenAggregator.TYPE_FIELD.getPreferredName(), childType);
-        return builder;
-    }
-
-    public static ChildrenAggregationBuilder parse(String aggregationName, QueryParseContext context) throws IOException {
-        String childType = null;
-
-        XContentParser.Token token;
-        String currentFieldName = null;
-        XContentParser parser = context.parser();
-        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-            if (token == XContentParser.Token.FIELD_NAME) {
-                currentFieldName = parser.currentName();
-            } else if (token == XContentParser.Token.VALUE_STRING) {
-                if ("type".equals(currentFieldName)) {
-                    childType = parser.text();
-                } else {
-                    throw new ParsingException(parser.getTokenLocation(),
-                            "Unknown key for a " + token + " in [" + aggregationName + "]: [" + currentFieldName + "].");
-                }
-            } else {
-                throw new ParsingException(parser.getTokenLocation(), "Unexpected token " + token + " in [" + aggregationName + "].");
-            }
-        }
-
-        if (childType == null) {
-            throw new ParsingException(parser.getTokenLocation(),
-                    "Missing [child_type] field for children aggregation [" + aggregationName + "]");
-        }
-
-
-        return new ChildrenAggregationBuilder(aggregationName, childType);
-    }
-
-    @Override
-    protected int innerHashCode() {
-        return Objects.hash(childType);
-    }
-
-    @Override
-    protected boolean innerEquals(Object obj) {
-        ChildrenAggregationBuilder other = (ChildrenAggregationBuilder) obj;
-        return Objects.equals(childType, other.childType);
-    }
-
-    @Override
-    public String getWriteableName() {
-        return NAME;
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/ChildrenAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/ChildrenAggregatorFactory.java
deleted file mode 100644
index cb87f1260b382..0000000000000
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/ChildrenAggregatorFactory.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.aggregations.bucket.children;
-
-import org.apache.lucene.search.Query;
-import org.elasticsearch.search.aggregations.Aggregator;
-import org.elasticsearch.search.aggregations.AggregatorFactories;
-import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
-import org.elasticsearch.search.aggregations.NonCollectingAggregator;
-import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
-import org.elasticsearch.search.aggregations.support.ValuesSource;
-import org.elasticsearch.search.aggregations.support.ValuesSource.Bytes.ParentChild;
-import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
-import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-
-public class ChildrenAggregatorFactory
-        extends ValuesSourceAggregatorFactory<ValuesSource.Bytes.WithOrdinals.ParentChild, ChildrenAggregatorFactory> {
-
-    private final String parentType;
-    private final Query parentFilter;
-    private final Query childFilter;
-
-    public ChildrenAggregatorFactory(String name, Type type, ValuesSourceConfig<ParentChild> config, String parentType, Query childFilter,
-            Query parentFilter, AggregationContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
-            Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
-        this.parentType = parentType;
-        this.childFilter = childFilter;
-        this.parentFilter = parentFilter;
-    }
-
-    @Override
-    protected Aggregator createUnmapped(Aggregator parent, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData)
-            throws IOException {
-        return new NonCollectingAggregator(name, context, parent, pipelineAggregators, metaData) {
-
-            @Override
-            public InternalAggregation buildEmptyAggregation() {
-                return new InternalChildren(name, 0, buildEmptySubAggregations(), pipelineAggregators(), metaData());
-            }
-
-        };
-    }
-
-    @Override
-    protected Aggregator doCreateInternal(ValuesSource.Bytes.WithOrdinals.ParentChild valuesSource, Aggregator parent,
-            boolean collectsFromSingleBucket, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData)
-                    throws IOException {
-        long maxOrd = valuesSource.globalMaxOrd(context.searchContext().searcher(), parentType);
-        return new ParentToChildrenAggregator(name, factories, context, parent, parentType, childFilter, parentFilter, valuesSource, maxOrd,
-                pipelineAggregators, metaData);
-    }
-
-}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregationBuilder.java
index 48be5365bb19f..bbf58384c8be2 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregationBuilder.java
@@ -28,15 +28,13 @@
 import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.Objects;
 
 public class FilterAggregationBuilder extends AbstractAggregationBuilder<FilterAggregationBuilder> {
     public static final String NAME = "filter";
-    private static final Type TYPE = new Type(NAME);
 
     private final QueryBuilder filter;
 
@@ -49,7 +47,7 @@ public class FilterAggregationBuilder extends AbstractAggregationBuilder<FilterA
      *            {@link Filter} aggregation.
      */
     public FilterAggregationBuilder(String name, QueryBuilder filter) {
-        super(name, TYPE);
+        super(name);
         if (filter == null) {
             throw new IllegalArgumentException("[filter] must not be null: [" + name + "]");
         }
@@ -60,7 +58,7 @@ public FilterAggregationBuilder(String name, QueryBuilder filter) {
      * Read from a stream.
      */
     public FilterAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE);
+        super(in);
         filter = in.readNamedWriteable(QueryBuilder.class);
     }
 
@@ -70,9 +68,11 @@ protected void doWriteTo(StreamOutput out) throws IOException {
     }
 
     @Override
-    protected AggregatorFactory<?> doBuild(AggregationContext context, AggregatorFactory<?> parent,
+    protected AggregatorFactory<?> doBuild(SearchContext context, AggregatorFactory<?> parent,
             AggregatorFactories.Builder subFactoriesBuilder) throws IOException {
-        return new FilterAggregatorFactory(name, type, filter, context, parent, subFactoriesBuilder, metaData);
+        // TODO this sucks we need a rewrite phase for aggregations too
+        final QueryBuilder rewrittenFilter = QueryBuilder.rewriteQuery(filter, context.getQueryShardContext());
+        return new FilterAggregatorFactory(name, rewrittenFilter, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
@@ -100,7 +100,7 @@ protected boolean doEquals(Object obj) {
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregator.java
index 1a79631f93d59..46a9049711f85 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregator.java
@@ -29,7 +29,7 @@
 import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
 import org.elasticsearch.search.aggregations.bucket.SingleBucketAggregator;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -45,10 +45,10 @@ public class FilterAggregator extends SingleBucketAggregator {
     public FilterAggregator(String name,
                             Weight filter,
                             AggregatorFactories factories,
-                            AggregationContext aggregationContext,
+                            SearchContext context,
                             Aggregator parent, List<PipelineAggregator> pipelineAggregators,
                             Map<String, Object> metaData) throws IOException {
-        super(name, factories, aggregationContext, parent, pipelineAggregators, metaData);
+        super(name, factories, context, parent, pipelineAggregators, metaData);
         this.filter = filter;
     }
 
@@ -56,7 +56,7 @@ public FilterAggregator(String name,
     public LeafBucketCollector getLeafCollector(LeafReaderContext ctx,
             final LeafBucketCollector sub) throws IOException {
         // no need to provide deleted docs to the filter
-        final Bits bits = Lucene.asSequentialAccessBits(ctx.reader().maxDoc(), filter.scorer(ctx));
+        final Bits bits = Lucene.asSequentialAccessBits(ctx.reader().maxDoc(), filter.scorerSupplier(ctx));
         return new LeafBucketCollectorBase(sub, null) {
             @Override
             public void collect(int doc, long bucket) throws IOException {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregatorFactory.java
index 212494ef48b7d..482bcb3d00951 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregatorFactory.java
@@ -26,9 +26,8 @@
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -36,13 +35,13 @@
 
 public class FilterAggregatorFactory extends AggregatorFactory<FilterAggregatorFactory> {
 
-    private final Weight weight;
+    final Weight weight;
 
-    public FilterAggregatorFactory(String name, Type type, QueryBuilder filterBuilder, AggregationContext context,
+    public FilterAggregatorFactory(String name, QueryBuilder filterBuilder, SearchContext context,
             AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
-        super(name, type, context, parent, subFactoriesBuilder, metaData);
-        IndexSearcher contextSearcher = context.searchContext().searcher();
-        Query filter = filterBuilder.toQuery(context.searchContext().getQueryShardContext());
+        super(name, context, parent, subFactoriesBuilder, metaData);
+        IndexSearcher contextSearcher = context.searcher();
+        Query filter = filterBuilder.toFilter(context.getQueryShardContext());
         weight = contextSearcher.createNormalizedWeight(filter, false);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/ParsedFilter.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/ParsedFilter.java
new file mode 100644
index 0000000000000..5f5cf104498e8
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/ParsedFilter.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.bucket.filter;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.bucket.ParsedSingleBucketAggregation;
+
+import java.io.IOException;
+
+public class ParsedFilter extends ParsedSingleBucketAggregation implements Filter {
+
+    @Override
+    public String getType() {
+        return FilterAggregationBuilder.NAME;
+    }
+
+    public static ParsedFilter fromXContent(XContentParser parser, final String name) throws IOException {
+        return parseXContent(parser, new ParsedFilter(), name);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregationBuilder.java
index 2cd4f508ccbea..12c977a95a7ad 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregationBuilder.java
@@ -29,10 +29,9 @@
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.bucket.filters.FiltersAggregator.KeyedFilter;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -45,7 +44,6 @@
 
 public class FiltersAggregationBuilder extends AbstractAggregationBuilder<FiltersAggregationBuilder> {
     public static final String NAME = "filters";
-    private static final Type TYPE = new Type(NAME);
 
     private static final ParseField FILTERS_FIELD = new ParseField("filters");
     private static final ParseField OTHER_BUCKET_FIELD = new ParseField("other_bucket");
@@ -67,7 +65,7 @@ public FiltersAggregationBuilder(String name, KeyedFilter... filters) {
     }
 
     private FiltersAggregationBuilder(String name, List<KeyedFilter> filters) {
-        super(name, TYPE);
+        super(name);
         // internally we want to have a fixed order of filters, regardless of the order of the filters in the request
         this.filters = new ArrayList<>(filters);
         Collections.sort(this.filters, (KeyedFilter kf1, KeyedFilter kf2) -> kf1.key().compareTo(kf2.key()));
@@ -81,7 +79,7 @@ private FiltersAggregationBuilder(String name, List<KeyedFilter> filters) {
      *            the filters to use with this aggregation
      */
     public FiltersAggregationBuilder(String name, QueryBuilder... filters) {
-        super(name, TYPE);
+        super(name);
         List<KeyedFilter> keyedFilters = new ArrayList<>(filters.length);
         for (int i = 0; i < filters.length; i++) {
             keyedFilters.add(new KeyedFilter(String.valueOf(i), filters[i]));
@@ -94,7 +92,7 @@ public FiltersAggregationBuilder(String name, QueryBuilder... filters) {
      * Read from a stream.
      */
     public FiltersAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE);
+        super(in);
         keyed = in.readBoolean();
         int filtersSize = in.readVInt();
         filters = new ArrayList<>(filtersSize);
@@ -171,9 +169,14 @@ public String otherBucketKey() {
     }
 
     @Override
-    protected AggregatorFactory<?> doBuild(AggregationContext context, AggregatorFactory<?> parent, Builder subFactoriesBuilder)
+    protected AggregatorFactory<?> doBuild(SearchContext context, AggregatorFactory<?> parent, Builder subFactoriesBuilder)
             throws IOException {
-        return new FiltersAggregatorFactory(name, type, filters, keyed, otherBucket, otherBucketKey, context, parent,
+        List<KeyedFilter> rewrittenFilters = new ArrayList<>(filters.size());
+        for(KeyedFilter kf : filters) {
+            rewrittenFilters.add(new KeyedFilter(kf.key(), QueryBuilder.rewriteQuery(kf.filter(),
+                    context.getQueryShardContext())));
+        }
+        return new FiltersAggregatorFactory(name, rewrittenFilters, keyed, otherBucket, otherBucketKey, context, parent,
                 subFactoriesBuilder, metaData);
     }
 
@@ -209,26 +212,26 @@ public static FiltersAggregationBuilder parse(String aggregationName, QueryParse
         XContentParser.Token token = null;
         String currentFieldName = null;
         String otherBucketKey = null;
-        Boolean otherBucket = false;
+        Boolean otherBucket = null;
         while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.VALUE_BOOLEAN) {
-                if (context.getParseFieldMatcher().match(currentFieldName, OTHER_BUCKET_FIELD)) {
+                if (OTHER_BUCKET_FIELD.match(currentFieldName)) {
                     otherBucket = parser.booleanValue();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
                             "Unknown key for a " + token + " in [" + aggregationName + "]: [" + currentFieldName + "].");
                 }
             } else if (token == XContentParser.Token.VALUE_STRING) {
-                if (context.getParseFieldMatcher().match(currentFieldName, OTHER_BUCKET_KEY_FIELD)) {
+                if (OTHER_BUCKET_KEY_FIELD.match(currentFieldName)) {
                     otherBucketKey = parser.text();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
                             "Unknown key for a " + token + " in [" + aggregationName + "]: [" + currentFieldName + "].");
                 }
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (context.getParseFieldMatcher().match(currentFieldName, FILTERS_FIELD)) {
+                if (FILTERS_FIELD.match(currentFieldName)) {
                     keyedFilters = new ArrayList<>();
                     String key = null;
                     while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
@@ -244,7 +247,7 @@ public static FiltersAggregationBuilder parse(String aggregationName, QueryParse
                             "Unknown key for a " + token + " in [" + aggregationName + "]: [" + currentFieldName + "].");
                 }
             } else if (token == XContentParser.Token.START_ARRAY) {
-                if (context.getParseFieldMatcher().match(currentFieldName, FILTERS_FIELD)) {
+                if (FILTERS_FIELD.match(currentFieldName)) {
                     nonKeyedFilters = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         QueryBuilder filter = context.parseInnerQueryBuilder().orElse(matchAllQuery());
@@ -260,8 +263,9 @@ public static FiltersAggregationBuilder parse(String aggregationName, QueryParse
             }
         }
 
-        if (otherBucket && otherBucketKey == null) {
-            otherBucketKey = "_other_";
+        if (otherBucket == null && otherBucketKey != null) {
+            // automatically enable the other bucket if a key is set, as per the doc
+            otherBucket = true;
         }
 
         FiltersAggregationBuilder factory;
@@ -296,7 +300,7 @@ protected boolean doEquals(Object obj) {
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregator.java
index 08bbdaf3e3b53..898acd8eb77aa 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregator.java
@@ -29,7 +29,6 @@
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
@@ -39,7 +38,7 @@
 import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
 import org.elasticsearch.search.aggregations.bucket.BucketsAggregator;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -126,10 +125,10 @@ public boolean equals(Object obj) {
     private final int totalNumKeys;
 
     public FiltersAggregator(String name, AggregatorFactories factories, String[] keys, Weight[] filters, boolean keyed, String otherBucketKey,
-            AggregationContext aggregationContext,
+            SearchContext context,
             Aggregator parent, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData)
             throws IOException {
-        super(name, factories, aggregationContext, parent, pipelineAggregators, metaData);
+        super(name, factories, context, parent, pipelineAggregators, metaData);
         this.keyed = keyed;
         this.keys = keys;
         this.filters = filters;
@@ -148,7 +147,7 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx,
         // no need to provide deleted docs to the filter
         final Bits[] bits = new Bits[filters.length];
         for (int i = 0; i < filters.length; ++i) {
-            bits[i] = Lucene.asSequentialAccessBits(ctx.reader().maxDoc(), filters[i].scorer(ctx));
+            bits[i] = Lucene.asSequentialAccessBits(ctx.reader().maxDoc(), filters[i].scorerSupplier(ctx));
         }
         return new LeafBucketCollectorBase(sub, null) {
             @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregatorFactory.java
index 9b7def2395e97..ded828d7623d3 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregatorFactory.java
@@ -25,10 +25,9 @@
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.bucket.filters.FiltersAggregator.KeyedFilter;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -37,25 +36,25 @@
 public class FiltersAggregatorFactory extends AggregatorFactory<FiltersAggregatorFactory> {
 
     private final String[] keys;
-    private final Weight[] weights;
+    final Weight[] weights;
     private final boolean keyed;
     private final boolean otherBucket;
     private final String otherBucketKey;
 
-    public FiltersAggregatorFactory(String name, Type type, List<KeyedFilter> filters, boolean keyed, boolean otherBucket,
-            String otherBucketKey, AggregationContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactories,
+    public FiltersAggregatorFactory(String name, List<KeyedFilter> filters, boolean keyed, boolean otherBucket,
+            String otherBucketKey, SearchContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactories,
             Map<String, Object> metaData) throws IOException {
-        super(name, type, context, parent, subFactories, metaData);
+        super(name, context, parent, subFactories, metaData);
         this.keyed = keyed;
         this.otherBucket = otherBucket;
         this.otherBucketKey = otherBucketKey;
-        IndexSearcher contextSearcher = context.searchContext().searcher();
+        IndexSearcher contextSearcher = context.searcher();
         weights = new Weight[filters.size()];
         keys = new String[filters.size()];
         for (int i = 0; i < filters.size(); ++i) {
             KeyedFilter keyedFilter = filters.get(i);
             this.keys[i] = keyedFilter.key();
-            Query filter = keyedFilter.filter().toFilter(context.searchContext().getQueryShardContext());
+            Query filter = keyedFilter.filter().toFilter(context.getQueryShardContext());
             this.weights[i] = contextSearcher.createNormalizedWeight(filter, false);
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/InternalFilters.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/InternalFilters.java
index bd33f1608bc8b..5153122272564 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/InternalFilters.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/InternalFilters.java
@@ -108,7 +108,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             } else {
                 builder.startObject();
             }
-            builder.field(CommonFields.DOC_COUNT, docCount);
+            builder.field(CommonFields.DOC_COUNT.getPreferredName(), docCount);
             aggregations.toXContentInternal(builder, params);
             builder.endObject();
             return builder;
@@ -210,9 +210,9 @@ public InternalAggregation doReduce(List<InternalAggregation> aggregations, Redu
     @Override
     public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
         if (keyed) {
-            builder.startObject(CommonFields.BUCKETS);
+            builder.startObject(CommonFields.BUCKETS.getPreferredName());
         } else {
-            builder.startArray(CommonFields.BUCKETS);
+            builder.startArray(CommonFields.BUCKETS.getPreferredName());
         }
         for (InternalBucket bucket : buckets) {
             bucket.toXContent(builder, params);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/ParsedFilters.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/ParsedFilters.java
new file mode 100644
index 0000000000000..e27706d162bff
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/ParsedFilters.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.filters;
+
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParserUtils;
+import org.elasticsearch.search.aggregations.Aggregation;
+import org.elasticsearch.search.aggregations.Aggregations;
+import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+
+public class ParsedFilters extends ParsedMultiBucketAggregation<ParsedFilters.ParsedBucket> implements Filters {
+
+    private Map<String, ParsedBucket> bucketMap;
+
+    @Override
+    public String getType() {
+        return FiltersAggregationBuilder.NAME;
+    }
+
+    @Override
+    public List<? extends Filters.Bucket> getBuckets() {
+        return buckets;
+    }
+
+    @Override
+    public ParsedBucket getBucketByKey(String key) {
+        if (bucketMap == null) {
+            bucketMap = new HashMap<>(buckets.size());
+            for (ParsedBucket bucket : buckets) {
+                bucketMap.put(bucket.getKey(), bucket);
+            }
+        }
+        return bucketMap.get(key);
+    }
+
+    private static ObjectParser<ParsedFilters, Void> PARSER =
+            new ObjectParser<>(ParsedFilters.class.getSimpleName(), true, ParsedFilters::new);
+    static {
+        declareMultiBucketAggregationFields(PARSER,
+                parser -> ParsedBucket.fromXContent(parser, false),
+                parser -> ParsedBucket.fromXContent(parser, true));
+    }
+
+    public static ParsedFilters fromXContent(XContentParser parser, String name) throws IOException {
+        ParsedFilters aggregation = PARSER.parse(parser, null);
+        aggregation.setName(name);
+        // in case this is not a keyed aggregation, we need to add numeric keys to the buckets
+        if (aggregation.keyed == false) {
+            int i = 0;
+            for (ParsedBucket bucket : aggregation.buckets) {
+                assert bucket.key == null;
+                bucket.key = String.valueOf(i);
+                i++;
+            }
+        }
+        return aggregation;
+    }
+
+    public static class ParsedBucket extends ParsedMultiBucketAggregation.ParsedBucket implements Filters.Bucket {
+
+        private String key;
+
+        @Override
+        public String getKey() {
+            return key;
+        }
+
+        @Override
+        public String getKeyAsString() {
+            return key;
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            if (isKeyed()) {
+                builder.startObject(key);
+            } else {
+                builder.startObject();
+            }
+            builder.field(CommonFields.DOC_COUNT.getPreferredName(), getDocCount());
+            getAggregations().toXContentInternal(builder, params);
+            builder.endObject();
+            return builder;
+        }
+
+
+        static ParsedBucket fromXContent(XContentParser parser, boolean keyed) throws IOException {
+            final ParsedBucket bucket = new ParsedBucket();
+            bucket.setKeyed(keyed);
+            XContentParser.Token token = parser.currentToken();
+            String currentFieldName = parser.currentName();
+            if (keyed) {
+                ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser::getTokenLocation);
+                bucket.key = currentFieldName;
+                ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
+            }
+
+            List<Aggregation> aggregations = new ArrayList<>();
+            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                if (token == XContentParser.Token.FIELD_NAME) {
+                    currentFieldName = parser.currentName();
+                } else if (token.isValue()) {
+                    if (CommonFields.DOC_COUNT.getPreferredName().equals(currentFieldName)) {
+                        bucket.setDocCount(parser.longValue());
+                    }
+                } else if (token == XContentParser.Token.START_OBJECT) {
+                    XContentParserUtils.parseTypedKeysObject(parser, Aggregation.TYPED_KEYS_DELIMITER, Aggregation.class,
+                            aggregations::add);
+                }
+            }
+            bucket.setAggregations(new Aggregations(aggregations));
+            return bucket;
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java
index 24d5999e698d5..602c3a81c66e9 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java
@@ -26,43 +26,59 @@
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.fielddata.MultiGeoPointValues;
 import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
 import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
 import org.elasticsearch.index.fielddata.SortingNumericDocValues;
+import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.bucket.BucketUtils;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.Objects;
 
 public class GeoGridAggregationBuilder extends ValuesSourceAggregationBuilder<ValuesSource.GeoPoint, GeoGridAggregationBuilder> {
     public static final String NAME = "geohash_grid";
-    private static final Type TYPE = new Type(NAME);
+    public static final int DEFAULT_PRECISION = 5;
+    public static final int DEFAULT_MAX_NUM_CELLS = 10000;
+
+    private static final ObjectParser<GeoGridAggregationBuilder, QueryParseContext> PARSER;
+    static {
+        PARSER = new ObjectParser<>(GeoGridAggregationBuilder.NAME);
+        ValuesSourceParserHelper.declareGeoFields(PARSER, false, false);
+        PARSER.declareInt(GeoGridAggregationBuilder::precision, GeoHashGridParams.FIELD_PRECISION);
+        PARSER.declareInt(GeoGridAggregationBuilder::size, GeoHashGridParams.FIELD_SIZE);
+        PARSER.declareInt(GeoGridAggregationBuilder::shardSize, GeoHashGridParams.FIELD_SHARD_SIZE);
+    }
+
+    public static GeoGridAggregationBuilder parse(String aggregationName, QueryParseContext context) throws IOException {
+        return PARSER.parse(context.parser(), new GeoGridAggregationBuilder(aggregationName), context);
+    }
 
-    private int precision = GeoHashGridParser.DEFAULT_PRECISION;
-    private int requiredSize = GeoHashGridParser.DEFAULT_MAX_NUM_CELLS;
+    private int precision = DEFAULT_PRECISION;
+    private int requiredSize = DEFAULT_MAX_NUM_CELLS;
     private int shardSize = -1;
 
     public GeoGridAggregationBuilder(String name) {
-        super(name, TYPE, ValuesSourceType.GEOPOINT, ValueType.GEOPOINT);
+        super(name, ValuesSourceType.GEOPOINT, ValueType.GEOPOINT);
     }
 
     /**
      * Read from a stream.
      */
     public GeoGridAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE, ValuesSourceType.GEOPOINT, ValueType.GEOPOINT);
+        super(in, ValuesSourceType.GEOPOINT, ValueType.GEOPOINT);
         precision = in.readVInt();
         requiredSize = in.readVInt();
         shardSize = in.readVInt();
@@ -111,7 +127,7 @@ public int shardSize() {
     }
 
     @Override
-    protected ValuesSourceAggregatorFactory<ValuesSource.GeoPoint, ?> innerBuild(AggregationContext context,
+    protected ValuesSourceAggregatorFactory<ValuesSource.GeoPoint, ?> innerBuild(SearchContext context,
             ValuesSourceConfig<ValuesSource.GeoPoint> config, AggregatorFactory<?> parent, Builder subFactoriesBuilder)
                     throws IOException {
         int shardSize = this.shardSize;
@@ -121,7 +137,7 @@ public int shardSize() {
         if (shardSize < 0) {
             // Use default heuristic to avoid any wrong-ranking caused by
             // distributed counting
-            shardSize = BucketUtils.suggestShardSideQueueSize(requiredSize, context.searchContext().numberOfShards());
+            shardSize = BucketUtils.suggestShardSideQueueSize(requiredSize, context.numberOfShards());
         }
 
         if (requiredSize <= 0 || shardSize <= 0) {
@@ -132,7 +148,7 @@ public int shardSize() {
         if (shardSize < requiredSize) {
             shardSize = requiredSize;
         }
-        return new GeoHashGridAggregatorFactory(name, type, config, precision, requiredSize, shardSize, context, parent,
+        return new GeoHashGridAggregatorFactory(name, config, precision, requiredSize, shardSize, context, parent,
                 subFactoriesBuilder, metaData);
     }
 
@@ -167,7 +183,7 @@ protected int innerHashCode() {
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 
@@ -196,7 +212,7 @@ static class CellIdSource extends ValuesSource.Numeric {
         private final ValuesSource.GeoPoint valuesSource;
         private final int precision;
 
-        public CellIdSource(ValuesSource.GeoPoint valuesSource, int precision) {
+        CellIdSource(ValuesSource.GeoPoint valuesSource, int precision) {
             this.valuesSource = valuesSource;
             //different GeoPoints could map to the same or different geohash cells.
             this.precision = precision;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGrid.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGrid.java
index 6456fba864021..9cce698957d70 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGrid.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGrid.java
@@ -38,6 +38,5 @@ interface Bucket extends MultiBucketsAggregation.Bucket {
      * @return  The buckets of this aggregation (each bucket representing a geohash grid cell)
      */
     @Override
-    List<Bucket> getBuckets();
-
+    List<? extends Bucket> getBuckets();
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java
index d3653e87422de..a24efe2df9bfc 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java
@@ -29,7 +29,7 @@
 import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
 import org.elasticsearch.search.aggregations.bucket.BucketsAggregator;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -50,7 +50,7 @@ public class GeoHashGridAggregator extends BucketsAggregator {
     private final LongHash bucketOrds;
 
     public GeoHashGridAggregator(String name, AggregatorFactories factories, GeoGridAggregationBuilder.CellIdSource valuesSource,
-            int requiredSize, int shardSize, AggregationContext aggregationContext, Aggregator parent, List<PipelineAggregator> pipelineAggregators,
+            int requiredSize, int shardSize, SearchContext aggregationContext, Aggregator parent, List<PipelineAggregator> pipelineAggregators,
             Map<String, Object> metaData) throws IOException {
         super(name, factories, aggregationContext, parent, pipelineAggregators, metaData);
         this.valuesSource = valuesSource;
@@ -98,7 +98,7 @@ static class OrdinalBucket extends InternalGeoHashGrid.Bucket {
 
         long bucketOrd;
 
-        public OrdinalBucket() {
+        OrdinalBucket() {
             super(0, 0, (InternalAggregations) null);
         }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java
index 1b2c4c263724f..4f87b82f7882f 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java
@@ -23,15 +23,14 @@
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.InternalAggregation;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.NonCollectingAggregator;
 import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder.CellIdSource;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.GeoPoint;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -44,10 +43,10 @@ public class GeoHashGridAggregatorFactory extends ValuesSourceAggregatorFactory<
     private final int requiredSize;
     private final int shardSize;
 
-    public GeoHashGridAggregatorFactory(String name, Type type, ValuesSourceConfig<GeoPoint> config, int precision, int requiredSize,
-            int shardSize, AggregationContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
+    public GeoHashGridAggregatorFactory(String name, ValuesSourceConfig<GeoPoint> config, int precision, int requiredSize,
+            int shardSize, SearchContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
             Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.precision = precision;
         this.requiredSize = requiredSize;
         this.shardSize = shardSize;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridParser.java
deleted file mode 100644
index e669ee8b9d9e0..0000000000000
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridParser.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.search.aggregations.bucket.geogrid;
-
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentParser.Token;
-import org.elasticsearch.index.query.GeoBoundingBoxQueryBuilder;
-import org.elasticsearch.search.aggregations.support.AbstractValuesSourceParser.GeoPointValuesSourceParser;
-import org.elasticsearch.search.aggregations.support.XContentParseContext;
-import org.elasticsearch.search.aggregations.support.ValueType;
-import org.elasticsearch.search.aggregations.support.ValuesSourceType;
-
-import java.io.IOException;
-import java.util.Map;
-
-/**
- * Aggregates Geo information into cells determined by geohashes of a given precision.
- * WARNING - for high-precision geohashes it may prove necessary to use a {@link GeoBoundingBoxQueryBuilder}
- * aggregation to focus in on a smaller area to avoid generating too many buckets and using too much RAM
- */
-public class GeoHashGridParser extends GeoPointValuesSourceParser {
-
-    public static final int DEFAULT_PRECISION = 5;
-    public static final int DEFAULT_MAX_NUM_CELLS = 10000;
-
-    public GeoHashGridParser() {
-        super(false, false);
-    }
-
-    @Override
-    protected GeoGridAggregationBuilder createFactory(
-            String aggregationName, ValuesSourceType valuesSourceType,
-            ValueType targetValueType, Map<ParseField, Object> otherOptions) {
-        GeoGridAggregationBuilder factory = new GeoGridAggregationBuilder(aggregationName);
-        Integer precision = (Integer) otherOptions.get(GeoHashGridParams.FIELD_PRECISION);
-        if (precision != null) {
-            factory.precision(precision);
-        }
-        Integer size = (Integer) otherOptions.get(GeoHashGridParams.FIELD_SIZE);
-        if (size != null) {
-            factory.size(size);
-        }
-        Integer shardSize = (Integer) otherOptions.get(GeoHashGridParams.FIELD_SHARD_SIZE);
-        if (shardSize != null) {
-            factory.shardSize(shardSize);
-        }
-        return factory;
-    }
-
-    @Override
-    protected boolean token(String aggregationName, String currentFieldName, Token token,
-                            XContentParseContext context, Map<ParseField, Object> otherOptions) throws IOException {
-        XContentParser parser = context.getParser();
-        if (token == XContentParser.Token.VALUE_NUMBER || token == XContentParser.Token.VALUE_STRING) {
-            if (context.matchField(currentFieldName, GeoHashGridParams.FIELD_PRECISION)) {
-                otherOptions.put(GeoHashGridParams.FIELD_PRECISION, parser.intValue());
-                return true;
-            } else if (context.matchField(currentFieldName, GeoHashGridParams.FIELD_SIZE)) {
-                otherOptions.put(GeoHashGridParams.FIELD_SIZE, parser.intValue());
-                return true;
-            } else if (context.matchField(currentFieldName, GeoHashGridParams.FIELD_SHARD_SIZE)) {
-                otherOptions.put(GeoHashGridParams.FIELD_SHARD_SIZE, parser.intValue());
-                return true;
-            }
-        }
-        return false;
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoHashGrid.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoHashGrid.java
index eaf00755f76dc..af98e4a1c249c 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoHashGrid.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoHashGrid.java
@@ -52,7 +52,7 @@ static class Bucket extends InternalMultiBucketAggregation.InternalBucket implem
         protected long docCount;
         protected InternalAggregations aggregations;
 
-        public Bucket(long geohashAsLong, long docCount, InternalAggregations aggregations) {
+        Bucket(long geohashAsLong, long docCount, InternalAggregations aggregations) {
             this.docCount = docCount;
             this.aggregations = aggregations;
             this.geohashAsLong = geohashAsLong;
@@ -120,8 +120,8 @@ public Bucket reduce(List<? extends Bucket> buckets, ReduceContext context) {
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.startObject();
-            builder.field(CommonFields.KEY, getKeyAsString());
-            builder.field(CommonFields.DOC_COUNT, docCount);
+            builder.field(CommonFields.KEY.getPreferredName(), getKeyAsString());
+            builder.field(CommonFields.DOC_COUNT.getPreferredName(), docCount);
             aggregations.toXContentInternal(builder, params);
             builder.endObject();
             return builder;
@@ -169,7 +169,7 @@ public Bucket createBucket(InternalAggregations aggregations, Bucket prototype)
     }
 
     @Override
-    public List<GeoHashGrid.Bucket> getBuckets() {
+    public List<InternalGeoHashGrid.Bucket> getBuckets() {
         return unmodifiableList(buckets);
     }
 
@@ -192,7 +192,7 @@ public InternalGeoHashGrid doReduce(List<InternalAggregation> aggregations, Redu
             }
         }
 
-        final int size = (int) Math.min(requiredSize, buckets.size());
+        final int size = Math.toIntExact(reduceContext.isFinalReduce() == false ? buckets.size() : Math.min(requiredSize, buckets.size()));
         BucketPriorityQueue ordered = new BucketPriorityQueue(size);
         for (LongObjectPagedHashMap.Cursor<List<Bucket>> cursor : buckets) {
             List<Bucket> sameCellBuckets = cursor.value;
@@ -208,7 +208,7 @@ public InternalGeoHashGrid doReduce(List<InternalAggregation> aggregations, Redu
 
     @Override
     public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
-        builder.startArray(CommonFields.BUCKETS);
+        builder.startArray(CommonFields.BUCKETS.getPreferredName());
         for (Bucket bucket : buckets) {
             bucket.toXContent(builder, params);
         }
@@ -218,7 +218,7 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th
 
     static class BucketPriorityQueue extends PriorityQueue<Bucket> {
 
-        public BucketPriorityQueue(int size) {
+        BucketPriorityQueue(int size) {
             super(size);
         }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/ParsedGeoHashGrid.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/ParsedGeoHashGrid.java
new file mode 100644
index 0000000000000..4551523e0fc8b
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/ParsedGeoHashGrid.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.geogrid;
+
+import org.elasticsearch.common.geo.GeoPoint;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation;
+
+import java.io.IOException;
+import java.util.List;
+
+public class ParsedGeoHashGrid extends ParsedMultiBucketAggregation<ParsedGeoHashGrid.ParsedBucket> implements GeoHashGrid {
+
+    @Override
+    public String getType() {
+        return GeoGridAggregationBuilder.NAME;
+    }
+
+    @Override
+    public List<? extends GeoHashGrid.Bucket> getBuckets() {
+        return buckets;
+    }
+
+    private static ObjectParser<ParsedGeoHashGrid, Void> PARSER =
+            new ObjectParser<>(ParsedGeoHashGrid.class.getSimpleName(), true, ParsedGeoHashGrid::new);
+    static {
+        declareMultiBucketAggregationFields(PARSER, ParsedBucket::fromXContent, ParsedBucket::fromXContent);
+    }
+
+    public static ParsedGeoHashGrid fromXContent(XContentParser parser, String name) throws IOException {
+        ParsedGeoHashGrid aggregation = PARSER.parse(parser, null);
+        aggregation.setName(name);
+        return aggregation;
+    }
+
+    public static class ParsedBucket extends ParsedMultiBucketAggregation.ParsedBucket implements GeoHashGrid.Bucket {
+
+        private String geohashAsString;
+
+        @Override
+        public GeoPoint getKey() {
+            return GeoPoint.fromGeohash(geohashAsString);
+        }
+
+        @Override
+        public String getKeyAsString() {
+            return geohashAsString;
+        }
+
+        @Override
+        protected XContentBuilder keyToXContent(XContentBuilder builder) throws IOException {
+            return builder.field(CommonFields.KEY.getPreferredName(), geohashAsString);
+        }
+
+        static ParsedBucket fromXContent(XContentParser parser) throws IOException {
+            return parseXContent(parser, false, ParsedBucket::new, (p, bucket) -> bucket.geohashAsString = p.textOrNull());
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregationBuilder.java
index 94211f39dd099..2363ed498a9dd 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregationBuilder.java
@@ -26,24 +26,22 @@
 import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 
 public class GlobalAggregationBuilder extends AbstractAggregationBuilder<GlobalAggregationBuilder> {
     public static final String NAME = "global";
-    private static final Type TYPE = new Type(NAME);
 
     public GlobalAggregationBuilder(String name) {
-        super(name, TYPE);
+        super(name);
     }
 
     /**
      * Read from a stream.
      */
     public GlobalAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE);
+        super(in);
     }
 
     @Override
@@ -52,9 +50,9 @@ protected void doWriteTo(StreamOutput out) throws IOException {
     }
 
     @Override
-    protected AggregatorFactory<?> doBuild(AggregationContext context, AggregatorFactory<?> parent, Builder subFactoriesBuilder)
+    protected AggregatorFactory<?> doBuild(SearchContext context, AggregatorFactory<?> parent, Builder subFactoriesBuilder)
             throws IOException {
-        return new GlobalAggregatorFactory(name, type, context, parent, subFactoriesBuilder, metaData);
+        return new GlobalAggregatorFactory(name, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
@@ -80,7 +78,7 @@ protected int doHashCode() {
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregator.java
index 6e4980ede2715..f89144bfae32a 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregator.java
@@ -25,7 +25,7 @@
 import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
 import org.elasticsearch.search.aggregations.bucket.SingleBucketAggregator;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -36,7 +36,7 @@
  */
 public class GlobalAggregator extends SingleBucketAggregator {
 
-    public GlobalAggregator(String name, AggregatorFactories subFactories, AggregationContext aggregationContext, List<PipelineAggregator> pipelineAggregators,
+    public GlobalAggregator(String name, AggregatorFactories subFactories, SearchContext aggregationContext, List<PipelineAggregator> pipelineAggregators,
             Map<String, Object> metaData) throws IOException {
         super(name, subFactories, aggregationContext, null, pipelineAggregators, metaData);
     }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregatorFactory.java
index f92241f534598..b11ffde6d7bc4 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregatorFactory.java
@@ -23,9 +23,8 @@
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -33,9 +32,9 @@
 
 public class GlobalAggregatorFactory extends AggregatorFactory<GlobalAggregatorFactory> {
 
-    public GlobalAggregatorFactory(String name, Type type, AggregationContext context, AggregatorFactory<?> parent,
+    public GlobalAggregatorFactory(String name, SearchContext context, AggregatorFactory<?> parent,
             AggregatorFactories.Builder subFactories, Map<String, Object> metaData) throws IOException {
-        super(name, type, context, parent, subFactories, metaData);
+        super(name, context, parent, subFactories, metaData);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/InternalGlobal.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/InternalGlobal.java
index f278e5e72b1da..6ba3b79e96863 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/InternalGlobal.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/InternalGlobal.java
@@ -32,7 +32,8 @@
  * regardless the query.
  */
 public class InternalGlobal extends InternalSingleBucketAggregation implements Global {
-    InternalGlobal(String name, long docCount, InternalAggregations aggregations, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
+    InternalGlobal(String name, long docCount, InternalAggregations aggregations, List<PipelineAggregator> pipelineAggregators,
+            Map<String, Object> metaData) {
         super(name, docCount, aggregations, pipelineAggregators, metaData);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/ParsedGlobal.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/ParsedGlobal.java
new file mode 100644
index 0000000000000..062752805b1c8
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/ParsedGlobal.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.bucket.global;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.bucket.ParsedSingleBucketAggregation;
+
+import java.io.IOException;
+
+public class ParsedGlobal extends ParsedSingleBucketAggregation implements Global {
+
+    @Override
+    public String getType() {
+        return GlobalAggregationBuilder.NAME;
+    }
+
+    public static ParsedGlobal fromXContent(XContentParser parser, final String name) throws IOException {
+        return parseXContent(parser, new ParsedGlobal(), name);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java
index d3b8857ccab91..e805abd86c981 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java
@@ -19,30 +19,107 @@
 
 package org.elasticsearch.search.aggregations.bucket.histogram;
 
+import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.rounding.DateTimeUnit;
+import org.elasticsearch.common.rounding.Rounding;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParser.Token;
+import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.Objects;
 
+import static java.util.Collections.unmodifiableMap;
+
 /**
  * A builder for histograms on date fields.
  */
 public class DateHistogramAggregationBuilder
         extends ValuesSourceAggregationBuilder<ValuesSource.Numeric, DateHistogramAggregationBuilder> {
-    public static final String NAME = InternalDateHistogram.TYPE.name();
+    public static final String NAME = "date_histogram";
+
+    public static final Map<String, DateTimeUnit> DATE_FIELD_UNITS;
+
+    static {
+        Map<String, DateTimeUnit> dateFieldUnits = new HashMap<>();
+        dateFieldUnits.put("year", DateTimeUnit.YEAR_OF_CENTURY);
+        dateFieldUnits.put("1y", DateTimeUnit.YEAR_OF_CENTURY);
+        dateFieldUnits.put("quarter", DateTimeUnit.QUARTER);
+        dateFieldUnits.put("1q", DateTimeUnit.QUARTER);
+        dateFieldUnits.put("month", DateTimeUnit.MONTH_OF_YEAR);
+        dateFieldUnits.put("1M", DateTimeUnit.MONTH_OF_YEAR);
+        dateFieldUnits.put("week", DateTimeUnit.WEEK_OF_WEEKYEAR);
+        dateFieldUnits.put("1w", DateTimeUnit.WEEK_OF_WEEKYEAR);
+        dateFieldUnits.put("day", DateTimeUnit.DAY_OF_MONTH);
+        dateFieldUnits.put("1d", DateTimeUnit.DAY_OF_MONTH);
+        dateFieldUnits.put("hour", DateTimeUnit.HOUR_OF_DAY);
+        dateFieldUnits.put("1h", DateTimeUnit.HOUR_OF_DAY);
+        dateFieldUnits.put("minute", DateTimeUnit.MINUTES_OF_HOUR);
+        dateFieldUnits.put("1m", DateTimeUnit.MINUTES_OF_HOUR);
+        dateFieldUnits.put("second", DateTimeUnit.SECOND_OF_MINUTE);
+        dateFieldUnits.put("1s", DateTimeUnit.SECOND_OF_MINUTE);
+        DATE_FIELD_UNITS = unmodifiableMap(dateFieldUnits);
+    }
+
+    private static final ObjectParser<DateHistogramAggregationBuilder, QueryParseContext> PARSER;
+    static {
+        PARSER = new ObjectParser<>(DateHistogramAggregationBuilder.NAME);
+        ValuesSourceParserHelper.declareNumericFields(PARSER, true, true, true);
+
+        PARSER.declareField((histogram, interval) -> {
+            if (interval instanceof Long) {
+                histogram.interval((long) interval);
+            } else {
+                histogram.dateHistogramInterval((DateHistogramInterval) interval);
+            }
+        }, p -> {
+            if (p.currentToken() == XContentParser.Token.VALUE_NUMBER) {
+                return p.longValue();
+            } else {
+                return new DateHistogramInterval(p.text());
+            }
+        }, Histogram.INTERVAL_FIELD, ObjectParser.ValueType.LONG);
+
+        PARSER.declareField(DateHistogramAggregationBuilder::offset, p -> {
+            if (p.currentToken() == XContentParser.Token.VALUE_NUMBER) {
+                return p.longValue();
+            } else {
+                return DateHistogramAggregationBuilder.parseStringOffset(p.text());
+            }
+        }, Histogram.OFFSET_FIELD, ObjectParser.ValueType.LONG);
+
+        PARSER.declareBoolean(DateHistogramAggregationBuilder::keyed, Histogram.KEYED_FIELD);
+
+        PARSER.declareLong(DateHistogramAggregationBuilder::minDocCount, Histogram.MIN_DOC_COUNT_FIELD);
+
+        PARSER.declareField(DateHistogramAggregationBuilder::extendedBounds, parser -> ExtendedBounds.PARSER.apply(parser, null),
+                ExtendedBounds.EXTENDED_BOUNDS_FIELD, ObjectParser.ValueType.OBJECT);
+
+        PARSER.declareField(DateHistogramAggregationBuilder::order, DateHistogramAggregationBuilder::parseOrder,
+                Histogram.ORDER_FIELD, ObjectParser.ValueType.OBJECT);
+    }
+
+    public static DateHistogramAggregationBuilder parse(String aggregationName, QueryParseContext context) throws IOException {
+        return PARSER.parse(context.parser(), new DateHistogramAggregationBuilder(aggregationName), context);
+    }
 
     private long interval;
     private DateHistogramInterval dateHistogramInterval;
@@ -54,12 +131,12 @@ public class DateHistogramAggregationBuilder
 
     /** Create a new builder with the given name. */
     public DateHistogramAggregationBuilder(String name) {
-        super(name, InternalDateHistogram.TYPE, ValuesSourceType.NUMERIC, ValueType.DATE);
+        super(name, ValuesSourceType.NUMERIC, ValueType.DATE);
     }
 
     /** Read from a stream, for internal use only. */
     public DateHistogramAggregationBuilder(StreamInput in) throws IOException {
-        super(in, InternalDateHistogram.TYPE, ValuesSourceType.NUMERIC, ValueType.DATE);
+        super(in, ValuesSourceType.NUMERIC, ValueType.DATE);
         if (in.readBoolean()) {
             order = InternalOrder.Streams.readOrder(in);
         }
@@ -238,15 +315,43 @@ protected XContentBuilder doXContentBody(XContentBuilder builder, Params params)
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 
     @Override
-    protected ValuesSourceAggregatorFactory<Numeric, ?> innerBuild(AggregationContext context, ValuesSourceConfig<Numeric> config,
+    protected ValuesSourceAggregatorFactory<Numeric, ?> innerBuild(SearchContext context, ValuesSourceConfig<Numeric> config,
             AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
-        return new DateHistogramAggregatorFactory(name, type, config, interval, dateHistogramInterval, offset, order, keyed, minDocCount,
-                extendedBounds, context, parent, subFactoriesBuilder, metaData);
+        Rounding rounding = createRounding();
+        ExtendedBounds roundedBounds = null;
+        if (this.extendedBounds != null) {
+            // parse any string bounds to longs and round
+            roundedBounds = this.extendedBounds.parseAndValidate(name, context, config.format()).round(rounding);
+        }
+        return new DateHistogramAggregatorFactory(name, config, interval, dateHistogramInterval, offset, order, keyed, minDocCount,
+                rounding, roundedBounds, context, parent, subFactoriesBuilder, metaData);
+    }
+
+    private Rounding createRounding() {
+        Rounding.Builder tzRoundingBuilder;
+        if (dateHistogramInterval != null) {
+            DateTimeUnit dateTimeUnit = DATE_FIELD_UNITS.get(dateHistogramInterval.toString());
+            if (dateTimeUnit != null) {
+                tzRoundingBuilder = Rounding.builder(dateTimeUnit);
+            } else {
+                // the interval is a time value?
+                tzRoundingBuilder = Rounding.builder(
+                        TimeValue.parseTimeValue(dateHistogramInterval.toString(), null, getClass().getSimpleName() + ".interval"));
+            }
+        } else {
+            // the interval is an integer time value in millis?
+            tzRoundingBuilder = Rounding.builder(TimeValue.timeValueMillis(interval));
+        }
+        if (timeZone() != null) {
+            tzRoundingBuilder.timeZone(timeZone());
+        }
+        Rounding rounding = tzRoundingBuilder.build();
+        return rounding;
     }
 
     @Override
@@ -265,4 +370,35 @@ protected boolean innerEquals(Object obj) {
                 && Objects.equals(offset, other.offset)
                 && Objects.equals(extendedBounds, other.extendedBounds);
     }
+
+    // similar to the parsing oh histogram orders, but also accepts _time as an alias for _key
+    private static InternalOrder parseOrder(XContentParser parser, QueryParseContext context) throws IOException {
+        InternalOrder order = null;
+        Token token;
+        String currentFieldName = null;
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token == XContentParser.Token.VALUE_STRING) {
+                String dir = parser.text();
+                boolean asc = "asc".equals(dir);
+                if (!asc && !"desc".equals(dir)) {
+                    throw new ParsingException(parser.getTokenLocation(), "Unknown order direction: [" + dir
+                            + "]. Should be either [asc] or [desc]");
+                }
+                order = resolveOrder(currentFieldName, asc);
+            }
+        }
+        return order;
+    }
+
+    static InternalOrder resolveOrder(String key, boolean asc) {
+        if ("_key".equals(key) || "_time".equals(key)) {
+            return (InternalOrder) (asc ? InternalOrder.KEY_ASC : InternalOrder.KEY_DESC);
+        }
+        if ("_count".equals(key)) {
+            return (InternalOrder) (asc ? InternalOrder.COUNT_ASC : InternalOrder.COUNT_DESC);
+        }
+        return new InternalOrder.Aggregation(key, asc);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java
index 0ea2fba719b09..a52f2b2cfb5aa 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java
@@ -33,8 +33,8 @@
 import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
 import org.elasticsearch.search.aggregations.bucket.BucketsAggregator;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -62,10 +62,10 @@ class DateHistogramAggregator extends BucketsAggregator {
     private final LongHash bucketOrds;
     private long offset;
 
-    public DateHistogramAggregator(String name, AggregatorFactories factories, Rounding rounding, long offset, InternalOrder order,
+    DateHistogramAggregator(String name, AggregatorFactories factories, Rounding rounding, long offset, InternalOrder order,
             boolean keyed,
             long minDocCount, @Nullable ExtendedBounds extendedBounds, @Nullable ValuesSource.Numeric valuesSource,
-            DocValueFormat formatter, AggregationContext aggregationContext,
+            DocValueFormat formatter, SearchContext aggregationContext,
             Aggregator parent, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException {
 
         super(name, factories, aggregationContext, parent, pipelineAggregators, metaData);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorFactory.java
index 79f81e28374c4..44bb3e02afece 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorFactory.java
@@ -19,54 +19,24 @@
 
 package org.elasticsearch.search.aggregations.bucket.histogram;
 
-import org.elasticsearch.common.rounding.DateTimeUnit;
 import org.elasticsearch.common.rounding.Rounding;
-import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
+import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-import static java.util.Collections.unmodifiableMap;
-
-import org.elasticsearch.search.aggregations.support.AggregationContext;
-import org.elasticsearch.search.aggregations.support.ValuesSource;
-import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
-import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
-
 public final class DateHistogramAggregatorFactory
         extends ValuesSourceAggregatorFactory<ValuesSource.Numeric, DateHistogramAggregatorFactory> {
 
-    public static final Map<String, DateTimeUnit> DATE_FIELD_UNITS;
-
-    static {
-        Map<String, DateTimeUnit> dateFieldUnits = new HashMap<>();
-        dateFieldUnits.put("year", DateTimeUnit.YEAR_OF_CENTURY);
-        dateFieldUnits.put("1y", DateTimeUnit.YEAR_OF_CENTURY);
-        dateFieldUnits.put("quarter", DateTimeUnit.QUARTER);
-        dateFieldUnits.put("1q", DateTimeUnit.QUARTER);
-        dateFieldUnits.put("month", DateTimeUnit.MONTH_OF_YEAR);
-        dateFieldUnits.put("1M", DateTimeUnit.MONTH_OF_YEAR);
-        dateFieldUnits.put("week", DateTimeUnit.WEEK_OF_WEEKYEAR);
-        dateFieldUnits.put("1w", DateTimeUnit.WEEK_OF_WEEKYEAR);
-        dateFieldUnits.put("day", DateTimeUnit.DAY_OF_MONTH);
-        dateFieldUnits.put("1d", DateTimeUnit.DAY_OF_MONTH);
-        dateFieldUnits.put("hour", DateTimeUnit.HOUR_OF_DAY);
-        dateFieldUnits.put("1h", DateTimeUnit.HOUR_OF_DAY);
-        dateFieldUnits.put("minute", DateTimeUnit.MINUTES_OF_HOUR);
-        dateFieldUnits.put("1m", DateTimeUnit.MINUTES_OF_HOUR);
-        dateFieldUnits.put("second", DateTimeUnit.SECOND_OF_MINUTE);
-        dateFieldUnits.put("1s", DateTimeUnit.SECOND_OF_MINUTE);
-        DATE_FIELD_UNITS = unmodifiableMap(dateFieldUnits);
-    }
-
     private final DateHistogramInterval dateHistogramInterval;
     private final long interval;
     private final long offset;
@@ -74,12 +44,13 @@ public final class DateHistogramAggregatorFactory
     private final boolean keyed;
     private final long minDocCount;
     private final ExtendedBounds extendedBounds;
+    private Rounding rounding;
 
-    public DateHistogramAggregatorFactory(String name, Type type, ValuesSourceConfig<Numeric> config, long interval,
+    public DateHistogramAggregatorFactory(String name, ValuesSourceConfig<Numeric> config, long interval,
             DateHistogramInterval dateHistogramInterval, long offset, InternalOrder order, boolean keyed, long minDocCount,
-            ExtendedBounds extendedBounds, AggregationContext context, AggregatorFactory<?> parent,
+            Rounding rounding, ExtendedBounds extendedBounds, SearchContext context, AggregatorFactory<?> parent,
             AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.interval = interval;
         this.dateHistogramInterval = dateHistogramInterval;
         this.offset = offset;
@@ -87,34 +58,13 @@ public DateHistogramAggregatorFactory(String name, Type type, ValuesSourceConfig
         this.keyed = keyed;
         this.minDocCount = minDocCount;
         this.extendedBounds = extendedBounds;
+        this.rounding = rounding;
     }
 
     public long minDocCount() {
         return minDocCount;
     }
 
-    private Rounding createRounding() {
-        Rounding.Builder tzRoundingBuilder;
-        if (dateHistogramInterval != null) {
-            DateTimeUnit dateTimeUnit = DATE_FIELD_UNITS.get(dateHistogramInterval.toString());
-            if (dateTimeUnit != null) {
-                tzRoundingBuilder = Rounding.builder(dateTimeUnit);
-            } else {
-                // the interval is a time value?
-                tzRoundingBuilder = Rounding.builder(
-                        TimeValue.parseTimeValue(dateHistogramInterval.toString(), null, getClass().getSimpleName() + ".interval"));
-            }
-        } else {
-            // the interval is an integer time value in millis?
-            tzRoundingBuilder = Rounding.builder(TimeValue.timeValueMillis(interval));
-        }
-        if (timeZone() != null) {
-            tzRoundingBuilder.timeZone(timeZone());
-        }
-        Rounding rounding = tzRoundingBuilder.build();
-        return rounding;
-    }
-
     @Override
     protected Aggregator doCreateInternal(ValuesSource.Numeric valuesSource, Aggregator parent, boolean collectsFromSingleBucket,
             List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException {
@@ -126,18 +76,7 @@ protected Aggregator doCreateInternal(ValuesSource.Numeric valuesSource, Aggrega
 
     private Aggregator createAggregator(ValuesSource.Numeric valuesSource, Aggregator parent, List<PipelineAggregator> pipelineAggregators,
             Map<String, Object> metaData) throws IOException {
-        Rounding rounding = createRounding();
-        // we need to round the bounds given by the user and we have to do it
-        // for every aggregator we create
-        // as the rounding is not necessarily an idempotent operation.
-        // todo we need to think of a better structure to the factory/agtor
-        // code so we won't need to do that
-        ExtendedBounds roundedBounds = null;
-        if (extendedBounds != null) {
-            // parse any string bounds to longs and round them
-            roundedBounds = extendedBounds.parseAndValidate(name, context.searchContext(), config.format()).round(rounding);
-        }
-        return new DateHistogramAggregator(name, factories, rounding, offset, order, keyed, minDocCount, roundedBounds, valuesSource,
+        return new DateHistogramAggregator(name, factories, rounding, offset, order, keyed, minDocCount, extendedBounds, valuesSource,
                 config.format(), context, parent, pipelineAggregators, metaData);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramParser.java
deleted file mode 100644
index 952a0e2568f31..0000000000000
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramParser.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.search.aggregations.bucket.histogram;
-
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentParser.Token;
-import org.elasticsearch.search.aggregations.support.AbstractValuesSourceParser.NumericValuesSourceParser;
-import org.elasticsearch.search.aggregations.support.XContentParseContext;
-import org.elasticsearch.search.aggregations.support.ValueType;
-import org.elasticsearch.search.aggregations.support.ValuesSourceType;
-
-import java.io.IOException;
-import java.util.Map;
-
-/**
- * A parser for date histograms. This translates json into a
- * {@link DateHistogramAggregationBuilder} instance.
- */
-public class DateHistogramParser extends NumericValuesSourceParser {
-
-    public DateHistogramParser() {
-        super(true, true, true);
-    }
-
-    @Override
-    protected DateHistogramAggregationBuilder createFactory(String aggregationName, ValuesSourceType valuesSourceType,
-                                                            ValueType targetValueType, Map<ParseField, Object> otherOptions) {
-        DateHistogramAggregationBuilder factory = new DateHistogramAggregationBuilder(aggregationName);
-        Object interval = otherOptions.get(Histogram.INTERVAL_FIELD);
-        if (interval == null) {
-            throw new ParsingException(null, "Missing required field [interval] for histogram aggregation [" + aggregationName + "]");
-        } else if (interval instanceof Long) {
-            factory.interval((Long) interval);
-        } else if (interval instanceof DateHistogramInterval) {
-            factory.dateHistogramInterval((DateHistogramInterval) interval);
-        } else {
-            throw new IllegalStateException("Unexpected interval class: " + interval.getClass());
-        }
-        Long offset = (Long) otherOptions.get(Histogram.OFFSET_FIELD);
-        if (offset != null) {
-            factory.offset(offset);
-        }
-
-        ExtendedBounds extendedBounds = (ExtendedBounds) otherOptions.get(ExtendedBounds.EXTENDED_BOUNDS_FIELD);
-        if (extendedBounds != null) {
-            factory.extendedBounds(extendedBounds);
-        }
-        Boolean keyed = (Boolean) otherOptions.get(Histogram.KEYED_FIELD);
-        if (keyed != null) {
-            factory.keyed(keyed);
-        }
-        Long minDocCount = (Long) otherOptions.get(Histogram.MIN_DOC_COUNT_FIELD);
-        if (minDocCount != null) {
-            factory.minDocCount(minDocCount);
-        }
-        InternalOrder order = (InternalOrder) otherOptions.get(Histogram.ORDER_FIELD);
-        if (order != null) {
-            factory.order(order);
-        }
-        return factory;
-    }
-
-    @Override
-    protected boolean token(String aggregationName, String currentFieldName, Token token,
-                            XContentParseContext context, Map<ParseField, Object> otherOptions) throws IOException {
-        XContentParser parser = context.getParser();
-        if (token.isValue()) {
-            if (context.matchField(currentFieldName, Histogram.INTERVAL_FIELD)) {
-                if (token == XContentParser.Token.VALUE_STRING) {
-                    otherOptions.put(Histogram.INTERVAL_FIELD, new DateHistogramInterval(parser.text()));
-                    return true;
-                } else {
-                    otherOptions.put(Histogram.INTERVAL_FIELD, parser.longValue());
-                    return true;
-                }
-            } else if (context.matchField(currentFieldName, Histogram.MIN_DOC_COUNT_FIELD)) {
-                otherOptions.put(Histogram.MIN_DOC_COUNT_FIELD, parser.longValue());
-                return true;
-            } else if (context.matchField(currentFieldName, Histogram.KEYED_FIELD)) {
-                otherOptions.put(Histogram.KEYED_FIELD, parser.booleanValue());
-                return true;
-            } else if (context.matchField(currentFieldName, Histogram.OFFSET_FIELD)) {
-                if (token == XContentParser.Token.VALUE_STRING) {
-                    otherOptions.put(Histogram.OFFSET_FIELD,
-                            DateHistogramAggregationBuilder.parseStringOffset(parser.text()));
-                    return true;
-                } else {
-                    otherOptions.put(Histogram.OFFSET_FIELD, parser.longValue());
-                    return true;
-                }
-            } else {
-                return false;
-            }
-        } else if (token == XContentParser.Token.START_OBJECT) {
-            if (context.matchField(currentFieldName, Histogram.ORDER_FIELD)) {
-                InternalOrder order = null;
-                while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                    if (token == XContentParser.Token.FIELD_NAME) {
-                        currentFieldName = parser.currentName();
-                    } else if (token == XContentParser.Token.VALUE_STRING) {
-                        String dir = parser.text();
-                        boolean asc = "asc".equals(dir);
-                        if (!asc && !"desc".equals(dir)) {
-                            throw new ParsingException(parser.getTokenLocation(), "Unknown order direction in aggregation ["
-                                    + aggregationName + "]: [" + dir
-                                    + "]. Should be either [asc] or [desc]");
-                        }
-                        order = resolveOrder(currentFieldName, asc);
-                    }
-                }
-                otherOptions.put(Histogram.ORDER_FIELD, order);
-                return true;
-            } else if (context.matchField(currentFieldName, ExtendedBounds.EXTENDED_BOUNDS_FIELD)) {
-                try {
-                    otherOptions.put(ExtendedBounds.EXTENDED_BOUNDS_FIELD,
-                            ExtendedBounds.PARSER.apply(parser, context::getParseFieldMatcher));
-                } catch (Exception e) {
-                    throw new ParsingException(parser.getTokenLocation(), "Error parsing [{}]", e, aggregationName);
-                }
-                return true;
-            } else {
-                return false;
-            }
-        } else {
-            return false;
-        }
-    }
-
-    static InternalOrder resolveOrder(String key, boolean asc) {
-        if ("_key".equals(key) || "_time".equals(key)) {
-            return (InternalOrder) (asc ? InternalOrder.KEY_ASC : InternalOrder.KEY_DESC);
-        }
-        if ("_count".equals(key)) {
-            return (InternalOrder) (asc ? InternalOrder.COUNT_ASC : InternalOrder.COUNT_DESC);
-        }
-        return new InternalOrder.Aggregation(key, asc);
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ExtendedBounds.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ExtendedBounds.java
index 46fae19e49f9d..eeba2994c8229 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ExtendedBounds.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ExtendedBounds.java
@@ -19,17 +19,17 @@
 
 package org.elasticsearch.search.aggregations.bucket.histogram;
 
+import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.rounding.Rounding;
-import org.elasticsearch.common.xcontent.AbstractObjectParser.NoContextParser;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentParser.Token;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.SearchParseException;
@@ -45,7 +45,7 @@ public class ExtendedBounds implements ToXContent, Writeable {
     static final ParseField MIN_FIELD = new ParseField("min");
     static final ParseField MAX_FIELD = new ParseField("max");
 
-    public static final ConstructingObjectParser<ExtendedBounds, ParseFieldMatcherSupplier> PARSER = new ConstructingObjectParser<>(
+    public static final ConstructingObjectParser<ExtendedBounds, Void> PARSER = new ConstructingObjectParser<>(
             "extended_bounds", a -> {
         assert a.length == 2;
         Long min = null;
@@ -73,7 +73,7 @@ public class ExtendedBounds implements ToXContent, Writeable {
         return new ExtendedBounds(min, max, minAsStr, maxAsStr);
     });
     static {
-        NoContextParser<Object> longOrString = p -> {
+        CheckedFunction<XContentParser, Object, IOException> longOrString = p -> {
             if (p.currentToken() == Token.VALUE_NUMBER) {
                 return p.longValue(false);
             }
@@ -153,11 +153,11 @@ ExtendedBounds parseAndValidate(String aggName, SearchContext context, DocValueF
         Long max = this.max;
         assert format != null;
         if (minAsStr != null) {
-            min = format.parseLong(minAsStr, false, context::nowInMillis);
+            min = format.parseLong(minAsStr, false, context.getQueryShardContext()::nowInMillis);
         }
         if (maxAsStr != null) {
             // TODO: Should we rather pass roundUp=true?
-            max = format.parseLong(maxAsStr, false, context::nowInMillis);
+            max = format.parseLong(maxAsStr, false, context.getQueryShardContext()::nowInMillis);
         }
         if (min != null && max != null && min.compareTo(max) > 0) {
             throw new SearchParseException(context, "[extended_bounds.min][" + min + "] cannot be greater than " +
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/Histogram.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/Histogram.java
index 9453ecef59690..3ac87de81ed23 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/Histogram.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/Histogram.java
@@ -48,8 +48,7 @@ interface Bucket extends MultiBucketsAggregation.Bucket {
      * @return  The buckets of this histogram (each bucket representing an interval in the histogram)
      */
     @Override
-    List<Bucket> getBuckets();
-
+    List<? extends Bucket> getBuckets();
 
     /**
      * A strategy defining the order in which the buckets in this histogram are ordered.
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregationBuilder.java
index 03ff1ee935a85..87c7404c088ba 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregationBuilder.java
@@ -19,19 +19,26 @@
 
 package org.elasticsearch.search.aggregations.bucket.histogram;
 
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParser.Token;
+import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -41,7 +48,40 @@
  */
 public class HistogramAggregationBuilder
         extends ValuesSourceAggregationBuilder<ValuesSource.Numeric, HistogramAggregationBuilder> {
-    public static final String NAME = InternalHistogram.TYPE.name();
+    public static final String NAME = "histogram";
+
+    private static final ObjectParser<double[], Void> EXTENDED_BOUNDS_PARSER = new ObjectParser<>(
+            Histogram.EXTENDED_BOUNDS_FIELD.getPreferredName(),
+            () -> new double[]{ Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY });
+    static {
+        EXTENDED_BOUNDS_PARSER.declareDouble((bounds, d) -> bounds[0] = d, new ParseField("min"));
+        EXTENDED_BOUNDS_PARSER.declareDouble((bounds, d) -> bounds[1] = d, new ParseField("max"));
+    }
+
+    private static final ObjectParser<HistogramAggregationBuilder, QueryParseContext> PARSER;
+    static {
+        PARSER = new ObjectParser<>(HistogramAggregationBuilder.NAME);
+        ValuesSourceParserHelper.declareNumericFields(PARSER, true, true, false);
+
+        PARSER.declareDouble(HistogramAggregationBuilder::interval, Histogram.INTERVAL_FIELD);
+
+        PARSER.declareDouble(HistogramAggregationBuilder::offset, Histogram.OFFSET_FIELD);
+
+        PARSER.declareBoolean(HistogramAggregationBuilder::keyed, Histogram.KEYED_FIELD);
+
+        PARSER.declareLong(HistogramAggregationBuilder::minDocCount, Histogram.MIN_DOC_COUNT_FIELD);
+
+        PARSER.declareField((histogram, extendedBounds) -> {
+            histogram.extendedBounds(extendedBounds[0], extendedBounds[1]);
+        }, parser -> EXTENDED_BOUNDS_PARSER.apply(parser, null), ExtendedBounds.EXTENDED_BOUNDS_FIELD, ObjectParser.ValueType.OBJECT);
+
+        PARSER.declareField(HistogramAggregationBuilder::order, HistogramAggregationBuilder::parseOrder,
+                Histogram.ORDER_FIELD, ObjectParser.ValueType.OBJECT);
+    }
+
+    public static HistogramAggregationBuilder parse(String aggregationName, QueryParseContext context) throws IOException {
+        return PARSER.parse(context.parser(), new HistogramAggregationBuilder(aggregationName), context);
+    }
 
     private double interval;
     private double offset = 0;
@@ -53,12 +93,12 @@ public class HistogramAggregationBuilder
 
     /** Create a new builder with the given name. */
     public HistogramAggregationBuilder(String name) {
-        super(name, InternalHistogram.TYPE, ValuesSourceType.NUMERIC, ValueType.DOUBLE);
+        super(name, ValuesSourceType.NUMERIC, ValueType.DOUBLE);
     }
 
     /** Read from a stream, for internal use only. */
     public HistogramAggregationBuilder(StreamInput in) throws IOException {
-        super(in, InternalHistogram.TYPE, ValuesSourceType.NUMERIC, ValueType.DOUBLE);
+        super(in, ValuesSourceType.NUMERIC, ValueType.DOUBLE);
         if (in.readBoolean()) {
             order = InternalOrder.Streams.readOrder(in);
         }
@@ -219,14 +259,14 @@ protected XContentBuilder doXContentBody(XContentBuilder builder, Params params)
     }
 
     @Override
-    public String getWriteableName() {
-        return InternalHistogram.TYPE.name();
+    public String getType() {
+        return NAME;
     }
 
     @Override
-    protected ValuesSourceAggregatorFactory<Numeric, ?> innerBuild(AggregationContext context, ValuesSourceConfig<Numeric> config,
+    protected ValuesSourceAggregatorFactory<Numeric, ?> innerBuild(SearchContext context, ValuesSourceConfig<Numeric> config,
             AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
-        return new HistogramAggregatorFactory(name, type, config, interval, offset, order, keyed, minDocCount, minBound, maxBound,
+        return new HistogramAggregatorFactory(name, config, interval, offset, order, keyed, minDocCount, minBound, maxBound,
                 context, parent, subFactoriesBuilder, metaData);
     }
 
@@ -246,4 +286,34 @@ protected boolean innerEquals(Object obj) {
                 && Objects.equals(minBound, other.minBound)
                 && Objects.equals(maxBound, other.maxBound);
     }
+
+    private static InternalOrder parseOrder(XContentParser parser, QueryParseContext context) throws IOException {
+        InternalOrder order = null;
+        Token token;
+        String currentFieldName = null;
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token == XContentParser.Token.VALUE_STRING) {
+                String dir = parser.text();
+                boolean asc = "asc".equals(dir);
+                if (!asc && !"desc".equals(dir)) {
+                    throw new ParsingException(parser.getTokenLocation(), "Unknown order direction: [" + dir
+                            + "]. Should be either [asc] or [desc]");
+                }
+                order = resolveOrder(currentFieldName, asc);
+            }
+        }
+        return order;
+    }
+
+    static InternalOrder resolveOrder(String key, boolean asc) {
+        if ("_key".equals(key)) {
+            return (InternalOrder) (asc ? InternalOrder.KEY_ASC : InternalOrder.KEY_DESC);
+        }
+        if ("_count".equals(key)) {
+            return (InternalOrder) (asc ? InternalOrder.COUNT_ASC : InternalOrder.COUNT_DESC);
+        }
+        return new InternalOrder.Aggregation(key, asc);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregator.java
index 7d102578a720c..4b547989d8bce 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregator.java
@@ -34,8 +34,8 @@
 import org.elasticsearch.search.aggregations.bucket.BucketsAggregator;
 import org.elasticsearch.search.aggregations.bucket.histogram.InternalHistogram.EmptyBucketInfo;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -61,13 +61,13 @@ class HistogramAggregator extends BucketsAggregator {
 
     private final LongHash bucketOrds;
 
-    public HistogramAggregator(String name, AggregatorFactories factories, double interval, double offset,
+    HistogramAggregator(String name, AggregatorFactories factories, double interval, double offset,
             InternalOrder order, boolean keyed, long minDocCount, double minBound, double maxBound,
             @Nullable ValuesSource.Numeric valuesSource, DocValueFormat formatter,
-            AggregationContext aggregationContext, Aggregator parent,
+            SearchContext context, Aggregator parent,
             List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException {
 
-        super(name, factories, aggregationContext, parent, pipelineAggregators, metaData);
+        super(name, factories, context, parent, pipelineAggregators, metaData);
         if (interval <= 0) {
             throw new IllegalArgumentException("interval must be positive, got: " + interval);
         }
@@ -81,7 +81,7 @@ public HistogramAggregator(String name, AggregatorFactories factories, double in
         this.valuesSource = valuesSource;
         this.formatter = formatter;
 
-        bucketOrds = new LongHash(1, aggregationContext.bigArrays());
+        bucketOrds = new LongHash(1, context.bigArrays());
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregatorFactory.java
index 805aab9ecf5df..939210b63a699 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregatorFactory.java
@@ -22,13 +22,12 @@
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
-import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -42,11 +41,11 @@ public final class HistogramAggregatorFactory extends ValuesSourceAggregatorFact
     private final long minDocCount;
     private final double minBound, maxBound;
 
-    HistogramAggregatorFactory(String name, Type type, ValuesSourceConfig<Numeric> config, double interval, double offset,
+    HistogramAggregatorFactory(String name, ValuesSourceConfig<Numeric> config, double interval, double offset,
             InternalOrder order, boolean keyed, long minDocCount, double minBound, double maxBound,
-            AggregationContext context, AggregatorFactory<?> parent,
+            SearchContext context, AggregatorFactory<?> parent,
             AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.interval = interval;
         this.offset = offset;
         this.order = order;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramParser.java
deleted file mode 100644
index f27677a1a660f..0000000000000
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramParser.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.search.aggregations.bucket.histogram;
-
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
-import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.xcontent.ObjectParser;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentParser.Token;
-import org.elasticsearch.search.aggregations.support.AbstractValuesSourceParser.NumericValuesSourceParser;
-import org.elasticsearch.search.aggregations.support.XContentParseContext;
-import org.elasticsearch.search.aggregations.support.ValueType;
-import org.elasticsearch.search.aggregations.support.ValuesSourceType;
-
-import java.io.IOException;
-import java.util.Map;
-
-/**
- * A parser for date histograms. This translates json into an
- * {@link HistogramAggregationBuilder} instance.
- */
-public class HistogramParser extends NumericValuesSourceParser {
-
-    private static final ObjectParser<double[], ParseFieldMatcherSupplier> EXTENDED_BOUNDS_PARSER = new ObjectParser<>(
-            Histogram.EXTENDED_BOUNDS_FIELD.getPreferredName(),
-            () -> new double[]{ Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY });
-    static {
-        EXTENDED_BOUNDS_PARSER.declareDouble((bounds, d) -> bounds[0] = d, new ParseField("min"));
-        EXTENDED_BOUNDS_PARSER.declareDouble((bounds, d) -> bounds[1] = d, new ParseField("max"));
-    }
-
-    public HistogramParser() {
-        super(true, true, false);
-    }
-
-    @Override
-    protected HistogramAggregationBuilder createFactory(String aggregationName, ValuesSourceType valuesSourceType,
-            ValueType targetValueType, Map<ParseField, Object> otherOptions) {
-        HistogramAggregationBuilder factory = new HistogramAggregationBuilder(aggregationName);
-        Double interval = (Double) otherOptions.get(Histogram.INTERVAL_FIELD);
-        if (interval == null) {
-            throw new ParsingException(null, "Missing required field [interval] for histogram aggregation [" + aggregationName + "]");
-        } else {
-            factory.interval(interval);
-        }
-        Double offset = (Double) otherOptions.get(Histogram.OFFSET_FIELD);
-        if (offset != null) {
-            factory.offset(offset);
-        }
-
-        double[] extendedBounds = (double[]) otherOptions.get(Histogram.EXTENDED_BOUNDS_FIELD);
-        if (extendedBounds != null) {
-            factory.extendedBounds(extendedBounds[0], extendedBounds[1]);
-        }
-        Boolean keyed = (Boolean) otherOptions.get(Histogram.KEYED_FIELD);
-        if (keyed != null) {
-            factory.keyed(keyed);
-        }
-        Long minDocCount = (Long) otherOptions.get(Histogram.MIN_DOC_COUNT_FIELD);
-        if (minDocCount != null) {
-            factory.minDocCount(minDocCount);
-        }
-        InternalOrder order = (InternalOrder) otherOptions.get(Histogram.ORDER_FIELD);
-        if (order != null) {
-            factory.order(order);
-        }
-        return factory;
-    }
-
-    @Override
-    protected boolean token(String aggregationName, String currentFieldName, Token token,
-                            XContentParseContext context, Map<ParseField, Object> otherOptions) throws IOException {
-        XContentParser parser = context.getParser();
-        if (token.isValue()) {
-            if (context.matchField(currentFieldName, Histogram.INTERVAL_FIELD)) {
-                otherOptions.put(Histogram.INTERVAL_FIELD, parser.doubleValue());
-                return true;
-            } else if (context.matchField(currentFieldName, Histogram.MIN_DOC_COUNT_FIELD)) {
-                otherOptions.put(Histogram.MIN_DOC_COUNT_FIELD, parser.longValue());
-                return true;
-            } else if (context.matchField(currentFieldName, Histogram.KEYED_FIELD)) {
-                otherOptions.put(Histogram.KEYED_FIELD, parser.booleanValue());
-                return true;
-            } else if (context.matchField(currentFieldName, Histogram.OFFSET_FIELD)) {
-                otherOptions.put(Histogram.OFFSET_FIELD, parser.doubleValue());
-                return true;
-            } else {
-                return false;
-            }
-        } else if (token == XContentParser.Token.START_OBJECT) {
-            if (context.matchField(currentFieldName, Histogram.ORDER_FIELD)) {
-                InternalOrder order = null;
-                while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                    if (token == XContentParser.Token.FIELD_NAME) {
-                        currentFieldName = parser.currentName();
-                    } else if (token == XContentParser.Token.VALUE_STRING) {
-                        String dir = parser.text();
-                        boolean asc = "asc".equals(dir);
-                        if (!asc && !"desc".equals(dir)) {
-                            throw new ParsingException(parser.getTokenLocation(), "Unknown order direction in aggregation ["
-                                    + aggregationName + "]: [" + dir
-                                    + "]. Should be either [asc] or [desc]");
-                        }
-                        order = resolveOrder(currentFieldName, asc);
-                    }
-                }
-                otherOptions.put(Histogram.ORDER_FIELD, order);
-                return true;
-            } else if (context.matchField(currentFieldName, Histogram.EXTENDED_BOUNDS_FIELD)) {
-                double[] bounds = EXTENDED_BOUNDS_PARSER.apply(parser, context::getParseFieldMatcher);
-                otherOptions.put(Histogram.EXTENDED_BOUNDS_FIELD, bounds);
-                return true;
-            } else {
-                return false;
-            }
-        } else {
-            return false;
-        }
-    }
-
-    static InternalOrder resolveOrder(String key, boolean asc) {
-        if ("_key".equals(key)) {
-            return (InternalOrder) (asc ? InternalOrder.KEY_ASC : InternalOrder.KEY_DESC);
-        }
-        if ("_count".equals(key)) {
-            return (InternalOrder) (asc ? InternalOrder.COUNT_ASC : InternalOrder.COUNT_DESC);
-        }
-        return new InternalOrder.Aggregation(key, asc);
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java
index 56d3792e0c6b3..dc876b50c97a6 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java
@@ -43,13 +43,11 @@
 import java.util.Map;
 
 /**
- * Imelementation of {@link Histogram}.
+ * Implementation of {@link Histogram}.
  */
 public final class InternalDateHistogram extends InternalMultiBucketAggregation<InternalDateHistogram, InternalDateHistogram.Bucket>
         implements Histogram, HistogramFactory {
 
-    static final Type TYPE = new Type("date_histogram");
-
     public static class Bucket extends InternalMultiBucketAggregation.InternalBucket implements Histogram.Bucket {
 
         final long key;
@@ -125,10 +123,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                 builder.startObject();
             }
             if (format != DocValueFormat.RAW) {
-                builder.field(CommonFields.KEY_AS_STRING, keyAsString);
+                builder.field(CommonFields.KEY_AS_STRING.getPreferredName(), keyAsString);
             }
-            builder.field(CommonFields.KEY, key);
-            builder.field(CommonFields.DOC_COUNT, docCount);
+            builder.field(CommonFields.KEY.getPreferredName(), key);
+            builder.field(CommonFields.DOC_COUNT.getPreferredName(), docCount);
             aggregations.toXContentInternal(builder, params);
             builder.endObject();
             return builder;
@@ -233,7 +231,7 @@ public String getWriteableName() {
     }
 
     @Override
-    public List<Histogram.Bucket> getBuckets() {
+    public List<InternalDateHistogram.Bucket> getBuckets() {
         return Collections.unmodifiableList(buckets);
     }
 
@@ -287,7 +285,7 @@ protected boolean lessThan(IteratorAndCurrent a, IteratorAndCurrent b) {
                 if (top.current.key != key) {
                     // the key changes, reduce what we already buffered and reset the buffer for current buckets
                     final Bucket reduced = currentBuckets.get(0).reduce(currentBuckets, reduceContext);
-                    if (reduced.getDocCount() >= minDocCount) {
+                    if (reduced.getDocCount() >= minDocCount || reduceContext.isFinalReduce() == false) {
                         reducedBuckets.add(reduced);
                     }
                     currentBuckets.clear();
@@ -308,7 +306,7 @@ protected boolean lessThan(IteratorAndCurrent a, IteratorAndCurrent b) {
 
             if (currentBuckets.isEmpty() == false) {
                 final Bucket reduced = currentBuckets.get(0).reduce(currentBuckets, reduceContext);
-                if (reduced.getDocCount() >= minDocCount) {
+                if (reduced.getDocCount() >= minDocCount || reduceContext.isFinalReduce() == false) {
                     reducedBuckets.add(reduced);
                 }
             }
@@ -329,8 +327,8 @@ private void addEmptyBuckets(List<Bucket> list, ReduceContext reduceContext) {
             Bucket firstBucket = iter.hasNext() ? list.get(iter.nextIndex()) : null;
             if (firstBucket == null) {
                 if (bounds.getMin() != null && bounds.getMax() != null) {
-                    long key = bounds.getMin();
-                    long max = bounds.getMax();
+                    long key = bounds.getMin() + offset;
+                    long max = bounds.getMax() + offset;
                     while (key <= max) {
                         iter.add(new InternalDateHistogram.Bucket(key, 0, keyed, format, reducedEmptySubAggs));
                         key = nextKey(key).longValue();
@@ -338,7 +336,7 @@ private void addEmptyBuckets(List<Bucket> list, ReduceContext reduceContext) {
                 }
             } else {
                 if (bounds.getMin() != null) {
-                    long key = bounds.getMin();
+                    long key = bounds.getMin() + offset;
                     if (key < firstBucket.key) {
                         while (key < firstBucket.key) {
                             iter.add(new InternalDateHistogram.Bucket(key, 0, keyed, format, reducedEmptySubAggs));
@@ -365,12 +363,12 @@ private void addEmptyBuckets(List<Bucket> list, ReduceContext reduceContext) {
         }
 
         // finally, adding the empty buckets *after* the actual data (based on the extended_bounds.max requested by the user)
-        if (bounds != null && lastBucket != null && bounds.getMax() != null && bounds.getMax() > lastBucket.key) {
-            long key = emptyBucketInfo.rounding.nextRoundingValue(lastBucket.key);
-            long max = bounds.getMax();
+        if (bounds != null && lastBucket != null && bounds.getMax() != null && bounds.getMax() + offset > lastBucket.key) {
+            long key = nextKey(lastBucket.key).longValue();
+            long max = bounds.getMax() + offset;
             while (key <= max) {
                 iter.add(new InternalDateHistogram.Bucket(key, 0, keyed, format, reducedEmptySubAggs));
-                key = emptyBucketInfo.rounding.nextRoundingValue(key);
+                key = nextKey(key).longValue();
             }
         }
     }
@@ -384,7 +382,7 @@ public InternalAggregation doReduce(List<InternalAggregation> aggregations, Redu
             addEmptyBuckets(reducedBuckets, reduceContext);
         }
 
-        if (order == InternalOrder.KEY_ASC) {
+        if (order == InternalOrder.KEY_ASC || reduceContext.isFinalReduce() == false) {
             // nothing to do, data are already sorted since shards return
             // sorted buckets and the merge-sort performed by reduceBuckets
             // maintains order
@@ -405,9 +403,9 @@ public InternalAggregation doReduce(List<InternalAggregation> aggregations, Redu
     @Override
     public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
         if (keyed) {
-            builder.startObject(CommonFields.BUCKETS);
+            builder.startObject(CommonFields.BUCKETS.getPreferredName());
         } else {
-            builder.startArray(CommonFields.BUCKETS);
+            builder.startArray(CommonFields.BUCKETS.getPreferredName());
         }
         for (Bucket bucket : buckets) {
             bucket.toXContent(builder, params);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogram.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogram.java
index 4dae51533db3b..a77217ec98035 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogram.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogram.java
@@ -40,13 +40,10 @@
 import java.util.Map;
 
 /**
- * Imelementation of {@link Histogram}.
+ * Implementation of {@link Histogram}.
  */
 public final class InternalHistogram extends InternalMultiBucketAggregation<InternalHistogram, InternalHistogram.Bucket>
         implements Histogram, HistogramFactory {
-
-    static final Type TYPE = new Type("histogram");
-
     public static class Bucket extends InternalMultiBucketAggregation.InternalBucket implements Histogram.Bucket {
 
         final double key;
@@ -122,10 +119,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                 builder.startObject();
             }
             if (format != DocValueFormat.RAW) {
-                builder.field(CommonFields.KEY_AS_STRING, keyAsString);
+                builder.field(CommonFields.KEY_AS_STRING.getPreferredName(), keyAsString);
             }
-            builder.field(CommonFields.KEY, key);
-            builder.field(CommonFields.DOC_COUNT, docCount);
+            builder.field(CommonFields.KEY.getPreferredName(), key);
+            builder.field(CommonFields.DOC_COUNT.getPreferredName(), docCount);
             aggregations.toXContentInternal(builder, params);
             builder.endObject();
             return builder;
@@ -222,7 +219,7 @@ public String getWriteableName() {
     }
 
     @Override
-    public List<Histogram.Bucket> getBuckets() {
+    public List<InternalHistogram.Bucket> getBuckets() {
         return Collections.unmodifiableList(buckets);
     }
 
@@ -272,10 +269,11 @@ protected boolean lessThan(IteratorAndCurrent a, IteratorAndCurrent b) {
             do {
                 final IteratorAndCurrent top = pq.top();
 
-                if (top.current.key != key) {
-                    // the key changes, reduce what we already buffered and reset the buffer for current buckets
+                if (Double.compare(top.current.key, key) != 0) {
+                    // The key changes, reduce what we already buffered and reset the buffer for current buckets.
+                    // Using Double.compare instead of != to handle NaN correctly.
                     final Bucket reduced = currentBuckets.get(0).reduce(currentBuckets, reduceContext);
-                    if (reduced.getDocCount() >= minDocCount) {
+                    if (reduced.getDocCount() >= minDocCount || reduceContext.isFinalReduce() == false) {
                         reducedBuckets.add(reduced);
                     }
                     currentBuckets.clear();
@@ -286,7 +284,7 @@ protected boolean lessThan(IteratorAndCurrent a, IteratorAndCurrent b) {
 
                 if (top.iterator.hasNext()) {
                     final Bucket next = top.iterator.next();
-                    assert next.key > top.current.key : "shards must return data sorted by key";
+                    assert Double.compare(next.key, top.current.key) > 0 : "shards must return data sorted by key";
                     top.current = next;
                     pq.updateTop();
                 } else {
@@ -296,7 +294,7 @@ protected boolean lessThan(IteratorAndCurrent a, IteratorAndCurrent b) {
 
             if (currentBuckets.isEmpty() == false) {
                 final Bucket reduced = currentBuckets.get(0).reduce(currentBuckets, reduceContext);
-                if (reduced.getDocCount() >= minDocCount) {
+                if (reduced.getDocCount() >= minDocCount || reduceContext.isFinalReduce() == false) {
                     reducedBuckets.add(reduced);
                 }
             }
@@ -367,7 +365,7 @@ public InternalAggregation doReduce(List<InternalAggregation> aggregations, Redu
             addEmptyBuckets(reducedBuckets, reduceContext);
         }
 
-        if (order == InternalOrder.KEY_ASC) {
+        if (order == InternalOrder.KEY_ASC || reduceContext.isFinalReduce() == false) {
             // nothing to do, data are already sorted since shards return
             // sorted buckets and the merge-sort performed by reduceBuckets
             // maintains order
@@ -388,9 +386,9 @@ public InternalAggregation doReduce(List<InternalAggregation> aggregations, Redu
     @Override
     public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
         if (keyed) {
-            builder.startObject(CommonFields.BUCKETS);
+            builder.startObject(CommonFields.BUCKETS.getPreferredName());
         } else {
-            builder.startArray(CommonFields.BUCKETS);
+            builder.startArray(CommonFields.BUCKETS.getPreferredName());
         }
         for (Bucket bucket : buckets) {
             bucket.toXContent(builder, params);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ParsedDateHistogram.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ParsedDateHistogram.java
new file mode 100644
index 0000000000000..ace0cb59907a8
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ParsedDateHistogram.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.histogram;
+
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+
+import java.io.IOException;
+import java.util.List;
+
+public class ParsedDateHistogram extends ParsedMultiBucketAggregation<ParsedDateHistogram.ParsedBucket> implements Histogram {
+
+    @Override
+    public String getType() {
+        return DateHistogramAggregationBuilder.NAME;
+    }
+
+    @Override
+    public List<? extends Histogram.Bucket> getBuckets() {
+        return buckets;
+    }
+
+    private static ObjectParser<ParsedDateHistogram, Void> PARSER =
+            new ObjectParser<>(ParsedDateHistogram.class.getSimpleName(), true, ParsedDateHistogram::new);
+    static {
+        declareMultiBucketAggregationFields(PARSER,
+                parser -> ParsedBucket.fromXContent(parser, false),
+                parser -> ParsedBucket.fromXContent(parser, true));
+    }
+
+    public static ParsedDateHistogram fromXContent(XContentParser parser, String name) throws IOException {
+        ParsedDateHistogram aggregation = PARSER.parse(parser, null);
+        aggregation.setName(name);
+        return aggregation;
+    }
+
+    public static class ParsedBucket extends ParsedMultiBucketAggregation.ParsedBucket implements Histogram.Bucket {
+
+        private Long key;
+
+        @Override
+        public Object getKey() {
+            if (key != null) {
+                return new DateTime(key, DateTimeZone.UTC);
+            }
+            return null;
+        }
+
+        @Override
+        public String getKeyAsString() {
+            String keyAsString = super.getKeyAsString();
+            if (keyAsString != null) {
+                return keyAsString;
+            }
+            if (key != null) {
+                return Long.toString(key);
+            }
+            return null;
+        }
+
+        @Override
+        protected XContentBuilder keyToXContent(XContentBuilder builder) throws IOException {
+            return builder.field(CommonFields.KEY.getPreferredName(), key);
+        }
+
+        static ParsedBucket fromXContent(XContentParser parser, boolean keyed) throws IOException {
+            return parseXContent(parser, keyed, ParsedBucket::new, (p, bucket) -> bucket.key = p.longValue());
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ParsedHistogram.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ParsedHistogram.java
new file mode 100644
index 0000000000000..6037c1558867a
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ParsedHistogram.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.histogram;
+
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation;
+
+import java.io.IOException;
+import java.util.List;
+
+public class ParsedHistogram extends ParsedMultiBucketAggregation<ParsedHistogram.ParsedBucket> implements Histogram {
+
+    @Override
+    public String getType() {
+        return HistogramAggregationBuilder.NAME;
+    }
+
+    @Override
+    public List<? extends Histogram.Bucket> getBuckets() {
+        return buckets;
+    }
+
+    private static ObjectParser<ParsedHistogram, Void> PARSER =
+            new ObjectParser<>(ParsedHistogram.class.getSimpleName(), true, ParsedHistogram::new);
+    static {
+        declareMultiBucketAggregationFields(PARSER,
+                parser -> ParsedBucket.fromXContent(parser, false),
+                parser -> ParsedBucket.fromXContent(parser, true));
+    }
+
+    public static ParsedHistogram fromXContent(XContentParser parser, String name) throws IOException {
+        ParsedHistogram aggregation = PARSER.parse(parser, null);
+        aggregation.setName(name);
+        return aggregation;
+    }
+
+    static class ParsedBucket extends ParsedMultiBucketAggregation.ParsedBucket implements Histogram.Bucket {
+
+        private Double key;
+
+        @Override
+        public Object getKey() {
+            return key;
+        }
+
+        @Override
+        public String getKeyAsString() {
+            String keyAsString = super.getKeyAsString();
+            if (keyAsString != null) {
+                return keyAsString;
+            }
+            if (key != null) {
+                return Double.toString(key);
+            }
+            return null;
+        }
+
+        static ParsedBucket fromXContent(XContentParser parser, boolean keyed) throws IOException {
+            return parseXContent(parser, keyed, ParsedBucket::new, (p, bucket) -> bucket.key = p.doubleValue());
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregationBuilder.java
index 09916acfdf8a0..9361acc8fcb4f 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregationBuilder.java
@@ -21,33 +21,44 @@
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 
 public class MissingAggregationBuilder extends ValuesSourceAggregationBuilder<ValuesSource, MissingAggregationBuilder> {
     public static final String NAME = "missing";
-    public static final Type TYPE = new Type(NAME);
+
+    private static final ObjectParser<MissingAggregationBuilder, QueryParseContext> PARSER;
+    static {
+        PARSER = new ObjectParser<>(MissingAggregationBuilder.NAME);
+        ValuesSourceParserHelper.declareAnyFields(PARSER, true, true);
+    }
+
+    public static MissingAggregationBuilder parse(String aggregationName, QueryParseContext context) throws IOException {
+        return PARSER.parse(context.parser(), new MissingAggregationBuilder(aggregationName, null), context);
+    }
 
     public MissingAggregationBuilder(String name, ValueType targetValueType) {
-        super(name, TYPE, ValuesSourceType.ANY, targetValueType);
+        super(name, ValuesSourceType.ANY, targetValueType);
     }
 
     /**
      * Read from a stream.
      */
     public MissingAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE, ValuesSourceType.ANY);
+        super(in, ValuesSourceType.ANY);
     }
 
     @Override
@@ -61,9 +72,9 @@ protected boolean serializeTargetValueType() {
     }
 
     @Override
-    protected ValuesSourceAggregatorFactory<ValuesSource, ?> innerBuild(AggregationContext context,
+    protected ValuesSourceAggregatorFactory<ValuesSource, ?> innerBuild(SearchContext context,
             ValuesSourceConfig<ValuesSource> config, AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
-        return new MissingAggregatorFactory(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        return new MissingAggregatorFactory(name, config, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
@@ -82,7 +93,7 @@ protected boolean innerEquals(Object obj) {
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregator.java
index e80be56f34199..c0be6b2bfb5c1 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregator.java
@@ -27,8 +27,8 @@
 import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
 import org.elasticsearch.search.aggregations.bucket.SingleBucketAggregator;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -42,7 +42,7 @@ public class MissingAggregator extends SingleBucketAggregator {
     private final ValuesSource valuesSource;
 
     public MissingAggregator(String name, AggregatorFactories factories, ValuesSource valuesSource,
-            AggregationContext aggregationContext, Aggregator parent, List<PipelineAggregator> pipelineAggregators,
+            SearchContext aggregationContext, Aggregator parent, List<PipelineAggregator> pipelineAggregators,
             Map<String, Object> metaData) throws IOException {
         super(name, factories, aggregationContext, parent, pipelineAggregators, metaData);
         this.valuesSource = valuesSource;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorFactory.java
index d7b478b78f8fd..5d05b79cef2d6 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorFactory.java
@@ -22,12 +22,11 @@
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -35,9 +34,9 @@
 
 public class MissingAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource, MissingAggregatorFactory> {
 
-    public MissingAggregatorFactory(String name, Type type, ValuesSourceConfig<ValuesSource> config, AggregationContext context,
+    public MissingAggregatorFactory(String name, ValuesSourceConfig<ValuesSource> config, SearchContext context,
             AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingParser.java
deleted file mode 100644
index 5d6844ebbd25f..0000000000000
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingParser.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.search.aggregations.bucket.missing;
-
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.search.aggregations.support.AbstractValuesSourceParser.AnyValuesSourceParser;
-import org.elasticsearch.search.aggregations.support.XContentParseContext;
-import org.elasticsearch.search.aggregations.support.ValueType;
-import org.elasticsearch.search.aggregations.support.ValuesSourceType;
-
-import java.io.IOException;
-import java.util.Map;
-
-public class MissingParser extends AnyValuesSourceParser {
-
-    public MissingParser() {
-        super(true, true);
-    }
-
-    @Override
-    protected boolean token(String aggregationName, String currentFieldName, XContentParser.Token token,
-                            XContentParseContext context, Map<ParseField, Object> otherOptions) throws IOException {
-        return false;
-    }
-
-    @Override
-    protected MissingAggregationBuilder createFactory(String aggregationName, ValuesSourceType valuesSourceType,
-                                                      ValueType targetValueType, Map<ParseField, Object> otherOptions) {
-        return new MissingAggregationBuilder(aggregationName, targetValueType);
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/ParsedMissing.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/ParsedMissing.java
new file mode 100644
index 0000000000000..2897372df8954
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/ParsedMissing.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.bucket.missing;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.bucket.ParsedSingleBucketAggregation;
+
+import java.io.IOException;
+
+public class ParsedMissing extends ParsedSingleBucketAggregation implements Missing {
+
+    @Override
+    public String getType() {
+        return MissingAggregationBuilder.NAME;
+    }
+
+    public static ParsedMissing fromXContent(XContentParser parser, final String name) throws IOException {
+        return parseXContent(parser, new ParsedMissing(), name);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregationBuilder.java
index c2bf201e48f0e..f8f2602a474c6 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregationBuilder.java
@@ -30,15 +30,13 @@
 import org.elasticsearch.search.aggregations.AggregationExecutionException;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.Objects;
 
 public class NestedAggregationBuilder extends AbstractAggregationBuilder<NestedAggregationBuilder> {
     public static final String NAME = "nested";
-    private static final Type TYPE = new Type(NAME);
 
     private final String path;
 
@@ -50,7 +48,7 @@ public class NestedAggregationBuilder extends AbstractAggregationBuilder<NestedA
      *            match the path to a nested object in the mappings.
      */
     public NestedAggregationBuilder(String name, String path) {
-        super(name, TYPE);
+        super(name);
         if (path == null) {
             throw new IllegalArgumentException("[path] must not be null: [" + name + "]");
         }
@@ -61,7 +59,7 @@ public NestedAggregationBuilder(String name, String path) {
      * Read from a stream.
      */
     public NestedAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE);
+        super(in);
         path = in.readString();
     }
 
@@ -78,23 +76,23 @@ public String path() {
     }
 
     @Override
-    protected AggregatorFactory<?> doBuild(AggregationContext context, AggregatorFactory<?> parent, Builder subFactoriesBuilder)
+    protected AggregatorFactory<?> doBuild(SearchContext context, AggregatorFactory<?> parent, Builder subFactoriesBuilder)
             throws IOException {
-        ObjectMapper childObjectMapper = context.searchContext().getObjectMapper(path);
+        ObjectMapper childObjectMapper = context.getObjectMapper(path);
         if (childObjectMapper == null) {
             // in case the path has been unmapped:
-            return new NestedAggregatorFactory(name, type, null, null, context, parent, subFactoriesBuilder, metaData);
+            return new NestedAggregatorFactory(name, null, null, context, parent, subFactoriesBuilder, metaData);
         }
 
         if (childObjectMapper.nested().isNested() == false) {
             throw new AggregationExecutionException("[nested] nested path [" + path + "] is not nested");
         }
         try {
-            ObjectMapper parentObjectMapper = context.searchContext().getQueryShardContext().nestedScope().nextLevel(childObjectMapper);
-            return new NestedAggregatorFactory(name, type, parentObjectMapper, childObjectMapper, context, parent, subFactoriesBuilder,
+            ObjectMapper parentObjectMapper = context.getQueryShardContext().nestedScope().nextLevel(childObjectMapper);
+            return new NestedAggregatorFactory(name, parentObjectMapper, childObjectMapper, context, parent, subFactoriesBuilder,
                     metaData);
         } finally {
-            context.searchContext().getQueryShardContext().nestedScope().previousLevel();
+            context.getQueryShardContext().nestedScope().previousLevel();
         }
     }
 
@@ -116,7 +114,7 @@ public static NestedAggregationBuilder parse(String aggregationName, QueryParseC
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.VALUE_STRING) {
-                if (context.getParseFieldMatcher().match(currentFieldName, NestedAggregator.PATH_FIELD)) {
+                if (NestedAggregator.PATH_FIELD.match(currentFieldName)) {
                     path = parser.text();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
@@ -148,7 +146,7 @@ protected boolean doEquals(Object obj) {
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregator.java
index 448ea44e7ebf2..3df4b28993bbc 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregator.java
@@ -38,7 +38,7 @@
 import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
 import org.elasticsearch.search.aggregations.bucket.SingleBucketAggregator;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -52,11 +52,11 @@ public class NestedAggregator extends SingleBucketAggregator {
     private final Query childFilter;
 
     public NestedAggregator(String name, AggregatorFactories factories, ObjectMapper parentObjectMapper, ObjectMapper childObjectMapper,
-                            AggregationContext aggregationContext, Aggregator parentAggregator,
+            SearchContext context, Aggregator parentAggregator,
                             List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException {
-        super(name, factories, aggregationContext, parentAggregator, pipelineAggregators, metaData);
+        super(name, factories, context, parentAggregator, pipelineAggregators, metaData);
         Query parentFilter = parentObjectMapper != null ? parentObjectMapper.nestedTypeFilter() : Queries.newNonNestedFilter();
-        this.parentFilter = context.searchContext().bitsetFilterCache().getBitSetProducer(parentFilter);
+        this.parentFilter = context.bitsetFilterCache().getBitSetProducer(parentFilter);
         this.childFilter = childObjectMapper.nestedTypeFilter();
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorFactory.java
index b4e9fa05f708e..0ca0ef0a71e91 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorFactory.java
@@ -20,15 +20,13 @@
 package org.elasticsearch.search.aggregations.bucket.nested;
 
 import org.elasticsearch.index.mapper.ObjectMapper;
-import org.elasticsearch.search.aggregations.AggregationExecutionException;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.NonCollectingAggregator;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -39,10 +37,10 @@ public class NestedAggregatorFactory extends AggregatorFactory<NestedAggregatorF
     private final ObjectMapper parentObjectMapper;
     private final ObjectMapper childObjectMapper;
 
-    public NestedAggregatorFactory(String name, Type type, ObjectMapper parentObjectMapper, ObjectMapper childObjectMapper,
-                                   AggregationContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactories,
+    public NestedAggregatorFactory(String name, ObjectMapper parentObjectMapper, ObjectMapper childObjectMapper,
+            SearchContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactories,
                                    Map<String, Object> metaData) throws IOException {
-        super(name, type, context, parent, subFactories, metaData);
+        super(name, context, parent, subFactories, metaData);
         this.parentObjectMapper = parentObjectMapper;
         this.childObjectMapper = childObjectMapper;
     }
@@ -61,7 +59,7 @@ public Aggregator createInternal(Aggregator parent, boolean collectsFromSingleBu
 
     private static final class Unmapped extends NonCollectingAggregator {
 
-        public Unmapped(String name, AggregationContext context, Aggregator parent, List<PipelineAggregator> pipelineAggregators,
+        Unmapped(String name, SearchContext context, Aggregator parent, List<PipelineAggregator> pipelineAggregators,
                 Map<String, Object> metaData) throws IOException {
             super(name, context, parent, pipelineAggregators, metaData);
         }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ParsedNested.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ParsedNested.java
new file mode 100644
index 0000000000000..f241675678cfe
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ParsedNested.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.bucket.nested;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.bucket.ParsedSingleBucketAggregation;
+
+import java.io.IOException;
+
+public class ParsedNested extends ParsedSingleBucketAggregation implements Nested {
+
+    @Override
+    public String getType() {
+        return NestedAggregationBuilder.NAME;
+    }
+
+    public static ParsedNested fromXContent(XContentParser parser, final String name) throws IOException {
+        return parseXContent(parser, new ParsedNested(), name);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ParsedReverseNested.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ParsedReverseNested.java
new file mode 100644
index 0000000000000..dec15c3eded10
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ParsedReverseNested.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.bucket.nested;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.bucket.ParsedSingleBucketAggregation;
+
+import java.io.IOException;
+
+public class ParsedReverseNested extends ParsedSingleBucketAggregation implements Nested {
+
+    @Override
+    public String getType() {
+        return ReverseNestedAggregationBuilder.NAME;
+    }
+
+    public static ParsedReverseNested fromXContent(XContentParser parser, final String name) throws IOException {
+        return parseXContent(parser, new ParsedReverseNested(), name);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregationBuilder.java
index 707d46338454b..30bd72c6c6894 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregationBuilder.java
@@ -32,27 +32,25 @@
 import org.elasticsearch.search.aggregations.AggregationExecutionException;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.Objects;
 
 public class ReverseNestedAggregationBuilder extends AbstractAggregationBuilder<ReverseNestedAggregationBuilder> {
     public static final String NAME = "reverse_nested";
-    private static final Type TYPE = new Type(NAME);
 
     private String path;
 
     public ReverseNestedAggregationBuilder(String name) {
-        super(name, TYPE);
+        super(name);
     }
 
     /**
      * Read from a stream.
      */
     public ReverseNestedAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE);
+        super(in);
         path = in.readOptionalString();
     }
 
@@ -82,28 +80,28 @@ public String path() {
     }
 
     @Override
-    protected AggregatorFactory<?> doBuild(AggregationContext context, AggregatorFactory<?> parent, Builder subFactoriesBuilder)
+    protected AggregatorFactory<?> doBuild(SearchContext context, AggregatorFactory<?> parent, Builder subFactoriesBuilder)
             throws IOException {
         if (findNestedAggregatorFactory(parent) == null) {
-            throw new SearchParseException(context.searchContext(),
+            throw new SearchParseException(context,
                     "Reverse nested aggregation [" + name + "] can only be used inside a [nested] aggregation", null);
         }
 
         ObjectMapper parentObjectMapper = null;
         if (path != null) {
-            parentObjectMapper = context.searchContext().getObjectMapper(path);
+            parentObjectMapper = context.getObjectMapper(path);
             if (parentObjectMapper == null) {
-                return new ReverseNestedAggregatorFactory(name, type, true, null, context, parent, subFactoriesBuilder, metaData);
+                return new ReverseNestedAggregatorFactory(name, true, null, context, parent, subFactoriesBuilder, metaData);
             }
             if (parentObjectMapper.nested().isNested() == false) {
                 throw new AggregationExecutionException("[reverse_nested] nested path [" + path + "] is not nested");
             }
         }
 
-        NestedScope nestedScope = context.searchContext().getQueryShardContext().nestedScope();
+        NestedScope nestedScope = context.getQueryShardContext().nestedScope();
         try {
             nestedScope.nextLevel(parentObjectMapper);
-            return new ReverseNestedAggregatorFactory(name, type, false, parentObjectMapper, context, parent, subFactoriesBuilder,
+            return new ReverseNestedAggregatorFactory(name, false, parentObjectMapper, context, parent, subFactoriesBuilder,
                     metaData);
         } finally {
             nestedScope.previousLevel();
@@ -172,7 +170,7 @@ protected boolean doEquals(Object obj) {
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregator.java
index d45f103ed5e9c..f55dadaa8eb61 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregator.java
@@ -19,6 +19,7 @@
 package org.elasticsearch.search.aggregations.bucket.nested;
 
 import com.carrotsearch.hppc.LongIntHashMap;
+
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.Query;
@@ -34,7 +35,7 @@
 import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
 import org.elasticsearch.search.aggregations.bucket.SingleBucketAggregator;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -51,15 +52,15 @@ public class ReverseNestedAggregator extends SingleBucketAggregator {
     private final BitSetProducer parentBitsetProducer;
 
     public ReverseNestedAggregator(String name, AggregatorFactories factories, ObjectMapper objectMapper,
-            AggregationContext aggregationContext, Aggregator parent, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData)
+            SearchContext context, Aggregator parent, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData)
             throws IOException {
-        super(name, factories, aggregationContext, parent, pipelineAggregators, metaData);
+        super(name, factories, context, parent, pipelineAggregators, metaData);
         if (objectMapper == null) {
             parentFilter = Queries.newNonNestedFilter();
         } else {
             parentFilter = objectMapper.nestedTypeFilter();
         }
-        parentBitsetProducer = context.searchContext().bitsetFilterCache().getBitSetProducer(parentFilter);
+        parentBitsetProducer = context.bitsetFilterCache().getBitSetProducer(parentFilter);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorFactory.java
index b077e755bb7d3..3ed6f21d99c8d 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorFactory.java
@@ -20,16 +20,13 @@
 package org.elasticsearch.search.aggregations.bucket.nested;
 
 import org.elasticsearch.index.mapper.ObjectMapper;
-import org.elasticsearch.search.SearchParseException;
-import org.elasticsearch.search.aggregations.AggregationExecutionException;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.NonCollectingAggregator;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -40,11 +37,11 @@ public class ReverseNestedAggregatorFactory extends AggregatorFactory<ReverseNes
     private final boolean unmapped;
     private final ObjectMapper parentObjectMapper;
 
-    public ReverseNestedAggregatorFactory(String name, Type type, boolean unmapped, ObjectMapper parentObjectMapper,
-                                          AggregationContext context, AggregatorFactory<?> parent,
+    public ReverseNestedAggregatorFactory(String name, boolean unmapped, ObjectMapper parentObjectMapper,
+                                          SearchContext context, AggregatorFactory<?> parent,
                                           AggregatorFactories.Builder subFactories,
                                           Map<String, Object> metaData) throws IOException {
-        super(name, type, context, parent, subFactories, metaData);
+        super(name, context, parent, subFactories, metaData);
         this.unmapped = unmapped;
         this.parentObjectMapper = parentObjectMapper;
     }
@@ -61,7 +58,7 @@ public Aggregator createInternal(Aggregator parent, boolean collectsFromSingleBu
 
     private static final class Unmapped extends NonCollectingAggregator {
 
-        public Unmapped(String name, AggregationContext context, Aggregator parent, List<PipelineAggregator> pipelineAggregators,
+        Unmapped(String name, SearchContext context, Aggregator parent, List<PipelineAggregator> pipelineAggregators,
                 Map<String, Object> metaData) throws IOException {
             super(name, context, parent, pipelineAggregators, metaData);
         }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeAggregatorFactory.java
index f4103d87fbde3..156adcdc4f3ea 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeAggregatorFactory.java
@@ -22,15 +22,14 @@
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator.Range;
 import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator.Unmapped;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -40,13 +39,13 @@ public class AbstractRangeAggregatorFactory<AF extends AbstractRangeAggregatorFa
         extends ValuesSourceAggregatorFactory<ValuesSource.Numeric, AF> {
 
     private final InternalRange.Factory<?, ?> rangeFactory;
-    private final List<R> ranges;
+    private final R[] ranges;
     private final boolean keyed;
 
-    public AbstractRangeAggregatorFactory(String name, Type type, ValuesSourceConfig<Numeric> config, List<R> ranges, boolean keyed,
-            InternalRange.Factory<?, ?> rangeFactory, AggregationContext context, AggregatorFactory<?> parent,
+    public AbstractRangeAggregatorFactory(String name, ValuesSourceConfig<Numeric> config, R[] ranges, boolean keyed,
+            InternalRange.Factory<?, ?> rangeFactory, SearchContext context, AggregatorFactory<?> parent,
             AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.ranges = ranges;
         this.keyed = keyed;
         this.rangeFactory = rangeFactory;
@@ -55,7 +54,7 @@ public AbstractRangeAggregatorFactory(String name, Type type, ValuesSourceConfig
     @Override
     protected Aggregator createUnmapped(Aggregator parent, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData)
             throws IOException {
-        return new Unmapped<R>(name, ranges, keyed, config.format(), context, parent, rangeFactory, pipelineAggregators, metaData);
+        return new Unmapped<>(name, ranges, keyed, config.format(), context, parent, rangeFactory, pipelineAggregators, metaData);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeBuilder.java
index 13d10bd0a0ccf..635a0a6015c78 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeBuilder.java
@@ -19,13 +19,17 @@
 
 package org.elasticsearch.search.aggregations.bucket.range;
 
+import org.apache.lucene.util.InPlaceMergeSorter;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator.Range;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
+import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -40,7 +44,7 @@ public abstract class AbstractRangeBuilder<AB extends AbstractRangeBuilder<AB, R
     protected boolean keyed = false;
 
     protected AbstractRangeBuilder(String name, InternalRange.Factory<?, ?> rangeFactory) {
-        super(name, rangeFactory.type(), rangeFactory.getValueSourceType(), rangeFactory.getValueType());
+        super(name, rangeFactory.getValueSourceType(), rangeFactory.getValueType());
         this.rangeFactory = rangeFactory;
     }
 
@@ -49,12 +53,46 @@ protected AbstractRangeBuilder(String name, InternalRange.Factory<?, ?> rangeFac
      */
     protected AbstractRangeBuilder(StreamInput in, InternalRange.Factory<?, ?> rangeFactory, Writeable.Reader<R> rangeReader)
             throws IOException {
-        super(in, rangeFactory.type(), rangeFactory.getValueSourceType(), rangeFactory.getValueType());
+        super(in, rangeFactory.getValueSourceType(), rangeFactory.getValueType());
         this.rangeFactory = rangeFactory;
         ranges = in.readList(rangeReader);
         keyed = in.readBoolean();
     }
 
+    /**
+     * Resolve any strings in the ranges so we have a number value for the from
+     * and to of each range. The ranges are also sorted before being returned.
+     */
+    protected Range[] processRanges(SearchContext context, ValuesSourceConfig<Numeric> config) {
+        Range[] ranges = new Range[this.ranges.size()];
+        for (int i = 0; i < ranges.length; i++) {
+            ranges[i] = this.ranges.get(i).process(config.format(), context);
+        }
+        sortRanges(ranges);
+        return ranges;
+    }
+
+    private static void sortRanges(final Range[] ranges) {
+        new InPlaceMergeSorter() {
+
+            @Override
+            protected void swap(int i, int j) {
+                final Range tmp = ranges[i];
+                ranges[i] = ranges[j];
+                ranges[j] = tmp;
+            }
+
+            @Override
+            protected int compare(int i, int j) {
+                int cmp = Double.compare(ranges[i].from, ranges[j].from);
+                if (cmp == 0) {
+                    cmp = Double.compare(ranges[i].to, ranges[j].to);
+                }
+                return cmp;
+            }
+        }.sort(0, ranges.length);
+    }
+
     @Override
     protected void innerWriteTo(StreamOutput out) throws IOException {
         out.writeVInt(ranges.size());
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregator.java
index 77a6fc547b031..7ec3b3bb9c821 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregator.java
@@ -39,8 +39,8 @@
 import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
 import org.elasticsearch.search.aggregations.bucket.BucketsAggregator;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.internal.SearchContext;
 
 /** A range aggregator for values that are stored in SORTED_SET doc values. */
 public final class BinaryRangeAggregator extends BucketsAggregator {
@@ -78,10 +78,10 @@ private static int compare(BytesRef a, BytesRef b, int m) {
 
     public BinaryRangeAggregator(String name, AggregatorFactories factories,
             ValuesSource.Bytes valuesSource, DocValueFormat format,
-            List<Range> ranges, boolean keyed, AggregationContext aggregationContext,
+            List<Range> ranges, boolean keyed, SearchContext context,
             Aggregator parent, List<PipelineAggregator> pipelineAggregators,
             Map<String, Object> metaData) throws IOException {
-        super(name, factories, aggregationContext, parent, pipelineAggregators, metaData);
+        super(name, factories, context, parent, pipelineAggregators, metaData);
         this.valuesSource = valuesSource;
         this.format = format;
         this.keyed = keyed;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregatorFactory.java
index fda822cf11bf2..a8d74a7d83615 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregatorFactory.java
@@ -18,19 +18,18 @@
  */
 package org.elasticsearch.search.aggregations.bucket.range;
 
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.internal.SearchContext;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
 
 public class BinaryRangeAggregatorFactory
         extends ValuesSourceAggregatorFactory<ValuesSource.Bytes, BinaryRangeAggregatorFactory> {
@@ -38,13 +37,13 @@ public class BinaryRangeAggregatorFactory
     private final List<BinaryRangeAggregator.Range> ranges;
     private final boolean keyed;
 
-    public BinaryRangeAggregatorFactory(String name, Type type,
+    public BinaryRangeAggregatorFactory(String name,
             ValuesSourceConfig<ValuesSource.Bytes> config,
             List<BinaryRangeAggregator.Range> ranges, boolean keyed,
-            AggregationContext context,
+            SearchContext context,
             AggregatorFactory<?> parent, Builder subFactoriesBuilder,
             Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.ranges = ranges;
         this.keyed = keyed;
     }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java
index ed46062ee0662..c8b50d5b327fc 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java
@@ -42,6 +42,7 @@
 public final class InternalBinaryRange
         extends InternalMultiBucketAggregation<InternalBinaryRange, InternalBinaryRange.Bucket>
         implements Range {
+
     public static class Bucket extends InternalMultiBucketAggregation.InternalBucket implements Range.Bucket {
 
         private final transient DocValueFormat format;
@@ -131,16 +132,16 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             } else {
                 builder.startObject();
                 if (key != null) {
-                    builder.field(CommonFields.KEY, key);
+                    builder.field(CommonFields.KEY.getPreferredName(), key);
                 }
             }
             if (from != null) {
-                builder.field(CommonFields.FROM, getFrom());
+                builder.field(CommonFields.FROM.getPreferredName(), getFrom());
             }
             if (to != null) {
-                builder.field(CommonFields.TO, getTo());
+                builder.field(CommonFields.TO.getPreferredName(), getTo());
             }
-            builder.field(CommonFields.DOC_COUNT, docCount);
+            builder.field(CommonFields.DOC_COUNT.getPreferredName(), docCount);
             aggregations.toXContentInternal(builder, params);
             builder.endObject();
             return builder;
@@ -204,7 +205,7 @@ public String getWriteableName() {
     }
 
     @Override
-    public List<Range.Bucket> getBuckets() {
+    public List<InternalBinaryRange.Bucket> getBuckets() {
         return unmodifiableList(buckets);
     }
 
@@ -249,9 +250,9 @@ public InternalAggregation doReduce(List<InternalAggregation> aggregations, Redu
     public XContentBuilder doXContentBody(XContentBuilder builder,
             Params params) throws IOException {
         if (keyed) {
-            builder.startObject(CommonFields.BUCKETS);
+            builder.startObject(CommonFields.BUCKETS.getPreferredName());
         } else {
-            builder.startArray(CommonFields.BUCKETS);
+            builder.startArray(CommonFields.BUCKETS.getPreferredName());
         }
         for (Bucket range : buckets) {
             range.toXContent(builder, params);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalRange.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalRange.java
index 54ee27bfa96fd..db1d2d087750b 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalRange.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalRange.java
@@ -144,21 +144,21 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                 builder.startObject(key);
             } else {
                 builder.startObject();
-                builder.field(CommonFields.KEY, key);
+                builder.field(CommonFields.KEY.getPreferredName(), key);
             }
             if (!Double.isInfinite(from)) {
-                builder.field(CommonFields.FROM, from);
+                builder.field(CommonFields.FROM.getPreferredName(), from);
                 if (format != DocValueFormat.RAW) {
-                    builder.field(CommonFields.FROM_AS_STRING, format.format(from));
+                    builder.field(CommonFields.FROM_AS_STRING.getPreferredName(), format.format(from));
                 }
             }
             if (!Double.isInfinite(to)) {
-                builder.field(CommonFields.TO, to);
+                builder.field(CommonFields.TO.getPreferredName(), to);
                 if (format != DocValueFormat.RAW) {
-                    builder.field(CommonFields.TO_AS_STRING, format.format(to));
+                    builder.field(CommonFields.TO_AS_STRING.getPreferredName(), format.format(to));
                 }
             }
-            builder.field(CommonFields.DOC_COUNT, docCount);
+            builder.field(CommonFields.DOC_COUNT.getPreferredName(), docCount);
             aggregations.toXContentInternal(builder, params);
             builder.endObject();
             return builder;
@@ -178,10 +178,6 @@ public void writeTo(StreamOutput out) throws IOException {
     }
 
     public static class Factory<B extends Bucket, R extends InternalRange<B, R>> {
-        public Type type() {
-            return RangeAggregationBuilder.TYPE;
-        }
-
         public ValuesSourceType getValueSourceType() {
             return ValuesSourceType.NUMERIC;
         }
@@ -309,9 +305,9 @@ public InternalAggregation doReduce(List<InternalAggregation> aggregations, Redu
     @Override
     public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
         if (keyed) {
-            builder.startObject(CommonFields.BUCKETS);
+            builder.startObject(CommonFields.BUCKETS.getPreferredName());
         } else {
-            builder.startArray(CommonFields.BUCKETS);
+            builder.startArray(CommonFields.BUCKETS.getPreferredName());
         }
         for (B range : ranges) {
             range.toXContent(builder, params);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ParsedBinaryRange.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ParsedBinaryRange.java
new file mode 100644
index 0000000000000..955724aa66fe1
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ParsedBinaryRange.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.range;
+
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParserUtils;
+import org.elasticsearch.search.aggregations.Aggregation;
+import org.elasticsearch.search.aggregations.Aggregations;
+import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation;
+import org.elasticsearch.search.aggregations.bucket.range.ip.IpRangeAggregationBuilder;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+
+public class ParsedBinaryRange extends ParsedMultiBucketAggregation<ParsedBinaryRange.ParsedBucket> implements Range {
+
+    @Override
+    public String getType() {
+        return IpRangeAggregationBuilder.NAME;
+    }
+
+    @Override
+    public List<? extends Range.Bucket> getBuckets() {
+        return buckets;
+    }
+
+    private static ObjectParser<ParsedBinaryRange, Void> PARSER =
+            new ObjectParser<>(ParsedBinaryRange.class.getSimpleName(), true, ParsedBinaryRange::new);
+    static {
+        declareMultiBucketAggregationFields(PARSER,
+                parser -> ParsedBucket.fromXContent(parser, false),
+                parser -> ParsedBucket.fromXContent(parser, true));
+    }
+
+    public static ParsedBinaryRange fromXContent(XContentParser parser, String name) throws IOException {
+        ParsedBinaryRange aggregation = PARSER.parse(parser, null);
+        aggregation.setName(name);
+        return aggregation;
+    }
+
+    public static class ParsedBucket extends ParsedMultiBucketAggregation.ParsedBucket implements Range.Bucket {
+
+        private String key;
+        private String from;
+        private String to;
+
+        @Override
+        public Object getKey() {
+            return key;
+        }
+
+        @Override
+        public String getKeyAsString() {
+            return key;
+        }
+
+        @Override
+        public Object getFrom() {
+            return from;
+        }
+
+        @Override
+        public String getFromAsString() {
+            return from;
+        }
+
+        @Override
+        public Object getTo() {
+            return to;
+        }
+
+        @Override
+        public String getToAsString() {
+            return to;
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            if (isKeyed()) {
+                builder.startObject(key != null ? key : rangeKey(from, to));
+            } else {
+                builder.startObject();
+                if (key != null) {
+                    builder.field(CommonFields.KEY.getPreferredName(), key);
+                }
+            }
+            if (from != null) {
+                builder.field(CommonFields.FROM.getPreferredName(), getFrom());
+            }
+            if (to != null) {
+                builder.field(CommonFields.TO.getPreferredName(), getTo());
+            }
+            builder.field(CommonFields.DOC_COUNT.getPreferredName(), getDocCount());
+            getAggregations().toXContentInternal(builder, params);
+            builder.endObject();
+            return builder;
+        }
+
+        static ParsedBucket fromXContent(final XContentParser parser, final boolean keyed) throws IOException {
+            final ParsedBucket bucket = new ParsedBucket();
+            bucket.setKeyed(keyed);
+            XContentParser.Token token = parser.currentToken();
+            String currentFieldName = parser.currentName();
+
+            String rangeKey = null;
+            if (keyed) {
+                ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser::getTokenLocation);
+                rangeKey = currentFieldName;
+                ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
+            }
+
+            List<Aggregation> aggregations = new ArrayList<>();
+            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                if (token == XContentParser.Token.FIELD_NAME) {
+                    currentFieldName = parser.currentName();
+                } else if (token.isValue()) {
+                    if (CommonFields.KEY.getPreferredName().equals(currentFieldName)) {
+                        bucket.key = parser.text();
+                    } else if (CommonFields.DOC_COUNT.getPreferredName().equals(currentFieldName)) {
+                        bucket.setDocCount(parser.longValue());
+                    } else if (CommonFields.FROM.getPreferredName().equals(currentFieldName)) {
+                        bucket.from = parser.text();
+                    } else if (CommonFields.TO.getPreferredName().equals(currentFieldName)) {
+                        bucket.to = parser.text();
+                    }
+                } else if (token == XContentParser.Token.START_OBJECT) {
+                    XContentParserUtils.parseTypedKeysObject(parser, Aggregation.TYPED_KEYS_DELIMITER, Aggregation.class,
+                            aggregations::add);
+                }
+            }
+            bucket.setAggregations(new Aggregations(aggregations));
+
+            if (keyed) {
+                if (rangeKey(bucket.from, bucket.to).equals(rangeKey)) {
+                    bucket.key = null;
+                } else {
+                    bucket.key = rangeKey;
+                }
+            }
+            return bucket;
+        }
+
+        private static String rangeKey(String from, String to) {
+            return (from == null ? "*" : from) + '-' + (to == null ? "*" : to);
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ParsedRange.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ParsedRange.java
new file mode 100644
index 0000000000000..6095348e68863
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ParsedRange.java
@@ -0,0 +1,194 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.range;
+
+import org.elasticsearch.common.CheckedFunction;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParserUtils;
+import org.elasticsearch.search.aggregations.Aggregation;
+import org.elasticsearch.search.aggregations.Aggregations;
+import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Supplier;
+
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+
+public class ParsedRange extends ParsedMultiBucketAggregation<ParsedRange.ParsedBucket> implements Range {
+
+    @Override
+    public String getType() {
+        return RangeAggregationBuilder.NAME;
+    }
+
+    @Override
+    public List<? extends Range.Bucket> getBuckets() {
+        return buckets;
+    }
+
+    protected static void declareParsedRangeFields(final ObjectParser<? extends ParsedRange, Void> objectParser,
+                                         final CheckedFunction<XContentParser, ParsedBucket, IOException> bucketParser,
+                                         final CheckedFunction<XContentParser, ParsedBucket, IOException> keyedBucketParser) {
+        declareMultiBucketAggregationFields(objectParser, bucketParser::apply, keyedBucketParser::apply);
+    }
+
+    private static ObjectParser<ParsedRange, Void> PARSER =
+            new ObjectParser<>(ParsedRange.class.getSimpleName(), true, ParsedRange::new);
+    static {
+        declareParsedRangeFields(PARSER,
+                parser -> ParsedBucket.fromXContent(parser, false),
+                parser -> ParsedBucket.fromXContent(parser, true));
+    }
+
+    public static ParsedRange fromXContent(XContentParser parser, String name) throws IOException {
+        ParsedRange aggregation = PARSER.parse(parser, null);
+        aggregation.setName(name);
+        return aggregation;
+    }
+
+    public static class ParsedBucket extends ParsedMultiBucketAggregation.ParsedBucket implements Range.Bucket {
+
+        protected String key;
+        protected double from = Double.NEGATIVE_INFINITY;
+        protected String fromAsString;
+        protected double to = Double.POSITIVE_INFINITY;
+        protected String toAsString;
+
+        @Override
+        public String getKey() {
+            return getKeyAsString();
+        }
+
+        @Override
+        public String getKeyAsString() {
+            String keyAsString = super.getKeyAsString();
+            if (keyAsString != null) {
+                return keyAsString;
+            }
+            return key;
+        }
+
+        @Override
+        public Object getFrom() {
+            return from;
+        }
+
+        @Override
+        public String getFromAsString() {
+            if (fromAsString != null) {
+                return fromAsString;
+            }
+            return doubleAsString(from);
+        }
+
+        @Override
+        public Object getTo() {
+            return to;
+        }
+
+        @Override
+        public String getToAsString() {
+            if (toAsString != null) {
+                return toAsString;
+            }
+            return doubleAsString(to);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            if (isKeyed()) {
+                builder.startObject(key);
+            } else {
+                builder.startObject();
+                builder.field(CommonFields.KEY.getPreferredName(), key);
+            }
+            if (Double.isInfinite(from) == false) {
+                builder.field(CommonFields.FROM.getPreferredName(), from);
+                if (fromAsString != null) {
+                    builder.field(CommonFields.FROM_AS_STRING.getPreferredName(), fromAsString);
+                }
+            }
+            if (Double.isInfinite(to) == false) {
+                builder.field(CommonFields.TO.getPreferredName(), to);
+                if (toAsString != null) {
+                    builder.field(CommonFields.TO_AS_STRING.getPreferredName(), toAsString);
+                }
+            }
+            builder.field(CommonFields.DOC_COUNT.getPreferredName(), getDocCount());
+            getAggregations().toXContentInternal(builder, params);
+            builder.endObject();
+            return builder;
+        }
+
+        private static String doubleAsString(double d) {
+            return Double.isInfinite(d) ? null : Double.toString(d);
+        }
+
+        protected static <B extends ParsedBucket> B parseRangeBucketXContent(final XContentParser parser,
+                                                                             final Supplier<B> bucketSupplier,
+                                                                             final boolean keyed) throws IOException {
+            final B bucket = bucketSupplier.get();
+            bucket.setKeyed(keyed);
+            XContentParser.Token token = parser.currentToken();
+            String currentFieldName = parser.currentName();
+            if (keyed) {
+                ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser::getTokenLocation);
+                bucket.key = currentFieldName;
+                ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
+            }
+
+            List<Aggregation> aggregations = new ArrayList<>();
+            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                if (token == XContentParser.Token.FIELD_NAME) {
+                    currentFieldName = parser.currentName();
+                } else if (token.isValue()) {
+                    if (CommonFields.KEY_AS_STRING.getPreferredName().equals(currentFieldName)) {
+                        bucket.setKeyAsString(parser.text());
+                    } else if (CommonFields.KEY.getPreferredName().equals(currentFieldName)) {
+                        bucket.key = parser.text();
+                    } else if (CommonFields.DOC_COUNT.getPreferredName().equals(currentFieldName)) {
+                        bucket.setDocCount(parser.longValue());
+                    } else if (CommonFields.FROM.getPreferredName().equals(currentFieldName)) {
+                        bucket.from = parser.doubleValue();
+                    } else if (CommonFields.FROM_AS_STRING.getPreferredName().equals(currentFieldName)) {
+                        bucket.fromAsString = parser.text();
+                    } else if (CommonFields.TO.getPreferredName().equals(currentFieldName)) {
+                        bucket.to = parser.doubleValue();
+                    } else if (CommonFields.TO_AS_STRING.getPreferredName().equals(currentFieldName)) {
+                        bucket.toAsString = parser.text();
+                    }
+                } else if (token == XContentParser.Token.START_OBJECT) {
+                    XContentParserUtils.parseTypedKeysObject(parser, Aggregation.TYPED_KEYS_DELIMITER, Aggregation.class,
+                            aggregations::add);
+                }
+            }
+            bucket.setAggregations(new Aggregations(aggregations));
+            return bucket;
+        }
+
+        static ParsedBucket fromXContent(final XContentParser parser, final boolean keyed) throws IOException {
+            return parseRangeBucketXContent(parser, ParsedBucket::new, keyed);
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregationBuilder.java
index c815ae9d3cf1e..105dbbc545830 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregationBuilder.java
@@ -20,19 +20,43 @@
 package org.elasticsearch.search.aggregations.bucket.range;
 
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator.Range;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 
 public class RangeAggregationBuilder extends AbstractRangeBuilder<RangeAggregationBuilder, Range> {
     public static final String NAME = "range";
-    static final Type TYPE = new Type(NAME);
+
+    private static final ObjectParser<RangeAggregationBuilder, QueryParseContext> PARSER;
+    static {
+        PARSER = new ObjectParser<>(RangeAggregationBuilder.NAME);
+        ValuesSourceParserHelper.declareNumericFields(PARSER, true, true, false);
+        PARSER.declareBoolean(RangeAggregationBuilder::keyed, RangeAggregator.KEYED_FIELD);
+
+        PARSER.declareObjectArray((agg, ranges) -> {
+            for (Range range : ranges) {
+                agg.addRange(range);
+            }
+        }, RangeAggregationBuilder::parseRange, RangeAggregator.RANGES_FIELD);
+    }
+
+    public static AggregationBuilder parse(String aggregationName, QueryParseContext context) throws IOException {
+        return PARSER.parse(context.parser(), new RangeAggregationBuilder(aggregationName), context);
+    }
+
+    private static Range parseRange(XContentParser parser, QueryParseContext context) throws IOException {
+        return Range.fromXContent(parser);
+    }
 
     public RangeAggregationBuilder(String name) {
         super(name, InternalRange.FACTORY);
@@ -112,14 +136,19 @@ public RangeAggregationBuilder addUnboundedFrom(double from) {
     }
 
     @Override
-    protected RangeAggregatorFactory innerBuild(AggregationContext context, ValuesSourceConfig<Numeric> config,
+    protected RangeAggregatorFactory innerBuild(SearchContext context, ValuesSourceConfig<Numeric> config,
             AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
-        return new RangeAggregatorFactory(name, type, config, ranges, keyed, rangeFactory, context, parent, subFactoriesBuilder,
+        // We need to call processRanges here so they are parsed before we make the decision of whether to cache the request
+        Range[] ranges = processRanges(context, config);
+        if (ranges.length == 0) {
+            throw new IllegalArgumentException("No [ranges] specified for the [" + this.getName() + "] aggregation");
+        }
+        return new RangeAggregatorFactory(name, config, ranges, keyed, rangeFactory, context, parent, subFactoriesBuilder,
                 metaData);
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java
index c83e2d2c72141..a2348c4d19d54 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java
@@ -19,9 +19,7 @@
 package org.elasticsearch.search.aggregations.bucket.range;
 
 import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.util.InPlaceMergeSorter;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -39,7 +37,6 @@
 import org.elasticsearch.search.aggregations.NonCollectingAggregator;
 import org.elasticsearch.search.aggregations.bucket.BucketsAggregator;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.internal.SearchContext;
 
@@ -119,15 +116,15 @@ public Range process(DocValueFormat parser, SearchContext context) {
             Double from = this.from;
             Double to = this.to;
             if (fromAsStr != null) {
-                from = parser.parseDouble(fromAsStr, false, context::nowInMillis);
+                from = parser.parseDouble(fromAsStr, false, context.getQueryShardContext()::nowInMillis);
             }
             if (toAsStr != null) {
-                to = parser.parseDouble(toAsStr, false, context::nowInMillis);
+                to = parser.parseDouble(toAsStr, false, context.getQueryShardContext()::nowInMillis);
             }
             return new Range(key, from, fromAsStr, to, toAsStr);
         }
 
-        public static Range fromXContent(XContentParser parser, ParseFieldMatcher parseFieldMatcher) throws IOException {
+        public static Range fromXContent(XContentParser parser) throws IOException {
             XContentParser.Token token;
             String currentFieldName = null;
             double from = Double.NEGATIVE_INFINITY;
@@ -139,17 +136,17 @@ public static Range fromXContent(XContentParser parser, ParseFieldMatcher parseF
                 if (token == XContentParser.Token.FIELD_NAME) {
                     currentFieldName = parser.currentName();
                 } else if (token == XContentParser.Token.VALUE_NUMBER) {
-                    if (parseFieldMatcher.match(currentFieldName, FROM_FIELD)) {
+                    if (FROM_FIELD.match(currentFieldName)) {
                         from = parser.doubleValue();
-                    } else if (parseFieldMatcher.match(currentFieldName, TO_FIELD)) {
+                    } else if (TO_FIELD.match(currentFieldName)) {
                         to = parser.doubleValue();
                     }
                 } else if (token == XContentParser.Token.VALUE_STRING) {
-                    if (parseFieldMatcher.match(currentFieldName, FROM_FIELD)) {
+                    if (FROM_FIELD.match(currentFieldName)) {
                         fromAsStr = parser.text();
-                    } else if (parseFieldMatcher.match(currentFieldName, TO_FIELD)) {
+                    } else if (TO_FIELD.match(currentFieldName)) {
                         toAsStr = parser.text();
-                    } else if (parseFieldMatcher.match(currentFieldName, KEY_FIELD)) {
+                    } else if (KEY_FIELD.match(currentFieldName)) {
                         key = parser.text();
                     }
                 }
@@ -210,21 +207,17 @@ public boolean equals(Object obj) {
     final double[] maxTo;
 
     public RangeAggregator(String name, AggregatorFactories factories, ValuesSource.Numeric valuesSource, DocValueFormat format,
-            InternalRange.Factory rangeFactory, List<? extends Range> ranges, boolean keyed, AggregationContext aggregationContext,
+            InternalRange.Factory rangeFactory, Range[] ranges, boolean keyed, SearchContext context,
             Aggregator parent, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException {
 
-        super(name, factories, aggregationContext, parent, pipelineAggregators, metaData);
+        super(name, factories, context, parent, pipelineAggregators, metaData);
         assert valuesSource != null;
         this.valuesSource = valuesSource;
         this.format = format;
         this.keyed = keyed;
         this.rangeFactory = rangeFactory;
 
-        this.ranges = new Range[ranges.size()];
-        for (int i = 0; i < this.ranges.length; i++) {
-            this.ranges[i] = ranges.get(i).process(format, context.searchContext());
-        }
-        sortRanges(this.ranges);
+        this.ranges = ranges;
 
         maxTo = new double[this.ranges.length];
         maxTo[0] = this.ranges[0].to;
@@ -337,45 +330,21 @@ public InternalAggregation buildEmptyAggregation() {
         return rangeFactory.create(name, buckets, format, keyed, pipelineAggregators(), metaData());
     }
 
-    private static void sortRanges(final Range[] ranges) {
-        new InPlaceMergeSorter() {
-
-            @Override
-            protected void swap(int i, int j) {
-                final Range tmp = ranges[i];
-                ranges[i] = ranges[j];
-                ranges[j] = tmp;
-            }
-
-            @Override
-            protected int compare(int i, int j) {
-                int cmp = Double.compare(ranges[i].from, ranges[j].from);
-                if (cmp == 0) {
-                    cmp = Double.compare(ranges[i].to, ranges[j].to);
-                }
-                return cmp;
-            }
-        }.sort(0, ranges.length);
-    }
-
     public static class Unmapped<R extends RangeAggregator.Range> extends NonCollectingAggregator {
 
-        private final List<R> ranges;
+        private final R[] ranges;
         private final boolean keyed;
         private final InternalRange.Factory factory;
         private final DocValueFormat format;
 
         @SuppressWarnings("unchecked")
-        public Unmapped(String name, List<R> ranges, boolean keyed, DocValueFormat format,
-                AggregationContext context,
+        public Unmapped(String name, R[] ranges, boolean keyed, DocValueFormat format,
+                SearchContext context,
                 Aggregator parent, InternalRange.Factory factory, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData)
                 throws IOException {
 
             super(name, context, parent, pipelineAggregators, metaData);
-            this.ranges = new ArrayList<>();
-            for (R range : ranges) {
-                this.ranges.add((R) range.process(format, context.searchContext()));
-            }
+            this.ranges = ranges;
             this.keyed = keyed;
             this.format = format;
             this.factory = factory;
@@ -384,7 +353,7 @@ public Unmapped(String name, List<R> ranges, boolean keyed, DocValueFormat forma
         @Override
         public InternalAggregation buildEmptyAggregation() {
             InternalAggregations subAggs = buildEmptySubAggregations();
-            List<org.elasticsearch.search.aggregations.bucket.range.Range.Bucket> buckets = new ArrayList<>(ranges.size());
+            List<org.elasticsearch.search.aggregations.bucket.range.Range.Bucket> buckets = new ArrayList<>(ranges.length);
             for (RangeAggregator.Range range : ranges) {
                 buckets.add(factory.createBucket(range.key, range.from, range.to, 0, subAggs, keyed, format));
             }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorFactory.java
index 5dec4c40c4580..d1dc3e71b5c11 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorFactory.java
@@ -21,23 +21,21 @@
 
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.bucket.range.InternalRange.Factory;
 import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator.Range;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
-import java.util.List;
 import java.util.Map;
 
 public class RangeAggregatorFactory extends AbstractRangeAggregatorFactory<RangeAggregatorFactory, RangeAggregator.Range> {
 
-    public RangeAggregatorFactory(String name, Type type, ValuesSourceConfig<Numeric> config, List<Range> ranges, boolean keyed,
-            Factory<?, ?> rangeFactory, AggregationContext context, AggregatorFactory<?> parent,
+    public RangeAggregatorFactory(String name, ValuesSourceConfig<Numeric> config, Range[] ranges, boolean keyed,
+            Factory<?, ?> rangeFactory, SearchContext context, AggregatorFactory<?> parent,
             AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
-        super(name, type, config, ranges, keyed, rangeFactory, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, ranges, keyed, rangeFactory, context, parent, subFactoriesBuilder, metaData);
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeParser.java
deleted file mode 100644
index c8cb2c767157a..0000000000000
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeParser.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.search.aggregations.bucket.range;
-
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentParser.Token;
-import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator.Range;
-import org.elasticsearch.search.aggregations.support.AbstractValuesSourceParser.NumericValuesSourceParser;
-import org.elasticsearch.search.aggregations.support.XContentParseContext;
-import org.elasticsearch.search.aggregations.support.ValueType;
-import org.elasticsearch.search.aggregations.support.ValuesSourceType;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-/**
- *
- */
-public class RangeParser extends NumericValuesSourceParser {
-
-    public RangeParser() {
-        this(true, true, false);
-    }
-
-    /**
-     * Used by subclasses that parse slightly different kinds of ranges.
-     */
-    protected RangeParser(boolean scriptable, boolean formattable, boolean timezoneAware) {
-        super(scriptable, formattable, timezoneAware);
-    }
-
-    @Override
-    protected AbstractRangeBuilder<?, ?> createFactory(String aggregationName, ValuesSourceType valuesSourceType,
-            ValueType targetValueType, Map<ParseField, Object> otherOptions) {
-        RangeAggregationBuilder factory = new RangeAggregationBuilder(aggregationName);
-        @SuppressWarnings("unchecked")
-        List<? extends Range> ranges = (List<? extends Range>) otherOptions.get(RangeAggregator.RANGES_FIELD);
-        for (Range range : ranges) {
-            factory.addRange(range);
-        }
-        Boolean keyed = (Boolean) otherOptions.get(RangeAggregator.KEYED_FIELD);
-        if (keyed != null) {
-            factory.keyed(keyed);
-        }
-        return factory;
-    }
-
-    @Override
-    protected boolean token(String aggregationName, String currentFieldName, Token token,
-                            XContentParseContext context, Map<ParseField, Object> otherOptions) throws IOException {
-        XContentParser parser = context.getParser();
-        if (token == XContentParser.Token.START_ARRAY) {
-            if (context.matchField(currentFieldName, RangeAggregator.RANGES_FIELD)) {
-                List<Range> ranges = new ArrayList<>();
-                while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
-                    Range range = parseRange(parser, context.getParseFieldMatcher());
-                    ranges.add(range);
-                }
-                otherOptions.put(RangeAggregator.RANGES_FIELD, ranges);
-                return true;
-            }
-        } else if (token == XContentParser.Token.VALUE_BOOLEAN) {
-            if (context.matchField(currentFieldName, RangeAggregator.KEYED_FIELD)) {
-                boolean keyed = parser.booleanValue();
-                otherOptions.put(RangeAggregator.KEYED_FIELD, keyed);
-                return true;
-            }
-        }
-        return false;
-    }
-
-    protected Range parseRange(XContentParser parser, ParseFieldMatcher parseFieldMatcher) throws IOException {
-        return Range.fromXContent(parser, parseFieldMatcher);
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/DateRangeAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/DateRangeAggregationBuilder.java
index a75b071569c62..2c686fbb97768 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/DateRangeAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/DateRangeAggregationBuilder.java
@@ -20,22 +20,46 @@
 package org.elasticsearch.search.aggregations.bucket.range.date;
 
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.bucket.range.AbstractRangeBuilder;
 import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator;
 import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator.Range;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper;
+import org.elasticsearch.search.internal.SearchContext;
 import org.joda.time.DateTime;
 
 import java.io.IOException;
 
 public class DateRangeAggregationBuilder extends AbstractRangeBuilder<DateRangeAggregationBuilder, RangeAggregator.Range> {
     public static final String NAME = "date_range";
-    static final Type TYPE = new Type(NAME);
+
+    private static final ObjectParser<DateRangeAggregationBuilder, QueryParseContext> PARSER;
+    static {
+        PARSER = new ObjectParser<>(DateRangeAggregationBuilder.NAME);
+        ValuesSourceParserHelper.declareNumericFields(PARSER, true, true, true);
+        PARSER.declareBoolean(DateRangeAggregationBuilder::keyed, RangeAggregator.KEYED_FIELD);
+
+        PARSER.declareObjectArray((agg, ranges) -> {
+            for (Range range : ranges) {
+                agg.addRange(range);
+            }
+        }, DateRangeAggregationBuilder::parseRange, RangeAggregator.RANGES_FIELD);
+    }
+
+    public static AggregationBuilder parse(String aggregationName, QueryParseContext context) throws IOException {
+        return PARSER.parse(context.parser(), new DateRangeAggregationBuilder(aggregationName), context);
+    }
+
+    private static Range parseRange(XContentParser parser, QueryParseContext context) throws IOException {
+        return Range.fromXContent(parser);
+    }
 
     public DateRangeAggregationBuilder(String name) {
         super(name, InternalDateRange.FACTORY);
@@ -49,7 +73,7 @@ public DateRangeAggregationBuilder(StreamInput in) throws IOException {
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 
@@ -257,9 +281,15 @@ public DateRangeAggregationBuilder addUnboundedFrom(DateTime from) {
     }
 
     @Override
-    protected DateRangeAggregatorFactory innerBuild(AggregationContext context, ValuesSourceConfig<Numeric> config,
+    protected DateRangeAggregatorFactory innerBuild(SearchContext context, ValuesSourceConfig<Numeric> config,
             AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
-        return new DateRangeAggregatorFactory(name, type, config, ranges, keyed, rangeFactory, context, parent, subFactoriesBuilder,
+        // We need to call processRanges here so they are parsed and we know whether `now` has been used before we make
+        // the decision of whether to cache the request
+        Range[] ranges = processRanges(context, config);
+        if (ranges.length == 0) {
+            throw new IllegalArgumentException("No [ranges] specified for the [" + this.getName() + "] aggregation");
+        }
+        return new DateRangeAggregatorFactory(name, config, ranges, keyed, rangeFactory, context, parent, subFactoriesBuilder,
                 metaData);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/DateRangeAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/DateRangeAggregatorFactory.java
index d3bb7ac6238d0..3d674c5a04006 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/DateRangeAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/DateRangeAggregatorFactory.java
@@ -21,24 +21,22 @@
 
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.bucket.range.AbstractRangeAggregatorFactory;
 import org.elasticsearch.search.aggregations.bucket.range.InternalRange.Factory;
 import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator.Range;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
-import java.util.List;
 import java.util.Map;
 
 public class DateRangeAggregatorFactory extends AbstractRangeAggregatorFactory<DateRangeAggregatorFactory, Range> {
 
-    public DateRangeAggregatorFactory(String name, Type type, ValuesSourceConfig<Numeric> config, List<Range> ranges, boolean keyed,
-            Factory<?, ?> rangeFactory, AggregationContext context, AggregatorFactory<?> parent,
+    public DateRangeAggregatorFactory(String name, ValuesSourceConfig<Numeric> config, Range[] ranges, boolean keyed,
+            Factory<?, ?> rangeFactory, SearchContext context, AggregatorFactory<?> parent,
             AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
-        super(name, type, config, ranges, keyed, rangeFactory, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, ranges, keyed, rangeFactory, context, parent, subFactoriesBuilder, metaData);
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/DateRangeParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/DateRangeParser.java
deleted file mode 100644
index f8eff715abb3b..0000000000000
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/DateRangeParser.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.search.aggregations.bucket.range.date;
-
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator;
-import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator.Range;
-import org.elasticsearch.search.aggregations.bucket.range.RangeParser;
-import org.elasticsearch.search.aggregations.support.ValueType;
-import org.elasticsearch.search.aggregations.support.ValuesSourceType;
-
-import java.util.List;
-import java.util.Map;
-
-/**
- *
- */
-public class DateRangeParser extends RangeParser {
-
-    public DateRangeParser() {
-        super(true, true, true);
-    }
-
-    @Override
-    protected DateRangeAggregationBuilder createFactory(String aggregationName, ValuesSourceType valuesSourceType,
-                                                        ValueType targetValueType, Map<ParseField, Object> otherOptions) {
-        DateRangeAggregationBuilder factory = new DateRangeAggregationBuilder(aggregationName);
-        @SuppressWarnings("unchecked")
-        List<Range> ranges = (List<Range>) otherOptions.get(RangeAggregator.RANGES_FIELD);
-        for (Range range : ranges) {
-            factory.addRange(range);
-        }
-        Boolean keyed = (Boolean) otherOptions.get(RangeAggregator.KEYED_FIELD);
-        if (keyed != null) {
-            factory.keyed(keyed);
-        }
-        return factory;
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/InternalDateRange.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/InternalDateRange.java
index f0dfec2312f4a..b0cb11624ef10 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/InternalDateRange.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/InternalDateRange.java
@@ -87,11 +87,6 @@ DocValueFormat format() {
     }
 
     public static class Factory extends InternalRange.Factory<InternalDateRange.Bucket, InternalDateRange> {
-        @Override
-        public Type type() {
-            return DateRangeAggregationBuilder.TYPE;
-        }
-
         @Override
         public ValueType getValueType() {
             return ValueType.DATE;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/ParsedDateRange.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/ParsedDateRange.java
new file mode 100644
index 0000000000000..b8f2f008cec65
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/ParsedDateRange.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.range.date;
+
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.bucket.range.ParsedRange;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+
+import java.io.IOException;
+
+public class ParsedDateRange extends ParsedRange {
+
+    @Override
+    public String getType() {
+        return DateRangeAggregationBuilder.NAME;
+    }
+
+    private static ObjectParser<ParsedDateRange, Void> PARSER =
+            new ObjectParser<>(ParsedDateRange.class.getSimpleName(), true, ParsedDateRange::new);
+    static {
+        declareParsedRangeFields(PARSER,
+                parser -> ParsedBucket.fromXContent(parser, false),
+                parser -> ParsedBucket.fromXContent(parser, true));
+    }
+
+    public static ParsedDateRange fromXContent(XContentParser parser, String name) throws IOException {
+        ParsedDateRange aggregation = PARSER.parse(parser, null);
+        aggregation.setName(name);
+        return aggregation;
+    }
+
+    public static class ParsedBucket extends ParsedRange.ParsedBucket {
+
+        @Override
+        public Object getFrom() {
+            return doubleAsDateTime(from);
+        }
+
+        @Override
+        public Object getTo() {
+            return doubleAsDateTime(to);
+        }
+
+        private static DateTime doubleAsDateTime(Double d) {
+            if (d == null || Double.isInfinite(d)) {
+                return null;
+            }
+            return new DateTime(d.longValue(), DateTimeZone.UTC);
+        }
+
+        static ParsedBucket fromXContent(final XContentParser parser, final boolean keyed) throws IOException {
+            return parseRangeBucketXContent(parser, ParsedBucket::new, keyed);
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/GeoDistanceAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/GeoDistanceAggregationBuilder.java
index 4a4cab2affa03..1484fae8d44fb 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/GeoDistanceAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/GeoDistanceAggregationBuilder.java
@@ -19,23 +19,29 @@
 
 package org.elasticsearch.search.aggregations.bucket.range.geodistance;
 
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.geo.GeoDistance;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.unit.DistanceUnit;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParser.Token;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.bucket.range.InternalRange;
 import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator;
-import org.elasticsearch.search.aggregations.bucket.range.geodistance.GeoDistanceParser.Range;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -44,24 +50,170 @@
 
 public class GeoDistanceAggregationBuilder extends ValuesSourceAggregationBuilder<ValuesSource.GeoPoint, GeoDistanceAggregationBuilder> {
     public static final String NAME = "geo_distance";
-    public static final Type TYPE = new Type(NAME);
+    static final ParseField ORIGIN_FIELD = new ParseField("origin", "center", "point", "por");
+    static final ParseField UNIT_FIELD = new ParseField("unit");
+    static final ParseField DISTANCE_TYPE_FIELD = new ParseField("distance_type");
 
-    private final GeoPoint origin;
+    private static final ObjectParser<GeoDistanceAggregationBuilder, QueryParseContext> PARSER;
+    static {
+        PARSER = new ObjectParser<>(GeoDistanceAggregationBuilder.NAME);
+        ValuesSourceParserHelper.declareGeoFields(PARSER, true, false);
+
+        PARSER.declareBoolean(GeoDistanceAggregationBuilder::keyed, RangeAggregator.KEYED_FIELD);
+
+        PARSER.declareObjectArray((agg, ranges) -> {
+            for (Range range : ranges) {
+                agg.addRange(range);
+            }
+        }, GeoDistanceAggregationBuilder::parseRange, RangeAggregator.RANGES_FIELD);
+
+        PARSER.declareField(GeoDistanceAggregationBuilder::unit, p -> DistanceUnit.fromString(p.text()),
+                UNIT_FIELD, ObjectParser.ValueType.STRING);
+
+        PARSER.declareField(GeoDistanceAggregationBuilder::distanceType, p -> GeoDistance.fromString(p.text()),
+                DISTANCE_TYPE_FIELD, ObjectParser.ValueType.STRING);
+
+        PARSER.declareField(GeoDistanceAggregationBuilder::origin, GeoDistanceAggregationBuilder::parseGeoPoint,
+                ORIGIN_FIELD, ObjectParser.ValueType.OBJECT_ARRAY_OR_STRING);
+    }
+
+    public static AggregationBuilder parse(String aggregationName, QueryParseContext context) throws IOException {
+        GeoDistanceAggregationBuilder builder = PARSER.parse(context.parser(), new GeoDistanceAggregationBuilder(aggregationName), context);
+        if (builder.origin() == null) {
+            throw new IllegalArgumentException("Aggregation [" + aggregationName + "] must define an [origin].");
+        }
+        return builder;
+    }
+
+    public static class Range extends RangeAggregator.Range {
+        public Range(String key, Double from, Double to) {
+            super(key(key, from, to), from == null ? 0 : from, to);
+        }
+
+        /**
+         * Read from a stream.
+         */
+        public Range(StreamInput in) throws IOException {
+            super(in.readOptionalString(), in.readDouble(), in.readDouble());
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeOptionalString(key);
+            out.writeDouble(from);
+            out.writeDouble(to);
+        }
+
+        private static String key(String key, Double from, Double to) {
+            if (key != null) {
+                return key;
+            }
+            StringBuilder sb = new StringBuilder();
+            sb.append((from == null || from == 0) ? "*" : from);
+            sb.append("-");
+            sb.append((to == null || Double.isInfinite(to)) ? "*" : to);
+            return sb.toString();
+        }
+    }
+
+    private static GeoPoint parseGeoPoint(XContentParser parser, QueryParseContext context) throws IOException {
+        Token token = parser.currentToken();
+        if (token == XContentParser.Token.VALUE_STRING) {
+            GeoPoint point = new GeoPoint();
+            point.resetFromString(parser.text());
+            return point;
+        }
+        if (token == XContentParser.Token.START_ARRAY) {
+            double lat = Double.NaN;
+            double lon = Double.NaN;
+            while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                if (Double.isNaN(lon)) {
+                    lon = parser.doubleValue();
+                } else if (Double.isNaN(lat)) {
+                    lat = parser.doubleValue();
+                } else {
+                    throw new ParsingException(parser.getTokenLocation(), "malformed [" + ORIGIN_FIELD.getPreferredName()
+                        + "]: a geo point array must be of the form [lon, lat]");
+                }
+            }
+            return new GeoPoint(lat, lon);
+        }
+        if (token == XContentParser.Token.START_OBJECT) {
+            String currentFieldName = null;
+            double lat = Double.NaN;
+            double lon = Double.NaN;
+            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                if (token == XContentParser.Token.FIELD_NAME) {
+                    currentFieldName = parser.currentName();
+                } else if (token == XContentParser.Token.VALUE_NUMBER) {
+                    if ("lat".equals(currentFieldName)) {
+                        lat = parser.doubleValue();
+                    } else if ("lon".equals(currentFieldName)) {
+                        lon = parser.doubleValue();
+                    }
+                }
+            }
+            if (Double.isNaN(lat) || Double.isNaN(lon)) {
+                throw new ParsingException(parser.getTokenLocation(),
+                        "malformed [" + currentFieldName + "] geo point object. either [lat] or [lon] (or both) are " + "missing");
+            }
+            return new GeoPoint(lat, lon);
+        }
+
+        // should not happen since we only parse geo points when we encounter a string, an object or an array
+        throw new IllegalArgumentException("Unexpected token [" + token + "] while parsing geo point");
+    }
+
+    private static Range parseRange(XContentParser parser, QueryParseContext context) throws IOException {
+        String fromAsStr = null;
+        String toAsStr = null;
+        double from = 0.0;
+        double to = Double.POSITIVE_INFINITY;
+        String key = null;
+        String toOrFromOrKey = null;
+        Token token;
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                toOrFromOrKey = parser.currentName();
+            } else if (token == XContentParser.Token.VALUE_NUMBER) {
+                if (Range.FROM_FIELD.match(toOrFromOrKey)) {
+                    from = parser.doubleValue();
+                } else if (Range.TO_FIELD.match(toOrFromOrKey)) {
+                    to = parser.doubleValue();
+                }
+            } else if (token == XContentParser.Token.VALUE_STRING) {
+                if (Range.KEY_FIELD.match(toOrFromOrKey)) {
+                    key = parser.text();
+                } else if (Range.FROM_FIELD.match(toOrFromOrKey)) {
+                    fromAsStr = parser.text();
+                } else if (Range.TO_FIELD.match(toOrFromOrKey)) {
+                    toAsStr = parser.text();
+                }
+            }
+        }
+        if (fromAsStr != null || toAsStr != null) {
+            return new Range(key, Double.parseDouble(fromAsStr), Double.parseDouble(toAsStr));
+        } else {
+            return new Range(key, from, to);
+        }
+    }
+
+    private GeoPoint origin;
     private List<Range> ranges = new ArrayList<>();
     private DistanceUnit unit = DistanceUnit.DEFAULT;
-    private GeoDistance distanceType = GeoDistance.DEFAULT;
+    private GeoDistance distanceType = GeoDistance.ARC;
     private boolean keyed = false;
 
     public GeoDistanceAggregationBuilder(String name, GeoPoint origin) {
         this(name, origin, InternalGeoDistance.FACTORY);
+        if (origin == null) {
+            throw new IllegalArgumentException("[origin] must not be null: [" + name + "]");
+        }
     }
 
     private GeoDistanceAggregationBuilder(String name, GeoPoint origin,
                                           InternalRange.Factory<InternalGeoDistance.Bucket, InternalGeoDistance> rangeFactory) {
-        super(name, rangeFactory.type(), rangeFactory.getValueSourceType(), rangeFactory.getValueType());
-        if (origin == null) {
-            throw new IllegalArgumentException("[origin] must not be null: [" + name + "]");
-        }
+        super(name, rangeFactory.getValueSourceType(), rangeFactory.getValueType());
         this.origin = origin;
     }
 
@@ -69,8 +221,7 @@ private GeoDistanceAggregationBuilder(String name, GeoPoint origin,
      * Read from a stream.
      */
     public GeoDistanceAggregationBuilder(StreamInput in) throws IOException {
-        super(in, InternalGeoDistance.FACTORY.type(), InternalGeoDistance.FACTORY.getValueSourceType(),
-                InternalGeoDistance.FACTORY.getValueType());
+        super(in, InternalGeoDistance.FACTORY.getValueSourceType(), InternalGeoDistance.FACTORY.getValueType());
         origin = new GeoPoint(in.readDouble(), in.readDouble());
         int size = in.readVInt();
         ranges = new ArrayList<>(size);
@@ -82,6 +233,23 @@ public GeoDistanceAggregationBuilder(StreamInput in) throws IOException {
         unit = DistanceUnit.readFromStream(in);
     }
 
+    // for parsing
+    GeoDistanceAggregationBuilder(String name) {
+        this(name, null, InternalGeoDistance.FACTORY);
+    }
+
+    GeoDistanceAggregationBuilder origin(GeoPoint origin) {
+        this.origin = origin;
+        return this;
+    }
+
+    /**
+     * Return the {@link GeoPoint} that is used for distance computations.
+     */
+    public GeoPoint origin() {
+        return origin;
+    }
+
     @Override
     protected void innerWriteTo(StreamOutput out) throws IOException {
         out.writeDouble(origin.lat());
@@ -174,7 +342,7 @@ public List<Range> range() {
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 
@@ -212,20 +380,24 @@ public boolean keyed() {
     }
 
     @Override
-    protected ValuesSourceAggregatorFactory<ValuesSource.GeoPoint, ?> innerBuild(AggregationContext context,
+    protected ValuesSourceAggregatorFactory<ValuesSource.GeoPoint, ?> innerBuild(SearchContext context,
             ValuesSourceConfig<ValuesSource.GeoPoint> config, AggregatorFactory<?> parent, Builder subFactoriesBuilder)
                     throws IOException {
-        return new GeoDistanceRangeAggregatorFactory(name, type, config, origin, ranges, unit, distanceType, keyed, context, parent,
+        Range[] ranges = this.ranges.toArray(new Range[this.range().size()]);
+        if (ranges.length == 0) {
+            throw new IllegalArgumentException("No [ranges] specified for the [" + this.getName() + "] aggregation");
+        }
+        return new GeoDistanceRangeAggregatorFactory(name, config, origin, ranges, unit, distanceType, keyed, context, parent,
                 subFactoriesBuilder, metaData);
     }
 
     @Override
     protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
-        builder.field(GeoDistanceParser.ORIGIN_FIELD.getPreferredName(), origin);
+        builder.field(ORIGIN_FIELD.getPreferredName(), origin);
         builder.field(RangeAggregator.RANGES_FIELD.getPreferredName(), ranges);
         builder.field(RangeAggregator.KEYED_FIELD.getPreferredName(), keyed);
-        builder.field(GeoDistanceParser.UNIT_FIELD.getPreferredName(), unit);
-        builder.field(GeoDistanceParser.DISTANCE_TYPE_FIELD.getPreferredName(), distanceType);
+        builder.field(UNIT_FIELD.getPreferredName(), unit);
+        builder.field(DISTANCE_TYPE_FIELD.getPreferredName(), distanceType);
         return builder;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/GeoDistanceParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/GeoDistanceParser.java
deleted file mode 100644
index 677731d64ef3b..0000000000000
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/GeoDistanceParser.java
+++ /dev/null
@@ -1,175 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.search.aggregations.bucket.range.geodistance;
-
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.geo.GeoDistance;
-import org.elasticsearch.common.geo.GeoPoint;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.unit.DistanceUnit;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentParser.Token;
-import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator;
-import org.elasticsearch.search.aggregations.support.AbstractValuesSourceParser.GeoPointValuesSourceParser;
-import org.elasticsearch.search.aggregations.support.GeoPointParser;
-import org.elasticsearch.search.aggregations.support.XContentParseContext;
-import org.elasticsearch.search.aggregations.support.ValueType;
-import org.elasticsearch.search.aggregations.support.ValuesSourceType;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-/**
- *
- */
-public class GeoDistanceParser extends GeoPointValuesSourceParser {
-
-    static final ParseField ORIGIN_FIELD = new ParseField("origin", "center", "point", "por");
-    static final ParseField UNIT_FIELD = new ParseField("unit");
-    static final ParseField DISTANCE_TYPE_FIELD = new ParseField("distance_type");
-
-    private GeoPointParser geoPointParser = new GeoPointParser(GeoDistanceAggregationBuilder.TYPE, ORIGIN_FIELD);
-
-    public GeoDistanceParser() {
-        super(true, false);
-    }
-
-    public static class Range extends RangeAggregator.Range {
-        public Range(String key, Double from, Double to) {
-            super(key(key, from, to), from == null ? 0 : from, to);
-        }
-
-        /**
-         * Read from a stream.
-         */
-        public Range(StreamInput in) throws IOException {
-            super(in.readOptionalString(), in.readDouble(), in.readDouble());
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            out.writeOptionalString(key);
-            out.writeDouble(from);
-            out.writeDouble(to);
-        }
-
-        private static String key(String key, Double from, Double to) {
-            if (key != null) {
-                return key;
-            }
-            StringBuilder sb = new StringBuilder();
-            sb.append((from == null || from == 0) ? "*" : from);
-            sb.append("-");
-            sb.append((to == null || Double.isInfinite(to)) ? "*" : to);
-            return sb.toString();
-        }
-    }
-
-    @Override
-    protected GeoDistanceAggregationBuilder createFactory(
-            String aggregationName, ValuesSourceType valuesSourceType, ValueType targetValueType, Map<ParseField, Object> otherOptions) {
-        GeoPoint origin = (GeoPoint) otherOptions.get(ORIGIN_FIELD);
-        GeoDistanceAggregationBuilder factory = new GeoDistanceAggregationBuilder(aggregationName, origin);
-        @SuppressWarnings("unchecked")
-        List<Range> ranges = (List<Range>) otherOptions.get(RangeAggregator.RANGES_FIELD);
-        for (Range range : ranges) {
-            factory.addRange(range);
-        }
-        Boolean keyed = (Boolean) otherOptions.get(RangeAggregator.KEYED_FIELD);
-        if (keyed != null) {
-            factory.keyed(keyed);
-        }
-        DistanceUnit unit = (DistanceUnit) otherOptions.get(UNIT_FIELD);
-        if (unit != null) {
-            factory.unit(unit);
-        }
-        GeoDistance distanceType = (GeoDistance) otherOptions.get(DISTANCE_TYPE_FIELD);
-        if (distanceType != null) {
-            factory.distanceType(distanceType);
-        }
-        return factory;
-    }
-
-    @Override
-    protected boolean token(String aggregationName, String currentFieldName, Token token,
-                            XContentParseContext context, Map<ParseField, Object> otherOptions) throws IOException {
-        XContentParser parser = context.getParser();
-        if (geoPointParser.token(aggregationName, currentFieldName, token, parser, context.getParseFieldMatcher(), otherOptions)) {
-            return true;
-        } else if (token == XContentParser.Token.VALUE_STRING) {
-            if (context.matchField(currentFieldName, UNIT_FIELD)) {
-                DistanceUnit unit = DistanceUnit.fromString(parser.text());
-                otherOptions.put(UNIT_FIELD, unit);
-                return true;
-            } else if (context.matchField(currentFieldName, DISTANCE_TYPE_FIELD)) {
-                GeoDistance distanceType = GeoDistance.fromString(parser.text());
-                otherOptions.put(DISTANCE_TYPE_FIELD, distanceType);
-                return true;
-            }
-        } else if (token == XContentParser.Token.VALUE_BOOLEAN) {
-            if (context.matchField(currentFieldName, RangeAggregator.KEYED_FIELD)) {
-                boolean keyed = parser.booleanValue();
-                otherOptions.put(RangeAggregator.KEYED_FIELD, keyed);
-                return true;
-            }
-        } else if (token == XContentParser.Token.START_ARRAY) {
-            if (context.matchField(currentFieldName, RangeAggregator.RANGES_FIELD)) {
-                List<Range> ranges = new ArrayList<>();
-                while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
-                    String fromAsStr = null;
-                    String toAsStr = null;
-                    double from = 0.0;
-                    double to = Double.POSITIVE_INFINITY;
-                    String key = null;
-                    String toOrFromOrKey = null;
-                    while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                        if (token == XContentParser.Token.FIELD_NAME) {
-                            toOrFromOrKey = parser.currentName();
-                        } else if (token == XContentParser.Token.VALUE_NUMBER) {
-                            if (context.matchField(toOrFromOrKey, Range.FROM_FIELD)) {
-                                from = parser.doubleValue();
-                            } else if (context.matchField(toOrFromOrKey, Range.TO_FIELD)) {
-                                to = parser.doubleValue();
-                            }
-                        } else if (token == XContentParser.Token.VALUE_STRING) {
-                            if (context.matchField(toOrFromOrKey, Range.KEY_FIELD)) {
-                                key = parser.text();
-                            } else if (context.matchField(toOrFromOrKey, Range.FROM_FIELD)) {
-                                fromAsStr = parser.text();
-                            } else if (context.matchField(toOrFromOrKey, Range.TO_FIELD)) {
-                                toAsStr = parser.text();
-                            }
-                        }
-                    }
-                    if (fromAsStr != null || toAsStr != null) {
-                        ranges.add(new Range(key, Double.parseDouble(fromAsStr), Double.parseDouble(toAsStr)));
-                    } else {
-                        ranges.add(new Range(key, from, to));
-                    }
-                }
-                otherOptions.put(RangeAggregator.RANGES_FIELD, ranges);
-                return true;
-            }
-        }
-        return false;
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/GeoDistanceRangeAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/GeoDistanceRangeAggregatorFactory.java
index 32c3592a8fcb4..e12853e5b17f5 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/GeoDistanceRangeAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/GeoDistanceRangeAggregatorFactory.java
@@ -23,7 +23,7 @@
 import org.apache.lucene.index.SortedNumericDocValues;
 import org.elasticsearch.common.geo.GeoDistance;
 import org.elasticsearch.common.geo.GeoPoint;
-import org.elasticsearch.common.geo.GeoDistance.FixedSourceDistance;
+import org.elasticsearch.common.geo.GeoUtils;
 import org.elasticsearch.common.unit.DistanceUnit;
 import org.elasticsearch.index.fielddata.MultiGeoPointValues;
 import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
@@ -31,16 +31,15 @@
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.bucket.range.InternalRange;
 import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator;
 import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator.Unmapped;
-import org.elasticsearch.search.aggregations.bucket.range.geodistance.GeoDistanceParser.Range;
+import org.elasticsearch.search.aggregations.bucket.range.geodistance.GeoDistanceAggregationBuilder.Range;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -51,15 +50,15 @@ public class GeoDistanceRangeAggregatorFactory
 
     private final InternalRange.Factory<InternalGeoDistance.Bucket, InternalGeoDistance> rangeFactory = InternalGeoDistance.FACTORY;
     private final GeoPoint origin;
-    private final List<Range> ranges;
+    private final Range[] ranges;
     private final DistanceUnit unit;
     private final GeoDistance distanceType;
     private final boolean keyed;
 
-    public GeoDistanceRangeAggregatorFactory(String name, Type type, ValuesSourceConfig<ValuesSource.GeoPoint> config, GeoPoint origin,
-            List<Range> ranges, DistanceUnit unit, GeoDistance distanceType, boolean keyed, AggregationContext context,
+    public GeoDistanceRangeAggregatorFactory(String name, ValuesSourceConfig<ValuesSource.GeoPoint> config, GeoPoint origin,
+            Range[] ranges, DistanceUnit unit, GeoDistance distanceType, boolean keyed, SearchContext context,
             AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.origin = origin;
         this.ranges = ranges;
         this.unit = unit;
@@ -70,7 +69,7 @@ public GeoDistanceRangeAggregatorFactory(String name, Type type, ValuesSourceCon
     @Override
     protected Aggregator createUnmapped(Aggregator parent, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData)
             throws IOException {
-        return new Unmapped<Range>(name, ranges, keyed, config.format(), context, parent, rangeFactory, pipelineAggregators, metaData);
+        return new Unmapped<>(name, ranges, keyed, config.format(), context, parent, rangeFactory, pipelineAggregators, metaData);
     }
 
     @Override
@@ -86,16 +85,16 @@ private static class DistanceSource extends ValuesSource.Numeric {
 
         private final ValuesSource.GeoPoint source;
         private final GeoDistance distanceType;
-        private final DistanceUnit unit;
+        private final DistanceUnit units;
         private final org.elasticsearch.common.geo.GeoPoint origin;
 
-        public DistanceSource(ValuesSource.GeoPoint source, GeoDistance distanceType, org.elasticsearch.common.geo.GeoPoint origin,
-                DistanceUnit unit) {
+        DistanceSource(ValuesSource.GeoPoint source, GeoDistance distanceType,
+                org.elasticsearch.common.geo.GeoPoint origin, DistanceUnit units) {
             this.source = source;
             // even if the geo points are unique, there's no guarantee the
             // distances are
             this.distanceType = distanceType;
-            this.unit = unit;
+            this.units = units;
             this.origin = origin;
         }
 
@@ -112,8 +111,7 @@ public SortedNumericDocValues longValues(LeafReaderContext ctx) {
         @Override
         public SortedNumericDoubleValues doubleValues(LeafReaderContext ctx) {
             final MultiGeoPointValues geoValues = source.geoPointValues(ctx);
-            final FixedSourceDistance distance = distanceType.fixedSourceDistance(origin.getLat(), origin.getLon(), unit);
-            return GeoDistance.distanceValues(geoValues, distance);
+            return GeoUtils.distanceValues(distanceType, units, geoValues, origin);
         }
 
         @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/InternalGeoDistance.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/InternalGeoDistance.java
index f01e0233afd79..e636d6d23c7d4 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/InternalGeoDistance.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/InternalGeoDistance.java
@@ -62,11 +62,6 @@ boolean keyed() {
     }
 
     public static class Factory extends InternalRange.Factory<InternalGeoDistance.Bucket, InternalGeoDistance> {
-        @Override
-        public Type type() {
-            return GeoDistanceAggregationBuilder.TYPE;
-        }
-
         @Override
         public ValuesSourceType getValueSourceType() {
             return ValuesSourceType.GEOPOINT;
@@ -119,4 +114,9 @@ public InternalGeoDistance(StreamInput in) throws IOException {
     public InternalRange.Factory<Bucket, InternalGeoDistance> getFactory() {
         return FACTORY;
     }
-}
\ No newline at end of file
+
+    @Override
+    public String getWriteableName() {
+        return GeoDistanceAggregationBuilder.NAME;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/ParsedGeoDistance.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/ParsedGeoDistance.java
new file mode 100644
index 0000000000000..a926499e9249a
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/ParsedGeoDistance.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.range.geodistance;
+
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.bucket.range.ParsedRange;
+
+import java.io.IOException;
+
+public class ParsedGeoDistance extends ParsedRange {
+
+    @Override
+    public String getType() {
+        return GeoDistanceAggregationBuilder.NAME;
+    }
+
+    private static ObjectParser<ParsedGeoDistance, Void> PARSER =
+            new ObjectParser<>(ParsedGeoDistance.class.getSimpleName(), true, ParsedGeoDistance::new);
+    static {
+        declareParsedRangeFields(PARSER,
+                parser -> ParsedBucket.fromXContent(parser, false),
+                parser -> ParsedBucket.fromXContent(parser, true));
+    }
+
+    public static ParsedGeoDistance fromXContent(XContentParser parser, String name) throws IOException {
+        ParsedGeoDistance aggregation = PARSER.parse(parser, null);
+        aggregation.setName(name);
+        return aggregation;
+    }
+
+    public static class ParsedBucket extends ParsedRange.ParsedBucket {
+
+        static ParsedBucket fromXContent(final XContentParser parser, final boolean keyed) throws IOException {
+            return parseRangeBucketXContent(parser, ParsedBucket::new, keyed);
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ip/IpRangeAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ip/IpRangeAggregationBuilder.java
index bd2353b50998a..c530ecbfa9946 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ip/IpRangeAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ip/IpRangeAggregationBuilder.java
@@ -20,25 +20,32 @@
 
 import org.apache.lucene.document.InetAddressPoint;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.network.InetAddresses;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParser.Token;
+import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.script.Script;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.bucket.range.BinaryRangeAggregator;
 import org.elasticsearch.search.aggregations.bucket.range.BinaryRangeAggregatorFactory;
 import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.net.InetAddress;
@@ -52,9 +59,61 @@
 public final class IpRangeAggregationBuilder
         extends ValuesSourceAggregationBuilder<ValuesSource.Bytes, IpRangeAggregationBuilder> {
     public static final String NAME = "ip_range";
-    private static final InternalAggregation.Type TYPE = new InternalAggregation.Type(NAME);
+    private static final ParseField MASK_FIELD = new ParseField("mask");
+
+    private static final ObjectParser<IpRangeAggregationBuilder, QueryParseContext> PARSER;
+    static {
+        PARSER = new ObjectParser<>(IpRangeAggregationBuilder.NAME);
+        ValuesSourceParserHelper.declareBytesFields(PARSER, false, false);
+
+        PARSER.declareBoolean(IpRangeAggregationBuilder::keyed, RangeAggregator.KEYED_FIELD);
+
+        PARSER.declareObjectArray((agg, ranges) -> {
+            for (Range range : ranges) agg.addRange(range);
+        }, IpRangeAggregationBuilder::parseRange, RangeAggregator.RANGES_FIELD);
+    }
+
+    public static AggregationBuilder parse(String aggregationName, QueryParseContext context) throws IOException {
+        return PARSER.parse(context.parser(), new IpRangeAggregationBuilder(aggregationName), context);
+    }
+
+    private static Range parseRange(XContentParser parser, QueryParseContext context) throws IOException {
+        String key = null;
+        String from = null;
+        String to = null;
+        String mask = null;
+
+        if (parser.currentToken() != Token.START_OBJECT) {
+            throw new ParsingException(parser.getTokenLocation(), "[ranges] must contain objects, but hit a " + parser.currentToken());
+        }
+        while (parser.nextToken() != Token.END_OBJECT) {
+            if (parser.currentToken() == Token.FIELD_NAME) {
+                continue;
+            }
+            if (RangeAggregator.Range.KEY_FIELD.match(parser.currentName())) {
+                key = parser.text();
+            } else if (RangeAggregator.Range.FROM_FIELD.match(parser.currentName())) {
+                from = parser.textOrNull();
+            } else if (RangeAggregator.Range.TO_FIELD.match(parser.currentName())) {
+                to = parser.textOrNull();
+            } else if (MASK_FIELD.match(parser.currentName())) {
+                mask = parser.text();
+            } else {
+                throw new ParsingException(parser.getTokenLocation(), "Unexpected ip range parameter: [" + parser.currentName() + "]");
+            }
+        }
+        if (mask != null) {
+            if (key == null) {
+                key = mask;
+            }
+            return new Range(key, mask);
+        } else {
+            return new Range(key, from, to);
+        }
+    }
 
     public static class Range implements ToXContent {
+
         private final String key;
         private final String from;
         private final String to;
@@ -94,8 +153,18 @@ public static class Range implements ToXContent {
             }
             this.key = key;
             try {
-                this.from = InetAddresses.toAddrString(InetAddress.getByAddress(lower));
-                this.to = InetAddresses.toAddrString(InetAddress.getByAddress(upper));
+                InetAddress fromAddress = InetAddress.getByAddress(lower);
+                if (fromAddress.equals(InetAddressPoint.MIN_VALUE)) {
+                    this.from = null;
+                } else {
+                    this.from = InetAddresses.toAddrString(fromAddress);
+                }
+                InetAddress inclusiveToAddress = InetAddress.getByAddress(upper);
+                if (inclusiveToAddress.equals(InetAddressPoint.MAX_VALUE)) {
+                    this.to = null;
+                } else {
+                    this.to = InetAddresses.toAddrString(InetAddressPoint.nextUp(inclusiveToAddress));
+                }
             } catch (UnknownHostException bogus) {
                 throw new AssertionError(bogus);
             }
@@ -162,11 +231,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     private List<Range> ranges = new ArrayList<>();
 
     public IpRangeAggregationBuilder(String name) {
-        super(name, TYPE, ValuesSourceType.BYTES, ValueType.IP);
+        super(name, ValuesSourceType.BYTES, ValueType.IP);
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 
@@ -268,7 +337,7 @@ public IpRangeAggregationBuilder addUnboundedFrom(String from) {
     }
 
     public IpRangeAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE, ValuesSourceType.BYTES, ValueType.IP);
+        super(in, ValuesSourceType.BYTES, ValueType.IP);
         final int numRanges = in.readVInt();
         for (int i = 0; i < numRanges; ++i) {
             addRange(new Range(in));
@@ -296,14 +365,17 @@ private static BytesRef toBytesRef(String ip) {
 
     @Override
     protected ValuesSourceAggregatorFactory<ValuesSource.Bytes, ?> innerBuild(
-            AggregationContext context, ValuesSourceConfig<ValuesSource.Bytes> config,
+            SearchContext context, ValuesSourceConfig<ValuesSource.Bytes> config,
             AggregatorFactory<?> parent, Builder subFactoriesBuilder)
                     throws IOException {
         List<BinaryRangeAggregator.Range> ranges = new ArrayList<>();
+        if(this.ranges.size() == 0){
+            throw new IllegalArgumentException("No [ranges] specified for the [" + this.getName() + "] aggregation");
+        }
         for (Range range : this.ranges) {
             ranges.add(new BinaryRangeAggregator.Range(range.key, toBytesRef(range.from), toBytesRef(range.to)));
         }
-        return new BinaryRangeAggregatorFactory(name, TYPE, config, ranges,
+        return new BinaryRangeAggregatorFactory(name, config, ranges,
                 keyed, context, parent, subFactoriesBuilder, metaData);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ip/IpRangeParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ip/IpRangeParser.java
deleted file mode 100644
index 5d95f0dd4944e..0000000000000
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ip/IpRangeParser.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.search.aggregations.bucket.range.ip;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentParser.Token;
-import org.elasticsearch.search.aggregations.support.AbstractValuesSourceParser.BytesValuesSourceParser;
-import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator;
-import org.elasticsearch.search.aggregations.bucket.range.ip.IpRangeAggregationBuilder.Range;
-import org.elasticsearch.search.aggregations.support.XContentParseContext;
-import org.elasticsearch.search.aggregations.support.ValueType;
-import org.elasticsearch.search.aggregations.support.ValuesSource;
-import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
-import org.elasticsearch.search.aggregations.support.ValuesSourceType;
-
-/**
- * A parser for ip range aggregations.
- */
-public class IpRangeParser extends BytesValuesSourceParser {
-
-    private static final ParseField MASK_FIELD = new ParseField("mask");
-
-    public IpRangeParser() {
-        super(false, false);
-    }
-
-    @Override
-    protected ValuesSourceAggregationBuilder<ValuesSource.Bytes, ?> createFactory(
-            String aggregationName, ValuesSourceType valuesSourceType,
-            ValueType targetValueType, Map<ParseField, Object> otherOptions) {
-        IpRangeAggregationBuilder range = new IpRangeAggregationBuilder(aggregationName);
-        @SuppressWarnings("unchecked")
-        Iterable<Range> ranges = (Iterable<Range>) otherOptions.get(RangeAggregator.RANGES_FIELD);
-        if (otherOptions.containsKey(RangeAggregator.RANGES_FIELD)) {
-            for (Range r : ranges) {
-                range.addRange(r);
-            }
-        }
-        if (otherOptions.containsKey(RangeAggregator.KEYED_FIELD)) {
-            range.keyed((Boolean) otherOptions.get(RangeAggregator.KEYED_FIELD));
-        }
-        return range;
-    }
-
-    private Range parseRange(XContentParser parser, ParseFieldMatcher parseFieldMatcher) throws IOException {
-        String key = null;
-        String from = null;
-        String to = null;
-        String mask = null;
-
-        if (parser.currentToken() != Token.START_OBJECT) {
-            throw new ParsingException(parser.getTokenLocation(), "[ranges] must contain objects, but hit a " + parser.currentToken());
-        }
-        while (parser.nextToken() != Token.END_OBJECT) {
-            if (parser.currentToken() == Token.FIELD_NAME) {
-                continue;
-            }
-            if (parseFieldMatcher.match(parser.currentName(), RangeAggregator.Range.KEY_FIELD)) {
-                key = parser.text();
-            } else if (parseFieldMatcher.match(parser.currentName(), RangeAggregator.Range.FROM_FIELD)) {
-                from = parser.text();
-            } else if (parseFieldMatcher.match(parser.currentName(), RangeAggregator.Range.TO_FIELD)) {
-                to = parser.text();
-            } else if (parseFieldMatcher.match(parser.currentName(), MASK_FIELD)) {
-                mask = parser.text();
-            } else {
-                throw new ParsingException(parser.getTokenLocation(), "Unexpected ip range parameter: [" + parser.currentName() + "]");
-            }
-        }
-        if (mask != null) {
-            if (key == null) {
-                key = mask;
-            }
-            return new Range(key, mask);
-        } else {
-            return new Range(key, from, to);
-        }
-    }
-
-    @Override
-    protected boolean token(String aggregationName, String currentFieldName,
-                            Token token,
-                            XContentParseContext context,
-                            Map<ParseField, Object> otherOptions) throws IOException {
-        XContentParser parser = context.getParser();
-        if (context.matchField(currentFieldName, RangeAggregator.RANGES_FIELD)) {
-            if (parser.currentToken() != Token.START_ARRAY) {
-                throw new ParsingException(parser.getTokenLocation(), "[ranges] must be passed as an array, but got a " + token);
-            }
-            List<Range> ranges = new ArrayList<>();
-            while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
-                Range range = parseRange(parser, context.getParseFieldMatcher());
-                ranges.add(range);
-            }
-            otherOptions.put(RangeAggregator.RANGES_FIELD, ranges);
-            return true;
-        } else if (context.matchField(parser.currentName(), RangeAggregator.KEYED_FIELD)) {
-            otherOptions.put(RangeAggregator.KEYED_FIELD, parser.booleanValue());
-            return true;
-        }
-        return false;
-    }
-
-}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedAggregationBuilder.java
index 8a83fabcab28e..78f5bd0a7afd8 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedAggregationBuilder.java
@@ -21,39 +21,54 @@
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.Objects;
 
 public class DiversifiedAggregationBuilder extends ValuesSourceAggregationBuilder<ValuesSource, DiversifiedAggregationBuilder> {
     public static final String NAME = "diversified_sampler";
-    public static final Type TYPE = new Type(NAME);
 
     public static final int MAX_DOCS_PER_VALUE_DEFAULT = 1;
 
+    private static final ObjectParser<DiversifiedAggregationBuilder, QueryParseContext> PARSER;
+    static {
+        PARSER = new ObjectParser<>(DiversifiedAggregationBuilder.NAME);
+        ValuesSourceParserHelper.declareAnyFields(PARSER, true, false);
+        PARSER.declareInt(DiversifiedAggregationBuilder::shardSize, SamplerAggregator.SHARD_SIZE_FIELD);
+        PARSER.declareInt(DiversifiedAggregationBuilder::maxDocsPerValue, SamplerAggregator.MAX_DOCS_PER_VALUE_FIELD);
+        PARSER.declareString(DiversifiedAggregationBuilder::executionHint, SamplerAggregator.EXECUTION_HINT_FIELD);
+    }
+
+    public static AggregationBuilder parse(String aggregationName, QueryParseContext context) throws IOException {
+        return PARSER.parse(context.parser(), new DiversifiedAggregationBuilder(aggregationName), context);
+    }
+
     private int shardSize = SamplerAggregationBuilder.DEFAULT_SHARD_SAMPLE_SIZE;
     private int maxDocsPerValue = MAX_DOCS_PER_VALUE_DEFAULT;
     private String executionHint = null;
 
     public DiversifiedAggregationBuilder(String name) {
-        super(name, TYPE, ValuesSourceType.ANY, null);
+        super(name, ValuesSourceType.ANY, null);
     }
 
     /**
      * Read from a stream.
      */
     public DiversifiedAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE, ValuesSourceType.ANY, null);
+        super(in, ValuesSourceType.ANY, null);
         shardSize = in.readVInt();
         maxDocsPerValue = in.readVInt();
         executionHint = in.readOptionalString();
@@ -120,9 +135,9 @@ public String executionHint() {
     }
 
     @Override
-    protected ValuesSourceAggregatorFactory<ValuesSource, ?> innerBuild(AggregationContext context,
+    protected ValuesSourceAggregatorFactory<ValuesSource, ?> innerBuild(SearchContext context,
             ValuesSourceConfig<ValuesSource> config, AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
-        return new DiversifiedAggregatorFactory(name, TYPE, config, shardSize, maxDocsPerValue, executionHint, context, parent,
+        return new DiversifiedAggregatorFactory(name, config, shardSize, maxDocsPerValue, executionHint, context, parent,
                 subFactoriesBuilder, metaData);
     }
 
@@ -150,7 +165,7 @@ protected boolean innerEquals(Object obj) {
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedAggregatorFactory.java
index a39135192bc19..97a68649ca233 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedAggregatorFactory.java
@@ -25,14 +25,13 @@
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.NonCollectingAggregator;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.bucket.sampler.SamplerAggregator.ExecutionMode;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
-import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -44,10 +43,10 @@ public class DiversifiedAggregatorFactory extends ValuesSourceAggregatorFactory<
     private final int maxDocsPerValue;
     private final String executionHint;
 
-    public DiversifiedAggregatorFactory(String name, Type type, ValuesSourceConfig<ValuesSource> config, int shardSize, int maxDocsPerValue,
-            String executionHint, AggregationContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
+    public DiversifiedAggregatorFactory(String name, ValuesSourceConfig<ValuesSource> config, int shardSize, int maxDocsPerValue,
+            String executionHint, SearchContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
             Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.shardSize = shardSize;
         this.maxDocsPerValue = maxDocsPerValue;
         this.executionHint = executionHint;
@@ -65,7 +64,7 @@ protected Aggregator doCreateInternal(ValuesSource valuesSource, Aggregator pare
         if (valuesSource instanceof ValuesSource.Bytes) {
             ExecutionMode execution = null;
             if (executionHint != null) {
-                execution = ExecutionMode.fromString(executionHint, context.searchContext().parseFieldMatcher());
+                execution = ExecutionMode.fromString(executionHint);
             }
 
             // In some cases using ordinals is just not supported: override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedBytesHashSamplerAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedBytesHashSamplerAggregator.java
index b2f7be614dba8..f4d45a7d471b8 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedBytesHashSamplerAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedBytesHashSamplerAggregator.java
@@ -32,8 +32,8 @@
 import org.elasticsearch.search.aggregations.bucket.BestDocsDeferringCollector;
 import org.elasticsearch.search.aggregations.bucket.DeferringBucketCollector;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -49,10 +49,10 @@ public class DiversifiedBytesHashSamplerAggregator extends SamplerAggregator {
     private int maxDocsPerValue;
 
     public DiversifiedBytesHashSamplerAggregator(String name, int shardSize, AggregatorFactories factories,
-            AggregationContext aggregationContext, Aggregator parent, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData,
+            SearchContext context, Aggregator parent, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData,
             ValuesSource valuesSource,
             int maxDocsPerValue) throws IOException {
-        super(name, shardSize, factories, aggregationContext, parent, pipelineAggregators, metaData);
+        super(name, shardSize, factories, context, parent, pipelineAggregators, metaData);
         this.valuesSource = valuesSource;
         this.maxDocsPerValue = maxDocsPerValue;
     }
@@ -70,7 +70,7 @@ public DeferringBucketCollector getDeferringCollector() {
      */
     class DiverseDocsDeferringCollector extends BestDocsDeferringCollector {
 
-        public DiverseDocsDeferringCollector() {
+        DiverseDocsDeferringCollector() {
             super(shardSize, context.bigArrays());
         }
 
@@ -86,7 +86,7 @@ class ValuesDiversifiedTopDocsCollector extends DiversifiedTopDocsCollector {
 
             private SortedBinaryDocValues values;
 
-            public ValuesDiversifiedTopDocsCollector(int numHits, int maxHitsPerValue) {
+            ValuesDiversifiedTopDocsCollector(int numHits, int maxHitsPerValue) {
                 super(numHits, maxHitsPerValue);
 
             }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedMapSamplerAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedMapSamplerAggregator.java
index 7578296cee823..08fa1bcb7fae0 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedMapSamplerAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedMapSamplerAggregator.java
@@ -34,8 +34,8 @@
 import org.elasticsearch.search.aggregations.bucket.BestDocsDeferringCollector;
 import org.elasticsearch.search.aggregations.bucket.DeferringBucketCollector;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -48,12 +48,12 @@ public class DiversifiedMapSamplerAggregator extends SamplerAggregator {
     private BytesRefHash bucketOrds;
 
     public DiversifiedMapSamplerAggregator(String name, int shardSize, AggregatorFactories factories,
-            AggregationContext aggregationContext, Aggregator parent, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData,
+            SearchContext context, Aggregator parent, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData,
             ValuesSource valuesSource, int maxDocsPerValue) throws IOException {
-        super(name, shardSize, factories, aggregationContext, parent, pipelineAggregators, metaData);
+        super(name, shardSize, factories, context, parent, pipelineAggregators, metaData);
         this.valuesSource = valuesSource;
         this.maxDocsPerValue = maxDocsPerValue;
-        bucketOrds = new BytesRefHash(shardSize, aggregationContext.bigArrays());
+        bucketOrds = new BytesRefHash(shardSize, context.bigArrays());
 
     }
 
@@ -76,7 +76,7 @@ public DeferringBucketCollector getDeferringCollector() {
      */
     class DiverseDocsDeferringCollector extends BestDocsDeferringCollector {
 
-        public DiverseDocsDeferringCollector() {
+        DiverseDocsDeferringCollector() {
             super(shardSize, context.bigArrays());
         }
 
@@ -92,7 +92,7 @@ class ValuesDiversifiedTopDocsCollector extends DiversifiedTopDocsCollector {
 
             private SortedBinaryDocValues values;
 
-            public ValuesDiversifiedTopDocsCollector(int numHits, int maxHitsPerKey) {
+            ValuesDiversifiedTopDocsCollector(int numHits, int maxHitsPerKey) {
                 super(numHits, maxHitsPerKey);
 
             }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedNumericSamplerAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedNumericSamplerAggregator.java
index 430288db83b83..c595fdb5c25d8 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedNumericSamplerAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedNumericSamplerAggregator.java
@@ -31,8 +31,8 @@
 import org.elasticsearch.search.aggregations.bucket.BestDocsDeferringCollector;
 import org.elasticsearch.search.aggregations.bucket.DeferringBucketCollector;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -44,9 +44,9 @@ public class DiversifiedNumericSamplerAggregator extends SamplerAggregator {
     private int maxDocsPerValue;
 
     public DiversifiedNumericSamplerAggregator(String name, int shardSize, AggregatorFactories factories,
-            AggregationContext aggregationContext, Aggregator parent, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData,
+            SearchContext context, Aggregator parent, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData,
             ValuesSource.Numeric valuesSource, int maxDocsPerValue) throws IOException {
-        super(name, shardSize, factories, aggregationContext, parent, pipelineAggregators, metaData);
+        super(name, shardSize, factories, context, parent, pipelineAggregators, metaData);
         this.valuesSource = valuesSource;
         this.maxDocsPerValue = maxDocsPerValue;
     }
@@ -63,7 +63,7 @@ public DeferringBucketCollector getDeferringCollector() {
      * This implementation is only for use with a single bucket aggregation.
      */
     class DiverseDocsDeferringCollector extends BestDocsDeferringCollector {
-        public DiverseDocsDeferringCollector() {
+        DiverseDocsDeferringCollector() {
             super(shardSize, context.bigArrays());
         }
 
@@ -78,7 +78,7 @@ class ValuesDiversifiedTopDocsCollector extends DiversifiedTopDocsCollector {
 
             private SortedNumericDocValues values;
 
-            public ValuesDiversifiedTopDocsCollector(int numHits, int maxHitsPerKey) {
+            ValuesDiversifiedTopDocsCollector(int numHits, int maxHitsPerKey) {
                 super(numHits, maxHitsPerKey);
             }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedOrdinalsSamplerAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedOrdinalsSamplerAggregator.java
index 3725e10335c1b..5eb37f310add5 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedOrdinalsSamplerAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedOrdinalsSamplerAggregator.java
@@ -32,8 +32,8 @@
 import org.elasticsearch.search.aggregations.bucket.BestDocsDeferringCollector;
 import org.elasticsearch.search.aggregations.bucket.DeferringBucketCollector;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -45,9 +45,9 @@ public class DiversifiedOrdinalsSamplerAggregator extends SamplerAggregator {
     private int maxDocsPerValue;
 
     public DiversifiedOrdinalsSamplerAggregator(String name, int shardSize, AggregatorFactories factories,
-            AggregationContext aggregationContext, Aggregator parent, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData,
+            SearchContext context, Aggregator parent, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData,
             ValuesSource.Bytes.WithOrdinals.FieldData valuesSource, int maxDocsPerValue) throws IOException {
-        super(name, shardSize, factories, aggregationContext, parent, pipelineAggregators, metaData);
+        super(name, shardSize, factories, context, parent, pipelineAggregators, metaData);
         this.valuesSource = valuesSource;
         this.maxDocsPerValue = maxDocsPerValue;
     }
@@ -65,7 +65,7 @@ public DeferringBucketCollector getDeferringCollector() {
      */
     class DiverseDocsDeferringCollector extends BestDocsDeferringCollector {
 
-        public DiverseDocsDeferringCollector() {
+        DiverseDocsDeferringCollector() {
             super(shardSize, context.bigArrays());
         }
 
@@ -79,7 +79,7 @@ protected TopDocsCollector<ScoreDocKey> createTopDocsCollector(int size) {
         class ValuesDiversifiedTopDocsCollector extends DiversifiedTopDocsCollector {
 
 
-            public ValuesDiversifiedTopDocsCollector(int numHits, int maxHitsPerKey) {
+            ValuesDiversifiedTopDocsCollector(int numHits, int maxHitsPerKey) {
                 super(numHits, maxHitsPerKey);
             }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedSamplerParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedSamplerParser.java
deleted file mode 100644
index a62035d7234fc..0000000000000
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedSamplerParser.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.search.aggregations.bucket.sampler;
-
-
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.search.aggregations.support.AbstractValuesSourceParser.AnyValuesSourceParser;
-import org.elasticsearch.search.aggregations.support.XContentParseContext;
-import org.elasticsearch.search.aggregations.support.ValueType;
-import org.elasticsearch.search.aggregations.support.ValuesSourceType;
-
-import java.io.IOException;
-import java.util.Map;
-
-/**
- *
- */
-public class DiversifiedSamplerParser extends AnyValuesSourceParser {
-    public DiversifiedSamplerParser() {
-        super(true, false);
-    }
-
-    @Override
-    protected DiversifiedAggregationBuilder createFactory(String aggregationName, ValuesSourceType valuesSourceType,
-                                                          ValueType targetValueType, Map<ParseField, Object> otherOptions) {
-        DiversifiedAggregationBuilder factory = new DiversifiedAggregationBuilder(aggregationName);
-        Integer shardSize = (Integer) otherOptions.get(SamplerAggregator.SHARD_SIZE_FIELD);
-        if (shardSize != null) {
-            factory.shardSize(shardSize);
-        }
-        Integer maxDocsPerValue = (Integer) otherOptions.get(SamplerAggregator.MAX_DOCS_PER_VALUE_FIELD);
-        if (maxDocsPerValue != null) {
-            factory.maxDocsPerValue(maxDocsPerValue);
-        }
-        String executionHint = (String) otherOptions.get(SamplerAggregator.EXECUTION_HINT_FIELD);
-        if (executionHint != null) {
-            factory.executionHint(executionHint);
-        }
-        return factory;
-    }
-
-    @Override
-    protected boolean token(String aggregationName, String currentFieldName, XContentParser.Token token,
-                            XContentParseContext context, Map<ParseField, Object> otherOptions) throws IOException {
-        XContentParser parser = context.getParser();
-        if (token == XContentParser.Token.VALUE_NUMBER) {
-            if (context.matchField(currentFieldName, SamplerAggregator.SHARD_SIZE_FIELD)) {
-                int shardSize = parser.intValue();
-                otherOptions.put(SamplerAggregator.SHARD_SIZE_FIELD, shardSize);
-                return true;
-            } else if (context.matchField(currentFieldName, SamplerAggregator.MAX_DOCS_PER_VALUE_FIELD)) {
-                int maxDocsPerValue = parser.intValue();
-                otherOptions.put(SamplerAggregator.MAX_DOCS_PER_VALUE_FIELD, maxDocsPerValue);
-                return true;
-            }
-        } else if (token == XContentParser.Token.VALUE_STRING) {
-            if (context.matchField(currentFieldName, SamplerAggregator.EXECUTION_HINT_FIELD)) {
-                String executionHint = parser.text();
-                otherOptions.put(SamplerAggregator.EXECUTION_HINT_FIELD, executionHint);
-                return true;
-            }
-        }
-        return false;
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/InternalSampler.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/InternalSampler.java
index 224de07e93d1c..1a04133e82bfc 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/InternalSampler.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/InternalSampler.java
@@ -29,7 +29,11 @@
 
 public class InternalSampler extends InternalSingleBucketAggregation implements Sampler {
     public static final String NAME = "mapped_sampler";
-    InternalSampler(String name, long docCount, InternalAggregations subAggregations, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
+    // InternalSampler and UnmappedSampler share the same parser name, so we use this when identifying the aggregation type
+    public static final String PARSER_NAME = "sampler";
+
+    InternalSampler(String name, long docCount, InternalAggregations subAggregations, List<PipelineAggregator> pipelineAggregators,
+            Map<String, Object> metaData) {
         super(name, docCount, subAggregations, pipelineAggregators, metaData);
     }
 
@@ -45,6 +49,11 @@ public String getWriteableName() {
         return NAME;
     }
 
+    @Override
+    public String getType() {
+        return PARSER_NAME;
+    }
+
     @Override
     protected InternalSingleBucketAggregation newAggregation(String name, long docCount,
             InternalAggregations subAggregations) {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/ParsedSampler.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/ParsedSampler.java
new file mode 100644
index 0000000000000..3d5e946beadfc
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/ParsedSampler.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.bucket.sampler;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.bucket.ParsedSingleBucketAggregation;
+
+import java.io.IOException;
+
+public class ParsedSampler extends ParsedSingleBucketAggregation implements Sampler {
+
+    @Override
+    public String getType() {
+        return InternalSampler.PARSER_NAME;
+    }
+
+    public static ParsedSampler fromXContent(XContentParser parser, final String name) throws IOException {
+        return parseXContent(parser, new ParsedSampler(), name);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregationBuilder.java
index 2230408b033b9..f69b66ffd1eee 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregationBuilder.java
@@ -28,29 +28,27 @@
 import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.Objects;
 
 public class SamplerAggregationBuilder extends AbstractAggregationBuilder<SamplerAggregationBuilder> {
     public static final String NAME = "sampler";
-    private static final Type TYPE = new Type(NAME);
 
     public static final int DEFAULT_SHARD_SAMPLE_SIZE = 100;
 
     private int shardSize = DEFAULT_SHARD_SAMPLE_SIZE;
 
     public SamplerAggregationBuilder(String name) {
-        super(name, TYPE);
+        super(name);
     }
 
     /**
      * Read from a stream.
      */
     public SamplerAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE);
+        super(in);
         shardSize = in.readVInt();
     }
 
@@ -75,9 +73,9 @@ public int shardSize() {
     }
 
     @Override
-    protected SamplerAggregatorFactory doBuild(AggregationContext context, AggregatorFactory<?> parent, Builder subFactoriesBuilder)
+    protected SamplerAggregatorFactory doBuild(SearchContext context, AggregatorFactory<?> parent, Builder subFactoriesBuilder)
             throws IOException {
-        return new SamplerAggregatorFactory(name, type, shardSize, context, parent, subFactoriesBuilder, metaData);
+        return new SamplerAggregatorFactory(name, shardSize, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
@@ -98,7 +96,7 @@ public static SamplerAggregationBuilder parse(String aggregationName, QueryParse
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.VALUE_NUMBER) {
-                if (context.getParseFieldMatcher().match(currentFieldName, SamplerAggregator.SHARD_SIZE_FIELD)) {
+                if (SamplerAggregator.SHARD_SIZE_FIELD.match(currentFieldName)) {
                     shardSize = parser.intValue();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
@@ -129,7 +127,7 @@ protected boolean doEquals(Object obj) {
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregator.java
index cec2c4577dfdf..9e0fb17a56234 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregator.java
@@ -20,7 +20,6 @@
 
 import org.apache.lucene.index.LeafReaderContext;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.search.aggregations.AggregationExecutionException;
 import org.elasticsearch.search.aggregations.Aggregator;
@@ -31,10 +30,11 @@
 import org.elasticsearch.search.aggregations.bucket.DeferringBucketCollector;
 import org.elasticsearch.search.aggregations.bucket.SingleBucketAggregator;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 
@@ -61,7 +61,7 @@ public enum ExecutionMode {
 
             @Override
             Aggregator create(String name, AggregatorFactories factories, int shardSize, int maxDocsPerValue, ValuesSource valuesSource,
-                    AggregationContext context, Aggregator parent, List<PipelineAggregator> pipelineAggregators,
+                    SearchContext context, Aggregator parent, List<PipelineAggregator> pipelineAggregators,
                     Map<String, Object> metaData) throws IOException {
 
                 return new DiversifiedMapSamplerAggregator(name, shardSize, factories, context, parent, pipelineAggregators, metaData,
@@ -79,7 +79,7 @@ boolean needsGlobalOrdinals() {
 
             @Override
             Aggregator create(String name, AggregatorFactories factories, int shardSize, int maxDocsPerValue, ValuesSource valuesSource,
-                    AggregationContext context, Aggregator parent, List<PipelineAggregator> pipelineAggregators,
+                    SearchContext context, Aggregator parent, List<PipelineAggregator> pipelineAggregators,
                     Map<String, Object> metaData) throws IOException {
 
                 return new DiversifiedBytesHashSamplerAggregator(name, shardSize, factories, context, parent, pipelineAggregators,
@@ -98,7 +98,7 @@ boolean needsGlobalOrdinals() {
 
             @Override
             Aggregator create(String name, AggregatorFactories factories, int shardSize, int maxDocsPerValue, ValuesSource valuesSource,
-                    AggregationContext context, Aggregator parent, List<PipelineAggregator> pipelineAggregators,
+                    SearchContext context, Aggregator parent, List<PipelineAggregator> pipelineAggregators,
                     Map<String, Object> metaData) throws IOException {
                 return new DiversifiedOrdinalsSamplerAggregator(name, shardSize, factories, context, parent, pipelineAggregators, metaData,
                         (ValuesSource.Bytes.WithOrdinals.FieldData) valuesSource, maxDocsPerValue);
@@ -111,13 +111,13 @@ boolean needsGlobalOrdinals() {
 
         };
 
-        public static ExecutionMode fromString(String value, ParseFieldMatcher parseFieldMatcher) {
+        public static ExecutionMode fromString(String value) {
             for (ExecutionMode mode : values()) {
-                if (parseFieldMatcher.match(value, mode.parseField)) {
+                if (mode.parseField.match(value)) {
                     return mode;
                 }
             }
-            throw new IllegalArgumentException("Unknown `execution_hint`: [" + value + "], expected any of " + values());
+            throw new IllegalArgumentException("Unknown `execution_hint`: [" + value + "], expected any of " + Arrays.toString(values()));
         }
 
         private final ParseField parseField;
@@ -126,8 +126,8 @@ public static ExecutionMode fromString(String value, ParseFieldMatcher parseFiel
             this.parseField = parseField;
         }
 
-        abstract Aggregator create(String name, AggregatorFactories factories, int shardSize, int maxDocsPerValue, ValuesSource valuesSource,
- AggregationContext context, Aggregator parent, List<PipelineAggregator> pipelineAggregators,
+        abstract Aggregator create(String name, AggregatorFactories factories, int shardSize, int maxDocsPerValue,
+                ValuesSource valuesSource, SearchContext context, Aggregator parent, List<PipelineAggregator> pipelineAggregators,
                 Map<String, Object> metaData) throws IOException;
 
         abstract boolean needsGlobalOrdinals();
@@ -142,9 +142,9 @@ public String toString() {
     protected final int shardSize;
     protected BestDocsDeferringCollector bdd;
 
-    public SamplerAggregator(String name, int shardSize, AggregatorFactories factories, AggregationContext aggregationContext,
+    public SamplerAggregator(String name, int shardSize, AggregatorFactories factories, SearchContext context,
             Aggregator parent, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException {
-        super(name, factories, aggregationContext, parent, pipelineAggregators, metaData);
+        super(name, factories, context, parent, pipelineAggregators, metaData);
         this.shardSize = shardSize;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregatorFactory.java
index 09f510f80e5a4..4fb7e28f3d638 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregatorFactory.java
@@ -22,9 +22,8 @@
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -34,9 +33,9 @@ public class SamplerAggregatorFactory extends AggregatorFactory<SamplerAggregato
 
     private final int shardSize;
 
-    public SamplerAggregatorFactory(String name, Type type, int shardSize, AggregationContext context, AggregatorFactory<?> parent,
+    public SamplerAggregatorFactory(String name, int shardSize, SearchContext context, AggregatorFactory<?> parent,
             AggregatorFactories.Builder subFactories, Map<String, Object> metaData) throws IOException {
-        super(name, type, context, parent, subFactories, metaData);
+        super(name, context, parent, subFactories, metaData);
         this.shardSize = shardSize;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/UnmappedSampler.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/UnmappedSampler.java
index 1cf77baa1f0fd..b30f1c48beec6 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/UnmappedSampler.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/UnmappedSampler.java
@@ -20,6 +20,7 @@
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.search.aggregations.Aggregation;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
@@ -59,7 +60,7 @@ public InternalAggregation doReduce(List<InternalAggregation> aggregations, Redu
 
     @Override
     public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
-        builder.field(InternalAggregation.CommonFields.DOC_COUNT, 0);
+        builder.field(Aggregation.CommonFields.DOC_COUNT.getPreferredName(), 0);
         return builder;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/GlobalOrdinalsSignificantTermsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/GlobalOrdinalsSignificantTermsAggregator.java
index 1aeacf4be60f2..0c5835a54e41e 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/GlobalOrdinalsSignificantTermsAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/GlobalOrdinalsSignificantTermsAggregator.java
@@ -22,7 +22,6 @@
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.lease.Releasables;
-import org.elasticsearch.common.util.LongHash;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
@@ -32,9 +31,9 @@
 import org.elasticsearch.search.aggregations.bucket.terms.GlobalOrdinalsStringTermsAggregator;
 import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.internal.ContextIndexSearcher;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -52,22 +51,28 @@ public class GlobalOrdinalsSignificantTermsAggregator extends GlobalOrdinalsStri
     protected final SignificantTermsAggregatorFactory termsAggFactory;
     private final SignificanceHeuristic significanceHeuristic;
 
-    public GlobalOrdinalsSignificantTermsAggregator(String name, AggregatorFactories factories,
-            ValuesSource.Bytes.WithOrdinals.FieldData valuesSource, DocValueFormat format,
-            BucketCountThresholds bucketCountThresholds, IncludeExclude.OrdinalsFilter includeExclude,
-            AggregationContext aggregationContext, Aggregator parent,
-            SignificanceHeuristic significanceHeuristic, SignificantTermsAggregatorFactory termsAggFactory,
-            List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException {
-
-        super(name, factories, valuesSource, null, format, bucketCountThresholds, includeExclude, aggregationContext, parent,
-                SubAggCollectionMode.DEPTH_FIRST, false, pipelineAggregators, metaData);
+    public GlobalOrdinalsSignificantTermsAggregator(String name,
+                                                    AggregatorFactories factories,
+                                                    ValuesSource.Bytes.WithOrdinals.FieldData valuesSource,
+                                                    DocValueFormat format,
+                                                    BucketCountThresholds bucketCountThresholds,
+                                                    IncludeExclude.OrdinalsFilter includeExclude,
+                                                    SearchContext context,
+                                                    Aggregator parent,
+                                                    boolean forceRemapGlobalOrds,
+                                                    SignificanceHeuristic significanceHeuristic,
+                                                    SignificantTermsAggregatorFactory termsAggFactory,
+                                                    List<PipelineAggregator> pipelineAggregators,
+                                                    Map<String, Object> metaData) throws IOException {
+        super(name, factories, valuesSource, null, format, bucketCountThresholds, includeExclude, context, parent,
+            forceRemapGlobalOrds, SubAggCollectionMode.DEPTH_FIRST, false, pipelineAggregators, metaData);
         this.significanceHeuristic = significanceHeuristic;
         this.termsAggFactory = termsAggFactory;
+        this.numCollectedDocs = 0;
     }
 
     @Override
-    public LeafBucketCollector getLeafCollector(LeafReaderContext ctx,
-            final LeafBucketCollector sub) throws IOException {
+    public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, final LeafBucketCollector sub) throws IOException {
         return new LeafBucketCollectorBase(super.getLeafCollector(ctx, sub), null) {
             @Override
             public void collect(int doc, long bucket) throws IOException {
@@ -77,18 +82,17 @@ public void collect(int doc, long bucket) throws IOException {
         };
     }
 
-
     @Override
     public SignificantStringTerms buildAggregation(long owningBucketOrdinal) throws IOException {
         assert owningBucketOrdinal == 0;
-        if (globalOrds == null) { // no context in this reader
+        if (valueCount == 0) { // no context in this reader
             return buildEmptyAggregation();
         }
 
         final int size;
         if (bucketCountThresholds.getMinDocCount() == 0) {
             // if minDocCount == 0 then we can end up with more buckets then maxBucketOrd() returns
-            size = (int) Math.min(globalOrds.getValueCount(), bucketCountThresholds.getShardSize());
+            size = (int) Math.min(valueCount, bucketCountThresholds.getShardSize());
         } else {
             size = (int) Math.min(maxBucketOrd(), bucketCountThresholds.getShardSize());
         }
@@ -97,7 +101,7 @@ public SignificantStringTerms buildAggregation(long owningBucketOrdinal) throws
 
         BucketSignificancePriorityQueue<SignificantStringTerms.Bucket> ordered = new BucketSignificancePriorityQueue<>(size);
         SignificantStringTerms.Bucket spare = null;
-        for (long globalTermOrd = 0; globalTermOrd < globalOrds.getValueCount(); ++globalTermOrd) {
+        for (long globalTermOrd = 0; globalTermOrd < valueCount; ++globalTermOrd) {
             if (includeExclude != null && !acceptedGlobalOrdinals.get(globalTermOrd)) {
                 continue;
             }
@@ -114,7 +118,7 @@ public SignificantStringTerms buildAggregation(long owningBucketOrdinal) throws
                 spare = new SignificantStringTerms.Bucket(new BytesRef(), 0, 0, 0, 0, null, format);
             }
             spare.bucketOrd = bucketOrd;
-            copy(globalOrds.lookupOrd(globalTermOrd), spare.termBytes);
+            copy(lookupGlobalOrd.apply(globalTermOrd), spare.termBytes);
             spare.subsetDf = bucketDocCount;
             spare.subsetSize = subsetSize;
             spare.supersetDf = termsAggFactory.getBackgroundFrequency(spare.termBytes);
@@ -143,66 +147,17 @@ public SignificantStringTerms buildAggregation(long owningBucketOrdinal) throws
     @Override
     public SignificantStringTerms buildEmptyAggregation() {
         // We need to account for the significance of a miss in our global stats - provide corpus size as context
-        ContextIndexSearcher searcher = context.searchContext().searcher();
+        ContextIndexSearcher searcher = context.searcher();
         IndexReader topReader = searcher.getIndexReader();
         int supersetSize = topReader.numDocs();
         return new SignificantStringTerms(name, bucketCountThresholds.getRequiredSize(), bucketCountThresholds.getMinDocCount(),
-                pipelineAggregators(), metaData(), format, 0, supersetSize, significanceHeuristic, emptyList());
+                pipelineAggregators(), metaData(), format, numCollectedDocs, supersetSize, significanceHeuristic, emptyList());
     }
 
     @Override
     protected void doClose() {
+        super.doClose();
         Releasables.close(termsAggFactory);
     }
-
-    public static class WithHash extends GlobalOrdinalsSignificantTermsAggregator {
-
-        private final LongHash bucketOrds;
-
-        public WithHash(String name, AggregatorFactories factories, ValuesSource.Bytes.WithOrdinals.FieldData valuesSource,
-                DocValueFormat format, BucketCountThresholds bucketCountThresholds, IncludeExclude.OrdinalsFilter includeExclude,
-                AggregationContext aggregationContext, Aggregator parent, SignificanceHeuristic significanceHeuristic,
-                SignificantTermsAggregatorFactory termsAggFactory, List<PipelineAggregator> pipelineAggregators,
-                Map<String, Object> metaData) throws IOException {
-            super(name, factories, valuesSource, format, bucketCountThresholds, includeExclude, aggregationContext, parent, significanceHeuristic,
-                    termsAggFactory, pipelineAggregators, metaData);
-            bucketOrds = new LongHash(1, aggregationContext.bigArrays());
-        }
-
-        @Override
-        public LeafBucketCollector getLeafCollector(LeafReaderContext ctx,
-                final LeafBucketCollector sub) throws IOException {
-            return new LeafBucketCollectorBase(super.getLeafCollector(ctx, sub), null) {
-                @Override
-                public void collect(int doc, long bucket) throws IOException {
-                    assert bucket == 0;
-                    numCollectedDocs++;
-                    globalOrds.setDocument(doc);
-                    final int numOrds = globalOrds.cardinality();
-                    for (int i = 0; i < numOrds; i++) {
-                        final long globalOrd = globalOrds.ordAt(i);
-                        long bucketOrd = bucketOrds.add(globalOrd);
-                        if (bucketOrd < 0) {
-                            bucketOrd = -1 - bucketOrd;
-                            collectExistingBucket(sub, doc, bucketOrd);
-                        } else {
-                            collectBucket(sub, doc, bucketOrd);
-                        }
-                    }
-                }
-            };
-        }
-
-        @Override
-        protected long getBucketOrd(long termOrd) {
-            return bucketOrds.find(termOrd);
-        }
-
-        @Override
-        protected void doClose() {
-            Releasables.close(termsAggFactory, bucketOrds);
-        }
-    }
-
 }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/InternalMappedSignificantTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/InternalMappedSignificantTerms.java
index 92995d5fab464..a15be066e7bbd 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/InternalMappedSignificantTerms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/InternalMappedSignificantTerms.java
@@ -21,11 +21,14 @@
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristic;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 
 import java.io.IOException;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.function.Function;
@@ -73,7 +76,12 @@ protected final void writeTermTypeInfoTo(StreamOutput out) throws IOException {
     }
 
     @Override
-    protected List<B> getBucketsInternal() {
+    public Iterator<SignificantTerms.Bucket> iterator() {
+        return buckets.stream().map(bucket -> (SignificantTerms.Bucket) bucket).collect(Collectors.toList()).iterator();
+    }
+
+    @Override
+    public List<B> getBuckets() {
         return buckets;
     }
 
@@ -99,4 +107,20 @@ protected long getSupersetSize() {
     protected SignificanceHeuristic getSignificanceHeuristic() {
         return significanceHeuristic;
     }
+
+    @Override
+    public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+        builder.field(CommonFields.DOC_COUNT.getPreferredName(), subsetSize);
+        builder.field(BG_COUNT, supersetSize);
+        builder.startArray(CommonFields.BUCKETS.getPreferredName());
+        for (Bucket bucket : buckets) {
+            //There is a condition (presumably when only one shard has a bucket?) where reduce is not called
+            // and I end up with buckets that contravene the user's min_doc_count criteria in my reducer
+            if (bucket.subsetDf >= minDocCount) {
+                bucket.toXContent(builder, params);
+            }
+        }
+        builder.endArray();
+        return builder;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/InternalSignificantTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/InternalSignificantTerms.java
index cd18386da1e48..ec91c8bbd625a 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/InternalSignificantTerms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/InternalSignificantTerms.java
@@ -21,6 +21,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.Aggregations;
 import org.elasticsearch.search.aggregations.InternalAggregation;
@@ -33,19 +34,21 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
-import static java.util.Collections.unmodifiableList;
-
 /**
  * Result of the significant terms aggregation.
  */
 public abstract class InternalSignificantTerms<A extends InternalSignificantTerms<A, B>, B extends InternalSignificantTerms.Bucket<B>>
         extends InternalMultiBucketAggregation<A, B> implements SignificantTerms, ToXContent {
+
+    public static final String SCORE = "score";
+    public static final String BG_COUNT = "bg_count";
+
     @SuppressWarnings("PMD.ConstructorCallsOverridableMethod")
-    public abstract static class Bucket<B extends Bucket<B>> extends SignificantTerms.Bucket {
+    public abstract static class Bucket<B extends Bucket<B>> extends InternalMultiBucketAggregation.InternalBucket
+            implements SignificantTerms.Bucket {
         /**
          * Reads a bucket. Should be a constructor reference.
          */
@@ -54,14 +57,21 @@ public interface Reader<B extends Bucket<B>> {
             B read(StreamInput in, long subsetSize, long supersetSize, DocValueFormat format) throws IOException;
         }
 
+        long subsetDf;
+        long subsetSize;
+        long supersetDf;
+        long supersetSize;
         long bucketOrd;
-        protected InternalAggregations aggregations;
         double score;
+        protected InternalAggregations aggregations;
         final transient DocValueFormat format;
 
         protected Bucket(long subsetDf, long subsetSize, long supersetDf, long supersetSize,
                 InternalAggregations aggregations, DocValueFormat format) {
-            super(subsetDf, subsetSize, supersetDf, supersetSize);
+            this.subsetSize = subsetSize;
+            this.supersetSize = supersetSize;
+            this.subsetDf = subsetDf;
+            this.supersetDf = supersetDf;
             this.aggregations = aggregations;
             this.format = format;
         }
@@ -70,7 +80,8 @@ protected Bucket(long subsetDf, long subsetSize, long supersetDf, long supersetS
          * Read from a stream.
          */
         protected Bucket(StreamInput in, long subsetSize, long supersetSize, DocValueFormat format) {
-            super(in, subsetSize, supersetSize);
+            this.subsetSize = subsetSize;
+            this.supersetSize = supersetSize;
             this.format = format;
         }
 
@@ -94,7 +105,7 @@ public long getSubsetSize() {
             return subsetSize;
         }
 
-        public void updateScore(SignificanceHeuristic significanceHeuristic) {
+        void updateScore(SignificanceHeuristic significanceHeuristic) {
             score = significanceHeuristic.getScore(subsetDf, subsetSize, supersetDf, supersetSize);
         }
 
@@ -127,6 +138,20 @@ public B reduce(List<B> buckets, ReduceContext context) {
         public double getSignificanceScore() {
             return score;
         }
+
+        @Override
+        public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            keyToXContent(builder);
+            builder.field(CommonFields.DOC_COUNT.getPreferredName(), getDocCount());
+            builder.field(SCORE, score);
+            builder.field(BG_COUNT, supersetDf);
+            aggregations.toXContentInternal(builder, params);
+            builder.endObject();
+            return builder;
+        }
+
+        protected abstract XContentBuilder keyToXContent(XContentBuilder builder) throws IOException;
     }
 
     protected final int requiredSize;
@@ -157,16 +182,7 @@ protected final void doWriteTo(StreamOutput out) throws IOException {
     protected abstract void writeTermTypeInfoTo(StreamOutput out) throws IOException;
 
     @Override
-    public Iterator<SignificantTerms.Bucket> iterator() {
-        return getBuckets().iterator();
-    }
-
-    @Override
-    public List<SignificantTerms.Bucket> getBuckets() {
-        return unmodifiableList(getBucketsInternal());
-    }
-
-    protected abstract List<B> getBucketsInternal();
+    public abstract List<B> getBuckets();
 
     @Override
     public InternalAggregation doReduce(List<InternalAggregation> aggregations, ReduceContext reduceContext) {
@@ -184,7 +200,7 @@ public InternalAggregation doReduce(List<InternalAggregation> aggregations, Redu
         for (InternalAggregation aggregation : aggregations) {
             @SuppressWarnings("unchecked")
             InternalSignificantTerms<A, B> terms = (InternalSignificantTerms<A, B>) aggregation;
-            for (B bucket : terms.getBucketsInternal()) {
+            for (B bucket : terms.getBuckets()) {
                 List<B> existingBuckets = buckets.get(bucket.getKeyAsString());
                 if (existingBuckets == null) {
                     existingBuckets = new ArrayList<>(aggregations.size());
@@ -196,15 +212,14 @@ public InternalAggregation doReduce(List<InternalAggregation> aggregations, Redu
                         bucket.aggregations));
             }
         }
-
-        getSignificanceHeuristic().initialize(reduceContext);
-        final int size = Math.min(requiredSize, buckets.size());
+        SignificanceHeuristic heuristic = getSignificanceHeuristic().rewrite(reduceContext);
+        final int size = reduceContext.isFinalReduce() == false ? buckets.size() : Math.min(requiredSize, buckets.size());
         BucketSignificancePriorityQueue<B> ordered = new BucketSignificancePriorityQueue<>(size);
         for (Map.Entry<String, List<B>> entry : buckets.entrySet()) {
             List<B> sameTermBuckets = entry.getValue();
             final B b = sameTermBuckets.get(0).reduce(sameTermBuckets, reduceContext);
-            b.updateScore(getSignificanceHeuristic());
-            if ((b.score > 0) && (b.subsetDf >= minDocCount)) {
+            b.updateScore(heuristic);
+            if (((b.score > 0) && (b.subsetDf >= minDocCount)) || reduceContext.isFinalReduce() == false) {
                 ordered.insertWithOverflow(b);
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/ParsedSignificantLongTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/ParsedSignificantLongTerms.java
new file mode 100644
index 0000000000000..9592d80c77625
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/ParsedSignificantLongTerms.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.significant;
+
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+
+public class ParsedSignificantLongTerms extends ParsedSignificantTerms {
+
+    @Override
+    public String getType() {
+        return SignificantLongTerms.NAME;
+    }
+
+    private static ObjectParser<ParsedSignificantLongTerms, Void> PARSER =
+            new ObjectParser<>(ParsedSignificantLongTerms.class.getSimpleName(), true, ParsedSignificantLongTerms::new);
+    static {
+        declareParsedSignificantTermsFields(PARSER, ParsedBucket::fromXContent);
+    }
+
+    public static ParsedSignificantLongTerms fromXContent(XContentParser parser, String name) throws IOException {
+        return parseSignificantTermsXContent(() -> PARSER.parse(parser, null), name);
+    }
+
+    public static class ParsedBucket extends ParsedSignificantTerms.ParsedBucket {
+
+        private Long key;
+
+        @Override
+        public Object getKey() {
+            return key;
+        }
+
+        @Override
+        public String getKeyAsString() {
+            String keyAsString = super.getKeyAsString();
+            if (keyAsString != null) {
+                return keyAsString;
+            }
+            return Long.toString(key);
+        }
+
+        public Number getKeyAsNumber() {
+            return key;
+        }
+
+        @Override
+        protected XContentBuilder keyToXContent(XContentBuilder builder) throws IOException {
+            builder.field(CommonFields.KEY.getPreferredName(), key);
+            if (super.getKeyAsString() != null) {
+                builder.field(CommonFields.KEY_AS_STRING.getPreferredName(), getKeyAsString());
+            }
+            return builder;
+        }
+
+        static ParsedBucket fromXContent(XContentParser parser) throws IOException {
+            return parseSignificantTermsBucketXContent(parser, new ParsedBucket(), (p, bucket) -> bucket.key = p.longValue());
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/ParsedSignificantStringTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/ParsedSignificantStringTerms.java
new file mode 100644
index 0000000000000..008a5a28e5d39
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/ParsedSignificantStringTerms.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.significant;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+
+public class ParsedSignificantStringTerms extends ParsedSignificantTerms {
+
+    @Override
+    public String getType() {
+        return SignificantStringTerms.NAME;
+    }
+
+    private static ObjectParser<ParsedSignificantStringTerms, Void> PARSER =
+            new ObjectParser<>(ParsedSignificantStringTerms.class.getSimpleName(), true, ParsedSignificantStringTerms::new);
+    static {
+        declareParsedSignificantTermsFields(PARSER, ParsedBucket::fromXContent);
+    }
+
+    public static ParsedSignificantStringTerms fromXContent(XContentParser parser, String name) throws IOException {
+        return parseSignificantTermsXContent(() -> PARSER.parse(parser, null), name);
+    }
+
+    public static class ParsedBucket extends ParsedSignificantTerms.ParsedBucket {
+
+        private BytesRef key;
+
+        @Override
+        public Object getKey() {
+            return getKeyAsString();
+        }
+
+        @Override
+        public String getKeyAsString() {
+            String keyAsString = super.getKeyAsString();
+            if (keyAsString != null) {
+                return keyAsString;
+            }
+            return key.utf8ToString();
+        }
+
+        public Number getKeyAsNumber() {
+            return Double.parseDouble(key.utf8ToString());
+        }
+
+        @Override
+        protected XContentBuilder keyToXContent(XContentBuilder builder) throws IOException {
+            return builder.field(CommonFields.KEY.getPreferredName(), getKey());
+        }
+
+        static ParsedBucket fromXContent(XContentParser parser) throws IOException {
+            return parseSignificantTermsBucketXContent(parser, new ParsedBucket(), (p, bucket) -> bucket.key = p.utf8BytesOrNull());
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/ParsedSignificantTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/ParsedSignificantTerms.java
new file mode 100644
index 0000000000000..1b4739c184d5e
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/ParsedSignificantTerms.java
@@ -0,0 +1,191 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.significant;
+
+import org.elasticsearch.common.CheckedBiConsumer;
+import org.elasticsearch.common.CheckedFunction;
+import org.elasticsearch.common.CheckedSupplier;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParserUtils;
+import org.elasticsearch.search.aggregations.Aggregation;
+import org.elasticsearch.search.aggregations.Aggregations;
+import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+public abstract class ParsedSignificantTerms extends ParsedMultiBucketAggregation<ParsedSignificantTerms.ParsedBucket>
+        implements SignificantTerms {
+
+    private Map<String, ParsedBucket> bucketMap;
+    protected long subsetSize;
+    protected long supersetSize;
+
+    protected long getSubsetSize() {
+        return subsetSize;
+    }
+
+    protected long getSupersetSize() {
+        return supersetSize;
+    }
+
+    @Override
+    public List<? extends SignificantTerms.Bucket> getBuckets() {
+        return buckets;
+    }
+
+    @Override
+    public SignificantTerms.Bucket getBucketByKey(String term) {
+        if (bucketMap == null) {
+            bucketMap = buckets.stream().collect(Collectors.toMap(SignificantTerms.Bucket::getKeyAsString, Function.identity()));
+        }
+        return bucketMap.get(term);
+    }
+
+    @Override
+    public Iterator<SignificantTerms.Bucket> iterator() {
+        return buckets.stream().map(bucket -> (SignificantTerms.Bucket) bucket).collect(Collectors.toList()).iterator();
+    }
+
+    @Override
+    protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+        builder.field(CommonFields.DOC_COUNT.getPreferredName(), subsetSize);
+        builder.field(InternalMappedSignificantTerms.BG_COUNT, supersetSize);
+        builder.startArray(CommonFields.BUCKETS.getPreferredName());
+        for (SignificantTerms.Bucket bucket : buckets) {
+            bucket.toXContent(builder, params);
+        }
+        builder.endArray();
+        return builder;
+    }
+
+    static <T extends ParsedSignificantTerms> T parseSignificantTermsXContent(final CheckedSupplier<T, IOException> aggregationSupplier,
+                                                                              final String name) throws IOException {
+        T aggregation = aggregationSupplier.get();
+        aggregation.setName(name);
+        for (ParsedBucket bucket : aggregation.buckets) {
+            bucket.subsetSize = aggregation.subsetSize;
+            bucket.supersetSize = aggregation.supersetSize;
+        }
+        return aggregation;
+    }
+
+    static void declareParsedSignificantTermsFields(final ObjectParser<? extends ParsedSignificantTerms, Void> objectParser,
+                                     final CheckedFunction<XContentParser, ParsedSignificantTerms.ParsedBucket, IOException> bucketParser) {
+        declareMultiBucketAggregationFields(objectParser, bucketParser::apply, bucketParser::apply);
+        objectParser.declareLong((parsedTerms, value) -> parsedTerms.subsetSize = value , CommonFields.DOC_COUNT);
+        objectParser.declareLong((parsedTerms, value) -> parsedTerms.supersetSize = value ,
+                new ParseField(InternalMappedSignificantTerms.BG_COUNT));
+    }
+
+    public abstract static class ParsedBucket extends ParsedMultiBucketAggregation.ParsedBucket implements SignificantTerms.Bucket {
+
+        protected long subsetDf;
+        protected long subsetSize;
+        protected long supersetDf;
+        protected long supersetSize;
+        protected double score;
+
+        @Override
+        public long getDocCount() {
+            return getSubsetDf();
+        }
+
+        @Override
+        public long getSubsetDf() {
+            return subsetDf;
+        }
+
+        @Override
+        public long getSupersetDf() {
+            return supersetDf;
+        }
+
+        @Override
+        public double getSignificanceScore() {
+            return score;
+        }
+
+        @Override
+        public long getSupersetSize() {
+            return supersetSize;
+        }
+
+        @Override
+        public long getSubsetSize() {
+            return subsetSize;
+        }
+
+        @Override
+        public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            keyToXContent(builder);
+            builder.field(CommonFields.DOC_COUNT.getPreferredName(), getDocCount());
+            builder.field(InternalSignificantTerms.SCORE, getSignificanceScore());
+            builder.field(InternalSignificantTerms.BG_COUNT, getSupersetDf());
+            getAggregations().toXContentInternal(builder, params);
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        protected abstract XContentBuilder keyToXContent(XContentBuilder builder) throws IOException;
+
+        static <B extends ParsedBucket> B parseSignificantTermsBucketXContent(final XContentParser parser, final B bucket,
+            final CheckedBiConsumer<XContentParser, B, IOException> keyConsumer) throws IOException {
+
+            final List<Aggregation> aggregations = new ArrayList<>();
+            XContentParser.Token token;
+            String currentFieldName = parser.currentName();
+            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                if (token == XContentParser.Token.FIELD_NAME) {
+                    currentFieldName = parser.currentName();
+                } else if (token.isValue()) {
+                    if (CommonFields.KEY_AS_STRING.getPreferredName().equals(currentFieldName)) {
+                        bucket.setKeyAsString(parser.text());
+                    } else if (CommonFields.KEY.getPreferredName().equals(currentFieldName)) {
+                        keyConsumer.accept(parser, bucket);
+                    } else if (CommonFields.DOC_COUNT.getPreferredName().equals(currentFieldName)) {
+                        long value = parser.longValue();
+                        bucket.subsetDf = value;
+                        bucket.setDocCount(value);
+                    } else if (InternalSignificantTerms.SCORE.equals(currentFieldName)) {
+                        bucket.score = parser.longValue();
+                    } else if (InternalSignificantTerms.BG_COUNT.equals(currentFieldName)) {
+                        bucket.supersetDf = parser.longValue();
+                    }
+                } else if (token == XContentParser.Token.START_OBJECT) {
+                    XContentParserUtils.parseTypedKeysObject(parser, Aggregation.TYPED_KEYS_DELIMITER, Aggregation.class,
+                            aggregations::add);
+                }
+            }
+            bucket.setAggregations(new Aggregations(aggregations));
+            return bucket;
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTerms.java
index ffca3740184ac..d20302f32b4ae 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTerms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTerms.java
@@ -40,19 +40,19 @@ static class Bucket extends InternalSignificantTerms.Bucket<Bucket> {
 
         long term;
 
-        public Bucket(long subsetDf, long subsetSize, long supersetDf, long supersetSize, long term, InternalAggregations aggregations,
+        Bucket(long subsetDf, long subsetSize, long supersetDf, long supersetSize, long term, InternalAggregations aggregations,
                 DocValueFormat format) {
             super(subsetDf, subsetSize, supersetDf, supersetSize, aggregations, format);
             this.term = term;
         }
 
-        public Bucket(long subsetDf, long subsetSize, long supersetDf, long supersetSize, long term, InternalAggregations aggregations,
+        Bucket(long subsetDf, long subsetSize, long supersetDf, long supersetSize, long term, InternalAggregations aggregations,
                 double score) {
             this(subsetDf, subsetSize, supersetDf, supersetSize, term, aggregations, null);
             this.score = score;
         }
 
-        public Bucket(StreamInput in, long subsetSize, long supersetSize, DocValueFormat format) throws IOException {
+        Bucket(StreamInput in, long subsetSize, long supersetSize, DocValueFormat format) throws IOException {
             super(in, subsetSize, supersetSize, format);
             subsetDf = in.readVLong();
             supersetDf = in.readVLong();
@@ -75,11 +75,6 @@ public Object getKey() {
             return term;
         }
 
-        @Override
-        int compareTerm(SignificantTerms.Bucket other) {
-            return Long.compare(term, ((Number) other.getKey()).longValue());
-        }
-
         @Override
         public String getKeyAsString() {
             return format.format(term);
@@ -96,17 +91,11 @@ Bucket newBucket(long subsetDf, long subsetSize, long supersetDf, long supersetS
         }
 
         @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startObject();
-            builder.field(CommonFields.KEY, term);
+        protected XContentBuilder keyToXContent(XContentBuilder builder) throws IOException {
+            builder.field(CommonFields.KEY.getPreferredName(), term);
             if (format != DocValueFormat.RAW) {
-                builder.field(CommonFields.KEY_AS_STRING, format.format(term));
+                builder.field(CommonFields.KEY_AS_STRING.getPreferredName(), format.format(term));
             }
-            builder.field(CommonFields.DOC_COUNT, getDocCount());
-            builder.field("score", score);
-            builder.field("bg_count", supersetDf);
-            aggregations.toXContentInternal(builder, params);
-            builder.endObject();
             return builder;
         }
     }
@@ -148,17 +137,6 @@ protected SignificantLongTerms create(long subsetSize, long supersetSize, List<B
                 supersetSize, significanceHeuristic, buckets);
     }
 
-    @Override
-    public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
-        builder.field("doc_count", subsetSize);
-        builder.startArray(CommonFields.BUCKETS);
-        for (Bucket bucket : buckets) {
-            bucket.toXContent(builder, params);
-        }
-        builder.endArray();
-        return builder;
-    }
-
     @Override
     protected Bucket[] createBucketsArray(int size) {
         return new Bucket[size];
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTermsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTermsAggregator.java
index 4de8f7954018a..70101e8a17bf6 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTermsAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTermsAggregator.java
@@ -30,9 +30,9 @@
 import org.elasticsearch.search.aggregations.bucket.terms.LongTermsAggregator;
 import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.internal.ContextIndexSearcher;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -47,12 +47,12 @@
 public class SignificantLongTermsAggregator extends LongTermsAggregator {
 
     public SignificantLongTermsAggregator(String name, AggregatorFactories factories, ValuesSource.Numeric valuesSource,
-            DocValueFormat format, BucketCountThresholds bucketCountThresholds, AggregationContext aggregationContext, Aggregator parent,
+            DocValueFormat format, BucketCountThresholds bucketCountThresholds, SearchContext context, Aggregator parent,
             SignificanceHeuristic significanceHeuristic, SignificantTermsAggregatorFactory termsAggFactory,
             IncludeExclude.LongFilter includeExclude,
             List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException {
 
-        super(name, factories, valuesSource, format, null, bucketCountThresholds, aggregationContext, parent,
+        super(name, factories, valuesSource, format, null, bucketCountThresholds, context, parent,
                 SubAggCollectionMode.DEPTH_FIRST, false, includeExclude, pipelineAggregators, metaData);
         this.significanceHeuristic = significanceHeuristic;
         this.termsAggFactory = termsAggFactory;
@@ -119,7 +119,7 @@ public SignificantLongTerms buildAggregation(long owningBucketOrdinal) throws IO
     @Override
     public SignificantLongTerms buildEmptyAggregation() {
         // We need to account for the significance of a miss in our global stats - provide corpus size as context
-        ContextIndexSearcher searcher = context.searchContext().searcher();
+        ContextIndexSearcher searcher = context.searcher();
         IndexReader topReader = searcher.getIndexReader();
         int supersetSize = topReader.numDocs();
         return new SignificantLongTerms(name, bucketCountThresholds.getRequiredSize(), bucketCountThresholds.getMinDocCount(),
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTerms.java
index e73da337a0fc0..299a2655ffc3d 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTerms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTerms.java
@@ -80,11 +80,6 @@ public Number getKeyAsNumber() {
             return Double.parseDouble(termBytes.utf8ToString());
         }
 
-        @Override
-        int compareTerm(SignificantTerms.Bucket other) {
-            return termBytes.compareTo(((Bucket) other).termBytes);
-        }
-
         @Override
         public String getKeyAsString() {
             return format.format(termBytes);
@@ -101,15 +96,8 @@ Bucket newBucket(long subsetDf, long subsetSize, long supersetDf, long supersetS
         }
 
         @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startObject();
-            builder.field(CommonFields.KEY, getKeyAsString());
-            builder.field(CommonFields.DOC_COUNT, getDocCount());
-            builder.field("score", score);
-            builder.field("bg_count", supersetDf);
-            aggregations.toXContentInternal(builder, params);
-            builder.endObject();
-            return builder;
+        protected XContentBuilder keyToXContent(XContentBuilder builder) throws IOException {
+            return builder.field(CommonFields.KEY.getPreferredName(), getKeyAsString());
         }
     }
 
@@ -150,21 +138,6 @@ protected SignificantStringTerms create(long subsetSize, long supersetSize, List
                 supersetSize, significanceHeuristic, buckets);
     }
 
-    @Override
-    public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
-        builder.field("doc_count", subsetSize);
-        builder.startArray(CommonFields.BUCKETS);
-        for (Bucket bucket : buckets) {
-            //There is a condition (presumably when only one shard has a bucket?) where reduce is not called
-            // and I end up with buckets that contravene the user's min_doc_count criteria in my reducer
-            if (bucket.subsetDf >= minDocCount) {
-                bucket.toXContent(builder, params);
-            }
-        }
-        builder.endArray();
-        return builder;
-    }
-
     @Override
     protected Bucket[] createBucketsArray(int size) {
         return new Bucket[size];
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTermsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTermsAggregator.java
index ad7645b1efe33..d9cd5b260d048 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTermsAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTermsAggregator.java
@@ -31,9 +31,9 @@
 import org.elasticsearch.search.aggregations.bucket.terms.StringTermsAggregator;
 import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.internal.ContextIndexSearcher;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -52,7 +52,7 @@ public class SignificantStringTermsAggregator extends StringTermsAggregator {
     private final SignificanceHeuristic significanceHeuristic;
 
     public SignificantStringTermsAggregator(String name, AggregatorFactories factories, ValuesSource valuesSource, DocValueFormat format,
-            BucketCountThresholds bucketCountThresholds, IncludeExclude.StringFilter includeExclude, AggregationContext aggregationContext,
+            BucketCountThresholds bucketCountThresholds, IncludeExclude.StringFilter includeExclude, SearchContext aggregationContext,
             Aggregator parent, SignificanceHeuristic significanceHeuristic, SignificantTermsAggregatorFactory termsAggFactory,
             List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException {
 
@@ -126,7 +126,7 @@ public SignificantStringTerms buildAggregation(long owningBucketOrdinal) throws
     @Override
     public SignificantStringTerms buildEmptyAggregation() {
         // We need to account for the significance of a miss in our global stats - provide corpus size as context
-        ContextIndexSearcher searcher = context.searchContext().searcher();
+        ContextIndexSearcher searcher = context.searcher();
         IndexReader topReader = searcher.getIndexReader();
         int supersetSize = topReader.numDocs();
         return new SignificantStringTerms(name, bucketCountThresholds.getRequiredSize(), bucketCountThresholds.getMinDocCount(),
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTerms.java
index 34d8361cbf05c..61cb4a9ca0a2d 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTerms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTerms.java
@@ -18,8 +18,6 @@
  */
 package org.elasticsearch.search.aggregations.bucket.significant;
 
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation;
 import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
 
 import java.util.List;
@@ -28,54 +26,46 @@
  * An aggregation that collects significant terms in comparison to a background set.
  */
 public interface SignificantTerms extends MultiBucketsAggregation, Iterable<SignificantTerms.Bucket> {
-    abstract static class Bucket extends InternalMultiBucketAggregation.InternalBucket {
 
-        long subsetDf;
-        long subsetSize;
-        long supersetDf;
-        long supersetSize;
-
-        Bucket(long subsetDf, long subsetSize, long supersetDf, long supersetSize) {
-            this.subsetSize = subsetSize;
-            this.supersetSize = supersetSize;
-            this.subsetDf = subsetDf;
-            this.supersetDf = supersetDf;
-        }
+    interface Bucket extends MultiBucketsAggregation.Bucket {
 
         /**
-         * Read from a stream.
+         * @return The significant score for the subset
          */
-        protected Bucket(StreamInput in, long subsetSize, long supersetSize) {
-            this.subsetSize = subsetSize;
-            this.supersetSize = supersetSize;
-        }
-
-        abstract int compareTerm(SignificantTerms.Bucket other);
-
-        public abstract double getSignificanceScore();
-
-        abstract Number getKeyAsNumber();
+        double getSignificanceScore();
 
-        public long getSubsetDf() {
-            return subsetDf;
-        }
+        /**
+         * @return The number of docs in the subset containing a particular term.
+         * This number is equal to the document count of the bucket.
+         */
+        long getSubsetDf();
 
-        public long getSupersetDf() {
-            return supersetDf;
-        }
+        /**
+         * @return The numbers of docs in the subset (also known as "foreground set").
+         * This number is equal to the document count of the containing aggregation.
+         */
+        long getSubsetSize();
 
-        public long getSupersetSize() {
-            return supersetSize;
-        }
+        /**
+         * @return The number of docs in the superset containing a particular term (also
+         * known as the "background count" of the bucket)
+         */
+        long getSupersetDf();
 
-        public long getSubsetSize() {
-            return subsetSize;
-        }
+        /**
+         * @return The numbers of docs in the superset (ordinarily the background count
+         * of the containing aggregation).
+         */
+        long getSupersetSize();
 
+        /**
+         * @return The key, expressed as a number
+         */
+        Number getKeyAsNumber();
     }
 
     @Override
-    List<Bucket> getBuckets();
+    List<? extends Bucket> getBuckets();
 
     /**
      * Get the bucket for the given term, or null if there is no such bucket.
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregationBuilder.java
index 245297d72cc28..f416d2dcf1095 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregationBuilder.java
@@ -21,25 +21,30 @@
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ParseFieldRegistry;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.bucket.significant.heuristics.JLHScore;
 import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristic;
+import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristicParser;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator.BucketCountThresholds;
 import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -49,7 +54,6 @@
  */
 public class SignificantTermsAggregationBuilder extends ValuesSourceAggregationBuilder<ValuesSource, SignificantTermsAggregationBuilder> {
     public static final String NAME = "significant_terms";
-    public static final InternalAggregation.Type TYPE = new Type(NAME);
 
     static final ParseField BACKGROUND_FILTER = new ParseField("background_filter");
     static final ParseField HEURISTIC = new ParseField("significance_heuristic");
@@ -58,6 +62,48 @@ public class SignificantTermsAggregationBuilder extends ValuesSourceAggregationB
             3, 0, 10, -1);
     static final SignificanceHeuristic DEFAULT_SIGNIFICANCE_HEURISTIC = new JLHScore();
 
+    public static Aggregator.Parser getParser(ParseFieldRegistry<SignificanceHeuristicParser> significanceHeuristicParserRegistry) {
+        ObjectParser<SignificantTermsAggregationBuilder, QueryParseContext> parser =
+                new ObjectParser<>(SignificantTermsAggregationBuilder.NAME);
+        ValuesSourceParserHelper.declareAnyFields(parser, true, true);
+
+        parser.declareInt(SignificantTermsAggregationBuilder::shardSize, TermsAggregationBuilder.SHARD_SIZE_FIELD_NAME);
+
+        parser.declareLong(SignificantTermsAggregationBuilder::minDocCount, TermsAggregationBuilder.MIN_DOC_COUNT_FIELD_NAME);
+
+        parser.declareLong(SignificantTermsAggregationBuilder::shardMinDocCount, TermsAggregationBuilder.SHARD_MIN_DOC_COUNT_FIELD_NAME);
+
+        parser.declareInt(SignificantTermsAggregationBuilder::size, TermsAggregationBuilder.REQUIRED_SIZE_FIELD_NAME);
+
+        parser.declareString(SignificantTermsAggregationBuilder::executionHint, TermsAggregationBuilder.EXECUTION_HINT_FIELD_NAME);
+
+        parser.declareObject((b, v) -> { if (v.isPresent()) b.backgroundFilter(v.get()); },
+                (p, context) -> context.parseInnerQueryBuilder(),
+                SignificantTermsAggregationBuilder.BACKGROUND_FILTER);
+
+        parser.declareField((b, v) -> b.includeExclude(IncludeExclude.merge(v, b.includeExclude())),
+                IncludeExclude::parseInclude, IncludeExclude.INCLUDE_FIELD, ObjectParser.ValueType.OBJECT_ARRAY_OR_STRING);
+
+        parser.declareField((b, v) -> b.includeExclude(IncludeExclude.merge(b.includeExclude(), v)),
+                IncludeExclude::parseExclude, IncludeExclude.EXCLUDE_FIELD, ObjectParser.ValueType.OBJECT_ARRAY_OR_STRING);
+
+        for (String name : significanceHeuristicParserRegistry.getNames()) {
+            parser.declareObject(SignificantTermsAggregationBuilder::significanceHeuristic,
+                    (p, context) -> {
+                        SignificanceHeuristicParser significanceHeuristicParser = significanceHeuristicParserRegistry
+                                .lookupReturningNullIfNotFound(name);
+                        return significanceHeuristicParser.parse(context);
+                    },
+                    new ParseField(name));
+        }
+        return new Aggregator.Parser() {
+            @Override
+            public AggregationBuilder parse(String aggregationName, QueryParseContext context) throws IOException {
+                return parser.parse(context.parser(), new SignificantTermsAggregationBuilder(aggregationName, null), context);
+            }
+        };
+    }
+
     private IncludeExclude includeExclude = null;
     private String executionHint = null;
     private QueryBuilder filterBuilder = null;
@@ -65,14 +111,14 @@ public class SignificantTermsAggregationBuilder extends ValuesSourceAggregationB
     private SignificanceHeuristic significanceHeuristic = DEFAULT_SIGNIFICANCE_HEURISTIC;
 
     public SignificantTermsAggregationBuilder(String name, ValueType valueType) {
-        super(name, TYPE, ValuesSourceType.ANY, valueType);
+        super(name, ValuesSourceType.ANY, valueType);
     }
 
     /**
      * Read from a Stream.
      */
     public SignificantTermsAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE, ValuesSourceType.ANY);
+        super(in, ValuesSourceType.ANY);
         bucketCountThresholds = new BucketCountThresholds(in);
         executionHint = in.readOptionalString();
         filterBuilder = in.readOptionalNamedWriteable(QueryBuilder.class);
@@ -218,10 +264,11 @@ public SignificanceHeuristic significanceHeuristic() {
     }
 
     @Override
-    protected ValuesSourceAggregatorFactory<ValuesSource, ?> innerBuild(AggregationContext context, ValuesSourceConfig<ValuesSource> config,
+    protected ValuesSourceAggregatorFactory<ValuesSource, ?> innerBuild(SearchContext context, ValuesSourceConfig<ValuesSource> config,
             AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
-        return new SignificantTermsAggregatorFactory(name, type, config, includeExclude, executionHint, filterBuilder,
-                bucketCountThresholds, significanceHeuristic, context, parent, subFactoriesBuilder, metaData);
+        SignificanceHeuristic executionHeuristic = this.significanceHeuristic.rewrite(context);
+        return new SignificantTermsAggregatorFactory(name, config, includeExclude, executionHint, filterBuilder,
+                bucketCountThresholds, executionHeuristic, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
@@ -256,7 +303,7 @@ protected boolean innerEquals(Object obj) {
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java
index ab30e1b2d4aad..65819736d5419 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java
@@ -19,17 +19,16 @@
 
 package org.elasticsearch.search.aggregations.bucket.significant;
 
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.PostingsEnum;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.BooleanClause.Occur;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.lucene.index.FilterableTermsEnum;
 import org.elasticsearch.common.lucene.index.FreqTermsEnum;
@@ -42,20 +41,19 @@
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.NonCollectingAggregator;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.bucket.BucketUtils;
 import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristic;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator.BucketCountThresholds;
 import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 
@@ -68,22 +66,29 @@ public class SignificantTermsAggregatorFactory extends ValuesSourceAggregatorFac
     private MappedFieldType fieldType;
     private FilterableTermsEnum termsEnum;
     private int numberOfAggregatorsCreated;
-    private final Query filter;
+    final Query filter;
     private final int supersetNumDocs;
     private final TermsAggregator.BucketCountThresholds bucketCountThresholds;
     private final SignificanceHeuristic significanceHeuristic;
 
-    public SignificantTermsAggregatorFactory(String name, Type type, ValuesSourceConfig<ValuesSource> config, IncludeExclude includeExclude,
-            String executionHint, QueryBuilder filterBuilder, TermsAggregator.BucketCountThresholds bucketCountThresholds,
-            SignificanceHeuristic significanceHeuristic, AggregationContext context, AggregatorFactory<?> parent,
-            AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+    public SignificantTermsAggregatorFactory(String name,
+                                             ValuesSourceConfig<ValuesSource> config,
+                                             IncludeExclude includeExclude,
+                                             String executionHint,
+                                             QueryBuilder filterBuilder,
+                                             TermsAggregator.BucketCountThresholds bucketCountThresholds,
+                                             SignificanceHeuristic significanceHeuristic,
+                                             SearchContext context,
+                                             AggregatorFactory<?> parent,
+                                             AggregatorFactories.Builder subFactoriesBuilder,
+                                             Map<String, Object> metaData) throws IOException {
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.includeExclude = includeExclude;
         this.executionHint = executionHint;
         this.filter = filterBuilder == null
                 ? null
-                : filterBuilder.toQuery(context.searchContext().getQueryShardContext());
-        IndexSearcher searcher = context.searchContext().searcher();
+                : filterBuilder.toFilter(context.getQueryShardContext());
+        IndexSearcher searcher = context.searcher();
         this.supersetNumDocs = filter == null
                 // Important - need to use the doc count that includes deleted docs
                 // or we have this issue: https://github.com/elastic/elasticsearch/issues/7951
@@ -91,15 +96,14 @@ public SignificantTermsAggregatorFactory(String name, Type type, ValuesSourceCon
                 : searcher.count(filter);
         this.bucketCountThresholds = bucketCountThresholds;
         this.significanceHeuristic = significanceHeuristic;
-        this.significanceHeuristic.initialize(context.searchContext());
-        setFieldInfo();
+        setFieldInfo(context);
 
     }
 
-    private void setFieldInfo() {
+    private void setFieldInfo(SearchContext context) {
         if (!config.unmapped()) {
             this.indexedFieldName = config.fieldContext().field();
-            fieldType = SearchContext.current().smartNameFieldType(indexedFieldName);
+            fieldType = context.smartNameFieldType(indexedFieldName);
         }
     }
 
@@ -114,7 +118,7 @@ private FilterableTermsEnum getTermsEnum(String field) throws IOException {
         if (termsEnum != null) {
             return termsEnum;
         }
-        IndexReader reader = context.searchContext().searcher().getIndexReader();
+        IndexReader reader = context.searcher().getIndexReader();
         if (numberOfAggregatorsCreated > 1) {
             termsEnum = new FreqTermsEnum(reader, field, true, false, filter, context.bigArrays());
         } else {
@@ -124,7 +128,7 @@ private FilterableTermsEnum getTermsEnum(String field) throws IOException {
     }
 
     private long getBackgroundFrequency(String value) throws IOException {
-        Query query = fieldType.termQuery(value, context.searchContext().getQueryShardContext());
+        Query query = fieldType.termQuery(value, context.getQueryShardContext());
         if (query instanceof TermQuery) {
             // for types that use the inverted index, we prefer using a caching terms
             // enum that will do a better job at reusing index inputs
@@ -143,7 +147,7 @@ private long getBackgroundFrequency(String value) throws IOException {
                     .add(filter, Occur.FILTER)
                     .build();
         }
-        return context.searchContext().searcher().count(query);
+        return context.searcher().count(query);
     }
 
     public long getBackgroundFrequency(BytesRef termBytes) throws IOException {
@@ -192,13 +196,13 @@ protected Aggregator doCreateInternal(ValuesSource valuesSource, Aggregator pare
             // such are impossible to differentiate from non-significant terms
             // at that early stage.
             bucketCountThresholds.setShardSize(2 * BucketUtils.suggestShardSideQueueSize(bucketCountThresholds.getRequiredSize(),
-                    context.searchContext().numberOfShards()));
+                    context.numberOfShards()));
         }
 
         if (valuesSource instanceof ValuesSource.Bytes) {
             ExecutionMode execution = null;
             if (executionHint != null) {
-                execution = ExecutionMode.fromString(executionHint, context.searchContext().parseFieldMatcher());
+                execution = ExecutionMode.fromString(executionHint);
             }
             if (!(valuesSource instanceof ValuesSource.Bytes.WithOrdinals)) {
                 execution = ExecutionMode.MAP;
@@ -211,13 +215,13 @@ protected Aggregator doCreateInternal(ValuesSource valuesSource, Aggregator pare
                 }
             }
             assert execution != null;
-            
+
             DocValueFormat format = config.format();
             if ((includeExclude != null) && (includeExclude.isRegexBased()) && format != DocValueFormat.RAW) {
                 throw new AggregationExecutionException("Aggregation [" + name + "] cannot support regular expression style include/exclude "
                         + "settings as they can only be applied to string fields. Use an array of values for include/exclude clauses");
             }
-            
+
             return execution.create(name, factories, valuesSource, format, bucketCountThresholds, includeExclude, context, parent,
                     significanceHeuristic, this, pipelineAggregators, metaData);
         }
@@ -250,54 +254,81 @@ public enum ExecutionMode {
         MAP(new ParseField("map")) {
 
             @Override
-            Aggregator create(String name, AggregatorFactories factories, ValuesSource valuesSource, DocValueFormat format,
-                    TermsAggregator.BucketCountThresholds bucketCountThresholds, IncludeExclude includeExclude,
-                    AggregationContext aggregationContext, Aggregator parent, SignificanceHeuristic significanceHeuristic,
-                    SignificantTermsAggregatorFactory termsAggregatorFactory, List<PipelineAggregator> pipelineAggregators,
-                    Map<String, Object> metaData) throws IOException {
+            Aggregator create(String name,
+                              AggregatorFactories factories,
+                              ValuesSource valuesSource,
+                              DocValueFormat format,
+                              TermsAggregator.BucketCountThresholds bucketCountThresholds,
+                              IncludeExclude includeExclude,
+                              SearchContext aggregationContext,
+                              Aggregator parent,
+                              SignificanceHeuristic significanceHeuristic,
+                              SignificantTermsAggregatorFactory termsAggregatorFactory,
+                              List<PipelineAggregator> pipelineAggregators,
+                              Map<String, Object> metaData) throws IOException {
+
                 final IncludeExclude.StringFilter filter = includeExclude == null ? null : includeExclude.convertToStringFilter(format);
                 return new SignificantStringTermsAggregator(name, factories, valuesSource, format, bucketCountThresholds, filter,
                         aggregationContext, parent, significanceHeuristic, termsAggregatorFactory, pipelineAggregators, metaData);
+
             }
 
         },
         GLOBAL_ORDINALS(new ParseField("global_ordinals")) {
 
             @Override
-            Aggregator create(String name, AggregatorFactories factories, ValuesSource valuesSource, DocValueFormat format,
-                    TermsAggregator.BucketCountThresholds bucketCountThresholds, IncludeExclude includeExclude,
-                    AggregationContext aggregationContext, Aggregator parent, SignificanceHeuristic significanceHeuristic,
-                    SignificantTermsAggregatorFactory termsAggregatorFactory, List<PipelineAggregator> pipelineAggregators,
-                    Map<String, Object> metaData) throws IOException {
+            Aggregator create(String name,
+                              AggregatorFactories factories,
+                              ValuesSource valuesSource,
+                              DocValueFormat format,
+                              TermsAggregator.BucketCountThresholds bucketCountThresholds,
+                              IncludeExclude includeExclude,
+                              SearchContext aggregationContext,
+                              Aggregator parent,
+                              SignificanceHeuristic significanceHeuristic,
+                              SignificantTermsAggregatorFactory termsAggregatorFactory,
+                              List<PipelineAggregator> pipelineAggregators,
+                              Map<String, Object> metaData) throws IOException {
+
                 final IncludeExclude.OrdinalsFilter filter = includeExclude == null ? null : includeExclude.convertToOrdinalsFilter(format);
                 return new GlobalOrdinalsSignificantTermsAggregator(name, factories,
                         (ValuesSource.Bytes.WithOrdinals.FieldData) valuesSource, format, bucketCountThresholds, filter,
-                        aggregationContext, parent, significanceHeuristic, termsAggregatorFactory, pipelineAggregators, metaData);
+                        aggregationContext, parent, false, significanceHeuristic, termsAggregatorFactory, pipelineAggregators, metaData);
+
             }
 
         },
         GLOBAL_ORDINALS_HASH(new ParseField("global_ordinals_hash")) {
 
             @Override
-            Aggregator create(String name, AggregatorFactories factories, ValuesSource valuesSource, DocValueFormat format,
-                    TermsAggregator.BucketCountThresholds bucketCountThresholds, IncludeExclude includeExclude,
-                    AggregationContext aggregationContext, Aggregator parent, SignificanceHeuristic significanceHeuristic,
-                    SignificantTermsAggregatorFactory termsAggregatorFactory, List<PipelineAggregator> pipelineAggregators,
-                    Map<String, Object> metaData) throws IOException {
+            Aggregator create(String name,
+                              AggregatorFactories factories,
+                              ValuesSource valuesSource,
+                              DocValueFormat format,
+                              TermsAggregator.BucketCountThresholds bucketCountThresholds,
+                              IncludeExclude includeExclude,
+                              SearchContext aggregationContext,
+                              Aggregator parent,
+                              SignificanceHeuristic significanceHeuristic,
+                              SignificantTermsAggregatorFactory termsAggregatorFactory,
+                              List<PipelineAggregator> pipelineAggregators,
+                              Map<String, Object> metaData) throws IOException {
+
                 final IncludeExclude.OrdinalsFilter filter = includeExclude == null ? null : includeExclude.convertToOrdinalsFilter(format);
-                return new GlobalOrdinalsSignificantTermsAggregator.WithHash(name, factories,
-                        (ValuesSource.Bytes.WithOrdinals.FieldData) valuesSource, format, bucketCountThresholds, filter,
-                        aggregationContext, parent, significanceHeuristic, termsAggregatorFactory, pipelineAggregators, metaData);
+                return new GlobalOrdinalsSignificantTermsAggregator(name, factories,
+                    (ValuesSource.Bytes.WithOrdinals.FieldData) valuesSource, format, bucketCountThresholds, filter, aggregationContext, parent,
+                    true, significanceHeuristic, termsAggregatorFactory, pipelineAggregators, metaData);
+
             }
         };
 
-        public static ExecutionMode fromString(String value, ParseFieldMatcher parseFieldMatcher) {
+        public static ExecutionMode fromString(String value) {
             for (ExecutionMode mode : values()) {
-                if (parseFieldMatcher.match(value, mode.parseField)) {
+                if (mode.parseField.match(value)) {
                     return mode;
                 }
             }
-            throw new IllegalArgumentException("Unknown `execution_hint`: [" + value + "], expected any of " + values());
+            throw new IllegalArgumentException("Unknown `execution_hint`: [" + value + "], expected any of " + Arrays.toString(values()));
         }
 
         private final ParseField parseField;
@@ -306,11 +337,18 @@ public static ExecutionMode fromString(String value, ParseFieldMatcher parseFiel
             this.parseField = parseField;
         }
 
-        abstract Aggregator create(String name, AggregatorFactories factories, ValuesSource valuesSource,  DocValueFormat format,
-                TermsAggregator.BucketCountThresholds bucketCountThresholds, IncludeExclude includeExclude,
-                AggregationContext aggregationContext, Aggregator parent, SignificanceHeuristic significanceHeuristic,
-                SignificantTermsAggregatorFactory termsAggregatorFactory, List<PipelineAggregator> pipelineAggregators,
-                Map<String, Object> metaData) throws IOException;
+        abstract Aggregator create(String name,
+                                   AggregatorFactories factories,
+                                   ValuesSource valuesSource,
+                                   DocValueFormat format,
+                                   TermsAggregator.BucketCountThresholds bucketCountThresholds,
+                                   IncludeExclude includeExclude,
+                                   SearchContext aggregationContext,
+                                   Aggregator parent,
+                                   SignificanceHeuristic significanceHeuristic,
+                                   SignificantTermsAggregatorFactory termsAggregatorFactory,
+                                   List<PipelineAggregator> pipelineAggregators,
+                                   Map<String, Object> metaData) throws IOException;
 
         @Override
         public String toString() {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsParser.java
deleted file mode 100644
index 0f08cf0a0a32b..0000000000000
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsParser.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.search.aggregations.bucket.significant;
-
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.xcontent.ParseFieldRegistry;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentParser.Token;
-import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
-import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode;
-import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristic;
-import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristicParser;
-import org.elasticsearch.search.aggregations.bucket.terms.AbstractTermsParser;
-import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator;
-import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator.BucketCountThresholds;
-import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude;
-import org.elasticsearch.search.aggregations.support.XContentParseContext;
-import org.elasticsearch.search.aggregations.support.ValueType;
-import org.elasticsearch.search.aggregations.support.ValuesSourceType;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.Optional;
-
-/**
- *
- */
-public class SignificantTermsParser extends AbstractTermsParser {
-    private final ParseFieldRegistry<SignificanceHeuristicParser> significanceHeuristicParserRegistry;
-    private final IndicesQueriesRegistry queriesRegistry;
-
-    public SignificantTermsParser(ParseFieldRegistry<SignificanceHeuristicParser> significanceHeuristicParserRegistry,
-            IndicesQueriesRegistry queriesRegistry) {
-        this.significanceHeuristicParserRegistry = significanceHeuristicParserRegistry;
-        this.queriesRegistry = queriesRegistry;
-    }
-
-    @Override
-    protected SignificantTermsAggregationBuilder doCreateFactory(String aggregationName, ValuesSourceType valuesSourceType,
-                                                                 ValueType targetValueType, BucketCountThresholds bucketCountThresholds,
-                                                                 SubAggCollectionMode collectMode, String executionHint,
-                                                                 IncludeExclude incExc, Map<ParseField, Object> otherOptions) {
-        SignificantTermsAggregationBuilder factory = new SignificantTermsAggregationBuilder(aggregationName, targetValueType);
-        if (bucketCountThresholds != null) {
-            factory.bucketCountThresholds(bucketCountThresholds);
-        }
-        if (executionHint != null) {
-            factory.executionHint(executionHint);
-        }
-        if (incExc != null) {
-            factory.includeExclude(incExc);
-        }
-        QueryBuilder backgroundFilter = (QueryBuilder) otherOptions.get(SignificantTermsAggregationBuilder.BACKGROUND_FILTER);
-        if (backgroundFilter != null) {
-            factory.backgroundFilter(backgroundFilter);
-        }
-        SignificanceHeuristic significanceHeuristic =
-            (SignificanceHeuristic) otherOptions.get(SignificantTermsAggregationBuilder.HEURISTIC);
-        if (significanceHeuristic != null) {
-            factory.significanceHeuristic(significanceHeuristic);
-        }
-        return factory;
-    }
-
-    @Override
-    public boolean parseSpecial(String aggregationName, XContentParseContext context, Token token,
-                                String currentFieldName, Map<ParseField, Object> otherOptions) throws IOException {
-        if (token == XContentParser.Token.START_OBJECT) {
-            SignificanceHeuristicParser significanceHeuristicParser = significanceHeuristicParserRegistry
-                    .lookupReturningNullIfNotFound(currentFieldName, context.getParseFieldMatcher());
-            if (significanceHeuristicParser != null) {
-                SignificanceHeuristic significanceHeuristic = significanceHeuristicParser.parse(context);
-                otherOptions.put(SignificantTermsAggregationBuilder.HEURISTIC, significanceHeuristic);
-                return true;
-            } else if (context.matchField(currentFieldName, SignificantTermsAggregationBuilder.BACKGROUND_FILTER)) {
-                QueryParseContext queryParseContext = new QueryParseContext(context.getDefaultScriptLanguage(), queriesRegistry,
-                        context.getParser(), context.getParseFieldMatcher());
-                Optional<QueryBuilder> filter = queryParseContext.parseInnerQueryBuilder();
-                if (filter.isPresent()) {
-                    otherOptions.put(SignificantTermsAggregationBuilder.BACKGROUND_FILTER, filter.get());
-                }
-                return true;
-            }
-        }
-        return false;
-    }
-
-    @Override
-    protected BucketCountThresholds getDefaultBucketCountThresholds() {
-        return new TermsAggregator.BucketCountThresholds(SignificantTermsAggregationBuilder.DEFAULT_BUCKET_COUNT_THRESHOLDS);
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/UnmappedSignificantTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/UnmappedSignificantTerms.java
index 1a1094e62b467..c2cf93b28105e 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/UnmappedSignificantTerms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/UnmappedSignificantTerms.java
@@ -31,15 +31,18 @@
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 
 import java.io.IOException;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
+import static java.util.Collections.emptyIterator;
 import static java.util.Collections.emptyList;
 
 /**
  * Result of the running the significant terms aggregation on an unmapped field.
  */
 public class UnmappedSignificantTerms extends InternalSignificantTerms<UnmappedSignificantTerms, UnmappedSignificantTerms.Bucket> {
+
     public static final String NAME = "umsigterms";
 
     /**
@@ -75,6 +78,11 @@ public String getWriteableName() {
         return NAME;
     }
 
+    @Override
+    public String getType() {
+        return SignificantStringTerms.NAME;
+    }
+
     @Override
     public UnmappedSignificantTerms create(List<Bucket> buckets) {
         return new UnmappedSignificantTerms(name, requiredSize, minDocCount, pipelineAggregators(), metaData);
@@ -102,7 +110,7 @@ public InternalAggregation doReduce(List<InternalAggregation> aggregations, Redu
 
     @Override
     public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
-        builder.startArray(CommonFields.BUCKETS).endArray();
+        builder.startArray(CommonFields.BUCKETS.getPreferredName()).endArray();
         return builder;
     }
 
@@ -112,7 +120,12 @@ protected Bucket[] createBucketsArray(int size) {
     }
 
     @Override
-    protected List<Bucket> getBucketsInternal() {
+    public Iterator<SignificantTerms.Bucket> iterator() {
+        return emptyIterator();
+    }
+
+    @Override
+    public List<Bucket> getBuckets() {
         return emptyList();
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/GND.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/GND.java
index 3ae26639aa9c4..5968f42211e54 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/GND.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/GND.java
@@ -26,8 +26,8 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardException;
-import org.elasticsearch.search.aggregations.support.XContentParseContext;
 
 import java.io.IOException;
 
@@ -113,13 +113,13 @@ protected SignificanceHeuristic newHeuristic(boolean includeNegatives, boolean b
         }
 
         @Override
-        public SignificanceHeuristic parse(XContentParseContext context) throws IOException, QueryShardException {
-            XContentParser parser = context.getParser();
+        public SignificanceHeuristic parse(QueryParseContext context) throws IOException, QueryShardException {
+            XContentParser parser = context.parser();
             String givenName = parser.currentName();
             boolean backgroundIsSuperset = true;
             XContentParser.Token token = parser.nextToken();
             while (!token.equals(XContentParser.Token.END_OBJECT)) {
-                if (context.matchField(parser.currentName(), BACKGROUND_IS_SUPERSET)) {
+                if (BACKGROUND_IS_SUPERSET.match(parser.currentName())) {
                     parser.nextToken();
                     backgroundIsSuperset = parser.booleanValue();
                 } else {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/JLHScore.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/JLHScore.java
index 58f8060a10874..d8009818f7f6b 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/JLHScore.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/JLHScore.java
@@ -26,8 +26,8 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardException;
-import org.elasticsearch.search.aggregations.support.XContentParseContext;
 
 import java.io.IOException;
 
@@ -104,9 +104,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         return builder;
     }
 
-    public static SignificanceHeuristic parse(XContentParseContext context)
+    public static SignificanceHeuristic parse(QueryParseContext context)
             throws IOException, QueryShardException {
-        XContentParser parser = context.getParser();
+        XContentParser parser = context.parser();
         // move to the closing bracket
         if (!parser.nextToken().equals(XContentParser.Token.END_OBJECT)) {
             throw new ElasticsearchParseException(
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/NXYSignificanceHeuristic.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/NXYSignificanceHeuristic.java
index d6064ca37fd52..5f92b5b40e632 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/NXYSignificanceHeuristic.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/NXYSignificanceHeuristic.java
@@ -27,8 +27,8 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardException;
-import org.elasticsearch.search.aggregations.support.XContentParseContext;
 
 import java.io.IOException;
 
@@ -152,18 +152,18 @@ protected void build(XContentBuilder builder) throws IOException {
     public abstract static class NXYParser implements SignificanceHeuristicParser {
 
         @Override
-        public SignificanceHeuristic parse(XContentParseContext context)
+        public SignificanceHeuristic parse(QueryParseContext context)
                 throws IOException, QueryShardException {
-            XContentParser parser = context.getParser();
+            XContentParser parser = context.parser();
             String givenName = parser.currentName();
             boolean includeNegatives = false;
             boolean backgroundIsSuperset = true;
             XContentParser.Token token = parser.nextToken();
             while (!token.equals(XContentParser.Token.END_OBJECT)) {
-                if (context.matchField(parser.currentName(), INCLUDE_NEGATIVES_FIELD)) {
+                if (INCLUDE_NEGATIVES_FIELD.match(parser.currentName())) {
                     parser.nextToken();
                     includeNegatives = parser.booleanValue();
-                } else if (context.matchField(parser.currentName(), BACKGROUND_IS_SUPERSET)) {
+                } else if (BACKGROUND_IS_SUPERSET.match(parser.currentName())) {
                     parser.nextToken();
                     backgroundIsSuperset = parser.booleanValue();
                 } else {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/PercentageScore.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/PercentageScore.java
index c7e5c7ead6f58..f4a61fbfae0cc 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/PercentageScore.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/PercentageScore.java
@@ -26,8 +26,8 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardException;
-import org.elasticsearch.search.aggregations.support.XContentParseContext;
 
 import java.io.IOException;
 
@@ -56,9 +56,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         return builder;
     }
 
-    public static SignificanceHeuristic parse(XContentParseContext context)
+    public static SignificanceHeuristic parse(QueryParseContext context)
             throws IOException, QueryShardException {
-        XContentParser parser = context.getParser();
+        XContentParser parser = context.parser();
         // move to the closing bracket
         if (!parser.nextToken().equals(XContentParser.Token.END_OBJECT)) {
             throw new ElasticsearchParseException("failed to parse [percentage] significance heuristic. expected an empty object, but got [{}] instead", parser.currentToken());
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/ScriptHeuristic.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/ScriptHeuristic.java
index c933f9ef59631..5f496ecbf8049 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/ScriptHeuristic.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/ScriptHeuristic.java
@@ -24,38 +24,58 @@
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardException;
+import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.Script.ScriptField;
 import org.elasticsearch.script.ScriptContext;
-import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.aggregations.InternalAggregation;
-import org.elasticsearch.search.aggregations.support.XContentParseContext;
 import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
-import java.util.Collections;
 import java.util.Objects;
 
 public class ScriptHeuristic extends SignificanceHeuristic {
     public static final String NAME = "script_heuristic";
 
-    private final LongAccessor subsetSizeHolder;
-    private final LongAccessor supersetSizeHolder;
-    private final LongAccessor subsetDfHolder;
-    private final LongAccessor supersetDfHolder;
     private final Script script;
-    ExecutableScript searchScript = null;
+
+    // This class holds an executable form of the script with private variables ready for execution
+    // on a single search thread.
+    static class ExecutableScriptHeuristic extends ScriptHeuristic {
+        private final LongAccessor subsetSizeHolder;
+        private final LongAccessor supersetSizeHolder;
+        private final LongAccessor subsetDfHolder;
+        private final LongAccessor supersetDfHolder;
+        private final ExecutableScript executableScript;
+
+        ExecutableScriptHeuristic(Script script, ExecutableScript executableScript){
+            super(script);
+            subsetSizeHolder = new LongAccessor();
+            supersetSizeHolder = new LongAccessor();
+            subsetDfHolder = new LongAccessor();
+            supersetDfHolder = new LongAccessor();
+            this.executableScript = executableScript;
+            executableScript.setNextVar("_subset_freq", subsetDfHolder);
+            executableScript.setNextVar("_subset_size", subsetSizeHolder);
+            executableScript.setNextVar("_superset_freq", supersetDfHolder);
+            executableScript.setNextVar("_superset_size", supersetSizeHolder);
+        }
+
+        @Override
+        public double getScore(long subsetFreq, long subsetSize, long supersetFreq, long supersetSize) {
+            subsetSizeHolder.value = subsetSize;
+            supersetSizeHolder.value = supersetSize;
+            subsetDfHolder.value = subsetFreq;
+            supersetDfHolder.value = supersetFreq;
+            return ((Number) executableScript.run()).doubleValue();
+       }
+    }
 
     public ScriptHeuristic(Script script) {
-        subsetSizeHolder = new LongAccessor();
-        supersetSizeHolder = new LongAccessor();
-        subsetDfHolder = new LongAccessor();
-        supersetDfHolder = new LongAccessor();
         this.script = script;
     }
 
@@ -72,21 +92,14 @@ public void writeTo(StreamOutput out) throws IOException {
     }
 
     @Override
-    public void initialize(InternalAggregation.ReduceContext context) {
-        initialize(context.scriptService());
+    public SignificanceHeuristic rewrite(InternalAggregation.ReduceContext context) {
+        CompiledScript compiledScript = context.scriptService().compile(script, ScriptContext.Standard.AGGS);
+        return new ExecutableScriptHeuristic(script, context.scriptService().executable(compiledScript, script.getParams()));
     }
 
     @Override
-    public void initialize(SearchContext context) {
-        initialize(context.scriptService());
-    }
-
-    public void initialize(ScriptService scriptService) {
-        searchScript = scriptService.executable(script, ScriptContext.Standard.AGGS, Collections.emptyMap());
-        searchScript.setNextVar("_subset_freq", subsetDfHolder);
-        searchScript.setNextVar("_subset_size", subsetSizeHolder);
-        searchScript.setNextVar("_superset_freq", supersetDfHolder);
-        searchScript.setNextVar("_superset_size", supersetSizeHolder);
+    public SignificanceHeuristic rewrite(SearchContext context) {
+        return new ExecutableScriptHeuristic(script, context.getQueryShardContext().getExecutableScript(script, ScriptContext.Standard.AGGS));
     }
 
     /**
@@ -100,19 +113,7 @@ public void initialize(ScriptService scriptService) {
      */
     @Override
     public double getScore(long subsetFreq, long subsetSize, long supersetFreq, long supersetSize) {
-        if (searchScript == null) {
-            //In tests, wehn calling assertSearchResponse(..) the response is streamed one additional time with an arbitrary version, see assertVersionSerializable(..).
-            // Now, for version before 1.5.0 the score is computed after streaming the response but for scripts the script does not exists yet.
-            // assertSearchResponse() might therefore fail although there is no problem.
-            // This should be replaced by an exception in 2.0.
-            ESLoggerFactory.getLogger("script heuristic").warn("cannot compute score - script has not been initialized yet.");
-            return 0;
-        }
-        subsetSizeHolder.value = subsetSize;
-        supersetSizeHolder.value = supersetSize;
-        subsetDfHolder.value = subsetFreq;
-        supersetDfHolder.value = supersetFreq;
-        return ((Number) searchScript.run()).doubleValue();
+        throw new UnsupportedOperationException("This scoring heuristic must have 'rewrite' called on it to provide a version ready for use");
     }
 
     @Override
@@ -123,7 +124,7 @@ public String getWriteableName() {
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params builderParams) throws IOException {
         builder.startObject(NAME);
-        builder.field(ScriptField.SCRIPT.getPreferredName());
+        builder.field(Script.SCRIPT_PARSE_FIELD.getPreferredName());
         script.toXContent(builder, builderParams);
         builder.endObject();
         return builder;
@@ -146,9 +147,9 @@ public boolean equals(Object obj) {
         return Objects.equals(script, other.script);
     }
 
-    public static SignificanceHeuristic parse(XContentParseContext context)
+    public static SignificanceHeuristic parse(QueryParseContext context)
             throws IOException, QueryShardException {
-        XContentParser parser = context.getParser();
+        XContentParser parser = context.parser();
         String heuristicName = parser.currentName();
         Script script = null;
         XContentParser.Token token;
@@ -157,8 +158,8 @@ public static SignificanceHeuristic parse(XContentParseContext context)
             if (token.equals(XContentParser.Token.FIELD_NAME)) {
                 currentFieldName = parser.currentName();
             } else {
-                if (context.matchField(currentFieldName, ScriptField.SCRIPT)) {
-                    script = Script.parse(parser, context.getParseFieldMatcher(), context.getDefaultScriptLanguage());
+                if (Script.SCRIPT_PARSE_FIELD.match(currentFieldName)) {
+                    script = Script.parse(parser, context.getDefaultScriptLanguage());
                 } else {
                     throw new ElasticsearchParseException("failed to parse [{}] significance heuristic. unknown object [{}]", heuristicName, currentFieldName);
                 }
@@ -171,26 +172,6 @@ public static SignificanceHeuristic parse(XContentParseContext context)
         return new ScriptHeuristic(script);
     }
 
-    public static class ScriptHeuristicBuilder implements SignificanceHeuristicBuilder {
-
-        private Script script = null;
-
-        public ScriptHeuristicBuilder setScript(Script script) {
-            this.script = script;
-            return this;
-        }
-
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params builderParams) throws IOException {
-            builder.startObject(NAME);
-            builder.field(ScriptField.SCRIPT.getPreferredName());
-            script.toXContent(builder, builderParams);
-            builder.endObject();
-            return builder;
-        }
-
-    }
-
     public final class LongAccessor extends Number {
         public long value;
         @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/SignificanceHeuristic.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/SignificanceHeuristic.java
index db9711c1a8de5..7b6cf699741c6 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/SignificanceHeuristic.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/SignificanceHeuristic.java
@@ -50,11 +50,23 @@ protected void checkFrequencyValidity(long subsetFreq, long subsetSize, long sup
         }
     }
 
-    public void initialize(InternalAggregation.ReduceContext reduceContext) {
-
+    /**
+     * Provides a hook for subclasses to provide a version of the heuristic
+     * prepared for execution on data on the coordinating node.
+     * @param reduceContext the reduce context on the coordinating node
+     * @return a version of this heuristic suitable for execution
+     */
+    public SignificanceHeuristic rewrite(InternalAggregation.ReduceContext reduceContext) {
+        return this;
     }
 
-    public void initialize(SearchContext context) {
-
+    /**
+     * Provides a hook for subclasses to provide a version of the heuristic
+     * prepared for execution on data on a shard. 
+     * @param context the search context on the data node
+     * @return a version of this heuristic suitable for execution
+     */
+    public SignificanceHeuristic rewrite(SearchContext context) {
+        return this;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/SignificanceHeuristicParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/SignificanceHeuristicParser.java
index 26fd552a6b142..3c6f98c155eac 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/SignificanceHeuristicParser.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/SignificanceHeuristicParser.java
@@ -22,7 +22,7 @@
 
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.search.aggregations.support.XContentParseContext;
+import org.elasticsearch.index.query.QueryParseContext;
 
 import java.io.IOException;
 
@@ -31,5 +31,5 @@
  */
 @FunctionalInterface
 public interface SignificanceHeuristicParser {
-    SignificanceHeuristic parse(XContentParseContext context) throws IOException, ParsingException;
+    SignificanceHeuristic parse(QueryParseContext context) throws IOException, ParsingException;
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/AbstractStringTermsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/AbstractStringTermsAggregator.java
index 5719fe7721b3e..9b40fcc42dcc6 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/AbstractStringTermsAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/AbstractStringTermsAggregator.java
@@ -24,7 +24,7 @@
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -36,7 +36,7 @@ abstract class AbstractStringTermsAggregator extends TermsAggregator {
 
     protected final boolean showTermDocCountError;
 
-    public AbstractStringTermsAggregator(String name, AggregatorFactories factories, AggregationContext context, Aggregator parent,
+    AbstractStringTermsAggregator(String name, AggregatorFactories factories, SearchContext context, Aggregator parent,
             Terms.Order order, DocValueFormat format, BucketCountThresholds bucketCountThresholds, SubAggCollectionMode subAggCollectMode,
             boolean showTermDocCountError, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException {
         super(name, factories, context, parent, bucketCountThresholds, order, format, subAggCollectMode, pipelineAggregators, metaData);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/AbstractTermsParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/AbstractTermsParser.java
deleted file mode 100644
index a106cea3a1596..0000000000000
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/AbstractTermsParser.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.aggregations.bucket.terms;
-
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentParser.Token;
-import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode;
-import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator.BucketCountThresholds;
-import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude;
-import org.elasticsearch.search.aggregations.support.AbstractValuesSourceParser.AnyValuesSourceParser;
-import org.elasticsearch.search.aggregations.support.XContentParseContext;
-import org.elasticsearch.search.aggregations.support.ValueType;
-import org.elasticsearch.search.aggregations.support.ValuesSource;
-import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
-import org.elasticsearch.search.aggregations.support.ValuesSourceType;
-
-import java.io.IOException;
-import java.util.Map;
-
-public abstract class AbstractTermsParser extends AnyValuesSourceParser {
-
-    public static final ParseField EXECUTION_HINT_FIELD_NAME = new ParseField("execution_hint");
-    public static final ParseField SHARD_SIZE_FIELD_NAME = new ParseField("shard_size");
-    public static final ParseField MIN_DOC_COUNT_FIELD_NAME = new ParseField("min_doc_count");
-    public static final ParseField SHARD_MIN_DOC_COUNT_FIELD_NAME = new ParseField("shard_min_doc_count");
-    public static final ParseField REQUIRED_SIZE_FIELD_NAME = new ParseField("size");
-
-    public IncludeExclude.Parser incExcParser = new IncludeExclude.Parser();
-
-    protected AbstractTermsParser() {
-        super(true, true);
-    }
-
-    @Override
-    protected final ValuesSourceAggregationBuilder<ValuesSource, ?> createFactory(String aggregationName,
-                                                                                  ValuesSourceType valuesSourceType,
-                                                                                  ValueType targetValueType,
-                                                                                  Map<ParseField, Object> otherOptions) {
-        BucketCountThresholds bucketCountThresholds = getDefaultBucketCountThresholds();
-        Integer requiredSize = (Integer) otherOptions.get(REQUIRED_SIZE_FIELD_NAME);
-        if (requiredSize != null && requiredSize != -1) {
-            bucketCountThresholds.setRequiredSize(requiredSize);
-        }
-        Integer shardSize = (Integer) otherOptions.get(SHARD_SIZE_FIELD_NAME);
-        if (shardSize != null && shardSize != -1) {
-            bucketCountThresholds.setShardSize(shardSize);
-        }
-        Long minDocCount = (Long) otherOptions.get(MIN_DOC_COUNT_FIELD_NAME);
-        if (minDocCount != null && minDocCount != -1) {
-            bucketCountThresholds.setMinDocCount(minDocCount);
-        }
-        Long shardMinDocCount = (Long) otherOptions.get(SHARD_MIN_DOC_COUNT_FIELD_NAME);
-        if (shardMinDocCount != null && shardMinDocCount != -1) {
-            bucketCountThresholds.setShardMinDocCount(shardMinDocCount);
-        }
-        SubAggCollectionMode collectMode = (SubAggCollectionMode) otherOptions.get(SubAggCollectionMode.KEY);
-        String executionHint = (String) otherOptions.get(EXECUTION_HINT_FIELD_NAME);
-        IncludeExclude incExc = incExcParser.createIncludeExclude(otherOptions);
-        return doCreateFactory(aggregationName, valuesSourceType, targetValueType, bucketCountThresholds, collectMode, executionHint,
-                incExc,
-                otherOptions);
-    }
-
-    protected abstract ValuesSourceAggregationBuilder<ValuesSource, ?> doCreateFactory(String aggregationName,
-                                                                                       ValuesSourceType valuesSourceType,
-                                                                                       ValueType targetValueType,
-                                                                                       BucketCountThresholds bucketCountThresholds,
-                                                                                       SubAggCollectionMode collectMode,
-                                                                                       String executionHint,
-                                                                                       IncludeExclude incExc,
-                                                                                       Map<ParseField, Object> otherOptions);
-
-    @Override
-    protected boolean token(String aggregationName, String currentFieldName, Token token,
-                            XContentParseContext context, Map<ParseField, Object> otherOptions) throws IOException {
-        XContentParser parser = context.getParser();
-        if (incExcParser.token(currentFieldName, token, parser, context.getParseFieldMatcher(), otherOptions)) {
-            return true;
-        } else if (token == XContentParser.Token.VALUE_STRING) {
-            if (context.matchField(currentFieldName, EXECUTION_HINT_FIELD_NAME)) {
-                otherOptions.put(EXECUTION_HINT_FIELD_NAME, parser.text());
-                return true;
-            } else if (context.matchField(currentFieldName, SubAggCollectionMode.KEY)) {
-                otherOptions.put(SubAggCollectionMode.KEY, SubAggCollectionMode.parse(parser.text(), context.getParseFieldMatcher()));
-                return true;
-            } else if (context.matchField(currentFieldName, REQUIRED_SIZE_FIELD_NAME)) {
-                otherOptions.put(REQUIRED_SIZE_FIELD_NAME, parser.intValue());
-                return true;
-            } else if (parseSpecial(aggregationName, context, token, currentFieldName, otherOptions)) {
-                return true;
-            }
-        } else if (token == XContentParser.Token.VALUE_NUMBER) {
-            if (context.matchField(currentFieldName, REQUIRED_SIZE_FIELD_NAME)) {
-                otherOptions.put(REQUIRED_SIZE_FIELD_NAME, parser.intValue());
-                return true;
-            } else if (context.matchField(currentFieldName, SHARD_SIZE_FIELD_NAME)) {
-                otherOptions.put(SHARD_SIZE_FIELD_NAME, parser.intValue());
-                return true;
-            } else if (context.matchField(currentFieldName, MIN_DOC_COUNT_FIELD_NAME)) {
-                otherOptions.put(MIN_DOC_COUNT_FIELD_NAME, parser.longValue());
-                return true;
-            } else if (context.matchField(currentFieldName, SHARD_MIN_DOC_COUNT_FIELD_NAME)) {
-                otherOptions.put(SHARD_MIN_DOC_COUNT_FIELD_NAME, parser.longValue());
-                return true;
-            } else if (parseSpecial(aggregationName, context, token, currentFieldName, otherOptions)) {
-                return true;
-            }
-        } else if (parseSpecial(aggregationName, context, token, currentFieldName, otherOptions)) {
-            return true;
-        }
-        return false;
-    }
-
-    public abstract boolean parseSpecial(String aggregationName, XContentParseContext context,
-                                         Token token, String currentFieldName, Map<ParseField, Object> otherOptions) throws IOException;
-
-    protected abstract TermsAggregator.BucketCountThresholds getDefaultBucketCountThresholds();
-
-}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTerms.java
index 7e3daf5034b83..830f4d359467a 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTerms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTerms.java
@@ -38,7 +38,7 @@ public class DoubleTerms extends InternalMappedTerms<DoubleTerms, DoubleTerms.Bu
     static class Bucket extends InternalTerms.Bucket<Bucket> {
         private final double term;
 
-        public Bucket(double term, long docCount, InternalAggregations aggregations, boolean showDocCountError, long docCountError,
+        Bucket(double term, long docCount, InternalAggregations aggregations, boolean showDocCountError, long docCountError,
                 DocValueFormat format) {
             super(docCount, aggregations, showDocCountError, docCountError, format);
             this.term = term;
@@ -47,7 +47,7 @@ public Bucket(double term, long docCount, InternalAggregations aggregations, boo
         /**
          * Read from a stream.
          */
-        public Bucket(StreamInput in, DocValueFormat format, boolean showDocCountError) throws IOException {
+        Bucket(StreamInput in, DocValueFormat format, boolean showDocCountError) throws IOException {
             super(in, format, showDocCountError);
             term = in.readDouble();
         }
@@ -73,7 +73,7 @@ public Number getKeyAsNumber() {
         }
 
         @Override
-        int compareTerm(Terms.Bucket other) {
+        public int compareTerm(Terms.Bucket other) {
             return Double.compare(term, ((Number) other.getKey()).doubleValue());
         }
 
@@ -83,18 +83,11 @@ Bucket newBucket(long docCount, InternalAggregations aggs, long docCountError) {
         }
 
         @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startObject();
-            builder.field(CommonFields.KEY, term);
+        protected final XContentBuilder keyToXContent(XContentBuilder builder) throws IOException {
+            builder.field(CommonFields.KEY.getPreferredName(), term);
             if (format != DocValueFormat.RAW) {
-                builder.field(CommonFields.KEY_AS_STRING, format.format(term));
+                builder.field(CommonFields.KEY_AS_STRING.getPreferredName(), format.format(term));
             }
-            builder.field(CommonFields.DOC_COUNT, getDocCount());
-            if (showDocCountError) {
-                builder.field(InternalTerms.DOC_COUNT_ERROR_UPPER_BOUND_FIELD_NAME, getDocCountError());
-            }
-            aggregations.toXContentInternal(builder, params);
-            builder.endObject();
             return builder;
         }
     }
@@ -136,20 +129,9 @@ protected DoubleTerms create(String name, List<Bucket> buckets, long docCountErr
                 shardSize, showTermDocCountError, otherDocCount, buckets, docCountError);
     }
 
-    @Override
-    public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
-        builder.field(InternalTerms.DOC_COUNT_ERROR_UPPER_BOUND_FIELD_NAME, docCountError);
-        builder.field(SUM_OF_OTHER_DOC_COUNTS, otherDocCount);
-        builder.startArray(CommonFields.BUCKETS);
-        for (Bucket bucket : buckets) {
-            bucket.toXContent(builder, params);
-        }
-        builder.endArray();
-        return builder;
-    }
-
     @Override
     protected Bucket[] createBucketsArray(int size) {
         return new Bucket[size];
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTermsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTermsAggregator.java
index b2b57f9d06004..3b8fdd795f2fa 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTermsAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTermsAggregator.java
@@ -27,9 +27,9 @@
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -42,7 +42,7 @@
 public class DoubleTermsAggregator extends LongTermsAggregator {
 
     public DoubleTermsAggregator(String name, AggregatorFactories factories, ValuesSource.Numeric valuesSource, DocValueFormat format,
-            Terms.Order order, BucketCountThresholds bucketCountThresholds, AggregationContext aggregationContext, Aggregator parent,
+            Terms.Order order, BucketCountThresholds bucketCountThresholds, SearchContext aggregationContext, Aggregator parent,
             SubAggCollectionMode collectionMode, boolean showTermDocCountError, IncludeExclude.LongFilter longFilter,
             List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException {
         super(name, factories, valuesSource, format, order, bucketCountThresholds, aggregationContext, parent, collectionMode,
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java
index 00a066c11ba38..62bad9313a774 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java
@@ -20,13 +20,13 @@
 package org.elasticsearch.search.aggregations.bucket.terms;
 
 import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.RandomAccessOrds;
 import org.apache.lucene.index.SortedDocValues;
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.LongBitSet;
-import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.util.IntArray;
@@ -44,8 +44,8 @@
 import org.elasticsearch.search.aggregations.bucket.terms.support.BucketPriorityQueue;
 import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -66,66 +66,106 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr
     // first defined one.
     // So currently for each instance of this aggregator the acceptedglobalValues will be computed, this is unnecessary
     // especially if this agg is on a second layer or deeper.
-    protected LongBitSet acceptedGlobalOrdinals;
+    protected final LongBitSet acceptedGlobalOrdinals;
+    protected final long valueCount;
+    protected final GlobalOrdLookupFunction lookupGlobalOrd;
 
-    protected RandomAccessOrds globalOrds;
+    private final LongHash bucketOrds;
 
-    public GlobalOrdinalsStringTermsAggregator(String name, AggregatorFactories factories, ValuesSource.Bytes.WithOrdinals valuesSource,
-           Terms.Order order, DocValueFormat format, BucketCountThresholds bucketCountThresholds,
-           IncludeExclude.OrdinalsFilter includeExclude, AggregationContext aggregationContext, Aggregator parent,
-           SubAggCollectionMode collectionMode, boolean showTermDocCountError, List<PipelineAggregator> pipelineAggregators,
-           Map<String, Object> metaData) throws IOException {
-        super(name, factories, aggregationContext, parent, order, format, bucketCountThresholds, collectionMode, showTermDocCountError,
+    public interface GlobalOrdLookupFunction {
+        BytesRef apply(long ord) throws IOException;
+    }
+
+    public GlobalOrdinalsStringTermsAggregator(String name, AggregatorFactories factories,
+                                               ValuesSource.Bytes.WithOrdinals valuesSource,
+                                               Terms.Order order,
+                                               DocValueFormat format,
+                                               BucketCountThresholds bucketCountThresholds,
+                                               IncludeExclude.OrdinalsFilter includeExclude,
+                                               SearchContext context,
+                                               Aggregator parent,
+                                               boolean forceRemapGlobalOrds,
+                                               SubAggCollectionMode collectionMode,
+                                               boolean showTermDocCountError,
+                                               List<PipelineAggregator> pipelineAggregators,
+                                               Map<String, Object> metaData) throws IOException {
+        super(name, factories, context, parent, order, format, bucketCountThresholds, collectionMode, showTermDocCountError,
                 pipelineAggregators, metaData);
         this.valuesSource = valuesSource;
         this.includeExclude = includeExclude;
+        final IndexReader reader = context.searcher().getIndexReader();
+        final RandomAccessOrds values = reader.leaves().size() > 0 ?
+            valuesSource.globalOrdinalsValues(context.searcher().getIndexReader().leaves().get(0)) : DocValues.emptySortedSet();
+        this.valueCount = values.getValueCount();
+        this.lookupGlobalOrd = values::lookupOrd;
+        this.acceptedGlobalOrdinals = includeExclude != null ? includeExclude.acceptedGlobalOrdinals(values) : null;
+
+        /**
+         * Remap global ords to dense bucket ordinals if any sub-aggregator cannot be deferred.
+         * Sub-aggregators expect dense buckets and allocate memories based on this assumption.
+         * Deferred aggregators are safe because the selected ordinals are remapped when the buckets
+         * are replayed.
+         */
+        boolean remapGlobalOrds = forceRemapGlobalOrds || Arrays.stream(subAggregators).anyMatch((a) -> shouldDefer(a) == false);
+        this.bucketOrds = remapGlobalOrds ? new LongHash(1, context.bigArrays()) : null;
     }
 
-    protected long getBucketOrd(long termOrd) {
-        return termOrd;
-    }
-
-    @Override
-    public LeafBucketCollector getLeafCollector(LeafReaderContext ctx,
-            final LeafBucketCollector sub) throws IOException {
 
-        globalOrds = valuesSource.globalOrdinalsValues(ctx);
+    boolean remapGlobalOrds() {
+        return bucketOrds != null;
+    }
 
-        if (acceptedGlobalOrdinals == null && includeExclude != null) {
-            acceptedGlobalOrdinals = includeExclude.acceptedGlobalOrdinals(globalOrds);
-        }
+    protected final long getBucketOrd(long globalOrd) {
+        return bucketOrds == null ? globalOrd : bucketOrds.find(globalOrd);
+    }
 
-        if (acceptedGlobalOrdinals != null) {
-            globalOrds = new FilteredOrdinals(globalOrds, acceptedGlobalOrdinals);
+    private void collectGlobalOrd(int doc, long globalOrd, LeafBucketCollector sub) throws IOException {
+        if (bucketOrds == null) {
+            collectExistingBucket(sub, doc, globalOrd);
+        } else {
+            long bucketOrd = bucketOrds.add(globalOrd);
+            if (bucketOrd < 0) {
+                bucketOrd = -1 - bucketOrd;
+                collectExistingBucket(sub, doc, bucketOrd);
+            } else {
+                collectBucket(sub, doc, bucketOrd);
+            }
         }
+    }
 
-        return newCollector(globalOrds, sub);
+    private RandomAccessOrds getGlobalOrds(LeafReaderContext ctx) throws IOException {
+        return acceptedGlobalOrdinals == null ?
+            valuesSource.globalOrdinalsValues(ctx) : new FilteredOrdinals(valuesSource.globalOrdinalsValues(ctx), acceptedGlobalOrdinals);
     }
 
-    protected LeafBucketCollector newCollector(final RandomAccessOrds ords, final LeafBucketCollector sub) {
-        grow(ords.getValueCount());
-        final SortedDocValues singleValues = DocValues.unwrapSingleton(ords);
+    @Override
+    public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, final LeafBucketCollector sub) throws IOException {
+        final RandomAccessOrds globalOrds = getGlobalOrds(ctx);
+        if (bucketOrds == null) {
+            grow(globalOrds.getValueCount());
+        }
+        final SortedDocValues singleValues = DocValues.unwrapSingleton(globalOrds);
         if (singleValues != null) {
-            return new LeafBucketCollectorBase(sub, ords) {
+            return new LeafBucketCollectorBase(sub, globalOrds) {
                 @Override
                 public void collect(int doc, long bucket) throws IOException {
                     assert bucket == 0;
                     final int ord = singleValues.getOrd(doc);
                     if (ord >= 0) {
-                        collectExistingBucket(sub, doc, ord);
+                        collectGlobalOrd(doc, ord, sub);
                     }
                 }
             };
         } else {
-            return new LeafBucketCollectorBase(sub, ords) {
+            return new LeafBucketCollectorBase(sub, globalOrds) {
                 @Override
                 public void collect(int doc, long bucket) throws IOException {
                     assert bucket == 0;
-                    ords.setDocument(doc);
-                    final int numOrds = ords.cardinality();
+                    globalOrds.setDocument(doc);
+                    final int numOrds = globalOrds.cardinality();
                     for (int i = 0; i < numOrds; i++) {
-                        final long globalOrd = ords.ordAt(i);
-                        collectExistingBucket(sub, doc, globalOrd);
+                        final long globalOrd = globalOrds.ordAt(i);
+                        collectGlobalOrd(doc, globalOrd, sub);
                     }
                 }
             };
@@ -143,21 +183,21 @@ protected static void copy(BytesRef from, BytesRef to) {
 
     @Override
     public InternalAggregation buildAggregation(long owningBucketOrdinal) throws IOException {
-        if (globalOrds == null) { // no context in this reader
+        if (valueCount == 0) { // no context in this reader
             return buildEmptyAggregation();
         }
 
         final int size;
         if (bucketCountThresholds.getMinDocCount() == 0) {
             // if minDocCount == 0 then we can end up with more buckets then maxBucketOrd() returns
-            size = (int) Math.min(globalOrds.getValueCount(), bucketCountThresholds.getShardSize());
+            size = (int) Math.min(valueCount, bucketCountThresholds.getShardSize());
         } else {
             size = (int) Math.min(maxBucketOrd(), bucketCountThresholds.getShardSize());
         }
         long otherDocCount = 0;
         BucketPriorityQueue<OrdBucket> ordered = new BucketPriorityQueue<>(size, order.comparator(this));
         OrdBucket spare = new OrdBucket(-1, 0, null, showTermDocCountError, 0);
-        for (long globalTermOrd = 0; globalTermOrd < globalOrds.getValueCount(); ++globalTermOrd) {
+        for (long globalTermOrd = 0; globalTermOrd < valueCount; ++globalTermOrd) {
             if (includeExclude != null && !acceptedGlobalOrdinals.get(globalTermOrd)) {
                 continue;
             }
@@ -182,10 +222,10 @@ public InternalAggregation buildAggregation(long owningBucketOrdinal) throws IOE
         final StringTerms.Bucket[] list = new StringTerms.Bucket[ordered.size()];
         long survivingBucketOrds[] = new long[ordered.size()];
         for (int i = ordered.size() - 1; i >= 0; --i) {
-            final OrdBucket bucket = (OrdBucket) ordered.pop();
+            final OrdBucket bucket = ordered.pop();
             survivingBucketOrds[i] = bucket.bucketOrd;
             BytesRef scratch = new BytesRef();
-            copy(globalOrds.lookupOrd(bucket.globalOrd), scratch);
+            copy(lookupGlobalOrd.apply(bucket.globalOrd), scratch);
             list[i] = new StringTerms.Bucket(scratch, bucket.docCount, null, showTermDocCountError, 0, format);
             list[i].bucketOrd = bucket.bucketOrd;
             otherDocCount -= list[i].docCount;
@@ -217,7 +257,7 @@ static class OrdBucket extends InternalTerms.Bucket<OrdBucket> {
         }
 
         @Override
-        int compareTerm(Terms.Bucket other) {
+        public int compareTerm(Terms.Bucket other) {
             return Long.compare(globalOrd, ((OrdBucket) other).globalOrd);
         }
 
@@ -247,79 +287,14 @@ protected void writeTermTo(StreamOutput out) throws IOException {
         }
 
         @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        protected final XContentBuilder keyToXContent(XContentBuilder builder) throws IOException {
             throw new UnsupportedOperationException();
         }
     }
 
-    /**
-     * Variant of {@link GlobalOrdinalsStringTermsAggregator} that rebases hashes in order to make them dense. Might be
-     * useful in case few hashes are visited.
-     */
-    public static class WithHash extends GlobalOrdinalsStringTermsAggregator {
-
-        private final LongHash bucketOrds;
-
-        public WithHash(String name, AggregatorFactories factories, ValuesSource.Bytes.WithOrdinals valuesSource, Terms.Order order,
-                DocValueFormat format, BucketCountThresholds bucketCountThresholds, IncludeExclude.OrdinalsFilter includeExclude,
-                AggregationContext aggregationContext, Aggregator parent, SubAggCollectionMode collectionMode,
-                boolean showTermDocCountError, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData)
-                        throws IOException {
-            super(name, factories, valuesSource, order, format, bucketCountThresholds, includeExclude, aggregationContext, parent, collectionMode,
-                    showTermDocCountError, pipelineAggregators, metaData);
-            bucketOrds = new LongHash(1, aggregationContext.bigArrays());
-        }
-
-        @Override
-        protected LeafBucketCollector newCollector(final RandomAccessOrds ords, final LeafBucketCollector sub) {
-            final SortedDocValues singleValues = DocValues.unwrapSingleton(ords);
-            if (singleValues != null) {
-                return new LeafBucketCollectorBase(sub, ords) {
-                    @Override
-                    public void collect(int doc, long bucket) throws IOException {
-                        final int globalOrd = singleValues.getOrd(doc);
-                        if (globalOrd >= 0) {
-                            long bucketOrd = bucketOrds.add(globalOrd);
-                            if (bucketOrd < 0) {
-                                bucketOrd = -1 - bucketOrd;
-                                collectExistingBucket(sub, doc, bucketOrd);
-                            } else {
-                                collectBucket(sub, doc, bucketOrd);
-                            }
-                        }
-                    }
-                };
-            } else {
-                return new LeafBucketCollectorBase(sub, ords) {
-                    @Override
-                    public void collect(int doc, long bucket) throws IOException {
-                        ords.setDocument(doc);
-                        final int numOrds = ords.cardinality();
-                        for (int i = 0; i < numOrds; i++) {
-                            final long globalOrd = ords.ordAt(i);
-                            long bucketOrd = bucketOrds.add(globalOrd);
-                            if (bucketOrd < 0) {
-                                bucketOrd = -1 - bucketOrd;
-                                collectExistingBucket(sub, doc, bucketOrd);
-                            } else {
-                                collectBucket(sub, doc, bucketOrd);
-                            }
-                        }
-                    }
-                };
-            }
-        }
-
-        @Override
-        protected long getBucketOrd(long termOrd) {
-            return bucketOrds.find(termOrd);
-        }
-
-        @Override
-        protected void doClose() {
-            Releasables.close(bucketOrds);
-        }
-
+    @Override
+    protected void doClose() {
+        Releasables.close(bucketOrds);
     }
 
     /**
@@ -327,31 +302,43 @@ protected void doClose() {
      * instead of on the fly for each match.This is beneficial for low cardinality fields, because it can reduce
      * the amount of look-ups significantly.
      */
-    public static class LowCardinality extends GlobalOrdinalsStringTermsAggregator {
+    static class LowCardinality extends GlobalOrdinalsStringTermsAggregator {
 
         private IntArray segmentDocCounts;
-
+        private RandomAccessOrds globalOrds;
         private RandomAccessOrds segmentOrds;
 
-        public LowCardinality(String name, AggregatorFactories factories, ValuesSource.Bytes.WithOrdinals valuesSource,
-                Terms.Order order, DocValueFormat format,
-                BucketCountThresholds bucketCountThresholds, AggregationContext aggregationContext, Aggregator parent,
-                SubAggCollectionMode collectionMode, boolean showTermDocCountError, List<PipelineAggregator> pipelineAggregators,
-                Map<String, Object> metaData) throws IOException {
-            super(name, factories, valuesSource, order, format, bucketCountThresholds, null, aggregationContext, parent, collectionMode,
-                    showTermDocCountError, pipelineAggregators, metaData);
-            assert factories == null || factories.countAggregators() == 0;
+        LowCardinality(String name,
+                       AggregatorFactories factories,
+                       ValuesSource.Bytes.WithOrdinals valuesSource,
+                       Terms.Order order,
+                       DocValueFormat format,
+                       BucketCountThresholds bucketCountThresholds,
+                       SearchContext context,
+                       Aggregator parent,
+                       boolean forceDenseMode,
+                       SubAggCollectionMode collectionMode,
+                       boolean showTermDocCountError,
+                       List<PipelineAggregator> pipelineAggregators,
+                       Map<String, Object> metaData) throws IOException {
+            super(name, factories, valuesSource, order, format, bucketCountThresholds, null, context, parent, forceDenseMode,
+                collectionMode, showTermDocCountError, pipelineAggregators, metaData);
             this.segmentDocCounts = context.bigArrays().newIntArray(1, true);
         }
 
-        // bucketOrd is ord + 1 to avoid a branch to deal with the missing ord
         @Override
-        protected LeafBucketCollector newCollector(final RandomAccessOrds ords, LeafBucketCollector sub) {
-            segmentDocCounts = context.bigArrays().grow(segmentDocCounts, 1 + ords.getValueCount());
+        public LeafBucketCollector getLeafCollector(LeafReaderContext ctx,
+                                                    final LeafBucketCollector sub) throws IOException {
+            if (segmentOrds != null) {
+                mapSegmentCountsToGlobalCounts();
+            }
+            globalOrds = valuesSource.globalOrdinalsValues(ctx);
+            segmentOrds = valuesSource.ordinalsValues(ctx);
+            segmentDocCounts = context.bigArrays().grow(segmentDocCounts, 1 + segmentOrds.getValueCount());
             assert sub == LeafBucketCollector.NO_OP_COLLECTOR;
-            final SortedDocValues singleValues = DocValues.unwrapSingleton(ords);
+            final SortedDocValues singleValues = DocValues.unwrapSingleton(segmentOrds);
             if (singleValues != null) {
-                return new LeafBucketCollectorBase(sub, ords) {
+                return new LeafBucketCollectorBase(sub, segmentOrds) {
                     @Override
                     public void collect(int doc, long bucket) throws IOException {
                         assert bucket == 0;
@@ -360,14 +347,14 @@ public void collect(int doc, long bucket) throws IOException {
                     }
                 };
             } else {
-                return new LeafBucketCollectorBase(sub, ords) {
+                return new LeafBucketCollectorBase(sub, segmentOrds) {
                     @Override
                     public void collect(int doc, long bucket) throws IOException {
                         assert bucket == 0;
-                        ords.setDocument(doc);
-                        final int numOrds = ords.cardinality();
+                        segmentOrds.setDocument(doc);
+                        final int numOrds = segmentOrds.cardinality();
                         for (int i = 0; i < numOrds; i++) {
-                            final long segmentOrd = ords.ordAt(i);
+                            final long segmentOrd = segmentOrds.ordAt(i);
                             segmentDocCounts.increment(segmentOrd + 1, 1);
                         }
                     }
@@ -375,18 +362,6 @@ public void collect(int doc, long bucket) throws IOException {
             }
         }
 
-        @Override
-        public LeafBucketCollector getLeafCollector(LeafReaderContext ctx,
-                final LeafBucketCollector sub) throws IOException {
-            if (segmentOrds != null) {
-                mapSegmentCountsToGlobalCounts();
-            }
-
-            globalOrds = valuesSource.globalOrdinalsValues(ctx);
-            segmentOrds = valuesSource.ordinalsValues(ctx);
-            return newCollector(segmentOrds, sub);
-        }
-
         @Override
         protected void doPostCollection() {
             if (segmentOrds != null) {
@@ -418,7 +393,8 @@ private void mapSegmentCountsToGlobalCounts() {
                 }
                 final long ord = i - 1; // remember we do +1 when counting
                 final long globalOrd = mapping == null ? ord : mapping.getGlobalOrd(ord);
-                incrementBucketDocCount(globalOrd, inc);
+                long bucketOrd = getBucketOrd(globalOrd);
+                incrementBucketDocCount(bucketOrd, inc);
             }
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalMappedTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalMappedTerms.java
index e3f842a08de03..a83aad541bd70 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalMappedTerms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalMappedTerms.java
@@ -21,10 +21,12 @@
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 
 import java.io.IOException;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.function.Function;
@@ -99,7 +101,7 @@ public long getSumOfOtherDocCounts() {
     }
 
     @Override
-    public List<B> getBucketsInternal() {
+    public List<B> getBuckets() {
         return buckets;
     }
 
@@ -110,4 +112,9 @@ public B getBucketByKey(String term) {
         }
         return bucketMap.get(term);
     }
+
+    @Override
+    public final XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+        return doXContentCommon(builder, params, docCountError, otherDocCount, buckets);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalOrder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalOrder.java
index f3f87c09dca7a..3b85fa71d99d1 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalOrder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalOrder.java
@@ -244,11 +244,11 @@ static class CompoundOrder extends Terms.Order {
 
         private final List<Terms.Order> orderElements;
 
-        public CompoundOrder(List<Terms.Order> compoundOrder) {
+        CompoundOrder(List<Terms.Order> compoundOrder) {
             this(compoundOrder, true);
         }
 
-        public CompoundOrder(List<Terms.Order> compoundOrder, boolean absoluteOrdering) {
+        CompoundOrder(List<Terms.Order> compoundOrder, boolean absoluteOrdering) {
             this.orderElements = new LinkedList<>(compoundOrder);
             Terms.Order lastElement = compoundOrder.get(compoundOrder.size() - 1);
             if (absoluteOrdering && !(InternalOrder.TERM_ASC == lastElement || InternalOrder.TERM_DESC == lastElement)) {
@@ -303,7 +303,7 @@ public static class CompoundOrderComparator implements Comparator<Terms.Bucket>
             private List<Terms.Order> compoundOrder;
             private Aggregator aggregator;
 
-            public CompoundOrderComparator(List<Terms.Order> compoundOrder, Aggregator aggregator) {
+            CompoundOrderComparator(List<Terms.Order> compoundOrder, Aggregator aggregator) {
                 this.compoundOrder = compoundOrder;
                 this.aggregator = aggregator;
             }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java
index a8b4c44ce462d..d62cd83473a38 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java
@@ -18,9 +18,11 @@
  */
 package org.elasticsearch.search.aggregations.bucket.terms;
 
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.AggregationExecutionException;
 import org.elasticsearch.search.aggregations.Aggregations;
@@ -37,15 +39,14 @@
 import java.util.List;
 import java.util.Map;
 
-import static java.util.Collections.unmodifiableList;
-
 public abstract class InternalTerms<A extends InternalTerms<A, B>, B extends InternalTerms.Bucket<B>>
         extends InternalMultiBucketAggregation<A, B> implements Terms, ToXContent {
 
-    protected static final String DOC_COUNT_ERROR_UPPER_BOUND_FIELD_NAME = "doc_count_error_upper_bound";
-    protected static final String SUM_OF_OTHER_DOC_COUNTS = "sum_other_doc_count";
+    protected static final ParseField DOC_COUNT_ERROR_UPPER_BOUND_FIELD_NAME = new ParseField("doc_count_error_upper_bound");
+    protected static final ParseField SUM_OF_OTHER_DOC_COUNTS = new ParseField("sum_other_doc_count");
+
+    public abstract static class Bucket<B extends Bucket<B>> extends InternalMultiBucketAggregation.InternalBucket implements Terms.Bucket {
 
-    public abstract static class Bucket<B extends Bucket<B>> extends Terms.Bucket {
         /**
          * Reads a bucket. Should be a constructor reference.
          */
@@ -119,6 +120,10 @@ public Aggregations getAggregations() {
 
         public B reduce(List<B> buckets, ReduceContext context) {
             long docCount = 0;
+            // For the per term doc count error we add up the errors from the
+            // shards that did not respond with the term. To do this we add up
+            // the errors from the shards that did respond with the terms and
+            // subtract that from the sum of the error from all shards
             long docCountError = 0;
             List<InternalAggregations> aggregationsList = new ArrayList<>(buckets.size());
             for (B bucket : buckets) {
@@ -135,6 +140,21 @@ public B reduce(List<B> buckets, ReduceContext context) {
             InternalAggregations aggs = InternalAggregations.reduce(aggregationsList, context);
             return newBucket(docCount, aggs, docCountError);
         }
+
+        @Override
+        public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            keyToXContent(builder);
+            builder.field(CommonFields.DOC_COUNT.getPreferredName(), getDocCount());
+            if (showDocCountError) {
+                builder.field(InternalTerms.DOC_COUNT_ERROR_UPPER_BOUND_FIELD_NAME.getPreferredName(), getDocCountError());
+            }
+            aggregations.toXContentInternal(builder, params);
+            builder.endObject();
+            return builder;
+        }
+
+        protected abstract XContentBuilder keyToXContent(XContentBuilder builder) throws IOException;
     }
 
     protected final Terms.Order order;
@@ -170,11 +190,7 @@ protected final void doWriteTo(StreamOutput out) throws IOException {
     protected abstract void writeTermTypeInfoTo(StreamOutput out) throws IOException;
 
     @Override
-    public final List<Terms.Bucket> getBuckets() {
-        return unmodifiableList(getBucketsInternal());
-    }
-
-    protected abstract List<B> getBucketsInternal();
+    public abstract List<B> getBuckets();
 
     @Override
     public abstract B getBucketByKey(String term);
@@ -202,10 +218,18 @@ public InternalAggregation doReduce(List<InternalAggregation> aggregations, Redu
             }
             otherDocCount += terms.getSumOfOtherDocCounts();
             final long thisAggDocCountError;
-            if (terms.getBucketsInternal().size() < getShardSize() || InternalOrder.isTermOrder(order)) {
+            if (terms.getBuckets().size() < getShardSize() || InternalOrder.isTermOrder(order)) {
                 thisAggDocCountError = 0;
             } else if (InternalOrder.isCountDesc(this.order)) {
-                thisAggDocCountError = terms.getBucketsInternal().get(terms.getBucketsInternal().size() - 1).docCount;
+                if (terms.getDocCountError() > 0) {
+                    // If there is an existing docCountError for this agg then
+                    // use this as the error for this aggregation
+                    thisAggDocCountError = terms.getDocCountError();
+                } else {
+                    // otherwise use the doc count of the last term in the
+                    // aggregation
+                    thisAggDocCountError = terms.getBuckets().get(terms.getBuckets().size() - 1).docCount;
+                }
             } else {
                 thisAggDocCountError = -1;
             }
@@ -217,8 +241,15 @@ public InternalAggregation doReduce(List<InternalAggregation> aggregations, Redu
                 }
             }
             setDocCountError(thisAggDocCountError);
-            for (B bucket : terms.getBucketsInternal()) {
-                bucket.docCountError = thisAggDocCountError;
+            for (B bucket : terms.getBuckets()) {
+                // If there is already a doc count error for this bucket
+                // subtract this aggs doc count error from it to make the
+                // new value for the bucket. This then means that when the
+                // final error for the bucket is calculated below we account
+                // for the existing error calculated in a previous reduce.
+                // Note that if the error is unbounded (-1) this will be fixed
+                // later in this method.
+                bucket.docCountError -= thisAggDocCountError;
                 List<B> bucketList = buckets.get(bucket.getKey());
                 if (bucketList == null) {
                     bucketList = new ArrayList<>();
@@ -228,18 +259,16 @@ public InternalAggregation doReduce(List<InternalAggregation> aggregations, Redu
             }
         }
 
-        final int size = Math.min(requiredSize, buckets.size());
-        BucketPriorityQueue<B> ordered = new BucketPriorityQueue<>(size, order.comparator(null));
+        final int size = reduceContext.isFinalReduce() == false ? buckets.size() : Math.min(requiredSize, buckets.size());
+        final BucketPriorityQueue<B> ordered = new BucketPriorityQueue<>(size, order.comparator(null));
         for (List<B> sameTermBuckets : buckets.values()) {
             final B b = sameTermBuckets.get(0).reduce(sameTermBuckets, reduceContext);
-            if (b.docCountError != -1) {
-                if (sumDocCountError == -1) {
-                    b.docCountError = -1;
-                } else {
-                    b.docCountError = sumDocCountError - b.docCountError;
-                }
+            if (sumDocCountError == -1) {
+                b.docCountError = -1;
+            } else {
+                b.docCountError += sumDocCountError;
             }
-            if (b.docCount >= minDocCount) {
+            if (b.docCount >= minDocCount || reduceContext.isFinalReduce() == false) {
                 B removed = ordered.insertWithOverflow(b);
                 if (removed != null) {
                     otherDocCount += removed.getDocCount();
@@ -269,4 +298,16 @@ public InternalAggregation doReduce(List<InternalAggregation> aggregations, Redu
      * Create an array to hold some buckets. Used in collecting the results.
      */
     protected abstract B[] createBucketsArray(int size);
+
+    protected static XContentBuilder doXContentCommon(XContentBuilder builder, Params params,
+                                               long docCountError, long otherDocCount, List<? extends Bucket> buckets) throws IOException {
+        builder.field(DOC_COUNT_ERROR_UPPER_BOUND_FIELD_NAME.getPreferredName(), docCountError);
+        builder.field(SUM_OF_OTHER_DOC_COUNTS.getPreferredName(), otherDocCount);
+        builder.startArray(CommonFields.BUCKETS.getPreferredName());
+        for (Bucket bucket : buckets) {
+            bucket.toXContent(builder, params);
+        }
+        builder.endArray();
+        return builder;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTerms.java
index 41092dba17653..1f950cc4e9824 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTerms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTerms.java
@@ -22,12 +22,15 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 
 /**
  * Result of the {@link TermsAggregator} when the field is some kind of whole number like a integer, long, or a date.
@@ -73,7 +76,7 @@ public Number getKeyAsNumber() {
         }
 
         @Override
-        int compareTerm(Terms.Bucket other) {
+        public int compareTerm(Terms.Bucket other) {
             return Long.compare(term, ((Number) other.getKey()).longValue());
         }
 
@@ -83,20 +86,23 @@ Bucket newBucket(long docCount, InternalAggregations aggs, long docCountError) {
         }
 
         @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startObject();
-            builder.field(CommonFields.KEY, term);
+        protected final XContentBuilder keyToXContent(XContentBuilder builder) throws IOException {
+            builder.field(CommonFields.KEY.getPreferredName(), term);
             if (format != DocValueFormat.RAW) {
-                builder.field(CommonFields.KEY_AS_STRING, format.format(term));
+                builder.field(CommonFields.KEY_AS_STRING.getPreferredName(), format.format(term));
             }
-            builder.field(CommonFields.DOC_COUNT, getDocCount());
-            if (showDocCountError) {
-                builder.field(InternalTerms.DOC_COUNT_ERROR_UPPER_BOUND_FIELD_NAME, getDocCountError());
-            }
-            aggregations.toXContentInternal(builder, params);
-            builder.endObject();
             return builder;
         }
+
+        @Override
+        public boolean equals(Object obj) {
+            return super.equals(obj) && Objects.equals(term, ((Bucket) obj).term);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(super.hashCode(), term);
+        }
     }
 
     public LongTerms(String name, Terms.Order order, int requiredSize, long minDocCount, List<PipelineAggregator> pipelineAggregators,
@@ -136,18 +142,6 @@ protected LongTerms create(String name, List<Bucket> buckets, long docCountError
                 showTermDocCountError, otherDocCount, buckets, docCountError);
     }
 
-    @Override
-    public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
-        builder.field(InternalTerms.DOC_COUNT_ERROR_UPPER_BOUND_FIELD_NAME, docCountError);
-        builder.field(SUM_OF_OTHER_DOC_COUNTS, otherDocCount);
-        builder.startArray(CommonFields.BUCKETS);
-        for (Bucket bucket : buckets) {
-            bucket.toXContent(builder, params);
-        }
-        builder.endArray();
-        return builder;
-    }
-
     @Override
     protected Bucket[] createBucketsArray(int size) {
         return new Bucket[size];
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTermsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTermsAggregator.java
index fd7296d07d9c3..44a7f29aeb6e4 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTermsAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTermsAggregator.java
@@ -32,8 +32,8 @@
 import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude;
 import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude.LongFilter;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -53,7 +53,7 @@ public class LongTermsAggregator extends TermsAggregator {
     private LongFilter longFilter;
 
     public LongTermsAggregator(String name, AggregatorFactories factories, ValuesSource.Numeric valuesSource, DocValueFormat format,
-            Terms.Order order, BucketCountThresholds bucketCountThresholds, AggregationContext aggregationContext, Aggregator parent,
+            Terms.Order order, BucketCountThresholds bucketCountThresholds, SearchContext aggregationContext, Aggregator parent,
             SubAggCollectionMode subAggCollectMode, boolean showTermDocCountError, IncludeExclude.LongFilter longFilter,
             List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException {
         super(name, factories, aggregationContext, parent, bucketCountThresholds, order, format, subAggCollectMode, pipelineAggregators, metaData);
@@ -110,13 +110,16 @@ public InternalAggregation buildAggregation(long owningBucketOrdinal) throws IOE
 
         if (bucketCountThresholds.getMinDocCount() == 0 && (order != InternalOrder.COUNT_DESC || bucketOrds.size() < bucketCountThresholds.getRequiredSize())) {
             // we need to fill-in the blanks
-            for (LeafReaderContext ctx : context.searchContext().searcher().getTopReaderContext().leaves()) {
+            for (LeafReaderContext ctx : context.searcher().getTopReaderContext().leaves()) {
                 final SortedNumericDocValues values = getValues(valuesSource, ctx);
                 for (int docId = 0; docId < ctx.reader().maxDoc(); ++docId) {
                     values.setDocument(docId);
                     final int valueCount = values.count();
                     for (int i = 0; i < valueCount; ++i) {
-                        bucketOrds.add(values.valueAt(i));
+                        long value = values.valueAt(i);
+                        if (longFilter == null || longFilter.accept(value)) {
+                            bucketOrds.add(value);
+                        }
                     }
                 }
             }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/ParsedDoubleTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/ParsedDoubleTerms.java
new file mode 100644
index 0000000000000..d3afe5c17603f
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/ParsedDoubleTerms.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.terms;
+
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+
+public class ParsedDoubleTerms extends ParsedTerms {
+
+    @Override
+    public String getType() {
+        return DoubleTerms.NAME;
+    }
+
+    private static ObjectParser<ParsedDoubleTerms, Void> PARSER =
+            new ObjectParser<>(ParsedDoubleTerms.class.getSimpleName(), true, ParsedDoubleTerms::new);
+    static {
+        declareParsedTermsFields(PARSER, ParsedBucket::fromXContent);
+    }
+
+    public static ParsedDoubleTerms fromXContent(XContentParser parser, String name) throws IOException {
+        ParsedDoubleTerms aggregation = PARSER.parse(parser, null);
+        aggregation.setName(name);
+        return aggregation;
+    }
+
+    public static class ParsedBucket extends ParsedTerms.ParsedBucket {
+
+        private Double key;
+
+        @Override
+        public Object getKey() {
+            return key;
+        }
+
+        @Override
+        public String getKeyAsString() {
+            String keyAsString = super.getKeyAsString();
+            if (keyAsString != null) {
+                return keyAsString;
+            }
+            if (key != null) {
+                return Double.toString(key);
+            }
+            return null;
+        }
+
+        public Number getKeyAsNumber() {
+            return key;
+        }
+
+        @Override
+        protected XContentBuilder keyToXContent(XContentBuilder builder) throws IOException {
+            builder.field(CommonFields.KEY.getPreferredName(), key);
+            if (super.getKeyAsString() != null) {
+                builder.field(CommonFields.KEY_AS_STRING.getPreferredName(), getKeyAsString());
+            }
+            return builder;
+        }
+
+        static ParsedBucket fromXContent(XContentParser parser) throws IOException {
+            return parseTermsBucketXContent(parser, ParsedBucket::new, (p, bucket) -> bucket.key = p.doubleValue());
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/ParsedLongTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/ParsedLongTerms.java
new file mode 100644
index 0000000000000..b5869fc6ee2d8
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/ParsedLongTerms.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.terms;
+
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+
+public class ParsedLongTerms extends ParsedTerms {
+
+    @Override
+    public String getType() {
+        return LongTerms.NAME;
+    }
+
+    private static ObjectParser<ParsedLongTerms, Void> PARSER =
+            new ObjectParser<>(ParsedLongTerms.class.getSimpleName(), true, ParsedLongTerms::new);
+    static {
+        declareParsedTermsFields(PARSER, ParsedBucket::fromXContent);
+    }
+
+    public static ParsedLongTerms fromXContent(XContentParser parser, String name) throws IOException {
+        ParsedLongTerms aggregation = PARSER.parse(parser, null);
+        aggregation.setName(name);
+        return aggregation;
+    }
+
+    public static class ParsedBucket extends ParsedTerms.ParsedBucket {
+
+        private Long key;
+
+        @Override
+        public Object getKey() {
+            return key;
+        }
+
+        @Override
+        public String getKeyAsString() {
+            String keyAsString = super.getKeyAsString();
+            if (keyAsString != null) {
+                return keyAsString;
+            }
+            if (key != null) {
+                return Long.toString(key);
+            }
+            return null;
+        }
+
+        public Number getKeyAsNumber() {
+            return key;
+        }
+
+        @Override
+        protected XContentBuilder keyToXContent(XContentBuilder builder) throws IOException {
+            builder.field(CommonFields.KEY.getPreferredName(), key);
+            if (super.getKeyAsString() != null) {
+                builder.field(CommonFields.KEY_AS_STRING.getPreferredName(), getKeyAsString());
+            }
+            return builder;
+        }
+
+        static ParsedBucket fromXContent(XContentParser parser) throws IOException {
+            return parseTermsBucketXContent(parser, ParsedBucket::new, (p, bucket) -> bucket.key = p.longValue());
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/ParsedStringTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/ParsedStringTerms.java
new file mode 100644
index 0000000000000..792365d6b1aa2
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/ParsedStringTerms.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.terms;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+
+public class ParsedStringTerms extends ParsedTerms {
+
+    @Override
+    public String getType() {
+        return StringTerms.NAME;
+    }
+
+    private static ObjectParser<ParsedStringTerms, Void> PARSER =
+            new ObjectParser<>(ParsedStringTerms.class.getSimpleName(), true, ParsedStringTerms::new);
+    static {
+        declareParsedTermsFields(PARSER, ParsedBucket::fromXContent);
+    }
+
+    public static ParsedStringTerms fromXContent(XContentParser parser, String name) throws IOException {
+        ParsedStringTerms aggregation = PARSER.parse(parser, null);
+        aggregation.setName(name);
+        return aggregation;
+    }
+
+    public static class ParsedBucket extends ParsedTerms.ParsedBucket {
+
+        private BytesRef key;
+
+        @Override
+        public Object getKey() {
+            return getKeyAsString();
+        }
+
+        @Override
+        public String getKeyAsString() {
+            String keyAsString = super.getKeyAsString();
+            if (keyAsString != null) {
+                return keyAsString;
+            }
+            if (key != null) {
+                return key.utf8ToString();
+            }
+            return null;
+        }
+
+        public Number getKeyAsNumber() {
+            if (key != null) {
+                return Double.parseDouble(key.utf8ToString());
+            }
+            return null;
+        }
+
+        @Override
+        protected XContentBuilder keyToXContent(XContentBuilder builder) throws IOException {
+            return builder.field(CommonFields.KEY.getPreferredName(), getKey());
+        }
+
+        static ParsedBucket fromXContent(XContentParser parser) throws IOException {
+            return parseTermsBucketXContent(parser, ParsedBucket::new, (p, bucket) -> bucket.key = p.utf8BytesOrNull());
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/ParsedTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/ParsedTerms.java
new file mode 100644
index 0000000000000..1ff4598295a77
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/ParsedTerms.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.terms;
+
+import org.elasticsearch.common.CheckedBiConsumer;
+import org.elasticsearch.common.CheckedFunction;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParserUtils;
+import org.elasticsearch.search.aggregations.Aggregation;
+import org.elasticsearch.search.aggregations.Aggregations;
+import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Supplier;
+
+import static org.elasticsearch.search.aggregations.bucket.terms.InternalTerms.DOC_COUNT_ERROR_UPPER_BOUND_FIELD_NAME;
+import static org.elasticsearch.search.aggregations.bucket.terms.InternalTerms.SUM_OF_OTHER_DOC_COUNTS;
+
+public abstract class ParsedTerms extends ParsedMultiBucketAggregation<ParsedTerms.ParsedBucket> implements Terms {
+
+    protected long docCountErrorUpperBound;
+    protected long sumOtherDocCount;
+
+    @Override
+    public long getDocCountError() {
+        return docCountErrorUpperBound;
+    }
+
+    @Override
+    public long getSumOfOtherDocCounts() {
+        return sumOtherDocCount;
+    }
+
+    @Override
+    public List<? extends Terms.Bucket> getBuckets() {
+        return buckets;
+    }
+
+    @Override
+    public Terms.Bucket getBucketByKey(String term) {
+        for (Terms.Bucket bucket : getBuckets()) {
+            if (bucket.getKeyAsString().equals(term)) {
+                return bucket;
+            }
+        }
+        return null;
+    }
+
+    @Override
+    protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+        builder.field(DOC_COUNT_ERROR_UPPER_BOUND_FIELD_NAME.getPreferredName(), getDocCountError());
+        builder.field(SUM_OF_OTHER_DOC_COUNTS.getPreferredName(), getSumOfOtherDocCounts());
+        builder.startArray(CommonFields.BUCKETS.getPreferredName());
+        for (Terms.Bucket bucket : getBuckets()) {
+            bucket.toXContent(builder, params);
+        }
+        builder.endArray();
+        return builder;
+    }
+
+    static void declareParsedTermsFields(final ObjectParser<? extends ParsedTerms, Void> objectParser,
+                                         final CheckedFunction<XContentParser, ParsedBucket, IOException> bucketParser) {
+        declareMultiBucketAggregationFields(objectParser, bucketParser::apply, bucketParser::apply);
+        objectParser.declareLong((parsedTerms, value) -> parsedTerms.docCountErrorUpperBound = value ,
+                DOC_COUNT_ERROR_UPPER_BOUND_FIELD_NAME);
+        objectParser.declareLong((parsedTerms, value) -> parsedTerms.sumOtherDocCount = value,
+                SUM_OF_OTHER_DOC_COUNTS);
+    }
+
+    public abstract static class ParsedBucket extends ParsedMultiBucketAggregation.ParsedBucket implements Terms.Bucket {
+
+        boolean showDocCountError = false;
+        protected long docCountError;
+
+        @Override
+        public int compareTerm(Terms.Bucket other) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public long getDocCountError() {
+            return docCountError;
+        }
+
+        @Override
+        public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            keyToXContent(builder);
+            builder.field(CommonFields.DOC_COUNT.getPreferredName(), getDocCount());
+            if (showDocCountError) {
+                builder.field(DOC_COUNT_ERROR_UPPER_BOUND_FIELD_NAME.getPreferredName(), getDocCountError());
+            }
+            getAggregations().toXContentInternal(builder, params);
+            builder.endObject();
+            return builder;
+        }
+
+
+        static <B extends ParsedBucket> B parseTermsBucketXContent(final XContentParser parser, final Supplier<B> bucketSupplier,
+                                                                   final CheckedBiConsumer<XContentParser, B, IOException> keyConsumer)
+                throws IOException {
+
+            final B bucket = bucketSupplier.get();
+            final List<Aggregation> aggregations = new ArrayList<>();
+
+            XContentParser.Token token;
+            String currentFieldName = parser.currentName();
+            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                if (token == XContentParser.Token.FIELD_NAME) {
+                    currentFieldName = parser.currentName();
+                } else if (token.isValue()) {
+                    if (CommonFields.KEY_AS_STRING.getPreferredName().equals(currentFieldName)) {
+                        bucket.setKeyAsString(parser.text());
+                    } else if (CommonFields.KEY.getPreferredName().equals(currentFieldName)) {
+                        keyConsumer.accept(parser, bucket);
+                    } else if (CommonFields.DOC_COUNT.getPreferredName().equals(currentFieldName)) {
+                        bucket.setDocCount(parser.longValue());
+                    } else if (DOC_COUNT_ERROR_UPPER_BOUND_FIELD_NAME.getPreferredName().equals(currentFieldName)) {
+                        bucket.docCountError = parser.longValue();
+                        bucket.showDocCountError = true;
+                    }
+                } else if (token == XContentParser.Token.START_OBJECT) {
+                    XContentParserUtils.parseTypedKeysObject(parser, Aggregation.TYPED_KEYS_DELIMITER, Aggregation.class,
+                            aggregations::add);
+                }
+            }
+            bucket.setAggregations(new Aggregations(aggregations));
+            return bucket;
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTerms.java
index 4a40f77b2b24b..62ad5afd4b0e8 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTerms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTerms.java
@@ -74,7 +74,7 @@ public String getKeyAsString() {
         }
 
         @Override
-        int compareTerm(Terms.Bucket other) {
+        public int compareTerm(Terms.Bucket other) {
             return termBytes.compareTo(((Bucket) other).termBytes);
         }
 
@@ -84,16 +84,8 @@ Bucket newBucket(long docCount, InternalAggregations aggs, long docCountError) {
         }
 
         @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startObject();
-            builder.field(CommonFields.KEY, getKeyAsString());
-            builder.field(CommonFields.DOC_COUNT, getDocCount());
-            if (showDocCountError) {
-                builder.field(InternalTerms.DOC_COUNT_ERROR_UPPER_BOUND_FIELD_NAME, getDocCountError());
-            }
-            aggregations.toXContentInternal(builder, params);
-            builder.endObject();
-            return builder;
+        protected final XContentBuilder keyToXContent(XContentBuilder builder) throws IOException {
+            return builder.field(CommonFields.KEY.getPreferredName(), getKeyAsString());
         }
     }
 
@@ -134,18 +126,6 @@ protected StringTerms create(String name, List<Bucket> buckets, long docCountErr
                 showTermDocCountError, otherDocCount, buckets, docCountError);
     }
 
-    @Override
-    public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
-        builder.field(InternalTerms.DOC_COUNT_ERROR_UPPER_BOUND_FIELD_NAME, docCountError);
-        builder.field(SUM_OF_OTHER_DOC_COUNTS, otherDocCount);
-        builder.startArray(CommonFields.BUCKETS);
-        for (Bucket bucket : buckets) {
-            bucket.toXContent(builder, params);
-        }
-        builder.endArray();
-        return builder;
-    }
-
     @Override
     protected Bucket[] createBucketsArray(int size) {
         return new Bucket[size];
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsAggregator.java
index 7b2b76a086154..c93fc94ff6187 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsAggregator.java
@@ -33,8 +33,8 @@
 import org.elasticsearch.search.aggregations.bucket.terms.support.BucketPriorityQueue;
 import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -52,15 +52,15 @@ public class StringTermsAggregator extends AbstractStringTermsAggregator {
 
     public StringTermsAggregator(String name, AggregatorFactories factories, ValuesSource valuesSource,
             Terms.Order order, DocValueFormat format, BucketCountThresholds bucketCountThresholds,
-            IncludeExclude.StringFilter includeExclude, AggregationContext aggregationContext,
+            IncludeExclude.StringFilter includeExclude, SearchContext context,
             Aggregator parent, SubAggCollectionMode collectionMode, boolean showTermDocCountError,
             List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException {
 
-        super(name, factories, aggregationContext, parent, order, format, bucketCountThresholds, collectionMode, showTermDocCountError,
+        super(name, factories, context, parent, order, format, bucketCountThresholds, collectionMode, showTermDocCountError,
                 pipelineAggregators, metaData);
         this.valuesSource = valuesSource;
         this.includeExclude = includeExclude;
-        bucketOrds = new BytesRefHash(1, aggregationContext.bigArrays());
+        bucketOrds = new BytesRefHash(1, context.bigArrays());
     }
 
     @Override
@@ -110,7 +110,7 @@ public InternalAggregation buildAggregation(long owningBucketOrdinal) throws IOE
 
         if (bucketCountThresholds.getMinDocCount() == 0 && (order != InternalOrder.COUNT_DESC || bucketOrds.size() < bucketCountThresholds.getRequiredSize())) {
             // we need to fill-in the blanks
-            for (LeafReaderContext ctx : context.searchContext().searcher().getTopReaderContext().leaves()) {
+            for (LeafReaderContext ctx : context.searcher().getTopReaderContext().leaves()) {
                 final SortedBinaryDocValues values = valuesSource.bytesValues(ctx);
                 // brute force
                 for (int docId = 0; docId < ctx.reader().maxDoc(); ++docId) {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/Terms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/Terms.java
index aa70309da95c0..166ece4e1122d 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/Terms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/Terms.java
@@ -20,7 +20,6 @@
 
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.search.aggregations.Aggregator;
-import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation;
 import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
 
 import java.util.Arrays;
@@ -33,50 +32,23 @@
  */
 public interface Terms extends MultiBucketsAggregation {
 
-    static enum ValueType {
-
-        STRING(org.elasticsearch.search.aggregations.support.ValueType.STRING),
-        LONG(org.elasticsearch.search.aggregations.support.ValueType.LONG),
-        DOUBLE(org.elasticsearch.search.aggregations.support.ValueType.DOUBLE);
-
-        final org.elasticsearch.search.aggregations.support.ValueType scriptValueType;
-
-        private ValueType(org.elasticsearch.search.aggregations.support.ValueType scriptValueType) {
-            this.scriptValueType = scriptValueType;
-        }
-
-        static ValueType resolveType(String type) {
-            if ("string".equals(type)) {
-                return STRING;
-            }
-            if ("double".equals(type) || "float".equals(type)) {
-                return DOUBLE;
-            }
-            if ("long".equals(type) || "integer".equals(type) || "short".equals(type) || "byte".equals(type)) {
-                return LONG;
-            }
-            return null;
-        }
-    }
-
     /**
      * A bucket that is associated with a single term
      */
-    abstract static class Bucket extends InternalMultiBucketAggregation.InternalBucket {
-
-        public abstract Number getKeyAsNumber();
+    interface Bucket extends MultiBucketsAggregation.Bucket {
 
-        abstract int compareTerm(Terms.Bucket other);
+        Number getKeyAsNumber();
 
-        public abstract long getDocCountError();
+        int compareTerm(Terms.Bucket other);
 
+        long getDocCountError();
     }
 
     /**
      * Return the sorted list of the buckets in this terms aggregation.
      */
     @Override
-    List<Bucket> getBuckets();
+    List<? extends Bucket> getBuckets();
 
     /**
      * Get the bucket for the given term, or null if there is no such bucket.
@@ -97,7 +69,7 @@ abstract static class Bucket extends InternalMultiBucketAggregation.InternalBuck
     /**
      * Determines the order by which the term buckets will be sorted
      */
-    abstract static class Order implements ToXContent {
+    abstract class Order implements ToXContent {
 
         /**
          * @return a bucket ordering strategy that sorts buckets by their document counts (ascending or descending)
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java
index e34c3bcbed4c5..210f45be3179c 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java
@@ -19,23 +19,29 @@
 package org.elasticsearch.search.aggregations.bucket.terms;
 
 import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
+import org.elasticsearch.search.aggregations.bucket.terms.InternalOrder.CompoundOrder;
+import org.elasticsearch.search.aggregations.bucket.terms.Terms.Order;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator.BucketCountThresholds;
 import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -43,7 +49,6 @@
 
 public class TermsAggregationBuilder extends ValuesSourceAggregationBuilder<ValuesSource, TermsAggregationBuilder> {
     public static final String NAME = "terms";
-    private static final InternalAggregation.Type TYPE = new Type("terms");
 
     public static final ParseField EXECUTION_HINT_FIELD_NAME = new ParseField("execution_hint");
     public static final ParseField SHARD_SIZE_FIELD_NAME = new ParseField("shard_size");
@@ -56,6 +61,42 @@ public class TermsAggregationBuilder extends ValuesSourceAggregationBuilder<Valu
     public static final ParseField SHOW_TERM_DOC_COUNT_ERROR = new ParseField("show_term_doc_count_error");
     public static final ParseField ORDER_FIELD = new ParseField("order");
 
+    private static final ObjectParser<TermsAggregationBuilder, QueryParseContext> PARSER;
+    static {
+        PARSER = new ObjectParser<>(TermsAggregationBuilder.NAME);
+        ValuesSourceParserHelper.declareAnyFields(PARSER, true, true);
+
+        PARSER.declareBoolean(TermsAggregationBuilder::showTermDocCountError,
+                TermsAggregationBuilder.SHOW_TERM_DOC_COUNT_ERROR);
+
+        PARSER.declareInt(TermsAggregationBuilder::shardSize, SHARD_SIZE_FIELD_NAME);
+
+        PARSER.declareLong(TermsAggregationBuilder::minDocCount, MIN_DOC_COUNT_FIELD_NAME);
+
+        PARSER.declareLong(TermsAggregationBuilder::shardMinDocCount, SHARD_MIN_DOC_COUNT_FIELD_NAME);
+
+        PARSER.declareInt(TermsAggregationBuilder::size, REQUIRED_SIZE_FIELD_NAME);
+
+        PARSER.declareString(TermsAggregationBuilder::executionHint, EXECUTION_HINT_FIELD_NAME);
+
+        PARSER.declareField(TermsAggregationBuilder::collectMode,
+                (p, c) -> SubAggCollectionMode.parse(p.text()),
+                SubAggCollectionMode.KEY, ObjectParser.ValueType.STRING);
+
+        PARSER.declareObjectArray(TermsAggregationBuilder::order, TermsAggregationBuilder::parseOrderParam,
+                TermsAggregationBuilder.ORDER_FIELD);
+
+        PARSER.declareField((b, v) -> b.includeExclude(IncludeExclude.merge(v, b.includeExclude())),
+                IncludeExclude::parseInclude, IncludeExclude.INCLUDE_FIELD, ObjectParser.ValueType.OBJECT_ARRAY_OR_STRING);
+
+        PARSER.declareField((b, v) -> b.includeExclude(IncludeExclude.merge(b.includeExclude(), v)),
+                IncludeExclude::parseExclude, IncludeExclude.EXCLUDE_FIELD, ObjectParser.ValueType.OBJECT_ARRAY_OR_STRING);
+    }
+
+    public static AggregationBuilder parse(String aggregationName, QueryParseContext context) throws IOException {
+        return PARSER.parse(context.parser(), new TermsAggregationBuilder(aggregationName, null), context);
+    }
+
     private Terms.Order order = Terms.Order.compound(Terms.Order.count(false), Terms.Order.term(true));
     private IncludeExclude includeExclude = null;
     private String executionHint = null;
@@ -65,14 +106,14 @@ public class TermsAggregationBuilder extends ValuesSourceAggregationBuilder<Valu
     private boolean showTermDocCountError = false;
 
     public TermsAggregationBuilder(String name, ValueType valueType) {
-        super(name, TYPE, ValuesSourceType.ANY, valueType);
+        super(name, ValuesSourceType.ANY, valueType);
     }
 
     /**
      * Read from a stream.
      */
     public TermsAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE, ValuesSourceType.ANY);
+        super(in, ValuesSourceType.ANY);
         bucketCountThresholds = new BucketCountThresholds(in);
         collectMode = in.readOptionalWriteable(SubAggCollectionMode::readFromStream);
         executionHint = in.readOptionalString();
@@ -96,18 +137,6 @@ protected void innerWriteTo(StreamOutput out) throws IOException {
         out.writeBoolean(showTermDocCountError);
     }
 
-    public TermsAggregator.BucketCountThresholds bucketCountThresholds() {
-        return bucketCountThresholds;
-    }
-
-    public TermsAggregationBuilder bucketCountThresholds(TermsAggregator.BucketCountThresholds bucketCountThresholds) {
-        if (bucketCountThresholds == null) {
-            throw new IllegalArgumentException("[bucketCountThresholds] must not be null: [" + name + "]");
-        }
-        this.bucketCountThresholds = bucketCountThresholds;
-        return this;
-    }
-
     /**
      * Sets the size - indicating how many term buckets should be returned
      * (defaults to 10)
@@ -148,6 +177,13 @@ public TermsAggregationBuilder minDocCount(long minDocCount) {
         return this;
     }
 
+    /**
+     * Returns the minimum document count required per term
+     */
+    public long minDocCount() {
+        return bucketCountThresholds.getMinDocCount();
+    }
+
     /**
      * Set the minimum document count terms should have on the shard in order to
      * appear in the response.
@@ -168,7 +204,11 @@ public TermsAggregationBuilder order(Terms.Order order) {
         if (order == null) {
             throw new IllegalArgumentException("[order] must not be null: [" + name + "]");
         }
-        this.order = order;
+        if(order instanceof CompoundOrder || InternalOrder.isTermOrder(order)) {
+            this.order = order; // if order already contains a tie-breaker we are good to go
+        } else { // otherwise add a tie-breaker by using a compound order
+            this.order = Terms.Order.compound(order);
+        }
         return this;
     }
 
@@ -254,9 +294,9 @@ public TermsAggregationBuilder showTermDocCountError(boolean showTermDocCountErr
     }
 
     @Override
-    protected ValuesSourceAggregatorFactory<ValuesSource, ?> innerBuild(AggregationContext context, ValuesSourceConfig<ValuesSource> config,
+    protected ValuesSourceAggregatorFactory<ValuesSource, ?> innerBuild(SearchContext context, ValuesSourceConfig<ValuesSource> config,
             AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
-        return new TermsAggregatorFactory(name, type, config, order, includeExclude, executionHint, collectMode,
+        return new TermsAggregatorFactory(name, config, order, includeExclude, executionHint, collectMode,
                 bucketCountThresholds, showTermDocCountError, context, parent, subFactoriesBuilder, metaData);
     }
 
@@ -295,7 +335,49 @@ protected boolean innerEquals(Object obj) {
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
+
+    private static Terms.Order parseOrderParam(XContentParser parser, QueryParseContext context) throws IOException {
+        XContentParser.Token token;
+        Terms.Order orderParam = null;
+        String orderKey = null;
+        boolean orderAsc = false;
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                orderKey = parser.currentName();
+            } else if (token == XContentParser.Token.VALUE_STRING) {
+                String dir = parser.text();
+                if ("asc".equalsIgnoreCase(dir)) {
+                    orderAsc = true;
+                } else if ("desc".equalsIgnoreCase(dir)) {
+                    orderAsc = false;
+                } else {
+                    throw new ParsingException(parser.getTokenLocation(),
+                            "Unknown terms order direction [" + dir + "]");
+                }
+            } else {
+                throw new ParsingException(parser.getTokenLocation(),
+                        "Unexpected token " + token + " for [order]");
+            }
+        }
+        if (orderKey == null) {
+            throw new ParsingException(parser.getTokenLocation(),
+                    "Must specify at least one field for [order]");
+        } else {
+            orderParam = resolveOrder(orderKey, orderAsc);
+        }
+        return orderParam;
+    }
+
+    static Terms.Order resolveOrder(String key, boolean asc) {
+        if ("_term".equals(key)) {
+            return Order.term(asc);
+        }
+        if ("_count".equals(key)) {
+            return Order.count(asc);
+        }
+        return Order.aggregation(key, asc);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregator.java
index 4b8ec77c46761..78d6cde211cde 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregator.java
@@ -33,8 +33,8 @@
 import org.elasticsearch.search.aggregations.bucket.terms.InternalOrder.Aggregation;
 import org.elasticsearch.search.aggregations.bucket.terms.InternalOrder.CompoundOrder;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.AggregationPath;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.HashSet;
@@ -137,7 +137,9 @@ public void setShardSize(int shardSize) {
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.field(TermsAggregationBuilder.REQUIRED_SIZE_FIELD_NAME.getPreferredName(), requiredSize);
-            builder.field(TermsAggregationBuilder.SHARD_SIZE_FIELD_NAME.getPreferredName(), shardSize);
+            if (shardSize != -1) {
+                builder.field(TermsAggregationBuilder.SHARD_SIZE_FIELD_NAME.getPreferredName(), shardSize);
+            }
             builder.field(TermsAggregationBuilder.MIN_DOC_COUNT_FIELD_NAME.getPreferredName(), minDocCount);
             builder.field(TermsAggregationBuilder.SHARD_MIN_DOC_COUNT_FIELD_NAME.getPreferredName(), shardMinDocCount);
             return builder;
@@ -170,7 +172,7 @@ public boolean equals(Object obj) {
     protected final Set<Aggregator> aggsUsedForSorting = new HashSet<>();
     protected final SubAggCollectionMode collectMode;
 
-    public TermsAggregator(String name, AggregatorFactories factories, AggregationContext context, Aggregator parent,
+    public TermsAggregator(String name, AggregatorFactories factories, SearchContext context, Aggregator parent,
             BucketCountThresholds bucketCountThresholds, Terms.Order order, DocValueFormat format, SubAggCollectionMode collectMode,
             List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException {
         super(name, factories, context, parent, pipelineAggregators, metaData);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java
index c01377d976189..0ef6d0b9bf35d 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java
@@ -21,26 +21,25 @@
 
 import org.apache.lucene.search.IndexSearcher;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.AggregationExecutionException;
 import org.elasticsearch.search.aggregations.Aggregator;
+import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.NonCollectingAggregator;
-import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.bucket.BucketUtils;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator.BucketCountThresholds;
 import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 
@@ -53,11 +52,19 @@ public class TermsAggregatorFactory extends ValuesSourceAggregatorFactory<Values
     private final TermsAggregator.BucketCountThresholds bucketCountThresholds;
     private boolean showTermDocCountError;
 
-    public TermsAggregatorFactory(String name, Type type, ValuesSourceConfig<ValuesSource> config, Terms.Order order,
-            IncludeExclude includeExclude, String executionHint, SubAggCollectionMode collectMode,
-            TermsAggregator.BucketCountThresholds bucketCountThresholds, boolean showTermDocCountError, AggregationContext context,
-            AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+    public TermsAggregatorFactory(String name,
+                                  ValuesSourceConfig<ValuesSource> config,
+                                  Terms.Order order,
+                                  IncludeExclude includeExclude,
+                                  String executionHint,
+                                  SubAggCollectionMode collectMode,
+                                  TermsAggregator.BucketCountThresholds bucketCountThresholds,
+                                  boolean showTermDocCountError,
+                                  SearchContext context,
+                                  AggregatorFactory<?> parent,
+                                  AggregatorFactories.Builder subFactoriesBuilder,
+                                  Map<String, Object> metaData) throws IOException {
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.order = order;
         this.includeExclude = includeExclude;
         this.executionHint = executionHint;
@@ -98,13 +105,13 @@ protected Aggregator doCreateInternal(ValuesSource valuesSource, Aggregator pare
             // heuristic to avoid any wrong-ranking caused by distributed
             // counting
             bucketCountThresholds.setShardSize(BucketUtils.suggestShardSideQueueSize(bucketCountThresholds.getRequiredSize(),
-                    context.searchContext().numberOfShards()));
+                    context.numberOfShards()));
         }
         bucketCountThresholds.ensureValidity();
         if (valuesSource instanceof ValuesSource.Bytes) {
             ExecutionMode execution = null;
             if (executionHint != null) {
-                execution = ExecutionMode.fromString(executionHint, context.searchContext().parseFieldMatcher());
+                execution = ExecutionMode.fromString(executionHint);
             }
 
             // In some cases, using ordinals is just not supported: override it
@@ -116,7 +123,7 @@ protected Aggregator doCreateInternal(ValuesSource valuesSource, Aggregator pare
             final double ratio;
             if (execution == null || execution.needsGlobalOrdinals()) {
                 ValuesSource.Bytes.WithOrdinals valueSourceWithOrdinals = (ValuesSource.Bytes.WithOrdinals) valuesSource;
-                IndexSearcher indexSearcher = context.searchContext().searcher();
+                IndexSearcher indexSearcher = context.searcher();
                 maxOrd = valueSourceWithOrdinals.globalMaxOrd(indexSearcher);
                 ratio = maxOrd / ((double) indexSearcher.getIndexReader().numDocs());
             } else {
@@ -131,7 +138,10 @@ protected Aggregator doCreateInternal(ValuesSource valuesSource, Aggregator pare
                 // to be unbounded and most instances may only aggregate few
                 // documents, so use hashed based
                 // global ordinals to keep the bucket ords dense.
-                if (Aggregator.descendsFromBucketAggregator(parent)) {
+                // Additionally, if using partitioned terms the regular global
+                // ordinals would be sparse so we opt for hash
+                if (Aggregator.descendsFromBucketAggregator(parent) ||
+                        (includeExclude != null && includeExclude.isPartitionBased())) {
                     execution = ExecutionMode.GLOBAL_ORDINALS_HASH;
                 } else {
                     if (factories == AggregatorFactories.EMPTY) {
@@ -222,14 +232,23 @@ public enum ExecutionMode {
         MAP(new ParseField("map")) {
 
             @Override
-            Aggregator create(String name, AggregatorFactories factories, ValuesSource valuesSource, Terms.Order order,
-                    DocValueFormat format, TermsAggregator.BucketCountThresholds bucketCountThresholds, IncludeExclude includeExclude,
-                    AggregationContext aggregationContext, Aggregator parent, SubAggCollectionMode subAggCollectMode,
-                    boolean showTermDocCountError, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData)
-                            throws IOException {
+            Aggregator create(String name,
+                              AggregatorFactories factories,
+                              ValuesSource valuesSource,
+                              Terms.Order order,
+                              DocValueFormat format,
+                              TermsAggregator.BucketCountThresholds bucketCountThresholds,
+                              IncludeExclude includeExclude,
+                              SearchContext context, Aggregator parent,
+                              SubAggCollectionMode subAggCollectMode,
+                              boolean showTermDocCountError,
+                              List<PipelineAggregator> pipelineAggregators,
+                              Map<String, Object> metaData) throws IOException {
+
                 final IncludeExclude.StringFilter filter = includeExclude == null ? null : includeExclude.convertToStringFilter(format);
                 return new StringTermsAggregator(name, factories, valuesSource, order, format, bucketCountThresholds, filter,
-                        aggregationContext, parent, subAggCollectMode, showTermDocCountError, pipelineAggregators, metaData);
+                        context, parent, subAggCollectMode, showTermDocCountError, pipelineAggregators, metaData);
+
             }
 
             @Override
@@ -241,15 +260,25 @@ boolean needsGlobalOrdinals() {
         GLOBAL_ORDINALS(new ParseField("global_ordinals")) {
 
             @Override
-            Aggregator create(String name, AggregatorFactories factories, ValuesSource valuesSource, Terms.Order order,
-                    DocValueFormat format, TermsAggregator.BucketCountThresholds bucketCountThresholds, IncludeExclude includeExclude,
-                    AggregationContext aggregationContext, Aggregator parent, SubAggCollectionMode subAggCollectMode,
-                    boolean showTermDocCountError, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData)
-                            throws IOException {
+            Aggregator create(String name,
+                              AggregatorFactories factories,
+                              ValuesSource valuesSource,
+                              Terms.Order order,
+                              DocValueFormat format,
+                              TermsAggregator.BucketCountThresholds bucketCountThresholds,
+                              IncludeExclude includeExclude,
+                              SearchContext context,
+                              Aggregator parent,
+                              SubAggCollectionMode subAggCollectMode,
+                              boolean showTermDocCountError,
+                              List<PipelineAggregator> pipelineAggregators,
+                              Map<String, Object> metaData) throws IOException {
+
                 final IncludeExclude.OrdinalsFilter filter = includeExclude == null ? null : includeExclude.convertToOrdinalsFilter(format);
                 return new GlobalOrdinalsStringTermsAggregator(name, factories, (ValuesSource.Bytes.WithOrdinals) valuesSource, order,
-                        format, bucketCountThresholds, filter, aggregationContext, parent, subAggCollectMode, showTermDocCountError,
+                        format, bucketCountThresholds, filter, context, parent, false, subAggCollectMode, showTermDocCountError,
                         pipelineAggregators, metaData);
+
             }
 
             @Override
@@ -261,15 +290,25 @@ boolean needsGlobalOrdinals() {
         GLOBAL_ORDINALS_HASH(new ParseField("global_ordinals_hash")) {
 
             @Override
-            Aggregator create(String name, AggregatorFactories factories, ValuesSource valuesSource, Terms.Order order,
-                    DocValueFormat format, TermsAggregator.BucketCountThresholds bucketCountThresholds, IncludeExclude includeExclude,
-                    AggregationContext aggregationContext, Aggregator parent, SubAggCollectionMode subAggCollectMode,
-                    boolean showTermDocCountError, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData)
-                            throws IOException {
+            Aggregator create(String name,
+                              AggregatorFactories factories,
+                              ValuesSource valuesSource,
+                              Terms.Order order,
+                              DocValueFormat format,
+                              TermsAggregator.BucketCountThresholds bucketCountThresholds,
+                              IncludeExclude includeExclude,
+                              SearchContext context,
+                              Aggregator parent,
+                              SubAggCollectionMode subAggCollectMode,
+                              boolean showTermDocCountError,
+                              List<PipelineAggregator> pipelineAggregators,
+                              Map<String, Object> metaData) throws IOException {
+
                 final IncludeExclude.OrdinalsFilter filter = includeExclude == null ? null : includeExclude.convertToOrdinalsFilter(format);
-                return new GlobalOrdinalsStringTermsAggregator.WithHash(name, factories, (ValuesSource.Bytes.WithOrdinals) valuesSource,
-                        order, format, bucketCountThresholds, filter, aggregationContext, parent, subAggCollectMode, showTermDocCountError,
-                        pipelineAggregators, metaData);
+                return new GlobalOrdinalsStringTermsAggregator(name, factories, (ValuesSource.Bytes.WithOrdinals) valuesSource,
+                    order, format, bucketCountThresholds, filter, context, parent, true, subAggCollectMode,
+                    showTermDocCountError, pipelineAggregators, metaData);
+
             }
 
             @Override
@@ -280,21 +319,31 @@ boolean needsGlobalOrdinals() {
         GLOBAL_ORDINALS_LOW_CARDINALITY(new ParseField("global_ordinals_low_cardinality")) {
 
             @Override
-            Aggregator create(String name, AggregatorFactories factories, ValuesSource valuesSource, Terms.Order order,
-                    DocValueFormat format, TermsAggregator.BucketCountThresholds bucketCountThresholds, IncludeExclude includeExclude,
-                    AggregationContext aggregationContext, Aggregator parent, SubAggCollectionMode subAggCollectMode,
-                    boolean showTermDocCountError, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData)
-                            throws IOException {
+            Aggregator create(String name,
+                              AggregatorFactories factories,
+                              ValuesSource valuesSource,
+                              Terms.Order order,
+                              DocValueFormat format,
+                              TermsAggregator.BucketCountThresholds bucketCountThresholds,
+                              IncludeExclude includeExclude,
+                              SearchContext context,
+                              Aggregator parent,
+                              SubAggCollectionMode subAggCollectMode,
+                              boolean showTermDocCountError,
+                              List<PipelineAggregator> pipelineAggregators,
+                              Map<String, Object> metaData) throws IOException {
+
                 if (includeExclude != null || factories.countAggregators() > 0
-                // we need the FieldData impl to be able to extract the
-                // segment to global ord mapping
+                        // we need the FieldData impl to be able to extract the
+                        // segment to global ord mapping
                         || valuesSource.getClass() != ValuesSource.Bytes.FieldData.class) {
                     return GLOBAL_ORDINALS.create(name, factories, valuesSource, order, format, bucketCountThresholds, includeExclude,
-                            aggregationContext, parent, subAggCollectMode, showTermDocCountError, pipelineAggregators, metaData);
+                            context, parent, subAggCollectMode, showTermDocCountError, pipelineAggregators, metaData);
                 }
                 return new GlobalOrdinalsStringTermsAggregator.LowCardinality(name, factories,
-                        (ValuesSource.Bytes.WithOrdinals) valuesSource, order, format, bucketCountThresholds, aggregationContext, parent,
-                        subAggCollectMode, showTermDocCountError, pipelineAggregators, metaData);
+                        (ValuesSource.Bytes.WithOrdinals) valuesSource, order, format, bucketCountThresholds, context, parent,
+                        false, subAggCollectMode, showTermDocCountError, pipelineAggregators, metaData);
+
             }
 
             @Override
@@ -303,13 +352,13 @@ boolean needsGlobalOrdinals() {
             }
         };
 
-        public static ExecutionMode fromString(String value, ParseFieldMatcher parseFieldMatcher) {
+        public static ExecutionMode fromString(String value) {
             for (ExecutionMode mode : values()) {
-                if (parseFieldMatcher.match(value, mode.parseField)) {
+                if (mode.parseField.match(value)) {
                     return mode;
                 }
             }
-            throw new IllegalArgumentException("Unknown `execution_hint`: [" + value + "], expected any of " + values());
+            throw new IllegalArgumentException("Unknown `execution_hint`: [" + value + "], expected any of " + Arrays.toString(values()));
         }
 
         private final ParseField parseField;
@@ -318,11 +367,19 @@ public static ExecutionMode fromString(String value, ParseFieldMatcher parseFiel
             this.parseField = parseField;
         }
 
-        abstract Aggregator create(String name, AggregatorFactories factories, ValuesSource valuesSource, Terms.Order order,
-                DocValueFormat format, TermsAggregator.BucketCountThresholds bucketCountThresholds, IncludeExclude includeExclude,
-                AggregationContext aggregationContext, Aggregator parent, SubAggCollectionMode subAggCollectMode,
-                boolean showTermDocCountError, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData)
-                        throws IOException;
+        abstract Aggregator create(String name,
+                                   AggregatorFactories factories,
+                                   ValuesSource valuesSource,
+                                   Terms.Order order,
+                                   DocValueFormat format,
+                                   TermsAggregator.BucketCountThresholds bucketCountThresholds,
+                                   IncludeExclude includeExclude,
+                                   SearchContext context,
+                                   Aggregator parent,
+                                   SubAggCollectionMode subAggCollectMode,
+                                   boolean showTermDocCountError,
+                                   List<PipelineAggregator> pipelineAggregators,
+                                   Map<String, Object> metaData) throws IOException;
 
         abstract boolean needsGlobalOrdinals();
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsParser.java
deleted file mode 100644
index bf8b06ab65aaa..0000000000000
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsParser.java
+++ /dev/null
@@ -1,175 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.search.aggregations.bucket.terms;
-
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentParser.Token;
-import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode;
-import org.elasticsearch.search.aggregations.bucket.terms.Terms.Order;
-import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator.BucketCountThresholds;
-import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude;
-import org.elasticsearch.search.aggregations.support.XContentParseContext;
-import org.elasticsearch.search.aggregations.support.ValueType;
-import org.elasticsearch.search.aggregations.support.ValuesSourceType;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
-/**
- *
- */
-public class TermsParser extends AbstractTermsParser {
-    @Override
-    protected TermsAggregationBuilder doCreateFactory(String aggregationName, ValuesSourceType valuesSourceType,
-                                                      ValueType targetValueType, BucketCountThresholds bucketCountThresholds,
-                                                      SubAggCollectionMode collectMode, String executionHint,
-                                                      IncludeExclude incExc, Map<ParseField, Object> otherOptions) {
-        TermsAggregationBuilder factory = new TermsAggregationBuilder(aggregationName, targetValueType);
-        @SuppressWarnings("unchecked")
-        List<OrderElement> orderElements = (List<OrderElement>) otherOptions.get(TermsAggregationBuilder.ORDER_FIELD);
-        if (orderElements != null) {
-            List<Terms.Order> orders = new ArrayList<>(orderElements.size());
-            for (OrderElement orderElement : orderElements) {
-                orders.add(resolveOrder(orderElement.key(), orderElement.asc()));
-            }
-            factory.order(orders);
-        }
-        if (bucketCountThresholds != null) {
-            factory.bucketCountThresholds(bucketCountThresholds);
-        }
-        if (collectMode != null) {
-            factory.collectMode(collectMode);
-        }
-        if (executionHint != null) {
-            factory.executionHint(executionHint);
-        }
-        if (incExc != null) {
-            factory.includeExclude(incExc);
-        }
-        Boolean showTermDocCountError = (Boolean) otherOptions.get(TermsAggregationBuilder.SHOW_TERM_DOC_COUNT_ERROR);
-        if (showTermDocCountError != null) {
-            factory.showTermDocCountError(showTermDocCountError);
-        }
-        return factory;
-    }
-
-    @Override
-    public boolean parseSpecial(String aggregationName, XContentParseContext context, Token token,
-                                String currentFieldName, Map<ParseField, Object> otherOptions) throws IOException {
-        XContentParser parser = context.getParser();
-        if (token == XContentParser.Token.START_OBJECT) {
-            if (context.matchField(currentFieldName, TermsAggregationBuilder.ORDER_FIELD)) {
-                otherOptions.put(TermsAggregationBuilder.ORDER_FIELD, Collections.singletonList(parseOrderParam(aggregationName, parser)));
-                return true;
-            }
-        } else if (token == XContentParser.Token.START_ARRAY) {
-            if (context.matchField(currentFieldName, TermsAggregationBuilder.ORDER_FIELD)) {
-                List<OrderElement> orderElements = new ArrayList<>();
-                while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
-                    if (token == XContentParser.Token.START_OBJECT) {
-                        OrderElement orderParam = parseOrderParam(aggregationName, parser);
-                        orderElements.add(orderParam);
-                    } else {
-                        throw new ParsingException(parser.getTokenLocation(),
-                                "Order elements must be of type object in [" + aggregationName + "] found token of type [" + token + "].");
-                    }
-                }
-                otherOptions.put(TermsAggregationBuilder.ORDER_FIELD, orderElements);
-                return true;
-            }
-        } else if (token == XContentParser.Token.VALUE_BOOLEAN) {
-            if (context.matchField(currentFieldName, TermsAggregationBuilder.SHOW_TERM_DOC_COUNT_ERROR)) {
-                otherOptions.put(TermsAggregationBuilder.SHOW_TERM_DOC_COUNT_ERROR, parser.booleanValue());
-                return true;
-            }
-        }
-        return false;
-    }
-
-    private OrderElement parseOrderParam(String aggregationName, XContentParser parser) throws IOException {
-        XContentParser.Token token;
-        OrderElement orderParam = null;
-        String orderKey = null;
-        boolean orderAsc = false;
-        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-            if (token == XContentParser.Token.FIELD_NAME) {
-                orderKey = parser.currentName();
-            } else if (token == XContentParser.Token.VALUE_STRING) {
-                String dir = parser.text();
-                if ("asc".equalsIgnoreCase(dir)) {
-                    orderAsc = true;
-                } else if ("desc".equalsIgnoreCase(dir)) {
-                    orderAsc = false;
-                } else {
-                    throw new ParsingException(parser.getTokenLocation(),
-                            "Unknown terms order direction [" + dir + "] in terms aggregation [" + aggregationName + "]");
-                }
-            } else {
-                throw new ParsingException(parser.getTokenLocation(),
-                        "Unexpected token " + token + " for [order] in [" + aggregationName + "].");
-            }
-        }
-        if (orderKey == null) {
-            throw new ParsingException(parser.getTokenLocation(),
-                    "Must specify at least one field for [order] in [" + aggregationName + "].");
-        } else {
-            orderParam = new OrderElement(orderKey, orderAsc);
-        }
-        return orderParam;
-    }
-
-    static class OrderElement {
-        private final String key;
-        private final boolean asc;
-
-        public OrderElement(String key, boolean asc) {
-            this.key = key;
-            this.asc = asc;
-        }
-
-        public String key() {
-            return key;
-        }
-
-        public boolean asc() {
-            return asc;
-        }
-
-    }
-
-    @Override
-    public TermsAggregator.BucketCountThresholds getDefaultBucketCountThresholds() {
-        return new TermsAggregator.BucketCountThresholds(TermsAggregationBuilder.DEFAULT_BUCKET_COUNT_THRESHOLDS);
-    }
-
-    static Terms.Order resolveOrder(String key, boolean asc) {
-        if ("_term".equals(key)) {
-            return Order.term(asc);
-        }
-        if ("_count".equals(key)) {
-            return Order.count(asc);
-        }
-        return Order.aggregation(key, asc);
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/UnmappedTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/UnmappedTerms.java
index c3d1a8937c4ed..ebd5e8a62ba52 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/UnmappedTerms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/UnmappedTerms.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
@@ -71,6 +72,11 @@ public String getWriteableName() {
         return NAME;
     }
 
+    @Override
+    public String getType() {
+        return StringTerms.NAME;
+    }
+
     @Override
     public UnmappedTerms create(List<Bucket> buckets) {
         return new UnmappedTerms(name, order, requiredSize, minDocCount, pipelineAggregators(), metaData);
@@ -97,11 +103,8 @@ public InternalAggregation doReduce(List<InternalAggregation> aggregations, Redu
     }
 
     @Override
-    public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
-        builder.field(InternalTerms.DOC_COUNT_ERROR_UPPER_BOUND_FIELD_NAME, 0);
-        builder.field(SUM_OF_OTHER_DOC_COUNTS, 0);
-        builder.startArray(CommonFields.BUCKETS).endArray();
-        return builder;
+    public final XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+        return doXContentCommon(builder, params, 0, 0, Collections.emptyList());
     }
 
     @Override
@@ -124,7 +127,7 @@ public long getSumOfOtherDocCounts() {
     }
 
     @Override
-    protected List<Bucket> getBucketsInternal() {
+    public List<Bucket> getBuckets() {
         return emptyList();
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/support/IncludeExclude.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/support/IncludeExclude.java
index e751c54fb16e9..7499575f642d9 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/support/IncludeExclude.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/support/IncludeExclude.java
@@ -18,9 +18,9 @@
  */
 package org.elasticsearch.search.aggregations.bucket.terms.support;
 
+import com.carrotsearch.hppc.BitMixer;
 import com.carrotsearch.hppc.LongHashSet;
 import com.carrotsearch.hppc.LongSet;
-
 import org.apache.lucene.index.RandomAccessOrds;
 import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.index.Terms;
@@ -28,6 +28,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.LongBitSet;
 import org.apache.lucene.util.NumericUtils;
+import org.apache.lucene.util.StringHelper;
 import org.apache.lucene.util.automaton.Automata;
 import org.apache.lucene.util.automaton.Automaton;
 import org.apache.lucene.util.automaton.ByteRunAutomaton;
@@ -35,8 +36,8 @@
 import org.apache.lucene.util.automaton.Operations;
 import org.apache.lucene.util.automaton.RegExp;
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -47,7 +48,6 @@
 
 import java.io.IOException;
 import java.util.HashSet;
-import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.SortedSet;
@@ -58,18 +58,143 @@
  * exclusion has precedence, where the {@code include} is evaluated first and then the {@code exclude}.
  */
 public class IncludeExclude implements Writeable, ToXContent {
-    private static final ParseField INCLUDE_FIELD = new ParseField("include");
-    private static final ParseField EXCLUDE_FIELD = new ParseField("exclude");
-    private static final ParseField PATTERN_FIELD = new ParseField("pattern");
+    public static final ParseField INCLUDE_FIELD = new ParseField("include");
+    public static final ParseField EXCLUDE_FIELD = new ParseField("exclude");
+    public static final ParseField PATTERN_FIELD = new ParseField("pattern").withAllDeprecated("Put patterns directly under the [include] or [exclude]");
+    public static final ParseField PARTITION_FIELD = new ParseField("partition");
+    public static final ParseField NUM_PARTITIONS_FIELD = new ParseField("num_partitions");
+    // Needed to add this seed for a deterministic term hashing policy
+    // otherwise tests fail to get expected results and worse, shards
+    // can disagree on which terms hash to the required partition.
+    private static final int HASH_PARTITIONING_SEED = 31;
+
+    // for parsing purposes only
+    // TODO: move all aggs to the same package so that this stuff could be pkg-private
+    public static IncludeExclude merge(IncludeExclude include, IncludeExclude exclude) {
+        if (include == null) {
+            return exclude;
+        }
+        if (exclude == null) {
+            return include;
+        }
+        if (include.isPartitionBased()) {
+            throw new IllegalArgumentException("Cannot specify any excludes when using a partition-based include");
+        }
+        String includeMethod = include.isRegexBased() ? "regex" : "set";
+        String excludeMethod = exclude.isRegexBased() ? "regex" : "set";
+        if (includeMethod.equals(excludeMethod) == false) {
+            throw new IllegalArgumentException("Cannot mix a " + includeMethod + "-based include with a "
+                    + excludeMethod + "-based method");
+        }
+        if (include.isRegexBased()) {
+            return new IncludeExclude(include.include, exclude.exclude);
+        } else {
+            return new IncludeExclude(include.includeValues, exclude.excludeValues);
+        }
+    }
+
+    public static IncludeExclude parseInclude(XContentParser parser) throws IOException {
+        XContentParser.Token token = parser.currentToken();
+        if (token == XContentParser.Token.VALUE_STRING) {
+            return new IncludeExclude(parser.text(), null);
+        } else if (token == XContentParser.Token.START_ARRAY) {
+            return new IncludeExclude(new TreeSet<>(parseArrayToSet(parser)), null);
+        } else if (token == XContentParser.Token.START_OBJECT) {
+            String currentFieldName = null;
+            Integer partition = null, numPartitions = null;
+            String pattern = null;
+            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                if (token == XContentParser.Token.FIELD_NAME) {
+                    currentFieldName = parser.currentName();
+                } else
+                // This "include":{"pattern":"foo.*"} syntax is undocumented since 2.0
+                // Regexes should be "include":"foo.*"
+                if (PATTERN_FIELD.match(currentFieldName)) {
+                    pattern = parser.text();
+                } else if (NUM_PARTITIONS_FIELD.match(currentFieldName)) {
+                    numPartitions = parser.intValue();
+                } else if (PARTITION_FIELD.match(currentFieldName)) {
+                    partition = parser.intValue();
+                } else {
+                    throw new ElasticsearchParseException(
+                            "Unknown parameter in Include/Exclude clause: " + currentFieldName);
+                }
+            }
+
+            final boolean hasPattern = pattern != null;
+            final boolean hasPartition = partition != null || numPartitions != null;
+            if (hasPattern && hasPartition) {
+                throw new IllegalArgumentException("Cannot mix pattern-based and partition-based includes");
+            }
+
+            if (pattern != null) {
+                return new IncludeExclude(pattern, null);
+            }
+
+            if (partition == null) {
+                throw new IllegalArgumentException("Missing [" + PARTITION_FIELD.getPreferredName()
+                    + "] parameter for partition-based include");
+            }
+            if (numPartitions == null) {
+                throw new IllegalArgumentException("Missing [" + NUM_PARTITIONS_FIELD.getPreferredName()
+                    + "] parameter for partition-based include");
+            }
+            return new IncludeExclude(partition, numPartitions);
+        } else {
+            throw new IllegalArgumentException("Unrecognized token for an include [" + token + "]");
+        }
+    }
+
+    public static IncludeExclude parseExclude(XContentParser parser) throws IOException {
+        XContentParser.Token token = parser.currentToken();
+        if (token == XContentParser.Token.VALUE_STRING) {
+            return new IncludeExclude(null, parser.text());
+        } else if (token == XContentParser.Token.START_ARRAY) {
+            return new IncludeExclude(null, new TreeSet<>(parseArrayToSet(parser)));
+        } else if (token == XContentParser.Token.START_OBJECT) {
+            String currentFieldName = null;
+            String pattern = null;
+            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                if (token == XContentParser.Token.FIELD_NAME) {
+                    currentFieldName = parser.currentName();
+                } else if (PATTERN_FIELD.match(currentFieldName)) {
+                    pattern = parser.text();
+                } else {
+                    throw new IllegalArgumentException("Unrecognized field [" + parser.currentName() + "]");
+                }
+            }
+            if (pattern == null) {
+                throw new IllegalArgumentException("Missing [pattern] element under [exclude]");
+            }
+            return new IncludeExclude(null, pattern);
+        } else {
+            throw new IllegalArgumentException("Unrecognized token for an exclude [" + token + "]");
+        }
+    }
 
     // The includeValue and excludeValue ByteRefs which are the result of the parsing
     // process are converted into a LongFilter when used on numeric fields
     // in the index.
-    public static class LongFilter {
+    public abstract static class LongFilter {
+        public abstract boolean accept(long value);
+
+    }
+
+    public class PartitionedLongFilter extends LongFilter {
+        @Override
+        public boolean accept(long value) {
+            // hash the value to keep even distributions
+            final long hashCode = BitMixer.mix64(value);
+            return Math.floorMod(hashCode, incNumPartitions) == incZeroBasedPartition;
+        }
+    }
+
+
+    public static class SetBackedLongFilter extends LongFilter {
         private LongSet valids;
         private LongSet invalids;
 
-        private LongFilter(int numValids, int numInvalids) {
+        private SetBackedLongFilter(int numValids, int numInvalids) {
             if (numValids > 0) {
                 valids = new LongHashSet(numValids);
             }
@@ -96,6 +221,13 @@ public abstract static class StringFilter {
         public abstract boolean accept(BytesRef value);
     }
 
+    class PartitionedStringFilter extends StringFilter {
+        @Override
+        public boolean accept(BytesRef value) {
+            return Math.floorMod(StringHelper.murmurhash3_x86_32(value, HASH_PARTITIONING_SEED), incNumPartitions) == incZeroBasedPartition;
+        }
+    }
+
     static class AutomatonBackedStringFilter extends StringFilter {
 
         private final ByteRunAutomaton runAutomaton;
@@ -118,7 +250,7 @@ static class TermListBackedStringFilter extends StringFilter {
         private final Set<BytesRef> valids;
         private final Set<BytesRef> invalids;
 
-        public TermListBackedStringFilter(Set<BytesRef> includeValues, Set<BytesRef> excludeValues) {
+        TermListBackedStringFilter(Set<BytesRef> includeValues, Set<BytesRef> excludeValues) {
             this.valids = includeValues;
             this.invalids = excludeValues;
         }
@@ -135,7 +267,25 @@ public boolean accept(BytesRef value) {
 
     public abstract static class OrdinalsFilter {
         public abstract LongBitSet acceptedGlobalOrdinals(RandomAccessOrds globalOrdinals) throws IOException;
+    }
+
+    class PartitionedOrdinalsFilter extends OrdinalsFilter {
 
+        @Override
+        public LongBitSet acceptedGlobalOrdinals(RandomAccessOrds globalOrdinals) throws IOException {
+            final long numOrds = globalOrdinals.getValueCount();
+            final LongBitSet acceptedGlobalOrdinals = new LongBitSet(numOrds);
+            final TermsEnum termEnum = globalOrdinals.termsEnum();
+
+            BytesRef term = termEnum.next();
+            while (term != null) {
+                if (Math.floorMod(StringHelper.murmurhash3_x86_32(term, HASH_PARTITIONING_SEED), incNumPartitions) == incZeroBasedPartition) {
+                    acceptedGlobalOrdinals.set(termEnum.ord());
+                }
+                term = termEnum.next();
+            }
+            return acceptedGlobalOrdinals;
+        }
     }
 
     static class AutomatonBackedOrdinalsFilter extends OrdinalsFilter {
@@ -151,8 +301,7 @@ private AutomatonBackedOrdinalsFilter(Automaton automaton) {
          *
          */
         @Override
-        public LongBitSet acceptedGlobalOrdinals(RandomAccessOrds globalOrdinals)
-                throws IOException {
+        public LongBitSet acceptedGlobalOrdinals(RandomAccessOrds globalOrdinals) throws IOException {
             LongBitSet acceptedGlobalOrdinals = new LongBitSet(globalOrdinals.getValueCount());
             TermsEnum globalTermsEnum;
             Terms globalTerms = new DocValuesTerms(globalOrdinals);
@@ -171,7 +320,7 @@ static class TermListBackedOrdinalsFilter extends OrdinalsFilter {
         private final SortedSet<BytesRef> includeValues;
         private final SortedSet<BytesRef> excludeValues;
 
-        public TermListBackedOrdinalsFilter(SortedSet<BytesRef> includeValues, SortedSet<BytesRef> excludeValues) {
+        TermListBackedOrdinalsFilter(SortedSet<BytesRef> includeValues, SortedSet<BytesRef> excludeValues) {
             this.includeValues = includeValues;
             this.excludeValues = excludeValues;
         }
@@ -205,6 +354,8 @@ public LongBitSet acceptedGlobalOrdinals(RandomAccessOrds globalOrdinals) throws
 
     private final RegExp include, exclude;
     private final SortedSet<BytesRef> includeValues, excludeValues;
+    private final int incZeroBasedPartition;
+    private final int incNumPartitions;
 
     /**
      * @param include   The regular expression pattern for the terms to be included
@@ -218,6 +369,8 @@ public IncludeExclude(RegExp include, RegExp exclude) {
         this.exclude = exclude;
         this.includeValues = null;
         this.excludeValues = null;
+        this.incZeroBasedPartition = 0;
+        this.incNumPartitions = 0;
     }
 
     public IncludeExclude(String include, String exclude) {
@@ -234,6 +387,8 @@ public IncludeExclude(SortedSet<BytesRef> includeValues, SortedSet<BytesRef> exc
         }
         this.include = null;
         this.exclude = null;
+        this.incZeroBasedPartition = 0;
+        this.incNumPartitions = 0;
         this.includeValues = includeValues;
         this.excludeValues = excludeValues;
     }
@@ -250,6 +405,21 @@ public IncludeExclude(long[] includeValues, long[] excludeValues) {
         this(convertToBytesRefSet(includeValues), convertToBytesRefSet(excludeValues));
     }
 
+    public IncludeExclude(int partition, int numPartitions) {
+        if (partition < 0 || partition >= numPartitions) {
+            throw new IllegalArgumentException("Partition must be >=0 and < numPartition which is "+numPartitions);
+        }
+        this.incZeroBasedPartition = partition;
+        this.incNumPartitions = numPartitions;
+        this.include = null;
+        this.exclude = null;
+        this.includeValues = null;
+        this.excludeValues = null;
+
+    }
+
+
+
     /**
      * Read from a stream.
      */
@@ -257,6 +427,8 @@ public IncludeExclude(StreamInput in) throws IOException {
         if (in.readBoolean()) {
             includeValues = null;
             excludeValues = null;
+            incZeroBasedPartition = 0;
+            incNumPartitions = 0;
             String includeString = in.readOptionalString();
             include = includeString == null ? null : new RegExp(includeString);
             String excludeString = in.readOptionalString();
@@ -283,6 +455,13 @@ public IncludeExclude(StreamInput in) throws IOException {
         } else {
             excludeValues = null;
         }
+        if (in.getVersion().onOrAfter(Version.V_5_2_0)) {
+            incNumPartitions = in.readVInt();
+            incZeroBasedPartition = in.readVInt();
+        } else {
+            incNumPartitions = 0;
+            incZeroBasedPartition = 0;
+        }
     }
 
     @Override
@@ -309,6 +488,10 @@ public void writeTo(StreamOutput out) throws IOException {
                     out.writeBytesRef(value);
                 }
             }
+            if (out.getVersion().onOrAfter(Version.V_5_2_0)) {
+                out.writeVInt(incNumPartitions);
+                out.writeVInt(incZeroBasedPartition);
+            }
         }
     }
 
@@ -403,120 +586,28 @@ public boolean hasPayloads() {
 
     }
 
-
-
-    public static class Parser {
-
-        public boolean token(String currentFieldName, XContentParser.Token token, XContentParser parser,
-                ParseFieldMatcher parseFieldMatcher, Map<ParseField, Object> otherOptions) throws IOException {
-
-            if (token == XContentParser.Token.VALUE_STRING) {
-                if (parseFieldMatcher.match(currentFieldName, INCLUDE_FIELD)) {
-                    otherOptions.put(INCLUDE_FIELD, parser.text());
-                } else if (parseFieldMatcher.match(currentFieldName, EXCLUDE_FIELD)) {
-                    otherOptions.put(EXCLUDE_FIELD, parser.text());
-                } else {
-                    return false;
-                }
-                return true;
-            }
-
-            if (token == XContentParser.Token.START_ARRAY) {
-                if (parseFieldMatcher.match(currentFieldName, INCLUDE_FIELD)) {
-                    otherOptions.put(INCLUDE_FIELD, new TreeSet<>(parseArrayToSet(parser)));
-                     return true;
-                }
-                if (parseFieldMatcher.match(currentFieldName, EXCLUDE_FIELD)) {
-                    otherOptions.put(EXCLUDE_FIELD, new TreeSet<>(parseArrayToSet(parser)));
-                      return true;
-                }
-                return false;
-            }
-
-            if (token == XContentParser.Token.START_OBJECT) {
-                if (parseFieldMatcher.match(currentFieldName, INCLUDE_FIELD)) {
-                    while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                        if (token == XContentParser.Token.FIELD_NAME) {
-                            currentFieldName = parser.currentName();
-                        } else if (token == XContentParser.Token.VALUE_STRING) {
-                            if (parseFieldMatcher.match(currentFieldName, PATTERN_FIELD)) {
-                                otherOptions.put(INCLUDE_FIELD, parser.text());
-                            }
-                        }
-                    }
-                } else if (parseFieldMatcher.match(currentFieldName, EXCLUDE_FIELD)) {
-                    while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                        if (token == XContentParser.Token.FIELD_NAME) {
-                            currentFieldName = parser.currentName();
-                        } else if (token == XContentParser.Token.VALUE_STRING) {
-                            if (parseFieldMatcher.match(currentFieldName, PATTERN_FIELD)) {
-                                otherOptions.put(EXCLUDE_FIELD, parser.text());
-                            }
-                        }
-                    }
-                } else {
-                    return false;
-                }
-                return true;
-            }
-
-            return false;
+    private static Set<BytesRef> parseArrayToSet(XContentParser parser) throws IOException {
+        final Set<BytesRef> set = new HashSet<>();
+        if (parser.currentToken() != XContentParser.Token.START_ARRAY) {
+            throw new ElasticsearchParseException("Missing start of array in include/exclude clause");
         }
-
-        private Set<BytesRef> parseArrayToSet(XContentParser parser) throws IOException {
-            final Set<BytesRef> set = new HashSet<>();
-            if (parser.currentToken() != XContentParser.Token.START_ARRAY) {
-                throw new ElasticsearchParseException("Missing start of array in include/exclude clause");
-            }
-            while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
-                if (!parser.currentToken().isValue()) {
-                    throw new ElasticsearchParseException("Array elements in include/exclude clauses should be string values");
-                }
-                set.add(new BytesRef(parser.text()));
-            }
-            return set;
-        }
-
-        public IncludeExclude createIncludeExclude(Map<ParseField, Object> otherOptions) {
-            Object includeObject = otherOptions.get(INCLUDE_FIELD);
-            String include = null;
-            SortedSet<BytesRef> includeValues = null;
-            if (includeObject != null) {
-                if (includeObject instanceof String) {
-                    include = (String) includeObject;
-                } else if (includeObject instanceof SortedSet) {
-                    includeValues = (SortedSet<BytesRef>) includeObject;
-                }
-            }
-            Object excludeObject = otherOptions.get(EXCLUDE_FIELD);
-            String exclude = null;
-            SortedSet<BytesRef> excludeValues = null;
-            if (excludeObject != null) {
-                if (excludeObject instanceof String) {
-                    exclude = (String) excludeObject;
-                } else if (excludeObject instanceof SortedSet) {
-                    excludeValues = (SortedSet<BytesRef>) excludeObject;
-                }
-            }
-            RegExp includePattern =  include != null ? new RegExp(include) : null;
-            RegExp excludePattern = exclude != null ? new RegExp(exclude) : null;
-            if (includePattern != null || excludePattern != null) {
-                if (includeValues != null || excludeValues != null) {
-                    throw new IllegalArgumentException("Can only use regular expression include/exclude or a set of values, not both");
-                }
-                return new IncludeExclude(includePattern, excludePattern);
-            } else if (includeValues != null || excludeValues != null) {
-                return new IncludeExclude(includeValues, excludeValues);
-            } else {
-                return null;
+        while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
+            if (!parser.currentToken().isValue()) {
+                throw new ElasticsearchParseException("Array elements in include/exclude clauses should be string values");
             }
+            set.add(new BytesRef(parser.text()));
         }
+        return set;
     }
 
     public boolean isRegexBased() {
         return include != null || exclude != null;
     }
 
+    public boolean isPartitionBased() {
+        return incNumPartitions > 0;
+    }
+
     private Automaton toAutomaton() {
         Automaton a = null;
         if (include != null) {
@@ -538,6 +629,9 @@ public StringFilter convertToStringFilter(DocValueFormat format) {
         if (isRegexBased()) {
             return new AutomatonBackedStringFilter(toAutomaton());
         }
+        if (isPartitionBased()){
+            return new PartitionedStringFilter();
+        }
         return new TermListBackedStringFilter(parseForDocValues(includeValues, format), parseForDocValues(excludeValues, format));
     }
 
@@ -559,13 +653,22 @@ public OrdinalsFilter convertToOrdinalsFilter(DocValueFormat format) {
         if (isRegexBased()) {
             return new AutomatonBackedOrdinalsFilter(toAutomaton());
         }
+        if (isPartitionBased()){
+            return new PartitionedOrdinalsFilter();
+        }
+
         return new TermListBackedOrdinalsFilter(parseForDocValues(includeValues, format), parseForDocValues(excludeValues, format));
     }
 
     public LongFilter convertToLongFilter(DocValueFormat format) {
+
+        if(isPartitionBased()){
+            return new PartitionedLongFilter();
+        }
+
         int numValids = includeValues == null ? 0 : includeValues.size();
         int numInvalids = excludeValues == null ? 0 : excludeValues.size();
-        LongFilter result = new LongFilter(numValids, numInvalids);
+        SetBackedLongFilter result = new SetBackedLongFilter(numValids, numInvalids);
         if (includeValues != null) {
             for (BytesRef val : includeValues) {
                 result.addAccept(format.parseLong(val.utf8ToString(), false, null));
@@ -580,9 +683,13 @@ public LongFilter convertToLongFilter(DocValueFormat format) {
     }
 
     public LongFilter convertToDoubleFilter() {
+        if(isPartitionBased()){
+            return new PartitionedLongFilter();
+        }
+
         int numValids = includeValues == null ? 0 : includeValues.size();
         int numInvalids = excludeValues == null ? 0 : excludeValues.size();
-        LongFilter result = new LongFilter(numValids, numInvalids);
+        SetBackedLongFilter result = new SetBackedLongFilter(numValids, numInvalids);
         if (includeValues != null) {
             for (BytesRef val : includeValues) {
                 double dval = Double.parseDouble(val.utf8ToString());
@@ -602,18 +709,21 @@ public LongFilter convertToDoubleFilter() {
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         if (include != null) {
             builder.field(INCLUDE_FIELD.getPreferredName(), include.getOriginalString());
-        }
-        if (includeValues != null) {
+        } else if (includeValues != null) {
             builder.startArray(INCLUDE_FIELD.getPreferredName());
             for (BytesRef value : includeValues) {
                 builder.value(value.utf8ToString());
             }
             builder.endArray();
+        } else if (isPartitionBased()) {
+            builder.startObject(INCLUDE_FIELD.getPreferredName());
+            builder.field(PARTITION_FIELD.getPreferredName(), incZeroBasedPartition);
+            builder.field(NUM_PARTITIONS_FIELD.getPreferredName(), incNumPartitions);
+            builder.endObject();
         }
         if (exclude != null) {
             builder.field(EXCLUDE_FIELD.getPreferredName(), exclude.getOriginalString());
-        }
-        if (excludeValues != null) {
+        } else if (excludeValues != null) {
             builder.startArray(EXCLUDE_FIELD.getPreferredName());
             for (BytesRef value : excludeValues) {
                 builder.value(value.utf8ToString());
@@ -625,8 +735,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
 
     @Override
     public int hashCode() {
-        return Objects.hash(include == null ? null : include.getOriginalString(), exclude == null ? null : exclude.getOriginalString(),
-                includeValues, excludeValues);
+        return Objects.hash(
+                include == null ? null : include.getOriginalString(),
+                exclude == null ? null : exclude.getOriginalString(),
+                includeValues, excludeValues, incZeroBasedPartition, incNumPartitions);
     }
 
     @Override
@@ -640,7 +752,9 @@ public boolean equals(Object obj) {
         return Objects.equals(include == null ? null : include.getOriginalString(), other.include == null ? null : other.include.getOriginalString())
                 && Objects.equals(exclude == null ? null : exclude.getOriginalString(), other.exclude == null ? null : other.exclude.getOriginalString())
                 && Objects.equals(includeValues, other.includeValues)
-                && Objects.equals(excludeValues, other.excludeValues);
+                && Objects.equals(excludeValues, other.excludeValues)
+                && Objects.equals(incZeroBasedPartition, other.incZeroBasedPartition)
+                && Objects.equals(incNumPartitions, other.incNumPartitions);
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalMetricsAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalMetricsAggregation.java
deleted file mode 100644
index ded69d9f75bfb..0000000000000
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalMetricsAggregation.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.aggregations.metrics;
-
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.search.aggregations.InternalAggregation;
-import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-
-public abstract class InternalMetricsAggregation extends InternalAggregation {
-    protected InternalMetricsAggregation(String name, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
-        super(name, pipelineAggregators, metaData);
-    }
-
-    /**
-     * Read from a stream.
-     */
-    protected InternalMetricsAggregation(StreamInput in) throws IOException {
-        super(in);
-    }
-
-}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalNumericMetricsAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalNumericMetricsAggregation.java
index 901c52a232d22..b5d1eeddae9d8 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalNumericMetricsAggregation.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalNumericMetricsAggregation.java
@@ -20,16 +20,14 @@
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
-/**
- *
- */
-public abstract class InternalNumericMetricsAggregation extends InternalMetricsAggregation {
+public abstract class InternalNumericMetricsAggregation extends InternalAggregation {
 
     private static final DocValueFormat DEFAULT_FORMAT = DocValueFormat.RAW;
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/MetricsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/MetricsAggregator.java
index 30330e617196e..0a8553e00547e 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/MetricsAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/MetricsAggregator.java
@@ -23,7 +23,7 @@
 import org.elasticsearch.search.aggregations.AggregatorBase;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -31,7 +31,7 @@
 
 public abstract class MetricsAggregator extends AggregatorBase {
     
-    protected MetricsAggregator(String name, AggregationContext context, Aggregator parent, List<PipelineAggregator> pipelineAggregators,
+    protected MetricsAggregator(String name, SearchContext context, Aggregator parent, List<PipelineAggregator> pipelineAggregators,
             Map<String, Object> metaData) throws IOException {
         super(name, AggregatorFactories.EMPTY, context, parent, pipelineAggregators, metaData);
     }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/NumericMetricsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/NumericMetricsAggregator.java
index 3ffd7f797503f..c81169ae13ace 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/NumericMetricsAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/NumericMetricsAggregator.java
@@ -20,7 +20,7 @@
 
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -31,14 +31,14 @@
  */
 public abstract class NumericMetricsAggregator extends MetricsAggregator {
 
-    private NumericMetricsAggregator(String name, AggregationContext context, Aggregator parent,
+    private NumericMetricsAggregator(String name, SearchContext context, Aggregator parent,
             List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException {
         super(name, context, parent, pipelineAggregators, metaData);
     }
 
     public abstract static class SingleValue extends NumericMetricsAggregator {
 
-        protected SingleValue(String name, AggregationContext context, Aggregator parent, List<PipelineAggregator> pipelineAggregators,
+        protected SingleValue(String name, SearchContext context, Aggregator parent, List<PipelineAggregator> pipelineAggregators,
                 Map<String, Object> metaData) throws IOException {
             super(name, context, parent, pipelineAggregators, metaData);
         }
@@ -48,7 +48,7 @@ protected SingleValue(String name, AggregationContext context, Aggregator parent
 
     public abstract static class MultiValue extends NumericMetricsAggregator {
 
-        protected MultiValue(String name, AggregationContext context, Aggregator parent, List<PipelineAggregator> pipelineAggregators,
+        protected MultiValue(String name, SearchContext context, Aggregator parent, List<PipelineAggregator> pipelineAggregators,
                 Map<String, Object> metaData) throws IOException {
             super(name, context, parent, pipelineAggregators, metaData);
         }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/ParsedSingleValueNumericMetricsAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/ParsedSingleValueNumericMetricsAggregation.java
new file mode 100644
index 0000000000000..3105a785e1903
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/ParsedSingleValueNumericMetricsAggregation.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.metrics;
+
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
+import org.elasticsearch.search.aggregations.ParsedAggregation;
+
+public abstract class ParsedSingleValueNumericMetricsAggregation extends ParsedAggregation
+        implements NumericMetricsAggregation.SingleValue {
+
+    protected double value;
+    protected String valueAsString;
+
+    @Override
+    public String getValueAsString() {
+        if (valueAsString != null) {
+            return valueAsString;
+        } else {
+            return Double.toString(value);
+        }
+    }
+
+    @Override
+    public double value() {
+        return value;
+    }
+
+    protected void setValue(double value) {
+        this.value = value;
+    }
+
+    protected void setValueAsString(String valueAsString) {
+        this.valueAsString = valueAsString;
+    }
+
+    protected static void declareSingleValueFields(ObjectParser<? extends ParsedSingleValueNumericMetricsAggregation, Void> objectParser,
+            double defaultNullValue) {
+        declareAggregationFields(objectParser);
+        objectParser.declareField(ParsedSingleValueNumericMetricsAggregation::setValue,
+                (parser, context) -> parseDouble(parser, defaultNullValue), CommonFields.VALUE, ValueType.DOUBLE_OR_NULL);
+        objectParser.declareString(ParsedSingleValueNumericMetricsAggregation::setValueAsString, CommonFields.VALUE_AS_STRING);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgAggregationBuilder.java
index 37d6788782202..0d9bd6fc1a9b7 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgAggregationBuilder.java
@@ -21,33 +21,45 @@
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 
 public class AvgAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOnly<ValuesSource.Numeric, AvgAggregationBuilder> {
     public static final String NAME = "avg";
-    private static final Type TYPE = new Type(NAME);
+
+    private static final ObjectParser<AvgAggregationBuilder, QueryParseContext> PARSER;
+    static {
+        PARSER = new ObjectParser<>(AvgAggregationBuilder.NAME);
+        ValuesSourceParserHelper.declareNumericFields(PARSER, true, true, false);
+    }
+
+    public static AggregationBuilder parse(String aggregationName, QueryParseContext context) throws IOException {
+        return PARSER.parse(context.parser(), new AvgAggregationBuilder(aggregationName), context);
+    }
 
     public AvgAggregationBuilder(String name) {
-        super(name, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(name, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     /**
      * Read from a stream.
      */
     public AvgAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(in, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     @Override
@@ -56,9 +68,9 @@ protected void innerWriteTo(StreamOutput out) {
     }
 
     @Override
-    protected AvgAggregatorFactory innerBuild(AggregationContext context, ValuesSourceConfig<Numeric> config,
+    protected AvgAggregatorFactory innerBuild(SearchContext context, ValuesSourceConfig<Numeric> config,
             AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
-        return new AvgAggregatorFactory(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        return new AvgAggregatorFactory(name, config, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
@@ -77,7 +89,7 @@ protected boolean innerEquals(Object obj) {
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgAggregator.java
index eb0fc42f9c20b..c948334bc5bce 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgAggregator.java
@@ -31,8 +31,8 @@
 import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
 import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregator;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -49,7 +49,7 @@ public class AvgAggregator extends NumericMetricsAggregator.SingleValue {
     DoubleArray sums;
     DocValueFormat format;
 
-    public AvgAggregator(String name, ValuesSource.Numeric valuesSource, DocValueFormat formatter, AggregationContext context,
+    public AvgAggregator(String name, ValuesSource.Numeric valuesSource, DocValueFormat formatter, SearchContext context,
             Aggregator parent, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException {
         super(name, context, parent, pipelineAggregators, metaData);
         this.valuesSource = valuesSource;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgAggregatorFactory.java
index 5d425f2732cbf..f1fc12ef4e505 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgAggregatorFactory.java
@@ -22,13 +22,12 @@
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -36,9 +35,9 @@
 
 public class AvgAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource.Numeric, AvgAggregatorFactory> {
 
-    public AvgAggregatorFactory(String name, Type type, ValuesSourceConfig<Numeric> config, AggregationContext context,
+    public AvgAggregatorFactory(String name, ValuesSourceConfig<Numeric> config, SearchContext context,
             AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgParser.java
deleted file mode 100644
index bc6f762295cd8..0000000000000
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgParser.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.search.aggregations.metrics.avg;
-
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.search.aggregations.support.AbstractValuesSourceParser.NumericValuesSourceParser;
-import org.elasticsearch.search.aggregations.support.XContentParseContext;
-import org.elasticsearch.search.aggregations.support.ValueType;
-import org.elasticsearch.search.aggregations.support.ValuesSourceType;
-
-import java.io.IOException;
-import java.util.Map;
-
-/**
- *
- */
-public class AvgParser extends NumericValuesSourceParser {
-
-    public AvgParser() {
-        super(true, true, false);
-    }
-
-    @Override
-    protected boolean token(String aggregationName, String currentFieldName, XContentParser.Token token,
-                            XContentParseContext context, Map<ParseField, Object> otherOptions) throws IOException {
-        return false;
-    }
-
-    @Override
-    protected AvgAggregationBuilder createFactory(String aggregationName, ValuesSourceType valuesSourceType,
-                                                  ValueType targetValueType, Map<ParseField, Object> otherOptions) {
-        return new AvgAggregationBuilder(aggregationName);
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/InternalAvg.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/InternalAvg.java
index 5f0d54db00329..f33bb931f9eda 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/InternalAvg.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/InternalAvg.java
@@ -69,6 +69,14 @@ public double getValue() {
         return sum / count;
     }
 
+    double getSum() {
+        return sum;
+    }
+
+    long getCount() {
+        return count;
+    }
+
     @Override
     public String getWriteableName() {
         return AvgAggregationBuilder.NAME;
@@ -87,11 +95,10 @@ public InternalAvg doReduce(List<InternalAggregation> aggregations, ReduceContex
 
     @Override
     public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
-        builder.field(CommonFields.VALUE, count != 0 ? getValue() : null);
+        builder.field(CommonFields.VALUE.getPreferredName(), count != 0 ? getValue() : null);
         if (count != 0 && format != DocValueFormat.RAW) {
-            builder.field(CommonFields.VALUE_AS_STRING, format.format(getValue()));
+            builder.field(CommonFields.VALUE_AS_STRING.getPreferredName(), format.format(getValue()));
         }
         return builder;
     }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/ParsedAvg.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/ParsedAvg.java
new file mode 100644
index 0000000000000..16d91bd08f0d3
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/ParsedAvg.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.avg;
+
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.metrics.ParsedSingleValueNumericMetricsAggregation;
+
+import java.io.IOException;
+
+public class ParsedAvg extends ParsedSingleValueNumericMetricsAggregation implements Avg {
+
+    @Override
+    public double getValue() {
+        return value();
+    }
+
+    @Override
+    public String getType() {
+        return AvgAggregationBuilder.NAME;
+    }
+
+    @Override
+    protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+        // InternalAvg renders value only if the avg normalizer (count) is not 0.
+        // We parse back `null` as Double.POSITIVE_INFINITY so we check for that value here to get the same xContent output
+        boolean hasValue = value != Double.POSITIVE_INFINITY;
+        builder.field(CommonFields.VALUE.getPreferredName(), hasValue ? value : null);
+        if (hasValue && valueAsString != null) {
+            builder.field(CommonFields.VALUE_AS_STRING.getPreferredName(), valueAsString);
+        }
+        return builder;
+    }
+
+    private static final ObjectParser<ParsedAvg, Void> PARSER = new ObjectParser<>(ParsedAvg.class.getSimpleName(), true, ParsedAvg::new);
+
+    static {
+        declareSingleValueFields(PARSER, Double.POSITIVE_INFINITY);
+    }
+
+    public static ParsedAvg fromXContent(XContentParser parser, final String name) {
+        ParsedAvg avg = PARSER.apply(parser, null);
+        avg.setName(name);
+        return avg;
+    }
+}
\ No newline at end of file
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregationBuilder.java
index d545cfce23ebd..1f76d8530f77f 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregationBuilder.java
@@ -22,16 +22,19 @@
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -40,21 +43,33 @@ public final class CardinalityAggregationBuilder
     extends ValuesSourceAggregationBuilder.LeafOnly<ValuesSource, CardinalityAggregationBuilder> {
 
     public static final String NAME = "cardinality";
-    private static final Type TYPE = new Type(NAME);
 
+    private static final ParseField REHASH = new ParseField("rehash").withAllDeprecated("no replacement - values will always be rehashed");
     public static final ParseField PRECISION_THRESHOLD_FIELD = new ParseField("precision_threshold");
 
+    private static final ObjectParser<CardinalityAggregationBuilder, QueryParseContext> PARSER;
+    static {
+        PARSER = new ObjectParser<>(CardinalityAggregationBuilder.NAME);
+        ValuesSourceParserHelper.declareAnyFields(PARSER, true, false);
+        PARSER.declareLong(CardinalityAggregationBuilder::precisionThreshold, CardinalityAggregationBuilder.PRECISION_THRESHOLD_FIELD);
+        PARSER.declareLong((b, v) -> {/*ignore*/}, REHASH);
+    }
+
+    public static AggregationBuilder parse(String aggregationName, QueryParseContext context) throws IOException {
+        return PARSER.parse(context.parser(), new CardinalityAggregationBuilder(aggregationName, null), context);
+    }
+
     private Long precisionThreshold = null;
 
     public CardinalityAggregationBuilder(String name, ValueType targetValueType) {
-        super(name, TYPE, ValuesSourceType.ANY, targetValueType);
+        super(name, ValuesSourceType.ANY, targetValueType);
     }
 
     /**
      * Read from a stream.
      */
     public CardinalityAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE, ValuesSourceType.ANY);
+        super(in, ValuesSourceType.ANY);
         if (in.readBoolean()) {
             precisionThreshold = in.readLong();
         }
@@ -105,9 +120,9 @@ public void rehash(boolean rehash) {
     }
 
     @Override
-    protected CardinalityAggregatorFactory innerBuild(AggregationContext context, ValuesSourceConfig<ValuesSource> config,
+    protected CardinalityAggregatorFactory innerBuild(SearchContext context, ValuesSourceConfig<ValuesSource> config,
             AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
-        return new CardinalityAggregatorFactory(name, type, config, precisionThreshold, context, parent, subFactoriesBuilder, metaData);
+        return new CardinalityAggregatorFactory(name, config, precisionThreshold, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
@@ -130,7 +145,7 @@ protected boolean innerEquals(Object obj) {
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregator.java
index 99922cc6aec1d..7d5db460ae6b0 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregator.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.search.aggregations.metrics.cardinality;
 
 import com.carrotsearch.hppc.BitMixer;
+
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.RandomAccessOrds;
 import org.apache.lucene.index.SortedNumericDocValues;
@@ -40,8 +41,8 @@
 import org.elasticsearch.search.aggregations.LeafBucketCollector;
 import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregator;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -62,7 +63,7 @@ public class CardinalityAggregator extends NumericMetricsAggregator.SingleValue
     private Collector collector;
 
     public CardinalityAggregator(String name, ValuesSource valuesSource, int precision,
-            AggregationContext context, Aggregator parent, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException {
+            SearchContext context, Aggregator parent, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException {
         super(name, context, parent, pipelineAggregators, metaData);
         this.valuesSource = valuesSource;
         this.precision = precision;
@@ -326,7 +327,7 @@ private static class Long extends MurmurHash3Values {
 
             private final SortedNumericDocValues values;
 
-            public Long(SortedNumericDocValues values) {
+            Long(SortedNumericDocValues values) {
                 this.values = values;
             }
 
@@ -350,7 +351,7 @@ private static class Double extends MurmurHash3Values {
 
             private final SortedNumericDoubleValues values;
 
-            public Double(SortedNumericDoubleValues values) {
+            Double(SortedNumericDoubleValues values) {
                 this.values = values;
             }
 
@@ -376,7 +377,7 @@ private static class Bytes extends MurmurHash3Values {
 
             private final SortedBinaryDocValues values;
 
-            public Bytes(SortedBinaryDocValues values) {
+            Bytes(SortedBinaryDocValues values) {
                 this.values = values;
             }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregatorFactory.java
index dad9ba46e6d2c..0d2d32f04697c 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregatorFactory.java
@@ -22,12 +22,11 @@
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -37,10 +36,10 @@ public class CardinalityAggregatorFactory extends ValuesSourceAggregatorFactory<
 
     private final Long precisionThreshold;
 
-    public CardinalityAggregatorFactory(String name, Type type, ValuesSourceConfig<ValuesSource> config, Long precisionThreshold,
-            AggregationContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
+    public CardinalityAggregatorFactory(String name, ValuesSourceConfig<ValuesSource> config, Long precisionThreshold,
+            SearchContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
             Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.precisionThreshold = precisionThreshold;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityParser.java
deleted file mode 100644
index e40e0767994dc..0000000000000
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityParser.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.aggregations.metrics.cardinality;
-
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.xcontent.XContentParser.Token;
-import org.elasticsearch.search.aggregations.support.AbstractValuesSourceParser.AnyValuesSourceParser;
-import org.elasticsearch.search.aggregations.support.XContentParseContext;
-import org.elasticsearch.search.aggregations.support.ValueType;
-import org.elasticsearch.search.aggregations.support.ValuesSourceType;
-
-import java.io.IOException;
-import java.util.Map;
-
-
-public class CardinalityParser extends AnyValuesSourceParser {
-
-    private static final ParseField REHASH = new ParseField("rehash").withAllDeprecated("no replacement - values will always be rehashed");
-
-    public CardinalityParser() {
-        super(true, false);
-    }
-
-    @Override
-    protected CardinalityAggregationBuilder createFactory(String aggregationName, ValuesSourceType valuesSourceType,
-                                                          ValueType targetValueType, Map<ParseField, Object> otherOptions) {
-        CardinalityAggregationBuilder factory = new CardinalityAggregationBuilder(aggregationName, targetValueType);
-        Long precisionThreshold = (Long) otherOptions.get(CardinalityAggregationBuilder.PRECISION_THRESHOLD_FIELD);
-        if (precisionThreshold != null) {
-            factory.precisionThreshold(precisionThreshold);
-        }
-        return factory;
-    }
-
-    @Override
-    protected boolean token(String aggregationName, String currentFieldName, Token token,
-                            XContentParseContext context, Map<ParseField, Object> otherOptions) throws IOException {
-        if (token.isValue()) {
-            if (context.matchField(currentFieldName, CardinalityAggregationBuilder.PRECISION_THRESHOLD_FIELD)) {
-                otherOptions.put(CardinalityAggregationBuilder.PRECISION_THRESHOLD_FIELD, context.getParser().longValue());
-                return true;
-            } else if (context.matchField(currentFieldName, REHASH)) {
-                // ignore
-                return true;
-            }
-        }
-        return false;
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/HyperLogLogPlusPlus.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/HyperLogLogPlusPlus.java
index 568ecdbec5901..42b4561e07b3c 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/HyperLogLogPlusPlus.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/HyperLogLogPlusPlus.java
@@ -433,7 +433,7 @@ private class Hashset {
         private final BytesRef readSpare;
         private final ByteBuffer writeSpare;
 
-        public Hashset(long initialBucketCount) {
+        Hashset(long initialBucketCount) {
             capacity = m / 4; // because ints take 4 bytes
             threshold = (int) (capacity * MAX_LOAD_FACTOR);
             mask = capacity - 1;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/InternalCardinality.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/InternalCardinality.java
index 02953abc2daa5..de72582b16fa3 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/InternalCardinality.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/InternalCardinality.java
@@ -109,8 +109,9 @@ public void merge(InternalCardinality other) {
     @Override
     public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
         final long cardinality = getValue();
-        builder.field(CommonFields.VALUE, cardinality);
+        builder.field(CommonFields.VALUE.getPreferredName(), cardinality);
         return builder;
     }
 
 }
+
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/ParsedCardinality.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/ParsedCardinality.java
new file mode 100644
index 0000000000000..5a615f61a4ae6
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/ParsedCardinality.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.cardinality;
+
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.ParsedAggregation;
+
+import java.io.IOException;
+
+public class ParsedCardinality extends ParsedAggregation implements Cardinality {
+
+    private long cardinalityValue;
+
+    @Override
+    public String getValueAsString() {
+        return Double.toString((double) cardinalityValue);
+    }
+
+    @Override
+    public double value() {
+        return getValue();
+    }
+
+    @Override
+    public long getValue() {
+        return cardinalityValue;
+    }
+
+    @Override
+    public String getType() {
+        return CardinalityAggregationBuilder.NAME;
+    }
+
+    private static final ObjectParser<ParsedCardinality, Void> PARSER = new ObjectParser<>(
+            ParsedCardinality.class.getSimpleName(), true, ParsedCardinality::new);
+
+    static {
+        declareAggregationFields(PARSER);
+        PARSER.declareLong((agg, value) -> agg.cardinalityValue = value, CommonFields.VALUE);
+    }
+
+    public static ParsedCardinality fromXContent(XContentParser parser, final String name) {
+        ParsedCardinality cardinality = PARSER.apply(parser, null);
+        cardinality.setName(name);
+        return cardinality;
+    }
+
+    @Override
+    protected XContentBuilder doXContentBody(XContentBuilder builder, Params params)
+            throws IOException {
+        builder.field(CommonFields.VALUE.getPreferredName(), cardinalityValue);
+        return builder;
+    }
+}
\ No newline at end of file
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/GeoBoundsAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/GeoBoundsAggregationBuilder.java
index c3565d794a835..be3ad4db802e2 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/GeoBoundsAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/GeoBoundsAggregationBuilder.java
@@ -21,35 +21,48 @@
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.Objects;
 
 public class GeoBoundsAggregationBuilder extends ValuesSourceAggregationBuilder<ValuesSource.GeoPoint, GeoBoundsAggregationBuilder> {
     public static final String NAME = "geo_bounds";
-    private static final Type TYPE = new Type(NAME);
+
+    private static final ObjectParser<GeoBoundsAggregationBuilder, QueryParseContext> PARSER;
+    static {
+        PARSER = new ObjectParser<>(GeoBoundsAggregationBuilder.NAME);
+        ValuesSourceParserHelper.declareGeoFields(PARSER, false, false);
+        PARSER.declareBoolean(GeoBoundsAggregationBuilder::wrapLongitude, GeoBoundsAggregator.WRAP_LONGITUDE_FIELD);
+    }
+
+    public static AggregationBuilder parse(String aggregationName, QueryParseContext context) throws IOException {
+        return PARSER.parse(context.parser(), new GeoBoundsAggregationBuilder(aggregationName), context);
+    }
 
     private boolean wrapLongitude = true;
 
     public GeoBoundsAggregationBuilder(String name) {
-        super(name, TYPE, ValuesSourceType.GEOPOINT, ValueType.GEOPOINT);
+        super(name, ValuesSourceType.GEOPOINT, ValueType.GEOPOINT);
     }
 
     /**
      * Read from a stream.
      */
     public GeoBoundsAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE, ValuesSourceType.GEOPOINT, ValueType.GEOPOINT);
+        super(in, ValuesSourceType.GEOPOINT, ValueType.GEOPOINT);
         wrapLongitude = in.readBoolean();
     }
 
@@ -74,9 +87,9 @@ public boolean wrapLongitude() {
     }
 
     @Override
-    protected GeoBoundsAggregatorFactory innerBuild(AggregationContext context, ValuesSourceConfig<ValuesSource.GeoPoint> config,
+    protected GeoBoundsAggregatorFactory innerBuild(SearchContext context, ValuesSourceConfig<ValuesSource.GeoPoint> config,
             AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
-        return new GeoBoundsAggregatorFactory(name, type, config, wrapLongitude, context, parent, subFactoriesBuilder, metaData);
+        return new GeoBoundsAggregatorFactory(name, config, wrapLongitude, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
@@ -97,7 +110,7 @@ protected boolean innerEquals(Object obj) {
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/GeoBoundsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/GeoBoundsAggregator.java
index 57e30fd58bd78..2083ea570d30a 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/GeoBoundsAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/GeoBoundsAggregator.java
@@ -32,8 +32,8 @@
 import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
 import org.elasticsearch.search.aggregations.metrics.MetricsAggregator;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -52,7 +52,7 @@ public final class GeoBoundsAggregator extends MetricsAggregator {
     DoubleArray negLefts;
     DoubleArray negRights;
 
-    protected GeoBoundsAggregator(String name, AggregationContext aggregationContext, Aggregator parent,
+    protected GeoBoundsAggregator(String name, SearchContext aggregationContext, Aggregator parent,
             ValuesSource.GeoPoint valuesSource, boolean wrapLongitude, List<PipelineAggregator> pipelineAggregators,
             Map<String, Object> metaData) throws IOException {
         super(name, aggregationContext, parent, pipelineAggregators, metaData);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/GeoBoundsAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/GeoBoundsAggregatorFactory.java
index e7420a9f1f779..e67ad49115ac6 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/GeoBoundsAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/GeoBoundsAggregatorFactory.java
@@ -22,12 +22,11 @@
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -37,10 +36,10 @@ public class GeoBoundsAggregatorFactory extends ValuesSourceAggregatorFactory<Va
 
     private final boolean wrapLongitude;
 
-    public GeoBoundsAggregatorFactory(String name, Type type, ValuesSourceConfig<ValuesSource.GeoPoint> config, boolean wrapLongitude,
-            AggregationContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
+    public GeoBoundsAggregatorFactory(String name, ValuesSourceConfig<ValuesSource.GeoPoint> config, boolean wrapLongitude,
+            SearchContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
             Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.wrapLongitude = wrapLongitude;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/GeoBoundsParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/GeoBoundsParser.java
deleted file mode 100644
index c42de23949b11..0000000000000
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/GeoBoundsParser.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.aggregations.metrics.geobounds;
-
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentParser.Token;
-import org.elasticsearch.search.aggregations.support.AbstractValuesSourceParser.GeoPointValuesSourceParser;
-import org.elasticsearch.search.aggregations.support.XContentParseContext;
-import org.elasticsearch.search.aggregations.support.ValueType;
-import org.elasticsearch.search.aggregations.support.ValuesSourceType;
-
-import java.io.IOException;
-import java.util.Map;
-
-public class GeoBoundsParser extends GeoPointValuesSourceParser {
-
-    public GeoBoundsParser() {
-        super(false, false);
-    }
-
-    @Override
-    protected GeoBoundsAggregationBuilder createFactory(String aggregationName, ValuesSourceType valuesSourceType,
-                                                        ValueType targetValueType, Map<ParseField, Object> otherOptions) {
-        GeoBoundsAggregationBuilder factory = new GeoBoundsAggregationBuilder(aggregationName);
-        Boolean wrapLongitude = (Boolean) otherOptions.get(GeoBoundsAggregator.WRAP_LONGITUDE_FIELD);
-        if (wrapLongitude != null) {
-            factory.wrapLongitude(wrapLongitude);
-        }
-        return factory;
-    }
-
-    @Override
-    protected boolean token(String aggregationName, String currentFieldName, Token token,
-                            XContentParseContext context, Map<ParseField, Object> otherOptions) throws IOException {
-        if (token == XContentParser.Token.VALUE_BOOLEAN) {
-            if (context.matchField(currentFieldName, GeoBoundsAggregator.WRAP_LONGITUDE_FIELD)) {
-                otherOptions.put(GeoBoundsAggregator.WRAP_LONGITUDE_FIELD, context.getParser().booleanValue());
-                return true;
-            }
-        }
-        return false;
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/InternalGeoBounds.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/InternalGeoBounds.java
index b999936693d06..92cf381d058dc 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/InternalGeoBounds.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/InternalGeoBounds.java
@@ -19,19 +19,26 @@
 
 package org.elasticsearch.search.aggregations.metrics.geobounds;
 
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.aggregations.InternalAggregation;
-import org.elasticsearch.search.aggregations.metrics.InternalMetricsAggregation;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
-public class InternalGeoBounds extends InternalMetricsAggregation implements GeoBounds {
+public class InternalGeoBounds extends InternalAggregation implements GeoBounds {
+
+    static final ParseField BOUNDS_FIELD = new ParseField("bounds");
+    static final ParseField TOP_LEFT_FIELD = new ParseField("top_left");
+    static final ParseField BOTTOM_RIGHT_FIELD = new ParseField("bottom_right");
+    static final ParseField LAT_FIELD = new ParseField("lat");
+    static final ParseField LON_FIELD = new ParseField("lon");
+
     private final double top;
     private final double bottom;
     private final double posLeft;
@@ -82,7 +89,7 @@ protected void doWriteTo(StreamOutput out) throws IOException {
     public String getWriteableName() {
         return GeoBoundsAggregationBuilder.NAME;
     }
-    
+
     @Override
     public InternalAggregation doReduce(List<InternalAggregation> aggregations, ReduceContext reduceContext) {
         double top = Double.NEGATIVE_INFINITY;
@@ -170,14 +177,14 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th
         GeoPoint topLeft = topLeft();
         GeoPoint bottomRight = bottomRight();
         if (topLeft != null) {
-            builder.startObject("bounds");
-            builder.startObject("top_left");
-            builder.field("lat", topLeft.lat());
-            builder.field("lon", topLeft.lon());
+            builder.startObject(BOUNDS_FIELD.getPreferredName());
+            builder.startObject(TOP_LEFT_FIELD.getPreferredName());
+            builder.field(LAT_FIELD.getPreferredName(), topLeft.lat());
+            builder.field(LON_FIELD.getPreferredName(), topLeft.lon());
             builder.endObject();
-            builder.startObject("bottom_right");
-            builder.field("lat", bottomRight.lat());
-            builder.field("lon", bottomRight.lon());
+            builder.startObject(BOTTOM_RIGHT_FIELD.getPreferredName());
+            builder.field(LAT_FIELD.getPreferredName(), bottomRight.lat());
+            builder.field(LON_FIELD.getPreferredName(), bottomRight.lon());
             builder.endObject();
             builder.endObject();
         }
@@ -187,21 +194,21 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th
     private static class BoundingBox {
         private final GeoPoint topLeft;
         private final GeoPoint bottomRight;
-        
-        public BoundingBox(GeoPoint topLeft, GeoPoint bottomRight) {
+
+        BoundingBox(GeoPoint topLeft, GeoPoint bottomRight) {
             this.topLeft = topLeft;
             this.bottomRight = bottomRight;
         }
-        
+
         public GeoPoint topLeft() {
             return topLeft;
         }
-        
+
         public GeoPoint bottomRight() {
             return bottomRight;
         }
     }
-    
+
     private BoundingBox resolveBoundingBox() {
         if (Double.isInfinite(top)) {
             return null;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/ParsedGeoBounds.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/ParsedGeoBounds.java
new file mode 100644
index 0000000000000..04d2b2448d2e2
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/ParsedGeoBounds.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.geobounds;
+
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.geo.GeoPoint;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.ParsedAggregation;
+
+import java.io.IOException;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.search.aggregations.metrics.geobounds.InternalGeoBounds.BOTTOM_RIGHT_FIELD;
+import static org.elasticsearch.search.aggregations.metrics.geobounds.InternalGeoBounds.BOUNDS_FIELD;
+import static org.elasticsearch.search.aggregations.metrics.geobounds.InternalGeoBounds.LAT_FIELD;
+import static org.elasticsearch.search.aggregations.metrics.geobounds.InternalGeoBounds.LON_FIELD;
+import static org.elasticsearch.search.aggregations.metrics.geobounds.InternalGeoBounds.TOP_LEFT_FIELD;
+
+public class ParsedGeoBounds extends ParsedAggregation implements GeoBounds {
+    private GeoPoint topLeft;
+    private GeoPoint bottomRight;
+
+    @Override
+    public String getType() {
+        return GeoBoundsAggregationBuilder.NAME;
+    }
+
+    @Override
+    public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+        if (topLeft != null) {
+            builder.startObject("bounds");
+            builder.startObject("top_left");
+            builder.field("lat", topLeft.getLat());
+            builder.field("lon", topLeft.getLon());
+            builder.endObject();
+            builder.startObject("bottom_right");
+            builder.field("lat", bottomRight.getLat());
+            builder.field("lon", bottomRight.getLon());
+            builder.endObject();
+            builder.endObject();
+        }
+        return builder;
+    }
+
+    @Override
+    public GeoPoint topLeft() {
+        return topLeft;
+    }
+
+    @Override
+    public GeoPoint bottomRight() {
+        return bottomRight;
+    }
+
+    private static final ObjectParser<ParsedGeoBounds, Void> PARSER = new ObjectParser<>(ParsedGeoBounds.class.getSimpleName(), true,
+            ParsedGeoBounds::new);
+
+    private static final ConstructingObjectParser<Tuple<GeoPoint, GeoPoint>, Void> BOUNDS_PARSER =
+            new ConstructingObjectParser<>(ParsedGeoBounds.class.getSimpleName() + "_BOUNDS", true,
+                    args -> new Tuple<>((GeoPoint) args[0], (GeoPoint) args[1]));
+
+    private static final ObjectParser<GeoPoint, Void> GEO_POINT_PARSER = new ObjectParser<>(
+            ParsedGeoBounds.class.getSimpleName() + "_POINT", true, GeoPoint::new);
+
+    static {
+        declareAggregationFields(PARSER);
+        PARSER.declareObject((agg, bbox) -> {
+            agg.topLeft = bbox.v1();
+            agg.bottomRight = bbox.v2();
+        }, BOUNDS_PARSER, BOUNDS_FIELD);
+
+        BOUNDS_PARSER.declareObject(constructorArg(), GEO_POINT_PARSER, TOP_LEFT_FIELD);
+        BOUNDS_PARSER.declareObject(constructorArg(), GEO_POINT_PARSER, BOTTOM_RIGHT_FIELD);
+
+        GEO_POINT_PARSER.declareDouble(GeoPoint::resetLat, LAT_FIELD);
+        GEO_POINT_PARSER.declareDouble(GeoPoint::resetLon, LON_FIELD);
+    }
+
+    public static ParsedGeoBounds fromXContent(XContentParser parser, final String name) {
+        ParsedGeoBounds geoBounds = PARSER.apply(parser, null);
+        geoBounds.setName(name);
+        return geoBounds;
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregationBuilder.java
index 3c5208f873856..8e173e65923ee 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregationBuilder.java
@@ -21,33 +21,45 @@
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 
 public class GeoCentroidAggregationBuilder
         extends ValuesSourceAggregationBuilder.LeafOnly<ValuesSource.GeoPoint, GeoCentroidAggregationBuilder> {
     public static final String NAME = "geo_centroid";
-    public static final Type TYPE = new Type(NAME);
+
+    private static final ObjectParser<GeoCentroidAggregationBuilder, QueryParseContext> PARSER;
+    static {
+        PARSER = new ObjectParser<>(GeoCentroidAggregationBuilder.NAME);
+        ValuesSourceParserHelper.declareGeoFields(PARSER, true, false);
+    }
+
+    public static AggregationBuilder parse(String aggregationName, QueryParseContext context) throws IOException {
+        return PARSER.parse(context.parser(), new GeoCentroidAggregationBuilder(aggregationName), context);
+    }
 
     public GeoCentroidAggregationBuilder(String name) {
-        super(name, TYPE, ValuesSourceType.GEOPOINT, ValueType.GEOPOINT);
+        super(name, ValuesSourceType.GEOPOINT, ValueType.GEOPOINT);
     }
 
     /**
      * Read from a stream.
      */
     public GeoCentroidAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE, ValuesSourceType.GEOPOINT, ValueType.GEOPOINT);
+        super(in, ValuesSourceType.GEOPOINT, ValueType.GEOPOINT);
     }
 
     @Override
@@ -56,9 +68,9 @@ protected void innerWriteTo(StreamOutput out) {
     }
 
     @Override
-    protected GeoCentroidAggregatorFactory innerBuild(AggregationContext context, ValuesSourceConfig<ValuesSource.GeoPoint> config,
+    protected GeoCentroidAggregatorFactory innerBuild(SearchContext context, ValuesSourceConfig<ValuesSource.GeoPoint> config,
             AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
-        return new GeoCentroidAggregatorFactory(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        return new GeoCentroidAggregatorFactory(name, config, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
@@ -77,7 +89,7 @@ protected boolean innerEquals(Object obj) {
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregator.java
index ec838e7dd41ad..9d537030fa948 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregator.java
@@ -32,8 +32,8 @@
 import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
 import org.elasticsearch.search.aggregations.metrics.MetricsAggregator;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -47,10 +47,10 @@ public final class GeoCentroidAggregator extends MetricsAggregator {
     LongArray centroids;
     LongArray counts;
 
-    protected GeoCentroidAggregator(String name, AggregationContext aggregationContext, Aggregator parent,
+    protected GeoCentroidAggregator(String name, SearchContext context, Aggregator parent,
                                     ValuesSource.GeoPoint valuesSource, List<PipelineAggregator> pipelineAggregators,
                                     Map<String, Object> metaData) throws IOException {
-        super(name, aggregationContext, parent, pipelineAggregators, metaData);
+        super(name, context, parent, pipelineAggregators, metaData);
         this.valuesSource = valuesSource;
         if (valuesSource != null) {
             final BigArrays bigArrays = context.bigArrays();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregatorFactory.java
index 6fa2d28856efd..c21999d3fb48e 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregatorFactory.java
@@ -22,12 +22,11 @@
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -35,10 +34,10 @@
 
 public class GeoCentroidAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource.GeoPoint, GeoCentroidAggregatorFactory> {
 
-    public GeoCentroidAggregatorFactory(String name, Type type, ValuesSourceConfig<ValuesSource.GeoPoint> config,
-            AggregationContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
+    public GeoCentroidAggregatorFactory(String name, ValuesSourceConfig<ValuesSource.GeoPoint> config,
+            SearchContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
             Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidParser.java
deleted file mode 100644
index 8e88a11c6b60a..0000000000000
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidParser.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.aggregations.metrics.geocentroid;
-
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.xcontent.XContentParser.Token;
-import org.elasticsearch.search.aggregations.support.AbstractValuesSourceParser.GeoPointValuesSourceParser;
-import org.elasticsearch.search.aggregations.support.XContentParseContext;
-import org.elasticsearch.search.aggregations.support.ValueType;
-import org.elasticsearch.search.aggregations.support.ValuesSourceType;
-
-import java.io.IOException;
-import java.util.Map;
-
-/**
- * Parser class for {@link org.elasticsearch.search.aggregations.metrics.geocentroid.GeoCentroidAggregator}
- */
-public class GeoCentroidParser extends GeoPointValuesSourceParser {
-
-    public GeoCentroidParser() {
-        super(true, false);
-    }
-
-    @Override
-    protected boolean token(String aggregationName, String currentFieldName, Token token,
-                            XContentParseContext context, Map<ParseField, Object> otherOptions) throws IOException {
-        return false;
-    }
-
-    @Override
-    protected GeoCentroidAggregationBuilder createFactory(String aggregationName, ValuesSourceType valuesSourceType,
-                                                          ValueType targetValueType, Map<ParseField, Object> otherOptions) {
-        return new GeoCentroidAggregationBuilder(aggregationName);
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroid.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroid.java
index 06d9d369029af..9beabf12887f0 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroid.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroid.java
@@ -20,12 +20,12 @@
 package org.elasticsearch.search.aggregations.metrics.geocentroid;
 
 import org.apache.lucene.spatial.geopoint.document.GeoPointField;
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.aggregations.InternalAggregation;
-import org.elasticsearch.search.aggregations.metrics.InternalMetricsAggregation;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 
 import java.io.IOException;
@@ -35,9 +35,9 @@
 /**
  * Serialization and merge logic for {@link GeoCentroidAggregator}.
  */
-public class InternalGeoCentroid extends InternalMetricsAggregation implements GeoCentroid {
-    protected final GeoPoint centroid;
-    protected final long count;
+public class InternalGeoCentroid extends InternalAggregation implements GeoCentroid {
+    private final GeoPoint centroid;
+    private final long count;
 
     public InternalGeoCentroid(String name, GeoPoint centroid, long count, List<PipelineAggregator>
             pipelineAggregators, Map<String, Object> metaData) {
@@ -124,6 +124,8 @@ public Object getProperty(List<String> path) {
                     return centroid.lat();
                 case "lon":
                     return centroid.lon();
+                case "count":
+                    return count;
                 default:
                     throw new IllegalArgumentException("Found unknown path element [" + coordinate + "] in [" + getName() + "]");
             }
@@ -133,14 +135,18 @@ public Object getProperty(List<String> path) {
     }
 
     static class Fields {
-        public static final String CENTROID = "location";
+        static final ParseField CENTROID = new ParseField("location");
+        static final ParseField COUNT = new ParseField("count");
+        static final ParseField CENTROID_LAT = new ParseField("lat");
+        static final ParseField CENTROID_LON = new ParseField("lon");
     }
 
     @Override
     public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
         if (centroid != null) {
-            builder.startObject(Fields.CENTROID).field("lat", centroid.lat()).field("lon", centroid.lon()).endObject();
+            builder.startObject(Fields.CENTROID.getPreferredName()).field("lat", centroid.lat()).field("lon", centroid.lon()).endObject();
         }
+        builder.field(Fields.COUNT.getPreferredName(), count);
         return builder;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/ParsedGeoCentroid.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/ParsedGeoCentroid.java
new file mode 100644
index 0000000000000..7ce1f5d86feb3
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/ParsedGeoCentroid.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.geocentroid;
+
+import org.elasticsearch.common.geo.GeoPoint;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.ParsedAggregation;
+import org.elasticsearch.search.aggregations.metrics.geocentroid.InternalGeoCentroid.Fields;
+
+import java.io.IOException;
+
+/**
+ * Serialization and merge logic for {@link GeoCentroidAggregator}.
+ */
+public class ParsedGeoCentroid extends ParsedAggregation implements GeoCentroid {
+    private GeoPoint centroid;
+    private long count;
+
+    @Override
+    public GeoPoint centroid() {
+        return centroid;
+    }
+
+    @Override
+    public long count() {
+        return count;
+    }
+
+    @Override
+    public String getType() {
+        return GeoCentroidAggregationBuilder.NAME;
+    }
+
+    @Override
+    public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+        if (centroid != null) {
+            builder.startObject(Fields.CENTROID.getPreferredName());
+            {
+                builder.field(Fields.CENTROID_LAT.getPreferredName(), centroid.lat());
+                builder.field(Fields.CENTROID_LON.getPreferredName(), centroid.lon());
+            }
+            builder.endObject();
+        }
+        builder.field(Fields.COUNT.getPreferredName(), count);
+        return builder;
+    }
+
+    private static final ObjectParser<ParsedGeoCentroid, Void> PARSER = new ObjectParser<>(ParsedGeoCentroid.class.getSimpleName(), true,
+            ParsedGeoCentroid::new);
+
+    private static final ObjectParser<GeoPoint, Void> GEO_POINT_PARSER = new ObjectParser<>(
+            ParsedGeoCentroid.class.getSimpleName() + "_POINT", true, GeoPoint::new);
+
+    static {
+        declareAggregationFields(PARSER);
+        PARSER.declareObject((agg, centroid) -> agg.centroid = centroid, GEO_POINT_PARSER, Fields.CENTROID);
+        PARSER.declareLong((agg, count) -> agg.count = count, Fields.COUNT);
+
+        GEO_POINT_PARSER.declareDouble(GeoPoint::resetLat, Fields.CENTROID_LAT);
+        GEO_POINT_PARSER.declareDouble(GeoPoint::resetLon, Fields.CENTROID_LON);
+    }
+
+    public static ParsedGeoCentroid fromXContent(XContentParser parser, final String name) {
+        ParsedGeoCentroid geoCentroid = PARSER.apply(parser, null);
+        geoCentroid.setName(name);
+        return geoCentroid;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/InternalMax.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/InternalMax.java
index c9d045a7235f1..0a51142ca056b 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/InternalMax.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/InternalMax.java
@@ -82,9 +82,9 @@ public InternalMax doReduce(List<InternalAggregation> aggregations, ReduceContex
     @Override
     public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
         boolean hasValue = !Double.isInfinite(max);
-        builder.field(CommonFields.VALUE, hasValue ? max : null);
+        builder.field(CommonFields.VALUE.getPreferredName(), hasValue ? max : null);
         if (hasValue && format != DocValueFormat.RAW) {
-            builder.field(CommonFields.VALUE_AS_STRING, format.format(max));
+            builder.field(CommonFields.VALUE_AS_STRING.getPreferredName(), format.format(max));
         }
         return builder;
     }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxAggregationBuilder.java
index 117b360992a6e..dafad9dacb59f 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxAggregationBuilder.java
@@ -21,33 +21,45 @@
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 
 public class MaxAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOnly<ValuesSource.Numeric, MaxAggregationBuilder> {
     public static final String NAME = "max";
-    public static final Type TYPE = new Type(NAME);
+
+    private static final ObjectParser<MaxAggregationBuilder, QueryParseContext> PARSER;
+    static {
+        PARSER = new ObjectParser<>(MaxAggregationBuilder.NAME);
+        ValuesSourceParserHelper.declareNumericFields(PARSER, true, true, false);
+    }
+
+    public static AggregationBuilder parse(String aggregationName, QueryParseContext context) throws IOException {
+        return PARSER.parse(context.parser(), new MaxAggregationBuilder(aggregationName), context);
+    }
 
     public MaxAggregationBuilder(String name) {
-        super(name, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(name, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     /**
      * Read from a stream.
      */
     public MaxAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(in, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     @Override
@@ -56,9 +68,9 @@ protected void innerWriteTo(StreamOutput out) {
     }
 
     @Override
-    protected MaxAggregatorFactory innerBuild(AggregationContext context, ValuesSourceConfig<Numeric> config,
+    protected MaxAggregatorFactory innerBuild(SearchContext context, ValuesSourceConfig<Numeric> config,
             AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
-        return new MaxAggregatorFactory(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        return new MaxAggregatorFactory(name, config, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
@@ -77,7 +89,7 @@ protected boolean innerEquals(Object obj) {
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxAggregator.java
index 73504f9a8f462..7d4f837e00c88 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxAggregator.java
@@ -24,16 +24,16 @@
 import org.elasticsearch.common.util.DoubleArray;
 import org.elasticsearch.index.fielddata.NumericDoubleValues;
 import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
-import org.elasticsearch.search.MultiValueMode;
 import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.MultiValueMode;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.LeafBucketCollector;
 import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
 import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregator;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -50,7 +50,7 @@ public class MaxAggregator extends NumericMetricsAggregator.SingleValue {
     DoubleArray maxes;
 
     public MaxAggregator(String name, ValuesSource.Numeric valuesSource, DocValueFormat formatter,
- AggregationContext context,
+            SearchContext context,
             Aggregator parent, List<PipelineAggregator> pipelineAggregators,
             Map<String, Object> metaData) throws IOException {
         super(name, context, parent, pipelineAggregators, metaData);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxAggregatorFactory.java
index 2a9bdd9d8dc32..aedba76e0c7d1 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxAggregatorFactory.java
@@ -22,13 +22,12 @@
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -36,9 +35,9 @@
 
 public class MaxAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource.Numeric, MaxAggregatorFactory> {
 
-    public MaxAggregatorFactory(String name, Type type, ValuesSourceConfig<Numeric> config, AggregationContext context,
+    public MaxAggregatorFactory(String name, ValuesSourceConfig<Numeric> config, SearchContext context,
             AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxParser.java
deleted file mode 100644
index f0290e93fa9e1..0000000000000
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxParser.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.search.aggregations.metrics.max;
-
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.search.aggregations.support.AbstractValuesSourceParser.NumericValuesSourceParser;
-import org.elasticsearch.search.aggregations.support.XContentParseContext;
-import org.elasticsearch.search.aggregations.support.ValueType;
-import org.elasticsearch.search.aggregations.support.ValuesSourceType;
-
-import java.io.IOException;
-import java.util.Map;
-
-/**
- *
- */
-public class MaxParser extends NumericValuesSourceParser {
-
-    public MaxParser() {
-        super(true, true, false);
-    }
-
-    @Override
-    protected boolean token(String aggregationName, String currentFieldName, XContentParser.Token token,
-                            XContentParseContext context, Map<ParseField, Object> otherOptions) throws IOException {
-        return false;
-    }
-
-    @Override
-    protected MaxAggregationBuilder createFactory(String aggregationName, ValuesSourceType valuesSourceType,
-                                                  ValueType targetValueType, Map<ParseField, Object> otherOptions) {
-        return new MaxAggregationBuilder(aggregationName);
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/ParsedMax.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/ParsedMax.java
new file mode 100644
index 0000000000000..f6a3190cd04d4
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/ParsedMax.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.max;
+
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.metrics.ParsedSingleValueNumericMetricsAggregation;
+
+import java.io.IOException;
+
+public class ParsedMax extends ParsedSingleValueNumericMetricsAggregation implements Max {
+
+    @Override
+    public double getValue() {
+        return value();
+    }
+
+    @Override
+    public String getType() {
+        return MaxAggregationBuilder.NAME;
+    }
+
+    @Override
+    protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+        boolean hasValue = !Double.isInfinite(value);
+        builder.field(CommonFields.VALUE.getPreferredName(), hasValue ? value : null);
+        if (hasValue && valueAsString != null) {
+            builder.field(CommonFields.VALUE_AS_STRING.getPreferredName(), valueAsString);
+        }
+        return builder;
+    }
+
+    private static final ObjectParser<ParsedMax, Void> PARSER = new ObjectParser<>(ParsedMax.class.getSimpleName(), true, ParsedMax::new);
+
+    static {
+        declareSingleValueFields(PARSER, Double.NEGATIVE_INFINITY);
+    }
+
+    public static ParsedMax fromXContent(XContentParser parser, final String name) {
+        ParsedMax max = PARSER.apply(parser, null);
+        max.setName(name);
+        return max;
+    }
+}
\ No newline at end of file
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/InternalMin.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/InternalMin.java
index 09f9c3915fdfd..9f9f6b3d31928 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/InternalMin.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/InternalMin.java
@@ -82,9 +82,9 @@ public InternalMin doReduce(List<InternalAggregation> aggregations, ReduceContex
     @Override
     public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
         boolean hasValue = !Double.isInfinite(min);
-        builder.field(CommonFields.VALUE, hasValue ? min : null);
+        builder.field(CommonFields.VALUE.getPreferredName(), hasValue ? min : null);
         if (hasValue && format != DocValueFormat.RAW) {
-            builder.field(CommonFields.VALUE_AS_STRING, format.format(min));
+            builder.field(CommonFields.VALUE_AS_STRING.getPreferredName(), format.format(min));
         }
         return builder;
     }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregationBuilder.java
index 248b83db0c936..0f85748416edc 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregationBuilder.java
@@ -21,33 +21,46 @@
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
+import org.elasticsearch.search.aggregations.metrics.avg.AvgAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 
 public class MinAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOnly<ValuesSource.Numeric, MinAggregationBuilder> {
     public static final String NAME = "min";
-    private static final Type TYPE = new Type(NAME);
+
+    private static final ObjectParser<MinAggregationBuilder, QueryParseContext> PARSER;
+    static {
+        PARSER = new ObjectParser<>(AvgAggregationBuilder.NAME);
+        ValuesSourceParserHelper.declareNumericFields(PARSER, true, true, false);
+    }
+
+    public static AggregationBuilder parse(String aggregationName, QueryParseContext context) throws IOException {
+        return PARSER.parse(context.parser(), new MinAggregationBuilder(aggregationName), context);
+    }
 
     public MinAggregationBuilder(String name) {
-        super(name, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(name, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     /**
      * Read from a stream.
      */
     public MinAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(in, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     @Override
@@ -56,9 +69,9 @@ protected void innerWriteTo(StreamOutput out) {
     }
 
     @Override
-    protected MinAggregatorFactory innerBuild(AggregationContext context, ValuesSourceConfig<Numeric> config,
+    protected MinAggregatorFactory innerBuild(SearchContext context, ValuesSourceConfig<Numeric> config,
             AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
-        return new MinAggregatorFactory(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        return new MinAggregatorFactory(name, config, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
@@ -77,7 +90,7 @@ protected boolean innerEquals(Object obj) {
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregator.java
index a32379b7d10a7..6626b7755d8d8 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregator.java
@@ -24,16 +24,16 @@
 import org.elasticsearch.common.util.DoubleArray;
 import org.elasticsearch.index.fielddata.NumericDoubleValues;
 import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
-import org.elasticsearch.search.MultiValueMode;
 import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.MultiValueMode;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.LeafBucketCollector;
 import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
 import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregator;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -50,7 +50,7 @@ public class MinAggregator extends NumericMetricsAggregator.SingleValue {
     DoubleArray mins;
 
     public MinAggregator(String name, ValuesSource.Numeric valuesSource, DocValueFormat formatter,
-            AggregationContext context, Aggregator parent, List<PipelineAggregator> pipelineAggregators,
+            SearchContext context, Aggregator parent, List<PipelineAggregator> pipelineAggregators,
             Map<String, Object> metaData) throws IOException {
         super(name, context, parent, pipelineAggregators, metaData);
         this.valuesSource = valuesSource;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregatorFactory.java
index 1bc01d1e9e35c..8f5538fb7a2bb 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregatorFactory.java
@@ -22,13 +22,12 @@
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -36,9 +35,9 @@
 
 public class MinAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource.Numeric, MinAggregatorFactory> {
 
-    public MinAggregatorFactory(String name, Type type, ValuesSourceConfig<Numeric> config, AggregationContext context,
+    public MinAggregatorFactory(String name, ValuesSourceConfig<Numeric> config, SearchContext context,
             AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinParser.java
deleted file mode 100644
index 4381ca4189926..0000000000000
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinParser.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.search.aggregations.metrics.min;
-
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.xcontent.XContentParser.Token;
-import org.elasticsearch.search.aggregations.support.AbstractValuesSourceParser.NumericValuesSourceParser;
-import org.elasticsearch.search.aggregations.support.XContentParseContext;
-import org.elasticsearch.search.aggregations.support.ValueType;
-import org.elasticsearch.search.aggregations.support.ValuesSourceType;
-
-import java.io.IOException;
-import java.util.Map;
-
-/**
- *
- */
-public class MinParser extends NumericValuesSourceParser {
-
-    public MinParser() {
-        super(true, true, false);
-    }
-
-    @Override
-    protected boolean token(String aggregationName, String currentFieldName, Token token,
-                            XContentParseContext context, Map<ParseField, Object> otherOptions) throws IOException {
-        return false;
-    }
-
-    @Override
-    protected MinAggregationBuilder createFactory(String aggregationName, ValuesSourceType valuesSourceType,
-                                                  ValueType targetValueType, Map<ParseField, Object> otherOptions) {
-        return new MinAggregationBuilder(aggregationName);
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/ParsedMin.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/ParsedMin.java
new file mode 100644
index 0000000000000..9b214bb346201
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/ParsedMin.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.min;
+
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.metrics.ParsedSingleValueNumericMetricsAggregation;
+
+import java.io.IOException;
+
+public class ParsedMin extends ParsedSingleValueNumericMetricsAggregation implements Min {
+
+    @Override
+    public double getValue() {
+        return value();
+    }
+
+    @Override
+    public String getType() {
+        return MinAggregationBuilder.NAME;
+    }
+
+    @Override
+    protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+        boolean hasValue = !Double.isInfinite(value);
+        builder.field(CommonFields.VALUE.getPreferredName(), hasValue ? value : null);
+        if (hasValue && valueAsString != null) {
+            builder.field(CommonFields.VALUE_AS_STRING.getPreferredName(), valueAsString);
+        }
+        return builder;
+    }
+
+    private static final ObjectParser<ParsedMin, Void> PARSER = new ObjectParser<>(ParsedMin.class.getSimpleName(), true, ParsedMin::new);
+
+    static {
+        declareSingleValueFields(PARSER, Double.POSITIVE_INFINITY);
+    }
+
+    public static ParsedMin fromXContent(XContentParser parser, final String name) {
+        ParsedMin min = PARSER.apply(parser, null);
+        min.setName(name);
+        return min;
+    }
+}
\ No newline at end of file
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractPercentilesParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractPercentilesParser.java
deleted file mode 100644
index 053a415c9719e..0000000000000
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractPercentilesParser.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.aggregations.metrics.percentiles;
-
-import com.carrotsearch.hppc.DoubleArrayList;
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentParser.Token;
-import org.elasticsearch.search.aggregations.support.AbstractValuesSourceParser.NumericValuesSourceParser;
-import org.elasticsearch.search.aggregations.support.XContentParseContext;
-import org.elasticsearch.search.aggregations.support.ValueType;
-import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
-import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
-import org.elasticsearch.search.aggregations.support.ValuesSourceType;
-
-import java.io.IOException;
-import java.util.Map;
-
-public abstract class AbstractPercentilesParser extends NumericValuesSourceParser {
-
-    public static final ParseField KEYED_FIELD = new ParseField("keyed");
-    public static final ParseField METHOD_FIELD = new ParseField("method");
-    public static final ParseField COMPRESSION_FIELD = new ParseField("compression");
-    public static final ParseField NUMBER_SIGNIFICANT_DIGITS_FIELD = new ParseField("number_of_significant_value_digits");
-
-    public AbstractPercentilesParser(boolean formattable) {
-        super(true, formattable, false);
-    }
-
-    @Override
-    protected boolean token(String aggregationName, String currentFieldName, Token token,
-                            XContentParseContext context, Map<ParseField, Object> otherOptions) throws IOException {
-        XContentParser parser = context.getParser();
-        if (token == XContentParser.Token.START_ARRAY) {
-            if (context.matchField(currentFieldName, keysField())) {
-                DoubleArrayList values = new DoubleArrayList(10);
-                while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
-                    double value = parser.doubleValue();
-                    values.add(value);
-                }
-                double[] keys = values.toArray();
-                otherOptions.put(keysField(), keys);
-                return true;
-            } else {
-                return false;
-            }
-        } else if (token == XContentParser.Token.VALUE_BOOLEAN) {
-            if (context.matchField(currentFieldName, KEYED_FIELD)) {
-                boolean keyed = parser.booleanValue();
-                otherOptions.put(KEYED_FIELD, keyed);
-                return true;
-            } else {
-                return false;
-            }
-        } else if (token == XContentParser.Token.START_OBJECT) {
-            PercentilesMethod method = PercentilesMethod.resolveFromName(currentFieldName);
-            if (method == null) {
-                return false;
-            } else {
-                otherOptions.put(METHOD_FIELD, method);
-                switch (method) {
-                case TDIGEST:
-                    while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                        if (token == XContentParser.Token.FIELD_NAME) {
-                            currentFieldName = parser.currentName();
-                        } else if (token == XContentParser.Token.VALUE_NUMBER) {
-                            if (context.matchField(currentFieldName, COMPRESSION_FIELD)) {
-                                double compression = parser.doubleValue();
-                                otherOptions.put(COMPRESSION_FIELD, compression);
-                            } else {
-                                return false;
-                            }
-                        } else {
-                            return false;
-                        }
-                    }
-                    break;
-                case HDR:
-                    while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                        if (token == XContentParser.Token.FIELD_NAME) {
-                            currentFieldName = parser.currentName();
-                        } else if (token == XContentParser.Token.VALUE_NUMBER) {
-                            if (context.matchField(currentFieldName, NUMBER_SIGNIFICANT_DIGITS_FIELD)) {
-                                int numberOfSignificantValueDigits = parser.intValue();
-                                otherOptions.put(NUMBER_SIGNIFICANT_DIGITS_FIELD, numberOfSignificantValueDigits);
-                            } else {
-                                return false;
-                            }
-                        } else {
-                            return false;
-                        }
-                    }
-                    break;
-                }
-                return true;
-            }
-        }
-        return false;
-    }
-
-    @Override
-    protected ValuesSourceAggregationBuilder<Numeric, ?> createFactory(String aggregationName, ValuesSourceType valuesSourceType,
-                                                                       ValueType targetValueType, Map<ParseField, Object> otherOptions) {
-        PercentilesMethod method = (PercentilesMethod) otherOptions.getOrDefault(METHOD_FIELD, PercentilesMethod.TDIGEST);
-
-        double[] cdfValues = (double[]) otherOptions.get(keysField());
-        Double compression = (Double) otherOptions.get(COMPRESSION_FIELD);
-        Integer numberOfSignificantValueDigits = (Integer) otherOptions.get(NUMBER_SIGNIFICANT_DIGITS_FIELD);
-        Boolean keyed = (Boolean) otherOptions.get(KEYED_FIELD);
-        return buildFactory(aggregationName, cdfValues, method, compression, numberOfSignificantValueDigits, keyed);
-    }
-
-    protected abstract ValuesSourceAggregationBuilder<Numeric, ?> buildFactory(String aggregationName, double[] cdfValues,
-                                                                               PercentilesMethod method,
-                                                                               Double compression,
-                                                                               Integer numberOfSignificantValueDigits, Boolean keyed);
-
-    protected abstract ParseField keysField();
-
-}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/InternalPercentile.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/InternalPercentile.java
deleted file mode 100644
index bb8876d82fded..0000000000000
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/InternalPercentile.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.aggregations.metrics.percentiles;
-
-public class InternalPercentile implements Percentile {
-
-    private final double percent;
-    private final double value;
-
-    public InternalPercentile(double percent, double value) {
-        this.percent = percent;
-        this.value = value;
-    }
-
-    @Override
-    public double getPercent() {
-        return percent;
-    }
-
-    @Override
-    public double getValue() {
-        return value;
-    }
-}
\ No newline at end of file
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/ParsedPercentileRanks.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/ParsedPercentileRanks.java
new file mode 100644
index 0000000000000..2c80d0328dd86
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/ParsedPercentileRanks.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.percentiles;
+
+public abstract class ParsedPercentileRanks extends ParsedPercentiles implements PercentileRanks {
+
+    @Override
+    public double percent(double value) {
+        return getPercentile(value);
+    }
+
+    @Override
+    public String percentAsString(double value) {
+        return getPercentileAsString(value);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/ParsedPercentiles.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/ParsedPercentiles.java
new file mode 100644
index 0000000000000..3f56b21dcd8a0
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/ParsedPercentiles.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.percentiles;
+
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.ParsedAggregation;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+public abstract class ParsedPercentiles extends ParsedAggregation implements Iterable<Percentile>  {
+
+    protected final Map<Double, Double> percentiles = new LinkedHashMap<>();
+    protected final Map<Double, String> percentilesAsString = new HashMap<>();
+
+    private boolean keyed;
+
+    void addPercentile(Double key, Double value) {
+        percentiles.put(key, value);
+    }
+
+    void addPercentileAsString(Double key, String valueAsString) {
+        percentilesAsString.put(key, valueAsString);
+    }
+
+    protected Double getPercentile(double percent) {
+        if (percentiles.isEmpty()) {
+            return Double.NaN;
+        }
+        return percentiles.get(percent);
+    }
+
+    protected String getPercentileAsString(double percent) {
+        String valueAsString = percentilesAsString.get(percent);
+        if (valueAsString != null) {
+            return valueAsString;
+        }
+        Double value = getPercentile(percent);
+        if (value != null) {
+            return Double.toString(value);
+        }
+        return null;
+    }
+
+    void setKeyed(boolean keyed) {
+        this.keyed = keyed;
+    }
+
+    @Override
+    public Iterator<Percentile> iterator() {
+        return new Iterator<Percentile>() {
+            final Iterator<Map.Entry<Double, Double>> iterator = percentiles.entrySet().iterator();
+            @Override
+            public boolean hasNext() {
+                return iterator.hasNext();
+            }
+
+            @Override
+            public Percentile next() {
+                Map.Entry<Double, Double> next = iterator.next();
+                return new Percentile(next.getKey(), next.getValue());
+            }
+        };
+    }
+
+    @Override
+    protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+        final boolean valuesAsString = (percentilesAsString.isEmpty() == false);
+        if (keyed) {
+            builder.startObject(CommonFields.VALUES.getPreferredName());
+            for (Map.Entry<Double, Double> percentile : percentiles.entrySet()) {
+                Double key = percentile.getKey();
+                builder.field(String.valueOf(key), percentile.getValue());
+
+                if (valuesAsString) {
+                    builder.field(key + "_as_string", getPercentileAsString(key));
+                }
+            }
+            builder.endObject();
+        } else {
+            builder.startArray(CommonFields.VALUES.getPreferredName());
+            for (Map.Entry<Double, Double> percentile : percentiles.entrySet()) {
+                Double key = percentile.getKey();
+                builder.startObject();
+                {
+                    builder.field(CommonFields.KEY.getPreferredName(), key);
+                    builder.field(CommonFields.VALUE.getPreferredName(), percentile.getValue());
+                    if (valuesAsString) {
+                        builder.field(CommonFields.VALUE_AS_STRING.getPreferredName(), getPercentileAsString(key));
+                    }
+                }
+                builder.endObject();
+            }
+            builder.endArray();
+        }
+        return builder;
+    }
+
+    protected static void declarePercentilesFields(ObjectParser<? extends ParsedPercentiles, Void> objectParser) {
+        ParsedAggregation.declareAggregationFields(objectParser);
+
+        objectParser.declareField((parser, aggregation, context) -> {
+            XContentParser.Token token = parser.currentToken();
+            if (token == XContentParser.Token.START_OBJECT) {
+                aggregation.setKeyed(true);
+                while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                    if (token.isValue()) {
+                        if (token == XContentParser.Token.VALUE_NUMBER) {
+                            aggregation.addPercentile(Double.valueOf(parser.currentName()), parser.doubleValue());
+                        } else if (token == XContentParser.Token.VALUE_STRING) {
+                            int i = parser.currentName().indexOf("_as_string");
+                            if (i > 0) {
+                                double key = Double.valueOf(parser.currentName().substring(0, i));
+                                aggregation.addPercentileAsString(key, parser.text());
+                            } else {
+                                aggregation.addPercentile(Double.valueOf(parser.currentName()), Double.valueOf(parser.text()));
+                            }
+                        }
+                    } else if (token == XContentParser.Token.VALUE_NULL) {
+                        aggregation.addPercentile(Double.valueOf(parser.currentName()), Double.NaN);
+                    } else {
+                        parser.skipChildren(); // skip potential inner objects and arrays for forward compatibility
+                    }
+                }
+            } else if (token == XContentParser.Token.START_ARRAY) {
+                aggregation.setKeyed(false);
+
+                String currentFieldName = null;
+                while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                    Double key = null;
+                    Double value = null;
+                    String valueAsString = null;
+
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                        if (token == XContentParser.Token.FIELD_NAME) {
+                            currentFieldName = parser.currentName();
+                        } else if (token.isValue()) {
+                            if (CommonFields.KEY.getPreferredName().equals(currentFieldName)) {
+                                key = parser.doubleValue();
+                            } else if (CommonFields.VALUE.getPreferredName().equals(currentFieldName)) {
+                                value = parser.doubleValue();
+                            } else if (CommonFields.VALUE_AS_STRING.getPreferredName().equals(currentFieldName)) {
+                                valueAsString = parser.text();
+                            }
+                        } else if (token == XContentParser.Token.VALUE_NULL) {
+                            value = Double.NaN;
+                        } else {
+                            parser.skipChildren(); // skip potential inner objects and arrays for forward compatibility
+                        }
+                    }
+                    if (key != null) {
+                        aggregation.addPercentile(key, value);
+                        if (valueAsString != null) {
+                            aggregation.addPercentileAsString(key, valueAsString);
+                        }
+                    }
+                }
+            }
+        }, CommonFields.VALUES, ObjectParser.ValueType.OBJECT_ARRAY);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/Percentile.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/Percentile.java
index 96ad4f261a64a..ca62ca6b2007e 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/Percentile.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/Percentile.java
@@ -19,10 +19,41 @@
 
 package org.elasticsearch.search.aggregations.metrics.percentiles;
 
-public interface Percentile {
+import java.util.Objects;
 
-    double getPercent();
+public class Percentile {
 
-    double getValue();
+    private final double percent;
+    private final double value;
 
+    public Percentile(double percent, double value) {
+        this.percent = percent;
+        this.value = value;
+    }
+
+    public double getPercent() {
+        return percent;
+    }
+
+    public double getValue() {
+        return value;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        Percentile that = (Percentile) o;
+        return Double.compare(that.percent, percent) == 0
+                && Double.compare(that.value, value) == 0;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(percent, value);
+    }
 }
\ No newline at end of file
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentileRanksAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentileRanksAggregationBuilder.java
index 8320774da0f73..db322a8e70e1c 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentileRanksAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentileRanksAggregationBuilder.java
@@ -19,22 +19,26 @@
 
 package org.elasticsearch.search.aggregations.metrics.percentiles;
 
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.metrics.percentiles.hdr.HDRPercentileRanksAggregatorFactory;
 import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.TDigestPercentileRanksAggregatorFactory;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder.LeafOnly;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -42,7 +46,59 @@
 
 public class PercentileRanksAggregationBuilder extends LeafOnly<ValuesSource.Numeric, PercentileRanksAggregationBuilder> {
     public static final String NAME = PercentileRanks.TYPE_NAME;
-    public static final Type TYPE = new Type(NAME);
+
+    public static final ParseField VALUES_FIELD = new ParseField("values");
+
+    private static class TDigestOptions {
+        Double compression;
+    }
+
+    private static final ObjectParser<TDigestOptions, QueryParseContext> TDIGEST_OPTIONS_PARSER =
+            new ObjectParser<>(PercentilesMethod.TDIGEST.getParseField().getPreferredName(), TDigestOptions::new);
+    static {
+        TDIGEST_OPTIONS_PARSER.declareDouble((opts, compression) -> opts.compression = compression, new ParseField("compression"));
+    }
+
+    private static class HDROptions {
+        Integer numberOfSigDigits;
+    }
+
+    private static final ObjectParser<HDROptions, QueryParseContext> HDR_OPTIONS_PARSER =
+            new ObjectParser<>(PercentilesMethod.HDR.getParseField().getPreferredName(), HDROptions::new);
+    static {
+        HDR_OPTIONS_PARSER.declareInt((opts, numberOfSigDigits) -> opts.numberOfSigDigits = numberOfSigDigits,
+                new ParseField("number_of_significant_value_digits"));
+    }
+
+    private static final ObjectParser<PercentileRanksAggregationBuilder, QueryParseContext> PARSER;
+    static {
+        PARSER = new ObjectParser<>(PercentileRanksAggregationBuilder.NAME);
+        ValuesSourceParserHelper.declareNumericFields(PARSER, true, false, false);
+
+        PARSER.declareDoubleArray(
+                (b, v) -> b.values(v.stream().mapToDouble(Double::doubleValue).toArray()),
+                VALUES_FIELD);
+
+        PARSER.declareBoolean(PercentileRanksAggregationBuilder::keyed, PercentilesAggregationBuilder.KEYED_FIELD);
+
+        PARSER.declareField((b, v) -> {
+            b.method(PercentilesMethod.TDIGEST);
+            if (v.compression != null) {
+                b.compression(v.compression);
+            }
+        }, TDIGEST_OPTIONS_PARSER::parse, PercentilesMethod.TDIGEST.getParseField(), ObjectParser.ValueType.OBJECT);
+
+        PARSER.declareField((b, v) -> {
+            b.method(PercentilesMethod.HDR);
+            if (v.numberOfSigDigits != null) {
+                b.numberOfSignificantValueDigits(v.numberOfSigDigits);
+            }
+        }, HDR_OPTIONS_PARSER::parse, PercentilesMethod.HDR.getParseField(), ObjectParser.ValueType.OBJECT);
+    }
+
+    public static AggregationBuilder parse(String aggregationName, QueryParseContext context) throws IOException {
+        return PARSER.parse(context.parser(), new PercentileRanksAggregationBuilder(aggregationName), context);
+    }
 
     private double[] values;
     private PercentilesMethod method = PercentilesMethod.TDIGEST;
@@ -51,14 +107,14 @@ public class PercentileRanksAggregationBuilder extends LeafOnly<ValuesSource.Num
     private boolean keyed = true;
 
     public PercentileRanksAggregationBuilder(String name) {
-        super(name, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(name, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     /**
      * Read from a stream.
      */
     public PercentileRanksAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(in, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
         values = in.readDoubleArray();
         keyed = in.readBoolean();
         numberOfSignificantValueDigits = in.readVInt();
@@ -164,29 +220,29 @@ public PercentilesMethod method() {
     }
 
     @Override
-    protected ValuesSourceAggregatorFactory<Numeric, ?> innerBuild(AggregationContext context, ValuesSourceConfig<Numeric> config,
+    protected ValuesSourceAggregatorFactory<Numeric, ?> innerBuild(SearchContext context, ValuesSourceConfig<Numeric> config,
             AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
         switch (method) {
         case TDIGEST:
-            return new TDigestPercentileRanksAggregatorFactory(name, type, config, values, compression, keyed, context, parent,
+            return new TDigestPercentileRanksAggregatorFactory(name, config, values, compression, keyed, context, parent,
                     subFactoriesBuilder, metaData);
         case HDR:
-            return new HDRPercentileRanksAggregatorFactory(name, type, config, values, numberOfSignificantValueDigits, keyed, context,
+            return new HDRPercentileRanksAggregatorFactory(name, config, values, numberOfSignificantValueDigits, keyed, context,
                     parent, subFactoriesBuilder, metaData);
         default:
-            throw new IllegalStateException("Illegal method [" + method.getName() + "]");
+            throw new IllegalStateException("Illegal method [" + method + "]");
         }
     }
 
     @Override
     protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
-        builder.field(PercentileRanksParser.VALUES_FIELD.getPreferredName(), values);
-        builder.field(AbstractPercentilesParser.KEYED_FIELD.getPreferredName(), keyed);
-        builder.startObject(method.getName());
+        builder.array(VALUES_FIELD.getPreferredName(), values);
+        builder.field(PercentilesAggregationBuilder.KEYED_FIELD.getPreferredName(), keyed);
+        builder.startObject(method.toString());
         if (method == PercentilesMethod.TDIGEST) {
-            builder.field(AbstractPercentilesParser.COMPRESSION_FIELD.getPreferredName(), compression);
+            builder.field(PercentilesAggregationBuilder.COMPRESSION_FIELD.getPreferredName(), compression);
         } else {
-            builder.field(AbstractPercentilesParser.NUMBER_SIGNIFICANT_DIGITS_FIELD.getPreferredName(), numberOfSignificantValueDigits);
+            builder.field(PercentilesAggregationBuilder.NUMBER_SIGNIFICANT_DIGITS_FIELD.getPreferredName(), numberOfSignificantValueDigits);
         }
         builder.endObject();
         return builder;
@@ -207,7 +263,7 @@ protected boolean innerEquals(Object obj) {
             equalSettings = Objects.equals(compression, other.compression);
             break;
         default:
-            throw new IllegalStateException("Illegal method [" + method.getName() + "]");
+            throw new IllegalStateException("Illegal method [" + method + "]");
         }
         return equalSettings
                 && Objects.deepEquals(values, other.values)
@@ -223,12 +279,12 @@ protected int innerHashCode() {
         case TDIGEST:
             return Objects.hash(Arrays.hashCode(values), keyed, compression, method);
         default:
-            throw new IllegalStateException("Illegal method [" + method.getName() + "]");
+            throw new IllegalStateException("Illegal method [" + method + "]");
         }
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentileRanksParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentileRanksParser.java
deleted file mode 100644
index 313b8e7b7c04a..0000000000000
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentileRanksParser.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.search.aggregations.metrics.percentiles;
-
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
-import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
-
-/**
- *
- */
-public class PercentileRanksParser extends AbstractPercentilesParser {
-
-    public static final ParseField VALUES_FIELD = new ParseField("values");
-
-    public PercentileRanksParser() {
-        super(false);
-    }
-
-    @Override
-    protected ParseField keysField() {
-        return VALUES_FIELD;
-    }
-
-    @Override
-    protected ValuesSourceAggregationBuilder<Numeric, ?> buildFactory(String aggregationName, double[] keys, PercentilesMethod method,
-                                                                      Double compression, Integer numberOfSignificantValueDigits,
-                                                                      Boolean keyed) {
-        PercentileRanksAggregationBuilder factory = new PercentileRanksAggregationBuilder(aggregationName);
-        if (keys != null) {
-            factory.values(keys);
-        }
-        if (method != null) {
-            factory.method(method);
-        }
-        if (compression != null) {
-            factory.compression(compression);
-        }
-        if (numberOfSignificantValueDigits != null) {
-            factory.numberOfSignificantValueDigits(numberOfSignificantValueDigits);
-        }
-        if (keyed != null) {
-            factory.keyed(keyed);
-        }
-        return factory;
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesAggregationBuilder.java
index f3fd9ad744b27..767e875c13274 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesAggregationBuilder.java
@@ -19,22 +19,26 @@
 
 package org.elasticsearch.search.aggregations.metrics.percentiles;
 
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.metrics.percentiles.hdr.HDRPercentilesAggregatorFactory;
 import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.TDigestPercentilesAggregatorFactory;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder.LeafOnly;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -42,23 +46,81 @@
 
 public class PercentilesAggregationBuilder extends LeafOnly<ValuesSource.Numeric, PercentilesAggregationBuilder> {
     public static final String NAME = Percentiles.TYPE_NAME;
-    public static final Type TYPE = new Type(NAME);
 
-    private double[] percents = PercentilesParser.DEFAULT_PERCENTS;
+    public static final double[] DEFAULT_PERCENTS = new double[] { 1, 5, 25, 50, 75, 95, 99 };
+    public static final ParseField PERCENTS_FIELD = new ParseField("percents");
+    public static final ParseField KEYED_FIELD = new ParseField("keyed");
+    public static final ParseField METHOD_FIELD = new ParseField("method");
+    public static final ParseField COMPRESSION_FIELD = new ParseField("compression");
+    public static final ParseField NUMBER_SIGNIFICANT_DIGITS_FIELD = new ParseField("number_of_significant_value_digits");
+
+    private static class TDigestOptions {
+        Double compression;
+    }
+
+    private static final ObjectParser<TDigestOptions, QueryParseContext> TDIGEST_OPTIONS_PARSER =
+            new ObjectParser<>(PercentilesMethod.TDIGEST.getParseField().getPreferredName(), TDigestOptions::new);
+    static {
+        TDIGEST_OPTIONS_PARSER.declareDouble((opts, compression) -> opts.compression = compression, COMPRESSION_FIELD);
+    }
+
+    private static class HDROptions {
+        Integer numberOfSigDigits;
+    }
+
+    private static final ObjectParser<HDROptions, QueryParseContext> HDR_OPTIONS_PARSER =
+            new ObjectParser<>(PercentilesMethod.HDR.getParseField().getPreferredName(), HDROptions::new);
+    static {
+        HDR_OPTIONS_PARSER.declareInt(
+                (opts, numberOfSigDigits) -> opts.numberOfSigDigits = numberOfSigDigits,
+                NUMBER_SIGNIFICANT_DIGITS_FIELD);
+    }
+
+    private static final ObjectParser<PercentilesAggregationBuilder, QueryParseContext> PARSER;
+    static {
+        PARSER = new ObjectParser<>(PercentilesAggregationBuilder.NAME);
+        ValuesSourceParserHelper.declareNumericFields(PARSER, true, true, false);
+
+        PARSER.declareDoubleArray(
+                (b, v) -> b.percentiles(v.stream().mapToDouble(Double::doubleValue).toArray()),
+                PERCENTS_FIELD);
+
+        PARSER.declareBoolean(PercentilesAggregationBuilder::keyed, KEYED_FIELD);
+
+        PARSER.declareField((b, v) -> {
+            b.method(PercentilesMethod.TDIGEST);
+            if (v.compression != null) {
+                b.compression(v.compression);
+            }
+        }, TDIGEST_OPTIONS_PARSER::parse, PercentilesMethod.TDIGEST.getParseField(), ObjectParser.ValueType.OBJECT);
+
+        PARSER.declareField((b, v) -> {
+            b.method(PercentilesMethod.HDR);
+            if (v.numberOfSigDigits != null) {
+                b.numberOfSignificantValueDigits(v.numberOfSigDigits);
+            }
+        }, HDR_OPTIONS_PARSER::parse, PercentilesMethod.HDR.getParseField(), ObjectParser.ValueType.OBJECT);
+    }
+
+    public static AggregationBuilder parse(String aggregationName, QueryParseContext context) throws IOException {
+        return PARSER.parse(context.parser(), new PercentilesAggregationBuilder(aggregationName), context);
+    }
+
+    private double[] percents = DEFAULT_PERCENTS;
     private PercentilesMethod method = PercentilesMethod.TDIGEST;
     private int numberOfSignificantValueDigits = 3;
     private double compression = 100.0;
     private boolean keyed = true;
 
     public PercentilesAggregationBuilder(String name) {
-        super(name, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(name, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     /**
      * Read from a stream.
      */
     public PercentilesAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(in, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
         percents = in.readDoubleArray();
         keyed = in.readBoolean();
         numberOfSignificantValueDigits = in.readVInt();
@@ -164,29 +226,29 @@ public PercentilesMethod method() {
     }
 
     @Override
-    protected ValuesSourceAggregatorFactory<Numeric, ?> innerBuild(AggregationContext context, ValuesSourceConfig<Numeric> config,
+    protected ValuesSourceAggregatorFactory<Numeric, ?> innerBuild(SearchContext context, ValuesSourceConfig<Numeric> config,
             AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
         switch (method) {
         case TDIGEST:
-            return new TDigestPercentilesAggregatorFactory(name, type, config, percents, compression, keyed, context, parent,
+            return new TDigestPercentilesAggregatorFactory(name, config, percents, compression, keyed, context, parent,
                     subFactoriesBuilder, metaData);
         case HDR:
-            return new HDRPercentilesAggregatorFactory(name, type, config, percents, numberOfSignificantValueDigits, keyed, context, parent,
+            return new HDRPercentilesAggregatorFactory(name, config, percents, numberOfSignificantValueDigits, keyed, context, parent,
                     subFactoriesBuilder, metaData);
         default:
-            throw new IllegalStateException("Illegal method [" + method.getName() + "]");
+            throw new IllegalStateException("Illegal method [" + method + "]");
         }
     }
 
     @Override
     protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
-        builder.field(PercentilesParser.PERCENTS_FIELD.getPreferredName(), percents);
-        builder.field(AbstractPercentilesParser.KEYED_FIELD.getPreferredName(), keyed);
-        builder.startObject(method.getName());
+        builder.array(PERCENTS_FIELD.getPreferredName(), percents);
+        builder.field(KEYED_FIELD.getPreferredName(), keyed);
+        builder.startObject(method.toString());
         if (method == PercentilesMethod.TDIGEST) {
-            builder.field(AbstractPercentilesParser.COMPRESSION_FIELD.getPreferredName(), compression);
+            builder.field(COMPRESSION_FIELD.getPreferredName(), compression);
         } else {
-            builder.field(AbstractPercentilesParser.NUMBER_SIGNIFICANT_DIGITS_FIELD.getPreferredName(), numberOfSignificantValueDigits);
+            builder.field(NUMBER_SIGNIFICANT_DIGITS_FIELD.getPreferredName(), numberOfSignificantValueDigits);
         }
         builder.endObject();
         return builder;
@@ -207,7 +269,7 @@ protected boolean innerEquals(Object obj) {
             equalSettings = Objects.equals(compression, other.compression);
             break;
         default:
-            throw new IllegalStateException("Illegal method [" + method.getName() + "]");
+            throw new IllegalStateException("Illegal method [" + method.toString() + "]");
         }
         return equalSettings
                 && Objects.deepEquals(percents, other.percents)
@@ -223,12 +285,12 @@ protected int innerHashCode() {
         case TDIGEST:
             return Objects.hash(Arrays.hashCode(percents), keyed, compression, method);
         default:
-            throw new IllegalStateException("Illegal method [" + method.getName() + "]");
+            throw new IllegalStateException("Illegal method [" + method.toString() + "]");
         }
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesMethod.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesMethod.java
index 97b8a727be331..3b8085793dc0a 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesMethod.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesMethod.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.search.aggregations.metrics.percentiles;
 
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -32,48 +33,36 @@ public enum PercentilesMethod implements Writeable {
     /**
      * The TDigest method for calculating percentiles
      */
-    TDIGEST("tdigest"),
+    TDIGEST("tdigest", "TDigest", "TDIGEST"),
     /**
      * The HDRHistogram method of calculating percentiles
      */
-    HDR("hdr");
+    HDR("hdr", "HDR");
 
-    private String name;
+    private final ParseField parseField;
 
-    private PercentilesMethod(String name) {
-        this.name = name;
+    PercentilesMethod(String name, String... deprecatedNames) {
+        this.parseField = new ParseField(name, deprecatedNames);
     }
 
     /**
      * @return the name of the method
      */
-    public String getName() {
-        return name;
+    public ParseField getParseField() {
+        return parseField;
     }
 
     public static PercentilesMethod readFromStream(StreamInput in) throws IOException {
-        int ordinal = in.readVInt();
-        if (ordinal < 0 || ordinal >= values().length) {
-            throw new IOException("Unknown PercentilesMethod ordinal [" + ordinal + "]");
-        }
-        return values()[ordinal];
+        return in.readEnum(PercentilesMethod.class);
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        out.writeVInt(ordinal());
+        out.writeEnum(this);
     }
 
-    /**
-     * Returns the {@link PercentilesMethod} for this method name. returns
-     * <code>null</code> if no {@link PercentilesMethod} exists for the name.
-     */
-    public static PercentilesMethod resolveFromName(String name) {
-        for (PercentilesMethod method : values()) {
-            if (method.name.equalsIgnoreCase(name)) {
-                return method;
-            }
-        }
-        return null;
+    @Override
+    public String toString() {
+        return parseField.getPreferredName();
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesParser.java
deleted file mode 100644
index 806fb26cd3fe8..0000000000000
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesParser.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.search.aggregations.metrics.percentiles;
-
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
-import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
-
-/**
- *
- */
-public class PercentilesParser extends AbstractPercentilesParser {
-
-    public static final ParseField PERCENTS_FIELD = new ParseField("percents");
-
-    public PercentilesParser() {
-        super(true);
-    }
-
-    public static final double[] DEFAULT_PERCENTS = new double[] { 1, 5, 25, 50, 75, 95, 99 };
-
-    @Override
-    protected ParseField keysField() {
-        return PERCENTS_FIELD;
-    }
-
-    @Override
-    protected ValuesSourceAggregationBuilder<Numeric, ?> buildFactory(String aggregationName, double[] keys, PercentilesMethod method,
-                                                                      Double compression, Integer numberOfSignificantValueDigits,
-                                                                      Boolean keyed) {
-        PercentilesAggregationBuilder factory = new PercentilesAggregationBuilder(aggregationName);
-        if (keys != null) {
-            factory.percentiles(keys);
-        }
-        if (method != null) {
-            factory.method(method);
-        }
-        if (compression != null) {
-            factory.compression(compression);
-        }
-        if (numberOfSignificantValueDigits != null) {
-            factory.numberOfSignificantValueDigits(numberOfSignificantValueDigits);
-        }
-        if (keyed != null) {
-            factory.keyed(keyed);
-        }
-        return factory;
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/AbstractHDRPercentilesAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/AbstractHDRPercentilesAggregator.java
index 26ac179a5613e..bf4443c88739f 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/AbstractHDRPercentilesAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/AbstractHDRPercentilesAggregator.java
@@ -32,8 +32,8 @@
 import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
 import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregator;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -52,7 +52,7 @@ private static int indexOfKey(double[] keys, double key) {
     protected final int numberOfSignificantValueDigits;
     protected final boolean keyed;
 
-    public AbstractHDRPercentilesAggregator(String name, ValuesSource.Numeric valuesSource, AggregationContext context, Aggregator parent,
+    public AbstractHDRPercentilesAggregator(String name, ValuesSource.Numeric valuesSource, SearchContext context, Aggregator parent,
             double[] keys, int numberOfSignificantValueDigits, boolean keyed, DocValueFormat formatter,
             List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException {
         super(name, context, parent, pipelineAggregators, metaData);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/AbstractInternalHDRPercentiles.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/AbstractInternalHDRPercentiles.java
index ba76f8b98632f..5f9e87e7c047a 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/AbstractInternalHDRPercentiles.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/AbstractInternalHDRPercentiles.java
@@ -40,7 +40,7 @@ abstract class AbstractInternalHDRPercentiles extends InternalNumericMetricsAggr
     protected final DoubleHistogram state;
     private final boolean keyed;
 
-    public AbstractInternalHDRPercentiles(String name, double[] keys, DoubleHistogram state, boolean keyed, DocValueFormat format,
+    AbstractInternalHDRPercentiles(String name, double[] keys, DoubleHistogram state, boolean keyed, DocValueFormat format,
             List<PipelineAggregator> pipelineAggregators,
             Map<String, Object> metaData) {
         super(name, pipelineAggregators, metaData);
@@ -113,7 +113,7 @@ protected abstract AbstractInternalHDRPercentiles createReduced(String name, dou
     @Override
     public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
         if (keyed) {
-            builder.startObject(CommonFields.VALUES);
+            builder.startObject(CommonFields.VALUES.getPreferredName());
             for(int i = 0; i < keys.length; ++i) {
                 String key = String.valueOf(keys[i]);
                 double value = value(keys[i]);
@@ -124,14 +124,14 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th
             }
             builder.endObject();
         } else {
-            builder.startArray(CommonFields.VALUES);
+            builder.startArray(CommonFields.VALUES.getPreferredName());
             for (int i = 0; i < keys.length; i++) {
                 double value = value(keys[i]);
                 builder.startObject();
-                builder.field(CommonFields.KEY, keys[i]);
-                builder.field(CommonFields.VALUE, value);
+                builder.field(CommonFields.KEY.getPreferredName(), keys[i]);
+                builder.field(CommonFields.VALUE.getPreferredName(), value);
                 if (format != DocValueFormat.RAW) {
-                    builder.field(CommonFields.VALUE_AS_STRING, format.format(value));
+                    builder.field(CommonFields.VALUE_AS_STRING.getPreferredName(), format.format(value));
                 }
                 builder.endObject();
             }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/HDRPercentileRanksAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/HDRPercentileRanksAggregator.java
index faa6039f56cd9..53da1ee81856c 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/HDRPercentileRanksAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/HDRPercentileRanksAggregator.java
@@ -23,8 +23,8 @@
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -35,7 +35,7 @@
  */
 public class HDRPercentileRanksAggregator extends AbstractHDRPercentilesAggregator {
 
-    public HDRPercentileRanksAggregator(String name, Numeric valuesSource, AggregationContext context, Aggregator parent,
+    public HDRPercentileRanksAggregator(String name, Numeric valuesSource, SearchContext context, Aggregator parent,
             double[] percents, int numberOfSignificantValueDigits, boolean keyed, DocValueFormat format,
             List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException {
         super(name, valuesSource, context, parent, percents, numberOfSignificantValueDigits, keyed, format, pipelineAggregators,
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/HDRPercentileRanksAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/HDRPercentileRanksAggregatorFactory.java
index 7f17220edad8e..d89a9a85b28a9 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/HDRPercentileRanksAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/HDRPercentileRanksAggregatorFactory.java
@@ -22,13 +22,12 @@
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -41,10 +40,10 @@ public class HDRPercentileRanksAggregatorFactory
     private final int numberOfSignificantValueDigits;
     private final boolean keyed;
 
-    public HDRPercentileRanksAggregatorFactory(String name, Type type, ValuesSourceConfig<Numeric> config, double[] values,
-            int numberOfSignificantValueDigits, boolean keyed, AggregationContext context, AggregatorFactory<?> parent,
+    public HDRPercentileRanksAggregatorFactory(String name, ValuesSourceConfig<Numeric> config, double[] values,
+            int numberOfSignificantValueDigits, boolean keyed, SearchContext context, AggregatorFactory<?> parent,
             AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.values = values;
         this.numberOfSignificantValueDigits = numberOfSignificantValueDigits;
         this.keyed = keyed;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/HDRPercentilesAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/HDRPercentilesAggregator.java
index 0b9c2c43d3438..d61b4ddade0b1 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/HDRPercentilesAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/HDRPercentilesAggregator.java
@@ -23,8 +23,8 @@
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -35,7 +35,7 @@
  */
 public class HDRPercentilesAggregator extends AbstractHDRPercentilesAggregator {
 
-    public HDRPercentilesAggregator(String name, Numeric valuesSource, AggregationContext context, Aggregator parent, double[] percents,
+    public HDRPercentilesAggregator(String name, Numeric valuesSource, SearchContext context, Aggregator parent, double[] percents,
             int numberOfSignificantValueDigits, boolean keyed, DocValueFormat formatter,
             List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException {
         super(name, valuesSource, context, parent, percents, numberOfSignificantValueDigits, keyed, formatter,
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/HDRPercentilesAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/HDRPercentilesAggregatorFactory.java
index 0e1bd1e532d31..1074b6e142db6 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/HDRPercentilesAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/HDRPercentilesAggregatorFactory.java
@@ -22,13 +22,12 @@
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -40,10 +39,10 @@ public class HDRPercentilesAggregatorFactory extends ValuesSourceAggregatorFacto
     private final int numberOfSignificantValueDigits;
     private final boolean keyed;
 
-    public HDRPercentilesAggregatorFactory(String name, Type type, ValuesSourceConfig<Numeric> config, double[] percents,
-            int numberOfSignificantValueDigits, boolean keyed, AggregationContext context, AggregatorFactory<?> parent,
+    public HDRPercentilesAggregatorFactory(String name, ValuesSourceConfig<Numeric> config, double[] percents,
+            int numberOfSignificantValueDigits, boolean keyed, SearchContext context, AggregatorFactory<?> parent,
             AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.percents = percents;
         this.numberOfSignificantValueDigits = numberOfSignificantValueDigits;
         this.keyed = keyed;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentileRanks.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentileRanks.java
index 35234d73accf7..cb058128c5a49 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentileRanks.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentileRanks.java
@@ -21,7 +21,6 @@
 import org.HdrHistogram.DoubleHistogram;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.metrics.percentiles.InternalPercentile;
 import org.elasticsearch.search.aggregations.metrics.percentiles.Percentile;
 import org.elasticsearch.search.aggregations.metrics.percentiles.PercentileRanks;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
@@ -109,7 +108,7 @@ public boolean hasNext() {
 
         @Override
         public Percentile next() {
-            final Percentile next = new InternalPercentile(percentileRank(state, values[i]), values[i]);
+            final Percentile next = new Percentile(percentileRank(state, values[i]), values[i]);
             ++i;
             return next;
         }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentiles.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentiles.java
index 579f25c1666de..a153e497f7bc8 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentiles.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentiles.java
@@ -21,7 +21,6 @@
 import org.HdrHistogram.DoubleHistogram;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.metrics.percentiles.InternalPercentile;
 import org.elasticsearch.search.aggregations.metrics.percentiles.Percentile;
 import org.elasticsearch.search.aggregations.metrics.percentiles.Percentiles;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
@@ -99,7 +98,9 @@ public boolean hasNext() {
 
         @Override
         public Percentile next() {
-            final Percentile next = new InternalPercentile(percents[i], state.getValueAtPercentile(percents[i]));
+            double percent = percents[i];
+            double value = (state.getTotalCount() == 0) ? Double.NaN : state.getValueAtPercentile(percent);
+            final Percentile next = new Percentile(percent, value);
             ++i;
             return next;
         }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/ParsedHDRPercentileRanks.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/ParsedHDRPercentileRanks.java
new file mode 100644
index 0000000000000..f5fd7717e04bf
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/ParsedHDRPercentileRanks.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.percentiles.hdr;
+
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.metrics.percentiles.ParsedPercentiles;
+import org.elasticsearch.search.aggregations.metrics.percentiles.ParsedPercentileRanks;
+import org.elasticsearch.search.aggregations.metrics.percentiles.Percentile;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+public class ParsedHDRPercentileRanks extends ParsedPercentileRanks {
+
+    @Override
+    public String getType() {
+        return InternalHDRPercentileRanks.NAME;
+    }
+
+    @Override
+    public Iterator<Percentile> iterator() {
+        final Iterator<Percentile> iterator = super.iterator();
+        return new Iterator<Percentile>() {
+            @Override
+            public boolean hasNext() {
+                return iterator.hasNext();
+            }
+
+            @Override
+            public Percentile next() {
+                Percentile percentile = iterator.next();
+                return new Percentile(percentile.getValue(), percentile.getPercent());
+            }
+        };
+    }
+
+    private static ObjectParser<ParsedHDRPercentileRanks, Void> PARSER =
+            new ObjectParser<>(ParsedHDRPercentileRanks.class.getSimpleName(), true, ParsedHDRPercentileRanks::new);
+    static {
+        ParsedPercentiles.declarePercentilesFields(PARSER);
+    }
+
+    public static ParsedHDRPercentileRanks fromXContent(XContentParser parser, String name) throws IOException {
+        ParsedHDRPercentileRanks aggregation = PARSER.parse(parser, null);
+        aggregation.setName(name);
+        return aggregation;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/ParsedHDRPercentiles.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/ParsedHDRPercentiles.java
new file mode 100644
index 0000000000000..1b1ba906aa087
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/ParsedHDRPercentiles.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.percentiles.hdr;
+
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.metrics.percentiles.ParsedPercentiles;
+import org.elasticsearch.search.aggregations.metrics.percentiles.Percentiles;
+
+import java.io.IOException;
+
+public class ParsedHDRPercentiles extends ParsedPercentiles implements Percentiles {
+
+    @Override
+    public String getType() {
+        return InternalHDRPercentiles.NAME;
+    }
+
+    @Override
+    public double percentile(double percent) {
+        return getPercentile(percent);
+    }
+
+    @Override
+    public String percentileAsString(double percent) {
+        return getPercentileAsString(percent);
+    }
+
+    private static ObjectParser<ParsedHDRPercentiles, Void> PARSER =
+            new ObjectParser<>(ParsedHDRPercentiles.class.getSimpleName(), true, ParsedHDRPercentiles::new);
+    static {
+        ParsedPercentiles.declarePercentilesFields(PARSER);
+    }
+
+    public static ParsedHDRPercentiles fromXContent(XContentParser parser, String name) throws IOException {
+        ParsedHDRPercentiles aggregation = PARSER.parse(parser, null);
+        aggregation.setName(name);
+        return aggregation;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/AbstractInternalTDigestPercentiles.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/AbstractInternalTDigestPercentiles.java
index 43bee101f4a01..b1cf553529d01 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/AbstractInternalTDigestPercentiles.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/AbstractInternalTDigestPercentiles.java
@@ -37,7 +37,7 @@ abstract class AbstractInternalTDigestPercentiles extends InternalNumericMetrics
     protected final TDigestState state;
     private final boolean keyed;
 
-    public AbstractInternalTDigestPercentiles(String name, double[] keys, TDigestState state, boolean keyed, DocValueFormat formatter,
+    AbstractInternalTDigestPercentiles(String name, double[] keys, TDigestState state, boolean keyed, DocValueFormat formatter,
             List<PipelineAggregator> pipelineAggregators,
             Map<String, Object> metaData) {
         super(name, pipelineAggregators, metaData);
@@ -96,7 +96,7 @@ protected abstract AbstractInternalTDigestPercentiles createReduced(String name,
     @Override
     public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
         if (keyed) {
-            builder.startObject(CommonFields.VALUES);
+            builder.startObject(CommonFields.VALUES.getPreferredName());
             for(int i = 0; i < keys.length; ++i) {
                 String key = String.valueOf(keys[i]);
                 double value = value(keys[i]);
@@ -107,14 +107,14 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th
             }
             builder.endObject();
         } else {
-            builder.startArray(CommonFields.VALUES);
+            builder.startArray(CommonFields.VALUES.getPreferredName());
             for (int i = 0; i < keys.length; i++) {
                 double value = value(keys[i]);
                 builder.startObject();
-                builder.field(CommonFields.KEY, keys[i]);
-                builder.field(CommonFields.VALUE, value);
+                builder.field(CommonFields.KEY.getPreferredName(), keys[i]);
+                builder.field(CommonFields.VALUE.getPreferredName(), value);
                 if (format != DocValueFormat.RAW) {
-                    builder.field(CommonFields.VALUE_AS_STRING, format.format(value));
+                    builder.field(CommonFields.VALUE_AS_STRING.getPreferredName(), format.format(value));
                 }
                 builder.endObject();
             }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/AbstractTDigestPercentilesAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/AbstractTDigestPercentilesAggregator.java
index 0676d291a1133..2c68d580e14b5 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/AbstractTDigestPercentilesAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/AbstractTDigestPercentilesAggregator.java
@@ -31,8 +31,8 @@
 import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
 import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregator;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -51,7 +51,7 @@ private static int indexOfKey(double[] keys, double key) {
     protected final double compression;
     protected final boolean keyed;
 
-    public AbstractTDigestPercentilesAggregator(String name, ValuesSource.Numeric valuesSource, AggregationContext context, Aggregator parent,
+    public AbstractTDigestPercentilesAggregator(String name, ValuesSource.Numeric valuesSource, SearchContext context, Aggregator parent,
             double[] keys, double compression, boolean keyed, DocValueFormat formatter,
             List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException {
         super(name, context, parent, pipelineAggregators, metaData);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentileRanks.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentileRanks.java
index 9e24ba5d86e0a..666993f41fda3 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentileRanks.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentileRanks.java
@@ -20,7 +20,6 @@
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.metrics.percentiles.InternalPercentile;
 import org.elasticsearch.search.aggregations.metrics.percentiles.Percentile;
 import org.elasticsearch.search.aggregations.metrics.percentiles.PercentileRanks;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
@@ -106,7 +105,7 @@ public boolean hasNext() {
 
         @Override
         public Percentile next() {
-            final Percentile next = new InternalPercentile(percentileRank(state, values[i]), values[i]);
+            final Percentile next = new Percentile(percentileRank(state, values[i]), values[i]);
             ++i;
             return next;
         }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentiles.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentiles.java
index ec619219111eb..5a62f24933b40 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentiles.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentiles.java
@@ -20,7 +20,6 @@
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.metrics.percentiles.InternalPercentile;
 import org.elasticsearch.search.aggregations.metrics.percentiles.Percentile;
 import org.elasticsearch.search.aggregations.metrics.percentiles.Percentiles;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
@@ -95,7 +94,7 @@ public boolean hasNext() {
 
         @Override
         public Percentile next() {
-            final Percentile next = new InternalPercentile(percents[i], state.quantile(percents[i] / 100));
+            final Percentile next = new Percentile(percents[i], state.quantile(percents[i] / 100));
             ++i;
             return next;
         }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/ParsedTDigestPercentileRanks.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/ParsedTDigestPercentileRanks.java
new file mode 100644
index 0000000000000..01929f374d486
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/ParsedTDigestPercentileRanks.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.percentiles.tdigest;
+
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.metrics.percentiles.ParsedPercentiles;
+import org.elasticsearch.search.aggregations.metrics.percentiles.ParsedPercentileRanks;
+import org.elasticsearch.search.aggregations.metrics.percentiles.Percentile;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+public class ParsedTDigestPercentileRanks extends ParsedPercentileRanks {
+
+    @Override
+    public String getType() {
+        return InternalTDigestPercentileRanks.NAME;
+    }
+
+    @Override
+    public Iterator<Percentile> iterator() {
+        final Iterator<Percentile> iterator = super.iterator();
+        return new Iterator<Percentile>() {
+            @Override
+            public boolean hasNext() {
+                return iterator.hasNext();
+            }
+
+            @Override
+            public Percentile next() {
+                Percentile percentile = iterator.next();
+                return new Percentile(percentile.getValue(), percentile.getPercent());
+            }
+        };
+    }
+
+    private static ObjectParser<ParsedTDigestPercentileRanks, Void> PARSER =
+            new ObjectParser<>(ParsedTDigestPercentileRanks.class.getSimpleName(), true, ParsedTDigestPercentileRanks::new);
+    static {
+        ParsedPercentiles.declarePercentilesFields(PARSER);
+    }
+
+    public static ParsedTDigestPercentileRanks fromXContent(XContentParser parser, String name) throws IOException {
+        ParsedTDigestPercentileRanks aggregation = PARSER.parse(parser, null);
+        aggregation.setName(name);
+        return aggregation;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/ParsedTDigestPercentiles.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/ParsedTDigestPercentiles.java
new file mode 100644
index 0000000000000..cbae25d61e046
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/ParsedTDigestPercentiles.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.percentiles.tdigest;
+
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.metrics.percentiles.ParsedPercentiles;
+import org.elasticsearch.search.aggregations.metrics.percentiles.Percentiles;
+
+import java.io.IOException;
+
+public class ParsedTDigestPercentiles extends ParsedPercentiles implements Percentiles {
+
+    @Override
+    public String getType() {
+        return InternalTDigestPercentiles.NAME;
+    }
+
+    @Override
+    public double percentile(double percent) {
+        return getPercentile(percent);
+    }
+
+    @Override
+    public String percentileAsString(double percent) {
+        return getPercentileAsString(percent);
+    }
+
+    private static ObjectParser<ParsedTDigestPercentiles, Void> PARSER =
+            new ObjectParser<>(ParsedTDigestPercentiles.class.getSimpleName(), true, ParsedTDigestPercentiles::new);
+    static {
+        ParsedPercentiles.declarePercentilesFields(PARSER);
+    }
+
+    public static ParsedTDigestPercentiles fromXContent(XContentParser parser, String name) throws IOException {
+        ParsedTDigestPercentiles aggregation = PARSER.parse(parser, null);
+        aggregation.setName(name);
+        return aggregation;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/TDigestPercentileRanksAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/TDigestPercentileRanksAggregator.java
index 1151e2272a442..c796742fbb21f 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/TDigestPercentileRanksAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/TDigestPercentileRanksAggregator.java
@@ -22,8 +22,8 @@
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -34,7 +34,7 @@
  */
 public class TDigestPercentileRanksAggregator extends AbstractTDigestPercentilesAggregator {
 
-    public TDigestPercentileRanksAggregator(String name, Numeric valuesSource, AggregationContext context, Aggregator parent, double[] percents,
+    public TDigestPercentileRanksAggregator(String name, Numeric valuesSource, SearchContext context, Aggregator parent, double[] percents,
             double compression, boolean keyed, DocValueFormat formatter, List<PipelineAggregator> pipelineAggregators,
             Map<String, Object> metaData)
             throws IOException {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/TDigestPercentileRanksAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/TDigestPercentileRanksAggregatorFactory.java
index a2605fce23923..223d25216bca2 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/TDigestPercentileRanksAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/TDigestPercentileRanksAggregatorFactory.java
@@ -22,13 +22,12 @@
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -41,10 +40,10 @@ public class TDigestPercentileRanksAggregatorFactory
     private final double compression;
     private final boolean keyed;
 
-    public TDigestPercentileRanksAggregatorFactory(String name, Type type, ValuesSourceConfig<Numeric> config, double[] percents,
-            double compression, boolean keyed, AggregationContext context, AggregatorFactory<?> parent,
+    public TDigestPercentileRanksAggregatorFactory(String name, ValuesSourceConfig<Numeric> config, double[] percents,
+            double compression, boolean keyed, SearchContext context, AggregatorFactory<?> parent,
             AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.percents = percents;
         this.compression = compression;
         this.keyed = keyed;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/TDigestPercentilesAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/TDigestPercentilesAggregator.java
index c0063102e070b..74a5bce0e519a 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/TDigestPercentilesAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/TDigestPercentilesAggregator.java
@@ -22,8 +22,8 @@
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -34,7 +34,7 @@
  */
 public class TDigestPercentilesAggregator extends AbstractTDigestPercentilesAggregator {
 
-    public TDigestPercentilesAggregator(String name, Numeric valuesSource, AggregationContext context,
+    public TDigestPercentilesAggregator(String name, Numeric valuesSource, SearchContext context,
             Aggregator parent, double[] percents,
             double compression, boolean keyed, DocValueFormat formatter, List<PipelineAggregator> pipelineAggregators,
             Map<String, Object> metaData) throws IOException {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/TDigestPercentilesAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/TDigestPercentilesAggregatorFactory.java
index 4513801fbfbbe..47b17d84f3b6b 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/TDigestPercentilesAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/TDigestPercentilesAggregatorFactory.java
@@ -22,13 +22,12 @@
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -41,10 +40,10 @@ public class TDigestPercentilesAggregatorFactory
     private final double compression;
     private final boolean keyed;
 
-    public TDigestPercentilesAggregatorFactory(String name, Type type, ValuesSourceConfig<Numeric> config, double[] percents,
-            double compression, boolean keyed, AggregationContext context, AggregatorFactory<?> parent,
+    public TDigestPercentilesAggregatorFactory(String name, ValuesSourceConfig<Numeric> config, double[] percents,
+            double compression, boolean keyed, SearchContext context, AggregatorFactory<?> parent,
             AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.percents = percents;
         this.compression = compression;
         this.keyed = keyed;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetric.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetric.java
index c5704d9f2b6ef..d86104a5b142a 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetric.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetric.java
@@ -27,7 +27,6 @@
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.search.aggregations.InternalAggregation;
-import org.elasticsearch.search.aggregations.metrics.InternalMetricsAggregation;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 
 import java.io.IOException;
@@ -37,11 +36,16 @@
 import java.util.List;
 import java.util.Map;
 
-public class InternalScriptedMetric extends InternalMetricsAggregation implements ScriptedMetric {
+public class InternalScriptedMetric extends InternalAggregation implements ScriptedMetric {
     private final Script reduceScript;
-    private final Object aggregation;
+    private final List<Object> aggregation;
 
     public InternalScriptedMetric(String name, Object aggregation, Script reduceScript, List<PipelineAggregator> pipelineAggregators,
+                                  Map<String, Object> metaData) {
+        this(name, Collections.singletonList(aggregation), reduceScript, pipelineAggregators, metaData);
+    }
+
+    private InternalScriptedMetric(String name, List<Object> aggregation, Script reduceScript, List<PipelineAggregator> pipelineAggregators,
             Map<String, Object> metaData) {
         super(name, pipelineAggregators, metaData);
         this.aggregation = aggregation;
@@ -54,13 +58,13 @@ public InternalScriptedMetric(String name, Object aggregation, Script reduceScri
     public InternalScriptedMetric(StreamInput in) throws IOException {
         super(in);
         reduceScript = in.readOptionalWriteable(Script::new);
-        aggregation = in.readGenericValue();
+        aggregation = Collections.singletonList(in.readGenericValue());
     }
 
     @Override
     protected void doWriteTo(StreamOutput out) throws IOException {
         out.writeOptionalWriteable(reduceScript);
-        out.writeGenericValue(aggregation);
+        out.writeGenericValue(aggregation());
     }
 
     @Override
@@ -70,7 +74,10 @@ public String getWriteableName() {
 
     @Override
     public Object aggregation() {
-        return aggregation;
+        if (aggregation.size() != 1) {
+            throw new IllegalStateException("aggregation was not reduced");
+        }
+        return aggregation.get(0);
     }
 
     @Override
@@ -78,26 +85,29 @@ public InternalAggregation doReduce(List<InternalAggregation> aggregations, Redu
         List<Object> aggregationObjects = new ArrayList<>();
         for (InternalAggregation aggregation : aggregations) {
             InternalScriptedMetric mapReduceAggregation = (InternalScriptedMetric) aggregation;
-            aggregationObjects.add(mapReduceAggregation.aggregation());
+            aggregationObjects.addAll(mapReduceAggregation.aggregation);
         }
         InternalScriptedMetric firstAggregation = ((InternalScriptedMetric) aggregations.get(0));
-        Object aggregation;
-        if (firstAggregation.reduceScript != null) {
+        List<Object> aggregation;
+        if (firstAggregation.reduceScript != null && reduceContext.isFinalReduce()) {
             Map<String, Object> vars = new HashMap<>();
             vars.put("_aggs", aggregationObjects);
             if (firstAggregation.reduceScript.getParams() != null) {
                 vars.putAll(firstAggregation.reduceScript.getParams());
             }
-            CompiledScript compiledScript = reduceContext.scriptService().compile(firstAggregation.reduceScript,
-                    ScriptContext.Standard.AGGS, Collections.emptyMap());
+            CompiledScript compiledScript = reduceContext.scriptService().compile(
+                firstAggregation.reduceScript, ScriptContext.Standard.AGGS);
             ExecutableScript script = reduceContext.scriptService().executable(compiledScript, vars);
-            aggregation = script.run();
+            aggregation = Collections.singletonList(script.run());
+        } else if (reduceContext.isFinalReduce())  {
+            aggregation = Collections.singletonList(aggregationObjects);
         } else {
+            // if we are not an final reduce we have to maintain all the aggs from all the incoming one
+            // until we hit the final reduce phase.
             aggregation = aggregationObjects;
         }
         return new InternalScriptedMetric(firstAggregation.getName(), aggregation, firstAggregation.reduceScript, pipelineAggregators(),
                 getMetaData());
-
     }
 
     @Override
@@ -105,7 +115,7 @@ public Object getProperty(List<String> path) {
         if (path.isEmpty()) {
             return this;
         } else if (path.size() == 1 && "value".equals(path.get(0))) {
-            return aggregation;
+            return aggregation();
         } else {
             throw new IllegalArgumentException("path not supported for [" + getName() + "]: " + path);
         }
@@ -113,7 +123,7 @@ public Object getProperty(List<String> path) {
 
     @Override
     public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
-        return builder.field("value", aggregation);
+        return builder.field(CommonFields.VALUE.getPreferredName(), aggregation());
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ParsedScriptedMetric.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ParsedScriptedMetric.java
new file mode 100644
index 0000000000000..f2aae9f5e8aa5
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ParsedScriptedMetric.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.scripted;
+
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParser.Token;
+import org.elasticsearch.search.aggregations.ParsedAggregation;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+
+public class ParsedScriptedMetric extends ParsedAggregation implements ScriptedMetric {
+    private List<Object> aggregation;
+
+    @Override
+    public String getType() {
+        return ScriptedMetricAggregationBuilder.NAME;
+    }
+
+    @Override
+    public Object aggregation() {
+        assert aggregation.size() == 1; // see InternalScriptedMetric#aggregations() for why we can assume this
+        return aggregation.get(0);
+    }
+
+    @Override
+    public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+        return builder.field(CommonFields.VALUE.getPreferredName(), aggregation());
+    }
+
+    private static final ObjectParser<ParsedScriptedMetric, Void> PARSER =
+            new ObjectParser<>(ParsedScriptedMetric.class.getSimpleName(), true, ParsedScriptedMetric::new);
+
+    static {
+        declareAggregationFields(PARSER);
+        PARSER.declareField((agg, value) -> agg.aggregation = Collections.singletonList(value),
+                ParsedScriptedMetric::parseValue, CommonFields.VALUE, ValueType.VALUE_OBJECT_ARRAY);
+    }
+
+    private static Object parseValue(XContentParser parser) throws IOException {
+        Token token = parser.currentToken();
+        Object value = null;
+        if (token == XContentParser.Token.VALUE_NULL) {
+            value = null;
+        } else if (token.isValue()) {
+            if (token == XContentParser.Token.VALUE_STRING) {
+                //binary values will be parsed back and returned as base64 strings when reading from json and yaml
+                value = parser.text();
+            } else if (token == XContentParser.Token.VALUE_NUMBER) {
+                value = parser.numberValue();
+            } else if (token == XContentParser.Token.VALUE_BOOLEAN) {
+                value = parser.booleanValue();
+            } else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
+                //binary values will be parsed back and returned as BytesArray when reading from cbor and smile
+                value = new BytesArray(parser.binaryValue());
+            }
+        } else if (token == XContentParser.Token.START_OBJECT) {
+            value = parser.map();
+        } else if (token == XContentParser.Token.START_ARRAY) {
+            value = parser.list();
+        }
+        return value;
+    }
+
+    public static ParsedScriptedMetric fromXContent(XContentParser parser, final String name) {
+        ParsedScriptedMetric aggregation = PARSER.apply(parser, null);
+        aggregation.setName(name);
+        return aggregation;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregationBuilder.java
index 244881a51557e..d7604a14d6883 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregationBuilder.java
@@ -26,23 +26,25 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptContext;
+import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import java.util.function.Function;
 
 public class ScriptedMetricAggregationBuilder extends AbstractAggregationBuilder<ScriptedMetricAggregationBuilder> {
-
     public static final String NAME = "scripted_metric";
-    private static final Type TYPE = new Type(NAME);
 
     private static final ParseField INIT_SCRIPT_FIELD = new ParseField("init_script");
     private static final ParseField MAP_SCRIPT_FIELD = new ParseField("map_script");
@@ -57,14 +59,14 @@ public class ScriptedMetricAggregationBuilder extends AbstractAggregationBuilder
     private Map<String, Object> params;
 
     public ScriptedMetricAggregationBuilder(String name) {
-        super(name, TYPE);
+        super(name);
     }
 
     /**
      * Read from a stream.
      */
     public ScriptedMetricAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE);
+        super(in);
         initScript = in.readOptionalWriteable(Script::new);
         mapScript = in.readOptionalWriteable(Script::new);
         combineScript = in.readOptionalWriteable(Script::new);
@@ -180,12 +182,29 @@ public Map<String, Object> params() {
     }
 
     @Override
-    protected ScriptedMetricAggregatorFactory doBuild(AggregationContext context, AggregatorFactory<?> parent,
+    protected ScriptedMetricAggregatorFactory doBuild(SearchContext context, AggregatorFactory<?> parent,
             Builder subfactoriesBuilder) throws IOException {
-        return new ScriptedMetricAggregatorFactory(name, type, initScript, mapScript, combineScript, reduceScript, params, context,
-                parent, subfactoriesBuilder, metaData);
+
+        QueryShardContext queryShardContext = context.getQueryShardContext();
+        Function<Map<String, Object>, ExecutableScript> executableInitScript;
+        if (initScript != null) {
+            executableInitScript = queryShardContext.getLazyExecutableScript(initScript, ScriptContext.Standard.AGGS);
+        } else {
+            executableInitScript = (p) -> null;
+        }
+        Function<Map<String, Object>, SearchScript> searchMapScript = queryShardContext.getLazySearchScript(mapScript,
+            ScriptContext.Standard.AGGS);
+        Function<Map<String, Object>, ExecutableScript> executableCombineScript;
+        if (combineScript != null) {
+            executableCombineScript = queryShardContext.getLazyExecutableScript(combineScript, ScriptContext.Standard.AGGS);
+        } else {
+            executableCombineScript = (p) -> null;
+        }
+        return new ScriptedMetricAggregatorFactory(name, searchMapScript, executableInitScript, executableCombineScript, reduceScript,
+                params, context, parent, subfactoriesBuilder, metaData);
     }
 
+
     @Override
     protected XContentBuilder internalXContent(XContentBuilder builder, Params builderParams) throws IOException {
         builder.startObject();
@@ -231,16 +250,16 @@ public static ScriptedMetricAggregationBuilder parse(String aggregationName, Que
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.START_OBJECT || token == XContentParser.Token.VALUE_STRING) {
-                if (context.getParseFieldMatcher().match(currentFieldName, INIT_SCRIPT_FIELD)) {
-                    initScript = Script.parse(parser, context.getParseFieldMatcher(), context.getDefaultScriptLanguage());
-                } else if (context.getParseFieldMatcher().match(currentFieldName, MAP_SCRIPT_FIELD)) {
-                    mapScript = Script.parse(parser, context.getParseFieldMatcher(), context.getDefaultScriptLanguage());
-                } else if (context.getParseFieldMatcher().match(currentFieldName, COMBINE_SCRIPT_FIELD)) {
-                    combineScript = Script.parse(parser, context.getParseFieldMatcher(), context.getDefaultScriptLanguage());
-                } else if (context.getParseFieldMatcher().match(currentFieldName, REDUCE_SCRIPT_FIELD)) {
-                    reduceScript = Script.parse(parser, context.getParseFieldMatcher(), context.getDefaultScriptLanguage());
+                if (INIT_SCRIPT_FIELD.match(currentFieldName)) {
+                    initScript = Script.parse(parser, context.getDefaultScriptLanguage());
+                } else if (MAP_SCRIPT_FIELD.match(currentFieldName)) {
+                    mapScript = Script.parse(parser, context.getDefaultScriptLanguage());
+                } else if (COMBINE_SCRIPT_FIELD.match(currentFieldName)) {
+                    combineScript = Script.parse(parser, context.getDefaultScriptLanguage());
+                } else if (REDUCE_SCRIPT_FIELD.match(currentFieldName)) {
+                    reduceScript = Script.parse(parser, context.getDefaultScriptLanguage());
                 } else if (token == XContentParser.Token.START_OBJECT &&
-                        context.getParseFieldMatcher().match(currentFieldName, PARAMS_FIELD)) {
+                        PARAMS_FIELD.match(currentFieldName)) {
                     params = parser.map();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
@@ -275,7 +294,7 @@ public static ScriptedMetricAggregationBuilder parse(String aggregationName, Que
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregator.java
index e2d3034fa1165..cee7b3402f3e4 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregator.java
@@ -23,8 +23,6 @@
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.LeafSearchScript;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptContext;
-import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.InternalAggregation;
@@ -32,10 +30,9 @@
 import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
 import org.elasticsearch.search.aggregations.metrics.MetricsAggregator;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
@@ -46,21 +43,14 @@ public class ScriptedMetricAggregator extends MetricsAggregator {
     private final Script reduceScript;
     private Map<String, Object> params;
 
-    protected ScriptedMetricAggregator(String name, Script initScript, Script mapScript, Script combineScript, Script reduceScript,
-            Map<String, Object> params, AggregationContext context, Aggregator parent, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData)
+    protected ScriptedMetricAggregator(String name, SearchScript mapScript, ExecutableScript combineScript,
+                                       Script reduceScript,
+            Map<String, Object> params, SearchContext context, Aggregator parent, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData)
             throws IOException {
         super(name, context, parent, pipelineAggregators, metaData);
         this.params = params;
-        ScriptService scriptService = context.searchContext().scriptService();
-        if (initScript != null) {
-            scriptService.executable(initScript, ScriptContext.Standard.AGGS, Collections.emptyMap()).run();
-        }
-        this.mapScript = scriptService.search(context.searchContext().lookup(), mapScript, ScriptContext.Standard.AGGS, Collections.emptyMap());
-        if (combineScript != null) {
-            this.combineScript = scriptService.executable(combineScript, ScriptContext.Standard.AGGS, Collections.emptyMap());
-        } else {
-            this.combineScript = null;
-        }
+        this.mapScript = mapScript;
+        this.combineScript = combineScript;
         this.reduceScript = reduceScript;
     }
 
@@ -73,7 +63,7 @@ public boolean needsScores() {
     public LeafBucketCollector getLeafCollector(LeafReaderContext ctx,
             final LeafBucketCollector sub) throws IOException {
         final LeafSearchScript leafMapScript = mapScript.getLeafSearchScript(ctx);
-        return new LeafBucketCollectorBase(sub, mapScript) {
+        return new LeafBucketCollectorBase(sub, leafMapScript) {
             @Override
             public void collect(int doc, long bucket) throws IOException {
                 assert bucket == 0 : bucket;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorFactory.java
index f89e99f44b31e..bac2becc8e4e9 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorFactory.java
@@ -19,14 +19,14 @@
 
 package org.elasticsearch.search.aggregations.metrics.scripted;
 
+import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.Script;
+import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.SearchParseException;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
@@ -34,22 +34,23 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Map.Entry;
+import java.util.function.Function;
 
 public class ScriptedMetricAggregatorFactory extends AggregatorFactory<ScriptedMetricAggregatorFactory> {
 
-    private final Script initScript;
-    private final Script mapScript;
-    private final Script combineScript;
+    private final Function<Map<String, Object>, SearchScript> mapScript;
+    private final Function<Map<String, Object>, ExecutableScript> combineScript;
     private final Script reduceScript;
     private final Map<String, Object> params;
+    private final Function<Map<String, Object>, ExecutableScript> initScript;
 
-    public ScriptedMetricAggregatorFactory(String name, Type type, Script initScript, Script mapScript, Script combineScript,
-            Script reduceScript, Map<String, Object> params, AggregationContext context, AggregatorFactory<?> parent,
+    public ScriptedMetricAggregatorFactory(String name, Function<Map<String, Object>, SearchScript> mapScript,
+            Function<Map<String, Object>, ExecutableScript> initScript, Function<Map<String, Object>, ExecutableScript> combineScript,
+            Script reduceScript, Map<String, Object> params, SearchContext context, AggregatorFactory<?> parent,
             AggregatorFactories.Builder subFactories, Map<String, Object> metaData) throws IOException {
-        super(name, type, context, parent, subFactories, metaData);
-        this.initScript = initScript;
+        super(name, context, parent, subFactories, metaData);
         this.mapScript = mapScript;
+        this.initScript = initScript;
         this.combineScript = combineScript;
         this.reduceScript = reduceScript;
         this.params = params;
@@ -63,21 +64,23 @@ public Aggregator createInternal(Aggregator parent, boolean collectsFromSingleBu
         }
         Map<String, Object> params = this.params;
         if (params != null) {
-            params = deepCopyParams(params, context.searchContext());
+            params = deepCopyParams(params, context);
         } else {
             params = new HashMap<>();
             params.put("_agg", new HashMap<String, Object>());
         }
-        return new ScriptedMetricAggregator(name, insertParams(initScript, params), insertParams(mapScript, params),
-                insertParams(combineScript, params), deepCopyScript(reduceScript, context.searchContext()), params, context, parent,
-                pipelineAggregators, metaData);
-    }
 
-    private static Script insertParams(Script script, Map<String, Object> params) {
-        if (script == null) {
-            return null;
+        final ExecutableScript initScript = this.initScript.apply(params);
+        final SearchScript mapScript = this.mapScript.apply(params);
+        final ExecutableScript combineScript = this.combineScript.apply(params);
+
+        final Script reduceScript = deepCopyScript(this.reduceScript, context);
+        if (initScript != null) {
+            initScript.run();
         }
-        return new Script(script.getScript(), script.getType(), script.getLang(), params);
+        return new ScriptedMetricAggregator(name, mapScript,
+                combineScript, reduceScript, params, context, parent,
+                pipelineAggregators, metaData);
     }
 
     private static Script deepCopyScript(Script script, SearchContext context) {
@@ -86,7 +89,7 @@ private static Script deepCopyScript(Script script, SearchContext context) {
             if (params != null) {
                 params = deepCopyParams(params, context);
             }
-            return new Script(script.getScript(), script.getType(), script.getLang(), params);
+            return new Script(script.getType(), script.getLang(), script.getIdOrCode(), params);
         } else {
             return null;
         }
@@ -98,26 +101,27 @@ private static <T> T deepCopyParams(T original, SearchContext context) {
         if (original instanceof Map) {
             Map<?, ?> originalMap = (Map<?, ?>) original;
             Map<Object, Object> clonedMap = new HashMap<>();
-            for (Entry<?, ?> e : originalMap.entrySet()) {
+            for (Map.Entry<?, ?> e : originalMap.entrySet()) {
                 clonedMap.put(deepCopyParams(e.getKey(), context), deepCopyParams(e.getValue(), context));
             }
             clone = (T) clonedMap;
         } else if (original instanceof List) {
             List<?> originalList = (List<?>) original;
-            List<Object> clonedList = new ArrayList<Object>();
+            List<Object> clonedList = new ArrayList<>();
             for (Object o : originalList) {
                 clonedList.add(deepCopyParams(o, context));
             }
             clone = (T) clonedList;
         } else if (original instanceof String || original instanceof Integer || original instanceof Long || original instanceof Short
-                || original instanceof Byte || original instanceof Float || original instanceof Double || original instanceof Character
-                || original instanceof Boolean) {
+            || original instanceof Byte || original instanceof Float || original instanceof Double || original instanceof Character
+            || original instanceof Boolean) {
             clone = original;
         } else {
             throw new SearchParseException(context,
-                    "Can only clone primitives, String, ArrayList, and HashMap. Found: " + original.getClass().getCanonicalName(), null);
+                "Can only clone primitives, String, ArrayList, and HashMap. Found: " + original.getClass().getCanonicalName(), null);
         }
         return clone;
     }
 
+
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/InternalStats.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/InternalStats.java
index e060826c24c0b..253825b75e1b2 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/InternalStats.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/InternalStats.java
@@ -111,11 +111,6 @@ public double getSum() {
         return sum;
     }
 
-    @Override
-    public String getCountAsString() {
-        return valueAsString(Metrics.count.name());
-    }
-
     @Override
     public String getMinAsString() {
         return valueAsString(Metrics.min.name());
@@ -181,21 +176,28 @@ static class Fields {
     @Override
     public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
         builder.field(Fields.COUNT, count);
-        builder.field(Fields.MIN, count != 0 ? min : null);
-        builder.field(Fields.MAX, count != 0 ? max : null);
-        builder.field(Fields.AVG, count != 0 ? getAvg() : null);
-        builder.field(Fields.SUM, count != 0 ? sum : null);
-        if (count != 0 && format != DocValueFormat.RAW) {
-            builder.field(Fields.MIN_AS_STRING, format.format(min));
-            builder.field(Fields.MAX_AS_STRING, format.format(max));
-            builder.field(Fields.AVG_AS_STRING, format.format(getAvg()));
-            builder.field(Fields.SUM_AS_STRING, format.format(sum));
+        if (count != 0) {
+            builder.field(Fields.MIN, min);
+            builder.field(Fields.MAX, max);
+            builder.field(Fields.AVG, getAvg());
+            builder.field(Fields.SUM, sum);
+            if (format != DocValueFormat.RAW) {
+                builder.field(Fields.MIN_AS_STRING, format.format(min));
+                builder.field(Fields.MAX_AS_STRING, format.format(max));
+                builder.field(Fields.AVG_AS_STRING, format.format(getAvg()));
+                builder.field(Fields.SUM_AS_STRING, format.format(sum));
+            }
+        } else {
+            builder.nullField(Fields.MIN);
+            builder.nullField(Fields.MAX);
+            builder.nullField(Fields.AVG);
+            builder.nullField(Fields.SUM);
         }
-        otherStatsToXCotent(builder, params);
+        otherStatsToXContent(builder, params);
         return builder;
     }
 
-    protected XContentBuilder otherStatsToXCotent(XContentBuilder builder, Params params) throws IOException {
+    protected XContentBuilder otherStatsToXContent(XContentBuilder builder, Params params) throws IOException {
         return builder;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/ParsedStats.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/ParsedStats.java
new file mode 100644
index 0000000000000..239548ecdebc6
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/ParsedStats.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.stats;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.ParsedAggregation;
+import org.elasticsearch.search.aggregations.metrics.stats.InternalStats.Fields;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+public class ParsedStats extends ParsedAggregation implements Stats {
+
+    protected long count;
+    protected double min;
+    protected double max;
+    protected double sum;
+    protected double avg;
+
+    protected final Map<String, String> valueAsString = new HashMap<>();
+
+    @Override
+    public long getCount() {
+        return count;
+    }
+
+    @Override
+    public double getMin() {
+        return min;
+    }
+
+    @Override
+    public double getMax() {
+        return max;
+    }
+
+    @Override
+    public double getAvg() {
+        return avg;
+    }
+
+    @Override
+    public double getSum() {
+        return sum;
+    }
+
+    @Override
+    public String getMinAsString() {
+        return valueAsString.getOrDefault(Fields.MIN_AS_STRING, Double.toString(min));
+    }
+
+    @Override
+    public String getMaxAsString() {
+        return valueAsString.getOrDefault(Fields.MAX_AS_STRING, Double.toString(max));
+    }
+
+    @Override
+    public String getAvgAsString() {
+        return valueAsString.getOrDefault(Fields.AVG_AS_STRING, Double.toString(avg));
+    }
+
+    @Override
+    public String getSumAsString() {
+        return valueAsString.getOrDefault(Fields.SUM_AS_STRING, Double.toString(sum));
+    }
+
+    @Override
+    public String getType() {
+        return StatsAggregationBuilder.NAME;
+    }
+
+    @Override
+    protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+        builder.field(Fields.COUNT, count);
+        if (count != 0) {
+            builder.field(Fields.MIN, min);
+            builder.field(Fields.MAX, max);
+            builder.field(Fields.AVG, avg);
+            builder.field(Fields.SUM, sum);
+            if (valueAsString.get(Fields.MIN_AS_STRING) != null) {
+                builder.field(Fields.MIN_AS_STRING, getMinAsString());
+                builder.field(Fields.MAX_AS_STRING, getMaxAsString());
+                builder.field(Fields.AVG_AS_STRING, getAvgAsString());
+                builder.field(Fields.SUM_AS_STRING, getSumAsString());
+            }
+        } else {
+            builder.nullField(Fields.MIN);
+            builder.nullField(Fields.MAX);
+            builder.nullField(Fields.AVG);
+            builder.nullField(Fields.SUM);
+        }
+        otherStatsToXContent(builder, params);
+        return builder;
+    }
+
+    private static final ObjectParser<ParsedStats, Void> PARSER = new ObjectParser<>(ParsedStats.class.getSimpleName(), true,
+            ParsedStats::new);
+
+    static {
+        declareStatsFields(PARSER);
+    }
+
+    protected static void declareStatsFields(ObjectParser<? extends ParsedStats, Void> objectParser) {
+        declareAggregationFields(objectParser);
+        objectParser.declareLong((agg, value) -> agg.count = value, new ParseField(Fields.COUNT));
+        objectParser.declareField((agg, value) -> agg.min = value, (parser, context) -> parseDouble(parser, Double.POSITIVE_INFINITY),
+                new ParseField(Fields.MIN), ValueType.DOUBLE_OR_NULL);
+        objectParser.declareField((agg, value) -> agg.max = value, (parser, context) -> parseDouble(parser, Double.NEGATIVE_INFINITY),
+                new ParseField(Fields.MAX), ValueType.DOUBLE_OR_NULL);
+        objectParser.declareField((agg, value) -> agg.avg = value, (parser, context) -> parseDouble(parser, 0), new ParseField(Fields.AVG),
+                ValueType.DOUBLE_OR_NULL);
+        objectParser.declareField((agg, value) -> agg.sum = value, (parser, context) -> parseDouble(parser, 0), new ParseField(Fields.SUM),
+                ValueType.DOUBLE_OR_NULL);
+        objectParser.declareString((agg, value) -> agg.valueAsString.put(Fields.MIN_AS_STRING, value),
+                new ParseField(Fields.MIN_AS_STRING));
+        objectParser.declareString((agg, value) -> agg.valueAsString.put(Fields.MAX_AS_STRING, value),
+                new ParseField(Fields.MAX_AS_STRING));
+        objectParser.declareString((agg, value) -> agg.valueAsString.put(Fields.AVG_AS_STRING, value),
+                new ParseField(Fields.AVG_AS_STRING));
+        objectParser.declareString((agg, value) -> agg.valueAsString.put(Fields.SUM_AS_STRING, value),
+                new ParseField(Fields.SUM_AS_STRING));
+    }
+
+    public static ParsedStats fromXContent(XContentParser parser, final String name) {
+        ParsedStats parsedStats = PARSER.apply(parser, null);
+        parsedStats.setName(name);
+        return parsedStats;
+    }
+
+    protected XContentBuilder otherStatsToXContent(XContentBuilder builder, Params params) throws IOException {
+        return builder;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/Stats.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/Stats.java
index 4910dc140026d..46620f51dc2fc 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/Stats.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/Stats.java
@@ -50,11 +50,6 @@ public interface Stats extends NumericMetricsAggregation.MultiValue {
      */
     double getSum();
 
-    /**
-     * @return The number of values that were aggregated as a String.
-     */
-    String getCountAsString();
-
     /**
      * @return The minimum value of all aggregated values as a String.
      */
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggregationBuilder.java
index e26edbcc290e8..390be44d74716 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggregationBuilder.java
@@ -21,33 +21,45 @@
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 
 public class StatsAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOnly<ValuesSource.Numeric, StatsAggregationBuilder> {
     public static final String NAME = "stats";
-    private static final Type TYPE = new Type(NAME);
+
+    private static final ObjectParser<StatsAggregationBuilder, QueryParseContext> PARSER;
+    static {
+        PARSER = new ObjectParser<>(StatsAggregationBuilder.NAME);
+        ValuesSourceParserHelper.declareNumericFields(PARSER, true, true, false);
+    }
+
+    public static AggregationBuilder parse(String aggregationName, QueryParseContext context) throws IOException {
+        return PARSER.parse(context.parser(), new StatsAggregationBuilder(aggregationName), context);
+    }
 
     public StatsAggregationBuilder(String name) {
-        super(name, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(name, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     /**
      * Read from a stream.
      */
     public StatsAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(in, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     @Override
@@ -56,9 +68,9 @@ protected void innerWriteTo(StreamOutput out) {
     }
 
     @Override
-    protected StatsAggregatorFactory innerBuild(AggregationContext context, ValuesSourceConfig<Numeric> config,
+    protected StatsAggregatorFactory innerBuild(SearchContext context, ValuesSourceConfig<Numeric> config,
             AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
-        return new StatsAggregatorFactory(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        return new StatsAggregatorFactory(name, config, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
@@ -77,7 +89,7 @@ protected boolean innerEquals(Object obj) {
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggregator.java
index 374cbcaf0e6c0..8357cae3cd344 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggregator.java
@@ -31,8 +31,8 @@
 import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
 import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregator;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -53,7 +53,7 @@ public class StatsAggregator extends NumericMetricsAggregator.MultiValue {
 
 
     public StatsAggregator(String name, ValuesSource.Numeric valuesSource, DocValueFormat format,
-                           AggregationContext context,
+            SearchContext context,
                            Aggregator parent, List<PipelineAggregator> pipelineAggregators,
                            Map<String, Object> metaData) throws IOException {
         super(name, context, parent, pipelineAggregators, metaData);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggregatorFactory.java
index a4f78f53d0788..a6e59d7c75bf0 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggregatorFactory.java
@@ -22,13 +22,12 @@
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -36,9 +35,9 @@
 
 public class StatsAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource.Numeric, StatsAggregatorFactory> {
 
-    public StatsAggregatorFactory(String name, Type type, ValuesSourceConfig<Numeric> config, AggregationContext context,
+    public StatsAggregatorFactory(String name, ValuesSourceConfig<Numeric> config, SearchContext context,
             AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsParser.java
deleted file mode 100644
index 60e3d2ef0aad0..0000000000000
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsParser.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.search.aggregations.metrics.stats;
-
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.search.aggregations.support.AbstractValuesSourceParser.NumericValuesSourceParser;
-import org.elasticsearch.search.aggregations.support.XContentParseContext;
-import org.elasticsearch.search.aggregations.support.ValueType;
-import org.elasticsearch.search.aggregations.support.ValuesSourceType;
-
-import java.io.IOException;
-import java.util.Map;
-
-/**
- *
- */
-public class StatsParser extends NumericValuesSourceParser {
-
-    public StatsParser() {
-        super(true, true, false);
-    }
-
-    @Override
-    protected boolean token(String aggregationName, String currentFieldName, XContentParser.Token token,
-                            XContentParseContext context, Map<ParseField, Object> otherOptions) throws IOException {
-        return false;
-    }
-
-    @Override
-    protected StatsAggregationBuilder createFactory(String aggregationName, ValuesSourceType valuesSourceType,
-                                                    ValueType targetValueType, Map<ParseField, Object> otherOptions) {
-        return new StatsAggregationBuilder(aggregationName);
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStats.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStats.java
index 1b235a6cfecaa..8a198a5825a3d 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStats.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStats.java
@@ -67,7 +67,7 @@ public interface ExtendedStats extends Stats {
     String getVarianceAsString();
 
 
-    public enum Bounds {
+    enum Bounds {
         UPPER, LOWER
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsAggregationBuilder.java
index a20b8fa676498..94857c8753f08 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsAggregationBuilder.java
@@ -21,17 +21,20 @@
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -39,19 +42,29 @@
 public class ExtendedStatsAggregationBuilder
         extends ValuesSourceAggregationBuilder.LeafOnly<ValuesSource.Numeric, ExtendedStatsAggregationBuilder> {
     public static final String NAME = "extended_stats";
-    public static final Type TYPE = new Type(NAME);
+
+    private static final ObjectParser<ExtendedStatsAggregationBuilder, QueryParseContext> PARSER;
+    static {
+        PARSER = new ObjectParser<>(ExtendedStatsAggregationBuilder.NAME);
+        ValuesSourceParserHelper.declareNumericFields(PARSER, true, true, false);
+        PARSER.declareDouble(ExtendedStatsAggregationBuilder::sigma, ExtendedStatsAggregator.SIGMA_FIELD);
+    }
+
+    public static AggregationBuilder parse(String aggregationName, QueryParseContext context) throws IOException {
+        return PARSER.parse(context.parser(), new ExtendedStatsAggregationBuilder(aggregationName), context);
+    }
 
     private double sigma = 2.0;
 
     public ExtendedStatsAggregationBuilder(String name) {
-        super(name, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(name, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     /**
      * Read from a stream.
      */
     public ExtendedStatsAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(in, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
         sigma = in.readDouble();
     }
 
@@ -73,9 +86,9 @@ public double sigma() {
     }
 
     @Override
-    protected ExtendedStatsAggregatorFactory innerBuild(AggregationContext context, ValuesSourceConfig<Numeric> config,
+    protected ExtendedStatsAggregatorFactory innerBuild(SearchContext context, ValuesSourceConfig<Numeric> config,
             AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
-        return new ExtendedStatsAggregatorFactory(name, type, config, sigma, context, parent, subFactoriesBuilder, metaData);
+        return new ExtendedStatsAggregatorFactory(name, config, sigma, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
@@ -96,7 +109,7 @@ protected boolean innerEquals(Object obj) {
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsAggregator.java
index 7715d4b713ec0..0cb73118632e0 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsAggregator.java
@@ -32,8 +32,8 @@
 import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
 import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregator;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -57,7 +57,7 @@ public class ExtendedStatsAggregator extends NumericMetricsAggregator.MultiValue
     DoubleArray sumOfSqrs;
 
     public ExtendedStatsAggregator(String name, ValuesSource.Numeric valuesSource, DocValueFormat formatter,
-            AggregationContext context, Aggregator parent, double sigma, List<PipelineAggregator> pipelineAggregators,
+            SearchContext context, Aggregator parent, double sigma, List<PipelineAggregator> pipelineAggregators,
             Map<String, Object> metaData)
             throws IOException {
         super(name, context, parent, pipelineAggregators, metaData);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsAggregatorFactory.java
index 82b0862482e56..521ea8f68a67d 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsAggregatorFactory.java
@@ -22,13 +22,12 @@
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -38,10 +37,10 @@ public class ExtendedStatsAggregatorFactory extends ValuesSourceAggregatorFactor
 
     private final double sigma;
 
-    public ExtendedStatsAggregatorFactory(String name, Type type, ValuesSourceConfig<Numeric> config, double sigma,
-            AggregationContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
+    public ExtendedStatsAggregatorFactory(String name, ValuesSourceConfig<Numeric> config, double sigma,
+            SearchContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
             Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.sigma = sigma;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsParser.java
deleted file mode 100644
index 9644d26e93a6a..0000000000000
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsParser.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.search.aggregations.metrics.stats.extended;
-
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.search.aggregations.support.AbstractValuesSourceParser.NumericValuesSourceParser;
-import org.elasticsearch.search.aggregations.support.XContentParseContext;
-import org.elasticsearch.search.aggregations.support.ValueType;
-import org.elasticsearch.search.aggregations.support.ValuesSourceType;
-
-import java.io.IOException;
-import java.util.Map;
-
-/**
- *
- */
-public class ExtendedStatsParser extends NumericValuesSourceParser {
-
-    public ExtendedStatsParser() {
-        super(true, true, false);
-    }
-
-    @Override
-    protected boolean token(String aggregationName, String currentFieldName, XContentParser.Token token,
-                            XContentParseContext context, Map<ParseField, Object> otherOptions) throws IOException {
-        if (context.matchField(currentFieldName, ExtendedStatsAggregator.SIGMA_FIELD)) {
-            if (token.isValue()) {
-                otherOptions.put(ExtendedStatsAggregator.SIGMA_FIELD, context.getParser().doubleValue());
-                return true;
-            }
-        }
-        return false;
-    }
-
-    @Override
-    protected ExtendedStatsAggregationBuilder createFactory(String aggregationName, ValuesSourceType valuesSourceType,
-                                                            ValueType targetValueType, Map<ParseField, Object> otherOptions) {
-        ExtendedStatsAggregationBuilder factory = new ExtendedStatsAggregationBuilder(aggregationName);
-        Double sigma = (Double) otherOptions.get(ExtendedStatsAggregator.SIGMA_FIELD);
-        if (sigma != null) {
-            factory.sigma(sigma);
-        }
-        return factory;
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/InternalExtendedStats.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/InternalExtendedStats.java
index d848001171c6b..be999849c7fa7 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/InternalExtendedStats.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/InternalExtendedStats.java
@@ -164,25 +164,38 @@ static class Fields {
     }
 
     @Override
-    protected XContentBuilder otherStatsToXCotent(XContentBuilder builder, Params params) throws IOException {
-        builder.field(Fields.SUM_OF_SQRS, count != 0 ? sumOfSqrs : null);
-        builder.field(Fields.VARIANCE, count != 0 ? getVariance() : null);
-        builder.field(Fields.STD_DEVIATION, count != 0 ? getStdDeviation() : null);
-        builder.startObject(Fields.STD_DEVIATION_BOUNDS)
-                .field(Fields.UPPER, count != 0 ? getStdDeviationBound(Bounds.UPPER) : null)
-                .field(Fields.LOWER, count != 0 ? getStdDeviationBound(Bounds.LOWER) : null)
-                .endObject();
-
-        if (count != 0 && format != DocValueFormat.RAW) {
-            builder.field(Fields.SUM_OF_SQRS_AS_STRING, format.format(sumOfSqrs));
-            builder.field(Fields.VARIANCE_AS_STRING, format.format(getVariance()));
-            builder.field(Fields.STD_DEVIATION_AS_STRING, getStdDeviationAsString());
-
-            builder.startObject(Fields.STD_DEVIATION_BOUNDS_AS_STRING)
-                    .field(Fields.UPPER, getStdDeviationBoundAsString(Bounds.UPPER))
-                    .field(Fields.LOWER, getStdDeviationBoundAsString(Bounds.LOWER))
-                    .endObject();
-
+    protected XContentBuilder otherStatsToXContent(XContentBuilder builder, Params params) throws IOException {
+        if (count != 0) {
+            builder.field(Fields.SUM_OF_SQRS, sumOfSqrs);
+            builder.field(Fields.VARIANCE, getVariance());
+            builder.field(Fields.STD_DEVIATION, getStdDeviation());
+            builder.startObject(Fields.STD_DEVIATION_BOUNDS);
+            {
+                builder.field(Fields.UPPER, getStdDeviationBound(Bounds.UPPER));
+                builder.field(Fields.LOWER, getStdDeviationBound(Bounds.LOWER));
+            }
+            builder.endObject();
+            if (format != DocValueFormat.RAW) {
+                builder.field(Fields.SUM_OF_SQRS_AS_STRING, format.format(sumOfSqrs));
+                builder.field(Fields.VARIANCE_AS_STRING, format.format(getVariance()));
+                builder.field(Fields.STD_DEVIATION_AS_STRING, getStdDeviationAsString());
+                builder.startObject(Fields.STD_DEVIATION_BOUNDS_AS_STRING);
+                {
+                    builder.field(Fields.UPPER, getStdDeviationBoundAsString(Bounds.UPPER));
+                    builder.field(Fields.LOWER, getStdDeviationBoundAsString(Bounds.LOWER));
+                }
+                builder.endObject();
+            }
+        } else {
+            builder.nullField(Fields.SUM_OF_SQRS);
+            builder.nullField(Fields.VARIANCE);
+            builder.nullField(Fields.STD_DEVIATION);
+            builder.startObject(Fields.STD_DEVIATION_BOUNDS);
+            {
+                builder.nullField(Fields.UPPER);
+                builder.nullField(Fields.LOWER);
+            }
+            builder.endObject();
         }
         return builder;
     }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ParsedExtendedStats.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ParsedExtendedStats.java
new file mode 100644
index 0000000000000..59311127368f5
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ParsedExtendedStats.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.stats.extended;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.metrics.stats.ParsedStats;
+import org.elasticsearch.search.aggregations.metrics.stats.extended.InternalExtendedStats.Fields;
+
+import java.io.IOException;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+
+public class ParsedExtendedStats extends ParsedStats implements ExtendedStats {
+
+    protected double sumOfSquares;
+    protected double variance;
+    protected double stdDeviation;
+    protected double stdDeviationBoundUpper;
+    protected double stdDeviationBoundLower;
+    protected double sum;
+    protected double avg;
+
+    @Override
+    public String getType() {
+        return ExtendedStatsAggregationBuilder.NAME;
+    }
+
+    @Override
+    public double getSumOfSquares() {
+        return sumOfSquares;
+    }
+
+    @Override
+    public double getVariance() {
+        return variance;
+    }
+
+    @Override
+    public double getStdDeviation() {
+        return stdDeviation;
+    }
+
+    private void setStdDeviationBounds(Tuple<Double, Double> bounds) {
+        this.stdDeviationBoundLower = bounds.v1();
+        this.stdDeviationBoundUpper = bounds.v2();
+    }
+
+    @Override
+    public double getStdDeviationBound(Bounds bound) {
+        return (bound.equals(Bounds.LOWER)) ? stdDeviationBoundLower : stdDeviationBoundUpper;
+    }
+
+    @Override
+    public String getStdDeviationAsString() {
+        return valueAsString.getOrDefault(Fields.STD_DEVIATION_AS_STRING, Double.toString(stdDeviation));
+    }
+
+    private void setStdDeviationBoundsAsString(Tuple<String, String> boundsAsString) {
+        this.valueAsString.put(Fields.STD_DEVIATION_BOUNDS_AS_STRING + "_lower", boundsAsString.v1());
+        this.valueAsString.put(Fields.STD_DEVIATION_BOUNDS_AS_STRING + "_upper", boundsAsString.v2());
+    }
+
+    @Override
+    public String getStdDeviationBoundAsString(Bounds bound) {
+        if (bound.equals(Bounds.LOWER)) {
+            return valueAsString.getOrDefault(Fields.STD_DEVIATION_BOUNDS_AS_STRING + "_lower", Double.toString(stdDeviationBoundLower));
+        } else {
+            return valueAsString.getOrDefault(Fields.STD_DEVIATION_BOUNDS_AS_STRING + "_upper", Double.toString(stdDeviationBoundUpper));
+        }
+    }
+
+    @Override
+    public String getSumOfSquaresAsString() {
+        return valueAsString.getOrDefault(Fields.SUM_OF_SQRS_AS_STRING, Double.toString(sumOfSquares));
+    }
+
+    @Override
+    public String getVarianceAsString() {
+        return valueAsString.getOrDefault(Fields.VARIANCE_AS_STRING, Double.toString(variance));
+    }
+
+    @Override
+    protected XContentBuilder otherStatsToXContent(XContentBuilder builder, Params params) throws IOException {
+        if (count != 0) {
+            builder.field(Fields.SUM_OF_SQRS, sumOfSquares);
+            builder.field(Fields.VARIANCE, getVariance());
+            builder.field(Fields.STD_DEVIATION, getStdDeviation());
+            builder.startObject(Fields.STD_DEVIATION_BOUNDS);
+            {
+                builder.field(Fields.UPPER, getStdDeviationBound(Bounds.UPPER));
+                builder.field(Fields.LOWER, getStdDeviationBound(Bounds.LOWER));
+            }
+            builder.endObject();
+            if (valueAsString.containsKey(Fields.SUM_OF_SQRS_AS_STRING)) {
+                builder.field(Fields.SUM_OF_SQRS_AS_STRING, getSumOfSquaresAsString());
+                builder.field(Fields.VARIANCE_AS_STRING, getVarianceAsString());
+                builder.field(Fields.STD_DEVIATION_AS_STRING, getStdDeviationAsString());
+                builder.startObject(Fields.STD_DEVIATION_BOUNDS_AS_STRING);
+                {
+                    builder.field(Fields.UPPER, getStdDeviationBoundAsString(Bounds.UPPER));
+                    builder.field(Fields.LOWER, getStdDeviationBoundAsString(Bounds.LOWER));
+                }
+                builder.endObject();
+            }
+        } else {
+            builder.nullField(Fields.SUM_OF_SQRS);
+            builder.nullField(Fields.VARIANCE);
+            builder.nullField(Fields.STD_DEVIATION);
+            builder.startObject(Fields.STD_DEVIATION_BOUNDS);
+            {
+                builder.nullField(Fields.UPPER);
+                builder.nullField(Fields.LOWER);
+            }
+            builder.endObject();
+        }
+        return builder;
+    }
+
+    private static final ObjectParser<ParsedExtendedStats, Void> PARSER = new ObjectParser<>(ParsedExtendedStats.class.getSimpleName(),
+            true, ParsedExtendedStats::new);
+
+    private static final ConstructingObjectParser<Tuple<Double, Double>, Void> STD_BOUNDS_PARSER = new ConstructingObjectParser<>(
+            ParsedExtendedStats.class.getSimpleName() + "_STD_BOUNDS", true, args -> new Tuple<>((Double) args[0], (Double) args[1]));
+
+    private static final ConstructingObjectParser<Tuple<String, String>, Void> STD_BOUNDS_AS_STRING_PARSER = new ConstructingObjectParser<>(
+            ParsedExtendedStats.class.getSimpleName() + "_STD_BOUNDS_AS_STRING", true,
+            args -> new Tuple<>((String) args[0], (String) args[1]));
+
+    static {
+        STD_BOUNDS_PARSER.declareField(constructorArg(), (parser, context) -> parseDouble(parser, 0),
+                new ParseField(Fields.LOWER), ValueType.DOUBLE_OR_NULL);
+        STD_BOUNDS_PARSER.declareField(constructorArg(), (parser, context) -> parseDouble(parser, 0),
+                new ParseField(Fields.UPPER), ValueType.DOUBLE_OR_NULL);
+        STD_BOUNDS_AS_STRING_PARSER.declareString(constructorArg(), new ParseField(Fields.LOWER));
+        STD_BOUNDS_AS_STRING_PARSER.declareString(constructorArg(), new ParseField(Fields.UPPER));
+        declareExtendedStatsFields(PARSER);
+    }
+
+    protected static void declareExtendedStatsFields(ObjectParser<? extends ParsedExtendedStats, Void> objectParser) {
+        declareAggregationFields(objectParser);
+        declareStatsFields(objectParser);
+        objectParser.declareField((agg, value) -> agg.sumOfSquares = value, (parser, context) -> parseDouble(parser, 0),
+                new ParseField(Fields.SUM_OF_SQRS), ValueType.DOUBLE_OR_NULL);
+        objectParser.declareField((agg, value) -> agg.variance = value, (parser, context) -> parseDouble(parser, 0),
+                new ParseField(Fields.VARIANCE), ValueType.DOUBLE_OR_NULL);
+        objectParser.declareField((agg, value) -> agg.stdDeviation = value, (parser, context) -> parseDouble(parser, 0),
+                new ParseField(Fields.STD_DEVIATION), ValueType.DOUBLE_OR_NULL);
+        objectParser.declareObject(ParsedExtendedStats::setStdDeviationBounds, STD_BOUNDS_PARSER,
+                new ParseField(Fields.STD_DEVIATION_BOUNDS));
+        objectParser.declareString((agg, value) -> agg.valueAsString.put(Fields.SUM_OF_SQRS_AS_STRING, value),
+                new ParseField(Fields.SUM_OF_SQRS_AS_STRING));
+        objectParser.declareString((agg, value) -> agg.valueAsString.put(Fields.VARIANCE_AS_STRING, value),
+                new ParseField(Fields.VARIANCE_AS_STRING));
+        objectParser.declareString((agg, value) -> agg.valueAsString.put(Fields.STD_DEVIATION_AS_STRING, value),
+                new ParseField(Fields.STD_DEVIATION_AS_STRING));
+        objectParser.declareObject(ParsedExtendedStats::setStdDeviationBoundsAsString, STD_BOUNDS_AS_STRING_PARSER,
+                new ParseField(Fields.STD_DEVIATION_BOUNDS_AS_STRING));
+    }
+
+    public static ParsedExtendedStats fromXContent(XContentParser parser, final String name) {
+        ParsedExtendedStats parsedStats = PARSER.apply(parser, null);
+        parsedStats.setName(name);
+        return parsedStats;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/InternalSum.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/InternalSum.java
index 70c69b65a1759..b93d97a663539 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/InternalSum.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/InternalSum.java
@@ -81,9 +81,9 @@ public InternalSum doReduce(List<InternalAggregation> aggregations, ReduceContex
 
     @Override
     public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
-        builder.field(CommonFields.VALUE, sum);
+        builder.field(CommonFields.VALUE.getPreferredName(), sum);
         if (format != DocValueFormat.RAW) {
-            builder.field(CommonFields.VALUE_AS_STRING, format.format(sum));
+            builder.field(CommonFields.VALUE_AS_STRING.getPreferredName(), format.format(sum));
         }
         return builder;
     }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/ParsedSum.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/ParsedSum.java
new file mode 100644
index 0000000000000..a51f03d356549
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/ParsedSum.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.sum;
+
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.metrics.ParsedSingleValueNumericMetricsAggregation;
+
+import java.io.IOException;
+
+public class ParsedSum extends ParsedSingleValueNumericMetricsAggregation implements Sum {
+
+    @Override
+    public double getValue() {
+        return value();
+    }
+
+    @Override
+    public String getType() {
+        return SumAggregationBuilder.NAME;
+    }
+
+    @Override
+    protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+        builder.field(CommonFields.VALUE.getPreferredName(), value);
+        if (valueAsString != null) {
+            builder.field(CommonFields.VALUE_AS_STRING.getPreferredName(), valueAsString);
+        }
+        return builder;
+    }
+
+    private static final ObjectParser<ParsedSum, Void> PARSER = new ObjectParser<>(ParsedSum.class.getSimpleName(), true, ParsedSum::new);
+
+    static {
+        declareSingleValueFields(PARSER, Double.NEGATIVE_INFINITY);
+    }
+
+    public static ParsedSum fromXContent(XContentParser parser, final String name) {
+        ParsedSum sum = PARSER.apply(parser, null);
+        sum.setName(name);
+        return sum;
+    }
+}
\ No newline at end of file
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumAggregationBuilder.java
index 2f285b146c1ae..7118b14d0cdca 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumAggregationBuilder.java
@@ -21,33 +21,45 @@
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 
 public class SumAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOnly<ValuesSource.Numeric, SumAggregationBuilder> {
     public static final String NAME = "sum";
-    private static final Type TYPE = new Type(NAME);
+
+    private static final ObjectParser<SumAggregationBuilder, QueryParseContext> PARSER;
+    static {
+        PARSER = new ObjectParser<>(SumAggregationBuilder.NAME);
+        ValuesSourceParserHelper.declareNumericFields(PARSER, true, true, false);
+    }
+
+    public static AggregationBuilder parse(String aggregationName, QueryParseContext context) throws IOException {
+        return PARSER.parse(context.parser(), new SumAggregationBuilder(aggregationName), context);
+    }
 
     public SumAggregationBuilder(String name) {
-        super(name, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(name, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     /**
      * Read from a stream.
      */
     public SumAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(in, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     @Override
@@ -56,9 +68,9 @@ protected void innerWriteTo(StreamOutput out) {
     }
 
     @Override
-    protected SumAggregatorFactory innerBuild(AggregationContext context, ValuesSourceConfig<Numeric> config,
+    protected SumAggregatorFactory innerBuild(SearchContext context, ValuesSourceConfig<Numeric> config,
             AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
-        return new SumAggregatorFactory(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        return new SumAggregatorFactory(name, config, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
@@ -77,7 +89,7 @@ protected boolean innerEquals(Object obj) {
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumAggregator.java
index 0d6b5de4a80b8..d47ffdb1bbae4 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumAggregator.java
@@ -30,8 +30,8 @@
 import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
 import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregator;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -47,7 +47,7 @@ public class SumAggregator extends NumericMetricsAggregator.SingleValue {
 
     DoubleArray sums;
 
-    public SumAggregator(String name, ValuesSource.Numeric valuesSource, DocValueFormat formatter, AggregationContext context,
+    public SumAggregator(String name, ValuesSource.Numeric valuesSource, DocValueFormat formatter, SearchContext context,
             Aggregator parent, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException {
         super(name, context, parent, pipelineAggregators, metaData);
         this.valuesSource = valuesSource;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumAggregatorFactory.java
index 07184849c5e91..8b6103214a754 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumAggregatorFactory.java
@@ -22,13 +22,12 @@
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -36,9 +35,9 @@
 
 public class SumAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource.Numeric, SumAggregatorFactory> {
 
-    public SumAggregatorFactory(String name, Type type, ValuesSourceConfig<Numeric> config, AggregationContext context,
+    public SumAggregatorFactory(String name, ValuesSourceConfig<Numeric> config, SearchContext context,
             AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumParser.java
deleted file mode 100644
index ee82829b0a71d..0000000000000
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumParser.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.search.aggregations.metrics.sum;
-
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.search.aggregations.support.AbstractValuesSourceParser.NumericValuesSourceParser;
-import org.elasticsearch.search.aggregations.support.XContentParseContext;
-import org.elasticsearch.search.aggregations.support.ValueType;
-import org.elasticsearch.search.aggregations.support.ValuesSourceType;
-
-import java.io.IOException;
-import java.util.Map;
-
-/**
- *
- */
-public class SumParser extends NumericValuesSourceParser {
-
-    public SumParser() {
-        super(true, true, false);
-    }
-
-    @Override
-    protected boolean token(String aggregationName, String currentFieldName, XContentParser.Token token,
-                            XContentParseContext context, Map<ParseField, Object> otherOptions) throws IOException {
-        return false;
-    }
-
-    @Override
-    protected SumAggregationBuilder createFactory(String aggregationName, ValuesSourceType valuesSourceType,
-                                                  ValueType targetValueType, Map<ParseField, Object> otherOptions) {
-        return new SumAggregationBuilder(aggregationName);
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/InternalTopHits.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/InternalTopHits.java
index 8c6758b4f3567..bcef9c1388ab1 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/InternalTopHits.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/InternalTopHits.java
@@ -22,17 +22,14 @@
 import org.apache.lucene.search.Sort;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.search.TopFieldDocs;
-import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.aggregations.InternalAggregation;
-import org.elasticsearch.search.aggregations.metrics.InternalMetricsAggregation;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.internal.InternalSearchHit;
-import org.elasticsearch.search.internal.InternalSearchHits;
 
 import java.io.IOException;
 import java.util.List;
@@ -41,13 +38,13 @@
 /**
  * Results of the {@link TopHitsAggregator}.
  */
-public class InternalTopHits extends InternalMetricsAggregation implements TopHits {
+public class InternalTopHits extends InternalAggregation implements TopHits {
     private int from;
     private int size;
     private TopDocs topDocs;
-    private InternalSearchHits searchHits;
+    private SearchHits searchHits;
 
-    public InternalTopHits(String name, int from, int size, TopDocs topDocs, InternalSearchHits searchHits,
+    public InternalTopHits(String name, int from, int size, TopDocs topDocs, SearchHits searchHits,
             List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
         super(name, pipelineAggregators, metaData);
         this.from = from;
@@ -65,7 +62,7 @@ public InternalTopHits(StreamInput in) throws IOException {
         size = in.readVInt();
         topDocs = Lucene.readTopDocs(in);
         assert topDocs != null;
-        searchHits = InternalSearchHits.readSearchHits(in);
+        searchHits = SearchHits.readSearchHits(in);
     }
 
     @Override
@@ -88,47 +85,54 @@ public SearchHits getHits() {
 
     @Override
     public InternalAggregation doReduce(List<InternalAggregation> aggregations, ReduceContext reduceContext) {
-        InternalSearchHits[] shardHits = new InternalSearchHits[aggregations.size()];
+        final SearchHits[] shardHits = new SearchHits[aggregations.size()];
+        final int from;
+        final int size;
+        if (reduceContext.isFinalReduce()) {
+            from = this.from;
+            size = this.size;
+        } else {
+            // if we are not in the final reduce we need to ensure we maintain all possible elements during reduce
+            // hence for pagination we need to maintain all hits until we are in the final phase.
+            from = 0;
+            size = this.from + this.size;
+        }
 
         final TopDocs reducedTopDocs;
         final TopDocs[] shardDocs;
 
-        try {
-            if (topDocs instanceof TopFieldDocs) {
-                Sort sort = new Sort(((TopFieldDocs) topDocs).fields);
-                shardDocs = new TopFieldDocs[aggregations.size()];
-                for (int i = 0; i < shardDocs.length; i++) {
-                    InternalTopHits topHitsAgg = (InternalTopHits) aggregations.get(i);
-                    shardDocs[i] = (TopFieldDocs) topHitsAgg.topDocs;
-                    shardHits[i] = topHitsAgg.searchHits;
-                }
-                reducedTopDocs = TopDocs.merge(sort, from, size, (TopFieldDocs[]) shardDocs);
-            } else {
-                shardDocs = new TopDocs[aggregations.size()];
-                for (int i = 0; i < shardDocs.length; i++) {
-                    InternalTopHits topHitsAgg = (InternalTopHits) aggregations.get(i);
-                    shardDocs[i] = topHitsAgg.topDocs;
-                    shardHits[i] = topHitsAgg.searchHits;
-                }
-                reducedTopDocs = TopDocs.merge(from, size, shardDocs);
+        if (topDocs instanceof TopFieldDocs) {
+            Sort sort = new Sort(((TopFieldDocs) topDocs).fields);
+            shardDocs = new TopFieldDocs[aggregations.size()];
+            for (int i = 0; i < shardDocs.length; i++) {
+                InternalTopHits topHitsAgg = (InternalTopHits) aggregations.get(i);
+                shardDocs[i] = topHitsAgg.topDocs;
+                shardHits[i] = topHitsAgg.searchHits;
             }
-
-            final int[] tracker = new int[shardHits.length];
-            InternalSearchHit[] hits = new InternalSearchHit[reducedTopDocs.scoreDocs.length];
-            for (int i = 0; i < reducedTopDocs.scoreDocs.length; i++) {
-                ScoreDoc scoreDoc = reducedTopDocs.scoreDocs[i];
-                int position;
-                do {
-                    position = tracker[scoreDoc.shardIndex]++;
-                } while (shardDocs[scoreDoc.shardIndex].scoreDocs[position] != scoreDoc);
-                hits[i] = (InternalSearchHit) shardHits[scoreDoc.shardIndex].getAt(position);
+            reducedTopDocs = TopDocs.merge(sort, from, size, (TopFieldDocs[]) shardDocs, true);
+        } else {
+            shardDocs = new TopDocs[aggregations.size()];
+            for (int i = 0; i < shardDocs.length; i++) {
+                InternalTopHits topHitsAgg = (InternalTopHits) aggregations.get(i);
+                shardDocs[i] = topHitsAgg.topDocs;
+                shardHits[i] = topHitsAgg.searchHits;
             }
-            return new InternalTopHits(name, from, size, reducedTopDocs, new InternalSearchHits(hits, reducedTopDocs.totalHits,
-                    reducedTopDocs.getMaxScore()),
-                    pipelineAggregators(), getMetaData());
-        } catch (IOException e) {
-            throw ExceptionsHelper.convertToElastic(e);
+            reducedTopDocs = TopDocs.merge(from, size, shardDocs, true);
+        }
+
+        final int[] tracker = new int[shardHits.length];
+        SearchHit[] hits = new SearchHit[reducedTopDocs.scoreDocs.length];
+        for (int i = 0; i < reducedTopDocs.scoreDocs.length; i++) {
+            ScoreDoc scoreDoc = reducedTopDocs.scoreDocs[i];
+            int position;
+            do {
+                position = tracker[scoreDoc.shardIndex]++;
+            } while (shardDocs[scoreDoc.shardIndex].scoreDocs[position] != scoreDoc);
+            hits[i] = shardHits[scoreDoc.shardIndex].getAt(position);
         }
+        return new InternalTopHits(name, this.from, this.size, reducedTopDocs, new SearchHits(hits, reducedTopDocs.totalHits,
+                reducedTopDocs.getMaxScore()),
+                pipelineAggregators(), getMetaData());
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/ParsedTopHits.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/ParsedTopHits.java
new file mode 100644
index 0000000000000..362423abca8a3
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/ParsedTopHits.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.tophits;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.search.aggregations.ParsedAggregation;
+
+import java.io.IOException;
+
+public class ParsedTopHits extends ParsedAggregation implements TopHits {
+
+    private SearchHits searchHits;
+
+    @Override
+    public String getType() {
+        return TopHitsAggregationBuilder.NAME;
+    }
+
+    @Override
+    public SearchHits getHits() {
+        return searchHits;
+    }
+
+    @Override
+    protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+        return searchHits.toXContent(builder, params);
+    }
+
+    private static ObjectParser<ParsedTopHits, Void> PARSER =
+            new ObjectParser<>(ParsedTopHits.class.getSimpleName(), true, ParsedTopHits::new);
+    static {
+        declareAggregationFields(PARSER);
+        PARSER.declareObject((topHit, searchHits) -> topHit.searchHits = searchHits, (parser, context) -> SearchHits.fromXContent(parser),
+                new ParseField(SearchHits.Fields.HITS));
+    }
+
+    public static ParsedTopHits fromXContent(XContentParser parser, String name) throws IOException {
+        ParsedTopHits aggregation = PARSER.parse(parser, null);
+        aggregation.setName(name);
+        return aggregation;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregationBuilder.java
index 828d56798463a..a8ec235c56399 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregationBuilder.java
@@ -28,19 +28,21 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptContext;
+import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregationInitializationException;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder.ScriptField;
 import org.elasticsearch.search.fetch.StoredFieldsContext;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
+import org.elasticsearch.search.fetch.subphase.ScriptFieldsContext;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.sort.ScoreSortBuilder;
+import org.elasticsearch.search.sort.SortAndFormats;
 import org.elasticsearch.search.sort.SortBuilder;
 import org.elasticsearch.search.sort.SortBuilders;
 import org.elasticsearch.search.sort.SortOrder;
@@ -51,11 +53,11 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 
 public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHitsAggregationBuilder> {
     public static final String NAME = "top_hits";
-    private static final InternalAggregation.Type TYPE = new Type(NAME);
 
     private int from = 0;
     private int size = 3;
@@ -70,16 +72,16 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
     private FetchSourceContext fetchSourceContext;
 
     public TopHitsAggregationBuilder(String name) {
-        super(name, TYPE);
+        super(name);
     }
 
     /**
      * Read from a stream.
      */
     public TopHitsAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE);
+        super(in);
         explain = in.readBoolean();
-        fetchSourceContext = in.readOptionalStreamable(FetchSourceContext::new);
+        fetchSourceContext = in.readOptionalWriteable(FetchSourceContext::new);
         if (in.readBoolean()) {
             int size = in.readVInt();
             fieldDataFields = new ArrayList<>(size);
@@ -112,7 +114,7 @@ public TopHitsAggregationBuilder(StreamInput in) throws IOException {
     @Override
     protected void doWriteTo(StreamOutput out) throws IOException {
         out.writeBoolean(explain);
-        out.writeOptionalStreamable(fetchSourceContext);
+        out.writeOptionalWriteable(fetchSourceContext);
         boolean hasFieldDataFields = fieldDataFields != null;
         out.writeBoolean(hasFieldDataFields);
         if (hasFieldDataFields) {
@@ -280,11 +282,9 @@ public HighlightBuilder highlighter() {
      * every hit
      */
     public TopHitsAggregationBuilder fetchSource(boolean fetch) {
-        if (this.fetchSourceContext == null) {
-            this.fetchSourceContext = new FetchSourceContext(fetch);
-        } else {
-            this.fetchSourceContext.fetchSource(fetch);
-        }
+        FetchSourceContext fetchSourceContext = this.fetchSourceContext != null ? this.fetchSourceContext
+            : FetchSourceContext.FETCH_SOURCE;
+        this.fetchSourceContext = new FetchSourceContext(fetch, fetchSourceContext.includes(), fetchSourceContext.excludes());
         return this;
     }
 
@@ -319,7 +319,9 @@ public TopHitsAggregationBuilder fetchSource(@Nullable String include, @Nullable
      *            pattern to filter the returned _source
      */
     public TopHitsAggregationBuilder fetchSource(@Nullable String[] includes, @Nullable String[] excludes) {
-        fetchSourceContext = new FetchSourceContext(includes, excludes);
+        FetchSourceContext fetchSourceContext = this.fetchSourceContext != null ? this.fetchSourceContext
+            : FetchSourceContext.FETCH_SOURCE;
+        this.fetchSourceContext = new FetchSourceContext(fetchSourceContext.fetchSource(), includes, excludes);
         return this;
     }
 
@@ -521,15 +523,31 @@ public boolean trackScores() {
 
     @Override
     public TopHitsAggregationBuilder subAggregations(Builder subFactories) {
-        throw new AggregationInitializationException("Aggregator [" + name + "] of type [" + type + "] cannot accept sub-aggregations");
+        throw new AggregationInitializationException("Aggregator [" + name + "] of type ["
+                + getType() + "] cannot accept sub-aggregations");
     }
 
     @Override
-    protected TopHitsAggregatorFactory doBuild(AggregationContext context, AggregatorFactory<?> parent, Builder subfactoriesBuilder)
+    protected TopHitsAggregatorFactory doBuild(SearchContext context, AggregatorFactory<?> parent, Builder subfactoriesBuilder)
             throws IOException {
-        return new TopHitsAggregatorFactory(name, type, from, size, explain, version, trackScores, sorts, highlightBuilder,
-            storedFieldsContext, fieldDataFields, scriptFields, fetchSourceContext, context,
-            parent, subfactoriesBuilder, metaData);
+        List<ScriptFieldsContext.ScriptField> fields = new ArrayList<>();
+        if (scriptFields != null) {
+            for (ScriptField field : scriptFields) {
+                SearchScript searchScript = context.getQueryShardContext().getSearchScript(field.script(),
+                    ScriptContext.Standard.SEARCH);
+                fields.add(new org.elasticsearch.search.fetch.subphase.ScriptFieldsContext.ScriptField(
+                    field.fieldName(), searchScript, field.ignoreFailure()));
+            }
+        }
+
+        final Optional<SortAndFormats> optionalSort;
+        if (sorts == null) {
+            optionalSort = Optional.empty();
+        } else {
+            optionalSort = SortBuilder.buildSort(sorts, context.getQueryShardContext());
+        }
+        return new TopHitsAggregatorFactory(name, from, size, explain, version, trackScores, optionalSort, highlightBuilder,
+                storedFieldsContext, fieldDataFields, fields, fetchSourceContext, context, parent, subfactoriesBuilder, metaData);
     }
 
     @Override
@@ -585,31 +603,31 @@ public static TopHitsAggregationBuilder parse(String aggregationName, QueryParse
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token.isValue()) {
-                if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.FROM_FIELD)) {
+                if (SearchSourceBuilder.FROM_FIELD.match(currentFieldName)) {
                     factory.from(parser.intValue());
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.SIZE_FIELD)) {
+                } else if (SearchSourceBuilder.SIZE_FIELD.match(currentFieldName)) {
                     factory.size(parser.intValue());
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.VERSION_FIELD)) {
+                } else if (SearchSourceBuilder.VERSION_FIELD.match(currentFieldName)) {
                     factory.version(parser.booleanValue());
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.EXPLAIN_FIELD)) {
+                } else if (SearchSourceBuilder.EXPLAIN_FIELD.match(currentFieldName)) {
                     factory.explain(parser.booleanValue());
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.TRACK_SCORES_FIELD)) {
+                } else if (SearchSourceBuilder.TRACK_SCORES_FIELD.match(currentFieldName)) {
                     factory.trackScores(parser.booleanValue());
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder._SOURCE_FIELD)) {
-                    factory.fetchSource(FetchSourceContext.parse(context));
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.STORED_FIELDS_FIELD)) {
+                } else if (SearchSourceBuilder._SOURCE_FIELD.match(currentFieldName)) {
+                    factory.fetchSource(FetchSourceContext.fromXContent(context.parser()));
+                } else if (SearchSourceBuilder.STORED_FIELDS_FIELD.match(currentFieldName)) {
                     factory.storedFieldsContext =
                         StoredFieldsContext.fromXContent(SearchSourceBuilder.STORED_FIELDS_FIELD.getPreferredName(), context);
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.SORT_FIELD)) {
+                } else if (SearchSourceBuilder.SORT_FIELD.match(currentFieldName)) {
                     factory.sort(parser.text());
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName + "].",
                             parser.getTokenLocation());
                 }
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder._SOURCE_FIELD)) {
-                    factory.fetchSource(FetchSourceContext.parse(context));
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.SCRIPT_FIELDS_FIELD)) {
+                if (SearchSourceBuilder._SOURCE_FIELD.match(currentFieldName)) {
+                    factory.fetchSource(FetchSourceContext.fromXContent(context.parser()));
+                } else if (SearchSourceBuilder.SCRIPT_FIELDS_FIELD.match(currentFieldName)) {
                     List<ScriptField> scriptFields = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
                         String scriptFieldName = parser.currentName();
@@ -621,10 +639,9 @@ public static TopHitsAggregationBuilder parse(String aggregationName, QueryParse
                                 if (token == XContentParser.Token.FIELD_NAME) {
                                     currentFieldName = parser.currentName();
                                 } else if (token.isValue()) {
-                                    if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.SCRIPT_FIELD)) {
-                                        script = Script.parse(parser, context.getParseFieldMatcher(), context.getDefaultScriptLanguage());
-                                    } else if (context.getParseFieldMatcher().match(currentFieldName,
-                                            SearchSourceBuilder.IGNORE_FAILURE_FIELD)) {
+                                    if (SearchSourceBuilder.SCRIPT_FIELD.match(currentFieldName)) {
+                                        script = Script.parse(parser, context.getDefaultScriptLanguage());
+                                    } else if (SearchSourceBuilder.IGNORE_FAILURE_FIELD.match(currentFieldName)) {
                                         ignoreFailure = parser.booleanValue();
                                     } else {
                                         throw new ParsingException(parser.getTokenLocation(),
@@ -632,8 +649,8 @@ public static TopHitsAggregationBuilder parse(String aggregationName, QueryParse
                                                 parser.getTokenLocation());
                                     }
                                 } else if (token == XContentParser.Token.START_OBJECT) {
-                                    if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.SCRIPT_FIELD)) {
-                                        script = Script.parse(parser, context.getParseFieldMatcher(), context.getDefaultScriptLanguage());
+                                    if (SearchSourceBuilder.SCRIPT_FIELD.match(currentFieldName)) {
+                                        script = Script.parse(parser, context.getDefaultScriptLanguage());
                                     } else {
                                         throw new ParsingException(parser.getTokenLocation(),
                                                 "Unknown key for a " + token + " in [" + currentFieldName + "].",
@@ -651,9 +668,9 @@ public static TopHitsAggregationBuilder parse(String aggregationName, QueryParse
                         }
                     }
                     factory.scriptFields(scriptFields);
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.HIGHLIGHT_FIELD)) {
+                } else if (SearchSourceBuilder.HIGHLIGHT_FIELD.match(currentFieldName)) {
                     factory.highlighter(HighlightBuilder.fromXContent(context));
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.SORT_FIELD)) {
+                } else if (SearchSourceBuilder.SORT_FIELD.match(currentFieldName)) {
                     List<SortBuilder<?>> sorts = SortBuilder.fromXContent(context);
                     factory.sorts(sorts);
                 } else {
@@ -662,10 +679,10 @@ public static TopHitsAggregationBuilder parse(String aggregationName, QueryParse
                 }
             } else if (token == XContentParser.Token.START_ARRAY) {
 
-                if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.STORED_FIELDS_FIELD)) {
+                if (SearchSourceBuilder.STORED_FIELDS_FIELD.match(currentFieldName)) {
                     factory.storedFieldsContext =
                         StoredFieldsContext.fromXContent(SearchSourceBuilder.STORED_FIELDS_FIELD.getPreferredName(), context);
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.DOCVALUE_FIELDS_FIELD)) {
+                } else if (SearchSourceBuilder.DOCVALUE_FIELDS_FIELD.match(currentFieldName)) {
                     List<String> fieldDataFields = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         if (token == XContentParser.Token.VALUE_STRING) {
@@ -676,11 +693,11 @@ public static TopHitsAggregationBuilder parse(String aggregationName, QueryParse
                         }
                     }
                     factory.fieldDataFields(fieldDataFields);
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.SORT_FIELD)) {
+                } else if (SearchSourceBuilder.SORT_FIELD.match(currentFieldName)) {
                     List<SortBuilder<?>> sorts = SortBuilder.fromXContent(context);
                     factory.sorts(sorts);
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder._SOURCE_FIELD)) {
-                    factory.fetchSource(FetchSourceContext.parse(context));
+                } else if (SearchSourceBuilder._SOURCE_FIELD.match(currentFieldName)) {
+                    factory.fetchSource(FetchSourceContext.fromXContent(context.parser()));
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName + "].",
                             parser.getTokenLocation());
@@ -716,7 +733,7 @@ protected boolean doEquals(Object obj) {
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregator.java
index 07292f1d29f82..f91a982dbd300 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregator.java
@@ -29,6 +29,7 @@
 import org.apache.lucene.search.TopFieldCollector;
 import org.apache.lucene.search.TopFieldDocs;
 import org.apache.lucene.search.TopScoreDocCollector;
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.util.LongObjectPagedHashMap;
@@ -38,12 +39,13 @@
 import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
 import org.elasticsearch.search.aggregations.metrics.MetricsAggregator;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.fetch.FetchPhase;
 import org.elasticsearch.search.fetch.FetchSearchResult;
-import org.elasticsearch.search.internal.InternalSearchHit;
-import org.elasticsearch.search.internal.InternalSearchHits;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.internal.SubSearchContext;
+import org.elasticsearch.search.rescore.RescoreSearchContext;
 import org.elasticsearch.search.sort.SortAndFormats;
 
 import java.io.IOException;
@@ -68,7 +70,7 @@ private static class TopDocsAndLeafCollector {
     final SubSearchContext subSearchContext;
     final LongObjectPagedHashMap<TopDocsAndLeafCollector> topDocsCollectors;
 
-    public TopHitsAggregator(FetchPhase fetchPhase, SubSearchContext subSearchContext, String name, AggregationContext context,
+    public TopHitsAggregator(FetchPhase fetchPhase, SubSearchContext subSearchContext, String name, SearchContext context,
             Aggregator parent, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException {
         super(name, context, parent, pipelineAggregators, metaData);
         this.fetchPhase = fetchPhase;
@@ -114,10 +116,21 @@ public void collect(int docId, long bucket) throws IOException {
                 if (collectors == null) {
                     SortAndFormats sort = subSearchContext.sort();
                     int topN = subSearchContext.from() + subSearchContext.size();
+                    if (sort == null) {
+                        for (RescoreSearchContext rescoreContext : context.rescore()) {
+                            topN = Math.max(rescoreContext.window(), topN);
+                        }
+                    }
                     // In the QueryPhase we don't need this protection, because it is build into the IndexSearcher,
                     // but here we create collectors ourselves and we need prevent OOM because of crazy an offset and size.
                     topN = Math.min(topN, subSearchContext.searcher().getIndexReader().maxDoc());
-                    TopDocsCollector<?> topLevelCollector = sort != null ? TopFieldCollector.create(sort.sort, topN, true, subSearchContext.trackScores(), subSearchContext.trackScores()) : TopScoreDocCollector.create(topN);
+                    TopDocsCollector<?> topLevelCollector;
+                    if (sort == null) {
+                        topLevelCollector = TopScoreDocCollector.create(topN);
+                    } else {
+                        topLevelCollector = TopFieldCollector.create(sort.sort, topN, true, subSearchContext.trackScores(),
+                                subSearchContext.trackScores());
+                    }
                     collectors = new TopDocsAndLeafCollector(topLevelCollector);
                     collectors.leafCollector = collectors.topLevelCollector.getLeafCollector(ctx);
                     collectors.leafCollector.setScorer(scorer);
@@ -135,9 +148,18 @@ public InternalAggregation buildAggregation(long owningBucketOrdinal) {
         if (topDocsCollector == null) {
             topHits = buildEmptyAggregation();
         } else {
-            final TopDocs topDocs = topDocsCollector.topLevelCollector.topDocs();
-
-            subSearchContext.queryResult().topDocs(topDocs, subSearchContext.sort() == null ? null : subSearchContext.sort().formats);
+            TopDocs topDocs = topDocsCollector.topLevelCollector.topDocs();
+            if (subSearchContext.sort() == null) {
+                for (RescoreSearchContext ctx : context().rescore()) {
+                    try {
+                        topDocs = ctx.rescorer().rescore(topDocs, context, ctx);
+                    } catch (IOException e) {
+                        throw new ElasticsearchException("Rescore TopHits Failed", e);
+                    }
+                }
+            }
+            subSearchContext.queryResult().topDocs(topDocs,
+                subSearchContext.sort() == null ? null : subSearchContext.sort().formats);
             int[] docIdsToLoad = new int[topDocs.scoreDocs.length];
             for (int i = 0; i < topDocs.scoreDocs.length; i++) {
                 docIdsToLoad[i] = topDocs.scoreDocs[i].doc;
@@ -145,10 +167,10 @@ public InternalAggregation buildAggregation(long owningBucketOrdinal) {
             subSearchContext.docIdsToLoad(docIdsToLoad, 0, docIdsToLoad.length);
             fetchPhase.execute(subSearchContext);
             FetchSearchResult fetchResult = subSearchContext.fetchResult();
-            InternalSearchHit[] internalHits = fetchResult.fetchResult().hits().internalHits();
+            SearchHit[] internalHits = fetchResult.fetchResult().hits().internalHits();
             for (int i = 0; i < internalHits.length; i++) {
                 ScoreDoc scoreDoc = topDocs.scoreDocs[i];
-                InternalSearchHit searchHitFields = internalHits[i];
+                SearchHit searchHitFields = internalHits[i];
                 searchHitFields.shard(subSearchContext.shardTarget());
                 searchHitFields.score(scoreDoc.score);
                 if (scoreDoc instanceof FieldDoc) {
@@ -156,8 +178,8 @@ public InternalAggregation buildAggregation(long owningBucketOrdinal) {
                     searchHitFields.sortValues(fieldDoc.fields, subSearchContext.sort().formats);
                 }
             }
-            topHits = new InternalTopHits(name, subSearchContext.from(), subSearchContext.size(), topDocs, fetchResult.hits(), pipelineAggregators(),
-                    metaData());
+            topHits = new InternalTopHits(name, subSearchContext.from(), subSearchContext.size(), topDocs, fetchResult.hits(),
+                    pipelineAggregators(), metaData());
         }
         return topHits;
     }
@@ -170,7 +192,8 @@ public InternalTopHits buildEmptyAggregation() {
         } else {
             topDocs = Lucene.EMPTY_TOP_DOCS;
         }
-        return new InternalTopHits(name, subSearchContext.from(), subSearchContext.size(), topDocs, InternalSearchHits.empty(), pipelineAggregators(), metaData());
+        return new InternalTopHits(name, subSearchContext.from(), subSearchContext.size(), topDocs, SearchHits.empty(),
+                pipelineAggregators(), metaData());
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java
index b3389322d9c2a..6a41cc97f8ec5 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java
@@ -19,31 +19,23 @@
 
 package org.elasticsearch.search.aggregations.metrics.tophits;
 
-import org.elasticsearch.search.fetch.StoredFieldsContext;
-import org.elasticsearch.script.ScriptContext;
-import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
-import org.elasticsearch.search.builder.SearchSourceBuilder.ScriptField;
+import org.elasticsearch.search.fetch.StoredFieldsContext;
 import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext;
-import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext.DocValueField;
-import org.elasticsearch.search.fetch.subphase.DocValueFieldsFetchSubPhase;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
+import org.elasticsearch.search.fetch.subphase.ScriptFieldsContext;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.internal.SubSearchContext;
 import org.elasticsearch.search.sort.SortAndFormats;
-import org.elasticsearch.search.sort.SortBuilder;
 
 import java.io.IOException;
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
-import java.util.Set;
 
 public class TopHitsAggregatorFactory extends AggregatorFactory<TopHitsAggregatorFactory> {
 
@@ -52,25 +44,25 @@ public class TopHitsAggregatorFactory extends AggregatorFactory<TopHitsAggregato
     private final boolean explain;
     private final boolean version;
     private final boolean trackScores;
-    private final List<SortBuilder<?>> sorts;
+    private final Optional<SortAndFormats> sort;
     private final HighlightBuilder highlightBuilder;
     private final StoredFieldsContext storedFieldsContext;
     private final List<String> docValueFields;
-    private final Set<ScriptField> scriptFields;
+    private final List<ScriptFieldsContext.ScriptField> scriptFields;
     private final FetchSourceContext fetchSourceContext;
 
-    public TopHitsAggregatorFactory(String name, Type type, int from, int size, boolean explain, boolean version, boolean trackScores,
-            List<SortBuilder<?>> sorts, HighlightBuilder highlightBuilder, StoredFieldsContext storedFieldsContext,
-            List<String> docValueFields, Set<ScriptField> scriptFields, FetchSourceContext fetchSourceContext,
-            AggregationContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactories,
-            Map<String, Object> metaData) throws IOException {
-        super(name, type, context, parent, subFactories, metaData);
+    public TopHitsAggregatorFactory(String name, int from, int size, boolean explain, boolean version, boolean trackScores,
+            Optional<SortAndFormats> sort, HighlightBuilder highlightBuilder, StoredFieldsContext storedFieldsContext,
+            List<String> docValueFields, List<ScriptFieldsContext.ScriptField> scriptFields, FetchSourceContext fetchSourceContext,
+            SearchContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactories, Map<String, Object> metaData)
+            throws IOException {
+        super(name, context, parent, subFactories, metaData);
         this.from = from;
         this.size = size;
         this.explain = explain;
         this.version = version;
         this.trackScores = trackScores;
-        this.sorts = sorts;
+        this.sort = sort;
         this.highlightBuilder = highlightBuilder;
         this.storedFieldsContext = storedFieldsContext;
         this.docValueFields = docValueFields;
@@ -81,45 +73,32 @@ public TopHitsAggregatorFactory(String name, Type type, int from, int size, bool
     @Override
     public Aggregator createInternal(Aggregator parent, boolean collectsFromSingleBucket, List<PipelineAggregator> pipelineAggregators,
             Map<String, Object> metaData) throws IOException {
-        SubSearchContext subSearchContext = new SubSearchContext(context.searchContext());
-        subSearchContext.parsedQuery(context.searchContext().parsedQuery());
+        SubSearchContext subSearchContext = new SubSearchContext(context);
+        subSearchContext.parsedQuery(context.parsedQuery());
         subSearchContext.explain(explain);
         subSearchContext.version(version);
         subSearchContext.trackScores(trackScores);
         subSearchContext.from(from);
         subSearchContext.size(size);
-        if (sorts != null) {
-            Optional<SortAndFormats> optionalSort = SortBuilder.buildSort(sorts, subSearchContext.getQueryShardContext());
-            if (optionalSort.isPresent()) {
-                subSearchContext.sort(optionalSort.get());
-            }
+        if (sort.isPresent()) {
+            subSearchContext.sort(sort.get());
         }
         if (storedFieldsContext != null) {
             subSearchContext.storedFieldsContext(storedFieldsContext);
         }
         if (docValueFields != null) {
-            DocValueFieldsContext docValueFieldsContext = subSearchContext
-                    .getFetchSubPhaseContext(DocValueFieldsFetchSubPhase.CONTEXT_FACTORY);
-            for (String field : docValueFields) {
-                docValueFieldsContext.add(new DocValueField(field));
-            }
-            docValueFieldsContext.setHitExecutionNeeded(true);
+            subSearchContext.docValueFieldsContext(new DocValueFieldsContext(docValueFields));
         }
-        if (scriptFields != null) {
-            for (ScriptField field : scriptFields) {
-                SearchScript searchScript = subSearchContext.scriptService().search(subSearchContext.lookup(), field.script(),
-                        ScriptContext.Standard.SEARCH, Collections.emptyMap());
-                subSearchContext.scriptFields().add(new org.elasticsearch.search.fetch.subphase.ScriptFieldsContext.ScriptField(
-                        field.fieldName(), searchScript, field.ignoreFailure()));
+        for (ScriptFieldsContext.ScriptField field : scriptFields) {
+            subSearchContext.scriptFields().add(field);
             }
-        }
         if (fetchSourceContext != null) {
             subSearchContext.fetchSourceContext(fetchSourceContext);
         }
         if (highlightBuilder != null) {
-            subSearchContext.highlight(highlightBuilder.build(context.searchContext().getQueryShardContext()));
+            subSearchContext.highlight(highlightBuilder.build(context.getQueryShardContext()));
         }
-        return new TopHitsAggregator(context.searchContext().fetchPhase(), subSearchContext, name, context, parent,
+        return new TopHitsAggregator(context.fetchPhase(), subSearchContext, name, context, parent,
                 pipelineAggregators, metaData);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/InternalValueCount.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/InternalValueCount.java
index bfd186df5f247..835d98b1b1912 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/InternalValueCount.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/InternalValueCount.java
@@ -80,7 +80,7 @@ public InternalAggregation doReduce(List<InternalAggregation> aggregations, Redu
 
     @Override
     public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
-        builder.field(CommonFields.VALUE, value);
+        builder.field(CommonFields.VALUE.getPreferredName(), value);
         return builder;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ParsedValueCount.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ParsedValueCount.java
new file mode 100644
index 0000000000000..7430bca08de32
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ParsedValueCount.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.valuecount;
+
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.ParsedAggregation;
+
+import java.io.IOException;
+
+public class ParsedValueCount extends ParsedAggregation implements ValueCount {
+
+    private long valueCount;
+
+    @Override
+    public double value() {
+        return getValue();
+    }
+
+    @Override
+    public long getValue() {
+        return valueCount;
+    }
+
+    @Override
+    public String getValueAsString() {
+        // InternalValueCount doesn't print "value_as_string", but you can get a formatted value using
+        // getValueAsString() using the raw formatter and converting the value to double
+        return Double.toString(valueCount);
+    }
+
+    @Override
+    public String getType() {
+        return ValueCountAggregationBuilder.NAME;
+    }
+
+    @Override
+    protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+        builder.field(CommonFields.VALUE.getPreferredName(), valueCount);
+        return builder;
+    }
+
+    private static final ObjectParser<ParsedValueCount, Void> PARSER = new ObjectParser<>(ParsedValueCount.class.getSimpleName(), true,
+            ParsedValueCount::new);
+
+    static {
+        declareAggregationFields(PARSER);
+        PARSER.declareLong((agg, value) -> agg.valueCount = value, CommonFields.VALUE);
+    }
+
+    public static ParsedValueCount fromXContent(XContentParser parser, final String name) {
+        ParsedValueCount sum = PARSER.apply(parser, null);
+        sum.setName(name);
+        return sum;
+    }
+}
\ No newline at end of file
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountAggregationBuilder.java
index ce0c1fd3e5ba6..50916b4063cc4 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountAggregationBuilder.java
@@ -21,32 +21,44 @@
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 
 public class ValueCountAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOnly<ValuesSource, ValueCountAggregationBuilder> {
     public static final String NAME = "value_count";
-    public static final Type TYPE = new Type(NAME);
+
+    private static final ObjectParser<ValueCountAggregationBuilder, QueryParseContext> PARSER;
+    static {
+        PARSER = new ObjectParser<>(ValueCountAggregationBuilder.NAME);
+        ValuesSourceParserHelper.declareAnyFields(PARSER, true, true);
+    }
+
+    public static AggregationBuilder parse(String aggregationName, QueryParseContext context) throws IOException {
+        return PARSER.parse(context.parser(), new ValueCountAggregationBuilder(aggregationName, null), context);
+    }
 
     public ValueCountAggregationBuilder(String name, ValueType targetValueType) {
-        super(name, TYPE, ValuesSourceType.ANY, targetValueType);
+        super(name, ValuesSourceType.ANY, targetValueType);
     }
 
     /**
      * Read from a stream.
      */
     public ValueCountAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE, ValuesSourceType.ANY);
+        super(in, ValuesSourceType.ANY);
     }
 
     @Override
@@ -60,9 +72,9 @@ protected boolean serializeTargetValueType() {
     }
 
     @Override
-    protected ValueCountAggregatorFactory innerBuild(AggregationContext context, ValuesSourceConfig<ValuesSource> config,
+    protected ValueCountAggregatorFactory innerBuild(SearchContext context, ValuesSourceConfig<ValuesSource> config,
             AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder) throws IOException {
-        return new ValueCountAggregatorFactory(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        return new ValueCountAggregatorFactory(name, config, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
@@ -81,7 +93,7 @@ protected boolean innerEquals(Object obj) {
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountAggregator.java
index 4dbfb54e8da63..54eba8d1fffe0 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountAggregator.java
@@ -29,8 +29,8 @@
 import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
 import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregator;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -50,7 +50,7 @@ public class ValueCountAggregator extends NumericMetricsAggregator.SingleValue {
     LongArray counts;
 
     public ValueCountAggregator(String name, ValuesSource valuesSource,
-            AggregationContext aggregationContext, Aggregator parent, List<PipelineAggregator> pipelineAggregators,
+            SearchContext aggregationContext, Aggregator parent, List<PipelineAggregator> pipelineAggregators,
             Map<String, Object> metaData)
             throws IOException {
         super(name, aggregationContext, parent, pipelineAggregators, metaData);
@@ -82,7 +82,7 @@ public void collect(int doc, long bucket) throws IOException {
 
     @Override
     public double metric(long owningBucketOrd) {
-        return valuesSource == null ? 0 : counts.get(owningBucketOrd);
+        return (valuesSource == null || owningBucketOrd >= counts.size()) ? 0 : counts.get(owningBucketOrd);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountAggregatorFactory.java
index 4569f3421d05f..80c8001b93c97 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountAggregatorFactory.java
@@ -22,12 +22,11 @@
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -35,9 +34,9 @@
 
 public class ValueCountAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource, ValueCountAggregatorFactory> {
 
-    public ValueCountAggregatorFactory(String name, Type type, ValuesSourceConfig<ValuesSource> config, AggregationContext context,
+    public ValueCountAggregatorFactory(String name, ValuesSourceConfig<ValuesSource> config, SearchContext context,
             AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountParser.java
deleted file mode 100644
index bd61e276fec75..0000000000000
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountParser.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.search.aggregations.metrics.valuecount;
-
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.search.aggregations.support.AbstractValuesSourceParser.AnyValuesSourceParser;
-import org.elasticsearch.search.aggregations.support.XContentParseContext;
-import org.elasticsearch.search.aggregations.support.ValueType;
-import org.elasticsearch.search.aggregations.support.ValuesSource;
-import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
-import org.elasticsearch.search.aggregations.support.ValuesSourceType;
-
-import java.io.IOException;
-import java.util.Map;
-
-/**
- *
- */
-public class ValueCountParser extends AnyValuesSourceParser {
-
-    public ValueCountParser() {
-        super(true, true);
-    }
-
-    @Override
-    protected boolean token(String aggregationName, String currentFieldName, XContentParser.Token token,
-                            XContentParseContext context, Map<ParseField, Object> otherOptions) throws IOException {
-        return false;
-    }
-
-    @Override
-    protected ValuesSourceAggregationBuilder<ValuesSource, ValueCountAggregationBuilder> createFactory(
-            String aggregationName, ValuesSourceType valuesSourceType, ValueType targetValueType, Map<ParseField, Object> otherOptions) {
-        return new ValueCountAggregationBuilder(aggregationName, targetValueType);
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/AbstractPipelineAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/AbstractPipelineAggregationBuilder.java
index 4fb2ed914011a..8d28195d55163 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/AbstractPipelineAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/AbstractPipelineAggregationBuilder.java
@@ -171,4 +171,8 @@ public boolean equals(Object obj) {
 
     protected abstract boolean doEquals(Object obj);
 
+    @Override
+    public String getType() {
+        return type;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketHelpers.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketHelpers.java
index 98b5b67b7cf5f..90665eeb9b0fc 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketHelpers.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketHelpers.java
@@ -52,7 +52,7 @@ public class BucketHelpers {
      * "insert_zeros": empty buckets will be filled with zeros for all metrics
      * "ignore": empty buckets will simply be ignored
      */
-    public static enum GapPolicy {
+    public enum GapPolicy {
         INSERT_ZEROS((byte) 0, "insert_zeros"), SKIP((byte) 1, "skip");
 
         /**
@@ -65,7 +65,7 @@ public static enum GapPolicy {
         public static GapPolicy parse(QueryParseContext context, String text, XContentLocation tokenLocation) {
             GapPolicy result = null;
             for (GapPolicy policy : values()) {
-                if (context.getParseFieldMatcher().match(text, policy.parseField)) {
+                if (policy.parseField.match(text)) {
                     if (result == null) {
                         result = policy;
                     } else {
@@ -87,7 +87,7 @@ public static GapPolicy parse(QueryParseContext context, String text, XContentLo
         private final byte id;
         private final ParseField parseField;
 
-        private GapPolicy(byte id, String name) {
+        GapPolicy(byte id, String name) {
             this.id = id;
             this.parseField = new ParseField(name);
         }
@@ -147,13 +147,13 @@ public String getName() {
      *         <code>aggPath</code>
      */
     public static Double resolveBucketValue(MultiBucketsAggregation agg,
-            InternalMultiBucketAggregation.Bucket bucket, String aggPath, GapPolicy gapPolicy) {
+            InternalMultiBucketAggregation.InternalBucket bucket, String aggPath, GapPolicy gapPolicy) {
         List<String> aggPathsList = AggregationPath.parse(aggPath).getPathElementsAsStringList();
         return resolveBucketValue(agg, bucket, aggPathsList, gapPolicy);
     }
 
     public static Double resolveBucketValue(MultiBucketsAggregation agg,
-            InternalMultiBucketAggregation.Bucket bucket, List<String> aggPathAsList, GapPolicy gapPolicy) {
+            InternalMultiBucketAggregation.InternalBucket bucket, List<String> aggPathAsList, GapPolicy gapPolicy) {
         try {
             Object propertyValue = bucket.getProperty(agg.getName(), aggPathAsList);
             if (propertyValue == null) {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/InternalSimpleValue.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/InternalSimpleValue.java
index b9d79b8e3297a..0f8eec4e66a34 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/InternalSimpleValue.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/InternalSimpleValue.java
@@ -79,9 +79,9 @@ public InternalMax doReduce(List<InternalAggregation> aggregations, ReduceContex
     @Override
     public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
         boolean hasValue = !(Double.isInfinite(value) || Double.isNaN(value));
-        builder.field(CommonFields.VALUE, hasValue ? value : null);
+        builder.field(CommonFields.VALUE.getPreferredName(), hasValue ? value : null);
         if (hasValue && format != DocValueFormat.RAW) {
-            builder.field(CommonFields.VALUE_AS_STRING, format.format(value));
+            builder.field(CommonFields.VALUE_AS_STRING.getPreferredName(), format.format(value));
         }
         return builder;
     }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/ParsedSimpleValue.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/ParsedSimpleValue.java
new file mode 100644
index 0000000000000..7449ce66666e6
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/ParsedSimpleValue.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.pipeline;
+
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.metrics.ParsedSingleValueNumericMetricsAggregation;
+
+import java.io.IOException;
+
+public class ParsedSimpleValue extends ParsedSingleValueNumericMetricsAggregation implements SimpleValue {
+
+    @Override
+    public String getType() {
+        return InternalSimpleValue.NAME;
+    }
+
+    private static final ObjectParser<ParsedSimpleValue, Void> PARSER = new ObjectParser<>(ParsedSimpleValue.class.getSimpleName(), true,
+            ParsedSimpleValue::new);
+
+    static {
+        declareSingleValueFields(PARSER, Double.NaN);
+    }
+
+    public static ParsedSimpleValue fromXContent(XContentParser parser, final String name) {
+        ParsedSimpleValue simpleValue = PARSER.apply(parser, null);
+        simpleValue.setName(name);
+        return simpleValue;
+    }
+
+    @Override
+    protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+        boolean hasValue = Double.isNaN(value) == false;
+        builder.field(CommonFields.VALUE.getPreferredName(), hasValue ? value : null);
+        if (hasValue && valueAsString != null) {
+            builder.field(CommonFields.VALUE_AS_STRING.getPreferredName(), valueAsString);
+        }
+        return builder;
+    }
+}
\ No newline at end of file
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/BucketMetricValue.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/BucketMetricValue.java
new file mode 100644
index 0000000000000..be22679a4e1bf
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/BucketMetricValue.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.pipeline.bucketmetrics;
+
+import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation;
+
+public interface BucketMetricValue extends NumericMetricsAggregation.SingleValue {
+
+    String[] keys();
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/BucketMetricsParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/BucketMetricsParser.java
index 9dee002ca2905..e7954174aa3c9 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/BucketMetricsParser.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/BucketMetricsParser.java
@@ -58,17 +58,17 @@ public final BucketMetricsPipelineAggregationBuilder<?> parse(String pipelineAgg
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.VALUE_STRING) {
-                if (context.getParseFieldMatcher().match(currentFieldName, FORMAT)) {
+                if (FORMAT.match(currentFieldName)) {
                     format = parser.text();
-                } else if (context.getParseFieldMatcher().match(currentFieldName, BUCKETS_PATH)) {
+                } else if (BUCKETS_PATH.match(currentFieldName)) {
                     bucketsPaths = new String[] { parser.text() };
-                } else if (context.getParseFieldMatcher().match(currentFieldName, GAP_POLICY)) {
+                } else if (GAP_POLICY.match(currentFieldName)) {
                     gapPolicy = GapPolicy.parse(context, parser.text(), parser.getTokenLocation());
                 } else {
                     parseToken(pipelineAggregatorName, parser, context, currentFieldName, token, params);
                 }
             } else if (token == XContentParser.Token.START_ARRAY) {
-                if (context.getParseFieldMatcher().match(currentFieldName, BUCKETS_PATH)) {
+                if (BUCKETS_PATH.match(currentFieldName)) {
                     List<String> paths = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         String path = parser.text();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/BucketMetricsPipelineAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/BucketMetricsPipelineAggregator.java
index 6bdbd5e6e4f9d..413862d3f1d2b 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/BucketMetricsPipelineAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/BucketMetricsPipelineAggregator.java
@@ -27,7 +27,6 @@
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext;
 import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation;
-import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
@@ -82,9 +81,8 @@ public final InternalAggregation doReduce(Aggregations aggregations, ReduceConte
             if (aggregation.getName().equals(bucketsPath.get(0))) {
                 bucketsPath = bucketsPath.subList(1, bucketsPath.size());
                 InternalMultiBucketAggregation<?, ?> multiBucketsAgg = (InternalMultiBucketAggregation<?, ?>) aggregation;
-                List<? extends Bucket> buckets = multiBucketsAgg.getBuckets();
-                for (int i = 0; i < buckets.size(); i++) {
-                    Bucket bucket = buckets.get(i);
+                List<? extends InternalMultiBucketAggregation.InternalBucket> buckets = multiBucketsAgg.getBuckets();
+                for (InternalMultiBucketAggregation.InternalBucket bucket : buckets) {
                     Double bucketValue = BucketHelpers.resolveBucketValue(multiBucketsAgg, bucket, bucketsPath, gapPolicy);
                     if (bucketValue != null && !Double.isNaN(bucketValue)) {
                         collectBucketValue(bucket.getKeyAsString(), bucketValue);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/InternalBucketMetricValue.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/InternalBucketMetricValue.java
index 6477728b3231d..8edcb684505d5 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/InternalBucketMetricValue.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/InternalBucketMetricValue.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.search.aggregations.pipeline.bucketmetrics;
 
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -31,8 +32,9 @@
 import java.util.List;
 import java.util.Map;
 
-public class InternalBucketMetricValue extends InternalNumericMetricsAggregation.SingleValue {
+public class InternalBucketMetricValue extends InternalNumericMetricsAggregation.SingleValue implements BucketMetricValue {
     public static final String NAME = "bucket_metric_value";
+    static final ParseField KEYS_FIELD = new ParseField("keys");
 
     private double value;
     private String[] keys;
@@ -72,6 +74,7 @@ public double value() {
         return value;
     }
 
+    @Override
     public String[] keys() {
         return keys;
     }
@@ -87,7 +90,7 @@ public Object getProperty(List<String> path) {
             return this;
         } else if (path.size() == 1 && "value".equals(path.get(0))) {
             return value();
-        } else if (path.size() == 1 && "keys".equals(path.get(0))) {
+        } else if (path.size() == 1 && KEYS_FIELD.getPreferredName().equals(path.get(0))) {
             return keys();
         } else {
             throw new IllegalArgumentException("path not supported for [" + getName() + "]: " + path);
@@ -97,16 +100,15 @@ public Object getProperty(List<String> path) {
     @Override
     public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
         boolean hasValue = !Double.isInfinite(value);
-        builder.field(CommonFields.VALUE, hasValue ? value : null);
+        builder.field(CommonFields.VALUE.getPreferredName(), hasValue ? value : null);
         if (hasValue && format != DocValueFormat.RAW) {
-            builder.field(CommonFields.VALUE_AS_STRING, format.format(value));
+            builder.field(CommonFields.VALUE_AS_STRING.getPreferredName(), format.format(value));
         }
-        builder.startArray("keys");
+        builder.startArray(KEYS_FIELD.getPreferredName());
         for (String key : keys) {
             builder.value(key);
         }
         builder.endArray();
         return builder;
     }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/ParsedBucketMetricValue.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/ParsedBucketMetricValue.java
new file mode 100644
index 0000000000000..69e99352636b6
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/ParsedBucketMetricValue.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.pipeline.bucketmetrics;
+
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.metrics.ParsedSingleValueNumericMetricsAggregation;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+
+public class ParsedBucketMetricValue extends ParsedSingleValueNumericMetricsAggregation implements BucketMetricValue {
+
+    private List<String> keys = Collections.emptyList();
+
+    @Override
+    public String[] keys() {
+        return this.keys.toArray(new String[keys.size()]);
+    }
+
+    @Override
+    public String getType() {
+        return InternalBucketMetricValue.NAME;
+    }
+
+    @Override
+    protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+        boolean hasValue = !Double.isInfinite(value);
+        builder.field(CommonFields.VALUE.getPreferredName(), hasValue ? value : null);
+        if (hasValue && valueAsString != null) {
+            builder.field(CommonFields.VALUE_AS_STRING.getPreferredName(), valueAsString);
+        }
+        builder.startArray(InternalBucketMetricValue.KEYS_FIELD.getPreferredName());
+        for (String key : keys) {
+            builder.value(key);
+        }
+        builder.endArray();
+        return builder;
+    }
+
+    private static final ObjectParser<ParsedBucketMetricValue, Void> PARSER = new ObjectParser<>(
+            ParsedBucketMetricValue.class.getSimpleName(), true, ParsedBucketMetricValue::new);
+
+    static {
+        declareSingleValueFields(PARSER, Double.NEGATIVE_INFINITY);
+        PARSER.declareStringArray((agg, value) -> agg.keys = value, InternalBucketMetricValue.KEYS_FIELD);
+    }
+
+    public static ParsedBucketMetricValue fromXContent(XContentParser parser, final String name) {
+        ParsedBucketMetricValue bucketMetricValue = PARSER.apply(parser, null);
+        bucketMetricValue.setName(name);
+        return bucketMetricValue;
+    }
+}
\ No newline at end of file
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/InternalPercentilesBucket.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/InternalPercentilesBucket.java
index 059d72a8e1013..4633aaa457dcf 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/InternalPercentilesBucket.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/InternalPercentilesBucket.java
@@ -26,12 +26,12 @@
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.metrics.InternalNumericMetricsAggregation;
 import org.elasticsearch.search.aggregations.metrics.max.InternalMax;
-import org.elasticsearch.search.aggregations.metrics.percentiles.InternalPercentile;
 import org.elasticsearch.search.aggregations.metrics.percentiles.Percentile;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -39,14 +39,26 @@
 public class InternalPercentilesBucket extends InternalNumericMetricsAggregation.MultiValue implements PercentilesBucket {
     private double[] percentiles;
     private double[] percents;
+    private final transient Map<Double, Double> percentileLookups = new HashMap<>();
 
     public InternalPercentilesBucket(String name, double[] percents, double[] percentiles,
                                      DocValueFormat formatter, List<PipelineAggregator> pipelineAggregators,
                                      Map<String, Object> metaData) {
         super(name, pipelineAggregators, metaData);
+        if ((percentiles.length == percents.length) == false) {
+            throw new IllegalArgumentException("The number of provided percents and percentiles didn't match. percents: "
+                    + Arrays.toString(percents) + ", percentiles: " + Arrays.toString(percentiles));
+        }
         this.format = formatter;
         this.percentiles = percentiles;
         this.percents = percents;
+        computeLookup();
+    }
+
+    private void computeLookup() {
+        for (int i = 0; i < percents.length; i++) {
+            percentileLookups.put(percents[i], percentiles[i]);
+        }
     }
 
     /**
@@ -57,6 +69,7 @@ public InternalPercentilesBucket(StreamInput in) throws IOException {
         format = in.readNamedWriteable(DocValueFormat.class);
         percentiles = in.readDoubleArray();
         percents = in.readDoubleArray();
+        computeLookup();
     }
 
     @Override
@@ -73,12 +86,12 @@ public String getWriteableName() {
 
     @Override
     public double percentile(double percent) throws IllegalArgumentException {
-        int index = Arrays.binarySearch(percents, percent);
-        if (index < 0) {
+        Double percentile = percentileLookups.get(percent);
+        if (percentile == null) {
             throw new IllegalArgumentException("Percent requested [" + String.valueOf(percent) + "] was not" +
                     " one of the computed percentiles.  Available keys are: " + Arrays.toString(percents));
         }
-        return percentiles[index];
+        return percentile;
     }
 
     @Override
@@ -136,7 +149,7 @@ public boolean hasNext() {
 
         @Override
         public Percentile next() {
-            final Percentile next = new InternalPercentile(percents[i], percentiles[i]);
+            final Percentile next = new Percentile(percents[i], percentiles[i]);
             ++i;
             return next;
         }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/ParsedPercentilesBucket.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/ParsedPercentilesBucket.java
new file mode 100644
index 0000000000000..eebe296e531fe
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/ParsedPercentilesBucket.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.pipeline.bucketmetrics.percentile;
+
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.metrics.percentiles.ParsedPercentiles;
+import org.elasticsearch.search.aggregations.metrics.percentiles.Percentiles;
+
+import java.io.IOException;
+import java.util.Map.Entry;
+
+public class ParsedPercentilesBucket extends ParsedPercentiles implements Percentiles {
+
+    @Override
+    public String getType() {
+        return PercentilesBucketPipelineAggregationBuilder.NAME;
+    }
+
+    @Override
+    public double percentile(double percent) throws IllegalArgumentException {
+        Double value = percentiles.get(percent);
+        if (value == null) {
+            throw new IllegalArgumentException("Percent requested [" + String.valueOf(percent) + "] was not" +
+                    " one of the computed percentiles. Available keys are: " + percentiles.keySet());
+        }
+        return value;
+    }
+
+    @Override
+    public String percentileAsString(double percent) {
+        double value = percentile(percent); // check availability as unformatted value
+        String valueAsString = percentilesAsString.get(percent);
+        if (valueAsString != null) {
+            return valueAsString;
+        } else {
+            return Double.toString(value);
+        }
+    }
+
+    @Override
+    public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject("values");
+        for (Entry<Double, Double> percent : percentiles.entrySet()) {
+            double value = percent.getValue();
+            boolean hasValue = !(Double.isNaN(value));
+            Double key = percent.getKey();
+            builder.field(Double.toString(key), hasValue ? value : null);
+            String valueAsString = percentilesAsString.get(key);
+            if (hasValue && valueAsString != null) {
+                builder.field(key + "_as_string", valueAsString);
+            }
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    private static ObjectParser<ParsedPercentilesBucket, Void> PARSER =
+            new ObjectParser<>(ParsedPercentilesBucket.class.getSimpleName(), true, ParsedPercentilesBucket::new);
+
+    static {
+        ParsedPercentiles.declarePercentilesFields(PARSER);
+    }
+
+    public static ParsedPercentilesBucket fromXContent(XContentParser parser, String name) throws IOException {
+        ParsedPercentilesBucket aggregation = PARSER.parse(parser, null);
+        aggregation.setName(name);
+        return aggregation;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/PercentilesBucketPipelineAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/PercentilesBucketPipelineAggregationBuilder.java
index 435d0239cbe51..cea7d01136786 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/PercentilesBucketPipelineAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/PercentilesBucketPipelineAggregationBuilder.java
@@ -113,7 +113,7 @@ public void doValidate(AggregatorFactory<?> parent, AggregatorFactory<?>[] aggFa
     @Override
     protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
         if (percents != null) {
-            builder.field(PERCENTS_FIELD.getPreferredName(), percents);
+            builder.array(PERCENTS_FIELD.getPreferredName(), percents);
         }
         return builder;
     }
@@ -138,7 +138,7 @@ protected PercentilesBucketPipelineAggregationBuilder buildFactory(String pipeli
         @Override
         protected boolean token(XContentParser parser, QueryParseContext context, String field,
                                 XContentParser.Token token, Map<String, Object> params) throws IOException {
-            if (context.getParseFieldMatcher().match(field, PERCENTS_FIELD) && token == XContentParser.Token.START_ARRAY) {
+            if (PERCENTS_FIELD.match(field) && token == XContentParser.Token.START_ARRAY) {
                 DoubleArrayList percents = new DoubleArrayList(10);
                 while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
                     percents.add(parser.doubleValue());
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/PercentilesBucketPipelineAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/PercentilesBucketPipelineAggregator.java
index 2818d0f3f5ed8..7f51a99d79867 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/PercentilesBucketPipelineAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/PercentilesBucketPipelineAggregator.java
@@ -88,7 +88,7 @@ protected InternalAggregation buildAggregation(List<PipelineAggregator> pipeline
             }
         } else {
             for (int i = 0; i < percents.length; i++) {
-                int index = (int)((percents[i] / 100.0) * data.size());
+                int index = (int) Math.round((percents[i] / 100.0) * (data.size() - 1));
                 percentiles[i] = data.get(index);
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/ParsedStatsBucket.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/ParsedStatsBucket.java
new file mode 100644
index 0000000000000..c7ddcc6ee9686
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/ParsedStatsBucket.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats;
+
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.metrics.stats.ParsedStats;
+
+
+public class ParsedStatsBucket extends ParsedStats implements StatsBucket {
+
+    @Override
+    public String getType() {
+        return StatsBucketPipelineAggregationBuilder.NAME;
+    }
+
+    private static final ObjectParser<ParsedStatsBucket, Void> PARSER = new ObjectParser<>(
+            ParsedStatsBucket.class.getSimpleName(), true, ParsedStatsBucket::new);
+
+    static {
+        declareStatsFields(PARSER);
+    }
+
+    public static ParsedStatsBucket fromXContent(XContentParser parser, final String name) {
+        ParsedStatsBucket parsedStatsBucket = PARSER.apply(parser, null);
+        parsedStatsBucket.setName(name);
+        return parsedStatsBucket;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/ExtendedStatsBucketParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/ExtendedStatsBucketParser.java
index b7fa49267dc55..dfa28c3dd2724 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/ExtendedStatsBucketParser.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/ExtendedStatsBucketParser.java
@@ -46,7 +46,7 @@ protected ExtendedStatsBucketPipelineAggregationBuilder buildFactory(String pipe
     @Override
     protected boolean token(XContentParser parser, QueryParseContext context, String field,
                             XContentParser.Token token, Map<String, Object> params) throws IOException {
-        if (context.getParseFieldMatcher().match(field, SIGMA) && token == XContentParser.Token.VALUE_NUMBER) {
+        if (SIGMA.match(field) && token == XContentParser.Token.VALUE_NUMBER) {
             params.put(SIGMA.getPreferredName(), parser.doubleValue());
             return true;
         }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/ParsedExtendedStatsBucket.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/ParsedExtendedStatsBucket.java
new file mode 100644
index 0000000000000..d292249242396
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/ParsedExtendedStatsBucket.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.extended;
+
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.metrics.stats.extended.ParsedExtendedStats;
+
+
+public class ParsedExtendedStatsBucket extends ParsedExtendedStats implements ExtendedStatsBucket {
+
+    @Override
+    public String getType() {
+        return ExtendedStatsBucketPipelineAggregationBuilder.NAME;
+    }
+
+    private static final ObjectParser<ParsedExtendedStatsBucket, Void> PARSER = new ObjectParser<>(
+            ParsedExtendedStatsBucket.class.getSimpleName(), true, ParsedExtendedStatsBucket::new);
+
+    static {
+        declareExtendedStatsFields(PARSER);
+    }
+
+    public static ParsedExtendedStatsBucket fromXContent(XContentParser parser, final String name) {
+        ParsedExtendedStatsBucket parsedStatsBucket = PARSER.apply(parser, null);
+        parsedStatsBucket.setName(name);
+        return parsedStatsBucket;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketscript/BucketScriptPipelineAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketscript/BucketScriptPipelineAggregationBuilder.java
index cd7b1bb828ed7..7c96b04b1561a 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketscript/BucketScriptPipelineAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketscript/BucketScriptPipelineAggregationBuilder.java
@@ -26,7 +26,6 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.Script.ScriptField;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
@@ -150,7 +149,7 @@ protected PipelineAggregator createInternal(Map<String, Object> metaData) throws
     @Override
     protected XContentBuilder internalXContent(XContentBuilder builder, Params params) throws IOException {
         builder.field(BUCKETS_PATH.getPreferredName(), bucketsPathsMap);
-        builder.field(ScriptField.SCRIPT.getPreferredName(), script);
+        builder.field(Script.SCRIPT_PARSE_FIELD.getPreferredName(), script);
         if (format != null) {
             builder.field(FORMAT.getPreferredName(), format);
         }
@@ -171,21 +170,21 @@ public static BucketScriptPipelineAggregationBuilder parse(String reducerName, Q
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.VALUE_STRING) {
-                if (context.getParseFieldMatcher().match(currentFieldName, FORMAT)) {
+                if (FORMAT.match(currentFieldName)) {
                     format = parser.text();
-                } else if (context.getParseFieldMatcher().match(currentFieldName, BUCKETS_PATH)) {
+                } else if (BUCKETS_PATH.match(currentFieldName)) {
                     bucketsPathsMap = new HashMap<>();
                     bucketsPathsMap.put("_value", parser.text());
-                } else if (context.getParseFieldMatcher().match(currentFieldName, GAP_POLICY)) {
+                } else if (GAP_POLICY.match(currentFieldName)) {
                     gapPolicy = GapPolicy.parse(context, parser.text(), parser.getTokenLocation());
-                } else if (context.getParseFieldMatcher().match(currentFieldName, ScriptField.SCRIPT)) {
-                    script = Script.parse(parser, context.getParseFieldMatcher(), context.getDefaultScriptLanguage());
+                } else if (Script.SCRIPT_PARSE_FIELD.match(currentFieldName)) {
+                    script = Script.parse(parser, context.getDefaultScriptLanguage());
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
                             "Unknown key for a " + token + " in [" + reducerName + "]: [" + currentFieldName + "].");
                 }
             } else if (token == XContentParser.Token.START_ARRAY) {
-                if (context.getParseFieldMatcher().match(currentFieldName, BUCKETS_PATH)) {
+                if (BUCKETS_PATH.match(currentFieldName)) {
                     List<String> paths = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         String path = parser.text();
@@ -200,9 +199,9 @@ public static BucketScriptPipelineAggregationBuilder parse(String reducerName, Q
                             "Unknown key for a " + token + " in [" + reducerName + "]: [" + currentFieldName + "].");
                 }
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (context.getParseFieldMatcher().match(currentFieldName, ScriptField.SCRIPT)) {
-                    script = Script.parse(parser, context.getParseFieldMatcher(), context.getDefaultScriptLanguage());
-                } else if (context.getParseFieldMatcher().match(currentFieldName, BUCKETS_PATH)) {
+                if (Script.SCRIPT_PARSE_FIELD.match(currentFieldName)) {
+                    script = Script.parse(parser, context.getDefaultScriptLanguage());
+                } else if (BUCKETS_PATH.match(currentFieldName)) {
                     Map<String, Object> map = parser.map();
                     bucketsPathsMap = new HashMap<>();
                     for (Map.Entry<String, Object> entry : map.entrySet()) {
@@ -223,7 +222,7 @@ public static BucketScriptPipelineAggregationBuilder parse(String reducerName, Q
         }
 
         if (script == null) {
-            throw new ParsingException(parser.getTokenLocation(), "Missing required field [" + ScriptField.SCRIPT.getPreferredName()
+            throw new ParsingException(parser.getTokenLocation(), "Missing required field [" + Script.SCRIPT_PARSE_FIELD.getPreferredName()
                     + "] for series_arithmetic aggregation [" + reducerName + "]");
         }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketscript/BucketScriptPipelineAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketscript/BucketScriptPipelineAggregator.java
index 008b0217c65d5..87df926ebab55 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketscript/BucketScriptPipelineAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketscript/BucketScriptPipelineAggregator.java
@@ -31,14 +31,12 @@
 import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext;
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation;
-import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
 import org.elasticsearch.search.aggregations.pipeline.InternalSimpleValue;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -89,13 +87,13 @@ public String getWriteableName() {
 
     @Override
     public InternalAggregation reduce(InternalAggregation aggregation, ReduceContext reduceContext) {
-        InternalMultiBucketAggregation<InternalMultiBucketAggregation, InternalMultiBucketAggregation.InternalBucket> originalAgg = (InternalMultiBucketAggregation<InternalMultiBucketAggregation, InternalMultiBucketAggregation.InternalBucket>) aggregation;
-        List<? extends Bucket> buckets = originalAgg.getBuckets();
+        InternalMultiBucketAggregation<InternalMultiBucketAggregation, InternalMultiBucketAggregation.InternalBucket> originalAgg =
+                (InternalMultiBucketAggregation<InternalMultiBucketAggregation, InternalMultiBucketAggregation.InternalBucket>) aggregation;
+        List<? extends InternalMultiBucketAggregation.InternalBucket> buckets = originalAgg.getBuckets();
 
-        CompiledScript compiledScript = reduceContext.scriptService().compile(script, ScriptContext.Standard.AGGS,
-                Collections.emptyMap());
-        List newBuckets = new ArrayList<>();
-        for (Bucket bucket : buckets) {
+        CompiledScript compiledScript = reduceContext.scriptService().compile(script, ScriptContext.Standard.AGGS);
+        List<InternalMultiBucketAggregation.InternalBucket> newBuckets = new ArrayList<>();
+        for (InternalMultiBucketAggregation.InternalBucket bucket : buckets) {
             Map<String, Object> vars = new HashMap<>();
             if (script.getParams() != null) {
                 vars.putAll(script.getParams());
@@ -123,13 +121,12 @@ public InternalAggregation reduce(InternalAggregation aggregation, ReduceContext
                         throw new AggregationExecutionException("series_arithmetic script for reducer [" + name()
                                 + "] must return a Number");
                     }
-                    final List<InternalAggregation> aggs = StreamSupport.stream(bucket.getAggregations().spliterator(), false).map((p) -> {
-                        return (InternalAggregation) p;
-                    }).collect(Collectors.toList());
+                    final List<InternalAggregation> aggs = StreamSupport.stream(bucket.getAggregations().spliterator(), false).map(
+                            (p) -> (InternalAggregation) p).collect(Collectors.toList());
                     aggs.add(new InternalSimpleValue(name(), ((Number) returned).doubleValue(), formatter,
                             new ArrayList<>(), metaData()));
                     InternalMultiBucketAggregation.InternalBucket newBucket = originalAgg.createBucket(new InternalAggregations(aggs),
-                            (InternalMultiBucketAggregation.InternalBucket) bucket);
+                            bucket);
                     newBuckets.add(newBucket);
                 }
             }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketselector/BucketSelectorPipelineAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketselector/BucketSelectorPipelineAggregationBuilder.java
index e3b423767286b..078f50a978f1e 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketselector/BucketSelectorPipelineAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketselector/BucketSelectorPipelineAggregationBuilder.java
@@ -26,7 +26,6 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.Script.ScriptField;
 import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
@@ -119,7 +118,7 @@ protected PipelineAggregator createInternal(Map<String, Object> metaData) throws
     @Override
     protected XContentBuilder internalXContent(XContentBuilder builder, Params params) throws IOException {
         builder.field(BUCKETS_PATH.getPreferredName(), bucketsPathsMap);
-        builder.field(ScriptField.SCRIPT.getPreferredName(), script);
+        builder.field(Script.SCRIPT_PARSE_FIELD.getPreferredName(), script);
         builder.field(GAP_POLICY.getPreferredName(), gapPolicy.getName());
         return builder;
     }
@@ -136,19 +135,19 @@ public static BucketSelectorPipelineAggregationBuilder parse(String reducerName,
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.VALUE_STRING) {
-                if (context.getParseFieldMatcher().match(currentFieldName, BUCKETS_PATH)) {
+                if (BUCKETS_PATH.match(currentFieldName)) {
                     bucketsPathsMap = new HashMap<>();
                     bucketsPathsMap.put("_value", parser.text());
-                } else if (context.getParseFieldMatcher().match(currentFieldName, GAP_POLICY)) {
+                } else if (GAP_POLICY.match(currentFieldName)) {
                     gapPolicy = GapPolicy.parse(context, parser.text(), parser.getTokenLocation());
-                } else if (context.getParseFieldMatcher().match(currentFieldName, ScriptField.SCRIPT)) {
-                    script = Script.parse(parser, context.getParseFieldMatcher(), context.getDefaultScriptLanguage());
+                } else if (Script.SCRIPT_PARSE_FIELD.match(currentFieldName)) {
+                    script = Script.parse(parser, context.getDefaultScriptLanguage());
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
                             "Unknown key for a " + token + " in [" + reducerName + "]: [" + currentFieldName + "].");
                 }
             } else if (token == XContentParser.Token.START_ARRAY) {
-                if (context.getParseFieldMatcher().match(currentFieldName, BUCKETS_PATH)) {
+                if (BUCKETS_PATH.match(currentFieldName)) {
                     List<String> paths = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         String path = parser.text();
@@ -163,9 +162,9 @@ public static BucketSelectorPipelineAggregationBuilder parse(String reducerName,
                             "Unknown key for a " + token + " in [" + reducerName + "]: [" + currentFieldName + "].");
                 }
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (context.getParseFieldMatcher().match(currentFieldName, ScriptField.SCRIPT)) {
-                    script = Script.parse(parser, context.getParseFieldMatcher(), context.getDefaultScriptLanguage());
-                } else if (context.getParseFieldMatcher().match(currentFieldName, BUCKETS_PATH)) {
+                if (Script.SCRIPT_PARSE_FIELD.match(currentFieldName)) {
+                    script = Script.parse(parser, context.getDefaultScriptLanguage());
+                } else if (BUCKETS_PATH.match(currentFieldName)) {
                     Map<String, Object> map = parser.map();
                     bucketsPathsMap = new HashMap<>();
                     for (Map.Entry<String, Object> entry : map.entrySet()) {
@@ -186,7 +185,7 @@ public static BucketSelectorPipelineAggregationBuilder parse(String reducerName,
         }
 
         if (script == null) {
-            throw new ParsingException(parser.getTokenLocation(), "Missing required field [" + ScriptField.SCRIPT.getPreferredName()
+            throw new ParsingException(parser.getTokenLocation(), "Missing required field [" + Script.SCRIPT_PARSE_FIELD.getPreferredName()
                     + "] for bucket_selector aggregation [" + reducerName + "]");
         }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketselector/BucketSelectorPipelineAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketselector/BucketSelectorPipelineAggregator.java
index eabbad7213a4e..62eed8d4e0a08 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketselector/BucketSelectorPipelineAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketselector/BucketSelectorPipelineAggregator.java
@@ -29,13 +29,11 @@
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext;
 import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation;
-import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -84,12 +82,11 @@ public String getWriteableName() {
     public InternalAggregation reduce(InternalAggregation aggregation, ReduceContext reduceContext) {
         InternalMultiBucketAggregation<InternalMultiBucketAggregation, InternalMultiBucketAggregation.InternalBucket> originalAgg =
                 (InternalMultiBucketAggregation<InternalMultiBucketAggregation, InternalMultiBucketAggregation.InternalBucket>) aggregation;
-        List<? extends Bucket> buckets = originalAgg.getBuckets();
+        List<? extends InternalMultiBucketAggregation.InternalBucket> buckets = originalAgg.getBuckets();
 
-        CompiledScript compiledScript = reduceContext.scriptService().compile(script, ScriptContext.Standard.AGGS,
-                Collections.emptyMap());
-        List newBuckets = new ArrayList<>();
-        for (Bucket bucket : buckets) {
+        CompiledScript compiledScript = reduceContext.scriptService().compile(script, ScriptContext.Standard.AGGS);
+        List<InternalMultiBucketAggregation.InternalBucket> newBuckets = new ArrayList<>();
+        for (InternalMultiBucketAggregation.InternalBucket bucket : buckets) {
             Map<String, Object> vars = new HashMap<>();
             if (script.getParams() != null) {
                 vars.putAll(script.getParams());
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/cumulativesum/CumulativeSumPipelineAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/cumulativesum/CumulativeSumPipelineAggregationBuilder.java
index a83787f13650e..5ac185990b488 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/cumulativesum/CumulativeSumPipelineAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/cumulativesum/CumulativeSumPipelineAggregationBuilder.java
@@ -141,16 +141,16 @@ public static CumulativeSumPipelineAggregationBuilder parse(String pipelineAggre
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.VALUE_STRING) {
-                if (context.getParseFieldMatcher().match(currentFieldName, FORMAT)) {
+                if (FORMAT.match(currentFieldName)) {
                     format = parser.text();
-                } else if (context.getParseFieldMatcher().match(currentFieldName, BUCKETS_PATH)) {
+                } else if (BUCKETS_PATH.match(currentFieldName)) {
                     bucketsPaths = new String[] { parser.text() };
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
                             "Unknown key for a " + token + " in [" + pipelineAggregatorName + "]: [" + currentFieldName + "].");
                 }
             } else if (token == XContentParser.Token.START_ARRAY) {
-                if (context.getParseFieldMatcher().match(currentFieldName, BUCKETS_PATH)) {
+                if (BUCKETS_PATH.match(currentFieldName)) {
                     List<String> paths = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         String path = parser.text();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/cumulativesum/CumulativeSumPipelineAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/cumulativesum/CumulativeSumPipelineAggregator.java
index 98c6f7b2fa29e..8a1b70fdd145b 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/cumulativesum/CumulativeSumPipelineAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/cumulativesum/CumulativeSumPipelineAggregator.java
@@ -25,7 +25,7 @@
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext;
 import org.elasticsearch.search.aggregations.InternalAggregations;
-import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
+import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation;
 import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket;
 import org.elasticsearch.search.aggregations.bucket.histogram.HistogramFactory;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
@@ -70,13 +70,14 @@ public String getWriteableName() {
 
     @Override
     public InternalAggregation reduce(InternalAggregation aggregation, ReduceContext reduceContext) {
-        MultiBucketsAggregation histo = (MultiBucketsAggregation) aggregation;
-        List<? extends Bucket> buckets = histo.getBuckets();
+        InternalMultiBucketAggregation<? extends InternalMultiBucketAggregation, ? extends InternalMultiBucketAggregation.InternalBucket>
+                histo = (InternalMultiBucketAggregation<? extends InternalMultiBucketAggregation, ? extends
+                InternalMultiBucketAggregation.InternalBucket>) aggregation;
+        List<? extends InternalMultiBucketAggregation.InternalBucket> buckets = histo.getBuckets();
         HistogramFactory factory = (HistogramFactory) histo;
-
-        List<Bucket> newBuckets = new ArrayList<>();
+        List<Bucket> newBuckets = new ArrayList<>(buckets.size());
         double sum = 0;
-        for (Bucket bucket : buckets) {
+        for (InternalMultiBucketAggregation.InternalBucket bucket : buckets) {
             Double thisBucketValue = resolveBucketValue(histo, bucket, bucketsPaths()[0], GapPolicy.INSERT_ZEROS);
             sum += thisBucketValue;
             List<InternalAggregation> aggs = StreamSupport.stream(bucket.getAggregations().spliterator(), false).map((p) -> {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/DerivativePipelineAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/DerivativePipelineAggregationBuilder.java
index 5ffc77669b8df..bb2a1c01a505d 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/DerivativePipelineAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/DerivativePipelineAggregationBuilder.java
@@ -32,6 +32,7 @@
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregatorFactory;
+import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
 import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregatorFactory;
 import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregationBuilder;
@@ -141,7 +142,7 @@ protected PipelineAggregator createInternal(Map<String, Object> metaData) throws
         }
         Long xAxisUnits = null;
         if (units != null) {
-            DateTimeUnit dateTimeUnit = DateHistogramAggregatorFactory.DATE_FIELD_UNITS.get(units);
+            DateTimeUnit dateTimeUnit = DateHistogramAggregationBuilder.DATE_FIELD_UNITS.get(units);
             if (dateTimeUnit != null) {
                 xAxisUnits = dateTimeUnit.field(DateTimeZone.UTC).getDurationField().getUnitMillis();
             } else {
@@ -206,20 +207,20 @@ public static DerivativePipelineAggregationBuilder parse(String pipelineAggregat
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.VALUE_STRING) {
-                if (context.getParseFieldMatcher().match(currentFieldName, FORMAT_FIELD)) {
+                if (FORMAT_FIELD.match(currentFieldName)) {
                     format = parser.text();
-                } else if (context.getParseFieldMatcher().match(currentFieldName, BUCKETS_PATH_FIELD)) {
+                } else if (BUCKETS_PATH_FIELD.match(currentFieldName)) {
                     bucketsPaths = new String[] { parser.text() };
-                } else if (context.getParseFieldMatcher().match(currentFieldName, GAP_POLICY_FIELD)) {
+                } else if (GAP_POLICY_FIELD.match(currentFieldName)) {
                     gapPolicy = GapPolicy.parse(context, parser.text(), parser.getTokenLocation());
-                } else if (context.getParseFieldMatcher().match(currentFieldName, UNIT_FIELD)) {
+                } else if (UNIT_FIELD.match(currentFieldName)) {
                     units = parser.text();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
                             "Unknown key for a " + token + " in [" + pipelineAggregatorName + "]: [" + currentFieldName + "].");
                 }
             } else if (token == XContentParser.Token.START_ARRAY) {
-                if (context.getParseFieldMatcher().match(currentFieldName, BUCKETS_PATH_FIELD)) {
+                if (BUCKETS_PATH_FIELD.match(currentFieldName)) {
                     List<String> paths = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         String path = parser.text();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/DerivativePipelineAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/DerivativePipelineAggregator.java
index 480f04f545a4e..3fe60f23cf31d 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/DerivativePipelineAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/DerivativePipelineAggregator.java
@@ -25,7 +25,7 @@
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext;
 import org.elasticsearch.search.aggregations.InternalAggregations;
-import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
+import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation;
 import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket;
 import org.elasticsearch.search.aggregations.bucket.histogram.HistogramFactory;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
@@ -77,14 +77,16 @@ public String getWriteableName() {
 
     @Override
     public InternalAggregation reduce(InternalAggregation aggregation, ReduceContext reduceContext) {
-        MultiBucketsAggregation histo = (MultiBucketsAggregation) aggregation;
-        List<? extends Bucket> buckets = histo.getBuckets();
+        InternalMultiBucketAggregation<? extends InternalMultiBucketAggregation, ? extends InternalMultiBucketAggregation.InternalBucket>
+                histo = (InternalMultiBucketAggregation<? extends InternalMultiBucketAggregation, ? extends
+                InternalMultiBucketAggregation.InternalBucket>) aggregation;
+        List<? extends InternalMultiBucketAggregation.InternalBucket> buckets = histo.getBuckets();
         HistogramFactory factory = (HistogramFactory) histo;
 
         List<Bucket> newBuckets = new ArrayList<>();
         Number lastBucketKey = null;
         Double lastBucketValue = null;
-        for (Bucket bucket : buckets) {
+        for (InternalMultiBucketAggregation.InternalBucket bucket : buckets) {
             Number thisBucketKey = factory.getKey(bucket);
             Double thisBucketValue = resolveBucketValue(histo, bucket, bucketsPaths()[0], gapPolicy);
             if (lastBucketValue != null && thisBucketValue != null) {
@@ -107,5 +109,4 @@ public InternalAggregation reduce(InternalAggregation aggregation, ReduceContext
         }
         return factory.createAggregation(newBuckets);
     }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/ParsedDerivative.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/ParsedDerivative.java
new file mode 100644
index 0000000000000..2b871a99d9a6a
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/ParsedDerivative.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.pipeline.derivative;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.pipeline.ParsedSimpleValue;
+
+import java.io.IOException;
+
+public class ParsedDerivative extends ParsedSimpleValue implements Derivative {
+
+    private double normalizedValue;
+    private String normalizedAsString;
+    private boolean hasNormalizationFactor;
+    private static final ParseField NORMALIZED_AS_STRING = new ParseField("normalized_value_as_string");
+    private static final ParseField NORMALIZED = new ParseField("normalized_value");
+
+    @Override
+    public double normalizedValue() {
+        return this.normalizedValue;
+    }
+
+    @Override
+    public String getType() {
+        return DerivativePipelineAggregationBuilder.NAME;
+    }
+
+    private static final ObjectParser<ParsedDerivative, Void> PARSER = new ObjectParser<>(ParsedDerivative.class.getSimpleName(), true,
+            ParsedDerivative::new);
+
+    static {
+        declareSingleValueFields(PARSER, Double.NaN);
+        PARSER.declareField((agg, normalized) -> {
+            agg.normalizedValue = normalized;
+            agg.hasNormalizationFactor = true;
+        }, (parser, context) -> parseDouble(parser, Double.NaN), NORMALIZED, ValueType.DOUBLE_OR_NULL);
+        PARSER.declareString((agg, normalAsString) -> agg.normalizedAsString = normalAsString, NORMALIZED_AS_STRING);
+    }
+
+    public static ParsedDerivative fromXContent(XContentParser parser, final String name) {
+        ParsedDerivative derivative = PARSER.apply(parser, null);
+        derivative.setName(name);
+        return derivative;
+    }
+
+    @Override
+    protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+        super.doXContentBody(builder, params);
+        if (hasNormalizationFactor) {
+            boolean hasValue = Double.isNaN(normalizedValue) == false;
+            builder.field(NORMALIZED.getPreferredName(), hasValue ? normalizedValue : null);
+            if (hasValue && normalizedAsString != null) {
+                builder.field(NORMALIZED_AS_STRING.getPreferredName(), normalizedAsString);
+            }
+        }
+        return builder;
+    }
+}
\ No newline at end of file
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/MovAvgPipelineAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/MovAvgPipelineAggregationBuilder.java
index f0aa1f8112639..bc973ad442f54 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/MovAvgPipelineAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/MovAvgPipelineAggregationBuilder.java
@@ -322,13 +322,13 @@ public static MovAvgPipelineAggregationBuilder parse(
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.VALUE_NUMBER) {
-                if (context.getParseFieldMatcher().match(currentFieldName, WINDOW)) {
+                if (WINDOW.match(currentFieldName)) {
                     window = parser.intValue();
                     if (window <= 0) {
                         throw new ParsingException(parser.getTokenLocation(), "[" + currentFieldName + "] value must be a positive, "
                                 + "non-zero integer.  Value supplied was [" + predict + "] in [" + pipelineAggregatorName + "].");
                     }
-                } else if (context.getParseFieldMatcher().match(currentFieldName, PREDICT)) {
+                } else if (PREDICT.match(currentFieldName)) {
                     predict = parser.intValue();
                     if (predict <= 0) {
                         throw new ParsingException(parser.getTokenLocation(), "[" + currentFieldName + "] value must be a positive integer."
@@ -339,20 +339,20 @@ public static MovAvgPipelineAggregationBuilder parse(
                             "Unknown key for a " + token + " in [" + pipelineAggregatorName + "]: [" + currentFieldName + "].");
                 }
             } else if (token == XContentParser.Token.VALUE_STRING) {
-                if (context.getParseFieldMatcher().match(currentFieldName, FORMAT)) {
+                if (FORMAT.match(currentFieldName)) {
                     format = parser.text();
-                } else if (context.getParseFieldMatcher().match(currentFieldName, BUCKETS_PATH)) {
+                } else if (BUCKETS_PATH.match(currentFieldName)) {
                     bucketsPaths = new String[] { parser.text() };
-                } else if (context.getParseFieldMatcher().match(currentFieldName, GAP_POLICY)) {
+                } else if (GAP_POLICY.match(currentFieldName)) {
                     gapPolicy = GapPolicy.parse(context, parser.text(), parser.getTokenLocation());
-                } else if (context.getParseFieldMatcher().match(currentFieldName, MODEL)) {
+                } else if (MODEL.match(currentFieldName)) {
                     model = parser.text();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
                             "Unknown key for a " + token + " in [" + pipelineAggregatorName + "]: [" + currentFieldName + "].");
                 }
             } else if (token == XContentParser.Token.START_ARRAY) {
-                if (context.getParseFieldMatcher().match(currentFieldName, BUCKETS_PATH)) {
+                if (BUCKETS_PATH.match(currentFieldName)) {
                     List<String> paths = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         String path = parser.text();
@@ -364,14 +364,14 @@ public static MovAvgPipelineAggregationBuilder parse(
                             "Unknown key for a " + token + " in [" + pipelineAggregatorName + "]: [" + currentFieldName + "].");
                 }
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (context.getParseFieldMatcher().match(currentFieldName, SETTINGS)) {
+                if (SETTINGS.match(currentFieldName)) {
                     settings = parser.map();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
                             "Unknown key for a " + token + " in [" + pipelineAggregatorName + "]: [" + currentFieldName + "].");
                 }
             } else if (token == XContentParser.Token.VALUE_BOOLEAN) {
-                if (context.getParseFieldMatcher().match(currentFieldName, MINIMIZE)) {
+                if (MINIMIZE.match(currentFieldName)) {
                     minimize = parser.booleanValue();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
@@ -403,11 +403,10 @@ public static MovAvgPipelineAggregationBuilder parse(
             factory.predict(predict);
         }
         if (model != null) {
-            MovAvgModel.AbstractModelParser modelParser = movingAverageMdelParserRegistry.lookup(model, context.getParseFieldMatcher(),
-                    parser.getTokenLocation());
+            MovAvgModel.AbstractModelParser modelParser = movingAverageMdelParserRegistry.lookup(model, parser.getTokenLocation());
             MovAvgModel movAvgModel;
             try {
-                movAvgModel = modelParser.parse(settings, pipelineAggregatorName, factory.window(), context.getParseFieldMatcher());
+                movAvgModel = modelParser.parse(settings, pipelineAggregatorName, factory.window());
             } catch (ParseException exception) {
                 throw new ParsingException(parser.getTokenLocation(), "Could not parse settings for model [" + model + "].", exception);
             }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/MovAvgPipelineAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/MovAvgPipelineAggregator.java
index 87aa5bfda63e0..196f7cca4737f 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/MovAvgPipelineAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/MovAvgPipelineAggregator.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext;
 import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation;
 import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
 import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket;
 import org.elasticsearch.search.aggregations.bucket.histogram.HistogramFactory;
@@ -93,8 +94,10 @@ public String getWriteableName() {
 
     @Override
     public InternalAggregation reduce(InternalAggregation aggregation, ReduceContext reduceContext) {
-        MultiBucketsAggregation histo = (MultiBucketsAggregation) aggregation;
-        List<? extends Bucket> buckets = histo.getBuckets();
+        InternalMultiBucketAggregation<? extends InternalMultiBucketAggregation, ? extends InternalMultiBucketAggregation.InternalBucket>
+                histo = (InternalMultiBucketAggregation<? extends InternalMultiBucketAggregation, ? extends
+                InternalMultiBucketAggregation.InternalBucket>) aggregation;
+        List<? extends InternalMultiBucketAggregation.InternalBucket> buckets = histo.getBuckets();
         HistogramFactory factory = (HistogramFactory) histo;
 
         List<Bucket> newBuckets = new ArrayList<>();
@@ -110,7 +113,7 @@ public InternalAggregation reduce(InternalAggregation aggregation, ReduceContext
             model = minimize(buckets, histo, model);
         }
 
-        for (Bucket bucket : buckets) {
+        for (InternalMultiBucketAggregation.InternalBucket bucket : buckets) {
             Double thisBucketValue = resolveBucketValue(histo, bucket, bucketsPaths()[0], gapPolicy);
 
             // Default is to reuse existing bucket.  Simplifies the rest of the logic,
@@ -158,7 +161,7 @@ public InternalAggregation reduce(InternalAggregation aggregation, ReduceContext
                     }).collect(Collectors.toList());
                     aggs.add(new InternalSimpleValue(name(), predictions[i], formatter, new ArrayList<PipelineAggregator>(), metaData()));
 
-                    Bucket newBucket = factory.createBucket(newKey, 0, new InternalAggregations(aggs));
+                    Bucket newBucket = factory.createBucket(newKey, bucket.getDocCount(), new InternalAggregations(aggs));
 
                     // Overwrite the existing bucket with the new version
                     newBuckets.set(lastValidPosition + i + 1, newBucket);
@@ -180,13 +183,14 @@ public InternalAggregation reduce(InternalAggregation aggregation, ReduceContext
         return factory.createAggregation(newBuckets);
     }
 
-    private MovAvgModel minimize(List<? extends Bucket> buckets, MultiBucketsAggregation histo, MovAvgModel model) {
+    private MovAvgModel minimize(List<? extends InternalMultiBucketAggregation.InternalBucket> buckets,
+                                 MultiBucketsAggregation histo, MovAvgModel model) {
 
         int counter = 0;
         EvictingQueue<Double> values = new EvictingQueue<>(this.window);
 
         double[] test = new double[window];
-        ListIterator<? extends Bucket> iter = buckets.listIterator(buckets.size());
+        ListIterator<? extends InternalMultiBucketAggregation.InternalBucket> iter = buckets.listIterator(buckets.size());
 
         // We have to walk the iterator backwards because we don't know if/how many buckets are empty.
         while (iter.hasPrevious() && counter < window) {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/EwmaModel.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/EwmaModel.java
index c7e6b0e898004..26fb0333b188b 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/EwmaModel.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/EwmaModel.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.search.aggregations.pipeline.movavg.models;
 
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -127,8 +126,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
 
     public static final AbstractModelParser PARSER = new AbstractModelParser() {
         @Override
-        public MovAvgModel parse(@Nullable Map<String, Object> settings, String pipelineName, int windowSize,
-                                 ParseFieldMatcher parseFieldMatcher) throws ParseException {
+        public MovAvgModel parse(@Nullable Map<String, Object> settings, String pipelineName, int windowSize) throws ParseException {
             double alpha = parseDoubleParam(settings, "alpha", DEFAULT_ALPHA);
             checkUnrecognizedParams(settings);
             return new EwmaModel(alpha);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/HoltLinearModel.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/HoltLinearModel.java
index d8a591972ecd0..1819333738502 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/HoltLinearModel.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/HoltLinearModel.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.search.aggregations.pipeline.movavg.models;
 
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -191,8 +190,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
 
     public static final AbstractModelParser PARSER = new AbstractModelParser() {
         @Override
-        public MovAvgModel parse(@Nullable Map<String, Object> settings, String pipelineName, int windowSize,
-                                 ParseFieldMatcher parseFieldMatcher) throws ParseException {
+        public MovAvgModel parse(@Nullable Map<String, Object> settings, String pipelineName, int windowSize) throws ParseException {
 
             double alpha = parseDoubleParam(settings, "alpha", DEFAULT_ALPHA);
             double beta = parseDoubleParam(settings, "beta", DEFAULT_BETA);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/HoltWintersModel.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/HoltWintersModel.java
index 2130faf8674d8..92b2e4d3ea26e 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/HoltWintersModel.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/HoltWintersModel.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -59,17 +58,16 @@ public enum SeasonalityType {
          * Parse a string SeasonalityType into the byte enum
          *
          * @param text                SeasonalityType in string format (e.g. "add")
-         * @param parseFieldMatcher   Matcher for field names
          * @return                    SeasonalityType enum
          */
         @Nullable
-        public static SeasonalityType parse(String text, ParseFieldMatcher parseFieldMatcher) {
+        public static SeasonalityType parse(String text) {
             if (text == null) {
                 return null;
             }
             SeasonalityType result = null;
             for (SeasonalityType policy : values()) {
-                if (parseFieldMatcher.match(text, policy.parseField)) {
+                if (policy.parseField.match(text)) {
                     result = policy;
                     break;
                 }
@@ -379,8 +377,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
 
     public static final AbstractModelParser PARSER = new AbstractModelParser() {
         @Override
-        public MovAvgModel parse(@Nullable Map<String, Object> settings, String pipelineName, int windowSize,
-                                 ParseFieldMatcher parseFieldMatcher) throws ParseException {
+        public MovAvgModel parse(@Nullable Map<String, Object> settings, String pipelineName, int windowSize) throws ParseException {
 
             double alpha = parseDoubleParam(settings, "alpha", DEFAULT_ALPHA);
             double beta = parseDoubleParam(settings, "beta", DEFAULT_BETA);
@@ -399,7 +396,7 @@ public MovAvgModel parse(@Nullable Map<String, Object> settings, String pipeline
                 Object value = settings.get("type");
                 if (value != null) {
                     if (value instanceof String) {
-                        seasonalityType = SeasonalityType.parse((String)value, parseFieldMatcher);
+                        seasonalityType = SeasonalityType.parse((String)value);
                         settings.remove("type");
                     } else {
                         throw new ParseException("Parameter [type] must be a String, type `"
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/LinearModel.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/LinearModel.java
index 089f3a430ca48..3eed0bf603baa 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/LinearModel.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/LinearModel.java
@@ -21,7 +21,6 @@
 
 
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -106,8 +105,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
 
     public static final AbstractModelParser PARSER = new AbstractModelParser() {
         @Override
-        public MovAvgModel parse(@Nullable Map<String, Object> settings, String pipelineName, int windowSize,
-                                 ParseFieldMatcher parseFieldMatcher) throws ParseException {
+        public MovAvgModel parse(@Nullable Map<String, Object> settings, String pipelineName, int windowSize) throws ParseException {
             checkUnrecognizedParams(settings);
             return new LinearModel();
         }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/MovAvgModel.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/MovAvgModel.java
index 0837eca38bd92..f64117236d6d4 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/MovAvgModel.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/MovAvgModel.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.search.aggregations.pipeline.movavg.models;
 
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ToXContent;
@@ -143,11 +142,10 @@ public abstract static class AbstractModelParser {
          * @param settings           Map of settings, extracted from the request
          * @param pipelineName       Name of the parent pipeline agg
          * @param windowSize         Size of the window for this moving avg
-         * @param parseFieldMatcher  Matcher for field names
          * @return                   A fully built moving average model
          */
         public abstract MovAvgModel parse(@Nullable Map<String, Object> settings, String pipelineName,
-                                          int windowSize, ParseFieldMatcher parseFieldMatcher) throws ParseException;
+                                          int windowSize) throws ParseException;
 
 
         /**
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/SimpleModel.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/SimpleModel.java
index 1454488188385..e30a59d288711 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/SimpleModel.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/SimpleModel.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.search.aggregations.pipeline.movavg.models;
 
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -99,8 +98,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
 
     public static final AbstractModelParser PARSER = new AbstractModelParser() {
         @Override
-        public MovAvgModel parse(@Nullable Map<String, Object> settings, String pipelineName, int windowSize,
-                                 ParseFieldMatcher parseFieldMatcher) throws ParseException {
+        public MovAvgModel parse(@Nullable Map<String, Object> settings, String pipelineName, int windowSize) throws ParseException {
             checkUnrecognizedParams(settings);
             return new SimpleModel();
         }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/serialdiff/SerialDiffPipelineAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/serialdiff/SerialDiffPipelineAggregationBuilder.java
index f20a4f8da42cf..0acd4c7f1b795 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/serialdiff/SerialDiffPipelineAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/serialdiff/SerialDiffPipelineAggregationBuilder.java
@@ -162,18 +162,18 @@ public static SerialDiffPipelineAggregationBuilder parse(String reducerName, Que
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.VALUE_STRING) {
-                if (context.getParseFieldMatcher().match(currentFieldName, FORMAT)) {
+                if (FORMAT.match(currentFieldName)) {
                     format = parser.text();
-                } else if (context.getParseFieldMatcher().match(currentFieldName, BUCKETS_PATH)) {
+                } else if (BUCKETS_PATH.match(currentFieldName)) {
                     bucketsPaths = new String[] { parser.text() };
-                } else if (context.getParseFieldMatcher().match(currentFieldName, GAP_POLICY)) {
+                } else if (GAP_POLICY.match(currentFieldName)) {
                     gapPolicy = GapPolicy.parse(context, parser.text(), parser.getTokenLocation());
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
                             "Unknown key for a " + token + " in [" + reducerName + "]: [" + currentFieldName + "].");
                 }
             } else if (token == XContentParser.Token.VALUE_NUMBER) {
-                if (context.getParseFieldMatcher().match(currentFieldName, LAG)) {
+                if (LAG.match(currentFieldName)) {
                     lag = parser.intValue(true);
                     if (lag <= 0) {
                         throw new ParsingException(parser.getTokenLocation(),
@@ -186,7 +186,7 @@ public static SerialDiffPipelineAggregationBuilder parse(String reducerName, Que
                             "Unknown key for a " + token + " in [" + reducerName + "]: [" + currentFieldName + "].");
                 }
             } else if (token == XContentParser.Token.START_ARRAY) {
-                if (context.getParseFieldMatcher().match(currentFieldName, BUCKETS_PATH)) {
+                if (BUCKETS_PATH.match(currentFieldName)) {
                     List<String> paths = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         String path = parser.text();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/serialdiff/SerialDiffPipelineAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/serialdiff/SerialDiffPipelineAggregator.java
index 3216d5527dc76..d438104be7fe4 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/serialdiff/SerialDiffPipelineAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/serialdiff/SerialDiffPipelineAggregator.java
@@ -26,10 +26,10 @@
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext;
-import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
+import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation;
 import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket;
 import org.elasticsearch.search.aggregations.bucket.histogram.HistogramFactory;
-import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
 import org.elasticsearch.search.aggregations.pipeline.InternalSimpleValue;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
@@ -80,15 +80,17 @@ public String getWriteableName() {
 
     @Override
     public InternalAggregation reduce(InternalAggregation aggregation, ReduceContext reduceContext) {
-        MultiBucketsAggregation histo = (MultiBucketsAggregation) aggregation;
-        List<? extends Bucket> buckets = histo.getBuckets();
+        InternalMultiBucketAggregation<? extends InternalMultiBucketAggregation, ? extends InternalMultiBucketAggregation.InternalBucket>
+                histo = (InternalMultiBucketAggregation<? extends InternalMultiBucketAggregation, ? extends
+                InternalMultiBucketAggregation.InternalBucket>) aggregation;
+        List<? extends InternalMultiBucketAggregation.InternalBucket> buckets = histo.getBuckets();
         HistogramFactory factory = (HistogramFactory) histo;
 
         List<Bucket> newBuckets = new ArrayList<>();
         EvictingQueue<Double> lagWindow = new EvictingQueue<>(lag);
         int counter = 0;
 
-        for (Bucket bucket : buckets) {
+        for (InternalMultiBucketAggregation.InternalBucket bucket : buckets) {
             Double thisBucketValue = resolveBucketValue(histo, bucket, bucketsPaths()[0], gapPolicy);
             Bucket newBucket = bucket;
 
@@ -111,17 +113,14 @@ public InternalAggregation reduce(InternalAggregation aggregation, ReduceContext
             if (!Double.isNaN(thisBucketValue) && !Double.isNaN(lagValue)) {
                 double diff = thisBucketValue - lagValue;
 
-                List<InternalAggregation> aggs = StreamSupport.stream(bucket.getAggregations().spliterator(), false).map((p) -> {
-                    return (InternalAggregation) p;
-                }).collect(Collectors.toList());
-                aggs.add(new InternalSimpleValue(name(), diff, formatter, new ArrayList<PipelineAggregator>(), metaData()));
+                List<InternalAggregation> aggs = StreamSupport.stream(bucket.getAggregations().spliterator(), false).map(
+                        (p) -> (InternalAggregation) p).collect(Collectors.toList());
+                aggs.add(new InternalSimpleValue(name(), diff, formatter, new ArrayList<>(), metaData()));
                 newBucket = factory.createBucket(factory.getKey(bucket), bucket.getDocCount(), new InternalAggregations(aggs));
             }
 
-
             newBuckets.add(newBucket);
             lagWindow.add(thisBucketValue);
-
         }
         return factory.createAggregation(newBuckets);
     }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/AbstractValuesSourceParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/AbstractValuesSourceParser.java
deleted file mode 100644
index 57eea9ccf65b4..0000000000000
--- a/core/src/main/java/org/elasticsearch/search/aggregations/support/AbstractValuesSourceParser.java
+++ /dev/null
@@ -1,220 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.aggregations.support;
-
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.script.Script;
-import org.elasticsearch.script.Script.ScriptField;
-import org.elasticsearch.search.aggregations.Aggregator;
-import org.joda.time.DateTimeZone;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- *
- */
-public abstract class AbstractValuesSourceParser<VS extends ValuesSource>
-        implements Aggregator.Parser {
-    static final ParseField TIME_ZONE = new ParseField("time_zone");
-
-    public abstract static class AnyValuesSourceParser extends AbstractValuesSourceParser<ValuesSource> {
-
-        protected AnyValuesSourceParser(boolean scriptable, boolean formattable) {
-            super(scriptable, formattable, false, ValuesSourceType.ANY, null);
-        }
-    }
-
-    public abstract static class NumericValuesSourceParser extends AbstractValuesSourceParser<ValuesSource.Numeric> {
-
-        protected NumericValuesSourceParser(boolean scriptable, boolean formattable, boolean timezoneAware) {
-            super(scriptable, formattable, timezoneAware, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
-        }
-    }
-
-    public abstract static class BytesValuesSourceParser extends AbstractValuesSourceParser<ValuesSource.Bytes> {
-
-        protected BytesValuesSourceParser(boolean scriptable, boolean formattable) {
-            super(scriptable, formattable, false, ValuesSourceType.BYTES, ValueType.STRING);
-        }
-    }
-
-    public abstract static class GeoPointValuesSourceParser extends AbstractValuesSourceParser<ValuesSource.GeoPoint> {
-
-        protected GeoPointValuesSourceParser(boolean scriptable, boolean formattable) {
-            super(scriptable, formattable, false, ValuesSourceType.GEOPOINT, ValueType.GEOPOINT);
-        }
-    }
-
-    private boolean scriptable = true;
-    private boolean formattable = false;
-    private boolean timezoneAware = false;
-    private ValuesSourceType valuesSourceType = null;
-    private ValueType targetValueType = null;
-
-    private AbstractValuesSourceParser(boolean scriptable, boolean formattable, boolean timezoneAware, ValuesSourceType valuesSourceType,
-            ValueType targetValueType) {
-        this.timezoneAware = timezoneAware;
-        this.valuesSourceType = valuesSourceType;
-        this.targetValueType = targetValueType;
-        this.scriptable = scriptable;
-        this.formattable = formattable;
-    }
-
-    @Override
-    public final ValuesSourceAggregationBuilder<VS, ?> parse(String aggregationName, QueryParseContext context)
-            throws IOException {
-
-        XContentParser parser = context.parser();
-        String field = null;
-        Script script = null;
-        ValueType valueType = null;
-        String format = null;
-        Object missing = null;
-        DateTimeZone timezone = null;
-        Map<ParseField, Object> otherOptions = new HashMap<>();
-        XContentParseContext parserContext =
-                new XContentParseContext(parser, context.getParseFieldMatcher(), context.getDefaultScriptLanguage());
-
-        XContentParser.Token token;
-        String currentFieldName = null;
-        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-            if (token == XContentParser.Token.FIELD_NAME) {
-                currentFieldName = parser.currentName();
-            } else if ("missing".equals(currentFieldName) && token.isValue()) {
-                missing = parser.objectText();
-            } else if (timezoneAware && context.getParseFieldMatcher().match(currentFieldName, TIME_ZONE)) {
-                if (token == XContentParser.Token.VALUE_STRING) {
-                    timezone = DateTimeZone.forID(parser.text());
-                } else if (token == XContentParser.Token.VALUE_NUMBER) {
-                    timezone = DateTimeZone.forOffsetHours(parser.intValue());
-                } else {
-                    throw new ParsingException(parser.getTokenLocation(),
-                            "Unexpected token " + token + " [" + currentFieldName + "] in [" + aggregationName + "].");
-                }
-            } else if (token == XContentParser.Token.VALUE_STRING) {
-                if ("field".equals(currentFieldName)) {
-                    field = parser.text();
-                } else if (formattable && "format".equals(currentFieldName)) {
-                    format = parser.text();
-                } else if (scriptable) {
-                    if ("value_type".equals(currentFieldName) || "valueType".equals(currentFieldName)) {
-                        valueType = ValueType.resolveForScript(parser.text());
-                        if (targetValueType != null && valueType.isNotA(targetValueType)) {
-                            throw new ParsingException(parser.getTokenLocation(),
-                                    "Aggregation [" + aggregationName + "] was configured with an incompatible value type ["
-                                            + valueType + "]. It can only work on value of type ["
-                                            + targetValueType + "]");
-                        }
-                    } else if (!token(aggregationName, currentFieldName, token, parserContext, otherOptions)) {
-                        throw new ParsingException(parser.getTokenLocation(),
-                                "Unexpected token " + token + " [" + currentFieldName + "] in [" + aggregationName + "].");
-                    }
-                } else if (!token(aggregationName, currentFieldName, token, parserContext, otherOptions)) {
-                    throw new ParsingException(parser.getTokenLocation(),
-                            "Unexpected token " + token + " [" + currentFieldName + "] in [" + aggregationName + "].");
-                }
-            } else if (scriptable && token == XContentParser.Token.START_OBJECT) {
-                if (context.getParseFieldMatcher().match(currentFieldName, ScriptField.SCRIPT)) {
-                    script = Script.parse(parser, context.getParseFieldMatcher(), context.getDefaultScriptLanguage());
-                } else if (!token(aggregationName, currentFieldName, token, parserContext, otherOptions)) {
-                    throw new ParsingException(parser.getTokenLocation(),
-                            "Unexpected token " + token + " [" + currentFieldName + "] in [" + aggregationName + "].");
-                }
-            } else if (!token(aggregationName, currentFieldName, token, parserContext, otherOptions)) {
-                throw new ParsingException(parser.getTokenLocation(),
-                        "Unexpected token " + token + " [" + currentFieldName + "] in [" + aggregationName + "].");
-            }
-        }
-
-        ValuesSourceAggregationBuilder<VS, ?> factory = createFactory(aggregationName, this.valuesSourceType, this.targetValueType,
-                otherOptions);
-        if (field != null) {
-            factory.field(field);
-        }
-        if (script != null) {
-            factory.script(script);
-        }
-        if (valueType != null) {
-            factory.valueType(valueType);
-        }
-        if (format != null) {
-            factory.format(format);
-        }
-        if (missing != null) {
-            factory.missing(missing);
-        }
-        if (timezone != null) {
-            factory.timeZone(timezone);
-        }
-        return factory;
-    }
-
-    /**
-     * Creates a {@link ValuesSourceAggregationBuilder} from the information
-     * gathered by the subclass. Options parsed in
-     * {@link AbstractValuesSourceParser} itself will be added to the factory
-     * after it has been returned by this method.
-     *
-     * @param aggregationName
-     *            the name of the aggregation
-     * @param valuesSourceType
-     *            the type of the {@link ValuesSource}
-     * @param targetValueType
-     *            the target type of the final value output by the aggregation
-     * @param otherOptions
-     *            a {@link Map} containing the extra options parsed by the
-     *            {@link #token(String, String, XContentParser.Token, XContentParseContext, Map)}
-     *            method
-     * @return the created factory
-     */
-    protected abstract ValuesSourceAggregationBuilder<VS, ?> createFactory(String aggregationName, ValuesSourceType valuesSourceType,
-                                                                           ValueType targetValueType, Map<ParseField, Object> otherOptions);
-
-    /**
-     * Allows subclasses of {@link AbstractValuesSourceParser} to parse extra
-     * parameters and store them in a {@link Map} which will later be passed to
-     * {@link #createFactory(String, ValuesSourceType, ValueType, Map)}.
-     *
-     * @param aggregationName
-     *            the name of the aggregation
-     * @param currentFieldName
-     *            the name of the current field being parsed
-     * @param token
-     *            the current token for the parser
-     * @param context
-     *            the query context
-     * @param otherOptions
-     *            a {@link Map} of options to be populated by successive calls
-     *            to this method which will then be passed to the
-     *            {@link #createFactory(String, ValuesSourceType, ValueType, Map)}
-     *            method
-     * @return <code>true</code> if the current token was correctly parsed,
-     *         <code>false</code> otherwise
-     * @throws IOException
-     *             if an error occurs whilst parsing
-     */
-    protected abstract boolean token(String aggregationName, String currentFieldName, XContentParser.Token token,
-                                     XContentParseContext context, Map<ParseField, Object> otherOptions) throws IOException;
-}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/AggregationContext.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/AggregationContext.java
deleted file mode 100644
index 79549f87392d0..0000000000000
--- a/core/src/main/java/org/elasticsearch/search/aggregations/support/AggregationContext.java
+++ /dev/null
@@ -1,189 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.search.aggregations.support;
-
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.geo.GeoPoint;
-import org.elasticsearch.common.geo.GeoUtils;
-import org.elasticsearch.common.util.BigArrays;
-import org.elasticsearch.index.fielddata.IndexFieldData;
-import org.elasticsearch.index.fielddata.IndexGeoPointFieldData;
-import org.elasticsearch.index.fielddata.IndexNumericFieldData;
-import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
-import org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData;
-import org.elasticsearch.search.SearchParseException;
-import org.elasticsearch.search.aggregations.AggregationExecutionException;
-import org.elasticsearch.search.internal.SearchContext;
-import org.joda.time.DateTimeZone;
-
-import java.io.IOException;
-
-public class AggregationContext {
-
-    private final SearchContext searchContext;
-
-    public AggregationContext(SearchContext searchContext) {
-        this.searchContext = searchContext;
-    }
-
-    public SearchContext searchContext() {
-        return searchContext;
-    }
-
-    public BigArrays bigArrays() {
-        return searchContext.bigArrays();
-    }
-
-    /** Get a value source given its configuration. A return value of null indicates that
-     *  no value source could be built. */
-    @Nullable
-    public <VS extends ValuesSource> VS valuesSource(ValuesSourceConfig<VS> config, SearchContext context) throws IOException {
-        if (!config.valid()) {
-            throw new IllegalStateException(
-                    "value source config is invalid; must have either a field context or a script or marked as unwrapped");
-        }
-
-        final VS vs;
-        if (config.unmapped()) {
-            if (config.missing() == null) {
-                // otherwise we will have values because of the missing value
-                vs = null;
-            } else if (config.valueSourceType() == ValuesSourceType.NUMERIC) {
-                vs = (VS) ValuesSource.Numeric.EMPTY;
-            } else if (config.valueSourceType() == ValuesSourceType.GEOPOINT) {
-                vs = (VS) ValuesSource.GeoPoint.EMPTY;
-            } else if (config.valueSourceType() == ValuesSourceType.ANY || config.valueSourceType() == ValuesSourceType.BYTES) {
-                vs = (VS) ValuesSource.Bytes.WithOrdinals.EMPTY;
-            } else {
-                throw new SearchParseException(searchContext, "Can't deal with unmapped ValuesSource type "
-                    + config.valueSourceType(), null);
-            }
-        } else {
-            vs = originalValuesSource(config);
-        }
-
-        if (config.missing() == null) {
-            return vs;
-        }
-
-        if (vs instanceof ValuesSource.Bytes) {
-            final BytesRef missing = new BytesRef(config.missing().toString());
-            if (vs instanceof ValuesSource.Bytes.WithOrdinals) {
-                return (VS) MissingValues.replaceMissing((ValuesSource.Bytes.WithOrdinals) vs, missing);
-            } else {
-                return (VS) MissingValues.replaceMissing((ValuesSource.Bytes) vs, missing);
-            }
-        } else if (vs instanceof ValuesSource.Numeric) {
-            Number missing = null;
-            if (config.missing() instanceof Number) {
-                missing = (Number) config.missing();
-            } else {
-                if (config.fieldContext() != null && config.fieldContext().fieldType() != null) {
-                    missing = config.fieldContext().fieldType().docValueFormat(null, DateTimeZone.UTC)
-                            .parseDouble(config.missing().toString(), false, context::nowInMillis);
-                } else {
-                    missing = Double.parseDouble(config.missing().toString());
-                }
-            }
-            return (VS) MissingValues.replaceMissing((ValuesSource.Numeric) vs, missing);
-        } else if (vs instanceof ValuesSource.GeoPoint) {
-            // TODO: also support the structured formats of geo points
-            final GeoPoint missing = GeoUtils.parseGeoPoint(config.missing().toString(), new GeoPoint());
-            return (VS) MissingValues.replaceMissing((ValuesSource.GeoPoint) vs, missing);
-        } else {
-            // Should not happen
-            throw new SearchParseException(searchContext, "Can't apply missing values on a " + vs.getClass(), null);
-        }
-    }
-
-    /**
-     * Return the original values source, before we apply `missing`.
-     */
-    private <VS extends ValuesSource> VS originalValuesSource(ValuesSourceConfig<VS> config) throws IOException {
-        if (config.fieldContext() == null) {
-            if (config.valueSourceType() == ValuesSourceType.NUMERIC) {
-                return (VS) numericScript(config);
-            }
-            if (config.valueSourceType() == ValuesSourceType.BYTES) {
-                return (VS) bytesScript(config);
-            }
-            throw new AggregationExecutionException("value source of type [" + config.valueSourceType().name()
-                    + "] is not supported by scripts");
-        }
-
-        if (config.valueSourceType() == ValuesSourceType.NUMERIC) {
-            return (VS) numericField(config);
-        }
-        if (config.valueSourceType() == ValuesSourceType.GEOPOINT) {
-            return (VS) geoPointField(config);
-        }
-        // falling back to bytes values
-        return (VS) bytesField(config);
-    }
-
-    private ValuesSource.Numeric numericScript(ValuesSourceConfig<?> config) throws IOException {
-        return new ValuesSource.Numeric.Script(config.script(), config.scriptValueType());
-    }
-
-    private ValuesSource.Numeric numericField(ValuesSourceConfig<?> config) throws IOException {
-
-        if (!(config.fieldContext().indexFieldData() instanceof IndexNumericFieldData)) {
-            throw new IllegalArgumentException("Expected numeric type on field [" + config.fieldContext().field() +
-                    "], but got [" + config.fieldContext().fieldType().typeName() + "]");
-        }
-
-        ValuesSource.Numeric dataSource = new ValuesSource.Numeric.FieldData((IndexNumericFieldData)config.fieldContext().indexFieldData());
-        if (config.script() != null) {
-            dataSource = new ValuesSource.Numeric.WithScript(dataSource, config.script());
-        }
-        return dataSource;
-    }
-
-    private ValuesSource bytesField(ValuesSourceConfig<?> config) throws IOException {
-        final IndexFieldData<?> indexFieldData = config.fieldContext().indexFieldData();
-        ValuesSource dataSource;
-        if (indexFieldData instanceof ParentChildIndexFieldData) {
-            dataSource = new ValuesSource.Bytes.WithOrdinals.ParentChild((ParentChildIndexFieldData) indexFieldData);
-        } else if (indexFieldData instanceof IndexOrdinalsFieldData) {
-            dataSource = new ValuesSource.Bytes.WithOrdinals.FieldData((IndexOrdinalsFieldData) indexFieldData);
-        } else {
-            dataSource = new ValuesSource.Bytes.FieldData(indexFieldData);
-        }
-        if (config.script() != null) {
-            dataSource = new ValuesSource.WithScript(dataSource, config.script());
-        }
-        return dataSource;
-    }
-
-    private ValuesSource.Bytes bytesScript(ValuesSourceConfig<?> config) throws IOException {
-        return new ValuesSource.Bytes.Script(config.script());
-    }
-
-    private ValuesSource.GeoPoint geoPointField(ValuesSourceConfig<?> config) throws IOException {
-
-        if (!(config.fieldContext().indexFieldData() instanceof IndexGeoPointFieldData)) {
-            throw new IllegalArgumentException("Expected geo_point type on field [" + config.fieldContext().field() +
-                    "], but got [" + config.fieldContext().fieldType().typeName() + "]");
-        }
-
-        return new ValuesSource.GeoPoint.Fielddata((IndexGeoPointFieldData) config.fieldContext().indexFieldData());
-    }
-
-}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/GeoPointParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/GeoPointParser.java
deleted file mode 100644
index fd2f3636d173f..0000000000000
--- a/core/src/main/java/org/elasticsearch/search/aggregations/support/GeoPointParser.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.aggregations.support;
-
-
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.geo.GeoPoint;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.search.aggregations.InternalAggregation;
-
-import java.io.IOException;
-import java.util.Map;
-
-/**
- *
- */
-public class GeoPointParser {
-
-    private final InternalAggregation.Type aggType;
-    private final ParseField field;
-
-    public GeoPointParser(InternalAggregation.Type aggType, ParseField field) {
-        this.aggType = aggType;
-        this.field = field;
-    }
-
-    public boolean token(String aggName, String currentFieldName, XContentParser.Token token, XContentParser parser,
-            ParseFieldMatcher parseFieldMatcher, Map<ParseField, Object> otherOptions) throws IOException {
-        if (!parseFieldMatcher.match(currentFieldName, field)) {
-            return false;
-        }
-        if (token == XContentParser.Token.VALUE_STRING) {
-            GeoPoint point = new GeoPoint();
-            point.resetFromString(parser.text());
-            otherOptions.put(field, point);
-            return true;
-        }
-        if (token == XContentParser.Token.START_ARRAY) {
-            double lat = Double.NaN;
-            double lon = Double.NaN;
-            while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
-                if (Double.isNaN(lon)) {
-                    lon = parser.doubleValue();
-                } else if (Double.isNaN(lat)) {
-                    lat = parser.doubleValue();
-                } else {
-                    throw new ParsingException(parser.getTokenLocation(), "malformed [" + currentFieldName + "] geo point array in ["
-                            + aggName + "] " + aggType + " aggregation. a geo point array must be of the form [lon, lat]");
-                }
-            }
-            GeoPoint point = new GeoPoint(lat, lon);
-            otherOptions.put(field, point);
-            return true;
-        }
-        if (token == XContentParser.Token.START_OBJECT) {
-            double lat = Double.NaN;
-            double lon = Double.NaN;
-            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                if (token == XContentParser.Token.FIELD_NAME) {
-                    currentFieldName = parser.currentName();
-                } else if (token == XContentParser.Token.VALUE_NUMBER) {
-                    if ("lat".equals(currentFieldName)) {
-                        lat = parser.doubleValue();
-                    } else if ("lon".equals(currentFieldName)) {
-                        lon = parser.doubleValue();
-                    }
-                }
-            }
-            if (Double.isNaN(lat) || Double.isNaN(lon)) {
-                throw new ParsingException(parser.getTokenLocation(),
-                        "malformed [" + currentFieldName + "] geo point object. either [lat] or [lon] (or both) are " + "missing in ["
-                                + aggName + "] " + aggType + " aggregation");
-            }
-            GeoPoint point = new GeoPoint(lat, lon);
-            otherOptions.put(field, point);
-            return true;
-        }
-        return false;
-    }
-
-}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java
index 28a4bd2567ce4..4a01b67b78078 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java
@@ -80,7 +80,7 @@ public int count() {
     }
 
     public static ValuesSource.Numeric replaceMissing(final ValuesSource.Numeric valuesSource, final Number missing) {
-        final boolean missingIsFloat = missing.longValue() != (long) missing.doubleValue();
+        final boolean missingIsFloat = missing.doubleValue() % 1 != 0;
         final boolean isFloatingPoint = valuesSource.isFloatingPoint() || missingIsFloat;
         return new ValuesSource.Numeric() {
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValueType.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValueType.java
index 9e0bf350beb96..6c918feb57a5c 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValueType.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValueType.java
@@ -71,6 +71,7 @@ public boolean isNumeric() {
         }
     },
     IP((byte) 6, "ip", "ip", ValuesSourceType.BYTES, IndexFieldData.class, DocValueFormat.IP),
+    // TODO: what is the difference between "number" and "numeric"?
     NUMERIC((byte) 7, "numeric", "numeric", ValuesSourceType.NUMERIC, IndexNumericFieldData.class, DocValueFormat.RAW) {
         @Override
         public boolean isNumeric() {
@@ -82,6 +83,12 @@ public boolean isNumeric() {
         public boolean isGeoPoint() {
             return true;
         }
+    },
+    BOOLEAN((byte) 9, "boolean", "boolean", ValuesSourceType.NUMERIC, IndexNumericFieldData.class, DocValueFormat.BOOLEAN) {
+        @Override
+        public boolean isNumeric() {
+            return super.isNumeric();
+        }
     };
 
     final String description;
@@ -91,7 +98,7 @@ public boolean isGeoPoint() {
     private final byte id;
     private String preferredName;
 
-    private ValueType(byte id, String description, String preferredName, ValuesSourceType valuesSourceType,
+    ValueType(byte id, String description, String preferredName, ValuesSourceType valuesSourceType,
             Class<? extends IndexFieldData> fieldDataType, DocValueFormat defaultFormat) {
         this.id = id;
         this.description = description;
@@ -153,7 +160,9 @@ public static ValueType resolveForScript(String type) {
             case "byte":    return LONG;
             case "date":    return DATE;
             case "ip":      return IP;
+            case "boolean": return BOOLEAN;
             default:
+                // TODO: do not be lenient here
                 return null;
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java
index 124b7ba71f4b5..ca3424ae7042b 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java
@@ -23,7 +23,6 @@
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.RandomAccessOrds;
-import org.apache.lucene.index.SortedDocValues;
 import org.apache.lucene.index.SortedNumericDocValues;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Scorer;
@@ -31,19 +30,16 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.lucene.ScorerAware;
 import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData;
-import org.elasticsearch.index.fielddata.AtomicParentChildFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexGeoPointFieldData;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
-import org.elasticsearch.index.fielddata.IndexParentChildFieldData;
 import org.elasticsearch.index.fielddata.MultiGeoPointValues;
 import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
 import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
 import org.elasticsearch.index.fielddata.SortingBinaryDocValues;
 import org.elasticsearch.index.fielddata.SortingNumericDocValues;
 import org.elasticsearch.index.fielddata.SortingNumericDoubleValues;
-import org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData;
 import org.elasticsearch.script.LeafSearchScript;
 import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.aggregations.support.ValuesSource.WithScript.BytesValues;
@@ -154,40 +150,6 @@ public RandomAccessOrds globalOrdinalsValues(LeafReaderContext context) {
             }
         }
 
-        public static class ParentChild extends Bytes {
-
-            protected final ParentChildIndexFieldData indexFieldData;
-
-            public ParentChild(ParentChildIndexFieldData indexFieldData) {
-                this.indexFieldData = indexFieldData;
-            }
-
-            public long globalMaxOrd(IndexSearcher indexSearcher, String type) {
-                DirectoryReader indexReader = (DirectoryReader) indexSearcher.getIndexReader();
-                if (indexReader.leaves().isEmpty()) {
-                    return 0;
-                } else {
-                    LeafReaderContext atomicReaderContext = indexReader.leaves().get(0);
-                    IndexParentChildFieldData globalFieldData = indexFieldData.loadGlobal(indexReader);
-                    AtomicParentChildFieldData afd = globalFieldData.load(atomicReaderContext);
-                    SortedDocValues values = afd.getOrdinalsValues(type);
-                    return values.getValueCount();
-                }
-            }
-
-            public SortedDocValues globalOrdinalsValues(String type, LeafReaderContext context) {
-                final IndexParentChildFieldData global = indexFieldData.loadGlobal((DirectoryReader)context.parent.reader());
-                final AtomicParentChildFieldData atomicFieldData = global.load(context);
-                return atomicFieldData.getOrdinalsValues(type);
-            }
-
-            @Override
-            public SortedBinaryDocValues bytesValues(LeafReaderContext context) {
-                final AtomicParentChildFieldData atomicFieldData = indexFieldData.load(context);
-                return atomicFieldData.getBytesValues();
-            }
-        }
-
         public static class FieldData extends Bytes {
 
             protected final IndexFieldData<?> indexFieldData;
@@ -318,7 +280,7 @@ static class LongValues extends SortingNumericDocValues implements ScorerAware {
                 private final SortedNumericDocValues longValues;
                 private final LeafSearchScript script;
 
-                public LongValues(SortedNumericDocValues values, LeafSearchScript script) {
+                LongValues(SortedNumericDocValues values, LeafSearchScript script) {
                     this.longValues = values;
                     this.script = script;
                 }
@@ -346,7 +308,7 @@ static class DoubleValues extends SortingNumericDoubleValues implements ScorerAw
                 private final SortedNumericDoubleValues doubleValues;
                 private final LeafSearchScript script;
 
-                public DoubleValues(SortedNumericDoubleValues values, LeafSearchScript script) {
+                DoubleValues(SortedNumericDoubleValues values, LeafSearchScript script) {
                     this.doubleValues = values;
                     this.script = script;
                 }
@@ -462,7 +424,7 @@ static class BytesValues extends SortingBinaryDocValues implements ScorerAware {
             private final SortedBinaryDocValues bytesValues;
             private final LeafSearchScript script;
 
-            public BytesValues(SortedBinaryDocValues bytesValues, LeafSearchScript script) {
+            BytesValues(SortedBinaryDocValues bytesValues, LeafSearchScript script) {
                 this.bytesValues = bytesValues;
                 this.script = script;
             }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregationBuilder.java
index 8f14a1ffaf9d1..de76d1490e942 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregationBuilder.java
@@ -18,29 +18,19 @@
  */
 package org.elasticsearch.search.aggregations.support;
 
-import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.index.fielddata.IndexFieldData;
-import org.elasticsearch.index.fielddata.IndexGeoPointFieldData;
-import org.elasticsearch.index.fielddata.IndexNumericFieldData;
-import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptContext;
-import org.elasticsearch.script.SearchScript;
-import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.AggregationInitializationException;
 import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
+import org.elasticsearch.search.aggregations.AggregationInitializationException;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.internal.SearchContext;
 import org.joda.time.DateTimeZone;
 
 import java.io.IOException;
-import java.util.Collections;
 import java.util.Objects;
 
 /**
@@ -52,28 +42,29 @@ public abstract class ValuesSourceAggregationBuilder<VS extends ValuesSource, AB
     public abstract static class LeafOnly<VS extends ValuesSource, AB extends ValuesSourceAggregationBuilder<VS, AB>>
             extends ValuesSourceAggregationBuilder<VS, AB> {
 
-        protected LeafOnly(String name, Type type, ValuesSourceType valuesSourceType, ValueType targetValueType) {
-            super(name, type, valuesSourceType, targetValueType);
+        protected LeafOnly(String name, ValuesSourceType valuesSourceType, ValueType targetValueType) {
+            super(name, valuesSourceType, targetValueType);
         }
 
         /**
          * Read an aggregation from a stream that does not serialize its targetValueType. This should be used by most subclasses.
          */
-        protected LeafOnly(StreamInput in, Type type, ValuesSourceType valuesSourceType, ValueType targetValueType) throws IOException {
-            super(in, type, valuesSourceType, targetValueType);
+        protected LeafOnly(StreamInput in, ValuesSourceType valuesSourceType, ValueType targetValueType) throws IOException {
+            super(in, valuesSourceType, targetValueType);
         }
 
         /**
          * Read an aggregation from a stream that serializes its targetValueType. This should only be used by subclasses that override
          * {@link #serializeTargetValueType()} to return true.
          */
-        protected LeafOnly(StreamInput in, Type type, ValuesSourceType valuesSourceType) throws IOException {
-            super(in, type, valuesSourceType);
+        protected LeafOnly(StreamInput in, ValuesSourceType valuesSourceType) throws IOException {
+            super(in, valuesSourceType);
         }
 
         @Override
         public AB subAggregations(Builder subFactories) {
-            throw new AggregationInitializationException("Aggregator [" + name + "] of type [" + type + "] cannot accept sub-aggregations");
+            throw new AggregationInitializationException("Aggregator [" + name + "] of type ["
+                    + getType() + "] cannot accept sub-aggregations");
         }
     }
 
@@ -87,8 +78,8 @@ public AB subAggregations(Builder subFactories) {
     private DateTimeZone timeZone = null;
     protected ValuesSourceConfig<VS> config;
 
-    protected ValuesSourceAggregationBuilder(String name, Type type, ValuesSourceType valuesSourceType, ValueType targetValueType) {
-        super(name, type);
+    protected ValuesSourceAggregationBuilder(String name, ValuesSourceType valuesSourceType, ValueType targetValueType) {
+        super(name);
         if (valuesSourceType == null) {
             throw new IllegalArgumentException("[valuesSourceType] must not be null: [" + name + "]");
         }
@@ -99,9 +90,9 @@ protected ValuesSourceAggregationBuilder(String name, Type type, ValuesSourceTyp
     /**
      * Read an aggregation from a stream that does not serialize its targetValueType. This should be used by most subclasses.
      */
-    protected ValuesSourceAggregationBuilder(StreamInput in, Type type, ValuesSourceType valuesSourceType, ValueType targetValueType)
+    protected ValuesSourceAggregationBuilder(StreamInput in, ValuesSourceType valuesSourceType, ValueType targetValueType)
             throws IOException {
-        super(in, type);
+        super(in);
         assert false == serializeTargetValueType() : "Wrong read constructor called for subclass that provides its targetValueType";
         this.valuesSourceType = valuesSourceType;
         this.targetValueType = targetValueType;
@@ -112,8 +103,8 @@ protected ValuesSourceAggregationBuilder(StreamInput in, Type type, ValuesSource
      * Read an aggregation from a stream that serializes its targetValueType. This should only be used by subclasses that override
      * {@link #serializeTargetValueType()} to return true.
      */
-    protected ValuesSourceAggregationBuilder(StreamInput in, Type type, ValuesSourceType valuesSourceType) throws IOException {
-        super(in, type);
+    protected ValuesSourceAggregationBuilder(StreamInput in, ValuesSourceType valuesSourceType) throws IOException {
+        super(in);
         assert serializeTargetValueType() : "Wrong read constructor called for subclass that serializes its targetValueType";
         this.valuesSourceType = valuesSourceType;
         this.targetValueType = in.readOptionalWriteable(ValueType::readFromStream);
@@ -294,102 +285,21 @@ public DateTimeZone timeZone() {
     }
 
     @Override
-    protected final ValuesSourceAggregatorFactory<VS, ?> doBuild(AggregationContext context, AggregatorFactory<?> parent,
+    protected final ValuesSourceAggregatorFactory<VS, ?> doBuild(SearchContext context, AggregatorFactory<?> parent,
             AggregatorFactories.Builder subFactoriesBuilder) throws IOException {
         ValuesSourceConfig<VS> config = resolveConfig(context);
         ValuesSourceAggregatorFactory<VS, ?> factory = innerBuild(context, config, parent, subFactoriesBuilder);
         return factory;
     }
 
-    protected ValuesSourceConfig<VS> resolveConfig(AggregationContext context) {
-        ValuesSourceConfig<VS> config = config(context);
-        return config;
-    }
-
-    protected abstract ValuesSourceAggregatorFactory<VS, ?> innerBuild(AggregationContext context, ValuesSourceConfig<VS> config,
-            AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder) throws IOException;
-
-    public ValuesSourceConfig<VS> config(AggregationContext context) {
-
+    protected ValuesSourceConfig<VS> resolveConfig(SearchContext context) {
         ValueType valueType = this.valueType != null ? this.valueType : targetValueType;
-
-        if (field == null) {
-            if (script == null) {
-                @SuppressWarnings("unchecked")
-                ValuesSourceConfig<VS> config = new ValuesSourceConfig(ValuesSourceType.ANY);
-                config.format(resolveFormat(null, valueType));
-                return config;
-            }
-            ValuesSourceType valuesSourceType = valueType != null ? valueType.getValuesSourceType() : this.valuesSourceType;
-            if (valuesSourceType == null || valuesSourceType == ValuesSourceType.ANY) {
-                // the specific value source type is undefined, but for scripts,
-                // we need to have a specific value source
-                // type to know how to handle the script values, so we fallback
-                // on Bytes
-                valuesSourceType = ValuesSourceType.BYTES;
-            }
-            ValuesSourceConfig<VS> config = new ValuesSourceConfig<VS>(valuesSourceType);
-            config.missing(missing);
-            config.timezone(timeZone);
-            config.format(resolveFormat(format, valueType));
-            config.script(createScript(script, context.searchContext()));
-            config.scriptValueType(valueType);
-            return config;
-        }
-
-        MappedFieldType fieldType = context.searchContext().smartNameFieldType(field);
-        if (fieldType == null) {
-            ValuesSourceType valuesSourceType = valueType != null ? valueType.getValuesSourceType() : this.valuesSourceType;
-            ValuesSourceConfig<VS> config = new ValuesSourceConfig<>(valuesSourceType);
-            config.missing(missing);
-            config.timezone(timeZone);
-            config.format(resolveFormat(format, valueType));
-            config.unmapped(true);
-            if (valueType != null) {
-                // todo do we really need this for unmapped?
-                config.scriptValueType(valueType);
-            }
-            return config;
-        }
-
-        IndexFieldData<?> indexFieldData = context.searchContext().fieldData().getForField(fieldType);
-
-        ValuesSourceConfig<VS> config;
-        if (valuesSourceType == ValuesSourceType.ANY) {
-            if (indexFieldData instanceof IndexNumericFieldData) {
-                config = new ValuesSourceConfig<>(ValuesSourceType.NUMERIC);
-            } else if (indexFieldData instanceof IndexGeoPointFieldData) {
-                config = new ValuesSourceConfig<>(ValuesSourceType.GEOPOINT);
-            } else {
-                config = new ValuesSourceConfig<>(ValuesSourceType.BYTES);
-            }
-        } else {
-            config = new ValuesSourceConfig(valuesSourceType);
-        }
-
-        config.fieldContext(new FieldContext(field, indexFieldData, fieldType));
-        config.missing(missing);
-        config.timezone(timeZone);
-        config.script(createScript(script, context.searchContext()));
-        config.format(fieldType.docValueFormat(format, timeZone));
-        return config;
-    }
-
-    private SearchScript createScript(Script script, SearchContext context) {
-        return script == null ? null
-                : context.scriptService().search(context.lookup(), script, ScriptContext.Standard.AGGS, Collections.emptyMap());
+        return ValuesSourceConfig.resolve(context.getQueryShardContext(),
+                valueType, field, script, missing, timeZone, format);
     }
 
-    private static DocValueFormat resolveFormat(@Nullable String format, @Nullable ValueType valueType) {
-        if (valueType == null) {
-            return DocValueFormat.RAW; // we can't figure it out
-        }
-        DocValueFormat valueFormat = valueType.defaultFormat;
-        if (valueFormat instanceof DocValueFormat.Decimal && format != null) {
-            valueFormat = new DocValueFormat.Decimal(format);
-        }
-        return valueFormat;
-    }
+    protected abstract ValuesSourceAggregatorFactory<VS, ?> innerBuild(SearchContext context, ValuesSourceConfig<VS> config,
+            AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder) throws IOException;
 
     @Override
     public final XContentBuilder internalXContent(XContentBuilder builder, Params params) throws IOException {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregatorFactory.java
index 36ab0d505b959..28d82f4cafd72 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregatorFactory.java
@@ -22,8 +22,8 @@
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.search.internal.SearchContext;
 import org.joda.time.DateTimeZone;
 
 import java.io.IOException;
@@ -35,9 +35,9 @@ public abstract class ValuesSourceAggregatorFactory<VS extends ValuesSource, AF
 
     protected ValuesSourceConfig<VS> config;
 
-    public ValuesSourceAggregatorFactory(String name, Type type, ValuesSourceConfig<VS> config, AggregationContext context,
+    public ValuesSourceAggregatorFactory(String name, ValuesSourceConfig<VS> config, SearchContext context,
             AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
-        super(name, type, context, parent, subFactoriesBuilder, metaData);
+        super(name, context, parent, subFactoriesBuilder, metaData);
         this.config = config;
     }
 
@@ -48,7 +48,7 @@ public DateTimeZone timeZone() {
     @Override
     public Aggregator createInternal(Aggregator parent, boolean collectsFromSingleBucket,
             List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException {
-        VS vs = context.valuesSource(config, context.searchContext());
+        VS vs = config.toValuesSource(context.getQueryShardContext());
         if (vs == null) {
             return createUnmapped(parent, pipelineAggregators, metaData);
         }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java
index cb6f76037cf03..e5fac62840fac 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java
@@ -18,15 +18,123 @@
  */
 package org.elasticsearch.search.aggregations.support;
 
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.geo.GeoPoint;
+import org.elasticsearch.common.geo.GeoUtils;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexGeoPointFieldData;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
+import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.AggregationExecutionException;
 import org.joda.time.DateTimeZone;
 
+import java.io.IOException;
+
 /**
- *
+ * A configuration that tells aggregations how to retrieve data from the index
+ * in order to run a specific aggregation.
  */
 public class ValuesSourceConfig<VS extends ValuesSource> {
 
+    /**
+     * Resolve a {@link ValuesSourceConfig} given configuration parameters.
+     */
+    public static <VS extends ValuesSource> ValuesSourceConfig<VS> resolve(
+            QueryShardContext context,
+            ValueType valueType,
+            String field, Script script,
+            Object missing,
+            DateTimeZone timeZone,
+            String format) {
+
+        if (field == null) {
+            if (script == null) {
+                @SuppressWarnings("unchecked")
+                ValuesSourceConfig<VS> config = new ValuesSourceConfig<>(ValuesSourceType.ANY);
+                config.format(resolveFormat(null, valueType));
+                return config;
+            }
+            ValuesSourceType valuesSourceType = valueType != null ? valueType.getValuesSourceType() : ValuesSourceType.ANY;
+            if (valuesSourceType == ValuesSourceType.ANY) {
+                // the specific value source type is undefined, but for scripts,
+                // we need to have a specific value source
+                // type to know how to handle the script values, so we fallback
+                // on Bytes
+                valuesSourceType = ValuesSourceType.BYTES;
+            }
+            ValuesSourceConfig<VS> config = new ValuesSourceConfig<VS>(valuesSourceType);
+            config.missing(missing);
+            config.timezone(timeZone);
+            config.format(resolveFormat(format, valueType));
+            config.script(createScript(script, context));
+            config.scriptValueType(valueType);
+            return config;
+        }
+
+        MappedFieldType fieldType = context.fieldMapper(field);
+        if (fieldType == null) {
+            ValuesSourceType valuesSourceType = valueType != null ? valueType.getValuesSourceType() : ValuesSourceType.ANY;
+            ValuesSourceConfig<VS> config = new ValuesSourceConfig<>(valuesSourceType);
+            config.missing(missing);
+            config.timezone(timeZone);
+            config.format(resolveFormat(format, valueType));
+            config.unmapped(true);
+            if (valueType != null) {
+                // todo do we really need this for unmapped?
+                config.scriptValueType(valueType);
+            }
+            return config;
+        }
+
+        IndexFieldData<?> indexFieldData = context.getForField(fieldType);
+
+        ValuesSourceConfig<VS> config;
+        if (valueType == null) {
+            if (indexFieldData instanceof IndexNumericFieldData) {
+                config = new ValuesSourceConfig<>(ValuesSourceType.NUMERIC);
+            } else if (indexFieldData instanceof IndexGeoPointFieldData) {
+                config = new ValuesSourceConfig<>(ValuesSourceType.GEOPOINT);
+            } else {
+                config = new ValuesSourceConfig<>(ValuesSourceType.BYTES);
+            }
+        } else {
+            config = new ValuesSourceConfig<>(valueType.getValuesSourceType());
+        }
+
+        config.fieldContext(new FieldContext(field, indexFieldData, fieldType));
+        config.missing(missing);
+        config.timezone(timeZone);
+        config.script(createScript(script, context));
+        config.format(fieldType.docValueFormat(format, timeZone));
+        return config;
+    }
+
+    private static SearchScript createScript(Script script, QueryShardContext context) {
+        if (script == null) {
+            return null;
+        } else {
+            return context.getSearchScript(script, ScriptContext.Standard.AGGS);
+        }
+    }
+
+    private static DocValueFormat resolveFormat(@Nullable String format, @Nullable ValueType valueType) {
+        if (valueType == null) {
+            return DocValueFormat.RAW; // we can't figure it out
+        }
+        DocValueFormat valueFormat = valueType.defaultFormat;
+        if (valueFormat instanceof DocValueFormat.Decimal && format != null) {
+            valueFormat = new DocValueFormat.Decimal(format);
+        }
+        return valueFormat;
+    }
+
     private final ValuesSourceType valueSourceType;
     private FieldContext fieldContext;
     private SearchScript script;
@@ -110,4 +218,126 @@ public DateTimeZone timezone() {
     public DocValueFormat format() {
         return format;
     }
+
+    /** Get a value source given its configuration. A return value of null indicates that
+     *  no value source could be built. */
+    @Nullable
+    public VS toValuesSource(QueryShardContext context) throws IOException {
+        if (!valid()) {
+            throw new IllegalStateException(
+                    "value source config is invalid; must have either a field context or a script or marked as unwrapped");
+        }
+
+        final VS vs;
+        if (unmapped()) {
+            if (missing() == null) {
+                // otherwise we will have values because of the missing value
+                vs = null;
+            } else if (valueSourceType() == ValuesSourceType.NUMERIC) {
+                vs = (VS) ValuesSource.Numeric.EMPTY;
+            } else if (valueSourceType() == ValuesSourceType.GEOPOINT) {
+                vs = (VS) ValuesSource.GeoPoint.EMPTY;
+            } else if (valueSourceType() == ValuesSourceType.ANY || valueSourceType() == ValuesSourceType.BYTES) {
+                vs = (VS) ValuesSource.Bytes.WithOrdinals.EMPTY;
+            } else {
+                throw new IllegalArgumentException("Can't deal with unmapped ValuesSource type " + valueSourceType());
+            }
+        } else {
+            vs = originalValuesSource();
+        }
+
+        if (missing() == null) {
+            return vs;
+        }
+
+        if (vs instanceof ValuesSource.Bytes) {
+            final BytesRef missing = new BytesRef(missing().toString());
+            if (vs instanceof ValuesSource.Bytes.WithOrdinals) {
+                return (VS) MissingValues.replaceMissing((ValuesSource.Bytes.WithOrdinals) vs, missing);
+            } else {
+                return (VS) MissingValues.replaceMissing((ValuesSource.Bytes) vs, missing);
+            }
+        } else if (vs instanceof ValuesSource.Numeric) {
+            Number missing = format.parseDouble(missing().toString(), false, context::nowInMillis);
+            return (VS) MissingValues.replaceMissing((ValuesSource.Numeric) vs, missing);
+        } else if (vs instanceof ValuesSource.GeoPoint) {
+            // TODO: also support the structured formats of geo points
+            final GeoPoint missing = GeoUtils.parseGeoPoint(missing().toString(), new GeoPoint());
+            return (VS) MissingValues.replaceMissing((ValuesSource.GeoPoint) vs, missing);
+        } else {
+            // Should not happen
+            throw new IllegalArgumentException("Can't apply missing values on a " + vs.getClass());
+        }
+    }
+
+    /**
+     * Return the original values source, before we apply `missing`.
+     */
+    private VS originalValuesSource() throws IOException {
+        if (fieldContext() == null) {
+            if (valueSourceType() == ValuesSourceType.NUMERIC) {
+                return (VS) numericScript();
+            }
+            if (valueSourceType() == ValuesSourceType.BYTES) {
+                return (VS) bytesScript();
+            }
+            throw new AggregationExecutionException("value source of type [" + valueSourceType().name()
+                    + "] is not supported by scripts");
+        }
+
+        if (valueSourceType() == ValuesSourceType.NUMERIC) {
+            return (VS) numericField();
+        }
+        if (valueSourceType() == ValuesSourceType.GEOPOINT) {
+            return (VS) geoPointField();
+        }
+        // falling back to bytes values
+        return (VS) bytesField();
+    }
+
+    private ValuesSource.Numeric numericScript() throws IOException {
+        return new ValuesSource.Numeric.Script(script(), scriptValueType());
+    }
+
+    private ValuesSource.Numeric numericField() throws IOException {
+
+        if (!(fieldContext().indexFieldData() instanceof IndexNumericFieldData)) {
+            throw new IllegalArgumentException("Expected numeric type on field [" + fieldContext().field() +
+                    "], but got [" + fieldContext().fieldType().typeName() + "]");
+        }
+
+        ValuesSource.Numeric dataSource = new ValuesSource.Numeric.FieldData((IndexNumericFieldData)fieldContext().indexFieldData());
+        if (script() != null) {
+            dataSource = new ValuesSource.Numeric.WithScript(dataSource, script());
+        }
+        return dataSource;
+    }
+
+    private ValuesSource bytesField() throws IOException {
+        final IndexFieldData<?> indexFieldData = fieldContext().indexFieldData();
+        ValuesSource dataSource;
+        if (indexFieldData instanceof IndexOrdinalsFieldData) {
+            dataSource = new ValuesSource.Bytes.WithOrdinals.FieldData((IndexOrdinalsFieldData) indexFieldData);
+        } else {
+            dataSource = new ValuesSource.Bytes.FieldData(indexFieldData);
+        }
+        if (script() != null) {
+            dataSource = new ValuesSource.WithScript(dataSource, script());
+        }
+        return dataSource;
+    }
+
+    private ValuesSource.Bytes bytesScript() throws IOException {
+        return new ValuesSource.Bytes.Script(script());
+    }
+
+    private ValuesSource.GeoPoint geoPointField() throws IOException {
+
+        if (!(fieldContext().indexFieldData() instanceof IndexGeoPointFieldData)) {
+            throw new IllegalArgumentException("Expected geo_point type on field [" + fieldContext().field() +
+                    "], but got [" + fieldContext().fieldType().typeName() + "]");
+        }
+
+        return new ValuesSource.GeoPoint.Fielddata((IndexGeoPointFieldData) fieldContext().indexFieldData());
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceParserHelper.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceParserHelper.java
new file mode 100644
index 0000000000000..7b174d789f4e4
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceParserHelper.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.support;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.script.Script;
+import org.joda.time.DateTimeZone;
+
+public final class ValuesSourceParserHelper {
+    static final ParseField TIME_ZONE = new ParseField("time_zone");
+
+    private ValuesSourceParserHelper() {} // utility class, no instantiation
+
+    public static void declareAnyFields(
+            ObjectParser<? extends ValuesSourceAggregationBuilder<ValuesSource, ?>, QueryParseContext> objectParser,
+            boolean scriptable, boolean formattable) {
+        declareFields(objectParser, scriptable, formattable, false, null);
+    }
+
+    public static void declareNumericFields(
+            ObjectParser<? extends ValuesSourceAggregationBuilder<ValuesSource.Numeric, ?>, QueryParseContext> objectParser,
+            boolean scriptable, boolean formattable, boolean timezoneAware) {
+        declareFields(objectParser, scriptable, formattable, timezoneAware, ValueType.NUMERIC);
+    }
+
+    public static void declareBytesFields(
+            ObjectParser<? extends ValuesSourceAggregationBuilder<ValuesSource.Bytes, ?>, QueryParseContext> objectParser,
+            boolean scriptable, boolean formattable) {
+        declareFields(objectParser, scriptable, formattable, false, ValueType.STRING);
+    }
+
+    public static void declareGeoFields(
+            ObjectParser<? extends ValuesSourceAggregationBuilder<ValuesSource.GeoPoint, ?>, QueryParseContext> objectParser,
+            boolean scriptable, boolean formattable) {
+        declareFields(objectParser, scriptable, formattable, false, ValueType.GEOPOINT);
+    }
+
+    private static <VS extends ValuesSource> void declareFields(
+            ObjectParser<? extends ValuesSourceAggregationBuilder<VS, ?>, QueryParseContext> objectParser,
+            boolean scriptable, boolean formattable, boolean timezoneAware, ValueType targetValueType) {
+
+
+        objectParser.declareField(ValuesSourceAggregationBuilder::field, XContentParser::text,
+                new ParseField("field"), ObjectParser.ValueType.STRING);
+
+        objectParser.declareField(ValuesSourceAggregationBuilder::missing, XContentParser::objectText,
+                new ParseField("missing"), ObjectParser.ValueType.VALUE);
+
+        objectParser.declareField(ValuesSourceAggregationBuilder::valueType, p -> {
+            ValueType valueType = ValueType.resolveForScript(p.text());
+            if (targetValueType != null && valueType.isNotA(targetValueType)) {
+                throw new ParsingException(p.getTokenLocation(),
+                        "Aggregation [" + objectParser.getName() + "] was configured with an incompatible value type ["
+                                + valueType + "]. It can only work on value of type ["
+                                + targetValueType + "]");
+            }
+            return valueType;
+        }, new ParseField("value_type", "valueType"), ObjectParser.ValueType.STRING);
+
+        if (formattable) {
+            objectParser.declareField(ValuesSourceAggregationBuilder::format, XContentParser::text,
+                    new ParseField("format"), ObjectParser.ValueType.STRING);
+        }
+
+        if (scriptable) {
+            objectParser.declareField(ValuesSourceAggregationBuilder::script,
+                    (parser, context) -> Script.parse(parser, context.getDefaultScriptLanguage()),
+                    Script.SCRIPT_PARSE_FIELD, ObjectParser.ValueType.OBJECT_OR_STRING);
+        }
+
+        if (timezoneAware) {
+            objectParser.declareField(ValuesSourceAggregationBuilder::timeZone, p -> {
+                if (p.currentToken() == XContentParser.Token.VALUE_STRING) {
+                    return DateTimeZone.forID(p.text());
+                } else {
+                    return DateTimeZone.forOffsetHours(p.intValue());
+                }
+            }, TIME_ZONE, ObjectParser.ValueType.LONG);
+        }
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/XContentParseContext.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/XContentParseContext.java
deleted file mode 100644
index 07c33f1f47384..0000000000000
--- a/core/src/main/java/org/elasticsearch/search/aggregations/support/XContentParseContext.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.aggregations.support;
-
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.xcontent.XContentParser;
-
-/**
- * A minimal context for parsing xcontent into aggregation builders.
- * Only a minimal set of dependencies and settings are available.
- */
-public final class XContentParseContext {
-
-    private final XContentParser parser;
-
-    private final ParseFieldMatcher parseFieldMatcher;
-
-    private final String defaultScriptLanguage;
-
-    public XContentParseContext(XContentParser parser, ParseFieldMatcher parseFieldMatcher, String defaultScriptLanguage) {
-        this.parser = parser;
-        this.parseFieldMatcher = parseFieldMatcher;
-        this.defaultScriptLanguage = defaultScriptLanguage;
-    }
-
-    public XContentParser getParser() {
-        return parser;
-    }
-
-    public ParseFieldMatcher getParseFieldMatcher() {
-        return parseFieldMatcher;
-    }
-
-    public String getDefaultScriptLanguage() {
-        return defaultScriptLanguage;
-    }
-
-    /**
-     * Returns whether the parse field we're looking for matches with the found field name.
-     *
-     * Helper that delegates to {@link ParseFieldMatcher#match(String, ParseField)}.
-     */
-    public boolean matchField(String fieldName, ParseField parseField) {
-        return parseFieldMatcher.match(fieldName, parseField);
-    }
-
-}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptLongValues.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptLongValues.java
index 06bf998d8dafc..503cec39b8d01 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptLongValues.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptLongValues.java
@@ -50,15 +50,10 @@ public void setDocument(int docId) {
             resize(0);
         }
 
-        else if (value instanceof Number) {
-            resize(1);
-            values[0] = ((Number) value).longValue();
-        }
-
         else if (value.getClass().isArray()) {
             resize(Array.getLength(value));
             for (int i = 0; i < count(); ++i) {
-                values[i] = ((Number) Array.get(value, i)).longValue();
+                values[i] = toLongValue(Array.get(value, i));
             }
         }
 
@@ -66,18 +61,33 @@ else if (value instanceof Collection) {
             resize(((Collection<?>) value).size());
             int i = 0;
             for (Iterator<?> it = ((Collection<?>) value).iterator(); it.hasNext(); ++i) {
-                values[i] = ((Number) it.next()).longValue();
+                values[i] = toLongValue(it.next());
             }
             assert i == count();
         }
 
         else {
-            throw new AggregationExecutionException("Unsupported script value [" + value + "]");
+            resize(1);
+            values[0] = toLongValue(value);
         }
 
         sort();
     }
 
+    private static long toLongValue(Object o) {
+        if (o instanceof Number) {
+            return ((Number) o).longValue();
+        } else if (o instanceof Boolean) {
+            // We do expose boolean fields as boolean in scripts, however aggregations still expect
+            // that scripts return the same internal representation as regular fields, so boolean
+            // values in scripts need to be converted to a number, and the value formatter will
+            // make sure of using true/false in the key_as_string field
+            return ((Boolean) o).booleanValue() ? 1L : 0L;
+        } else {
+            throw new AggregationExecutionException("Unsupported script value [" + o + "], expected a number");
+        }
+    }
+
     @Override
     public void setScorer(Scorer scorer) {
         script.setScorer(scorer);
diff --git a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
index 309e49448e9da..de476a103dc6c 100644
--- a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
@@ -19,31 +19,31 @@
 
 package org.elasticsearch.search.builder;
 
-import com.carrotsearch.hppc.ObjectFloatHashMap;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.support.ToXContentToBytes;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.script.Script;
+import org.elasticsearch.search.SearchExtBuilder;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
-import org.elasticsearch.search.aggregations.AggregatorParsers;
 import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
+import org.elasticsearch.search.collapse.CollapseBuilder;
 import org.elasticsearch.search.fetch.StoredFieldsContext;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
@@ -56,17 +56,12 @@
 import org.elasticsearch.search.sort.SortBuilders;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.search.suggest.SuggestBuilder;
-import org.elasticsearch.search.suggest.Suggesters;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
-import java.util.stream.Collectors;
-import java.util.stream.StreamSupport;
-
-import static org.elasticsearch.common.collect.Tuple.tuple;
 
 /**
  * A search source builder allowing to easily build search source. Simple
@@ -75,7 +70,9 @@
  *
  * @see org.elasticsearch.action.search.SearchRequest#source(SearchSourceBuilder)
  */
-public final class SearchSourceBuilder extends ToXContentToBytes implements Writeable {
+public final class SearchSourceBuilder extends ToXContentToBytes implements Writeable, ToXContentObject {
+    private static final DeprecationLogger DEPRECATION_LOGGER =
+        new DeprecationLogger(Loggers.getLogger(SearchSourceBuilder.class));
 
     public static final ParseField FROM_FIELD = new ParseField("from");
     public static final ParseField SIZE_FIELD = new ParseField("size");
@@ -105,12 +102,13 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
     public static final ParseField EXT_FIELD = new ParseField("ext");
     public static final ParseField PROFILE_FIELD = new ParseField("profile");
     public static final ParseField SEARCH_AFTER = new ParseField("search_after");
+    public static final ParseField COLLAPSE = new ParseField("collapse");
     public static final ParseField SLICE = new ParseField("slice");
+    public static final ParseField ALL_FIELDS_FIELDS = new ParseField("all_fields");
 
-    public static SearchSourceBuilder fromXContent(QueryParseContext context, AggregatorParsers aggParsers,
-            Suggesters suggesters) throws IOException {
+    public static SearchSourceBuilder fromXContent(QueryParseContext context) throws IOException {
         SearchSourceBuilder builder = new SearchSourceBuilder();
-        builder.parseXContent(context, aggParsers, suggesters);
+        builder.parseXContent(context);
         return builder;
     }
 
@@ -164,16 +162,17 @@ public static HighlightBuilder highlight() {
 
     private SuggestBuilder suggestBuilder;
 
-    private List<RescoreBuilder<?>> rescoreBuilders;
+    private List<RescoreBuilder> rescoreBuilders;
 
-    private ObjectFloatHashMap<String> indexBoost = null;
+    private List<IndexBoost> indexBoosts = new ArrayList<>();
 
     private List<String> stats;
 
-    private BytesReference ext = null;
+    private List<SearchExtBuilder> extBuilders = Collections.emptyList();
 
     private boolean profile = false;
 
+    private CollapseBuilder collapse = null;
 
     /**
      * Constructs a new search source builder.
@@ -187,34 +186,20 @@ public SearchSourceBuilder() {
     public SearchSourceBuilder(StreamInput in) throws IOException {
         aggregations = in.readOptionalWriteable(AggregatorFactories.Builder::new);
         explain = in.readOptionalBoolean();
-        fetchSourceContext = in.readOptionalStreamable(FetchSourceContext::new);
+        fetchSourceContext = in.readOptionalWriteable(FetchSourceContext::new);
         docValueFields = (List<String>) in.readGenericValue();
         storedFieldsContext = in.readOptionalWriteable(StoredFieldsContext::new);
         from = in.readVInt();
         highlightBuilder = in.readOptionalWriteable(HighlightBuilder::new);
-        int indexBoostSize = in.readVInt();
-        if (indexBoostSize > 0) {
-            indexBoost = new ObjectFloatHashMap<>(indexBoostSize);
-            for (int i = 0; i < indexBoostSize; i++) {
-                indexBoost.put(in.readString(), in.readFloat());
-            }
-        }
+        indexBoosts = in.readList(IndexBoost::new);
         minScore = in.readOptionalFloat();
         postQueryBuilder = in.readOptionalNamedWriteable(QueryBuilder.class);
         queryBuilder = in.readOptionalNamedWriteable(QueryBuilder.class);
         if (in.readBoolean()) {
-            int size = in.readVInt();
-            rescoreBuilders = new ArrayList<>();
-            for (int i = 0; i < size; i++) {
-                rescoreBuilders.add(in.readNamedWriteable(RescoreBuilder.class));
-            }
+            rescoreBuilders = in.readNamedWriteableList(RescoreBuilder.class);
         }
         if (in.readBoolean()) {
-            int size = in.readVInt();
-            scriptFields = new ArrayList<>(size);
-            for (int i = 0; i < size; i++) {
-                scriptFields.add(new ScriptField(in));
-            }
+            scriptFields = in.readList(ScriptField::new);
         }
         size = in.readVInt();
         if (in.readBoolean()) {
@@ -225,55 +210,44 @@ public SearchSourceBuilder(StreamInput in) throws IOException {
             }
         }
         if (in.readBoolean()) {
-            int size = in.readVInt();
-            stats = new ArrayList<>();
-            for (int i = 0; i < size; i++) {
-                stats.add(in.readString());
-            }
+            stats = in.readList(StreamInput::readString);
         }
         suggestBuilder = in.readOptionalWriteable(SuggestBuilder::new);
         terminateAfter = in.readVInt();
         timeout = in.readOptionalWriteable(TimeValue::new);
         trackScores = in.readBoolean();
         version = in.readOptionalBoolean();
-        ext = in.readOptionalBytesReference();
+        extBuilders = in.readNamedWriteableList(SearchExtBuilder.class);
         profile = in.readBoolean();
         searchAfterBuilder = in.readOptionalWriteable(SearchAfterBuilder::new);
         sliceBuilder = in.readOptionalWriteable(SliceBuilder::new);
+        if (in.getVersion().onOrAfter(Version.V_5_3_0)) {
+            collapse = in.readOptionalWriteable(CollapseBuilder::new);
+        }
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalWriteable(aggregations);
         out.writeOptionalBoolean(explain);
-        out.writeOptionalStreamable(fetchSourceContext);
+        out.writeOptionalWriteable(fetchSourceContext);
         out.writeGenericValue(docValueFields);
         out.writeOptionalWriteable(storedFieldsContext);
         out.writeVInt(from);
         out.writeOptionalWriteable(highlightBuilder);
-        int indexBoostSize = indexBoost == null ? 0 : indexBoost.size();
-        out.writeVInt(indexBoostSize);
-        if (indexBoostSize > 0) {
-            writeIndexBoost(out);
-        }
+        out.writeList(indexBoosts);
         out.writeOptionalFloat(minScore);
         out.writeOptionalNamedWriteable(postQueryBuilder);
         out.writeOptionalNamedWriteable(queryBuilder);
         boolean hasRescoreBuilders = rescoreBuilders != null;
         out.writeBoolean(hasRescoreBuilders);
         if (hasRescoreBuilders) {
-            out.writeVInt(rescoreBuilders.size());
-            for (RescoreBuilder<?> rescoreBuilder : rescoreBuilders) {
-                out.writeNamedWriteable(rescoreBuilder);
-            }
+            out.writeNamedWriteableList(rescoreBuilders);
         }
         boolean hasScriptFields = scriptFields != null;
         out.writeBoolean(hasScriptFields);
         if (hasScriptFields) {
-            out.writeVInt(scriptFields.size());
-            for (ScriptField scriptField : scriptFields) {
-                scriptField.writeTo(out);
-            }
+            out.writeList(scriptFields);
         }
         out.writeVInt(size);
         boolean hasSorts = sorts != null;
@@ -287,30 +261,19 @@ public void writeTo(StreamOutput out) throws IOException {
         boolean hasStats = stats != null;
         out.writeBoolean(hasStats);
         if (hasStats) {
-            out.writeVInt(stats.size());
-            for (String stat : stats) {
-                out.writeString(stat);
-            }
+            out.writeStringList(stats);
         }
         out.writeOptionalWriteable(suggestBuilder);
         out.writeVInt(terminateAfter);
         out.writeOptionalWriteable(timeout);
         out.writeBoolean(trackScores);
         out.writeOptionalBoolean(version);
-        out.writeOptionalBytesReference(ext);
+        out.writeNamedWriteableList(extBuilders);
         out.writeBoolean(profile);
         out.writeOptionalWriteable(searchAfterBuilder);
         out.writeOptionalWriteable(sliceBuilder);
-    }
-
-    private void writeIndexBoost(StreamOutput out) throws IOException {
-        List<Tuple<String, Float>> ibs = StreamSupport
-            .stream(indexBoost.spliterator(), false)
-            .map(i -> tuple(i.key, i.value)).sorted((o1, o2) -> o1.v1().compareTo(o2.v1()))
-            .collect(Collectors.toList());
-        for (Tuple<String, Float> ib : ibs) {
-            out.writeString(ib.v1());
-            out.writeFloat(ib.v2());
+        if (out.getVersion().onOrAfter(Version.V_5_3_0)) {
+            out.writeOptionalWriteable(collapse);
         }
     }
 
@@ -352,6 +315,9 @@ public QueryBuilder postFilter() {
      * From index to start the search from. Defaults to <tt>0</tt>.
      */
     public SearchSourceBuilder from(int from) {
+        if (from < 0) {
+            throw new IllegalArgumentException("[from] parameter cannot be negative");
+        }
         this.from = from;
         return this;
     }
@@ -561,6 +527,16 @@ public SliceBuilder slice() {
         return sliceBuilder;
     }
 
+
+    public CollapseBuilder collapse() {
+        return collapse;
+    }
+
+    public SearchSourceBuilder collapse(CollapseBuilder collapse) {
+        this.collapse = collapse;
+        return this;
+    }
+
     /**
      * Add an aggregation to perform as part of the search.
      */
@@ -649,7 +625,7 @@ public boolean profile() {
     /**
      * Gets the bytes representing the rescore builders for this request.
      */
-    public List<RescoreBuilder<?>> rescores() {
+    public List<RescoreBuilder> rescores() {
         return rescoreBuilders;
     }
 
@@ -658,11 +634,9 @@ public List<RescoreBuilder<?>> rescores() {
      * every hit
      */
     public SearchSourceBuilder fetchSource(boolean fetch) {
-        if (this.fetchSourceContext == null) {
-            this.fetchSourceContext = new FetchSourceContext(fetch);
-        } else {
-            this.fetchSourceContext.fetchSource(fetch);
-        }
+        FetchSourceContext fetchSourceContext = this.fetchSourceContext != null ? this.fetchSourceContext
+            : FetchSourceContext.FETCH_SOURCE;
+        this.fetchSourceContext = new FetchSourceContext(fetch, fetchSourceContext.includes(), fetchSourceContext.excludes());
         return this;
     }
 
@@ -696,7 +670,9 @@ public SearchSourceBuilder fetchSource(@Nullable String include, @Nullable Strin
      *            filter the returned _source
      */
     public SearchSourceBuilder fetchSource(@Nullable String[] includes, @Nullable String[] excludes) {
-        fetchSourceContext = new FetchSourceContext(includes, excludes);
+        FetchSourceContext fetchSourceContext = this.fetchSourceContext != null ? this.fetchSourceContext
+            : FetchSourceContext.FETCH_SOURCE;
+        this.fetchSourceContext = new FetchSourceContext(fetchSourceContext.fetchSource(), includes, excludes);
         return this;
     }
 
@@ -836,28 +812,26 @@ public List<ScriptField> scriptFields() {
     }
 
     /**
-     * Sets the boost a specific index will receive when the query is executed
+     * Sets the boost a specific index or alias will receive when the query is executed
      * against it.
      *
      * @param index
-     *            The index to apply the boost against
+     *            The index or alias to apply the boost against
      * @param indexBoost
      *            The boost to apply to the index
      */
     public SearchSourceBuilder indexBoost(String index, float indexBoost) {
-        if (this.indexBoost == null) {
-            this.indexBoost = new ObjectFloatHashMap<>();
-        }
-        this.indexBoost.put(index, indexBoost);
+        Objects.requireNonNull(index, "index must not be null");
+        this.indexBoosts.add(new IndexBoost(index, indexBoost));
         return this;
     }
 
     /**
-     * Gets the boost a specific indices will receive when the query is
+     * Gets the boost a specific indices or aliases will receive when the query is
      * executed against them.
      */
-    public ObjectFloatHashMap<String> indexBoost() {
-        return indexBoost;
+    public List<IndexBoost> indexBoosts() {
+        return indexBoosts;
     }
 
     /**
@@ -875,13 +849,13 @@ public List<String> stats() {
         return stats;
     }
 
-    public SearchSourceBuilder ext(XContentBuilder ext) {
-        this.ext = ext.bytes();
+    public SearchSourceBuilder ext(List<SearchExtBuilder> searchExtBuilders) {
+        this.extBuilders = Objects.requireNonNull(searchExtBuilders, "searchExtBuilders must not be null");
         return this;
     }
 
-    public BytesReference ext() {
-        return ext;
+    public List<SearchExtBuilder> ext() {
+        return extBuilders;
     }
 
     /**
@@ -899,7 +873,7 @@ public boolean isSuggestOnly() {
      * infinitely.
      */
     public SearchSourceBuilder rewrite(QueryShardContext context) throws IOException {
-        assert (this.equals(shallowCopy(queryBuilder, postQueryBuilder)));
+        assert (this.equals(shallowCopy(queryBuilder, postQueryBuilder, sliceBuilder)));
         QueryBuilder queryBuilder = null;
         if (this.queryBuilder != null) {
             queryBuilder = this.queryBuilder.rewrite(context);
@@ -910,86 +884,96 @@ public SearchSourceBuilder rewrite(QueryShardContext context) throws IOException
         }
         boolean rewritten = queryBuilder != this.queryBuilder || postQueryBuilder != this.postQueryBuilder;
         if (rewritten) {
-            return shallowCopy(queryBuilder, postQueryBuilder);
+            return shallowCopy(queryBuilder, postQueryBuilder, sliceBuilder);
         }
         return this;
     }
 
-    private SearchSourceBuilder shallowCopy(QueryBuilder queryBuilder, QueryBuilder postQueryBuilder) {
-            SearchSourceBuilder rewrittenBuilder = new SearchSourceBuilder();
-            rewrittenBuilder.aggregations = aggregations;
-            rewrittenBuilder.explain = explain;
-            rewrittenBuilder.ext = ext;
-            rewrittenBuilder.fetchSourceContext = fetchSourceContext;
-            rewrittenBuilder.docValueFields = docValueFields;
-            rewrittenBuilder.storedFieldsContext = storedFieldsContext;
-            rewrittenBuilder.from = from;
-            rewrittenBuilder.highlightBuilder = highlightBuilder;
-            rewrittenBuilder.indexBoost = indexBoost;
-            rewrittenBuilder.minScore = minScore;
-            rewrittenBuilder.postQueryBuilder = postQueryBuilder;
-            rewrittenBuilder.profile = profile;
-            rewrittenBuilder.queryBuilder = queryBuilder;
-            rewrittenBuilder.rescoreBuilders = rescoreBuilders;
-            rewrittenBuilder.scriptFields = scriptFields;
-            rewrittenBuilder.searchAfterBuilder = searchAfterBuilder;
-            rewrittenBuilder.sliceBuilder = sliceBuilder;
-            rewrittenBuilder.size = size;
-            rewrittenBuilder.sorts = sorts;
-            rewrittenBuilder.stats = stats;
-            rewrittenBuilder.suggestBuilder = suggestBuilder;
-            rewrittenBuilder.terminateAfter = terminateAfter;
-            rewrittenBuilder.timeout = timeout;
-            rewrittenBuilder.trackScores = trackScores;
-            rewrittenBuilder.version = version;
-            return rewrittenBuilder;
-        }
+    /**
+     * Create a shallow copy of this builder with a new slice configuration.
+     */
+    public SearchSourceBuilder copyWithNewSlice(SliceBuilder slice) {
+        return shallowCopy(queryBuilder, postQueryBuilder, slice);
+    }
+
+    /**
+     * Create a shallow copy of this source replaced {@link #queryBuilder}, {@link #postQueryBuilder}, and {@link #sliceBuilder}. Used by
+     * {@link #rewrite(QueryShardContext)} and {@link #copyWithNewSlice(SliceBuilder)}.
+     */
+    private SearchSourceBuilder shallowCopy(QueryBuilder queryBuilder, QueryBuilder postQueryBuilder, SliceBuilder slice) {
+        SearchSourceBuilder rewrittenBuilder = new SearchSourceBuilder();
+        rewrittenBuilder.aggregations = aggregations;
+        rewrittenBuilder.explain = explain;
+        rewrittenBuilder.extBuilders = extBuilders;
+        rewrittenBuilder.fetchSourceContext = fetchSourceContext;
+        rewrittenBuilder.docValueFields = docValueFields;
+        rewrittenBuilder.storedFieldsContext = storedFieldsContext;
+        rewrittenBuilder.from = from;
+        rewrittenBuilder.highlightBuilder = highlightBuilder;
+        rewrittenBuilder.indexBoosts = indexBoosts;
+        rewrittenBuilder.minScore = minScore;
+        rewrittenBuilder.postQueryBuilder = postQueryBuilder;
+        rewrittenBuilder.profile = profile;
+        rewrittenBuilder.queryBuilder = queryBuilder;
+        rewrittenBuilder.rescoreBuilders = rescoreBuilders;
+        rewrittenBuilder.scriptFields = scriptFields;
+        rewrittenBuilder.searchAfterBuilder = searchAfterBuilder;
+        rewrittenBuilder.sliceBuilder = slice;
+        rewrittenBuilder.size = size;
+        rewrittenBuilder.sorts = sorts;
+        rewrittenBuilder.stats = stats;
+        rewrittenBuilder.suggestBuilder = suggestBuilder;
+        rewrittenBuilder.terminateAfter = terminateAfter;
+        rewrittenBuilder.timeout = timeout;
+        rewrittenBuilder.trackScores = trackScores;
+        rewrittenBuilder.version = version;
+        rewrittenBuilder.collapse = collapse;
+        return rewrittenBuilder;
+    }
 
     /**
      * Parse some xContent into this SearchSourceBuilder, overwriting any values specified in the xContent. Use this if you need to set up
      * different defaults than a regular SearchSourceBuilder would have and use
-     * {@link #fromXContent(QueryParseContext, AggregatorParsers, Suggesters)} if you have normal defaults.
+     * {@link #fromXContent(QueryParseContext)} if you have normal defaults.
      */
-    public void parseXContent(QueryParseContext context, AggregatorParsers aggParsers, Suggesters suggesters)
-        throws IOException {
-
+    public void parseXContent(QueryParseContext context) throws IOException {
         XContentParser parser = context.parser();
         XContentParser.Token token = parser.currentToken();
         String currentFieldName = null;
         if (token != XContentParser.Token.START_OBJECT && (token = parser.nextToken()) != XContentParser.Token.START_OBJECT) {
-            throw new ParsingException(parser.getTokenLocation(), "Expected [" + XContentParser.Token.START_OBJECT + "] but found [" + token + "]",
-                    parser.getTokenLocation());
+            throw new ParsingException(parser.getTokenLocation(), "Expected [" + XContentParser.Token.START_OBJECT +
+                    "] but found [" + token + "]", parser.getTokenLocation());
         }
         while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token.isValue()) {
-                if (context.getParseFieldMatcher().match(currentFieldName, FROM_FIELD)) {
+                if (FROM_FIELD.match(currentFieldName)) {
                     from = parser.intValue();
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SIZE_FIELD)) {
+                } else if (SIZE_FIELD.match(currentFieldName)) {
                     size = parser.intValue();
-                } else if (context.getParseFieldMatcher().match(currentFieldName, TIMEOUT_FIELD)) {
+                } else if (TIMEOUT_FIELD.match(currentFieldName)) {
                     timeout = TimeValue.parseTimeValue(parser.text(), null, TIMEOUT_FIELD.getPreferredName());
-                } else if (context.getParseFieldMatcher().match(currentFieldName, TERMINATE_AFTER_FIELD)) {
+                } else if (TERMINATE_AFTER_FIELD.match(currentFieldName)) {
                     terminateAfter = parser.intValue();
-                } else if (context.getParseFieldMatcher().match(currentFieldName, MIN_SCORE_FIELD)) {
+                } else if (MIN_SCORE_FIELD.match(currentFieldName)) {
                     minScore = parser.floatValue();
-                } else if (context.getParseFieldMatcher().match(currentFieldName, VERSION_FIELD)) {
+                } else if (VERSION_FIELD.match(currentFieldName)) {
                     version = parser.booleanValue();
-                } else if (context.getParseFieldMatcher().match(currentFieldName, EXPLAIN_FIELD)) {
+                } else if (EXPLAIN_FIELD.match(currentFieldName)) {
                     explain = parser.booleanValue();
-                } else if (context.getParseFieldMatcher().match(currentFieldName, TRACK_SCORES_FIELD)) {
+                } else if (TRACK_SCORES_FIELD.match(currentFieldName)) {
                     trackScores = parser.booleanValue();
-                } else if (context.getParseFieldMatcher().match(currentFieldName, _SOURCE_FIELD)) {
-                    fetchSourceContext = FetchSourceContext.parse(context);
-                } else if (context.getParseFieldMatcher().match(currentFieldName, STORED_FIELDS_FIELD)) {
+                } else if (_SOURCE_FIELD.match(currentFieldName)) {
+                    fetchSourceContext = FetchSourceContext.fromXContent(context.parser());
+                } else if (STORED_FIELDS_FIELD.match(currentFieldName)) {
                     storedFieldsContext =
                         StoredFieldsContext.fromXContent(SearchSourceBuilder.STORED_FIELDS_FIELD.getPreferredName(), context);
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SORT_FIELD)) {
+                } else if (SORT_FIELD.match(currentFieldName)) {
                     sort(parser.text());
-                } else if (context.getParseFieldMatcher().match(currentFieldName, PROFILE_FIELD)) {
+                } else if (PROFILE_FIELD.match(currentFieldName)) {
                     profile = parser.booleanValue();
-                } else if (context.getParseFieldMatcher().match(currentFieldName, FIELDS_FIELD)) {
+                } else if (FIELDS_FIELD.match(currentFieldName)) {
                     throw new ParsingException(parser.getTokenLocation(), "Deprecated field [" +
                         SearchSourceBuilder.FIELDS_FIELD + "] used, expected [" +
                         SearchSourceBuilder.STORED_FIELDS_FIELD + "] instead");
@@ -998,85 +982,105 @@ public void parseXContent(QueryParseContext context, AggregatorParsers aggParser
                             parser.getTokenLocation());
                 }
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (context.getParseFieldMatcher().match(currentFieldName, QUERY_FIELD)) {
+                if (QUERY_FIELD.match(currentFieldName)) {
                     queryBuilder = context.parseInnerQueryBuilder().orElse(null);
-                } else if (context.getParseFieldMatcher().match(currentFieldName, POST_FILTER_FIELD)) {
+                } else if (POST_FILTER_FIELD.match(currentFieldName)) {
                     postQueryBuilder = context.parseInnerQueryBuilder().orElse(null);
-                } else if (context.getParseFieldMatcher().match(currentFieldName, _SOURCE_FIELD)) {
-                    fetchSourceContext = FetchSourceContext.parse(context);
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SCRIPT_FIELDS_FIELD)) {
+                } else if (_SOURCE_FIELD.match(currentFieldName)) {
+                    fetchSourceContext = FetchSourceContext.fromXContent(context.parser());
+                } else if (SCRIPT_FIELDS_FIELD.match(currentFieldName)) {
                     scriptFields = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
                         scriptFields.add(new ScriptField(context));
                     }
-                } else if (context.getParseFieldMatcher().match(currentFieldName, INDICES_BOOST_FIELD)) {
-                    indexBoost = new ObjectFloatHashMap<>();
+                } else if (INDICES_BOOST_FIELD.match(currentFieldName)) {
+                    DEPRECATION_LOGGER.deprecated(
+                        "Object format in indices_boost is deprecated, please use array format instead");
                     while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
                         if (token == XContentParser.Token.FIELD_NAME) {
                             currentFieldName = parser.currentName();
                         } else if (token.isValue()) {
-                            indexBoost.put(currentFieldName, parser.floatValue());
+                            indexBoosts.add(new IndexBoost(currentFieldName, parser.floatValue()));
                         } else {
-                            throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName + "].",
-                                    parser.getTokenLocation());
+                            throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token +
+                                " in [" + currentFieldName + "].", parser.getTokenLocation());
                         }
                     }
-                } else if (context.getParseFieldMatcher().match(currentFieldName, AGGREGATIONS_FIELD)
-                        || context.getParseFieldMatcher().match(currentFieldName, AGGS_FIELD)) {
-                    aggregations = aggParsers.parseAggregators(context);
-                } else if (context.getParseFieldMatcher().match(currentFieldName, HIGHLIGHT_FIELD)) {
+                } else if (AGGREGATIONS_FIELD.match(currentFieldName)
+                        || AGGS_FIELD.match(currentFieldName)) {
+                    aggregations = AggregatorFactories.parseAggregators(context);
+                } else if (HIGHLIGHT_FIELD.match(currentFieldName)) {
                     highlightBuilder = HighlightBuilder.fromXContent(context);
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SUGGEST_FIELD)) {
-                    suggestBuilder = SuggestBuilder.fromXContent(context, suggesters);
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SORT_FIELD)) {
+                } else if (SUGGEST_FIELD.match(currentFieldName)) {
+                    suggestBuilder = SuggestBuilder.fromXContent(context.parser());
+                } else if (SORT_FIELD.match(currentFieldName)) {
                     sorts = new ArrayList<>(SortBuilder.fromXContent(context));
-                } else if (context.getParseFieldMatcher().match(currentFieldName, RESCORE_FIELD)) {
+                } else if (RESCORE_FIELD.match(currentFieldName)) {
                     rescoreBuilders = new ArrayList<>();
                     rescoreBuilders.add(RescoreBuilder.parseFromXContent(context));
-                } else if (context.getParseFieldMatcher().match(currentFieldName, EXT_FIELD)) {
-                    XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().copyCurrentStructure(parser);
-                    ext = xContentBuilder.bytes();
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SLICE)) {
+                } else if (EXT_FIELD.match(currentFieldName)) {
+                    extBuilders = new ArrayList<>();
+                    String extSectionName = null;
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                        if (token == XContentParser.Token.FIELD_NAME) {
+                            extSectionName = parser.currentName();
+                        } else {
+                            SearchExtBuilder searchExtBuilder = parser.namedObject(SearchExtBuilder.class, extSectionName, null);
+                            if (searchExtBuilder.getWriteableName().equals(extSectionName) == false) {
+                                throw new IllegalStateException("The parsed [" + searchExtBuilder.getClass().getName() + "] object has a "
+                                        + "different writeable name compared to the name of the section that it was parsed from: found ["
+                                        + searchExtBuilder.getWriteableName() + "] expected [" + extSectionName + "]");
+                            }
+                            extBuilders.add(searchExtBuilder);
+                        }
+                    }
+                } else if (SLICE.match(currentFieldName)) {
                     sliceBuilder = SliceBuilder.fromXContent(context);
+                } else if (COLLAPSE.match(currentFieldName)) {
+                    collapse = CollapseBuilder.fromXContent(context);
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName + "].",
                             parser.getTokenLocation());
                 }
             } else if (token == XContentParser.Token.START_ARRAY) {
-                if (context.getParseFieldMatcher().match(currentFieldName, STORED_FIELDS_FIELD)) {
+                if (STORED_FIELDS_FIELD.match(currentFieldName)) {
                     storedFieldsContext = StoredFieldsContext.fromXContent(STORED_FIELDS_FIELD.getPreferredName(), context);
-                } else if (context.getParseFieldMatcher().match(currentFieldName, DOCVALUE_FIELDS_FIELD)) {
+                } else if (DOCVALUE_FIELDS_FIELD.match(currentFieldName)) {
                     docValueFields = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         if (token == XContentParser.Token.VALUE_STRING) {
                             docValueFields.add(parser.text());
                         } else {
-                            throw new ParsingException(parser.getTokenLocation(), "Expected [" + XContentParser.Token.VALUE_STRING + "] in ["
-                                    + currentFieldName + "] but found [" + token + "]", parser.getTokenLocation());
+                            throw new ParsingException(parser.getTokenLocation(), "Expected [" + XContentParser.Token.VALUE_STRING +
+                                "] in [" + currentFieldName + "] but found [" + token + "]", parser.getTokenLocation());
                         }
                     }
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SORT_FIELD)) {
+                } else if (INDICES_BOOST_FIELD.match(currentFieldName)) {
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                        indexBoosts.add(new IndexBoost(context));
+                    }
+                } else if (SORT_FIELD.match(currentFieldName)) {
                     sorts = new ArrayList<>(SortBuilder.fromXContent(context));
-                } else if (context.getParseFieldMatcher().match(currentFieldName, RESCORE_FIELD)) {
+                } else if (RESCORE_FIELD.match(currentFieldName)) {
                     rescoreBuilders = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         rescoreBuilders.add(RescoreBuilder.parseFromXContent(context));
                     }
-                } else if (context.getParseFieldMatcher().match(currentFieldName, STATS_FIELD)) {
+                } else if (STATS_FIELD.match(currentFieldName)) {
                     stats = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         if (token == XContentParser.Token.VALUE_STRING) {
                             stats.add(parser.text());
                         } else {
-                            throw new ParsingException(parser.getTokenLocation(), "Expected [" + XContentParser.Token.VALUE_STRING + "] in ["
-                                    + currentFieldName + "] but found [" + token + "]", parser.getTokenLocation());
+                            throw new ParsingException(parser.getTokenLocation(), "Expected [" + XContentParser.Token.VALUE_STRING +
+                                    "] in [" + currentFieldName + "] but found [" + token + "]", parser.getTokenLocation());
                         }
                     }
-                } else if (context.getParseFieldMatcher().match(currentFieldName, _SOURCE_FIELD)) {
-                    fetchSourceContext = FetchSourceContext.parse(context);
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SEARCH_AFTER)) {
-                    searchAfterBuilder = SearchAfterBuilder.fromXContent(parser, context.getParseFieldMatcher());
-                } else if (context.getParseFieldMatcher().match(currentFieldName, FIELDS_FIELD)) {
+                } else if (_SOURCE_FIELD.match(currentFieldName)) {
+                    fetchSourceContext = FetchSourceContext.fromXContent(context.parser());
+                } else if (SEARCH_AFTER.match(currentFieldName)) {
+                    searchAfterBuilder = SearchAfterBuilder.fromXContent(parser);
+                } else if (FIELDS_FIELD.match(currentFieldName)) {
                     throw new ParsingException(parser.getTokenLocation(), "The field [" +
                         SearchSourceBuilder.FIELDS_FIELD + "] is no longer supported, please use [" +
                         SearchSourceBuilder.STORED_FIELDS_FIELD + "] to retrieve stored fields or _source filtering " +
@@ -1095,12 +1099,6 @@ public void parseXContent(QueryParseContext context, AggregatorParsers aggParser
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
-        innerToXContent(builder, params);
-        builder.endObject();
-        return builder;
-    }
-
-    public void innerToXContent(XContentBuilder builder, Params params) throws IOException {
         if (from != -1) {
             builder.field(FROM_FIELD.getPreferredName(), from);
         }
@@ -1177,29 +1175,26 @@ public void innerToXContent(XContentBuilder builder, Params params) throws IOExc
         }
 
         if (searchAfterBuilder != null) {
-            builder.field(SEARCH_AFTER.getPreferredName(), searchAfterBuilder.getSortValues());
+            builder.array(SEARCH_AFTER.getPreferredName(), searchAfterBuilder.getSortValues());
         }
 
         if (sliceBuilder != null) {
             builder.field(SLICE.getPreferredName(), sliceBuilder);
         }
 
-        if (indexBoost != null) {
-            builder.startObject(INDICES_BOOST_FIELD.getPreferredName());
-            assert !indexBoost.containsKey(null);
-            final Object[] keys = indexBoost.keys;
-            final float[] values = indexBoost.values;
-            for (int i = 0; i < keys.length; i++) {
-                if (keys[i] != null) {
-                    builder.field((String) keys[i], values[i]);
-                }
+        if (!indexBoosts.isEmpty()) {
+            builder.startArray(INDICES_BOOST_FIELD.getPreferredName());
+            for (IndexBoost ib : indexBoosts) {
+                builder.startObject();
+                builder.field(ib.index, ib.boost);
+                builder.endObject();
             }
-            builder.endObject();
+            builder.endArray();
         }
 
         if (aggregations != null) {
             builder.field(AGGREGATIONS_FIELD.getPreferredName(), aggregations);
-            }
+        }
 
         if (highlightBuilder != null) {
             builder.field(HIGHLIGHT_FIELD.getPreferredName(), highlightBuilder);
@@ -1221,15 +1216,106 @@ public void innerToXContent(XContentBuilder builder, Params params) throws IOExc
             builder.field(STATS_FIELD.getPreferredName(), stats);
         }
 
-        if (ext != null) {
-            builder.field(EXT_FIELD.getPreferredName());
-            try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(ext)) {
-                parser.nextToken();
-                builder.copyCurrentStructure(parser);
+        if (extBuilders != null && extBuilders.isEmpty() == false) {
+            builder.startObject(EXT_FIELD.getPreferredName());
+            for (SearchExtBuilder extBuilder : extBuilders) {
+                extBuilder.toXContent(builder, params);
             }
+            builder.endObject();
         }
+
+        if (collapse != null) {
+            builder.field(COLLAPSE.getPreferredName(), collapse);
+        }
+        builder.endObject();
+        return builder;
     }
 
+    public static class IndexBoost implements Writeable, ToXContent {
+        private final String index;
+        private final float boost;
+
+        IndexBoost(String index, float boost) {
+            this.index = index;
+            this.boost = boost;
+        }
+
+        IndexBoost(StreamInput in) throws IOException {
+            index = in.readString();
+            boost = in.readFloat();
+        }
+
+        IndexBoost(QueryParseContext context) throws IOException {
+            XContentParser parser = context.parser();
+            XContentParser.Token token = parser.currentToken();
+
+            if (token == XContentParser.Token.START_OBJECT) {
+                token = parser.nextToken();
+                if (token == XContentParser.Token.FIELD_NAME) {
+                    index = parser.currentName();
+                } else {
+                    throw new ParsingException(parser.getTokenLocation(), "Expected [" + XContentParser.Token.FIELD_NAME +
+                        "] in [" + INDICES_BOOST_FIELD + "] but found [" + token + "]", parser.getTokenLocation());
+                }
+                token = parser.nextToken();
+                if (token == XContentParser.Token.VALUE_NUMBER) {
+                    boost = parser.floatValue();
+                } else {
+                    throw new ParsingException(parser.getTokenLocation(), "Expected [" + XContentParser.Token.VALUE_NUMBER +
+                        "] in [" + INDICES_BOOST_FIELD + "] but found [" + token + "]", parser.getTokenLocation());
+                }
+                token = parser.nextToken();
+                if (token != XContentParser.Token.END_OBJECT) {
+                    throw new ParsingException(parser.getTokenLocation(), "Expected [" + XContentParser.Token.END_OBJECT +
+                        "] in [" + INDICES_BOOST_FIELD + "] but found [" + token + "]", parser.getTokenLocation());
+                }
+            } else {
+                throw new ParsingException(parser.getTokenLocation(), "Expected [" + XContentParser.Token.START_OBJECT +
+                    "] in [" + parser.currentName() + "] but found [" + token + "]", parser.getTokenLocation());
+            }
+        }
+
+        public String getIndex() {
+            return index;
+        }
+
+        public float getBoost() {
+            return boost;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeString(index);
+            out.writeFloat(boost);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            builder.field(index, boost);
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(index, boost);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null) {
+                return false;
+            }
+            if (getClass() != obj.getClass()) {
+                return false;
+            }
+            IndexBoost other = (IndexBoost) obj;
+            return Objects.equals(index, other.index)
+                && Objects.equals(boost, other.boost);
+        }
+
+    }
     public static class ScriptField implements Writeable, ToXContent {
 
         private final boolean ignoreFailure;
@@ -1272,17 +1358,17 @@ public ScriptField(QueryParseContext context) throws IOException {
                     if (token == XContentParser.Token.FIELD_NAME) {
                         currentFieldName = parser.currentName();
                     } else if (token.isValue()) {
-                        if (context.getParseFieldMatcher().match(currentFieldName, SCRIPT_FIELD)) {
-                            script = Script.parse(parser, context.getParseFieldMatcher(), context.getDefaultScriptLanguage());
-                        } else if (context.getParseFieldMatcher().match(currentFieldName, IGNORE_FAILURE_FIELD)) {
+                        if (SCRIPT_FIELD.match(currentFieldName)) {
+                            script = Script.parse(parser, context.getDefaultScriptLanguage());
+                        } else if (IGNORE_FAILURE_FIELD.match(currentFieldName)) {
                             ignoreFailure = parser.booleanValue();
                         } else {
                             throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName
                                     + "].", parser.getTokenLocation());
                         }
                     } else if (token == XContentParser.Token.START_OBJECT) {
-                        if (context.getParseFieldMatcher().match(currentFieldName, SCRIPT_FIELD)) {
-                            script = Script.parse(parser, context.getParseFieldMatcher(), context.getDefaultScriptLanguage());
+                        if (SCRIPT_FIELD.match(currentFieldName)) {
+                            script = Script.parse(parser, context.getDefaultScriptLanguage());
                         } else {
                             throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName
                                     + "].", parser.getTokenLocation());
@@ -1344,9 +1430,10 @@ public boolean equals(Object obj) {
 
     @Override
     public int hashCode() {
-        return Objects.hash(aggregations, explain, fetchSourceContext, docValueFields, storedFieldsContext, from,
-                highlightBuilder, indexBoost, minScore, postQueryBuilder, queryBuilder, rescoreBuilders, scriptFields,
-                size, sorts, searchAfterBuilder, sliceBuilder, stats, suggestBuilder, terminateAfter, timeout, trackScores, version, profile);
+        return Objects.hash(aggregations, explain, fetchSourceContext, docValueFields, storedFieldsContext, from, highlightBuilder,
+                indexBoosts, minScore, postQueryBuilder, queryBuilder, rescoreBuilders, scriptFields, size,
+                sorts, searchAfterBuilder, sliceBuilder, stats, suggestBuilder, terminateAfter, timeout, trackScores, version,
+                profile, extBuilders, collapse);
     }
 
     @Override
@@ -1365,7 +1452,7 @@ public boolean equals(Object obj) {
                 && Objects.equals(storedFieldsContext, other.storedFieldsContext)
                 && Objects.equals(from, other.from)
                 && Objects.equals(highlightBuilder, other.highlightBuilder)
-                && Objects.equals(indexBoost, other.indexBoost)
+                && Objects.equals(indexBoosts, other.indexBoosts)
                 && Objects.equals(minScore, other.minScore)
                 && Objects.equals(postQueryBuilder, other.postQueryBuilder)
                 && Objects.equals(queryBuilder, other.queryBuilder)
@@ -1381,7 +1468,8 @@ public boolean equals(Object obj) {
                 && Objects.equals(timeout, other.timeout)
                 && Objects.equals(trackScores, other.trackScores)
                 && Objects.equals(version, other.version)
-                && Objects.equals(profile, other.profile);
+                && Objects.equals(profile, other.profile)
+                && Objects.equals(extBuilders, other.extBuilders)
+                && Objects.equals(collapse, other.collapse);
     }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/search/collapse/CollapseBuilder.java b/core/src/main/java/org/elasticsearch/search/collapse/CollapseBuilder.java
new file mode 100644
index 0000000000000..93f4b8bf41c9a
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/collapse/CollapseBuilder.java
@@ -0,0 +1,254 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.collapse;
+
+import org.apache.lucene.index.IndexOptions;
+import org.elasticsearch.Version;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.mapper.KeywordFieldMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.NumberFieldMapper;
+import org.elasticsearch.index.query.InnerHitBuilder;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.search.SearchContextException;
+import org.elasticsearch.search.internal.SearchContext;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * A builder that enables field collapsing on search request.
+ */
+public class CollapseBuilder implements Writeable, ToXContentObject {
+    public static final ParseField FIELD_FIELD = new ParseField("field");
+    public static final ParseField INNER_HITS_FIELD = new ParseField("inner_hits");
+    public static final ParseField MAX_CONCURRENT_GROUP_REQUESTS_FIELD = new ParseField("max_concurrent_group_searches");
+    private static final ObjectParser<CollapseBuilder, QueryParseContext> PARSER =
+        new ObjectParser<>("collapse", CollapseBuilder::new);
+
+    static {
+        PARSER.declareString(CollapseBuilder::setField, FIELD_FIELD);
+        PARSER.declareInt(CollapseBuilder::setMaxConcurrentGroupRequests, MAX_CONCURRENT_GROUP_REQUESTS_FIELD);
+        PARSER.declareField((parser, builder, context) -> {
+            XContentParser.Token currentToken = parser.currentToken();
+            if (currentToken == XContentParser.Token.START_OBJECT) {
+                builder.setInnerHits(InnerHitBuilder.fromXContent(context));
+            } else if (currentToken == XContentParser.Token.START_ARRAY) {
+                List<InnerHitBuilder> innerHitBuilders = new ArrayList<>();
+                for (currentToken = parser.nextToken(); currentToken != XContentParser.Token.END_ARRAY; currentToken = parser.nextToken()) {
+                    if (currentToken == XContentParser.Token.START_OBJECT) {
+                        innerHitBuilders.add(InnerHitBuilder.fromXContent(context));
+                    } else {
+                        throw new ParsingException(parser.getTokenLocation(), "Invalid token in inner_hits array");
+                    }
+                }
+
+                builder.setInnerHits(innerHitBuilders);
+            }
+        }, INNER_HITS_FIELD, ObjectParser.ValueType.OBJECT_ARRAY);
+    }
+
+    private String field;
+    private List<InnerHitBuilder> innerHits = Collections.emptyList();
+    private int maxConcurrentGroupRequests = 0;
+
+    private CollapseBuilder() {}
+
+    /**
+     * Public constructor
+     * @param field The name of the field to collapse on
+     */
+    public CollapseBuilder(String field) {
+        Objects.requireNonNull(field, "field must be non-null");
+        this.field = field;
+    }
+
+    public CollapseBuilder(StreamInput in) throws IOException {
+        this.field = in.readString();
+        this.maxConcurrentGroupRequests = in.readVInt();
+        if (in.getVersion().onOrAfter(Version.V_5_5_0)) {
+            this.innerHits = in.readList(InnerHitBuilder::new);
+        } else {
+            InnerHitBuilder innerHitBuilder = in.readOptionalWriteable(InnerHitBuilder::new);
+            if (innerHitBuilder != null) {
+                this.innerHits = Collections.singletonList(innerHitBuilder);
+            } else {
+                this.innerHits = Collections.emptyList();
+            }
+        }
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(field);
+        out.writeVInt(maxConcurrentGroupRequests);
+        if (out.getVersion().onOrAfter(Version.V_5_5_0)) {
+            out.writeList(innerHits);
+        } else {
+            boolean hasInnerHit = innerHits.isEmpty() == false;
+            out.writeBoolean(hasInnerHit);
+            if (hasInnerHit) {
+                innerHits.get(0).writeToCollapseBWC(out);
+            }
+       }
+    }
+
+    public static CollapseBuilder fromXContent(QueryParseContext context) throws IOException {
+        CollapseBuilder builder = PARSER.parse(context.parser(), new CollapseBuilder(), context);
+        return builder;
+    }
+
+    // for object parser only
+    private CollapseBuilder setField(String field) {
+        if (Strings.isEmpty(field)) {
+            throw new IllegalArgumentException("field name is null or empty");
+        }
+        this.field = field;
+        return this;
+    }
+
+    public CollapseBuilder setInnerHits(InnerHitBuilder innerHit) {
+        this.innerHits = Collections.singletonList(innerHit);
+        return this;
+    }
+
+    public CollapseBuilder setInnerHits(List<InnerHitBuilder> innerHits) {
+        this.innerHits = innerHits;
+        return this;
+    }
+
+    public CollapseBuilder setMaxConcurrentGroupRequests(int num) {
+        if (num < 1) {
+            throw new IllegalArgumentException("maxConcurrentGroupRequests` must be positive");
+        }
+        this.maxConcurrentGroupRequests = num;
+        return this;
+    }
+
+    /**
+     * The name of the field to collapse against
+     */
+    public String getField() {
+        return this.field;
+    }
+
+    /**
+     * The inner hit options to expand the collapsed results
+     */
+    public List<InnerHitBuilder> getInnerHits() {
+        return this.innerHits;
+    }
+
+    /**
+     * Returns the amount of group requests that are allowed to be ran concurrently in the inner_hits phase.
+     */
+    public int getMaxConcurrentGroupRequests() {
+        return maxConcurrentGroupRequests;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
+        builder.startObject();
+        innerToXContent(builder);
+        builder.endObject();
+        return builder;
+    }
+
+    private void innerToXContent(XContentBuilder builder) throws IOException {
+        builder.field(FIELD_FIELD.getPreferredName(), field);
+        if (maxConcurrentGroupRequests > 0) {
+            builder.field(MAX_CONCURRENT_GROUP_REQUESTS_FIELD.getPreferredName(), maxConcurrentGroupRequests);
+        }
+        if (innerHits.isEmpty() == false) {
+            if (innerHits.size() == 1) {
+                builder.field(INNER_HITS_FIELD.getPreferredName(), innerHits.get(0));
+            } else {
+                builder.startArray(INNER_HITS_FIELD.getPreferredName());
+                for (InnerHitBuilder innerHit : innerHits) {
+                    innerHit.toXContent(builder, ToXContent.EMPTY_PARAMS);
+                }
+                builder.endArray();
+            }
+        }
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        CollapseBuilder that = (CollapseBuilder) o;
+
+        if (maxConcurrentGroupRequests != that.maxConcurrentGroupRequests) return false;
+        if (!field.equals(that.field)) return false;
+        return Objects.equals(innerHits, that.innerHits);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = Objects.hash(field, innerHits);
+        result = 31 * result + maxConcurrentGroupRequests;
+        return result;
+    }
+
+    public CollapseContext build(SearchContext context) {
+        if (context.scrollContext() != null) {
+            throw new SearchContextException(context, "cannot use `collapse` in a scroll context");
+        }
+        if (context.searchAfter() != null) {
+            throw new SearchContextException(context, "cannot use `collapse` in conjunction with `search_after`");
+        }
+        if (context.rescore() != null && context.rescore().isEmpty() == false) {
+            throw new SearchContextException(context, "cannot use `collapse` in conjunction with `rescore`");
+        }
+
+        MappedFieldType fieldType = context.getQueryShardContext().fieldMapper(field);
+        if (fieldType == null) {
+            throw new SearchContextException(context, "no mapping found for `" + field + "` in order to collapse on");
+        }
+        if (fieldType instanceof KeywordFieldMapper.KeywordFieldType == false &&
+            fieldType instanceof NumberFieldMapper.NumberFieldType == false) {
+            throw new SearchContextException(context, "unknown type for collapse field `" + field +
+                "`, only keywords and numbers are accepted");
+        }
+
+        if (fieldType.hasDocValues() == false) {
+            throw new SearchContextException(context, "cannot collapse on field `" + field + "` without `doc_values`");
+        }
+        if (fieldType.indexOptions() == IndexOptions.NONE && (innerHits != null && !innerHits.isEmpty())) {
+            throw new SearchContextException(context, "cannot expand `inner_hits` for collapse field `"
+                + field + "`, " + "only indexed field can retrieve `inner_hits`");
+        }
+
+        return new CollapseContext(fieldType, innerHits);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/collapse/CollapseContext.java b/core/src/main/java/org/elasticsearch/search/collapse/CollapseContext.java
new file mode 100644
index 0000000000000..cb1587cd7d9e9
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/collapse/CollapseContext.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.collapse;
+
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.grouping.CollapsingTopDocsCollector;
+import org.elasticsearch.index.mapper.KeywordFieldMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.NumberFieldMapper;
+import org.elasticsearch.index.query.InnerHitBuilder;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Context used for field collapsing
+ */
+public class CollapseContext {
+    private final MappedFieldType fieldType;
+    private final List<InnerHitBuilder> innerHits;
+
+    public CollapseContext(MappedFieldType fieldType, InnerHitBuilder innerHit) {
+        this.fieldType = fieldType;
+        this.innerHits = Collections.singletonList(innerHit);
+    }
+
+    public CollapseContext(MappedFieldType fieldType, List<InnerHitBuilder> innerHits) {
+        this.fieldType = fieldType;
+        this.innerHits = innerHits;
+    }
+
+    /** The field type used for collapsing **/
+    public MappedFieldType getFieldType() {
+        return fieldType;
+    }
+
+    /** The inner hit options to expand the collapsed results **/
+    public List<InnerHitBuilder> getInnerHit() {
+        return innerHits;
+    }
+
+    public CollapsingTopDocsCollector<?> createTopDocs(Sort sort, int topN, boolean trackMaxScore) throws IOException {
+        if (fieldType instanceof KeywordFieldMapper.KeywordFieldType) {
+            return CollapsingTopDocsCollector.createKeyword(fieldType.name(), sort, topN, trackMaxScore);
+        } else if (fieldType instanceof NumberFieldMapper.NumberFieldType) {
+            return CollapsingTopDocsCollector.createNumeric(fieldType.name(), sort, topN, trackMaxScore);
+        } else {
+            throw new IllegalStateException("unknown type for collapse field " + fieldType.name() +
+                ", only keywords and numbers are accepted");
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/controller/SearchPhaseController.java b/core/src/main/java/org/elasticsearch/search/controller/SearchPhaseController.java
deleted file mode 100644
index 1766d41dde376..0000000000000
--- a/core/src/main/java/org/elasticsearch/search/controller/SearchPhaseController.java
+++ /dev/null
@@ -1,537 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.controller;
-
-import com.carrotsearch.hppc.IntArrayList;
-import com.carrotsearch.hppc.ObjectObjectHashMap;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.CollectionStatistics;
-import org.apache.lucene.search.FieldDoc;
-import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.Sort;
-import org.apache.lucene.search.SortField;
-import org.apache.lucene.search.TermStatistics;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.search.TopFieldDocs;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.collect.HppcMaps;
-import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.lucene.Lucene;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.BigArrays;
-import org.elasticsearch.common.util.concurrent.AtomicArray;
-import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.search.aggregations.InternalAggregation;
-import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext;
-import org.elasticsearch.search.aggregations.InternalAggregations;
-import org.elasticsearch.search.aggregations.pipeline.SiblingPipelineAggregator;
-import org.elasticsearch.search.dfs.AggregatedDfs;
-import org.elasticsearch.search.dfs.DfsSearchResult;
-import org.elasticsearch.search.fetch.FetchSearchResult;
-import org.elasticsearch.search.fetch.FetchSearchResultProvider;
-import org.elasticsearch.search.internal.InternalSearchHit;
-import org.elasticsearch.search.internal.InternalSearchHits;
-import org.elasticsearch.search.internal.InternalSearchResponse;
-import org.elasticsearch.search.profile.ProfileShardResult;
-import org.elasticsearch.search.profile.SearchProfileShardResults;
-import org.elasticsearch.search.query.QuerySearchResult;
-import org.elasticsearch.search.query.QuerySearchResultProvider;
-import org.elasticsearch.search.suggest.Suggest;
-import org.elasticsearch.search.suggest.Suggest.Suggestion;
-import org.elasticsearch.search.suggest.Suggest.Suggestion.Entry;
-import org.elasticsearch.search.suggest.completion.CompletionSuggestion;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-import java.util.stream.StreamSupport;
-
-/**
- *
- */
-public class SearchPhaseController extends AbstractComponent {
-
-    public static final Comparator<AtomicArray.Entry<? extends QuerySearchResultProvider>> QUERY_RESULT_ORDERING = (o1, o2) -> {
-        int i = o1.value.shardTarget().index().compareTo(o2.value.shardTarget().index());
-        if (i == 0) {
-            i = o1.value.shardTarget().shardId().id() - o2.value.shardTarget().shardId().id();
-        }
-        return i;
-    };
-
-    public static final ScoreDoc[] EMPTY_DOCS = new ScoreDoc[0];
-
-    private final BigArrays bigArrays;
-    private final ScriptService scriptService;
-    private final ClusterService clusterService;
-
-    @Inject
-    public SearchPhaseController(Settings settings, BigArrays bigArrays, ScriptService scriptService, ClusterService clusterService) {
-        super(settings);
-        this.bigArrays = bigArrays;
-        this.scriptService = scriptService;
-        this.clusterService = clusterService;
-    }
-
-    public AggregatedDfs aggregateDfs(AtomicArray<DfsSearchResult> results) {
-        ObjectObjectHashMap<Term, TermStatistics> termStatistics = HppcMaps.newNoNullKeysMap();
-        ObjectObjectHashMap<String, CollectionStatistics> fieldStatistics = HppcMaps.newNoNullKeysMap();
-        long aggMaxDoc = 0;
-        for (AtomicArray.Entry<DfsSearchResult> lEntry : results.asList()) {
-            final Term[] terms = lEntry.value.terms();
-            final TermStatistics[] stats = lEntry.value.termStatistics();
-            assert terms.length == stats.length;
-            for (int i = 0; i < terms.length; i++) {
-                assert terms[i] != null;
-                TermStatistics existing = termStatistics.get(terms[i]);
-                if (existing != null) {
-                    assert terms[i].bytes().equals(existing.term());
-                    // totalTermFrequency is an optional statistic we need to check if either one or both
-                    // are set to -1 which means not present and then set it globally to -1
-                    termStatistics.put(terms[i], new TermStatistics(existing.term(),
-                            existing.docFreq() + stats[i].docFreq(),
-                            optionalSum(existing.totalTermFreq(), stats[i].totalTermFreq())));
-                } else {
-                    termStatistics.put(terms[i], stats[i]);
-                }
-
-            }
-
-            assert !lEntry.value.fieldStatistics().containsKey(null);
-            final Object[] keys = lEntry.value.fieldStatistics().keys;
-            final Object[] values = lEntry.value.fieldStatistics().values;
-            for (int i = 0; i < keys.length; i++) {
-                if (keys[i] != null) {
-                    String key = (String) keys[i];
-                    CollectionStatistics value = (CollectionStatistics) values[i];
-                    assert key != null;
-                    CollectionStatistics existing = fieldStatistics.get(key);
-                    if (existing != null) {
-                        CollectionStatistics merged = new CollectionStatistics(
-                                key, existing.maxDoc() + value.maxDoc(),
-                                optionalSum(existing.docCount(), value.docCount()),
-                                optionalSum(existing.sumTotalTermFreq(), value.sumTotalTermFreq()),
-                                optionalSum(existing.sumDocFreq(), value.sumDocFreq())
-                        );
-                        fieldStatistics.put(key, merged);
-                    } else {
-                        fieldStatistics.put(key, value);
-                    }
-                }
-            }
-            aggMaxDoc += lEntry.value.maxDoc();
-        }
-        return new AggregatedDfs(termStatistics, fieldStatistics, aggMaxDoc);
-    }
-
-    private static long optionalSum(long left, long right) {
-        return Math.min(left, right) == -1 ? -1 : left + right;
-    }
-
-    /**
-     * Returns a score doc array of top N search docs across all shards, followed by top suggest docs for each
-     * named completion suggestion across all shards. If more than one named completion suggestion is specified in the
-     * request, the suggest docs for a named suggestion are ordered by the suggestion name.
-     *
-     * @param ignoreFrom Whether to ignore the from and sort all hits in each shard result.
-     *                   Enabled only for scroll search, because that only retrieves hits of length 'size' in the query phase.
-     * @param resultsArr Shard result holder
-     */
-    public ScoreDoc[] sortDocs(boolean ignoreFrom, AtomicArray<? extends QuerySearchResultProvider> resultsArr) throws IOException {
-        List<? extends AtomicArray.Entry<? extends QuerySearchResultProvider>> results = resultsArr.asList();
-        if (results.isEmpty()) {
-            return EMPTY_DOCS;
-        }
-
-        boolean canOptimize = false;
-        QuerySearchResult result = null;
-        int shardIndex = -1;
-        if (results.size() == 1) {
-            canOptimize = true;
-            result = results.get(0).value.queryResult();
-            shardIndex = results.get(0).index;
-        } else {
-            // lets see if we only got hits from a single shard, if so, we can optimize...
-            for (AtomicArray.Entry<? extends QuerySearchResultProvider> entry : results) {
-                if (entry.value.queryResult().hasHits()) {
-                    if (result != null) { // we already have one, can't really optimize
-                        canOptimize = false;
-                        break;
-                    }
-                    canOptimize = true;
-                    result = entry.value.queryResult();
-                    shardIndex = entry.index;
-                }
-            }
-        }
-        if (canOptimize) {
-            int offset = result.from();
-            if (ignoreFrom) {
-                offset = 0;
-            }
-            ScoreDoc[] scoreDocs = result.topDocs().scoreDocs;
-            ScoreDoc[] docs;
-            int numSuggestDocs = 0;
-            final Suggest suggest = result.queryResult().suggest();
-            final List<CompletionSuggestion> completionSuggestions;
-            if (suggest != null) {
-                completionSuggestions = suggest.filter(CompletionSuggestion.class);
-                for (CompletionSuggestion suggestion : completionSuggestions) {
-                    numSuggestDocs += suggestion.getOptions().size();
-                }
-            } else {
-                completionSuggestions = Collections.emptyList();
-            }
-            int docsOffset = 0;
-            if (scoreDocs.length == 0 || scoreDocs.length < offset) {
-                docs = new ScoreDoc[numSuggestDocs];
-            } else {
-                int resultDocsSize = result.size();
-                if ((scoreDocs.length - offset) < resultDocsSize) {
-                    resultDocsSize = scoreDocs.length - offset;
-                }
-                docs = new ScoreDoc[resultDocsSize + numSuggestDocs];
-                for (int i = 0; i < resultDocsSize; i++) {
-                    ScoreDoc scoreDoc = scoreDocs[offset + i];
-                    scoreDoc.shardIndex = shardIndex;
-                    docs[i] = scoreDoc;
-                    docsOffset++;
-                }
-            }
-            for (CompletionSuggestion suggestion: completionSuggestions) {
-                for (CompletionSuggestion.Entry.Option option : suggestion.getOptions()) {
-                    ScoreDoc doc = option.getDoc();
-                    doc.shardIndex = shardIndex;
-                    docs[docsOffset++] = doc;
-                }
-            }
-            return docs;
-        }
-
-        @SuppressWarnings("unchecked")
-        AtomicArray.Entry<? extends QuerySearchResultProvider>[] sortedResults = results.toArray(new AtomicArray.Entry[results.size()]);
-        Arrays.sort(sortedResults, QUERY_RESULT_ORDERING);
-        QuerySearchResultProvider firstResult = sortedResults[0].value;
-
-        int topN = topN(results);
-        int from = firstResult.queryResult().from();
-        if (ignoreFrom) {
-            from = 0;
-        }
-
-        final TopDocs mergedTopDocs;
-        if (firstResult.queryResult().topDocs() instanceof TopFieldDocs) {
-            TopFieldDocs firstTopDocs = (TopFieldDocs) firstResult.queryResult().topDocs();
-            final Sort sort = new Sort(firstTopDocs.fields);
-
-            final TopFieldDocs[] shardTopDocs = new TopFieldDocs[resultsArr.length()];
-            for (AtomicArray.Entry<? extends QuerySearchResultProvider> sortedResult : sortedResults) {
-                TopDocs topDocs = sortedResult.value.queryResult().topDocs();
-                // the 'index' field is the position in the resultsArr atomic array
-                shardTopDocs[sortedResult.index] = (TopFieldDocs) topDocs;
-            }
-            // TopDocs#merge can't deal with null shard TopDocs
-            for (int i = 0; i < shardTopDocs.length; ++i) {
-                if (shardTopDocs[i] == null) {
-                    shardTopDocs[i] = new TopFieldDocs(0, new FieldDoc[0], sort.getSort(), Float.NaN);
-                }
-            }
-            mergedTopDocs = TopDocs.merge(sort, from, topN, shardTopDocs);
-        } else {
-            final TopDocs[] shardTopDocs = new TopDocs[resultsArr.length()];
-            for (AtomicArray.Entry<? extends QuerySearchResultProvider> sortedResult : sortedResults) {
-                TopDocs topDocs = sortedResult.value.queryResult().topDocs();
-                // the 'index' field is the position in the resultsArr atomic array
-                shardTopDocs[sortedResult.index] = topDocs;
-            }
-            // TopDocs#merge can't deal with null shard TopDocs
-            for (int i = 0; i < shardTopDocs.length; ++i) {
-                if (shardTopDocs[i] == null) {
-                    shardTopDocs[i] = Lucene.EMPTY_TOP_DOCS;
-                }
-            }
-            mergedTopDocs = TopDocs.merge(from, topN, shardTopDocs);
-        }
-
-        ScoreDoc[] scoreDocs = mergedTopDocs.scoreDocs;
-        final Map<String, List<Suggestion<CompletionSuggestion.Entry>>> groupedCompletionSuggestions = new HashMap<>();
-        // group suggestions and assign shard index
-        for (AtomicArray.Entry<? extends QuerySearchResultProvider> sortedResult : sortedResults) {
-            Suggest shardSuggest = sortedResult.value.queryResult().suggest();
-            if (shardSuggest != null) {
-                for (CompletionSuggestion suggestion : shardSuggest.filter(CompletionSuggestion.class)) {
-                    suggestion.setShardIndex(sortedResult.index);
-                    List<Suggestion<CompletionSuggestion.Entry>> suggestions =
-                        groupedCompletionSuggestions.computeIfAbsent(suggestion.getName(), s -> new ArrayList<>());
-                    suggestions.add(suggestion);
-                }
-            }
-        }
-        if (groupedCompletionSuggestions.isEmpty() == false) {
-            int numSuggestDocs = 0;
-            List<Suggestion<? extends Entry<? extends Entry.Option>>> completionSuggestions =
-                new ArrayList<>(groupedCompletionSuggestions.size());
-            for (List<Suggestion<CompletionSuggestion.Entry>> groupedSuggestions : groupedCompletionSuggestions.values()) {
-                final CompletionSuggestion completionSuggestion = CompletionSuggestion.reduceTo(groupedSuggestions);
-                assert completionSuggestion != null;
-                numSuggestDocs += completionSuggestion.getOptions().size();
-                completionSuggestions.add(completionSuggestion);
-            }
-            scoreDocs = new ScoreDoc[mergedTopDocs.scoreDocs.length + numSuggestDocs];
-            System.arraycopy(mergedTopDocs.scoreDocs, 0, scoreDocs, 0, mergedTopDocs.scoreDocs.length);
-            int offset = mergedTopDocs.scoreDocs.length;
-            Suggest suggestions = new Suggest(completionSuggestions);
-            for (CompletionSuggestion completionSuggestion : suggestions.filter(CompletionSuggestion.class)) {
-                for (CompletionSuggestion.Entry.Option option : completionSuggestion.getOptions()) {
-                    scoreDocs[offset++] = option.getDoc();
-                }
-            }
-        }
-        return scoreDocs;
-    }
-
-    public ScoreDoc[] getLastEmittedDocPerShard(List<? extends AtomicArray.Entry<? extends QuerySearchResultProvider>> queryResults,
-                                                ScoreDoc[] sortedScoreDocs, int numShards) {
-        ScoreDoc[] lastEmittedDocPerShard = new ScoreDoc[numShards];
-        if (queryResults.isEmpty() == false) {
-            long fetchHits = 0;
-            for (AtomicArray.Entry<? extends QuerySearchResultProvider> queryResult : queryResults) {
-                fetchHits += queryResult.value.queryResult().topDocs().scoreDocs.length;
-            }
-            // from is always zero as when we use scroll, we ignore from
-            long size = Math.min(fetchHits, topN(queryResults));
-            for (int sortedDocsIndex = 0; sortedDocsIndex < size; sortedDocsIndex++) {
-                ScoreDoc scoreDoc = sortedScoreDocs[sortedDocsIndex];
-                lastEmittedDocPerShard[scoreDoc.shardIndex] = scoreDoc;
-            }
-        }
-        return lastEmittedDocPerShard;
-
-    }
-
-    /**
-     * Builds an array, with potential null elements, with docs to load.
-     */
-    public void fillDocIdsToLoad(AtomicArray<IntArrayList> docIdsToLoad, ScoreDoc[] shardDocs) {
-        for (ScoreDoc shardDoc : shardDocs) {
-            IntArrayList shardDocIdsToLoad = docIdsToLoad.get(shardDoc.shardIndex);
-            if (shardDocIdsToLoad == null) {
-                shardDocIdsToLoad = new IntArrayList(); // can't be shared!, uses unsafe on it later on
-                docIdsToLoad.set(shardDoc.shardIndex, shardDocIdsToLoad);
-            }
-            shardDocIdsToLoad.add(shardDoc.doc);
-        }
-    }
-
-    /**
-     * Enriches search hits and completion suggestion hits from <code>sortedDocs</code> using <code>fetchResultsArr</code>,
-     * merges suggestions, aggregations and profile results
-     *
-     * Expects sortedDocs to have top search docs across all shards, optionally followed by top suggest docs for each named
-     * completion suggestion ordered by suggestion name
-     */
-    public InternalSearchResponse merge(boolean ignoreFrom, ScoreDoc[] sortedDocs,
-                                        AtomicArray<? extends QuerySearchResultProvider> queryResultsArr,
-                                        AtomicArray<? extends FetchSearchResultProvider> fetchResultsArr) {
-
-        List<? extends AtomicArray.Entry<? extends QuerySearchResultProvider>> queryResults = queryResultsArr.asList();
-        List<? extends AtomicArray.Entry<? extends FetchSearchResultProvider>> fetchResults = fetchResultsArr.asList();
-
-        if (queryResults.isEmpty()) {
-            return InternalSearchResponse.empty();
-        }
-
-        QuerySearchResult firstResult = queryResults.get(0).value.queryResult();
-
-        boolean sorted = false;
-        int sortScoreIndex = -1;
-        if (firstResult.topDocs() instanceof TopFieldDocs) {
-            sorted = true;
-            TopFieldDocs fieldDocs = (TopFieldDocs) firstResult.queryResult().topDocs();
-            for (int i = 0; i < fieldDocs.fields.length; i++) {
-                if (fieldDocs.fields[i].getType() == SortField.Type.SCORE) {
-                    sortScoreIndex = i;
-                }
-            }
-        }
-
-        // count the total (we use the query result provider here, since we might not get any hits (we scrolled past them))
-        long totalHits = 0;
-        long fetchHits = 0;
-        float maxScore = Float.NEGATIVE_INFINITY;
-        boolean timedOut = false;
-        Boolean terminatedEarly = null;
-        for (AtomicArray.Entry<? extends QuerySearchResultProvider> entry : queryResults) {
-            QuerySearchResult result = entry.value.queryResult();
-            if (result.searchTimedOut()) {
-                timedOut = true;
-            }
-            if (result.terminatedEarly() != null) {
-                if (terminatedEarly == null) {
-                    terminatedEarly = result.terminatedEarly();
-                } else if (result.terminatedEarly()) {
-                    terminatedEarly = true;
-                }
-            }
-            totalHits += result.topDocs().totalHits;
-            fetchHits += result.topDocs().scoreDocs.length;
-            if (!Float.isNaN(result.topDocs().getMaxScore())) {
-                maxScore = Math.max(maxScore, result.topDocs().getMaxScore());
-            }
-        }
-        if (Float.isInfinite(maxScore)) {
-            maxScore = Float.NaN;
-        }
-
-        // clean the fetch counter
-        for (AtomicArray.Entry<? extends FetchSearchResultProvider> entry : fetchResults) {
-            entry.value.fetchResult().initCounter();
-        }
-        int from = ignoreFrom ? 0 : firstResult.queryResult().from();
-        int numSearchHits = (int) Math.min(fetchHits - from, topN(queryResults));
-        // merge hits
-        List<InternalSearchHit> hits = new ArrayList<>();
-        if (!fetchResults.isEmpty()) {
-            for (int i = 0; i < numSearchHits; i++) {
-                ScoreDoc shardDoc = sortedDocs[i];
-                FetchSearchResultProvider fetchResultProvider = fetchResultsArr.get(shardDoc.shardIndex);
-                if (fetchResultProvider == null) {
-                    continue;
-                }
-                FetchSearchResult fetchResult = fetchResultProvider.fetchResult();
-                int index = fetchResult.counterGetAndIncrement();
-                if (index < fetchResult.hits().internalHits().length) {
-                    InternalSearchHit searchHit = fetchResult.hits().internalHits()[index];
-                    searchHit.score(shardDoc.score);
-                    searchHit.shard(fetchResult.shardTarget());
-                    if (sorted) {
-                        FieldDoc fieldDoc = (FieldDoc) shardDoc;
-                        searchHit.sortValues(fieldDoc.fields, firstResult.sortValueFormats());
-                        if (sortScoreIndex != -1) {
-                            searchHit.score(((Number) fieldDoc.fields[sortScoreIndex]).floatValue());
-                        }
-                    }
-                    hits.add(searchHit);
-                }
-            }
-        }
-
-        // merge suggest results
-        Suggest suggest = null;
-        if (firstResult.suggest() != null) {
-            final Map<String, List<Suggestion>> groupedSuggestions = new HashMap<>();
-            for (AtomicArray.Entry<? extends QuerySearchResultProvider> queryResult : queryResults) {
-                Suggest shardSuggest = queryResult.value.queryResult().suggest();
-                if (shardSuggest != null) {
-                    for (Suggestion<? extends Suggestion.Entry<? extends Suggestion.Entry.Option>> suggestion : shardSuggest) {
-                        List<Suggestion> suggestionList = groupedSuggestions.computeIfAbsent(suggestion.getName(), s -> new ArrayList<>());
-                        suggestionList.add(suggestion);
-                    }
-                }
-            }
-            if (groupedSuggestions.isEmpty() == false) {
-                suggest = new Suggest(Suggest.reduce(groupedSuggestions));
-                if (!fetchResults.isEmpty()) {
-                    int currentOffset = numSearchHits;
-                    for (CompletionSuggestion suggestion : suggest.filter(CompletionSuggestion.class)) {
-                        final List<CompletionSuggestion.Entry.Option> suggestionOptions = suggestion.getOptions();
-                        for (int scoreDocIndex = currentOffset; scoreDocIndex < currentOffset + suggestionOptions.size(); scoreDocIndex++) {
-                            ScoreDoc shardDoc = sortedDocs[scoreDocIndex];
-                            FetchSearchResultProvider fetchSearchResultProvider = fetchResultsArr.get(shardDoc.shardIndex);
-                            if (fetchSearchResultProvider == null) {
-                                continue;
-                            }
-                            FetchSearchResult fetchResult = fetchSearchResultProvider.fetchResult();
-                            int fetchResultIndex = fetchResult.counterGetAndIncrement();
-                            if (fetchResultIndex < fetchResult.hits().internalHits().length) {
-                                InternalSearchHit hit = fetchResult.hits().internalHits()[fetchResultIndex];
-                                CompletionSuggestion.Entry.Option suggestOption =
-                                    suggestionOptions.get(scoreDocIndex - currentOffset);
-                                hit.score(shardDoc.score);
-                                hit.shard(fetchResult.shardTarget());
-                                suggestOption.setHit(hit);
-                            }
-                        }
-                        currentOffset += suggestionOptions.size();
-                    }
-                    assert currentOffset == sortedDocs.length : "expected no more score doc slices";
-                }
-            }
-        }
-
-        // merge Aggregation
-        InternalAggregations aggregations = null;
-        if (firstResult.aggregations() != null && firstResult.aggregations().asList() != null) {
-            List<InternalAggregations> aggregationsList = new ArrayList<>(queryResults.size());
-            for (AtomicArray.Entry<? extends QuerySearchResultProvider> entry : queryResults) {
-                aggregationsList.add((InternalAggregations) entry.value.queryResult().aggregations());
-            }
-            ReduceContext reduceContext = new ReduceContext(bigArrays, scriptService, clusterService.state());
-            aggregations = InternalAggregations.reduce(aggregationsList, reduceContext);
-            List<SiblingPipelineAggregator> pipelineAggregators = firstResult.pipelineAggregators();
-            if (pipelineAggregators != null) {
-                List<InternalAggregation> newAggs = StreamSupport.stream(aggregations.spliterator(), false)
-                    .map((p) -> (InternalAggregation) p)
-                    .collect(Collectors.toList());
-                for (SiblingPipelineAggregator pipelineAggregator : pipelineAggregators) {
-                    InternalAggregation newAgg = pipelineAggregator.doReduce(new InternalAggregations(newAggs), reduceContext);
-                    newAggs.add(newAgg);
-                }
-                aggregations = new InternalAggregations(newAggs);
-            }
-        }
-
-        //Collect profile results
-        SearchProfileShardResults shardResults = null;
-        if (firstResult.profileResults() != null) {
-            Map<String, ProfileShardResult> profileResults = new HashMap<>(queryResults.size());
-            for (AtomicArray.Entry<? extends QuerySearchResultProvider> entry : queryResults) {
-                String key = entry.value.queryResult().shardTarget().toString();
-                profileResults.put(key, entry.value.queryResult().profileResults());
-            }
-            shardResults = new SearchProfileShardResults(profileResults);
-        }
-
-        InternalSearchHits searchHits = new InternalSearchHits(hits.toArray(new InternalSearchHit[hits.size()]), totalHits, maxScore);
-
-        return new InternalSearchResponse(searchHits, aggregations, suggest, shardResults, timedOut, terminatedEarly);
-    }
-
-    /**
-     * returns the number of top results to be considered across all shards
-     */
-    private static int topN(List<? extends AtomicArray.Entry<? extends QuerySearchResultProvider>> queryResults) {
-        QuerySearchResultProvider firstResult = queryResults.get(0).value;
-        int topN = firstResult.queryResult().size();
-        if (firstResult.includeFetch()) {
-            // if we did both query and fetch on the same go, we have fetched all the docs from each shards already, use them...
-            // this is also important since we shortcut and fetch only docs from "from" and up to "size"
-            topN *= queryResults.size();
-        }
-        return topN;
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/dfs/AggregatedDfs.java b/core/src/main/java/org/elasticsearch/search/dfs/AggregatedDfs.java
index d762540caabef..f7bfb6a805251 100644
--- a/core/src/main/java/org/elasticsearch/search/dfs/AggregatedDfs.java
+++ b/core/src/main/java/org/elasticsearch/search/dfs/AggregatedDfs.java
@@ -85,10 +85,10 @@ public void writeTo(final StreamOutput out) throws IOException {
         out.writeVInt(termStatistics.size());
 
         for (ObjectObjectCursor<Term, TermStatistics> c : termStatistics()) {
-            Term term = (Term) c.key;
+            Term term = c.key;
             out.writeString(term.field());
             out.writeBytesRef(term.bytes());
-            TermStatistics stats = (TermStatistics) c.value;
+            TermStatistics stats = c.value;
             out.writeBytesRef(stats.term());
             out.writeVLong(stats.docFreq());
             out.writeVLong(DfsSearchResult.addOne(stats.totalTermFreq()));
diff --git a/core/src/main/java/org/elasticsearch/search/dfs/DfsPhase.java b/core/src/main/java/org/elasticsearch/search/dfs/DfsPhase.java
index de06655f4146d..6be95a8bceb20 100644
--- a/core/src/main/java/org/elasticsearch/search/dfs/DfsPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/dfs/DfsPhase.java
@@ -28,16 +28,19 @@
 import org.apache.lucene.search.CollectionStatistics;
 import org.apache.lucene.search.TermStatistics;
 import org.elasticsearch.common.collect.HppcMaps;
+import org.elasticsearch.search.SearchContextException;
 import org.elasticsearch.search.SearchPhase;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.rescore.RescoreSearchContext;
+import org.elasticsearch.tasks.TaskCancelledException;
 
 import java.util.AbstractSet;
 import java.util.Collection;
 import java.util.Iterator;
 
 /**
- *
+ * Dfs phase of a search request, used to make scoring 100% accurate by collecting additional info from each shard before the query phase.
+ * The additional information is used to better compare the scores coming from all the shards, which depend on local factors (e.g. idf)
  */
 public class DfsPhase implements SearchPhase {
 
@@ -58,6 +61,9 @@ public void execute(SearchContext context) {
             TermStatistics[] termStatistics = new TermStatistics[terms.length];
             IndexReaderContext indexReaderContext = context.searcher().getTopReaderContext();
             for (int i = 0; i < terms.length; i++) {
+                if(context.isCancelled()) {
+                    throw new TaskCancelledException("cancelled");
+                }
                 // LUCENE 4 UPGRADE: cache TermContext?
                 TermContext termContext = TermContext.build(indexReaderContext, terms[i]);
                 termStatistics[i] = context.searcher().termStatistics(terms[i], termContext);
@@ -69,6 +75,9 @@ public void execute(SearchContext context) {
                 if (!fieldStatistics.containsKey(term.field())) {
                     final CollectionStatistics collectionStatistics = context.searcher().collectionStatistics(term.field());
                     fieldStatistics.put(term.field(), collectionStatistics);
+                    if(context.isCancelled()) {
+                        throw new TaskCancelledException("cancelled");
+                    }
                 }
             }
 
diff --git a/core/src/main/java/org/elasticsearch/search/dfs/DfsPhaseExecutionException.java b/core/src/main/java/org/elasticsearch/search/dfs/DfsPhaseExecutionException.java
index a6020b4498acd..dace1e271cff9 100644
--- a/core/src/main/java/org/elasticsearch/search/dfs/DfsPhaseExecutionException.java
+++ b/core/src/main/java/org/elasticsearch/search/dfs/DfsPhaseExecutionException.java
@@ -34,7 +34,11 @@ public DfsPhaseExecutionException(SearchContext context, String msg, Throwable t
         super(context, "Dfs Failed [" + msg + "]", t);
     }
 
+    public DfsPhaseExecutionException(SearchContext context, String msg) {
+        super(context, "Dfs Failed [" + msg + "]");
+    }
+
     public DfsPhaseExecutionException(StreamInput in) throws IOException {
         super(in);
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/search/dfs/DfsSearchResult.java b/core/src/main/java/org/elasticsearch/search/dfs/DfsSearchResult.java
index 6e93e41058725..0cd624b00a36b 100644
--- a/core/src/main/java/org/elasticsearch/search/dfs/DfsSearchResult.java
+++ b/core/src/main/java/org/elasticsearch/search/dfs/DfsSearchResult.java
@@ -30,47 +30,24 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
-import org.elasticsearch.transport.TransportResponse;
 
 import java.io.IOException;
 
-/**
- *
- */
-public class DfsSearchResult extends TransportResponse implements SearchPhaseResult {
+public class DfsSearchResult extends SearchPhaseResult {
 
     private static final Term[] EMPTY_TERMS = new Term[0];
     private static final TermStatistics[] EMPTY_TERM_STATS = new TermStatistics[0];
-
-    private SearchShardTarget shardTarget;
-    private long id;
     private Term[] terms;
     private TermStatistics[] termStatistics;
     private ObjectObjectHashMap<String, CollectionStatistics> fieldStatistics = HppcMaps.newNoNullKeysMap();
     private int maxDoc;
 
     public DfsSearchResult() {
-
     }
 
     public DfsSearchResult(long id, SearchShardTarget shardTarget) {
-        this.id = id;
-        this.shardTarget = shardTarget;
-    }
-
-    @Override
-    public long id() {
-        return this.id;
-    }
-
-    @Override
-    public SearchShardTarget shardTarget() {
-        return shardTarget;
-    }
-
-    @Override
-    public void shardTarget(SearchShardTarget shardTarget) {
-        this.shardTarget = shardTarget;
+        this.setSearchShardTarget(shardTarget);
+        this.requestId = id;
     }
 
     public DfsSearchResult maxDoc(int maxDoc) {
@@ -105,16 +82,10 @@ public ObjectObjectHashMap<String, CollectionStatistics> fieldStatistics() {
         return fieldStatistics;
     }
 
-    public static DfsSearchResult readDfsSearchResult(StreamInput in) throws IOException, ClassNotFoundException {
-        DfsSearchResult result = new DfsSearchResult();
-        result.readFrom(in);
-        return result;
-    }
-
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
-        id = in.readLong();
+        requestId = in.readLong();
         int termsSize = in.readVInt();
         if (termsSize == 0) {
             terms = EMPTY_TERMS;
@@ -131,11 +102,10 @@ public void readFrom(StreamInput in) throws IOException {
         maxDoc = in.readVInt();
     }
 
-
   @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        out.writeLong(id);
+        out.writeLong(requestId);
         out.writeVInt(terms.length);
         for (Term term : terms) {
             out.writeString(term.field());
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java
index 997cb7caa4b99..d16eeb04605e7 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java
@@ -39,19 +39,20 @@
 import org.elasticsearch.index.fieldvisitor.FieldsVisitor;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ObjectMapper;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
+import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHitField;
-import org.elasticsearch.search.SearchParseElement;
 import org.elasticsearch.search.SearchPhase;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
+import org.elasticsearch.search.fetch.subphase.InnerHitsContext;
 import org.elasticsearch.search.fetch.subphase.InnerHitsFetchSubPhase;
-import org.elasticsearch.search.internal.InternalSearchHit;
-import org.elasticsearch.search.internal.InternalSearchHitField;
-import org.elasticsearch.search.internal.InternalSearchHits;
+import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.tasks.TaskCancelledException;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -62,11 +63,11 @@
 import java.util.Map;
 import java.util.Set;
 
-import static java.util.Collections.unmodifiableMap;
 import static org.elasticsearch.common.xcontent.XContentFactory.contentBuilder;
 
 /**
- *
+ * Fetch phase of a search request, used to fetch the actual top matching documents to be returned to the client, identified
+ * after reducing all of the matches returned by the query phase
  */
 public class FetchPhase implements SearchPhase {
 
@@ -77,15 +78,6 @@ public FetchPhase(List<FetchSubPhase> fetchSubPhases) {
         this.fetchSubPhases[fetchSubPhases.size()] = new InnerHitsFetchSubPhase(this);
     }
 
-    @Override
-    public Map<String, ? extends SearchParseElement> parseElements() {
-        Map<String, SearchParseElement> parseElements = new HashMap<>();
-        for (FetchSubPhase fetchSubPhase : fetchSubPhases) {
-            parseElements.putAll(fetchSubPhase.parseElements());
-        }
-        return unmodifiableMap(parseElements);
-    }
-
     @Override
     public void preProcess(SearchContext context) {
     }
@@ -109,11 +101,9 @@ public void execute(SearchContext context) {
         } else {
             for (String fieldName : context.storedFieldsContext().fieldNames()) {
                 if (fieldName.equals(SourceFieldMapper.NAME)) {
-                    if (context.hasFetchSourceContext()) {
-                        context.fetchSourceContext().fetchSource(true);
-                    } else {
-                        context.fetchSourceContext(new FetchSourceContext(true));
-                    }
+                    FetchSourceContext fetchSourceContext = context.hasFetchSourceContext() ? context.fetchSourceContext()
+                            : FetchSourceContext.FETCH_SOURCE;
+                    context.fetchSourceContext(new FetchSourceContext(true, fetchSourceContext.includes(), fetchSourceContext.excludes()));
                     continue;
                 }
                 if (Regex.isSimpleMatchPattern(fieldName)) {
@@ -141,23 +131,27 @@ public void execute(SearchContext context) {
                 fieldsVisitor = new FieldsVisitor(loadSource);
             } else {
                 fieldsVisitor = new CustomFieldsVisitor(fieldNames == null ? Collections.emptySet() : fieldNames,
-                    fieldNamePatterns == null ? Collections.emptyList() : fieldNamePatterns, loadSource);
+                        fieldNamePatterns == null ? Collections.emptyList() : fieldNamePatterns, loadSource);
             }
         }
 
-        InternalSearchHit[] hits = new InternalSearchHit[context.docIdsToLoadSize()];
+        SearchHit[] hits = new SearchHit[context.docIdsToLoadSize()];
         FetchSubPhase.HitContext hitContext = new FetchSubPhase.HitContext();
         for (int index = 0; index < context.docIdsToLoadSize(); index++) {
+            if (context.isCancelled()) {
+                throw new TaskCancelledException("cancelled");
+            }
             int docId = context.docIdsToLoad()[context.docIdsToLoadFrom() + index];
             int readerIndex = ReaderUtil.subIndex(docId, context.searcher().getIndexReader().leaves());
             LeafReaderContext subReaderContext = context.searcher().getIndexReader().leaves().get(readerIndex);
             int subDocId = docId - subReaderContext.docBase;
 
-            final InternalSearchHit searchHit;
+            final SearchHit searchHit;
             try {
                 int rootDocId = findRootDocumentIfNested(context, subReaderContext, subDocId);
                 if (rootDocId != -1) {
-                    searchHit = createNestedSearchHit(context, docId, subDocId, rootDocId, fieldNames, fieldNamePatterns, subReaderContext);
+                    searchHit = createNestedSearchHit(context, docId, subDocId, rootDocId, fieldNames, fieldNamePatterns,
+                            subReaderContext);
                 } else {
                     searchHit = createSearchHit(context, fieldsVisitor, docId, subDocId, subReaderContext);
                 }
@@ -176,7 +170,7 @@ public void execute(SearchContext context) {
             fetchSubPhase.hitsExecute(context, hits);
         }
 
-        context.fetchResult().hits(new InternalSearchHits(hits, context.queryResult().topDocs().totalHits, context.queryResult().topDocs().getMaxScore()));
+        context.fetchResult().hits(new SearchHits(hits, context.queryResult().getTotalHits(), context.queryResult().getMaxScore()));
     }
 
     private int findRootDocumentIfNested(SearchContext context, LeafReaderContext subReaderContext, int subDocId) throws IOException {
@@ -189,9 +183,10 @@ private int findRootDocumentIfNested(SearchContext context, LeafReaderContext su
         return -1;
     }
 
-    private InternalSearchHit createSearchHit(SearchContext context, FieldsVisitor fieldsVisitor, int docId, int subDocId, LeafReaderContext subReaderContext) {
+    private SearchHit createSearchHit(SearchContext context, FieldsVisitor fieldsVisitor, int docId, int subDocId,
+                                      LeafReaderContext subReaderContext) {
         if (fieldsVisitor == null) {
-            return new InternalSearchHit(docId);
+            return new SearchHit(docId);
         }
         loadStoredFields(context, subReaderContext, fieldsVisitor, subDocId);
         fieldsVisitor.postProcess(context.mapperService());
@@ -200,7 +195,7 @@ private InternalSearchHit createSearchHit(SearchContext context, FieldsVisitor f
         if (!fieldsVisitor.fields().isEmpty()) {
             searchFields = new HashMap<>(fieldsVisitor.fields().size());
             for (Map.Entry<String, List<Object>> entry : fieldsVisitor.fields().entrySet()) {
-                searchFields.put(entry.getKey(), new InternalSearchHitField(entry.getKey(), entry.getValue()));
+                searchFields.put(entry.getKey(), new SearchHitField(entry.getKey(), entry.getValue()));
             }
         }
 
@@ -211,7 +206,7 @@ private InternalSearchHit createSearchHit(SearchContext context, FieldsVisitor f
         } else {
             typeText = documentMapper.typeText();
         }
-        InternalSearchHit searchHit = new InternalSearchHit(docId, fieldsVisitor.uid().id(), typeText, searchFields);
+        SearchHit searchHit = new SearchHit(docId, fieldsVisitor.uid().id(), typeText, searchFields);
         // Set _source if requested.
         SourceLookup sourceLookup = context.lookup().source();
         sourceLookup.setSegmentAndDocument(subReaderContext, subDocId);
@@ -221,24 +216,39 @@ private InternalSearchHit createSearchHit(SearchContext context, FieldsVisitor f
         return searchHit;
     }
 
-    private InternalSearchHit createNestedSearchHit(SearchContext context, int nestedTopDocId, int nestedSubDocId, int rootSubDocId, Set<String> fieldNames, List<String> fieldNamePatterns, LeafReaderContext subReaderContext) throws IOException {
+    private SearchHit createNestedSearchHit(SearchContext context, int nestedTopDocId, int nestedSubDocId,
+                                            int rootSubDocId, Set<String> fieldNames,
+                                            List<String> fieldNamePatterns, LeafReaderContext subReaderContext) throws IOException {
         // Also if highlighting is requested on nested documents we need to fetch the _source from the root document,
         // otherwise highlighting will attempt to fetch the _source from the nested doc, which will fail,
         // because the entire _source is only stored with the root document.
-        final FieldsVisitor rootFieldsVisitor = new FieldsVisitor(context.sourceRequested() || context.highlight() != null);
-        loadStoredFields(context, subReaderContext, rootFieldsVisitor, rootSubDocId);
-        rootFieldsVisitor.postProcess(context.mapperService());
+        final Uid uid;
+        final BytesReference source;
+        final boolean needSource = context.sourceRequested() || context.highlight() != null;
+        if (needSource || (context instanceof InnerHitsContext.InnerHitSubContext == false)) {
+            FieldsVisitor rootFieldsVisitor = new FieldsVisitor(needSource);
+            loadStoredFields(context, subReaderContext, rootFieldsVisitor, rootSubDocId);
+            rootFieldsVisitor.postProcess(context.mapperService());
+            uid = rootFieldsVisitor.uid();
+            source = rootFieldsVisitor.source();
+        } else {
+            // In case of nested inner hits we already know the uid, so no need to fetch it from stored fields again!
+            uid = ((InnerHitsContext.InnerHitSubContext) context).getUid();
+            source = null;
+        }
 
-        Map<String, SearchHitField> searchFields = getSearchFields(context, nestedSubDocId, fieldNames, fieldNamePatterns, subReaderContext);
-        DocumentMapper documentMapper = context.mapperService().documentMapper(rootFieldsVisitor.uid().type());
+
+        Map<String, SearchHitField> searchFields =
+                getSearchFields(context, nestedSubDocId, fieldNames, fieldNamePatterns, subReaderContext);
+        DocumentMapper documentMapper = context.mapperService().documentMapper(uid.type());
         SourceLookup sourceLookup = context.lookup().source();
         sourceLookup.setSegmentAndDocument(subReaderContext, nestedSubDocId);
 
         ObjectMapper nestedObjectMapper = documentMapper.findNestedObjectMapper(nestedSubDocId, context, subReaderContext);
         assert nestedObjectMapper != null;
-        InternalSearchHit.InternalNestedIdentity nestedIdentity = getInternalNestedIdentity(context, nestedSubDocId, subReaderContext, documentMapper, nestedObjectMapper);
+        SearchHit.NestedIdentity nestedIdentity =
+                getInternalNestedIdentity(context, nestedSubDocId, subReaderContext, context.mapperService(), nestedObjectMapper);
 
-        BytesReference source = rootFieldsVisitor.source();
         if (source != null) {
             Tuple<XContentType, Map<String, Object>> tuple = XContentHelper.convertToMap(source, true);
             Map<String, Object> sourceAsMap = tuple.v2();
@@ -253,17 +263,28 @@ private InternalSearchHit createNestedSearchHit(SearchContext context, int neste
                 String nestedPath = nested.getField().string();
                 current.put(nestedPath, new HashMap<>());
                 Object extractedValue = XContentMapValues.extractValue(nestedPath, sourceAsMap);
-                List<Map<String, Object>> nestedParsedSource;
+                List<?> nestedParsedSource;
                 if (extractedValue instanceof List) {
                     // nested field has an array value in the _source
-                    nestedParsedSource = (List<Map<String, Object>>) extractedValue;
+                    nestedParsedSource = (List<?>) extractedValue;
                 } else if (extractedValue instanceof Map) {
-                    // nested field has an object value in the _source. This just means the nested field has just one inner object, which is valid, but uncommon.
-                    nestedParsedSource = Collections.singletonList((Map<String, Object>) extractedValue);
+                    // nested field has an object value in the _source. This just means the nested field has just one inner object,
+                    // which is valid, but uncommon.
+                    nestedParsedSource = Collections.singletonList(extractedValue);
                 } else {
                     throw new IllegalStateException("extracted source isn't an object or an array");
                 }
-                sourceAsMap = nestedParsedSource.get(nested.getOffset());
+                if ((nestedParsedSource.get(0) instanceof Map) == false &&
+                    nestedObjectMapper.parentObjectMapperAreNested(context.mapperService()) == false) {
+                    // When one of the parent objects are not nested then XContentMapValues.extractValue(...) extracts the values
+                    // from two or more layers resulting in a list of list being returned. This is because nestedPath
+                    // encapsulates two or more object layers in the _source.
+                    //
+                    // This is why only the first element of nestedParsedSource needs to be checked.
+                    throw new IllegalArgumentException("Cannot execute inner hits. One or more parent object fields of nested field [" +
+                        nestedObjectMapper.name() + "] are not nested. All parent fields need to be nested fields too");
+                }
+                sourceAsMap = (Map<String, Object>) nestedParsedSource.get(nested.getOffset());
                 if (nested.getChild() == null) {
                     current.put(nestedPath, sourceAsMap);
                 } else {
@@ -278,22 +299,22 @@ private InternalSearchHit createNestedSearchHit(SearchContext context, int neste
             context.lookup().source().setSource(nestedSource);
             context.lookup().source().setSourceContentType(contentType);
         }
-
-        return new InternalSearchHit(nestedTopDocId, rootFieldsVisitor.uid().id(), documentMapper.typeText(), nestedIdentity, searchFields);
+        return new SearchHit(nestedTopDocId, uid.id(), documentMapper.typeText(), nestedIdentity, searchFields);
     }
 
-    private Map<String, SearchHitField> getSearchFields(SearchContext context, int nestedSubDocId, Set<String> fieldNames, List<String> fieldNamePatterns, LeafReaderContext subReaderContext) {
+    private Map<String, SearchHitField> getSearchFields(SearchContext context, int nestedSubDocId, Set<String> fieldNames,
+                                                       List<String> fieldNamePatterns, LeafReaderContext subReaderContext) {
         Map<String, SearchHitField> searchFields = null;
         if (context.hasStoredFields() && !context.storedFieldsContext().fieldNames().isEmpty()) {
             FieldsVisitor nestedFieldsVisitor = new CustomFieldsVisitor(fieldNames == null ? Collections.emptySet() : fieldNames,
-                fieldNamePatterns == null ? Collections.emptyList() : fieldNamePatterns, false);
+                    fieldNamePatterns == null ? Collections.emptyList() : fieldNamePatterns, false);
             if (nestedFieldsVisitor != null) {
                 loadStoredFields(context, subReaderContext, nestedFieldsVisitor, nestedSubDocId);
                 nestedFieldsVisitor.postProcess(context.mapperService());
                 if (!nestedFieldsVisitor.fields().isEmpty()) {
                     searchFields = new HashMap<>(nestedFieldsVisitor.fields().size());
                     for (Map.Entry<String, List<Object>> entry : nestedFieldsVisitor.fields().entrySet()) {
-                        searchFields.put(entry.getKey(), new InternalSearchHitField(entry.getKey(), entry.getValue()));
+                        searchFields.put(entry.getKey(), new SearchHitField(entry.getKey(), entry.getValue()));
                     }
                 }
             }
@@ -301,15 +322,18 @@ private Map<String, SearchHitField> getSearchFields(SearchContext context, int n
         return searchFields;
     }
 
-    private InternalSearchHit.InternalNestedIdentity getInternalNestedIdentity(SearchContext context, int nestedSubDocId, LeafReaderContext subReaderContext, DocumentMapper documentMapper, ObjectMapper nestedObjectMapper) throws IOException {
+    private SearchHit.NestedIdentity getInternalNestedIdentity(SearchContext context, int nestedSubDocId,
+                                                               LeafReaderContext subReaderContext,
+                                                               MapperService mapperService,
+                                                               ObjectMapper nestedObjectMapper) throws IOException {
         int currentParent = nestedSubDocId;
         ObjectMapper nestedParentObjectMapper;
         ObjectMapper current = nestedObjectMapper;
         String originalName = nestedObjectMapper.name();
-        InternalSearchHit.InternalNestedIdentity nestedIdentity = null;
+        SearchHit.NestedIdentity nestedIdentity = null;
         do {
             Query parentFilter;
-            nestedParentObjectMapper = documentMapper.findParentObjectMapper(current);
+            nestedParentObjectMapper = current.getParentObjectMapper(mapperService);
             if (nestedParentObjectMapper != null) {
                 if (nestedParentObjectMapper.nested().isNested() == false) {
                     current = nestedParentObjectMapper;
@@ -337,13 +361,14 @@ private InternalSearchHit.InternalNestedIdentity getInternalNestedIdentity(Searc
 
             int offset = 0;
             int nextParent = parentBits.nextSetBit(currentParent);
-            for (int docId = childIter.advance(currentParent + 1); docId < nextParent && docId != DocIdSetIterator.NO_MORE_DOCS; docId = childIter.nextDoc()) {
+            for (int docId = childIter.advance(currentParent + 1); docId < nextParent && docId != DocIdSetIterator.NO_MORE_DOCS;
+                 docId = childIter.nextDoc()) {
                 offset++;
             }
             currentParent = nextParent;
             current = nestedObjectMapper = nestedParentObjectMapper;
             int currentPrefix = current == null ? 0 : current.name().length() + 1;
-            nestedIdentity = new InternalSearchHit.InternalNestedIdentity(originalName.substring(currentPrefix), offset, nestedIdentity);
+            nestedIdentity = new SearchHit.NestedIdentity(originalName.substring(currentPrefix), offset, nestedIdentity);
             if (current != null) {
                 originalName = current.name();
             }
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/FetchPhaseExecutionException.java b/core/src/main/java/org/elasticsearch/search/fetch/FetchPhaseExecutionException.java
index 1f56ac871429b..5d17a1c95bf6c 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/FetchPhaseExecutionException.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/FetchPhaseExecutionException.java
@@ -34,6 +34,10 @@ public FetchPhaseExecutionException(SearchContext context, String msg, Throwable
         super(context, "Fetch Failed [" + msg + "]", t);
     }
 
+    public FetchPhaseExecutionException(SearchContext context, String msg) {
+        super(context, "Fetch Failed [" + msg + "]");
+    }
+
     public FetchPhaseExecutionException(StreamInput in) throws IOException {
         super(in);
     }
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/FetchSearchResult.java b/core/src/main/java/org/elasticsearch/search/fetch/FetchSearchResult.java
index ed8c0358dbb6e..72960334ea14c 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/FetchSearchResult.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/FetchSearchResult.java
@@ -21,59 +21,51 @@
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
-import org.elasticsearch.search.internal.InternalSearchHits;
-import org.elasticsearch.transport.TransportResponse;
+import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.search.query.QuerySearchResult;
 
 import java.io.IOException;
 
-import static org.elasticsearch.search.internal.InternalSearchHits.StreamContext;
+public final class FetchSearchResult extends SearchPhaseResult {
 
-/**
- *
- */
-public class FetchSearchResult extends TransportResponse implements FetchSearchResultProvider {
-
-    private long id;
-    private SearchShardTarget shardTarget;
-    private InternalSearchHits hits;
+    private SearchHits hits;
     // client side counter
     private transient int counter;
 
     public FetchSearchResult() {
-
     }
 
     public FetchSearchResult(long id, SearchShardTarget shardTarget) {
-        this.id = id;
-        this.shardTarget = shardTarget;
-    }
-
-    @Override
-    public FetchSearchResult fetchResult() {
-        return this;
+        this.requestId = id;
+        setSearchShardTarget(shardTarget);
     }
 
     @Override
-    public long id() {
-        return this.id;
+    public QuerySearchResult queryResult() {
+        return null;
     }
 
     @Override
-    public SearchShardTarget shardTarget() {
-        return this.shardTarget;
+    public FetchSearchResult fetchResult() {
+        return this;
     }
 
-    @Override
-    public void shardTarget(SearchShardTarget shardTarget) {
-        this.shardTarget = shardTarget;
+    public void hits(SearchHits hits) {
+        assert assertNoSearchTarget(hits);
+        this.hits = hits;
     }
 
-    public void hits(InternalSearchHits hits) {
-        this.hits = hits;
+    private boolean assertNoSearchTarget(SearchHits hits) {
+        for (SearchHit hit : hits.hits()) {
+            assert hit.getShard() == null : "expected null but got: " + hit.getShard();
+        }
+        return true;
     }
 
-    public InternalSearchHits hits() {
+    public SearchHits hits() {
         return hits;
     }
 
@@ -95,14 +87,14 @@ public static FetchSearchResult readFetchSearchResult(StreamInput in) throws IOE
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
-        id = in.readLong();
-        hits = InternalSearchHits.readSearchHits(in, InternalSearchHits.streamContext().streamShardTarget(StreamContext.ShardTargetType.NO_STREAM));
+        requestId = in.readLong();
+        hits = SearchHits.readSearchHits(in);
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        out.writeLong(id);
-        hits.writeTo(out, InternalSearchHits.streamContext().streamShardTarget(StreamContext.ShardTargetType.NO_STREAM));
+        out.writeLong(requestId);
+        hits.writeTo(out);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/FetchSearchResultProvider.java b/core/src/main/java/org/elasticsearch/search/fetch/FetchSearchResultProvider.java
deleted file mode 100644
index 5f4b81012989c..0000000000000
--- a/core/src/main/java/org/elasticsearch/search/fetch/FetchSearchResultProvider.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.fetch;
-
-import org.elasticsearch.search.SearchPhaseResult;
-
-/**
- *
- */
-public interface FetchSearchResultProvider extends SearchPhaseResult {
-
-    FetchSearchResult fetchResult();
-}
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java
index 2180cc34128ad..6f34eba2129b8 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java
@@ -22,11 +22,9 @@
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.IndexSearcher;
-import org.elasticsearch.search.SearchParseElement;
-import org.elasticsearch.search.internal.InternalSearchHit;
+import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.internal.SearchContext;
 
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -36,20 +34,20 @@
 public interface FetchSubPhase {
 
     class HitContext {
-        private InternalSearchHit hit;
+        private SearchHit hit;
         private IndexSearcher searcher;
         private LeafReaderContext readerContext;
         private int docId;
         private Map<String, Object> cache;
 
-        public void reset(InternalSearchHit hit, LeafReaderContext context, int docId, IndexSearcher searcher) {
+        public void reset(SearchHit hit, LeafReaderContext context, int docId, IndexSearcher searcher) {
             this.hit = hit;
             this.readerContext = context;
             this.docId = docId;
             this.searcher = searcher;
         }
 
-        public InternalSearchHit hit() {
+        public SearchHit hit() {
             return hit;
         }
 
@@ -69,10 +67,6 @@ public IndexReader topLevelReader() {
             return searcher.getIndexReader();
         }
 
-        public IndexSearcher topLevelSearcher() {
-            return searcher;
-        }
-
         public Map<String, Object> cache() {
             if (cache == null) {
                 cache = new HashMap<>();
@@ -82,34 +76,11 @@ public Map<String, Object> cache() {
 
     }
 
-    default Map<String, ? extends SearchParseElement> parseElements() {
-        return Collections.emptyMap();
-    }
-
     /**
      * Executes the hit level phase, with a reader and doc id (note, its a low level reader, and the matching doc).
      */
     default void hitExecute(SearchContext context, HitContext hitContext) {}
 
 
-    default void hitsExecute(SearchContext context, InternalSearchHit[] hits) {}
-
-    /**
-     * This interface is in the fetch phase plugin mechanism.
-     * Whenever a new search is executed we create a new {@link SearchContext} that holds individual contexts for each {@link org.elasticsearch.search.fetch.FetchSubPhase}.
-     * Fetch phases that use the plugin mechanism must provide a ContextFactory to the SearchContext that creates the fetch phase context and also associates them with a name.
-     * See {@link SearchContext#getFetchSubPhaseContext(FetchSubPhase.ContextFactory)}
-     */
-    interface ContextFactory<SubPhaseContext extends FetchSubPhaseContext> {
-
-        /**
-         * The name of the context.
-         */
-        String getName();
-
-        /**
-         * Creates a new instance of a FetchSubPhaseContext that holds all information a FetchSubPhase needs to execute on hits.
-         */
-        SubPhaseContext newContextInstance();
-    }
+    default void hitsExecute(SearchContext context, SearchHit[] hits) {}
 }
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/FetchSubPhaseContext.java b/core/src/main/java/org/elasticsearch/search/fetch/FetchSubPhaseContext.java
deleted file mode 100644
index 856c0ad902f33..0000000000000
--- a/core/src/main/java/org/elasticsearch/search/fetch/FetchSubPhaseContext.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.fetch;
-
-import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext;
-
-/**
- * All configuration and context needed by the FetchSubPhase to execute on hits.
- * The only required information in this base class is whether or not the sub phase needs to be run at all.
- * It can be extended by FetchSubPhases to hold information the phase needs to execute on hits.
- * See {@link org.elasticsearch.search.fetch.FetchSubPhase.ContextFactory} and also {@link DocValueFieldsContext} for an example.
- */
-public class FetchSubPhaseContext {
-
-    // This is to store if the FetchSubPhase should be executed at all.
-    private boolean hitExecutionNeeded = false;
-
-    /**
-     * Set if this phase should be executed at all.
-     */
-    public void setHitExecutionNeeded(boolean hitExecutionNeeded) {
-        this.hitExecutionNeeded = hitExecutionNeeded;
-    }
-
-    /**
-     * Returns if this phase be executed at all.
-     */
-    public boolean hitExecutionNeeded() {
-        return hitExecutionNeeded;
-    }
-
-}
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/QueryFetchSearchResult.java b/core/src/main/java/org/elasticsearch/search/fetch/QueryFetchSearchResult.java
index f3271f933febc..8d1e6276e65d9 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/QueryFetchSearchResult.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/QueryFetchSearchResult.java
@@ -21,25 +21,21 @@
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.query.QuerySearchResult;
-import org.elasticsearch.search.query.QuerySearchResultProvider;
 
 import java.io.IOException;
 
 import static org.elasticsearch.search.fetch.FetchSearchResult.readFetchSearchResult;
 import static org.elasticsearch.search.query.QuerySearchResult.readQuerySearchResult;
 
-/**
- *
- */
-public class QueryFetchSearchResult extends QuerySearchResultProvider implements FetchSearchResultProvider {
+public final class QueryFetchSearchResult extends SearchPhaseResult {
 
     private QuerySearchResult queryResult;
     private FetchSearchResult fetchResult;
 
     public QueryFetchSearchResult() {
-
     }
 
     public QueryFetchSearchResult(QuerySearchResult queryResult, FetchSearchResult fetchResult) {
@@ -48,24 +44,27 @@ public QueryFetchSearchResult(QuerySearchResult queryResult, FetchSearchResult f
     }
 
     @Override
-    public long id() {
-        return queryResult.id();
+    public long getRequestId() {
+        return queryResult.getRequestId();
     }
 
     @Override
-    public SearchShardTarget shardTarget() {
-        return queryResult.shardTarget();
+    public SearchShardTarget getSearchShardTarget() {
+        return queryResult.getSearchShardTarget();
     }
 
     @Override
-    public void shardTarget(SearchShardTarget shardTarget) {
-        queryResult.shardTarget(shardTarget);
-        fetchResult.shardTarget(shardTarget);
+    public void setSearchShardTarget(SearchShardTarget shardTarget) {
+        super.setSearchShardTarget(shardTarget);
+        queryResult.setSearchShardTarget(shardTarget);
+        fetchResult.setSearchShardTarget(shardTarget);
     }
 
     @Override
-    public boolean includeFetch() {
-        return true;
+    public void setShardIndex(int requestIndex) {
+        super.setShardIndex(requestIndex);
+        queryResult.setShardIndex(requestIndex);
+        fetchResult.setShardIndex(requestIndex);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/ScrollQueryFetchSearchResult.java b/core/src/main/java/org/elasticsearch/search/fetch/ScrollQueryFetchSearchResult.java
index dbaee5b64bb6c..55aa4a96d018c 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/ScrollQueryFetchSearchResult.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/ScrollQueryFetchSearchResult.java
@@ -21,49 +21,64 @@
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
-import org.elasticsearch.transport.TransportResponse;
+import org.elasticsearch.search.query.QuerySearchResult;
 
 import java.io.IOException;
 
 import static org.elasticsearch.search.fetch.QueryFetchSearchResult.readQueryFetchSearchResult;
 
-/**
- *
- */
-public class ScrollQueryFetchSearchResult extends TransportResponse {
+public final class ScrollQueryFetchSearchResult extends SearchPhaseResult {
 
     private QueryFetchSearchResult result;
-    private SearchShardTarget shardTarget;
 
     public ScrollQueryFetchSearchResult() {
     }
 
     public ScrollQueryFetchSearchResult(QueryFetchSearchResult result, SearchShardTarget shardTarget) {
         this.result = result;
-        this.shardTarget = shardTarget;
+        setSearchShardTarget(shardTarget);
     }
 
     public QueryFetchSearchResult result() {
         return result;
     }
 
-    public SearchShardTarget shardTarget() {
-        return shardTarget;
+    @Override
+    public void setSearchShardTarget(SearchShardTarget shardTarget) {
+        super.setSearchShardTarget(shardTarget);
+        result.setSearchShardTarget(shardTarget);
+    }
+
+    @Override
+    public void setShardIndex(int shardIndex) {
+        super.setShardIndex(shardIndex);
+        result.setShardIndex(shardIndex);
+    }
+
+    @Override
+    public QuerySearchResult queryResult() {
+        return result.queryResult();
+    }
+
+    @Override
+    public FetchSearchResult fetchResult() {
+        return result.fetchResult();
     }
 
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
-        shardTarget = new SearchShardTarget(in);
+        SearchShardTarget searchShardTarget = new SearchShardTarget(in);
         result = readQueryFetchSearchResult(in);
-        result.shardTarget(shardTarget);
+        setSearchShardTarget(searchShardTarget);
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        shardTarget.writeTo(out);
+        getSearchShardTarget().writeTo(out);
         result.writeTo(out);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/ShardFetchRequest.java b/core/src/main/java/org/elasticsearch/search/fetch/ShardFetchRequest.java
index 4087eb9a01cf8..dcea42e5ecb7f 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/ShardFetchRequest.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/ShardFetchRequest.java
@@ -22,9 +22,12 @@
 import com.carrotsearch.hppc.IntArrayList;
 import org.apache.lucene.search.FieldDoc;
 import org.apache.lucene.search.ScoreDoc;
+import org.elasticsearch.action.search.SearchTask;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lucene.Lucene;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.transport.TransportRequest;
 
 import java.io.IOException;
@@ -106,4 +109,15 @@ public void writeTo(StreamOutput out) throws IOException {
             Lucene.writeScoreDoc(out, lastEmittedDoc);
         }
     }
+
+    @Override
+    public Task createTask(long id, String type, String action, TaskId parentTaskId) {
+        return new SearchTask(id, type, action, getDescription(), parentTaskId);
+    }
+
+    @Override
+    public String getDescription() {
+        return "id[" + id + "], size[" + size + "], lastEmittedDoc[" + lastEmittedDoc + "]";
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/ShardFetchSearchRequest.java b/core/src/main/java/org/elasticsearch/search/fetch/ShardFetchSearchRequest.java
index f6738f9972511..fdfc582c95295 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/ShardFetchSearchRequest.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/ShardFetchSearchRequest.java
@@ -23,7 +23,6 @@
 import org.apache.lucene.search.ScoreDoc;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.OriginalIndices;
-import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -42,9 +41,9 @@ public ShardFetchSearchRequest() {
 
     }
 
-    public ShardFetchSearchRequest(SearchRequest request, long id, IntArrayList list, ScoreDoc lastEmittedDoc) {
+    public ShardFetchSearchRequest(OriginalIndices originalIndices, long id, IntArrayList list, ScoreDoc lastEmittedDoc) {
         super(id, list, lastEmittedDoc);
-        this.originalIndices = new OriginalIndices(request);
+        this.originalIndices = originalIndices;
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsContext.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsContext.java
index 54185734f97b4..325d28e459282 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsContext.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsContext.java
@@ -18,38 +18,23 @@
  */
 package org.elasticsearch.search.fetch.subphase;
 
-import org.elasticsearch.search.fetch.FetchSubPhaseContext;
-
-import java.util.ArrayList;
 import java.util.List;
 
 /**
  * All the required context to pull a field from the doc values.
  */
-public class DocValueFieldsContext extends FetchSubPhaseContext {
-
-    public static class DocValueField {
-        private final String name;
+public class DocValueFieldsContext {
 
-        public DocValueField(String name) {
-            this.name = name;
-        }
-
-        public String name() {
-            return name;
-        }
-    }
-
-    private List<DocValueField> fields = new ArrayList<>();
-
-    public DocValueFieldsContext() {
-    }
+    private final List<String> fields;
 
-    public void add(DocValueField field) {
-        this.fields.add(field);
+    public DocValueFieldsContext(List<String> fields) {
+        this.fields = fields;
     }
 
-    public List<DocValueField> fields() {
+    /**
+     * Returns the required docvalue fields
+     */
+    public List<String> fields() {
         return this.fields;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java
index 803cbb4348f9f..f5425b3c7e9ab 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java
@@ -23,10 +23,10 @@
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.search.SearchHitField;
 import org.elasticsearch.search.fetch.FetchSubPhase;
-import org.elasticsearch.search.internal.InternalSearchHitField;
 import org.elasticsearch.search.internal.SearchContext;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 
 /**
@@ -36,35 +36,30 @@
  */
 public final class DocValueFieldsFetchSubPhase implements FetchSubPhase {
 
-    public static final String NAME = "docvalue_fields";
-    public static final ContextFactory<DocValueFieldsContext> CONTEXT_FACTORY = new ContextFactory<DocValueFieldsContext>() {
-
-        @Override
-        public String getName() {
-            return NAME;
-        }
-
-        @Override
-        public DocValueFieldsContext newContextInstance() {
-            return new DocValueFieldsContext();
-        }
-    };
-
     @Override
     public void hitExecute(SearchContext context, HitContext hitContext) {
-        if (context.getFetchSubPhaseContext(CONTEXT_FACTORY).hitExecutionNeeded() == false) {
+        if (context.collapse() != null) {
+            // retrieve the `doc_value` associated with the collapse field
+            String name = context.collapse().getFieldType().name();
+            if (context.docValueFieldsContext() == null) {
+                context.docValueFieldsContext(new DocValueFieldsContext(Collections.singletonList(name)));
+            } else if (context.docValueFieldsContext().fields().contains(name) == false) {
+                context.docValueFieldsContext().fields().add(name);
+            }
+        }
+        if (context.docValueFieldsContext() == null) {
             return;
         }
-        for (DocValueFieldsContext.DocValueField field : context.getFetchSubPhaseContext(CONTEXT_FACTORY).fields()) {
+        for (String field : context.docValueFieldsContext().fields()) {
             if (hitContext.hit().fieldsOrNull() == null) {
                 hitContext.hit().fields(new HashMap<>(2));
             }
-            SearchHitField hitField = hitContext.hit().fields().get(field.name());
+            SearchHitField hitField = hitContext.hit().fields().get(field);
             if (hitField == null) {
-                hitField = new InternalSearchHitField(field.name(), new ArrayList<>(2));
-                hitContext.hit().fields().put(field.name(), hitField);
+                hitField = new SearchHitField(field, new ArrayList<>(2));
+                hitContext.hit().fields().put(field, hitField);
             }
-            MappedFieldType fieldType = context.mapperService().fullName(field.name());
+            MappedFieldType fieldType = context.mapperService().fullName(field);
             if (fieldType != null) {
                 AtomicFieldData data = context.fieldData().getForField(fieldType).load(hitContext.readerContext());
                 ScriptDocValues values = data.getScriptValues();
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourceContext.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourceContext.java
index 864de1628a723..128bef9ba146f 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourceContext.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourceContext.java
@@ -25,96 +25,73 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Streamable;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.rest.RestRequest;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
 
 /**
  * Context used to fetch the {@code _source}.
  */
-public class FetchSourceContext implements Streamable, ToXContent {
+public class FetchSourceContext implements Writeable, ToXContent {
+    private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(FetchSourceContext.class));
 
     public static final ParseField INCLUDES_FIELD = new ParseField("includes", "include");
     public static final ParseField EXCLUDES_FIELD = new ParseField("excludes", "exclude");
 
     public static final FetchSourceContext FETCH_SOURCE = new FetchSourceContext(true);
     public static final FetchSourceContext DO_NOT_FETCH_SOURCE = new FetchSourceContext(false);
-    private boolean fetchSource;
-    private String[] includes;
-    private String[] excludes;
+    private final boolean fetchSource;
+    private final String[] includes;
+    private final String[] excludes;
+    private Function<Map<String, ?>, Map<String, Object>> filter;
 
-    public static FetchSourceContext parse(QueryParseContext context) throws IOException {
-        FetchSourceContext fetchSourceContext = new FetchSourceContext();
-        fetchSourceContext.fromXContent(context);
-        return fetchSourceContext;
-    }
-
-    public FetchSourceContext() {
+    public FetchSourceContext(boolean fetchSource, String[] includes, String[] excludes) {
+        this.fetchSource = fetchSource;
+        this.includes = includes == null ? Strings.EMPTY_ARRAY : includes;
+        this.excludes = excludes == null ? Strings.EMPTY_ARRAY : excludes;
     }
 
     public FetchSourceContext(boolean fetchSource) {
         this(fetchSource, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY);
     }
 
-    public FetchSourceContext(String include) {
-        this(include, null);
-    }
-
-    public FetchSourceContext(String include, String exclude) {
-        this(true,
-                include == null ? Strings.EMPTY_ARRAY : new String[]{include},
-                exclude == null ? Strings.EMPTY_ARRAY : new String[]{exclude});
-    }
-
-    public FetchSourceContext(String[] includes) {
-        this(true, includes, Strings.EMPTY_ARRAY);
-    }
-
-    public FetchSourceContext(String[] includes, String[] excludes) {
-        this(true, includes, excludes);
+    public FetchSourceContext(StreamInput in) throws IOException {
+        fetchSource = in.readBoolean();
+        includes = in.readStringArray();
+        excludes = in.readStringArray();
     }
 
-    public FetchSourceContext(boolean fetchSource, String[] includes, String[] excludes) {
-        this.fetchSource = fetchSource;
-        this.includes = includes == null ? Strings.EMPTY_ARRAY : includes;
-        this.excludes = excludes == null ? Strings.EMPTY_ARRAY : excludes;
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeBoolean(fetchSource);
+        out.writeStringArray(includes);
+        out.writeStringArray(excludes);
     }
 
     public boolean fetchSource() {
         return this.fetchSource;
     }
 
-    public FetchSourceContext fetchSource(boolean fetchSource) {
-        this.fetchSource = fetchSource;
-        return this;
-    }
-
     public String[] includes() {
         return this.includes;
     }
 
-    public FetchSourceContext includes(String[] includes) {
-        this.includes = includes;
-        return this;
-    }
-
     public String[] excludes() {
         return this.excludes;
     }
 
-    public FetchSourceContext excludes(String[] excludes) {
-        this.excludes = excludes;
-        return this;
-    }
-
     public static FetchSourceContext parseFromRestRequest(RestRequest request) {
         Boolean fetchSource = null;
         String[] source_excludes = null;
@@ -129,6 +106,10 @@ public static FetchSourceContext parseFromRestRequest(RestRequest request) {
             } else {
                 source_includes = Strings.splitStringByCommaToArray(source);
             }
+            if (fetchSource != null && Booleans.isStrictlyBoolean(source) == false) {
+                DEPRECATION_LOGGER.deprecated("Expected a boolean [true/false] for request parameter [_source] but got [{}]", source);
+            }
+
         }
         String sIncludes = request.param("_source_includes");
         sIncludes = request.param("_source_include", sIncludes);
@@ -148,8 +129,7 @@ public static FetchSourceContext parseFromRestRequest(RestRequest request) {
         return null;
     }
 
-    public void fromXContent(QueryParseContext context) throws IOException {
-        XContentParser parser = context.parser();
+    public static FetchSourceContext fromXContent(XContentParser parser) throws IOException {
         XContentParser.Token token = parser.currentToken();
         boolean fetchSource = true;
         String[] includes = Strings.EMPTY_ARRAY;
@@ -170,7 +150,7 @@ public void fromXContent(QueryParseContext context) throws IOException {
                 if (token == XContentParser.Token.FIELD_NAME) {
                     currentFieldName = parser.currentName();
                 } else if (token == XContentParser.Token.START_ARRAY) {
-                    if (context.getParseFieldMatcher().match(currentFieldName, INCLUDES_FIELD)) {
+                    if (INCLUDES_FIELD.match(currentFieldName)) {
                         List<String> includesList = new ArrayList<>();
                         while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                             if (token == XContentParser.Token.VALUE_STRING) {
@@ -181,7 +161,7 @@ public void fromXContent(QueryParseContext context) throws IOException {
                             }
                         }
                         includes = includesList.toArray(new String[includesList.size()]);
-                    } else if (context.getParseFieldMatcher().match(currentFieldName, EXCLUDES_FIELD)) {
+                    } else if (EXCLUDES_FIELD.match(currentFieldName)) {
                         List<String> excludesList = new ArrayList<>();
                         while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                             if (token == XContentParser.Token.VALUE_STRING) {
@@ -197,10 +177,13 @@ public void fromXContent(QueryParseContext context) throws IOException {
                                 + " in [" + currentFieldName + "].", parser.getTokenLocation());
                     }
                 } else if (token == XContentParser.Token.VALUE_STRING) {
-                    if (context.getParseFieldMatcher().match(currentFieldName, INCLUDES_FIELD)) {
+                    if (INCLUDES_FIELD.match(currentFieldName)) {
                         includes = new String[] {parser.text()};
-                    } else if (context.getParseFieldMatcher().match(currentFieldName, EXCLUDES_FIELD)) {
+                    } else if (EXCLUDES_FIELD.match(currentFieldName)) {
                         excludes = new String[] {parser.text()};
+                    } else {
+                        throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token
+                            + " in [" + currentFieldName + "].", parser.getTokenLocation());
                     }
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName + "].",
@@ -211,9 +194,7 @@ public void fromXContent(QueryParseContext context) throws IOException {
             throw new ParsingException(parser.getTokenLocation(), "Expected one of [" + XContentParser.Token.VALUE_BOOLEAN + ", "
                     + XContentParser.Token.START_OBJECT + "] but found [" + token + "]", parser.getTokenLocation());
         }
-        this.fetchSource = fetchSource;
-        this.includes = includes;
-        this.excludes = excludes;
+        return new FetchSourceContext(fetchSource, includes, excludes);
     }
 
     @Override
@@ -229,22 +210,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         return builder;
     }
 
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        fetchSource = in.readBoolean();
-        includes = in.readStringArray();
-        excludes = in.readStringArray();
-        in.readBoolean(); // Used to be transformSource but that was dropped in 2.1
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeBoolean(fetchSource);
-        out.writeStringArray(includes);
-        out.writeStringArray(excludes);
-        out.writeBoolean(false); // Used to be transformSource but that was dropped in 2.1
-    }
-
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
@@ -266,4 +231,15 @@ public int hashCode() {
         result = 31 * result + (excludes != null ? Arrays.hashCode(excludes) : 0);
         return result;
     }
+
+    /**
+     * Returns a filter function that expects the source map as an input and returns
+     * the filtered map.
+     */
+    public Function<Map<String, ?>, Map<String, Object>> getFilter() {
+        if (filter == null) {
+            filter = XContentMapValues.filter(includes, excludes);
+        }
+        return filter;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourceSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourceSubPhase.java
index c67b96c7af588..3171ca4b00834 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourceSubPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourceSubPhase.java
@@ -36,9 +36,6 @@ public void hitExecute(SearchContext context, HitContext hitContext) {
             return;
         }
         SourceLookup source = context.lookup().source();
-        if (source.internalSourceRef() == null) {
-            return; // source disabled in the mapping
-        }
         FetchSourceContext fetchSourceContext = context.fetchSourceContext();
         assert fetchSourceContext.fetchSource();
         if (fetchSourceContext.includes().length == 0 && fetchSourceContext.excludes().length == 0) {
@@ -46,7 +43,12 @@ public void hitExecute(SearchContext context, HitContext hitContext) {
             return;
         }
 
-        Object value = source.filter(fetchSourceContext.includes(), fetchSourceContext.excludes());
+        if (source.internalSourceRef() == null) {
+            throw new IllegalArgumentException("unable to fetch fields from _source field: _source is disabled in the mappings " +
+                    "for index [" + context.indexShard().shardId().getIndexName() + "]");
+        }
+
+        final Object value = source.filter(fetchSourceContext);
         try {
             final int initialCapacity = Math.min(1024, source.internalSourceRef().length());
             BytesStreamOutput streamOutput = new BytesStreamOutput(initialCapacity);
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsContext.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsContext.java
index 44a6b13fd403d..9ec940b8f5b56 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsContext.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsContext.java
@@ -19,84 +19,72 @@
 
 package org.elasticsearch.search.fetch.subphase;
 
-import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.BooleanClause.Occur;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.ConstantScoreScorer;
-import org.apache.lucene.search.ConstantScoreWeight;
+import org.apache.lucene.search.CollectionTerminatedException;
+import org.apache.lucene.search.Collector;
+import org.apache.lucene.search.ConjunctionDISI;
 import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.DocValuesTermsQuery;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Query;
+import org.apache.lucene.search.LeafCollector;
 import org.apache.lucene.search.Scorer;
-import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.ScorerSupplier;
 import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.search.TopDocsCollector;
-import org.apache.lucene.search.TopFieldCollector;
-import org.apache.lucene.search.TopScoreDocCollector;
 import org.apache.lucene.search.Weight;
-import org.apache.lucene.search.join.BitSetProducer;
-import org.apache.lucene.util.BitSet;
-import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.common.lucene.Lucene;
-import org.elasticsearch.common.lucene.search.Queries;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.ObjectMapper;
-import org.elasticsearch.index.mapper.ParentFieldMapper;
+import org.apache.lucene.util.Bits;
 import org.elasticsearch.index.mapper.Uid;
-import org.elasticsearch.index.mapper.UidFieldMapper;
-import org.elasticsearch.search.SearchHitField;
-import org.elasticsearch.search.fetch.FetchSubPhase;
-import org.elasticsearch.search.internal.InternalSearchHit;
+import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.internal.SubSearchContext;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
 
-/**
- */
 public final class InnerHitsContext {
-
-    private final Map<String, BaseInnerHits> innerHits;
+    private final Map<String, InnerHitSubContext> innerHits;
 
     public InnerHitsContext() {
         this.innerHits = new HashMap<>();
     }
 
-    public InnerHitsContext(Map<String, BaseInnerHits> innerHits) {
+    InnerHitsContext(Map<String, InnerHitSubContext> innerHits) {
         this.innerHits = Objects.requireNonNull(innerHits);
     }
 
-    public Map<String, BaseInnerHits> getInnerHits() {
+    public Map<String, InnerHitSubContext> getInnerHits() {
         return innerHits;
     }
 
-    public void addInnerHitDefinition(BaseInnerHits innerHit) {
+    public void addInnerHitDefinition(InnerHitSubContext innerHit) {
         if (innerHits.containsKey(innerHit.getName())) {
             throw new IllegalArgumentException("inner_hit definition with the name [" + innerHit.getName() +
-                    "] already exists. Use a different inner_hit name or define one explicitly");
+                "] already exists. Use a different inner_hit name or define one explicitly");
         }
 
         innerHits.put(innerHit.getName(), innerHit);
     }
 
-    public abstract static class BaseInnerHits extends SubSearchContext {
+    /**
+     * A {@link SubSearchContext} that associates {@link TopDocs} to each {@link SearchHit}
+     * in the parent search context
+     */
+    public abstract static class InnerHitSubContext extends SubSearchContext {
 
         private final String name;
+        protected final SearchContext context;
         private InnerHitsContext childInnerHits;
 
-        protected BaseInnerHits(String name, SearchContext context) {
+        // TODO: when types are complete removed just use String instead for the id:
+        private Uid uid;
+
+        protected InnerHitSubContext(String name, SearchContext context) {
             super(context);
             this.name = name;
+            this.context = context;
         }
 
-        public abstract TopDocs topDocs(SearchContext context, FetchSubPhase.HitContext hitContext) throws IOException;
+        public abstract TopDocs[] topDocs(SearchHit[] hits) throws IOException;
 
         public String getName() {
             return name;
@@ -107,236 +95,63 @@ public InnerHitsContext innerHits() {
             return childInnerHits;
         }
 
-        public void setChildInnerHits(Map<String, InnerHitsContext.BaseInnerHits> childInnerHits) {
+        public void setChildInnerHits(Map<String, InnerHitSubContext> childInnerHits) {
             this.childInnerHits = new InnerHitsContext(childInnerHits);
         }
-    }
-
-    public static final class NestedInnerHits extends BaseInnerHits {
-
-        private final ObjectMapper parentObjectMapper;
-        private final ObjectMapper childObjectMapper;
 
-        public NestedInnerHits(String name, SearchContext context, ObjectMapper parentObjectMapper, ObjectMapper childObjectMapper) {
-            super(name != null ? name : childObjectMapper.fullPath(), context);
-            this.parentObjectMapper = parentObjectMapper;
-            this.childObjectMapper = childObjectMapper;
+        protected Weight createInnerHitQueryWeight() throws IOException {
+            final boolean needsScores = size() != 0 && (sort() == null || sort().sort.needsScores());
+            return context.searcher().createNormalizedWeight(query(), needsScores);
         }
 
-        @Override
-        public TopDocs topDocs(SearchContext context, FetchSubPhase.HitContext hitContext) throws IOException {
-            Query rawParentFilter;
-            if (parentObjectMapper == null) {
-                rawParentFilter = Queries.newNonNestedFilter();
-            } else {
-                rawParentFilter = parentObjectMapper.nestedTypeFilter();
-            }
-            BitSetProducer parentFilter = context.bitsetFilterCache().getBitSetProducer(rawParentFilter);
-            Query childFilter = childObjectMapper.nestedTypeFilter();
-            Query q = Queries.filtered(query(), new NestedChildrenQuery(parentFilter, childFilter, hitContext));
-
-            if (size() == 0) {
-                return new TopDocs(context.searcher().count(q), Lucene.EMPTY_SCORE_DOCS, 0);
-            } else {
-                int topN = Math.min(from() + size(), context.searcher().getIndexReader().maxDoc());
-                TopDocsCollector topDocsCollector;
-                if (sort() != null) {
-                    try {
-                        topDocsCollector = TopFieldCollector.create(sort().sort, topN, true, trackScores(), trackScores());
-                    } catch (IOException e) {
-                        throw ExceptionsHelper.convertToElastic(e);
-                    }
-                } else {
-                    topDocsCollector = TopScoreDocCollector.create(topN);
-                }
-                try {
-                    context.searcher().search(q, topDocsCollector);
-                } finally {
-                    clearReleasables(Lifetime.COLLECTION);
-                }
-                return topDocsCollector.topDocs(from(), size());
-            }
+        public SearchContext parentSearchContext() {
+            return context;
         }
 
-        // A filter that only emits the nested children docs of a specific nested parent doc
-        static class NestedChildrenQuery extends Query {
-
-            private final BitSetProducer parentFilter;
-            private final Query childFilter;
-            private final int docId;
-            private final LeafReader leafReader;
-
-            NestedChildrenQuery(BitSetProducer parentFilter, Query childFilter, FetchSubPhase.HitContext hitContext) {
-                this.parentFilter = parentFilter;
-                this.childFilter = childFilter;
-                this.docId = hitContext.docId();
-                this.leafReader = hitContext.readerContext().reader();
-            }
-
-            @Override
-            public boolean equals(Object obj) {
-                if (sameClassAs(obj) == false) {
-                    return false;
-                }
-                NestedChildrenQuery other = (NestedChildrenQuery) obj;
-                return parentFilter.equals(other.parentFilter)
-                        && childFilter.equals(other.childFilter)
-                        && docId == other.docId
-                        && leafReader.getCoreCacheKey() == other.leafReader.getCoreCacheKey();
-            }
-
-            @Override
-            public int hashCode() {
-                int hash = classHash();
-                hash = 31 * hash + parentFilter.hashCode();
-                hash = 31 * hash + childFilter.hashCode();
-                hash = 31 * hash + docId;
-                hash = 31 * hash + leafReader.getCoreCacheKey().hashCode();
-                return hash;
-            }
-
-            @Override
-            public String toString(String field) {
-                return "NestedChildren(parent=" + parentFilter + ",child=" + childFilter + ")";
-            }
-
-            @Override
-            public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException {
-                final Weight childWeight = childFilter.createWeight(searcher, false);
-                return new ConstantScoreWeight(this) {
-                    @Override
-                    public Scorer scorer(LeafReaderContext context) throws IOException {
-                        // Nested docs only reside in a single segment, so no need to evaluate all segments
-                        if (!context.reader().getCoreCacheKey().equals(leafReader.getCoreCacheKey())) {
-                            return null;
-                        }
-
-                        // If docId == 0 then we a parent doc doesn't have child docs, because child docs are stored
-                        // before the parent doc and because parent doc is 0 we can safely assume that there are no child docs.
-                        if (docId == 0) {
-                            return null;
-                        }
-
-                        final BitSet parents = parentFilter.getBitSet(context);
-                        final int firstChildDocId = parents.prevSetBit(docId - 1) + 1;
-                        // A parent doc doesn't have child docs, so we can early exit here:
-                        if (firstChildDocId == docId) {
-                            return null;
-                        }
-
-                        final Scorer childrenScorer = childWeight.scorer(context);
-                        if (childrenScorer == null) {
-                            return null;
-                        }
-                        DocIdSetIterator childrenIterator = childrenScorer.iterator();
-                        final DocIdSetIterator it = new DocIdSetIterator() {
-
-                            int doc = -1;
-
-                            @Override
-                            public int docID() {
-                                return doc;
-                            }
-
-                            @Override
-                            public int nextDoc() throws IOException {
-                                return advance(doc + 1);
-                            }
-
-                            @Override
-                            public int advance(int target) throws IOException {
-                                target = Math.max(firstChildDocId, target);
-                                if (target >= docId) {
-                                    // We're outside the child nested scope, so it is done
-                                    return doc = NO_MORE_DOCS;
-                                } else {
-                                    int advanced = childrenIterator.advance(target);
-                                    if (advanced >= docId) {
-                                        // We're outside the child nested scope, so it is done
-                                        return doc = NO_MORE_DOCS;
-                                    } else {
-                                        return doc = advanced;
-                                    }
-                                }
-                            }
-
-                            @Override
-                            public long cost() {
-                                return Math.min(childrenIterator.cost(), docId - firstChildDocId);
-                            }
-
-                        };
-                        return new ConstantScoreScorer(this, score(), it);
-                    }
-                };
-            }
+        public Uid getUid() {
+            return uid;
         }
 
+        public void setUid(Uid uid) {
+            this.uid = uid;
+        }
     }
 
-    public static final class ParentChildInnerHits extends BaseInnerHits {
-
-        private final MapperService mapperService;
-        private final DocumentMapper documentMapper;
-
-        public ParentChildInnerHits(String name, SearchContext context, MapperService mapperService, DocumentMapper documentMapper) {
-            super(name != null ? name : documentMapper.type(), context);
-            this.mapperService = mapperService;
-            this.documentMapper = documentMapper;
+    public static void intersect(Weight weight, Weight innerHitQueryWeight, Collector collector, LeafReaderContext ctx) throws IOException {
+        ScorerSupplier scorerSupplier = weight.scorerSupplier(ctx);
+        if (scorerSupplier == null) {
+            return;
         }
+        // use random access since this scorer will be consumed on a minority of documents
+        Scorer scorer = scorerSupplier.get(true);
 
-        @Override
-        public TopDocs topDocs(SearchContext context, FetchSubPhase.HitContext hitContext) throws IOException {
-            final Query hitQuery;
-            if (isParentHit(hitContext.hit())) {
-                String field = ParentFieldMapper.joinField(hitContext.hit().type());
-                hitQuery = new DocValuesTermsQuery(field, hitContext.hit().id());
-            } else if (isChildHit(hitContext.hit())) {
-                DocumentMapper hitDocumentMapper = mapperService.documentMapper(hitContext.hit().type());
-                final String parentType = hitDocumentMapper.parentFieldMapper().type();
-                SearchHitField parentField = hitContext.hit().field(ParentFieldMapper.NAME);
-                if (parentField == null) {
-                    throw new IllegalStateException("All children must have a _parent");
-                }
-                hitQuery = new TermQuery(new Term(UidFieldMapper.NAME, Uid.createUid(parentType, parentField.getValue())));
-            } else {
-                return Lucene.EMPTY_TOP_DOCS;
-            }
-
-            BooleanQuery q = new BooleanQuery.Builder()
-                .add(query(), Occur.MUST)
-                // Only include docs that have the current hit as parent
-                .add(hitQuery, Occur.FILTER)
-                // Only include docs that have this inner hits type
-                .add(documentMapper.typeFilter(), Occur.FILTER)
-                .build();
-            if (size() == 0) {
-                final int count = context.searcher().count(q);
-                return new TopDocs(count, Lucene.EMPTY_SCORE_DOCS, 0);
-            } else {
-                int topN = Math.min(from() + size(), context.searcher().getIndexReader().maxDoc());
-                TopDocsCollector topDocsCollector;
-                if (sort() != null) {
-                    topDocsCollector = TopFieldCollector.create(sort().sort, topN, true, trackScores(), trackScores());
-                } else {
-                    topDocsCollector = TopScoreDocCollector.create(topN);
-                }
-                try {
-                    context.searcher().search(q, topDocsCollector);
-                } finally {
-                    clearReleasables(Lifetime.COLLECTION);
-                }
-                return topDocsCollector.topDocs(from(), size());
-            }
+        ScorerSupplier innerHitQueryScorerSupplier = innerHitQueryWeight.scorerSupplier(ctx);
+        if (innerHitQueryScorerSupplier == null) {
+            return;
         }
-
-        private boolean isParentHit(InternalSearchHit hit) {
-            return hit.type().equals(documentMapper.parentFieldMapper().type());
+        // use random access since this scorer will be consumed on a minority of documents
+        Scorer innerHitQueryScorer = innerHitQueryScorerSupplier.get(true);
+
+        final LeafCollector leafCollector;
+        try {
+            leafCollector = collector.getLeafCollector(ctx);
+            // Just setting the innerHitQueryScorer is ok, because that is the actual scoring part of the query
+            leafCollector.setScorer(innerHitQueryScorer);
+        } catch (CollectionTerminatedException e) {
+            return;
         }
 
-        private boolean isChildHit(InternalSearchHit hit) {
-            DocumentMapper hitDocumentMapper = mapperService.documentMapper(hit.type());
-            return documentMapper.type().equals(hitDocumentMapper.parentFieldMapper().type());
+        try {
+            Bits acceptDocs = ctx.reader().getLiveDocs();
+            DocIdSetIterator iterator = ConjunctionDISI.intersectIterators(Arrays.asList(innerHitQueryScorer.iterator(),
+                scorer.iterator()));
+            for (int docId = iterator.nextDoc(); docId < DocIdSetIterator.NO_MORE_DOCS; docId = iterator.nextDoc()) {
+                if (acceptDocs == null || acceptDocs.get(docId)) {
+                    leafCollector.collect(docId);
+                }
+            }
+        } catch (CollectionTerminatedException e) {
+            // ignore and continue
         }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsFetchSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsFetchSubPhase.java
index 23c63bc7eef0f..e5907425fee9b 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsFetchSubPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsFetchSubPhase.java
@@ -23,11 +23,12 @@
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.TopDocs;
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.index.mapper.Uid;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.fetch.FetchPhase;
 import org.elasticsearch.search.fetch.FetchSearchResult;
 import org.elasticsearch.search.fetch.FetchSubPhase;
-import org.elasticsearch.search.internal.InternalSearchHit;
-import org.elasticsearch.search.internal.InternalSearchHits;
 import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
@@ -43,39 +44,48 @@ public InnerHitsFetchSubPhase(FetchPhase fetchPhase) {
     }
 
     @Override
-    public void hitExecute(SearchContext context, HitContext hitContext) {
+    public void hitsExecute(SearchContext context, SearchHit[] hits) {
         if ((context.innerHits() != null && context.innerHits().getInnerHits().size() > 0) == false) {
             return;
         }
-        Map<String, InternalSearchHits> results = new HashMap<>();
-        for (Map.Entry<String, InnerHitsContext.BaseInnerHits> entry : context.innerHits().getInnerHits().entrySet()) {
-            InnerHitsContext.BaseInnerHits innerHits = entry.getValue();
-            TopDocs topDocs;
+
+        for (Map.Entry<String, InnerHitsContext.InnerHitSubContext> entry : context.innerHits().getInnerHits().entrySet()) {
+            InnerHitsContext.InnerHitSubContext innerHits = entry.getValue();
+            TopDocs[] topDocs;
             try {
-                topDocs = innerHits.topDocs(context, hitContext);
+                topDocs = innerHits.topDocs(hits);
             } catch (IOException e) {
                 throw ExceptionsHelper.convertToElastic(e);
             }
-            innerHits.queryResult().topDocs(topDocs, innerHits.sort() == null ? null : innerHits.sort().formats);
-            int[] docIdsToLoad = new int[topDocs.scoreDocs.length];
-            for (int i = 0; i < topDocs.scoreDocs.length; i++) {
-                docIdsToLoad[i] = topDocs.scoreDocs[i].doc;
-            }
-            innerHits.docIdsToLoad(docIdsToLoad, 0, docIdsToLoad.length);
-            fetchPhase.execute(innerHits);
-            FetchSearchResult fetchResult = innerHits.fetchResult();
-            InternalSearchHit[] internalHits = fetchResult.fetchResult().hits().internalHits();
-            for (int i = 0; i < internalHits.length; i++) {
-                ScoreDoc scoreDoc = topDocs.scoreDocs[i];
-                InternalSearchHit searchHitFields = internalHits[i];
-                searchHitFields.score(scoreDoc.score);
-                if (scoreDoc instanceof FieldDoc) {
-                    FieldDoc fieldDoc = (FieldDoc) scoreDoc;
-                    searchHitFields.sortValues(fieldDoc.fields, innerHits.sort().formats);
+            for (int i = 0; i < hits.length; i++) {
+                SearchHit hit = hits[i];
+                TopDocs topDoc = topDocs[i];
+
+                Map<String, SearchHits> results = hit.getInnerHits();
+                if (results == null) {
+                    hit.setInnerHits(results = new HashMap<>());
+                }
+                innerHits.queryResult().topDocs(topDoc, innerHits.sort() == null ? null : innerHits.sort().formats);
+                int[] docIdsToLoad = new int[topDoc.scoreDocs.length];
+                for (int j = 0; j < topDoc.scoreDocs.length; j++) {
+                    docIdsToLoad[j] = topDoc.scoreDocs[j].doc;
+                }
+                innerHits.docIdsToLoad(docIdsToLoad, 0, docIdsToLoad.length);
+                innerHits.setUid(new Uid(hit.getType(), hit.getId()));
+                fetchPhase.execute(innerHits);
+                FetchSearchResult fetchResult = innerHits.fetchResult();
+                SearchHit[] internalHits = fetchResult.fetchResult().hits().internalHits();
+                for (int j = 0; j < internalHits.length; j++) {
+                    ScoreDoc scoreDoc = topDoc.scoreDocs[j];
+                    SearchHit searchHitFields = internalHits[j];
+                    searchHitFields.score(scoreDoc.score);
+                    if (scoreDoc instanceof FieldDoc) {
+                        FieldDoc fieldDoc = (FieldDoc) scoreDoc;
+                        searchHitFields.sortValues(fieldDoc.fields, innerHits.sort().formats);
+                    }
                 }
+                results.put(entry.getKey(), fetchResult.hits());
             }
-            results.put(entry.getKey(), fetchResult.hits());
         }
-        hitContext.hit().setInnerHits(results);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesFetchSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesFetchSubPhase.java
index 56223b1ec4617..c28e07ff45526 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesFetchSubPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesFetchSubPhase.java
@@ -22,13 +22,13 @@
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.ReaderUtil;
 import org.apache.lucene.search.Query;
-import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.ScorerSupplier;
 import org.apache.lucene.search.Weight;
 import org.apache.lucene.util.Bits;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.search.fetch.FetchSubPhase;
-import org.elasticsearch.search.internal.InternalSearchHit;
+import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.internal.SearchContext.Lifetime;
 
@@ -42,7 +42,7 @@
 public final class MatchedQueriesFetchSubPhase implements FetchSubPhase {
 
     @Override
-    public void hitsExecute(SearchContext context, InternalSearchHit[] hits) {
+    public void hitsExecute(SearchContext context, SearchHit[] hits) {
         if (hits.length == 0 ||
             // in case the request has only suggest, parsed query is null
             context.parsedQuery() == null) {
@@ -71,15 +71,15 @@ public void hitsExecute(SearchContext context, InternalSearchHit[] hits) {
                 Bits matchingDocs = null;
                 final IndexReader indexReader = context.searcher().getIndexReader();
                 for (int i = 0; i < hits.length; ++i) {
-                    InternalSearchHit hit = hits[i];
+                    SearchHit hit = hits[i];
                     int hitReaderIndex = ReaderUtil.subIndex(hit.docId(), indexReader.leaves());
                     if (readerIndex != hitReaderIndex) {
                         readerIndex = hitReaderIndex;
                         LeafReaderContext ctx = indexReader.leaves().get(readerIndex);
                         docBase = ctx.docBase;
                         // scorers can be costly to create, so reuse them across docs of the same segment
-                        Scorer scorer = weight.scorer(ctx);
-                        matchingDocs = Lucene.asSequentialAccessBits(ctx.reader().maxDoc(), scorer);
+                        ScorerSupplier scorerSupplier = weight.scorerSupplier(ctx);
+                        matchingDocs = Lucene.asSequentialAccessBits(ctx.reader().maxDoc(), scorerSupplier);
                     }
                     if (matchingDocs.get(hit.docId() - docBase)) {
                         matchedQueries[i].add(name);
@@ -92,7 +92,7 @@ public void hitsExecute(SearchContext context, InternalSearchHit[] hits) {
         } catch (IOException e) {
             throw ExceptionsHelper.convertToElastic(e);
         } finally {
-            SearchContext.current().clearReleasables(Lifetime.COLLECTION);
+            context.clearReleasables(Lifetime.COLLECTION);
         }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/ParentFieldSubFetchPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/ParentFieldSubFetchPhase.java
index ccfbf3515fc35..9cfab2ff9812b 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/subphase/ParentFieldSubFetchPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/subphase/ParentFieldSubFetchPhase.java
@@ -26,7 +26,6 @@
 import org.elasticsearch.index.mapper.ParentFieldMapper;
 import org.elasticsearch.search.SearchHitField;
 import org.elasticsearch.search.fetch.FetchSubPhase;
-import org.elasticsearch.search.internal.InternalSearchHitField;
 import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
@@ -47,20 +46,28 @@ public void hitExecute(SearchContext context, HitContext hitContext) {
         }
 
         String parentId = getParentId(parentFieldMapper, hitContext.reader(), hitContext.docId());
+        if (parentId == null) {
+            // hit has no _parent field. Can happen for nested inner hits if parent hit is a p/c document.
+            return;
+        }
+
         Map<String, SearchHitField> fields = hitContext.hit().fieldsOrNull();
         if (fields == null) {
             fields = new HashMap<>();
             hitContext.hit().fields(fields);
         }
-        fields.put(ParentFieldMapper.NAME, new InternalSearchHitField(ParentFieldMapper.NAME, Collections.singletonList(parentId)));
+        fields.put(ParentFieldMapper.NAME, new SearchHitField(ParentFieldMapper.NAME, Collections.singletonList(parentId)));
     }
 
     public static String getParentId(ParentFieldMapper fieldMapper, LeafReader reader, int docId) {
         try {
             SortedDocValues docValues = reader.getSortedDocValues(fieldMapper.name());
+            if (docValues == null) {
+                // hit has no _parent field.
+                return null;
+            }
             BytesRef parentId = docValues.get(docId);
-            assert parentId.length > 0;
-            return parentId.utf8ToString();
+            return parentId.length > 0 ? parentId.utf8ToString() : null;
         } catch (IOException e) {
             throw ExceptionsHelper.convertToElastic(e);
         }
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/ScriptFieldsFetchSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/ScriptFieldsFetchSubPhase.java
index 80638860f6cfa..d5f587db29dab 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/subphase/ScriptFieldsFetchSubPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/subphase/ScriptFieldsFetchSubPhase.java
@@ -21,7 +21,6 @@
 import org.elasticsearch.script.LeafSearchScript;
 import org.elasticsearch.search.SearchHitField;
 import org.elasticsearch.search.fetch.FetchSubPhase;
-import org.elasticsearch.search.internal.InternalSearchHitField;
 import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
@@ -70,7 +69,7 @@ public void hitExecute(SearchContext context, HitContext hitContext) {
                 } else {
                     values = Collections.singletonList(value);
                 }
-                hitField = new InternalSearchHitField(scriptField.name(), values);
+                hitField = new SearchHitField(scriptField.name(), values);
                 hitContext.hit().fields().put(scriptField.name(), hitField);
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/AbstractHighlighterBuilder.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/AbstractHighlighterBuilder.java
index 72bd436a88cb2..4699843153f25 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/AbstractHighlighterBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/AbstractHighlighterBuilder.java
@@ -21,6 +21,7 @@
 
 import org.apache.lucene.search.highlight.SimpleFragmenter;
 import org.apache.lucene.search.highlight.SimpleSpanFragmenter;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.support.ToXContentToBytes;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParsingException;
@@ -32,10 +33,12 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder.BoundaryScannerType;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder.Order;
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 import java.util.function.BiFunction;
@@ -57,8 +60,10 @@ public abstract class AbstractHighlighterBuilder<HB extends AbstractHighlighterB
     public static final ParseField NUMBER_OF_FRAGMENTS_FIELD = new ParseField("number_of_fragments");
     public static final ParseField ENCODER_FIELD = new ParseField("encoder");
     public static final ParseField REQUIRE_FIELD_MATCH_FIELD = new ParseField("require_field_match");
+    public static final ParseField BOUNDARY_SCANNER_FIELD = new ParseField("boundary_scanner");
     public static final ParseField BOUNDARY_MAX_SCAN_FIELD = new ParseField("boundary_max_scan");
     public static final ParseField BOUNDARY_CHARS_FIELD = new ParseField("boundary_chars");
+    public static final ParseField BOUNDARY_SCANNER_LOCALE_FIELD = new ParseField("boundary_scanner_locale");
     public static final ParseField TYPE_FIELD = new ParseField("type");
     public static final ParseField FRAGMENTER_FIELD = new ParseField("fragmenter");
     public static final ParseField NO_MATCH_SIZE_FIELD = new ParseField("no_match_size");
@@ -88,10 +93,14 @@ public abstract class AbstractHighlighterBuilder<HB extends AbstractHighlighterB
 
     protected Boolean forceSource;
 
+    protected BoundaryScannerType boundaryScannerType;
+
     protected Integer boundaryMaxScan;
 
     protected char[] boundaryChars;
 
+    protected Locale boundaryScannerLocale;
+
     protected Integer noMatchSize;
 
     protected Integer phraseLimit;
@@ -119,10 +128,18 @@ protected AbstractHighlighterBuilder(StreamInput in) throws IOException {
         order(in.readOptionalWriteable(Order::readFromStream));
         highlightFilter(in.readOptionalBoolean());
         forceSource(in.readOptionalBoolean());
+        if (in.getVersion().onOrAfter(Version.V_5_4_0)) {
+            boundaryScannerType(in.readOptionalWriteable(BoundaryScannerType::readFromStream));
+        }
         boundaryMaxScan(in.readOptionalVInt());
         if (in.readBoolean()) {
             boundaryChars(in.readString().toCharArray());
         }
+        if (in.getVersion().onOrAfter(Version.V_5_4_0)) {
+            if (in.readBoolean()) {
+                boundaryScannerLocale(in.readString());
+            }
+        }
         noMatchSize(in.readOptionalVInt());
         phraseLimit(in.readOptionalVInt());
         if (in.readBoolean()) {
@@ -150,12 +167,22 @@ public final void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalWriteable(order);
         out.writeOptionalBoolean(highlightFilter);
         out.writeOptionalBoolean(forceSource);
+        if (out.getVersion().onOrAfter(Version.V_5_4_0)) {
+            out.writeOptionalWriteable(boundaryScannerType);
+        }
         out.writeOptionalVInt(boundaryMaxScan);
         boolean hasBounaryChars = boundaryChars != null;
         out.writeBoolean(hasBounaryChars);
         if (hasBounaryChars) {
             out.writeString(String.valueOf(boundaryChars));
         }
+        if (out.getVersion().onOrAfter(Version.V_5_4_0)) {
+            boolean hasBoundaryScannerLocale = boundaryScannerLocale != null;
+            out.writeBoolean(hasBoundaryScannerLocale);
+            if (hasBoundaryScannerLocale) {
+                out.writeString(boundaryScannerLocale.toLanguageTag());
+            }
+        }
         out.writeOptionalVInt(noMatchSize);
         out.writeOptionalVInt(phraseLimit);
         boolean hasOptions = options != null;
@@ -331,6 +358,33 @@ public Boolean highlightFilter() {
         return this.highlightFilter;
     }
 
+    /**
+     * When using the highlighterType <tt>fvh</tt> this setting
+     * controls which scanner to use for fragment boundaries, and defaults to "simple".
+     */
+    @SuppressWarnings("unchecked")
+    public HB boundaryScannerType(String boundaryScannerType) {
+        this.boundaryScannerType = BoundaryScannerType.fromString(boundaryScannerType);
+        return (HB) this;
+    }
+
+    /**
+     * When using the highlighterType <tt>fvh</tt> this setting
+     * controls which scanner to use for fragment boundaries, and defaults to "simple".
+     */
+    @SuppressWarnings("unchecked")
+    public HB boundaryScannerType(BoundaryScannerType boundaryScannerType) {
+        this.boundaryScannerType = boundaryScannerType;
+        return (HB) this;
+    }
+
+    /**
+     * @return the value set by {@link #boundaryScannerType(String)}
+     */
+    public BoundaryScannerType boundaryScannerType() {
+        return this.boundaryScannerType;
+    }
+
     /**
      * When using the highlighterType <tt>fvh</tt> this setting
      * controls how far to look for boundary characters, and defaults to 20.
@@ -366,6 +420,25 @@ public char[] boundaryChars() {
         return this.boundaryChars;
     }
 
+    /**
+     * When using the highlighterType <tt>fvh</tt> and boundaryScannerType <tt>break_iterator</tt>, this setting
+     * controls the locale to use by the BreakIterator, defaults to "root".
+     */
+    @SuppressWarnings("unchecked")
+    public HB boundaryScannerLocale(String boundaryScannerLocale) {
+        if (boundaryScannerLocale != null) {
+            this.boundaryScannerLocale = Locale.forLanguageTag(boundaryScannerLocale);
+        }
+        return (HB) this;
+    }
+
+    /**
+     * @return the value set by {@link #boundaryScannerLocale(String)}
+     */
+    public Locale boundaryScannerLocale() {
+        return this.boundaryScannerLocale;
+    }
+
     /**
      * Allows to set custom options for custom highlighters.
      */
@@ -491,12 +564,18 @@ void commonOptionsToXContent(XContentBuilder builder) throws IOException {
         if (highlightFilter != null) {
             builder.field(HIGHLIGHT_FILTER_FIELD.getPreferredName(), highlightFilter);
         }
+        if (boundaryScannerType != null) {
+            builder.field(BOUNDARY_SCANNER_FIELD.getPreferredName(), boundaryScannerType.name());
+        }
         if (boundaryMaxScan != null) {
             builder.field(BOUNDARY_MAX_SCAN_FIELD.getPreferredName(), boundaryMaxScan);
         }
         if (boundaryChars != null) {
             builder.field(BOUNDARY_CHARS_FIELD.getPreferredName(), new String(boundaryChars));
         }
+        if (boundaryScannerLocale != null) {
+            builder.field(BOUNDARY_SCANNER_LOCALE_FIELD.getPreferredName(), boundaryScannerLocale.toLanguageTag());
+        }
         if (options != null && options.size() > 0) {
             builder.field(OPTIONS_FIELD.getPreferredName(), options);
         }
@@ -523,8 +602,10 @@ static <HB extends AbstractHighlighterBuilder<HB>> BiFunction<QueryParseContext,
         parser.declareInt(HB::fragmentSize, FRAGMENT_SIZE_FIELD);
         parser.declareInt(HB::numOfFragments, NUMBER_OF_FRAGMENTS_FIELD);
         parser.declareBoolean(HB::requireFieldMatch, REQUIRE_FIELD_MATCH_FIELD);
+        parser.declareString(HB::boundaryScannerType, BOUNDARY_SCANNER_FIELD);
         parser.declareInt(HB::boundaryMaxScan, BOUNDARY_MAX_SCAN_FIELD);
         parser.declareString((HB hb, String bc) -> hb.boundaryChars(bc.toCharArray()) , BOUNDARY_CHARS_FIELD);
+        parser.declareString(HB::boundaryScannerLocale, BOUNDARY_SCANNER_LOCALE_FIELD);
         parser.declareString(HB::highlighterType, TYPE_FIELD);
         parser.declareString(HB::fragmenter, FRAGMENTER_FIELD);
         parser.declareInt(HB::noMatchSize, NO_MATCH_SIZE_FIELD);
@@ -562,8 +643,8 @@ static <HB extends AbstractHighlighterBuilder<HB>> BiFunction<QueryParseContext,
     public final int hashCode() {
         return Objects.hash(getClass(), Arrays.hashCode(preTags), Arrays.hashCode(postTags), fragmentSize,
                 numOfFragments, highlighterType, fragmenter, highlightQuery, order, highlightFilter,
-                forceSource, boundaryMaxScan, Arrays.hashCode(boundaryChars), noMatchSize,
-                phraseLimit, options, requireFieldMatch, doHashCode());
+                forceSource, boundaryScannerType, boundaryMaxScan, Arrays.hashCode(boundaryChars), boundaryScannerLocale,
+                noMatchSize, phraseLimit, options, requireFieldMatch, doHashCode());
     }
 
     /**
@@ -591,8 +672,10 @@ public final boolean equals(Object obj) {
                Objects.equals(order, other.order) &&
                Objects.equals(highlightFilter, other.highlightFilter) &&
                Objects.equals(forceSource, other.forceSource) &&
+               Objects.equals(boundaryScannerType, other.boundaryScannerType) &&
                Objects.equals(boundaryMaxScan, other.boundaryMaxScan) &&
                Arrays.equals(boundaryChars, other.boundaryChars) &&
+               Objects.equals(boundaryScannerLocale, other.boundaryScannerLocale) &&
                Objects.equals(noMatchSize, other.noMatchSize) &&
                Objects.equals(phraseLimit, other.phraseLimit) &&
                Objects.equals(options, other.options) &&
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/CustomQueryScorer.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/CustomQueryScorer.java
index b62d28f8ab4ef..463200081f0a0 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/CustomQueryScorer.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/CustomQueryScorer.java
@@ -24,10 +24,9 @@
 import org.apache.lucene.search.highlight.QueryScorer;
 import org.apache.lucene.search.highlight.WeightedSpanTerm;
 import org.apache.lucene.search.highlight.WeightedSpanTermExtractor;
-import org.apache.lucene.spatial.geopoint.search.GeoPointInBBoxQuery;
 import org.elasticsearch.common.lucene.search.function.FiltersFunctionScoreQuery;
 import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery;
-import org.elasticsearch.index.query.HasChildQueryBuilder;
+import org.elasticsearch.index.search.ESToParentBlockJoinQuery;
 
 import java.io.IOException;
 import java.util.Map;
@@ -67,39 +66,43 @@ protected WeightedSpanTermExtractor newTermExtractor(String defaultField) {
 
     private static class CustomWeightedSpanTermExtractor extends WeightedSpanTermExtractor {
 
-        public CustomWeightedSpanTermExtractor() {
+        CustomWeightedSpanTermExtractor() {
             super();
         }
 
-        public CustomWeightedSpanTermExtractor(String defaultField) {
+        CustomWeightedSpanTermExtractor(String defaultField) {
             super(defaultField);
         }
 
         @Override
         protected void extractUnknownQuery(Query query,
                                            Map<String, WeightedSpanTerm> terms) throws IOException {
-            if (query instanceof FunctionScoreQuery) {
-                query = ((FunctionScoreQuery) query).getSubQuery();
-                extract(query, 1F, terms);
-            } else if (query instanceof FiltersFunctionScoreQuery) {
-                query = ((FiltersFunctionScoreQuery) query).getSubQuery();
-                extract(query, 1F, terms);
-            } else if (terms.isEmpty()) {
+            if (terms.isEmpty()) {
                 extractWeightedTerms(terms, query, 1F);
             }
         }
 
         protected void extract(Query query, float boost, Map<String, WeightedSpanTerm> terms) throws IOException {
-            if (query instanceof GeoPointInBBoxQuery) {
-                // skip all geo queries, see https://issues.apache.org/jira/browse/LUCENE-7293 and
-                // https://github.com/elastic/elasticsearch/issues/17537
-                return;
-            } else if (query instanceof HasChildQueryBuilder.LateParsingQuery) {
+            if (isChildOrParentQuery(query.getClass())) {
                 // skip has_child or has_parent queries, see: https://github.com/elastic/elasticsearch/issues/14999
                 return;
+            } else if (query instanceof FunctionScoreQuery) {
+                super.extract(((FunctionScoreQuery) query).getSubQuery(), boost, terms);
+            } else if (query instanceof FiltersFunctionScoreQuery) {
+                super.extract(((FiltersFunctionScoreQuery) query).getSubQuery(), boost, terms);
+            } else if (query instanceof ESToParentBlockJoinQuery) {
+                super.extract(((ESToParentBlockJoinQuery) query).getChildQuery(), boost, terms);
+            } else {
+                super.extract(query, boost, terms);
             }
+        }
 
-            super.extract(query, boost, terms);
+        /**
+         * Workaround to detect parent/child query
+         */
+        private static final String PARENT_CHILD_QUERY_NAME = "HasChildQueryBuilder$LateParsingQuery";
+        private static boolean isChildOrParentQuery(Class<?> clazz) {
+            return clazz.getName().endsWith(PARENT_CHILD_QUERY_NAME);
         }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FastVectorHighlighter.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FastVectorHighlighter.java
index 873567de44e51..1a2f3cbf78a78 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FastVectorHighlighter.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FastVectorHighlighter.java
@@ -21,6 +21,7 @@
 import org.apache.lucene.search.highlight.Encoder;
 import org.apache.lucene.search.vectorhighlight.BaseFragmentsBuilder;
 import org.apache.lucene.search.vectorhighlight.BoundaryScanner;
+import org.apache.lucene.search.vectorhighlight.BreakIteratorBoundaryScanner;
 import org.apache.lucene.search.vectorhighlight.CustomFieldQuery;
 import org.apache.lucene.search.vectorhighlight.FieldFragList;
 import org.apache.lucene.search.vectorhighlight.FieldPhraseList.WeightedPhraseInfo;
@@ -32,15 +33,20 @@
 import org.apache.lucene.search.vectorhighlight.SimpleFieldFragList;
 import org.apache.lucene.search.vectorhighlight.SimpleFragListBuilder;
 import org.apache.lucene.search.vectorhighlight.SingleFragListBuilder;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.text.Text;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.search.fetch.FetchPhaseExecutionException;
 import org.elasticsearch.search.fetch.FetchSubPhase;
+import org.elasticsearch.search.fetch.subphase.highlight.SearchContextHighlight.Field;
+import org.elasticsearch.search.fetch.subphase.highlight.SearchContextHighlight.FieldOptions;
 import org.elasticsearch.search.internal.SearchContext;
 
+import java.text.BreakIterator;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.Locale;
 import java.util.Map;
 
 /**
@@ -48,13 +54,20 @@
  */
 public class FastVectorHighlighter implements Highlighter {
 
-    private static final SimpleBoundaryScanner DEFAULT_BOUNDARY_SCANNER = new SimpleBoundaryScanner();
+    private static final BoundaryScanner DEFAULT_SIMPLE_BOUNDARY_SCANNER = new SimpleBoundaryScanner();
+    private static final BoundaryScanner DEFAULT_SENTENCE_BOUNDARY_SCANNER =
+        new BreakIteratorBoundaryScanner(BreakIterator.getSentenceInstance(Locale.ROOT));
+    private static final BoundaryScanner DEFAULT_WORD_BOUNDARY_SCANNER =
+        new BreakIteratorBoundaryScanner(BreakIterator.getWordInstance(Locale.ROOT));
+
+    public static final Setting<Boolean> SETTING_TV_HIGHLIGHT_MULTI_VALUE =
+        Setting.boolSetting("search.highlight.term_vector_multi_value", true, Setting.Property.NodeScope);
 
     private static final String CACHE_KEY = "highlight-fsv";
     private final Boolean termVectorMultiValue;
 
     public FastVectorHighlighter(Settings settings) {
-        this.termVectorMultiValue = settings.getAsBoolean("search.highlight.term_vector_multi_value", true);
+        this.termVectorMultiValue = SETTING_TV_HIGHLIGHT_MULTI_VALUE.get(settings);
     }
 
     @Override
@@ -65,11 +78,12 @@ public HighlightField highlight(HighlighterContext highlighterContext) {
         FieldMapper mapper = highlighterContext.mapper;
 
         if (canHighlight(mapper) == false) {
-            throw new IllegalArgumentException("the field [" + highlighterContext.fieldName
-                    + "] should be indexed with term vector with position offsets to be used with fast vector highlighter");
+            throw new IllegalArgumentException("the field [" + highlighterContext.fieldName +
+                "] should be indexed with term vector with position offsets to be used with fast vector highlighter");
         }
 
-        Encoder encoder = field.fieldOptions().encoder().equals("html") ? HighlightUtils.Encoders.HTML : HighlightUtils.Encoders.DEFAULT;
+        Encoder encoder = field.fieldOptions().encoder().equals("html") ?
+            HighlightUtils.Encoders.HTML : HighlightUtils.Encoders.DEFAULT;
 
         if (!hitContext.cache().containsKey(CACHE_KEY)) {
             hitContext.cache().put(CACHE_KEY, new HighlighterEntry());
@@ -77,40 +91,12 @@ public HighlightField highlight(HighlighterContext highlighterContext) {
         HighlighterEntry cache = (HighlighterEntry) hitContext.cache().get(CACHE_KEY);
 
         try {
-            FieldQuery fieldQuery;
-            if (field.fieldOptions().requireFieldMatch()) {
-                if (cache.fieldMatchFieldQuery == null) {
-                    /*
-                     * we use top level reader to rewrite the query against all readers, with use caching it across hits (and across
-                     * readers...)
-                     */
-                    cache.fieldMatchFieldQuery = new CustomFieldQuery(highlighterContext.query, hitContext.topLevelReader(),
-                            true, field.fieldOptions().requireFieldMatch());
-                }
-                fieldQuery = cache.fieldMatchFieldQuery;
-            } else {
-                if (cache.noFieldMatchFieldQuery == null) {
-                    /*
-                     * we use top level reader to rewrite the query against all readers, with use caching it across hits (and across
-                     * readers...)
-                     */
-                    cache.noFieldMatchFieldQuery = new CustomFieldQuery(highlighterContext.query, hitContext.topLevelReader(),
-                            true, field.fieldOptions().requireFieldMatch());
-                }
-                fieldQuery = cache.noFieldMatchFieldQuery;
-            }
-
             MapperHighlightEntry entry = cache.mappers.get(mapper);
             if (entry == null) {
                 FragListBuilder fragListBuilder;
                 BaseFragmentsBuilder fragmentsBuilder;
 
-                BoundaryScanner boundaryScanner = DEFAULT_BOUNDARY_SCANNER;
-                if (field.fieldOptions().boundaryMaxScan() != SimpleBoundaryScanner.DEFAULT_MAX_SCAN
-                        || field.fieldOptions().boundaryChars() != SimpleBoundaryScanner.DEFAULT_BOUNDARY_CHARS) {
-                    boundaryScanner = new SimpleBoundaryScanner(field.fieldOptions().boundaryMaxScan(),
-                            field.fieldOptions().boundaryChars());
-                }
+                final BoundaryScanner boundaryScanner = getBoundaryScanner(field);
                 boolean forceSource = context.highlight().forceSource(field);
                 if (field.fieldOptions().numberOfFragments() == 0) {
                     fragListBuilder = new SingleFragListBuilder();
@@ -124,7 +110,7 @@ public HighlightField highlight(HighlighterContext highlighterContext) {
                     }
                 } else {
                     fragListBuilder = field.fieldOptions().fragmentOffset() == -1 ?
-                            new SimpleFragListBuilder() : new SimpleFragListBuilder(field.fieldOptions().fragmentOffset());
+                        new SimpleFragListBuilder() : new SimpleFragListBuilder(field.fieldOptions().fragmentOffset());
                     if (field.fieldOptions().scoreOrdered()) {
                         if (!forceSource && mapper.fieldType().stored()) {
                             fragmentsBuilder = new ScoreOrderFragmentsBuilder(field.fieldOptions().preTags(),
@@ -138,24 +124,46 @@ public HighlightField highlight(HighlighterContext highlighterContext) {
                             fragmentsBuilder = new SimpleFragmentsBuilder(mapper, field.fieldOptions().preTags(),
                                     field.fieldOptions().postTags(), boundaryScanner);
                         } else {
-                            fragmentsBuilder = new SourceSimpleFragmentsBuilder(mapper, context, field.fieldOptions().preTags(),
+                            fragmentsBuilder =
+                                new SourceSimpleFragmentsBuilder(mapper, context, field.fieldOptions().preTags(),
                                     field.fieldOptions().postTags(), boundaryScanner);
                         }
                     }
                 }
                 fragmentsBuilder.setDiscreteMultiValueHighlighting(termVectorMultiValue);
                 entry = new MapperHighlightEntry();
+                if (field.fieldOptions().requireFieldMatch()) {
+                    /**
+                     * we use top level reader to rewrite the query against all readers,
+                     * with use caching it across hits (and across readers...)
+                     */
+                    entry.fieldMatchFieldQuery = new CustomFieldQuery(highlighterContext.query,
+                        hitContext.topLevelReader(), true, field.fieldOptions().requireFieldMatch());
+                } else {
+                    /**
+                     * we use top level reader to rewrite the query against all readers,
+                     * with use caching it across hits (and across readers...)
+                     */
+                    entry.noFieldMatchFieldQuery = new CustomFieldQuery(highlighterContext.query,
+                        hitContext.topLevelReader(), true, field.fieldOptions().requireFieldMatch());
+                }
                 entry.fragListBuilder = fragListBuilder;
                 entry.fragmentsBuilder = fragmentsBuilder;
                 if (cache.fvh == null) {
                     // parameters to FVH are not requires since:
-                    // first two booleans are not relevant since they are set on the CustomFieldQuery (phrase and fieldMatch)
-                    // fragment builders are used explicitly
+                    // first two booleans are not relevant since they are set on the CustomFieldQuery
+                    // (phrase and fieldMatch) fragment builders are used explicitly
                     cache.fvh = new org.apache.lucene.search.vectorhighlight.FastVectorHighlighter();
                 }
                 CustomFieldQuery.highlightFilters.set(field.fieldOptions().highlightFilter());
                 cache.mappers.put(mapper, entry);
             }
+            final FieldQuery fieldQuery;
+            if (field.fieldOptions().requireFieldMatch()) {
+                fieldQuery = entry.fieldMatchFieldQuery;
+            } else {
+                fieldQuery = entry.noFieldMatchFieldQuery;
+            }
             cache.fvh.setPhraseLimit(field.fieldOptions().phraseLimit());
 
             String[] fragments;
@@ -168,13 +176,14 @@ public HighlightField highlight(HighlighterContext highlighterContext) {
             // we highlight against the low level reader and docId, because if we load source, we want to reuse it if possible
             // Only send matched fields if they were requested to save time.
             if (field.fieldOptions().matchedFields() != null && !field.fieldOptions().matchedFields().isEmpty()) {
-                fragments = cache.fvh.getBestFragments(fieldQuery, hitContext.reader(), hitContext.docId(), mapper.fieldType().name(),
-                        field.fieldOptions().matchedFields(), fragmentCharSize, numberOfFragments, entry.fragListBuilder,
-                        entry.fragmentsBuilder, field.fieldOptions().preTags(), field.fieldOptions().postTags(), encoder);
+                fragments = cache.fvh.getBestFragments(fieldQuery, hitContext.reader(), hitContext.docId(),
+                    mapper.fieldType().name(), field.fieldOptions().matchedFields(), fragmentCharSize,
+                    numberOfFragments, entry.fragListBuilder, entry.fragmentsBuilder, field.fieldOptions().preTags(),
+                    field.fieldOptions().postTags(), encoder);
             } else {
-                fragments = cache.fvh.getBestFragments(fieldQuery, hitContext.reader(), hitContext.docId(), mapper.fieldType().name(),
-                        fragmentCharSize, numberOfFragments, entry.fragListBuilder, entry.fragmentsBuilder, field.fieldOptions().preTags(),
-                        field.fieldOptions().postTags(), encoder);
+                fragments = cache.fvh.getBestFragments(fieldQuery, hitContext.reader(), hitContext.docId(),
+                    mapper.fieldType().name(), fragmentCharSize, numberOfFragments, entry.fragListBuilder,
+                    entry.fragmentsBuilder, field.fieldOptions().preTags(), field.fieldOptions().postTags(), encoder);
             }
 
             if (fragments != null && fragments.length > 0) {
@@ -183,11 +192,13 @@ public HighlightField highlight(HighlighterContext highlighterContext) {
 
             int noMatchSize = highlighterContext.field.fieldOptions().noMatchSize();
             if (noMatchSize > 0) {
-                // Essentially we just request that a fragment is built from 0 to noMatchSize using the normal fragmentsBuilder
+                // Essentially we just request that a fragment is built from 0 to noMatchSize using
+                // the normal fragmentsBuilder
                 FieldFragList fieldFragList = new SimpleFieldFragList(-1 /*ignored*/);
                 fieldFragList.add(0, noMatchSize, Collections.<WeightedPhraseInfo>emptyList());
-                fragments = entry.fragmentsBuilder.createFragments(hitContext.reader(), hitContext.docId(), mapper.fieldType().name(),
-                        fieldFragList, 1, field.fieldOptions().preTags(), field.fieldOptions().postTags(), encoder);
+                fragments = entry.fragmentsBuilder.createFragments(hitContext.reader(), hitContext.docId(),
+                    mapper.fieldType().name(), fieldFragList, 1, field.fieldOptions().preTags(),
+                    field.fieldOptions().postTags(), encoder);
                 if (fragments != null && fragments.length > 0) {
                     return new HighlightField(highlighterContext.fieldName, Text.convertFromStringArray(fragments));
                 }
@@ -196,7 +207,8 @@ public HighlightField highlight(HighlighterContext highlighterContext) {
             return null;
 
         } catch (Exception e) {
-            throw new FetchPhaseExecutionException(context, "Failed to highlight field [" + highlighterContext.fieldName + "]", e);
+            throw new FetchPhaseExecutionException(context,
+                "Failed to highlight field [" + highlighterContext.fieldName + "]", e);
         }
     }
 
@@ -206,15 +218,45 @@ public boolean canHighlight(FieldMapper fieldMapper) {
                 && fieldMapper.fieldType().storeTermVectorPositions();
     }
 
+    private static BoundaryScanner getBoundaryScanner(Field field) {
+        final FieldOptions fieldOptions = field.fieldOptions();
+        final Locale boundaryScannerLocale =
+            fieldOptions.boundaryScannerLocale() != null ? fieldOptions.boundaryScannerLocale() :
+                Locale.ROOT;
+        final HighlightBuilder.BoundaryScannerType type =
+            fieldOptions.boundaryScannerType()  != null ? fieldOptions.boundaryScannerType() :
+                HighlightBuilder.BoundaryScannerType.CHARS;
+        switch(type) {
+            case SENTENCE:
+                if (boundaryScannerLocale != null) {
+                    return new BreakIteratorBoundaryScanner(BreakIterator.getSentenceInstance(boundaryScannerLocale));
+                }
+                return DEFAULT_SENTENCE_BOUNDARY_SCANNER;
+            case WORD:
+                if (boundaryScannerLocale != null) {
+                    return new BreakIteratorBoundaryScanner(BreakIterator.getWordInstance(boundaryScannerLocale));
+                }
+                return DEFAULT_WORD_BOUNDARY_SCANNER;
+            case CHARS:
+                if (fieldOptions.boundaryMaxScan() != SimpleBoundaryScanner.DEFAULT_MAX_SCAN
+                    || fieldOptions.boundaryChars() != SimpleBoundaryScanner.DEFAULT_BOUNDARY_CHARS) {
+                    return new SimpleBoundaryScanner(fieldOptions.boundaryMaxScan(), fieldOptions.boundaryChars());
+                }
+                return DEFAULT_SIMPLE_BOUNDARY_SCANNER;
+            default:
+                throw new IllegalArgumentException("Invalid boundary scanner type: " + type.toString());
+        }
+    }
+
     private class MapperHighlightEntry {
         public FragListBuilder fragListBuilder;
         public FragmentsBuilder fragmentsBuilder;
+        public FieldQuery noFieldMatchFieldQuery;
+        public FieldQuery fieldMatchFieldQuery;
     }
 
     private class HighlighterEntry {
         public org.apache.lucene.search.vectorhighlight.FastVectorHighlighter fvh;
-        public FieldQuery noFieldMatchFieldQuery;
-        public FieldQuery fieldMatchFieldQuery;
         public Map<FieldMapper, MapperHighlightEntry> mappers = new HashMap<>();
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilder.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilder.java
index fe4587826c7b8..5e49aa7395d57 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilder.java
@@ -97,7 +97,7 @@ public class HighlightBuilder extends AbstractHighlighterBuilder<HighlightBuilde
             .forceSource(DEFAULT_FORCE_SOURCE).fragmentCharSize(DEFAULT_FRAGMENT_CHAR_SIZE)
             .numberOfFragments(DEFAULT_NUMBER_OF_FRAGMENTS).encoder(DEFAULT_ENCODER)
             .boundaryMaxScan(SimpleBoundaryScanner.DEFAULT_MAX_SCAN).boundaryChars(SimpleBoundaryScanner.DEFAULT_BOUNDARY_CHARS)
-            .noMatchSize(DEFAULT_NO_MATCH_SIZE).phraseLimit(DEFAULT_PHRASE_LIMIT).build();
+            .boundaryScannerLocale(Locale.ROOT).noMatchSize(DEFAULT_NO_MATCH_SIZE).phraseLimit(DEFAULT_PHRASE_LIMIT).build();
 
     private final List<Field> fields = new ArrayList<>();
 
@@ -327,12 +327,18 @@ private static void transferOptions(AbstractHighlighterBuilder highlighterBuilde
         if (highlighterBuilder.requireFieldMatch != null) {
             targetOptionsBuilder.requireFieldMatch(highlighterBuilder.requireFieldMatch);
         }
+        if (highlighterBuilder.boundaryScannerType != null) {
+            targetOptionsBuilder.boundaryScannerType(highlighterBuilder.boundaryScannerType);
+        }
         if (highlighterBuilder.boundaryMaxScan != null) {
             targetOptionsBuilder.boundaryMaxScan(highlighterBuilder.boundaryMaxScan);
         }
         if (highlighterBuilder.boundaryChars != null) {
             targetOptionsBuilder.boundaryChars(convertCharArray(highlighterBuilder.boundaryChars));
         }
+        if (highlighterBuilder.boundaryScannerLocale != null) {
+            targetOptionsBuilder.boundaryScannerLocale(highlighterBuilder.boundaryScannerLocale);
+        }
         if (highlighterBuilder.highlighterType != null) {
             targetOptionsBuilder.highlighterType(highlighterBuilder.highlighterType);
         }
@@ -476,7 +482,7 @@ public void innerXContent(XContentBuilder builder) throws IOException {
                 builder.field(FRAGMENT_OFFSET_FIELD.getPreferredName(), fragmentOffset);
             }
             if (matchedFields != null) {
-                builder.field(MATCHED_FIELDS_FIELD.getPreferredName(), matchedFields);
+                builder.array(MATCHED_FIELDS_FIELD.getPreferredName(), matchedFields);
             }
             builder.endObject();
         }
@@ -498,16 +504,12 @@ public enum Order implements Writeable {
         NONE, SCORE;
 
         public static Order readFromStream(StreamInput in) throws IOException {
-            int ordinal = in.readVInt();
-            if (ordinal < 0 || ordinal >= values().length) {
-                throw new IOException("Unknown Order ordinal [" + ordinal + "]");
-            }
-            return values()[ordinal];
+            return in.readEnum(Order.class);
         }
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
-            out.writeVInt(this.ordinal());
+            out.writeEnum(this);
         }
 
         public static Order fromString(String order) {
@@ -522,4 +524,26 @@ public String toString() {
             return name().toLowerCase(Locale.ROOT);
         }
     }
+
+    public enum BoundaryScannerType implements Writeable {
+        CHARS, WORD, SENTENCE;
+
+        public static BoundaryScannerType readFromStream(StreamInput in) throws IOException {
+            return in.readEnum(BoundaryScannerType.class);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeEnum(this);
+        }
+
+        public static BoundaryScannerType fromString(String boundaryScannerType) {
+            return valueOf(boundaryScannerType.toUpperCase(Locale.ROOT));
+        }
+
+        @Override
+        public String toString() {
+            return name().toLowerCase(Locale.ROOT);
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightField.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightField.java
index 91fde32c8885d..7ff2147868a5b 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightField.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightField.java
@@ -19,18 +19,27 @@
 
 package org.elasticsearch.search.fetch.subphase.highlight;
 
+import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
 import org.elasticsearch.common.text.Text;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
 
 /**
  * A field highlighted with its highlighted fragments.
  */
-public class HighlightField implements Streamable {
+public class HighlightField implements ToXContent, Streamable {
 
     private String name;
 
@@ -40,7 +49,7 @@ public class HighlightField implements Streamable {
     }
 
     public HighlightField(String name, Text[] fragments) {
-        this.name = name;
+        this.name = Objects.requireNonNull(name, "missing highlight field name");
         this.fragments = fragments;
     }
 
@@ -112,4 +121,57 @@ public void writeTo(StreamOutput out) throws IOException {
             }
         }
     }
+
+    public static HighlightField fromXContent(XContentParser parser) throws IOException {
+        ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser::getTokenLocation);
+        String fieldName = parser.currentName();
+        Text[] fragments = null;
+        XContentParser.Token token = parser.nextToken();
+        if (token == XContentParser.Token.START_ARRAY) {
+            List<Text> values = new ArrayList<>();
+            while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
+                values.add(new Text(parser.text()));
+            }
+            fragments = values.toArray(new Text[values.size()]);
+        } else if (token == XContentParser.Token.VALUE_NULL) {
+            fragments = null;
+        } else {
+            throw new ParsingException(parser.getTokenLocation(),
+                    "unexpected token type [" + token + "]");
+        }
+        return new HighlightField(fieldName, fragments);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.field(name);
+        if (fragments == null) {
+            builder.nullValue();
+        } else {
+            builder.startArray();
+            for (Text fragment : fragments) {
+                builder.value(fragment);
+            }
+            builder.endArray();
+        }
+        return builder;
+    }
+
+    @Override
+    public final boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+        HighlightField other = (HighlightField) obj;
+        return Objects.equals(name, other.name) && Arrays.equals(fragments, other.fragments);
+    }
+
+    @Override
+    public final int hashCode() {
+        return Objects.hash(name, Arrays.hashCode(fragments));
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightPhase.java
index 84890857c793f..cef765862f5e7 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightPhase.java
@@ -29,6 +29,7 @@
 import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.index.mapper.StringFieldMapper;
 import org.elasticsearch.index.mapper.TextFieldMapper;
+import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.fetch.FetchSubPhase;
 import org.elasticsearch.search.internal.SearchContext;
 
@@ -68,13 +69,13 @@ public void hitExecute(SearchContext context, HitContext hitContext) {
                 SourceFieldMapper sourceFieldMapper = context.mapperService().documentMapper(hitContext.hit().type()).sourceMapper();
                 if (!sourceFieldMapper.enabled()) {
                     throw new IllegalArgumentException("source is forced for fields " +  fieldNamesToHighlight
-                            + " but type [" + hitContext.hit().type() + "] has disabled _source");
+                        + " but type [" + hitContext.hit().type() + "] has disabled _source");
                 }
             }
 
             boolean fieldNameContainsWildcards = field.field().contains("*");
             for (String fieldName : fieldNamesToHighlight) {
-                FieldMapper fieldMapper = getMapperForField(fieldName, context, hitContext);
+                FieldMapper fieldMapper = getMapperForField(fieldName, context, hitContext.hit());
                 if (fieldMapper == null) {
                     continue;
                 }
@@ -107,7 +108,7 @@ public void hitExecute(SearchContext context, HitContext hitContext) {
                 Highlighter highlighter = highlighters.get(highlighterType);
                 if (highlighter == null) {
                     throw new IllegalArgumentException("unknown highlighter type [" + highlighterType
-                            + "] for the field [" + fieldName + "]");
+                        + "] for the field [" + fieldName + "]");
                 }
 
                 Query highlightQuery = field.fieldOptions().highlightQuery();
@@ -115,7 +116,7 @@ public void hitExecute(SearchContext context, HitContext hitContext) {
                     highlightQuery = context.parsedQuery().query();
                 }
                 HighlighterContext highlighterContext = new HighlighterContext(fieldName, field, fieldMapper, context,
-                        hitContext, highlightQuery);
+                    hitContext, highlightQuery);
 
                 if ((highlighter.canHighlight(fieldMapper) == false) && fieldNameContainsWildcards) {
                     // if several fieldnames matched the wildcard then we want to skip those that we cannot highlight
@@ -130,9 +131,85 @@ public void hitExecute(SearchContext context, HitContext hitContext) {
         hitContext.hit().highlightFields(highlightFields);
     }
 
-    private FieldMapper getMapperForField(String fieldName, SearchContext searchContext, HitContext hitContext) {
-        DocumentMapper documentMapper = searchContext.mapperService().documentMapper(hitContext.hit().type());
+    @Override
+    public void hitsExecute(SearchContext context, SearchHit[] hits) {
+        if (context.highlight() == null) {
+            return;
+        }
+
+        /**
+         * This is used to make sure that we log the deprecation of the {@link PostingsHighlighter}
+         * only once per request even though multiple documents/fields use the deprecated highlighter.
+         */
+        for (SearchHit hit : hits) {
+            for (SearchContextHighlight.Field field : context.highlight().fields()) {
+                for (String fieldName : getFieldsToHighlight(context, field, hit)) {
+                    FieldMapper fieldMapper = getMapperForField(fieldName, context, hit);
+                    if (fieldMapper == null) {
+                        continue;
+                    }
+
+                    String highlighterType = field.fieldOptions().highlighterType();
+                    Highlighter highlighter = getHighlighterForField(fieldMapper, highlighterType);
+                    if (highlighter instanceof PostingsHighlighter) {
+                        deprecationLogger.deprecated("[postings] highlighter is deprecated, please use [unified] instead");
+                        return;
+                    }
+                }
+            }
+        }
+    }
+
+    /**
+     * Returns the list of field names that match the provided field definition in the mapping of the {@link SearchHit}
+     */
+    private Collection<String> getFieldsToHighlight(SearchContext context, SearchContextHighlight.Field field, SearchHit hit) {
+        Collection<String> fieldNamesToHighlight;
+        if (Regex.isSimpleMatchPattern(field.field())) {
+            DocumentMapper documentMapper = context.mapperService().documentMapper(hit.type());
+            fieldNamesToHighlight = documentMapper.mappers().simpleMatchToFullName(field.field());
+        } else {
+            fieldNamesToHighlight = Collections.singletonList(field.field());
+        }
+
+        if (context.highlight().forceSource(field)) {
+            SourceFieldMapper sourceFieldMapper = context.mapperService().documentMapper(hit.type()).sourceMapper();
+            if (!sourceFieldMapper.enabled()) {
+                throw new IllegalArgumentException("source is forced for fields " + fieldNamesToHighlight
+                    + " but type [" + hit.type() + "] has disabled _source");
+            }
+        }
+        return fieldNamesToHighlight;
+    }
+
+    /**
+     * Returns the {@link FieldMapper} associated with the provided <code>fieldName</code>
+     */
+    private FieldMapper getMapperForField(String fieldName, SearchContext searchContext, SearchHit hit) {
+        DocumentMapper documentMapper = searchContext.mapperService().documentMapper(hit.getType());
         // TODO: no need to lookup the doc mapper with unambiguous field names? just look at the mapper service
         return documentMapper.mappers().smartNameFieldMapper(fieldName);
     }
+
+    /**
+     * Returns the {@link Highlighter} for the given {@link FieldMapper}
+     */
+    private Highlighter getHighlighterForField(FieldMapper fieldMapper, String name) {
+        if (name == null) {
+            for (String highlighterCandidate : STANDARD_HIGHLIGHTERS_BY_PRECEDENCE) {
+                if (highlighters.get(highlighterCandidate).canHighlight(fieldMapper)) {
+                    name = highlighterCandidate;
+                    break;
+                }
+            }
+            assert name != null;
+        }
+        Highlighter highlighter = highlighters.get(name);
+        if (highlighter == null) {
+            throw new IllegalArgumentException("unknown highlighter type [" + name
+                + "] for the field [" + fieldMapper.name() + "]");
+        }
+        return highlighter;
+    }
 }
+
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighter.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighter.java
index 631d716f6f77a..f5b8abc8b4ae4 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighter.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighter.java
@@ -31,6 +31,7 @@
 import org.apache.lucene.search.highlight.SimpleHTMLFormatter;
 import org.apache.lucene.search.highlight.SimpleSpanFragmenter;
 import org.apache.lucene.search.highlight.TextFragment;
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefHash;
 import org.apache.lucene.util.CollectionUtil;
 import org.elasticsearch.ExceptionsHelper;
@@ -109,7 +110,12 @@ public HighlightField highlight(HighlighterContext highlighterContext) {
             textsToHighlight = HighlightUtils.loadFieldValues(field, mapper, context, hitContext);
 
             for (Object textToHighlight : textsToHighlight) {
-                String text = textToHighlight.toString();
+                String text;
+                if (textToHighlight instanceof BytesRef) {
+                    text = mapper.fieldType().valueForDisplay(textToHighlight).toString();
+                } else {
+                    text = textToHighlight.toString();
+                }
 
                 try (TokenStream tokenStream = analyzer.tokenStream(mapper.fieldType().name(), text)) {
                     if (!tokenStream.hasAttribute(CharTermAttribute.class) || !tokenStream.hasAttribute(OffsetAttribute.class)) {
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/PostingsHighlighter.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/PostingsHighlighter.java
index 7ed50c7a1ddcc..6e15fe930b547 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/PostingsHighlighter.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/PostingsHighlighter.java
@@ -25,7 +25,7 @@
 import org.apache.lucene.search.postingshighlight.CustomPassageFormatter;
 import org.apache.lucene.search.postingshighlight.CustomPostingsHighlighter;
 import org.apache.lucene.search.postingshighlight.CustomSeparatorBreakIterator;
-import org.apache.lucene.search.postingshighlight.Snippet;
+import org.apache.lucene.search.highlight.Snippet;
 import org.apache.lucene.util.CollectionUtil;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.text.Text;
@@ -44,6 +44,7 @@
 import java.util.Locale;
 import java.util.Map;
 
+@Deprecated
 public class PostingsHighlighter implements Highlighter {
 
     private static final String CACHE_KEY = "highlight-postings";
@@ -139,14 +140,14 @@ public boolean canHighlight(FieldMapper fieldMapper) {
         return fieldMapper.fieldType().indexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
     }
 
-    private static String mergeFieldValues(List<Object> fieldValues, char valuesSeparator) {
+    static String mergeFieldValues(List<Object> fieldValues, char valuesSeparator) {
         //postings highlighter accepts all values in a single string, as offsets etc. need to match with content
         //loaded from stored fields, we merge all values using a proper separator
         String rawValue = Strings.collectionToDelimitedString(fieldValues, String.valueOf(valuesSeparator));
         return rawValue.substring(0, Math.min(rawValue.length(), Integer.MAX_VALUE - 1));
     }
 
-    private static List<Snippet> filterSnippets(List<Snippet> snippets, int numberOfFragments) {
+    static List<Snippet> filterSnippets(List<Snippet> snippets, int numberOfFragments) {
 
         //We need to filter the snippets as due to no_match_size we could have
         //either highlighted snippets or non highlighted ones and we don't want to mix those up
@@ -181,11 +182,11 @@ private static List<Snippet> filterSnippets(List<Snippet> snippets, int numberOf
         return filteredSnippets;
     }
 
-    private static class HighlighterEntry {
+    static class HighlighterEntry {
         Map<FieldMapper, MapperHighlighterEntry> mappers = new HashMap<>();
     }
 
-    private static class MapperHighlighterEntry {
+    static class MapperHighlighterEntry {
         final CustomPassageFormatter passageFormatter;
 
         private MapperHighlighterEntry(CustomPassageFormatter passageFormatter) {
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/SearchContextHighlight.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/SearchContextHighlight.java
index 9f2074d741248..7e7a1c62592d1 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/SearchContextHighlight.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/SearchContextHighlight.java
@@ -20,11 +20,13 @@
 package org.elasticsearch.search.fetch.subphase.highlight;
 
 import org.apache.lucene.search.Query;
+import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder.BoundaryScannerType;
 
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 
@@ -113,10 +115,14 @@ public static class FieldOptions {
 
         private String fragmenter;
 
+        private BoundaryScannerType boundaryScannerType;
+
         private int boundaryMaxScan = -1;
 
         private Character[] boundaryChars = null;
 
+        private Locale boundaryScannerLocale;
+
         private Query highlightQuery;
 
         private int noMatchSize = -1;
@@ -171,6 +177,10 @@ public String fragmenter() {
             return fragmenter;
         }
 
+        public BoundaryScannerType boundaryScannerType() {
+            return boundaryScannerType;
+        }
+
         public int boundaryMaxScan() {
             return boundaryMaxScan;
         }
@@ -179,6 +189,10 @@ public Character[] boundaryChars() {
             return boundaryChars;
         }
 
+        public Locale boundaryScannerLocale() {
+            return boundaryScannerLocale;
+        }
+
         public Query highlightQuery() {
             return highlightQuery;
         }
@@ -263,6 +277,11 @@ Builder fragmenter(String fragmenter) {
                 return this;
             }
 
+            Builder boundaryScannerType(BoundaryScannerType boundaryScanner) {
+                fieldOptions.boundaryScannerType = boundaryScanner;
+                return this;
+            }
+
             Builder boundaryMaxScan(int boundaryMaxScan) {
                 fieldOptions.boundaryMaxScan = boundaryMaxScan;
                 return this;
@@ -273,6 +292,11 @@ Builder boundaryChars(Character[] boundaryChars) {
                 return this;
             }
 
+            Builder boundaryScannerLocale(Locale boundaryScannerLocale) {
+                fieldOptions.boundaryScannerLocale = boundaryScannerLocale;
+                return this;
+            }
+
             Builder highlightQuery(Query highlightQuery) {
                 fieldOptions.highlightQuery = highlightQuery;
                 return this;
@@ -327,12 +351,18 @@ Builder merge(FieldOptions globalOptions) {
                 if (fieldOptions.requireFieldMatch == null) {
                     fieldOptions.requireFieldMatch = globalOptions.requireFieldMatch;
                 }
+                if (fieldOptions.boundaryScannerType == null) {
+                    fieldOptions.boundaryScannerType = globalOptions.boundaryScannerType;
+                }
                 if (fieldOptions.boundaryMaxScan == -1) {
                     fieldOptions.boundaryMaxScan = globalOptions.boundaryMaxScan;
                 }
                 if (fieldOptions.boundaryChars == null && globalOptions.boundaryChars != null) {
                     fieldOptions.boundaryChars = Arrays.copyOf(globalOptions.boundaryChars, globalOptions.boundaryChars.length);
                 }
+                if (fieldOptions.boundaryScannerLocale == null) {
+                    fieldOptions.boundaryScannerLocale = globalOptions.boundaryScannerLocale;
+                }
                 if (fieldOptions.highlighterType == null) {
                     fieldOptions.highlighterType = globalOptions.highlighterType;
                 }
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/UnifiedHighlighter.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/UnifiedHighlighter.java
new file mode 100644
index 0000000000000..e5b16dacec70c
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/UnifiedHighlighter.java
@@ -0,0 +1,202 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.fetch.subphase.highlight;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.highlight.Encoder;
+import org.apache.lucene.search.highlight.Snippet;
+import org.apache.lucene.search.uhighlight.BoundedBreakIteratorScanner;
+import org.apache.lucene.search.uhighlight.CustomPassageFormatter;
+import org.apache.lucene.search.uhighlight.CustomUnifiedHighlighter;
+import org.apache.lucene.search.uhighlight.UnifiedHighlighter.OffsetSource;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.CollectionUtil;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.text.Text;
+import org.elasticsearch.index.mapper.FieldMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.search.fetch.FetchPhaseExecutionException;
+import org.elasticsearch.search.fetch.FetchSubPhase;
+import org.elasticsearch.search.internal.SearchContext;
+
+import java.io.IOException;
+import java.text.BreakIterator;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static org.apache.lucene.search.uhighlight.CustomUnifiedHighlighter.MULTIVAL_SEP_CHAR;
+import static org.elasticsearch.search.fetch.subphase.highlight.PostingsHighlighter.filterSnippets;
+import static org.elasticsearch.search.fetch.subphase.highlight.PostingsHighlighter.mergeFieldValues;
+
+public class UnifiedHighlighter implements Highlighter {
+    private static final String CACHE_KEY = "highlight-unified";
+
+    @Override
+    public boolean canHighlight(FieldMapper fieldMapper) {
+        return true;
+    }
+
+    @Override
+    public HighlightField highlight(HighlighterContext highlighterContext) {
+        FieldMapper fieldMapper = highlighterContext.mapper;
+        SearchContextHighlight.Field field = highlighterContext.field;
+        SearchContext context = highlighterContext.context;
+        FetchSubPhase.HitContext hitContext = highlighterContext.hitContext;
+
+        if (!hitContext.cache().containsKey(CACHE_KEY)) {
+            hitContext.cache().put(CACHE_KEY, new HighlighterEntry());
+        }
+
+        HighlighterEntry highlighterEntry = (HighlighterEntry) hitContext.cache().get(CACHE_KEY);
+        MapperHighlighterEntry mapperHighlighterEntry = highlighterEntry.mappers.get(fieldMapper);
+
+        if (mapperHighlighterEntry == null) {
+            Encoder encoder = field.fieldOptions().encoder().equals("html") ?
+                HighlightUtils.Encoders.HTML : HighlightUtils.Encoders.DEFAULT;
+            CustomPassageFormatter passageFormatter =
+                new CustomPassageFormatter(field.fieldOptions().preTags()[0],
+                    field.fieldOptions().postTags()[0], encoder);
+            mapperHighlighterEntry = new MapperHighlighterEntry(passageFormatter);
+        }
+
+        List<Snippet> snippets = new ArrayList<>();
+        int numberOfFragments;
+        try {
+            Analyzer analyzer =
+                context.mapperService().documentMapper(hitContext.hit().type()).mappers().indexAnalyzer();
+            List<Object> fieldValues = HighlightUtils.loadFieldValues(field, fieldMapper, context, hitContext);
+            fieldValues = fieldValues.stream().map(obj -> {
+                if (obj instanceof BytesRef) {
+                    return fieldMapper.fieldType().valueForDisplay(obj).toString();
+                } else {
+                    return obj;
+                }
+            }).collect(Collectors.toList());
+            final IndexSearcher searcher = new IndexSearcher(hitContext.reader());
+            final CustomUnifiedHighlighter highlighter;
+            final String fieldValue = mergeFieldValues(fieldValues, MULTIVAL_SEP_CHAR);
+            final OffsetSource offsetSource = getOffsetSource(fieldMapper.fieldType());
+            if (field.fieldOptions().numberOfFragments() == 0) {
+                // we use a control char to separate values, which is the only char that the custom break iterator
+                // breaks the text on, so we don't lose the distinction between the different values of a field and we
+                // get back a snippet per value
+                org.apache.lucene.search.postingshighlight.CustomSeparatorBreakIterator breakIterator =
+                    new org.apache.lucene.search.postingshighlight
+                        .CustomSeparatorBreakIterator(MULTIVAL_SEP_CHAR);
+                highlighter = new CustomUnifiedHighlighter(searcher, analyzer, offsetSource,
+                        mapperHighlighterEntry.passageFormatter, field.fieldOptions().boundaryScannerLocale(),
+                        breakIterator, fieldValue, field.fieldOptions().noMatchSize());
+                numberOfFragments = fieldValues.size(); // we are highlighting the whole content, one snippet per value
+            } else {
+                //using paragraph separator we make sure that each field value holds a discrete passage for highlighting
+                BreakIterator bi = getBreakIterator(field);
+                highlighter = new CustomUnifiedHighlighter(searcher, analyzer, offsetSource,
+                    mapperHighlighterEntry.passageFormatter, field.fieldOptions().boundaryScannerLocale(), bi,
+                    fieldValue, field.fieldOptions().noMatchSize());
+                numberOfFragments = field.fieldOptions().numberOfFragments();
+            }
+
+            if (field.fieldOptions().requireFieldMatch()) {
+                final String fieldName = highlighterContext.fieldName;
+                highlighter.setFieldMatcher((name) -> fieldName.equals(name));
+            } else {
+                highlighter.setFieldMatcher((name) -> true);
+            }
+
+            Snippet[] fieldSnippets = highlighter.highlightField(highlighterContext.fieldName,
+                highlighterContext.query, hitContext.docId(), numberOfFragments);
+            for (Snippet fieldSnippet : fieldSnippets) {
+                if (Strings.hasText(fieldSnippet.getText())) {
+                    snippets.add(fieldSnippet);
+                }
+            }
+        } catch (IOException e) {
+            throw new FetchPhaseExecutionException(context,
+                "Failed to highlight field [" + highlighterContext.fieldName + "]", e);
+        }
+
+        snippets = filterSnippets(snippets, field.fieldOptions().numberOfFragments());
+
+        if (field.fieldOptions().scoreOrdered()) {
+            //let's sort the snippets by score if needed
+            CollectionUtil.introSort(snippets, (o1, o2) -> Double.compare(o2.getScore(), o1.getScore()));
+        }
+
+        String[] fragments = new String[snippets.size()];
+        for (int i = 0; i < fragments.length; i++) {
+            fragments[i] = snippets.get(i).getText();
+        }
+
+        if (fragments.length > 0) {
+            return new HighlightField(highlighterContext.fieldName, Text.convertFromStringArray(fragments));
+        }
+        return null;
+    }
+
+    private BreakIterator getBreakIterator(SearchContextHighlight.Field field) {
+        final SearchContextHighlight.FieldOptions fieldOptions = field.fieldOptions();
+        final Locale locale =
+            fieldOptions.boundaryScannerLocale() != null ? fieldOptions.boundaryScannerLocale() :
+                Locale.ROOT;
+        final HighlightBuilder.BoundaryScannerType type =
+            fieldOptions.boundaryScannerType()  != null ? fieldOptions.boundaryScannerType() :
+                HighlightBuilder.BoundaryScannerType.SENTENCE;
+        int maxLen = fieldOptions.fragmentCharSize();
+        switch (type) {
+            case SENTENCE:
+                if (maxLen > 0) {
+                    return BoundedBreakIteratorScanner.getSentence(locale, maxLen);
+                }
+                return BreakIterator.getSentenceInstance(locale);
+            case WORD:
+                // ignore maxLen
+                return BreakIterator.getWordInstance(locale);
+            default:
+                throw new IllegalArgumentException("Invalid boundary scanner type: " + type.toString());
+        }
+    }
+
+    private OffsetSource getOffsetSource(MappedFieldType fieldType) {
+        if (fieldType.indexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) {
+            return fieldType.storeTermVectors() ? OffsetSource.POSTINGS_WITH_TERM_VECTORS : OffsetSource.POSTINGS;
+        }
+        if (fieldType.storeTermVectorOffsets()) {
+            return OffsetSource.TERM_VECTORS;
+        }
+        return OffsetSource.ANALYSIS;
+    }
+
+    private static class HighlighterEntry {
+        Map<FieldMapper, MapperHighlighterEntry> mappers = new HashMap<>();
+    }
+
+    private static class MapperHighlighterEntry {
+        final CustomPassageFormatter passageFormatter;
+
+        private MapperHighlighterEntry(CustomPassageFormatter passageFormatter) {
+            this.passageFormatter = passageFormatter;
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/internal/AliasFilter.java b/core/src/main/java/org/elasticsearch/search/internal/AliasFilter.java
new file mode 100644
index 0000000000000..ee598486acfad
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/internal/AliasFilter.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.internal;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.CheckedFunction;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryRewriteContext;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Objects;
+import java.util.Optional;
+
+/**
+ * Represents a {@link QueryBuilder} and a list of alias names that filters the builder is composed of.
+ */
+public final class AliasFilter implements Writeable {
+
+    private final String[] aliases;
+    private final QueryBuilder filter;
+    private final boolean reparseAliases;
+
+    public static final AliasFilter EMPTY = new AliasFilter(null, Strings.EMPTY_ARRAY);
+
+    public AliasFilter(QueryBuilder filter, String... aliases) {
+        this.aliases = aliases == null ? Strings.EMPTY_ARRAY : aliases;
+        this.filter = filter;
+        reparseAliases = false; // no bwc here - we only do this if we parse the filter
+    }
+
+    public AliasFilter(StreamInput input) throws IOException {
+        aliases = input.readStringArray();
+        if (input.getVersion().onOrAfter(Version.V_5_1_1)) {
+            filter = input.readOptionalNamedWriteable(QueryBuilder.class);
+            reparseAliases = false;
+        } else {
+            reparseAliases = true; // alright we read from 5.0
+            filter = null;
+        }
+    }
+
+    private QueryBuilder reparseFilter(QueryRewriteContext context) {
+        if (reparseAliases) {
+            // we are processing a filter received from a 5.0 node - we need to reparse this on the executing node
+            final IndexMetaData indexMetaData = context.getIndexSettings().getIndexMetaData();
+            /* Being static, parseAliasFilter doesn't have access to whatever guts it needs to parse a query. Instead of passing in a bunch
+             * of dependencies we pass in a function that can perform the parsing. */
+            CheckedFunction<byte[], Optional<QueryBuilder>, IOException> filterParser = bytes -> {
+                try (XContentParser parser = XContentFactory.xContent(bytes).createParser(context.getXContentRegistry(), bytes)) {
+                    return context.newParseContext(parser).parseInnerQueryBuilder();
+                }
+            };
+            return ShardSearchRequest.parseAliasFilter(filterParser, indexMetaData, aliases);
+        }
+        return filter;
+    }
+
+    AliasFilter rewrite(QueryRewriteContext context) throws IOException {
+        QueryBuilder queryBuilder = reparseFilter(context);
+        if (queryBuilder != null) {
+            return new AliasFilter(QueryBuilder.rewriteQuery(queryBuilder, context), aliases);
+        }
+        return new AliasFilter(filter, aliases);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeStringArray(aliases);
+        if (out.getVersion().onOrAfter(Version.V_5_1_1)) {
+            out.writeOptionalNamedWriteable(filter);
+        }
+    }
+
+    /**
+     * Returns the aliases patters that are used to compose the {@link QueryBuilder}
+     * returned from {@link #getQueryBuilder()}
+     */
+    public String[] getAliases() {
+        return aliases;
+    }
+
+    /**
+     * Returns the alias filter {@link QueryBuilder} or <code>null</code> if there is no such filter
+     */
+    public QueryBuilder getQueryBuilder() {
+        if (reparseAliases) {
+            // this is only for BWC since 5.0 still  only sends aliases so this must be rewritten on the executing node
+            // if we talk to an older node we also only forward/write the string array which is compatible with the consumers
+            // in 5.0 see ExplainRequest and QueryValidationRequest
+            throw new IllegalStateException("alias filter for aliases: " + Arrays.toString(aliases) + " must be rewritten first");
+        }
+        return filter;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        AliasFilter that = (AliasFilter) o;
+        return reparseAliases == that.reparseAliases &&
+            Arrays.equals(aliases, that.aliases) &&
+            Objects.equals(filter, that.filter);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(reparseAliases, Arrays.hashCode(aliases), filter);
+    }
+
+    @Override
+    public String toString() {
+        return "AliasFilter{" +
+            "aliases=" + Arrays.toString(aliases) +
+            ", filter=" + filter +
+            ", reparseAliases=" + reparseAliases +
+            '}';
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/internal/CancellableBulkScorer.java b/core/src/main/java/org/elasticsearch/search/internal/CancellableBulkScorer.java
new file mode 100644
index 0000000000000..f5eceed52ea32
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/internal/CancellableBulkScorer.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.internal;
+
+import org.apache.lucene.search.BulkScorer;
+import org.apache.lucene.search.LeafCollector;
+import org.apache.lucene.util.Bits;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * A {@link BulkScorer} wrapper that runs a {@link Runnable} on a regular basis
+ * so that the query can be interrupted.
+ */
+final class CancellableBulkScorer extends BulkScorer {
+
+    // we use the BooleanScorer window size as a base interval in order to make sure that we do not
+    // slow down boolean queries
+    private static final int INITIAL_INTERVAL = 1 << 11;
+
+    // No point in having intervals that are larger than 1M
+    private static final int MAX_INTERVAL = 1 << 20;
+
+    private final BulkScorer scorer;
+    private final Runnable checkCancelled;
+
+    CancellableBulkScorer(BulkScorer scorer, Runnable checkCancelled) {
+        this.scorer = Objects.requireNonNull(scorer);
+        this.checkCancelled = Objects.requireNonNull(checkCancelled);
+    }
+
+    @Override
+    public int score(LeafCollector collector, Bits acceptDocs, int min, int max) throws IOException {
+        int interval = INITIAL_INTERVAL;
+        while (min < max) {
+            checkCancelled.run();
+            final int newMax = (int) Math.min((long) min + interval, max);
+            min = scorer.score(collector, acceptDocs, min, newMax);
+            interval = Math.min(interval << 1, MAX_INTERVAL);
+        }
+        checkCancelled.run();
+        return min;
+    }
+
+    @Override
+    public long cost() {
+        return scorer.cost();
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java b/core/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java
index 8d33140e3eec7..5adf25e03b0aa 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java
@@ -20,25 +20,32 @@
 package org.elasticsearch.search.internal;
 
 import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.TermContext;
+import org.apache.lucene.search.BulkScorer;
 import org.apache.lucene.search.CollectionStatistics;
+import org.apache.lucene.search.Collector;
 import org.apache.lucene.search.Explanation;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.QueryCache;
 import org.apache.lucene.search.QueryCachingPolicy;
+import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.TermStatistics;
 import org.apache.lucene.search.Weight;
 import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.search.dfs.AggregatedDfs;
+import org.elasticsearch.search.profile.Timer;
 import org.elasticsearch.search.profile.query.ProfileWeight;
 import org.elasticsearch.search.profile.query.QueryProfileBreakdown;
 import org.elasticsearch.search.profile.query.QueryProfiler;
 import org.elasticsearch.search.profile.query.QueryTimingType;
 
 import java.io.IOException;
+import java.util.List;
+import java.util.Set;
 
 /**
  * Context-aware extension of {@link IndexSearcher}.
@@ -57,6 +64,8 @@ public class ContextIndexSearcher extends IndexSearcher implements Releasable {
     // TODO revisit moving the profiler to inheritance or wrapping model in the future
     private QueryProfiler profiler;
 
+    private Runnable checkCancelled;
+
     public ContextIndexSearcher(Engine.Searcher searcher,
             QueryCache queryCache, QueryCachingPolicy queryCachingPolicy) {
         super(searcher.reader());
@@ -75,6 +84,14 @@ public void setProfiler(QueryProfiler profiler) {
         this.profiler = profiler;
     }
 
+    /**
+     * Set a {@link Runnable} that will be run on a regular basis while
+     * collecting documents.
+     */
+    public void setCheckCancelled(Runnable checkCancelled) {
+        this.checkCancelled = checkCancelled;
+    }
+
     public void setAggregatedDfs(AggregatedDfs aggregatedDfs) {
         this.aggregatedDfs = aggregatedDfs;
     }
@@ -116,12 +133,13 @@ public Weight createWeight(Query query, boolean needsScores) throws IOException
             // each invocation so that it can build an internal representation of the query
             // tree
             QueryProfileBreakdown profile = profiler.getQueryBreakdown(query);
-            profile.startTime(QueryTimingType.CREATE_WEIGHT);
+            Timer timer = profile.getTimer(QueryTimingType.CREATE_WEIGHT);
+            timer.start();
             final Weight weight;
             try {
                 weight = super.createWeight(query, needsScores);
             } finally {
-                profile.stopAndRecordTime();
+                timer.stop();
                 profiler.pollLastElement();
             }
             return new ProfileWeight(query, weight, profile);
@@ -131,6 +149,53 @@ public Weight createWeight(Query query, boolean needsScores) throws IOException
         }
     }
 
+    @Override
+    protected void search(List<LeafReaderContext> leaves, Weight weight, Collector collector) throws IOException {
+        final Weight cancellableWeight;
+        if (checkCancelled != null) {
+            cancellableWeight = new Weight(weight.getQuery()) {
+
+                @Override
+                public void extractTerms(Set<Term> terms) {
+                    throw new UnsupportedOperationException();
+                }
+
+                @Override
+                public Explanation explain(LeafReaderContext context, int doc) throws IOException {
+                    throw new UnsupportedOperationException();
+                }
+
+                @Override
+                public Scorer scorer(LeafReaderContext context) throws IOException {
+                    throw new UnsupportedOperationException();
+                }
+
+                @Override
+                public float getValueForNormalization() throws IOException {
+                    throw new UnsupportedOperationException();
+                }
+
+                @Override
+                public void normalize(float norm, float boost) {
+                    throw new UnsupportedOperationException();
+                }
+
+                @Override
+                public BulkScorer bulkScorer(LeafReaderContext context) throws IOException {
+                    BulkScorer in = weight.bulkScorer(context);
+                    if (in != null) {
+                        return new CancellableBulkScorer(in, checkCancelled);
+                    } else {
+                        return null;
+                    }
+                }
+            };
+        } else {
+            cancellableWeight = weight;
+        }
+        super.search(leaves, cancellableWeight, collector);
+    }
+
     @Override
     public Explanation explain(Query query, int doc) throws IOException {
         if (aggregatedDfs != null) {
diff --git a/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java
index 4e52ab0d53413..fadf979d911d2 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java
@@ -23,11 +23,10 @@
 import org.apache.lucene.search.FieldDoc;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.Counter;
+import org.elasticsearch.action.search.SearchTask;
 import org.elasticsearch.action.search.SearchType;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
-import org.elasticsearch.index.analysis.AnalysisService;
 import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
 import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.mapper.MappedFieldType;
@@ -35,17 +34,16 @@
 import org.elasticsearch.index.mapper.ObjectMapper;
 import org.elasticsearch.index.query.ParsedQuery;
 import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.search.fetch.StoredFieldsContext;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.similarity.SimilarityService;
-import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.search.SearchExtBuilder;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.aggregations.SearchContextAggregations;
+import org.elasticsearch.search.collapse.CollapseContext;
 import org.elasticsearch.search.dfs.DfsSearchResult;
 import org.elasticsearch.search.fetch.FetchPhase;
 import org.elasticsearch.search.fetch.FetchSearchResult;
-import org.elasticsearch.search.fetch.FetchSubPhase;
-import org.elasticsearch.search.fetch.FetchSubPhaseContext;
+import org.elasticsearch.search.fetch.StoredFieldsContext;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 import org.elasticsearch.search.fetch.subphase.InnerHitsContext;
 import org.elasticsearch.search.fetch.subphase.ScriptFieldsContext;
@@ -65,8 +63,6 @@ public abstract class FilteredSearchContext extends SearchContext {
     private final SearchContext in;
 
     public FilteredSearchContext(SearchContext in) {
-        //inner_hits in percolator ends up with null inner search context
-        super(in == null ? ParseFieldMatcher.EMPTY : in.parseFieldMatcher());
         this.in = in;
     }
 
@@ -101,13 +97,13 @@ protected void doClose() {
     }
 
     @Override
-    public void preProcess() {
-        in.preProcess();
+    public void preProcess(boolean rewrite) {
+        in.preProcess(rewrite);
     }
 
     @Override
-    public Query searchFilter(String[] types) {
-        return in.searchFilter(types);
+    public Query buildFilteredQuery(Query query) {
+        return in.buildFilteredQuery(query);
     }
 
     @Override
@@ -145,21 +141,11 @@ public float queryBoost() {
         return in.queryBoost();
     }
 
-    @Override
-    public SearchContext queryBoost(float queryBoost) {
-        return in.queryBoost(queryBoost);
-    }
-
     @Override
     public long getOriginNanoTime() {
         return in.getOriginNanoTime();
     }
 
-    @Override
-    protected long nowInMillisImpl() {
-        return in.nowInMillisImpl();
-    }
-
     @Override
     public ScrollContext scrollContext() {
         return in.scrollContext();
@@ -260,21 +246,11 @@ public MapperService mapperService() {
         return in.mapperService();
     }
 
-    @Override
-    public AnalysisService analysisService() {
-        return in.analysisService();
-    }
-
     @Override
     public SimilarityService similarityService() {
         return in.similarityService();
     }
 
-    @Override
-    public ScriptService scriptService() {
-        return in.scriptService();
-    }
-
     @Override
     public BigArrays bigArrays() {
         return in.bigArrays();
@@ -310,6 +286,11 @@ public void terminateAfter(int terminateAfter) {
         in.terminateAfter(terminateAfter);
     }
 
+    @Override
+    public boolean lowLevelCancellation() {
+        return in.lowLevelCancellation();
+    }
+
     @Override
     public SearchContext minimumScore(float minimumScore) {
         return in.minimumScore(minimumScore);
@@ -512,8 +493,13 @@ public Counter timeEstimateCounter() {
     }
 
     @Override
-    public <SubPhaseContext extends FetchSubPhaseContext> SubPhaseContext getFetchSubPhaseContext(FetchSubPhase.ContextFactory<SubPhaseContext> contextFactory) {
-        return in.getFetchSubPhaseContext(contextFactory);
+    public void addSearchExt(SearchExtBuilder searchExtBuilder) {
+        in.addSearchExt(searchExtBuilder);
+    }
+
+    @Override
+    public SearchExtBuilder getSearchExt(String name) {
+        return in.getSearchExt(name);
     }
 
     @Override
@@ -528,4 +514,29 @@ public Profilers getProfilers() {
     public QueryShardContext getQueryShardContext() {
         return in.getQueryShardContext();
     }
+
+    @Override
+    public void setTask(SearchTask task) {
+        in.setTask(task);
+    }
+
+    @Override
+    public SearchTask getTask() {
+        return in.getTask();
+    }
+
+    @Override
+    public boolean isCancelled() {
+        return in.isCancelled();
+    }
+
+    @Override
+    public SearchContext collapse(CollapseContext collapse) {
+        return in.collapse(collapse);
+    }
+
+    @Override
+    public CollapseContext collapse() {
+        return in.collapse();
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/internal/InternalScrollSearchRequest.java b/core/src/main/java/org/elasticsearch/search/internal/InternalScrollSearchRequest.java
index 82a74de73d295..f112c97dd0f63 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/InternalScrollSearchRequest.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/InternalScrollSearchRequest.java
@@ -20,9 +20,12 @@
 package org.elasticsearch.search.internal;
 
 import org.elasticsearch.action.search.SearchScrollRequest;
+import org.elasticsearch.action.search.SearchTask;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.Scroll;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.transport.TransportRequest;
 
 import java.io.IOException;
@@ -67,4 +70,15 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeLong(id);
         out.writeOptionalWriteable(scroll);
     }
+
+    @Override
+    public Task createTask(long id, String type, String action, TaskId parentTaskId) {
+        return new SearchTask(id, type, action, getDescription(), parentTaskId);
+    }
+
+    @Override
+    public String getDescription() {
+        return "id[" + id + "], scroll[" + scroll + "]";
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java b/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java
deleted file mode 100644
index e8ba4d88aa702..0000000000000
--- a/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java
+++ /dev/null
@@ -1,853 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.internal;
-
-import org.apache.lucene.search.Explanation;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.compress.CompressorFactory;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Streamable;
-import org.elasticsearch.common.text.Text;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.SearchHit;
-import org.elasticsearch.search.SearchHitField;
-import org.elasticsearch.search.SearchHits;
-import org.elasticsearch.search.SearchShardTarget;
-import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
-import org.elasticsearch.search.internal.InternalSearchHits.StreamContext.ShardTargetType;
-import org.elasticsearch.search.lookup.SourceLookup;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-import static java.util.Collections.emptyMap;
-import static java.util.Collections.singletonMap;
-import static java.util.Collections.unmodifiableMap;
-import static org.elasticsearch.common.lucene.Lucene.readExplanation;
-import static org.elasticsearch.common.lucene.Lucene.writeExplanation;
-import static org.elasticsearch.search.fetch.subphase.highlight.HighlightField.readHighlightField;
-import static org.elasticsearch.search.internal.InternalSearchHitField.readSearchHitField;
-
-/**
- *
- */
-public class InternalSearchHit implements SearchHit {
-
-    private static final Object[] EMPTY_SORT_VALUES = new Object[0];
-
-    private transient int docId;
-
-    private float score = Float.NEGATIVE_INFINITY;
-
-    private Text id;
-    private Text type;
-
-    private InternalNestedIdentity nestedIdentity;
-
-    private long version = -1;
-
-    private BytesReference source;
-
-    private Map<String, SearchHitField> fields = emptyMap();
-
-    private Map<String, HighlightField> highlightFields = null;
-
-    private Object[] sortValues = EMPTY_SORT_VALUES;
-
-    private String[] matchedQueries = Strings.EMPTY_ARRAY;
-
-    private Explanation explanation;
-
-    @Nullable
-    private SearchShardTarget shard;
-
-    private Map<String, Object> sourceAsMap;
-    private byte[] sourceAsBytes;
-
-    private Map<String, InternalSearchHits> innerHits;
-
-    private InternalSearchHit() {
-
-    }
-
-    public InternalSearchHit(int docId) {
-        this(docId, null, null, null);
-    }
-
-    public InternalSearchHit(int docId, String id, Text type, Map<String, SearchHitField> fields) {
-        this.docId = docId;
-        if (id != null) {
-            this.id = new Text(id);
-        } else {
-            this.id = null;
-        }
-        this.type = type;
-        this.fields = fields;
-    }
-
-    public InternalSearchHit(int nestedTopDocId, String id, Text type, InternalNestedIdentity nestedIdentity, Map<String, SearchHitField> fields) {
-        this.docId = nestedTopDocId;
-        this.id = new Text(id);
-        this.type = type;
-        this.nestedIdentity = nestedIdentity;
-        this.fields = fields;
-    }
-
-    public int docId() {
-        return this.docId;
-    }
-
-    public void shardTarget(SearchShardTarget shardTarget) {
-        this.shard = shardTarget;
-        if (innerHits != null) {
-            for (InternalSearchHits searchHits : innerHits.values()) {
-                searchHits.shardTarget(shardTarget);
-            }
-        }
-    }
-
-    public void score(float score) {
-        this.score = score;
-    }
-
-    @Override
-    public float score() {
-        return this.score;
-    }
-
-    @Override
-    public float getScore() {
-        return score();
-    }
-
-    public void version(long version) {
-        this.version = version;
-    }
-
-    @Override
-    public long version() {
-        return this.version;
-    }
-
-    @Override
-    public long getVersion() {
-        return this.version;
-    }
-
-    @Override
-    public String index() {
-        return shard.index();
-    }
-
-    @Override
-    public String getIndex() {
-        return index();
-    }
-
-    @Override
-    public String id() {
-        return id != null ? id.string() : null;
-    }
-
-    @Override
-    public String getId() {
-        return id();
-    }
-
-    @Override
-    public String type() {
-        return type != null ? type.string() : null;
-    }
-
-    @Override
-    public String getType() {
-        return type();
-    }
-
-    @Override
-    public NestedIdentity getNestedIdentity() {
-        return nestedIdentity;
-    }
-
-    /**
-     * Returns bytes reference, also un compress the source if needed.
-     */
-    @Override
-    public BytesReference sourceRef() {
-        try {
-            this.source = CompressorFactory.uncompressIfNeeded(this.source);
-            return this.source;
-        } catch (IOException e) {
-            throw new ElasticsearchParseException("failed to decompress source", e);
-        }
-    }
-
-    /**
-     * Sets representation, might be compressed....
-     */
-    public InternalSearchHit sourceRef(BytesReference source) {
-        this.source = source;
-        this.sourceAsBytes = null;
-        this.sourceAsMap = null;
-        return this;
-    }
-
-    @Override
-    public BytesReference getSourceRef() {
-        return sourceRef();
-    }
-
-    /**
-     * Internal source representation, might be compressed....
-     */
-    public BytesReference internalSourceRef() {
-        return source;
-    }
-
-
-    @Override
-    public byte[] source() {
-        if (source == null) {
-            return null;
-        }
-        if (sourceAsBytes != null) {
-            return sourceAsBytes;
-        }
-        this.sourceAsBytes = BytesReference.toBytes(sourceRef());
-        return this.sourceAsBytes;
-    }
-
-    @Override
-    public boolean hasSource() {
-        return source == null;
-    }
-
-    @Override
-    public Map<String, Object> getSource() {
-        return sourceAsMap();
-    }
-
-    @Override
-    public String sourceAsString() {
-        if (source == null) {
-            return null;
-        }
-        try {
-            return XContentHelper.convertToJson(sourceRef(), false);
-        } catch (IOException e) {
-            throw new ElasticsearchParseException("failed to convert source to a json string");
-        }
-    }
-
-    @Override
-    public String getSourceAsString() {
-        return sourceAsString();
-    }
-
-    @SuppressWarnings({"unchecked"})
-    @Override
-    public Map<String, Object> sourceAsMap() throws ElasticsearchParseException {
-        if (source == null) {
-            return null;
-        }
-        if (sourceAsMap != null) {
-            return sourceAsMap;
-        }
-
-        sourceAsMap = SourceLookup.sourceAsMap(source);
-        return sourceAsMap;
-    }
-
-    @Override
-    public Iterator<SearchHitField> iterator() {
-        return fields.values().iterator();
-    }
-
-    @Override
-    public SearchHitField field(String fieldName) {
-        return fields().get(fieldName);
-    }
-
-    @Override
-    public Map<String, SearchHitField> fields() {
-        return fields == null ? emptyMap() : fields;
-    }
-
-    // returns the fields without handling null cases
-    public Map<String, SearchHitField> fieldsOrNull() {
-        return fields;
-    }
-
-    @Override
-    public Map<String, SearchHitField> getFields() {
-        return fields();
-    }
-
-    public void fields(Map<String, SearchHitField> fields) {
-        this.fields = fields;
-    }
-
-    public Map<String, HighlightField> internalHighlightFields() {
-        return highlightFields;
-    }
-
-    @Override
-    public Map<String, HighlightField> highlightFields() {
-        return highlightFields == null ? emptyMap() : highlightFields;
-    }
-
-    @Override
-    public Map<String, HighlightField> getHighlightFields() {
-        return highlightFields();
-    }
-
-    public void highlightFields(Map<String, HighlightField> highlightFields) {
-        this.highlightFields = highlightFields;
-    }
-
-    public void sortValues(Object[] sortValues, DocValueFormat[] sortValueFormats) {
-        this.sortValues = Arrays.copyOf(sortValues, sortValues.length);
-        for (int i = 0; i < sortValues.length; ++i) {
-            if (this.sortValues[i] instanceof BytesRef) {
-                this.sortValues[i] = sortValueFormats[i].format((BytesRef) sortValues[i]);
-            }
-        }
-    }
-
-    @Override
-    public Object[] sortValues() {
-        return sortValues;
-    }
-
-    @Override
-    public Object[] getSortValues() {
-        return sortValues();
-    }
-
-    @Override
-    public Explanation explanation() {
-        return explanation;
-    }
-
-    @Override
-    public Explanation getExplanation() {
-        return explanation();
-    }
-
-    public void explanation(Explanation explanation) {
-        this.explanation = explanation;
-    }
-
-    @Override
-    public SearchShardTarget shard() {
-        return shard;
-    }
-
-    @Override
-    public SearchShardTarget getShard() {
-        return shard();
-    }
-
-    public void shard(SearchShardTarget target) {
-        this.shard = target;
-    }
-
-    public void matchedQueries(String[] matchedQueries) {
-        this.matchedQueries = matchedQueries;
-    }
-
-    @Override
-    public String[] matchedQueries() {
-        return this.matchedQueries;
-    }
-
-    @Override
-    public String[] getMatchedQueries() {
-        return this.matchedQueries;
-    }
-
-    @Override
-    @SuppressWarnings("unchecked")
-    public Map<String, SearchHits> getInnerHits() {
-        return (Map) innerHits;
-    }
-
-    public void setInnerHits(Map<String, InternalSearchHits> innerHits) {
-        this.innerHits = innerHits;
-    }
-
-    public static class Fields {
-        static final String _INDEX = "_index";
-        static final String _TYPE = "_type";
-        static final String _ID = "_id";
-        static final String _VERSION = "_version";
-        static final String _SCORE = "_score";
-        static final String FIELDS = "fields";
-        static final String HIGHLIGHT = "highlight";
-        static final String SORT = "sort";
-        static final String MATCHED_QUERIES = "matched_queries";
-        static final String _EXPLANATION = "_explanation";
-        static final String VALUE = "value";
-        static final String DESCRIPTION = "description";
-        static final String DETAILS = "details";
-        static final String INNER_HITS = "inner_hits";
-    }
-
-    // public because we render hit as part of completion suggestion option
-    public XContentBuilder toInnerXContent(XContentBuilder builder, Params params) throws IOException {
-        List<SearchHitField> metaFields = new ArrayList<>();
-        List<SearchHitField> otherFields = new ArrayList<>();
-        if (fields != null && !fields.isEmpty()) {
-            for (SearchHitField field : fields.values()) {
-                if (field.values().isEmpty()) {
-                    continue;
-                }
-                if (field.isMetadataField()) {
-                    metaFields.add(field);
-                } else {
-                    otherFields.add(field);
-                }
-            }
-        }
-
-        // For inner_hit hits shard is null and that is ok, because the parent search hit has all this information.
-        // Even if this was included in the inner_hit hits this would be the same, so better leave it out.
-        if (explanation() != null && shard != null) {
-            builder.field("_shard", shard.shardId());
-            builder.field("_node", shard.nodeIdText());
-        }
-        if (nestedIdentity != null) {
-            nestedIdentity.toXContent(builder, params);
-        } else {
-            if (shard != null) {
-                builder.field(Fields._INDEX, shard.indexText());
-            }
-            if (type != null) {
-                builder.field(Fields._TYPE, type);
-            }
-            if (id != null) {
-                builder.field(Fields._ID, id);
-            }
-        }
-        if (version != -1) {
-            builder.field(Fields._VERSION, version);
-        }
-        if (Float.isNaN(score)) {
-            builder.nullField(Fields._SCORE);
-        } else {
-            builder.field(Fields._SCORE, score);
-        }
-        for (SearchHitField field : metaFields) {
-            builder.field(field.name(), (Object) field.value());
-        }
-        if (source != null) {
-            XContentHelper.writeRawField("_source", source, builder, params);
-        }
-        if (!otherFields.isEmpty()) {
-            builder.startObject(Fields.FIELDS);
-            for (SearchHitField field : otherFields) {
-                builder.startArray(field.name());
-                for (Object value : field.getValues()) {
-                    builder.value(value);
-                }
-                builder.endArray();
-            }
-            builder.endObject();
-        }
-        if (highlightFields != null && !highlightFields.isEmpty()) {
-            builder.startObject(Fields.HIGHLIGHT);
-            for (HighlightField field : highlightFields.values()) {
-                builder.field(field.name());
-                if (field.fragments() == null) {
-                    builder.nullValue();
-                } else {
-                    builder.startArray();
-                    for (Text fragment : field.fragments()) {
-                        builder.value(fragment);
-                    }
-                    builder.endArray();
-                }
-            }
-            builder.endObject();
-        }
-        if (sortValues != null && sortValues.length > 0) {
-            builder.startArray(Fields.SORT);
-            for (Object sortValue : sortValues) {
-                builder.value(sortValue);
-            }
-            builder.endArray();
-        }
-        if (matchedQueries.length > 0) {
-            builder.startArray(Fields.MATCHED_QUERIES);
-            for (String matchedFilter : matchedQueries) {
-                builder.value(matchedFilter);
-            }
-            builder.endArray();
-        }
-        if (explanation() != null) {
-            builder.field(Fields._EXPLANATION);
-            buildExplanation(builder, explanation());
-        }
-        if (innerHits != null) {
-            builder.startObject(Fields.INNER_HITS);
-            for (Map.Entry<String, InternalSearchHits> entry : innerHits.entrySet()) {
-                builder.startObject(entry.getKey());
-                entry.getValue().toXContent(builder, params);
-                builder.endObject();
-            }
-            builder.endObject();
-        }
-        return builder;
-    }
-
-    private void buildExplanation(XContentBuilder builder, Explanation explanation) throws IOException {
-        builder.startObject();
-        builder.field(Fields.VALUE, explanation.getValue());
-        builder.field(Fields.DESCRIPTION, explanation.getDescription());
-        Explanation[] innerExps = explanation.getDetails();
-        if (innerExps != null) {
-            builder.startArray(Fields.DETAILS);
-            for (Explanation exp : innerExps) {
-                buildExplanation(builder, exp);
-            }
-            builder.endArray();
-        }
-        builder.endObject();
-
-    }
-
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject();
-        toInnerXContent(builder, params);
-        builder.endObject();
-        return builder;
-    }
-
-    public static InternalSearchHit readSearchHit(StreamInput in, InternalSearchHits.StreamContext context) throws IOException {
-        InternalSearchHit hit = new InternalSearchHit();
-        hit.readFrom(in, context);
-        return hit;
-    }
-
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        readFrom(in, InternalSearchHits.streamContext().streamShardTarget(ShardTargetType.STREAM));
-    }
-
-    public void readFrom(StreamInput in, InternalSearchHits.StreamContext context) throws IOException {
-        score = in.readFloat();
-        id = in.readOptionalText();
-        type = in.readOptionalText();
-        nestedIdentity = in.readOptionalStreamable(InternalNestedIdentity::new);
-        version = in.readLong();
-        source = in.readBytesReference();
-        if (source.length() == 0) {
-            source = null;
-        }
-        if (in.readBoolean()) {
-            explanation = readExplanation(in);
-        }
-        int size = in.readVInt();
-        if (size == 0) {
-            fields = emptyMap();
-        } else if (size == 1) {
-            SearchHitField hitField = readSearchHitField(in);
-            fields = singletonMap(hitField.name(), hitField);
-        } else {
-            Map<String, SearchHitField> fields = new HashMap<>();
-            for (int i = 0; i < size; i++) {
-                SearchHitField hitField = readSearchHitField(in);
-                fields.put(hitField.name(), hitField);
-            }
-            this.fields = unmodifiableMap(fields);
-        }
-
-        size = in.readVInt();
-        if (size == 0) {
-            highlightFields = emptyMap();
-        } else if (size == 1) {
-            HighlightField field = readHighlightField(in);
-            highlightFields = singletonMap(field.name(), field);
-        } else {
-            Map<String, HighlightField> highlightFields = new HashMap<>();
-            for (int i = 0; i < size; i++) {
-                HighlightField field = readHighlightField(in);
-                highlightFields.put(field.name(), field);
-            }
-            this.highlightFields = unmodifiableMap(highlightFields);
-        }
-
-        size = in.readVInt();
-        if (size > 0) {
-            sortValues = new Object[size];
-            for (int i = 0; i < sortValues.length; i++) {
-                byte type = in.readByte();
-                if (type == 0) {
-                    sortValues[i] = null;
-                } else if (type == 1) {
-                    sortValues[i] = in.readString();
-                } else if (type == 2) {
-                    sortValues[i] = in.readInt();
-                } else if (type == 3) {
-                    sortValues[i] = in.readLong();
-                } else if (type == 4) {
-                    sortValues[i] = in.readFloat();
-                } else if (type == 5) {
-                    sortValues[i] = in.readDouble();
-                } else if (type == 6) {
-                    sortValues[i] = in.readByte();
-                } else if (type == 7) {
-                    sortValues[i] = in.readShort();
-                } else if (type == 8) {
-                    sortValues[i] = in.readBoolean();
-                } else {
-                    throw new IOException("Can't match type [" + type + "]");
-                }
-            }
-        }
-
-        size = in.readVInt();
-        if (size > 0) {
-            matchedQueries = new String[size];
-            for (int i = 0; i < size; i++) {
-                matchedQueries[i] = in.readString();
-            }
-        }
-
-        if (context.streamShardTarget() == ShardTargetType.STREAM) {
-            if (in.readBoolean()) {
-                shard = new SearchShardTarget(in);
-            }
-        } else if (context.streamShardTarget() == ShardTargetType.LOOKUP) {
-            int lookupId = in.readVInt();
-            if (lookupId > 0) {
-                shard = context.handleShardLookup().get(lookupId);
-            }
-        }
-
-        size = in.readVInt();
-        if (size > 0) {
-            innerHits = new HashMap<>(size);
-            for (int i = 0; i < size; i++) {
-                String key = in.readString();
-                ShardTargetType shardTarget = context.streamShardTarget();
-                InternalSearchHits value = InternalSearchHits.readSearchHits(in, context.streamShardTarget(ShardTargetType.NO_STREAM));
-                context.streamShardTarget(shardTarget);
-                innerHits.put(key, value);
-            }
-        }
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        writeTo(out, InternalSearchHits.streamContext().streamShardTarget(ShardTargetType.STREAM));
-    }
-
-    public void writeTo(StreamOutput out, InternalSearchHits.StreamContext context) throws IOException {
-        out.writeFloat(score);
-        out.writeOptionalText(id);
-        out.writeOptionalText(type);
-        out.writeOptionalStreamable(nestedIdentity);
-        out.writeLong(version);
-        out.writeBytesReference(source);
-        if (explanation == null) {
-            out.writeBoolean(false);
-        } else {
-            out.writeBoolean(true);
-            writeExplanation(out, explanation);
-        }
-        if (fields == null) {
-            out.writeVInt(0);
-        } else {
-            out.writeVInt(fields.size());
-            for (SearchHitField hitField : fields().values()) {
-                hitField.writeTo(out);
-            }
-        }
-        if (highlightFields == null) {
-            out.writeVInt(0);
-        } else {
-            out.writeVInt(highlightFields.size());
-            for (HighlightField highlightField : highlightFields.values()) {
-                highlightField.writeTo(out);
-            }
-        }
-
-        if (sortValues.length == 0) {
-            out.writeVInt(0);
-        } else {
-            out.writeVInt(sortValues.length);
-            for (Object sortValue : sortValues) {
-                if (sortValue == null) {
-                    out.writeByte((byte) 0);
-                } else {
-                    Class type = sortValue.getClass();
-                    if (type == String.class) {
-                        out.writeByte((byte) 1);
-                        out.writeString((String) sortValue);
-                    } else if (type == Integer.class) {
-                        out.writeByte((byte) 2);
-                        out.writeInt((Integer) sortValue);
-                    } else if (type == Long.class) {
-                        out.writeByte((byte) 3);
-                        out.writeLong((Long) sortValue);
-                    } else if (type == Float.class) {
-                        out.writeByte((byte) 4);
-                        out.writeFloat((Float) sortValue);
-                    } else if (type == Double.class) {
-                        out.writeByte((byte) 5);
-                        out.writeDouble((Double) sortValue);
-                    } else if (type == Byte.class) {
-                        out.writeByte((byte) 6);
-                        out.writeByte((Byte) sortValue);
-                    } else if (type == Short.class) {
-                        out.writeByte((byte) 7);
-                        out.writeShort((Short) sortValue);
-                    } else if (type == Boolean.class) {
-                        out.writeByte((byte) 8);
-                        out.writeBoolean((Boolean) sortValue);
-                    } else {
-                        throw new IOException("Can't handle sort field value of type [" + type + "]");
-                    }
-                }
-            }
-        }
-
-        if (matchedQueries.length == 0) {
-            out.writeVInt(0);
-        } else {
-            out.writeVInt(matchedQueries.length);
-            for (String matchedFilter : matchedQueries) {
-                out.writeString(matchedFilter);
-            }
-        }
-
-        if (context.streamShardTarget() == ShardTargetType.STREAM) {
-            if (shard == null) {
-                out.writeBoolean(false);
-            } else {
-                out.writeBoolean(true);
-                shard.writeTo(out);
-            }
-        } else if (context.streamShardTarget() == ShardTargetType.LOOKUP) {
-            if (shard == null) {
-                out.writeVInt(0);
-            } else {
-                out.writeVInt(context.shardHandleLookup().get(shard));
-            }
-        }
-
-        if (innerHits == null) {
-            out.writeVInt(0);
-        } else {
-            out.writeVInt(innerHits.size());
-            for (Map.Entry<String, InternalSearchHits> entry : innerHits.entrySet()) {
-                out.writeString(entry.getKey());
-                ShardTargetType shardTarget = context.streamShardTarget();
-                entry.getValue().writeTo(out, context.streamShardTarget(ShardTargetType.NO_STREAM));
-                context.streamShardTarget(shardTarget);
-            }
-        }
-    }
-
-    public static final class InternalNestedIdentity implements NestedIdentity, Streamable, ToXContent {
-
-        private Text field;
-        private int offset;
-        private InternalNestedIdentity child;
-
-        public InternalNestedIdentity(String field, int offset, InternalNestedIdentity child) {
-            this.field = new Text(field);
-            this.offset = offset;
-            this.child = child;
-        }
-
-        InternalNestedIdentity() {
-        }
-
-        @Override
-        public Text getField() {
-            return field;
-        }
-
-        @Override
-        public int getOffset() {
-            return offset;
-        }
-
-        @Override
-        public NestedIdentity getChild() {
-            return child;
-        }
-
-        @Override
-        public void readFrom(StreamInput in) throws IOException {
-            field = in.readOptionalText();
-            offset = in.readInt();
-            child = in.readOptionalStreamable(InternalNestedIdentity::new);
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            out.writeOptionalText(field);
-            out.writeInt(offset);
-            out.writeOptionalStreamable(child);
-        }
-
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startObject(Fields._NESTED);
-            if (field != null) {
-                builder.field(Fields._NESTED_FIELD, field);
-            }
-            if (offset != -1) {
-                builder.field(Fields._NESTED_OFFSET, offset);
-            }
-            if (child != null) {
-                builder = child.toXContent(builder, params);
-            }
-            builder.endObject();
-            return builder;
-        }
-
-        public static class Fields {
-
-            static final String _NESTED = "_nested";
-            static final String _NESTED_FIELD = "field";
-            static final String _NESTED_OFFSET = "offset";
-
-        }
-    }
-
-}
diff --git a/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHitField.java b/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHitField.java
deleted file mode 100644
index 114aa4999d1b6..0000000000000
--- a/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHitField.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.internal;
-
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.search.SearchHitField;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-/**
- *
- */
-public class InternalSearchHitField implements SearchHitField {
-
-    private String name;
-    private List<Object> values;
-
-    private InternalSearchHitField() {
-    }
-
-    public InternalSearchHitField(String name, List<Object> values) {
-        this.name = name;
-        this.values = values;
-    }
-
-    @Override
-    public String name() {
-        return name;
-    }
-
-    @Override
-    public String getName() {
-        return name();
-    }
-
-    @Override
-    public Object value() {
-        if (values == null || values.isEmpty()) {
-            return null;
-        }
-        return values.get(0);
-    }
-
-    @Override
-    public Object getValue() {
-        return value();
-    }
-
-    @Override
-    public List<Object> values() {
-        return values;
-    }
-
-    @Override
-    public List<Object> getValues() {
-        return values();
-    }
-
-    @Override
-    public boolean isMetadataField() {
-        return MapperService.isMetadataField(name);
-    }
-
-    @Override
-    public Iterator<Object> iterator() {
-        return values.iterator();
-    }
-
-    public static InternalSearchHitField readSearchHitField(StreamInput in) throws IOException {
-        InternalSearchHitField result = new InternalSearchHitField();
-        result.readFrom(in);
-        return result;
-    }
-
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        name = in.readString();
-        int size = in.readVInt();
-        values = new ArrayList<>(size);
-        for (int i = 0; i < size; i++) {
-            values.add(in.readGenericValue());
-        }
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeString(name);
-        out.writeVInt(values.size());
-        for (Object value : values) {
-            out.writeGenericValue(value);
-        }
-    }
-}
\ No newline at end of file
diff --git a/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHits.java b/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHits.java
deleted file mode 100644
index 592d4b0751ef3..0000000000000
--- a/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHits.java
+++ /dev/null
@@ -1,263 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.internal;
-
-import com.carrotsearch.hppc.IntObjectHashMap;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.search.SearchHit;
-import org.elasticsearch.search.SearchHits;
-import org.elasticsearch.search.SearchShardTarget;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.IdentityHashMap;
-import java.util.Iterator;
-import java.util.Map;
-
-import static org.elasticsearch.search.internal.InternalSearchHit.readSearchHit;
-
-/**
- *
- */
-public class InternalSearchHits implements SearchHits {
-
-    public static class StreamContext {
-
-        public static enum ShardTargetType {
-            STREAM,
-            LOOKUP,
-            NO_STREAM
-        }
-
-        private IdentityHashMap<SearchShardTarget, Integer> shardHandleLookup = new IdentityHashMap<>();
-        private IntObjectHashMap<SearchShardTarget> handleShardLookup = new IntObjectHashMap<>();
-        private ShardTargetType streamShardTarget = ShardTargetType.STREAM;
-
-        public StreamContext reset() {
-            shardHandleLookup.clear();
-            handleShardLookup.clear();
-            streamShardTarget = ShardTargetType.STREAM;
-            return this;
-        }
-
-        public IdentityHashMap<SearchShardTarget, Integer> shardHandleLookup() {
-            return shardHandleLookup;
-        }
-
-        public IntObjectHashMap<SearchShardTarget> handleShardLookup() {
-            return handleShardLookup;
-        }
-
-        public ShardTargetType streamShardTarget() {
-            return streamShardTarget;
-        }
-
-        public StreamContext streamShardTarget(ShardTargetType streamShardTarget) {
-            this.streamShardTarget = streamShardTarget;
-            return this;
-        }
-    }
-
-    private static final ThreadLocal<StreamContext> cache = new ThreadLocal<StreamContext>() {
-        @Override
-        protected StreamContext initialValue() {
-            return new StreamContext();
-        }
-    };
-
-    public static StreamContext streamContext() {
-        return cache.get().reset();
-    }
-
-    public static InternalSearchHits empty() {
-        // We shouldn't use static final instance, since that could directly be returned by native transport clients
-        return new InternalSearchHits(EMPTY, 0, 0);
-    }
-
-    public static final InternalSearchHit[] EMPTY = new InternalSearchHit[0];
-
-    private InternalSearchHit[] hits;
-
-    public long totalHits;
-
-    private float maxScore;
-
-    InternalSearchHits() {
-
-    }
-
-    public InternalSearchHits(InternalSearchHit[] hits, long totalHits, float maxScore) {
-        this.hits = hits;
-        this.totalHits = totalHits;
-        this.maxScore = maxScore;
-    }
-
-    public void shardTarget(SearchShardTarget shardTarget) {
-        for (InternalSearchHit hit : hits) {
-            hit.shard(shardTarget);
-        }
-    }
-
-    @Override
-    public long totalHits() {
-        return totalHits;
-    }
-
-    @Override
-    public long getTotalHits() {
-        return totalHits();
-    }
-
-    @Override
-    public float maxScore() {
-        return this.maxScore;
-    }
-
-    @Override
-    public float getMaxScore() {
-        return maxScore();
-    }
-
-    @Override
-    public SearchHit[] hits() {
-        return this.hits;
-    }
-
-    @Override
-    public SearchHit getAt(int position) {
-        return hits[position];
-    }
-
-    @Override
-    public SearchHit[] getHits() {
-        return hits();
-    }
-
-    @Override
-    public Iterator<SearchHit> iterator() {
-        return Arrays.stream(hits()).iterator();
-    }
-
-    public InternalSearchHit[] internalHits() {
-        return this.hits;
-    }
-
-    static final class Fields {
-        static final String HITS = "hits";
-        static final String TOTAL = "total";
-        static final String MAX_SCORE = "max_score";
-    }
-
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject(Fields.HITS);
-        builder.field(Fields.TOTAL, totalHits);
-        if (Float.isNaN(maxScore)) {
-            builder.nullField(Fields.MAX_SCORE);
-        } else {
-            builder.field(Fields.MAX_SCORE, maxScore);
-        }
-        builder.field(Fields.HITS);
-        builder.startArray();
-        for (SearchHit hit : hits) {
-            hit.toXContent(builder, params);
-        }
-        builder.endArray();
-        builder.endObject();
-        return builder;
-    }
-
-    public static InternalSearchHits readSearchHits(StreamInput in, StreamContext context) throws IOException {
-        InternalSearchHits hits = new InternalSearchHits();
-        hits.readFrom(in, context);
-        return hits;
-    }
-
-    public static InternalSearchHits readSearchHits(StreamInput in) throws IOException {
-        InternalSearchHits hits = new InternalSearchHits();
-        hits.readFrom(in);
-        return hits;
-    }
-
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        readFrom(in, streamContext().streamShardTarget(StreamContext.ShardTargetType.LOOKUP));
-    }
-
-    public void readFrom(StreamInput in, StreamContext context) throws IOException {
-        totalHits = in.readVLong();
-        maxScore = in.readFloat();
-        int size = in.readVInt();
-        if (size == 0) {
-            hits = EMPTY;
-        } else {
-            if (context.streamShardTarget() == StreamContext.ShardTargetType.LOOKUP) {
-                // read the lookup table first
-                int lookupSize = in.readVInt();
-                for (int i = 0; i < lookupSize; i++) {
-                    context.handleShardLookup().put(in.readVInt(), new SearchShardTarget(in));
-                }
-            }
-
-            hits = new InternalSearchHit[size];
-            for (int i = 0; i < hits.length; i++) {
-                hits[i] = readSearchHit(in, context);
-            }
-        }
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        writeTo(out, streamContext().streamShardTarget(StreamContext.ShardTargetType.LOOKUP));
-    }
-
-    public void writeTo(StreamOutput out, StreamContext context) throws IOException {
-        out.writeVLong(totalHits);
-        out.writeFloat(maxScore);
-        out.writeVInt(hits.length);
-        if (hits.length > 0) {
-            if (context.streamShardTarget() == StreamContext.ShardTargetType.LOOKUP) {
-                // start from 1, 0 is for null!
-                int counter = 1;
-                for (InternalSearchHit hit : hits) {
-                    if (hit.shard() != null) {
-                        Integer handle = context.shardHandleLookup().get(hit.shard());
-                        if (handle == null) {
-                            context.shardHandleLookup().put(hit.shard(), counter++);
-                        }
-                    }
-                }
-                out.writeVInt(context.shardHandleLookup().size());
-                if (!context.shardHandleLookup().isEmpty()) {
-                    for (Map.Entry<SearchShardTarget, Integer> entry : context.shardHandleLookup().entrySet()) {
-                        out.writeVInt(entry.getValue());
-                        entry.getKey().writeTo(out);
-                    }
-                }
-            }
-
-            for (InternalSearchHit hit : hits) {
-                hit.writeTo(out, context);
-            }
-        }
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/internal/InternalSearchResponse.java b/core/src/main/java/org/elasticsearch/search/internal/InternalSearchResponse.java
index 09a787ac3cb8a..eae65c2e95288 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/InternalSearchResponse.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/InternalSearchResponse.java
@@ -19,144 +19,57 @@
 
 package org.elasticsearch.search.internal;
 
+
 import org.elasticsearch.Version;
+import org.elasticsearch.action.search.SearchResponseSections;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Streamable;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.SearchHits;
-import org.elasticsearch.search.aggregations.Aggregations;
 import org.elasticsearch.search.aggregations.InternalAggregations;
-import org.elasticsearch.search.profile.ProfileShardResult;
 import org.elasticsearch.search.profile.SearchProfileShardResults;
 import org.elasticsearch.search.suggest.Suggest;
 
 import java.io.IOException;
-import java.util.Collections;
-import java.util.Map;
-
-import static org.elasticsearch.search.internal.InternalSearchHits.readSearchHits;
 
 /**
- *
+ * {@link SearchResponseSections} subclass that can be serialized over the wire.
  */
-public class InternalSearchResponse implements Streamable, ToXContent {
+public class InternalSearchResponse extends SearchResponseSections implements Writeable, ToXContent {
 
     public static InternalSearchResponse empty() {
-        return new InternalSearchResponse(InternalSearchHits.empty(), null, null, null, false, null);
-    }
-
-    private InternalSearchHits hits;
-
-    private InternalAggregations aggregations;
-
-    private Suggest suggest;
-
-    private SearchProfileShardResults profileResults;
-
-    private boolean timedOut;
-
-    private Boolean terminatedEarly = null;
-
-    private InternalSearchResponse() {
-    }
-
-    public InternalSearchResponse(InternalSearchHits hits, InternalAggregations aggregations, Suggest suggest,
-                                  SearchProfileShardResults profileResults, boolean timedOut, Boolean terminatedEarly) {
-        this.hits = hits;
-        this.aggregations = aggregations;
-        this.suggest = suggest;
-        this.profileResults = profileResults;
-        this.timedOut = timedOut;
-        this.terminatedEarly = terminatedEarly;
-    }
-
-    public boolean timedOut() {
-        return this.timedOut;
-    }
-
-    public Boolean terminatedEarly() {
-        return this.terminatedEarly;
-    }
-
-    public SearchHits hits() {
-        return hits;
+        return new InternalSearchResponse(SearchHits.empty(), null, null, null, false, null, 1);
     }
 
-    public Aggregations aggregations() {
-        return aggregations;
+    public InternalSearchResponse(SearchHits hits, InternalAggregations aggregations, Suggest suggest,
+                                  SearchProfileShardResults profileResults, boolean timedOut, Boolean terminatedEarly,
+                                  int numReducePhases) {
+        super(hits, aggregations, suggest, timedOut, terminatedEarly, profileResults, numReducePhases);
     }
 
-    public Suggest suggest() {
-        return suggest;
-    }
-
-    /**
-     * Returns the profile results for this search response (including all shards).
-     * An empty map is returned if profiling was not enabled
-     *
-     * @return Profile results
-     */
-    public Map<String, ProfileShardResult> profile() {
-        if (profileResults == null) {
-            return Collections.emptyMap();
-        }
-        return profileResults.getShardResults();
-    }
-
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        hits.toXContent(builder, params);
-        if (aggregations != null) {
-            aggregations.toXContent(builder, params);
-        }
-        if (suggest != null) {
-            suggest.toXContent(builder, params);
-        }
-        if (profileResults != null) {
-            profileResults.toXContent(builder, params);
-        }
-        return builder;
-    }
-
-    public static InternalSearchResponse readInternalSearchResponse(StreamInput in) throws IOException {
-        InternalSearchResponse response = new InternalSearchResponse();
-        response.readFrom(in);
-        return response;
-    }
-
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        hits = readSearchHits(in);
-        if (in.readBoolean()) {
-            aggregations = InternalAggregations.readAggregations(in);
-        }
-        if (in.readBoolean()) {
-            suggest = Suggest.readSuggest(in);
-        }
-        timedOut = in.readBoolean();
-        terminatedEarly = in.readOptionalBoolean();
-        profileResults = in.readOptionalWriteable(SearchProfileShardResults::new);
+    public InternalSearchResponse(StreamInput in) throws IOException {
+        super(
+                SearchHits.readSearchHits(in),
+                in.readBoolean() ? InternalAggregations.readAggregations(in) : null,
+                in.readBoolean() ? Suggest.readSuggest(in) : null,
+                in.readBoolean(),
+                in.readOptionalBoolean(),
+                in.readOptionalWriteable(SearchProfileShardResults::new),
+                in.getVersion().onOrAfter(Version.V_5_4_0) ? in.readVInt() : 1
+        );
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         hits.writeTo(out);
-        if (aggregations == null) {
-            out.writeBoolean(false);
-        } else {
-            out.writeBoolean(true);
-            aggregations.writeTo(out);
-        }
-        if (suggest == null) {
-            out.writeBoolean(false);
-        } else {
-            out.writeBoolean(true);
-            suggest.writeTo(out);
-        }
+        out.writeOptionalStreamable((InternalAggregations)aggregations);
+        out.writeOptionalStreamable(suggest);
         out.writeBoolean(timedOut);
         out.writeOptionalBoolean(terminatedEarly);
         out.writeOptionalWriteable(profileResults);
+        if (out.getVersion().onOrAfter(Version.V_5_4_0)) {
+            out.writeVInt(numReducePhases);
+        }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/internal/ScrollContext.java b/core/src/main/java/org/elasticsearch/search/internal/ScrollContext.java
index 1b7bcfb93c788..163dbcc73d924 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/ScrollContext.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/ScrollContext.java
@@ -22,11 +22,35 @@
 import org.apache.lucene.search.ScoreDoc;
 import org.elasticsearch.search.Scroll;
 
+import java.util.HashMap;
+import java.util.Map;
+
 /** Wrapper around information that needs to stay around when scrolling. */
-public class ScrollContext {
+public final class ScrollContext {
+
+    private Map<String, Object> context = null;
 
     public int totalHits = -1;
     public float maxScore;
     public ScoreDoc lastEmittedDoc;
     public Scroll scroll;
+
+    /**
+     * Returns the object or <code>null</code> if the given key does not have a
+     * value in the context
+     */
+    @SuppressWarnings("unchecked") // (T)object
+    public <T> T getFromContext(String key) {
+        return context != null ? (T) context.get(key) : null;
+    }
+
+    /**
+     * Puts the object into the context
+     */
+    public void putInContext(String key, Object value) {
+        if (context == null) {
+            context = new HashMap<>();
+        }
+        context.put(key, value);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java
index 65fe7ddad1530..ebb2157d981e7 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java
@@ -22,12 +22,10 @@
 import org.apache.lucene.search.Collector;
 import org.apache.lucene.search.FieldDoc;
 import org.apache.lucene.search.Query;
-import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.util.Counter;
-import org.apache.lucene.util.RefCount;
+import org.elasticsearch.action.search.SearchTask;
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.unit.TimeValue;
@@ -35,25 +33,24 @@
 import org.elasticsearch.common.util.concurrent.AbstractRefCounted;
 import org.elasticsearch.common.util.concurrent.RefCounted;
 import org.elasticsearch.common.util.iterable.Iterables;
-import org.elasticsearch.index.analysis.AnalysisService;
 import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
 import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ObjectMapper;
+import org.elasticsearch.search.collapse.CollapseContext;
 import org.elasticsearch.index.query.ParsedQuery;
 import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.search.fetch.StoredFieldsContext;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.similarity.SimilarityService;
-import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.search.SearchExtBuilder;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.aggregations.SearchContextAggregations;
 import org.elasticsearch.search.dfs.DfsSearchResult;
 import org.elasticsearch.search.fetch.FetchPhase;
 import org.elasticsearch.search.fetch.FetchSearchResult;
-import org.elasticsearch.search.fetch.FetchSubPhase;
-import org.elasticsearch.search.fetch.FetchSubPhaseContext;
+import org.elasticsearch.search.fetch.StoredFieldsContext;
+import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 import org.elasticsearch.search.fetch.subphase.InnerHitsContext;
 import org.elasticsearch.search.fetch.subphase.ScriptFieldsContext;
@@ -66,7 +63,7 @@
 import org.elasticsearch.search.suggest.SuggestionSearchContext;
 
 import java.util.ArrayList;
-import java.util.HashMap;
+import java.util.EnumMap;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -84,35 +81,20 @@
 // For reference why we use RefCounted here see #20095
 public abstract class SearchContext extends AbstractRefCounted implements Releasable {
 
-    private static ThreadLocal<SearchContext> current = new ThreadLocal<>();
     public static final int DEFAULT_TERMINATE_AFTER = 0;
-
-    public static void setCurrent(SearchContext value) {
-        current.set(value);
-    }
-
-    public static void removeCurrent() {
-        current.remove();
-    }
-
-    public static SearchContext current() {
-        return current.get();
-    }
-
     private Map<Lifetime, List<Releasable>> clearables = null;
     private final AtomicBoolean closed = new AtomicBoolean(false);
     private InnerHitsContext innerHitsContext;
 
-    protected final ParseFieldMatcher parseFieldMatcher;
-
-    protected SearchContext(ParseFieldMatcher parseFieldMatcher) {
+    protected SearchContext() {
         super("search_context");
-        this.parseFieldMatcher = parseFieldMatcher;
     }
 
-    public ParseFieldMatcher parseFieldMatcher() {
-        return parseFieldMatcher;
-    }
+    public abstract void setTask(SearchTask task);
+
+    public abstract SearchTask getTask();
+
+    public abstract boolean isCancelled();
 
     @Override
     public final void close() {
@@ -121,8 +103,6 @@ public final void close() {
         }
     }
 
-    private boolean nowInMillisUsed;
-
     @Override
     protected final void closeInternal() {
         try {
@@ -141,10 +121,13 @@ protected void alreadyClosed() {
 
     /**
      * Should be called before executing the main query and after all other parameters have been set.
+     * @param rewrite if the set query should be rewritten against the searcher returned from {@link #searcher()}
      */
-    public abstract void preProcess();
+    public abstract void preProcess(boolean rewrite);
 
-    public abstract Query searchFilter(String[] types);
+    /** Automatically apply all required filters to the given query such as
+     *  alias filters, types filters, etc. */
+    public abstract Query buildFilteredQuery(Query query);
 
     public abstract long id();
 
@@ -160,25 +143,8 @@ protected void alreadyClosed() {
 
     public abstract float queryBoost();
 
-    public abstract SearchContext queryBoost(float queryBoost);
-
     public abstract long getOriginNanoTime();
 
-    public final long nowInMillis() {
-        nowInMillisUsed = true;
-        return nowInMillisImpl();
-    }
-
-    public final boolean nowInMillisUsed() {
-        return nowInMillisUsed;
-    }
-
-    public final void resetNowInMillisUsed() {
-        this.nowInMillisUsed = false;
-    }
-
-    protected abstract long nowInMillisImpl();
-
     public abstract ScrollContext scrollContext();
 
     public abstract SearchContext scrollContext(ScrollContext scroll);
@@ -187,7 +153,9 @@ public final void resetNowInMillisUsed() {
 
     public abstract SearchContext aggregations(SearchContextAggregations aggregations);
 
-    public abstract  <SubPhaseContext extends FetchSubPhaseContext> SubPhaseContext getFetchSubPhaseContext(FetchSubPhase.ContextFactory<SubPhaseContext> contextFactory);
+    public abstract void addSearchExt(SearchExtBuilder searchExtBuilder);
+
+    public abstract SearchExtBuilder getSearchExt(String name);
 
     public abstract SearchContextHighlight highlight();
 
@@ -226,18 +194,18 @@ public InnerHitsContext innerHits() {
 
     public abstract SearchContext fetchSourceContext(FetchSourceContext fetchSourceContext);
 
+    public abstract DocValueFieldsContext docValueFieldsContext();
+
+    public abstract SearchContext docValueFieldsContext(DocValueFieldsContext docValueFieldsContext);
+
     public abstract ContextIndexSearcher searcher();
 
     public abstract IndexShard indexShard();
 
     public abstract MapperService mapperService();
 
-    public abstract AnalysisService analysisService();
-
     public abstract SimilarityService similarityService();
 
-    public abstract ScriptService scriptService();
-
     public abstract BigArrays bigArrays();
 
     public abstract BitsetFilterCache bitsetFilterCache();
@@ -252,6 +220,14 @@ public InnerHitsContext innerHits() {
 
     public abstract void terminateAfter(int terminateAfter);
 
+    /**
+     * Indicates if the current index should perform frequent low level search cancellation check.
+     *
+     * Enabling low-level checks will make long running searches to react to the cancellation request faster. However,
+     * since it will produce more cancellation checks it might slow the search performance down.
+     */
+    public abstract boolean lowLevelCancellation();
+
     public abstract SearchContext minimumScore(float minimumScore);
 
     public abstract Float minimumScore();
@@ -268,6 +244,10 @@ public InnerHitsContext innerHits() {
 
     public abstract FieldDoc searchAfter();
 
+    public abstract SearchContext collapse(CollapseContext collapse);
+
+    public abstract CollapseContext collapse();
+
     public abstract SearchContext parsedPostFilter(ParsedQuery postFilter);
 
     public abstract ParsedQuery parsedPostFilter();
@@ -333,7 +313,9 @@ public InnerHitsContext innerHits() {
 
     public abstract void keepAlive(long keepAlive);
 
-    public abstract SearchLookup lookup();
+    public SearchLookup lookup() {
+        return getQueryShardContext().lookup();
+    }
 
     public abstract DfsSearchResult dfsResult();
 
@@ -354,7 +336,7 @@ public InnerHitsContext innerHits() {
      */
     public void addReleasable(Releasable releasable, Lifetime lifetime) {
         if (clearables == null) {
-            clearables = new HashMap<>();
+            clearables = new EnumMap<>(Lifetime.class);
         }
         List<Releasable> releasables = clearables.get(lifetime);
         if (releasables == null) {
@@ -427,7 +409,11 @@ public String toString() {
             result.append("searchType=[").append(searchType()).append("]");
         }
         if (scrollContext() != null) {
-            result.append("scroll=[").append(scrollContext().scroll.keepAlive()).append("]");
+            if (scrollContext().scroll != null) {
+                result.append("scroll=[").append(scrollContext().scroll.keepAlive()).append("]");
+            } else {
+                result.append("scroll=[null]");
+            }
         }
         result.append(" query=[").append(query()).append("]");
         return result.toString();
diff --git a/core/src/main/java/org/elasticsearch/search/internal/ShardSearchLocalRequest.java b/core/src/main/java/org/elasticsearch/search/internal/ShardSearchLocalRequest.java
index d025d573c14ce..5a8d91fbaa86c 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/ShardSearchLocalRequest.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/ShardSearchLocalRequest.java
@@ -19,21 +19,23 @@
 
 package org.elasticsearch.search.internal;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchType;
-import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 
 import java.io.IOException;
+import java.util.Optional;
 
 /**
  * Shard level search request that gets created and consumed on the local node.
@@ -57,12 +59,14 @@
 
 public class ShardSearchLocalRequest implements ShardSearchRequest {
 
+    private String clusterAlias;
     private ShardId shardId;
     private int numberOfShards;
     private SearchType searchType;
     private Scroll scroll;
     private String[] types = Strings.EMPTY_ARRAY;
-    private String[] filteringAliases;
+    private AliasFilter aliasFilter;
+    private float indexBoost;
     private SearchSourceBuilder source;
     private Boolean requestCache;
     private long nowInMillis;
@@ -72,33 +76,33 @@ public class ShardSearchLocalRequest implements ShardSearchRequest {
     ShardSearchLocalRequest() {
     }
 
-    ShardSearchLocalRequest(SearchRequest searchRequest, ShardRouting shardRouting, int numberOfShards,
-                            String[] filteringAliases, long nowInMillis) {
-        this(shardRouting.shardId(), numberOfShards, searchRequest.searchType(),
-                searchRequest.source(), searchRequest.types(), searchRequest.requestCache());
+    ShardSearchLocalRequest(SearchRequest searchRequest, ShardId shardId, int numberOfShards,
+                            AliasFilter aliasFilter, float indexBoost, long nowInMillis, String clusterAlias) {
+        this(shardId, numberOfShards, searchRequest.searchType(),
+                searchRequest.source(), searchRequest.types(), searchRequest.requestCache(), aliasFilter, indexBoost);
         this.scroll = searchRequest.scroll();
-        this.filteringAliases = filteringAliases;
         this.nowInMillis = nowInMillis;
+        this.clusterAlias = clusterAlias;
     }
 
-    public ShardSearchLocalRequest(String[] types, long nowInMillis) {
+    public ShardSearchLocalRequest(ShardId shardId, String[] types, long nowInMillis, AliasFilter aliasFilter) {
         this.types = types;
         this.nowInMillis = nowInMillis;
-    }
-
-    public ShardSearchLocalRequest(String[] types, long nowInMillis, String[] filteringAliases) {
-        this(types, nowInMillis);
-        this.filteringAliases = filteringAliases;
+        this.aliasFilter = aliasFilter;
+        this.shardId = shardId;
+        indexBoost = 1.0f;
     }
 
     public ShardSearchLocalRequest(ShardId shardId, int numberOfShards, SearchType searchType, SearchSourceBuilder source, String[] types,
-            Boolean requestCache) {
+            Boolean requestCache, AliasFilter aliasFilter, float indexBoost) {
         this.shardId = shardId;
         this.numberOfShards = numberOfShards;
         this.searchType = searchType;
         this.source = source;
         this.types = types;
         this.requestCache = requestCache;
+        this.aliasFilter = aliasFilter;
+        this.indexBoost = indexBoost;
     }
 
 
@@ -133,8 +137,13 @@ public SearchType searchType() {
     }
 
     @Override
-    public String[] filteringAliases() {
-        return filteringAliases;
+    public QueryBuilder filteringAliases() {
+        return aliasFilter.getQueryBuilder();
+    }
+
+    @Override
+    public float indexBoost() {
+        return indexBoost;
     }
 
     @Override
@@ -162,6 +171,10 @@ public boolean isProfile() {
         return profile;
     }
 
+    void setSearchType(SearchType type) {
+        this.searchType = type;
+    }
+
     protected void innerReadFrom(StreamInput in) throws IOException {
         shardId = ShardId.readShardId(in);
         searchType = SearchType.fromId(in.readByte());
@@ -169,9 +182,26 @@ protected void innerReadFrom(StreamInput in) throws IOException {
         scroll = in.readOptionalWriteable(Scroll::new);
         source = in.readOptionalWriteable(SearchSourceBuilder::new);
         types = in.readStringArray();
-        filteringAliases = in.readStringArray();
+        aliasFilter = new AliasFilter(in);
+        if (in.getVersion().onOrAfter(Version.V_5_2_0)) {
+            indexBoost = in.readFloat();
+        } else {
+            // Nodes < 5.2.0 doesn't send index boost. Read it from source.
+            if (source != null) {
+                Optional<SearchSourceBuilder.IndexBoost> boost = source.indexBoosts()
+                    .stream()
+                    .filter(ib -> ib.getIndex().equals(shardId.getIndexName()))
+                    .findFirst();
+                indexBoost = boost.isPresent() ? boost.get().getBoost() : 1.0f;
+            } else {
+                indexBoost = 1.0f;
+            }
+        }
         nowInMillis = in.readVLong();
         requestCache = in.readOptionalBoolean();
+        if (in.getVersion().onOrAfter(Version.V_5_6_0)) {
+            clusterAlias = in.readOptionalString();
+        }
     }
 
     protected void innerWriteTo(StreamOutput out, boolean asKey) throws IOException {
@@ -183,11 +213,17 @@ protected void innerWriteTo(StreamOutput out, boolean asKey) throws IOException
         out.writeOptionalWriteable(scroll);
         out.writeOptionalWriteable(source);
         out.writeStringArray(types);
-        out.writeStringArrayNullable(filteringAliases);
+        aliasFilter.writeTo(out);
+        if (out.getVersion().onOrAfter(Version.V_5_2_0)) {
+            out.writeFloat(indexBoost);
+        }
         if (!asKey) {
             out.writeVLong(nowInMillis);
         }
         out.writeOptionalBoolean(requestCache);
+        if (out.getVersion().onOrAfter(Version.V_5_6_0)) {
+            out.writeOptionalString(clusterAlias);
+        }
     }
 
     @Override
@@ -203,10 +239,16 @@ public BytesReference cacheKey() throws IOException {
     public void rewrite(QueryShardContext context) throws IOException {
         SearchSourceBuilder source = this.source;
         SearchSourceBuilder rewritten = null;
+        aliasFilter = aliasFilter.rewrite(context);
         while (rewritten != source) {
             rewritten = source.rewrite(context);
             source = rewritten;
         }
         this.source = source;
     }
+
+    @Override
+    public String getClusterAlias() {
+        return clusterAlias;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java b/core/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java
index 6c237322f0458..6c5c310eb1e72 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java
@@ -20,13 +20,24 @@
 package org.elasticsearch.search.internal;
 
 import org.elasticsearch.action.search.SearchType;
+import org.elasticsearch.cluster.metadata.AliasMetaData;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.indices.AliasFilterParsingException;
+import org.elasticsearch.indices.InvalidAliasNameException;
 import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 
 import java.io.IOException;
+import java.util.Optional;
+import java.util.function.Function;
 
 /**
  * Shard level request that represents a search.
@@ -47,7 +58,9 @@ public interface ShardSearchRequest {
 
     SearchType searchType();
 
-    String[] filteringAliases();
+    QueryBuilder filteringAliases();
+
+    float indexBoost();
 
     long nowInMillis();
 
@@ -76,4 +89,63 @@ public interface ShardSearchRequest {
      * QueryBuilder.
      */
     void rewrite(QueryShardContext context) throws IOException;
+
+    /**
+     * Returns the filter associated with listed filtering aliases.
+     * <p>
+     * The list of filtering aliases should be obtained by calling MetaData.filteringAliases.
+     * Returns <tt>null</tt> if no filtering is required.</p>
+     */
+    static QueryBuilder parseAliasFilter(CheckedFunction<byte[], Optional<QueryBuilder>, IOException> filterParser,
+                                         IndexMetaData metaData, String... aliasNames) {
+        if (aliasNames == null || aliasNames.length == 0) {
+            return null;
+        }
+        Index index = metaData.getIndex();
+        ImmutableOpenMap<String, AliasMetaData> aliases = metaData.getAliases();
+        Function<AliasMetaData, QueryBuilder> parserFunction = (alias) -> {
+            if (alias.filter() == null) {
+                return null;
+            }
+            try {
+                return filterParser.apply(alias.filter().uncompressed()).orElse(null);
+            } catch (IOException ex) {
+                throw new AliasFilterParsingException(index, alias.getAlias(), "Invalid alias filter", ex);
+            }
+        };
+        if (aliasNames.length == 1) {
+            AliasMetaData alias = aliases.get(aliasNames[0]);
+            if (alias == null) {
+                // This shouldn't happen unless alias disappeared after filteringAliases was called.
+                throw new InvalidAliasNameException(index, aliasNames[0], "Unknown alias name was passed to alias Filter");
+            }
+            return parserFunction.apply(alias);
+        } else {
+            // we need to bench here a bit, to see maybe it makes sense to use OrFilter
+            BoolQueryBuilder combined = new BoolQueryBuilder();
+            for (String aliasName : aliasNames) {
+                AliasMetaData alias = aliases.get(aliasName);
+                if (alias == null) {
+                    // This shouldn't happen unless alias disappeared after filteringAliases was called.
+                    throw new InvalidAliasNameException(index, aliasNames[0],
+                        "Unknown alias name was passed to alias Filter");
+                }
+                QueryBuilder parsedFilter = parserFunction.apply(alias);
+                if (parsedFilter != null) {
+                    combined.should(parsedFilter);
+                } else {
+                    // The filter might be null only if filter was removed after filteringAliases was called
+                    return null;
+                }
+            }
+            return combined;
+        }
+    }
+
+    /**
+     * Returns the cluster alias if this request is for a remote cluster or <code>null</code> if the request if targeted to the local
+     * cluster.
+     */
+    String getClusterAlias();
+
 }
diff --git a/core/src/main/java/org/elasticsearch/search/internal/ShardSearchTransportRequest.java b/core/src/main/java/org/elasticsearch/search/internal/ShardSearchTransportRequest.java
index 93013b94b36a8..95f2f26e9aec0 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/ShardSearchTransportRequest.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/ShardSearchTransportRequest.java
@@ -22,16 +22,19 @@
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.OriginalIndices;
 import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchTask;
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.transport.TransportRequest;
 
 import java.io.IOException;
@@ -50,10 +53,15 @@ public class ShardSearchTransportRequest extends TransportRequest implements Sha
     public ShardSearchTransportRequest(){
     }
 
-    public ShardSearchTransportRequest(SearchRequest searchRequest, ShardRouting shardRouting, int numberOfShards,
-                                       String[] filteringAliases, long nowInMillis) {
-        this.shardSearchLocalRequest = new ShardSearchLocalRequest(searchRequest, shardRouting, numberOfShards, filteringAliases, nowInMillis);
-        this.originalIndices = new OriginalIndices(searchRequest);
+    public ShardSearchTransportRequest(OriginalIndices originalIndices, SearchRequest searchRequest, ShardId shardId, int numberOfShards,
+                                       AliasFilter aliasFilter, float indexBoost, long nowInMillis, String clusterAlias) {
+        this.shardSearchLocalRequest = new ShardSearchLocalRequest(searchRequest, shardId, numberOfShards, aliasFilter, indexBoost,
+            nowInMillis, clusterAlias);
+        this.originalIndices = originalIndices;
+    }
+
+    public void searchType(SearchType searchType) {
+        shardSearchLocalRequest.setSearchType(searchType);
     }
 
     @Override
@@ -72,7 +80,6 @@ public IndicesOptions indicesOptions() {
         return originalIndices.indicesOptions();
     }
 
-
     @Override
     public ShardId shardId() {
         return shardSearchLocalRequest.shardId();
@@ -104,10 +111,15 @@ public SearchType searchType() {
     }
 
     @Override
-    public String[] filteringAliases() {
+    public QueryBuilder filteringAliases() {
         return shardSearchLocalRequest.filteringAliases();
     }
 
+    @Override
+    public float indexBoost() {
+        return shardSearchLocalRequest.indexBoost();
+    }
+
     @Override
     public long nowInMillis() {
         return shardSearchLocalRequest.nowInMillis();
@@ -129,6 +141,7 @@ public void readFrom(StreamInput in) throws IOException {
         shardSearchLocalRequest = new ShardSearchLocalRequest();
         shardSearchLocalRequest.innerReadFrom(in);
         originalIndices = OriginalIndices.readOriginalIndices(in);
+
     }
 
     @Override
@@ -157,4 +170,20 @@ public boolean isProfile() {
     public void rewrite(QueryShardContext context) throws IOException {
         shardSearchLocalRequest.rewrite(context);
     }
+
+    @Override
+    public Task createTask(long id, String type, String action, TaskId parentTaskId) {
+        return new SearchTask(id, type, action, getDescription(), parentTaskId);
+    }
+
+    @Override
+    public String getDescription() {
+        // Shard id is enough here, the request itself can be found by looking at the parent task description
+        return "shardId[" + shardSearchLocalRequest.shardId() + "]";
+    }
+
+    @Override
+    public String getClusterAlias() {
+        return shardSearchLocalRequest.getClusterAlias();
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java
index 077a0941a0bde..0a6d684ccf525 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java
@@ -22,13 +22,14 @@
 import org.apache.lucene.util.Counter;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.query.ParsedQuery;
-import org.elasticsearch.search.fetch.StoredFieldsContext;
 import org.elasticsearch.search.aggregations.SearchContextAggregations;
+import org.elasticsearch.search.collapse.CollapseContext;
 import org.elasticsearch.search.fetch.FetchSearchResult;
+import org.elasticsearch.search.fetch.StoredFieldsContext;
+import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 import org.elasticsearch.search.fetch.subphase.ScriptFieldsContext;
 import org.elasticsearch.search.fetch.subphase.highlight.SearchContextHighlight;
-import org.elasticsearch.search.lookup.SearchLookup;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.search.rescore.RescoreSearchContext;
 import org.elasticsearch.search.sort.SortAndFormats;
@@ -36,8 +37,6 @@
 
 import java.util.List;
 
-/**
- */
 public class SubSearchContext extends FilteredSearchContext {
 
     // By default return 3 hits per bucket. A higher default would make the response really large by default, since
@@ -60,6 +59,7 @@ public class SubSearchContext extends FilteredSearchContext {
     private StoredFieldsContext storedFields;
     private ScriptFieldsContext scriptFields;
     private FetchSourceContext fetchSourceContext;
+    private DocValueFieldsContext docValueFieldsContext;
     private SearchContextHighlight highlight;
 
     private boolean explain;
@@ -77,19 +77,14 @@ protected void doClose() {
     }
 
     @Override
-    public void preProcess() {
+    public void preProcess(boolean rewrite) {
     }
 
     @Override
-    public Query searchFilter(String[] types) {
+    public Query buildFilteredQuery(Query query) {
         throw new UnsupportedOperationException("this context should be read only");
     }
 
-    @Override
-    public SearchContext queryBoost(float queryBoost) {
-        throw new UnsupportedOperationException("Not supported");
-    }
-
     @Override
     public SearchContext scrollContext(ScrollContext scrollContext) {
         throw new UnsupportedOperationException("Not supported");
@@ -154,6 +149,17 @@ public SearchContext fetchSourceContext(FetchSourceContext fetchSourceContext) {
         return this;
     }
 
+    @Override
+    public DocValueFieldsContext docValueFieldsContext() {
+        return docValueFieldsContext;
+    }
+
+    @Override
+    public SearchContext docValueFieldsContext(DocValueFieldsContext docValueFieldsContext) {
+        this.docValueFieldsContext = docValueFieldsContext;
+        return this;
+    }
+
     @Override
     public void timeout(TimeValue timeout) {
         throw new UnsupportedOperationException("Not supported");
@@ -311,6 +317,11 @@ public SearchContext docIdsToLoad(int[] docIdsToLoad, int docsIdsToLoadFrom, int
         return this;
     }
 
+    @Override
+    public CollapseContext collapse() {
+        return null;
+    }
+
     @Override
     public void accessed(long accessTime) {
         throw new UnsupportedOperationException("Not supported");
@@ -331,16 +342,6 @@ public FetchSearchResult fetchResult() {
         return fetchSearchResult;
     }
 
-    private SearchLookup searchLookup;
-
-    @Override
-    public SearchLookup lookup() {
-        if (searchLookup == null) {
-            searchLookup = new SearchLookup(mapperService(), fieldData(), request().types());
-        }
-        return searchLookup;
-    }
-
     @Override
     public Counter timeEstimateCounter() {
         throw new UnsupportedOperationException("Not supported");
diff --git a/core/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java b/core/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java
index db20a03f8256b..5300202924b9c 100644
--- a/core/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java
+++ b/core/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java
@@ -33,12 +33,9 @@
 import java.util.Map;
 import java.util.Set;
 
-/**
- *
- */
-public class LeafDocLookup implements Map {
+public class LeafDocLookup implements Map<String, ScriptDocValues<?>> {
 
-    private final Map<String, ScriptDocValues> localCacheFieldData = new HashMap<>(4);
+    private final Map<String, ScriptDocValues<?>> localCacheFieldData = new HashMap<>(4);
 
     private final MapperService mapperService;
     private final IndexFieldDataService fieldDataService;
@@ -70,20 +67,20 @@ public void setDocument(int docId) {
     }
 
     @Override
-    public Object get(Object key) {
+    public ScriptDocValues<?> get(Object key) {
         // assume its a string...
         String fieldName = key.toString();
-        ScriptDocValues scriptValues = localCacheFieldData.get(fieldName);
+        ScriptDocValues<?> scriptValues = localCacheFieldData.get(fieldName);
         if (scriptValues == null) {
             final MappedFieldType fieldType = mapperService.fullName(fieldName);
             if (fieldType == null) {
-                throw new IllegalArgumentException("No field found for [" + fieldName + "] in mapping with types " + Arrays.toString(types) + "");
+                throw new IllegalArgumentException("No field found for [" + fieldName + "] in mapping with types " + Arrays.toString(types));
             }
             // load fielddata on behalf of the script: otherwise it would need additional permissions
             // to deal with pagedbytes/ramusagestimator/etc
-            scriptValues = AccessController.doPrivileged(new PrivilegedAction<ScriptDocValues>() {
+            scriptValues = AccessController.doPrivileged(new PrivilegedAction<ScriptDocValues<?>>() {
                 @Override
-                public ScriptDocValues run() {
+                public ScriptDocValues<?> run() {
                     return fieldDataService.getForField(fieldType).load(reader).getScriptValues();
                 }
             });
@@ -97,7 +94,7 @@ public ScriptDocValues run() {
     public boolean containsKey(Object key) {
         // assume its a string...
         String fieldName = key.toString();
-        ScriptDocValues scriptValues = localCacheFieldData.get(fieldName);
+        ScriptDocValues<?> scriptValues = localCacheFieldData.get(fieldName);
         if (scriptValues == null) {
             MappedFieldType fieldType = mapperService.fullName(fieldName);
             if (fieldType == null) {
@@ -123,17 +120,17 @@ public boolean containsValue(Object value) {
     }
 
     @Override
-    public Object put(Object key, Object value) {
+    public ScriptDocValues<?> put(String key, ScriptDocValues<?> value) {
         throw new UnsupportedOperationException();
     }
 
     @Override
-    public Object remove(Object key) {
+    public ScriptDocValues<?> remove(Object key) {
         throw new UnsupportedOperationException();
     }
 
     @Override
-    public void putAll(Map m) {
+    public void putAll(Map<? extends String, ? extends ScriptDocValues<?>> m) {
         throw new UnsupportedOperationException();
     }
 
@@ -143,17 +140,17 @@ public void clear() {
     }
 
     @Override
-    public Set keySet() {
+    public Set<String> keySet() {
         throw new UnsupportedOperationException();
     }
 
     @Override
-    public Collection values() {
+    public Collection<ScriptDocValues<?>> values() {
         throw new UnsupportedOperationException();
     }
 
     @Override
-    public Set entrySet() {
+    public Set<Map.Entry<String, ScriptDocValues<?>>> entrySet() {
         throw new UnsupportedOperationException();
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/lookup/LeafFieldsLookup.java b/core/src/main/java/org/elasticsearch/search/lookup/LeafFieldsLookup.java
index a5f90aa2c909a..446a51d451fc7 100644
--- a/core/src/main/java/org/elasticsearch/search/lookup/LeafFieldsLookup.java
+++ b/core/src/main/java/org/elasticsearch/search/lookup/LeafFieldsLookup.java
@@ -22,13 +22,16 @@
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.index.fieldvisitor.SingleFieldsVisitor;
+import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.UidFieldMapper;
 
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
@@ -40,6 +43,7 @@
 public class LeafFieldsLookup implements Map {
 
     private final MapperService mapperService;
+    private final boolean singleType;
 
     @Nullable
     private final String[] types;
@@ -54,6 +58,7 @@ public class LeafFieldsLookup implements Map {
 
     LeafFieldsLookup(MapperService mapperService, @Nullable String[] types, LeafReader reader) {
         this.mapperService = mapperService;
+        this.singleType = mapperService.getIndexSettings().isSingleType();
         this.types = types;
         this.reader = reader;
         this.fieldVisitor = new SingleFieldsVisitor(null);
@@ -138,18 +143,22 @@ private FieldLookup loadFieldData(String name) {
         if (data == null) {
             MappedFieldType fieldType = mapperService.fullName(name);
             if (fieldType == null) {
-                throw new IllegalArgumentException("No field found for [" + name + "] in mapping with types " + Arrays.toString(types) + "");
+                throw new IllegalArgumentException("No field found for [" + name + "] in mapping with types " + Arrays.toString(types));
             }
             data = new FieldLookup(fieldType);
             cachedFieldData.put(name, data);
         }
         if (data.fields() == null) {
             String fieldName = data.fieldType().name();
+            if (singleType && UidFieldMapper.NAME.equals(fieldName)) {
+                fieldName = IdFieldMapper.NAME;
+            }
             fieldVisitor.reset(fieldName);
             try {
                 reader.document(docId, fieldVisitor);
-                fieldVisitor.postProcess(data.fieldType());
-                data.fields(singletonMap(name, fieldVisitor.fields().get(data.fieldType().name())));
+                fieldVisitor.postProcess(mapperService);
+                List<Object> storedFields = fieldVisitor.fields().get(data.fieldType().name());
+                data.fields(singletonMap(name, storedFields));
             } catch (IOException e) {
                 throw new ElasticsearchParseException("failed to load field [{}]", e, name);
             }
diff --git a/core/src/main/java/org/elasticsearch/search/lookup/LeafIndexLookup.java b/core/src/main/java/org/elasticsearch/search/lookup/LeafIndexLookup.java
index a7b965dbfb1b0..9908f2830f85b 100644
--- a/core/src/main/java/org/elasticsearch/search/lookup/LeafIndexLookup.java
+++ b/core/src/main/java/org/elasticsearch/search/lookup/LeafIndexLookup.java
@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.search.lookup;
 
+import org.apache.logging.log4j.Logger;
 import org.apache.lucene.index.Fields;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexReaderContext;
@@ -26,6 +27,8 @@
 import org.apache.lucene.index.ReaderUtil;
 import org.apache.lucene.search.IndexSearcher;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.util.MinimalMap;
 
 import java.io.IOException;
@@ -64,7 +67,19 @@ public class LeafIndexLookup extends MinimalMap<String, IndexField> {
     // computation is expensive
     private int numDeletedDocs = -1;
 
+    private boolean deprecationEmitted = false;
+
+    private void logDeprecation() {
+        if (deprecationEmitted == false) {
+            Logger logger = Loggers.getLogger(getClass());
+            DeprecationLogger deprecationLogger = new DeprecationLogger(logger);
+            deprecationLogger.deprecated("Using _index is deprecated. Create a custom ScriptEngine to access index internals.");
+            deprecationEmitted = true;
+        }
+    }
+
     public int numDocs() {
+        logDeprecation();
         if (numDocs == -1) {
             numDocs = parentReader.numDocs();
         }
@@ -72,6 +87,7 @@ public int numDocs() {
     }
 
     public int maxDoc() {
+        logDeprecation();
         if (maxDoc == -1) {
             maxDoc = parentReader.maxDoc();
         }
@@ -79,6 +95,7 @@ public int maxDoc() {
     }
 
     public int numDeletedDocs() {
+        logDeprecation();
         if (numDeletedDocs == -1) {
             numDeletedDocs = parentReader.numDeletedDocs();
         }
@@ -127,6 +144,7 @@ protected void setNextDocIdInFields() {
      */
     @Override
     public IndexField get(Object key) {
+        logDeprecation();
         String stringField = (String) key;
         IndexField indexField = indexFields.get(key);
         if (indexField == null) {
@@ -146,19 +164,23 @@ public IndexField get(Object key) {
      * *
      */
     public Fields termVectors() throws IOException {
+        logDeprecation();
         assert reader != null;
         return reader.getTermVectors(docId);
     }
 
     LeafReader getReader() {
+        logDeprecation();
         return reader;
     }
 
     public int getDocId() {
+        logDeprecation();
         return docId;
     }
 
     public IndexReader getParentReader() {
+        logDeprecation();
         if (parentReader == null) {
             return reader;
         }
@@ -166,10 +188,12 @@ public IndexReader getParentReader() {
     }
 
     public IndexSearcher getIndexSearcher() {
+        logDeprecation();
         return indexSearcher;
     }
 
     public IndexReaderContext getReaderContext() {
+        logDeprecation();
         return getParentReader().getContext();
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java b/core/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java
index 910f5daf7a3af..01610c6405f2b 100644
--- a/core/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java
+++ b/core/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.index.fieldvisitor.FieldsVisitor;
+import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 
 import java.util.Collection;
 import java.util.List;
@@ -130,8 +131,8 @@ public List<Object> extractRawValues(String path) {
         return XContentMapValues.extractRawValues(path, loadSourceIfNeeded());
     }
 
-    public Object filter(String[] includes, String[] excludes) {
-        return XContentMapValues.filter(loadSourceIfNeeded(), includes, excludes);
+    public Object filter(FetchSourceContext context) {
+        return context.getFilter().apply(loadSourceIfNeeded());
     }
 
     public Object extractValue(String path) {
diff --git a/core/src/main/java/org/elasticsearch/search/profile/AbstractInternalProfileTree.java b/core/src/main/java/org/elasticsearch/search/profile/AbstractInternalProfileTree.java
index 5d20080408b94..a93d3c419ad19 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/AbstractInternalProfileTree.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/AbstractInternalProfileTree.java
@@ -162,10 +162,9 @@ private ProfileResult doGetTree(int token) {
 
         // TODO this would be better done bottom-up instead of top-down to avoid
         // calculating the same times over and over...but worth the effort?
-        long nodeTime = getNodeTime(timings);
         String type = getTypeFromElement(element);
         String description = getDescriptionFromElement(element);
-        return new ProfileResult(type, description, timings, childrenProfileResults, nodeTime);
+        return new ProfileResult(type, description, timings, childrenProfileResults);
     }
 
     protected abstract String getTypeFromElement(E element);
@@ -184,19 +183,4 @@ private void updateParent(int childToken) {
         tree.set(parent, parentNode);
     }
 
-    /**
-     * Internal helper to calculate the time of a node, inclusive of children
-     *
-     * @param timings
-     *            A map of breakdown timing for the node
-     * @return The total time at this node, inclusive of children
-     */
-    private static long getNodeTime(Map<String, Long> timings) {
-        long nodeTime = 0;
-        for (long time : timings.values()) {
-            nodeTime += time;
-        }
-        return nodeTime;
-    }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/search/profile/AbstractProfileBreakdown.java b/core/src/main/java/org/elasticsearch/search/profile/AbstractProfileBreakdown.java
index 3698ea07da11f..f49ad4a8718a0 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/AbstractProfileBreakdown.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/AbstractProfileBreakdown.java
@@ -33,77 +33,29 @@ public abstract class AbstractProfileBreakdown<T extends Enum<T>> {
     /**
      * The accumulated timings for this query node
      */
-    private final long[] timings;
-
-    private final long[] counts;
-
-    /** Scratch to store the current timing type. */
-    private T currentTimingType;
-
-    /**
-     * The temporary scratch space for holding start-times
-     */
-    private long scratch;
-
-    private T[] timingTypes;
+    private final Timer[] timings;
+    private final T[] timingTypes;
 
     /** Sole constructor. */
-    public AbstractProfileBreakdown(T[] timingTypes) {
-        this.timingTypes = timingTypes;
-        timings = new long[timingTypes.length];
-        counts = new long[timingTypes.length];
-    }
-
-    /**
-     * Begin timing a query for a specific Timing context
-     * @param timing    The timing context being profiled
-     */
-    public void startTime(T timing) {
-        assert currentTimingType == null;
-        assert scratch == 0;
-        counts[timing.ordinal()] += 1;
-        currentTimingType = timing;
-        scratch = System.nanoTime();
+    public AbstractProfileBreakdown(Class<T> clazz) {
+        this.timingTypes = clazz.getEnumConstants();
+        timings = new Timer[timingTypes.length];
+        for (int i = 0; i < timings.length; ++i) {
+            timings[i] = new Timer();
+        }
     }
 
-    /**
-     * Halt the timing process and save the elapsed time.
-     * startTime() must be called for a particular context prior to calling
-     * stopAndRecordTime(), otherwise the elapsed time will be negative and
-     * nonsensical
-     *
-     * @return          The elapsed time
-     */
-    public long stopAndRecordTime() {
-        long time = Math.max(1, System.nanoTime() - scratch);
-        timings[currentTimingType.ordinal()] += time;
-        currentTimingType = null;
-        scratch = 0L;
-        return time;
+    public Timer getTimer(T timing) {
+        return timings[timing.ordinal()];
     }
 
     /** Convert this record to a map from timingType to times. */
     public Map<String, Long> toTimingMap() {
         Map<String, Long> map = new HashMap<>();
         for (T timingType : timingTypes) {
-            map.put(timingType.toString(), timings[timingType.ordinal()]);
-            map.put(timingType.toString() + "_count", counts[timingType.ordinal()]);
+            map.put(timingType.toString(), timings[timingType.ordinal()].getApproximateTiming());
+            map.put(timingType.toString() + "_count", timings[timingType.ordinal()].getCount());
         }
         return Collections.unmodifiableMap(map);
     }
-
-    /**
-     * Add <code>other</code>'s timings into this breakdown
-     * @param other Another Breakdown to merge with this one
-     */
-    public void merge(AbstractProfileBreakdown<T> other) {
-        assert(timings.length == other.timings.length);
-        for (int i = 0; i < timings.length; ++i) {
-            timings[i] += other.timings[i];
-        }
-        assert(counts.length == other.counts.length);
-        for (int i = 0; i < counts.length; ++i) {
-            counts[i] += other.counts[i];
-        }
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/profile/ProfileResult.java b/core/src/main/java/org/elasticsearch/search/profile/ProfileResult.java
index 4c745a5127dab..9ffc58bf7386d 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/ProfileResult.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/ProfileResult.java
@@ -23,8 +23,9 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -33,6 +34,9 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Objects;
+
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
 
 /**
  * This class is the internal representation of a profiled Query, corresponding
@@ -43,13 +47,14 @@
  * Each InternalProfileResult has a List of InternalProfileResults, which will contain
  * "children" queries if applicable
  */
-public final class ProfileResult implements Writeable, ToXContent {
+public final class ProfileResult implements Writeable, ToXContentObject {
 
-    private static final ParseField TYPE = new ParseField("type");
-    private static final ParseField DESCRIPTION = new ParseField("description");
-    private static final ParseField NODE_TIME = new ParseField("time");
-    private static final ParseField CHILDREN = new ParseField("children");
-    private static final ParseField BREAKDOWN = new ParseField("breakdown");
+    static final ParseField TYPE = new ParseField("type");
+    static final ParseField DESCRIPTION = new ParseField("description");
+    static final ParseField NODE_TIME = new ParseField("time");
+    static final ParseField NODE_TIME_NANOS = new ParseField("time_in_nanos");
+    static final ParseField CHILDREN = new ParseField("children");
+    static final ParseField BREAKDOWN = new ParseField("breakdown");
 
     private final String type;
     private final String description;
@@ -57,13 +62,12 @@ public final class ProfileResult implements Writeable, ToXContent {
     private final long nodeTime;
     private final List<ProfileResult> children;
 
-    public ProfileResult(String type, String description, Map<String, Long> timings, List<ProfileResult> children,
-            long nodeTime) {
+    public ProfileResult(String type, String description, Map<String, Long> timings, List<ProfileResult> children) {
         this.type = type;
         this.description = description;
-        this.timings = timings;
+        this.timings = Objects.requireNonNull(timings, "required timings argument missing");
         this.children = children;
-        this.nodeTime = nodeTime;
+        this.nodeTime = getTotalTime(timings);
     }
 
     /**
@@ -147,6 +151,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                 .field(TYPE.getPreferredName(), type)
                 .field(DESCRIPTION.getPreferredName(), description)
                 .field(NODE_TIME.getPreferredName(), String.format(Locale.US, "%.10gms", getTime() / 1000000.0))
+                .field(NODE_TIME_NANOS.getPreferredName(), getTime())
                 .field(BREAKDOWN.getPreferredName(), timings);
 
         if (!children.isEmpty()) {
@@ -161,4 +166,65 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         return builder;
     }
 
+    public static ProfileResult fromXContent(XContentParser parser) throws IOException {
+        XContentParser.Token token = parser.currentToken();
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser::getTokenLocation);
+        String currentFieldName = null;
+        String type = null, description = null;
+        Map<String, Long> timings =  new HashMap<>();
+        List<ProfileResult> children = new ArrayList<>();
+        while((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token.isValue()) {
+                if (TYPE.match(currentFieldName)) {
+                    type = parser.text();
+                } else if (DESCRIPTION.match(currentFieldName)) {
+                    description = parser.text();
+                } else if (NODE_TIME.match(currentFieldName)) {
+                    // skip, total time is calculate by adding up 'timings' values in ProfileResult ctor
+                    parser.text();
+                } else if (NODE_TIME_NANOS.match(currentFieldName)) {
+                    // skip, total time is calculate by adding up 'timings' values in ProfileResult ctor
+                    parser.longValue();
+                } else {
+                    parser.skipChildren();
+                }
+            } else if (token == XContentParser.Token.START_OBJECT) {
+                if (BREAKDOWN.match(currentFieldName)) {
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                        ensureExpectedToken(parser.currentToken(), XContentParser.Token.FIELD_NAME, parser::getTokenLocation);
+                        String name = parser.currentName();
+                        ensureExpectedToken(parser.nextToken(), XContentParser.Token.VALUE_NUMBER, parser::getTokenLocation);
+                        long value = parser.longValue();
+                        timings.put(name, value);
+                    }
+                } else {
+                    parser.skipChildren();
+                }
+            } else if (token == XContentParser.Token.START_ARRAY) {
+                if (CHILDREN.match(currentFieldName)) {
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                        children.add(ProfileResult.fromXContent(parser));
+                    }
+                } else {
+                    parser.skipChildren();
+                }
+            }
+        }
+        return new ProfileResult(type, description, timings, children);
+    }
+
+    /**
+     * @param timings a map of breakdown timing for the node
+     * @return The total time at this node
+     */
+    private static long getTotalTime(Map<String, Long> timings) {
+        long nodeTime = 0;
+        for (long time : timings.values()) {
+            nodeTime += time;
+        }
+        return nodeTime;
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/search/profile/SearchProfileShardResults.java b/core/src/main/java/org/elasticsearch/search/profile/SearchProfileShardResults.java
index 6794aa49399cb..b7fa39c42f3ab 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/SearchProfileShardResults.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/SearchProfileShardResults.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.search.profile.aggregation.AggregationProfileShardResult;
 import org.elasticsearch.search.profile.aggregation.AggregationProfiler;
 import org.elasticsearch.search.profile.query.QueryProfileShardResult;
@@ -35,6 +36,9 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.TreeSet;
+
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
 
 /**
  * A container class to hold all the profile results across all shards.  Internally
@@ -42,6 +46,11 @@
  */
 public final class SearchProfileShardResults implements Writeable, ToXContent{
 
+    private static final String SEARCHES_FIELD = "searches";
+    private static final String ID_FIELD = "id";
+    private static final String SHARDS_FIELD = "shards";
+    public static final String PROFILE_FIELD = "profile";
+
     private Map<String, ProfileShardResult> shardResults;
 
     public SearchProfileShardResults(Map<String, ProfileShardResult> shardResults) {
@@ -75,26 +84,80 @@ public void writeTo(StreamOutput out) throws IOException {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject("profile").startArray("shards");
-
-        for (Map.Entry<String, ProfileShardResult> entry : shardResults.entrySet()) {
+        builder.startObject(PROFILE_FIELD).startArray(SHARDS_FIELD);
+        // shardResults is a map, but we print entries in a json array, which is ordered.
+        // we sort the keys of the map, so that toXContent always prints out the same array order
+        TreeSet<String> sortedKeys = new TreeSet<>(shardResults.keySet());
+        for (String key : sortedKeys) {
             builder.startObject();
-            builder.field("id", entry.getKey());
-            builder.startArray("searches");
-            for (QueryProfileShardResult result : entry.getValue().getQueryProfileResults()) {
-                builder.startObject();
+            builder.field(ID_FIELD, key);
+            builder.startArray(SEARCHES_FIELD);
+            ProfileShardResult profileShardResult = shardResults.get(key);
+            for (QueryProfileShardResult result : profileShardResult.getQueryProfileResults()) {
                 result.toXContent(builder, params);
-                builder.endObject();
             }
             builder.endArray();
-            entry.getValue().getAggregationProfileResults().toXContent(builder, params);
+            profileShardResult.getAggregationProfileResults().toXContent(builder, params);
             builder.endObject();
         }
-
         builder.endArray().endObject();
         return builder;
     }
 
+    public static SearchProfileShardResults fromXContent(XContentParser parser) throws IOException {
+        XContentParser.Token token = parser.currentToken();
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser::getTokenLocation);
+        Map<String, ProfileShardResult> searchProfileResults = new HashMap<>();
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.START_ARRAY) {
+                if (SHARDS_FIELD.equals(parser.currentName())) {
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                        parseSearchProfileResultsEntry(parser, searchProfileResults);
+                    }
+                } else {
+                    parser.skipChildren();
+                }
+            } else if (token == XContentParser.Token.START_OBJECT) {
+                parser.skipChildren();
+            }
+        }
+        return new SearchProfileShardResults(searchProfileResults);
+    }
+
+    private static void parseSearchProfileResultsEntry(XContentParser parser,
+            Map<String, ProfileShardResult> searchProfileResults) throws IOException {
+        XContentParser.Token token = parser.currentToken();
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser::getTokenLocation);
+        List<QueryProfileShardResult> queryProfileResults = new ArrayList<>();
+        AggregationProfileShardResult aggProfileShardResult = null;
+        String id = null;
+        String currentFieldName = null;
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token.isValue()) {
+                if (ID_FIELD.equals(currentFieldName)) {
+                    id = parser.text();
+                } else {
+                    parser.skipChildren();
+                }
+            } else if (token == XContentParser.Token.START_ARRAY) {
+                if (SEARCHES_FIELD.equals(currentFieldName)) {
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                        queryProfileResults.add(QueryProfileShardResult.fromXContent(parser));
+                    }
+                } else if (AggregationProfileShardResult.AGGREGATIONS.equals(currentFieldName)) {
+                    aggProfileShardResult = AggregationProfileShardResult.fromXContent(parser);
+                } else {
+                    parser.skipChildren();
+                }
+            } else {
+                parser.skipChildren();
+            }
+        }
+        searchProfileResults.put(id, new ProfileShardResult(queryProfileResults, aggProfileShardResult));
+    }
+
     /**
      * Helper method to convert Profiler into InternalProfileShardResults, which
      * can be serialized to other nodes, emitted as JSON, etc.
diff --git a/core/src/main/java/org/elasticsearch/search/profile/Timer.java b/core/src/main/java/org/elasticsearch/search/profile/Timer.java
new file mode 100644
index 0000000000000..72b62ec562244
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/profile/Timer.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.profile;
+
+/** Helps measure how much time is spent running some methods.
+ *  The {@link #start()} and {@link #stop()} methods should typically be called
+ *  in a try/finally clause with {@link #start()} being called right before the
+ *  try block and {@link #stop()} being called at the beginning of the finally
+ *  block:
+ *  <pre>
+ *  timer.start();
+ *  try {
+ *    // code to time
+ *  } finally {
+ *    timer.stop();
+ *  }
+ *  </pre>
+ */
+public class Timer {
+
+    private boolean doTiming;
+    private long timing, count, lastCount, start;
+
+    /** pkg-private for testing */
+    long nanoTime() {
+        return System.nanoTime();
+    }
+
+    /** Start the timer. */
+    public final void start() {
+        assert start == 0 : "#start call misses a matching #stop call";
+        // We measure the timing of each method call for the first 256
+        // calls, then 1/2 call up to 512 then 1/3 up to 768, etc. with
+        // a maximum interval of 1024, which is reached for 1024*2^8 ~= 262000
+        // This allows to not slow down things too much because of calls
+        // to System.nanoTime() when methods are called millions of time
+        // in tight loops, while still providing useful timings for methods
+        // that are only called a couple times per search execution.
+        doTiming = (count - lastCount) >= Math.min(lastCount >>> 8, 1024);
+        if (doTiming) {
+            start = nanoTime();
+        }
+        count++;
+    }
+
+    /** Stop the timer. */
+    public final void stop() {
+        if (doTiming) {
+            timing += (count - lastCount) * Math.max(nanoTime() - start, 1L);
+            lastCount = count;
+            start = 0;
+        }
+    }
+
+    /** Return the number of times that {@link #start()} has been called. */
+    public final long getCount() {
+        if (start != 0) {
+            throw new IllegalStateException("#start call misses a matching #stop call");
+        }
+        return count;
+    }
+
+    /** Return an approximation of the total time spent between consecutive calls of #start and #stop. */
+    public final long getApproximateTiming() {
+        if (start != 0) {
+            throw new IllegalStateException("#start call misses a matching #stop call");
+        }
+        // We don't have timings for the last `count-lastCount` method calls
+        // so we assume that they had the same timing as the lastCount first
+        // calls. This approximation is ok since at most 1/256th of method
+        // calls have not been timed.
+        long timing = this.timing;
+        if (count > lastCount) {
+            assert lastCount > 0;
+            timing += (count - lastCount) * timing / lastCount;
+        }
+        return timing;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationProfileBreakdown.java b/core/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationProfileBreakdown.java
index b4cb1efe5d350..84a525bf90784 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationProfileBreakdown.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationProfileBreakdown.java
@@ -24,7 +24,7 @@
 public class AggregationProfileBreakdown extends AbstractProfileBreakdown<AggregationTimingType> {
 
     public AggregationProfileBreakdown() {
-        super(AggregationTimingType.values());
+        super(AggregationTimingType.class);
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationProfileShardResult.java b/core/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationProfileShardResult.java
index df55c5592d6a4..95a41401d11bc 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationProfileShardResult.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationProfileShardResult.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.search.profile.ProfileResult;
 
 import java.io.IOException;
@@ -31,12 +32,15 @@
 import java.util.Collections;
 import java.util.List;
 
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+
 /**
  * A container class to hold the profile results for a single shard in the request.
  * Contains a list of query profiles, a collector tree and a total rewrite tree.
  */
 public final class AggregationProfileShardResult implements Writeable, ToXContent {
 
+    public static final String AGGREGATIONS = "aggregations";
     private final List<ProfileResult> aggProfileResults;
 
     public AggregationProfileShardResult(List<ProfileResult> aggProfileResults) {
@@ -69,11 +73,21 @@ public List<ProfileResult> getProfileResults() {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startArray("aggregations");
+        builder.startArray(AGGREGATIONS);
         for (ProfileResult p : aggProfileResults) {
             p.toXContent(builder, params);
         }
         builder.endArray();
         return builder;
     }
+
+    public static AggregationProfileShardResult fromXContent(XContentParser parser) throws IOException {
+        XContentParser.Token token = parser.currentToken();
+        ensureExpectedToken(XContentParser.Token.START_ARRAY, token, parser::getTokenLocation);
+        List<ProfileResult> aggProfileResults = new ArrayList<>();
+        while((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+            aggProfileResults.add(ProfileResult.fromXContent(parser));
+        }
+        return new AggregationProfileShardResult(aggProfileResults);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/profile/aggregation/ProfilingAggregator.java b/core/src/main/java/org/elasticsearch/search/profile/aggregation/ProfilingAggregator.java
index 2883c2903e8f1..d96fbe0d86697 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/aggregation/ProfilingAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/aggregation/ProfilingAggregator.java
@@ -23,7 +23,8 @@
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.LeafBucketCollector;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
+import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.search.profile.Timer;
 
 import java.io.IOException;
 
@@ -54,7 +55,7 @@ public String name() {
     }
 
     @Override
-    public AggregationContext context() {
+    public SearchContext context() {
         return delegate.context();
     }
 
@@ -70,9 +71,14 @@ public Aggregator subAggregator(String name) {
 
     @Override
     public InternalAggregation buildAggregation(long bucket) throws IOException {
-        profileBreakdown.startTime(AggregationTimingType.BUILD_AGGREGATION);
-        InternalAggregation result = delegate.buildAggregation(bucket);
-        profileBreakdown.stopAndRecordTime();
+        Timer timer = profileBreakdown.getTimer(AggregationTimingType.BUILD_AGGREGATION);
+        timer.start();
+        InternalAggregation result;
+        try {
+            result = delegate.buildAggregation(bucket);
+        } finally {
+            timer.stop();
+        }
         return result;
     }
 
@@ -89,9 +95,13 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx) throws IOExce
     @Override
     public void preCollection() throws IOException {
         this.profileBreakdown = profiler.getQueryBreakdown(delegate);
-        profileBreakdown.startTime(AggregationTimingType.INITIALIZE);
-        delegate.preCollection();
-        profileBreakdown.stopAndRecordTime();
+        Timer timer = profileBreakdown.getTimer(AggregationTimingType.INITIALIZE);
+        timer.start();
+        try {
+            delegate.preCollection();
+        } finally {
+            timer.stop();
+        }
         profiler.pollLastElement();
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/profile/aggregation/ProfilingLeafBucketCollector.java b/core/src/main/java/org/elasticsearch/search/profile/aggregation/ProfilingLeafBucketCollector.java
index addf910bc56a5..4db67967dcb2b 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/aggregation/ProfilingLeafBucketCollector.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/aggregation/ProfilingLeafBucketCollector.java
@@ -21,24 +21,28 @@
 
 import org.apache.lucene.search.Scorer;
 import org.elasticsearch.search.aggregations.LeafBucketCollector;
+import org.elasticsearch.search.profile.Timer;
 
 import java.io.IOException;
 
 public class ProfilingLeafBucketCollector extends LeafBucketCollector {
 
     private LeafBucketCollector delegate;
-    private AggregationProfileBreakdown profileBreakdown;
+    private Timer collectTimer;
 
     public ProfilingLeafBucketCollector(LeafBucketCollector delegate, AggregationProfileBreakdown profileBreakdown) {
         this.delegate = delegate;
-        this.profileBreakdown = profileBreakdown;
+        this.collectTimer = profileBreakdown.getTimer(AggregationTimingType.COLLECT);
     }
 
     @Override
     public void collect(int doc, long bucket) throws IOException {
-        profileBreakdown.startTime(AggregationTimingType.COLLECT);
-        delegate.collect(doc, bucket);
-        profileBreakdown.stopAndRecordTime();
+        collectTimer.start();
+        try {
+            delegate.collect(doc, bucket);
+        } finally {
+            collectTimer.stop();
+        }
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/profile/query/CollectorResult.java b/core/src/main/java/org/elasticsearch/search/profile/query/CollectorResult.java
index c517c8730e481..26623d300a159 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/query/CollectorResult.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/query/CollectorResult.java
@@ -24,19 +24,23 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Locale;
 
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+
 /**
  * Public interface and serialization container for profiled timings of the
  * Collectors used in the search.  Children CollectorResult's may be
  * embedded inside of a parent CollectorResult
  */
-public class CollectorResult implements ToXContent, Writeable {
+public class CollectorResult implements ToXContentObject, Writeable {
 
     public static final String REASON_SEARCH_COUNT = "search_count";
     public static final String REASON_SEARCH_TOP_HITS = "search_top_hits";
@@ -45,12 +49,14 @@ public class CollectorResult implements ToXContent, Writeable {
     public static final String REASON_SEARCH_MIN_SCORE = "search_min_score";
     public static final String REASON_SEARCH_MULTI = "search_multi";
     public static final String REASON_SEARCH_TIMEOUT = "search_timeout";
+    public static final String REASON_SEARCH_CANCELLED = "search_cancelled";
     public static final String REASON_AGGREGATION = "aggregation";
     public static final String REASON_AGGREGATION_GLOBAL = "aggregation_global";
 
     private static final ParseField NAME = new ParseField("name");
     private static final ParseField REASON = new ParseField("reason");
     private static final ParseField TIME = new ParseField("time");
+    private static final ParseField TIME_NANOS = new ParseField("time_in_nanos");
     private static final ParseField CHILDREN = new ParseField("children");
 
     /**
@@ -139,7 +145,8 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par
         builder = builder.startObject()
                 .field(NAME.getPreferredName(), getName())
                 .field(REASON.getPreferredName(), getReason())
-                .field(TIME.getPreferredName(), String.format(Locale.US, "%.10gms", (double) (getTime() / 1000000.0)));
+                .field(TIME.getPreferredName(), String.format(Locale.US, "%.10gms", getTime() / 1000000.0))
+                .field(TIME_NANOS.getPreferredName(), getTime());
 
         if (!children.isEmpty()) {
             builder = builder.startArray(CHILDREN.getPreferredName());
@@ -151,4 +158,42 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par
         builder = builder.endObject();
         return builder;
     }
+
+    public static CollectorResult fromXContent(XContentParser parser) throws IOException {
+        XContentParser.Token token = parser.currentToken();
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser::getTokenLocation);
+        String currentFieldName = null;
+        String name = null, reason = null;
+        long time = -1;
+        List<CollectorResult> children = new ArrayList<>();
+        while((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token.isValue()) {
+                if (NAME.match(currentFieldName)) {
+                    name = parser.text();
+                } else if (REASON.match(currentFieldName)) {
+                    reason = parser.text();
+                } else if (TIME.match(currentFieldName)) {
+                    // we need to consume this value, but we use the raw nanosecond value
+                    parser.text();
+                } else if (TIME_NANOS.match(currentFieldName)) {
+                    time = parser.longValue();
+                } else {
+                    parser.skipChildren();
+                }
+            } else if (token == XContentParser.Token.START_ARRAY) {
+                if (CHILDREN.match(currentFieldName)) {
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                        children.add(CollectorResult.fromXContent(parser));
+                    }
+                } else {
+                    parser.skipChildren();
+                }
+            } else {
+                parser.skipChildren();
+            }
+        }
+        return new CollectorResult(name, reason, time, children);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/profile/query/ProfileCollector.java b/core/src/main/java/org/elasticsearch/search/profile/query/ProfileCollector.java
index c65d097890627..ea8dbb2f335ca 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/query/ProfileCollector.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/query/ProfileCollector.java
@@ -34,7 +34,7 @@ final class ProfileCollector extends FilterCollector {
     private long time;
 
     /** Sole constructor. */
-    public ProfileCollector(Collector in) {
+    ProfileCollector(Collector in) {
         super(in);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/profile/query/ProfileScorer.java b/core/src/main/java/org/elasticsearch/search/profile/query/ProfileScorer.java
index 51d0b14fc9601..e475bb6b7d90c 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/query/ProfileScorer.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/query/ProfileScorer.java
@@ -23,6 +23,7 @@
 import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.TwoPhaseIterator;
 import org.apache.lucene.search.Weight;
+import org.elasticsearch.search.profile.Timer;
 
 import java.io.IOException;
 import java.util.Collection;
@@ -35,13 +36,16 @@ final class ProfileScorer extends Scorer {
 
     private final Scorer scorer;
     private ProfileWeight profileWeight;
-    private final QueryProfileBreakdown profile;
+    private final Timer scoreTimer, nextDocTimer, advanceTimer, matchTimer;
 
     ProfileScorer(ProfileWeight w, Scorer scorer, QueryProfileBreakdown profile) throws IOException {
         super(w);
         this.scorer = scorer;
         this.profileWeight = w;
-        this.profile = profile;
+        scoreTimer = profile.getTimer(QueryTimingType.SCORE);
+        nextDocTimer = profile.getTimer(QueryTimingType.NEXT_DOC);
+        advanceTimer = profile.getTimer(QueryTimingType.ADVANCE);
+        matchTimer = profile.getTimer(QueryTimingType.MATCH);
     }
 
     @Override
@@ -51,11 +55,11 @@ public int docID() {
 
     @Override
     public float score() throws IOException {
-        profile.startTime(QueryTimingType.SCORE);
+        scoreTimer.start();
         try {
             return scorer.score();
         } finally {
-            profile.stopAndRecordTime();
+            scoreTimer.stop();
         }
     }
 
@@ -70,7 +74,7 @@ public Weight getWeight() {
     }
 
     @Override
-    public Collection<ChildScorer> getChildren() {
+    public Collection<ChildScorer> getChildren() throws IOException {
         return scorer.getChildren();
     }
 
@@ -81,21 +85,21 @@ public DocIdSetIterator iterator() {
 
             @Override
             public int advance(int target) throws IOException {
-                profile.startTime(QueryTimingType.ADVANCE);
+                advanceTimer.start();
                 try {
                     return in.advance(target);
                 } finally {
-                    profile.stopAndRecordTime();
+                    advanceTimer.stop();
                 }
             }
 
             @Override
             public int nextDoc() throws IOException {
-                profile.startTime(QueryTimingType.NEXT_DOC);
+                nextDocTimer.start();
                 try {
                     return in.nextDoc();
                 } finally {
-                    profile.stopAndRecordTime();
+                    nextDocTimer.stop();
                 }
             }
 
@@ -122,21 +126,21 @@ public TwoPhaseIterator twoPhaseIterator() {
 
             @Override
             public int advance(int target) throws IOException {
-                profile.startTime(QueryTimingType.ADVANCE);
+                advanceTimer.start();
                 try {
                     return inApproximation.advance(target);
                 } finally {
-                    profile.stopAndRecordTime();
+                    advanceTimer.stop();
                 }
             }
 
             @Override
             public int nextDoc() throws IOException {
-                profile.startTime(QueryTimingType.NEXT_DOC);
+                nextDocTimer.start();
                 try {
                     return inApproximation.nextDoc();
                 } finally {
-                    profile.stopAndRecordTime();
+                    nextDocTimer.stop();
                 }
             }
 
@@ -153,11 +157,11 @@ public long cost() {
         return new TwoPhaseIterator(approximation) {
             @Override
             public boolean matches() throws IOException {
-                profile.startTime(QueryTimingType.MATCH);
+                matchTimer.start();
                 try {
                     return in.matches();
                 } finally {
-                    profile.stopAndRecordTime();
+                    matchTimer.stop();
                 }
             }
 
diff --git a/core/src/main/java/org/elasticsearch/search/profile/query/ProfileWeight.java b/core/src/main/java/org/elasticsearch/search/profile/query/ProfileWeight.java
index 9ca33f8454253..a080c22f78f72 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/query/ProfileWeight.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/query/ProfileWeight.java
@@ -25,7 +25,9 @@
 import org.apache.lucene.search.Explanation;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.ScorerSupplier;
 import org.apache.lucene.search.Weight;
+import org.elasticsearch.search.profile.Timer;
 
 import java.io.IOException;
 import java.util.Set;
@@ -48,18 +50,50 @@ public ProfileWeight(Query query, Weight subQueryWeight, QueryProfileBreakdown p
 
     @Override
     public Scorer scorer(LeafReaderContext context) throws IOException {
-        profile.startTime(QueryTimingType.BUILD_SCORER);
-        final Scorer subQueryScorer;
+        ScorerSupplier supplier = scorerSupplier(context);
+        if (supplier == null) {
+            return null;
+        }
+        return supplier.get(false);
+    }
+
+    @Override
+    public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
+        Timer timer = profile.getTimer(QueryTimingType.BUILD_SCORER);
+        timer.start();
+        final ScorerSupplier subQueryScorerSupplier;
         try {
-            subQueryScorer = subQueryWeight.scorer(context);
+            subQueryScorerSupplier = subQueryWeight.scorerSupplier(context);
         } finally {
-            profile.stopAndRecordTime();
+            timer.stop();
         }
-        if (subQueryScorer == null) {
+        if (subQueryScorerSupplier == null) {
             return null;
         }
 
-        return new ProfileScorer(this, subQueryScorer, profile);
+        final ProfileWeight weight = this;
+        return new ScorerSupplier() {
+
+            @Override
+            public Scorer get(boolean randomAccess) throws IOException {
+                timer.start();
+                try {
+                    return new ProfileScorer(weight, subQueryScorerSupplier.get(randomAccess), profile);
+                } finally {
+                    timer.stop();
+                }
+            }
+
+            @Override
+            public long cost() {
+                timer.start();
+                try {
+                    return subQueryScorerSupplier.cost();
+                } finally {
+                    timer.stop();
+                }
+            }
+        };
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/profile/query/QueryProfileBreakdown.java b/core/src/main/java/org/elasticsearch/search/profile/query/QueryProfileBreakdown.java
index d0608eb01afa1..21ec507dbafef 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/query/QueryProfileBreakdown.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/query/QueryProfileBreakdown.java
@@ -30,6 +30,6 @@ public final class QueryProfileBreakdown extends AbstractProfileBreakdown<QueryT
 
     /** Sole constructor. */
     public QueryProfileBreakdown() {
-        super(QueryTimingType.values());
+        super(QueryTimingType.class);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/profile/query/QueryProfileShardResult.java b/core/src/main/java/org/elasticsearch/search/profile/query/QueryProfileShardResult.java
index d5e00aca33632..81062ed75c6d6 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/query/QueryProfileShardResult.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/query/QueryProfileShardResult.java
@@ -22,8 +22,9 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.search.profile.ProfileResult;
 
 import java.io.IOException;
@@ -31,11 +32,17 @@
 import java.util.Collections;
 import java.util.List;
 
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+
 /**
  * A container class to hold the profile results for a single shard in the request.
  * Contains a list of query profiles, a collector tree and a total rewrite tree.
  */
-public final class QueryProfileShardResult implements Writeable, ToXContent {
+public final class QueryProfileShardResult implements Writeable, ToXContentObject {
+
+    public static final String COLLECTOR = "collector";
+    public static final String REWRITE_TIME = "rewrite_time";
+    public static final String QUERY_ARRAY = "query";
 
     private final List<ProfileResult> queryProfileResults;
 
@@ -90,15 +97,52 @@ public CollectorResult getCollectorResult() {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startArray("query");
+        builder.startObject();
+        builder.startArray(QUERY_ARRAY);
         for (ProfileResult p : queryProfileResults) {
             p.toXContent(builder, params);
         }
         builder.endArray();
-        builder.field("rewrite_time", rewriteTime);
-        builder.startArray("collector");
+        builder.field(REWRITE_TIME, rewriteTime);
+        builder.startArray(COLLECTOR);
         profileCollector.toXContent(builder, params);
         builder.endArray();
+        builder.endObject();
         return builder;
     }
+
+    public static QueryProfileShardResult fromXContent(XContentParser parser) throws IOException {
+        XContentParser.Token token = parser.currentToken();
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser::getTokenLocation);
+        String currentFieldName = null;
+        List<ProfileResult> queryProfileResults = new ArrayList<>();
+        long rewriteTime = 0;
+        CollectorResult collector = null;
+        while((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token.isValue()) {
+                if (REWRITE_TIME.equals(currentFieldName)) {
+                    rewriteTime = parser.longValue();
+                } else {
+                    parser.skipChildren();
+                }
+            } else if (token == XContentParser.Token.START_ARRAY) {
+                if (QUERY_ARRAY.equals(currentFieldName)) {
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                        queryProfileResults.add(ProfileResult.fromXContent(parser));
+                    }
+                } else if (COLLECTOR.equals(currentFieldName)) {
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                        collector = CollectorResult.fromXContent(parser);
+                    }
+                } else {
+                    parser.skipChildren();
+                }
+            } else {
+                parser.skipChildren();
+            }
+        }
+        return new QueryProfileShardResult(queryProfileResults, rewriteTime, collector);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/query/CancellableCollector.java b/core/src/main/java/org/elasticsearch/search/query/CancellableCollector.java
new file mode 100644
index 0000000000000..504a7f3d13da5
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/query/CancellableCollector.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.query;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.Collector;
+import org.apache.lucene.search.FilterCollector;
+import org.apache.lucene.search.LeafCollector;
+import org.elasticsearch.tasks.TaskCancelledException;
+
+import java.io.IOException;
+import java.util.function.BooleanSupplier;
+
+/**
+ * Collector that checks if the task it is executed under is cancelled.
+ */
+public class CancellableCollector extends FilterCollector {
+    private final BooleanSupplier cancelled;
+
+    /**
+     * Constructor
+     * @param cancelled supplier of the cancellation flag, the supplier will be called for each segment
+     * @param in wrapped collector
+     */
+    public CancellableCollector(BooleanSupplier cancelled, Collector in) {
+        super(in);
+        this.cancelled = cancelled;
+    }
+
+    @Override
+    public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException {
+        if (cancelled.getAsBoolean()) {
+            throw new TaskCancelledException("cancelled");
+        }
+        return super.getLeafCollector(context);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java
index 189fead781396..99253867b80d6 100644
--- a/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java
@@ -34,22 +34,26 @@
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.Sort;
 import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.TimeLimitingCollector;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.search.TopDocsCollector;
 import org.apache.lucene.search.TopFieldCollector;
 import org.apache.lucene.search.TopScoreDocCollector;
 import org.apache.lucene.search.TotalHitCountCollector;
 import org.apache.lucene.search.Weight;
+import org.apache.lucene.search.grouping.CollapsingTopDocsCollector;
+import org.apache.lucene.util.Counter;
+import org.elasticsearch.action.search.SearchTask;
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.lucene.MinimumScoreCollector;
 import org.elasticsearch.common.lucene.search.FilteredCollector;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.search.collapse.CollapseContext;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.SearchPhase;
 import org.elasticsearch.search.SearchService;
 import org.elasticsearch.search.aggregations.AggregationPhase;
+import org.elasticsearch.search.internal.ContextIndexSearcher;
 import org.elasticsearch.search.internal.ScrollContext;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.profile.ProfileShardResult;
@@ -60,15 +64,19 @@
 import org.elasticsearch.search.rescore.RescoreSearchContext;
 import org.elasticsearch.search.sort.SortAndFormats;
 import org.elasticsearch.search.suggest.SuggestPhase;
+import org.elasticsearch.tasks.TaskCancelledException;
 
 import java.util.AbstractList;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.Callable;
+import java.util.function.Consumer;
+
 
 /**
- *
+ * Query phase of a search request, used to run the query and get back from each shard information about the matching documents
+ * (document ids and score or sort criteria) so that matches can be reduced on the coordinating node
  */
 public class QueryPhase implements SearchPhase {
 
@@ -84,7 +92,7 @@ public QueryPhase(Settings settings) {
 
     @Override
     public void preProcess(SearchContext context) {
-        context.preProcess();
+        context.preProcess(true);
     }
 
     @Override
@@ -102,7 +110,8 @@ public void execute(SearchContext searchContext) throws QueryPhaseExecutionExcep
         // here to make sure it happens during the QUERY phase
         aggregationPhase.preProcess(searchContext);
 
-        boolean rescore = execute(searchContext, searchContext.searcher());
+        final ContextIndexSearcher searcher = searchContext.searcher();
+        boolean rescore = execute(searchContext, searcher, searcher::setCheckCancelled);
 
         if (rescore) { // only if we do a regular search
             rescorePhase.execute(searchContext);
@@ -134,12 +143,12 @@ private static boolean returnsDocsInOrder(Query query, SortAndFormats sf) {
      * wire everything (mapperService, etc.)
      * @return whether the rescoring phase should be executed
      */
-    static boolean execute(SearchContext searchContext, final IndexSearcher searcher) throws QueryPhaseExecutionException {
+    static boolean execute(SearchContext searchContext, final IndexSearcher searcher,
+            Consumer<Runnable> checkCancellationSetter) throws QueryPhaseExecutionException {
         QuerySearchResult queryResult = searchContext.queryResult();
         queryResult.searchTimedOut(false);
 
         final boolean doProfile = searchContext.getProfilers() != null;
-        final SearchType searchType = searchContext.searchType();
         boolean rescore = false;
         try {
             queryResult.from(searchContext.from());
@@ -162,17 +171,12 @@ static boolean execute(SearchContext searchContext, final IndexSearcher searcher
                 if (searchContext.getProfilers() != null) {
                     collector = new InternalProfileCollector(collector, CollectorResult.REASON_SEARCH_COUNT, Collections.emptyList());
                 }
-                topDocsCallable = new Callable<TopDocs>() {
-                    @Override
-                    public TopDocs call() throws Exception {
-                        return new TopDocs(totalHitCountCollector.getTotalHits(), Lucene.EMPTY_SCORE_DOCS, 0);
-                    }
-                };
+                topDocsCallable = () -> new TopDocs(totalHitCountCollector.getTotalHits(), Lucene.EMPTY_SCORE_DOCS, 0);
             } else {
                 // Perhaps have a dedicated scroll phase?
                 final ScrollContext scrollContext = searchContext.scrollContext();
                 assert (scrollContext != null) == (searchContext.request().scroll() != null);
-                final TopDocsCollector<?> topDocsCollector;
+                final Collector topDocsCollector;
                 ScoreDoc after = null;
                 if (searchContext.request().scroll() != null) {
                     numDocs = Math.min(searchContext.size(), totalNumDocs);
@@ -205,52 +209,65 @@ public TopDocs call() throws Exception {
                     numDocs = 1;
                 }
                 assert numDocs > 0;
-                if (searchContext.sort() != null) {
-                    SortAndFormats sf = searchContext.sort();
-                    topDocsCollector = TopFieldCollector.create(sf.sort, numDocs,
+                if (searchContext.collapse() == null) {
+                    if (searchContext.sort() != null) {
+                        SortAndFormats sf = searchContext.sort();
+                        topDocsCollector = TopFieldCollector.create(sf.sort, numDocs,
                             (FieldDoc) after, true, searchContext.trackScores(), searchContext.trackScores());
-                    sortValueFormats = sf.formats;
+                        sortValueFormats = sf.formats;
+                    } else {
+                        rescore = !searchContext.rescore().isEmpty();
+                        for (RescoreSearchContext rescoreContext : searchContext.rescore()) {
+                            numDocs = Math.max(rescoreContext.window(), numDocs);
+                        }
+                        topDocsCollector = TopScoreDocCollector.create(numDocs, after);
+                    }
                 } else {
-                    rescore = !searchContext.rescore().isEmpty();
-                    for (RescoreSearchContext rescoreContext : searchContext.rescore()) {
-                        numDocs = Math.max(rescoreContext.window(), numDocs);
+                    Sort sort = Sort.RELEVANCE;
+                    if (searchContext.sort() != null) {
+                        sort = searchContext.sort().sort;
+                    }
+                    CollapseContext collapse = searchContext.collapse();
+                    topDocsCollector = collapse.createTopDocs(sort, numDocs, searchContext.trackScores());
+                    if (searchContext.sort() == null) {
+                        sortValueFormats = new DocValueFormat[] {DocValueFormat.RAW};
+                    } else {
+                        sortValueFormats = searchContext.sort().formats;
                     }
-                    topDocsCollector = TopScoreDocCollector.create(numDocs, after);
                 }
                 collector = topDocsCollector;
                 if (doProfile) {
                     collector = new InternalProfileCollector(collector, CollectorResult.REASON_SEARCH_TOP_HITS, Collections.emptyList());
                 }
-                topDocsCallable = new Callable<TopDocs>() {
-                    @Override
-                    public TopDocs call() throws Exception {
-                        TopDocs topDocs = topDocsCollector.topDocs();
-                        if (scrollContext != null) {
-                            if (scrollContext.totalHits == -1) {
-                                // first round
-                                scrollContext.totalHits = topDocs.totalHits;
-                                scrollContext.maxScore = topDocs.getMaxScore();
-                            } else {
-                                // subsequent round: the total number of hits and
-                                // the maximum score were computed on the first round
-                                topDocs.totalHits = scrollContext.totalHits;
-                                topDocs.setMaxScore(scrollContext.maxScore);
-                            }
-                            switch (searchType) {
-                            case QUERY_AND_FETCH:
-                            case DFS_QUERY_AND_FETCH:
-                                // for (DFS_)QUERY_AND_FETCH, we already know the last emitted doc
-                                if (topDocs.scoreDocs.length > 0) {
-                                    // set the last emitted doc
-                                    scrollContext.lastEmittedDoc = topDocs.scoreDocs[topDocs.scoreDocs.length - 1];
-                                }
-                                break;
-                            default:
-                                break;
+                topDocsCallable = () -> {
+                    final TopDocs topDocs;
+                    if (topDocsCollector instanceof TopDocsCollector) {
+                        topDocs = ((TopDocsCollector<?>) topDocsCollector).topDocs();
+                    } else if (topDocsCollector instanceof CollapsingTopDocsCollector) {
+                        topDocs = ((CollapsingTopDocsCollector) topDocsCollector).getTopDocs();
+                    } else {
+                        throw new IllegalStateException("Unknown top docs collector " + topDocsCollector.getClass().getName());
+                    }
+                    if (scrollContext != null) {
+                        if (scrollContext.totalHits == -1) {
+                            // first round
+                            scrollContext.totalHits = topDocs.totalHits;
+                            scrollContext.maxScore = topDocs.getMaxScore();
+                        } else {
+                            // subsequent round: the total number of hits and
+                            // the maximum score were computed on the first round
+                            topDocs.totalHits = scrollContext.totalHits;
+                            topDocs.setMaxScore(scrollContext.maxScore);
+                        }
+                        if (searchContext.request().numberOfShards() == 1) {
+                            // if we fetch the document in the same roundtrip, we already know the last emitted doc
+                            if (topDocs.scoreDocs.length > 0) {
+                                // set the last emitted doc
+                                scrollContext.lastEmittedDoc = topDocs.scoreDocs[topDocs.scoreDocs.length - 1];
                             }
                         }
-                        return topDocs;
                     }
+                    return topDocs;
                 };
             }
 
@@ -350,14 +367,49 @@ public TopDocs call() throws Exception {
             }
 
             final boolean timeoutSet = searchContext.timeout() != null && !searchContext.timeout().equals(SearchService.NO_TIMEOUT);
-            if (timeoutSet && collector != null) { // collector might be null if no collection is actually needed
+            if (collector != null) { // collector might be null if no collection is actually needed
+                final Runnable timeoutRunnable;
+                if (timeoutSet) {
+                    final Counter counter = searchContext.timeEstimateCounter();
+                    final long startTime = counter.get();
+                    final long timeout = searchContext.timeout().millis();
+                    final long maxTime = startTime + timeout;
+                    timeoutRunnable = () -> {
+                        final long time = counter.get();
+                        if (time > maxTime) {
+                            throw new TimeExceededException();
+                        }
+                    };
+                } else {
+                    timeoutRunnable = null;
+                }
+
+                final Runnable cancellationRunnable;
+                if (searchContext.lowLevelCancellation()) {
+                    SearchTask task = searchContext.getTask();
+                    cancellationRunnable = () -> { if (task.isCancelled()) throw new TaskCancelledException("cancelled"); };
+                } else {
+                    cancellationRunnable = null;
+                }
+
+                final Runnable checkCancelled;
+                if (timeoutRunnable != null && cancellationRunnable != null) {
+                    checkCancelled = () -> { timeoutRunnable.run(); cancellationRunnable.run(); };
+                } else if (timeoutRunnable != null) {
+                    checkCancelled = timeoutRunnable;
+                } else if (cancellationRunnable != null) {
+                    checkCancelled = cancellationRunnable;
+                } else {
+                    checkCancelled = null;
+                }
+
+                checkCancellationSetter.accept(checkCancelled);
+
                 final Collector child = collector;
-                // TODO: change to use our own counter that uses the scheduler in ThreadPool
-                // throws TimeLimitingCollector.TimeExceededException when timeout has reached
-                collector = Lucene.wrapTimeLimitingCollector(collector, searchContext.timeEstimateCounter(), searchContext.timeout().millis());
+                collector = new CancellableCollector(searchContext.getTask()::isCancelled, collector);
                 if (doProfile) {
-                    collector = new InternalProfileCollector(collector, CollectorResult.REASON_SEARCH_TIMEOUT,
-                            Collections.singletonList((InternalProfileCollector) child));
+                    collector = new InternalProfileCollector(collector, CollectorResult.REASON_SEARCH_CANCELLED,
+                        Collections.singletonList((InternalProfileCollector) child));
                 }
             }
 
@@ -368,7 +420,7 @@ public TopDocs call() throws Exception {
                     }
                     searcher.search(query, collector);
                 }
-            } catch (TimeLimitingCollector.TimeExceededException e) {
+            } catch (TimeExceededException e) {
                 assert timeoutSet : "TimeExceededException thrown even though timeout wasn't set";
                 queryResult.searchTimedOut(true);
             } catch (Lucene.EarlyTerminationException e) {
@@ -395,4 +447,6 @@ public TopDocs call() throws Exception {
             throw new QueryPhaseExecutionException(searchContext, "Failed to execute main query", e);
         }
     }
+
+    private static class TimeExceededException extends RuntimeException {}
 }
diff --git a/core/src/main/java/org/elasticsearch/search/query/QuerySearchRequest.java b/core/src/main/java/org/elasticsearch/search/query/QuerySearchRequest.java
index 15593abf0da5e..6229f29566162 100644
--- a/core/src/main/java/org/elasticsearch/search/query/QuerySearchRequest.java
+++ b/core/src/main/java/org/elasticsearch/search/query/QuerySearchRequest.java
@@ -21,11 +21,14 @@
 
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.OriginalIndices;
-import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchTask;
 import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.dfs.AggregatedDfs;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.transport.TransportRequest;
 
 import java.io.IOException;
@@ -46,10 +49,10 @@ public class QuerySearchRequest extends TransportRequest implements IndicesReque
     public QuerySearchRequest() {
     }
 
-    public QuerySearchRequest(SearchRequest request, long id, AggregatedDfs dfs) {
+    public QuerySearchRequest(OriginalIndices originalIndices, long id, AggregatedDfs dfs) {
         this.id = id;
         this.dfs = dfs;
-        this.originalIndices = new OriginalIndices(request);
+        this.originalIndices = originalIndices;
     }
 
     public long id() {
@@ -85,4 +88,21 @@ public void writeTo(StreamOutput out) throws IOException {
         dfs.writeTo(out);
         OriginalIndices.writeOriginalIndices(originalIndices, out);
     }
+
+    @Override
+    public Task createTask(long id, String type, String action, TaskId parentTaskId) {
+        return new SearchTask(id, type, action, getDescription(), parentTaskId);
+    }
+
+    public String getDescription() {
+        StringBuilder sb = new StringBuilder();
+        sb.append("id[");
+        sb.append(id);
+        sb.append("], ");
+        sb.append("indices[");
+        Strings.arrayToDelimitedString(originalIndices.indices(), ",", sb);
+        sb.append("]");
+        return sb.toString();
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java b/core/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
index e583cfbf13e7e..f071c62f12c16 100644
--- a/core/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
+++ b/core/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
@@ -21,11 +21,10 @@
 
 import org.apache.lucene.search.FieldDoc;
 import org.apache.lucene.search.TopDocs;
-import org.elasticsearch.Version;
-import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.aggregations.Aggregations;
 import org.elasticsearch.search.aggregations.InternalAggregations;
@@ -42,36 +41,30 @@
 import static org.elasticsearch.common.lucene.Lucene.readTopDocs;
 import static org.elasticsearch.common.lucene.Lucene.writeTopDocs;
 
-/**
- *
- */
-public class QuerySearchResult extends QuerySearchResultProvider {
+public final class QuerySearchResult extends SearchPhaseResult {
 
-    private long id;
-    private SearchShardTarget shardTarget;
     private int from;
     private int size;
     private TopDocs topDocs;
     private DocValueFormat[] sortValueFormats;
     private InternalAggregations aggregations;
+    private boolean hasAggs;
     private List<SiblingPipelineAggregator> pipelineAggregators;
     private Suggest suggest;
     private boolean searchTimedOut;
     private Boolean terminatedEarly = null;
     private ProfileShardResult profileShardResults;
+    private boolean hasProfileResults;
+    private boolean hasScoreDocs;
+    private int totalHits;
+    private float maxScore;
 
     public QuerySearchResult() {
-
     }
 
     public QuerySearchResult(long id, SearchShardTarget shardTarget) {
-        this.id = id;
-        this.shardTarget = shardTarget;
-    }
-
-    @Override
-    public boolean includeFetch() {
-        return false;
+        this.requestId = id;
+        setSearchShardTarget(shardTarget);
     }
 
     @Override
@@ -79,20 +72,6 @@ public QuerySearchResult queryResult() {
         return this;
     }
 
-    @Override
-    public long id() {
-        return this.id;
-    }
-
-    @Override
-    public SearchShardTarget shardTarget() {
-        return shardTarget;
-    }
-
-    @Override
-    public void shardTarget(SearchShardTarget shardTarget) {
-        this.shardTarget = shardTarget;
-    }
 
     public void searchTimedOut(boolean searchTimedOut) {
         this.searchTimedOut = searchTimedOut;
@@ -111,11 +90,34 @@ public Boolean terminatedEarly() {
     }
 
     public TopDocs topDocs() {
+        if (topDocs == null) {
+            throw new IllegalStateException("topDocs already consumed");
+        }
+        return topDocs;
+    }
+
+    /**
+     * Returns <code>true</code> iff the top docs have already been consumed.
+     */
+    public boolean hasConsumedTopDocs() {
+        return topDocs == null;
+    }
+
+    /**
+     * Returns and nulls out the top docs for this search results. This allows to free up memory once the top docs are consumed.
+     * @throws IllegalStateException if the top docs have already been consumed.
+     */
+    public TopDocs consumeTopDocs() {
+        TopDocs topDocs = this.topDocs;
+        if (topDocs == null) {
+            throw new IllegalStateException("topDocs already consumed");
+        }
+        this.topDocs = null;
         return topDocs;
     }
 
     public void topDocs(TopDocs topDocs, DocValueFormat[] sortValueFormats) {
-        this.topDocs = topDocs;
+        setTopDocs(topDocs);
         if (topDocs.scoreDocs.length > 0 && topDocs.scoreDocs[0] instanceof FieldDoc) {
             int numFields = ((FieldDoc) topDocs.scoreDocs[0]).fields.length;
             if (numFields != sortValueFormats.length) {
@@ -126,24 +128,58 @@ public void topDocs(TopDocs topDocs, DocValueFormat[] sortValueFormats) {
         this.sortValueFormats = sortValueFormats;
     }
 
+    private void setTopDocs(TopDocs topDocs) {
+        this.topDocs = topDocs;
+        hasScoreDocs = topDocs.scoreDocs.length > 0;
+        this.totalHits = topDocs.totalHits;
+        this.maxScore = topDocs.getMaxScore();
+    }
+
     public DocValueFormat[] sortValueFormats() {
         return sortValueFormats;
     }
 
-    public Aggregations aggregations() {
-        return aggregations;
+    /**
+     * Returns <code>true</code> if this query result has unconsumed aggregations
+     */
+    public boolean hasAggs() {
+        return hasAggs;
+    }
+
+    /**
+     * Returns and nulls out the aggregation for this search results. This allows to free up memory once the aggregation is consumed.
+     * @throws IllegalStateException if the aggregations have already been consumed.
+     */
+    public Aggregations consumeAggs() {
+        if (aggregations == null) {
+            throw new IllegalStateException("aggs already consumed");
+        }
+        Aggregations aggs = aggregations;
+        aggregations = null;
+        return aggs;
     }
 
     public void aggregations(InternalAggregations aggregations) {
         this.aggregations = aggregations;
+        hasAggs = aggregations != null;
     }
 
     /**
-     * Returns the profiled results for this search, or potentially null if result was empty
-     * @return The profiled results, or null
+     * Returns and nulls out the profiled results for this search, or potentially null if result was empty.
+     * This allows to free up memory once the profiled result is consumed.
+     * @throws IllegalStateException if the profiled result has already been consumed.
      */
-    @Nullable public ProfileShardResult profileResults() {
-        return profileShardResults;
+    public ProfileShardResult consumeProfileResult() {
+        if (profileShardResults == null) {
+            throw new IllegalStateException("profile results already consumed");
+        }
+        ProfileShardResult result = profileShardResults;
+        profileShardResults = null;
+        return result;
+    }
+
+    public boolean hasProfileResults() {
+        return hasProfileResults;
     }
 
     /**
@@ -152,6 +188,7 @@ public void aggregations(InternalAggregations aggregations) {
      */
     public void profileResults(ProfileShardResult shardResults) {
         this.profileShardResults = shardResults;
+        hasProfileResults = shardResults != null;
     }
 
     public List<SiblingPipelineAggregator> pipelineAggregators() {
@@ -179,6 +216,9 @@ public QuerySearchResult from(int from) {
         return this;
     }
 
+    /**
+     * Returns the maximum size of this results top docs.
+     */
     public int size() {
         return size;
     }
@@ -188,10 +228,15 @@ public QuerySearchResult size(int size) {
         return this;
     }
 
-    /** Returns true iff the result has hits */
-    public boolean hasHits() {
-        return (topDocs != null && topDocs.scoreDocs.length > 0) ||
-            (suggest != null && suggest.hasScoreDocs());
+    /**
+     * Returns <code>true</code> if this result has any suggest score docs
+     */
+    public boolean hasSuggestHits() {
+      return (suggest != null && suggest.hasScoreDocs());
+    }
+
+    public boolean hasSearchContext() {
+        return hasScoreDocs || hasSuggestHits();
     }
 
     public static QuerySearchResult readQuerySearchResult(StreamInput in) throws IOException {
@@ -208,8 +253,7 @@ public void readFrom(StreamInput in) throws IOException {
     }
 
     public void readFromWithId(long id, StreamInput in) throws IOException {
-        this.id = id;
-//        shardTarget = readSearchShardTarget(in);
+        this.requestId = id;
         from = in.readVInt();
         size = in.readVInt();
         int numSortFieldsPlus1 = in.readVInt();
@@ -221,8 +265,8 @@ public void readFromWithId(long id, StreamInput in) throws IOException {
                 sortValueFormats[i] = in.readNamedWriteable(DocValueFormat.class);
             }
         }
-        topDocs = readTopDocs(in);
-        if (in.readBoolean()) {
+        setTopDocs(readTopDocs(in));
+        if (hasAggs = in.readBoolean()) {
             aggregations = InternalAggregations.readAggregations(in);
         }
         pipelineAggregators = in.readNamedWriteableList(PipelineAggregator.class).stream().map(a -> (SiblingPipelineAggregator) a)
@@ -232,21 +276,18 @@ public void readFromWithId(long id, StreamInput in) throws IOException {
         }
         searchTimedOut = in.readBoolean();
         terminatedEarly = in.readOptionalBoolean();
-
-        if (in.getVersion().onOrAfter(Version.V_2_2_0) && in.readBoolean()) {
-            profileShardResults = new ProfileShardResult(in);
-        }
+        profileShardResults = in.readOptionalWriteable(ProfileShardResult::new);
+        hasProfileResults = profileShardResults != null;
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        out.writeLong(id);
+        out.writeLong(requestId);
         writeToNoId(out);
     }
 
     public void writeToNoId(StreamOutput out) throws IOException {
-//        shardTarget.writeTo(out);
         out.writeVInt(from);
         out.writeVInt(size);
         if (sortValueFormats == null) {
@@ -273,14 +314,14 @@ public void writeToNoId(StreamOutput out) throws IOException {
         }
         out.writeBoolean(searchTimedOut);
         out.writeOptionalBoolean(terminatedEarly);
+        out.writeOptionalWriteable(profileShardResults);
+    }
 
-        if (out.getVersion().onOrAfter(Version.V_2_2_0)) {
-            if (profileShardResults == null) {
-                out.writeBoolean(false);
-            } else {
-                out.writeBoolean(true);
-                profileShardResults.writeTo(out);
-            }
-        }
+    public int getTotalHits() {
+        return totalHits;
+    }
+
+    public float getMaxScore() {
+        return maxScore;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/query/QuerySearchResultProvider.java b/core/src/main/java/org/elasticsearch/search/query/QuerySearchResultProvider.java
deleted file mode 100644
index 1ae3157fa5355..0000000000000
--- a/core/src/main/java/org/elasticsearch/search/query/QuerySearchResultProvider.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.query;
-
-import org.elasticsearch.search.SearchPhaseResult;
-import org.elasticsearch.transport.TransportResponse;
-
-/**
- *
- */
-public abstract class QuerySearchResultProvider extends TransportResponse implements SearchPhaseResult {
-
-    /**
-     * If both query and fetch happened on the same call.
-     */
-    public abstract boolean includeFetch();
-
-    public abstract QuerySearchResult queryResult();
-}
diff --git a/core/src/main/java/org/elasticsearch/search/query/ScrollQuerySearchResult.java b/core/src/main/java/org/elasticsearch/search/query/ScrollQuerySearchResult.java
index bcdd94adf891f..6401459489955 100644
--- a/core/src/main/java/org/elasticsearch/search/query/ScrollQuerySearchResult.java
+++ b/core/src/main/java/org/elasticsearch/search/query/ScrollQuerySearchResult.java
@@ -21,49 +21,54 @@
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
-import org.elasticsearch.transport.TransportResponse;
 
 import java.io.IOException;
 
 import static org.elasticsearch.search.query.QuerySearchResult.readQuerySearchResult;
 
-/**
- *
- */
-public class ScrollQuerySearchResult extends TransportResponse {
+public final class ScrollQuerySearchResult extends SearchPhaseResult {
 
-    private QuerySearchResult queryResult;
-    private SearchShardTarget shardTarget;
+    private QuerySearchResult result;
 
     public ScrollQuerySearchResult() {
     }
 
-    public ScrollQuerySearchResult(QuerySearchResult queryResult, SearchShardTarget shardTarget) {
-        this.queryResult = queryResult;
-        this.shardTarget = shardTarget;
+    public ScrollQuerySearchResult(QuerySearchResult result, SearchShardTarget shardTarget) {
+        this.result = result;
+        setSearchShardTarget(shardTarget);
     }
 
-    public QuerySearchResult queryResult() {
-        return queryResult;
+    @Override
+    public void setSearchShardTarget(SearchShardTarget shardTarget) {
+        super.setSearchShardTarget(shardTarget);
+        result.setSearchShardTarget(shardTarget);
     }
 
-    public SearchShardTarget shardTarget() {
-        return shardTarget;
+    @Override
+    public void setShardIndex(int shardIndex) {
+        super.setShardIndex(shardIndex);
+        result.setShardIndex(shardIndex);
+    }
+
+    @Override
+    public QuerySearchResult queryResult() {
+        return result;
     }
 
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
-        shardTarget = new SearchShardTarget(in);
-        queryResult = readQuerySearchResult(in);
-        queryResult.shardTarget(shardTarget);
+        SearchShardTarget shardTarget = new SearchShardTarget(in);
+        result = readQuerySearchResult(in);
+        setSearchShardTarget(shardTarget);
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        shardTarget.writeTo(out);
-        queryResult.writeTo(out);
+        getSearchShardTarget().writeTo(out);
+        result.writeTo(out);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/rescore/QueryRescoreMode.java b/core/src/main/java/org/elasticsearch/search/rescore/QueryRescoreMode.java
index 70718b56c0c5c..51db82652fce0 100644
--- a/core/src/main/java/org/elasticsearch/search/rescore/QueryRescoreMode.java
+++ b/core/src/main/java/org/elasticsearch/search/rescore/QueryRescoreMode.java
@@ -86,16 +86,12 @@ public String toString() {
     public abstract float combine(float primary, float secondary);
 
     public static QueryRescoreMode readFromStream(StreamInput in) throws IOException {
-        int ordinal = in.readVInt();
-        if (ordinal < 0 || ordinal >= values().length) {
-            throw new IOException("Unknown ScoreMode ordinal [" + ordinal + "]");
-        }
-        return values()[ordinal];
+        return in.readEnum(QueryRescoreMode.class);
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        out.writeVInt(this.ordinal());
+        out.writeEnum(this);
     }
 
     public static QueryRescoreMode fromString(String scoreMode) {
@@ -111,4 +107,4 @@ public static QueryRescoreMode fromString(String scoreMode) {
     public String toString() {
         return name().toLowerCase(Locale.ROOT);
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/search/rescore/QueryRescorer.java b/core/src/main/java/org/elasticsearch/search/rescore/QueryRescorer.java
index 2a57e41cfafc8..fe1b0577aa79f 100644
--- a/core/src/main/java/org/elasticsearch/search/rescore/QueryRescorer.java
+++ b/core/src/main/java/org/elasticsearch/search/rescore/QueryRescorer.java
@@ -24,10 +24,6 @@
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.TopDocs;
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.xcontent.ObjectParser;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.internal.ContextIndexSearcher;
 import org.elasticsearch.search.internal.SearchContext;
 
@@ -159,6 +155,8 @@ private TopDocs combine(TopDocs in, TopDocs resorted, QueryRescoreContext ctx) {
             // incoming first pass hits, instead of allowing recoring of just the top subset:
             Arrays.sort(in.scoreDocs, SCORE_DOC_COMPARATOR);
         }
+        // update the max score after the resort
+        in.setMaxScore(in.scoreDocs[0].score);
         return in;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/rescore/RescoreBuilder.java b/core/src/main/java/org/elasticsearch/search/rescore/RescoreBuilder.java
index 16c9c9ba8c70e..ac136a7601e3b 100644
--- a/core/src/main/java/org/elasticsearch/search/rescore/RescoreBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/rescore/RescoreBuilder.java
@@ -85,7 +85,7 @@ public static RescoreBuilder<?> parseFromXContent(QueryParseContext parseContext
             if (token == XContentParser.Token.FIELD_NAME) {
                 fieldName = parser.currentName();
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(fieldName, WINDOW_SIZE_FIELD)) {
+                if (WINDOW_SIZE_FIELD.match(fieldName)) {
                     windowSize = parser.intValue();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "rescore doesn't support [" + fieldName + "]");
diff --git a/core/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java b/core/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java
index 395db4cdcd8c2..d3d4c75cd7b5a 100644
--- a/core/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java
+++ b/core/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java
@@ -29,6 +29,7 @@
 import java.io.IOException;
 
 /**
+ * Rescore phase of a search request, used to run potentially expensive scoring models against the top matching documents.
  */
 public class RescorePhase extends AbstractComponent implements SearchPhase {
 
diff --git a/core/src/main/java/org/elasticsearch/search/rescore/Rescorer.java b/core/src/main/java/org/elasticsearch/search/rescore/Rescorer.java
index bf15e06bcb3a4..43a9dd9c64115 100644
--- a/core/src/main/java/org/elasticsearch/search/rescore/Rescorer.java
+++ b/core/src/main/java/org/elasticsearch/search/rescore/Rescorer.java
@@ -66,7 +66,6 @@ Explanation explain(int topLevelDocId, SearchContext context, RescoreSearchConte
     /**
      * Extracts all terms needed to execute this {@link Rescorer}. This method
      * is executed in a distributed frequency collection roundtrip for
-     * {@link SearchType#DFS_QUERY_AND_FETCH} and
      * {@link SearchType#DFS_QUERY_THEN_FETCH}
      */
     void extractTerms(SearchContext context, RescoreSearchContext rescoreContext, Set<Term> termsSet);
diff --git a/core/src/main/java/org/elasticsearch/search/searchafter/SearchAfterBuilder.java b/core/src/main/java/org/elasticsearch/search/searchafter/SearchAfterBuilder.java
index 6ed4b0db5bc7d..e6eb2cfcb35de 100644
--- a/core/src/main/java/org/elasticsearch/search/searchafter/SearchAfterBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/searchafter/SearchAfterBuilder.java
@@ -21,9 +21,10 @@
 
 import org.apache.lucene.search.FieldDoc;
 import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.SortedNumericSortField;
+import org.apache.lucene.search.SortedSetSortField;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -132,12 +133,27 @@ public static FieldDoc buildFieldDoc(SortAndFormats sort, Object[] values) {
         return new FieldDoc(Integer.MAX_VALUE, 0, fieldValues);
     }
 
-    private static Object convertValueFromSortField(Object value, SortField sortField, DocValueFormat format) {
+    /**
+     * Returns the inner {@link SortField.Type} expected for this sort field.
+     */
+    static SortField.Type extractSortType(SortField sortField) {
         if (sortField.getComparatorSource() instanceof IndexFieldData.XFieldComparatorSource) {
-            IndexFieldData.XFieldComparatorSource cmpSource = (IndexFieldData.XFieldComparatorSource) sortField.getComparatorSource();
-            return convertValueFromSortType(sortField.getField(), cmpSource.reducedType(), value, format);
+            return ((IndexFieldData.XFieldComparatorSource) sortField.getComparatorSource()).reducedType();
+        } else if (sortField instanceof SortedSetSortField) {
+            return SortField.Type.STRING;
+        } else if (sortField instanceof SortedNumericSortField) {
+            return ((SortedNumericSortField) sortField).getNumericType();
+        } else if ("LatLonPointSortField".equals(sortField.getClass().getSimpleName())) {
+            // for geo distance sorting
+            return SortField.Type.DOUBLE;
+        } else {
+            return sortField.getType();
         }
-        return convertValueFromSortType(sortField.getField(), sortField.getType(), value, format);
+    }
+
+    static Object convertValueFromSortField(Object value, SortField sortField, DocValueFormat format) {
+        SortField.Type sortType = extractSortType(sortField);
+        return convertValueFromSortType(sortField.getField(), sortType, value, format);
     }
 
     private static Object convertValueFromSortType(String fieldName, SortField.Type sortType, Object value, DocValueFormat format) {
@@ -202,10 +218,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     }
 
     void innerToXContent(XContentBuilder builder) throws IOException {
-        builder.field(SEARCH_AFTER.getPreferredName(), sortValues);
+        builder.array(SEARCH_AFTER.getPreferredName(), sortValues);
     }
 
-    public static SearchAfterBuilder fromXContent(XContentParser parser, ParseFieldMatcher parseFieldMatcher) throws IOException {
+    public static SearchAfterBuilder fromXContent(XContentParser parser) throws IOException {
         SearchAfterBuilder builder = new SearchAfterBuilder();
         XContentParser.Token token = parser.currentToken();
         List<Object> values = new ArrayList<> ();
diff --git a/core/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java b/core/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java
index 905ac8991bf64..20d67c494e712 100644
--- a/core/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.search.slice;
 
 import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.action.support.ToXContentToBytes;
 import org.elasticsearch.common.ParseField;
@@ -27,12 +28,13 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
+import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.UidFieldMapper;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
@@ -194,9 +196,14 @@ public Query toFilter(QueryShardContext context, int shardId, int numShards) {
             throw new IllegalArgumentException("field " + field + " not found");
         }
 
+        String field = this.field;
         boolean useTermQuery = false;
         if (UidFieldMapper.NAME.equals(field)) {
-           useTermQuery = true;
+            if (context.getIndexSettings().isSingleType()) {
+                // on new indices, the _id acts as a _uid
+                field = IdFieldMapper.NAME;
+            }
+            useTermQuery = true;
         } else if (type.hasDocValues() == false) {
             throw new IllegalArgumentException("cannot load numeric doc values on " + field);
         } else {
diff --git a/core/src/main/java/org/elasticsearch/search/slice/TermsSliceQuery.java b/core/src/main/java/org/elasticsearch/search/slice/TermsSliceQuery.java
index 429a3ebe89264..ddc02d32e55a9 100644
--- a/core/src/main/java/org/elasticsearch/search/slice/TermsSliceQuery.java
+++ b/core/src/main/java/org/elasticsearch/search/slice/TermsSliceQuery.java
@@ -33,6 +33,7 @@
 import org.apache.lucene.search.ConstantScoreScorer;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.DocIdSetBuilder;
+import org.apache.lucene.util.StringHelper;
 
 import java.io.IOException;
 
@@ -46,6 +47,9 @@
  * <b>NOTE</b>: Documents with no value for that field are ignored.
  */
 public final class TermsSliceQuery extends SliceQuery {
+    // Fixed seed for computing term hashCode
+    public static final int SEED = 7919;
+
     public TermsSliceQuery(String field, int id, int max) {
         super(field, id, max);
     }
@@ -71,7 +75,9 @@ private DocIdSet build(LeafReader reader) throws IOException {
         final TermsEnum te = terms.iterator();
         PostingsEnum docsEnum = null;
         for (BytesRef term = te.next(); term != null; term = te.next()) {
-            int hashCode = term.hashCode();
+            // use a fixed seed instead of term.hashCode() otherwise this query may return inconsistent results when
+            // running on another replica (StringHelper sets its default seed at startup with current time)
+            int hashCode = StringHelper.murmurhash3_x86_32(term, SEED);
             if (contains(hashCode)) {
                 docsEnum = te.postings(docsEnum, PostingsEnum.NONE);
                 builder.add(docsEnum);
diff --git a/core/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java
index d519f74087003..db6177ab36f53 100644
--- a/core/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java
@@ -21,11 +21,11 @@
 
 import org.apache.lucene.search.SortField;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
@@ -39,17 +39,13 @@
 
 import java.io.IOException;
 import java.util.Objects;
-import java.util.Optional;
 
 /**
  * A sort builder to sort based on a document field.
  */
 public class FieldSortBuilder extends SortBuilder<FieldSortBuilder> {
     public static final String NAME = "field_sort";
-    public static final ParseField NESTED_PATH = new ParseField("nested_path");
-    public static final ParseField NESTED_FILTER = new ParseField("nested_filter");
     public static final ParseField MISSING = new ParseField("missing");
-    public static final ParseField ORDER = new ParseField("order");
     public static final ParseField SORT_MODE = new ParseField("mode");
     public static final ParseField UNMAPPED_TYPE = new ParseField("unmapped_type");
 
@@ -239,10 +235,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.field(SORT_MODE.getPreferredName(), sortMode);
         }
         if (nestedFilter != null) {
-            builder.field(NESTED_FILTER.getPreferredName(), nestedFilter, params);
+            builder.field(NESTED_FILTER_FIELD.getPreferredName(), nestedFilter, params);
         }
         if (nestedPath != null) {
-            builder.field(NESTED_PATH.getPreferredName(), nestedPath);
+            builder.field(NESTED_PATH_FIELD.getPreferredName(), nestedPath);
         }
         builder.endObject();
         builder.endObject();
@@ -283,9 +279,7 @@ public SortFieldAndFormat build(QueryShardContext context) throws IOException {
                     && (sortMode == SortMode.SUM || sortMode == SortMode.AVG || sortMode == SortMode.MEDIAN)) {
                 throw new QueryShardException(context, "we only support AVG, MEDIAN and SUM on number based fields");
             }
-            IndexFieldData.XFieldComparatorSource fieldComparatorSource = fieldData
-                    .comparatorSource(missing, localSortMode, nested);
-            SortField field = new SortField(fieldType.name(), fieldComparatorSource, reverse);
+            SortField field = fieldData.sortField(missing, localSortMode, nested, reverse);
             return new SortFieldAndFormat(field, fieldType.docValueFormat(null, null));
         }
     }
@@ -327,67 +321,17 @@ public String getWriteableName() {
      *        in '{ "foo": { "order" : "asc"} }'. When parsing the inner object, the field name can be passed in via this argument
      */
     public static FieldSortBuilder fromXContent(QueryParseContext context, String fieldName) throws IOException {
-        XContentParser parser = context.parser();
-
-        Optional<QueryBuilder> nestedFilter = Optional.empty();
-        String nestedPath = null;
-        Object missing = null;
-        SortOrder order = null;
-        SortMode sortMode = null;
-        String unmappedType = null;
-
-        String currentFieldName = null;
-        XContentParser.Token token;
-        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-            if (token == XContentParser.Token.FIELD_NAME) {
-                currentFieldName = parser.currentName();
-            } else if (token == XContentParser.Token.START_OBJECT) {
-                if (context.getParseFieldMatcher().match(currentFieldName, NESTED_FILTER)) {
-                    nestedFilter = context.parseInnerQueryBuilder();
-                } else {
-                    throw new ParsingException(parser.getTokenLocation(), "Expected " + NESTED_FILTER.getPreferredName() + " element.");
-                }
-            } else if (token.isValue()) {
-                if (context.getParseFieldMatcher().match(currentFieldName, NESTED_PATH)) {
-                    nestedPath = parser.text();
-                } else if (context.getParseFieldMatcher().match(currentFieldName, MISSING)) {
-                    missing = parser.objectText();
-                } else if (context.getParseFieldMatcher().match(currentFieldName, ORDER)) {
-                    String sortOrder = parser.text();
-                    if ("asc".equals(sortOrder)) {
-                        order = SortOrder.ASC;
-                    } else if ("desc".equals(sortOrder)) {
-                        order = SortOrder.DESC;
-                    } else {
-                        throw new ParsingException(parser.getTokenLocation(), "Sort order [{}] not supported.", sortOrder);
-                    }
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SORT_MODE)) {
-                    sortMode = SortMode.fromString(parser.text());
-                } else if (context.getParseFieldMatcher().match(currentFieldName, UNMAPPED_TYPE)) {
-                    unmappedType = parser.text();
-                } else {
-                    throw new ParsingException(parser.getTokenLocation(), "Option [{}] not supported.", currentFieldName);
-                }
-            }
-        }
+        return PARSER.parse(context.parser(), new FieldSortBuilder(fieldName), context);
+    }
 
-        FieldSortBuilder builder = new FieldSortBuilder(fieldName);
-        nestedFilter.ifPresent(builder::setNestedFilter);
-        if (nestedPath != null) {
-            builder.setNestedPath(nestedPath);
-        }
-        if (missing != null) {
-            builder.missing(missing);
-        }
-        if (order != null) {
-            builder.order(order);
-        }
-        if (sortMode != null) {
-            builder.sortMode(sortMode);
-        }
-        if (unmappedType != null) {
-            builder.unmappedType(unmappedType);
-        }
-        return builder;
+    private static ObjectParser<FieldSortBuilder, QueryParseContext> PARSER = new ObjectParser<>(NAME);
+
+    static {
+        PARSER.declareField(FieldSortBuilder::missing, p -> p.objectText(),  MISSING, ValueType.VALUE);
+        PARSER.declareString(FieldSortBuilder::setNestedPath , NESTED_PATH_FIELD);
+        PARSER.declareString(FieldSortBuilder::unmappedType , UNMAPPED_TYPE);
+        PARSER.declareString((b, v) -> b.order(SortOrder.fromString(v)) , ORDER_FIELD);
+        PARSER.declareString((b, v) -> b.sortMode(SortMode.fromString(v)), SORT_MODE);
+        PARSER.declareObject(FieldSortBuilder::setNestedFilter, SortBuilder::parseNestedFilter, NESTED_FILTER_FIELD);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java
index f33dd0e2b1531..f175c8c7c0e6d 100644
--- a/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.search.sort;
 
+import org.apache.lucene.document.LatLonDocValuesField;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.NumericDocValues;
 import org.apache.lucene.search.DocIdSetIterator;
@@ -28,10 +29,8 @@
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.geo.GeoDistance;
-import org.elasticsearch.common.geo.GeoDistance.FixedSourceDistance;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.geo.GeoUtils;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -46,6 +45,7 @@
 import org.elasticsearch.index.fielddata.MultiGeoPointValues;
 import org.elasticsearch.index.fielddata.NumericDoubleValues;
 import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
+import org.elasticsearch.index.fielddata.plain.AbstractLatLonPointDVIndexFieldData.LatLonPointDVIndexFieldData;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.query.GeoValidationMethod;
 import org.elasticsearch.index.query.QueryBuilder;
@@ -73,18 +73,14 @@ public class GeoDistanceSortBuilder extends SortBuilder<GeoDistanceSortBuilder>
     private static final ParseField UNIT_FIELD = new ParseField("unit");
     private static final ParseField DISTANCE_TYPE_FIELD = new ParseField("distance_type");
     private static final ParseField VALIDATION_METHOD_FIELD = new ParseField("validation_method");
-    private static final ParseField IGNORE_MALFORMED_FIELD = new ParseField("ignore_malformed")
-            .withAllDeprecated("use validation_method instead");
-    private static final ParseField COERCE_FIELD = new ParseField("coerce", "normalize")
-            .withAllDeprecated("use validation_method instead");
+    private static final ParseField IGNORE_MALFORMED_FIELD = new ParseField("ignore_malformed").withAllDeprecated("validation_method");
+    private static final ParseField COERCE_FIELD = new ParseField("coerce", "normalize").withAllDeprecated("validation_method");
     private static final ParseField SORTMODE_FIELD = new ParseField("mode", "sort_mode");
-    private static final ParseField NESTED_PATH_FIELD = new ParseField("nested_path");
-    private static final ParseField NESTED_FILTER_FIELD = new ParseField("nested_filter");
 
     private final String fieldName;
     private final List<GeoPoint> points = new ArrayList<>();
 
-    private GeoDistance geoDistance = GeoDistance.DEFAULT;
+    private GeoDistance geoDistance = GeoDistance.ARC;
     private DistanceUnit unit = DistanceUnit.DEFAULT;
 
     private SortMode sortMode = null;
@@ -243,7 +239,7 @@ public GeoDistance geoDistance() {
     }
 
     /**
-     * The distance unit to use. Defaults to {@link org.elasticsearch.common.unit.DistanceUnit#KILOMETERS}
+     * The distance unit to use. Defaults to {@link org.elasticsearch.common.unit.DistanceUnit#METERS}
      */
     public GeoDistanceSortBuilder unit(DistanceUnit unit) {
         this.unit = unit;
@@ -251,7 +247,7 @@ public GeoDistanceSortBuilder unit(DistanceUnit unit) {
     }
 
     /**
-     * Returns the distance unit to use. Defaults to {@link org.elasticsearch.common.unit.DistanceUnit#KILOMETERS}
+     * Returns the distance unit to use. Defaults to {@link org.elasticsearch.common.unit.DistanceUnit#METERS}
      */
     public DistanceUnit unit() {
         return this.unit;
@@ -394,18 +390,18 @@ public int hashCode() {
      * Creates a new {@link GeoDistanceSortBuilder} from the query held by the {@link QueryParseContext} in
      * {@link org.elasticsearch.common.xcontent.XContent} format.
      *
-     * @param context the input parse context. The state on the parser contained in this context will be changed as a side effect of this
-     *        method call
-     * @param elementName in some sort syntax variations the field name precedes the xContent object that specifies further parameters, e.g.
-     *        in '{ "foo": { "order" : "asc"} }'. When parsing the inner object, the field name can be passed in via this argument
+     * @param context the input parse context. The state on the parser contained in this context will be changed as a
+     *                side effect of this method call
+     * @param elementName in some sort syntax variations the field name precedes the xContent object that specifies
+     *                    further parameters, e.g. in '{ "foo": { "order" : "asc"} }'. When parsing the inner object,
+     *                    the field name can be passed in via this argument
      */
     public static GeoDistanceSortBuilder fromXContent(QueryParseContext context, String elementName) throws IOException {
         XContentParser parser = context.parser();
-        ParseFieldMatcher parseFieldMatcher = context.getParseFieldMatcher();
         String fieldName = null;
         List<GeoPoint> geoPoints = new ArrayList<>();
         DistanceUnit unit = DistanceUnit.DEFAULT;
-        GeoDistance geoDistance = GeoDistance.DEFAULT;
+        GeoDistance geoDistance = GeoDistance.ARC;
         SortOrder order = SortOrder.ASC;
         SortMode sortMode = null;
         Optional<QueryBuilder> nestedFilter = Optional.empty();
@@ -425,7 +421,7 @@ public static GeoDistanceSortBuilder fromXContent(QueryParseContext context, Str
 
                 fieldName = currentName;
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (parseFieldMatcher.match(currentName, NESTED_FILTER_FIELD)) {
+                if (NESTED_FILTER_FIELD.match(currentName)) {
                     nestedFilter = context.parseInnerQueryBuilder();
                 } else {
                     // the json in the format of -> field : { lat : 30, lon : 12 }
@@ -442,27 +438,27 @@ public static GeoDistanceSortBuilder fromXContent(QueryParseContext context, Str
                     geoPoints.add(point);
                 }
             } else if (token.isValue()) {
-                if (parseFieldMatcher.match(currentName, ORDER_FIELD)) {
+                if (ORDER_FIELD.match(currentName)) {
                     order = SortOrder.fromString(parser.text());
-                } else if (parseFieldMatcher.match(currentName, UNIT_FIELD)) {
+                } else if (UNIT_FIELD.match(currentName)) {
                     unit = DistanceUnit.fromString(parser.text());
-                } else if (parseFieldMatcher.match(currentName, DISTANCE_TYPE_FIELD)) {
+                } else if (DISTANCE_TYPE_FIELD.match(currentName)) {
                     geoDistance = GeoDistance.fromString(parser.text());
-                } else if (parseFieldMatcher.match(currentName, COERCE_FIELD)) {
+                } else if (COERCE_FIELD.match(currentName)) {
                     coerce = parser.booleanValue();
-                    if (coerce == true) {
+                    if (coerce) {
                         ignoreMalformed = true;
                     }
-                } else if (parseFieldMatcher.match(currentName, IGNORE_MALFORMED_FIELD)) {
+                } else if (IGNORE_MALFORMED_FIELD.match(currentName)) {
                     boolean ignore_malformed_value = parser.booleanValue();
                     if (coerce == false) {
                         ignoreMalformed = ignore_malformed_value;
                     }
-                } else if (parseFieldMatcher.match(currentName, VALIDATION_METHOD_FIELD)) {
+                } else if (VALIDATION_METHOD_FIELD.match(currentName)) {
                     validation = GeoValidationMethod.fromString(parser.text());
-                } else if (parseFieldMatcher.match(currentName, SORTMODE_FIELD)) {
+                } else if (SORTMODE_FIELD.match(currentName)) {
                     sortMode = SortMode.fromString(parser.text());
-                } else if (parseFieldMatcher.match(currentName, NESTED_PATH_FIELD)) {
+                } else if (NESTED_PATH_FIELD.match(currentName)) {
                     nestedPath = parser.text();
                 } else if (token == Token.VALUE_STRING){
                     if (fieldName != null && fieldName.equals(currentName) == false) {
@@ -508,12 +504,9 @@ public static GeoDistanceSortBuilder fromXContent(QueryParseContext context, Str
     @Override
     public SortFieldAndFormat build(QueryShardContext context) throws IOException {
         final boolean indexCreatedBeforeV2_0 = context.indexVersionCreated().before(Version.V_2_0_0);
-        // validation was not available prior to 2.x, so to support bwc percolation queries we only ignore_malformed on 2.x created indexes
-        List<GeoPoint> localPoints = new ArrayList<GeoPoint>();
-        for (GeoPoint geoPoint : this.points) {
-            localPoints.add(new GeoPoint(geoPoint));
-        }
-
+        // validation was not available prior to 2.x, so to support bwc percolation queries we only ignore_malformed
+        // on 2.x created indexes
+        GeoPoint[] localPoints =  points.toArray(new GeoPoint[points.size()]);
         if (!indexCreatedBeforeV2_0 && !GeoValidationMethod.isIgnoreMalformed(validation)) {
             for (GeoPoint point : localPoints) {
                 if (GeoUtils.isValidLatitude(point.lat()) == false) {
@@ -547,16 +540,23 @@ public SortFieldAndFormat build(QueryShardContext context) throws IOException {
 
         MappedFieldType fieldType = context.fieldMapper(fieldName);
         if (fieldType == null) {
-            throw new IllegalArgumentException("failed to find mapper for [" + fieldName + "] for geo distance based sort");
+            throw new IllegalArgumentException("failed to find mapper for [" + fieldName
+                + "] for geo distance based sort");
         }
         final IndexGeoPointFieldData geoIndexFieldData = context.getForField(fieldType);
-        final FixedSourceDistance[] distances = new FixedSourceDistance[localPoints.size()];
-        for (int i = 0; i< localPoints.size(); i++) {
-            distances[i] = geoDistance.fixedSourceDistance(localPoints.get(i).lat(), localPoints.get(i).lon(), unit);
-        }
-
         final Nested nested = resolveNested(context, nestedPath, nestedFilter);
 
+        if (geoIndexFieldData.getClass() == LatLonPointDVIndexFieldData.class // only works with 5.x geo_point
+                && nested == null
+                && finalSortMode == MultiValueMode.MIN // LatLonDocValuesField internally picks the closest point
+                && unit == DistanceUnit.METERS
+                && reverse == false
+                && localPoints.length == 1) {
+            return new SortFieldAndFormat(
+                    LatLonDocValuesField.newDistanceSort(fieldName, localPoints[0].lat(), localPoints[0].lon()),
+                    DocValueFormat.RAW);
+        }
+
         IndexFieldData.XFieldComparatorSource geoDistanceComparatorSource = new IndexFieldData.XFieldComparatorSource() {
 
             @Override
@@ -565,19 +565,21 @@ public SortField.Type reducedType() {
             }
 
             @Override
-            public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
+            public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) {
                 return new FieldComparator.DoubleComparator(numHits, null, null) {
                     @Override
-                    protected NumericDocValues getNumericDocValues(LeafReaderContext context, String field) throws IOException {
+                    protected NumericDocValues getNumericDocValues(LeafReaderContext context, String field)
+                        throws IOException {
                         final MultiGeoPointValues geoPointValues = geoIndexFieldData.load(context).getGeoPointValues();
-                        final SortedNumericDoubleValues distanceValues = GeoDistance.distanceValues(geoPointValues, distances);
+                        final SortedNumericDoubleValues distanceValues = GeoUtils.distanceValues(geoDistance, unit,
+                            geoPointValues, localPoints);
                         final NumericDoubleValues selectedValues;
                         if (nested == null) {
-                            selectedValues = finalSortMode.select(distanceValues, Double.MAX_VALUE);
+                            selectedValues = finalSortMode.select(distanceValues, Double.POSITIVE_INFINITY);
                         } else {
                             final BitSet rootDocs = nested.rootDocs(context);
                             final DocIdSetIterator innerDocs = nested.innerDocs(context);
-                            selectedValues = finalSortMode.select(distanceValues, Double.MAX_VALUE, rootDocs, innerDocs,
+                            selectedValues = finalSortMode.select(distanceValues, Double.POSITIVE_INFINITY, rootDocs, innerDocs,
                                     context.reader().maxDoc());
                         }
                         return selectedValues.getRawDoubleValues();
@@ -587,14 +589,16 @@ protected NumericDocValues getNumericDocValues(LeafReaderContext context, String
 
         };
 
-        return new SortFieldAndFormat(new SortField(fieldName, geoDistanceComparatorSource, reverse), DocValueFormat.RAW);
+        return new SortFieldAndFormat(new SortField(fieldName, geoDistanceComparatorSource, reverse),
+            DocValueFormat.RAW);
     }
 
     static void parseGeoPoints(XContentParser parser, List<GeoPoint> geoPoints) throws IOException {
         while (!parser.nextToken().equals(XContentParser.Token.END_ARRAY)) {
             if (parser.currentToken() == XContentParser.Token.VALUE_NUMBER) {
-                // we might get here if the geo point is " number, number] " and the parser already moved over the opening bracket
-                // in this case we cannot use GeoUtils.parseGeoPoint(..) because this expects an opening bracket
+                // we might get here if the geo point is " number, number] " and the parser already moved over the
+                // opening bracket in this case we cannot use GeoUtils.parseGeoPoint(..) because this expects an opening
+                // bracket
                 double lon = parser.doubleValue();
                 parser.nextToken();
                 if (!parser.currentToken().equals(XContentParser.Token.VALUE_NUMBER)) {
diff --git a/core/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java
index 5b9b139e4958b..52f25301783c1 100644
--- a/core/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java
@@ -20,13 +20,10 @@
 package org.elasticsearch.search.sort;
 
 import org.apache.lucene.search.SortField;
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.DocValueFormat;
@@ -40,7 +37,6 @@
 public class ScoreSortBuilder extends SortBuilder<ScoreSortBuilder> {
 
     public static final String NAME = "_score";
-    public static final ParseField ORDER_FIELD = new ParseField("order");
     private static final SortFieldAndFormat SORT_SCORE = new SortFieldAndFormat(
             new SortField(null, SortField.Type.SCORE), DocValueFormat.RAW);
     private static final SortFieldAndFormat SORT_SCORE_REVERSE = new SortFieldAndFormat(
@@ -86,26 +82,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
      *        in '{ "foo": { "order" : "asc"} }'. When parsing the inner object, the field name can be passed in via this argument
      */
     public static ScoreSortBuilder fromXContent(QueryParseContext context, String fieldName) throws IOException {
-        XContentParser parser = context.parser();
-        ParseFieldMatcher matcher = context.getParseFieldMatcher();
-
-        XContentParser.Token token;
-        String currentName = parser.currentName();
-        ScoreSortBuilder result = new ScoreSortBuilder();
-        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-            if (token == XContentParser.Token.FIELD_NAME) {
-                currentName = parser.currentName();
-            } else if (token.isValue()) {
-                if (matcher.match(currentName, ORDER_FIELD)) {
-                    result.order(SortOrder.fromString(parser.text()));
-                } else {
-                    throw new ParsingException(parser.getTokenLocation(), "[" + NAME + "] failed to parse field [" + currentName + "]");
-                }
-            } else {
-                throw new ParsingException(parser.getTokenLocation(), "[" + NAME + "] unexpected token [" + token + "]");
-            }
-        }
-        return result;
+        return PARSER.apply(context.parser(), context);
+    }
+
+    private static ObjectParser<ScoreSortBuilder, QueryParseContext> PARSER = new ObjectParser<>(NAME, ScoreSortBuilder::new);
+
+    static {
+        PARSER.declareString((builder, order) -> builder.order(SortOrder.fromString(order)), ORDER_FIELD);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java
index 0a7cb5e1b3691..0527217bc14de 100644
--- a/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java
@@ -26,13 +26,12 @@
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefBuilder;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.fielddata.FieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
@@ -47,19 +46,16 @@
 import org.elasticsearch.index.query.QueryShardException;
 import org.elasticsearch.script.LeafSearchScript;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.Script.ScriptField;
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.MultiValueMode;
 
 import java.io.IOException;
-import java.util.Collections;
-import java.util.HashMap;
 import java.util.Locale;
-import java.util.Map;
 import java.util.Objects;
-import java.util.Optional;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
 
 /**
  * Script sort builder allows to sort based on a custom script expression.
@@ -70,8 +66,6 @@ public class ScriptSortBuilder extends SortBuilder<ScriptSortBuilder> {
     public static final ParseField TYPE_FIELD = new ParseField("type");
     public static final ParseField SCRIPT_FIELD = new ParseField("script");
     public static final ParseField SORTMODE_FIELD = new ParseField("mode");
-    public static final ParseField NESTED_PATH_FIELD = new ParseField("nested_path");
-    public static final ParseField NESTED_FILTER_FIELD = new ParseField("nested_filter");
 
     private final Script script;
 
@@ -218,6 +212,19 @@ public XContentBuilder toXContent(XContentBuilder builder, Params builderParams)
         return builder;
     }
 
+    private static ConstructingObjectParser<ScriptSortBuilder, QueryParseContext> PARSER = new ConstructingObjectParser<>(NAME,
+            a -> new ScriptSortBuilder((Script) a[0], (ScriptSortType) a[1]));
+
+    static {
+        PARSER.declareField(constructorArg(), (parser, context) -> Script.parse(parser, context.getDefaultScriptLanguage()),
+                Script.SCRIPT_PARSE_FIELD, ValueType.OBJECT_OR_STRING);
+        PARSER.declareField(constructorArg(), p -> ScriptSortType.fromString(p.text()), TYPE_FIELD, ValueType.STRING);
+        PARSER.declareString((b, v) -> b.order(SortOrder.fromString(v)), ORDER_FIELD);
+        PARSER.declareString((b, v) -> b.sortMode(SortMode.fromString(v)), SORTMODE_FIELD);
+        PARSER.declareString(ScriptSortBuilder::setNestedPath , NESTED_PATH_FIELD);
+        PARSER.declareObject(ScriptSortBuilder::setNestedFilter, SortBuilder::parseNestedFilter, NESTED_FILTER_FIELD);
+    }
+
     /**
      * Creates a new {@link ScriptSortBuilder} from the query held by the {@link QueryParseContext} in
      * {@link org.elasticsearch.common.xcontent.XContent} format.
@@ -228,66 +235,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params builderParams)
      *        in '{ "foo": { "order" : "asc"} }'. When parsing the inner object, the field name can be passed in via this argument
      */
     public static ScriptSortBuilder fromXContent(QueryParseContext context, String elementName) throws IOException {
-        XContentParser parser = context.parser();
-        ParseFieldMatcher parseField = context.getParseFieldMatcher();
-        Script script = null;
-        ScriptSortType type = null;
-        SortMode sortMode = null;
-        SortOrder order = null;
-        Optional<QueryBuilder> nestedFilter = Optional.empty();
-        String nestedPath = null;
-
-        XContentParser.Token token;
-        String currentName = parser.currentName();
-        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-            if (token == XContentParser.Token.FIELD_NAME) {
-                currentName = parser.currentName();
-            } else if (token == XContentParser.Token.START_OBJECT) {
-                if (parseField.match(currentName, ScriptField.SCRIPT)) {
-                    script = Script.parse(parser, parseField, context.getDefaultScriptLanguage());
-                } else if (parseField.match(currentName, NESTED_FILTER_FIELD)) {
-                    nestedFilter = context.parseInnerQueryBuilder();
-                } else {
-                    throw new ParsingException(parser.getTokenLocation(), "[" + NAME + "] failed to parse field [" + currentName + "]");
-                }
-            } else if (token.isValue()) {
-                if (parseField.match(currentName, ORDER_FIELD)) {
-                    order = SortOrder.fromString(parser.text());
-                } else if (parseField.match(currentName, TYPE_FIELD)) {
-                    type = ScriptSortType.fromString(parser.text());
-                } else if (parseField.match(currentName, SORTMODE_FIELD)) {
-                    sortMode = SortMode.fromString(parser.text());
-                } else if (parseField.match(currentName, NESTED_PATH_FIELD)) {
-                    nestedPath = parser.text();
-                } else if (parseField.match(currentName, ScriptField.SCRIPT)) {
-                    script = Script.parse(parser, parseField, context.getDefaultScriptLanguage());
-                } else {
-                    throw new ParsingException(parser.getTokenLocation(), "[" + NAME + "] failed to parse field [" + currentName + "]");
-                }
-            } else {
-                throw new ParsingException(parser.getTokenLocation(), "[" + NAME + "] unexpected token [" + token + "]");
-            }
-        }
-
-        ScriptSortBuilder result = new ScriptSortBuilder(script, type);
-        if (order != null) {
-            result.order(order);
-        }
-        if (sortMode != null) {
-            result.sortMode(sortMode);
-        }
-        nestedFilter.ifPresent(result::setNestedFilter);
-        if (nestedPath != null) {
-            result.setNestedPath(nestedPath);
-        }
-        return result;
+        return PARSER.apply(context.parser(), context);
     }
 
 
     @Override
     public SortFieldAndFormat build(QueryShardContext context) throws IOException {
-        final SearchScript searchScript = context.getScriptService().search(
-                context.lookup(), script, ScriptContext.Standard.SEARCH, Collections.emptyMap());
+        final SearchScript searchScript = context.getSearchScript(script, ScriptContext.Standard.SEARCH);
 
         MultiValueMode valueMode = null;
         if (sortMode != null) {
@@ -387,18 +341,14 @@ public enum ScriptSortType implements Writeable {
 
         @Override
         public void writeTo(final StreamOutput out) throws IOException {
-            out.writeVInt(ordinal());
+            out.writeEnum(this);
         }
 
         /**
          * Read from a stream.
          */
         static ScriptSortType readFromStream(final StreamInput in) throws IOException {
-            int ordinal = in.readVInt();
-            if (ordinal < 0 || ordinal >= values().length) {
-                throw new IOException("Unknown ScriptSortType ordinal [" + ordinal + "]");
-            }
-            return values()[ordinal];
+            return in.readEnum(ScriptSortType.class);
         }
 
         public static ScriptSortType fromString(final String str) {
diff --git a/core/src/main/java/org/elasticsearch/search/sort/SortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/SortBuilder.java
index c8f15f3a1e8c2..3af96e7eb4ead 100644
--- a/core/src/main/java/org/elasticsearch/search/sort/SortBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/sort/SortBuilder.java
@@ -25,6 +25,7 @@
 import org.apache.lucene.search.join.BitSetProducer;
 import org.elasticsearch.action.support.ToXContentToBytes;
 import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -52,7 +53,11 @@
 public abstract class SortBuilder<T extends SortBuilder<T>> extends ToXContentToBytes implements NamedWriteable {
 
     protected SortOrder order = SortOrder.ASC;
+
+    // parse fields common to more than one SortBuilder
     public static final ParseField ORDER_FIELD = new ParseField("order");
+    public static final ParseField NESTED_FILTER_FIELD = new ParseField("nested_filter");
+    public static final ParseField NESTED_PATH_FIELD = new ParseField("nested_path");
 
     private static final Map<String, Parser<?>> PARSERS;
     static {
@@ -199,6 +204,16 @@ protected static Nested resolveNested(QueryShardContext context, String nestedPa
         return nested;
     }
 
+    protected static QueryBuilder parseNestedFilter(XContentParser parser, QueryParseContext context) {
+        try {
+            QueryBuilder builder = context.parseInnerQueryBuilder().orElseThrow(() -> new ParsingException(parser.getTokenLocation(),
+                    "Expected " + NESTED_FILTER_FIELD.getPreferredName() + " element."));
+            return builder;
+        } catch (Exception e) {
+            throw new ParsingException(parser.getTokenLocation(), "Expected " + NESTED_FILTER_FIELD.getPreferredName() + " element.", e);
+        }
+    }
+
     @FunctionalInterface
     private interface Parser<T extends SortBuilder<?>> {
         T fromXContent(QueryParseContext context, String elementName) throws IOException;
diff --git a/core/src/main/java/org/elasticsearch/search/sort/SortMode.java b/core/src/main/java/org/elasticsearch/search/sort/SortMode.java
index 21495798a89f9..07b5bfa98c23f 100644
--- a/core/src/main/java/org/elasticsearch/search/sort/SortMode.java
+++ b/core/src/main/java/org/elasticsearch/search/sort/SortMode.java
@@ -52,15 +52,11 @@ public enum SortMode implements Writeable {
 
     @Override
     public void writeTo(final StreamOutput out) throws IOException {
-        out.writeVInt(ordinal());
+        out.writeEnum(this);
     }
 
     public static SortMode readFromStream(StreamInput in) throws IOException {
-        int ordinal = in.readVInt();
-        if (ordinal < 0 || ordinal >= values().length) {
-            throw new IOException("Unknown SortMode ordinal [" + ordinal + "]");
-        }
-        return values()[ordinal];
+        return in.readEnum(SortMode.class);
     }
 
     public static SortMode fromString(final String str) {
@@ -85,4 +81,4 @@ public static SortMode fromString(final String str) {
     public String toString() {
         return name().toLowerCase(Locale.ROOT);
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/search/sort/SortOrder.java b/core/src/main/java/org/elasticsearch/search/sort/SortOrder.java
index cd0a3bb6d4642..fbcb7b4288e31 100644
--- a/core/src/main/java/org/elasticsearch/search/sort/SortOrder.java
+++ b/core/src/main/java/org/elasticsearch/search/sort/SortOrder.java
@@ -52,16 +52,12 @@ public String toString() {
     };
 
     static SortOrder readFromStream(StreamInput in) throws IOException {
-        int ordinal = in.readVInt();
-        if (ordinal < 0 || ordinal >= values().length) {
-            throw new IOException("Unknown SortOrder ordinal [" + ordinal + "]");
-        }
-        return values()[ordinal];
+        return in.readEnum(SortOrder.class);
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        out.writeVInt(this.ordinal());
+        out.writeEnum(this);
     }
 
     public static SortOrder fromString(String op) {
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SortBy.java b/core/src/main/java/org/elasticsearch/search/suggest/SortBy.java
index 3cd19c5c2fb22..328fc4e8218ed 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/SortBy.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/SortBy.java
@@ -38,15 +38,11 @@ public enum SortBy implements Writeable {
 
     @Override
     public void writeTo(final StreamOutput out) throws IOException {
-        out.writeVInt(ordinal());
+        out.writeEnum(this);
     }
 
     public static SortBy readFromStream(final StreamInput in) throws IOException {
-        int ordinal = in.readVInt();
-        if (ordinal < 0 || ordinal >= values().length) {
-            throw new IOException("Unknown SortBy ordinal [" + ordinal + "]");
-        }
-        return values()[ordinal];
+        return in.readEnum(SortBy.class);
     }
 
     public static SortBy resolve(final String str) {
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/Suggest.java b/core/src/main/java/org/elasticsearch/search/suggest/Suggest.java
index 95612693f8b4c..943fde8dbf56d 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/Suggest.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/Suggest.java
@@ -19,13 +19,24 @@
 package org.elasticsearch.search.suggest;
 
 import org.apache.lucene.util.CollectionUtil;
+import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.common.CheckedFunction;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
 import org.elasticsearch.common.text.Text;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParserUtils;
+import org.elasticsearch.rest.action.search.RestSearchAction;
+import org.elasticsearch.search.aggregations.Aggregation;
 import org.elasticsearch.search.suggest.Suggest.Suggestion.Entry;
 import org.elasticsearch.search.suggest.Suggest.Suggestion.Entry.Option;
 import org.elasticsearch.search.suggest.completion.CompletionSuggestion;
@@ -39,15 +50,20 @@
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+
 /**
  * Top level suggest result, containing the result for each suggestion.
  */
 public class Suggest implements Iterable<Suggest.Suggestion<? extends Entry<? extends Option>>>, Streamable, ToXContent {
 
-    private static final String NAME = "suggest";
+    public static final String NAME = "suggest";
 
     public static final Comparator<Option> COMPARATOR = (first, second) -> {
         int cmp = Float.compare(second.getScore(), first.getScore());
@@ -62,7 +78,7 @@ public class Suggest implements Iterable<Suggest.Suggestion<? extends Entry<? ex
 
     private Map<String, Suggestion<? extends Entry<? extends Option>>> suggestMap;
 
-    public Suggest() {
+    private Suggest() {
         this(Collections.emptyList());
     }
 
@@ -143,7 +159,7 @@ public void readFrom(StreamInput in) throws IOException {
     public void writeTo(StreamOutput out) throws IOException {
         out.writeVInt(suggestions.size());
         for (Suggestion<?> command : suggestions) {
-            out.writeVInt(command.getType());
+            out.writeVInt(command.getWriteableType());
             command.writeTo(out);
         }
     }
@@ -166,6 +182,27 @@ public XContentBuilder toInnerXContent(XContentBuilder builder, Params params) t
         return builder;
     }
 
+    /**
+     * this parsing method assumes that the leading "suggest" field name has already been parsed by the caller
+     */
+    public static Suggest fromXContent(XContentParser parser) throws IOException {
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation);
+        List<Suggestion<? extends Entry<? extends Option>>> suggestions = new ArrayList<>();
+        while ((parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser::getTokenLocation);
+            String currentField = parser.currentName();
+            ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.nextToken(), parser::getTokenLocation);
+            Suggestion<? extends Entry<? extends Option>> suggestion = Suggestion.fromXContent(parser);
+            if (suggestion != null) {
+                suggestions.add(suggestion);
+            } else {
+                throw new ParsingException(parser.getTokenLocation(),
+                        String.format(Locale.ROOT, "Could not parse suggestion keyed as [%s]", currentField));
+            }
+        }
+        return new Suggest(suggestions);
+    }
+
     public static Suggest readSuggest(StreamInput in) throws IOException {
         Suggest result = new Suggest();
         result.readFrom(in);
@@ -208,12 +245,14 @@ public <T extends Suggestion> List<T> filter(Class<T> suggestionType) {
      */
     public static class Suggestion<T extends Suggestion.Entry> implements Iterable<T>, Streamable, ToXContent {
 
+        private static final String NAME = "suggestion";
+
         public static final int TYPE = 0;
         protected String name;
         protected int size;
         protected final List<T> entries = new ArrayList<>(5);
 
-        public Suggestion() {
+        protected Suggestion() {
         }
 
         public Suggestion(String name, int size) {
@@ -225,10 +264,23 @@ public void addTerm(T entry) {
             entries.add(entry);
         }
 
-        public int getType() {
+        /**
+         * Returns a integer representing the type of the suggestion. This is used for
+         * internal serialization over the network.
+         */
+        public int getWriteableType() { // TODO remove this in favor of NamedWriteable
             return TYPE;
         }
 
+        /**
+         * Returns a string representing the type of the suggestion. This type is added to
+         * the suggestion name in the XContent response, so that it can later be used by
+         * REST clients to determine the internal type of the suggestion.
+         */
+        protected String getType() {
+            return NAME;
+        }
+
         @Override
         public Iterator<T> iterator() {
             return entries.iterator();
@@ -340,7 +392,12 @@ public void innerWriteTo(StreamOutput out) throws IOException {
 
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startArray(name);
+            if (params.paramAsBoolean(RestSearchAction.TYPED_KEYS_PARAM, false)) {
+                // Concatenates the type and the name of the suggestion (ex: completion#foo)
+                builder.startArray(String.join(Aggregation.TYPED_KEYS_DELIMITER, getType(), getName()));
+            } else {
+                builder.startArray(getName());
+            }
             for (Entry<?> entry : entries) {
                 entry.toXContent(builder, params);
             }
@@ -348,40 +405,58 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             return builder;
         }
 
+        @SuppressWarnings("unchecked")
+        public static Suggestion<? extends Entry<? extends Option>> fromXContent(XContentParser parser) throws IOException {
+            ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser::getTokenLocation);
+            SetOnce<Suggestion> suggestion = new SetOnce<>();
+            XContentParserUtils.parseTypedKeysObject(parser, Aggregation.TYPED_KEYS_DELIMITER, Suggestion.class, suggestion::set);
+            return suggestion.get();
+        }
+
+        protected static <E extends Suggestion.Entry<?>> void parseEntries(XContentParser parser, Suggestion<E> suggestion,
+                                                                           CheckedFunction<XContentParser, E, IOException> entryParser)
+                throws IOException {
+            ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser::getTokenLocation);
+            while ((parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                suggestion.addTerm(entryParser.apply(parser));
+            }
+        }
+
         /**
          * Represents a part from the suggest text with suggested options.
          */
-        public static class Entry<O extends Entry.Option> implements Iterable<O>, Streamable, ToXContent {
-
-            static class Fields {
+        public static class Entry<O extends Entry.Option> implements Iterable<O>, Streamable, ToXContentObject {
 
-                static final String TEXT = "text";
-                static final String OFFSET = "offset";
-                static final String LENGTH = "length";
-                static final String OPTIONS = "options";
-
-            }
+            private static final String TEXT = "text";
+            private static final String OFFSET = "offset";
+            private static final String LENGTH = "length";
+            protected static final String OPTIONS = "options";
 
             protected Text text;
             protected int offset;
             protected int length;
 
-            protected List<O> options;
+            protected List<O> options = new ArrayList<>(5);
 
             public Entry(Text text, int offset, int length) {
                 this.text = text;
                 this.offset = offset;
                 this.length = length;
-                this.options = new ArrayList<>(5);
             }
 
-            public Entry() {
+            protected Entry() {
             }
 
             public void addOption(O option) {
                 options.add(option);
             }
 
+            protected void addOptions(List<O> options) {
+                for (O option : options) {
+                    addOption(option);
+                }
+            }
+
             protected void sort(Comparator<O> comparator) {
                 CollectionUtil.timSort(options, comparator);
             }
@@ -501,6 +576,7 @@ public void readFrom(StreamInput in) throws IOException {
                 }
             }
 
+            @SuppressWarnings("unchecked")
             protected O newOption(){
                 return (O) new Option();
             }
@@ -519,10 +595,10 @@ public void writeTo(StreamOutput out) throws IOException {
             @Override
             public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
                 builder.startObject();
-                builder.field(Fields.TEXT, text);
-                builder.field(Fields.OFFSET, offset);
-                builder.field(Fields.LENGTH, length);
-                builder.startArray(Fields.OPTIONS);
+                builder.field(TEXT, text);
+                builder.field(OFFSET, offset);
+                builder.field(LENGTH, length);
+                builder.startArray(OPTIONS);
                 for (Option option : options) {
                     option.toXContent(builder, params);
                 }
@@ -531,19 +607,32 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                 return builder;
             }
 
+            private static ObjectParser<Entry<Option>, Void> PARSER = new ObjectParser<>("SuggestionEntryParser", true, Entry::new);
+
+            static {
+                declareCommonFields(PARSER);
+                PARSER.declareObjectArray(Entry::addOptions, (p,c) -> Option.fromXContent(p), new ParseField(OPTIONS));
+            }
+
+            protected static void declareCommonFields(ObjectParser<? extends Entry<? extends Option>, Void> parser) {
+                parser.declareString((entry, text) -> entry.text = new Text(text), new ParseField(TEXT));
+                parser.declareInt((entry, offset) -> entry.offset = offset, new ParseField(OFFSET));
+                parser.declareInt((entry, length) -> entry.length = length, new ParseField(LENGTH));
+            }
+
+            public static Entry<? extends Option> fromXContent(XContentParser parser) {
+                return PARSER.apply(parser, null);
+            }
+
             /**
              * Contains the suggested text with its document frequency and score.
              */
-            public static class Option implements Streamable, ToXContent {
-
-                static class Fields {
+            public static class Option implements Streamable, ToXContentObject {
 
-                    static final String TEXT = "text";
-                    static final String HIGHLIGHTED = "highlighted";
-                    static final String SCORE = "score";
-                    static final String COLLATE_MATCH = "collate_match";
-
-                }
+                public static final ParseField TEXT = new ParseField("text");
+                public static final ParseField HIGHLIGHTED = new ParseField("highlighted");
+                public static final ParseField SCORE = new ParseField("score");
+                public static final ParseField COLLATE_MATCH = new ParseField("collate_match");
 
                 private Text text;
                 private Text highlighted;
@@ -627,17 +716,38 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                 }
 
                 protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
-                    builder.field(Fields.TEXT, text);
+                    builder.field(TEXT.getPreferredName(), text);
                     if (highlighted != null) {
-                        builder.field(Fields.HIGHLIGHTED, highlighted);
+                        builder.field(HIGHLIGHTED.getPreferredName(), highlighted);
                     }
-                    builder.field(Fields.SCORE, score);
+                    builder.field(SCORE.getPreferredName(), score);
                     if (collateMatch != null) {
-                        builder.field(Fields.COLLATE_MATCH, collateMatch.booleanValue());
+                        builder.field(COLLATE_MATCH.getPreferredName(), collateMatch.booleanValue());
                     }
                     return builder;
                 }
 
+                private static final ConstructingObjectParser<Option, Void> PARSER = new ConstructingObjectParser<>("SuggestOptionParser",
+                        true, args -> {
+                            Text text = new Text((String) args[0]);
+                            float score = (Float) args[1];
+                            String highlighted = (String) args[2];
+                            Text highlightedText = highlighted == null ? null : new Text(highlighted);
+                            Boolean collateMatch = (Boolean) args[3];
+                            return new Option(text, highlightedText, score, collateMatch);
+                        });
+
+                static {
+                    PARSER.declareString(constructorArg(), TEXT);
+                    PARSER.declareFloat(constructorArg(), SCORE);
+                    PARSER.declareString(optionalConstructorArg(), HIGHLIGHTED);
+                    PARSER.declareBoolean(optionalConstructorArg(), COLLATE_MATCH);
+                }
+
+                public static Option fromXContent(XContentParser parser) {
+                    return PARSER.apply(parser, null);
+                }
+
                 protected void mergeInto(Option otherOption) {
                     score = Math.max(score, otherOption.score);
                 }
@@ -649,7 +759,6 @@ public boolean equals(Object o) {
 
                     Option that = (Option) o;
                     return text.equals(that.text);
-
                 }
 
                 @Override
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/SuggestBuilder.java
index 332d18f256d2c..fec2b0f3bf31d 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/SuggestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/SuggestBuilder.java
@@ -21,7 +21,6 @@
 import org.elasticsearch.action.support.ToXContentToBytes;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -29,7 +28,6 @@
 import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
 
@@ -138,9 +136,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         return builder;
     }
 
-    public static SuggestBuilder fromXContent(QueryParseContext parseContext, Suggesters suggesters) throws IOException {
-        XContentParser parser = parseContext.parser();
-        ParseFieldMatcher parseFieldMatcher = parseContext.getParseFieldMatcher();
+    public static SuggestBuilder fromXContent(XContentParser parser) throws IOException {
         SuggestBuilder suggestBuilder = new SuggestBuilder();
         String fieldName = null;
 
@@ -154,7 +150,7 @@ public static SuggestBuilder fromXContent(QueryParseContext parseContext, Sugges
             if (token == XContentParser.Token.FIELD_NAME) {
                 fieldName = parser.currentName();
             } else if (token.isValue()) {
-                if (parseFieldMatcher.match(fieldName, GLOBAL_TEXT_FIELD)) {
+                if (GLOBAL_TEXT_FIELD.match(fieldName)) {
                     suggestBuilder.setGlobalText(parser.text());
                 } else {
                     throw new IllegalArgumentException("[suggest] does not support [" + fieldName + "]");
@@ -164,7 +160,7 @@ public static SuggestBuilder fromXContent(QueryParseContext parseContext, Sugges
                 if (suggestionName == null) {
                     throw new IllegalArgumentException("suggestion must have name");
                 }
-                suggestBuilder.addSuggestion(suggestionName, SuggestionBuilder.fromXContent(parseContext, suggesters));
+                suggestBuilder.addSuggestion(suggestionName, SuggestionBuilder.fromXContent(parser));
             } else {
                 throw new ParsingException(parser.getTokenLocation(), "unexpected token [" + token + "] after [" + fieldName + "]");
             }
@@ -195,8 +191,7 @@ public boolean equals(Object other) {
         if (other == null || getClass() != other.getClass()) {
             return false;
         }
-        @SuppressWarnings("unchecked")
-        SuggestBuilder o = (SuggestBuilder)other;
+        SuggestBuilder o = (SuggestBuilder) other;
         return Objects.equals(globalText, o.globalText) &&
                Objects.equals(suggestions, o.suggestions);
     }
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestPhase.java b/core/src/main/java/org/elasticsearch/search/suggest/SuggestPhase.java
index c0567e59e8c62..874448b924c06 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/SuggestPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/SuggestPhase.java
@@ -35,6 +35,7 @@
 import java.util.Map;
 
 /**
+ * Suggest phase of a search request, used to collect suggestions
  */
 public class SuggestPhase extends AbstractComponent implements SearchPhase {
 
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/Suggester.java b/core/src/main/java/org/elasticsearch/search/suggest/Suggester.java
index 711e3d9db7fd2..6c007bf4cb2b1 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/Suggester.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/Suggester.java
@@ -21,23 +21,16 @@
 
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.util.CharsRefBuilder;
-import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.index.query.QueryParseContext;
 
 import java.io.IOException;
 
-public abstract class Suggester<T extends SuggestionSearchContext.SuggestionContext> implements Writeable.Reader<SuggestionBuilder<?>> {
+public abstract class Suggester<T extends SuggestionSearchContext.SuggestionContext> {
 
     protected abstract Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>>
         innerExecute(String name, T suggestion, IndexSearcher searcher, CharsRefBuilder spare) throws IOException;
 
-    /**
-     * Read the SuggestionBuilder paired with this Suggester XContent.
-     */
-    public abstract SuggestionBuilder<?> innerFromXContent(QueryParseContext context) throws IOException;
-
     public Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>>
-        execute(String name, T suggestion, IndexSearcher searcher, CharsRefBuilder spare) throws IOException {
+                execute(String name, T suggestion, IndexSearcher searcher, CharsRefBuilder spare) throws IOException {
         // #3469 We want to ignore empty shards
 
         if (searcher.getIndexReader().numDocs() == 0) {
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/Suggesters.java b/core/src/main/java/org/elasticsearch/search/suggest/Suggesters.java
deleted file mode 100644
index 3f3f805d8f70b..0000000000000
--- a/core/src/main/java/org/elasticsearch/search/suggest/Suggesters.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.search.suggest;
-
-import java.util.Map;
-
-/**
- * Registry of Suggesters. This is only its own class to make Guice happy.
- */
-public final class Suggesters {
-    private final Map<String, Suggester<?>> suggesters;
-
-    public Suggesters(Map<String, Suggester<?>> suggesters) {
-        this.suggesters = suggesters;
-    }
-
-    public Suggester<?> getSuggester(String suggesterName) {
-        Suggester<?> suggester = suggesters.get(suggesterName);
-        if (suggester == null) {
-            throw new IllegalArgumentException("suggester with name [" + suggesterName + "] not supported");
-        }
-        return suggester;
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestionBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/SuggestionBuilder.java
index 59555e049c2cc..6714778c1b303 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/SuggestionBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/SuggestionBuilder.java
@@ -21,19 +21,17 @@
 
 import org.apache.lucene.analysis.Analyzer;
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.action.support.ToXContentToBytes;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lucene.BytesRefs;
+import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
 
@@ -43,7 +41,7 @@
 /**
  * Base class for the different suggestion implementations.
  */
-public abstract class SuggestionBuilder<T extends SuggestionBuilder<T>> extends ToXContentToBytes implements NamedWriteable {
+public abstract class SuggestionBuilder<T extends SuggestionBuilder<T>> implements NamedWriteable, ToXContent {
 
     protected final String field;
     protected String text;
@@ -254,10 +252,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
 
     protected abstract XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException;
 
-    static SuggestionBuilder<?> fromXContent(QueryParseContext parseContext, Suggesters suggesters)
-            throws IOException {
-        XContentParser parser = parseContext.parser();
-        ParseFieldMatcher parsefieldMatcher = parseContext.getParseFieldMatcher();
+    static SuggestionBuilder<?> fromXContent(XContentParser parser) throws IOException {
         XContentParser.Token token;
         String currentFieldName = null;
         String suggestText = null;
@@ -269,17 +264,17 @@ static SuggestionBuilder<?> fromXContent(QueryParseContext parseContext, Suggest
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token.isValue()) {
-                if (parsefieldMatcher.match(currentFieldName, TEXT_FIELD)) {
+                if (TEXT_FIELD.match(currentFieldName)) {
                     suggestText = parser.text();
-                } else if (parsefieldMatcher.match(currentFieldName, PREFIX_FIELD)) {
+                } else if (PREFIX_FIELD.match(currentFieldName)) {
                     prefix = parser.text();
-                } else if (parsefieldMatcher.match(currentFieldName, REGEX_FIELD)) {
+                } else if (REGEX_FIELD.match(currentFieldName)) {
                     regex = parser.text();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "suggestion does not support [" + currentFieldName + "]");
                 }
             } else if (token == XContentParser.Token.START_OBJECT) {
-                suggestionBuilder = suggesters.getSuggester(currentFieldName).innerFromXContent(parseContext);
+                suggestionBuilder = parser.namedObject(SuggestionBuilder.class, currentFieldName, null);
             }
         }
         if (suggestionBuilder == null) {
@@ -319,7 +314,7 @@ protected void populateCommonFields(MapperService mapperService,
                 suggestionContext.setAnalyzer(fieldType.searchAnalyzer());
             }
         } else {
-            Analyzer luceneAnalyzer = mapperService.analysisService().analyzer(analyzer);
+            Analyzer luceneAnalyzer = mapperService.getIndexAnalyzers().get(analyzer);
             if (luceneAnalyzer == null) {
                 throw new IllegalArgumentException("analyzer [" + analyzer + "] doesn't exists");
             }
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionFieldStats.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionFieldStats.java
index e5e1b1b919922..8b5761a7e9a5f 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionFieldStats.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionFieldStats.java
@@ -27,6 +27,7 @@
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.search.suggest.document.CompletionTerms;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.common.FieldMemoryStats;
 import org.elasticsearch.common.regex.Regex;
 
 import java.io.IOException;
@@ -64,6 +65,6 @@ public static CompletionStats completionStats(IndexReader indexReader, String ..
                 throw new ElasticsearchException(ioe);
             }
         }
-        return new CompletionStats(sizeInBytes, completionFields);
+        return new CompletionStats(sizeInBytes, completionFields == null ? null : new FieldMemoryStats(completionFields));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionStats.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionStats.java
index c8ed562f9cc09..62cc62b184129 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionStats.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionStats.java
@@ -18,7 +18,7 @@
  */
 package org.elasticsearch.search.suggest.completion;
 
-import com.carrotsearch.hppc.ObjectLongHashMap;
+import org.elasticsearch.common.FieldMemoryStats;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -34,15 +34,19 @@
  */
 public class CompletionStats implements Streamable, ToXContent {
 
-    private long sizeInBytes;
+    private static final String COMPLETION = "completion";
+    private static final String SIZE_IN_BYTES = "size_in_bytes";
+    private static final String SIZE = "size";
+    private static final String FIELDS = "fields";
 
+    private long sizeInBytes;
     @Nullable
-    private ObjectLongHashMap<String> fields;
+    private FieldMemoryStats fields;
 
     public CompletionStats() {
     }
 
-    public CompletionStats(long size, @Nullable ObjectLongHashMap<String> fields) {
+    public CompletionStats(long size, @Nullable FieldMemoryStats fields) {
         this.sizeInBytes = size;
         this.fields = fields;
     }
@@ -55,98 +59,43 @@ public ByteSizeValue getSize() {
         return new ByteSizeValue(sizeInBytes);
     }
 
-    public ObjectLongHashMap<String> getFields() {
+    public FieldMemoryStats getFields() {
         return fields;
     }
 
     @Override
     public void readFrom(StreamInput in) throws IOException {
         sizeInBytes = in.readVLong();
-        if (in.readBoolean()) {
-            int size = in.readVInt();
-            fields = new ObjectLongHashMap<>(size);
-            for (int i = 0; i < size; i++) {
-                fields.put(in.readString(), in.readVLong());
-            }
-        }
+        fields = in.readOptionalWriteable(FieldMemoryStats::new);
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeVLong(sizeInBytes);
-        if (fields == null) {
-            out.writeBoolean(false);
-        } else {
-            out.writeBoolean(true);
-            out.writeVInt(fields.size());
-
-            assert !fields.containsKey(null);
-            final Object[] keys = fields.keys;
-            final long[] values = fields.values;
-            for (int i = 0; i < keys.length; i++) {
-                if (keys[i] != null) {
-                    out.writeString((String) keys[i]);
-                    out.writeVLong(values[i]);
-                }
-            }
-        }
+        out.writeOptionalWriteable(fields);
     }
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject(Fields.COMPLETION);
-        builder.byteSizeField(Fields.SIZE_IN_BYTES, Fields.SIZE, sizeInBytes);
+        builder.startObject(COMPLETION);
+        builder.byteSizeField(SIZE_IN_BYTES, SIZE, sizeInBytes);
         if (fields != null) {
-            builder.startObject(Fields.FIELDS);
-
-            assert !fields.containsKey(null);
-            final Object[] keys = fields.keys;
-            final long[] values = fields.values;
-            for (int i = 0; i < keys.length; i++) {
-                if (keys[i] != null) {
-                    builder.startObject((String) keys[i]);
-                    builder.byteSizeField(Fields.SIZE_IN_BYTES, Fields.SIZE, values[i]);
-                    builder.endObject();
-                }
-            }
-            builder.endObject();
+            fields.toXContent(builder, FIELDS, SIZE_IN_BYTES, SIZE);
         }
         builder.endObject();
         return builder;
     }
 
-    public static CompletionStats readCompletionStats(StreamInput in) throws IOException {
-        CompletionStats stats = new CompletionStats();
-        stats.readFrom(in);
-        return stats;
-    }
-
-    static final class Fields {
-        static final String COMPLETION = "completion";
-        static final String SIZE_IN_BYTES = "size_in_bytes";
-        static final String SIZE = "size";
-        static final String FIELDS = "fields";
-    }
-
     public void add(CompletionStats completion) {
         if (completion == null) {
             return;
         }
-
         sizeInBytes += completion.getSizeInBytes();
-
         if (completion.fields != null) {
             if (fields == null) {
-                fields = completion.fields.clone();
+                fields = completion.fields.copy();
             } else {
-                assert !completion.fields.containsKey(null);
-                final Object[] keys = completion.fields.keys;
-                final long[] values = completion.fields.values;
-                for (int i = 0; i < keys.length; i++) {
-                    if (keys[i] != null) {
-                        fields.addTo((String) keys[i], values[i]);
-                    }
-                }
+                fields.add(completion.fields);
             }
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java
index f471d91c3b33b..19d73b5bc4122 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java
@@ -21,7 +21,6 @@
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.ReaderUtil;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.search.BulkScorer;
 import org.apache.lucene.search.CollectionTerminatedException;
@@ -35,18 +34,10 @@
 import org.apache.lucene.util.CollectionUtil;
 import org.apache.lucene.util.PriorityQueue;
 import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.text.Text;
-import org.elasticsearch.index.fielddata.AtomicFieldData;
-import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.index.mapper.CompletionFieldMapper;
-import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.suggest.Suggest;
 import org.elasticsearch.search.suggest.Suggester;
-import org.elasticsearch.search.suggest.SuggestionBuilder;
 import org.elasticsearch.search.suggest.completion2x.Completion090PostingsFormat;
 
 import java.io.IOException;
@@ -187,7 +178,7 @@ private static final class SuggestDoc extends TopSuggestDocs.SuggestScoreDoc {
 
             private List<TopSuggestDocs.SuggestScoreDoc> suggestScoreDocs;
 
-            public SuggestDoc(int doc, CharSequence key, CharSequence context, float score) {
+            SuggestDoc(int doc, CharSequence key, CharSequence context, float score) {
                 super(doc, key, context, score);
             }
 
@@ -231,7 +222,7 @@ public List<CharSequence> getContexts() {
 
         private static final class SuggestDocPriorityQueue extends PriorityQueue<SuggestDoc> {
 
-            public SuggestDocPriorityQueue(int maxSize) {
+            SuggestDocPriorityQueue(int maxSize) {
                 super(maxSize);
             }
 
@@ -263,8 +254,10 @@ public SuggestDoc[] getResults() {
         private final SuggestDocPriorityQueue pq;
         private final Map<Integer, SuggestDoc> scoreDocMap;
 
-        public TopDocumentsCollector(int num) {
-            super(1); // TODO hack, we don't use the underlying pq, so we allocate a size of 1
+        // TODO: expose dup removal
+        
+        TopDocumentsCollector(int num) {
+            super(1, false); // TODO hack, we don't use the underlying pq, so we allocate a size of 1
             this.num = num;
             this.scoreDocMap = new LinkedHashMap<>(num);
             this.pq = new SuggestDocPriorityQueue(num);
@@ -316,14 +309,4 @@ public TopSuggestDocs get() throws IOException {
             }
         }
     }
-
-    @Override
-    public SuggestionBuilder<?> innerFromXContent(QueryParseContext context) throws IOException {
-        return CompletionSuggestionBuilder.innerFromXContent(context);
-    }
-
-    @Override
-    public SuggestionBuilder<?> read(StreamInput in) throws IOException {
-        return new CompletionSuggestionBuilder(in);
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestion.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestion.java
index 84533710781d7..229b77aad2850 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestion.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestion.java
@@ -20,26 +20,30 @@
 
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.suggest.Lookup;
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.text.Text;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.search.internal.InternalSearchHit;
-import org.elasticsearch.search.internal.InternalSearchHits;
-import org.elasticsearch.search.internal.InternalSearchHits.StreamContext.ShardTargetType;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.suggest.Suggest;
+import org.elasticsearch.search.suggest.Suggest.Suggestion;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
 import static org.elasticsearch.search.suggest.Suggest.COMPARATOR;
 
 /**
@@ -60,6 +64,8 @@
  */
 public final class CompletionSuggestion extends Suggest.Suggestion<CompletionSuggestion.Entry> {
 
+    public static final String NAME = "completion";
+
     public static final int TYPE = 4;
 
     public CompletionSuggestion() {
@@ -88,6 +94,12 @@ public boolean hasScoreDocs() {
         return getOptions().size() > 0;
     }
 
+    public static CompletionSuggestion fromXContent(XContentParser parser, String name) throws IOException {
+        CompletionSuggestion suggestion = new CompletionSuggestion(name, -1);
+        parseEntries(parser, suggestion, CompletionSuggestion.Entry::fromXContent);
+        return suggestion;
+    }
+
     private static final class OptionPriorityQueue extends org.apache.lucene.util.PriorityQueue<Entry.Option> {
 
         private final Comparator<Suggest.Suggestion.Entry.Option> comparator;
@@ -168,10 +180,15 @@ public void setShardIndex(int shardIndex) {
     }
 
     @Override
-    public int getType() {
+    public int getWriteableType() {
         return TYPE;
     }
 
+    @Override
+    protected String getType() {
+        return NAME;
+    }
+
     @Override
     protected Entry newEntry() {
         return new Entry();
@@ -183,8 +200,7 @@ public Entry(Text text, int offset, int length) {
             super(text, offset, length);
         }
 
-        protected Entry() {
-            super();
+        Entry() {
         }
 
         @Override
@@ -192,15 +208,29 @@ protected Option newOption() {
             return new Option();
         }
 
+        private static ObjectParser<Entry, Void> PARSER = new ObjectParser<>("CompletionSuggestionEntryParser", true,
+                Entry::new);
+
+        static {
+            declareCommonFields(PARSER);
+            PARSER.declareObjectArray(Entry::addOptions, (p,c) -> Option.fromXContent(p), new ParseField(OPTIONS));
+        }
+
+        public static Entry fromXContent(XContentParser parser) {
+            return PARSER.apply(parser, null);
+        }
+
         public static class Option extends Suggest.Suggestion.Entry.Option {
-            private Map<String, Set<CharSequence>> contexts;
+            private Map<String, Set<CharSequence>> contexts = Collections.emptyMap();
             private ScoreDoc doc;
-            private InternalSearchHit hit;
+            private SearchHit hit;
+
+            public static final ParseField CONTEXTS = new ParseField("contexts");
 
             public Option(int docID, Text text, float score, Map<String, Set<CharSequence>> contexts) {
                 super(text, score);
                 this.doc = new ScoreDoc(docID, score);
-                this.contexts = contexts;
+                this.contexts = Objects.requireNonNull(contexts, "context map cannot be null");
             }
 
             protected Option() {
@@ -222,7 +252,7 @@ public ScoreDoc getDoc() {
                 return doc;
             }
 
-            public InternalSearchHit getHit() {
+            public SearchHit getHit() {
                 return hit;
             }
 
@@ -230,20 +260,20 @@ public void setShardIndex(int shardIndex) {
                 this.doc.shardIndex = shardIndex;
             }
 
-            public void setHit(InternalSearchHit hit) {
+            public void setHit(SearchHit hit) {
                 this.hit = hit;
             }
 
             @Override
             protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
-                builder.field("text", getText());
+                builder.field(TEXT.getPreferredName(), getText());
                 if (hit != null) {
                     hit.toInnerXContent(builder, params);
                 } else {
-                    builder.field("score", getScore());
+                    builder.field(SCORE.getPreferredName(), getScore());
                 }
                 if (contexts.size() > 0) {
-                    builder.startObject("contexts");
+                    builder.startObject(CONTEXTS.getPreferredName());
                     for (Map.Entry<String, Set<CharSequence>> entry : contexts.entrySet()) {
                         builder.startArray(entry.getKey());
                         for (CharSequence context : entry.getValue()) {
@@ -256,13 +286,64 @@ protected XContentBuilder innerToXContent(XContentBuilder builder, Params params
                 return builder;
             }
 
+            private static ObjectParser<Map<String, Object>, Void> PARSER = new ObjectParser<>("CompletionOptionParser",
+                    true, HashMap::new);
+
+            static {
+                SearchHit.declareInnerHitsParseFields(PARSER);
+                PARSER.declareString((map, value) -> map.put(Suggestion.Entry.Option.TEXT.getPreferredName(), value),
+                        Suggestion.Entry.Option.TEXT);
+                PARSER.declareFloat((map, value) -> map.put(Suggestion.Entry.Option.SCORE.getPreferredName(), value),
+                        Suggestion.Entry.Option.SCORE);
+                PARSER.declareObject((map, value) -> map.put(CompletionSuggestion.Entry.Option.CONTEXTS.getPreferredName(), value),
+                        (p,c) -> parseContexts(p), CompletionSuggestion.Entry.Option.CONTEXTS);
+            }
+
+            private static Map<String, Set<CharSequence>> parseContexts(XContentParser parser) throws IOException {
+                Map<String, Set<CharSequence>> contexts = new HashMap<>();
+                while((parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                    ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser::getTokenLocation);
+                    String key = parser.currentName();
+                    ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.nextToken(), parser::getTokenLocation);
+                    Set<CharSequence> values = new HashSet<>();
+                    while((parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                        ensureExpectedToken(XContentParser.Token.VALUE_STRING, parser.currentToken(), parser::getTokenLocation);
+                        values.add(parser.text());
+                    }
+                    contexts.put(key, values);
+                }
+                return contexts;
+            }
+
+            public static Option fromXContent(XContentParser parser) {
+                Map<String, Object> values = PARSER.apply(parser, null);
+
+                Text text = new Text((String) values.get(Suggestion.Entry.Option.TEXT.getPreferredName()));
+                Float score = (Float) values.get(Suggestion.Entry.Option.SCORE.getPreferredName());
+                @SuppressWarnings("unchecked")
+                Map<String, Set<CharSequence>> contexts = (Map<String, Set<CharSequence>>) values
+                        .get(CompletionSuggestion.Entry.Option.CONTEXTS.getPreferredName());
+                if (contexts == null) {
+                    contexts = Collections.emptyMap();
+                }
+
+                SearchHit hit = null;
+                // the option either prints SCORE or inlines the search hit
+                if (score == null) {
+                    hit = SearchHit.createFromMap(values);
+                    score = hit.getScore();
+                }
+                CompletionSuggestion.Entry.Option option = new CompletionSuggestion.Entry.Option(-1, text, score, contexts);
+                option.setHit(hit);
+                return option;
+            }
+
             @Override
             public void readFrom(StreamInput in) throws IOException {
                 super.readFrom(in);
                 this.doc = Lucene.readScoreDoc(in);
                 if (in.readBoolean()) {
-                    this.hit = InternalSearchHit.readSearchHit(in,
-                        InternalSearchHits.streamContext().streamShardTarget(ShardTargetType.STREAM));
+                    this.hit = SearchHit.readSearchHit(in);
                 }
                 int contextSize = in.readInt();
                 this.contexts = new LinkedHashMap<>(contextSize);
@@ -283,7 +364,7 @@ public void writeTo(StreamOutput out) throws IOException {
                 Lucene.writeScoreDoc(out, doc);
                 if (hit != null) {
                     out.writeBoolean(true);
-                    hit.writeTo(out, InternalSearchHits.streamContext().streamShardTarget(ShardTargetType.STREAM));
+                    hit.writeTo(out);
                 } else {
                     out.writeBoolean(false);
                 }
@@ -314,7 +395,6 @@ public String toString() {
                 stringBuilder.append("]");
                 return stringBuilder.toString();
             }
-
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java
index 3b216d9186ab0..22abf42e27328 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java
@@ -20,7 +20,6 @@
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -35,7 +34,6 @@
 import org.elasticsearch.index.mapper.CompletionFieldMapper2x;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.suggest.SuggestionBuilder;
 import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
@@ -72,27 +70,26 @@ public class CompletionSuggestionBuilder extends SuggestionBuilder<CompletionSug
      *     "payload" : STRING_ARRAY
      * }
      */
-    private static ObjectParser<CompletionSuggestionBuilder.InnerBuilder, ParseFieldMatcherSupplier> TLP_PARSER =
-        new ObjectParser<>(SUGGESTION_NAME, null);
+    private static final ObjectParser<CompletionSuggestionBuilder.InnerBuilder, Void> PARSER = new ObjectParser<>(SUGGESTION_NAME, null);
     static {
-        TLP_PARSER.declareField((parser, completionSuggestionContext, context) -> {
+        PARSER.declareField((parser, completionSuggestionContext, context) -> {
                 if (parser.currentToken() == XContentParser.Token.VALUE_BOOLEAN) {
                     if (parser.booleanValue()) {
                         completionSuggestionContext.fuzzyOptions = new FuzzyOptions.Builder().build();
                     }
                 } else {
-                    completionSuggestionContext.fuzzyOptions = FuzzyOptions.parse(parser, context);
+                    completionSuggestionContext.fuzzyOptions = FuzzyOptions.parse(parser);
                 }
             },
             FuzzyOptions.FUZZY_OPTIONS, ObjectParser.ValueType.OBJECT_OR_BOOLEAN);
-        TLP_PARSER.declareField((parser, completionSuggestionContext, context) ->
-            completionSuggestionContext.regexOptions = RegexOptions.parse(parser, context),
+        PARSER.declareField((parser, completionSuggestionContext, context) ->
+            completionSuggestionContext.regexOptions = RegexOptions.parse(parser),
             RegexOptions.REGEX_OPTIONS, ObjectParser.ValueType.OBJECT);
-        TLP_PARSER.declareString(CompletionSuggestionBuilder.InnerBuilder::field, FIELDNAME_FIELD);
-        TLP_PARSER.declareString(CompletionSuggestionBuilder.InnerBuilder::analyzer, ANALYZER_FIELD);
-        TLP_PARSER.declareInt(CompletionSuggestionBuilder.InnerBuilder::size, SIZE_FIELD);
-        TLP_PARSER.declareInt(CompletionSuggestionBuilder.InnerBuilder::shardSize, SHARDSIZE_FIELD);
-        TLP_PARSER.declareField((p, v, c) -> {
+        PARSER.declareString(CompletionSuggestionBuilder.InnerBuilder::field, FIELDNAME_FIELD);
+        PARSER.declareString(CompletionSuggestionBuilder.InnerBuilder::analyzer, ANALYZER_FIELD);
+        PARSER.declareInt(CompletionSuggestionBuilder.InnerBuilder::size, SIZE_FIELD);
+        PARSER.declareInt(CompletionSuggestionBuilder.InnerBuilder::shardSize, SHARDSIZE_FIELD);
+        PARSER.declareField((p, v, c) -> {
             // Copy the current structure. We will parse, once the mapping is provided
             XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
             builder.copyCurrentStructure(p);
@@ -317,7 +314,7 @@ public Contexts2x addContextField(String name, Iterable<? extends CharSequence>
     private static class InnerBuilder extends CompletionSuggestionBuilder {
         private String field;
 
-        public InnerBuilder() {
+        InnerBuilder() {
             super("_na_");
         }
 
@@ -336,17 +333,14 @@ protected XContentBuilder innerToXContent(XContentBuilder builder, Params params
             regexOptions.toXContent(builder, params);
         }
         if (contextBytes != null) {
-            try (XContentParser contextParser = XContentFactory.xContent(XContentType.JSON).createParser(contextBytes)) {
-                builder.field(CONTEXTS_FIELD.getPreferredName());
-                builder.copyCurrentStructure(contextParser);
-            }
+            builder.rawField(CONTEXTS_FIELD.getPreferredName(), contextBytes);
         }
         return builder;
     }
 
-    static CompletionSuggestionBuilder innerFromXContent(QueryParseContext parseContext) throws IOException {
+    public static CompletionSuggestionBuilder fromXContent(XContentParser parser) throws IOException {
         CompletionSuggestionBuilder.InnerBuilder builder = new CompletionSuggestionBuilder.InnerBuilder();
-        TLP_PARSER.parse(parseContext.parser(), builder, parseContext);
+        PARSER.parse(parser, builder, null);
         String field = builder.field;
         // now we should have field name, check and copy fields over to the suggestion builder we return
         if (field == null) {
@@ -374,7 +368,8 @@ public SuggestionContext build(QueryShardContext context) throws IOException {
             CompletionFieldMapper.CompletionFieldType type = (CompletionFieldMapper.CompletionFieldType) mappedFieldType;
             suggestionContext.setFieldType(type);
             if (type.hasContextMappings() && contextBytes != null) {
-                try (XContentParser contextParser = XContentFactory.xContent(contextBytes).createParser(contextBytes)) {
+                try (XContentParser contextParser = XContentFactory.xContent(contextBytes).createParser(context.getXContentRegistry(),
+                        contextBytes)) {
                     if (type.hasContextMappings() && contextParser != null) {
                         ContextMappings contextMappings = type.getContextMappings();
                         contextParser.nextToken();
@@ -400,7 +395,8 @@ public SuggestionContext build(QueryShardContext context) throws IOException {
             suggestionContext.setFieldType2x(type);
             if (type.requiresContext()) {
                 if (contextBytes != null) {
-                    try (XContentParser contextParser = XContentFactory.xContent(contextBytes).createParser(contextBytes)) {
+                    try (XContentParser contextParser = XContentFactory.xContent(contextBytes)
+                            .createParser(suggestionContext.getShardContext().getXContentRegistry(), contextBytes)) {
                         contextParser.nextToken();
                         suggestionContext.setContextQueries(ContextQuery.parseQueries(type.getContextMapping(), contextParser));
                     }
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionContext.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionContext.java
index 84eb9de6a5947..d318d23b7e412 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionContext.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionContext.java
@@ -19,6 +19,7 @@
 package org.elasticsearch.search.suggest.completion;
 
 import org.apache.lucene.search.suggest.document.CompletionQuery;
+import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.index.mapper.CompletionFieldMapper;
 import org.elasticsearch.index.mapper.CompletionFieldMapper2x;
@@ -88,15 +89,7 @@ CompletionQuery toQuery() {
         CompletionFieldMapper.CompletionFieldType fieldType = getFieldType();
         final CompletionQuery query;
         if (getPrefix() != null) {
-            if (fuzzyOptions != null) {
-                query = fieldType.fuzzyQuery(getPrefix().utf8ToString(),
-                        Fuzziness.fromEdits(fuzzyOptions.getEditDistance()),
-                        fuzzyOptions.getFuzzyPrefixLength(), fuzzyOptions.getFuzzyMinLength(),
-                        fuzzyOptions.getMaxDeterminizedStates(), fuzzyOptions.isTranspositions(),
-                        fuzzyOptions.isUnicodeAware());
-            } else {
-                query = fieldType.prefixQuery(getPrefix());
-            }
+            query = createCompletionQuery(getPrefix(), fieldType);
         } else if (getRegex() != null) {
             if (fuzzyOptions != null) {
                 throw new IllegalArgumentException("can not use 'fuzzy' options with 'regex");
@@ -106,8 +99,10 @@ CompletionQuery toQuery() {
             }
             query = fieldType.regexpQuery(getRegex(), regexOptions.getFlagsValue(),
                     regexOptions.getMaxDeterminizedStates());
+        } else if (getText() != null) {
+            query = createCompletionQuery(getText(), fieldType);
         } else {
-            throw new IllegalArgumentException("'prefix' or 'regex' must be defined");
+            throw new IllegalArgumentException("'prefix/text' or 'regex' must be defined");
         }
         if (fieldType.hasContextMappings()) {
             ContextMappings contextMappings = fieldType.getContextMappings();
@@ -116,6 +111,20 @@ CompletionQuery toQuery() {
         return query;
     }
 
+    private CompletionQuery createCompletionQuery(BytesRef prefix, CompletionFieldMapper.CompletionFieldType fieldType) {
+        final CompletionQuery query;
+        if (fuzzyOptions != null) {
+            query = fieldType.fuzzyQuery(prefix.utf8ToString(),
+                    Fuzziness.fromEdits(fuzzyOptions.getEditDistance()),
+                    fuzzyOptions.getFuzzyPrefixLength(), fuzzyOptions.getFuzzyMinLength(),
+                    fuzzyOptions.getMaxDeterminizedStates(), fuzzyOptions.isTranspositions(),
+                    fuzzyOptions.isUnicodeAware());
+        } else {
+            query = fieldType.prefixQuery(prefix);
+        }
+        return query;
+    }
+
     public void setFieldType2x(CompletionFieldMapper2x.CompletionFieldType type) {
         this.fieldType2x = type;
     }
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/FuzzyOptions.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/FuzzyOptions.java
index 3ae05f9ecf5d1..8d121b24f1229 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/completion/FuzzyOptions.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/FuzzyOptions.java
@@ -22,7 +22,6 @@
 import org.apache.lucene.search.suggest.document.FuzzyCompletionQuery;
 import org.apache.lucene.util.automaton.Operations;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -56,7 +55,7 @@ public class FuzzyOptions implements ToXContent, Writeable {
      *     "max_determinized_states" : INT
      * }
      */
-    private static ObjectParser<Builder, ParseFieldMatcherSupplier> PARSER = new ObjectParser<>(FUZZY_OPTIONS.getPreferredName(),
+    private static final ObjectParser<Builder, Void> PARSER = new ObjectParser<>(FUZZY_OPTIONS.getPreferredName(),
             Builder::new);
     static {
         PARSER.declareInt(Builder::setFuzzyMinLength, MIN_LENGTH_FIELD);
@@ -67,6 +66,14 @@ public class FuzzyOptions implements ToXContent, Writeable {
         PARSER.declareField(Builder::setFuzziness, Fuzziness::parse, Fuzziness.FIELD, ObjectParser.ValueType.VALUE);
     }
 
+    static FuzzyOptions parse(XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null).build();
+    }
+
+    public static Builder builder() {
+        return new Builder();
+    }
+
     private int editDistance;
     private boolean transpositions;
     private int fuzzyMinLength;
@@ -106,14 +113,6 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeVInt(maxDeterminizedStates);
     }
 
-    static FuzzyOptions parse(XContentParser parser, ParseFieldMatcherSupplier context) throws IOException {
-        return PARSER.parse(parser, context).build();
-    }
-
-    public static Builder builder() {
-        return new Builder();
-    }
-
     /**
      * Returns the maximum number of edits
      */
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/RegexOptions.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/RegexOptions.java
index a9209044c8b18..f330322a9f065 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/completion/RegexOptions.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/RegexOptions.java
@@ -23,7 +23,6 @@
 import org.apache.lucene.util.automaton.RegExp;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -49,7 +48,7 @@ public class RegexOptions implements ToXContent, Writeable {
      *     "max_determinized_states" : INT
      * }
      */
-    private static ObjectParser<Builder, ParseFieldMatcherSupplier> PARSER = new ObjectParser<>(REGEX_OPTIONS.getPreferredName(),
+    private static final ObjectParser<Builder, Void> PARSER = new ObjectParser<>(REGEX_OPTIONS.getPreferredName(),
             Builder::new);
     static {
         PARSER.declareInt(Builder::setMaxDeterminizedStates, MAX_DETERMINIZED_STATES);
@@ -66,6 +65,14 @@ public class RegexOptions implements ToXContent, Writeable {
         PARSER.declareStringOrNull(Builder::setFlags, FLAGS_VALUE);
     }
 
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    static RegexOptions parse(XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null).build();
+    }
+
     private int flagsValue;
     private int maxDeterminizedStates;
 
@@ -103,14 +110,6 @@ public int getMaxDeterminizedStates() {
         return maxDeterminizedStates;
     }
 
-    public static Builder builder() {
-        return new Builder();
-    }
-
-    static RegexOptions parse(XContentParser parser, ParseFieldMatcherSupplier context) throws IOException {
-        return PARSER.parse(parser, context).build();
-    }
-
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/context/CategoryContextMapping.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/CategoryContextMapping.java
index 150b7bf4f907a..9b839511607e3 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/completion/context/CategoryContextMapping.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/CategoryContextMapping.java
@@ -19,15 +19,21 @@
 
 package org.elasticsearch.search.suggest.completion.context;
 
+import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.SortedSetDocValuesField;
+import org.apache.lucene.document.StoredField;
 import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.search.SortedSetSortField;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentParser.Token;
+import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.ParseContext.Document;
 import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.index.mapper.StringFieldType;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -107,21 +113,24 @@ protected XContentBuilder toInnerXContent(XContentBuilder builder, Params params
      *  </ul>
      */
     @Override
-    public Set<CharSequence> parseContext(ParseContext parseContext, XContentParser parser) throws IOException, ElasticsearchParseException {
+    public Set<CharSequence> parseContext(ParseContext parseContext, XContentParser parser)
+            throws IOException, ElasticsearchParseException {
         final Set<CharSequence> contexts = new HashSet<>();
         Token token = parser.currentToken();
-        if (token == Token.VALUE_STRING) {
+        if (token == Token.VALUE_STRING || token == Token.VALUE_NUMBER || token == Token.VALUE_BOOLEAN) {
             contexts.add(parser.text());
         } else if (token == Token.START_ARRAY) {
             while ((token = parser.nextToken()) != Token.END_ARRAY) {
-                if (token == Token.VALUE_STRING) {
+                if (token == Token.VALUE_STRING || token == Token.VALUE_NUMBER || token == Token.VALUE_BOOLEAN) {
                     contexts.add(parser.text());
                 } else {
-                    throw new ElasticsearchParseException("context array must have string values");
+                    throw new ElasticsearchParseException(
+                            "context array must have string, number or boolean values, but was [" + token + "]");
                 }
             }
         } else {
-            throw new ElasticsearchParseException("contexts must be a string or a list of strings");
+            throw new ElasticsearchParseException(
+                    "contexts must be a string, number or boolean or a list of string, number or boolean, but was [" + token + "]");
         }
         return contexts;
     }
@@ -133,10 +142,20 @@ public Set<CharSequence> parseContext(Document document) {
             IndexableField[] fields = document.getFields(fieldName);
             values = new HashSet<>(fields.length);
             for (IndexableField field : fields) {
-                values.add(field.stringValue());
+                if (field instanceof SortedDocValuesField ||
+                        field instanceof SortedSetDocValuesField ||
+                        field instanceof StoredField) {
+                    // Ignore doc values and stored fields
+                } else if (field.fieldType() instanceof KeywordFieldMapper.KeywordFieldType) {
+                    values.add(field.binaryValue().utf8ToString());
+                } else if (field.fieldType() instanceof StringFieldType) {
+                    values.add(field.stringValue());
+                } else {
+                    throw new IllegalArgumentException("Failed to parse context field [" + fieldName + "], only keyword and text fields are accepted");
+                }
             }
         }
-        return (values == null) ? Collections.<CharSequence>emptySet() : values;
+        return (values == null) ? Collections.emptySet() : values;
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/context/CategoryQueryContext.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/CategoryQueryContext.java
index a17b7a87b432a..51b740a3529e9 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/completion/context/CategoryQueryContext.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/CategoryQueryContext.java
@@ -21,8 +21,7 @@
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
+import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -98,9 +97,10 @@ public int hashCode() {
         return result;
     }
 
-    private static ObjectParser<Builder, ParseFieldMatcherSupplier> CATEGORY_PARSER = new ObjectParser<>(NAME, null);
+    private static ObjectParser<Builder, Void> CATEGORY_PARSER = new ObjectParser<>(NAME, null);
     static {
-        CATEGORY_PARSER.declareString(Builder::setCategory, new ParseField(CONTEXT_VALUE));
+        CATEGORY_PARSER.declareField(Builder::setCategory, XContentParser::text, new ParseField(CONTEXT_VALUE),
+                ObjectParser.ValueType.VALUE);
         CATEGORY_PARSER.declareInt(Builder::setBoost, new ParseField(CONTEXT_BOOST));
         CATEGORY_PARSER.declareBoolean(Builder::setPrefix, new ParseField(CONTEXT_PREFIX));
     }
@@ -110,11 +110,16 @@ public static CategoryQueryContext fromXContent(QueryParseContext context) throw
         XContentParser.Token token = parser.currentToken();
         Builder builder = builder();
         if (token == XContentParser.Token.START_OBJECT) {
-            CATEGORY_PARSER.parse(parser, builder, () -> ParseFieldMatcher.STRICT);
-        } else if (token == XContentParser.Token.VALUE_STRING) {
+            try {
+                CATEGORY_PARSER.parse(parser, builder, null);
+            } catch(ParsingException e) {
+                throw new ElasticsearchParseException("category context must be a string, number or boolean");
+            } 
+        } else if (token == XContentParser.Token.VALUE_STRING || token == XContentParser.Token.VALUE_BOOLEAN
+                || token == XContentParser.Token.VALUE_NUMBER) {
             builder.setCategory(parser.text());
         } else {
-            throw new ElasticsearchParseException("category context must be an object or string");
+            throw new ElasticsearchParseException("category context must be an object, string, number or boolean");
         }
         return builder.build();
     }
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/context/ContextMapping.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/ContextMapping.java
index f41273662a4c8..273138bbb7969 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/completion/context/ContextMapping.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/ContextMapping.java
@@ -109,13 +109,14 @@ public final List<InternalQueryContext> parseQueryContext(QueryParseContext cont
         List<T> queryContexts = new ArrayList<>();
         XContentParser parser = context.parser();
         Token token = parser.nextToken();
-        if (token == Token.START_OBJECT || token == Token.VALUE_STRING) {
-            queryContexts.add(fromXContent(context));
-        } else if (token == Token.START_ARRAY) {
+        if (token == Token.START_ARRAY) {
             while (parser.nextToken() != Token.END_ARRAY) {
                 queryContexts.add(fromXContent(context));
             }
+        } else {
+            queryContexts.add(fromXContent(context));
         }
+        
         return toInternalQueryContexts(queryContexts);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/context/ContextMappings.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/ContextMappings.java
index 5a7cfccf3156a..bbefbf41b8907 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/completion/context/ContextMappings.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/ContextMappings.java
@@ -81,7 +81,9 @@ public int size() {
     public ContextMapping get(String name) {
         ContextMapping contextMapping = contextNameMap.get(name);
         if (contextMapping == null) {
-            throw new IllegalArgumentException("Unknown context name[" + name + "], must be one of " + contextNameMap.size());
+            List<String> keys = new ArrayList<>(contextNameMap.keySet());
+            Collections.sort(keys);
+            throw new IllegalArgumentException("Unknown context name [" + name + "], must be one of " + keys.toString());
         }
         return contextMapping;
     }
@@ -115,7 +117,7 @@ private class TypedContextField extends ContextSuggestField {
         private final Map<String, Set<CharSequence>> contexts;
         private final ParseContext.Document document;
 
-        public TypedContextField(String name, String value, int weight, Map<String, Set<CharSequence>> contexts,
+        TypedContextField(String name, String value, int weight, Map<String, Set<CharSequence>> contexts,
                                  ParseContext.Document document) {
             super(name, value, weight);
             this.contexts = contexts;
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/context/GeoContextMapping.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/GeoContextMapping.java
index 854a5fb0e1a49..bd1449bbfe7a5 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/completion/context/GeoContextMapping.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/GeoContextMapping.java
@@ -207,7 +207,8 @@ public Set<CharSequence> parseContext(Document document) {
                     if (field instanceof StringField) {
                         spare.resetFromString(field.stringValue());
                     } else {
-                        spare.resetFromIndexHash(Long.parseLong(field.stringValue()));
+                        // todo return this to .stringValue() once LatLonPoint implements it
+                        spare.resetFromIndexableField(field);
                     }
                     geohashes.add(spare.geohash());
                 }
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/context/GeoQueryContext.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/GeoQueryContext.java
index 6ed8754c9786a..e2f00c0001cf4 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/completion/context/GeoQueryContext.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/GeoQueryContext.java
@@ -21,7 +21,6 @@
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.geo.GeoUtils;
 import org.elasticsearch.common.xcontent.ObjectParser;
@@ -113,7 +112,7 @@ public static Builder builder() {
         return new Builder();
     }
 
-    private static ObjectParser<GeoQueryContext.Builder, ParseFieldMatcherSupplier> GEO_CONTEXT_PARSER = new ObjectParser<>(NAME, null);
+    private static ObjectParser<GeoQueryContext.Builder, Void> GEO_CONTEXT_PARSER = new ObjectParser<>(NAME, null);
     static {
         GEO_CONTEXT_PARSER.declareField((parser, geoQueryContext, geoContextMapping) -> geoQueryContext.setGeoPoint(GeoUtils.parseGeoPoint(parser)), new ParseField(CONTEXT_VALUE), ObjectParser.ValueType.OBJECT);
         GEO_CONTEXT_PARSER.declareInt(GeoQueryContext.Builder::setBoost, new ParseField(CONTEXT_BOOST));
@@ -130,7 +129,7 @@ public static GeoQueryContext fromXContent(QueryParseContext context) throws IOE
         XContentParser.Token token = parser.currentToken();
         GeoQueryContext.Builder builder = new Builder();
         if (token == XContentParser.Token.START_OBJECT) {
-            GEO_CONTEXT_PARSER.parse(parser, builder, context);
+            GEO_CONTEXT_PARSER.parse(parser, builder, null);
         } else if (token == XContentParser.Token.VALUE_STRING) {
             builder.setGeoPoint(GeoPoint.fromGeohash(parser.text()));
         } else {
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion2x/AnalyzingCompletionLookupProvider.java b/core/src/main/java/org/elasticsearch/search/suggest/completion2x/AnalyzingCompletionLookupProvider.java
index e9095bfb7d576..dcbca0c842c9e 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/completion2x/AnalyzingCompletionLookupProvider.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion2x/AnalyzingCompletionLookupProvider.java
@@ -45,6 +45,7 @@
 import org.apache.lucene.util.fst.PairOutputs;
 import org.apache.lucene.util.fst.PairOutputs.Pair;
 import org.apache.lucene.util.fst.PositiveIntOutputs;
+import org.elasticsearch.common.FieldMemoryStats;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.index.mapper.CompletionFieldMapper2x;
 import org.elasticsearch.index.mapper.MappedFieldType;
@@ -321,8 +322,7 @@ public CompletionStats stats(String... fields) {
                         completionFields.addTo(entry.getKey(), fstSize);
                     }
                 }
-
-                return new CompletionStats(sizeInBytes, completionFields);
+                return new CompletionStats(sizeInBytes, completionFields == null ? null : new FieldMemoryStats(completionFields));
             }
 
             @Override
@@ -355,7 +355,7 @@ static class AnalyzingSuggestHolder implements Accountable {
         final int endByte;
         final int holeCharacter;
 
-        public AnalyzingSuggestHolder(boolean preserveSep, boolean preservePositionIncrements,
+        AnalyzingSuggestHolder(boolean preserveSep, boolean preservePositionIncrements,
                                       int maxSurfaceFormsPerAnalyzedForm, int maxGraphExpansions,
                                       boolean hasPayloads, int maxAnalyzedPathsForOneInput,
                                       FST<Pair<Long, BytesRef>> fst, int sepLabel, int payloadSep,
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion2x/Completion090PostingsFormat.java b/core/src/main/java/org/elasticsearch/search/suggest/completion2x/Completion090PostingsFormat.java
index 245f2416b40cb..34f8d56223ba4 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/completion2x/Completion090PostingsFormat.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion2x/Completion090PostingsFormat.java
@@ -121,7 +121,7 @@ private class CompletionFieldsConsumer extends FieldsConsumer {
         private FieldsConsumer delegatesFieldsConsumer;
         private FieldsConsumer suggestFieldsConsumer;
 
-        public CompletionFieldsConsumer(SegmentWriteState state) throws IOException {
+        CompletionFieldsConsumer(SegmentWriteState state) throws IOException {
             this.delegatesFieldsConsumer = delegatePostingsFormat.fieldsConsumer(state);
             String suggestFSTFile = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, EXTENSION);
             IndexOutput output = null;
@@ -163,7 +163,7 @@ private static class CompletionFieldsProducer extends FieldsProducer {
         private final LookupFactory lookupFactory;
         private final int version;
 
-        public CompletionFieldsProducer(SegmentReadState state) throws IOException {
+        CompletionFieldsProducer(SegmentReadState state) throws IOException {
             String suggestFSTFile = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, EXTENSION);
             IndexInput input = state.directory.openInput(suggestFSTFile, state.context);
             if (state.segmentInfo.getVersion().onOrAfter(Version.LUCENE_6_2_0)) {
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion2x/CompletionSuggestion.java b/core/src/main/java/org/elasticsearch/search/suggest/completion2x/CompletionSuggestion.java
index cf0939b85f9a5..a57e163cce9f3 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/completion2x/CompletionSuggestion.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion2x/CompletionSuggestion.java
@@ -46,7 +46,7 @@ public CompletionSuggestion(String name, int size) {
     }
 
     @Override
-    public int getType() {
+    public int getWriteableType() {
         return TYPE;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion2x/PayloadProcessor.java b/core/src/main/java/org/elasticsearch/search/suggest/completion2x/PayloadProcessor.java
index eb857ce61e6c7..819e781d545a2 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/completion2x/PayloadProcessor.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion2x/PayloadProcessor.java
@@ -30,7 +30,7 @@ interface PayloadProcessor {
 
     void parsePayload(BytesRef payload, SuggestPayload ref) throws IOException;
 
-    static class SuggestPayload {
+    class SuggestPayload {
         final BytesRefBuilder payload = new BytesRefBuilder();
         long weight = 0;
         final BytesRefBuilder surfaceForm = new BytesRefBuilder();
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion2x/context/CategoryContextMapping.java b/core/src/main/java/org/elasticsearch/search/suggest/completion2x/context/CategoryContextMapping.java
index 775d8b031ad64..cba6cb61949d4 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/completion2x/context/CategoryContextMapping.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion2x/context/CategoryContextMapping.java
@@ -234,7 +234,7 @@ private static class FieldConfig extends ContextConfig {
         private final Iterable<String> defaultValues;
         private final Iterable<String> values;
 
-        public FieldConfig(String fieldname, Iterable<String> defaultValues, Iterable<String> values) {
+        FieldConfig(String fieldname, Iterable<String> defaultValues, Iterable<String> values) {
             this.fieldname = fieldname;
             this.defaultValues = defaultValues;
             this.values = values;
@@ -293,7 +293,7 @@ private static class FieldQuery extends ContextQuery {
 
         private final Iterable<? extends CharSequence> values;
 
-        public FieldQuery(String name, Iterable<? extends CharSequence> values) {
+        FieldQuery(String name, Iterable<? extends CharSequence> values) {
             super(name);
             this.values = values;
         }
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion2x/context/GeolocationContextMapping.java b/core/src/main/java/org/elasticsearch/search/suggest/completion2x/context/GeolocationContextMapping.java
index 21031d36cc80f..1a06203d7c274 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/completion2x/context/GeolocationContextMapping.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion2x/context/GeolocationContextMapping.java
@@ -208,7 +208,7 @@ protected static GeolocationContextMapping load(String name, Map<String, Object>
 
     @Override
     protected XContentBuilder toInnerXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.field(FIELD_PRECISION, precision);
+        builder.array(FIELD_PRECISION, precision);
         builder.field(FIELD_NEIGHBORS, neighbors);
         if (defaultLocations != null) {
             builder.startArray(FIELD_MISSING);
@@ -625,7 +625,7 @@ private static class GeoConfig extends ContextConfig {
         private final GeolocationContextMapping mapping;
         private final Collection<String> locations;
 
-        public GeoConfig(GeolocationContextMapping mapping, Collection<String> locations) {
+        GeoConfig(GeolocationContextMapping mapping, Collection<String> locations) {
             this.locations = locations;
             this.mapping = mapping;
         }
@@ -712,7 +712,7 @@ private static class GeoQuery extends ContextQuery {
         private final String location;
         private final int[] precisions;
 
-        public GeoQuery(String name, String location, int...precisions) {
+        GeoQuery(String name, String location, int...precisions) {
             super(name);
             this.location = location;
             this.precisions = precisions;
@@ -741,7 +741,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             } else {
                 builder.startObject(name);
                 builder.field(FIELD_VALUE, location);
-                builder.field(FIELD_PRECISION, precisions);
+                builder.array(FIELD_PRECISION, precisions);
                 builder.endObject();
             }
             return builder;
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/CandidateScorer.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/CandidateScorer.java
index b3dad43ad340a..d24ce6b3c29a0 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/CandidateScorer.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/CandidateScorer.java
@@ -29,7 +29,7 @@ final class CandidateScorer {
     private final int maxNumCorrections;
     private final int gramSize;
 
-    public CandidateScorer(WordScorer scorer, int maxNumCorrections, int gramSize) {
+    CandidateScorer(WordScorer scorer, int maxNumCorrections, int gramSize) {
         this.scorer = scorer;
         this.maxNumCorrections = maxNumCorrections;
         this.gramSize = gramSize;
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGenerator.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGenerator.java
index f6faaaeea5cef..b874c3aeca311 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGenerator.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGenerator.java
@@ -58,7 +58,7 @@ public final class DirectCandidateGenerator extends CandidateGenerator {
     private final TermsEnum termsEnum;
     private final IndexReader reader;
     private final long dictSize;
-    private final double logBase = 5;
+    private static final double LOG_BASE = 5;
     private final long frequencyPlateau;
     private final Analyzer preFilter;
     private final Analyzer postFilter;
@@ -189,7 +189,7 @@ private double score(long frequency, double errorScore, long dictionarySize) {
 
     protected long thresholdFrequency(long termFrequency, long dictionarySize) {
         if (termFrequency > 0) {
-            return max(0, round(termFrequency * (log10(termFrequency - frequencyPlateau) * (1.0 / log10(logBase))) + 1));
+            return max(0, round(termFrequency * (log10(termFrequency - frequencyPlateau) * (1.0 / log10(LOG_BASE))) + 1));
         }
         return 0;
 
@@ -229,9 +229,8 @@ public void addCandidates(List<Candidate> candidates) {
             // Merge new candidates into existing ones,
             // deduping:
             final Set<Candidate> set = new HashSet<>(candidates);
-            for (int i = 0; i < this.candidates.length; i++) {
-                set.add(this.candidates[i]);
-            }
+            Collections.addAll(set, this.candidates);
+
             this.candidates = set.toArray(new Candidate[set.size()]);
             // Sort strongest to weakest:
             Arrays.sort(this.candidates, Collections.reverseOrder());
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorBuilder.java
index 9e3beb2ccf1ab..0c5d5ba832a62 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorBuilder.java
@@ -35,8 +35,8 @@
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.suggest.SortBy;
 import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.CandidateGenerator;
 
@@ -151,6 +151,10 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalString(postFilter);
     }
 
+    String field() {
+        return this.field;
+    }
+
     /**
      * The global suggest mode controls what suggested terms are included or
      * controls for what suggest text tokens, terms should be suggested for.
@@ -169,6 +173,10 @@ public DirectCandidateGeneratorBuilder suggestMode(String suggestMode) {
         return this;
     }
 
+    String suggestMode() {
+        return suggestMode;
+    }
+
     /**
      * Sets how similar the suggested terms at least need to be compared to
      * the original suggest text tokens. A value between 0 and 1 can be
@@ -182,6 +190,10 @@ public DirectCandidateGeneratorBuilder accuracy(float accuracy) {
         return this;
     }
 
+    Float accuracy() {
+        return this.accuracy;
+    }
+
     /**
      * Sets the maximum suggestions to be returned per suggest text term.
      */
@@ -193,6 +205,10 @@ public DirectCandidateGeneratorBuilder size(int size) {
         return this;
     }
 
+    Integer size() {
+        return size;
+    }
+
     /**
      * Sets how to sort the suggest terms per suggest text token. Two
      * possible values:
@@ -210,6 +226,10 @@ public DirectCandidateGeneratorBuilder sort(String sort) {
         return this;
     }
 
+    String sort() {
+        return sort;
+    }
+
     /**
      * Sets what string distance implementation to use for comparing how
      * similar suggested terms are. Four possible values can be specified:
@@ -232,6 +252,10 @@ public DirectCandidateGeneratorBuilder stringDistance(String stringDistance) {
         return this;
     }
 
+    String stringDistance() {
+        return stringDistance;
+    }
+
     /**
      * Sets the maximum edit distance candidate suggestions can have in
      * order to be considered as a suggestion. Can only be a value between 1
@@ -246,6 +270,10 @@ public DirectCandidateGeneratorBuilder maxEdits(Integer maxEdits) {
         return this;
     }
 
+    Integer maxEdits() {
+        return maxEdits;
+    }
+
     /**
      * A factor that is used to multiply with the size in order to inspect
      * more candidate suggestions. Can improve accuracy at the cost of
@@ -256,6 +284,10 @@ public DirectCandidateGeneratorBuilder maxInspections(Integer maxInspections) {
         return this;
     }
 
+    Integer maxInspections() {
+        return maxInspections;
+    }
+
     /**
      * Sets a maximum threshold in number of documents a suggest text token
      * can exist in order to be corrected. Can be a relative percentage
@@ -272,6 +304,10 @@ public DirectCandidateGeneratorBuilder maxTermFreq(float maxTermFreq) {
         return this;
     }
 
+    Float maxTermFreq() {
+        return maxTermFreq;
+    }
+
     /**
      * Sets the number of minimal prefix characters that must match in order
      * be a candidate suggestion. Defaults to 1. Increasing this number
@@ -283,6 +319,10 @@ public DirectCandidateGeneratorBuilder prefixLength(int prefixLength) {
         return this;
     }
 
+    Integer prefixLength() {
+        return prefixLength;
+    }
+
     /**
      * The minimum length a suggest text term must have in order to be
      * corrected. Defaults to <tt>4</tt>.
@@ -292,6 +332,10 @@ public DirectCandidateGeneratorBuilder minWordLength(int minWordLength) {
         return this;
     }
 
+    Integer minWordLength() {
+        return minWordLength;
+    }
+
     /**
      * Sets a minimal threshold in number of documents a suggested term
      * should appear in. This can be specified as an absolute number or as a
@@ -305,6 +349,10 @@ public DirectCandidateGeneratorBuilder minDocFreq(float minDocFreq) {
         return this;
     }
 
+    Float minDocFreq() {
+        return minDocFreq;
+    }
+
     /**
      * Sets a filter (analyzer) that is applied to each of the tokens passed to this candidate generator.
      * This filter is applied to the original token before candidates are generated.
@@ -314,6 +362,10 @@ public DirectCandidateGeneratorBuilder preFilter(String preFilter) {
         return this;
     }
 
+    String preFilter() {
+        return preFilter;
+    }
+
     /**
      * Sets a filter (analyzer) that is applied to each of the generated tokens
      * before they are passed to the actual phrase scorer.
@@ -323,6 +375,10 @@ public DirectCandidateGeneratorBuilder postFilter(String postFilter) {
         return this;
     }
 
+    String postFilter() {
+        return postFilter;
+    }
+
     /**
      * gets the type identifier of this {@link CandidateGenerator}
      */
@@ -358,8 +414,7 @@ private static <T> void outputFieldIfNotNull(T value, ParseField field, XContent
         }
     }
 
-    private static ObjectParser<Tuple<Set<String>, DirectCandidateGeneratorBuilder>, QueryParseContext> PARSER = new ObjectParser<>(TYPE);
-
+    private static final ObjectParser<Tuple<Set<String>, DirectCandidateGeneratorBuilder>, Void> PARSER = new ObjectParser<>(TYPE);
     static {
         PARSER.declareString((tp, s) -> tp.v1().add(s), FIELDNAME_FIELD);
         PARSER.declareString((tp, s) -> tp.v2().preFilter(s), PREFILTER_FIELD);
@@ -377,12 +432,11 @@ private static <T> void outputFieldIfNotNull(T value, ParseField field, XContent
         PARSER.declareInt((tp, i) -> tp.v2().prefixLength(i), PREFIX_LENGTH_FIELD);
     }
 
-    public static DirectCandidateGeneratorBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+    public static DirectCandidateGeneratorBuilder fromXContent(XContentParser parser) throws IOException {
         DirectCandidateGeneratorBuilder tempGenerator = new DirectCandidateGeneratorBuilder("_na_");
         // bucket for the field name, needed as constructor arg later
         Set<String> tmpFieldName = new HashSet<>(1);
-        PARSER.parse(parseContext.parser(), new Tuple<>(tmpFieldName, tempGenerator),
-                parseContext);
+        PARSER.parse(parser, new Tuple<>(tmpFieldName, tempGenerator), null);
         if (tmpFieldName.size() != 1) {
             throw new IllegalArgumentException("[" + TYPE + "] expects exactly one field parameter, but found " + tmpFieldName);
         }
@@ -395,13 +449,13 @@ public PhraseSuggestionContext.DirectCandidateGenerator build(MapperService mapp
         generator.setField(this.field);
         transferIfNotNull(this.size, generator::size);
         if (this.preFilter != null) {
-            generator.preFilter(mapperService.analysisService().analyzer(this.preFilter));
+            generator.preFilter(mapperService.getIndexAnalyzers().get(this.preFilter));
             if (generator.preFilter() == null) {
                 throw new IllegalArgumentException("Analyzer [" + this.preFilter + "] doesn't exists");
             }
         }
         if (this.postFilter != null) {
-            generator.postFilter(mapperService.analysisService().analyzer(this.postFilter));
+            generator.postFilter(mapperService.getIndexAnalyzers().get(this.postFilter));
             if (generator.postFilter() == null) {
                 throw new IllegalArgumentException("Analyzer [" + this.postFilter + "] doesn't exists");
             }
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/Laplace.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/Laplace.java
index 6c781a8ee9159..222f69019164a 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/Laplace.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/Laplace.java
@@ -28,7 +28,6 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentParser.Token;
-import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.suggest.phrase.WordScorer.WordScorerFactory;
 
 import java.io.IOException;
@@ -103,8 +102,7 @@ protected int doHashCode() {
         return Objects.hash(alpha);
     }
 
-    public static SmoothingModel innerFromXContent(QueryParseContext parseContext) throws IOException {
-        XContentParser parser = parseContext.parser();
+    public static SmoothingModel fromXContent(XContentParser parser) throws IOException {
         XContentParser.Token token;
         String fieldName = null;
         double alpha = DEFAULT_LAPLACE_ALPHA;
@@ -112,7 +110,7 @@ public static SmoothingModel innerFromXContent(QueryParseContext parseContext) t
             if (token == XContentParser.Token.FIELD_NAME) {
                 fieldName = parser.currentName();
             }
-            if (token.isValue() && parseContext.getParseFieldMatcher().match(fieldName, ALPHA_FIELD)) {
+            if (token.isValue() && ALPHA_FIELD.match(fieldName)) {
                 alpha = parser.doubleValue();
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/LinearInterpolation.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/LinearInterpolation.java
index 72802e66eaacc..10a170d766dea 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/LinearInterpolation.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/LinearInterpolation.java
@@ -23,14 +23,12 @@
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentParser.Token;
-import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.suggest.phrase.WordScorer.WordScorerFactory;
 
 import java.io.IOException;
@@ -131,29 +129,27 @@ protected int doHashCode() {
         return Objects.hash(trigramLambda, bigramLambda, unigramLambda);
     }
 
-    public static LinearInterpolation innerFromXContent(QueryParseContext parseContext) throws IOException {
-        XContentParser parser = parseContext.parser();
+    public static LinearInterpolation fromXContent(XContentParser parser) throws IOException {
         XContentParser.Token token;
         String fieldName = null;
         double trigramLambda = 0.0;
         double bigramLambda = 0.0;
         double unigramLambda = 0.0;
-        ParseFieldMatcher matcher = parseContext.getParseFieldMatcher();
         while ((token = parser.nextToken()) != Token.END_OBJECT) {
             if (token == XContentParser.Token.FIELD_NAME) {
                 fieldName = parser.currentName();
             } else if (token.isValue()) {
-                if (matcher.match(fieldName, TRIGRAM_FIELD)) {
+                if (TRIGRAM_FIELD.match(fieldName)) {
                     trigramLambda = parser.doubleValue();
                     if (trigramLambda < 0) {
                         throw new IllegalArgumentException("trigram_lambda must be positive");
                     }
-                } else if (matcher.match(fieldName, BIGRAM_FIELD)) {
+                } else if (BIGRAM_FIELD.match(fieldName)) {
                     bigramLambda = parser.doubleValue();
                     if (bigramLambda < 0) {
                         throw new IllegalArgumentException("bigram_lambda must be positive");
                     }
-                } else if (matcher.match(fieldName, UNIGRAM_FIELD)) {
+                } else if (UNIGRAM_FIELD.match(fieldName)) {
                     unigramLambda = parser.doubleValue();
                     if (unigramLambda < 0) {
                         throw new IllegalArgumentException("unigram_lambda must be positive");
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java
index a9f5accb9180e..e9ef9257157db 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java
@@ -29,7 +29,6 @@
 import org.apache.lucene.util.BytesRefBuilder;
 import org.apache.lucene.util.CharsRefBuilder;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.text.Text;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -37,16 +36,12 @@
 import org.elasticsearch.index.query.MatchNoneQueryBuilder;
 import org.elasticsearch.index.query.ParsedQuery;
 import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.ExecutableScript;
-import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.suggest.Suggest.Suggestion;
 import org.elasticsearch.search.suggest.Suggest.Suggestion.Entry;
 import org.elasticsearch.search.suggest.Suggest.Suggestion.Entry.Option;
 import org.elasticsearch.search.suggest.Suggester;
-import org.elasticsearch.search.suggest.SuggestionBuilder;
 import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
 import org.elasticsearch.search.suggest.phrase.NoisyChannelSpellChecker.Result;
 
@@ -55,6 +50,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.function.Function;
 
 public final class PhraseSuggester extends Suggester<PhraseSuggestionContext> {
     private final BytesRef SEPARATOR = new BytesRef(" ");
@@ -109,7 +105,7 @@ public Suggestion<? extends Entry<? extends Option>> innerExecute(String name, P
             response.addTerm(resultEntry);
 
             final BytesRefBuilder byteSpare = new BytesRefBuilder();
-            final CompiledScript collateScript = suggestion.getCollateQueryScript();
+            final Function<Map<String, Object>, ExecutableScript> collateScript = suggestion.getCollateQueryScript();
             final boolean collatePrune = (collateScript != null) && suggestion.collatePrune();
             for (int i = 0; i < checkerResult.corrections.length; i++) {
                 Correction correction = checkerResult.corrections[i];
@@ -121,10 +117,10 @@ public Suggestion<? extends Entry<? extends Option>> innerExecute(String name, P
                     final Map<String, Object> vars = suggestion.getCollateScriptParams();
                     vars.put(SUGGESTION_TEMPLATE_VAR_NAME, spare.toString());
                     QueryShardContext shardContext = suggestion.getShardContext();
-                    ScriptService scriptService = shardContext.getScriptService();
-                    final ExecutableScript executable = scriptService.executable(collateScript, vars);
+                    final ExecutableScript executable = collateScript.apply(vars);
                     final BytesReference querySource = (BytesReference) executable.run();
-                    try (XContentParser parser = XContentFactory.xContent(querySource).createParser(querySource)) {
+                    try (XContentParser parser = XContentFactory.xContent(querySource).createParser(shardContext.getXContentRegistry(),
+                            querySource)) {
                         Optional<QueryBuilder> innerQueryBuilder = shardContext.newParseContext(parser).parseInnerQueryBuilder();
                         final ParsedQuery parsedQuery = shardContext.toQuery(innerQueryBuilder.orElse(new MatchNoneQueryBuilder()));
                         collateMatch = Lucene.exists(searcher, parsedQuery.query());
@@ -155,14 +151,4 @@ private static PhraseSuggestion.Entry buildResultEntry(SuggestionContext suggest
         spare.copyUTF8Bytes(suggestion.getText());
         return new PhraseSuggestion.Entry(new Text(spare.toString()), 0, spare.length(), cutoffScore);
     }
-
-    @Override
-    public SuggestionBuilder<?> innerFromXContent(QueryParseContext context) throws IOException {
-        return PhraseSuggestionBuilder.innerFromXContent(context);
-    }
-
-    @Override
-    public SuggestionBuilder<?> read(StreamInput in) throws IOException {
-        return new PhraseSuggestionBuilder(in);
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestion.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestion.java
index 23949f1a09688..bd6c828bd42bb 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestion.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestion.java
@@ -19,9 +19,12 @@
 
 package org.elasticsearch.search.suggest.phrase;
 
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.text.Text;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.search.suggest.Suggest;
 import org.elasticsearch.search.suggest.Suggest.Suggestion;
 
@@ -31,6 +34,8 @@
  * Suggestion entry returned from the {@link PhraseSuggester}.
  */
 public class PhraseSuggestion extends Suggest.Suggestion<PhraseSuggestion.Entry> {
+
+    public static final String NAME = "phrase";
     public static final int TYPE = 3;
 
     public PhraseSuggestion() {
@@ -41,19 +46,27 @@ public PhraseSuggestion(String name, int size) {
     }
 
     @Override
-    public int getType() {
+    public int getWriteableType() {
         return TYPE;
     }
 
+    @Override
+    protected String getType() {
+        return NAME;
+    }
+
     @Override
     protected Entry newEntry() {
         return new Entry();
     }
 
+    public static PhraseSuggestion fromXContent(XContentParser parser, String name) throws IOException {
+        PhraseSuggestion suggestion = new PhraseSuggestion(name, -1);
+        parseEntries(parser, suggestion, PhraseSuggestion.Entry::fromXContent);
+        return suggestion;
+    }
+
     public static class Entry extends Suggestion.Entry<Suggestion.Entry.Option> {
-        static class Fields {
-            static final String CUTOFF_SCORE = "cutoff_score";
-        }
 
         protected double cutoffScore = Double.MIN_VALUE;
 
@@ -62,7 +75,7 @@ public Entry(Text text, int offset, int length, double cutoffScore) {
             this.cutoffScore = cutoffScore;
         }
 
-        public Entry() {
+        Entry() {
         }
 
         /**
@@ -93,6 +106,17 @@ public void addOption(Suggestion.Entry.Option option) {
             }
         }
 
+        private static ObjectParser<Entry, Void> PARSER = new ObjectParser<>("PhraseSuggestionEntryParser", true, Entry::new);
+
+        static {
+            declareCommonFields(PARSER);
+            PARSER.declareObjectArray(Entry::addOptions, (p,c) -> Option.fromXContent(p), new ParseField(OPTIONS));
+        }
+
+        public static Entry fromXContent(XContentParser parser) {
+            return PARSER.apply(parser, null);
+        }
+
         @Override
         public void readFrom(StreamInput in) throws IOException {
             super.readFrom(in);
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java
index b0cd6a2049988..0ffa2d44fdc2d 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java
@@ -22,7 +22,6 @@
 import org.apache.lucene.analysis.Analyzer;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -37,12 +36,11 @@
 import org.elasticsearch.index.analysis.ShingleTokenFilterFactory;
 import org.elasticsearch.index.analysis.TokenFilterFactory;
 import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.script.CompiledScript;
+import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptContext;
-import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.suggest.SuggestionBuilder;
 import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
 import org.elasticsearch.search.suggest.phrase.PhraseSuggestionContext.DirectCandidateGenerator;
@@ -56,6 +54,7 @@
 import java.util.Map.Entry;
 import java.util.Objects;
 import java.util.Set;
+import java.util.function.Function;
 
 /**
  * Defines the actual suggest command for phrase suggestions ( <tt>phrase</tt>).
@@ -125,7 +124,7 @@ private PhraseSuggestionBuilder(String fieldname, PhraseSuggestionBuilder in) {
     /**
      * Read from a stream.
      */
-    PhraseSuggestionBuilder(StreamInput in) throws IOException {
+    public PhraseSuggestionBuilder(StreamInput in) throws IOException {
         super(in);
         maxErrors = in.readFloat();
         realWordErrorLikelihood = in.readFloat();
@@ -392,7 +391,7 @@ public String postTag() {
      * Sets a query used for filtering out suggested phrases (collation).
      */
     public PhraseSuggestionBuilder collateQuery(String collateQuery) {
-        this.collateQuery = new Script(collateQuery, ScriptService.ScriptType.INLINE, "mustache", Collections.emptyMap());
+        this.collateQuery = new Script(ScriptType.INLINE, "mustache", collateQuery, Collections.emptyMap());
         return this;
     }
 
@@ -487,10 +486,8 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t
         return builder;
     }
 
-    static PhraseSuggestionBuilder innerFromXContent(QueryParseContext parseContext) throws IOException {
-        XContentParser parser = parseContext.parser();
+    public static PhraseSuggestionBuilder fromXContent(XContentParser parser) throws IOException {
         PhraseSuggestionBuilder tmpSuggestion = new PhraseSuggestionBuilder("_na_");
-        ParseFieldMatcher parseFieldMatcher = parseContext.getParseFieldMatcher();
         XContentParser.Token token;
         String currentFieldName = null;
         String fieldname = null;
@@ -498,56 +495,56 @@ static PhraseSuggestionBuilder innerFromXContent(QueryParseContext parseContext)
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token.isValue()) {
-                if (parseFieldMatcher.match(currentFieldName, SuggestionBuilder.ANALYZER_FIELD)) {
+                if (SuggestionBuilder.ANALYZER_FIELD.match(currentFieldName)) {
                     tmpSuggestion.analyzer(parser.text());
-                } else if (parseFieldMatcher.match(currentFieldName, SuggestionBuilder.FIELDNAME_FIELD)) {
+                } else if (SuggestionBuilder.FIELDNAME_FIELD.match(currentFieldName)) {
                     fieldname = parser.text();
-                } else if (parseFieldMatcher.match(currentFieldName, SuggestionBuilder.SIZE_FIELD)) {
+                } else if (SuggestionBuilder.SIZE_FIELD.match(currentFieldName)) {
                     tmpSuggestion.size(parser.intValue());
-                } else if (parseFieldMatcher.match(currentFieldName, SuggestionBuilder.SHARDSIZE_FIELD)) {
+                } else if (SuggestionBuilder.SHARDSIZE_FIELD.match(currentFieldName)) {
                     tmpSuggestion.shardSize(parser.intValue());
-                } else if (parseFieldMatcher.match(currentFieldName, PhraseSuggestionBuilder.RWE_LIKELIHOOD_FIELD)) {
+                } else if (PhraseSuggestionBuilder.RWE_LIKELIHOOD_FIELD.match(currentFieldName)) {
                     tmpSuggestion.realWordErrorLikelihood(parser.floatValue());
-                } else if (parseFieldMatcher.match(currentFieldName, PhraseSuggestionBuilder.CONFIDENCE_FIELD)) {
+                } else if (PhraseSuggestionBuilder.CONFIDENCE_FIELD.match(currentFieldName)) {
                     tmpSuggestion.confidence(parser.floatValue());
-                } else if (parseFieldMatcher.match(currentFieldName, PhraseSuggestionBuilder.SEPARATOR_FIELD)) {
+                } else if (PhraseSuggestionBuilder.SEPARATOR_FIELD.match(currentFieldName)) {
                     tmpSuggestion.separator(parser.text());
-                } else if (parseFieldMatcher.match(currentFieldName, PhraseSuggestionBuilder.MAXERRORS_FIELD)) {
+                } else if (PhraseSuggestionBuilder.MAXERRORS_FIELD.match(currentFieldName)) {
                     tmpSuggestion.maxErrors(parser.floatValue());
-                } else if (parseFieldMatcher.match(currentFieldName, PhraseSuggestionBuilder.GRAMSIZE_FIELD)) {
+                } else if (PhraseSuggestionBuilder.GRAMSIZE_FIELD.match(currentFieldName)) {
                     tmpSuggestion.gramSize(parser.intValue());
-                } else if (parseFieldMatcher.match(currentFieldName, PhraseSuggestionBuilder.FORCE_UNIGRAM_FIELD)) {
+                } else if (PhraseSuggestionBuilder.FORCE_UNIGRAM_FIELD.match(currentFieldName)) {
                     tmpSuggestion.forceUnigrams(parser.booleanValue());
-                } else if (parseFieldMatcher.match(currentFieldName, PhraseSuggestionBuilder.TOKEN_LIMIT_FIELD)) {
+                } else if (PhraseSuggestionBuilder.TOKEN_LIMIT_FIELD.match(currentFieldName)) {
                     tmpSuggestion.tokenLimit(parser.intValue());
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
                             "suggester[phrase] doesn't support field [" + currentFieldName + "]");
                 }
             } else if (token == Token.START_ARRAY) {
-                if (parseFieldMatcher.match(currentFieldName, DirectCandidateGeneratorBuilder.DIRECT_GENERATOR_FIELD)) {
+                if (DirectCandidateGeneratorBuilder.DIRECT_GENERATOR_FIELD.match(currentFieldName)) {
                     // for now we only have a single type of generators
                     while ((token = parser.nextToken()) == Token.START_OBJECT) {
-                        tmpSuggestion.addCandidateGenerator(DirectCandidateGeneratorBuilder.fromXContent(parseContext));
+                        tmpSuggestion.addCandidateGenerator(DirectCandidateGeneratorBuilder.fromXContent(parser));
                     }
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
                             "suggester[phrase]  doesn't support array field [" + currentFieldName + "]");
                 }
             } else if (token == Token.START_OBJECT) {
-                if (parseFieldMatcher.match(currentFieldName, PhraseSuggestionBuilder.SMOOTHING_MODEL_FIELD)) {
+                if (PhraseSuggestionBuilder.SMOOTHING_MODEL_FIELD.match(currentFieldName)) {
                     ensureNoSmoothing(tmpSuggestion);
-                    tmpSuggestion.smoothingModel(SmoothingModel.fromXContent(parseContext));
-                } else if (parseFieldMatcher.match(currentFieldName, PhraseSuggestionBuilder.HIGHLIGHT_FIELD)) {
+                    tmpSuggestion.smoothingModel(SmoothingModel.fromXContent(parser));
+                } else if (PhraseSuggestionBuilder.HIGHLIGHT_FIELD.match(currentFieldName)) {
                     String preTag = null;
                     String postTag = null;
                     while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
                         if (token == XContentParser.Token.FIELD_NAME) {
                             currentFieldName = parser.currentName();
                         } else if (token.isValue()) {
-                            if (parseFieldMatcher.match(currentFieldName, PhraseSuggestionBuilder.PRE_TAG_FIELD)) {
+                            if (PhraseSuggestionBuilder.PRE_TAG_FIELD.match(currentFieldName)) {
                                 preTag = parser.text();
-                            } else if (parseFieldMatcher.match(currentFieldName, PhraseSuggestionBuilder.POST_TAG_FIELD)) {
+                            } else if (PhraseSuggestionBuilder.POST_TAG_FIELD.match(currentFieldName)) {
                                 postTag = parser.text();
                             } else {
                                 throw new ParsingException(parser.getTokenLocation(),
@@ -556,21 +553,21 @@ static PhraseSuggestionBuilder innerFromXContent(QueryParseContext parseContext)
                         }
                     }
                     tmpSuggestion.highlight(preTag, postTag);
-                } else if (parseFieldMatcher.match(currentFieldName, PhraseSuggestionBuilder.COLLATE_FIELD)) {
+                } else if (PhraseSuggestionBuilder.COLLATE_FIELD.match(currentFieldName)) {
                     while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
                         if (token == XContentParser.Token.FIELD_NAME) {
                             currentFieldName = parser.currentName();
-                        } else if (parseFieldMatcher.match(currentFieldName, PhraseSuggestionBuilder.COLLATE_QUERY_FIELD)) {
+                        } else if (PhraseSuggestionBuilder.COLLATE_QUERY_FIELD.match(currentFieldName)) {
                             if (tmpSuggestion.collateQuery() != null) {
                                 throw new ParsingException(parser.getTokenLocation(),
                                         "suggester[phrase][collate] query already set, doesn't support additional ["
                                         + currentFieldName + "]");
                             }
-                            Script template = Script.parse(parser, parseFieldMatcher, "mustache");
+                            Script template = Script.parse(parser, "mustache");
                             tmpSuggestion.collateQuery(template);
-                        } else if (parseFieldMatcher.match(currentFieldName, PhraseSuggestionBuilder.COLLATE_QUERY_PARAMS)) {
+                        } else if (PhraseSuggestionBuilder.COLLATE_QUERY_PARAMS.match(currentFieldName)) {
                             tmpSuggestion.collateParams(parser.map());
-                        } else if (parseFieldMatcher.match(currentFieldName, PhraseSuggestionBuilder.COLLATE_QUERY_PRUNE)) {
+                        } else if (PhraseSuggestionBuilder.COLLATE_QUERY_PRUNE.match(currentFieldName)) {
                             if (parser.isBooleanValue()) {
                                 tmpSuggestion.collatePrune(parser.booleanValue());
                             } else {
@@ -633,8 +630,8 @@ public SuggestionContext build(QueryShardContext context) throws IOException {
         }
 
         if (this.collateQuery != null) {
-            CompiledScript compiledScript = context.getScriptService().compile(this.collateQuery, ScriptContext.Standard.SEARCH,
-                    Collections.emptyMap());
+            Function<Map<String, Object>, ExecutableScript> compiledScript = context.getLazyExecutableScript(this.collateQuery,
+                ScriptContext.Standard.SEARCH);
             suggestionContext.setCollateQueryScript(compiledScript);
             if (this.collateParams != null) {
                 suggestionContext.setCollateScriptParams(this.collateParams);
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionContext.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionContext.java
index e2d7ff1c41d24..99e2e18496b61 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionContext.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionContext.java
@@ -24,6 +24,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.script.CompiledScript;
+import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.search.suggest.DirectSpellcheckerSettings;
 import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
 
@@ -31,6 +32,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Function;
 
 class PhraseSuggestionContext extends SuggestionContext {
     static final boolean DEFAULT_COLLATE_PRUNE = false;
@@ -52,13 +54,13 @@ class PhraseSuggestionContext extends SuggestionContext {
     private boolean requireUnigram = DEFAULT_REQUIRE_UNIGRAM;
     private BytesRef preTag;
     private BytesRef postTag;
-    private CompiledScript collateQueryScript;
+    private Function<Map<String, Object>, ExecutableScript>  collateQueryScript;
     private boolean prune = DEFAULT_COLLATE_PRUNE;
     private List<DirectCandidateGenerator> generators = new ArrayList<>();
     private Map<String, Object> collateScriptParams = new HashMap<>(1);
     private WordScorer.WordScorerFactory scorer = DEFAULT_SCORER;
 
-    public PhraseSuggestionContext(QueryShardContext shardContext) {
+    PhraseSuggestionContext(QueryShardContext shardContext) {
         super(PhraseSuggester.INSTANCE, shardContext);
     }
 
@@ -192,11 +194,11 @@ public BytesRef getPostTag() {
         return postTag;
     }
 
-    CompiledScript getCollateQueryScript() {
+    Function<Map<String, Object>, ExecutableScript> getCollateQueryScript() {
         return collateQueryScript;
     }
 
-    void setCollateQueryScript(CompiledScript collateQueryScript) {
+    void setCollateQueryScript( Function<Map<String, Object>, ExecutableScript>  collateQueryScript) {
         this.collateQueryScript = collateQueryScript;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/SmoothingModel.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/SmoothingModel.java
index 0f55c9ed8fe5d..6fd7d6ea5397a 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/SmoothingModel.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/SmoothingModel.java
@@ -19,13 +19,11 @@
 
 package org.elasticsearch.search.suggest.phrase;
 
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.suggest.phrase.WordScorer.WordScorerFactory;
 
 import java.io.IOException;
@@ -65,9 +63,7 @@ public final int hashCode() {
 
     protected abstract int doHashCode();
 
-    public static SmoothingModel fromXContent(QueryParseContext parseContext) throws IOException {
-        XContentParser parser = parseContext.parser();
-        ParseFieldMatcher parseFieldMatcher = parseContext.getParseFieldMatcher();
+    public static SmoothingModel fromXContent(XContentParser parser) throws IOException {
         XContentParser.Token token;
         String fieldName = null;
         SmoothingModel model = null;
@@ -75,12 +71,12 @@ public static SmoothingModel fromXContent(QueryParseContext parseContext) throws
             if (token == XContentParser.Token.FIELD_NAME) {
                 fieldName = parser.currentName();
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (parseFieldMatcher.match(fieldName, LinearInterpolation.PARSE_FIELD)) {
-                    model = LinearInterpolation.innerFromXContent(parseContext);
-                } else if (parseFieldMatcher.match(fieldName, Laplace.PARSE_FIELD)) {
-                    model = Laplace.innerFromXContent(parseContext);
-                } else if (parseFieldMatcher.match(fieldName, StupidBackoff.PARSE_FIELD)) {
-                    model = StupidBackoff.innerFromXContent(parseContext);
+                if (LinearInterpolation.PARSE_FIELD.match(fieldName)) {
+                    model = LinearInterpolation.fromXContent(parser);
+                } else if (Laplace.PARSE_FIELD.match(fieldName)) {
+                    model = Laplace.fromXContent(parser);
+                } else if (StupidBackoff.PARSE_FIELD.match(fieldName)) {
+                    model = StupidBackoff.fromXContent(parser);
                 } else {
                     throw new IllegalArgumentException("suggester[phrase] doesn't support object field [" + fieldName + "]");
                 }
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/StupidBackoff.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/StupidBackoff.java
index 3e67110ce7dc8..ebdd1c9ec2892 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/StupidBackoff.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/StupidBackoff.java
@@ -28,7 +28,6 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentParser.Token;
-import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.suggest.phrase.WordScorer.WordScorerFactory;
 
 import java.io.IOException;
@@ -106,8 +105,7 @@ protected int doHashCode() {
         return Objects.hash(discount);
     }
 
-    public static SmoothingModel innerFromXContent(QueryParseContext parseContext) throws IOException {
-        XContentParser parser = parseContext.parser();
+    public static SmoothingModel fromXContent(XContentParser parser) throws IOException {
         XContentParser.Token token;
         String fieldName = null;
         double discount = DEFAULT_BACKOFF_DISCOUNT;
@@ -115,7 +113,7 @@ public static SmoothingModel innerFromXContent(QueryParseContext parseContext) t
             if (token == XContentParser.Token.FIELD_NAME) {
                 fieldName = parser.currentName();
             }
-            if (token.isValue() && parseContext.getParseFieldMatcher().match(fieldName, DISCOUNT_FIELD)) {
+            if (token.isValue() && DISCOUNT_FIELD.match(fieldName)) {
                 discount = parser.doubleValue();
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/StupidBackoffScorer.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/StupidBackoffScorer.java
index 8eb08ef06889f..166a4182c8f81 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/StupidBackoffScorer.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/StupidBackoffScorer.java
@@ -28,7 +28,7 @@
 class StupidBackoffScorer extends WordScorer {
     private final double discount;
 
-    public StupidBackoffScorer(IndexReader reader, Terms terms,String field, double realWordLikelyhood, BytesRef separator, double discount)
+    StupidBackoffScorer(IndexReader reader, Terms terms,String field, double realWordLikelyhood, BytesRef separator, double discount)
             throws IOException {
         super(reader, terms, field, realWordLikelyhood, separator);
         this.discount = discount;
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggester.java b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggester.java
index 0d58e0f5cac03..d6edf352d1911 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggester.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggester.java
@@ -27,11 +27,8 @@
 import org.apache.lucene.util.BytesRefBuilder;
 import org.apache.lucene.util.CharsRefBuilder;
 import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.text.Text;
-import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.suggest.Suggester;
-import org.elasticsearch.search.suggest.SuggestionBuilder;
 import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
 import org.elasticsearch.search.suggest.phrase.DirectCandidateGenerator;
 
@@ -84,16 +81,6 @@ public void nextToken() {
        return result;
     }
 
-    @Override
-    public SuggestionBuilder<?> innerFromXContent(QueryParseContext context) throws IOException {
-        return TermSuggestionBuilder.innerFromXContent(context);
-    }
-
-    @Override
-    public SuggestionBuilder<?> read(StreamInput in) throws IOException {
-        return new TermSuggestionBuilder(in);
-    }
-
     private static class Token {
 
         public final Term term;
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestion.java b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestion.java
index 7673267ecf09c..185e4c76b3c62 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestion.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestion.java
@@ -19,10 +19,14 @@
 package org.elasticsearch.search.suggest.term;
 
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.text.Text;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.search.suggest.SortBy;
 import org.elasticsearch.search.suggest.Suggest.Suggestion;
 import org.elasticsearch.search.suggest.Suggest.Suggestion.Entry.Option;
@@ -30,11 +34,15 @@
 import java.io.IOException;
 import java.util.Comparator;
 
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+
 /**
  * The suggestion responses corresponding with the suggestions in the request.
  */
 public class TermSuggestion extends Suggestion<TermSuggestion.Entry> {
 
+    public static final String NAME = "term";
+
     public static final Comparator<Suggestion.Entry.Option> SCORE = new Score();
     public static final Comparator<Suggestion.Entry.Option> FREQUENCY = new Frequency();
     public static final int TYPE = 1;
@@ -91,10 +99,15 @@ public int compare(Suggestion.Entry.Option first, Suggestion.Entry.Option second
     }
 
     @Override
-    public int getType() {
+    public int getWriteableType() {
         return TYPE;
     }
 
+    @Override
+    protected String getType() {
+        return NAME;
+    }
+
     @Override
     protected Comparator<Option> sortComparator() {
         switch (sort) {
@@ -119,6 +132,13 @@ public void innerWriteTo(StreamOutput out) throws IOException {
         sort.writeTo(out);
     }
 
+    public static TermSuggestion fromXContent(XContentParser parser, String name) throws IOException {
+        // the "size" parameter and the SortBy for TermSuggestion cannot be parsed from the response, use default values
+        TermSuggestion suggestion = new TermSuggestion(name, -1, SortBy.SCORE);
+        parseEntries(parser, suggestion, TermSuggestion.Entry::fromXContent);
+        return suggestion;
+    }
+
     @Override
     protected Entry newEntry() {
         return new Entry();
@@ -127,10 +147,9 @@ protected Entry newEntry() {
     /**
      * Represents a part from the suggest text with suggested options.
      */
-    public static class Entry extends
-            org.elasticsearch.search.suggest.Suggest.Suggestion.Entry<TermSuggestion.Entry.Option> {
+    public static class Entry extends org.elasticsearch.search.suggest.Suggest.Suggestion.Entry<TermSuggestion.Entry.Option> {
 
-        Entry(Text text, int offset, int length) {
+        public Entry(Text text, int offset, int length) {
             super(text, offset, length);
         }
 
@@ -142,18 +161,27 @@ protected Option newOption() {
             return new Option();
         }
 
+        private static ObjectParser<Entry, Void> PARSER = new ObjectParser<>("TermSuggestionEntryParser", true, Entry::new);
+
+        static {
+            declareCommonFields(PARSER);
+            PARSER.declareObjectArray(Entry::addOptions, (p,c) -> Option.fromXContent(p), new ParseField(OPTIONS));
+        }
+
+        public static Entry fromXContent(XContentParser parser) {
+            return PARSER.apply(parser, null);
+        }
+
         /**
          * Contains the suggested text with its document frequency and score.
          */
         public static class Option extends org.elasticsearch.search.suggest.Suggest.Suggestion.Entry.Option {
 
-            static class Fields {
-                static final String FREQ = "freq";
-            }
+            public static final ParseField FREQ = new ParseField("freq");
 
             private int freq;
 
-            protected Option(Text text, int freq, float score) {
+            public Option(Text text, int freq, float score) {
                 super(text, score);
                 this.freq = freq;
             }
@@ -194,9 +222,28 @@ public void writeTo(StreamOutput out) throws IOException {
             @Override
             protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
                 builder = super.innerToXContent(builder, params);
-                builder.field(Fields.FREQ, freq);
+                builder.field(FREQ.getPreferredName(), freq);
                 return builder;
             }
+
+            private static final ConstructingObjectParser<Option, Void> PARSER = new ConstructingObjectParser<>(
+                    "TermSuggestionOptionParser", true,
+                    args -> {
+                        Text text = new Text((String) args[0]);
+                        int freq = (Integer) args[1];
+                        float score = (Float) args[2];
+                        return new Option(text, freq, score);
+                    });
+
+            static {
+                PARSER.declareString(constructorArg(), Suggestion.Entry.Option.TEXT);
+                PARSER.declareInt(constructorArg(), FREQ);
+                PARSER.declareFloat(constructorArg(), Suggestion.Entry.Option.SCORE);
+            }
+
+            public static Option fromXContent(XContentParser parser) {
+                return PARSER.apply(parser, null);
+            }
         }
 
     }
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilder.java
index 31e6c3718e885..f701ff3642624 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilder.java
@@ -26,14 +26,12 @@
 import org.apache.lucene.search.spell.NGramDistance;
 import org.apache.lucene.search.spell.StringDistance;
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.suggest.DirectSpellcheckerSettings;
 import org.elasticsearch.search.suggest.SortBy;
@@ -105,7 +103,7 @@ private TermSuggestionBuilder(String field, TermSuggestionBuilder in) {
     /**
      * Read from a stream.
      */
-    TermSuggestionBuilder(StreamInput in) throws IOException {
+    public TermSuggestionBuilder(StreamInput in) throws IOException {
         super(in);
         suggestMode = SuggestMode.readFromStream(in);
         accuracy = in.readFloat();
@@ -388,10 +386,8 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t
         return builder;
     }
 
-    static TermSuggestionBuilder innerFromXContent(QueryParseContext parseContext) throws IOException {
-        XContentParser parser = parseContext.parser();
+    public static TermSuggestionBuilder fromXContent(XContentParser parser) throws IOException {
         TermSuggestionBuilder tmpSuggestion = new TermSuggestionBuilder("_na_");
-        ParseFieldMatcher parseFieldMatcher = parseContext.getParseFieldMatcher();
         XContentParser.Token token;
         String currentFieldName = null;
         String fieldname = null;
@@ -399,33 +395,33 @@ static TermSuggestionBuilder innerFromXContent(QueryParseContext parseContext) t
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token.isValue()) {
-                if (parseFieldMatcher.match(currentFieldName, SuggestionBuilder.ANALYZER_FIELD)) {
+                if (SuggestionBuilder.ANALYZER_FIELD.match(currentFieldName)) {
                     tmpSuggestion.analyzer(parser.text());
-                } else if (parseFieldMatcher.match(currentFieldName, SuggestionBuilder.FIELDNAME_FIELD)) {
+                } else if (SuggestionBuilder.FIELDNAME_FIELD.match(currentFieldName)) {
                     fieldname = parser.text();
-                } else if (parseFieldMatcher.match(currentFieldName, SuggestionBuilder.SIZE_FIELD)) {
+                } else if (SuggestionBuilder.SIZE_FIELD.match(currentFieldName)) {
                     tmpSuggestion.size(parser.intValue());
-                } else if (parseFieldMatcher.match(currentFieldName, SuggestionBuilder.SHARDSIZE_FIELD)) {
+                } else if (SuggestionBuilder.SHARDSIZE_FIELD.match(currentFieldName)) {
                     tmpSuggestion.shardSize(parser.intValue());
-                } else if (parseFieldMatcher.match(currentFieldName, SUGGESTMODE_FIELD)) {
+                } else if (SUGGESTMODE_FIELD.match(currentFieldName)) {
                     tmpSuggestion.suggestMode(SuggestMode.resolve(parser.text()));
-                } else if (parseFieldMatcher.match(currentFieldName, ACCURACY_FIELD)) {
+                } else if (ACCURACY_FIELD.match(currentFieldName)) {
                     tmpSuggestion.accuracy(parser.floatValue());
-                } else if (parseFieldMatcher.match(currentFieldName, SORT_FIELD)) {
+                } else if (SORT_FIELD.match(currentFieldName)) {
                     tmpSuggestion.sort(SortBy.resolve(parser.text()));
-                } else if (parseFieldMatcher.match(currentFieldName, STRING_DISTANCE_FIELD)) {
+                } else if (STRING_DISTANCE_FIELD.match(currentFieldName)) {
                     tmpSuggestion.stringDistance(StringDistanceImpl.resolve(parser.text()));
-                } else if (parseFieldMatcher.match(currentFieldName, MAX_EDITS_FIELD)) {
+                } else if (MAX_EDITS_FIELD.match(currentFieldName)) {
                     tmpSuggestion.maxEdits(parser.intValue());
-                } else if (parseFieldMatcher.match(currentFieldName, MAX_INSPECTIONS_FIELD)) {
+                } else if (MAX_INSPECTIONS_FIELD.match(currentFieldName)) {
                     tmpSuggestion.maxInspections(parser.intValue());
-                } else if (parseFieldMatcher.match(currentFieldName, MAX_TERM_FREQ_FIELD)) {
+                } else if (MAX_TERM_FREQ_FIELD.match(currentFieldName)) {
                     tmpSuggestion.maxTermFreq(parser.floatValue());
-                } else if (parseFieldMatcher.match(currentFieldName, PREFIX_LENGTH_FIELD)) {
+                } else if (PREFIX_LENGTH_FIELD.match(currentFieldName)) {
                     tmpSuggestion.prefixLength(parser.intValue());
-                } else if (parseFieldMatcher.match(currentFieldName, MIN_WORD_LENGTH_FIELD)) {
+                } else if (MIN_WORD_LENGTH_FIELD.match(currentFieldName)) {
                     tmpSuggestion.minWordLength(parser.intValue());
-                } else if (parseFieldMatcher.match(currentFieldName, MIN_DOC_FREQ_FIELD)) {
+                } else if (MIN_DOC_FREQ_FIELD.match(currentFieldName)) {
                     tmpSuggestion.minDocFreq(parser.floatValue());
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
@@ -515,15 +511,11 @@ public org.apache.lucene.search.spell.SuggestMode toLucene() {
 
         @Override
         public void writeTo(final StreamOutput out) throws IOException {
-            out.writeVInt(ordinal());
+            out.writeEnum(this);
         }
 
         public static SuggestMode readFromStream(final StreamInput in) throws IOException {
-            int ordinal = in.readVInt();
-            if (ordinal < 0 || ordinal >= values().length) {
-                throw new IOException("Unknown SuggestMode ordinal [" + ordinal + "]");
-            }
-            return values()[ordinal];
+            return in.readEnum(SuggestMode.class);
         }
 
         public static SuggestMode resolve(final String str) {
@@ -575,15 +567,11 @@ public StringDistance toLucene() {
 
         @Override
         public void writeTo(final StreamOutput out) throws IOException {
-            out.writeVInt(ordinal());
+            out.writeEnum(this);
         }
 
         public static StringDistanceImpl readFromStream(final StreamInput in) throws IOException {
-            int ordinal = in.readVInt();
-            if (ordinal < 0 || ordinal >= values().length) {
-                throw new IOException("Unknown StringDistanceImpl ordinal [" + ordinal + "]");
-            }
-            return values()[ordinal];
+            return in.readEnum(StringDistanceImpl.class);
         }
 
         public static StringDistanceImpl resolve(final String str) {
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionContext.java b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionContext.java
index c950e5729da14..19ad03e1b4716 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionContext.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionContext.java
@@ -26,7 +26,7 @@ final class TermSuggestionContext extends SuggestionContext {
 
     private final DirectSpellcheckerSettings settings = new DirectSpellcheckerSettings();
 
-    public TermSuggestionContext(QueryShardContext shardContext) {
+    TermSuggestionContext(QueryShardContext shardContext) {
         super(TermSuggester.INSTANCE, shardContext);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java
index 180f8f0af11b8..275a65a792dcd 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java
@@ -22,6 +22,7 @@
 import com.carrotsearch.hppc.IntSet;
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
+import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
 import org.elasticsearch.Version;
@@ -29,10 +30,14 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.ClusterStateListener;
+import org.elasticsearch.cluster.ClusterStateApplier;
+import org.elasticsearch.cluster.ClusterStateTaskConfig;
+import org.elasticsearch.cluster.ClusterStateTaskExecutor;
+import org.elasticsearch.cluster.ClusterStateTaskListener;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.RestoreInProgress;
 import org.elasticsearch.cluster.RestoreInProgress.ShardRestoreStatus;
+import org.elasticsearch.cluster.SnapshotDeletionsInProgress;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.metadata.AliasMetaData;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
@@ -41,27 +46,29 @@
 import org.elasticsearch.cluster.metadata.MetaDataCreateIndexService;
 import org.elasticsearch.cluster.metadata.MetaDataIndexUpgradeService;
 import org.elasticsearch.cluster.metadata.RepositoriesMetaData;
-import org.elasticsearch.cluster.routing.IndexRoutingTable;
+import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
+import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.RecoverySource.SnapshotRecoverySource;
+import org.elasticsearch.cluster.routing.RoutingChangesObserver;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
-import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
-import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
+import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
@@ -84,12 +91,10 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
-import java.util.Map.Entry;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.stream.Collectors;
 
 import static java.util.Collections.unmodifiableSet;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS;
@@ -98,7 +103,6 @@
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_VERSION_CREATED;
-import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_VERSION_MINIMUM_COMPATIBLE;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_VERSION_UPGRADED;
 import static org.elasticsearch.common.util.set.Sets.newHashSet;
 
@@ -118,11 +122,13 @@
  * method, which detects that shard should be restored from snapshot rather than recovered from gateway by looking
  * at the {@link ShardRouting#recoverySource()} property.
  * <p>
- * At the end of the successful restore process {@code IndexShardSnapshotAndRestoreService} calls {@link #indexShardRestoreCompleted(Snapshot, ShardId)},
- * which updates {@link RestoreInProgress} in cluster state or removes it when all shards are completed. In case of
+ * At the end of the successful restore process {@code RestoreService} calls {@link #cleanupRestoreState(ClusterChangedEvent)},
+ * which removes {@link RestoreInProgress} when all shards are completed. In case of
  * restore failure a normal recovery fail-over process kicks in.
  */
-public class RestoreService extends AbstractComponent implements ClusterStateListener {
+public class RestoreService extends AbstractComponent implements ClusterStateApplier {
+
+    public static Version V_5_1_0_UNRELEASED = Version.fromId(5010099);
 
     public static final String UPDATE_RESTORE_ACTION_NAME = "internal:cluster/snapshot/update_restore";
 
@@ -141,7 +147,6 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
         unremovable.add(SETTING_NUMBER_OF_REPLICAS);
         unremovable.add(SETTING_AUTO_EXPAND_REPLICAS);
         unremovable.add(SETTING_VERSION_UPGRADED);
-        unremovable.add(SETTING_VERSION_MINIMUM_COMPATIBLE);
         UNREMOVABLE_SETTINGS = unmodifiableSet(unremovable);
     }
 
@@ -157,11 +162,10 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
 
     private final MetaDataIndexUpgradeService metaDataIndexUpgradeService;
 
-    private final CopyOnWriteArrayList<ActionListener<RestoreCompletionResponse>> listeners = new CopyOnWriteArrayList<>();
-
-    private final BlockingQueue<UpdateIndexShardRestoreStatusRequest> updatedSnapshotStateQueue = ConcurrentCollections.newBlockingQueue();
     private final ClusterSettings clusterSettings;
 
+    private final CleanRestoreStateTaskExecutor cleanRestoreStateTaskExecutor;
+
     @Inject
     public RestoreService(Settings settings, ClusterService clusterService, RepositoriesService repositoriesService, TransportService transportService,
                           AllocationService allocationService, MetaDataCreateIndexService createIndexService,
@@ -174,8 +178,9 @@ public RestoreService(Settings settings, ClusterService clusterService, Reposito
         this.createIndexService = createIndexService;
         this.metaDataIndexUpgradeService = metaDataIndexUpgradeService;
         transportService.registerRequestHandler(UPDATE_RESTORE_ACTION_NAME, UpdateIndexShardRestoreStatusRequest::new, ThreadPool.Names.SAME, new UpdateRestoreStateRequestHandler());
-        clusterService.add(this);
+        clusterService.addStateApplier(this);
         this.clusterSettings = clusterSettings;
+        this.cleanRestoreStateTaskExecutor = new CleanRestoreStateTaskExecutor(logger);
     }
 
     /**
@@ -184,11 +189,16 @@ public RestoreService(Settings settings, ClusterService clusterService, Reposito
      * @param request  restore request
      * @param listener restore listener
      */
-    public void restoreSnapshot(final RestoreRequest request, final ActionListener<RestoreInfo> listener) {
+    public void restoreSnapshot(final RestoreRequest request, final ActionListener<RestoreCompletionResponse> listener) {
         try {
             // Read snapshot info and metadata from the repository
             Repository repository = repositoriesService.repository(request.repositoryName);
             final RepositoryData repositoryData = repository.getRepositoryData();
+            final Optional<SnapshotId> incompatibleSnapshotId =
+                repositoryData.getIncompatibleSnapshotIds().stream().filter(s -> request.snapshotName.equals(s.getName())).findFirst();
+            if (incompatibleSnapshotId.isPresent()) {
+                throw new SnapshotRestoreException(request.repositoryName, request.snapshotName, "cannot restore incompatible snapshot");
+            }
             final Optional<SnapshotId> matchingSnapshotId = repositoryData.getSnapshotIds().stream()
                 .filter(s -> request.snapshotName.equals(s.getName())).findFirst();
             if (matchingSnapshotId.isPresent() == false) {
@@ -228,6 +238,13 @@ public ClusterState execute(ClusterState currentState) {
                     if (restoreInProgress != null && !restoreInProgress.entries().isEmpty()) {
                         throw new ConcurrentSnapshotExecutionException(snapshot, "Restore process is already running in this cluster");
                     }
+                    // Check if the snapshot to restore is currently being deleted
+                    SnapshotDeletionsInProgress deletionsInProgress = currentState.custom(SnapshotDeletionsInProgress.TYPE);
+                    if (deletionsInProgress != null && deletionsInProgress.hasDeletionsInProgress()) {
+                        throw new ConcurrentSnapshotExecutionException(snapshot,
+                            "cannot restore a snapshot while a snapshot deletion is in-progress [" +
+                                deletionsInProgress.getEntries().get(0).getSnapshot() + "]");
+                    }
 
                     // Updating cluster state
                     ClusterState.Builder builder = ClusterState.builder(currentState);
@@ -239,6 +256,8 @@ public ClusterState execute(ClusterState currentState) {
                     if (!renamedIndices.isEmpty()) {
                         // We have some indices to restore
                         ImmutableOpenMap.Builder<ShardId, RestoreInProgress.ShardRestoreStatus> shardsBuilder = ImmutableOpenMap.builder();
+                        final Version minIndexCompatibilityVersion = currentState.getNodes().getMaxNodeVersion()
+                            .minimumIndexCompatibilityVersion();
                         for (Map.Entry<String, String> indexEntry : renamedIndices.entrySet()) {
                             String index = indexEntry.getValue();
                             boolean partial = checkPartial(index);
@@ -247,7 +266,8 @@ public ClusterState execute(ClusterState currentState) {
                             IndexMetaData snapshotIndexMetaData = metaData.index(index);
                             snapshotIndexMetaData = updateIndexSettings(snapshotIndexMetaData, request.indexSettings, request.ignoreIndexSettings);
                             try {
-                                snapshotIndexMetaData = metaDataIndexUpgradeService.upgradeIndexMetaData(snapshotIndexMetaData);
+                                snapshotIndexMetaData = metaDataIndexUpgradeService.upgradeIndexMetaData(snapshotIndexMetaData,
+                                    minIndexCompatibilityVersion);
                             } catch (Exception ex) {
                                 throw new SnapshotRestoreException(snapshot, "cannot restore index [" + index + "] because it cannot be upgraded", ex);
                             }
@@ -315,7 +335,7 @@ public ClusterState execute(ClusterState currentState) {
                         }
 
                         shards = shardsBuilder.build();
-                        RestoreInProgress.Entry restoreEntry = new RestoreInProgress.Entry(snapshot, RestoreInProgress.State.INIT, Collections.unmodifiableList(new ArrayList<>(renamedIndices.keySet())), shards);
+                        RestoreInProgress.Entry restoreEntry = new RestoreInProgress.Entry(snapshot, overallState(RestoreInProgress.State.INIT, shards), Collections.unmodifiableList(new ArrayList<>(renamedIndices.keySet())), shards);
                         builder.putCustom(RestoreInProgress.TYPE, new RestoreInProgress(restoreEntry));
                     } else {
                         shards = ImmutableOpenMap.of();
@@ -336,10 +356,7 @@ public ClusterState execute(ClusterState currentState) {
 
                     RoutingTable rt = rtBuilder.build();
                     ClusterState updatedState = builder.metaData(mdBuilder).blocks(blocks).routingTable(rt).build();
-                    RoutingAllocation.Result routingResult = allocationService.reroute(
-                            ClusterState.builder(updatedState).routingTable(rt).build(),
-                            "restored snapshot [" + snapshot + "]");
-                    return ClusterState.builder(updatedState).routingResult(routingResult).build();
+                    return allocationService.reroute(updatedState, "restored snapshot [" + snapshot + "]");
                 }
 
                 private void checkAliasNameConflicts(Map<String, String> renamedIndices, Set<String> aliases) {
@@ -473,7 +490,7 @@ public TimeValue timeout() {
 
                 @Override
                 public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
-                    listener.onResponse(restoreInfo);
+                    listener.onResponse(new RestoreCompletionResponse(snapshot, restoreInfo));
                 }
             });
 
@@ -484,19 +501,33 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS
         }
     }
 
-    /**
-     * This method is used by {@link IndexShard} to notify
-     * {@code RestoreService} about shard restore completion.
-     *
-     * @param snapshot   snapshot
-     * @param shardId    shard id
-     */
-    public void indexShardRestoreCompleted(Snapshot snapshot, ShardId shardId) {
-        logger.trace("[{}] successfully restored shard  [{}]", snapshot, shardId);
-        UpdateIndexShardRestoreStatusRequest request = new UpdateIndexShardRestoreStatusRequest(snapshot, shardId,
-                new ShardRestoreStatus(clusterService.state().nodes().getLocalNodeId(), RestoreInProgress.State.SUCCESS));
-        transportService.sendRequest(clusterService.state().nodes().getMasterNode(),
-                UPDATE_RESTORE_ACTION_NAME, request, EmptyTransportResponseHandler.INSTANCE_SAME);
+    public static RestoreInProgress updateRestoreStateWithDeletedIndices(RestoreInProgress oldRestore, Set<Index> deletedIndices) {
+        boolean changesMade = false;
+        final List<RestoreInProgress.Entry> entries = new ArrayList<>();
+        for (RestoreInProgress.Entry entry : oldRestore.entries()) {
+            ImmutableOpenMap.Builder<ShardId, ShardRestoreStatus> shardsBuilder = null;
+            for (ObjectObjectCursor<ShardId, ShardRestoreStatus> cursor : entry.shards()) {
+                ShardId shardId = cursor.key;
+                if (deletedIndices.contains(shardId.getIndex())) {
+                    changesMade = true;
+                    if (shardsBuilder == null) {
+                        shardsBuilder = ImmutableOpenMap.builder(entry.shards());
+                    }
+                    shardsBuilder.put(shardId, new ShardRestoreStatus(null, RestoreInProgress.State.FAILURE, "index was deleted"));
+                }
+            }
+            if (shardsBuilder != null) {
+                ImmutableOpenMap<ShardId, ShardRestoreStatus> shards = shardsBuilder.build();
+                entries.add(new RestoreInProgress.Entry(entry.snapshot(), overallState(RestoreInProgress.State.STARTED, shards), entry.indices(), shards));
+            } else {
+                entries.add(entry);
+            }
+        }
+        if (changesMade) {
+            return new RestoreInProgress(entries.toArray(new RestoreInProgress.Entry[entries.size()]));
+        } else {
+            return oldRestore;
+        }
     }
 
     public static final class RestoreCompletionResponse {
@@ -517,168 +548,269 @@ public RestoreInfo getRestoreInfo() {
         }
     }
 
-    /**
-     * Updates shard restore record in the cluster state.
-     *
-     * @param request update shard status request
-     */
-    private void updateRestoreStateOnMaster(final UpdateIndexShardRestoreStatusRequest request) {
-        logger.trace("received updated snapshot restore state [{}]", request);
-        updatedSnapshotStateQueue.add(request);
+    public static class RestoreInProgressUpdater extends RoutingChangesObserver.AbstractRoutingChangesObserver {
+        private final Map<Snapshot, Updates> shardChanges = new HashMap<>();
 
-        clusterService.submitStateUpdateTask("update snapshot state", new ClusterStateUpdateTask() {
-            private final List<UpdateIndexShardRestoreStatusRequest> drainedRequests = new ArrayList<>();
-            private Map<Snapshot, Tuple<RestoreInfo, ImmutableOpenMap<ShardId, ShardRestoreStatus>>> batchedRestoreInfo = null;
-
-            @Override
-            public ClusterState execute(ClusterState currentState) {
+        @Override
+        public void shardStarted(ShardRouting initializingShard, ShardRouting startedShard) {
+            // mark snapshot as completed
+            if (initializingShard.primary()) {
+                RecoverySource recoverySource = initializingShard.recoverySource();
+                if (recoverySource.getType() == RecoverySource.Type.SNAPSHOT) {
+                    Snapshot snapshot = ((SnapshotRecoverySource) recoverySource).snapshot();
+                    changes(snapshot).startedShards.put(initializingShard.shardId(),
+                        new ShardRestoreStatus(initializingShard.currentNodeId(), RestoreInProgress.State.SUCCESS));
+                }
+            }
+        }
 
-                if (request.processed) {
-                    return currentState;
+        @Override
+        public void shardFailed(ShardRouting failedShard, UnassignedInfo unassignedInfo) {
+            if (failedShard.primary() && failedShard.initializing()) {
+                RecoverySource recoverySource = failedShard.recoverySource();
+                if (recoverySource.getType() == RecoverySource.Type.SNAPSHOT) {
+                    Snapshot snapshot = ((SnapshotRecoverySource) recoverySource).snapshot();
+                    // mark restore entry for this shard as failed when it's due to a file corruption. There is no need wait on retries
+                    // to restore this shard on another node if the snapshot files are corrupt. In case where a node just left or crashed,
+                    // however, we only want to acknowledge the restore operation once it has been successfully restored on another node.
+                    if (unassignedInfo.getFailure() != null && Lucene.isCorruptionException(unassignedInfo.getFailure().getCause())) {
+                        changes(snapshot).failedShards.put(failedShard.shardId(), new ShardRestoreStatus(failedShard.currentNodeId(),
+                            RestoreInProgress.State.FAILURE, unassignedInfo.getFailure().getCause().getMessage()));
+                    }
                 }
+            }
+        }
 
-                updatedSnapshotStateQueue.drainTo(drainedRequests);
+        @Override
+        public void shardInitialized(ShardRouting unassignedShard, ShardRouting initializedShard) {
+            // if we force an empty primary, we should also fail the restore entry
+            if (unassignedShard.recoverySource().getType() == RecoverySource.Type.SNAPSHOT &&
+                initializedShard.recoverySource().getType() != RecoverySource.Type.SNAPSHOT) {
+                Snapshot snapshot = ((SnapshotRecoverySource) unassignedShard.recoverySource()).snapshot();
+                changes(snapshot).failedShards.put(unassignedShard.shardId(), new ShardRestoreStatus(null,
+                    RestoreInProgress.State.FAILURE, "recovery source type changed from snapshot to " + initializedShard.recoverySource()));
+            }
+        }
 
-                final int batchSize = drainedRequests.size();
+        /**
+         * Helper method that creates update entry for the given shard id if such an entry does not exist yet.
+         */
+        private Updates changes(Snapshot snapshot) {
+            return shardChanges.computeIfAbsent(snapshot, k -> new Updates());
+        }
 
-                // nothing to process (a previous event has processed it already)
-                if (batchSize == 0) {
-                    return currentState;
-                }
+        private static class Updates {
+            private Map<ShardId, ShardRestoreStatus> failedShards = new HashMap<>();
+            private Map<ShardId, ShardRestoreStatus> startedShards = new HashMap<>();
+        }
 
-                final RestoreInProgress restore = currentState.custom(RestoreInProgress.TYPE);
-                if (restore != null) {
-                    int changedCount = 0;
-                    final List<RestoreInProgress.Entry> entries = new ArrayList<>();
-                    for (RestoreInProgress.Entry entry : restore.entries()) {
-                        ImmutableOpenMap.Builder<ShardId, ShardRestoreStatus> shardsBuilder = null;
-
-                        for (int i = 0; i < batchSize; i++) {
-                            final UpdateIndexShardRestoreStatusRequest updateSnapshotState = drainedRequests.get(i);
-                            updateSnapshotState.processed = true;
-
-                            if (entry.snapshot().equals(updateSnapshotState.snapshot())) {
-                                logger.trace("[{}] Updating shard [{}] with status [{}]", updateSnapshotState.snapshot(), updateSnapshotState.shardId(), updateSnapshotState.status().state());
-                                if (shardsBuilder == null) {
-                                    shardsBuilder = ImmutableOpenMap.builder(entry.shards());
-                                }
-                                shardsBuilder.put(updateSnapshotState.shardId(), updateSnapshotState.status());
-                                changedCount++;
-                            }
+        public RestoreInProgress applyChanges(RestoreInProgress oldRestore) {
+            if (shardChanges.isEmpty() == false) {
+                final List<RestoreInProgress.Entry> entries = new ArrayList<>();
+                for (RestoreInProgress.Entry entry : oldRestore.entries()) {
+                    Snapshot snapshot = entry.snapshot();
+                    Updates updates = shardChanges.get(snapshot);
+                    assert Sets.haveEmptyIntersection(updates.startedShards.keySet(), updates.failedShards.keySet());
+                    if (updates.startedShards.isEmpty() == false || updates.failedShards.isEmpty() == false) {
+                        ImmutableOpenMap.Builder<ShardId, ShardRestoreStatus> shardsBuilder = ImmutableOpenMap.builder(entry.shards());
+                        for (Map.Entry<ShardId, ShardRestoreStatus> startedShardEntry : updates.startedShards.entrySet()) {
+                            shardsBuilder.put(startedShardEntry.getKey(), startedShardEntry.getValue());
                         }
-
-                        if (shardsBuilder != null) {
-                            ImmutableOpenMap<ShardId, ShardRestoreStatus> shards = shardsBuilder.build();
-                            if (!completed(shards)) {
-                                entries.add(new RestoreInProgress.Entry(entry.snapshot(), RestoreInProgress.State.STARTED, entry.indices(), shards));
-                            } else {
-                                logger.info("restore [{}] is done", entry.snapshot());
-                                if (batchedRestoreInfo == null) {
-                                    batchedRestoreInfo = new HashMap<>();
-                                }
-                                assert !batchedRestoreInfo.containsKey(entry.snapshot());
-                                batchedRestoreInfo.put(entry.snapshot(),
-                                    new Tuple<>(
-                                        new RestoreInfo(entry.snapshot().getSnapshotId().getName(),
-                                                        entry.indices(),
-                                                        shards.size(),
-                                                        shards.size() - failedShards(shards)),
-                                        shards));
-                            }
-                        } else {
-                            entries.add(entry);
+                        for (Map.Entry<ShardId, ShardRestoreStatus> failedShardEntry : updates.failedShards.entrySet()) {
+                            shardsBuilder.put(failedShardEntry.getKey(), failedShardEntry.getValue());
                         }
+                        ImmutableOpenMap<ShardId, ShardRestoreStatus> shards = shardsBuilder.build();
+                        RestoreInProgress.State newState = overallState(RestoreInProgress.State.STARTED, shards);
+                        entries.add(new RestoreInProgress.Entry(entry.snapshot(), newState, entry.indices(), shards));
+                    } else {
+                        entries.add(entry);
                     }
+                }
+                return new RestoreInProgress(entries.toArray(new RestoreInProgress.Entry[entries.size()]));
+            } else {
+                return oldRestore;
+            }
+        }
 
-                    if (changedCount > 0) {
-                        logger.trace("changed cluster state triggered by {} snapshot restore state updates", changedCount);
+    }
 
-                        final RestoreInProgress updatedRestore = new RestoreInProgress(entries.toArray(new RestoreInProgress.Entry[entries.size()]));
-                        return ClusterState.builder(currentState).putCustom(RestoreInProgress.TYPE, updatedRestore).build();
-                    }
+    public static RestoreInProgress.Entry restoreInProgress(ClusterState state, Snapshot snapshot) {
+        final RestoreInProgress restoreInProgress = state.custom(RestoreInProgress.TYPE);
+        if (restoreInProgress != null) {
+            for (RestoreInProgress.Entry e : restoreInProgress.entries()) {
+                if (e.snapshot().equals(snapshot)) {
+                    return e;
                 }
-                return currentState;
             }
+        }
+        return null;
+    }
 
-            @Override
-            public void onFailure(String source, @Nullable Exception e) {
-                for (UpdateIndexShardRestoreStatusRequest request : drainedRequests) {
-                    logger.warn((Supplier<?>) () -> new ParameterizedMessage("[{}][{}] failed to update snapshot status to [{}]", request.snapshot(), request.shardId(), request.status()), e);
-                }
+    static class CleanRestoreStateTaskExecutor implements ClusterStateTaskExecutor<CleanRestoreStateTaskExecutor.Task>, ClusterStateTaskListener {
+
+        static class Task {
+            final Snapshot snapshot;
+
+            Task(Snapshot snapshot) {
+                this.snapshot = snapshot;
             }
 
             @Override
-            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
-                if (batchedRestoreInfo != null) {
-                    for (final Entry<Snapshot, Tuple<RestoreInfo, ImmutableOpenMap<ShardId, ShardRestoreStatus>>> entry : batchedRestoreInfo.entrySet()) {
-                        final Snapshot snapshot = entry.getKey();
-                        final RestoreInfo restoreInfo = entry.getValue().v1();
-                        final ImmutableOpenMap<ShardId, ShardRestoreStatus> shards = entry.getValue().v2();
-                        RoutingTable routingTable = newState.getRoutingTable();
-                        final List<ShardId> waitForStarted = new ArrayList<>();
-                        for (ObjectObjectCursor<ShardId, ShardRestoreStatus> shard : shards) {
-                            if (shard.value.state() == RestoreInProgress.State.SUCCESS ) {
-                                ShardId shardId = shard.key;
-                                ShardRouting shardRouting = findPrimaryShard(routingTable, shardId);
-                                if (shardRouting != null && !shardRouting.active()) {
-                                    logger.trace("[{}][{}] waiting for the shard to start", snapshot, shardId);
-                                    waitForStarted.add(shardId);
-                                }
-                            }
-                        }
-                        if (waitForStarted.isEmpty()) {
-                            notifyListeners(snapshot, restoreInfo);
-                        } else {
-                            clusterService.addLast(new ClusterStateListener() {
-                                @Override
-                                public void clusterChanged(ClusterChangedEvent event) {
-                                    if (event.routingTableChanged()) {
-                                        RoutingTable routingTable = event.state().getRoutingTable();
-                                        for (Iterator<ShardId> iterator = waitForStarted.iterator(); iterator.hasNext();) {
-                                            ShardId shardId = iterator.next();
-                                            ShardRouting shardRouting = findPrimaryShard(routingTable, shardId);
-                                            // Shard disappeared (index deleted) or became active
-                                            if (shardRouting == null || shardRouting.active()) {
-                                                iterator.remove();
-                                                logger.trace("[{}][{}] shard disappeared or started - removing", snapshot, shardId);
-                                            }
-                                        }
-                                    }
-                                    if (waitForStarted.isEmpty()) {
-                                        notifyListeners(snapshot, restoreInfo);
-                                        clusterService.remove(this);
-                                    }
-                                }
-                            });
-                        }
+            public String toString() {
+                return "clean restore state for restoring snapshot " + snapshot;
+            }
+        }
+
+        private final Logger logger;
+
+        CleanRestoreStateTaskExecutor(Logger logger) {
+            this.logger = logger;
+        }
+
+        @Override
+        public ClusterTasksResult<Task> execute(final ClusterState currentState, final List<Task> tasks) throws Exception {
+            final ClusterTasksResult.Builder<Task> resultBuilder = ClusterTasksResult.<Task>builder().successes(tasks);
+            Set<Snapshot> completedSnapshots = tasks.stream().map(e -> e.snapshot).collect(Collectors.toSet());
+            final List<RestoreInProgress.Entry> entries = new ArrayList<>();
+            final RestoreInProgress restoreInProgress = currentState.custom(RestoreInProgress.TYPE);
+            boolean changed = false;
+            if (restoreInProgress != null) {
+                for (RestoreInProgress.Entry entry : restoreInProgress.entries()) {
+                    if (completedSnapshots.contains(entry.snapshot()) == false) {
+                        entries.add(entry);
+                    } else {
+                        changed = true;
                     }
                 }
             }
+            if (changed == false) {
+                return resultBuilder.build(currentState);
+            }
+            RestoreInProgress updatedRestoreInProgress = new RestoreInProgress(entries.toArray(new RestoreInProgress.Entry[entries.size()]));
+            ImmutableOpenMap.Builder<String, ClusterState.Custom> builder = ImmutableOpenMap.builder(currentState.getCustoms());
+            builder.put(RestoreInProgress.TYPE, updatedRestoreInProgress);
+            ImmutableOpenMap<String, ClusterState.Custom> customs = builder.build();
+            return resultBuilder.build(ClusterState.builder(currentState).customs(customs).build());
+        }
 
-            private ShardRouting findPrimaryShard(RoutingTable routingTable, ShardId shardId) {
-                IndexRoutingTable indexRoutingTable = routingTable.index(shardId.getIndex());
-                if (indexRoutingTable != null) {
-                    IndexShardRoutingTable indexShardRoutingTable = indexRoutingTable.shard(shardId.id());
-                    if (indexShardRoutingTable != null) {
-                        return indexShardRoutingTable.primaryShard();
-                    }
+        @Override
+        public void onFailure(final String source, final Exception e) {
+            logger.error((Supplier<?>) () -> new ParameterizedMessage("unexpected failure during [{}]", source), e);
+        }
+
+        @Override
+        public void onNoLongerMaster(String source) {
+            logger.debug("no longer master while processing restore state update [{}]", source);
+        }
+
+    }
+
+    private void cleanupRestoreState(ClusterChangedEvent event) {
+        ClusterState state = event.state();
+
+        RestoreInProgress restoreInProgress = state.custom(RestoreInProgress.TYPE);
+        if (restoreInProgress != null) {
+            for (RestoreInProgress.Entry entry : restoreInProgress.entries()) {
+                if (entry.state().completed()) {
+                    assert completed(entry.shards()) : "state says completed but restore entries are not";
+                    clusterService.submitStateUpdateTask(
+                        "clean up snapshot restore state",
+                        new CleanRestoreStateTaskExecutor.Task(entry.snapshot()),
+                        ClusterStateTaskConfig.build(Priority.URGENT),
+                        cleanRestoreStateTaskExecutor,
+                        cleanRestoreStateTaskExecutor);
                 }
-                return null;
             }
 
-            private void notifyListeners(Snapshot snapshot, RestoreInfo restoreInfo) {
-                for (ActionListener<RestoreCompletionResponse> listener : listeners) {
-                    try {
-                        listener.onResponse(new RestoreCompletionResponse(snapshot, restoreInfo));
-                    } catch (Exception e) {
-                        logger.warn((Supplier<?>) () -> new ParameterizedMessage("failed to update snapshot status for [{}]", listener), e);
+            if (event.localNodeMaster() && !event.previousState().nodes().isLocalNodeElectedMaster()) {
+                // old master (before 5.1.0) might have failed to update RestoreInProgress after updating the routing table
+                // try to reconcile routing table and RestoreInProgress here.
+                clusterService.submitStateUpdateTask("update restore state after master switch", new ClusterStateUpdateTask() {
+                    @Override
+                    public ClusterState execute(ClusterState currentState) throws Exception {
+                        return resyncRestoreInProgressWithRoutingTable(currentState);
+                    }
+
+                    @Override
+                    public void onFailure(String source, Exception e) {
+                        logger.warn("failed to sync restore state after master switch", e);
+                    }
+                });
+            }
+        }
+    }
+
+    private ClusterState resyncRestoreInProgressWithRoutingTable(ClusterState currentState) {
+        RoutingTable routingTable = currentState.routingTable();
+        RestoreInProgress restoreInProgress = currentState.custom(RestoreInProgress.TYPE);
+        if (restoreInProgress != null) {
+            final List<RestoreInProgress.Entry> entries = new ArrayList<>();
+            for (RestoreInProgress.Entry entry : restoreInProgress.entries()) {
+                Snapshot snapshot = entry.snapshot();
+                ImmutableOpenMap.Builder<ShardId, ShardRestoreStatus> shardsBuilder = ImmutableOpenMap.builder(entry.shards());
+                for (ObjectObjectCursor<ShardId, ShardRestoreStatus> restoreEntry : entry.shards()) {
+                    ShardId shardId = restoreEntry.key;
+                    IndexShardRoutingTable indexShardRoutingTable = routingTable.shardRoutingTableOrNull(shardId);
+                    if (indexShardRoutingTable == null) {
+                        shardsBuilder.put(shardId, new ShardRestoreStatus(null, RestoreInProgress.State.FAILURE, "index was deleted"));
+                    } else {
+                        if (restoreEntry.value.state().completed() == false) {
+                            // must be INIT (restore state for ShardRestoreStatus is never STARTED)
+                            assert restoreEntry.value.state() == RestoreInProgress.State.INIT;
+                            ShardRouting primaryShard = indexShardRoutingTable.primaryShard();
+                            if (primaryShard.active()) {
+                                // assume shard was started after successful restore from snapshot
+                                shardsBuilder.put(shardId,
+                                    new ShardRestoreStatus(primaryShard.currentNodeId(), RestoreInProgress.State.SUCCESS));
+                            } else {
+                                assert primaryShard.unassigned() || primaryShard.initializing();
+                                if (primaryShard.recoverySource().getType() == RecoverySource.Type.SNAPSHOT &&
+                                    ((SnapshotRecoverySource) primaryShard.recoverySource()).snapshot().equals(snapshot)) {
+                                    // if the primary is unassigned we assume that the restore wasn't started yet and try the restore again
+                                    // once the primary is assigned to a node.
+                                    // if the primary is initializing we assume that the restore is in progress.
+                                } else {
+                                    // for example if an empty primary was forced (see AllocateEmptyPrimaryAllocationCommand)
+                                    shardsBuilder.put(shardId, new ShardRestoreStatus(null, RestoreInProgress.State.FAILURE,
+                                        "recovery source type changed from snapshot to " + primaryShard.recoverySource()));
+                                }
+                            }
+                        }
                     }
                 }
+
+                ImmutableOpenMap<ShardId, ShardRestoreStatus> shards = shardsBuilder.build();
+                RestoreInProgress.State newState = overallState(RestoreInProgress.State.STARTED, shards);
+                entries.add(new RestoreInProgress.Entry(entry.snapshot(), newState, entry.indices(), shards));
             }
-        });
+            RestoreInProgress updatedRestoreInProgress = new RestoreInProgress(entries.toArray(new RestoreInProgress.Entry[entries.size()]));
+            ImmutableOpenMap.Builder<String, ClusterState.Custom> builder = ImmutableOpenMap.builder(currentState.getCustoms());
+            builder.put(RestoreInProgress.TYPE, updatedRestoreInProgress);
+            ImmutableOpenMap<String, ClusterState.Custom> customs = builder.build();
+            return ClusterState.builder(currentState).customs(customs).build();
+        }
+        return currentState;
     }
 
-    private boolean completed(ImmutableOpenMap<ShardId, RestoreInProgress.ShardRestoreStatus> shards) {
+    public static RestoreInProgress.State overallState(RestoreInProgress.State nonCompletedState,
+                                                       ImmutableOpenMap<ShardId, RestoreInProgress.ShardRestoreStatus> shards) {
+        boolean hasFailed = false;
+        for (ObjectCursor<RestoreInProgress.ShardRestoreStatus> status : shards.values()) {
+            if (!status.value.state().completed()) {
+                return nonCompletedState;
+            }
+            if (status.value.state() == RestoreInProgress.State.FAILURE) {
+                hasFailed = true;
+            }
+        }
+        if (hasFailed) {
+            return RestoreInProgress.State.FAILURE;
+        } else {
+            return RestoreInProgress.State.SUCCESS;
+        }
+    }
+
+    public static boolean completed(ImmutableOpenMap<ShardId, RestoreInProgress.ShardRestoreStatus> shards) {
         for (ObjectCursor<RestoreInProgress.ShardRestoreStatus> status : shards.values()) {
             if (!status.value.state().completed()) {
                 return false;
@@ -687,7 +819,7 @@ private boolean completed(ImmutableOpenMap<ShardId, RestoreInProgress.ShardResto
         return true;
     }
 
-    private int failedShards(ImmutableOpenMap<ShardId, RestoreInProgress.ShardRestoreStatus> shards) {
+    public static int failedShards(ImmutableOpenMap<ShardId, RestoreInProgress.ShardRestoreStatus> shards) {
         int failedShards = 0;
         for (ObjectCursor<RestoreInProgress.ShardRestoreStatus> status : shards.values()) {
             if (status.value.state() == RestoreInProgress.State.FAILURE) {
@@ -732,38 +864,20 @@ private void validateSnapshotRestorable(final String repository, final SnapshotI
     }
 
     /**
-     * Checks if any of the deleted indices are still recovering and fails recovery on the shards of these indices
+     * This method is used by {@link IndexShard} to notify
+     * {@code RestoreService} about shard restore completion.
      *
-     * @param event cluster changed event
+     * @param snapshot   snapshot
+     * @param shardId    shard id
      */
-    private void processDeletedIndices(ClusterChangedEvent event) {
-        RestoreInProgress restore = event.state().custom(RestoreInProgress.TYPE);
-        if (restore == null) {
-            // Not restoring - nothing to do
-            return;
-        }
-
-        if (!event.indicesDeleted().isEmpty()) {
-            // Some indices were deleted, let's make sure all indices that we are restoring still exist
-            for (RestoreInProgress.Entry entry : restore.entries()) {
-                List<ShardId> shardsToFail = null;
-                for (ObjectObjectCursor<ShardId, ShardRestoreStatus> shard : entry.shards()) {
-                    if (!shard.value.state().completed()) {
-                        if (!event.state().metaData().hasIndex(shard.key.getIndex().getName())) {
-                            if (shardsToFail == null) {
-                                shardsToFail = new ArrayList<>();
-                            }
-                            shardsToFail.add(shard.key);
-                        }
-                    }
-                }
-                if (shardsToFail != null) {
-                    for (ShardId shardId : shardsToFail) {
-                        logger.trace("[{}] failing running shard restore [{}]", entry.snapshot(), shardId);
-                        updateRestoreStateOnMaster(new UpdateIndexShardRestoreStatusRequest(entry.snapshot(), shardId, new ShardRestoreStatus(null, RestoreInProgress.State.FAILURE, "index was deleted")));
-                    }
-                }
-            }
+    public void indexShardRestoreCompleted(Snapshot snapshot, ShardId shardId) {
+        logger.trace("[{}] successfully restored shard  [{}]", snapshot, shardId);
+        DiscoveryNode masterNode = clusterService.state().nodes().getMasterNode();
+        if (masterNode != null && masterNode.getVersion().before(V_5_1_0_UNRELEASED)) {
+            // just here for backward compatibility with versions before 5.1.0
+            UpdateIndexShardRestoreStatusRequest request = new UpdateIndexShardRestoreStatusRequest(snapshot, shardId,
+                new ShardRestoreStatus(clusterService.state().nodes().getLocalNodeId(), RestoreInProgress.State.SUCCESS));
+            transportService.sendRequest(masterNode, UPDATE_RESTORE_ACTION_NAME, request, EmptyTransportResponseHandler.INSTANCE_SAME);
         }
     }
 
@@ -772,10 +886,13 @@ private void processDeletedIndices(ClusterChangedEvent event) {
      */
     public void failRestore(Snapshot snapshot, ShardId shardId) {
         logger.debug("[{}] failed to restore shard  [{}]", snapshot, shardId);
-        UpdateIndexShardRestoreStatusRequest request = new UpdateIndexShardRestoreStatusRequest(snapshot, shardId,
+        DiscoveryNode masterNode = clusterService.state().nodes().getMasterNode();
+        if (masterNode != null && masterNode.getVersion().before(V_5_1_0_UNRELEASED)) {
+            // just here for backward compatibility with versions before 5.1.0
+            UpdateIndexShardRestoreStatusRequest request = new UpdateIndexShardRestoreStatusRequest(snapshot, shardId,
                 new ShardRestoreStatus(clusterService.state().nodes().getLocalNodeId(), RestoreInProgress.State.FAILURE));
-        transportService.sendRequest(clusterService.state().nodes().getMasterNode(),
-                                     UPDATE_RESTORE_ACTION_NAME, request, EmptyTransportResponseHandler.INSTANCE_SAME);
+            transportService.sendRequest(masterNode, UPDATE_RESTORE_ACTION_NAME, request, EmptyTransportResponseHandler.INSTANCE_SAME);
+        }
     }
 
     private boolean failed(SnapshotInfo snapshot, String index) {
@@ -814,34 +931,11 @@ public static void checkIndexClosing(ClusterState currentState, Set<IndexMetaDat
         }
     }
 
-    /**
-     * Adds restore completion listener
-     * <p>
-     * This listener is called for each snapshot that finishes restore operation in the cluster. It's responsibility of
-     * the listener to decide if it's called for the appropriate snapshot or not.
-     *
-     * @param listener restore completion listener
-     */
-    public void addListener(ActionListener<RestoreCompletionResponse> listener) {
-        this.listeners.add(listener);
-    }
-
-    /**
-     * Removes restore completion listener
-     * <p>
-     * This listener is called for each snapshot that finishes restore operation in the cluster.
-     *
-     * @param listener restore completion listener
-     */
-    public void removeListener(ActionListener<RestoreCompletionResponse> listener) {
-        this.listeners.remove(listener);
-    }
-
     @Override
-    public void clusterChanged(ClusterChangedEvent event) {
+    public void applyClusterState(ClusterChangedEvent event) {
         try {
             if (event.localNodeMaster()) {
-                processDeletedIndices(event);
+                cleanupRestoreState(event);
             }
         } catch (Exception t) {
             logger.warn("Failed to update restore state ", t);
@@ -1126,7 +1220,8 @@ public String toString() {
     class UpdateRestoreStateRequestHandler implements TransportRequestHandler<UpdateIndexShardRestoreStatusRequest> {
         @Override
         public void messageReceived(UpdateIndexShardRestoreStatusRequest request, final TransportChannel channel) throws Exception {
-            updateRestoreStateOnMaster(request);
+            // just here for backward compatibility, no need to do anything, there is already a parallel shard started / failed request
+            // that contains all relevant information needed.
             channel.sendResponse(TransportResponse.Empty.INSTANCE);
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotId.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotId.java
index 4866a79afb95a..ffd7547099c66 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotId.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotId.java
@@ -32,7 +32,7 @@
 /**
  * SnapshotId - snapshot name + snapshot UUID
  */
-public final class SnapshotId implements Writeable, ToXContent {
+public final class SnapshotId implements Comparable<SnapshotId>, Writeable, ToXContent {
 
     private static final String NAME = "name";
     private static final String UUID = "uuid";
@@ -106,6 +106,11 @@ public int hashCode() {
         return hashCode;
     }
 
+    @Override
+    public int compareTo(final SnapshotId other) {
+        return this.name.compareTo(other.name);
+    }
+
     private int computeHashCode() {
         return Objects.hash(name, uuid);
     }
diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java
index ddcee4b035366..de0a52ed0e46a 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java
@@ -21,13 +21,13 @@
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ShardOperationFailedException;
-import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.joda.FormatDateTimeFormatter;
 import org.elasticsearch.common.joda.Joda;
-import org.elasticsearch.common.xcontent.FromXContentBuilder;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -36,15 +36,15 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.List;
 import java.util.Objects;
 
 /**
  * Information about a snapshot
  */
-public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent, FromXContentBuilder<SnapshotInfo>, Writeable {
+public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent, Writeable {
 
-    public static final SnapshotInfo PROTO = new SnapshotInfo(new SnapshotId("", ""), Collections.emptyList(), 0);
     public static final String CONTEXT_MODE_PARAM = "context_mode";
     public static final String CONTEXT_MODE_SNAPSHOT = "SNAPSHOT";
     private static final FormatDateTimeFormatter DATE_TIME_FORMATTER = Joda.forPattern("strictDateOptionalTime");
@@ -70,10 +70,18 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
     private static final String TOTAL_SHARDS = "total_shards";
     private static final String SUCCESSFUL_SHARDS = "successful_shards";
 
+    private static final Version VERSION_INCOMPATIBLE_INTRODUCED = Version.V_5_2_0;
+    public static final Version VERBOSE_INTRODUCED = Version.V_5_5_0;
+
+    private static final Comparator<SnapshotInfo> COMPARATOR =
+        Comparator.comparing(SnapshotInfo::startTime).thenComparing(SnapshotInfo::snapshotId);
+
     private final SnapshotId snapshotId;
 
+    @Nullable
     private final SnapshotState state;
 
+    @Nullable
     private final String reason;
 
     private final List<String> indices;
@@ -86,10 +94,15 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
 
     private final int successfulShards;
 
+    @Nullable
     private final Version version;
 
     private final List<SnapshotShardFailure> shardFailures;
 
+    public SnapshotInfo(SnapshotId snapshotId, List<String> indices, SnapshotState state) {
+        this(snapshotId, indices, state, null, null, 0L, 0L, 0, 0, Collections.emptyList());
+    }
+
     public SnapshotInfo(SnapshotId snapshotId, List<String> indices, long startTime) {
         this(snapshotId, indices, SnapshotState.IN_PROGRESS, null, Version.CURRENT, startTime, 0L, 0, 0, Collections.emptyList());
     }
@@ -103,8 +116,8 @@ public SnapshotInfo(SnapshotId snapshotId, List<String> indices, long startTime,
     private SnapshotInfo(SnapshotId snapshotId, List<String> indices, SnapshotState state, String reason, Version version,
                          long startTime, long endTime, int totalShards, int successfulShards, List<SnapshotShardFailure> shardFailures) {
         this.snapshotId = Objects.requireNonNull(snapshotId);
-        this.indices = Objects.requireNonNull(indices);
-        this.state = Objects.requireNonNull(state);
+        this.indices = Collections.unmodifiableList(Objects.requireNonNull(indices));
+        this.state = state;
         this.reason = reason;
         this.version = version;
         this.startTime = startTime;
@@ -125,7 +138,11 @@ public SnapshotInfo(final StreamInput in) throws IOException {
             indicesListBuilder.add(in.readString());
         }
         indices = Collections.unmodifiableList(indicesListBuilder);
-        state = SnapshotState.fromValue(in.readByte());
+        if (in.getVersion().onOrAfter(VERBOSE_INTRODUCED)) {
+            state = in.readBoolean() ? SnapshotState.fromValue(in.readByte()) : null;
+        } else {
+            state = SnapshotState.fromValue(in.readByte());
+        }
         reason = in.readOptionalString();
         startTime = in.readVLong();
         endTime = in.readVLong();
@@ -141,7 +158,29 @@ public SnapshotInfo(final StreamInput in) throws IOException {
         } else {
             shardFailures = Collections.emptyList();
         }
-        version = Version.readVersion(in);
+        if (in.getVersion().before(VERSION_INCOMPATIBLE_INTRODUCED)) {
+            version = Version.readVersion(in);
+        } else {
+            version = in.readBoolean() ? Version.readVersion(in) : null;
+        }
+    }
+
+    /**
+     * Gets a new {@link SnapshotInfo} instance for a snapshot that is incompatible with the
+     * current version of the cluster.
+     */
+    public static SnapshotInfo incompatible(SnapshotId snapshotId) {
+        return new SnapshotInfo(snapshotId, Collections.emptyList(), SnapshotState.INCOMPATIBLE,
+                                "the snapshot is incompatible with the current version of Elasticsearch and its exact version is unknown",
+                                null, 0L, 0L, 0, 0, Collections.emptyList());
+    }
+
+    /**
+     * Gets a new {@link SnapshotInfo} instance from the given {@link SnapshotInfo} with
+     * all information stripped out except the snapshot id, state, and indices.
+     */
+    public SnapshotInfo basic() {
+        return new SnapshotInfo(snapshotId, indices, state);
     }
 
     /**
@@ -154,25 +193,27 @@ public SnapshotId snapshotId() {
     }
 
     /**
-     * Returns snapshot state
+     * Returns snapshot state; {@code null} if the state is unknown.
      *
      * @return snapshot state
      */
+    @Nullable
     public SnapshotState state() {
         return state;
     }
 
     /**
-     * Returns snapshot failure reason
+     * Returns snapshot failure reason; {@code null} if the snapshot succeeded.
      *
      * @return snapshot failure reason
      */
+    @Nullable
     public String reason() {
         return reason;
     }
 
     /**
-     * Returns indices that were included into this snapshot
+     * Returns indices that were included in this snapshot.
      *
      * @return list of indices
      */
@@ -181,7 +222,8 @@ public List<String> indices() {
     }
 
     /**
-     * Returns time when snapshot started
+     * Returns time when snapshot started; a value of {@code 0L} will be returned if
+     * {@link #state()} returns {@code null}.
      *
      * @return snapshot start time
      */
@@ -190,9 +232,8 @@ public long startTime() {
     }
 
     /**
-     * Returns time when snapshot ended
-     * <p>
-     * Can be 0L if snapshot is still running
+     * Returns time when snapshot ended; a value of {@code 0L} will be returned if the
+     * snapshot is still running or if {@link #state()} returns {@code null}.
      *
      * @return snapshot end time
      */
@@ -201,7 +242,8 @@ public long endTime() {
     }
 
     /**
-     * Returns total number of shards that were snapshotted
+     * Returns total number of shards that were snapshotted; a value of {@code 0} will
+     * be returned if {@link #state()} returns {@code null}.
      *
      * @return number of shards
      */
@@ -210,7 +252,8 @@ public int totalShards() {
     }
 
     /**
-     * Number of failed shards
+     * Number of failed shards; a value of {@code 0} will be returned if there were no
+     * failed shards, or if {@link #state()} returns {@code null}.
      *
      * @return number of failed shards
      */
@@ -219,7 +262,8 @@ public int failedShards() {
     }
 
     /**
-     * Returns total number of shards that were successfully snapshotted
+     * Returns total number of shards that were successfully snapshotted; a value of
+     * {@code 0} will be returned if {@link #state()} returns {@code null}.
      *
      * @return number of successful shards
      */
@@ -228,7 +272,8 @@ public int successfulShards() {
     }
 
     /**
-     * Returns shard failures
+     * Returns shard failures; an empty list will be returned if there were no shard
+     * failures, or if {@link #state()} returns {@code null}.
      *
      * @return shard failures
      */
@@ -237,25 +282,23 @@ public List<SnapshotShardFailure> shardFailures() {
     }
 
     /**
-     * Returns the version of elasticsearch that the snapshot was created with
+     * Returns the version of elasticsearch that the snapshot was created with.  Will only
+     * return {@code null} if {@link #state()} returns {@code null} or {@link SnapshotState#INCOMPATIBLE}.
      *
      * @return version of elasticsearch that the snapshot was created with
      */
+    @Nullable
     public Version version() {
         return version;
     }
 
     /**
-     * Compares two snapshots by their start time
-     *
-     * @param o other snapshot
-     * @return the value {@code 0} if snapshots were created at the same time;
-     * a value less than {@code 0} if this snapshot was created before snapshot {@code o}; and
-     * a value greater than {@code 0} if this snapshot was created after snapshot {@code o};
+     * Compares two snapshots by their start time; if the start times are the same, then
+     * compares the two snapshots by their snapshot ids.
      */
     @Override
     public int compareTo(final SnapshotInfo o) {
-        return Long.compare(startTime, o.startTime);
+        return COMPARATOR.compare(this, o);
     }
 
     @Override
@@ -304,42 +347,51 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa
             return toXContentSnapshot(builder, params);
         }
 
+        final boolean verbose = params.paramAsBoolean("verbose", GetSnapshotsRequest.DEFAULT_VERBOSE_MODE);
         // write snapshot info for the API and any other situations
         builder.startObject();
         builder.field(SNAPSHOT, snapshotId.getName());
         builder.field(UUID, snapshotId.getUUID());
-        builder.field(VERSION_ID, version.id);
-        builder.field(VERSION, version.toString());
+        if (version != null) {
+            builder.field(VERSION_ID, version.id);
+            builder.field(VERSION, version.toString());
+        }
         builder.startArray(INDICES);
         for (String index : indices) {
             builder.value(index);
         }
         builder.endArray();
-        builder.field(STATE, state);
+        if (verbose || state != null) {
+            builder.field(STATE, state);
+        }
         if (reason != null) {
             builder.field(REASON, reason);
         }
-        if (startTime != 0) {
+        if (verbose || startTime != 0) {
             builder.field(START_TIME, DATE_TIME_FORMATTER.printer().print(startTime));
             builder.field(START_TIME_IN_MILLIS, startTime);
         }
-        if (endTime != 0) {
+        if (verbose || endTime != 0) {
             builder.field(END_TIME, DATE_TIME_FORMATTER.printer().print(endTime));
             builder.field(END_TIME_IN_MILLIS, endTime);
             builder.timeValueField(DURATION_IN_MILLIS, DURATION, endTime - startTime);
         }
-        builder.startArray(FAILURES);
-        for (SnapshotShardFailure shardFailure : shardFailures) {
-            builder.startObject();
-            shardFailure.toXContent(builder, params);
+        if (verbose || !shardFailures.isEmpty()) {
+            builder.startArray(FAILURES);
+            for (SnapshotShardFailure shardFailure : shardFailures) {
+                builder.startObject();
+                shardFailure.toXContent(builder, params);
+                builder.endObject();
+            }
+            builder.endArray();
+        }
+        if (verbose || totalShards != 0) {
+            builder.startObject(SHARDS);
+            builder.field(TOTAL, totalShards);
+            builder.field(FAILED, failedShards());
+            builder.field(SUCCESSFUL, successfulShards);
             builder.endObject();
         }
-        builder.endArray();
-        builder.startObject(SHARDS);
-        builder.field(TOTAL, totalShards);
-        builder.field(FAILED, failedShards());
-        builder.field(SUCCESSFUL, successfulShards);
-        builder.endObject();
         builder.endObject();
         return builder;
     }
@@ -348,6 +400,7 @@ private XContentBuilder toXContentSnapshot(final XContentBuilder builder, final
         builder.startObject(SNAPSHOT);
         builder.field(NAME, snapshotId.getName());
         builder.field(UUID, snapshotId.getUUID());
+        assert version != null : "version must always be known when writing a snapshot metadata blob";
         builder.field(VERSION_ID, version.id);
         builder.startArray(INDICES);
         for (String index : indices) {
@@ -373,11 +426,6 @@ private XContentBuilder toXContentSnapshot(final XContentBuilder builder, final
         return builder;
     }
 
-    @Override
-    public SnapshotInfo fromXContent(final XContentParser parser, final ParseFieldMatcher matcher) throws IOException {
-        return fromXContent(parser);
-    }
-
     /**
      * This method creates a SnapshotInfo from internal x-content.  It does not
      * handle x-content written with the external version as external x-content
@@ -479,7 +527,20 @@ public void writeTo(final StreamOutput out) throws IOException {
         for (String index : indices) {
             out.writeString(index);
         }
-        out.writeByte(state.value());
+        if (out.getVersion().onOrAfter(VERBOSE_INTRODUCED)) {
+            if (state != null) {
+                out.writeBoolean(true);
+                out.writeByte(state.value());
+            } else {
+                out.writeBoolean(false);
+            }
+        } else {
+            if (out.getVersion().before(VERSION_INCOMPATIBLE_INTRODUCED) && state == SnapshotState.INCOMPATIBLE) {
+                out.writeByte(SnapshotState.FAILED.value());
+            } else {
+                out.writeByte(state.value());
+            }
+        }
         out.writeOptionalString(reason);
         out.writeVLong(startTime);
         out.writeVLong(endTime);
@@ -489,7 +550,20 @@ public void writeTo(final StreamOutput out) throws IOException {
         for (SnapshotShardFailure failure : shardFailures) {
             failure.writeTo(out);
         }
-        Version.writeVersion(version, out);
+        if (out.getVersion().before(VERSION_INCOMPATIBLE_INTRODUCED)) {
+            Version versionToWrite = version;
+            if (versionToWrite == null) {
+                versionToWrite = Version.CURRENT;
+            }
+            Version.writeVersion(versionToWrite, out);
+        } else {
+            if (version != null) {
+                out.writeBoolean(true);
+                Version.writeVersion(version, out);
+            } else {
+                out.writeBoolean(false);
+            }
+        }
     }
 
     private static SnapshotState snapshotState(final String reason, final List<SnapshotShardFailure> shardFailures) {
diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardFailure.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardFailure.java
index 7facd49088f13..b7b0561db0b7f 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardFailure.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardFailure.java
@@ -62,6 +62,7 @@ public SnapshotShardFailure(@Nullable String nodeId, ShardId shardId, String rea
         this.nodeId = nodeId;
         this.shardId = shardId;
         this.reason = reason;
+        assert reason != null;
         status = RestStatus.INTERNAL_SERVER_ERROR;
     }
 
@@ -192,7 +193,9 @@ public static SnapshotShardFailure fromXContent(XContentParser parser) throws IO
                         } else if ("node_id".equals(currentFieldName)) {
                             snapshotShardFailure.nodeId = parser.text();
                         } else if ("reason".equals(currentFieldName)) {
-                            snapshotShardFailure.reason = parser.text();
+                            // Workaround for https://github.com/elastic/elasticsearch/issues/25878
+                            // Some old snapshot might still have null in shard failure reasons
+                            snapshotShardFailure.reason = parser.textOrNull();
                         } else if ("shard_id".equals(currentFieldName)) {
                             shardId = parser.intValue();
                         } else if ("status".equals(currentFieldName)) {
@@ -215,6 +218,11 @@ public static SnapshotShardFailure fromXContent(XContentParser parser) throws IO
             throw new ElasticsearchParseException("index shard was not set");
         }
         snapshotShardFailure.shardId = new ShardId(index, index_uuid, shardId);
+        // Workaround for https://github.com/elastic/elasticsearch/issues/25878
+        // Some old snapshot might still have null in shard failure reasons
+        if (snapshotShardFailure.reason == null) {
+            snapshotShardFailure.reason = "";
+        }
         return snapshotShardFailure;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java
index 1f7a4ee4fd6af..c0e59d64d4be3 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java
@@ -26,11 +26,14 @@
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.ClusterStateListener;
+import org.elasticsearch.cluster.ClusterStateApplier;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.SnapshotsInProgress;
+import org.elasticsearch.cluster.SnapshotsInProgress.ShardSnapshotStatus;
+import org.elasticsearch.cluster.SnapshotsInProgress.State;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.inject.Inject;
@@ -42,11 +45,13 @@
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.index.engine.SnapshotFailedEngineException;
+import org.elasticsearch.index.shard.IndexEventListener;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.IndexShardState;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.snapshots.IndexShardSnapshotFailedException;
 import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus;
+import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus.Stage;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.repositories.IndexId;
 import org.elasticsearch.repositories.Repository;
@@ -80,7 +85,7 @@
  * This service runs on data and master nodes and controls currently snapshotted shards on these nodes. It is responsible for
  * starting and stopping shard level snapshots
  */
-public class SnapshotShardsService extends AbstractLifecycleComponent implements ClusterStateListener {
+public class SnapshotShardsService extends AbstractLifecycleComponent implements ClusterStateApplier, IndexEventListener {
 
     public static final String UPDATE_SNAPSHOT_ACTION_NAME = "internal:cluster/snapshot/update_snapshot";
 
@@ -114,8 +119,8 @@ public SnapshotShardsService(Settings settings, ClusterService clusterService, S
         this.threadPool = threadPool;
         if (DiscoveryNode.isDataNode(settings)) {
             // this is only useful on the nodes that can hold data
-            // addLast to make sure that Repository will be created before snapshot
-            clusterService.addLast(this);
+            // addLowPriorityApplier to make sure that Repository will be created before snapshot
+            clusterService.addLowPriorityApplier(this);
         }
 
         if (DiscoveryNode.isMasterNode(settings)) {
@@ -147,21 +152,17 @@ protected void doStop() {
 
     @Override
     protected void doClose() {
-        clusterService.remove(this);
+        clusterService.removeApplier(this);
     }
 
     @Override
-    public void clusterChanged(ClusterChangedEvent event) {
+    public void applyClusterState(ClusterChangedEvent event) {
         try {
             SnapshotsInProgress prev = event.previousState().custom(SnapshotsInProgress.TYPE);
             SnapshotsInProgress curr = event.state().custom(SnapshotsInProgress.TYPE);
 
-            if (prev == null) {
-                if (curr != null) {
-                    processIndexShardSnapshots(event);
-                }
-            } else if (prev.equals(curr) == false) {
-                    processIndexShardSnapshots(event);
+            if ((prev == null && curr != null) || (prev != null && prev.equals(curr) == false)) {
+                processIndexShardSnapshots(event);
             }
             String masterNodeId = event.state().nodes().getMasterNodeId();
             if (masterNodeId != null && masterNodeId.equals(event.previousState().nodes().getMasterNodeId()) == false) {
@@ -173,6 +174,18 @@ public void clusterChanged(ClusterChangedEvent event) {
         }
     }
 
+    @Override
+    public void beforeIndexShardClosed(ShardId shardId, @Nullable IndexShard indexShard, Settings indexSettings) {
+        // abort any snapshots occurring on the soon-to-be closed shard
+        Map<Snapshot, SnapshotShards> snapshotShardsMap = shardSnapshots;
+        for (Map.Entry<Snapshot, SnapshotShards> snapshotShards : snapshotShardsMap.entrySet()) {
+            Map<ShardId, IndexShardSnapshotStatus> shards = snapshotShards.getValue().shards;
+            if (shards.containsKey(shardId)) {
+                logger.debug("[{}] shard closing, abort snapshotting for snapshot [{}]", shardId, snapshotShards.getKey().getSnapshotId());
+                shards.get(shardId).abort();
+            }
+        }
+    }
 
     /**
      * Returns status of shards that are snapshotted on the node and belong to the given snapshot
@@ -205,6 +218,16 @@ private void processIndexShardSnapshots(ClusterChangedEvent event) {
             final Snapshot snapshot = entry.getKey();
             if (snapshotsInProgress != null && snapshotsInProgress.snapshot(snapshot) != null) {
                 survivors.put(entry.getKey(), entry.getValue());
+            } else {
+                // abort any running snapshots of shards for the removed entry;
+                // this could happen if for some reason the cluster state update for aborting
+                // running shards is missed, then the snapshot is removed is a subsequent cluster
+                // state update, which is being processed here
+                for (IndexShardSnapshotStatus snapshotStatus : entry.getValue().shards.values()) {
+                    if (snapshotStatus.stage() == Stage.INIT || snapshotStatus.stage() == Stage.STARTED) {
+                        snapshotStatus.abort();
+                    }
+                }
             }
         }
 
@@ -212,19 +235,20 @@ private void processIndexShardSnapshots(ClusterChangedEvent event) {
         // snapshots in the future
         Map<Snapshot, Map<ShardId, IndexShardSnapshotStatus>> newSnapshots = new HashMap<>();
         // Now go through all snapshots and update existing or create missing
-        final String localNodeId = clusterService.localNode().getId();
+        final String localNodeId = event.state().nodes().getLocalNodeId();
+        final DiscoveryNode masterNode = event.state().nodes().getMasterNode();
         final Map<Snapshot, Map<String, IndexId>> snapshotIndices = new HashMap<>();
         if (snapshotsInProgress != null) {
             for (SnapshotsInProgress.Entry entry : snapshotsInProgress.entries()) {
                 snapshotIndices.put(entry.snapshot(),
                                     entry.indices().stream().collect(Collectors.toMap(IndexId::getName, Function.identity())));
-                if (entry.state() == SnapshotsInProgress.State.STARTED) {
+                if (entry.state() == State.STARTED) {
                     Map<ShardId, IndexShardSnapshotStatus> startedShards = new HashMap<>();
                     SnapshotShards snapshotShards = shardSnapshots.get(entry.snapshot());
-                    for (ObjectObjectCursor<ShardId, SnapshotsInProgress.ShardSnapshotStatus> shard : entry.shards()) {
+                    for (ObjectObjectCursor<ShardId, ShardSnapshotStatus> shard : entry.shards()) {
                         // Add all new shards to start processing on
                         if (localNodeId.equals(shard.value.nodeId())) {
-                            if (shard.value.state() == SnapshotsInProgress.State.INIT && (snapshotShards == null || !snapshotShards.shards.containsKey(shard.key))) {
+                            if (shard.value.state() == State.INIT && (snapshotShards == null || !snapshotShards.shards.containsKey(shard.key))) {
                                 logger.trace("[{}] - Adding shard to the queue", shard.key);
                                 startedShards.put(shard.key, new IndexShardSnapshotStatus());
                             }
@@ -245,11 +269,11 @@ private void processIndexShardSnapshots(ClusterChangedEvent event) {
                             survivors.put(entry.snapshot(), new SnapshotShards(unmodifiableMap(startedShards)));
                         }
                     }
-                } else if (entry.state() == SnapshotsInProgress.State.ABORTED) {
+                } else if (entry.state() == State.ABORTED) {
                     // Abort all running shards for this snapshot
                     SnapshotShards snapshotShards = shardSnapshots.get(entry.snapshot());
                     if (snapshotShards != null) {
-                        for (ObjectObjectCursor<ShardId, SnapshotsInProgress.ShardSnapshotStatus> shard : entry.shards()) {
+                        for (ObjectObjectCursor<ShardId, ShardSnapshotStatus> shard : entry.shards()) {
                             IndexShardSnapshotStatus snapshotStatus = snapshotShards.shards.get(shard.key);
                             if (snapshotStatus != null) {
                                 switch (snapshotStatus.stage()) {
@@ -263,12 +287,12 @@ private void processIndexShardSnapshots(ClusterChangedEvent event) {
                                     case DONE:
                                         logger.debug("[{}] trying to cancel snapshot on the shard [{}] that is already done, updating status on the master", entry.snapshot(), shard.key);
                                         updateIndexShardSnapshotStatus(entry.snapshot(), shard.key,
-                                                new SnapshotsInProgress.ShardSnapshotStatus(event.state().nodes().getLocalNodeId(), SnapshotsInProgress.State.SUCCESS));
+                                                new ShardSnapshotStatus(localNodeId, State.SUCCESS), masterNode);
                                         break;
                                     case FAILURE:
                                         logger.debug("[{}] trying to cancel snapshot on the shard [{}] that has already failed, updating status on the master", entry.snapshot(), shard.key);
                                         updateIndexShardSnapshotStatus(entry.snapshot(), shard.key,
-                                                new SnapshotsInProgress.ShardSnapshotStatus(event.state().nodes().getLocalNodeId(), SnapshotsInProgress.State.FAILED, snapshotStatus.failure()));
+                                            new ShardSnapshotStatus(localNodeId, State.FAILED, snapshotStatus.failure()), masterNode);
                                         break;
                                     default:
                                         throw new IllegalStateException("Unknown snapshot shard stage " + snapshotStatus.stage());
@@ -309,18 +333,21 @@ private void processIndexShardSnapshots(ClusterChangedEvent event) {
                             @Override
                             public void doRun() {
                                 snapshot(indexShard, entry.getKey(), indexId, shardEntry.getValue());
-                                updateIndexShardSnapshotStatus(entry.getKey(), shardId, new SnapshotsInProgress.ShardSnapshotStatus(localNodeId, SnapshotsInProgress.State.SUCCESS));
+                                updateIndexShardSnapshotStatus(entry.getKey(), shardId,
+                                    new ShardSnapshotStatus(localNodeId, State.SUCCESS), masterNode);
                             }
 
                             @Override
                             public void onFailure(Exception e) {
                                 logger.warn((Supplier<?>) () -> new ParameterizedMessage("[{}] [{}] failed to create snapshot", shardId, entry.getKey()), e);
-                                updateIndexShardSnapshotStatus(entry.getKey(), shardId, new SnapshotsInProgress.ShardSnapshotStatus(localNodeId, SnapshotsInProgress.State.FAILED, ExceptionsHelper.detailedMessage(e)));
+                                updateIndexShardSnapshotStatus(entry.getKey(), shardId,
+                                    new ShardSnapshotStatus(localNodeId, State.FAILED, ExceptionsHelper.detailedMessage(e)), masterNode);
                             }
 
                         });
                     } catch (Exception e) {
-                        updateIndexShardSnapshotStatus(entry.getKey(), shardId, new SnapshotsInProgress.ShardSnapshotStatus(localNodeId, SnapshotsInProgress.State.FAILED, ExceptionsHelper.detailedMessage(e)));
+                        updateIndexShardSnapshotStatus(entry.getKey(), shardId,
+                            new ShardSnapshotStatus(localNodeId, State.FAILED, ExceptionsHelper.detailedMessage(e)), masterNode);
                     }
                 }
             }
@@ -379,27 +406,29 @@ private void syncShardStatsOnNewMaster(ClusterChangedEvent event) {
         if (snapshotsInProgress == null) {
             return;
         }
+        final String localNodeId = event.state().nodes().getLocalNodeId();
+        final DiscoveryNode masterNode = event.state().nodes().getMasterNode();
         for (SnapshotsInProgress.Entry snapshot : snapshotsInProgress.entries()) {
-            if (snapshot.state() == SnapshotsInProgress.State.STARTED || snapshot.state() == SnapshotsInProgress.State.ABORTED) {
+            if (snapshot.state() == State.STARTED || snapshot.state() == State.ABORTED) {
                 Map<ShardId, IndexShardSnapshotStatus> localShards = currentSnapshotShards(snapshot.snapshot());
                 if (localShards != null) {
-                    ImmutableOpenMap<ShardId, SnapshotsInProgress.ShardSnapshotStatus> masterShards = snapshot.shards();
+                    ImmutableOpenMap<ShardId, ShardSnapshotStatus> masterShards = snapshot.shards();
                     for(Map.Entry<ShardId, IndexShardSnapshotStatus> localShard : localShards.entrySet()) {
                         ShardId shardId = localShard.getKey();
                         IndexShardSnapshotStatus localShardStatus = localShard.getValue();
-                        SnapshotsInProgress.ShardSnapshotStatus masterShard = masterShards.get(shardId);
+                        ShardSnapshotStatus masterShard = masterShards.get(shardId);
                         if (masterShard != null && masterShard.state().completed() == false) {
                             // Master knows about the shard and thinks it has not completed
-                            if (localShardStatus.stage() == IndexShardSnapshotStatus.Stage.DONE) {
+                            if (localShardStatus.stage() == Stage.DONE) {
                                 // but we think the shard is done - we need to make new master know that the shard is done
                                 logger.debug("[{}] new master thinks the shard [{}] is not completed but the shard is done locally, updating status on the master", snapshot.snapshot(), shardId);
                                 updateIndexShardSnapshotStatus(snapshot.snapshot(), shardId,
-                                        new SnapshotsInProgress.ShardSnapshotStatus(event.state().nodes().getLocalNodeId(), SnapshotsInProgress.State.SUCCESS));
-                            } else if (localShard.getValue().stage() == IndexShardSnapshotStatus.Stage.FAILURE) {
+                                        new ShardSnapshotStatus(localNodeId, State.SUCCESS), masterNode);
+                            } else if (localShard.getValue().stage() == Stage.FAILURE) {
                                 // but we think the shard failed - we need to make new master know that the shard failed
                                 logger.debug("[{}] new master thinks the shard [{}] is not completed but the shard failed locally, updating status on master", snapshot.snapshot(), shardId);
                                 updateIndexShardSnapshotStatus(snapshot.snapshot(), shardId,
-                                        new SnapshotsInProgress.ShardSnapshotStatus(event.state().nodes().getLocalNodeId(), SnapshotsInProgress.State.FAILED, localShardStatus.failure()));
+                                        new ShardSnapshotStatus(localNodeId, State.FAILED, localShardStatus.failure()), masterNode);
 
                             }
                         }
@@ -427,7 +456,7 @@ private SnapshotShards(Map<ShardId, IndexShardSnapshotStatus> shards) {
     public static class UpdateIndexShardSnapshotStatusRequest extends TransportRequest {
         private Snapshot snapshot;
         private ShardId shardId;
-        private SnapshotsInProgress.ShardSnapshotStatus status;
+        private ShardSnapshotStatus status;
 
         private volatile boolean processed; // state field, no need to serialize
 
@@ -435,7 +464,7 @@ public UpdateIndexShardSnapshotStatusRequest() {
 
         }
 
-        public UpdateIndexShardSnapshotStatusRequest(Snapshot snapshot, ShardId shardId, SnapshotsInProgress.ShardSnapshotStatus status) {
+        public UpdateIndexShardSnapshotStatusRequest(Snapshot snapshot, ShardId shardId, ShardSnapshotStatus status) {
             this.snapshot = snapshot;
             this.shardId = shardId;
             this.status = status;
@@ -446,7 +475,7 @@ public void readFrom(StreamInput in) throws IOException {
             super.readFrom(in);
             snapshot = new Snapshot(in);
             shardId = ShardId.readShardId(in);
-            status = SnapshotsInProgress.ShardSnapshotStatus.readShardSnapshotStatus(in);
+            status = new ShardSnapshotStatus(in);
         }
 
         @Override
@@ -465,13 +494,13 @@ public ShardId shardId() {
             return shardId;
         }
 
-        public SnapshotsInProgress.ShardSnapshotStatus status() {
+        public ShardSnapshotStatus status() {
             return status;
         }
 
         @Override
         public String toString() {
-            return "" + snapshot + ", shardId [" + shardId + "], status [" + status.state() + "]";
+            return snapshot + ", shardId [" + shardId + "], status [" + status.state() + "]";
         }
 
         public void markAsProcessed() {
@@ -486,15 +515,10 @@ public boolean isProcessed() {
     /**
      * Updates the shard status
      */
-    public void updateIndexShardSnapshotStatus(Snapshot snapshot, ShardId shardId, SnapshotsInProgress.ShardSnapshotStatus status) {
+    public void updateIndexShardSnapshotStatus(Snapshot snapshot, ShardId shardId, ShardSnapshotStatus status, DiscoveryNode master) {
         UpdateIndexShardSnapshotStatusRequest request = new UpdateIndexShardSnapshotStatusRequest(snapshot, shardId, status);
         try {
-            if (clusterService.state().nodes().isLocalNodeElectedMaster()) {
-                innerUpdateSnapshotState(request);
-            } else {
-                transportService.sendRequest(clusterService.state().nodes().getMasterNode(),
-                        UPDATE_SNAPSHOT_ACTION_NAME, request, EmptyTransportResponseHandler.INSTANCE_SAME);
-            }
+            transportService.sendRequest(master, UPDATE_SNAPSHOT_ACTION_NAME, request, EmptyTransportResponseHandler.INSTANCE_SAME);
         } catch (Exception e) {
             logger.warn((Supplier<?>) () -> new ParameterizedMessage("[{}] [{}] failed to update snapshot state", request.snapshot(), request.status()), e);
         }
@@ -533,7 +557,7 @@ public ClusterState execute(ClusterState currentState) {
                     int changedCount = 0;
                     final List<SnapshotsInProgress.Entry> entries = new ArrayList<>();
                     for (SnapshotsInProgress.Entry entry : snapshots.entries()) {
-                        ImmutableOpenMap.Builder<ShardId, SnapshotsInProgress.ShardSnapshotStatus> shards = ImmutableOpenMap.builder();
+                        ImmutableOpenMap.Builder<ShardId, ShardSnapshotStatus> shards = ImmutableOpenMap.builder();
                         boolean updated = false;
 
                         for (int i = 0; i < batchSize; i++) {
@@ -557,7 +581,7 @@ public ClusterState execute(ClusterState currentState) {
                             } else {
                                 // Snapshot is finished - mark it as done
                                 // TODO: Add PARTIAL_SUCCESS status?
-                                SnapshotsInProgress.Entry updatedEntry = new SnapshotsInProgress.Entry(entry, SnapshotsInProgress.State.SUCCESS, shards.build());
+                                SnapshotsInProgress.Entry updatedEntry = new SnapshotsInProgress.Entry(entry, State.SUCCESS, shards.build());
                                 entries.add(updatedEntry);
                                 // Finalize snapshot in the repository
                                 snapshotsService.endSnapshot(updatedEntry);
diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotState.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotState.java
index b893a372d1374..3df5f8fff042f 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotState.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotState.java
@@ -39,7 +39,11 @@ public enum SnapshotState {
     /**
      * Snapshot was partial successful
      */
-    PARTIAL((byte) 3, true, true);
+    PARTIAL((byte) 3, true, true),
+    /**
+     * Snapshot is incompatible with the current version of the cluster
+     */
+    INCOMPATIBLE((byte) 4, true, false);
 
     private byte value;
 
@@ -47,7 +51,7 @@ public enum SnapshotState {
 
     private boolean restorable;
 
-    private SnapshotState(byte value, boolean completed, boolean restorable) {
+    SnapshotState(byte value, boolean completed, boolean restorable) {
         this.value = value;
         this.completed = completed;
         this.restorable = restorable;
@@ -97,6 +101,8 @@ public static SnapshotState fromValue(byte value) {
                 return FAILED;
             case 3:
                 return PARTIAL;
+            case 4:
+                return INCOMPATIBLE;
             default:
                 throw new IllegalArgumentException("No snapshot state for value [" + value + "]");
         }
diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotUtils.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotUtils.java
index 66e9ac88de620..616e5ffecc814 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotUtils.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotUtils.java
@@ -82,8 +82,7 @@ public static List<String> filterIndices(List<String> availableIndices, String[]
                     } else {
                         if (result == null) {
                             // add all the previous ones...
-                            result = new HashSet<>();
-                            result.addAll(availableIndices.subList(0, i));
+                            result = new HashSet<>(availableIndices.subList(0, i));
                         }
                     }
                 } else {
@@ -99,8 +98,7 @@ public static List<String> filterIndices(List<String> availableIndices, String[]
             }
             if (result == null) {
                 // add all the previous ones...
-                result = new HashSet<>();
-                result.addAll(availableIndices.subList(0, i));
+                result = new HashSet<>(availableIndices.subList(0, i));
             }
             boolean found = false;
             for (String index : availableIndices) {
diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
index ea8deea566140..037db4d5caf66 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
@@ -26,13 +26,13 @@
 import org.apache.lucene.util.CollectionUtil;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.ClusterStateListener;
+import org.elasticsearch.cluster.ClusterStateApplier;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.RestoreInProgress;
+import org.elasticsearch.cluster.SnapshotDeletionsInProgress;
 import org.elasticsearch.cluster.SnapshotsInProgress;
 import org.elasticsearch.cluster.SnapshotsInProgress.ShardSnapshotStatus;
 import org.elasticsearch.cluster.SnapshotsInProgress.State;
@@ -48,6 +48,7 @@
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
@@ -64,7 +65,6 @@
 import org.elasticsearch.repositories.Repository;
 import org.elasticsearch.repositories.RepositoryData;
 import org.elasticsearch.repositories.RepositoryMissingException;
-import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.IOException;
@@ -102,7 +102,7 @@
  * notifies all {@link #snapshotCompletionListeners} that snapshot is completed, and finally calls {@link #removeSnapshotFromClusterState(Snapshot, SnapshotInfo, Exception)} to remove snapshot from cluster state</li>
  * </ul>
  */
-public class SnapshotsService extends AbstractLifecycleComponent implements ClusterStateListener {
+public class SnapshotsService extends AbstractLifecycleComponent implements ClusterStateApplier {
 
     private final ClusterService clusterService;
 
@@ -123,21 +123,21 @@ public SnapshotsService(Settings settings, ClusterService clusterService, IndexN
         this.threadPool = threadPool;
 
         if (DiscoveryNode.isMasterNode(settings)) {
-            // addLast to make sure that Repository will be created before snapshot
-            clusterService.addLast(this);
+            // addLowPriorityApplier to make sure that Repository will be created before snapshot
+            clusterService.addLowPriorityApplier(this);
         }
     }
 
     /**
-     * Retrieves list of snapshot ids that are present in a repository
+     * Gets the {@link RepositoryData} for the given repository.
      *
      * @param repositoryName repository name
-     * @return list of snapshot ids
+     * @return repository data
      */
-    public List<SnapshotId> snapshotIds(final String repositoryName) {
+    public RepositoryData getRepositoryData(final String repositoryName) {
         Repository repository = repositoriesService.repository(repositoryName);
         assert repository != null; // should only be called once we've validated the repository exists
-        return repository.getRepositoryData().getSnapshotIds();
+        return repository.getRepositoryData();
     }
 
     /**
@@ -161,11 +161,15 @@ public SnapshotInfo snapshot(final String repositoryName, final SnapshotId snaps
      *
      * @param repositoryName repository name
      * @param snapshotIds       snapshots for which to fetch snapshot information
+     * @param incompatibleSnapshotIds   snapshots for which not to fetch snapshot information
      * @param ignoreUnavailable if true, snapshots that could not be read will only be logged with a warning,
      *                          if false, they will throw an error
      * @return list of snapshots
      */
-    public List<SnapshotInfo> snapshots(final String repositoryName, List<SnapshotId> snapshotIds, final boolean ignoreUnavailable) {
+    public List<SnapshotInfo> snapshots(final String repositoryName,
+                                        final List<SnapshotId> snapshotIds,
+                                        final Set<SnapshotId> incompatibleSnapshotIds,
+                                        final boolean ignoreUnavailable) {
         final Set<SnapshotInfo> snapshotSet = new HashSet<>();
         final Set<SnapshotId> snapshotIdsToIterate = new HashSet<>(snapshotIds);
         // first, look at the snapshots in progress
@@ -179,7 +183,13 @@ public List<SnapshotInfo> snapshots(final String repositoryName, List<SnapshotId
         final Repository repository = repositoriesService.repository(repositoryName);
         for (SnapshotId snapshotId : snapshotIdsToIterate) {
             try {
-                snapshotSet.add(repository.getSnapshotInfo(snapshotId));
+                if (incompatibleSnapshotIds.contains(snapshotId)) {
+                    // an incompatible snapshot - cannot read its snapshot metadata file, just return
+                    // a SnapshotInfo indicating its incompatible
+                    snapshotSet.add(SnapshotInfo.incompatible(snapshotId));
+                } else {
+                    snapshotSet.add(repository.getSnapshotInfo(snapshotId));
+                }
             } catch (Exception ex) {
                 if (ignoreUnavailable) {
                     logger.warn((Supplier<?>) () -> new ParameterizedMessage("failed to get snapshot [{}]", snapshotId), ex);
@@ -232,7 +242,11 @@ public void createSnapshot(final SnapshotRequest request, final CreateSnapshotLi
             @Override
             public ClusterState execute(ClusterState currentState) {
                 validate(request, currentState);
-
+                SnapshotDeletionsInProgress deletionsInProgress = currentState.custom(SnapshotDeletionsInProgress.TYPE);
+                if (deletionsInProgress != null && deletionsInProgress.hasDeletionsInProgress()) {
+                    throw new ConcurrentSnapshotExecutionException(repositoryName, snapshotName,
+                        "cannot snapshot while a snapshot deletion is in-progress");
+                }
                 SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE);
                 if (snapshots == null || snapshots.entries().isEmpty()) {
                     // Store newSnapshot here to be processed in clusterStateProcessed
@@ -245,10 +259,10 @@ public ClusterState execute(ClusterState currentState) {
                                                                 State.INIT,
                                                                 snapshotIndices,
                                                                 System.currentTimeMillis(),
+                                                                repositoryData.getGenId(),
                                                                 null);
                     snapshots = new SnapshotsInProgress(newSnapshot);
                 } else {
-                    // TODO: What should we do if a snapshot is already running?
                     throw new ConcurrentSnapshotExecutionException(repositoryName, snapshotName, "a snapshot is already running");
                 }
                 return ClusterState.builder(currentState).putCustom(SnapshotsInProgress.TYPE, snapshots).build();
@@ -441,7 +455,7 @@ private class CleanupAfterErrorListener implements ActionListener<SnapshotInfo>
         private final CreateSnapshotListener userCreateSnapshotListener;
         private final Exception e;
 
-        public CleanupAfterErrorListener(SnapshotsInProgress.Entry snapshot, boolean snapshotCreated, CreateSnapshotListener userCreateSnapshotListener, Exception e) {
+        CleanupAfterErrorListener(SnapshotsInProgress.Entry snapshot, boolean snapshotCreated, CreateSnapshotListener userCreateSnapshotListener, Exception e) {
             this.snapshot = snapshot;
             this.snapshotCreated = snapshotCreated;
             this.userCreateSnapshotListener = userCreateSnapshotListener;
@@ -468,7 +482,8 @@ private void cleanupAfterError(Exception exception) {
                                                          snapshot.startTime(),
                                                          ExceptionsHelper.detailedMessage(exception),
                                                          0,
-                                                         Collections.emptyList());
+                                                         Collections.emptyList(),
+                                                         snapshot.getRepositoryStateId());
                 } catch (Exception inner) {
                     inner.addSuppressed(exception);
                     logger.warn((Supplier<?>) () -> new ParameterizedMessage("[{}] failed to close snapshot in repository", snapshot.snapshot()), inner);
@@ -543,7 +558,8 @@ public List<SnapshotsInProgress.Entry> currentSnapshots(final String repository,
     /**
      * Returns status of shards  currently finished snapshots
      * <p>
-     * This method is executed on master node and it's complimentary to the {@link SnapshotShardsService#currentSnapshotShards(Snapshot)} because it
+     * This method is executed on master node and it's complimentary to the
+     * {@link SnapshotShardsService#currentSnapshotShards(Snapshot)} because it
      * returns similar information but for already finished snapshots.
      * </p>
      *
@@ -571,8 +587,25 @@ public Map<ShardId, IndexShardSnapshotStatus> snapshotShards(final String reposi
                         shardSnapshotStatus.failure(shardFailure.reason());
                         shardStatus.put(shardId, shardSnapshotStatus);
                     } else {
-                        IndexShardSnapshotStatus shardSnapshotStatus =
-                            repository.getShardSnapshotStatus(snapshotInfo.snapshotId(), snapshotInfo.version(), indexId, shardId);
+                        final IndexShardSnapshotStatus shardSnapshotStatus;
+                        if (snapshotInfo.state() == SnapshotState.FAILED) {
+                            // If the snapshot failed, but the shard's snapshot does
+                            // not have an exception, it means that partial snapshots
+                            // were disabled and in this case, the shard snapshot will
+                            // *not* have any metadata, so attempting to read the shard
+                            // snapshot status will throw an exception.  Instead, we create
+                            // a status for the shard to indicate that the shard snapshot
+                            // could not be taken due to partial being set to false.
+                            shardSnapshotStatus = new IndexShardSnapshotStatus();
+                            shardSnapshotStatus.updateStage(IndexShardSnapshotStatus.Stage.FAILURE);
+                            shardSnapshotStatus.failure("skipped");
+                        } else {
+                            shardSnapshotStatus = repository.getShardSnapshotStatus(
+                                snapshotInfo.snapshotId(),
+                                snapshotInfo.version(),
+                                indexId,
+                                shardId);
+                        }
                         shardStatus.put(shardId, shardSnapshotStatus);
                     }
                 }
@@ -592,7 +625,7 @@ private SnapshotShardFailure findShardFailure(List<SnapshotShardFailure> shardFa
     }
 
     @Override
-    public void clusterChanged(ClusterChangedEvent event) {
+    public void applyClusterState(ClusterChangedEvent event) {
         try {
             if (event.localNodeMaster()) {
                 if (event.nodesRemoved()) {
@@ -601,12 +634,56 @@ public void clusterChanged(ClusterChangedEvent event) {
                 if (event.routingTableChanged()) {
                     processStartedShards(event);
                 }
+                removeFinishedSnapshotFromClusterState(event);
+                finalizeSnapshotDeletionFromPreviousMaster(event);
             }
         } catch (Exception e) {
             logger.warn("Failed to update snapshot state ", e);
         }
     }
 
+    /**
+     * Finalizes a snapshot deletion in progress if the current node is the master but it
+     * was not master in the previous cluster state and there is still a lingering snapshot
+     * deletion in progress in the cluster state.  This means that the old master failed
+     * before it could clean up an in-progress snapshot deletion.  We attempt to delete the
+     * snapshot files and remove the deletion from the cluster state.  It is possible that the
+     * old master was in a state of long GC and then it resumes and tries to delete the snapshot
+     * that has already been deleted by the current master.  This is acceptable however, since
+     * the old master's snapshot deletion will just respond with an error but in actuality, the
+     * snapshot was deleted and a call to GET snapshots would reveal that the snapshot no longer exists.
+     */
+    private void finalizeSnapshotDeletionFromPreviousMaster(ClusterChangedEvent event) {
+        if (event.localNodeMaster() && event.previousState().nodes().isLocalNodeElectedMaster() == false) {
+            SnapshotDeletionsInProgress deletionsInProgress = event.state().custom(SnapshotDeletionsInProgress.TYPE);
+            if (deletionsInProgress != null && deletionsInProgress.hasDeletionsInProgress()) {
+                assert deletionsInProgress.getEntries().size() == 1 : "only one in-progress deletion allowed per cluster";
+                SnapshotDeletionsInProgress.Entry entry = deletionsInProgress.getEntries().get(0);
+                deleteSnapshotFromRepository(entry.getSnapshot(), null, entry.getRepositoryStateId());
+            }
+        }
+    }
+
+    /**
+     * Removes a finished snapshot from the cluster state.  This can happen if the previous
+     * master node processed a cluster state update that marked the snapshot as finished,
+     * but the previous master node died before removing the snapshot in progress from the
+     * cluster state.  It is then the responsibility of the new master node to end the
+     * snapshot and remove it from the cluster state.
+     */
+    private void removeFinishedSnapshotFromClusterState(ClusterChangedEvent event) {
+        if (event.localNodeMaster() && !event.previousState().nodes().isLocalNodeElectedMaster()) {
+            SnapshotsInProgress snapshotsInProgress = event.state().custom(SnapshotsInProgress.TYPE);
+            if (snapshotsInProgress != null && !snapshotsInProgress.entries().isEmpty()) {
+                for (SnapshotsInProgress.Entry entry : snapshotsInProgress.entries()) {
+                    if (entry.state().completed()) {
+                        endSnapshot(entry);
+                    }
+                }
+            }
+        }
+    }
+
     /**
      * Cleans up shard snapshots that were running on removed nodes
      *
@@ -667,10 +744,7 @@ public void onResponse() {
                                 public void onFailure(Exception e) {
                                     logger.warn("failed to clean up abandoned snapshot {} in INIT state", snapshot.snapshot());
                                 }
-                            });
-                        } else if (snapshot.state() == State.SUCCESS && newMaster) {
-                            // Finalize the snapshot
-                            endSnapshot(snapshot);
+                            }, updatedSnapshot.getRepositoryStateId(), false);
                         }
                     }
                     if (changed) {
@@ -793,12 +867,12 @@ private boolean waitingShardsStartedOrUnassigned(ClusterChangedEvent event) {
     }
 
     private boolean removedNodesCleanupNeeded(ClusterChangedEvent event) {
-        // Check if we just became the master
-        boolean newMaster = !event.previousState().nodes().isLocalNodeElectedMaster();
         SnapshotsInProgress snapshotsInProgress = event.state().custom(SnapshotsInProgress.TYPE);
         if (snapshotsInProgress == null) {
             return false;
         }
+        // Check if we just became the master
+        boolean newMaster = !event.previousState().nodes().isLocalNodeElectedMaster();
         for (SnapshotsInProgress.Entry snapshot : snapshotsInProgress.entries()) {
             if (newMaster && (snapshot.state() == State.SUCCESS || snapshot.state() == State.INIT)) {
                 // We just replaced old master and snapshots in intermediate states needs to be cleaned
@@ -865,17 +939,22 @@ public void run() {
                 try {
                     final Repository repository = repositoriesService.repository(snapshot.getRepository());
                     logger.trace("[{}] finalizing snapshot in repository, state: [{}], failure[{}]", snapshot, entry.state(), failure);
-                    ArrayList<ShardSearchFailure> failures = new ArrayList<>();
                     ArrayList<SnapshotShardFailure> shardFailures = new ArrayList<>();
                     for (ObjectObjectCursor<ShardId, ShardSnapshotStatus> shardStatus : entry.shards()) {
                         ShardId shardId = shardStatus.key;
                         ShardSnapshotStatus status = shardStatus.value;
                         if (status.state().failed()) {
-                            failures.add(new ShardSearchFailure(status.reason(), new SearchShardTarget(status.nodeId(), shardId.getIndex(), shardId.id())));
                             shardFailures.add(new SnapshotShardFailure(status.nodeId(), shardId, status.reason()));
                         }
                     }
-                    SnapshotInfo snapshotInfo = repository.finalizeSnapshot(snapshot.getSnapshotId(), entry.indices(), entry.startTime(), failure, entry.shards().size(), Collections.unmodifiableList(shardFailures));
+                    SnapshotInfo snapshotInfo = repository.finalizeSnapshot(
+                        snapshot.getSnapshotId(),
+                        entry.indices(),
+                        entry.startTime(),
+                        failure,
+                        entry.shards().size(),
+                        Collections.unmodifiableList(shardFailures),
+                        entry.getRepositoryStateId());
                     removeSnapshotFromClusterState(snapshot, snapshotInfo, null);
                 } catch (Exception e) {
                     logger.warn((Supplier<?>) () -> new ParameterizedMessage("[{}] failed to finalize snapshot", snapshot), e);
@@ -904,6 +983,7 @@ private void removeSnapshotFromClusterState(final Snapshot snapshot, final Snaps
     private void removeSnapshotFromClusterState(final Snapshot snapshot, final SnapshotInfo snapshotInfo, final Exception failure,
                                                 @Nullable ActionListener<SnapshotInfo> listener) {
         clusterService.submitStateUpdateTask("remove snapshot metadata", new ClusterStateUpdateTask() {
+
             @Override
             public ClusterState execute(ClusterState currentState) {
                 SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE);
@@ -961,10 +1041,17 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS
      * @param snapshotName    snapshotName
      * @param listener        listener
      */
-    public void deleteSnapshot(final String repositoryName, final String snapshotName, final DeleteSnapshotListener listener) {
+    public void deleteSnapshot(final String repositoryName, final String snapshotName, final DeleteSnapshotListener listener,
+                               final boolean immediatePriority) {
         // First, look for the snapshot in the repository
         final Repository repository = repositoriesService.repository(repositoryName);
-        Optional<SnapshotId> matchedEntry = repository.getRepositoryData().getSnapshotIds()
+        final RepositoryData repositoryData = repository.getRepositoryData();
+        final Optional<SnapshotId> incompatibleSnapshotId =
+            repositoryData.getIncompatibleSnapshotIds().stream().filter(s -> snapshotName.equals(s.getName())).findFirst();
+        if (incompatibleSnapshotId.isPresent()) {
+            throw new SnapshotException(repositoryName, snapshotName, "cannot delete incompatible snapshot");
+        }
+        Optional<SnapshotId> matchedEntry = repositoryData.getSnapshotIds()
                                                 .stream()
                                                 .filter(s -> s.getName().equals(snapshotName))
                                                 .findFirst();
@@ -976,7 +1063,7 @@ public void deleteSnapshot(final String repositoryName, final String snapshotNam
         if (matchedEntry.isPresent() == false) {
             throw new SnapshotMissingException(repositoryName, snapshotName);
         }
-        deleteSnapshot(new Snapshot(repositoryName, matchedEntry.get()), listener);
+        deleteSnapshot(new Snapshot(repositoryName, matchedEntry.get()), listener, repositoryData.getGenId(), immediatePriority);
     }
 
     /**
@@ -984,16 +1071,23 @@ public void deleteSnapshot(final String repositoryName, final String snapshotNam
      * <p>
      * If the snapshot is still running cancels the snapshot first and then deletes it from the repository.
      *
-     * @param snapshot  snapshot
-     * @param listener  listener
+     * @param snapshot snapshot
+     * @param listener listener
+     * @param repositoryStateId the unique id for the state of the repository
      */
-    public void deleteSnapshot(final Snapshot snapshot, final DeleteSnapshotListener listener) {
-        clusterService.submitStateUpdateTask("delete snapshot", new ClusterStateUpdateTask() {
+    private void deleteSnapshot(final Snapshot snapshot, final DeleteSnapshotListener listener, final long repositoryStateId,
+                                final boolean immediatePriority) {
+        Priority priority = immediatePriority ? Priority.IMMEDIATE : Priority.NORMAL;
+        clusterService.submitStateUpdateTask("delete snapshot", new ClusterStateUpdateTask(priority) {
 
             boolean waitForSnapshot = false;
 
             @Override
             public ClusterState execute(ClusterState currentState) throws Exception {
+                SnapshotDeletionsInProgress deletionsInProgress = currentState.custom(SnapshotDeletionsInProgress.TYPE);
+                if (deletionsInProgress != null && deletionsInProgress.hasDeletionsInProgress()) {
+                    throw new ConcurrentSnapshotExecutionException(snapshot, "cannot delete - another snapshot is currently being deleted");
+                }
                 RestoreInProgress restoreInProgress = currentState.custom(RestoreInProgress.TYPE);
                 if (restoreInProgress != null) {
                     // don't allow snapshot deletions while a restore is taking place,
@@ -1003,19 +1097,27 @@ public ClusterState execute(ClusterState currentState) throws Exception {
                         throw new ConcurrentSnapshotExecutionException(snapshot, "cannot delete snapshot during a restore");
                     }
                 }
+                ClusterState.Builder clusterStateBuilder = ClusterState.builder(currentState);
                 SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE);
-                if (snapshots == null) {
-                    // No snapshots running - we can continue
-                    return currentState;
-                }
-                SnapshotsInProgress.Entry snapshotEntry = snapshots.snapshot(snapshot);
+                SnapshotsInProgress.Entry snapshotEntry = snapshots != null ? snapshots.snapshot(snapshot) : null;
                 if (snapshotEntry == null) {
-                    // This snapshot is not running - continue
-                    if (!snapshots.entries().isEmpty()) {
+                    // This snapshot is not running - delete
+                    if (snapshots != null && !snapshots.entries().isEmpty()) {
                         // However other snapshots are running - cannot continue
                         throw new ConcurrentSnapshotExecutionException(snapshot, "another snapshot is currently running cannot delete");
                     }
-                    return currentState;
+                    // add the snapshot deletion to the cluster state
+                    SnapshotDeletionsInProgress.Entry entry = new SnapshotDeletionsInProgress.Entry(
+                        snapshot,
+                        System.currentTimeMillis(),
+                        repositoryStateId
+                    );
+                    if (deletionsInProgress != null) {
+                        deletionsInProgress = deletionsInProgress.withAddedEntry(entry);
+                    } else {
+                        deletionsInProgress = SnapshotDeletionsInProgress.newInstance(entry);
+                    }
+                    clusterStateBuilder.putCustom(SnapshotDeletionsInProgress.TYPE, deletionsInProgress);
                 } else {
                     // This snapshot is currently running - stopping shards first
                     waitForSnapshot = true;
@@ -1026,7 +1128,8 @@ public ClusterState execute(ClusterState currentState) throws Exception {
                         for (ObjectObjectCursor<ShardId, ShardSnapshotStatus> shardEntry : snapshotEntry.shards()) {
                             ShardSnapshotStatus status = shardEntry.value;
                             if (!status.state().completed()) {
-                                shardsBuilder.put(shardEntry.key, new ShardSnapshotStatus(status.nodeId(), State.ABORTED));
+                                shardsBuilder.put(shardEntry.key, new ShardSnapshotStatus(status.nodeId(), State.ABORTED,
+                                    "aborted by snapshot deletion"));
                             } else {
                                 shardsBuilder.put(shardEntry.key, status);
                             }
@@ -1060,8 +1163,9 @@ public ClusterState execute(ClusterState currentState) throws Exception {
                     }
                     SnapshotsInProgress.Entry newSnapshot = new SnapshotsInProgress.Entry(snapshotEntry, State.ABORTED, shards);
                     snapshots = new SnapshotsInProgress(newSnapshot);
-                    return ClusterState.builder(currentState).putCustom(SnapshotsInProgress.TYPE, snapshots).build();
+                    clusterStateBuilder.putCustom(SnapshotsInProgress.TYPE, snapshots);
                 }
+                return clusterStateBuilder.build();
             }
 
             @Override
@@ -1077,24 +1181,42 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS
                         @Override
                         public void onSnapshotCompletion(Snapshot completedSnapshot, SnapshotInfo snapshotInfo) {
                             if (completedSnapshot.equals(snapshot)) {
-                                logger.trace("deleted snapshot completed - deleting files");
+                                logger.debug("deleted snapshot completed - deleting files");
                                 removeListener(this);
-                                deleteSnapshotFromRepository(snapshot, listener);
+                                threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(() ->
+                                    deleteSnapshot(completedSnapshot.getRepository(), completedSnapshot.getSnapshotId().getName(),
+                                        listener, true)
+                                );
                             }
                         }
 
                         @Override
                         public void onSnapshotFailure(Snapshot failedSnapshot, Exception e) {
                             if (failedSnapshot.equals(snapshot)) {
-                                logger.trace("deleted snapshot failed - deleting files", e);
+                                logger.warn("deleted snapshot failed - deleting files", e);
                                 removeListener(this);
-                                deleteSnapshotFromRepository(snapshot, listener);
+                                threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(() -> {
+                                    try {
+                                        deleteSnapshot(failedSnapshot.getRepository(),
+                                                       failedSnapshot.getSnapshotId().getName(),
+                                                       listener,
+                                                       true);
+                                    } catch (SnapshotMissingException smex) {
+                                        logger.info((Supplier<?>) () -> new ParameterizedMessage(
+                                            "Tried deleting in-progress snapshot [{}], but it " +
+                                            "could not be found after failing to abort.",
+                                            smex.getSnapshotName()), e);
+                                        listener.onFailure(new SnapshotException(snapshot,
+                                            "Tried deleting in-progress snapshot [{}], but it " +
+                                            "could not be found after failing to abort.", smex));
+                                    }
+                                });
                             }
                         }
                     });
                 } else {
-                    logger.trace("deleted snapshot is not running - deleting files");
-                    deleteSnapshotFromRepository(snapshot, listener);
+                    logger.debug("deleted snapshot is not running - deleting files");
+                    deleteSnapshotFromRepository(snapshot, listener, repositoryStateId);
                 }
             }
         });
@@ -1116,6 +1238,14 @@ public static boolean isRepositoryInUse(ClusterState clusterState, String reposi
                 }
             }
         }
+        SnapshotDeletionsInProgress deletionsInProgress = clusterState.custom(SnapshotDeletionsInProgress.TYPE);
+        if (deletionsInProgress != null) {
+            for (SnapshotDeletionsInProgress.Entry entry : deletionsInProgress.getEntries()) {
+                if (entry.getSnapshot().getRepository().equals(repository)) {
+                    return true;
+                }
+            }
+        }
         return false;
     }
 
@@ -1124,15 +1254,62 @@ public static boolean isRepositoryInUse(ClusterState clusterState, String reposi
      *
      * @param snapshot   snapshot
      * @param listener   listener
+     * @param repositoryStateId the unique id representing the state of the repository at the time the deletion began
      */
-    private void deleteSnapshotFromRepository(final Snapshot snapshot, final DeleteSnapshotListener listener) {
+    private void deleteSnapshotFromRepository(final Snapshot snapshot, @Nullable final DeleteSnapshotListener listener,
+                                              long repositoryStateId) {
         threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(() -> {
             try {
                 Repository repository = repositoriesService.repository(snapshot.getRepository());
-                repository.deleteSnapshot(snapshot.getSnapshotId());
-                listener.onResponse();
-            } catch (Exception t) {
-                listener.onFailure(t);
+                repository.deleteSnapshot(snapshot.getSnapshotId(), repositoryStateId);
+                removeSnapshotDeletionFromClusterState(snapshot, null, listener);
+            } catch (Exception ex) {
+                removeSnapshotDeletionFromClusterState(snapshot, ex, listener);
+            }
+        });
+    }
+
+    /**
+     * Removes the snapshot deletion from {@link SnapshotDeletionsInProgress} in the cluster state.
+     */
+    private void removeSnapshotDeletionFromClusterState(final Snapshot snapshot, @Nullable final Exception failure,
+                                                        @Nullable final DeleteSnapshotListener listener) {
+        clusterService.submitStateUpdateTask("remove snapshot deletion metadata", new ClusterStateUpdateTask() {
+            @Override
+            public ClusterState execute(ClusterState currentState) {
+                SnapshotDeletionsInProgress deletions = currentState.custom(SnapshotDeletionsInProgress.TYPE);
+                if (deletions != null) {
+                    boolean changed = false;
+                    if (deletions.hasDeletionsInProgress()) {
+                        assert deletions.getEntries().size() == 1 : "should have exactly one deletion in progress";
+                        SnapshotDeletionsInProgress.Entry entry = deletions.getEntries().get(0);
+                        deletions = deletions.withRemovedEntry(entry);
+                        changed = true;
+                    }
+                    if (changed) {
+                        return ClusterState.builder(currentState).putCustom(SnapshotDeletionsInProgress.TYPE, deletions).build();
+                    }
+                }
+                return currentState;
+            }
+
+            @Override
+            public void onFailure(String source, Exception e) {
+                logger.warn((Supplier<?>) () -> new ParameterizedMessage("[{}] failed to remove snapshot deletion metadata", snapshot), e);
+                if (listener != null) {
+                    listener.onFailure(e);
+                }
+            }
+
+            @Override
+            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                if (listener != null) {
+                    if (failure != null) {
+                        listener.onFailure(failure);
+                    } else {
+                        listener.onResponse();
+                    }
+                }
             }
         });
     }
@@ -1273,7 +1450,7 @@ protected void doStop() {
 
     @Override
     protected void doClose() {
-        clusterService.remove(this);
+        clusterService.removeApplier(this);
     }
 
     public RepositoriesService getRepositoriesService() {
diff --git a/core/src/main/java/org/elasticsearch/tasks/CancellableTask.java b/core/src/main/java/org/elasticsearch/tasks/CancellableTask.java
index eb084a57e1034..685e9bcf35251 100644
--- a/core/src/main/java/org/elasticsearch/tasks/CancellableTask.java
+++ b/core/src/main/java/org/elasticsearch/tasks/CancellableTask.java
@@ -26,7 +26,7 @@
 /**
  * A task that can be canceled
  */
-public class CancellableTask extends Task {
+public abstract class CancellableTask extends Task {
 
     private final AtomicReference<String> reason = new AtomicReference<>();
 
@@ -51,6 +51,11 @@ public boolean cancelOnParentLeaving() {
         return true;
     }
 
+    /**
+     * Returns true if this task can potentially have children that need to be cancelled when it parent is cancelled.
+     */
+    public abstract boolean shouldCancelChildrenOnCancellation();
+
     public boolean isCancelled() {
         return reason.get() != null;
     }
diff --git a/core/src/main/java/org/elasticsearch/tasks/Task.java b/core/src/main/java/org/elasticsearch/tasks/Task.java
index 53e5855c025df..bc2e8418141dc 100644
--- a/core/src/main/java/org/elasticsearch/tasks/Task.java
+++ b/core/src/main/java/org/elasticsearch/tasks/Task.java
@@ -42,8 +42,14 @@ public class Task {
 
     private final TaskId parentTask;
 
+    /**
+     * The task's start time as a wall clock time since epoch ({@link System#currentTimeMillis()} style).
+     */
     private final long startTime;
 
+    /**
+     * The task's start time as a relative time ({@link System#nanoTime()} style).
+     */
     private final long startTimeNanos;
 
     public Task(long id, String type, String action, String description, TaskId parentTask) {
@@ -64,20 +70,27 @@ public Task(long id, String type, String action, String description, TaskId pare
      * Build a version of the task status you can throw over the wire and back
      * to the user.
      *
-     * @param node
-     *            the node this task is running on
+     * @param localNodeId
+     *            the id of the node this task is running on
      * @param detailed
      *            should the information include detailed, potentially slow to
      *            generate data?
      */
-    public TaskInfo taskInfo(DiscoveryNode node, boolean detailed) {
+    public final TaskInfo taskInfo(String localNodeId, boolean detailed) {
         String description = null;
         Task.Status status = null;
         if (detailed) {
             description = getDescription();
             status = getStatus();
         }
-        return new TaskInfo(new TaskId(node.getId(), getId()), getType(), getAction(), description, status, startTime,
+        return taskInfo(localNodeId, description, status);
+    }
+
+    /**
+     * Build a proper {@link TaskInfo} for this task.
+     */
+    protected final TaskInfo taskInfo(String localNodeId, String description, Status status) {
+        return new TaskInfo(new TaskId(localNodeId, getId()), getType(), getAction(), description, status, startTime,
                 System.nanoTime() - startTimeNanos, this instanceof CancellableTask, parentTask);
     }
 
@@ -110,7 +123,7 @@ public String getDescription() {
     }
 
     /**
-     * Returns the task start time
+     * Returns the task's start time as a wall clock time since epoch ({@link System#currentTimeMillis()} style).
      */
     public long getStartTime() {
         return startTime;
@@ -136,12 +149,12 @@ public Status getStatus() {
     public interface Status extends ToXContent, NamedWriteable {}
 
     public TaskResult result(DiscoveryNode node, Exception error) throws IOException {
-        return new TaskResult(taskInfo(node, true), error);
+        return new TaskResult(taskInfo(node.getId(), true), error);
     }
 
     public TaskResult result(DiscoveryNode node, ActionResponse response) throws IOException {
         if (response instanceof ToXContent) {
-            return new TaskResult(taskInfo(node, true), (ToXContent) response);
+            return new TaskResult(taskInfo(node.getId(), true), (ToXContent) response);
         } else {
             throw new IllegalStateException("response has to implement ToXContent to be able to store the results");
         }
diff --git a/core/src/main/java/org/elasticsearch/tasks/TaskAwareRequest.java b/core/src/main/java/org/elasticsearch/tasks/TaskAwareRequest.java
new file mode 100644
index 0000000000000..a2364ac8e4047
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/tasks/TaskAwareRequest.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.tasks;
+
+/**
+ * An interface for a request that can be used to register a task manager task
+ */
+public interface TaskAwareRequest {
+    /**
+     * Set a reference to task that caused this task to be run.
+     */
+    default void setParentTask(String parentTaskNode, long parentTaskId) {
+        setParentTask(new TaskId(parentTaskNode, parentTaskId));
+    }
+
+    /**
+     * Set a reference to task that created this request.
+     */
+    void setParentTask(TaskId taskId);
+
+    /**
+     * Get a reference to the task that created this request. Implementers should default to
+     * {@link TaskId#EMPTY_TASK_ID}, meaning "there is no parent".
+     */
+    TaskId getParentTask();
+
+    /**
+     * Returns the task object that should be used to keep track of the processing of the request.
+     *
+     * A request can override this method and return null to avoid being tracked by the task
+     * manager.
+     */
+    default Task createTask(long id, String type, String action, TaskId parentTaskId) {
+        return new Task(id, type, action, getDescription(), parentTaskId);
+    }
+
+    /**
+     * Returns optional description of the request to be displayed by the task manager
+     */
+    default String getDescription() {
+        return "";
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/tasks/TaskCancelledException.java b/core/src/main/java/org/elasticsearch/tasks/TaskCancelledException.java
new file mode 100644
index 0000000000000..4da9eb05a6870
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/tasks/TaskCancelledException.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.tasks;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.common.io.stream.StreamInput;
+
+import java.io.IOException;
+
+/**
+ * A generic exception that can be thrown by a task when it's cancelled by the task manager API
+ */
+public class TaskCancelledException  extends ElasticsearchException {
+
+    public TaskCancelledException(String msg) {
+        super(msg);
+    }
+
+    public TaskCancelledException(StreamInput in) throws IOException{
+        super(in);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/tasks/TaskInfo.java b/core/src/main/java/org/elasticsearch/tasks/TaskInfo.java
index 271586779e273..bfd2addb9c598 100644
--- a/core/src/main/java/org/elasticsearch/tasks/TaskInfo.java
+++ b/core/src/main/java/org/elasticsearch/tasks/TaskInfo.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.tasks;
 
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -164,12 +163,6 @@ public TaskId getParentTaskId() {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject();
-        innerToXContent(builder, params);
-        return builder.endObject();
-    }
-
-    public XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
         builder.field("node", taskId.getNodeId());
         builder.field("id", taskId.getId());
         builder.field("type", type);
@@ -180,7 +173,7 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t
         if (description != null) {
             builder.field("description", description);
         }
-        builder.dateValueField("start_time_in_millis", "start_time", startTime);
+        builder.dateField("start_time_in_millis", "start_time", startTime);
         builder.timeValueField("running_time_in_nanos", "running_time", runningTimeNanos, TimeUnit.NANOSECONDS);
         builder.field("cancellable", cancellable);
         if (parentTaskId.isSet()) {
@@ -189,8 +182,8 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t
         return builder;
     }
 
-    public static final ConstructingObjectParser<TaskInfo, ParseFieldMatcherSupplier> PARSER = new ConstructingObjectParser<>(
-            "task_info", a -> {
+    public static final ConstructingObjectParser<TaskInfo, Void> PARSER = new ConstructingObjectParser<>(
+            "task_info", true, a -> {
                 int i = 0;
                 TaskId id = new TaskId((String) a[i++], (Long) a[i++]);
                 String type = (String) a[i++];
@@ -203,11 +196,11 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t
                 String parentTaskIdString = (String) a[i++];
 
                 RawTaskStatus status = statusBytes == null ? null : new RawTaskStatus(statusBytes);
-                TaskId parentTaskId = parentTaskIdString == null ? TaskId.EMPTY_TASK_ID : new TaskId((String) parentTaskIdString);
+                TaskId parentTaskId = parentTaskIdString == null ? TaskId.EMPTY_TASK_ID : new TaskId(parentTaskIdString);
                 return new TaskInfo(id, type, action, description, status, startTime, runningTimeNanos, cancellable, parentTaskId);
             });
     static {
-        // Note for the future: this has to be backwards compatible with all changes to the task storage format
+        // Note for the future: this has to be backwards and forwards compatible with all changes to the task storage format
         PARSER.declareString(constructorArg(), new ParseField("node"));
         PARSER.declareLong(constructorArg(), new ParseField("id"));
         PARSER.declareString(constructorArg(), new ParseField("type"));
diff --git a/core/src/main/java/org/elasticsearch/tasks/TaskManager.java b/core/src/main/java/org/elasticsearch/tasks/TaskManager.java
index 003a51c31756b..afeeeeedd1168 100644
--- a/core/src/main/java/org/elasticsearch/tasks/TaskManager.java
+++ b/core/src/main/java/org/elasticsearch/tasks/TaskManager.java
@@ -27,7 +27,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.cluster.ClusterChangedEvent;
-import org.elasticsearch.cluster.ClusterStateListener;
+import org.elasticsearch.cluster.ClusterStateApplier;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.component.AbstractComponent;
@@ -35,25 +35,21 @@
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.common.util.concurrent.ConcurrentMapLong;
-import org.elasticsearch.transport.TransportRequest;
 
 import java.io.IOException;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Map;
-import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicLong;
-import java.util.function.Consumer;
 
 import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
 
 /**
  * Task Manager service for keeping track of currently running tasks on the nodes
  */
-public class TaskManager extends AbstractComponent implements ClusterStateListener {
+public class TaskManager extends AbstractComponent implements ClusterStateApplier {
     private static final TimeValue WAIT_FOR_COMPLETION_POLL = timeValueMillis(100);
 
     private final ConcurrentMapLong<Task> tasks = ConcurrentCollections.newConcurrentMapLongWithAggressiveConcurrency();
@@ -83,7 +79,7 @@ public void setTaskResultsService(TaskResultsService taskResultsService) {
      * <p>
      * Returns the task manager tracked task or null if the task doesn't support the task manager
      */
-    public Task register(String type, String action, TransportRequest request) {
+    public Task register(String type, String action, TaskAwareRequest request) {
         Task task = request.createTask(taskIdGenerator.incrementAndGet(), type, action, request.getParentTask());
         if (task == null) {
             return null;
@@ -125,15 +121,18 @@ private void registerCancellableTask(Task task) {
     /**
      * Cancels a task
      * <p>
-     * Returns a set of nodes with child tasks where this task should be cancelled if cancellation was successful, null otherwise.
+     * Returns true if cancellation was started successful, null otherwise.
+     *
+     * After starting cancellation on the parent task, the task manager tries to cancel all children tasks
+     * of the current task. Once cancellation of the children tasks is done, the listener is triggered.
      */
-    public Set<String> cancel(CancellableTask task, String reason, Consumer<Set<String>> listener) {
+    public boolean cancel(CancellableTask task, String reason, Runnable listener) {
         CancellableTaskHolder holder = cancellableTasks.get(task.getId());
         if (holder != null) {
             logger.trace("cancelling task with id {}", task.getId());
             return holder.cancel(reason, listener);
         }
-        return null;
+        return false;
     }
 
     /**
@@ -314,7 +313,7 @@ public void removeBan(TaskId parentTaskId) {
     }
 
     @Override
-    public void clusterChanged(ClusterChangedEvent event) {
+    public void applyClusterState(ClusterChangedEvent event) {
         lastDiscoveryNodes = event.state().getNodes();
         if (event.nodesRemoved()) {
             synchronized (banedParents) {
@@ -344,17 +343,6 @@ public void clusterChanged(ClusterChangedEvent event) {
         }
     }
 
-    public void registerChildTask(Task task, String node) {
-        if (task == null || task instanceof CancellableTask == false) {
-            // We don't have a cancellable task - not much we can do here
-            return;
-        }
-        CancellableTaskHolder holder = cancellableTasks.get(task.getId());
-        if (holder != null) {
-            holder.registerChildTaskNode(node);
-        }
-    }
-
     /**
      * Blocks the calling thread, waiting for the task to vanish from the TaskManager.
      */
@@ -378,46 +366,44 @@ private static class CancellableTaskHolder {
 
         private final CancellableTask task;
 
-        private final Set<String> nodesWithChildTasks = new HashSet<>();
-
         private volatile String cancellationReason = null;
 
-        private volatile Consumer<Set<String>> cancellationListener = null;
+        private volatile Runnable cancellationListener = null;
 
-        public CancellableTaskHolder(CancellableTask task) {
+        CancellableTaskHolder(CancellableTask task) {
             this.task = task;
         }
 
         /**
          * Marks task as cancelled.
          * <p>
-         * Returns a set of nodes with child tasks where this task should be cancelled if cancellation was successful, null otherwise.
+         * Returns true if cancellation was successful, false otherwise.
          */
-        public Set<String> cancel(String reason, Consumer<Set<String>> listener) {
-            Set<String> nodes;
+        public boolean cancel(String reason, Runnable listener) {
+            final boolean cancelled;
             synchronized (this) {
                 assert reason != null;
                 if (cancellationReason == null) {
                     cancellationReason = reason;
                     cancellationListener = listener;
-                    nodes = Collections.unmodifiableSet(nodesWithChildTasks);
+                    cancelled = true;
                 } else {
                     // Already cancelled by somebody else
-                    nodes = null;
+                    cancelled = false;
                 }
             }
-            if (nodes != null) {
+            if (cancelled) {
                 task.cancel(reason);
             }
-            return nodes;
+            return cancelled;
         }
 
         /**
          * Marks task as cancelled.
          * <p>
-         * Returns a set of nodes with child tasks where this task should be cancelled if cancellation was successful, null otherwise.
+         * Returns true if cancellation was successful, false otherwise.
          */
-        public Set<String> cancel(String reason) {
+        public boolean cancel(String reason) {
             return cancel(reason, null);
         }
 
@@ -425,14 +411,12 @@ public Set<String> cancel(String reason) {
          * Marks task as finished.
          */
         public void finish() {
-            Consumer<Set<String>> listener = null;
-            Set<String> nodes = null;
+            Runnable listener = null;
             synchronized (this) {
                 if (cancellationReason != null) {
                     // The task was cancelled, we need to notify the listener
                     if (cancellationListener != null) {
                         listener = cancellationListener;
-                        nodes = Collections.unmodifiableSet(nodesWithChildTasks);
                         cancellationListener = null;
                     }
                 } else {
@@ -442,7 +426,7 @@ public void finish() {
             // We need to call the listener outside of the synchronised section to avoid potential bottle necks
             // in the listener synchronization
             if (listener != null) {
-                listener.accept(nodes);
+                listener.run();
             }
 
         }
@@ -454,14 +438,6 @@ public boolean hasParent(TaskId parentTaskId) {
         public CancellableTask getTask() {
             return task;
         }
-
-        public synchronized void registerChildTaskNode(String nodeId) {
-            if (cancellationReason == null) {
-                nodesWithChildTasks.add(nodeId);
-            } else {
-                throw new IllegalStateException("cannot register child task request, the task is already cancelled");
-            }
-        }
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/tasks/TaskResult.java b/core/src/main/java/org/elasticsearch/tasks/TaskResult.java
index b5918ad959308..ba80879b5df8f 100644
--- a/core/src/main/java/org/elasticsearch/tasks/TaskResult.java
+++ b/core/src/main/java/org/elasticsearch/tasks/TaskResult.java
@@ -22,7 +22,6 @@
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -161,7 +160,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
 
     public XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
         builder.field("completed", completed);
-        builder.field("task", task);
+        builder.startObject("task");
+        task.toXContent(builder, params);
+        builder.endObject();
         if (error != null) {
             XContentHelper.writeRawField("error", error, builder, params);
         }
@@ -171,7 +172,7 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t
         return builder;
     }
 
-    public static final ConstructingObjectParser<TaskResult, ParseFieldMatcherSupplier> PARSER = new ConstructingObjectParser<>(
+    public static final ConstructingObjectParser<TaskResult, Void> PARSER = new ConstructingObjectParser<>(
             "stored_task_result", a -> {
                 int i = 0;
                 boolean completed = (boolean) a[i++];
@@ -231,7 +232,7 @@ private static BytesReference toXContent(ToXContent result) throws IOException {
     private static BytesReference toXContent(Exception error) throws IOException {
         try (XContentBuilder builder = XContentFactory.contentBuilder(Requests.INDEX_CONTENT_TYPE)) {
             builder.startObject();
-            ElasticsearchException.toXContent(builder, ToXContent.EMPTY_PARAMS, error);
+            ElasticsearchException.generateThrowableXContent(builder, ToXContent.EMPTY_PARAMS, error);
             builder.endObject();
             return builder.bytes();
         }
diff --git a/core/src/main/java/org/elasticsearch/tasks/TaskResultsService.java b/core/src/main/java/org/elasticsearch/tasks/TaskResultsService.java
index fd515c5733aa2..58f623bd436ef 100644
--- a/core/src/main/java/org/elasticsearch/tasks/TaskResultsService.java
+++ b/core/src/main/java/org/elasticsearch/tasks/TaskResultsService.java
@@ -42,7 +42,7 @@
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.indices.IndexAlreadyExistsException;
+import org.elasticsearch.ResourceAlreadyExistsException;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
@@ -93,7 +93,7 @@ public void onResponse(CreateIndexResponse result) {
 
                 @Override
                 public void onFailure(Exception e) {
-                    if (ExceptionsHelper.unwrapCause(e) instanceof IndexAlreadyExistsException) {
+                    if (ExceptionsHelper.unwrapCause(e) instanceof ResourceAlreadyExistsException) {
                         // we have the index, do it
                         try {
                             doStoreResult(taskResult, listener);
diff --git a/core/src/main/java/org/elasticsearch/template/CompiledTemplate.java b/core/src/main/java/org/elasticsearch/template/CompiledTemplate.java
new file mode 100644
index 0000000000000..52f46202eb8ba
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/template/CompiledTemplate.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.template;
+
+import java.util.Map;
+
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.script.CompiledScript;
+import org.elasticsearch.script.ScriptType;
+
+/**
+ * A template that may be executed.
+ */
+public interface CompiledTemplate {
+
+    /** Run a template and return the resulting string, encoded in utf8 bytes. */
+    BytesReference run(Map<String, Object> params);
+}
diff --git a/core/src/main/java/org/elasticsearch/threadpool/ExecutorBuilder.java b/core/src/main/java/org/elasticsearch/threadpool/ExecutorBuilder.java
index 0f9b79156e61c..54f5ab0af38ba 100644
--- a/core/src/main/java/org/elasticsearch/threadpool/ExecutorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/threadpool/ExecutorBuilder.java
@@ -82,7 +82,7 @@ abstract static class ExecutorSettings {
 
         protected final String nodeName;
 
-        public ExecutorSettings(String nodeName) {
+        ExecutorSettings(String nodeName) {
             this.nodeName = nodeName;
         }
 
diff --git a/core/src/main/java/org/elasticsearch/threadpool/FixedExecutorBuilder.java b/core/src/main/java/org/elasticsearch/threadpool/FixedExecutorBuilder.java
index de7dbbaefc938..10cd21361b312 100644
--- a/core/src/main/java/org/elasticsearch/threadpool/FixedExecutorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/threadpool/FixedExecutorBuilder.java
@@ -30,7 +30,7 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.Locale;
-import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorService;
 import java.util.concurrent.ThreadFactory;
 
 /**
@@ -103,7 +103,7 @@ ThreadPool.ExecutorHolder build(final FixedExecutorSettings settings, final Thre
         int size = settings.size;
         int queueSize = settings.queueSize;
         final ThreadFactory threadFactory = EsExecutors.daemonThreadFactory(EsExecutors.threadName(settings.nodeName, name()));
-        Executor executor = EsExecutors.newFixed(name(), size, queueSize, threadFactory, threadContext);
+        final ExecutorService executor = EsExecutors.newFixed(name(), size, queueSize, threadFactory, threadContext);
         final ThreadPool.Info info =
             new ThreadPool.Info(name(), ThreadPool.ThreadPoolType.FIXED, size, size, null, queueSize < 0 ? null : new SizeValue(queueSize));
         return new ThreadPool.ExecutorHolder(executor, info);
@@ -124,7 +124,7 @@ static class FixedExecutorSettings extends ExecutorBuilder.ExecutorSettings {
         private final int size;
         private final int queueSize;
 
-        public FixedExecutorSettings(final String nodeName, final int size, final int queueSize) {
+        FixedExecutorSettings(final String nodeName, final int size, final int queueSize) {
             super(nodeName);
             this.size = size;
             this.queueSize = queueSize;
diff --git a/core/src/main/java/org/elasticsearch/threadpool/ScalingExecutorBuilder.java b/core/src/main/java/org/elasticsearch/threadpool/ScalingExecutorBuilder.java
index 6ab108e94b425..06df1fcf8b1c6 100644
--- a/core/src/main/java/org/elasticsearch/threadpool/ScalingExecutorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/threadpool/ScalingExecutorBuilder.java
@@ -29,7 +29,7 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.Locale;
-import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorService;
 import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.TimeUnit;
 
@@ -96,7 +96,7 @@ ThreadPool.ExecutorHolder build(final ScalingExecutorSettings settings, final Th
         int max = settings.max;
         final ThreadPool.Info info = new ThreadPool.Info(name(), ThreadPool.ThreadPoolType.SCALING, core, max, keepAlive, null);
         final ThreadFactory threadFactory = EsExecutors.daemonThreadFactory(EsExecutors.threadName(settings.nodeName, name()));
-        final Executor executor =
+        final ExecutorService executor =
             EsExecutors.newScaling(name(), core, max, keepAlive.millis(), TimeUnit.MILLISECONDS, threadFactory, threadContext);
         return new ThreadPool.ExecutorHolder(executor, info);
     }
@@ -118,7 +118,7 @@ static class ScalingExecutorSettings extends ExecutorBuilder.ExecutorSettings {
         private final int max;
         private final TimeValue keepAlive;
 
-        public ScalingExecutorSettings(final String nodeName, final int core, final int max, final TimeValue keepAlive) {
+        ScalingExecutorSettings(final String nodeName, final int core, final int max, final TimeValue keepAlive) {
             super(nodeName);
             this.core = core;
             this.max = max;
diff --git a/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java b/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
index d2ff4defc9ec5..128c78a4fd6ca 100644
--- a/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
+++ b/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
@@ -142,9 +142,9 @@ public static ThreadPoolType fromType(String type) {
 
     private final ScheduledThreadPoolExecutor scheduler;
 
-    private final EstimatedTimeThread estimatedTimeThread;
+    private final CachedTimeThread cachedTimeThread;
 
-    static final Executor DIRECT_EXECUTOR = command -> command.run();
+    static final ExecutorService DIRECT_EXECUTOR = EsExecutors.newDirectExecutorService();
 
     private final ThreadContext threadContext;
 
@@ -169,7 +169,7 @@ public ThreadPool(final Settings settings, final ExecutorBuilder<?>... customBui
         final int genericThreadPoolMax = boundedBy(4 * availableProcessors, 128, 512);
         builders.put(Names.GENERIC, new ScalingExecutorBuilder(Names.GENERIC, 4, genericThreadPoolMax, TimeValue.timeValueSeconds(30)));
         builders.put(Names.INDEX, new FixedExecutorBuilder(settings, Names.INDEX, availableProcessors, 200));
-        builders.put(Names.BULK, new FixedExecutorBuilder(settings, Names.BULK, availableProcessors, 50));
+        builders.put(Names.BULK, new FixedExecutorBuilder(settings, Names.BULK, availableProcessors, 200)); // now that we reuse bulk for index/delete ops
         builders.put(Names.GET, new FixedExecutorBuilder(settings, Names.GET, availableProcessors, 1000));
         builders.put(Names.SEARCH, new FixedExecutorBuilder(settings, Names.SEARCH, searchThreadPoolSize(availableProcessors), 1000));
         builders.put(Names.MANAGEMENT, new ScalingExecutorBuilder(Names.MANAGEMENT, 1, 5, TimeValue.timeValueMinutes(5)));
@@ -213,16 +213,33 @@ public ThreadPool(final Settings settings, final ExecutorBuilder<?>... customBui
         this.scheduler.setRemoveOnCancelPolicy(true);
 
         TimeValue estimatedTimeInterval = ESTIMATED_TIME_INTERVAL_SETTING.get(settings);
-        this.estimatedTimeThread = new EstimatedTimeThread(EsExecutors.threadName(settings, "[timer]"), estimatedTimeInterval.millis());
-        this.estimatedTimeThread.start();
+        this.cachedTimeThread = new CachedTimeThread(EsExecutors.threadName(settings, "[timer]"), estimatedTimeInterval.millis());
+        this.cachedTimeThread.start();
     }
 
-    public long estimatedTimeInMillis() {
-        return estimatedTimeThread.estimatedTimeInMillis();
+    /**
+     * Returns a value of milliseconds that may be used for relative time calculations.
+     *
+     * This method should only be used for calculating time deltas. For an epoch based
+     * timestamp, see {@link #absoluteTimeInMillis()}.
+     */
+    public long relativeTimeInMillis() {
+        return cachedTimeThread.relativeTimeInMillis();
+    }
+
+    /**
+     * Returns the value of milliseconds since UNIX epoch.
+     *
+     * This method should only be used for exact date/time formatting. For calculating
+     * time deltas that should not suffer from negative deltas, which are possible with
+     * this method, see {@link #relativeTimeInMillis()}.
+     */
+    public long absoluteTimeInMillis() {
+        return cachedTimeThread.absoluteTimeInMillis();
     }
 
     public Counter estimatedTimeInMillisCounter() {
-        return estimatedTimeThread.counter;
+        return cachedTimeThread.counter;
     }
 
     public ThreadPoolInfo info() {
@@ -278,23 +295,26 @@ public ThreadPoolStats stats() {
     }
 
     /**
-     * Get the generic executor. This executor's {@link Executor#execute(Runnable)} method will run the Runnable it is given in
-     * the {@link ThreadContext} of the thread that queues it.
+     * Get the generic executor service. This executor service {@link Executor#execute(Runnable)} method will run the {@link Runnable} it
+     * is given in the {@link ThreadContext} of the thread that queues it.
      */
-    public Executor generic() {
+    public ExecutorService generic() {
         return executor(Names.GENERIC);
     }
 
     /**
-     * Get the executor with the given name. This executor's {@link Executor#execute(Runnable)} method will run the Runnable it is given in
-     * the {@link ThreadContext} of the thread that queues it.
+     * Get the executor service with the given name. This executor service's {@link Executor#execute(Runnable)} method will run the
+     * {@link Runnable} it is given in the {@link ThreadContext} of the thread that queues it.
+     *
+     * @param name the name of the executor service to obtain
+     * @throws IllegalArgumentException if no executor service with the specified name exists
      */
-    public Executor executor(String name) {
-        Executor executor = executors.get(name).executor();
-        if (executor == null) {
-            throw new IllegalArgumentException("No executor found for [" + name + "]");
+    public ExecutorService executor(String name) {
+        final ExecutorHolder holder = executors.get(name);
+        if (holder == null) {
+            throw new IllegalArgumentException("no executor service found for [" + name + "]");
         }
-        return executor;
+        return holder.executor();
     }
 
     public ScheduledExecutorService scheduler() {
@@ -339,8 +359,8 @@ public ScheduledFuture<?> schedule(TimeValue delay, String executor, Runnable co
     }
 
     public void shutdown() {
-        estimatedTimeThread.running = false;
-        estimatedTimeThread.interrupt();
+        cachedTimeThread.running = false;
+        cachedTimeThread.interrupt();
         scheduler.shutdown();
         for (ExecutorHolder executor : executors.values()) {
             if (executor.executor() instanceof ThreadPoolExecutor) {
@@ -350,8 +370,8 @@ public void shutdown() {
     }
 
     public void shutdownNow() {
-        estimatedTimeThread.running = false;
-        estimatedTimeThread.interrupt();
+        cachedTimeThread.running = false;
+        cachedTimeThread.interrupt();
         scheduler.shutdownNow();
         for (ExecutorHolder executor : executors.values()) {
             if (executor.executor() instanceof ThreadPoolExecutor) {
@@ -368,7 +388,7 @@ public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedE
             }
         }
 
-        estimatedTimeThread.join(unit.toMillis(timeout));
+        cachedTimeThread.join(unit.toMillis(timeout));
         return result;
     }
 
@@ -468,29 +488,50 @@ public String toString() {
         }
     }
 
-    static class EstimatedTimeThread extends Thread {
+    /**
+     * A thread to cache millisecond time values from
+     * {@link System#nanoTime()} and {@link System#currentTimeMillis()}.
+     *
+     * The values are updated at a specified interval.
+     */
+    static class CachedTimeThread extends Thread {
 
         final long interval;
         final TimeCounter counter;
         volatile boolean running = true;
-        volatile long estimatedTimeInMillis;
+        volatile long relativeMillis;
+        volatile long absoluteMillis;
 
-        EstimatedTimeThread(String name, long interval) {
+        CachedTimeThread(String name, long interval) {
             super(name);
             this.interval = interval;
-            this.estimatedTimeInMillis = TimeValue.nsecToMSec(System.nanoTime());
+            this.relativeMillis = TimeValue.nsecToMSec(System.nanoTime());
+            this.absoluteMillis = System.currentTimeMillis();
             this.counter = new TimeCounter();
             setDaemon(true);
         }
 
-        public long estimatedTimeInMillis() {
-            return this.estimatedTimeInMillis;
+        /**
+         * Return the current time used for relative calculations. This is
+         * {@link System#nanoTime()} truncated to milliseconds.
+         */
+        long relativeTimeInMillis() {
+            return relativeMillis;
+        }
+
+        /**
+         * Return the current epoch time, used to find absolute time. This is
+         * a cached version of {@link System#currentTimeMillis()}.
+         */
+        long absoluteTimeInMillis() {
+            return absoluteMillis;
         }
 
         @Override
         public void run() {
             while (running) {
-                estimatedTimeInMillis = TimeValue.nsecToMSec(System.nanoTime());
+                relativeMillis = TimeValue.nsecToMSec(System.nanoTime());
+                absoluteMillis = System.currentTimeMillis();
                 try {
                     Thread.sleep(interval);
                 } catch (InterruptedException e) {
@@ -509,22 +550,22 @@ public long addAndGet(long delta) {
 
             @Override
             public long get() {
-                return estimatedTimeInMillis;
+                return relativeMillis;
             }
         }
     }
 
     static class ExecutorHolder {
-        private final Executor executor;
+        private final ExecutorService executor;
         public final Info info;
 
-        ExecutorHolder(Executor executor, Info info) {
+        ExecutorHolder(ExecutorService executor, Info info) {
             assert executor instanceof EsThreadPoolExecutor || executor == DIRECT_EXECUTOR;
             this.executor = executor;
             this.info = info;
         }
 
-        Executor executor() {
+        ExecutorService executor() {
             return executor;
         }
     }
@@ -638,14 +679,23 @@ static final class Fields {
     public static boolean terminate(ExecutorService service, long timeout, TimeUnit timeUnit) {
         if (service != null) {
             service.shutdown();
-            try {
-                if (service.awaitTermination(timeout, timeUnit)) {
-                    return true;
-                }
-            } catch (InterruptedException e) {
-                Thread.currentThread().interrupt();
-            }
+            if (awaitTermination(service, timeout, timeUnit)) return true;
             service.shutdownNow();
+            return awaitTermination(service, timeout, timeUnit);
+        }
+        return false;
+    }
+
+    private static boolean awaitTermination(
+            final ExecutorService service,
+            final long timeout,
+            final TimeUnit timeUnit) {
+        try {
+            if (service.awaitTermination(timeout, timeUnit)) {
+                return true;
+            }
+        } catch (final InterruptedException e) {
+            Thread.currentThread().interrupt();
         }
         return false;
     }
@@ -658,15 +708,10 @@ public static boolean terminate(ThreadPool pool, long timeout, TimeUnit timeUnit
         if (pool != null) {
             try {
                 pool.shutdown();
-                try {
-                    if (pool.awaitTermination(timeout, timeUnit)) {
-                        return true;
-                    }
-                } catch (InterruptedException e) {
-                    Thread.currentThread().interrupt();
-                }
+                if (awaitTermination(pool, timeout, timeUnit)) return true;
                 // last resort
                 pool.shutdownNow();
+                return awaitTermination(pool, timeout, timeUnit);
             } finally {
                 IOUtils.closeWhileHandlingException(pool);
             }
@@ -674,6 +719,20 @@ public static boolean terminate(ThreadPool pool, long timeout, TimeUnit timeUnit
         return false;
     }
 
+    private static boolean awaitTermination(
+            final ThreadPool pool,
+            final long timeout,
+            final TimeUnit timeUnit) {
+        try {
+            if (pool.awaitTermination(timeout, timeUnit)) {
+                return true;
+            }
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+        }
+        return false;
+    }
+
     @Override
     public void close() throws IOException {
         threadContext.close();
diff --git a/core/src/main/java/org/elasticsearch/transport/ConnectionProfile.java b/core/src/main/java/org/elasticsearch/transport/ConnectionProfile.java
new file mode 100644
index 0000000000000..095b7d2d6ed24
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/transport/ConnectionProfile.java
@@ -0,0 +1,230 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.transport;
+
+import org.elasticsearch.common.inject.internal.Nullable;
+import org.elasticsearch.common.unit.TimeValue;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * A connection profile describes how many connection are established to specific node for each of the available request types.
+ * ({@link org.elasticsearch.transport.TransportRequestOptions.Type}). This allows to tailor a connection towards a specific usage.
+ */
+public final class ConnectionProfile {
+
+    /**
+     * Builds a connection profile that is dedicated to a single channel type. Use this
+     * when opening single use connections
+     */
+    public static ConnectionProfile buildSingleChannelProfile(TransportRequestOptions.Type channelType,
+                                                              @Nullable TimeValue connectTimeout,
+                                                               @Nullable TimeValue handshakeTimeout) {
+        Builder builder = new Builder();
+        builder.addConnections(1, channelType);
+        final EnumSet<TransportRequestOptions.Type> otherTypes = EnumSet.allOf(TransportRequestOptions.Type.class);
+        otherTypes.remove(channelType);
+        builder.addConnections(0, otherTypes.stream().toArray(TransportRequestOptions.Type[]::new));
+        if (connectTimeout != null) {
+            builder.setConnectTimeout(connectTimeout);
+        }
+        if (handshakeTimeout != null) {
+            builder.setHandshakeTimeout(handshakeTimeout);
+        }
+        return builder.build();
+    }
+
+    private final List<ConnectionTypeHandle> handles;
+    private final int numConnections;
+    private final TimeValue connectTimeout;
+    private final TimeValue handshakeTimeout;
+
+    private ConnectionProfile(List<ConnectionTypeHandle> handles, int numConnections, TimeValue connectTimeout, TimeValue handshakeTimeout)
+    {
+        this.handles = handles;
+        this.numConnections = numConnections;
+        this.connectTimeout = connectTimeout;
+        this.handshakeTimeout = handshakeTimeout;
+    }
+
+    /**
+     * A builder to build a new {@link ConnectionProfile}
+     */
+    public static class Builder {
+        private final List<ConnectionTypeHandle> handles = new ArrayList<>();
+        private final Set<TransportRequestOptions.Type> addedTypes = EnumSet.noneOf(TransportRequestOptions.Type.class);
+        private int offset = 0;
+        private TimeValue connectTimeout;
+        private TimeValue handshakeTimeout;
+
+        /** create an empty builder */
+        public Builder() {
+        }
+
+        /** copy constructor, using another profile as a base */
+        public Builder(ConnectionProfile source) {
+            handles.addAll(source.getHandles());
+            offset = source.getNumConnections();
+            handles.forEach(th -> addedTypes.addAll(th.types));
+            connectTimeout = source.getConnectTimeout();
+            handshakeTimeout = source.getHandshakeTimeout();
+        }
+        /**
+         * Sets a connect timeout for this connection profile
+         */
+        public Builder setConnectTimeout(TimeValue connectTimeout) {
+            if (connectTimeout.millis() < 0) {
+                throw new IllegalArgumentException("connectTimeout must be non-negative but was: " + connectTimeout);
+            }
+            this.connectTimeout = connectTimeout;
+            return this;
+        }
+
+        /**
+         * Sets a handshake timeout for this connection profile
+         */
+        public Builder setHandshakeTimeout(TimeValue handshakeTimeout) {
+            if (handshakeTimeout.millis() < 0) {
+                throw new IllegalArgumentException("handshakeTimeout must be non-negative but was: " + handshakeTimeout);
+            }
+            this.handshakeTimeout = handshakeTimeout;
+            return this;
+        }
+
+        /**
+         * Adds a number of connections for one or more types. Each type can only be added once.
+         * @param numConnections the number of connections to use in the pool for the given connection types
+         * @param types a set of types that should share the given number of connections
+         */
+        public Builder addConnections(int numConnections, TransportRequestOptions.Type... types) {
+            if (types == null || types.length == 0) {
+                throw new IllegalArgumentException("types must not be null");
+            }
+            for (TransportRequestOptions.Type type : types) {
+                if (addedTypes.contains(type)) {
+                    throw new IllegalArgumentException("type [" + type + "] is already registered");
+                }
+            }
+            addedTypes.addAll(Arrays.asList(types));
+            handles.add(new ConnectionTypeHandle(offset, numConnections, EnumSet.copyOf(Arrays.asList(types))));
+            offset += numConnections;
+            return this;
+        }
+
+        /**
+         * Creates a new {@link ConnectionProfile} based on the added connections.
+         * @throws IllegalStateException if any of the {@link org.elasticsearch.transport.TransportRequestOptions.Type} enum is missing
+         */
+        public ConnectionProfile build() {
+            EnumSet<TransportRequestOptions.Type> types = EnumSet.allOf(TransportRequestOptions.Type.class);
+            types.removeAll(addedTypes);
+            if (types.isEmpty() == false) {
+                throw new IllegalStateException("not all types are added for this connection profile - missing types: " + types);
+            }
+            return new ConnectionProfile(Collections.unmodifiableList(handles), offset, connectTimeout, handshakeTimeout);
+        }
+
+    }
+
+    /**
+     * Returns the connect timeout or <code>null</code> if no explicit timeout is set on this profile.
+     */
+    public TimeValue getConnectTimeout() {
+        return connectTimeout;
+    }
+
+    /**
+     * Returns the handshake timeout or <code>null</code> if no explicit timeout is set on this profile.
+     */
+    public TimeValue getHandshakeTimeout() {
+        return handshakeTimeout;
+    }
+
+    /**
+     * Returns the total number of connections for this profile
+     */
+    public int getNumConnections() {
+        return numConnections;
+    }
+
+    /**
+     * Returns the number of connections per type for this profile. This might return a count that is shared with other types such
+     * that the sum of all connections per type might be higher than {@link #getNumConnections()}. For instance if
+     * {@link org.elasticsearch.transport.TransportRequestOptions.Type#BULK} shares connections with
+     * {@link org.elasticsearch.transport.TransportRequestOptions.Type#REG} they will return both the same number of connections from
+     * this method but the connections are not distinct.
+     */
+    public int getNumConnectionsPerType(TransportRequestOptions.Type type) {
+        for (ConnectionTypeHandle handle : handles) {
+            if (handle.getTypes().contains(type)) {
+                return handle.length;
+            }
+        }
+        throw new AssertionError("no handle found for type: "  + type);
+    }
+
+    /**
+     * Returns the type handles for this connection profile
+     */
+    List<ConnectionTypeHandle> getHandles() {
+        return Collections.unmodifiableList(handles);
+    }
+
+    /**
+     * Connection type handle encapsulates the logic which connection
+     */
+    static final class ConnectionTypeHandle {
+        public final int length;
+        public final int offset;
+        private final Set<TransportRequestOptions.Type> types;
+        private final AtomicInteger counter = new AtomicInteger();
+
+        private ConnectionTypeHandle(int offset, int length, Set<TransportRequestOptions.Type> types) {
+            this.length = length;
+            this.offset = offset;
+            this.types = types;
+        }
+
+        /**
+         * Returns one of the channels out configured for this handle. The channel is selected in a round-robin
+         * fashion.
+         */
+        <T> T getChannel(T[] channels) {
+            if (length == 0) {
+                throw new IllegalStateException("can't select channel size is 0 for types: " + types);
+            }
+            assert channels.length >= offset + length : "illegal size: " + channels.length + " expected >= " + (offset + length);
+            return channels[offset + Math.floorMod(counter.incrementAndGet(), length)];
+        }
+
+        /**
+         * Returns all types for this handle
+         */
+        Set<TransportRequestOptions.Type> getTypes() {
+            return types;
+        }
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/transport/DelegatingTransportChannel.java b/core/src/main/java/org/elasticsearch/transport/DelegatingTransportChannel.java
index af85b438618be..bc3b4b3c2ab07 100644
--- a/core/src/main/java/org/elasticsearch/transport/DelegatingTransportChannel.java
+++ b/core/src/main/java/org/elasticsearch/transport/DelegatingTransportChannel.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.transport;
 
+import org.elasticsearch.Version;
+
 import java.io.IOException;
 
 /**
@@ -71,4 +73,9 @@ public void sendResponse(Exception exception) throws IOException {
     public TransportChannel getChannel() {
         return channel;
     }
+
+    @Override
+    public Version getVersion() {
+        return channel.getVersion();
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java b/core/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java
new file mode 100644
index 0000000000000..84aff5814c765
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.transport;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.ClusterNameExpressionResolver;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.component.AbstractComponent;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.elasticsearch.common.transport.TransportAddress;
+
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * Base class for all services and components that need up-to-date information about the registered remote clusters
+ */
+public abstract class RemoteClusterAware extends AbstractComponent {
+
+    /**
+     * A list of initial seed nodes to discover eligible nodes from the remote cluster
+     */
+    public static final Setting.AffixSetting<List<InetSocketAddress>> REMOTE_CLUSTERS_SEEDS = Setting.affixKeySetting("search.remote.",
+        "seeds", (key) -> Setting.listSetting(key, Collections.emptyList(), RemoteClusterAware::parseSeedAddress,
+            Setting.Property.NodeScope, Setting.Property.Dynamic));
+    public static final char REMOTE_CLUSTER_INDEX_SEPARATOR = ':';
+    public static final String LOCAL_CLUSTER_GROUP_KEY = "";
+    protected final ClusterNameExpressionResolver clusterNameResolver;
+
+    /**
+     * Creates a new {@link RemoteClusterAware} instance
+     * @param settings the nodes level settings
+     */
+    protected RemoteClusterAware(Settings settings) {
+        super(settings);
+        this.clusterNameResolver = new ClusterNameExpressionResolver(settings);
+    }
+
+    protected static Map<String, List<DiscoveryNode>> buildRemoteClustersSeeds(Settings settings) {
+        Stream<Setting<List<InetSocketAddress>>> allConcreteSettings = REMOTE_CLUSTERS_SEEDS.getAllConcreteSettings(settings);
+        return allConcreteSettings.collect(
+            Collectors.toMap(REMOTE_CLUSTERS_SEEDS::getNamespace, concreteSetting -> {
+                String clusterName = REMOTE_CLUSTERS_SEEDS.getNamespace(concreteSetting);
+                List<DiscoveryNode> nodes = new ArrayList<>();
+                for (InetSocketAddress address : concreteSetting.get(settings)) {
+                    TransportAddress transportAddress = new InetSocketTransportAddress(address);
+                    DiscoveryNode node = new DiscoveryNode(clusterName + "#" + transportAddress.toString(),
+                        transportAddress,
+                        Version.CURRENT.minimumCompatibilityVersion());
+                    nodes.add(node);
+                }
+                return nodes;
+            }));
+    }
+
+    /**
+     * Groups indices per cluster by splitting remote cluster-alias, index-name pairs on {@link #REMOTE_CLUSTER_INDEX_SEPARATOR}. All
+     * indices per cluster are collected as a list in the returned map keyed by the cluster alias. Local indices are grouped under
+     * {@link #LOCAL_CLUSTER_GROUP_KEY}. The returned map is mutable.
+     *
+     * @param requestIndices the indices in the search request to filter
+     * @param indexExists a predicate that can test if a certain index or alias exists in the local cluster
+     *
+     * @return a map of grouped remote and local indices
+     */
+    public Map<String, List<String>> groupClusterIndices(String[] requestIndices, Predicate<String> indexExists) {
+        Map<String, List<String>> perClusterIndices = new HashMap<>();
+        Set<String> remoteClusterNames = getRemoteClusterNames();
+        for (String index : requestIndices) {
+            int i = index.indexOf(RemoteClusterService.REMOTE_CLUSTER_INDEX_SEPARATOR);
+            if (i >= 0) {
+                String remoteClusterName = index.substring(0, i);
+                List<String> clusters = clusterNameResolver.resolveClusterNames(remoteClusterNames, remoteClusterName);
+                if (clusters.isEmpty() == false) {
+                    if (indexExists.test(index)) {
+                        // we use : as a separator for remote clusters. might conflict if there is an index that is actually named
+                        // remote_cluster_alias:index_name - for this case we fail the request. the user can easily change the cluster alias
+                        // if that happens
+                        throw new IllegalArgumentException("Can not filter indices; index " + index +
+                            " exists but there is also a remote cluster named: " + remoteClusterName);
+                        }
+                    String indexName = index.substring(i + 1);
+                    for (String clusterName : clusters) {
+                        perClusterIndices.computeIfAbsent(clusterName, k -> new ArrayList<>()).add(indexName);
+                    }
+                } else {
+                    perClusterIndices.computeIfAbsent(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY, k -> new ArrayList<>()).add(index);
+                }
+            } else {
+                perClusterIndices.computeIfAbsent(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY, k -> new ArrayList<>()).add(index);
+            }
+        }
+        return perClusterIndices;
+    }
+
+    protected abstract Set<String> getRemoteClusterNames();
+
+    /**
+     * Subclasses must implement this to receive information about updated cluster aliases. If the given address list is
+     * empty the cluster alias is unregistered and should be removed.
+     */
+    protected abstract void updateRemoteCluster(String clusterAlias, List<InetSocketAddress> addresses);
+
+    /**
+     * Registers this instance to listen to updates on the cluster settings.
+     */
+    public void listenForUpdates(ClusterSettings clusterSettings) {
+        clusterSettings.addAffixUpdateConsumer(RemoteClusterAware.REMOTE_CLUSTERS_SEEDS, this::updateRemoteCluster,
+            (namespace, value) -> {});
+    }
+
+    private static InetSocketAddress parseSeedAddress(String remoteHost) {
+        int portSeparator = remoteHost.lastIndexOf(':'); // in case we have a IPv6 address ie. [::1]:9300
+        if (portSeparator == -1 || portSeparator == remoteHost.length()) {
+            throw new IllegalArgumentException("remote hosts need to be configured as [host:port], found [" + remoteHost + "] instead");
+        }
+        String host = remoteHost.substring(0, portSeparator);
+        InetAddress hostAddress;
+        try {
+            hostAddress = InetAddress.getByName(host);
+        } catch (UnknownHostException e) {
+            throw new IllegalArgumentException("unknown host [" + host + "]", e);
+        }
+        try {
+            int port = Integer.valueOf(remoteHost.substring(portSeparator + 1));
+            if (port <= 0) {
+                throw new IllegalArgumentException("port number must be > 0 but was: [" + port + "]");
+            }
+            return new InetSocketAddress(hostAddress, port);
+        } catch (NumberFormatException e) {
+            throw new IllegalArgumentException("port must be a number", e);
+        }
+    }
+
+    public static final String buildRemoteIndexName(String clusterAlias, String indexName) {
+        return clusterAlias != null ? clusterAlias + REMOTE_CLUSTER_INDEX_SEPARATOR + indexName : indexName;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java b/core/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java
new file mode 100644
index 0000000000000..c08bf9b737e95
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java
@@ -0,0 +1,724 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.transport;
+
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.apache.logging.log4j.util.Supplier;
+import org.apache.lucene.store.AlreadyClosedException;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
+import org.elasticsearch.action.admin.cluster.node.info.NodesInfoAction;
+import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest;
+import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
+import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsAction;
+import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsRequest;
+import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsResponse;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.component.AbstractComponent;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.util.CancellableThreads;
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.threadpool.ThreadPool;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Function;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+
+/**
+ * Represents a connection to a single remote cluster. In contrast to a local cluster a remote cluster is not joined such that the
+ * current node is part of the cluster and it won't receive cluster state updates from the remote cluster. Remote clusters are also not
+ * fully connected with the current node. From a connection perspective a local cluster forms a bi-directional star network while in the
+ * remote case we only connect to a subset of the nodes in the cluster in an uni-directional fashion.
+ *
+ * This class also handles the discovery of nodes from the remote cluster. The initial list of seed nodes is only used to discover all nodes
+ * in the remote cluster and connects to all eligible nodes, for details see {@link RemoteClusterService#REMOTE_NODE_ATTRIBUTE}.
+ *
+ * In the case of a disconnection, this class will issue a re-connect task to establish at most
+ * {@link RemoteClusterService#REMOTE_CONNECTIONS_PER_CLUSTER} until either all eligible nodes are exhausted or the maximum number of
+ * connections per cluster has been reached.
+ */
+final class RemoteClusterConnection extends AbstractComponent implements TransportConnectionListener, Closeable {
+
+    private final TransportService transportService;
+    private final ConnectionProfile remoteProfile;
+    private final ConnectedNodes connectedNodes;
+    private final String clusterAlias;
+    private final int maxNumRemoteConnections;
+    private final Predicate<DiscoveryNode> nodePredicate;
+    private volatile List<DiscoveryNode> seedNodes;
+    private final ConnectHandler connectHandler;
+    private SetOnce<ClusterName> remoteClusterName = new SetOnce<>();
+
+    /**
+     * Creates a new {@link RemoteClusterConnection}
+     * @param settings the nodes settings object
+     * @param clusterAlias the configured alias of the cluster to connect to
+     * @param seedNodes a list of seed nodes to discover eligible nodes from
+     * @param transportService the local nodes transport service
+     * @param maxNumRemoteConnections the maximum number of connections to the remote cluster
+     * @param nodePredicate a predicate to filter eligible remote nodes to connect to
+     */
+    RemoteClusterConnection(Settings settings, String clusterAlias, List<DiscoveryNode> seedNodes,
+                            TransportService transportService, int maxNumRemoteConnections, Predicate<DiscoveryNode> nodePredicate) {
+        super(settings);
+        this.transportService = transportService;
+        this.maxNumRemoteConnections = maxNumRemoteConnections;
+        this.nodePredicate = nodePredicate;
+        this.clusterAlias = clusterAlias;
+        ConnectionProfile.Builder builder = new ConnectionProfile.Builder();
+        builder.setConnectTimeout(TcpTransport.TCP_CONNECT_TIMEOUT.get(settings));
+        builder.setHandshakeTimeout(TcpTransport.TCP_CONNECT_TIMEOUT.get(settings));
+        builder.addConnections(6, TransportRequestOptions.Type.REG, TransportRequestOptions.Type.PING); // TODO make this configurable?
+        builder.addConnections(0, // we don't want this to be used for anything else but search
+            TransportRequestOptions.Type.BULK,
+            TransportRequestOptions.Type.STATE,
+            TransportRequestOptions.Type.RECOVERY);
+        remoteProfile = builder.build();
+        connectedNodes = new ConnectedNodes(clusterAlias);
+        this.seedNodes = Collections.unmodifiableList(seedNodes);
+        this.connectHandler = new ConnectHandler();
+        transportService.addConnectionListener(this);
+    }
+
+    /**
+     * Updates the list of seed nodes for this cluster connection
+     */
+    synchronized void updateSeedNodes(List<DiscoveryNode> seedNodes, ActionListener<Void> connectListener) {
+        this.seedNodes = Collections.unmodifiableList(new ArrayList<>(seedNodes));
+        connectHandler.connect(connectListener);
+    }
+
+    @Override
+    public void onNodeDisconnected(DiscoveryNode node) {
+        boolean remove = connectedNodes.remove(node);
+        if (remove && connectedNodes.size() < maxNumRemoteConnections) {
+            // try to reconnect and fill up the slot of the disconnected node
+            connectHandler.forceConnect();
+        }
+    }
+
+    /**
+     * Fetches all shards for the search request from this remote connection. This is used to later run the search on the remote end.
+     */
+    public void fetchSearchShards(ClusterSearchShardsRequest searchRequest,
+                                  ActionListener<ClusterSearchShardsResponse> listener) {
+        if (connectedNodes.size() == 0) {
+            // just in case if we are not connected for some reason we try to connect and if we fail we have to notify the listener
+            // this will cause some back pressure on the search end and eventually will cause rejections but that's fine
+            // we can't proceed with a search on a cluster level.
+            // in the future we might want to just skip the remote nodes in such a case but that can already be implemented on the caller
+            // end since they provide the listener.
+            ensureConnected(ActionListener.wrap((x) -> fetchShardsInternal(searchRequest, listener), listener::onFailure));
+        } else {
+            fetchShardsInternal(searchRequest, listener);
+        }
+    }
+
+    /**
+     * Ensures that this cluster is connected. If the cluster is connected this operation
+     * will invoke the listener immediately.
+     */
+    public void ensureConnected(ActionListener<Void> voidActionListener) {
+        if (connectedNodes.size() == 0) {
+            connectHandler.connect(voidActionListener);
+        } else {
+            voidActionListener.onResponse(null);
+        }
+    }
+
+    private void fetchShardsInternal(ClusterSearchShardsRequest searchShardsRequest,
+                                     final ActionListener<ClusterSearchShardsResponse> listener) {
+        final DiscoveryNode node = connectedNodes.get();
+        transportService.sendRequest(node, ClusterSearchShardsAction.NAME, searchShardsRequest,
+            new TransportResponseHandler<ClusterSearchShardsResponse>() {
+
+                @Override
+                public ClusterSearchShardsResponse newInstance() {
+                    return new ClusterSearchShardsResponse();
+                }
+
+                @Override
+                public void handleResponse(ClusterSearchShardsResponse clusterSearchShardsResponse) {
+                    listener.onResponse(clusterSearchShardsResponse);
+                }
+
+                @Override
+                public void handleException(TransportException e) {
+                    listener.onFailure(e);
+                }
+
+                @Override
+                public String executor() {
+                    return ThreadPool.Names.SEARCH;
+                }
+            });
+    }
+
+    /**
+     * Collects all nodes on the connected cluster and returns / passes a nodeID to {@link DiscoveryNode} lookup function
+     * that returns <code>null</code> if the node ID is not found.
+     */
+    void collectNodes(ActionListener<Function<String, DiscoveryNode>> listener) {
+        Runnable runnable = () -> {
+            final ClusterStateRequest request = new ClusterStateRequest();
+            request.clear();
+            request.nodes(true);
+            request.local(true); // run this on the node that gets the request it's as good as any other
+            final DiscoveryNode node = connectedNodes.get();
+            transportService.sendRequest(node, ClusterStateAction.NAME, request, TransportRequestOptions.EMPTY,
+                new TransportResponseHandler<ClusterStateResponse>() {
+                    @Override
+                    public ClusterStateResponse newInstance() {
+                        return new ClusterStateResponse();
+                    }
+
+                    @Override
+                    public void handleResponse(ClusterStateResponse response) {
+                        DiscoveryNodes nodes = response.getState().nodes();
+                        listener.onResponse(nodes::get);
+                    }
+
+                    @Override
+                    public void handleException(TransportException exp) {
+                        listener.onFailure(exp);
+                    }
+
+                    @Override
+                    public String executor() {
+                        return ThreadPool.Names.SAME;
+                    }
+                });
+        };
+        try {
+            if (connectedNodes.size() == 0) {
+                // just in case if we are not connected for some reason we try to connect and if we fail we have to notify the listener
+                // this will cause some back pressure on the search end and eventually will cause rejections but that's fine
+                // we can't proceed with a search on a cluster level.
+                // in the future we might want to just skip the remote nodes in such a case but that can already be implemented on the
+                // caller end since they provide the listener.
+                ensureConnected(ActionListener.wrap((x) -> runnable.run(), listener::onFailure));
+            } else {
+                runnable.run();
+            }
+        } catch (Exception ex) {
+            listener.onFailure(ex);
+        }
+    }
+
+    /**
+     * Returns a connection to the remote cluster. This connection might be a proxy connection that redirects internally to the
+     * given node.
+     */
+    Transport.Connection getConnection(DiscoveryNode remoteClusterNode) {
+        DiscoveryNode discoveryNode = connectedNodes.get();
+        Transport.Connection connection = transportService.getConnection(discoveryNode);
+        return new Transport.Connection() {
+            @Override
+            public DiscoveryNode getNode() {
+                return remoteClusterNode;
+            }
+
+            @Override
+            public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options)
+                throws IOException, TransportException {
+                connection.sendRequest(requestId, TransportActionProxy.getProxyAction(action),
+                    TransportActionProxy.wrapRequest(remoteClusterNode, request), options);
+            }
+
+            @Override
+            public void close() throws IOException {
+                assert false: "proxy connections must not be closed";
+            }
+        };
+    }
+
+    Transport.Connection getConnection() {
+        DiscoveryNode discoveryNode = connectedNodes.get();
+        return transportService.getConnection(discoveryNode);
+    }
+
+    @Override
+    public void close() throws IOException {
+        connectHandler.close();
+    }
+
+    public boolean isClosed() {
+        return connectHandler.isClosed();
+    }
+
+    /**
+     * The connect handler manages node discovery and the actual connect to the remote cluster.
+     * There is at most one connect job running at any time. If such a connect job is triggered
+     * while another job is running the provided listeners are queued and batched up until the current running job returns.
+     *
+     * The handler has a built-in queue that can hold up to 100 connect attempts and will reject requests once the queue is full.
+     * In a scenario when a remote cluster becomes unavailable we will queue requests up but if we can't connect quick enough
+     * we will just reject the connect trigger which will lead to failing searches.
+     */
+    private class ConnectHandler implements Closeable {
+        private final Semaphore running = new Semaphore(1);
+        private final AtomicBoolean closed = new AtomicBoolean(false);
+        private final BlockingQueue<ActionListener<Void>> queue = new ArrayBlockingQueue<>(100);
+        private final CancellableThreads cancellableThreads = new CancellableThreads();
+
+        /**
+         * Triggers a connect round iff there are pending requests queued up and if there is no
+         * connect round currently running.
+         */
+        void maybeConnect() {
+            connect(null);
+        }
+
+        /**
+         * Triggers a connect round unless there is one running already. If there is a connect round running, the listener will either
+         * be queued or rejected and failed.
+         */
+        void connect(ActionListener<Void> connectListener) {
+            connect(connectListener, false);
+        }
+
+        /**
+         * Triggers a connect round unless there is one already running. In contrast to {@link #maybeConnect()} will this method also
+         * trigger a connect round if there is no listener queued up.
+         */
+        void forceConnect() {
+            connect(null, true);
+        }
+
+        private void connect(ActionListener<Void> connectListener, boolean forceRun) {
+            final boolean runConnect;
+            final Collection<ActionListener<Void>> toNotify;
+            synchronized (queue) {
+                if (connectListener != null && queue.offer(connectListener) == false) {
+                    connectListener.onFailure(new RejectedExecutionException("connect queue is full"));
+                    return;
+                }
+                if (forceRun == false && queue.isEmpty()) {
+                    return;
+                }
+                runConnect = running.tryAcquire();
+                if (runConnect) {
+                    toNotify = new ArrayList<>();
+                    queue.drainTo(toNotify);
+                    if (closed.get()) {
+                        running.release();
+                        ActionListener.onFailure(toNotify, new AlreadyClosedException("connect handler is already closed"));
+                        return;
+                    }
+                } else {
+                    toNotify = Collections.emptyList();
+                }
+            }
+            if (runConnect) {
+                forkConnect(toNotify);
+            }
+        }
+
+        private void forkConnect(final Collection<ActionListener<Void>> toNotify) {
+            ThreadPool threadPool = transportService.getThreadPool();
+            ExecutorService executor = threadPool.executor(ThreadPool.Names.MANAGEMENT);
+            executor.submit(new AbstractRunnable() {
+                @Override
+                public void onFailure(Exception e) {
+                    synchronized (queue) {
+                        running.release();
+                    }
+                    try {
+                        ActionListener.onFailure(toNotify, e);
+                    } finally {
+                        maybeConnect();
+                    }
+                }
+
+                @Override
+                protected void doRun() throws Exception {
+                    ActionListener<Void> listener = ActionListener.wrap((x) -> {
+                        synchronized (queue) {
+                            running.release();
+                        }
+                        try {
+                            ActionListener.onResponse(toNotify, x);
+                        } finally {
+                            maybeConnect();
+                        }
+
+                    }, (e) -> {
+                        synchronized (queue) {
+                            running.release();
+                        }
+                        try {
+                            ActionListener.onFailure(toNotify, e);
+                        } finally {
+                            maybeConnect();
+                        }
+                    });
+                    collectRemoteNodes(seedNodes.iterator(), transportService, listener);
+                }
+            });
+
+        }
+
+        void collectRemoteNodes(Iterator<DiscoveryNode> seedNodes,
+                                final TransportService transportService, ActionListener<Void> listener) {
+            if (Thread.currentThread().isInterrupted()) {
+                listener.onFailure(new InterruptedException("remote connect thread got interrupted"));
+            }
+            try {
+                if (seedNodes.hasNext()) {
+                    cancellableThreads.executeIO(() -> {
+                        final DiscoveryNode seedNode = seedNodes.next();
+                        final DiscoveryNode handshakeNode;
+                        Transport.Connection connection = transportService.openConnection(seedNode,
+                            ConnectionProfile.buildSingleChannelProfile(TransportRequestOptions.Type.REG, null, null));
+                        boolean success = false;
+                        try {
+                            try {
+                                handshakeNode = transportService.handshake(connection, remoteProfile.getHandshakeTimeout().millis(),
+                                    (c) -> remoteClusterName.get() == null ? true : c.equals(remoteClusterName.get()));
+                            } catch (IllegalStateException ex) {
+                                logger.warn((Supplier<?>) () -> new ParameterizedMessage("seed node {} cluster name mismatch expected " +
+                                    "cluster name {}", connection.getNode(), remoteClusterName.get()), ex);
+                                throw ex;
+                            }
+                            if (nodePredicate.test(handshakeNode) && connectedNodes.size() < maxNumRemoteConnections) {
+                                transportService.connectToNode(handshakeNode, remoteProfile);
+                                connectedNodes.add(handshakeNode);
+                            }
+                            ClusterStateRequest request = new ClusterStateRequest();
+                            request.clear();
+                            request.nodes(true);
+                            // here we pass on the connection since we can only close it once the sendRequest returns otherwise
+                            // due to the async nature (it will return before it's actually sent) this can cause the request to fail
+                            // due to an already closed connection.
+                            ThreadPool threadPool = transportService.getThreadPool();
+                            ThreadContext threadContext = threadPool.getThreadContext();
+                            TransportService.ContextRestoreResponseHandler<ClusterStateResponse> responseHandler = new TransportService
+                                .ContextRestoreResponseHandler<>(threadContext.newRestorableContext(false),
+                                new SniffClusterStateResponseHandler(transportService, connection, listener, seedNodes,
+                                    cancellableThreads));
+                            try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
+                                // we stash any context here since this is an internal execution and should not leak any
+                                // existing context information.
+                                threadContext.markAsSystemContext();
+                                transportService.sendRequest(connection, ClusterStateAction.NAME, request, TransportRequestOptions.EMPTY,
+                                    responseHandler);
+                            }
+                            success = true;
+                        } finally {
+                            if (success == false) {
+                                connection.close();
+                            }
+                        }
+                    });
+                } else {
+                    listener.onFailure(new IllegalStateException("no seed node left"));
+                }
+            } catch (CancellableThreads.ExecutionCancelledException ex) {
+                listener.onFailure(ex); // we got canceled - fail the listener and step out
+            } catch (ConnectTransportException | IOException | IllegalStateException ex) {
+                // ISE if we fail the handshake with an version incompatible node
+                if (seedNodes.hasNext()) {
+                    logger.debug((Supplier<?>) () -> new ParameterizedMessage("fetching nodes from external cluster {} failed",
+                        clusterAlias), ex);
+                    collectRemoteNodes(seedNodes, transportService, listener);
+                } else {
+                    listener.onFailure(ex);
+                }
+            }
+        }
+
+        @Override
+        public void close() throws IOException {
+            try {
+                if (closed.compareAndSet(false, true)) {
+                    cancellableThreads.cancel("connect handler is closed");
+                    running.acquire(); // acquire the semaphore to ensure all connections are closed and all thread joined
+                    running.release();
+                    maybeConnect(); // now go and notify pending listeners
+                }
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+            }
+        }
+
+        final boolean isClosed() {
+            return closed.get();
+        }
+
+        /* This class handles the _state response from the remote cluster when sniffing nodes to connect to */
+        private class SniffClusterStateResponseHandler implements TransportResponseHandler<ClusterStateResponse> {
+
+            private final TransportService transportService;
+            private final Transport.Connection connection;
+            private final ActionListener<Void> listener;
+            private final Iterator<DiscoveryNode> seedNodes;
+            private final CancellableThreads cancellableThreads;
+
+            SniffClusterStateResponseHandler(TransportService transportService, Transport.Connection connection,
+                                             ActionListener<Void> listener, Iterator<DiscoveryNode> seedNodes,
+                                             CancellableThreads cancellableThreads) {
+                this.transportService = transportService;
+                this.connection = connection;
+                this.listener = listener;
+                this.seedNodes = seedNodes;
+                this.cancellableThreads = cancellableThreads;
+            }
+
+            @Override
+            public ClusterStateResponse newInstance() {
+                return new ClusterStateResponse();
+            }
+
+            @Override
+            public void handleResponse(ClusterStateResponse response) {
+                assert transportService.getThreadPool().getThreadContext().isSystemContext() == false : "context is a system context";
+                try {
+                    if (remoteClusterName.get() == null) {
+                        assert response.getClusterName().value() != null;
+                        remoteClusterName.set(response.getClusterName());
+                    }
+                    try (Closeable theConnection = connection) { // the connection is unused - see comment in #collectRemoteNodes
+                        // we have to close this connection before we notify listeners - this is mainly needed for test correctness
+                        // since if we do it afterwards we might fail assertions that check if all high level connections are closed.
+                        // from a code correctness perspective we could also close it afterwards. This try/with block will
+                        // maintain the possibly exceptions thrown from within the try block and suppress the ones that are possible thrown
+                        // by closing the connection
+                        cancellableThreads.executeIO(() -> {
+                            DiscoveryNodes nodes = response.getState().nodes();
+                            Iterable<DiscoveryNode> nodesIter = nodes.getNodes()::valuesIt;
+                            for (DiscoveryNode node : nodesIter) {
+                                if (nodePredicate.test(node) && connectedNodes.size() < maxNumRemoteConnections) {
+                                    try {
+                                        transportService.connectToNode(node, remoteProfile); // noop if node is connected
+                                        connectedNodes.add(node);
+                                    } catch (ConnectTransportException | IllegalStateException ex) {
+                                        // ISE if we fail the handshake with an version incompatible node
+                                        // fair enough we can't connect just move on
+                                        logger.debug((Supplier<?>)
+                                            () -> new ParameterizedMessage("failed to connect to node {}", node), ex);
+                                    }
+                                }
+                            }
+                        });
+                    }
+                    listener.onResponse(null);
+                } catch (CancellableThreads.ExecutionCancelledException ex) {
+                    listener.onFailure(ex); // we got canceled - fail the listener and step out
+                } catch (Exception ex) {
+                    logger.warn((Supplier<?>)
+                        () -> new ParameterizedMessage("fetching nodes from external cluster {} failed",
+                            clusterAlias), ex);
+                    collectRemoteNodes(seedNodes, transportService, listener);
+                }
+            }
+
+            @Override
+            public void handleException(TransportException exp) {
+                assert transportService.getThreadPool().getThreadContext().isSystemContext() == false : "context is a system context";
+                logger.warn((Supplier<?>)
+                    () -> new ParameterizedMessage("fetching nodes from external cluster {} failed", clusterAlias),
+                    exp);
+                try {
+                    IOUtils.closeWhileHandlingException(connection);
+                } finally {
+                    // once the connection is closed lets try the next node
+                    collectRemoteNodes(seedNodes, transportService, listener);
+                }
+            }
+
+            @Override
+            public String executor() {
+                return ThreadPool.Names.MANAGEMENT;
+            }
+        }
+    }
+
+    boolean assertNoRunningConnections() { // for testing only
+        assert connectHandler.running.availablePermits() == 1;
+        return true;
+    }
+
+    boolean isNodeConnected(final DiscoveryNode node) {
+        return connectedNodes.contains(node);
+    }
+
+    DiscoveryNode getConnectedNode() {
+        return connectedNodes.get();
+    }
+
+    void addConnectedNode(DiscoveryNode node) {
+        connectedNodes.add(node);
+    }
+
+    /**
+     * Fetches connection info for this connection
+     */
+    public void getConnectionInfo(ActionListener<RemoteConnectionInfo> listener) {
+        final Optional<DiscoveryNode> anyNode = connectedNodes.getAny();
+        if (anyNode.isPresent() == false) {
+            // not connected we return immediately
+            RemoteConnectionInfo remoteConnectionStats = new RemoteConnectionInfo(clusterAlias,
+                Collections.emptyList(), Collections.emptyList(), maxNumRemoteConnections, 0,
+                RemoteClusterService.REMOTE_INITIAL_CONNECTION_TIMEOUT_SETTING.get(settings));
+            listener.onResponse(remoteConnectionStats);
+        } else {
+            NodesInfoRequest request = new NodesInfoRequest();
+            request.clear();
+            request.http(true);
+
+            transportService.sendRequest(anyNode.get(), NodesInfoAction.NAME, request, new TransportResponseHandler<NodesInfoResponse>() {
+                @Override
+                public NodesInfoResponse newInstance() {
+                    return new NodesInfoResponse();
+                }
+
+                @Override
+                public void handleResponse(NodesInfoResponse response) {
+                    Collection<TransportAddress> httpAddresses = new HashSet<>();
+                    for (NodeInfo info : response.getNodes()) {
+                        if (connectedNodes.contains(info.getNode()) && info.getHttp() != null) {
+                            httpAddresses.add(info.getHttp().getAddress().publishAddress());
+                        }
+                    }
+
+                    if (httpAddresses.size() < maxNumRemoteConnections) {
+                        // just in case non of the connected nodes have http enabled we get other http enabled nodes instead.
+                        for (NodeInfo info : response.getNodes()) {
+                            if (nodePredicate.test(info.getNode()) && info.getHttp() != null) {
+                                httpAddresses.add(info.getHttp().getAddress().publishAddress());
+                            }
+                            if (httpAddresses.size() == maxNumRemoteConnections) {
+                                break; // once we have enough return...
+                            }
+                        }
+                    }
+                    RemoteConnectionInfo remoteConnectionInfo = new RemoteConnectionInfo(clusterAlias,
+                        seedNodes.stream().map(n -> n.getAddress()).collect(Collectors.toList()), new ArrayList<>(httpAddresses),
+                        maxNumRemoteConnections, connectedNodes.size(),
+                        RemoteClusterService.REMOTE_INITIAL_CONNECTION_TIMEOUT_SETTING.get(settings));
+                    listener.onResponse(remoteConnectionInfo);
+                }
+
+                @Override
+                public void handleException(TransportException exp) {
+                    listener.onFailure(exp);
+                }
+
+                @Override
+                public String executor() {
+                    return ThreadPool.Names.SAME;
+                }
+            });
+        }
+
+    }
+
+    int getNumNodesConnected() {
+        return connectedNodes.size();
+    }
+
+    private static class ConnectedNodes implements Supplier<DiscoveryNode> {
+
+        private final Set<DiscoveryNode> nodeSet = new HashSet<>();
+        private final String clusterAlias;
+
+        private Iterator<DiscoveryNode> currentIterator = null;
+
+        private ConnectedNodes(String clusterAlias) {
+            this.clusterAlias = clusterAlias;
+        }
+
+        @Override
+        public synchronized DiscoveryNode get() {
+            ensureIteratorAvailable();
+            if (currentIterator.hasNext()) {
+                return currentIterator.next();
+            } else {
+                throw new IllegalStateException("No node available for cluster: " + clusterAlias);
+            }
+        }
+
+        synchronized boolean remove(DiscoveryNode node) {
+            final boolean setRemoval = nodeSet.remove(node);
+            if (setRemoval) {
+                currentIterator = null;
+            }
+            return setRemoval;
+        }
+
+        synchronized boolean add(DiscoveryNode node) {
+            final boolean added = nodeSet.add(node);
+            if (added) {
+                currentIterator = null;
+            }
+            return added;
+        }
+
+        synchronized int size() {
+            return nodeSet.size();
+        }
+
+        synchronized boolean contains(DiscoveryNode node) {
+            return nodeSet.contains(node);
+        }
+
+        synchronized Optional<DiscoveryNode> getAny() {
+            ensureIteratorAvailable();
+            if (currentIterator.hasNext()) {
+                return Optional.of(currentIterator.next());
+            } else {
+                return Optional.empty();
+            }
+        }
+
+        private synchronized void ensureIteratorAvailable() {
+            if (currentIterator == null) {
+                currentIterator = nodeSet.iterator();
+            } else if (currentIterator.hasNext() == false && nodeSet.isEmpty() == false) {
+                // iterator rollover
+                currentIterator = nodeSet.iterator();
+            }
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/transport/RemoteClusterService.java b/core/src/main/java/org/elasticsearch/transport/RemoteClusterService.java
new file mode 100644
index 0000000000000..2c933180049f9
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/transport/RemoteClusterService.java
@@ -0,0 +1,391 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.transport;
+
+import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.OriginalIndices;
+import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsRequest;
+import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsResponse;
+import org.elasticsearch.action.support.GroupedActionListener;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.Booleans;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.CountDown;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+
+/**
+ * Basic service for accessing remote clusters via gateway nodes
+ */
+public final class RemoteClusterService extends RemoteClusterAware implements Closeable {
+
+    /**
+     * The maximum number of connections that will be established to a remote cluster. For instance if there is only a single
+     * seed node, other nodes will be discovered up to the given number of nodes in this setting. The default is 3.
+     */
+    public static final Setting<Integer> REMOTE_CONNECTIONS_PER_CLUSTER = Setting.intSetting("search.remote.connections_per_cluster",
+        3, 1, Setting.Property.NodeScope);
+
+    /**
+     * The initial connect timeout for remote cluster connections
+     */
+    public static final Setting<TimeValue> REMOTE_INITIAL_CONNECTION_TIMEOUT_SETTING =
+        Setting.positiveTimeSetting("search.remote.initial_connect_timeout", TimeValue.timeValueSeconds(30), Setting.Property.NodeScope);
+
+    /**
+     * The name of a node attribute to select nodes that should be connected to in the remote cluster.
+     * For instance a node can be configured with <tt>node.attr.gateway: true</tt> in order to be eligible as a gateway node between
+     * clusters. In that case <tt>search.remote.node.attr: gateway</tt> can be used to filter out other nodes in the remote cluster.
+     * The value of the setting is expected to be a boolean, <tt>true</tt> for nodes that can become gateways, <tt>false</tt> otherwise.
+     */
+    public static final Setting<String> REMOTE_NODE_ATTRIBUTE = Setting.simpleString("search.remote.node.attr",
+        Setting.Property.NodeScope);
+
+    /**
+     * If <code>true</code> connecting to remote clusters is supported on this node. If <code>false</code> this node will not establish
+     * connections to any remote clusters configured. Search requests executed against this node (where this node is the coordinating node)
+     * will fail if remote cluster syntax is used as an index pattern. The default is <code>true</code>
+     */
+    public static final Setting<Boolean> ENABLE_REMOTE_CLUSTERS = Setting.boolSetting("search.remote.connect", true,
+        Setting.Property.NodeScope);
+
+    private final TransportService transportService;
+    private final int numRemoteConnections;
+    private volatile Map<String, RemoteClusterConnection> remoteClusters = Collections.emptyMap();
+
+    RemoteClusterService(Settings settings, TransportService transportService) {
+        super(settings);
+        this.transportService = transportService;
+        numRemoteConnections = REMOTE_CONNECTIONS_PER_CLUSTER.get(settings);
+    }
+
+    /**
+     * This method updates the list of remote clusters. It's intended to be used as an update consumer on the settings infrastructure
+     * @param seeds a cluster alias to discovery node mapping representing the remote clusters seeds nodes
+     * @param connectionListener a listener invoked once every configured cluster has been connected to
+     */
+    private synchronized void updateRemoteClusters(Map<String, List<DiscoveryNode>> seeds, ActionListener<Void> connectionListener) {
+        if (seeds.containsKey(LOCAL_CLUSTER_GROUP_KEY)) {
+            throw new IllegalArgumentException("remote clusters must not have the empty string as its key");
+        }
+        Map<String, RemoteClusterConnection> remoteClusters = new HashMap<>();
+        if (seeds.isEmpty()) {
+            connectionListener.onResponse(null);
+        } else {
+            CountDown countDown = new CountDown(seeds.size());
+            Predicate<DiscoveryNode> nodePredicate = (node) -> Version.CURRENT.isCompatible(node.getVersion())
+                && node.getVersion().onOrAfter(Version.V_5_3_0); // only 5.3 an onwards has the proxy actions for _search
+            if (REMOTE_NODE_ATTRIBUTE.exists(settings)) {
+                // nodes can be tagged with node.attr.remote_gateway: true to allow a node to be a gateway node for
+                // cross cluster search
+                String attribute = REMOTE_NODE_ATTRIBUTE.get(settings);
+                nodePredicate = nodePredicate.and((node) -> {
+                    final String value = node.getAttributes().getOrDefault(attribute, "false");
+                    boolean booleanValue = Booleans.parseBooleanExact(value);
+                    if (Booleans.isStrictlyBoolean(value) == false) {
+                        DeprecationLogger deprecationLogger = new DeprecationLogger(logger);
+                        deprecationLogger.deprecated("Expected a boolean [true/false] for setting [{}] but got [{}]", attribute, value);
+                    }
+                    return booleanValue;
+                });
+            }
+            remoteClusters.putAll(this.remoteClusters);
+            for (Map.Entry<String, List<DiscoveryNode>> entry : seeds.entrySet()) {
+                RemoteClusterConnection remote = this.remoteClusters.get(entry.getKey());
+                if (entry.getValue().isEmpty()) { // with no seed nodes we just remove the connection
+                    try {
+                        IOUtils.close(remote);
+                    } catch (IOException e) {
+                        logger.warn("failed to close remote cluster connections for cluster: " + entry.getKey(), e);
+                    }
+                    remoteClusters.remove(entry.getKey());
+                    continue;
+                }
+
+                if (remote == null) { // this is a new cluster we have to add a new representation
+                    remote = new RemoteClusterConnection(settings, entry.getKey(), entry.getValue(), transportService, numRemoteConnections,
+                        nodePredicate);
+                    remoteClusters.put(entry.getKey(), remote);
+                }
+
+                // now update the seed nodes no matter if it's new or already existing
+                RemoteClusterConnection finalRemote = remote;
+                remote.updateSeedNodes(entry.getValue(), ActionListener.wrap(
+                    response -> {
+                        if (countDown.countDown()) {
+                            connectionListener.onResponse(response);
+                        }
+                    },
+                    exception -> {
+                        if (countDown.fastForward()) {
+                            connectionListener.onFailure(exception);
+                        }
+                        if (finalRemote.isClosed() == false) {
+                            logger.warn("failed to update seed list for cluster: " + entry.getKey(), exception);
+                        }
+                    }));
+            }
+        }
+        this.remoteClusters = Collections.unmodifiableMap(remoteClusters);
+    }
+
+    /**
+     * Returns <code>true</code> if at least one remote cluster is configured
+     */
+    public boolean isCrossClusterSearchEnabled() {
+        return remoteClusters.isEmpty() == false;
+    }
+
+    boolean isRemoteNodeConnected(final String remoteCluster, final DiscoveryNode node) {
+        return remoteClusters.get(remoteCluster).isNodeConnected(node);
+    }
+
+    public Map<String, OriginalIndices> groupIndices(IndicesOptions indicesOptions, String[] indices, Predicate<String> indexExists) {
+        Map<String, OriginalIndices> originalIndicesMap = new HashMap<>();
+        if (isCrossClusterSearchEnabled()) {
+            final Map<String, List<String>> groupedIndices = groupClusterIndices(indices, indexExists);
+            for (Map.Entry<String, List<String>> entry : groupedIndices.entrySet()) {
+                String clusterAlias = entry.getKey();
+                List<String> originalIndices = entry.getValue();
+                originalIndicesMap.put(clusterAlias,
+                    new OriginalIndices(originalIndices.toArray(new String[originalIndices.size()]), indicesOptions));
+            }
+            if (originalIndicesMap.containsKey(LOCAL_CLUSTER_GROUP_KEY) == false) {
+                originalIndicesMap.put(LOCAL_CLUSTER_GROUP_KEY, new OriginalIndices(Strings.EMPTY_ARRAY, indicesOptions));
+            }
+        } else {
+            originalIndicesMap.put(LOCAL_CLUSTER_GROUP_KEY, new OriginalIndices(indices, indicesOptions));
+        }
+        return originalIndicesMap;
+    }
+
+    /**
+     * Returns <code>true</code> iff the given cluster is configured as a remote cluster. Otherwise <code>false</code>
+     */
+    boolean isRemoteClusterRegistered(String clusterName) {
+        return remoteClusters.containsKey(clusterName);
+    }
+
+    public void collectSearchShards(IndicesOptions indicesOptions, String preference, String routing,
+                                    Map<String, OriginalIndices> remoteIndicesByCluster,
+                                    ActionListener<Map<String, ClusterSearchShardsResponse>> listener) {
+        final CountDown responsesCountDown = new CountDown(remoteIndicesByCluster.size());
+        final Map<String, ClusterSearchShardsResponse> searchShardsResponses = new ConcurrentHashMap<>();
+        final AtomicReference<TransportException> transportException = new AtomicReference<>();
+        for (Map.Entry<String, OriginalIndices> entry : remoteIndicesByCluster.entrySet()) {
+            final String clusterName = entry.getKey();
+            RemoteClusterConnection remoteClusterConnection = remoteClusters.get(clusterName);
+            if (remoteClusterConnection == null) {
+                throw new IllegalArgumentException("no such remote cluster: " + clusterName);
+            }
+            final String[] indices = entry.getValue().indices();
+            ClusterSearchShardsRequest searchShardsRequest = new ClusterSearchShardsRequest(indices)
+                .indicesOptions(indicesOptions).local(true).preference(preference)
+                .routing(routing);
+            remoteClusterConnection.fetchSearchShards(searchShardsRequest,
+                new ActionListener<ClusterSearchShardsResponse>() {
+                    @Override
+                    public void onResponse(ClusterSearchShardsResponse clusterSearchShardsResponse) {
+                        searchShardsResponses.put(clusterName, clusterSearchShardsResponse);
+                        if (responsesCountDown.countDown()) {
+                            TransportException exception = transportException.get();
+                            if (exception == null) {
+                                listener.onResponse(searchShardsResponses);
+                            } else {
+                                listener.onFailure(transportException.get());
+                            }
+                        }
+                    }
+
+                    @Override
+                    public void onFailure(Exception e) {
+                        TransportException exception = new TransportException("unable to communicate with remote cluster [" +
+                            clusterName + "]", e);
+                        if (transportException.compareAndSet(null, exception) == false) {
+                            exception = transportException.accumulateAndGet(exception, (previous, current) -> {
+                                current.addSuppressed(previous);
+                                return current;
+                            });
+                        }
+                        if (responsesCountDown.countDown()) {
+                            listener.onFailure(exception);
+                        }
+                    }
+                });
+        }
+    }
+
+    /**
+     * Returns a connection to the given node on the given remote cluster
+     * @throws IllegalArgumentException if the remote cluster is unknown
+     */
+    public Transport.Connection getConnection(DiscoveryNode node, String cluster) {
+        RemoteClusterConnection connection = remoteClusters.get(cluster);
+        if (connection == null) {
+            throw new IllegalArgumentException("no such remote cluster: " + cluster);
+        }
+        return connection.getConnection(node);
+    }
+
+    /**
+     * Ensures that the given cluster alias is connected. If the cluster is connected this operation
+     * will invoke the listener immediately.
+     */
+    public void ensureConnected(String clusterAlias, ActionListener<Void> listener) {
+        RemoteClusterConnection remoteClusterConnection = remoteClusters.get(clusterAlias);
+        if (remoteClusterConnection == null) {
+            throw new IllegalArgumentException("no such remote cluster: " + clusterAlias);
+        }
+        remoteClusterConnection.ensureConnected(listener);
+    }
+
+    public Transport.Connection getConnection(String cluster) {
+        RemoteClusterConnection connection = remoteClusters.get(cluster);
+        if (connection == null) {
+            throw new IllegalArgumentException("no such remote cluster: " + cluster);
+        }
+        return connection.getConnection();
+    }
+
+    @Override
+    protected Set<String> getRemoteClusterNames() {
+        return this.remoteClusters.keySet();
+    }
+
+    protected void updateRemoteCluster(String clusterAlias, List<InetSocketAddress> addresses) {
+        updateRemoteCluster(clusterAlias, addresses, ActionListener.wrap((x) -> {}, (x) -> {}));
+    }
+
+    void updateRemoteCluster(
+            final String clusterAlias,
+            final List<InetSocketAddress> addresses,
+            final ActionListener<Void> connectionListener) {
+        final List<DiscoveryNode> nodes = addresses.stream().map(address -> {
+            final TransportAddress transportAddress = new InetSocketTransportAddress(address);
+            final String id = clusterAlias + "#" + transportAddress.toString();
+            final Version version = Version.CURRENT.minimumCompatibilityVersion();
+            return new DiscoveryNode(id, transportAddress, version);
+        }).collect(Collectors.toList());
+        updateRemoteClusters(Collections.singletonMap(clusterAlias, nodes), connectionListener);
+    }
+
+    /**
+     * Connects to all remote clusters in a blocking fashion. This should be called on node startup to establish an initial connection
+     * to all configured seed nodes.
+     */
+    void initializeRemoteClusters() {
+        final TimeValue timeValue = REMOTE_INITIAL_CONNECTION_TIMEOUT_SETTING.get(settings);
+        final PlainActionFuture<Void> future = new PlainActionFuture<>();
+        Map<String, List<DiscoveryNode>> seeds = RemoteClusterAware.buildRemoteClustersSeeds(settings);
+        updateRemoteClusters(seeds, future);
+        try {
+            future.get(timeValue.millis(), TimeUnit.MILLISECONDS);
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+        } catch (TimeoutException ex) {
+            logger.warn("failed to connect to remote clusters within {}", timeValue.toString());
+        } catch (Exception e) {
+            throw new IllegalStateException("failed to connect to remote clusters", e);
+        }
+    }
+
+    @Override
+    public void close() throws IOException {
+        IOUtils.close(remoteClusters.values());
+    }
+
+    public void getRemoteConnectionInfos(ActionListener<Collection<RemoteConnectionInfo>> listener) {
+        final Map<String, RemoteClusterConnection> remoteClusters = this.remoteClusters;
+        if (remoteClusters.isEmpty()) {
+            listener.onResponse(Collections.emptyList());
+        } else {
+            final GroupedActionListener<RemoteConnectionInfo> actionListener = new GroupedActionListener<>(listener,
+                remoteClusters.size(), Collections.emptyList());
+            for (RemoteClusterConnection connection : remoteClusters.values()) {
+                connection.getConnectionInfo(actionListener);
+            }
+        }
+    }
+
+    /**
+     * Collects all nodes of the given clusters and returns / passes a (clusterAlias, nodeId) to {@link DiscoveryNode}
+     * function on success.
+     */
+    public void collectNodes(Set<String> clusters, ActionListener<BiFunction<String, String, DiscoveryNode>> listener) {
+        Map<String, RemoteClusterConnection> remoteClusters = this.remoteClusters;
+        for (String cluster : clusters) {
+            if (remoteClusters.containsKey(cluster) == false) {
+                listener.onFailure(new IllegalArgumentException("no such remote cluster: [" + cluster + "]"));
+                return;
+            }
+        }
+
+        final Map<String, Function<String, DiscoveryNode>> clusterMap = new HashMap<>();
+        CountDown countDown = new CountDown(clusters.size());
+        Function<String, DiscoveryNode> nullFunction = s -> null;
+        for (final String cluster : clusters) {
+            RemoteClusterConnection connection = remoteClusters.get(cluster);
+            connection.collectNodes(new ActionListener<Function<String, DiscoveryNode>>() {
+                @Override
+                public void onResponse(Function<String, DiscoveryNode> nodeLookup) {
+                    synchronized (clusterMap) {
+                        clusterMap.put(cluster, nodeLookup);
+                    }
+                    if (countDown.countDown()) {
+                        listener.onResponse((clusterAlias, nodeId)
+                            -> clusterMap.getOrDefault(clusterAlias, nullFunction).apply(nodeId));
+                    }
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    if (countDown.fastForward()) { // we need to check if it's true since we could have multiple failures
+                        listener.onFailure(e);
+                    }
+                }
+            });
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/transport/RemoteConnectionInfo.java b/core/src/main/java/org/elasticsearch/transport/RemoteConnectionInfo.java
new file mode 100644
index 0000000000000..3f4ff64f162c5
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/transport/RemoteConnectionInfo.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.transport;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.transport.TransportAddressSerializers;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+/**
+ * This class encapsulates all remote cluster information to be rendered on
+ * <tt>_remote/info</tt> requests.
+ */
+public final class RemoteConnectionInfo implements ToXContent, Writeable {
+    final List<TransportAddress> seedNodes;
+    final List<TransportAddress> httpAddresses;
+    final int connectionsPerCluster;
+    final TimeValue initialConnectionTimeout;
+    final int numNodesConnected;
+    final String clusterAlias;
+
+    RemoteConnectionInfo(String clusterAlias, List<TransportAddress> seedNodes,
+                         List<TransportAddress> httpAddresses,
+                         int connectionsPerCluster, int numNodesConnected,
+                         TimeValue initialConnectionTimeout) {
+        this.clusterAlias = clusterAlias;
+        this.seedNodes = seedNodes;
+        this.httpAddresses = httpAddresses;
+        this.connectionsPerCluster = connectionsPerCluster;
+        this.numNodesConnected = numNodesConnected;
+        this.initialConnectionTimeout = initialConnectionTimeout;
+    }
+
+    public RemoteConnectionInfo(StreamInput input) throws IOException {
+        seedNodes = input.readList(in -> TransportAddressSerializers.addressFromStream(in));
+        httpAddresses = input.readList(in -> TransportAddressSerializers.addressFromStream(in));
+        connectionsPerCluster = input.readVInt();
+        initialConnectionTimeout = new TimeValue(input);
+        numNodesConnected = input.readVInt();
+        clusterAlias = input.readString();
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject(clusterAlias);
+        {
+            builder.startArray("seeds");
+            for (TransportAddress addr : seedNodes) {
+                builder.value(addr.toString());
+            }
+            builder.endArray();
+            builder.startArray("http_addresses");
+            for (TransportAddress addr : httpAddresses) {
+                builder.value(addr.toString());
+            }
+            builder.endArray();
+            builder.field("connected", numNodesConnected > 0);
+            builder.field("num_nodes_connected", numNodesConnected);
+            builder.field("max_connections_per_cluster", connectionsPerCluster);
+            builder.field("initial_connect_timeout", initialConnectionTimeout);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeList(seedNodes.stream().map(a -> getWriteable(a)).collect(Collectors.toList()));
+        out.writeList(httpAddresses.stream().map(a -> getWriteable(a)).collect(Collectors.toList()));
+        out.writeVInt(connectionsPerCluster);
+        initialConnectionTimeout.writeTo(out);
+        out.writeVInt(numNodesConnected);
+        out.writeString(clusterAlias);
+    }
+
+    private static Writeable getWriteable(TransportAddress address) {
+        return (out) -> TransportAddressSerializers.addressToStream(out, address);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        RemoteConnectionInfo that = (RemoteConnectionInfo) o;
+        return connectionsPerCluster == that.connectionsPerCluster &&
+            numNodesConnected == that.numNodesConnected &&
+            Objects.equals(seedNodes, that.seedNodes) &&
+            Objects.equals(httpAddresses, that.httpAddresses) &&
+            Objects.equals(initialConnectionTimeout, that.initialConnectionTimeout) &&
+            Objects.equals(clusterAlias, that.clusterAlias);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(seedNodes, httpAddresses, connectionsPerCluster, initialConnectionTimeout, numNodesConnected, clusterAlias);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/transport/RequestHandlerRegistry.java b/core/src/main/java/org/elasticsearch/transport/RequestHandlerRegistry.java
index 27342af1bdd96..9531c9c4f76cd 100644
--- a/core/src/main/java/org/elasticsearch/transport/RequestHandlerRegistry.java
+++ b/core/src/main/java/org/elasticsearch/transport/RequestHandlerRegistry.java
@@ -99,7 +99,7 @@ private static class TransportChannelWrapper extends DelegatingTransportChannel
 
         private final TaskManager taskManager;
 
-        public TransportChannelWrapper(TaskManager taskManager, Task task, TransportChannel channel) {
+        TransportChannelWrapper(TaskManager taskManager, Task task, TransportChannel channel) {
             super(channel);
             this.task = task;
             this.taskManager = taskManager;
diff --git a/core/src/main/java/org/elasticsearch/transport/TaskAwareTransportRequestHandler.java b/core/src/main/java/org/elasticsearch/transport/TaskAwareTransportRequestHandler.java
new file mode 100644
index 0000000000000..12899d86d430d
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/transport/TaskAwareTransportRequestHandler.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.transport;
+
+/**
+ * Transport request handlers that is using task context
+ */
+public abstract class TaskAwareTransportRequestHandler<T extends TransportRequest> implements TransportRequestHandler<T>  {
+    @Override
+    public final void messageReceived(T request, TransportChannel channel) throws Exception {
+        throw new UnsupportedOperationException("the task parameter is required");
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/transport/TcpTransport.java b/core/src/main/java/org/elasticsearch/transport/TcpTransport.java
index 00409f42995b8..86e574de786bf 100644
--- a/core/src/main/java/org/elasticsearch/transport/TcpTransport.java
+++ b/core/src/main/java/org/elasticsearch/transport/TcpTransport.java
@@ -24,9 +24,12 @@
 import org.apache.logging.log4j.util.Supplier;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.NotifyOnceListener;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.CheckedBiConsumer;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.bytes.BytesArray;
@@ -37,7 +40,7 @@
 import org.elasticsearch.common.compress.Compressor;
 import org.elasticsearch.common.compress.CompressorFactory;
 import org.elasticsearch.common.compress.NotCompressedException;
-import org.elasticsearch.common.io.ReleasableBytesStream;
+import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -61,15 +64,14 @@
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.AbstractLifecycleRunnable;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
+import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.common.util.concurrent.KeyedLock;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.monitor.jvm.JvmInfo;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.support.TransportStatus;
 
-import java.io.Closeable;
 import java.io.IOException;
 import java.io.StreamCorruptedException;
 import java.net.BindException;
@@ -81,22 +83,28 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.EnumMap;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.function.Consumer;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
 import static java.util.Collections.unmodifiableMap;
 import static org.elasticsearch.common.settings.Setting.boolSetting;
@@ -145,15 +153,8 @@ public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent i
         Setting.byteSizeSetting("transport.tcp.receive_buffer_size", NetworkService.TcpSettings.TCP_RECEIVE_BUFFER_SIZE,
             Setting.Property.NodeScope);
 
-    private static final long NINETY_PER_HEAP_SIZE = (long) (JvmInfo.jvmInfo().getMem().getHeapMax().bytes() * 0.9);
-    private static final int PING_DATA_SIZE = -1;
-
-    protected final int connectionsPerNodeRecovery;
-    protected final int connectionsPerNodeBulk;
-    protected final int connectionsPerNodeReg;
-    protected final int connectionsPerNodeState;
-    protected final int connectionsPerNodePing;
-    protected final TimeValue connectTimeout;
+    private static final long NINETY_PER_HEAP_SIZE = (long) (JvmInfo.jvmInfo().getMem().getHeapMax().getBytes() * 0.9);
+    public static final int PING_DATA_SIZE = -1;
     protected final boolean blockingClient;
     private final CircuitBreakerService circuitBreakerService;
     // package visibility for tests
@@ -166,6 +167,7 @@ public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent i
     protected volatile TransportServiceAdapter transportServiceAdapter;
     // node id to actual channel
     protected final ConcurrentMap<DiscoveryNode, NodeChannels> connectedNodes = newConcurrentMap();
+
     protected final Map<String, List<Channel>> serverChannels = newConcurrentMap();
     protected final ConcurrentMap<String, BoundTransportAddress> profileBoundAddresses = newConcurrentMap();
 
@@ -174,10 +176,16 @@ public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent i
 
     // this lock is here to make sure we close this transport and disconnect all the client nodes
     // connections while no connect operations is going on... (this might help with 100% CPU when stopping the transport?)
-    protected final ReadWriteLock globalLock = new ReentrantReadWriteLock();
+    protected final ReadWriteLock closeLock = new ReentrantReadWriteLock();
     protected final boolean compress;
     protected volatile BoundTransportAddress boundAddress;
     private final String transportName;
+    protected final ConnectionProfile defaultConnectionProfile;
+
+    private final ConcurrentMap<Long, HandshakeResponseHandler> pendingHandshakes = new ConcurrentHashMap<>();
+    private final AtomicLong requestIdGenerator = new AtomicLong();
+    private final CounterMetric numHandshakes = new CounterMetric();
+    private static final String HANDSHAKE_ACTION_NAME = "internal:tcp/handshake";
 
     public TcpTransport(String transportName, Settings settings, ThreadPool threadPool, BigArrays bigArrays,
                         CircuitBreakerService circuitBreakerService, NamedWriteableRegistry namedWriteableRegistry,
@@ -192,14 +200,27 @@ public TcpTransport(String transportName, Settings settings, ThreadPool threadPo
         this.compress = Transport.TRANSPORT_TCP_COMPRESS.get(settings);
         this.networkService = networkService;
         this.transportName = transportName;
-
-        this.connectionsPerNodeRecovery = CONNECTIONS_PER_NODE_RECOVERY.get(settings);
-        this.connectionsPerNodeBulk = CONNECTIONS_PER_NODE_BULK.get(settings);
-        this.connectionsPerNodeReg = CONNECTIONS_PER_NODE_REG.get(settings);
-        this.connectionsPerNodeState = CONNECTIONS_PER_NODE_STATE.get(settings);
-        this.connectionsPerNodePing = CONNECTIONS_PER_NODE_PING.get(settings);
-        this.connectTimeout = TCP_CONNECT_TIMEOUT.get(settings);
         this.blockingClient = TCP_BLOCKING_CLIENT.get(settings);
+        defaultConnectionProfile = buildDefaultConnectionProfile(settings);
+    }
+
+    static ConnectionProfile buildDefaultConnectionProfile(Settings settings) {
+        int connectionsPerNodeRecovery = CONNECTIONS_PER_NODE_RECOVERY.get(settings);
+        int connectionsPerNodeBulk = CONNECTIONS_PER_NODE_BULK.get(settings);
+        int connectionsPerNodeReg = CONNECTIONS_PER_NODE_REG.get(settings);
+        int connectionsPerNodeState = CONNECTIONS_PER_NODE_STATE.get(settings);
+        int connectionsPerNodePing = CONNECTIONS_PER_NODE_PING.get(settings);
+        ConnectionProfile.Builder builder = new ConnectionProfile.Builder();
+        builder.setConnectTimeout(TCP_CONNECT_TIMEOUT.get(settings));
+        builder.setHandshakeTimeout(TCP_CONNECT_TIMEOUT.get(settings));
+        builder.addConnections(connectionsPerNodeBulk, TransportRequestOptions.Type.BULK);
+        builder.addConnections(connectionsPerNodePing, TransportRequestOptions.Type.PING);
+        // if we are not master eligible we don't need a dedicated channel to publish the state
+        builder.addConnections(DiscoveryNode.isMasterNode(settings) ? connectionsPerNodeState : 0, TransportRequestOptions.Type.STATE);
+        // if we are not a data-node we don't need any dedicated channels for recovery
+        builder.addConnections(DiscoveryNode.isDataNode(settings) ? connectionsPerNodeRecovery : 0, TransportRequestOptions.Type.RECOVERY);
+        builder.addConnections(connectionsPerNodeReg, TransportRequestOptions.Type.REG);
+        return builder.build();
     }
 
     @Override
@@ -217,15 +238,55 @@ public CircuitBreaker getInFlightRequestBreaker() {
 
     @Override
     public void transportServiceAdapter(TransportServiceAdapter service) {
+        if (service.getRequestHandler(HANDSHAKE_ACTION_NAME) != null) {
+            throw new IllegalStateException(HANDSHAKE_ACTION_NAME + " is a reserved request handler and must not be registered");
+        }
         this.transportServiceAdapter = service;
     }
 
-    public Settings settings() {
-        return this.settings;
-    }
+    private static class HandshakeResponseHandler<Channel> implements TransportResponseHandler<VersionHandshakeResponse> {
+        final AtomicReference<Version> versionRef = new AtomicReference<>();
+        final CountDownLatch latch = new CountDownLatch(1);
+        final AtomicBoolean handshakeNotSupported = new AtomicBoolean(false);
+        final AtomicReference<Exception> exceptionRef = new AtomicReference<>();
+        final Channel channel;
+
+        HandshakeResponseHandler(Channel channel) {
+            this.channel = channel;
+        }
+
+        @Override
+        public VersionHandshakeResponse newInstance() {
+            return new VersionHandshakeResponse();
+        }
 
-    public boolean isCompressed() {
-        return compress;
+        @Override
+        public void handleResponse(VersionHandshakeResponse response) {
+            final boolean success = versionRef.compareAndSet(null, response.version);
+            assert success;
+            latch.countDown();
+        }
+
+        @Override
+        public void handleException(TransportException exp) {
+            Throwable cause = exp.getCause();
+            if (cause != null
+                && cause instanceof ActionNotFoundTransportException
+                // this will happen if we talk to a node (pre 5.2) that doesn't have a handshake handler
+                // we will just treat the node as a 5.0.0 node unless the discovery node that is used to connect has a higher version.
+                && cause.getMessage().equals("No handler for action [internal:tcp/handshake]")) {
+                    handshakeNotSupported.set(true);
+            } else {
+                final boolean success = exceptionRef.compareAndSet(null, exp);
+                assert success;
+            }
+            latch.countDown();
+        }
+
+        @Override
+        public String executor() {
+            return ThreadPool.Names.SAME;
+        }
     }
 
     public class ScheduledPing extends AbstractLifecycleRunnable {
@@ -255,20 +316,26 @@ protected void doRunInLifecycle() throws Exception {
             for (Map.Entry<DiscoveryNode, NodeChannels> entry : connectedNodes.entrySet()) {
                 DiscoveryNode node = entry.getKey();
                 NodeChannels channels = entry.getValue();
-                for (Channel channel : channels.allChannels) {
-                    try {
-                        sendMessage(channel, pingHeader, successfulPings::inc, false);
-                    } catch (Exception e) {
-                        if (isOpen(channel)) {
-                            logger.debug(
-                                (Supplier<?>) () -> new ParameterizedMessage("[{}] failed to send ping transport message", node), e);
-                            failedPings.inc();
-                        } else {
-                            logger.trace(
-                                (Supplier<?>) () -> new ParameterizedMessage(
-                                    "[{}] failed to send ping transport message (channel closed)", node), e);
+                for (Channel channel : channels.getChannels()) {
+                    internalSendMessage(channel, pingHeader, new NotifyOnceListener<Channel>() {
+                        @Override
+                        public void innerOnResponse(Channel channel) {
+                            successfulPings.inc();
                         }
-                    }
+
+                        @Override
+                        public void innerOnFailure(Exception e) {
+                            if (isOpen(channel)) {
+                                logger.debug(
+                                    (Supplier<?>) () -> new ParameterizedMessage("[{}] failed to send ping transport message", node), e);
+                                failedPings.inc();
+                            } else {
+                                logger.trace((Supplier<?>) () ->
+                                    new ParameterizedMessage("[{}] failed to send ping transport message (channel closed)", node), e);
+                            }
+
+                        }
+                    });
                 }
             }
         }
@@ -283,7 +350,15 @@ public long getFailedPings() {
 
         @Override
         protected void onAfterInLifecycle() {
-            threadPool.schedule(pingSchedule, ThreadPool.Names.GENERIC, this);
+            try {
+                threadPool.schedule(pingSchedule, ThreadPool.Names.GENERIC, this);
+            } catch (EsRejectedExecutionException ex) {
+                if (ex.isExecutorShutdown()) {
+                    logger.debug("couldn't schedule new ping execution, executor is shutting down", ex);
+                } else {
+                    throw ex;
+                }
+            }
         }
 
         @Override
@@ -296,69 +371,79 @@ public void onFailure(Exception e) {
         }
     }
 
-    public class NodeChannels implements Closeable {
-
-        public List<Channel> allChannels = Collections.emptyList();
-        public Channel[] recovery;
-        public final AtomicInteger recoveryCounter = new AtomicInteger();
-        public Channel[] bulk;
-        public final AtomicInteger bulkCounter = new AtomicInteger();
-        public Channel[] reg;
-        public final AtomicInteger regCounter = new AtomicInteger();
-        public Channel[] state;
-        public final AtomicInteger stateCounter = new AtomicInteger();
-        public Channel[] ping;
-        public final AtomicInteger pingCounter = new AtomicInteger();
-
-        public NodeChannels(Channel[] recovery, Channel[] bulk, Channel[] reg, Channel[] state, Channel[] ping) {
-            this.recovery = recovery;
-            this.bulk = bulk;
-            this.reg = reg;
-            this.state = state;
-            this.ping = ping;
+    public final class NodeChannels implements Connection {
+        private final Map<TransportRequestOptions.Type, ConnectionProfile.ConnectionTypeHandle> typeMapping;
+        private final Channel[] channels;
+        private final DiscoveryNode node;
+        private final AtomicBoolean closed = new AtomicBoolean(false);
+        private final Version version;
+
+        public NodeChannels(DiscoveryNode node, Channel[] channels, ConnectionProfile connectionProfile) {
+            this.node = node;
+            this.channels = channels;
+            assert channels.length == connectionProfile.getNumConnections() : "expected channels size to be == "
+                + connectionProfile.getNumConnections() + " but was: [" + channels.length + "]";
+            typeMapping = new EnumMap<>(TransportRequestOptions.Type.class);
+            for (ConnectionProfile.ConnectionTypeHandle handle : connectionProfile.getHandles()) {
+                for (TransportRequestOptions.Type type : handle.getTypes())
+                    typeMapping.put(type, handle);
+            }
+            version = node.getVersion();
         }
 
-        public void start() {
-            List<Channel> newAllChannels = new ArrayList<>();
-            newAllChannels.addAll(Arrays.asList(recovery));
-            newAllChannels.addAll(Arrays.asList(bulk));
-            newAllChannels.addAll(Arrays.asList(reg));
-            newAllChannels.addAll(Arrays.asList(state));
-            newAllChannels.addAll(Arrays.asList(ping));
-            this.allChannels = Collections.unmodifiableList(newAllChannels);
+        NodeChannels(NodeChannels channels, Version handshakeVersion) {
+            this.node = channels.node;
+            this.channels = channels.channels;
+            this.typeMapping = channels.typeMapping;
+            this.version = handshakeVersion;
+            assert handshakeVersion != null : "handshakeVersion must not be null";
         }
 
-        public boolean hasChannel(Channel channel) {
-            for (Channel channel1 : allChannels) {
-                if (channel.equals(channel1)) {
-                    return true;
-                }
-            }
-            return false;
+        @Override
+        public Version getVersion() {
+            return version;
+        }
+
+        public List<Channel> getChannels() {
+            return Arrays.asList(channels);
         }
 
         public Channel channel(TransportRequestOptions.Type type) {
-            if (type == TransportRequestOptions.Type.REG) {
-                return reg[Math.floorMod(regCounter.incrementAndGet(), reg.length)];
-            } else if (type == TransportRequestOptions.Type.STATE) {
-                return state[Math.floorMod(stateCounter.incrementAndGet(), state.length)];
-            } else if (type == TransportRequestOptions.Type.PING) {
-                return ping[Math.floorMod(pingCounter.incrementAndGet(), ping.length)];
-            } else if (type == TransportRequestOptions.Type.BULK) {
-                return bulk[Math.floorMod(bulkCounter.incrementAndGet(), bulk.length)];
-            } else if (type == TransportRequestOptions.Type.RECOVERY) {
-                return recovery[Math.floorMod(recoveryCounter.incrementAndGet(), recovery.length)];
-            } else {
+            ConnectionProfile.ConnectionTypeHandle connectionTypeHandle = typeMapping.get(type);
+            if (connectionTypeHandle == null) {
                 throw new IllegalArgumentException("no type channel for [" + type + "]");
             }
+            return connectionTypeHandle.getChannel(channels);
         }
 
-        public List<Channel[]> getChannelArrays() {
-            return Arrays.asList(recovery, bulk, reg, state, ping);
+        @Override
+        public void close() throws IOException {
+            if (closed.compareAndSet(false, true)) {
+                try {
+                    closeChannels(Arrays.stream(channels).filter(Objects::nonNull).collect(Collectors.toList()), false);
+                } finally {
+                    transportServiceAdapter.onConnectionClosed(this);
+                }
+            }
         }
 
-        public synchronized void close() throws IOException {
-            closeChannels(allChannels);
+        @Override
+        public DiscoveryNode getNode() {
+            return this.node;
+        }
+
+        @Override
+        public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options)
+            throws IOException, TransportException {
+            if (closed.get()) {
+                throw new NodeNotConnectedException(node, "connection already closed");
+            }
+            Channel channel = channel(options.type());
+            sendRequestToChannel(this.node, channel, requestId, action, request, options, getVersion(), (byte) 0);
+        }
+
+        boolean isClosed() {
+            return closed.get();
         }
     }
 
@@ -368,133 +453,205 @@ public boolean nodeConnected(DiscoveryNode node) {
     }
 
     @Override
-    public void connectToNodeLight(DiscoveryNode node) throws ConnectTransportException {
-        connectToNode(node, true);
-    }
-
-    @Override
-    public void connectToNode(DiscoveryNode node) {
-        connectToNode(node, false);
-    }
-
-    public void connectToNode(DiscoveryNode node, boolean light) {
-        if (!lifecycle.started()) {
-            throw new IllegalStateException("can't add nodes to a stopped transport");
-        }
+    public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile,
+                              CheckedBiConsumer<Connection, ConnectionProfile, IOException> connectionValidator)
+        throws ConnectTransportException {
+        connectionProfile = resolveConnectionProfile(connectionProfile, defaultConnectionProfile);
         if (node == null) {
             throw new ConnectTransportException(null, "can't connect to a null node");
         }
-        globalLock.readLock().lock();
+        closeLock.readLock().lock(); // ensure we don't open connections while we are closing
         try {
-
+            ensureOpen();
             try (Releasable ignored = connectionLock.acquire(node.getId())) {
-                if (!lifecycle.started()) {
-                    throw new IllegalStateException("can't add nodes to a stopped transport");
-                }
                 NodeChannels nodeChannels = connectedNodes.get(node);
                 if (nodeChannels != null) {
                     return;
                 }
+                boolean success = false;
                 try {
-                    if (light) {
-                        nodeChannels = connectToChannelsLight(node);
-                    } else {
-                        try {
-                            nodeChannels = connectToChannels(node);
-                        } catch (Exception e) {
-                            logger.trace(
-                                (Supplier<?>) () -> new ParameterizedMessage(
-                                    "failed to connect to [{}], cleaning dangling connections", node), e);
-                            throw e;
-                        }
-                    }
+                    nodeChannels = openConnection(node, connectionProfile);
+                    connectionValidator.accept(nodeChannels, connectionProfile);
                     // we acquire a connection lock, so no way there is an existing connection
-                    nodeChannels.start();
                     connectedNodes.put(node, nodeChannels);
                     if (logger.isDebugEnabled()) {
                         logger.debug("connected to node [{}]", node);
                     }
-                    transportServiceAdapter.raiseNodeConnected(node);
+                    try {
+                        transportServiceAdapter.onNodeConnected(node);
+                    } finally {
+                        if (nodeChannels.isClosed()) {
+                            // we got closed concurrently due to a disconnect or some other event on the channel.
+                            // the close callback will close the NodeChannel instance first and then try to remove
+                            // the connection from the connected nodes. It will NOT acquire the connectionLock for
+                            // the node to prevent any blocking calls on network threads. Yet, we still establish a happens
+                            // before relationship to the connectedNodes.put since we check if we can remove the
+                            // (DiscoveryNode, NodeChannels) tuple from the map after we closed. Here we check if it's closed an if so we
+                            // try to remove it first either way one of the two wins even if the callback has run before we even added the
+                            // tuple to the map since in that case we remove it here again
+                            if (connectedNodes.remove(node, nodeChannels)) {
+                                transportServiceAdapter.onNodeDisconnected(node);
+                            }
+                            throw new NodeNotConnectedException(node, "connection concurrently closed");
+                        }
+                    }
+                    success = true;
                 } catch (ConnectTransportException e) {
                     throw e;
                 } catch (Exception e) {
                     throw new ConnectTransportException(node, "general node connection failure", e);
+                } finally {
+                    if (success == false) { // close the connection if there is a failure
+                        logger.trace(
+                            (Supplier<?>) () -> new ParameterizedMessage(
+                                "failed to connect to [{}], cleaning dangling connections", node));
+                        IOUtils.closeWhileHandlingException(nodeChannels);
+                    }
                 }
             }
         } finally {
-            globalLock.readLock().unlock();
+            closeLock.readLock().unlock();
         }
     }
+
     /**
-     * Disconnects from a node, only if the relevant channel is found to be part of the node channels.
+     * takes a {@link ConnectionProfile} that have been passed as a parameter to the public methods
+     * and resolves it to a fully specified (i.e., no nulls) profile
      */
-    protected boolean disconnectFromNode(DiscoveryNode node, Channel channel, String reason) {
-        // this might be called multiple times from all the node channels, so do a lightweight
-        // check outside of the lock
-        NodeChannels nodeChannels = connectedNodes.get(node);
-        if (nodeChannels != null && nodeChannels.hasChannel(channel)) {
-            try (Releasable ignored = connectionLock.acquire(node.getId())) {
-                nodeChannels = connectedNodes.get(node);
-                // check again within the connection lock, if its still applicable to remove it
-                if (nodeChannels != null && nodeChannels.hasChannel(channel)) {
-                    connectedNodes.remove(node);
+    static ConnectionProfile resolveConnectionProfile(@Nullable ConnectionProfile connectionProfile,
+                                                      ConnectionProfile defaultConnectionProfile) {
+        Objects.requireNonNull(defaultConnectionProfile);
+        if (connectionProfile == null) {
+            return defaultConnectionProfile;
+        } else if (connectionProfile.getConnectTimeout() != null && connectionProfile.getHandshakeTimeout() != null) {
+            return connectionProfile;
+        } else {
+            ConnectionProfile.Builder builder = new ConnectionProfile.Builder(connectionProfile);
+            if (connectionProfile.getConnectTimeout() == null) {
+                builder.setConnectTimeout(defaultConnectionProfile.getConnectTimeout());
+            }
+            if (connectionProfile.getHandshakeTimeout() == null) {
+                builder.setHandshakeTimeout(defaultConnectionProfile.getHandshakeTimeout());
+            }
+            return builder.build();
+        }
+    }
+
+    @Override
+    public final NodeChannels openConnection(DiscoveryNode node, ConnectionProfile connectionProfile) throws IOException {
+        if (node == null) {
+            throw new ConnectTransportException(null, "can't open connection to a null node");
+        }
+        boolean success = false;
+        NodeChannels nodeChannels = null;
+        connectionProfile = resolveConnectionProfile(connectionProfile, defaultConnectionProfile);
+        closeLock.readLock().lock(); // ensure we don't open connections while we are closing
+        try {
+            ensureOpen();
+            try {
+                final AtomicBoolean runOnce = new AtomicBoolean(false);
+                final AtomicReference<NodeChannels> connectionRef = new AtomicReference<>();
+                Consumer<Channel> onClose = c -> {
                     try {
-                        logger.debug("disconnecting from [{}], {}", node, reason);
-                        IOUtils.closeWhileHandlingException(nodeChannels);
+                        // we can't assert that the channel is closed here since netty3 has a different behavior that doesn't
+                        // consider a channel closed while it's close future is running.
+                        onChannelClosed(c);
                     } finally {
-                        logger.trace("disconnected from [{}], {}", node, reason);
-                        transportServiceAdapter.raiseNodeDisconnected(node);
+                        // we only need to disconnect from the nodes once since all other channels
+                        // will also try to run this we protect it from running multiple times.
+                        if (runOnce.compareAndSet(false, true)) {
+                            NodeChannels connection = connectionRef.get();
+                            if (connection != null) {
+                                disconnectFromNodeCloseAndNotify(node, connection);
+                            }
+                        }
                     }
-                    return true;
+                };
+                nodeChannels = connectToChannels(node, connectionProfile, onClose);
+                final Channel channel = nodeChannels.getChannels().get(0); // one channel is guaranteed by the connection profile
+                final TimeValue connectTimeout = connectionProfile.getConnectTimeout() == null ?
+                    defaultConnectionProfile.getConnectTimeout() :
+                    connectionProfile.getConnectTimeout();
+                final TimeValue handshakeTimeout = connectionProfile.getHandshakeTimeout() == null ?
+                    connectTimeout : connectionProfile.getHandshakeTimeout();
+                final Version version = executeHandshake(node, channel, handshakeTimeout);
+                if (version != null) {
+                    // if we are talking to a pre 5.2 node we won't be able to retrieve the version since it doesn't implement the handshake
+                    // we do since 5.2 - in this case we just go with the version provided by the node.
+                    nodeChannels = new NodeChannels(nodeChannels, version); // clone the channels - we now have the correct version
+                }
+                transportServiceAdapter.onConnectionOpened(nodeChannels);
+                connectionRef.set(nodeChannels);
+                if (Arrays.stream(nodeChannels.channels).allMatch(this::isOpen) == false) {
+                    throw new ConnectTransportException(node, "a channel closed while connecting");
+                }
+                success = true;
+                return nodeChannels;
+            } catch (ConnectTransportException e) {
+                throw e;
+            } catch (Exception e) {
+                // ConnectTransportExceptions are handled specifically on the caller end - we wrap the actual exception to ensure
+                // only relevant exceptions are logged on the caller end.. this is the same as in connectToNode
+                throw new ConnectTransportException(node, "general node connection failure", e);
+            } finally {
+                    if (success == false) {
+                        IOUtils.closeWhileHandlingException(nodeChannels);
+                    }
+            }
+        } finally {
+            closeLock.readLock().unlock();
+        }
+    }
+
+    private void disconnectFromNodeCloseAndNotify(DiscoveryNode node, NodeChannels nodeChannels) {
+        assert nodeChannels != null : "nodeChannels must not be null";
+        try {
+            IOUtils.closeWhileHandlingException(nodeChannels);
+        } finally {
+            if (closeLock.readLock().tryLock()) {
+                try {
+                    if (connectedNodes.remove(node, nodeChannels)) {
+                        transportServiceAdapter.onNodeDisconnected(node);
+                    }
+                } finally {
+                    closeLock.readLock().unlock();
                 }
             }
         }
-        return false;
     }
 
     /**
      * Disconnects from a node if a channel is found as part of that nodes channels.
      */
-    protected final void disconnectFromNodeChannel(final Channel channel, final Exception failure) {
-        threadPool.generic().execute(() -> {
+    protected final void closeChannelWhileHandlingExceptions(final Channel channel) {
+        if (isOpen(channel)) {
             try {
-                try {
-                    closeChannels(Collections.singletonList(channel));
-                } finally {
-                    for (DiscoveryNode node : connectedNodes.keySet()) {
-                        if (disconnectFromNode(node, channel, ExceptionsHelper.detailedMessage(failure))) {
-                            // if we managed to find this channel and disconnect from it, then break, no need to check on
-                            // the rest of the nodes
-                            break;
-                        }
-                    }
-                }
+                closeChannels(Collections.singletonList(channel), false);
             } catch (IOException e) {
                 logger.warn("failed to close channel", e);
             }
-        });
+        }
     }
 
-    protected Channel nodeChannel(DiscoveryNode node, TransportRequestOptions options) throws ConnectTransportException {
+    @Override
+    public NodeChannels getConnection(DiscoveryNode node) {
         NodeChannels nodeChannels = connectedNodes.get(node);
         if (nodeChannels == null) {
             throw new NodeNotConnectedException(node, "Node not connected");
         }
-        return nodeChannels.channel(options.type());
+        return nodeChannels;
     }
 
     @Override
     public void disconnectFromNode(DiscoveryNode node) {
+        closeLock.readLock().lock();
+        NodeChannels nodeChannels = null;
         try (Releasable ignored = connectionLock.acquire(node.getId())) {
-            NodeChannels nodeChannels = connectedNodes.remove(node);
-            if (nodeChannels != null) {
-                try {
-                    logger.debug("disconnecting from [{}] due to explicit disconnect call", node);
-                    IOUtils.closeWhileHandlingException(nodeChannels);
-                } finally {
-                    logger.trace("disconnected from [{}] due to explicit disconnect call", node);
-                    transportServiceAdapter.raiseNodeDisconnected(node);
-                }
+            nodeChannels = connectedNodes.remove(node);
+        } finally {
+            closeLock.readLock().unlock();
+            if (nodeChannels != null) { // if we found it and removed it we close and notify
+                IOUtils.closeWhileHandlingException(nodeChannels, () -> transportServiceAdapter.onNodeDisconnected(node));
             }
         }
     }
@@ -521,7 +678,7 @@ public Map<String, BoundTransportAddress> profileBoundAddresses() {
 
     protected Map<String, Settings> buildProfileSettings() {
         // extract default profile first and create standard bootstrap
-        Map<String, Settings> profiles = TransportSettings.TRANSPORT_PROFILES_SETTING.get(settings()).getAsGroups(true);
+        Map<String, Settings> profiles = TransportSettings.TRANSPORT_PROFILES_SETTING.get(settings).getAsGroups(true);
         if (!profiles.containsKey(TransportSettings.DEFAULT_PROFILE)) {
             profiles = new HashMap<>(profiles);
             profiles.put(TransportSettings.DEFAULT_PROFILE, Settings.EMPTY);
@@ -548,8 +705,8 @@ protected Map<String, Settings> buildProfileSettings() {
                 continue;
             }
             Settings mergedSettings = Settings.builder()
-                .put(defaultSettings)
-                .put(profileSettings)
+                .put(defaultSettings.getAsMap())
+                .put(profileSettings.getAsMap())
                 .build();
             result.put(name, mergedSettings);
         }
@@ -574,14 +731,14 @@ protected void bindServer(final String name, final Settings settings) {
         try {
             hostAddresses = networkService.resolveBindHostAddresses(bindHosts);
         } catch (IOException e) {
-            throw new BindTransportException("Failed to resolve host " + Arrays.toString(bindHosts) + "", e);
+            throw new BindTransportException("Failed to resolve host " + Arrays.toString(bindHosts), e);
         }
         if (logger.isDebugEnabled()) {
             String[] addresses = new String[hostAddresses.length];
             for (int i = 0; i < hostAddresses.length; i++) {
                 addresses[i] = NetworkAddress.format(hostAddresses[i]);
             }
-            logger.debug("binding server bootstrap to: {}", (Object)addresses);
+            logger.debug("binding server bootstrap to: {}", (Object) addresses);
         }
 
         assert hostAddresses.length > 0;
@@ -705,7 +862,7 @@ public static int resolvePublishPort(String profileName, Settings settings, Sett
     }
 
     @Override
-    public TransportAddress[] addressesFromString(String address, int perAddressLimit) throws Exception {
+    public TransportAddress[] addressesFromString(String address, int perAddressLimit) throws UnknownHostException {
         return parse(address, settings.get("transport.profiles.default.port", TransportSettings.PORT.get(settings)), perAddressLimit);
     }
 
@@ -772,27 +929,33 @@ protected final void doStop() {
         final CountDownLatch latch = new CountDownLatch(1);
         // make sure we run it on another thread than a possible IO handler thread
         threadPool.generic().execute(() -> {
-            globalLock.writeLock().lock();
+            closeLock.writeLock().lock();
             try {
                 // first stop to accept any incoming connections so nobody can connect to this transport
                 for (Map.Entry<String, List<Channel>> entry : serverChannels.entrySet()) {
                     try {
-                        closeChannels(entry.getValue());
+                        closeChannels(entry.getValue(), true);
                     } catch (Exception e) {
                         logger.debug(
                             (Supplier<?>) () -> new ParameterizedMessage(
                                 "Error closing serverChannel for profile [{}]", entry.getKey()), e);
                     }
                 }
-
-                for (Iterator<NodeChannels> it = connectedNodes.values().iterator(); it.hasNext(); ) {
-                    NodeChannels nodeChannels = it.next();
-                    it.remove();
-                    IOUtils.closeWhileHandlingException(nodeChannels);
+                // we are holding a write lock so nobody modifies the connectedNodes / openConnections map - it's safe to first close
+                // all instances and then clear them maps
+                Iterator<Map.Entry<DiscoveryNode, NodeChannels>> iterator = connectedNodes.entrySet().iterator();
+                while (iterator.hasNext()) {
+                    Map.Entry<DiscoveryNode, NodeChannels> next = iterator.next();
+                    try {
+                        IOUtils.closeWhileHandlingException(next.getValue());
+                        transportServiceAdapter.onNodeDisconnected(next.getKey());
+                    } finally {
+                        iterator.remove();
+                    }
                 }
                 stopInternal();
             } finally {
-                globalLock.writeLock().unlock();
+                closeLock.writeLock().unlock();
                 latch.countDown();
             }
         });
@@ -805,11 +968,13 @@ protected final void doStop() {
         }
     }
 
-    protected void onException(Channel channel, Exception e) throws IOException {
+    protected void onException(Channel channel, Exception e) {
         if (!lifecycle.started()) {
-            // ignore
+            // just close and ignore - we are already stopped and just need to make sure we release all resources
+            closeChannelWhileHandlingExceptions(channel);
             return;
         }
+
         if (isCloseConnectionException(e)) {
             logger.trace(
                 (Supplier<?>) () -> new ParameterizedMessage(
@@ -817,15 +982,15 @@ protected void onException(Channel channel, Exception e) throws IOException {
                     channel),
                 e);
             // close the channel, which will cause a node to be disconnected if relevant
-            disconnectFromNodeChannel(channel, e);
+            closeChannelWhileHandlingExceptions(channel);
         } else if (isConnectException(e)) {
             logger.trace((Supplier<?>) () -> new ParameterizedMessage("connect exception caught on transport layer [{}]", channel), e);
             // close the channel as safe measure, which will cause a node to be disconnected if relevant
-            disconnectFromNodeChannel(channel, e);
+            closeChannelWhileHandlingExceptions(channel);
         } else if (e instanceof BindException) {
             logger.trace((Supplier<?>) () -> new ParameterizedMessage("bind exception caught on transport layer [{}]", channel), e);
             // close the channel as safe measure, which will cause a node to be disconnected if relevant
-            disconnectFromNodeChannel(channel, e);
+            closeChannelWhileHandlingExceptions(channel);
         } else if (e instanceof CancelledKeyException) {
             logger.trace(
                 (Supplier<?>) () -> new ParameterizedMessage(
@@ -833,17 +998,37 @@ protected void onException(Channel channel, Exception e) throws IOException {
                     channel),
                 e);
             // close the channel as safe measure, which will cause a node to be disconnected if relevant
-            disconnectFromNodeChannel(channel, e);
+            closeChannelWhileHandlingExceptions(channel);
         } else if (e instanceof TcpTransport.HttpOnTransportException) {
             // in case we are able to return data, serialize the exception content and sent it back to the client
             if (isOpen(channel)) {
-                sendMessage(channel, new BytesArray(e.getMessage().getBytes(StandardCharsets.UTF_8)), () -> {}, true);
+                final NotifyOnceListener<Channel> closeChannel = new NotifyOnceListener<Channel>() {
+                    @Override
+                    public void innerOnResponse(Channel channel) {
+                        try {
+                            closeChannels(Collections.singletonList(channel), false);
+                        } catch (IOException e1) {
+                            logger.debug("failed to close httpOnTransport channel", e1);
+                        }
+                    }
+
+                    @Override
+                    public void innerOnFailure(Exception e) {
+                        try {
+                            closeChannels(Collections.singletonList(channel), false);
+                        } catch (IOException e1) {
+                            e.addSuppressed(e1);
+                            logger.debug("failed to close httpOnTransport channel", e1);
+                        }
+                    }
+                };
+                internalSendMessage(channel, new BytesArray(e.getMessage().getBytes(StandardCharsets.UTF_8)), closeChannel);
             }
         } else {
             logger.warn(
                 (Supplier<?>) () -> new ParameterizedMessage("exception caught on transport layer [{}], closing connection", channel), e);
             // close the channel, which will cause a node to be disconnected if relevant
-            disconnectFromNodeChannel(channel, e);
+            closeChannelWhileHandlingExceptions(channel);
         }
     }
 
@@ -854,31 +1039,44 @@ protected void onException(Channel channel, Exception e) throws IOException {
 
     /**
      * Binds to the given {@link InetSocketAddress}
-     * @param name the profile name
+     *
+     * @param name    the profile name
      * @param address the address to bind to
      */
     protected abstract Channel bind(String name, InetSocketAddress address) throws IOException;
 
     /**
-     * Closes all channels in this list
+     * Closes all channels in this list. If the blocking boolean is set to true, the channels must be
+     * closed before the method returns. This should never be called with blocking set to true from a
+     * network thread.
+     *
+     * @param channels the channels to close
+     * @param blocking whether the channels should be closed synchronously
      */
-    protected abstract void closeChannels(List<Channel> channel) throws IOException;
+    protected abstract void closeChannels(List<Channel> channels, boolean blocking) throws IOException;
 
     /**
-     * Connects to the given node in a light way. This means we are not creating multiple connections like we do
-     * for production connections. This connection is for pings or handshakes
+     * Sends message to channel. The listener's onResponse method will be called when the send is complete unless an exception
+     * is thrown during the send. If an exception is thrown, the listener's onException method will be called.
+     * @param channel the destination channel
+     * @param reference the byte reference for the message
+     * @param listener the listener to call when the operation has completed
      */
-    protected abstract NodeChannels connectToChannelsLight(DiscoveryNode node) throws IOException;
-
-
-    protected abstract void sendMessage(Channel channel, BytesReference reference, Runnable sendListener, boolean close) throws IOException;
+    protected abstract void sendMessage(Channel channel, BytesReference reference, ActionListener<Channel> listener);
 
     /**
-     * Connects to the node in a <tt>heavy</tt> way.
+     * Connect to the node with channels as defined by the specified connection profile. Implementations must invoke the specified channel
+     * close callback when a channel is closed.
      *
-     * @see #connectToChannelsLight(DiscoveryNode)
+     * @param node              the node to connect to
+     * @param connectionProfile the connection profile
+     * @param onChannelClose    callback to invoke when a channel is closed
+     * @return the channels
+     * @throws IOException if an I/O exception occurs while opening channels
      */
-    protected abstract NodeChannels connectToChannels(DiscoveryNode node) throws IOException;
+    protected abstract NodeChannels connectToChannels(DiscoveryNode node,
+                                                      ConnectionProfile connectionProfile,
+                                                      Consumer<Channel> onChannelClose) throws IOException;
 
     /**
      * Called to tear down internal resources
@@ -889,18 +1087,19 @@ public boolean canCompress(TransportRequest request) {
         return compress && (!(request instanceof BytesTransportRequest));
     }
 
-    @Override
-    public void sendRequest(final DiscoveryNode node, final long requestId, final String action, final TransportRequest request,
-                            TransportRequestOptions options) throws IOException, TransportException {
-        Channel targetChannel = nodeChannel(node, options);
+    private void sendRequestToChannel(final DiscoveryNode node, final Channel targetChannel, final long requestId, final String action,
+                                      final TransportRequest request, TransportRequestOptions options, Version channelVersion,
+                                      byte status) throws IOException,
+        TransportException {
         if (compress) {
             options = TransportRequestOptions.builder(options).withCompress(true).build();
         }
-        byte status = 0;
         status = TransportStatus.setRequest(status);
         ReleasableBytesStreamOutput bStream = new ReleasableBytesStreamOutput(bigArrays);
-        boolean addedReleaseListener = false;
-        StreamOutput stream = bStream;
+        // we wrap this in a release once since if the onRequestSent callback throws an exception
+        // we might release things twice and this should be prevented
+        final Releasable toRelease = Releasables.releaseOnce(() -> Releasables.close(bStream.bytes()));
+        StreamOutput stream = Streams.flushOnCloseStream(bStream);
         try {
             // only compress if asked, and, the request is not bytes, since then only
             // the header part is compressed, and the "body" can't be extracted as compressed
@@ -912,50 +1111,38 @@ public void sendRequest(final DiscoveryNode node, final long requestId, final St
             // we pick the smallest of the 2, to support both backward and forward compatibility
             // note, this is the only place we need to do this, since from here on, we use the serialized version
             // as the version to use also when the node receiving this request will send the response with
-            Version version = Version.smallest(getCurrentVersion(), node.getVersion());
+            Version version = Version.min(getCurrentVersion(), channelVersion);
 
             stream.setVersion(version);
             threadPool.getThreadContext().writeTo(stream);
             stream.writeString(action);
             BytesReference message = buildMessage(requestId, status, node.getVersion(), request, stream, bStream);
             final TransportRequestOptions finalOptions = options;
-            Runnable onRequestSent = () -> {
-                try {
-                    Releasables.close(bStream.bytes());
-                } finally {
-                    transportServiceAdapter.onRequestSent(node, requestId, action, request, finalOptions);
-                }
-            };
-            addedReleaseListener = internalSendMessage(targetChannel, message, onRequestSent);
+            // this might be called in a different thread
+            SendListener onRequestSent = new SendListener(toRelease,
+                    () -> transportServiceAdapter.onRequestSent(node, requestId, action, request, finalOptions));
+            internalSendMessage(targetChannel, message, onRequestSent);
         } finally {
             IOUtils.close(stream);
-            if (!addedReleaseListener) {
-                Releasables.close(bStream.bytes());
-            }
         }
     }
 
     /**
-     * sends a message view the given channel, using the given callbacks.
-     *
-     * @return true if the message was successfully sent or false when an error occurred and the error hanlding logic was activated
-     *
+     * sends a message to the given channel, using the given callbacks.
      */
-    private boolean internalSendMessage(Channel targetChannel, BytesReference message, Runnable onRequestSent) throws IOException {
-        boolean success;
+    private void internalSendMessage(Channel targetChannel, BytesReference message, NotifyOnceListener<Channel> listener) {
         try {
-            sendMessage(targetChannel, message, onRequestSent, false);
-            success = true;
-        } catch (IOException ex) {
-            // passing exception handling to deal with this and raise disconnect events and decide the right logging level
+            sendMessage(targetChannel, message, listener);
+        } catch (Exception ex) {
+            // call listener to ensure that any resources are released
+            listener.onFailure(ex);
             onException(targetChannel, ex);
-            success = false;
         }
-        return success;
     }
 
     /**
      * Sends back an error response to the caller via the given channel
+     *
      * @param nodeVersion the caller node version
      * @param channel the channel to send the response to
      * @param error the error to return
@@ -975,8 +1162,9 @@ public void sendErrorResponse(Version nodeVersion, Channel channel, final Except
             status = TransportStatus.setError(status);
             final BytesReference bytes = stream.bytes();
             final BytesReference header = buildHeader(requestId, status, nodeVersion, bytes.length());
-            Runnable onRequestSent = () -> transportServiceAdapter.onResponseSent(requestId, action, error);
-            sendMessage(channel, new CompositeBytesReference(header, bytes), onRequestSent, false);
+            SendListener onResponseSent = new SendListener(null,
+                () -> transportServiceAdapter.onResponseSent(requestId, action, error));
+            internalSendMessage(channel, new CompositeBytesReference(header, bytes), onResponseSent);
         }
     }
 
@@ -987,14 +1175,20 @@ public void sendErrorResponse(Version nodeVersion, Channel channel, final Except
      */
     public void sendResponse(Version nodeVersion, Channel channel, final TransportResponse response, final long requestId,
                              final String action, TransportResponseOptions options) throws IOException {
+        sendResponse(nodeVersion, channel, response, requestId, action, options, (byte) 0);
+    }
+
+    private void sendResponse(Version nodeVersion, Channel channel, final TransportResponse response, final long requestId,
+        final String action, TransportResponseOptions options, byte status) throws IOException {
         if (compress) {
             options = TransportResponseOptions.builder(options).withCompress(true).build();
         }
-        byte status = 0;
         status = TransportStatus.setResponse(status); // TODO share some code with sendRequest
         ReleasableBytesStreamOutput bStream = new ReleasableBytesStreamOutput(bigArrays);
-        boolean addedReleaseListener = false;
-        StreamOutput stream = bStream;
+        // we wrap this in a release once since if the onRequestSent callback throws an exception
+        // we might release things twice and this should be prevented
+        final Releasable toRelease = Releasables.releaseOnce(() -> Releasables.close(bStream.bytes()));
+        StreamOutput stream = Streams.flushOnCloseStream(bStream);
         try {
             if (options.compress()) {
                 status = TransportStatus.setCompress(status);
@@ -1005,19 +1199,12 @@ public void sendResponse(Version nodeVersion, Channel channel, final TransportRe
             BytesReference reference = buildMessage(requestId, status, nodeVersion, response, stream, bStream);
 
             final TransportResponseOptions finalOptions = options;
-            Runnable onRequestSent = () -> {
-                try {
-                    Releasables.close(bStream.bytes());
-                } finally {
-                    transportServiceAdapter.onResponseSent(requestId, action, response, finalOptions);
-                }
-            };
-            addedReleaseListener = internalSendMessage(channel, reference, onRequestSent);
+            // this might be called in a different thread
+            SendListener listener = new SendListener(toRelease,
+                    () -> transportServiceAdapter.onResponseSent(requestId, action, response, finalOptions));
+            internalSendMessage(channel, reference, listener);
         } finally {
             IOUtils.close(stream);
-            if (!addedReleaseListener) {
-                Releasables.close(bStream.bytes());
-            }
         }
     }
 
@@ -1030,7 +1217,7 @@ public void sendResponse(Version nodeVersion, Channel channel, final TransportRe
      * @param length          the payload length in bytes
      * @see TcpHeader
      */
-    private BytesReference buildHeader(long requestId, byte status, Version protocolVersion, int length) throws IOException {
+    final BytesReference buildHeader(long requestId, byte status, Version protocolVersion, int length) throws IOException {
         try (BytesStreamOutput headerOutput = new BytesStreamOutput(TcpHeader.HEADER_SIZE)) {
             headerOutput.setVersion(protocolVersion);
             TcpHeader.writeHeader(headerOutput, requestId, status, protocolVersion, length);
@@ -1045,7 +1232,7 @@ private BytesReference buildHeader(long requestId, byte status, Version protocol
      * Serializes the given message into a bytes representation
      */
     private BytesReference buildMessage(long requestId, byte status, Version nodeVersion, TransportMessage message, StreamOutput stream,
-                                        ReleasableBytesStream writtenBytes) throws IOException {
+                                        ReleasableBytesStreamOutput writtenBytes) throws IOException {
         final BytesReference zeroCopyBuffer;
         if (message instanceof BytesTransportRequest) { // what a shitty optimization - we should use a direct send method instead
             BytesTransportRequest bRequest = (BytesTransportRequest) message;
@@ -1113,13 +1300,14 @@ public static boolean validateMessageHeader(BytesReference buffer) throws IOExce
                 return false;
             }
         }
+
         if (dataLen <= 0) {
             throw new StreamCorruptedException("invalid data length: " + dataLen);
         }
         // safety against too large frames being sent
         if (dataLen > NINETY_PER_HEAP_SIZE) {
             throw new IllegalArgumentException("transport content length received [" + new ByteSizeValue(dataLen) + "] exceeded ["
-                    + new ByteSizeValue(NINETY_PER_HEAP_SIZE) + "]");
+                + new ByteSizeValue(NINETY_PER_HEAP_SIZE) + "]");
         }
 
         if (buffer.length() < dataLen + sizeHeaderLength) {
@@ -1131,7 +1319,7 @@ public static boolean validateMessageHeader(BytesReference buffer) throws IOExce
     private static boolean bufferStartsWith(BytesReference buffer, int offset, String method) {
         char[] chars = method.toCharArray();
         for (int i = 0; i < chars.length; i++) {
-            if (buffer.get(offset+ i) != chars[i]) {
+            if (buffer.get(offset + i) != chars[i]) {
                 return false;
             }
         }
@@ -1154,7 +1342,7 @@ public RestStatus status() {
             return RestStatus.BAD_REQUEST;
         }
 
-        public HttpOnTransportException(StreamInput in) throws IOException{
+        public HttpOnTransportException(StreamInput in) throws IOException {
             super(in);
         }
     }
@@ -1202,9 +1390,19 @@ public final void messageReceived(BytesReference reference, Channel channel, Str
             streamIn.setVersion(version);
             threadPool.getThreadContext().readHeaders(streamIn);
             if (TransportStatus.isRequest(status)) {
-                handleRequest(channel, profileName, streamIn, requestId, messageLengthBytes, version, remoteAddress);
+                handleRequest(channel, profileName, streamIn, requestId, messageLengthBytes, version, remoteAddress, status);
             } else {
-                final TransportResponseHandler<?> handler = transportServiceAdapter.onResponseReceived(requestId);
+                final TransportResponseHandler<?> handler;
+                if (TransportStatus.isHandshake(status)) {
+                    handler = pendingHandshakes.remove(requestId);
+                } else {
+                    TransportResponseHandler theHandler = transportServiceAdapter.onResponseReceived(requestId);
+                    if (theHandler == null && TransportStatus.isError(status)) {
+                        handler = pendingHandshakes.remove(requestId);
+                    } else {
+                        handler = theHandler;
+                    }
+                }
                 // ignore if its null, the adapter logs it
                 if (handler != null) {
                     if (TransportStatus.isError(status)) {
@@ -1250,7 +1448,8 @@ public void onFailure(Exception e) {
             @Override
             protected void doRun() throws Exception {
                 handler.handleResponse(response);
-            }});
+            }
+        });
 
     }
 
@@ -1281,29 +1480,35 @@ private void handleException(final TransportResponseHandler handler, Throwable e
         });
     }
 
-    protected String handleRequest(Channel channel, String profileName, final StreamInput stream, long requestId,
-                                   int messageLengthBytes, Version version, InetSocketAddress remoteAddress) throws IOException {
+    protected String handleRequest(Channel channel, String profileName, final StreamInput stream, long requestId, int messageLengthBytes,
+                                   Version version, InetSocketAddress remoteAddress, byte status) throws IOException {
         final String action = stream.readString();
         transportServiceAdapter.onRequestReceived(requestId, action);
         TransportChannel transportChannel = null;
         try {
-            final RequestHandlerRegistry reg = transportServiceAdapter.getRequestHandler(action);
-            if (reg == null) {
-                throw new ActionNotFoundTransportException(action);
-            }
-            if (reg.canTripCircuitBreaker()) {
-                getInFlightRequestBreaker().addEstimateBytesAndMaybeBreak(messageLengthBytes, "<transport_request>");
+            if (TransportStatus.isHandshake(status)) {
+                final VersionHandshakeResponse response = new VersionHandshakeResponse(getCurrentVersion());
+                sendResponse(version, channel, response, requestId, HANDSHAKE_ACTION_NAME, TransportResponseOptions.EMPTY,
+                    TransportStatus.setHandshake((byte) 0));
             } else {
-                getInFlightRequestBreaker().addWithoutBreaking(messageLengthBytes);
+                final RequestHandlerRegistry reg = transportServiceAdapter.getRequestHandler(action);
+                if (reg == null) {
+                    throw new ActionNotFoundTransportException(action);
+                }
+                if (reg.canTripCircuitBreaker()) {
+                    getInFlightRequestBreaker().addEstimateBytesAndMaybeBreak(messageLengthBytes, "<transport_request>");
+                } else {
+                    getInFlightRequestBreaker().addWithoutBreaking(messageLengthBytes);
+                }
+                transportChannel = new TcpTransportChannel<>(this, channel, transportName, action, requestId, version, profileName,
+                    messageLengthBytes);
+                final TransportRequest request = reg.newRequest();
+                request.remoteAddress(new InetSocketTransportAddress(remoteAddress));
+                request.readFrom(stream);
+                // in case we throw an exception, i.e. when the limit is hit, we don't want to verify
+                validateRequest(stream, requestId, action);
+                threadPool.executor(reg.getExecutor()).execute(new RequestHandler(reg, request, transportChannel));
             }
-            transportChannel = new TcpTransportChannel<>(this, channel, transportName, action, requestId, version, profileName,
-                messageLengthBytes);
-            final TransportRequest request = reg.newRequest();
-            request.remoteAddress(new InetSocketTransportAddress(remoteAddress));
-            request.readFrom(stream);
-            // in case we throw an exception, i.e. when the limit is hit, we don't want to verify
-            validateRequest(stream, requestId, action);
-            threadPool.executor(reg.getExecutor()).execute(new RequestHandler(reg, request, transportChannel));
         } catch (Exception e) {
             // the circuit breaker tripped
             if (transportChannel == null) {
@@ -1336,7 +1541,7 @@ class RequestHandler extends AbstractRunnable {
         private final TransportRequest request;
         private final TransportChannel transportChannel;
 
-        public RequestHandler(RequestHandlerRegistry reg, TransportRequest request, TransportChannel transportChannel) {
+        RequestHandler(RequestHandlerRegistry reg, TransportRequest request, TransportChannel transportChannel) {
             this.reg = reg;
             this.request = request;
             this.transportChannel = transportChannel;
@@ -1368,4 +1573,142 @@ public void onFailure(Exception e) {
             }
         }
     }
+
+    private static final class VersionHandshakeResponse extends TransportResponse {
+        private Version version;
+
+        private VersionHandshakeResponse(Version version) {
+            this.version = version;
+        }
+
+        private VersionHandshakeResponse() {}
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            super.readFrom(in);
+            version = Version.readVersion(in);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            assert version != null;
+            Version.writeVersion(version, out);
+        }
+    }
+
+    protected Version executeHandshake(DiscoveryNode node, Channel channel, TimeValue timeout) throws IOException, InterruptedException {
+        numHandshakes.inc();
+        final long requestId = newRequestId();
+        final HandshakeResponseHandler handler = new HandshakeResponseHandler(channel);
+        AtomicReference<Version> versionRef = handler.versionRef;
+        AtomicReference<Exception> exceptionRef = handler.exceptionRef;
+        pendingHandshakes.put(requestId, handler);
+        boolean success = false;
+        try {
+            if (isOpen(channel) == false) {
+                // we have to protect us here since sendRequestToChannel won't barf if the channel is closed.
+                // it's weird but to change it will cause a lot of impact on the exception handling code all over the codebase.
+                // yet, if we don't check the state here we might have registered a pending handshake handler but the close
+                // listener calling #onChannelClosed might have already run and we are waiting on the latch below unitl we time out.
+                throw new IllegalStateException("handshake failed, channel already closed");
+            }
+            // for the request we use the minCompatVersion since we don't know what's the version of the node we talk to
+            // we also have no payload on the request but the response will contain the actual version of the node we talk
+            // to as the payload.
+            final Version minCompatVersion = getCurrentVersion().minimumCompatibilityVersion();
+            sendRequestToChannel(node, channel, requestId, HANDSHAKE_ACTION_NAME, TransportRequest.Empty.INSTANCE,
+                TransportRequestOptions.EMPTY, minCompatVersion, TransportStatus.setHandshake((byte) 0));
+            if (handler.latch.await(timeout.millis(), TimeUnit.MILLISECONDS) == false) {
+                throw new ConnectTransportException(node, "handshake_timeout[" + timeout + "]");
+            }
+            success = true;
+            if (handler.handshakeNotSupported.get()) {
+                // this is a BWC layer, if we talk to a pre 5.2 node then the handshake is not supported
+                // this will go away in master once it's all ported to 5.2 but for now we keep this to make
+                // the backport straight forward
+                return null;
+            }
+            if (exceptionRef.get() != null) {
+                throw new IllegalStateException("handshake failed", exceptionRef.get());
+            } else {
+                Version version = versionRef.get();
+                if (getCurrentVersion().isCompatible(version) == false) {
+                    throw new IllegalStateException("Received message from unsupported version: [" + version
+                        + "] minimal compatible version is: [" + getCurrentVersion().minimumCompatibilityVersion() + "]");
+                }
+                return version;
+            }
+        } finally {
+            final TransportResponseHandler<?> removedHandler = pendingHandshakes.remove(requestId);
+            // in the case of a timeout or an exception on the send part the handshake has not been removed yet.
+            // but the timeout is tricky since it's basically a race condition so we only assert on the success case.
+            assert success && removedHandler == null || success == false : "handler for requestId [" + requestId + "] is not been removed";
+        }
+    }
+
+    final int getNumPendingHandshakes() { // for testing
+        return pendingHandshakes.size();
+    }
+
+    final long getNumHandshakes() {
+        return numHandshakes.count(); // for testing
+    }
+
+    @Override
+    public long newRequestId() {
+        return requestIdGenerator.incrementAndGet();
+    }
+
+    /**
+     * Called once the channel is closed for instance due to a disconnect or a closed socket etc.
+     */
+    private void onChannelClosed(Channel channel) {
+        final Optional<Long> first = pendingHandshakes.entrySet().stream()
+            .filter((entry) -> entry.getValue().channel == channel).map((e) -> e.getKey()).findFirst();
+        if (first.isPresent()) {
+            final Long requestId = first.get();
+            final HandshakeResponseHandler handler = pendingHandshakes.remove(requestId);
+            if (handler != null) {
+                // there might be a race removing this or this method might be called twice concurrently depending on how
+                // the channel is closed ie. due to connection reset or broken pipes
+                handler.handleException(new TransportException("connection reset"));
+            }
+        }
+    }
+
+    /**
+     * Ensures this transport is still started / open
+     *
+     * @throws IllegalStateException if the transport is not started / open
+     */
+    protected final void ensureOpen() {
+        if (lifecycle.started() == false) {
+            throw new IllegalStateException("transport has been stopped");
+        }
+    }
+
+    private final class SendListener extends NotifyOnceListener<Channel> {
+        private final Releasable optionalReleasable;
+        private final Runnable transportAdaptorCallback;
+
+        private SendListener(Releasable optionalReleasable, Runnable transportAdaptorCallback) {
+            this.optionalReleasable = optionalReleasable;
+            this.transportAdaptorCallback = transportAdaptorCallback;
+        }
+
+        @Override
+        public void innerOnResponse(Channel channel) {
+            release();
+        }
+
+        @Override
+        public void innerOnFailure(Exception e) {
+            release();
+        }
+
+        private void release() {
+            Releasables.close(optionalReleasable, transportAdaptorCallback::run);
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/transport/TcpTransportChannel.java b/core/src/main/java/org/elasticsearch/transport/TcpTransportChannel.java
index 1fceb5aa1a31d..03632eb951a72 100644
--- a/core/src/main/java/org/elasticsearch/transport/TcpTransportChannel.java
+++ b/core/src/main/java/org/elasticsearch/transport/TcpTransportChannel.java
@@ -66,7 +66,7 @@ public void sendResponse(TransportResponse response, TransportResponseOptions op
         try {
             transport.sendResponse(version, channel, response, requestId, action, options);
         } finally {
-            release();
+            release(false);
         }
     }
 
@@ -75,19 +75,20 @@ public void sendResponse(Exception exception) throws IOException {
         try {
             transport.sendErrorResponse(version, channel, exception, requestId, action);
         } finally {
-            release();
+            release(true);
         }
     }
     private Exception releaseBy;
 
-    private void release() {
-        // attempt to release once atomically
-        if (released.compareAndSet(false, true) == false) {
-            throw new IllegalStateException("reserved bytes are already released", releaseBy);
-        } else {
+    private void release(boolean isExceptionResponse) {
+        if (released.compareAndSet(false, true)) {
             assert (releaseBy = new Exception()) != null; // easier to debug if it's already closed
+            transport.getInFlightRequestBreaker().addWithoutBreaking(-reservedBytes);
+        } else if (isExceptionResponse == false) {
+            // only fail if we are not sending an error - we might send the error triggered by the previous
+            // sendResponse call
+            throw new IllegalStateException("reserved bytes are already released", releaseBy);
         }
-        transport.getInFlightRequestBreaker().addWithoutBreaking(-reservedBytes);
     }
 
     @Override
@@ -104,5 +105,9 @@ public Channel getChannel() {
         return channel;
     }
 
+    @Override
+    public Version getVersion() {
+        return version;
+    }
 }
 
diff --git a/core/src/main/java/org/elasticsearch/transport/Transport.java b/core/src/main/java/org/elasticsearch/transport/Transport.java
index d0b2edf09bbf4..5fdb5293685e0 100644
--- a/core/src/main/java/org/elasticsearch/transport/Transport.java
+++ b/core/src/main/java/org/elasticsearch/transport/Transport.java
@@ -19,7 +19,9 @@
 
 package org.elasticsearch.transport;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.CheckedBiConsumer;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.breaker.NoopCircuitBreaker;
 import org.elasticsearch.common.component.LifecycleComponent;
@@ -28,7 +30,9 @@
 import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
 
+import java.io.Closeable;
 import java.io.IOException;
+import java.net.UnknownHostException;
 import java.util.List;
 import java.util.Map;
 
@@ -37,7 +41,6 @@
  */
 public interface Transport extends LifecycleComponent {
 
-
     Setting<Boolean> TRANSPORT_TCP_COMPRESS = Setting.boolSetting("transport.tcp.compress", false, Property.NodeScope);
 
     void transportServiceAdapter(TransportServiceAdapter service);
@@ -56,7 +59,7 @@ public interface Transport extends LifecycleComponent {
     /**
      * Returns an address from its string representation.
      */
-    TransportAddress[] addressesFromString(String address, int perAddressLimit) throws Exception;
+    TransportAddress[] addressesFromString(String address, int perAddressLimit) throws UnknownHostException;
 
     /**
      * Is the address type supported.
@@ -69,28 +72,17 @@ public interface Transport extends LifecycleComponent {
     boolean nodeConnected(DiscoveryNode node);
 
     /**
-     * Connects to the given node, if already connected, does nothing.
+     * Connects to a node with the given connection profile. If the node is already connected this method has no effect.
+     * Once a successful is established, it can be validated before being exposed.
      */
-    void connectToNode(DiscoveryNode node) throws ConnectTransportException;
-
-    /**
-     * Connects to a node in a light manner. Used when just connecting for ping and then
-     * disconnecting.
-     */
-    void connectToNodeLight(DiscoveryNode node) throws ConnectTransportException;
+    void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile,
+                       CheckedBiConsumer<Connection, ConnectionProfile, IOException> connectionValidator) throws ConnectTransportException;
 
     /**
      * Disconnected from the given node, if not connected, will do nothing.
      */
     void disconnectFromNode(DiscoveryNode node);
 
-    /**
-     * Sends the request to the node.
-     * @throws NodeNotConnectedException if the given node is not connected
-     */
-    void sendRequest(DiscoveryNode node, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws
-        IOException, TransportException;
-
     /**
      * Returns count of currently open connections
      */
@@ -102,4 +94,59 @@ default CircuitBreaker getInFlightRequestBreaker() {
         return new NoopCircuitBreaker("in-flight-noop");
     }
 
+    /**
+     * Returns a new request ID to use when sending a message via {@link Connection#sendRequest(long, String,
+     * TransportRequest, TransportRequestOptions)}
+     */
+    long newRequestId();
+    /**
+     * Returns a connection for the given node if the node is connected.
+     * Connections returned from this method must not be closed. The lifecylce of this connection is maintained by the Transport
+     * implementation.
+     *
+     * @throws NodeNotConnectedException if the node is not connected
+     * @see #connectToNode(DiscoveryNode, ConnectionProfile, CheckedBiConsumer)
+     */
+    Connection getConnection(DiscoveryNode node);
+
+    /**
+     * Opens a new connection to the given node and returns it. In contrast to
+     * {@link #connectToNode(DiscoveryNode, ConnectionProfile, CheckedBiConsumer)} the returned connection is not managed by
+     * the transport implementation. This connection must be closed once it's not needed anymore.
+     * This connection type can be used to execute a handshake between two nodes before the node will be published via
+     * {@link #connectToNode(DiscoveryNode, ConnectionProfile, CheckedBiConsumer)}.
+     */
+    Connection openConnection(DiscoveryNode node, ConnectionProfile profile) throws IOException;
+
+    /**
+     * A unidirectional connection to a {@link DiscoveryNode}
+     */
+    interface Connection extends Closeable {
+        /**
+         * The node this connection is associated with
+         */
+        DiscoveryNode getNode();
+
+        /**
+         * Sends the request to the node this connection is associated with
+         * @throws NodeNotConnectedException if the given node is not connected
+         */
+        void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options) throws
+            IOException, TransportException;
+
+        /**
+         * Returns the version of the node this connection was established with.
+         */
+        default Version getVersion() {
+            return getNode().getVersion();
+        }
+
+        /**
+         * Returns a key that this connection can be cached on. Delegating subclasses must delegate method call to
+         * the original connection.
+         */
+        default Object getCacheKey() {
+            return this;
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/transport/TransportActionProxy.java b/core/src/main/java/org/elasticsearch/transport/TransportActionProxy.java
new file mode 100644
index 0000000000000..3cca1810ef30f
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/transport/TransportActionProxy.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.transport;
+
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.threadpool.ThreadPool;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.function.Function;
+import java.util.function.Supplier;
+
+/**
+ * TransportActionProxy allows an arbitrary action to be executed on a defined target node while the initial request is sent to a second
+ * node that acts as a request proxy to the target node. This is useful if a node is not directly connected to a target node but is
+ * connected to an intermediate node that establishes a transitive connection.
+ */
+public final class TransportActionProxy {
+
+    private TransportActionProxy() {} // no instance
+
+    private static class ProxyRequestHandler<T extends ProxyRequest> implements TransportRequestHandler<T> {
+
+        private final TransportService service;
+        private final String action;
+        private final Function<TransportRequest, Supplier<TransportResponse>> responseFunction;
+
+        ProxyRequestHandler(TransportService service, String action, Function<TransportRequest,
+                Supplier<TransportResponse>> responseFunction) {
+            this.service = service;
+            this.action = action;
+            this.responseFunction = responseFunction;
+        }
+
+        @Override
+        public void messageReceived(T request, TransportChannel channel) throws Exception {
+            DiscoveryNode targetNode = request.targetNode;
+            TransportRequest wrappedRequest = request.wrapped;
+            service.sendRequest(targetNode, action, wrappedRequest,
+                    new ProxyResponseHandler<>(channel, responseFunction.apply(wrappedRequest)));
+        }
+    }
+
+    private static class ProxyResponseHandler<T extends TransportResponse> implements TransportResponseHandler<T> {
+
+        private final Supplier<T> responseFactory;
+        private final TransportChannel channel;
+
+        ProxyResponseHandler(TransportChannel channel, Supplier<T> responseFactory) {
+            this.responseFactory = responseFactory;
+            this.channel = channel;
+
+        }
+        @Override
+        public T newInstance() {
+            return responseFactory.get();
+        }
+
+        @Override
+        public void handleResponse(T response) {
+            try {
+                channel.sendResponse(response);
+            } catch (IOException e) {
+                throw new UncheckedIOException(e);
+            }
+        }
+
+        @Override
+        public void handleException(TransportException exp) {
+            try {
+                channel.sendResponse(exp);
+            } catch (IOException e) {
+                throw new UncheckedIOException(e);
+            }
+        }
+
+        @Override
+        public String executor() {
+            return ThreadPool.Names.SAME;
+        }
+    }
+
+    static class ProxyRequest<T extends TransportRequest> extends TransportRequest {
+        T wrapped;
+        Supplier<T> supplier;
+        DiscoveryNode targetNode;
+
+        ProxyRequest(Supplier<T> supplier) {
+            this.supplier = supplier;
+        }
+
+        ProxyRequest(T wrapped, DiscoveryNode targetNode) {
+            this.wrapped = wrapped;
+            this.targetNode = targetNode;
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            super.readFrom(in);
+            targetNode = new DiscoveryNode(in);
+            wrapped = supplier.get();
+            wrapped.readFrom(in);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            targetNode.writeTo(out);
+            wrapped.writeTo(out);
+        }
+    }
+
+    /**
+     * Registers a proxy request handler that allows to forward requests for the given action to another node. To be used when the
+     * response type changes based on the upcoming request (quite rare)
+     */
+    public static void registerProxyAction(TransportService service, String action,
+                                           Function<TransportRequest, Supplier<TransportResponse>> responseFunction) {
+        RequestHandlerRegistry requestHandler = service.getRequestHandler(action);
+        service.registerRequestHandler(getProxyAction(action), () -> new ProxyRequest(requestHandler::newRequest), ThreadPool.Names.SAME,
+            true, false, new ProxyRequestHandler<>(service, action, responseFunction));
+    }
+
+    /**
+     * Registers a proxy request handler that allows to forward requests for the given action to another node. To be used when the
+     * response type is always the same (most of the cases).
+     */
+    public static void registerProxyAction(TransportService service, String action, Supplier<TransportResponse> responseSupplier) {
+        RequestHandlerRegistry requestHandler = service.getRequestHandler(action);
+        service.registerRequestHandler(getProxyAction(action), () -> new ProxyRequest(requestHandler::newRequest), ThreadPool.Names.SAME,
+                true, false, new ProxyRequestHandler<>(service, action, request -> responseSupplier));
+    }
+
+    private static final String PROXY_ACTION_PREFIX = "internal:transport/proxy/";
+    /**
+     * Returns the corresponding proxy action for the given action
+     */
+    public static String getProxyAction(String action) {
+        return  PROXY_ACTION_PREFIX + action;
+    }
+
+    /**
+     * Wraps the actual request in a proxy request object that encodes the target node.
+     */
+    public static TransportRequest wrapRequest(DiscoveryNode node, TransportRequest request) {
+        return new ProxyRequest<>(request, node);
+    }
+
+    /**
+     * Unwraps a proxy request and returns the original request
+     */
+    public static TransportRequest unwrapRequest(TransportRequest request) {
+        if (request instanceof ProxyRequest) {
+            return ((ProxyRequest)request).wrapped;
+        }
+        return request;
+    }
+
+    /**
+     * Returns <code>true</code> iff the given action is a proxy action
+     */
+    public static boolean isProxyAction(String action) {
+        return action.startsWith(PROXY_ACTION_PREFIX);
+    }
+
+    /**
+     * Returns <code>true</code> iff the given request is a proxy request
+     */
+    public static boolean isProxyRequest(TransportRequest request) {
+        return request instanceof ProxyRequest;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/transport/TransportChannel.java b/core/src/main/java/org/elasticsearch/transport/TransportChannel.java
index d9fbcad30e0fd..dbf12f297c139 100644
--- a/core/src/main/java/org/elasticsearch/transport/TransportChannel.java
+++ b/core/src/main/java/org/elasticsearch/transport/TransportChannel.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.transport;
 
+import org.elasticsearch.Version;
+
 import java.io.IOException;
 
 /**
@@ -39,4 +41,11 @@ public interface TransportChannel {
     void sendResponse(TransportResponse response, TransportResponseOptions options) throws IOException;
 
     void sendResponse(Exception exception) throws IOException;
+
+    /**
+     * Returns the version of the other party that this channel will send a response to.
+     */
+    default Version getVersion() {
+        return Version.CURRENT;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/transport/TransportConnectionListener.java b/core/src/main/java/org/elasticsearch/transport/TransportConnectionListener.java
index ad9c7991fa4fc..56389f35dc7a9 100644
--- a/core/src/main/java/org/elasticsearch/transport/TransportConnectionListener.java
+++ b/core/src/main/java/org/elasticsearch/transport/TransportConnectionListener.java
@@ -26,7 +26,24 @@
  */
 public interface TransportConnectionListener {
 
-    void onNodeConnected(DiscoveryNode node);
+    /**
+     * Called once a node connection is opened and registered.
+     */
+    default void onNodeConnected(DiscoveryNode node) {}
 
-    void onNodeDisconnected(DiscoveryNode node);
+    /**
+     * Called once a node connection is closed and unregistered.
+     */
+    default void onNodeDisconnected(DiscoveryNode node) {}
+
+    /**
+     * Called once a node connection is closed. The connection might not have been registered in the
+     * transport as a shared connection to a specific node
+     */
+    default void onConnectionClosed(Transport.Connection connection) {}
+
+    /**
+     * Called once a node connection is opened.
+     */
+    default void onConnectionOpened(Transport.Connection connection) {}
 }
diff --git a/core/src/main/java/org/elasticsearch/transport/TransportInterceptor.java b/core/src/main/java/org/elasticsearch/transport/TransportInterceptor.java
new file mode 100644
index 0000000000000..cc90a6c6d2331
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/transport/TransportInterceptor.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.transport;
+
+import org.elasticsearch.cluster.node.DiscoveryNode;
+
+import java.util.function.Supplier;
+
+/**
+ * This interface allows plugins to intercept requests on both the sender and the receiver side.
+ */
+public interface TransportInterceptor {
+    /**
+     * This is called for each handler that is registered via
+     * {@link TransportService#registerRequestHandler(String, Supplier, String, boolean, boolean, TransportRequestHandler)} or
+     * {@link TransportService#registerRequestHandler(String, Supplier, String, TransportRequestHandler)}. The returned handler is
+     * used instead of the passed in handler. By default the provided handler is returned.
+     */
+    default <T extends TransportRequest> TransportRequestHandler<T> interceptHandler(String action, String executor,
+                                                                                     boolean forceExecution,
+                                                                                     TransportRequestHandler<T> actualHandler) {
+        return actualHandler;
+    }
+
+    /**
+     * This is called up-front providing the actual low level {@link AsyncSender} that performs the low level send request.
+     * The returned sender is used to send all requests that come in via
+     * {@link TransportService#sendRequest(DiscoveryNode, String, TransportRequest, TransportResponseHandler)} or
+     * {@link TransportService#sendRequest(DiscoveryNode, String, TransportRequest, TransportRequestOptions, TransportResponseHandler)}.
+     * This allows plugins to perform actions on each send request including modifying the request context etc.
+     */
+    default AsyncSender interceptSender(AsyncSender sender) {
+        return sender;
+    }
+
+    /**
+     * A simple interface to decorate
+     * {@link #sendRequest(Transport.Connection, String, TransportRequest, TransportRequestOptions, TransportResponseHandler)}
+     */
+    interface AsyncSender {
+        <T extends TransportResponse> void sendRequest(Transport.Connection connection, String action,
+                                                       TransportRequest request, TransportRequestOptions options,
+                                                       TransportResponseHandler<T> handler);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/transport/TransportRequest.java b/core/src/main/java/org/elasticsearch/transport/TransportRequest.java
index 83eb7cf0d81c3..89b7d890c6a52 100644
--- a/core/src/main/java/org/elasticsearch/transport/TransportRequest.java
+++ b/core/src/main/java/org/elasticsearch/transport/TransportRequest.java
@@ -21,14 +21,14 @@
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskAwareRequest;
 import org.elasticsearch.tasks.TaskId;
 
 import java.io.IOException;
 
 /**
  */
-public abstract class TransportRequest extends TransportMessage {
+public abstract class TransportRequest extends TransportMessage implements TaskAwareRequest {
     public static class Empty extends TransportRequest {
         public static final Empty INSTANCE = new Empty();
     }
@@ -41,16 +41,10 @@ public static class Empty extends TransportRequest {
     public TransportRequest() {
     }
 
-    /**
-     * Set a reference to task that caused this task to be run.
-     */
-    public void setParentTask(String parentTaskNode, long parentTaskId) {
-        setParentTask(new TaskId(parentTaskNode, parentTaskId));
-    }
-
     /**
      * Set a reference to task that created this request.
      */
+    @Override
     public void setParentTask(TaskId taskId) {
         this.parentTaskId = taskId;
     }
@@ -58,26 +52,11 @@ public void setParentTask(TaskId taskId) {
     /**
      * Get a reference to the task that created this request. Defaults to {@link TaskId#EMPTY_TASK_ID}, meaning "there is no parent".
      */
+    @Override
     public TaskId getParentTask() {
         return parentTaskId;
     }
 
-    /**
-     * Returns the task object that should be used to keep track of the processing of the request.
-     *
-     * A request can override this method and return null to avoid being tracked by the task manager.
-     */
-    public Task createTask(long id, String type, String action, TaskId parentTaskId) {
-        return new Task(id, type, action, getDescription(), parentTaskId);
-    }
-
-    /**
-     * Returns optional description of the request to be displayed by the task manager
-     */
-    public String getDescription() {
-        return "";
-    }
-
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
diff --git a/core/src/main/java/org/elasticsearch/transport/TransportService.java b/core/src/main/java/org/elasticsearch/transport/TransportService.java
index 1c807553a2428..10aa66a34d14e 100644
--- a/core/src/main/java/org/elasticsearch/transport/TransportService.java
+++ b/core/src/main/java/org/elasticsearch/transport/TransportService.java
@@ -21,14 +21,15 @@
 
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.cluster.node.liveness.TransportLivenessAction;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.logging.Loggers;
@@ -46,10 +47,13 @@
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.common.util.concurrent.FutureUtils;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskCancelledException;
 import org.elasticsearch.tasks.TaskManager;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.IOException;
+import java.net.UnknownHostException;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.LinkedHashMap;
@@ -58,37 +62,39 @@
 import java.util.Objects;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
 import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.Function;
+import java.util.function.Predicate;
 import java.util.function.Supplier;
+import java.util.stream.Stream;
 
 import static java.util.Collections.emptyList;
 import static org.elasticsearch.common.settings.Setting.listSetting;
 
-/**
- *
- */
 public class TransportService extends AbstractLifecycleComponent {
 
     public static final String DIRECT_RESPONSE_PROFILE = ".direct";
-    private static final String HANDSHAKE_ACTION_NAME = "internal:transport/handshake";
+    public static final String HANDSHAKE_ACTION_NAME = "internal:transport/handshake";
 
     private final CountDownLatch blockIncomingRequestsLatch = new CountDownLatch(1);
     protected final Transport transport;
     protected final ThreadPool threadPool;
     protected final ClusterName clusterName;
     protected final TaskManager taskManager;
+    private final TransportInterceptor.AsyncSender asyncSender;
+    private final Function<BoundTransportAddress, DiscoveryNode> localNodeFactory;
+    private final boolean connectToRemoteCluster;
 
     volatile Map<String, RequestHandlerRegistry> requestHandlers = Collections.emptyMap();
     final Object requestHandlerMutex = new Object();
 
     final ConcurrentMapLong<RequestHolder> clientHandlers = ConcurrentCollections.newConcurrentMapLongWithAggressiveConcurrency();
 
-    final AtomicLong requestIds = new AtomicLong();
-
     final CopyOnWriteArrayList<TransportConnectionListener> connectionListeners = new CopyOnWriteArrayList<>();
 
+    private final TransportInterceptor interceptor;
+
     // An LRU (don't really care about concurrency here) that holds the latest timed out requests so if they
     // do show up, we can print more descriptive information about them
     final Map<Long, TimeoutInfoHolder> timeoutInfoHandlers =
@@ -101,6 +107,8 @@ protected boolean removeEldestEntry(Map.Entry eldest) {
 
     private final TransportService.Adapter adapter;
 
+    public static final TransportInterceptor NOOP_TRANSPORT_INTERCEPTOR = new TransportInterceptor() {};
+
     // tracer log
 
     public static final Setting<List<String>> TRACE_LOG_INCLUDE_SETTING =
@@ -112,34 +120,65 @@ protected boolean removeEldestEntry(Map.Entry eldest) {
     private final Logger tracerLog;
 
     volatile String[] tracerLogInclude;
-    volatile String[] tracelLogExclude;
+    volatile String[] tracerLogExclude;
+
+    private final RemoteClusterService remoteClusterService;
 
     /** if set will call requests sent to this id to shortcut and executed locally */
     volatile DiscoveryNode localNode = null;
+    private final Transport.Connection localNodeConnection = new Transport.Connection() {
+        @Override
+        public DiscoveryNode getNode() {
+            return localNode;
+        }
 
-    @Inject
-    public TransportService(Settings settings, Transport transport, ThreadPool threadPool) {
+        @Override
+        public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options)
+            throws IOException, TransportException {
+            sendLocalRequest(requestId, action, request, options);
+        }
+
+        @Override
+        public void close() throws IOException {
+        }
+    };
+
+    /**
+     * Build the service.
+     *
+     * @param clusterSettings if non null the the {@linkplain TransportService} will register with the {@link ClusterSettings} for settings
+     *        updates for {@link #TRACE_LOG_EXCLUDE_SETTING} and {@link #TRACE_LOG_INCLUDE_SETTING}.
+     */
+    public TransportService(Settings settings, Transport transport, ThreadPool threadPool, TransportInterceptor transportInterceptor,
+                            Function<BoundTransportAddress, DiscoveryNode> localNodeFactory, @Nullable ClusterSettings clusterSettings) {
         super(settings);
         this.transport = transport;
         this.threadPool = threadPool;
+        this.localNodeFactory = localNodeFactory;
         this.clusterName = ClusterName.CLUSTER_NAME_SETTING.get(settings);
         setTracerLogInclude(TRACE_LOG_INCLUDE_SETTING.get(settings));
         setTracerLogExclude(TRACE_LOG_EXCLUDE_SETTING.get(settings));
         tracerLog = Loggers.getLogger(logger, ".tracer");
         adapter = createAdapter();
         taskManager = createTaskManager();
+        this.interceptor = transportInterceptor;
+        this.asyncSender = interceptor.interceptSender(this::sendRequestInternal);
+        this.connectToRemoteCluster = RemoteClusterService.ENABLE_REMOTE_CLUSTERS.get(settings);
+        remoteClusterService = new RemoteClusterService(settings, this);
+        if (clusterSettings != null) {
+            clusterSettings.addSettingsUpdateConsumer(TRACE_LOG_INCLUDE_SETTING, this::setTracerLogInclude);
+            clusterSettings.addSettingsUpdateConsumer(TRACE_LOG_EXCLUDE_SETTING, this::setTracerLogExclude);
+            if (connectToRemoteCluster) {
+                remoteClusterService.listenForUpdates(clusterSettings);
+            }
+        }
     }
 
-    /**
-     * makes the transport service aware of the local node. this allows it to optimize requests sent
-     * from the local node to it self and by pass the network stack/ serialization
-     */
-    public void setLocalNode(DiscoveryNode localNode) {
-        this.localNode = localNode;
+    public RemoteClusterService getRemoteClusterService() {
+        return remoteClusterService;
     }
 
-    // for testing
-    DiscoveryNode getLocalNode() {
+    public DiscoveryNode getLocalNode() {
         return localNode;
     }
 
@@ -155,19 +194,21 @@ protected TaskManager createTaskManager() {
         return new TaskManager(settings);
     }
 
-    // These need to be optional as they don't exist in the context of a transport client
-    @Inject(optional = true)
-    public void setDynamicSettings(ClusterSettings clusterSettings) {
-        clusterSettings.addSettingsUpdateConsumer(TRACE_LOG_INCLUDE_SETTING, this::setTracerLogInclude);
-        clusterSettings.addSettingsUpdateConsumer(TRACE_LOG_EXCLUDE_SETTING, this::setTracerLogExclude);
+    /**
+     * The executor service for this transport service.
+     *
+     * @return the executor service
+     */
+    protected ExecutorService getExecutorService() {
+        return threadPool.generic();
     }
 
     void setTracerLogInclude(List<String> tracerLogInclude) {
         this.tracerLogInclude = tracerLogInclude.toArray(Strings.EMPTY_ARRAY);
     }
 
-    void setTracerLogExclude(List<String> tracelLogExclude) {
-        this.tracelLogExclude = tracelLogExclude.toArray(Strings.EMPTY_ARRAY);
+    void setTracerLogExclude(List<String> tracerLogExclude) {
+        this.tracerLogExclude = tracerLogExclude.toArray(Strings.EMPTY_ARRAY);
     }
 
     @Override
@@ -176,18 +217,25 @@ protected void doStart() {
         adapter.txMetric.clear();
         transport.transportServiceAdapter(adapter);
         transport.start();
+
         if (transport.boundAddress() != null && logger.isInfoEnabled()) {
             logger.info("{}", transport.boundAddress());
             for (Map.Entry<String, BoundTransportAddress> entry : transport.profileBoundAddresses().entrySet()) {
                 logger.info("profile [{}]: {}", entry.getKey(), entry.getValue());
             }
         }
+        localNode = localNodeFactory.apply(transport.boundAddress());
         registerRequestHandler(
             HANDSHAKE_ACTION_NAME,
             () -> HandshakeRequest.INSTANCE,
             ThreadPool.Names.SAME,
+            false, false,
             (request, channel) -> channel.sendResponse(
-                    new HandshakeResponse(localNode, clusterName, localNode != null ? localNode.getVersion() : Version.CURRENT)));
+                    new HandshakeResponse(localNode, clusterName, localNode.getVersion())));
+        if (connectToRemoteCluster) {
+            // here we start to connect to the remote clusters
+            remoteClusterService.initializeRemoteClusters();
+        }
     }
 
     @Override
@@ -202,7 +250,7 @@ protected void doStop() {
                 if (holderToNotify != null) {
                     // callback that an exception happened, but on a different thread since we don't
                     // want handlers to worry about stack overflows
-                    threadPool.generic().execute(new AbstractRunnable() {
+                    getExecutorService().execute(new AbstractRunnable() {
                         @Override
                         public void onRejection(Exception e) {
                             // if we get rejected during node shutdown we don't wanna bubble it up
@@ -232,8 +280,8 @@ public void doRun() {
     }
 
     @Override
-    protected void doClose() {
-        transport.close();
+    protected void doClose() throws IOException {
+        IOUtils.close(remoteClusterService, transport);
     }
 
     /**
@@ -241,11 +289,11 @@ protected void doClose() {
      * when the transport layer starts up it will block any incoming requests until
      * this method is called
      */
-    public void acceptIncomingRequests() {
+    public final void acceptIncomingRequests() {
         blockIncomingRequestsLatch.countDown();
     }
 
-    public boolean addressSupported(Class<? extends TransportAddress> address) {
+    public final boolean addressSupported(Class<? extends TransportAddress> address) {
         return transport.addressSupported(address);
     }
 
@@ -270,110 +318,110 @@ public List<String> getLocalAddresses() {
         return transport.getLocalAddresses();
     }
 
+    /**
+     * Returns <code>true</code> iff the given node is already connected.
+     */
     public boolean nodeConnected(DiscoveryNode node) {
-        return node.equals(localNode) || transport.nodeConnected(node);
+        return isLocalNode(node) || transport.nodeConnected(node);
     }
 
     public void connectToNode(DiscoveryNode node) throws ConnectTransportException {
-        if (node.equals(localNode)) {
-            return;
-        }
-        transport.connectToNode(node);
+        connectToNode(node, null);
     }
 
     /**
-     * Lightly connect to the specified node
+     * Connect to the specified node with the given connection profile
      *
      * @param node the node to connect to
+     * @param connectionProfile the connection profile to use when connecting to this node
      */
-    public void connectToNodeLight(final DiscoveryNode node) {
-        if (node.equals(localNode)) {
+    public void connectToNode(final DiscoveryNode node, ConnectionProfile connectionProfile) {
+        if (isLocalNode(node)) {
             return;
         }
-        transport.connectToNodeLight(node);
+        transport.connectToNode(node, connectionProfile, (newConnection, actualProfile) -> {
+            // We don't validate cluster names to allow for tribe node connections.
+            final DiscoveryNode remote = handshake(newConnection, actualProfile.getHandshakeTimeout().millis(), cn -> true);
+            if (node.equals(remote) == false) {
+                throw new ConnectTransportException(node, "handshake failed. unexpected remote node " + remote);
+            }
+        });
     }
 
     /**
-     * Lightly connect to the specified node, and handshake cluster
-     * name and version
+     * Establishes and returns a new connection to the given node. The connection is NOT maintained by this service, it's the callers
+     * responsibility to close the connection once it goes out of scope.
+     * @param node the node to connect to
+     * @param profile the connection profile to use
+     */
+    public Transport.Connection openConnection(final DiscoveryNode node, ConnectionProfile profile) throws IOException {
+        if (isLocalNode(node)) {
+            return localNodeConnection;
+        } else {
+            return transport.openConnection(node, profile);
+        }
+    }
+
+    /**
+     * Executes a high-level handshake using the given connection
+     * and returns the discovery node of the node the connection
+     * was established with. The handshake will fail if the cluster
+     * name on the target node mismatches the local cluster name.
      *
-     * @param node             the node to connect to
+     * @param connection       the connection to a specific node
      * @param handshakeTimeout handshake timeout
-     * @return the connected node with version set
-     * @throws ConnectTransportException if the connection or the
-     *                                   handshake failed
+     * @return the connected node
+     * @throws ConnectTransportException if the connection failed
+     * @throws IllegalStateException if the handshake failed
      */
-    public DiscoveryNode connectToNodeLightAndHandshake(
-            final DiscoveryNode node,
+    public DiscoveryNode handshake(
+            final Transport.Connection connection,
             final long handshakeTimeout) throws ConnectTransportException {
-        return connectToNodeLightAndHandshake(node, handshakeTimeout, true);
+        return handshake(connection, handshakeTimeout, clusterName::equals);
     }
 
     /**
-     * Lightly connect to the specified node, returning updated node
-     * information. The handshake will fail if the cluster name on the
-     * target node mismatches the local cluster name and
-     * {@code checkClusterName} is {@code true}.
+     * Executes a high-level handshake using the given connection
+     * and returns the discovery node of the node the connection
+     * was established with. The handshake will fail if the cluster
+     * name on the target node doesn't match the local cluster name.
      *
-     * @param node             the node to connect to
+     * @param connection       the connection to a specific node
      * @param handshakeTimeout handshake timeout
-     * @param checkClusterName whether or not to ignore cluster name
-     *                         mismatches
+     * @param clusterNamePredicate cluster name validation predicate
      * @return the connected node
      * @throws ConnectTransportException if the connection failed
      * @throws IllegalStateException if the handshake failed
      */
-    public DiscoveryNode connectToNodeLightAndHandshake(
-            final DiscoveryNode node,
-            final long handshakeTimeout,
-            final boolean checkClusterName) {
-        if (node.equals(localNode)) {
-            return localNode;
-        }
-        transport.connectToNodeLight(node);
-        try {
-            return handshake(node, handshakeTimeout, checkClusterName);
-        } catch (ConnectTransportException | IllegalStateException e) {
-            transport.disconnectFromNode(node);
-            throw e;
-        }
-    }
-
-    private DiscoveryNode handshake(
-            final DiscoveryNode node,
-            final long handshakeTimeout,
-            final boolean checkClusterName) throws ConnectTransportException {
+    public DiscoveryNode handshake(
+        final Transport.Connection connection,
+        final long handshakeTimeout, Predicate<ClusterName> clusterNamePredicate) throws ConnectTransportException {
         final HandshakeResponse response;
+        final DiscoveryNode node = connection.getNode();
         try {
-            response = this.submitRequest(
-                node,
-                HANDSHAKE_ACTION_NAME,
-                HandshakeRequest.INSTANCE,
-                TransportRequestOptions.builder().withTimeout(handshakeTimeout).build(),
+            PlainTransportFuture<HandshakeResponse> futureHandler = new PlainTransportFuture<>(
                 new FutureTransportResponseHandler<HandshakeResponse>() {
-                    @Override
-                    public HandshakeResponse newInstance() {
-                        return new HandshakeResponse();
-                    }
-                }).txGet();
+                @Override
+                public HandshakeResponse newInstance() {
+                    return new HandshakeResponse();
+                }
+            });
+            sendRequest(connection, HANDSHAKE_ACTION_NAME, HandshakeRequest.INSTANCE,
+                TransportRequestOptions.builder().withTimeout(handshakeTimeout).build(), futureHandler);
+            response = futureHandler.txGet();
         } catch (Exception e) {
             throw new IllegalStateException("handshake failed with " + node, e);
         }
 
-        if (checkClusterName && !Objects.equals(clusterName, response.clusterName)) {
+        if (!clusterNamePredicate.test(response.clusterName)) {
             throw new IllegalStateException("handshake failed, mismatched cluster name [" + response.clusterName + "] - " + node);
-        } else if (!isVersionCompatible(response.version)) {
+        } else if (response.version.isCompatible(localNode.getVersion()) == false) {
             throw new IllegalStateException("handshake failed, incompatible version [" + response.version + "] - " + node);
         }
 
         return response.discoveryNode;
     }
 
-    private boolean isVersionCompatible(Version version) {
-        return version.minimumCompatibilityVersion().equals(
-                localNode != null ? localNode.getVersion().minimumCompatibilityVersion() : Version.CURRENT.minimumCompatibilityVersion());
-    }
-
     static class HandshakeRequest extends TransportRequest {
 
         public static final HandshakeRequest INSTANCE = new HandshakeRequest();
@@ -383,12 +431,12 @@ private HandshakeRequest() {
 
     }
 
-    static class HandshakeResponse extends TransportResponse {
+    public static class HandshakeResponse extends TransportResponse {
         private DiscoveryNode discoveryNode;
         private ClusterName clusterName;
         private Version version;
 
-        public HandshakeResponse() {
+        HandshakeResponse() {
         }
 
         public HandshakeResponse(DiscoveryNode discoveryNode, ClusterName clusterName, Version version) {
@@ -415,7 +463,7 @@ public void writeTo(StreamOutput out) throws IOException {
     }
 
     public void disconnectFromNode(DiscoveryNode node) {
-        if (node.equals(localNode)) {
+        if (isLocalNode(node)) {
             return;
         }
         transport.disconnectFromNode(node);
@@ -438,21 +486,93 @@ public <T extends TransportResponse> TransportFuture<T> submitRequest(DiscoveryN
                                                                           TransportRequestOptions options,
                                                                           TransportResponseHandler<T> handler) throws TransportException {
         PlainTransportFuture<T> futureHandler = new PlainTransportFuture<>(handler);
-        sendRequest(node, action, request, options, futureHandler);
+        try {
+            Transport.Connection connection = getConnection(node);
+            sendRequest(connection, action, request, options, futureHandler);
+        } catch (NodeNotConnectedException ex) {
+            // the caller might not handle this so we invoke the handler
+            futureHandler.handleException(ex);
+        }
         return futureHandler;
     }
 
-    public <T extends TransportResponse> void sendRequest(final DiscoveryNode node, final String action, final TransportRequest request,
-                                                          final TransportResponseHandler<T> handler) {
-        sendRequest(node, action, request, TransportRequestOptions.EMPTY, handler);
+    public <T extends TransportResponse> void sendRequest(final DiscoveryNode node, final String action,
+                                                                final TransportRequest request,
+                                                                final TransportResponseHandler<T> handler) {
+        try {
+            Transport.Connection connection = getConnection(node);
+            sendRequest(connection, action, request, TransportRequestOptions.EMPTY, handler);
+        } catch (NodeNotConnectedException ex) {
+            // the caller might not handle this so we invoke the handler
+            handler.handleException(ex);
+        }
+    }
+
+    public final <T extends TransportResponse> void sendRequest(final DiscoveryNode node, final String action,
+                                                                final TransportRequest request,
+                                                                final TransportRequestOptions options,
+                                                                TransportResponseHandler<T> handler) {
+        try {
+            Transport.Connection connection = getConnection(node);
+            sendRequest(connection, action, request, options, handler);
+        } catch (NodeNotConnectedException ex) {
+            // the caller might not handle this so we invoke the handler
+            handler.handleException(ex);
+        }
+    }
+
+    public final <T extends TransportResponse> void sendRequest(final Transport.Connection connection, final String action,
+                                                                final TransportRequest request,
+                                                                final TransportRequestOptions options,
+                                                                TransportResponseHandler<T> handler) {
+
+        asyncSender.sendRequest(connection, action, request, options, handler);
+    }
+
+    /**
+     * Returns either a real transport connection or a local node connection if we are using the local node optimization.
+     * @throws NodeNotConnectedException if the given node is not connected
+     */
+    public Transport.Connection getConnection(DiscoveryNode node) {
+        if (isLocalNode(node)) {
+            return localNodeConnection;
+        } else {
+            return transport.getConnection(node);
+        }
     }
 
-    public <T extends TransportResponse> void sendRequest(final DiscoveryNode node, final String action, final TransportRequest request,
-                                                          final TransportRequestOptions options, TransportResponseHandler<T> handler) {
-        if (node == null) {
-            throw new IllegalStateException("can't send request to a null node");
+    public <T extends TransportResponse> void sendChildRequest(final Transport.Connection connection, final String action,
+                                                               final TransportRequest request, final Task parentTask,
+                                                               final TransportResponseHandler<T> handler) {
+        sendChildRequest(connection, action, request, parentTask, TransportRequestOptions.EMPTY, handler);
+    }
+
+    public <T extends TransportResponse> void sendChildRequest(final Transport.Connection connection, final String action,
+                                                               final TransportRequest request, final Task parentTask,
+                                                               final TransportRequestOptions options,
+                                                               final TransportResponseHandler<T> handler) {
+        request.setParentTask(localNode.getId(), parentTask.getId());
+        try {
+            sendRequest(connection, action, request, options, handler);
+        } catch (TaskCancelledException ex) {
+            // The parent task is already cancelled - just fail the request
+            handler.handleException(new TransportException(ex));
+        } catch (NodeNotConnectedException ex) {
+            // the caller might not handle this so we invoke the handler
+            handler.handleException(ex);
         }
-        final long requestId = newRequestId();
+
+    }
+
+    private <T extends TransportResponse> void sendRequestInternal(final Transport.Connection connection, final String action,
+                                                                   final TransportRequest request,
+                                                                   final TransportRequestOptions options,
+                                                                   TransportResponseHandler<T> handler) {
+        if (connection == null) {
+            throw new IllegalStateException("can't send request to a null connection");
+        }
+        DiscoveryNode node = connection.getNode();
+        final long requestId = transport.newRequestId();
         final TimeoutHandler timeoutHandler;
         try {
 
@@ -461,9 +581,9 @@ public <T extends TransportResponse> void sendRequest(final DiscoveryNode node,
             } else {
                 timeoutHandler = new TimeoutHandler(requestId);
             }
-            TransportResponseHandler<T> responseHandler =
-                new ContextRestoreResponseHandler<>(threadPool.getThreadContext().newStoredContext(), handler);
-            clientHandlers.put(requestId, new RequestHolder<>(responseHandler, node, action, timeoutHandler));
+            Supplier<ThreadContext.StoredContext> storedContextSupplier = threadPool.getThreadContext().newRestorableContext(true);
+            TransportResponseHandler<T> responseHandler = new ContextRestoreResponseHandler<>(storedContextSupplier, handler);
+            clientHandlers.put(requestId, new RequestHolder<>(responseHandler, connection, action, timeoutHandler));
             if (lifecycle.stoppedOrClosed()) {
                 // if we are not started the exception handling will remove the RequestHolder again and calls the handler to notify
                 // the caller. It will only notify if the toStop code hasn't done the work yet.
@@ -473,11 +593,7 @@ public <T extends TransportResponse> void sendRequest(final DiscoveryNode node,
                 assert options.timeout() != null;
                 timeoutHandler.future = threadPool.schedule(options.timeout(), ThreadPool.Names.GENERIC, timeoutHandler);
             }
-            if (node.equals(localNode)) {
-                sendLocalRequest(requestId, action, request);
-            } else {
-                transport.sendRequest(node, requestId, action, request, options);
-            }
+            connection.sendRequest(requestId, action, request, options); // local node optimization happens upstream
         } catch (final Exception e) {
             // usually happen either because we failed to connect to the node
             // or because we failed serializing the message
@@ -511,13 +627,17 @@ protected void doRun() throws Exception {
                         holderToNotify.handler().handleException(sendRequestException);
                     }
                 });
+            } else {
+                logger.debug("Exception while sending request, handler likely already notified due to timeout", e);
             }
         }
     }
 
-    private void sendLocalRequest(long requestId, final String action, final TransportRequest request) {
+    private void sendLocalRequest(long requestId, final String action, final TransportRequest request, TransportRequestOptions options) {
         final DirectResponseChannel channel = new DirectResponseChannel(logger, localNode, action, requestId, adapter, threadPool);
         try {
+            adapter.onRequestSent(localNode, requestId, action, request, options);
+            adapter.onRequestReceived(requestId, action);
             final RequestHandlerRegistry reg = adapter.getRequestHandler(action);
             if (reg == null) {
                 throw new ActionNotFoundTransportException("Action [" + action + "] not found");
@@ -563,7 +683,6 @@ public void onFailure(Exception e) {
                         "failed to notify channel of error message for action [{}]", action), inner);
             }
         }
-
     }
 
     private boolean shouldTraceAction(String action) {
@@ -572,17 +691,13 @@ private boolean shouldTraceAction(String action) {
                 return false;
             }
         }
-        if (tracelLogExclude.length > 0) {
-            return !Regex.simpleMatch(tracelLogExclude, action);
+        if (tracerLogExclude.length > 0) {
+            return !Regex.simpleMatch(tracerLogExclude, action);
         }
         return true;
     }
 
-    private long newRequestId() {
-        return requestIds.getAndIncrement();
-    }
-
-    public TransportAddress[] addressesFromString(String address, int perAddressLimit) throws Exception {
+    public TransportAddress[] addressesFromString(String address, int perAddressLimit) throws UnknownHostException {
         return transport.addressesFromString(address, perAddressLimit);
     }
 
@@ -594,8 +709,9 @@ public TransportAddress[] addressesFromString(String address, int perAddressLimi
      * @param executor       The executor the request handling will be executed on
      * @param handler        The handler itself that implements the request handling
      */
-    public <Request extends TransportRequest> void registerRequestHandler(String action, Supplier<Request> requestFactory, String executor,
-                                                                          TransportRequestHandler<Request> handler) {
+    public <Request extends TransportRequest> void registerRequestHandler(String action, Supplier<Request> requestFactory,
+                                                    String executor, TransportRequestHandler<Request> handler) {
+        handler = interceptor.interceptHandler(action, executor, false, handler);
         RequestHandlerRegistry<Request> reg = new RequestHandlerRegistry<>(
             action, requestFactory, taskManager, handler, executor, false, true);
         registerRequestHandler(reg);
@@ -605,7 +721,7 @@ public <Request extends TransportRequest> void registerRequestHandler(String act
      * Registers a new request handler
      *
      * @param action                The action the request handler is associated with
-     * @param request               The request class that will be used to constrcut new instances for streaming
+     * @param request               The request class that will be used to construct new instances for streaming
      * @param executor              The executor the request handling will be executed on
      * @param forceExecution        Force execution on the executor queue and never reject it
      * @param canTripCircuitBreaker Check the request size and raise an exception in case the limit is breached.
@@ -615,24 +731,18 @@ public <Request extends TransportRequest> void registerRequestHandler(String act
                                                                           String executor, boolean forceExecution,
                                                                           boolean canTripCircuitBreaker,
                                                                           TransportRequestHandler<Request> handler) {
+        handler = interceptor.interceptHandler(action, executor, forceExecution, handler);
         RequestHandlerRegistry<Request> reg = new RequestHandlerRegistry<>(
             action, request, taskManager, handler, executor, forceExecution, canTripCircuitBreaker);
         registerRequestHandler(reg);
     }
 
-    protected <Request extends TransportRequest> void registerRequestHandler(RequestHandlerRegistry<Request> reg) {
+    private <Request extends TransportRequest> void registerRequestHandler(RequestHandlerRegistry<Request> reg) {
         synchronized (requestHandlerMutex) {
-            RequestHandlerRegistry replaced = requestHandlers.get(reg.getAction());
-            requestHandlers = MapBuilder.newMapBuilder(requestHandlers).put(reg.getAction(), reg).immutableMap();
-            if (replaced != null) {
-                logger.warn("registered two transport handlers for action {}, handlers: {}, {}", reg.getAction(), reg, replaced);
+            if (requestHandlers.containsKey(reg.getAction())) {
+                throw new IllegalArgumentException("transport handlers for action " + reg.getAction() + " is already registered");
             }
-        }
-    }
-
-    public void removeHandler(String action) {
-        synchronized (requestHandlerMutex) {
-            requestHandlers = MapBuilder.newMapBuilder(requestHandlers).remove(action).immutableMap();
+            requestHandlers = MapBuilder.newMapBuilder(requestHandlers).put(reg.getAction(), reg).immutableMap();
         }
     }
 
@@ -714,7 +824,7 @@ public TransportResponseHandler onResponseReceived(final long requestId) {
             }
             holder.cancelTimeout();
             if (traceEnabled() && shouldTraceAction(holder.action())) {
-                traceReceivedResponse(requestId, holder.node(), holder.action());
+                traceReceivedResponse(requestId, holder.connection().getNode(), holder.action());
             }
             return holder.handler();
         }
@@ -750,46 +860,53 @@ protected void checkForTimeout(long requestId) {
         }
 
         @Override
-        public void raiseNodeConnected(final DiscoveryNode node) {
-            threadPool.generic().execute(new Runnable() {
-                @Override
-                public void run() {
-                    for (TransportConnectionListener connectionListener : connectionListeners) {
-                        connectionListener.onNodeConnected(node);
+        public void onNodeConnected(final DiscoveryNode node) {
+            // capture listeners before spawning the background callback so the following pattern won't trigger a call
+            // connectToNode(); connection is completed successfully
+            // addConnectionListener(); this listener shouldn't be called
+            final Stream<TransportConnectionListener> listenersToNotify = TransportService.this.connectionListeners.stream();
+            getExecutorService().execute(() -> listenersToNotify.forEach(listener -> listener.onNodeConnected(node)));
+        }
+
+        @Override
+        public void onConnectionOpened(Transport.Connection connection) {
+            // capture listeners before spawning the background callback so the following pattern won't trigger a call
+            // connectToNode(); connection is completed successfully
+            // addConnectionListener(); this listener shouldn't be called
+            final Stream<TransportConnectionListener> listenersToNotify = TransportService.this.connectionListeners.stream();
+            getExecutorService().execute(() -> listenersToNotify.forEach(listener -> listener.onConnectionOpened(connection)));
+        }
+
+        @Override
+        public void onNodeDisconnected(final DiscoveryNode node) {
+            try {
+                getExecutorService().execute( () -> {
+                    for (final TransportConnectionListener connectionListener : connectionListeners) {
+                        connectionListener.onNodeDisconnected(node);
                     }
-                }
-            });
+                });
+            } catch (EsRejectedExecutionException ex) {
+                logger.debug("Rejected execution on NodeDisconnected", ex);
+            }
         }
 
         @Override
-        public void raiseNodeDisconnected(final DiscoveryNode node) {
+        public void onConnectionClosed(Transport.Connection connection) {
             try {
-                for (final TransportConnectionListener connectionListener : connectionListeners) {
-                    threadPool.generic().execute(new Runnable() {
-                        @Override
-                        public void run() {
-                            connectionListener.onNodeDisconnected(node);
-                        }
-                    });
-                }
                 for (Map.Entry<Long, RequestHolder> entry : clientHandlers.entrySet()) {
                     RequestHolder holder = entry.getValue();
-                    if (holder.node().equals(node)) {
+                    if (holder.connection().getCacheKey().equals(connection.getCacheKey())) {
                         final RequestHolder holderToNotify = clientHandlers.remove(entry.getKey());
                         if (holderToNotify != null) {
                             // callback that an exception happened, but on a different thread since we don't
                             // want handlers to worry about stack overflows
-                            threadPool.generic().execute(new Runnable() {
-                                @Override
-                                public void run() {
-                                    holderToNotify.handler().handleException(new NodeDisconnectedException(node, holderToNotify.action()));
-                                }
-                            });
+                            getExecutorService().execute(() -> holderToNotify.handler().handleException(new NodeDisconnectedException(
+                                connection.getNode(), holderToNotify.action())));
                         }
                     }
                 }
             } catch (EsRejectedExecutionException ex) {
-                logger.debug("Rejected execution on NodeDisconnected", ex);
+                logger.debug("Rejected execution on onConnectionClosed", ex);
             }
         }
 
@@ -834,13 +951,14 @@ public void run() {
             if (holder != null) {
                 // add it to the timeout information holder, in case we are going to get a response later
                 long timeoutTime = System.currentTimeMillis();
-                timeoutInfoHandlers.put(requestId, new TimeoutInfoHolder(holder.node(), holder.action(), sentTime, timeoutTime));
+                timeoutInfoHandlers.put(requestId, new TimeoutInfoHolder(holder.connection().getNode(), holder.action(), sentTime,
+                    timeoutTime));
                 // now that we have the information visible via timeoutInfoHandlers, we try to remove the request id
                 final RequestHolder removedHolder = clientHandlers.remove(requestId);
                 if (removedHolder != null) {
                     assert removedHolder == holder : "two different holder instances for request [" + requestId + "]";
                     removedHolder.handler().handleException(
-                        new ReceiveTimeoutTransportException(holder.node(), holder.action(),
+                        new ReceiveTimeoutTransportException(holder.connection().getNode(), holder.action(),
                             "request_id [" + requestId + "] timed out after [" + (timeoutTime - sentTime) + "ms]"));
                 } else {
                     // response was processed, remove timeout info.
@@ -895,15 +1013,15 @@ static class RequestHolder<T extends TransportResponse> {
 
         private final TransportResponseHandler<T> handler;
 
-        private final DiscoveryNode node;
+        private final Transport.Connection connection;
 
         private final String action;
 
         private final TimeoutHandler timeoutHandler;
 
-        RequestHolder(TransportResponseHandler<T> handler, DiscoveryNode node, String action, TimeoutHandler timeoutHandler) {
+        RequestHolder(TransportResponseHandler<T> handler, Transport.Connection connection, String action, TimeoutHandler timeoutHandler) {
             this.handler = handler;
-            this.node = node;
+            this.connection = connection;
             this.action = action;
             this.timeoutHandler = timeoutHandler;
         }
@@ -912,8 +1030,8 @@ public TransportResponseHandler<T> handler() {
             return handler;
         }
 
-        public DiscoveryNode node() {
-            return this.node;
+        public Transport.Connection connection() {
+            return this.connection;
         }
 
         public String action() {
@@ -931,13 +1049,14 @@ public void cancelTimeout() {
      * This handler wrapper ensures that the response thread executes with the correct thread context. Before any of the4 handle methods
      * are invoked we restore the context.
      */
-    private static final class ContextRestoreResponseHandler<T extends TransportResponse> implements TransportResponseHandler<T> {
+    public static final class ContextRestoreResponseHandler<T extends TransportResponse> implements TransportResponseHandler<T> {
+
         private final TransportResponseHandler<T> delegate;
-        private final ThreadContext.StoredContext threadContext;
+        private final Supplier<ThreadContext.StoredContext> contextSupplier;
 
-        private ContextRestoreResponseHandler(ThreadContext.StoredContext threadContext, TransportResponseHandler<T> delegate) {
+        public ContextRestoreResponseHandler(Supplier<ThreadContext.StoredContext> contextSupplier, TransportResponseHandler<T> delegate) {
             this.delegate = delegate;
-            this.threadContext = threadContext;
+            this.contextSupplier = contextSupplier;
         }
 
         @Override
@@ -947,20 +1066,28 @@ public T newInstance() {
 
         @Override
         public void handleResponse(T response) {
-            threadContext.restore();
-            delegate.handleResponse(response);
+            try (ThreadContext.StoredContext ignore = contextSupplier.get()) {
+                delegate.handleResponse(response);
+            }
         }
 
         @Override
         public void handleException(TransportException exp) {
-            threadContext.restore();
-            delegate.handleException(exp);
+            try (ThreadContext.StoredContext ignore = contextSupplier.get()) {
+                delegate.handleException(exp);
+            }
         }
 
         @Override
         public String executor() {
             return delegate.executor();
         }
+
+        @Override
+        public String toString() {
+            return getClass().getName() + "/" + delegate.toString();
+        }
+
     }
 
     static class DirectResponseChannel implements TransportChannel {
@@ -971,7 +1098,7 @@ static class DirectResponseChannel implements TransportChannel {
         final TransportServiceAdapter adapter;
         final ThreadPool threadPool;
 
-        public DirectResponseChannel(Logger logger, DiscoveryNode localNode, String action, long requestId,
+        DirectResponseChannel(Logger logger, DiscoveryNode localNode, String action, long requestId,
                                      TransportServiceAdapter adapter, ThreadPool threadPool) {
             this.logger = logger;
             this.localNode = localNode;
@@ -998,6 +1125,7 @@ public void sendResponse(TransportResponse response) throws IOException {
 
         @Override
         public void sendResponse(final TransportResponse response, TransportResponseOptions options) throws IOException {
+            adapter.onResponseSent(requestId, action, response, options);
             final TransportResponseHandler handler = adapter.onResponseReceived(requestId);
             // ignore if its null, the adapter logs it
             if (handler != null) {
@@ -1005,13 +1133,7 @@ public void sendResponse(final TransportResponse response, TransportResponseOpti
                 if (ThreadPool.Names.SAME.equals(executor)) {
                     processResponse(handler, response);
                 } else {
-                    threadPool.executor(executor).execute(new Runnable() {
-                        @SuppressWarnings({"unchecked"})
-                        @Override
-                        public void run() {
-                            processResponse(handler, response);
-                        }
-                    });
+                    threadPool.executor(executor).execute(() -> processResponse(handler, response));
                 }
             }
         }
@@ -1027,6 +1149,7 @@ protected void processResponse(TransportResponseHandler handler, TransportRespon
 
         @Override
         public void sendResponse(Exception exception) throws IOException {
+            adapter.onResponseSent(requestId, action, exception);
             final TransportResponseHandler handler = adapter.onResponseReceived(requestId);
             // ignore if its null, the adapter logs it
             if (handler != null) {
@@ -1073,5 +1196,20 @@ public String getChannelType() {
             return "direct";
         }
 
+        @Override
+        public Version getVersion() {
+            return localNode.getVersion();
+        }
+    }
+
+    /**
+     * Returns the internal thread pool
+     */
+    public ThreadPool getThreadPool() {
+        return threadPool;
+    }
+
+    private boolean isLocalNode(DiscoveryNode discoveryNode) {
+        return Objects.requireNonNull(discoveryNode, "discovery node must not be null").equals(localNode);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/transport/TransportServiceAdapter.java b/core/src/main/java/org/elasticsearch/transport/TransportServiceAdapter.java
index 34910532a0289..70748b01a6802 100644
--- a/core/src/main/java/org/elasticsearch/transport/TransportServiceAdapter.java
+++ b/core/src/main/java/org/elasticsearch/transport/TransportServiceAdapter.java
@@ -21,7 +21,7 @@
 
 import org.elasticsearch.cluster.node.DiscoveryNode;
 
-public interface TransportServiceAdapter {
+public interface TransportServiceAdapter extends TransportConnectionListener {
 
     void addBytesReceived(long size);
 
@@ -50,9 +50,4 @@ public interface TransportServiceAdapter {
     void onRequestReceived(long requestId, String action);
 
     RequestHandlerRegistry getRequestHandler(String action);
-
-    void raiseNodeConnected(DiscoveryNode node);
-
-    void raiseNodeDisconnected(DiscoveryNode node);
-
 }
diff --git a/core/src/main/java/org/elasticsearch/transport/support/TransportStatus.java b/core/src/main/java/org/elasticsearch/transport/TransportStatus.java
similarity index 79%
rename from core/src/main/java/org/elasticsearch/transport/support/TransportStatus.java
rename to core/src/main/java/org/elasticsearch/transport/TransportStatus.java
index 87494456ff9b2..2f5f6d6bd9bb5 100644
--- a/core/src/main/java/org/elasticsearch/transport/support/TransportStatus.java
+++ b/core/src/main/java/org/elasticsearch/transport/TransportStatus.java
@@ -17,15 +17,14 @@
  * under the License.
  */
 
-package org.elasticsearch.transport.support;
+package org.elasticsearch.transport;
 
-/**
- */
-public class TransportStatus {
+public final class TransportStatus {
 
     private static final byte STATUS_REQRES = 1 << 0;
     private static final byte STATUS_ERROR = 1 << 1;
     private static final byte STATUS_COMPRESS = 1 << 2;
+    private static final byte STATUS_HANDSHAKE = 1 << 3;
 
     public static boolean isRequest(byte value) {
         return (value & STATUS_REQRES) == 0;
@@ -58,4 +57,15 @@ public static byte setCompress(byte value) {
         value |= STATUS_COMPRESS;
         return value;
     }
+
+    static boolean isHandshake(byte value) { // pkg private since it's only used internally
+        return (value & STATUS_HANDSHAKE) != 0;
+    }
+
+    static byte setHandshake(byte value) { // pkg private since it's only used internally
+        value |= STATUS_HANDSHAKE;
+        return value;
+    }
+
+
 }
diff --git a/core/src/main/java/org/elasticsearch/transport/local/LocalTransport.java b/core/src/main/java/org/elasticsearch/transport/local/LocalTransport.java
index e556ec9676a61..90b498ae97ee1 100644
--- a/core/src/main/java/org/elasticsearch/transport/local/LocalTransport.java
+++ b/core/src/main/java/org/elasticsearch/transport/local/LocalTransport.java
@@ -24,12 +24,12 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.CheckedBiConsumer;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.component.Lifecycle;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -46,6 +46,7 @@
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.ActionNotFoundTransportException;
 import org.elasticsearch.transport.ConnectTransportException;
+import org.elasticsearch.transport.ConnectionProfile;
 import org.elasticsearch.transport.NodeNotConnectedException;
 import org.elasticsearch.transport.RemoteTransportException;
 import org.elasticsearch.transport.RequestHandlerRegistry;
@@ -58,8 +59,8 @@
 import org.elasticsearch.transport.TransportResponseHandler;
 import org.elasticsearch.transport.TransportSerializationException;
 import org.elasticsearch.transport.TransportServiceAdapter;
+import org.elasticsearch.transport.TransportStatus;
 import org.elasticsearch.transport.Transports;
-import org.elasticsearch.transport.support.TransportStatus;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -86,12 +87,12 @@ public class LocalTransport extends AbstractLifecycleComponent implements Transp
     private final ConcurrentMap<DiscoveryNode, LocalTransport> connectedNodes = newConcurrentMap();
     protected final NamedWriteableRegistry namedWriteableRegistry;
     private final CircuitBreakerService circuitBreakerService;
+    private final AtomicLong requestIdGenerator = new AtomicLong();
 
     public static final String TRANSPORT_LOCAL_ADDRESS = "transport.local.address";
     public static final String TRANSPORT_LOCAL_WORKERS = "transport.local.workers";
     public static final String TRANSPORT_LOCAL_QUEUE = "transport.local.queue";
 
-    @Inject
     public LocalTransport(Settings settings, ThreadPool threadPool,
                           NamedWriteableRegistry namedWriteableRegistry, CircuitBreakerService circuitBreakerService) {
         super(settings);
@@ -169,12 +170,9 @@ public boolean nodeConnected(DiscoveryNode node) {
     }
 
     @Override
-    public void connectToNodeLight(DiscoveryNode node) throws ConnectTransportException {
-        connectToNode(node);
-    }
-
-    @Override
-    public void connectToNode(DiscoveryNode node) throws ConnectTransportException {
+    public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile,
+                              CheckedBiConsumer<Connection, ConnectionProfile, IOException> connectionValidator)
+        throws ConnectTransportException {
         synchronized (this) {
             if (connectedNodes.containsKey(node)) {
                 return;
@@ -184,7 +182,7 @@ public void connectToNode(DiscoveryNode node) throws ConnectTransportException {
                 throw new ConnectTransportException(node, "Failed to connect");
             }
             connectedNodes.put(node, targetTransport);
-            transportServiceAdapter.raiseNodeConnected(node);
+            transportServiceAdapter.onNodeConnected(node);
         }
     }
 
@@ -193,7 +191,9 @@ public void disconnectFromNode(DiscoveryNode node) {
         synchronized (this) {
             LocalTransport removed = connectedNodes.remove(node);
             if (removed != null) {
-                transportServiceAdapter.raiseNodeDisconnected(node);
+                transportServiceAdapter.onNodeDisconnected(node);
+                // here we notify that the connection to the node is closed -- use the remove transport as cache key
+                transportServiceAdapter.onConnectionClosed(getConnectionForTransport(removed, node, true));
             }
         }
     }
@@ -204,9 +204,63 @@ public long serverOpen() {
     }
 
     @Override
-    public void sendRequest(final DiscoveryNode node, final long requestId, final String action, final TransportRequest request,
+    public Connection getConnection(DiscoveryNode node) {
+        final LocalTransport targetTransport = connectedNodes.get(node);
+        if (targetTransport == null) {
+            throw new NodeNotConnectedException(node, "Node not connected");
+        }
+        return getConnectionForTransport(targetTransport, node, true);
+    }
+
+    @Override
+    public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) throws IOException {
+        final LocalTransport targetTransport = transports.get(node.getAddress());
+        if (targetTransport == null) {
+            throw new ConnectTransportException(node, "Failed to connect");
+        }
+        return getConnectionForTransport(targetTransport, node, false);
+    }
+
+    private Connection getConnectionForTransport(LocalTransport targetTransport, DiscoveryNode node, boolean registered) {
+        return new Connection() {
+            @Override
+            public DiscoveryNode getNode() {
+                return node;
+            }
+
+            @Override
+            public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options)
+                throws IOException, TransportException {
+                if (transports.get(node.getAddress()) != targetTransport) {
+                    // we double check that we are still connected since due to the nature of local transport we won't get any errors
+                    // when we send to a closed / disconnected transport. Yet, if the target transport is closed concurrently before
+                    // the transport service registers the client handler local transport will simply ignore the request which can cause
+                    // requests to hang. By checking again here we guarantee to throw an exception which causes the client handler to run
+                    // in such a situation, if the client handler is not registered yet when the connections is obtained.
+                    throw new NodeNotConnectedException(node, " got disconnected");
+                }
+                LocalTransport.this.sendRequest(targetTransport, node, requestId, action, request, options);
+            }
+
+            @Override
+            public Object getCacheKey() {
+                // the cache key depends if it's registered connection or a single connection (not shared)
+                // we use the actual LocalTransport if it's a regsitered connection otherwise we use the identity
+                return registered ? targetTransport : this;
+            }
+
+            @Override
+            public void close() throws IOException {
+                transportServiceAdapter.onConnectionClosed(this);
+                LocalTransport.this.disconnectFromNode(node);
+            }
+        };
+    }
+
+    protected void sendRequest(LocalTransport targetTransport, final DiscoveryNode node, final long requestId, final String action,
+                             final TransportRequest request,
             TransportRequestOptions options) throws IOException, TransportException {
-        final Version version = Version.smallest(node.getVersion(), getVersion());
+        final Version version = Version.min(node.getVersion(), getVersion());
 
         try (BytesStreamOutput stream = new BytesStreamOutput()) {
             stream.setVersion(version);
@@ -222,11 +276,6 @@ public void sendRequest(final DiscoveryNode node, final long requestId, final St
 
             stream.close();
 
-            final LocalTransport targetTransport = connectedNodes.get(node);
-            if (targetTransport == null) {
-                throw new NodeNotConnectedException(node, "Node not connected");
-            }
-
             final byte[] data = BytesReference.toBytes(stream.bytes());
             transportServiceAdapter.addBytesSent(data.length);
             transportServiceAdapter.onRequestSent(node, requestId, action, request, options);
@@ -426,6 +475,11 @@ public List<String> getLocalAddresses() {
         return Collections.singletonList("0.0.0.0");
     }
 
+    @Override
+    public long newRequestId() {
+        return requestIdGenerator.incrementAndGet();
+    }
+
     protected Version getVersion() { // for tests
         return Version.CURRENT;
     }
diff --git a/core/src/main/java/org/elasticsearch/transport/local/LocalTransportChannel.java b/core/src/main/java/org/elasticsearch/transport/local/LocalTransportChannel.java
index 0c1e8747a121a..674e9532dd74b 100644
--- a/core/src/main/java/org/elasticsearch/transport/local/LocalTransportChannel.java
+++ b/core/src/main/java/org/elasticsearch/transport/local/LocalTransportChannel.java
@@ -28,14 +28,11 @@
 import org.elasticsearch.transport.TransportResponse;
 import org.elasticsearch.transport.TransportResponseOptions;
 import org.elasticsearch.transport.TransportServiceAdapter;
-import org.elasticsearch.transport.support.TransportStatus;
+import org.elasticsearch.transport.TransportStatus;
 
 import java.io.IOException;
 import java.util.concurrent.atomic.AtomicBoolean;
 
-/**
- *
- */
 public class LocalTransportChannel implements TransportChannel {
 
     private static final String LOCAL_TRANSPORT_PROFILE = "default";
@@ -97,6 +94,7 @@ public void sendResponse(TransportResponse response, TransportResponseOptions op
     @Override
     public void sendResponse(Exception exception) throws IOException {
         BytesStreamOutput stream = new BytesStreamOutput();
+        stream.setVersion(version);
         writeResponseExceptionHeader(stream);
         RemoteTransportException tx = new RemoteTransportException(targetTransport.nodeName(),
                 targetTransport.boundAddress().boundAddresses()[0], action, exception);
@@ -128,6 +126,11 @@ public String getChannelType() {
         return "local";
     }
 
+    @Override
+    public Version getVersion() {
+        return version;
+    }
+
     private void writeResponseExceptionHeader(BytesStreamOutput stream) throws IOException {
         stream.writeLong(requestId);
         byte status = 0;
diff --git a/core/src/main/java/org/elasticsearch/tribe/TribeClientNode.java b/core/src/main/java/org/elasticsearch/tribe/TribeClientNode.java
deleted file mode 100644
index 02957ab99a4cf..0000000000000
--- a/core/src/main/java/org/elasticsearch/tribe/TribeClientNode.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.tribe;
-
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.node.Node;
-import org.elasticsearch.plugins.Plugin;
-
-import java.util.Collections;
-
-/**
- * An internal node that connects to a remove cluster, as part of a tribe node.
- */
-class TribeClientNode extends Node {
-    TribeClientNode(Settings settings) {
-        super(new Environment(settings), Collections.<Class<? extends Plugin>>emptyList());
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/tribe/TribeService.java b/core/src/main/java/org/elasticsearch/tribe/TribeService.java
index 99f6c696e6624..821499cc20775 100644
--- a/core/src/main/java/org/elasticsearch/tribe/TribeService.java
+++ b/core/src/main/java/org/elasticsearch/tribe/TribeService.java
@@ -44,6 +44,12 @@
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.hash.MurmurHash3;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.regex.Regex;
@@ -61,7 +67,9 @@
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.transport.TransportSettings;
 
+import java.io.IOException;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
@@ -70,6 +78,7 @@
 import java.util.Set;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.function.Function;
+import java.util.stream.Collectors;
 
 import static java.util.Collections.unmodifiableMap;
 
@@ -94,9 +103,9 @@
 public class TribeService extends AbstractLifecycleComponent {
 
     public static final ClusterBlock TRIBE_METADATA_BLOCK = new ClusterBlock(10, "tribe node, metadata not allowed", false, false,
-            RestStatus.BAD_REQUEST, EnumSet.of(ClusterBlockLevel.METADATA_READ, ClusterBlockLevel.METADATA_WRITE));
+        false, RestStatus.BAD_REQUEST, EnumSet.of(ClusterBlockLevel.METADATA_READ, ClusterBlockLevel.METADATA_WRITE));
     public static final ClusterBlock TRIBE_WRITE_BLOCK = new ClusterBlock(11, "tribe node, write not allowed", false, false,
-            RestStatus.BAD_REQUEST, EnumSet.of(ClusterBlockLevel.WRITE));
+        false, RestStatus.BAD_REQUEST, EnumSet.of(ClusterBlockLevel.WRITE));
 
     public static Settings processSettings(Settings settings) {
         if (TRIBE_NAME_SETTING.exists(settings)) {
@@ -122,7 +131,7 @@ public static Settings processSettings(Settings settings) {
         if (!NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.exists(settings)) {
             sb.put(NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.getKey(), nodesSettings.size());
         }
-        sb.put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), "local"); // a tribe node should not use zen discovery
+        sb.put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), "none"); // a tribe node should not use zen discovery
         // nothing is going to be discovered, since no master will be elected
         sb.put(DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING.getKey(), 0);
         if (sb.get("cluster.name") == null) {
@@ -132,6 +141,29 @@ public static Settings processSettings(Settings settings) {
         return sb.build();
     }
 
+    /**
+     * Interface to allow merging {@link org.elasticsearch.cluster.metadata.MetaData.Custom} in tribe node
+     * When multiple Mergable Custom metadata of the same type is found (from underlying clusters), the
+     * Custom metadata will be merged using {@link #merge(MetaData.Custom)} and the result will be stored
+     * in the tribe cluster state
+     *
+     * @param <T> type of custom meta data
+     */
+    public interface MergableCustomMetaData<T extends MetaData.Custom> {
+
+        /**
+         * Merges this custom metadata with other, returning either this or <code>other</code> custom metadata
+         * for tribe cluster state. This method should not mutate either <code>this</code> or the
+         * <code>other</code> custom metadata.
+         *
+         * @param other custom meta data
+         * @return the same instance or <code>other</code> custom metadata based on implementation
+         *         if both the instances are considered equal, implementations should return this
+         *         instance to avoid redundant cluster state changes.
+         */
+        T merge(T other);
+    }
+
     // internal settings only
     public static final Setting<String> TRIBE_NAME_SETTING = Setting.simpleString("tribe.name", Property.NodeScope);
     private final ClusterService clusterService;
@@ -182,15 +214,19 @@ public static Settings processSettings(Settings settings) {
 
     private final List<Node> nodes = new CopyOnWriteArrayList<>();
 
-    public TribeService(Settings settings, ClusterService clusterService, final String tribeNodeId) {
+    private final NamedWriteableRegistry namedWriteableRegistry;
+
+    public TribeService(Settings settings, ClusterService clusterService, final String tribeNodeId,
+                        NamedWriteableRegistry namedWriteableRegistry, Function<Settings, Node> clientNodeBuilder) {
         super(settings);
         this.clusterService = clusterService;
+        this.namedWriteableRegistry = namedWriteableRegistry;
         Map<String, Settings> nodesSettings = new HashMap<>(settings.getGroups("tribe", true));
         nodesSettings.remove("blocks"); // remove prefix settings that don't indicate a client
         nodesSettings.remove("on_conflict"); // remove prefix settings that don't indicate a client
         for (Map.Entry<String, Settings> entry : nodesSettings.entrySet()) {
             Settings clientSettings = buildClientSettings(entry.getKey(), tribeNodeId, settings, entry.getValue());
-            nodes.add(new TribeClientNode(clientSettings));
+            nodes.add(clientNodeBuilder.apply(clientSettings));
         }
 
         this.blockIndicesMetadata = BLOCKS_METADATA_INDICES_SETTING.get(settings).toArray(Strings.EMPTY_ARRAY);
@@ -204,6 +240,8 @@ public TribeService(Settings settings, ClusterService clusterService, final Stri
             if (BLOCKS_METADATA_SETTING.get(settings)) {
                 clusterService.addInitialStateBlock(TRIBE_METADATA_BLOCK);
             }
+            new DeprecationLogger(Loggers.getLogger(TribeService.class))
+                .deprecated("tribe nodes are deprecated in favor of cross-cluster search and will be removed in Elasticsearch 7.0.0");
         }
 
         this.onConflict = ON_CONFLICT_SETTING.get(settings);
@@ -267,7 +305,7 @@ protected void doStart() {
     public void startNodes() {
         for (Node node : nodes) {
             try {
-                node.injector().getInstance(ClusterService.class).add(new TribeClusterStateListener(node));
+                getClusterService(node).addListener(new TribeClusterStateListener(node));
                 node.start();
             } catch (Exception e) {
                 // calling close is safe for non started nodes, we can just iterate over all
@@ -333,35 +371,31 @@ class TribeNodeClusterStateTaskExecutor implements ClusterStateTaskExecutor<Clus
             this.tribeName = tribeName;
         }
 
-        @Override
-        public boolean runOnlyOnMaster() {
-            return false;
-        }
-
         @Override
         public String describeTasks(List<ClusterChangedEvent> tasks) {
             return tasks.stream().map(ClusterChangedEvent::source).reduce((s1, s2) -> s1 + ", " + s2).orElse("");
         }
 
         @Override
-        public BatchResult<ClusterChangedEvent> execute(ClusterState currentState, List<ClusterChangedEvent> tasks) throws Exception {
-            ClusterState accumulator = ClusterState.builder(currentState).build();
-            BatchResult.Builder<ClusterChangedEvent> builder = BatchResult.builder();
-
-            try {
-                // we only need to apply the latest cluster state update
-                accumulator = applyUpdate(accumulator, tasks.get(tasks.size() - 1));
-                builder.successes(tasks);
-            } catch (Exception e) {
-                builder.failures(tasks, e);
-            }
+        public boolean runOnlyOnMaster() {
+            return false;
+        }
 
-            return builder.build(accumulator);
+        @Override
+        public ClusterTasksResult<ClusterChangedEvent> execute(ClusterState currentState, List<ClusterChangedEvent> tasks) throws Exception {
+            ClusterTasksResult.Builder<ClusterChangedEvent> builder = ClusterTasksResult.builder();
+            ClusterState.Builder newState = ClusterState.builder(currentState).incrementVersion();
+            boolean clusterStateChanged = updateNodes(currentState, tasks, newState);
+            clusterStateChanged |= updateIndicesAndMetaData(currentState, tasks, newState);
+            builder.successes(tasks);
+            return builder.build(clusterStateChanged ? newState.build() : currentState);
         }
 
-        private ClusterState applyUpdate(ClusterState currentState, ClusterChangedEvent task) {
+        private boolean updateNodes(ClusterState currentState, List<ClusterChangedEvent> tasks, ClusterState.Builder newState) {
             boolean clusterStateChanged = false;
-            ClusterState tribeState = task.state();
+            // we only need to apply the latest cluster state update
+            ClusterChangedEvent latestTask = tasks.get(tasks.size() - 1);
+            ClusterState tribeState = latestTask.state();
             DiscoveryNodes.Builder nodes = DiscoveryNodes.builder(currentState.nodes());
             // -- merge nodes
             // go over existing nodes, and see if they need to be removed
@@ -382,16 +416,25 @@ private ClusterState applyUpdate(ClusterState currentState, ClusterChangedEvent
                     Map<String, String> tribeAttr = new HashMap<>(tribe.getAttributes());
                     tribeAttr.put(TRIBE_NAME_SETTING.getKey(), tribeName);
                     DiscoveryNode discoNode = new DiscoveryNode(tribe.getName(), tribe.getId(), tribe.getEphemeralId(),
-                        tribe.getHostName(), tribe.getHostAddress(), tribe.getAddress(), unmodifiableMap(tribeAttr), tribe.getRoles(),
-                        tribe.getVersion());
+                            tribe.getHostName(), tribe.getHostAddress(), tribe.getAddress(), unmodifiableMap(tribeAttr), tribe.getRoles(),
+                            tribe.getVersion());
                     clusterStateChanged = true;
                     logger.info("[{}] adding node [{}]", tribeName, discoNode);
                     nodes.remove(tribe.getId()); // remove any existing node with the same id but different ephemeral id
                     nodes.add(discoNode);
                 }
             }
+            if (clusterStateChanged) {
+                newState.nodes(nodes);
+            }
+            return clusterStateChanged;
+        }
 
-            // -- merge metadata
+        private boolean updateIndicesAndMetaData(ClusterState currentState, List<ClusterChangedEvent> tasks, ClusterState.Builder newState) {
+            // we only need to apply the latest cluster state update
+            ClusterChangedEvent latestTask = tasks.get(tasks.size() - 1);
+            ClusterState tribeState = latestTask.state();
+            boolean clusterStateChanged = false;
             ClusterBlocks.Builder blocks = ClusterBlocks.builder().blocks(currentState.blocks());
             MetaData.Builder metaData = MetaData.builder(currentState.metaData());
             RoutingTable.Builder routingTable = RoutingTable.builder(currentState.routingTable());
@@ -459,13 +502,49 @@ private ClusterState applyUpdate(ClusterState currentState, ClusterChangedEvent
                     }
                 }
             }
+            clusterStateChanged |= updateCustoms(currentState, tasks, metaData);
+            if (clusterStateChanged) {
+                newState.blocks(blocks);
+                newState.metaData(metaData);
+                newState.routingTable(routingTable.build());
+            }
+            return clusterStateChanged;
+        }
 
-            if (!clusterStateChanged) {
-                return currentState;
-            } else {
-                return ClusterState.builder(currentState).incrementVersion().blocks(blocks).nodes(nodes).metaData(metaData)
-                        .routingTable(routingTable.build()).build();
+        private boolean updateCustoms(ClusterState currentState, List<ClusterChangedEvent> tasks, MetaData.Builder metaData) {
+            boolean clusterStateChanged = false;
+            Set<String> changedCustomMetaDataTypeSet = tasks.stream()
+                    .map(ClusterChangedEvent::changedCustomMetaDataSet)
+                    .flatMap(Collection::stream)
+                    .collect(Collectors.toSet());
+            final List<Node> tribeClientNodes = TribeService.this.nodes;
+            Map<String, MetaData.Custom> mergedCustomMetaDataMap = mergeChangedCustomMetaData(changedCustomMetaDataTypeSet,
+                    customMetaDataType -> tribeClientNodes.stream()
+                            .map(TribeService::getClusterService).map(ClusterService::state)
+                            .map(ClusterState::metaData)
+                            .map(clusterMetaData -> ((MetaData.Custom) clusterMetaData.custom(customMetaDataType)))
+                            .filter(custom1 -> custom1 != null && custom1 instanceof MergableCustomMetaData)
+                            .map(custom2 -> (MergableCustomMetaData) marshal(custom2))
+                            .collect(Collectors.toList())
+            );
+            for (String changedCustomMetaDataType : changedCustomMetaDataTypeSet) {
+                MetaData.Custom mergedCustomMetaData = mergedCustomMetaDataMap.get(changedCustomMetaDataType);
+                if (mergedCustomMetaData == null) {
+                    // we ignore merging custom md which doesn't implement MergableCustomMetaData interface
+                    if (currentState.metaData().custom(changedCustomMetaDataType) instanceof MergableCustomMetaData) {
+                        // custom md has been removed
+                        clusterStateChanged = true;
+                        logger.info("[{}] removing custom meta data type [{}]", tribeName, changedCustomMetaDataType);
+                        metaData.removeCustom(changedCustomMetaDataType);
+                    }
+                } else {
+                    // custom md has been changed
+                    clusterStateChanged = true;
+                    logger.info("[{}] updating custom meta data type [{}] data [{}]", tribeName, changedCustomMetaDataType, mergedCustomMetaData);
+                    metaData.putCustom(changedCustomMetaDataType, mergedCustomMetaData);
+                }
             }
+            return clusterStateChanged;
         }
 
         private void removeIndex(ClusterBlocks.Builder blocks, MetaData.Builder metaData, RoutingTable.Builder routingTable,
@@ -491,4 +570,40 @@ private void addNewIndex(ClusterState tribeState, ClusterBlocks.Builder blocks,
             }
         }
     }
+
+    private static ClusterService getClusterService(Node node) {
+        return node.injector().getInstance(ClusterService.class);
+    }
+
+    // pkg-private for testing
+    static Map<String, MetaData.Custom> mergeChangedCustomMetaData(Set<String> changedCustomMetaDataTypeSet,
+                                                                   Function<String, List<MergableCustomMetaData>> customMetaDataByTribeNode) {
+
+        Map<String, MetaData.Custom> changedCustomMetaDataMap = new HashMap<>(changedCustomMetaDataTypeSet.size());
+        for (String customMetaDataType : changedCustomMetaDataTypeSet) {
+            customMetaDataByTribeNode.apply(customMetaDataType).stream()
+                    .reduce((mergableCustomMD, mergableCustomMD2) ->
+                            ((MergableCustomMetaData) mergableCustomMD.merge((MetaData.Custom) mergableCustomMD2)))
+                    .ifPresent(mergedCustomMetaData ->
+                            changedCustomMetaDataMap.put(customMetaDataType, ((MetaData.Custom) mergedCustomMetaData)));
+        }
+        return changedCustomMetaDataMap;
+    }
+
+    /**
+     * Since custom metadata can be loaded by a plugin class loader that resides in a sub-node, we need to
+     * marshal this object into something the tribe node can work with
+     */
+    private MetaData.Custom marshal(MetaData.Custom custom)  {
+        try (BytesStreamOutput bytesStreamOutput = new BytesStreamOutput()){
+            bytesStreamOutput.writeNamedWriteable(custom);
+            try(StreamInput input = bytesStreamOutput.bytes().streamInput()) {
+                StreamInput namedInput = new NamedWriteableAwareStreamInput(input, namedWriteableRegistry);
+                MetaData.Custom marshaled = namedInput.readNamedWriteable(MetaData.Custom.class);
+                return marshaled;
+            }
+        } catch (IOException ex) {
+            throw new IllegalStateException("cannot marshal object with type " + custom.getWriteableName() + " to tribe node");
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/watcher/FileWatcher.java b/core/src/main/java/org/elasticsearch/watcher/FileWatcher.java
index 8d5c04e770890..0b0504f460908 100644
--- a/core/src/main/java/org/elasticsearch/watcher/FileWatcher.java
+++ b/core/src/main/java/org/elasticsearch/watcher/FileWatcher.java
@@ -80,7 +80,7 @@ private class FileObserver {
         private boolean isDirectory;
         private FileObserver[] children;
 
-        public FileObserver(Path file) {
+        FileObserver(Path file) {
             this.file = file;
         }
 
diff --git a/core/src/main/java/org/joda/time/format/StrictISODateTimeFormat.java b/core/src/main/java/org/joda/time/format/StrictISODateTimeFormat.java
index a417fa8f10352..90507b65f5fd2 100644
--- a/core/src/main/java/org/joda/time/format/StrictISODateTimeFormat.java
+++ b/core/src/main/java/org/joda/time/format/StrictISODateTimeFormat.java
@@ -32,7 +32,7 @@
  * All methods have been marked with an "// ES change" commentary
  *
  * In case you compare this with the original ISODateTimeFormat, make sure you use a diff
- * call, that ignores whitespaces/tabs/indendetations like 'diff -b'
+ * call, that ignores whitespaces/tabs/indentations like 'diff -b'
  */
 /**
  * Factory that creates instances of DateTimeFormatter based on the ISO8601 standard.
diff --git a/core/src/main/resources/config/favicon.ico b/core/src/main/resources/config/favicon.ico
index 0eabd4c1edbe4..41320478b5fef 100644
Binary files a/core/src/main/resources/config/favicon.ico and b/core/src/main/resources/config/favicon.ico differ
diff --git a/core/src/main/resources/org/elasticsearch/bootstrap/security.policy b/core/src/main/resources/org/elasticsearch/bootstrap/security.policy
index 97ccfb31bf2b2..e60b4cc7aade9 100644
--- a/core/src/main/resources/org/elasticsearch/bootstrap/security.policy
+++ b/core/src/main/resources/org/elasticsearch/bootstrap/security.policy
@@ -31,7 +31,7 @@ grant codeBase "${codebase.securesm-1.1.jar}" {
 //// Very special jar permissions:
 //// These are dangerous permissions that we don't want to grant to everything.
 
-grant codeBase "${codebase.lucene-core-6.2.0.jar}" {
+grant codeBase "${codebase.lucene-core-6.6.1.jar}" {
   // needed to allow MMapDirectory's "unmap hack" (die unmap hack, die)
   // java 8 package
   permission java.lang.RuntimePermission "accessClassInPackage.sun.misc";
@@ -42,7 +42,7 @@ grant codeBase "${codebase.lucene-core-6.2.0.jar}" {
   permission java.lang.RuntimePermission "accessDeclaredMembers";
 };
 
-grant codeBase "${codebase.lucene-misc-6.2.0.jar}" {
+grant codeBase "${codebase.lucene-misc-6.6.1.jar}" {
   // needed to allow shard shrinking to use hard-links if possible via lucenes HardlinkCopyDirectoryWrapper
   permission java.nio.file.LinkPermission "hard";
 };
@@ -92,10 +92,10 @@ grant {
   permission java.lang.RuntimePermission "modifyThreadGroup";
 
   // needed by ExceptionSerializationTests and RestTestCase for
-  // some hackish things they do. otherwise only needed by groovy 
+  // some hackish things they do. otherwise only needed by groovy
   // (TODO: clean this up?)
   permission java.lang.RuntimePermission "getProtectionDomain";
-  
+
   // needed by HotThreads and potentially more
   // otherwise can be provided only to test libraries
   permission java.lang.RuntimePermission "getStackTrace";
@@ -116,12 +116,16 @@ grant {
   // load averages on Linux
   permission java.io.FilePermission "/proc/loadavg", "read";
 
-  // load averages on FreeBSD
-  permission java.io.FilePermission "/compat/linux/proc/loadavg", "read";
-
   // read max virtual memory areas
   permission java.io.FilePermission "/proc/sys/vm/max_map_count", "read";
 
   // io stats on Linux
   permission java.io.FilePermission "/proc/diskstats", "read";
+
+  // control group stats on Linux
+  permission java.io.FilePermission "/proc/self/cgroup", "read";
+  permission java.io.FilePermission "/sys/fs/cgroup/cpu", "read";
+  permission java.io.FilePermission "/sys/fs/cgroup/cpu/-", "read";
+  permission java.io.FilePermission "/sys/fs/cgroup/cpuacct", "read";
+  permission java.io.FilePermission "/sys/fs/cgroup/cpuacct/-", "read";
 };
diff --git a/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy b/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
index 43f6b62c3c33c..686119da3290e 100644
--- a/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
+++ b/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
- 
+
 //// additional test framework permissions.
 //// These are mock objects and test management that we allow test framework libs
 //// to provide on our behalf. But tests themselves cannot do this stuff!
@@ -33,7 +33,7 @@ grant codeBase "${codebase.securemock-1.2.jar}" {
   permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
 };
 
-grant codeBase "${codebase.lucene-test-framework-6.2.0.jar}" {
+grant codeBase "${codebase.lucene-test-framework-6.6.1.jar}" {
   // needed by RamUsageTester
   permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
   // needed for testing hardlinks in StoreRecoveryTests since we install MockFS
@@ -42,7 +42,7 @@ grant codeBase "${codebase.lucene-test-framework-6.2.0.jar}" {
   permission java.lang.RuntimePermission "accessDeclaredMembers";
 };
 
-grant codeBase "${codebase.randomizedtesting-runner-2.3.2.jar}" {
+grant codeBase "${codebase.randomizedtesting-runner-2.5.0.jar}" {
   // optionally needed for access to private test methods (e.g. beforeClass)
   permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
   // needed to fail tests on uncaught exceptions from other threads
@@ -53,12 +53,19 @@ grant codeBase "${codebase.randomizedtesting-runner-2.3.2.jar}" {
   permission java.lang.RuntimePermission "accessDeclaredMembers";
 };
 
-grant codeBase "${codebase.junit4-ant-2.3.2.jar}" {
-  // needed for stream redirection
-  permission java.lang.RuntimePermission "setIO";
-};
-
-grant codeBase "${codebase.junit-4.11.jar}" {
+grant codeBase "${codebase.junit-4.12.jar}" {
   // needed for TestClass creation
   permission java.lang.RuntimePermission "accessDeclaredMembers";
 };
+
+grant codeBase "${codebase.elasticsearch-rest-client}" {
+  // rest makes socket connections for rest tests
+  permission java.net.SocketPermission "*", "connect";
+  // rest client uses system properties which gets the default proxy
+  permission java.net.NetPermission "getProxySelector";
+};
+
+grant codeBase "${codebase.httpasyncclient-4.1.2.jar}" {
+  // rest client uses system properties which gets the default proxy
+  permission java.net.NetPermission "getProxySelector";
+};
diff --git a/core/src/test/java/org/apache/lucene/grouping/CollapsingTopDocsCollectorTests.java b/core/src/test/java/org/apache/lucene/grouping/CollapsingTopDocsCollectorTests.java
new file mode 100644
index 0000000000000..aef354a04951f
--- /dev/null
+++ b/core/src/test/java/org/apache/lucene/grouping/CollapsingTopDocsCollectorTests.java
@@ -0,0 +1,426 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lucene.grouping;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.SortedNumericDocValuesField;
+import org.apache.lucene.document.SortedSetDocValuesField;
+import org.apache.lucene.index.CompositeReaderContext;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexReaderContext;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.search.Collector;
+import org.apache.lucene.search.FieldDoc;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.SortedNumericSortField;
+import org.apache.lucene.search.SortedSetSortField;
+import org.apache.lucene.search.TopFieldCollector;
+import org.apache.lucene.search.TopFieldDocs;
+import org.apache.lucene.search.Weight;
+import org.apache.lucene.search.grouping.CollapseTopFieldDocs;
+import org.apache.lucene.search.grouping.CollapsingTopDocsCollector;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.NumericUtils;
+import org.apache.lucene.util.TestUtil;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+public class CollapsingTopDocsCollectorTests extends ESTestCase {
+    private static class SegmentSearcher extends IndexSearcher {
+        private final List<LeafReaderContext> ctx;
+
+        SegmentSearcher(LeafReaderContext ctx, IndexReaderContext parent) {
+            super(parent);
+            this.ctx = Collections.singletonList(ctx);
+        }
+
+        public void search(Weight weight, Collector collector) throws IOException {
+            search(ctx, weight, collector);
+        }
+
+        @Override
+        public String toString() {
+            return "ShardSearcher(" + ctx.get(0) + ")";
+        }
+    }
+
+    interface CollapsingDocValuesProducer<T extends Comparable> {
+        T randomGroup(int maxGroup);
+
+        void add(Document doc, T value, boolean multivalued);
+
+        SortField sortField(boolean multivalued);
+    }
+
+    <T extends Comparable> void assertSearchCollapse(CollapsingDocValuesProducer<T> dvProducers, boolean numeric) throws IOException {
+        assertSearchCollapse(dvProducers, numeric, true);
+        assertSearchCollapse(dvProducers, numeric, false);
+    }
+
+    private <T extends Comparable> void assertSearchCollapse(CollapsingDocValuesProducer<T> dvProducers,
+                                                             boolean numeric, boolean multivalued) throws IOException {
+        final int numDocs = randomIntBetween(1000, 2000);
+        int maxGroup = randomIntBetween(2, 500);
+        final Directory dir = newDirectory();
+        final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
+        Set<T> values = new HashSet<>();
+        int totalHits = 0;
+        for (int i = 0; i < numDocs; i++) {
+            final T value = dvProducers.randomGroup(maxGroup);
+            values.add(value);
+            Document doc = new Document();
+            dvProducers.add(doc, value, multivalued);
+            doc.add(new NumericDocValuesField("sort1", randomIntBetween(0, 10)));
+            doc.add(new NumericDocValuesField("sort2", randomLong()));
+            w.addDocument(doc);
+            totalHits++;
+        }
+        List<T> valueList = new ArrayList<>(values);
+        Collections.sort(valueList);
+        final IndexReader reader = w.getReader();
+        final IndexSearcher searcher = newSearcher(reader);
+        final SortField collapseField = dvProducers.sortField(multivalued);
+        final SortField sort1 = new SortField("sort1", SortField.Type.INT);
+        final SortField sort2 = new SortField("sort2", SortField.Type.LONG);
+        Sort sort = new Sort(sort1, sort2, collapseField);
+
+        int expectedNumGroups = values.size();
+
+        final CollapsingTopDocsCollector collapsingCollector;
+        if (numeric) {
+            collapsingCollector =
+                CollapsingTopDocsCollector.createNumeric(collapseField.getField(), sort, expectedNumGroups, false);
+        } else {
+            collapsingCollector =
+                CollapsingTopDocsCollector.createKeyword(collapseField.getField(), sort, expectedNumGroups, false);
+        }
+
+        TopFieldCollector topFieldCollector =
+            TopFieldCollector.create(sort, totalHits, true, false, false);
+
+        searcher.search(new MatchAllDocsQuery(), collapsingCollector);
+        searcher.search(new MatchAllDocsQuery(), topFieldCollector);
+        CollapseTopFieldDocs collapseTopFieldDocs = collapsingCollector.getTopDocs();
+        TopFieldDocs topDocs = topFieldCollector.topDocs();
+        assertEquals(collapseField.getField(), collapseTopFieldDocs.field);
+        assertEquals(expectedNumGroups, collapseTopFieldDocs.scoreDocs.length);
+        assertEquals(totalHits, collapseTopFieldDocs.totalHits);
+        assertEquals(totalHits, topDocs.scoreDocs.length);
+        assertEquals(totalHits, topDocs.totalHits);
+
+        Set<Object> seen = new HashSet<>();
+        // collapse field is the last sort
+        int collapseIndex = sort.getSort().length - 1;
+        int topDocsIndex = 0;
+        for (int i = 0; i < expectedNumGroups; i++) {
+            FieldDoc fieldDoc = null;
+            for (; topDocsIndex < totalHits; topDocsIndex++) {
+                fieldDoc = (FieldDoc) topDocs.scoreDocs[topDocsIndex];
+                if (seen.contains(fieldDoc.fields[collapseIndex]) == false) {
+                    break;
+                }
+            }
+            FieldDoc collapseFieldDoc = (FieldDoc) collapseTopFieldDocs.scoreDocs[i];
+            assertNotNull(fieldDoc);
+            assertEquals(collapseFieldDoc.doc, fieldDoc.doc);
+            assertArrayEquals(collapseFieldDoc.fields, fieldDoc.fields);
+            seen.add(fieldDoc.fields[fieldDoc.fields.length - 1]);
+        }
+        for (; topDocsIndex < totalHits; topDocsIndex++) {
+            FieldDoc fieldDoc = (FieldDoc) topDocs.scoreDocs[topDocsIndex];
+            assertTrue(seen.contains(fieldDoc.fields[collapseIndex]));
+        }
+
+
+        // check merge
+        final IndexReaderContext ctx = searcher.getTopReaderContext();
+        final SegmentSearcher[] subSearchers;
+        final int[] docStarts;
+
+        if (ctx instanceof LeafReaderContext) {
+            subSearchers = new SegmentSearcher[1];
+            docStarts = new int[1];
+            subSearchers[0] = new SegmentSearcher((LeafReaderContext) ctx, ctx);
+            docStarts[0] = 0;
+        } else {
+            final CompositeReaderContext compCTX = (CompositeReaderContext) ctx;
+            final int size = compCTX.leaves().size();
+            subSearchers = new SegmentSearcher[size];
+            docStarts = new int[size];
+            int docBase = 0;
+            for (int searcherIDX = 0; searcherIDX < subSearchers.length; searcherIDX++) {
+                final LeafReaderContext leave = compCTX.leaves().get(searcherIDX);
+                subSearchers[searcherIDX] = new SegmentSearcher(leave, compCTX);
+                docStarts[searcherIDX] = docBase;
+                docBase += leave.reader().maxDoc();
+            }
+        }
+
+        final CollapseTopFieldDocs[] shardHits = new CollapseTopFieldDocs[subSearchers.length];
+        final Weight weight = searcher.createNormalizedWeight(new MatchAllDocsQuery(), false);
+        for (int shardIDX = 0; shardIDX < subSearchers.length; shardIDX++) {
+            final SegmentSearcher subSearcher = subSearchers[shardIDX];
+            final CollapsingTopDocsCollector c;
+            if (numeric) {
+                c = CollapsingTopDocsCollector.createNumeric(collapseField.getField(), sort, expectedNumGroups, false);
+            } else {
+                c = CollapsingTopDocsCollector.createKeyword(collapseField.getField(), sort, expectedNumGroups, false);
+            }
+            subSearcher.search(weight, c);
+            shardHits[shardIDX] = c.getTopDocs();
+        }
+        CollapseTopFieldDocs mergedFieldDocs = CollapseTopFieldDocs.merge(sort, 0, expectedNumGroups, shardHits, true);
+        assertTopDocsEquals(mergedFieldDocs, collapseTopFieldDocs);
+        w.close();
+        reader.close();
+        dir.close();
+    }
+
+    private static void assertTopDocsEquals(CollapseTopFieldDocs topDocs1, CollapseTopFieldDocs topDocs2) {
+        TestUtil.assertEquals(topDocs1, topDocs2);
+        assertArrayEquals(topDocs1.collapseValues, topDocs2.collapseValues);
+    }
+
+    public void testCollapseLong() throws Exception {
+        CollapsingDocValuesProducer producer = new CollapsingDocValuesProducer<Long>() {
+            @Override
+            public Long randomGroup(int maxGroup) {
+                return randomNonNegativeLong() % maxGroup;
+            }
+
+            @Override
+            public void add(Document doc, Long value, boolean multivalued) {
+                if (multivalued) {
+                    doc.add(new SortedNumericDocValuesField("field", value));
+                } else {
+                    doc.add(new NumericDocValuesField("field", value));
+                }
+            }
+
+            @Override
+            public SortField sortField(boolean multivalued) {
+                if (multivalued) {
+                    return new SortedNumericSortField("field", SortField.Type.LONG);
+                } else {
+                    return new SortField("field", SortField.Type.LONG);
+                }
+            }
+        };
+        assertSearchCollapse(producer, true);
+    }
+
+    public void testCollapseInt() throws Exception {
+        CollapsingDocValuesProducer producer = new CollapsingDocValuesProducer<Integer>() {
+            @Override
+            public Integer randomGroup(int maxGroup) {
+                return randomIntBetween(0, maxGroup - 1);
+            }
+
+            @Override
+            public void add(Document doc, Integer value, boolean multivalued) {
+                if (multivalued) {
+                    doc.add(new SortedNumericDocValuesField("field", value));
+                } else {
+                    doc.add(new NumericDocValuesField("field", value));
+                }
+            }
+
+            @Override
+            public SortField sortField(boolean multivalued) {
+                if (multivalued) {
+                    return new SortedNumericSortField("field", SortField.Type.INT);
+                } else {
+                    return new SortField("field", SortField.Type.INT);
+                }
+            }
+        };
+        assertSearchCollapse(producer, true);
+    }
+
+    public void testCollapseFloat() throws Exception {
+        CollapsingDocValuesProducer producer = new CollapsingDocValuesProducer<Float>() {
+            @Override
+            public Float randomGroup(int maxGroup) {
+                return new Float(randomIntBetween(0, maxGroup - 1));
+            }
+
+            @Override
+            public void add(Document doc, Float value, boolean multivalued) {
+                if (multivalued) {
+                    doc.add(new SortedNumericDocValuesField("field", NumericUtils.floatToSortableInt(value)));
+                } else {
+                    doc.add(new NumericDocValuesField("field", Float.floatToIntBits(value)));
+                }
+            }
+
+            @Override
+            public SortField sortField(boolean multivalued) {
+                if (multivalued) {
+                    return new SortedNumericSortField("field", SortField.Type.FLOAT);
+                } else {
+                    return new SortField("field", SortField.Type.FLOAT);
+                }
+            }
+        };
+        assertSearchCollapse(producer, true);
+    }
+
+    public void testCollapseDouble() throws Exception {
+        CollapsingDocValuesProducer producer = new CollapsingDocValuesProducer<Double>() {
+            @Override
+            public Double randomGroup(int maxGroup) {
+                return new Double(randomIntBetween(0, maxGroup - 1));
+            }
+
+            @Override
+            public void add(Document doc, Double value, boolean multivalued) {
+                if (multivalued) {
+                    doc.add(new SortedNumericDocValuesField("field", NumericUtils.doubleToSortableLong(value)));
+                } else {
+                    doc.add(new NumericDocValuesField("field", Double.doubleToLongBits(value)));
+                }
+            }
+
+            @Override
+            public SortField sortField(boolean multivalued) {
+                if (multivalued) {
+                    return new SortedNumericSortField("field", SortField.Type.DOUBLE);
+                } else {
+                    return new SortField("field", SortField.Type.DOUBLE);
+                }
+            }
+        };
+        assertSearchCollapse(producer, true);
+    }
+
+    public void testCollapseString() throws Exception {
+        CollapsingDocValuesProducer producer = new CollapsingDocValuesProducer<BytesRef>() {
+            @Override
+            public BytesRef randomGroup(int maxGroup) {
+                return new BytesRef(Integer.toString(randomIntBetween(0, maxGroup - 1)));
+            }
+
+            @Override
+            public void add(Document doc, BytesRef value, boolean multivalued) {
+                if (multivalued) {
+                    doc.add(new SortedSetDocValuesField("field", value));
+                } else {
+                    doc.add(new SortedDocValuesField("field", value));
+                }
+            }
+
+            @Override
+            public SortField sortField(boolean multivalued) {
+                if (multivalued) {
+                    return new SortedSetSortField("field", false);
+                } else {
+                    return new SortField("field", SortField.Type.STRING_VAL);
+                }
+            }
+        };
+        assertSearchCollapse(producer, false);
+    }
+
+    public void testEmptyNumericSegment() throws Exception {
+        final Directory dir = newDirectory();
+        final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
+        Document doc = new Document();
+        doc.add(new NumericDocValuesField("group", 0));
+        w.addDocument(doc);
+        doc.clear();
+        doc.add(new NumericDocValuesField("group", 1));
+        w.addDocument(doc);
+        w.commit();
+        doc.clear();
+        doc.add(new NumericDocValuesField("group", 10));
+        w.addDocument(doc);
+        w.commit();
+        doc.clear();
+        doc.add(new NumericDocValuesField("category", 0));
+        w.addDocument(doc);
+        w.commit();
+        final IndexReader reader = w.getReader();
+        final IndexSearcher searcher = newSearcher(reader);
+        SortField sortField = new SortField("group", SortField.Type.LONG);
+        sortField.setMissingValue(Long.MAX_VALUE);
+        Sort sort = new Sort(sortField);
+        final CollapsingTopDocsCollector collapsingCollector =
+                CollapsingTopDocsCollector.createNumeric("group", sort, 10, false);
+        searcher.search(new MatchAllDocsQuery(), collapsingCollector);
+        CollapseTopFieldDocs collapseTopFieldDocs = collapsingCollector.getTopDocs();
+        assertEquals(4, collapseTopFieldDocs.scoreDocs.length);
+        assertEquals(4, collapseTopFieldDocs.collapseValues.length);
+        assertEquals(0L, collapseTopFieldDocs.collapseValues[0]);
+        assertEquals(1L, collapseTopFieldDocs.collapseValues[1]);
+        assertEquals(10L, collapseTopFieldDocs.collapseValues[2]);
+        assertNull(collapseTopFieldDocs.collapseValues[3]);
+        w.close();
+        reader.close();
+        dir.close();
+    }
+
+    public void testEmptySortedSegment() throws Exception {
+        final Directory dir = newDirectory();
+        final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
+        Document doc = new Document();
+        doc.add(new SortedDocValuesField("group", new BytesRef("0")));
+        w.addDocument(doc);
+        doc.clear();
+        doc.add(new SortedDocValuesField("group", new BytesRef("1")));
+        w.addDocument(doc);
+        w.commit();
+        doc.clear();
+        doc.add(new SortedDocValuesField("group", new BytesRef("10")));
+        w.addDocument(doc);
+        w.commit();
+        doc.clear();
+        doc.add(new NumericDocValuesField("category", 0));
+        w.addDocument(doc);
+        w.commit();
+        final IndexReader reader = w.getReader();
+        final IndexSearcher searcher = newSearcher(reader);
+        Sort sort = new Sort(new SortField("group", SortField.Type.STRING_VAL));
+        final CollapsingTopDocsCollector collapsingCollector =
+            CollapsingTopDocsCollector.createKeyword("group", sort, 10, false);
+        searcher.search(new MatchAllDocsQuery(), collapsingCollector);
+        CollapseTopFieldDocs collapseTopFieldDocs = collapsingCollector.getTopDocs();
+        assertEquals(4, collapseTopFieldDocs.scoreDocs.length);
+        assertEquals(4, collapseTopFieldDocs.collapseValues.length);
+        assertNull(collapseTopFieldDocs.collapseValues[0]);
+        assertEquals(new BytesRef("0"), collapseTopFieldDocs.collapseValues[1]);
+        assertEquals(new BytesRef("1"), collapseTopFieldDocs.collapseValues[2]);
+        assertEquals(new BytesRef("10"), collapseTopFieldDocs.collapseValues[3]);
+        w.close();
+        reader.close();
+        dir.close();
+    }
+}
diff --git a/core/src/test/java/org/apache/lucene/queries/MinDocQueryTests.java b/core/src/test/java/org/apache/lucene/queries/MinDocQueryTests.java
index 3b9671d195a4f..b5bddcaebfe7e 100644
--- a/core/src/test/java/org/apache/lucene/queries/MinDocQueryTests.java
+++ b/core/src/test/java/org/apache/lucene/queries/MinDocQueryTests.java
@@ -21,8 +21,10 @@
 
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.MultiReader;
 import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
 import org.apache.lucene.search.QueryUtils;
 import org.apache.lucene.store.Directory;
 import org.elasticsearch.test.ESTestCase;
@@ -38,6 +40,19 @@ public void testBasics() {
         QueryUtils.check(query1);
         QueryUtils.checkEqual(query1, query2);
         QueryUtils.checkUnequal(query1, query3);
+
+        MinDocQuery query4 = new MinDocQuery(42, new Object());
+        MinDocQuery query5 = new MinDocQuery(42, new Object());
+        QueryUtils.checkUnequal(query4, query5);
+    }
+
+    public void testRewrite() throws Exception {
+        IndexReader reader = new MultiReader();
+        MinDocQuery query = new MinDocQuery(42);
+        Query rewritten = query.rewrite(reader);
+        QueryUtils.checkUnequal(query, rewritten);
+        Query rewritten2 = rewritten.rewrite(reader);
+        assertSame(rewritten, rewritten2);
     }
 
     public void testRandom() throws IOException {
diff --git a/core/src/test/java/org/apache/lucene/search/QueriesTests.java b/core/src/test/java/org/apache/lucene/search/QueriesTests.java
new file mode 100644
index 0000000000000..43723ddcdbb92
--- /dev/null
+++ b/core/src/test/java/org/apache/lucene/search/QueriesTests.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lucene.search;
+
+import org.elasticsearch.common.lucene.search.Queries;
+import org.elasticsearch.test.ESTestCase;
+
+public class QueriesTests extends ESTestCase {
+
+    public void testNonNestedQuery() {
+        // This is a custom query that extends AutomatonQuery and want to make sure the equals method works
+        assertEquals(Queries.newNonNestedFilter(), Queries.newNonNestedFilter());
+        assertEquals(Queries.newNonNestedFilter().hashCode(), Queries.newNonNestedFilter().hashCode());
+    }
+
+}
diff --git a/core/src/test/java/org/apache/lucene/search/postingshighlight/CustomPassageFormatterTests.java b/core/src/test/java/org/apache/lucene/search/postingshighlight/CustomPassageFormatterTests.java
index fcddc58f77a99..2d43a1ca64ef4 100644
--- a/core/src/test/java/org/apache/lucene/search/postingshighlight/CustomPassageFormatterTests.java
+++ b/core/src/test/java/org/apache/lucene/search/postingshighlight/CustomPassageFormatterTests.java
@@ -19,6 +19,7 @@
 
 package org.apache.lucene.search.postingshighlight;
 
+import org.apache.lucene.search.highlight.Snippet;
 import org.apache.lucene.search.highlight.DefaultEncoder;
 import org.apache.lucene.search.highlight.SimpleHTMLEncoder;
 import org.apache.lucene.util.BytesRef;
diff --git a/core/src/test/java/org/apache/lucene/search/postingshighlight/CustomPostingsHighlighterTests.java b/core/src/test/java/org/apache/lucene/search/postingshighlight/CustomPostingsHighlighterTests.java
index c95819a0e5399..315e38d12feb5 100644
--- a/core/src/test/java/org/apache/lucene/search/postingshighlight/CustomPostingsHighlighterTests.java
+++ b/core/src/test/java/org/apache/lucene/search/postingshighlight/CustomPostingsHighlighterTests.java
@@ -31,6 +31,7 @@
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.highlight.Snippet;
 import org.apache.lucene.search.Sort;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
diff --git a/core/src/test/java/org/apache/lucene/search/uhighlight/BoundedBreakIteratorScannerTests.java b/core/src/test/java/org/apache/lucene/search/uhighlight/BoundedBreakIteratorScannerTests.java
new file mode 100644
index 0000000000000..69e41962cc506
--- /dev/null
+++ b/core/src/test/java/org/apache/lucene/search/uhighlight/BoundedBreakIteratorScannerTests.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lucene.search.uhighlight;
+
+import org.elasticsearch.test.ESTestCase;
+
+import java.text.BreakIterator;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Locale;
+
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
+
+public class BoundedBreakIteratorScannerTests extends ESTestCase {
+    private static final String[] WORD_BOUNDARIES =
+        new String[] { " ", "  ",  "\t", "#", "\n" };
+    private static final String[] SENTENCE_BOUNDARIES =
+        new String[] { "! ", "? ", ". ", ".\n", ".\n\n" };
+
+    private void testRandomAsciiTextCase(BreakIterator bi, int maxLen) {
+        // Generate a random set of unique terms with ascii character
+        int maxSize = randomIntBetween(5, 100);
+        String[] vocabulary = new String[maxSize];
+        for (int i = 0; i < maxSize; i++) {
+            if (rarely()) {
+                vocabulary[i] = randomAlphaOfLengthBetween(50, 200);
+            } else {
+                vocabulary[i] = randomAlphaOfLengthBetween(1, 30);
+            }
+        }
+
+        // Generate a random text made of random terms separated with word-boundaries
+        // and sentence-boundaries.
+        StringBuilder text = new StringBuilder();
+        List<Integer> offsetList = new ArrayList<> ();
+        List<Integer> sizeList = new ArrayList<> ();
+        // the number of sentences to generate
+        int numSentences = randomIntBetween(10, 100);
+        int maxTermLen = 0;
+        for (int i = 0; i < numSentences; i++) {
+            // the number of terms in the sentence
+            int numTerms = randomIntBetween(5, 10);
+            for (int j = 0; j < numTerms; j++) {
+                int termId = randomIntBetween(0, vocabulary.length - 1);
+                String term = vocabulary[termId].toLowerCase(Locale.ROOT);
+                if (j == 0) {
+                    // capitalize the first letter of the first term in the sentence
+                    term = term.substring(0, 1).toUpperCase(Locale.ROOT) + term.substring(1);
+                } else {
+                    String sep = randomFrom(WORD_BOUNDARIES);
+                    text.append(sep);
+                }
+                maxTermLen = Math.max(term.length(), maxTermLen);
+                offsetList.add(text.length());
+                sizeList.add(term.length());
+                text.append(term);
+            }
+            String boundary = randomFrom(SENTENCE_BOUNDARIES);
+            text.append(boundary);
+        }
+
+        int[] sizes = sizeList.stream().mapToInt(i->i).toArray();
+        int[] offsets = offsetList.stream().mapToInt(i->i).toArray();
+
+        bi.setText(text.toString());
+        int currentPos = randomIntBetween(0, 20);
+        int lastEnd = -1;
+        int maxPassageLen = maxLen+(maxTermLen*2);
+        while (currentPos < offsets.length) {
+            // find the passage that contains the current term
+            int nextOffset = offsets[currentPos];
+            int start = bi.preceding(nextOffset+1);
+            int end = bi.following(nextOffset);
+
+            // check that the passage is valid
+            assertThat(start, greaterThanOrEqualTo(lastEnd));
+            assertThat(end, greaterThan(start));
+            assertThat(start, lessThanOrEqualTo(nextOffset));
+            assertThat(end, greaterThanOrEqualTo(nextOffset));
+            int passageLen = end-start;
+            assertThat(passageLen, lessThanOrEqualTo(maxPassageLen));
+
+            // checks that the start and end of the passage are on word boundaries.
+            int startPos = Arrays.binarySearch(offsets, start);
+            int endPos = Arrays.binarySearch(offsets, end);
+            if (startPos < 0) {
+                int lastWordEnd =
+                    offsets[Math.abs(startPos)-2] + sizes[Math.abs(startPos)-2];
+                assertThat(start, greaterThanOrEqualTo(lastWordEnd));
+            }
+            if (endPos < 0) {
+                if (Math.abs(endPos)-2 < offsets.length) {
+                    int lastWordEnd =
+                        offsets[Math.abs(endPos) - 2] + sizes[Math.abs(endPos) - 2];
+                    assertThat(end, greaterThanOrEqualTo(lastWordEnd));
+                }
+                // advance the position to the end of the current passage
+                currentPos = (Math.abs(endPos) - 1);
+            } else {
+                // advance the position to the end of the current passage
+                currentPos = endPos;
+            }
+            // randomly advance to the next term to highlight
+            currentPos += randomIntBetween(0, 20);
+            lastEnd = end;
+        }
+    }
+
+    public void testBoundedSentence() {
+        for (int i = 0; i < 20; i++) {
+            int maxLen = randomIntBetween(10, 500);
+            testRandomAsciiTextCase(
+                BoundedBreakIteratorScanner.getSentence(Locale.ROOT, maxLen),
+                maxLen
+            );
+        }
+    }
+}
diff --git a/core/src/test/java/org/apache/lucene/search/uhighlight/CustomPassageFormatterTests.java b/core/src/test/java/org/apache/lucene/search/uhighlight/CustomPassageFormatterTests.java
new file mode 100644
index 0000000000000..4e664c3e24101
--- /dev/null
+++ b/core/src/test/java/org/apache/lucene/search/uhighlight/CustomPassageFormatterTests.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lucene.search.uhighlight;
+
+import org.apache.lucene.search.highlight.Snippet;
+import org.apache.lucene.search.highlight.DefaultEncoder;
+import org.apache.lucene.search.highlight.SimpleHTMLEncoder;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.test.ESTestCase;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.notNullValue;
+
+
+public class CustomPassageFormatterTests extends ESTestCase {
+    public void testSimpleFormat() {
+        String content = "This is a really cool highlighter. Unified highlighter gives nice snippets back. No matches here.";
+
+        CustomPassageFormatter passageFormatter = new CustomPassageFormatter("<em>", "</em>", new DefaultEncoder());
+
+        Passage[] passages = new Passage[3];
+        String match = "highlighter";
+        BytesRef matchBytesRef = new BytesRef(match);
+
+        Passage passage1 = new Passage();
+        int start = content.indexOf(match);
+        int end = start + match.length();
+        passage1.setStartOffset(0);
+        passage1.setEndOffset(end + 2); //lets include the whitespace at the end to make sure we trim it
+        passage1.addMatch(start, end, matchBytesRef);
+        passages[0] = passage1;
+
+        Passage passage2 = new Passage();
+        start = content.lastIndexOf(match);
+        end = start + match.length();
+        passage2.setStartOffset(passage1.getEndOffset());
+        passage2.setEndOffset(end + 26);
+        passage2.addMatch(start, end, matchBytesRef);
+        passages[1] = passage2;
+
+        Passage passage3 = new Passage();
+        passage3.setStartOffset(passage2.getEndOffset());
+        passage3.setEndOffset(content.length());
+        passages[2] = passage3;
+
+        Snippet[] fragments = passageFormatter.format(passages, content);
+        assertThat(fragments, notNullValue());
+        assertThat(fragments.length, equalTo(3));
+        assertThat(fragments[0].getText(), equalTo("This is a really cool <em>highlighter</em>."));
+        assertThat(fragments[0].isHighlighted(), equalTo(true));
+        assertThat(fragments[1].getText(), equalTo("Unified <em>highlighter</em> gives nice snippets back."));
+        assertThat(fragments[1].isHighlighted(), equalTo(true));
+        assertThat(fragments[2].getText(), equalTo("No matches here."));
+        assertThat(fragments[2].isHighlighted(), equalTo(false));
+    }
+
+    public void testHtmlEncodeFormat() {
+        String content = "<b>This is a really cool highlighter.</b> Unified highlighter gives nice snippets back.";
+
+        CustomPassageFormatter passageFormatter = new CustomPassageFormatter("<em>", "</em>", new SimpleHTMLEncoder());
+
+        Passage[] passages = new Passage[2];
+        String match = "highlighter";
+        BytesRef matchBytesRef = new BytesRef(match);
+
+        Passage passage1 = new Passage();
+        int start = content.indexOf(match);
+        int end = start + match.length();
+        passage1.setStartOffset(0);
+        passage1.setEndOffset(end + 6); //lets include the whitespace at the end to make sure we trim it
+        passage1.addMatch(start, end, matchBytesRef);
+        passages[0] = passage1;
+
+        Passage passage2 = new Passage();
+        start = content.lastIndexOf(match);
+        end = start + match.length();
+        passage2.setStartOffset(passage1.getEndOffset());
+        passage2.setEndOffset(content.length());
+        passage2.addMatch(start, end, matchBytesRef);
+        passages[1] = passage2;
+
+        Snippet[] fragments = passageFormatter.format(passages, content);
+        assertThat(fragments, notNullValue());
+        assertThat(fragments.length, equalTo(2));
+        assertThat(fragments[0].getText(), equalTo("&lt;b&gt;This is a really cool <em>highlighter</em>.&lt;&#x2F;b&gt;"));
+        assertThat(fragments[1].getText(), equalTo("Unified <em>highlighter</em> gives nice snippets back."));
+    }
+}
diff --git a/core/src/test/java/org/apache/lucene/search/uhighlight/CustomUnifiedHighlighterTests.java b/core/src/test/java/org/apache/lucene/search/uhighlight/CustomUnifiedHighlighterTests.java
new file mode 100644
index 0000000000000..0824794e592a9
--- /dev/null
+++ b/core/src/test/java/org/apache/lucene/search/uhighlight/CustomUnifiedHighlighterTests.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lucene.search.uhighlight;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.queries.CommonTermsQuery;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.PhraseQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.highlight.DefaultEncoder;
+import org.apache.lucene.search.highlight.Snippet;
+import org.apache.lucene.store.Directory;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.lucene.all.AllTermQuery;
+import org.elasticsearch.common.lucene.search.MultiPhrasePrefixQuery;
+import org.elasticsearch.test.ESTestCase;
+
+import java.text.BreakIterator;
+import java.util.Locale;
+
+import static org.apache.lucene.search.uhighlight.CustomUnifiedHighlighter.MULTIVAL_SEP_CHAR;
+import static org.hamcrest.CoreMatchers.equalTo;
+
+public class CustomUnifiedHighlighterTests extends ESTestCase {
+    private void assertHighlightOneDoc(String fieldName, String[] inputs, Analyzer analyzer, Query query,
+                                       Locale locale, BreakIterator breakIterator,
+                                       int noMatchSize, String[] expectedPassages) throws Exception {
+        Directory dir = newDirectory();
+        IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
+        iwc.setMergePolicy(newTieredMergePolicy(random()));
+        RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
+        FieldType ft = new FieldType(TextField.TYPE_STORED);
+        ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
+        ft.freeze();
+        Document doc = new Document();
+        for (String input : inputs) {
+            Field field = new Field(fieldName, "", ft);
+            field.setStringValue(input);
+            doc.add(field);
+        }
+        iw.addDocument(doc);
+        DirectoryReader reader = iw.getReader();
+        IndexSearcher searcher = newSearcher(reader);
+        iw.close();
+        TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), 1, Sort.INDEXORDER);
+        assertThat(topDocs.totalHits, equalTo(1));
+        String rawValue = Strings.arrayToDelimitedString(inputs, String.valueOf(MULTIVAL_SEP_CHAR));
+        CustomUnifiedHighlighter highlighter = new CustomUnifiedHighlighter(searcher, analyzer, null,
+                new CustomPassageFormatter("<b>", "</b>", new DefaultEncoder()), locale,
+                breakIterator, rawValue, noMatchSize);
+        highlighter.setFieldMatcher((name) -> "text".equals(name));
+        final Snippet[] snippets =
+            highlighter.highlightField("text", query, topDocs.scoreDocs[0].doc, expectedPassages.length);
+        assertEquals(snippets.length, expectedPassages.length);
+        for (int i = 0; i < snippets.length; i++) {
+            assertEquals(snippets[i].getText(), expectedPassages[i]);
+        }
+        reader.close();
+        dir.close();
+    }
+
+    public void testSimple() throws Exception {
+        final String[] inputs = {
+            "This is a test. Just a test1 highlighting from unified highlighter.",
+            "This is the second highlighting value to perform highlighting on a longer text that gets scored lower.",
+            "This is highlighting the third short highlighting value.",
+            "Just a test4 highlighting from unified highlighter."
+        };
+
+        String[] expectedPassages = {
+            "Just a test1 <b>highlighting</b> from unified highlighter.",
+            "This is the second <b>highlighting</b> value to perform <b>highlighting</b> on a" +
+                " longer text that gets scored lower.",
+            "This is <b>highlighting</b> the third short <b>highlighting</b> value.",
+            "Just a test4 <b>highlighting</b> from unified highlighter."
+        };
+        Query query = new TermQuery(new Term("text", "highlighting"));
+        assertHighlightOneDoc("text", inputs, new StandardAnalyzer(), query, Locale.ROOT,
+            BreakIterator.getSentenceInstance(Locale.ROOT), 0, expectedPassages);
+    }
+
+    public void testNoMatchSize() throws Exception {
+        final String[] inputs = {
+            "This is a test. Just a test highlighting from unified. Feel free to ignore."
+        };
+        Query query = new TermQuery(new Term("body", "highlighting"));
+        assertHighlightOneDoc("text", inputs, new StandardAnalyzer(), query, Locale.ROOT,
+            BreakIterator.getSentenceInstance(Locale.ROOT), 100, inputs);
+    }
+
+    public void testMultiPhrasePrefixQuerySingleTerm() throws Exception {
+        final String[] inputs = {
+            "The quick brown fox."
+        };
+        final String[] outputs = {
+            "The quick <b>brown</b> fox."
+        };
+        MultiPhrasePrefixQuery query = new MultiPhrasePrefixQuery();
+        query.add(new Term("text", "bro"));
+        assertHighlightOneDoc("text", inputs, new StandardAnalyzer(), query, Locale.ROOT,
+            BreakIterator.getSentenceInstance(Locale.ROOT), 0, outputs);
+    }
+
+    public void testMultiPhrasePrefixQuery() throws Exception {
+        final String[] inputs = {
+            "The quick brown fox."
+        };
+        final String[] outputs = {
+            "The <b>quick</b> <b>brown</b> <b>fox</b>."
+        };
+        MultiPhrasePrefixQuery query = new MultiPhrasePrefixQuery();
+        query.add(new Term("text", "quick"));
+        query.add(new Term("text", "brown"));
+        query.add(new Term("text", "fo"));
+        assertHighlightOneDoc("text", inputs, new StandardAnalyzer(), query, Locale.ROOT,
+            BreakIterator.getSentenceInstance(Locale.ROOT), 0, outputs);
+    }
+
+    public void testAllTermQuery() throws Exception {
+        final String[] inputs = {
+            "The quick brown fox."
+        };
+        final String[] outputs = {
+            "The quick brown <b>fox</b>."
+        };
+        AllTermQuery query = new AllTermQuery(new Term("text", "fox"));
+        assertHighlightOneDoc("text", inputs, new StandardAnalyzer(), query, Locale.ROOT,
+            BreakIterator.getSentenceInstance(Locale.ROOT), 0, outputs);
+    }
+
+    public void testCommonTermsQuery() throws Exception {
+        final String[] inputs = {
+            "The quick brown fox."
+        };
+        final String[] outputs = {
+            "The <b>quick</b> <b>brown</b> <b>fox</b>."
+        };
+        CommonTermsQuery query = new CommonTermsQuery(BooleanClause.Occur.SHOULD, BooleanClause.Occur.SHOULD, 128);
+        query.add(new Term("text", "quick"));
+        query.add(new Term("text", "brown"));
+        query.add(new Term("text", "fox"));
+        assertHighlightOneDoc("text", inputs, new StandardAnalyzer(), query, Locale.ROOT,
+            BreakIterator.getSentenceInstance(Locale.ROOT), 0, outputs);
+    }
+
+    public void testSentenceBoundedBreakIterator() throws Exception {
+        final String[] inputs = {
+            "The quick brown fox in a long sentence with another quick brown fox. " +
+                "Another sentence with brown fox."
+        };
+        final String[] outputs = {
+            "The <b>quick</b> <b>brown</b>",
+            "<b>fox</b> in a long",
+            "with another <b>quick</b>",
+            "<b>brown</b> <b>fox</b>.",
+            "sentence with <b>brown</b>",
+            "<b>fox</b>.",
+        };
+        BooleanQuery query = new BooleanQuery.Builder()
+            .add(new TermQuery(new Term("text", "quick")), BooleanClause.Occur.SHOULD)
+            .add(new TermQuery(new Term("text", "brown")), BooleanClause.Occur.SHOULD)
+            .add(new TermQuery(new Term("text", "fox")), BooleanClause.Occur.SHOULD)
+            .build();
+        assertHighlightOneDoc("text", inputs, new StandardAnalyzer(), query, Locale.ROOT,
+            BoundedBreakIteratorScanner.getSentence(Locale.ROOT, 10), 0, outputs);
+    }
+
+    public void testRepeat() throws Exception {
+        final String[] inputs = {
+            "Fun  fun fun  fun  fun  fun  fun  fun  fun  fun"
+        };
+        final String[] outputs = {
+            "<b>Fun</b>  <b>fun</b> <b>fun</b>",
+            "<b>fun</b>  <b>fun</b>",
+            "<b>fun</b>  <b>fun</b>  <b>fun</b>",
+            "<b>fun</b>  <b>fun</b>"
+        };
+        Query query = new TermQuery(new Term("text", "fun"));
+        assertHighlightOneDoc("text", inputs, new StandardAnalyzer(), query, Locale.ROOT,
+            BoundedBreakIteratorScanner.getSentence(Locale.ROOT, 10), 0, outputs);
+
+        query = new PhraseQuery.Builder()
+            .add(new Term("text", "fun"))
+            .add(new Term("text", "fun"))
+            .build();
+        assertHighlightOneDoc("text", inputs, new StandardAnalyzer(), query, Locale.ROOT,
+            BoundedBreakIteratorScanner.getSentence(Locale.ROOT, 10), 0, outputs);
+    }
+}
diff --git a/core/src/test/java/org/apache/lucene/util/SloppyMathTests.java b/core/src/test/java/org/apache/lucene/util/SloppyMathTests.java
deleted file mode 100644
index 502d11e3b44c1..0000000000000
--- a/core/src/test/java/org/apache/lucene/util/SloppyMathTests.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.lucene.util;
-
-import org.elasticsearch.common.geo.GeoDistance;
-import org.elasticsearch.common.unit.DistanceUnit;
-import org.elasticsearch.test.ESTestCase;
-
-import static org.hamcrest.number.IsCloseTo.closeTo;
-
-public class SloppyMathTests extends ESTestCase {
-    public void testAccuracy() {
-        for (double lat1 = -89; lat1 <= 89; lat1+=1) {
-            final double lon1 = randomLongitude();
-
-            for (double i = -180; i <= 180; i+=1) {
-                final double lon2 = i;
-                final double lat2 = randomLatitude();
-
-                assertAccurate(lat1, lon1, lat2, lon2);
-            }
-        }
-    }
-
-    public void testSloppyMath() {
-        testSloppyMath(DistanceUnit.METERS, 0.01, 5, 45, 90);
-        testSloppyMath(DistanceUnit.KILOMETERS, 0.01, 5, 45, 90);
-        testSloppyMath(DistanceUnit.INCH, 0.01, 5, 45, 90);
-        testSloppyMath(DistanceUnit.MILES, 0.01, 5, 45, 90);
-    }
-
-    private static double maxError(double distance) {
-        return distance / 1000.0;
-    }
-
-    private void testSloppyMath(DistanceUnit unit, double...deltaDeg) {
-        final double lat1 = randomLatitude();
-        final double lon1 = randomLongitude();
-        logger.info("testing SloppyMath with {} at \"{}, {}\"", unit, lat1, lon1);
-
-        for (int test = 0; test < deltaDeg.length; test++) {
-            for (int i = 0; i < 100; i++) {
-                // crop pole areas, sine we now there the function
-                // is not accurate around lat(89°, 90°) and lat(-90°, -89°)
-                final double lat2 = Math.max(-89.0, Math.min(+89.0, lat1 + (random().nextDouble() - 0.5) * 2 * deltaDeg[test]));
-                final double lon2 = lon1 + (random().nextDouble() - 0.5) * 2 * deltaDeg[test];
-
-                final double accurate = GeoDistance.ARC.calculate(lat1, lon1, lat2, lon2, unit);
-                final double dist = GeoDistance.SLOPPY_ARC.calculate(lat1, lon1, lat2, lon2, unit);
-
-                assertThat("distance between("+lat1+", "+lon1+") and ("+lat2+", "+lon2+"))", dist, closeTo(accurate, maxError(accurate)));
-            }
-        }
-    }
-
-    private static void assertAccurate(double lat1, double lon1, double lat2, double lon2) {
-        double accurate = GeoDistance.ARC.calculate(lat1, lon1, lat2, lon2, DistanceUnit.METERS);
-        double sloppy = GeoDistance.SLOPPY_ARC.calculate(lat1, lon1, lat2, lon2, DistanceUnit.METERS);
-        assertThat("distance between("+lat1+", "+lon1+") and ("+lat2+", "+lon2+"))", sloppy, closeTo(accurate, maxError(accurate)));
-    }
-
-    private static double randomLatitude() {
-        // crop pole areas, sine we now there the function
-        // is not accurate around lat(89°, 90°) and lat(-90°, -89°)
-        return (random().nextDouble() - 0.5) * 178.0;
-    }
-
-    private static double randomLongitude() {
-        return (random().nextDouble() - 0.5) * 360.0;
-    }
-}
diff --git a/core/src/test/java/org/elasticsearch/BuildTests.java b/core/src/test/java/org/elasticsearch/BuildTests.java
index 7bff83262b317..7adfd883cdc27 100644
--- a/core/src/test/java/org/elasticsearch/BuildTests.java
+++ b/core/src/test/java/org/elasticsearch/BuildTests.java
@@ -43,7 +43,7 @@ public void testEqualsAndHashCode() {
         assertEquals(build, another);
         assertEquals(build.hashCode(), another.hashCode());
 
-        Build differentHash = new Build(randomAsciiOfLengthBetween(3, 10), build.date(), build.isSnapshot());
+        Build differentHash = new Build(randomAlphaOfLengthBetween(3, 10), build.date(), build.isSnapshot());
         assertNotEquals(build, differentHash);
 
         Build differentDate = new Build(build.shortHash(), "1970-01-01", build.isSnapshot());
diff --git a/core/src/test/java/org/elasticsearch/ESExceptionTests.java b/core/src/test/java/org/elasticsearch/ESExceptionTests.java
deleted file mode 100644
index 44ccc4ef34241..0000000000000
--- a/core/src/test/java/org/elasticsearch/ESExceptionTests.java
+++ /dev/null
@@ -1,377 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch;
-
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.IndexFormatTooNewException;
-import org.apache.lucene.index.IndexFormatTooOldException;
-import org.apache.lucene.store.AlreadyClosedException;
-import org.apache.lucene.store.LockObtainFailedException;
-import org.elasticsearch.action.search.SearchPhaseExecutionException;
-import org.elasticsearch.action.search.ShardSearchFailure;
-import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.io.stream.NotSerializableExceptionWrapper;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentLocation;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.index.IndexNotFoundException;
-import org.elasticsearch.index.query.QueryShardException;
-import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.search.SearchParseException;
-import org.elasticsearch.search.SearchShardTarget;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.TestSearchContext;
-import org.elasticsearch.test.VersionUtils;
-import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
-import org.elasticsearch.transport.RemoteTransportException;
-import org.hamcrest.Matchers;
-
-import java.io.EOFException;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.nio.file.NoSuchFileException;
-
-import static org.hamcrest.Matchers.equalTo;
-
-public class ESExceptionTests extends ESTestCase {
-    private static final ToXContent.Params PARAMS = ToXContent.EMPTY_PARAMS;
-
-    private class UnknownException extends Exception {
-
-        UnknownException(final String message, final Exception cause) {
-            super(message, cause);
-        }
-
-    }
-
-    public void testStatus() {
-        ElasticsearchException exception = new ElasticsearchException("test");
-        assertThat(exception.status(), equalTo(RestStatus.INTERNAL_SERVER_ERROR));
-
-        exception = new ElasticsearchException("test", new RuntimeException());
-        assertThat(exception.status(), equalTo(RestStatus.INTERNAL_SERVER_ERROR));
-
-        exception = new ElasticsearchException("test", new ResourceNotFoundException("test"));
-        assertThat(exception.status(), equalTo(RestStatus.INTERNAL_SERVER_ERROR));
-
-        exception = new RemoteTransportException("test", new ResourceNotFoundException("test"));
-        assertThat(exception.status(), equalTo(RestStatus.NOT_FOUND));
-
-        exception = new RemoteTransportException("test", new IllegalArgumentException("foobar"));
-        assertThat(exception.status(), equalTo(RestStatus.BAD_REQUEST));
-
-        exception = new RemoteTransportException("test", new IllegalStateException("foobar"));
-        assertThat(exception.status(), equalTo(RestStatus.INTERNAL_SERVER_ERROR));
-    }
-
-    public void testGuessRootCause() {
-        {
-            ElasticsearchException exception = new ElasticsearchException("foo", new ElasticsearchException("bar", new IndexNotFoundException("foo", new RuntimeException("foobar"))));
-            ElasticsearchException[] rootCauses = exception.guessRootCauses();
-            assertEquals(rootCauses.length, 1);
-            assertEquals(ElasticsearchException.getExceptionName(rootCauses[0]), "index_not_found_exception");
-            assertEquals(rootCauses[0].getMessage(), "no such index");
-            ShardSearchFailure failure = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null),
-                    new SearchShardTarget("node_1", new Index("foo", "_na_"), 1));
-            ShardSearchFailure failure1 = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null),
-                    new SearchShardTarget("node_1", new Index("foo", "_na_"), 2));
-            SearchPhaseExecutionException ex = new SearchPhaseExecutionException("search", "all shards failed", new ShardSearchFailure[]{failure, failure1});
-            if (randomBoolean()) {
-                rootCauses = (randomBoolean() ? new RemoteTransportException("remoteboom", ex) : ex).guessRootCauses();
-            } else {
-                rootCauses = ElasticsearchException.guessRootCauses(randomBoolean() ? new RemoteTransportException("remoteboom", ex) : ex);
-            }
-            assertEquals(ElasticsearchException.getExceptionName(rootCauses[0]), "parsing_exception");
-            assertEquals(rootCauses[0].getMessage(), "foobar");
-
-            ElasticsearchException oneLevel = new ElasticsearchException("foo", new RuntimeException("foobar"));
-            rootCauses = oneLevel.guessRootCauses();
-            assertEquals(ElasticsearchException.getExceptionName(rootCauses[0]), "exception");
-            assertEquals(rootCauses[0].getMessage(), "foo");
-        }
-        {
-            ShardSearchFailure failure = new ShardSearchFailure(
-                    new ParsingException(1, 2, "foobar", null),
-                    new SearchShardTarget("node_1", new Index("foo", "_na_"), 1));
-            ShardSearchFailure failure1 = new ShardSearchFailure(new QueryShardException(new Index("foo1", "_na_"), "foobar", null),
-                    new SearchShardTarget("node_1", new Index("foo1", "_na_"), 1));
-            ShardSearchFailure failure2 = new ShardSearchFailure(new QueryShardException(new Index("foo1", "_na_"), "foobar", null),
-                    new SearchShardTarget("node_1", new Index("foo1", "_na_"), 2));
-            SearchPhaseExecutionException ex = new SearchPhaseExecutionException("search", "all shards failed", new ShardSearchFailure[]{failure, failure1, failure2});
-            final ElasticsearchException[] rootCauses = ex.guessRootCauses();
-            assertEquals(rootCauses.length, 2);
-            assertEquals(ElasticsearchException.getExceptionName(rootCauses[0]), "parsing_exception");
-            assertEquals(rootCauses[0].getMessage(), "foobar");
-            assertEquals(((ParsingException) rootCauses[0]).getLineNumber(), 1);
-            assertEquals(((ParsingException) rootCauses[0]).getColumnNumber(), 2);
-            assertEquals(ElasticsearchException.getExceptionName(rootCauses[1]), "query_shard_exception");
-            assertEquals((rootCauses[1]).getIndex().getName(), "foo1");
-            assertEquals(rootCauses[1].getMessage(), "foobar");
-        }
-
-        {
-            final ElasticsearchException[] foobars = ElasticsearchException.guessRootCauses(new IllegalArgumentException("foobar"));
-            assertEquals(foobars.length, 1);
-            assertTrue(foobars[0] instanceof ElasticsearchException);
-            assertEquals(foobars[0].getMessage(), "foobar");
-            assertEquals(foobars[0].getCause().getClass(), IllegalArgumentException.class);
-            assertEquals(foobars[0].getExceptionName(), "illegal_argument_exception");
-        }
-
-    }
-
-    public void testDeduplicate() throws IOException {
-        {
-            ShardSearchFailure failure = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null),
-                    new SearchShardTarget("node_1", new Index("foo", "_na_"), 1));
-            ShardSearchFailure failure1 = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null),
-                    new SearchShardTarget("node_1", new Index("foo", "_na_"), 2));
-            SearchPhaseExecutionException ex = new SearchPhaseExecutionException("search", "all shards failed", randomBoolean() ? failure1.getCause() : failure.getCause(), new ShardSearchFailure[]{failure, failure1});
-            XContentBuilder builder = XContentFactory.jsonBuilder();
-            builder.startObject();
-            ex.toXContent(builder, PARAMS);
-            builder.endObject();
-            String expected = "{\"type\":\"search_phase_execution_exception\",\"reason\":\"all shards failed\",\"phase\":\"search\",\"grouped\":true,\"failed_shards\":[{\"shard\":1,\"index\":\"foo\",\"node\":\"node_1\",\"reason\":{\"type\":\"parsing_exception\",\"reason\":\"foobar\",\"line\":1,\"col\":2}}]}";
-            assertEquals(expected, builder.string());
-        }
-        {
-            ShardSearchFailure failure = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null),
-                    new SearchShardTarget("node_1", new Index("foo", "_na_"), 1));
-            ShardSearchFailure failure1 = new ShardSearchFailure(new QueryShardException(new Index("foo1", "_na_"), "foobar", null),
-                    new SearchShardTarget("node_1", new Index("foo1", "_na_"), 1));
-            ShardSearchFailure failure2 = new ShardSearchFailure(new QueryShardException(new Index("foo1", "_na_"), "foobar", null),
-                    new SearchShardTarget("node_1", new Index("foo1", "_na_"), 2));
-            SearchPhaseExecutionException ex = new SearchPhaseExecutionException("search", "all shards failed", new ShardSearchFailure[]{failure, failure1, failure2});
-            XContentBuilder builder = XContentFactory.jsonBuilder();
-            builder.startObject();
-            ex.toXContent(builder, PARAMS);
-            builder.endObject();
-            String expected = "{\"type\":\"search_phase_execution_exception\",\"reason\":\"all shards failed\",\"phase\":\"search\",\"grouped\":true,\"failed_shards\":[{\"shard\":1,\"index\":\"foo\",\"node\":\"node_1\",\"reason\":{\"type\":\"parsing_exception\",\"reason\":\"foobar\",\"line\":1,\"col\":2}},{\"shard\":1,\"index\":\"foo1\",\"node\":\"node_1\",\"reason\":{\"type\":\"query_shard_exception\",\"reason\":\"foobar\",\"index_uuid\":\"_na_\",\"index\":\"foo1\"}}]}";
-            assertEquals(expected, builder.string());
-        }
-        {
-            ShardSearchFailure failure = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null),
-                    new SearchShardTarget("node_1", new Index("foo", "_na_"), 1));
-            ShardSearchFailure failure1 = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null),
-                    new SearchShardTarget("node_1", new Index("foo", "_na_"), 2));
-            NullPointerException nullPointerException = new NullPointerException();
-            SearchPhaseExecutionException ex = new SearchPhaseExecutionException("search", "all shards failed", nullPointerException, new ShardSearchFailure[]{failure, failure1});
-            assertEquals(nullPointerException, ex.getCause());
-            XContentBuilder builder = XContentFactory.jsonBuilder();
-            builder.startObject();
-            ex.toXContent(builder, PARAMS);
-            builder.endObject();
-            String expected = "{\"type\":\"search_phase_execution_exception\",\"reason\":\"all shards failed\",\"phase\":\"search\",\"grouped\":true,\"failed_shards\":[{\"shard\":1,\"index\":\"foo\",\"node\":\"node_1\",\"reason\":{\"type\":\"parsing_exception\",\"reason\":\"foobar\",\"line\":1,\"col\":2}}],\"caused_by\":{\"type\":\"null_pointer_exception\",\"reason\":null}}";
-            assertEquals(expected, builder.string());
-        }
-    }
-
-    /**
-     * Check whether this exception contains an exception of the given type:
-     * either it is of the given class itself or it contains a nested cause
-     * of the given type.
-     *
-     * @param exType the exception type to look for
-     * @return whether there is a nested exception of the specified type
-     */
-    private boolean contains(Throwable t, Class<? extends Throwable> exType) {
-        if (exType == null) {
-            return false;
-        }
-        for (Throwable cause = t; t != null; t = t.getCause()) {
-            if (exType.isInstance(cause)) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-    public void testGetRootCause() {
-        Exception root = new RuntimeException("foobar");
-        ElasticsearchException exception = new ElasticsearchException("foo", new ElasticsearchException("bar", new IllegalArgumentException("index is closed", root)));
-        assertEquals(root, exception.getRootCause());
-        assertTrue(contains(exception, RuntimeException.class));
-        assertFalse(contains(exception, EOFException.class));
-    }
-
-    public void testToString() {
-        ElasticsearchException exception = new ElasticsearchException("foo", new ElasticsearchException("bar", new IllegalArgumentException("index is closed", new RuntimeException("foobar"))));
-        assertEquals("ElasticsearchException[foo]; nested: ElasticsearchException[bar]; nested: IllegalArgumentException[index is closed]; nested: RuntimeException[foobar];", exception.toString());
-    }
-
-    public void testToXContent() throws IOException {
-        {
-            ElasticsearchException ex = new SearchParseException(new TestSearchContext(null), "foo", new XContentLocation(1,0));
-            XContentBuilder builder = XContentFactory.jsonBuilder();
-            builder.startObject();
-            ex.toXContent(builder, PARAMS);
-            builder.endObject();
-
-            String expected = "{\"type\":\"search_parse_exception\",\"reason\":\"foo\",\"line\":1,\"col\":0}";
-            assertEquals(expected, builder.string());
-        }
-        {
-            ElasticsearchException ex = new ElasticsearchException("foo", new ElasticsearchException("bar", new IllegalArgumentException("index is closed", new RuntimeException("foobar"))));
-            XContentBuilder builder = XContentFactory.jsonBuilder();
-            builder.startObject();
-            ex.toXContent(builder, PARAMS);
-            builder.endObject();
-
-            String expected = "{\"type\":\"exception\",\"reason\":\"foo\",\"caused_by\":{\"type\":\"exception\",\"reason\":\"bar\",\"caused_by\":{\"type\":\"illegal_argument_exception\",\"reason\":\"index is closed\",\"caused_by\":{\"type\":\"runtime_exception\",\"reason\":\"foobar\"}}}}";
-            assertEquals(expected, builder.string());
-        }
-
-        {
-            Exception ex = new FileNotFoundException("foo not found");
-            if (randomBoolean()) {
-                // just a wrapper which is omitted
-                ex = new RemoteTransportException("foobar", ex);
-            }
-            XContentBuilder builder = XContentFactory.jsonBuilder();
-            builder.startObject();
-            ElasticsearchException.toXContent(builder, PARAMS, ex);
-            builder.endObject();
-
-            String expected = "{\"type\":\"file_not_found_exception\",\"reason\":\"foo not found\"}";
-            assertEquals(expected, builder.string());
-        }
-
-        {
-            ParsingException ex = new ParsingException(1, 2, "foobar", null);
-            XContentBuilder builder = XContentFactory.jsonBuilder();
-            builder.startObject();
-            ElasticsearchException.toXContent(builder, PARAMS, ex);
-            builder.endObject();
-            String expected = "{\"type\":\"parsing_exception\",\"reason\":\"foobar\",\"line\":1,\"col\":2}";
-            assertEquals(expected, builder.string());
-        }
-
-        { // test equivalence
-            ElasticsearchException ex =  new RemoteTransportException("foobar", new FileNotFoundException("foo not found"));
-            XContentBuilder builder = XContentFactory.jsonBuilder();
-            builder.startObject();
-            ElasticsearchException.toXContent(builder, PARAMS, ex);
-            builder.endObject();
-
-            XContentBuilder otherBuilder = XContentFactory.jsonBuilder();
-
-            otherBuilder.startObject();
-            ex.toXContent(otherBuilder, PARAMS);
-            otherBuilder.endObject();
-            assertEquals(otherBuilder.string(), builder.string());
-            assertEquals("{\"type\":\"file_not_found_exception\",\"reason\":\"foo not found\"}", builder.string());
-        }
-
-        { // render header
-            ParsingException ex = new ParsingException(1, 2, "foobar", null);
-            ex.addHeader("test", "some value");
-            ex.addHeader("test_multi", "some value", "another value");
-            XContentBuilder builder = XContentFactory.jsonBuilder();
-            builder.startObject();
-            ElasticsearchException.toXContent(builder, PARAMS, ex);
-            builder.endObject();
-            assertThat(builder.string(), Matchers.anyOf( // iteration order depends on platform
-                            equalTo("{\"type\":\"parsing_exception\",\"reason\":\"foobar\",\"line\":1,\"col\":2,\"header\":{\"test_multi\":[\"some value\",\"another value\"],\"test\":\"some value\"}}"),
-                            equalTo("{\"type\":\"parsing_exception\",\"reason\":\"foobar\",\"line\":1,\"col\":2,\"header\":{\"test\":\"some value\",\"test_multi\":[\"some value\",\"another value\"]}}")
-            ));
-        }
-    }
-
-    public void testSerializeElasticsearchException() throws IOException {
-        BytesStreamOutput out = new BytesStreamOutput();
-        ParsingException ex = new ParsingException(1, 2, "foobar", null);
-        out.writeException(ex);
-
-        StreamInput in = out.bytes().streamInput();
-        ParsingException e = in.readException();
-        assertEquals(ex.getIndex(), e.getIndex());
-        assertEquals(ex.getMessage(), e.getMessage());
-        assertEquals(ex.getLineNumber(), e.getLineNumber());
-        assertEquals(ex.getColumnNumber(), e.getColumnNumber());
-    }
-
-    public void testSerializeUnknownException() throws IOException {
-        BytesStreamOutput out = new BytesStreamOutput();
-        ParsingException parsingException = new ParsingException(1, 2, "foobar", null);
-        final Exception ex = new UnknownException("eggplant", parsingException);
-        out.writeException(ex);
-
-        StreamInput in = out.bytes().streamInput();
-        Throwable throwable = in.readException();
-        assertEquals("unknown_exception: eggplant", throwable.getMessage());
-        assertTrue(throwable instanceof ElasticsearchException);
-        ParsingException e = (ParsingException)throwable.getCause();
-                assertEquals(parsingException.getIndex(), e.getIndex());
-        assertEquals(parsingException.getMessage(), e.getMessage());
-        assertEquals(parsingException.getLineNumber(), e.getLineNumber());
-        assertEquals(parsingException.getColumnNumber(), e.getColumnNumber());
-    }
-
-    public void testWriteThrowable() throws IOException {
-
-        final QueryShardException queryShardException = new QueryShardException(new Index("foo", "_na_"), "foobar", null);
-        final UnknownException unknownException = new UnknownException("this exception is unknown", queryShardException);
-
-        final Exception[] causes = new Exception[]{
-                new IllegalStateException("foobar"),
-                new IllegalArgumentException("alalaal"),
-                new NullPointerException("boom"),
-                new EOFException("dadada"),
-                new ElasticsearchSecurityException("nono!"),
-                new NumberFormatException("not a number"),
-                new CorruptIndexException("baaaam booom", "this is my resource"),
-                new IndexFormatTooNewException("tooo new", 1, 2, 3),
-                new IndexFormatTooOldException("tooo new", 1, 2, 3),
-                new IndexFormatTooOldException("tooo new", "very old version"),
-                new ArrayIndexOutOfBoundsException("booom"),
-                new StringIndexOutOfBoundsException("booom"),
-                new FileNotFoundException("booom"),
-                new NoSuchFileException("booom"),
-                new AlreadyClosedException("closed!!", new NullPointerException()),
-                new LockObtainFailedException("can't lock directory", new NullPointerException()),
-                unknownException};
-        for (final Exception cause : causes) {
-            BytesStreamOutput out = new BytesStreamOutput();
-            ElasticsearchException ex = new ElasticsearchException("topLevel", cause);
-            out.writeException(ex);
-            StreamInput in = out.bytes().streamInput();
-            ElasticsearchException e = in.readException();
-            assertEquals(e.getMessage(), ex.getMessage());
-            assertTrue("Expected: " + e.getCause().getMessage() + " to contain: " +
-                            ex.getCause().getClass().getName() + " but it didn't",
-                    e.getCause().getMessage().contains(ex.getCause().getMessage()));
-            if (ex.getCause().getClass() != UnknownException.class) { // unknown exception is not directly mapped
-                assertEquals(e.getCause().getClass(), ex.getCause().getClass());
-            } else {
-                assertEquals(e.getCause().getClass(), NotSerializableExceptionWrapper.class);
-            }
-            assertArrayEquals(e.getStackTrace(), ex.getStackTrace());
-            assertTrue(e.getStackTrace().length > 1);
-            ElasticsearchAssertions.assertVersionSerializable(VersionUtils.randomVersion(random()), cause);
-            ElasticsearchAssertions.assertVersionSerializable(VersionUtils.randomVersion(random()), ex);
-            ElasticsearchAssertions.assertVersionSerializable(VersionUtils.randomVersion(random()), e);
-        }
-    }
-
-}
diff --git a/core/src/test/java/org/elasticsearch/ElasticsearchExceptionTests.java b/core/src/test/java/org/elasticsearch/ElasticsearchExceptionTests.java
new file mode 100644
index 0000000000000..c01d075115cb7
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/ElasticsearchExceptionTests.java
@@ -0,0 +1,982 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch;
+
+import org.apache.lucene.util.Constants;
+import org.elasticsearch.action.NoShardAvailableActionException;
+import org.elasticsearch.action.OriginalIndices;
+import org.elasticsearch.action.RoutingMissingException;
+import org.elasticsearch.action.search.SearchPhaseExecutionException;
+import org.elasticsearch.action.search.ShardSearchFailure;
+import org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedException;
+import org.elasticsearch.client.transport.NoNodeAvailableException;
+import org.elasticsearch.cluster.block.ClusterBlockException;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.breaker.CircuitBreakingException;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.transport.LocalTransportAddress;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentLocation;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.discovery.DiscoverySettings;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.index.query.QueryShardException;
+import org.elasticsearch.index.shard.IndexShardRecoveringException;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.node.NodeClosedException;
+import org.elasticsearch.repositories.RepositoryException;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.script.ScriptException;
+import org.elasticsearch.search.SearchContextMissingException;
+import org.elasticsearch.search.SearchParseException;
+import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.TestSearchContext;
+import org.elasticsearch.transport.RemoteTransportException;
+
+import java.io.EOFException;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.file.FileAlreadyExistsException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static java.util.Collections.emptyList;
+import static java.util.Collections.singleton;
+import static java.util.Collections.singletonList;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
+import static org.hamcrest.CoreMatchers.hasItem;
+import static org.hamcrest.CoreMatchers.hasItems;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.startsWith;
+
+public class ElasticsearchExceptionTests extends ESTestCase {
+
+    public void testStatus() {
+        ElasticsearchException exception = new ElasticsearchException("test");
+        assertThat(exception.status(), equalTo(RestStatus.INTERNAL_SERVER_ERROR));
+
+        exception = new ElasticsearchException("test", new RuntimeException());
+        assertThat(exception.status(), equalTo(RestStatus.INTERNAL_SERVER_ERROR));
+
+        exception = new ElasticsearchException("test", new ResourceNotFoundException("test"));
+        assertThat(exception.status(), equalTo(RestStatus.INTERNAL_SERVER_ERROR));
+
+        exception = new RemoteTransportException("test", new ResourceNotFoundException("test"));
+        assertThat(exception.status(), equalTo(RestStatus.NOT_FOUND));
+
+        exception = new RemoteTransportException("test", new ResourceAlreadyExistsException("test"));
+        assertThat(exception.status(), equalTo(RestStatus.BAD_REQUEST));
+
+        exception = new RemoteTransportException("test", new IllegalArgumentException("foobar"));
+        assertThat(exception.status(), equalTo(RestStatus.BAD_REQUEST));
+
+        exception = new RemoteTransportException("test", new IllegalStateException("foobar"));
+        assertThat(exception.status(), equalTo(RestStatus.INTERNAL_SERVER_ERROR));
+    }
+
+    public void testGuessRootCause() {
+        {
+            ElasticsearchException exception = new ElasticsearchException("foo", new ElasticsearchException("bar",
+                    new IndexNotFoundException("foo", new RuntimeException("foobar"))));
+            ElasticsearchException[] rootCauses = exception.guessRootCauses();
+            assertEquals(rootCauses.length, 1);
+            assertEquals(ElasticsearchException.getExceptionName(rootCauses[0]), "index_not_found_exception");
+            assertEquals(rootCauses[0].getMessage(), "no such index");
+            ShardSearchFailure failure = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null),
+                    new SearchShardTarget("node_1", new Index("foo", "_na_"), 1, null));
+            ShardSearchFailure failure1 = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null),
+                    new SearchShardTarget("node_1", new Index("foo", "_na_"), 2, null));
+            SearchPhaseExecutionException ex = new SearchPhaseExecutionException("search", "all shards failed",
+                    new ShardSearchFailure[]{failure, failure1});
+            if (randomBoolean()) {
+                rootCauses = (randomBoolean() ? new RemoteTransportException("remoteboom", ex) : ex).guessRootCauses();
+            } else {
+                rootCauses = ElasticsearchException.guessRootCauses(randomBoolean() ? new RemoteTransportException("remoteboom", ex) : ex);
+            }
+            assertEquals(ElasticsearchException.getExceptionName(rootCauses[0]), "parsing_exception");
+            assertEquals(rootCauses[0].getMessage(), "foobar");
+
+            ElasticsearchException oneLevel = new ElasticsearchException("foo", new RuntimeException("foobar"));
+            rootCauses = oneLevel.guessRootCauses();
+            assertEquals(ElasticsearchException.getExceptionName(rootCauses[0]), "exception");
+            assertEquals(rootCauses[0].getMessage(), "foo");
+        }
+        {
+            ShardSearchFailure failure = new ShardSearchFailure(
+                    new ParsingException(1, 2, "foobar", null),
+                    new SearchShardTarget("node_1", new Index("foo", "_na_"), 1, null));
+            ShardSearchFailure failure1 = new ShardSearchFailure(new QueryShardException(new Index("foo1", "_na_"), "foobar", null),
+                    new SearchShardTarget("node_1", new Index("foo1", "_na_"), 1, null));
+            ShardSearchFailure failure2 = new ShardSearchFailure(new QueryShardException(new Index("foo1", "_na_"), "foobar", null),
+                    new SearchShardTarget("node_1", new Index("foo1", "_na_"), 2, null));
+            SearchPhaseExecutionException ex = new SearchPhaseExecutionException("search", "all shards failed",
+                    new ShardSearchFailure[]{failure, failure1, failure2});
+            final ElasticsearchException[] rootCauses = ex.guessRootCauses();
+            assertEquals(rootCauses.length, 2);
+            assertEquals(ElasticsearchException.getExceptionName(rootCauses[0]), "parsing_exception");
+            assertEquals(rootCauses[0].getMessage(), "foobar");
+            assertEquals(((ParsingException) rootCauses[0]).getLineNumber(), 1);
+            assertEquals(((ParsingException) rootCauses[0]).getColumnNumber(), 2);
+            assertEquals(ElasticsearchException.getExceptionName(rootCauses[1]), "query_shard_exception");
+            assertEquals((rootCauses[1]).getIndex().getName(), "foo1");
+            assertEquals(rootCauses[1].getMessage(), "foobar");
+        }
+
+        {
+            final ElasticsearchException[] foobars = ElasticsearchException.guessRootCauses(new IllegalArgumentException("foobar"));
+            assertEquals(foobars.length, 1);
+            assertTrue(foobars[0] instanceof ElasticsearchException);
+            assertEquals(foobars[0].getMessage(), "foobar");
+            assertEquals(foobars[0].getCause().getClass(), IllegalArgumentException.class);
+            assertEquals(foobars[0].getExceptionName(), "illegal_argument_exception");
+        }
+    }
+
+    public void testDeduplicate() throws IOException {
+        {
+            ShardSearchFailure failure = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null),
+                    new SearchShardTarget("node_1", new Index("foo", "_na_"), 1, null));
+            ShardSearchFailure failure1 = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null),
+                    new SearchShardTarget("node_1", new Index("foo", "_na_"), 2, null));
+            SearchPhaseExecutionException ex = new SearchPhaseExecutionException("search", "all shards failed",
+                    randomBoolean() ? failure1.getCause() : failure.getCause(), new ShardSearchFailure[]{failure, failure1});
+            XContentBuilder builder = XContentFactory.jsonBuilder();
+            builder.startObject();
+            ex.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            builder.endObject();
+            String expected = "{\"type\":\"search_phase_execution_exception\",\"reason\":\"all shards failed\",\"phase\":\"search\"," +
+                    "\"grouped\":true,\"failed_shards\":[{\"shard\":1,\"index\":\"foo\",\"node\":\"node_1\",\"reason\":" +
+                    "{\"type\":\"parsing_exception\",\"reason\":\"foobar\",\"line\":1,\"col\":2}}]}";
+            assertEquals(expected, builder.string());
+        }
+        {
+            ShardSearchFailure failure = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null),
+                    new SearchShardTarget("node_1", new Index("foo", "_na_"), 1, null));
+            ShardSearchFailure failure1 = new ShardSearchFailure(new QueryShardException(new Index("foo1", "_na_"), "foobar", null),
+                    new SearchShardTarget("node_1", new Index("foo1", "_na_"), 1, null));
+            ShardSearchFailure failure2 = new ShardSearchFailure(new QueryShardException(new Index("foo1", "_na_"), "foobar", null),
+                    new SearchShardTarget("node_1", new Index("foo1", "_na_"), 2, null));
+            SearchPhaseExecutionException ex = new SearchPhaseExecutionException("search", "all shards failed",
+                    new ShardSearchFailure[]{failure, failure1, failure2});
+            XContentBuilder builder = XContentFactory.jsonBuilder();
+            builder.startObject();
+            ex.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            builder.endObject();
+            String expected = "{\"type\":\"search_phase_execution_exception\",\"reason\":\"all shards failed\"," +
+                    "\"phase\":\"search\",\"grouped\":true,\"failed_shards\":[{\"shard\":1,\"index\":\"foo\",\"node\":\"node_1\"," +
+                    "\"reason\":{\"type\":\"parsing_exception\",\"reason\":\"foobar\",\"line\":1,\"col\":2}},{\"shard\":1," +
+                    "\"index\":\"foo1\",\"node\":\"node_1\",\"reason\":{\"type\":\"query_shard_exception\",\"reason\":\"foobar\"," +
+                    "\"index_uuid\":\"_na_\",\"index\":\"foo1\"}}]}";
+            assertEquals(expected, builder.string());
+        }
+        {
+            ShardSearchFailure failure = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null),
+                    new SearchShardTarget("node_1", new Index("foo", "_na_"), 1, null));
+            ShardSearchFailure failure1 = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null),
+                    new SearchShardTarget("node_1", new Index("foo", "_na_"), 2, null));
+            NullPointerException nullPointerException = new NullPointerException();
+            SearchPhaseExecutionException ex = new SearchPhaseExecutionException("search", "all shards failed", nullPointerException,
+                    new ShardSearchFailure[]{failure, failure1});
+            assertEquals(nullPointerException, ex.getCause());
+            XContentBuilder builder = XContentFactory.jsonBuilder();
+            builder.startObject();
+            ex.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            builder.endObject();
+            String expected = "{\"type\":\"search_phase_execution_exception\",\"reason\":\"all shards failed\"," +
+                    "\"phase\":\"search\",\"grouped\":true,\"failed_shards\":[{\"shard\":1,\"index\":\"foo\",\"node\":\"node_1\"," +
+                    "\"reason\":{\"type\":\"parsing_exception\",\"reason\":\"foobar\",\"line\":1,\"col\":2}}]," +
+                    "\"caused_by\":{\"type\":\"null_pointer_exception\",\"reason\":null}}";
+            assertEquals(expected, builder.string());
+        }
+    }
+
+    /**
+     * Check whether this exception contains an exception of the given type:
+     * either it is of the given class itself or it contains a nested cause
+     * of the given type.
+     *
+     * @param exType the exception type to look for
+     * @return whether there is a nested exception of the specified type
+     */
+    private static boolean contains(Throwable t, Class<? extends Throwable> exType) {
+        if (exType == null) {
+            return false;
+        }
+        for (Throwable cause = t; t != null; t = t.getCause()) {
+            if (exType.isInstance(cause)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    public void testGetRootCause() {
+        Exception root = new RuntimeException("foobar");
+        ElasticsearchException exception = new ElasticsearchException("foo", new ElasticsearchException("bar",
+                new IllegalArgumentException("index is closed", root)));
+        assertEquals(root, exception.getRootCause());
+        assertTrue(contains(exception, RuntimeException.class));
+        assertFalse(contains(exception, EOFException.class));
+    }
+
+    public void testToString() {
+        ElasticsearchException exception = new ElasticsearchException("foo", new ElasticsearchException("bar",
+                new IllegalArgumentException("index is closed", new RuntimeException("foobar"))));
+        assertEquals("ElasticsearchException[foo]; nested: ElasticsearchException[bar]; nested: IllegalArgumentException" +
+                "[index is closed]; nested: RuntimeException[foobar];", exception.toString());
+    }
+
+    public void testToXContent() throws IOException {
+        {
+            ElasticsearchException e = new ElasticsearchException("test");
+            assertExceptionAsJson(e, "{\"type\":\"exception\",\"reason\":\"test\"}");
+        }
+        {
+            ElasticsearchException e = new IndexShardRecoveringException(new ShardId("_test", "_0", 5));
+            assertExceptionAsJson(e, "{\"type\":\"index_shard_recovering_exception\"," +
+                    "\"reason\":\"CurrentState[RECOVERING] Already recovering\",\"index_uuid\":\"_0\"," +
+                    "\"shard\":\"5\",\"index\":\"_test\"}");
+        }
+        {
+            ElasticsearchException e = new BroadcastShardOperationFailedException(new ShardId("_index", "_uuid", 12), "foo",
+                    new IllegalStateException("bar"));
+            assertExceptionAsJson(e, "{\"type\":\"illegal_state_exception\",\"reason\":\"bar\"}");
+        }
+        {
+            ElasticsearchException e = new ElasticsearchException(new IllegalArgumentException("foo"));
+            assertExceptionAsJson(e, "{\"type\":\"exception\",\"reason\":\"java.lang.IllegalArgumentException: foo\"," +
+                    "\"caused_by\":{\"type\":\"illegal_argument_exception\",\"reason\":\"foo\"}}");
+        }
+        {
+            ElasticsearchException e = new SearchParseException(new TestSearchContext(null), "foo", new XContentLocation(1,0));
+            assertExceptionAsJson(e, "{\"type\":\"search_parse_exception\",\"reason\":\"foo\",\"line\":1,\"col\":0}");
+        }
+        {
+            ElasticsearchException ex = new ElasticsearchException("foo",
+                    new ElasticsearchException("bar", new IllegalArgumentException("index is closed", new RuntimeException("foobar"))));
+            assertExceptionAsJson(ex, "{\"type\":\"exception\",\"reason\":\"foo\",\"caused_by\":{\"type\":\"exception\"," +
+                    "\"reason\":\"bar\",\"caused_by\":{\"type\":\"illegal_argument_exception\",\"reason\":\"index is closed\"," +
+                    "\"caused_by\":{\"type\":\"runtime_exception\",\"reason\":\"foobar\"}}}}");
+        }
+        {
+            ElasticsearchException e = new ElasticsearchException("foo", new IllegalStateException("bar"));
+            assertExceptionAsJson(e, "{\"type\":\"exception\",\"reason\":\"foo\"," +
+                    "\"caused_by\":{\"type\":\"illegal_state_exception\",\"reason\":\"bar\"}}");
+
+            // Test the same exception but with the "rest.exception.stacktrace.skip" parameter disabled: the stack_trace must be present
+            // in the JSON. Since the stack can be large, it only checks the beginning of the JSON.
+            ToXContent.Params params = new ToXContent.MapParams(
+                    Collections.singletonMap(ElasticsearchException.REST_EXCEPTION_SKIP_STACK_TRACE, "false"));
+            String actual;
+            try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) {
+                builder.startObject();
+                e.toXContent(builder, params);
+                builder.endObject();
+                actual = builder.string();
+            }
+            assertThat(actual, startsWith("{\"type\":\"exception\",\"reason\":\"foo\"," +
+                    "\"caused_by\":{\"type\":\"illegal_state_exception\",\"reason\":\"bar\"," +
+                    "\"stack_trace\":\"java.lang.IllegalStateException: bar" +
+                    (Constants.WINDOWS ? "\\r\\n" : "\\n") +
+                    "\\tat org.elasticsearch."));
+        }
+    }
+
+    public void testGenerateThrowableToXContent() throws IOException {
+        {
+            Exception ex;
+            if (randomBoolean()) {
+                // just a wrapper which is omitted
+                ex = new RemoteTransportException("foobar", new FileNotFoundException("foo not found"));
+            } else {
+                ex = new FileNotFoundException("foo not found");
+            }
+            assertExceptionAsJson(ex, "{\"type\":\"file_not_found_exception\",\"reason\":\"foo not found\"}");
+        }
+        {
+            ParsingException ex = new ParsingException(1, 2, "foobar", null);
+            assertExceptionAsJson(ex, "{\"type\":\"parsing_exception\",\"reason\":\"foobar\",\"line\":1,\"col\":2}");
+        }
+
+        { // test equivalence
+            ElasticsearchException ex =  new RemoteTransportException("foobar", new FileNotFoundException("foo not found"));
+            String toXContentString = Strings.toString(ex);
+            String throwableString = Strings.toString((builder, params) -> {
+                ElasticsearchException.generateThrowableXContent(builder, params, ex);
+                return builder;
+            });
+
+            assertEquals(throwableString, toXContentString);
+            assertEquals("{\"type\":\"file_not_found_exception\",\"reason\":\"foo not found\"}", toXContentString);
+        }
+
+        { // render header and metadata
+            ParsingException ex = new ParsingException(1, 2, "foobar", null);
+            ex.addMetadata("es.test1", "value1");
+            ex.addMetadata("es.test2", "value2");
+            ex.addHeader("test", "some value");
+            ex.addHeader("test_multi", "some value", "another value");
+            String expected = "{\"type\":\"parsing_exception\",\"reason\":\"foobar\",\"line\":1,\"col\":2," +
+                    "\"test1\":\"value1\",\"test2\":\"value2\"," +
+                    "\"header\":{\"test_multi\":" +
+                    "[\"some value\",\"another value\"],\"test\":\"some value\"}}";
+            assertExceptionAsJson(ex, expected);
+        }
+    }
+
+    public void testToXContentWithHeadersAndMetadata() throws IOException {
+        ElasticsearchException e = new ElasticsearchException("foo",
+                                        new ElasticsearchException("bar",
+                                                new ElasticsearchException("baz",
+                                                        new ClusterBlockException(singleton(DiscoverySettings.NO_MASTER_BLOCK_WRITES)))));
+        e.addHeader("foo_0", "0");
+        e.addHeader("foo_1", "1");
+        e.addMetadata("es.metadata_foo_0", "foo_0");
+        e.addMetadata("es.metadata_foo_1", "foo_1");
+
+        final String expectedJson = "{"
+            + "\"type\":\"exception\","
+            + "\"reason\":\"foo\","
+            + "\"metadata_foo_0\":\"foo_0\","
+            + "\"metadata_foo_1\":\"foo_1\","
+            + "\"caused_by\":{"
+                + "\"type\":\"exception\","
+                + "\"reason\":\"bar\","
+                + "\"caused_by\":{"
+                    + "\"type\":\"exception\","
+                    + "\"reason\":\"baz\","
+                    + "\"caused_by\":{"
+                        + "\"type\":\"cluster_block_exception\","
+                        + "\"reason\":\"blocked by: [SERVICE_UNAVAILABLE/2/no master];\""
+                    + "}"
+                + "}"
+            + "},"
+            + "\"header\":{"
+                    + "\"foo_0\":\"0\","
+                    + "\"foo_1\":\"1\""
+                + "}"
+        + "}";
+
+        assertExceptionAsJson(e, expectedJson);
+
+        ElasticsearchException parsed;
+        try (XContentParser parser = createParser(XContentType.JSON.xContent(), expectedJson)) {
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            parsed = ElasticsearchException.fromXContent(parser);
+            assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());
+            assertNull(parser.nextToken());
+        }
+
+        assertNotNull(parsed);
+        assertEquals(parsed.getMessage(), "Elasticsearch exception [type=exception, reason=foo]");
+        assertThat(parsed.getHeaderKeys(), hasSize(2));
+        assertEquals(parsed.getHeader("foo_0").get(0), "0");
+        assertEquals(parsed.getHeader("foo_1").get(0), "1");
+        assertThat(parsed.getMetadataKeys(), hasSize(2));
+        assertEquals(parsed.getMetadata("es.metadata_foo_0").get(0), "foo_0");
+        assertEquals(parsed.getMetadata("es.metadata_foo_1").get(0), "foo_1");
+
+        ElasticsearchException cause = (ElasticsearchException) parsed.getCause();
+        assertEquals(cause.getMessage(), "Elasticsearch exception [type=exception, reason=bar]");
+
+        cause = (ElasticsearchException) cause.getCause();
+        assertEquals(cause.getMessage(), "Elasticsearch exception [type=exception, reason=baz]");
+
+        cause = (ElasticsearchException) cause.getCause();
+        assertEquals(cause.getMessage(),
+                "Elasticsearch exception [type=cluster_block_exception, reason=blocked by: [SERVICE_UNAVAILABLE/2/no master];]");
+    }
+
+    public void testFromXContent() throws IOException {
+        final XContent xContent = randomFrom(XContentType.values()).xContent();
+        XContentBuilder builder = XContentBuilder.builder(xContent)
+                                                    .startObject()
+                                                        .field("type", "foo")
+                                                        .field("reason", "something went wrong")
+                                                        .field("stack_trace", "...")
+                                                    .endObject();
+
+        builder = shuffleXContent(builder);
+        ElasticsearchException parsed;
+        try (XContentParser parser = createParser(xContent, builder.bytes())) {
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            parsed = ElasticsearchException.fromXContent(parser);
+            assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());
+            assertNull(parser.nextToken());
+        }
+
+        assertNotNull(parsed);
+        assertEquals(parsed.getMessage(), "Elasticsearch exception [type=foo, reason=something went wrong, stack_trace=...]");
+    }
+
+    public void testFromXContentWithCause() throws IOException {
+        ElasticsearchException e = new ElasticsearchException("foo",
+                new ElasticsearchException("bar",
+                        new ElasticsearchException("baz",
+                                new RoutingMissingException("_test", "_type", "_id"))));
+
+        final XContent xContent = randomFrom(XContentType.values()).xContent();
+        XContentBuilder builder = XContentBuilder.builder(xContent).startObject().value(e).endObject();
+        builder = shuffleXContent(builder);
+
+        ElasticsearchException parsed;
+        try (XContentParser parser = createParser(builder)) {
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            parsed = ElasticsearchException.fromXContent(parser);
+            assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());
+            assertNull(parser.nextToken());
+        }
+
+        assertNotNull(parsed);
+        assertEquals(parsed.getMessage(), "Elasticsearch exception [type=exception, reason=foo]");
+
+        ElasticsearchException cause = (ElasticsearchException) parsed.getCause();
+        assertEquals(cause.getMessage(), "Elasticsearch exception [type=exception, reason=bar]");
+
+        cause = (ElasticsearchException) cause.getCause();
+        assertEquals(cause.getMessage(), "Elasticsearch exception [type=exception, reason=baz]");
+
+        cause = (ElasticsearchException) cause.getCause();
+        assertEquals(cause.getMessage(),
+                "Elasticsearch exception [type=routing_missing_exception, reason=routing is required for [_test]/[_type]/[_id]]");
+        assertThat(cause.getHeaderKeys(), hasSize(0));
+        assertThat(cause.getMetadataKeys(), hasSize(2));
+        assertThat(cause.getMetadata("es.index"), hasItem("_test"));
+        assertThat(cause.getMetadata("es.index_uuid"), hasItem("_na_"));
+    }
+
+    public void testFromXContentWithHeadersAndMetadata() throws IOException {
+        RoutingMissingException routing = new RoutingMissingException("_test", "_type", "_id");
+        ElasticsearchException baz = new ElasticsearchException("baz", routing);
+        baz.addHeader("baz_0", "baz0");
+        baz.addMetadata("es.baz_1", "baz1");
+        baz.addHeader("baz_2", "baz2");
+        baz.addMetadata("es.baz_3", "baz3");
+        ElasticsearchException bar = new ElasticsearchException("bar", baz);
+        bar.addMetadata("es.bar_0", "bar0");
+        bar.addHeader("bar_1", "bar1");
+        bar.addMetadata("es.bar_2", "bar2");
+        ElasticsearchException foo = new ElasticsearchException("foo", bar);
+        foo.addMetadata("es.foo_0", "foo0");
+        foo.addHeader("foo_1", "foo1");
+
+        final XContent xContent = randomFrom(XContentType.values()).xContent();
+        XContentBuilder builder = XContentBuilder.builder(xContent).startObject().value(foo).endObject();
+        builder = shuffleXContent(builder);
+
+        ElasticsearchException parsed;
+        try (XContentParser parser = createParser(builder)) {
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            parsed = ElasticsearchException.fromXContent(parser);
+            assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());
+            assertNull(parser.nextToken());
+        }
+
+        assertNotNull(parsed);
+        assertEquals(parsed.getMessage(), "Elasticsearch exception [type=exception, reason=foo]");
+        assertThat(parsed.getHeaderKeys(), hasSize(1));
+        assertThat(parsed.getHeader("foo_1"), hasItem("foo1"));
+        assertThat(parsed.getMetadataKeys(), hasSize(1));
+        assertThat(parsed.getMetadata("es.foo_0"), hasItem("foo0"));
+
+        ElasticsearchException cause = (ElasticsearchException) parsed.getCause();
+        assertEquals(cause.getMessage(), "Elasticsearch exception [type=exception, reason=bar]");
+        assertThat(cause.getHeaderKeys(), hasSize(1));
+        assertThat(cause.getHeader("bar_1"), hasItem("bar1"));
+        assertThat(cause.getMetadataKeys(), hasSize(2));
+        assertThat(cause.getMetadata("es.bar_0"), hasItem("bar0"));
+        assertThat(cause.getMetadata("es.bar_2"), hasItem("bar2"));
+
+        cause = (ElasticsearchException) cause.getCause();
+        assertEquals(cause.getMessage(), "Elasticsearch exception [type=exception, reason=baz]");
+        assertThat(cause.getHeaderKeys(), hasSize(2));
+        assertThat(cause.getHeader("baz_0"), hasItem("baz0"));
+        assertThat(cause.getHeader("baz_2"), hasItem("baz2"));
+        assertThat(cause.getMetadataKeys(), hasSize(2));
+        assertThat(cause.getMetadata("es.baz_1"), hasItem("baz1"));
+        assertThat(cause.getMetadata("es.baz_3"), hasItem("baz3"));
+
+        cause = (ElasticsearchException) cause.getCause();
+        assertEquals(cause.getMessage(),
+                "Elasticsearch exception [type=routing_missing_exception, reason=routing is required for [_test]/[_type]/[_id]]");
+        assertThat(cause.getHeaderKeys(), hasSize(0));
+        assertThat(cause.getMetadataKeys(), hasSize(2));
+        assertThat(cause.getMetadata("es.index"), hasItem("_test"));
+        assertThat(cause.getMetadata("es.index_uuid"), hasItem("_na_"));
+    }
+
+    /**
+     * Test that some values like arrays of numbers are ignored when parsing back
+     * an exception.
+     */
+    public void testFromXContentWithIgnoredMetadataAndHeaders() throws IOException {
+        final XContent xContent = randomFrom(XContentType.values()).xContent();
+
+        // The exception content to parse is built using a XContentBuilder
+        // because the current Java API does not allow to add metadata/headers
+        // of other types than list of strings.
+        BytesReference originalBytes;
+        try (XContentBuilder builder = XContentBuilder.builder(xContent)) {
+            builder.startObject()
+                    .field("metadata_int", 1)
+                    .array("metadata_array_of_ints", new int[]{8, 13, 21})
+                    .field("reason", "Custom reason")
+                    .array("metadata_array_of_boolean", new boolean[]{false, false})
+                    .startArray("metadata_array_of_objects")
+                        .startObject()
+                            .field("object_array_one", "value_one")
+                        .endObject()
+                        .startObject()
+                            .field("object_array_two", "value_two")
+                        .endObject()
+                    .endArray()
+                    .field("type", "custom_exception")
+                    .field("metadata_long", 1L)
+                    .array("metadata_array_of_longs", new long[]{2L, 3L, 5L})
+                    .field("metadata_other", "some metadata")
+                    .startObject("header")
+                        .field("header_string", "some header")
+                        .array("header_array_of_strings", new String[]{"foo", "bar", "baz"})
+                    .endObject()
+                    .startObject("metadata_object")
+                        .field("object_field", "value")
+                    .endObject()
+            .endObject();
+            try (XContentBuilder shuffledBuilder = shuffleXContent(builder)) {
+                originalBytes = shuffledBuilder.bytes();
+            }
+        }
+
+        ElasticsearchException parsedException;
+        try (XContentParser parser = createParser(xContent, originalBytes)) {
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            parsedException = ElasticsearchException.fromXContent(parser);
+            assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());
+            assertNull(parser.nextToken());
+        }
+
+        assertNotNull(parsedException);
+        assertEquals("Elasticsearch exception [type=custom_exception, reason=Custom reason]", parsedException.getMessage());
+        assertEquals(2, parsedException.getHeaderKeys().size());
+        assertThat(parsedException.getHeader("header_string"), hasItem("some header"));
+        assertThat(parsedException.getHeader("header_array_of_strings"), hasItems("foo", "bar", "baz"));
+        assertEquals(1, parsedException.getMetadataKeys().size());
+        assertThat(parsedException.getMetadata("es.metadata_other"), hasItem("some metadata"));
+    }
+
+    public void testThrowableToAndFromXContent() throws IOException {
+        final XContent xContent = randomFrom(XContentType.values()).xContent();
+
+        final Tuple<Throwable, ElasticsearchException> exceptions = randomExceptions();
+        final Throwable throwable = exceptions.v1();
+
+        BytesReference throwableBytes = toShuffledXContent((builder, params) -> {
+            ElasticsearchException.generateThrowableXContent(builder, params, throwable);
+            return builder;
+        }, xContent.type(), ToXContent.EMPTY_PARAMS, randomBoolean());
+
+        ElasticsearchException parsedException;
+        try (XContentParser parser = createParser(xContent, throwableBytes)) {
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            parsedException = ElasticsearchException.fromXContent(parser);
+            assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());
+            assertNull(parser.nextToken());
+        }
+        assertDeepEquals(exceptions.v2(), parsedException);
+    }
+
+    public void testUnknownFailureToAndFromXContent() throws IOException {
+        final XContent xContent = randomFrom(XContentType.values()).xContent();
+
+        BytesReference failureBytes = toShuffledXContent((builder, params) -> {
+            // Prints a null failure using generateFailureXContent()
+            ElasticsearchException.generateFailureXContent(builder, params, null, randomBoolean());
+            return builder;
+        }, xContent.type(), ToXContent.EMPTY_PARAMS, randomBoolean());
+
+        ElasticsearchException parsedFailure;
+        try (XContentParser parser = createParser(xContent, failureBytes)) {
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
+            parsedFailure = ElasticsearchException.failureFromXContent(parser);
+            assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
+            assertNull(parser.nextToken());
+        }
+
+        // Failure was null, expecting a "unknown" reason
+        assertEquals("Elasticsearch exception [type=exception, reason=unknown]", parsedFailure.getMessage());
+        assertEquals(0, parsedFailure.getHeaders().size());
+        assertEquals(0, parsedFailure.getMetadata().size());
+    }
+
+    public void testFailureToAndFromXContentWithNoDetails() throws IOException {
+        final XContent xContent = randomFrom(XContentType.values()).xContent();
+
+        final Exception failure = (Exception) randomExceptions().v1();
+        BytesReference failureBytes = toShuffledXContent((builder, params) -> {
+            ElasticsearchException.generateFailureXContent(builder, params, failure, false);
+            return builder;
+        }, xContent.type(), ToXContent.EMPTY_PARAMS, randomBoolean());
+
+        try (XContentParser parser = createParser(xContent, failureBytes)) {
+            failureBytes = shuffleXContent(parser, randomBoolean()).bytes();
+        }
+
+        ElasticsearchException parsedFailure;
+        try (XContentParser parser = createParser(xContent, failureBytes)) {
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
+            parsedFailure = ElasticsearchException.failureFromXContent(parser);
+            assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
+            assertNull(parser.nextToken());
+        }
+        assertNotNull(parsedFailure);
+
+        String reason;
+        if (failure instanceof ElasticsearchException) {
+            reason = failure.getClass().getSimpleName() + "[" + failure.getMessage() + "]";
+        } else {
+            reason = "No ElasticsearchException found";
+        }
+        assertEquals(ElasticsearchException.buildMessage("exception", reason, null), parsedFailure.getMessage());
+        assertEquals(0, parsedFailure.getHeaders().size());
+        assertEquals(0, parsedFailure.getMetadata().size());
+        assertNull(parsedFailure.getCause());
+    }
+
+    public void testFailureToAndFromXContentWithDetails() throws IOException {
+        final XContent xContent = randomFrom(XContentType.values()).xContent();
+
+        Exception failure;
+        Throwable failureCause;
+        ElasticsearchException expected;
+        ElasticsearchException expectedCause;
+        ElasticsearchException suppressed;
+
+        switch (randomIntBetween(0, 6)) {
+            case 0: // Simple elasticsearch exception without cause
+                failure = new NoNodeAvailableException("A");
+
+                expected = new ElasticsearchException("Elasticsearch exception [type=no_node_available_exception, reason=A]");
+                expected.addSuppressed(new ElasticsearchException("Elasticsearch exception [type=no_node_available_exception, reason=A]"));
+                break;
+
+            case 1: // Simple elasticsearch exception with headers (other metadata of type number are not parsed)
+                failure = new CircuitBreakingException("B", 5_000, 2_000);
+                ((ElasticsearchException) failure).addHeader("header_name", "0", "1");
+                expected = new ElasticsearchException("Elasticsearch exception [type=circuit_breaking_exception, reason=B]");
+                expected.addHeader("header_name", "0", "1");
+                suppressed = new ElasticsearchException("Elasticsearch exception [type=circuit_breaking_exception, reason=B]");
+                suppressed.addHeader("header_name", "0", "1");
+                expected.addSuppressed(suppressed);
+                break;
+
+            case 2: // Elasticsearch exception with a cause, headers and parsable metadata
+                failureCause = new NullPointerException("var is null");
+                failure = new ScriptException("C", failureCause, singletonList("stack"), "test", "painless");
+                ((ElasticsearchException) failure).addHeader("script_name", "my_script");
+
+                expectedCause = new ElasticsearchException("Elasticsearch exception [type=null_pointer_exception, reason=var is null]");
+                expected = new ElasticsearchException("Elasticsearch exception [type=script_exception, reason=C]", expectedCause);
+                expected.addHeader("script_name", "my_script");
+                expected.addMetadata("es.lang", "painless");
+                expected.addMetadata("es.script", "test");
+                expected.addMetadata("es.script_stack", "stack");
+                suppressed = new ElasticsearchException("Elasticsearch exception [type=script_exception, reason=C]");
+                suppressed.addHeader("script_name", "my_script");
+                suppressed.addMetadata("es.lang", "painless");
+                suppressed.addMetadata("es.script", "test");
+                suppressed.addMetadata("es.script_stack", "stack");
+                expected.addSuppressed(suppressed);
+                break;
+
+            case 3: // JDK exception without cause
+                failure = new IllegalStateException("D");
+
+                expected = new ElasticsearchException("Elasticsearch exception [type=illegal_state_exception, reason=D]");
+                suppressed = new ElasticsearchException("Elasticsearch exception [type=illegal_state_exception, reason=D]");
+                expected.addSuppressed(suppressed);
+                break;
+
+            case 4: // JDK exception with cause
+                failureCause = new RoutingMissingException("idx", "type", "id");
+                failure = new RuntimeException("E", failureCause);
+
+                expectedCause = new ElasticsearchException("Elasticsearch exception [type=routing_missing_exception, " +
+                        "reason=routing is required for [idx]/[type]/[id]]");
+                expectedCause.addMetadata("es.index", "idx");
+                expectedCause.addMetadata("es.index_uuid", "_na_");
+                expected = new ElasticsearchException("Elasticsearch exception [type=runtime_exception, reason=E]", expectedCause);
+                suppressed = new ElasticsearchException("Elasticsearch exception [type=runtime_exception, reason=E]");
+                expected.addSuppressed(suppressed);
+                break;
+
+            case 5: // Wrapped exception with cause
+                failureCause = new FileAlreadyExistsException("File exists");
+                failure = new BroadcastShardOperationFailedException(new ShardId("_index", "_uuid", 5), "F", failureCause);
+
+                expected = new ElasticsearchException("Elasticsearch exception [type=file_already_exists_exception, reason=File exists]");
+                // strangely, the wrapped exception appears as the root cause...
+                suppressed = new ElasticsearchException("Elasticsearch exception [type=broadcast_shard_operation_failed_exception, " +
+                        "reason=F]");
+                expected.addSuppressed(suppressed);
+                break;
+
+            case 6: // SearchPhaseExecutionException with cause and multiple failures
+                DiscoveryNode node = new DiscoveryNode("node_g", LocalTransportAddress.buildUnique(), Version.CURRENT);
+                failureCause = new NodeClosedException(node);
+                failureCause = new NoShardAvailableActionException(new ShardId("_index_g", "_uuid_g", 6), "node_g", failureCause);
+                ShardSearchFailure[] shardFailures = new ShardSearchFailure[]{
+                        new ShardSearchFailure(new ParsingException(0, 0, "Parsing g", null),
+                                new SearchShardTarget("node_g", new ShardId(new Index("_index_g", "_uuid_g"), 61), null,
+                                    OriginalIndices.NONE)), new ShardSearchFailure(new RepositoryException("repository_g", "Repo"),
+                                new SearchShardTarget("node_g", new ShardId(new Index("_index_g", "_uuid_g"), 62), null,
+                                    OriginalIndices.NONE)), new ShardSearchFailure(new SearchContextMissingException(0L), null)
+                };
+                failure = new SearchPhaseExecutionException("phase_g", "G", failureCause, shardFailures);
+
+                expectedCause = new ElasticsearchException("Elasticsearch exception [type=node_closed_exception, " +
+                        "reason=node closed " + node + "]");
+                expectedCause = new ElasticsearchException("Elasticsearch exception [type=no_shard_available_action_exception, " +
+                        "reason=node_g]", expectedCause);
+                expectedCause.addMetadata("es.index", "_index_g");
+                expectedCause.addMetadata("es.index_uuid", "_uuid_g");
+                expectedCause.addMetadata("es.shard", "6");
+
+                expected = new ElasticsearchException("Elasticsearch exception [type=search_phase_execution_exception, " +
+                        "reason=G]", expectedCause);
+                expected.addMetadata("es.phase", "phase_g");
+
+                expected.addSuppressed(new ElasticsearchException("Elasticsearch exception [type=parsing_exception, reason=Parsing g]"));
+                expected.addSuppressed(new ElasticsearchException("Elasticsearch exception [type=repository_exception, " +
+                        "reason=[repository_g] Repo]"));
+                expected.addSuppressed(new ElasticsearchException("Elasticsearch exception [type=search_context_missing_exception, " +
+                        "reason=No search context found for id [0]]"));
+                break;
+            default:
+                throw new UnsupportedOperationException("Failed to generate randomized failure");
+        }
+
+        Exception finalFailure = failure;
+        BytesReference failureBytes = toShuffledXContent((builder, params) -> {
+            ElasticsearchException.generateFailureXContent(builder, params, finalFailure, true);
+            return builder;
+        }, xContent.type(), ToXContent.EMPTY_PARAMS, randomBoolean());
+
+        try (XContentParser parser = createParser(xContent, failureBytes)) {
+            failureBytes = shuffleXContent(parser, randomBoolean()).bytes();
+        }
+
+        ElasticsearchException parsedFailure;
+        try (XContentParser parser = createParser(xContent, failureBytes)) {
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
+            parsedFailure = ElasticsearchException.failureFromXContent(parser);
+            assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
+            assertNull(parser.nextToken());
+        }
+
+        assertDeepEquals(expected, parsedFailure);
+    }
+
+    /**
+     * Builds a {@link ToXContent} using a JSON XContentBuilder and compares the result to the given json in string format.
+     *
+     * By default, the stack trace of the exception is not rendered. The parameter `errorTrace` forces the stack trace to
+     * be rendered like the REST API does when the "error_trace" parameter is set to true.
+     */
+    private static void assertToXContentAsJson(ToXContent e, String expectedJson) throws IOException {
+        BytesReference actual = XContentHelper.toXContent(e, XContentType.JSON, randomBoolean());
+        assertToXContentEquivalent(new BytesArray(expectedJson), actual, XContentType.JSON);
+    }
+
+    private static void assertExceptionAsJson(Exception e, String expectedJson) throws IOException {
+        assertToXContentAsJson((builder, params) -> {
+            ElasticsearchException.generateThrowableXContent(builder, params, e);
+            return builder;
+        }, expectedJson);
+    }
+
+    public static void assertDeepEquals(ElasticsearchException expected, ElasticsearchException actual) {
+        do {
+            if (expected == null) {
+                assertNull(actual);
+            } else {
+                assertNotNull(actual);
+            }
+
+            assertEquals(expected.getMessage(), actual.getMessage());
+            assertEquals(expected.getHeaders(), actual.getHeaders());
+            assertEquals(expected.getMetadata(), actual.getMetadata());
+            assertEquals(expected.getResourceType(), actual.getResourceType());
+            assertEquals(expected.getResourceId(), actual.getResourceId());
+
+            Throwable[] expectedSuppressed = expected.getSuppressed();
+            Throwable[] actualSuppressed = actual.getSuppressed();
+
+            if (expectedSuppressed == null) {
+                assertNull(actualSuppressed);
+            } else {
+                assertNotNull(actualSuppressed);
+                assertEquals(expectedSuppressed.length, actualSuppressed.length);
+                for (int i = 0; i < expectedSuppressed.length; i++) {
+                    assertDeepEquals((ElasticsearchException) expectedSuppressed[i], (ElasticsearchException) actualSuppressed[i]);
+                }
+            }
+
+            expected = (ElasticsearchException) expected.getCause();
+            actual = (ElasticsearchException) actual.getCause();
+            if (expected == null) {
+                assertNull(actual);
+            }
+        } while (expected != null);
+    }
+
+    public static Tuple<Throwable, ElasticsearchException> randomExceptions() {
+        Throwable actual;
+        ElasticsearchException expected;
+
+        int type = randomIntBetween(0, 5);
+        switch (type) {
+            case 0:
+                actual = new ClusterBlockException(singleton(DiscoverySettings.NO_MASTER_BLOCK_WRITES));
+                expected = new ElasticsearchException("Elasticsearch exception [type=cluster_block_exception, " +
+                        "reason=blocked by: [SERVICE_UNAVAILABLE/2/no master];]");
+                break;
+            case 1:
+                actual = new CircuitBreakingException("Data too large", 123, 456);
+                expected = new ElasticsearchException("Elasticsearch exception [type=circuit_breaking_exception, reason=Data too large]");
+                break;
+            case 2:
+                actual = new SearchParseException(new TestSearchContext(null), "Parse failure", new XContentLocation(12, 98));
+                expected = new ElasticsearchException("Elasticsearch exception [type=search_parse_exception, reason=Parse failure]");
+                break;
+            case 3:
+                actual = new IllegalArgumentException("Closed resource", new RuntimeException("Resource"));
+                expected = new ElasticsearchException("Elasticsearch exception [type=illegal_argument_exception, reason=Closed resource]",
+                                new ElasticsearchException("Elasticsearch exception [type=runtime_exception, reason=Resource]"));
+                break;
+            case 4:
+                actual = new SearchPhaseExecutionException("search", "all shards failed",
+                            new ShardSearchFailure[]{
+                                    new ShardSearchFailure(new ParsingException(1, 2, "foobar", null),
+                                            new SearchShardTarget("node_1", new Index("foo", "_na_"), 1, null))
+                            });
+                expected = new ElasticsearchException("Elasticsearch exception [type=search_phase_execution_exception, " +
+                        "reason=all shards failed]");
+                expected.addMetadata("es.phase", "search");
+                break;
+            case 5:
+                actual = new ElasticsearchException("Parsing failed",
+                            new ParsingException(9, 42, "Wrong state",
+                                new NullPointerException("Unexpected null value")));
+
+                ElasticsearchException expectedCause = new ElasticsearchException("Elasticsearch exception [type=parsing_exception, " +
+                        "reason=Wrong state]", new ElasticsearchException("Elasticsearch exception [type=null_pointer_exception, " +
+                        "reason=Unexpected null value]"));
+                expected = new ElasticsearchException("Elasticsearch exception [type=exception, reason=Parsing failed]", expectedCause);
+                break;
+            default:
+                throw new UnsupportedOperationException("No randomized exceptions generated for type [" + type + "]");
+        }
+
+        if (actual instanceof ElasticsearchException) {
+            ElasticsearchException actualException = (ElasticsearchException) actual;
+            if (randomBoolean()) {
+                int nbHeaders = randomIntBetween(1, 5);
+                Map<String, List<String>> randomHeaders = new HashMap<>(nbHeaders);
+
+                for (int i = 0; i < nbHeaders; i++) {
+                    List<String> values = new ArrayList<>();
+
+                    int nbValues = randomIntBetween(1, 3);
+                    for (int j = 0; j < nbValues; j++) {
+                        values.add(frequently() ? randomAlphaOfLength(5) : "");
+                    }
+                    randomHeaders.put("header_" + i, values);
+                }
+
+                for (Map.Entry<String, List<String>> entry : randomHeaders.entrySet()) {
+                    actualException.addHeader(entry.getKey(), entry.getValue());
+                    expected.addHeader(entry.getKey(), entry.getValue());
+                }
+
+                if (rarely()) {
+                    // Empty or null headers are not printed out by the toXContent method
+                    actualException.addHeader("ignored", randomBoolean() ? emptyList() : null);
+                }
+            }
+
+            if (randomBoolean()) {
+                int nbMetadata = randomIntBetween(1, 5);
+                Map<String, List<String>> randomMetadata = new HashMap<>(nbMetadata);
+
+                for (int i = 0; i < nbMetadata; i++) {
+                    List<String> values = new ArrayList<>();
+
+                    int nbValues = randomIntBetween(1, 3);
+                    for (int j = 0; j < nbValues; j++) {
+                        values.add(frequently() ? randomAlphaOfLength(5) : "");
+                    }
+                    randomMetadata.put("es.metadata_" + i, values);
+                }
+
+                for (Map.Entry<String, List<String>> entry : randomMetadata.entrySet()) {
+                    actualException.addMetadata(entry.getKey(), entry.getValue());
+                    expected.addMetadata(entry.getKey(), entry.getValue());
+                }
+
+                if (rarely()) {
+                    // Empty or null metadata are not printed out by the toXContent method
+                    actualException.addMetadata("es.ignored", randomBoolean() ? emptyList() : null);
+                }
+            }
+
+            if (randomBoolean()) {
+                int nbResources = randomIntBetween(1, 5);
+                for (int i = 0; i < nbResources; i++) {
+                    String resourceType = "type_" + i;
+                    String[] resourceIds = new String[randomIntBetween(1, 3)];
+                    for (int j = 0; j < resourceIds.length; j++) {
+                        resourceIds[j] = frequently() ? randomAlphaOfLength(5) : "";
+                    }
+                    actualException.setResources(resourceType, resourceIds);
+                    expected.setResources(resourceType, resourceIds);
+                }
+            }
+        }
+        return new Tuple<>(actual, expected);
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java b/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
index a3b0629e8abfb..e3c9e877f7ead 100644
--- a/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
+++ b/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
@@ -18,6 +18,11 @@
  */
 package org.elasticsearch;
 
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.IndexFormatTooNewException;
+import org.apache.lucene.index.IndexFormatTooOldException;
+import org.apache.lucene.store.AlreadyClosedException;
+import org.apache.lucene.store.LockObtainFailedException;
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.RoutingMissingException;
 import org.elasticsearch.action.TimestampParsingException;
@@ -33,8 +38,12 @@
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.breaker.CircuitBreakingException;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.NotSerializableExceptionWrapper;
@@ -44,14 +53,11 @@
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.CancellableThreadsTests;
 import org.elasticsearch.common.util.set.Sets;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentLocation;
 import org.elasticsearch.discovery.DiscoverySettings;
+import org.elasticsearch.env.ShardLockObtainFailedException;
 import org.elasticsearch.index.AlreadyExpiredException;
 import org.elasticsearch.index.Index;
-import org.elasticsearch.index.engine.IndexFailedEngineException;
 import org.elasticsearch.index.engine.RecoveryEngineException;
 import org.elasticsearch.index.query.QueryShardException;
 import org.elasticsearch.index.shard.IllegalIndexShardStateException;
@@ -82,6 +88,8 @@
 import org.elasticsearch.transport.ConnectTransportException;
 import org.elasticsearch.transport.TcpTransport;
 
+import java.io.EOFException;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.net.URISyntaxException;
 import java.nio.file.AccessDeniedException;
@@ -98,6 +106,7 @@
 import java.nio.file.Path;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.util.Arrays;
+import java.util.Base64;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
@@ -108,7 +117,9 @@
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
 import static java.util.Collections.singleton;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertVersionSerializable;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.instanceOf;
 
 public class ExceptionSerializationTests extends ESTestCase {
 
@@ -121,7 +132,6 @@ public void testExceptionRegistration()
         final Path startPath = PathUtils.get(ElasticsearchException.class.getProtectionDomain().getCodeSource().getLocation().toURI())
                 .resolve("org").resolve("elasticsearch");
         final Set<? extends Class<?>> ignore = Sets.newHashSet(
-                org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException.class,
                 CancellableThreadsTests.CustomException.class,
                 org.elasticsearch.rest.BytesRestResponseTests.WithHeadersException.class,
                 AbstractClientHeadersTestCase.InternalException.class);
@@ -162,10 +172,10 @@ private void checkClass(Class<?> clazz) {
                 if (isEsException(clazz) == false) {
                     return;
                 }
-                if (ElasticsearchException.isRegistered(clazz.asSubclass(Throwable.class)) == false
+                if (ElasticsearchException.isRegistered(clazz.asSubclass(Throwable.class), Version.CURRENT) == false
                         && ElasticsearchException.class.equals(clazz.getEnclosingClass()) == false) {
                     notRegistered.add(clazz);
-                } else if (ElasticsearchException.isRegistered(clazz.asSubclass(Throwable.class))) {
+                } else if (ElasticsearchException.isRegistered(clazz.asSubclass(Throwable.class), Version.CURRENT)) {
                     registered.add(clazz);
                     try {
                         if (clazz.getMethod("writeTo", StreamOutput.class) != null) {
@@ -220,10 +230,17 @@ public TestException(StreamInput in) throws IOException {
     }
 
     private <T extends Exception> T serialize(T exception) throws IOException {
-        ElasticsearchAssertions.assertVersionSerializable(VersionUtils.randomVersion(random()), exception);
+       return serialize(exception, VersionUtils.randomVersion(random()));
+    }
+
+    private <T extends Exception> T serialize(T exception, Version version) throws IOException {
+        ElasticsearchAssertions.assertVersionSerializable(version, exception);
         BytesStreamOutput out = new BytesStreamOutput();
+        out.setVersion(version);
         out.writeException(exception);
+
         StreamInput in = out.bytes().streamInput();
+        in.setVersion(version);
         return in.readException();
     }
 
@@ -269,7 +286,7 @@ public void testQueryShardException() throws IOException {
     }
 
     public void testSearchException() throws IOException {
-        SearchShardTarget target = new SearchShardTarget("foo", new Index("bar", "_na_"), 1);
+        SearchShardTarget target = new SearchShardTarget("foo", new Index("bar", "_na_"), 1, null);
         SearchException ex = serialize(new SearchException(target, "hello world"));
         assertEquals(target, ex.shard());
         assertEquals(ex.getMessage(), "hello world");
@@ -399,21 +416,6 @@ public void testTimestampParsingException() throws IOException {
         assertEquals("TIMESTAMP", ex.timestamp());
     }
 
-    public void testIndexFailedEngineException() throws IOException {
-        ShardId id = new ShardId("foo", "_na_", 1);
-        IndexFailedEngineException ex = serialize(new IndexFailedEngineException(id, "type", "id", null));
-        assertEquals(ex.getShardId(), new ShardId("foo", "_na_", 1));
-        assertEquals("type", ex.type());
-        assertEquals("id", ex.id());
-        assertNull(ex.getCause());
-
-        ex = serialize(new IndexFailedEngineException(null, "type", "id", new NullPointerException()));
-        assertNull(ex.getShardId());
-        assertEquals("type", ex.type());
-        assertEquals("id", ex.id());
-        assertTrue(ex.getCause() instanceof NullPointerException);
-    }
-
     public void testAliasesMissingException() throws IOException {
         AliasesNotFoundException ex = serialize(new AliasesNotFoundException("one", "two", "three"));
         assertEquals("aliases [one, two, three] missing", ex.getMessage());
@@ -528,30 +530,16 @@ public void testClusterBlockException() throws IOException {
         assertEquals(1, ex.blocks().size());
     }
 
-    private String toXContent(ToXContent x) {
-        try {
-            XContentBuilder builder = XContentFactory.jsonBuilder();
-            builder.startObject();
-            x.toXContent(builder, ToXContent.EMPTY_PARAMS);
-            builder.endObject();
-            return builder.string();
-        } catch (IOException e) {
-            return "{ \"error\" : \"" + e.getMessage() + "\"}";
-        }
-    }
-
     public void testNotSerializableExceptionWrapper() throws IOException {
         NotSerializableExceptionWrapper ex = serialize(new NotSerializableExceptionWrapper(new NullPointerException()));
-        assertEquals("{\"type\":\"null_pointer_exception\",\"reason\":\"null_pointer_exception: null\"}", toXContent(ex));
+        assertEquals("{\"type\":\"null_pointer_exception\",\"reason\":\"null_pointer_exception: null\"}", Strings.toString(ex));
         ex = serialize(new NotSerializableExceptionWrapper(new IllegalArgumentException("nono!")));
-        assertEquals("{\"type\":\"illegal_argument_exception\",\"reason\":\"illegal_argument_exception: nono!\"}", toXContent(ex));
+        assertEquals("{\"type\":\"illegal_argument_exception\",\"reason\":\"illegal_argument_exception: nono!\"}", Strings.toString(ex));
 
         class UnknownException extends Exception {
-
-            public UnknownException(final String message) {
+            UnknownException(final String message) {
                 super(message);
             }
-
         }
 
         Exception[] unknowns = new Exception[]{
@@ -576,33 +564,99 @@ public UnknownException(final String message) {
         }
     }
 
+    public void testUnknownException() throws IOException {
+        ParsingException parsingException = new ParsingException(1, 2, "foobar", null);
+        final Exception ex = new UnknownException("eggplant", parsingException);
+        Exception exception = serialize(ex);
+        assertEquals("unknown_exception: eggplant", exception.getMessage());
+        assertTrue(exception instanceof ElasticsearchException);
+        ParsingException e = (ParsingException)exception.getCause();
+        assertEquals(parsingException.getIndex(), e.getIndex());
+        assertEquals(parsingException.getMessage(), e.getMessage());
+        assertEquals(parsingException.getLineNumber(), e.getLineNumber());
+        assertEquals(parsingException.getColumnNumber(), e.getColumnNumber());
+    }
+
+    public void testWriteThrowable() throws IOException {
+        final QueryShardException queryShardException = new QueryShardException(new Index("foo", "_na_"), "foobar", null);
+        final UnknownException unknownException = new UnknownException("this exception is unknown", queryShardException);
+
+        final Exception[] causes = new Exception[]{
+                new IllegalStateException("foobar"),
+                new IllegalArgumentException("alalaal"),
+                new NullPointerException("boom"),
+                new EOFException("dadada"),
+                new ElasticsearchSecurityException("nono!"),
+                new NumberFormatException("not a number"),
+                new CorruptIndexException("baaaam booom", "this is my resource"),
+                new IndexFormatTooNewException("tooo new", 1, 2, 3),
+                new IndexFormatTooOldException("tooo new", 1, 2, 3),
+                new IndexFormatTooOldException("tooo new", "very old version"),
+                new ArrayIndexOutOfBoundsException("booom"),
+                new StringIndexOutOfBoundsException("booom"),
+                new FileNotFoundException("booom"),
+                new NoSuchFileException("booom"),
+                new AlreadyClosedException("closed!!", new NullPointerException()),
+                new LockObtainFailedException("can't lock directory", new NullPointerException()),
+                unknownException};
+        for (final Exception cause : causes) {
+            ElasticsearchException ex = new ElasticsearchException("topLevel", cause);
+            ElasticsearchException deserialized = serialize(ex);
+            assertEquals(deserialized.getMessage(), ex.getMessage());
+            assertTrue("Expected: " + deserialized.getCause().getMessage() + " to contain: " +
+                            ex.getCause().getClass().getName() + " but it didn't",
+                    deserialized.getCause().getMessage().contains(ex.getCause().getMessage()));
+            if (ex.getCause().getClass() != UnknownException.class) { // unknown exception is not directly mapped
+                assertEquals(deserialized.getCause().getClass(), ex.getCause().getClass());
+            } else {
+                assertEquals(deserialized.getCause().getClass(), NotSerializableExceptionWrapper.class);
+            }
+            assertArrayEquals(deserialized.getStackTrace(), ex.getStackTrace());
+            assertTrue(deserialized.getStackTrace().length > 1);
+            assertVersionSerializable(VersionUtils.randomVersion(random()), cause);
+            assertVersionSerializable(VersionUtils.randomVersion(random()), ex);
+            assertVersionSerializable(VersionUtils.randomVersion(random()), deserialized);
+        }
+    }
+
     public void testWithRestHeadersException() throws IOException {
-        ElasticsearchException ex = new ElasticsearchException("msg");
-        ex.addHeader("foo", "foo", "bar");
-        ex = serialize(ex);
-        assertEquals("msg", ex.getMessage());
-        assertEquals(2, ex.getHeader("foo").size());
-        assertEquals("foo", ex.getHeader("foo").get(0));
-        assertEquals("bar", ex.getHeader("foo").get(1));
-
-        RestStatus status = randomFrom(RestStatus.values());
-        // ensure we are carrying over the headers even if not serialized
-        UnknownHeaderException uhe = new UnknownHeaderException("msg", status);
-        uhe.addHeader("foo", "foo", "bar");
-
-        ElasticsearchException serialize = serialize((ElasticsearchException) uhe);
-        assertTrue(serialize instanceof NotSerializableExceptionWrapper);
-        NotSerializableExceptionWrapper e = (NotSerializableExceptionWrapper) serialize;
-        assertEquals("unknown_header_exception: msg", e.getMessage());
-        assertEquals(2, e.getHeader("foo").size());
-        assertEquals("foo", e.getHeader("foo").get(0));
-        assertEquals("bar", e.getHeader("foo").get(1));
-        assertSame(status, e.status());
+        {
+            ElasticsearchException ex = new ElasticsearchException("msg");
+            ex.addHeader("foo", "foo", "bar");
+            ex.addMetadata("es.foo_metadata", "value1", "value2");
+            ex = serialize(ex);
+            assertEquals("msg", ex.getMessage());
+            assertEquals(2, ex.getHeader("foo").size());
+            assertEquals("foo", ex.getHeader("foo").get(0));
+            assertEquals("bar", ex.getHeader("foo").get(1));
+            assertEquals(2, ex.getMetadata("es.foo_metadata").size());
+            assertEquals("value1", ex.getMetadata("es.foo_metadata").get(0));
+            assertEquals("value2", ex.getMetadata("es.foo_metadata").get(1));
+        }
+        {
+            RestStatus status = randomFrom(RestStatus.values());
+            // ensure we are carrying over the headers and metadata even if not serialized
+            UnknownHeaderException uhe = new UnknownHeaderException("msg", status);
+            uhe.addHeader("foo", "foo", "bar");
+            uhe.addMetadata("es.foo_metadata", "value1", "value2");
+
+            ElasticsearchException serialize = serialize((ElasticsearchException) uhe);
+            assertTrue(serialize instanceof NotSerializableExceptionWrapper);
+            NotSerializableExceptionWrapper e = (NotSerializableExceptionWrapper) serialize;
+            assertEquals("unknown_header_exception: msg", e.getMessage());
+            assertEquals(2, e.getHeader("foo").size());
+            assertEquals("foo", e.getHeader("foo").get(0));
+            assertEquals("bar", e.getHeader("foo").get(1));
+            assertEquals(2, e.getMetadata("es.foo_metadata").size());
+            assertEquals("value1", e.getMetadata("es.foo_metadata").get(0));
+            assertEquals("value2", e.getMetadata("es.foo_metadata").get(1));
+            assertSame(status, e.status());
+        }
     }
 
     public void testNoLongerPrimaryShardException() throws IOException {
-        ShardId shardId = new ShardId(new Index(randomAsciiOfLength(4), randomAsciiOfLength(4)), randomIntBetween(0, Integer.MAX_VALUE));
-        String msg = randomAsciiOfLength(4);
+        ShardId shardId = new ShardId(new Index(randomAlphaOfLength(4), randomAlphaOfLength(4)), randomIntBetween(0, Integer.MAX_VALUE));
+        String msg = randomAlphaOfLength(4);
         ShardStateAction.NoLongerPrimaryShardException ex = serialize(new ShardStateAction.NoLongerPrimaryShardException(shardId, msg));
         assertEquals(shardId, ex.getShardId());
         assertEquals(msg, ex.getMessage());
@@ -611,7 +665,7 @@ public void testNoLongerPrimaryShardException() throws IOException {
     public static class UnknownHeaderException extends ElasticsearchException {
         private final RestStatus status;
 
-        public UnknownHeaderException(String msg, RestStatus status) {
+        UnknownHeaderException(String msg, RestStatus status) {
             super(msg);
             this.status = status;
         }
@@ -636,15 +690,15 @@ public void testInterruptedException() throws IOException {
     }
 
     public void testThatIdsArePositive() {
-        for (ElasticsearchException.ElasticsearchExceptionHandle handle : ElasticsearchException.ElasticsearchExceptionHandle.values()) {
-            assertThat("negative id", handle.id, greaterThanOrEqualTo(0));
+        for (final int id : ElasticsearchException.ids()) {
+            assertThat("negative id", id, greaterThanOrEqualTo(0));
         }
     }
 
     public void testThatIdsAreUnique() {
-        Set<Integer> ids = new HashSet<>();
-        for (ElasticsearchException.ElasticsearchExceptionHandle handle : ElasticsearchException.ElasticsearchExceptionHandle.values()) {
-            assertTrue("duplicate id", ids.add(handle.id));
+        final Set<Integer> ids = new HashSet<>();
+        for (final int id: ElasticsearchException.ids()) {
+            assertTrue("duplicate id", ids.add(id));
         }
     }
 
@@ -678,7 +732,7 @@ public void testIds() {
         ids.put(25, org.elasticsearch.script.GeneralScriptException.class);
         ids.put(26, org.elasticsearch.index.shard.TranslogRecoveryPerformer.BatchOperationException.class);
         ids.put(27, org.elasticsearch.snapshots.SnapshotCreationException.class);
-        ids.put(28, org.elasticsearch.index.engine.DeleteFailedEngineException.class);
+        ids.put(28, org.elasticsearch.index.engine.DeleteFailedEngineException.class); //deprecated in 6.0
         ids.put(29, org.elasticsearch.index.engine.DocumentMissingException.class);
         ids.put(30, org.elasticsearch.snapshots.SnapshotException.class);
         ids.put(31, org.elasticsearch.indices.InvalidAliasNameException.class);
@@ -730,7 +784,7 @@ public void testIds() {
         ids.put(77, org.elasticsearch.common.util.concurrent.UncategorizedExecutionException.class);
         ids.put(78, org.elasticsearch.action.TimestampParsingException.class);
         ids.put(79, org.elasticsearch.action.RoutingMissingException.class);
-        ids.put(80, org.elasticsearch.index.engine.IndexFailedEngineException.class);
+        ids.put(80, org.elasticsearch.index.engine.IndexFailedEngineException.class); //deprecated in 6.0
         ids.put(81, org.elasticsearch.index.snapshots.IndexShardRestoreFailedException.class);
         ids.put(82, org.elasticsearch.repositories.RepositoryException.class);
         ids.put(83, org.elasticsearch.transport.ReceiveTimeoutTransportException.class);
@@ -757,7 +811,7 @@ public void testIds() {
         ids.put(107, org.elasticsearch.repositories.RepositoryMissingException.class);
         ids.put(108, null);
         ids.put(109, org.elasticsearch.index.engine.DocumentSourceMissingException.class);
-        ids.put(110, org.elasticsearch.index.engine.FlushNotAllowedEngineException.class);
+        ids.put(110, null); // FlushNotAllowedEngineException was removed in 5.0
         ids.put(111, org.elasticsearch.common.settings.NoClassSettingsException.class);
         ids.put(112, org.elasticsearch.transport.BindTransportException.class);
         ids.put(113, org.elasticsearch.rest.action.admin.indices.AliasesNotFoundException.class);
@@ -770,7 +824,7 @@ public void testIds() {
         ids.put(120, org.elasticsearch.repositories.RepositoryVerificationException.class);
         ids.put(121, org.elasticsearch.search.aggregations.InvalidAggregationPathException.class);
         ids.put(122, null);
-        ids.put(123, org.elasticsearch.indices.IndexAlreadyExistsException.class);
+        ids.put(123, org.elasticsearch.ResourceAlreadyExistsException.class);
         ids.put(124, null);
         ids.put(125, TcpTransport.HttpOnTransportException.class);
         ids.put(126, org.elasticsearch.index.mapper.MapperParsingException.class);
@@ -793,6 +847,9 @@ public void testIds() {
         ids.put(143, org.elasticsearch.script.ScriptException.class);
         ids.put(144, org.elasticsearch.cluster.NotMasterException.class);
         ids.put(145, org.elasticsearch.ElasticsearchStatusException.class);
+        ids.put(146, org.elasticsearch.tasks.TaskCancelledException.class);
+        ids.put(147, org.elasticsearch.env.ShardLockObtainFailedException.class);
+        ids.put(148, org.elasticsearch.common.xcontent.NamedXContentRegistry.UnknownNamedObjectException.class);
 
         Map<Class<? extends ElasticsearchException>, Integer> reverse = new HashMap<>();
         for (Map.Entry<Integer, Class<? extends ElasticsearchException>> entry : ids.entrySet()) {
@@ -801,8 +858,9 @@ public void testIds() {
             }
         }
 
-        for (ElasticsearchException.ElasticsearchExceptionHandle handle : ElasticsearchException.ElasticsearchExceptionHandle.values()) {
-            assertEquals((int) reverse.get(handle.exceptionClass), handle.id);
+        for (final Tuple<Integer, Class<? extends ElasticsearchException>> tuple : ElasticsearchException.classes()) {
+            assertNotNull(tuple.v1());
+            assertEquals((int) reverse.get(tuple.v2()), (int)tuple.v1());
         }
 
         for (Map.Entry<Integer, Class<? extends ElasticsearchException>> entry : ids.entrySet()) {
@@ -850,4 +908,96 @@ public void testElasticsearchRemoteException() throws IOException {
         assertEquals(ex.status(), e.status());
         assertEquals(RestStatus.TOO_MANY_REQUESTS, e.status());
     }
+
+    public void testShardLockObtainFailedException() throws IOException {
+        ShardId shardId = new ShardId("foo", "_na_", 1);
+        ShardLockObtainFailedException orig = new ShardLockObtainFailedException(shardId, "boom");
+        Version version = VersionUtils.randomVersionBetween(random(), Version.V_5_0_0, Version.CURRENT);
+        if (version.before(Version.V_5_0_2)) {
+            version = Version.V_5_0_2;
+        }
+        ShardLockObtainFailedException ex = serialize(orig, version);
+        assertEquals(orig.getMessage(), ex.getMessage());
+        assertEquals(orig.getShardId(), ex.getShardId());
+    }
+
+    public void testBWCShardLockObtainFailedException() throws IOException {
+        ShardId shardId = new ShardId("foo", "_na_", 1);
+        ShardLockObtainFailedException orig = new ShardLockObtainFailedException(shardId, "boom");
+        Exception ex = serialize((Exception)orig, randomFrom(Version.V_5_0_0, Version.V_5_0_1));
+        assertThat(ex, instanceOf(NotSerializableExceptionWrapper.class));
+        assertEquals("shard_lock_obtain_failed_exception: [foo][1]: boom", ex.getMessage());
+    }
+
+    public void testBWCHeadersAndMetadata() throws IOException {
+        //this is a request serialized with headers only, no metadata as they were added in 5.3.0
+        BytesReference decoded = new BytesArray(Base64.getDecoder().decode
+                ("AQ10ZXN0ICBtZXNzYWdlACYtb3JnLmVsYXN0aWNzZWFyY2guRXhjZXB0aW9uU2VyaWFsaXphdGlvblRlc3RzASBFeGNlcHRpb25TZXJpYWxpemF0aW9uVG" +
+                        "VzdHMuamF2YQR0ZXN03wYkc3VuLnJlZmxlY3QuTmF0aXZlTWV0aG9kQWNjZXNzb3JJbXBsAR1OYXRpdmVNZXRob2RBY2Nlc3NvckltcGwuamF2Y" +
+                        "QdpbnZva2Uw/v///w8kc3VuLnJlZmxlY3QuTmF0aXZlTWV0aG9kQWNjZXNzb3JJbXBsAR1OYXRpdmVNZXRob2RBY2Nlc3NvckltcGwuamF2YQZp" +
+                        "bnZva2U+KHN1bi5yZWZsZWN0LkRlbGVnYXRpbmdNZXRob2RBY2Nlc3NvckltcGwBIURlbGVnYXRpbmdNZXRob2RBY2Nlc3NvckltcGwuamF2YQZ" +
+                        "pbnZva2UrGGphdmEubGFuZy5yZWZsZWN0Lk1ldGhvZAELTWV0aG9kLmphdmEGaW52b2tl8QMzY29tLmNhcnJvdHNlYXJjaC5yYW5kb21pemVkdG" +
+                        "VzdGluZy5SYW5kb21pemVkUnVubmVyARVSYW5kb21pemVkUnVubmVyLmphdmEGaW52b2tlsQ01Y29tLmNhcnJvdHNlYXJjaC5yYW5kb21pemVkd" +
+                        "GVzdGluZy5SYW5kb21pemVkUnVubmVyJDgBFVJhbmRvbWl6ZWRSdW5uZXIuamF2YQhldmFsdWF0ZYsHNWNvbS5jYXJyb3RzZWFyY2gucmFuZG9t" +
+                        "aXplZHRlc3RpbmcuUmFuZG9taXplZFJ1bm5lciQ5ARVSYW5kb21pemVkUnVubmVyLmphdmEIZXZhbHVhdGWvBzZjb20uY2Fycm90c2VhcmNoLnJ" +
+                        "hbmRvbWl6ZWR0ZXN0aW5nLlJhbmRvbWl6ZWRSdW5uZXIkMTABFVJhbmRvbWl6ZWRSdW5uZXIuamF2YQhldmFsdWF0Zb0HOWNvbS5jYXJyb3RzZW" +
+                        "FyY2gucmFuZG9taXplZHRlc3RpbmcucnVsZXMuU3RhdGVtZW50QWRhcHRlcgEVU3RhdGVtZW50QWRhcHRlci5qYXZhCGV2YWx1YXRlJDVvcmcuY" +
+                        "XBhY2hlLmx1Y2VuZS51dGlsLlRlc3RSdWxlU2V0dXBUZWFyZG93bkNoYWluZWQkMQEhVGVzdFJ1bGVTZXR1cFRlYXJkb3duQ2hhaW5lZC5qYXZh" +
+                        "CGV2YWx1YXRlMTBvcmcuYXBhY2hlLmx1Y2VuZS51dGlsLkFic3RyYWN0QmVmb3JlQWZ0ZXJSdWxlJDEBHEFic3RyYWN0QmVmb3JlQWZ0ZXJSdWx" +
+                        "lLmphdmEIZXZhbHVhdGUtMm9yZy5hcGFjaGUubHVjZW5lLnV0aWwuVGVzdFJ1bGVUaHJlYWRBbmRUZXN0TmFtZSQxAR5UZXN0UnVsZVRocmVhZE" +
+                        "FuZFRlc3ROYW1lLmphdmEIZXZhbHVhdGUwN29yZy5hcGFjaGUubHVjZW5lLnV0aWwuVGVzdFJ1bGVJZ25vcmVBZnRlck1heEZhaWx1cmVzJDEBI" +
+                        "1Rlc3RSdWxlSWdub3JlQWZ0ZXJNYXhGYWlsdXJlcy5qYXZhCGV2YWx1YXRlQCxvcmcuYXBhY2hlLmx1Y2VuZS51dGlsLlRlc3RSdWxlTWFya0Zh" +
+                        "aWx1cmUkMQEYVGVzdFJ1bGVNYXJrRmFpbHVyZS5qYXZhCGV2YWx1YXRlLzljb20uY2Fycm90c2VhcmNoLnJhbmRvbWl6ZWR0ZXN0aW5nLnJ1bGV" +
+                        "zLlN0YXRlbWVudEFkYXB0ZXIBFVN0YXRlbWVudEFkYXB0ZXIuamF2YQhldmFsdWF0ZSREY29tLmNhcnJvdHNlYXJjaC5yYW5kb21pemVkdGVzdG" +
+                        "luZy5UaHJlYWRMZWFrQ29udHJvbCRTdGF0ZW1lbnRSdW5uZXIBFlRocmVhZExlYWtDb250cm9sLmphdmEDcnVu7wI0Y29tLmNhcnJvdHNlYXJja" +
+                        "C5yYW5kb21pemVkdGVzdGluZy5UaHJlYWRMZWFrQ29udHJvbAEWVGhyZWFkTGVha0NvbnRyb2wuamF2YRJmb3JrVGltZW91dGluZ1Rhc2urBjZj" +
+                        "b20uY2Fycm90c2VhcmNoLnJhbmRvbWl6ZWR0ZXN0aW5nLlRocmVhZExlYWtDb250cm9sJDMBFlRocmVhZExlYWtDb250cm9sLmphdmEIZXZhbHV" +
+                        "hdGXOAzNjb20uY2Fycm90c2VhcmNoLnJhbmRvbWl6ZWR0ZXN0aW5nLlJhbmRvbWl6ZWRSdW5uZXIBFVJhbmRvbWl6ZWRSdW5uZXIuamF2YQ1ydW" +
+                        "5TaW5nbGVUZXN0lAc1Y29tLmNhcnJvdHNlYXJjaC5yYW5kb21pemVkdGVzdGluZy5SYW5kb21pemVkUnVubmVyJDUBFVJhbmRvbWl6ZWRSdW5uZ" +
+                        "XIuamF2YQhldmFsdWF0ZaIGNWNvbS5jYXJyb3RzZWFyY2gucmFuZG9taXplZHRlc3RpbmcuUmFuZG9taXplZFJ1bm5lciQ2ARVSYW5kb21pemVk" +
+                        "UnVubmVyLmphdmEIZXZhbHVhdGXUBjVjb20uY2Fycm90c2VhcmNoLnJhbmRvbWl6ZWR0ZXN0aW5nLlJhbmRvbWl6ZWRSdW5uZXIkNwEVUmFuZG9" +
+                        "taXplZFJ1bm5lci5qYXZhCGV2YWx1YXRl3wYwb3JnLmFwYWNoZS5sdWNlbmUudXRpbC5BYnN0cmFjdEJlZm9yZUFmdGVyUnVsZSQxARxBYnN0cm" +
+                        "FjdEJlZm9yZUFmdGVyUnVsZS5qYXZhCGV2YWx1YXRlLTljb20uY2Fycm90c2VhcmNoLnJhbmRvbWl6ZWR0ZXN0aW5nLnJ1bGVzLlN0YXRlbWVud" +
+                        "EFkYXB0ZXIBFVN0YXRlbWVudEFkYXB0ZXIuamF2YQhldmFsdWF0ZSQvb3JnLmFwYWNoZS5sdWNlbmUudXRpbC5UZXN0UnVsZVN0b3JlQ2xhc3NO" +
+                        "YW1lJDEBG1Rlc3RSdWxlU3RvcmVDbGFzc05hbWUuamF2YQhldmFsdWF0ZSlOY29tLmNhcnJvdHNlYXJjaC5yYW5kb21pemVkdGVzdGluZy5ydWx" +
+                        "lcy5Ob1NoYWRvd2luZ09yT3ZlcnJpZGVzT25NZXRob2RzUnVsZSQxAShOb1NoYWRvd2luZ09yT3ZlcnJpZGVzT25NZXRob2RzUnVsZS5qYXZhCG" +
+                        "V2YWx1YXRlKE5jb20uY2Fycm90c2VhcmNoLnJhbmRvbWl6ZWR0ZXN0aW5nLnJ1bGVzLk5vU2hhZG93aW5nT3JPdmVycmlkZXNPbk1ldGhvZHNSd" +
+                        "WxlJDEBKE5vU2hhZG93aW5nT3JPdmVycmlkZXNPbk1ldGhvZHNSdWxlLmphdmEIZXZhbHVhdGUoOWNvbS5jYXJyb3RzZWFyY2gucmFuZG9taXpl" +
+                        "ZHRlc3RpbmcucnVsZXMuU3RhdGVtZW50QWRhcHRlcgEVU3RhdGVtZW50QWRhcHRlci5qYXZhCGV2YWx1YXRlJDljb20uY2Fycm90c2VhcmNoLnJ" +
+                        "hbmRvbWl6ZWR0ZXN0aW5nLnJ1bGVzLlN0YXRlbWVudEFkYXB0ZXIBFVN0YXRlbWVudEFkYXB0ZXIuamF2YQhldmFsdWF0ZSQ5Y29tLmNhcnJvdH" +
+                        "NlYXJjaC5yYW5kb21pemVkdGVzdGluZy5ydWxlcy5TdGF0ZW1lbnRBZGFwdGVyARVTdGF0ZW1lbnRBZGFwdGVyLmphdmEIZXZhbHVhdGUkM29yZ" +
+                        "y5hcGFjaGUubHVjZW5lLnV0aWwuVGVzdFJ1bGVBc3NlcnRpb25zUmVxdWlyZWQkMQEfVGVzdFJ1bGVBc3NlcnRpb25zUmVxdWlyZWQuamF2YQhl" +
+                        "dmFsdWF0ZTUsb3JnLmFwYWNoZS5sdWNlbmUudXRpbC5UZXN0UnVsZU1hcmtGYWlsdXJlJDEBGFRlc3RSdWxlTWFya0ZhaWx1cmUuamF2YQhldmF" +
+                        "sdWF0ZS83b3JnLmFwYWNoZS5sdWNlbmUudXRpbC5UZXN0UnVsZUlnbm9yZUFmdGVyTWF4RmFpbHVyZXMkMQEjVGVzdFJ1bGVJZ25vcmVBZnRlck" +
+                        "1heEZhaWx1cmVzLmphdmEIZXZhbHVhdGVAMW9yZy5hcGFjaGUubHVjZW5lLnV0aWwuVGVzdFJ1bGVJZ25vcmVUZXN0U3VpdGVzJDEBHVRlc3RSd" +
+                        "WxlSWdub3JlVGVzdFN1aXRlcy5qYXZhCGV2YWx1YXRlNjljb20uY2Fycm90c2VhcmNoLnJhbmRvbWl6ZWR0ZXN0aW5nLnJ1bGVzLlN0YXRlbWVu" +
+                        "dEFkYXB0ZXIBFVN0YXRlbWVudEFkYXB0ZXIuamF2YQhldmFsdWF0ZSREY29tLmNhcnJvdHNlYXJjaC5yYW5kb21pemVkdGVzdGluZy5UaHJlYWR" +
+                        "MZWFrQ29udHJvbCRTdGF0ZW1lbnRSdW5uZXIBFlRocmVhZExlYWtDb250cm9sLmphdmEDcnVu7wIQamF2YS5sYW5nLlRocmVhZAELVGhyZWFkLm" +
+                        "phdmEDcnVu6QUABAdoZWFkZXIyAQZ2YWx1ZTIKZXMuaGVhZGVyMwEGdmFsdWUzB2hlYWRlcjEBBnZhbHVlMQplcy5oZWFkZXI0AQZ2YWx1ZTQAA" +
+                        "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" +
+                        "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" +
+                        "AAAAA"));
+
+        try (StreamInput in = decoded.streamInput()) {
+            //randomize the version across released and unreleased ones
+            Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2,
+                    Version.V_5_1_1, Version.V_5_1_2, Version.V_5_2_0);
+            in.setVersion(version);
+            ElasticsearchException exception = new ElasticsearchException(in);
+            assertEquals("test  message", exception.getMessage());
+            //the headers received as part of a single set get split based on their prefix
+            assertEquals(2, exception.getHeaderKeys().size());
+            assertEquals("value1", exception.getHeader("header1").get(0));
+            assertEquals("value2", exception.getHeader("header2").get(0));
+            assertEquals(2, exception.getMetadataKeys().size());
+            assertEquals("value3", exception.getMetadata("es.header3").get(0));
+            assertEquals("value4", exception.getMetadata("es.header4").get(0));
+        }
+    }
+
+    private static class UnknownException extends Exception {
+        UnknownException(final String message, final Exception cause) {
+            super(message, cause);
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/ResourceAlreadyExistsExceptionTests.java b/core/src/test/java/org/elasticsearch/ResourceAlreadyExistsExceptionTests.java
new file mode 100644
index 0000000000000..d89770e68a46e
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/ResourceAlreadyExistsExceptionTests.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+package org.elasticsearch;
+
+import org.elasticsearch.index.Index;
+import org.elasticsearch.test.ESTestCase;
+
+public class ResourceAlreadyExistsExceptionTests extends ESTestCase {
+
+    public void testGetExceptionName() {
+        assertEquals("resource_already_exists_exception", new ResourceAlreadyExistsException("with a plain msg").getExceptionName());
+        assertEquals("index_already_exists_exception", new ResourceAlreadyExistsException(new Index("myIndex", "1")).getExceptionName());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/VersionTests.java b/core/src/test/java/org/elasticsearch/VersionTests.java
index 862cccab318a2..160868958b018 100644
--- a/core/src/test/java/org/elasticsearch/VersionTests.java
+++ b/core/src/test/java/org/elasticsearch/VersionTests.java
@@ -27,12 +27,18 @@
 import org.hamcrest.Matchers;
 
 import java.lang.reflect.Modifier;
+import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Set;
 
 import static org.elasticsearch.Version.V_2_2_0;
 import static org.elasticsearch.Version.V_5_0_0_alpha1;
+import static org.elasticsearch.test.VersionUtils.allVersions;
 import static org.elasticsearch.test.VersionUtils.randomVersion;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.containsString;
@@ -62,7 +68,37 @@ public void testVersionComparison() throws Exception {
         assertTrue(Version.fromString("5.0.0").onOrAfter(Version.fromString("5.0.0-beta2")));
         assertTrue(Version.fromString("5.0.0-rc1").onOrAfter(Version.fromString("5.0.0-beta24")));
         assertTrue(Version.fromString("5.0.0-alpha24").before(Version.fromString("5.0.0-beta0")));
+    }
+
+    public void testMin() {
+        assertEquals(VersionUtils.getPreviousVersion(), Version.min(Version.CURRENT, VersionUtils.getPreviousVersion()));
+        assertEquals(Version.fromString("1.0.1"), Version.min(Version.fromString("1.0.1"), Version.CURRENT));
+        Version version = VersionUtils.randomVersion(random());
+        Version version1 = VersionUtils.randomVersion(random());
+        if (version.id <= version1.id) {
+            assertEquals(version, Version.min(version1, version));
+        } else {
+            assertEquals(version1, Version.min(version1, version));
+        }
+    }
+
+    public void testMax() {
+        assertEquals(Version.CURRENT, Version.max(Version.CURRENT, VersionUtils.getPreviousVersion()));
+        assertEquals(Version.CURRENT, Version.max(Version.fromString("1.0.1"), Version.CURRENT));
+        Version version = VersionUtils.randomVersion(random());
+        Version version1 = VersionUtils.randomVersion(random());
+        if (version.id >= version1.id) {
+            assertEquals(version, Version.max(version1, version));
+        } else {
+            assertEquals(version1, Version.max(version1, version));
+        }
+    }
 
+    public void testMinimumIndexCompatibilityVersion() {
+        assertEquals(Version.V_5_0_0, Version.fromString("6.0.0").minimumIndexCompatibilityVersion());
+        assertEquals(Version.V_2_0_0_beta1, Version.V_5_0_0.minimumIndexCompatibilityVersion());
+        assertEquals(Version.V_2_0_0_beta1, Version.V_5_1_1.minimumIndexCompatibilityVersion());
+        assertEquals(Version.V_2_0_0_beta1, Version.V_5_0_0_alpha1.minimumIndexCompatibilityVersion());
     }
 
     public void testVersionConstantPresent() {
@@ -95,40 +131,23 @@ public void testVersionFromString() {
     }
 
     public void testTooLongVersionFromString() {
-        try {
-            Version.fromString("1.0.0.1.3");
-            fail("Expected IllegalArgumentException");
-        } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), containsString("needs to contain major, minor, and revision"));
-        }
+        Exception e = expectThrows(IllegalArgumentException.class, () -> Version.fromString("1.0.0.1.3"));
+        assertThat(e.getMessage(), containsString("needs to contain major, minor, and revision"));
     }
 
     public void testTooShortVersionFromString() {
-        try {
-            Version.fromString("1.0");
-            fail("Expected IllegalArgumentException");
-        } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), containsString("needs to contain major, minor, and revision"));
-        }
-
+        Exception e = expectThrows(IllegalArgumentException.class, () -> Version.fromString("1.0"));
+        assertThat(e.getMessage(), containsString("needs to contain major, minor, and revision"));
     }
 
     public void testWrongVersionFromString() {
-        try {
-            Version.fromString("WRONG.VERSION");
-            fail("Expected IllegalArgumentException");
-        } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), containsString("needs to contain major, minor, and revision"));
-        }
+        Exception e = expectThrows(IllegalArgumentException.class, () -> Version.fromString("WRONG.VERSION"));
+        assertThat(e.getMessage(), containsString("needs to contain major, minor, and revision"));
     }
 
     public void testVersionNoPresentInSettings() {
-        try {
-            Version.indexCreated(Settings.builder().build());
-            fail("Expected IllegalArgumentException");
-        } catch (IllegalStateException e) {
-            assertThat(e.getMessage(), containsString("[index.version.created] is not present"));
-        }
+        Exception e = expectThrows(IllegalStateException.class, () -> Version.indexCreated(Settings.builder().build()));
+        assertThat(e.getMessage(), containsString("[index.version.created] is not present"));
     }
 
     public void testIndexCreatedVersion() {
@@ -209,7 +228,7 @@ public void testParseVersion() {
 
     public void testParseLenient() {
         // note this is just a silly sanity check, we test it in lucene
-        for (Version version : VersionUtils.allVersions()) {
+        for (Version version : VersionUtils.allReleasedVersions()) {
             org.apache.lucene.util.Version luceneVersion = version.luceneVersion;
             String string = luceneVersion.toString().toUpperCase(Locale.ROOT)
                     .replaceFirst("^LUCENE_(\\d+)_(\\d+)$", "$1.$2");
@@ -218,20 +237,27 @@ public void testParseLenient() {
     }
 
     public void testAllVersionsMatchId() throws Exception {
+        final Set<Version> releasedVersions = new HashSet<>(VersionUtils.allReleasedVersions());
+        final Set<Version> unreleasedVersions = new HashSet<>(VersionUtils.allUnreleasedVersions());
         Map<String, Version> maxBranchVersions = new HashMap<>();
         for (java.lang.reflect.Field field : Version.class.getFields()) {
-            if (field.getName().endsWith("_ID")) {
+            if (field.getName().matches("_ID(_UNRELEASED)?")) {
                 assertTrue(field.getName() + " should be static", Modifier.isStatic(field.getModifiers()));
                 assertTrue(field.getName() + " should be final", Modifier.isFinal(field.getModifiers()));
                 int versionId = (Integer)field.get(Version.class);
 
-                String constantName = field.getName().substring(0, field.getName().length() - 3);
+                String constantName = field.getName().substring(0, field.getName().indexOf("_ID"));
                 java.lang.reflect.Field versionConstant = Version.class.getField(constantName);
                 assertTrue(constantName + " should be static", Modifier.isStatic(versionConstant.getModifiers()));
                 assertTrue(constantName + " should be final", Modifier.isFinal(versionConstant.getModifiers()));
 
-                Version v = (Version) versionConstant.get(Version.class);
-                logger.info("Checking {}", v);
+                Version v = (Version) versionConstant.get(null);
+                logger.debug("Checking {}", v);
+                if (field.getName().endsWith("_UNRELEASED")) {
+                    assertTrue(unreleasedVersions.contains(v));
+                } else {
+                    assertTrue(releasedVersions.contains(v));
+                }
                 assertEquals("Version id " + field.getName() + " does not point to " + constantName, v, Version.fromId(versionId));
                 assertEquals("Version " + constantName + " does not have correct id", versionId, v.id);
                 if (v.major >= 2) {
@@ -264,8 +290,8 @@ public void testAllVersionsMatchId() throws Exception {
 
     // this test ensures we never bump the lucene version in a bugfix release
     public void testLuceneVersionIsSameOnMinorRelease() {
-        for (Version version : VersionUtils.allVersions()) {
-            for (Version other : VersionUtils.allVersions()) {
+        for (Version version : VersionUtils.allReleasedVersions()) {
+            for (Version other : VersionUtils.allReleasedVersions()) {
                 if (other.onOrAfter(version)) {
                     assertTrue("lucene versions must be "  + other + " >= " + version,
                         other.luceneVersion.onOrAfter(version.luceneVersion));
@@ -279,4 +305,34 @@ public void testLuceneVersionIsSameOnMinorRelease() {
             }
         }
     }
+
+    public static void assertUnknownVersion(Version version) {
+        assertFalse("Version " + version + " has been released don't use a new instance of this version",
+            VersionUtils.allReleasedVersions().contains(version));
+    }
+
+    public void testIsCompatible() {
+        assertTrue(isCompatible(Version.CURRENT, Version.CURRENT.minimumCompatibilityVersion()));
+        assertTrue(isCompatible(Version.CURRENT, Version.fromString("6.0.0")));
+        assertFalse("only compatible with the latest minor",
+            isCompatible(VersionUtils.getPreviousMinorVersion(), Version.fromString("6.0.0")));
+        assertFalse(isCompatible(Version.V_2_0_0, Version.fromString("6.0.0")));
+        assertFalse(isCompatible(Version.V_2_0_0, Version.V_5_0_0));
+    }
+
+    /* tests that if a new version's minCompatVersion is always equal or higher to any older version */
+    public void testMinCompatVersionOrderRespectsVersionOrder() {
+        List<Version> versionsByMinCompat = new ArrayList<>(allVersions());
+        versionsByMinCompat.sort(Comparator.comparing(Version::minimumCompatibilityVersion));
+        assertThat(versionsByMinCompat, equalTo(allVersions()));
+
+        versionsByMinCompat.sort(Comparator.comparing(Version::minimumIndexCompatibilityVersion));
+        assertThat(versionsByMinCompat, equalTo(allVersions()));
+    }
+
+    public boolean isCompatible(Version left, Version right) {
+        boolean result = left.isCompatible(right);
+        assert result == right.isCompatible(left);
+        return result;
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/ActionListenerTests.java b/core/src/test/java/org/elasticsearch/action/ActionListenerTests.java
new file mode 100644
index 0000000000000..6414c81058bec
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/ActionListenerTests.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action;
+
+import org.elasticsearch.common.CheckedConsumer;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+public class ActionListenerTests extends ESTestCase {
+
+    public void testWrap() {
+        AtomicReference<Boolean> reference = new AtomicReference<>();
+        AtomicReference<Exception> exReference = new AtomicReference<>();
+
+        CheckedConsumer<Boolean, ? extends Exception> handler = (o) -> {
+            if (Boolean.FALSE.equals(o)) {
+                throw new IllegalArgumentException("must not be false");
+            }
+            reference.set(o);
+        };
+        ActionListener<Boolean> wrap = ActionListener.wrap(handler, exReference::set);
+        wrap.onResponse(Boolean.FALSE);
+        assertNull(reference.get());
+        assertNotNull(exReference.get());
+        assertEquals("must not be false", exReference.get().getMessage());
+        exReference.set(null);
+
+        wrap.onResponse(Boolean.TRUE);
+        assertTrue(reference.get());
+        assertNull(exReference.get());
+    }
+
+    public void testOnResponse() {
+        final int numListeners = randomIntBetween(1, 20);
+        List<AtomicReference<Boolean>> refList = new ArrayList<>();
+        List<AtomicReference<Exception>> excList = new ArrayList<>();
+        List<ActionListener<Boolean>> listeners = new ArrayList<>();
+        List<Boolean> failOnTrue = new ArrayList<>();
+        AtomicInteger exceptionCounter = new AtomicInteger(0);
+        for (int i = 0; i < numListeners; i++) {
+            boolean doFailOnTrue = rarely();
+            failOnTrue.add(doFailOnTrue);
+            AtomicReference<Boolean> reference = new AtomicReference<>();
+            AtomicReference<Exception> exReference = new AtomicReference<>();
+            refList.add(reference);
+            excList.add(exReference);
+            CheckedConsumer<Boolean, ? extends Exception> handler = (o) -> {
+                if (Boolean.FALSE.equals(o)) {
+                    throw new IllegalArgumentException("must not be false " + exceptionCounter.getAndIncrement());
+                }
+                if (doFailOnTrue) {
+                    throw new IllegalStateException("must not be true");
+                }
+                reference.set(o);
+            };
+            listeners.add(ActionListener.wrap(handler, exReference::set));
+        }
+
+        ActionListener.onResponse(listeners, Boolean.TRUE);
+        for (int i = 0; i < numListeners; i++) {
+            if (failOnTrue.get(i) == false) {
+                assertTrue("listener index " + i, refList.get(i).get());
+                refList.get(i).set(null);
+            } else {
+                assertNull("listener index " + i, refList.get(i).get());
+            }
+
+        }
+
+        for (int i = 0; i < numListeners; i++) {
+            if (failOnTrue.get(i) == false) {
+                assertNull("listener index " + i, excList.get(i).get());
+            } else {
+                assertEquals("listener index " + i, "must not be true", excList.get(i).get().getMessage());
+            }
+        }
+
+        ActionListener.onResponse(listeners, Boolean.FALSE);
+        for (int i = 0; i < numListeners; i++) {
+            assertNull("listener index " + i, refList.get(i).get());
+        }
+
+        assertEquals(numListeners, exceptionCounter.get());
+        for (int i = 0; i < numListeners; i++) {
+            assertNotNull(excList.get(i).get());
+            assertEquals("listener index " + i, "must not be false " + i, excList.get(i).get().getMessage());
+        }
+    }
+
+    public void testOnFailure() {
+        final int numListeners = randomIntBetween(1, 20);
+        List<AtomicReference<Boolean>> refList = new ArrayList<>();
+        List<AtomicReference<Exception>> excList = new ArrayList<>();
+        List<ActionListener<Boolean>> listeners = new ArrayList<>();
+
+        final int listenerToFail = randomBoolean() ? -1 : randomIntBetween(0, numListeners-1);
+        for (int i = 0; i < numListeners; i++) {
+            AtomicReference<Boolean> reference = new AtomicReference<>();
+            AtomicReference<Exception> exReference = new AtomicReference<>();
+            refList.add(reference);
+            excList.add(exReference);
+            boolean fail = i == listenerToFail;
+            CheckedConsumer<Boolean, ? extends Exception> handler = (o) -> {
+                reference.set(o);
+            };
+            listeners.add(ActionListener.wrap(handler, (e) -> {
+                exReference.set(e);
+                if (fail) {
+                    throw new RuntimeException("double boom");
+                }
+            }));
+        }
+
+        try {
+            ActionListener.onFailure(listeners, new Exception("booom"));
+            assertTrue("unexpected succces listener to fail: " + listenerToFail, listenerToFail == -1);
+        } catch (RuntimeException ex) {
+            assertTrue("listener to fail: " + listenerToFail, listenerToFail >= 0);
+            assertNotNull(ex.getCause());
+            assertEquals("double boom", ex.getCause().getMessage());
+        }
+
+        for (int i = 0; i < numListeners; i++) {
+            assertNull("listener index " + i, refList.get(i).get());
+        }
+
+        for (int i = 0; i < numListeners; i++) {
+            assertEquals("listener index " + i, "booom", excList.get(i).get().getMessage());
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/ActionModuleTests.java b/core/src/test/java/org/elasticsearch/action/ActionModuleTests.java
index 2c17cea5ef54c..7c9d54434644a 100644
--- a/core/src/test/java/org/elasticsearch/action/ActionModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/action/ActionModuleTests.java
@@ -25,23 +25,33 @@
 import org.elasticsearch.action.support.TransportAction;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsFilter;
+import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.ActionPlugin.ActionHandler;
 import org.elasticsearch.rest.RestChannel;
+import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestHandler;
 import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestRequest.Method;
 import org.elasticsearch.rest.action.RestMainAction;
 import org.elasticsearch.tasks.TaskManager;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 
+import java.io.IOException;
 import java.util.List;
+import java.util.function.Supplier;
 
 import static java.util.Collections.emptyList;
 import static java.util.Collections.singletonList;
 import static org.hamcrest.Matchers.hasEntry;
-import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.startsWith;
 
 public class ActionModuleTests extends ESTestCase {
     public void testSetupActionsContainsKnownBuiltin() {
@@ -52,7 +62,7 @@ public void testSetupActionsContainsKnownBuiltin() {
     public void testPluginCantOverwriteBuiltinAction() {
         ActionPlugin dupsMainAction = new ActionPlugin() {
             @Override
-            public List<ActionHandler<? extends ActionRequest<?>, ? extends ActionResponse>> getActions() {
+            public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
                 return singletonList(new ActionHandler<>(MainAction.INSTANCE, TransportMainAction.class));
             }
         };
@@ -61,7 +71,7 @@ public void testPluginCantOverwriteBuiltinAction() {
     }
 
     public void testPluginCanRegisterAction() {
-        class FakeRequest extends ActionRequest<FakeRequest> {
+        class FakeRequest extends ActionRequest {
             @Override
             public ActionRequestValidationException validate() {
                 return null;
@@ -90,7 +100,7 @@ public ActionResponse newResponse() {
         FakeAction action = new FakeAction();
         ActionPlugin registersFakeAction = new ActionPlugin() {
             @Override
-            public List<ActionHandler<? extends ActionRequest<?>, ? extends ActionResponse>> getActions() {
+            public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
                 return singletonList(new ActionHandler<>(action, FakeTransportAction.class));
             }
         };
@@ -99,32 +109,70 @@ public ActionResponse newResponse() {
     }
 
     public void testSetupRestHandlerContainsKnownBuiltin() {
-        assertThat(ActionModule.setupRestHandlers(emptyList()), hasItem(RestMainAction.class));
+        SettingsModule settings = new SettingsModule(Settings.EMPTY);
+        ActionModule actionModule = new ActionModule(false, settings.getSettings(), new IndexNameExpressionResolver(Settings.EMPTY),
+                settings.getIndexScopedSettings(), settings.getClusterSettings(), settings.getSettingsFilter(), null, emptyList(), null,
+                null);
+        actionModule.initRestHandlers(null);
+        // At this point the easiest way to confirm that a handler is loaded is to try to register another one on top of it and to fail
+        Exception e = expectThrows(IllegalArgumentException.class, () ->
+            actionModule.getRestController().registerHandler(Method.GET, "/", null));
+        assertThat(e.getMessage(), startsWith("Path [/] already has a value [" + RestMainAction.class.getName()));
     }
 
-    public void testPluginCantOverwriteBuiltinRestHandler() {
+    public void testPluginCantOverwriteBuiltinRestHandler() throws IOException {
         ActionPlugin dupsMainAction = new ActionPlugin() {
             @Override
-            public List<Class<? extends RestHandler>> getRestHandlers() {
-                return singletonList(RestMainAction.class);
+            public List<RestHandler> getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings,
+                    IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter,
+                    IndexNameExpressionResolver indexNameExpressionResolver, Supplier<DiscoveryNodes> nodesInCluster) {
+                return singletonList(new RestMainAction(settings, restController));
             }
         };
-        Exception e = expectThrows(IllegalArgumentException.class, () -> ActionModule.setupRestHandlers(singletonList(dupsMainAction)));
-        assertEquals("can't register the same [rest_handler] more than once for [" + RestMainAction.class.getName() + "]", e.getMessage());
+        SettingsModule settings = new SettingsModule(Settings.EMPTY);
+        ThreadPool threadPool = new TestThreadPool(getTestName());
+        try {
+            ActionModule actionModule = new ActionModule(false, settings.getSettings(), new IndexNameExpressionResolver(Settings.EMPTY),
+                    settings.getIndexScopedSettings(), settings.getClusterSettings(), settings.getSettingsFilter(), threadPool,
+                    singletonList(dupsMainAction), null, null);
+            Exception e = expectThrows(IllegalArgumentException.class, () -> actionModule.initRestHandlers(null));
+            assertThat(e.getMessage(), startsWith("Path [/] already has a value [" + RestMainAction.class.getName()));
+        } finally {
+            threadPool.shutdown();
+        }
     }
 
     public void testPluginCanRegisterRestHandler() {
         class FakeHandler implements RestHandler {
+            FakeHandler(RestController restController) {
+                restController.registerHandler(Method.GET, "/_dummy", this);
+            }
             @Override
             public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
             }
         }
         ActionPlugin registersFakeHandler = new ActionPlugin() {
             @Override
-            public List<Class<? extends RestHandler>> getRestHandlers() {
-                return singletonList(FakeHandler.class);
+            public List<RestHandler> getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings,
+                    IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter,
+                    IndexNameExpressionResolver indexNameExpressionResolver, Supplier<DiscoveryNodes> nodesInCluster) {
+                return singletonList(new FakeHandler(restController));
             }
         };
-        assertThat(ActionModule.setupRestHandlers(singletonList(registersFakeHandler)), hasItem(FakeHandler.class));
+
+        SettingsModule settings = new SettingsModule(Settings.EMPTY);
+        ThreadPool threadPool = new TestThreadPool(getTestName());
+        try {
+            ActionModule actionModule = new ActionModule(false, settings.getSettings(), new IndexNameExpressionResolver(Settings.EMPTY),
+                    settings.getIndexScopedSettings(), settings.getClusterSettings(), settings.getSettingsFilter(), threadPool,
+                    singletonList(registersFakeHandler), null, null);
+            actionModule.initRestHandlers(null);
+            // At this point the easiest way to confirm that a handler is loaded is to try to register another one on top of it and to fail
+            Exception e = expectThrows(IllegalArgumentException.class, () ->
+                actionModule.getRestController().registerHandler(Method.GET, "/_dummy", null));
+            assertThat(e.getMessage(), startsWith("Path [/_dummy] already has a value [" + FakeHandler.class.getName()));
+        } finally {
+            threadPool.shutdown();
+        }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/DocWriteResponseTests.java b/core/src/test/java/org/elasticsearch/action/DocWriteResponseTests.java
index ce50fbc7e5d9c..594e734f6425d 100644
--- a/core/src/test/java/org/elasticsearch/action/DocWriteResponseTests.java
+++ b/core/src/test/java/org/elasticsearch/action/DocWriteResponseTests.java
@@ -35,14 +35,43 @@
 import static org.hamcrest.Matchers.not;
 
 public class DocWriteResponseTests extends ESTestCase {
+
     public void testGetLocation() {
-        DocWriteResponse response = new DocWriteResponse(new ShardId("index", "uuid", 0), "type", "id", 0, Result.CREATED) {
-            // DocWriteResponse is abstract so we have to sneak a subclass in here to test it.
-        };
+        final DocWriteResponse response =
+            new DocWriteResponse(
+                new ShardId("index", "uuid", 0),
+                "type",
+                "id",
+                0,
+                Result.CREATED) {};
         assertEquals("/index/type/id", response.getLocation(null));
         assertEquals("/index/type/id?routing=test_routing", response.getLocation("test_routing"));
     }
 
+    public void testGetLocationNonAscii() {
+        final DocWriteResponse response =
+            new DocWriteResponse(
+                new ShardId("index", "uuid", 0),
+                "type",
+                "❤",
+                0,
+                Result.CREATED) {};
+        assertEquals("/index/type/%E2%9D%A4", response.getLocation(null));
+        assertEquals("/index/type/%E2%9D%A4?routing=%C3%A4", response.getLocation("ä"));
+    }
+
+    public void testGetLocationWithSpaces() {
+        final DocWriteResponse response =
+                new DocWriteResponse(
+                        new ShardId("index", "uuid", 0),
+                        "type",
+                        "a b",
+                        0,
+                        Result.CREATED) {};
+        assertEquals("/index/type/a+b", response.getLocation(null));
+        assertEquals("/index/type/a+b?routing=c+d", response.getLocation("c d"));
+    }
+
     /**
      * Tests that {@link DocWriteResponse#toXContent(XContentBuilder, ToXContent.Params)} doesn't include {@code forced_refresh} unless it
      * is true. We can't assert this in the yaml tests because "not found" is also "false" there....
@@ -54,19 +83,15 @@ public void testToXContentDoesntIncludeForcedRefreshUnlessForced() throws IOExce
         response.setShardInfo(new ShardInfo(1, 1));
         response.setForcedRefresh(false);
         try (XContentBuilder builder = JsonXContent.contentBuilder()) {
-            builder.startObject();
             response.toXContent(builder, ToXContent.EMPTY_PARAMS);
-            builder.endObject();
-            try (XContentParser parser = JsonXContent.jsonXContent.createParser(builder.bytes())) {
+            try (XContentParser parser = createParser(JsonXContent.jsonXContent, builder.bytes())) {
                 assertThat(parser.map(), not(hasKey("forced_refresh")));
             }
         }
         response.setForcedRefresh(true);
         try (XContentBuilder builder = JsonXContent.contentBuilder()) {
-            builder.startObject();
             response.toXContent(builder, ToXContent.EMPTY_PARAMS);
-            builder.endObject();
-            try (XContentParser parser = JsonXContent.jsonXContent.createParser(builder.bytes())) {
+            try (XContentParser parser = createParser(JsonXContent.jsonXContent, builder.bytes())) {
                 assertThat(parser.map(), hasEntry("forced_refresh", true));
             }
         }
diff --git a/core/src/test/java/org/elasticsearch/action/ExplainRequestTests.java b/core/src/test/java/org/elasticsearch/action/ExplainRequestTests.java
new file mode 100644
index 0000000000000..adab07f52a38d
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/ExplainRequestTests.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.explain.ExplainRequest;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.indices.IndicesModule;
+import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
+import org.elasticsearch.search.internal.AliasFilter;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Base64;
+import java.util.Collections;
+import java.util.List;
+
+public class ExplainRequestTests extends ESTestCase {
+    private NamedWriteableRegistry namedWriteableRegistry;
+
+    public void setUp() throws Exception {
+        super.setUp();
+        IndicesModule indicesModule = new IndicesModule(Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
+        entries.addAll(indicesModule.getNamedWriteables());
+        entries.addAll(searchModule.getNamedWriteables());
+        namedWriteableRegistry = new NamedWriteableRegistry(entries);
+    }
+
+    public void testSerialize() throws IOException {
+        try (BytesStreamOutput output = new BytesStreamOutput()) {
+            ExplainRequest request = new ExplainRequest("index", "type", "id");
+            request.fetchSourceContext(new FetchSourceContext(true, new String[]{"field1.*"}, new String[] {"field2.*"}));
+            request.filteringAlias(new AliasFilter(QueryBuilders.termQuery("filter_field", "value"), new String[] {"alias0", "alias1"}));
+            request.preference("the_preference");
+            request.query(QueryBuilders.termQuery("field", "value"));
+            request.storedFields(new String[] {"field1", "field2"});
+            request.routing("some_routing");
+            request.writeTo(output);
+            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
+                ExplainRequest readRequest = new ExplainRequest();
+                readRequest.readFrom(in);
+                assertEquals(request.filteringAlias(), readRequest.filteringAlias());
+                assertArrayEquals(request.storedFields(), readRequest.storedFields());
+                assertEquals(request.preference(), readRequest.preference());
+                assertEquals(request.query(), readRequest.query());
+                assertEquals(request.routing(), readRequest.routing());
+                assertEquals(request.fetchSourceContext(), readRequest.fetchSourceContext());
+            }
+        }
+    }
+
+    // BWC test for changes from #20916
+    public void testSerialize50Request() throws IOException {
+        ExplainRequest request = new ExplainRequest("index", "type", "id");
+        request.fetchSourceContext(new FetchSourceContext(true, new String[]{"field1.*"}, new String[] {"field2.*"}));
+        request.filteringAlias(new AliasFilter(QueryBuilders.termQuery("filter_field", "value"), new String[] {"alias0", "alias1"}));
+        request.preference("the_preference");
+        request.query(QueryBuilders.termQuery("field", "value"));
+        request.storedFields(new String[] {"field1", "field2"});
+        request.routing("some_routing");
+        BytesArray requestBytes = new BytesArray(Base64.getDecoder()
+            // this is a base64 encoded request generated with the same input
+            .decode("AAABBWluZGV4BHR5cGUCaWQBDHNvbWVfcm91dGluZwEOdGhlX3ByZWZlcmVuY2UEdGVybT" +
+                "+AAAAABWZpZWxkFQV2YWx1ZQIGYWxpYXMwBmFsaWFzMQECBmZpZWxkMQZmaWVsZDIBAQEIZmllbGQxLioBCGZpZWxkMi4qAA"));
+        try (StreamInput in = new NamedWriteableAwareStreamInput(requestBytes.streamInput(), namedWriteableRegistry)) {
+            in.setVersion(Version.V_5_0_0);
+            ExplainRequest readRequest = new ExplainRequest();
+            readRequest.readFrom(in);
+            assertEquals(0, in.available());
+            assertArrayEquals(request.filteringAlias().getAliases(), readRequest.filteringAlias().getAliases());
+            expectThrows(IllegalStateException.class, () -> readRequest.filteringAlias().getQueryBuilder());
+            assertArrayEquals(request.storedFields(), readRequest.storedFields());
+            assertEquals(request.preference(), readRequest.preference());
+            assertEquals(request.query(), readRequest.query());
+            assertEquals(request.routing(), readRequest.routing());
+            assertEquals(request.fetchSourceContext(), readRequest.fetchSourceContext());
+            BytesStreamOutput output = new BytesStreamOutput();
+            output.setVersion(Version.V_5_0_0);
+            readRequest.writeTo(output);
+            assertEquals(output.bytes().toBytesRef(), requestBytes.toBytesRef());
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/IndicesRequestIT.java b/core/src/test/java/org/elasticsearch/action/IndicesRequestIT.java
index 1a11e3f48034c..b77a8f8530140 100644
--- a/core/src/test/java/org/elasticsearch/action/IndicesRequestIT.java
+++ b/core/src/test/java/org/elasticsearch/action/IndicesRequestIT.java
@@ -77,26 +77,26 @@
 import org.elasticsearch.action.update.UpdateAction;
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.action.update.UpdateResponse;
+import org.elasticsearch.client.Requests;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.plugins.NetworkPlugin;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.PluginsService;
 import org.elasticsearch.script.MockScriptPlugin;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.search.action.SearchTransportService;
+import org.elasticsearch.action.search.SearchTransportService;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportChannel;
+import org.elasticsearch.transport.TransportInterceptor;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.transport.TransportRequestHandler;
-import org.elasticsearch.transport.TransportService;
 import org.junit.After;
 import org.junit.Before;
 
@@ -110,7 +110,6 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.function.Function;
-import java.util.function.Supplier;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
@@ -143,8 +142,7 @@ protected Settings nodeSettings(int ordinal) {
         return Settings.builder().put(super.nodeSettings(ordinal))
             // InternalClusterInfoService sends IndicesStatsRequest periodically which messes with this test
             // this setting disables it...
-            .put("cluster.routing.allocation.disk.threshold_enabled", false)
-            .put(NetworkModule.TRANSPORT_SERVICE_TYPE_KEY, "intercepting").build();
+            .put("cluster.routing.allocation.disk.threshold_enabled", false).build();
     }
 
     @Override
@@ -204,10 +202,11 @@ public void testAnalyze() {
     }
 
     public void testIndex() {
-        String[] indexShardActions = new String[]{IndexAction.NAME, IndexAction.NAME + "[p]", IndexAction.NAME + "[r]"};
+        String[] indexShardActions = new String[]{BulkAction.NAME + "[s][p]", BulkAction.NAME + "[s][r]"};
         interceptTransportActions(indexShardActions);
 
-        IndexRequest indexRequest = new IndexRequest(randomIndexOrAlias(), "type", "id").source("field", "value");
+        IndexRequest indexRequest = new IndexRequest(randomIndexOrAlias(), "type", "id")
+            .source(Requests.INDEX_CONTENT_TYPE, "field", "value");
         internalCluster().coordOnlyNodeClient().index(indexRequest).actionGet();
 
         clearInterceptedActions();
@@ -215,7 +214,7 @@ public void testIndex() {
     }
 
     public void testDelete() {
-        String[] deleteShardActions = new String[]{DeleteAction.NAME, DeleteAction.NAME + "[p]", DeleteAction.NAME + "[r]"};
+        String[] deleteShardActions = new String[]{BulkAction.NAME + "[s][p]", BulkAction.NAME + "[s][r]"};
         interceptTransportActions(deleteShardActions);
 
         DeleteRequest deleteRequest = new DeleteRequest(randomIndexOrAlias(), "type", "id");
@@ -227,12 +226,12 @@ public void testDelete() {
 
     public void testUpdate() {
         //update action goes to the primary, index op gets executed locally, then replicated
-        String[] updateShardActions = new String[]{UpdateAction.NAME + "[s]", IndexAction.NAME + "[r]"};
+        String[] updateShardActions = new String[]{UpdateAction.NAME + "[s]", BulkAction.NAME + "[s][p]", BulkAction.NAME + "[s][r]"};
         interceptTransportActions(updateShardActions);
 
         String indexOrAlias = randomIndexOrAlias();
         client().prepareIndex(indexOrAlias, "type", "id").setSource("field", "value").get();
-        UpdateRequest updateRequest = new UpdateRequest(indexOrAlias, "type", "id").doc("field1", "value1");
+        UpdateRequest updateRequest = new UpdateRequest(indexOrAlias, "type", "id").doc(Requests.INDEX_CONTENT_TYPE, "field1", "value1");
         UpdateResponse updateResponse = internalCluster().coordOnlyNodeClient().update(updateRequest).actionGet();
         assertEquals(DocWriteResponse.Result.UPDATED, updateResponse.getResult());
 
@@ -242,11 +241,12 @@ public void testUpdate() {
 
     public void testUpdateUpsert() {
         //update action goes to the primary, index op gets executed locally, then replicated
-        String[] updateShardActions = new String[]{UpdateAction.NAME + "[s]", IndexAction.NAME + "[r]"};
+        String[] updateShardActions = new String[]{UpdateAction.NAME + "[s]", BulkAction.NAME + "[s][p]", BulkAction.NAME + "[s][r]"};
         interceptTransportActions(updateShardActions);
 
         String indexOrAlias = randomIndexOrAlias();
-        UpdateRequest updateRequest = new UpdateRequest(indexOrAlias, "type", "id").upsert("field", "value").doc("field1", "value1");
+        UpdateRequest updateRequest = new UpdateRequest(indexOrAlias, "type", "id").upsert(Requests.INDEX_CONTENT_TYPE, "field", "value")
+            .doc(Requests.INDEX_CONTENT_TYPE, "field1", "value1");
         UpdateResponse updateResponse = internalCluster().coordOnlyNodeClient().update(updateRequest).actionGet();
         assertEquals(DocWriteResponse.Result.CREATED, updateResponse.getResult());
 
@@ -256,13 +256,13 @@ public void testUpdateUpsert() {
 
     public void testUpdateDelete() {
         //update action goes to the primary, delete op gets executed locally, then replicated
-        String[] updateShardActions = new String[]{UpdateAction.NAME + "[s]", DeleteAction.NAME + "[r]"};
+        String[] updateShardActions = new String[]{UpdateAction.NAME + "[s]", BulkAction.NAME + "[s][p]", BulkAction.NAME + "[s][r]"};
         interceptTransportActions(updateShardActions);
 
         String indexOrAlias = randomIndexOrAlias();
         client().prepareIndex(indexOrAlias, "type", "id").setSource("field", "value").get();
         UpdateRequest updateRequest = new UpdateRequest(indexOrAlias, "type", "id")
-                .script(new Script("ctx.op='delete'", ScriptService.ScriptType.INLINE, CustomScriptPlugin.NAME, Collections.emptyMap()));
+                .script(new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "ctx.op='delete'", Collections.emptyMap()));
         UpdateResponse updateResponse = internalCluster().coordOnlyNodeClient().update(updateRequest).actionGet();
         assertEquals(DocWriteResponse.Result.DELETED, updateResponse.getResult());
 
@@ -279,7 +279,7 @@ public void testBulk() {
         int numIndexRequests = iterations(1, 10);
         for (int i = 0; i < numIndexRequests; i++) {
             String indexOrAlias = randomIndexOrAlias();
-            bulkRequest.add(new IndexRequest(indexOrAlias, "type", "id").source("field", "value"));
+            bulkRequest.add(new IndexRequest(indexOrAlias, "type", "id").source(Requests.INDEX_CONTENT_TYPE, "field", "value"));
             indices.add(indexOrAlias);
         }
         int numDeleteRequests = iterations(1, 10);
@@ -291,7 +291,7 @@ public void testBulk() {
         int numUpdateRequests = iterations(1, 10);
         for (int i = 0; i < numUpdateRequests; i++) {
             String indexOrAlias = randomIndexOrAlias();
-            bulkRequest.add(new UpdateRequest(indexOrAlias, "type", "id").doc("field1", "value1"));
+            bulkRequest.add(new UpdateRequest(indexOrAlias, "type", "id").doc(Requests.INDEX_CONTENT_TYPE, "field1", "value1"));
             indices.add(indexOrAlias);
         }
 
@@ -581,48 +581,6 @@ public void testSearchDfsQueryThenFetch() throws Exception {
         assertSameIndicesOptionalRequests(searchRequest, SearchTransportService.FREE_CONTEXT_ACTION_NAME);
     }
 
-    public void testSearchQueryAndFetch() throws Exception {
-        interceptTransportActions(SearchTransportService.QUERY_FETCH_ACTION_NAME,
-                SearchTransportService.FREE_CONTEXT_ACTION_NAME);
-
-        String[] randomIndicesOrAliases = randomIndicesOrAliases();
-        for (int i = 0; i < randomIndicesOrAliases.length; i++) {
-            client().prepareIndex(randomIndicesOrAliases[i], "type", "id-" + i).setSource("field", "value").get();
-        }
-        refresh();
-
-        SearchRequest searchRequest = new SearchRequest(randomIndicesOrAliases).searchType(SearchType.QUERY_AND_FETCH);
-        SearchResponse searchResponse = internalCluster().coordOnlyNodeClient().search(searchRequest).actionGet();
-        assertNoFailures(searchResponse);
-        assertThat(searchResponse.getHits().totalHits(), greaterThan(0L));
-
-        clearInterceptedActions();
-        assertSameIndices(searchRequest, SearchTransportService.QUERY_FETCH_ACTION_NAME);
-        //free context messages are not necessarily sent, but if they are, check their indices
-        assertSameIndicesOptionalRequests(searchRequest, SearchTransportService.FREE_CONTEXT_ACTION_NAME);
-    }
-
-    public void testSearchDfsQueryAndFetch() throws Exception {
-        interceptTransportActions(SearchTransportService.QUERY_QUERY_FETCH_ACTION_NAME,
-                SearchTransportService.FREE_CONTEXT_ACTION_NAME);
-
-        String[] randomIndicesOrAliases = randomIndicesOrAliases();
-        for (int i = 0; i < randomIndicesOrAliases.length; i++) {
-            client().prepareIndex(randomIndicesOrAliases[i], "type", "id-" + i).setSource("field", "value").get();
-        }
-        refresh();
-
-        SearchRequest searchRequest = new SearchRequest(randomIndicesOrAliases).searchType(SearchType.DFS_QUERY_AND_FETCH);
-        SearchResponse searchResponse = internalCluster().coordOnlyNodeClient().search(searchRequest).actionGet();
-        assertNoFailures(searchResponse);
-        assertThat(searchResponse.getHits().totalHits(), greaterThan(0L));
-
-        clearInterceptedActions();
-        assertSameIndices(searchRequest, SearchTransportService.QUERY_QUERY_FETCH_ACTION_NAME);
-        //free context messages are not necessarily sent, but if they are, check their indices
-        assertSameIndicesOptionalRequests(searchRequest, SearchTransportService.FREE_CONTEXT_ACTION_NAME);
-    }
-
     private static void assertSameIndices(IndicesRequest originalRequest, String... actions) {
         assertSameIndices(originalRequest, false, actions);
     }
@@ -701,31 +659,39 @@ private String[] randomUniqueIndicesOrAliases() {
     }
 
     private static void assertAllRequestsHaveBeenConsumed() {
-        Iterable<TransportService> transportServices = internalCluster().getInstances(TransportService.class);
-        for (TransportService transportService : transportServices) {
-            assertThat(((InterceptingTransportService)transportService).requests.entrySet(), emptyIterable());
+        Iterable<PluginsService> pluginsServices = internalCluster().getInstances(PluginsService.class);
+        for (PluginsService pluginsService : pluginsServices) {
+            Set<Map.Entry<String, List<TransportRequest>>> entries =
+                pluginsService.filterPlugins(InterceptingTransportService.TestPlugin.class).stream().findFirst().get()
+                .instance.requests.entrySet();
+            assertThat(entries, emptyIterable());
+
         }
     }
 
     private static void clearInterceptedActions() {
-        Iterable<TransportService> transportServices = internalCluster().getInstances(TransportService.class);
-        for (TransportService transportService : transportServices) {
-            ((InterceptingTransportService) transportService).clearInterceptedActions();
+        Iterable<PluginsService> pluginsServices = internalCluster().getInstances(PluginsService.class);
+        for (PluginsService pluginsService : pluginsServices) {
+            pluginsService.filterPlugins(InterceptingTransportService.TestPlugin.class).stream().findFirst().get()
+                .instance.clearInterceptedActions();
         }
     }
 
     private static void interceptTransportActions(String... actions) {
-        Iterable<TransportService> transportServices = internalCluster().getInstances(TransportService.class);
-        for (TransportService transportService : transportServices) {
-            ((InterceptingTransportService) transportService).interceptTransportActions(actions);
+        Iterable<PluginsService> pluginsServices = internalCluster().getInstances(PluginsService.class);
+        for (PluginsService pluginsService : pluginsServices) {
+            pluginsService.filterPlugins(InterceptingTransportService.TestPlugin.class).stream().findFirst().get()
+                .instance.interceptTransportActions(actions);
         }
     }
 
     private static List<TransportRequest> consumeTransportRequests(String action) {
         List<TransportRequest> requests = new ArrayList<>();
-        Iterable<TransportService> transportServices = internalCluster().getInstances(TransportService.class);
-        for (TransportService transportService : transportServices) {
-            List<TransportRequest> transportRequests = ((InterceptingTransportService) transportService).consumeRequests(action);
+
+        Iterable<PluginsService> pluginsServices = internalCluster().getInstances(PluginsService.class);
+        for (PluginsService pluginsService : pluginsServices) {
+            List<TransportRequest> transportRequests = pluginsService.filterPlugins(InterceptingTransportService.TestPlugin.class)
+                .stream().findFirst().get().instance.consumeRequests(action);
             if (transportRequests != null) {
                 requests.addAll(transportRequests);
             }
@@ -733,12 +699,13 @@ private static List<TransportRequest> consumeTransportRequests(String action) {
         return requests;
     }
 
-    public static class InterceptingTransportService extends TransportService {
-
-        public static class TestPlugin extends Plugin {
+    public static class InterceptingTransportService implements TransportInterceptor {
 
-            public void onModule(NetworkModule module) {
-                module.registerTransportService("intercepting", InterceptingTransportService.class);
+        public static class TestPlugin extends Plugin implements NetworkPlugin {
+            public final InterceptingTransportService instance = new InterceptingTransportService();
+            @Override
+            public List<TransportInterceptor> getTransportInterceptors(ThreadContext threadContext) {
+                return Collections.singletonList(instance);
             }
         }
 
@@ -746,9 +713,11 @@ public void onModule(NetworkModule module) {
 
         private final Map<String, List<TransportRequest>> requests = new HashMap<>();
 
-        @Inject
-        public InterceptingTransportService(Settings settings, Transport transport, ThreadPool threadPool) {
-            super(settings, transport, threadPool);
+        @Override
+        public <T extends TransportRequest> TransportRequestHandler<T> interceptHandler(String action, String executor,
+                                                                                        boolean forceExecution,
+                                                                                        TransportRequestHandler<T> actualHandler) {
+            return new InterceptingRequestHandler<>(action, actualHandler);
         }
 
         synchronized List<TransportRequest> consumeRequests(String action) {
@@ -763,19 +732,6 @@ synchronized void clearInterceptedActions() {
             actions.clear();
         }
 
-        @Override
-        public <Request extends TransportRequest> void registerRequestHandler(String action, Supplier<Request> request, String executor,
-                                                                              boolean forceExecution, boolean canTripCircuitBreaker,
-                                                                              TransportRequestHandler<Request> handler) {
-            super.registerRequestHandler(action, request, executor, forceExecution, canTripCircuitBreaker, new
-                    InterceptingRequestHandler<>(action, handler));
-        }
-
-        @Override
-        public <Request extends TransportRequest> void registerRequestHandler(String action, Supplier<Request> requestFactory, String
-                executor, TransportRequestHandler<Request> handler) {
-            super.registerRequestHandler(action, requestFactory, executor, new InterceptingRequestHandler<>(action, handler));
-        }
 
         private class InterceptingRequestHandler<T extends TransportRequest> implements TransportRequestHandler<T> {
 
diff --git a/core/src/test/java/org/elasticsearch/action/ListenerActionIT.java b/core/src/test/java/org/elasticsearch/action/ListenerActionIT.java
index 93389f898e907..ad4f4547c5686 100644
--- a/core/src/test/java/org/elasticsearch/action/ListenerActionIT.java
+++ b/core/src/test/java/org/elasticsearch/action/ListenerActionIT.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.client.Requests;
 import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.test.ESIntegTestCase;
 
@@ -40,7 +41,7 @@ public void testThreadedListeners() throws Throwable {
         IndexRequest request = new IndexRequest("test", "type", "1");
         if (randomBoolean()) {
             // set the source, without it, we will have a verification failure
-            request.source("field1", "value1");
+            request.source(Requests.INDEX_CONTENT_TYPE, "field1", "value1");
         }
 
         client.index(request, new ActionListener<IndexResponse>() {
diff --git a/core/src/test/java/org/elasticsearch/action/NotifyOnceListenerTests.java b/core/src/test/java/org/elasticsearch/action/NotifyOnceListenerTests.java
new file mode 100644
index 0000000000000..501a40686ed96
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/NotifyOnceListenerTests.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action;
+
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.concurrent.atomic.AtomicReference;
+
+public class NotifyOnceListenerTests extends ESTestCase {
+
+    public void testWhenSuccessCannotNotifyMultipleTimes() {
+        AtomicReference<String> response = new AtomicReference<>();
+        AtomicReference<Exception> exception = new AtomicReference<>();
+
+        NotifyOnceListener<String> listener = new NotifyOnceListener<String>() {
+            @Override
+            public void innerOnResponse(String s) {
+                response.set(s);
+            }
+
+            @Override
+            public void innerOnFailure(Exception e) {
+                exception.set(e);
+            }
+        };
+
+        listener.onResponse("response");
+        listener.onResponse("wrong-response");
+        listener.onFailure(new RuntimeException());
+
+        assertNull(exception.get());
+        assertEquals("response", response.get());
+    }
+
+    public void testWhenErrorCannotNotifyMultipleTimes() {
+        AtomicReference<String> response = new AtomicReference<>();
+        AtomicReference<Exception> exception = new AtomicReference<>();
+
+        NotifyOnceListener<String> listener = new NotifyOnceListener<String>() {
+            @Override
+            public void innerOnResponse(String s) {
+                response.set(s);
+            }
+
+            @Override
+            public void innerOnFailure(Exception e) {
+                exception.set(e);
+            }
+        };
+
+        RuntimeException expected = new RuntimeException();
+        listener.onFailure(expected);
+        listener.onFailure(new IllegalArgumentException());
+        listener.onResponse("response");
+
+        assertNull(response.get());
+        assertSame(expected, exception.get());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/OriginalIndicesTests.java b/core/src/test/java/org/elasticsearch/action/OriginalIndicesTests.java
index ceba9278a1627..c82f184eff418 100644
--- a/core/src/test/java/org/elasticsearch/action/OriginalIndicesTests.java
+++ b/core/src/test/java/org/elasticsearch/action/OriginalIndicesTests.java
@@ -57,7 +57,7 @@ private static OriginalIndices randomOriginalIndices() {
         int numIndices = randomInt(10);
         String[] indices = new String[numIndices];
         for (int j = 0; j < indices.length; j++) {
-            indices[j] = randomAsciiOfLength(randomIntBetween(1, 10));
+            indices[j] = randomAlphaOfLength(randomIntBetween(1, 10));
         }
         IndicesOptions indicesOptions = randomFrom(indicesOptionsValues);
         return new OriginalIndices(indices, indicesOptions);
diff --git a/core/src/test/java/org/elasticsearch/action/RejectionActionIT.java b/core/src/test/java/org/elasticsearch/action/RejectionActionIT.java
index 6745906488ac8..7bb4eb2b1c308 100644
--- a/core/src/test/java/org/elasticsearch/action/RejectionActionIT.java
+++ b/core/src/test/java/org/elasticsearch/action/RejectionActionIT.java
@@ -82,7 +82,7 @@ public void onFailure(Exception e) {
                     });
         }
         latch.await();
-        assertThat(responses.size(), equalTo(numberOfAsyncOps));
+
 
         // validate all responses
         for (Object response : responses) {
@@ -104,5 +104,6 @@ public void onFailure(Exception e) {
                 }
             }
         }
+        assertThat(responses.size(), equalTo(numberOfAsyncOps));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/ShardValidateQueryRequestTests.java b/core/src/test/java/org/elasticsearch/action/ShardValidateQueryRequestTests.java
new file mode 100644
index 0000000000000..55bc37df18aed
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/ShardValidateQueryRequestTests.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.admin.indices.validate.query.ShardValidateQueryRequest;
+import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.indices.IndicesModule;
+import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.search.internal.AliasFilter;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Base64;
+import java.util.Collections;
+import java.util.List;
+
+public class ShardValidateQueryRequestTests extends ESTestCase {
+    protected NamedWriteableRegistry namedWriteableRegistry;
+
+    public void setUp() throws Exception {
+        super.setUp();
+        IndicesModule indicesModule = new IndicesModule(Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
+        entries.addAll(indicesModule.getNamedWriteables());
+        entries.addAll(searchModule.getNamedWriteables());
+        namedWriteableRegistry = new NamedWriteableRegistry(entries);
+    }
+
+    public void testSerialize() throws IOException {
+        try (BytesStreamOutput output = new BytesStreamOutput()) {
+            ValidateQueryRequest validateQueryRequest = new ValidateQueryRequest("indices");
+            validateQueryRequest.query(QueryBuilders.termQuery("field", "value"));
+            validateQueryRequest.rewrite(true);
+            validateQueryRequest.explain(false);
+            validateQueryRequest.types("type1", "type2");
+            ShardValidateQueryRequest request = new ShardValidateQueryRequest(new ShardId("index", "foobar", 1),
+                new AliasFilter(QueryBuilders.termQuery("filter_field", "value"), new String[] {"alias0", "alias1"}), validateQueryRequest);
+            request.writeTo(output);
+            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
+                ShardValidateQueryRequest readRequest = new ShardValidateQueryRequest();
+                readRequest.readFrom(in);
+                assertEquals(request.filteringAliases(), readRequest.filteringAliases());
+                assertArrayEquals(request.types(), readRequest.types());
+                assertEquals(request.explain(), readRequest.explain());
+                assertEquals(request.query(), readRequest.query());
+                assertEquals(request.rewrite(), readRequest.rewrite());
+                assertEquals(request.shardId(), readRequest.shardId());
+            }
+        }
+    }
+
+    // BWC test for changes from #20916
+    public void testSerialize50Request() throws IOException {
+        ValidateQueryRequest validateQueryRequest = new ValidateQueryRequest("indices");
+        validateQueryRequest.query(QueryBuilders.termQuery("field", "value"));
+        validateQueryRequest.rewrite(true);
+        validateQueryRequest.explain(false);
+        validateQueryRequest.types("type1", "type2");
+        ShardValidateQueryRequest request = new ShardValidateQueryRequest(new ShardId("index", "foobar", 1),
+            new AliasFilter(QueryBuilders.termQuery("filter_field", "value"), new String[] {"alias0", "alias1"}), validateQueryRequest);
+        BytesArray requestBytes = new BytesArray(Base64.getDecoder()
+            // this is a base64 encoded request generated with the same input
+            .decode("AAVpbmRleAZmb29iYXIBAQdpbmRpY2VzBAR0ZXJtP4AAAAAFZmllbGQVBXZhbHVlAgV0eXBlMQV0eXBlMgIGYWxpYXMwBmFsaWFzMQABAA"));
+        try (StreamInput in = new NamedWriteableAwareStreamInput(requestBytes.streamInput(), namedWriteableRegistry)) {
+            in.setVersion(Version.V_5_0_0);
+            ShardValidateQueryRequest readRequest = new ShardValidateQueryRequest();
+            readRequest.readFrom(in);
+            assertEquals(0, in.available());
+            assertArrayEquals(request.filteringAliases().getAliases(), readRequest.filteringAliases().getAliases());
+            expectThrows(IllegalStateException.class, () -> readRequest.filteringAliases().getQueryBuilder());
+            assertArrayEquals(request.types(), readRequest.types());
+            assertEquals(request.explain(), readRequest.explain());
+            assertEquals(request.query(), readRequest.query());
+            assertEquals(request.rewrite(), readRequest.rewrite());
+            assertEquals(request.shardId(), readRequest.shardId());
+            BytesStreamOutput output = new BytesStreamOutput();
+            output.setVersion(Version.V_5_0_0);
+            readRequest.writeTo(output);
+            assertEquals(output.bytes().toBytesRef(), requestBytes.toBytesRef());
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainActionTests.java
new file mode 100644
index 0000000000000..f9785df6495b2
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainActionTests.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.cluster.allocation;
+
+import org.elasticsearch.action.support.replication.ClusterStateCreationUtils;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.routing.RoutingNode;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
+import org.elasticsearch.cluster.routing.allocation.ShardAllocationDecision;
+import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocator;
+import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.gateway.TestGatewayAllocator;
+
+import java.util.Collections;
+import java.util.Locale;
+
+import static org.elasticsearch.action.admin.cluster.allocation.TransportClusterAllocationExplainAction.findShardToExplain;
+
+/**
+ * Tests for the {@link TransportClusterAllocationExplainAction} class.
+ */
+public class ClusterAllocationExplainActionTests extends ESTestCase {
+
+    private static final AllocationDeciders NOOP_DECIDERS = new AllocationDeciders(Settings.EMPTY, Collections.emptyList());
+
+    public void testInitializingOrRelocatingShardExplanation() throws Exception {
+        ShardRoutingState shardRoutingState = randomFrom(ShardRoutingState.INITIALIZING, ShardRoutingState.RELOCATING);
+        ClusterState clusterState = ClusterStateCreationUtils.state("idx", randomBoolean(), shardRoutingState);
+        ShardRouting shard = clusterState.getRoutingTable().index("idx").shard(0).primaryShard();
+        RoutingAllocation allocation = new RoutingAllocation(new AllocationDeciders(Settings.EMPTY, Collections.emptyList()),
+            clusterState.getRoutingNodes(), clusterState, null, System.nanoTime(), randomBoolean());
+        ClusterAllocationExplanation cae = TransportClusterAllocationExplainAction.explainShard(shard, allocation, null, randomBoolean(),
+            new TestGatewayAllocator(), new ShardsAllocator() {
+                @Override
+                public void allocate(RoutingAllocation allocation) {
+                    // no-op
+                }
+
+                @Override
+                public ShardAllocationDecision decideShardAllocation(ShardRouting shard, RoutingAllocation allocation) {
+                    if (shard.initializing() || shard.relocating()) {
+                        return ShardAllocationDecision.NOT_TAKEN;
+                    } else {
+                        throw new UnsupportedOperationException("cannot explain");
+                    }
+                }
+            });
+
+        assertEquals(shard.currentNodeId(), cae.getCurrentNode().getId());
+        assertFalse(cae.getShardAllocationDecision().isDecisionTaken());
+        assertFalse(cae.getShardAllocationDecision().getAllocateDecision().isDecisionTaken());
+        assertFalse(cae.getShardAllocationDecision().getMoveDecision().isDecisionTaken());
+        XContentBuilder builder = XContentFactory.jsonBuilder();
+        cae.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        String explanation;
+        if (shardRoutingState == ShardRoutingState.RELOCATING) {
+            explanation = "the shard is in the process of relocating from node [] to node [], wait until " +
+                              "relocation has completed";
+        } else {
+            explanation = "the shard is in the process of initializing on node [], " +
+                              "wait until initialization has completed";
+        }
+        assertEquals("{\"index\":\"idx\",\"shard\":0,\"primary\":true,\"current_state\":\"" +
+                         shardRoutingState.toString().toLowerCase(Locale.ROOT) + "\",\"current_node\":" +
+                         "{\"id\":\"" + cae.getCurrentNode().getId() + "\",\"name\":\"" + cae.getCurrentNode().getName() +
+                         "\",\"transport_address\":\"" + cae.getCurrentNode().getAddress() +
+                         "\"},\"explanation\":\"" + explanation + "\"}", builder.string());
+    }
+
+    public void testFindAnyUnassignedShardToExplain() {
+        // find unassigned primary
+        ClusterState clusterState = ClusterStateCreationUtils.state("idx", randomBoolean(), ShardRoutingState.UNASSIGNED);
+        ClusterAllocationExplainRequest request = new ClusterAllocationExplainRequest();
+        ShardRouting shard = findShardToExplain(request, routingAllocation(clusterState));
+        assertEquals(clusterState.getRoutingTable().index("idx").shard(0).primaryShard(), shard);
+
+        // find unassigned replica
+        clusterState = ClusterStateCreationUtils.state("idx", randomBoolean(), ShardRoutingState.STARTED, ShardRoutingState.UNASSIGNED);
+        request = new ClusterAllocationExplainRequest();
+        shard = findShardToExplain(request, routingAllocation(clusterState));
+        assertEquals(clusterState.getRoutingTable().index("idx").shard(0).replicaShards().get(0), shard);
+
+        // no unassigned shard to explain
+        final ClusterState allStartedClusterState = ClusterStateCreationUtils.state("idx", randomBoolean(),
+            ShardRoutingState.STARTED, ShardRoutingState.STARTED);
+        final ClusterAllocationExplainRequest anyUnassignedShardsRequest = new ClusterAllocationExplainRequest();
+        expectThrows(IllegalArgumentException.class, () ->
+            findShardToExplain(anyUnassignedShardsRequest, routingAllocation(allStartedClusterState)));
+    }
+
+    public void testFindPrimaryShardToExplain() {
+        ClusterState clusterState = ClusterStateCreationUtils.state("idx", randomBoolean(), randomFrom(ShardRoutingState.values()));
+        ClusterAllocationExplainRequest request = new ClusterAllocationExplainRequest("idx", 0, true, null);
+        ShardRouting shard = findShardToExplain(request, routingAllocation(clusterState));
+        assertEquals(clusterState.getRoutingTable().index("idx").shard(0).primaryShard(), shard);
+    }
+
+    public void testFindAnyReplicaToExplain() {
+        // prefer unassigned replicas to started replicas
+        ClusterState clusterState = ClusterStateCreationUtils.state("idx", randomBoolean(), ShardRoutingState.STARTED,
+            ShardRoutingState.STARTED, ShardRoutingState.UNASSIGNED);
+        ClusterAllocationExplainRequest request = new ClusterAllocationExplainRequest("idx", 0, false, null);
+        ShardRouting shard = findShardToExplain(request, routingAllocation(clusterState));
+        assertEquals(clusterState.getRoutingTable().index("idx").shard(0).replicaShards()
+                         .stream().filter(ShardRouting::unassigned).findFirst().get(), shard);
+
+        // prefer started replicas to initializing/relocating replicas
+        clusterState = ClusterStateCreationUtils.state("idx", randomBoolean(), ShardRoutingState.STARTED,
+            randomFrom(ShardRoutingState.RELOCATING, ShardRoutingState.INITIALIZING), ShardRoutingState.STARTED);
+        request = new ClusterAllocationExplainRequest("idx", 0, false, null);
+        shard = findShardToExplain(request, routingAllocation(clusterState));
+        assertEquals(clusterState.getRoutingTable().index("idx").shard(0).replicaShards()
+                         .stream().filter(ShardRouting::started).findFirst().get(), shard);
+    }
+
+    public void testFindShardAssignedToNode() {
+        // find shard with given node
+        final boolean primary = randomBoolean();
+        ShardRoutingState[] replicaStates = new ShardRoutingState[0];
+        if (primary == false) {
+            replicaStates = new ShardRoutingState[] { ShardRoutingState.STARTED };
+        }
+        ClusterState clusterState = ClusterStateCreationUtils.state("idx", randomBoolean(), ShardRoutingState.STARTED, replicaStates);
+        ShardRouting shardToExplain = primary ? clusterState.getRoutingTable().index("idx").shard(0).primaryShard() :
+                                          clusterState.getRoutingTable().index("idx").shard(0).replicaShards().get(0);
+        ClusterAllocationExplainRequest request = new ClusterAllocationExplainRequest("idx", 0, primary, shardToExplain.currentNodeId());
+        RoutingAllocation allocation = routingAllocation(clusterState);
+        ShardRouting foundShard = findShardToExplain(request, allocation);
+        assertEquals(shardToExplain, foundShard);
+
+        // shard is not assigned to given node
+        String explainNode = null;
+        for (RoutingNode routingNode : clusterState.getRoutingNodes()) {
+            if (routingNode.nodeId().equals(shardToExplain.currentNodeId()) == false) {
+                explainNode = routingNode.nodeId();
+                break;
+            }
+        }
+        final ClusterAllocationExplainRequest failingRequest = new ClusterAllocationExplainRequest("idx", 0, primary, explainNode);
+        expectThrows(IllegalArgumentException.class, () -> findShardToExplain(failingRequest, allocation));
+    }
+
+    private static RoutingAllocation routingAllocation(ClusterState clusterState) {
+        return new RoutingAllocation(NOOP_DECIDERS, clusterState.getRoutingNodes(), clusterState, null, System.nanoTime(), randomBoolean());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java
index 3b26345148775..ebbf4f7446850 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java
@@ -19,150 +19,1383 @@
 
 package org.elasticsearch.action.admin.cluster.allocation;
 
-import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
-import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresResponse;
 import org.elasticsearch.action.support.ActiveShardCount;
-import org.elasticsearch.client.Requests;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.ClusterInfo;
+import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
+import org.elasticsearch.cluster.routing.UnassignedInfo.AllocationStatus;
+import org.elasticsearch.cluster.routing.UnassignedInfo.Reason;
+import org.elasticsearch.cluster.routing.allocation.AllocateUnassignedDecision;
+import org.elasticsearch.cluster.routing.allocation.AllocationDecision;
+import org.elasticsearch.cluster.routing.allocation.MoveDecision;
+import org.elasticsearch.cluster.routing.allocation.NodeAllocationResult;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParser.Token;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.InternalTestCluster;
 
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
+import java.util.Set;
 
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.isOneOf;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
+import static org.hamcrest.Matchers.startsWith;
 
 /**
  * Tests for the cluster allocation explanation
  */
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0)
 public final class ClusterAllocationExplainIT extends ESIntegTestCase {
-    public void testDelayShards() throws Exception {
+
+    public void testUnassignedPrimaryWithExistingIndex() throws Exception {
+        logger.info("--> starting 2 nodes");
+        internalCluster().startNodes(2);
+
+        logger.info("--> creating an index with 1 primary, 0 replicas");
+        createIndexAndIndexData(1, 0);
+
+        logger.info("--> stopping the node with the primary");
+        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName()));
+        ensureStableCluster(1);
+
+        boolean includeYesDecisions = randomBoolean();
+        boolean includeDiskInfo = randomBoolean();
+        ClusterAllocationExplanation explanation = runExplain(true, includeYesDecisions, includeDiskInfo);
+
+        ShardId shardId = explanation.getShard();
+        boolean isPrimary = explanation.isPrimary();
+        ShardRoutingState shardState = explanation.getShardState();
+        DiscoveryNode currentNode = explanation.getCurrentNode();
+        UnassignedInfo unassignedInfo = explanation.getUnassignedInfo();
+        ClusterInfo clusterInfo = explanation.getClusterInfo();
+        AllocateUnassignedDecision allocateDecision = explanation.getShardAllocationDecision().getAllocateDecision();
+        MoveDecision moveDecision = explanation.getShardAllocationDecision().getMoveDecision();
+
+        // verify shard info
+        assertEquals("idx", shardId.getIndexName());
+        assertEquals(0, shardId.getId());
+        assertTrue(isPrimary);
+
+        // verify current node info
+        assertNotEquals(ShardRoutingState.STARTED, shardState);
+        assertNull(currentNode);
+
+        // verify unassigned info
+        assertNotNull(unassignedInfo);
+        assertEquals(Reason.NODE_LEFT, unassignedInfo.getReason());
+        assertTrue(unassignedInfo.getLastAllocationStatus() == AllocationStatus.FETCHING_SHARD_DATA
+                       || unassignedInfo.getLastAllocationStatus() == AllocationStatus.NO_VALID_SHARD_COPY);
+
+        // verify cluster info
+        verifyClusterInfo(clusterInfo, includeDiskInfo, 1);
+
+        // verify decision objects
+        assertTrue(allocateDecision.isDecisionTaken());
+        assertFalse(moveDecision.isDecisionTaken());
+        assertTrue(allocateDecision.getAllocationDecision() == AllocationDecision.NO_VALID_SHARD_COPY
+                       || allocateDecision.getAllocationDecision() == AllocationDecision.AWAITING_INFO);
+        if (allocateDecision.getAllocationDecision() == AllocationDecision.NO_VALID_SHARD_COPY) {
+            assertEquals("cannot allocate because a previous copy of the primary shard existed but can no longer be " +
+                             "found on the nodes in the cluster", allocateDecision.getExplanation());
+        } else {
+            assertEquals("cannot allocate because information about existing shard data is still being retrieved from some of the nodes",
+                allocateDecision.getExplanation());
+        }
+        assertNull(allocateDecision.getAllocationId());
+        assertNull(allocateDecision.getTargetNode());
+        assertEquals(0L, allocateDecision.getConfiguredDelayInMillis());
+        assertEquals(0L, allocateDecision.getRemainingDelayInMillis());
+
+        if (allocateDecision.getAllocationDecision() == AllocationDecision.NO_VALID_SHARD_COPY) {
+            assertEquals(1, allocateDecision.getNodeDecisions().size());
+
+            // verify JSON output
+            try (XContentParser parser = getParser(explanation)) {
+                verifyShardInfo(parser, true, includeDiskInfo, ShardRoutingState.UNASSIGNED);
+                parser.nextToken();
+                assertEquals("can_allocate", parser.currentName());
+                parser.nextToken();
+                assertEquals(AllocationDecision.NO_VALID_SHARD_COPY.toString(), parser.text());
+                parser.nextToken();
+                assertEquals("allocate_explanation", parser.currentName());
+                parser.nextToken();
+                assertEquals("cannot allocate because a previous copy of the primary shard existed but can no longer be found " +
+                                 "on the nodes in the cluster", parser.text());
+                verifyStaleShardCopyNodeDecisions(parser, 1, Collections.emptySet());
+            }
+        }
+    }
+
+    public void testUnassignedReplicaDelayedAllocation() throws Exception {
         logger.info("--> starting 3 nodes");
-        List<String> nodes = internalCluster().startNodesAsync(3).get();
+        internalCluster().startNodes(3);
+
+        logger.info("--> creating an index with 1 primary, 1 replica");
+        createIndexAndIndexData(1, 1);
+        logger.info("--> stopping the node with the replica");
+        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(replicaNode().getName()));
+        ensureStableCluster(2);
+        assertBusy(() ->
+            // wait till we have passed any pending shard data fetching
+            assertEquals(AllocationDecision.ALLOCATION_DELAYED, client().admin().cluster().prepareAllocationExplain()
+                .setIndex("idx").setShard(0).setPrimary(false).get().getExplanation()
+                .getShardAllocationDecision().getAllocateDecision().getAllocationDecision())
+        );
+
+        logger.info("--> observing delayed allocation...");
+        boolean includeYesDecisions = randomBoolean();
+        boolean includeDiskInfo = randomBoolean();
+
+        ClusterAllocationExplanation explanation = runExplain(false, includeYesDecisions, includeDiskInfo);
+
+        ShardId shardId = explanation.getShard();
+        boolean isPrimary = explanation.isPrimary();
+        ShardRoutingState shardRoutingState = explanation.getShardState();
+        DiscoveryNode currentNode = explanation.getCurrentNode();
+        UnassignedInfo unassignedInfo = explanation.getUnassignedInfo();
+        ClusterInfo clusterInfo = explanation.getClusterInfo();
+        AllocateUnassignedDecision allocateDecision = explanation.getShardAllocationDecision().getAllocateDecision();
+        MoveDecision moveDecision = explanation.getShardAllocationDecision().getMoveDecision();
+
+        // verify shard info
+        assertEquals("idx", shardId.getIndexName());
+        assertEquals(0, shardId.getId());
+        assertFalse(isPrimary);
+
+        // verify current node info
+        assertNotEquals(ShardRoutingState.STARTED, shardRoutingState);
+        assertNull(currentNode);
+
+        // verify unassigned info
+        assertNotNull(unassignedInfo);
+        assertEquals(Reason.NODE_LEFT, unassignedInfo.getReason());
+        assertEquals(AllocationStatus.NO_ATTEMPT, unassignedInfo.getLastAllocationStatus());
+
+        // verify cluster info
+        verifyClusterInfo(clusterInfo, includeDiskInfo, 2);
+
+        // verify decision objects
+        assertTrue(allocateDecision.isDecisionTaken());
+        assertFalse(moveDecision.isDecisionTaken());
+        assertEquals(AllocationDecision.ALLOCATION_DELAYED, allocateDecision.getAllocationDecision());
+        assertThat(allocateDecision.getExplanation(), startsWith("cannot allocate because the cluster is still waiting"));
+        assertThat(allocateDecision.getExplanation(), containsString(
+            "despite being allowed to allocate the shard to at least one other node"));
+        assertNull(allocateDecision.getAllocationId());
+        assertNull(allocateDecision.getTargetNode());
+        assertEquals(60000L, allocateDecision.getConfiguredDelayInMillis());
+        assertThat(allocateDecision.getRemainingDelayInMillis(), greaterThan(0L));
+        assertEquals(2, allocateDecision.getNodeDecisions().size());
+        String primaryNodeName = primaryNodeName();
+        for (NodeAllocationResult result : allocateDecision.getNodeDecisions()) {
+            assertNotNull(result.getNode());
+            boolean nodeHoldingPrimary = result.getNode().getName().equals(primaryNodeName);
+            if (nodeHoldingPrimary) {
+                // shouldn't be able to allocate to the same node as the primary, the same shard decider should say no
+                assertEquals(AllocationDecision.NO, result.getNodeDecision());
+                assertThat(result.getShardStoreInfo().getMatchingBytes(), greaterThan(0L));
+            } else {
+                assertEquals(AllocationDecision.YES, result.getNodeDecision());
+                assertNull(result.getShardStoreInfo());
+            }
+            if (includeYesDecisions) {
+                assertThat(result.getCanAllocateDecision().getDecisions().size(), greaterThan(1));
+            } else {
+                // if we are not including YES decisions, then the node holding the primary should have 1 NO decision,
+                // the other node should have zero NO decisions
+                assertEquals(nodeHoldingPrimary ? 1 : 0, result.getCanAllocateDecision().getDecisions().size());
+            }
+            for (Decision d : result.getCanAllocateDecision().getDecisions()) {
+                if (d.label().equals("same_shard") && nodeHoldingPrimary) {
+                    assertEquals(Decision.Type.NO, d.type());
+                    assertThat(d.getExplanation(), startsWith(
+                        "the shard cannot be allocated to the same node on which a copy of the shard already exists"));
+                } else {
+                    assertEquals(Decision.Type.YES, d.type());
+                    assertNotNull(d.getExplanation());
+                }
+            }
+        }
+
+        // verify JSON output
+        try (XContentParser parser = getParser(explanation)) {
+            verifyShardInfo(parser, false, includeDiskInfo, ShardRoutingState.UNASSIGNED);
+            parser.nextToken();
+            assertEquals("can_allocate", parser.currentName());
+            parser.nextToken();
+            assertEquals(AllocationDecision.ALLOCATION_DELAYED.toString(), parser.text());
+            parser.nextToken();
+            assertEquals("allocate_explanation", parser.currentName());
+            parser.nextToken();
+            assertThat(parser.text(), startsWith("cannot allocate because the cluster is still waiting"));
+            parser.nextToken();
+            assertEquals("configured_delay_in_millis", parser.currentName());
+            parser.nextToken();
+            assertEquals(60000L, parser.longValue());
+            parser.nextToken();
+            assertEquals("remaining_delay_in_millis", parser.currentName());
+            parser.nextToken();
+            assertThat(parser.longValue(), greaterThan(0L));
+            Map<String, AllocationDecision> nodes = new HashMap<>();
+            nodes.put(primaryNodeName, AllocationDecision.NO);
+            String[] currentNodes = internalCluster().getNodeNames();
+            nodes.put(currentNodes[0].equals(primaryNodeName) ? currentNodes[1] : currentNodes[0], AllocationDecision.YES);
+            verifyNodeDecisions(parser, nodes, includeYesDecisions, true);
+            assertEquals(Token.END_OBJECT, parser.nextToken());
+        }
+    }
+
+    public void testUnassignedReplicaWithPriorCopy() throws Exception {
+        logger.info("--> starting 3 nodes");
+        List<String> nodes = internalCluster().startNodes(3);
+
+        logger.info("--> creating an index with 1 primary and 1 replica");
+        createIndexAndIndexData(1, 1);
+        String primaryNodeName = primaryNodeName();
+        nodes.remove(primaryNodeName);
+
+        logger.info("--> shutting down all nodes except the one that holds the primary");
+        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodes.get(0)));
+        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodes.get(1)));
+        ensureStableCluster(1);
+
+        logger.info("--> setting allocation filtering to only allow allocation on the currently running node");
+        client().admin().indices().prepareUpdateSettings("idx").setSettings(
+            Settings.builder().put("index.routing.allocation.include._name", primaryNodeName)).get();
+
+        logger.info("--> restarting the stopped nodes");
+        internalCluster().startNode(Settings.builder().put("node.name", nodes.get(0)).build());
+        internalCluster().startNode(Settings.builder().put("node.name", nodes.get(1)).build());
+        ensureStableCluster(3);
+
+        boolean includeYesDecisions = randomBoolean();
+        boolean includeDiskInfo = randomBoolean();
+        ClusterAllocationExplanation explanation = runExplain(false, includeYesDecisions, includeDiskInfo);
+
+        ShardId shardId = explanation.getShard();
+        boolean isPrimary = explanation.isPrimary();
+        ShardRoutingState shardRoutingState = explanation.getShardState();
+        DiscoveryNode currentNode = explanation.getCurrentNode();
+        UnassignedInfo unassignedInfo = explanation.getUnassignedInfo();
+        ClusterInfo clusterInfo = explanation.getClusterInfo();
+        AllocateUnassignedDecision allocateDecision = explanation.getShardAllocationDecision().getAllocateDecision();
+        MoveDecision moveDecision = explanation.getShardAllocationDecision().getMoveDecision();
+
+        // verify shard info
+        assertEquals("idx", shardId.getIndexName());
+        assertEquals(0, shardId.getId());
+        assertFalse(isPrimary);
+
+        // verify current node info
+        assertNotEquals(ShardRoutingState.STARTED, shardRoutingState);
+        assertNull(currentNode);
+
+        // verify unassigned info
+        assertNotNull(unassignedInfo);
+        assertEquals(Reason.NODE_LEFT, unassignedInfo.getReason());
+        assertEquals(AllocationStatus.NO_ATTEMPT, unassignedInfo.getLastAllocationStatus());
+
+        // verify cluster info
+        verifyClusterInfo(clusterInfo, includeDiskInfo, 3);
+
+        // verify decision objects
+        assertTrue(allocateDecision.isDecisionTaken());
+        assertFalse(moveDecision.isDecisionTaken());
+        AllocationDecision decisionToAllocate = allocateDecision.getAllocationDecision();
+        assertTrue(decisionToAllocate == AllocationDecision.AWAITING_INFO || decisionToAllocate == AllocationDecision.NO);
+        if (decisionToAllocate == AllocationDecision.AWAITING_INFO) {
+            assertEquals("cannot allocate because information about existing shard data is still being retrieved from some of the nodes",
+                allocateDecision.getExplanation());
+        } else {
+            assertEquals("cannot allocate because allocation is not permitted to any of the nodes", allocateDecision.getExplanation());
+        }
+        assertNull(allocateDecision.getAllocationId());
+        assertNull(allocateDecision.getTargetNode());
+        assertEquals(0L, allocateDecision.getConfiguredDelayInMillis());
+        assertEquals(0L, allocateDecision.getRemainingDelayInMillis());
+        assertEquals(3, allocateDecision.getNodeDecisions().size());
+        for (NodeAllocationResult result : allocateDecision.getNodeDecisions()) {
+            assertNotNull(result.getNode());
+            boolean nodeHoldingPrimary = result.getNode().getName().equals(primaryNodeName);
+            assertEquals(AllocationDecision.NO, result.getNodeDecision());
+            if (includeYesDecisions) {
+                assertThat(result.getCanAllocateDecision().getDecisions().size(), greaterThan(1));
+            } else {
+                assertEquals(1, result.getCanAllocateDecision().getDecisions().size());
+            }
+            for (Decision d : result.getCanAllocateDecision().getDecisions()) {
+                if (d.label().equals("same_shard") && nodeHoldingPrimary) {
+                    assertEquals(Decision.Type.NO, d.type());
+                    assertThat(d.getExplanation(), startsWith(
+                        "the shard cannot be allocated to the same node on which a copy of the shard already exists"));
+                } else if (d.label().equals("filter") && nodeHoldingPrimary == false) {
+                    assertEquals(Decision.Type.NO, d.type());
+                    assertEquals("node does not match index setting [index.routing.allocation.include] " +
+                                     "filters [_name:\"" + primaryNodeName + "\"]", d.getExplanation());
+                } else {
+                    assertEquals(Decision.Type.YES, d.type());
+                    assertNotNull(d.getExplanation());
+                }
+            }
+        }
+
+        // verify JSON output
+        try (XContentParser parser = getParser(explanation)) {
+            verifyShardInfo(parser, false, includeDiskInfo, ShardRoutingState.UNASSIGNED);
+            parser.nextToken();
+            assertEquals("can_allocate", parser.currentName());
+            parser.nextToken();
+            String allocationDecision = parser.text();
+            assertTrue(allocationDecision.equals(AllocationDecision.NO.toString())
+                           || allocationDecision.equals(AllocationDecision.AWAITING_INFO.toString()));
+            parser.nextToken();
+            assertEquals("allocate_explanation", parser.currentName());
+            parser.nextToken();
+            if (allocationDecision.equals("awaiting_info")) {
+                assertEquals("cannot allocate because information about existing shard data is still being retrieved " +
+                                 "from some of the nodes", parser.text());
+            } else {
+                assertEquals("cannot allocate because allocation is not permitted to any of the nodes", parser.text());
+            }
+            Map<String, AllocationDecision> nodeDecisions = new HashMap<>();
+            for (String nodeName : internalCluster().getNodeNames()) {
+                nodeDecisions.put(nodeName, AllocationDecision.NO);
+            }
+            verifyNodeDecisions(parser, nodeDecisions, includeYesDecisions, true);
+            assertEquals(Token.END_OBJECT, parser.nextToken());
+        }
+    }
+
+    public void testAllocationFilteringOnIndexCreation() throws Exception {
+        logger.info("--> starting 2 nodes");
+        internalCluster().startNodes(2);
+
+        logger.info("--> creating an index with 1 primary, 0 replicas, with allocation filtering so the primary can't be assigned");
+        createIndexAndIndexData(1, 0, Settings.builder().put("index.routing.allocation.include._name", "non_existent_node").build(),
+            ActiveShardCount.NONE);
+
+        boolean includeYesDecisions = randomBoolean();
+        boolean includeDiskInfo = randomBoolean();
+        ClusterAllocationExplanation explanation = runExplain(true, includeYesDecisions, includeDiskInfo);
+
+        ShardId shardId = explanation.getShard();
+        boolean isPrimary = explanation.isPrimary();
+        ShardRoutingState shardRoutingState = explanation.getShardState();
+        DiscoveryNode currentNode = explanation.getCurrentNode();
+        UnassignedInfo unassignedInfo = explanation.getUnassignedInfo();
+        ClusterInfo clusterInfo = explanation.getClusterInfo();
+        AllocateUnassignedDecision allocateDecision = explanation.getShardAllocationDecision().getAllocateDecision();
+        MoveDecision moveDecision = explanation.getShardAllocationDecision().getMoveDecision();
+
+        // verify shard info
+        assertEquals("idx", shardId.getIndexName());
+        assertEquals(0, shardId.getId());
+        assertTrue(isPrimary);
+
+        // verify current node info
+        assertNotEquals(ShardRoutingState.STARTED, shardRoutingState);
+        assertNull(currentNode);
+
+        // verify unassigned info
+        assertNotNull(unassignedInfo);
+        assertEquals(Reason.INDEX_CREATED, unassignedInfo.getReason());
+        assertEquals(AllocationStatus.DECIDERS_NO, unassignedInfo.getLastAllocationStatus());
+
+        // verify cluster info
+        verifyClusterInfo(clusterInfo, includeDiskInfo, 2);
+
+        // verify decision objects
+        assertTrue(allocateDecision.isDecisionTaken());
+        assertFalse(moveDecision.isDecisionTaken());
+        assertEquals(AllocationDecision.NO, allocateDecision.getAllocationDecision());
+        assertEquals("cannot allocate because allocation is not permitted to any of the nodes", allocateDecision.getExplanation());
+        assertNull(allocateDecision.getAllocationId());
+        assertNull(allocateDecision.getTargetNode());
+        assertEquals(0L, allocateDecision.getConfiguredDelayInMillis());
+        assertEquals(0L, allocateDecision.getRemainingDelayInMillis());
+        assertEquals(2, allocateDecision.getNodeDecisions().size());
+        for (NodeAllocationResult result : allocateDecision.getNodeDecisions()) {
+            assertNotNull(result.getNode());
+            assertEquals(AllocationDecision.NO, result.getNodeDecision());
+            if (includeYesDecisions) {
+                assertThat(result.getCanAllocateDecision().getDecisions().size(), greaterThan(1));
+            } else {
+                assertEquals(1, result.getCanAllocateDecision().getDecisions().size());
+            }
+            for (Decision d : result.getCanAllocateDecision().getDecisions()) {
+                if (d.label().equals("filter")) {
+                    assertEquals(Decision.Type.NO, d.type());
+                    assertEquals("node does not match index setting [index.routing.allocation.include] filters " +
+                                     "[_name:\"non_existent_node\"]", d.getExplanation());
+                }
+            }
+        }
+
+        // verify JSON output
+        try (XContentParser parser = getParser(explanation)) {
+            verifyShardInfo(parser, true, includeDiskInfo, ShardRoutingState.UNASSIGNED);
+            parser.nextToken();
+            assertEquals("can_allocate", parser.currentName());
+            parser.nextToken();
+            String allocationDecision = parser.text();
+            assertTrue(allocationDecision.equals(AllocationDecision.NO.toString())
+                           || allocationDecision.equals(AllocationDecision.AWAITING_INFO.toString()));
+            parser.nextToken();
+            assertEquals("allocate_explanation", parser.currentName());
+            parser.nextToken();
+            if (allocationDecision.equals("awaiting_info")) {
+                assertEquals("cannot allocate because information about existing shard data is still being retrieved " +
+                                 "from some of the nodes", parser.text());
+            } else {
+                assertEquals("cannot allocate because allocation is not permitted to any of the nodes", parser.text());
+            }
+            Map<String, AllocationDecision> nodeDecisions = new HashMap<>();
+            for (String nodeName : internalCluster().getNodeNames()) {
+                nodeDecisions.put(nodeName, AllocationDecision.NO);
+            }
+            verifyNodeDecisions(parser, nodeDecisions, includeYesDecisions, false);
+            assertEquals(Token.END_OBJECT, parser.nextToken());
+        }
+    }
+
+    public void testAllocationFilteringPreventsShardMove() throws Exception {
+        logger.info("--> starting 2 nodes");
+        internalCluster().startNodes(2);
+
+        logger.info("--> creating an index with 1 primary and 0 replicas");
+        createIndexAndIndexData(1, 0);
+
+        logger.info("--> setting up allocation filtering to prevent allocation to both nodes");
+        client().admin().indices().prepareUpdateSettings("idx").setSettings(
+            Settings.builder().put("index.routing.allocation.include._name", "non_existent_node")).get();
+
+        boolean includeYesDecisions = randomBoolean();
+        boolean includeDiskInfo = randomBoolean();
+        ClusterAllocationExplanation explanation = runExplain(true, includeYesDecisions, includeDiskInfo);
+
+        ShardId shardId = explanation.getShard();
+        boolean isPrimary = explanation.isPrimary();
+        ShardRoutingState shardRoutingState = explanation.getShardState();
+        DiscoveryNode currentNode = explanation.getCurrentNode();
+        UnassignedInfo unassignedInfo = explanation.getUnassignedInfo();
+        ClusterInfo clusterInfo = explanation.getClusterInfo();
+        AllocateUnassignedDecision allocateDecision = explanation.getShardAllocationDecision().getAllocateDecision();
+        MoveDecision moveDecision = explanation.getShardAllocationDecision().getMoveDecision();
+
+        // verify shard info
+        assertEquals("idx", shardId.getIndexName());
+        assertEquals(0, shardId.getId());
+        assertTrue(isPrimary);
+
+        // verify current node info
+        assertEquals(ShardRoutingState.STARTED, shardRoutingState);
+        assertNotNull(currentNode);
+
+        // verify unassigned info
+        assertNull(unassignedInfo);
+
+        // verify cluster info
+        verifyClusterInfo(clusterInfo, includeDiskInfo, 2);
+
+        // verify decision object
+        assertFalse(allocateDecision.isDecisionTaken());
+        assertTrue(moveDecision.isDecisionTaken());
+        assertEquals(AllocationDecision.NO, moveDecision.getAllocationDecision());
+        assertEquals("cannot move shard to another node, even though it is not allowed to remain on its current node",
+            moveDecision.getExplanation());
+        assertFalse(moveDecision.canRemain());
+        assertFalse(moveDecision.forceMove());
+        assertFalse(moveDecision.canRebalanceCluster());
+        assertNull(moveDecision.getClusterRebalanceDecision());
+        assertNull(moveDecision.getTargetNode());
+        assertEquals(0, moveDecision.getCurrentNodeRanking());
+        // verifying can remain decision object
+        assertNotNull(moveDecision.getCanRemainDecision());
+        assertEquals(Decision.Type.NO, moveDecision.getCanRemainDecision().type());
+        for (Decision d : moveDecision.getCanRemainDecision().getDecisions()) {
+            if (d.label().equals("filter")) {
+                assertEquals(Decision.Type.NO, d.type());
+                assertEquals("node does not match index setting [index.routing.allocation.include] filters [_name:\"non_existent_node\"]",
+                    d.getExplanation());
+            } else {
+                assertEquals(Decision.Type.YES, d.type());
+                assertNotNull(d.getExplanation());
+            }
+        }
+        // verify node decisions
+        assertEquals(1, moveDecision.getNodeDecisions().size());
+        NodeAllocationResult result = moveDecision.getNodeDecisions().get(0);
+        assertNotNull(result.getNode());
+        assertEquals(1, result.getWeightRanking());
+        assertEquals(AllocationDecision.NO, result.getNodeDecision());
+        if (includeYesDecisions) {
+            assertThat(result.getCanAllocateDecision().getDecisions().size(), greaterThan(1));
+        } else {
+            assertEquals(1, result.getCanAllocateDecision().getDecisions().size());
+        }
+        for (Decision d : result.getCanAllocateDecision().getDecisions()) {
+            if (d.label().equals("filter")) {
+                assertEquals(Decision.Type.NO, d.type());
+                assertEquals("node does not match index setting [index.routing.allocation.include] filters [_name:\"non_existent_node\"]",
+                    d.getExplanation());
+            } else {
+                assertEquals(Decision.Type.YES, d.type());
+                assertNotNull(d.getExplanation());
+            }
+        }
+
+        // verify JSON output
+        try (XContentParser parser = getParser(explanation)) {
+            verifyShardInfo(parser, true, includeDiskInfo, ShardRoutingState.STARTED);
+            parser.nextToken();
+            assertEquals("can_remain_on_current_node", parser.currentName());
+            parser.nextToken();
+            assertEquals(AllocationDecision.NO.toString(), parser.text());
+            parser.nextToken();
+            assertEquals("can_remain_decisions", parser.currentName());
+            verifyDeciders(parser, AllocationDecision.NO);
+            parser.nextToken();
+            assertEquals("can_move_to_other_node", parser.currentName());
+            parser.nextToken();
+            assertEquals(AllocationDecision.NO.toString(), parser.text());
+            parser.nextToken();
+            assertEquals("move_explanation", parser.currentName());
+            parser.nextToken();
+            assertEquals("cannot move shard to another node, even though it is not allowed to remain on its current node", parser.text());
+            verifyNodeDecisions(parser, allNodeDecisions(AllocationDecision.NO, true), includeYesDecisions, false);
+            assertEquals(Token.END_OBJECT, parser.nextToken());
+        }
+    }
+
+    public void testRebalancingNotAllowed() throws Exception {
+        logger.info("--> starting a single node");
+        internalCluster().startNode();
+        ensureStableCluster(1);
+
+        logger.info("--> creating an index with 5 shards, all allocated to the single node");
+        createIndexAndIndexData(5, 0);
+
+        logger.info("--> disabling rebalancing on the index");
+        client().admin().indices().prepareUpdateSettings("idx").setSettings(
+            Settings.builder().put("index.routing.rebalance.enable", "none")).get();
+
+        logger.info("--> starting another node, with rebalancing disabled, it should get no shards");
+        internalCluster().startNode();
+        ensureStableCluster(2);
+
+        boolean includeYesDecisions = randomBoolean();
+        boolean includeDiskInfo = randomBoolean();
+        ClusterAllocationExplanation explanation = runExplain(true, includeYesDecisions, includeDiskInfo);
+
+        ShardId shardId = explanation.getShard();
+        boolean isPrimary = explanation.isPrimary();
+        ShardRoutingState shardRoutingState = explanation.getShardState();
+        DiscoveryNode currentNode = explanation.getCurrentNode();
+        UnassignedInfo unassignedInfo = explanation.getUnassignedInfo();
+        ClusterInfo clusterInfo = explanation.getClusterInfo();
+        AllocateUnassignedDecision allocateDecision = explanation.getShardAllocationDecision().getAllocateDecision();
+        MoveDecision moveDecision = explanation.getShardAllocationDecision().getMoveDecision();
+
+        // verify shard info
+        assertEquals("idx", shardId.getIndexName());
+        assertEquals(0, shardId.getId());
+        assertTrue(isPrimary);
+
+        // verify current node info
+        assertEquals(ShardRoutingState.STARTED, shardRoutingState);
+        assertNotNull(currentNode);
+
+        // verify unassigned info
+        assertNull(unassignedInfo);
+
+        // verify cluster info
+        verifyClusterInfo(clusterInfo, includeDiskInfo, 2);
+
+        // verify decision object
+        assertFalse(allocateDecision.isDecisionTaken());
+        assertTrue(moveDecision.isDecisionTaken());
+        assertEquals(AllocationDecision.NO, moveDecision.getAllocationDecision());
+        assertEquals("rebalancing is not allowed, even though there is at least one node on which the shard can be allocated",
+            moveDecision.getExplanation());
+        assertTrue(moveDecision.canRemain());
+        assertFalse(moveDecision.forceMove());
+        assertFalse(moveDecision.canRebalanceCluster());
+        assertNotNull(moveDecision.getCanRemainDecision());
+        assertNull(moveDecision.getTargetNode());
+        assertEquals(2, moveDecision.getCurrentNodeRanking());
+        // verifying cluster rebalance decision object
+        assertNotNull(moveDecision.getClusterRebalanceDecision());
+        assertEquals(Decision.Type.NO, moveDecision.getClusterRebalanceDecision().type());
+        for (Decision d : moveDecision.getClusterRebalanceDecision().getDecisions()) {
+            if (d.label().equals("enable")) {
+                assertEquals(Decision.Type.NO, d.type());
+                assertEquals("no rebalancing is allowed due to index setting [index.routing.rebalance.enable=none]",
+                    d.getExplanation());
+            } else {
+                assertEquals(Decision.Type.YES, d.type());
+                assertNotNull(d.getExplanation());
+            }
+        }
+        // verify node decisions
+        assertEquals(1, moveDecision.getNodeDecisions().size());
+        NodeAllocationResult result = moveDecision.getNodeDecisions().get(0);
+        assertNotNull(result.getNode());
+        assertEquals(1, result.getWeightRanking());
+        assertEquals(AllocationDecision.YES, result.getNodeDecision());
+        if (includeYesDecisions) {
+            assertThat(result.getCanAllocateDecision().getDecisions().size(), greaterThan(0));
+        } else {
+            assertEquals(0, result.getCanAllocateDecision().getDecisions().size());
+        }
+        for (Decision d : result.getCanAllocateDecision().getDecisions()) {
+            assertEquals(Decision.Type.YES, d.type());
+            assertNotNull(d.getExplanation());
+        }
+
+        // verify JSON output
+        try (XContentParser parser = getParser(explanation)) {
+            verifyShardInfo(parser, true, includeDiskInfo, ShardRoutingState.STARTED);
+            parser.nextToken();
+            assertEquals("can_remain_on_current_node", parser.currentName());
+            parser.nextToken();
+            assertEquals(AllocationDecision.YES.toString(), parser.text());
+            parser.nextToken();
+            assertEquals("can_rebalance_cluster", parser.currentName());
+            parser.nextToken();
+            assertEquals(AllocationDecision.NO.toString(), parser.text());
+            parser.nextToken();
+            assertEquals("can_rebalance_cluster_decisions", parser.currentName());
+            verifyDeciders(parser, AllocationDecision.NO);
+            parser.nextToken();
+            assertEquals("can_rebalance_to_other_node", parser.currentName());
+            parser.nextToken();
+            assertEquals(AllocationDecision.NO.toString(), parser.text());
+            parser.nextToken();
+            assertEquals("rebalance_explanation", parser.currentName());
+            parser.nextToken();
+            assertEquals("rebalancing is not allowed, even though there is at least one node on which the shard can be allocated",
+                parser.text());
+            verifyNodeDecisions(parser, allNodeDecisions(AllocationDecision.YES, true), includeYesDecisions, false);
+            assertEquals(Token.END_OBJECT, parser.nextToken());
+        }
+    }
+
+    public void testWorseBalance() throws Exception {
+        logger.info("--> starting a single node");
+        internalCluster().startNode();
+        ensureStableCluster(1);
 
-        // Wait for all 3 nodes to be up
-        logger.info("--> waiting for 3 nodes to be up");
+        logger.info("--> creating an index with 5 shards, all allocated to the single node");
+        createIndexAndIndexData(5, 0);
+
+        logger.info("--> setting balancing threshold really high, so it won't be met");
+        client().admin().cluster().prepareUpdateSettings().setTransientSettings(
+            Settings.builder().put("cluster.routing.allocation.balance.threshold", 1000.0f)).get();
+
+        logger.info("--> starting another node, with the rebalance threshold so high, it should not get any shards");
+        internalCluster().startNode();
+        ensureStableCluster(2);
+
+        boolean includeYesDecisions = randomBoolean();
+        boolean includeDiskInfo = randomBoolean();
+        ClusterAllocationExplanation explanation = runExplain(true, includeYesDecisions, includeDiskInfo);
+
+        ShardId shardId = explanation.getShard();
+        boolean isPrimary = explanation.isPrimary();
+        ShardRoutingState shardRoutingState = explanation.getShardState();
+        DiscoveryNode currentNode = explanation.getCurrentNode();
+        UnassignedInfo unassignedInfo = explanation.getUnassignedInfo();
+        ClusterInfo clusterInfo = explanation.getClusterInfo();
+        AllocateUnassignedDecision allocateDecision = explanation.getShardAllocationDecision().getAllocateDecision();
+        MoveDecision moveDecision = explanation.getShardAllocationDecision().getMoveDecision();
+
+        // verify shard info
+        assertEquals("idx", shardId.getIndexName());
+        assertEquals(0, shardId.getId());
+        assertTrue(isPrimary);
+
+        // verify current node info
+        assertEquals(ShardRoutingState.STARTED, shardRoutingState);
+        assertNotNull(currentNode);
+
+        // verify unassigned info
+        assertNull(unassignedInfo);
+
+        // verify cluster info
+        verifyClusterInfo(clusterInfo, includeDiskInfo, 2);
+
+        // verify decision object
+        assertFalse(allocateDecision.isDecisionTaken());
+        assertTrue(moveDecision.isDecisionTaken());
+        assertEquals(AllocationDecision.NO, moveDecision.getAllocationDecision());
+        assertEquals("cannot rebalance as no target node exists that can both allocate this shard and improve the cluster balance",
+            moveDecision.getExplanation());
+        assertTrue(moveDecision.canRemain());
+        assertFalse(moveDecision.forceMove());
+        assertTrue(moveDecision.canRebalanceCluster());
+        assertNotNull(moveDecision.getCanRemainDecision());
+        assertNull(moveDecision.getTargetNode());
+        assertEquals(1, moveDecision.getCurrentNodeRanking());
+        // verifying cluster rebalance decision object
+        assertNotNull(moveDecision.getClusterRebalanceDecision());
+        assertEquals(Decision.Type.YES, moveDecision.getClusterRebalanceDecision().type());
+        for (Decision d : moveDecision.getClusterRebalanceDecision().getDecisions()) {
+            assertEquals(Decision.Type.YES, d.type());
+            assertNotNull(d.getExplanation());
+        }
+        // verify node decisions
+        assertEquals(1, moveDecision.getNodeDecisions().size());
+        NodeAllocationResult result = moveDecision.getNodeDecisions().get(0);
+        assertNotNull(result.getNode());
+        assertEquals(1, result.getWeightRanking());
+        assertEquals(AllocationDecision.WORSE_BALANCE, result.getNodeDecision());
+        if (includeYesDecisions) {
+            assertThat(result.getCanAllocateDecision().getDecisions().size(), greaterThan(0));
+        } else {
+            assertEquals(0, result.getCanAllocateDecision().getDecisions().size());
+        }
+        for (Decision d : result.getCanAllocateDecision().getDecisions()) {
+            assertEquals(Decision.Type.YES, d.type());
+            assertNotNull(d.getExplanation());
+        }
+
+        // verify JSON output
+        try (XContentParser parser = getParser(explanation)) {
+            verifyShardInfo(parser, true, includeDiskInfo, ShardRoutingState.STARTED);
+            parser.nextToken();
+            assertEquals("can_remain_on_current_node", parser.currentName());
+            parser.nextToken();
+            assertEquals(AllocationDecision.YES.toString(), parser.text());
+            parser.nextToken();
+            assertEquals("can_rebalance_cluster", parser.currentName());
+            parser.nextToken();
+            assertEquals(AllocationDecision.YES.toString(), parser.text());
+            parser.nextToken();
+            assertEquals("can_rebalance_to_other_node", parser.currentName());
+            parser.nextToken();
+            assertEquals(AllocationDecision.NO.toString(), parser.text());
+            parser.nextToken();
+            assertEquals("rebalance_explanation", parser.currentName());
+            parser.nextToken();
+            assertEquals("cannot rebalance as no target node exists that can both allocate this shard and improve the cluster balance",
+                parser.text());
+            verifyNodeDecisions(parser, allNodeDecisions(AllocationDecision.WORSE_BALANCE, true), includeYesDecisions, false);
+            assertEquals(Token.END_OBJECT, parser.nextToken());
+        }
+    }
+
+    public void testBetterBalanceButCannotAllocate() throws Exception {
+        logger.info("--> starting a single node");
+        String firstNode = internalCluster().startNode();
+        ensureStableCluster(1);
+
+        logger.info("--> creating an index with 5 shards, all allocated to the single node");
+        createIndexAndIndexData(5, 0);
+
+        logger.info("--> setting up allocation filtering to only allow allocation to the current node");
+        client().admin().indices().prepareUpdateSettings("idx").setSettings(
+            Settings.builder().put("index.routing.allocation.include._name", firstNode)).get();
+
+        logger.info("--> starting another node, with filtering not allowing allocation to the new node, it should not get any shards");
+        internalCluster().startNode();
+        ensureStableCluster(2);
+
+        boolean includeYesDecisions = randomBoolean();
+        boolean includeDiskInfo = randomBoolean();
+        ClusterAllocationExplanation explanation = runExplain(true, includeYesDecisions, includeDiskInfo);
+
+        ShardId shardId = explanation.getShard();
+        boolean isPrimary = explanation.isPrimary();
+        ShardRoutingState shardRoutingState = explanation.getShardState();
+        DiscoveryNode currentNode = explanation.getCurrentNode();
+        UnassignedInfo unassignedInfo = explanation.getUnassignedInfo();
+        ClusterInfo clusterInfo = explanation.getClusterInfo();
+        AllocateUnassignedDecision allocateDecision = explanation.getShardAllocationDecision().getAllocateDecision();
+        MoveDecision moveDecision = explanation.getShardAllocationDecision().getMoveDecision();
+
+        // verify shard info
+        assertEquals("idx", shardId.getIndexName());
+        assertEquals(0, shardId.getId());
+        assertTrue(isPrimary);
+
+        // verify current node info
+        assertEquals(ShardRoutingState.STARTED, shardRoutingState);
+        assertNotNull(currentNode);
+
+        // verify unassigned info
+        assertNull(unassignedInfo);
+
+        // verify cluster info
+        verifyClusterInfo(clusterInfo, includeDiskInfo, 2);
+
+        // verify decision object
+        assertFalse(allocateDecision.isDecisionTaken());
+        assertTrue(moveDecision.isDecisionTaken());
+        assertEquals(AllocationDecision.NO, moveDecision.getAllocationDecision());
+        assertEquals("cannot rebalance as no target node exists that can both allocate this shard and improve the cluster balance",
+            moveDecision.getExplanation());
+        assertTrue(moveDecision.canRemain());
+        assertFalse(moveDecision.forceMove());
+        assertTrue(moveDecision.canRebalanceCluster());
+        assertNotNull(moveDecision.getCanRemainDecision());
+        assertNull(moveDecision.getTargetNode());
+        assertEquals(2, moveDecision.getCurrentNodeRanking());
+        // verifying cluster rebalance decision object
+        assertNotNull(moveDecision.getClusterRebalanceDecision());
+        assertEquals(Decision.Type.YES, moveDecision.getClusterRebalanceDecision().type());
+        for (Decision d : moveDecision.getClusterRebalanceDecision().getDecisions()) {
+            assertEquals(Decision.Type.YES, d.type());
+            assertNotNull(d.getExplanation());
+        }
+        // verify node decisions
+        assertEquals(1, moveDecision.getNodeDecisions().size());
+        NodeAllocationResult result = moveDecision.getNodeDecisions().get(0);
+        assertNotNull(result.getNode());
+        assertEquals(1, result.getWeightRanking());
+        assertEquals(AllocationDecision.NO, result.getNodeDecision());
+        if (includeYesDecisions) {
+            assertThat(result.getCanAllocateDecision().getDecisions().size(), greaterThan(1));
+        } else {
+            assertEquals(1, result.getCanAllocateDecision().getDecisions().size());
+        }
+        String primaryNodeName = primaryNodeName();
+        for (Decision d : result.getCanAllocateDecision().getDecisions()) {
+            if (d.label().equals("filter")) {
+                assertEquals(Decision.Type.NO, d.type());
+                assertEquals("node does not match index setting [index.routing.allocation.include] filters [_name:\"" +
+                                 primaryNodeName + "\"]", d.getExplanation());
+            } else {
+                assertEquals(Decision.Type.YES, d.type());
+                assertNotNull(d.getExplanation());
+            }
+        }
+
+        // verify JSON output
+        try (XContentParser parser = getParser(explanation)) {
+            verifyShardInfo(parser, true, includeDiskInfo, ShardRoutingState.STARTED);
+            parser.nextToken();
+            assertEquals("can_remain_on_current_node", parser.currentName());
+            parser.nextToken();
+            assertEquals(AllocationDecision.YES.toString(), parser.text());
+            parser.nextToken();
+            assertEquals("can_rebalance_cluster", parser.currentName());
+            parser.nextToken();
+            assertEquals(AllocationDecision.YES.toString(), parser.text());
+            parser.nextToken();
+            assertEquals("can_rebalance_to_other_node", parser.currentName());
+            parser.nextToken();
+            assertEquals(AllocationDecision.NO.toString(), parser.text());
+            parser.nextToken();
+            assertEquals("rebalance_explanation", parser.currentName());
+            parser.nextToken();
+            assertEquals("cannot rebalance as no target node exists that can both allocate this shard and improve the cluster balance",
+                parser.text());
+            verifyNodeDecisions(parser, allNodeDecisions(AllocationDecision.NO, true), includeYesDecisions, false);
+            assertEquals(Token.END_OBJECT, parser.nextToken());
+        }
+    }
+
+    public void testAssignedReplicaOnSpecificNode() throws Exception {
+        logger.info("--> starting 3 nodes");
+        List<String> nodes = internalCluster().startNodes(3);
+
+        logger.info("--> creating an index with 1 primary and 2 replicas");
+        String excludedNode = nodes.get(randomIntBetween(0, 2));
+        createIndexAndIndexData(1, 2, Settings.builder().put("index.routing.allocation.exclude._name", excludedNode).build(),
+            ActiveShardCount.from(2));
+
+        boolean includeYesDecisions = randomBoolean();
+        boolean includeDiskInfo = randomBoolean();
+        ClusterAllocationExplanation explanation = runExplain(false, replicaNode().getId(), includeYesDecisions, includeDiskInfo);
+
+        ShardId shardId = explanation.getShard();
+        boolean isPrimary = explanation.isPrimary();
+        ShardRoutingState shardRoutingState = explanation.getShardState();
+        DiscoveryNode currentNode = explanation.getCurrentNode();
+        UnassignedInfo unassignedInfo = explanation.getUnassignedInfo();
+        ClusterInfo clusterInfo = explanation.getClusterInfo();
+        AllocateUnassignedDecision allocateDecision = explanation.getShardAllocationDecision().getAllocateDecision();
+        MoveDecision moveDecision = explanation.getShardAllocationDecision().getMoveDecision();
+
+        // verify shard info
+        assertEquals("idx", shardId.getIndexName());
+        assertEquals(0, shardId.getId());
+        assertFalse(isPrimary);
+
+        // verify current node info
+        assertEquals(ShardRoutingState.STARTED, shardRoutingState);
+        assertNotNull(currentNode);
+        assertEquals(replicaNode().getName(), currentNode.getName());
+
+        // verify unassigned info
+        assertNull(unassignedInfo);
+
+        // verify cluster info
+        verifyClusterInfo(clusterInfo, includeDiskInfo, 3);
+
+        // verify decision objects
+        assertFalse(allocateDecision.isDecisionTaken());
+        assertTrue(moveDecision.isDecisionTaken());
+        assertEquals(AllocationDecision.NO, moveDecision.getAllocationDecision());
+        assertEquals("rebalancing is not allowed", moveDecision.getExplanation());
+        assertTrue(moveDecision.canRemain());
+        assertFalse(moveDecision.forceMove());
+        assertFalse(moveDecision.canRebalanceCluster());
+        assertNotNull(moveDecision.getCanRemainDecision());
+        assertNull(moveDecision.getTargetNode());
+        // verifying cluster rebalance decision object
+        assertNotNull(moveDecision.getClusterRebalanceDecision());
+        assertEquals(Decision.Type.NO, moveDecision.getClusterRebalanceDecision().type());
+        // verify node decisions
+        assertEquals(2, moveDecision.getNodeDecisions().size());
+        for (NodeAllocationResult result : moveDecision.getNodeDecisions()) {
+            assertNotNull(result.getNode());
+            assertEquals(1, result.getWeightRanking());
+            assertEquals(AllocationDecision.NO, result.getNodeDecision());
+            if (includeYesDecisions) {
+                assertThat(result.getCanAllocateDecision().getDecisions().size(), greaterThan(1));
+            } else {
+                assertEquals(1, result.getCanAllocateDecision().getDecisions().size());
+            }
+            for (Decision d : result.getCanAllocateDecision().getDecisions()) {
+                if (d.type() == Decision.Type.NO) {
+                    assertThat(d.label(), isOneOf("filter", "same_shard"));
+                }
+                assertNotNull(d.getExplanation());
+            }
+        }
+
+        // verify JSON output
+        try (XContentParser parser = getParser(explanation)) {
+            verifyShardInfo(parser, false, includeDiskInfo, ShardRoutingState.STARTED);
+            parser.nextToken();
+            assertEquals("can_remain_on_current_node", parser.currentName());
+            parser.nextToken();
+            assertEquals(AllocationDecision.YES.toString(), parser.text());
+            parser.nextToken();
+            assertEquals("can_rebalance_cluster", parser.currentName());
+            parser.nextToken();
+            assertEquals(AllocationDecision.NO.toString(), parser.text());
+            parser.nextToken();
+            assertEquals("can_rebalance_cluster_decisions", parser.currentName());
+            verifyDeciders(parser, AllocationDecision.NO);
+            parser.nextToken();
+            assertEquals("can_rebalance_to_other_node", parser.currentName());
+            parser.nextToken();
+            assertEquals(AllocationDecision.NO.toString(), parser.text());
+            parser.nextToken();
+            assertEquals("rebalance_explanation", parser.currentName());
+            parser.nextToken();
+            assertEquals("rebalancing is not allowed", parser.text());
+            verifyNodeDecisions(parser, allNodeDecisions(AllocationDecision.NO, false), includeYesDecisions, false);
+            assertEquals(Token.END_OBJECT, parser.nextToken());
+        }
+    }
+
+    public void testCannotAllocateStaleReplicaExplanation() throws Exception {
+        logger.info("--> starting 3 nodes");
+        final String masterNode = internalCluster().startNode();
+        // start replica node first, so it's path will be used first when we start a node after
+        // stopping all of them at end of test.
+        final String replicaNode = internalCluster().startNode();
+        final String primaryNode = internalCluster().startNode();
+
+        logger.info("--> creating an index with 1 primary and 1 replica");
+        createIndexAndIndexData(1, 1,
+            Settings.builder()
+                .put("index.routing.allocation.include._name", primaryNode)
+                .put("index.routing.allocation.exclude._name", masterNode)
+                .build(),
+            ActiveShardCount.ONE);
+
+        client().admin().indices().prepareUpdateSettings("idx").setSettings(
+            Settings.builder().put("index.routing.allocation.include._name", (String) null)).get();
+        ensureGreen();
+
+        assertThat(replicaNode().getName(), equalTo(replicaNode));
+        assertThat(primaryNodeName(), equalTo(primaryNode));
+
+        logger.info("--> stop node with the replica shard");
+        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(replicaNode));
+
+        logger.info("--> index more data, now the replica is stale");
+        indexData();
+
+        logger.info("--> stop the node with the primary");
+        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNode));
+
+        logger.info("--> restart the node with the stale replica");
+        String restartedNode = internalCluster().startDataOnlyNode();
+        ensureClusterSizeConsistency(); // wait for the master to finish processing join.
+
+        // wait until the system has fetched shard data and we know there is no valid shard copy
         assertBusy(() -> {
-            NodesStatsResponse resp = client().admin().cluster().prepareNodesStats().get();
-            assertThat(resp.getNodes().size(), equalTo(3));
+            ClusterAllocationExplanation explanation = client().admin().cluster().prepareAllocationExplain()
+                .setIndex("idx").setShard(0).setPrimary(true).get().getExplanation();
+            assertTrue(explanation.getShardAllocationDecision().getAllocateDecision().isDecisionTaken());
+            assertEquals(AllocationDecision.NO_VALID_SHARD_COPY,
+                explanation.getShardAllocationDecision().getAllocateDecision().getAllocationDecision());
         });
+        boolean includeYesDecisions = randomBoolean();
+        boolean includeDiskInfo = randomBoolean();
+        ClusterAllocationExplanation explanation = runExplain(true, includeYesDecisions, includeDiskInfo);
 
-        logger.info("--> creating 'test' index");
-        assertAcked(prepareCreate("test").setSettings(Settings.builder()
-                .put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "1m")
-                .put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 5)
-                .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1))
-                .setWaitForActiveShards(ActiveShardCount.ALL).get());
+        ShardId shardId = explanation.getShard();
+        boolean isPrimary = explanation.isPrimary();
+        ShardRoutingState shardRoutingState = explanation.getShardState();
+        DiscoveryNode currentNode = explanation.getCurrentNode();
+        UnassignedInfo unassignedInfo = explanation.getUnassignedInfo();
+        AllocateUnassignedDecision allocateDecision = explanation.getShardAllocationDecision().getAllocateDecision();
+        MoveDecision moveDecision = explanation.getShardAllocationDecision().getMoveDecision();
 
-        logger.info("--> stopping a random node");
-        assertTrue(internalCluster().stopRandomDataNode());
+        // verify shard info
+        assertEquals("idx", shardId.getIndexName());
+        assertEquals(0, shardId.getId());
+        assertTrue(isPrimary);
 
-        ensureYellow("test");
+        // verify current node info
+        assertEquals(ShardRoutingState.UNASSIGNED, shardRoutingState);
+        assertNull(currentNode);
+        // verify unassigned info
+        assertNotNull(unassignedInfo);
+
+        // verify decision object
+        assertTrue(allocateDecision.isDecisionTaken());
+        assertFalse(moveDecision.isDecisionTaken());
+        assertEquals(AllocationDecision.NO_VALID_SHARD_COPY, allocateDecision.getAllocationDecision());
+        assertEquals(2, allocateDecision.getNodeDecisions().size());
+        for (NodeAllocationResult nodeAllocationResult : allocateDecision.getNodeDecisions()) {
+            if (nodeAllocationResult.getNode().getName().equals(restartedNode)) {
+                assertNotNull(nodeAllocationResult.getShardStoreInfo());
+                assertNotNull(nodeAllocationResult.getShardStoreInfo().getAllocationId());
+                assertFalse(nodeAllocationResult.getShardStoreInfo().isInSync());
+                assertNull(nodeAllocationResult.getShardStoreInfo().getStoreException());
+            } else {
+                assertNotNull(nodeAllocationResult.getShardStoreInfo());
+                assertNull(nodeAllocationResult.getShardStoreInfo().getAllocationId());
+                assertFalse(nodeAllocationResult.getShardStoreInfo().isInSync());
+                assertNull(nodeAllocationResult.getShardStoreInfo().getStoreException());
+            }
+        }
 
-        ClusterAllocationExplainResponse resp = client().admin().cluster().prepareAllocationExplain().useAnyUnassignedShard().get();
-        ClusterAllocationExplanation cae = resp.getExplanation();
-        assertThat(cae.getShard().getIndexName(), equalTo("test"));
-        assertFalse(cae.isPrimary());
-        assertFalse(cae.isAssigned());
-        assertThat("expecting a remaining delay, got: " + cae.getRemainingDelayMillis(), cae.getRemainingDelayMillis(), greaterThan(0L));
+        // verify JSON output
+        try (XContentParser parser = getParser(explanation)) {
+            verifyShardInfo(parser, true, includeDiskInfo, ShardRoutingState.UNASSIGNED);
+            parser.nextToken();
+            assertEquals("can_allocate", parser.currentName());
+            parser.nextToken();
+            assertEquals(AllocationDecision.NO_VALID_SHARD_COPY.toString(), parser.text());
+            parser.nextToken();
+            assertEquals("allocate_explanation", parser.currentName());
+            parser.nextToken();
+            assertEquals("cannot allocate because all found copies of the shard are either stale or corrupt", parser.text());
+            verifyStaleShardCopyNodeDecisions(parser, 2, Collections.singleton(restartedNode));
+        }
     }
 
-    public void testUnassignedShards() throws Exception {
-        logger.info("--> starting 3 nodes");
-        String noAttrNode = internalCluster().startNode();
-        String barAttrNode = internalCluster().startNode(Settings.builder().put("node.attr.bar", "baz"));
-        String fooBarAttrNode = internalCluster().startNode(Settings.builder()
-                .put("node.attr.foo", "bar")
-                .put("node.attr.bar", "baz"));
-
-        // Wait for all 3 nodes to be up
-        logger.info("--> waiting for 3 nodes to be up");
-        client().admin().cluster().health(Requests.clusterHealthRequest().waitForNodes("3")).actionGet();
-
-        client().admin().indices().prepareCreate("anywhere")
-                .setSettings(Settings.builder()
-                        .put("index.number_of_shards", 5)
-                        .put("index.number_of_replicas", 1))
-                .setWaitForActiveShards(ActiveShardCount.ALL)  // wait on all shards
-                .get();
-
-        client().admin().indices().prepareCreate("only-baz")
-                .setSettings(Settings.builder()
-                        .put("index.routing.allocation.include.bar", "baz")
-                        .put("index.number_of_shards", 5)
-                        .put("index.number_of_replicas", 1))
-                .setWaitForActiveShards(ActiveShardCount.ALL)
-                .get();
-
-        client().admin().indices().prepareCreate("only-foo")
-                .setSettings(Settings.builder()
-                        .put("index.routing.allocation.include.foo", "bar")
-                        .put("index.number_of_shards", 1)
-                        .put("index.number_of_replicas", 1))
-                .get();
-
-        ClusterAllocationExplainResponse resp = client().admin().cluster().prepareAllocationExplain()
-                .setIndex("only-foo")
-                .setShard(0)
-                .setPrimary(false)
-                .get();
-        ClusterAllocationExplanation cae = resp.getExplanation();
-        assertThat(cae.getShard().getIndexName(), equalTo("only-foo"));
-        assertFalse(cae.isPrimary());
-        assertFalse(cae.isAssigned());
-        assertFalse(cae.isStillFetchingShardData());
-        assertThat(UnassignedInfo.Reason.INDEX_CREATED, equalTo(cae.getUnassignedInfo().getReason()));
-        assertThat("expecting no remaining delay: " + cae.getRemainingDelayMillis(), cae.getRemainingDelayMillis(), equalTo(0L));
-
-        Map<DiscoveryNode, NodeExplanation> explanations = cae.getNodeExplanations();
-
-        Float barAttrWeight = -1f;
-        Float fooBarAttrWeight = -1f;
-        for (Map.Entry<DiscoveryNode, NodeExplanation> entry : explanations.entrySet()) {
-            DiscoveryNode node = entry.getKey();
-            String nodeName = node.getName();
-            NodeExplanation explanation = entry.getValue();
-            ClusterAllocationExplanation.FinalDecision finalDecision = explanation.getFinalDecision();
-            ClusterAllocationExplanation.StoreCopy storeCopy = explanation.getStoreCopy();
-            Decision d = explanation.getDecision();
-            float weight = explanation.getWeight();
-            IndicesShardStoresResponse.StoreStatus storeStatus = explanation.getStoreStatus();
-
-            assertEquals(d.type(), Decision.Type.NO);
-            if (noAttrNode.equals(nodeName)) {
-                assertThat(d.toString(), containsString("node does not match index include filters [foo:\"bar\"]"));
-                assertNull(storeStatus);
-                assertEquals("the shard cannot be assigned because one or more allocation decider returns a 'NO' decision",
-                        explanation.getFinalExplanation());
-                assertEquals(ClusterAllocationExplanation.FinalDecision.NO, finalDecision);
-            } else if (barAttrNode.equals(nodeName)) {
-                assertThat(d.toString(), containsString("node does not match index include filters [foo:\"bar\"]"));
-                barAttrWeight = weight;
-                assertNull(storeStatus);
-                assertEquals("the shard cannot be assigned because one or more allocation decider returns a 'NO' decision",
-                        explanation.getFinalExplanation());
-                assertEquals(ClusterAllocationExplanation.FinalDecision.NO, finalDecision);
-            } else if (fooBarAttrNode.equals(nodeName)) {
-                assertThat(d.toString(), containsString("the shard cannot be allocated on the same node id"));
-                fooBarAttrWeight = weight;
-                assertEquals(storeStatus.getAllocationStatus(),
-                        IndicesShardStoresResponse.StoreStatus.AllocationStatus.PRIMARY);
-                assertEquals(ClusterAllocationExplanation.FinalDecision.NO, finalDecision);
-                assertEquals(ClusterAllocationExplanation.StoreCopy.AVAILABLE, storeCopy);
-                assertEquals("the shard cannot be assigned because one or more allocation decider returns a 'NO' decision",
-                        explanation.getFinalExplanation());
+    private void verifyClusterInfo(ClusterInfo clusterInfo, boolean includeDiskInfo, int numNodes) {
+        if (includeDiskInfo) {
+            assertThat(clusterInfo.getNodeMostAvailableDiskUsages().size(), greaterThanOrEqualTo(0));
+            assertThat(clusterInfo.getNodeLeastAvailableDiskUsages().size(), greaterThanOrEqualTo(0));
+            assertThat(clusterInfo.getNodeMostAvailableDiskUsages().size(), lessThanOrEqualTo(numNodes));
+            assertThat(clusterInfo.getNodeLeastAvailableDiskUsages().size(), lessThanOrEqualTo(numNodes));
+        } else {
+            assertNull(clusterInfo);
+        }
+    }
+
+    private ClusterAllocationExplanation runExplain(boolean primary, boolean includeYesDecisions, boolean includeDiskInfo)
+        throws Exception {
+
+        return runExplain(primary, null, includeYesDecisions, includeDiskInfo);
+    }
+
+    private ClusterAllocationExplanation runExplain(boolean primary, String nodeId, boolean includeYesDecisions, boolean includeDiskInfo)
+        throws Exception {
+
+        ClusterAllocationExplanation explanation = client().admin().cluster().prepareAllocationExplain()
+            .setIndex("idx").setShard(0).setPrimary(primary)
+            .setIncludeYesDecisions(includeYesDecisions)
+            .setIncludeDiskInfo(includeDiskInfo)
+            .setCurrentNode(nodeId)
+            .get().getExplanation();
+        if (logger.isDebugEnabled()) {
+            XContentBuilder builder = JsonXContent.contentBuilder();
+            builder.prettyPrint();
+            builder.humanReadable(true);
+            logger.debug("--> explain json output: \n{}", explanation.toXContent(builder, ToXContent.EMPTY_PARAMS).string());
+        }
+        return explanation;
+    }
+
+    private void createIndexAndIndexData(int numPrimaries, int numReplicas) {
+        createIndexAndIndexData(numPrimaries, numReplicas, Settings.EMPTY, ActiveShardCount.ALL);
+    }
+
+    private void createIndexAndIndexData(int numPrimaries, int numReplicas, Settings settings, ActiveShardCount activeShardCount) {
+        client().admin().indices().prepareCreate("idx")
+            .setSettings(Settings.builder()
+                             .put("index.number_of_shards", numPrimaries)
+                             .put("index.number_of_replicas", numReplicas)
+                             .put(settings))
+            .setWaitForActiveShards(activeShardCount)
+            .get();
+        if (activeShardCount != ActiveShardCount.NONE) {
+            indexData();
+        }
+    }
+
+    private void indexData() {
+        for (int i = 0; i < 10; i++) {
+            index("idx", "t", Integer.toString(i), Collections.singletonMap("f1", Integer.toString(i)));
+        }
+        flushAndRefresh("idx");
+    }
+
+    private String primaryNodeName() {
+        ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
+        String nodeId = clusterState.getRoutingTable().index("idx").shard(0).primaryShard().currentNodeId();
+        return clusterState.getRoutingNodes().node(nodeId).node().getName();
+    }
+
+    private DiscoveryNode replicaNode() {
+        ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
+        String nodeId = clusterState.getRoutingTable().index("idx").shard(0).replicaShards().get(0).currentNodeId();
+        return clusterState.getRoutingNodes().node(nodeId).node();
+    }
+
+    private XContentParser getParser(ClusterAllocationExplanation explanation) throws IOException {
+        XContentBuilder builder = JsonXContent.contentBuilder();
+        return createParser(explanation.toXContent(builder, ToXContent.EMPTY_PARAMS));
+    }
+
+    private void verifyShardInfo(XContentParser parser, boolean primary, boolean includeDiskInfo, ShardRoutingState state)
+        throws IOException {
+
+        parser.nextToken();
+        assertEquals(Token.START_OBJECT, parser.currentToken());
+        parser.nextToken();
+        assertEquals("index", parser.currentName());
+        parser.nextToken();
+        assertEquals("idx", parser.text());
+        parser.nextToken();
+        assertEquals("shard", parser.currentName());
+        parser.nextToken();
+        assertEquals(0, parser.intValue());
+        parser.nextToken();
+        assertEquals("primary", parser.currentName());
+        parser.nextToken();
+        assertEquals(primary, parser.booleanValue());
+        parser.nextToken();
+        assertEquals("current_state", parser.currentName());
+        parser.nextToken();
+        assertEquals(state.toString().toLowerCase(Locale.ROOT), parser.text());
+        if (state == ShardRoutingState.UNASSIGNED) {
+            parser.nextToken();
+            assertEquals("unassigned_info", parser.currentName());
+            assertEquals(Token.START_OBJECT, parser.nextToken());
+            Token token;
+            while ((token = parser.nextToken()) != Token.END_OBJECT) { // until we reach end of unassigned_info
+                if (token == XContentParser.Token.FIELD_NAME) {
+                    assertNotEquals("delayed", parser.currentName()); // we should never display "delayed" from unassigned info
+                    if (parser.currentName().equals("last_allocation_status")) {
+                        parser.nextToken();
+                        assertThat(parser.text(), isOneOf(AllocationDecision.NO.toString(),
+                            AllocationDecision.NO_VALID_SHARD_COPY.toString(),
+                            AllocationDecision.AWAITING_INFO.toString(),
+                            AllocationDecision.NO_ATTEMPT.toString()));
+                    }
+                }
+            }
+        } else {
+            assertEquals(ShardRoutingState.STARTED, state);
+            parser.nextToken();
+            assertEquals("current_node", parser.currentName());
+            assertEquals(Token.START_OBJECT, parser.nextToken());
+            Token token;
+            while ((token = parser.nextToken()) != Token.END_OBJECT) { // until we reach end of current_node
+                if (token == Token.FIELD_NAME) {
+                    assertTrue(parser.currentName().equals("id")
+                                   || parser.currentName().equals("name")
+                                   || parser.currentName().equals("transport_address")
+                                   || parser.currentName().equals("weight_ranking"));
+                } else {
+                    assertTrue(token.isValue());
+                    assertNotNull(parser.text());
+                }
+            }
+        }
+        if (includeDiskInfo) {
+            // disk info is included, just verify the object is there
+            parser.nextToken();
+            assertEquals("cluster_info", parser.currentName());
+            assertEquals(Token.START_OBJECT, parser.nextToken());
+            int numObjects = 1;
+            while (numObjects > 0) {
+                Token token = parser.nextToken();
+                if (token == Token.START_OBJECT) {
+                    ++numObjects;
+                } else if (token == Token.END_OBJECT) {
+                    --numObjects;
+                }
+            }
+        }
+    }
+
+    private void verifyStaleShardCopyNodeDecisions(XContentParser parser, int numNodes, Set<String> foundStores) throws IOException {
+        parser.nextToken();
+        assertEquals("node_allocation_decisions", parser.currentName());
+        assertEquals(Token.START_ARRAY, parser.nextToken());
+        for (int i = 0; i < numNodes; i++) {
+            String nodeName = verifyNodeDecisionPrologue(parser);
+            assertEquals(AllocationDecision.NO.toString(), parser.text());
+            parser.nextToken();
+            assertEquals("store", parser.currentName());
+            assertEquals(Token.START_OBJECT, parser.nextToken());
+            parser.nextToken();
+            if (foundStores.contains(nodeName)) {
+                // shard data was found on the node, but it is stale
+                assertEquals("in_sync", parser.currentName());
+                parser.nextToken();
+                assertFalse(parser.booleanValue());
+                parser.nextToken();
+                assertEquals("allocation_id", parser.currentName());
+                parser.nextToken();
+                assertNotNull(parser.text());
             } else {
-                fail("unexpected node with name: " + nodeName +
-                     ", I have: " + noAttrNode + ", " + barAttrNode + ", " + fooBarAttrNode);
+                // no shard data was found on the node
+                assertEquals("found", parser.currentName());
+                parser.nextToken();
+                assertFalse(parser.booleanValue());
             }
+            assertEquals(Token.END_OBJECT, parser.nextToken());
+            parser.nextToken();
+            assertEquals(Token.END_OBJECT, parser.currentToken());
         }
-        assertFalse(barAttrWeight == fooBarAttrWeight);
+        assertEquals(Token.END_ARRAY, parser.nextToken());
     }
+
+    private void verifyNodeDecisions(XContentParser parser, Map<String, AllocationDecision> expectedNodeDecisions,
+                                     boolean includeYesDecisions, boolean reuseStore) throws IOException {
+        parser.nextToken();
+        assertEquals("node_allocation_decisions", parser.currentName());
+        assertEquals(Token.START_ARRAY, parser.nextToken());
+        boolean encounteredNo = false;
+        final int numNodes = expectedNodeDecisions.size();
+        for (int i = 0; i < numNodes; i++) {
+            String nodeName = verifyNodeDecisionPrologue(parser);
+            AllocationDecision allocationDecision = expectedNodeDecisions.get(nodeName);
+            assertEquals(allocationDecision.toString(), parser.text());
+            if (allocationDecision != AllocationDecision.YES) {
+                encounteredNo = true;
+            } else {
+                assertFalse("encountered a YES node decision after a NO node decision - sort order is wrong", encounteredNo);
+            }
+            parser.nextToken();
+            if ("store".equals(parser.currentName())) {
+                assertTrue("store info should not be present", reuseStore);
+                assertEquals(Token.START_OBJECT, parser.nextToken());
+                parser.nextToken();
+                assertEquals("matching_size_in_bytes", parser.currentName());
+                parser.nextToken();
+                assertThat(parser.longValue(), greaterThan(0L));
+                assertEquals(Token.END_OBJECT, parser.nextToken());
+                parser.nextToken();
+            }
+            if (reuseStore == false) {
+                assertEquals("weight_ranking", parser.currentName());
+                parser.nextToken();
+                assertThat(parser.intValue(), greaterThan(0));
+                parser.nextToken();
+            }
+            if (allocationDecision == AllocationDecision.NO || allocationDecision == AllocationDecision.THROTTLED || includeYesDecisions) {
+                assertEquals("deciders", parser.currentName());
+                boolean atLeastOneMatchingDecisionFound = verifyDeciders(parser, allocationDecision);
+                parser.nextToken();
+                if (allocationDecision == AllocationDecision.NO || allocationDecision == AllocationDecision.THROTTLED) {
+                    assertTrue("decision was " + allocationDecision + " but found no node's with that decision",
+                        atLeastOneMatchingDecisionFound);
+                }
+            }
+            assertEquals(Token.END_OBJECT, parser.currentToken());
+        }
+        assertEquals(Token.END_ARRAY, parser.nextToken());
+    }
+
+    private String verifyNodeDecisionPrologue(XContentParser parser) throws IOException {
+        assertEquals(Token.START_OBJECT, parser.nextToken());
+        parser.nextToken();
+        assertEquals("node_id", parser.currentName());
+        parser.nextToken();
+        assertNotNull(parser.text());
+        parser.nextToken();
+        assertEquals("node_name", parser.currentName());
+        parser.nextToken();
+        String nodeName = parser.text();
+        assertNotNull(nodeName);
+        parser.nextToken();
+        assertEquals("transport_address", parser.currentName());
+        parser.nextToken();
+        assertNotNull(parser.text());
+        parser.nextToken();
+        assertEquals("node_decision", parser.currentName());
+        parser.nextToken();
+        return nodeName;
+    }
+
+    private boolean verifyDeciders(XContentParser parser, AllocationDecision allocationDecision) throws IOException {
+        assertEquals(Token.START_ARRAY, parser.nextToken());
+        boolean atLeastOneMatchingDecisionFound = false;
+        while (parser.nextToken() != Token.END_ARRAY) {
+            assertEquals(Token.START_OBJECT, parser.currentToken());
+            parser.nextToken();
+            assertEquals("decider", parser.currentName());
+            parser.nextToken();
+            assertNotNull(parser.text());
+            parser.nextToken();
+            assertEquals("decision", parser.currentName());
+            parser.nextToken();
+            String decisionText = parser.text();
+            if ((allocationDecision == AllocationDecision.NO && decisionText.equals("NO")
+                     || (allocationDecision == AllocationDecision.THROTTLED && decisionText.equals("THROTTLE")))) {
+                atLeastOneMatchingDecisionFound = true;
+            }
+            assertNotNull(decisionText);
+            parser.nextToken();
+            assertEquals("explanation", parser.currentName());
+            parser.nextToken();
+            assertNotNull(parser.text());
+            assertEquals(Token.END_OBJECT, parser.nextToken());
+        }
+        return atLeastOneMatchingDecisionFound;
+    }
+
+    private Map<String, AllocationDecision> allNodeDecisions(AllocationDecision allocationDecision, boolean removePrimary) {
+        Map<String, AllocationDecision> nodeDecisions = new HashMap<>();
+        Set<String> allNodes = Sets.newHashSet(internalCluster().getNodeNames());
+        allNodes.remove(removePrimary ? primaryNodeName() : replicaNode().getName());
+        for (String nodeName : allNodes) {
+            nodeDecisions.put(nodeName, allocationDecision);
+        }
+        return nodeDecisions;
+    }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainRequestTests.java
index 926de8b253e77..0c47caa54665c 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainRequestTests.java
@@ -26,7 +26,8 @@ public class ClusterAllocationExplainRequestTests extends ESTestCase {
 
     public void testSerialization() throws Exception {
         ClusterAllocationExplainRequest request =
-                new ClusterAllocationExplainRequest(randomAsciiOfLength(4), randomIntBetween(0, Integer.MAX_VALUE), randomBoolean());
+                new ClusterAllocationExplainRequest(randomAlphaOfLength(4), randomIntBetween(0, Integer.MAX_VALUE), randomBoolean(),
+                                                       randomBoolean() ? randomAlphaOfLength(5) : null);
         request.includeYesDecisions(randomBoolean());
         request.includeDiskInfo(randomBoolean());
         BytesStreamOutput output = new BytesStreamOutput();
@@ -39,6 +40,7 @@ public void testSerialization() throws Exception {
         assertEquals(request.isPrimary(), actual.isPrimary());
         assertEquals(request.includeYesDecisions(), actual.includeYesDecisions());
         assertEquals(request.includeDiskInfo(), actual.includeDiskInfo());
+        assertEquals(request.getCurrentNode(), actual.getCurrentNode());
     }
 
 }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainTests.java
deleted file mode 100644
index 895450e6d5b71..0000000000000
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainTests.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.action.admin.cluster.allocation;
-
-import org.elasticsearch.client.Requests;
-import org.elasticsearch.cluster.routing.allocation.decider.Decision;
-import org.elasticsearch.test.ESSingleNodeTestCase;
-
-
-/**
- * Tests for the cluster allocation explanation
- */
-public final class ClusterAllocationExplainTests extends ESSingleNodeTestCase {
-
-    public void testShardExplain() throws Exception {
-        client().admin().indices().prepareCreate("test")
-                .setSettings("index.number_of_shards", 1, "index.number_of_replicas", 1).get();
-        ClusterAllocationExplainResponse resp = client().admin().cluster().prepareAllocationExplain()
-                .setIndex("test").setShard(0).setPrimary(false).get();
-
-        ClusterAllocationExplanation cae = resp.getExplanation();
-        assertNotNull("should always have an explanation", cae);
-        assertEquals("test", cae.getShard().getIndexName());
-        assertEquals(0, cae.getShard().getId());
-        assertEquals(false, cae.isPrimary());
-        assertNull(cae.getAssignedNodeId());
-        assertFalse(cae.isStillFetchingShardData());
-        assertNotNull(cae.getUnassignedInfo());
-        NodeExplanation explanation = cae.getNodeExplanations().values().iterator().next();
-        ClusterAllocationExplanation.FinalDecision fd = explanation.getFinalDecision();
-        ClusterAllocationExplanation.StoreCopy storeCopy = explanation.getStoreCopy();
-        Decision d = explanation.getDecision();
-        assertNotNull("should have a decision", d);
-        assertEquals(Decision.Type.NO, d.type());
-        assertEquals(ClusterAllocationExplanation.FinalDecision.NO, fd);
-        assertEquals(ClusterAllocationExplanation.StoreCopy.AVAILABLE, storeCopy);
-        assertTrue(d.toString(), d.toString().contains("NO(the shard cannot be allocated on the same node id"));
-        assertTrue(d instanceof Decision.Multi);
-        Decision.Multi md = (Decision.Multi) d;
-        Decision ssd = md.getDecisions().get(0);
-        assertEquals(Decision.Type.NO, ssd.type());
-        assertTrue(ssd.toString(), ssd.toString().contains("NO(the shard cannot be allocated on the same node id"));
-        Float weight = explanation.getWeight();
-        assertNotNull("should have a weight", weight);
-
-        resp = client().admin().cluster().prepareAllocationExplain().setIndex("test").setShard(0).setPrimary(true).get();
-
-        cae = resp.getExplanation();
-        assertNotNull("should always have an explanation", cae);
-        assertEquals("test", cae.getShard().getIndexName());
-        assertEquals(0, cae.getShard().getId());
-        assertEquals(true, cae.isPrimary());
-        assertFalse(cae.isStillFetchingShardData());
-        assertNotNull("shard should have assigned node id", cae.getAssignedNodeId());
-        assertNull("assigned shard should not have unassigned info", cae.getUnassignedInfo());
-        explanation = cae.getNodeExplanations().values().iterator().next();
-        d = explanation.getDecision();
-        fd = explanation.getFinalDecision();
-        storeCopy = explanation.getStoreCopy();
-        assertNotNull("should have a decision", d);
-        assertEquals(Decision.Type.NO, d.type());
-        assertEquals(ClusterAllocationExplanation.FinalDecision.ALREADY_ASSIGNED, fd);
-        assertEquals(ClusterAllocationExplanation.StoreCopy.AVAILABLE, storeCopy);
-        assertTrue(d.toString(), d.toString().contains("NO(the shard cannot be allocated on the same node id"));
-        assertTrue(d instanceof Decision.Multi);
-        md = (Decision.Multi) d;
-        ssd = md.getDecisions().get(0);
-        assertEquals(Decision.Type.NO, ssd.type());
-        assertTrue(ssd.toString(), ssd.toString().contains("NO(the shard cannot be allocated on the same node id"));
-        weight = explanation.getWeight();
-        assertNotNull("should have a weight", weight);
-
-        resp = client().admin().cluster().prepareAllocationExplain().useAnyUnassignedShard().get();
-        cae = resp.getExplanation();
-        assertNotNull("should always have an explanation", cae);
-        assertEquals("test", cae.getShard().getIndexName());
-        assertEquals(0, cae.getShard().getId());
-        assertEquals(false, cae.isPrimary());
-    }
-}
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java
index 577f73a89ed4a..7656ce5164ba4 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java
@@ -19,22 +19,20 @@
 
 package org.elasticsearch.action.admin.cluster.allocation;
 
-import org.apache.lucene.index.CorruptIndexException;
-import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
-import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresResponse;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.routing.RecoverySource.PeerRecoverySource;
-import org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
+import org.elasticsearch.cluster.routing.allocation.AllocateUnassignedDecision;
+import org.elasticsearch.cluster.routing.allocation.AllocationDecision;
+import org.elasticsearch.cluster.routing.allocation.MoveDecision;
+import org.elasticsearch.cluster.routing.allocation.ShardAllocationDecision;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.LocalTransportAddress;
-import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -42,12 +40,6 @@
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESTestCase;
 
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
 
@@ -56,123 +48,6 @@
  */
 public final class ClusterAllocationExplanationTests extends ESTestCase {
 
-    private Index i = new Index("foo", "uuid");
-    private ShardRouting primaryShard = ShardRouting.newUnassigned(new ShardId(i, 0), true, StoreRecoverySource.EMPTY_STORE_INSTANCE,
-        new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
-    private IndexMetaData indexMetaData = IndexMetaData.builder("foo")
-            .settings(Settings.builder()
-                    .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
-                    .put(IndexMetaData.SETTING_INDEX_UUID, "uuid"))
-            .putInSyncAllocationIds(0, Sets.newHashSet("aid1", "aid2"))
-            .numberOfShards(1)
-            .numberOfReplicas(1)
-            .build();
-    private DiscoveryNode node = new DiscoveryNode("node-0", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
-    private static Decision.Multi yesDecision = new Decision.Multi();
-    private static Decision.Multi noDecision = new Decision.Multi();
-
-    static {
-        yesDecision.add(Decision.single(Decision.Type.YES, "yes label", "yes please"));
-        noDecision.add(Decision.single(Decision.Type.NO, "no label", "no thanks"));
-    }
-
-    private void assertExplanations(NodeExplanation ne, String finalExplanation, ClusterAllocationExplanation.FinalDecision finalDecision,
-                                    ClusterAllocationExplanation.StoreCopy storeCopy) {
-        assertEquals(finalExplanation, ne.getFinalExplanation());
-        assertEquals(finalDecision, ne.getFinalDecision());
-        assertEquals(storeCopy, ne.getStoreCopy());
-    }
-
-    public void testDecisionAndExplanation() {
-        Exception e = new IOException("stuff's broke, yo");
-        Exception corruptE = new CorruptIndexException("stuff's corrupt, yo", "");
-        Float nodeWeight = randomFloat();
-        Set<String> activeAllocationIds = new HashSet<>();
-        activeAllocationIds.add("eggplant");
-        ShardRouting primaryStartedShard = ShardRouting.newUnassigned(new ShardId(i, 0), true, StoreRecoverySource.EXISTING_STORE_INSTANCE,
-                new UnassignedInfo(UnassignedInfo.Reason.INDEX_REOPENED, "foo"));
-        ShardRouting replicaStartedShard = ShardRouting.newUnassigned(new ShardId(i, 0), false, PeerRecoverySource.INSTANCE,
-                new UnassignedInfo(UnassignedInfo.Reason.INDEX_REOPENED, "foo"));
-
-        IndicesShardStoresResponse.StoreStatus storeStatus = new IndicesShardStoresResponse.StoreStatus(node, 42, "eggplant",
-                IndicesShardStoresResponse.StoreStatus.AllocationStatus.PRIMARY, e);
-        NodeExplanation ne = TransportClusterAllocationExplainAction.calculateNodeExplanation(primaryStartedShard, indexMetaData, node,
-                yesDecision, nodeWeight, storeStatus, "", activeAllocationIds, false);
-        assertExplanations(ne, "the copy of the shard cannot be read",
-                ClusterAllocationExplanation.FinalDecision.NO, ClusterAllocationExplanation.StoreCopy.IO_ERROR);
-
-        ne = TransportClusterAllocationExplainAction.calculateNodeExplanation(primaryShard, indexMetaData, node, yesDecision, nodeWeight,
-                null, "", activeAllocationIds, false);
-        assertExplanations(ne, "the shard can be assigned",
-                ClusterAllocationExplanation.FinalDecision.YES, ClusterAllocationExplanation.StoreCopy.NONE);
-
-        ne = TransportClusterAllocationExplainAction.calculateNodeExplanation(primaryStartedShard, indexMetaData, node, yesDecision,
-                nodeWeight, null, "", activeAllocationIds, false);
-        assertExplanations(ne, "there is no copy of the shard available",
-                ClusterAllocationExplanation.FinalDecision.NO, ClusterAllocationExplanation.StoreCopy.NONE);
-
-        ne = TransportClusterAllocationExplainAction.calculateNodeExplanation(primaryShard, indexMetaData, node, noDecision, nodeWeight,
-                null, "", activeAllocationIds, false);
-        assertExplanations(ne, "the shard cannot be assigned because one or more allocation decider returns a 'NO' decision",
-                ClusterAllocationExplanation.FinalDecision.NO, ClusterAllocationExplanation.StoreCopy.NONE);
-
-        storeStatus = new IndicesShardStoresResponse.StoreStatus(node, 42, "eggplant",
-                IndicesShardStoresResponse.StoreStatus.AllocationStatus.PRIMARY, null);
-        ne = TransportClusterAllocationExplainAction.calculateNodeExplanation(primaryShard, indexMetaData, node, noDecision, nodeWeight,
-                storeStatus, "", activeAllocationIds, false);
-        assertExplanations(ne, "the shard cannot be assigned because one or more allocation decider returns a 'NO' decision",
-                ClusterAllocationExplanation.FinalDecision.NO, ClusterAllocationExplanation.StoreCopy.AVAILABLE);
-
-        storeStatus = new IndicesShardStoresResponse.StoreStatus(node, 42, "eggplant",
-                IndicesShardStoresResponse.StoreStatus.AllocationStatus.PRIMARY, corruptE);
-        ne = TransportClusterAllocationExplainAction.calculateNodeExplanation(primaryStartedShard, indexMetaData, node, yesDecision,
-                nodeWeight, storeStatus, "", activeAllocationIds, false);
-        assertExplanations(ne, "the copy of the shard is corrupt",
-                ClusterAllocationExplanation.FinalDecision.NO, ClusterAllocationExplanation.StoreCopy.CORRUPT);
-
-        storeStatus = new IndicesShardStoresResponse.StoreStatus(node, 42, "banana",
-                IndicesShardStoresResponse.StoreStatus.AllocationStatus.PRIMARY, null);
-        ne = TransportClusterAllocationExplainAction.calculateNodeExplanation(primaryShard, indexMetaData, node, yesDecision, nodeWeight,
-                storeStatus, "", activeAllocationIds, false);
-        assertExplanations(ne, "the shard can be assigned",
-                ClusterAllocationExplanation.FinalDecision.YES, ClusterAllocationExplanation.StoreCopy.STALE);
-
-        storeStatus = new IndicesShardStoresResponse.StoreStatus(node, 42, "banana",
-                IndicesShardStoresResponse.StoreStatus.AllocationStatus.PRIMARY, null);
-        ne = TransportClusterAllocationExplainAction.calculateNodeExplanation(primaryStartedShard, indexMetaData, node, yesDecision,
-                nodeWeight, storeStatus, "", activeAllocationIds, false);
-        assertExplanations(ne, "the copy of the shard is stale, allocation ids do not match",
-                ClusterAllocationExplanation.FinalDecision.NO, ClusterAllocationExplanation.StoreCopy.STALE);
-
-        storeStatus = new IndicesShardStoresResponse.StoreStatus(node, 42, "eggplant",
-                IndicesShardStoresResponse.StoreStatus.AllocationStatus.PRIMARY, null);
-        ne = TransportClusterAllocationExplainAction.calculateNodeExplanation(primaryShard, indexMetaData, node, yesDecision, nodeWeight,
-                storeStatus, "node-0", activeAllocationIds, false);
-        assertExplanations(ne, "the shard is already assigned to this node",
-                ClusterAllocationExplanation.FinalDecision.ALREADY_ASSIGNED, ClusterAllocationExplanation.StoreCopy.AVAILABLE);
-
-        storeStatus = new IndicesShardStoresResponse.StoreStatus(node, 42, "eggplant",
-                IndicesShardStoresResponse.StoreStatus.AllocationStatus.PRIMARY, null);
-        ne = TransportClusterAllocationExplainAction.calculateNodeExplanation(primaryShard, indexMetaData, node, yesDecision, nodeWeight,
-                storeStatus, "", activeAllocationIds, false);
-        assertExplanations(ne, "the shard can be assigned and the node contains a valid copy of the shard data",
-                ClusterAllocationExplanation.FinalDecision.YES, ClusterAllocationExplanation.StoreCopy.AVAILABLE);
-
-        storeStatus = new IndicesShardStoresResponse.StoreStatus(node, 42, "eggplant",
-                IndicesShardStoresResponse.StoreStatus.AllocationStatus.PRIMARY, null);
-        ne = TransportClusterAllocationExplainAction.calculateNodeExplanation(primaryStartedShard, indexMetaData, node, yesDecision,
-                nodeWeight, storeStatus, "", activeAllocationIds, true);
-        assertExplanations(ne, "the shard's state is still being fetched so it cannot be allocated",
-                ClusterAllocationExplanation.FinalDecision.NO, ClusterAllocationExplanation.StoreCopy.AVAILABLE);
-
-        storeStatus = new IndicesShardStoresResponse.StoreStatus(node, 42, "eggplant",
-                IndicesShardStoresResponse.StoreStatus.AllocationStatus.REPLICA, null);
-        ne = TransportClusterAllocationExplainAction.calculateNodeExplanation(replicaStartedShard, indexMetaData, node, noDecision,
-                nodeWeight, storeStatus, "", activeAllocationIds, true);
-        assertExplanations(ne, "the shard cannot be assigned because allocation deciders return a NO decision",
-                ClusterAllocationExplanation.FinalDecision.NO, ClusterAllocationExplanation.StoreCopy.AVAILABLE);
-    }
-
     public void testDecisionEquality() {
         Decision.Multi d = new Decision.Multi();
         Decision.Multi d2 = new Decision.Multi();
@@ -186,67 +61,53 @@ public void testDecisionEquality() {
     }
 
     public void testExplanationSerialization() throws Exception {
-        ShardId shard = new ShardId("test", "uuid", 0);
-        long allocationDelay = randomIntBetween(0, 500);
-        long remainingDelay = randomIntBetween(0, 500);
-        Map<DiscoveryNode, NodeExplanation> nodeExplanations = new HashMap<>(1);
-        Float nodeWeight = randomFloat();
-        Set<String> activeAllocationIds = new HashSet<>();
-        activeAllocationIds.add("eggplant");
-
-        IndicesShardStoresResponse.StoreStatus storeStatus = new IndicesShardStoresResponse.StoreStatus(node, 42, "eggplant",
-                IndicesShardStoresResponse.StoreStatus.AllocationStatus.PRIMARY, null);
-        NodeExplanation ne = TransportClusterAllocationExplainAction.calculateNodeExplanation(primaryShard, indexMetaData, node,
-                yesDecision, nodeWeight, storeStatus, "", activeAllocationIds, false);
-        nodeExplanations.put(ne.getNode(), ne);
-        ClusterAllocationExplanation cae = new ClusterAllocationExplanation(shard, true,
-                "assignedNode", allocationDelay, remainingDelay, null, false, nodeExplanations, null);
+        ClusterAllocationExplanation cae = randomClusterAllocationExplanation(randomBoolean());
         BytesStreamOutput out = new BytesStreamOutput();
         cae.writeTo(out);
         StreamInput in = out.bytes().streamInput();
         ClusterAllocationExplanation cae2 = new ClusterAllocationExplanation(in);
-        assertEquals(shard, cae2.getShard());
+        assertEquals(cae.getShard(), cae2.getShard());
+        assertEquals(cae.isPrimary(), cae2.isPrimary());
         assertTrue(cae2.isPrimary());
-        assertTrue(cae2.isAssigned());
-        assertEquals("assignedNode", cae2.getAssignedNodeId());
-        assertNull(cae2.getUnassignedInfo());
-        assertEquals(allocationDelay, cae2.getAllocationDelayMillis());
-        assertEquals(remainingDelay, cae2.getRemainingDelayMillis());
-        for (Map.Entry<DiscoveryNode, NodeExplanation> entry : cae2.getNodeExplanations().entrySet()) {
-            NodeExplanation explanation = entry.getValue();
-            assertNotNull(explanation.getStoreStatus());
-            assertNotNull(explanation.getDecision());
-            assertEquals(nodeWeight, explanation.getWeight());
+        assertEquals(cae.getUnassignedInfo(), cae2.getUnassignedInfo());
+        assertEquals(cae.getCurrentNode(), cae2.getCurrentNode());
+        assertEquals(cae.getShardState(), cae2.getShardState());
+        if (cae.getClusterInfo() == null) {
+            assertNull(cae2.getClusterInfo());
+        } else {
+            assertNotNull(cae2.getClusterInfo());
+            assertEquals(cae.getClusterInfo().getNodeMostAvailableDiskUsages().size(),
+                cae2.getClusterInfo().getNodeMostAvailableDiskUsages().size());
         }
+        assertEquals(cae.getShardAllocationDecision().getAllocateDecision(), cae2.getShardAllocationDecision().getAllocateDecision());
+        assertEquals(cae.getShardAllocationDecision().getMoveDecision(), cae2.getShardAllocationDecision().getMoveDecision());
     }
 
     public void testExplanationToXContent() throws Exception {
-        ShardId shardId = new ShardId("foo", "uuid", 0);
-        Decision.Multi d = new Decision.Multi();
-        d.add(Decision.single(Decision.Type.NO, "no label", "because I said no"));
-        d.add(Decision.single(Decision.Type.YES, "yes label", "yes please"));
-        d.add(Decision.single(Decision.Type.THROTTLE, "throttle label", "wait a sec"));
-        Float nodeWeight = 1.5f;
-        Set<String> allocationIds = new HashSet<>();
-        allocationIds.add("bar");
-        IndicesShardStoresResponse.StoreStatus storeStatus = new IndicesShardStoresResponse.StoreStatus(node, 42, "eggplant",
-                IndicesShardStoresResponse.StoreStatus.AllocationStatus.PRIMARY, new ElasticsearchException("stuff's broke, yo"));
-        NodeExplanation ne = TransportClusterAllocationExplainAction.calculateNodeExplanation(primaryShard, indexMetaData, node,
-                d, nodeWeight, storeStatus, "node-0", allocationIds, false);
-        Map<DiscoveryNode, NodeExplanation> nodeExplanations = new HashMap<>(1);
-        nodeExplanations.put(ne.getNode(), ne);
-        ClusterAllocationExplanation cae = new ClusterAllocationExplanation(shardId, true,
-                "assignedNode", 42, 42, null, false, nodeExplanations, null);
+        ClusterAllocationExplanation cae = randomClusterAllocationExplanation(true);
         XContentBuilder builder = XContentFactory.jsonBuilder();
         cae.toXContent(builder, ToXContent.EMPTY_PARAMS);
-        assertEquals("{\"shard\":{\"index\":\"foo\",\"index_uuid\":\"uuid\",\"id\":0,\"primary\":true},\"assigned\":true," +
-                        "\"assigned_node_id\":\"assignedNode\",\"shard_state_fetch_pending\":false,\"nodes\":{\"node-0\":" +
-                        "{\"node_name\":\"\",\"node_attributes\":{},\"store\":{\"shard_copy\":\"IO_ERROR\",\"store_except" +
-                        "ion\":\"ElasticsearchException[stuff's broke, yo]\"},\"final_decision\":\"ALREADY_ASSIGNED\",\"f" +
-                        "inal_explanation\":\"the shard is already assigned to this node\",\"weight\":1.5,\"decisions\":[" +
-                        "{\"decider\":\"no label\",\"decision\":\"NO\",\"explanation\":\"because I said no\"},{\"decider" +
-                        "\":\"yes label\",\"decision\":\"YES\",\"explanation\":\"yes please\"},{\"decider\":\"throttle la" +
-                        "bel\",\"decision\":\"THROTTLE\",\"explanation\":\"wait a sec\"}]}}}",
-                builder.string());
+        assertEquals("{\"index\":\"idx\",\"shard\":0,\"primary\":true,\"current_state\":\"started\",\"current_node\":" +
+                         "{\"id\":\"node-0\",\"name\":\"\",\"transport_address\":\"" + cae.getCurrentNode().getAddress() +
+                         "\",\"weight_ranking\":3},\"can_remain_on_current_node\":\"yes\",\"can_rebalance_cluster\":\"yes\"," +
+                         "\"can_rebalance_to_other_node\":\"no\",\"rebalance_explanation\":\"cannot rebalance as no target node exists " +
+                         "that can both allocate this shard and improve the cluster balance\"}", builder.string());
+    }
+
+    private static ClusterAllocationExplanation randomClusterAllocationExplanation(boolean assignedShard) {
+        ShardRouting shardRouting = TestShardRouting.newShardRouting(new ShardId(new Index("idx", "123"), 0),
+            assignedShard ? "node-0" : null, true, assignedShard ? ShardRoutingState.STARTED : ShardRoutingState.UNASSIGNED);
+        DiscoveryNode node = assignedShard ? new DiscoveryNode("node-0", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(),
+                                                                  Version.CURRENT) : null;
+        ShardAllocationDecision shardAllocationDecision;
+        if (assignedShard) {
+            MoveDecision moveDecision = MoveDecision.cannotRebalance(Decision.YES, AllocationDecision.NO, 3, null)
+                                            .withRemainDecision(Decision.YES);
+            shardAllocationDecision = new ShardAllocationDecision(AllocateUnassignedDecision.NOT_TAKEN, moveDecision);
+        } else {
+            AllocateUnassignedDecision allocateDecision = AllocateUnassignedDecision.no(UnassignedInfo.AllocationStatus.DECIDERS_NO, null);
+            shardAllocationDecision = new ShardAllocationDecision(allocateDecision, MoveDecision.NOT_TAKEN);
+        }
+        return new ClusterAllocationExplanation(shardRouting, node, null, null, shardAllocationDecision);
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java
index 04905de18c278..92023a5561ebf 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java
@@ -24,7 +24,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.discovery.DiscoveryStats;
-import org.elasticsearch.discovery.zen.publish.PendingClusterStateStats;
+import org.elasticsearch.discovery.zen.PendingClusterStateStats;
 import org.elasticsearch.http.HttpStats;
 import org.elasticsearch.indices.breaker.AllCircuitBreakerStats;
 import org.elasticsearch.indices.breaker.CircuitBreakerStats;
@@ -73,6 +73,30 @@ public void testSerialization() throws IOException {
                     assertEquals(nodeStats.getOs().getMem().getFreePercent(), deserializedNodeStats.getOs().getMem().getFreePercent());
                     assertEquals(nodeStats.getOs().getMem().getUsedPercent(), deserializedNodeStats.getOs().getMem().getUsedPercent());
                     assertEquals(nodeStats.getOs().getCpu().getPercent(), deserializedNodeStats.getOs().getCpu().getPercent());
+                    assertEquals(
+                        nodeStats.getOs().getCgroup().getCpuAcctControlGroup(),
+                        deserializedNodeStats.getOs().getCgroup().getCpuAcctControlGroup());
+                    assertEquals(
+                        nodeStats.getOs().getCgroup().getCpuAcctUsageNanos(),
+                        deserializedNodeStats.getOs().getCgroup().getCpuAcctUsageNanos());
+                    assertEquals(
+                        nodeStats.getOs().getCgroup().getCpuControlGroup(),
+                        deserializedNodeStats.getOs().getCgroup().getCpuControlGroup());
+                    assertEquals(
+                        nodeStats.getOs().getCgroup().getCpuCfsPeriodMicros(),
+                        deserializedNodeStats.getOs().getCgroup().getCpuCfsPeriodMicros());
+                    assertEquals(
+                        nodeStats.getOs().getCgroup().getCpuCfsQuotaMicros(),
+                        deserializedNodeStats.getOs().getCgroup().getCpuCfsQuotaMicros());
+                    assertEquals(
+                        nodeStats.getOs().getCgroup().getCpuStat().getNumberOfElapsedPeriods(),
+                        deserializedNodeStats.getOs().getCgroup().getCpuStat().getNumberOfElapsedPeriods());
+                    assertEquals(
+                        nodeStats.getOs().getCgroup().getCpuStat().getNumberOfTimesThrottled(),
+                        deserializedNodeStats.getOs().getCgroup().getCpuStat().getNumberOfTimesThrottled());
+                    assertEquals(
+                        nodeStats.getOs().getCgroup().getCpuStat().getTimeThrottledNanos(),
+                        deserializedNodeStats.getOs().getCgroup().getCpuStat().getTimeThrottledNanos());
                     assertArrayEquals(nodeStats.getOs().getCpu().getLoadAverage(),
                             deserializedNodeStats.getOs().getCpu().getLoadAverage(), 0);
                 }
@@ -265,36 +289,43 @@ private static NodeStats createNodeStats() {
             }
             osStats = new OsStats(System.currentTimeMillis(), new OsStats.Cpu(randomShort(), loadAverages),
                     new OsStats.Mem(randomLong(), randomLong()),
-                    new OsStats.Swap(randomLong(), randomLong()));
+                    new OsStats.Swap(randomLong(), randomLong()),
+                    new OsStats.Cgroup(
+                        randomAlphaOfLength(8),
+                        randomNonNegativeLong(),
+                        randomAlphaOfLength(8),
+                        randomNonNegativeLong(),
+                        randomNonNegativeLong(),
+                        new OsStats.Cgroup.CpuStat(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong())));
         }
-        ProcessStats processStats = frequently() ? new ProcessStats(randomPositiveLong(), randomPositiveLong(), randomPositiveLong(),
-                new ProcessStats.Cpu(randomShort(), randomPositiveLong()),
-                new ProcessStats.Mem(randomPositiveLong())) : null;
+        ProcessStats processStats = frequently() ? new ProcessStats(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
+                new ProcessStats.Cpu(randomShort(), randomNonNegativeLong()),
+                new ProcessStats.Mem(randomNonNegativeLong())) : null;
         JvmStats jvmStats = null;
         if (frequently()) {
             int numMemoryPools = randomIntBetween(0, 10);
             List<JvmStats.MemoryPool> memoryPools = new ArrayList<>(numMemoryPools);
             for (int i = 0; i < numMemoryPools; i++) {
-                memoryPools.add(new JvmStats.MemoryPool(randomAsciiOfLengthBetween(3, 10), randomPositiveLong(),
-                        randomPositiveLong(), randomPositiveLong(), randomPositiveLong()));
+                memoryPools.add(new JvmStats.MemoryPool(randomAlphaOfLengthBetween(3, 10), randomNonNegativeLong(),
+                        randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()));
             }
             JvmStats.Threads threads = new JvmStats.Threads(randomIntBetween(1, 1000), randomIntBetween(1, 1000));
             int numGarbageCollectors = randomIntBetween(0, 10);
             JvmStats.GarbageCollector[] garbageCollectorsArray = new JvmStats.GarbageCollector[numGarbageCollectors];
             for (int i = 0; i < numGarbageCollectors; i++) {
-                garbageCollectorsArray[i] = new JvmStats.GarbageCollector(randomAsciiOfLengthBetween(3, 10),
-                        randomPositiveLong(), randomPositiveLong());
+                garbageCollectorsArray[i] = new JvmStats.GarbageCollector(randomAlphaOfLengthBetween(3, 10),
+                        randomNonNegativeLong(), randomNonNegativeLong());
             }
             JvmStats.GarbageCollectors garbageCollectors = new JvmStats.GarbageCollectors(garbageCollectorsArray);
             int numBufferPools = randomIntBetween(0, 10);
             List<JvmStats.BufferPool> bufferPoolList = new ArrayList<>();
             for (int i = 0; i < numBufferPools; i++) {
-                bufferPoolList.add(new JvmStats.BufferPool(randomAsciiOfLengthBetween(3, 10), randomPositiveLong(), randomPositiveLong(),
-                        randomPositiveLong()));
+                bufferPoolList.add(new JvmStats.BufferPool(randomAlphaOfLengthBetween(3, 10), randomNonNegativeLong(), randomNonNegativeLong(),
+                        randomNonNegativeLong()));
             }
-            JvmStats.Classes classes = new JvmStats.Classes(randomPositiveLong(), randomPositiveLong(), randomPositiveLong());
-            jvmStats = frequently() ? new JvmStats(randomPositiveLong(), randomPositiveLong(), new JvmStats.Mem(randomPositiveLong(),
-                    randomPositiveLong(), randomPositiveLong(), randomPositiveLong(), randomPositiveLong(), memoryPools), threads,
+            JvmStats.Classes classes = new JvmStats.Classes(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong());
+            jvmStats = frequently() ? new JvmStats(randomNonNegativeLong(), randomNonNegativeLong(), new JvmStats.Mem(randomNonNegativeLong(),
+                    randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), memoryPools), threads,
                     garbageCollectors, randomBoolean() ? Collections.emptyList() : bufferPoolList, classes) : null;
         }
         ThreadPoolStats threadPoolStats = null;
@@ -302,8 +333,8 @@ private static NodeStats createNodeStats() {
             int numThreadPoolStats = randomIntBetween(0, 10);
             List<ThreadPoolStats.Stats> threadPoolStatsList = new ArrayList<>();
             for (int i = 0; i < numThreadPoolStats; i++) {
-                threadPoolStatsList.add(new ThreadPoolStats.Stats(randomAsciiOfLengthBetween(3, 10), randomIntBetween(1, 1000),
-                        randomIntBetween(1, 1000), randomIntBetween(1, 1000), randomPositiveLong(),
+                threadPoolStatsList.add(new ThreadPoolStats.Stats(randomAlphaOfLengthBetween(3, 10), randomIntBetween(1, 1000),
+                        randomIntBetween(1, 1000), randomIntBetween(1, 1000), randomNonNegativeLong(),
                         randomIntBetween(1, 1000), randomIntBetween(1, 1000)));
             }
             threadPoolStats = new ThreadPoolStats(threadPoolStatsList);
@@ -314,51 +345,51 @@ private static NodeStats createNodeStats() {
             FsInfo.DeviceStats[] deviceStatsArray = new FsInfo.DeviceStats[numDeviceStats];
             for (int i = 0; i < numDeviceStats; i++) {
                 FsInfo.DeviceStats previousDeviceStats = randomBoolean() ? null :
-                        new FsInfo.DeviceStats(randomInt(), randomInt(), randomAsciiOfLengthBetween(3, 10),
-                                randomPositiveLong(), randomPositiveLong(), randomPositiveLong(), randomPositiveLong(), null);
-                deviceStatsArray[i] = new FsInfo.DeviceStats(randomInt(), randomInt(), randomAsciiOfLengthBetween(3, 10),
-                        randomPositiveLong(), randomPositiveLong(), randomPositiveLong(), randomPositiveLong(), previousDeviceStats);
+                        new FsInfo.DeviceStats(randomInt(), randomInt(), randomAlphaOfLengthBetween(3, 10),
+                                randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), null);
+                deviceStatsArray[i] = new FsInfo.DeviceStats(randomInt(), randomInt(), randomAlphaOfLengthBetween(3, 10),
+                        randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), previousDeviceStats);
             }
             FsInfo.IoStats ioStats = new FsInfo.IoStats(deviceStatsArray);
             int numPaths = randomIntBetween(0, 10);
             FsInfo.Path[] paths = new FsInfo.Path[numPaths];
             for (int i = 0; i < numPaths; i++) {
-                paths[i] = new FsInfo.Path(randomAsciiOfLengthBetween(3, 10), randomBoolean() ? randomAsciiOfLengthBetween(3, 10) : null,
-                        randomPositiveLong(), randomPositiveLong(), randomPositiveLong());
+                paths[i] = new FsInfo.Path(randomAlphaOfLengthBetween(3, 10), randomBoolean() ? randomAlphaOfLengthBetween(3, 10) : null,
+                        randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong());
             }
-            fsInfo = new FsInfo(randomPositiveLong(), ioStats, paths);
+            fsInfo = new FsInfo(randomNonNegativeLong(), ioStats, paths);
         }
-        TransportStats transportStats = frequently() ? new TransportStats(randomPositiveLong(), randomPositiveLong(),
-                randomPositiveLong(), randomPositiveLong(), randomPositiveLong()) : null;
-        HttpStats httpStats = frequently() ? new HttpStats(randomPositiveLong(), randomPositiveLong()) : null;
+        TransportStats transportStats = frequently() ? new TransportStats(randomNonNegativeLong(), randomNonNegativeLong(),
+                randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()) : null;
+        HttpStats httpStats = frequently() ? new HttpStats(randomNonNegativeLong(), randomNonNegativeLong()) : null;
         AllCircuitBreakerStats allCircuitBreakerStats = null;
         if (frequently()) {
             int numCircuitBreakerStats = randomIntBetween(0, 10);
             CircuitBreakerStats[] circuitBreakerStatsArray = new CircuitBreakerStats[numCircuitBreakerStats];
             for (int i = 0; i < numCircuitBreakerStats; i++) {
-                circuitBreakerStatsArray[i] = new CircuitBreakerStats(randomAsciiOfLengthBetween(3, 10), randomPositiveLong(),
-                        randomPositiveLong(), randomDouble(), randomPositiveLong());
+                circuitBreakerStatsArray[i] = new CircuitBreakerStats(randomAlphaOfLengthBetween(3, 10), randomNonNegativeLong(),
+                        randomNonNegativeLong(), randomDouble(), randomNonNegativeLong());
             }
             allCircuitBreakerStats = new AllCircuitBreakerStats(circuitBreakerStatsArray);
         }
-        ScriptStats scriptStats = frequently() ? new ScriptStats(randomPositiveLong(), randomPositiveLong()) : null;
+        ScriptStats scriptStats = frequently() ? new ScriptStats(randomNonNegativeLong(), randomNonNegativeLong()) : null;
         DiscoveryStats discoveryStats = frequently() ? new DiscoveryStats(randomBoolean() ? new PendingClusterStateStats(randomInt(),
                 randomInt(), randomInt()) : null) : null;
         IngestStats ingestStats = null;
         if (frequently()) {
-            IngestStats.Stats totalStats = new IngestStats.Stats(randomPositiveLong(), randomPositiveLong(), randomPositiveLong(),
-                    randomPositiveLong());
+            IngestStats.Stats totalStats = new IngestStats.Stats(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
+                    randomNonNegativeLong());
 
             int numStatsPerPipeline = randomIntBetween(0, 10);
             Map<String, IngestStats.Stats> statsPerPipeline = new HashMap<>();
             for (int i = 0; i < numStatsPerPipeline; i++) {
-                statsPerPipeline.put(randomAsciiOfLengthBetween(3, 10), new IngestStats.Stats(randomPositiveLong(),
-                        randomPositiveLong(), randomPositiveLong(), randomPositiveLong()));
+                statsPerPipeline.put(randomAlphaOfLengthBetween(3, 10), new IngestStats.Stats(randomNonNegativeLong(),
+                        randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()));
             }
             ingestStats = new IngestStats(totalStats, statsPerPipeline);
         }
         //TODO NodeIndicesStats are not tested here, way too complicated to create, also they need to be migrated to Writeable yet
-        return new NodeStats(node, randomPositiveLong(), null, osStats, processStats, jvmStats, threadPoolStats, fsInfo,
+        return new NodeStats(node, randomNonNegativeLong(), null, osStats, processStats, jvmStats, threadPoolStats, fsInfo,
                 transportStats, httpStats, allCircuitBreakerStats, scriptStats, discoveryStats, ingestStats);
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java
index 9027b3d372e37..c28fddf68ad72 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java
@@ -19,7 +19,7 @@
 package org.elasticsearch.action.admin.cluster.node.tasks;
 
 import com.carrotsearch.randomizedtesting.RandomizedContext;
-import com.carrotsearch.randomizedtesting.generators.RandomInts;
+import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest;
 import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse;
@@ -35,6 +35,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskCancelledException;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.tasks.TaskInfo;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -50,6 +51,7 @@
 import java.util.concurrent.atomic.AtomicReference;
 
 import static org.elasticsearch.test.ClusterServiceUtils.setState;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 
@@ -90,7 +92,12 @@ public String getDescription() {
 
         @Override
         public Task createTask(long id, String type, String action, TaskId parentTaskId) {
-            return new CancellableTask(id, type, action, getDescription(), parentTaskId);
+            return new CancellableTask(id, type, action, getDescription(), parentTaskId) {
+                @Override
+                public boolean shouldCancelChildrenOnCancellation() {
+                    return false;
+                }
+            };
         }
     }
 
@@ -125,7 +132,12 @@ public String getDescription() {
 
         @Override
         public Task createTask(long id, String type, String action, TaskId parentTaskId) {
-            return new CancellableTask(id, type, action, getDescription(), parentTaskId);
+            return new CancellableTask(id, type, action, getDescription(), parentTaskId) {
+                @Override
+                public boolean shouldCancelChildrenOnCancellation() {
+                    return true;
+                }
+            };
         }
     }
 
@@ -168,7 +180,7 @@ protected NodeResponse nodeOperation(CancellableNodeRequest request, Task task)
                 try {
                     awaitBusy(() -> {
                         if (((CancellableTask) task).isCancelled()) {
-                            throw new RuntimeException("Cancelled");
+                            throw new TaskCancelledException("Cancelled");
                         }
                         return false;
                     });
@@ -200,7 +212,7 @@ private Task startCancellableTestNodesAction(boolean waitForActionToStart, Colle
         CancellableTestNodesAction[] actions = new CancellableTestNodesAction[nodesCount];
         for (int i = 0; i < testNodes.length; i++) {
             boolean shouldBlock = blockOnNodes.contains(testNodes[i]);
-            logger.info("The action in the node [{}] should block: [{}]", testNodes[i].discoveryNode.getId(), shouldBlock);
+            logger.info("The action in the node [{}] should block: [{}]", testNodes[i].getNodeId(), shouldBlock);
             actions[i] = new CancellableTestNodesAction(CLUSTER_SETTINGS, "testAction", threadPool, testNodes[i]
                 .clusterService, testNodes[i].transportService, shouldBlock, actionLatch);
         }
@@ -239,7 +251,7 @@ public void onFailure(Exception e) {
         // Cancel main task
         CancelTasksRequest request = new CancelTasksRequest();
         request.setReason("Testing Cancellation");
-        request.setTaskId(new TaskId(testNodes[0].discoveryNode.getId(), mainTask.getId()));
+        request.setTaskId(new TaskId(testNodes[0].getNodeId(), mainTask.getId()));
         // And send the cancellation request to a random node
         CancelTasksResponse response = testNodes[randomIntBetween(0, testNodes.length - 1)].transportCancelTasksAction.execute(request)
             .get();
@@ -276,7 +288,7 @@ public void onFailure(Exception e) {
         // Make sure that tasks are no longer running
         ListTasksResponse listTasksResponse = testNodes[randomIntBetween(0, testNodes.length - 1)]
             .transportListTasksAction.execute(new ListTasksRequest().setTaskId(
-                new TaskId(testNodes[0].discoveryNode.getId(), mainTask.getId()))).get();
+                new TaskId(testNodes[0].getNodeId(), mainTask.getId()))).get();
         assertEquals(0, listTasksResponse.getTasks().size());
 
         // Make sure that there are no leftover bans, the ban removal is async, so we might return from the cancellation
@@ -288,6 +300,54 @@ public void onFailure(Exception e) {
         });
     }
 
+    public void testChildTasksCancellation() throws Exception {
+        setupTestNodes(Settings.EMPTY);
+        connectNodes(testNodes);
+        CountDownLatch responseLatch = new CountDownLatch(1);
+        final AtomicReference<NodesResponse> responseReference = new AtomicReference<>();
+        final AtomicReference<Throwable> throwableReference = new AtomicReference<>();
+        Task mainTask = startCancellableTestNodesAction(true, nodesCount, new ActionListener<NodesResponse>() {
+            @Override
+            public void onResponse(NodesResponse listTasksResponse) {
+                responseReference.set(listTasksResponse);
+                responseLatch.countDown();
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                throwableReference.set(e);
+                responseLatch.countDown();
+            }
+        });
+
+        // Cancel all child tasks without cancelling the main task, which should quit on its own
+        CancelTasksRequest request = new CancelTasksRequest();
+        request.setReason("Testing Cancellation");
+        request.setParentTaskId(new TaskId(testNodes[0].getNodeId(), mainTask.getId()));
+        // And send the cancellation request to a random node
+        CancelTasksResponse response = testNodes[randomIntBetween(1, testNodes.length - 1)].transportCancelTasksAction.execute(request)
+            .get();
+
+        // Awaiting for the main task to finish
+        responseLatch.await();
+
+        // Should have cancelled tasks on all nodes
+        assertThat(response.getTasks().size(), equalTo(testNodes.length));
+
+        assertBusy(() -> {
+            try {
+            // Make sure that main task is no longer running
+                ListTasksResponse listTasksResponse = testNodes[randomIntBetween(0, testNodes.length - 1)]
+                    .transportListTasksAction.execute(new ListTasksRequest().setTaskId(
+                        new TaskId(testNodes[0].getNodeId(), mainTask.getId()))).get();
+                assertEquals(0, listTasksResponse.getTasks().size());
+
+            } catch (ExecutionException | InterruptedException ex) {
+                throw new RuntimeException(ex);
+            }
+        });
+    }
+
     public void testTaskCancellationOnCoordinatingNodeLeavingTheCluster() throws Exception {
         setupTestNodes(Settings.EMPTY);
         connectNodes(testNodes);
@@ -314,7 +374,7 @@ public void onFailure(Exception e) {
             }
         });
 
-        String mainNode = testNodes[0].discoveryNode.getId();
+        String mainNode = testNodes[0].getNodeId();
 
         // Make sure that tasks are running
         ListTasksResponse listTasksResponse = testNodes[randomIntBetween(0, testNodes.length - 1)]
@@ -324,12 +384,12 @@ public void onFailure(Exception e) {
         // Simulate the coordinating node leaving the cluster
         DiscoveryNode[] discoveryNodes = new DiscoveryNode[testNodes.length - 1];
         for (int i = 1; i < testNodes.length; i++) {
-            discoveryNodes[i - 1] = testNodes[i].discoveryNode;
+            discoveryNodes[i - 1] = testNodes[i].discoveryNode();
         }
         DiscoveryNode master = discoveryNodes[0];
         for (int i = 1; i < testNodes.length; i++) {
             // Notify only nodes that should remain in the cluster
-            setState(testNodes[i].clusterService, ClusterStateCreationUtils.state(testNodes[i].discoveryNode, master, discoveryNodes));
+            setState(testNodes[i].clusterService, ClusterStateCreationUtils.state(testNodes[i].discoveryNode(), master, discoveryNodes));
         }
 
         if (simulateBanBeforeLeaving) {
@@ -337,7 +397,7 @@ public void onFailure(Exception e) {
             // Simulate issuing cancel request on the node that is about to leave the cluster
             CancelTasksRequest request = new CancelTasksRequest();
             request.setReason("Testing Cancellation");
-            request.setTaskId(new TaskId(testNodes[0].discoveryNode.getId(), mainTask.getId()));
+            request.setTaskId(new TaskId(testNodes[0].getNodeId(), mainTask.getId()));
             // And send the cancellation request to a random node
             CancelTasksResponse response = testNodes[0].transportCancelTasksAction.execute(request).get();
             logger.info("--> Done simulating issuing cancel request on the node that is about to leave the cluster");
@@ -379,9 +439,9 @@ private static void debugDelay(String nodeId, String name) {
         // Introduce an additional pseudo random repeatable race conditions
         String delayName = RandomizedContext.current().getRunnerSeedAsString() + ":" + nodeId + ":" + name;
         Random random = new Random(delayName.hashCode());
-        if (RandomInts.randomIntBetween(random, 0, 10) < 1) {
+        if (RandomNumbers.randomIntBetween(random, 0, 10) < 1) {
             try {
-                Thread.sleep(RandomInts.randomIntBetween(random, 20, 50));
+                Thread.sleep(RandomNumbers.randomIntBetween(random, 20, 50));
             } catch (InterruptedException ex) {
                 Thread.currentThread().interrupt();
             }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/RecordingTaskManagerListener.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/RecordingTaskManagerListener.java
index 656a5ab9ec46b..5046a48b938f6 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/RecordingTaskManagerListener.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/RecordingTaskManagerListener.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.action.admin.cluster.node.tasks;
 
-import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.tasks.Task;
@@ -37,26 +36,26 @@
 public class RecordingTaskManagerListener implements MockTaskManagerListener {
 
     private String[] actionMasks;
-    private DiscoveryNode localNode;
+    private String localNodeId;
 
     private List<Tuple<Boolean, TaskInfo>> events  = new ArrayList<>();
 
-    public RecordingTaskManagerListener(DiscoveryNode localNode, String... actionMasks) {
+    public RecordingTaskManagerListener(String localNodeId, String... actionMasks) {
         this.actionMasks = actionMasks;
-        this.localNode = localNode;
+        this.localNodeId = localNodeId;
     }
 
     @Override
     public synchronized void onTaskRegistered(Task task) {
         if (Regex.simpleMatch(actionMasks, task.getAction())) {
-            events.add(new Tuple<>(true, task.taskInfo(localNode, true)));
+            events.add(new Tuple<>(true, task.taskInfo(localNodeId, true)));
         }
     }
 
     @Override
     public synchronized void onTaskUnregistered(Task task) {
         if (Regex.simpleMatch(actionMasks, task.getAction())) {
-            events.add(new Tuple<>(false, task.taskInfo(localNode, true)));
+            events.add(new Tuple<>(false, task.taskInfo(localNodeId, true)));
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java
index 8bb32c240f217..0b06f9eb54274 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java
@@ -18,13 +18,7 @@
  */
 package org.elasticsearch.action.admin.cluster.node.tasks;
 
-import java.io.IOException;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-import java.util.function.Supplier;
-
+import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.admin.cluster.node.tasks.cancel.TransportCancelTasksAction;
@@ -36,6 +30,7 @@
 import org.elasticsearch.action.support.nodes.BaseNodesResponse;
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
 import org.elasticsearch.action.support.replication.ClusterStateCreationUtils;
+import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -45,6 +40,8 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.BoundTransportAddress;
+import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.tasks.TaskManager;
 import org.elasticsearch.test.ESTestCase;
@@ -57,6 +54,14 @@
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
+import java.util.function.Supplier;
+
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
 import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
@@ -158,18 +163,19 @@ protected NodeResponse newNodeResponse() {
         @Override
         protected abstract NodeResponse nodeOperation(NodeRequest request);
 
-        @Override
-        protected boolean accumulateExceptions() {
-            return true;
-        }
     }
 
     public static class TestNode implements Releasable {
         public TestNode(String name, ThreadPool threadPool, Settings settings) {
-            clusterService = createClusterService(threadPool);
+            final Function<BoundTransportAddress, DiscoveryNode> boundTransportAddressDiscoveryNodeFunction =
+                address -> {
+                 discoveryNode.set(new DiscoveryNode(name, address.publishAddress(), emptyMap(), emptySet(), Version.CURRENT));
+                 return discoveryNode.get();
+                };
             transportService = new TransportService(settings,
                     new LocalTransport(settings, threadPool, new NamedWriteableRegistry(Collections.emptyList()),
-                        new NoneCircuitBreakerService()), threadPool) {
+                        new NoneCircuitBreakerService()), threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR,
+                boundTransportAddressDiscoveryNodeFunction, null) {
                 @Override
                 protected TaskManager createTaskManager() {
                     if (MockTaskManager.USE_MOCK_TASK_MANAGER_SETTING.get(settings)) {
@@ -180,9 +186,8 @@ protected TaskManager createTaskManager() {
                 }
             };
             transportService.start();
-            clusterService.add(transportService.getTaskManager());
-            discoveryNode = new DiscoveryNode(name, transportService.boundAddress().publishAddress(),
-                    emptyMap(), emptySet(), Version.CURRENT);
+            clusterService = createClusterService(threadPool, discoveryNode.get());
+            clusterService.addStateApplier(transportService.getTaskManager());
             IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(settings);
             ActionFilters actionFilters = new ActionFilters(emptySet());
             transportListTasksAction = new TransportListTasksAction(settings, threadPool, clusterService, transportService,
@@ -194,7 +199,7 @@ protected TaskManager createTaskManager() {
 
         public final ClusterService clusterService;
         public final TransportService transportService;
-        public final DiscoveryNode discoveryNode;
+        private final SetOnce<DiscoveryNode> discoveryNode = new SetOnce<>();
         public final TransportListTasksAction transportListTasksAction;
         public final TransportCancelTasksAction transportCancelTasksAction;
 
@@ -205,22 +210,24 @@ public void close() {
         }
 
         public String getNodeId() {
-            return discoveryNode.getId();
+            return discoveryNode().getId();
         }
+
+        public DiscoveryNode discoveryNode() { return  discoveryNode.get(); }
     }
 
     public static void connectNodes(TestNode... nodes) {
         DiscoveryNode[] discoveryNodes = new DiscoveryNode[nodes.length];
         for (int i = 0; i < nodes.length; i++) {
-            discoveryNodes[i] = nodes[i].discoveryNode;
+            discoveryNodes[i] = nodes[i].discoveryNode();
         }
         DiscoveryNode master = discoveryNodes[0];
         for (TestNode node : nodes) {
-            setState(node.clusterService, ClusterStateCreationUtils.state(node.discoveryNode, master, discoveryNodes));
+            setState(node.clusterService, ClusterStateCreationUtils.state(node.discoveryNode(), master, discoveryNodes));
         }
         for (TestNode nodeA : nodes) {
             for (TestNode nodeB : nodes) {
-                nodeA.transportService.connectToNode(nodeB.discoveryNode);
+                nodeA.transportService.connectToNode(nodeB.discoveryNode());
             }
         }
     }
@@ -228,7 +235,7 @@ public static void connectNodes(TestNode... nodes) {
     public static RecordingTaskManagerListener[] setupListeners(TestNode[] nodes, String... actionMasks) {
         RecordingTaskManagerListener[] listeners = new RecordingTaskManagerListener[nodes.length];
         for (int i = 0; i < nodes.length; i++) {
-            listeners[i] = new RecordingTaskManagerListener(nodes[i].discoveryNode, actionMasks);
+            listeners[i] = new RecordingTaskManagerListener(nodes[i].getNodeId(), actionMasks);
             ((MockTaskManager) (nodes[i].transportService.getTaskManager())).addListener(listeners[i]);
         }
         return listeners;
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskTests.java
new file mode 100644
index 0000000000000..c5d8b39c3da39
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskTests.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.admin.cluster.node.tasks;
+
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.tasks.TaskId;
+import org.elasticsearch.tasks.TaskInfo;
+import org.elasticsearch.test.ESTestCase;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Map;
+
+public class TaskTests extends ESTestCase {
+
+    public void testTaskInfoToString() {
+        String nodeId = randomAlphaOfLength(10);
+        long taskId = randomIntBetween(0, 100000);
+        long startTime = randomNonNegativeLong();
+        long runningTime = randomNonNegativeLong();
+        boolean cancellable = randomBoolean();
+        TaskInfo taskInfo = new TaskInfo(new TaskId(nodeId, taskId), "test_type",
+            "test_action", "test_description", null, startTime, runningTime, cancellable, TaskId.EMPTY_TASK_ID);
+        String taskInfoString = taskInfo.toString();
+        Map<String, Object> map = XContentHelper.convertToMap(new BytesArray(taskInfoString.getBytes(StandardCharsets.UTF_8)), true).v2();
+        assertEquals(((Number)map.get("id")).longValue(), taskId);
+        assertEquals(map.get("type"), "test_type");
+        assertEquals(map.get("action"), "test_action");
+        assertEquals(map.get("description"), "test_description");
+        assertEquals(((Number)map.get("start_time_in_millis")).longValue(), startTime);
+        assertEquals(((Number)map.get("running_time_in_nanos")).longValue(), runningTime);
+        assertEquals(map.get("cancellable"), cancellable);
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
index a7234b20ab578..dcf2eddf7b2d2 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
@@ -37,19 +37,27 @@
 import org.elasticsearch.action.fieldstats.FieldStatsAction;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.index.IndexAction;
+import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.action.search.SearchAction;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.SearchTransportService;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.action.support.replication.ReplicationResponse;
+import org.elasticsearch.action.support.replication.TransportReplicationActionTests;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.elasticsearch.tasks.TaskResult;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.tasks.TaskInfo;
+import org.elasticsearch.tasks.TaskResult;
 import org.elasticsearch.tasks.TaskResultsService;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.tasks.MockTaskManager;
@@ -69,32 +77,39 @@
 import java.util.concurrent.BrokenBarrierException;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.CyclicBarrier;
-import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.ReentrantLock;
 import java.util.function.Consumer;
 import java.util.function.Function;
 
+import static java.util.Collections.emptyList;
 import static java.util.Collections.singleton;
 import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
 import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows;
 import static org.hamcrest.Matchers.allOf;
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.emptyCollectionOf;
+import static org.hamcrest.Matchers.endsWith;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.startsWith;
 
 /**
  * Integration tests for task management API
  * <p>
  * We need at least 2 nodes so we have a master node a non-master node
  */
-@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, minNumDataNodes = 2)
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, minNumDataNodes = 2, transportClientRatio = 0.0)
 public class TasksIT extends ESIntegTestCase {
 
     private Map<Tuple<String, String>, RecordingTaskManagerListener> listeners = new HashMap<>();
@@ -284,7 +299,8 @@ public void testTransportBulkTasks() {
         registerTaskManageListeners(BulkAction.NAME + "[s][r]");  // shard task on replica
         createIndex("test");
         ensureGreen("test"); // Make sure all shards are allocated to catch replication tasks
-        client().prepareBulk().add(client().prepareIndex("test", "doc", "test_id").setSource("{\"foo\": \"bar\"}")).get();
+        client().prepareBulk().add(client().prepareIndex("test", "doc", "test_id")
+            .setSource("{\"foo\": \"bar\"}", XContentType.JSON)).get();
 
         // the bulk operation should produce one main task
         assertEquals(1, numberOfEvents(BulkAction.NAME, Tuple::v1));
@@ -325,49 +341,81 @@ public void testTransportBulkTasks() {
         assertParentTask(findEvents(BulkAction.NAME + "[s][r]", Tuple::v1), shardTask);
     }
 
+
+    public void testSearchTaskDescriptions() {
+        registerTaskManageListeners(SearchAction.NAME);  // main task
+        registerTaskManageListeners(SearchAction.NAME + "[*]");  // shard task
+        createIndex("test");
+        ensureGreen("test"); // Make sure all shards are allocated to catch replication tasks
+        client().prepareIndex("test", "doc", "test_id").setSource("{\"foo\": \"bar\"}", XContentType.JSON)
+            .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get();
+
+        assertSearchResponse(client().prepareSearch("test").setTypes("doc").setQuery(QueryBuilders.matchAllQuery()).get());
+
+        // the search operation should produce one main task
+        List<TaskInfo> mainTask = findEvents(SearchAction.NAME, Tuple::v1);
+        assertEquals(1, mainTask.size());
+        assertThat(mainTask.get(0).getDescription(), startsWith("indices[test], types[doc], search_type["));
+        assertThat(mainTask.get(0).getDescription(), containsString("\"query\":{\"match_all\""));
+
+        // check that if we have any shard-level requests they all have non-zero length description
+        List<TaskInfo> shardTasks = findEvents(SearchAction.NAME + "[*]", Tuple::v1);
+        for (TaskInfo taskInfo : shardTasks) {
+            assertThat(taskInfo.getParentTaskId(), notNullValue());
+            assertEquals(mainTask.get(0).getTaskId(), taskInfo.getParentTaskId());
+            switch (taskInfo.getAction()) {
+                case SearchTransportService.QUERY_ACTION_NAME:
+                case SearchTransportService.DFS_ACTION_NAME:
+                    assertTrue(taskInfo.getDescription(), Regex.simpleMatch("shardId[[test][*]]", taskInfo.getDescription()));
+                    break;
+                case SearchTransportService.QUERY_ID_ACTION_NAME:
+                    assertTrue(taskInfo.getDescription(), Regex.simpleMatch("id[*], indices[test]", taskInfo.getDescription()));
+                    break;
+                case SearchTransportService.FETCH_ID_ACTION_NAME:
+                    assertTrue(taskInfo.getDescription(), Regex.simpleMatch("id[*], size[1], lastEmittedDoc[null]",
+                        taskInfo.getDescription()));
+                    break;
+                case SearchTransportService.QUERY_CAN_MATCH_NAME:
+                    assertTrue(taskInfo.getDescription(), Regex.simpleMatch("shardId[[test][*]]", taskInfo.getDescription()));
+                    break;
+                default:
+                    fail("Unexpected action [" + taskInfo.getAction() + "] with description [" + taskInfo.getDescription() + "]");
+            }
+            // assert that all task descriptions have non-zero length
+            assertThat(taskInfo.getDescription().length(), greaterThan(0));
+        }
+
+    }
+
     /**
-     * Very basic "is it plugged in" style test that indexes a document and
-     * makes sure that you can fetch the status of the process. The goal here is
-     * to verify that the large moving parts that make fetching task status work
-     * fit together rather than to verify any particular status results from
-     * indexing. For that, look at
-     * {@link org.elasticsearch.action.support.replication.TransportReplicationActionTests}
-     * . We intentionally don't use the task recording mechanism used in other
-     * places in this test so we can make sure that the status fetching works
-     * properly over the wire.
+     * Very basic "is it plugged in" style test that indexes a document and makes sure that you can fetch the status of the process. The
+     * goal here is to verify that the large moving parts that make fetching task status work fit together rather than to verify any
+     * particular status results from indexing. For that, look at {@link TransportReplicationActionTests}. We intentionally don't use the
+     * task recording mechanism used in other places in this test so we can make sure that the status fetching works properly over the wire.
      */
-    public void testCanFetchIndexStatus() throws InterruptedException, ExecutionException, IOException {
-        /*
-         * We prevent any tasks from unregistering until the test is done so we
-         * can fetch them. This will gum up the server if we leave it enabled
-         * but we'll be quick so it'll be OK (TM).
-         */
-        ReentrantLock taskFinishLock = new ReentrantLock();
-        taskFinishLock.lock();
-        ListenableActionFuture<?> indexFuture = null;
+    public void testCanFetchIndexStatus() throws Exception {
+        // First latch waits for the task to start, second on blocks it from finishing.
+        CountDownLatch taskRegistered = new CountDownLatch(1);
+        CountDownLatch letTaskFinish = new CountDownLatch(1);
+        Thread index = null;
         try {
-            CountDownLatch taskRegistered = new CountDownLatch(1);
             for (TransportService transportService : internalCluster().getInstances(TransportService.class)) {
                 ((MockTaskManager) transportService.getTaskManager()).addListener(new MockTaskManagerListener() {
                     @Override
                     public void onTaskRegistered(Task task) {
                         if (task.getAction().startsWith(IndexAction.NAME)) {
                             taskRegistered.countDown();
+                            logger.debug("Blocking [{}] starting", task);
+                            try {
+                                assertTrue(letTaskFinish.await(10, TimeUnit.SECONDS));
+                            } catch (InterruptedException e) {
+                                throw new RuntimeException(e);
+                            }
                         }
                     }
 
                     @Override
                     public void onTaskUnregistered(Task task) {
-                        /*
-                         * We can't block all tasks here or the task listing task
-                         * would never return.
-                         */
-                        if (false == task.getAction().startsWith(IndexAction.NAME)) {
-                            return;
-                        }
-                        logger.debug("Blocking {} from being unregistered", task);
-                        taskFinishLock.lock();
-                        taskFinishLock.unlock();
                     }
 
                     @Override
@@ -375,8 +423,13 @@ public void waitForTaskCompletion(Task task) {
                     }
                 });
             }
-            indexFuture = client().prepareIndex("test", "test").setSource("test", "test").execute();
-            taskRegistered.await(10, TimeUnit.SECONDS); // waiting for at least one task to be registered
+            // Need to run the task in a separate thread because node client's .execute() is blocked by our task listener
+            index = new Thread(() -> {
+                IndexResponse indexResponse = client().prepareIndex("test", "test").setSource("test", "test").get();
+                assertArrayEquals(ReplicationResponse.EMPTY, indexResponse.getShardInfo().getFailures());
+            });
+            index.start();
+            assertTrue(taskRegistered.await(10, TimeUnit.SECONDS)); // waiting for at least one task to be registered
 
             ListTasksResponse listResponse = client().admin().cluster().prepareListTasks().setActions("indices:data/write/index*")
                     .setDetailed(true).get();
@@ -390,17 +443,21 @@ public void waitForTaskCompletion(Task task) {
                 assertEquals(task.getType(), fetchedWithGet.getType());
                 assertEquals(task.getAction(), fetchedWithGet.getAction());
                 assertEquals(task.getDescription(), fetchedWithGet.getDescription());
-                // The status won't always be equal - it might change between the list and the get.
+                assertEquals(task.getStatus(), fetchedWithGet.getStatus());
                 assertEquals(task.getStartTime(), fetchedWithGet.getStartTime());
                 assertThat(fetchedWithGet.getRunningTimeNanos(), greaterThanOrEqualTo(task.getRunningTimeNanos()));
                 assertEquals(task.isCancellable(), fetchedWithGet.isCancellable());
                 assertEquals(task.getParentTaskId(), fetchedWithGet.getParentTaskId());
             }
         } finally {
-            taskFinishLock.unlock();
-            if (indexFuture != null) {
-                indexFuture.get();
+            letTaskFinish.countDown();
+            if (index != null) {
+                index.join();
             }
+            assertBusy(() -> {
+                assertEquals(emptyList(),
+                        client().admin().cluster().prepareListTasks().setActions("indices:data/write/index*").get().getTasks());
+            });
         }
     }
 
@@ -449,6 +506,9 @@ public void testListTasksWaitForCompletion() throws Exception {
         }, response -> {
             assertThat(response.getNodeFailures(), empty());
             assertThat(response.getTaskFailures(), empty());
+            assertThat(response.getTasks(), hasSize(1));
+            TaskInfo task = response.getTasks().get(0);
+            assertEquals(TestTaskPlugin.TestTaskAction.NAME, task.getAction());
         });
     }
 
@@ -456,10 +516,12 @@ public void testGetTaskWaitForCompletionWithoutStoringResult() throws Exception
         waitForCompletionTestCase(false, id -> {
             return client().admin().cluster().prepareGetTask(id).setWaitForCompletion(true).execute();
         }, response -> {
-            assertNotNull(response.getTask().getTask());
             assertTrue(response.getTask().isCompleted());
             // We didn't store the result so it won't come back when we wait
             assertNull(response.getTask().getResponse());
+            // But the task's details should still be there because we grabbed a reference to the task before waiting for it to complete.
+            assertNotNull(response.getTask().getTask());
+            assertEquals(TestTaskPlugin.TestTaskAction.NAME, response.getTask().getTask().getAction());
         });
     }
 
@@ -467,10 +529,12 @@ public void testGetTaskWaitForCompletionWithStoringResult() throws Exception {
         waitForCompletionTestCase(true, id -> {
             return client().admin().cluster().prepareGetTask(id).setWaitForCompletion(true).execute();
         }, response -> {
-            assertNotNull(response.getTask().getTask());
             assertTrue(response.getTask().isCompleted());
             // We stored the task so we should get its results
             assertEquals(0, response.getTask().getResponseAsMap().get("failure_count"));
+            // The task's details should also be there
+            assertNotNull(response.getTask().getTask());
+            assertEquals(TestTaskPlugin.TestTaskAction.NAME, response.getTask().getTask().getAction());
         });
     }
 
@@ -500,6 +564,7 @@ private <T> void waitForCompletionTestCase(boolean storeResult, Function<TaskId,
                 ((MockTaskManager) transportService.getTaskManager()).addListener(new MockTaskManagerListener() {
                     @Override
                     public void waitForTaskCompletion(Task task) {
+                        waitForWaitingToStart.countDown();
                     }
 
                     @Override
@@ -508,7 +573,6 @@ public void onTaskRegistered(Task task) {
 
                     @Override
                     public void onTaskUnregistered(Task task) {
-                        waitForWaitingToStart.countDown();
                     }
                 });
             }
@@ -516,7 +580,9 @@ public void onTaskUnregistered(Task task) {
             // Spin up a request to wait for the test task to finish
             waitResponseFuture = wait.apply(taskId);
 
-            // Wait for the wait to start
+            /* Wait for the wait to start. This should count down just *before* we wait for completion but after the list/get has got a
+             * reference to the running task. Because we unblock immediately after this the task may no longer be running for us to wait
+             * on which is fine. */
             waitForWaitingToStart.await();
         } finally {
             // Unblock the request so the wait for completion request can finish
@@ -527,7 +593,8 @@ public void onTaskUnregistered(Task task) {
         T waitResponse = waitResponseFuture.get();
         validator.accept(waitResponse);
 
-        future.get();
+        TestTaskPlugin.NodesResponse response = future.get();
+        assertEquals(emptyList(), response.failures());
     }
 
     public void testListTasksWaitForTimeout() throws Exception {
@@ -773,7 +840,7 @@ public void tearDown() throws Exception {
     private void registerTaskManageListeners(String actionMasks) {
         for (String nodeName : internalCluster().getNodeNames()) {
             DiscoveryNode node = internalCluster().getInstance(ClusterService.class, nodeName).localNode();
-            RecordingTaskManagerListener listener = new RecordingTaskManagerListener(node, actionMasks.split(","));
+            RecordingTaskManagerListener listener = new RecordingTaskManagerListener(node.getId(), actionMasks.split(","));
             ((MockTaskManager) internalCluster().getInstance(TransportService.class, nodeName).getTaskManager()).addListener(listener);
             RecordingTaskManagerListener oldListener = listeners.put(new Tuple<>(node.getName(), actionMasks), listener);
             assertNull(oldListener);
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java
index e91105291e363..f113f49a4158a 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java
@@ -70,7 +70,7 @@
 public class TestTaskPlugin extends Plugin implements ActionPlugin {
 
     @Override
-    public List<ActionHandler<? extends ActionRequest<?>, ? extends ActionResponse>> getActions() {
+    public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
         return Arrays.asList(new ActionHandler<>(TestTaskAction.INSTANCE, TransportTestTaskAction.class),
                 new ActionHandler<>(UnblockTestTasksAction.INSTANCE, TransportUnblockTestTasksAction.class));
     }
@@ -79,10 +79,15 @@ static class TestTask extends CancellableTask {
 
         private volatile boolean blocked = true;
 
-        public TestTask(long id, String type, String action, String description, TaskId parentTaskId) {
+        TestTask(long id, String type, String action, String description, TaskId parentTaskId) {
             super(id, type, action, description, parentTaskId);
         }
 
+        @Override
+        public boolean shouldCancelChildrenOnCancellation() {
+            return false;
+        }
+
         public boolean isBlocked() {
             return blocked;
         }
@@ -242,7 +247,12 @@ public String getDescription() {
 
         @Override
         public Task createTask(long id, String type, String action, TaskId parentTaskId) {
-            return new CancellableTask(id, type, action, getDescription(), parentTaskId);
+            return new CancellableTask(id, type, action, getDescription(), parentTaskId) {
+                @Override
+                public boolean shouldCancelChildrenOnCancellation() {
+                    return true;
+                }
+            };
         }
     }
 
@@ -303,10 +313,6 @@ protected NodeResponse nodeOperation(NodeRequest request) {
             throw new UnsupportedOperationException("the task parameter is required");
         }
 
-        @Override
-        protected boolean accumulateExceptions() {
-            return true;
-        }
     }
 
     public static class TestTaskAction extends Action<NodesRequest, NodesResponse, NodesRequestBuilder> {
@@ -380,7 +386,7 @@ public static class UnblockTestTasksResponse extends BaseTasksResponse {
         private List<UnblockTestTaskResponse> tasks;
 
         public UnblockTestTasksResponse() {
-
+            super(null, null);
         }
 
         public UnblockTestTasksResponse(List<UnblockTestTaskResponse> tasks, List<TaskOperationFailure> taskFailures, List<? extends
@@ -438,15 +444,11 @@ protected UnblockTestTaskResponse readTaskResponse(StreamInput in) throws IOExce
         }
 
         @Override
-        protected UnblockTestTaskResponse taskOperation(UnblockTestTasksRequest request, Task task) {
+        protected void taskOperation(UnblockTestTasksRequest request, Task task, ActionListener<UnblockTestTaskResponse> listener) {
             ((TestTask) task).unblock();
-            return new UnblockTestTaskResponse();
+            listener.onResponse(new UnblockTestTaskResponse());
         }
 
-        @Override
-        protected boolean accumulateExceptions() {
-            return true;
-        }
     }
 
     public static class UnblockTestTasksAction extends Action<UnblockTestTasksRequest, UnblockTestTasksResponse,
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java
index f964eeba9f73f..d0d5be5b4178d 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java
@@ -47,7 +47,6 @@
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.rest.action.admin.cluster.RestListTasksAction;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.tasks.TaskInfo;
@@ -66,12 +65,12 @@
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
-import java.util.function.Consumer;
 
 import static org.elasticsearch.action.support.PlainActionFuture.newFuture;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.endsWith;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.not;
 
 public class TransportTasksActionTests extends TaskManagerTestCase {
@@ -191,7 +190,7 @@ static class TestTaskResponse implements Writeable {
 
         private final String status;
 
-        public TestTaskResponse(StreamInput in) throws IOException {
+        TestTaskResponse(StreamInput in) throws IOException {
             status = in.readString();
         }
 
@@ -200,7 +199,7 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeString(status);
         }
 
-        public TestTaskResponse(String status) {
+        TestTaskResponse(String status) {
             this.status = status;
         }
 
@@ -217,11 +216,11 @@ static class TestTasksResponse extends BaseTasksResponse {
 
         private List<TestTaskResponse> tasks;
 
-        public TestTasksResponse() {
-
+        TestTasksResponse() {
+            super(null, null);
         }
 
-        public TestTasksResponse(List<TestTaskResponse> tasks, List<TaskOperationFailure> taskFailures,
+        TestTasksResponse(List<TestTaskResponse> tasks, List<TaskOperationFailure> taskFailures,
                 List<? extends FailedNodeException> nodeFailures) {
             super(taskFailures, nodeFailures);
             this.tasks = tasks == null ? Collections.emptyList() : Collections.unmodifiableList(new ArrayList<>(tasks));
@@ -271,10 +270,6 @@ protected TestTaskResponse readTaskResponse(StreamInput in) throws IOException {
             return new TestTaskResponse(in);
         }
 
-        @Override
-        protected boolean accumulateExceptions() {
-            return true;
-        }
     }
 
     private ActionFuture<NodesResponse> startBlockingTestNodesAction(CountDownLatch checkLatch) throws InterruptedException {
@@ -311,7 +306,7 @@ protected NodeResponse nodeOperation(NodeRequest request) {
                         Thread.currentThread().interrupt();
                     }
                     logger.info("Action on node {} finished", node);
-                    return new NodeResponse(testNodes[node].discoveryNode);
+                    return new NodeResponse(testNodes[node].discoveryNode());
                 }
             };
         }
@@ -371,10 +366,10 @@ public void onFailure(Exception e) {
         assertEquals(testNodes.length, response.getPerNodeTasks().size());
 
         // Coordinating node
-        assertEquals(2, response.getPerNodeTasks().get(testNodes[0].discoveryNode.getId()).size());
+        assertEquals(2, response.getPerNodeTasks().get(testNodes[0].getNodeId()).size());
         // Other nodes node
         for (int i = 1; i < testNodes.length; i++) {
-            assertEquals(1, response.getPerNodeTasks().get(testNodes[i].discoveryNode.getId()).size());
+            assertEquals(1, response.getPerNodeTasks().get(testNodes[i].getNodeId()).size());
         }
         // There should be a single main task when grouped by tasks
         assertEquals(1, response.getTaskGroups().size());
@@ -536,7 +531,7 @@ public void onFailure(Exception e) {
 
         // Try to cancel main task using action name
         CancelTasksRequest request = new CancelTasksRequest();
-        request.setNodesIds(testNodes[0].discoveryNode.getId());
+        request.setNodes(testNodes[0].getNodeId());
         request.setReason("Testing Cancellation");
         request.setActions(actionName);
         CancelTasksResponse response = testNodes[randomIntBetween(0, testNodes.length - 1)].transportCancelTasksAction.execute(request)
@@ -551,7 +546,7 @@ public void onFailure(Exception e) {
         // Try to cancel main task using id
         request = new CancelTasksRequest();
         request.setReason("Testing Cancellation");
-        request.setTaskId(new TaskId(testNodes[0].discoveryNode.getId(), task.getId()));
+        request.setTaskId(new TaskId(testNodes[0].getNodeId(), task.getId()));
         response = testNodes[randomIntBetween(0, testNodes.length - 1)].transportCancelTasksAction.execute(request).get();
 
         // Shouldn't match any tasks since testAction doesn't support cancellation
@@ -627,13 +622,34 @@ public void testTaskLevelActionFailures() throws ExecutionException, Interrupted
             tasksActions[i] = new TestTasksAction(CLUSTER_SETTINGS, "testTasksAction", threadPool, testNodes[i].clusterService,
                     testNodes[i].transportService) {
                 @Override
-                protected TestTaskResponse taskOperation(TestTasksRequest request, Task task) {
+                protected void taskOperation(TestTasksRequest request, Task task, ActionListener<TestTaskResponse> listener) {
                     logger.info("Task action on node {}", node);
                     if (failTaskOnNode == node && task.getParentTaskId().isSet()) {
                         logger.info("Failing on node {}", node);
-                        throw new RuntimeException("Task level failure");
+                        // Fail in a random way to make sure we can handle all these ways
+                        Runnable failureMode = randomFrom(
+                                () -> {
+                                    logger.info("Throwing exception from taskOperation");
+                                    throw new RuntimeException("Task level failure (direct)");
+                                },
+                                () -> {
+                                    logger.info("Calling listener synchronously with exception from taskOperation");
+                                    listener.onFailure(new RuntimeException("Task level failure (sync listener)"));
+                                },
+                                () -> {
+                                    logger.info("Calling listener asynchronously with exception from taskOperation");
+                                    threadPool.generic()
+                                            .execute(() -> listener.onFailure(new RuntimeException("Task level failure (async listener)")));
+                                }
+                                );
+                        failureMode.run();
+                    } else {
+                        if (randomBoolean()) {
+                            listener.onResponse(new TestTaskResponse("Success on node (sync)" + node));
+                        } else {
+                            threadPool.generic().execute(() -> listener.onResponse(new TestTaskResponse("Success on node (async)" + node)));
+                        }
                     }
-                    return new TestTaskResponse("Success on node " + node);
                 }
             };
         }
@@ -643,10 +659,10 @@ protected TestTaskResponse taskOperation(TestTasksRequest request, Task task) {
         TestTasksRequest testTasksRequest = new TestTasksRequest();
         testTasksRequest.setActions("testAction[n]"); // pick all test actions
         TestTasksResponse response = tasksActions[0].execute(testTasksRequest).get();
+        assertThat(response.getTaskFailures(), hasSize(1)); // one task failed
+        assertThat(response.getTaskFailures().get(0).getReason(), containsString("Task level failure"));
         // Get successful responses from all nodes except one
         assertEquals(testNodes.length - 1, response.tasks.size());
-        assertEquals(1, response.getTaskFailures().size()); // one task failed
-        assertThat(response.getTaskFailures().get(0).getReason(), containsString("Task level failure"));
         assertEquals(0, response.getNodeFailures().size()); // no nodes failed
 
         // Release all node tasks and wait for response
@@ -698,8 +714,12 @@ protected String[] filterNodeIds(DiscoveryNodes nodes, String[] nodesIds) {
                 }
 
                 @Override
-                protected TestTaskResponse taskOperation(TestTasksRequest request, Task task) {
-                    return new TestTaskResponse(testNodes[node].getNodeId());
+                protected void taskOperation(TestTasksRequest request, Task task, ActionListener<TestTaskResponse> listener) {
+                    if (randomBoolean()) {
+                        listener.onResponse(new TestTaskResponse(testNodes[node].getNodeId()));
+                    } else {
+                        threadPool.generic().execute(() -> listener.onResponse(new TestTaskResponse(testNodes[node].getNodeId())));
+                    }
                 }
             };
         }
@@ -742,11 +762,11 @@ public void testTasksToXContentGrouping() throws Exception {
         byNodes = (Map<String, Object>) byNodes.get("nodes");
         // One element on the top level
         assertEquals(testNodes.length, byNodes.size());
-        Map<String, Object> firstNode = (Map<String, Object>) byNodes.get(testNodes[0].discoveryNode.getId());
+        Map<String, Object> firstNode = (Map<String, Object>) byNodes.get(testNodes[0].getNodeId());
         firstNode = (Map<String, Object>) firstNode.get("tasks");
         assertEquals(2, firstNode.size()); // two tasks for the first node
         for (int i = 1; i < testNodes.length; i++) {
-            Map<String, Object> otherNode = (Map<String, Object>) byNodes.get(testNodes[i].discoveryNode.getId());
+            Map<String, Object> otherNode = (Map<String, Object>) byNodes.get(testNodes[i].getNodeId());
             otherNode = (Map<String, Object>) otherNode.get("tasks");
             assertEquals(1, otherNode.size()); // one tasks for the all other nodes
         }
@@ -777,6 +797,6 @@ private Map<String, Object> serialize(ListTasksResponse response, boolean byPare
         builder.endObject();
         builder.flush();
         logger.info(builder.string());
-        return XContentHelper.convertToMap(builder.bytes(), false).v2();
+        return XContentHelper.convertToMap(builder.bytes(), false, builder.contentType()).v2();
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java
index ca0242c066586..77b5ccc09abad 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java
@@ -25,15 +25,14 @@
 import org.elasticsearch.cluster.routing.allocation.command.AllocateReplicaAllocationCommand;
 import org.elasticsearch.cluster.routing.allocation.command.AllocateStalePrimaryAllocationCommand;
 import org.elasticsearch.cluster.routing.allocation.command.AllocationCommand;
-import org.elasticsearch.cluster.routing.allocation.command.AllocationCommandRegistry;
 import org.elasticsearch.cluster.routing.allocation.command.CancelAllocationCommand;
 import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.network.NetworkModule;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -45,7 +44,6 @@
 
 import java.io.IOException;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -59,24 +57,22 @@
  */
 public class ClusterRerouteRequestTests extends ESTestCase {
     private static final int ROUNDS = 30;
-    private final List<Supplier<AllocationCommand>> RANDOM_COMMAND_GENERATORS = unmodifiableList(Arrays.asList(
-            () -> new AllocateReplicaAllocationCommand(randomAsciiOfLengthBetween(2, 10), between(0, 1000),
-                    randomAsciiOfLengthBetween(2, 10)),
-            () -> new AllocateEmptyPrimaryAllocationCommand(randomAsciiOfLengthBetween(2, 10), between(0, 1000),
-                    randomAsciiOfLengthBetween(2, 10), randomBoolean()),
-            () -> new AllocateStalePrimaryAllocationCommand(randomAsciiOfLengthBetween(2, 10), between(0, 1000),
-                    randomAsciiOfLengthBetween(2, 10), randomBoolean()),
-            () -> new CancelAllocationCommand(randomAsciiOfLengthBetween(2, 10), between(0, 1000),
-                    randomAsciiOfLengthBetween(2, 10), randomBoolean()),
-            () -> new MoveAllocationCommand(randomAsciiOfLengthBetween(2, 10), between(0, 1000),
-                    randomAsciiOfLengthBetween(2, 10), randomAsciiOfLengthBetween(2, 10))));
+    private final List<Supplier<AllocationCommand>> RANDOM_COMMAND_GENERATORS = unmodifiableList(
+            Arrays.<Supplier<AllocationCommand>> asList(
+            () -> new AllocateReplicaAllocationCommand(randomAlphaOfLengthBetween(2, 10), between(0, 1000),
+                    randomAlphaOfLengthBetween(2, 10)),
+            () -> new AllocateEmptyPrimaryAllocationCommand(randomAlphaOfLengthBetween(2, 10), between(0, 1000),
+                    randomAlphaOfLengthBetween(2, 10), randomBoolean()),
+            () -> new AllocateStalePrimaryAllocationCommand(randomAlphaOfLengthBetween(2, 10), between(0, 1000),
+                    randomAlphaOfLengthBetween(2, 10), randomBoolean()),
+            () -> new CancelAllocationCommand(randomAlphaOfLengthBetween(2, 10), between(0, 1000),
+                    randomAlphaOfLengthBetween(2, 10), randomBoolean()),
+            () -> new MoveAllocationCommand(randomAlphaOfLengthBetween(2, 10), between(0, 1000),
+                    randomAlphaOfLengthBetween(2, 10), randomAlphaOfLengthBetween(2, 10))));
     private final NamedWriteableRegistry namedWriteableRegistry;
-    private final AllocationCommandRegistry allocationCommandRegistry;
 
     public ClusterRerouteRequestTests() {
-        NetworkModule networkModule = new NetworkModule(null, null, true);
-        allocationCommandRegistry = networkModule.getAllocationCommandRegistry();
-        namedWriteableRegistry = new NamedWriteableRegistry(networkModule.getNamedWriteables());
+        namedWriteableRegistry = new NamedWriteableRegistry(NetworkModule.getNamedWriteables());
     }
 
     private ClusterRerouteRequest randomRequest() {
@@ -177,10 +173,10 @@ private ClusterRerouteRequest roundTripThroughBytes(ClusterRerouteRequest origin
 
     private ClusterRerouteRequest roundTripThroughRestRequest(ClusterRerouteRequest original) throws IOException {
         RestRequest restRequest = toRestRequest(original);
-        return RestClusterRerouteAction.createRequest(restRequest, allocationCommandRegistry, ParseFieldMatcher.STRICT);
+        return RestClusterRerouteAction.createRequest(restRequest);
     }
 
-    private static RestRequest toRestRequest(ClusterRerouteRequest original) throws IOException {
+    private RestRequest toRestRequest(ClusterRerouteRequest original) throws IOException {
         Map<String, String> params = new HashMap<>();
         XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
         boolean hasBody = false;
@@ -210,11 +206,16 @@ private static RestRequest toRestRequest(ClusterRerouteRequest original) throws
         }
         builder.endObject();
 
-        FakeRestRequest.Builder requestBuilder = new FakeRestRequest.Builder();
+        FakeRestRequest.Builder requestBuilder = new FakeRestRequest.Builder(xContentRegistry());
         requestBuilder.withParams(params);
         if (hasBody) {
-            requestBuilder.withContent(builder.bytes());
+            requestBuilder.withContent(builder.bytes(), builder.contentType());
         }
         return requestBuilder.build();
     }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return new NamedXContentRegistry(NetworkModule.getNamedXContents());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteTests.java
index 58ea5035292b4..a9054879941fa 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteTests.java
@@ -21,14 +21,14 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.EmptyClusterInfoService;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
-import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation;
-import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
+import org.elasticsearch.cluster.routing.allocation.FailedShard;
 import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
 import org.elasticsearch.cluster.routing.allocation.command.AllocateEmptyPrimaryAllocationCommand;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
@@ -41,8 +41,7 @@
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.cluster.ESAllocationTestCase;
-import org.elasticsearch.test.gateway.NoopGatewayAllocator;
+import org.elasticsearch.test.gateway.TestGatewayAllocator;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -51,6 +50,8 @@
 
 import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.UNASSIGNED;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
 
 public class ClusterRerouteTests extends ESAllocationTestCase {
 
@@ -64,8 +65,7 @@ public void testSerializeRequest() throws IOException {
         BytesStreamOutput out = new BytesStreamOutput();
         req.writeTo(out);
         BytesReference bytes = out.bytes();
-        NetworkModule networkModule = new NetworkModule(null, Settings.EMPTY, true);
-        NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(networkModule.getNamedWriteables());
+        NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(NetworkModule.getNamedWriteables());
         StreamInput wrap = new NamedWriteableAwareStreamInput(bytes.streamInput(),
             namedWriteableRegistry);
         ClusterRerouteRequest deserializedReq = new ClusterRerouteRequest();
@@ -82,7 +82,7 @@ public void testSerializeRequest() throws IOException {
     public void testClusterStateUpdateTask() {
         AllocationService allocationService = new AllocationService(Settings.builder().build(), new AllocationDeciders(Settings.EMPTY,
             Collections.singleton(new MaxRetryAllocationDecider(Settings.EMPTY))),
-            NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
+            new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
         ClusterState clusterState = createInitialClusterState(allocationService);
         ClusterRerouteRequest req = new ClusterRerouteRequest();
         req.dryRun(true);
@@ -118,12 +118,12 @@ public void onFailure(Exception e) {
             assertEquals(routingTable.index("idx").shards().size(), 1);
             assertEquals(routingTable.index("idx").shard(0).shards().get(0).state(), INITIALIZING);
             assertEquals(routingTable.index("idx").shard(0).shards().get(0).unassignedInfo().getNumFailedAllocations(), i);
-            List<FailedRerouteAllocation.FailedShard> failedShards = Collections.singletonList(
-                new FailedRerouteAllocation.FailedShard(routingTable.index("idx").shard(0).shards().get(0), "boom" + i,
+            List<FailedShard> failedShards = Collections.singletonList(
+                new FailedShard(routingTable.index("idx").shard(0).shards().get(0), "boom" + i,
                     new UnsupportedOperationException()));
-            RoutingAllocation.Result result = allocationService.applyFailedShards(clusterState, failedShards);
-            assertTrue(result.changed());
-            clusterState = ClusterState.builder(clusterState).routingTable(result.routingTable()).build();
+            newState = allocationService.applyFailedShards(clusterState, failedShards);
+            assertThat(newState, not(equalTo(clusterState)));
+            clusterState = newState;
             routingTable = clusterState.routingTable();
             assertEquals(routingTable.index("idx").shards().size(), 1);
             if (i == retries-1) {
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/settings/SettingsUpdaterTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/settings/SettingsUpdaterTests.java
index bd1377b89feb4..ad03d4b001db6 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/settings/SettingsUpdaterTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/settings/SettingsUpdaterTests.java
@@ -122,5 +122,14 @@ public void testClusterBlock() {
             Settings.builder().put(MetaData.SETTING_READ_ONLY_SETTING.getKey(), false).build());
         assertEquals(clusterState.blocks().global().size(), 0);
 
+
+        clusterState = updater.updateSettings(build, Settings.builder().put(MetaData.SETTING_READ_ONLY_ALLOW_DELETE_SETTING.getKey(), true).build(),
+            Settings.builder().put(BalancedShardsAllocator.INDEX_BALANCE_FACTOR_SETTING.getKey(), 1.6).put(BalancedShardsAllocator.SHARD_BALANCE_FACTOR_SETTING.getKey(), 1.0f).build());
+        assertEquals(clusterState.blocks().global().size(), 1);
+        assertEquals(clusterState.blocks().global().iterator().next(), MetaData.CLUSTER_READ_ONLY_ALLOW_DELETE_BLOCK);
+        clusterState = updater.updateSettings(build, Settings.EMPTY,
+            Settings.builder().put(MetaData.SETTING_READ_ONLY_ALLOW_DELETE_SETTING.getKey(), false).build());
+        assertEquals(clusterState.blocks().global().size(), 0);
+
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequestTests.java
new file mode 100644
index 0000000000000..e21635596b9e7
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequestTests.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.cluster.shards;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.VersionUtils;
+
+public class ClusterSearchShardsRequestTests extends ESTestCase {
+
+    public void testSerialization() throws Exception {
+        ClusterSearchShardsRequest request = new ClusterSearchShardsRequest();
+        if (randomBoolean()) {
+            int numIndices = randomIntBetween(1, 5);
+            String[] indices = new String[numIndices];
+            for (int i = 0; i < numIndices; i++) {
+                indices[i] = randomAlphaOfLengthBetween(3, 10);
+            }
+            request.indices(indices);
+        }
+        if (randomBoolean()) {
+            request.indicesOptions(
+                    IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()));
+        }
+        if (randomBoolean()) {
+            request.preference(randomAlphaOfLengthBetween(3, 10));
+        }
+        if (randomBoolean()) {
+            int numRoutings = randomIntBetween(1, 3);
+            String[] routings = new String[numRoutings];
+            for (int i = 0; i < numRoutings; i++) {
+                routings[i] = randomAlphaOfLengthBetween(3, 10);
+            }
+            request.routing(routings);
+        }
+
+        Version version = VersionUtils.randomVersionBetween(random(), Version.V_5_0_0, Version.CURRENT);
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            out.setVersion(version);
+            request.writeTo(out);
+            try (StreamInput in = out.bytes().streamInput()) {
+                in.setVersion(version);
+                ClusterSearchShardsRequest deserialized = new ClusterSearchShardsRequest();
+                deserialized.readFrom(in);
+                assertArrayEquals(request.indices(), deserialized.indices());
+                assertSame(request.indicesOptions(), deserialized.indicesOptions());
+                assertEquals(request.routing(), deserialized.routing());
+                assertEquals(request.preference(), deserialized.preference());
+            }
+        }
+    }
+
+    public void testIndicesMustNotBeNull() {
+        ClusterSearchShardsRequest request = new ClusterSearchShardsRequest();
+        assertNotNull(request.indices());
+        expectThrows(NullPointerException.class, () -> request.indices((String[])null));
+        expectThrows(NullPointerException.class, () -> request.indices((String)null));
+        expectThrows(NullPointerException.class, () -> request.indices(new String[]{"index1", null, "index3"}));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsResponseTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsResponseTests.java
new file mode 100644
index 0000000000000..52fdf405f8113
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsResponseTests.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.cluster.shards;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.TestShardRouting;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.LocalTransportAddress;
+import org.elasticsearch.index.query.RandomQueryBuilder;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.search.internal.AliasFilter;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.VersionUtils;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class ClusterSearchShardsResponseTests extends ESTestCase {
+
+    public void testSerialization() throws Exception {
+        Map<String, AliasFilter> indicesAndFilters = new HashMap<>();
+        Set<DiscoveryNode> nodes = new HashSet<>();
+        int numShards = randomIntBetween(1, 10);
+        ClusterSearchShardsGroup[] clusterSearchShardsGroups = new ClusterSearchShardsGroup[numShards];
+        for (int i = 0; i < numShards; i++) {
+            String index = randomAlphaOfLengthBetween(3, 10);
+            ShardId shardId = new ShardId(index, randomAlphaOfLength(12), i);
+            String nodeId = randomAlphaOfLength(10);
+            ShardRouting shardRouting = TestShardRouting.newShardRouting(shardId, nodeId, randomBoolean(), ShardRoutingState.STARTED);
+            clusterSearchShardsGroups[i] = new ClusterSearchShardsGroup(shardId, new ShardRouting[]{shardRouting});
+            DiscoveryNode node = new DiscoveryNode(shardRouting.currentNodeId(),
+                    new LocalTransportAddress(randomAlphaOfLength(3)), VersionUtils.randomVersion(random()));
+            nodes.add(node);
+            AliasFilter aliasFilter;
+            if (randomBoolean()) {
+                aliasFilter = new AliasFilter(RandomQueryBuilder.createQuery(random()), "alias-" + index);
+            } else {
+                aliasFilter = new AliasFilter(null, Strings.EMPTY_ARRAY);
+            }
+            indicesAndFilters.put(index, aliasFilter);
+        }
+        ClusterSearchShardsResponse clusterSearchShardsResponse = new ClusterSearchShardsResponse(clusterSearchShardsGroups,
+                nodes.toArray(new DiscoveryNode[nodes.size()]), indicesAndFilters);
+
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
+        entries.addAll(searchModule.getNamedWriteables());
+        NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(entries);
+        Version version = VersionUtils.randomVersionBetween(random(), Version.V_5_0_0, Version.CURRENT);
+        try(BytesStreamOutput out = new BytesStreamOutput()) {
+            out.setVersion(version);
+            clusterSearchShardsResponse.writeTo(out);
+            try(StreamInput in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), namedWriteableRegistry)) {
+                in.setVersion(version);
+                ClusterSearchShardsResponse deserialized = new ClusterSearchShardsResponse();
+                deserialized.readFrom(in);
+                assertArrayEquals(clusterSearchShardsResponse.getNodes(), deserialized.getNodes());
+                assertEquals(clusterSearchShardsResponse.getGroups().length, deserialized.getGroups().length);
+                for (int i = 0; i < clusterSearchShardsResponse.getGroups().length; i++) {
+                    ClusterSearchShardsGroup clusterSearchShardsGroup = clusterSearchShardsResponse.getGroups()[i];
+                    ClusterSearchShardsGroup deserializedGroup = deserialized.getGroups()[i];
+                    assertEquals(clusterSearchShardsGroup.getShardId(), deserializedGroup.getShardId());
+                    assertArrayEquals(clusterSearchShardsGroup.getShards(), deserializedGroup.getShards());
+                }
+                if (version.onOrAfter(Version.V_5_1_1)) {
+                    assertEquals(clusterSearchShardsResponse.getIndicesAndFilters(), deserialized.getIndicesAndFilters());
+                } else {
+                    assertNull(deserialized.getIndicesAndFilters());
+                }
+            }
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java
index 2690dc63bdc32..26a882f00456f 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java
@@ -162,7 +162,7 @@ public void testIndicesShardStats() throws ExecutionException, InterruptedExcept
     }
 
     public void testValuesSmokeScreen() throws IOException, ExecutionException, InterruptedException {
-        internalCluster().startNodesAsync(randomIntBetween(1, 3)).get();
+        internalCluster().startNodes(randomIntBetween(1, 3));
         index("test1", "type", "1", "f", "f");
 
         ClusterStatsResponse response = client().admin().cluster().prepareClusterStats().get();
@@ -170,7 +170,7 @@ public void testValuesSmokeScreen() throws IOException, ExecutionException, Inte
         assertThat(msg, response.getTimestamp(), Matchers.greaterThan(946681200000L)); // 1 Jan 2000
         assertThat(msg, response.indicesStats.getStore().getSizeInBytes(), Matchers.greaterThan(0L));
 
-        assertThat(msg, response.nodesStats.getFs().getTotal().bytes(), Matchers.greaterThan(0L));
+        assertThat(msg, response.nodesStats.getFs().getTotal().getBytes(), Matchers.greaterThan(0L));
         assertThat(msg, response.nodesStats.getJvm().getVersions().size(), Matchers.greaterThan(0));
 
         assertThat(msg, response.nodesStats.getVersions().size(), Matchers.greaterThan(0));
@@ -189,20 +189,20 @@ public void testValuesSmokeScreen() throws IOException, ExecutionException, Inte
         long free = 0;
         long used = 0;
         for (NodeStats nodeStats : nodesStatsResponse.getNodes()) {
-            total += nodeStats.getOs().getMem().getTotal().bytes();
-            free += nodeStats.getOs().getMem().getFree().bytes();
-            used += nodeStats.getOs().getMem().getUsed().bytes();
+            total += nodeStats.getOs().getMem().getTotal().getBytes();
+            free += nodeStats.getOs().getMem().getFree().getBytes();
+            used += nodeStats.getOs().getMem().getUsed().getBytes();
         }
-        assertEquals(msg, free, response.nodesStats.getOs().getMem().getFree().bytes());
-        assertEquals(msg, total, response.nodesStats.getOs().getMem().getTotal().bytes());
-        assertEquals(msg, used, response.nodesStats.getOs().getMem().getUsed().bytes());
+        assertEquals(msg, free, response.nodesStats.getOs().getMem().getFree().getBytes());
+        assertEquals(msg, total, response.nodesStats.getOs().getMem().getTotal().getBytes());
+        assertEquals(msg, used, response.nodesStats.getOs().getMem().getUsed().getBytes());
         assertEquals(msg, OsStats.calculatePercentage(used, total), response.nodesStats.getOs().getMem().getUsedPercent());
         assertEquals(msg, OsStats.calculatePercentage(free, total), response.nodesStats.getOs().getMem().getFreePercent());
     }
 
     public void testAllocatedProcessors() throws Exception {
         // start one node with 7 processors.
-        internalCluster().startNodesAsync(Settings.builder().put(EsExecutors.PROCESSORS_SETTING.getKey(), 7).build()).get();
+        internalCluster().startNode(Settings.builder().put(EsExecutors.PROCESSORS_SETTING.getKey(), 7).build());
         waitForNodes(1);
 
         ClusterStatsResponse response = client().admin().cluster().prepareClusterStats().get();
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodesTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodesTests.java
new file mode 100644
index 0000000000000..e9719dd96b209
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodesTests.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.cluster.stats;
+
+import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.network.NetworkModule;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.LocalTransportAddress;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Arrays;
+import java.util.List;
+
+import static java.util.Collections.emptyList;
+import static java.util.Collections.singletonList;
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+
+public class ClusterStatsNodesTests extends ESTestCase {
+
+    /**
+     * Test that empty transport/http types are not printed out as part
+     * of the cluster stats xcontent output.
+     */
+    public void testNetworkTypesToXContent() throws Exception {
+        ClusterStatsNodes.NetworkTypes stats = new ClusterStatsNodes.NetworkTypes(emptyList());
+        assertEquals("{\"transport_types\":{},\"http_types\":{}}",
+                toXContent(stats, XContentType.JSON, randomBoolean()).utf8ToString());
+
+        List<NodeInfo> nodeInfos = singletonList(createNodeInfo("node_0", null, null));
+        stats = new ClusterStatsNodes.NetworkTypes(nodeInfos);
+        assertEquals("{\"transport_types\":{},\"http_types\":{}}",
+                toXContent(stats, XContentType.JSON, randomBoolean()).utf8ToString());
+
+        nodeInfos = Arrays.asList(createNodeInfo("node_1", "", ""),
+                                  createNodeInfo("node_2", "custom", "custom"),
+                                  createNodeInfo("node_3", null, "custom"));
+        stats = new ClusterStatsNodes.NetworkTypes(nodeInfos);
+        assertEquals("{"
+                + "\"transport_types\":{\"custom\":1},"
+                + "\"http_types\":{\"custom\":2}"
+        + "}", toXContent(stats, XContentType.JSON, randomBoolean()).utf8ToString());
+    }
+
+    private static NodeInfo createNodeInfo(String nodeId, String transportType, String httpType) {
+        Settings.Builder settings = Settings.builder();
+        if (transportType != null) {
+            settings.put(randomFrom(NetworkModule.TRANSPORT_TYPE_KEY,
+                    NetworkModule.TRANSPORT_TYPE_DEFAULT_KEY), transportType);
+        }
+        if (httpType != null) {
+            settings.put(randomFrom(NetworkModule.HTTP_TYPE_KEY,
+                    NetworkModule.HTTP_TYPE_DEFAULT_KEY), httpType);
+        }
+        return new NodeInfo(null, null,
+                new DiscoveryNode(nodeId, LocalTransportAddress.buildUnique(), null),
+                settings.build(), null, null, null, null, null, null, null, null, null);
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestTests.java
new file mode 100644
index 0000000000000..aec8349ea8d51
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestTests.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.cluster.storedscripts;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.Base64;
+
+public class PutStoredScriptRequestTests extends ESTestCase {
+
+    public void testSerialization() throws IOException {
+        PutStoredScriptRequest storedScriptRequest = new PutStoredScriptRequest("foo", "bar", new BytesArray("{}"), XContentType.JSON);
+
+        assertEquals(XContentType.JSON, storedScriptRequest.xContentType());
+        try (BytesStreamOutput output = new BytesStreamOutput()) {
+            storedScriptRequest.writeTo(output);
+
+            try (StreamInput in = output.bytes().streamInput()) {
+                PutStoredScriptRequest serialized = new PutStoredScriptRequest();
+                serialized.readFrom(in);
+                assertEquals(XContentType.JSON, serialized.xContentType());
+                assertEquals(storedScriptRequest.lang(), serialized.lang());
+                assertEquals(storedScriptRequest.id(), serialized.id());
+            }
+        }
+    }
+
+    public void testSerializationBwc() throws IOException {
+        final byte[] rawStreamBytes = Base64.getDecoder().decode("ADwDCG11c3RhY2hlAQZzY3JpcHQCe30A");
+        final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2,
+            Version.V_5_1_1, Version.V_5_1_2, Version.V_5_2_0);
+        try (StreamInput in = StreamInput.wrap(rawStreamBytes)) {
+            in.setVersion(version);
+            PutStoredScriptRequest serialized = new PutStoredScriptRequest();
+            serialized.readFrom(in);
+            assertEquals(XContentType.JSON, serialized.xContentType());
+            assertEquals("mustache", serialized.lang());
+            assertEquals("script", serialized.id());
+            assertEquals(new BytesArray("{}"), serialized.content());
+
+            try (BytesStreamOutput out = new BytesStreamOutput()) {
+                out.setVersion(version);
+                serialized.writeTo(out);
+                out.flush();
+                assertArrayEquals(rawStreamBytes, out.bytes().toBytesRef().bytes);
+            }
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/tasks/PendingTasksBlocksIT.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/tasks/PendingTasksBlocksIT.java
index 95fa5b2600f02..2aaf2507e3ba7 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/tasks/PendingTasksBlocksIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/tasks/PendingTasksBlocksIT.java
@@ -28,6 +28,7 @@
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_READ;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_WRITE;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY_ALLOW_DELETE;
 
 @ClusterScope(scope = ESIntegTestCase.Scope.TEST)
 public class PendingTasksBlocksIT extends ESIntegTestCase {
@@ -36,7 +37,8 @@ public void testPendingTasksWithBlocks() {
         ensureGreen("test");
 
         // This test checks that the Pending Cluster Tasks operation is never blocked, even if an index is read only or whatever.
-        for (String blockSetting : Arrays.asList(SETTING_BLOCKS_READ, SETTING_BLOCKS_WRITE, SETTING_READ_ONLY, SETTING_BLOCKS_METADATA)) {
+        for (String blockSetting : Arrays.asList(SETTING_BLOCKS_READ, SETTING_BLOCKS_WRITE, SETTING_READ_ONLY, SETTING_BLOCKS_METADATA,
+            SETTING_READ_ONLY_ALLOW_DELETE)) {
             try {
                 enableIndexBlock("test", blockSetting);
                 PendingClusterTasksResponse response = client().admin().cluster().preparePendingClusterTasks().execute().actionGet();
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java
index 9adf77cf20bd1..bcd7bba8d3808 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java
@@ -28,7 +28,7 @@
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.AnalysisRegistry;
-import org.elasticsearch.index.analysis.AnalysisService;
+import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.mapper.AllFieldMapper;
 import org.elasticsearch.indices.analysis.AnalysisModule;
 import org.elasticsearch.test.ESTestCase;
@@ -41,7 +41,7 @@
 
 public class TransportAnalyzeActionTests extends ESTestCase {
 
-    private AnalysisService analysisService;
+    private IndexAnalyzers indexAnalyzers;
     private AnalysisRegistry registry;
     private Environment environment;
 
@@ -70,10 +70,10 @@ public void setUp() throws Exception {
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings);
         environment = new Environment(settings);
         registry = new AnalysisModule(environment, emptyList()).getAnalysisRegistry();
-        analysisService = registry.build(idxSettings);
+        indexAnalyzers = registry.build(idxSettings);
     }
 
-    public void testNoAnalysisService() throws IOException {
+    public void testNoIndexAnalyzers() throws IOException {
         AnalyzeRequest request = new AnalyzeRequest();
         request.analyzer("standard");
         request.text("the quick brown fox");
@@ -86,7 +86,7 @@ public void testNoAnalysisService() throws IOException {
         request.addTokenFilter("lowercase");
         request.addTokenFilter("word_delimiter");
         request.text("the qu1ck brown fox");
-        analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, randomBoolean() ? analysisService : null, registry, environment);
+        analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, randomBoolean() ? indexAnalyzers : null, registry, environment);
         tokens = analyze.getTokens();
         assertEquals(6, tokens.size());
         assertEquals("qu", tokens.get(1).getTerm());
@@ -99,7 +99,7 @@ public void testNoAnalysisService() throws IOException {
         request.addTokenFilter("lowercase");
         request.addTokenFilter("word_delimiter");
         request.text("<p>the qu1ck brown fox</p>");
-        analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, randomBoolean() ? analysisService : null, registry, environment);
+        analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, randomBoolean() ? indexAnalyzers : null, registry, environment);
         tokens = analyze.getTokens();
         assertEquals(6, tokens.size());
         assertEquals("the", tokens.get(0).getTerm());
@@ -142,26 +142,26 @@ public void testFillsAttributes() throws IOException {
         assertEquals("<ALPHANUM>", tokens.get(3).getType());
     }
 
-    public void testWithAnalysisService() throws IOException {
+    public void testWithIndexAnalyzers() throws IOException {
 
         AnalyzeRequest request = new AnalyzeRequest();
         request.analyzer("standard");
         request.text("the quick brown fox");
         request.analyzer("custom_analyzer");
         request.text("the qu1ck brown fox");
-        AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, analysisService, registry, environment);
+        AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment);
         List<AnalyzeResponse.AnalyzeToken> tokens = analyze.getTokens();
         assertEquals(4, tokens.size());
 
         request.analyzer("whitespace");
         request.text("the qu1ck brown fox-dog");
-        analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, analysisService, registry, environment);
+        analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment);
         tokens = analyze.getTokens();
         assertEquals(4, tokens.size());
 
         request.analyzer("custom_analyzer");
         request.text("the qu1ck brown fox-dog");
-        analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, analysisService, registry, environment);
+        analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment);
         tokens = analyze.getTokens();
         assertEquals(5, tokens.size());
 
@@ -170,7 +170,7 @@ public void testWithAnalysisService() throws IOException {
         request.addTokenFilter("lowercase");
         request.addTokenFilter("wordDelimiter");
         request.text("the qu1ck brown fox-dog");
-        analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, analysisService, registry, environment);
+        analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment);
         tokens = analyze.getTokens();
         assertEquals(5, tokens.size());
         assertEquals("the", tokens.get(0).getTerm());
@@ -183,14 +183,14 @@ public void testWithAnalysisService() throws IOException {
         request.tokenizer("trigram");
         request.addTokenFilter("synonym");
         request.text("kimchy");
-        analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, analysisService, registry, environment);
+        analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment);
         tokens = analyze.getTokens();
         assertEquals(2, tokens.size());
         assertEquals("sha", tokens.get(0).getTerm());
         assertEquals("hay", tokens.get(1).getTerm());
     }
 
-    public void testGetIndexAnalyserWithoutAnalysisService() throws IOException {
+    public void testGetIndexAnalyserWithoutIndexAnalyzers() throws IOException {
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
             () -> TransportAnalyzeAction.analyze(
                 new AnalyzeRequest()
@@ -207,7 +207,7 @@ public void testUnknown() throws IOException {
                 new AnalyzeRequest()
                     .analyzer("foobar")
                     .text("the qu1ck brown fox"),
-                AllFieldMapper.NAME, null, notGlobal ? analysisService : null, registry, environment));
+                AllFieldMapper.NAME, null, notGlobal ? indexAnalyzers : null, registry, environment));
         if (notGlobal) {
             assertEquals(e.getMessage(), "failed to find analyzer [foobar]");
         } else {
@@ -219,7 +219,7 @@ public void testUnknown() throws IOException {
                 new AnalyzeRequest()
                     .tokenizer("foobar")
                     .text("the qu1ck brown fox"),
-                AllFieldMapper.NAME, null, notGlobal ? analysisService : null, registry, environment));
+                AllFieldMapper.NAME, null, notGlobal ? indexAnalyzers : null, registry, environment));
         if (notGlobal) {
             assertEquals(e.getMessage(), "failed to find tokenizer under [foobar]");
         } else {
@@ -232,7 +232,7 @@ public void testUnknown() throws IOException {
                     .tokenizer("whitespace")
                     .addTokenFilter("foobar")
                     .text("the qu1ck brown fox"),
-                AllFieldMapper.NAME, null, notGlobal ? analysisService : null, registry, environment));
+                AllFieldMapper.NAME, null, notGlobal ? indexAnalyzers : null, registry, environment));
         if (notGlobal) {
             assertEquals(e.getMessage(), "failed to find token filter under [foobar]");
         } else {
@@ -246,11 +246,25 @@ public void testUnknown() throws IOException {
                     .addTokenFilter("lowercase")
                     .addCharFilter("foobar")
                     .text("the qu1ck brown fox"),
-                AllFieldMapper.NAME, null, notGlobal ? analysisService : null, registry, environment));
+                AllFieldMapper.NAME, null, notGlobal ? indexAnalyzers : null, registry, environment));
         if (notGlobal) {
             assertEquals(e.getMessage(), "failed to find char filter under [foobar]");
         } else {
             assertEquals(e.getMessage(), "failed to find global char filter under [foobar]");
         }
     }
+
+    public void testNonPreBuildTokenFilter() throws IOException {
+        AnalyzeRequest request = new AnalyzeRequest();
+        request.tokenizer("whitespace");
+        request.addTokenFilter("min_hash");
+        request.text("the quick brown fox");
+        AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment);
+        List<AnalyzeResponse.AnalyzeToken> tokens = analyze.getTokens();
+        int default_hash_count = 1;
+        int default_bucket_size = 512;
+        int default_hash_set_size = 1;
+        assertEquals(default_hash_count * default_bucket_size * default_hash_set_size, tokens.size());
+
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/alias/AliasActionsTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/alias/AliasActionsTests.java
index 7ac93f38978f1..4a4aa736332cf 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/alias/AliasActionsTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/alias/AliasActionsTests.java
@@ -20,13 +20,11 @@
 package org.elasticsearch.action.admin.indices.alias;
 
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESTestCase;
@@ -49,9 +47,9 @@ public void testValidate() {
             assertEquals("One of [index] or [indices] is required", e.getMessage());
         } else {
             Exception e = expectThrows(IllegalArgumentException.class,
-                    () -> new AliasActions(type).alias(randomAsciiOfLength(5)).validate());
+                    () -> new AliasActions(type).alias(randomAlphaOfLength(5)).validate());
             assertEquals("One of [index] or [indices] is required", e.getMessage());
-            e = expectThrows(IllegalArgumentException.class, () -> new AliasActions(type).index(randomAsciiOfLength(5)).validate());
+            e = expectThrows(IllegalArgumentException.class, () -> new AliasActions(type).index(randomAlphaOfLength(5)).validate());
             assertEquals("One of [alias] or [aliases] is required", e.getMessage());
         }
     }
@@ -115,12 +113,12 @@ public void testParseAdd() throws IOException {
         b.startObject(); {
             b.startObject("add"); {
                 if (indices.length > 1 || randomBoolean()) {
-                    b.field("indices", indices);
+                    b.array("indices", indices);
                 } else {
                     b.field("index", indices[0]);
                 }
                 if (aliases.length > 1 || randomBoolean()) {
-                    b.field("aliases", aliases);
+                    b.array("aliases", aliases);
                 } else {
                     b.field("alias", aliases[0]);
                 }
@@ -142,8 +140,8 @@ public void testParseAdd() throws IOException {
         }
         b.endObject();
         b = shuffleXContent(b, "filter");
-        try (XContentParser parser = XContentHelper.createParser(b.bytes())) {
-            AliasActions action = AliasActions.PARSER.apply(parser, () -> ParseFieldMatcher.STRICT);
+        try (XContentParser parser = createParser(b)) {
+            AliasActions action = AliasActions.PARSER.apply(parser, null);
             assertEquals(AliasActions.Type.ADD, action.actionType());
             assertThat(action.indices(), equalTo(indices));
             assertThat(action.aliases(), equalTo(aliases));
@@ -158,8 +156,8 @@ public void testParseAdd() throws IOException {
     }
 
     public void testParseAddDefaultRouting() throws IOException {
-        String index = randomAsciiOfLength(5);
-        String alias = randomAsciiOfLength(5);
+        String index = randomAlphaOfLength(5);
+        String alias = randomAlphaOfLength(5);
         Object searchRouting = randomRouting();
         Object indexRouting = randomRouting();
         XContentBuilder b = XContentBuilder.builder(randomFrom(XContentType.values()).xContent());
@@ -179,8 +177,8 @@ public void testParseAddDefaultRouting() throws IOException {
         }
         b.endObject();
         b = shuffleXContent(b);
-        try (XContentParser parser = XContentHelper.createParser(b.bytes())) {
-            AliasActions action = AliasActions.PARSER.apply(parser, () -> ParseFieldMatcher.STRICT);
+        try (XContentParser parser = createParser(b)) {
+            AliasActions action = AliasActions.PARSER.apply(parser, null);
             assertEquals(AliasActions.Type.ADD, action.actionType());
             assertThat(action.indices(), arrayContaining(index));
             assertThat(action.aliases(), arrayContaining(alias));
@@ -196,12 +194,12 @@ public void testParseRemove() throws IOException {
         b.startObject(); {
             b.startObject("remove"); {
                 if (indices.length > 1 || randomBoolean()) {
-                    b.field("indices", indices);
+                    b.array("indices", indices);
                 } else {
                     b.field("index", indices[0]);
                 }
                 if (aliases.length > 1 || randomBoolean()) {
-                    b.field("aliases", aliases);
+                    b.array("aliases", aliases);
                 } else {
                     b.field("alias", aliases[0]);
                 }
@@ -210,8 +208,8 @@ public void testParseRemove() throws IOException {
         }
         b.endObject();
         b = shuffleXContent(b);
-        try (XContentParser parser = XContentHelper.createParser(b.bytes())) {
-            AliasActions action = AliasActions.PARSER.apply(parser, () -> ParseFieldMatcher.STRICT);
+        try (XContentParser parser = createParser(b)) {
+            AliasActions action = AliasActions.PARSER.apply(parser, null);
             assertEquals(AliasActions.Type.REMOVE, action.actionType());
             assertThat(action.indices(), equalTo(indices));
             assertThat(action.aliases(), equalTo(aliases));
@@ -219,12 +217,12 @@ public void testParseRemove() throws IOException {
     }
 
     public void testParseRemoveIndex() throws IOException {
-        String[] indices = randomBoolean() ? new String[] {randomAsciiOfLength(5)} : generateRandomStringArray(10, 5, false, false);
+        String[] indices = randomBoolean() ? new String[] {randomAlphaOfLength(5)} : generateRandomStringArray(10, 5, false, false);
         XContentBuilder b = XContentBuilder.builder(randomFrom(XContentType.values()).xContent());
         b.startObject(); {
             b.startObject("remove_index"); {
                 if (indices.length > 1 || randomBoolean()) {
-                    b.field("indices", indices);
+                    b.array("indices", indices);
                 } else {
                     b.field("index", indices[0]);
                 }
@@ -233,8 +231,8 @@ public void testParseRemoveIndex() throws IOException {
         }
         b.endObject();
         b = shuffleXContent(b);
-        try (XContentParser parser = XContentHelper.createParser(b.bytes())) {
-            AliasActions action = AliasActions.PARSER.apply(parser, () -> ParseFieldMatcher.STRICT);
+        try (XContentParser parser = createParser(b)) {
+            AliasActions action = AliasActions.PARSER.apply(parser, null);
             assertEquals(AliasActions.Type.REMOVE_INDEX, action.actionType());
             assertArrayEquals(indices, action.indices());
             assertThat(action.aliases(), arrayWithSize(0));
@@ -245,15 +243,15 @@ public void testParseIndexAndIndicesThrowsError() throws IOException {
         XContentBuilder b = XContentBuilder.builder(randomFrom(XContentType.values()).xContent());
         b.startObject(); {
             b.startObject(randomFrom("add", "remove")); {
-                b.field("index", randomAsciiOfLength(5));
-                b.field("indices", generateRandomStringArray(10, 5, false, false));
-                b.field("alias", randomAsciiOfLength(5));
+                b.field("index", randomAlphaOfLength(5));
+                b.array("indices", generateRandomStringArray(10, 5, false, false));
+                b.field("alias", randomAlphaOfLength(5));
             }
             b.endObject();
         }
         b.endObject();
-        try (XContentParser parser = XContentHelper.createParser(b.bytes())) {
-            Exception e = expectThrows(ParsingException.class, () -> AliasActions.PARSER.apply(parser, () -> ParseFieldMatcher.STRICT));
+        try (XContentParser parser = createParser(b)) {
+            Exception e = expectThrows(ParsingException.class, () -> AliasActions.PARSER.apply(parser, null));
             assertThat(e.getCause().getCause(), instanceOf(IllegalArgumentException.class));
             assertEquals("Only one of [index] and [indices] is supported", e.getCause().getCause().getMessage());
         }
@@ -263,15 +261,15 @@ public void testParseAliasAndAliasesThrowsError() throws IOException {
         XContentBuilder b = XContentBuilder.builder(randomFrom(XContentType.values()).xContent());
         b.startObject(); {
             b.startObject(randomFrom("add", "remove")); {
-                b.field("index", randomAsciiOfLength(5));
-                b.field("alias", randomAsciiOfLength(5));
-                b.field("aliases", generateRandomStringArray(10, 5, false, false));
+                b.field("index", randomAlphaOfLength(5));
+                b.field("alias", randomAlphaOfLength(5));
+                b.array("aliases", generateRandomStringArray(10, 5, false, false));
             }
             b.endObject();
         }
         b.endObject();
-        try (XContentParser parser = XContentHelper.createParser(b.bytes())) {
-            Exception e = expectThrows(ParsingException.class, () -> AliasActions.PARSER.apply(parser, () -> ParseFieldMatcher.STRICT));
+        try (XContentParser parser = createParser(b)) {
+            Exception e = expectThrows(ParsingException.class, () -> AliasActions.PARSER.apply(parser, null));
             assertThat(e.getCause().getCause(), instanceOf(IllegalArgumentException.class));
             assertEquals("Only one of [alias] and [aliases] is supported", e.getCause().getCause().getMessage());
         }
@@ -280,27 +278,27 @@ public void testParseAliasAndAliasesThrowsError() throws IOException {
     public void testRoundTrip() throws IOException {
         AliasActions action = new AliasActions(randomFrom(AliasActions.Type.values()));
         if (randomBoolean()) {
-            action.index(randomAsciiOfLength(5));
+            action.index(randomAlphaOfLength(5));
         } else {
             action.indices(generateRandomStringArray(5, 5, false, false));
         }
         if (action.actionType() != AliasActions.Type.REMOVE_INDEX) {
             if (randomBoolean()) {
-                action.alias(randomAsciiOfLength(5));
+                action.alias(randomAlphaOfLength(5));
             } else {
                 action.aliases(generateRandomStringArray(5, 5, false, false));
             }
         }
         if (action.actionType() == AliasActions.Type.ADD) {
             if (randomBoolean()) {
-                action.filter(randomAsciiOfLength(10));
+                action.filter(randomAlphaOfLength(10));
             }
             if (randomBoolean()) {
                 if (randomBoolean()) {
-                    action.routing(randomAsciiOfLength(5));
+                    action.routing(randomAlphaOfLength(5));
                 } else {
-                    action.searchRouting(randomAsciiOfLength(5));
-                    action.indexRouting(randomAsciiOfLength(5));
+                    action.searchRouting(randomAlphaOfLength(5));
+                    action.indexRouting(randomAlphaOfLength(5));
                 }
             }
         }
@@ -324,11 +322,11 @@ private Map<String, Object> randomMap(int maxDepth) {
                 if (maxDepth > 0) {
                     value = randomMap(maxDepth - 1);
                 } else {
-                    value = randomAsciiOfLength(5);
+                    value = randomAlphaOfLength(5);
                 }
                 break;
             case 1:
-                value = randomAsciiOfLength(5);
+                value = randomAlphaOfLength(5);
                 break;
             case 2:
                 value = randomBoolean();
@@ -339,12 +337,12 @@ private Map<String, Object> randomMap(int maxDepth) {
             default:
                 throw new UnsupportedOperationException();
             }
-            result.put(randomAsciiOfLength(5), value);
+            result.put(randomAlphaOfLength(5), value);
         }
         return result;
     }
 
     private Object randomRouting() {
-        return randomBoolean() ? randomAsciiOfLength(5) : randomInt();
+        return randomBoolean() ? randomAlphaOfLength(5) : randomInt();
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/cache/clear/ClearIndicesCacheBlocksIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/cache/clear/ClearIndicesCacheBlocksIT.java
index dbc7e5cddc3f5..ee1f4dd24e2f4 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/cache/clear/ClearIndicesCacheBlocksIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/cache/clear/ClearIndicesCacheBlocksIT.java
@@ -28,6 +28,7 @@
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_READ;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_WRITE;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY_ALLOW_DELETE;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBlocked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
 import static org.hamcrest.Matchers.equalTo;
@@ -52,7 +53,7 @@ public void testClearIndicesCacheWithBlocks() {
             }
         }
         // Request is blocked
-        for (String blockSetting : Arrays.asList(SETTING_READ_ONLY, SETTING_BLOCKS_METADATA)) {
+        for (String blockSetting : Arrays.asList(SETTING_READ_ONLY, SETTING_BLOCKS_METADATA, SETTING_READ_ONLY_ALLOW_DELETE)) {
             try {
                 enableIndexBlock("test", blockSetting);
                 assertBlocked(client().admin().indices().prepareClearCache("test").setFieldDataCache(true).setQueryCache(true).setFieldDataCache(true));
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
index ba497ffca3313..3c55d0df9c142 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
@@ -21,27 +21,19 @@
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.UnavailableShardsException;
-import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteResponse;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.cluster.ClusterInfoService;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.InternalClusterInfoService;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.routing.RoutingTable;
-import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.IndexNotFoundException;
-import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.query.RangeQueryBuilder;
-import org.elasticsearch.index.query.TermsQueryBuilder;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
@@ -49,12 +41,12 @@
 import java.util.HashMap;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.BiFunction;
 
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_WAIT_FOR_ACTIVE_SHARDS;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBlocked;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -64,6 +56,7 @@
 
 @ClusterScope(scope = Scope.TEST)
 public class CreateIndexIT extends ESIntegTestCase {
+
     public void testCreationDateGivenFails() {
         try {
             prepareCreate("test").setSettings(Settings.builder().put(IndexMetaData.SETTING_CREATION_DATE, 4L)).get();
@@ -277,15 +270,8 @@ public void testMappingConflictRootCause() throws Exception {
                 .startObject("text")
                     .field("type", "text")
                 .endObject().endObject().endObject());
-        try {
-            b.get();
-        } catch (MapperParsingException e) {
-            StringBuilder messages = new StringBuilder();
-            for (Exception rootCause: e.guessRootCauses()) {
-                messages.append(rootCause.getMessage());
-            }
-            assertThat(messages.toString(), containsString("mapper [text] is used by multiple types"));
-        }
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> b.get());
+        assertThat(e.getMessage(), containsString("mapper [text] is used by multiple types"));
     }
 
     public void testRestartIndexCreationAfterFullClusterRestart() throws Exception {
@@ -296,187 +282,6 @@ public void testRestartIndexCreationAfterFullClusterRestart() throws Exception {
         ensureGreen("test");
     }
 
-    public void testCreateShrinkIndexToN() {
-        int[][] possibleShardSplits = new int[][] {{8,4,2}, {9, 3, 1}, {4, 2, 1}, {15,5,1}};
-        int[] shardSplits = randomFrom(possibleShardSplits);
-        assertEquals(shardSplits[0], (shardSplits[0] / shardSplits[1]) * shardSplits[1]);
-        assertEquals(shardSplits[1], (shardSplits[1] / shardSplits[2]) * shardSplits[2]);
-        internalCluster().ensureAtLeastNumDataNodes(2);
-        prepareCreate("source").setSettings(Settings.builder().put(indexSettings()).put("number_of_shards", shardSplits[0])).get();
-        for (int i = 0; i < 20; i++) {
-            client().prepareIndex("source", "t1", Integer.toString(i)).setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}").get();
-        }
-        ImmutableOpenMap<String, DiscoveryNode> dataNodes = client().admin().cluster().prepareState().get().getState().nodes()
-            .getDataNodes();
-        assertTrue("at least 2 nodes but was: " + dataNodes.size(), dataNodes.size() >= 2);
-        DiscoveryNode[] discoveryNodes = dataNodes.values().toArray(DiscoveryNode.class);
-        String mergeNode = discoveryNodes[0].getName();
-        // ensure all shards are allocated otherwise the ensure green below might not succeed since we require the merge node
-        // if we change the setting too quickly we will end up with one replica unassigned which can't be assigned anymore due
-        // to the require._name below.
-        ensureGreen();
-        // relocate all shards to one node such that we can merge it.
-        client().admin().indices().prepareUpdateSettings("source")
-            .setSettings(Settings.builder()
-                .put("index.routing.allocation.require._name", mergeNode)
-                .put("index.blocks.write", true)).get();
-        ensureGreen();
-        // now merge source into a 4 shard index
-        assertAcked(client().admin().indices().prepareShrinkIndex("source", "first_shrink")
-            .setSettings(Settings.builder()
-                .put("index.number_of_replicas", 0)
-                .put("index.number_of_shards", shardSplits[1]).build()).get());
-        ensureGreen();
-        assertHitCount(client().prepareSearch("first_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
-
-        for (int i = 0; i < 20; i++) { // now update
-            client().prepareIndex("first_shrink", "t1", Integer.toString(i)).setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}").get();
-        }
-        flushAndRefresh();
-        assertHitCount(client().prepareSearch("first_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
-        assertHitCount(client().prepareSearch("source").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
-
-        // relocate all shards to one node such that we can merge it.
-        client().admin().indices().prepareUpdateSettings("first_shrink")
-            .setSettings(Settings.builder()
-                .put("index.routing.allocation.require._name", mergeNode)
-                .put("index.blocks.write", true)).get();
-        ensureGreen();
-        // now merge source into a 2 shard index
-        assertAcked(client().admin().indices().prepareShrinkIndex("first_shrink", "second_shrink")
-            .setSettings(Settings.builder()
-                .put("index.number_of_replicas", 0)
-                .put("index.number_of_shards", shardSplits[2]).build()).get());
-        ensureGreen();
-        assertHitCount(client().prepareSearch("second_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
-        // let it be allocated anywhere and bump replicas
-        client().admin().indices().prepareUpdateSettings("second_shrink")
-            .setSettings(Settings.builder()
-                .putNull("index.routing.allocation.include._id")
-                .put("index.number_of_replicas", 1)).get();
-        ensureGreen();
-        assertHitCount(client().prepareSearch("second_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
-
-        for (int i = 0; i < 20; i++) { // now update
-            client().prepareIndex("second_shrink", "t1", Integer.toString(i)).setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}").get();
-        }
-        flushAndRefresh();
-        assertHitCount(client().prepareSearch("second_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
-        assertHitCount(client().prepareSearch("first_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
-        assertHitCount(client().prepareSearch("source").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
-    }
-
-    public void testCreateShrinkIndex() {
-        internalCluster().ensureAtLeastNumDataNodes(2);
-        prepareCreate("source").setSettings(Settings.builder().put(indexSettings()).put("number_of_shards", randomIntBetween(2, 7))).get();
-        for (int i = 0; i < 20; i++) {
-            client().prepareIndex("source", randomFrom("t1", "t2", "t3")).setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}").get();
-        }
-        ImmutableOpenMap<String, DiscoveryNode> dataNodes = client().admin().cluster().prepareState().get().getState().nodes()
-            .getDataNodes();
-        assertTrue("at least 2 nodes but was: " + dataNodes.size(), dataNodes.size() >= 2);
-        DiscoveryNode[] discoveryNodes = dataNodes.values().toArray(DiscoveryNode.class);
-        String mergeNode = discoveryNodes[0].getName();
-        // ensure all shards are allocated otherwise the ensure green below might not succeed since we require the merge node
-        // if we change the setting too quickly we will end up with one replica unassigned which can't be assigned anymore due
-        // to the require._name below.
-        ensureGreen();
-        // relocate all shards to one node such that we can merge it.
-        client().admin().indices().prepareUpdateSettings("source")
-            .setSettings(Settings.builder()
-                .put("index.routing.allocation.require._name", mergeNode)
-                .put("index.blocks.write", true)).get();
-        ensureGreen();
-        // now merge source into a single shard index
-        assertAcked(client().admin().indices().prepareShrinkIndex("source", "target")
-            .setSettings(Settings.builder().put("index.number_of_replicas", 0).build()).get());
-        ensureGreen();
-        assertHitCount(client().prepareSearch("target").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
-        // bump replicas
-        client().admin().indices().prepareUpdateSettings("target")
-            .setSettings(Settings.builder()
-                .put("index.number_of_replicas", 1)).get();
-        ensureGreen();
-        assertHitCount(client().prepareSearch("target").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
-
-        for (int i = 20; i < 40; i++) {
-            client().prepareIndex("target", randomFrom("t1", "t2", "t3")).setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}").get();
-        }
-        flushAndRefresh();
-        assertHitCount(client().prepareSearch("target").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 40);
-        assertHitCount(client().prepareSearch("source").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
-
-    }
-    /**
-     * Tests that we can manually recover from a failed allocation due to shards being moved away etc.
-     */
-    public void testCreateShrinkIndexFails() throws Exception {
-        internalCluster().ensureAtLeastNumDataNodes(2);
-        prepareCreate("source").setSettings(Settings.builder().put(indexSettings())
-            .put("number_of_shards", randomIntBetween(2, 7))
-            .put("number_of_replicas", 0)).get();
-        for (int i = 0; i < 20; i++) {
-            client().prepareIndex("source", randomFrom("t1", "t2", "t3")).setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}").get();
-        }
-        ImmutableOpenMap<String, DiscoveryNode> dataNodes = client().admin().cluster().prepareState().get().getState().nodes()
-            .getDataNodes();
-        assertTrue("at least 2 nodes but was: " + dataNodes.size(), dataNodes.size() >= 2);
-        DiscoveryNode[] discoveryNodes = dataNodes.values().toArray(DiscoveryNode.class);
-        String spareNode = discoveryNodes[0].getName();
-        String mergeNode = discoveryNodes[1].getName();
-        // ensure all shards are allocated otherwise the ensure green below might not succeed since we require the merge node
-        // if we change the setting too quickly we will end up with one replica unassigned which can't be assigned anymore due
-        // to the require._name below.
-        ensureGreen();
-        // relocate all shards to one node such that we can merge it.
-        client().admin().indices().prepareUpdateSettings("source")
-            .setSettings(Settings.builder().put("index.routing.allocation.require._name", mergeNode)
-                .put("index.blocks.write", true)).get();
-        ensureGreen();
-
-        // now merge source into a single shard index
-        client().admin().indices().prepareShrinkIndex("source", "target")
-            .setSettings(Settings.builder()
-            .put("index.routing.allocation.exclude._name", mergeNode) // we manually exclude the merge node to forcefully fuck it up
-            .put("index.number_of_replicas", 0)
-            .put("index.allocation.max_retries", 1).build()).get();
-
-        // now we move all shards away from the merge node
-        client().admin().indices().prepareUpdateSettings("source")
-            .setSettings(Settings.builder().put("index.routing.allocation.require._name", spareNode)
-                .put("index.blocks.write", true)).get();
-        ensureGreen("source");
-
-        client().admin().indices().prepareUpdateSettings("target") // erase the forcefully fuckup!
-            .setSettings(Settings.builder().putNull("index.routing.allocation.exclude._name")).get();
-        // wait until it fails
-        assertBusy(() -> {
-            ClusterStateResponse clusterStateResponse = client().admin().cluster().prepareState().get();
-            RoutingTable routingTables = clusterStateResponse.getState().routingTable();
-            assertTrue(routingTables.index("target").shard(0).getShards().get(0).unassigned());
-            assertEquals(UnassignedInfo.Reason.ALLOCATION_FAILED,
-                routingTables.index("target").shard(0).getShards().get(0).unassignedInfo().getReason());
-            assertEquals(1,
-                routingTables.index("target").shard(0).getShards().get(0).unassignedInfo().getNumFailedAllocations());
-        });
-        client().admin().indices().prepareUpdateSettings("source") // now relocate them all to the right node
-            .setSettings(Settings.builder()
-                .put("index.routing.allocation.require._name", mergeNode)).get();
-        ensureGreen("source");
-
-        final InternalClusterInfoService infoService = (InternalClusterInfoService) internalCluster().getInstance(ClusterInfoService.class,
-            internalCluster().getMasterName());
-        infoService.refresh();
-        // kick off a retry and wait until it's done!
-        ClusterRerouteResponse clusterRerouteResponse = client().admin().cluster().prepareReroute().setRetryFailed(true).get();
-        long expectedShardSize = clusterRerouteResponse.getState().routingTable().index("target")
-            .shard(0).getShards().get(0).getExpectedShardSize();
-        // we support the expected shard size in the allocator to sum up over the source index shards
-        assertTrue("expected shard size must be set but wasn't: " + expectedShardSize, expectedShardSize > 0);
-        ensureGreen();
-        assertHitCount(client().prepareSearch("target").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
-    }
-
     /**
      * This test ensures that index creation adheres to the {@link IndexMetaData#SETTING_WAIT_FOR_ACTIVE_SHARDS}.
      */
@@ -503,4 +308,39 @@ public void testDefaultWaitForActiveShardsUsesIndexSetting() throws Exception {
                        .build();
         assertFalse(client().admin().indices().prepareCreate("test-idx-3").setSettings(settings).setTimeout("100ms").get().isShardsAcked());
     }
+
+    public void testInvalidPartitionSize() {
+        BiFunction<Integer, Integer, Boolean> createPartitionedIndex = (shards, partitionSize) -> {
+            CreateIndexResponse response;
+
+            try {
+                response = prepareCreate("test_" + shards + "_" + partitionSize)
+                    .setSettings(Settings.builder()
+                        .put("index.number_of_shards", shards)
+                        .put("index.routing_partition_size", partitionSize))
+                    .execute().actionGet();
+            } catch (IllegalStateException | IllegalArgumentException e) {
+                return false;
+            }
+
+            return response.isAcknowledged();
+        };
+
+        assertFalse(createPartitionedIndex.apply(3, 6));
+        assertFalse(createPartitionedIndex.apply(3, 0));
+        assertFalse(createPartitionedIndex.apply(3, 3));
+
+        assertTrue(createPartitionedIndex.apply(3, 1));
+        assertTrue(createPartitionedIndex.apply(3, 2));
+
+        assertTrue(createPartitionedIndex.apply(1, 1));
+    }
+
+    public void testIndexNameInResponse() {
+        CreateIndexResponse response = prepareCreate("foo")
+            .setSettings(Settings.builder().build())
+            .get();
+
+        assertEquals("Should have index name in response", "foo", response.index());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilderTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilderTests.java
index ad17689bb789c..1ae60f42eb63d 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilderTests.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.client.NoOpClient;
 import org.junit.After;
@@ -57,7 +58,7 @@ public void tearDown() throws Exception {
      */
     public void testSetSource() throws IOException {
         CreateIndexRequestBuilder builder = new CreateIndexRequestBuilder(this.testClient, CreateIndexAction.INSTANCE);
-        builder.setSource("{\""+KEY+"\" : \""+VALUE+"\"}");
+        builder.setSource("{\""+KEY+"\" : \""+VALUE+"\"}", XContentType.JSON);
         assertEquals(VALUE, builder.request().settings().get(KEY));
 
         XContentBuilder xContent = XContentFactory.jsonBuilder().startObject().field(KEY, VALUE).endObject();
@@ -68,7 +69,7 @@ public void testSetSource() throws IOException {
         ByteArrayOutputStream docOut = new ByteArrayOutputStream();
         XContentBuilder doc = XContentFactory.jsonBuilder(docOut).startObject().field(KEY, VALUE).endObject();
         doc.close();
-        builder.setSource(docOut.toByteArray());
+        builder.setSource(docOut.toByteArray(), XContentType.JSON);
         assertEquals(VALUE, builder.request().settings().get(KEY));
 
         Map<String, String> settingsMap = new HashMap<>();
@@ -85,7 +86,7 @@ public void testSetSettings() throws IOException {
         builder.setSettings(KEY, VALUE);
         assertEquals(VALUE, builder.request().settings().get(KEY));
 
-        builder.setSettings("{\""+KEY+"\" : \""+VALUE+"\"}");
+        builder.setSettings("{\""+KEY+"\" : \""+VALUE+"\"}", XContentType.JSON);
         assertEquals(VALUE, builder.request().settings().get(KEY));
 
         builder.setSettings(Settings.builder().put(KEY, VALUE));
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestTests.java
new file mode 100644
index 0000000000000..2675eb9d45bf9
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestTests.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.indices.create;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.Base64;
+
+public class CreateIndexRequestTests extends ESTestCase {
+
+    public void testSerialization() throws IOException {
+        CreateIndexRequest request = new CreateIndexRequest("foo");
+        String mapping = JsonXContent.contentBuilder().startObject().startObject("type").endObject().endObject().string();
+        request.mapping("my_type", mapping, XContentType.JSON);
+
+        try (BytesStreamOutput output = new BytesStreamOutput()) {
+            request.writeTo(output);
+
+            try (StreamInput in = output.bytes().streamInput()) {
+                CreateIndexRequest serialized = new CreateIndexRequest();
+                serialized.readFrom(in);
+                assertEquals(request.index(), serialized.index());
+                assertEquals(mapping, serialized.mappings().get("my_type"));
+            }
+        }
+    }
+
+    public void testSerializationBwc() throws IOException {
+        final byte[] data = Base64.getDecoder().decode("ADwDAANmb28APAMBB215X3R5cGULeyJ0eXBlIjp7fX0AAAD////+AA==");
+        final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2,
+            Version.V_5_1_1, Version.V_5_1_2, Version.V_5_2_0);
+        try (StreamInput in = StreamInput.wrap(data)) {
+            in.setVersion(version);
+            CreateIndexRequest serialized = new CreateIndexRequest();
+            serialized.readFrom(in);
+            assertEquals("foo", serialized.index());
+            BytesReference bytesReference = JsonXContent.contentBuilder().startObject().startObject("type").endObject().endObject().bytes();
+            assertEquals(bytesReference.utf8ToString(), serialized.mappings().get("my_type"));
+
+            try (BytesStreamOutput out = new BytesStreamOutput()) {
+                out.setVersion(version);
+                serialized.writeTo(out);
+                out.flush();
+                assertArrayEquals(data, out.bytes().toBytesRef().bytes);
+            }
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexResponseTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexResponseTests.java
new file mode 100644
index 0000000000000..588659335e499
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexResponseTests.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.indices.create;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+public class CreateIndexResponseTests extends ESTestCase {
+
+    public void testSerialization() throws IOException {
+        CreateIndexResponse response = new CreateIndexResponse(true, true, "foo");
+
+        try (BytesStreamOutput output = new BytesStreamOutput()) {
+            response.writeTo(output);
+
+            try (StreamInput in = output.bytes().streamInput()) {
+                CreateIndexResponse serialized = new CreateIndexResponse();
+                serialized.readFrom(in);
+                assertEquals(response.isShardsAcked(), serialized.isShardsAcked());
+                assertEquals(response.isAcknowledged(), serialized.isAcknowledged());
+                assertEquals(response.index(), serialized.index());
+            }
+        }
+    }
+
+    public void testSerializationWithOldVersion() throws IOException {
+        Version oldVersion = Version.V_5_4_0;
+        CreateIndexResponse response = new CreateIndexResponse(true, true, "foo");
+
+        try (BytesStreamOutput output = new BytesStreamOutput()) {
+            output.setVersion(oldVersion);
+            response.writeTo(output);
+
+            try (StreamInput in = output.bytes().streamInput()) {
+                in.setVersion(oldVersion);
+                CreateIndexResponse serialized = new CreateIndexResponse();
+                serialized.readFrom(in);
+                assertEquals(response.isShardsAcked(), serialized.isShardsAcked());
+                assertEquals(response.isAcknowledged(), serialized.isAcknowledged());
+                assertNull(serialized.index());
+            }
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java
new file mode 100644
index 0000000000000..9ab6551d6fd07
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java
@@ -0,0 +1,253 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.indices.create;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteResponse;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
+import org.elasticsearch.action.support.ActiveShardCount;
+import org.elasticsearch.cluster.ClusterInfoService;
+import org.elasticsearch.cluster.InternalClusterInfoService;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.RoutingTable;
+import org.elasticsearch.cluster.routing.UnassignedInfo;
+import org.elasticsearch.common.Priority;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.query.TermsQueryBuilder;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.InternalSettingsPlugin;
+import org.elasticsearch.test.VersionUtils;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+
+public class ShrinkIndexIT extends ESIntegTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Arrays.asList(InternalSettingsPlugin.class);
+    }
+
+    public void testCreateShrinkIndexToN() {
+        int[][] possibleShardSplits = new int[][] {{8,4,2}, {9, 3, 1}, {4, 2, 1}, {15,5,1}};
+        int[] shardSplits = randomFrom(possibleShardSplits);
+        assertEquals(shardSplits[0], (shardSplits[0] / shardSplits[1]) * shardSplits[1]);
+        assertEquals(shardSplits[1], (shardSplits[1] / shardSplits[2]) * shardSplits[2]);
+        internalCluster().ensureAtLeastNumDataNodes(2);
+        prepareCreate("source").setSettings(Settings.builder().put(indexSettings()).put("number_of_shards", shardSplits[0])).get();
+        for (int i = 0; i < 20; i++) {
+            client().prepareIndex("source", "t1", Integer.toString(i))
+                .setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get();
+        }
+        ImmutableOpenMap<String, DiscoveryNode> dataNodes = client().admin().cluster().prepareState().get().getState().nodes()
+            .getDataNodes();
+        assertTrue("at least 2 nodes but was: " + dataNodes.size(), dataNodes.size() >= 2);
+        DiscoveryNode[] discoveryNodes = dataNodes.values().toArray(DiscoveryNode.class);
+        String mergeNode = discoveryNodes[0].getName();
+        // ensure all shards are allocated otherwise the ensure green below might not succeed since we require the merge node
+        // if we change the setting too quickly we will end up with one replica unassigned which can't be assigned anymore due
+        // to the require._name below.
+        ensureGreen();
+        // relocate all shards to one node such that we can merge it.
+        client().admin().indices().prepareUpdateSettings("source")
+            .setSettings(Settings.builder()
+                .put("index.routing.allocation.require._name", mergeNode)
+                .put("index.blocks.write", true)).get();
+        ensureGreen();
+        // now merge source into a 4 shard index
+        assertAcked(client().admin().indices().prepareShrinkIndex("source", "first_shrink")
+            .setSettings(Settings.builder()
+                .put("index.number_of_replicas", 0)
+                .put("index.number_of_shards", shardSplits[1]).build()).get());
+        ensureGreen();
+        assertHitCount(client().prepareSearch("first_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
+
+        for (int i = 0; i < 20; i++) { // now update
+            client().prepareIndex("first_shrink", "t1", Integer.toString(i))
+                .setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get();
+        }
+        flushAndRefresh();
+        assertHitCount(client().prepareSearch("first_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
+        assertHitCount(client().prepareSearch("source").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
+
+        // relocate all shards to one node such that we can merge it.
+        client().admin().indices().prepareUpdateSettings("first_shrink")
+            .setSettings(Settings.builder()
+                .put("index.routing.allocation.require._name", mergeNode)
+                .put("index.blocks.write", true)).get();
+        ensureGreen();
+        // now merge source into a 2 shard index
+        assertAcked(client().admin().indices().prepareShrinkIndex("first_shrink", "second_shrink")
+            .setSettings(Settings.builder()
+                .put("index.number_of_replicas", 0)
+                .put("index.number_of_shards", shardSplits[2]).build()).get());
+        ensureGreen();
+        assertHitCount(client().prepareSearch("second_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
+        // let it be allocated anywhere and bump replicas
+        client().admin().indices().prepareUpdateSettings("second_shrink")
+            .setSettings(Settings.builder()
+                .putNull("index.routing.allocation.include._id")
+                .put("index.number_of_replicas", 1)).get();
+        ensureGreen();
+        assertHitCount(client().prepareSearch("second_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
+
+        for (int i = 0; i < 20; i++) { // now update
+            client().prepareIndex("second_shrink", "t1", Integer.toString(i))
+                .setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get();
+        }
+        flushAndRefresh();
+        assertHitCount(client().prepareSearch("second_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
+        assertHitCount(client().prepareSearch("first_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
+        assertHitCount(client().prepareSearch("source").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
+    }
+
+    public void testCreateShrinkIndex() {
+        internalCluster().ensureAtLeastNumDataNodes(2);
+        Version version = VersionUtils.randomVersion(random());
+        prepareCreate("source").setSettings(Settings.builder().put(indexSettings())
+            .put("number_of_shards", randomIntBetween(2, 7))
+            .put("index.version.created", version)
+        ).get();
+        for (int i = 0; i < 20; i++) {
+            client().prepareIndex("source", randomFrom("t1", "t2", "t3"))
+                .setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get();
+        }
+        ImmutableOpenMap<String, DiscoveryNode> dataNodes = client().admin().cluster().prepareState().get().getState().nodes()
+            .getDataNodes();
+        assertTrue("at least 2 nodes but was: " + dataNodes.size(), dataNodes.size() >= 2);
+        DiscoveryNode[] discoveryNodes = dataNodes.values().toArray(DiscoveryNode.class);
+        String mergeNode = discoveryNodes[0].getName();
+        // ensure all shards are allocated otherwise the ensure green below might not succeed since we require the merge node
+        // if we change the setting too quickly we will end up with one replica unassigned which can't be assigned anymore due
+        // to the require._name below.
+        ensureGreen();
+        // relocate all shards to one node such that we can merge it.
+        client().admin().indices().prepareUpdateSettings("source")
+            .setSettings(Settings.builder()
+                .put("index.routing.allocation.require._name", mergeNode)
+                .put("index.blocks.write", true)).get();
+        ensureGreen();
+        // now merge source into a single shard index
+
+        final boolean createWithReplicas = randomBoolean();
+        assertAcked(client().admin().indices().prepareShrinkIndex("source", "target")
+            .setSettings(Settings.builder().put("index.number_of_replicas", createWithReplicas ? 1 : 0).build()).get());
+        ensureGreen();
+        assertHitCount(client().prepareSearch("target").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
+
+        if (createWithReplicas == false) {
+            // bump replicas
+            client().admin().indices().prepareUpdateSettings("target")
+                .setSettings(Settings.builder()
+                    .put("index.number_of_replicas", 1)).get();
+            ensureGreen();
+            assertHitCount(client().prepareSearch("target").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
+        }
+
+        for (int i = 20; i < 40; i++) {
+            client().prepareIndex("target", randomFrom("t1", "t2", "t3"))
+                .setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get();
+        }
+        flushAndRefresh();
+        assertHitCount(client().prepareSearch("target").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 40);
+        assertHitCount(client().prepareSearch("source").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
+        GetSettingsResponse target = client().admin().indices().prepareGetSettings("target").get();
+        assertEquals(version, target.getIndexToSettings().get("target").getAsVersion("index.version.created", null));
+    }
+    /**
+     * Tests that we can manually recover from a failed allocation due to shards being moved away etc.
+     */
+    public void testCreateShrinkIndexFails() throws Exception {
+        internalCluster().ensureAtLeastNumDataNodes(2);
+        prepareCreate("source").setSettings(Settings.builder().put(indexSettings())
+            .put("number_of_shards", randomIntBetween(2, 7))
+            .put("number_of_replicas", 0)).get();
+        for (int i = 0; i < 20; i++) {
+            client().prepareIndex("source", randomFrom("t1", "t2", "t3"))
+                .setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get();
+        }
+        ImmutableOpenMap<String, DiscoveryNode> dataNodes = client().admin().cluster().prepareState().get().getState().nodes()
+            .getDataNodes();
+        assertTrue("at least 2 nodes but was: " + dataNodes.size(), dataNodes.size() >= 2);
+        DiscoveryNode[] discoveryNodes = dataNodes.values().toArray(DiscoveryNode.class);
+        String spareNode = discoveryNodes[0].getName();
+        String mergeNode = discoveryNodes[1].getName();
+        // ensure all shards are allocated otherwise the ensure green below might not succeed since we require the merge node
+        // if we change the setting too quickly we will end up with one replica unassigned which can't be assigned anymore due
+        // to the require._name below.
+        ensureGreen();
+        // relocate all shards to one node such that we can merge it.
+        client().admin().indices().prepareUpdateSettings("source")
+            .setSettings(Settings.builder().put("index.routing.allocation.require._name", mergeNode)
+                .put("index.blocks.write", true)).get();
+        ensureGreen();
+
+        // now merge source into a single shard index
+        client().admin().indices().prepareShrinkIndex("source", "target")
+            .setWaitForActiveShards(ActiveShardCount.NONE)
+            .setSettings(Settings.builder()
+                .put("index.routing.allocation.exclude._name", mergeNode) // we manually exclude the merge node to forcefully fuck it up
+                .put("index.number_of_replicas", 0)
+                .put("index.allocation.max_retries", 1).build()).get();
+        client().admin().cluster().prepareHealth("target").setWaitForEvents(Priority.LANGUID).get();
+
+        // now we move all shards away from the merge node
+        client().admin().indices().prepareUpdateSettings("source")
+            .setSettings(Settings.builder().put("index.routing.allocation.require._name", spareNode)
+                .put("index.blocks.write", true)).get();
+        ensureGreen("source");
+
+        client().admin().indices().prepareUpdateSettings("target") // erase the forcefully fuckup!
+            .setSettings(Settings.builder().putNull("index.routing.allocation.exclude._name")).get();
+        // wait until it fails
+        assertBusy(() -> {
+            ClusterStateResponse clusterStateResponse = client().admin().cluster().prepareState().get();
+            RoutingTable routingTables = clusterStateResponse.getState().routingTable();
+            assertTrue(routingTables.index("target").shard(0).getShards().get(0).unassigned());
+            assertEquals(UnassignedInfo.Reason.ALLOCATION_FAILED,
+                routingTables.index("target").shard(0).getShards().get(0).unassignedInfo().getReason());
+            assertEquals(1,
+                routingTables.index("target").shard(0).getShards().get(0).unassignedInfo().getNumFailedAllocations());
+        });
+        client().admin().indices().prepareUpdateSettings("source") // now relocate them all to the right node
+            .setSettings(Settings.builder()
+                .put("index.routing.allocation.require._name", mergeNode)).get();
+        ensureGreen("source");
+
+        final InternalClusterInfoService infoService = (InternalClusterInfoService) internalCluster().getInstance(ClusterInfoService.class,
+            internalCluster().getMasterName());
+        infoService.refresh();
+        // kick off a retry and wait until it's done!
+        ClusterRerouteResponse clusterRerouteResponse = client().admin().cluster().prepareReroute().setRetryFailed(true).get();
+        long expectedShardSize = clusterRerouteResponse.getState().routingTable().index("target")
+            .shard(0).getShards().get(0).getExpectedShardSize();
+        // we support the expected shard size in the allocator to sum up over the source index shards
+        assertTrue("expected shard size must be set but wasn't: " + expectedShardSize, expectedShardSize > 0);
+        ensureGreen();
+        assertHitCount(client().prepareSearch("target").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexBlocksIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexBlocksIT.java
index a83c209a3c22d..63cfc5da43b2f 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexBlocksIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexBlocksIT.java
@@ -19,22 +19,68 @@
 
 package org.elasticsearch.action.admin.indices.delete;
 
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBlocked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits;
 
-@ClusterScope(scope = ESIntegTestCase.Scope.TEST)
 public class DeleteIndexBlocksIT extends ESIntegTestCase {
     public void testDeleteIndexWithBlocks() {
         createIndex("test");
         ensureGreen("test");
-
         try {
             setClusterReadOnly(true);
-            assertBlocked(client().admin().indices().prepareDelete("test"));
+            assertBlocked(client().admin().indices().prepareDelete("test"), MetaData.CLUSTER_READ_ONLY_BLOCK);
         } finally {
             setClusterReadOnly(false);
         }
     }
+
+    public void testDeleteIndexOnIndexReadOnlyAllowDeleteSetting() {
+        createIndex("test");
+        ensureGreen("test");
+        client().prepareIndex().setIndex("test").setType("doc").setId("1").setSource("foo", "bar").get();
+        refresh();
+        try {
+            Settings settings = Settings.builder().put(IndexMetaData.SETTING_READ_ONLY_ALLOW_DELETE, true).build();
+            assertAcked(client().admin().indices().prepareUpdateSettings("test").setSettings(settings).get());
+            assertSearchHits(client().prepareSearch().get(), "1");
+            assertBlocked(client().prepareIndex().setIndex("test").setType("doc").setId("2").setSource("foo", "bar"),
+                IndexMetaData.INDEX_READ_ONLY_ALLOW_DELETE_BLOCK);
+            assertBlocked(client().admin().indices().prepareUpdateSettings("test")
+                    .setSettings(Settings.builder().put("index.number_of_replicas", 2)), IndexMetaData.INDEX_READ_ONLY_ALLOW_DELETE_BLOCK);
+            assertSearchHits(client().prepareSearch().get(), "1");
+            assertAcked(client().admin().indices().prepareDelete("test"));
+        } finally {
+            Settings settings = Settings.builder().putNull(IndexMetaData.SETTING_READ_ONLY_ALLOW_DELETE).build();
+            assertAcked(client().admin().indices().prepareUpdateSettings("test").setIndicesOptions(IndicesOptions.lenientExpandOpen()).
+                setSettings(settings).get());
+        }
+    }
+
+    public void testDeleteIndexOnReadOnlyAllowDeleteSetting() {
+        createIndex("test");
+        ensureGreen("test");
+        client().prepareIndex().setIndex("test").setType("doc").setId("1").setSource("foo", "bar").get();
+        refresh();
+        try {
+            Settings settings = Settings.builder().put(MetaData.SETTING_READ_ONLY_ALLOW_DELETE_SETTING.getKey(), true).build();
+            assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(settings).get());
+            assertSearchHits(client().prepareSearch().get(), "1");
+            assertBlocked(client().prepareIndex().setIndex("test").setType("doc").setId("2").setSource("foo", "bar"),
+                MetaData.CLUSTER_READ_ONLY_ALLOW_DELETE_BLOCK);
+            assertBlocked(client().admin().indices().prepareUpdateSettings("test")
+                .setSettings(Settings.builder().put("index.number_of_replicas", 2)), MetaData.CLUSTER_READ_ONLY_ALLOW_DELETE_BLOCK);
+            assertSearchHits(client().prepareSearch().get(), "1");
+            assertAcked(client().admin().indices().prepareDelete("test"));
+        } finally {
+            Settings settings = Settings.builder().putNull(MetaData.SETTING_READ_ONLY_ALLOW_DELETE_SETTING.getKey()).build();
+            assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(settings).get());
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/exists/IndicesExistsIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/exists/IndicesExistsIT.java
index e289d90c7a845..1116c88b6fc87 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/exists/IndicesExistsIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/exists/IndicesExistsIT.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.discovery.MasterNotDiscoveredException;
+import org.elasticsearch.discovery.zen.ElectMasterService;
 import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
@@ -31,11 +32,14 @@
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows;
 
-@ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0)
+@ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0,
+    autoMinMasterNodes = false)
 public class IndicesExistsIT extends ESIntegTestCase {
 
     public void testIndexExistsWithBlocksInPlace() throws IOException {
-        Settings settings = Settings.builder().put(GatewayService.RECOVER_AFTER_NODES_SETTING.getKey(), 99).build();
+        Settings settings = Settings.builder()
+            .put(GatewayService.RECOVER_AFTER_NODES_SETTING.getKey(), 99)
+            .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), 1).build();
         String node = internalCluster().startNode(settings);
 
         assertThrows(client(node).admin().indices().prepareExists("test").setMasterNodeTimeout(TimeValue.timeValueSeconds(0)),
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/flush/FlushBlocksIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/flush/FlushBlocksIT.java
index 7a55b22b60039..1ace701572c45 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/flush/FlushBlocksIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/flush/FlushBlocksIT.java
@@ -28,6 +28,7 @@
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_READ;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_WRITE;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY_ALLOW_DELETE;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBlocked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
 import static org.hamcrest.Matchers.equalTo;
@@ -46,38 +47,16 @@ public void testFlushWithBlocks() {
         }
 
         // Request is not blocked
-        for (String blockSetting : Arrays.asList(SETTING_BLOCKS_READ, SETTING_BLOCKS_WRITE)) {
+        for (String blockSetting : Arrays.asList(SETTING_BLOCKS_READ, SETTING_BLOCKS_WRITE, SETTING_READ_ONLY, SETTING_BLOCKS_METADATA,
+            SETTING_READ_ONLY_ALLOW_DELETE)) {
             try {
                 enableIndexBlock("test", blockSetting);
-                FlushResponse response = client().admin().indices().prepareFlush("test").setWaitIfOngoing(true).execute().actionGet();
+                FlushResponse response = client().admin().indices().prepareFlush("test").execute().actionGet();
                 assertNoFailures(response);
                 assertThat(response.getSuccessfulShards(), equalTo(numShards.totalNumShards));
             } finally {
                 disableIndexBlock("test", blockSetting);
             }
         }
-
-        // Request is blocked
-        for (String blockSetting : Arrays.asList(SETTING_READ_ONLY, SETTING_BLOCKS_METADATA)) {
-            try {
-                enableIndexBlock("test", blockSetting);
-                FlushResponse flushResponse = client().admin().indices().prepareFlush("test").get();
-                assertBlocked(flushResponse);
-            } finally {
-                disableIndexBlock("test", blockSetting);
-            }
-        }
-
-        // Flushing all indices is blocked when the cluster is read-only
-        try {
-            FlushResponse response = client().admin().indices().prepareFlush().execute().actionGet();
-            assertNoFailures(response);
-            assertThat(response.getSuccessfulShards(), equalTo(numShards.totalNumShards));
-
-            setClusterReadOnly(true);
-            assertBlocked(client().admin().indices().prepareFlush().get());
-        } finally {
-            setClusterReadOnly(false);
-        }
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeBlocksIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeBlocksIT.java
index e1f498b09bbfc..aa6b7c61386bc 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeBlocksIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeBlocksIT.java
@@ -28,6 +28,7 @@
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_READ;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_WRITE;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY_ALLOW_DELETE;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBlocked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
 import static org.hamcrest.Matchers.equalTo;
@@ -59,7 +60,7 @@ public void testForceMergeWithBlocks() {
         }
 
         // Request is blocked
-        for (String blockSetting : Arrays.asList(SETTING_READ_ONLY, SETTING_BLOCKS_METADATA)) {
+        for (String blockSetting : Arrays.asList(SETTING_READ_ONLY, SETTING_BLOCKS_METADATA, SETTING_READ_ONLY_ALLOW_DELETE)) {
             try {
                 enableIndexBlock("test", blockSetting);
                 assertBlocked(client().admin().indices().prepareForceMerge("test"));
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexIT.java
index 74416742d12cb..00ca800cfb4f6 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexIT.java
@@ -37,6 +37,7 @@
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_READ;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_WRITE;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY_ALLOW_DELETE;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBlocked;
 import static org.hamcrest.Matchers.anyOf;
@@ -177,7 +178,7 @@ public void testEmptyMixedFeatures() {
     }
 
     public void testGetIndexWithBlocks() {
-        for (String block : Arrays.asList(SETTING_BLOCKS_READ, SETTING_BLOCKS_WRITE, SETTING_READ_ONLY)) {
+        for (String block : Arrays.asList(SETTING_BLOCKS_READ, SETTING_BLOCKS_WRITE, SETTING_READ_ONLY, SETTING_READ_ONLY_ALLOW_DELETE)) {
             try {
                 enableIndexBlock("idx", block);
                 GetIndexResponse response = client().admin().indices().prepareGetIndex().addIndices("idx")
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java
new file mode 100644
index 0000000000000..4dc396323c048
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.indices.mapping.get;
+
+import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+public class GetFieldMappingsResponseTests extends ESTestCase {
+
+    public void testSerialization() throws IOException {
+        Map<String, Map<String, Map<String, FieldMappingMetaData>>> mappings = new HashMap<>();
+        FieldMappingMetaData fieldMappingMetaData = new FieldMappingMetaData("my field", new BytesArray("{}"));
+        mappings.put("index", Collections.singletonMap("type", Collections.singletonMap("field", fieldMappingMetaData)));
+        GetFieldMappingsResponse response = new GetFieldMappingsResponse(mappings);
+
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            response.writeTo(out);
+            GetFieldMappingsResponse serialized = new GetFieldMappingsResponse();
+            try (StreamInput in = StreamInput.wrap(out.bytes().toBytesRef().bytes)) {
+                serialized.readFrom(in);
+                FieldMappingMetaData metaData = serialized.fieldMappings("index", "type", "field");
+                assertNotNull(metaData);
+                assertEquals(new BytesArray("{}"), metaData.getSource());
+            }
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java
index 9c93e5c73d9f5..2870b04fdb747 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java
@@ -19,10 +19,20 @@
 
 package org.elasticsearch.action.admin.indices.mapping.put;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.yaml.YamlXContent;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.test.ESTestCase;
 
+import java.io.IOException;
+import java.util.Base64;
+
 public class PutMappingRequestTests extends ESTestCase {
 
     public void testValidation() {
@@ -41,12 +51,12 @@ public void testValidation() {
         assertNotNull("source validation should fail", ex);
         assertTrue(ex.getMessage().contains("source is missing"));
 
-        r.source("");
+        r.source("", XContentType.JSON);
         ex = r.validate();
         assertNotNull("source validation should fail", ex);
         assertTrue(ex.getMessage().contains("source is empty"));
 
-        r.source("somevalidmapping");
+        r.source("somevalidmapping", XContentType.JSON);
         ex = r.validate();
         assertNull("validation should succeed", ex);
 
@@ -64,4 +74,33 @@ public void testBuildFromSimplifiedDef() {
                 () -> PutMappingRequest.buildFromSimplifiedDef("type", "only_field"));
         assertEquals("mapping source must be pairs of fieldnames and properties definition.", e.getMessage());
     }
+
+    public void testPutMappingRequestSerialization() throws IOException {
+        PutMappingRequest request = new PutMappingRequest("foo");
+        String mapping = YamlXContent.contentBuilder().startObject().field("foo", "bar").endObject().string();
+        request.source(mapping, XContentType.YAML);
+        assertEquals(XContentHelper.convertToJson(new BytesArray(mapping), false, XContentType.YAML), request.source());
+
+        BytesStreamOutput bytesStreamOutput = new BytesStreamOutput();
+        request.writeTo(bytesStreamOutput);
+        StreamInput in = StreamInput.wrap(bytesStreamOutput.bytes().toBytesRef().bytes);
+        PutMappingRequest serialized = new PutMappingRequest();
+        serialized.readFrom(in);
+
+        String source = serialized.source();
+        assertEquals(XContentHelper.convertToJson(new BytesArray(mapping), false, XContentType.YAML), source);
+    }
+
+    public void testSerializationBwc() throws IOException {
+        final byte[] data = Base64.getDecoder().decode("ADwDAQNmb28MAA8tLS0KZm9vOiAiYmFyIgoAPAMAAAA=");
+        final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2,
+            Version.V_5_1_1, Version.V_5_1_2, Version.V_5_2_0);
+        try (StreamInput in = StreamInput.wrap(data)) {
+            in.setVersion(version);
+            PutMappingRequest request = new PutMappingRequest();
+            request.readFrom(in);
+            String mapping = YamlXContent.contentBuilder().startObject().field("foo", "bar").endObject().string();
+            assertEquals(XContentHelper.convertToJson(new BytesArray(mapping), false, XContentType.YAML), request.source());
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/refresh/RefreshBlocksIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/refresh/RefreshBlocksIT.java
index 33e03010d594d..d69f7842bb662 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/refresh/RefreshBlocksIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/refresh/RefreshBlocksIT.java
@@ -29,7 +29,7 @@
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_READ;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_WRITE;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBlocked;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY_ALLOW_DELETE;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
 import static org.hamcrest.Matchers.equalTo;
 
@@ -42,7 +42,8 @@ public void testRefreshWithBlocks() {
         NumShards numShards = getNumShards("test");
 
         // Request is not blocked
-        for (String blockSetting : Arrays.asList(SETTING_BLOCKS_READ, SETTING_BLOCKS_WRITE)) {
+        for (String blockSetting : Arrays.asList(SETTING_BLOCKS_READ, SETTING_BLOCKS_WRITE, SETTING_READ_ONLY, SETTING_BLOCKS_METADATA,
+            SETTING_READ_ONLY_ALLOW_DELETE)) {
             try {
                 enableIndexBlock("test", blockSetting);
                 RefreshResponse response = client().admin().indices().prepareRefresh("test").execute().actionGet();
@@ -52,27 +53,5 @@ public void testRefreshWithBlocks() {
                 disableIndexBlock("test", blockSetting);
             }
         }
-
-        // Request is blocked
-        for (String blockSetting : Arrays.asList(SETTING_READ_ONLY, SETTING_BLOCKS_METADATA)) {
-            try {
-                enableIndexBlock("test", blockSetting);
-                assertBlocked(client().admin().indices().prepareRefresh("test").get());
-            } finally {
-                disableIndexBlock("test", blockSetting);
-            }
-        }
-
-        // Refreshing all indices is blocked when the cluster is read-only
-        try {
-            RefreshResponse response = client().admin().indices().prepareRefresh().execute().actionGet();
-            assertNoFailures(response);
-            assertThat(response.getSuccessfulShards(), equalTo(numShards.totalNumShards));
-
-            setClusterReadOnly(true);
-            assertBlocked(client().admin().indices().prepareRefresh().get());
-        } finally {
-            setClusterReadOnly(false);
-        }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java
index 6566eb96db850..c449147cbbd5e 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java
@@ -20,13 +20,21 @@
 package org.elasticsearch.action.admin.indices.rollover;
 
 import org.elasticsearch.action.admin.indices.alias.Alias;
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.indices.IndexAlreadyExistsException;
+import org.elasticsearch.ResourceAlreadyExistsException;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.InternalSettingsPlugin;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+import org.joda.time.format.DateTimeFormat;
 
+import java.util.Collection;
+import java.util.Collections;
 import java.util.Map;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@@ -35,6 +43,12 @@
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST)
 public class RolloverIT extends ESIntegTestCase {
 
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Collections.singleton(InternalSettingsPlugin.class);
+    }
+
+
     public void testRolloverOnEmptyIndex() throws Exception {
         assertAcked(prepareCreate("test_index-1").addAlias(new Alias("test_alias")).get());
         final RolloverResponse response = client().admin().indices().prepareRolloverIndex("test_alias").get();
@@ -116,7 +130,7 @@ public void testRolloverConditionsNotMet() throws Exception {
         final RolloverResponse response = client().admin().indices().prepareRolloverIndex("test_alias")
             .addMaxIndexAgeCondition(TimeValue.timeValueHours(4)).get();
         assertThat(response.getOldIndex(), equalTo("test_index-0"));
-        assertThat(response.getNewIndex(), equalTo("test_index-0"));
+        assertThat(response.getNewIndex(), equalTo("test_index-000001"));
         assertThat(response.isDryRun(), equalTo(false));
         assertThat(response.isRolledOver(), equalTo(false));
         assertThat(response.getConditionStatus().size(), equalTo(1));
@@ -157,8 +171,51 @@ public void testRolloverOnExistingIndex() throws Exception {
         try {
             client().admin().indices().prepareRolloverIndex("test_alias").get();
             fail("expected failure due to existing rollover index");
-        } catch (IndexAlreadyExistsException e) {
+        } catch (ResourceAlreadyExistsException e) {
             assertThat(e.getIndex().getName(), equalTo("test_index-000001"));
         }
     }
+
+    public void testRolloverWithDateMath() {
+        DateTime now = new DateTime(DateTimeZone.UTC);
+        String index = "test-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now) + "-1";
+        String dateMathExp = "<test-{now/d}-1>";
+        assertAcked(prepareCreate(dateMathExp).addAlias(new Alias("test_alias")).get());
+        ensureGreen(index);
+        // now we modify the provided name such that we can test that the pattern is carried on
+        client().admin().indices().prepareClose(index).get();
+        client().admin().indices().prepareUpdateSettings(index).setSettings(Settings.builder()
+            .put(IndexMetaData.SETTING_INDEX_PROVIDED_NAME,
+            "<test-{now/M{YYYY.MM}}-1>")).get();
+
+        client().admin().indices().prepareOpen(index).get();
+        ensureGreen(index);
+        RolloverResponse response = client().admin().indices().prepareRolloverIndex("test_alias").get();
+        assertThat(response.getOldIndex(), equalTo(index));
+        assertThat(response.getNewIndex(), equalTo("test-" + DateTimeFormat.forPattern("YYYY.MM").print(now) + "-000002"));
+        assertThat(response.isDryRun(), equalTo(false));
+        assertThat(response.isRolledOver(), equalTo(true));
+        assertThat(response.getConditionStatus().size(), equalTo(0));
+
+        response = client().admin().indices().prepareRolloverIndex("test_alias").get();
+        assertThat(response.getOldIndex(), equalTo("test-" + DateTimeFormat.forPattern("YYYY.MM").print(now) + "-000002"));
+        assertThat(response.getNewIndex(), equalTo("test-" + DateTimeFormat.forPattern("YYYY.MM").print(now) + "-000003"));
+        assertThat(response.isDryRun(), equalTo(false));
+        assertThat(response.isRolledOver(), equalTo(true));
+        assertThat(response.getConditionStatus().size(), equalTo(0));
+
+        GetSettingsResponse getSettingsResponse = client().admin().indices().prepareGetSettings(response.getOldIndex(),
+            response.getNewIndex()).get();
+        assertEquals("<test-{now/M{YYYY.MM}}-000002>", getSettingsResponse.getSetting(response.getOldIndex(),
+            IndexMetaData.SETTING_INDEX_PROVIDED_NAME));
+        assertEquals("<test-{now/M{YYYY.MM}}-000003>", getSettingsResponse.getSetting(response.getNewIndex(),
+            IndexMetaData.SETTING_INDEX_PROVIDED_NAME));
+
+        response = client().admin().indices().prepareRolloverIndex("test_alias").setNewIndexName("<test-{now/d}-000004>").get();
+        assertThat(response.getOldIndex(), equalTo("test-" + DateTimeFormat.forPattern("YYYY.MM").print(now) + "-000003"));
+        assertThat(response.getNewIndex(), equalTo("test-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now) + "-000004"));
+        assertThat(response.isDryRun(), equalTo(false));
+        assertThat(response.isRolledOver(), equalTo(true));
+        assertThat(response.getConditionStatus().size(), equalTo(0));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java
index 7f4db3b840900..920ba2e9715f0 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java
@@ -31,15 +31,15 @@
 public class RolloverRequestTests extends ESTestCase {
 
     public void testConditionsParsing() throws Exception {
-        final RolloverRequest request = new RolloverRequest(randomAsciiOfLength(10), randomAsciiOfLength(10));
+        final RolloverRequest request = new RolloverRequest(randomAlphaOfLength(10), randomAlphaOfLength(10));
         final XContentBuilder builder = XContentFactory.jsonBuilder()
             .startObject()
-            .startObject("conditions")
-                .field("max_age", "10d")
-                .field("max_docs", 100)
-            .endObject()
+                .startObject("conditions")
+                    .field("max_age", "10d")
+                    .field("max_docs", 100)
+                .endObject()
             .endObject();
-        request.source(builder.bytes());
+        RolloverRequest.PARSER.parse(createParser(builder), request, null);
         Set<Condition> conditions = request.getConditions();
         assertThat(conditions.size(), equalTo(2));
         for (Condition condition : conditions) {
@@ -56,31 +56,31 @@ public void testConditionsParsing() throws Exception {
     }
 
     public void testParsingWithIndexSettings() throws Exception {
-        final RolloverRequest request = new RolloverRequest(randomAsciiOfLength(10), randomAsciiOfLength(10));
+        final RolloverRequest request = new RolloverRequest(randomAlphaOfLength(10), randomAlphaOfLength(10));
         final XContentBuilder builder = XContentFactory.jsonBuilder()
             .startObject()
-            .startObject("conditions")
-                .field("max_age", "10d")
-                .field("max_docs", 100)
-            .endObject()
-            .startObject("mappings")
-                .startObject("type1")
-                    .startObject("properties")
-                        .startObject("field1")
-                            .field("type", "string")
-                            .field("index", "not_analyzed")
+                .startObject("conditions")
+                    .field("max_age", "10d")
+                    .field("max_docs", 100)
+                .endObject()
+                .startObject("mappings")
+                    .startObject("type1")
+                        .startObject("properties")
+                            .startObject("field1")
+                                .field("type", "string")
+                                .field("index", "not_analyzed")
+                            .endObject()
                         .endObject()
                     .endObject()
                 .endObject()
-            .endObject()
-            .startObject("settings")
-                .field("number_of_shards", 10)
-            .endObject()
-            .startObject("aliases")
-                .startObject("alias1").endObject()
-            .endObject()
+                .startObject("settings")
+                    .field("number_of_shards", 10)
+                .endObject()
+                .startObject("aliases")
+                    .startObject("alias1").endObject()
+                .endObject()
             .endObject();
-        request.source(builder.bytes());
+        RolloverRequest.PARSER.parse(createParser(builder), request, null);
         Set<Condition> conditions = request.getConditions();
         assertThat(conditions.size(), equalTo(2));
         assertThat(request.getCreateIndexRequest().mappings().size(), equalTo(1));
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
index 2bd4c2883fc5e..d33987c92adbd 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
@@ -22,10 +22,13 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesClusterStateUpdateRequest;
 import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest;
+import org.elasticsearch.action.admin.indices.stats.CommonStats;
+import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.cluster.metadata.AliasAction;
 import org.elasticsearch.cluster.metadata.AliasMetaData;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.Settings;
@@ -39,12 +42,30 @@
 import java.util.Locale;
 import java.util.Set;
 
+import org.mockito.ArgumentCaptor;
 import static org.elasticsearch.action.admin.indices.rollover.TransportRolloverAction.evaluateConditions;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
 
 public class TransportRolloverActionTests extends ESTestCase {
 
+    public void testDocStatsSelectionFromPrimariesOnly() throws Exception {
+        long docsInPrimaryShards = 100;
+        long docsInShards = 200;
+
+        final Condition condition = createTestCondition();
+        evaluateConditions(Sets.newHashSet(condition), createMetaData(), createIndecesStatResponse(docsInShards, docsInPrimaryShards));
+        final ArgumentCaptor<Condition.Stats> argument = ArgumentCaptor.forClass(Condition.Stats.class);
+        verify(condition).evaluate(argument.capture());
+
+        assertEquals(docsInPrimaryShards, argument.getValue().numDocs);
+    }
+
     public void testEvaluateConditions() throws Exception {
         MaxDocsCondition maxDocsCondition = new MaxDocsCondition(100L);
         MaxAgeCondition maxAgeCondition = new MaxAgeCondition(TimeValue.timeValueHours(2));
@@ -56,7 +77,7 @@ public void testEvaluateConditions() throws Exception {
             .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
             .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
             .build();
-        final IndexMetaData metaData = IndexMetaData.builder(randomAsciiOfLength(10))
+        final IndexMetaData metaData = IndexMetaData.builder(randomAlphaOfLength(10))
             .creationDate(System.currentTimeMillis() - TimeValue.timeValueHours(3).getMillis())
             .settings(settings)
             .build();
@@ -91,9 +112,9 @@ public void testEvaluateConditions() throws Exception {
     }
 
     public void testCreateUpdateAliasRequest() throws Exception {
-        String sourceAlias = randomAsciiOfLength(10);
-        String sourceIndex = randomAsciiOfLength(10);
-        String targetIndex = randomAsciiOfLength(10);
+        String sourceAlias = randomAlphaOfLength(10);
+        String sourceIndex = randomAlphaOfLength(10);
+        String targetIndex = randomAlphaOfLength(10);
         final RolloverRequest rolloverRequest = new RolloverRequest(sourceAlias, targetIndex);
         final IndicesAliasesClusterStateUpdateRequest updateRequest =
             TransportRolloverAction.prepareRolloverAliasesUpdateRequest(sourceIndex, targetIndex, rolloverRequest);
@@ -118,10 +139,10 @@ public void testCreateUpdateAliasRequest() throws Exception {
     }
 
     public void testValidation() throws Exception {
-        String index1 = randomAsciiOfLength(10);
-        String alias = randomAsciiOfLength(10);
-        String index2 = randomAsciiOfLength(10);
-        String aliasWithMultipleIndices = randomAsciiOfLength(10);
+        String index1 = randomAlphaOfLength(10);
+        String alias = randomAlphaOfLength(10);
+        String index2 = randomAlphaOfLength(10);
+        String aliasWithMultipleIndices = randomAlphaOfLength(10);
         final Settings settings = Settings.builder()
             .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
             .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
@@ -141,34 +162,39 @@ public void testValidation() throws Exception {
 
         expectThrows(IllegalArgumentException.class, () ->
             TransportRolloverAction.validate(metaData, new RolloverRequest(aliasWithMultipleIndices,
-                randomAsciiOfLength(10))));
+                randomAlphaOfLength(10))));
         expectThrows(IllegalArgumentException.class, () ->
             TransportRolloverAction.validate(metaData, new RolloverRequest(randomFrom(index1, index2),
-                randomAsciiOfLength(10))));
+                randomAlphaOfLength(10))));
         expectThrows(IllegalArgumentException.class, () ->
-            TransportRolloverAction.validate(metaData, new RolloverRequest(randomAsciiOfLength(5),
-                randomAsciiOfLength(10)))
+            TransportRolloverAction.validate(metaData, new RolloverRequest(randomAlphaOfLength(5),
+                randomAlphaOfLength(10)))
         );
-        TransportRolloverAction.validate(metaData, new RolloverRequest(alias, randomAsciiOfLength(10)));
+        TransportRolloverAction.validate(metaData, new RolloverRequest(alias, randomAlphaOfLength(10)));
     }
 
     public void testGenerateRolloverIndexName() throws Exception {
-        String invalidIndexName = randomAsciiOfLength(10) + "A";
+        String invalidIndexName = randomAlphaOfLength(10) + "A";
+        IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(Settings.EMPTY);
         expectThrows(IllegalArgumentException.class, () ->
-            TransportRolloverAction.generateRolloverIndexName(invalidIndexName));
+            TransportRolloverAction.generateRolloverIndexName(invalidIndexName, indexNameExpressionResolver));
         int num = randomIntBetween(0, 100);
-        final String indexPrefix = randomAsciiOfLength(10);
+        final String indexPrefix = randomAlphaOfLength(10);
         String indexEndingInNumbers = indexPrefix + "-" + num;
-        assertThat(TransportRolloverAction.generateRolloverIndexName(indexEndingInNumbers),
+        assertThat(TransportRolloverAction.generateRolloverIndexName(indexEndingInNumbers, indexNameExpressionResolver),
             equalTo(indexPrefix + "-" + String.format(Locale.ROOT, "%06d", num + 1)));
-        assertThat(TransportRolloverAction.generateRolloverIndexName("index-name-1"), equalTo("index-name-000002"));
-        assertThat(TransportRolloverAction.generateRolloverIndexName("index-name-2"), equalTo("index-name-000003"));
+        assertThat(TransportRolloverAction.generateRolloverIndexName("index-name-1", indexNameExpressionResolver),
+            equalTo("index-name-000002"));
+        assertThat(TransportRolloverAction.generateRolloverIndexName("index-name-2", indexNameExpressionResolver),
+            equalTo("index-name-000003"));
+        assertEquals( "<index-name-{now/d}-000002>", TransportRolloverAction.generateRolloverIndexName("<index-name-{now/d}-1>",
+            indexNameExpressionResolver));
     }
 
     public void testCreateIndexRequest() throws Exception {
-        String alias = randomAsciiOfLength(10);
-        String rolloverIndex = randomAsciiOfLength(10);
-        final RolloverRequest rolloverRequest = new RolloverRequest(alias, randomAsciiOfLength(10));
+        String alias = randomAlphaOfLength(10);
+        String rolloverIndex = randomAlphaOfLength(10);
+        final RolloverRequest rolloverRequest = new RolloverRequest(alias, randomAlphaOfLength(10));
         final ActiveShardCount activeShardCount = randomBoolean() ? ActiveShardCount.ALL : ActiveShardCount.ONE;
         rolloverRequest.setWaitForActiveShards(activeShardCount);
         final Settings settings = Settings.builder()
@@ -179,9 +205,42 @@ public void testCreateIndexRequest() throws Exception {
             .build();
         rolloverRequest.getCreateIndexRequest().settings(settings);
         final CreateIndexClusterStateUpdateRequest createIndexRequest =
-            TransportRolloverAction.prepareCreateIndexRequest(rolloverIndex, rolloverRequest);
+            TransportRolloverAction.prepareCreateIndexRequest(rolloverIndex, rolloverIndex, rolloverRequest);
         assertThat(createIndexRequest.settings(), equalTo(settings));
         assertThat(createIndexRequest.index(), equalTo(rolloverIndex));
         assertThat(createIndexRequest.cause(), equalTo("rollover_index"));
     }
+
+    private IndicesStatsResponse createIndecesStatResponse(long totalDocs, long primaryDocs) {
+        final CommonStats primaryStats = mock(CommonStats.class);
+        when(primaryStats.getDocs()).thenReturn(new DocsStats(primaryDocs, 0));
+
+        final CommonStats totalStats = mock(CommonStats.class);
+        when(totalStats.getDocs()).thenReturn(new DocsStats(totalDocs, 0));
+
+        final IndicesStatsResponse response = mock(IndicesStatsResponse.class);
+        when(response.getPrimaries()).thenReturn(primaryStats);
+        when(response.getTotal()).thenReturn(totalStats);
+
+        return response;
+    }
+
+    private IndexMetaData createMetaData() {
+        final Settings settings = Settings.builder()
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+            .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
+            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+            .build();
+        return IndexMetaData.builder(randomAlphaOfLength(10))
+            .creationDate(System.currentTimeMillis() - TimeValue.timeValueHours(3).getMillis())
+            .settings(settings)
+            .build();
+    }
+
+    private Condition createTestCondition() {
+        final Condition condition = mock(Condition.class);
+        when(condition.evaluate(any())).thenReturn(new Condition.Result(condition, true));
+        return condition;
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsBlocksIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsBlocksIT.java
index 035c760d84b4c..bcf106eda8040 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsBlocksIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsBlocksIT.java
@@ -28,6 +28,7 @@
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_READ;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_WRITE;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY_ALLOW_DELETE;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBlocked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
 
@@ -44,7 +45,8 @@ public void testIndicesSegmentsWithBlocks() {
         client().admin().indices().prepareFlush("test-blocks").get();
 
         // Request is not blocked
-        for (String blockSetting : Arrays.asList(SETTING_BLOCKS_READ, SETTING_BLOCKS_WRITE, SETTING_READ_ONLY)) {
+        for (String blockSetting : Arrays.asList(SETTING_BLOCKS_READ, SETTING_BLOCKS_WRITE, SETTING_READ_ONLY,
+            SETTING_READ_ONLY_ALLOW_DELETE)) {
             try {
                 enableIndexBlock("test-blocks", blockSetting);
                 IndicesSegmentResponse response = client().admin().indices().prepareSegments("test-blocks").execute().actionGet();
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsRequestTests.java
index 36aad4fb36ef9..4a2895ad7ee41 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsRequestTests.java
@@ -54,7 +54,7 @@ public void setupIndex() {
             String id = Integer.toString(j);
             client().prepareIndex("test", "type1", id).setSource("text", "sometext").get();
         }
-        client().admin().indices().prepareFlush("test").setWaitIfOngoing(true).get();
+        client().admin().indices().prepareFlush("test").get();
     }
 
     public void testBasic() {
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java
index a887b2f01eb91..7cd18f0254e07 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java
@@ -60,7 +60,7 @@
 import static org.hamcrest.Matchers.nullValue;
 
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST)
-@TestLogging("_root:DEBUG,action.admin.indices.shards:TRACE,cluster.service:TRACE")
+@TestLogging("_root:DEBUG,org.elasticsearch.action.admin.indices.shards:TRACE,org.elasticsearch.cluster.service:TRACE")
 public class IndicesShardStoreRequestIT extends ESIntegTestCase {
 
     @Override
@@ -213,13 +213,13 @@ private void indexRandomData(String index) throws ExecutionException, Interrupte
             builders[i] = client().prepareIndex(index, "type").setSource("field", "value");
         }
         indexRandom(true, builders);
-        client().admin().indices().prepareFlush().setForce(true).setWaitIfOngoing(true).execute().actionGet();
+        client().admin().indices().prepareFlush().setForce(true).execute().actionGet();
     }
 
     private static final class IndexNodePredicate implements Predicate<Settings> {
         private final Set<String> nodesWithShard;
 
-        public IndexNodePredicate(String index) {
+        IndexNodePredicate(String index) {
             this.nodesWithShard = findNodesWithShard(index);
         }
 
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreResponseTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreResponseTests.java
index 9705009a044bb..574a27f7e2855 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreResponseTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreResponseTests.java
@@ -32,6 +32,7 @@
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.shard.ShardStateMetaData;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.transport.NodeDisconnectedException;
@@ -74,7 +75,7 @@ public void testBasicSerialization() throws Exception {
         contentBuilder.endObject();
         BytesReference bytes = contentBuilder.bytes();
 
-        try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(bytes)) {
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, bytes)) {
             Map<String, Object> map = parser.map();
             List failureList = (List) map.get("failures");
             assertThat(failureList.size(), equalTo(1));
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkActionTests.java
index 49d0ce447bade..44efcb679653a 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkActionTests.java
@@ -42,7 +42,7 @@
 import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.index.shard.DocsStats;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.gateway.NoopGatewayAllocator;
+import org.elasticsearch.test.gateway.TestGatewayAllocator;
 
 import java.util.Arrays;
 import java.util.Collections;
@@ -97,7 +97,7 @@ public void testErrorCondition() {
             .build();
         AllocationService service = new AllocationService(Settings.builder().build(), new AllocationDeciders(Settings.EMPTY,
             Collections.singleton(new MaxRetryAllocationDecider(Settings.EMPTY))),
-            NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
+            new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
 
         RoutingTable routingTable = service.reroute(clusterState, "reroute").routingTable();
         clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
@@ -111,7 +111,7 @@ public void testErrorCondition() {
     }
 
     public void testShrinkIndexSettings() {
-        String indexName = randomAsciiOfLength(10);
+        String indexName = randomAlphaOfLength(10);
         // create one that won't fail
         ClusterState clusterState = ClusterState.builder(createClusterState(indexName, randomIntBetween(2, 10), 0,
             Settings.builder()
@@ -120,7 +120,7 @@ public void testShrinkIndexSettings() {
             .build();
         AllocationService service = new AllocationService(Settings.builder().build(), new AllocationDeciders(Settings.EMPTY,
             Collections.singleton(new MaxRetryAllocationDecider(Settings.EMPTY))),
-            NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
+            new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
 
         RoutingTable routingTable = service.reroute(clusterState, "reroute").routingTable();
         clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsBlocksIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsBlocksIT.java
index 25fdb7a84dbc3..e7b477f61b2a0 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsBlocksIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsBlocksIT.java
@@ -29,6 +29,7 @@
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_READ;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_WRITE;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY_ALLOW_DELETE;
 
 @ClusterScope(scope = ESIntegTestCase.Scope.TEST)
 public class IndicesStatsBlocksIT extends ESIntegTestCase {
@@ -37,7 +38,8 @@ public void testIndicesStatsWithBlocks() {
         ensureGreen("ro");
 
         // Request is not blocked
-        for (String blockSetting : Arrays.asList(SETTING_BLOCKS_READ, SETTING_BLOCKS_WRITE, SETTING_READ_ONLY)) {
+        for (String blockSetting : Arrays.asList(SETTING_BLOCKS_READ, SETTING_BLOCKS_WRITE, SETTING_READ_ONLY,
+            SETTING_READ_ONLY_ALLOW_DELETE)) {
             try {
                 enableIndexBlock("ro", blockSetting);
                 IndicesStatsResponse indicesStatsResponse = client().admin().indices().prepareStats("ro").execute().actionGet();
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponseTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponseTests.java
new file mode 100644
index 0000000000000..0f24a520b84b7
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponseTests.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.indices.stats;
+
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Collections;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.object.HasToString.hasToString;
+
+
+public class IndicesStatsResponseTests extends ESTestCase {
+
+    public void testInvalidLevel() {
+        final IndicesStatsResponse response = new IndicesStatsResponse();
+        final String level = randomAlphaOfLength(16);
+        final ToXContent.Params params = new ToXContent.MapParams(Collections.singletonMap("level", level));
+        final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> response.toXContent(null, params));
+        assertThat(
+            e,
+            hasToString(containsString("level parameter must be one of [cluster] or [indices] or [shards] but was [" + level + "]")));
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java
index dfc10169e709f..2d70e72c8d1fb 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java
@@ -19,15 +19,22 @@
 
 package org.elasticsearch.action.admin.indices.stats;
 
+import org.elasticsearch.action.ListenableActionFuture;
 import org.elasticsearch.action.ShardOperationFailedException;
+import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.engine.CommitStats;
 import org.elasticsearch.index.engine.SegmentsStats;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.elasticsearch.test.junit.annotations.TestLogging;
 
 import java.util.List;
+import java.util.concurrent.TimeUnit;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.greaterThan;
@@ -111,6 +118,42 @@ public void testCommitStats() throws Exception {
         }
     }
 
+    public void testRefreshListeners() throws Exception {
+        // Create an index without automatic refreshes
+        createIndex("test", Settings.builder().put("refresh_interval", -1).build());
+
+        // Index a document asynchronously so the request will only return when document is refreshed
+        ListenableActionFuture<IndexResponse> index = client().prepareIndex("test", "test", "test").setSource("test", "test")
+                .setRefreshPolicy(RefreshPolicy.WAIT_UNTIL).execute();
+
+        // Wait for the refresh listener to appear in the stats. Wait a long time because NFS tests can be quite slow!
+        logger.info("starting to wait");
+        long end = System.nanoTime() + TimeUnit.MINUTES.toNanos(1);
+        while (true) {
+            IndicesStatsResponse stats = client().admin().indices().prepareStats("test").clear().setRefresh(true).setDocs(true).get();
+            CommonStats common = stats.getIndices().get("test").getTotal();
+            // There shouldn't be a doc. If there is then we did *something* weird.
+            assertEquals(0, common.docs.getCount());
+            if (1 == common.refresh.getListeners()) {
+                break;
+            }
+            if (end - System.nanoTime() < 0) {
+                logger.info("timed out");
+                fail("didn't get a refresh listener in time: " + Strings.toString(common));
+            }
+        }
+
+        // Refresh the index and wait for the request to come back
+        client().admin().indices().prepareRefresh("test").get();
+        index.get();
+
+        // The document should appear in the statistics and the refresh listener should be gone
+        IndicesStatsResponse stats = client().admin().indices().prepareStats("test").clear().setRefresh(true).setDocs(true).get();
+        CommonStats common = stats.getIndices().get("test").getTotal();
+        assertEquals(1, common.docs.getCount());
+        assertEquals(0, common.refresh.getListeners());
+    }
+
     /**
      * Gives access to package private IndicesStatsResponse constructor for test purpose.
      **/
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/template/BWCTemplateTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/template/BWCTemplateTests.java
new file mode 100644
index 0000000000000..bb2b0d58573fd
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/template/BWCTemplateTests.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.indices.template;
+
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+
+import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath;
+
+/**
+ * Rudimentary tests that the templates used by Logstash and Beats
+ * prior to their 5.x releases work for newly created indices
+ */
+public class BWCTemplateTests extends ESSingleNodeTestCase {
+    public void testBeatsTemplatesBWC() throws Exception {
+        byte[] topBeat = copyToBytesFromClasspath("/org/elasticsearch/action/admin/indices/template/topbeat-1.3.template.json");
+        byte[] packetBeat = copyToBytesFromClasspath("/org/elasticsearch/action/admin/indices/template/packetbeat-1.3.template.json");
+        byte[] fileBeat = copyToBytesFromClasspath("/org/elasticsearch/action/admin/indices/template/filebeat-1.3.template.json");
+        byte[] winLogBeat = copyToBytesFromClasspath("/org/elasticsearch/action/admin/indices/template/winlogbeat-1.3.template.json");
+        client().admin().indices().preparePutTemplate("topbeat").setSource(topBeat, XContentType.JSON).get();
+        client().admin().indices().preparePutTemplate("packetbeat").setSource(packetBeat, XContentType.JSON).get();
+        client().admin().indices().preparePutTemplate("filebeat").setSource(fileBeat, XContentType.JSON).get();
+        client().admin().indices().preparePutTemplate("winlogbeat").setSource(winLogBeat, XContentType.JSON).get();
+
+        client().prepareIndex("topbeat-foo", "doc", "1").setSource("message", "foo").get();
+        client().prepareIndex("packetbeat-foo", "doc", "1").setSource("message", "foo").get();
+        client().prepareIndex("filebeat-foo", "doc", "1").setSource("message", "foo").get();
+        client().prepareIndex("winlogbeat-foo", "doc", "1").setSource("message", "foo").get();
+    }
+
+    public void testLogstashTemplatesBWC() throws Exception {
+        byte[] ls2x = copyToBytesFromClasspath("/org/elasticsearch/action/admin/indices/template/logstash-2.x.template.json");
+        byte[] ls5x = copyToBytesFromClasspath("/org/elasticsearch/action/admin/indices/template/logstash-5.x.template.json");
+        client().admin().indices().preparePutTemplate("logstash-2x").setSource(ls2x, XContentType.JSON).get();
+        client().prepareIndex("logstash-foo", "doc", "1").setSource("message", "foo").get();
+        client().admin().indices().prepareDeleteTemplate("*").get();
+
+        client().admin().indices().preparePutTemplate("logstash-5x").setSource(ls5x, XContentType.JSON).get();
+        client().prepareIndex("logstash-foo", "doc", "1").setSource("message", "foo").get();
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java
index d895ef9cbf580..d82c1d198d52b 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java
@@ -26,9 +26,10 @@
 import org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService;
 import org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService.PutRequest;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.index.NodeServicesProvider;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.InvalidIndexTemplateException;
@@ -54,12 +55,17 @@ public void testIndexTemplateInvalidNumberOfShards() {
 
         Map<String, Object> map = new HashMap<>();
         map.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "0");
+        map.put("index.shard.check_on_startup", "blargh");
         request.settings(Settings.builder().put(map).build());
 
-        List<Throwable> throwables = putTemplate(request);
+        List<Throwable> throwables = putTemplate(xContentRegistry(), request);
         assertEquals(throwables.size(), 1);
         assertThat(throwables.get(0), instanceOf(InvalidIndexTemplateException.class));
-        assertThat(throwables.get(0).getMessage(), containsString("index must have 1 or more primary shards"));
+        assertThat(throwables.get(0).getMessage(),
+                containsString("Failed to parse value [0] for setting [index.number_of_shards] must be >= 1"));
+        assertThat(throwables.get(0).getMessage(),
+                containsString("unknown value for [index.shard.check_on_startup] " +
+                                "must be one of [true, false, fix, checksum] but was: blargh"));
     }
 
     public void testIndexTemplateValidationAccumulatesValidationErrors() {
@@ -70,12 +76,13 @@ public void testIndexTemplateValidationAccumulatesValidationErrors() {
         map.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "0");
         request.settings(Settings.builder().put(map).build());
 
-        List<Throwable> throwables = putTemplate(request);
+        List<Throwable> throwables = putTemplate(xContentRegistry(), request);
         assertEquals(throwables.size(), 1);
         assertThat(throwables.get(0), instanceOf(InvalidIndexTemplateException.class));
         assertThat(throwables.get(0).getMessage(), containsString("name must not contain a space"));
         assertThat(throwables.get(0).getMessage(), containsString("template must not start with '_'"));
-        assertThat(throwables.get(0).getMessage(), containsString("index must have 1 or more primary shards"));
+        assertThat(throwables.get(0).getMessage(),
+                containsString("Failed to parse value [0] for setting [index.number_of_shards] must be >= 1"));
     }
 
     public void testIndexTemplateWithAliasNameEqualToTemplatePattern() {
@@ -83,7 +90,7 @@ public void testIndexTemplateWithAliasNameEqualToTemplatePattern() {
         request.template("foobar");
         request.aliases(Collections.singleton(new Alias("foobar")));
 
-        List<Throwable> errors = putTemplate(request);
+        List<Throwable> errors = putTemplate(xContentRegistry(), request);
         assertThat(errors.size(), equalTo(1));
         assertThat(errors.get(0), instanceOf(IllegalArgumentException.class));
         assertThat(errors.get(0).getMessage(), equalTo("Alias [foobar] cannot be the same as the template pattern [foobar]"));
@@ -151,16 +158,17 @@ public void testAliasInvalidFilterInvalidJson() throws Exception {
     }
 
 
-    private static List<Throwable> putTemplate(PutRequest request) {
+    private static List<Throwable> putTemplate(NamedXContentRegistry xContentRegistry, PutRequest request) {
         MetaDataCreateIndexService createIndexService = new MetaDataCreateIndexService(
                 Settings.EMPTY,
                 null,
                 null,
                 null,
                 null,
-                null,
-                null, null, null);
-        MetaDataIndexTemplateService service = new MetaDataIndexTemplateService(Settings.EMPTY, null, createIndexService, new AliasValidator(Settings.EMPTY), null, null);
+                null, null, null, xContentRegistry);
+        MetaDataIndexTemplateService service = new MetaDataIndexTemplateService(Settings.EMPTY, null, createIndexService,
+                new AliasValidator(Settings.EMPTY), null,
+                new IndexScopedSettings(Settings.EMPTY, IndexScopedSettings.BUILT_IN_INDEX_SETTINGS), xContentRegistry);
 
         final List<Throwable> throwables = new ArrayList<>();
         service.putTemplate(request, new MetaDataIndexTemplateService.PutListener() {
@@ -180,7 +188,6 @@ public void onFailure(Exception e) {
     private List<Throwable> putTemplateDetail(PutRequest request) throws Exception {
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
         ClusterService clusterService = getInstanceFromNode(ClusterService.class);
-        NodeServicesProvider nodeServicesProvider = getInstanceFromNode(NodeServicesProvider.class);
         MetaDataCreateIndexService createIndexService = new MetaDataCreateIndexService(
             Settings.EMPTY,
             clusterService,
@@ -188,11 +195,12 @@ private List<Throwable> putTemplateDetail(PutRequest request) throws Exception {
             null,
             null,
             null,
-            nodeServicesProvider,
             null,
-            null);
+            null,
+            xContentRegistry());
         MetaDataIndexTemplateService service = new MetaDataIndexTemplateService(
-            Settings.EMPTY, clusterService, createIndexService, new AliasValidator(Settings.EMPTY), indicesService, nodeServicesProvider);
+                Settings.EMPTY, clusterService, createIndexService, new AliasValidator(Settings.EMPTY), indicesService,
+                new IndexScopedSettings(Settings.EMPTY, IndexScopedSettings.BUILT_IN_INDEX_SETTINGS), xContentRegistry());
 
         final List<Throwable> throwables = new ArrayList<>();
         final CountDownLatch latch = new CountDownLatch(1);
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestTests.java
new file mode 100644
index 0000000000000..178988ca9ec7d
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestTests.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.admin.indices.template.put;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.yaml.YamlXContent;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.Base64;
+
+public class PutIndexTemplateRequestTests extends ESTestCase {
+
+    // bwc for #21009
+    public void testPutIndexTemplateRequest510() throws IOException {
+        PutIndexTemplateRequest putRequest = new PutIndexTemplateRequest("test");
+        putRequest.template("test*");
+        putRequest.order(5);
+
+        // These bytes were retrieved by Base64 encoding the result of the above with 5_0_0 code.
+        // Note: Instead of a list for the template, in 5_0_0 the element was provided as a string.
+        String putRequestBytes = "ADwDAAR0ZXN0BXRlc3QqAAAABQAAAAAAAA==";
+        BytesArray bytes = new BytesArray(Base64.getDecoder().decode(putRequestBytes));
+
+        try (StreamInput in = bytes.streamInput()) {
+            in.setVersion(Version.V_5_0_0);
+            PutIndexTemplateRequest readRequest = new PutIndexTemplateRequest();
+            readRequest.readFrom(in);
+            assertEquals(putRequest.template(), readRequest.template());
+            assertEquals(putRequest.order(), readRequest.order());
+
+            BytesStreamOutput output = new BytesStreamOutput();
+            output.setVersion(Version.V_5_0_0);
+            readRequest.writeTo(output);
+            assertEquals(bytes.toBytesRef(), output.bytes().toBytesRef());
+        }
+    }
+
+    public void testPutIndexTemplateRequestSerializationXContent() throws IOException {
+        PutIndexTemplateRequest request = new PutIndexTemplateRequest("foo");
+        String mapping = YamlXContent.contentBuilder().startObject().field("foo", "bar").endObject().string();
+        request.template("foo");
+        request.mapping("bar", mapping, XContentType.YAML);
+        assertNotEquals(mapping, request.mappings().get("bar"));
+        assertEquals(XContentHelper.convertToJson(new BytesArray(mapping), false, XContentType.YAML), request.mappings().get("bar"));
+
+        BytesStreamOutput out = new BytesStreamOutput();
+        request.writeTo(out);
+
+        StreamInput in = StreamInput.wrap(out.bytes().toBytesRef().bytes);
+        PutIndexTemplateRequest serialized = new PutIndexTemplateRequest();
+        serialized.readFrom(in);
+        assertEquals(XContentHelper.convertToJson(new BytesArray(mapping), false, XContentType.YAML), serialized.mappings().get("bar"));
+    }
+
+    public void testPutIndexTemplateRequestSerializationXContentBwc() throws IOException {
+        final byte[] data = Base64.getDecoder().decode("ADwDAANmb28IdGVtcGxhdGUAAAAAAAABA2Jhcg8tLS0KZm9vOiAiYmFyIgoAAAAAAAAAAAAAAAA=");
+        final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2,
+            Version.V_5_1_1, Version.V_5_1_2, Version.V_5_2_0);
+        try (StreamInput in = StreamInput.wrap(data)) {
+            in.setVersion(version);
+            PutIndexTemplateRequest request = new PutIndexTemplateRequest();
+            request.readFrom(in);
+            String mapping = YamlXContent.contentBuilder().startObject().field("foo", "bar").endObject().string();
+            assertNotEquals(mapping, request.mappings().get("bar"));
+            assertEquals(XContentHelper.convertToJson(new BytesArray(mapping), false, XContentType.YAML), request.mappings().get("bar"));
+            assertEquals("foo", request.name());
+            assertEquals("template", request.template());
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/upgrade/UpgradeIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/upgrade/UpgradeIT.java
index 9af2289c8daa3..72f75a363a4b2 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/upgrade/UpgradeIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/upgrade/UpgradeIT.java
@@ -173,7 +173,7 @@ static class UpgradeStatus {
         public final int toUpgradeBytes;
         public final int toUpgradeBytesAncient;
 
-        public UpgradeStatus(String indexName, int totalBytes, int toUpgradeBytes, int toUpgradeBytesAncient) {
+        UpgradeStatus(String indexName, int totalBytes, int toUpgradeBytes, int toUpgradeBytesAncient) {
             this.indexName = indexName;
             this.totalBytes = totalBytes;
             this.toUpgradeBytes = toUpgradeBytes;
diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java
index 4300a629fbd8c..8fcc76e018a6c 100644
--- a/core/src/test/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java
+++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java
@@ -21,6 +21,7 @@
 package org.elasticsearch.action.bulk;
 
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESIntegTestCase;
 
 import java.nio.charset.StandardCharsets;
@@ -31,15 +32,12 @@ public class BulkIntegrationIT extends ESIntegTestCase {
     public void testBulkIndexCreatesMapping() throws Exception {
         String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/bulk-log.json");
         BulkRequestBuilder bulkBuilder = client().prepareBulk();
-        bulkBuilder.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null);
+        bulkBuilder.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON);
         bulkBuilder.get();
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                GetMappingsResponse mappingsResponse = client().admin().indices().prepareGetMappings().get();
-                assertTrue(mappingsResponse.getMappings().containsKey("logstash-2014.03.30"));
-                assertTrue(mappingsResponse.getMappings().get("logstash-2014.03.30").containsKey("logs"));
-            }
+        assertBusy(() -> {
+            GetMappingsResponse mappingsResponse = client().admin().indices().prepareGetMappings().get();
+            assertTrue(mappingsResponse.getMappings().containsKey("logstash-2014.03.30"));
+            assertTrue(mappingsResponse.getMappings().get("logstash-2014.03.30").containsKey("logs"));
         });
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkItemResponseTests.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkItemResponseTests.java
index ec82c481e6c40..4a55f0c8b95bf 100644
--- a/core/src/test/java/org/elasticsearch/action/bulk/BulkItemResponseTests.java
+++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkItemResponseTests.java
@@ -19,12 +19,30 @@
 
 package org.elasticsearch.action.bulk;
 
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
+import org.elasticsearch.action.delete.DeleteResponseTests;
+import org.elasticsearch.action.index.IndexResponseTests;
+import org.elasticsearch.action.update.UpdateResponse;
+import org.elasticsearch.action.update.UpdateResponseTests;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESTestCase;
 
+import java.io.IOException;
+
+import static org.elasticsearch.ElasticsearchExceptionTests.assertDeepEquals;
+import static org.elasticsearch.ElasticsearchExceptionTests.randomExceptions;
 import static org.hamcrest.Matchers.containsString;
 
 public class BulkItemResponseTests extends ESTestCase {
+
     public void testFailureToString() {
         Failure failure = new Failure("index", "type", "id", new RuntimeException("test"));
         String toString = failure.toString();
@@ -32,4 +50,101 @@ public void testFailureToString() {
         assertThat(toString, containsString("\"reason\":\"test\""));
         assertThat(toString, containsString("\"status\":500"));
     }
+
+    public void testToAndFromXContent() throws IOException {
+        final XContentType xContentType = randomFrom(XContentType.values());
+
+        for (DocWriteRequest.OpType opType : DocWriteRequest.OpType.values()) {
+            int bulkItemId = randomIntBetween(0, 100);
+            boolean humanReadable = randomBoolean();
+
+            Tuple<? extends DocWriteResponse, ? extends DocWriteResponse> randomDocWriteResponses = null;
+            if (opType == DocWriteRequest.OpType.INDEX || opType == DocWriteRequest.OpType.CREATE) {
+                randomDocWriteResponses = IndexResponseTests.randomIndexResponse();
+            } else if (opType == DocWriteRequest.OpType.DELETE) {
+                randomDocWriteResponses = DeleteResponseTests.randomDeleteResponse();
+            } else if (opType == DocWriteRequest.OpType.UPDATE) {
+                randomDocWriteResponses = UpdateResponseTests.randomUpdateResponse(xContentType);
+            } else {
+                fail("Test does not support opType [" + opType + "]");
+            }
+
+            BulkItemResponse bulkItemResponse = new BulkItemResponse(bulkItemId, opType, randomDocWriteResponses.v1());
+            BulkItemResponse expectedBulkItemResponse = new BulkItemResponse(bulkItemId, opType, randomDocWriteResponses.v2());
+            BytesReference originalBytes = toShuffledXContent(bulkItemResponse, xContentType, ToXContent.EMPTY_PARAMS, humanReadable);
+
+            BulkItemResponse parsedBulkItemResponse;
+            try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) {
+                assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+                parsedBulkItemResponse = BulkItemResponse.fromXContent(parser, bulkItemId);
+                assertNull(parser.nextToken());
+            }
+            assertBulkItemResponse(expectedBulkItemResponse, parsedBulkItemResponse);
+        }
+    }
+
+    public void testFailureToAndFromXContent() throws IOException {
+        final XContentType xContentType = randomFrom(XContentType.values());
+
+        int itemId = randomIntBetween(0, 100);
+        String index = randomAlphaOfLength(5);
+        String type = randomAlphaOfLength(5);
+        String id = randomAlphaOfLength(5);
+        DocWriteRequest.OpType opType = randomFrom(DocWriteRequest.OpType.values());
+
+        final Tuple<Throwable, ElasticsearchException> exceptions = randomExceptions();
+
+        Exception bulkItemCause = (Exception) exceptions.v1();
+        Failure bulkItemFailure = new Failure(index, type, id, bulkItemCause);
+        BulkItemResponse bulkItemResponse = new BulkItemResponse(itemId, opType, bulkItemFailure);
+        Failure expectedBulkItemFailure = new Failure(index, type, id, exceptions.v2(), ExceptionsHelper.status(bulkItemCause));
+        BulkItemResponse expectedBulkItemResponse = new BulkItemResponse(itemId, opType, expectedBulkItemFailure);
+        BytesReference originalBytes = toShuffledXContent(bulkItemResponse, xContentType, ToXContent.EMPTY_PARAMS, randomBoolean());
+
+        // Shuffle the XContent fields
+        if (randomBoolean()) {
+            try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) {
+                originalBytes = shuffleXContent(parser, randomBoolean()).bytes();
+            }
+        }
+
+        BulkItemResponse parsedBulkItemResponse;
+        try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) {
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            parsedBulkItemResponse = BulkItemResponse.fromXContent(parser, itemId);
+            assertNull(parser.nextToken());
+        }
+        assertBulkItemResponse(expectedBulkItemResponse, parsedBulkItemResponse);
+    }
+
+    public static void assertBulkItemResponse(BulkItemResponse expected, BulkItemResponse actual) {
+        assertEquals(expected.getItemId(), actual.getItemId());
+        assertEquals(expected.getIndex(), actual.getIndex());
+        assertEquals(expected.getType(), actual.getType());
+        assertEquals(expected.getId(), actual.getId());
+        assertEquals(expected.getOpType(), actual.getOpType());
+        assertEquals(expected.getVersion(), actual.getVersion());
+        assertEquals(expected.isFailed(), actual.isFailed());
+
+        if (expected.isFailed()) {
+            BulkItemResponse.Failure expectedFailure = expected.getFailure();
+            BulkItemResponse.Failure actualFailure = actual.getFailure();
+
+            assertEquals(expectedFailure.getIndex(), actualFailure.getIndex());
+            assertEquals(expectedFailure.getType(), actualFailure.getType());
+            assertEquals(expectedFailure.getId(), actualFailure.getId());
+            assertEquals(expectedFailure.getMessage(), actualFailure.getMessage());
+            assertEquals(expectedFailure.getStatus(), actualFailure.getStatus());
+
+            assertDeepEquals((ElasticsearchException) expectedFailure.getCause(), (ElasticsearchException) actualFailure.getCause());
+        } else {
+            DocWriteResponse expectedDocResponse = expected.getResponse();
+            DocWriteResponse actualDocResponse = expected.getResponse();
+
+            IndexResponseTests.assertDocWriteResponse(expectedDocResponse, actualDocResponse);
+            if (expected.getOpType() == DocWriteRequest.OpType.UPDATE) {
+                assertEquals(((UpdateResponse) expectedDocResponse).getGetResult(), ((UpdateResponse)actualDocResponse).getGetResult());
+            }
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorClusterSettingsIT.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorClusterSettingsIT.java
index 48d989209cbba..ca1630b00641c 100644
--- a/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorClusterSettingsIT.java
+++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorClusterSettingsIT.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.action.bulk;
 
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
@@ -36,9 +37,9 @@ public void testBulkProcessorAutoCreateRestrictions() throws Exception {
         client().admin().cluster().prepareHealth("willwork").setWaitForGreenStatus().execute().actionGet();
 
         BulkRequestBuilder bulkRequestBuilder = client().prepareBulk();
-        bulkRequestBuilder.add(client().prepareIndex("willwork", "type1", "1").setSource("{\"foo\":1}"));
-        bulkRequestBuilder.add(client().prepareIndex("wontwork", "type1", "2").setSource("{\"foo\":2}"));
-        bulkRequestBuilder.add(client().prepareIndex("willwork", "type1", "3").setSource("{\"foo\":3}"));
+        bulkRequestBuilder.add(client().prepareIndex("willwork", "type1", "1").setSource("{\"foo\":1}", XContentType.JSON));
+        bulkRequestBuilder.add(client().prepareIndex("wontwork", "type1", "2").setSource("{\"foo\":2}", XContentType.JSON));
+        bulkRequestBuilder.add(client().prepareIndex("willwork", "type1", "3").setSource("{\"foo\":3}", XContentType.JSON));
         BulkResponse br = bulkRequestBuilder.get();
         BulkItemResponse[] responses = br.getItems();
         assertEquals(3, responses.length);
diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorIT.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorIT.java
index 6bac7c2f8a436..44e0bbf823063 100644
--- a/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorIT.java
+++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorIT.java
@@ -25,11 +25,15 @@
 import org.elasticsearch.action.get.MultiGetResponse;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.client.Requests;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.transport.MockTransportClient;
@@ -53,12 +57,12 @@
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 
 public class BulkProcessorIT extends ESIntegTestCase {
-    public void testThatBulkProcessorCountIsCorrect() throws InterruptedException {
+    public void testThatBulkProcessorCountIsCorrect() throws Exception {
         final CountDownLatch latch = new CountDownLatch(1);
         BulkProcessorTestListener listener = new BulkProcessorTestListener(latch);
 
         int numDocs = randomIntBetween(10, 100);
-        try (BulkProcessor processor = BulkProcessor.builder(client(), listener).setName("foo")
+        try (BulkProcessor processor = BulkProcessor.builder(client(), listener)
                 //let's make sure that the bulk action limit trips, one single execution will index all the documents
                 .setConcurrentRequests(randomIntBetween(0, 1)).setBulkActions(numDocs)
                 .setFlushInterval(TimeValue.timeValueHours(24)).setBulkSize(new ByteSizeValue(1, ByteSizeUnit.GB))
@@ -76,13 +80,13 @@ public void testThatBulkProcessorCountIsCorrect() throws InterruptedException {
         }
     }
 
-    public void testBulkProcessorFlush() throws InterruptedException {
+    public void testBulkProcessorFlush() throws Exception {
         final CountDownLatch latch = new CountDownLatch(1);
         BulkProcessorTestListener listener = new BulkProcessorTestListener(latch);
 
         int numDocs = randomIntBetween(10, 100);
 
-        try (BulkProcessor processor = BulkProcessor.builder(client(), listener).setName("foo")
+        try (BulkProcessor processor = BulkProcessor.builder(client(), listener)
                 //let's make sure that this bulk won't be automatically flushed
                 .setConcurrentRequests(randomIntBetween(0, 10)).setBulkActions(numDocs + randomIntBetween(1, 100))
                 .setFlushInterval(TimeValue.timeValueHours(24)).setBulkSize(new ByteSizeValue(1, ByteSizeUnit.GB)).build()) {
@@ -199,7 +203,7 @@ public void testBulkProcessorWaitOnClose() throws Exception {
         BulkProcessorTestListener listener = new BulkProcessorTestListener();
 
         int numDocs = randomIntBetween(10, 100);
-        BulkProcessor processor = BulkProcessor.builder(client(), listener).setName("foo")
+        BulkProcessor processor = BulkProcessor.builder(client(), listener)
                 //let's make sure that the bulk action limit trips, one single execution will index all the documents
                 .setConcurrentRequests(randomIntBetween(0, 1)).setBulkActions(numDocs)
                 .setFlushInterval(TimeValue.timeValueHours(24)).setBulkSize(new ByteSizeValue(randomIntBetween(1, 10),
@@ -254,11 +258,13 @@ public void testBulkProcessorConcurrentRequestsReadOnlyIndex() throws Exception
             for (int i = 1; i <= numDocs; i++) {
                 if (randomBoolean()) {
                     testDocs++;
-                    processor.add(new IndexRequest("test", "test", Integer.toString(testDocs)).source("field", "value"));
+                    processor.add(new IndexRequest("test", "test", Integer.toString(testDocs))
+                        .source(Requests.INDEX_CONTENT_TYPE, "field", "value"));
                     multiGetRequestBuilder.add("test", "test", Integer.toString(testDocs));
                 } else {
                     testReadOnlyDocs++;
-                    processor.add(new IndexRequest("test-ro", "test", Integer.toString(testReadOnlyDocs)).source("field", "value"));
+                    processor.add(new IndexRequest("test-ro", "test", Integer.toString(testReadOnlyDocs))
+                        .source(Requests.INDEX_CONTENT_TYPE, "field", "value"));
                 }
             }
         }
@@ -293,10 +299,18 @@ public void testBulkProcessorConcurrentRequestsReadOnlyIndex() throws Exception
         assertMultiGetResponse(multiGetRequestBuilder.get(), testDocs);
     }
 
-    private static MultiGetRequestBuilder indexDocs(Client client, BulkProcessor processor, int numDocs) {
+    private static MultiGetRequestBuilder indexDocs(Client client, BulkProcessor processor, int numDocs) throws Exception {
         MultiGetRequestBuilder multiGetRequestBuilder = client.prepareMultiGet();
         for (int i = 1; i <= numDocs; i++) {
-            processor.add(new IndexRequest("test", "test", Integer.toString(i)).source("field", randomRealisticUnicodeOfLengthBetween(1, 30)));
+            if (randomBoolean()) {
+                processor.add(new IndexRequest("test", "test", Integer.toString(i))
+                    .source(Requests.INDEX_CONTENT_TYPE, "field", randomRealisticUnicodeOfLengthBetween(1, 30)));
+            } else {
+                final String source = "{ \"index\":{\"_index\":\"test\",\"_type\":\"test\",\"_id\":\"" + Integer.toString(i) + "\"} }\n"
+                    + JsonXContent.contentBuilder()
+                        .startObject().field("field", randomRealisticUnicodeOfLengthBetween(1, 30)).endObject().string() + "\n";
+                processor.add(new BytesArray(source), null, null, XContentType.JSON);
+            }
             multiGetRequestBuilder.add("test", "test", Integer.toString(i));
         }
         return multiGetRequestBuilder;
@@ -309,7 +323,8 @@ private static void assertResponseItems(List<BulkItemResponse> bulkItemResponses
             assertThat(bulkItemResponse.getIndex(), equalTo("test"));
             assertThat(bulkItemResponse.getType(), equalTo("test"));
             assertThat(bulkItemResponse.getId(), equalTo(Integer.toString(i++)));
-            assertThat(bulkItemResponse.isFailed(), equalTo(false));
+            assertThat("item " + i + " failed with cause: " + bulkItemResponse.getFailureMessage(),
+                bulkItemResponse.isFailed(), equalTo(false));
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorRetryIT.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorRetryIT.java
index 87249bc8b5a13..bb2aff8026978 100644
--- a/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorRetryIT.java
+++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorRetryIT.java
@@ -59,7 +59,6 @@ protected Settings nodeSettings(int nodeOrdinal) {
                 .build();
     }
 
-
     public void testBulkRejectionLoadWithoutBackoff() throws Throwable {
         boolean rejectedExecutionExpected = true;
         executeBulkRejectionLoad(BackoffPolicy.noBackoff(), rejectedExecutionExpected);
diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorTests.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorTests.java
new file mode 100644
index 0000000000000..9b37a2b9f4c48
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorTests.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.bulk;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.junit.After;
+import org.junit.Before;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.function.BiConsumer;
+
+public class BulkProcessorTests extends ESTestCase {
+
+    private ThreadPool threadPool;
+
+    @Before
+    public void startThreadPool() {
+        threadPool = new TestThreadPool("BulkProcessorTests");
+    }
+
+    @After
+    public void stopThreadPool() throws InterruptedException {
+        terminate(threadPool);
+    }
+
+    public void testBulkProcessorFlushPreservesContext() throws InterruptedException {
+        final CountDownLatch latch = new CountDownLatch(1);
+        final String headerKey = randomAlphaOfLengthBetween(1, 8);
+        final String transientKey = randomAlphaOfLengthBetween(1, 8);
+        final String headerValue = randomAlphaOfLengthBetween(1, 32);
+        final Object transientValue = new Object();
+
+        BiConsumer<BulkRequest, ActionListener<BulkResponse>> consumer = (request, listener) -> {
+            ThreadContext threadContext = threadPool.getThreadContext();
+            assertEquals(headerValue, threadContext.getHeader(headerKey));
+            assertSame(transientValue, threadContext.getTransient(transientKey));
+            latch.countDown();
+        };
+
+        final int bulkSize = randomIntBetween(2, 32);
+        final TimeValue flushInterval = TimeValue.timeValueSeconds(1L);
+        final BulkProcessor bulkProcessor;
+        assertNull(threadPool.getThreadContext().getHeader(headerKey));
+        assertNull(threadPool.getThreadContext().getTransient(transientKey));
+        try (ThreadContext.StoredContext ctx = threadPool.getThreadContext().stashContext()) {
+            threadPool.getThreadContext().putHeader(headerKey, headerValue);
+            threadPool.getThreadContext().putTransient(transientKey, transientValue);
+            bulkProcessor = new BulkProcessor(consumer, BackoffPolicy.noBackoff(), new BulkProcessor.Listener() {
+                @Override
+                public void beforeBulk(long executionId, BulkRequest request) {
+                }
+
+                @Override
+                public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
+                }
+
+                @Override
+                public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
+                }
+            }, 1, bulkSize, new ByteSizeValue(5, ByteSizeUnit.MB), flushInterval, threadPool);
+        }
+        assertNull(threadPool.getThreadContext().getHeader(headerKey));
+        assertNull(threadPool.getThreadContext().getTransient(transientKey));
+
+        // add a single item which won't be over the size or number of items
+        bulkProcessor.add(new IndexRequest());
+
+        // wait for flush to execute
+        latch.await();
+
+        assertNull(threadPool.getThreadContext().getHeader(headerKey));
+        assertNull(threadPool.getThreadContext().getTransient(transientKey));
+        bulkProcessor.close();
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/ingest/BulkRequestModifierTests.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestModifierTests.java
similarity index 86%
rename from core/src/test/java/org/elasticsearch/action/ingest/BulkRequestModifierTests.java
rename to core/src/test/java/org/elasticsearch/action/bulk/BulkRequestModifierTests.java
index 9ee5036131df2..1c37e7f4f28c3 100644
--- a/core/src/test/java/org/elasticsearch/action/ingest/BulkRequestModifierTests.java
+++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestModifierTests.java
@@ -1,5 +1,3 @@
-package org.elasticsearch.action.ingest;
-
 /*
  * Licensed to Elasticsearch under one or more contributor
  * license agreements. See the NOTICE file distributed with
@@ -19,16 +17,7 @@
  * under the License.
  */
 
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ActionRequest;
-import org.elasticsearch.action.bulk.BulkItemResponse;
-import org.elasticsearch.action.bulk.BulkRequest;
-import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.index.IndexResponse;
-import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.test.ESTestCase;
-import org.hamcrest.Matchers;
+package org.elasticsearch.action.bulk;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -36,6 +25,15 @@
 import java.util.List;
 import java.util.Set;
 
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.test.ESTestCase;
+import org.hamcrest.Matchers;
+
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
@@ -47,10 +45,10 @@ public void testBulkRequestModifier() {
         int numRequests = scaledRandomIntBetween(8, 64);
         BulkRequest bulkRequest = new BulkRequest();
         for (int i = 0; i < numRequests; i++) {
-            bulkRequest.add(new IndexRequest("_index", "_type", String.valueOf(i)).source("{}"));
+            bulkRequest.add(new IndexRequest("_index", "_type", String.valueOf(i)).source("{}", XContentType.JSON));
         }
         CaptureActionListener actionListener = new CaptureActionListener();
-        IngestActionFilter.BulkRequestModifier bulkRequestModifier = new IngestActionFilter.BulkRequestModifier(bulkRequest);
+        TransportBulkAction.BulkRequestModifier bulkRequestModifier = new TransportBulkAction.BulkRequestModifier(bulkRequest);
 
         int i = 0;
         Set<Integer> failedSlots = new HashSet<>();
@@ -91,7 +89,7 @@ public void testPipelineFailures() {
             originalBulkRequest.add(new IndexRequest("index", "type", String.valueOf(i)));
         }
 
-        IngestActionFilter.BulkRequestModifier modifier = new IngestActionFilter.BulkRequestModifier(originalBulkRequest);
+        TransportBulkAction.BulkRequestModifier modifier = new TransportBulkAction.BulkRequestModifier(originalBulkRequest);
         for (int i = 0; modifier.hasNext(); i++) {
             modifier.next();
             if (i % 2 == 0) {
@@ -116,10 +114,12 @@ public void onFailure(Exception e) {
         });
 
         List<BulkItemResponse> originalResponses = new ArrayList<>();
-        for (ActionRequest actionRequest : bulkRequest.requests()) {
+        for (DocWriteRequest actionRequest : bulkRequest.requests()) {
             IndexRequest indexRequest = (IndexRequest) actionRequest;
-            IndexResponse indexResponse = new IndexResponse(new ShardId("index", "_na_", 0), indexRequest.type(), indexRequest.id(), 1, true);
-            originalResponses.add(new BulkItemResponse(Integer.parseInt(indexRequest.id()), indexRequest.opType().lowercase(), indexResponse));
+            IndexResponse indexResponse =
+                new IndexResponse(new ShardId("index", "_na_", 0), indexRequest.type(), indexRequest.id(), 1, true);
+            originalResponses.add(new BulkItemResponse(Integer.parseInt(indexRequest.id()),
+                indexRequest.opType(), indexResponse));
         }
         bulkResponseListener.onResponse(new BulkResponse(originalResponses.toArray(new BulkItemResponse[originalResponses.size()]), 0));
 
@@ -135,7 +135,7 @@ public void testNoFailures() {
             originalBulkRequest.add(new IndexRequest("index", "type", String.valueOf(i)));
         }
 
-        IngestActionFilter.BulkRequestModifier modifier = new IngestActionFilter.BulkRequestModifier(originalBulkRequest);
+        TransportBulkAction.BulkRequestModifier modifier = new TransportBulkAction.BulkRequestModifier(originalBulkRequest);
         while (modifier.hasNext()) {
             modifier.next();
         }
diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java
index 142fb282c20cb..da3f0fed3ac4a 100644
--- a/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java
@@ -19,19 +19,25 @@
 
 package org.elasticsearch.action.bulk;
 
-import org.apache.lucene.util.Constants;
-import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.client.Requests;
-import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.test.ESTestCase;
 
+import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.List;
@@ -39,6 +45,7 @@
 
 import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath;
 import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
@@ -48,36 +55,43 @@
 public class BulkRequestTests extends ESTestCase {
     public void testSimpleBulk1() throws Exception {
         String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk.json");
-        // translate Windows line endings (\r\n) to standard ones (\n)
-        if (Constants.WINDOWS) {
-            bulkAction = Strings.replace(bulkAction, "\r\n", "\n");
-        }
         BulkRequest bulkRequest = new BulkRequest();
-        bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null);
+        bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON);
         assertThat(bulkRequest.numberOfActions(), equalTo(3));
         assertThat(((IndexRequest) bulkRequest.requests().get(0)).source(), equalTo(new BytesArray("{ \"field1\" : \"value1\" }")));
         assertThat(bulkRequest.requests().get(1), instanceOf(DeleteRequest.class));
         assertThat(((IndexRequest) bulkRequest.requests().get(2)).source(), equalTo(new BytesArray("{ \"field1\" : \"value3\" }")));
     }
 
+    public void testSimpleBulkWithCarriageReturn() throws Exception {
+        String bulkAction = "{ \"index\":{\"_index\":\"test\",\"_type\":\"type1\",\"_id\":\"1\"} }\r\n{ \"field1\" : \"value1\" }\r\n";
+        BulkRequest bulkRequest = new BulkRequest();
+        bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON);
+        assertThat(bulkRequest.numberOfActions(), equalTo(1));
+        assertThat(((IndexRequest) bulkRequest.requests().get(0)).source(), equalTo(new BytesArray("{ \"field1\" : \"value1\" }")));
+        Map<String, Object> sourceMap = XContentHelper.convertToMap(((IndexRequest) bulkRequest.requests().get(0)).source(),
+            false, XContentType.JSON).v2();
+        assertEquals("value1", sourceMap.get("field1"));
+    }
+
     public void testSimpleBulk2() throws Exception {
         String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk2.json");
         BulkRequest bulkRequest = new BulkRequest();
-        bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null);
+        bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON);
         assertThat(bulkRequest.numberOfActions(), equalTo(3));
     }
 
     public void testSimpleBulk3() throws Exception {
         String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk3.json");
         BulkRequest bulkRequest = new BulkRequest();
-        bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null);
+        bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON);
         assertThat(bulkRequest.numberOfActions(), equalTo(3));
     }
 
     public void testSimpleBulk4() throws Exception {
         String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk4.json");
         BulkRequest bulkRequest = new BulkRequest();
-        bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null);
+        bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON);
         assertThat(bulkRequest.numberOfActions(), equalTo(4));
         assertThat(((UpdateRequest) bulkRequest.requests().get(0)).id(), equalTo("1"));
         assertThat(((UpdateRequest) bulkRequest.requests().get(0)).retryOnConflict(), equalTo(2));
@@ -87,7 +101,7 @@ public void testSimpleBulk4() throws Exception {
         assertThat(((UpdateRequest) bulkRequest.requests().get(1)).index(), equalTo("index1"));
         Script script = ((UpdateRequest) bulkRequest.requests().get(1)).script();
         assertThat(script, notNullValue());
-        assertThat(script.getScript(), equalTo("counter += param1"));
+        assertThat(script.getIdOrCode(), equalTo("counter += param1"));
         assertThat(script.getLang(), equalTo("javascript"));
         Map<String, Object> scriptParams = script.getParams();
         assertThat(scriptParams, notNullValue());
@@ -99,21 +113,21 @@ public void testSimpleBulk4() throws Exception {
     public void testBulkAllowExplicitIndex() throws Exception {
         String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk.json");
         try {
-            new BulkRequest().add(new BytesArray(bulkAction.getBytes(StandardCharsets.UTF_8)), null, null, false);
+            new BulkRequest().add(new BytesArray(bulkAction.getBytes(StandardCharsets.UTF_8)), null, null, false, XContentType.JSON);
             fail();
         } catch (Exception e) {
 
         }
 
         bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk5.json");
-        new BulkRequest().add(new BytesArray(bulkAction.getBytes(StandardCharsets.UTF_8)), "test", null, false);
+        new BulkRequest().add(new BytesArray(bulkAction.getBytes(StandardCharsets.UTF_8)), "test", null, false, XContentType.JSON);
     }
 
     public void testBulkAddIterable() {
         BulkRequest bulkRequest = Requests.bulkRequest();
-        List<ActionRequest<?>> requests = new ArrayList<>();
-        requests.add(new IndexRequest("test", "test", "id").source("field", "value"));
-        requests.add(new UpdateRequest("test", "test", "id").doc("field", "value"));
+        List<DocWriteRequest> requests = new ArrayList<>();
+        requests.add(new IndexRequest("test", "test", "id").source(Requests.INDEX_CONTENT_TYPE, "field", "value"));
+        requests.add(new UpdateRequest("test", "test", "id").doc(Requests.INDEX_CONTENT_TYPE, "field", "value"));
         requests.add(new DeleteRequest("test", "test", "id"));
         bulkRequest.add(requests);
         assertThat(bulkRequest.requests().size(), equalTo(3));
@@ -125,58 +139,68 @@ public void testBulkAddIterable() {
     public void testSimpleBulk6() throws Exception {
         String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk6.json");
         BulkRequest bulkRequest = new BulkRequest();
-        try {
-            bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null);
-            fail("should have thrown an exception about the wrong format of line 1");
-        } catch (IllegalArgumentException e) {
-            assertThat("message contains error about the wrong format of line 1: " + e.getMessage(),
-                    e.getMessage().contains("Malformed action/metadata line [1], expected a simple value for field [_source] but found [START_OBJECT]"), equalTo(true));
-        }
+        ParsingException exc = expectThrows(ParsingException.class,
+            () -> bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON));
+        assertThat(exc.getMessage(), containsString("Unknown key for a VALUE_STRING in [hello]"));
     }
 
     public void testSimpleBulk7() throws Exception {
         String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk7.json");
         BulkRequest bulkRequest = new BulkRequest();
-        try {
-            bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null);
-            fail("should have thrown an exception about the wrong format of line 5");
-        } catch (IllegalArgumentException e) {
-            assertThat("message contains error about the wrong format of line 5: " + e.getMessage(),
-                    e.getMessage().contains("Malformed action/metadata line [5], expected a simple value for field [_unkown] but found [START_ARRAY]"), equalTo(true));
-        }
+        IllegalArgumentException exc = expectThrows(IllegalArgumentException.class,
+            () -> bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON));
+        assertThat(exc.getMessage(),
+            containsString("Malformed action/metadata line [5], expected a simple value for field [_unkown] but found [START_ARRAY]"));
     }
 
     public void testSimpleBulk8() throws Exception {
         String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk8.json");
         BulkRequest bulkRequest = new BulkRequest();
-        try {
-            bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null);
-            fail("should have thrown an exception about the unknown parameter _foo");
-        } catch (IllegalArgumentException e) {
-            assertThat("message contains error about the unknown parameter _foo: " + e.getMessage(),
-                    e.getMessage().contains("Action/metadata line [3] contains an unknown parameter [_foo]"), equalTo(true));
-        }
+        IllegalArgumentException exc = expectThrows(IllegalArgumentException.class,
+            () -> bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON));
+        assertThat(exc.getMessage(), containsString("Action/metadata line [3] contains an unknown parameter [_foo]"));
     }
 
     public void testSimpleBulk9() throws Exception {
         String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk9.json");
         BulkRequest bulkRequest = new BulkRequest();
-        try {
-            bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null);
-            fail("should have thrown an exception about the wrong format of line 3");
-        } catch (IllegalArgumentException e) {
-            assertThat("message contains error about the wrong format of line 3: " + e.getMessage(),
-                    e.getMessage().contains("Malformed action/metadata line [3], expected START_OBJECT or END_OBJECT but found [START_ARRAY]"), equalTo(true));
-        }
+        IllegalArgumentException exc = expectThrows(IllegalArgumentException.class,
+            () -> bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON));
+        assertThat(exc.getMessage(), containsString("Malformed action/metadata line [3], expected START_OBJECT or END_OBJECT but found [START_ARRAY]"));
     }
 
     public void testSimpleBulk10() throws Exception {
         String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk10.json");
         BulkRequest bulkRequest = new BulkRequest();
-        bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null);
+        bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON);
         assertThat(bulkRequest.numberOfActions(), equalTo(9));
     }
 
+    public void testBulkEmptyObject() throws Exception {
+        String bulkIndexAction = "{ \"index\":{\"_index\":\"test\",\"_type\":\"type1\",\"_id\":\"1\"} }\r\n";
+        String bulkIndexSource = "{ \"field1\" : \"value1\" }\r\n";
+        String emptyObject = "{}\r\n";
+        StringBuilder bulk = new StringBuilder();
+        int emptyLine;
+        if (randomBoolean()) {
+            bulk.append(emptyObject);
+            emptyLine = 1;
+        } else {
+            int actions = randomIntBetween(1, 10);
+            int emptyAction = randomIntBetween(1, actions);
+            emptyLine = emptyAction * 2 - 1;
+            for (int i = 1; i <= actions; i++) {
+                bulk.append(i == emptyAction ? emptyObject : bulkIndexAction);
+                bulk.append(randomBoolean() ? emptyObject : bulkIndexSource);
+            }
+        }
+        String bulkAction = bulk.toString();
+        BulkRequest bulkRequest = new BulkRequest();
+        IllegalArgumentException exc = expectThrows(IllegalArgumentException.class,
+            () -> bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON));
+        assertThat(exc.getMessage(), containsString("Malformed action/metadata line [" + emptyLine + "], expected FIELD_NAME but found [END_OBJECT]"));
+    }
+
     // issue 7361
     public void testBulkRequestWithRefresh() throws Exception {
         BulkRequest bulkRequest = new BulkRequest();
@@ -185,8 +209,8 @@ public void testBulkRequestWithRefresh() throws Exception {
         // We force here a "type is missing" validation error
         bulkRequest.add(new DeleteRequest("index", null, "id"));
         bulkRequest.add(new DeleteRequest("index", "type", "id").setRefreshPolicy(RefreshPolicy.IMMEDIATE));
-        bulkRequest.add(new UpdateRequest("index", "type", "id").doc("{}").setRefreshPolicy(RefreshPolicy.IMMEDIATE));
-        bulkRequest.add(new IndexRequest("index", "type", "id").source("{}").setRefreshPolicy(RefreshPolicy.IMMEDIATE));
+        bulkRequest.add(new UpdateRequest("index", "type", "id").doc("{}", XContentType.JSON).setRefreshPolicy(RefreshPolicy.IMMEDIATE));
+        bulkRequest.add(new IndexRequest("index", "type", "id").source("{}", XContentType.JSON).setRefreshPolicy(RefreshPolicy.IMMEDIATE));
         ActionRequestValidationException validate = bulkRequest.validate();
         assertThat(validate, notNullValue());
         assertThat(validate.validationErrors(), not(empty()));
@@ -209,7 +233,8 @@ public void testBulkNoSource() throws Exception {
         assertThat(validate.validationErrors(), not(empty()));
         assertThat(validate.validationErrors(), contains(
                 "script or doc is missing",
-                "source is missing"));
+                "source is missing",
+                "content type is missing"));
     }
 
     public void testCannotAddNullRequests() throws Exception {
@@ -218,4 +243,40 @@ public void testCannotAddNullRequests() throws Exception {
         expectThrows(NullPointerException.class, () -> bulkRequest.add((UpdateRequest) null));
         expectThrows(NullPointerException.class, () -> bulkRequest.add((DeleteRequest) null));
     }
+
+    public void testSmileIsSupported() throws IOException {
+        XContentType xContentType = XContentType.SMILE;
+        BytesReference data;
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            try(XContentBuilder  builder = XContentFactory.contentBuilder(xContentType, out)) {
+                builder.startObject();
+                builder.startObject("index");
+                builder.field("_index", "index");
+                builder.field("_type", "type");
+                builder.field("_id", "test");
+                builder.endObject();
+                builder.endObject();
+            }
+            out.write(xContentType.xContent().streamSeparator());
+            try(XContentBuilder builder = XContentFactory.contentBuilder(xContentType, out)) {
+                builder.startObject();
+                builder.field("field", "value");
+                builder.endObject();
+            }
+            out.write(xContentType.xContent().streamSeparator());
+            data = out.bytes();
+        }
+
+        BulkRequest bulkRequest = new BulkRequest();
+        bulkRequest.add(data, null, null);
+        assertEquals(1, bulkRequest.requests().size());
+        assertThat(bulkRequest.requests().get(0), instanceOf(IndexRequest.class));
+        IndexRequest request = (IndexRequest) bulkRequest.requests().get(0);
+        assertEquals(IndexRequest.OpType.INDEX, request.opType());
+        assertEquals("index", request.index());
+        assertEquals("type", request.type());
+        assertEquals("test", request.id());
+        assertEquals(1, request.sourceAsMap().size());
+        assertEquals("value", request.sourceAsMap().get("field"));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkResponseTests.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkResponseTests.java
new file mode 100644
index 0000000000000..dd8f92bb2e507
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkResponseTests.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.bulk;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.action.delete.DeleteResponseTests;
+import org.elasticsearch.action.index.IndexResponseTests;
+import org.elasticsearch.action.update.UpdateResponseTests;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+import static org.elasticsearch.ElasticsearchExceptionTests.randomExceptions;
+import static org.elasticsearch.action.bulk.BulkItemResponseTests.assertBulkItemResponse;
+import static org.elasticsearch.action.bulk.BulkResponse.NO_INGEST_TOOK;
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
+
+public class BulkResponseTests extends ESTestCase {
+
+    public void testToAndFromXContent() throws IOException {
+        XContentType xContentType = randomFrom(XContentType.values());
+        boolean humanReadable = randomBoolean();
+
+        long took = randomFrom(randomNonNegativeLong(), -1L);
+        long ingestTook = randomFrom(randomNonNegativeLong(), NO_INGEST_TOOK);
+        int nbBulkItems = randomIntBetween(1, 10);
+
+        BulkItemResponse[] bulkItems = new BulkItemResponse[nbBulkItems];
+        BulkItemResponse[] expectedBulkItems = new BulkItemResponse[nbBulkItems];
+
+        for (int i = 0; i < nbBulkItems; i++) {
+            DocWriteRequest.OpType opType = randomFrom(DocWriteRequest.OpType.values());
+
+            if (frequently()) {
+                Tuple<? extends DocWriteResponse, ? extends DocWriteResponse> randomDocWriteResponses = null;
+                if (opType == DocWriteRequest.OpType.INDEX || opType == DocWriteRequest.OpType.CREATE) {
+                    randomDocWriteResponses = IndexResponseTests.randomIndexResponse();
+                } else if (opType == DocWriteRequest.OpType.DELETE) {
+                    randomDocWriteResponses = DeleteResponseTests.randomDeleteResponse();
+                } else if (opType == DocWriteRequest.OpType.UPDATE) {
+                    randomDocWriteResponses = UpdateResponseTests.randomUpdateResponse(xContentType);
+                } else {
+                    fail("Test does not support opType [" + opType + "]");
+                }
+
+                bulkItems[i] = new BulkItemResponse(i, opType, randomDocWriteResponses.v1());
+                expectedBulkItems[i] = new BulkItemResponse(i, opType, randomDocWriteResponses.v2());
+            } else {
+                String index = randomAlphaOfLength(5);
+                String type = randomAlphaOfLength(5);
+                String id = randomAlphaOfLength(5);
+
+                Tuple<Throwable, ElasticsearchException> failures = randomExceptions();
+
+                Exception bulkItemCause = (Exception) failures.v1();
+                bulkItems[i] = new BulkItemResponse(i, opType,
+                        new BulkItemResponse.Failure(index, type, id, bulkItemCause));
+                expectedBulkItems[i] = new BulkItemResponse(i, opType,
+                        new BulkItemResponse.Failure(index, type, id, failures.v2(), ExceptionsHelper.status(bulkItemCause)));
+            }
+        }
+
+        BulkResponse bulkResponse = new BulkResponse(bulkItems, took, ingestTook);
+        BytesReference originalBytes = toShuffledXContent(bulkResponse, xContentType, ToXContent.EMPTY_PARAMS, humanReadable);
+
+        BulkResponse parsedBulkResponse;
+        try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) {
+            parsedBulkResponse = BulkResponse.fromXContent(parser);
+            assertNull(parser.nextToken());
+        }
+
+        assertEquals(took, parsedBulkResponse.getTookInMillis());
+        assertEquals(ingestTook, parsedBulkResponse.getIngestTookInMillis());
+        assertEquals(expectedBulkItems.length, parsedBulkResponse.getItems().length);
+
+        for (int i = 0; i < expectedBulkItems.length; i++) {
+            assertBulkItemResponse(expectedBulkItems[i], parsedBulkResponse.getItems()[i]);
+        }
+
+        BytesReference finalBytes = toXContent(parsedBulkResponse, xContentType, humanReadable);
+        BytesReference expectedFinalBytes = toXContent(parsedBulkResponse, xContentType, humanReadable);
+        assertToXContentEquivalent(expectedFinalBytes, finalBytes, xContentType);
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkShardRequestTests.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkShardRequestTests.java
index bb406366d2570..1f1789fcc6b65 100644
--- a/core/src/test/java/org/elasticsearch/action/bulk/BulkShardRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkShardRequestTests.java
@@ -28,12 +28,13 @@
 public class BulkShardRequestTests extends ESTestCase {
     public void testToString() {
         String index = randomSimpleString(random(), 10);
-        int count = between(1, 100);
-        BulkShardRequest r = new BulkShardRequest(new ShardId(index, "ignored", 0), RefreshPolicy.NONE, new BulkItemRequest[count]);
-        assertEquals("BulkShardRequest to [" + index + "] containing [" + count + "] requests", r.toString());
-        r = new BulkShardRequest(new ShardId(index, "ignored", 0), RefreshPolicy.IMMEDIATE, new BulkItemRequest[count]);
-        assertEquals("BulkShardRequest to [" + index + "] containing [" + count + "] requests and a refresh", r.toString());
-        r = new BulkShardRequest(new ShardId(index, "ignored", 0), RefreshPolicy.WAIT_UNTIL, new BulkItemRequest[count]);
-        assertEquals("BulkShardRequest to [" + index + "] containing [" + count + "] requests blocking until refresh", r.toString());
+        int count = between(2, 100);
+        final ShardId shardId = new ShardId(index, "ignored", 0);
+        BulkShardRequest r = new BulkShardRequest(shardId, RefreshPolicy.NONE, new BulkItemRequest[count]);
+        assertEquals("BulkShardRequest [" + shardId + "] containing [" + count + "] requests", r.toString());
+        r = new BulkShardRequest(shardId, RefreshPolicy.IMMEDIATE, new BulkItemRequest[count]);
+        assertEquals("BulkShardRequest [" + shardId + "] containing [" + count + "] requests and a refresh", r.toString());
+        r = new BulkShardRequest(shardId, RefreshPolicy.WAIT_UNTIL, new BulkItemRequest[count]);
+        assertEquals("BulkShardRequest [" + shardId + "] containing [" + count + "] requests blocking until refresh", r.toString());
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java
index 3b88efca20201..83d57a69e4ea2 100644
--- a/core/src/test/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java
+++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java
@@ -29,10 +29,11 @@
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.action.update.UpdateRequestBuilder;
 import org.elasticsearch.action.update.UpdateResponse;
+import org.elasticsearch.client.Requests;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.VersionType;
-import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.MockScriptPlugin;
 import org.elasticsearch.script.Script;
@@ -47,12 +48,13 @@
 import java.util.concurrent.CyclicBarrier;
 import java.util.function.Function;
 
+import static org.elasticsearch.action.DocWriteRequest.OpType;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.elasticsearch.script.ScriptService.ScriptType;
+
+import org.elasticsearch.script.ScriptType;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
@@ -120,7 +122,7 @@ public void testBulkUpdateSimple() throws Exception {
             assertThat(bulkItemResponse.getIndex(), equalTo("test"));
         }
 
-        final Script script = new Script("ctx._source.field += 1", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
+        final Script script = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "ctx._source.field += 1", Collections.emptyMap());
 
         bulkResponse = client().prepareBulk()
                 .add(client().prepareUpdate().setIndex(indexOrAlias()).setType("type1").setId("1").setScript(script))
@@ -207,9 +209,9 @@ public void testBulkVersioning() throws Exception {
         assertThat(bulkResponse.getItems()[2].getResponse().getVersion(), equalTo(2L));
 
         bulkResponse = client().prepareBulk()
-                .add(client().prepareUpdate("test", "type", "1").setVersion(4L).setDoc("field", "2"))
-                .add(client().prepareUpdate("test", "type", "2").setDoc("field", "2"))
-                .add(client().prepareUpdate("test", "type", "1").setVersion(2L).setDoc("field", "3")).get();
+                .add(client().prepareUpdate("test", "type", "1").setVersion(4L).setDoc(Requests.INDEX_CONTENT_TYPE, "field", "2"))
+                .add(client().prepareUpdate("test", "type", "2").setDoc(Requests.INDEX_CONTENT_TYPE, "field", "2"))
+                .add(client().prepareUpdate("test", "type", "1").setVersion(2L).setDoc(Requests.INDEX_CONTENT_TYPE, "field", "3")).get();
 
         assertThat(bulkResponse.getItems()[0].getFailureMessage(), containsString("version conflict"));
         assertThat(bulkResponse.getItems()[1].getResponse().getVersion(), equalTo(2L));
@@ -233,13 +235,16 @@ public void testBulkVersioning() throws Exception {
 
         bulkResponse = client().prepareBulk()
                 .add(client().prepareUpdate("test", "type", "e1")
-                        .setDoc("field", "2").setVersion(10)) // INTERNAL
+                        .setDoc(Requests.INDEX_CONTENT_TYPE, "field", "2").setVersion(10)) // INTERNAL
+                .add(client().prepareUpdate("test", "type", "e1")
+                        .setDoc(Requests.INDEX_CONTENT_TYPE, "field", "3").setVersion(20).setVersionType(VersionType.FORCE))
                 .add(client().prepareUpdate("test", "type", "e1")
-                        .setDoc("field", "3").setVersion(13).setVersionType(VersionType.INTERNAL))
+                        .setDoc("field", "4").setVersion(20).setVersionType(VersionType.INTERNAL))
                 .get();
 
         assertThat(bulkResponse.getItems()[0].getFailureMessage(), containsString("version conflict"));
-        assertThat(bulkResponse.getItems()[1].getFailureMessage(), containsString("version conflict"));
+        assertThat(bulkResponse.getItems()[1].getResponse().getVersion(), equalTo(20L));
+        assertThat(bulkResponse.getItems()[2].getResponse().getVersion(), equalTo(21L));
     }
 
     public void testBulkUpdateMalformedScripts() throws Exception {
@@ -257,11 +262,14 @@ public void testBulkUpdateMalformedScripts() throws Exception {
 
         bulkResponse = client().prepareBulk()
                 .add(client().prepareUpdate().setIndex("test").setType("type1").setId("1").setFields("field")
-                        .setScript(new Script("throw script exception on unknown var", ScriptType.INLINE, CustomScriptPlugin.NAME, null)))
+                        .setScript(new Script(
+                            ScriptType.INLINE, CustomScriptPlugin.NAME, "throw script exception on unknown var", Collections.emptyMap())))
                 .add(client().prepareUpdate().setIndex("test").setType("type1").setId("2").setFields("field")
-                        .setScript(new Script("ctx._source.field += 1", ScriptType.INLINE, CustomScriptPlugin.NAME, null)))
+                        .setScript(new Script(
+                            ScriptType.INLINE, CustomScriptPlugin.NAME, "ctx._source.field += 1", Collections.emptyMap())))
                 .add(client().prepareUpdate().setIndex("test").setType("type1").setId("3").setFields("field")
-                        .setScript(new Script("throw script exception on unknown var", ScriptType.INLINE, CustomScriptPlugin.NAME, null)))
+                        .setScript(new Script(
+                            ScriptType.INLINE, CustomScriptPlugin.NAME, "throw script exception on unknown var", Collections.emptyMap())))
                 .execute().actionGet();
 
         assertThat(bulkResponse.hasFailures(), equalTo(true));
@@ -289,13 +297,14 @@ public void testBulkUpdateLargerVolume() throws Exception {
             numDocs++; // this test needs an even num of docs
         }
 
-        final Script script = new Script("ctx._source.counter += 1", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
+        final Script script = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "ctx._source.counter += 1", Collections.emptyMap());
 
         BulkRequestBuilder builder = client().prepareBulk();
         for (int i = 0; i < numDocs; i++) {
             builder.add(
                     client().prepareUpdate()
-                            .setIndex("test").setType("type1").setId(Integer.toString(i)).setFields("counter")
+                            .setIndex("test").setType("type1").setId(Integer.toString(i))
+                            .setFields("counter")
                             .setScript(script)
                             .setUpsert(jsonBuilder().startObject().field("counter", 1).endObject()));
         }
@@ -308,7 +317,7 @@ public void testBulkUpdateLargerVolume() throws Exception {
             assertThat(response.getItems()[i].getVersion(), equalTo(1L));
             assertThat(response.getItems()[i].getIndex(), equalTo("test"));
             assertThat(response.getItems()[i].getType(), equalTo("type1"));
-            assertThat(response.getItems()[i].getOpType(), equalTo("update"));
+            assertThat(response.getItems()[i].getOpType(), equalTo(OpType.UPDATE));
             assertThat(response.getItems()[i].getResponse().getId(), equalTo(Integer.toString(i)));
             assertThat(response.getItems()[i].getResponse().getVersion(), equalTo(1L));
             assertThat(((UpdateResponse) response.getItems()[i].getResponse()).getGetResult().field("counter").getValue(), equalTo(1));
@@ -346,7 +355,7 @@ public void testBulkUpdateLargerVolume() throws Exception {
             assertThat(response.getItems()[i].getVersion(), equalTo(2L));
             assertThat(response.getItems()[i].getIndex(), equalTo("test"));
             assertThat(response.getItems()[i].getType(), equalTo("type1"));
-            assertThat(response.getItems()[i].getOpType(), equalTo("update"));
+            assertThat(response.getItems()[i].getOpType(), equalTo(OpType.UPDATE));
             assertThat(response.getItems()[i].getResponse().getId(), equalTo(Integer.toString(i)));
             assertThat(response.getItems()[i].getResponse().getVersion(), equalTo(2L));
             assertThat(((UpdateResponse) response.getItems()[i].getResponse()).getGetResult().field("counter").getValue(), equalTo(2));
@@ -370,14 +379,14 @@ public void testBulkUpdateLargerVolume() throws Exception {
                 assertThat(response.getItems()[i].getVersion(), equalTo(3L));
                 assertThat(response.getItems()[i].getIndex(), equalTo("test"));
                 assertThat(response.getItems()[i].getType(), equalTo("type1"));
-                assertThat(response.getItems()[i].getOpType(), equalTo("update"));
+                assertThat(response.getItems()[i].getOpType(), equalTo(OpType.UPDATE));
             }
         }
 
         builder = client().prepareBulk();
         for (int i = 0; i < numDocs; i++) {
             builder.add(client().prepareUpdate().setIndex("test").setType("type1").setId(Integer.toString(i))
-                    .setScript(new Script("ctx.op = \"none\"", ScriptType.INLINE, CustomScriptPlugin.NAME, null)));
+                    .setScript(new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "ctx.op = \"none\"", Collections.emptyMap())));
         }
         response = builder.execute().actionGet();
         assertThat(response.buildFailureMessage(), response.hasFailures(), equalTo(false));
@@ -387,13 +396,13 @@ public void testBulkUpdateLargerVolume() throws Exception {
             assertThat(response.getItems()[i].getId(), equalTo(Integer.toString(i)));
             assertThat(response.getItems()[i].getIndex(), equalTo("test"));
             assertThat(response.getItems()[i].getType(), equalTo("type1"));
-            assertThat(response.getItems()[i].getOpType(), equalTo("update"));
+            assertThat(response.getItems()[i].getOpType(), equalTo(OpType.UPDATE));
         }
 
         builder = client().prepareBulk();
         for (int i = 0; i < numDocs; i++) {
             builder.add(client().prepareUpdate().setIndex("test").setType("type1").setId(Integer.toString(i))
-                    .setScript(new Script("ctx.op = \"delete\"", ScriptType.INLINE, CustomScriptPlugin.NAME, null)));
+                    .setScript(new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "ctx.op = \"delete\"", Collections.emptyMap())));
         }
         response = builder.execute().actionGet();
         assertThat(response.hasFailures(), equalTo(false));
@@ -403,10 +412,9 @@ public void testBulkUpdateLargerVolume() throws Exception {
             assertThat(response.getItems()[i].getId(), equalTo(Integer.toString(i)));
             assertThat(response.getItems()[i].getIndex(), equalTo("test"));
             assertThat(response.getItems()[i].getType(), equalTo("type1"));
-            assertThat(response.getItems()[i].getOpType(), equalTo("update"));
+            assertThat(response.getItems()[i].getOpType(), equalTo(OpType.UPDATE));
             for (int j = 0; j < 5; j++) {
-                GetResponse getResponse = client().prepareGet("test", "type1", Integer.toString(i)).setFields("counter").execute()
-                        .actionGet();
+                GetResponse getResponse = client().prepareGet("test", "type1", Integer.toString(i)).get();
                 assertThat(getResponse.isExists(), equalTo(false));
             }
         }
@@ -443,155 +451,6 @@ public void testBulkIndexingWhileInitializing() throws Exception {
         assertHitCount(countResponse, numDocs);
     }
 
-    /*
-    Test for https://github.com/elastic/elasticsearch/issues/3444
-     */
-    public void testBulkUpdateDocAsUpsertWithParent() throws Exception {
-        client().admin().indices().prepareCreate("test")
-                .addMapping("parent", "{\"parent\":{}}")
-                .addMapping("child", "{\"child\": {\"_parent\": {\"type\": \"parent\"}}}")
-                .execute().actionGet();
-        ensureGreen();
-
-        BulkRequestBuilder builder = client().prepareBulk();
-
-        // It's important to use JSON parsing here and request objects: issue 3444 is related to incomplete option parsing
-        byte[] addParent = new BytesArray(
-                "{" +
-                "  \"index\" : {" +
-                "    \"_index\" : \"test\"," +
-                "    \"_type\"  : \"parent\"," +
-                "    \"_id\"    : \"parent1\"" +
-                "  }" +
-                "}" +
-                "\n" +
-                "{" +
-                "  \"field1\" : \"value1\"" +
-                "}" +
-                "\n").array();
-
-        byte[] addChild = new BytesArray(
-                "{" +
-                "  \"update\" : {" +
-                "    \"_index\" : \"test\"," +
-                "    \"_type\"  : \"child\"," +
-                "    \"_id\"    : \"child1\"," +
-                "    \"parent\" : \"parent1\"" +
-                "  }" +
-                "}" +
-                "\n" +
-                "{" +
-                "  \"doc\" : {" +
-                "    \"field1\" : \"value1\"" +
-                "  }," +
-                "  \"doc_as_upsert\" : \"true\"" +
-                "}" +
-                "\n").array();
-
-        builder.add(addParent, 0, addParent.length);
-        builder.add(addChild, 0, addChild.length);
-
-        BulkResponse bulkResponse = builder.get();
-        assertThat(bulkResponse.getItems().length, equalTo(2));
-        assertThat(bulkResponse.getItems()[0].isFailed(), equalTo(false));
-        assertThat(bulkResponse.getItems()[1].isFailed(), equalTo(false));
-
-        client().admin().indices().prepareRefresh("test").get();
-
-        //we check that the _parent field was set on the child document by using the has parent query
-        SearchResponse searchResponse = client().prepareSearch("test")
-                .setQuery(QueryBuilders.hasParentQuery("parent", QueryBuilders.matchAllQuery(), false))
-                .get();
-
-        assertNoFailures(searchResponse);
-        assertSearchHits(searchResponse, "child1");
-    }
-
-    /*
-    Test for https://github.com/elastic/elasticsearch/issues/3444
-     */
-    public void testBulkUpdateUpsertWithParent() throws Exception {
-        assertAcked(prepareCreate("test")
-                .addMapping("parent", "{\"parent\":{}}")
-                .addMapping("child", "{\"child\": {\"_parent\": {\"type\": \"parent\"}}}"));
-        ensureGreen();
-
-        BulkRequestBuilder builder = client().prepareBulk();
-
-        byte[] addParent = new BytesArray(
-                "{" +
-                "  \"index\" : {" +
-                "    \"_index\" : \"test\"," +
-                "    \"_type\"  : \"parent\"," +
-                "    \"_id\"    : \"parent1\"" +
-                "  }" +
-                "}" +
-                "\n" +
-                "{" +
-                "  \"field1\" : \"value1\"" +
-                "}" +
-                "\n").array();
-
-        byte[] addChild1 = new BytesArray(
-                "{" +
-                "  \"update\" : {" +
-                "    \"_index\" : \"test\"," +
-                "    \"_type\"  : \"child\"," +
-                "    \"_id\"    : \"child1\"," +
-                "    \"parent\" : \"parent1\"" +
-                "  }" +
-                "}" +
-                "\n" +
-                "{" +
-                "  \"script\" : {" +
-                "    \"inline\" : \"ctx._source.field2 = 'value2'\"" +
-                "  }," +
-                "  \"lang\" : \"" + CustomScriptPlugin.NAME + "\"," +
-                "  \"upsert\" : {" +
-                "    \"field1\" : \"value1'\"" +
-                "  }" +
-                "}" +
-                "\n").array();
-
-        byte[] addChild2 = new BytesArray(
-                "{" +
-                "  \"update\" : {" +
-                "    \"_index\" : \"test\"," +
-                "    \"_type\"  : \"child\"," +
-                "    \"_id\"    : \"child1\"," +
-                "    \"parent\" : \"parent1\"" +
-                "  }" +
-                "}" +
-                "\n" +
-                "{" +
-                "  \"script\" : \"ctx._source.field2 = 'value2'\"," +
-                "  \"upsert\" : {" +
-                "    \"field1\" : \"value1'\"" +
-                "  }" +
-                "}" +
-                "\n").array();
-
-        builder.add(addParent, 0, addParent.length);
-        builder.add(addChild1, 0, addChild1.length);
-        builder.add(addChild2, 0, addChild2.length);
-
-        BulkResponse bulkResponse = builder.get();
-        assertThat(bulkResponse.getItems().length, equalTo(3));
-        assertThat(bulkResponse.getItems()[0].isFailed(), equalTo(false));
-        assertThat(bulkResponse.getItems()[1].isFailed(), equalTo(false));
-        assertThat(bulkResponse.getItems()[2].isFailed(), equalTo(true));
-        assertThat(bulkResponse.getItems()[2].getFailure().getCause().getCause().getMessage(),
-                equalTo("script_lang not supported [painless]"));
-
-        client().admin().indices().prepareRefresh("test").get();
-
-        SearchResponse searchResponse = client().prepareSearch("test")
-                .setQuery(QueryBuilders.hasParentQuery("parent", QueryBuilders.matchAllQuery(), false))
-                .get();
-
-        assertSearchHits(searchResponse, "child1");
-    }
-
     /*
      * Test for https://github.com/elastic/elasticsearch/issues/8365
      */
@@ -645,10 +504,10 @@ public void testBulkUpdateChildMissingParentRouting() throws Exception {
                 "}" +
                 "\n").array();
 
-        builder.add(addParent, 0, addParent.length);
-        builder.add(addChildOK, 0, addChildOK.length);
-        builder.add(addChildMissingRouting, 0, addChildMissingRouting.length);
-        builder.add(addChildOK, 0, addChildOK.length);
+        builder.add(addParent, 0, addParent.length, XContentType.JSON);
+        builder.add(addChildOK, 0, addChildOK.length, XContentType.JSON);
+        builder.add(addChildMissingRouting, 0, addChildMissingRouting.length, XContentType.JSON);
+        builder.add(addChildOK, 0, addChildOK.length, XContentType.JSON);
 
         BulkResponse bulkResponse = builder.get();
         assertThat(bulkResponse.getItems().length, equalTo(4));
@@ -675,7 +534,8 @@ public void testFailingVersionedUpdatedOnBulk() throws Exception {
                     return;
                 }
                 BulkRequestBuilder requestBuilder = client().prepareBulk();
-                requestBuilder.add(client().prepareUpdate("test", "type", "1").setVersion(1).setDoc("field", threadID));
+                requestBuilder.add(client().prepareUpdate("test", "type", "1").setVersion(1)
+                    .setDoc(Requests.INDEX_CONTENT_TYPE, "field", threadID));
                 responses[threadID] = requestBuilder.get();
 
             });
@@ -728,31 +588,31 @@ public void testThatInvalidIndexNamesShouldNotBreakCompleteBulkRequest() {
     // issue 6630
     public void testThatFailedUpdateRequestReturnsCorrectType() throws Exception {
         BulkResponse indexBulkItemResponse = client().prepareBulk()
-                .add(new IndexRequest("test", "type", "3").source("{ \"title\" : \"Great Title of doc 3\" }"))
-                .add(new IndexRequest("test", "type", "4").source("{ \"title\" : \"Great Title of doc 4\" }"))
-                .add(new IndexRequest("test", "type", "5").source("{ \"title\" : \"Great Title of doc 5\" }"))
-                .add(new IndexRequest("test", "type", "6").source("{ \"title\" : \"Great Title of doc 6\" }"))
+                .add(new IndexRequest("test", "type", "3").source("{ \"title\" : \"Great Title of doc 3\" }", XContentType.JSON))
+                .add(new IndexRequest("test", "type", "4").source("{ \"title\" : \"Great Title of doc 4\" }", XContentType.JSON))
+                .add(new IndexRequest("test", "type", "5").source("{ \"title\" : \"Great Title of doc 5\" }", XContentType.JSON))
+                .add(new IndexRequest("test", "type", "6").source("{ \"title\" : \"Great Title of doc 6\" }", XContentType.JSON))
                 .setRefreshPolicy(RefreshPolicy.IMMEDIATE)
                 .get();
         assertNoFailures(indexBulkItemResponse);
 
         BulkResponse bulkItemResponse = client().prepareBulk()
-                .add(new IndexRequest("test", "type", "1").source("{ \"title\" : \"Great Title of doc 1\" }"))
-                .add(new IndexRequest("test", "type", "2").source("{ \"title\" : \"Great Title of doc 2\" }"))
-                .add(new UpdateRequest("test", "type", "3").doc("{ \"date\" : \"2014-01-30T23:59:57\"}"))
-                .add(new UpdateRequest("test", "type", "4").doc("{ \"date\" : \"2014-13-30T23:59:57\"}"))
+                .add(new IndexRequest("test", "type", "1").source("{ \"title\" : \"Great Title of doc 1\" }", XContentType.JSON))
+                .add(new IndexRequest("test", "type", "2").source("{ \"title\" : \"Great Title of doc 2\" }", XContentType.JSON))
+                .add(new UpdateRequest("test", "type", "3").doc("{ \"date\" : \"2014-01-30T23:59:57\"}", XContentType.JSON))
+                .add(new UpdateRequest("test", "type", "4").doc("{ \"date\" : \"2014-13-30T23:59:57\"}", XContentType.JSON))
                 .add(new DeleteRequest("test", "type", "5"))
                 .add(new DeleteRequest("test", "type", "6"))
         .get();
 
         assertNoFailures(indexBulkItemResponse);
         assertThat(bulkItemResponse.getItems().length, is(6));
-        assertThat(bulkItemResponse.getItems()[0].getOpType(), is("index"));
-        assertThat(bulkItemResponse.getItems()[1].getOpType(), is("index"));
-        assertThat(bulkItemResponse.getItems()[2].getOpType(), is("update"));
-        assertThat(bulkItemResponse.getItems()[3].getOpType(), is("update"));
-        assertThat(bulkItemResponse.getItems()[4].getOpType(), is("delete"));
-        assertThat(bulkItemResponse.getItems()[5].getOpType(), is("delete"));
+        assertThat(bulkItemResponse.getItems()[0].getOpType(), is(OpType.INDEX));
+        assertThat(bulkItemResponse.getItems()[1].getOpType(), is(OpType.INDEX));
+        assertThat(bulkItemResponse.getItems()[2].getOpType(), is(OpType.UPDATE));
+        assertThat(bulkItemResponse.getItems()[3].getOpType(), is(OpType.UPDATE));
+        assertThat(bulkItemResponse.getItems()[4].getOpType(), is(OpType.DELETE));
+        assertThat(bulkItemResponse.getItems()[5].getOpType(), is(OpType.DELETE));
     }
 
     private static String indexOrAlias() {
@@ -763,10 +623,10 @@ private static String indexOrAlias() {
     public void testThatMissingIndexDoesNotAbortFullBulkRequest() throws Exception{
         createIndex("bulkindex1", "bulkindex2");
         BulkRequest bulkRequest = new BulkRequest();
-        bulkRequest.add(new IndexRequest("bulkindex1", "index1_type", "1").source("text", "hallo1"))
-                   .add(new IndexRequest("bulkindex2", "index2_type", "1").source("text", "hallo2"))
-                   .add(new IndexRequest("bulkindex2", "index2_type").source("text", "hallo2"))
-                   .add(new UpdateRequest("bulkindex2", "index2_type", "2").doc("foo", "bar"))
+        bulkRequest.add(new IndexRequest("bulkindex1", "index1_type", "1").source(Requests.INDEX_CONTENT_TYPE, "text", "hallo1"))
+                   .add(new IndexRequest("bulkindex2", "index2_type", "1").source(Requests.INDEX_CONTENT_TYPE, "text", "hallo2"))
+                   .add(new IndexRequest("bulkindex2", "index2_type").source(Requests.INDEX_CONTENT_TYPE, "text", "hallo2"))
+                   .add(new UpdateRequest("bulkindex2", "index2_type", "2").doc(Requests.INDEX_CONTENT_TYPE, "foo", "bar"))
                    .add(new DeleteRequest("bulkindex2", "index2_type", "3"))
                    .setRefreshPolicy(RefreshPolicy.IMMEDIATE);
 
@@ -789,29 +649,31 @@ public void testFailedRequestsOnClosedIndex() throws Exception {
         assertAcked(client().admin().indices().prepareClose("bulkindex1"));
 
         BulkRequest bulkRequest = new BulkRequest().setRefreshPolicy(RefreshPolicy.IMMEDIATE);
-        bulkRequest.add(new IndexRequest("bulkindex1", "index1_type", "1").source("text", "hallo1"))
-                .add(new UpdateRequest("bulkindex1", "index1_type", "1").doc("foo", "bar"))
+        bulkRequest.add(new IndexRequest("bulkindex1", "index1_type", "1").source(Requests.INDEX_CONTENT_TYPE, "text", "hallo1"))
+                .add(new UpdateRequest("bulkindex1", "index1_type", "1").doc(Requests.INDEX_CONTENT_TYPE, "foo", "bar"))
                 .add(new DeleteRequest("bulkindex1", "index1_type", "1"));
 
         BulkResponse bulkResponse = client().bulk(bulkRequest).get();
         assertThat(bulkResponse.hasFailures(), is(true));
         BulkItemResponse[] responseItems = bulkResponse.getItems();
         assertThat(responseItems.length, is(3));
-        assertThat(responseItems[0].getOpType(), is("index"));
-        assertThat(responseItems[1].getOpType(), is("update"));
-        assertThat(responseItems[2].getOpType(), is("delete"));
+        assertThat(responseItems[0].getOpType(), is(OpType.INDEX));
+        assertThat(responseItems[1].getOpType(), is(OpType.UPDATE));
+        assertThat(responseItems[2].getOpType(), is(OpType.DELETE));
     }
 
     // issue 9821
     public void testInvalidIndexNamesCorrectOpType() {
         BulkResponse bulkResponse = client().prepareBulk()
-                .add(client().prepareIndex().setIndex("INVALID.NAME").setType("type1").setId("1").setSource("field", 1))
-                .add(client().prepareUpdate().setIndex("INVALID.NAME").setType("type1").setId("1").setDoc("field", randomInt()))
+                .add(client().prepareIndex().setIndex("INVALID.NAME").setType("type1").setId("1")
+                    .setSource(Requests.INDEX_CONTENT_TYPE, "field", 1))
+                .add(client().prepareUpdate().setIndex("INVALID.NAME").setType("type1").setId("1")
+                    .setDoc(Requests.INDEX_CONTENT_TYPE, "field", randomInt()))
                 .add(client().prepareDelete().setIndex("INVALID.NAME").setType("type1").setId("1")).get();
         assertThat(bulkResponse.getItems().length, is(3));
-        assertThat(bulkResponse.getItems()[0].getOpType(), is("index"));
-        assertThat(bulkResponse.getItems()[1].getOpType(), is("update"));
-        assertThat(bulkResponse.getItems()[2].getOpType(), is("delete"));
+        assertThat(bulkResponse.getItems()[0].getOpType(), is(OpType.INDEX));
+        assertThat(bulkResponse.getItems()[1].getOpType(), is(OpType.UPDATE));
+        assertThat(bulkResponse.getItems()[2].getOpType(), is(OpType.DELETE));
     }
 }
 
diff --git a/core/src/test/java/org/elasticsearch/action/bulk/RetryTests.java b/core/src/test/java/org/elasticsearch/action/bulk/RetryTests.java
index 4fa640b3adc50..51d3709ba1b2d 100644
--- a/core/src/test/java/org/elasticsearch/action/bulk/RetryTests.java
+++ b/core/src/test/java/org/elasticsearch/action/bulk/RetryTests.java
@@ -20,6 +20,7 @@
 
 import org.elasticsearch.action.ActionFuture;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.DocWriteRequest.OpType;
 import org.elasticsearch.action.delete.DeleteResponse;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.action.update.UpdateRequest;
@@ -30,6 +31,8 @@
 import org.junit.After;
 import org.junit.Before;
 
+import java.util.HashMap;
+import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicInteger;
 
@@ -43,12 +46,21 @@ public class RetryTests extends ESTestCase {
     private static final int CALLS_TO_FAIL = 5;
 
     private MockBulkClient bulkClient;
+    /**
+     * Headers that are expected to be sent with all bulk requests.
+     */
+    private Map<String, String> expectedHeaders = new HashMap<>();
 
     @Override
     @Before
     public void setUp() throws Exception {
         super.setUp();
         this.bulkClient = new MockBulkClient(getTestName(), CALLS_TO_FAIL);
+        // Stash some random headers so we can assert that we preserve them
+        bulkClient.threadPool().getThreadContext().stashContext();
+        expectedHeaders.clear();
+        expectedHeaders.put(randomAlphaOfLength(5), randomAlphaOfLength(5));
+        bulkClient.threadPool().getThreadContext().putHeader(expectedHeaders);
     }
 
     @Override
@@ -75,7 +87,8 @@ public void testSyncRetryBacksOff() throws Exception {
         BulkResponse response = Retry
                 .on(EsRejectedExecutionException.class)
                 .policy(backoff)
-                .withSyncBackoff(bulkClient, bulkRequest);
+                .using(bulkClient.threadPool())
+                .withSyncBackoff(bulkClient::bulk, bulkRequest, bulkClient.settings());
 
         assertFalse(response.hasFailures());
         assertThat(response.getItems().length, equalTo(bulkRequest.numberOfActions()));
@@ -88,7 +101,8 @@ public void testSyncRetryFailsAfterBackoff() throws Exception {
         BulkResponse response = Retry
                 .on(EsRejectedExecutionException.class)
                 .policy(backoff)
-                .withSyncBackoff(bulkClient, bulkRequest);
+                .using(bulkClient.threadPool())
+                .withSyncBackoff(bulkClient::bulk, bulkRequest, bulkClient.settings());
 
         assertTrue(response.hasFailures());
         assertThat(response.getItems().length, equalTo(bulkRequest.numberOfActions()));
@@ -101,7 +115,8 @@ public void testAsyncRetryBacksOff() throws Exception {
         BulkRequest bulkRequest = createBulkRequest();
         Retry.on(EsRejectedExecutionException.class)
                 .policy(backoff)
-                .withAsyncBackoff(bulkClient, bulkRequest, listener);
+                .using(bulkClient.threadPool())
+                .withAsyncBackoff(bulkClient::bulk, bulkRequest, listener, bulkClient.settings());
 
         listener.awaitCallbacksCalled();
         listener.assertOnResponseCalled();
@@ -117,7 +132,8 @@ public void testAsyncRetryFailsAfterBacksOff() throws Exception {
         BulkRequest bulkRequest = createBulkRequest();
         Retry.on(EsRejectedExecutionException.class)
                 .policy(backoff)
-                .withAsyncBackoff(bulkClient, bulkRequest, listener);
+                .using(bulkClient.threadPool())
+                .withAsyncBackoff(bulkClient::bulk, bulkRequest, listener, bulkClient.settings());
 
         listener.awaitCallbacksCalled();
 
@@ -177,7 +193,7 @@ public void assertOnFailureNeverCalled() {
         }
     }
 
-    private static class MockBulkClient extends NoOpClient {
+    private class MockBulkClient extends NoOpClient {
         private int numberOfCallsToFail;
 
         private MockBulkClient(String testName, int numberOfCallsToFail) {
@@ -194,6 +210,12 @@ public ActionFuture<BulkResponse> bulk(BulkRequest request) {
 
         @Override
         public void bulk(BulkRequest request, ActionListener<BulkResponse> listener) {
+            if (false == expectedHeaders.equals(threadPool().getThreadContext().getHeaders())) {
+                listener.onFailure(
+                        new RuntimeException("Expected " + expectedHeaders + " but got " + threadPool().getThreadContext().getHeaders()));
+                return;
+            }
+
             // do everything synchronously, that's fine for a test
             boolean shouldFail = numberOfCallsToFail > 0;
             numberOfCallsToFail--;
@@ -212,11 +234,11 @@ public void bulk(BulkRequest request, ActionListener<BulkResponse> listener) {
         }
 
         private BulkItemResponse successfulResponse() {
-            return new BulkItemResponse(1, "update", new DeleteResponse());
+            return new BulkItemResponse(1, OpType.DELETE, new DeleteResponse());
         }
 
         private BulkItemResponse failedResponse() {
-            return new BulkItemResponse(1, "update", new BulkItemResponse.Failure("test", "test", "1", new EsRejectedExecutionException("pool full")));
+            return new BulkItemResponse(1, OpType.INDEX, new BulkItemResponse.Failure("test", "test", "1", new EsRejectedExecutionException("pool full")));
         }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java b/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java
new file mode 100644
index 0000000000000..aa150ad6844e7
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.bulk;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
+import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.update.UpdateRequest;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.AtomicArray;
+import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.transport.TransportService;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Function;
+
+import static java.util.Collections.emptySet;
+import static java.util.Collections.singleton;
+import static org.mockito.Mockito.mock;
+
+public class TransportBulkActionIndicesThatCannotBeCreatedTests extends ESTestCase {
+    public void testNonExceptional() {
+        BulkRequest bulkRequest = new BulkRequest();
+        bulkRequest.add(new IndexRequest(randomAlphaOfLength(5)));
+        bulkRequest.add(new IndexRequest(randomAlphaOfLength(5)));
+        bulkRequest.add(new DeleteRequest(randomAlphaOfLength(5)));
+        bulkRequest.add(new UpdateRequest(randomAlphaOfLength(5), randomAlphaOfLength(5), randomAlphaOfLength(5)));
+        // Test emulating auto_create_index=false
+        indicesThatCannotBeCreatedTestCase(emptySet(), bulkRequest, null);
+        // Test emulating auto_create_index=true
+        indicesThatCannotBeCreatedTestCase(emptySet(), bulkRequest, index -> true);
+        // Test emulating all indices already created
+        indicesThatCannotBeCreatedTestCase(emptySet(), bulkRequest, index -> false);
+        // Test emulating auto_create_index=true with some indices already created.
+        indicesThatCannotBeCreatedTestCase(emptySet(), bulkRequest, index -> randomBoolean());
+    }
+
+    public void testAllFail() {
+        BulkRequest bulkRequest = new BulkRequest();
+        bulkRequest.add(new IndexRequest("no"));
+        bulkRequest.add(new IndexRequest("can't"));
+        bulkRequest.add(new DeleteRequest("do"));
+        bulkRequest.add(new UpdateRequest("nothin", randomAlphaOfLength(5), randomAlphaOfLength(5)));
+        indicesThatCannotBeCreatedTestCase(new HashSet<>(Arrays.asList("no", "can't", "do", "nothin")), bulkRequest, index -> {
+            throw new IndexNotFoundException("Can't make it because I say so");
+        });
+    }
+
+    public void testSomeFail() {
+        BulkRequest bulkRequest = new BulkRequest();
+        bulkRequest.add(new IndexRequest("ok"));
+        bulkRequest.add(new IndexRequest("bad"));
+        // Emulate auto_create_index=-bad,+*
+        indicesThatCannotBeCreatedTestCase(singleton("bad"), bulkRequest, index -> {
+            if (index.equals("bad")) {
+                throw new IndexNotFoundException("Can't make it because I say so");
+            }
+            return true;
+        });
+        // Emulate auto_create_index=false but the "ok" index already exists
+        indicesThatCannotBeCreatedTestCase(singleton("bad"), bulkRequest, index -> {
+            if (index.equals("bad")) {
+                throw new IndexNotFoundException("Can't make it because I say so");
+            }
+            return false;
+        });
+    }
+
+
+    private void indicesThatCannotBeCreatedTestCase(Set<String> expected,
+            BulkRequest bulkRequest, Function<String, Boolean> shouldAutoCreate) {
+        TransportBulkAction action = new TransportBulkAction(Settings.EMPTY, null, mock(TransportService.class), mock(ClusterService.class),
+                null, null, null, mock(ActionFilters.class), null, null) {
+            @Override
+            void executeBulk(Task task, BulkRequest bulkRequest, long startTimeNanos, ActionListener<BulkResponse> listener,
+                    AtomicArray<BulkItemResponse> responses, Map<String, IndexNotFoundException> indicesThatCannotBeCreated) {
+                assertEquals(expected, indicesThatCannotBeCreated.keySet());
+            }
+
+            @Override
+            boolean needToCheck() {
+                return null != shouldAutoCreate; // Use "null" to mean "no indices can be created so don't bother checking"
+            }
+
+            @Override
+            boolean shouldAutoCreate(String index, ClusterState state) {
+                return shouldAutoCreate.apply(index);
+            }
+
+            @Override
+            void createIndex(String index, TimeValue timeout, ActionListener<CreateIndexResponse> listener) {
+                // If we try to create an index just immediately assume it worked
+                listener.onResponse(new CreateIndexResponse(true, true, index) {});
+            }
+        };
+        action.doExecute(null, bulkRequest, null);
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java b/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java
new file mode 100644
index 0000000000000..728d02a13d69a
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java
@@ -0,0 +1,362 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.bulk;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.index.IndexAction;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateApplier;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.AtomicArray;
+import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.ingest.IngestService;
+import org.elasticsearch.ingest.PipelineExecutionService;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportResponseHandler;
+import org.elasticsearch.transport.TransportService;
+import org.junit.Before;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Captor;
+import org.mockito.MockitoAnnotations;
+
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.BiConsumer;
+import java.util.function.Consumer;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.sameInstance;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyZeroInteractions;
+import static org.mockito.Mockito.when;
+
+public class TransportBulkActionIngestTests extends ESTestCase {
+
+    /** Services needed by bulk action */
+    TransportService transportService;
+    ClusterService clusterService;
+    IngestService ingestService;
+
+    /** The ingest execution service we can capture calls to */
+    PipelineExecutionService executionService;
+
+    /** Arguments to callbacks we want to capture, but which require generics, so we must use @Captor */
+    @Captor
+    ArgumentCaptor<BiConsumer<IndexRequest, Exception>> failureHandler;
+    @Captor
+    ArgumentCaptor<Consumer<Exception>> completionHandler;
+    @Captor
+    ArgumentCaptor<TransportResponseHandler<BulkResponse>> remoteResponseHandler;
+    @Captor
+    ArgumentCaptor<Iterable<DocWriteRequest>> bulkDocsItr;
+
+    /** The actual action we want to test, with real indexing mocked */
+    TestTransportBulkAction action;
+
+    /** Single item bulk write action that wraps index requests */
+    TestSingleItemBulkWriteAction singleItemBulkWriteAction;
+
+    /** True if the next call to the index action should act as an ingest node */
+    boolean localIngest;
+
+    /** The nodes that forwarded index requests should be cycled through. */
+    DiscoveryNodes nodes;
+    DiscoveryNode remoteNode1;
+    DiscoveryNode remoteNode2;
+
+    /** A subclass of the real bulk action to allow skipping real bulk indexing, and marking when it would have happened. */
+    class TestTransportBulkAction extends TransportBulkAction {
+        boolean isExecuted = false; // set when the "real" bulk execution happens
+        TestTransportBulkAction() {
+            super(Settings.EMPTY, null, transportService, clusterService, ingestService,
+                null, null, new ActionFilters(Collections.emptySet()), null, null);
+        }
+        @Override
+        protected boolean needToCheck() {
+            return false;
+        }
+        @Override
+        void executeBulk(Task task, final BulkRequest bulkRequest, final long startTimeNanos, final ActionListener<BulkResponse> listener,
+                final AtomicArray<BulkItemResponse> responses, Map<String, IndexNotFoundException> indicesThatCannotBeCreated) {
+            isExecuted = true;
+        }
+    }
+
+    class TestSingleItemBulkWriteAction extends TransportSingleItemBulkWriteAction<IndexRequest, IndexResponse> {
+
+        TestSingleItemBulkWriteAction(TestTransportBulkAction bulkAction) {
+            super(Settings.EMPTY, IndexAction.NAME, transportService, TransportBulkActionIngestTests.this.clusterService,
+                    null, null, null, new ActionFilters(Collections.emptySet()), null,
+                    IndexRequest::new, IndexRequest::new, ThreadPool.Names.INDEX, bulkAction, null);
+        }
+
+        @Override
+        protected IndexResponse newResponseInstance() {
+            return new IndexResponse();
+        }
+    }
+
+    @Before
+    public void setupAction() {
+        // initialize captors, which must be members to use @Capture because of generics
+        MockitoAnnotations.initMocks(this);
+        // setup services that will be called by action
+        transportService = mock(TransportService.class);
+        clusterService = mock(ClusterService.class);
+        localIngest = true;
+        // setup nodes for local and remote
+        DiscoveryNode localNode = mock(DiscoveryNode.class);
+        when(localNode.isIngestNode()).thenAnswer(stub -> localIngest);
+        when(clusterService.localNode()).thenReturn(localNode);
+        remoteNode1 = mock(DiscoveryNode.class);
+        remoteNode2 = mock(DiscoveryNode.class);
+        nodes = mock(DiscoveryNodes.class);
+        ImmutableOpenMap<String, DiscoveryNode> ingestNodes = ImmutableOpenMap.<String, DiscoveryNode>builder(2)
+            .fPut("node1", remoteNode1).fPut("node2", remoteNode2).build();
+        when(nodes.getIngestNodes()).thenReturn(ingestNodes);
+        ClusterState state = mock(ClusterState.class);
+        when(state.getNodes()).thenReturn(nodes);
+        when(clusterService.state()).thenReturn(state);
+        doAnswer(invocation -> {
+            ClusterChangedEvent event = mock(ClusterChangedEvent.class);
+            when(event.state()).thenReturn(state);
+            ((ClusterStateApplier)invocation.getArguments()[0]).applyClusterState(event);
+            return null;
+        }).when(clusterService).addStateApplier(any(ClusterStateApplier.class));
+        // setup the mocked ingest service for capturing calls
+        ingestService = mock(IngestService.class);
+        executionService = mock(PipelineExecutionService.class);
+        when(ingestService.getPipelineExecutionService()).thenReturn(executionService);
+        action = new TestTransportBulkAction();
+        singleItemBulkWriteAction = new TestSingleItemBulkWriteAction(action);
+        reset(transportService); // call on construction of action
+    }
+
+    public void testIngestSkipped() throws Exception {
+        BulkRequest bulkRequest = new BulkRequest();
+        IndexRequest indexRequest = new IndexRequest("index", "type", "id");
+        indexRequest.source(Collections.emptyMap());
+        bulkRequest.add(indexRequest);
+        action.execute(null, bulkRequest, ActionListener.wrap(response -> {}, exception -> {
+            throw new AssertionError(exception);
+        }));
+        assertTrue(action.isExecuted);
+        verifyZeroInteractions(ingestService);
+    }
+
+    public void testSingleItemBulkActionIngestSkipped() throws Exception {
+        IndexRequest indexRequest = new IndexRequest("index", "type", "id");
+        indexRequest.source(Collections.emptyMap());
+        singleItemBulkWriteAction.execute(null, indexRequest, ActionListener.wrap(response -> {}, exception -> {
+            throw new AssertionError(exception);
+        }));
+        assertTrue(action.isExecuted);
+        verifyZeroInteractions(ingestService);
+    }
+
+    public void testIngestLocal() throws Exception {
+        Exception exception = new Exception("fake exception");
+        BulkRequest bulkRequest = new BulkRequest();
+        IndexRequest indexRequest1 = new IndexRequest("index", "type", "id");
+        indexRequest1.source(Collections.emptyMap());
+        indexRequest1.setPipeline("testpipeline");
+        IndexRequest indexRequest2 = new IndexRequest("index", "type", "id");
+        indexRequest2.source(Collections.emptyMap());
+        indexRequest2.setPipeline("testpipeline");
+        bulkRequest.add(indexRequest1);
+        bulkRequest.add(indexRequest2);
+
+        AtomicBoolean responseCalled = new AtomicBoolean(false);
+        AtomicBoolean failureCalled = new AtomicBoolean(false);
+        action.execute(null, bulkRequest, ActionListener.wrap(
+            response -> {
+                BulkItemResponse itemResponse = response.iterator().next();
+                assertThat(itemResponse.getFailure().getMessage(), containsString("fake exception"));
+                responseCalled.set(true);
+            },
+            e -> {
+                assertThat(e, sameInstance(exception));
+                failureCalled.set(true);
+            }));
+
+        // check failure works, and passes through to the listener
+        assertFalse(action.isExecuted); // haven't executed yet
+        assertFalse(responseCalled.get());
+        assertFalse(failureCalled.get());
+        verify(executionService).executeBulkRequest(bulkDocsItr.capture(), failureHandler.capture(), completionHandler.capture());
+        completionHandler.getValue().accept(exception);
+        assertTrue(failureCalled.get());
+
+        // now check success
+        Iterator<DocWriteRequest> req = bulkDocsItr.getValue().iterator();
+        failureHandler.getValue().accept((IndexRequest)req.next(), exception); // have an exception for our one index request
+        indexRequest2.setPipeline(null); // this is done by the real pipeline execution service when processing
+        completionHandler.getValue().accept(null);
+        assertTrue(action.isExecuted);
+        assertFalse(responseCalled.get()); // listener would only be called by real index action, not our mocked one
+        verifyZeroInteractions(transportService);
+    }
+
+    public void testSingleItemBulkActionIngestLocal() throws Exception {
+        Exception exception = new Exception("fake exception");
+        IndexRequest indexRequest = new IndexRequest("index", "type", "id");
+        indexRequest.source(Collections.emptyMap());
+        indexRequest.setPipeline("testpipeline");
+        AtomicBoolean responseCalled = new AtomicBoolean(false);
+        AtomicBoolean failureCalled = new AtomicBoolean(false);
+        singleItemBulkWriteAction.execute(null, indexRequest, ActionListener.wrap(
+                response -> {
+                    responseCalled.set(true);
+                },
+                e -> {
+                    assertThat(e, sameInstance(exception));
+                    failureCalled.set(true);
+                }));
+
+        // check failure works, and passes through to the listener
+        assertFalse(action.isExecuted); // haven't executed yet
+        assertFalse(responseCalled.get());
+        assertFalse(failureCalled.get());
+        verify(executionService).executeBulkRequest(bulkDocsItr.capture(), failureHandler.capture(), completionHandler.capture());
+        completionHandler.getValue().accept(exception);
+        assertTrue(failureCalled.get());
+
+        // now check success
+        indexRequest.setPipeline(null); // this is done by the real pipeline execution service when processing
+        completionHandler.getValue().accept(null);
+        assertTrue(action.isExecuted);
+        assertFalse(responseCalled.get()); // listener would only be called by real index action, not our mocked one
+        verifyZeroInteractions(transportService);
+    }
+
+    public void testIngestForward() throws Exception {
+        localIngest = false;
+        BulkRequest bulkRequest = new BulkRequest();
+        IndexRequest indexRequest = new IndexRequest("index", "type", "id");
+        indexRequest.source(Collections.emptyMap());
+        indexRequest.setPipeline("testpipeline");
+        bulkRequest.add(indexRequest);
+        BulkResponse bulkResponse = mock(BulkResponse.class);
+        AtomicBoolean responseCalled = new AtomicBoolean(false);
+        ActionListener<BulkResponse> listener = ActionListener.wrap(
+            response -> {
+                responseCalled.set(true);
+                assertSame(bulkResponse, response);
+            },
+            e -> {
+                throw new AssertionError(e);
+            });
+        action.execute(null, bulkRequest, listener);
+
+        // should not have executed ingest locally
+        verify(executionService, never()).executeBulkRequest(any(), any(), any());
+        // but instead should have sent to a remote node with the transport service
+        ArgumentCaptor<DiscoveryNode> node = ArgumentCaptor.forClass(DiscoveryNode.class);
+        verify(transportService).sendRequest(node.capture(), eq(BulkAction.NAME), any(), remoteResponseHandler.capture());
+        boolean usedNode1 = node.getValue() == remoteNode1; // make sure we used one of the nodes
+        if (usedNode1 == false) {
+            assertSame(remoteNode2, node.getValue());
+        }
+        assertFalse(action.isExecuted); // no local index execution
+        assertFalse(responseCalled.get()); // listener not called yet
+
+        remoteResponseHandler.getValue().handleResponse(bulkResponse); // call the listener for the remote node
+        assertTrue(responseCalled.get()); // now the listener we passed should have been delegated to by the remote listener
+        assertFalse(action.isExecuted); // still no local index execution
+
+        // now make sure ingest nodes are rotated through with a subsequent request
+        reset(transportService);
+        action.execute(null, bulkRequest, listener);
+        verify(transportService).sendRequest(node.capture(), eq(BulkAction.NAME), any(), remoteResponseHandler.capture());
+        if (usedNode1) {
+            assertSame(remoteNode2, node.getValue());
+        } else {
+            assertSame(remoteNode1, node.getValue());
+        }
+    }
+
+    public void testSingleItemBulkActionIngestForward() throws Exception {
+        localIngest = false;
+        IndexRequest indexRequest = new IndexRequest("index", "type", "id");
+        indexRequest.source(Collections.emptyMap());
+        indexRequest.setPipeline("testpipeline");
+        IndexResponse indexResponse = mock(IndexResponse.class);
+        AtomicBoolean responseCalled = new AtomicBoolean(false);
+        ActionListener<IndexResponse> listener = ActionListener.wrap(
+                response -> {
+                    responseCalled.set(true);
+                    assertSame(indexResponse, response);
+                },
+                e -> {
+                    throw new AssertionError(e);
+                });
+        singleItemBulkWriteAction.execute(null, indexRequest, listener);
+
+        // should not have executed ingest locally
+        verify(executionService, never()).executeBulkRequest(any(), any(), any());
+        // but instead should have sent to a remote node with the transport service
+        ArgumentCaptor<DiscoveryNode> node = ArgumentCaptor.forClass(DiscoveryNode.class);
+        verify(transportService).sendRequest(node.capture(), eq(BulkAction.NAME), any(), remoteResponseHandler.capture());
+        boolean usedNode1 = node.getValue() == remoteNode1; // make sure we used one of the nodes
+        if (usedNode1 == false) {
+            assertSame(remoteNode2, node.getValue());
+        }
+        assertFalse(action.isExecuted); // no local index execution
+        assertFalse(responseCalled.get()); // listener not called yet
+
+        BulkItemResponse itemResponse = new BulkItemResponse(0, DocWriteRequest.OpType.CREATE, indexResponse);
+        BulkItemResponse[] bulkItemResponses = new BulkItemResponse[1];
+        bulkItemResponses[0] = itemResponse;
+        remoteResponseHandler.getValue().handleResponse(new BulkResponse(bulkItemResponses, 0)); // call the listener for the remote node
+        assertTrue(responseCalled.get()); // now the listener we passed should have been delegated to by the remote listener
+        assertFalse(action.isExecuted); // still no local index execution
+
+        // now make sure ingest nodes are rotated through with a subsequent request
+        reset(transportService);
+        singleItemBulkWriteAction.execute(null, indexRequest, listener);
+        verify(transportService).sendRequest(node.capture(), eq(BulkAction.NAME), any(), remoteResponseHandler.capture());
+        if (usedNode1) {
+            assertSame(remoteNode2, node.getValue());
+        } else {
+            assertSame(remoteNode1, node.getValue());
+        }
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java b/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java
index 7c39adc76f650..e35f98e220e03 100644
--- a/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java
+++ b/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java
@@ -35,6 +35,8 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.transport.CapturingTransport;
@@ -48,6 +50,7 @@
 
 import java.nio.charset.StandardCharsets;
 import java.util.HashSet;
+import java.util.Map;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.LongSupplier;
@@ -87,7 +90,9 @@ public void tearDown() throws Exception {
 
     private TransportBulkAction createAction(boolean controlled, AtomicLong expected) {
         CapturingTransport capturingTransport = new CapturingTransport();
-        TransportService transportService = new TransportService(clusterService.getSettings(), capturingTransport, threadPool);
+        TransportService transportService = new TransportService(clusterService.getSettings(), capturingTransport, threadPool,
+                TransportService.NOOP_TRANSPORT_INTERCEPTOR,
+            boundAddress -> clusterService.localNode(), null);
         transportService.start();
         transportService.acceptIncomingRequests();
         IndexNameExpressionResolver resolver = new Resolver(Settings.EMPTY);
@@ -115,11 +120,6 @@ private TransportBulkAction createAction(boolean controlled, AtomicLong expected
                     resolver,
                     null,
                     expected::get) {
-                @Override
-                public void executeBulk(BulkRequest bulkRequest, ActionListener<BulkResponse> listener) {
-                    expected.set(1000000);
-                    super.executeBulk(bulkRequest, listener);
-                }
 
                 @Override
                 void executeBulk(
@@ -127,9 +127,10 @@ void executeBulk(
                         BulkRequest bulkRequest,
                         long startTimeNanos,
                         ActionListener<BulkResponse> listener,
-                        AtomicArray<BulkItemResponse> responses) {
+                        AtomicArray<BulkItemResponse> responses,
+                        Map<String, IndexNotFoundException> indicesThatCannotBeCreated) {
                     expected.set(1000000);
-                    super.executeBulk(task, bulkRequest, startTimeNanos, listener, responses);
+                    super.executeBulk(task, bulkRequest, startTimeNanos, listener, responses, indicesThatCannotBeCreated);
                 }
             };
         } else {
@@ -144,12 +145,6 @@ void executeBulk(
                     resolver,
                     null,
                     System::nanoTime) {
-                @Override
-                public void executeBulk(BulkRequest bulkRequest, ActionListener<BulkResponse> listener) {
-                    long elapsed = spinForAtLeastOneMillisecond();
-                    expected.set(elapsed);
-                    super.executeBulk(bulkRequest, listener);
-                }
 
                 @Override
                 void executeBulk(
@@ -157,10 +152,11 @@ void executeBulk(
                         BulkRequest bulkRequest,
                         long startTimeNanos,
                         ActionListener<BulkResponse> listener,
-                        AtomicArray<BulkItemResponse> responses) {
+                        AtomicArray<BulkItemResponse> responses,
+                        Map<String, IndexNotFoundException> indicesThatCannotBeCreated) {
                     long elapsed = spinForAtLeastOneMillisecond();
                     expected.set(elapsed);
-                    super.executeBulk(task, bulkRequest, startTimeNanos, listener, responses);
+                    super.executeBulk(task, bulkRequest, startTimeNanos, listener, responses, indicesThatCannotBeCreated);
                 }
             };
         }
@@ -183,7 +179,7 @@ private void runTestTook(boolean controlled) throws Exception {
             bulkAction = Strings.replace(bulkAction, "\r\n", "\n");
         }
         BulkRequest bulkRequest = new BulkRequest();
-        bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null);
+        bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON);
         AtomicLong expected = new AtomicLong();
         TransportBulkAction action = createAction(controlled, expected);
         action.doExecute(null, bulkRequest, new ActionListener<BulkResponse>() {
@@ -208,7 +204,7 @@ public void onFailure(Exception e) {
     }
 
     static class Resolver extends IndexNameExpressionResolver {
-        public Resolver(Settings settings) {
+        Resolver(Settings settings) {
             super(settings);
         }
 
@@ -220,7 +216,7 @@ public String[] concreteIndexNames(ClusterState state, IndicesRequest request) {
 
     static class TestTransportBulkAction extends TransportBulkAction {
 
-        public TestTransportBulkAction(
+        TestTransportBulkAction(
                 Settings settings,
                 ThreadPool threadPool,
                 TransportService transportService,
@@ -236,6 +232,7 @@ public TestTransportBulkAction(
                     threadPool,
                     transportService,
                     clusterService,
+                    null,
                     shardBulkAction,
                     createIndexAction,
                     actionFilters,
@@ -258,7 +255,7 @@ boolean shouldAutoCreate(String index, ClusterState state) {
 
     static class TestTransportCreateIndexAction extends TransportCreateIndexAction {
 
-        public TestTransportCreateIndexAction(
+        TestTransportCreateIndexAction(
                 Settings settings,
                 TransportService transportService,
                 ClusterService clusterService,
diff --git a/core/src/test/java/org/elasticsearch/action/delete/DeleteResponseTests.java b/core/src/test/java/org/elasticsearch/action/delete/DeleteResponseTests.java
new file mode 100644
index 0000000000000..344b1c4aa93bf
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/delete/DeleteResponseTests.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.delete;
+
+import org.elasticsearch.action.support.replication.ReplicationResponse;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.RandomObjects;
+
+import java.io.IOException;
+import java.util.function.Predicate;
+
+import static org.elasticsearch.action.index.IndexResponseTests.assertDocWriteResponse;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.INDEX_UUID_NA_VALUE;
+import static org.elasticsearch.test.XContentTestUtils.insertRandomFields;
+
+public class DeleteResponseTests extends ESTestCase {
+
+    public void testToXContent() {
+        {
+            DeleteResponse response = new DeleteResponse(new ShardId("index", "index_uuid", 0), "type", "id", 5, true);
+            String output = Strings.toString(response);
+            assertEquals("{\"found\":true,\"_index\":\"index\",\"_type\":\"type\",\"_id\":\"id\",\"_version\":5,\"result\":\"deleted\"," +
+                "\"_shards\":null}", output);
+        }
+        {
+            DeleteResponse response = new DeleteResponse(new ShardId("index", "index_uuid", 0), "type", "id", 7, true);
+            response.setForcedRefresh(true);
+            response.setShardInfo(new ReplicationResponse.ShardInfo(10, 5));
+            String output = Strings.toString(response);
+            assertEquals("{\"found\":true,\"_index\":\"index\",\"_type\":\"type\",\"_id\":\"id\",\"_version\":7,\"result\":\"deleted\"," +
+                "\"forced_refresh\":true,\"_shards\":{\"total\":10,\"successful\":5,\"failed\":0}}", output);
+        }
+    }
+
+    public void testToAndFromXContent() throws IOException {
+        doFromXContentTestWithRandomFields(false);
+    }
+
+    /**
+     * This test adds random fields and objects to the xContent rendered out to
+     * ensure we can parse it back to be forward compatible with additions to
+     * the xContent
+     */
+    public void testFromXContentWithRandomFields() throws IOException {
+        doFromXContentTestWithRandomFields(true);
+    }
+
+    private void doFromXContentTestWithRandomFields(boolean addRandomFields) throws IOException {
+        final Tuple<DeleteResponse, DeleteResponse> tuple = randomDeleteResponse();
+        DeleteResponse deleteResponse = tuple.v1();
+        DeleteResponse expectedDeleteResponse = tuple.v2();
+
+        boolean humanReadable = randomBoolean();
+        final XContentType xContentType = randomFrom(XContentType.values());
+        BytesReference originalBytes = toShuffledXContent(deleteResponse, xContentType, ToXContent.EMPTY_PARAMS, humanReadable);
+
+        BytesReference mutated;
+        if (addRandomFields) {
+            // The ShardInfo.Failure's exception is rendered out in a "reason" object. We shouldn't add anything random there
+            // because exception rendering and parsing are very permissive: any extra object or field would be rendered as
+            // a exception custom metadata and be parsed back as a custom header, making it impossible to compare the results
+            // in this test.
+            Predicate<String> excludeFilter = path -> path.contains("reason");
+            mutated = insertRandomFields(xContentType, originalBytes, excludeFilter, random());
+        } else {
+            mutated = originalBytes;
+        }
+        DeleteResponse parsedDeleteResponse;
+        try (XContentParser parser = createParser(xContentType.xContent(), mutated)) {
+            parsedDeleteResponse = DeleteResponse.fromXContent(parser);
+            assertNull(parser.nextToken());
+        }
+
+        // We can't use equals() to compare the original and the parsed delete response
+        // because the random delete response can contain shard failures with exceptions,
+        // and those exceptions are not parsed back with the same types.
+        assertDocWriteResponse(expectedDeleteResponse, parsedDeleteResponse);
+    }
+
+    /**
+     * Returns a tuple of {@link DeleteResponse}s.
+     * <p>
+     * The left element is the actual {@link DeleteResponse} to serialize while the right element is the
+     * expected {@link DeleteResponse} after parsing.
+     */
+    public static Tuple<DeleteResponse, DeleteResponse> randomDeleteResponse() {
+        String index = randomAlphaOfLength(5);
+        String indexUUid = randomAlphaOfLength(5);
+        int shardId = randomIntBetween(0, 5);
+        String type = randomAlphaOfLength(5);
+        String id = randomAlphaOfLength(5);
+        long version = randomBoolean() ? randomNonNegativeLong() : randomIntBetween(0, 10000);
+        boolean found = randomBoolean();
+        boolean forcedRefresh = randomBoolean();
+
+        Tuple<ReplicationResponse.ShardInfo, ReplicationResponse.ShardInfo> shardInfos = RandomObjects.randomShardInfo(random());
+
+        DeleteResponse actual = new DeleteResponse(new ShardId(index, indexUUid, shardId), type, id, version, found);
+        actual.setForcedRefresh(forcedRefresh);
+        actual.setShardInfo(shardInfos.v1());
+
+        DeleteResponse expected = new DeleteResponse(new ShardId(index, INDEX_UUID_NA_VALUE, -1), type, id, version, found);
+        expected.setForcedRefresh(forcedRefresh);
+        expected.setShardInfo(shardInfos.v2());
+
+        return Tuple.tuple(actual, expected);
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestTests.java b/core/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestTests.java
new file mode 100644
index 0000000000000..8543b35569a31
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestTests.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.fieldcaps;
+
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+public class FieldCapabilitiesRequestTests extends ESTestCase {
+    private FieldCapabilitiesRequest randomRequest() {
+        FieldCapabilitiesRequest request =  new FieldCapabilitiesRequest();
+        int size = randomIntBetween(1, 20);
+        String[] randomFields = new String[size];
+        for (int i = 0; i < size; i++) {
+            randomFields[i] = randomAlphaOfLengthBetween(5, 10);
+        }
+
+        size = randomIntBetween(0, 20);
+        String[] randomIndices = new String[size];
+        for (int i = 0; i < size; i++) {
+            randomIndices[i] = randomAlphaOfLengthBetween(5, 10);
+        }
+        request.fields(randomFields);
+        request.indices(randomIndices);
+        if (randomBoolean()) {
+            request.indicesOptions(randomBoolean() ? IndicesOptions.strictExpand() : IndicesOptions.lenientExpandOpen());
+        }
+        return request;
+    }
+
+    public void testEqualsAndHashcode() {
+        FieldCapabilitiesRequest request = new FieldCapabilitiesRequest();
+        request.indices("foo");
+        request.indicesOptions(IndicesOptions.lenientExpandOpen());
+        request.fields("bar");
+
+        FieldCapabilitiesRequest other = new FieldCapabilitiesRequest();
+        other.indices("foo");
+        other.indicesOptions(IndicesOptions.lenientExpandOpen());
+        other.fields("bar");
+        assertEquals(request, request);
+        assertEquals(request, other);
+        assertEquals(request.hashCode(), other.hashCode());
+
+        // change indices
+        other.indices("foo", "bar");
+        assertNotEquals(request, other);
+        other.indices("foo");
+        assertEquals(request, other);
+
+        // change fields
+        other.fields("foo", "bar");
+        assertNotEquals(request, other);
+        other.fields("bar");
+        assertEquals(request, request);
+
+        // change indices options
+        other.indicesOptions(IndicesOptions.strictExpand());
+        assertNotEquals(request, other);
+
+    }
+
+    public void testFieldCapsRequestSerialization() throws IOException {
+        for (int i = 0; i < 20; i++) {
+            FieldCapabilitiesRequest request = randomRequest();
+            BytesStreamOutput output = new BytesStreamOutput();
+            request.writeTo(output);
+            output.flush();
+            StreamInput input = output.bytes().streamInput();
+            FieldCapabilitiesRequest deserialized = new FieldCapabilitiesRequest();
+            deserialized.readFrom(input);
+            assertEquals(deserialized, request);
+            assertEquals(deserialized.hashCode(), request.hashCode());
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java b/core/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java
new file mode 100644
index 0000000000000..2eaf1d4832f3f
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.fieldcaps;
+
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+public class FieldCapabilitiesResponseTests extends ESTestCase {
+    private FieldCapabilitiesResponse randomResponse() {
+        Map<String, Map<String, FieldCapabilities> > fieldMap = new HashMap<> ();
+        int numFields = randomInt(10);
+        for (int i = 0; i < numFields; i++) {
+            String fieldName = randomAlphaOfLengthBetween(5, 10);
+            int numIndices = randomIntBetween(1, 5);
+            Map<String, FieldCapabilities> indexFieldMap = new HashMap<> ();
+            for (int j = 0; j < numIndices; j++) {
+                String index = randomAlphaOfLengthBetween(10, 20);
+                indexFieldMap.put(index, FieldCapabilitiesTests.randomFieldCaps());
+            }
+            fieldMap.put(fieldName, indexFieldMap);
+        }
+        return new FieldCapabilitiesResponse(fieldMap);
+    }
+
+    public void testSerialization() throws IOException {
+        for (int i = 0; i < 20; i++) {
+            FieldCapabilitiesResponse response = randomResponse();
+            BytesStreamOutput output = new BytesStreamOutput();
+            response.writeTo(output);
+            output.flush();
+            StreamInput input = output.bytes().streamInput();
+            FieldCapabilitiesResponse deserialized = new FieldCapabilitiesResponse();
+            deserialized.readFrom(input);
+            assertEquals(deserialized, response);
+            assertEquals(deserialized.hashCode(), response.hashCode());
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTests.java b/core/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTests.java
new file mode 100644
index 0000000000000..f637f2e7cafd4
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTests.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.fieldcaps;
+
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteable;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.Collections;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class FieldCapabilitiesTests extends ESTestCase {
+    protected static final int NUMBER_OF_TEST_RUNS = 20;
+
+    public void testBuilder() {
+        FieldCapabilities.Builder builder =
+            new FieldCapabilities.Builder("field", "type");
+        builder.add("index1", true, false);
+        builder.add("index2", true, false);
+        builder.add("index3", true, false);
+
+        {
+            FieldCapabilities cap1 = builder.build(false);
+            assertThat(cap1.isSearchable(), equalTo(true));
+            assertThat(cap1.isAggregatable(), equalTo(false));
+            assertNull(cap1.indices());
+            assertNull(cap1.nonSearchableIndices());
+            assertNull(cap1.nonAggregatableIndices());
+
+            FieldCapabilities cap2 = builder.build(true);
+            assertThat(cap2.isSearchable(), equalTo(true));
+            assertThat(cap2.isAggregatable(), equalTo(false));
+            assertThat(cap2.indices().length, equalTo(3));
+            assertThat(cap2.indices(), equalTo(new String[]{"index1", "index2", "index3"}));
+            assertNull(cap2.nonSearchableIndices());
+            assertNull(cap2.nonAggregatableIndices());
+        }
+
+        builder = new FieldCapabilities.Builder("field", "type");
+        builder.add("index1", false, true);
+        builder.add("index2", true, false);
+        builder.add("index3", false, false);
+        {
+            FieldCapabilities cap1 = builder.build(false);
+            assertThat(cap1.isSearchable(), equalTo(false));
+            assertThat(cap1.isAggregatable(), equalTo(false));
+            assertNull(cap1.indices());
+            assertThat(cap1.nonSearchableIndices(), equalTo(new String[]{"index1", "index3"}));
+            assertThat(cap1.nonAggregatableIndices(), equalTo(new String[]{"index2", "index3"}));
+
+            FieldCapabilities cap2 = builder.build(true);
+            assertThat(cap2.isSearchable(), equalTo(false));
+            assertThat(cap2.isAggregatable(), equalTo(false));
+            assertThat(cap2.indices().length, equalTo(3));
+            assertThat(cap2.indices(), equalTo(new String[]{"index1", "index2", "index3"}));
+            assertThat(cap1.nonSearchableIndices(), equalTo(new String[]{"index1", "index3"}));
+            assertThat(cap1.nonAggregatableIndices(), equalTo(new String[]{"index2", "index3"}));
+        }
+    }
+
+    /**
+     * Tests that the equals and hashcode methods are consistent and copied
+     * versions of the instance have are equal.
+     */
+    public void testEqualsAndHashcode() throws IOException {
+        for (int runs = 0; runs < NUMBER_OF_TEST_RUNS; runs++) {
+            FieldCapabilities firstInstance = randomFieldCaps();
+            assertFalse("instance is equal to null",
+                firstInstance.equals(null));
+            assertFalse("instance is equal to incompatible type",
+                firstInstance.equals(""));
+            assertEquals("instance is not equal to self",
+                firstInstance, firstInstance);
+            assertThat("same instance's hashcode returns different values " +
+                    "if called multiple times",
+                firstInstance.hashCode(),
+                equalTo(firstInstance.hashCode()));
+
+            FieldCapabilities secondInstance = copyInstance(firstInstance);
+            assertEquals("instance is not equal to self",
+                secondInstance, secondInstance);
+            assertEquals("instance is not equal to its copy",
+                firstInstance, secondInstance);
+            assertEquals("equals is not symmetric",
+                secondInstance, firstInstance);
+            assertThat("instance copy's hashcode is different from original hashcode",
+                secondInstance.hashCode(),
+                equalTo(firstInstance.hashCode()));
+
+            FieldCapabilities thirdInstance = copyInstance(secondInstance);
+            assertEquals("instance is not equal to self",
+                thirdInstance, thirdInstance);
+            assertEquals("instance is not equal to its copy",
+                secondInstance, thirdInstance);
+            assertThat("instance copy's hashcode is different from original hashcode",
+                secondInstance.hashCode(),
+                equalTo(thirdInstance.hashCode()));
+            assertEquals("equals is not transitive",
+                firstInstance, thirdInstance);
+            assertThat("instance copy's hashcode is different from original hashcode",
+                firstInstance.hashCode(),
+                equalTo(thirdInstance.hashCode()));
+            assertEquals("equals is not symmetric",
+                thirdInstance, secondInstance);
+            assertEquals("equals is not symmetric",
+                thirdInstance, firstInstance);
+        }
+    }
+
+    /**
+     * Test serialization and deserialization of the test instance.
+     */
+    public void testSerialization() throws IOException {
+        for (int runs = 0; runs < NUMBER_OF_TEST_RUNS; runs++) {
+            FieldCapabilities testInstance = randomFieldCaps();
+            assertSerialization(testInstance);
+        }
+    }
+
+    /**
+     * Serialize the given instance and asserts that both are equal
+     */
+    protected FieldCapabilities assertSerialization(FieldCapabilities testInstance)
+        throws IOException {
+        FieldCapabilities deserializedInstance = copyInstance(testInstance);
+        assertEquals(testInstance, deserializedInstance);
+        assertEquals(testInstance.hashCode(), deserializedInstance.hashCode());
+        assertNotSame(testInstance, deserializedInstance);
+        return deserializedInstance;
+    }
+
+    private FieldCapabilities copyInstance(FieldCapabilities instance) throws IOException {
+        try (BytesStreamOutput output = new BytesStreamOutput()) {
+            instance.writeTo(output);
+            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(),
+                getNamedWriteableRegistry())) {
+                return new FieldCapabilities(in);
+            }
+        }
+    }
+
+    /**
+     * Get the {@link NamedWriteableRegistry} to use when de-serializing the object.
+     *
+     * Override this method if you need to register {@link NamedWriteable}s for the
+     * test object to de-serialize.
+     *
+     * By default this will return a {@link NamedWriteableRegistry} with no
+     * registered {@link NamedWriteable}s
+     */
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return new NamedWriteableRegistry(Collections.emptyList());
+    }
+
+    static FieldCapabilities randomFieldCaps() {
+        String[] indices = null;
+        if (randomBoolean()) {
+            indices = new String[randomIntBetween(1, 5)];
+            for (int i = 0; i < indices.length; i++) {
+                indices[i] = randomAlphaOfLengthBetween(5, 20);
+            }
+        }
+        String[] nonSearchableIndices = null;
+        if (randomBoolean()) {
+            nonSearchableIndices = new String[randomIntBetween(0, 5)];
+            for (int i = 0; i < nonSearchableIndices.length; i++) {
+                nonSearchableIndices[i] = randomAlphaOfLengthBetween(5, 20);
+            }
+        }
+        String[] nonAggregatableIndices = null;
+        if (randomBoolean()) {
+            nonAggregatableIndices = new String[randomIntBetween(0, 5)];
+            for (int i = 0; i < nonAggregatableIndices.length; i++) {
+                nonAggregatableIndices[i] = randomAlphaOfLengthBetween(5, 20);
+            }
+        }
+        return new FieldCapabilities(randomAlphaOfLengthBetween(5, 20),
+            randomAlphaOfLengthBetween(5, 20), randomBoolean(), randomBoolean(),
+            indices, nonSearchableIndices, nonAggregatableIndices);
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/fieldstats/FieldStatsRequestTests.java b/core/src/test/java/org/elasticsearch/action/fieldstats/FieldStatsRequestTests.java
index 702c0801054f9..142ca7dc7f1d2 100644
--- a/core/src/test/java/org/elasticsearch/action/fieldstats/FieldStatsRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/fieldstats/FieldStatsRequestTests.java
@@ -19,9 +19,19 @@
 
 package org.elasticsearch.action.fieldstats;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.fieldstats.FieldStatsTests;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.StreamsUtils;
+import org.elasticsearch.test.VersionUtils;
+
+import java.util.HashMap;
+import java.util.Map;
 
 import static org.elasticsearch.action.fieldstats.IndexConstraint.Comparison.GT;
 import static org.elasticsearch.action.fieldstats.IndexConstraint.Comparison.GTE;
@@ -34,11 +44,11 @@
 public class FieldStatsRequestTests extends ESTestCase {
 
     public void testFieldsParsing() throws Exception {
-        byte[] data =
+        BytesArray data = new BytesArray(
             StreamsUtils.copyToBytesFromClasspath("/org/elasticsearch/action/fieldstats/" +
-                "fieldstats-index-constraints-request.json");
+                "fieldstats-index-constraints-request.json"));
         FieldStatsRequest request = new FieldStatsRequest();
-        request.source(new BytesArray(data));
+        request.source(createParser(JsonXContent.jsonXContent, data));
 
         assertThat(request.getFields().length, equalTo(5));
         assertThat(request.getFields()[0], equalTo("field1"));
@@ -84,4 +94,33 @@ public void testFieldsParsing() throws Exception {
         assertThat(request.getIndexConstraints()[7].getOptionalFormat(), equalTo("date_optional_time"));
     }
 
+    public void testFieldStatsBWC() throws Exception {
+        int size = randomIntBetween(5, 20);
+        Map<String, FieldStats<?> > stats = new HashMap<> ();
+        for (int i = 0; i < size; i++) {
+            stats.put(Integer.toString(i), FieldStatsTests.randomFieldStats(true));
+        }
+
+        FieldStatsShardResponse response = new FieldStatsShardResponse(new ShardId("test", "test", 0), stats);
+        for (int i = 0; i < 10; i++) {
+            Version version = VersionUtils.randomVersionBetween(random(), Version.V_5_0_0, Version.CURRENT);
+            BytesStreamOutput output = new BytesStreamOutput();
+            output.setVersion(version);
+            response.writeTo(output);
+            output.flush();
+            StreamInput input = output.bytes().streamInput();
+            input.setVersion(version);
+            FieldStatsShardResponse deserialized = new FieldStatsShardResponse();
+            deserialized.readFrom(input);
+            final Map<String, FieldStats<?>> expected;
+            if (version.before(Version.V_5_2_0)) {
+                expected = deserialized.filterNullMinMax();
+            } else {
+                expected = deserialized.getFieldStats();
+            }
+            assertEquals(expected.size(), deserialized.getFieldStats().size());
+            assertThat(expected, equalTo(deserialized.getFieldStats()));
+        }
+    }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/action/get/GetResponseTests.java b/core/src/test/java/org/elasticsearch/action/get/GetResponseTests.java
new file mode 100644
index 0000000000000..aa753ef817f52
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/get/GetResponseTests.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.get;
+
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.get.GetField;
+import org.elasticsearch.index.get.GetResult;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.function.Predicate;
+
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+import static org.elasticsearch.index.get.GetResultTests.copyGetResult;
+import static org.elasticsearch.index.get.GetResultTests.mutateGetResult;
+import static org.elasticsearch.index.get.GetResultTests.randomGetResult;
+import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
+import static org.elasticsearch.test.XContentTestUtils.insertRandomFields;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
+
+public class GetResponseTests extends ESTestCase {
+
+    public void testToAndFromXContent() throws Exception {
+        doFromXContentTestWithRandomFields(false);
+    }
+
+    /**
+     * This test adds random fields and objects to the xContent rendered out to
+     * ensure we can parse it back to be forward compatible with additions to
+     * the xContent
+     */
+    public void testFromXContentWithRandomFields() throws IOException {
+        doFromXContentTestWithRandomFields(true);
+    }
+
+    private void doFromXContentTestWithRandomFields(boolean addRandomFields) throws IOException {
+        XContentType xContentType = randomFrom(XContentType.values());
+        Tuple<GetResult, GetResult> tuple = randomGetResult(xContentType);
+        GetResponse getResponse = new GetResponse(tuple.v1());
+        GetResponse expectedGetResponse = new GetResponse(tuple.v2());
+        boolean humanReadable = randomBoolean();
+        BytesReference originalBytes = toShuffledXContent(getResponse, xContentType, ToXContent.EMPTY_PARAMS, humanReadable, "_source");
+
+        BytesReference mutated;
+        if (addRandomFields) {
+            // "_source" and "fields" just consists of key/value pairs, we shouldn't add anything random there. It is already
+            // randomized in the randomGetResult() method anyway. Also, we cannot add anything in the root object since this is
+            // where GetResult's metadata fields are rendered out while            // other fields are rendered out in a "fields" object.
+            Predicate<String> excludeFilter = (s) -> s.isEmpty() || s.contains("fields") || s.contains("_source");
+            mutated = insertRandomFields(xContentType, originalBytes, excludeFilter, random());
+        } else {
+            mutated = originalBytes;
+        }
+        GetResponse parsedGetResponse;
+        try (XContentParser parser = createParser(xContentType.xContent(), mutated)) {
+            parsedGetResponse = GetResponse.fromXContent(parser);
+            assertNull(parser.nextToken());
+        }
+        assertEquals(expectedGetResponse, parsedGetResponse);
+        //print the parsed object out and test that the output is the same as the original output
+        BytesReference finalBytes = toXContent(parsedGetResponse, xContentType, humanReadable);
+        assertToXContentEquivalent(originalBytes, finalBytes, xContentType);
+        //check that the source stays unchanged, no shuffling of keys nor anything like that
+        assertEquals(expectedGetResponse.getSourceAsString(), parsedGetResponse.getSourceAsString());
+    }
+
+    public void testToXContent() {
+        {
+            GetResponse getResponse = new GetResponse(new GetResult("index", "type", "id", 1, true, new BytesArray("{ \"field1\" : " +
+                    "\"value1\", \"field2\":\"value2\"}"), Collections.singletonMap("field1", new GetField("field1",
+                    Collections.singletonList("value1")))));
+            String output = Strings.toString(getResponse);
+            assertEquals("{\"_index\":\"index\",\"_type\":\"type\",\"_id\":\"id\",\"_version\":1,\"found\":true,\"_source\":{ \"field1\" " +
+                    ": \"value1\", \"field2\":\"value2\"},\"fields\":{\"field1\":[\"value1\"]}}", output);
+        }
+        {
+            GetResponse getResponse = new GetResponse(new GetResult("index", "type", "id", 1, false, null, null));
+            String output = Strings.toString(getResponse);
+            assertEquals("{\"_index\":\"index\",\"_type\":\"type\",\"_id\":\"id\",\"found\":false}", output);
+        }
+    }
+
+    public void testToString() {
+        GetResponse getResponse = new GetResponse(
+                new GetResult("index", "type", "id", 1, true, new BytesArray("{ \"field1\" : " + "\"value1\", \"field2\":\"value2\"}"),
+                        Collections.singletonMap("field1", new GetField("field1", Collections.singletonList("value1")))));
+        assertEquals("{\"_index\":\"index\",\"_type\":\"type\",\"_id\":\"id\",\"_version\":1,\"found\":true,\"_source\":{ \"field1\" "
+                + ": \"value1\", \"field2\":\"value2\"},\"fields\":{\"field1\":[\"value1\"]}}", getResponse.toString());
+    }
+
+    public void testEqualsAndHashcode() {
+        checkEqualsAndHashCode(new GetResponse(randomGetResult(XContentType.JSON).v1()), GetResponseTests::copyGetResponse,
+                GetResponseTests::mutateGetResponse);
+    }
+
+    public void testFromXContentThrowsParsingException() throws IOException {
+        GetResponse getResponse = new GetResponse(new GetResult(null, null, null, randomIntBetween(1, 5), randomBoolean(), null, null));
+
+        XContentType xContentType = randomFrom(XContentType.values());
+        BytesReference originalBytes = toShuffledXContent(getResponse, xContentType, ToXContent.EMPTY_PARAMS, randomBoolean());
+
+        try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) {
+            ParsingException exception = expectThrows(ParsingException.class, () -> GetResponse.fromXContent(parser));
+            assertEquals("Missing required fields [_index,_type,_id]", exception.getMessage());
+        }
+    }
+
+    private static GetResponse copyGetResponse(GetResponse getResponse) {
+        return new GetResponse(copyGetResult(getResponse.getResult));
+    }
+
+    private static GetResponse mutateGetResponse(GetResponse getResponse) {
+        return new GetResponse(mutateGetResult(getResponse.getResult));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/get/MultiGetRequestTests.java b/core/src/test/java/org/elasticsearch/action/get/MultiGetRequestTests.java
new file mode 100644
index 0000000000000..603ee44d173b8
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/get/MultiGetRequestTests.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.get;
+
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.containsString;
+
+public class MultiGetRequestTests extends ESTestCase {
+
+    public void testAddWithInvalidKey() throws IOException {
+        final XContentBuilder builder = XContentFactory.jsonBuilder();
+        builder.startObject();
+        {
+            builder.startArray("doc");
+            {
+                builder.startObject();
+                {
+                    builder.field("_type", "type");
+                    builder.field("_id", "1");
+                }
+                builder.endObject();
+            }
+            builder.endArray();
+        }
+        builder.endObject();
+        final XContentParser parser = createParser(builder);
+        final MultiGetRequest mgr = new MultiGetRequest();
+        final ParsingException e = expectThrows(
+                ParsingException.class,
+                () -> {
+                    final String defaultIndex = randomAlphaOfLength(5);
+                    final String defaultType = randomAlphaOfLength(3);
+                    final FetchSourceContext fetchSource = FetchSourceContext.FETCH_SOURCE;
+                    mgr.add(defaultIndex, defaultType, null, fetchSource, null, parser, true);
+                });
+        assertThat(
+                e.toString(),
+                containsString("unknown key [doc] for a START_ARRAY, expected [docs] or [ids]"));
+    }
+
+    public void testUnexpectedField() throws IOException {
+        final XContentBuilder builder = XContentFactory.jsonBuilder();
+        builder.startObject();
+        {
+            builder.startObject("docs");
+            {
+                builder.field("_type", "type");
+                builder.field("_id", "1");
+            }
+            builder.endObject();
+        }
+        builder.endObject();
+        final XContentParser parser = createParser(builder);
+        final MultiGetRequest mgr = new MultiGetRequest();
+        final ParsingException e = expectThrows(
+                ParsingException.class,
+                () -> {
+                    final String defaultIndex = randomAlphaOfLength(5);
+                    final String defaultType = randomAlphaOfLength(3);
+                    final FetchSourceContext fetchSource = FetchSourceContext.FETCH_SOURCE;
+                    mgr.add(defaultIndex, defaultType, null, fetchSource, null, parser, true);
+                });
+        assertThat(
+                e.toString(),
+                containsString(
+                        "unexpected token [START_OBJECT], expected [FIELD_NAME] or [START_ARRAY]"));
+    }
+
+    public void testAddWithInvalidSourceValueIsRejected() throws Exception {
+        String sourceValue = randomFrom("on", "off", "0", "1");
+        XContentParser parser = createParser(XContentFactory.jsonBuilder()
+            .startObject()
+                .startArray("docs")
+                    .startObject()
+                        .field("_source", sourceValue)
+                    .endObject()
+                .endArray()
+            .endObject()
+        );
+
+        MultiGetRequest multiGetRequest = new MultiGetRequest();
+        multiGetRequest.add(
+            randomAlphaOfLength(5), randomAlphaOfLength(3), null, FetchSourceContext.FETCH_SOURCE, null, parser, true);
+
+        assertEquals(1, multiGetRequest.getItems().size());
+        assertWarnings("Expected a boolean [true/false] for property [_source] but got ["+ sourceValue  + "]");
+    }
+
+    public void testAddWithValidSourceValueIsAccepted() throws Exception {
+        XContentParser parser = createParser(XContentFactory.jsonBuilder()
+            .startObject()
+                .startArray("docs")
+                    .startObject()
+                        .field("_source", randomFrom("false", "true"))
+                    .endObject()
+                    .startObject()
+                        .field("_source", randomBoolean())
+                    .endObject()
+                .endArray()
+            .endObject()
+        );
+
+        MultiGetRequest multiGetRequest = new MultiGetRequest();
+        multiGetRequest.add(
+            randomAlphaOfLength(5), randomAlphaOfLength(3), null, FetchSourceContext.FETCH_SOURCE, null, parser, true);
+
+        assertEquals(2, multiGetRequest.getItems().size());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/get/MultiGetShardRequestTests.java b/core/src/test/java/org/elasticsearch/action/get/MultiGetShardRequestTests.java
index dab737cf7f50c..a130f10d65908 100644
--- a/core/src/test/java/org/elasticsearch/action/get/MultiGetShardRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/get/MultiGetShardRequestTests.java
@@ -34,7 +34,7 @@ public class MultiGetShardRequestTests extends ESTestCase {
     public void testSerialization() throws IOException {
         MultiGetRequest multiGetRequest = new MultiGetRequest();
         if (randomBoolean()) {
-            multiGetRequest.preference(randomAsciiOfLength(randomIntBetween(1, 10)));
+            multiGetRequest.preference(randomAlphaOfLength(randomIntBetween(1, 10)));
         }
         if (randomBoolean()) {
             multiGetRequest.realtime(false);
@@ -42,19 +42,17 @@ public void testSerialization() throws IOException {
         if (randomBoolean()) {
             multiGetRequest.refresh(true);
         }
-        multiGetRequest.ignoreErrorsOnGeneratedFields(randomBoolean());
-
         MultiGetShardRequest multiGetShardRequest = new MultiGetShardRequest(multiGetRequest, "index", 0);
         int numItems = iterations(10, 30);
         for (int i = 0; i < numItems; i++) {
-            MultiGetRequest.Item item = new MultiGetRequest.Item("alias-" + randomAsciiOfLength(randomIntBetween(1, 10)), "type", "id-" + i);
+            MultiGetRequest.Item item = new MultiGetRequest.Item("alias-" + randomAlphaOfLength(randomIntBetween(1, 10)), "type", "id-" + i);
             if (randomBoolean()) {
                 int numFields = randomIntBetween(1, 5);
                 String[] fields = new String[numFields];
                 for (int j = 0; j < fields.length; j++) {
-                    fields[j] = randomAsciiOfLength(randomIntBetween(1, 10));
+                    fields[j] = randomAlphaOfLength(randomIntBetween(1, 10));
                 }
-                item.fields(fields);
+                item.storedFields(fields);
             }
             if (randomBoolean()) {
                 item.version(randomIntBetween(1, Integer.MAX_VALUE));
@@ -79,7 +77,6 @@ public void testSerialization() throws IOException {
         assertThat(multiGetShardRequest2.preference(), equalTo(multiGetShardRequest.preference()));
         assertThat(multiGetShardRequest2.realtime(), equalTo(multiGetShardRequest.realtime()));
         assertThat(multiGetShardRequest2.refresh(), equalTo(multiGetShardRequest.refresh()));
-        assertThat(multiGetShardRequest2.ignoreErrorsOnGeneratedFields(), equalTo(multiGetShardRequest.ignoreErrorsOnGeneratedFields()));
         assertThat(multiGetShardRequest2.items.size(), equalTo(multiGetShardRequest.items.size()));
         for (int i = 0; i < multiGetShardRequest2.items.size(); i++) {
             MultiGetRequest.Item item = multiGetShardRequest.items.get(i);
@@ -87,7 +84,7 @@ public void testSerialization() throws IOException {
                 assertThat(item2.index(), equalTo(item.index()));
             assertThat(item2.type(), equalTo(item.type()));
             assertThat(item2.id(), equalTo(item.id()));
-            assertThat(item2.fields(), equalTo(item.fields()));
+            assertThat(item2.storedFields(), equalTo(item.storedFields()));
             assertThat(item2.version(), equalTo(item.version()));
             assertThat(item2.versionType(), equalTo(item.versionType()));
             assertThat(item2.fetchSourceContext(), equalTo(item.fetchSourceContext()));
diff --git a/core/src/test/java/org/elasticsearch/action/index/IndexRequestBuilderTests.java b/core/src/test/java/org/elasticsearch/action/index/IndexRequestBuilderTests.java
index 06e9d586e3671..5341588f7226b 100644
--- a/core/src/test/java/org/elasticsearch/action/index/IndexRequestBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/action/index/IndexRequestBuilderTests.java
@@ -74,8 +74,9 @@ public void testSetSource() throws Exception {
         ByteArrayOutputStream docOut = new ByteArrayOutputStream();
         XContentBuilder doc = XContentFactory.jsonBuilder(docOut).startObject().field("SomeKey", "SomeValue").endObject();
         doc.close();
-        indexRequestBuilder.setSource(docOut.toByteArray());
-        assertEquals(EXPECTED_SOURCE, XContentHelper.convertToJson(indexRequestBuilder.request().source(), true));
+        indexRequestBuilder.setSource(docOut.toByteArray(), XContentType.JSON);
+        assertEquals(EXPECTED_SOURCE, XContentHelper.convertToJson(indexRequestBuilder.request().source(), true,
+            indexRequestBuilder.request().getContentType()));
 
         doc = XContentFactory.jsonBuilder().startObject().field("SomeKey", "SomeValue").endObject();
         doc.close();
diff --git a/core/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java b/core/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java
index 355daf4068319..03b19df760c11 100644
--- a/core/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java
@@ -18,24 +18,36 @@
  */
 package org.elasticsearch.action.index;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.support.ActiveShardCount;
+import org.elasticsearch.action.support.replication.ReplicationResponse;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.VersionType;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESTestCase;
 
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
 import java.util.Arrays;
+import java.util.Base64;
 import java.util.HashSet;
 import java.util.Set;
 
+import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
 
-/**
- */
 public class IndexRequestTests extends ESTestCase {
     public void testIndexRequestOpTypeFromString() throws Exception {
         String create = "create";
@@ -43,18 +55,24 @@ public void testIndexRequestOpTypeFromString() throws Exception {
         String createUpper = "CREATE";
         String indexUpper = "INDEX";
 
-        assertThat(IndexRequest.OpType.fromString(create), equalTo(IndexRequest.OpType.CREATE));
-        assertThat(IndexRequest.OpType.fromString(index), equalTo(IndexRequest.OpType.INDEX));
-        assertThat(IndexRequest.OpType.fromString(createUpper), equalTo(IndexRequest.OpType.CREATE));
-        assertThat(IndexRequest.OpType.fromString(indexUpper), equalTo(IndexRequest.OpType.INDEX));
+        IndexRequest indexRequest = new IndexRequest("");
+        indexRequest.opType(create);
+        assertThat(indexRequest.opType() , equalTo(DocWriteRequest.OpType.CREATE));
+        indexRequest.opType(createUpper);
+        assertThat(indexRequest.opType() , equalTo(DocWriteRequest.OpType.CREATE));
+        indexRequest.opType(index);
+        assertThat(indexRequest.opType() , equalTo(DocWriteRequest.OpType.INDEX));
+        indexRequest.opType(indexUpper);
+        assertThat(indexRequest.opType() , equalTo(DocWriteRequest.OpType.INDEX));
     }
 
     public void testReadBogusString() {
         try {
-            IndexRequest.OpType.fromString("foobar");
+            IndexRequest indexRequest = new IndexRequest("");
+            indexRequest.opType("foobar");
             fail("Expected IllegalArgumentException");
         } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), containsString("opType [foobar] not allowed"));
+            assertThat(e.getMessage(), equalTo("opType must be 'create' or 'index', found: [foobar]"));
         }
     }
 
@@ -73,26 +91,26 @@ public void testCreateOperationRejectsVersions() {
     }
 
     public void testIndexingRejectsLongIds() {
-        String id = randomAsciiOfLength(511);
+        String id = randomAlphaOfLength(511);
         IndexRequest request = new IndexRequest("index", "type", id);
-        request.source("{}");
+        request.source("{}", XContentType.JSON);
         ActionRequestValidationException validate = request.validate();
         assertNull(validate);
 
-        id = randomAsciiOfLength(512);
+        id = randomAlphaOfLength(512);
         request = new IndexRequest("index", "type", id);
-        request.source("{}");
+        request.source("{}", XContentType.JSON);
         validate = request.validate();
         assertNull(validate);
 
-        id = randomAsciiOfLength(513);
+        id = randomAlphaOfLength(513);
         request = new IndexRequest("index", "type", id);
-        request.source("{}");
+        request.source("{}", XContentType.JSON);
         validate = request.validate();
         assertThat(validate, notNullValue());
         assertThat(validate.getMessage(),
                 containsString("id is too long, must be no longer than 512 bytes but was: 513"));
-}
+    }
 
     public void testSetTTLAsTimeValue() {
         IndexRequest indexRequest = new IndexRequest();
@@ -112,9 +130,9 @@ public void testSetTTLAsString() {
     public void testSetTTLAsLong() {
         IndexRequest indexRequest = new IndexRequest();
         String ttlAsString = randomTimeValue();
-        TimeValue ttl = TimeValue.parseTimeValue(ttlAsString, null, "ttl");
-        indexRequest.ttl(ttl.millis());
-        assertThat(indexRequest.ttl(), equalTo(ttl));
+        long millis = TimeValue.parseTimeValue(ttlAsString, null, "ttl").millis();
+        indexRequest.ttl(millis);
+        assertEquals(timeValueMillis(millis), indexRequest.ttl());
     }
 
     public void testValidateTTL() {
@@ -150,4 +168,93 @@ public void testAutoGenIdTimestampIsSet() {
         request.process(null, true, "index");
         assertEquals(IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, request.getAutoGeneratedTimestamp());
     }
+
+    public void testIndexResponse() {
+        ShardId shardId = new ShardId(randomAlphaOfLengthBetween(3, 10), randomAlphaOfLengthBetween(3, 10), randomIntBetween(0, 1000));
+        String type = randomAlphaOfLengthBetween(3, 10);
+        String id = randomAlphaOfLengthBetween(3, 10);
+        long version = randomLong();
+        boolean created = randomBoolean();
+        IndexResponse indexResponse = new IndexResponse(shardId, type, id, version, created);
+        int total = randomIntBetween(1, 10);
+        int successful = randomIntBetween(1, 10);
+        ReplicationResponse.ShardInfo shardInfo = new ReplicationResponse.ShardInfo(total, successful);
+        indexResponse.setShardInfo(shardInfo);
+        boolean forcedRefresh = false;
+        if (randomBoolean()) {
+            forcedRefresh = randomBoolean();
+            indexResponse.setForcedRefresh(forcedRefresh);
+        }
+        assertEquals(type, indexResponse.getType());
+        assertEquals(id, indexResponse.getId());
+        assertEquals(version, indexResponse.getVersion());
+        assertEquals(shardId, indexResponse.getShardId());
+        assertEquals(created ? RestStatus.CREATED : RestStatus.OK, indexResponse.status());
+        assertEquals(total, indexResponse.getShardInfo().getTotal());
+        assertEquals(successful, indexResponse.getShardInfo().getSuccessful());
+        assertEquals(forcedRefresh, indexResponse.forcedRefresh());
+        assertEquals("IndexResponse[index=" + shardId.getIndexName() + ",type=" + type + ",id="+ id +
+                ",version=" + version + ",result=" + (created ? "created" : "updated") +
+                ",shards={\"total\":" + total + ",\"successful\":" + successful + ",\"failed\":0}]",
+                indexResponse.toString());
+    }
+
+    public void testIndexRequestXContentSerialization() throws IOException {
+        IndexRequest indexRequest = new IndexRequest("foo", "bar", "1");
+        indexRequest.source("{}", XContentType.JSON);
+        assertEquals(XContentType.JSON, indexRequest.getContentType());
+
+        BytesStreamOutput out = new BytesStreamOutput();
+        indexRequest.writeTo(out);
+        StreamInput in = StreamInput.wrap(out.bytes().toBytesRef().bytes);
+        IndexRequest serialized = new IndexRequest();
+        serialized.readFrom(in);
+        assertEquals(XContentType.JSON, serialized.getContentType());
+        assertEquals(new BytesArray("{}"), serialized.source());
+    }
+
+    public void testIndexRequestXContentSerializationBwc() throws IOException {
+        final byte[] data = Base64.getDecoder().decode("AAD////+AgQDZm9vAAAAAQNiYXIBATEAAAAAAnt9AP/////////9AAAA//////////8AAAAAAAA=");
+        final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2,
+            Version.V_5_1_1, Version.V_5_1_2, Version.V_5_2_0);
+        try (StreamInput in = StreamInput.wrap(data)) {
+            in.setVersion(version);
+            IndexRequest serialized = new IndexRequest();
+            serialized.readFrom(in);
+            assertEquals(XContentType.JSON, serialized.getContentType());
+            assertEquals("{}", serialized.source().utf8ToString());
+            // don't test writing to earlier versions since output differs due to no timestamp
+        }
+    }
+
+    // reindex makes use of index requests without a source so this needs to be handled
+    public void testSerializationOfEmptyRequestWorks() throws IOException {
+        IndexRequest request = new IndexRequest("index", "type");
+        assertNull(request.getContentType());
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            request.writeTo(out);
+
+            try (StreamInput in = out.bytes().streamInput()) {
+                IndexRequest serialized = new IndexRequest();
+                serialized.readFrom(in);
+                assertNull(request.getContentType());
+                assertEquals("index", request.index());
+                assertEquals("type", request.type());
+            }
+        }
+    }
+
+    public void testToStringSizeLimit() throws UnsupportedEncodingException {
+        IndexRequest request = new IndexRequest("index", "type");
+
+        String source = "{\"name\":\"value\"}";
+        request.source(source);
+        assertEquals("index {[index][type][null], source[" + source + "]}", request.toString());
+
+        source = "{\"name\":\"" + randomUnicodeOfLength(IndexRequest.MAX_SOURCE_LENGTH_IN_TOSTRING) + "\"}";
+        request.source(source);
+        int actualBytes = source.getBytes("UTF-8").length;
+        assertEquals("index {[index][type][null], source[n/a, actual length: [" + new ByteSizeValue(actualBytes).toString() +
+                "], max length: " + new ByteSizeValue(IndexRequest.MAX_SOURCE_LENGTH_IN_TOSTRING).toString() + "]}", request.toString());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/index/IndexResponseTests.java b/core/src/test/java/org/elasticsearch/action/index/IndexResponseTests.java
new file mode 100644
index 0000000000000..8680eeec37379
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/index/IndexResponseTests.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.index;
+
+import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.action.support.replication.ReplicationResponse;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.RandomObjects;
+
+import java.io.IOException;
+import java.util.function.Predicate;
+
+import static org.elasticsearch.action.support.replication.ReplicationResponseTests.assertShardInfo;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.INDEX_UUID_NA_VALUE;
+import static org.elasticsearch.test.XContentTestUtils.insertRandomFields;
+
+public class IndexResponseTests extends ESTestCase {
+
+    public void testToXContent() {
+        {
+            IndexResponse indexResponse = new IndexResponse(new ShardId("index", "index_uuid", 0), "type", "id", 5, true);
+            String output = Strings.toString(indexResponse);
+            assertEquals("{\"_index\":\"index\",\"_type\":\"type\",\"_id\":\"id\",\"_version\":5,\"result\":\"created\",\"_shards\":null," +
+                    "\"created\":true}", output);
+        }
+        {
+            IndexResponse indexResponse = new IndexResponse(new ShardId("index", "index_uuid", 0), "type", "id", 7, true);
+            indexResponse.setForcedRefresh(true);
+            indexResponse.setShardInfo(new ReplicationResponse.ShardInfo(10, 5));
+            String output = Strings.toString(indexResponse);
+            assertEquals("{\"_index\":\"index\",\"_type\":\"type\",\"_id\":\"id\",\"_version\":7,\"result\":\"created\"," +
+                    "\"forced_refresh\":true,\"_shards\":{\"total\":10,\"successful\":5,\"failed\":0},\"created\":true}", output);
+        }
+    }
+
+    public void testToAndFromXContent() throws IOException {
+        doFromXContentTestWithRandomFields(false);
+    }
+
+    /**
+     * This test adds random fields and objects to the xContent rendered out to
+     * ensure we can parse it back to be forward compatible with additions to
+     * the xContent
+     */
+    public void testFromXContentWithRandomFields() throws IOException {
+        doFromXContentTestWithRandomFields(true);
+    }
+
+    private void doFromXContentTestWithRandomFields(boolean addRandomFields) throws IOException {
+        final Tuple<IndexResponse, IndexResponse> tuple = randomIndexResponse();
+        IndexResponse indexResponse = tuple.v1();
+        IndexResponse expectedIndexResponse = tuple.v2();
+
+        boolean humanReadable = randomBoolean();
+        XContentType xContentType = randomFrom(XContentType.values());
+        BytesReference originalBytes = toShuffledXContent(indexResponse, xContentType, ToXContent.EMPTY_PARAMS, humanReadable);
+
+        BytesReference mutated;
+        if (addRandomFields) {
+            // The ShardInfo.Failure's exception is rendered out in a "reason" object. We shouldn't add anything random there
+            // because exception rendering and parsing are very permissive: any extra object or field would be rendered as
+            // a exception custom metadata and be parsed back as a custom header, making it impossible to compare the results
+            // in this test.
+            Predicate<String> excludeFilter = path -> path.contains("reason");
+            mutated = insertRandomFields(xContentType, originalBytes, excludeFilter, random());
+        } else {
+            mutated = originalBytes;
+        }
+        IndexResponse parsedIndexResponse;
+        try (XContentParser parser = createParser(xContentType.xContent(), mutated)) {
+            parsedIndexResponse = IndexResponse.fromXContent(parser);
+            assertNull(parser.nextToken());
+        }
+
+        // We can't use equals() to compare the original and the parsed index response
+        // because the random index response can contain shard failures with exceptions,
+        // and those exceptions are not parsed back with the same types.
+        assertDocWriteResponse(expectedIndexResponse, parsedIndexResponse);
+    }
+
+    public static void assertDocWriteResponse(DocWriteResponse expected, DocWriteResponse actual) {
+        assertEquals(expected.getIndex(), actual.getIndex());
+        assertEquals(expected.getType(), actual.getType());
+        assertEquals(expected.getId(), actual.getId());
+        assertEquals(expected.getResult(), actual.getResult());
+        assertEquals(expected.getShardId(), actual.getShardId());
+        assertEquals(expected.forcedRefresh(), actual.forcedRefresh());
+        assertEquals(expected.status(), actual.status());
+        assertShardInfo(expected.getShardInfo(), actual.getShardInfo());
+    }
+
+    /**
+     * Returns a tuple of {@link IndexResponse}s.
+     * <p>
+     * The left element is the actual {@link IndexResponse} to serialize while the right element is the
+     * expected {@link IndexResponse} after parsing.
+     */
+    public static Tuple<IndexResponse, IndexResponse> randomIndexResponse() {
+        String index = randomAlphaOfLength(5);
+        String indexUUid = randomAlphaOfLength(5);
+        int shardId = randomIntBetween(0, 5);
+        String type = randomAlphaOfLength(5);
+        String id = randomAlphaOfLength(5);
+        long version = randomBoolean() ? randomNonNegativeLong() : randomIntBetween(0, 10000);
+        boolean created = randomBoolean();
+        boolean forcedRefresh = randomBoolean();
+
+        Tuple<ReplicationResponse.ShardInfo, ReplicationResponse.ShardInfo> shardInfos = RandomObjects.randomShardInfo(random());
+
+        IndexResponse actual = new IndexResponse(new ShardId(index, indexUUid, shardId), type, id, version, created);
+        actual.setForcedRefresh(forcedRefresh);
+        actual.setShardInfo(shardInfos.v1());
+
+        IndexResponse expected = new IndexResponse(new ShardId(index, INDEX_UUID_NA_VALUE, -1), type, id, version, created);
+        expected.setForcedRefresh(forcedRefresh);
+        expected.setShardInfo(shardInfos.v2());
+
+        return Tuple.tuple(actual, expected);
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/ingest/IngestActionFilterTests.java b/core/src/test/java/org/elasticsearch/action/ingest/IngestActionFilterTests.java
deleted file mode 100644
index 1316c87e2aa00..0000000000000
--- a/core/src/test/java/org/elasticsearch/action/ingest/IngestActionFilterTests.java
+++ /dev/null
@@ -1,232 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.action.ingest;
-
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ActionRequest;
-import org.elasticsearch.action.bulk.BulkAction;
-import org.elasticsearch.action.bulk.BulkRequest;
-import org.elasticsearch.action.delete.DeleteRequest;
-import org.elasticsearch.action.index.IndexAction;
-import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.support.ActionFilterChain;
-import org.elasticsearch.action.update.UpdateRequest;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.ingest.IngestService;
-import org.elasticsearch.ingest.PipelineExecutionService;
-import org.elasticsearch.ingest.PipelineStore;
-import org.elasticsearch.ingest.TestProcessor;
-import org.elasticsearch.ingest.CompoundProcessor;
-import org.elasticsearch.ingest.Pipeline;
-import org.elasticsearch.ingest.Processor;
-import org.elasticsearch.node.service.NodeService;
-import org.elasticsearch.tasks.Task;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.junit.Before;
-import org.mockito.stubbing.Answer;
-
-import java.util.function.Consumer;
-
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.nullValue;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Matchers.same;
-import static org.mockito.Mockito.any;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.verifyZeroInteractions;
-import static org.mockito.Mockito.when;
-
-public class IngestActionFilterTests extends ESTestCase {
-
-    private IngestActionFilter filter;
-    private PipelineExecutionService executionService;
-
-    @Before
-    public void setup() {
-        executionService = mock(PipelineExecutionService.class);
-        IngestService ingestService = mock(IngestService.class);
-        when(ingestService.getPipelineExecutionService()).thenReturn(executionService);
-        NodeService nodeService = mock(NodeService.class);
-        when(nodeService.getIngestService()).thenReturn(ingestService);
-        filter = new IngestActionFilter(Settings.EMPTY, nodeService);
-    }
-
-    public void testApplyNoPipelineId() throws Exception {
-        IndexRequest indexRequest = new IndexRequest();
-        Task task = mock(Task.class);
-        ActionListener actionListener = mock(ActionListener.class);
-        ActionFilterChain actionFilterChain = mock(ActionFilterChain.class);
-
-        filter.apply(task, IndexAction.NAME, indexRequest, actionListener, actionFilterChain);
-
-        verify(actionFilterChain).proceed(task, IndexAction.NAME, indexRequest, actionListener);
-        verifyZeroInteractions(executionService, actionFilterChain);
-    }
-
-    public void testApplyBulkNoPipelineId() throws Exception {
-        BulkRequest bulkRequest = new BulkRequest();
-        bulkRequest.add(new IndexRequest());
-        Task task = mock(Task.class);
-        ActionListener actionListener = mock(ActionListener.class);
-        ActionFilterChain actionFilterChain = mock(ActionFilterChain.class);
-
-        filter.apply(task, BulkAction.NAME, bulkRequest, actionListener, actionFilterChain);
-
-        verify(actionFilterChain).proceed(task, BulkAction.NAME, bulkRequest, actionListener);
-        verifyZeroInteractions(executionService, actionFilterChain);
-    }
-
-    @SuppressWarnings("unchecked")
-    public void testApplyIngestIdViaRequestParam() throws Exception {
-        Task task = mock(Task.class);
-        IndexRequest indexRequest = new IndexRequest("_index", "_type", "_id").setPipeline("_id");
-        indexRequest.source("field", "value");
-        ActionListener actionListener = mock(ActionListener.class);
-        ActionFilterChain actionFilterChain = mock(ActionFilterChain.class);
-
-        filter.apply(task, IndexAction.NAME, indexRequest, actionListener, actionFilterChain);
-
-        verify(executionService).executeIndexRequest(same(indexRequest), any(Consumer.class), any(Consumer.class));
-        verifyZeroInteractions(actionFilterChain);
-    }
-
-    @SuppressWarnings("unchecked")
-    public void testApplyExecuted() throws Exception {
-        Task task = mock(Task.class);
-        IndexRequest indexRequest = new IndexRequest("_index", "_type", "_id").setPipeline("_id");
-        indexRequest.source("field", "value");
-        ActionListener actionListener = mock(ActionListener.class);
-        ActionFilterChain actionFilterChain = mock(ActionFilterChain.class);
-
-        Answer answer = invocationOnMock -> {
-            @SuppressWarnings("unchecked")
-            Consumer<Boolean> listener = (Consumer) invocationOnMock.getArguments()[2];
-            listener.accept(true);
-            return null;
-        };
-        doAnswer(answer).when(executionService).executeIndexRequest(any(IndexRequest.class), any(Consumer.class), any(Consumer.class));
-        filter.apply(task, IndexAction.NAME, indexRequest, actionListener, actionFilterChain);
-
-        verify(executionService).executeIndexRequest(same(indexRequest), any(Consumer.class), any(Consumer.class));
-        verify(actionFilterChain).proceed(task, IndexAction.NAME, indexRequest, actionListener);
-        verifyZeroInteractions(actionListener);
-    }
-
-    @SuppressWarnings("unchecked")
-    public void testApplyFailed() throws Exception {
-        Task task = mock(Task.class);
-        IndexRequest indexRequest = new IndexRequest("_index", "_type", "_id").setPipeline("_id");
-        indexRequest.source("field", "value");
-        ActionListener actionListener = mock(ActionListener.class);
-        ActionFilterChain actionFilterChain = mock(ActionFilterChain.class);
-
-        RuntimeException exception = new RuntimeException();
-        Answer answer = invocationOnMock -> {
-            Consumer<Throwable> handler = (Consumer) invocationOnMock.getArguments()[1];
-            handler.accept(exception);
-            return null;
-        };
-        doAnswer(answer).when(executionService).executeIndexRequest(same(indexRequest), any(Consumer.class), any(Consumer.class));
-        filter.apply(task, IndexAction.NAME, indexRequest, actionListener, actionFilterChain);
-
-        verify(executionService).executeIndexRequest(same(indexRequest), any(Consumer.class), any(Consumer.class));
-        verify(actionListener).onFailure(exception);
-        verifyZeroInteractions(actionFilterChain);
-    }
-
-    public void testApplyWithBulkRequest() throws Exception {
-        Task task = mock(Task.class);
-        ThreadPool threadPool = mock(ThreadPool.class);
-        when(threadPool.executor(any())).thenReturn(Runnable::run);
-        PipelineStore store = mock(PipelineStore.class);
-
-        Processor processor = new TestProcessor(ingestDocument -> ingestDocument.setFieldValue("field2", "value2"));
-        when(store.get("_id")).thenReturn(new Pipeline("_id", "_description", randomInt(), new CompoundProcessor(processor)));
-        executionService = new PipelineExecutionService(store, threadPool);
-        IngestService ingestService = mock(IngestService.class);
-        when(ingestService.getPipelineExecutionService()).thenReturn(executionService);
-        NodeService nodeService = mock(NodeService.class);
-        when(nodeService.getIngestService()).thenReturn(ingestService);
-        filter = new IngestActionFilter(Settings.EMPTY, nodeService);
-
-        BulkRequest bulkRequest = new BulkRequest();
-        int numRequest = scaledRandomIntBetween(8, 64);
-        for (int i = 0; i < numRequest; i++) {
-            if (rarely()) {
-                ActionRequest request;
-                if (randomBoolean()) {
-                    request = new DeleteRequest("_index", "_type", "_id");
-                } else {
-                    request = new UpdateRequest("_index", "_type", "_id");
-                }
-                bulkRequest.add(request);
-            } else {
-                IndexRequest indexRequest = new IndexRequest("_index", "_type", "_id").setPipeline("_id");
-                indexRequest.source("field1", "value1");
-                bulkRequest.add(indexRequest);
-            }
-        }
-
-        ActionListener actionListener = mock(ActionListener.class);
-        ActionFilterChain actionFilterChain = mock(ActionFilterChain.class);
-
-        filter.apply(task, BulkAction.NAME, bulkRequest, actionListener, actionFilterChain);
-        verify(actionFilterChain).proceed(eq(task), eq(BulkAction.NAME), eq(bulkRequest), any());
-        verifyZeroInteractions(actionListener);
-
-        int assertedRequests = 0;
-        for (ActionRequest actionRequest : bulkRequest.requests()) {
-            if (actionRequest instanceof IndexRequest) {
-                IndexRequest indexRequest = (IndexRequest) actionRequest;
-                assertThat(indexRequest.sourceAsMap().size(), equalTo(2));
-                assertThat(indexRequest.sourceAsMap().get("field1"), equalTo("value1"));
-                assertThat(indexRequest.sourceAsMap().get("field2"), equalTo("value2"));
-            }
-            assertedRequests++;
-        }
-        assertThat(assertedRequests, equalTo(numRequest));
-    }
-
-    @SuppressWarnings("unchecked")
-    public void testIndexApiSinglePipelineExecution() {
-        Answer answer = invocationOnMock -> {
-            @SuppressWarnings("unchecked")
-            Consumer<Boolean> listener = (Consumer) invocationOnMock.getArguments()[2];
-            listener.accept(true);
-            return null;
-        };
-        doAnswer(answer).when(executionService).executeIndexRequest(any(IndexRequest.class), any(Consumer.class), any(Consumer.class));
-
-        Task task = mock(Task.class);
-        ActionListener actionListener = mock(ActionListener.class);
-        ActionFilterChain actionFilterChain = mock(ActionFilterChain.class);
-
-        IndexRequest indexRequest = new IndexRequest("_index", "_type", "_id").setPipeline("_id").source("field", "value");
-        filter.apply(task, IndexAction.NAME, indexRequest, actionListener, actionFilterChain);
-        assertThat(indexRequest.getPipeline(), nullValue());
-        filter.apply(task, IndexAction.NAME, indexRequest, actionListener, actionFilterChain);
-        verify(executionService, times(1)).executeIndexRequest(same(indexRequest), any(Consumer.class), any(Consumer.class));
-        verify(actionFilterChain, times(2)).proceed(task, IndexAction.NAME, indexRequest, actionListener);
-    }
-}
diff --git a/core/src/test/java/org/elasticsearch/action/ingest/IngestProxyActionFilterTests.java b/core/src/test/java/org/elasticsearch/action/ingest/IngestProxyActionFilterTests.java
deleted file mode 100644
index a602465197aa5..0000000000000
--- a/core/src/test/java/org/elasticsearch/action/ingest/IngestProxyActionFilterTests.java
+++ /dev/null
@@ -1,252 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.action.ingest;
-
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ActionRequest;
-import org.elasticsearch.action.bulk.BulkAction;
-import org.elasticsearch.action.bulk.BulkRequest;
-import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.action.index.IndexAction;
-import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.index.IndexResponse;
-import org.elasticsearch.action.support.ActionFilterChain;
-import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.transport.LocalTransportAddress;
-import org.elasticsearch.tasks.Task;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.VersionUtils;
-import org.elasticsearch.transport.TransportException;
-import org.elasticsearch.transport.TransportRequest;
-import org.elasticsearch.transport.TransportResponse;
-import org.elasticsearch.transport.TransportResponseHandler;
-import org.elasticsearch.transport.TransportService;
-import org.hamcrest.CustomTypeSafeMatcher;
-import org.mockito.stubbing.Answer;
-
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.argThat;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Matchers.same;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.never;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.verifyZeroInteractions;
-import static org.mockito.Mockito.when;
-
-public class IngestProxyActionFilterTests extends ESTestCase {
-
-    private TransportService transportService;
-
-    @SuppressWarnings("unchecked")
-    private IngestProxyActionFilter buildFilter(int ingestNodes, int totalNodes) {
-        ClusterState.Builder clusterState = new ClusterState.Builder(new ClusterName("_name"));
-        DiscoveryNodes.Builder builder = new DiscoveryNodes.Builder();
-        DiscoveryNode localNode = null;
-        for (int i = 0; i < totalNodes; i++) {
-            String nodeId = "node" + i;
-            Map<String, String> attributes = new HashMap<>();
-            Set<DiscoveryNode.Role> roles = new HashSet<>();
-            if (i < ingestNodes) {
-                roles.add(DiscoveryNode.Role.INGEST);
-            }
-            DiscoveryNode node = new DiscoveryNode(nodeId, nodeId, LocalTransportAddress.buildUnique(), attributes, roles, VersionUtils.randomVersion(random()));
-            builder.add(node);
-            if (i == totalNodes - 1) {
-                localNode = node;
-            }
-        }
-        clusterState.nodes(builder);
-        ClusterService clusterService = mock(ClusterService.class);
-        when(clusterService.localNode()).thenReturn(localNode);
-        when(clusterService.state()).thenReturn(clusterState.build());
-        transportService = mock(TransportService.class);
-        return new IngestProxyActionFilter(clusterService, transportService);
-    }
-
-    public void testApplyNoIngestNodes() {
-        Task task = mock(Task.class);
-        ActionListener actionListener = mock(ActionListener.class);
-        ActionFilterChain actionFilterChain = mock(ActionFilterChain.class);
-        int totalNodes = randomIntBetween(1, 5);
-        IngestProxyActionFilter filter = buildFilter(0, totalNodes);
-
-        String action;
-        ActionRequest request;
-        if (randomBoolean()) {
-            action = IndexAction.NAME;
-            request = new IndexRequest().setPipeline("_id");
-        } else {
-            action = BulkAction.NAME;
-            request = new BulkRequest().add(new IndexRequest().setPipeline("_id"));
-        }
-        try {
-            filter.apply(task, action, request, actionListener, actionFilterChain);
-            fail("should have failed because there are no ingest nodes");
-        } catch(IllegalStateException e) {
-            assertThat(e.getMessage(), equalTo("There are no ingest nodes in this cluster, unable to forward request to an ingest node."));
-        }
-        verifyZeroInteractions(transportService);
-        verifyZeroInteractions(actionFilterChain);
-        verifyZeroInteractions(actionListener);
-    }
-
-    public void testApplyNoPipelineId() {
-        Task task = mock(Task.class);
-        ActionListener actionListener = mock(ActionListener.class);
-        ActionFilterChain actionFilterChain = mock(ActionFilterChain.class);
-        int totalNodes = randomIntBetween(1, 5);
-        IngestProxyActionFilter filter = buildFilter(randomIntBetween(0, totalNodes - 1), totalNodes);
-
-        String action;
-        ActionRequest request;
-        if (randomBoolean()) {
-            action = IndexAction.NAME;
-            request = new IndexRequest();
-        } else {
-            action = BulkAction.NAME;
-            request = new BulkRequest().add(new IndexRequest());
-        }
-        filter.apply(task, action, request, actionListener, actionFilterChain);
-        verifyZeroInteractions(transportService);
-        verify(actionFilterChain).proceed(any(Task.class), eq(action), same(request), same(actionListener));
-        verifyZeroInteractions(actionListener);
-    }
-
-    public void testApplyAnyAction() {
-        Task task = mock(Task.class);
-        ActionListener actionListener = mock(ActionListener.class);
-        ActionFilterChain actionFilterChain = mock(ActionFilterChain.class);
-        ActionRequest request = mock(ActionRequest.class);
-        int totalNodes = randomIntBetween(1, 5);
-        IngestProxyActionFilter filter = buildFilter(randomIntBetween(0, totalNodes - 1), totalNodes);
-
-        String action = randomAsciiOfLengthBetween(1, 20);
-        filter.apply(task, action, request, actionListener, actionFilterChain);
-        verifyZeroInteractions(transportService);
-        verify(actionFilterChain).proceed(any(Task.class), eq(action), same(request), same(actionListener));
-        verifyZeroInteractions(actionListener);
-    }
-
-    @SuppressWarnings("unchecked")
-    public void testApplyIndexRedirect() {
-        Task task = mock(Task.class);
-        ActionListener actionListener = mock(ActionListener.class);
-        ActionFilterChain actionFilterChain = mock(ActionFilterChain.class);
-        int totalNodes = randomIntBetween(2, 5);
-        IngestProxyActionFilter filter = buildFilter(randomIntBetween(1, totalNodes - 1), totalNodes);
-        Answer<Void> answer = invocationOnMock -> {
-            TransportResponseHandler transportResponseHandler = (TransportResponseHandler) invocationOnMock.getArguments()[3];
-            transportResponseHandler.handleResponse(new IndexResponse());
-            return null;
-        };
-        doAnswer(answer).when(transportService).sendRequest(any(DiscoveryNode.class), any(String.class), any(TransportRequest.class), any(TransportResponseHandler.class));
-
-        IndexRequest indexRequest = new IndexRequest().setPipeline("_id");
-        filter.apply(task, IndexAction.NAME, indexRequest, actionListener, actionFilterChain);
-
-        verify(transportService).sendRequest(argThat(new IngestNodeMatcher()), eq(IndexAction.NAME), same(indexRequest), any(TransportResponseHandler.class));
-        verifyZeroInteractions(actionFilterChain);
-        verify(actionListener).onResponse(any(IndexResponse.class));
-        verify(actionListener, never()).onFailure(any(TransportException.class));
-    }
-
-    @SuppressWarnings("unchecked")
-    public void testApplyBulkRedirect() {
-        Task task = mock(Task.class);
-        ActionListener actionListener = mock(ActionListener.class);
-        ActionFilterChain actionFilterChain = mock(ActionFilterChain.class);
-        int totalNodes = randomIntBetween(2, 5);
-        IngestProxyActionFilter filter = buildFilter(randomIntBetween(1, totalNodes - 1), totalNodes);
-        Answer<Void> answer = invocationOnMock -> {
-            TransportResponseHandler transportResponseHandler = (TransportResponseHandler) invocationOnMock.getArguments()[3];
-            transportResponseHandler.handleResponse(new BulkResponse(null, -1));
-            return null;
-        };
-        doAnswer(answer).when(transportService).sendRequest(any(DiscoveryNode.class), any(String.class), any(TransportRequest.class), any(TransportResponseHandler.class));
-
-        BulkRequest bulkRequest = new BulkRequest();
-        bulkRequest.add(new IndexRequest().setPipeline("_id"));
-        int numNoPipelineRequests = randomIntBetween(0, 10);
-        for (int i = 0; i < numNoPipelineRequests; i++) {
-            bulkRequest.add(new IndexRequest());
-        }
-        filter.apply(task, BulkAction.NAME, bulkRequest, actionListener, actionFilterChain);
-
-        verify(transportService).sendRequest(argThat(new IngestNodeMatcher()), eq(BulkAction.NAME), same(bulkRequest), any(TransportResponseHandler.class));
-        verifyZeroInteractions(actionFilterChain);
-        verify(actionListener).onResponse(any(BulkResponse.class));
-        verify(actionListener, never()).onFailure(any(TransportException.class));
-    }
-
-    @SuppressWarnings("unchecked")
-    public void testApplyFailures() {
-        Task task = mock(Task.class);
-        ActionListener actionListener = mock(ActionListener.class);
-        ActionFilterChain actionFilterChain = mock(ActionFilterChain.class);
-        int totalNodes = randomIntBetween(2, 5);
-        IngestProxyActionFilter filter = buildFilter(randomIntBetween(1, totalNodes - 1), totalNodes);
-        Answer<Void> answer = invocationOnMock -> {
-            TransportResponseHandler transportResponseHandler = (TransportResponseHandler) invocationOnMock.getArguments()[3];
-            transportResponseHandler.handleException(new TransportException(new IllegalArgumentException()));
-            return null;
-        };
-        doAnswer(answer).when(transportService).sendRequest(any(DiscoveryNode.class), any(String.class), any(TransportRequest.class), any(TransportResponseHandler.class));
-
-        String action;
-        ActionRequest request;
-        if (randomBoolean()) {
-            action = IndexAction.NAME;
-            request = new IndexRequest().setPipeline("_id");
-        } else {
-            action = BulkAction.NAME;
-            request = new BulkRequest().add(new IndexRequest().setPipeline("_id"));
-        }
-
-        filter.apply(task, action, request, actionListener, actionFilterChain);
-
-        verify(transportService).sendRequest(argThat(new IngestNodeMatcher()), eq(action), same(request), any(TransportResponseHandler.class));
-        verifyZeroInteractions(actionFilterChain);
-        verify(actionListener).onFailure(any(TransportException.class));
-        verify(actionListener, never()).onResponse(any(TransportResponse.class));
-    }
-
-    private static class IngestNodeMatcher extends CustomTypeSafeMatcher<DiscoveryNode> {
-        private IngestNodeMatcher() {
-            super("discovery node should be an ingest node");
-        }
-
-        @Override
-        protected boolean matchesSafely(DiscoveryNode node) {
-            return node.isIngestNode();
-        }
-    }
-}
diff --git a/core/src/test/java/org/elasticsearch/action/ingest/PutPipelineRequestTests.java b/core/src/test/java/org/elasticsearch/action/ingest/PutPipelineRequestTests.java
new file mode 100644
index 0000000000000..37a15c5b3592f
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/ingest/PutPipelineRequestTests.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.ingest;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+
+public class PutPipelineRequestTests extends ESTestCase {
+
+    public void testSerializationWithXContent() throws IOException {
+        PutPipelineRequest request = new PutPipelineRequest("1", new BytesArray("{}".getBytes(StandardCharsets.UTF_8)), XContentType.JSON);
+        assertEquals(XContentType.JSON, request.getXContentType());
+
+        BytesStreamOutput output = new BytesStreamOutput();
+        request.writeTo(output);
+        StreamInput in = StreamInput.wrap(output.bytes().toBytesRef().bytes);
+
+        PutPipelineRequest serialized = new PutPipelineRequest();
+        serialized.readFrom(in);
+        assertEquals(XContentType.JSON, serialized.getXContentType());
+        assertEquals("{}", serialized.getSource().utf8ToString());
+    }
+
+    public void testSerializationBwc() throws IOException {
+        final byte[] data = Base64.getDecoder().decode("ADwDATECe30=");
+        final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2,
+            Version.V_5_1_1, Version.V_5_1_2, Version.V_5_2_0);
+        try (StreamInput in = StreamInput.wrap(data)) {
+            in.setVersion(version);
+            PutPipelineRequest request = new PutPipelineRequest();
+            request.readFrom(in);
+            assertEquals(XContentType.JSON, request.getXContentType());
+            assertEquals("{}", request.getSource().utf8ToString());
+
+            try (BytesStreamOutput out = new BytesStreamOutput()) {
+                out.setVersion(version);
+                request.writeTo(out);
+                assertArrayEquals(data, out.bytes().toBytesRef().bytes);
+            }
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentSimpleResultTests.java b/core/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentSimpleResultTests.java
index 544e2932b4415..83aad26f6a07b 100644
--- a/core/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentSimpleResultTests.java
+++ b/core/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentSimpleResultTests.java
@@ -27,7 +27,7 @@
 
 import java.io.IOException;
 
-import static org.elasticsearch.ingest.IngestDocumentTests.assertIngestDocument;
+import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.instanceOf;
 
diff --git a/core/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java b/core/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java
index 8e32334a9db5e..a4320d264110e 100644
--- a/core/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java
@@ -34,7 +34,7 @@
 import java.util.Collections;
 import java.util.Map;
 
-import static org.elasticsearch.ingest.IngestDocumentTests.assertIngestDocument;
+import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.not;
diff --git a/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java b/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java
index ab5d30c6f9b7e..6f8280277e0fc 100644
--- a/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java
+++ b/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java
@@ -71,14 +71,14 @@ public void testParseUsingPipelineStore() throws Exception {
         requestContent.put(Fields.DOCS, docs);
         for (int i = 0; i < numDocs; i++) {
             Map<String, Object> doc = new HashMap<>();
-            String index = randomAsciiOfLengthBetween(1, 10);
-            String type = randomAsciiOfLengthBetween(1, 10);
-            String id = randomAsciiOfLengthBetween(1, 10);
+            String index = randomAlphaOfLengthBetween(1, 10);
+            String type = randomAlphaOfLengthBetween(1, 10);
+            String id = randomAlphaOfLengthBetween(1, 10);
             doc.put(INDEX.getFieldName(), index);
             doc.put(TYPE.getFieldName(), type);
             doc.put(ID.getFieldName(), id);
-            String fieldName = randomAsciiOfLengthBetween(1, 10);
-            String fieldValue = randomAsciiOfLengthBetween(1, 10);
+            String fieldName = randomAlphaOfLengthBetween(1, 10);
+            String fieldValue = randomAlphaOfLengthBetween(1, 10);
             doc.put(Fields.SOURCE, Collections.singletonMap(fieldName, fieldValue));
             docs.add(doc);
             Map<String, Object> expectedDoc = new HashMap<>();
@@ -116,14 +116,14 @@ public void testParseWithProvidedPipeline() throws Exception {
         requestContent.put(Fields.DOCS, docs);
         for (int i = 0; i < numDocs; i++) {
             Map<String, Object> doc = new HashMap<>();
-            String index = randomAsciiOfLengthBetween(1, 10);
-            String type = randomAsciiOfLengthBetween(1, 10);
-            String id = randomAsciiOfLengthBetween(1, 10);
+            String index = randomAlphaOfLengthBetween(1, 10);
+            String type = randomAlphaOfLengthBetween(1, 10);
+            String id = randomAlphaOfLengthBetween(1, 10);
             doc.put(INDEX.getFieldName(), index);
             doc.put(TYPE.getFieldName(), type);
             doc.put(ID.getFieldName(), id);
-            String fieldName = randomAsciiOfLengthBetween(1, 10);
-            String fieldValue = randomAsciiOfLengthBetween(1, 10);
+            String fieldName = randomAlphaOfLengthBetween(1, 10);
+            String fieldValue = randomAlphaOfLengthBetween(1, 10);
             doc.put(Fields.SOURCE, Collections.singletonMap(fieldName, fieldValue));
             docs.add(doc);
             Map<String, Object> expectedDoc = new HashMap<>();
@@ -190,7 +190,7 @@ public void testNullPipelineId() {
     }
 
     public void testNonExistentPipelineId() {
-        String pipelineId = randomAsciiOfLengthBetween(1, 10);
+        String pipelineId = randomAlphaOfLengthBetween(1, 10);
         Map<String, Object> requestContent = new HashMap<>();
         List<Map<String, Object>> docs = new ArrayList<>();
         requestContent.put(Fields.DOCS, docs);
diff --git a/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestTests.java b/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestTests.java
index 2c2506308ac78..ecd0256b11068 100644
--- a/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestTests.java
@@ -19,31 +19,26 @@
 
 package org.elasticsearch.action.ingest;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.ingest.IngestDocument;
-import org.elasticsearch.ingest.RandomDocumentPicks;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
 
-import static org.elasticsearch.ingest.IngestDocumentTests.assertIngestDocument;
 import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.CoreMatchers.instanceOf;
-import static org.hamcrest.CoreMatchers.nullValue;
 
 public class SimulatePipelineRequestTests extends ESTestCase {
 
     public void testSerialization() throws IOException {
-        SimulatePipelineRequest request = new SimulatePipelineRequest(new BytesArray(""));
+        SimulatePipelineRequest request = new SimulatePipelineRequest(new BytesArray(""), XContentType.JSON);
         // Sometimes we set an id
         if (randomBoolean()) {
-            request.setId(randomAsciiOfLengthBetween(1, 10));
+            request.setId(randomAlphaOfLengthBetween(1, 10));
         }
 
         // Sometimes we explicitly set a boolean (with whatever value)
@@ -60,4 +55,38 @@ public void testSerialization() throws IOException {
         assertThat(otherRequest.getId(), equalTo(request.getId()));
         assertThat(otherRequest.isVerbose(), equalTo(request.isVerbose()));
     }
+
+    public void testSerializationWithXContent() throws IOException {
+        SimulatePipelineRequest request =
+            new SimulatePipelineRequest(new BytesArray("{}".getBytes(StandardCharsets.UTF_8)), XContentType.JSON);
+        assertEquals(XContentType.JSON, request.getXContentType());
+
+        BytesStreamOutput output = new BytesStreamOutput();
+        request.writeTo(output);
+        StreamInput in = StreamInput.wrap(output.bytes().toBytesRef().bytes);
+
+        SimulatePipelineRequest serialized = new SimulatePipelineRequest();
+        serialized.readFrom(in);
+        assertEquals(XContentType.JSON, serialized.getXContentType());
+        assertEquals("{}", serialized.getSource().utf8ToString());
+    }
+
+    public void testSerializationWithXContentBwc() throws IOException {
+        final byte[] data = Base64.getDecoder().decode("AAAAAnt9AAA=");
+        final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2,
+            Version.V_5_1_1, Version.V_5_1_2, Version.V_5_2_0);
+        try (StreamInput in = StreamInput.wrap(data)) {
+            in.setVersion(version);
+            SimulatePipelineRequest request = new SimulatePipelineRequest();
+            request.readFrom(in);
+            assertEquals(XContentType.JSON, request.getXContentType());
+            assertEquals("{}", request.getSource().utf8ToString());
+
+            try (BytesStreamOutput out = new BytesStreamOutput()) {
+                out.setVersion(version);
+                request.writeTo(out);
+                assertArrayEquals(data, out.bytes().toBytesRef().bytes);
+            }
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineResponseTests.java b/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineResponseTests.java
index 485dc8934c484..be448a09db892 100644
--- a/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineResponseTests.java
+++ b/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineResponseTests.java
@@ -30,7 +30,7 @@
 import java.util.Iterator;
 import java.util.List;
 
-import static org.elasticsearch.ingest.IngestDocumentTests.assertIngestDocument;
+import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.hamcrest.CoreMatchers.nullValue;
@@ -39,7 +39,7 @@ public class SimulatePipelineResponseTests extends ESTestCase {
 
     public void testSerialization() throws IOException {
         boolean isVerbose = randomBoolean();
-        String id = randomBoolean() ? randomAsciiOfLengthBetween(1, 10) : null;
+        String id = randomBoolean() ? randomAlphaOfLengthBetween(1, 10) : null;
         int numResults = randomIntBetween(1, 10);
         List<SimulateDocumentResult> results = new ArrayList<>(numResults);
         for (int i = 0; i < numResults; i++) {
@@ -49,7 +49,7 @@ public void testSerialization() throws IOException {
                 int numProcessors = randomIntBetween(1, 10);
                 List<SimulateProcessorResult> processorResults = new ArrayList<>(numProcessors);
                 for (int j = 0; j < numProcessors; j++) {
-                    String processorTag = randomAsciiOfLengthBetween(1, 10);
+                    String processorTag = randomAlphaOfLengthBetween(1, 10);
                     SimulateProcessorResult processorResult;
                     if (isFailure) {
                         processorResult = new SimulateProcessorResult(processorTag, new IllegalArgumentException("test"));
diff --git a/core/src/test/java/org/elasticsearch/action/ingest/SimulateProcessorResultTests.java b/core/src/test/java/org/elasticsearch/action/ingest/SimulateProcessorResultTests.java
index f6ffc035342ff..3014a1a4ae61d 100644
--- a/core/src/test/java/org/elasticsearch/action/ingest/SimulateProcessorResultTests.java
+++ b/core/src/test/java/org/elasticsearch/action/ingest/SimulateProcessorResultTests.java
@@ -27,7 +27,7 @@
 
 import java.io.IOException;
 
-import static org.elasticsearch.ingest.IngestDocumentTests.assertIngestDocument;
+import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
@@ -36,7 +36,7 @@
 public class SimulateProcessorResultTests extends ESTestCase {
 
     public void testSerialization() throws IOException {
-        String processorTag = randomAsciiOfLengthBetween(1, 10);
+        String processorTag = randomAlphaOfLengthBetween(1, 10);
         boolean isSuccessful = randomBoolean();
         boolean isIgnoredException = randomBoolean();
         SimulateProcessorResult simulateProcessorResult;
diff --git a/core/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java b/core/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java
index 5df0aa1de0cf0..c5f67e235061e 100644
--- a/core/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java
+++ b/core/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java
@@ -34,7 +34,7 @@
 import java.util.Map;
 
 import static org.elasticsearch.common.xcontent.ToXContent.EMPTY_PARAMS;
-import static org.elasticsearch.ingest.IngestDocumentTests.assertIngestDocument;
+import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
@@ -45,12 +45,12 @@ public void testEqualsAndHashcode() throws Exception {
         Map<String, Object> sourceAndMetadata = RandomDocumentPicks.randomSource(random());
         int numFields = randomIntBetween(1, IngestDocument.MetaData.values().length);
         for (int i = 0; i < numFields; i++) {
-            sourceAndMetadata.put(randomFrom(IngestDocument.MetaData.values()).getFieldName(), randomAsciiOfLengthBetween(5, 10));
+            sourceAndMetadata.put(randomFrom(IngestDocument.MetaData.values()).getFieldName(), randomAlphaOfLengthBetween(5, 10));
         }
         Map<String, Object> ingestMetadata = new HashMap<>();
         numFields = randomIntBetween(1, 5);
         for (int i = 0; i < numFields; i++) {
-            ingestMetadata.put(randomAsciiOfLengthBetween(5, 10), randomAsciiOfLengthBetween(5, 10));
+            ingestMetadata.put(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLengthBetween(5, 10));
         }
         WriteableIngestDocument ingestDocument = new WriteableIngestDocument(new IngestDocument(sourceAndMetadata, ingestMetadata));
 
@@ -65,7 +65,7 @@ public void testEqualsAndHashcode() throws Exception {
         if (randomBoolean()) {
             numFields = randomIntBetween(1, IngestDocument.MetaData.values().length);
             for (int i = 0; i < numFields; i++) {
-                otherSourceAndMetadata.put(randomFrom(IngestDocument.MetaData.values()).getFieldName(), randomAsciiOfLengthBetween(5, 10));
+                otherSourceAndMetadata.put(randomFrom(IngestDocument.MetaData.values()).getFieldName(), randomAlphaOfLengthBetween(5, 10));
             }
             changed = true;
         }
@@ -75,7 +75,7 @@ public void testEqualsAndHashcode() throws Exception {
             otherIngestMetadata = new HashMap<>();
             numFields = randomIntBetween(1, 5);
             for (int i = 0; i < numFields; i++) {
-                otherIngestMetadata.put(randomAsciiOfLengthBetween(5, 10), randomAsciiOfLengthBetween(5, 10));
+                otherIngestMetadata.put(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLengthBetween(5, 10));
             }
             changed = true;
         } else {
@@ -101,12 +101,12 @@ public void testSerialization() throws IOException {
         Map<String, Object> sourceAndMetadata = RandomDocumentPicks.randomSource(random());
         int numFields = randomIntBetween(1, IngestDocument.MetaData.values().length);
         for (int i = 0; i < numFields; i++) {
-            sourceAndMetadata.put(randomFrom(IngestDocument.MetaData.values()).getFieldName(), randomAsciiOfLengthBetween(5, 10));
+            sourceAndMetadata.put(randomFrom(IngestDocument.MetaData.values()).getFieldName(), randomAlphaOfLengthBetween(5, 10));
         }
         Map<String, Object> ingestMetadata = new HashMap<>();
         numFields = randomIntBetween(1, 5);
         for (int i = 0; i < numFields; i++) {
-            ingestMetadata.put(randomAsciiOfLengthBetween(5, 10), randomAsciiOfLengthBetween(5, 10));
+            ingestMetadata.put(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLengthBetween(5, 10));
         }
         WriteableIngestDocument writeableIngestDocument = new WriteableIngestDocument(new IngestDocument(sourceAndMetadata, ingestMetadata));
 
@@ -127,7 +127,7 @@ public void testToXContent() throws IOException {
         builder.startObject();
         writeableIngestDocument.toXContent(builder, EMPTY_PARAMS);
         builder.endObject();
-        Map<String, Object> toXContentMap = XContentHelper.convertToMap(builder.bytes(), false).v2();
+        Map<String, Object> toXContentMap = XContentHelper.convertToMap(builder.bytes(), false, builder.contentType()).v2();
 
         Map<String, Object> toXContentDoc = (Map<String, Object>) toXContentMap.get("doc");
         Map<String, Object> toXContentSource = (Map<String, Object>) toXContentDoc.get("_source");
diff --git a/core/src/test/java/org/elasticsearch/action/main/MainActionTests.java b/core/src/test/java/org/elasticsearch/action/main/MainActionTests.java
index a8c550e01c556..92e093350aa3b 100644
--- a/core/src/test/java/org/elasticsearch/action/main/MainActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/main/MainActionTests.java
@@ -30,7 +30,6 @@
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.io.stream.ByteBufferStreamInput;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ToXContent;
@@ -42,7 +41,6 @@
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
-import java.nio.ByteBuffer;
 import java.util.concurrent.atomic.AtomicReference;
 
 import static org.hamcrest.Matchers.equalTo;
@@ -56,11 +54,12 @@ public class MainActionTests extends ESTestCase {
     public void testMainResponseSerialization() throws IOException {
         final String nodeName = "node1";
         final ClusterName clusterName = new ClusterName("cluster1");
+        final String clusterUUID = randomAlphaOfLengthBetween(10, 20);
         final boolean available = randomBoolean();
         final Version version = Version.CURRENT;
         final Build build = Build.CURRENT;
 
-        final MainResponse mainResponse = new MainResponse(nodeName, version, clusterName, build, available);
+        final MainResponse mainResponse = new MainResponse(nodeName, version, clusterName, clusterUUID, build, available);
         BytesStreamOutput streamOutput = new BytesStreamOutput();
         mainResponse.writeTo(streamOutput);
         final MainResponse serialized = new MainResponse();
@@ -74,11 +73,21 @@ public void testMainResponseSerialization() throws IOException {
     }
 
     public void testMainResponseXContent() throws IOException {
-        final MainResponse mainResponse = new MainResponse("node1", Version.CURRENT, new ClusterName("cluster1"), Build.CURRENT, false);
-        final String expected = "{\"name\":\"node1\",\"cluster_name\":\"cluster1\",\"version\":{\"number\":\"" + Version.CURRENT.toString()
-            + "\",\"build_hash\":\"" + Build.CURRENT.shortHash() + "\",\"build_date\":\"" + Build.CURRENT.date() + "\"," +
-            "\"build_snapshot\":" + Build.CURRENT.isSnapshot() + ",\"lucene_version\":\"" + Version.CURRENT.luceneVersion.toString() +
-            "\"},\"tagline\":\"You Know, for Search\"}";
+        String clusterUUID = randomAlphaOfLengthBetween(10, 20);
+        final MainResponse mainResponse = new MainResponse("node1", Version.CURRENT, new ClusterName("cluster1"), clusterUUID,
+                Build.CURRENT, false);
+        final String expected = "{" +
+                "\"name\":\"node1\"," +
+                "\"cluster_name\":\"cluster1\"," +
+                "\"cluster_uuid\":\"" + clusterUUID + "\"," +
+                "\"version\":{" +
+                "\"number\":\"" + Version.CURRENT.toString() + "\"," +
+                "\"build_hash\":\"" + Build.CURRENT.shortHash() + "\"," +
+                "\"build_date\":\"" + Build.CURRENT.date() + "\"," +
+                "\"build_snapshot\":" + Build.CURRENT.isSnapshot() +
+                ",\"lucene_version\":\"" + Version.CURRENT.luceneVersion.toString() +
+                "\"}," +
+                "\"tagline\":\"You Know, for Search\"}";
 
         XContentBuilder builder = XContentFactory.jsonBuilder();
         mainResponse.toXContent(builder, ToXContent.EMPTY_PARAMS);
@@ -99,20 +108,22 @@ public void testMainActionClusterAvailable() {
             } else {
                 blocks = ClusterBlocks.builder()
                     .addGlobalBlock(new ClusterBlock(randomIntBetween(1, 16), "test global block 400", randomBoolean(), randomBoolean(),
-                        RestStatus.BAD_REQUEST, ClusterBlockLevel.ALL))
+                        false, RestStatus.BAD_REQUEST, ClusterBlockLevel.ALL))
                     .build();
             }
         } else {
             blocks = ClusterBlocks.builder()
                 .addGlobalBlock(new ClusterBlock(randomIntBetween(1, 16), "test global block 503", randomBoolean(), randomBoolean(),
-                    RestStatus.SERVICE_UNAVAILABLE, ClusterBlockLevel.ALL))
+                    false, RestStatus.SERVICE_UNAVAILABLE, ClusterBlockLevel.ALL))
                 .build();
         }
         ClusterState state = ClusterState.builder(clusterName).blocks(blocks).build();
         when(clusterService.state()).thenReturn(state);
 
-        TransportMainAction action = new TransportMainAction(settings, mock(ThreadPool.class), mock(TransportService.class),
-            mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), clusterService);
+        TransportService transportService = new TransportService(Settings.EMPTY, null, null, TransportService.NOOP_TRANSPORT_INTERCEPTOR,
+            x -> null, null);
+        TransportMainAction action = new TransportMainAction(settings, mock(ThreadPool.class), transportService, mock(ActionFilters.class),
+                mock(IndexNameExpressionResolver.class), clusterService);
         AtomicReference<MainResponse> responseRef = new AtomicReference<>();
         action.doExecute(new MainRequest(), new ActionListener<MainResponse>() {
             @Override
diff --git a/core/src/test/java/org/elasticsearch/action/main/MainResponseTests.java b/core/src/test/java/org/elasticsearch/action/main/MainResponseTests.java
new file mode 100644
index 0000000000000..c68fd187beac7
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/main/MainResponseTests.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.main;
+
+import org.elasticsearch.Build;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.VersionUtils;
+
+import java.io.IOException;
+import java.util.Date;
+
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
+import static org.elasticsearch.test.XContentTestUtils.insertRandomFields;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
+
+public class MainResponseTests extends ESTestCase {
+
+    public static MainResponse createTestItem() {
+        String clusterUuid = randomAlphaOfLength(10);
+        ClusterName clusterName = new ClusterName(randomAlphaOfLength(10));
+        String nodeName = randomAlphaOfLength(10);
+        Build build = new Build(randomAlphaOfLength(8), new Date(randomNonNegativeLong()).toString(), randomBoolean());
+        Version version = VersionUtils.randomVersion(random());
+        boolean available = randomBoolean();
+        return new MainResponse(nodeName, version, clusterName, clusterUuid , build, available);
+    }
+
+    public void testFromXContent() throws IOException {
+        MainResponse mainResponse = createTestItem();
+        XContentType xContentType = randomFrom(XContentType.values());
+        boolean humanReadable = randomBoolean();
+        BytesReference originalBytes = toShuffledXContent(mainResponse, xContentType, ToXContent.EMPTY_PARAMS, humanReadable);
+        // we add a few random fields to check that parser is lenient on new fields
+        BytesReference withRandomFields = insertRandomFields(xContentType, originalBytes, null, random());
+        MainResponse parsed;
+        try (XContentParser parser = createParser(xContentType.xContent(), withRandomFields)) {
+            parsed = MainResponse.fromXContent(parser);
+            assertNull(parser.nextToken());
+        }
+        assertEquals(mainResponse.getClusterUuid(), parsed.getClusterUuid());
+        assertEquals(mainResponse.getClusterName(), parsed.getClusterName());
+        assertEquals(mainResponse.getNodeName(), parsed.getNodeName());
+        assertEquals(mainResponse.getBuild(), parsed.getBuild());
+        assertEquals(mainResponse.getVersion(), parsed.getVersion());
+        // we cannot recreate the "available" flag from xContent, but should be "true" if request came through
+        assertEquals(true, parsed.isAvailable());
+        assertToXContentEquivalent(originalBytes, toXContent(parsed, xContentType, humanReadable), xContentType);
+    }
+
+    public void testToXContent() throws IOException {
+        Build build = new Build("buildHash", "2016-11-15".toString(), true);
+        Version version = Version.V_2_4_4;
+        MainResponse response = new MainResponse("nodeName", version, new ClusterName("clusterName"), "clusterUuid", build, true);
+        XContentBuilder builder = XContentFactory.jsonBuilder();
+        response.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        assertEquals("{"
+                + "\"name\":\"nodeName\","
+                + "\"cluster_name\":\"clusterName\","
+                + "\"cluster_uuid\":\"clusterUuid\","
+                + "\"version\":{"
+                    + "\"number\":\"2.4.4\","
+                    + "\"build_hash\":\"buildHash\","
+                    + "\"build_date\":\"2016-11-15\","
+                    + "\"build_snapshot\":true,"
+                    + "\"lucene_version\":\"5.5.2\"},"
+                + "\"tagline\":\"You Know, for Search\""
+          + "}", builder.string());
+    }
+
+    public void testEqualsAndHashcode() {
+        MainResponse original = createTestItem();
+        checkEqualsAndHashCode(original, MainResponseTests::copy, MainResponseTests::mutate);
+    }
+
+    private static MainResponse copy(MainResponse o) {
+        return new MainResponse(o.getNodeName(), o.getVersion(), o.getClusterName(), o.getClusterUuid(), o.getBuild(), o.isAvailable());
+    }
+
+    private static MainResponse mutate(MainResponse o) {
+        String clusterUuid = o.getClusterUuid();
+        boolean available = o.isAvailable();
+        Build build = o.getBuild();
+        Version version = o.getVersion();
+        String nodeName = o.getNodeName();
+        ClusterName clusterName = o.getClusterName();
+        switch (randomIntBetween(0, 5)) {
+        case 0:
+            clusterUuid = clusterUuid + randomAlphaOfLength(5);
+            break;
+        case 1:
+            nodeName = nodeName + randomAlphaOfLength(5);
+            break;
+        case 2:
+            available = !available;
+            break;
+        case 3:
+            // toggle the snapshot flag of the original Build parameter
+            build = new Build(build.shortHash(), build.date(), !build.isSnapshot());
+            break;
+        case 4:
+            version = randomValueOtherThan(version, () -> VersionUtils.randomVersion(random()));
+            break;
+        case 5:
+            clusterName = new ClusterName(clusterName + randomAlphaOfLength(5));
+            break;
+        }
+        return new MainResponse(nodeName, version, clusterName, clusterUuid, build, available);
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java b/core/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java
new file mode 100644
index 0000000000000..8445fb08fabd1
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.search;
+
+import org.elasticsearch.action.OriginalIndices;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.cluster.routing.GroupShardsIterator;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.search.SearchPhaseResult;
+import org.elasticsearch.search.internal.AliasFilter;
+import org.elasticsearch.search.internal.ShardSearchTransportRequest;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Collections;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+
+public class AbstractSearchAsyncActionTests extends ESTestCase {
+
+    private AbstractSearchAsyncAction<SearchPhaseResult> createAction(
+            final boolean controlled,
+            final AtomicLong expected) {
+
+        final Runnable runnable;
+        final TransportSearchAction.SearchTimeProvider timeProvider;
+        if (controlled) {
+            runnable = () -> expected.set(randomNonNegativeLong());
+            timeProvider = new TransportSearchAction.SearchTimeProvider(0, 0, expected::get);
+        } else {
+            runnable = () -> {
+                long elapsed = spinForAtLeastNMilliseconds(randomIntBetween(1, 10));
+                expected.set(elapsed);
+            };
+            timeProvider = new TransportSearchAction.SearchTimeProvider(
+                    0,
+                    System.nanoTime(),
+                    System::nanoTime);
+        }
+
+        return new AbstractSearchAsyncAction<SearchPhaseResult>("test", null, null, null,
+                Collections.singletonMap("foo", new AliasFilter(new MatchAllQueryBuilder())), Collections.singletonMap("foo", 2.0f), null,
+                new SearchRequest(), null, new GroupShardsIterator<>(Collections.singletonList(
+                new SearchShardIterator(null, null, Collections.emptyList(), null))), timeProvider, 0, null,
+            new InitialSearchPhase.ArraySearchPhaseResults<>(10)) {
+            @Override
+            protected SearchPhase getNextPhase(final SearchPhaseResults<SearchPhaseResult> results, final SearchPhaseContext context) {
+                return null;
+            }
+
+            @Override
+            protected void executePhaseOnShard(final SearchShardIterator shardIt, final ShardRouting shard,
+                                               final SearchActionListener<SearchPhaseResult> listener) {
+            }
+
+            @Override
+            long buildTookInMillis() {
+                runnable.run();
+                return super.buildTookInMillis();
+            }
+        };
+    }
+
+    public void testTookWithControlledClock() {
+        runTestTook(true);
+    }
+
+    public void testTookWithRealClock() {
+        runTestTook(false);
+    }
+
+    private void runTestTook(final boolean controlled) {
+        final AtomicLong expected = new AtomicLong();
+        AbstractSearchAsyncAction<SearchPhaseResult> action = createAction(controlled, expected);
+        final long actual = action.buildTookInMillis();
+        if (controlled) {
+            // with a controlled clock, we can assert the exact took time
+            assertThat(actual, equalTo(TimeUnit.NANOSECONDS.toMillis(expected.get())));
+        } else {
+            // with a real clock, the best we can say is that it took as long as we spun for
+            assertThat(actual, greaterThanOrEqualTo(TimeUnit.NANOSECONDS.toMillis(expected.get())));
+        }
+    }
+
+    public void testBuildShardSearchTransportRequest() {
+        final AtomicLong expected = new AtomicLong();
+        AbstractSearchAsyncAction<SearchPhaseResult> action = createAction(false, expected);
+        SearchShardIterator iterator = new SearchShardIterator("test-cluster", new ShardId(new Index("name", "foo"), 1),
+            Collections.emptyList(), new OriginalIndices(new String[] {"name", "name1"}, IndicesOptions.strictExpand()));
+        ShardSearchTransportRequest shardSearchTransportRequest = action.buildShardSearchRequest(iterator);
+        assertEquals(IndicesOptions.strictExpand(), shardSearchTransportRequest.indicesOptions());
+        assertArrayEquals(new String[] {"name", "name1"}, shardSearchTransportRequest.indices());
+        assertEquals(new MatchAllQueryBuilder(), shardSearchTransportRequest.filteringAliases());
+        assertEquals(2.0f, shardSearchTransportRequest.indexBoost(), 0.0f);
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java b/core/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java
new file mode 100644
index 0000000000000..41f11ee39d568
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.search;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.OriginalIndices;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.GroupShardsIterator;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.LocalTransportAddress;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.search.internal.AliasFilter;
+import org.elasticsearch.search.internal.ShardSearchTransportRequest;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.VersionUtils;
+import org.elasticsearch.transport.Transport;
+import org.elasticsearch.transport.local.LocalTransport;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicReference;
+
+public class CanMatchPreFilterSearchPhaseTests extends ESTestCase {
+
+
+    public void testFilterShards() throws InterruptedException {
+
+        final TransportSearchAction.SearchTimeProvider timeProvider = new TransportSearchAction.SearchTimeProvider(0, System.nanoTime(),
+            System::nanoTime);
+
+        Map<String, Transport.Connection> lookup = new ConcurrentHashMap<>();
+        DiscoveryNode primaryNode = new DiscoveryNode("node_1", LocalTransportAddress.buildUnique(), Version.CURRENT);
+        DiscoveryNode replicaNode = new DiscoveryNode("node_2", LocalTransportAddress.buildUnique(), Version.CURRENT);
+        lookup.put("node1", new SearchAsyncActionTests.MockConnection(primaryNode));
+        lookup.put("node2", new SearchAsyncActionTests.MockConnection(replicaNode));
+        final boolean shard1 = randomBoolean();
+        final boolean shard2 = randomBoolean();
+
+        SearchTransportService searchTransportService = new SearchTransportService(
+            Settings.builder().put("search.remote.connect", false).build(), null) {
+
+            @Override
+            public void sendCanMatch(Transport.Connection connection, ShardSearchTransportRequest request, SearchTask task,
+                                     ActionListener<CanMatchResponse> listener) {
+                new Thread(() -> listener.onResponse(new CanMatchResponse(request.shardId().id() == 0 ? shard1 :
+                    shard2))).start();
+            }
+        };
+
+        AtomicReference<GroupShardsIterator<SearchShardIterator>> result = new AtomicReference<>();
+        CountDownLatch latch = new CountDownLatch(1);
+        GroupShardsIterator<SearchShardIterator> shardsIter = SearchAsyncActionTests.getShardsIter("idx",
+            new OriginalIndices(new String[]{"idx"}, IndicesOptions.strictExpandOpenAndForbidClosed()),
+            2, randomBoolean(), primaryNode, replicaNode);
+        CanMatchPreFilterSearchPhase canMatchPhase = new CanMatchPreFilterSearchPhase(logger,
+            searchTransportService,
+            (clusterAlias, node) -> lookup.get(node),
+            Collections.singletonMap("_na_", new AliasFilter(null, Strings.EMPTY_ARRAY)),
+            Collections.emptyMap(), EsExecutors.newDirectExecutorService(),
+            new SearchRequest(), null, shardsIter, timeProvider, 0, null,
+            (iter) -> new SearchPhase("test") {
+                    @Override
+                    public void run() throws IOException {
+                        result.set(iter);
+                        latch.countDown();
+                    }});
+
+        canMatchPhase.start();
+        latch.await();
+
+        if (shard1 && shard2) {
+            for (SearchShardIterator i : result.get()) {
+                assertFalse(i.skip());
+            }
+        } else if (shard1 == false &&  shard2 == false) {
+            assertFalse(result.get().get(0).skip());
+            assertTrue(result.get().get(1).skip());
+        } else {
+            assertEquals(0, result.get().get(0).shardId().id());
+            assertEquals(1, result.get().get(1).shardId().id());
+            assertEquals(shard1, !result.get().get(0).skip());
+            assertEquals(shard2, !result.get().get(1).skip());
+        }
+    }
+
+    public void testOldNodesTriggerException() {
+        SearchTransportService searchTransportService = new SearchTransportService(
+            Settings.builder().put("search.remote.connect", false).build(), null);
+        DiscoveryNode node = new DiscoveryNode("node_1", LocalTransportAddress.buildUnique(),
+            VersionUtils.getPreviousVersion(Version.V_5_6_0));
+        SearchAsyncActionTests.MockConnection mockConnection = new SearchAsyncActionTests.MockConnection(node);
+        IllegalArgumentException illegalArgumentException = expectThrows(IllegalArgumentException.class,
+            () -> searchTransportService.sendCanMatch(mockConnection, null, null, null));
+        assertEquals("can_match is not supported on pre 5.6.0 nodes", illegalArgumentException.getMessage());
+    }
+
+    public void testFilterWithFailure() throws InterruptedException {
+        final TransportSearchAction.SearchTimeProvider timeProvider = new TransportSearchAction.SearchTimeProvider(0, System.nanoTime(),
+            System::nanoTime);
+        Map<String, Transport.Connection> lookup = new ConcurrentHashMap<>();
+        DiscoveryNode primaryNode = new DiscoveryNode("node_1", LocalTransportAddress.buildUnique(), Version.CURRENT);
+        DiscoveryNode replicaNode = new DiscoveryNode("node_2", LocalTransportAddress.buildUnique(), Version.CURRENT);
+        lookup.put("node1", new SearchAsyncActionTests.MockConnection(primaryNode));
+        lookup.put("node2", new SearchAsyncActionTests.MockConnection(replicaNode));
+        final boolean shard1 = randomBoolean();
+        SearchTransportService searchTransportService = new SearchTransportService(
+            Settings.builder().put("search.remote.connect", false).build(), null) {
+
+            @Override
+            public void sendCanMatch(Transport.Connection connection, ShardSearchTransportRequest request, SearchTask task,
+                                     ActionListener<CanMatchResponse> listener) {
+                boolean throwException = request.shardId().id() != 0;
+                if (throwException && randomBoolean()) {
+                    throw new IllegalArgumentException("boom");
+                } else {
+                    new Thread(() -> {
+                        if (throwException == false) {
+                            listener.onResponse(new CanMatchResponse(shard1));
+                        } else {
+                            listener.onFailure(new NullPointerException());
+                        }
+                    }).start();
+                }
+            }
+        };
+
+        AtomicReference<GroupShardsIterator<SearchShardIterator>> result = new AtomicReference<>();
+        CountDownLatch latch = new CountDownLatch(1);
+        GroupShardsIterator<SearchShardIterator> shardsIter = SearchAsyncActionTests.getShardsIter("idx",
+            new OriginalIndices(new String[]{"idx"}, IndicesOptions.strictExpandOpenAndForbidClosed()),
+            2, randomBoolean(), primaryNode, replicaNode);
+        CanMatchPreFilterSearchPhase canMatchPhase = new CanMatchPreFilterSearchPhase(logger,
+            searchTransportService,
+            (clusterAlias, node) -> lookup.get(node),
+            Collections.singletonMap("_na_", new AliasFilter(null, Strings.EMPTY_ARRAY)),
+            Collections.emptyMap(), EsExecutors.newDirectExecutorService(),
+            new SearchRequest(), null, shardsIter, timeProvider, 0, null,
+            (iter) -> new SearchPhase("test") {
+                @Override
+                public void run() throws IOException {
+                    result.set(iter);
+                    latch.countDown();
+                }});
+
+        canMatchPhase.start();
+        latch.await();
+
+        assertEquals(0, result.get().get(0).shardId().id());
+        assertEquals(1, result.get().get(1).shardId().id());
+        assertEquals(shard1, !result.get().get(0).skip());
+        assertFalse(result.get().get(1).skip()); // never skip the failure
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/search/ClearScrollControllerTests.java b/core/src/test/java/org/elasticsearch/action/search/ClearScrollControllerTests.java
new file mode 100644
index 0000000000000..2ba4ca0f6203a
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/search/ClearScrollControllerTests.java
@@ -0,0 +1,257 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.search;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.LocalTransportAddress;
+import org.elasticsearch.common.util.concurrent.AtomicArray;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.search.SearchPhaseResult;
+import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.transport.NodeNotConnectedException;
+import org.elasticsearch.transport.Transport;
+import org.elasticsearch.transport.TransportResponse;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class ClearScrollControllerTests extends ESTestCase {
+
+    public void testClearAll() throws IOException, InterruptedException {
+        DiscoveryNode node1 = new DiscoveryNode("node_1", LocalTransportAddress.buildUnique(), Version.CURRENT);
+        DiscoveryNode node2 = new DiscoveryNode("node_2", LocalTransportAddress.buildUnique(), Version.CURRENT);
+        DiscoveryNode node3 = new DiscoveryNode("node_3", LocalTransportAddress.buildUnique(), Version.CURRENT);
+        DiscoveryNodes nodes = DiscoveryNodes.builder().add(node1).add(node2).add(node3).build();
+        CountDownLatch latch = new CountDownLatch(1);
+        ActionListener<ClearScrollResponse> listener = new ActionListener<ClearScrollResponse>() {
+            @Override
+            public void onResponse(ClearScrollResponse clearScrollResponse) {
+                try {
+                    assertEquals(3, clearScrollResponse.getNumFreed());
+                    assertTrue(clearScrollResponse.isSucceeded());
+                } finally {
+                    latch.countDown();
+                }
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                try {
+                    throw new AssertionError(e);
+                } finally {
+                    latch.countDown();
+                }
+            }
+        };
+        List<DiscoveryNode> nodesInvoked = new CopyOnWriteArrayList<>();
+        SearchTransportService searchTransportService = new SearchTransportService(Settings.EMPTY, null) {
+            @Override
+            public void sendClearAllScrollContexts(Transport.Connection connection, ActionListener<TransportResponse> listener) {
+                nodesInvoked.add(connection.getNode());
+                Thread t = new Thread(() -> listener.onResponse(TransportResponse.Empty.INSTANCE)); // response is unused
+                t.start();
+            }
+
+            @Override
+            Transport.Connection getConnection(String clusterAlias, DiscoveryNode node) {
+                return new SearchAsyncActionTests.MockConnection(node);
+            }
+        };
+        ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
+        clearScrollRequest.scrollIds(Arrays.asList("_all"));
+        ClearScrollController controller = new ClearScrollController(clearScrollRequest, listener,
+            nodes, logger, searchTransportService);
+        controller.run();
+        latch.await();
+        assertEquals(3, nodesInvoked.size());
+        Collections.sort(nodesInvoked, Comparator.comparing(DiscoveryNode::getId));
+        assertEquals(nodesInvoked, Arrays.asList(node1, node2, node3));
+    }
+
+
+    public void testClearScrollIds() throws IOException, InterruptedException {
+        DiscoveryNode node1 = new DiscoveryNode("node_1", LocalTransportAddress.buildUnique(), Version.CURRENT);
+        DiscoveryNode node2 = new DiscoveryNode("node_2", LocalTransportAddress.buildUnique(), Version.CURRENT);
+        DiscoveryNode node3 = new DiscoveryNode("node_3", LocalTransportAddress.buildUnique(), Version.CURRENT);
+        AtomicArray<SearchPhaseResult> array = new AtomicArray<>(3);
+        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult1 = new SearchAsyncActionTests.TestSearchPhaseResult(1, node1);
+        testSearchPhaseResult1.setSearchShardTarget(new SearchShardTarget("node_1", new ShardId("idx", "uuid1", 2), null, null));
+        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult2 = new SearchAsyncActionTests.TestSearchPhaseResult(12, node2);
+        testSearchPhaseResult2.setSearchShardTarget(new SearchShardTarget("node_2", new ShardId("idy", "uuid2", 42), null, null));
+        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult3 = new SearchAsyncActionTests.TestSearchPhaseResult(42, node3);
+        testSearchPhaseResult3.setSearchShardTarget(new SearchShardTarget("node_3", new ShardId("idy", "uuid2", 43), null, null));
+        array.setOnce(0, testSearchPhaseResult1);
+        array.setOnce(1, testSearchPhaseResult2);
+        array.setOnce(2, testSearchPhaseResult3);
+        AtomicInteger numFreed = new AtomicInteger(0);
+        String scrollId = TransportSearchHelper.buildScrollId(array);
+        DiscoveryNodes nodes = DiscoveryNodes.builder().add(node1).add(node2).add(node3).build();
+        CountDownLatch latch = new CountDownLatch(1);
+        ActionListener<ClearScrollResponse> listener = new ActionListener<ClearScrollResponse>() {
+            @Override
+            public void onResponse(ClearScrollResponse clearScrollResponse) {
+                try {
+                    assertEquals(numFreed.get(), clearScrollResponse.getNumFreed());
+                    assertTrue(clearScrollResponse.isSucceeded());
+                } finally {
+                    latch.countDown();
+                }
+
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                try {
+                    throw new AssertionError(e);
+                } finally {
+                    latch.countDown();
+                }
+            }
+        };
+        List<DiscoveryNode> nodesInvoked = new CopyOnWriteArrayList<>();
+        SearchTransportService searchTransportService = new SearchTransportService(Settings.EMPTY, null) {
+
+            @Override
+            public void sendFreeContext(Transport.Connection connection, long contextId,
+                                        ActionListener<SearchFreeContextResponse> listener) {
+                nodesInvoked.add(connection.getNode());
+                boolean freed = randomBoolean();
+                if (freed) {
+                    numFreed.incrementAndGet();
+                }
+                Thread t = new Thread(() -> listener.onResponse(new SearchFreeContextResponse(freed)));
+                t.start();
+            }
+
+            @Override
+            Transport.Connection getConnection(String clusterAlias, DiscoveryNode node) {
+                return new SearchAsyncActionTests.MockConnection(node);
+            }
+        };
+        ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
+        clearScrollRequest.scrollIds(Arrays.asList(scrollId));
+        ClearScrollController controller = new ClearScrollController(clearScrollRequest, listener,
+            nodes, logger, searchTransportService);
+        controller.run();
+        latch.await();
+        assertEquals(3, nodesInvoked.size());
+        Collections.sort(nodesInvoked, Comparator.comparing(DiscoveryNode::getId));
+        assertEquals(nodesInvoked, Arrays.asList(node1, node2, node3));
+    }
+
+    public void testClearScrollIdsWithFailure() throws IOException, InterruptedException {
+        DiscoveryNode node1 = new DiscoveryNode("node_1", LocalTransportAddress.buildUnique(), Version.CURRENT);
+        DiscoveryNode node2 = new DiscoveryNode("node_2", LocalTransportAddress.buildUnique(), Version.CURRENT);
+        DiscoveryNode node3 = new DiscoveryNode("node_3", LocalTransportAddress.buildUnique(), Version.CURRENT);
+        AtomicArray<SearchPhaseResult> array = new AtomicArray<>(3);
+        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult1 = new SearchAsyncActionTests.TestSearchPhaseResult(1, node1);
+        testSearchPhaseResult1.setSearchShardTarget(new SearchShardTarget("node_1", new ShardId("idx", "uuid1", 2), null, null));
+        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult2 = new SearchAsyncActionTests.TestSearchPhaseResult(12, node2);
+        testSearchPhaseResult2.setSearchShardTarget(new SearchShardTarget("node_2", new ShardId("idy", "uuid2", 42), null, null));
+        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult3 = new SearchAsyncActionTests.TestSearchPhaseResult(42, node3);
+        testSearchPhaseResult3.setSearchShardTarget(new SearchShardTarget("node_3", new ShardId("idy", "uuid2", 43), null, null));
+        array.setOnce(0, testSearchPhaseResult1);
+        array.setOnce(1, testSearchPhaseResult2);
+        array.setOnce(2, testSearchPhaseResult3);
+        AtomicInteger numFreed = new AtomicInteger(0);
+        AtomicInteger numFailures = new AtomicInteger(0);
+        AtomicInteger numConnectionFailures = new AtomicInteger(0);
+        String scrollId = TransportSearchHelper.buildScrollId(array);
+        DiscoveryNodes nodes = DiscoveryNodes.builder().add(node1).add(node2).add(node3).build();
+        CountDownLatch latch = new CountDownLatch(1);
+
+        ActionListener<ClearScrollResponse> listener = new ActionListener<ClearScrollResponse>() {
+            @Override
+            public void onResponse(ClearScrollResponse clearScrollResponse) {
+                try {
+                    assertEquals(numFreed.get(), clearScrollResponse.getNumFreed());
+                    if (numFailures.get() > 0) {
+                        assertFalse(clearScrollResponse.isSucceeded());
+                    } else {
+                        assertTrue(clearScrollResponse.isSucceeded());
+                    }
+
+                } finally {
+                    latch.countDown();
+                }
+
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                try {
+                    throw new AssertionError(e);
+                } finally {
+                    latch.countDown();
+                }
+            }
+        };
+        List<DiscoveryNode> nodesInvoked = new CopyOnWriteArrayList<>();
+        SearchTransportService searchTransportService = new SearchTransportService(Settings.EMPTY, null) {
+
+            @Override
+            public void sendFreeContext(Transport.Connection connection, long contextId,
+                                        ActionListener<SearchFreeContextResponse> listener) {
+                nodesInvoked.add(connection.getNode());
+                boolean freed = randomBoolean();
+                boolean fail = randomBoolean();
+                Thread t = new Thread(() -> {
+                    if (fail) {
+                        numFailures.incrementAndGet();
+                        listener.onFailure(new IllegalArgumentException("boom"));
+                    } else {
+                        if (freed) {
+                            numFreed.incrementAndGet();
+                        }
+                        listener.onResponse(new SearchFreeContextResponse(freed));
+                    }
+                });
+                t.start();
+            }
+
+            @Override
+            Transport.Connection getConnection(String clusterAlias, DiscoveryNode node) {
+                if (randomBoolean()) {
+                    numFailures.incrementAndGet();
+                    numConnectionFailures.incrementAndGet();
+                    throw new NodeNotConnectedException(node, "boom");
+                }
+                return new SearchAsyncActionTests.MockConnection(node);
+            }
+        };
+        ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
+        clearScrollRequest.scrollIds(Arrays.asList(scrollId));
+        ClearScrollController controller = new ClearScrollController(clearScrollRequest, listener,
+            nodes, logger, searchTransportService);
+        controller.run();
+        latch.await();
+        assertEquals(3 - numConnectionFailures.get(), nodesInvoked.size());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/search/ClearScrollRequestTests.java b/core/src/test/java/org/elasticsearch/action/search/ClearScrollRequestTests.java
new file mode 100644
index 0000000000000..6414e510069a0
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/search/ClearScrollRequestTests.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.search;
+
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.startsWith;
+
+public class ClearScrollRequestTests extends ESTestCase {
+
+    public void testFromXContent() throws Exception {
+        ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
+        if (randomBoolean()) {
+            //test that existing values get overridden
+            clearScrollRequest = createClearScrollRequest();
+        }
+        try (XContentParser parser = createParser(XContentFactory.jsonBuilder()
+                .startObject()
+                .array("scroll_id", "value_1", "value_2")
+                .endObject())) {
+            clearScrollRequest.fromXContent(parser);
+        }
+        assertThat(clearScrollRequest.scrollIds(), contains("value_1", "value_2"));
+    }
+
+    public void testFromXContentWithoutArray() throws Exception {
+        ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
+        if (randomBoolean()) {
+            //test that existing values get overridden
+            clearScrollRequest = createClearScrollRequest();
+        }
+        try (XContentParser parser = createParser(XContentFactory.jsonBuilder()
+                .startObject()
+                .field("scroll_id", "value_1")
+                .endObject())) {
+            clearScrollRequest.fromXContent(parser);
+        }
+        assertThat(clearScrollRequest.scrollIds(), contains("value_1"));
+    }
+
+    public void testFromXContentWithUnknownParamThrowsException() throws Exception {
+        XContentParser invalidContent = createParser(XContentFactory.jsonBuilder()
+                .startObject()
+                .array("scroll_id", "value_1", "value_2")
+                .field("unknown", "keyword")
+                .endObject());
+        ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
+
+        Exception e = expectThrows(IllegalArgumentException.class, () -> clearScrollRequest.fromXContent(invalidContent));
+        assertThat(e.getMessage(), startsWith("Unknown parameter [unknown]"));
+    }
+
+    public void testToXContent() throws IOException {
+        ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
+        clearScrollRequest.addScrollId("SCROLL_ID");
+        try (XContentBuilder builder = JsonXContent.contentBuilder()) {
+            clearScrollRequest.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            assertEquals("{\"scroll_id\":[\"SCROLL_ID\"]}", builder.string());
+        }
+    }
+
+    public void testFromAndToXContent() throws IOException {
+        XContentType xContentType = randomFrom(XContentType.values());
+        ClearScrollRequest originalRequest = createClearScrollRequest();
+        BytesReference originalBytes = toShuffledXContent(originalRequest, xContentType, ToXContent.EMPTY_PARAMS, randomBoolean());
+        ClearScrollRequest parsedRequest = new ClearScrollRequest();
+        try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) {
+            parsedRequest.fromXContent(parser);
+        }
+        assertEquals(originalRequest.scrollIds(), parsedRequest.scrollIds());
+        BytesReference parsedBytes = XContentHelper.toXContent(parsedRequest, xContentType, randomBoolean());
+        assertToXContentEquivalent(originalBytes, parsedBytes, xContentType);
+    }
+
+    public static ClearScrollRequest createClearScrollRequest() {
+        ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
+        int numScrolls = randomIntBetween(1, 10);
+        for (int i = 0; i < numScrolls; i++) {
+            clearScrollRequest.addScrollId(randomAlphaOfLengthBetween(3, 10));
+        }
+        return clearScrollRequest;
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/search/CountedCollectorTests.java b/core/src/test/java/org/elasticsearch/action/search/CountedCollectorTests.java
new file mode 100644
index 0000000000000..951b5997508ca
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/search/CountedCollectorTests.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.search;
+
+import org.elasticsearch.common.util.concurrent.AtomicArray;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.search.SearchPhaseResult;
+import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.search.dfs.DfsSearchResult;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executor;
+
+public class CountedCollectorTests extends ESTestCase {
+    public void testCollect() throws InterruptedException {
+        AtomicArray<SearchPhaseResult> results = new AtomicArray<>(randomIntBetween(1, 100));
+        List<Integer> state = new ArrayList<>();
+        int numResultsExpected = randomIntBetween(1, results.length());
+        MockSearchPhaseContext context = new MockSearchPhaseContext(results.length());
+        CountDownLatch latch = new CountDownLatch(1);
+        boolean maybeFork = randomBoolean();
+        Executor executor = (runnable) -> {
+            if (randomBoolean() && maybeFork) {
+                new Thread(runnable).start();
+
+            } else {
+                runnable.run();
+            }
+        };
+        CountedCollector<SearchPhaseResult> collector = new CountedCollector<>(r -> results.set(r.getShardIndex(), r), numResultsExpected,
+            latch::countDown, context);
+        for (int i = 0; i < numResultsExpected; i++) {
+            int shardID = i;
+            switch (randomIntBetween(0, 2)) {
+                case 0:
+                    state.add(0);
+                    executor.execute(() -> collector.countDown());
+                    break;
+                case 1:
+                    state.add(1);
+                    executor.execute(() -> {
+                        DfsSearchResult dfsSearchResult = new DfsSearchResult(shardID, null);
+                        dfsSearchResult.setShardIndex(shardID);
+                        dfsSearchResult.setSearchShardTarget(new SearchShardTarget("foo",
+                            new Index("bar", "baz"), shardID, null));
+                        collector.onResult(dfsSearchResult);});
+                    break;
+                case 2:
+                    state.add(2);
+                    executor.execute(() -> collector.onFailure(shardID, new SearchShardTarget("foo", new Index("bar", "baz"),
+                        shardID, null), new RuntimeException("boom")));
+                    break;
+                default:
+                    fail("unknown state");
+            }
+        }
+        latch.await();
+        assertEquals(numResultsExpected, state.size());
+
+        for (int i = 0; i < numResultsExpected; i++) {
+            switch (state.get(i)) {
+                case 0:
+                    assertNull(results.get(i));
+                    break;
+                case 1:
+                    assertNotNull(results.get(i));
+                    assertEquals(i, results.get(i).getRequestId());
+                    break;
+                case 2:
+                    final int shardId = i;
+                    assertEquals(1, context.failures.stream().filter(f -> f.shardId() == shardId).count());
+                    break;
+                default:
+                    fail("unknown state");
+            }
+        }
+
+        for (int i = numResultsExpected; i < results.length(); i++) {
+            assertNull("index: " + i, results.get(i));
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java b/core/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java
new file mode 100644
index 0000000000000..185dae410d55e
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java
@@ -0,0 +1,206 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.search;
+
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.TermStatistics;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.store.MockDirectoryWrapper;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.concurrent.AtomicArray;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.SearchPhaseResult;
+import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.search.dfs.DfsSearchResult;
+import org.elasticsearch.search.query.QuerySearchRequest;
+import org.elasticsearch.search.query.QuerySearchResult;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.transport.Transport;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.concurrent.atomic.AtomicReference;
+
+public class DfsQueryPhaseTests extends ESTestCase {
+
+    private static DfsSearchResult newSearchResult(int shardIndex, long requestId, SearchShardTarget target) {
+        DfsSearchResult result = new DfsSearchResult(requestId, target);
+        result.setShardIndex(shardIndex);
+        return result;
+    }
+
+    public void testDfsWith2Shards() throws IOException {
+        AtomicArray<DfsSearchResult> results = new AtomicArray<>(2);
+        AtomicReference<AtomicArray<SearchPhaseResult>> responseRef = new AtomicReference<>();
+        results.set(0, newSearchResult(0, 1, new SearchShardTarget("node1", new Index("test", "na"), 0, null)));
+        results.set(1, newSearchResult(1, 2, new SearchShardTarget("node2", new Index("test", "na"), 0, null)));
+        results.get(0).termsStatistics(new Term[0], new TermStatistics[0]);
+        results.get(1).termsStatistics(new Term[0], new TermStatistics[0]);
+
+        SearchPhaseController controller = new SearchPhaseController(Settings.EMPTY, BigArrays.NON_RECYCLING_INSTANCE, null);
+        SearchTransportService searchTransportService = new SearchTransportService(
+            Settings.builder().put("search.remote.connect", false).build(), null) {
+
+            @Override
+            public void sendExecuteQuery(Transport.Connection connection, QuerySearchRequest request, SearchTask task,
+                                         SearchActionListener<QuerySearchResult> listener) {
+                if (request.id() == 1) {
+                    QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new Index("test", "na"), 0,
+                        null));
+                    queryResult.topDocs(new TopDocs(1, new ScoreDoc[] {new ScoreDoc(42, 1.0F)}, 2.0F), new DocValueFormat[0]);
+                    queryResult.size(2); // the size of the result set
+                    listener.onResponse(queryResult);
+                } else if (request.id() == 2) {
+                    QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node2", new Index("test", "na"), 0,
+                        null));
+                    queryResult.topDocs(new TopDocs(1, new ScoreDoc[] {new ScoreDoc(84, 2.0F)}, 2.0F), new DocValueFormat[0]);
+                    queryResult.size(2); // the size of the result set
+                    listener.onResponse(queryResult);
+                } else {
+                    fail("no such request ID: " + request.id());
+                }
+            }
+        };
+        MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(2);
+        mockSearchPhaseContext.searchTransport = searchTransportService;
+        DfsQueryPhase phase = new DfsQueryPhase(results, controller,
+            (response) -> new SearchPhase("test") {
+            @Override
+            public void run() throws IOException {
+                responseRef.set(response.results);
+            }
+        }, mockSearchPhaseContext);
+        assertEquals("dfs_query", phase.getName());
+        phase.run();
+        mockSearchPhaseContext.assertNoFailure();
+        assertNotNull(responseRef.get());
+        assertNotNull(responseRef.get().get(0));
+        assertNull(responseRef.get().get(0).fetchResult());
+        assertEquals(1, responseRef.get().get(0).queryResult().topDocs().totalHits);
+        assertEquals(42, responseRef.get().get(0).queryResult().topDocs().scoreDocs[0].doc);
+        assertNotNull(responseRef.get().get(1));
+        assertNull(responseRef.get().get(1).fetchResult());
+        assertEquals(1, responseRef.get().get(1).queryResult().topDocs().totalHits);
+        assertEquals(84, responseRef.get().get(1).queryResult().topDocs().scoreDocs[0].doc);
+        assertTrue(mockSearchPhaseContext.releasedSearchContexts.isEmpty());
+        assertEquals(2, mockSearchPhaseContext.numSuccess.get());
+    }
+
+    public void testDfsWith1ShardFailed() throws IOException {
+        AtomicArray<DfsSearchResult> results = new AtomicArray<>(2);
+        AtomicReference<AtomicArray<SearchPhaseResult>> responseRef = new AtomicReference<>();
+        results.set(0, newSearchResult(0, 1, new SearchShardTarget("node1", new Index("test", "na"), 0, null)));
+        results.set(1, newSearchResult(1, 2, new SearchShardTarget("node2", new Index("test", "na"), 0, null)));
+        results.get(0).termsStatistics(new Term[0], new TermStatistics[0]);
+        results.get(1).termsStatistics(new Term[0], new TermStatistics[0]);
+
+        SearchPhaseController controller = new SearchPhaseController(Settings.EMPTY, BigArrays.NON_RECYCLING_INSTANCE, null);
+        SearchTransportService searchTransportService = new SearchTransportService(
+            Settings.builder().put("search.remote.connect", false).build(), null) {
+
+            @Override
+            public void sendExecuteQuery(Transport.Connection connection, QuerySearchRequest request, SearchTask task,
+                                         SearchActionListener<QuerySearchResult> listener) {
+                if (request.id() == 1) {
+                    QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new Index("test", "na"), 0,
+                        null));
+                    queryResult.topDocs(new TopDocs(1, new ScoreDoc[] {new ScoreDoc(42, 1.0F)}, 2.0F), new DocValueFormat[0]);
+                    queryResult.size(2); // the size of the result set
+                    listener.onResponse(queryResult);
+                } else if (request.id() == 2) {
+                    listener.onFailure(new MockDirectoryWrapper.FakeIOException());
+                } else {
+                    fail("no such request ID: " + request.id());
+                }
+            }
+        };
+        MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(2);
+        mockSearchPhaseContext.searchTransport = searchTransportService;
+        DfsQueryPhase phase = new DfsQueryPhase(results, controller,
+            (response) -> new SearchPhase("test") {
+                @Override
+                public void run() throws IOException {
+                    responseRef.set(response.results);
+                }
+            }, mockSearchPhaseContext);
+        assertEquals("dfs_query", phase.getName());
+        phase.run();
+        mockSearchPhaseContext.assertNoFailure();
+        assertNotNull(responseRef.get());
+        assertNotNull(responseRef.get().get(0));
+        assertNull(responseRef.get().get(0).fetchResult());
+        assertEquals(1, responseRef.get().get(0).queryResult().topDocs().totalHits);
+        assertEquals(42, responseRef.get().get(0).queryResult().topDocs().scoreDocs[0].doc);
+        assertNull(responseRef.get().get(1));
+
+        assertEquals(1, mockSearchPhaseContext.numSuccess.get());
+        assertEquals(1, mockSearchPhaseContext.failures.size());
+        assertTrue(mockSearchPhaseContext.failures.get(0).getCause() instanceof MockDirectoryWrapper.FakeIOException);
+        assertEquals(1, mockSearchPhaseContext.releasedSearchContexts.size());
+        assertTrue(mockSearchPhaseContext.releasedSearchContexts.contains(2L));
+        assertNull(responseRef.get().get(1));
+    }
+
+
+    public void testFailPhaseOnException() throws IOException {
+        AtomicArray<DfsSearchResult> results = new AtomicArray<>(2);
+        AtomicReference<AtomicArray<SearchPhaseResult>> responseRef = new AtomicReference<>();
+        results.set(0, newSearchResult(0, 1, new SearchShardTarget("node1", new Index("test", "na"), 0, null)));
+        results.set(1, newSearchResult(1, 2, new SearchShardTarget("node2", new Index("test", "na"), 0, null)));
+        results.get(0).termsStatistics(new Term[0], new TermStatistics[0]);
+        results.get(1).termsStatistics(new Term[0], new TermStatistics[0]);
+
+        SearchPhaseController controller = new SearchPhaseController(Settings.EMPTY, BigArrays.NON_RECYCLING_INSTANCE, null);
+        SearchTransportService searchTransportService = new SearchTransportService(
+            Settings.builder().put("search.remote.connect", false).build(), null) {
+
+            @Override
+            public void sendExecuteQuery(Transport.Connection connection, QuerySearchRequest request, SearchTask task,
+                                         SearchActionListener<QuerySearchResult> listener) {
+                if (request.id() == 1) {
+                    QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new Index("test", "na"), 0,
+                        null));
+                    queryResult.topDocs(new TopDocs(1, new ScoreDoc[] {new ScoreDoc(42, 1.0F)}, 2.0F), new DocValueFormat[0]);
+                    queryResult.size(2); // the size of the result set
+                    listener.onResponse(queryResult);
+                } else if (request.id() == 2) {
+                   throw new UncheckedIOException(new MockDirectoryWrapper.FakeIOException());
+                } else {
+                    fail("no such request ID: " + request.id());
+                }
+            }
+        };
+        MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(2);
+        mockSearchPhaseContext.searchTransport = searchTransportService;
+        DfsQueryPhase phase = new DfsQueryPhase(results, controller,
+            (response) -> new SearchPhase("test") {
+                @Override
+                public void run() throws IOException {
+                    responseRef.set(response.results);
+                }
+            }, mockSearchPhaseContext);
+        assertEquals("dfs_query", phase.getName());
+        expectThrows(UncheckedIOException.class, () -> phase.run());
+        assertTrue(mockSearchPhaseContext.releasedSearchContexts.isEmpty()); // phase execution will clean up on the contexts
+    }
+
+ }
diff --git a/core/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java b/core/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java
new file mode 100644
index 0000000000000..fef4cff6a4e4b
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java
@@ -0,0 +1,245 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.search;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.text.Text;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.InnerHitBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchHitField;
+import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.collapse.CollapseBuilder;
+import org.elasticsearch.search.internal.InternalSearchResponse;
+import org.elasticsearch.test.ESTestCase;
+import org.hamcrest.Matchers;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+public class ExpandSearchPhaseTests extends ESTestCase {
+
+    public void testCollapseSingleHit() throws IOException {
+        final int iters = randomIntBetween(5, 10);
+        for (int i = 0; i < iters; i++) {
+            final int numInnerHits = randomIntBetween(1, 5);
+            List<SearchHits> collapsedHits = new ArrayList<>(numInnerHits);
+            for (int innerHitNum = 0; innerHitNum < numInnerHits; innerHitNum++) {
+                SearchHits hits = new SearchHits(new SearchHit[]{new SearchHit(innerHitNum, "ID", new Text("type"),
+                    Collections.emptyMap()), new SearchHit(innerHitNum + 1, "ID", new Text("type"),
+                    Collections.emptyMap())}, 2, 1.0F);
+                collapsedHits.add(hits);
+            }
+
+            AtomicBoolean executedMultiSearch = new AtomicBoolean(false);
+            QueryBuilder originalQuery = randomBoolean() ? null : QueryBuilders.termQuery("foo", "bar");
+
+            final MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(1);
+            String collapseValue = randomBoolean() ? null : "boom";
+
+            mockSearchPhaseContext.getRequest().source(new SearchSourceBuilder()
+                .collapse(new CollapseBuilder("someField")
+                    .setInnerHits(IntStream.range(0, numInnerHits).mapToObj(hitNum -> new InnerHitBuilder().setName("innerHit" + hitNum))
+                        .collect(Collectors.toList()))));
+            mockSearchPhaseContext.getRequest().source().query(originalQuery);
+            mockSearchPhaseContext.searchTransport = new SearchTransportService(
+                Settings.builder().put("search.remote.connect", false).build(), null) {
+
+                @Override
+                void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionListener<MultiSearchResponse> listener) {
+                    assertTrue(executedMultiSearch.compareAndSet(false, true));
+                    assertEquals(numInnerHits, request.requests().size());
+                    SearchRequest searchRequest = request.requests().get(0);
+                    assertTrue(searchRequest.source().query() instanceof BoolQueryBuilder);
+
+                    BoolQueryBuilder groupBuilder = (BoolQueryBuilder) searchRequest.source().query();
+                    if (collapseValue == null) {
+                        assertThat(groupBuilder.mustNot(), Matchers.contains(QueryBuilders.existsQuery("someField")));
+                    } else {
+                        assertThat(groupBuilder.filter(), Matchers.contains(QueryBuilders.matchQuery("someField", "boom")));
+                    }
+                    if (originalQuery != null) {
+                        assertThat(groupBuilder.must(), Matchers.contains(QueryBuilders.termQuery("foo", "bar")));
+                    }
+                    assertArrayEquals(mockSearchPhaseContext.getRequest().indices(), searchRequest.indices());
+                    assertArrayEquals(mockSearchPhaseContext.getRequest().types(), searchRequest.types());
+
+
+                    List<MultiSearchResponse.Item> mSearchResponses = new ArrayList<>(numInnerHits);
+                    for (int innerHitNum = 0; innerHitNum < numInnerHits; innerHitNum++) {
+                        InternalSearchResponse internalSearchResponse = new InternalSearchResponse(collapsedHits.get(innerHitNum),
+                            null, null, null, false, null, 1);
+                        SearchResponse response = mockSearchPhaseContext.buildSearchResponse(internalSearchResponse, null);
+                        mSearchResponses.add(new MultiSearchResponse.Item(response, null));
+                    }
+
+                    listener.onResponse(new MultiSearchResponse(mSearchResponses.toArray(new MultiSearchResponse.Item[0])));
+                }
+            };
+
+            SearchHits hits = new SearchHits(new SearchHit[]{new SearchHit(1, "ID", new Text("type"),
+                Collections.singletonMap("someField", new SearchHitField("someField", Collections.singletonList(collapseValue))))},
+                1, 1.0F);
+            InternalSearchResponse internalSearchResponse = new InternalSearchResponse(hits, null, null, null, false, null, 1);
+            AtomicReference<SearchResponse> reference = new AtomicReference<>();
+            ExpandSearchPhase phase = new ExpandSearchPhase(mockSearchPhaseContext, internalSearchResponse, (r) ->
+                new SearchPhase("test") {
+                    @Override
+                    public void run() throws IOException {
+                        reference.set(mockSearchPhaseContext.buildSearchResponse(r, null));
+                    }
+                }
+            );
+
+            phase.run();
+            mockSearchPhaseContext.assertNoFailure();
+            assertNotNull(reference.get());
+            SearchResponse theResponse = reference.get();
+            assertEquals(numInnerHits, theResponse.getHits().getHits()[0].getInnerHits().size());
+
+            for (int innerHitNum = 0; innerHitNum < numInnerHits; innerHitNum++) {
+                assertSame(theResponse.getHits().getHits()[0].getInnerHits().get("innerHit" + innerHitNum), collapsedHits.get(innerHitNum));
+            }
+
+            assertTrue(executedMultiSearch.get());
+            assertEquals(1, mockSearchPhaseContext.phasesExecuted.get());
+        }
+    }
+
+    public void testFailOneItemFailsEntirePhase() throws IOException {
+        AtomicBoolean executedMultiSearch = new AtomicBoolean(false);
+
+        SearchHits collapsedHits = new SearchHits(new SearchHit[]{new SearchHit(2, "ID", new Text("type"),
+            Collections.emptyMap()), new SearchHit(3, "ID", new Text("type"),
+            Collections.emptyMap())}, 1, 1.0F);
+        MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(1);
+        String collapseValue = randomBoolean() ? null : "boom";
+        mockSearchPhaseContext.getRequest().source(new SearchSourceBuilder()
+            .collapse(new CollapseBuilder("someField").setInnerHits(new InnerHitBuilder().setName("foobarbaz"))));
+        mockSearchPhaseContext.searchTransport = new SearchTransportService(
+            Settings.builder().put("search.remote.connect", false).build(), null) {
+
+            @Override
+            void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionListener<MultiSearchResponse> listener) {
+                assertTrue(executedMultiSearch.compareAndSet(false, true));
+                InternalSearchResponse internalSearchResponse = new InternalSearchResponse(collapsedHits,
+                    null, null, null, false, null, 1);
+                SearchResponse response = mockSearchPhaseContext.buildSearchResponse(internalSearchResponse, null);
+                listener.onResponse(new MultiSearchResponse(new MultiSearchResponse.Item[]{
+                    new MultiSearchResponse.Item(null, new RuntimeException("boom")),
+                    new MultiSearchResponse.Item(response, null)
+                }));
+            }
+        };
+
+        SearchHits hits = new SearchHits(new SearchHit[]{new SearchHit(1, "ID", new Text("type"),
+            Collections.singletonMap("someField", new SearchHitField("someField", Collections.singletonList(collapseValue)))),
+            new SearchHit(2, "ID2", new Text("type"),
+                Collections.singletonMap("someField", new SearchHitField("someField", Collections.singletonList(collapseValue))))}, 1,
+            1.0F);
+        InternalSearchResponse internalSearchResponse = new InternalSearchResponse(hits, null, null, null, false, null, 1);
+        AtomicReference<SearchResponse> reference = new AtomicReference<>();
+        ExpandSearchPhase phase = new ExpandSearchPhase(mockSearchPhaseContext, internalSearchResponse, r ->
+            new SearchPhase("test") {
+                @Override
+                public void run() throws IOException {
+                    reference.set(mockSearchPhaseContext.buildSearchResponse(r, null));
+                }
+            }
+        );
+        phase.run();
+        assertThat(mockSearchPhaseContext.phaseFailure.get(), Matchers.instanceOf(RuntimeException.class));
+        assertEquals("boom", mockSearchPhaseContext.phaseFailure.get().getMessage());
+        assertNotNull(mockSearchPhaseContext.phaseFailure.get());
+        assertNull(reference.get());
+        assertEquals(0, mockSearchPhaseContext.phasesExecuted.get());
+    }
+
+    public void testSkipPhase() throws IOException {
+        MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(1);
+        mockSearchPhaseContext.searchTransport = new SearchTransportService(
+            Settings.builder().put("search.remote.connect", false).build(), null) {
+
+            @Override
+            void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionListener<MultiSearchResponse> listener) {
+              fail("no collapsing here");
+            }
+        };
+
+        SearchHits hits = new SearchHits(new SearchHit[]{new SearchHit(1, "ID", new Text("type"),
+            Collections.singletonMap("someField", new SearchHitField("someField", Collections.singletonList(null)))),
+            new SearchHit(2, "ID2", new Text("type"),
+                Collections.singletonMap("someField", new SearchHitField("someField", Collections.singletonList(null))))}, 1, 1.0F);
+        InternalSearchResponse internalSearchResponse = new InternalSearchResponse(hits, null, null, null, false, null, 1);
+        AtomicReference<SearchResponse> reference = new AtomicReference<>();
+        ExpandSearchPhase phase = new ExpandSearchPhase(mockSearchPhaseContext, internalSearchResponse, r ->
+            new SearchPhase("test") {
+                @Override
+                public void run() throws IOException {
+                    reference.set(mockSearchPhaseContext.buildSearchResponse(r, null));
+                }
+            }
+        );
+        phase.run();
+        mockSearchPhaseContext.assertNoFailure();
+        assertNotNull(reference.get());
+        assertEquals(1, mockSearchPhaseContext.phasesExecuted.get());
+    }
+
+    public void testSkipExpandCollapseNoHits() throws IOException {
+        MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(1);
+        mockSearchPhaseContext.searchTransport = new SearchTransportService(
+            Settings.builder().put("search.remote.connect", false).build(), null) {
+
+            @Override
+            void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionListener<MultiSearchResponse> listener) {
+                fail("expand should not try to send empty multi search request");
+            }
+        };
+        mockSearchPhaseContext.getRequest().source(new SearchSourceBuilder()
+            .collapse(new CollapseBuilder("someField").setInnerHits(new InnerHitBuilder().setName("foobarbaz"))));
+
+        SearchHits hits = new SearchHits(new SearchHit[0], 1, 1.0f);
+        InternalSearchResponse internalSearchResponse = new InternalSearchResponse(hits, null, null, null, false, null, 1);
+        AtomicReference<SearchResponse> reference = new AtomicReference<>();
+        ExpandSearchPhase phase = new ExpandSearchPhase(mockSearchPhaseContext, internalSearchResponse, r ->
+            new SearchPhase("test") {
+                @Override
+                public void run() throws IOException {
+                    reference.set(mockSearchPhaseContext.buildSearchResponse(r, null));
+                }
+            }
+        );
+        phase.run();
+        mockSearchPhaseContext.assertNoFailure();
+        assertNotNull(reference.get());
+        assertEquals(1, mockSearchPhaseContext.phasesExecuted.get());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java b/core/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java
new file mode 100644
index 0000000000000..c0c0ec44e58e7
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java
@@ -0,0 +1,361 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.search;
+
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.store.MockDirectoryWrapper;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.search.SearchPhaseResult;
+import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.search.fetch.FetchSearchResult;
+import org.elasticsearch.search.fetch.QueryFetchSearchResult;
+import org.elasticsearch.search.fetch.ShardFetchSearchRequest;
+import org.elasticsearch.search.query.QuerySearchResult;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.transport.Transport;
+
+import java.io.IOException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+public class FetchSearchPhaseTests extends ESTestCase {
+
+    public void testShortcutQueryAndFetchOptimization() throws IOException {
+        SearchPhaseController controller = new SearchPhaseController(Settings.EMPTY, BigArrays.NON_RECYCLING_INSTANCE, null);
+        MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(1);
+        InitialSearchPhase.ArraySearchPhaseResults<SearchPhaseResult> results =
+            controller.newSearchPhaseResults(mockSearchPhaseContext.getRequest(), 1);
+        AtomicReference<SearchResponse> responseRef = new AtomicReference<>();
+        boolean hasHits = randomBoolean();
+        final int numHits;
+        if (hasHits) {
+            QuerySearchResult queryResult = new QuerySearchResult();
+            queryResult.topDocs(new TopDocs(1, new ScoreDoc[] {new ScoreDoc(42, 1.0F)}, 1.0F), new DocValueFormat[0]);
+            queryResult.size(1);
+            FetchSearchResult fetchResult = new FetchSearchResult();
+            fetchResult.hits(new SearchHits(new SearchHit[] {new SearchHit(42)}, 1, 1.0F));
+            QueryFetchSearchResult fetchSearchResult = new QueryFetchSearchResult(queryResult, fetchResult);
+            fetchSearchResult.setShardIndex(0);
+            results.consumeResult(fetchSearchResult);
+            numHits = 1;
+        } else {
+            numHits = 0;
+        }
+
+        FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext,
+            (searchResponse, scrollId) -> new SearchPhase("test") {
+            @Override
+            public void run() throws IOException {
+                responseRef.set(mockSearchPhaseContext.buildSearchResponse(searchResponse, null));
+            }
+        });
+        assertEquals("fetch", phase.getName());
+        phase.run();
+        mockSearchPhaseContext.assertNoFailure();
+        assertNotNull(responseRef.get());
+        assertEquals(numHits, responseRef.get().getHits().getTotalHits());
+        if (numHits != 0) {
+            assertEquals(42, responseRef.get().getHits().getAt(0).docId());
+        }
+        assertTrue(mockSearchPhaseContext.releasedSearchContexts.isEmpty());
+    }
+
+    public void testFetchTwoDocument() throws IOException {
+        MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(2);
+        SearchPhaseController controller = new SearchPhaseController(Settings.EMPTY, BigArrays.NON_RECYCLING_INSTANCE, null);
+        InitialSearchPhase.ArraySearchPhaseResults<SearchPhaseResult> results =
+            controller.newSearchPhaseResults(mockSearchPhaseContext.getRequest(), 2);
+        AtomicReference<SearchResponse> responseRef = new AtomicReference<>();
+        int resultSetSize = randomIntBetween(2, 10);
+        QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new Index("test", "na"), 0, null));
+        queryResult.topDocs(new TopDocs(1, new ScoreDoc[] {new ScoreDoc(42, 1.0F)}, 2.0F), new DocValueFormat[0]);
+        queryResult.size(resultSetSize); // the size of the result set
+        queryResult.setShardIndex(0);
+        results.consumeResult(queryResult);
+
+        queryResult = new QuerySearchResult(321, new SearchShardTarget("node2", new Index("test", "na"), 1, null));
+        queryResult.topDocs(new TopDocs(1, new ScoreDoc[] {new ScoreDoc(84, 2.0F)}, 2.0F), new DocValueFormat[0]);
+        queryResult.size(resultSetSize);
+        queryResult.setShardIndex(1);
+        results.consumeResult(queryResult);
+
+        SearchTransportService searchTransportService = new SearchTransportService(
+            Settings.builder().put("search.remote.connect", false).build(), null) {
+            @Override
+            public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRequest request, SearchTask task,
+                                         SearchActionListener<FetchSearchResult> listener) {
+                FetchSearchResult fetchResult = new FetchSearchResult();
+                if (request.id() == 321) {
+                    fetchResult.hits(new SearchHits(new SearchHit[] {new SearchHit(84)}, 1, 2.0F));
+                } else {
+                    assertEquals(123, request.id());
+                    fetchResult.hits(new SearchHits(new SearchHit[] {new SearchHit(42)}, 1, 1.0F));
+                }
+                listener.onResponse(fetchResult);
+            }
+        };
+        mockSearchPhaseContext.searchTransport = searchTransportService;
+        FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext,
+            (searchResponse, scrollId) -> new SearchPhase("test") {
+                @Override
+                public void run() throws IOException {
+                    responseRef.set(mockSearchPhaseContext.buildSearchResponse(searchResponse, null));
+                }
+            });
+        assertEquals("fetch", phase.getName());
+        phase.run();
+        mockSearchPhaseContext.assertNoFailure();
+        assertNotNull(responseRef.get());
+        assertEquals(2, responseRef.get().getHits().getTotalHits());
+        assertEquals(84, responseRef.get().getHits().getAt(0).docId());
+        assertEquals(42, responseRef.get().getHits().getAt(1).docId());
+        assertEquals(0, responseRef.get().getFailedShards());
+        assertEquals(2, responseRef.get().getSuccessfulShards());
+        assertTrue(mockSearchPhaseContext.releasedSearchContexts.isEmpty());
+    }
+
+    public void testFailFetchOneDoc() throws IOException {
+        MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(2);
+        SearchPhaseController controller = new SearchPhaseController(Settings.EMPTY, BigArrays.NON_RECYCLING_INSTANCE, null);
+        InitialSearchPhase.ArraySearchPhaseResults<SearchPhaseResult> results =
+            controller.newSearchPhaseResults(mockSearchPhaseContext.getRequest(), 2);
+        AtomicReference<SearchResponse> responseRef = new AtomicReference<>();
+        int resultSetSize = randomIntBetween(2, 10);
+        QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new Index("test", "na"), 0, null));
+        queryResult.topDocs(new TopDocs(1, new ScoreDoc[] {new ScoreDoc(42, 1.0F)}, 2.0F), new DocValueFormat[0]);
+        queryResult.size(resultSetSize); // the size of the result set
+        queryResult.setShardIndex(0);
+        results.consumeResult(queryResult);
+
+        queryResult = new QuerySearchResult(321, new SearchShardTarget("node2", new Index("test", "na"), 1, null));
+        queryResult.topDocs(new TopDocs(1, new ScoreDoc[] {new ScoreDoc(84, 2.0F)}, 2.0F), new DocValueFormat[0]);
+        queryResult.size(resultSetSize);
+        queryResult.setShardIndex(1);
+        results.consumeResult(queryResult);
+
+        SearchTransportService searchTransportService = new SearchTransportService(
+            Settings.builder().put("search.remote.connect", false).build(), null) {
+            @Override
+            public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRequest request, SearchTask task,
+                                         SearchActionListener<FetchSearchResult> listener) {
+                if (request.id() == 321) {
+                    FetchSearchResult fetchResult = new FetchSearchResult();
+                    fetchResult.hits(new SearchHits(new SearchHit[] {new SearchHit(84)}, 1, 2.0F));
+                    listener.onResponse(fetchResult);
+                } else {
+                    listener.onFailure(new MockDirectoryWrapper.FakeIOException());
+                }
+
+            }
+        };
+        mockSearchPhaseContext.searchTransport = searchTransportService;
+        FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext,
+            (searchResponse, scrollId) -> new SearchPhase("test") {
+                @Override
+                public void run() throws IOException {
+                    responseRef.set(mockSearchPhaseContext.buildSearchResponse(searchResponse, null));
+                }
+            });
+        assertEquals("fetch", phase.getName());
+        phase.run();
+        mockSearchPhaseContext.assertNoFailure();
+        assertNotNull(responseRef.get());
+        assertEquals(2, responseRef.get().getHits().getTotalHits());
+        assertEquals(84, responseRef.get().getHits().getAt(0).docId());
+        assertEquals(1, responseRef.get().getFailedShards());
+        assertEquals(1, responseRef.get().getSuccessfulShards());
+        assertEquals(1, responseRef.get().getShardFailures().length);
+        assertTrue(responseRef.get().getShardFailures()[0].getCause() instanceof MockDirectoryWrapper.FakeIOException);
+        assertEquals(1, mockSearchPhaseContext.releasedSearchContexts.size());
+        assertTrue(mockSearchPhaseContext.releasedSearchContexts.contains(123L));
+    }
+
+    public void testFetchDocsConcurrently() throws IOException, InterruptedException {
+        int resultSetSize = randomIntBetween(0, 100);
+        // we use at least 2 hits otherwise this is subject to single shard optimization and we trip an assert...
+        int numHits = randomIntBetween(2, 100); // also numshards --> 1 hit per shard
+        SearchPhaseController controller = new SearchPhaseController(Settings.EMPTY, BigArrays.NON_RECYCLING_INSTANCE, null);
+        MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(numHits);
+        InitialSearchPhase.ArraySearchPhaseResults<SearchPhaseResult> results =
+            controller.newSearchPhaseResults(mockSearchPhaseContext.getRequest(), numHits);
+        AtomicReference<SearchResponse> responseRef = new AtomicReference<>();
+        for (int i = 0; i < numHits; i++) {
+            QuerySearchResult queryResult = new QuerySearchResult(i, new SearchShardTarget("node1", new Index("test", "na"), 0, null));
+            queryResult.topDocs(new TopDocs(1, new ScoreDoc[] {new ScoreDoc(i+1, i)}, i), new DocValueFormat[0]);
+            queryResult.size(resultSetSize); // the size of the result set
+            queryResult.setShardIndex(i);
+            results.consumeResult(queryResult);
+        }
+        SearchTransportService searchTransportService = new SearchTransportService(
+            Settings.builder().put("search.remote.connect", false).build(), null) {
+            @Override
+            public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRequest request, SearchTask task,
+                                         SearchActionListener<FetchSearchResult> listener) {
+                new Thread(() -> {
+                    FetchSearchResult fetchResult = new FetchSearchResult();
+                    fetchResult.hits(new SearchHits(new SearchHit[] {new SearchHit((int) (request.id()+1))}, 1,
+                        100F));
+                    listener.onResponse(fetchResult);
+                }).start();
+            }
+        };
+        mockSearchPhaseContext.searchTransport = searchTransportService;
+        CountDownLatch latch = new CountDownLatch(1);
+        FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext,
+            (searchResponse, scrollId) -> new SearchPhase("test") {
+                @Override
+                public void run() throws IOException {
+                    responseRef.set(mockSearchPhaseContext.buildSearchResponse(searchResponse, null));
+                    latch.countDown();
+                }
+            });
+        assertEquals("fetch", phase.getName());
+        phase.run();
+        latch.await();
+        mockSearchPhaseContext.assertNoFailure();
+        assertNotNull(responseRef.get());
+        assertEquals(numHits, responseRef.get().getHits().getTotalHits());
+        assertEquals(Math.min(numHits, resultSetSize), responseRef.get().getHits().getHits().length);
+        SearchHit[] hits = responseRef.get().getHits().getHits();
+        for (int i = 0; i < hits.length; i++) {
+            assertNotNull(hits[i]);
+            assertEquals("index: " + i, numHits-i, hits[i].docId());
+            assertEquals("index: " + i, numHits-1-i, (int)hits[i].getScore());
+        }
+        assertEquals(0, responseRef.get().getFailedShards());
+        assertEquals(numHits, responseRef.get().getSuccessfulShards());
+        int sizeReleasedContexts = Math.max(0, numHits - resultSetSize); // all non fetched results will be freed
+        assertEquals(mockSearchPhaseContext.releasedSearchContexts.toString(),
+            sizeReleasedContexts, mockSearchPhaseContext.releasedSearchContexts.size());
+    }
+
+    public void testExceptionFailsPhase() throws IOException {
+        MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(2);
+        SearchPhaseController controller = new SearchPhaseController(Settings.EMPTY, BigArrays.NON_RECYCLING_INSTANCE, null);
+        InitialSearchPhase.ArraySearchPhaseResults<SearchPhaseResult> results =
+            controller.newSearchPhaseResults(mockSearchPhaseContext.getRequest(), 2);
+        AtomicReference<SearchResponse> responseRef = new AtomicReference<>();
+        int resultSetSize = randomIntBetween(2, 10);
+        QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new Index("test", "na"), 0, null));
+        queryResult.topDocs(new TopDocs(1, new ScoreDoc[] {new ScoreDoc(42, 1.0F)}, 2.0F), new DocValueFormat[0]);
+        queryResult.size(resultSetSize); // the size of the result set
+        queryResult.setShardIndex(0);
+        results.consumeResult(queryResult);
+
+        queryResult = new QuerySearchResult(321, new SearchShardTarget("node2", new Index("test", "na"), 1, null));
+        queryResult.topDocs(new TopDocs(1, new ScoreDoc[] {new ScoreDoc(84, 2.0F)}, 2.0F), new DocValueFormat[0]);
+        queryResult.size(resultSetSize);
+        queryResult.setShardIndex(1);
+        results.consumeResult(queryResult);
+        AtomicInteger numFetches = new AtomicInteger(0);
+        SearchTransportService searchTransportService = new SearchTransportService(
+            Settings.builder().put("search.remote.connect", false).build(), null) {
+            @Override
+            public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRequest request, SearchTask task,
+                                         SearchActionListener<FetchSearchResult> listener) {
+                FetchSearchResult fetchResult = new FetchSearchResult();
+                if (numFetches.incrementAndGet() == 1) {
+                    throw new RuntimeException("BOOM");
+                }
+                if (request.id() == 321) {
+                    fetchResult.hits(new SearchHits(new SearchHit[] {new SearchHit(84)}, 1, 2.0F));
+                } else {
+                    assertEquals(request, 123);
+                    fetchResult.hits(new SearchHits(new SearchHit[] {new SearchHit(42)}, 1, 1.0F));
+                }
+                listener.onResponse(fetchResult);
+            }
+        };
+        mockSearchPhaseContext.searchTransport = searchTransportService;
+        FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext,
+            (searchResponse, scrollId) -> new SearchPhase("test") {
+                @Override
+                public void run() throws IOException {
+                    responseRef.set(mockSearchPhaseContext.buildSearchResponse(searchResponse, null));
+                }
+            });
+        assertEquals("fetch", phase.getName());
+        phase.run();
+        assertNotNull(mockSearchPhaseContext.phaseFailure.get());
+        assertEquals(mockSearchPhaseContext.phaseFailure.get().getMessage(), "BOOM");
+        assertNull(responseRef.get());
+        assertTrue(mockSearchPhaseContext.releasedSearchContexts.isEmpty());
+    }
+
+    public void testCleanupIrrelevantContexts() throws IOException { // contexts that are not fetched should be cleaned up
+        MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(2);
+        SearchPhaseController controller = new SearchPhaseController(Settings.EMPTY, BigArrays.NON_RECYCLING_INSTANCE, null);
+        InitialSearchPhase.ArraySearchPhaseResults<SearchPhaseResult> results =
+            controller.newSearchPhaseResults(mockSearchPhaseContext.getRequest(), 2);
+        AtomicReference<SearchResponse> responseRef = new AtomicReference<>();
+        int resultSetSize = 1;
+        QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new Index("test", "na"), 0, null));
+        queryResult.topDocs(new TopDocs(1, new ScoreDoc[] {new ScoreDoc(42, 1.0F)}, 2.0F), new DocValueFormat[0]);
+        queryResult.size(resultSetSize); // the size of the result set
+        queryResult.setShardIndex(0);
+        results.consumeResult(queryResult);
+
+        queryResult = new QuerySearchResult(321, new SearchShardTarget("node2", new Index("test", "na"), 1, null));
+        queryResult.topDocs(new TopDocs(1, new ScoreDoc[] {new ScoreDoc(84, 2.0F)}, 2.0F), new DocValueFormat[0]);
+        queryResult.size(resultSetSize);
+        queryResult.setShardIndex(1);
+        results.consumeResult(queryResult);
+
+        SearchTransportService searchTransportService = new SearchTransportService(
+            Settings.builder().put("search.remote.connect", false).build(), null) {
+            @Override
+            public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRequest request, SearchTask task,
+                                         SearchActionListener<FetchSearchResult> listener) {
+                FetchSearchResult fetchResult = new FetchSearchResult();
+                if (request.id() == 321) {
+                    fetchResult.hits(new SearchHits(new SearchHit[] {new SearchHit(84)}, 1, 2.0F));
+                } else {
+                    fail("requestID 123 should not be fetched but was");
+                }
+                listener.onResponse(fetchResult);
+            }
+        };
+        mockSearchPhaseContext.searchTransport = searchTransportService;
+        FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext,
+            (searchResponse, scrollId) -> new SearchPhase("test") {
+                @Override
+                public void run() throws IOException {
+                    responseRef.set(mockSearchPhaseContext.buildSearchResponse(searchResponse, null));
+                }
+            });
+        assertEquals("fetch", phase.getName());
+        phase.run();
+        mockSearchPhaseContext.assertNoFailure();
+        assertNotNull(responseRef.get());
+        assertEquals(2, responseRef.get().getHits().getTotalHits());
+        assertEquals(1, responseRef.get().getHits().getHits().length);
+        assertEquals(84, responseRef.get().getHits().getAt(0).docId());
+        assertEquals(0, responseRef.get().getFailedShards());
+        assertEquals(2, responseRef.get().getSuccessfulShards());
+        assertEquals(1, mockSearchPhaseContext.releasedSearchContexts.size());
+        assertTrue(mockSearchPhaseContext.releasedSearchContexts.contains(123L));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/search/MockSearchPhaseContext.java b/core/src/test/java/org/elasticsearch/action/search/MockSearchPhaseContext.java
new file mode 100644
index 0000000000000..2f06dd873215e
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/search/MockSearchPhaseContext.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.search;
+
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.OriginalIndices;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.search.internal.InternalSearchResponse;
+import org.elasticsearch.search.internal.ShardSearchTransportRequest;
+import org.elasticsearch.transport.Transport;
+import org.junit.Assert;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ * SearchPhaseContext for tests
+ */
+public final class MockSearchPhaseContext implements SearchPhaseContext {
+    private static final Logger logger = Loggers.getLogger(MockSearchPhaseContext.class);
+    public AtomicReference<Throwable> phaseFailure = new AtomicReference<>();
+    final int numShards;
+    final AtomicInteger numSuccess;
+    List<ShardSearchFailure> failures = Collections.synchronizedList(new ArrayList<>());
+    SearchTransportService searchTransport;
+    Set<Long> releasedSearchContexts = new HashSet<>();
+    SearchRequest searchRequest = new SearchRequest();
+    AtomicInteger phasesExecuted = new AtomicInteger();
+
+    public MockSearchPhaseContext(int numShards) {
+        this.numShards = numShards;
+        numSuccess = new AtomicInteger(numShards);
+    }
+
+    public void assertNoFailure() {
+        if (phaseFailure.get() != null) {
+            throw new AssertionError(phaseFailure.get());
+        }
+    }
+
+    @Override
+    public int getNumShards() {
+        return numShards;
+    }
+
+    @Override
+    public Logger getLogger() {
+        return logger;
+    }
+
+    @Override
+    public SearchTask getTask() {
+        return new SearchTask(0, "n/a", "n/a", "test", null);
+    }
+
+    @Override
+    public SearchRequest getRequest() {
+        return searchRequest;
+    }
+
+    @Override
+    public SearchResponse buildSearchResponse(InternalSearchResponse internalSearchResponse, String scrollId) {
+        return new SearchResponse(internalSearchResponse, scrollId, numShards, numSuccess.get(), 0, 0,
+            failures.toArray(new ShardSearchFailure[0]));
+    }
+
+    @Override
+    public void onPhaseFailure(SearchPhase phase, String msg, Throwable cause) {
+        phaseFailure.set(cause);
+    }
+
+    @Override
+    public void onShardFailure(int shardIndex, @Nullable SearchShardTarget shardTarget, Exception e) {
+        failures.add(new ShardSearchFailure(e, shardTarget));
+        numSuccess.decrementAndGet();
+    }
+
+    @Override
+    public Transport.Connection getConnection(String clusterAlias, String nodeId) {
+        return null; // null is ok here for this test
+    }
+
+    @Override
+    public SearchTransportService getSearchTransport() {
+        Assert.assertNotNull(searchTransport);
+        return searchTransport;
+    }
+
+    @Override
+    public ShardSearchTransportRequest buildShardSearchRequest(SearchShardIterator shardIt) {
+        Assert.fail("should not be called");
+        return null;
+    }
+
+    @Override
+    public void executeNextPhase(SearchPhase currentPhase, SearchPhase nextPhase) {
+        phasesExecuted.incrementAndGet();
+        try {
+            nextPhase.run();
+        } catch (Exception e) {
+           onPhaseFailure(nextPhase, "phase failed", e);
+        }
+    }
+
+    @Override
+    public void execute(Runnable command) {
+        command.run();
+    }
+
+    @Override
+    public void onResponse(SearchResponse response) {
+        Assert.fail("should not be called");
+    }
+
+    @Override
+    public void onFailure(Exception e) {
+        Assert.fail("should not be called");
+    }
+
+    @Override
+    public void sendReleaseSearchContext(long contextId, Transport.Connection connection, OriginalIndices originalIndices) {
+        releasedSearchContexts.add(contextId);
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java b/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java
index 3c9acb4104b6f..5e707bb523c92 100644
--- a/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java
@@ -20,24 +20,23 @@
 package org.elasticsearch.action.search;
 
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
-import org.elasticsearch.index.query.QueryParser;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
+import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.search.RestMultiSearchAction;
-import org.elasticsearch.search.SearchRequestParsers;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.StreamsUtils;
 import org.elasticsearch.test.rest.FakeRestRequest;
 
 import java.io.IOException;
+import java.util.Optional;
 
+import static java.util.Collections.singletonList;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
@@ -81,6 +80,19 @@ public void testSimpleAdd() throws Exception {
         assertThat(request.requests().get(7).types().length, equalTo(0));
     }
 
+    public void testSimpleAddWithCarriageReturn() throws Exception {
+        final String requestContent = "{\"index\":\"test\", \"ignore_unavailable\" : true, \"expand_wildcards\" : \"open,closed\"}}\r\n" +
+            "{\"query\" : {\"match_all\" :{}}}\r\n";
+        FakeRestRequest restRequest = new FakeRestRequest.Builder(xContentRegistry())
+            .withContent(new BytesArray(requestContent), XContentType.JSON).build();
+        MultiSearchRequest request = RestMultiSearchAction.parseRequest(restRequest, true);
+        assertThat(request.requests().size(), equalTo(1));
+        assertThat(request.requests().get(0).indices()[0], equalTo("test"));
+        assertThat(request.requests().get(0).indicesOptions(),
+            equalTo(IndicesOptions.fromOptions(true, true, true, true, IndicesOptions.strictExpandOpenAndForbidClosed())));
+        assertThat(request.requests().get(0).types().length, equalTo(0));
+    }
+
     public void testSimpleAdd2() throws Exception {
         MultiSearchRequest request = parseMultiSearchRequest("/org/elasticsearch/action/search/simple-msearch2.json");
         assertThat(request.requests().size(), equalTo(5));
@@ -140,11 +152,6 @@ public void testResponseErrorToXContent() throws IOException {
                     new MultiSearchResponse.Item(null, new IllegalStateException("baaaaaazzzz"))
         });
 
-        XContentBuilder builder = XContentFactory.jsonBuilder();
-        builder.startObject();
-        response.toXContent(builder, ToXContent.EMPTY_PARAMS);
-        builder.endObject();
-
         assertEquals("{\"responses\":["
                         + "{"
                         + "\"error\":{\"root_cause\":[{\"type\":\"illegal_state_exception\",\"reason\":\"foobar\"}],"
@@ -155,7 +162,7 @@ public void testResponseErrorToXContent() throws IOException {
                         + "\"type\":\"illegal_state_exception\",\"reason\":\"baaaaaazzzz\"},\"status\":500"
                         + "}"
                         + "]}",
-                builder.string());
+                Strings.toString(response));
     }
 
     public void testMaxConcurrentSearchRequests() {
@@ -167,14 +174,14 @@ public void testMaxConcurrentSearchRequests() {
 
     private MultiSearchRequest parseMultiSearchRequest(String sample) throws IOException {
         byte[] data = StreamsUtils.copyToBytesFromClasspath(sample);
-        RestRequest restRequest = new FakeRestRequest.Builder().withContent(new BytesArray(data)).build();
-        return RestMultiSearchAction.parseRequest(restRequest, true, parsers(), ParseFieldMatcher.EMPTY);
+        RestRequest restRequest = new FakeRestRequest.Builder(xContentRegistry())
+            .withContent(new BytesArray(data), XContentType.JSON).build();
+        return RestMultiSearchAction.parseRequest(restRequest, true);
     }
 
-    private SearchRequestParsers parsers() {
-        IndicesQueriesRegistry registry = new IndicesQueriesRegistry();
-        QueryParser<MatchAllQueryBuilder> parser = MatchAllQueryBuilder::fromXContent;
-        registry.register(parser, MatchAllQueryBuilder.NAME);
-        return new SearchRequestParsers(registry, null, null);
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return new NamedXContentRegistry(singletonList(new NamedXContentRegistry.Entry(Optional.class,
+                new ParseField(MatchAllQueryBuilder.NAME), (p, c) -> MatchAllQueryBuilder.fromXContent((QueryParseContext) c))));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java b/core/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java
new file mode 100644
index 0000000000000..1abab96b9d5da
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java
@@ -0,0 +1,434 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.search;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.OriginalIndices;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.GroupShardsIterator;
+import org.elasticsearch.cluster.routing.RecoverySource;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.UnassignedInfo;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.LocalTransportAddress;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.search.SearchPhaseResult;
+import org.elasticsearch.search.internal.AliasFilter;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.transport.Transport;
+import org.elasticsearch.transport.TransportException;
+import org.elasticsearch.transport.TransportRequest;
+import org.elasticsearch.transport.TransportRequestOptions;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+public class SearchAsyncActionTests extends ESTestCase {
+
+    public void testSkipSearchShards() throws InterruptedException {
+        SearchRequest request = new SearchRequest();
+        CountDownLatch latch = new CountDownLatch(1);
+        AtomicReference<TestSearchResponse> response = new AtomicReference<>();
+        ActionListener<SearchResponse> responseListener = new ActionListener<SearchResponse>() {
+            @Override
+            public void onResponse(SearchResponse searchResponse) {
+                response.set((TestSearchResponse) searchResponse);
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                logger.warn("test failed", e);
+                fail(e.getMessage());
+            }
+        };
+        DiscoveryNode primaryNode = new DiscoveryNode("node_1", LocalTransportAddress.buildUnique(), Version.CURRENT);
+        DiscoveryNode replicaNode = new DiscoveryNode("node_2", LocalTransportAddress.buildUnique(), Version.CURRENT);
+
+        AtomicInteger contextIdGenerator = new AtomicInteger(0);
+        GroupShardsIterator<SearchShardIterator> shardsIter = getShardsIter("idx",
+            new OriginalIndices(new String[]{"idx"}, IndicesOptions.strictExpandOpenAndForbidClosed()),
+            10, randomBoolean(), primaryNode, replicaNode);
+        int numSkipped = 0;
+        for (SearchShardIterator iter : shardsIter) {
+            if (iter.shardId().id() % 2 == 0) {
+                iter.resetAndSkip();
+                numSkipped++;
+            }
+        }
+
+        SearchTransportService transportService = new SearchTransportService(Settings.EMPTY, null);
+        Map<String, Transport.Connection> lookup = new HashMap<>();
+        Map<ShardId, Boolean> seenShard = new ConcurrentHashMap<>();
+        lookup.put(primaryNode.getId(), new MockConnection(primaryNode));
+        lookup.put(replicaNode.getId(), new MockConnection(replicaNode));
+        Map<String, AliasFilter> aliasFilters = Collections.singletonMap("_na_", new AliasFilter(null, Strings.EMPTY_ARRAY));
+        AtomicInteger numRequests = new AtomicInteger(0);
+        AbstractSearchAsyncAction asyncAction =
+            new AbstractSearchAsyncAction<TestSearchPhaseResult>(
+                "test",
+                logger,
+                transportService,
+                (cluster, node) -> {
+                    assert cluster == null : "cluster was not null: " + cluster;
+                    return lookup.get(node); },
+                aliasFilters,
+                Collections.emptyMap(),
+                null,
+                request,
+                responseListener,
+                shardsIter,
+                new TransportSearchAction.SearchTimeProvider(0, 0, () -> 0),
+                0,
+                null,
+                new InitialSearchPhase.ArraySearchPhaseResults<>(shardsIter.size())) {
+
+                @Override
+                protected void executePhaseOnShard(SearchShardIterator shardIt, ShardRouting shard,
+                                                   SearchActionListener<TestSearchPhaseResult> listener) {
+                    seenShard.computeIfAbsent(shard.shardId(), (i) -> {
+                        numRequests.incrementAndGet(); // only count this once per replica
+                        return Boolean.TRUE;
+                    });
+
+                    new Thread(() -> {
+                        Transport.Connection connection = getConnection(null, shard.currentNodeId());
+                        TestSearchPhaseResult testSearchPhaseResult = new TestSearchPhaseResult(contextIdGenerator.incrementAndGet(),
+                        connection.getNode());
+                        listener.onResponse(testSearchPhaseResult);
+
+                    }).start();
+                }
+
+                @Override
+                protected SearchPhase getNextPhase(SearchPhaseResults<TestSearchPhaseResult> results, SearchPhaseContext context) {
+                    return new SearchPhase("test") {
+                        @Override
+                        public void run() throws IOException {
+                            latch.countDown();
+                        }
+                    };
+                }
+            };
+        asyncAction.start();
+        latch.await();
+        SearchResponse searchResponse = asyncAction.buildSearchResponse(null, null);
+        assertEquals(shardsIter.size()-numSkipped, numRequests.get());
+        assertEquals(0, searchResponse.getFailedShards());
+        assertEquals(numSkipped, searchResponse.getSkippedShards());
+        assertEquals(shardsIter.size(), searchResponse.getSuccessfulShards());
+    }
+
+    public void testLimitConcurrentShardRequests() throws InterruptedException {
+        SearchRequest request = new SearchRequest();
+        int numConcurrent = randomIntBetween(1, 5);
+        request.setMaxConcurrentShardRequests(numConcurrent);
+        CountDownLatch latch = new CountDownLatch(1);
+        AtomicReference<TestSearchResponse> response = new AtomicReference<>();
+        ActionListener<SearchResponse> responseListener = new ActionListener<SearchResponse>() {
+            @Override
+            public void onResponse(SearchResponse searchResponse) {
+                response.set((TestSearchResponse) searchResponse);
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                logger.warn("test failed", e);
+                fail(e.getMessage());
+            }
+        };
+        DiscoveryNode primaryNode = new DiscoveryNode("node_1", LocalTransportAddress.buildUnique(), Version.CURRENT);
+        DiscoveryNode replicaNode = new DiscoveryNode("node_2", LocalTransportAddress.buildUnique(), Version.CURRENT);
+
+        AtomicInteger contextIdGenerator = new AtomicInteger(0);
+        GroupShardsIterator<SearchShardIterator> shardsIter = getShardsIter("idx",
+            new OriginalIndices(new String[]{"idx"}, IndicesOptions.strictExpandOpenAndForbidClosed()),
+            10, randomBoolean(), primaryNode, replicaNode);
+        SearchTransportService transportService = new SearchTransportService(Settings.EMPTY, null);
+        Map<String, Transport.Connection> lookup = new HashMap<>();
+        Map<ShardId, Boolean> seenShard = new ConcurrentHashMap<>();
+        lookup.put(primaryNode.getId(), new MockConnection(primaryNode));
+        lookup.put(replicaNode.getId(), new MockConnection(replicaNode));
+        Map<String, AliasFilter> aliasFilters = Collections.singletonMap("_na_", new AliasFilter(null, Strings.EMPTY_ARRAY));
+        CountDownLatch awaitInitialRequests = new CountDownLatch(1);
+        AtomicInteger numRequests = new AtomicInteger(0);
+        AtomicInteger numResponses = new AtomicInteger(0);
+        AbstractSearchAsyncAction asyncAction =
+            new AbstractSearchAsyncAction<TestSearchPhaseResult>(
+                "test",
+                logger,
+                transportService,
+                (cluster, node) -> {
+                    assert cluster == null : "cluster was not null: " + cluster;
+                    return lookup.get(node); },
+                aliasFilters,
+                Collections.emptyMap(),
+                null,
+                request,
+                responseListener,
+                shardsIter,
+                new TransportSearchAction.SearchTimeProvider(0, 0, () -> 0),
+                0,
+                null,
+                new InitialSearchPhase.ArraySearchPhaseResults<>(shardsIter.size())) {
+
+                @Override
+                protected void executePhaseOnShard(SearchShardIterator shardIt, ShardRouting shard,
+                                                   SearchActionListener<TestSearchPhaseResult> listener) {
+                    seenShard.computeIfAbsent(shard.shardId(), (i) -> {
+                        numRequests.incrementAndGet(); // only count this once per replica
+                        return Boolean.TRUE;
+                    });
+
+                    new Thread(() -> {
+                        try {
+                            awaitInitialRequests.await();
+                        } catch (InterruptedException e) {
+                            throw new AssertionError(e);
+                        }
+                        Transport.Connection connection = getConnection(null, shard.currentNodeId());
+                        TestSearchPhaseResult testSearchPhaseResult = new TestSearchPhaseResult(contextIdGenerator.incrementAndGet(),
+                            connection.getNode());
+                        if (numResponses.getAndIncrement() > 0 && randomBoolean()) { // at least one response otherwise the entire
+                            // request fails
+                            listener.onFailure(new RuntimeException());
+                        } else {
+                            listener.onResponse(testSearchPhaseResult);
+                        }
+
+                    }).start();
+                }
+
+                @Override
+                protected SearchPhase getNextPhase(SearchPhaseResults<TestSearchPhaseResult> results, SearchPhaseContext context) {
+                    return new SearchPhase("test") {
+                        @Override
+                        public void run() throws IOException {
+                            latch.countDown();
+                        }
+                    };
+                }
+            };
+        asyncAction.start();
+        assertEquals(numConcurrent, numRequests.get());
+        awaitInitialRequests.countDown();
+        latch.await();
+        assertEquals(10, numRequests.get());
+    }
+
+    public void testFanOutAndCollect() throws InterruptedException {
+        SearchRequest request = new SearchRequest();
+        request.setMaxConcurrentShardRequests(randomIntBetween(1, 100));
+        CountDownLatch latch = new CountDownLatch(1);
+        AtomicReference<TestSearchResponse> response = new AtomicReference<>();
+        ActionListener<SearchResponse> responseListener = new ActionListener<SearchResponse>() {
+            @Override
+            public void onResponse(SearchResponse searchResponse) {
+                response.set((TestSearchResponse) searchResponse);
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                logger.warn("test failed", e);
+                fail(e.getMessage());
+            }
+        };
+        DiscoveryNode primaryNode = new DiscoveryNode("node_1", new LocalTransportAddress("foo"), Version.CURRENT);
+        DiscoveryNode replicaNode = new DiscoveryNode("node_2", new LocalTransportAddress("bar"), Version.CURRENT);
+
+        Map<DiscoveryNode, Set<Long>> nodeToContextMap = new HashMap<>();
+        AtomicInteger contextIdGenerator = new AtomicInteger(0);
+        GroupShardsIterator<SearchShardIterator> shardsIter = getShardsIter("idx",
+                new OriginalIndices(new String[]{"idx"}, IndicesOptions.strictExpandOpenAndForbidClosed()),
+                randomIntBetween(1, 10), randomBoolean(), primaryNode, replicaNode);
+        AtomicInteger numFreedContext = new AtomicInteger();
+        SearchTransportService transportService = new SearchTransportService(Settings.EMPTY, null) {
+            @Override
+            public void sendFreeContext(Transport.Connection connection, long contextId, OriginalIndices originalIndices) {
+                numFreedContext.incrementAndGet();
+                assertTrue(nodeToContextMap.containsKey(connection.getNode()));
+                assertTrue(nodeToContextMap.get(connection.getNode()).remove(contextId));
+            }
+        };
+        Map<String, Transport.Connection> lookup = new HashMap<>();
+        lookup.put(primaryNode.getId(), new MockConnection(primaryNode));
+        lookup.put(replicaNode.getId(), new MockConnection(replicaNode));
+        Map<String, AliasFilter> aliasFilters = Collections.singletonMap("_na_", new AliasFilter(null, Strings.EMPTY_ARRAY));
+        AbstractSearchAsyncAction asyncAction =
+                new AbstractSearchAsyncAction<TestSearchPhaseResult>(
+                        "test",
+                        logger,
+                        transportService,
+                        (cluster, node) -> {
+                            assert cluster == null : "cluster was not null: " + cluster;
+                            return lookup.get(node); },
+                        aliasFilters,
+                        Collections.emptyMap(),
+                        null,
+                        request,
+                        responseListener,
+                        shardsIter,
+                        new TransportSearchAction.SearchTimeProvider(0, 0, () -> 0),
+                        0,
+                        null,
+                        new InitialSearchPhase.ArraySearchPhaseResults<>(shardsIter.size())) {
+            TestSearchResponse response = new TestSearchResponse();
+
+            @Override
+            protected void executePhaseOnShard(SearchShardIterator shardIt, ShardRouting shard, SearchActionListener<TestSearchPhaseResult>
+                listener) {
+                assertTrue("shard: " + shard.shardId() + " has been queried twice", response.queried.add(shard.shardId()));
+                Transport.Connection connection = getConnection(null, shard.currentNodeId());
+                TestSearchPhaseResult testSearchPhaseResult = new TestSearchPhaseResult(contextIdGenerator.incrementAndGet(),
+                    connection.getNode());
+                Set<Long> ids = nodeToContextMap.computeIfAbsent(connection.getNode(), (n) -> new HashSet<>());
+                ids.add(testSearchPhaseResult.getRequestId());
+                if (randomBoolean()) {
+                    listener.onResponse(testSearchPhaseResult);
+                } else {
+                    new Thread(() -> listener.onResponse(testSearchPhaseResult)).start();
+                }
+            }
+
+            @Override
+            protected SearchPhase getNextPhase(SearchPhaseResults<TestSearchPhaseResult> results, SearchPhaseContext context) {
+                return new SearchPhase("test") {
+                    @Override
+                    public void run() throws IOException {
+                        for (int i = 0; i < results.getNumShards(); i++) {
+                            TestSearchPhaseResult result = results.getAtomicArray().get(i);
+                            assertEquals(result.node.getId(), result.getSearchShardTarget().getNodeId());
+                            sendReleaseSearchContext(result.getRequestId(), new MockConnection(result.node), OriginalIndices.NONE);
+                        }
+                        responseListener.onResponse(response);
+                        latch.countDown();
+                    }
+                };
+            }
+        };
+        asyncAction.start();
+        latch.await();
+        assertNotNull(response.get());
+        assertFalse(nodeToContextMap.isEmpty());
+        assertTrue(nodeToContextMap.toString(), nodeToContextMap.containsKey(primaryNode) || nodeToContextMap.containsKey(replicaNode));
+        assertEquals(shardsIter.size(), numFreedContext.get());
+        if (nodeToContextMap.containsKey(primaryNode)) {
+            assertTrue(nodeToContextMap.get(primaryNode).toString(), nodeToContextMap.get(primaryNode).isEmpty());
+        } else {
+            assertTrue(nodeToContextMap.get(replicaNode).toString(), nodeToContextMap.get(replicaNode).isEmpty());
+        }
+    }
+
+    static GroupShardsIterator<SearchShardIterator> getShardsIter(String index, OriginalIndices originalIndices, int numShards,
+                                                     boolean doReplicas, DiscoveryNode primaryNode, DiscoveryNode replicaNode) {
+        ArrayList<SearchShardIterator> list = new ArrayList<>();
+        for (int i = 0; i < numShards; i++) {
+            ArrayList<ShardRouting> started = new ArrayList<>();
+            ArrayList<ShardRouting> initializing = new ArrayList<>();
+            ArrayList<ShardRouting> unassigned = new ArrayList<>();
+
+            ShardRouting routing = ShardRouting.newUnassigned(new ShardId(new Index(index, "_na_"), i), true,
+                RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar"));
+            routing = routing.initialize(primaryNode.getId(), i + "p", 0);
+            routing.started();
+            started.add(routing);
+            if (doReplicas) {
+                routing = ShardRouting.newUnassigned(new ShardId(new Index(index, "_na_"), i), false,
+                    RecoverySource.PeerRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar"));
+                if (replicaNode != null) {
+                    routing = routing.initialize(replicaNode.getId(), i + "r", 0);
+                    if (randomBoolean()) {
+                        routing.started();
+                        started.add(routing);
+                    } else {
+                        initializing.add(routing);
+                    }
+                } else {
+                    unassigned.add(routing); // unused yet
+                }
+            }
+            Collections.shuffle(started, random());
+            started.addAll(initializing);
+            list.add(new SearchShardIterator(null, new ShardId(new Index(index, "_na_"), i), started, originalIndices));
+        }
+        return new GroupShardsIterator<>(list);
+    }
+
+    public static class TestSearchResponse extends SearchResponse {
+        public final Set<ShardId> queried = new HashSet<>();
+    }
+
+    public static class TestSearchPhaseResult extends SearchPhaseResult {
+        final DiscoveryNode node;
+
+        public TestSearchPhaseResult(long id, DiscoveryNode node) {
+            this.requestId = id;
+            this.node = node;
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+
+        }
+    }
+
+    public static final class MockConnection implements Transport.Connection {
+
+        private final DiscoveryNode node;
+
+        public MockConnection(DiscoveryNode node) {
+            this.node = node;
+        }
+
+        @Override
+        public DiscoveryNode getNode() {
+            return node;
+        }
+
+        @Override
+        public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options)
+            throws IOException, TransportException {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public void close() throws IOException {
+            throw new UnsupportedOperationException();
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java b/core/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java
new file mode 100644
index 0000000000000..c614ddafc4462
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java
@@ -0,0 +1,490 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.search;
+
+import com.carrotsearch.randomizedtesting.RandomizedContext;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.TopDocs;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.text.Text;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.concurrent.AtomicArray;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.SearchPhaseResult;
+import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.search.aggregations.AggregationBuilders;
+import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.aggregations.metrics.max.InternalMax;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.fetch.FetchSearchResult;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.search.internal.InternalSearchResponse;
+import org.elasticsearch.search.query.QuerySearchResult;
+import org.elasticsearch.search.suggest.Suggest;
+import org.elasticsearch.search.suggest.completion.CompletionSuggestion;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
+import static org.hamcrest.Matchers.not;
+
+public class SearchPhaseControllerTests extends ESTestCase {
+    private SearchPhaseController searchPhaseController;
+
+    @Before
+    public void setup() {
+        searchPhaseController = new SearchPhaseController(Settings.EMPTY, BigArrays.NON_RECYCLING_INSTANCE, null);
+    }
+
+    public void testSort() throws Exception {
+        List<CompletionSuggestion> suggestions = new ArrayList<>();
+        for (int i = 0; i < randomIntBetween(1, 5); i++) {
+            suggestions.add(new CompletionSuggestion(randomAlphaOfLength(randomIntBetween(1, 5)), randomIntBetween(1, 20)));
+        }
+        int nShards = randomIntBetween(1, 20);
+        int queryResultSize = randomBoolean() ? 0 : randomIntBetween(1, nShards * 2);
+        AtomicArray<SearchPhaseResult> results = generateQueryResults(nShards, suggestions, queryResultSize, false);
+        Optional<SearchPhaseResult> first = results.asList().stream().findFirst();
+        int from = 0, size = 0;
+        if (first.isPresent()) {
+            from = first.get().queryResult().from();
+            size = first.get().queryResult().size();
+        }
+        int accumulatedLength = Math.min(queryResultSize, getTotalQueryHits(results));
+        ScoreDoc[] sortedDocs = searchPhaseController.sortDocs(true, results.asList(), null, new SearchPhaseController.TopDocsStats(),
+            from, size)
+            .scoreDocs;
+        for (Suggest.Suggestion<?> suggestion : reducedSuggest(results)) {
+            int suggestionSize = suggestion.getEntries().get(0).getOptions().size();
+            accumulatedLength += suggestionSize;
+        }
+        assertThat(sortedDocs.length, equalTo(accumulatedLength));
+    }
+
+    public void testSortIsIdempotent() throws Exception {
+        int nShards = randomIntBetween(1, 20);
+        int queryResultSize = randomBoolean() ? 0 : randomIntBetween(1, nShards * 2);
+        long randomSeed = randomLong();
+        boolean useConstantScore = randomBoolean();
+        AtomicArray<SearchPhaseResult> results = generateSeededQueryResults(randomSeed, nShards, Collections.emptyList(), queryResultSize,
+            useConstantScore);
+        boolean ignoreFrom = randomBoolean();
+        Optional<SearchPhaseResult> first = results.asList().stream().findFirst();
+        int from = 0, size = 0;
+        if (first.isPresent()) {
+            from = first.get().queryResult().from();
+            size = first.get().queryResult().size();
+        }
+        SearchPhaseController.TopDocsStats topDocsStats = new SearchPhaseController.TopDocsStats();
+        ScoreDoc[] sortedDocs = searchPhaseController.sortDocs(ignoreFrom, results.asList(), null, topDocsStats, from, size).scoreDocs;
+
+        results = generateSeededQueryResults(randomSeed, nShards, Collections.emptyList(), queryResultSize,
+            useConstantScore);
+        SearchPhaseController.TopDocsStats topDocsStats2 = new SearchPhaseController.TopDocsStats();
+        ScoreDoc[] sortedDocs2 = searchPhaseController.sortDocs(ignoreFrom, results.asList(), null, topDocsStats2, from, size).scoreDocs;
+        assertEquals(sortedDocs.length, sortedDocs2.length);
+        for (int i = 0; i < sortedDocs.length; i++) {
+            assertEquals(sortedDocs[i].doc, sortedDocs2[i].doc);
+            assertEquals(sortedDocs[i].shardIndex, sortedDocs2[i].shardIndex);
+            assertEquals(sortedDocs[i].score, sortedDocs2[i].score, 0.0f);
+        }
+        assertEquals(topDocsStats.maxScore, topDocsStats2.maxScore, 0.0f);
+        assertEquals(topDocsStats.totalHits, topDocsStats2.totalHits);
+        assertEquals(topDocsStats.fetchHits, topDocsStats2.fetchHits);
+    }
+
+    private AtomicArray<SearchPhaseResult> generateSeededQueryResults(long seed, int nShards,
+                                                                      List<CompletionSuggestion> suggestions,
+                                                                      int searchHitsSize, boolean useConstantScore) throws Exception {
+        return RandomizedContext.current().runWithPrivateRandomness(seed,
+            () -> generateQueryResults(nShards, suggestions, searchHitsSize, useConstantScore));
+    }
+
+    public void testMerge() throws IOException {
+        List<CompletionSuggestion> suggestions = new ArrayList<>();
+        int maxSuggestSize = 0;
+        for (int i = 0; i < randomIntBetween(1, 5); i++) {
+            int size = randomIntBetween(1, 20);
+            maxSuggestSize += size;
+            suggestions.add(new CompletionSuggestion(randomAlphaOfLength(randomIntBetween(1, 5)), size));
+        }
+        int nShards = randomIntBetween(1, 20);
+        int queryResultSize = randomBoolean() ? 0 : randomIntBetween(1, nShards * 2);
+        AtomicArray<SearchPhaseResult> queryResults = generateQueryResults(nShards, suggestions, queryResultSize, false);
+        SearchPhaseController.ReducedQueryPhase reducedQueryPhase = searchPhaseController.reducedQueryPhase(queryResults.asList(), false);
+        AtomicArray<SearchPhaseResult> searchPhaseResultAtomicArray = generateFetchResults(nShards, reducedQueryPhase.scoreDocs,
+            reducedQueryPhase.suggest);
+        InternalSearchResponse mergedResponse = searchPhaseController.merge(false,
+            reducedQueryPhase,
+            searchPhaseResultAtomicArray.asList(), searchPhaseResultAtomicArray::get);
+        int suggestSize = 0;
+        for (Suggest.Suggestion s : reducedQueryPhase.suggest) {
+            Stream<CompletionSuggestion.Entry> stream = s.getEntries().stream();
+            suggestSize += stream.collect(Collectors.summingInt(e -> e.getOptions().size()));
+        }
+        assertThat(suggestSize, lessThanOrEqualTo(maxSuggestSize));
+        assertThat(mergedResponse.hits().getHits().length, equalTo(reducedQueryPhase.scoreDocs.length-suggestSize));
+        Suggest suggestResult = mergedResponse.suggest();
+        for (Suggest.Suggestion<?> suggestion : reducedQueryPhase.suggest) {
+            assertThat(suggestion, instanceOf(CompletionSuggestion.class));
+            if (suggestion.getEntries().get(0).getOptions().size() > 0) {
+                CompletionSuggestion suggestionResult = suggestResult.getSuggestion(suggestion.getName());
+                assertNotNull(suggestionResult);
+                List<CompletionSuggestion.Entry.Option> options = suggestionResult.getEntries().get(0).getOptions();
+                assertThat(options.size(), equalTo(suggestion.getEntries().get(0).getOptions().size()));
+                for (CompletionSuggestion.Entry.Option option : options) {
+                    assertNotNull(option.getHit());
+                }
+            }
+        }
+    }
+
+    private AtomicArray<SearchPhaseResult> generateQueryResults(int nShards,
+                                                                List<CompletionSuggestion> suggestions,
+                                                                int searchHitsSize, boolean useConstantScore) {
+        AtomicArray<SearchPhaseResult> queryResults = new AtomicArray<>(nShards);
+        for (int shardIndex = 0; shardIndex < nShards; shardIndex++) {
+            QuerySearchResult querySearchResult = new QuerySearchResult(shardIndex,
+                new SearchShardTarget("", new Index("", ""), shardIndex, null));
+            TopDocs topDocs = new TopDocs(0, new ScoreDoc[0], 0);
+            if (searchHitsSize > 0) {
+                int nDocs = randomIntBetween(0, searchHitsSize);
+                ScoreDoc[] scoreDocs = new ScoreDoc[nDocs];
+                float maxScore = 0F;
+                for (int i = 0; i < nDocs; i++) {
+                    float score = useConstantScore ? 1.0F : Math.abs(randomFloat());
+                    scoreDocs[i] = new ScoreDoc(i, score);
+                    if (score > maxScore) {
+                        maxScore = score;
+                    }
+                }
+                topDocs = new TopDocs(scoreDocs.length, scoreDocs, maxScore);
+            }
+            List<CompletionSuggestion> shardSuggestion = new ArrayList<>();
+            for (CompletionSuggestion completionSuggestion : suggestions) {
+                CompletionSuggestion suggestion = new CompletionSuggestion(
+                    completionSuggestion.getName(), completionSuggestion.getSize());
+                final CompletionSuggestion.Entry completionEntry = new CompletionSuggestion.Entry(new Text(""), 0, 5);
+                suggestion.addTerm(completionEntry);
+                int optionSize = randomIntBetween(1, suggestion.getSize());
+                float maxScore = randomIntBetween(suggestion.getSize(), (int) Float.MAX_VALUE);
+                for (int i = 0; i < optionSize; i++) {
+                    completionEntry.addOption(new CompletionSuggestion.Entry.Option(i, new Text(""), maxScore,
+                        Collections.emptyMap()));
+                    float dec = randomIntBetween(0, optionSize);
+                    if (dec <= maxScore) {
+                        maxScore -= dec;
+                    }
+                }
+                suggestion.setShardIndex(shardIndex);
+                shardSuggestion.add(suggestion);
+            }
+            querySearchResult.topDocs(topDocs, null);
+            querySearchResult.size(searchHitsSize);
+            querySearchResult.suggest(new Suggest(new ArrayList<>(shardSuggestion)));
+            querySearchResult.setShardIndex(shardIndex);
+            queryResults.set(shardIndex, querySearchResult);
+        }
+        return queryResults;
+    }
+
+    private int getTotalQueryHits(AtomicArray<SearchPhaseResult> results) {
+        int resultCount = 0;
+        for (SearchPhaseResult shardResult : results.asList()) {
+            resultCount += shardResult.queryResult().topDocs().totalHits;
+        }
+        return resultCount;
+    }
+
+    private Suggest reducedSuggest(AtomicArray<SearchPhaseResult> results) {
+        Map<String, List<Suggest.Suggestion<CompletionSuggestion.Entry>>> groupedSuggestion = new HashMap<>();
+        for (SearchPhaseResult entry : results.asList()) {
+            for (Suggest.Suggestion<?> suggestion : entry.queryResult().suggest()) {
+                List<Suggest.Suggestion<CompletionSuggestion.Entry>> suggests =
+                    groupedSuggestion.computeIfAbsent(suggestion.getName(), s -> new ArrayList<>());
+                suggests.add((Suggest.Suggestion<CompletionSuggestion.Entry>) suggestion);
+            }
+        }
+        return new Suggest(groupedSuggestion.values().stream().map(CompletionSuggestion::reduceTo)
+            .collect(Collectors.toList()));
+    }
+
+    private AtomicArray<SearchPhaseResult> generateFetchResults(int nShards, ScoreDoc[] mergedSearchDocs, Suggest mergedSuggest) {
+        AtomicArray<SearchPhaseResult> fetchResults = new AtomicArray<>(nShards);
+        for (int shardIndex = 0; shardIndex < nShards; shardIndex++) {
+            float maxScore = -1F;
+            SearchShardTarget shardTarget = new SearchShardTarget("", new Index("", ""), shardIndex, null);
+            FetchSearchResult fetchSearchResult = new FetchSearchResult(shardIndex, shardTarget);
+            List<SearchHit> searchHits = new ArrayList<>();
+            for (ScoreDoc scoreDoc : mergedSearchDocs) {
+                if (scoreDoc.shardIndex == shardIndex) {
+                    searchHits.add(new SearchHit(scoreDoc.doc, "", new Text(""), Collections.emptyMap()));
+                    if (scoreDoc.score > maxScore) {
+                        maxScore = scoreDoc.score;
+                    }
+                }
+            }
+            for (Suggest.Suggestion<?> suggestion : mergedSuggest) {
+                if (suggestion instanceof CompletionSuggestion) {
+                    for (CompletionSuggestion.Entry.Option option : ((CompletionSuggestion) suggestion).getOptions()) {
+                        ScoreDoc doc = option.getDoc();
+                        if (doc.shardIndex == shardIndex) {
+                            searchHits.add(new SearchHit(doc.doc, "", new Text(""), Collections.emptyMap()));
+                            if (doc.score > maxScore) {
+                                maxScore = doc.score;
+                            }
+                        }
+                    }
+                }
+            }
+            SearchHit[] hits = searchHits.toArray(new SearchHit[searchHits.size()]);
+            fetchSearchResult.hits(new SearchHits(hits, hits.length, maxScore));
+            fetchResults.set(shardIndex, fetchSearchResult);
+        }
+        return fetchResults;
+    }
+
+    public void testConsumer() {
+        int bufferSize = randomIntBetween(2, 3);
+        SearchRequest request = new SearchRequest();
+        request.source(new SearchSourceBuilder().aggregation(AggregationBuilders.avg("foo")));
+        request.setBatchedReduceSize(bufferSize);
+        InitialSearchPhase.ArraySearchPhaseResults<SearchPhaseResult> consumer = searchPhaseController.newSearchPhaseResults(request, 3);
+        QuerySearchResult result = new QuerySearchResult(0, new SearchShardTarget("node", new Index("a", "b"), 0, null));
+        result.topDocs(new TopDocs(0, new ScoreDoc[0], 0.0F), new DocValueFormat[0]);
+        InternalAggregations aggs = new InternalAggregations(Arrays.asList(new InternalMax("test", 1.0D, DocValueFormat.RAW,
+            Collections.emptyList(), Collections.emptyMap())));
+        result.aggregations(aggs);
+        result.setShardIndex(0);
+        consumer.consumeResult(result);
+
+        result = new QuerySearchResult(1, new SearchShardTarget("node", new Index("a", "b"), 0, null));
+        result.topDocs(new TopDocs(0, new ScoreDoc[0], 0.0F), new DocValueFormat[0]);
+        aggs = new InternalAggregations(Arrays.asList(new InternalMax("test", 3.0D, DocValueFormat.RAW,
+            Collections.emptyList(), Collections.emptyMap())));
+        result.aggregations(aggs);
+        result.setShardIndex(2);
+        consumer.consumeResult(result);
+
+        result = new QuerySearchResult(1, new SearchShardTarget("node", new Index("a", "b"), 0, null));
+        result.topDocs(new TopDocs(0, new ScoreDoc[0], 0.0F), new DocValueFormat[0]);
+        aggs = new InternalAggregations(Arrays.asList(new InternalMax("test", 2.0D, DocValueFormat.RAW,
+            Collections.emptyList(), Collections.emptyMap())));
+        result.aggregations(aggs);
+        result.setShardIndex(1);
+        consumer.consumeResult(result);
+        int numTotalReducePhases = 1;
+        if (bufferSize == 2) {
+            assertThat(consumer, instanceOf(SearchPhaseController.QueryPhaseResultConsumer.class));
+            assertEquals(1, ((SearchPhaseController.QueryPhaseResultConsumer)consumer).getNumReducePhases());
+            assertEquals(2, ((SearchPhaseController.QueryPhaseResultConsumer)consumer).getNumBuffered());
+            numTotalReducePhases++;
+        } else {
+            assertThat(consumer, not(instanceOf(SearchPhaseController.QueryPhaseResultConsumer.class)));
+        }
+
+        SearchPhaseController.ReducedQueryPhase reduce = consumer.reduce();
+        assertEquals(numTotalReducePhases, reduce.numReducePhases);
+        InternalMax max = (InternalMax) reduce.aggregations.asList().get(0);
+        assertEquals(3.0D, max.getValue(), 0.0D);
+    }
+
+    public void testConsumerConcurrently() throws InterruptedException {
+        int expectedNumResults = randomIntBetween(1, 100);
+        int bufferSize = randomIntBetween(2, 200);
+
+        SearchRequest request = new SearchRequest();
+        request.source(new SearchSourceBuilder().aggregation(AggregationBuilders.avg("foo")));
+        request.setBatchedReduceSize(bufferSize);
+        InitialSearchPhase.ArraySearchPhaseResults<SearchPhaseResult> consumer =
+            searchPhaseController.newSearchPhaseResults(request, expectedNumResults);
+        AtomicInteger max = new AtomicInteger();
+        Thread[] threads = new Thread[expectedNumResults];
+        for (int i = 0; i < expectedNumResults; i++) {
+            int id = i;
+            threads[i] = new Thread(() -> {
+                int number = randomIntBetween(1, 1000);
+                max.updateAndGet(prev -> Math.max(prev, number));
+                QuerySearchResult result = new QuerySearchResult(id, new SearchShardTarget("node", new Index("a", "b"), id, null));
+                result.topDocs(new TopDocs(1, new ScoreDoc[] {new ScoreDoc(0, number)}, number), new DocValueFormat[0]);
+                InternalAggregations aggs = new InternalAggregations(Arrays.asList(new InternalMax("test", (double) number,
+                    DocValueFormat.RAW, Collections.emptyList(), Collections.emptyMap())));
+                result.aggregations(aggs);
+                result.setShardIndex(id);
+                result.size(1);
+                consumer.consumeResult(result);
+
+            });
+            threads[i].start();
+        }
+        for (int i = 0; i < expectedNumResults; i++) {
+            threads[i].join();
+        }
+        SearchPhaseController.ReducedQueryPhase reduce = consumer.reduce();
+        InternalMax internalMax = (InternalMax) reduce.aggregations.asList().get(0);
+        assertEquals(max.get(), internalMax.getValue(), 0.0D);
+        assertEquals(1, reduce.scoreDocs.length);
+        assertEquals(max.get(), reduce.maxScore, 0.0f);
+        assertEquals(expectedNumResults, reduce.totalHits);
+        assertEquals(max.get(), reduce.scoreDocs[0].score, 0.0f);
+    }
+
+    public void testConsumerOnlyAggs() throws InterruptedException {
+        int expectedNumResults = randomIntBetween(1, 100);
+        int bufferSize = randomIntBetween(2, 200);
+        SearchRequest request = new SearchRequest();
+        request.source(new SearchSourceBuilder().aggregation(AggregationBuilders.avg("foo")).size(0));
+        request.setBatchedReduceSize(bufferSize);
+        InitialSearchPhase.ArraySearchPhaseResults<SearchPhaseResult> consumer =
+            searchPhaseController.newSearchPhaseResults(request, expectedNumResults);
+        AtomicInteger max = new AtomicInteger();
+        for (int i = 0; i < expectedNumResults; i++) {
+            int id = i;
+            int number = randomIntBetween(1, 1000);
+            max.updateAndGet(prev -> Math.max(prev, number));
+            QuerySearchResult result = new QuerySearchResult(id, new SearchShardTarget("node", new Index("a", "b"), id, null));
+            result.topDocs(new TopDocs(1, new ScoreDoc[0], number), new DocValueFormat[0]);
+            InternalAggregations aggs = new InternalAggregations(Arrays.asList(new InternalMax("test", (double) number,
+                DocValueFormat.RAW, Collections.emptyList(), Collections.emptyMap())));
+            result.aggregations(aggs);
+            result.setShardIndex(id);
+            result.size(1);
+            consumer.consumeResult(result);
+        }
+        SearchPhaseController.ReducedQueryPhase reduce = consumer.reduce();
+        InternalMax internalMax = (InternalMax) reduce.aggregations.asList().get(0);
+        assertEquals(max.get(), internalMax.getValue(), 0.0D);
+        assertEquals(0, reduce.scoreDocs.length);
+        assertEquals(max.get(), reduce.maxScore, 0.0f);
+        assertEquals(expectedNumResults, reduce.totalHits);
+    }
+
+
+    public void testConsumerOnlyHits() throws InterruptedException {
+        int expectedNumResults = randomIntBetween(1, 100);
+        int bufferSize = randomIntBetween(2, 200);
+        SearchRequest request = new SearchRequest();
+        if (randomBoolean()) {
+            request.source(new SearchSourceBuilder().size(randomIntBetween(1, 10)));
+        }
+        request.setBatchedReduceSize(bufferSize);
+        InitialSearchPhase.ArraySearchPhaseResults<SearchPhaseResult> consumer =
+            searchPhaseController.newSearchPhaseResults(request, expectedNumResults);
+        AtomicInteger max = new AtomicInteger();
+        for (int i = 0; i < expectedNumResults; i++) {
+            int id = i;
+            int number = randomIntBetween(1, 1000);
+            max.updateAndGet(prev -> Math.max(prev, number));
+            QuerySearchResult result = new QuerySearchResult(id, new SearchShardTarget("node", new Index("a", "b"), id, null));
+            result.topDocs(new TopDocs(1, new ScoreDoc[] {new ScoreDoc(0, number)}, number), new DocValueFormat[0]);
+            result.setShardIndex(id);
+            result.size(1);
+            consumer.consumeResult(result);
+        }
+        SearchPhaseController.ReducedQueryPhase reduce = consumer.reduce();
+        assertEquals(1, reduce.scoreDocs.length);
+        assertEquals(max.get(), reduce.maxScore, 0.0f);
+        assertEquals(expectedNumResults, reduce.totalHits);
+        assertEquals(max.get(), reduce.scoreDocs[0].score, 0.0f);
+    }
+
+
+    public void testNewSearchPhaseResults() {
+        for (int i = 0; i < 10; i++) {
+            int expectedNumResults = randomIntBetween(1, 10);
+            int bufferSize = randomIntBetween(2, 10);
+            SearchRequest request = new SearchRequest();
+            final boolean hasAggs;
+            if ((hasAggs = randomBoolean())) {
+                request.source(new SearchSourceBuilder().aggregation(AggregationBuilders.avg("foo")));
+            }
+            final boolean hasTopDocs;
+            if ((hasTopDocs = randomBoolean())) {
+                if (request.source() != null) {
+                    request.source().size(randomIntBetween(1, 100));
+                } // no source means size = 10
+            } else {
+                if (request.source() == null) {
+                    request.source(new SearchSourceBuilder().size(0));
+                } else {
+                    request.source().size(0);
+                }
+            }
+            request.setBatchedReduceSize(bufferSize);
+            InitialSearchPhase.ArraySearchPhaseResults<SearchPhaseResult> consumer
+                = searchPhaseController.newSearchPhaseResults(request, expectedNumResults);
+            if ((hasAggs || hasTopDocs) && expectedNumResults > bufferSize) {
+                assertThat("expectedNumResults: " + expectedNumResults + " bufferSize: " + bufferSize,
+                    consumer, instanceOf(SearchPhaseController.QueryPhaseResultConsumer.class));
+            } else {
+                assertThat("expectedNumResults: " + expectedNumResults + " bufferSize: " + bufferSize,
+                    consumer, not(instanceOf(SearchPhaseController.QueryPhaseResultConsumer.class)));
+            }
+        }
+    }
+
+    public void testReduceTopNWithFromOffset() {
+        SearchRequest request = new SearchRequest();
+        request.source(new SearchSourceBuilder().size(5).from(5));
+        request.setBatchedReduceSize(randomIntBetween(2, 4));
+        InitialSearchPhase.ArraySearchPhaseResults<SearchPhaseResult> consumer =
+            searchPhaseController.newSearchPhaseResults(request, 4);
+        int score = 100;
+        for (int i = 0; i < 4; i++) {
+            QuerySearchResult result = new QuerySearchResult(i, new SearchShardTarget("node", new Index("a", "b"), i, null));
+            ScoreDoc[] docs = new ScoreDoc[3];
+            for (int j = 0; j < docs.length; j++) {
+                docs[j] = new ScoreDoc(0, score--);
+            }
+            result.topDocs(new TopDocs(3, docs, docs[0].score), new DocValueFormat[0]);
+            result.setShardIndex(i);
+            result.size(5);
+            result.from(5);
+            consumer.consumeResult(result);
+        }
+        // 4*3 results = 12 we get result 5 to 10 here with from=5 and size=5
+
+        SearchPhaseController.ReducedQueryPhase reduce = consumer.reduce();
+        assertEquals(5, reduce.scoreDocs.length);
+        assertEquals(100.f, reduce.maxScore, 0.0f);
+        assertEquals(12, reduce.totalHits);
+        assertEquals(95.0f, reduce.scoreDocs[0].score, 0.0f);
+        assertEquals(94.0f, reduce.scoreDocs[1].score, 0.0f);
+        assertEquals(93.0f, reduce.scoreDocs[2].score, 0.0f);
+        assertEquals(92.0f, reduce.scoreDocs[3].score, 0.0f);
+        assertEquals(91.0f, reduce.scoreDocs[4].score, 0.0f);
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/search/SearchPhaseExecutionExceptionTests.java b/core/src/test/java/org/elasticsearch/action/search/SearchPhaseExecutionExceptionTests.java
new file mode 100644
index 0000000000000..c542f3b72f9f2
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/search/SearchPhaseExecutionExceptionTests.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.search;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.TimestampParsingException;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.shard.IndexShardClosedException;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.indices.InvalidIndexTemplateException;
+import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+import static java.util.Collections.singletonMap;
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.hamcrest.CoreMatchers.hasItem;
+import static org.hamcrest.Matchers.hasSize;
+
+public class SearchPhaseExecutionExceptionTests extends ESTestCase {
+
+    public void testToXContent() throws IOException {
+        SearchPhaseExecutionException exception = new SearchPhaseExecutionException("test", "all shards failed",
+                new ShardSearchFailure[]{
+                        new ShardSearchFailure(new ParsingException(1, 2, "foobar", null),
+                                new SearchShardTarget("node_1", new Index("foo", "_na_"), 0, null)),
+                        new ShardSearchFailure(new IndexShardClosedException(new ShardId(new Index("foo", "_na_"), 1)),
+                                new SearchShardTarget("node_2", new Index("foo", "_na_"), 1, null)),
+                        new ShardSearchFailure(new ParsingException(5, 7, "foobar", null),
+                                new SearchShardTarget("node_3", new Index("foo", "_na_"), 2, null)),
+                });
+
+        // Failures are grouped (by default)
+        assertEquals("{" +
+                "\"type\":\"search_phase_execution_exception\"," +
+                "\"reason\":\"all shards failed\"," +
+                "\"phase\":\"test\"," +
+                "\"grouped\":true," +
+                "\"failed_shards\":[" +
+                        "{" +
+                            "\"shard\":0," +
+                            "\"index\":\"foo\"," +
+                            "\"node\":\"node_1\"," +
+                            "\"reason\":{" +
+                                        "\"type\":\"parsing_exception\"," +
+                                        "\"reason\":\"foobar\"," +
+                                        "\"line\":1," +
+                                        "\"col\":2" +
+                            "}" +
+                        "}," +
+                        "{" +
+                            "\"shard\":1," +
+                            "\"index\":\"foo\"," +
+                            "\"node\":\"node_2\"," +
+                            "\"reason\":{" +
+                                        "\"type\":\"index_shard_closed_exception\"," +
+                                        "\"reason\":\"CurrentState[CLOSED] Closed\"," +
+                                        "\"index_uuid\":\"_na_\"," +
+                                        "\"shard\":\"1\"," +
+                                        "\"index\":\"foo\"" +
+                            "}" +
+                        "}" +
+                "]}", Strings.toString(exception));
+
+        // Failures are NOT grouped
+        ToXContent.MapParams params = new ToXContent.MapParams(singletonMap("group_shard_failures", "false"));
+        try (XContentBuilder builder = jsonBuilder()) {
+            builder.startObject();
+            exception.toXContent(builder, params);
+            builder.endObject();
+
+            assertEquals("{" +
+                    "\"type\":\"search_phase_execution_exception\"," +
+                    "\"reason\":\"all shards failed\"," +
+                    "\"phase\":\"test\"," +
+                    "\"grouped\":false," +
+                    "\"failed_shards\":[" +
+                            "{" +
+                                "\"shard\":0," +
+                                "\"index\":\"foo\"," +
+                                "\"node\":\"node_1\"," +
+                                "\"reason\":{" +
+                                            "\"type\":\"parsing_exception\"," +
+                                            "\"reason\":\"foobar\"," +
+                                            "\"line\":1," +
+                                            "\"col\":2" +
+                                "}" +
+                            "}," +
+                            "{" +
+                                "\"shard\":1," +
+                                "\"index\":\"foo\"," +
+                                "\"node\":\"node_2\"," +
+                                "\"reason\":{" +
+                                            "\"type\":\"index_shard_closed_exception\"," +
+                                            "\"reason\":\"CurrentState[CLOSED] Closed\"," +
+                                            "\"index_uuid\":\"_na_\"," +
+                                            "\"shard\":\"1\"," +
+                                            "\"index\":\"foo\"" +
+                                "}" +
+                            "}," +
+                            "{" +
+                                "\"shard\":2," +
+                                "\"index\":\"foo\"," +
+                                "\"node\":\"node_3\"," +
+                                "\"reason\":{" +
+                                            "\"type\":\"parsing_exception\"," +
+                                            "\"reason\":\"foobar\"," +
+                                            "\"line\":5," +
+                                            "\"col\":7" +
+                                "}" +
+                            "}" +
+                    "]}", builder.string());
+        }
+    }
+
+    public void testToAndFromXContent() throws IOException {
+        final XContent xContent = randomFrom(XContentType.values()).xContent();
+
+        ShardSearchFailure[] shardSearchFailures = new ShardSearchFailure[randomIntBetween(1, 5)];
+        for (int i = 0; i < shardSearchFailures.length; i++) {
+            Exception cause = randomFrom(
+                    new ParsingException(1, 2, "foobar", null),
+                    new InvalidIndexTemplateException("foo", "bar"),
+                    new TimestampParsingException("foo", null),
+                    new NullPointerException()
+            );
+            shardSearchFailures[i] = new  ShardSearchFailure(cause, new SearchShardTarget("node_" + i, new Index("test", "_na_"), i, null));
+        }
+
+        final String phase = randomFrom("query", "search", "other");
+        SearchPhaseExecutionException actual = new SearchPhaseExecutionException(phase, "unexpected failures", shardSearchFailures);
+
+        BytesReference exceptionBytes = toShuffledXContent(actual, xContent.type(), ToXContent.EMPTY_PARAMS, randomBoolean());
+
+        ElasticsearchException parsedException;
+        try (XContentParser parser = createParser(xContent, exceptionBytes)) {
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            parsedException = ElasticsearchException.fromXContent(parser);
+            assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());
+            assertNull(parser.nextToken());
+        }
+
+        assertNotNull(parsedException);
+        assertThat(parsedException.getHeaderKeys(), hasSize(0));
+        assertThat(parsedException.getMetadataKeys(), hasSize(1));
+        assertThat(parsedException.getMetadata("es.phase"), hasItem(phase));
+        // SearchPhaseExecutionException has no cause field
+        assertNull(parsedException.getCause());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/search/SearchResponseTests.java b/core/src/test/java/org/elasticsearch/action/search/SearchResponseTests.java
new file mode 100644
index 0000000000000..02c4964af3cc5
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/search/SearchResponseTests.java
@@ -0,0 +1,221 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.search;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.text.Text;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.rest.action.search.RestSearchAction;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.search.SearchHitsTests;
+import org.elasticsearch.search.aggregations.AggregationsTests;
+import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.internal.InternalSearchResponse;
+import org.elasticsearch.search.profile.SearchProfileShardResults;
+import org.elasticsearch.search.profile.SearchProfileShardResultsTests;
+import org.elasticsearch.search.suggest.Suggest;
+import org.elasticsearch.search.suggest.SuggestTests;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.InternalAggregationTestCase;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import static java.util.Collections.singletonMap;
+import static org.elasticsearch.test.XContentTestUtils.insertRandomFields;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
+
+public class SearchResponseTests extends ESTestCase {
+
+    private static final NamedXContentRegistry xContentRegistry;
+    static {
+        List<NamedXContentRegistry.Entry> namedXContents = new ArrayList<>(InternalAggregationTestCase.getDefaultNamedXContents());
+        namedXContents.addAll(SuggestTests.getDefaultNamedXContents());
+        xContentRegistry = new NamedXContentRegistry(namedXContents);
+    }
+
+    private AggregationsTests aggregationsTests = new AggregationsTests();
+
+    @Before
+    public void init() throws Exception {
+        aggregationsTests.init();
+    }
+
+    @After
+    public void cleanUp() throws Exception {
+        aggregationsTests.cleanUp();
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return xContentRegistry;
+    }
+
+    private SearchResponse createTestItem(ShardSearchFailure... shardSearchFailures) {
+        return createTestItem(false, shardSearchFailures);
+    }
+
+    /**
+     * This SearchResponse doesn't include SearchHits, Aggregations, Suggestions, ShardSearchFailures, SearchProfileShardResults
+     * to make it possible to only test properties of the SearchResponse itself
+     */
+    private SearchResponse createMinimalTestItem() {
+        return createTestItem(true);
+    }
+
+    /**
+     * if minimal is set, don't include search hits, aggregations, suggest etc... to make test simpler
+     */
+    private SearchResponse createTestItem(boolean minimal, ShardSearchFailure... shardSearchFailures) {
+        boolean timedOut = randomBoolean();
+        Boolean terminatedEarly = randomBoolean() ? null : randomBoolean();
+        int numReducePhases = randomIntBetween(1, 10);
+        long tookInMillis = randomNonNegativeLong();
+        int totalShards = randomIntBetween(1, Integer.MAX_VALUE);
+        int successfulShards = randomIntBetween(0, totalShards);
+        int skippedShards = randomIntBetween(0, totalShards);
+        InternalSearchResponse internalSearchResponse;
+        if (minimal == false) {
+            SearchHits hits = SearchHitsTests.createTestItem();
+            InternalAggregations aggregations = aggregationsTests.createTestInstance();
+            Suggest suggest = SuggestTests.createTestItem();
+            SearchProfileShardResults profileShardResults = SearchProfileShardResultsTests.createTestItem();
+            internalSearchResponse = new InternalSearchResponse(hits, aggregations, suggest, profileShardResults,
+                timedOut, terminatedEarly, numReducePhases);
+        } else {
+            internalSearchResponse = InternalSearchResponse.empty();
+        }
+        return new SearchResponse(internalSearchResponse, null, totalShards, successfulShards, skippedShards, tookInMillis,
+            shardSearchFailures);
+    }
+
+    /**
+     * the "_shard/total/failures" section makes it impossible to directly
+     * compare xContent, so we omit it here
+     */
+    public void testFromXContent() throws IOException {
+        doFromXContentTestWithRandomFields(createTestItem(), false);
+    }
+
+    /**
+     * This test adds random fields and objects to the xContent rendered out to
+     * ensure we can parse it back to be forward compatible with additions to
+     * the xContent. We test this with a "minimal" SearchResponse, adding random
+     * fields to SearchHits, Aggregations etc... is tested in their own tests
+     */
+    public void testFromXContentWithRandomFields() throws IOException {
+        doFromXContentTestWithRandomFields(createMinimalTestItem(), true);
+    }
+
+    private void doFromXContentTestWithRandomFields(SearchResponse response, boolean addRandomFields) throws IOException {
+        XContentType xcontentType = randomFrom(XContentType.values());
+        boolean humanReadable = randomBoolean();
+        final ToXContent.Params params = new ToXContent.MapParams(singletonMap(RestSearchAction.TYPED_KEYS_PARAM, "true"));
+        BytesReference originalBytes = toShuffledXContent(response, xcontentType, params, humanReadable);
+        BytesReference mutated;
+        if (addRandomFields) {
+            mutated = insertRandomFields(xcontentType, originalBytes, null, random());
+        } else {
+            mutated = originalBytes;
+        }
+        try (XContentParser parser = createParser(xcontentType.xContent(), mutated)) {
+            SearchResponse parsed = SearchResponse.fromXContent(parser);
+            assertToXContentEquivalent(originalBytes, XContentHelper.toXContent(parsed, xcontentType, params, humanReadable), xcontentType);
+            assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());
+            assertNull(parser.nextToken());
+        }
+    }
+
+    /**
+     * The "_shard/total/failures" section makes if impossible to directly compare xContent, because
+     * the failures in the parsed SearchResponse are wrapped in an extra ElasticSearchException on the client side.
+     * Because of this, in this special test case we compare the "top level" fields for equality
+     * and the subsections xContent equivalence independently
+     */
+    public void testFromXContentWithFailures() throws IOException {
+        int numFailures = randomIntBetween(1, 5);
+        ShardSearchFailure[] failures = new ShardSearchFailure[numFailures];
+        for (int i = 0; i < failures.length; i++) {
+            failures[i] = ShardSearchFailureTests.createTestItem();
+        }
+        SearchResponse response = createTestItem(failures);
+        XContentType xcontentType = randomFrom(XContentType.values());
+        final ToXContent.Params params = new ToXContent.MapParams(singletonMap(RestSearchAction.TYPED_KEYS_PARAM, "true"));
+        BytesReference originalBytes = toShuffledXContent(response, xcontentType, params, randomBoolean());
+        try (XContentParser parser = createParser(xcontentType.xContent(), originalBytes)) {
+            SearchResponse parsed = SearchResponse.fromXContent(parser);
+            for (int i = 0; i < parsed.getShardFailures().length; i++) {
+                ShardSearchFailure parsedFailure = parsed.getShardFailures()[i];
+                ShardSearchFailure originalFailure = failures[i];
+                assertEquals(originalFailure.index(), parsedFailure.index());
+                assertEquals(originalFailure.shard().getNodeId(), parsedFailure.shard().getNodeId());
+                assertEquals(originalFailure.shardId(), parsedFailure.shardId());
+                String originalMsg = originalFailure.getCause().getMessage();
+                assertEquals(parsedFailure.getCause().getMessage(), "Elasticsearch exception [type=parsing_exception, reason=" +
+                        originalMsg + "]");
+                String nestedMsg = originalFailure.getCause().getCause().getMessage();
+                assertEquals(parsedFailure.getCause().getCause().getMessage(),
+                        "Elasticsearch exception [type=illegal_argument_exception, reason=" + nestedMsg + "]");
+            }
+            assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());
+            assertNull(parser.nextToken());
+        }
+    }
+
+    public void testToXContent() {
+        SearchHit hit = new SearchHit(1, "id1", new Text("type"), Collections.emptyMap());
+        hit.score(2.0f);
+        SearchHit[] hits = new SearchHit[] { hit };
+        SearchResponse response = new SearchResponse(
+                new InternalSearchResponse(new SearchHits(hits, 100, 1.5f), null, null, null, false, null, 1), null, 0, 0, 0, 0,
+                new ShardSearchFailure[0]);
+        StringBuilder expectedString = new StringBuilder();
+        expectedString.append("{");
+        {
+            expectedString.append("\"took\":0,");
+            expectedString.append("\"timed_out\":false,");
+            expectedString.append("\"_shards\":");
+            {
+                expectedString.append("{\"total\":0,");
+                expectedString.append("\"successful\":0,");
+                expectedString.append("\"skipped\":0,");
+                expectedString.append("\"failed\":0},");
+            }
+            expectedString.append("\"hits\":");
+            {
+                expectedString.append("{\"total\":100,");
+                expectedString.append("\"max_score\":1.5,");
+                expectedString.append("\"hits\":[{\"_type\":\"type\",\"_id\":\"id1\",\"_score\":2.0}]}");
+            }
+        }
+        expectedString.append("}");
+        assertEquals(expectedString.toString(), Strings.toString(response));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/search/SearchScrollAsyncActionTests.java b/core/src/test/java/org/elasticsearch/action/search/SearchScrollAsyncActionTests.java
new file mode 100644
index 0000000000000..df408be34b8e8
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/search/SearchScrollAsyncActionTests.java
@@ -0,0 +1,460 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.search;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.transport.LocalTransportAddress;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.AtomicArray;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.search.Scroll;
+import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.search.internal.InternalScrollSearchRequest;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.transport.Transport;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.BiFunction;
+
+public class SearchScrollAsyncActionTests extends ESTestCase {
+
+    public void testSendRequestsToNodes() throws InterruptedException {
+
+        ParsedScrollId scrollId = getParsedScrollId(
+            new ScrollIdForNode(null, "node1", 1),
+            new ScrollIdForNode(null, "node2", 2),
+            new ScrollIdForNode(null, "node3", 17),
+            new ScrollIdForNode(null, "node1", 0),
+            new ScrollIdForNode(null, "node3", 0));
+        DiscoveryNodes discoveryNodes = DiscoveryNodes.builder()
+            .add(new DiscoveryNode("node1", LocalTransportAddress.buildUnique(), Version.CURRENT))
+            .add(new DiscoveryNode("node2", LocalTransportAddress.buildUnique(), Version.CURRENT))
+            .add(new DiscoveryNode("node3", LocalTransportAddress.buildUnique(), Version.CURRENT)).build();
+
+        AtomicArray<SearchAsyncActionTests.TestSearchPhaseResult> results = new AtomicArray<>(scrollId.getContext().length);
+        SearchScrollRequest request = new SearchScrollRequest();
+        request.scroll(new Scroll(TimeValue.timeValueMinutes(1)));
+        CountDownLatch latch = new CountDownLatch(1);
+        AtomicInteger movedCounter = new AtomicInteger(0);
+        SearchScrollAsyncAction<SearchAsyncActionTests.TestSearchPhaseResult> action =
+            new SearchScrollAsyncAction<SearchAsyncActionTests.TestSearchPhaseResult>(scrollId, logger, discoveryNodes, dummyListener(),
+                null, request, null)
+            {
+                @Override
+                protected void executeInitialPhase(Transport.Connection connection, InternalScrollSearchRequest internalRequest,
+                                                   SearchActionListener<SearchAsyncActionTests.TestSearchPhaseResult> searchActionListener)
+                {
+                    new Thread(() -> {
+                        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult =
+                            new SearchAsyncActionTests.TestSearchPhaseResult(internalRequest.id(), connection.getNode());
+                        testSearchPhaseResult.setSearchShardTarget(new SearchShardTarget(connection.getNode().getId(),
+                            new Index("test", "_na_"), 1, null));
+                        searchActionListener.onResponse(testSearchPhaseResult);
+                    }).start();
+                }
+
+                @Override
+                protected Transport.Connection getConnection(String clusterAlias, DiscoveryNode node) {
+                    return new SearchAsyncActionTests.MockConnection(node);
+                }
+
+                @Override
+                protected SearchPhase moveToNextPhase(BiFunction<String, String, DiscoveryNode> clusterNodeLookup) {
+                    assertEquals(1, movedCounter.incrementAndGet());
+                    return new SearchPhase("test") {
+                        @Override
+                        public void run() throws IOException {
+                            latch.countDown();
+                        }
+                    };
+                }
+
+                @Override
+                protected void onFirstPhaseResult(int shardId, SearchAsyncActionTests.TestSearchPhaseResult result) {
+                    results.setOnce(shardId, result);
+                }
+            };
+
+        action.run();
+        latch.await();
+        ShardSearchFailure[] shardSearchFailures = action.buildShardFailures();
+        assertEquals(0, shardSearchFailures.length);
+        ScrollIdForNode[] context = scrollId.getContext();
+        for (int i = 0; i < results.length(); i++) {
+            assertNotNull(results.get(i));
+            assertEquals(context[i].getScrollId(), results.get(i).getRequestId());
+            assertEquals(context[i].getNode(), results.get(i).node.getId());
+        }
+    }
+
+    public void testFailNextPhase() throws InterruptedException {
+
+        ParsedScrollId scrollId = getParsedScrollId(
+            new ScrollIdForNode(null, "node1", 1),
+            new ScrollIdForNode(null, "node2", 2),
+            new ScrollIdForNode(null, "node3", 17),
+            new ScrollIdForNode(null, "node1", 0),
+            new ScrollIdForNode(null, "node3", 0));
+        DiscoveryNodes discoveryNodes = DiscoveryNodes.builder()
+            .add(new DiscoveryNode("node1", LocalTransportAddress.buildUnique(), Version.CURRENT))
+            .add(new DiscoveryNode("node2", LocalTransportAddress.buildUnique(), Version.CURRENT))
+            .add(new DiscoveryNode("node3", LocalTransportAddress.buildUnique(), Version.CURRENT)).build();
+
+        AtomicArray<SearchAsyncActionTests.TestSearchPhaseResult> results = new AtomicArray<>(scrollId.getContext().length);
+        SearchScrollRequest request = new SearchScrollRequest();
+        request.scroll(new Scroll(TimeValue.timeValueMinutes(1)));
+        CountDownLatch latch = new CountDownLatch(1);
+        AtomicInteger movedCounter = new AtomicInteger(0);
+        ActionListener listener = new ActionListener() {
+            @Override
+            public void onResponse(Object o) {
+                try {
+                    fail("got a result");
+                } finally {
+                    latch.countDown();
+                }
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                try {
+                    assertTrue(e instanceof SearchPhaseExecutionException);
+                    SearchPhaseExecutionException ex = (SearchPhaseExecutionException) e;
+                    assertEquals("BOOM", ex.getCause().getMessage());
+                    assertEquals("TEST_PHASE", ex.getPhaseName());
+                    assertEquals("Phase failed", ex.getMessage());
+                } finally {
+                    latch.countDown();
+                }
+            }
+        };
+        SearchScrollAsyncAction<SearchAsyncActionTests.TestSearchPhaseResult> action =
+            new SearchScrollAsyncAction<SearchAsyncActionTests.TestSearchPhaseResult>(scrollId, logger, discoveryNodes, listener, null,
+                request, null) {
+                @Override
+                protected void executeInitialPhase(Transport.Connection connection, InternalScrollSearchRequest internalRequest,
+                                                   SearchActionListener<SearchAsyncActionTests.TestSearchPhaseResult> searchActionListener)
+                {
+                    new Thread(() -> {
+                        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult =
+                            new SearchAsyncActionTests.TestSearchPhaseResult(internalRequest.id(), connection.getNode());
+                        testSearchPhaseResult.setSearchShardTarget(new SearchShardTarget(connection.getNode().getId(),
+                            new Index("test", "_na_"), 1, null));
+                        searchActionListener.onResponse(testSearchPhaseResult);
+                    }).start();
+                }
+
+                @Override
+                protected Transport.Connection getConnection(String clusterAlias, DiscoveryNode node) {
+                    return new SearchAsyncActionTests.MockConnection(node);
+                }
+
+                @Override
+                protected SearchPhase moveToNextPhase(BiFunction<String, String, DiscoveryNode> clusterNodeLookup) {
+                    assertEquals(1, movedCounter.incrementAndGet());
+                    return new SearchPhase("TEST_PHASE") {
+                        @Override
+                        public void run() throws IOException {
+                            throw new IllegalArgumentException("BOOM");
+                        }
+                    };
+                }
+
+                @Override
+                protected void onFirstPhaseResult(int shardId, SearchAsyncActionTests.TestSearchPhaseResult result) {
+                    results.setOnce(shardId, result);
+                }
+            };
+
+        action.run();
+        latch.await();
+        ShardSearchFailure[] shardSearchFailures = action.buildShardFailures();
+        assertEquals(0, shardSearchFailures.length);
+        ScrollIdForNode[] context = scrollId.getContext();
+        for (int i = 0; i < results.length(); i++) {
+            assertNotNull(results.get(i));
+            assertEquals(context[i].getScrollId(), results.get(i).getRequestId());
+            assertEquals(context[i].getNode(), results.get(i).node.getId());
+        }
+    }
+
+    public void testNodeNotAvailable() throws InterruptedException {
+        ParsedScrollId scrollId = getParsedScrollId(
+            new ScrollIdForNode(null, "node1", 1),
+            new ScrollIdForNode(null, "node2", 2),
+            new ScrollIdForNode(null, "node3", 17),
+            new ScrollIdForNode(null, "node1", 0),
+            new ScrollIdForNode(null, "node3", 0));
+        // node2 is not available
+        DiscoveryNodes discoveryNodes = DiscoveryNodes.builder()
+            .add(new DiscoveryNode("node1", LocalTransportAddress.buildUnique(), Version.CURRENT))
+            .add(new DiscoveryNode("node3", LocalTransportAddress.buildUnique(), Version.CURRENT)).build();
+
+        AtomicArray<SearchAsyncActionTests.TestSearchPhaseResult> results = new AtomicArray<>(scrollId.getContext().length);
+        SearchScrollRequest request = new SearchScrollRequest();
+        request.scroll(new Scroll(TimeValue.timeValueMinutes(1)));
+        CountDownLatch latch = new CountDownLatch(1);
+        AtomicInteger movedCounter = new AtomicInteger(0);
+        SearchScrollAsyncAction<SearchAsyncActionTests.TestSearchPhaseResult> action =
+            new SearchScrollAsyncAction<SearchAsyncActionTests.TestSearchPhaseResult>(scrollId, logger, discoveryNodes, dummyListener()
+                , null, request, null)
+            {
+                @Override
+                protected void executeInitialPhase(Transport.Connection connection, InternalScrollSearchRequest internalRequest,
+                                                   SearchActionListener<SearchAsyncActionTests.TestSearchPhaseResult> searchActionListener)
+                {
+                    try {
+                        assertNotEquals("node2 is not available", "node2", connection.getNode().getId());
+                    } catch (NullPointerException e) {
+                        logger.warn(e);
+                    }
+                    new Thread(() -> {
+                        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult =
+                            new SearchAsyncActionTests.TestSearchPhaseResult(internalRequest.id(), connection.getNode());
+                        testSearchPhaseResult.setSearchShardTarget(new SearchShardTarget(connection.getNode().getId(),
+                            new Index("test", "_na_"), 1, null));
+                        searchActionListener.onResponse(testSearchPhaseResult);
+                    }).start();
+                }
+
+                @Override
+                protected Transport.Connection getConnection(String clusterAlias, DiscoveryNode node) {
+                    return new SearchAsyncActionTests.MockConnection(node);
+                }
+
+                @Override
+                protected SearchPhase moveToNextPhase(BiFunction<String, String, DiscoveryNode> clusterNodeLookup) {
+                    assertEquals(1, movedCounter.incrementAndGet());
+                    return new SearchPhase("test") {
+                        @Override
+                        public void run() throws IOException {
+                            latch.countDown();
+                        }
+                    };
+                }
+
+                @Override
+                protected void onFirstPhaseResult(int shardId, SearchAsyncActionTests.TestSearchPhaseResult result) {
+                    results.setOnce(shardId, result);
+                }
+            };
+
+        action.run();
+        latch.await();
+        ShardSearchFailure[] shardSearchFailures = action.buildShardFailures();
+        assertEquals(1, shardSearchFailures.length);
+        assertEquals("IllegalStateException[node [node2] is not available]", shardSearchFailures[0].reason());
+
+        ScrollIdForNode[] context = scrollId.getContext();
+        for (int i = 0; i < results.length(); i++) {
+            if (context[i].getNode().equals("node2")) {
+                assertNull(results.get(i));
+            } else {
+                assertNotNull(results.get(i));
+                assertEquals(context[i].getScrollId(), results.get(i).getRequestId());
+                assertEquals(context[i].getNode(), results.get(i).node.getId());
+            }
+        }
+    }
+
+    public void testShardFailures() throws InterruptedException {
+        ParsedScrollId scrollId = getParsedScrollId(
+            new ScrollIdForNode(null, "node1", 1),
+            new ScrollIdForNode(null, "node2", 2),
+            new ScrollIdForNode(null, "node3", 17),
+            new ScrollIdForNode(null, "node1", 0),
+            new ScrollIdForNode(null, "node3", 0));
+        DiscoveryNodes discoveryNodes = DiscoveryNodes.builder()
+            .add(new DiscoveryNode("node1", LocalTransportAddress.buildUnique(), Version.CURRENT))
+            .add(new DiscoveryNode("node2", LocalTransportAddress.buildUnique(), Version.CURRENT))
+            .add(new DiscoveryNode("node3", LocalTransportAddress.buildUnique(), Version.CURRENT)).build();
+
+        AtomicArray<SearchAsyncActionTests.TestSearchPhaseResult> results = new AtomicArray<>(scrollId.getContext().length);
+        SearchScrollRequest request = new SearchScrollRequest();
+        request.scroll(new Scroll(TimeValue.timeValueMinutes(1)));
+        CountDownLatch latch = new CountDownLatch(1);
+        AtomicInteger movedCounter = new AtomicInteger(0);
+        SearchScrollAsyncAction<SearchAsyncActionTests.TestSearchPhaseResult> action =
+            new SearchScrollAsyncAction<SearchAsyncActionTests.TestSearchPhaseResult>(scrollId, logger, discoveryNodes, dummyListener(),
+                null, request, null)
+            {
+                @Override
+                protected void executeInitialPhase(Transport.Connection connection, InternalScrollSearchRequest internalRequest,
+                                                   SearchActionListener<SearchAsyncActionTests.TestSearchPhaseResult> searchActionListener)
+                {
+                    new Thread(() -> {
+                        if (internalRequest.id() == 17) {
+                            searchActionListener.onFailure(new IllegalArgumentException("BOOM on shard"));
+                        } else {
+                            SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult =
+                                new SearchAsyncActionTests.TestSearchPhaseResult(internalRequest.id(), connection.getNode());
+                            testSearchPhaseResult.setSearchShardTarget(new SearchShardTarget(connection.getNode().getId(),
+                                new Index("test", "_na_"), 1, null));
+                            searchActionListener.onResponse(testSearchPhaseResult);
+                        }
+                    }).start();
+                }
+
+                @Override
+                protected Transport.Connection getConnection(String clusterAlias, DiscoveryNode node) {
+                    return new SearchAsyncActionTests.MockConnection(node);
+                }
+
+                @Override
+                protected SearchPhase moveToNextPhase(BiFunction<String, String, DiscoveryNode> clusterNodeLookup) {
+                    assertEquals(1, movedCounter.incrementAndGet());
+                    return new SearchPhase("test") {
+                        @Override
+                        public void run() throws IOException {
+                            latch.countDown();
+                        }
+                    };
+                }
+
+                @Override
+                protected void onFirstPhaseResult(int shardId, SearchAsyncActionTests.TestSearchPhaseResult result) {
+                    results.setOnce(shardId, result);
+                }
+            };
+
+        action.run();
+        latch.await();
+        ShardSearchFailure[] shardSearchFailures = action.buildShardFailures();
+        assertEquals(1, shardSearchFailures.length);
+        assertEquals("IllegalArgumentException[BOOM on shard]", shardSearchFailures[0].reason());
+
+        ScrollIdForNode[] context = scrollId.getContext();
+        for (int i = 0; i < results.length(); i++) {
+            if (context[i].getScrollId() == 17) {
+                assertNull(results.get(i));
+            } else {
+                assertNotNull(results.get(i));
+                assertEquals(context[i].getScrollId(), results.get(i).getRequestId());
+                assertEquals(context[i].getNode(), results.get(i).node.getId());
+            }
+        }
+    }
+
+    public void testAllShardsFailed() throws InterruptedException {
+        ParsedScrollId scrollId = getParsedScrollId(
+            new ScrollIdForNode(null, "node1", 1),
+            new ScrollIdForNode(null, "node2", 2),
+            new ScrollIdForNode(null, "node3", 17),
+            new ScrollIdForNode(null, "node1", 0),
+            new ScrollIdForNode(null, "node3", 0));
+        DiscoveryNodes discoveryNodes = DiscoveryNodes.builder()
+            .add(new DiscoveryNode("node1", LocalTransportAddress.buildUnique(), Version.CURRENT))
+            .add(new DiscoveryNode("node2", LocalTransportAddress.buildUnique(), Version.CURRENT))
+            .add(new DiscoveryNode("node3", LocalTransportAddress.buildUnique(), Version.CURRENT)).build();
+
+        AtomicArray<SearchAsyncActionTests.TestSearchPhaseResult> results = new AtomicArray<>(scrollId.getContext().length);
+        SearchScrollRequest request = new SearchScrollRequest();
+        request.scroll(new Scroll(TimeValue.timeValueMinutes(1)));
+        CountDownLatch latch = new CountDownLatch(1);
+        ActionListener listener = new ActionListener() {
+            @Override
+            public void onResponse(Object o) {
+                try {
+                    fail("got a result");
+                } finally {
+                    latch.countDown();
+                }
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                try {
+                    assertTrue(e instanceof SearchPhaseExecutionException);
+                    SearchPhaseExecutionException ex = (SearchPhaseExecutionException) e;
+                    assertEquals("BOOM on shard", ex.getCause().getMessage());
+                    assertEquals("query", ex.getPhaseName());
+                    assertEquals("all shards failed", ex.getMessage());
+                } finally {
+                    latch.countDown();
+                }
+            }
+        };
+        SearchScrollAsyncAction<SearchAsyncActionTests.TestSearchPhaseResult> action =
+            new SearchScrollAsyncAction<SearchAsyncActionTests.TestSearchPhaseResult>(scrollId, logger, discoveryNodes, listener, null,
+                request, null) {
+                @Override
+                protected void executeInitialPhase(Transport.Connection connection, InternalScrollSearchRequest internalRequest,
+                                                   SearchActionListener<SearchAsyncActionTests.TestSearchPhaseResult> searchActionListener)
+                {
+                    new Thread(() -> searchActionListener.onFailure(new IllegalArgumentException("BOOM on shard"))).start();
+                }
+
+                @Override
+                protected Transport.Connection getConnection(String clusterAlias, DiscoveryNode node) {
+                    return new SearchAsyncActionTests.MockConnection(node);
+                }
+
+                @Override
+                protected SearchPhase moveToNextPhase(BiFunction<String, String, DiscoveryNode> clusterNodeLookup) {
+                   fail("don't move all shards failed");
+                   return null;
+                }
+
+                @Override
+                protected void onFirstPhaseResult(int shardId, SearchAsyncActionTests.TestSearchPhaseResult result) {
+                    results.setOnce(shardId, result);
+                }
+            };
+
+        action.run();
+        latch.await();
+        ScrollIdForNode[] context = scrollId.getContext();
+
+        ShardSearchFailure[] shardSearchFailures = action.buildShardFailures();
+        assertEquals(context.length, shardSearchFailures.length);
+        assertEquals("IllegalArgumentException[BOOM on shard]", shardSearchFailures[0].reason());
+
+        for (int i = 0; i < results.length(); i++) {
+            assertNull(results.get(i));
+        }
+    }
+
+    private static ParsedScrollId getParsedScrollId(ScrollIdForNode... idsForNodes) {
+        List<ScrollIdForNode> scrollIdForNodes = Arrays.asList(idsForNodes);
+        Collections.shuffle(scrollIdForNodes, random());
+        return new ParsedScrollId("", "test", scrollIdForNodes.toArray(new ScrollIdForNode[0]));
+    }
+
+    private ActionListener<SearchResponse> dummyListener() {
+        return new ActionListener<SearchResponse>() {
+            @Override
+            public void onResponse(SearchResponse response) {
+                fail("dummy");
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                throw new AssertionError(e);
+            }
+        };
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/search/SearchScrollRequestTests.java b/core/src/test/java/org/elasticsearch/action/search/SearchScrollRequestTests.java
index 8e91b68d092c4..6ec9f95f489de 100644
--- a/core/src/test/java/org/elasticsearch/action/search/SearchScrollRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/search/SearchScrollRequestTests.java
@@ -19,13 +19,26 @@
 
 package org.elasticsearch.action.search;
 
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.search.internal.InternalScrollSearchRequest;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
 
+import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
+import static org.hamcrest.Matchers.startsWith;
+
 public class SearchScrollRequestTests extends ESTestCase {
 
     public void testSerialization() throws Exception {
@@ -57,57 +70,66 @@ public void testInternalScrollSearchRequestSerialization() throws IOException {
         }
     }
 
-    public void testEqualsAndHashcode() {
-        SearchScrollRequest firstSearchScrollRequest = createSearchScrollRequest();
-        assertNotNull("search scroll request is equal to null", firstSearchScrollRequest);
-        assertNotEquals("search scroll request  is equal to incompatible type", firstSearchScrollRequest, "");
-        assertEquals("search scroll request is not equal to self", firstSearchScrollRequest, firstSearchScrollRequest);
-        assertEquals("same source builder's hashcode returns different values if called multiple times",
-                firstSearchScrollRequest.hashCode(), firstSearchScrollRequest.hashCode());
-
-        SearchScrollRequest secondSearchScrollRequest = copyRequest(firstSearchScrollRequest);
-        assertEquals("search scroll request  is not equal to self", secondSearchScrollRequest, secondSearchScrollRequest);
-        assertEquals("search scroll request is not equal to its copy", firstSearchScrollRequest, secondSearchScrollRequest);
-        assertEquals("search scroll request is not symmetric", secondSearchScrollRequest, firstSearchScrollRequest);
-        assertEquals("search scroll request copy's hashcode is different from original hashcode",
-                firstSearchScrollRequest.hashCode(), secondSearchScrollRequest.hashCode());
-
-        SearchScrollRequest thirdSearchScrollRequest = copyRequest(secondSearchScrollRequest);
-        assertEquals("search scroll request is not equal to self", thirdSearchScrollRequest, thirdSearchScrollRequest);
-        assertEquals("search scroll request is not equal to its copy", secondSearchScrollRequest, thirdSearchScrollRequest);
-        assertEquals("search scroll request copy's hashcode is different from original hashcode",
-                secondSearchScrollRequest.hashCode(), thirdSearchScrollRequest.hashCode());
-        assertEquals("equals is not transitive", firstSearchScrollRequest, thirdSearchScrollRequest);
-        assertEquals("search scroll request copy's hashcode is different from original hashcode",
-                firstSearchScrollRequest.hashCode(), thirdSearchScrollRequest.hashCode());
-        assertEquals("equals is not symmetric", thirdSearchScrollRequest, secondSearchScrollRequest);
-        assertEquals("equals is not symmetric", thirdSearchScrollRequest, firstSearchScrollRequest);
-
-        boolean changed = false;
+    public void testFromXContent() throws Exception {
+        SearchScrollRequest searchScrollRequest = new SearchScrollRequest();
         if (randomBoolean()) {
-            secondSearchScrollRequest.scrollId(randomAsciiOfLengthBetween(3, 10));
-            if (secondSearchScrollRequest.scrollId().equals(firstSearchScrollRequest.scrollId()) == false) {
-                changed = true;
-            }
+            //test that existing values get overridden
+            searchScrollRequest = createSearchScrollRequest();
         }
-        if (randomBoolean()) {
-            secondSearchScrollRequest.scroll(randomPositiveTimeValue());
-            if (secondSearchScrollRequest.scroll().equals(firstSearchScrollRequest.scroll()) == false) {
-                changed = true;
-            }
+        try (XContentParser parser = createParser(XContentFactory.jsonBuilder()
+                .startObject()
+                .field("scroll_id", "SCROLL_ID")
+                .field("scroll", "1m")
+                .endObject())) {
+            searchScrollRequest.fromXContent(parser);
         }
-        
-        if (changed) {
-            assertNotEquals(firstSearchScrollRequest, secondSearchScrollRequest);
-            assertNotEquals(firstSearchScrollRequest.hashCode(), secondSearchScrollRequest.hashCode());
-        } else {
-            assertEquals(firstSearchScrollRequest, secondSearchScrollRequest);
-            assertEquals(firstSearchScrollRequest.hashCode(), secondSearchScrollRequest.hashCode());
+        assertEquals("SCROLL_ID", searchScrollRequest.scrollId());
+        assertEquals(TimeValue.parseTimeValue("1m", null, "scroll"), searchScrollRequest.scroll().keepAlive());
+    }
+
+    public void testFromXContentWithUnknownParamThrowsException() throws Exception {
+        SearchScrollRequest searchScrollRequest = new SearchScrollRequest();
+        XContentParser invalidContent = createParser(XContentFactory.jsonBuilder()
+                .startObject()
+                .field("scroll_id", "value_2")
+                .field("unknown", "keyword")
+                .endObject());
+
+        Exception e = expectThrows(IllegalArgumentException.class,
+                () -> searchScrollRequest.fromXContent(invalidContent));
+        assertThat(e.getMessage(), startsWith("Unknown parameter [unknown]"));
+    }
+
+    public void testToXContent() throws IOException {
+        SearchScrollRequest searchScrollRequest = new SearchScrollRequest();
+        searchScrollRequest.scrollId("SCROLL_ID");
+        searchScrollRequest.scroll("1m");
+        try (XContentBuilder builder = JsonXContent.contentBuilder()) {
+            searchScrollRequest.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            assertEquals("{\"scroll_id\":\"SCROLL_ID\",\"scroll\":\"1m\"}", builder.string());
         }
     }
-    
+
+    public void testToAndFromXContent() throws IOException {
+        XContentType xContentType = randomFrom(XContentType.values());
+        boolean humanReadable = randomBoolean();
+        SearchScrollRequest originalRequest = createSearchScrollRequest();
+        BytesReference originalBytes = toShuffledXContent(originalRequest, xContentType, ToXContent.EMPTY_PARAMS, humanReadable);
+        SearchScrollRequest parsedRequest = new SearchScrollRequest();
+        try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) {
+            parsedRequest.fromXContent(parser);
+        }
+        assertEquals(originalRequest, parsedRequest);
+        BytesReference parsedBytes = XContentHelper.toXContent(parsedRequest, xContentType, humanReadable);
+        assertToXContentEquivalent(originalBytes, parsedBytes, xContentType);
+    }
+
+    public void testEqualsAndHashcode() {
+        checkEqualsAndHashCode(createSearchScrollRequest(), SearchScrollRequestTests::copyRequest, SearchScrollRequestTests::mutate);
+    }
+
     public static SearchScrollRequest createSearchScrollRequest() {
-        SearchScrollRequest searchScrollRequest = new SearchScrollRequest(randomAsciiOfLengthBetween(3, 10));
+        SearchScrollRequest searchScrollRequest = new SearchScrollRequest(randomAlphaOfLengthBetween(3, 10));
         searchScrollRequest.scroll(randomPositiveTimeValue());
         return searchScrollRequest;
     }
@@ -118,4 +140,13 @@ private static SearchScrollRequest copyRequest(SearchScrollRequest searchScrollR
         result.scroll(searchScrollRequest.scroll());
         return result;
     }
+
+    private static SearchScrollRequest mutate(SearchScrollRequest original) {
+        SearchScrollRequest copy = copyRequest(original);
+        if (randomBoolean()) {
+            return copy.scrollId(original.scrollId() + "xyz");
+        } else {
+            return copy.scroll(new TimeValue(original.scroll().keepAlive().getMillis() + 1));
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/search/ShardSearchFailureTests.java b/core/src/test/java/org/elasticsearch/action/search/ShardSearchFailureTests.java
new file mode 100644
index 0000000000000..9a8c0b1feb1d3
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/search/ShardSearchFailureTests.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.search;
+
+import org.elasticsearch.action.OriginalIndices;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+import static org.elasticsearch.test.XContentTestUtils.insertRandomFields;
+
+public class ShardSearchFailureTests extends ESTestCase {
+
+    public static ShardSearchFailure createTestItem() {
+        String randomMessage = randomAlphaOfLengthBetween(3, 20);
+        Exception ex = new ParsingException(0, 0, randomMessage , new IllegalArgumentException("some bad argument"));
+        String nodeId = randomAlphaOfLengthBetween(5, 10);
+        String indexName = randomAlphaOfLengthBetween(5, 10);
+        String indexUuid = randomAlphaOfLengthBetween(5, 10);
+        int shardId = randomInt();
+        return new ShardSearchFailure(ex,
+                new SearchShardTarget(nodeId, new ShardId(new Index(indexName, indexUuid), shardId), null, null));
+    }
+
+    public void testFromXContent() throws IOException {
+        doFromXContentTestWithRandomFields(false);
+    }
+
+    /**
+     * This test adds random fields and objects to the xContent rendered out to
+     * ensure we can parse it back to be forward compatible with additions to
+     * the xContent
+     */
+    public void testFromXContentWithRandomFields() throws IOException {
+        doFromXContentTestWithRandomFields(true);
+    }
+
+    private void doFromXContentTestWithRandomFields(boolean addRandomFields) throws IOException {
+        ShardSearchFailure response = createTestItem();
+        XContentType xContentType = randomFrom(XContentType.values());
+        boolean humanReadable = randomBoolean();
+        BytesReference originalBytes = toShuffledXContent(response, xContentType, ToXContent.EMPTY_PARAMS, humanReadable);
+        BytesReference mutated;
+        if (addRandomFields) {
+            mutated = insertRandomFields(xContentType, originalBytes, null, random());
+        } else {
+            mutated = originalBytes;
+        }
+        ShardSearchFailure parsed;
+        try (XContentParser parser = createParser(xContentType.xContent(), mutated)) {
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            parsed = ShardSearchFailure.fromXContent(parser);
+            assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());
+            assertNull(parser.nextToken());
+        }
+        assertEquals(response.index(), parsed.index());
+        assertEquals(response.shard().getNodeId(), parsed.shard().getNodeId());
+        assertEquals(response.shardId(), parsed.shardId());
+
+        /**
+         * we cannot compare the cause, because it will be wrapped in an outer
+         * ElasticSearchException best effort: try to check that the original
+         * message appears somewhere in the rendered xContent
+         */
+        String originalMsg = response.getCause().getMessage();
+        assertEquals(parsed.getCause().getMessage(), "Elasticsearch exception [type=parsing_exception, reason=" + originalMsg + "]");
+        String nestedMsg = response.getCause().getCause().getMessage();
+        assertEquals(parsed.getCause().getCause().getMessage(),
+                "Elasticsearch exception [type=illegal_argument_exception, reason=" + nestedMsg + "]");
+    }
+
+    public void testToXContent() throws IOException {
+        ShardSearchFailure failure = new ShardSearchFailure(new ParsingException(0, 0, "some message", null),
+                new SearchShardTarget("nodeId", new ShardId(new Index("indexName", "indexUuid"), 123), null, OriginalIndices.NONE));
+        BytesReference xContent = toXContent(failure, XContentType.JSON, randomBoolean());
+        assertEquals(
+                "{\"shard\":123,"
+                        + "\"index\":\"indexName\","
+                        + "\"node\":\"nodeId\","
+                        + "\"reason\":{"
+                            + "\"type\":\"parsing_exception\","
+                            + "\"reason\":\"some message\","
+                            + "\"line\":0,"
+                            + "\"col\":0"
+                        + "}"
+                + "}",
+                xContent.utf8ToString());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/search/TransportMultiSearchActionTests.java b/core/src/test/java/org/elasticsearch/action/search/TransportMultiSearchActionTests.java
index 011fb17251449..a56a0b7e72209 100644
--- a/core/src/test/java/org/elasticsearch/action/search/TransportMultiSearchActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/search/TransportMultiSearchActionTests.java
@@ -30,6 +30,8 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Randomness;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.tasks.TaskManager;
@@ -37,7 +39,12 @@
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.util.Arrays;
 import java.util.Collections;
+import java.util.IdentityHashMap;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 
@@ -49,7 +56,7 @@
 public class TransportMultiSearchActionTests extends ESTestCase {
 
     public void testBatchExecute() throws Exception {
-        // Initialize depedencies of TransportMultiSearchAction
+        // Initialize dependencies of TransportMultiSearchAction
         Settings settings = Settings.builder()
                 .put("node.name", TransportMultiSearchActionTests.class.getSimpleName())
                 .build();
@@ -57,36 +64,43 @@ public void testBatchExecute() throws Exception {
         when(actionFilters.filters()).thenReturn(new ActionFilter[0]);
         ThreadPool threadPool = new ThreadPool(settings);
         TaskManager taskManager = mock(TaskManager.class);
-        TransportService transportService = mock(TransportService.class);
-        when(transportService.getTaskManager()).thenReturn(taskManager);
+        TransportService transportService = new TransportService(Settings.EMPTY, null, null, TransportService.NOOP_TRANSPORT_INTERCEPTOR,
+            boundAddress -> DiscoveryNode.createLocal(settings, boundAddress.publishAddress(), UUIDs.randomBase64UUID()), null) {
+            @Override
+            public TaskManager getTaskManager() {
+                return taskManager;
+            }
+        };
         ClusterService clusterService = mock(ClusterService.class);
         when(clusterService.state()).thenReturn(ClusterState.builder(new ClusterName("test")).build());
         IndexNameExpressionResolver resolver = new IndexNameExpressionResolver(Settings.EMPTY);
 
         // Keep track of the number of concurrent searches started by multi search api,
         // and if there are more searches than is allowed create an error and remember that.
-        int maxAllowedConcurrentSearches = scaledRandomIntBetween(1, 20);
+        int maxAllowedConcurrentSearches = scaledRandomIntBetween(1, 16);
         AtomicInteger counter = new AtomicInteger();
         AtomicReference<AssertionError> errorHolder = new AtomicReference<>();
+        // randomize whether or not requests are executed asynchronously
+        final List<String> threadPoolNames = Arrays.asList(ThreadPool.Names.GENERIC, ThreadPool.Names.SAME);
+        Randomness.shuffle(threadPoolNames);
+        final ExecutorService commonExecutor = threadPool.executor(threadPoolNames.get(0));
+        final ExecutorService rarelyExecutor = threadPool.executor(threadPoolNames.get(1));
+        final Set<SearchRequest> requests = Collections.newSetFromMap(Collections.synchronizedMap(new IdentityHashMap<>()));
         TransportAction<SearchRequest, SearchResponse> searchAction = new TransportAction<SearchRequest, SearchResponse>
                 (Settings.EMPTY, "action", threadPool, actionFilters, resolver, taskManager) {
             @Override
             protected void doExecute(SearchRequest request, ActionListener<SearchResponse> listener) {
+                requests.add(request);
                 int currentConcurrentSearches = counter.incrementAndGet();
                 if (currentConcurrentSearches > maxAllowedConcurrentSearches) {
                     errorHolder.set(new AssertionError("Current concurrent search [" + currentConcurrentSearches +
                             "] is higher than is allowed [" + maxAllowedConcurrentSearches + "]"));
                 }
-                threadPool.executor(ThreadPool.Names.GENERIC).execute(
-                        () -> {
-                            try {
-                                Thread.sleep(scaledRandomIntBetween(10, 1000));
-                            } catch (InterruptedException e) {
-                            }
-                            counter.decrementAndGet();
-                            listener.onResponse(new SearchResponse());
-                        }
-                );
+                final ExecutorService executorService = rarely() ? rarelyExecutor : commonExecutor;
+                executorService.execute(() -> {
+                    counter.decrementAndGet();
+                    listener.onResponse(new SearchResponse());
+                });
             }
         };
         TransportMultiSearchAction action =
@@ -94,7 +108,11 @@ protected void doExecute(SearchRequest request, ActionListener<SearchResponse> l
 
         // Execute the multi search api and fail if we find an error after executing:
         try {
-            int numSearchRequests = randomIntBetween(16, 128);
+            /*
+             * Allow for a large number of search requests in a single batch as previous implementations could stack overflow if the number
+             * of requests in a single batch was large
+             */
+            int numSearchRequests = scaledRandomIntBetween(1, 8192);
             MultiSearchRequest multiSearchRequest = new MultiSearchRequest();
             multiSearchRequest.maxConcurrentSearchRequests(maxAllowedConcurrentSearches);
             for (int i = 0; i < numSearchRequests; i++) {
@@ -103,6 +121,7 @@ protected void doExecute(SearchRequest request, ActionListener<SearchResponse> l
 
             MultiSearchResponse response = action.execute(multiSearchRequest).actionGet();
             assertThat(response.getResponses().length, equalTo(numSearchRequests));
+            assertThat(requests.size(), equalTo(numSearchRequests));
             assertThat(errorHolder.get(), nullValue());
         } finally {
             assertTrue(ESTestCase.terminate(threadPool));
diff --git a/core/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java b/core/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java
new file mode 100644
index 0000000000000..2e39a61b6f590
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java
@@ -0,0 +1,245 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.search;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.OriginalIndices;
+import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsGroup;
+import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsResponse;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.GroupShardsIterator;
+import org.elasticsearch.cluster.routing.PlainShardIterator;
+import org.elasticsearch.cluster.routing.ShardIterator;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.TestShardRouting;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.LocalTransportAddress;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
+import org.elasticsearch.index.query.TermsQueryBuilder;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.search.internal.AliasFilter;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.transport.MockTransportService;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.RemoteClusterService;
+import org.elasticsearch.transport.TransportService;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED;
+
+public class TransportSearchActionTests extends ESTestCase {
+
+    private final ThreadPool threadPool = new TestThreadPool(getClass().getName());
+
+    @Override
+    public void tearDown() throws Exception {
+        super.tearDown();
+        ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS);
+    }
+
+    public void testMergeShardsIterators() throws IOException {
+        List<ShardIterator> localShardIterators = new ArrayList<>();
+        {
+            ShardId shardId = new ShardId("local_index", "local_index_uuid", 0);
+            ShardRouting shardRouting = TestShardRouting.newShardRouting(shardId, "local_node", true, STARTED);
+            ShardIterator shardIterator = new PlainShardIterator(shardId, Collections.singletonList(shardRouting));
+            localShardIterators.add(shardIterator);
+        }
+        {
+            ShardId shardId2 = new ShardId("local_index_2", "local_index_2_uuid", 1);
+            ShardRouting shardRouting2 = TestShardRouting.newShardRouting(shardId2, "local_node", true, STARTED);
+            ShardIterator shardIterator2 = new PlainShardIterator(shardId2, Collections.singletonList(shardRouting2));
+            localShardIterators.add(shardIterator2);
+        }
+        GroupShardsIterator<ShardIterator> localShardsIterator = new GroupShardsIterator<>(localShardIterators);
+
+        OriginalIndices localIndices = new OriginalIndices(new String[]{"local_alias", "local_index_2"},
+                IndicesOptions.strictExpandOpenAndForbidClosed());
+
+        OriginalIndices remoteIndices = new OriginalIndices(new String[]{"remote_alias", "remote_index_2"},
+                IndicesOptions.strictExpandOpen());
+        List<SearchShardIterator> remoteShardIterators = new ArrayList<>();
+        {
+            ShardId remoteShardId = new ShardId("remote_index", "remote_index_uuid", 2);
+            ShardRouting remoteShardRouting = TestShardRouting.newShardRouting(remoteShardId, "remote_node", true, STARTED);
+            SearchShardIterator remoteShardIterator = new SearchShardIterator("remote", remoteShardId,
+                    Collections.singletonList(remoteShardRouting), remoteIndices);
+            remoteShardIterators.add(remoteShardIterator);
+        }
+        {
+            ShardId remoteShardId2 = new ShardId("remote_index_2", "remote_index_2_uuid", 3);
+            ShardRouting remoteShardRouting2 = TestShardRouting.newShardRouting(remoteShardId2, "remote_node", true, STARTED);
+            SearchShardIterator remoteShardIterator2 = new SearchShardIterator("remote", remoteShardId2,
+                    Collections.singletonList(remoteShardRouting2), remoteIndices);
+            remoteShardIterators.add(remoteShardIterator2);
+        }
+        OriginalIndices remoteIndices2 = new OriginalIndices(new String[]{"remote_index_3"}, IndicesOptions.strictExpand());
+
+        {
+            ShardId remoteShardId3 = new ShardId("remote_index_3", "remote_index_3_uuid", 4);
+            ShardRouting remoteShardRouting3 = TestShardRouting.newShardRouting(remoteShardId3, "remote_node", true, STARTED);
+            SearchShardIterator remoteShardIterator3 = new SearchShardIterator("remote", remoteShardId3,
+                    Collections.singletonList(remoteShardRouting3), remoteIndices2);
+            remoteShardIterators.add(remoteShardIterator3);
+        }
+
+        GroupShardsIterator<SearchShardIterator> searchShardIterators = TransportSearchAction.mergeShardsIterators(localShardsIterator,
+                localIndices, remoteShardIterators);
+
+        assertEquals(searchShardIterators.size(), 5);
+        int i = 0;
+        for (SearchShardIterator searchShardIterator : searchShardIterators) {
+            switch(i++) {
+                case 0:
+                    assertEquals("local_index", searchShardIterator.shardId().getIndexName());
+                    assertEquals(0, searchShardIterator.shardId().getId());
+                    assertSame(localIndices, searchShardIterator.getOriginalIndices());
+                    break;
+                case 1:
+                    assertEquals("local_index_2", searchShardIterator.shardId().getIndexName());
+                    assertEquals(1, searchShardIterator.shardId().getId());
+                    assertSame(localIndices, searchShardIterator.getOriginalIndices());
+                    break;
+                case 2:
+                    assertEquals("remote_index", searchShardIterator.shardId().getIndexName());
+                    assertEquals(2, searchShardIterator.shardId().getId());
+                    assertSame(remoteIndices, searchShardIterator.getOriginalIndices());
+                    break;
+                case 3:
+                    assertEquals("remote_index_2", searchShardIterator.shardId().getIndexName());
+                    assertEquals(3, searchShardIterator.shardId().getId());
+                    assertSame(remoteIndices, searchShardIterator.getOriginalIndices());
+                    break;
+                case 4:
+                    assertEquals("remote_index_3", searchShardIterator.shardId().getIndexName());
+                    assertEquals(4, searchShardIterator.shardId().getId());
+                    assertSame(remoteIndices2, searchShardIterator.getOriginalIndices());
+                    break;
+            }
+        }
+    }
+
+    public void testProcessRemoteShards() throws IOException {
+        try (TransportService transportService = MockTransportService.local(Settings.EMPTY, Version.CURRENT, threadPool,
+            null)) {
+            RemoteClusterService service = transportService.getRemoteClusterService();
+            assertFalse(service.isCrossClusterSearchEnabled());
+            List<SearchShardIterator> iteratorList = new ArrayList<>();
+            Map<String, ClusterSearchShardsResponse> searchShardsResponseMap = new HashMap<>();
+            DiscoveryNode[] nodes = new DiscoveryNode[] {
+                new DiscoveryNode("node1", LocalTransportAddress.buildUnique(), Version.CURRENT),
+                new DiscoveryNode("node2",  LocalTransportAddress.buildUnique(), Version.CURRENT)
+            };
+            Map<String, AliasFilter> indicesAndAliases = new HashMap<>();
+            indicesAndAliases.put("foo", new AliasFilter(new TermsQueryBuilder("foo", "bar"), "some_alias_for_foo",
+                "some_other_foo_alias"));
+            indicesAndAliases.put("bar", new AliasFilter(new MatchAllQueryBuilder(), Strings.EMPTY_ARRAY));
+            ClusterSearchShardsGroup[] groups = new ClusterSearchShardsGroup[] {
+                new ClusterSearchShardsGroup(new ShardId("foo", "foo_id", 0),
+                    new ShardRouting[] {TestShardRouting.newShardRouting("foo", 0, "node1", true, ShardRoutingState.STARTED),
+                        TestShardRouting.newShardRouting("foo", 0, "node2", false, ShardRoutingState.STARTED)}),
+                new ClusterSearchShardsGroup(new ShardId("foo", "foo_id", 1),
+                    new ShardRouting[] {TestShardRouting.newShardRouting("foo", 0, "node1", true, ShardRoutingState.STARTED),
+                        TestShardRouting.newShardRouting("foo", 1, "node2", false, ShardRoutingState.STARTED)}),
+                new ClusterSearchShardsGroup(new ShardId("bar", "bar_id", 0),
+                    new ShardRouting[] {TestShardRouting.newShardRouting("bar", 0, "node2", true, ShardRoutingState.STARTED),
+                        TestShardRouting.newShardRouting("bar", 0, "node1", false, ShardRoutingState.STARTED)})
+            };
+            searchShardsResponseMap.put("test_cluster_1", new ClusterSearchShardsResponse(groups, nodes, indicesAndAliases));
+            DiscoveryNode[] nodes2 = new DiscoveryNode[] {
+                new DiscoveryNode("node3",  LocalTransportAddress.buildUnique(), Version.CURRENT)
+            };
+            ClusterSearchShardsGroup[] groups2 = new ClusterSearchShardsGroup[] {
+                new ClusterSearchShardsGroup(new ShardId("xyz", "xyz_id", 0),
+                    new ShardRouting[] {TestShardRouting.newShardRouting("xyz", 0, "node3", true, ShardRoutingState.STARTED)})
+            };
+            Map<String, AliasFilter> filter = new HashMap<>();
+            filter.put("xyz", new AliasFilter(null, "some_alias_for_xyz"));
+            searchShardsResponseMap.put("test_cluster_2", new ClusterSearchShardsResponse(groups2, nodes2, filter));
+
+            Map<String, OriginalIndices> remoteIndicesByCluster = new HashMap<>();
+            remoteIndicesByCluster.put("test_cluster_1",
+                new OriginalIndices(new String[]{"fo*", "ba*"}, IndicesOptions.strictExpandOpenAndForbidClosed()));
+            remoteIndicesByCluster.put("test_cluster_2",
+                new OriginalIndices(new String[]{"x*"}, IndicesOptions.strictExpandOpenAndForbidClosed()));
+            Map<String, AliasFilter> remoteAliases = new HashMap<>();
+            TransportSearchAction.processRemoteShards(searchShardsResponseMap, remoteIndicesByCluster, iteratorList,
+                remoteAliases);
+            assertEquals(4, iteratorList.size());
+            for (SearchShardIterator iterator : iteratorList) {
+                if (iterator.shardId().getIndexName().endsWith("foo")) {
+                    assertArrayEquals(new String[]{"some_alias_for_foo", "some_other_foo_alias"},
+                        iterator.getOriginalIndices().indices());
+                    assertTrue(iterator.shardId().getId() == 0 || iterator.shardId().getId() == 1);
+                    assertEquals("test_cluster_1", iterator.getClusterAlias());
+                    assertEquals("foo", iterator.shardId().getIndexName());
+                    ShardRouting shardRouting = iterator.nextOrNull();
+                    assertNotNull(shardRouting);
+                    assertEquals(shardRouting.getIndexName(), "foo");
+                    shardRouting = iterator.nextOrNull();
+                    assertNotNull(shardRouting);
+                    assertEquals(shardRouting.getIndexName(), "foo");
+                    assertNull(iterator.nextOrNull());
+                } else if (iterator.shardId().getIndexName().endsWith("bar")) {
+                    assertArrayEquals(new String[]{"bar"}, iterator.getOriginalIndices().indices());
+                    assertEquals(0, iterator.shardId().getId());
+                    assertEquals("test_cluster_1", iterator.getClusterAlias());
+                    assertEquals("bar", iterator.shardId().getIndexName());
+                    ShardRouting shardRouting = iterator.nextOrNull();
+                    assertNotNull(shardRouting);
+                    assertEquals(shardRouting.getIndexName(), "bar");
+                    shardRouting = iterator.nextOrNull();
+                    assertNotNull(shardRouting);
+                    assertEquals(shardRouting.getIndexName(), "bar");
+                    assertNull(iterator.nextOrNull());
+                } else if (iterator.shardId().getIndexName().endsWith("xyz")) {
+                    assertArrayEquals(new String[]{"some_alias_for_xyz"}, iterator.getOriginalIndices().indices());
+                    assertEquals(0, iterator.shardId().getId());
+                    assertEquals("xyz", iterator.shardId().getIndexName());
+                    assertEquals("test_cluster_2", iterator.getClusterAlias());
+                    ShardRouting shardRouting = iterator.nextOrNull();
+                    assertNotNull(shardRouting);
+                    assertEquals(shardRouting.getIndexName(), "xyz");
+                    assertNull(iterator.nextOrNull());
+                }
+            }
+            assertEquals(3, remoteAliases.size());
+            assertTrue(remoteAliases.toString(), remoteAliases.containsKey("foo_id"));
+            assertTrue(remoteAliases.toString(), remoteAliases.containsKey("bar_id"));
+            assertTrue(remoteAliases.toString(), remoteAliases.containsKey("xyz_id"));
+            assertEquals(new TermsQueryBuilder("foo", "bar"), remoteAliases.get("foo_id").getQueryBuilder());
+            assertEquals(new MatchAllQueryBuilder(), remoteAliases.get("bar_id").getQueryBuilder());
+            assertNull(remoteAliases.get("xyz_id").getQueryBuilder());
+        }
+
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/action/search/TransportSearchHelperTests.java b/core/src/test/java/org/elasticsearch/action/search/TransportSearchHelperTests.java
new file mode 100644
index 0000000000000..3ab5b5266540d
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/search/TransportSearchHelperTests.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.search;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.transport.LocalTransportAddress;
+import org.elasticsearch.common.util.concurrent.AtomicArray;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.search.SearchPhaseResult;
+import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+public class TransportSearchHelperTests extends ESTestCase {
+
+    public void testParseScrollId() throws IOException {
+        AtomicArray<SearchPhaseResult> array = new AtomicArray<>(3);
+        DiscoveryNode node1 = new DiscoveryNode("node_1", LocalTransportAddress.buildUnique(), Version.CURRENT);
+        DiscoveryNode node2 = new DiscoveryNode("node_2", LocalTransportAddress.buildUnique(), Version.CURRENT);
+        DiscoveryNode node3 = new DiscoveryNode("node_3", LocalTransportAddress.buildUnique(), Version.CURRENT);
+        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult1 = new SearchAsyncActionTests.TestSearchPhaseResult(1, node1);
+        testSearchPhaseResult1.setSearchShardTarget(new SearchShardTarget("node_1", new ShardId("idx", "uuid1", 2), "cluster_x", null));
+        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult2 = new SearchAsyncActionTests.TestSearchPhaseResult(12, node2);
+        testSearchPhaseResult2.setSearchShardTarget(new SearchShardTarget("node_2", new ShardId("idy", "uuid2", 42), "cluster_y", null));
+        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult3 = new SearchAsyncActionTests.TestSearchPhaseResult(42, node3);
+        testSearchPhaseResult3.setSearchShardTarget(new SearchShardTarget("node_3", new ShardId("idy", "uuid2", 43), null, null));
+        array.setOnce(0, testSearchPhaseResult1);
+        array.setOnce(1, testSearchPhaseResult2);
+        array.setOnce(2, testSearchPhaseResult3);
+
+
+        String scrollId = TransportSearchHelper.buildScrollId(array);
+        ParsedScrollId parseScrollId = TransportSearchHelper.parseScrollId(scrollId);
+        assertEquals(3, parseScrollId.getContext().length);
+        assertEquals("node_1", parseScrollId.getContext()[0].getNode());
+        assertEquals("cluster_x", parseScrollId.getContext()[0].getClusterAlias());
+        assertEquals(1, parseScrollId.getContext()[0].getScrollId());
+
+        assertEquals("node_2", parseScrollId.getContext()[1].getNode());
+        assertEquals("cluster_y", parseScrollId.getContext()[1].getClusterAlias());
+        assertEquals(12, parseScrollId.getContext()[1].getScrollId());
+
+        assertEquals("node_3", parseScrollId.getContext()[2].getNode());
+        assertNull(parseScrollId.getContext()[2].getClusterAlias());
+        assertEquals(42, parseScrollId.getContext()[2].getScrollId());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/support/ActiveShardCountTests.java b/core/src/test/java/org/elasticsearch/action/support/ActiveShardCountTests.java
index 598a672fb2d98..4fb03bf393b3c 100644
--- a/core/src/test/java/org/elasticsearch/action/support/ActiveShardCountTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/ActiveShardCountTests.java
@@ -62,7 +62,7 @@ public void testParseString() {
         assertSame(ActiveShardCount.parseString("0"), ActiveShardCount.NONE);
         int value = randomIntBetween(1, 50);
         assertEquals(ActiveShardCount.parseString(value + ""), ActiveShardCount.from(value));
-        expectThrows(IllegalArgumentException.class, () -> ActiveShardCount.parseString(randomAsciiOfLengthBetween(4, 8)));
+        expectThrows(IllegalArgumentException.class, () -> ActiveShardCount.parseString(randomAlphaOfLengthBetween(4, 8)));
         expectThrows(IllegalArgumentException.class, () -> ActiveShardCount.parseString("-1")); // magic numbers not exposed through API
         expectThrows(IllegalArgumentException.class, () -> ActiveShardCount.parseString("-2"));
         expectThrows(IllegalArgumentException.class, () -> ActiveShardCount.parseString(randomIntBetween(-10, -3) + ""));
@@ -200,7 +200,7 @@ private ClusterState startPrimaries(final ClusterState clusterState, final Strin
             final IndexShardRoutingTable shardRoutingTable = shardEntry.value;
             for (ShardRouting shardRouting : shardRoutingTable.getShards()) {
                 if (shardRouting.primary()) {
-                    shardRouting = shardRouting.initialize(randomAsciiOfLength(8), null, shardRouting.getExpectedShardSize())
+                    shardRouting = shardRouting.initialize(randomAlphaOfLength(8), null, shardRouting.getExpectedShardSize())
                                        .moveToStarted();
                 }
                 newIndexRoutingTable.addShard(shardRouting);
@@ -224,7 +224,7 @@ private ClusterState startLessThanWaitOnShards(final ClusterState clusterState,
                     assertTrue(shardRouting.active());
                 } else {
                     if (numToStart > 0) {
-                        shardRouting = shardRouting.initialize(randomAsciiOfLength(8), null, shardRouting.getExpectedShardSize())
+                        shardRouting = shardRouting.initialize(randomAlphaOfLength(8), null, shardRouting.getExpectedShardSize())
                                            .moveToStarted();
                         numToStart--;
                     }
@@ -250,7 +250,7 @@ private ClusterState startWaitOnShards(final ClusterState clusterState, final St
                 } else {
                     if (shardRouting.active() == false) {
                         if (numToStart > 0) {
-                            shardRouting = shardRouting.initialize(randomAsciiOfLength(8), null, shardRouting.getExpectedShardSize())
+                            shardRouting = shardRouting.initialize(randomAlphaOfLength(8), null, shardRouting.getExpectedShardSize())
                                                .moveToStarted();
                             numToStart--;
                         }
@@ -276,7 +276,7 @@ private ClusterState startAllShards(final ClusterState clusterState, final Strin
                     assertTrue(shardRouting.active());
                 } else {
                     if (shardRouting.active() == false) {
-                        shardRouting = shardRouting.initialize(randomAsciiOfLength(8), null, shardRouting.getExpectedShardSize())
+                        shardRouting = shardRouting.initialize(randomAlphaOfLength(8), null, shardRouting.getExpectedShardSize())
                                            .moveToStarted();
                     }
                 }
diff --git a/core/src/test/java/org/elasticsearch/action/support/AdapterActionFutureTests.java b/core/src/test/java/org/elasticsearch/action/support/AdapterActionFutureTests.java
new file mode 100644
index 0000000000000..a7405ddae8cce
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/support/AdapterActionFutureTests.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.support;
+
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Objects;
+import java.util.concurrent.BrokenBarrierException;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+public class AdapterActionFutureTests extends ESTestCase {
+
+    public void testInterruption() throws Exception {
+        final AdapterActionFuture<String, Integer> adapter =
+                new AdapterActionFuture<String, Integer>() {
+                    @Override
+                    protected String convert(final Integer listenerResponse) {
+                        return Objects.toString(listenerResponse);
+                    }
+                };
+
+        // test all possible methods that can be interrupted
+        final Runnable runnable = () -> {
+            final int method = randomIntBetween(0, 4);
+            switch (method) {
+                case 0:
+                    adapter.actionGet();
+                    break;
+                case 1:
+                    adapter.actionGet("30s");
+                    break;
+                case 2:
+                    adapter.actionGet(30000);
+                    break;
+                case 3:
+                    adapter.actionGet(TimeValue.timeValueSeconds(30));
+                    break;
+                case 4:
+                    adapter.actionGet(30, TimeUnit.SECONDS);
+                    break;
+                default:
+                    throw new AssertionError(method);
+            }
+        };
+
+        final CyclicBarrier barrier = new CyclicBarrier(2);
+        final Thread main = Thread.currentThread();
+        final Thread thread = new Thread(() -> {
+            try {
+                barrier.await();
+            } catch (final BrokenBarrierException | InterruptedException e) {
+                throw new RuntimeException(e);
+            }
+            main.interrupt();
+        });
+        thread.start();
+
+        final AtomicBoolean interrupted = new AtomicBoolean();
+
+        barrier.await();
+
+        try {
+            runnable.run();
+        } catch (final IllegalStateException e) {
+            interrupted.set(Thread.interrupted());
+        }
+        // we check this here instead of in the catch block to ensure that the catch block executed
+        assertTrue(interrupted.get());
+
+        thread.join();
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/action/support/AutoCreateIndexTests.java b/core/src/test/java/org/elasticsearch/action/support/AutoCreateIndexTests.java
index 8a45ca4753565..159be84de0791 100644
--- a/core/src/test/java/org/elasticsearch/action/support/AutoCreateIndexTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/AutoCreateIndexTests.java
@@ -25,11 +25,17 @@
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.test.ESTestCase;
 
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
 import static org.hamcrest.CoreMatchers.equalTo;
 
 public class AutoCreateIndexTests extends ESTestCase {
@@ -57,26 +63,46 @@ public void testParseFailedMissingIndex() {
         }
     }
 
+    public void testHandleSpaces() { // see #21449
+        Settings settings = Settings.builder().put(AutoCreateIndex.AUTO_CREATE_INDEX_SETTING.getKey(),
+            randomFrom(".marvel-, .security, .watches, .triggered_watches, .watcher-history-",
+                ".marvel-,.security,.watches,.triggered_watches,.watcher-history-")).build();
+        AutoCreateIndex autoCreateIndex = newAutoCreateIndex(settings);
+        List<Tuple<String, Boolean>> expressions = autoCreateIndex.getAutoCreate().getExpressions();
+        Map<String, Boolean> map = new HashMap<>();
+        for (Tuple<String, Boolean> t : expressions) {
+            map.put(t.v1(), t.v2());
+        }
+        assertTrue(map.get(".marvel-"));
+        assertTrue(map.get(".security"));
+        assertTrue(map.get(".watches"));
+        assertTrue(map.get(".triggered_watches"));
+        assertTrue(map.get(".watcher-history-"));
+        assertEquals(5, map.size());
+    }
+
     public void testAutoCreationDisabled() {
         Settings settings = Settings.builder().put(AutoCreateIndex.AUTO_CREATE_INDEX_SETTING.getKey(), false).build();
         AutoCreateIndex autoCreateIndex = newAutoCreateIndex(settings);
-        assertThat(autoCreateIndex.shouldAutoCreate(randomAsciiOfLengthBetween(1, 10), buildClusterState()), equalTo(false));
+        IndexNotFoundException e = expectThrows(IndexNotFoundException.class, () ->
+            autoCreateIndex.shouldAutoCreate(randomAlphaOfLengthBetween(1, 10), buildClusterState()));
+        assertEquals("no such index and [action.auto_create_index] is [false]", e.getMessage());
     }
 
     public void testAutoCreationEnabled() {
         Settings settings = Settings.builder().put(AutoCreateIndex.AUTO_CREATE_INDEX_SETTING.getKey(), true).build();
         AutoCreateIndex autoCreateIndex = newAutoCreateIndex(settings);
-        assertThat(autoCreateIndex.shouldAutoCreate(randomAsciiOfLengthBetween(1, 10), buildClusterState()), equalTo(true));
+        assertThat(autoCreateIndex.shouldAutoCreate(randomAlphaOfLengthBetween(1, 10), buildClusterState()), equalTo(true));
     }
 
     public void testDefaultAutoCreation() {
         AutoCreateIndex autoCreateIndex = newAutoCreateIndex(Settings.EMPTY);
-        assertThat(autoCreateIndex.shouldAutoCreate(randomAsciiOfLengthBetween(1, 10), buildClusterState()), equalTo(true));
+        assertThat(autoCreateIndex.shouldAutoCreate(randomAlphaOfLengthBetween(1, 10), buildClusterState()), equalTo(true));
     }
 
     public void testExistingIndex() {
         Settings settings = Settings.builder().put(AutoCreateIndex.AUTO_CREATE_INDEX_SETTING.getKey(), randomFrom(true, false,
-                randomAsciiOfLengthBetween(7, 10))).build();
+                randomAlphaOfLengthBetween(7, 10))).build();
         AutoCreateIndex autoCreateIndex = newAutoCreateIndex(settings);
         assertThat(autoCreateIndex.shouldAutoCreate(randomFrom("index1", "index2", "index3"),
                 buildClusterState("index1", "index2", "index3")), equalTo(false));
@@ -84,10 +110,12 @@ public void testExistingIndex() {
 
     public void testDynamicMappingDisabled() {
         Settings settings = Settings.builder().put(AutoCreateIndex.AUTO_CREATE_INDEX_SETTING.getKey(), randomFrom(true,
-                randomAsciiOfLengthBetween(1, 10)))
+                randomAlphaOfLengthBetween(1, 10)))
             .put(MapperService.INDEX_MAPPER_DYNAMIC_SETTING.getKey(), false).build();
         AutoCreateIndex autoCreateIndex = newAutoCreateIndex(settings);
-        assertThat(autoCreateIndex.shouldAutoCreate(randomAsciiOfLengthBetween(1, 10), buildClusterState()), equalTo(false));
+        IndexNotFoundException e = expectThrows(IndexNotFoundException.class, () ->
+            autoCreateIndex.shouldAutoCreate(randomAlphaOfLengthBetween(1, 10), buildClusterState()));
+        assertEquals("no such index and [index.mapper.dynamic] is [false]", e.getMessage());
     }
 
     public void testAutoCreationPatternEnabled() {
@@ -95,17 +123,18 @@ public void testAutoCreationPatternEnabled() {
                 .build();
         AutoCreateIndex autoCreateIndex = newAutoCreateIndex(settings);
         ClusterState clusterState = ClusterState.builder(new ClusterName("test")).metaData(MetaData.builder()).build();
-        assertThat(autoCreateIndex.shouldAutoCreate("index" + randomAsciiOfLengthBetween(1, 5), clusterState), equalTo(true));
-        assertThat(autoCreateIndex.shouldAutoCreate("does_not_match" + randomAsciiOfLengthBetween(1, 5), clusterState), equalTo(false));
+        assertThat(autoCreateIndex.shouldAutoCreate("index" + randomAlphaOfLengthBetween(1, 5), clusterState), equalTo(true));
+        expectNotMatch(clusterState, autoCreateIndex, "does_not_match" + randomAlphaOfLengthBetween(1, 5));
     }
 
     public void testAutoCreationPatternDisabled() {
         Settings settings = Settings.builder().put(AutoCreateIndex.AUTO_CREATE_INDEX_SETTING.getKey(), "-index*").build();
         AutoCreateIndex autoCreateIndex = newAutoCreateIndex(settings);
         ClusterState clusterState = ClusterState.builder(new ClusterName("test")).metaData(MetaData.builder()).build();
-        assertThat(autoCreateIndex.shouldAutoCreate("index" + randomAsciiOfLengthBetween(1, 5), clusterState), equalTo(false));
-        //default is false when patterns are specified
-        assertThat(autoCreateIndex.shouldAutoCreate("does_not_match" + randomAsciiOfLengthBetween(1, 5), clusterState), equalTo(false));
+        expectForbidden(clusterState, autoCreateIndex, "index" + randomAlphaOfLengthBetween(1, 5), "-index*");
+        /* When patterns are specified, even if the are all negative, the default is can't create. So a pure negative pattern is the same
+         * as false, really. */
+        expectNotMatch(clusterState, autoCreateIndex, "does_not_match" + randomAlphaOfLengthBetween(1, 5));
     }
 
     public void testAutoCreationMultiplePatternsWithWildcards() {
@@ -113,9 +142,9 @@ public void testAutoCreationMultiplePatternsWithWildcards() {
                 randomFrom("+test*,-index*", "test*,-index*")).build();
         AutoCreateIndex autoCreateIndex = newAutoCreateIndex(settings);
         ClusterState clusterState = ClusterState.builder(new ClusterName("test")).metaData(MetaData.builder()).build();
-        assertThat(autoCreateIndex.shouldAutoCreate("index" + randomAsciiOfLengthBetween(1, 5), clusterState), equalTo(false));
-        assertThat(autoCreateIndex.shouldAutoCreate("test" + randomAsciiOfLengthBetween(1, 5), clusterState), equalTo(true));
-        assertThat(autoCreateIndex.shouldAutoCreate("does_not_match" + randomAsciiOfLengthBetween(1, 5), clusterState), equalTo(false));
+        expectForbidden(clusterState, autoCreateIndex, "index" + randomAlphaOfLengthBetween(1, 5), "-index*");
+        assertThat(autoCreateIndex.shouldAutoCreate("test" + randomAlphaOfLengthBetween(1, 5), clusterState), equalTo(true));
+        expectNotMatch(clusterState, autoCreateIndex, "does_not_match" + randomAlphaOfLengthBetween(1, 5));
     }
 
     public void testAutoCreationMultiplePatternsNoWildcards() {
@@ -123,9 +152,9 @@ public void testAutoCreationMultiplePatternsNoWildcards() {
         AutoCreateIndex autoCreateIndex = newAutoCreateIndex(settings);
         ClusterState clusterState = ClusterState.builder(new ClusterName("test")).metaData(MetaData.builder()).build();
         assertThat(autoCreateIndex.shouldAutoCreate("test1", clusterState), equalTo(true));
-        assertThat(autoCreateIndex.shouldAutoCreate("index" + randomAsciiOfLengthBetween(1, 5), clusterState), equalTo(false));
-        assertThat(autoCreateIndex.shouldAutoCreate("test" + randomAsciiOfLengthBetween(2, 5), clusterState), equalTo(false));
-        assertThat(autoCreateIndex.shouldAutoCreate("does_not_match" + randomAsciiOfLengthBetween(1, 5), clusterState), equalTo(false));
+        expectNotMatch(clusterState, autoCreateIndex, "index" + randomAlphaOfLengthBetween(1, 5));
+        expectNotMatch(clusterState, autoCreateIndex, "test" + randomAlphaOfLengthBetween(2, 5));
+        expectNotMatch(clusterState, autoCreateIndex, "does_not_match" + randomAlphaOfLengthBetween(1, 5));
     }
 
     public void testAutoCreationMultipleIndexNames() {
@@ -134,7 +163,7 @@ public void testAutoCreationMultipleIndexNames() {
         ClusterState clusterState = ClusterState.builder(new ClusterName("test")).metaData(MetaData.builder()).build();
         assertThat(autoCreateIndex.shouldAutoCreate("test1", clusterState), equalTo(true));
         assertThat(autoCreateIndex.shouldAutoCreate("test2", clusterState), equalTo(true));
-        assertThat(autoCreateIndex.shouldAutoCreate("does_not_match" + randomAsciiOfLengthBetween(1, 5), clusterState), equalTo(false));
+        expectNotMatch(clusterState, autoCreateIndex, "does_not_match" + randomAlphaOfLengthBetween(1, 5));
     }
 
     public void testAutoCreationConflictingPatternsFirstWins() {
@@ -143,8 +172,8 @@ public void testAutoCreationConflictingPatternsFirstWins() {
         AutoCreateIndex autoCreateIndex = newAutoCreateIndex(settings);
         ClusterState clusterState = ClusterState.builder(new ClusterName("test")).metaData(MetaData.builder()).build();
         assertThat(autoCreateIndex.shouldAutoCreate("test1", clusterState), equalTo(true));
-        assertThat(autoCreateIndex.shouldAutoCreate("test2", clusterState), equalTo(false));
-        assertThat(autoCreateIndex.shouldAutoCreate("does_not_match" + randomAsciiOfLengthBetween(1, 5), clusterState), equalTo(false));
+        expectForbidden(clusterState, autoCreateIndex, "test2", "-test2");
+        expectNotMatch(clusterState, autoCreateIndex, "does_not_match" + randomAlphaOfLengthBetween(1, 5));
     }
 
     public void testUpdate() {
@@ -185,4 +214,18 @@ private AutoCreateIndex newAutoCreateIndex(Settings settings) {
         return new AutoCreateIndex(settings, new ClusterSettings(settings,
                 ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), new IndexNameExpressionResolver(settings));
     }
+
+    private void expectNotMatch(ClusterState clusterState, AutoCreateIndex autoCreateIndex, String index) {
+        IndexNotFoundException e = expectThrows(IndexNotFoundException.class, () ->
+            autoCreateIndex.shouldAutoCreate(index, clusterState));
+        assertEquals("no such index and [action.auto_create_index] ([" + autoCreateIndex.getAutoCreate() + "]) doesn't match",
+                e.getMessage());
+    }
+
+    private void expectForbidden(ClusterState clusterState, AutoCreateIndex autoCreateIndex, String index, String forbiddingPattern) {
+        IndexNotFoundException e = expectThrows(IndexNotFoundException.class, () ->
+            autoCreateIndex.shouldAutoCreate(index, clusterState));
+        assertEquals("no such index and [action.auto_create_index] contains [" + forbiddingPattern
+                + "] which forbids automatic creation of the index", e.getMessage());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/support/ContextPreservingActionListenerTests.java b/core/src/test/java/org/elasticsearch/action/support/ContextPreservingActionListenerTests.java
new file mode 100644
index 0000000000000..dd627041f2262
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/support/ContextPreservingActionListenerTests.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.support;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+public class ContextPreservingActionListenerTests extends ESTestCase {
+
+    public void testOriginalContextIsPreservedAfterOnResponse() throws IOException {
+        try (ThreadContext threadContext = new ThreadContext(Settings.EMPTY)) {
+            final boolean nonEmptyContext = randomBoolean();
+            if (nonEmptyContext) {
+                threadContext.putHeader("not empty", "value");
+            }
+            final ContextPreservingActionListener<Void> actionListener;
+            try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
+                threadContext.putHeader("foo", "bar");
+                final ActionListener<Void> delegate = new ActionListener<Void>() {
+                    @Override
+                    public void onResponse(Void aVoid) {
+                        assertEquals("bar", threadContext.getHeader("foo"));
+                        assertNull(threadContext.getHeader("not empty"));
+                    }
+
+                    @Override
+                    public void onFailure(Exception e) {
+                        throw new RuntimeException("onFailure shouldn't be called", e);
+                    }
+                };
+                if (randomBoolean()) {
+                    actionListener = new ContextPreservingActionListener<>(threadContext.newRestorableContext(true), delegate);
+                } else {
+                    actionListener = ContextPreservingActionListener.wrapPreservingContext(delegate, threadContext);
+                }
+            }
+
+            assertNull(threadContext.getHeader("foo"));
+            assertEquals(nonEmptyContext ? "value" : null, threadContext.getHeader("not empty"));
+
+            actionListener.onResponse(null);
+
+            assertNull(threadContext.getHeader("foo"));
+            assertEquals(nonEmptyContext ? "value" : null, threadContext.getHeader("not empty"));
+        }
+    }
+
+    public void testOriginalContextIsPreservedAfterOnFailure() throws Exception {
+        try (ThreadContext threadContext = new ThreadContext(Settings.EMPTY)) {
+            final boolean nonEmptyContext = randomBoolean();
+            if (nonEmptyContext) {
+                threadContext.putHeader("not empty", "value");
+            }
+            final ContextPreservingActionListener<Void> actionListener;
+            try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
+                threadContext.putHeader("foo", "bar");
+                final ActionListener<Void> delegate = new ActionListener<Void>() {
+                    @Override
+                    public void onResponse(Void aVoid) {
+                        throw new RuntimeException("onResponse shouldn't be called");
+                    }
+
+                    @Override
+                    public void onFailure(Exception e) {
+                        assertEquals("bar", threadContext.getHeader("foo"));
+                        assertNull(threadContext.getHeader("not empty"));
+                    }
+                };
+
+                if (randomBoolean()) {
+                    actionListener = new ContextPreservingActionListener<>(threadContext.newRestorableContext(true), delegate);
+                } else {
+                    actionListener = ContextPreservingActionListener.wrapPreservingContext(delegate, threadContext);
+                }
+
+            }
+
+            assertNull(threadContext.getHeader("foo"));
+            assertEquals(nonEmptyContext ? "value" : null, threadContext.getHeader("not empty"));
+
+            actionListener.onFailure(null);
+
+            assertNull(threadContext.getHeader("foo"));
+            assertEquals(nonEmptyContext ? "value" : null, threadContext.getHeader("not empty"));
+        }
+    }
+
+    public void testOriginalContextIsWhenListenerThrows() throws Exception {
+        try (ThreadContext threadContext = new ThreadContext(Settings.EMPTY)) {
+            final boolean nonEmptyContext = randomBoolean();
+            if (nonEmptyContext) {
+                threadContext.putHeader("not empty", "value");
+            }
+            final ContextPreservingActionListener<Void> actionListener;
+            try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
+                threadContext.putHeader("foo", "bar");
+                final ActionListener<Void> delegate = new ActionListener<Void>() {
+                    @Override
+                    public void onResponse(Void aVoid) {
+                        assertEquals("bar", threadContext.getHeader("foo"));
+                        assertNull(threadContext.getHeader("not empty"));
+                        throw new RuntimeException("onResponse called");
+                    }
+
+                    @Override
+                    public void onFailure(Exception e) {
+                        assertEquals("bar", threadContext.getHeader("foo"));
+                        assertNull(threadContext.getHeader("not empty"));
+                        throw new RuntimeException("onFailure called");
+                    }
+                };
+
+                if (randomBoolean()) {
+                    actionListener = new ContextPreservingActionListener<>(threadContext.newRestorableContext(true), delegate);
+                } else {
+                    actionListener = ContextPreservingActionListener.wrapPreservingContext(delegate, threadContext);
+                }
+            }
+
+            assertNull(threadContext.getHeader("foo"));
+            assertEquals(nonEmptyContext ? "value" : null, threadContext.getHeader("not empty"));
+
+            RuntimeException e = expectThrows(RuntimeException.class, () -> actionListener.onResponse(null));
+            assertEquals("onResponse called", e.getMessage());
+
+            assertNull(threadContext.getHeader("foo"));
+            assertEquals(nonEmptyContext ? "value" : null, threadContext.getHeader("not empty"));
+
+            e = expectThrows(RuntimeException.class, () -> actionListener.onFailure(null));
+            assertEquals("onFailure called", e.getMessage());
+
+            assertNull(threadContext.getHeader("foo"));
+            assertEquals(nonEmptyContext ? "value" : null, threadContext.getHeader("not empty"));
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/support/GroupedActionListenerTests.java b/core/src/test/java/org/elasticsearch/action/support/GroupedActionListenerTests.java
new file mode 100644
index 0000000000000..2af2da7ba0939
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/support/GroupedActionListenerTests.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.support;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+public class GroupedActionListenerTests extends ESTestCase {
+
+    public void testNotifications() throws InterruptedException {
+        AtomicReference<Collection<Integer>> resRef = new AtomicReference<>();
+        ActionListener<Collection<Integer>> result = new ActionListener<Collection<Integer>>() {
+            @Override
+            public void onResponse(Collection<Integer> integers) {
+                resRef.set(integers);
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                throw new AssertionError(e);
+            }
+        };
+        final int groupSize = randomIntBetween(10, 1000);
+        AtomicInteger count = new AtomicInteger();
+        Collection<Integer> defaults = randomBoolean() ? Collections.singletonList(-1) :
+            Collections.emptyList();
+        GroupedActionListener<Integer> listener = new GroupedActionListener<>(result, groupSize,
+            defaults);
+        int numThreads = randomIntBetween(2, 5);
+        Thread[] threads = new Thread[numThreads];
+        CyclicBarrier barrier = new CyclicBarrier(numThreads);
+        for (int i = 0; i < numThreads; i++) {
+            threads[i] = new Thread()  {
+                @Override
+                public void run() {
+                    try {
+                        barrier.await(10, TimeUnit.SECONDS);
+                    } catch (Exception e) {
+                        throw new AssertionError(e);
+                    }
+                    int c = 0;
+                    while((c = count.incrementAndGet()) <= groupSize) {
+                        listener.onResponse(c-1);
+                    }
+                }
+            };
+            threads[i].start();
+        }
+        for (Thread t : threads) {
+            t.join();
+        }
+        assertNotNull(resRef.get());
+        ArrayList<Integer> list = new ArrayList<>(resRef.get());
+        Collections.sort(list);
+        int expectedSize = groupSize + defaults.size();
+        assertEquals(expectedSize, resRef.get().size());
+        int expectedValue = defaults.isEmpty() ? 0 : -1;
+        for (int i = 0; i < expectedSize; i++) {
+            assertEquals(Integer.valueOf(expectedValue++), list.get(i));
+        }
+    }
+
+    public void testFailed() {
+        AtomicReference<Collection<Integer>> resRef = new AtomicReference<>();
+        AtomicReference<Exception> excRef = new AtomicReference<>();
+
+        ActionListener<Collection<Integer>> result = new ActionListener<Collection<Integer>>() {
+            @Override
+            public void onResponse(Collection<Integer> integers) {
+                resRef.set(integers);
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                excRef.set(e);
+            }
+        };
+        Collection<Integer> defaults = randomBoolean() ? Collections.singletonList(-1) :
+            Collections.emptyList();
+        int size = randomIntBetween(3, 4);
+        GroupedActionListener<Integer> listener = new GroupedActionListener<>(result, size,
+            defaults);
+        listener.onResponse(0);
+        IOException ioException = new IOException();
+        RuntimeException rtException = new RuntimeException();
+        listener.onFailure(rtException);
+        listener.onFailure(ioException);
+        if (size == 4) {
+            listener.onResponse(2);
+        }
+        assertNotNull(excRef.get());
+        assertEquals(rtException, excRef.get());
+        assertEquals(1, excRef.get().getSuppressed().length);
+        assertEquals(ioException, excRef.get().getSuppressed()[0]);
+        assertNull(resRef.get());
+        listener.onResponse(1);
+        assertNull(resRef.get());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/support/RefreshPolicyTests.java b/core/src/test/java/org/elasticsearch/action/support/RefreshPolicyTests.java
new file mode 100644
index 0000000000000..a6e25036c5317
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/support/RefreshPolicyTests.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.support;
+
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+public class RefreshPolicyTests extends ESTestCase {
+
+    public void testSerialization() throws IOException {
+        final WriteRequest.RefreshPolicy refreshPolicy = randomFrom(WriteRequest.RefreshPolicy.values());
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            refreshPolicy.writeTo(out);
+            try (StreamInput in = out.bytes().streamInput()) {
+                WriteRequest.RefreshPolicy deserializedRefreshPolicy = WriteRequest.RefreshPolicy.readFrom(in);
+                assertEquals(refreshPolicy, deserializedRefreshPolicy);
+            }
+        }
+    }
+
+    public void testParse() throws IOException {
+        final String refreshPolicyValue = randomFrom(WriteRequest.RefreshPolicy.values()).getValue();
+        assertEquals(refreshPolicyValue, WriteRequest.RefreshPolicy.parse(refreshPolicyValue).getValue());
+    }
+
+    public void testParseEmpty() throws IOException {
+        assertEquals(WriteRequest.RefreshPolicy.IMMEDIATE, WriteRequest.RefreshPolicy.parse(""));
+    }
+
+    public void testParseUnknown() throws IOException {
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> WriteRequest.RefreshPolicy.parse("unknown"));
+        assertEquals("Unknown value for refresh: [unknown].", e.getMessage());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/support/TransportActionFilterChainTests.java b/core/src/test/java/org/elasticsearch/action/support/TransportActionFilterChainTests.java
index bbf1d2f19427e..aa61c2569d818 100644
--- a/core/src/test/java/org/elasticsearch/action/support/TransportActionFilterChainTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/TransportActionFilterChainTests.java
@@ -41,8 +41,6 @@
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.Function;
-import java.util.stream.IntStream;
 
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.instanceOf;
@@ -69,7 +67,7 @@ public void testActionFiltersRequest() throws ExecutionException, InterruptedExc
             filters.add(new RequestTestFilter(order, randomFrom(RequestOperation.values())));
         }
 
-        String actionName = randomAsciiOfLength(randomInt(30));
+        String actionName = randomAlphaOfLength(randomInt(30));
         ActionFilters actionFilters = new ActionFilters(filters);
         TransportAction<TestRequest, TestResponse> transportAction = new TransportAction<TestRequest, TestResponse>(Settings.EMPTY, actionName, null, actionFilters, null, new TaskManager(Settings.EMPTY)) {
             @Override
@@ -137,92 +135,12 @@ public int compare(RequestTestFilter o1, RequestTestFilter o2) {
         }
     }
 
-    public void testActionFiltersResponse() throws ExecutionException, InterruptedException {
-        int numFilters = randomInt(10);
-        Set<Integer> orders = new HashSet<>(numFilters);
-        while (orders.size() < numFilters) {
-            orders.add(randomInt(10));
-        }
-
-        Set<ActionFilter> filters = new HashSet<>();
-        for (Integer order : orders) {
-            filters.add(new ResponseTestFilter(order, randomFrom(ResponseOperation.values())));
-        }
-
-        String actionName = randomAsciiOfLength(randomInt(30));
-        ActionFilters actionFilters = new ActionFilters(filters);
-        TransportAction<TestRequest, TestResponse> transportAction = new TransportAction<TestRequest, TestResponse>(Settings.EMPTY, actionName, null, actionFilters, null, new TaskManager(Settings.EMPTY)) {
-            @Override
-            protected void doExecute(TestRequest request, ActionListener<TestResponse> listener) {
-                listener.onResponse(new TestResponse());
-            }
-        };
-
-        ArrayList<ActionFilter> actionFiltersByOrder = new ArrayList<>(filters);
-        Collections.sort(actionFiltersByOrder, new Comparator<ActionFilter>() {
-            @Override
-            public int compare(ActionFilter o1, ActionFilter o2) {
-                return Integer.compare(o2.order(), o1.order());
-            }
-        });
-
-        List<ActionFilter> expectedActionFilters = new ArrayList<>();
-        boolean errorExpected = false;
-        for (ActionFilter filter : actionFiltersByOrder) {
-            ResponseTestFilter testFilter = (ResponseTestFilter) filter;
-            expectedActionFilters.add(testFilter);
-            if (testFilter.callback == ResponseOperation.LISTENER_FAILURE) {
-                errorExpected = true;
-            }
-            if (testFilter.callback != ResponseOperation.CONTINUE_PROCESSING) {
-                break;
-            }
-        }
-
-        PlainListenableActionFuture<TestResponse> future = new PlainListenableActionFuture<>(null);
-        transportAction.execute(new TestRequest(), future);
-        try {
-            assertThat(future.get(), notNullValue());
-            assertThat("shouldn't get here if an error is expected", errorExpected, equalTo(false));
-        } catch(ExecutionException e) {
-            assertThat("shouldn't get here if an error is not expected " + e.getMessage(), errorExpected, equalTo(true));
-        }
-
-        List<ResponseTestFilter> testFiltersByLastExecution = new ArrayList<>();
-        for (ActionFilter actionFilter : actionFilters.filters()) {
-            testFiltersByLastExecution.add((ResponseTestFilter) actionFilter);
-        }
-        Collections.sort(testFiltersByLastExecution, new Comparator<ResponseTestFilter>() {
-            @Override
-            public int compare(ResponseTestFilter o1, ResponseTestFilter o2) {
-                return Integer.compare(o1.executionToken, o2.executionToken);
-            }
-        });
-
-        ArrayList<ResponseTestFilter> finalTestFilters = new ArrayList<>();
-        for (ActionFilter filter : testFiltersByLastExecution) {
-            ResponseTestFilter testFilter = (ResponseTestFilter) filter;
-            finalTestFilters.add(testFilter);
-            if (testFilter.callback != ResponseOperation.CONTINUE_PROCESSING) {
-                break;
-            }
-        }
-
-        assertThat(finalTestFilters.size(), equalTo(expectedActionFilters.size()));
-        for (int i = 0; i < finalTestFilters.size(); i++) {
-            ResponseTestFilter testFilter = finalTestFilters.get(i);
-            assertThat(testFilter, equalTo(expectedActionFilters.get(i)));
-            assertThat(testFilter.runs.get(), equalTo(1));
-            assertThat(testFilter.lastActionName, equalTo(actionName));
-        }
-    }
-
     public void testTooManyContinueProcessingRequest() throws ExecutionException, InterruptedException {
         final int additionalContinueCount = randomInt(10);
 
         RequestTestFilter testFilter = new RequestTestFilter(randomInt(), new RequestCallback() {
             @Override
-            public <Request extends ActionRequest<Request>, Response extends ActionResponse> void execute(Task task, String action, Request request,
+            public <Request extends ActionRequest, Response extends ActionResponse> void execute(Task task, String action, Request request,
                     ActionListener<Response> listener, ActionFilterChain<Request, Response> actionFilterChain) {
                 for (int i = 0; i <= additionalContinueCount; i++) {
                     actionFilterChain.proceed(task, action, request, listener);
@@ -233,64 +151,7 @@ public <Request extends ActionRequest<Request>, Response extends ActionResponse>
         Set<ActionFilter> filters = new HashSet<>();
         filters.add(testFilter);
 
-        String actionName = randomAsciiOfLength(randomInt(30));
-        ActionFilters actionFilters = new ActionFilters(filters);
-        TransportAction<TestRequest, TestResponse> transportAction = new TransportAction<TestRequest, TestResponse>(Settings.EMPTY, actionName, null, actionFilters, null, new TaskManager(Settings.EMPTY)) {
-            @Override
-            protected void doExecute(TestRequest request, ActionListener<TestResponse> listener) {
-                listener.onResponse(new TestResponse());
-            }
-        };
-
-        final CountDownLatch latch = new CountDownLatch(additionalContinueCount + 1);
-        final AtomicInteger responses = new AtomicInteger();
-        final List<Throwable> failures = new CopyOnWriteArrayList<>();
-
-        transportAction.execute(new TestRequest(), new ActionListener<TestResponse>() {
-            @Override
-            public void onResponse(TestResponse testResponse) {
-                responses.incrementAndGet();
-                latch.countDown();
-            }
-
-            @Override
-            public void onFailure(Exception e) {
-                failures.add(e);
-                latch.countDown();
-            }
-        });
-
-        if (!latch.await(10, TimeUnit.SECONDS)) {
-            fail("timeout waiting for the filter to notify the listener as many times as expected");
-        }
-
-        assertThat(testFilter.runs.get(), equalTo(1));
-        assertThat(testFilter.lastActionName, equalTo(actionName));
-
-        assertThat(responses.get(), equalTo(1));
-        assertThat(failures.size(), equalTo(additionalContinueCount));
-        for (Throwable failure : failures) {
-            assertThat(failure, instanceOf(IllegalStateException.class));
-        }
-    }
-
-    public void testTooManyContinueProcessingResponse() throws ExecutionException, InterruptedException {
-        final int additionalContinueCount = randomInt(10);
-
-        ResponseTestFilter testFilter = new ResponseTestFilter(randomInt(), new ResponseCallback() {
-            @Override
-            public <Response extends ActionResponse> void execute(String action, Response response, ActionListener<Response> listener,
-                    ActionFilterChain<?, Response> chain) {
-                for (int i = 0; i <= additionalContinueCount; i++) {
-                    chain.proceed(action, response, listener);
-                }
-            }
-        });
-
-        Set<ActionFilter> filters = new HashSet<>();
-        filters.add(testFilter);
-
-        String actionName = randomAsciiOfLength(randomInt(30));
+        String actionName = randomAlphaOfLength(randomInt(30));
         ActionFilters actionFilters = new ActionFilters(filters);
         TransportAction<TestRequest, TestResponse> transportAction = new TransportAction<TestRequest, TestResponse>(Settings.EMPTY, actionName, null, actionFilters, null, new TaskManager(Settings.EMPTY)) {
             @Override
@@ -349,58 +210,19 @@ public int order() {
         }
 
         @Override
-        public <Request extends ActionRequest<Request>, Response extends ActionResponse> void apply(Task task, String action, Request request,
+        public <Request extends ActionRequest, Response extends ActionResponse> void apply(Task task, String action, Request request,
                 ActionListener<Response> listener, ActionFilterChain<Request, Response> chain) {
             this.runs.incrementAndGet();
             this.lastActionName = action;
             this.executionToken = counter.incrementAndGet();
             this.callback.execute(task, action, request, listener, chain);
         }
-
-        @Override
-        public <Response extends ActionResponse> void apply(String action, Response response, ActionListener<Response> listener,
-                ActionFilterChain<?, Response> chain) {
-            chain.proceed(action, response, listener);
-        }
-    }
-
-    private class ResponseTestFilter implements ActionFilter {
-        private final ResponseCallback callback;
-        private final int order;
-        AtomicInteger runs = new AtomicInteger();
-        volatile String lastActionName;
-        volatile int executionToken = Integer.MAX_VALUE; //the filters that don't run will go last in the sorted list
-
-        ResponseTestFilter(int order, ResponseCallback callback) {
-            this.order = order;
-            this.callback = callback;
-        }
-
-        @Override
-        public int order() {
-            return order;
-        }
-
-        @Override
-        public <Request extends ActionRequest<Request>, Response extends ActionResponse> void apply(Task task, String action, Request request,
-                ActionListener<Response> listener, ActionFilterChain<Request, Response> chain) {
-            chain.proceed(task, action, request, listener);
-        }
-
-        @Override
-        public <Response extends ActionResponse> void apply(String action, Response response, ActionListener<Response> listener,
-                ActionFilterChain<?, Response> chain) {
-            this.runs.incrementAndGet();
-            this.lastActionName = action;
-            this.executionToken = counter.incrementAndGet();
-            this.callback.execute(action, response, listener, chain);
-        }
     }
 
-    private static enum RequestOperation implements RequestCallback {
+    private enum RequestOperation implements RequestCallback {
         CONTINUE_PROCESSING {
             @Override
-            public <Request extends ActionRequest<Request>, Response extends ActionResponse> void execute(Task task, String action, Request request,
+            public <Request extends ActionRequest, Response extends ActionResponse> void execute(Task task, String action, Request request,
                     ActionListener<Response> listener, ActionFilterChain<Request, Response> actionFilterChain) {
                 actionFilterChain.proceed(task, action, request, listener);
             }
@@ -408,56 +230,26 @@ public <Request extends ActionRequest<Request>, Response extends ActionResponse>
         LISTENER_RESPONSE {
             @Override
             @SuppressWarnings("unchecked")  // Safe because its all we test with
-            public <Request extends ActionRequest<Request>, Response extends ActionResponse> void execute(Task task, String action, Request request,
+            public <Request extends ActionRequest, Response extends ActionResponse> void execute(Task task, String action, Request request,
                     ActionListener<Response> listener, ActionFilterChain<Request, Response> actionFilterChain) {
                 ((ActionListener<TestResponse>) listener).onResponse(new TestResponse());
             }
         },
         LISTENER_FAILURE {
             @Override
-            public <Request extends ActionRequest<Request>, Response extends ActionResponse> void execute(Task task, String action, Request request,
+            public <Request extends ActionRequest, Response extends ActionResponse> void execute(Task task, String action, Request request,
                     ActionListener<Response> listener, ActionFilterChain<Request, Response> actionFilterChain) {
                 listener.onFailure(new ElasticsearchTimeoutException(""));
             }
         }
     }
 
-    private static enum ResponseOperation implements ResponseCallback {
-        CONTINUE_PROCESSING {
-            @Override
-            public <Response extends ActionResponse> void execute(String action, Response response, ActionListener<Response> listener,
-                    ActionFilterChain<?, Response> chain) {
-                chain.proceed(action, response, listener);
-            }
-        },
-        LISTENER_RESPONSE {
-            @Override
-            @SuppressWarnings("unchecked") // Safe because its all we test with
-            public <Response extends ActionResponse> void execute(String action, Response response, ActionListener<Response> listener,
-                    ActionFilterChain<?, Response> chain) {
-                ((ActionListener<TestResponse>) listener).onResponse(new TestResponse());
-            }
-        },
-        LISTENER_FAILURE {
-            @Override
-            public <Response extends ActionResponse> void execute(String action, Response response, ActionListener<Response> listener,
-                    ActionFilterChain<?, Response> chain) {
-                listener.onFailure(new ElasticsearchTimeoutException(""));
-            }
-        }
-    }
-
     private interface RequestCallback {
-        <Request extends ActionRequest<Request>, Response extends ActionResponse> void execute(Task task, String action, Request request,
+        <Request extends ActionRequest, Response extends ActionResponse> void execute(Task task, String action, Request request,
                 ActionListener<Response> listener, ActionFilterChain<Request, Response> actionFilterChain);
     }
 
-    private interface ResponseCallback {
-        <Response extends ActionResponse> void execute(String action, Response response, ActionListener<Response> listener,
-                ActionFilterChain<?, Response> chain);
-    }
-
-    public static class TestRequest extends ActionRequest<TestRequest> {
+    public static class TestRequest extends ActionRequest {
         @Override
         public ActionRequestValidationException validate() {
             return null;
diff --git a/core/src/test/java/org/elasticsearch/action/support/WaitActiveShardCountIT.java b/core/src/test/java/org/elasticsearch/action/support/WaitActiveShardCountIT.java
index fbeac88dbe104..6a33b8ae1df65 100644
--- a/core/src/test/java/org/elasticsearch/action/support/WaitActiveShardCountIT.java
+++ b/core/src/test/java/org/elasticsearch/action/support/WaitActiveShardCountIT.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESIntegTestCase;
 
@@ -32,6 +33,7 @@
 import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.startsWith;
 
 /**
  * Tests setting the active shard count for replication operations (e.g. index) operates correctly.
@@ -44,15 +46,15 @@ public void testReplicationWaitsForActiveShardCount() throws Exception {
         assertAcked(createIndexResponse);
 
         // indexing, by default, will work (waiting for one shard copy only)
-        client().prepareIndex("test", "type1", "1").setSource(source("1", "test")).execute().actionGet();
+        client().prepareIndex("test", "type1", "1").setSource(source("1", "test"), XContentType.JSON).execute().actionGet();
         try {
-            client().prepareIndex("test", "type1", "1").setSource(source("1", "test"))
+            client().prepareIndex("test", "type1", "1").setSource(source("1", "test"), XContentType.JSON)
                     .setWaitForActiveShards(2) // wait for 2 active shard copies
                     .setTimeout(timeValueMillis(100)).execute().actionGet();
             fail("can't index, does not enough active shard copies");
         } catch (UnavailableShardsException e) {
             assertThat(e.status(), equalTo(RestStatus.SERVICE_UNAVAILABLE));
-            assertThat(e.getMessage(), equalTo("[test][0] Not enough active copies to meet shard count of [2] (have 1, needed 2). Timeout: [100ms], request: [index {[test][type1][1], source[{ \"type1\" : { \"id\" : \"1\", \"name\" : \"test\" } }]}]"));
+            assertThat(e.getMessage(), startsWith("[test][0] Not enough active copies to meet shard count of [2] (have 1, needed 2). Timeout: [100ms], request:"));
             // but really, all is well
         }
 
@@ -70,18 +72,18 @@ public void testReplicationWaitsForActiveShardCount() throws Exception {
         assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.YELLOW));
 
         // this should work, since we now have two
-        client().prepareIndex("test", "type1", "1").setSource(source("1", "test"))
+        client().prepareIndex("test", "type1", "1").setSource(source("1", "test"), XContentType.JSON)
                 .setWaitForActiveShards(2)
                 .setTimeout(timeValueSeconds(1)).execute().actionGet();
 
         try {
-            client().prepareIndex("test", "type1", "1").setSource(source("1", "test"))
+            client().prepareIndex("test", "type1", "1").setSource(source("1", "test"), XContentType.JSON)
                     .setWaitForActiveShards(ActiveShardCount.ALL)
                     .setTimeout(timeValueMillis(100)).execute().actionGet();
             fail("can't index, not enough active shard copies");
         } catch (UnavailableShardsException e) {
             assertThat(e.status(), equalTo(RestStatus.SERVICE_UNAVAILABLE));
-            assertThat(e.getMessage(), equalTo("[test][0] Not enough active copies to meet shard count of [" + ActiveShardCount.ALL + "] (have 2, needed 3). Timeout: [100ms], request: [index {[test][type1][1], source[{ \"type1\" : { \"id\" : \"1\", \"name\" : \"test\" } }]}]"));
+            assertThat(e.getMessage(), startsWith("[test][0] Not enough active copies to meet shard count of [" + ActiveShardCount.ALL + "] (have 2, needed 3). Timeout: [100ms], request:"));
             // but really, all is well
         }
 
@@ -92,7 +94,7 @@ public void testReplicationWaitsForActiveShardCount() throws Exception {
         assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.GREEN));
 
         // this should work, since we now have all shards started
-        client().prepareIndex("test", "type1", "1").setSource(source("1", "test"))
+        client().prepareIndex("test", "type1", "1").setSource(source("1", "test"), XContentType.JSON)
                 .setWaitForActiveShards(ActiveShardCount.ALL)
                 .setTimeout(timeValueSeconds(1)).execute().actionGet();
     }
diff --git a/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java b/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java
index 7b23738303410..2426e7bbf8c1b 100644
--- a/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java
@@ -117,7 +117,7 @@ public Response(int totalShards, int successfulShards, int failedShards, List<Sh
     class TestTransportBroadcastByNodeAction extends TransportBroadcastByNodeAction<Request, Response, TransportBroadcastByNodeAction.EmptyResult> {
         private final Map<ShardRouting, Object> shards = new HashMap<>();
 
-        public TestTransportBroadcastByNodeAction(Settings settings, TransportService transportService, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, Supplier<Request> request, String executor) {
+        TestTransportBroadcastByNodeAction(Settings settings, TransportService transportService, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, Supplier<Request> request, String executor) {
             super(settings, "indices:admin/test", THREAD_POOL, TransportBroadcastByNodeActionTests.this.clusterService, transportService, actionFilters, indexNameExpressionResolver, request, executor);
         }
 
@@ -171,7 +171,7 @@ public Map<ShardRouting, Object> getResults() {
     }
 
     class MyResolver extends IndexNameExpressionResolver {
-        public MyResolver() {
+        MyResolver() {
             super(Settings.EMPTY);
         }
 
@@ -191,7 +191,8 @@ public void setUp() throws Exception {
         super.setUp();
         transport = new CapturingTransport();
         clusterService = createClusterService(THREAD_POOL);
-        final TransportService transportService = new TransportService(clusterService.getSettings(), transport, THREAD_POOL);
+        final TransportService transportService = new TransportService(clusterService.getSettings(), transport, THREAD_POOL,
+            TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null);
         transportService.start();
         transportService.acceptIncomingRequests();
         setClusterState(clusterService, TEST_INDEX);
@@ -262,7 +263,7 @@ public void testGlobalBlock() {
         PlainActionFuture<Response> listener = new PlainActionFuture<>();
 
         ClusterBlocks.Builder block = ClusterBlocks.builder()
-                .addGlobalBlock(new ClusterBlock(1, "test-block", false, true, RestStatus.SERVICE_UNAVAILABLE, ClusterBlockLevel.ALL));
+                .addGlobalBlock(new ClusterBlock(1, "test-block", false, true, false, RestStatus.SERVICE_UNAVAILABLE, ClusterBlockLevel.ALL));
         setState(clusterService, ClusterState.builder(clusterService.state()).blocks(block));
         try {
             action.new AsyncAction(null, request, listener).start();
@@ -277,7 +278,7 @@ public void testRequestBlock() {
         PlainActionFuture<Response> listener = new PlainActionFuture<>();
 
         ClusterBlocks.Builder block = ClusterBlocks.builder()
-                .addIndexBlock(TEST_INDEX, new ClusterBlock(1, "test-block", false, true, RestStatus.SERVICE_UNAVAILABLE, ClusterBlockLevel.ALL));
+                .addIndexBlock(TEST_INDEX, new ClusterBlock(1, "test-block", false, true, false, RestStatus.SERVICE_UNAVAILABLE, ClusterBlockLevel.ALL));
         setState(clusterService, ClusterState.builder(clusterService.state()).blocks(block));
         try {
             action.new AsyncAction(null, request, listener).start();
@@ -296,7 +297,7 @@ public void testOneRequestIsSentToEachNodeHoldingAShard() {
 
         ShardsIterator shardIt = clusterService.state().routingTable().allShards(new String[]{TEST_INDEX});
         Set<String> set = new HashSet<>();
-        for (ShardRouting shard : shardIt.asUnordered()) {
+        for (ShardRouting shard : shardIt) {
             set.add(shard.currentNodeId());
         }
 
@@ -332,7 +333,7 @@ public void testRequestsAreNotSentToFailedMaster() {
         // the master should not be in the list of nodes that requests were sent to
         ShardsIterator shardIt = clusterService.state().routingTable().allShards(new String[]{TEST_INDEX});
         Set<String> set = new HashSet<>();
-        for (ShardRouting shard : shardIt.asUnordered()) {
+        for (ShardRouting shard : shardIt) {
             if (!shard.currentNodeId().equals(masterNode.getId())) {
                 set.add(shard.currentNodeId());
             }
@@ -352,8 +353,8 @@ public void testRequestsAreNotSentToFailedMaster() {
     public void testOperationExecution() throws Exception {
         ShardsIterator shardIt = clusterService.state().routingTable().allShards(new String[]{TEST_INDEX});
         Set<ShardRouting> shards = new HashSet<>();
-        String nodeId = shardIt.asUnordered().iterator().next().currentNodeId();
-        for (ShardRouting shard : shardIt.asUnordered()) {
+        String nodeId = shardIt.iterator().next().currentNodeId();
+        for (ShardRouting shard : shardIt) {
             if (nodeId.equals(shard.currentNodeId())) {
                 shards.add(shard);
             }
@@ -417,7 +418,7 @@ public void testResultAggregation() throws ExecutionException, InterruptedExcept
 
         ShardsIterator shardIt = clusterService.state().getRoutingTable().allShards(new String[]{TEST_INDEX});
         Map<String, List<ShardRouting>> map = new HashMap<>();
-        for (ShardRouting shard : shardIt.asUnordered()) {
+        for (ShardRouting shard : shardIt) {
             if (!map.containsKey(shard.currentNodeId())) {
                 map.put(shard.currentNodeId(), new ArrayList<>());
             }
diff --git a/core/src/test/java/org/elasticsearch/action/support/master/IndexingMasterFailoverIT.java b/core/src/test/java/org/elasticsearch/action/support/master/IndexingMasterFailoverIT.java
index b30a343547973..ded42cc2354b4 100644
--- a/core/src/test/java/org/elasticsearch/action/support/master/IndexingMasterFailoverIT.java
+++ b/core/src/test/java/org/elasticsearch/action/support/master/IndexingMasterFailoverIT.java
@@ -23,10 +23,11 @@
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.discovery.DiscoverySettings;
-import org.elasticsearch.discovery.zen.elect.ElectMasterService;
-import org.elasticsearch.discovery.zen.fd.FaultDetection;
+import org.elasticsearch.discovery.zen.ElectMasterService;
+import org.elasticsearch.discovery.zen.FaultDetection;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.discovery.TestZenDiscovery;
 import org.elasticsearch.test.disruption.NetworkDisruption;
 import org.elasticsearch.test.disruption.NetworkDisruption.NetworkDisconnect;
 import org.elasticsearch.test.disruption.NetworkDisruption.TwoPartitions;
@@ -42,7 +43,7 @@
 
 import static org.hamcrest.Matchers.equalTo;
 
-@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0)
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoMinMasterNodes = false)
 @ESIntegTestCase.SuppressLocalMode
 public class IndexingMasterFailoverIT extends ESIntegTestCase {
 
@@ -53,6 +54,12 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
         return classes;
     }
 
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        return Settings.builder().put(super.nodeSettings(nodeOrdinal))
+            .put(TestZenDiscovery.USE_MOCK_PINGS.getKey(), false).build();
+    }
+
     /**
      * Indexing operations which entail mapping changes require a blocking request to the master node to update the mapping.
      * If the master node is being disrupted or if it cannot commit cluster state changes, it needs to retry within timeout limits.
@@ -69,7 +76,7 @@ public void testMasterFailoverDuringIndexingWithMappingChanges() throws Throwabl
                 .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), 2)
                 .build();
 
-        internalCluster().startMasterOnlyNodesAsync(3, sharedSettings).get();
+        internalCluster().startMasterOnlyNodes(3, sharedSettings);
 
         String dataNode = internalCluster().startDataOnlyNode(sharedSettings);
 
diff --git a/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java b/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java
index 9aeafcac0e4c0..ccc09e744d7c3 100644
--- a/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java
@@ -36,6 +36,7 @@
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.LocalTransportAddress;
@@ -87,7 +88,8 @@ public void setUp() throws Exception {
         super.setUp();
         transport = new CapturingTransport();
         clusterService = createClusterService(threadPool);
-        transportService = new TransportService(clusterService.getSettings(), transport, threadPool);
+        transportService = new TransportService(clusterService.getSettings(), transport, threadPool,
+                TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null);
         transportService.start();
         transportService.acceptIncomingRequests();
         localNode = new DiscoveryNode("local_node", LocalTransportAddress.buildUnique(), Collections.emptyMap(),
@@ -205,7 +207,7 @@ public void testLocalOperationWithBlocks() throws ExecutionException, Interrupte
         PlainActionFuture<Response> listener = new PlainActionFuture<>();
 
         ClusterBlock block = new ClusterBlock(1, "", retryableBlock, true,
-                randomFrom(RestStatus.values()), ClusterBlockLevel.ALL);
+            false, randomFrom(RestStatus.values()), ClusterBlockLevel.ALL);
         ClusterState stateWithBlock = ClusterState.builder(ClusterStateCreationUtils.state(localNode, localNode, allNodes))
                 .blocks(ClusterBlocks.builder().addGlobalBlock(block)).build();
         setState(clusterService, stateWithBlock);
@@ -298,21 +300,57 @@ public void testDelegateToMaster() throws ExecutionException, InterruptedExcepti
     }
 
     public void testDelegateToFailingMaster() throws ExecutionException, InterruptedException {
-        boolean failsWithConnectTransportException = randomBoolean();
+        final boolean failsWithConnectTransportException = randomBoolean();
+        final boolean rejoinSameMaster = failsWithConnectTransportException && randomBoolean();
         Request request = new Request().masterNodeTimeout(TimeValue.timeValueSeconds(failsWithConnectTransportException ? 60 : 0));
-        setState(clusterService, ClusterStateCreationUtils.state(localNode, remoteNode, allNodes));
+        DiscoveryNode masterNode = this.remoteNode;
+        setState(clusterService, ClusterState.builder(ClusterStateCreationUtils.state(localNode, masterNode, allNodes))
+            .version(randomIntBetween(0, 10))); // use a random base version so it can go down when simulating a restart.
 
         PlainActionFuture<Response> listener = new PlainActionFuture<>();
         new Action(Settings.EMPTY, "testAction", transportService, clusterService, threadPool).execute(request, listener);
 
-        assertThat(transport.capturedRequests().length, equalTo(1));
-        CapturingTransport.CapturedRequest capturedRequest = transport.capturedRequests()[0];
+        CapturingTransport.CapturedRequest[] capturedRequests = transport.getCapturedRequestsAndClear();
+        assertThat(capturedRequests.length, equalTo(1));
+        CapturingTransport.CapturedRequest capturedRequest = capturedRequests[0];
         assertTrue(capturedRequest.node.isMasterNode());
         assertThat(capturedRequest.request, equalTo(request));
         assertThat(capturedRequest.action, equalTo("testAction"));
 
-        if (failsWithConnectTransportException) {
-            transport.handleRemoteError(capturedRequest.requestId, new ConnectTransportException(remoteNode, "Fake error"));
+        if (rejoinSameMaster) {
+            transport.handleRemoteError(capturedRequest.requestId, new ConnectTransportException(masterNode, "Fake error"));
+            assertFalse(listener.isDone());
+            if (randomBoolean()) {
+                // simulate master node removal
+                final DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(clusterService.state().nodes());
+                nodesBuilder.masterNodeId(null);
+                setState(clusterService, ClusterState.builder(clusterService.state()).nodes(nodesBuilder));
+            }
+            if (randomBoolean()) {
+                // reset the same state to increment a version simulating a join of an existing node
+                // simulating use being disconnected
+                final DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(clusterService.state().nodes());
+                nodesBuilder.masterNodeId(masterNode.getId());
+                setState(clusterService, ClusterState.builder(clusterService.state()).nodes(nodesBuilder));
+            } else {
+                // simulate master restart followed by a state recovery - this will reset the cluster state version
+                final DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(clusterService.state().nodes());
+                nodesBuilder.remove(masterNode);
+                masterNode = new DiscoveryNode(masterNode.getId(), masterNode.getAddress(), masterNode.getVersion());
+                nodesBuilder.add(masterNode);
+                nodesBuilder.masterNodeId(masterNode.getId());
+                final ClusterState.Builder builder = ClusterState.builder(clusterService.state()).nodes(nodesBuilder);
+                setState(clusterService, builder.version(0));
+            }
+            assertFalse(listener.isDone());
+            capturedRequests = transport.getCapturedRequestsAndClear();
+            assertThat(capturedRequests.length, equalTo(1));
+            capturedRequest = capturedRequests[0];
+            assertTrue(capturedRequest.node.isMasterNode());
+            assertThat(capturedRequest.request, equalTo(request));
+            assertThat(capturedRequest.action, equalTo("testAction"));
+        } else if (failsWithConnectTransportException) {
+            transport.handleRemoteError(capturedRequest.requestId, new ConnectTransportException(masterNode, "Fake error"));
             assertFalse(listener.isDone());
             setState(clusterService, ClusterStateCreationUtils.state(localNode, localNode, allNodes));
             assertTrue(listener.isDone());
diff --git a/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java b/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java
index 744a116f4a7c7..f46b40f4dcaa2 100644
--- a/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java
@@ -58,7 +58,6 @@
 
 import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
 import static org.elasticsearch.test.ClusterServiceUtils.setState;
-import static org.mockito.Mockito.mock;
 
 public class TransportNodesActionTests extends ESTestCase {
 
@@ -107,12 +106,17 @@ public void testNewResponseNullArray() {
     public void testNewResponse() {
         TestTransportNodesAction action = getTestTransportNodesAction();
         TestNodesRequest request = new TestNodesRequest();
-        List<TestNodeResponse> expectedNodeResponses = mockList(TestNodeResponse.class, randomIntBetween(0, 2));
+        List<TestNodeResponse> expectedNodeResponses = mockList(TestNodeResponse::new, randomIntBetween(0, 2));
         expectedNodeResponses.add(new TestNodeResponse());
         List<BaseNodeResponse> nodeResponses = new ArrayList<>(expectedNodeResponses);
         // This should be ignored:
         nodeResponses.add(new OtherNodeResponse());
-        List<FailedNodeException> failures = mockList(FailedNodeException.class, randomIntBetween(0, 2));
+        List<FailedNodeException> failures = mockList(
+            () -> new FailedNodeException(
+                randomAlphaOfLength(8),
+                randomAlphaOfLength(8),
+                new IllegalStateException(randomAlphaOfLength(8))),
+            randomIntBetween(0, 2));
 
         List<Object> allResponses = new ArrayList<>(expectedNodeResponses);
         allResponses.addAll(failures);
@@ -142,10 +146,10 @@ public void testCustomResolving() throws Exception {
         assertEquals(clusterService.state().nodes().getDataNodes().size(), capturedRequests.size());
     }
 
-    private <T> List<T> mockList(Class<T> clazz, int size) {
+    private <T> List<T> mockList(Supplier<T> supplier, int size) {
         List<T> failures = new ArrayList<>(size);
         for (int i = 0; i < size; ++i) {
-            failures.add(mock(clazz));
+            failures.add(supplier.get());
         }
         return failures;
     }
@@ -177,7 +181,8 @@ public void setUp() throws Exception {
         super.setUp();
         transport = new CapturingTransport();
         clusterService = createClusterService(THREAD_POOL);
-        transportService = new TransportService(clusterService.getSettings(), transport, THREAD_POOL);
+        transportService = new TransportService(clusterService.getSettings(), transport, THREAD_POOL,
+            TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null);
         transportService.start();
         transportService.acceptIncomingRequests();
         int numNodes = randomIntBetween(3, 10);
@@ -187,7 +192,7 @@ public void setUp() throws Exception {
             Map<String, String> attributes = new HashMap<>();
             Set<DiscoveryNode.Role> roles = new HashSet<>(randomSubsetOf(Arrays.asList(DiscoveryNode.Role.values())));
             if (frequently()) {
-                attributes.put("custom", randomBoolean() ? "match" : randomAsciiOfLengthBetween(3, 5));
+                attributes.put("custom", randomBoolean() ? "match" : randomAlphaOfLengthBetween(3, 5));
             }
             final DiscoveryNode node = newNode(i, attributes, roles);
             discoBuilder = discoBuilder.add(node);
@@ -270,10 +275,6 @@ protected TestNodeResponse nodeOperation(TestNodeRequest request) {
             return new TestNodeResponse();
         }
 
-        @Override
-        protected boolean accumulateExceptions() {
-            return false;
-        }
     }
 
     private static class DataNodesOnlyTransportNodesAction
diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java b/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java
index b5b2cbeb737a5..5de5bb7746b42 100644
--- a/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java
@@ -18,7 +18,6 @@
  */
 package org.elasticsearch.action.support.replication;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.NoShardAvailableActionException;
 import org.elasticsearch.action.ShardOperationFailedException;
@@ -26,7 +25,6 @@
 import org.elasticsearch.action.admin.indices.flush.FlushRequest;
 import org.elasticsearch.action.admin.indices.flush.FlushResponse;
 import org.elasticsearch.action.admin.indices.flush.TransportFlushAction;
-import org.elasticsearch.action.support.ActionFilter;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.broadcast.BroadcastRequest;
 import org.elasticsearch.action.support.broadcast.BroadcastResponse;
@@ -92,10 +90,12 @@ public void setUp() throws Exception {
         super.setUp();
         LocalTransport transport = new LocalTransport(Settings.EMPTY, threadPool, new NamedWriteableRegistry(Collections.emptyList()), circuitBreakerService);
         clusterService = createClusterService(threadPool);
-        transportService = new TransportService(clusterService.getSettings(), transport, threadPool);
+        transportService = new TransportService(clusterService.getSettings(), transport, threadPool,
+                TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null);
         transportService.start();
         transportService.acceptIncomingRequests();
-        broadcastReplicationAction = new TestBroadcastReplicationAction(Settings.EMPTY, threadPool, clusterService, transportService, new ActionFilters(new HashSet<ActionFilter>()), new IndexNameExpressionResolver(Settings.EMPTY), null);
+        broadcastReplicationAction = new TestBroadcastReplicationAction(Settings.EMPTY, threadPool, clusterService, transportService,
+                new ActionFilters(new HashSet<>()), new IndexNameExpressionResolver(Settings.EMPTY), null);
     }
 
     @After
@@ -115,7 +115,7 @@ public void testNotStartedPrimary() throws InterruptedException, ExecutionExcept
         final String index = "test";
         setState(clusterService, state(index, randomBoolean(),
                 randomBoolean() ? ShardRoutingState.INITIALIZING : ShardRoutingState.UNASSIGNED, ShardRoutingState.UNASSIGNED));
-        logger.debug("--> using initial state:\n{}", clusterService.state().prettyPrint());
+        logger.debug("--> using initial state:\n{}", clusterService.state());
         Future<BroadcastResponse> response = (broadcastReplicationAction.execute(new DummyBroadcastRequest().indices(index)));
         for (Tuple<ShardId, ActionListener<ReplicationResponse>> shardRequests : broadcastReplicationAction.capturedShardRequests) {
             if (randomBoolean()) {
@@ -134,11 +134,11 @@ public void testStartedPrimary() throws InterruptedException, ExecutionException
         final String index = "test";
         setState(clusterService, state(index, randomBoolean(),
                 ShardRoutingState.STARTED));
-        logger.debug("--> using initial state:\n{}", clusterService.state().prettyPrint());
+        logger.debug("--> using initial state:\n{}", clusterService.state());
         Future<BroadcastResponse> response = (broadcastReplicationAction.execute(new DummyBroadcastRequest().indices(index)));
         for (Tuple<ShardId, ActionListener<ReplicationResponse>> shardRequests : broadcastReplicationAction.capturedShardRequests) {
             ReplicationResponse replicationResponse = new ReplicationResponse();
-            replicationResponse.setShardInfo(new ReplicationResponse.ShardInfo(1, 1, new ReplicationResponse.ShardInfo.Failure[0]));
+            replicationResponse.setShardInfo(new ReplicationResponse.ShardInfo(1, 1));
             shardRequests.v2().onResponse(replicationResponse);
         }
         logger.info("total shards: {}, ", response.get().getTotalShards());
@@ -149,7 +149,7 @@ public void testResultCombine() throws InterruptedException, ExecutionException,
         final String index = "test";
         int numShards = 1 + randomInt(3);
         setState(clusterService, stateWithAssignedPrimariesAndOneReplica(index, numShards));
-        logger.debug("--> using initial state:\n{}", clusterService.state().prettyPrint());
+        logger.debug("--> using initial state:\n{}", clusterService.state());
         Future<BroadcastResponse> response = (broadcastReplicationAction.execute(new DummyBroadcastRequest().indices(index)));
         int succeeded = 0;
         int failed = 0;
@@ -179,7 +179,7 @@ public void testResultCombine() throws InterruptedException, ExecutionException,
 
     public void testNoShards() throws InterruptedException, ExecutionException, IOException {
         setState(clusterService, stateWithNoShard());
-        logger.debug("--> using initial state:\n{}", clusterService.state().prettyPrint());
+        logger.debug("--> using initial state:\n{}", clusterService.state());
         BroadcastResponse response = executeAndAssertImmediateResponse(broadcastReplicationAction, new DummyBroadcastRequest());
         assertBroadcastResponse(0, 0, 0, response, null);
     }
@@ -189,7 +189,7 @@ public void testShardsList() throws InterruptedException, ExecutionException {
         final ShardId shardId = new ShardId(index, "_na_", 0);
         ClusterState clusterState = state(index, randomBoolean(),
                 randomBoolean() ? ShardRoutingState.INITIALIZING : ShardRoutingState.UNASSIGNED, ShardRoutingState.UNASSIGNED);
-        logger.debug("--> using initial state:\n{}", clusterService.state().prettyPrint());
+        logger.debug("--> using initial state:\n{}", clusterService.state());
         List<ShardId> shards = broadcastReplicationAction.shards(new DummyBroadcastRequest().indices(shardId.getIndexName()), clusterState);
         assertThat(shards.size(), equalTo(1));
         assertThat(shards.get(0), equalTo(shardId));
@@ -198,7 +198,7 @@ public void testShardsList() throws InterruptedException, ExecutionException {
     private class TestBroadcastReplicationAction extends TransportBroadcastReplicationAction<DummyBroadcastRequest, BroadcastResponse, BasicReplicationRequest, ReplicationResponse> {
         protected final Set<Tuple<ShardId, ActionListener<ReplicationResponse>>> capturedShardRequests = ConcurrentCollections.newConcurrentSet();
 
-        public TestBroadcastReplicationAction(Settings settings, ThreadPool threadPool, ClusterService clusterService,
+        TestBroadcastReplicationAction(Settings settings, ThreadPool threadPool, ClusterService clusterService,
                                               TransportService transportService, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
                                               TransportReplicationAction replicatedBroadcastShardAction) {
             super("test-broadcast-replication-action", DummyBroadcastRequest::new, settings, threadPool, clusterService, transportService,
diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java b/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java
index 646ad23a48b27..c0fa00dc90e41 100644
--- a/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java
+++ b/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java
@@ -37,9 +37,11 @@
 import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.index.shard.ShardId;
 
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Set;
 
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_CREATION_DATE;
@@ -144,6 +146,96 @@ public static ClusterState state(String index, boolean activePrimaryLocal, Shard
         return state.build();
     }
 
+    /**
+     * Creates cluster state with an index that has #(numberOfPrimaries) primary shards in the started state and no replicas.
+     * The cluster state contains #(numberOfNodes) nodes and assigns primaries to those nodes.
+     */
+    public static ClusterState state(String index, final int numberOfNodes, final int numberOfPrimaries) {
+        DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder();
+        Set<String> nodes = new HashSet<>();
+        for (int i = 0; i < numberOfNodes; i++) {
+            final DiscoveryNode node = newNode(i);
+            discoBuilder = discoBuilder.add(node);
+            nodes.add(node.getId());
+        }
+        discoBuilder.localNodeId(newNode(0).getId());
+        discoBuilder.masterNodeId(randomFrom(nodes));
+        IndexMetaData indexMetaData = IndexMetaData.builder(index).settings(Settings.builder()
+            .put(SETTING_VERSION_CREATED, Version.CURRENT)
+            .put(SETTING_NUMBER_OF_SHARDS, numberOfPrimaries).put(SETTING_NUMBER_OF_REPLICAS, 0)
+            .put(SETTING_CREATION_DATE, System.currentTimeMillis())).build();
+
+        RoutingTable.Builder routing = new RoutingTable.Builder();
+        routing.addAsNew(indexMetaData);
+
+        IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(indexMetaData.getIndex());
+        for (int i = 0; i < numberOfPrimaries; i++) {
+            ShardId shardId = new ShardId(indexMetaData.getIndex(), i);
+            IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(shardId);
+            indexShardRoutingBuilder.addShard(
+                TestShardRouting.newShardRouting(shardId, randomFrom(nodes), true, ShardRoutingState.STARTED));
+            indexRoutingTable.addIndexShard(indexShardRoutingBuilder.build());
+        }
+
+        ClusterState.Builder state = ClusterState.builder(new ClusterName("test"));
+        state.nodes(discoBuilder);
+        state.metaData(MetaData.builder().put(indexMetaData, false).generateClusterUuidIfNeeded());
+        state.routingTable(RoutingTable.builder().add(indexRoutingTable).build());
+        return state.build();
+    }
+
+
+
+    /**
+     * Creates cluster state with the given indices, each index containing #(numberOfPrimaries)
+     * started primary shards and no replicas.  The cluster state contains #(numberOfNodes) nodes
+     * and assigns primaries to those nodes.
+     */
+    public static ClusterState state(final int numberOfNodes, final String[] indices, final int numberOfPrimaries) {
+        DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder();
+        Set<String> nodes = new HashSet<>();
+        for (int i = 0; i < numberOfNodes; i++) {
+            final DiscoveryNode node = newNode(i);
+            discoBuilder = discoBuilder.add(node);
+            nodes.add(node.getId());
+        }
+        discoBuilder.localNodeId(newNode(0).getId());
+        discoBuilder.masterNodeId(newNode(0).getId());
+        MetaData.Builder metaData = MetaData.builder();
+        RoutingTable.Builder routingTable = RoutingTable.builder();
+        List<String> nodesList = new ArrayList<>(nodes);
+        int currentNodeToAssign = 0;
+        for (String index : indices) {
+            IndexMetaData indexMetaData = IndexMetaData.builder(index).settings(Settings.builder()
+                .put(SETTING_VERSION_CREATED, Version.CURRENT)
+                .put(SETTING_NUMBER_OF_SHARDS, numberOfPrimaries).put(SETTING_NUMBER_OF_REPLICAS, 0)
+                .put(SETTING_CREATION_DATE, System.currentTimeMillis())).build();
+
+            RoutingTable.Builder routing = new RoutingTable.Builder();
+            routing.addAsNew(indexMetaData);
+
+            IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(indexMetaData.getIndex());
+            for (int i = 0; i < numberOfPrimaries; i++) {
+                ShardId shardId = new ShardId(indexMetaData.getIndex(), i);
+                IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(shardId);
+                indexShardRoutingBuilder.addShard(
+                    TestShardRouting.newShardRouting(shardId, nodesList.get(currentNodeToAssign++), true, ShardRoutingState.STARTED));
+                if (currentNodeToAssign == nodesList.size()) {
+                    currentNodeToAssign = 0;
+                }
+                indexRoutingTable.addIndexShard(indexShardRoutingBuilder.build());
+            }
+
+            metaData.put(indexMetaData, false);
+            routingTable.add(indexRoutingTable);
+        }
+        ClusterState.Builder state = ClusterState.builder(new ClusterName("test"));
+        state.nodes(discoBuilder);
+        state.metaData(metaData.generateClusterUuidIfNeeded().build());
+        state.routingTable(routingTable.build());
+        return state.build();
+    }
+
     /**
      * Creates cluster state with several shards and one replica and all shards STARTED.
      */
diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/ReplicationOperationTests.java b/core/src/test/java/org/elasticsearch/action/support/replication/ReplicationOperationTests.java
index b5edc1b53c5fd..b261639627592 100644
--- a/core/src/test/java/org/elasticsearch/action/support/replication/ReplicationOperationTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/replication/ReplicationOperationTests.java
@@ -111,12 +111,11 @@ public void testReplication() throws Exception {
         Request request = new Request(shardId);
         PlainActionFuture<TestPrimary.Result> listener = new PlainActionFuture<>();
         final ClusterState finalState = state;
-        final TestReplicaProxy replicasProxy = new TestReplicaProxy(expectedFailures);
+        final TestReplicaProxy replicasProxy = new TestReplicaProxy(primaryTerm, expectedFailures);
         final TestReplicationOperation op = new TestReplicationOperation(request,
-            new TestPrimary(primaryShard, primaryTerm), listener, replicasProxy, () -> finalState);
+            new TestPrimary(primaryShard), listener, replicasProxy, () -> finalState);
         op.execute();
 
-        assertThat(request.primaryTerm(), equalTo(primaryTerm));
         assertThat("request was not processed on primary", request.processedOnPrimary.get(), equalTo(true));
         assertThat(request.processedOnReplicas, equalTo(expectedReplicas));
         assertThat(replicasProxy.failedReplicas, equalTo(expectedFailedShards));
@@ -145,8 +144,8 @@ public void testReplicationWithShadowIndex() throws Exception {
         Request request = new Request(shardId);
         PlainActionFuture<TestPrimary.Result> listener = new PlainActionFuture<>();
         final TestReplicationOperation op = new TestReplicationOperation(request,
-            new TestPrimary(primaryShard, primaryTerm), listener, false,
-            new TestReplicaProxy(), () -> state, logger, "test");
+            new TestPrimary(primaryShard), listener, false,
+            new TestReplicaProxy(primaryTerm), () -> state, logger, "test");
         op.execute();
         assertThat("request was not processed on primary", request.processedOnPrimary.get(), equalTo(true));
         assertThat(request.processedOnReplicas, equalTo(Collections.emptySet()));
@@ -175,7 +174,7 @@ public void testDemotedPrimary() throws Exception {
         }
         // add in-sync allocation id that doesn't have a corresponding routing entry
         state = ClusterState.builder(state).metaData(MetaData.builder(state.metaData()).put(IndexMetaData.builder(indexMetaData)
-            .putInSyncAllocationIds(0, Sets.union(indexMetaData.inSyncAllocationIds(0), Sets.newHashSet(randomAsciiOfLength(10))))))
+            .putInSyncAllocationIds(0, Sets.union(indexMetaData.inSyncAllocationIds(0), Sets.newHashSet(randomAlphaOfLength(10))))))
             .build();
 
         final Set<ShardRouting> expectedReplicas = getExpectedReplicas(shardId, state);
@@ -188,13 +187,13 @@ public void testDemotedPrimary() throws Exception {
         PlainActionFuture<TestPrimary.Result> listener = new PlainActionFuture<>();
         final ClusterState finalState = state;
         final boolean testPrimaryDemotedOnStaleShardCopies = randomBoolean();
-        final TestReplicaProxy replicasProxy = new TestReplicaProxy(expectedFailures) {
+        final TestReplicaProxy replicasProxy = new TestReplicaProxy(primaryTerm, expectedFailures) {
             @Override
-            public void failShard(ShardRouting replica, long primaryTerm, String message, Exception exception,
+            public void failShard(ShardRouting replica, String message, Exception exception,
                                   Runnable onSuccess, Consumer<Exception> onPrimaryDemoted,
                                   Consumer<Exception> onIgnoredFailure) {
                 if (testPrimaryDemotedOnStaleShardCopies) {
-                    super.failShard(replica, primaryTerm, message, exception, onSuccess, onPrimaryDemoted, onIgnoredFailure);
+                    super.failShard(replica, message, exception, onSuccess, onPrimaryDemoted, onIgnoredFailure);
                 } else {
                     assertThat(replica, equalTo(failedReplica));
                     onPrimaryDemoted.accept(new ElasticsearchException("the king is dead"));
@@ -202,17 +201,17 @@ public void failShard(ShardRouting replica, long primaryTerm, String message, Ex
             }
 
             @Override
-            public void markShardCopyAsStale(ShardId shardId, String allocationId, long primaryTerm, Runnable onSuccess,
+            public void markShardCopyAsStale(ShardId shardId, String allocationId, Runnable onSuccess,
                                              Consumer<Exception> onPrimaryDemoted, Consumer<Exception> onIgnoredFailure) {
                 if (testPrimaryDemotedOnStaleShardCopies) {
                     onPrimaryDemoted.accept(new ElasticsearchException("the king is dead"));
                 } else {
-                    super.markShardCopyAsStale(shardId, allocationId, primaryTerm, onSuccess, onPrimaryDemoted, onIgnoredFailure);
+                    super.markShardCopyAsStale(shardId, allocationId, onSuccess, onPrimaryDemoted, onIgnoredFailure);
                 }
             }
         };
         AtomicBoolean primaryFailed = new AtomicBoolean();
-        final TestPrimary primary = new TestPrimary(primaryShard, primaryTerm) {
+        final TestPrimary primary = new TestPrimary(primaryShard) {
             @Override
             public void failShard(String message, Exception exception) {
                 assertTrue(primaryFailed.compareAndSet(false, true));
@@ -256,15 +255,15 @@ private void testClusterStateChangeAfterPrimaryOperation(final ShardId shardId,
                                                              final ClusterState initialState,
                                                              final ClusterState changedState) throws Exception {
         AtomicReference<ClusterState> state = new AtomicReference<>(initialState);
-        logger.debug("--> using initial state:\n{}", state.get().prettyPrint());
+        logger.debug("--> using initial state:\n{}", state.get());
         final long primaryTerm = initialState.getMetaData().index(shardId.getIndexName()).primaryTerm(shardId.id());
         final ShardRouting primaryShard = state.get().routingTable().shardRoutingTable(shardId).primaryShard();
-        final TestPrimary primary = new TestPrimary(primaryShard, primaryTerm) {
+        final TestPrimary primary = new TestPrimary(primaryShard) {
             @Override
             public Result perform(Request request) throws Exception {
                 Result result = super.perform(request);
                 state.set(changedState);
-                logger.debug("--> state after primary operation:\n{}", state.get().prettyPrint());
+                logger.debug("--> state after primary operation:\n{}", state.get());
                 return result;
             }
         };
@@ -272,7 +271,7 @@ public Result perform(Request request) throws Exception {
         Request request = new Request(shardId);
         PlainActionFuture<TestPrimary.Result> listener = new PlainActionFuture<>();
         final TestReplicationOperation op = new TestReplicationOperation(request, primary, listener,
-            new TestReplicaProxy(), state::get);
+            new TestReplicaProxy(primaryTerm), state::get);
         op.execute();
 
         assertThat("request was not processed on primary", request.processedOnPrimary.get(), equalTo(true));
@@ -303,15 +302,14 @@ public void testWaitForActiveShards() throws Exception {
         logger.debug("using active shard count of [{}], assigned shards [{}], total shards [{}]." +
                 " expecting op to [{}]. using state: \n{}",
             request.waitForActiveShards(), 1 + assignedReplicas, 1 + assignedReplicas + unassignedReplicas,
-            passesActiveShardCheck ? "succeed" : "retry",
-            state.prettyPrint());
+            passesActiveShardCheck ? "succeed" : "retry", state);
         final long primaryTerm = state.metaData().index(index).primaryTerm(shardId.id());
         final IndexShardRoutingTable shardRoutingTable = state.routingTable().index(index).shard(shardId.id());
         PlainActionFuture<TestPrimary.Result> listener = new PlainActionFuture<>();
         final ShardRouting primaryShard = shardRoutingTable.primaryShard();
         final TestReplicationOperation op = new TestReplicationOperation(request,
-            new TestPrimary(primaryShard, primaryTerm),
-            listener, randomBoolean(), new TestReplicaProxy(), () -> state, logger, "test");
+            new TestPrimary(primaryShard),
+            listener, randomBoolean(), new TestReplicaProxy(primaryTerm), () -> state, logger, "test");
 
         if (passesActiveShardCheck) {
             assertThat(op.checkActiveShardCount(), nullValue());
@@ -376,11 +374,9 @@ public void readFrom(StreamInput in) throws IOException {
 
     static class TestPrimary implements ReplicationOperation.Primary<Request, Request, TestPrimary.Result> {
         final ShardRouting routing;
-        final long term;
 
-        TestPrimary(ShardRouting routing, long term) {
+        TestPrimary(ShardRouting routing) {
             this.routing = routing;
-            this.term = term;
         }
 
         @Override
@@ -398,7 +394,6 @@ public Result perform(Request request) throws Exception {
             if (request.processedOnPrimary.compareAndSet(false, true) == false) {
                 fail("processed [" + request + "] twice");
             }
-            request.primaryTerm(term);
             return new Result(request);
         }
 
@@ -406,7 +401,7 @@ static class Result implements ReplicationOperation.PrimaryResult<Request> {
             private final Request request;
             private ShardInfo shardInfo;
 
-            public Result(Request request) {
+            Result(Request request) {
                 this.request = request;
             }
 
@@ -434,11 +429,14 @@ static class TestReplicaProxy implements ReplicationOperation.Replicas<Request>
 
         final Set<String> markedAsStaleCopies = ConcurrentCollections.newConcurrentSet();
 
-        TestReplicaProxy() {
-            this(Collections.emptyMap());
+        final long primaryTerm;
+
+        TestReplicaProxy(long primaryTerm) {
+            this(primaryTerm, Collections.emptyMap());
         }
 
-        TestReplicaProxy(Map<ShardRouting, Exception> opFailures) {
+        TestReplicaProxy(long primaryTerm, Map<ShardRouting, Exception> opFailures) {
+            this.primaryTerm = primaryTerm;
             this.opFailures = opFailures;
         }
 
@@ -453,7 +451,7 @@ public void performOn(ShardRouting replica, Request request, ActionListener<Tran
         }
 
         @Override
-        public void failShard(ShardRouting replica, long primaryTerm, String message, Exception exception, Runnable onSuccess,
+        public void failShard(ShardRouting replica, String message, Exception exception, Runnable onSuccess,
                               Consumer<Exception> onPrimaryDemoted, Consumer<Exception> onIgnoredFailure) {
             if (failedReplicas.add(replica) == false) {
                 fail("replica [" + replica + "] was failed twice");
@@ -470,7 +468,7 @@ public void failShard(ShardRouting replica, long primaryTerm, String message, Ex
         }
 
         @Override
-        public void markShardCopyAsStale(ShardId shardId, String allocationId, long primaryTerm, Runnable onSuccess,
+        public void markShardCopyAsStale(ShardId shardId, String allocationId, Runnable onSuccess,
                                          Consumer<Exception> onPrimaryDemoted, Consumer<Exception> onIgnoredFailure) {
             if (markedAsStaleCopies.add(allocationId) == false) {
                 fail("replica [" + allocationId + "] was marked as stale twice");
@@ -484,12 +482,12 @@ public void markShardCopyAsStale(ShardId shardId, String allocationId, long prim
     }
 
     class TestReplicationOperation extends ReplicationOperation<Request, Request, TestPrimary.Result> {
-        public TestReplicationOperation(Request request, Primary<Request, Request, TestPrimary.Result> primary,
+        TestReplicationOperation(Request request, Primary<Request, Request, TestPrimary.Result> primary,
                 ActionListener<TestPrimary.Result> listener, Replicas<Request> replicas, Supplier<ClusterState> clusterStateSupplier) {
             this(request, primary, listener, true, replicas, clusterStateSupplier, ReplicationOperationTests.this.logger, "test");
         }
 
-        public TestReplicationOperation(Request request, Primary<Request, Request, TestPrimary.Result> primary,
+        TestReplicationOperation(Request request, Primary<Request, Request, TestPrimary.Result> primary,
                 ActionListener<TestPrimary.Result> listener, boolean executeOnReplicas,
                 Replicas<Request> replicas, Supplier<ClusterState> clusterStateSupplier, Logger logger, String opType) {
             super(request, primary, listener, executeOnReplicas, replicas, clusterStateSupplier, logger, opType);
diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/ReplicationResponseTests.java b/core/src/test/java/org/elasticsearch/action/support/replication/ReplicationResponseTests.java
new file mode 100644
index 0000000000000..fd61bbc6d71e9
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/support/replication/ReplicationResponseTests.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.support.replication;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.support.replication.ReplicationResponse.ShardInfo;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.breaker.CircuitBreakingException;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.RandomObjects;
+
+import java.io.IOException;
+import java.util.Locale;
+
+import static org.elasticsearch.ElasticsearchExceptionTests.assertDeepEquals;
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
+
+public class ReplicationResponseTests extends ESTestCase {
+
+    public void testShardInfoToString() {
+        final int total = 5;
+        final int successful = randomIntBetween(1, total);
+        final ShardInfo shardInfo = new ShardInfo(total, successful);
+        assertEquals(String.format(Locale.ROOT, "ShardInfo{total=5, successful=%d, failures=[]}", successful), shardInfo.toString());
+    }
+
+    public void testShardInfoToXContent() throws IOException {
+        {
+            ShardInfo shardInfo = new ShardInfo(5, 3);
+            String output = Strings.toString(shardInfo);
+            assertEquals("{\"total\":5,\"successful\":3,\"failed\":0}", output);
+        }
+        {
+            ShardInfo shardInfo = new ShardInfo(6, 4,
+                    new ShardInfo.Failure(new ShardId("index", "_uuid", 3),
+                            "_node_id", new IllegalArgumentException("Wrong"), RestStatus.BAD_REQUEST, false),
+                    new ShardInfo.Failure(new ShardId("index", "_uuid", 1),
+                            "_node_id", new CircuitBreakingException("Wrong", 12, 21), RestStatus.NOT_ACCEPTABLE, true));
+            String output = Strings.toString(shardInfo);
+            assertEquals("{\"total\":6,\"successful\":4,\"failed\":2,\"failures\":[{\"_index\":\"index\",\"_shard\":3," +
+                    "\"_node\":\"_node_id\",\"reason\":{\"type\":\"illegal_argument_exception\",\"reason\":\"Wrong\"}," +
+                    "\"status\":\"BAD_REQUEST\",\"primary\":false},{\"_index\":\"index\",\"_shard\":1,\"_node\":\"_node_id\"," +
+                    "\"reason\":{\"type\":\"circuit_breaking_exception\",\"reason\":\"Wrong\",\"bytes_wanted\":12,\"bytes_limit\":21}," +
+                    "\"status\":\"NOT_ACCEPTABLE\",\"primary\":true}]}", output);
+        }
+    }
+
+    public void testShardInfoToAndFromXContent() throws IOException {
+        final Tuple<ShardInfo, ShardInfo> tuple = RandomObjects.randomShardInfo(random());
+        ShardInfo shardInfo = tuple.v1();
+        ShardInfo expectedShardInfo = tuple.v2();
+
+        final XContentType xContentType = randomFrom(XContentType.values());
+        boolean humanReadable = randomBoolean();
+        BytesReference originalBytes = toShuffledXContent(shardInfo, xContentType, ToXContent.EMPTY_PARAMS, humanReadable);
+
+        // Shuffle the XContent fields
+        if (randomBoolean()) {
+            try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) {
+                originalBytes = shuffleXContent(parser, randomBoolean()).bytes();
+            }
+        }
+
+        ShardInfo parsedShardInfo;
+        try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) {
+            // Move to the first start object
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            parsedShardInfo = ShardInfo.fromXContent(parser);
+            assertNull(parser.nextToken());
+        }
+        assertShardInfo(expectedShardInfo, parsedShardInfo);
+
+        BytesReference expectedFinalBytes = toXContent(expectedShardInfo, xContentType, humanReadable);
+        BytesReference finalBytes = toXContent(parsedShardInfo, xContentType, humanReadable);
+        assertToXContentEquivalent(expectedFinalBytes, finalBytes, xContentType);
+    }
+
+    public static void assertShardInfo(ShardInfo expected, ShardInfo actual) {
+        if (expected == null) {
+            assertNull(actual);
+        } else {
+            assertEquals(expected.getTotal(), actual.getTotal());
+            assertEquals(expected.getSuccessful(), actual.getSuccessful());
+            assertEquals(expected.getFailed(), actual.getFailed());
+
+            ReplicationResponse.ShardInfo.Failure[] expectedFailures = expected.getFailures();
+            ReplicationResponse.ShardInfo.Failure[] actualFailures = actual.getFailures();
+            assertEquals(expectedFailures.length, actualFailures.length);
+
+            for (int i = 0; i < expectedFailures.length; i++) {
+                ReplicationResponse.ShardInfo.Failure expectedFailure = expectedFailures[i];
+                ReplicationResponse.ShardInfo.Failure actualFailure = actualFailures[i];
+
+                assertEquals(expectedFailure.fullShardId(), actualFailure.fullShardId());
+                assertEquals(expectedFailure.status(), actualFailure.status());
+                assertEquals(expectedFailure.nodeId(), actualFailure.nodeId());
+                assertEquals(expectedFailure.primary(), actualFailure.primary());
+
+                ElasticsearchException expectedCause = (ElasticsearchException) expectedFailure.getCause();
+                ElasticsearchException actualCause = (ElasticsearchException) actualFailure.getCause();
+                assertDeepEquals(expectedCause, actualCause);
+            }
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java b/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
index bda117642a00f..44cb0f7bd9db9 100644
--- a/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
@@ -18,9 +18,11 @@
  */
 package org.elasticsearch.action.support.replication;
 
+import org.apache.lucene.store.AlreadyClosedException;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.UnavailableShardsException;
+import org.elasticsearch.action.admin.indices.close.CloseIndexRequest;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.action.support.PlainActionFuture;
@@ -42,7 +44,6 @@
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
-import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -52,13 +53,14 @@
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.IndexService;
-import org.elasticsearch.index.engine.EngineClosedException;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.IndexShardClosedException;
 import org.elasticsearch.index.shard.IndexShardState;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.ShardNotFoundException;
+import org.elasticsearch.indices.IndexClosedException;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.indices.cluster.ClusterStateChanges;
 import org.elasticsearch.node.NodeClosedException;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESTestCase;
@@ -78,7 +80,7 @@
 import org.junit.BeforeClass;
 
 import java.io.IOException;
-import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
@@ -131,6 +133,8 @@ public static <R extends ReplicationRequest> R resolveRequest(TransportRequest r
     private TransportService transportService;
     private CapturingTransport transport;
     private Action action;
+    private ShardStateAction shardStateAction;
+
     /* *
     * TransportReplicationAction needs an instance of IndexShard to count operations.
     * indexShards is reset to null before each test and will be initialized upon request in the tests.
@@ -147,10 +151,12 @@ public void setUp() throws Exception {
         super.setUp();
         transport = new CapturingTransport();
         clusterService = createClusterService(threadPool);
-        transportService = new TransportService(clusterService.getSettings(), transport, threadPool);
+        transportService = new TransportService(clusterService.getSettings(), transport, threadPool,
+            TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null);
         transportService.start();
         transportService.acceptIncomingRequests();
-        action = new Action(Settings.EMPTY, "testAction", transportService, clusterService, threadPool);
+        shardStateAction = new ShardStateAction(Settings.EMPTY, clusterService, transportService, null, null, threadPool);
+        action = new Action(Settings.EMPTY, "testAction", transportService, clusterService, shardStateAction, threadPool);
     }
 
     @After
@@ -178,9 +184,15 @@ public void testBlocks() throws ExecutionException, InterruptedException {
         Request request = new Request();
         PlainActionFuture<Response> listener = new PlainActionFuture<>();
         ReplicationTask task = maybeTask();
+        Action action = new Action(Settings.EMPTY, "testActionWithBlocks", transportService, clusterService, shardStateAction, threadPool) {
+            @Override
+            protected ClusterBlockLevel globalBlockLevel() {
+                return ClusterBlockLevel.WRITE;
+            }
+        };
 
-        ClusterBlocks.Builder block = ClusterBlocks.builder()
-            .addGlobalBlock(new ClusterBlock(1, "non retryable", false, true, RestStatus.SERVICE_UNAVAILABLE, ClusterBlockLevel.ALL));
+        ClusterBlocks.Builder block = ClusterBlocks.builder().addGlobalBlock(new ClusterBlock(1, "non retryable", false, true,
+            false, RestStatus.SERVICE_UNAVAILABLE, ClusterBlockLevel.ALL));
         setState(clusterService, ClusterState.builder(clusterService.state()).blocks(block));
         Action.ReroutePhase reroutePhase = action.new ReroutePhase(task, request, listener);
         reroutePhase.run();
@@ -188,7 +200,7 @@ public void testBlocks() throws ExecutionException, InterruptedException {
         assertPhase(task, "failed");
 
         block = ClusterBlocks.builder()
-            .addGlobalBlock(new ClusterBlock(1, "retryable", true, true, RestStatus.SERVICE_UNAVAILABLE, ClusterBlockLevel.ALL));
+            .addGlobalBlock(new ClusterBlock(1, "retryable", true, true, false, RestStatus.SERVICE_UNAVAILABLE, ClusterBlockLevel.ALL));
         setState(clusterService, ClusterState.builder(clusterService.state()).blocks(block));
         listener = new PlainActionFuture<>();
         reroutePhase = action.new ReroutePhase(task, new Request().timeout("5ms"), listener);
@@ -204,12 +216,23 @@ public void testBlocks() throws ExecutionException, InterruptedException {
         assertPhase(task, "waiting_for_retry");
         assertTrue(request.isRetrySet.get());
 
-        block = ClusterBlocks.builder()
-            .addGlobalBlock(new ClusterBlock(1, "non retryable", false, true, RestStatus.SERVICE_UNAVAILABLE, ClusterBlockLevel.ALL));
+        block = ClusterBlocks.builder().addGlobalBlock(new ClusterBlock(1, "non retryable", false, true, false,
+            RestStatus.SERVICE_UNAVAILABLE, ClusterBlockLevel.ALL));
         setState(clusterService, ClusterState.builder(clusterService.state()).blocks(block));
         assertListenerThrows("primary phase should fail operation when moving from a retryable block to a non-retryable one", listener,
             ClusterBlockException.class);
         assertIndexShardUninitialized();
+
+        action = new Action(Settings.EMPTY, "testActionWithNoBlocks", transportService, clusterService, shardStateAction, threadPool) {
+            @Override
+            protected ClusterBlockLevel globalBlockLevel() {
+                return null;
+            }
+        };
+        listener = new PlainActionFuture<>();
+        reroutePhase = action.new ReroutePhase(task, new Request().timeout("5ms"), listener);
+        reroutePhase.run();
+        assertListenerThrows("should fail with an IndexNotFoundException when no blocks checked", listener, IndexNotFoundException.class);
     }
 
     public void assertIndexShardUninitialized() {
@@ -224,7 +247,7 @@ public void testNotStartedPrimary() throws InterruptedException, ExecutionExcept
             randomBoolean() ? ShardRoutingState.INITIALIZING : ShardRoutingState.UNASSIGNED));
         ReplicationTask task = maybeTask();
 
-        logger.debug("--> using initial state:\n{}", clusterService.state().prettyPrint());
+        logger.debug("--> using initial state:\n{}", clusterService.state());
 
         Request request = new Request(shardId).timeout("1ms");
         PlainActionFuture<Response> listener = new PlainActionFuture<>();
@@ -243,7 +266,7 @@ public void testNotStartedPrimary() throws InterruptedException, ExecutionExcept
         assertTrue(request.isRetrySet.get());
 
         setState(clusterService, state(index, true, ShardRoutingState.STARTED));
-        logger.debug("--> primary assigned state:\n{}", clusterService.state().prettyPrint());
+        logger.debug("--> primary assigned state:\n{}", clusterService.state());
 
         final IndexShardRoutingTable shardRoutingTable = clusterService.state().routingTable().index(index).shard(shardId.id());
         final String primaryNodeId = shardRoutingTable.primaryShard().currentNodeId();
@@ -272,7 +295,7 @@ public void testNoRerouteOnStaleClusterState() throws InterruptedException, Exec
         String relocationTargetNode = state.getRoutingTable().shardRoutingTable(shardId).primaryShard().relocatingNodeId();
         state = ClusterState.builder(state).nodes(DiscoveryNodes.builder(state.nodes()).localNodeId(relocationTargetNode)).build();
         setState(clusterService, state);
-        logger.debug("--> relocation ongoing state:\n{}", clusterService.state().prettyPrint());
+        logger.debug("--> relocation ongoing state:\n{}", clusterService.state());
 
         Request request = new Request(shardId).timeout("1ms").routedBasedOnClusterVersion(clusterService.state().version() + 1);
         PlainActionFuture<Response> listener = new PlainActionFuture<>();
@@ -292,11 +315,10 @@ public void testNoRerouteOnStaleClusterState() throws InterruptedException, Exec
         ShardRouting relocationTarget = clusterService.state().getRoutingTable().shardRoutingTable(shardId)
             .shardsWithState(ShardRoutingState.INITIALIZING).get(0);
         AllocationService allocationService = ESAllocationTestCase.createAllocationService();
-        RoutingAllocation.Result result = allocationService.applyStartedShards(state, Arrays.asList(relocationTarget));
-        ClusterState updatedState = ClusterState.builder(clusterService.state()).routingResult(result).build();
+        ClusterState updatedState = allocationService.applyStartedShards(state, Collections.singletonList(relocationTarget));
 
         setState(clusterService, updatedState);
-        logger.debug("--> relocation complete state:\n{}", clusterService.state().prettyPrint());
+        logger.debug("--> relocation complete state:\n{}", clusterService.state());
 
         IndexShardRoutingTable shardRoutingTable = clusterService.state().routingTable().index(index).shard(shardId.id());
         final String primaryNodeId = shardRoutingTable.primaryShard().currentNodeId();
@@ -313,7 +335,7 @@ public void testUnknownIndexOrShardOnReroute() throws InterruptedException {
         // no replicas in oder to skip the replication part
         setState(clusterService, state(index, true,
             randomBoolean() ? ShardRoutingState.INITIALIZING : ShardRoutingState.UNASSIGNED));
-        logger.debug("--> using initial state:\n{}", clusterService.state().prettyPrint());
+        logger.debug("--> using initial state:\n{}", clusterService.state());
         Request request = new Request(new ShardId("unknown_index", "_na_", 0)).timeout("1ms");
         PlainActionFuture<Response> listener = new PlainActionFuture<>();
         ReplicationTask task = maybeTask();
@@ -332,12 +354,40 @@ public void testUnknownIndexOrShardOnReroute() throws InterruptedException {
 
     }
 
+    public void testClosedIndexOnReroute() throws InterruptedException {
+        final String index = "test";
+        // no replicas in oder to skip the replication part
+        setState(clusterService, new ClusterStateChanges(xContentRegistry(), threadPool).closeIndices(state(index, true,
+            ShardRoutingState.UNASSIGNED), new CloseIndexRequest(index)));
+        logger.debug("--> using initial state:\n{}", clusterService.state());
+        Request request = new Request(new ShardId("test", "_na_", 0)).timeout("1ms");
+        PlainActionFuture<Response> listener = new PlainActionFuture<>();
+        ReplicationTask task = maybeTask();
+
+        ClusterBlockLevel indexBlockLevel = randomBoolean() ? ClusterBlockLevel.WRITE : null;
+        Action action = new Action(Settings.EMPTY, "testActionWithBlocks", transportService, clusterService, shardStateAction, threadPool) {
+            @Override
+            protected ClusterBlockLevel indexBlockLevel() {
+                return indexBlockLevel;
+            }
+        };
+        Action.ReroutePhase reroutePhase = action.new ReroutePhase(task, request, listener);
+        reroutePhase.run();
+        if (indexBlockLevel == ClusterBlockLevel.WRITE) {
+            assertListenerThrows("must throw block exception", listener, ClusterBlockException.class);
+        } else {
+            assertListenerThrows("must throw index closed exception", listener, IndexClosedException.class);
+        }
+        assertPhase(task, "failed");
+        assertFalse(request.isRetrySet.get());
+    }
+
     public void testStalePrimaryShardOnReroute() throws InterruptedException {
         final String index = "test";
         final ShardId shardId = new ShardId(index, "_na_", 0);
         // no replicas in order to skip the replication part
         setState(clusterService, stateWithActivePrimary(index, true, randomInt(3)));
-        logger.debug("--> using initial state:\n{}", clusterService.state().prettyPrint());
+        logger.debug("--> using initial state:\n{}", clusterService.state());
         Request request = new Request(shardId);
         boolean timeout = randomBoolean();
         if (timeout) {
@@ -378,12 +428,12 @@ public void testStalePrimaryShardOnReroute() throws InterruptedException {
         }
     }
 
-    private ElasticsearchException randomRetryPrimaryException(ShardId shardId) {
+    private Exception randomRetryPrimaryException(ShardId shardId) {
         return randomFrom(
             new ShardNotFoundException(shardId),
             new IndexNotFoundException(shardId.getIndex()),
             new IndexShardClosedException(shardId),
-            new EngineClosedException(shardId),
+            new AlreadyClosedException(shardId + " primary is closed"),
             new ReplicationOperation.RetryOnPrimaryException(shardId, "hello")
         );
     }
@@ -394,7 +444,7 @@ public void testRoutePhaseExecutesRequest() {
         ReplicationTask task = maybeTask();
 
         setState(clusterService, stateWithActivePrimary(index, randomBoolean(), 3));
-        logger.debug("using state: \n{}", clusterService.state().prettyPrint());
+        logger.debug("using state: \n{}", clusterService.state());
 
         final IndexShardRoutingTable shardRoutingTable = clusterService.state().routingTable().index(index).shard(shardId.id());
         final String primaryNodeId = shardRoutingTable.primaryShard().currentNodeId();
@@ -431,17 +481,20 @@ public void testPrimaryPhaseExecutesOrDelegatesRequestToRelocationTarget() throw
         AtomicBoolean executed = new AtomicBoolean();
 
         ShardRouting primaryShard = state.getRoutingTable().shardRoutingTable(shardId).primaryShard();
+        final long primaryTerm = state.metaData().index(index).primaryTerm(shardId.id());
         boolean executeOnPrimary = true;
         // whether shard has been marked as relocated already (i.e. relocation completed)
         if (primaryShard.relocating() && randomBoolean()) {
             isRelocated.set(true);
             executeOnPrimary = false;
         }
-        action.new AsyncPrimaryAction(request, primaryShard.allocationId().getId(), createTransportChannel(listener), task) {
+        action.new AsyncPrimaryAction(request, primaryShard.allocationId().getId(), primaryTerm, createTransportChannel(listener), task) {
             @Override
-            protected ReplicationOperation<Request, Request, Action.PrimaryResult> createReplicatedOperation(Request request,
-                    ActionListener<Action.PrimaryResult> actionListener, Action.PrimaryShardReference primaryShardReference,
-                    boolean executeOnReplicas) {
+            protected ReplicationOperation<Request, Request, TransportReplicationAction.PrimaryResult<Request, Response>>
+            createReplicatedOperation(Request request,
+                                      ActionListener<TransportReplicationAction.PrimaryResult<Request, Response>> actionListener,
+                                      TransportReplicationAction<Request, Request, Response>.PrimaryShardReference primaryShardReference,
+                                      boolean executeOnReplicas) {
                 return new NoopReplicationOperation(request, actionListener) {
                     public void execute() throws Exception {
                         assertPhase(task, "primary");
@@ -465,6 +518,8 @@ public void execute() throws Exception {
             assertThat(requests, notNullValue());
             assertThat(requests.size(), equalTo(1));
             assertThat("primary request was not delegated to relocation target", requests.get(0).action, equalTo("testAction[p]"));
+            assertThat("primary term not properly set on primary delegation",
+                ((TransportReplicationAction.ConcreteShardRequest<Request>)requests.get(0).request).getPrimaryTerm(), equalTo(primaryTerm));
             assertPhase(task, "primary_delegation");
             transport.handleResponse(requests.get(0).requestId, new Response());
             assertTrue(listener.isDone());
@@ -479,6 +534,7 @@ public void testPrimaryPhaseExecutesDelegatedRequestOnRelocationTarget() throws
         final ShardId shardId = new ShardId(index, "_na_", 0);
         ClusterState state = state(index, true, ShardRoutingState.RELOCATING);
         final ShardRouting primaryShard = state.getRoutingTable().shardRoutingTable(shardId).primaryShard();
+        final long primaryTerm = state.metaData().index(index).primaryTerm(shardId.id());
         String primaryTargetNodeId = primaryShard.relocatingNodeId();
         // simulate execution of the primary phase on the relocation target node
         state = ClusterState.builder(state).nodes(DiscoveryNodes.builder(state.nodes()).localNodeId(primaryTargetNodeId)).build();
@@ -487,11 +543,14 @@ public void testPrimaryPhaseExecutesDelegatedRequestOnRelocationTarget() throws
         PlainActionFuture<Response> listener = new PlainActionFuture<>();
         ReplicationTask task = maybeTask();
         AtomicBoolean executed = new AtomicBoolean();
-        action.new AsyncPrimaryAction(request, primaryShard.allocationId().getRelocationId(), createTransportChannel(listener), task) {
+        action.new AsyncPrimaryAction(request, primaryShard.allocationId().getRelocationId(), primaryTerm,
+            createTransportChannel(listener), task) {
             @Override
-            protected ReplicationOperation<Request, Request, Action.PrimaryResult> createReplicatedOperation(Request request,
-                    ActionListener<Action.PrimaryResult> actionListener, Action.PrimaryShardReference primaryShardReference,
-                    boolean executeOnReplicas) {
+            protected ReplicationOperation<Request, Request, TransportReplicationAction.PrimaryResult<Request, Response>>
+            createReplicatedOperation(Request request,
+                                      ActionListener<TransportReplicationAction.PrimaryResult<Request, Response>> actionListener,
+                                      TransportReplicationAction<Request, Request, Response>.PrimaryShardReference primaryShardReference,
+                                      boolean executeOnReplicas) {
                 return new NoopReplicationOperation(request, actionListener) {
                     public void execute() throws Exception {
                         assertPhase(task, "primary");
@@ -524,9 +583,7 @@ public void testPrimaryReference() throws Exception {
         };
         Action.PrimaryShardReference primary = action.new PrimaryShardReference(shard, releasable);
         final Request request = new Request();
-        Request replicaRequest = primary.perform(request).replicaRequest;
-
-        assertThat(replicaRequest.primaryTerm(), equalTo(primaryTerm));
+        Request replicaRequest = (Request) primary.perform(request).replicaRequest;
 
         final ElasticsearchException exception = new ElasticsearchException("testing");
         primary.failShard("test", exception);
@@ -539,12 +596,12 @@ public void testPrimaryReference() throws Exception {
     }
 
     public void testReplicaProxy() throws InterruptedException, ExecutionException {
-        Action.ReplicasProxy proxy = action.new ReplicasProxy();
         final String index = "test";
         final ShardId shardId = new ShardId(index, "_na_", 0);
         ClusterState state = stateWithActivePrimary(index, true, 1 + randomInt(3), randomInt(2));
-        logger.info("using state: {}", state.prettyPrint());
+        logger.info("using state: {}", state);
         setState(clusterService, state);
+        Action.ReplicasProxy proxy = action.new ReplicasProxy(state.metaData().index(index).primaryTerm(0));
 
         // check that at unknown node fails
         PlainActionFuture<TransportResponse.Empty> listener = new PlainActionFuture<>();
@@ -580,7 +637,7 @@ public void testReplicaProxy() throws InterruptedException, ExecutionException {
         AtomicReference<Throwable> failure = new AtomicReference<>();
         AtomicReference<Throwable> ignoredFailure = new AtomicReference<>();
         AtomicBoolean success = new AtomicBoolean();
-        proxy.failShard(replica, randomIntBetween(1, 10), "test", new ElasticsearchException("simulated"),
+        proxy.failShard(replica, "test", new ElasticsearchException("simulated"),
             () -> success.set(true), failure::set, ignoredFailure::set
         );
         CapturingTransport.CapturedRequest[] shardFailedRequests = transport.getCapturedRequestsAndClear();
@@ -629,16 +686,19 @@ public void testShadowIndexDisablesReplication() throws Exception {
         setState(clusterService, state);
         AtomicBoolean executed = new AtomicBoolean();
         ShardRouting primaryShard = state.routingTable().shardRoutingTable(shardId).primaryShard();
-        action.new AsyncPrimaryAction(new Request(shardId), primaryShard.allocationId().getId(),
+        final long primaryTerm = state.metaData().index(index).primaryTerm(shardId.id());
+        action.new AsyncPrimaryAction(new Request(shardId), primaryShard.allocationId().getId(), primaryTerm,
             createTransportChannel(new PlainActionFuture<>()), null) {
             @Override
-            protected ReplicationOperation<Request, Request, Action.PrimaryResult> createReplicatedOperation(Request request,
-                    ActionListener<Action.PrimaryResult> actionListener, Action.PrimaryShardReference primaryShardReference,
+            protected ReplicationOperation<Request, Request, Action.PrimaryResult<Request, Response>> createReplicatedOperation(
+                    Request request, ActionListener<TransportReplicationAction.PrimaryResult<Request, Response>> actionListener,
+                    TransportReplicationAction<Request, Request, Response>.PrimaryShardReference primaryShardReference,
                     boolean executeOnReplicas) {
                 assertFalse(executeOnReplicas);
                 assertFalse(executed.getAndSet(true));
                 return new NoopReplicationOperation(request, actionListener);
             }
+
         }.run();
         assertThat(executed.get(), equalTo(true));
     }
@@ -649,8 +709,9 @@ public void testCounterOnPrimary() throws Exception {
         // no replica, we only want to test on primary
         final ClusterState state = state(index, true, ShardRoutingState.STARTED);
         setState(clusterService, state);
-        logger.debug("--> using initial state:\n{}", clusterService.state().prettyPrint());
+        logger.debug("--> using initial state:\n{}", clusterService.state());
         final ShardRouting primaryShard = state.routingTable().shardRoutingTable(shardId).primaryShard();
+        final long primaryTerm = state.metaData().index(index).primaryTerm(shardId.id());
         Request request = new Request(shardId);
         PlainActionFuture<Response> listener = new PlainActionFuture<>();
         ReplicationTask task = maybeTask();
@@ -658,11 +719,13 @@ public void testCounterOnPrimary() throws Exception {
         final boolean throwExceptionOnCreation = i == 1;
         final boolean throwExceptionOnRun = i == 2;
         final boolean respondWithError = i == 3;
-        action.new AsyncPrimaryAction(request, primaryShard.allocationId().getId(), createTransportChannel(listener), task) {
+        action.new AsyncPrimaryAction(request, primaryShard.allocationId().getId(), primaryTerm, createTransportChannel(listener), task) {
             @Override
-            protected ReplicationOperation<Request, Request, Action.PrimaryResult> createReplicatedOperation(Request request,
-                    ActionListener<Action.PrimaryResult> actionListener, Action.PrimaryShardReference primaryShardReference,
-                    boolean executeOnReplicas) {
+            protected ReplicationOperation<Request, Request, TransportReplicationAction.PrimaryResult<Request, Response>>
+            createReplicatedOperation(Request request,
+                                      ActionListener<TransportReplicationAction.PrimaryResult<Request, Response>> actionListener,
+                                      TransportReplicationAction<Request, Request, Response>.PrimaryShardReference primaryShardReference,
+                                      boolean executeOnReplicas) {
                 assertIndexShardCounter(1);
                 if (throwExceptionOnCreation) {
                     throw new ElasticsearchException("simulated exception, during createReplicatedOperation");
@@ -705,11 +768,13 @@ public void testReplicasCounter() throws Exception {
         final ClusterState state = state(shardId.getIndexName(), true, ShardRoutingState.STARTED, ShardRoutingState.STARTED);
         setState(clusterService, state);
         final ShardRouting replicaRouting = state.getRoutingTable().shardRoutingTable(shardId).replicaShards().get(0);
+        final long primaryTerm = state.metaData().index(shardId.getIndexName()).primaryTerm(shardId.id());
         boolean throwException = randomBoolean();
         final ReplicationTask task = maybeTask();
-        Action action = new Action(Settings.EMPTY, "testActionWithExceptions", transportService, clusterService, threadPool) {
+        Action action = new Action(Settings.EMPTY, "testActionWithExceptions", transportService, clusterService, shardStateAction,
+            threadPool) {
             @Override
-            protected ReplicaResult shardOperationOnReplica(Request request) {
+            protected ReplicaResult shardOperationOnReplica(Request request, IndexShard replica) {
                 assertIndexShardCounter(1);
                 assertPhase(task, "replica");
                 if (throwException) {
@@ -722,7 +787,7 @@ protected ReplicaResult shardOperationOnReplica(Request request) {
         try {
             replicaOperationTransportHandler.messageReceived(
                 new TransportReplicationAction.ConcreteShardRequest<>(
-                        new Request().setShardId(shardId), replicaRouting.allocationId().getId()),
+                        new Request().setShardId(shardId), replicaRouting.allocationId().getId(), primaryTerm),
                 createTransportChannel(new PlainActionFuture<>()), task);
         } catch (ElasticsearchException e) {
             assertThat(e.getMessage(), containsString("simulated"));
@@ -764,15 +829,21 @@ public void testDefaultWaitForActiveShardsUsesIndexSetting() throws Exception {
         assertEquals(ActiveShardCount.from(requestWaitForActiveShards), request.waitForActiveShards());
     }
 
-    /** test that a primary request is rejected if it arrives at a shard with a wrong allocation id */
-    public void testPrimaryActionRejectsWrongAid() throws Exception {
+    /** test that a primary request is rejected if it arrives at a shard with a wrong allocation id or term */
+    public void testPrimaryActionRejectsWrongAidOrWrongTerm() throws Exception {
         final String index = "test";
         final ShardId shardId = new ShardId(index, "_na_", 0);
         setState(clusterService, state(index, true, ShardRoutingState.STARTED));
+        final ShardRouting primary = clusterService.state().routingTable().shardRoutingTable(shardId).primaryShard();
+        final long primaryTerm = clusterService.state().metaData().index(shardId.getIndexName()).primaryTerm(shardId.id());
         PlainActionFuture<Response> listener = new PlainActionFuture<>();
+        final boolean wrongAllocationId = randomBoolean();
+        final long requestTerm = wrongAllocationId && randomBoolean() ? primaryTerm : primaryTerm + randomIntBetween(1, 10);
         Request request = new Request(shardId).timeout("1ms");
             action.new PrimaryOperationTransportHandler().messageReceived(
-                new TransportReplicationAction.ConcreteShardRequest<>(request, "_not_a_valid_aid_"),
+                new TransportReplicationAction.ConcreteShardRequest<>(request,
+                    wrongAllocationId ? "_not_a_valid_aid_" : primary.allocationId().getId(),
+                    requestTerm),
                 createTransportChannel(listener), maybeTask()
             );
         try {
@@ -782,6 +853,13 @@ action.new PrimaryOperationTransportHandler().messageReceived(
             Throwable throwable = execException.getCause();
             logger.debug("got exception:" , throwable);
             assertTrue(throwable.getClass() + " is not a retry exception", action.retryPrimaryException(throwable));
+            if (wrongAllocationId) {
+                assertThat(throwable.getMessage(), containsString("expected aID [_not_a_valid_aid_] but found [" +
+                primary.allocationId().getId() + "]"));
+            } else {
+                assertThat(throwable.getMessage(), containsString("expected aID [" + primary.allocationId().getId() + "] with term [" +
+                    requestTerm + "] but found [" + primaryTerm + "]"));
+            }
         }
     }
 
@@ -798,7 +876,7 @@ public void testReplicaActionRejectsWrongAid() throws Exception {
         PlainActionFuture<Response> listener = new PlainActionFuture<>();
         Request request = new Request(shardId).timeout("1ms");
         action.new ReplicaOperationTransportHandler().messageReceived(
-            new TransportReplicationAction.ConcreteShardRequest<>(request, "_not_a_valid_aid_"),
+            new TransportReplicationAction.ConcreteShardRequest<>(request, "_not_a_valid_aid_", randomNonNegativeLong()),
             createTransportChannel(listener), maybeTask()
         );
         try {
@@ -821,14 +899,16 @@ public void testRetryOnReplica() throws Exception {
         final ShardId shardId = new ShardId("test", "_na_", 0);
         ClusterState state = state(shardId.getIndexName(), true, ShardRoutingState.STARTED, ShardRoutingState.STARTED);
         final ShardRouting replica = state.getRoutingTable().shardRoutingTable(shardId).replicaShards().get(0);
+        final long primaryTerm = state.metaData().getIndexSafe(shardId.getIndex()).primaryTerm(shardId.id());
         // simulate execution of the node holding the replica
         state = ClusterState.builder(state).nodes(DiscoveryNodes.builder(state.nodes()).localNodeId(replica.currentNodeId())).build();
         setState(clusterService, state);
         AtomicBoolean throwException = new AtomicBoolean(true);
         final ReplicationTask task = maybeTask();
-        Action action = new Action(Settings.EMPTY, "testActionWithExceptions", transportService, clusterService, threadPool) {
+        Action action = new Action(Settings.EMPTY, "testActionWithExceptions", transportService, clusterService, shardStateAction,
+            threadPool) {
             @Override
-            protected ReplicaResult shardOperationOnReplica(Request request) {
+            protected ReplicaResult shardOperationOnReplica(Request request, IndexShard replica) {
                 assertPhase(task, "replica");
                 if (throwException.get()) {
                     throw new RetryOnReplicaException(shardId, "simulation");
@@ -839,9 +919,8 @@ protected ReplicaResult shardOperationOnReplica(Request request) {
         final Action.ReplicaOperationTransportHandler replicaOperationTransportHandler = action.new ReplicaOperationTransportHandler();
         final PlainActionFuture<Response> listener = new PlainActionFuture<>();
         final Request request = new Request().setShardId(shardId);
-        request.primaryTerm(state.metaData().getIndexSafe(shardId.getIndex()).primaryTerm(shardId.id()));
         replicaOperationTransportHandler.messageReceived(
-                new TransportReplicationAction.ConcreteShardRequest<>(request, replica.allocationId().getId()),
+                new TransportReplicationAction.ConcreteShardRequest<>(request, replica.allocationId().getId(), primaryTerm),
                 createTransportChannel(listener), task);
         if (listener.isDone()) {
             listener.get(); // fail with the exception if there
@@ -863,8 +942,11 @@ protected ReplicaResult shardOperationOnReplica(Request request) {
         final CapturingTransport.CapturedRequest capturedRequest = capturedRequests.get(0);
         assertThat(capturedRequest.action, equalTo("testActionWithExceptions[r]"));
         assertThat(capturedRequest.request, instanceOf(TransportReplicationAction.ConcreteShardRequest.class));
-        assertThat(((TransportReplicationAction.ConcreteShardRequest<?>) capturedRequest.request).getRequest(), equalTo(request));
-        assertThat(((TransportReplicationAction.ConcreteShardRequest<?>) capturedRequest.request).getTargetAllocationID(),
+        final TransportReplicationAction.ConcreteShardRequest<Request> concreteShardRequest =
+            (TransportReplicationAction.ConcreteShardRequest<Request>) capturedRequest.request;
+        assertThat(concreteShardRequest.getRequest(), equalTo(request));
+        assertThat(concreteShardRequest.getRequest().isRetrySet.get(), equalTo(true));
+        assertThat(concreteShardRequest.getTargetAllocationID(),
             equalTo(replica.allocationId().getId()));
     }
 
@@ -940,9 +1022,10 @@ class Action extends TransportReplicationAction<Request, Request, Response> {
 
         Action(Settings settings, String actionName, TransportService transportService,
                ClusterService clusterService,
+               ShardStateAction shardStateAction,
                ThreadPool threadPool) {
             super(settings, actionName, transportService, clusterService, mockIndicesService(clusterService), threadPool,
-                new ShardStateAction(settings, clusterService, transportService, null, null, threadPool),
+                shardStateAction,
                 new ActionFilters(new HashSet<>()), new IndexNameExpressionResolver(Settings.EMPTY),
                 Request::new, Request::new, ThreadPool.Names.SAME);
         }
@@ -953,14 +1036,14 @@ protected Response newResponseInstance() {
         }
 
         @Override
-        protected PrimaryResult shardOperationOnPrimary(Request shardRequest) throws Exception {
+        protected PrimaryResult shardOperationOnPrimary(Request shardRequest, IndexShard primary) throws Exception {
             boolean executedBefore = shardRequest.processedOnPrimary.getAndSet(true);
             assert executedBefore == false : "request has already been executed on the primary";
             return new PrimaryResult(shardRequest, new Response());
         }
 
         @Override
-        protected ReplicaResult shardOperationOnReplica(Request request) {
+        protected ReplicaResult shardOperationOnReplica(Request request, IndexShard replica) {
             request.processedOnReplicas.incrementAndGet();
             return new ReplicaResult();
         }
@@ -1041,14 +1124,15 @@ private IndexShard mockIndexShard(ShardId shardId, ClusterService clusterService
         return indexShard;
     }
 
-    class NoopReplicationOperation extends ReplicationOperation<Request, Request, Action.PrimaryResult> {
-        public NoopReplicationOperation(Request request, ActionListener<Action.PrimaryResult> listener) {
+    class NoopReplicationOperation extends ReplicationOperation<Request, Request, Action.PrimaryResult<Request, Response>> {
+        NoopReplicationOperation(Request request, ActionListener<Action.PrimaryResult<Request, Response>> listener) {
             super(request, null, listener, true, null, null, TransportReplicationActionTests.this.logger, "noop");
         }
 
         @Override
         public void execute() throws Exception {
-            this.resultListener.onResponse(action.new PrimaryResult(null, new Response()));
+            // Eclipse can't infer the types below so we have to list them explicitly
+            this.resultListener.onResponse(new TransportReplicationAction.PrimaryResult<Request, Response>(null, new Response()));
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java b/core/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java
index 80e689743fd94..e0fec46bfb74f 100644
--- a/core/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java
@@ -28,15 +28,14 @@
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.translog.Translog;
-import org.elasticsearch.index.translog.Translog.Location;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportResponse;
 import org.elasticsearch.transport.TransportService;
 import org.junit.Before;
 import org.mockito.ArgumentCaptor;
 
 import java.util.HashSet;
-import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 
 import static org.mockito.Matchers.any;
@@ -55,67 +54,94 @@ public void initCommonMocks() {
     }
 
     public void testPrimaryNoRefreshCall() throws Exception {
-        noRefreshCall(TestAction::shardOperationOnPrimary, TestAction.WritePrimaryResult::respond);
+        TestRequest request = new TestRequest();
+        request.setRefreshPolicy(RefreshPolicy.NONE); // The default, but we'll set it anyway just to be explicit
+        TestAction testAction = new TestAction();
+        TransportWriteAction.WritePrimaryResult<TestRequest, TestResponse> result =
+                testAction.shardOperationOnPrimary(request, indexShard);
+        CapturingActionListener<TestResponse> listener = new CapturingActionListener<>();
+        result.respond(listener);
+        assertNotNull(listener.response);
+        assertNull(listener.failure);
+        verify(indexShard, never()).refresh(any());
+        verify(indexShard, never()).addRefreshListener(any(), any());
     }
 
     public void testReplicaNoRefreshCall() throws Exception {
-        noRefreshCall(TestAction::shardOperationOnReplica, TestAction.WriteReplicaResult::respond);
-    }
-
-    private <Result, Response> void noRefreshCall(ThrowingBiFunction<TestAction, TestRequest, Result> action,
-                                        BiConsumer<Result, CapturingActionListener<Response>> responder)
-            throws Exception {
         TestRequest request = new TestRequest();
         request.setRefreshPolicy(RefreshPolicy.NONE); // The default, but we'll set it anyway just to be explicit
-        Result result = action.apply(new TestAction(), request);
-        CapturingActionListener<Response> listener = new CapturingActionListener<>();
-        responder.accept(result, listener);
+        TestAction testAction = new TestAction();
+        TransportWriteAction.WriteReplicaResult<TestRequest> result =
+                testAction.shardOperationOnReplica(request, indexShard);
+        CapturingActionListener<TransportResponse.Empty> listener = new CapturingActionListener<>();
+        result.respond(listener);
         assertNotNull(listener.response);
+        assertNull(listener.failure);
         verify(indexShard, never()).refresh(any());
         verify(indexShard, never()).addRefreshListener(any(), any());
     }
 
     public void testPrimaryImmediateRefresh() throws Exception {
-        immediateRefresh(TestAction::shardOperationOnPrimary, TestAction.WritePrimaryResult::respond, r -> assertTrue(r.forcedRefresh));
+        TestRequest request = new TestRequest();
+        request.setRefreshPolicy(RefreshPolicy.IMMEDIATE);
+        TestAction testAction = new TestAction();
+        TransportWriteAction.WritePrimaryResult<TestRequest, TestResponse> result =
+                testAction.shardOperationOnPrimary(request, indexShard);
+        CapturingActionListener<TestResponse> listener = new CapturingActionListener<>();
+        result.respond(listener);
+        assertNotNull(listener.response);
+        assertNull(listener.failure);
+        assertTrue(listener.response.forcedRefresh);
+        verify(indexShard).refresh("refresh_flag_index");
+        verify(indexShard, never()).addRefreshListener(any(), any());
     }
 
     public void testReplicaImmediateRefresh() throws Exception {
-        immediateRefresh(TestAction::shardOperationOnReplica, TestAction.WriteReplicaResult::respond, r -> {});
-    }
-
-    private <Result, Response> void immediateRefresh(ThrowingBiFunction<TestAction, TestRequest, Result> action,
-                                                     BiConsumer<Result, CapturingActionListener<Response>> responder,
-                                                     Consumer<Response> responseChecker) throws Exception {
         TestRequest request = new TestRequest();
         request.setRefreshPolicy(RefreshPolicy.IMMEDIATE);
-        Result result = action.apply(new TestAction(), request);
-        CapturingActionListener<Response> listener = new CapturingActionListener<>();
-        responder.accept(result, listener);
+        TestAction testAction = new TestAction();
+        TransportWriteAction.WriteReplicaResult<TestRequest> result =
+                testAction.shardOperationOnReplica(request, indexShard);
+        CapturingActionListener<TransportResponse.Empty> listener = new CapturingActionListener<>();
+        result.respond(listener);
         assertNotNull(listener.response);
-        responseChecker.accept(listener.response);
+        assertNull(listener.failure);
         verify(indexShard).refresh("refresh_flag_index");
         verify(indexShard, never()).addRefreshListener(any(), any());
     }
 
     public void testPrimaryWaitForRefresh() throws Exception {
-        waitForRefresh(TestAction::shardOperationOnPrimary, TestAction.WritePrimaryResult::respond,
-            (r, forcedRefresh) -> assertEquals(forcedRefresh, r.forcedRefresh));
-    }
+        TestRequest request = new TestRequest();
+        request.setRefreshPolicy(RefreshPolicy.WAIT_UNTIL);
 
-    public void testReplicaWaitForRefresh() throws Exception {
-        waitForRefresh(TestAction::shardOperationOnReplica, TestAction.WriteReplicaResult::respond, (r, forcedRefresh) -> {});
+        TestAction testAction = new TestAction();
+        TransportWriteAction.WritePrimaryResult<TestRequest, TestResponse> result =
+                testAction.shardOperationOnPrimary(request, indexShard);
+        CapturingActionListener<TestResponse> listener = new CapturingActionListener<>();
+        result.respond(listener);
+        assertNull(listener.response); // Haven't reallresponded yet
+
+        @SuppressWarnings({ "unchecked", "rawtypes" })
+        ArgumentCaptor<Consumer<Boolean>> refreshListener = ArgumentCaptor.forClass((Class) Consumer.class);
+        verify(indexShard, never()).refresh(any());
+        verify(indexShard).addRefreshListener(any(), refreshListener.capture());
+
+        // Now we can fire the listener manually and we'll get a response
+        boolean forcedRefresh = randomBoolean();
+        refreshListener.getValue().accept(forcedRefresh);
+        assertNotNull(listener.response);
+        assertNull(listener.failure);
+        assertEquals(forcedRefresh, listener.response.forcedRefresh);
     }
 
-    private <Result, Response> void waitForRefresh(ThrowingBiFunction<TestAction, TestRequest, Result> action,
-                                                   BiConsumer<Result, CapturingActionListener<Response>> responder,
-                                         BiConsumer<Response, Boolean> resultChecker) throws Exception {
+    public void testReplicaWaitForRefresh() throws Exception {
         TestRequest request = new TestRequest();
         request.setRefreshPolicy(RefreshPolicy.WAIT_UNTIL);
-        Result result = action.apply(new TestAction(), request);
-        CapturingActionListener<Response> listener = new CapturingActionListener<>();
-        responder.accept(result, listener);
-        assertNull(listener.response); // Haven't reallresponded yet
-
+        TestAction testAction = new TestAction();
+        TransportWriteAction.WriteReplicaResult<TestRequest> result = testAction.shardOperationOnReplica(request, indexShard);
+        CapturingActionListener<TransportResponse.Empty> listener = new CapturingActionListener<>();
+        result.respond(listener);
+        assertNull(listener.response); // Haven't responded yet
         @SuppressWarnings({ "unchecked", "rawtypes" })
         ArgumentCaptor<Consumer<Boolean>> refreshListener = ArgumentCaptor.forClass((Class) Consumer.class);
         verify(indexShard, never()).refresh(any());
@@ -125,38 +151,79 @@ private <Result, Response> void waitForRefresh(ThrowingBiFunction<TestAction, Te
         boolean forcedRefresh = randomBoolean();
         refreshListener.getValue().accept(forcedRefresh);
         assertNotNull(listener.response);
-        resultChecker.accept(listener.response, forcedRefresh);
+        assertNull(listener.failure);
+    }
+
+    public void testDocumentFailureInShardOperationOnPrimary() throws Exception {
+        TestRequest request = new TestRequest();
+        TestAction testAction = new TestAction(true, true);
+        TransportWriteAction.WritePrimaryResult<TestRequest, TestResponse> writePrimaryResult =
+                testAction.shardOperationOnPrimary(request, indexShard);
+        CapturingActionListener<TestResponse> listener = new CapturingActionListener<>();
+        writePrimaryResult.respond(listener);
+        assertNull(listener.response);
+        assertNotNull(listener.failure);
+    }
+
+    public void testDocumentFailureInShardOperationOnReplica() throws Exception {
+        TestRequest request = new TestRequest();
+        TestAction testAction = new TestAction(randomBoolean(), true);
+        TransportWriteAction.WriteReplicaResult<TestRequest> writeReplicaResult =
+                testAction.shardOperationOnReplica(request, indexShard);
+        CapturingActionListener<TransportResponse.Empty> listener = new CapturingActionListener<>();
+        writeReplicaResult.respond(listener);
+        assertNull(listener.response);
+        assertNotNull(listener.failure);
     }
 
-    private class TestAction extends TransportWriteAction<TestRequest, TestResponse> {
+    private class TestAction extends TransportWriteAction<TestRequest, TestRequest, TestResponse> {
+
+        private final boolean withDocumentFailureOnPrimary;
+        private final boolean withDocumentFailureOnReplica;
+
         protected TestAction() {
-            super(Settings.EMPTY, "test", mock(TransportService.class), null, null, null, null, new ActionFilters(new HashSet<>()),
-                    new IndexNameExpressionResolver(Settings.EMPTY), TestRequest::new, ThreadPool.Names.SAME);
+            this(false, false);
         }
-
-        @Override
-        protected IndexShard indexShard(TestRequest request) {
-            return indexShard;
+        protected TestAction(boolean withDocumentFailureOnPrimary, boolean withDocumentFailureOnReplica) {
+            super(Settings.EMPTY, "test",
+                    new TransportService(Settings.EMPTY, null, null, TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> null, null),
+                    null, null, null, null, new ActionFilters(new HashSet<>()), new IndexNameExpressionResolver(Settings.EMPTY),
+                    TestRequest::new, TestRequest::new, ThreadPool.Names.SAME);
+            this.withDocumentFailureOnPrimary = withDocumentFailureOnPrimary;
+            this.withDocumentFailureOnReplica = withDocumentFailureOnReplica;
         }
 
         @Override
-        protected WriteResult<TestResponse> onPrimaryShard(TestRequest request, IndexShard indexShard) throws Exception {
-            return new WriteResult<>(new TestResponse(), location);
+        protected TestResponse newResponseInstance() {
+            return new TestResponse();
         }
 
         @Override
-        protected Location onReplicaShard(TestRequest request, IndexShard indexShard) {
-            return location;
+        protected WritePrimaryResult<TestRequest, TestResponse> shardOperationOnPrimary(
+                TestRequest request, IndexShard primary) throws Exception {
+            final WritePrimaryResult<TestRequest, TestResponse> primaryResult;
+            if (withDocumentFailureOnPrimary) {
+                primaryResult = new WritePrimaryResult<>(request, null, null, new RuntimeException("simulated"), primary, logger);
+            } else {
+                primaryResult = new WritePrimaryResult<>(request, new TestResponse(), location, null, primary, logger);
+            }
+            return primaryResult;
         }
 
         @Override
-        protected TestResponse newResponseInstance() {
-            return new TestResponse();
+        protected WriteReplicaResult<TestRequest> shardOperationOnReplica(TestRequest request, IndexShard replica) throws Exception {
+            final WriteReplicaResult<TestRequest> replicaResult;
+            if (withDocumentFailureOnReplica) {
+                replicaResult = new WriteReplicaResult<>(request, null, new RuntimeException("simulated"), replica, logger);
+            } else {
+                replicaResult = new WriteReplicaResult<>(request, location, null, replica, logger);
+            }
+            return replicaResult;
         }
     }
 
     private static class TestRequest extends ReplicatedWriteRequest<TestRequest> {
-        public TestRequest() {
+        TestRequest() {
             setShardId(new ShardId("test", "test", 1));
         }
     }
@@ -172,6 +239,7 @@ public void setForcedRefresh(boolean forcedRefresh) {
 
     private static class CapturingActionListener<R> implements ActionListener<R> {
         private R response;
+        private Exception failure;
 
         @Override
         public void onResponse(R response) {
@@ -179,12 +247,8 @@ public void onResponse(R response) {
         }
 
         @Override
-        public void onFailure(Exception e) {
-            throw new RuntimeException(e);
+        public void onFailure(Exception failure) {
+            this.failure = failure;
         }
     }
-
-    private interface ThrowingBiFunction<A, B, R> {
-        R apply(A a, B b) throws Exception;
-    }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java b/core/src/test/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java
index 37abc4d5eed34..29235329d6669 100644
--- a/core/src/test/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java
@@ -88,7 +88,7 @@ public Response() {
     class TestTransportInstanceSingleOperationAction extends TransportInstanceSingleOperationAction<Request, Response> {
         private final Map<ShardId, Object> shards = new HashMap<>();
 
-        public TestTransportInstanceSingleOperationAction(Settings settings, String actionName, TransportService transportService, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, Supplier<Request> request) {
+        TestTransportInstanceSingleOperationAction(Settings settings, String actionName, TransportService transportService, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, Supplier<Request> request) {
             super(settings, actionName, THREAD_POOL, TransportInstanceSingleOperationActionTests.this.clusterService, transportService, actionFilters, indexNameExpressionResolver, request);
         }
 
@@ -122,7 +122,7 @@ protected ShardIterator shards(ClusterState clusterState, Request request) {
     }
 
     class MyResolver extends IndexNameExpressionResolver {
-        public MyResolver() {
+        MyResolver() {
             super(Settings.EMPTY);
         }
 
@@ -137,12 +137,14 @@ public static void startThreadPool() {
         THREAD_POOL = new TestThreadPool(TransportInstanceSingleOperationActionTests.class.getSimpleName());
     }
 
+    @Override
     @Before
     public void setUp() throws Exception {
         super.setUp();
         transport = new CapturingTransport();
         clusterService = createClusterService(THREAD_POOL);
-        transportService = new TransportService(clusterService.getSettings(), transport, THREAD_POOL);
+        transportService = new TransportService(clusterService.getSettings(), transport, THREAD_POOL,
+                TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null);
         transportService.start();
         transportService.acceptIncomingRequests();
         action = new TestTransportInstanceSingleOperationAction(
@@ -155,6 +157,7 @@ public void setUp() throws Exception {
         );
     }
 
+    @Override
     @After
     public void tearDown() throws Exception {
         super.tearDown();
@@ -173,7 +176,7 @@ public void testGlobalBlock() {
         Request request = new Request();
         PlainActionFuture<Response> listener = new PlainActionFuture<>();
         ClusterBlocks.Builder block = ClusterBlocks.builder()
-                .addGlobalBlock(new ClusterBlock(1, "", false, true, RestStatus.SERVICE_UNAVAILABLE, ClusterBlockLevel.ALL));
+                .addGlobalBlock(new ClusterBlock(1, "", false, true, false, RestStatus.SERVICE_UNAVAILABLE, ClusterBlockLevel.ALL));
         setState(clusterService, ClusterState.builder(clusterService.state()).blocks(block));
         try {
             action.new AsyncSingleAction(request, listener).start();
@@ -272,12 +275,7 @@ public void testRetryOfAnAlreadyTimedOutRequest() throws Exception {
         transport.handleLocalError(requestId, new ConnectTransportException(node, "test exception"));
 
         // wait until the timeout was triggered and we actually tried to send for the second time
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                assertThat(transport.capturedRequests().length, equalTo(1));
-            }
-        });
+        assertBusy(() -> assertThat(transport.capturedRequests().length, equalTo(1)));
 
         // let it fail the second time too
         requestId = transport.capturedRequests()[0].requestId;
diff --git a/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsCheckDocFreqIT.java b/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsCheckDocFreqIT.java
index 1611c63d2bac4..294a0ffde8f30 100644
--- a/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsCheckDocFreqIT.java
+++ b/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsCheckDocFreqIT.java
@@ -137,9 +137,7 @@ private void checkWithoutFieldStatistics(int numDocs, String[] values, int[] fre
         assertThat(iterator.next(), Matchers.nullValue());
 
         XContentBuilder xBuilder = XContentFactory.jsonBuilder();
-        xBuilder.startObject();
         response.toXContent(xBuilder, null);
-        xBuilder.endObject();
         String utf8 = xBuilder.bytes().utf8ToString().replaceFirst("\"took\":\\d+,", "");;
         String expectedString = "{\"_index\":\"test\",\"_type\":\"type1\",\"_id\":\""
                 + i
@@ -193,9 +191,7 @@ private void checkWithoutTermStatistics(int numDocs, String[] values, int[] freq
         assertThat(iterator.next(), Matchers.nullValue());
 
         XContentBuilder xBuilder = XContentFactory.jsonBuilder();
-        xBuilder.startObject();
         response.toXContent(xBuilder, null);
-        xBuilder.endObject();
         String utf8 = xBuilder.bytes().utf8ToString().replaceFirst("\"took\":\\d+,", "");;
         String expectedString = "{\"_index\":\"test\",\"_type\":\"type1\",\"_id\":\""
                 + i
@@ -252,9 +248,7 @@ private void checkAllInfo(int numDocs, String[] values, int[] freq, int[][] pos,
         assertThat(iterator.next(), Matchers.nullValue());
 
         XContentBuilder xBuilder = XContentFactory.jsonBuilder();
-        xBuilder.startObject();
         response.toXContent(xBuilder, ToXContent.EMPTY_PARAMS);
-        xBuilder.endObject();
         String utf8 = xBuilder.bytes().utf8ToString().replaceFirst("\"took\":\\d+,", "");;
         String expectedString = "{\"_index\":\"test\",\"_type\":\"type1\",\"_id\":\""
                 + i
diff --git a/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java b/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java
index 3835edbbe9a33..ba2f5de24ba03 100644
--- a/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java
+++ b/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.action.termvectors;
 
 import com.carrotsearch.hppc.ObjectIntHashMap;
+
 import org.apache.lucene.analysis.payloads.PayloadHelper;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.index.DirectoryReader;
@@ -30,6 +31,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionFuture;
+import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsResponse;
 import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.common.Strings;
@@ -42,6 +44,7 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -49,6 +52,7 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
+import java.util.stream.Collectors;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@@ -78,7 +82,7 @@ public void testNoSuchDoc() throws Exception {
             assertThat(actionGet.getIndex(), equalTo("test"));
             assertThat(actionGet.isExists(), equalTo(false));
             // check response is nevertheless serializable to json
-            actionGet.toXContent(jsonBuilder().startObject(), ToXContent.EMPTY_PARAMS);
+            actionGet.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS);
         }
     }
 
@@ -848,6 +852,16 @@ public void testArtificialNoDoc() throws IOException {
                 .get();
         assertThat(resp.isExists(), equalTo(true));
         checkBrownFoxTermVector(resp.getFields(), "field1", false);
+
+        // Since the index is empty, all of artificial document's "term_statistics" should be 0/absent
+        Terms terms = resp.getFields().terms("field1");
+        assertEquals("sumDocFreq should be 0 for a non-existing field!", 0, terms.getSumDocFreq());
+        assertEquals("sumTotalTermFreq should be 0 for a non-existing field!", 0, terms.getSumTotalTermFreq());
+        TermsEnum termsEnum = terms.iterator(); // we're guaranteed to receive terms for that field
+        while (termsEnum.next() != null) {
+            String term = termsEnum.term().utf8ToString();
+            assertEquals("term [" + term + "] does not exist in the index; ttf should be 0!", 0, termsEnum.totalTermFreq());
+        }
     }
 
     public void testPerFieldAnalyzer() throws IOException {
@@ -1160,6 +1174,48 @@ public void testFilterDocFreq() throws ExecutionException, InterruptedException,
         }
     }
 
+    public void testArtificialDocWithPreference() throws ExecutionException, InterruptedException, IOException {
+        // setup indices
+        Settings.Builder settings = Settings.builder()
+                .put(indexSettings())
+                .put("index.analysis.analyzer", "standard");
+        assertAcked(prepareCreate("test")
+                .setSettings(settings)
+                .addMapping("type1", "field1", "type=text,term_vector=with_positions_offsets"));
+        ensureGreen();
+
+        // index document
+        indexRandom(true, client().prepareIndex("test", "type1", "1").setSource("field1", "random permutation"));
+
+        // Get search shards
+        ClusterSearchShardsResponse searchShardsResponse = client().admin().cluster().prepareSearchShards("test").get();
+        List<Integer> shardIds = Arrays.stream(searchShardsResponse.getGroups()).map(s -> s.getShardId().id()).collect(Collectors.toList());
+
+        // request termvectors of artificial document from each shard
+        int sumTotalTermFreq = 0;
+        int sumDocFreq = 0;
+        for (Integer shardId : shardIds) {
+            TermVectorsResponse tvResponse = client().prepareTermVectors()
+                    .setIndex("test")
+                    .setType("type1")
+                    .setPreference("_shards:" + shardId)
+                    .setDoc(jsonBuilder().startObject().field("field1", "random permutation").endObject())
+                    .setFieldStatistics(true)
+                    .setTermStatistics(true)
+                    .get();
+            Fields fields = tvResponse.getFields();
+            Terms terms = fields.terms("field1");
+            assertNotNull(terms);
+            TermsEnum termsEnum = terms.iterator();
+            while (termsEnum.next() != null) {
+                sumTotalTermFreq += termsEnum.totalTermFreq();
+                sumDocFreq += termsEnum.docFreq();
+            }
+        }
+        assertEquals("expected to find term statistics in exactly one shard!", 2, sumTotalTermFreq);
+        assertEquals("expected to find term statistics in exactly one shard!", 2, sumDocFreq);
+    }
+
     private void checkBestTerms(Terms terms, List<String> expectedTerms) throws IOException {
         final TermsEnum termsEnum = terms.iterator();
         List<String> bestTerms = new ArrayList<>();
diff --git a/core/src/test/java/org/elasticsearch/action/termvectors/TermVectorsUnitTests.java b/core/src/test/java/org/elasticsearch/action/termvectors/TermVectorsUnitTests.java
index a1b0752617ccc..2018218cc5456 100644
--- a/core/src/test/java/org/elasticsearch/action/termvectors/TermVectorsUnitTests.java
+++ b/core/src/test/java/org/elasticsearch/action/termvectors/TermVectorsUnitTests.java
@@ -36,14 +36,17 @@
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.store.Directory;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.termvectors.TermVectorsRequest.Flag;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.InputStreamStreamInput;
 import org.elasticsearch.common.io.stream.OutputStreamStreamOutput;
-import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.mapper.AllFieldMapper;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
@@ -57,6 +60,7 @@
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Base64;
 import java.util.EnumSet;
 import java.util.HashSet;
 import java.util.Set;
@@ -178,7 +182,7 @@ public void testRestRequestParsing() throws Exception {
                 " {\"fields\" : [\"a\",  \"b\",\"c\"], \"offsets\":false, \"positions\":false, \"payloads\":true}");
 
         TermVectorsRequest tvr = new TermVectorsRequest(null, null, null);
-        XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(inputBytes);
+        XContentParser parser = createParser(JsonXContent.jsonXContent, inputBytes);
         TermVectorsRequest.parseRequest(tvr, parser);
 
         Set<String> fields = tvr.selectedFields();
@@ -199,7 +203,7 @@ public void testRestRequestParsing() throws Exception {
 
         inputBytes = new BytesArray(" {\"offsets\":false, \"positions\":false, \"payloads\":true}");
         tvr = new TermVectorsRequest(null, null, null);
-        parser = XContentFactory.xContent(XContentType.JSON).createParser(inputBytes);
+        parser = createParser(JsonXContent.jsonXContent, inputBytes);
         TermVectorsRequest.parseRequest(tvr, parser);
         additionalFields = "";
         RestTermVectorsAction.addFieldStringsFromParameter(tvr, additionalFields);
@@ -216,7 +220,7 @@ public void testRequestParsingThrowsException() throws Exception {
         TermVectorsRequest tvr = new TermVectorsRequest(null, null, null);
         boolean threwException = false;
         try {
-            XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(inputBytes);
+            XContentParser parser = createParser(JsonXContent.jsonXContent, inputBytes);
             TermVectorsRequest.parseRequest(tvr, parser);
         } catch (Exception e) {
             threwException = true;
@@ -237,6 +241,7 @@ public void testStreamRequest() throws IOException {
             request.parent(parent);
             String pref = random().nextBoolean() ? "somePreference" : null;
             request.preference(pref);
+            request.doc(new BytesArray("{}"), randomBoolean(), XContentType.JSON);
 
             // write
             ByteArrayOutputStream outBuffer = new ByteArrayOutputStream();
@@ -256,7 +261,37 @@ public void testStreamRequest() throws IOException {
             assertThat(request.termStatistics(), equalTo(req2.termStatistics()));
             assertThat(request.preference(), equalTo(pref));
             assertThat(request.routing(), equalTo(null));
+            assertEquals(new BytesArray("{}"), request.doc());
+            assertEquals(XContentType.JSON, request.xContentType());
+        }
+    }
 
+    public void testStreamRequestWithXContentBwc() throws IOException {
+        final byte[] data = Base64.getDecoder().decode("AAABBWluZGV4BHR5cGUCaWQBAnt9AAABDnNvbWVQcmVmZXJlbmNlFgAAAAEA//////////0AAAA=");
+        final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2,
+            Version.V_5_1_1, Version.V_5_1_2, Version.V_5_2_0);
+        try (StreamInput in = StreamInput.wrap(data)) {
+            in.setVersion(version);
+            TermVectorsRequest request = new TermVectorsRequest();
+            request.readFrom(in);
+            assertEquals("index", request.index());
+            assertEquals("type", request.type());
+            assertEquals("id", request.id());
+            assertTrue(request.offsets());
+            assertFalse(request.fieldStatistics());
+            assertTrue(request.payloads());
+            assertFalse(request.positions());
+            assertTrue(request.termStatistics());
+            assertNull(request.parent());
+            assertEquals("somePreference", request.preference());
+            assertEquals("{}", request.doc().utf8ToString());
+            assertEquals(XContentType.JSON, request.xContentType());
+
+            try (BytesStreamOutput out = new BytesStreamOutput()) {
+                out.setVersion(version);
+                request.writeTo(out);
+                assertArrayEquals(data, out.bytes().toBytesRef().bytes);
+            }
         }
     }
 
@@ -289,16 +324,16 @@ public void testTermVectorStringGenerationWithoutPositions() throws Exception {
     }
 
     public void testMultiParser() throws Exception {
-        byte[] data = StreamsUtils.copyToBytesFromClasspath("/org/elasticsearch/action/termvectors/multiRequest1.json");
-        BytesReference bytes = new BytesArray(data);
+        byte[] bytes = StreamsUtils.copyToBytesFromClasspath("/org/elasticsearch/action/termvectors/multiRequest1.json");
+        XContentParser data = createParser(JsonXContent.jsonXContent, bytes);
         MultiTermVectorsRequest request = new MultiTermVectorsRequest();
-        request.add(new TermVectorsRequest(), bytes);
+        request.add(new TermVectorsRequest(), data);
         checkParsedParameters(request);
 
-        data = StreamsUtils.copyToBytesFromClasspath("/org/elasticsearch/action/termvectors/multiRequest2.json");
-        bytes = new BytesArray(data);
+        bytes = StreamsUtils.copyToBytesFromClasspath("/org/elasticsearch/action/termvectors/multiRequest2.json");
+        data = createParser(JsonXContent.jsonXContent, new BytesArray(bytes));
         request = new MultiTermVectorsRequest();
-        request.add(new TermVectorsRequest(), bytes);
+        request.add(new TermVectorsRequest(), data);
         checkParsedParameters(request);
     }
 
@@ -325,10 +360,10 @@ void checkParsedParameters(MultiTermVectorsRequest request) {
 
     // issue #12311
     public void testMultiParserFilter() throws Exception {
-        byte[] data = StreamsUtils.copyToBytesFromClasspath("/org/elasticsearch/action/termvectors/multiRequest3.json");
-        BytesReference bytes = new BytesArray(data);
+        byte[] bytes = StreamsUtils.copyToBytesFromClasspath("/org/elasticsearch/action/termvectors/multiRequest3.json");
+        XContentParser data = createParser(JsonXContent.jsonXContent, bytes);
         MultiTermVectorsRequest request = new MultiTermVectorsRequest();
-        request.add(new TermVectorsRequest(), bytes);
+        request.add(new TermVectorsRequest(), data);
         checkParsedFilterParameters(request);
     }
 
diff --git a/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java b/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java
index 7b606ee415968..4ea43ff2b03af 100644
--- a/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java
@@ -19,65 +19,156 @@
 
 package org.elasticsearch.action.update;
 
-import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.support.replication.ReplicationRequest;
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.Streamable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.get.GetResult;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.script.MockScriptEngine;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.script.ScriptContextRegistry;
+import org.elasticsearch.script.ScriptEngineRegistry;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptSettings;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.RandomObjects;
+import org.elasticsearch.watcher.ResourceWatcherService;
+import org.junit.Before;
 
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.HashMap;
 import java.util.Map;
+import java.util.function.Function;
 
+import static java.util.Collections.emptyList;
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.singletonList;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+import static org.elasticsearch.script.MockScriptEngine.mockInlineScript;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
 import static org.hamcrest.Matchers.arrayContaining;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
-import static org.hamcrest.Matchers.nullValue;
 
 public class UpdateRequestTests extends ESTestCase {
-    public void testUpdateRequest() throws Exception {
+
+    private UpdateHelper updateHelper;
+
+    @Before
+    public void setUp() throws Exception {
+        super.setUp();
+        final Path genericConfigFolder = createTempDir();
+        final Settings baseSettings = Settings.builder()
+                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+                .put(Environment.PATH_CONF_SETTING.getKey(), genericConfigFolder)
+                .build();
+        final Environment environment = new Environment(baseSettings);
+        final Map<String, Function<Map<String, Object>, Object>> scripts =  new HashMap<>();
+        scripts.put(
+                "ctx._source.update_timestamp = ctx._now",
+                vars -> {
+                    @SuppressWarnings("unchecked")
+                    final Map<String, Object> ctx = (Map<String, Object>) vars.get("ctx");
+                    @SuppressWarnings("unchecked")
+                    final Map<String, Object> source = (Map<String, Object>) ctx.get("_source");
+                    source.put("update_timestamp", ctx.get("_now"));
+                    return null;
+                });
+        scripts.put(
+                "ctx._timestamp = ctx._now",
+                vars -> {
+                    @SuppressWarnings("unchecked")
+                    final Map<String, Object> ctx = (Map<String, Object>) vars.get("ctx");
+                    ctx.put("_timestamp", ctx.get("_now"));
+                    return null;
+                });
+        scripts.put(
+                "ctx.op = delete",
+                vars -> {
+                    @SuppressWarnings("unchecked")
+                    final Map<String, Object> ctx = (Map<String, Object>) vars.get("ctx");
+                    ctx.put("op", "delete");
+                    return null;
+                });
+        scripts.put("return", vars -> null);
+        final ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(emptyList());
+        final MockScriptEngine engine = new MockScriptEngine("mock", scripts);
+        final ScriptEngineRegistry scriptEngineRegistry =
+                new ScriptEngineRegistry(singletonList(engine));
+
+        final ScriptSettings scriptSettings =
+                new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
+        final ResourceWatcherService watcherService =
+                new ResourceWatcherService(baseSettings, null);
+        ScriptService scriptService = new ScriptService(
+                baseSettings,
+                environment,
+                watcherService,
+                scriptEngineRegistry,
+                scriptContextRegistry,
+                scriptSettings);
+        final Settings settings = settings(Version.CURRENT).build();
+
+        updateHelper = new UpdateHelper(settings, scriptService);
+    }
+
+    public void testFromXContent() throws Exception {
         UpdateRequest request = new UpdateRequest("test", "type", "1");
         // simple script
-        request.source(XContentFactory.jsonBuilder().startObject()
-                .field("script", "script1")
-                .endObject());
+        request.fromXContent(createParser(XContentFactory.jsonBuilder()
+                .startObject()
+                    .field("script", "script1")
+                .endObject()));
         Script script = request.script();
         assertThat(script, notNullValue());
-        assertThat(script.getScript(), equalTo("script1"));
+        assertThat(script.getIdOrCode(), equalTo("script1"));
         assertThat(script.getType(), equalTo(ScriptType.INLINE));
         assertThat(script.getLang(), equalTo(Script.DEFAULT_SCRIPT_LANG));
         Map<String, Object> params = script.getParams();
-        assertThat(params, nullValue());
+        assertThat(params, equalTo(emptyMap()));
 
         // simple verbose script
-        request.source(XContentFactory.jsonBuilder().startObject()
-                .startObject("script").field("inline", "script1").endObject()
-                .endObject());
+        request.fromXContent(createParser(XContentFactory.jsonBuilder().startObject()
+                    .startObject("script").field("source", "script1").endObject()
+                .endObject()));
         script = request.script();
         assertThat(script, notNullValue());
-        assertThat(script.getScript(), equalTo("script1"));
+        assertThat(script.getIdOrCode(), equalTo("script1"));
         assertThat(script.getType(), equalTo(ScriptType.INLINE));
         assertThat(script.getLang(), equalTo(Script.DEFAULT_SCRIPT_LANG));
         params = script.getParams();
-        assertThat(params, nullValue());
+        assertThat(params, equalTo(emptyMap()));
 
         // script with params
         request = new UpdateRequest("test", "type", "1");
-        request.source(XContentFactory.jsonBuilder().startObject().startObject("script").field("inline", "script1").startObject("params")
-                .field("param1", "value1").endObject().endObject().endObject());
+        request.fromXContent(createParser(XContentFactory.jsonBuilder().startObject()
+            .startObject("script")
+                .field("source", "script1")
+                .startObject("params")
+                    .field("param1", "value1")
+                .endObject()
+            .endObject().endObject()));
         script = request.script();
         assertThat(script, notNullValue());
-        assertThat(script.getScript(), equalTo("script1"));
+        assertThat(script.getIdOrCode(), equalTo("script1"));
         assertThat(script.getType(), equalTo(ScriptType.INLINE));
         assertThat(script.getLang(), equalTo(Script.DEFAULT_SCRIPT_LANG));
         params = script.getParams();
@@ -86,11 +177,18 @@ public void testUpdateRequest() throws Exception {
         assertThat(params.get("param1").toString(), equalTo("value1"));
 
         request = new UpdateRequest("test", "type", "1");
-        request.source(XContentFactory.jsonBuilder().startObject().startObject("script").startObject("params").field("param1", "value1")
-                .endObject().field("inline", "script1").endObject().endObject());
+        request.fromXContent(createParser(XContentFactory.jsonBuilder()
+                .startObject()
+                    .startObject("script")
+                        .startObject("params")
+                            .field("param1", "value1")
+                        .endObject()
+                        .field("source", "script1")
+                    .endObject()
+                .endObject()));
         script = request.script();
         assertThat(script, notNullValue());
-        assertThat(script.getScript(), equalTo("script1"));
+        assertThat(script.getIdOrCode(), equalTo("script1"));
         assertThat(script.getType(), equalTo(ScriptType.INLINE));
         assertThat(script.getLang(), equalTo(Script.DEFAULT_SCRIPT_LANG));
         params = script.getParams();
@@ -100,43 +198,71 @@ public void testUpdateRequest() throws Exception {
 
         // script with params and upsert
         request = new UpdateRequest("test", "type", "1");
-        request.source(XContentFactory.jsonBuilder().startObject().startObject("script").startObject("params").field("param1", "value1")
-                .endObject().field("inline", "script1").endObject().startObject("upsert").field("field1", "value1").startObject("compound")
-                .field("field2", "value2").endObject().endObject().endObject());
+        request.fromXContent(createParser(XContentFactory.jsonBuilder().startObject()
+            .startObject("script")
+                .startObject("params")
+                    .field("param1", "value1")
+                .endObject()
+                .field("source", "script1")
+            .endObject()
+            .startObject("upsert")
+                .field("field1", "value1")
+                .startObject("compound")
+                    .field("field2", "value2")
+                .endObject()
+            .endObject().endObject()));
         script = request.script();
         assertThat(script, notNullValue());
-        assertThat(script.getScript(), equalTo("script1"));
+        assertThat(script.getIdOrCode(), equalTo("script1"));
         assertThat(script.getType(), equalTo(ScriptType.INLINE));
         assertThat(script.getLang(), equalTo(Script.DEFAULT_SCRIPT_LANG));
         params = script.getParams();
         assertThat(params, notNullValue());
         assertThat(params.size(), equalTo(1));
         assertThat(params.get("param1").toString(), equalTo("value1"));
-        Map<String, Object> upsertDoc = XContentHelper.convertToMap(request.upsertRequest().source(), true).v2();
+        Map<String, Object> upsertDoc =
+            XContentHelper.convertToMap(request.upsertRequest().source(), true, request.upsertRequest().getContentType()).v2();
         assertThat(upsertDoc.get("field1").toString(), equalTo("value1"));
         assertThat(((Map) upsertDoc.get("compound")).get("field2").toString(), equalTo("value2"));
 
         request = new UpdateRequest("test", "type", "1");
-        request.source(XContentFactory.jsonBuilder().startObject().startObject("upsert").field("field1", "value1").startObject("compound")
-                .field("field2", "value2").endObject().endObject().startObject("script").startObject("params").field("param1", "value1")
-                .endObject().field("inline", "script1").endObject().endObject());
+        request.fromXContent(createParser(XContentFactory.jsonBuilder().startObject()
+            .startObject("upsert")
+                .field("field1", "value1")
+                .startObject("compound")
+                    .field("field2", "value2")
+                .endObject()
+            .endObject()
+            .startObject("script")
+                .startObject("params")
+                    .field("param1", "value1")
+                .endObject()
+                .field("source", "script1")
+            .endObject().endObject()));
         script = request.script();
         assertThat(script, notNullValue());
-        assertThat(script.getScript(), equalTo("script1"));
+        assertThat(script.getIdOrCode(), equalTo("script1"));
         assertThat(script.getType(), equalTo(ScriptType.INLINE));
         assertThat(script.getLang(), equalTo(Script.DEFAULT_SCRIPT_LANG));
         params = script.getParams();
         assertThat(params, notNullValue());
         assertThat(params.size(), equalTo(1));
         assertThat(params.get("param1").toString(), equalTo("value1"));
-        upsertDoc = XContentHelper.convertToMap(request.upsertRequest().source(), true).v2();
+        upsertDoc = XContentHelper.convertToMap(request.upsertRequest().source(), true, request.upsertRequest().getContentType()).v2();
         assertThat(upsertDoc.get("field1").toString(), equalTo("value1"));
         assertThat(((Map) upsertDoc.get("compound")).get("field2").toString(), equalTo("value2"));
 
         // script with doc
         request = new UpdateRequest("test", "type", "1");
-        request.source(XContentFactory.jsonBuilder().startObject().startObject("doc").field("field1", "value1").startObject("compound")
-                .field("field2", "value2").endObject().endObject().endObject());
+        request.fromXContent(createParser(XContentFactory.jsonBuilder()
+                .startObject()
+                    .startObject("doc")
+                        .field("field1", "value1")
+                        .startObject("compound")
+                            .field("field2", "value2")
+                        .endObject()
+                    .endObject()
+                .endObject()));
         Map<String, Object> doc = request.doc().sourceAsMap();
         assertThat(doc.get("field1").toString(), equalTo("value1"));
         assertThat(((Map) doc.get("compound")).get("field2").toString(), equalTo("value2"));
@@ -157,9 +283,10 @@ public void testUpdateRequestWithTTL() throws Exception {
                 .doc(jsonBuilder().startObject().field("fooz", "baz").endObject())
                 .upsert(indexRequest);
 
+        long nowInMillis = randomNonNegativeLong();
         // We simulate that the document is not existing yet
         GetResult getResult = new GetResult("test", "type1", "1", 0, false, null, null);
-        UpdateHelper.Result result = updateHelper.prepare(new ShardId("test", "_na_", 0),updateRequest, getResult);
+        UpdateHelper.Result result = updateHelper.prepare(new ShardId("test", "_na_", 0),updateRequest, getResult, () -> nowInMillis);
         Streamable action = result.action();
         assertThat(action, instanceOf(IndexRequest.class));
         IndexRequest indexAction = (IndexRequest) action;
@@ -176,34 +303,235 @@ public void testUpdateRequestWithTTL() throws Exception {
 
         // We simulate that the document is not existing yet
         getResult = new GetResult("test", "type1", "2", 0, false, null, null);
-        result = updateHelper.prepare(new ShardId("test", "_na_", 0), updateRequest, getResult);
+        result = updateHelper.prepare(new ShardId("test", "_na_", 0), updateRequest, getResult, () -> nowInMillis);
         action = result.action();
         assertThat(action, instanceOf(IndexRequest.class));
         indexAction = (IndexRequest) action;
         assertThat(indexAction.ttl(), is(providedTTLValue));
     }
 
-    // Related to issue #15822
-    public void testInvalidBodyThrowsParseException() throws Exception {
-        UpdateRequest request = new UpdateRequest("test", "type", "1");
-        try {
-            request.source(new byte[] { (byte) '"' });
-            fail("Should have thrown a ElasticsearchParseException");
-        } catch (ElasticsearchParseException e) {
-            assertThat(e.getMessage(), equalTo("Failed to derive xcontent"));
-        }
-    }
-
     // Related to issue 15338
     public void testFieldsParsing() throws Exception {
-        UpdateRequest request = new UpdateRequest("test", "type1", "1")
-                .source(new BytesArray("{\"doc\": {\"field1\": \"value1\"}, \"fields\": \"_source\"}"));
+        UpdateRequest request = new UpdateRequest("test", "type1", "1").fromXContent(
+                createParser(JsonXContent.jsonXContent, new BytesArray("{\"doc\": {\"field1\": \"value1\"}, \"fields\": \"_source\"}")));
         assertThat(request.doc().sourceAsMap().get("field1").toString(), equalTo("value1"));
         assertThat(request.fields(), arrayContaining("_source"));
 
-        request = new UpdateRequest("test", "type2", "2")
-                .source(new BytesArray("{\"doc\": {\"field2\": \"value2\"}, \"fields\": [\"field1\", \"field2\"]}"));
+        request = new UpdateRequest("test", "type2", "2").fromXContent(createParser(JsonXContent.jsonXContent,
+                new BytesArray("{\"doc\": {\"field2\": \"value2\"}, \"fields\": [\"field1\", \"field2\"]}")));
         assertThat(request.doc().sourceAsMap().get("field2").toString(), equalTo("value2"));
         assertThat(request.fields(), arrayContaining("field1", "field2"));
     }
+
+    public void testFetchSourceParsing() throws Exception {
+        UpdateRequest request = new UpdateRequest("test", "type1", "1");
+        request.fromXContent(createParser(XContentFactory.jsonBuilder()
+                .startObject()
+                    .field("_source", true)
+                .endObject()));
+        assertThat(request.fetchSource(), notNullValue());
+        assertThat(request.fetchSource().includes().length, equalTo(0));
+        assertThat(request.fetchSource().excludes().length, equalTo(0));
+        assertThat(request.fetchSource().fetchSource(), equalTo(true));
+
+        request.fromXContent(createParser(XContentFactory.jsonBuilder()
+                .startObject()
+                    .field("_source", false)
+                .endObject()));
+        assertThat(request.fetchSource(), notNullValue());
+        assertThat(request.fetchSource().includes().length, equalTo(0));
+        assertThat(request.fetchSource().excludes().length, equalTo(0));
+        assertThat(request.fetchSource().fetchSource(), equalTo(false));
+
+        request.fromXContent(createParser(XContentFactory.jsonBuilder()
+                .startObject()
+                    .field("_source", "path.inner.*")
+                .endObject()));
+        assertThat(request.fetchSource(), notNullValue());
+        assertThat(request.fetchSource().fetchSource(), equalTo(true));
+        assertThat(request.fetchSource().includes().length, equalTo(1));
+        assertThat(request.fetchSource().excludes().length, equalTo(0));
+        assertThat(request.fetchSource().includes()[0], equalTo("path.inner.*"));
+
+        request.fromXContent(createParser(XContentFactory.jsonBuilder()
+                .startObject()
+                    .startObject("_source")
+                        .field("includes", "path.inner.*")
+                        .field("excludes", "another.inner.*")
+                    .endObject()
+                .endObject()));
+        assertThat(request.fetchSource(), notNullValue());
+        assertThat(request.fetchSource().fetchSource(), equalTo(true));
+        assertThat(request.fetchSource().includes().length, equalTo(1));
+        assertThat(request.fetchSource().excludes().length, equalTo(1));
+        assertThat(request.fetchSource().includes()[0], equalTo("path.inner.*"));
+        assertThat(request.fetchSource().excludes()[0], equalTo("another.inner.*"));
+    }
+
+    public void testNowInScript() throws IOException {
+        final TimeValue providedTTLValue = TimeValue.parseTimeValue(randomTimeValue(), null, "ttl");
+        // We just upsert one document with now() using a script
+        IndexRequest indexRequest = new IndexRequest("test", "type1", "2")
+            .source(jsonBuilder().startObject().field("foo", "bar").endObject())
+            .ttl(providedTTLValue);
+
+        {
+            UpdateRequest updateRequest = new UpdateRequest("test", "type1", "2")
+                .upsert(indexRequest)
+                .script(mockInlineScript("ctx._source.update_timestamp = ctx._now"))
+                .scriptedUpsert(true);
+            long nowInMillis = randomNonNegativeLong();
+            // We simulate that the document is not existing yet
+            GetResult getResult = new GetResult("test", "type1", "2", 0, false, null, null);
+            UpdateHelper.Result result = updateHelper.prepare(new ShardId("test", "_na_", 0), updateRequest, getResult, () -> nowInMillis);
+            Streamable action = result.action();
+            assertThat(action, instanceOf(IndexRequest.class));
+            IndexRequest indexAction = (IndexRequest) action;
+            assertEquals(nowInMillis, indexAction.sourceAsMap().get("update_timestamp"));
+        }
+        {
+            UpdateRequest updateRequest = new UpdateRequest("test", "type1", "2")
+                .upsert(indexRequest)
+                .script(mockInlineScript("ctx._timestamp = ctx._now"))
+                .scriptedUpsert(true);
+            long nowInMillis = randomNonNegativeLong();
+            // We simulate that the document is not existing yet
+            GetResult getResult = new GetResult("test", "type1", "2", 0, true, new BytesArray("{}"), null);
+            UpdateHelper.Result result = updateHelper.prepare(new ShardId("test", "_na_", 0), updateRequest, getResult, () -> nowInMillis);
+            Streamable action = result.action();
+            assertThat(action, instanceOf(IndexRequest.class));
+            IndexRequest indexAction = (IndexRequest) action;
+            assertEquals(Long.toString(nowInMillis), indexAction.timestamp());
+        }
+    }
+
+    public void testIndexTimeout() {
+        final GetResult getResult =
+                new GetResult("test", "type", "1", 0, true, new BytesArray("{\"f\":\"v\"}"), null);
+        final UpdateRequest updateRequest =
+                new UpdateRequest("test", "type", "1")
+                        .script(mockInlineScript("return"))
+                        .timeout(randomTimeValue());
+        runTimeoutTest(getResult, updateRequest);
+    }
+
+    public void testDeleteTimeout() {
+        final GetResult getResult =
+                new GetResult("test", "type", "1", 0, true, new BytesArray("{\"f\":\"v\"}"), null);
+        final UpdateRequest updateRequest =
+                new UpdateRequest("test", "type", "1")
+                        .script(mockInlineScript("ctx.op = delete"))
+                        .timeout(randomTimeValue());
+        runTimeoutTest(getResult, updateRequest);
+    }
+
+    public void testUpsertTimeout() throws IOException {
+        final boolean exists = randomBoolean();
+        final BytesReference source = exists ? new BytesArray("{\"f\":\"v\"}") : null;
+        final GetResult getResult = new GetResult("test", "type", "1", 0, exists, source, null);
+        final XContentBuilder sourceBuilder = jsonBuilder();
+        sourceBuilder.startObject();
+        {
+            sourceBuilder.field("f", "v");
+        }
+        sourceBuilder.endObject();
+        final IndexRequest upsert = new IndexRequest("test", "type", "1").source(sourceBuilder);
+        final UpdateRequest updateRequest =
+                new UpdateRequest("test", "type", "1")
+                .upsert(upsert)
+                .script(mockInlineScript("return"))
+                .timeout(randomTimeValue());
+        runTimeoutTest(getResult, updateRequest);
+    }
+
+    private void runTimeoutTest(final GetResult getResult, final UpdateRequest updateRequest) {
+        final UpdateHelper.Result result = updateHelper.prepare(
+                new ShardId("test", "", 0),
+                updateRequest,
+                getResult,
+                ESTestCase::randomNonNegativeLong);
+        final Streamable action = result.action();
+        assertThat(action, instanceOf(ReplicationRequest.class));
+        final ReplicationRequest request = (ReplicationRequest) action;
+        assertThat(request.timeout(), equalTo(updateRequest.timeout()));
+    }
+
+    public void testToAndFromXContent() throws IOException {
+        UpdateRequest updateRequest = new UpdateRequest();
+        updateRequest.detectNoop(randomBoolean());
+
+        if (randomBoolean()) {
+            XContentType xContentType = randomFrom(XContentType.values());
+            BytesReference source = RandomObjects.randomSource(random(), xContentType);
+            updateRequest.doc(new IndexRequest().source(source, xContentType));
+            updateRequest.docAsUpsert(randomBoolean());
+        } else {
+            ScriptType scriptType = randomFrom(ScriptType.values());
+            String scriptLang = (scriptType != ScriptType.STORED) ? randomAlphaOfLength(10) : null;
+            String scriptIdOrCode = randomAlphaOfLength(10);
+            int nbScriptParams = randomIntBetween(0, 5);
+            Map<String, Object> scriptParams = new HashMap<>(nbScriptParams);
+            for (int i = 0; i < nbScriptParams; i++) {
+                scriptParams.put(randomAlphaOfLength(5), randomAlphaOfLength(5));
+            }
+            updateRequest.script(new Script(scriptType, scriptLang, scriptIdOrCode, scriptParams));
+            updateRequest.scriptedUpsert(randomBoolean());
+        }
+        if (randomBoolean()) {
+            XContentType xContentType = randomFrom(XContentType.values());
+            BytesReference source = RandomObjects.randomSource(random(), xContentType);
+            updateRequest.upsert(new IndexRequest().source(source, xContentType));
+        }
+        if (randomBoolean()) {
+            String[] fields = new String[randomIntBetween(0, 5)];
+            for (int i = 0; i < fields.length; i++) {
+                fields[i] = randomAlphaOfLength(5);
+            }
+            updateRequest.fields(fields);
+        }
+        if (randomBoolean()) {
+            if (randomBoolean()) {
+                updateRequest.fetchSource(randomBoolean());
+            } else {
+                String[] includes = new String[randomIntBetween(0, 5)];
+                for (int i = 0; i < includes.length; i++) {
+                    includes[i] = randomAlphaOfLength(5);
+                }
+                String[] excludes = new String[randomIntBetween(0, 5)];
+                for (int i = 0; i < excludes.length; i++) {
+                    excludes[i] = randomAlphaOfLength(5);
+                }
+                if (randomBoolean()) {
+                    updateRequest.fetchSource(includes, excludes);
+                }
+            }
+        }
+
+        XContentType xContentType = randomFrom(XContentType.values());
+        boolean humanReadable = randomBoolean();
+        BytesReference originalBytes = toShuffledXContent(updateRequest, xContentType, ToXContent.EMPTY_PARAMS, humanReadable);
+
+        if (randomBoolean()) {
+            try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) {
+                originalBytes = shuffleXContent(parser, randomBoolean()).bytes();
+            }
+        }
+
+        UpdateRequest parsedUpdateRequest = new UpdateRequest();
+        try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) {
+            parsedUpdateRequest.fromXContent(parser);
+            assertNull(parser.nextToken());
+        }
+
+        assertEquals(updateRequest.detectNoop(), parsedUpdateRequest.detectNoop());
+        assertEquals(updateRequest.docAsUpsert(), parsedUpdateRequest.docAsUpsert());
+        assertEquals(updateRequest.docAsUpsert(), parsedUpdateRequest.docAsUpsert());
+        assertEquals(updateRequest.script(), parsedUpdateRequest.script());
+        assertEquals(updateRequest.scriptedUpsert(), parsedUpdateRequest.scriptedUpsert());
+        assertArrayEquals(updateRequest.fields(), parsedUpdateRequest.fields());
+        assertEquals(updateRequest.fetchSource(), parsedUpdateRequest.fetchSource());
+
+        BytesReference finalBytes = toXContent(parsedUpdateRequest, xContentType, humanReadable);
+        assertToXContentEquivalent(originalBytes, finalBytes, xContentType);
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/update/UpdateResponseTests.java b/core/src/test/java/org/elasticsearch/action/update/UpdateResponseTests.java
new file mode 100644
index 0000000000000..ba72ac2fd4f06
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/update/UpdateResponseTests.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.update;
+
+import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.action.index.IndexResponseTests;
+import org.elasticsearch.action.support.replication.ReplicationResponse;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.get.GetField;
+import org.elasticsearch.index.get.GetResult;
+import org.elasticsearch.index.get.GetResultTests;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.RandomObjects;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.Predicate;
+
+import static org.elasticsearch.action.DocWriteResponse.Result.DELETED;
+import static org.elasticsearch.action.DocWriteResponse.Result.NOT_FOUND;
+import static org.elasticsearch.action.DocWriteResponse.Result.UPDATED;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.INDEX_UUID_NA_VALUE;
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+import static org.elasticsearch.test.XContentTestUtils.insertRandomFields;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
+
+public class UpdateResponseTests extends ESTestCase {
+
+    public void testToXContent() throws IOException {
+        {
+            UpdateResponse updateResponse = new UpdateResponse(new ShardId("index", "index_uuid", 0), "type", "id", 0, NOT_FOUND);
+            String output = Strings.toString(updateResponse);
+            assertEquals("{\"_index\":\"index\",\"_type\":\"type\",\"_id\":\"id\",\"_version\":0,\"result\":\"not_found\"," +
+                    "\"_shards\":{\"total\":0,\"successful\":0,\"failed\":0}}", output);
+        }
+        {
+            UpdateResponse updateResponse = new UpdateResponse(new ReplicationResponse.ShardInfo(10, 6),
+                    new ShardId("index", "index_uuid", 1), "type", "id", 1, DELETED);
+            String output = Strings.toString(updateResponse);
+            assertEquals("{\"_index\":\"index\",\"_type\":\"type\",\"_id\":\"id\",\"_version\":1,\"result\":\"deleted\"," +
+                    "\"_shards\":{\"total\":10,\"successful\":6,\"failed\":0}}", output);
+        }
+        {
+            BytesReference source = new BytesArray("{\"title\":\"Book title\",\"isbn\":\"ABC-123\"}");
+            Map<String, GetField> fields = new HashMap<>();
+            fields.put("title", new GetField("title", Collections.singletonList("Book title")));
+            fields.put("isbn", new GetField("isbn", Collections.singletonList("ABC-123")));
+
+            UpdateResponse updateResponse = new UpdateResponse(new ReplicationResponse.ShardInfo(3, 2),
+                    new ShardId("books", "books_uuid", 2), "book", "1", 2, UPDATED);
+            updateResponse.setGetResult(new GetResult("books", "book", "1", 2, true, source, fields));
+
+            String output = Strings.toString(updateResponse);
+            assertEquals("{\"_index\":\"books\",\"_type\":\"book\",\"_id\":\"1\",\"_version\":2,\"result\":\"updated\"," +
+                    "\"_shards\":{\"total\":3,\"successful\":2,\"failed\":0},\"get\":{\"found\":true," +
+                    "\"_source\":{\"title\":\"Book title\",\"isbn\":\"ABC-123\"},\"fields\":{\"isbn\":[\"ABC-123\"],\"title\":[\"Book " +
+                    "title\"]}}}", output);
+        }
+    }
+
+    public void testToAndFromXContent() throws IOException {
+        doFromXContentTestWithRandomFields(false);
+    }
+
+    /**
+     * This test adds random fields and objects to the xContent rendered out to
+     * ensure we can parse it back to be forward compatible with additions to
+     * the xContent
+     */
+    public void testFromXContentWithRandomFields() throws IOException {
+        doFromXContentTestWithRandomFields(true);
+    }
+
+    private void doFromXContentTestWithRandomFields(boolean addRandomFields) throws IOException {
+        final XContentType xContentType = randomFrom(XContentType.JSON);
+        final Tuple<UpdateResponse, UpdateResponse> tuple = randomUpdateResponse(xContentType);
+        UpdateResponse updateResponse = tuple.v1();
+        UpdateResponse expectedUpdateResponse = tuple.v2();
+
+        boolean humanReadable = randomBoolean();
+        BytesReference originalBytes = toShuffledXContent(updateResponse, xContentType, ToXContent.EMPTY_PARAMS, humanReadable);
+
+        BytesReference mutated;
+        if (addRandomFields) {
+            // - The ShardInfo.Failure's exception is rendered out in a "reason" object. We shouldn't add anything random there
+            // because exception rendering and parsing are very permissive: any extra object or field would be rendered as
+            // a exception custom metadata and be parsed back as a custom header, making it impossible to compare the results
+            // in this test.
+            // - The GetResult's "_source" and "fields" just consists of key/value pairs, we shouldn't add anything random there.
+            // It is already randomized in the randomGetResult() method anyway. Also, we cannot add anything within the "get"
+            // object since this is where GetResult's metadata fields are rendered out and they would be parsed back as
+            // extra metadata fields.
+            Predicate<String> excludeFilter = path -> path.contains("reason") || path.contains("get");
+            mutated = insertRandomFields(xContentType, originalBytes, excludeFilter, random());
+        } else {
+            mutated = originalBytes;
+        }
+        UpdateResponse parsedUpdateResponse;
+        try (XContentParser parser = createParser(xContentType.xContent(), mutated)) {
+            parsedUpdateResponse = UpdateResponse.fromXContent(parser);
+            assertNull(parser.nextToken());
+        }
+
+        IndexResponseTests.assertDocWriteResponse(expectedUpdateResponse, parsedUpdateResponse);
+        if (addRandomFields == false) {
+            assertEquals(expectedUpdateResponse.getGetResult(), parsedUpdateResponse.getGetResult());
+        }
+
+        // Prints out the parsed UpdateResponse object to verify that it is the same as the expected output.
+        // If random fields have been inserted, it checks that they have been filtered out and that they do
+        // not alter the final output of the parsed object.
+        BytesReference parsedBytes = toXContent(parsedUpdateResponse, xContentType, humanReadable);
+        BytesReference expectedBytes = toXContent(expectedUpdateResponse, xContentType, humanReadable);
+        assertToXContentEquivalent(expectedBytes, parsedBytes, xContentType);
+    }
+
+    /**
+     * Returns a tuple of {@link UpdateResponse}s.
+     * <p>
+     * The left element is the actual {@link UpdateResponse} to serialize while the right element is the
+     * expected {@link UpdateResponse} after parsing.
+     */
+    public static Tuple<UpdateResponse, UpdateResponse> randomUpdateResponse(XContentType xContentType) {
+        Tuple<GetResult, GetResult> getResults = GetResultTests.randomGetResult(xContentType);
+        GetResult actualGetResult = getResults.v1();
+        GetResult expectedGetResult = getResults.v2();
+
+        String index = actualGetResult.getIndex();
+        String type = actualGetResult.getType();
+        String id = actualGetResult.getId();
+        long version = actualGetResult.getVersion();
+        DocWriteResponse.Result result = actualGetResult.isExists() ? DocWriteResponse.Result.UPDATED : DocWriteResponse.Result.NOT_FOUND;
+        String indexUUid = randomAlphaOfLength(5);
+        int shardId = randomIntBetween(0, 5);
+
+        ShardId actualShardId = new ShardId(index, indexUUid, shardId);
+        ShardId expectedShardId = new ShardId(index, INDEX_UUID_NA_VALUE, -1);
+
+        UpdateResponse actual, expected;
+        if (randomBoolean()) {
+            Tuple<ReplicationResponse.ShardInfo, ReplicationResponse.ShardInfo> shardInfos = RandomObjects.randomShardInfo(random());
+
+            actual = new UpdateResponse(shardInfos.v1(), actualShardId, type, id, version, result);
+            expected = new UpdateResponse(shardInfos.v2(), expectedShardId, type, id, version, result);
+        } else {
+            actual = new UpdateResponse(actualShardId, type, id, version, result);
+            expected = new UpdateResponse(expectedShardId, type, id, version, result);
+        }
+
+        if (actualGetResult.isExists()) {
+            actual.setGetResult(actualGetResult);
+        }
+
+        if (expectedGetResult.isExists()) {
+            expected.setGetResult(expectedGetResult);
+        }
+
+        boolean forcedRefresh = randomBoolean();
+        actual.setForcedRefresh(forcedRefresh);
+        expected.setForcedRefresh(forcedRefresh);
+
+        return Tuple.tuple(actual, expected);
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java b/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java
index ebca512baec79..73e110ba8757e 100644
--- a/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java
+++ b/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.aliases;
 
-import org.apache.lucene.search.join.ScoreMode;
 import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions;
 import org.elasticsearch.action.admin.indices.alias.exists.AliasesExistResponse;
@@ -35,6 +34,7 @@
 import org.elasticsearch.common.StopWatch;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.rest.action.admin.indices.AliasesNotFoundException;
@@ -62,9 +62,6 @@
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_READ;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_WRITE;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY;
-import static org.elasticsearch.index.query.QueryBuilders.hasChildQuery;
-import static org.elasticsearch.index.query.QueryBuilders.hasParentQuery;
-import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.index.query.QueryBuilders.rangeQuery;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
 import static org.elasticsearch.test.hamcrest.CollectionAssertions.hasKey;
@@ -90,7 +87,8 @@ public void testAliases() throws Exception {
         assertAcked(admin().indices().prepareAliases().addAlias("test", "alias1"));
 
         logger.info("--> indexing against [alias1], should work now");
-        IndexResponse indexResponse = client().index(indexRequest("alias1").type("type1").id("1").source(source("1", "test"))).actionGet();
+        IndexResponse indexResponse = client().index(indexRequest("alias1").type("type1").id("1")
+            .source(source("1", "test"), XContentType.JSON)).actionGet();
         assertThat(indexResponse.getIndex(), equalTo("test"));
 
         logger.info("--> creating index [test_x]");
@@ -102,7 +100,8 @@ public void testAliases() throws Exception {
         assertAcked(admin().indices().prepareAliases().removeAlias("test", "alias1").addAlias("test_x", "alias1"));
 
         logger.info("--> indexing against [alias1], should work against [test_x]");
-        indexResponse = client().index(indexRequest("alias1").type("type1").id("1").source(source("1", "test"))).actionGet();
+        indexResponse = client().index(indexRequest("alias1").type("type1").id("1")
+            .source(source("1", "test"), XContentType.JSON)).actionGet();
         assertThat(indexResponse.getIndex(), equalTo("test_x"));
     }
 
@@ -162,15 +161,14 @@ public void testSearchingFilteringAliasesSingleIndex() throws Exception {
         assertAcked(admin().indices().prepareAliases().addAlias("test", "tests", termQuery("name", "test")));
 
         logger.info("--> indexing against [test]");
-        client().index(indexRequest("test").type("type1").id("1").source(source("1", "foo test")).setRefreshPolicy(RefreshPolicy.IMMEDIATE))
-                .actionGet();
-        client().index(indexRequest("test").type("type1").id("2").source(source("2", "bar test")).setRefreshPolicy(RefreshPolicy.IMMEDIATE))
-                .actionGet();
-        client().index(indexRequest("test").type("type1").id("3").source(source("3", "baz test")).setRefreshPolicy(RefreshPolicy.IMMEDIATE))
-                .actionGet();
-        client().index(
-                indexRequest("test").type("type1").id("4").source(source("4", "something else")).setRefreshPolicy(RefreshPolicy.IMMEDIATE))
-                .actionGet();
+        client().index(indexRequest("test").type("type1").id("1").source(source("1", "foo test"), XContentType.JSON)
+            .setRefreshPolicy(RefreshPolicy.IMMEDIATE)).actionGet();
+        client().index(indexRequest("test").type("type1").id("2").source(source("2", "bar test"), XContentType.JSON)
+            .setRefreshPolicy(RefreshPolicy.IMMEDIATE)).actionGet();
+        client().index(indexRequest("test").type("type1").id("3").source(source("3", "baz test"), XContentType.JSON)
+            .setRefreshPolicy(RefreshPolicy.IMMEDIATE)).actionGet();
+        client().index(indexRequest("test").type("type1").id("4").source(source("4", "something else"), XContentType.JSON)
+                    .setRefreshPolicy(RefreshPolicy.IMMEDIATE)).actionGet();
 
         logger.info("--> checking single filtering alias search");
         SearchResponse searchResponse = client().prepareSearch("foos").setQuery(QueryBuilders.matchAllQuery()).get();
@@ -184,7 +182,7 @@ public void testSearchingFilteringAliasesSingleIndex() throws Exception {
         assertHits(searchResponse.getHits(), "1", "2", "3");
 
         logger.info("--> checking single filtering alias search with sort");
-        searchResponse = client().prepareSearch("tests").setQuery(QueryBuilders.matchAllQuery()).addSort("_uid", SortOrder.ASC).get();
+        searchResponse = client().prepareSearch("tests").setQuery(QueryBuilders.matchAllQuery()).addSort("_index", SortOrder.ASC).get();
         assertHits(searchResponse.getHits(), "1", "2", "3");
 
         logger.info("--> checking single filtering alias search with global facets");
@@ -199,7 +197,7 @@ public void testSearchingFilteringAliasesSingleIndex() throws Exception {
         logger.info("--> checking single filtering alias search with global facets and sort");
         searchResponse = client().prepareSearch("tests").setQuery(QueryBuilders.matchQuery("name", "bar"))
                 .addAggregation(AggregationBuilders.global("global").subAggregation(AggregationBuilders.terms("test").field("name")))
-                .addSort("_uid", SortOrder.ASC).get();
+                .addSort("_index", SortOrder.ASC).get();
         assertSearchResponse(searchResponse);
         global = searchResponse.getAggregations().get("global");
         terms = global.getAggregations().get("test");
@@ -208,7 +206,7 @@ public void testSearchingFilteringAliasesSingleIndex() throws Exception {
         logger.info("--> checking single filtering alias search with non-global facets");
         searchResponse = client().prepareSearch("tests").setQuery(QueryBuilders.matchQuery("name", "bar"))
                 .addAggregation(AggregationBuilders.terms("test").field("name"))
-                .addSort("_uid", SortOrder.ASC).get();
+                .addSort("_index", SortOrder.ASC).get();
         assertSearchResponse(searchResponse);
         terms = searchResponse.getAggregations().get("test");
         assertThat(terms.getBuckets().size(), equalTo(2));
@@ -252,16 +250,16 @@ public void testSearchingFilteringAliasesTwoIndices() throws Exception {
         assertAcked(admin().indices().prepareAliases().addAlias("test2", "foos", termQuery("name", "foo")));
 
         logger.info("--> indexing against [test1]");
-        client().index(indexRequest("test1").type("type1").id("1").source(source("1", "foo test"))).get();
-        client().index(indexRequest("test1").type("type1").id("2").source(source("2", "bar test"))).get();
-        client().index(indexRequest("test1").type("type1").id("3").source(source("3", "baz test"))).get();
-        client().index(indexRequest("test1").type("type1").id("4").source(source("4", "something else"))).get();
+        client().index(indexRequest("test1").type("type1").id("1").source(source("1", "foo test"), XContentType.JSON)).get();
+        client().index(indexRequest("test1").type("type1").id("2").source(source("2", "bar test"), XContentType.JSON)).get();
+        client().index(indexRequest("test1").type("type1").id("3").source(source("3", "baz test"), XContentType.JSON)).get();
+        client().index(indexRequest("test1").type("type1").id("4").source(source("4", "something else"), XContentType.JSON)).get();
 
         logger.info("--> indexing against [test2]");
-        client().index(indexRequest("test2").type("type1").id("5").source(source("5", "foo test"))).get();
-        client().index(indexRequest("test2").type("type1").id("6").source(source("6", "bar test"))).get();
-        client().index(indexRequest("test2").type("type1").id("7").source(source("7", "baz test"))).get();
-        client().index(indexRequest("test2").type("type1").id("8").source(source("8", "something else"))).get();
+        client().index(indexRequest("test2").type("type1").id("5").source(source("5", "foo test"), XContentType.JSON)).get();
+        client().index(indexRequest("test2").type("type1").id("6").source(source("6", "bar test"), XContentType.JSON)).get();
+        client().index(indexRequest("test2").type("type1").id("7").source(source("7", "baz test"), XContentType.JSON)).get();
+        client().index(indexRequest("test2").type("type1").id("8").source(source("8", "something else"), XContentType.JSON)).get();
 
         refresh();
 
@@ -320,17 +318,17 @@ public void testSearchingFilteringAliasesMultipleIndices() throws Exception {
         assertAcked(admin().indices().prepareAliases().addAlias("test3", "filter13", termQuery("name", "baz")));
 
         logger.info("--> indexing against [test1]");
-        client().index(indexRequest("test1").type("type1").id("11").source(source("11", "foo test1"))).get();
-        client().index(indexRequest("test1").type("type1").id("12").source(source("12", "bar test1"))).get();
-        client().index(indexRequest("test1").type("type1").id("13").source(source("13", "baz test1"))).get();
+        client().index(indexRequest("test1").type("type1").id("11").source(source("11", "foo test1"), XContentType.JSON)).get();
+        client().index(indexRequest("test1").type("type1").id("12").source(source("12", "bar test1"), XContentType.JSON)).get();
+        client().index(indexRequest("test1").type("type1").id("13").source(source("13", "baz test1"), XContentType.JSON)).get();
 
-        client().index(indexRequest("test2").type("type1").id("21").source(source("21", "foo test2"))).get();
-        client().index(indexRequest("test2").type("type1").id("22").source(source("22", "bar test2"))).get();
-        client().index(indexRequest("test2").type("type1").id("23").source(source("23", "baz test2"))).get();
+        client().index(indexRequest("test2").type("type1").id("21").source(source("21", "foo test2"), XContentType.JSON)).get();
+        client().index(indexRequest("test2").type("type1").id("22").source(source("22", "bar test2"), XContentType.JSON)).get();
+        client().index(indexRequest("test2").type("type1").id("23").source(source("23", "baz test2"), XContentType.JSON)).get();
 
-        client().index(indexRequest("test3").type("type1").id("31").source(source("31", "foo test3"))).get();
-        client().index(indexRequest("test3").type("type1").id("32").source(source("32", "bar test3"))).get();
-        client().index(indexRequest("test3").type("type1").id("33").source(source("33", "baz test3"))).get();
+        client().index(indexRequest("test3").type("type1").id("31").source(source("31", "foo test3"), XContentType.JSON)).get();
+        client().index(indexRequest("test3").type("type1").id("32").source(source("32", "bar test3"), XContentType.JSON)).get();
+        client().index(indexRequest("test3").type("type1").id("33").source(source("33", "baz test3"), XContentType.JSON)).get();
 
         refresh();
 
@@ -380,16 +378,16 @@ public void testDeletingByQueryFilteringAliases() throws Exception {
         assertAcked(admin().indices().prepareAliases().addAlias("test2", "tests", termQuery("name", "test")));
 
         logger.info("--> indexing against [test1]");
-        client().index(indexRequest("test1").type("type1").id("1").source(source("1", "foo test"))).get();
-        client().index(indexRequest("test1").type("type1").id("2").source(source("2", "bar test"))).get();
-        client().index(indexRequest("test1").type("type1").id("3").source(source("3", "baz test"))).get();
-        client().index(indexRequest("test1").type("type1").id("4").source(source("4", "something else"))).get();
+        client().index(indexRequest("test1").type("type1").id("1").source(source("1", "foo test"), XContentType.JSON)).get();
+        client().index(indexRequest("test1").type("type1").id("2").source(source("2", "bar test"), XContentType.JSON)).get();
+        client().index(indexRequest("test1").type("type1").id("3").source(source("3", "baz test"), XContentType.JSON)).get();
+        client().index(indexRequest("test1").type("type1").id("4").source(source("4", "something else"), XContentType.JSON)).get();
 
         logger.info("--> indexing against [test2]");
-        client().index(indexRequest("test2").type("type1").id("5").source(source("5", "foo test"))).get();
-        client().index(indexRequest("test2").type("type1").id("6").source(source("6", "bar test"))).get();
-        client().index(indexRequest("test2").type("type1").id("7").source(source("7", "baz test"))).get();
-        client().index(indexRequest("test2").type("type1").id("8").source(source("8", "something else"))).get();
+        client().index(indexRequest("test2").type("type1").id("5").source(source("5", "foo test"), XContentType.JSON)).get();
+        client().index(indexRequest("test2").type("type1").id("6").source(source("6", "bar test"), XContentType.JSON)).get();
+        client().index(indexRequest("test2").type("type1").id("7").source(source("7", "baz test"), XContentType.JSON)).get();
+        client().index(indexRequest("test2").type("type1").id("8").source(source("8", "something else"), XContentType.JSON)).get();
 
         refresh();
 
@@ -433,7 +431,7 @@ public void testWaitForAliasCreationMultipleShards() throws Exception {
 
         for (int i = 0; i < 10; i++) {
             assertAcked(admin().indices().prepareAliases().addAlias("test", "alias" + i));
-            client().index(indexRequest("alias" + i).type("type1").id("1").source(source("1", "test"))).get();
+            client().index(indexRequest("alias" + i).type("type1").id("1").source(source("1", "test"), XContentType.JSON)).get();
         }
     }
 
@@ -445,7 +443,7 @@ public void testWaitForAliasCreationSingleShard() throws Exception {
 
         for (int i = 0; i < 10; i++) {
             assertAcked(admin().indices().prepareAliases().addAlias("test", "alias" + i));
-            client().index(indexRequest("alias" + i).type("type1").id("1").source(source("1", "test"))).get();
+            client().index(indexRequest("alias" + i).type("type1").id("1").source(source("1", "test"), XContentType.JSON)).get();
         }
     }
 
@@ -464,7 +462,8 @@ public void testWaitForAliasSimultaneousUpdate() throws Exception {
                 @Override
                 public void run() {
                     assertAcked(admin().indices().prepareAliases().addAlias("test", aliasName));
-                    client().index(indexRequest(aliasName).type("type1").id("1").source(source("1", "test"))).actionGet();
+                    client().index(indexRequest(aliasName).type("type1").id("1").source(source("1", "test"), XContentType.JSON))
+                        .actionGet();
                 }
             });
         }
@@ -743,7 +742,7 @@ public void testCreateIndexWithAliasesInSource() throws Exception {
                 "        \"alias2\" : {\"filter\" : {\"match_all\": {}}},\n" +
                 "        \"alias3\" : { \"index_routing\" : \"index\", \"search_routing\" : \"search\"}\n" +
                 "    }\n" +
-                "}"));
+                "}", XContentType.JSON));
 
         checkAliases();
     }
@@ -819,26 +818,6 @@ public void testAliasFilterWithNowInRangeFilterAndQuery() throws Exception {
         }
     }
 
-    public void testAliasesFilterWithHasChildQuery() throws Exception {
-        assertAcked(prepareCreate("my-index")
-                        .addMapping("parent")
-                        .addMapping("child", "_parent", "type=parent")
-        );
-        client().prepareIndex("my-index", "parent", "1").setSource("{}").get();
-        client().prepareIndex("my-index", "child", "2").setSource("{}").setParent("1").get();
-        refresh();
-
-        assertAcked(admin().indices().prepareAliases().addAlias("my-index", "filter1", hasChildQuery("child", matchAllQuery(), ScoreMode.None)));
-        assertAcked(admin().indices().prepareAliases().addAlias("my-index", "filter2", hasParentQuery("parent", matchAllQuery(), false)));
-
-        SearchResponse response = client().prepareSearch("filter1").get();
-        assertHitCount(response, 1);
-        assertThat(response.getHits().getAt(0).id(), equalTo("1"));
-        response = client().prepareSearch("filter2").get();
-        assertHitCount(response, 1);
-        assertThat(response.getHits().getAt(0).id(), equalTo("2"));
-    }
-
     public void testAliasesWithBlocks() {
         createIndex("test");
         ensureGreen();
diff --git a/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java b/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java
deleted file mode 100644
index ab305c65cbb88..0000000000000
--- a/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java
+++ /dev/null
@@ -1,598 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.bootstrap;
-
-import org.apache.logging.log4j.Logger;
-import org.apache.lucene.util.Constants;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.BoundTransportAddress;
-import org.elasticsearch.common.transport.TransportAddress;
-import org.elasticsearch.test.ESTestCase;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.function.Consumer;
-
-import static org.hamcrest.CoreMatchers.allOf;
-import static org.hamcrest.CoreMatchers.containsString;
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.CoreMatchers.instanceOf;
-import static org.hamcrest.Matchers.hasToString;
-import static org.hamcrest.Matchers.not;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.reset;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.verifyNoMoreInteractions;
-import static org.mockito.Mockito.when;
-
-public class BootstrapCheckTests extends ESTestCase {
-
-    public void testNonProductionMode() {
-        // nothing should happen since we are in non-production mode
-        final List<TransportAddress> transportAddresses = new ArrayList<>();
-        for (int i = 0; i < randomIntBetween(1, 8); i++) {
-            TransportAddress localTransportAddress = mock(TransportAddress.class);
-            when(localTransportAddress.isLoopbackOrLinkLocalAddress()).thenReturn(true);
-            transportAddresses.add(localTransportAddress);
-        }
-
-        TransportAddress publishAddress = mock(TransportAddress.class);
-        when(publishAddress.isLoopbackOrLinkLocalAddress()).thenReturn(true);
-        BoundTransportAddress boundTransportAddress = mock(BoundTransportAddress.class);
-        when(boundTransportAddress.boundAddresses()).thenReturn(transportAddresses.toArray(new TransportAddress[0]));
-        when(boundTransportAddress.publishAddress()).thenReturn(publishAddress);
-        BootstrapCheck.check(Settings.EMPTY, boundTransportAddress);
-    }
-
-    public void testNoLogMessageInNonProductionMode() {
-        final Logger logger = mock(Logger.class);
-        BootstrapCheck.check(false, randomBoolean(), Collections.emptyList(), logger);
-        verifyNoMoreInteractions(logger);
-    }
-
-    public void testLogMessageInProductionMode() {
-        final Logger logger = mock(Logger.class);
-        final boolean ignoreSystemChecks = randomBoolean();
-        BootstrapCheck.check(true, ignoreSystemChecks, Collections.emptyList(), logger);
-        verify(logger).info("bound or publishing to a non-loopback or non-link-local address, enforcing bootstrap checks");
-        if (ignoreSystemChecks) {
-            verify(logger).warn("enforcing bootstrap checks but ignoring system bootstrap checks, consider not ignoring system checks");
-        }
-        verifyNoMoreInteractions(logger);
-    }
-
-    public void testEnforceLimitsWhenBoundToNonLocalAddress() {
-        final List<TransportAddress> transportAddresses = new ArrayList<>();
-        final TransportAddress nonLocalTransportAddress = mock(TransportAddress.class);
-        when(nonLocalTransportAddress.isLoopbackOrLinkLocalAddress()).thenReturn(false);
-        transportAddresses.add(nonLocalTransportAddress);
-
-        for (int i = 0; i < randomIntBetween(0, 7); i++) {
-            final TransportAddress randomTransportAddress = mock(TransportAddress.class);
-            when(randomTransportAddress.isLoopbackOrLinkLocalAddress()).thenReturn(randomBoolean());
-            transportAddresses.add(randomTransportAddress);
-        }
-
-        final TransportAddress publishAddress = mock(TransportAddress.class);
-        when(publishAddress.isLoopbackOrLinkLocalAddress()).thenReturn(randomBoolean());
-
-        final BoundTransportAddress boundTransportAddress = mock(BoundTransportAddress.class);
-        Collections.shuffle(transportAddresses, random());
-        when(boundTransportAddress.boundAddresses()).thenReturn(transportAddresses.toArray(new TransportAddress[0]));
-        when(boundTransportAddress.publishAddress()).thenReturn(publishAddress);
-
-        assertTrue(BootstrapCheck.enforceLimits(boundTransportAddress));
-    }
-
-    public void testEnforceLimitsWhenPublishingToNonLocalAddress() {
-        final List<TransportAddress> transportAddresses = new ArrayList<>();
-
-        for (int i = 0; i < randomIntBetween(1, 8); i++) {
-            final TransportAddress randomTransportAddress = mock(TransportAddress.class);
-            when(randomTransportAddress.isLoopbackOrLinkLocalAddress()).thenReturn(false);
-            transportAddresses.add(randomTransportAddress);
-        }
-
-        final TransportAddress publishAddress = mock(TransportAddress.class);
-        when(publishAddress.isLoopbackOrLinkLocalAddress()).thenReturn(true);
-
-        final BoundTransportAddress boundTransportAddress = mock(BoundTransportAddress.class);
-        when(boundTransportAddress.boundAddresses()).thenReturn(transportAddresses.toArray(new TransportAddress[0]));
-        when(boundTransportAddress.publishAddress()).thenReturn(publishAddress);
-
-        assertTrue(BootstrapCheck.enforceLimits(boundTransportAddress));
-    }
-
-    public void testExceptionAggregation() {
-        final List<BootstrapCheck.Check> checks = Arrays.asList(
-                new BootstrapCheck.Check() {
-                    @Override
-                    public boolean check() {
-                        return true;
-                    }
-
-                    @Override
-                    public String errorMessage() {
-                        return "first";
-                    }
-
-                    @Override
-                    public boolean isSystemCheck() {
-                        return false;
-                    }
-                },
-                new BootstrapCheck.Check() {
-                    @Override
-                    public boolean check() {
-                        return true;
-                    }
-
-                    @Override
-                    public String errorMessage() {
-                        return "second";
-                    }
-
-                    @Override
-                    public boolean isSystemCheck() {
-                        return false;
-                    }
-                }
-        );
-
-        final RuntimeException e =
-                expectThrows(RuntimeException.class, () -> BootstrapCheck.check(true, false, checks, "testExceptionAggregation"));
-        assertThat(e, hasToString(allOf(containsString("bootstrap checks failed"), containsString("first"), containsString("second"))));
-        final Throwable[] suppressed = e.getSuppressed();
-        assertThat(suppressed.length, equalTo(2));
-        assertThat(suppressed[0], instanceOf(IllegalStateException.class));
-        assertThat(suppressed[0], hasToString(containsString("first")));
-        assertThat(suppressed[1], instanceOf(IllegalStateException.class));
-        assertThat(suppressed[1], hasToString(containsString("second")));
-    }
-
-    public void testHeapSizeCheck() {
-        final int initial = randomIntBetween(0, Integer.MAX_VALUE - 1);
-        final int max = randomIntBetween(initial + 1, Integer.MAX_VALUE);
-        final AtomicLong initialHeapSize = new AtomicLong(initial);
-        final AtomicLong maxHeapSize = new AtomicLong(max);
-
-        final BootstrapCheck.HeapSizeCheck check = new BootstrapCheck.HeapSizeCheck() {
-            @Override
-            long getInitialHeapSize() {
-                return initialHeapSize.get();
-            }
-
-            @Override
-            long getMaxHeapSize() {
-                return maxHeapSize.get();
-            }
-        };
-
-        final RuntimeException e =
-                expectThrows(
-                        RuntimeException.class,
-                        () -> BootstrapCheck.check(true, false, Collections.singletonList(check), "testHeapSizeCheck"));
-        assertThat(
-                e.getMessage(),
-                containsString("initial heap size [" + initialHeapSize.get() + "] " +
-                        "not equal to maximum heap size [" + maxHeapSize.get() + "]"));
-
-        initialHeapSize.set(maxHeapSize.get());
-
-        BootstrapCheck.check(true, false, Collections.singletonList(check), "testHeapSizeCheck");
-
-        // nothing should happen if the initial heap size or the max
-        // heap size is not available
-        if (randomBoolean()) {
-            initialHeapSize.set(0);
-        } else {
-            maxHeapSize.set(0);
-        }
-        BootstrapCheck.check(true, false, Collections.singletonList(check), "testHeapSizeCheck");
-    }
-
-    public void testFileDescriptorLimits() {
-        final boolean osX = randomBoolean(); // simulates OS X versus non-OS X
-        final int limit = osX ? 10240 : 1 << 16;
-        final AtomicLong maxFileDescriptorCount = new AtomicLong(randomIntBetween(1, limit - 1));
-        final BootstrapCheck.FileDescriptorCheck check;
-        if (osX) {
-            check = new BootstrapCheck.OsXFileDescriptorCheck() {
-                @Override
-                long getMaxFileDescriptorCount() {
-                    return maxFileDescriptorCount.get();
-                }
-            };
-        } else {
-            check = new BootstrapCheck.FileDescriptorCheck() {
-                @Override
-                long getMaxFileDescriptorCount() {
-                    return maxFileDescriptorCount.get();
-                }
-            };
-        }
-
-        final RuntimeException e =
-                expectThrows(RuntimeException.class,
-                        () -> BootstrapCheck.check(true, false, Collections.singletonList(check), "testFileDescriptorLimits"));
-        assertThat(e.getMessage(), containsString("max file descriptors"));
-
-        maxFileDescriptorCount.set(randomIntBetween(limit + 1, Integer.MAX_VALUE));
-
-        BootstrapCheck.check(true, false, Collections.singletonList(check), "testFileDescriptorLimits");
-
-        // nothing should happen if current file descriptor count is
-        // not available
-        maxFileDescriptorCount.set(-1);
-        BootstrapCheck.check(true, false, Collections.singletonList(check), "testFileDescriptorLimits");
-    }
-
-    public void testFileDescriptorLimitsThrowsOnInvalidLimit() {
-        final IllegalArgumentException e =
-            expectThrows(
-                IllegalArgumentException.class,
-                () -> new BootstrapCheck.FileDescriptorCheck(-randomIntBetween(0, Integer.MAX_VALUE)));
-        assertThat(e.getMessage(), containsString("limit must be positive but was"));
-    }
-
-    public void testMlockallCheck() {
-        class MlockallCheckTestCase {
-
-            private final boolean mlockallSet;
-            private final boolean isMemoryLocked;
-            private final boolean shouldFail;
-
-            public MlockallCheckTestCase(final boolean mlockallSet, final boolean isMemoryLocked, final boolean shouldFail) {
-                this.mlockallSet = mlockallSet;
-                this.isMemoryLocked = isMemoryLocked;
-                this.shouldFail = shouldFail;
-            }
-
-        }
-
-        final List<MlockallCheckTestCase> testCases = new ArrayList<>();
-        testCases.add(new MlockallCheckTestCase(true, true, false));
-        testCases.add(new MlockallCheckTestCase(true, false, true));
-        testCases.add(new MlockallCheckTestCase(false, true, false));
-        testCases.add(new MlockallCheckTestCase(false, false, false));
-
-        for (final MlockallCheckTestCase testCase : testCases) {
-            final BootstrapCheck.MlockallCheck check = new BootstrapCheck.MlockallCheck(testCase.mlockallSet) {
-                @Override
-                boolean isMemoryLocked() {
-                    return testCase.isMemoryLocked;
-                }
-            };
-
-            if (testCase.shouldFail) {
-                final RuntimeException e = expectThrows(
-                        RuntimeException.class,
-                        () -> BootstrapCheck.check(
-                                true,
-                                false,
-                                Collections.singletonList(check),
-                                "testFileDescriptorLimitsThrowsOnInvalidLimit"));
-                assertThat(
-                        e.getMessage(),
-                        containsString("memory locking requested for elasticsearch process but memory is not locked"));
-            } else {
-                // nothing should happen
-                BootstrapCheck.check(true, false, Collections.singletonList(check), "testFileDescriptorLimitsThrowsOnInvalidLimit");
-            }
-        }
-    }
-
-    public void testMaxNumberOfThreadsCheck() {
-        final int limit = 1 << 11;
-        final AtomicLong maxNumberOfThreads = new AtomicLong(randomIntBetween(1, limit - 1));
-        final BootstrapCheck.MaxNumberOfThreadsCheck check = new BootstrapCheck.MaxNumberOfThreadsCheck() {
-            @Override
-            long getMaxNumberOfThreads() {
-                return maxNumberOfThreads.get();
-            }
-        };
-
-        final RuntimeException e = expectThrows(
-                RuntimeException.class,
-                () -> BootstrapCheck.check(true, false, Collections.singletonList(check), "testMaxNumberOfThreadsCheck"));
-        assertThat(e.getMessage(), containsString("max number of threads"));
-
-        maxNumberOfThreads.set(randomIntBetween(limit + 1, Integer.MAX_VALUE));
-
-        BootstrapCheck.check(true, false, Collections.singletonList(check), "testMaxNumberOfThreadsCheck");
-
-        // nothing should happen if current max number of threads is
-        // not available
-        maxNumberOfThreads.set(-1);
-        BootstrapCheck.check(true, false, Collections.singletonList(check), "testMaxNumberOfThreadsCheck");
-    }
-
-    public void testMaxSizeVirtualMemory() {
-        final long rlimInfinity = Constants.MAC_OS_X ? 9223372036854775807L : -1L;
-        final AtomicLong maxSizeVirtualMemory = new AtomicLong(randomIntBetween(0, Integer.MAX_VALUE));
-        final BootstrapCheck.MaxSizeVirtualMemoryCheck check = new BootstrapCheck.MaxSizeVirtualMemoryCheck() {
-            @Override
-            long getMaxSizeVirtualMemory() {
-                return maxSizeVirtualMemory.get();
-            }
-
-            @Override
-            long getRlimInfinity() {
-                return rlimInfinity;
-            }
-        };
-
-
-        final RuntimeException e = expectThrows(
-                RuntimeException.class,
-                () -> BootstrapCheck.check(true, false, Collections.singletonList(check), "testMaxSizeVirtualMemory"));
-        assertThat(e.getMessage(), containsString("max size virtual memory"));
-
-        maxSizeVirtualMemory.set(rlimInfinity);
-
-        BootstrapCheck.check(true, false, Collections.singletonList(check), "testMaxSizeVirtualMemory");
-
-        // nothing should happen if max size virtual memory is not
-        // available
-        maxSizeVirtualMemory.set(Long.MIN_VALUE);
-        BootstrapCheck.check(true, false, Collections.singletonList(check), "testMaxSizeVirtualMemory");
-    }
-
-    public void testMaxMapCountCheck() {
-        final int limit = 1 << 18;
-        final AtomicLong maxMapCount = new AtomicLong(randomIntBetween(1, limit - 1));
-        final BootstrapCheck.MaxMapCountCheck check = new BootstrapCheck.MaxMapCountCheck() {
-            @Override
-            long getMaxMapCount() {
-                return maxMapCount.get();
-            }
-        };
-
-        RuntimeException e = expectThrows(
-                RuntimeException.class,
-                () -> BootstrapCheck.check(true, false, Collections.singletonList(check), "testMaxMapCountCheck"));
-        assertThat(e.getMessage(), containsString("max virtual memory areas vm.max_map_count"));
-
-        maxMapCount.set(randomIntBetween(limit + 1, Integer.MAX_VALUE));
-
-        BootstrapCheck.check(true, false, Collections.singletonList(check), "testMaxMapCountCheck");
-
-        // nothing should happen if current vm.max_map_count is not
-        // available
-        maxMapCount.set(-1);
-        BootstrapCheck.check(true, false, Collections.singletonList(check), "testMaxMapCountCheck");
-    }
-
-    public void testClientJvmCheck() {
-        final AtomicReference<String> vmName = new AtomicReference<>("Java HotSpot(TM) 32-Bit Client VM");
-        final BootstrapCheck.Check check = new BootstrapCheck.ClientJvmCheck() {
-            @Override
-            String getVmName() {
-                return vmName.get();
-            }
-        };
-
-        final RuntimeException e = expectThrows(
-                RuntimeException.class,
-                () -> BootstrapCheck.check(true, false, Collections.singletonList(check), "testClientJvmCheck"));
-        assertThat(
-                e.getMessage(),
-                containsString("JVM is using the client VM [Java HotSpot(TM) 32-Bit Client VM] " +
-                        "but should be using a server VM for the best performance"));
-
-        vmName.set("Java HotSpot(TM) 32-Bit Server VM");
-        BootstrapCheck.check(true, false, Collections.singletonList(check), "testClientJvmCheck");
-    }
-
-    public void testMightForkCheck() {
-        final AtomicBoolean isSeccompInstalled = new AtomicBoolean();
-        final AtomicBoolean mightFork = new AtomicBoolean();
-        final BootstrapCheck.MightForkCheck check = new BootstrapCheck.MightForkCheck() {
-            @Override
-            boolean isSeccompInstalled() {
-                return isSeccompInstalled.get();
-            }
-
-            @Override
-            boolean mightFork() {
-                return mightFork.get();
-            }
-
-            @Override
-            public String errorMessage() {
-                return "error";
-            }
-        };
-
-        runMightForkTest(
-            check,
-            isSeccompInstalled,
-            () -> mightFork.set(false),
-            () -> mightFork.set(true),
-            e -> assertThat(e.getMessage(), containsString("error")));
-    }
-
-    public void testOnErrorCheck() {
-        final AtomicBoolean isSeccompInstalled = new AtomicBoolean();
-        final AtomicReference<String> onError = new AtomicReference<>();
-        final BootstrapCheck.MightForkCheck check = new BootstrapCheck.OnErrorCheck() {
-            @Override
-            boolean isSeccompInstalled() {
-                return isSeccompInstalled.get();
-            }
-
-            @Override
-            String onError() {
-                return onError.get();
-            }
-        };
-
-        final String command = randomAsciiOfLength(16);
-        runMightForkTest(
-            check,
-            isSeccompInstalled,
-            () -> onError.set(randomBoolean() ? "" : null),
-            () -> onError.set(command),
-            e -> assertThat(
-                e.getMessage(),
-                containsString(
-                    "OnError [" + command + "] requires forking but is prevented by system call filters ([bootstrap.seccomp=true]);"
-                        + " upgrade to at least Java 8u92 and use ExitOnOutOfMemoryError")));
-    }
-
-    public void testOnOutOfMemoryErrorCheck() {
-        final AtomicBoolean isSeccompInstalled = new AtomicBoolean();
-        final AtomicReference<String> onOutOfMemoryError = new AtomicReference<>();
-        final BootstrapCheck.MightForkCheck check = new BootstrapCheck.OnOutOfMemoryErrorCheck() {
-            @Override
-            boolean isSeccompInstalled() {
-                return isSeccompInstalled.get();
-            }
-
-            @Override
-            String onOutOfMemoryError() {
-                return onOutOfMemoryError.get();
-            }
-        };
-
-        final String command = randomAsciiOfLength(16);
-        runMightForkTest(
-            check,
-            isSeccompInstalled,
-            () -> onOutOfMemoryError.set(randomBoolean() ? "" : null),
-            () -> onOutOfMemoryError.set(command),
-            e -> assertThat(
-                e.getMessage(),
-                containsString(
-                    "OnOutOfMemoryError [" + command + "]"
-                        + " requires forking but is prevented by system call filters ([bootstrap.seccomp=true]);"
-                        + " upgrade to at least Java 8u92 and use ExitOnOutOfMemoryError")));
-    }
-
-    private void runMightForkTest(
-        final BootstrapCheck.MightForkCheck check,
-        final AtomicBoolean isSeccompInstalled,
-        final Runnable disableMightFork,
-        final Runnable enableMightFork,
-        final Consumer<RuntimeException> consumer) {
-
-        final String methodName = Thread.currentThread().getStackTrace()[2].getMethodName();
-
-        // if seccomp is disabled, nothing should happen
-        isSeccompInstalled.set(false);
-        if (randomBoolean()) {
-            disableMightFork.run();
-        } else {
-            enableMightFork.run();
-        }
-        BootstrapCheck.check(true, randomBoolean(), Collections.singletonList(check), methodName);
-
-        // if seccomp is enabled, but we will not fork, nothing should
-        // happen
-        isSeccompInstalled.set(true);
-        disableMightFork.run();
-        BootstrapCheck.check(true, randomBoolean(), Collections.singletonList(check), methodName);
-
-        // if seccomp is enabled, and we might fork, the check should
-        // be enforced, regardless of bootstrap checks being enabled or
-        // not
-        isSeccompInstalled.set(true);
-        enableMightFork.run();
-
-        final RuntimeException e = expectThrows(
-            RuntimeException.class,
-            () -> BootstrapCheck.check(randomBoolean(), randomBoolean(), Collections.singletonList(check), methodName));
-        consumer.accept(e);
-    }
-
-    public void testIgnoringSystemChecks() {
-        final BootstrapCheck.Check check = new BootstrapCheck.Check() {
-            @Override
-            public boolean check() {
-                return true;
-            }
-
-            @Override
-            public String errorMessage() {
-                return "error";
-            }
-
-            @Override
-            public boolean isSystemCheck() {
-                return true;
-            }
-        };
-
-        final RuntimeException notIgnored = expectThrows(
-                RuntimeException.class,
-                () -> BootstrapCheck.check(true, false, Collections.singletonList(check), "testIgnoringSystemChecks"));
-        assertThat(notIgnored, hasToString(containsString("error")));
-
-        final Logger logger = mock(Logger.class);
-
-        // nothing should happen if we ignore system checks
-        BootstrapCheck.check(true, true, Collections.singletonList(check), logger);
-        verify(logger).info("bound or publishing to a non-loopback or non-link-local address, enforcing bootstrap checks");
-        verify(logger).warn("enforcing bootstrap checks but ignoring system bootstrap checks, consider not ignoring system checks");
-        verify(logger).warn("error");
-        verifyNoMoreInteractions(logger);
-        reset(logger);
-
-        // nothing should happen if we ignore all checks
-        BootstrapCheck.check(false, randomBoolean(), Collections.singletonList(check), logger);
-        verify(logger).warn("error");
-        verifyNoMoreInteractions(logger);
-    }
-
-    public void testAlwaysEnforcedChecks() {
-        final BootstrapCheck.Check check = new BootstrapCheck.Check() {
-            @Override
-            public boolean check() {
-                return true;
-            }
-
-            @Override
-            public String errorMessage() {
-                return "error";
-            }
-
-            @Override
-            public boolean isSystemCheck() {
-                return randomBoolean();
-            }
-
-            @Override
-            public boolean alwaysEnforce() {
-                return true;
-            }
-        };
-
-        final RuntimeException alwaysEnforced = expectThrows(
-            RuntimeException.class,
-            () -> BootstrapCheck.check(randomBoolean(), randomBoolean(), Collections.singletonList(check), "testAlwaysEnforcedChecks"));
-        assertThat(alwaysEnforced, hasToString(containsString("error")));
-    }
-
-}
diff --git a/core/src/test/java/org/elasticsearch/bootstrap/BootstrapChecksTests.java b/core/src/test/java/org/elasticsearch/bootstrap/BootstrapChecksTests.java
new file mode 100644
index 0000000000000..6f00211adf799
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/bootstrap/BootstrapChecksTests.java
@@ -0,0 +1,732 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.bootstrap;
+
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.util.Constants;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.BoundTransportAddress;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.monitor.jvm.JvmInfo;
+import org.elasticsearch.node.NodeValidationException;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Locale;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
+
+import static org.hamcrest.CoreMatchers.allOf;
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.hamcrest.Matchers.hasToString;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
+
+public class BootstrapChecksTests extends ESTestCase {
+
+    public void testNonProductionMode() throws NodeValidationException {
+        // nothing should happen since we are in non-production mode
+        final List<TransportAddress> transportAddresses = new ArrayList<>();
+        for (int i = 0; i < randomIntBetween(1, 8); i++) {
+            TransportAddress localTransportAddress = mock(TransportAddress.class);
+            when(localTransportAddress.isLoopbackOrLinkLocalAddress()).thenReturn(true);
+            transportAddresses.add(localTransportAddress);
+        }
+
+        TransportAddress publishAddress = mock(TransportAddress.class);
+        when(publishAddress.isLoopbackOrLinkLocalAddress()).thenReturn(true);
+        BoundTransportAddress boundTransportAddress = mock(BoundTransportAddress.class);
+        when(boundTransportAddress.boundAddresses()).thenReturn(transportAddresses.toArray(new TransportAddress[0]));
+        when(boundTransportAddress.publishAddress()).thenReturn(publishAddress);
+        BootstrapChecks.check(Settings.EMPTY, boundTransportAddress, Collections.emptyList());
+    }
+
+    public void testNoLogMessageInNonProductionMode() throws NodeValidationException {
+        final Logger logger = mock(Logger.class);
+        BootstrapChecks.check(false, Collections.emptyList(), logger);
+        verifyNoMoreInteractions(logger);
+    }
+
+    public void testLogMessageInProductionMode() throws NodeValidationException {
+        final Logger logger = mock(Logger.class);
+        BootstrapChecks.check(true, Collections.emptyList(), logger);
+        verify(logger).info("bound or publishing to a non-loopback or non-link-local address, enforcing bootstrap checks");
+        verifyNoMoreInteractions(logger);
+    }
+
+    public void testEnforceLimitsWhenBoundToNonLocalAddress() {
+        final List<TransportAddress> transportAddresses = new ArrayList<>();
+        final TransportAddress nonLocalTransportAddress = mock(TransportAddress.class);
+        when(nonLocalTransportAddress.isLoopbackOrLinkLocalAddress()).thenReturn(false);
+        transportAddresses.add(nonLocalTransportAddress);
+
+        for (int i = 0; i < randomIntBetween(0, 7); i++) {
+            final TransportAddress randomTransportAddress = mock(TransportAddress.class);
+            when(randomTransportAddress.isLoopbackOrLinkLocalAddress()).thenReturn(randomBoolean());
+            transportAddresses.add(randomTransportAddress);
+        }
+
+        final TransportAddress publishAddress = mock(TransportAddress.class);
+        when(publishAddress.isLoopbackOrLinkLocalAddress()).thenReturn(randomBoolean());
+
+        final BoundTransportAddress boundTransportAddress = mock(BoundTransportAddress.class);
+        Collections.shuffle(transportAddresses, random());
+        when(boundTransportAddress.boundAddresses()).thenReturn(transportAddresses.toArray(new TransportAddress[0]));
+        when(boundTransportAddress.publishAddress()).thenReturn(publishAddress);
+
+        final String discoveryType = randomFrom("zen", "single-node");
+
+        assertEquals(BootstrapChecks.enforceLimits(boundTransportAddress, discoveryType), !"single-node".equals(discoveryType));
+    }
+
+    public void testEnforceLimitsWhenPublishingToNonLocalAddress() {
+        final List<TransportAddress> transportAddresses = new ArrayList<>();
+
+        for (int i = 0; i < randomIntBetween(1, 8); i++) {
+            final TransportAddress randomTransportAddress = mock(TransportAddress.class);
+            when(randomTransportAddress.isLoopbackOrLinkLocalAddress()).thenReturn(false);
+            transportAddresses.add(randomTransportAddress);
+        }
+
+        final TransportAddress publishAddress = mock(TransportAddress.class);
+        when(publishAddress.isLoopbackOrLinkLocalAddress()).thenReturn(true);
+
+        final BoundTransportAddress boundTransportAddress = mock(BoundTransportAddress.class);
+        when(boundTransportAddress.boundAddresses()).thenReturn(transportAddresses.toArray(new TransportAddress[0]));
+        when(boundTransportAddress.publishAddress()).thenReturn(publishAddress);
+
+        final String discoveryType = randomFrom("zen", "single-node");
+
+        assertEquals(BootstrapChecks.enforceLimits(boundTransportAddress, discoveryType), !"single-node".equals(discoveryType));
+    }
+
+    public void testExceptionAggregation() {
+        final List<BootstrapCheck> checks = Arrays.asList(
+                new BootstrapCheck() {
+                    @Override
+                    public boolean check() {
+                        return true;
+                    }
+
+                    @Override
+                    public String errorMessage() {
+                        return "first";
+                    }
+                },
+                new BootstrapCheck() {
+                    @Override
+                    public boolean check() {
+                        return true;
+                    }
+
+                    @Override
+                    public String errorMessage() {
+                        return "second";
+                    }
+                }
+        );
+
+        final NodeValidationException e =
+                expectThrows(NodeValidationException.class, () -> BootstrapChecks.check(true, checks, "testExceptionAggregation"));
+        assertThat(e, hasToString(allOf(containsString("bootstrap checks failed"), containsString("first"), containsString("second"))));
+        final Throwable[] suppressed = e.getSuppressed();
+        assertThat(suppressed.length, equalTo(2));
+        assertThat(suppressed[0], instanceOf(IllegalStateException.class));
+        assertThat(suppressed[0], hasToString(containsString("first")));
+        assertThat(suppressed[1], instanceOf(IllegalStateException.class));
+        assertThat(suppressed[1], hasToString(containsString("second")));
+    }
+
+    public void testHeapSizeCheck() throws NodeValidationException {
+        final int initial = randomIntBetween(0, Integer.MAX_VALUE - 1);
+        final int max = randomIntBetween(initial + 1, Integer.MAX_VALUE);
+        final AtomicLong initialHeapSize = new AtomicLong(initial);
+        final AtomicLong maxHeapSize = new AtomicLong(max);
+
+        final BootstrapChecks.HeapSizeCheck check = new BootstrapChecks.HeapSizeCheck() {
+            @Override
+            long getInitialHeapSize() {
+                return initialHeapSize.get();
+            }
+
+            @Override
+            long getMaxHeapSize() {
+                return maxHeapSize.get();
+            }
+        };
+
+        final NodeValidationException e =
+                expectThrows(
+                        NodeValidationException.class,
+                        () -> BootstrapChecks.check(true, Collections.singletonList(check), "testHeapSizeCheck"));
+        assertThat(
+                e.getMessage(),
+                containsString("initial heap size [" + initialHeapSize.get() + "] " +
+                        "not equal to maximum heap size [" + maxHeapSize.get() + "]"));
+
+        initialHeapSize.set(maxHeapSize.get());
+
+        BootstrapChecks.check(true, Collections.singletonList(check), "testHeapSizeCheck");
+
+        // nothing should happen if the initial heap size or the max
+        // heap size is not available
+        if (randomBoolean()) {
+            initialHeapSize.set(0);
+        } else {
+            maxHeapSize.set(0);
+        }
+        BootstrapChecks.check(true, Collections.singletonList(check), "testHeapSizeCheck");
+    }
+
+    public void testFileDescriptorLimits() throws NodeValidationException {
+        final boolean osX = randomBoolean(); // simulates OS X versus non-OS X
+        final int limit = osX ? 10240 : 1 << 16;
+        final AtomicLong maxFileDescriptorCount = new AtomicLong(randomIntBetween(1, limit - 1));
+        final BootstrapChecks.FileDescriptorCheck check;
+        if (osX) {
+            check = new BootstrapChecks.OsXFileDescriptorCheck() {
+                @Override
+                long getMaxFileDescriptorCount() {
+                    return maxFileDescriptorCount.get();
+                }
+            };
+        } else {
+            check = new BootstrapChecks.FileDescriptorCheck() {
+                @Override
+                long getMaxFileDescriptorCount() {
+                    return maxFileDescriptorCount.get();
+                }
+            };
+        }
+
+        final NodeValidationException e =
+                expectThrows(NodeValidationException.class,
+                        () -> BootstrapChecks.check(true, Collections.singletonList(check), "testFileDescriptorLimits"));
+        assertThat(e.getMessage(), containsString("max file descriptors"));
+
+        maxFileDescriptorCount.set(randomIntBetween(limit + 1, Integer.MAX_VALUE));
+
+        BootstrapChecks.check(true, Collections.singletonList(check), "testFileDescriptorLimits");
+
+        // nothing should happen if current file descriptor count is
+        // not available
+        maxFileDescriptorCount.set(-1);
+        BootstrapChecks.check(true, Collections.singletonList(check), "testFileDescriptorLimits");
+    }
+
+    public void testFileDescriptorLimitsThrowsOnInvalidLimit() {
+        final IllegalArgumentException e =
+            expectThrows(
+                IllegalArgumentException.class,
+                () -> new BootstrapChecks.FileDescriptorCheck(-randomIntBetween(0, Integer.MAX_VALUE)));
+        assertThat(e.getMessage(), containsString("limit must be positive but was"));
+    }
+
+    public void testMlockallCheck() throws NodeValidationException {
+        class MlockallCheckTestCase {
+
+            private final boolean mlockallSet;
+            private final boolean isMemoryLocked;
+            private final boolean shouldFail;
+
+            MlockallCheckTestCase(final boolean mlockallSet, final boolean isMemoryLocked, final boolean shouldFail) {
+                this.mlockallSet = mlockallSet;
+                this.isMemoryLocked = isMemoryLocked;
+                this.shouldFail = shouldFail;
+            }
+
+        }
+
+        final List<MlockallCheckTestCase> testCases = new ArrayList<>();
+        testCases.add(new MlockallCheckTestCase(true, true, false));
+        testCases.add(new MlockallCheckTestCase(true, false, true));
+        testCases.add(new MlockallCheckTestCase(false, true, false));
+        testCases.add(new MlockallCheckTestCase(false, false, false));
+
+        for (final MlockallCheckTestCase testCase : testCases) {
+            final BootstrapChecks.MlockallCheck check = new BootstrapChecks.MlockallCheck(testCase.mlockallSet) {
+                @Override
+                boolean isMemoryLocked() {
+                    return testCase.isMemoryLocked;
+                }
+            };
+
+            if (testCase.shouldFail) {
+                final NodeValidationException e = expectThrows(
+                        NodeValidationException.class,
+                        () -> BootstrapChecks.check(
+                                true,
+                                Collections.singletonList(check),
+                                "testFileDescriptorLimitsThrowsOnInvalidLimit"));
+                assertThat(
+                        e.getMessage(),
+                        containsString("memory locking requested for elasticsearch process but memory is not locked"));
+            } else {
+                // nothing should happen
+                BootstrapChecks.check(true, Collections.singletonList(check), "testFileDescriptorLimitsThrowsOnInvalidLimit");
+            }
+        }
+    }
+
+    public void testMaxNumberOfThreadsCheck() throws NodeValidationException {
+        final int limit = 1 << 11;
+        final AtomicLong maxNumberOfThreads = new AtomicLong(randomIntBetween(1, limit - 1));
+        final BootstrapChecks.MaxNumberOfThreadsCheck check = new BootstrapChecks.MaxNumberOfThreadsCheck() {
+            @Override
+            long getMaxNumberOfThreads() {
+                return maxNumberOfThreads.get();
+            }
+        };
+
+        final NodeValidationException e = expectThrows(
+                NodeValidationException.class,
+                () -> BootstrapChecks.check(true, Collections.singletonList(check), "testMaxNumberOfThreadsCheck"));
+        assertThat(e.getMessage(), containsString("max number of threads"));
+
+        maxNumberOfThreads.set(randomIntBetween(limit + 1, Integer.MAX_VALUE));
+
+        BootstrapChecks.check(true, Collections.singletonList(check), "testMaxNumberOfThreadsCheck");
+
+        // nothing should happen if current max number of threads is
+        // not available
+        maxNumberOfThreads.set(-1);
+        BootstrapChecks.check(true, Collections.singletonList(check), "testMaxNumberOfThreadsCheck");
+    }
+
+    public void testMaxSizeVirtualMemory() throws NodeValidationException {
+        final long rlimInfinity = Constants.MAC_OS_X ? 9223372036854775807L : -1L;
+        final AtomicLong maxSizeVirtualMemory = new AtomicLong(randomIntBetween(0, Integer.MAX_VALUE));
+        final BootstrapChecks.MaxSizeVirtualMemoryCheck check = new BootstrapChecks.MaxSizeVirtualMemoryCheck() {
+            @Override
+            long getMaxSizeVirtualMemory() {
+                return maxSizeVirtualMemory.get();
+            }
+
+            @Override
+            long getRlimInfinity() {
+                return rlimInfinity;
+            }
+        };
+
+        final NodeValidationException e = expectThrows(
+                NodeValidationException.class,
+                () -> BootstrapChecks.check(true, Collections.singletonList(check), "testMaxSizeVirtualMemory"));
+        assertThat(e.getMessage(), containsString("max size virtual memory"));
+
+        maxSizeVirtualMemory.set(rlimInfinity);
+
+        BootstrapChecks.check(true, Collections.singletonList(check), "testMaxSizeVirtualMemory");
+
+        // nothing should happen if max size virtual memory is not available
+        maxSizeVirtualMemory.set(Long.MIN_VALUE);
+        BootstrapChecks.check(true, Collections.singletonList(check), "testMaxSizeVirtualMemory");
+    }
+
+    public void testMaxFileSizeCheck() throws NodeValidationException {
+        final long rlimInfinity = Constants.MAC_OS_X ? 9223372036854775807L : -1L;
+        final AtomicLong maxFileSize = new AtomicLong(randomIntBetween(0, Integer.MAX_VALUE));
+        final BootstrapChecks.MaxFileSizeCheck check = new BootstrapChecks.MaxFileSizeCheck() {
+            @Override
+            long getMaxFileSize() {
+                return maxFileSize.get();
+            }
+
+            @Override
+            long getRlimInfinity() {
+                return rlimInfinity;
+            }
+        };
+
+        final NodeValidationException e = expectThrows(
+                NodeValidationException.class,
+                () -> BootstrapChecks.check(true, Collections.singletonList(check), "testMaxFileSize"));
+        assertThat(e.getMessage(), containsString("max file size"));
+
+        maxFileSize.set(rlimInfinity);
+
+        BootstrapChecks.check(true, Collections.singletonList(check), "testMaxFileSize");
+
+        // nothing should happen if max file size is not available
+        maxFileSize.set(Long.MIN_VALUE);
+        BootstrapChecks.check(true, Collections.singletonList(check), "testMaxFileSize");
+    }
+
+    public void testMaxMapCountCheck() throws NodeValidationException {
+        final int limit = 1 << 18;
+        final AtomicLong maxMapCount = new AtomicLong(randomIntBetween(1, limit - 1));
+        final BootstrapChecks.MaxMapCountCheck check = new BootstrapChecks.MaxMapCountCheck() {
+            @Override
+            long getMaxMapCount() {
+                return maxMapCount.get();
+            }
+        };
+
+        final NodeValidationException e = expectThrows(
+                NodeValidationException.class,
+                () -> BootstrapChecks.check(true, Collections.singletonList(check), "testMaxMapCountCheck"));
+        assertThat(e.getMessage(), containsString("max virtual memory areas vm.max_map_count"));
+
+        maxMapCount.set(randomIntBetween(limit + 1, Integer.MAX_VALUE));
+
+        BootstrapChecks.check(true, Collections.singletonList(check), "testMaxMapCountCheck");
+
+        // nothing should happen if current vm.max_map_count is not
+        // available
+        maxMapCount.set(-1);
+        BootstrapChecks.check(true, Collections.singletonList(check), "testMaxMapCountCheck");
+    }
+
+    public void testClientJvmCheck() throws NodeValidationException {
+        final AtomicReference<String> vmName = new AtomicReference<>("Java HotSpot(TM) 32-Bit Client VM");
+        final BootstrapCheck check = new BootstrapChecks.ClientJvmCheck() {
+            @Override
+            String getVmName() {
+                return vmName.get();
+            }
+        };
+
+        final NodeValidationException e = expectThrows(
+                NodeValidationException.class,
+                () -> BootstrapChecks.check(true, Collections.singletonList(check), "testClientJvmCheck"));
+        assertThat(
+                e.getMessage(),
+                containsString("JVM is using the client VM [Java HotSpot(TM) 32-Bit Client VM] " +
+                        "but should be using a server VM for the best performance"));
+
+        vmName.set("Java HotSpot(TM) 32-Bit Server VM");
+        BootstrapChecks.check(true, Collections.singletonList(check), "testClientJvmCheck");
+    }
+
+    public void testUseSerialGCCheck() throws NodeValidationException {
+        final AtomicReference<String> useSerialGC = new AtomicReference<>("true");
+        final BootstrapCheck check = new BootstrapChecks.UseSerialGCCheck() {
+            @Override
+            String getUseSerialGC() {
+                return useSerialGC.get();
+            }
+        };
+
+        final NodeValidationException e = expectThrows(
+            NodeValidationException.class,
+            () -> BootstrapChecks.check(true, Collections.singletonList(check), "testUseSerialGCCheck"));
+        assertThat(
+            e.getMessage(),
+            containsString("JVM is using the serial collector but should not be for the best performance; " + "" +
+                "either it's the default for the VM [" + JvmInfo.jvmInfo().getVmName() +"] or -XX:+UseSerialGC was explicitly specified"));
+
+        useSerialGC.set("false");
+        BootstrapChecks.check(true, Collections.singletonList(check), "testUseSerialGCCheck");
+    }
+
+    public void testSystemCallFilterCheck() throws NodeValidationException {
+        final AtomicBoolean isSystemCallFilterInstalled = new AtomicBoolean();
+        final BootstrapChecks.SystemCallFilterCheck systemCallFilterEnabledCheck = new BootstrapChecks.SystemCallFilterCheck(true) {
+            @Override
+            boolean isSystemCallFilterInstalled() {
+                return isSystemCallFilterInstalled.get();
+            }
+        };
+
+        final NodeValidationException e = expectThrows(
+            NodeValidationException.class,
+            () -> BootstrapChecks.check(true, Collections.singletonList(systemCallFilterEnabledCheck), "testSystemCallFilterCheck"));
+        assertThat(
+            e.getMessage(),
+            containsString("system call filters failed to install; " +
+                "check the logs and fix your configuration or disable system call filters at your own risk"));
+
+        isSystemCallFilterInstalled.set(true);
+        BootstrapChecks.check(true, Collections.singletonList(systemCallFilterEnabledCheck), "testSystemCallFilterCheck");
+
+        final BootstrapChecks.SystemCallFilterCheck systemCallFilterNotEnabledCheck = new BootstrapChecks.SystemCallFilterCheck(false) {
+            @Override
+            boolean isSystemCallFilterInstalled() {
+                return isSystemCallFilterInstalled.get();
+            }
+        };
+        isSystemCallFilterInstalled.set(false);
+        BootstrapChecks.check(true, Collections.singletonList(systemCallFilterNotEnabledCheck), "testSystemCallFilterCheck");
+        isSystemCallFilterInstalled.set(true);
+        BootstrapChecks.check(true, Collections.singletonList(systemCallFilterNotEnabledCheck), "testSystemCallFilterCheck");
+    }
+
+    public void testMightForkCheck() throws NodeValidationException {
+        final AtomicBoolean isSystemCallFilterInstalled = new AtomicBoolean();
+        final AtomicBoolean mightFork = new AtomicBoolean();
+        final BootstrapChecks.MightForkCheck check = new BootstrapChecks.MightForkCheck() {
+            @Override
+            boolean isSystemCallFilterInstalled() {
+                return isSystemCallFilterInstalled.get();
+            }
+
+            @Override
+            boolean mightFork() {
+                return mightFork.get();
+            }
+
+            @Override
+            public String errorMessage() {
+                return "error";
+            }
+        };
+
+        runMightForkTest(
+            check,
+            isSystemCallFilterInstalled,
+            () -> mightFork.set(false),
+            () -> mightFork.set(true),
+            e -> assertThat(e.getMessage(), containsString("error")));
+    }
+
+    public void testOnErrorCheck() throws NodeValidationException {
+        final AtomicBoolean isSystemCallFilterInstalled = new AtomicBoolean();
+        final AtomicReference<String> onError = new AtomicReference<>();
+        final BootstrapChecks.MightForkCheck check = new BootstrapChecks.OnErrorCheck() {
+            @Override
+            boolean isSystemCallFilterInstalled() {
+                return isSystemCallFilterInstalled.get();
+            }
+
+            @Override
+            String onError() {
+                return onError.get();
+            }
+        };
+
+        final String command = randomAlphaOfLength(16);
+        runMightForkTest(
+            check,
+            isSystemCallFilterInstalled,
+            () -> onError.set(randomBoolean() ? "" : null),
+            () -> onError.set(command),
+            e -> assertThat(
+                e.getMessage(),
+                containsString(
+                    "OnError [" + command + "] requires forking but is prevented by system call filters " +
+                        "([bootstrap.system_call_filter=true]); upgrade to at least Java 8u92 and use ExitOnOutOfMemoryError")));
+    }
+
+    public void testOnOutOfMemoryErrorCheck() throws NodeValidationException {
+        final AtomicBoolean isSystemCallFilterInstalled = new AtomicBoolean();
+        final AtomicReference<String> onOutOfMemoryError = new AtomicReference<>();
+        final BootstrapChecks.MightForkCheck check = new BootstrapChecks.OnOutOfMemoryErrorCheck() {
+            @Override
+            boolean isSystemCallFilterInstalled() {
+                return isSystemCallFilterInstalled.get();
+            }
+
+            @Override
+            String onOutOfMemoryError() {
+                return onOutOfMemoryError.get();
+            }
+        };
+
+        final String command = randomAlphaOfLength(16);
+        runMightForkTest(
+            check,
+            isSystemCallFilterInstalled,
+            () -> onOutOfMemoryError.set(randomBoolean() ? "" : null),
+            () -> onOutOfMemoryError.set(command),
+            e -> assertThat(
+                e.getMessage(),
+                containsString(
+                    "OnOutOfMemoryError [" + command + "]"
+                        + " requires forking but is prevented by system call filters ([bootstrap.system_call_filter=true]);"
+                        + " upgrade to at least Java 8u92 and use ExitOnOutOfMemoryError")));
+    }
+
+    private void runMightForkTest(
+        final BootstrapChecks.MightForkCheck check,
+        final AtomicBoolean isSystemCallFilterInstalled,
+        final Runnable disableMightFork,
+        final Runnable enableMightFork,
+        final Consumer<NodeValidationException> consumer) throws NodeValidationException {
+
+        final String methodName = Thread.currentThread().getStackTrace()[2].getMethodName();
+
+        // if system call filter is disabled, nothing should happen
+        isSystemCallFilterInstalled.set(false);
+        if (randomBoolean()) {
+            disableMightFork.run();
+        } else {
+            enableMightFork.run();
+        }
+        BootstrapChecks.check(true, Collections.singletonList(check), methodName);
+
+        // if system call filter is enabled, but we will not fork, nothing should
+        // happen
+        isSystemCallFilterInstalled.set(true);
+        disableMightFork.run();
+        BootstrapChecks.check(true, Collections.singletonList(check), methodName);
+
+        // if system call filter is enabled, and we might fork, the check should be enforced, regardless of bootstrap checks being enabled
+        // or not
+        isSystemCallFilterInstalled.set(true);
+        enableMightFork.run();
+
+        final NodeValidationException e = expectThrows(
+            NodeValidationException.class,
+            () -> BootstrapChecks.check(randomBoolean(), Collections.singletonList(check), methodName));
+        consumer.accept(e);
+    }
+
+    public void testEarlyAccessCheck() throws NodeValidationException {
+        final AtomicReference<String> javaVersion
+                = new AtomicReference<>(randomFrom("1.8.0_152-ea", "9-ea"));
+        final BootstrapChecks.EarlyAccessCheck eaCheck = new BootstrapChecks.EarlyAccessCheck() {
+
+            @Override
+            String jvmVendor() {
+                return "Oracle Corporation";
+            }
+
+            @Override
+            String javaVersion() {
+                return javaVersion.get();
+            }
+
+        };
+
+        final List<BootstrapCheck> checks = Collections.singletonList(eaCheck);
+        final NodeValidationException e = expectThrows(
+                NodeValidationException.class,
+                () -> {
+                    BootstrapChecks.check(true, checks, "testEarlyAccessCheck");
+                });
+        assertThat(
+                e.getMessage(),
+                containsString(
+                        "Java version ["
+                                + javaVersion.get()
+                                + "] is an early-access build, only use release builds"));
+
+        // if not on an early-access build, nothing should happen
+        javaVersion.set(randomFrom("1.8.0_152", "9"));
+        BootstrapChecks.check(true, checks, "testEarlyAccessCheck");
+
+    }
+
+    public void testG1GCCheck() throws NodeValidationException {
+        final AtomicBoolean isG1GCEnabled = new AtomicBoolean(true);
+        final AtomicBoolean isJava8 = new AtomicBoolean(true);
+        final AtomicReference<String> jvmVersion =
+            new AtomicReference<>(String.format(Locale.ROOT, "25.%d-b%d", randomIntBetween(0, 39), randomIntBetween(1, 128)));
+        final BootstrapChecks.G1GCCheck g1GCCheck = new BootstrapChecks.G1GCCheck() {
+
+            @Override
+            String jvmVendor() {
+                return "Oracle Corporation";
+            }
+
+            @Override
+            boolean isG1GCEnabled() {
+                return isG1GCEnabled.get();
+            }
+
+            @Override
+            String jvmVersion() {
+                return jvmVersion.get();
+            }
+
+            @Override
+            boolean isJava8() {
+                return isJava8.get();
+            }
+
+        };
+
+        final NodeValidationException e =
+            expectThrows(
+                NodeValidationException.class,
+                () -> BootstrapChecks.check(true, Collections.singletonList(g1GCCheck), "testG1GCCheck"));
+        assertThat(
+            e.getMessage(),
+            containsString(
+                "JVM version [" + jvmVersion.get() + "] can cause data corruption when used with G1GC; upgrade to at least Java 8u40"));
+
+        // if G1GC is disabled, nothing should happen
+        isG1GCEnabled.set(false);
+        BootstrapChecks.check(true, Collections.singletonList(g1GCCheck), "testG1GCCheck");
+
+        // if on or after update 40, nothing should happen independent of whether or not G1GC is enabled
+        isG1GCEnabled.set(randomBoolean());
+        jvmVersion.set(String.format(Locale.ROOT, "25.%d-b%d", randomIntBetween(40, 112), randomIntBetween(1, 128)));
+        BootstrapChecks.check(true, Collections.singletonList(g1GCCheck), "testG1GCCheck");
+
+        final BootstrapChecks.G1GCCheck nonOracleCheck = new BootstrapChecks.G1GCCheck() {
+
+            @Override
+            String jvmVendor() {
+                return randomAlphaOfLength(8);
+            }
+
+        };
+
+        // if not on an Oracle JVM, nothing should happen
+        BootstrapChecks.check(true, Collections.singletonList(nonOracleCheck), "testG1GCCheck");
+
+        final BootstrapChecks.G1GCCheck nonJava8Check = new BootstrapChecks.G1GCCheck() {
+
+            @Override
+            boolean isJava8() {
+                return false;
+            }
+
+        };
+
+        // if not Java 8, nothing should happen
+        BootstrapChecks.check(true, Collections.singletonList(nonJava8Check), "testG1GCCheck");
+    }
+
+    public void testAlwaysEnforcedChecks() {
+        final BootstrapCheck check = new BootstrapCheck() {
+            @Override
+            public boolean check() {
+                return true;
+            }
+
+            @Override
+            public String errorMessage() {
+                return "error";
+            }
+
+            @Override
+            public boolean alwaysEnforce() {
+                return true;
+            }
+        };
+
+        final NodeValidationException alwaysEnforced = expectThrows(
+            NodeValidationException.class,
+            () -> BootstrapChecks.check(randomBoolean(), Collections.singletonList(check), "testAlwaysEnforcedChecks"));
+        assertThat(alwaysEnforced, hasToString(containsString("error")));
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/bootstrap/BootstrapSettingsTests.java b/core/src/test/java/org/elasticsearch/bootstrap/BootstrapSettingsTests.java
index 128c82e953363..fb3d3628dd1b2 100644
--- a/core/src/test/java/org/elasticsearch/bootstrap/BootstrapSettingsTests.java
+++ b/core/src/test/java/org/elasticsearch/bootstrap/BootstrapSettingsTests.java
@@ -27,7 +27,7 @@ public class BootstrapSettingsTests extends ESTestCase {
     public void testDefaultSettings() {
         assertTrue(BootstrapSettings.SECURITY_FILTER_BAD_DEFAULTS_SETTING.get(Settings.EMPTY));
         assertFalse(BootstrapSettings.MEMORY_LOCK_SETTING.get(Settings.EMPTY));
-        assertTrue(BootstrapSettings.SECCOMP_SETTING.get(Settings.EMPTY));
+        assertTrue(BootstrapSettings.SYSTEM_CALL_FILTER_SETTING.get(Settings.EMPTY));
         assertTrue(BootstrapSettings.CTRLHANDLER_SETTING.get(Settings.EMPTY));
     }
 
diff --git a/core/src/test/java/org/elasticsearch/bootstrap/BootstrapTests.java b/core/src/test/java/org/elasticsearch/bootstrap/BootstrapTests.java
new file mode 100644
index 0000000000000..6506ed8d05689
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/bootstrap/BootstrapTests.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.bootstrap;
+
+import org.elasticsearch.test.ESTestCase;
+
+public class BootstrapTests extends ESTestCase {
+
+    public void testConfigDeprecation() {
+        Bootstrap.checkConfigExtension(".json");
+        assertWarnings("elasticsearch.json is deprecated; rename your configuration file to elasticsearch.yml");
+        Bootstrap.checkConfigExtension(".yaml");
+        assertWarnings("elasticsearch.yaml is deprecated; rename your configuration file to elasticsearch.yml");
+        Bootstrap.checkConfigExtension(".yml"); // no warnings, will be checked in @After
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/bootstrap/ElasticsearchCliTests.java b/core/src/test/java/org/elasticsearch/bootstrap/ElasticsearchCliTests.java
index f8bdf24499964..07c5a7e157f69 100644
--- a/core/src/test/java/org/elasticsearch/bootstrap/ElasticsearchCliTests.java
+++ b/core/src/test/java/org/elasticsearch/bootstrap/ElasticsearchCliTests.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.monitor.jvm.JvmInfo;
 
 import java.nio.file.Path;
+import java.util.Map;
 import java.util.function.Consumer;
 
 import static org.hamcrest.CoreMatchers.containsString;
@@ -43,6 +44,9 @@ public void testVersion() throws Exception {
         runTestThatVersionIsMutuallyExclusiveToOtherOptions("--version", "--daemonize");
         runTestThatVersionIsMutuallyExclusiveToOtherOptions("--version", "-p", "/tmp/pid");
         runTestThatVersionIsMutuallyExclusiveToOtherOptions("--version", "--pidfile", "/tmp/pid");
+        runTestThatVersionIsMutuallyExclusiveToOtherOptions("--version", "-q");
+        runTestThatVersionIsMutuallyExclusiveToOtherOptions("--version", "--quiet");
+
         runTestThatVersionIsReturned("-V");
         runTestThatVersionIsReturned("--version");
     }
@@ -66,7 +70,7 @@ private void runTestThatVersionIsReturned(String... args) throws Exception {
     }
 
     private void runTestVersion(int expectedStatus, Consumer<String> outputConsumer, String... args) throws Exception {
-        runTest(expectedStatus, false, outputConsumer, (foreground, pidFile, esSettings) -> {}, args);
+        runTest(expectedStatus, false, outputConsumer, (foreground, pidFile, quiet, esSettings) -> {}, args);
     }
 
     public void testPositionalArgs() throws Exception {
@@ -74,23 +78,20 @@ public void testPositionalArgs() throws Exception {
             ExitCodes.USAGE,
             false,
             output -> assertThat(output, containsString("Positional arguments not allowed, found [foo]")),
-            (foreground, pidFile, esSettings) -> {},
-            "foo"
-        );
+            (foreground, pidFile, quiet, esSettings) -> {},
+            "foo");
         runTest(
             ExitCodes.USAGE,
             false,
             output -> assertThat(output, containsString("Positional arguments not allowed, found [foo, bar]")),
-            (foreground, pidFile, esSettings) -> {},
-            "foo", "bar"
-        );
+            (foreground, pidFile, quiet, esSettings) -> {},
+            "foo", "bar");
         runTest(
             ExitCodes.USAGE,
             false,
             output -> assertThat(output, containsString("Positional arguments not allowed, found [foo]")),
-            (foreground, pidFile, esSettings) -> {},
-            "-E", "foo=bar", "foo", "-E", "baz=qux"
-        );
+            (foreground, pidFile, quiet, esSettings) -> {},
+            "-E", "foo=bar", "foo", "-E", "baz=qux");
     }
 
     public void testThatPidFileCanBeConfigured() throws Exception {
@@ -109,7 +110,7 @@ private void runPidFileTest(final int expectedStatus, final boolean expectedInit
                 expectedStatus,
                 expectedInit,
                 outputConsumer,
-                (foreground, pidFile, esSettings) -> assertThat(pidFile.toString(), equalTo(expectedPidFile.toString())),
+                (foreground, pidFile, quiet, esSettings) -> assertThat(pidFile.toString(), equalTo(expectedPidFile.toString())),
                 args);
     }
 
@@ -124,7 +125,22 @@ private void runDaemonizeTest(final boolean expectedDaemonize, final String... a
                 ExitCodes.OK,
                 true,
                 output -> {},
-                (foreground, pidFile, esSettings) -> assertThat(foreground, equalTo(!expectedDaemonize)),
+                (foreground, pidFile, quiet, esSettings) -> assertThat(foreground, equalTo(!expectedDaemonize)),
+                args);
+    }
+
+    public void testThatParsingQuietOptionWorks() throws Exception {
+        runQuietTest(true, "-q");
+        runQuietTest(true, "--quiet");
+        runQuietTest(false);
+    }
+
+    private void runQuietTest(final boolean expectedQuiet, final String... args) throws Exception {
+        runTest(
+                ExitCodes.OK,
+                true,
+                output -> {},
+                (foreground, pidFile, quiet, esSettings) -> assertThat(quiet, equalTo(expectedQuiet)),
                 args);
     }
 
@@ -133,23 +149,30 @@ public void testElasticsearchSettings() throws Exception {
                 ExitCodes.OK,
                 true,
                 output -> {},
-                (foreground, pidFile, esSettings) -> {
-                    assertThat(esSettings.size(), equalTo(2));
-                    assertThat(esSettings, hasEntry("foo", "bar"));
-                    assertThat(esSettings, hasEntry("baz", "qux"));
+                (foreground, pidFile, quiet, env) -> {
+                    Map<String, String> settings = env.settings().getAsMap();
+                    assertThat(settings, hasEntry("foo", "bar"));
+                    assertThat(settings, hasEntry("baz", "qux"));
                 },
-                "-Efoo=bar", "-E", "baz=qux"
-        );
+                "-Efoo=bar", "-E", "baz=qux");
     }
 
     public void testElasticsearchSettingCanNotBeEmpty() throws Exception {
         runTest(
                 ExitCodes.USAGE,
                 false,
-                output -> assertThat(output, containsString("Setting [foo] must not be empty")),
-                (foreground, pidFile, esSettings) -> {},
-                "-E", "foo="
-        );
+                output -> assertThat(output, containsString("setting [foo] must not be empty")),
+                (foreground, pidFile, quiet, esSettings) -> {},
+                "-E", "foo=");
+    }
+
+    public void testElasticsearchSettingCanNotBeDuplicated() throws Exception {
+        runTest(
+                ExitCodes.USAGE,
+                false,
+                output -> assertThat(output, containsString("setting [foo] already set, saw [bar] and [baz]")),
+                (foreground, pidFile, quiet, initialEnv) -> {},
+                "-E", "foo=bar", "-E", "foo=baz");
     }
 
     public void testUnknownOption() throws Exception {
@@ -157,7 +180,7 @@ public void testUnknownOption() throws Exception {
                 ExitCodes.USAGE,
                 false,
                 output -> assertThat(output, containsString("network.host is not a recognized option")),
-                (foreground, pidFile, esSettings) -> {},
+                (foreground, pidFile, quiet, esSettings) -> {},
                 "--network.host");
     }
 
diff --git a/core/src/test/java/org/elasticsearch/bootstrap/ElasticsearchUncaughtExceptionHandlerTests.java b/core/src/test/java/org/elasticsearch/bootstrap/ElasticsearchUncaughtExceptionHandlerTests.java
index e4ff83e9b4057..6e40153b467d9 100644
--- a/core/src/test/java/org/elasticsearch/bootstrap/ElasticsearchUncaughtExceptionHandlerTests.java
+++ b/core/src/test/java/org/elasticsearch/bootstrap/ElasticsearchUncaughtExceptionHandlerTests.java
@@ -60,7 +60,7 @@ public void testUncaughtError() throws InterruptedException {
             new IOError(new IOException("fatal")),
             new Error() {});
         final Thread thread = new Thread(() -> { throw error; });
-        final String name = randomAsciiOfLength(10);
+        final String name = randomAlphaOfLength(10);
         thread.setName(name);
         final AtomicBoolean halt = new AtomicBoolean();
         final AtomicInteger observedStatus = new AtomicInteger();
@@ -103,7 +103,7 @@ void onNonFatalUncaught(String threadName, Throwable t) {
     public void testUncaughtException() throws InterruptedException {
         final RuntimeException e = new RuntimeException("boom");
         final Thread thread = new Thread(() -> { throw e; });
-        final String name = randomAsciiOfLength(10);
+        final String name = randomAlphaOfLength(10);
         thread.setName(name);
         final AtomicReference<String> threadNameReference = new AtomicReference<>();
         final AtomicReference<Throwable> throwableReference = new AtomicReference<>();
diff --git a/core/src/test/java/org/elasticsearch/bootstrap/JarHellTests.java b/core/src/test/java/org/elasticsearch/bootstrap/JarHellTests.java
index 467608922d15d..7003ef3d81efe 100644
--- a/core/src/test/java/org/elasticsearch/bootstrap/JarHellTests.java
+++ b/core/src/test/java/org/elasticsearch/bootstrap/JarHellTests.java
@@ -30,7 +30,9 @@
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
+import java.util.Set;
 import java.util.jar.Attributes;
 import java.util.jar.JarOutputStream;
 import java.util.jar.Manifest;
@@ -62,7 +64,8 @@ URL makeFile(Path dir, String name) throws IOException {
 
     public void testDifferentJars() throws Exception {
         Path dir = createTempDir();
-        URL[] jars = {makeJar(dir, "foo.jar", null, "DuplicateClass.class"), makeJar(dir, "bar.jar", null, "DuplicateClass.class")};
+        Set<URL> jars = asSet(makeJar(dir, "foo.jar", null, "DuplicateClass.class"),
+                              makeJar(dir, "bar.jar", null, "DuplicateClass.class"));
         try {
             JarHell.checkJarHell(jars);
             fail("did not get expected exception");
@@ -74,17 +77,11 @@ public void testDifferentJars() throws Exception {
         }
     }
 
-    public void testDuplicateClasspathLeniency() throws Exception {
-        Path dir = createTempDir();
-        URL jar = makeJar(dir, "foo.jar", null, "Foo.class");
-        URL[] jars = {jar, jar};
-        JarHell.checkJarHell(jars);
-    }
-
     public void testDirsOnClasspath() throws Exception {
         Path dir1 = createTempDir();
         Path dir2 = createTempDir();
-        URL[] dirs = {makeFile(dir1, "DuplicateClass.class"), makeFile(dir2, "DuplicateClass.class")};
+        Set<URL> dirs = asSet(makeFile(dir1, "DuplicateClass.class"),
+                              makeFile(dir2, "DuplicateClass.class"));
         try {
             JarHell.checkJarHell(dirs);
             fail("did not get expected exception");
@@ -99,7 +96,8 @@ public void testDirsOnClasspath() throws Exception {
     public void testDirAndJar() throws Exception {
         Path dir1 = createTempDir();
         Path dir2 = createTempDir();
-        URL[] dirs = {makeJar(dir1, "foo.jar", null, "DuplicateClass.class"), makeFile(dir2, "DuplicateClass.class")};
+        Set<URL> dirs = asSet(makeJar(dir1, "foo.jar", null, "DuplicateClass.class"),
+                              makeFile(dir2, "DuplicateClass.class"));
         try {
             JarHell.checkJarHell(dirs);
             fail("did not get expected exception");
@@ -111,16 +109,10 @@ public void testDirAndJar() throws Exception {
         }
     }
 
-    public void testLog4jThrowableProxyLeniency() throws Exception {
-        Path dir = createTempDir();
-        URL[] jars = {makeJar(dir, "foo.jar", null, "org.apache.logging.log4j.core.impl.ThrowableProxy.class"), makeJar(dir, "bar.jar", null, "org.apache.logging.log4j.core.impl.ThrowableProxy.class")};
-        JarHell.checkJarHell(jars);
-    }
-
     public void testWithinSingleJar() throws Exception {
         // the java api for zip file does not allow creating duplicate entries (good!) so
-        // this bogus jar had to be constructed with ant
-        URL[] jars = {JarHellTests.class.getResource("duplicate-classes.jar")};
+        // this bogus jar had to be with https://github.com/jasontedor/duplicate-classes
+        Set<URL> jars = Collections.singleton(JarHellTests.class.getResource("duplicate-classes.jar"));
         try {
             JarHell.checkJarHell(jars);
             fail("did not get expected exception");
@@ -133,7 +125,7 @@ public void testWithinSingleJar() throws Exception {
     }
 
     public void testXmlBeansLeniency() throws Exception {
-        URL[] jars = {JarHellTests.class.getResource("duplicate-xmlbeans-classes.jar")};
+        Set<URL> jars = Collections.singleton(JarHellTests.class.getResource("duplicate-xmlbeans-classes.jar"));
         JarHell.checkJarHell(jars);
     }
 
@@ -151,7 +143,7 @@ public void testRequiredJDKVersionTooOld() throws Exception {
         Attributes attributes = manifest.getMainAttributes();
         attributes.put(Attributes.Name.MANIFEST_VERSION, "1.0.0");
         attributes.put(new Attributes.Name("X-Compile-Target-JDK"), targetVersion.toString());
-        URL[] jars = {makeJar(dir, "foo.jar", manifest, "Foo.class")};
+        Set<URL> jars = Collections.singleton(makeJar(dir, "foo.jar", manifest, "Foo.class"));
         try {
             JarHell.checkJarHell(jars);
             fail("did not get expected exception");
@@ -167,7 +159,7 @@ public void testBadJDKVersionInJar() throws Exception {
         Attributes attributes = manifest.getMainAttributes();
         attributes.put(Attributes.Name.MANIFEST_VERSION, "1.0.0");
         attributes.put(new Attributes.Name("X-Compile-Target-JDK"), "bogus");
-        URL[] jars = {makeJar(dir, "foo.jar", manifest, "Foo.class")};
+        Set<URL> jars = Collections.singleton(makeJar(dir, "foo.jar", manifest, "Foo.class"));
         try {
             JarHell.checkJarHell(jars);
             fail("did not get expected exception");
@@ -182,8 +174,7 @@ public void testRequiredJDKVersionIsOK() throws Exception {
         Attributes attributes = manifest.getMainAttributes();
         attributes.put(Attributes.Name.MANIFEST_VERSION, "1.0.0");
         attributes.put(new Attributes.Name("X-Compile-Target-JDK"), "1.7");
-        URL[] jars = {makeJar(dir, "foo.jar", manifest, "Foo.class")};
-
+        Set<URL> jars = Collections.singleton(makeJar(dir, "foo.jar", manifest, "Foo.class"));
         JarHell.checkJarHell(jars);
     }
 
@@ -194,7 +185,7 @@ public void testGoodESVersionInJar() throws Exception {
         Attributes attributes = manifest.getMainAttributes();
         attributes.put(Attributes.Name.MANIFEST_VERSION, "1.0.0");
         attributes.put(new Attributes.Name("X-Compile-Elasticsearch-Version"), Version.CURRENT.toString());
-        URL[] jars = {makeJar(dir, "foo.jar", manifest, "Foo.class")};
+        Set<URL> jars = Collections.singleton(makeJar(dir, "foo.jar", manifest, "Foo.class"));
         JarHell.checkJarHell(jars);
     }
 
@@ -205,7 +196,7 @@ public void testBadESVersionInJar() throws Exception {
         Attributes attributes = manifest.getMainAttributes();
         attributes.put(Attributes.Name.MANIFEST_VERSION, "1.0.0");
         attributes.put(new Attributes.Name("X-Compile-Elasticsearch-Version"), "1.0-bogus");
-        URL[] jars = {makeJar(dir, "foo.jar", manifest, "Foo.class")};
+        Set<URL> jars = Collections.singleton(makeJar(dir, "foo.jar", manifest, "Foo.class"));
         try {
             JarHell.checkJarHell(jars);
             fail("did not get expected exception");
@@ -248,8 +239,8 @@ public void testParseClassPathUnix() throws Exception {
         Path element1 = createTempDir();
         Path element2 = createTempDir();
 
-        URL expected[] = { element1.toUri().toURL(), element2.toUri().toURL() };
-        assertArrayEquals(expected, JarHell.parseClassPath(element1.toString() + ":" + element2.toString()));
+        Set<URL> expected = asSet(element1.toUri().toURL(), element2.toUri().toURL());
+        assertEquals(expected, JarHell.parseClassPath(element1.toString() + ":" + element2.toString()));
     }
 
     /**
@@ -277,8 +268,8 @@ public void testParseClassPathWindows() throws Exception {
         Path element1 = createTempDir();
         Path element2 = createTempDir();
 
-        URL expected[] = { element1.toUri().toURL(), element2.toUri().toURL() };
-        assertArrayEquals(expected, JarHell.parseClassPath(element1.toString() + ";" + element2.toString()));
+        Set<URL> expected = asSet(element1.toUri().toURL(), element2.toUri().toURL());
+        assertEquals(expected, JarHell.parseClassPath(element1.toString() + ";" + element2.toString()));
     }
 
     /**
@@ -304,13 +295,13 @@ public void testCrazyEclipseClassPathWindows() throws Exception {
         assumeTrue("test is designed for windows-like systems only", ";".equals(System.getProperty("path.separator")));
         assumeTrue("test is designed for windows-like systems only", "\\".equals(System.getProperty("file.separator")));
 
-        URL expected[] = {
+        Set<URL> expected = asSet(
             PathUtils.get("c:\\element1").toUri().toURL(),
             PathUtils.get("c:\\element2").toUri().toURL(),
             PathUtils.get("c:\\element3").toUri().toURL(),
-            PathUtils.get("c:\\element 4").toUri().toURL(),
-        };
-        URL actual[] = JarHell.parseClassPath("c:\\element1;c:\\element2;/c:/element3;/c:/element 4");
-        assertArrayEquals(expected, actual);
+            PathUtils.get("c:\\element 4").toUri().toURL()
+        );
+        Set<URL> actual = JarHell.parseClassPath("c:\\element1;c:\\element2;/c:/element3;/c:/element 4");
+        assertEquals(expected, actual);
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/bootstrap/MaxMapCountCheckTests.java b/core/src/test/java/org/elasticsearch/bootstrap/MaxMapCountCheckTests.java
index 5a2b8a5a14330..c5b99a91ffa3b 100644
--- a/core/src/test/java/org/elasticsearch/bootstrap/MaxMapCountCheckTests.java
+++ b/core/src/test/java/org/elasticsearch/bootstrap/MaxMapCountCheckTests.java
@@ -26,7 +26,7 @@
 import org.apache.lucene.util.Constants;
 import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.logging.ESLoggerFactory;
-import org.elasticsearch.common.logging.TestLoggers;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.MockLogAppender;
 
@@ -47,7 +47,7 @@ public class MaxMapCountCheckTests extends ESTestCase {
 
     public void testGetMaxMapCountOnLinux() {
         if (Constants.LINUX) {
-            final BootstrapCheck.MaxMapCountCheck check = new BootstrapCheck.MaxMapCountCheck();
+            final BootstrapChecks.MaxMapCountCheck check = new BootstrapChecks.MaxMapCountCheck();
             assertThat(check.getMaxMapCount(), greaterThan(0L));
         }
     }
@@ -57,7 +57,7 @@ public void testGetMaxMapCount() throws IOException, IllegalAccessException {
         final BufferedReader reader = mock(BufferedReader.class);
         when(reader.readLine()).thenReturn(Long.toString(procSysVmMaxMapCount));
         final Path procSysVmMaxMapCountPath = PathUtils.get("/proc/sys/vm/max_map_count");
-        BootstrapCheck.MaxMapCountCheck check = new BootstrapCheck.MaxMapCountCheck() {
+        BootstrapChecks.MaxMapCountCheck check = new BootstrapChecks.MaxMapCountCheck() {
             @Override
             BufferedReader getBufferedReader(Path path) throws IOException {
                 assertEquals(path, procSysVmMaxMapCountPath);
@@ -74,6 +74,7 @@ BufferedReader getBufferedReader(Path path) throws IOException {
             when(reader.readLine()).thenThrow(ioException);
             final Logger logger = ESLoggerFactory.getLogger("testGetMaxMapCountIOException");
             final MockLogAppender appender = new MockLogAppender();
+            appender.start();
             appender.addExpectation(
                     new ParameterizedMessageLoggingExpectation(
                             "expected logged I/O exception",
@@ -82,11 +83,12 @@ BufferedReader getBufferedReader(Path path) throws IOException {
                             "I/O exception while trying to read [{}]",
                             new Object[] { procSysVmMaxMapCountPath },
                             e -> ioException == e));
-            TestLoggers.addAppender(logger, appender);
+            Loggers.addAppender(logger, appender);
             assertThat(check.getMaxMapCount(logger), equalTo(-1L));
             appender.assertAllExpectationsMatched();
             verify(reader).close();
-            TestLoggers.removeAppender(logger, appender);
+            Loggers.removeAppender(logger, appender);
+            appender.stop();
         }
 
         {
@@ -94,6 +96,7 @@ BufferedReader getBufferedReader(Path path) throws IOException {
             when(reader.readLine()).thenReturn("eof");
             final Logger logger = ESLoggerFactory.getLogger("testGetMaxMapCountNumberFormatException");
             final MockLogAppender appender = new MockLogAppender();
+            appender.start();
             appender.addExpectation(
                     new ParameterizedMessageLoggingExpectation(
                             "expected logged number format exception",
@@ -102,11 +105,12 @@ BufferedReader getBufferedReader(Path path) throws IOException {
                             "unable to parse vm.max_map_count [{}]",
                             new Object[] { "eof" },
                             e -> e instanceof NumberFormatException && e.getMessage().equals("For input string: \"eof\"")));
-            TestLoggers.addAppender(logger, appender);
+            Loggers.addAppender(logger, appender);
             assertThat(check.getMaxMapCount(logger), equalTo(-1L));
             appender.assertAllExpectationsMatched();
             verify(reader).close();
-            TestLoggers.removeAppender(logger, appender);
+            Loggers.removeAppender(logger, appender);
+            appender.stop();
         }
 
     }
@@ -160,13 +164,13 @@ public void testMaxMapCountCheckRead() throws IOException {
         final String rawProcSysVmMaxMapCount = Long.toString(randomIntBetween(1, Integer.MAX_VALUE));
         final BufferedReader reader = mock(BufferedReader.class);
         when(reader.readLine()).thenReturn(rawProcSysVmMaxMapCount);
-        final BootstrapCheck.MaxMapCountCheck check = new BootstrapCheck.MaxMapCountCheck();
+        final BootstrapChecks.MaxMapCountCheck check = new BootstrapChecks.MaxMapCountCheck();
         assertThat(check.readProcSysVmMaxMapCount(reader), equalTo(rawProcSysVmMaxMapCount));
     }
 
     public void testMaxMapCountCheckParse() {
         final long procSysVmMaxMapCount = randomIntBetween(1, Integer.MAX_VALUE);
-        final BootstrapCheck.MaxMapCountCheck check = new BootstrapCheck.MaxMapCountCheck();
+        final BootstrapChecks.MaxMapCountCheck check = new BootstrapChecks.MaxMapCountCheck();
         assertThat(check.parseProcSysVmMaxMapCount(Long.toString(procSysVmMaxMapCount)), equalTo(procSysVmMaxMapCount));
     }
 
diff --git a/core/src/test/java/org/elasticsearch/broadcast/BroadcastActionsIT.java b/core/src/test/java/org/elasticsearch/broadcast/BroadcastActionsIT.java
index 63091a97818b7..0cd942328db56 100644
--- a/core/src/test/java/org/elasticsearch/broadcast/BroadcastActionsIT.java
+++ b/core/src/test/java/org/elasticsearch/broadcast/BroadcastActionsIT.java
@@ -39,7 +39,7 @@ protected int maximumNumberOfReplicas() {
     }
 
     public void testBroadcastOperations() throws IOException {
-        assertAcked(prepareCreate("test", 1).execute().actionGet(5000));
+        assertAcked(prepareCreate("test", 1));
 
         NumShards numShards = getNumShards("test");
 
diff --git a/core/src/test/java/org/elasticsearch/bwcompat/BasicAnalysisBackwardCompatibilityIT.java b/core/src/test/java/org/elasticsearch/bwcompat/BasicAnalysisBackwardCompatibilityIT.java
index f1451255b60b8..8286317f80ae6 100644
--- a/core/src/test/java/org/elasticsearch/bwcompat/BasicAnalysisBackwardCompatibilityIT.java
+++ b/core/src/test/java/org/elasticsearch/bwcompat/BasicAnalysisBackwardCompatibilityIT.java
@@ -112,7 +112,7 @@ private static final class InputOutput {
         final String input;
         final String field;
 
-        public InputOutput(AnalyzeResponse response, String input, String field) {
+        InputOutput(AnalyzeResponse response, String input, String field) {
             this.response = response;
             this.input = input;
             this.field = field;
diff --git a/core/src/test/java/org/elasticsearch/bwcompat/BasicBackwardsCompatibilityIT.java b/core/src/test/java/org/elasticsearch/bwcompat/BasicBackwardsCompatibilityIT.java
index 75f089b3488a2..efb0f8462d662 100644
--- a/core/src/test/java/org/elasticsearch/bwcompat/BasicBackwardsCompatibilityIT.java
+++ b/core/src/test/java/org/elasticsearch/bwcompat/BasicBackwardsCompatibilityIT.java
@@ -461,13 +461,13 @@ public void testDeleteRoutingRequired() throws ExecutionException, InterruptedEx
         IndexRequestBuilder[] indexRequestBuilders = new IndexRequestBuilder[numDocs];
         for (int i = 0; i < numDocs - 2; i++) {
             indexRequestBuilders[i] = client().prepareIndex("test", "test", Integer.toString(i))
-                    .setRouting(randomAsciiOfLength(randomIntBetween(1, 10))).setSource("field", "value");
+                    .setRouting(randomAlphaOfLength(randomIntBetween(1, 10))).setSource("field", "value");
         }
         String firstDocId = Integer.toString(numDocs - 2);
         indexRequestBuilders[numDocs - 2] = client().prepareIndex("test", "test", firstDocId)
                 .setRouting("routing").setSource("field", "value");
         String secondDocId = Integer.toString(numDocs - 1);
-        String secondRouting = randomAsciiOfLength(randomIntBetween(1, 10));
+        String secondRouting = randomAlphaOfLength(randomIntBetween(1, 10));
         indexRequestBuilders[numDocs - 1] = client().prepareIndex("test", "test", secondDocId)
                 .setRouting(secondRouting).setSource("field", "value");
 
diff --git a/core/src/test/java/org/elasticsearch/bwcompat/ClusterStateBackwardsCompatIT.java b/core/src/test/java/org/elasticsearch/bwcompat/ClusterStateBackwardsCompatIT.java
index eb6648cad0258..fd7d93d65cd68 100644
--- a/core/src/test/java/org/elasticsearch/bwcompat/ClusterStateBackwardsCompatIT.java
+++ b/core/src/test/java/org/elasticsearch/bwcompat/ClusterStateBackwardsCompatIT.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
 import org.elasticsearch.client.transport.TransportClient;
-import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlock;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.block.ClusterBlocks;
@@ -38,7 +37,6 @@
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_METADATA;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_READ;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_WRITE;
-import static org.hamcrest.Matchers.equalTo;
 
 public class ClusterStateBackwardsCompatIT extends ESBackcompatTestCase {
     public void testClusterState() throws Exception {
@@ -50,7 +48,6 @@ public void testClusterState() throws Exception {
                 tc.addTransportAddress(n.getNode().getAddress());
                 ClusterStateResponse response = tc.admin().cluster().prepareState().execute().actionGet();
 
-                assertThat(response.getState().status(), equalTo(ClusterState.ClusterStateStatus.UNKNOWN));
                 assertNotNull(response.getClusterName());
                 assertTrue(response.getState().getMetaData().hasIndex("test"));
             }
diff --git a/core/src/test/java/org/elasticsearch/bwcompat/OldIndexBackwardsCompatibilityIT.java b/core/src/test/java/org/elasticsearch/bwcompat/OldIndexBackwardsCompatibilityIT.java
index e4e7927872bf0..e11d6feef43fd 100644
--- a/core/src/test/java/org/elasticsearch/bwcompat/OldIndexBackwardsCompatibilityIT.java
+++ b/core/src/test/java/org/elasticsearch/bwcompat/OldIndexBackwardsCompatibilityIT.java
@@ -23,6 +23,7 @@
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.TestUtil;
 import org.elasticsearch.Version;
+import org.elasticsearch.VersionTests;
 import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
 import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
 import org.elasticsearch.action.admin.indices.segments.IndexSegments;
@@ -33,9 +34,11 @@
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.client.Requests;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.routing.RecoverySource;
+import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
@@ -54,13 +57,14 @@
 import org.elasticsearch.node.Node;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchHitField;
+import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
-import org.elasticsearch.test.InternalTestCluster;
 import org.elasticsearch.test.OldIndexUtils;
 import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
@@ -82,7 +86,9 @@
 import java.util.TreeSet;
 
 import static org.elasticsearch.test.OldIndexUtils.assertUpgradeWorks;
+import static org.elasticsearch.test.OldIndexUtils.getIndexDir;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 
 // needs at least 2 nodes since it bumps replicas to 1
@@ -92,7 +98,6 @@ public class OldIndexBackwardsCompatibilityIT extends ESIntegTestCase {
     // TODO: test for proper exception on unsupported indexes (maybe via separate test?)
     // We have a 0.20.6.zip etc for this.
 
-
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
         return Arrays.asList(InternalSettingsPlugin.class);
@@ -125,24 +130,23 @@ public Settings nodeSettings(int ord) {
     }
 
     void setupCluster() throws Exception {
-        InternalTestCluster.Async<List<String>> replicas = internalCluster().startNodesAsync(1); // for replicas
+        List<String> replicas = internalCluster().startNodes(1); // for replicas
 
         Path baseTempDir = createTempDir();
         // start single data path node
         Settings.Builder nodeSettings = Settings.builder()
                 .put(Environment.PATH_DATA_SETTING.getKey(), baseTempDir.resolve("single-path").toAbsolutePath())
                 .put(Node.NODE_MASTER_SETTING.getKey(), false); // workaround for dangling index loading issue when node is master
-        InternalTestCluster.Async<String> singleDataPathNode = internalCluster().startNodeAsync(nodeSettings.build());
+        singleDataPathNodeName = internalCluster().startNode(nodeSettings);
 
         // start multi data path node
         nodeSettings = Settings.builder()
                 .put(Environment.PATH_DATA_SETTING.getKey(), baseTempDir.resolve("multi-path1").toAbsolutePath() + "," + baseTempDir
                         .resolve("multi-path2").toAbsolutePath())
                 .put(Node.NODE_MASTER_SETTING.getKey(), false); // workaround for dangling index loading issue when node is master
-        InternalTestCluster.Async<String> multiDataPathNode = internalCluster().startNodeAsync(nodeSettings.build());
+        multiDataPathNodeName = internalCluster().startNode(nodeSettings);
 
         // find single data path dir
-        singleDataPathNodeName = singleDataPathNode.get();
         Path[] nodePaths = internalCluster().getInstance(NodeEnvironment.class, singleDataPathNodeName).nodeDataPaths();
         assertEquals(1, nodePaths.length);
         singleDataPath = nodePaths[0].resolve(NodeEnvironment.INDICES_FOLDER);
@@ -151,7 +155,6 @@ void setupCluster() throws Exception {
         logger.info("--> Single data path: {}", singleDataPath);
 
         // find multi data path dirs
-        multiDataPathNodeName = multiDataPathNode.get();
         nodePaths = internalCluster().getInstance(NodeEnvironment.class, multiDataPathNodeName).nodeDataPaths();
         assertEquals(2, nodePaths.length);
         multiDataPath = new Path[]{nodePaths[0].resolve(NodeEnvironment.INDICES_FOLDER),
@@ -161,8 +164,7 @@ void setupCluster() throws Exception {
         Files.createDirectories(multiDataPath[0]);
         Files.createDirectories(multiDataPath[1]);
         logger.info("--> Multi data paths: {}, {}", multiDataPath[0], multiDataPath[1]);
-
-        replicas.get(); // wait for replicas
+        ensureGreen();
     }
 
     void upgradeIndexFolder() throws Exception {
@@ -182,10 +184,10 @@ void unloadIndex(String indexName) throws Exception {
 
     public void testAllVersionsTested() throws Exception {
         SortedSet<String> expectedVersions = new TreeSet<>();
-        for (Version v : VersionUtils.allVersions()) {
+        for (Version v : VersionUtils.allReleasedVersions()) {
             if (VersionUtils.isSnapshot(v)) continue;  // snapshots are unreleased, so there is no backcompat yet
-            if (v.isAlpha()) continue; // no guarantees for alpha releases
-            if (v.onOrBefore(Version.V_2_0_0_beta1)) continue; // we can only test back one major lucene version
+            if (v.isRelease() == false) continue; // no guarantees for prereleases
+            if (v.before(Version.CURRENT.minimumIndexCompatibilityVersion())) continue; // we can only support one major version backward
             if (v.equals(Version.CURRENT)) continue; // the current version is always compatible with itself
             expectedVersions.add("index-" + v.toString() + ".zip");
         }
@@ -243,6 +245,8 @@ void assertOldIndexWorks(String index) throws Exception {
         assertUpgradeWorks(client(), indexName, version);
         assertDeleteByQueryWorked(indexName, version);
         assertPositionIncrementGapDefaults(indexName, version);
+        assertAliasWithBadName(indexName, version);
+        assertStoredBinaryFields(indexName, version);
         unloadIndex(indexName);
     }
 
@@ -385,7 +389,7 @@ void assertRealtimeGetWorks(String indexName) {
         SearchHit hit = searchReq.get().getHits().getAt(0);
         String docId = hit.getId();
         // foo is new, it is not a field in the generated index
-        client().prepareUpdate(indexName, "doc", docId).setDoc("foo", "bar").get();
+        client().prepareUpdate(indexName, "doc", docId).setDoc(Requests.INDEX_CONTENT_TYPE, "foo", "bar").get();
         GetResponse getRsp = client().prepareGet(indexName, "doc", docId).get();
         Map<String, Object> source = getRsp.getSourceAsMap();
         assertThat(source, Matchers.hasKey("foo"));
@@ -428,6 +432,46 @@ void assertPositionIncrementGapDefaults(String indexName, Version version) throw
         }
     }
 
+    /**
+     * Search on an alias that contains illegal characters that would prevent it from being created after 5.1.0. It should still be
+     * search-able though.
+     */
+    void assertAliasWithBadName(String indexName, Version version) throws Exception {
+        Version v510Unreleased = Version.fromId(5010099);
+        VersionTests.assertUnknownVersion(v510Unreleased);
+        if (version.onOrAfter(v510Unreleased)) {
+            return;
+        }
+        // We can read from the alias just like we can read from the index.
+        String aliasName = "#" + indexName;
+        long totalDocs = client().prepareSearch(indexName).setSize(0).get().getHits().totalHits();
+        assertHitCount(client().prepareSearch(aliasName).setSize(0).get(), totalDocs);
+        assertThat(totalDocs, greaterThanOrEqualTo(2000L));
+
+        // We can remove the alias.
+        assertAcked(client().admin().indices().prepareAliases().removeAlias(indexName, aliasName).get());
+        assertFalse(client().admin().indices().prepareAliasesExist(aliasName).get().exists());
+    }
+
+    /**
+     * Make sure we can load stored binary fields.
+     */
+    void assertStoredBinaryFields(String indexName, Version version) throws Exception {
+        SearchRequestBuilder builder = client().prepareSearch(indexName);
+        builder.setQuery(QueryBuilders.matchAllQuery());
+        builder.setSize(100);
+        builder.addStoredField("binary");
+        SearchHits hits = builder.get().getHits();
+        assertEquals(100, hits.hits().length);
+        for(SearchHit hit : hits) {
+            SearchHitField field = hit.field("binary");
+            assertNotNull(field);
+            Object value = field.value();
+            assertTrue(value instanceof BytesArray);
+            assertEquals(16, ((BytesArray) value).length());
+        }
+    }
+
     private Path getNodeDir(String indexFile) throws IOException {
         Path unzipDir = createTempDir();
         Path unzipDataDir = unzipDir.resolve("data");
@@ -445,8 +489,15 @@ private Path getNodeDir(String indexFile) throws IOException {
             throw new IllegalStateException("Backwards index must contain exactly one cluster");
         }
 
-        // the bwc scripts packs the indices under this path
-        return list[0].resolve("nodes/0/");
+        int zipIndex = indexFile.indexOf(".zip");
+        final Version version = Version.fromString(indexFile.substring("index-".length(), zipIndex));
+        if (version.before(Version.V_5_0_0_alpha1)) {
+            // the bwc scripts packs the indices under this path
+            return list[0].resolve("nodes/0/");
+        } else {
+            // after 5.0.0, data folders do not include the cluster name
+            return list[0].resolve("0");
+        }
     }
 
     public void testOldClusterStates() throws Exception {
@@ -481,9 +532,19 @@ public IndexMetaData fromXContent(XContentParser parser) throws IOException {
             String indexName = indexFile.replace(".zip", "").toLowerCase(Locale.ROOT).replace("unsupported-", "index-");
             Path nodeDir = getNodeDir(indexFile);
             logger.info("Parsing cluster state files from index [{}]", indexName);
-            assertNotNull(globalFormat.loadLatestState(logger, nodeDir)); // no exception
-            Path indexDir = nodeDir.resolve("indices").resolve(indexName);
-            assertNotNull(indexFormat.loadLatestState(logger, indexDir)); // no exception
+            final MetaData metaData = globalFormat.loadLatestState(logger, xContentRegistry(), nodeDir);
+            assertNotNull(metaData);
+
+            final Version version = Version.fromString(indexName.substring("index-".length()));
+            final Path dataDir;
+            if (version.before(Version.V_5_0_0_alpha1)) {
+                dataDir = nodeDir.getParent().getParent();
+            } else {
+                dataDir = nodeDir.getParent();
+            }
+            final Path indexDir = getIndexDir(logger, indexName, indexFile, dataDir);
+            assertNotNull(indexFormat.loadLatestState(logger, xContentRegistry(), indexDir));
         }
     }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/bwcompat/RepositoryUpgradabilityIT.java b/core/src/test/java/org/elasticsearch/bwcompat/RepositoryUpgradabilityIT.java
index 9bfcc554998b5..02ba7f5ea4fa6 100644
--- a/core/src/test/java/org/elasticsearch/bwcompat/RepositoryUpgradabilityIT.java
+++ b/core/src/test/java/org/elasticsearch/bwcompat/RepositoryUpgradabilityIT.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.bwcompat;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.io.FileTestUtils;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
@@ -47,7 +48,7 @@
  */
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST)
 // this test sometimes fails in recovery when the recovery is reset, increasing the logging level to help debug
-@TestLogging("indices.recovery:DEBUG")
+@TestLogging("org.elasticsearch.indices.recovery:DEBUG")
 public class RepositoryUpgradabilityIT extends AbstractSnapshotIntegTestCase {
 
     /**
@@ -70,7 +71,12 @@ public void testRepositoryWorksWithCrossVersions() throws Exception {
             final Set<SnapshotInfo> snapshotInfos = Sets.newHashSet(getSnapshots(repoName));
             assertThat(snapshotInfos.size(), equalTo(1));
             SnapshotInfo originalSnapshot = snapshotInfos.iterator().next();
-            assertThat(originalSnapshot.snapshotId(), equalTo(new SnapshotId("test_1", "test_1")));
+            if (Version.fromString(version).before(Version.V_5_0_0_alpha1)) {
+                assertThat(originalSnapshot.snapshotId(), equalTo(new SnapshotId("test_1", "test_1")));
+            } else {
+                assertThat(originalSnapshot.snapshotId().getName(), equalTo("test_1"));
+                assertNotNull(originalSnapshot.snapshotId().getUUID()); // it's a random UUID now
+            }
             assertThat(Sets.newHashSet(originalSnapshot.indices()), equalTo(indices));
 
             logger.info("--> restore the original snapshot");
@@ -148,7 +154,7 @@ private List<String> listRepoVersions() throws Exception {
         final String prefix = "repo";
         final List<String> repoVersions = new ArrayList<>();
         final Path repoFiles = getBwcIndicesPath();
-        try (final DirectoryStream<Path> dirStream = Files.newDirectoryStream(repoFiles, prefix + "-*.zip")) {
+        try (DirectoryStream<Path> dirStream = Files.newDirectoryStream(repoFiles, prefix + "-*.zip")) {
             for (final Path entry : dirStream) {
                 final String fileName = entry.getFileName().toString();
                 String version = fileName.substring(prefix.length() + 1);
diff --git a/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java b/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java
index 494aa7d1095d6..f0c7d4054a47e 100644
--- a/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java
+++ b/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java
@@ -32,8 +32,10 @@
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase;
 import org.elasticsearch.snapshots.RestoreInfo;
+import org.elasticsearch.snapshots.SnapshotException;
 import org.elasticsearch.snapshots.SnapshotInfo;
 import org.elasticsearch.snapshots.SnapshotRestoreException;
+import org.elasticsearch.snapshots.SnapshotState;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
 import org.elasticsearch.test.VersionUtils;
@@ -52,6 +54,7 @@
 import java.util.TreeSet;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
@@ -94,10 +97,10 @@ public void testRestoreOldSnapshots() throws Exception {
         }
 
         SortedSet<String> expectedVersions = new TreeSet<>();
-        for (Version v : VersionUtils.allVersions()) {
+        for (Version v : VersionUtils.allReleasedVersions()) {
             if (VersionUtils.isSnapshot(v)) continue;  // snapshots are unreleased, so there is no backcompat yet
-            if (v.isAlpha()) continue; // no guarantees for alpha releases
-            if (v.onOrBefore(Version.V_2_0_0_beta1)) continue; // we can only test back one major lucene version
+            if (v.isRelease() == false) continue; // no guarantees for prereleases
+            if (v.before(Version.CURRENT.minimumIndexCompatibilityVersion())) continue; // we only support versions N and N-1
             if (v.equals(Version.CURRENT)) continue; // the current version is always compatible with itself
             expectedVersions.add(v.toString());
         }
@@ -165,6 +168,32 @@ public void testRestoreSnapshotWithMissingChecksum() throws Exception {
         cluster().wipeTemplates();
     }
 
+    public void testSnapshotWithUnsupportedCompression() throws Exception {
+        final String repo = "test_repo";
+        final String repoFileId = "compressed-repo-1.7.4";
+        Path repoFile = getBwcIndicesPath().resolve(repoFileId + ".zip");
+        URI repoFileUri = repoFile.toUri();
+        URI repoJarUri = new URI("jar:" + repoFileUri.toString() + "!/repo/");
+        logger.info("-->  creating repository [{}] for repo file [{}]", repo, repoFileId);
+        assertAcked(client().admin().cluster().preparePutRepository(repo)
+                        .setType("url")
+                        .setSettings(Settings.builder().put("url", repoJarUri.toString()).put("compress", true)));
+
+        logger.info("--> get snapshots, only uncompressed one should be retrieved");
+        GetSnapshotsResponse getSnapshotsResponse = client().admin().cluster().prepareGetSnapshots(repo).setSnapshots("_all").get();
+        assertEquals(2, getSnapshotsResponse.getSnapshots().size());
+        for (SnapshotInfo snapshotInfo : getSnapshotsResponse.getSnapshots()) {
+            if (snapshotInfo.snapshotId().getName().equals("snap1x_uncompressed")) {
+                assertEquals(SnapshotState.SUCCESS, snapshotInfo.state());
+            } else {
+                assertEquals(SnapshotState.INCOMPATIBLE, snapshotInfo.state());
+            }
+        }
+        SnapshotException ex = expectThrows(SnapshotException.class, () ->
+            client().admin().cluster().prepareSnapshotStatus(repo).setSnapshots("snap1x").get());
+        assertThat(ex.getMessage(), containsString("cannot get the status for an incompatible snapshot"));
+    }
+
     private List<String> repoVersions() throws Exception {
         return listRepoVersions("repo");
     }
@@ -204,6 +233,14 @@ private void testOldSnapshot(String version, String repo, String snapshot) throw
         SnapshotInfo snapshotInfo = getSnapshotsResponse.getSnapshots().get(0);
         assertThat(snapshotInfo.version().toString(), equalTo(version));
 
+        logger.info("--> get less verbose snapshot info");
+        getSnapshotsResponse = client().admin().cluster().prepareGetSnapshots(repo)
+            .setSnapshots(snapshot).setVerbose(false).get();
+        assertEquals(1, getSnapshotsResponse.getSnapshots().size());
+        snapshotInfo = getSnapshotsResponse.getSnapshots().get(0);
+        assertEquals(snapshot, snapshotInfo.snapshotId().getName());
+        assertNull(snapshotInfo.version()); // in verbose=false mode, version doesn't exist
+
         logger.info("--> restoring snapshot");
         RestoreSnapshotResponse response = client().admin().cluster().prepareRestoreSnapshot(repo, snapshot).setRestoreGlobalState(true).setWaitForCompletion(true).get();
         assertThat(response.status(), equalTo(RestStatus.OK));
@@ -227,7 +264,15 @@ private void testOldSnapshot(String version, String repo, String snapshot) throw
         assertThat(template.template(), equalTo("te*"));
         assertThat(template.settings().getAsInt(IndexMetaData.SETTING_NUMBER_OF_SHARDS, -1), equalTo(1));
         assertThat(template.mappings().size(), equalTo(1));
-        assertThat(template.mappings().get("type1").string(), equalTo("{\"type1\":{\"_source\":{\"enabled\":false}}}"));
+        assertThat(template.mappings().get("type1").string(),
+            anyOf(
+                equalTo("{\"type1\":{\"_source\":{\"enabled\":false}}}"),
+                equalTo("{\"type1\":{\"_source\":{\"enabled\":\"false\"}}}"),
+                equalTo("{\"type1\":{\"_source\":{\"enabled\":\"0\"}}}"),
+                equalTo("{\"type1\":{\"_source\":{\"enabled\":0}}}"),
+                equalTo("{\"type1\":{\"_source\":{\"enabled\":\"off\"}}}"),
+                equalTo("{\"type1\":{\"_source\":{\"enabled\":\"no\"}}}")
+        ));
         assertThat(template.aliases().size(), equalTo(3));
         assertThat(template.aliases().get("alias1"), notNullValue());
         assertThat(template.aliases().get("alias2").filter().string(), containsString(version));
@@ -244,7 +289,7 @@ private void assertUnsupportedIndexFailsToRestore(String repo, String snapshot)
         logger.info("--> restoring unsupported snapshot");
         try {
             client().admin().cluster().prepareRestoreSnapshot(repo, snapshot).setRestoreGlobalState(true).setWaitForCompletion(true).get();
-            fail("should have failed to restore");
+            fail("should have failed to restore - " + repo);
         } catch (SnapshotRestoreException ex) {
             assertThat(ex.getMessage(), containsString("cannot restore index"));
             assertThat(ex.getMessage(), containsString("because it cannot be upgraded"));
diff --git a/core/src/test/java/org/elasticsearch/cli/CommandTests.java b/core/src/test/java/org/elasticsearch/cli/CommandTests.java
index 0da487e7b9d63..e3c5c254d32c2 100644
--- a/core/src/test/java/org/elasticsearch/cli/CommandTests.java
+++ b/core/src/test/java/org/elasticsearch/cli/CommandTests.java
@@ -26,30 +26,49 @@
 public class CommandTests extends ESTestCase {
 
     static class UserErrorCommand extends Command {
+
         UserErrorCommand() {
             super("Throws a user error");
         }
+
         @Override
         protected void execute(Terminal terminal, OptionSet options) throws Exception {
             throw new UserException(ExitCodes.DATA_ERROR, "Bad input");
         }
+
+        @Override
+        protected boolean addShutdownHook() {
+            return false;
+        }
+
     }
 
     static class UsageErrorCommand extends Command {
+
         UsageErrorCommand() {
             super("Throws a usage error");
         }
+
         @Override
         protected void execute(Terminal terminal, OptionSet options) throws Exception {
             throw new UserException(ExitCodes.USAGE, "something was no good");
         }
+
+        @Override
+        protected boolean addShutdownHook() {
+            return false;
+        }
+
     }
 
     static class NoopCommand extends Command {
+
         boolean executed = false;
+
         NoopCommand() {
             super("Does nothing");
         }
+
         @Override
         protected void execute(Terminal terminal, OptionSet options) throws Exception {
             terminal.println("Normal output");
@@ -57,10 +76,17 @@ protected void execute(Terminal terminal, OptionSet options) throws Exception {
             terminal.println(Terminal.Verbosity.VERBOSE, "Verbose output");
             executed = true;
         }
+
         @Override
         protected void printAdditionalHelp(Terminal terminal) {
             terminal.println("Some extra help");
         }
+
+        @Override
+        protected boolean addShutdownHook() {
+            return false;
+        }
+
     }
 
     public void testHelp() throws Exception {
@@ -92,7 +118,7 @@ public void testVerbositySilentAndVerbose() throws Exception {
             command.mainWithoutErrorHandling(args, terminal);
         });
         assertTrue(e.getMessage(),
-                e.getMessage().contains("Option(s) [v/verbose] are unavailable given other options on the command line"));
+            e.getMessage().contains("Option(s) [v/verbose] are unavailable given other options on the command line"));
     }
 
     public void testSilentVerbosity() throws Exception {
@@ -143,4 +169,5 @@ public void testUsageError() throws Exception {
         assertTrue(output, output.contains("Throws a usage error"));
         assertTrue(output, output.contains("ERROR: something was no good"));
     }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/cli/TerminalTests.java b/core/src/test/java/org/elasticsearch/cli/TerminalTests.java
index 6673bdbc858aa..3b409c2add636 100644
--- a/core/src/test/java/org/elasticsearch/cli/TerminalTests.java
+++ b/core/src/test/java/org/elasticsearch/cli/TerminalTests.java
@@ -46,6 +46,35 @@ public void testEscaping() throws Exception {
         assertPrinted(terminal, Terminal.Verbosity.NORMAL, "This message contains percent like %20n");
     }
 
+    public void testPromptYesNoDefault() throws Exception {
+        MockTerminal terminal = new MockTerminal();
+        terminal.addTextInput("");
+        assertTrue(terminal.promptYesNo("Answer?", true));
+        terminal.addTextInput("");
+        assertFalse(terminal.promptYesNo("Answer?", false));
+        terminal.addTextInput(null);
+        assertFalse(terminal.promptYesNo("Answer?", false));
+    }
+
+    public void testPromptYesNoReprompt() throws Exception {
+        MockTerminal terminal = new MockTerminal();
+        terminal.addTextInput("blah");
+        terminal.addTextInput("y");
+        assertTrue(terminal.promptYesNo("Answer? [Y/n]\nDid not understand answer 'blah'\nAnswer? [Y/n]", true));
+    }
+
+    public void testPromptYesNoCase() throws Exception {
+        MockTerminal terminal = new MockTerminal();
+        terminal.addTextInput("Y");
+        assertTrue(terminal.promptYesNo("Answer?", false));
+        terminal.addTextInput("y");
+        assertTrue(terminal.promptYesNo("Answer?", false));
+        terminal.addTextInput("N");
+        assertFalse(terminal.promptYesNo("Answer?", true));
+        terminal.addTextInput("n");
+        assertFalse(terminal.promptYesNo("Answer?", true));
+    }
+
     private void assertPrinted(MockTerminal logTerminal, Terminal.Verbosity verbosity, String text) throws Exception {
         logTerminal.println(verbosity, text);
         String output = logTerminal.getOutput();
diff --git a/core/src/test/java/org/elasticsearch/client/AbstractClientHeadersTestCase.java b/core/src/test/java/org/elasticsearch/client/AbstractClientHeadersTestCase.java
index 276a43581a691..29ffb7f7711b0 100644
--- a/core/src/test/java/org/elasticsearch/client/AbstractClientHeadersTestCase.java
+++ b/core/src/test/java/org/elasticsearch/client/AbstractClientHeadersTestCase.java
@@ -36,11 +36,10 @@
 import org.elasticsearch.action.search.SearchAction;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.junit.After;
-import org.junit.Before;
 
 import java.util.HashMap;
 import java.util.Map;
@@ -73,8 +72,9 @@ public abstract class AbstractClientHeadersTestCase extends ESTestCase {
     protected ThreadPool threadPool;
     private Client client;
 
-    @Before
-    public void initClient() {
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
         Settings settings = Settings.builder()
                 .put(HEADER_SETTINGS)
                 .put("path.home", createTempDir().toString())
@@ -85,8 +85,10 @@ public void initClient() {
         client = buildClient(settings, ACTIONS);
     }
 
-    @After
-    public void cleanupClient() throws Exception {
+
+    @Override
+    public void tearDown() throws Exception {
+        super.tearDown();
         client.close();
         terminate(threadPool);
     }
@@ -106,7 +108,7 @@ public void testActions() {
         client.prepareSearch().execute().addListener(new AssertingActionListener<>(SearchAction.NAME, client.threadPool()));
         client.prepareDelete("idx", "type", "id").execute().addListener(new AssertingActionListener<>(DeleteAction.NAME, client.threadPool()));
         client.admin().cluster().prepareDeleteStoredScript("lang", "id").execute().addListener(new AssertingActionListener<>(DeleteStoredScriptAction.NAME, client.threadPool()));
-        client.prepareIndex("idx", "type", "id").setSource("source").execute().addListener(new AssertingActionListener<>(IndexAction.NAME, client.threadPool()));
+        client.prepareIndex("idx", "type", "id").setSource("source", XContentType.JSON).execute().addListener(new AssertingActionListener<>(IndexAction.NAME, client.threadPool()));
 
         // choosing arbitrary cluster admin actions to test
         client.admin().cluster().prepareClusterStats().execute().addListener(new AssertingActionListener<>(ClusterStatsAction.NAME, client.threadPool()));
@@ -121,7 +123,7 @@ public void testActions() {
     }
 
     public void testOverrideHeader() throws Exception {
-        String key1Val = randomAsciiOfLength(5);
+        String key1Val = randomAlphaOfLength(5);
         Map<String, String> expected = new HashMap<>();
         expected.put("key1", key1Val);
         expected.put("key2", "val 2");
diff --git a/core/src/test/java/org/elasticsearch/client/ParentTaskAssigningClientTests.java b/core/src/test/java/org/elasticsearch/client/ParentTaskAssigningClientTests.java
index 360137b89047d..ee577d4df2f46 100644
--- a/core/src/test/java/org/elasticsearch/client/ParentTaskAssigningClientTests.java
+++ b/core/src/test/java/org/elasticsearch/client/ParentTaskAssigningClientTests.java
@@ -33,12 +33,12 @@
 
 public class ParentTaskAssigningClientTests extends ESTestCase {
     public void testSetsParentId() {
-        TaskId[] parentTaskId = new TaskId[] {new TaskId(randomAsciiOfLength(3), randomLong())};
+        TaskId[] parentTaskId = new TaskId[] {new TaskId(randomAlphaOfLength(3), randomLong())};
 
         // This mock will do nothing but verify that parentTaskId is set on all requests sent to it.
         NoOpClient mock = new NoOpClient(getTestName()) {
             @Override
-            protected <     Request extends ActionRequest<Request>,
+            protected <     Request extends ActionRequest,
                             Response extends ActionResponse,
                             RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>
                         > void doExecute( Action<Request, Response, RequestBuilder> action, Request request,
diff --git a/core/src/test/java/org/elasticsearch/client/node/NodeClientHeadersTests.java b/core/src/test/java/org/elasticsearch/client/node/NodeClientHeadersTests.java
index 04f7b73b1f26f..5e5610d49e6d1 100644
--- a/core/src/test/java/org/elasticsearch/client/node/NodeClientHeadersTests.java
+++ b/core/src/test/java/org/elasticsearch/client/node/NodeClientHeadersTests.java
@@ -28,7 +28,6 @@
 import org.elasticsearch.client.AbstractClientHeadersTestCase;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskManager;
 import org.elasticsearch.threadpool.ThreadPool;
 
@@ -47,7 +46,7 @@ protected Client buildClient(Settings headersSettings, GenericAction[] testedAct
         Settings settings = HEADER_SETTINGS;
         Actions actions = new Actions(settings, threadPool, testedActions);
         NodeClient client = new NodeClient(settings, threadPool);
-        client.intialize(actions);
+        client.initialize(actions, () -> "test");
         return client;
     }
 
diff --git a/core/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java b/core/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java
index 9d2c176dffba6..9827d2162891c 100644
--- a/core/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java
+++ b/core/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java
@@ -19,34 +19,38 @@
 
 package org.elasticsearch.client.transport;
 
-import com.carrotsearch.randomizedtesting.generators.RandomInts;
-import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.cluster.node.liveness.LivenessResponse;
 import org.elasticsearch.action.admin.cluster.node.liveness.TransportLivenessAction;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
 import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.CheckedBiConsumer;
 import org.elasticsearch.common.component.Lifecycle;
 import org.elasticsearch.common.component.LifecycleListener;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.transport.ConnectTransportException;
+import org.elasticsearch.transport.ConnectionProfile;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportException;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportResponse;
 import org.elasticsearch.transport.TransportResponseHandler;
-import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.transport.TransportServiceAdapter;
 
 import java.io.IOException;
+import java.net.UnknownHostException;
 import java.util.Collections;
 import java.util.Map;
 import java.util.Random;
 import java.util.Set;
 import java.util.concurrent.CopyOnWriteArraySet;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
 
 abstract class FailAndRetryMockTransport<Response extends TransportResponse> implements Transport {
 
@@ -61,47 +65,78 @@ abstract class FailAndRetryMockTransport<Response extends TransportResponse> imp
     private final AtomicInteger failures = new AtomicInteger();
     private final AtomicInteger successes = new AtomicInteger();
     private final Set<DiscoveryNode> triedNodes = new CopyOnWriteArraySet<>();
+    private final AtomicLong requestId = new AtomicLong();
 
     FailAndRetryMockTransport(Random random, ClusterName clusterName) {
         this.random = new Random(random.nextLong());
         this.clusterName = clusterName;
     }
 
+    protected abstract ClusterState getMockClusterState(DiscoveryNode node);
+
     @Override
-    @SuppressWarnings("unchecked")
-    public void sendRequest(DiscoveryNode node, long requestId, String action, TransportRequest request, TransportRequestOptions options)
-        throws IOException, TransportException {
-
-        //we make sure that nodes get added to the connected ones when calling addTransportAddress, by returning proper nodes info
-        if (connectMode) {
-            TransportResponseHandler transportResponseHandler = transportServiceAdapter.onResponseReceived(requestId);
-            transportResponseHandler.handleResponse(new LivenessResponse(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY),
-                node));
-            return;
-        }
-
-        //once nodes are connected we'll just return errors for each sendRequest call
-        triedNodes.add(node);
-
-        if (RandomInts.randomInt(random, 100) > 10) {
-            connectTransportExceptions.incrementAndGet();
-            throw new ConnectTransportException(node, "node not available");
-        } else {
-            if (random.nextBoolean()) {
-                failures.incrementAndGet();
-                //throw whatever exception that is not a subclass of ConnectTransportException
-                throw new IllegalStateException();
-            } else {
-                TransportResponseHandler transportResponseHandler = transportServiceAdapter.onResponseReceived(requestId);
-                if (random.nextBoolean()) {
-                    successes.incrementAndGet();
-                    transportResponseHandler.handleResponse(newResponse());
+    public Connection getConnection(DiscoveryNode node) {
+        return new Connection() {
+            @Override
+            public DiscoveryNode getNode() {
+                return node;
+            }
+
+            @Override
+            public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options)
+                throws IOException, TransportException {
+
+                //we make sure that nodes get added to the connected ones when calling addTransportAddress, by returning proper nodes info
+                if (connectMode) {
+                    if (TransportLivenessAction.NAME.equals(action)) {
+                        TransportResponseHandler transportResponseHandler = transportServiceAdapter.onResponseReceived(requestId);
+                        transportResponseHandler.handleResponse(new LivenessResponse(ClusterName.CLUSTER_NAME_SETTING.
+                            getDefault(Settings.EMPTY),
+                            node));
+                    } else if (ClusterStateAction.NAME.equals(action)) {
+                        TransportResponseHandler transportResponseHandler = transportServiceAdapter.onResponseReceived(requestId);
+                        ClusterState clusterState = getMockClusterState(node);
+                        transportResponseHandler.handleResponse(new ClusterStateResponse(clusterName, clusterState));
+                    } else {
+                        throw new UnsupportedOperationException("Mock transport does not understand action " + action);
+                    }
+                    return;
+                }
+
+                //once nodes are connected we'll just return errors for each sendRequest call
+                triedNodes.add(node);
+
+                if (random.nextInt(100) > 10) {
+                    connectTransportExceptions.incrementAndGet();
+                    throw new ConnectTransportException(node, "node not available");
                 } else {
-                    failures.incrementAndGet();
-                    transportResponseHandler.handleException(new TransportException("transport exception"));
+                    if (random.nextBoolean()) {
+                        failures.incrementAndGet();
+                        //throw whatever exception that is not a subclass of ConnectTransportException
+                        throw new IllegalStateException();
+                    } else {
+                        TransportResponseHandler transportResponseHandler = transportServiceAdapter.onResponseReceived(requestId);
+                        if (random.nextBoolean()) {
+                            successes.incrementAndGet();
+                            transportResponseHandler.handleResponse(newResponse());
+                        } else {
+                            failures.incrementAndGet();
+                            transportResponseHandler.handleException(new TransportException("transport exception"));
+                        }
+                    }
                 }
             }
-        }
+
+            @Override
+            public void close() throws IOException {
+
+            }
+        };
+    }
+
+    @Override
+    public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) throws IOException {
+        return getConnection(node);
     }
 
     protected abstract Response newResponse();
@@ -137,7 +172,7 @@ public BoundTransportAddress boundAddress() {
     }
 
     @Override
-    public TransportAddress[] addressesFromString(String address, int perAddressLimit) throws Exception {
+    public TransportAddress[] addressesFromString(String address, int perAddressLimit) throws UnknownHostException {
         throw new UnsupportedOperationException();
     }
 
@@ -152,12 +187,9 @@ public boolean nodeConnected(DiscoveryNode node) {
     }
 
     @Override
-    public void connectToNode(DiscoveryNode node) throws ConnectTransportException {
-
-    }
-
-    @Override
-    public void connectToNodeLight(DiscoveryNode node) throws ConnectTransportException {
+    public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile,
+                              CheckedBiConsumer<Connection, ConnectionProfile, IOException> connectionValidator)
+        throws ConnectTransportException {
 
     }
 
@@ -199,4 +231,9 @@ public void close() {}
     public Map<String, BoundTransportAddress> profileBoundAddresses() {
         return Collections.emptyMap();
     }
+
+    @Override
+    public long newRequestId() {
+        return requestId.incrementAndGet();
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/client/transport/NodeDisconnectIT.java b/core/src/test/java/org/elasticsearch/client/transport/NodeDisconnectIT.java
new file mode 100644
index 0000000000000..b91e36b632f74
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/client/transport/NodeDisconnectIT.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.transport;
+
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.transport.MockTransportClient;
+import org.elasticsearch.transport.TransportService;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.client.transport.TransportClient.CLIENT_TRANSPORT_NODES_SAMPLER_INTERVAL;
+
+public class NodeDisconnectIT  extends ESIntegTestCase {
+
+    public void testNotifyOnDisconnect() throws IOException {
+        internalCluster().ensureAtLeastNumDataNodes(2);
+
+        final Set<DiscoveryNode> disconnectedNodes = Collections.synchronizedSet(new HashSet<>());
+        try (TransportClient client = new MockTransportClient(Settings.builder()
+            .put("cluster.name", internalCluster().getClusterName())
+            .put(CLIENT_TRANSPORT_NODES_SAMPLER_INTERVAL.getKey(), "1h") // disable sniffing for better control
+            .build(),
+            Collections.emptySet(), (n, e) -> disconnectedNodes.add(n))) {
+            for (TransportService service : internalCluster().getInstances(TransportService.class)) {
+                client.addTransportAddress(service.boundAddress().publishAddress());
+            }
+            internalCluster().stopRandomDataNode();
+            for (int i = 0; i < 20; i++) { // fire up requests such that we hit the node and pass it to the listener
+                client.admin().cluster().prepareState().get();
+            }
+            assertEquals(1, disconnectedNodes.size());
+        }
+        assertEquals(1, disconnectedNodes.size());
+    }
+
+    public void testNotifyOnDisconnectInSniffer() throws IOException {
+        internalCluster().ensureAtLeastNumDataNodes(2);
+
+        final Set<DiscoveryNode> disconnectedNodes = Collections.synchronizedSet(new HashSet<>());
+        try (TransportClient client = new MockTransportClient(Settings.builder()
+            .put("cluster.name", internalCluster().getClusterName()).build(), Collections.emptySet(), (n, e) -> disconnectedNodes.add(n))) {
+            int numNodes = 0;
+            for (TransportService service : internalCluster().getInstances(TransportService.class)) {
+                numNodes++;
+                client.addTransportAddress(service.boundAddress().publishAddress());
+            }
+            Set<TransportAddress> discoveryNodes = client.connectedNodes().stream().map(n -> n.getAddress()).collect(Collectors.toSet());
+            assertEquals(numNodes, discoveryNodes.size());
+            assertEquals(0, disconnectedNodes.size());
+            internalCluster().stopRandomDataNode();
+            client.getNodesService().doSample();
+            assertEquals(1, disconnectedNodes.size());
+            assertTrue(discoveryNodes.contains(disconnectedNodes.stream().findAny().get().getAddress()));
+        }
+        assertEquals(1, disconnectedNodes.size());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/client/transport/TransportClientHeadersTests.java b/core/src/test/java/org/elasticsearch/client/transport/TransportClientHeadersTests.java
index e736e4b86a1cf..b7bbf1009d632 100644
--- a/core/src/test/java/org/elasticsearch/client/transport/TransportClientHeadersTests.java
+++ b/core/src/test/java/org/elasticsearch/client/transport/TransportClientHeadersTests.java
@@ -31,48 +31,58 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.plugins.NetworkPlugin;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.PluginsService;
+import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.ConnectTransportException;
 import org.elasticsearch.transport.MockTransportClient;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportException;
+import org.elasticsearch.transport.TransportInterceptor;
 import org.elasticsearch.transport.TransportRequest;
+import org.elasticsearch.transport.TransportRequestHandler;
 import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportResponse;
 import org.elasticsearch.transport.TransportResponseHandler;
 import org.elasticsearch.transport.TransportService;
 
 import java.util.Collections;
+import java.util.List;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.is;
-
-/**
- *
- */
 public class TransportClientHeadersTests extends AbstractClientHeadersTestCase {
 
-    private static final LocalTransportAddress address = new LocalTransportAddress("test");
+    private MockTransportService transportService;
+
+    @Override
+    public void tearDown() throws Exception {
+        super.tearDown();
+        transportService.stop();
+        transportService.close();
+    }
 
     @Override
     protected Client buildClient(Settings headersSettings, GenericAction[] testedActions) {
+        transportService = MockTransportService.local(Settings.EMPTY, Version.CURRENT, threadPool, null);
+        transportService.start();
+        transportService.acceptIncomingRequests();
         TransportClient client = new MockTransportClient(Settings.builder()
                 .put("client.transport.sniff", false)
                 .put("cluster.name", "cluster1")
                 .put("node.name", "transport_client_" + this.getTestName())
                 .put(headersSettings)
-                .build(), InternalTransportService.TestPlugin.class);
-
-        client.addTransportAddress(address);
+                .build(), InternalTransportServiceInterceptor.TestPlugin.class);
+        InternalTransportServiceInterceptor.TestPlugin plugin = client.injector.getInstance(PluginsService.class)
+            .filterPlugins(InternalTransportServiceInterceptor.TestPlugin.class).stream().findFirst().get();
+        plugin.instance.threadPool = client.threadPool();
+        plugin.instance.address = transportService.boundAddress().publishAddress();
+        client.addTransportAddress(transportService.boundAddress().publishAddress());
         return client;
     }
 
@@ -85,72 +95,83 @@ public void testWithSniffing() throws Exception {
                         .put("client.transport.nodes_sampler_interval", "1s")
                         .put(HEADER_SETTINGS)
                         .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build(),
-                InternalTransportService.TestPlugin.class)) {
-            client.addTransportAddress(address);
-
-            InternalTransportService service = (InternalTransportService) client.injector.getInstance(TransportService.class);
-
-            if (!service.clusterStateLatch.await(5, TimeUnit.SECONDS)) {
+                InternalTransportServiceInterceptor.TestPlugin.class)) {
+            InternalTransportServiceInterceptor.TestPlugin plugin = client.injector.getInstance(PluginsService.class)
+                .filterPlugins(InternalTransportServiceInterceptor.TestPlugin.class).stream().findFirst().get();
+            plugin.instance.threadPool = client.threadPool();
+            plugin.instance.address = transportService.boundAddress().publishAddress();
+            client.addTransportAddress(transportService.boundAddress().publishAddress());
+
+            if (!plugin.instance.clusterStateLatch.await(5, TimeUnit.SECONDS)) {
                 fail("takes way too long to get the cluster state");
             }
 
-            assertThat(client.connectedNodes().size(), is(1));
-            assertThat(client.connectedNodes().get(0).getAddress(), is((TransportAddress) address));
+            assertEquals(1, client.connectedNodes().size());
+            assertEquals(client.connectedNodes().get(0).getAddress(), transportService.boundAddress().publishAddress());
         }
     }
 
-    public static class InternalTransportService extends TransportService {
+    public static class InternalTransportServiceInterceptor implements TransportInterceptor {
 
-        public static class TestPlugin extends Plugin {
-            public void onModule(NetworkModule transportModule) {
-                transportModule.registerTransportService("internal", InternalTransportService.class);
-            }
-            @Override
-            public Settings additionalSettings() {
-                return Settings.builder().put(NetworkModule.TRANSPORT_SERVICE_TYPE_KEY, "internal").build();
-            }
-        }
+        ThreadPool threadPool;
+        TransportAddress address;
 
-        CountDownLatch clusterStateLatch = new CountDownLatch(1);
 
-        @Inject
-        public InternalTransportService(Settings settings, Transport transport, ThreadPool threadPool) {
-            super(settings, transport, threadPool);
-        }
+        public static class TestPlugin extends Plugin implements NetworkPlugin {
+            private InternalTransportServiceInterceptor instance = new InternalTransportServiceInterceptor();
 
-        @Override @SuppressWarnings("unchecked")
-        public <T extends TransportResponse> void sendRequest(DiscoveryNode node, String action, TransportRequest request,
-                                                              TransportRequestOptions options, TransportResponseHandler<T> handler) {
-            if (TransportLivenessAction.NAME.equals(action)) {
-                assertHeaders(threadPool);
-                ((TransportResponseHandler<LivenessResponse>) handler).handleResponse(new LivenessResponse(clusterName, node));
-                return;
-            }
-            if (ClusterStateAction.NAME.equals(action)) {
-                assertHeaders(threadPool);
-                ClusterName cluster1 = new ClusterName("cluster1");
-                ClusterState.Builder builder = ClusterState.builder(cluster1);
-                //the sniffer detects only data nodes
-                builder.nodes(DiscoveryNodes.builder().add(new DiscoveryNode("node_id", address, Collections.emptyMap(),
-                        Collections.singleton(DiscoveryNode.Role.DATA), Version.CURRENT)));
-                ((TransportResponseHandler<ClusterStateResponse>) handler)
-                        .handleResponse(new ClusterStateResponse(cluster1, builder.build()));
-                clusterStateLatch.countDown();
-                return;
+            @Override
+            public List<TransportInterceptor> getTransportInterceptors(ThreadContext threadContext) {
+                return Collections.singletonList(new TransportInterceptor() {
+                    @Override
+                    public <T extends TransportRequest> TransportRequestHandler<T> interceptHandler(String action, String executor,
+                                                                                                boolean forceExecution,
+                                                                                                TransportRequestHandler<T> actualHandler) {
+                        return instance.interceptHandler(action, executor, forceExecution, actualHandler);
+                    }
+
+                    @Override
+                    public AsyncSender interceptSender(AsyncSender sender) {
+                        return instance.interceptSender(sender);
+                    }
+                });
             }
-
-            handler.handleException(new TransportException("", new InternalException(action)));
         }
 
-        @Override
-        public boolean nodeConnected(DiscoveryNode node) {
-            assertThat(node.getAddress(), equalTo(address));
-            return true;
-        }
+        final CountDownLatch clusterStateLatch = new CountDownLatch(1);
 
         @Override
-        public void connectToNode(DiscoveryNode node) throws ConnectTransportException {
-            assertThat(node.getAddress(), equalTo(address));
+        public AsyncSender interceptSender(AsyncSender sender) {
+            return new AsyncSender() {
+                @Override
+                public <T extends TransportResponse> void sendRequest(Transport.Connection connection, String action,
+                                                                      TransportRequest request,
+                                                                      TransportRequestOptions options,
+                                                                      TransportResponseHandler<T> handler) {
+                    final ClusterName clusterName = new ClusterName("cluster1");
+                    if (TransportLivenessAction.NAME.equals(action)) {
+                        assertHeaders(threadPool);
+                        ((TransportResponseHandler<LivenessResponse>) handler).handleResponse(
+                            new LivenessResponse(clusterName, connection.getNode()));
+                    } else if (ClusterStateAction.NAME.equals(action)) {
+                        assertHeaders(threadPool);
+                        ClusterName cluster1 = clusterName;
+                        ClusterState.Builder builder = ClusterState.builder(cluster1);
+                        //the sniffer detects only data nodes
+                        builder.nodes(DiscoveryNodes.builder().add(new DiscoveryNode("node_id", "someId", "some_ephemeralId_id",
+                            address.getHost(), address.getAddress(), address, Collections.emptyMap(),
+                                Collections.singleton(DiscoveryNode.Role.DATA), Version.CURRENT)));
+                        ((TransportResponseHandler<ClusterStateResponse>) handler)
+                                .handleResponse(new ClusterStateResponse(cluster1, builder.build()));
+                        clusterStateLatch.countDown();
+                    } else if (TransportService.HANDSHAKE_ACTION_NAME .equals(action)) {
+                        ((TransportResponseHandler<TransportService.HandshakeResponse>) handler).handleResponse(
+                            new TransportService.HandshakeResponse(connection.getNode(), clusterName, connection.getNode().getVersion()));
+                    } else {
+                        handler.handleException(new TransportException("", new InternalException(action)));
+                    }
+                }
+            };
         }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/client/transport/TransportClientIT.java b/core/src/test/java/org/elasticsearch/client/transport/TransportClientIT.java
index 9b5b764b88e0b..143c0ff942114 100644
--- a/core/src/test/java/org/elasticsearch/client/transport/TransportClientIT.java
+++ b/core/src/test/java/org/elasticsearch/client/transport/TransportClientIT.java
@@ -19,6 +19,9 @@
 
 package org.elasticsearch.client.transport;
 
+import java.io.IOException;
+import java.util.Collections;
+
 import org.elasticsearch.Version;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -26,15 +29,16 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.node.MockNode;
 import org.elasticsearch.node.Node;
+import org.elasticsearch.node.NodeValidationException;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
+import org.elasticsearch.test.discovery.TestZenDiscovery;
 import org.elasticsearch.transport.MockTransportClient;
 import org.elasticsearch.transport.TransportService;
 
-import java.io.IOException;
-
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.is;
@@ -51,9 +55,9 @@ public void testPickingUpChangesInDiscoveryNode() {
 
     }
 
-    public void testNodeVersionIsUpdated() throws IOException {
+    public void testNodeVersionIsUpdated() throws IOException, NodeValidationException {
         TransportClient client = (TransportClient)  internalCluster().client();
-        try (Node node = new Node(Settings.builder()
+        try (Node node = new MockNode(Settings.builder()
                 .put(internalCluster().getDefaultSettings())
                 .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
                 .put("node.name", "testNodeVersionIsUpdated")
@@ -61,7 +65,7 @@ public void testNodeVersionIsUpdated() throws IOException {
                 .put(NetworkModule.HTTP_ENABLED.getKey(), false)
                 .put(Node.NODE_DATA_SETTING.getKey(), false)
                 .put("cluster.name", "foobar")
-                .build()).start()) {
+                .build(), Collections.singletonList(TestZenDiscovery.TestPlugin.class)).start()) {
             TransportAddress transportAddress = node.injector().getInstance(TransportService.class).boundAddress().publishAddress();
             client.addTransportAddress(transportAddress);
             // since we force transport clients there has to be one node started that we connect to.
diff --git a/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java b/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java
index 41891c5831fdf..f562d3514e57e 100644
--- a/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java
@@ -19,34 +19,59 @@
 
 package org.elasticsearch.client.transport;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.cluster.node.liveness.LivenessResponse;
 import org.elasticsearch.action.admin.cluster.node.liveness.TransportLivenessAction;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
 import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.LocalTransportAddress;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.node.Node;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.TransportResponseHandler;
+import org.elasticsearch.transport.ConnectionProfile;
+import org.elasticsearch.transport.Transport;
+import org.elasticsearch.transport.TransportChannel;
 import org.elasticsearch.transport.TransportException;
+import org.elasticsearch.transport.TransportInterceptor;
 import org.elasticsearch.transport.TransportRequest;
+import org.elasticsearch.transport.TransportRequestHandler;
 import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportResponse;
 import org.elasticsearch.transport.TransportResponseHandler;
 import org.elasticsearch.transport.TransportService;
+import org.hamcrest.CustomMatcher;
 
 import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 
+import static org.elasticsearch.test.transport.MockTransportService.createNewService;
 import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.everyItem;
 import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.hamcrest.CoreMatchers.not;
 import static org.hamcrest.CoreMatchers.nullValue;
 import static org.hamcrest.CoreMatchers.startsWith;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
@@ -59,11 +84,43 @@ private static class TestIteration implements Closeable {
         private final FailAndRetryMockTransport<TestResponse> transport;
         private final TransportService transportService;
         private final TransportClientNodesService transportClientNodesService;
-        private final int nodesCount;
-
-        TestIteration() {
-            Settings settings = Settings.builder().put("cluster.name", "test").build();
+        private final int listNodesCount;
+        private final int sniffNodesCount;
+        final List<TransportAddress> listNodeAddresses;
+        // map for each address of the nodes a cluster state request should respond with
+        final Map<TransportAddress, DiscoveryNodes> nodeMap;
+
+        TestIteration(Object... extraSettings) {
+            Settings settings = Settings.builder().put(extraSettings).put("cluster.name", "test").build();
             ClusterName clusterName = ClusterName.CLUSTER_NAME_SETTING.get(settings);
+            List<TransportAddress> listNodes = new ArrayList<>();
+            Map<TransportAddress, DiscoveryNodes> nodeMap = new HashMap<>();
+            int counter = 0;
+            this.listNodesCount = randomIntBetween(1, 10);
+            int sniffNodesCount = 0;
+            for (int i = 0; i < listNodesCount; i++) {
+                TransportAddress transportAddress = new LocalTransportAddress("node" + counter++);
+                listNodes.add(transportAddress);
+                DiscoveryNodes.Builder discoNodes = DiscoveryNodes.builder();
+                discoNodes.add(new DiscoveryNode("#list-node#-" + transportAddress, transportAddress, Version.CURRENT));
+
+                if (TransportClient.CLIENT_TRANSPORT_SNIFF.get(settings)) {
+                    final int numSniffNodes = randomIntBetween(0, 3);
+                    for (int j = 0; j < numSniffNodes; ++j) {
+                        TransportAddress sniffAddress = new LocalTransportAddress("node" + counter++);
+                        DiscoveryNode sniffNode = new DiscoveryNode("#sniff-node#-" + sniffAddress, sniffAddress, Version.CURRENT);
+                        discoNodes.add(sniffNode);
+                        // also allow sniffing of the sniff node itself
+                        nodeMap.put(sniffAddress, DiscoveryNodes.builder().add(sniffNode).build());
+                        ++sniffNodesCount;
+                    }
+                }
+                nodeMap.put(transportAddress, discoNodes.build());
+            }
+            listNodeAddresses = listNodes;
+            this.nodeMap = nodeMap;
+            this.sniffNodesCount = sniffNodesCount;
+
             threadPool = new TestThreadPool("transport-client-nodes-service-tests");
             transport = new FailAndRetryMockTransport<TestResponse>(random(), clusterName) {
                 @Override
@@ -73,40 +130,41 @@ public List<String> getLocalAddresses() {
 
                 @Override
                 protected TestResponse newResponse() {
-                    return  new TestResponse();
-                }
-            };
-            transportService = new TransportService(settings, transport, threadPool) {
-                @Override
-                public <T extends TransportResponse> void sendRequest(DiscoveryNode node, String action,
-                                                                      TransportRequest request, final TransportResponseHandler<T> handler) {
-                    if (TransportLivenessAction.NAME.equals(action)) {
-                        super.sendRequest(node, action, request, wrapLivenessResponseHandler(handler, node, clusterName));
-                    } else {
-                        super.sendRequest(node, action, request, handler);
-                    }
+                    return new TestResponse();
                 }
 
                 @Override
-                public <T extends TransportResponse> void sendRequest(DiscoveryNode node, String action, TransportRequest request,
-                                                                      TransportRequestOptions options,
-                                                                      TransportResponseHandler<T> handler) {
-                    if (TransportLivenessAction.NAME.equals(action)) {
-                        super.sendRequest(node, action, request, options, wrapLivenessResponseHandler(handler, node, clusterName));
-                    } else {
-                        super.sendRequest(node, action, request, options, handler);
-                    }
+                protected ClusterState getMockClusterState(DiscoveryNode node) {
+                    return ClusterState.builder(clusterName).nodes(TestIteration.this.nodeMap.get(node.getAddress())).build();
                 }
             };
+            transportService = new TransportService(settings, transport, threadPool, new TransportInterceptor() {
+                @Override
+                public AsyncSender interceptSender(AsyncSender sender) {
+                    return new AsyncSender() {
+                        @Override
+                        public <T extends TransportResponse> void sendRequest(Transport.Connection connection, String action,
+                                                                              TransportRequest request,
+                                                                              TransportRequestOptions options,
+                                                                              TransportResponseHandler<T> handler) {
+                            if (TransportLivenessAction.NAME.equals(action)) {
+                                sender.sendRequest(connection, action, request, options, wrapLivenessResponseHandler(handler,
+                                    connection.getNode(), clusterName));
+                            } else {
+                                sender.sendRequest(connection, action, request, options, handler);
+                            }
+                        }
+                    };
+                }
+            }, (addr) -> {
+                assert addr == null : "boundAddress: " + addr;
+                return DiscoveryNode.createLocal(settings, LocalTransportAddress.buildUnique(), UUIDs.randomBase64UUID());
+            }, null);
             transportService.start();
             transportService.acceptIncomingRequests();
             transportClientNodesService =
-                    new TransportClientNodesService(settings, transportService, threadPool);
-            this.nodesCount = randomIntBetween(1, 10);
-            for (int i = 0; i < nodesCount; i++) {
-                transportClientNodesService.addTransportAddresses(new LocalTransportAddress("node" + i));
-            }
-            transport.endConnectMode();
+                new TransportClientNodesService(settings, transportService, threadPool, (a, b) -> {});
+            transportClientNodesService.addTransportAddresses(listNodeAddresses.toArray(new TransportAddress[0]));
         }
 
         private <T extends TransportResponse> TransportResponseHandler wrapLivenessResponseHandler(TransportResponseHandler<T> handler,
@@ -143,7 +201,7 @@ public String executor() {
 
         @Override
         public void close() {
-
+            transport.endConnectMode();
             transportService.stop();
             transportClientNodesService.close();
             try {
@@ -157,7 +215,8 @@ public void close() {
     public void testListenerFailures() throws InterruptedException {
         int iters = iterations(10, 100);
         for (int i = 0; i <iters; i++) {
-            try(final TestIteration iteration = new TestIteration()) {
+            try(TestIteration iteration = new TestIteration()) {
+                iteration.transport.endConnectMode(); // stop transport from responding early
                 final CountDownLatch latch = new CountDownLatch(1);
                 final AtomicInteger finalFailures = new AtomicInteger();
                 final AtomicReference<Throwable> finalFailure = new AtomicReference<>();
@@ -210,7 +269,7 @@ public String executor() {
                     });
                 }, actionListener);
 
-                assertThat(latch.await(1, TimeUnit.SECONDS), equalTo(true));
+                latch.await();
 
                 //there can be only either one failure that causes the request to fail straightaway or success
                 assertThat(preSendFailures.get() + iteration.transport.failures() + iteration.transport.successes(), lessThanOrEqualTo(1));
@@ -228,7 +287,7 @@ public String executor() {
                     }
                 }
 
-                assertThat(iteration.transport.triedNodes().size(), lessThanOrEqualTo(iteration.nodesCount));
+                assertThat(iteration.transport.triedNodes().size(), lessThanOrEqualTo(iteration.listNodesCount));
                 assertThat(iteration.transport.triedNodes().size(), equalTo(iteration.transport.connectTransportExceptions() +
                         iteration.transport.failures() + iteration.transport.successes()));
             }
@@ -238,8 +297,8 @@ public String executor() {
     public void testConnectedNodes() {
         int iters = iterations(10, 100);
         for (int i = 0; i <iters; i++) {
-            try(final TestIteration iteration = new TestIteration()) {
-                assertThat(iteration.transportClientNodesService.connectedNodes().size(), lessThanOrEqualTo(iteration.nodesCount));
+            try(TestIteration iteration = new TestIteration()) {
+                assertThat(iteration.transportClientNodesService.connectedNodes().size(), lessThanOrEqualTo(iteration.listNodesCount));
                 for (DiscoveryNode discoveryNode : iteration.transportClientNodesService.connectedNodes()) {
                     assertThat(discoveryNode.getHostName(), startsWith("liveness-"));
                     assertThat(discoveryNode.getHostAddress(), startsWith("liveness-"));
@@ -252,6 +311,182 @@ public void testConnectedNodes() {
         }
     }
 
+    public void testRemoveAddressSniff() {
+        checkRemoveAddress(true);
+    }
+
+    public void testRemoveAddressSimple() {
+        checkRemoveAddress(false);
+    }
+
+    private void checkRemoveAddress(boolean sniff) {
+        Object[] extraSettings = {TransportClient.CLIENT_TRANSPORT_SNIFF.getKey(), sniff};
+        try(TestIteration iteration = new TestIteration(extraSettings)) {
+            final TransportClientNodesService service = iteration.transportClientNodesService;
+            assertEquals(iteration.listNodesCount + iteration.sniffNodesCount, service.connectedNodes().size());
+            final TransportAddress addressToRemove = randomFrom(iteration.listNodeAddresses);
+            service.removeTransportAddress(addressToRemove);
+            assertThat(service.connectedNodes(), everyItem(not(new CustomMatcher<DiscoveryNode>("removed address") {
+                @Override
+                public boolean matches(Object item) {
+                    return item instanceof DiscoveryNode && ((DiscoveryNode)item).getAddress().equals(addressToRemove);
+                }
+            })));
+            assertEquals(iteration.listNodesCount + iteration.sniffNodesCount - 1, service.connectedNodes().size());
+        }
+    }
+
+    public void testSniffNodesSamplerClosesConnections() throws Exception {
+        final TestThreadPool threadPool = new TestThreadPool("testSniffNodesSamplerClosesConnections");
+
+        Settings remoteSettings = Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), "remote").build();
+        try (MockTransportService remoteService = createNewService(remoteSettings, Version.CURRENT, threadPool, null)) {
+            final MockHandler handler = new MockHandler(remoteService);
+            remoteService.registerRequestHandler(ClusterStateAction.NAME, ClusterStateRequest::new, ThreadPool.Names.SAME, handler);
+            remoteService.start();
+            remoteService.acceptIncomingRequests();
+
+            Settings clientSettings = Settings.builder()
+                    .put(TransportClient.CLIENT_TRANSPORT_SNIFF.getKey(), true)
+                    .put(TransportClient.CLIENT_TRANSPORT_PING_TIMEOUT.getKey(), TimeValue.timeValueSeconds(1))
+                    .put(TransportClient.CLIENT_TRANSPORT_NODES_SAMPLER_INTERVAL.getKey(), TimeValue.timeValueSeconds(30))
+                    .build();
+
+            try (MockTransportService clientService = createNewService(clientSettings, Version.CURRENT, threadPool, null)) {
+                final List<MockConnection> establishedConnections = new CopyOnWriteArrayList<>();
+                final List<MockConnection> reusedConnections = new CopyOnWriteArrayList<>();
+
+                clientService.addDelegate(remoteService,  new MockTransportService.DelegateTransport(clientService.original()) {
+                    @Override
+                    public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) throws IOException {
+                        MockConnection connection = new MockConnection(super.openConnection(node, profile));
+                        establishedConnections.add(connection);
+                        return connection;
+                    }
+
+                    @Override
+                    public Connection getConnection(DiscoveryNode node) {
+                        MockConnection connection = new MockConnection(super.getConnection(node));
+                        reusedConnections.add(connection);
+                        return connection;
+                    }
+                });
+
+                clientService.start();
+                clientService.acceptIncomingRequests();
+
+                try (TransportClientNodesService transportClientNodesService =
+                        new TransportClientNodesService(clientSettings, clientService, threadPool, (a, b) -> {})) {
+                    assertEquals(0, transportClientNodesService.connectedNodes().size());
+                    assertEquals(0, establishedConnections.size());
+                    assertEquals(0, reusedConnections.size());
+
+                    transportClientNodesService.addTransportAddresses(remoteService.getLocalDiscoNode().getAddress());
+                    assertEquals(1, transportClientNodesService.connectedNodes().size());
+                    assertClosedConnections(establishedConnections, 1);
+
+                    transportClientNodesService.doSample();
+                    assertClosedConnections(establishedConnections, 2);
+                    assertOpenConnections(reusedConnections, 1);
+
+                    handler.blockRequest();
+                    Thread thread = new Thread(transportClientNodesService::doSample);
+                    thread.start();
+
+                    assertBusy(() ->  assertEquals(3, establishedConnections.size()));
+                    assertFalse("Temporary ping connection must be opened", establishedConnections.get(2).isClosed());
+
+                    handler.releaseRequest();
+                    thread.join();
+
+                    assertClosedConnections(establishedConnections, 3);
+                }
+            }
+        } finally {
+            terminate(threadPool);
+        }
+    }
+
+    private void assertClosedConnections(final List<MockConnection> connections, final int size) {
+        assertEquals("Expecting " + size + " closed connections but got " + connections.size(), size, connections.size());
+        connections.forEach(c -> assertConnection(c, true));
+    }
+
+    private void assertOpenConnections(final List<MockConnection> connections, final int size) {
+        assertEquals("Expecting " + size + " open connections but got " + connections.size(), size, connections.size());
+        connections.forEach(c -> assertConnection(c, false));
+    }
+
+    private static void assertConnection(final MockConnection connection, final boolean closed) {
+        assertEquals("Connection [" + connection + "] must be " + (closed ? "closed" : "open"), closed, connection.isClosed());
+    }
+
+    class MockConnection implements Transport.Connection {
+        private final AtomicBoolean closed = new AtomicBoolean(false);
+        private final Transport.Connection connection;
+
+        private MockConnection(Transport.Connection connection) {
+            this.connection = connection;
+        }
+
+        @Override
+        public DiscoveryNode getNode() {
+            return connection.getNode();
+        }
+
+        @Override
+        public Version getVersion() {
+            return connection.getVersion();
+        }
+
+        @Override
+        public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options)
+                throws IOException, TransportException {
+            connection.sendRequest(requestId, action, request, options);
+        }
+
+        @Override
+        public void close() throws IOException {
+            if (closed.compareAndSet(false, true)) {
+                connection.close();
+            }
+        }
+
+        boolean isClosed() {
+            return closed.get();
+        }
+    }
+
+    class MockHandler implements TransportRequestHandler<ClusterStateRequest> {
+        private final AtomicBoolean block = new AtomicBoolean(false);
+        private final CountDownLatch release = new CountDownLatch(1);
+        private final MockTransportService transportService;
+
+        MockHandler(MockTransportService transportService) {
+            this.transportService = transportService;
+        }
+
+        @Override
+        public void messageReceived(ClusterStateRequest request, TransportChannel channel) throws Exception {
+            if (block.get()) {
+                release.await();
+                return;
+            }
+            DiscoveryNodes discoveryNodes = DiscoveryNodes.builder().add(transportService.getLocalDiscoNode()).build();
+            ClusterState build = ClusterState.builder(ClusterName.DEFAULT).nodes(discoveryNodes).build();
+            channel.sendResponse(new ClusterStateResponse(ClusterName.DEFAULT, build));
+        }
+
+        void blockRequest() {
+            if (block.compareAndSet(false, true) == false) {
+                throw new AssertionError("Request handler is already marked as blocking");
+            }
+        }
+        void releaseRequest() {
+            release.countDown();
+        }
+    }
+
     public static class TestRequest extends TransportRequest {
 
     }
diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java b/core/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java
index 6326d96f31765..71f877953dd5f 100644
--- a/core/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.cluster;
 
 import com.carrotsearch.hppc.cursors.ObjectCursor;
+import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.metadata.IndexGraveyard;
@@ -34,6 +35,7 @@
 import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.TestCustomMetaData;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -223,6 +225,112 @@ public void testRoutingTableChanges() {
         assertTrue("index routing table should not be the same object", event.indexRoutingTableChanged(initialIndices.get(0).getName()));
     }
 
+    /**
+     * Test custom metadata change checks
+     */
+    public void testChangedCustomMetaDataSet() {
+        final int numNodesInCluster = 3;
+
+        final ClusterState originalState = createState(numNodesInCluster, randomBoolean(), initialIndices);
+        CustomMetaData1 customMetaData1 = new CustomMetaData1("data");
+        final ClusterState stateWithCustomMetaData = nextState(originalState, Collections.singletonList(customMetaData1));
+
+        // no custom metadata present in any state
+        ClusterState nextState = ClusterState.builder(originalState).build();
+        ClusterChangedEvent event = new ClusterChangedEvent("_na_", originalState, nextState);
+        assertTrue(event.changedCustomMetaDataSet().isEmpty());
+
+        // next state has new custom metadata
+        nextState = nextState(originalState, Collections.singletonList(customMetaData1));
+        event = new ClusterChangedEvent("_na_", originalState, nextState);
+        Set<String> changedCustomMetaDataTypeSet = event.changedCustomMetaDataSet();
+        assertTrue(changedCustomMetaDataTypeSet.size() == 1);
+        assertTrue(changedCustomMetaDataTypeSet.contains(customMetaData1.getWriteableName()));
+
+        // next state has same custom metadata
+        nextState = nextState(originalState, Collections.singletonList(customMetaData1));
+        event = new ClusterChangedEvent("_na_", stateWithCustomMetaData, nextState);
+        changedCustomMetaDataTypeSet = event.changedCustomMetaDataSet();
+        assertTrue(changedCustomMetaDataTypeSet.isEmpty());
+
+        // next state has equivalent custom metadata
+        nextState = nextState(originalState, Collections.singletonList(new CustomMetaData1("data")));
+        event = new ClusterChangedEvent("_na_", stateWithCustomMetaData, nextState);
+        changedCustomMetaDataTypeSet = event.changedCustomMetaDataSet();
+        assertTrue(changedCustomMetaDataTypeSet.isEmpty());
+
+        // next state removes custom metadata
+        nextState = originalState;
+        event = new ClusterChangedEvent("_na_", stateWithCustomMetaData, nextState);
+        changedCustomMetaDataTypeSet = event.changedCustomMetaDataSet();
+        assertTrue(changedCustomMetaDataTypeSet.size() == 1);
+        assertTrue(changedCustomMetaDataTypeSet.contains(customMetaData1.getWriteableName()));
+
+        // next state updates custom metadata
+        nextState = nextState(stateWithCustomMetaData, Collections.singletonList(new CustomMetaData1("data1")));
+        event = new ClusterChangedEvent("_na_", stateWithCustomMetaData, nextState);
+        changedCustomMetaDataTypeSet = event.changedCustomMetaDataSet();
+        assertTrue(changedCustomMetaDataTypeSet.size() == 1);
+        assertTrue(changedCustomMetaDataTypeSet.contains(customMetaData1.getWriteableName()));
+
+        // next state adds new custom metadata type
+        CustomMetaData2 customMetaData2 = new CustomMetaData2("data2");
+        nextState = nextState(stateWithCustomMetaData, Arrays.asList(customMetaData1, customMetaData2));
+        event = new ClusterChangedEvent("_na_", stateWithCustomMetaData, nextState);
+        changedCustomMetaDataTypeSet = event.changedCustomMetaDataSet();
+        assertTrue(changedCustomMetaDataTypeSet.size() == 1);
+        assertTrue(changedCustomMetaDataTypeSet.contains(customMetaData2.getWriteableName()));
+
+        // next state adds two custom metadata type
+        nextState = nextState(originalState, Arrays.asList(customMetaData1, customMetaData2));
+        event = new ClusterChangedEvent("_na_", originalState, nextState);
+        changedCustomMetaDataTypeSet = event.changedCustomMetaDataSet();
+        assertTrue(changedCustomMetaDataTypeSet.size() == 2);
+        assertTrue(changedCustomMetaDataTypeSet.contains(customMetaData2.getWriteableName()));
+        assertTrue(changedCustomMetaDataTypeSet.contains(customMetaData1.getWriteableName()));
+
+        // next state removes two custom metadata type
+        nextState = originalState;
+        event = new ClusterChangedEvent("_na_",
+                nextState(originalState, Arrays.asList(customMetaData1, customMetaData2)), nextState);
+        changedCustomMetaDataTypeSet = event.changedCustomMetaDataSet();
+        assertTrue(changedCustomMetaDataTypeSet.size() == 2);
+        assertTrue(changedCustomMetaDataTypeSet.contains(customMetaData2.getWriteableName()));
+        assertTrue(changedCustomMetaDataTypeSet.contains(customMetaData1.getWriteableName()));
+    }
+
+    private static class CustomMetaData2 extends TestCustomMetaData {
+        protected CustomMetaData2(String data) {
+            super(data);
+        }
+
+        @Override
+        public String getWriteableName() {
+            return "2";
+        }
+
+        @Override
+        public EnumSet<MetaData.XContentContext> context() {
+            return EnumSet.of(MetaData.XContentContext.GATEWAY);
+        }
+    }
+
+    private static class CustomMetaData1 extends TestCustomMetaData {
+        protected CustomMetaData1(String data) {
+            super(data);
+        }
+
+        @Override
+        public String getWriteableName() {
+            return "1";
+        }
+
+        @Override
+        public EnumSet<MetaData.XContentContext> context() {
+            return EnumSet.of(MetaData.XContentContext.GATEWAY);
+        }
+    }
+
     private static ClusterState createSimpleClusterState() {
         return ClusterState.builder(TEST_CLUSTER_NAME).build();
     }
@@ -245,6 +353,22 @@ private static ClusterState createNonInitializedState(final int numNodes, final
                            .build();
     }
 
+    private static ClusterState nextState(final ClusterState previousState, List<TestCustomMetaData> customMetaDataList) {
+        final ClusterState.Builder builder = ClusterState.builder(previousState);
+        builder.stateUUID(UUIDs.randomBase64UUID());
+        MetaData.Builder metaDataBuilder = new MetaData.Builder(previousState.metaData());
+        for (ObjectObjectCursor<String, MetaData.Custom> customMetaData : previousState.metaData().customs()) {
+            if (customMetaData.value instanceof TestCustomMetaData) {
+                metaDataBuilder.removeCustom(customMetaData.key);
+            }
+        }
+        for (TestCustomMetaData testCustomMetaData : customMetaDataList) {
+            metaDataBuilder.putCustom(testCustomMetaData.getWriteableName(), testCustomMetaData);
+        }
+        builder.metaData(metaDataBuilder);
+        return builder.build();
+    }
+
     // Create a modified cluster state from another one, but with some number of indices added and deleted.
     private static ClusterState nextState(final ClusterState previousState, final boolean changeClusterUUID,
                                           final List<Index> addedIndices, final List<Index> deletedIndices, final int numNodesToRemove) {
@@ -406,7 +530,7 @@ private static ClusterState executeIndicesChangesTest(final ClusterState previou
             default: throw new AssertionError("Unhandled mode [" + deletionQuantity + "]");
         }
         final boolean changeClusterUUID = randomBoolean();
-        final List<Index> addedIndices = addIndices(numAdd, randomAsciiOfLengthBetween(5, 10));
+        final List<Index> addedIndices = addIndices(numAdd, randomAlphaOfLengthBetween(5, 10));
         List<Index> delIndices;
         if (changeClusterUUID) {
             delIndices = new ArrayList<>();
diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java b/core/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java
index 3cafff08a07a5..637b6733e228c 100644
--- a/core/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java
@@ -48,6 +48,8 @@
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.InternalTestCluster;
 import org.elasticsearch.test.transport.MockTransportService;
+import org.elasticsearch.transport.ConnectionProfile;
+import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportException;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.transport.TransportRequestOptions;
@@ -93,18 +95,13 @@ public BlockingActionFilter(Settings settings) {
         }
 
         @Override
-        protected boolean apply(String action, ActionRequest<?> request, ActionListener<?> listener) {
+        protected boolean apply(String action, ActionRequest request, ActionListener<?> listener) {
             if (blockedActions.contains(action)) {
                 throw new ElasticsearchException("force exception on [" + action + "]");
             }
             return true;
         }
 
-        @Override
-        protected boolean apply(String action, ActionResponse response, ActionListener<?> listener) {
-            return true;
-        }
-
         @Override
         public int order() {
             return 0;
@@ -131,7 +128,7 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
     }
 
     public void testClusterInfoServiceCollectsInformation() throws Exception {
-        internalCluster().startNodesAsync(2).get();
+        internalCluster().startNodes(2);
         assertAcked(prepareCreate("test").setSettings(Settings.builder()
                 .put(Store.INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING.getKey(), 0)
                 .put(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), EnableAllocationDecider.Rebalance.NONE).build()));
@@ -179,10 +176,9 @@ public void testClusterInfoServiceCollectsInformation() throws Exception {
     }
 
     public void testClusterInfoServiceInformationClearOnError() throws InterruptedException, ExecutionException {
-        internalCluster().startNodesAsync(2,
+        internalCluster().startNodes(2,
                 // manually control publishing
-                Settings.builder().put(InternalClusterInfoService.INTERNAL_CLUSTER_INFO_UPDATE_INTERVAL_SETTING.getKey(), "60m").build())
-                .get();
+                Settings.builder().put(InternalClusterInfoService.INTERNAL_CLUSTER_INFO_UPDATE_INTERVAL_SETTING.getKey(), "60m").build());
         prepareCreate("test").setSettings(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1).get();
         ensureGreen("test");
         InternalTestCluster internalTestCluster = internalCluster();
@@ -202,15 +198,15 @@ public void testClusterInfoServiceInformationClearOnError() throws InterruptedEx
         for (DiscoveryNode node : internalTestCluster.clusterService().state().getNodes()) {
             mockTransportService.addDelegate(internalTestCluster.getInstance(TransportService.class, node.getName()), new MockTransportService.DelegateTransport(mockTransportService.original()) {
                 @Override
-                public void sendRequest(DiscoveryNode node, long requestId, String action, TransportRequest request,
-                                        TransportRequestOptions options) throws IOException, TransportException {
+                protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request,
+                                           TransportRequestOptions options) throws IOException {
                     if (blockedActions.contains(action)) {
                         if (timeout.get()) {
                             logger.info("dropping [{}] to [{}]", action, node);
                             return;
                         }
                     }
-                    super.sendRequest(node, requestId, action, request, options);
+                    super.sendRequest(connection, requestId, action, request, options);
                 }
             });
         }
diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterInfoTests.java b/core/src/test/java/org/elasticsearch/cluster/ClusterInfoTests.java
index 1fcab355d65ac..35a7ea2aab80c 100644
--- a/core/src/test/java/org/elasticsearch/cluster/ClusterInfoTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/ClusterInfoTests.java
@@ -46,9 +46,9 @@ private static ImmutableOpenMap<String, DiskUsage> randomDiskUsage() {
         int numEntries = randomIntBetween(0, 128);
         ImmutableOpenMap.Builder<String, DiskUsage> builder = ImmutableOpenMap.builder(numEntries);
         for (int i = 0; i < numEntries; i++) {
-            String key = randomAsciiOfLength(32);
+            String key = randomAlphaOfLength(32);
             DiskUsage diskUsage = new DiskUsage(
-                    randomAsciiOfLength(4), randomAsciiOfLength(4), randomAsciiOfLength(4),
+                    randomAlphaOfLength(4), randomAlphaOfLength(4), randomAlphaOfLength(4),
                     randomIntBetween(0, Integer.MAX_VALUE), randomIntBetween(0, Integer.MAX_VALUE)
             );
             builder.put(key, diskUsage);
@@ -60,7 +60,7 @@ private static ImmutableOpenMap<String, Long> randomShardSizes() {
         int numEntries = randomIntBetween(0, 128);
         ImmutableOpenMap.Builder<String, Long> builder = ImmutableOpenMap.builder(numEntries);
         for (int i = 0; i < numEntries; i++) {
-            String key = randomAsciiOfLength(32);
+            String key = randomAlphaOfLength(32);
             long shardSize = randomIntBetween(0, Integer.MAX_VALUE);
             builder.put(key, shardSize);
         }
@@ -71,9 +71,9 @@ private static ImmutableOpenMap<ShardRouting, String> randomRoutingToDataPath()
         int numEntries = randomIntBetween(0, 128);
         ImmutableOpenMap.Builder<ShardRouting, String> builder = ImmutableOpenMap.builder(numEntries);
         for (int i = 0; i < numEntries; i++) {
-            ShardId shardId = new ShardId(randomAsciiOfLength(32), randomAsciiOfLength(32), randomIntBetween(0, Integer.MAX_VALUE));
+            ShardId shardId = new ShardId(randomAlphaOfLength(32), randomAlphaOfLength(32), randomIntBetween(0, Integer.MAX_VALUE));
             ShardRouting shardRouting = TestShardRouting.newShardRouting(shardId, null, randomBoolean(), ShardRoutingState.UNASSIGNED);
-            builder.put(shardRouting, randomAsciiOfLength(32));
+            builder.put(shardRouting, randomAlphaOfLength(32));
         }
         return builder.build();
     }
diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java b/core/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java
index 5c710ec92daf2..d753c2b9c0209 100644
--- a/core/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java
@@ -19,18 +19,29 @@
 
 package org.elasticsearch.cluster;
 
-import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest;
-import org.elasticsearch.cluster.metadata.IndexTemplateFilter;
-import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
+import org.elasticsearch.Version;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
-import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
+import org.elasticsearch.cluster.routing.allocation.ShardAllocationDecision;
 import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocator;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.decider.AwarenessAllocationDecider;
+import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider;
+import org.elasticsearch.cluster.routing.allocation.decider.ConcurrentRebalanceAllocationDecider;
+import org.elasticsearch.cluster.routing.allocation.decider.DiskThresholdDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
+import org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider;
+import org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider;
+import org.elasticsearch.cluster.routing.allocation.decider.NodeVersionAllocationDecider;
+import org.elasticsearch.cluster.routing.allocation.decider.RebalanceOnlyWhenActiveAllocationDecider;
+import org.elasticsearch.cluster.routing.allocation.decider.ReplicaAfterPrimaryActiveAllocationDecider;
+import org.elasticsearch.cluster.routing.allocation.decider.SameShardAllocationDecider;
+import org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider;
+import org.elasticsearch.cluster.routing.allocation.decider.SnapshotInProgressAllocationDecider;
+import org.elasticsearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.inject.ModuleTestCase;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.IndexScopedSettings;
@@ -38,17 +49,21 @@
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
+import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.plugins.ClusterPlugin;
 
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 import java.util.function.Supplier;
 
 public class ClusterModuleTests extends ModuleTestCase {
     private ClusterService clusterService = new ClusterService(Settings.EMPTY,
-        new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null);
+        new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null, () ->
+        new DiscoveryNode(UUIDs.randomBase64UUID(), LocalTransportAddress.buildUnique(), Version.CURRENT));
     static class FakeAllocationDecider extends AllocationDecider {
         protected FakeAllocationDecider(Settings settings) {
             super(settings);
@@ -60,10 +75,9 @@ static class FakeShardsAllocator implements ShardsAllocator {
         public void allocate(RoutingAllocation allocation) {
             // noop
         }
-
         @Override
-        public Map<DiscoveryNode, Float> weighShard(RoutingAllocation allocation, ShardRouting shard) {
-            return new HashMap<>();
+        public ShardAllocationDecision decideShardAllocation(ShardRouting shard, RoutingAllocation allocation) {
+            throw new UnsupportedOperationException("explain API not supported on FakeShardsAllocator");
         }
     }
 
@@ -79,7 +93,7 @@ public void testRegisterClusterDynamicSettingDuplicate() {
     public void testRegisterClusterDynamicSetting() {
         SettingsModule module = new SettingsModule(Settings.EMPTY,
             Setting.boolSetting("foo.bar", false, Property.Dynamic, Property.NodeScope));
-        assertInstanceBinding(module, ClusterSettings.class, service -> service.hasDynamicSetting("foo.bar"));
+        assertInstanceBinding(module, ClusterSettings.class, service -> service.isDynamicSetting("foo.bar"));
     }
 
     public void testRegisterIndexDynamicSettingDuplicate() {
@@ -94,7 +108,7 @@ public void testRegisterIndexDynamicSettingDuplicate() {
     public void testRegisterIndexDynamicSetting() {
         SettingsModule module = new SettingsModule(Settings.EMPTY,
             Setting.boolSetting("index.foo.bar", false, Property.Dynamic, Property.IndexScope));
-        assertInstanceBinding(module, IndexScopedSettings.class, service -> service.hasDynamicSetting("index.foo.bar"));
+        assertInstanceBinding(module, IndexScopedSettings.class, service -> service.isDynamicSetting("index.foo.bar"));
     }
 
     public void testRegisterAllocationDeciderDuplicate() {
@@ -156,4 +170,34 @@ public void testShardsAllocatorFactoryNull() {
         NullPointerException e = expectThrows(NullPointerException.class, () ->
             newClusterModuleWithShardsAllocator(settings, "bad", () -> null));
     }
+
+    // makes sure that the allocation deciders are setup in the correct order, such that the
+    // slower allocation deciders come last and we can exit early if there is a NO decision without
+    // running them. If the order of the deciders is changed for a valid reason, the order should be
+    // changed in the test too.
+    public void testAllocationDeciderOrder() {
+        List<Class<? extends AllocationDecider>> expectedDeciders = Arrays.asList(
+            MaxRetryAllocationDecider.class,
+            ReplicaAfterPrimaryActiveAllocationDecider.class,
+            RebalanceOnlyWhenActiveAllocationDecider.class,
+            ClusterRebalanceAllocationDecider.class,
+            ConcurrentRebalanceAllocationDecider.class,
+            EnableAllocationDecider.class,
+            NodeVersionAllocationDecider.class,
+            SnapshotInProgressAllocationDecider.class,
+            FilterAllocationDecider.class,
+            SameShardAllocationDecider.class,
+            DiskThresholdDecider.class,
+            ThrottlingAllocationDecider.class,
+            ShardsLimitAllocationDecider.class,
+            AwarenessAllocationDecider.class);
+        Collection<AllocationDecider> deciders = ClusterModule.createAllocationDeciders(Settings.EMPTY,
+            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), Collections.emptyList());
+        Iterator<AllocationDecider> iter = deciders.iterator();
+        int idx = 0;
+        while (iter.hasNext()) {
+            AllocationDecider decider = iter.next();
+            assertSame(decider.getClass(), expectedDeciders.get(idx++));
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java b/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java
index db59e7858716b..3d59e383501cd 100644
--- a/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java
@@ -31,6 +31,9 @@
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.metadata.RepositoriesMetaData;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -57,6 +60,7 @@
 
 import java.util.Collections;
 import java.util.List;
+import java.util.Set;
 
 import static java.util.Collections.emptyList;
 import static java.util.Collections.emptyMap;
@@ -74,13 +78,15 @@
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, numDataNodes = 0, numClientNodes = 0)
 public class ClusterStateDiffIT extends ESIntegTestCase {
     public void testClusterStateDiffSerialization() throws Exception {
+        NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(ClusterModule.getNamedWriteables());
         DiscoveryNode masterNode = new DiscoveryNode("master", new LocalTransportAddress("master"),
                 emptyMap(), emptySet(), Version.CURRENT);
         DiscoveryNode otherNode = new DiscoveryNode("other", new LocalTransportAddress("other"),
                 emptyMap(), emptySet(), Version.CURRENT);
         DiscoveryNodes discoveryNodes = DiscoveryNodes.builder().add(masterNode).add(otherNode).localNodeId(masterNode.getId()).build();
         ClusterState clusterState = ClusterState.builder(new ClusterName("test")).nodes(discoveryNodes).build();
-        ClusterState clusterStateFromDiffs = ClusterState.Builder.fromBytes(ClusterState.Builder.toBytes(clusterState), otherNode);
+        ClusterState clusterStateFromDiffs =
+            ClusterState.Builder.fromBytes(ClusterState.Builder.toBytes(clusterState), otherNode, namedWriteableRegistry);
 
         int iterationCount = randomIntBetween(10, 300);
         for (int iteration = 0; iteration < iterationCount; iteration++) {
@@ -116,7 +122,8 @@ public void testClusterStateDiffSerialization() throws Exception {
 
             if (randomIntBetween(0, 10) < 1) {
                 // Update cluster state via full serialization from time to time
-                clusterStateFromDiffs = ClusterState.Builder.fromBytes(ClusterState.Builder.toBytes(clusterState), previousClusterStateFromDiffs.nodes().getLocalNode());
+                clusterStateFromDiffs = ClusterState.Builder.fromBytes(ClusterState.Builder.toBytes(clusterState),
+                    previousClusterStateFromDiffs.nodes().getLocalNode(), namedWriteableRegistry);
             } else {
                 // Update cluster states using diffs
                 Diff<ClusterState> diffBeforeSerialization = clusterState.diff(previousClusterState);
@@ -125,7 +132,8 @@ public void testClusterStateDiffSerialization() throws Exception {
                 byte[] diffBytes = BytesReference.toBytes(os.bytes());
                 Diff<ClusterState> diff;
                 try (StreamInput input = StreamInput.wrap(diffBytes)) {
-                    diff = previousClusterStateFromDiffs.readDiffFrom(input);
+                    StreamInput namedInput = new NamedWriteableAwareStreamInput(input, namedWriteableRegistry);
+                    diff = ClusterState.readDiffFrom(namedInput, previousClusterStateFromDiffs.nodes().getLocalNode());
                     clusterStateFromDiffs = diff.apply(previousClusterStateFromDiffs);
                 }
             }
@@ -193,14 +201,14 @@ private ClusterState.Builder randomNodes(ClusterState clusterState) {
             if (nodeId.startsWith("node-")) {
                 nodes.remove(nodeId);
                 if (randomBoolean()) {
-                    nodes.add(new DiscoveryNode(nodeId, new LocalTransportAddress(randomAsciiOfLength(10)), emptyMap(),
+                    nodes.add(new DiscoveryNode(nodeId, new LocalTransportAddress(randomAlphaOfLength(10)), emptyMap(),
                             emptySet(), randomVersion(random())));
                 }
             }
         }
         int additionalNodeCount = randomIntBetween(1, 20);
         for (int i = 0; i < additionalNodeCount; i++) {
-            nodes.add(new DiscoveryNode("node-" + randomAsciiOfLength(10), new LocalTransportAddress(randomAsciiOfLength(10)),
+            nodes.add(new DiscoveryNode("node-" + randomAlphaOfLength(10), new LocalTransportAddress(randomAlphaOfLength(10)),
                     emptyMap(), emptySet(), randomVersion(random())));
         }
         return ClusterState.builder(clusterState).nodes(nodes);
@@ -239,13 +247,19 @@ private IndexRoutingTable randomIndexRoutingTable(String index, String[] nodeIds
         for (int i = 0; i < shardCount; i++) {
             IndexShardRoutingTable.Builder indexShard = new IndexShardRoutingTable.Builder(new ShardId(index, "_na_", i));
             int replicaCount = randomIntBetween(1, 10);
+            Set<String> availableNodeIds = Sets.newHashSet(nodeIds);
             for (int j = 0; j < replicaCount; j++) {
                 UnassignedInfo unassignedInfo = null;
                 if (randomInt(5) == 1) {
-                    unassignedInfo = new UnassignedInfo(randomReason(), randomAsciiOfLength(10));
+                    unassignedInfo = new UnassignedInfo(randomReason(), randomAlphaOfLength(10));
                 }
+                if (availableNodeIds.isEmpty()) {
+                    break;
+                }
+                String nodeId = randomFrom(availableNodeIds);
+                availableNodeIds.remove(nodeId);
                 indexShard.addShard(
-                        TestShardRouting.newShardRouting(index, i, randomFrom(nodeIds), null, j == 0,
+                        TestShardRouting.newShardRouting(index, i, nodeId, null, j == 0,
                                 ShardRoutingState.fromValue((byte) randomIntBetween(2, 3)), unassignedInfo));
             }
             builder.addIndexShard(indexShard.build());
@@ -259,8 +273,20 @@ private IndexRoutingTable randomIndexRoutingTable(String index, String[] nodeIds
     private IndexRoutingTable randomChangeToIndexRoutingTable(IndexRoutingTable original, String[] nodes) {
         IndexRoutingTable.Builder builder = IndexRoutingTable.builder(original.getIndex());
         for (ObjectCursor<IndexShardRoutingTable> indexShardRoutingTable :  original.shards().values()) {
+            Set<String> availableNodes = Sets.newHashSet(nodes);
             for (ShardRouting shardRouting : indexShardRoutingTable.value.shards()) {
-                final ShardRouting updatedShardRouting = randomChange(shardRouting, nodes);
+                availableNodes.remove(shardRouting.currentNodeId());
+                if (shardRouting.relocating()) {
+                    availableNodes.remove(shardRouting.relocatingNodeId());
+                }
+            }
+
+            for (ShardRouting shardRouting : indexShardRoutingTable.value.shards()) {
+                final ShardRouting updatedShardRouting = randomChange(shardRouting, availableNodes);
+                availableNodes.remove(updatedShardRouting.currentNodeId());
+                if (shardRouting.relocating()) {
+                    availableNodes.remove(updatedShardRouting.relocatingNodeId());
+                }
                 builder.addShard(updatedShardRouting);
             }
         }
@@ -395,7 +421,7 @@ private Settings randomSettings(Settings settings) {
         }
         int settingsCount = randomInt(10);
         for (int i = 0; i < settingsCount; i++) {
-            builder.put(randomAsciiOfLength(10), randomAsciiOfLength(10));
+            builder.put(randomAlphaOfLength(10), randomAlphaOfLength(10));
         }
         return builder.build();
 
@@ -516,7 +542,7 @@ public IndexMetaData randomChange(IndexMetaData part) {
                         if (randomBoolean() && part.getAliases().isEmpty() == false) {
                             builder.removeAlias(randomFrom(part.getAliases().keys().toArray(String.class)));
                         } else {
-                            builder.putAlias(AliasMetaData.builder(randomAsciiOfLength(10)));
+                            builder.putAlias(AliasMetaData.builder(randomAlphaOfLength(10)));
                         }
                         break;
                     case 2:
@@ -581,7 +607,7 @@ private AliasMetaData randomAlias() {
             builder.filter(QueryBuilders.termQuery("test", randomRealisticUnicodeOfCodepointLength(10)).toString());
         }
         if (randomBoolean()) {
-            builder.routing(randomAsciiOfLength(10));
+            builder.routing(randomAlphaOfLength(10));
         }
         return builder.build();
     }
@@ -599,7 +625,7 @@ public ImmutableOpenMap<String, MetaData.Custom> parts(MetaData metaData) {
 
             @Override
             public MetaData.Builder put(MetaData.Builder builder, MetaData.Custom part) {
-                return builder.putCustom(part.type(), part);
+                return builder.putCustom(part.getWriteableName(), part);
             }
 
             @Override
@@ -641,7 +667,7 @@ public ImmutableOpenMap<String, ClusterState.Custom> parts(ClusterState clusterS
 
             @Override
             public ClusterState.Builder put(ClusterState.Builder builder, ClusterState.Custom part) {
-                return builder.putCustom(part.type(), part);
+                return builder.putCustom(part.getWriteableName(), part);
             }
 
             @Override
@@ -660,6 +686,7 @@ public ClusterState.Custom randomCreate(String name) {
                                 SnapshotsInProgress.State.fromValue((byte) randomIntBetween(0, 6)),
                                 Collections.emptyList(),
                                 Math.abs(randomLong()),
+                                (long) randomIntBetween(0, 1000),
                                 ImmutableOpenMap.of()));
                     case 1:
                         return new RestoreInProgress(new RestoreInProgress.Entry(
diff --git a/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java b/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java
index 201cdd7966e0e..b9e07a9207f93 100644
--- a/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java
@@ -26,7 +26,6 @@
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.RecoverySource.PeerRecoverySource;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingHelper;
@@ -269,8 +268,8 @@ private void assertDiskUsage(DiskUsage usage, FsInfo.Path path) {
         assertNotNull(usage);
         assertNotNull(path);
         assertEquals(usage.toString(), usage.getPath(), path.getPath());
-        assertEquals(usage.toString(), usage.getTotalBytes(), path.getTotal().bytes());
-        assertEquals(usage.toString(), usage.getFreeBytes(), path.getAvailable().bytes());
+        assertEquals(usage.toString(), usage.getTotalBytes(), path.getTotal().getBytes());
+        assertEquals(usage.toString(), usage.getFreeBytes(), path.getAvailable().getBytes());
 
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java b/core/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java
index 33141107b2e04..a847d342174ed 100644
--- a/core/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java
@@ -24,21 +24,18 @@
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.discovery.Discovery;
 import org.elasticsearch.discovery.DiscoverySettings;
+import org.elasticsearch.discovery.zen.ElectMasterService;
 import org.elasticsearch.discovery.zen.ZenDiscovery;
-import org.elasticsearch.discovery.zen.elect.ElectMasterService;
-import org.elasticsearch.discovery.zen.fd.FaultDetection;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
+import org.elasticsearch.test.discovery.TestZenDiscovery;
 import org.elasticsearch.test.disruption.NetworkDisruption;
-import org.elasticsearch.test.disruption.NetworkDisruption.NetworkDelay;
 import org.elasticsearch.test.disruption.NetworkDisruption.TwoPartitions;
 import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.test.transport.MockTransportService;
@@ -63,9 +60,9 @@
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.nullValue;
 
-@ClusterScope(scope = Scope.TEST, numDataNodes = 0)
+@ClusterScope(scope = Scope.TEST, numDataNodes = 0, autoMinMasterNodes = false)
 @ESIntegTestCase.SuppressLocalMode
-@TestLogging("_root:DEBUG,cluster.service:TRACE,discovery.zen:TRACE")
+@TestLogging("_root:DEBUG,org.elasticsearch.cluster.service:TRACE,org.elasticsearch.discovery.zen:TRACE")
 public class MinimumMasterNodesIT extends ESIntegTestCase {
 
     @Override
@@ -75,10 +72,15 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
         return classes;
     }
 
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        return Settings.builder().put(super.nodeSettings(nodeOrdinal))
+            .put(TestZenDiscovery.USE_MOCK_PINGS.getKey(), false).build();
+    }
+
     public void testSimpleMinimumMasterNodes() throws Exception {
 
         Settings settings = Settings.builder()
-                .put("discovery.type", "zen")
                 .put("discovery.zen.minimum_master_nodes", 2)
                 .put(ZenDiscovery.PING_TIMEOUT_SETTING.getKey(), "200ms")
                 .put("discovery.initial_state_timeout", "500ms")
@@ -130,7 +132,9 @@ public void testSimpleMinimumMasterNodes() throws Exception {
         });
         state = client().admin().cluster().prepareState().setLocal(true).execute().actionGet().getState();
         assertThat(state.blocks().hasGlobalBlock(DiscoverySettings.NO_MASTER_BLOCK_ID), equalTo(true));
-        assertThat(state.nodes().getSize(), equalTo(1)); // verify that we still see the local node in the cluster state
+        // verify that both nodes are still in the cluster state but there is no master
+        assertThat(state.nodes().getSize(), equalTo(2));
+        assertThat(state.nodes().getMasterNode(), equalTo(null));
 
         logger.info("--> starting the previous master node again...");
         internalCluster().startNode(settings);
@@ -155,12 +159,9 @@ public void testSimpleMinimumMasterNodes() throws Exception {
         }
 
         internalCluster().stopRandomNonMasterNode();
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                ClusterState state = client().admin().cluster().prepareState().setLocal(true).execute().actionGet().getState();
-                assertThat(state.blocks().hasGlobalBlock(DiscoverySettings.NO_MASTER_BLOCK_ID), equalTo(true));
-            }
+        assertBusy(() -> {
+            ClusterState state1 = client().admin().cluster().prepareState().setLocal(true).execute().actionGet().getState();
+            assertThat(state1.blocks().hasGlobalBlock(DiscoverySettings.NO_MASTER_BLOCK_ID), equalTo(true));
         });
 
         logger.info("--> starting the previous master node again...");
@@ -190,29 +191,25 @@ public void run() {
 
     public void testMultipleNodesShutdownNonMasterNodes() throws Exception {
         Settings settings = Settings.builder()
-                .put("discovery.type", "zen")
                 .put("discovery.zen.minimum_master_nodes", 3)
                 .put(ZenDiscovery.PING_TIMEOUT_SETTING.getKey(), "1s")
                 .put("discovery.initial_state_timeout", "500ms")
                 .build();
 
         logger.info("--> start first 2 nodes");
-        internalCluster().startNodesAsync(2, settings).get();
+        internalCluster().startNodes(2, settings);
 
         ClusterState state;
 
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                for (Client client : clients()) {
-                    ClusterState state = client.admin().cluster().prepareState().setLocal(true).execute().actionGet().getState();
-                    assertThat(state.blocks().hasGlobalBlock(DiscoverySettings.NO_MASTER_BLOCK_ID), equalTo(true));
-                }
+        assertBusy(() -> {
+            for (Client client : clients()) {
+                ClusterState state1 = client.admin().cluster().prepareState().setLocal(true).execute().actionGet().getState();
+                assertThat(state1.blocks().hasGlobalBlock(DiscoverySettings.NO_MASTER_BLOCK_ID), equalTo(true));
             }
         });
 
         logger.info("--> start two more nodes");
-        internalCluster().startNodesAsync(2, settings).get();
+        internalCluster().startNodes(2, settings);
 
         ensureGreen();
         ClusterHealthResponse clusterHealthResponse = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForNodes("4").execute().actionGet();
@@ -247,11 +244,10 @@ public void run() {
         assertNoMasterBlockOnAllNodes();
 
         logger.info("--> start back the 2 nodes ");
-        String[] newNodes = internalCluster().startNodesAsync(2, settings).get().toArray(Strings.EMPTY_ARRAY);
+        String[] newNodes = internalCluster().startNodes(2, settings).stream().toArray(String[]::new);
 
+        internalCluster().validateClusterFormed();
         ensureGreen();
-        clusterHealthResponse = client().admin().cluster().prepareHealth().setWaitForNodes("4").execute().actionGet();
-        assertThat(clusterHealthResponse.isTimedOut(), equalTo(false));
 
         state = client().admin().cluster().prepareState().execute().actionGet().getState();
         assertThat(state.nodes().getSize(), equalTo(4));
@@ -265,18 +261,18 @@ public void run() {
     }
 
     public void testDynamicUpdateMinimumMasterNodes() throws Exception {
-        Settings settings = Settings.builder()
-                .put("discovery.type", "zen")
-                .put(ZenDiscovery.PING_TIMEOUT_SETTING.getKey(), "400ms")
-                .put("discovery.initial_state_timeout", "500ms")
-                .build();
+        Settings settingsWithMinMaster1 = Settings.builder()
+            .put(ZenDiscovery.PING_TIMEOUT_SETTING.getKey(), "400ms")
+            .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), 1)
+            .build();
 
-        logger.info("--> start 2 nodes");
-        internalCluster().startNodesAsync(2, settings).get();
+        Settings settingsWithMinMaster2 = Settings.builder()
+            .put(settingsWithMinMaster1).put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), 2)
+            .build();
 
-        // wait until second node join the cluster
-        ClusterHealthResponse clusterHealthResponse = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForNodes("2").get();
-        assertThat(clusterHealthResponse.isTimedOut(), equalTo(false));
+        logger.info("--> start two nodes and wait for them to form a cluster");
+        internalCluster().startNodes(settingsWithMinMaster1, settingsWithMinMaster2);
+        ensureClusterSizeConsistency();
 
         logger.info("--> setting minimum master node to 2");
         setMinimumMasterNodes(2);
@@ -293,9 +289,8 @@ public void testDynamicUpdateMinimumMasterNodes() throws Exception {
         assertNoMasterBlockOnAllNodes();
 
         logger.info("--> bringing another node up");
-        internalCluster().startNode(Settings.builder().put(settings).put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), 2).build());
-        clusterHealthResponse = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForNodes("2").get();
-        assertThat(clusterHealthResponse.isTimedOut(), equalTo(false));
+        internalCluster().startNode(settingsWithMinMaster2);
+        ensureClusterSizeConsistency();
     }
 
     private void assertNoMasterBlockOnAllNodes() throws InterruptedException {
@@ -324,7 +319,6 @@ private void assertNoMasterBlockOnAllNodes() throws InterruptedException {
     public void testCanNotBringClusterDown() throws ExecutionException, InterruptedException {
         int nodeCount = scaledRandomIntBetween(1, 5);
         Settings.Builder settings = Settings.builder()
-                .put("discovery.type", "zen")
                 .put(ZenDiscovery.PING_TIMEOUT_SETTING.getKey(), "200ms")
                 .put("discovery.initial_state_timeout", "500ms");
 
@@ -334,7 +328,7 @@ public void testCanNotBringClusterDown() throws ExecutionException, InterruptedE
 
 
         logger.info("--> starting [{}] nodes. min_master_nodes set to [{}]", nodeCount, initialMinMasterNodes);
-        internalCluster().startNodesAsync(nodeCount, settings.build()).get();
+        internalCluster().startNodes(nodeCount, settings.build());
 
         logger.info("--> waiting for nodes to join");
         assertFalse(client().admin().cluster().prepareHealth().setWaitForNodes(Integer.toString(nodeCount)).get().isTimedOut());
@@ -363,13 +357,11 @@ public void testCanNotBringClusterDown() throws ExecutionException, InterruptedE
 
     public void testCanNotPublishWithoutMinMastNodes() throws Exception {
         Settings settings = Settings.builder()
-                .put("discovery.type", "zen")
-                .put(FaultDetection.PING_TIMEOUT_SETTING.getKey(), "1h") // disable it
                 .put(ZenDiscovery.PING_TIMEOUT_SETTING.getKey(), "200ms")
                 .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), 2)
                 .put(DiscoverySettings.COMMIT_TIMEOUT_SETTING.getKey(), "100ms") // speed things up
                 .build();
-        internalCluster().startNodesAsync(3, settings).get();
+        internalCluster().startNodes(3, settings);
         ensureGreen(); // ensure cluster state is recovered before we disrupt things
 
         final String master = internalCluster().getMasterName();
@@ -377,9 +369,8 @@ public void testCanNotPublishWithoutMinMastNodes() throws Exception {
         otherNodes.remove(master);
         NetworkDisruption partition = new NetworkDisruption(
             new TwoPartitions(Collections.singleton(master), otherNodes),
-            new NetworkDelay(TimeValue.timeValueMinutes(1)));
+            new NetworkDisruption.NetworkDisconnect());
         internalCluster().setDisruptionScheme(partition);
-        partition.startDisrupting();
 
         final CountDownLatch latch = new CountDownLatch(1);
         final AtomicReference<Exception> failure = new AtomicReference<>();
@@ -393,6 +384,8 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS
 
             @Override
             public ClusterState execute(ClusterState currentState) throws Exception {
+                logger.debug("--> starting the disruption, preventing cluster state publishing");
+                partition.startDisrupting();
                 MetaData.Builder metaData = MetaData.builder(currentState.metaData()).persistentSettings(
                         Settings.builder().put(currentState.metaData().persistentSettings()).put("_SHOULD_NOT_BE_THERE_", true).build()
                 );
@@ -410,12 +403,7 @@ public void onFailure(String source, Exception e) {
         latch.await();
 
         assertThat(failure.get(), instanceOf(Discovery.FailedToCommitClusterStateException.class));
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                assertThat(masterClusterService.state().nodes().getMasterNode(), nullValue());
-            }
-        });
+        assertBusy(() -> assertThat(masterClusterService.state().nodes().getMasterNode(), nullValue()));
 
         partition.stopDisrupting();
 
diff --git a/core/src/test/java/org/elasticsearch/cluster/NoMasterNodeIT.java b/core/src/test/java/org/elasticsearch/cluster/NoMasterNodeIT.java
index 84410a92c8328..09003adc6122f 100644
--- a/core/src/test/java/org/elasticsearch/cluster/NoMasterNodeIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/NoMasterNodeIT.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.action.bulk.BulkRequestBuilder;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.client.Requests;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
@@ -34,38 +35,33 @@
 import org.elasticsearch.discovery.zen.ZenDiscovery;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
 
+import java.util.Collections;
+
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertExists;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
-import static org.hamcrest.Matchers.lessThan;
 
-/**
- */
-@ClusterScope(scope = Scope.TEST, numDataNodes = 0)
+@ClusterScope(scope = Scope.TEST, numDataNodes = 0, autoMinMasterNodes = false)
 @ESIntegTestCase.SuppressLocalMode
 public class NoMasterNodeIT extends ESIntegTestCase {
     public void testNoMasterActions() throws Exception {
-        // note, sometimes, we want to check with the fact that an index gets created, sometimes not...
-        boolean autoCreateIndex = randomBoolean();
-        logger.info("auto_create_index set to {}", autoCreateIndex);
-
         Settings settings = Settings.builder()
                 .put("discovery.type", "zen")
-                .put("action.auto_create_index", autoCreateIndex)
+                .put("action.auto_create_index", true)
                 .put("discovery.zen.minimum_master_nodes", 2)
                 .put(ZenDiscovery.PING_TIMEOUT_SETTING.getKey(), "200ms")
                 .put("discovery.initial_state_timeout", "500ms")
                 .put(DiscoverySettings.NO_MASTER_BLOCK_SETTING.getKey(), "all")
                 .build();
 
-        TimeValue timeout = TimeValue.timeValueMillis(200);
+        final TimeValue timeout = TimeValue.timeValueMillis(10);
 
         internalCluster().startNode(settings);
         // start a second node, create an index, and then shut it down so we have no master block
@@ -73,12 +69,9 @@ public void testNoMasterActions() throws Exception {
         createIndex("test");
         client().admin().cluster().prepareHealth("test").setWaitForGreenStatus().execute().actionGet();
         internalCluster().stopRandomDataNode();
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                ClusterState state = client().admin().cluster().prepareState().setLocal(true).execute().actionGet().getState();
-                assertTrue(state.blocks().hasGlobalBlock(DiscoverySettings.NO_MASTER_BLOCK_ID));
-            }
+        assertBusy(() -> {
+            ClusterState state = client().admin().cluster().prepareState().setLocal(true).execute().actionGet().getState();
+            assertTrue(state.blocks().hasGlobalBlock(DiscoverySettings.NO_MASTER_BLOCK_ID));
         });
 
         assertThrows(client().prepareGet("test", "type1", "1"),
@@ -114,40 +107,42 @@ public void run() {
                 ClusterBlockException.class, RestStatus.SERVICE_UNAVAILABLE
         );
 
-        checkWriteAction(
-                false, timeout,
+        checkUpdateAction(false, timeout,
                 client().prepareUpdate("test", "type1", "1")
-                        .setScript(new Script("test script", ScriptService.ScriptType.INLINE, null, null)).setTimeout(timeout));
+                        .setScript(new Script(
+                            ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, "test script", Collections.emptyMap())).setTimeout(timeout));
 
-        checkWriteAction(
-                autoCreateIndex, timeout,
+        checkUpdateAction(
+                true, timeout,
                 client().prepareUpdate("no_index", "type1", "1")
-                        .setScript(new Script("test script", ScriptService.ScriptType.INLINE, null, null)).setTimeout(timeout));
+                        .setScript(new Script(
+                            ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, "test script", Collections.emptyMap())).setTimeout(timeout));
 
 
-        checkWriteAction(false, timeout,
-                client().prepareIndex("test", "type1", "1").setSource(XContentFactory.jsonBuilder().startObject().endObject()).setTimeout(timeout));
+        checkWriteAction(
+            client().prepareIndex("test", "type1", "1").setSource(XContentFactory.jsonBuilder().startObject().endObject()).setTimeout(timeout));
 
-        checkWriteAction(autoCreateIndex, timeout,
-                client().prepareIndex("no_index", "type1", "1").setSource(XContentFactory.jsonBuilder().startObject().endObject()).setTimeout(timeout));
+        checkWriteAction(
+            client().prepareIndex("no_index", "type1", "1").setSource(XContentFactory.jsonBuilder().startObject().endObject()).setTimeout(timeout));
 
         BulkRequestBuilder bulkRequestBuilder = client().prepareBulk();
         bulkRequestBuilder.add(client().prepareIndex("test", "type1", "1").setSource(XContentFactory.jsonBuilder().startObject().endObject()));
         bulkRequestBuilder.add(client().prepareIndex("test", "type1", "2").setSource(XContentFactory.jsonBuilder().startObject().endObject()));
-        checkBulkAction(false, bulkRequestBuilder);
+        bulkRequestBuilder.setTimeout(timeout);
+        checkWriteAction(bulkRequestBuilder);
 
         bulkRequestBuilder = client().prepareBulk();
         bulkRequestBuilder.add(client().prepareIndex("no_index", "type1", "1").setSource(XContentFactory.jsonBuilder().startObject().endObject()));
         bulkRequestBuilder.add(client().prepareIndex("no_index", "type1", "2").setSource(XContentFactory.jsonBuilder().startObject().endObject()));
-        checkBulkAction(autoCreateIndex, bulkRequestBuilder);
+        bulkRequestBuilder.setTimeout(timeout);
+        checkWriteAction(bulkRequestBuilder);
 
         internalCluster().startNode(settings);
         client().admin().cluster().prepareHealth().setWaitForGreenStatus().setWaitForNodes("2").execute().actionGet();
     }
 
-    void checkWriteAction(boolean autoCreateIndex, TimeValue timeout, ActionRequestBuilder<?, ?, ?> builder) {
+    void checkUpdateAction(boolean autoCreateIndex, TimeValue timeout, ActionRequestBuilder<?, ?, ?> builder) {
         // we clean the metadata when loosing a master, therefore all operations on indices will auto create it, if allowed
-        long now = System.currentTimeMillis();
         try {
             builder.get();
             fail("expected ClusterBlockException or MasterNotDiscoveredException");
@@ -157,37 +152,16 @@ void checkWriteAction(boolean autoCreateIndex, TimeValue timeout, ActionRequestB
             } else {
                 assertFalse(autoCreateIndex);
             }
-            // verify we waited before giving up...
             assertThat(e.status(), equalTo(RestStatus.SERVICE_UNAVAILABLE));
-            assertThat(System.currentTimeMillis() - now, greaterThan(timeout.millis() - 50));
         }
     }
 
-    void checkBulkAction(boolean indexShouldBeAutoCreated, BulkRequestBuilder builder) {
-        // bulk operation do not throw MasterNotDiscoveredException exceptions. The only test that auto create kicked in and failed is
-        // via the timeout, as bulk operation do not wait on blocks.
-        TimeValue timeout;
-        if (indexShouldBeAutoCreated) {
-            // we expect the bulk to fail because it will try to go to the master. Use small timeout and detect it has passed
-            timeout = new TimeValue(200);
-        } else {
-            // the request should fail very quickly - use a large timeout and make sure it didn't pass...
-            timeout = new TimeValue(5000);
-        }
-        builder.setTimeout(timeout);
-        long now = System.currentTimeMillis();
+    void checkWriteAction(ActionRequestBuilder<?, ?, ?> builder) {
         try {
             builder.get();
             fail("Expected ClusterBlockException");
         } catch (ClusterBlockException e) {
-            if (indexShouldBeAutoCreated) {
-                // timeout is 200
-                assertThat(System.currentTimeMillis() - now, greaterThan(timeout.millis() - 50));
-                assertThat(e.status(), equalTo(RestStatus.SERVICE_UNAVAILABLE));
-            } else {
-                // timeout is 5000
-                assertThat(System.currentTimeMillis() - now, lessThan(timeout.millis() - 50));
-            }
+            assertThat(e.status(), equalTo(RestStatus.SERVICE_UNAVAILABLE));
         }
     }
 
@@ -214,7 +188,7 @@ public void testNoMasterActionsWriteMasterBlock() throws Exception {
         ensureSearchable("test1", "test2");
 
         ClusterStateResponse clusterState = client().admin().cluster().prepareState().get();
-        logger.info("Cluster state:\n{}", clusterState.getState().prettyPrint());
+        logger.info("Cluster state:\n{}", clusterState.getState());
 
         internalCluster().stopRandomDataNode();
         assertTrue(awaitBusy(() -> {
@@ -239,19 +213,17 @@ public void testNoMasterActionsWriteMasterBlock() throws Exception {
         TimeValue timeout = TimeValue.timeValueMillis(200);
         long now = System.currentTimeMillis();
         try {
-            client().prepareUpdate("test1", "type1", "1").setDoc("field", "value2").setTimeout(timeout).get();
+            client().prepareUpdate("test1", "type1", "1").setDoc(Requests.INDEX_CONTENT_TYPE, "field", "value2").setTimeout(timeout).get();
             fail("Expected ClusterBlockException");
         } catch (ClusterBlockException e) {
             assertThat(System.currentTimeMillis() - now, greaterThan(timeout.millis() - 50));
             assertThat(e.status(), equalTo(RestStatus.SERVICE_UNAVAILABLE));
         }
 
-        now = System.currentTimeMillis();
         try {
             client().prepareIndex("test1", "type1", "1").setSource(XContentFactory.jsonBuilder().startObject().endObject()).setTimeout(timeout).get();
             fail("Expected ClusterBlockException");
         } catch (ClusterBlockException e) {
-            assertThat(System.currentTimeMillis() - now, greaterThan(timeout.millis() - 50));
             assertThat(e.status(), equalTo(RestStatus.SERVICE_UNAVAILABLE));
         }
 
diff --git a/core/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java
index ed7a20dc87e81..d44b027f8ca95 100644
--- a/core/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java
@@ -22,6 +22,8 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.CheckedBiConsumer;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.component.Lifecycle;
 import org.elasticsearch.common.component.LifecycleListener;
 import org.elasticsearch.common.settings.Settings;
@@ -30,8 +32,10 @@
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.ConnectTransportException;
+import org.elasticsearch.transport.ConnectionProfile;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportException;
 import org.elasticsearch.transport.TransportRequest;
@@ -39,10 +43,10 @@
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.transport.TransportServiceAdapter;
 import org.junit.After;
-import org.junit.AfterClass;
 import org.junit.Before;
 
 import java.io.IOException;
+import java.net.UnknownHostException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -51,12 +55,13 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
 
 import static org.hamcrest.Matchers.equalTo;
 
 public class NodeConnectionsServiceTests extends ESTestCase {
 
-    private static ThreadPool THREAD_POOL;
+    private ThreadPool threadPool;
     private MockTransport transport;
     private TransportService transportService;
 
@@ -80,38 +85,38 @@ private ClusterState clusterStateFromNodes(List<DiscoveryNode> nodes) {
 
     public void testConnectAndDisconnect() {
         List<DiscoveryNode> nodes = generateNodes();
-        NodeConnectionsService service = new NodeConnectionsService(Settings.EMPTY, THREAD_POOL, transportService);
+        NodeConnectionsService service = new NodeConnectionsService(Settings.EMPTY, threadPool, transportService);
 
         ClusterState current = clusterStateFromNodes(Collections.emptyList());
         ClusterChangedEvent event = new ClusterChangedEvent("test", clusterStateFromNodes(randomSubsetOf(nodes)), current);
 
-        service.connectToAddedNodes(event);
-        assertConnected(event.nodesDelta().addedNodes());
+        service.connectToNodes(event.state().nodes());
+        assertConnected(event.state().nodes());
 
-        service.disconnectFromRemovedNodes(event);
+        service.disconnectFromNodesExcept(event.state().nodes());
         assertConnectedExactlyToNodes(event.state());
 
         current = event.state();
         event = new ClusterChangedEvent("test", clusterStateFromNodes(randomSubsetOf(nodes)), current);
 
-        service.connectToAddedNodes(event);
-        assertConnected(event.nodesDelta().addedNodes());
+        service.connectToNodes(event.state().nodes());
+        assertConnected(event.state().nodes());
 
-        service.disconnectFromRemovedNodes(event);
+        service.disconnectFromNodesExcept(event.state().nodes());
         assertConnectedExactlyToNodes(event.state());
     }
 
 
     public void testReconnect() {
         List<DiscoveryNode> nodes = generateNodes();
-        NodeConnectionsService service = new NodeConnectionsService(Settings.EMPTY, THREAD_POOL, transportService);
+        NodeConnectionsService service = new NodeConnectionsService(Settings.EMPTY, threadPool, transportService);
 
         ClusterState current = clusterStateFromNodes(Collections.emptyList());
         ClusterChangedEvent event = new ClusterChangedEvent("test", clusterStateFromNodes(randomSubsetOf(nodes)), current);
 
         transport.randomConnectionExceptions = true;
 
-        service.connectToAddedNodes(event);
+        service.connectToNodes(event.state().nodes());
 
         for (int i = 0; i < 3; i++) {
             // simulate disconnects
@@ -148,8 +153,10 @@ private void assertNotConnected(Iterable<DiscoveryNode> nodes) {
     @Before
     public void setUp() throws Exception {
         super.setUp();
+        this.threadPool = new TestThreadPool(getClass().getName());
         this.transport = new MockTransport();
-        transportService = new TransportService(Settings.EMPTY, transport, THREAD_POOL);
+        transportService = new TransportService(Settings.EMPTY, transport, threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR,
+            boundAddress -> DiscoveryNode.createLocal(Settings.EMPTY, LocalTransportAddress.buildUnique(), UUIDs.randomBase64UUID()), null);
         transportService.start();
         transportService.acceptIncomingRequests();
     }
@@ -158,18 +165,13 @@ public void setUp() throws Exception {
     @After
     public void tearDown() throws Exception {
         transportService.stop();
+        ThreadPool.terminate(threadPool, 30, TimeUnit.SECONDS);
+        threadPool = null;
         super.tearDown();
     }
 
-    @AfterClass
-    public static void stopThreadPool() {
-        ThreadPool.terminate(THREAD_POOL, 30, TimeUnit.SECONDS);
-        THREAD_POOL = null;
-    }
-
-
     final class MockTransport implements Transport {
-
+        private final AtomicLong requestId = new AtomicLong();
         Set<DiscoveryNode> connectedNodes = ConcurrentCollections.newConcurrentSet();
         volatile boolean randomConnectionExceptions = false;
 
@@ -189,7 +191,7 @@ public Map<String, BoundTransportAddress> profileBoundAddresses() {
         }
 
         @Override
-        public TransportAddress[] addressesFromString(String address, int perAddressLimit) throws Exception {
+        public TransportAddress[] addressesFromString(String address, int perAddressLimit) throws UnknownHostException {
             return new TransportAddress[0];
         }
 
@@ -204,16 +206,15 @@ public boolean nodeConnected(DiscoveryNode node) {
         }
 
         @Override
-        public void connectToNode(DiscoveryNode node) throws ConnectTransportException {
-            if (connectedNodes.contains(node) == false && randomConnectionExceptions && randomBoolean()) {
-                throw new ConnectTransportException(node, "simulated");
+        public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile,
+                                  CheckedBiConsumer<Connection, ConnectionProfile, IOException> connectionValidator)
+            throws ConnectTransportException {
+            if (connectionProfile == null) {
+                if (connectedNodes.contains(node) == false && randomConnectionExceptions && randomBoolean()) {
+                    throw new ConnectTransportException(node, "simulated");
+                }
+                connectedNodes.add(node);
             }
-            connectedNodes.add(node);
-        }
-
-        @Override
-        public void connectToNodeLight(DiscoveryNode node) throws ConnectTransportException {
-
         }
 
         @Override
@@ -222,9 +223,29 @@ public void disconnectFromNode(DiscoveryNode node) {
         }
 
         @Override
-        public void sendRequest(DiscoveryNode node, long requestId, String action, TransportRequest request,
-                                TransportRequestOptions options) throws IOException, TransportException {
+        public Connection getConnection(DiscoveryNode node) {
+            return new Connection() {
+                @Override
+                public DiscoveryNode getNode() {
+                    return node;
+                }
+
+                @Override
+                public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options)
+                    throws IOException, TransportException {
 
+                }
+
+                @Override
+                public void close() throws IOException {
+
+                }
+            };
+        }
+
+        @Override
+        public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) throws IOException {
+            return getConnection(node);
         }
 
         @Override
@@ -237,6 +258,11 @@ public List<String> getLocalAddresses() {
             return null;
         }
 
+        @Override
+        public long newRequestId() {
+            return requestId.incrementAndGet();
+        }
+
         @Override
         public Lifecycle.State lifecycleState() {
             return null;
diff --git a/core/src/test/java/org/elasticsearch/cluster/SimpleClusterStateIT.java b/core/src/test/java/org/elasticsearch/cluster/SimpleClusterStateIT.java
index c396350f9de05..ecdb476dcc33a 100644
--- a/core/src/test/java/org/elasticsearch/cluster/SimpleClusterStateIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/SimpleClusterStateIT.java
@@ -94,18 +94,37 @@ public void testIndexTemplates() throws Exception {
         client().admin().indices().preparePutTemplate("foo_template")
                 .setTemplate("te*")
                 .setOrder(0)
-                .addMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties")
-                        .startObject("field1").field("type", "text").field("store", true).endObject()
-                        .startObject("field2").field("type", "keyword").field("store", true).endObject()
-                        .endObject().endObject().endObject())
+                .addMapping("type1", XContentFactory.jsonBuilder()
+                    .startObject()
+                        .startObject("type1")
+                            .startObject("properties")
+                                .startObject("field1")
+                                    .field("type", "text")
+                                    .field("store", true)
+                                .endObject()
+                                .startObject("field2")
+                                    .field("type", "keyword")
+                                    .field("store", true)
+                                .endObject()
+                            .endObject()
+                        .endObject()
+                    .endObject())
                 .get();
 
         client().admin().indices().preparePutTemplate("fuu_template")
                 .setTemplate("test*")
                 .setOrder(1)
-                .addMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties")
-                        .startObject("field2").field("type", "text").field("store", "no").endObject()
-                        .endObject().endObject().endObject())
+                .addMapping("type1", XContentFactory.jsonBuilder()
+                        .startObject()
+                            .startObject("type1")
+                                .startObject("properties")
+                                    .startObject("field2")
+                                        .field("type", "text")
+                                        .field("store", false)
+                                    .endObject()
+                                .endObject()
+                            .endObject()
+                        .endObject())
                 .get();
 
         ClusterStateResponse clusterStateResponseUnfiltered = client().admin().cluster().prepareState().get();
diff --git a/core/src/test/java/org/elasticsearch/cluster/SimpleDataNodesIT.java b/core/src/test/java/org/elasticsearch/cluster/SimpleDataNodesIT.java
index f411e00468eb7..8813efd1b382d 100644
--- a/core/src/test/java/org/elasticsearch/cluster/SimpleDataNodesIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/SimpleDataNodesIT.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
@@ -43,7 +44,8 @@ public void testDataNodes() throws Exception {
         internalCluster().startNode(Settings.builder().put(Node.NODE_DATA_SETTING.getKey(), false).build());
         client().admin().indices().create(createIndexRequest("test").waitForActiveShards(ActiveShardCount.NONE)).actionGet();
         try {
-            client().index(Requests.indexRequest("test").type("type1").id("1").source(source("1", "test")).timeout(timeValueSeconds(1))).actionGet();
+            client().index(Requests.indexRequest("test").type("type1").id("1").source(source("1", "test"), XContentType.JSON)
+                .timeout(timeValueSeconds(1))).actionGet();
             fail("no allocation should happen");
         } catch (UnavailableShardsException e) {
             // all is well
@@ -54,7 +56,8 @@ public void testDataNodes() throws Exception {
 
         // still no shard should be allocated
         try {
-            client().index(Requests.indexRequest("test").type("type1").id("1").source(source("1", "test")).timeout(timeValueSeconds(1))).actionGet();
+            client().index(Requests.indexRequest("test").type("type1").id("1").source(source("1", "test"), XContentType.JSON)
+                .timeout(timeValueSeconds(1))).actionGet();
             fail("no allocation should happen");
         } catch (UnavailableShardsException e) {
             // all is well
@@ -64,7 +67,8 @@ public void testDataNodes() throws Exception {
         internalCluster().startNode(Settings.builder().put(Node.NODE_DATA_SETTING.getKey(), true).build());
         assertThat(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForNodes("3").setLocal(true).execute().actionGet().isTimedOut(), equalTo(false));
 
-        IndexResponse indexResponse = client().index(Requests.indexRequest("test").type("type1").id("1").source(source("1", "test"))).actionGet();
+        IndexResponse indexResponse = client().index(Requests.indexRequest("test").type("type1").id("1")
+            .source(source("1", "test"), XContentType.JSON)).actionGet();
         assertThat(indexResponse.getId(), equalTo("1"));
         assertThat(indexResponse.getType(), equalTo("type1"));
     }
diff --git a/core/src/test/java/org/elasticsearch/cluster/SnapshotsInProgressTests.java b/core/src/test/java/org/elasticsearch/cluster/SnapshotsInProgressTests.java
new file mode 100644
index 0000000000000..fcf70909b31d6
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/cluster/SnapshotsInProgressTests.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster;
+
+import org.elasticsearch.cluster.SnapshotsInProgress.Entry;
+import org.elasticsearch.cluster.SnapshotsInProgress.ShardSnapshotStatus;
+import org.elasticsearch.cluster.SnapshotsInProgress.State;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.repositories.IndexId;
+import org.elasticsearch.snapshots.Snapshot;
+import org.elasticsearch.snapshots.SnapshotId;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * Unit tests for the {@link SnapshotsInProgress} class and its inner classes.
+ */
+public class SnapshotsInProgressTests extends ESTestCase {
+
+    /**
+     * Makes sure that the indices being waited on before snapshotting commences
+     * are populated with all shards in the relocating or initializing state.
+     */
+    public void testWaitingIndices() {
+        final Snapshot snapshot = new Snapshot("repo", new SnapshotId("snap", randomAlphaOfLength(5)));
+        final String idx1Name = "idx1";
+        final String idx2Name = "idx2";
+        final String idx3Name = "idx3";
+        final String idx1UUID = randomAlphaOfLength(5);
+        final String idx2UUID = randomAlphaOfLength(5);
+        final String idx3UUID = randomAlphaOfLength(5);
+        final List<IndexId> indices = Arrays.asList(new IndexId(idx1Name, randomAlphaOfLength(5)),
+            new IndexId(idx2Name, randomAlphaOfLength(5)), new IndexId(idx3Name, randomAlphaOfLength(5)));
+        ImmutableOpenMap.Builder<ShardId, ShardSnapshotStatus> shards = ImmutableOpenMap.builder();
+
+        // test more than one waiting shard in an index
+        shards.put(new ShardId(idx1Name, idx1UUID, 0), new ShardSnapshotStatus(randomAlphaOfLength(2), State.WAITING));
+        shards.put(new ShardId(idx1Name, idx1UUID, 1), new ShardSnapshotStatus(randomAlphaOfLength(2), State.WAITING));
+        shards.put(new ShardId(idx1Name, idx1UUID, 2), new ShardSnapshotStatus(randomAlphaOfLength(2), randomNonWaitingState(), ""));
+        // test exactly one waiting shard in an index
+        shards.put(new ShardId(idx2Name, idx2UUID, 0), new ShardSnapshotStatus(randomAlphaOfLength(2), State.WAITING));
+        shards.put(new ShardId(idx2Name, idx2UUID, 1), new ShardSnapshotStatus(randomAlphaOfLength(2), randomNonWaitingState(), ""));
+        // test no waiting shards in an index
+        shards.put(new ShardId(idx3Name, idx3UUID, 0), new ShardSnapshotStatus(randomAlphaOfLength(2), randomNonWaitingState(), ""));
+        Entry entry = new Entry(snapshot, randomBoolean(), randomBoolean(), State.INIT,
+                                indices, System.currentTimeMillis(), randomLong(), shards.build());
+
+        ImmutableOpenMap<String, List<ShardId>> waitingIndices = entry.waitingIndices();
+        assertEquals(2, waitingIndices.get(idx1Name).size());
+        assertEquals(1, waitingIndices.get(idx2Name).size());
+        assertFalse(waitingIndices.containsKey(idx3Name));
+    }
+
+    private State randomNonWaitingState() {
+        return randomFrom(Arrays.stream(State.values()).filter(s -> s != State.WAITING).collect(Collectors.toSet()));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java b/core/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java
index 4773aafbf3f9f..7a91454531411 100644
--- a/core/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java
@@ -32,21 +32,26 @@
 import java.io.IOException;
 import java.util.Map;
 
+import static org.elasticsearch.discovery.zen.ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
-@ClusterScope(scope = Scope.TEST, numDataNodes = 0)
+@ClusterScope(scope = Scope.TEST, numDataNodes = 0, autoMinMasterNodes = false)
 @ESIntegTestCase.SuppressLocalMode
 public class SpecificMasterNodesIT extends ESIntegTestCase {
-    protected final Settings.Builder settingsBuilder() {
-        return Settings.builder().put("discovery.type", "zen");
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        return Settings.builder().put(super.nodeSettings(nodeOrdinal))
+            .put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), 1).build();
     }
 
     public void testSimpleOnlyMasterNodeElection() throws IOException {
         logger.info("--> start data node / non master node");
-        internalCluster().startNode(settingsBuilder().put(Node.NODE_DATA_SETTING.getKey(), true).put(Node.NODE_MASTER_SETTING.getKey(), false).put("discovery.initial_state_timeout", "1s"));
+        internalCluster().startNode(Settings.builder().put(Node.NODE_DATA_SETTING.getKey(), true).put(Node.NODE_MASTER_SETTING.getKey(), false)
+            .put("discovery.initial_state_timeout", "1s"));
         try {
             assertThat(client().admin().cluster().prepareState().setMasterNodeTimeout("100ms").execute().actionGet().getState().nodes().getMasterNodeId(), nullValue());
             fail("should not be able to find master");
@@ -54,7 +59,7 @@ public void testSimpleOnlyMasterNodeElection() throws IOException {
             // all is well, no master elected
         }
         logger.info("--> start master node");
-        final String masterNodeName = internalCluster().startNode(settingsBuilder().put(Node.NODE_DATA_SETTING.getKey(), false).put(Node.NODE_MASTER_SETTING.getKey(), true));
+        final String masterNodeName = internalCluster().startNode(Settings.builder().put(Node.NODE_DATA_SETTING.getKey(), false).put(Node.NODE_MASTER_SETTING.getKey(), true));
         assertThat(internalCluster().nonMasterClient().admin().cluster().prepareState().execute().actionGet().getState().nodes().getMasterNode().getName(), equalTo(masterNodeName));
         assertThat(internalCluster().masterClient().admin().cluster().prepareState().execute().actionGet().getState().nodes().getMasterNode().getName(), equalTo(masterNodeName));
 
@@ -69,14 +74,14 @@ public void testSimpleOnlyMasterNodeElection() throws IOException {
         }
 
         logger.info("--> start master node");
-        final String nextMasterEligibleNodeName = internalCluster().startNode(settingsBuilder().put(Node.NODE_DATA_SETTING.getKey(), false).put(Node.NODE_MASTER_SETTING.getKey(), true));
+        final String nextMasterEligibleNodeName = internalCluster().startNode(Settings.builder().put(Node.NODE_DATA_SETTING.getKey(), false).put(Node.NODE_MASTER_SETTING.getKey(), true));
         assertThat(internalCluster().nonMasterClient().admin().cluster().prepareState().execute().actionGet().getState().nodes().getMasterNode().getName(), equalTo(nextMasterEligibleNodeName));
         assertThat(internalCluster().masterClient().admin().cluster().prepareState().execute().actionGet().getState().nodes().getMasterNode().getName(), equalTo(nextMasterEligibleNodeName));
     }
 
     public void testElectOnlyBetweenMasterNodes() throws IOException {
         logger.info("--> start data node / non master node");
-        internalCluster().startNode(settingsBuilder().put(Node.NODE_DATA_SETTING.getKey(), true).put(Node.NODE_MASTER_SETTING.getKey(), false).put("discovery.initial_state_timeout", "1s"));
+        internalCluster().startNode(Settings.builder().put(Node.NODE_DATA_SETTING.getKey(), true).put(Node.NODE_MASTER_SETTING.getKey(), false).put("discovery.initial_state_timeout", "1s"));
         try {
             assertThat(client().admin().cluster().prepareState().setMasterNodeTimeout("100ms").execute().actionGet().getState().nodes().getMasterNodeId(), nullValue());
             fail("should not be able to find master");
@@ -84,12 +89,12 @@ public void testElectOnlyBetweenMasterNodes() throws IOException {
             // all is well, no master elected
         }
         logger.info("--> start master node (1)");
-        final String masterNodeName = internalCluster().startNode(settingsBuilder().put(Node.NODE_DATA_SETTING.getKey(), false).put(Node.NODE_MASTER_SETTING.getKey(), true));
+        final String masterNodeName = internalCluster().startNode(Settings.builder().put(Node.NODE_DATA_SETTING.getKey(), false).put(Node.NODE_MASTER_SETTING.getKey(), true));
         assertThat(internalCluster().nonMasterClient().admin().cluster().prepareState().execute().actionGet().getState().nodes().getMasterNode().getName(), equalTo(masterNodeName));
         assertThat(internalCluster().masterClient().admin().cluster().prepareState().execute().actionGet().getState().nodes().getMasterNode().getName(), equalTo(masterNodeName));
 
         logger.info("--> start master node (2)");
-        final String nextMasterEligableNodeName = internalCluster().startNode(settingsBuilder().put(Node.NODE_DATA_SETTING.getKey(), false).put(Node.NODE_MASTER_SETTING.getKey(), true));
+        final String nextMasterEligableNodeName = internalCluster().startNode(Settings.builder().put(Node.NODE_DATA_SETTING.getKey(), false).put(Node.NODE_MASTER_SETTING.getKey(), true));
         assertThat(internalCluster().nonMasterClient().admin().cluster().prepareState().execute().actionGet().getState().nodes().getMasterNode().getName(), equalTo(masterNodeName));
         assertThat(internalCluster().nonMasterClient().admin().cluster().prepareState().execute().actionGet().getState().nodes().getMasterNode().getName(), equalTo(masterNodeName));
         assertThat(internalCluster().masterClient().admin().cluster().prepareState().execute().actionGet().getState().nodes().getMasterNode().getName(), equalTo(masterNodeName));
@@ -106,10 +111,10 @@ public void testElectOnlyBetweenMasterNodes() throws IOException {
      */
     public void testCustomDefaultMapping() throws Exception {
         logger.info("--> start master node / non data");
-        internalCluster().startNode(settingsBuilder().put(Node.NODE_DATA_SETTING.getKey(), false).put(Node.NODE_MASTER_SETTING.getKey(), true));
+        internalCluster().startNode(Settings.builder().put(Node.NODE_DATA_SETTING.getKey(), false).put(Node.NODE_MASTER_SETTING.getKey(), true));
 
         logger.info("--> start data node / non master node");
-        internalCluster().startNode(settingsBuilder().put(Node.NODE_DATA_SETTING.getKey(), true).put(Node.NODE_MASTER_SETTING.getKey(), false));
+        internalCluster().startNode(Settings.builder().put(Node.NODE_DATA_SETTING.getKey(), true).put(Node.NODE_MASTER_SETTING.getKey(), false));
 
         createIndex("test");
         assertAcked(client().admin().indices().preparePutMapping("test").setType("_default_").setSource("timestamp", "type=date"));
@@ -128,10 +133,10 @@ public void testCustomDefaultMapping() throws Exception {
 
     public void testAliasFilterValidation() throws Exception {
         logger.info("--> start master node / non data");
-        internalCluster().startNode(settingsBuilder().put(Node.NODE_DATA_SETTING.getKey(), false).put(Node.NODE_MASTER_SETTING.getKey(), true));
+        internalCluster().startNode(Settings.builder().put(Node.NODE_DATA_SETTING.getKey(), false).put(Node.NODE_MASTER_SETTING.getKey(), true));
 
         logger.info("--> start data node / non master node");
-        internalCluster().startNode(settingsBuilder().put(Node.NODE_DATA_SETTING.getKey(), true).put(Node.NODE_MASTER_SETTING.getKey(), false));
+        internalCluster().startNode(Settings.builder().put(Node.NODE_DATA_SETTING.getKey(), true).put(Node.NODE_MASTER_SETTING.getKey(), false));
 
         assertAcked(prepareCreate("test").addMapping("type1", "{\"type1\" : {\"properties\" : {\"table_a\" : { \"type\" : \"nested\", \"properties\" : {\"field_a\" : { \"type\" : \"keyword\" },\"field_b\" :{ \"type\" : \"keyword\" }}}}}}"));
         client().admin().indices().prepareAliases().addAlias("test", "a_test", QueryBuilders.nestedQuery("table_a", QueryBuilders.termQuery("table_a.field_b", "y"), ScoreMode.Avg)).get();
diff --git a/core/src/test/java/org/elasticsearch/cluster/UpdateSettingsValidationIT.java b/core/src/test/java/org/elasticsearch/cluster/UpdateSettingsValidationIT.java
index 89f072e2a354a..1e9faf81aaaab 100644
--- a/core/src/test/java/org/elasticsearch/cluster/UpdateSettingsValidationIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/UpdateSettingsValidationIT.java
@@ -27,7 +27,6 @@
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
 
-
 import static org.hamcrest.Matchers.equalTo;
 
 /**
@@ -35,11 +34,11 @@
 @ClusterScope(scope= Scope.TEST, numDataNodes =0)
 public class UpdateSettingsValidationIT extends ESIntegTestCase {
     public void testUpdateSettingsValidation() throws Exception {
-        internalCluster().startNodesAsync(
+        internalCluster().startNodes(
                 Settings.builder().put(Node.NODE_DATA_SETTING.getKey(), false).build(),
                 Settings.builder().put(Node.NODE_MASTER_SETTING.getKey(), false).build(),
                 Settings.builder().put(Node.NODE_MASTER_SETTING.getKey(), false).build()
-        ).get();
+        );
 
         createIndex("test");
         NumShards test = getNumShards("test");
diff --git a/core/src/test/java/org/elasticsearch/cluster/ack/AckClusterUpdateSettingsIT.java b/core/src/test/java/org/elasticsearch/cluster/ack/AckClusterUpdateSettingsIT.java
index f47134a45b8ed..ab3f82fff75f5 100644
--- a/core/src/test/java/org/elasticsearch/cluster/ack/AckClusterUpdateSettingsIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/ack/AckClusterUpdateSettingsIT.java
@@ -69,7 +69,10 @@ protected int numberOfReplicas() {
     private void removePublishTimeout() {
         //to test that the acknowledgement mechanism is working we better disable the wait for publish
         //otherwise the operation is most likely acknowledged even if it doesn't support ack
-        assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put(DiscoverySettings.PUBLISH_TIMEOUT_SETTING.getKey(), "0")));
+        assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder()
+            .put(DiscoverySettings.PUBLISH_TIMEOUT_SETTING.getKey(), "0")
+            .put(DiscoverySettings.COMMIT_TIMEOUT_SETTING.getKey(), "30s")
+        ));
     }
 
     public void testClusterUpdateSettingsAcknowledgement() {
diff --git a/core/src/test/java/org/elasticsearch/cluster/ack/AckIT.java b/core/src/test/java/org/elasticsearch/cluster/ack/AckIT.java
index 2ea4f7551568d..f51a4f11ae1b6 100644
--- a/core/src/test/java/org/elasticsearch/cluster/ack/AckIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/ack/AckIT.java
@@ -36,7 +36,6 @@
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand;
-import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.discovery.DiscoverySettings;
 import org.elasticsearch.index.Index;
@@ -61,7 +60,9 @@ protected Settings nodeSettings(int nodeOrdinal) {
         //to test that the acknowledgement mechanism is working we better disable the wait for publish
         //otherwise the operation is most likely acknowledged even if it doesn't support ack
         return Settings.builder().put(super.nodeSettings(nodeOrdinal))
-                .put(DiscoverySettings.PUBLISH_TIMEOUT_SETTING.getKey(), 0).build();
+            .put(DiscoverySettings.COMMIT_TIMEOUT_SETTING.getKey(), "30s") // explicitly set so it won't default to publish timeout
+            .put(DiscoverySettings.PUBLISH_TIMEOUT_SETTING.getKey(), "0s") // don't wait post commit to check acking
+            .build();
     }
 
     public void testUpdateSettingsAcknowledgement() {
diff --git a/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardFailedClusterStateTaskExecutorTests.java b/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardFailedClusterStateTaskExecutorTests.java
index ff0e0ba7a247a..fe11cc9cd5fd5 100644
--- a/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardFailedClusterStateTaskExecutorTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardFailedClusterStateTaskExecutorTests.java
@@ -37,8 +37,8 @@
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
-import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation;
-import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
+import org.elasticsearch.cluster.routing.allocation.FailedShard;
+import org.elasticsearch.cluster.routing.allocation.StaleShard;
 import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.Settings;
@@ -88,7 +88,7 @@ public void setUp() throws Exception {
 
     public void testEmptyTaskListProducesSameClusterState() throws Exception {
         List<ShardStateAction.ShardEntry> tasks = Collections.emptyList();
-        ClusterStateTaskExecutor.BatchResult<ShardStateAction.ShardEntry> result =
+        ClusterStateTaskExecutor.ClusterTasksResult<ShardStateAction.ShardEntry> result =
             executor.execute(clusterState, tasks);
         assertTasksSuccessful(tasks, result, clusterState, false);
     }
@@ -97,7 +97,7 @@ public void testDuplicateFailuresAreOkay() throws Exception {
         String reason = "test duplicate failures are okay";
         ClusterState currentState = createClusterStateWithStartedShards(reason);
         List<ShardStateAction.ShardEntry> tasks = createExistingShards(currentState, reason);
-        ClusterStateTaskExecutor.BatchResult<ShardStateAction.ShardEntry> result = executor.execute(currentState, tasks);
+        ClusterStateTaskExecutor.ClusterTasksResult<ShardStateAction.ShardEntry> result = executor.execute(currentState, tasks);
         assertTasksSuccessful(tasks, result, clusterState, true);
     }
 
@@ -105,7 +105,7 @@ public void testNonExistentShardsAreMarkedAsSuccessful() throws Exception {
         String reason = "test non existent shards are marked as successful";
         ClusterState currentState = createClusterStateWithStartedShards(reason);
         List<ShardStateAction.ShardEntry> tasks = createNonExistentShards(currentState, reason);
-        ClusterStateTaskExecutor.BatchResult<ShardStateAction.ShardEntry> result = executor.execute(clusterState, tasks);
+        ClusterStateTaskExecutor.ClusterTasksResult<ShardStateAction.ShardEntry> result = executor.execute(clusterState, tasks);
         assertTasksSuccessful(tasks, result, clusterState, false);
     }
 
@@ -116,15 +116,14 @@ public void testTriviallySuccessfulTasksBatchedWithFailingTasks() throws Excepti
         List<ShardStateAction.ShardEntry> nonExistentTasks = createNonExistentShards(currentState, reason);
         ShardStateAction.ShardFailedClusterStateTaskExecutor failingExecutor = new ShardStateAction.ShardFailedClusterStateTaskExecutor(allocationService, null, logger) {
             @Override
-            RoutingAllocation.Result applyFailedShards(ClusterState currentState, List<FailedRerouteAllocation.FailedShard> failedShards,
-                                                       List<FailedRerouteAllocation.StaleShard> staleShards) {
+            ClusterState applyFailedShards(ClusterState currentState, List<FailedShard> failedShards, List<StaleShard> staleShards) {
                 throw new RuntimeException("simulated applyFailedShards failure");
             }
         };
         List<ShardStateAction.ShardEntry> tasks = new ArrayList<>();
         tasks.addAll(failingTasks);
         tasks.addAll(nonExistentTasks);
-        ClusterStateTaskExecutor.BatchResult<ShardStateAction.ShardEntry> result = failingExecutor.execute(currentState, tasks);
+        ClusterStateTaskExecutor.ClusterTasksResult<ShardStateAction.ShardEntry> result = failingExecutor.execute(currentState, tasks);
         Map<ShardStateAction.ShardEntry, ClusterStateTaskExecutor.TaskResult> taskResultMap =
             failingTasks.stream().collect(Collectors.toMap(Function.identity(), task -> ClusterStateTaskExecutor.TaskResult.failure(new RuntimeException("simulated applyFailedShards failure"))));
         taskResultMap.putAll(nonExistentTasks.stream().collect(Collectors.toMap(Function.identity(), task -> ClusterStateTaskExecutor.TaskResult.success())));
@@ -147,7 +146,7 @@ public void testIllegalShardFailureRequests() throws Exception {
                 task -> ClusterStateTaskExecutor.TaskResult.failure(new ShardStateAction.NoLongerPrimaryShardException(task.shardId,
                     "primary term [" + task.primaryTerm + "] did not match current primary term [" +
                         currentState.metaData().index(task.shardId.getIndex()).primaryTerm(task.shardId.id()) + "]"))));
-        ClusterStateTaskExecutor.BatchResult<ShardStateAction.ShardEntry> result = executor.execute(currentState, tasks);
+        ClusterStateTaskExecutor.ClusterTasksResult<ShardStateAction.ShardEntry> result = executor.execute(currentState, tasks);
         assertTaskResults(taskResultMap, result, currentState, false);
     }
 
@@ -161,16 +160,14 @@ private ClusterState createClusterStateWithStartedShards(String reason) {
             allocationService.reroute(stateAfterAddingNode, reason).routingTable();
         ClusterState stateAfterReroute = ClusterState.builder(stateAfterAddingNode).routingTable(afterReroute).build();
         RoutingNodes routingNodes = stateAfterReroute.getRoutingNodes();
-        RoutingAllocation.Result afterStart = allocationService.applyStartedShards(stateAfterReroute, routingNodes.shardsWithState(ShardRoutingState.INITIALIZING));
-        return ClusterState.builder(stateAfterReroute).routingResult(afterStart).build();
+        return allocationService.applyStartedShards(stateAfterReroute, routingNodes.shardsWithState(ShardRoutingState.INITIALIZING));
     }
 
     private List<ShardStateAction.ShardEntry> createExistingShards(ClusterState currentState, String reason) {
         List<ShardRouting> shards = new ArrayList<>();
-        GroupShardsIterator shardGroups =
-            currentState.routingTable().allAssignedShardsGrouped(new String[] { INDEX }, true);
+        GroupShardsIterator<ShardIterator> shardGroups = currentState.routingTable().allAssignedShardsGrouped(new String[] { INDEX }, true);
         for (ShardIterator shardIt : shardGroups) {
-            for (ShardRouting shard : shardIt.asUnordered()) {
+            for (ShardRouting shard : shardIt) {
                 shards.add(shard);
             }
         }
@@ -216,7 +213,7 @@ private ShardRouting nonExistentShardRouting(Index index, List<String> nodeIds,
 
     private static void assertTasksSuccessful(
         List<ShardStateAction.ShardEntry> tasks,
-        ClusterStateTaskExecutor.BatchResult<ShardStateAction.ShardEntry> result,
+        ClusterStateTaskExecutor.ClusterTasksResult<ShardStateAction.ShardEntry> result,
         ClusterState clusterState,
         boolean clusterStateChanged
     ) {
@@ -227,7 +224,7 @@ private static void assertTasksSuccessful(
 
     private static void assertTaskResults(
         Map<ShardStateAction.ShardEntry, ClusterStateTaskExecutor.TaskResult> taskResultMap,
-        ClusterStateTaskExecutor.BatchResult<ShardStateAction.ShardEntry> result,
+        ClusterStateTaskExecutor.ClusterTasksResult<ShardStateAction.ShardEntry> result,
         ClusterState clusterState,
         boolean clusterStateChanged
     ) {
diff --git a/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java b/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java
index e0a04dc2f3c1a..58c9651a82af1 100644
--- a/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java
@@ -54,6 +54,7 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.LongConsumer;
+import java.util.function.Predicate;
 
 import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
 import static org.elasticsearch.test.ClusterServiceUtils.setState;
@@ -71,7 +72,7 @@ public class ShardStateActionTests extends ESTestCase {
     private ClusterService clusterService;
 
     private static class TestShardStateAction extends ShardStateAction {
-        public TestShardStateAction(Settings settings, ClusterService clusterService, TransportService transportService, AllocationService allocationService, RoutingService routingService) {
+        TestShardStateAction(Settings settings, ClusterService clusterService, TransportService transportService, AllocationService allocationService, RoutingService routingService) {
             super(settings, clusterService, transportService, allocationService, routingService, THREAD_POOL);
         }
 
@@ -88,9 +89,9 @@ public void setOnAfterWaitForNewMasterAndRetry(Runnable onAfterWaitForNewMasterA
         }
 
         @Override
-        protected void waitForNewMasterAndRetry(String actionName, ClusterStateObserver observer, ShardEntry shardEntry, Listener listener) {
+        protected void waitForNewMasterAndRetry(String actionName, ClusterStateObserver observer, ShardEntry shardEntry, Listener listener, Predicate<ClusterState> changePredicate) {
             onBeforeWaitForNewMasterAndRetry.run();
-            super.waitForNewMasterAndRetry(actionName, observer, shardEntry, listener);
+            super.waitForNewMasterAndRetry(actionName, observer, shardEntry, listener, changePredicate);
             onAfterWaitForNewMasterAndRetry.run();
         }
     }
@@ -106,7 +107,8 @@ public void setUp() throws Exception {
         super.setUp();
         this.transport = new CapturingTransport();
         clusterService = createClusterService(THREAD_POOL);
-        transportService = new TransportService(clusterService.getSettings(), transport, THREAD_POOL);
+        transportService = new TransportService(clusterService.getSettings(), transport, THREAD_POOL,
+                TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null);
         transportService.start();
         transportService.acceptIncomingRequests();
         shardStateAction = new TestShardStateAction(Settings.EMPTY, clusterService, transportService, null, null);
diff --git a/core/src/test/java/org/elasticsearch/cluster/allocation/AwarenessAllocationIT.java b/core/src/test/java/org/elasticsearch/cluster/allocation/AwarenessAllocationIT.java
index 98c7b1a3d67a0..60b863c11d681 100644
--- a/core/src/test/java/org/elasticsearch/cluster/allocation/AwarenessAllocationIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/allocation/AwarenessAllocationIT.java
@@ -31,7 +31,6 @@
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.discovery.zen.ZenDiscovery;
-import org.elasticsearch.discovery.zen.elect.ElectMasterService;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 
@@ -60,7 +59,7 @@ public void testSimpleAwareness() throws Exception {
 
 
         logger.info("--> starting 2 nodes on the same rack");
-        internalCluster().startNodesAsync(2, Settings.builder().put(commonSettings).put("node.attr.rack_id", "rack_1").build()).get();
+        internalCluster().startNodes(2, Settings.builder().put(commonSettings).put("node.attr.rack_id", "rack_1").build());
 
         createIndex("test1");
         createIndex("test2");
@@ -106,17 +105,16 @@ public void testAwarenessZones() throws Exception {
         Settings commonSettings = Settings.builder()
                 .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING.getKey() + "zone.values", "a,b")
                 .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.getKey(), "zone")
-                .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), 3)
                 .put(ZenDiscovery.JOIN_TIMEOUT_SETTING.getKey(), "10s")
                 .build();
 
         logger.info("--> starting 4 nodes on different zones");
-        List<String> nodes = internalCluster().startNodesAsync(
+        List<String> nodes = internalCluster().startNodes(
                 Settings.builder().put(commonSettings).put("node.attr.zone", "a").build(),
                 Settings.builder().put(commonSettings).put("node.attr.zone", "b").build(),
                 Settings.builder().put(commonSettings).put("node.attr.zone", "b").build(),
                 Settings.builder().put(commonSettings).put("node.attr.zone", "a").build()
-        ).get();
+        );
         String A_0 = nodes.get(0);
         String B_0 = nodes.get(1);
         String B_1 = nodes.get(2);
@@ -157,10 +155,10 @@ public void testAwarenessZonesIncrementalNodes() throws Exception {
                 .build();
 
         logger.info("--> starting 2 nodes on zones 'a' & 'b'");
-        List<String> nodes = internalCluster().startNodesAsync(
+        List<String> nodes = internalCluster().startNodes(
                 Settings.builder().put(commonSettings).put("node.attr.zone", "a").build(),
                 Settings.builder().put(commonSettings).put("node.attr.zone", "b").build()
-        ).get();
+        );
         String A_0 = nodes.get(0);
         String B_0 = nodes.get(1);
         client().admin().indices().prepareCreate("test")
diff --git a/core/src/test/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java b/core/src/test/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java
index 22d02f5146973..e585508e85da1 100644
--- a/core/src/test/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java
@@ -59,6 +59,7 @@
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_READ;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_WRITE;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY_ALLOW_DELETE;
 import static org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBlocked;
@@ -87,7 +88,7 @@ public void testRerouteWithCommands_enableAllocationSettings() throws Exception
     }
 
     private void rerouteWithCommands(Settings commonSettings) throws Exception {
-        List<String> nodesIds = internalCluster().startNodesAsync(2, commonSettings).get();
+        List<String> nodesIds = internalCluster().startNodes(2, commonSettings);
         final String node_1 = nodesIds.get(0);
         final String node_2 = nodesIds.get(1);
 
@@ -306,7 +307,7 @@ public void testRerouteExplain() {
     }
 
     public void testClusterRerouteWithBlocks() throws Exception {
-        List<String> nodesIds = internalCluster().startNodesAsync(2).get();
+        List<String> nodesIds = internalCluster().startNodes(2);
 
         logger.info("--> create an index with 1 shard and 0 replicas");
         assertAcked(prepareCreate("test-blocks").setSettings(Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0)));
@@ -329,7 +330,8 @@ public void testClusterRerouteWithBlocks() throws Exception {
         int toggle = nodesIds.indexOf(node.getName());
 
         // Rerouting shards is not blocked
-        for (String blockSetting : Arrays.asList(SETTING_BLOCKS_READ, SETTING_BLOCKS_WRITE, SETTING_READ_ONLY, SETTING_BLOCKS_METADATA)) {
+        for (String blockSetting : Arrays.asList(SETTING_BLOCKS_READ, SETTING_BLOCKS_WRITE, SETTING_READ_ONLY, SETTING_BLOCKS_METADATA,
+            SETTING_READ_ONLY_ALLOW_DELETE)) {
             try {
                 enableIndexBlock("test-blocks", blockSetting);
                 assertAcked(client().admin().cluster().prepareReroute()
diff --git a/core/src/test/java/org/elasticsearch/cluster/allocation/FilteringAllocationIT.java b/core/src/test/java/org/elasticsearch/cluster/allocation/FilteringAllocationIT.java
index 627fc03701c20..93771019ef437 100644
--- a/core/src/test/java/org/elasticsearch/cluster/allocation/FilteringAllocationIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/allocation/FilteringAllocationIT.java
@@ -24,8 +24,10 @@
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider;
 import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -43,7 +45,7 @@ public class FilteringAllocationIT extends ESIntegTestCase {
 
     public void testDecommissionNodeNoReplicas() throws Exception {
         logger.info("--> starting 2 nodes");
-        List<String> nodesIds = internalCluster().startNodesAsync(2).get();
+        List<String> nodesIds = internalCluster().startNodes(2);
         final String node_0 = nodesIds.get(0);
         final String node_1 = nodesIds.get(1);
         assertThat(cluster().size(), equalTo(2));
@@ -82,7 +84,7 @@ public void testDecommissionNodeNoReplicas() throws Exception {
 
     public void testDisablingAllocationFiltering() throws Exception {
         logger.info("--> starting 2 nodes");
-        List<String> nodesIds = internalCluster().startNodesAsync(2).get();
+        List<String> nodesIds = internalCluster().startNodes(2);
         final String node_0 = nodesIds.get(0);
         final String node_1 = nodesIds.get(1);
         assertThat(cluster().size(), equalTo(2));
@@ -144,5 +146,15 @@ public void testDisablingAllocationFiltering() throws Exception {
         clusterState = client().admin().cluster().prepareState().execute().actionGet().getState();
         assertThat(clusterState.routingTable().index("test").numberOfNodesShardsAreAllocatedOn(), equalTo(2));
     }
+
+    public void testInvalidIPFilterClusterSettings() {
+        String ipKey = randomFrom("_ip", "_host_ip", "_publish_ip");
+        Setting<Settings> filterSetting = randomFrom(FilterAllocationDecider.CLUSTER_ROUTING_REQUIRE_GROUP_SETTING,
+            FilterAllocationDecider.CLUSTER_ROUTING_INCLUDE_GROUP_SETTING, FilterAllocationDecider.CLUSTER_ROUTING_EXCLUDE_GROUP_SETTING);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> client().admin().cluster().prepareUpdateSettings()
+            .setTransientSettings(Settings.builder().put(filterSetting.getKey() + ipKey, "192.168.1.1."))
+            .execute().actionGet());
+        assertEquals("invalid IP address [192.168.1.1.] for [" + ipKey + "]", e.getMessage());
+    }
 }
 
diff --git a/core/src/test/java/org/elasticsearch/cluster/block/ClusterBlockTests.java b/core/src/test/java/org/elasticsearch/cluster/block/ClusterBlockTests.java
index a7fe1b918c0f0..a84d160cf0c95 100644
--- a/core/src/test/java/org/elasticsearch/cluster/block/ClusterBlockTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/block/ClusterBlockTests.java
@@ -49,7 +49,7 @@ public void testSerialization() throws Exception {
             }
 
             ClusterBlock clusterBlock = new ClusterBlock(randomInt(), "cluster block #" + randomInt(), randomBoolean(),
-                    randomBoolean(), randomFrom(RestStatus.values()), levels);
+                    randomBoolean(), false, randomFrom(RestStatus.values()), levels);
 
             BytesStreamOutput out = new BytesStreamOutput();
             out.setVersion(version);
@@ -75,7 +75,7 @@ public void testToStringDanglingComma() {
             levels.add(randomFrom(ClusterBlockLevel.values()));
         }
         ClusterBlock clusterBlock = new ClusterBlock(randomInt(), "cluster block #" + randomInt(), randomBoolean(),
-                randomBoolean(), randomFrom(RestStatus.values()), levels);
+                randomBoolean(), false, randomFrom(RestStatus.values()), levels);
         assertThat(clusterBlock.toString(), not(endsWith(",")));
     }
 
@@ -86,7 +86,7 @@ public void testGlobalBlocksCheckedIfNoIndicesSpecified() {
             levels.add(randomFrom(ClusterBlockLevel.values()));
         }
         ClusterBlock globalBlock = new ClusterBlock(randomInt(), "cluster block #" + randomInt(), randomBoolean(),
-            randomBoolean(), randomFrom(RestStatus.values()), levels);
+            randomBoolean(), false, randomFrom(RestStatus.values()), levels);
         ClusterBlocks clusterBlocks = new ClusterBlocks(Collections.singleton(globalBlock), ImmutableOpenMap.of());
         ClusterBlockException exception = clusterBlocks.indicesBlockedException(randomFrom(globalBlock.levels()), new String[0]);
         assertNotNull(exception);
diff --git a/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java b/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java
index df5ebb0193a00..f2cd88a27a7c1 100644
--- a/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java
@@ -29,10 +29,11 @@
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.LocalClusterUpdateTask;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.RecoverySource;
@@ -48,7 +49,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.gateway.NoopGatewayAllocator;
+import org.elasticsearch.test.gateway.TestGatewayAllocator;
 import org.elasticsearch.test.transport.CapturingTransport;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -68,6 +69,7 @@
 import java.util.concurrent.TimeUnit;
 
 import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
+import static org.elasticsearch.test.ClusterServiceUtils.setState;
 import static org.hamcrest.CoreMatchers.allOf;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
@@ -92,7 +94,8 @@ public static void setupThreadPool() {
     public void setUp() throws Exception {
         super.setUp();
         clusterService = createClusterService(threadPool);
-        transportService = new TransportService(clusterService.getSettings(), new CapturingTransport(), threadPool);
+        transportService = new TransportService(clusterService.getSettings(), new CapturingTransport(), threadPool,
+            TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null);
         transportService.start();
         transportService.acceptIncomingRequests();
     }
@@ -113,7 +116,11 @@ public static void terminateThreadPool() {
     public void testClusterHealthWaitsForClusterStateApplication() throws InterruptedException, ExecutionException {
         final CountDownLatch applyLatch = new CountDownLatch(1);
         final CountDownLatch listenerCalled = new CountDownLatch(1);
-        clusterService.add(event -> {
+
+        setState(clusterService, ClusterState.builder(clusterService.state())
+            .nodes(DiscoveryNodes.builder(clusterService.state().nodes()).masterNodeId(null)).build());
+
+        clusterService.addStateApplier(event -> {
             listenerCalled.countDown();
             try {
                 applyLatch.await();
@@ -122,10 +129,12 @@ public void testClusterHealthWaitsForClusterStateApplication() throws Interrupte
             }
         });
 
-        clusterService.submitStateUpdateTask("test", new ClusterStateUpdateTask() {
+        logger.info("--> submit task to restore master");
+        clusterService.submitStateUpdateTask("restore master", new LocalClusterUpdateTask() {
             @Override
-            public ClusterState execute(ClusterState currentState) throws Exception {
-                return ClusterState.builder(currentState).build();
+            public ClusterTasksResult<LocalClusterUpdateTask> execute(ClusterState currentState) throws Exception {
+                return newState(ClusterState.builder(currentState).nodes(
+                    DiscoveryNodes.builder(currentState.nodes()).masterNodeId(currentState.nodes().getLocalNodeId())).build());
             }
 
             @Override
@@ -138,12 +147,13 @@ public void onFailure(String source, Exception e) {
         listenerCalled.await();
 
         TransportClusterHealthAction action = new TransportClusterHealthAction(Settings.EMPTY, transportService,
-            clusterService, threadPool, new ActionFilters(new HashSet<>()), indexNameExpressionResolver, NoopGatewayAllocator.INSTANCE);
+            clusterService, threadPool, new ActionFilters(new HashSet<>()), indexNameExpressionResolver, new TestGatewayAllocator());
         PlainActionFuture<ClusterHealthResponse> listener = new PlainActionFuture<>();
-        action.execute(new ClusterHealthRequest(), listener);
+        action.execute(new ClusterHealthRequest().waitForGreenStatus(), listener);
 
         assertFalse(listener.isDone());
 
+        logger.info("--> realising task to restore master");
         applyLatch.countDown();
         listener.get();
     }
@@ -276,9 +286,9 @@ public void testClusterHealthOnClusterRecoveryWithPreviousAllocationIdsAndAlloca
             // if the inactive primaries are due solely to recovery (not failed allocation or previously being allocated)
             // then cluster health is YELLOW, otherwise RED
             if (primaryInactiveDueToRecovery(indexName, clusterState)) {
-                assertThat(health.getStatus(), equalTo(ClusterHealthStatus.YELLOW));
+                assertThat("clusterState is:\n" + clusterState, health.getStatus(), equalTo(ClusterHealthStatus.YELLOW));
             } else {
-                assertThat(health.getStatus(), equalTo(ClusterHealthStatus.RED));
+                assertThat("clusterState is:\n" + clusterState, health.getStatus(), equalTo(ClusterHealthStatus.RED));
             }
         }
     }
@@ -352,10 +362,10 @@ private List<ClusterState> generateClusterStates(final ClusterState originalClus
                                                      final int numberOfReplicas,
                                                      final boolean withPrimaryAllocationFailures) {
         // generate random node ids
-        final List<String> nodeIds = new ArrayList<>();
+        final Set<String> nodeIds = new HashSet<>();
         final int numNodes = randomIntBetween(numberOfReplicas + 1, 10);
         for (int i = 0; i < numNodes; i++) {
-            nodeIds.add(randomAsciiOfLength(8));
+            nodeIds.add(randomAlphaOfLength(8));
         }
 
         final List<ClusterState> clusterStates = new ArrayList<>();
@@ -371,7 +381,7 @@ private List<ClusterState> generateClusterStates(final ClusterState originalClus
             for (final ShardRouting shardRouting : shardRoutingTable.getShards()) {
                 if (shardRouting.primary()) {
                     newIndexRoutingTable.addShard(
-                        shardRouting.initialize(nodeIds.get(randomIntBetween(0, numNodes - 1)), null, shardRouting.getExpectedShardSize())
+                        shardRouting.initialize(randomFrom(nodeIds), null, shardRouting.getExpectedShardSize())
                     );
                 } else {
                     newIndexRoutingTable.addShard(shardRouting);
@@ -459,17 +469,15 @@ private List<ClusterState> generateClusterStates(final ClusterState originalClus
         newIndexRoutingTable = IndexRoutingTable.builder(indexRoutingTable.getIndex());
         for (final ObjectCursor<IndexShardRoutingTable> shardEntry : indexRoutingTable.getShards().values()) {
             final IndexShardRoutingTable shardRoutingTable = shardEntry.value;
+            final String primaryNodeId = shardRoutingTable.primaryShard().currentNodeId();
+            Set<String> allocatedNodes = new HashSet<>();
+            allocatedNodes.add(primaryNodeId);
             for (final ShardRouting shardRouting : shardRoutingTable.getShards()) {
                 if (shardRouting.primary() == false) {
                     // give the replica a different node id than the primary
-                    final String primaryNodeId = shardRoutingTable.primaryShard().currentNodeId();
-                    String replicaNodeId;
-                    do {
-                        replicaNodeId = nodeIds.get(randomIntBetween(0, numNodes - 1));
-                    } while (primaryNodeId.equals(replicaNodeId));
-                    newIndexRoutingTable.addShard(
-                        shardRouting.initialize(replicaNodeId, null, shardRouting.getExpectedShardSize())
-                    );
+                    String replicaNodeId = randomFrom(Sets.difference(nodeIds, allocatedNodes));
+                    newIndexRoutingTable.addShard(shardRouting.initialize(replicaNodeId, null, shardRouting.getExpectedShardSize()));
+                    allocatedNodes.add(replicaNodeId);
                 } else {
                     newIndexRoutingTable.addShard(shardRouting);
                 }
@@ -531,6 +539,12 @@ private boolean primaryInactiveDueToRecovery(final String indexName, final Clust
                     primaryShard.recoverySource().getType() == RecoverySource.Type.EXISTING_STORE) {
                     return false;
                 }
+                if (primaryShard.unassignedInfo().getNumFailedAllocations() > 0) {
+                    return false;
+                }
+                if (primaryShard.unassignedInfo().getLastAllocationStatus() == UnassignedInfo.AllocationStatus.DECIDERS_NO) {
+                    return false;
+                }
             }
         }
         return true;
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/AliasValidatorTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/AliasValidatorTests.java
new file mode 100644
index 0000000000000..05bd9eeab8c3d
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/AliasValidatorTests.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.metadata;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.indices.InvalidAliasNameException;
+import org.elasticsearch.test.ESTestCase;
+
+import static org.hamcrest.Matchers.startsWith;
+
+public class AliasValidatorTests extends ESTestCase {
+    public void testValidatesAliasNames() {
+        AliasValidator validator = new AliasValidator(Settings.EMPTY);
+        Exception e = expectThrows(InvalidAliasNameException.class, () -> validator.validateAliasStandalone(".", null));
+        assertEquals("Invalid alias name [.]: must not be '.' or '..'", e.getMessage());
+        e = expectThrows(InvalidAliasNameException.class, () -> validator.validateAliasStandalone("..", null));
+        assertEquals("Invalid alias name [..]: must not be '.' or '..'", e.getMessage());
+        e = expectThrows(InvalidAliasNameException.class, () -> validator.validateAliasStandalone("_cat", null));
+        assertEquals("Invalid alias name [_cat]: must not start with '_', '-', or '+'", e.getMessage());
+        e = expectThrows(InvalidAliasNameException.class, () -> validator.validateAliasStandalone("-cat", null));
+        assertEquals("Invalid alias name [-cat]: must not start with '_', '-', or '+'", e.getMessage());
+        e = expectThrows(InvalidAliasNameException.class, () -> validator.validateAliasStandalone("+cat", null));
+        assertEquals("Invalid alias name [+cat]: must not start with '_', '-', or '+'", e.getMessage());
+        e = expectThrows(InvalidAliasNameException.class, () -> validator.validateAliasStandalone("c*t", null));
+        assertThat(e.getMessage(), startsWith("Invalid alias name [c*t]: must not contain the following characters "));
+
+        // Doesn't throw an exception because we allow upper case alias names
+        validator.validateAliasStandalone("CAT", null);
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/ClusterNameExpressionResolverTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/ClusterNameExpressionResolverTests.java
new file mode 100644
index 0000000000000..d6c8707c1d76e
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/ClusterNameExpressionResolverTests.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.metadata;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+public class ClusterNameExpressionResolverTests extends ESTestCase {
+
+    private ClusterNameExpressionResolver clusterNameResolver = new ClusterNameExpressionResolver(Settings.EMPTY);
+    private static final Set<String> remoteClusters = new HashSet<>();
+
+    static {
+        remoteClusters.add("cluster1");
+        remoteClusters.add("cluster2");
+        remoteClusters.add("totallyDifferent");
+    }
+
+    public void testExactMatch() {
+        List<String> clusters = clusterNameResolver.resolveClusterNames(remoteClusters, "totallyDifferent");
+        assertEquals(new HashSet<>(Arrays.asList("totallyDifferent")), new HashSet<>(clusters));
+    }
+
+    public void testNoWildCardNoMatch() {
+        List<String> clusters = clusterNameResolver.resolveClusterNames(remoteClusters, "totallyDifferent2");
+        assertTrue(clusters.isEmpty());
+    }
+
+    public void testWildCardNoMatch() {
+        List<String> clusters = clusterNameResolver.resolveClusterNames(remoteClusters, "totally*2");
+        assertTrue(clusters.isEmpty());
+    }
+
+    public void testSimpleWildCard() {
+        List<String> clusters = clusterNameResolver.resolveClusterNames(remoteClusters, "*");
+        assertEquals(new HashSet<>(Arrays.asList("cluster1", "cluster2", "totallyDifferent")), new HashSet<>(clusters));
+    }
+
+    public void testSuffixWildCard() {
+        List<String> clusters = clusterNameResolver.resolveClusterNames(remoteClusters, "cluster*");
+        assertEquals(new HashSet<>(Arrays.asList("cluster1", "cluster2")), new HashSet<>(clusters));
+    }
+
+    public void testPrefixWildCard() {
+        List<String> clusters = clusterNameResolver.resolveClusterNames(remoteClusters, "*Different");
+        assertEquals(new HashSet<>(Arrays.asList("totallyDifferent")), new HashSet<>(clusters));
+    }
+
+    public void testMiddleWildCard() {
+        List<String> clusters = clusterNameResolver.resolveClusterNames(remoteClusters, "clu*1");
+        assertEquals(new HashSet<>(Arrays.asList("cluster1")), new HashSet<>(clusters));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/DateMathExpressionResolverTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/DateMathExpressionResolverTests.java
index 0215f94d4dab3..4f235e52636b3 100644
--- a/core/src/test/java/org/elasticsearch/cluster/metadata/DateMathExpressionResolverTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/DateMathExpressionResolverTests.java
@@ -51,7 +51,7 @@ public void testNormal() throws Exception {
         int numIndexExpressions = randomIntBetween(1, 9);
         List<String> indexExpressions = new ArrayList<>(numIndexExpressions);
         for (int i = 0; i < numIndexExpressions; i++) {
-            indexExpressions.add(randomAsciiOfLength(10));
+            indexExpressions.add(randomAlphaOfLength(10));
         }
         List<String> result = expressionResolver.resolve(context, indexExpressions);
         assertThat(result.size(), equalTo(indexExpressions.size()));
@@ -177,43 +177,31 @@ public void testExpression_CustomTimeZoneInIndexName() throws Exception {
     }
 
     public void testExpressionInvalidUnescaped() throws Exception {
-        try {
-            expressionResolver.resolve(context, Arrays.asList("<.mar}vel-{now/d}>"));
-            fail("Expected ElasticsearchParseException");
-        } catch (ElasticsearchParseException e) {
-            assertThat(e.getMessage(), containsString("invalid dynamic name expression"));
-            assertThat(e.getMessage(), containsString("invalid character at position ["));
-        }
+        Exception e = expectThrows(ElasticsearchParseException.class,
+                () -> expressionResolver.resolve(context, Arrays.asList("<.mar}vel-{now/d}>")));
+        assertThat(e.getMessage(), containsString("invalid dynamic name expression"));
+        assertThat(e.getMessage(), containsString("invalid character at position ["));
     }
 
     public void testExpressionInvalidDateMathFormat() throws Exception {
-        try {
-            expressionResolver.resolve(context, Arrays.asList("<.marvel-{now/d{}>"));
-            fail("Expected ElasticsearchParseException");
-        } catch (ElasticsearchParseException e) {
-            assertThat(e.getMessage(), containsString("invalid dynamic name expression"));
-            assertThat(e.getMessage(), containsString("date math placeholder is open ended"));
-        }
+        Exception e = expectThrows(ElasticsearchParseException.class,
+                () -> expressionResolver.resolve(context, Arrays.asList("<.marvel-{now/d{}>")));
+        assertThat(e.getMessage(), containsString("invalid dynamic name expression"));
+        assertThat(e.getMessage(), containsString("date math placeholder is open ended"));
     }
 
     public void testExpressionInvalidEmptyDateMathFormat() throws Exception {
-        try {
-            expressionResolver.resolve(context, Arrays.asList("<.marvel-{now/d{}}>"));
-            fail("Expected ElasticsearchParseException");
-        } catch (ElasticsearchParseException e) {
-            assertThat(e.getMessage(), containsString("invalid dynamic name expression"));
-            assertThat(e.getMessage(), containsString("missing date format"));
-        }
+        Exception e = expectThrows(ElasticsearchParseException.class,
+                () -> expressionResolver.resolve(context, Arrays.asList("<.marvel-{now/d{}}>")));
+        assertThat(e.getMessage(), containsString("invalid dynamic name expression"));
+        assertThat(e.getMessage(), containsString("missing date format"));
     }
 
     public void testExpressionInvalidOpenEnded() throws Exception {
-        try {
-            expressionResolver.resolve(context, Arrays.asList("<.marvel-{now/d>"));
-            fail("Expected ElasticsearchParseException");
-        } catch (ElasticsearchParseException e) {
-            assertThat(e.getMessage(), containsString("invalid dynamic name expression"));
-            assertThat(e.getMessage(), containsString("date math placeholder is open ended"));
-        }
+        Exception e = expectThrows(ElasticsearchParseException.class,
+                () -> expressionResolver.resolve(context, Arrays.asList("<.marvel-{now/d>")));
+        assertThat(e.getMessage(), containsString("invalid dynamic name expression"));
+        assertThat(e.getMessage(), containsString("date math placeholder is open ended"));
     }
 
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexGraveyardTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexGraveyardTests.java
index 8dd950ba8e6ac..9178d112acade 100644
--- a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexGraveyardTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexGraveyardTests.java
@@ -20,19 +20,16 @@
 package org.elasticsearch.cluster.metadata;
 
 import org.elasticsearch.common.UUIDs;
-import org.elasticsearch.common.io.stream.ByteBufferStreamInput;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
-import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
@@ -52,7 +49,7 @@ public void testEquals() {
         final IndexGraveyard graveyard = createRandom();
         assertThat(graveyard, equalTo(IndexGraveyard.builder(graveyard).build()));
         final IndexGraveyard.Builder newGraveyard = IndexGraveyard.builder(graveyard);
-        newGraveyard.addTombstone(new Index(randomAsciiOfLengthBetween(4, 15), UUIDs.randomBase64UUID()));
+        newGraveyard.addTombstone(new Index(randomAlphaOfLengthBetween(4, 15), UUIDs.randomBase64UUID()));
         assertThat(newGraveyard.build(), not(graveyard));
     }
 
@@ -60,7 +57,7 @@ public void testSerialization() throws IOException {
         final IndexGraveyard graveyard = createRandom();
         final BytesStreamOutput out = new BytesStreamOutput();
         graveyard.writeTo(out);
-        assertThat(IndexGraveyard.fromStream(out.bytes().streamInput()), equalTo(graveyard));
+        assertThat(new IndexGraveyard(out.bytes().streamInput()), equalTo(graveyard));
     }
 
     public void testXContent() throws IOException {
@@ -69,9 +66,9 @@ public void testXContent() throws IOException {
         builder.startObject();
         graveyard.toXContent(builder, ToXContent.EMPTY_PARAMS);
         builder.endObject();
-        XContentParser parser = XContentType.JSON.xContent().createParser(builder.bytes());
+        XContentParser parser = createParser(JsonXContent.jsonXContent, builder.bytes());
         parser.nextToken(); // the beginning of the parser
-        assertThat(IndexGraveyard.PROTO.fromXContent(parser), equalTo(graveyard));
+        assertThat(IndexGraveyard.fromXContent(parser), equalTo(graveyard));
     }
 
     public void testAddTombstones() {
@@ -143,7 +140,7 @@ public void testContains() {
         for (final Index index : indices) {
             assertTrue(indexGraveyard.containsIndex(index));
         }
-        assertFalse(indexGraveyard.containsIndex(new Index(randomAsciiOfLength(6), UUIDs.randomBase64UUID())));
+        assertFalse(indexGraveyard.containsIndex(new Index(randomAlphaOfLength(6), UUIDs.randomBase64UUID())));
     }
 
     public static IndexGraveyard createRandom() {
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java
index 5fef33be38873..45eca3b5c151e 100644
--- a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java
@@ -26,7 +26,6 @@
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESTestCase;
@@ -34,6 +33,9 @@
 import java.io.IOException;
 import java.util.Set;
 
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.nullValue;
+
 public class IndexMetaDataTests extends ESTestCase {
 
     public void testIndexMetaDataSerialization() throws IOException {
@@ -54,8 +56,8 @@ public void testIndexMetaDataSerialization() throws IOException {
         builder.startObject();
         metaData.toXContent(builder, ToXContent.EMPTY_PARAMS);
         builder.endObject();
-        XContentParser parser = XContentType.JSON.xContent().createParser(builder.bytes());
-        final IndexMetaData fromXContentMeta = IndexMetaData.PROTO.fromXContent(parser, null);
+        XContentParser parser = createParser(JsonXContent.jsonXContent, builder.bytes());
+        final IndexMetaData fromXContentMeta = IndexMetaData.fromXContent(parser);
         assertEquals(metaData, fromXContentMeta);
         assertEquals(metaData.hashCode(), fromXContentMeta.hashCode());
 
@@ -69,7 +71,7 @@ public void testIndexMetaDataSerialization() throws IOException {
 
         final BytesStreamOutput out = new BytesStreamOutput();
         metaData.writeTo(out);
-        IndexMetaData deserialized = IndexMetaData.PROTO.readFrom(out.bytes().streamInput());
+        IndexMetaData deserialized = IndexMetaData.readFrom(out.bytes().streamInput());
         assertEquals(metaData, deserialized);
         assertEquals(metaData.hashCode(), deserialized.hashCode());
 
@@ -123,4 +125,36 @@ public void testSelectShrinkShards() {
         assertEquals("the number of target shards (8) must be greater than the shard id: 8",
             expectThrows(IllegalArgumentException.class, () -> IndexMetaData.selectShrinkShards(8, metaData, 8)).getMessage());
     }
+
+    public void testIndexFormat() {
+        Settings defaultSettings = Settings.builder()
+                .put("index.version.created", 1)
+                .put("index.number_of_shards", 1)
+                .put("index.number_of_replicas", 1)
+                .build();
+
+        // matching version
+        {
+            IndexMetaData metaData = IndexMetaData.builder("foo")
+                    .settings(Settings.builder()
+                            .put(defaultSettings)
+                            // intentionally not using the constant, so upgrading requires you to look at this test
+                            // where you have to update this part and the next one
+                            .put("index.format", 6)
+                            .build())
+                    .build();
+
+            assertThat(metaData.getSettings().getAsInt(IndexMetaData.INDEX_FORMAT_SETTING.getKey(), 0), is(6));
+        }
+
+        // no setting configured
+        {
+            IndexMetaData metaData = IndexMetaData.builder("foo")
+                    .settings(Settings.builder()
+                            .put(defaultSettings)
+                            .build())
+                    .build();
+            assertThat(metaData.getSettings().getAsInt(IndexMetaData.INDEX_FORMAT_SETTING.getKey(), 0), is(0));
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java
index 0e0c9fb442bc3..edbe5d6dc8a76 100644
--- a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java
@@ -307,7 +307,7 @@ public void testIndexOptionsWildcardExpansion() {
         assertEquals(1, results.length);
         assertEquals("bar", results[0]);
 
-        results = indexNameExpressionResolver.concreteIndexNames(context, "-foo*");
+        results = indexNameExpressionResolver.concreteIndexNames(context, "*", "-foo*");
         assertEquals(1, results.length);
         assertEquals("bar", results[0]);
 
@@ -587,6 +587,64 @@ public void testConcreteIndicesWildcardExpansion() {
         assertThat(newHashSet(indexNameExpressionResolver.concreteIndexNames(context, "testX*")), equalTo(newHashSet("testXXX", "testXXY", "testXYY")));
     }
 
+    public void testConcreteIndicesWildcardWithNegation() {
+        MetaData.Builder mdBuilder = MetaData.builder()
+                .put(indexBuilder("testXXX").state(State.OPEN))
+                .put(indexBuilder("testXXY").state(State.OPEN))
+                .put(indexBuilder("testXYY").state(State.OPEN))
+                .put(indexBuilder("-testXYZ").state(State.OPEN))
+                .put(indexBuilder("-testXZZ").state(State.OPEN))
+                .put(indexBuilder("-testYYY").state(State.OPEN))
+                .put(indexBuilder("testYYY").state(State.OPEN))
+                .put(indexBuilder("testYYX").state(State.OPEN));
+        ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build();
+
+        IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state,
+                IndicesOptions.fromOptions(true, true, true, true));
+        assertThat(newHashSet(indexNameExpressionResolver.concreteIndexNames(context, "testX*")),
+                equalTo(newHashSet("testXXX", "testXXY", "testXYY")));
+
+        assertThat(newHashSet(indexNameExpressionResolver.concreteIndexNames(context, "test*", "-testX*")),
+                equalTo(newHashSet("testYYY", "testYYX")));
+
+        assertThat(newHashSet(indexNameExpressionResolver.concreteIndexNames(context, "-testX*")),
+                equalTo(newHashSet("-testXYZ", "-testXZZ")));
+
+        assertThat(newHashSet(indexNameExpressionResolver.concreteIndexNames(context, "testXXY", "-testX*")),
+                equalTo(newHashSet("testXXY", "-testXYZ", "-testXZZ")));
+
+        assertThat(newHashSet(indexNameExpressionResolver.concreteIndexNames(context, "*", "--testX*")),
+                equalTo(newHashSet("testXXX", "testXXY", "testXYY", "testYYX", "testYYY", "-testYYY")));
+
+        assertThat(newHashSet(indexNameExpressionResolver.concreteIndexNames(context, "-testXXX", "test*")),
+                equalTo(newHashSet("testYYX", "testXXX", "testXYY", "testYYY", "testXXY")));
+
+        assertThat(newHashSet(indexNameExpressionResolver.concreteIndexNames(context, "test*", "-testXXX")),
+                equalTo(newHashSet("testYYX", "testXYY", "testYYY", "testXXY")));
+
+        assertThat(newHashSet(indexNameExpressionResolver.concreteIndexNames(context, "testXXX", "testXXY", "testYYY", "-testYYY")),
+                equalTo(newHashSet("testXXX", "testXXY", "testYYY", "-testYYY")));
+
+        assertThat(newHashSet(indexNameExpressionResolver.concreteIndexNames(context, "testYYY", "testYYX", "testX*", "-testXXX")),
+                equalTo(newHashSet("testYYY", "testYYX", "testXXY", "testXYY")));
+
+        assertThat(newHashSet(indexNameExpressionResolver.concreteIndexNames(context, "-testXXX", "*testY*", "-testYYY")),
+                equalTo(newHashSet("testYYX", "testYYY", "-testYYY")));
+
+        String[] indexNames = indexNameExpressionResolver.concreteIndexNames(state, IndicesOptions.lenientExpandOpen(), "-doesnotexist");
+        assertEquals(0, indexNames.length);
+
+        assertThat(newHashSet(indexNameExpressionResolver.concreteIndexNames(state, IndicesOptions.lenientExpandOpen(), "-*")),
+                equalTo(newHashSet("-testXYZ", "-testXZZ", "-testYYY")));
+
+        assertThat(newHashSet(indexNameExpressionResolver.concreteIndexNames(state, IndicesOptions.lenientExpandOpen(),
+                                "testXXX", "testXXY", "testXYY", "-testXXY")),
+                equalTo(newHashSet("testXXX", "testXYY", "testXXY")));
+
+        indexNames = indexNameExpressionResolver.concreteIndexNames(state, IndicesOptions.lenientExpandOpen(), "*", "-*");
+        assertEquals(0, indexNames.length);
+    }
+
     /**
      * test resolving _all pattern (null, empty array or "_all") for random IndicesOptions
      */
@@ -760,7 +818,7 @@ public void testIsPatternMatchingAllIndicesNonMatchingTrailingWildcard() throws
     }
 
     public void testIsPatternMatchingAllIndicesMatchingSingleExclusion() throws Exception {
-        String[] indicesOrAliases = new String[]{"-index1", "+index1"};
+        String[] indicesOrAliases = new String[]{"-index1", "index1"};
         String[] concreteIndices = new String[]{"index1", "index2", "index3"};
         MetaData metaData = metaDataBuilder(concreteIndices);
         assertThat(indexNameExpressionResolver.isPatternMatchingAllIndices(metaData, indicesOrAliases, concreteIndices), equalTo(true));
@@ -775,7 +833,7 @@ public void testIsPatternMatchingAllIndicesNonMatchingSingleExclusion() throws E
     }
 
     public void testIsPatternMatchingAllIndicesMatchingTrailingWildcardAndExclusion() throws Exception {
-        String[] indicesOrAliases = new String[]{"index*", "-index1", "+index1"};
+        String[] indicesOrAliases = new String[]{"index*", "-index1", "index1"};
         String[] concreteIndices = new String[]{"index1", "index2", "index3"};
         MetaData metaData = metaDataBuilder(concreteIndices);
         assertThat(indexNameExpressionResolver.isPatternMatchingAllIndices(metaData, indicesOrAliases, concreteIndices), equalTo(true));
@@ -900,4 +958,35 @@ public void testFilteringAliases() {
         strings = indexNameExpressionResolver.filteringAliases(state, "test-0", "test-*,alias-*");
         assertNull(strings);
     }
+
+    public void testIndexAliases() {
+        MetaData.Builder mdBuilder = MetaData.builder()
+            .put(indexBuilder("test-0").state(State.OPEN)
+                .putAlias(AliasMetaData.builder("test-alias-0").filter("{ \"term\": \"foo\"}"))
+                .putAlias(AliasMetaData.builder("test-alias-1").filter("{ \"term\": \"foo\"}"))
+                .putAlias(AliasMetaData.builder("test-alias-non-filtering"))
+            );
+        ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build();
+        String[] strings = indexNameExpressionResolver.indexAliases(state, "test-0", x -> true, true, "test-*");
+        Arrays.sort(strings);
+        assertArrayEquals(new String[] {"test-alias-0", "test-alias-1", "test-alias-non-filtering"}, strings);
+    }
+
+    public void testConcreteIndicesForDeprecatedPattern() {
+        MetaData.Builder mdBuilder = MetaData.builder()
+            .put(indexBuilder("testXXX").state(State.OPEN))
+            .put(indexBuilder("testXXY").state(State.OPEN))
+            .put(indexBuilder("testYYY").state(State.OPEN));
+        ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build();
+
+        IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state,
+            IndicesOptions.fromOptions(true, true, true, true));
+        assertThat(newHashSet(indexNameExpressionResolver.concreteIndexNames(context, "+testX*")),
+            equalTo(newHashSet("testXXX", "testXXY")));
+        assertThat(newHashSet(indexNameExpressionResolver.concreteIndexNames(context, "+testXXX", "+testXXY", "+testYYY", "-testYYY")),
+            equalTo(newHashSet("testXXX", "testXXY", "testYYY")));
+        assertThat(newHashSet(indexNameExpressionResolver.concreteIndexNames(context, "+testXX*", "+testY*")),
+            equalTo(newHashSet("testXXX", "testXXY", "testYYY")));
+        assertWarnings("support for '+' as part of index expressions is deprecated");
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaDataTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaDataTests.java
new file mode 100644
index 0000000000000..d29263fde71d1
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaDataTests.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.cluster.metadata;
+
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.test.ESTestCase;
+
+import static java.util.Collections.singletonMap;
+import static org.hamcrest.CoreMatchers.equalTo;
+
+public class IndexTemplateMetaDataTests extends ESTestCase {
+
+    public void testIndexTemplateMetaDataXContentRoundTrip() throws Exception {
+        ToXContent.Params params = new ToXContent.MapParams(singletonMap("reduce_mappings", "true"));
+
+        String template = "{\"template\" : [ \".test-*\" ],\"order\" : 1000," +
+            "\"settings\" : {\"number_of_shards\" : 1,\"number_of_replicas\" : 0}," +
+            "\"mappings\" : {\"doc\" :" +
+            "{\"properties\":{\"" +
+            randomAlphaOfLength(10) + "\":{\"type\":\"text\"},\"" +
+            randomAlphaOfLength(10) + "\":{\"type\":\"keyword\"}}" +
+            "}}}";
+
+        BytesReference templateBytes = new BytesArray(template);
+        final IndexTemplateMetaData indexTemplateMetaData;
+        try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, templateBytes, XContentType.JSON)) {
+            indexTemplateMetaData = IndexTemplateMetaData.Builder.fromXContent(parser, "test");
+        }
+
+        final BytesReference templateBytesRoundTrip;
+        try (XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent)) {
+            builder.startObject();
+            IndexTemplateMetaData.Builder.toXContent(indexTemplateMetaData, builder, params);
+            builder.endObject();
+            templateBytesRoundTrip = builder.bytes();
+        }
+
+        final IndexTemplateMetaData indexTemplateMetaDataRoundTrip;
+        try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, templateBytesRoundTrip, XContentType.JSON)) {
+            indexTemplateMetaDataRoundTrip = IndexTemplateMetaData.Builder.fromXContent(parser, "test");
+        }
+        assertThat(indexTemplateMetaData, equalTo(indexTemplateMetaDataRoundTrip));
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java
index de82ce8d073f4..515c94ae5382c 100644
--- a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java
@@ -36,16 +36,19 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.index.IndexNotFoundException;
-import org.elasticsearch.indices.IndexAlreadyExistsException;
+import org.elasticsearch.ResourceAlreadyExistsException;
 import org.elasticsearch.indices.InvalidIndexNameException;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.gateway.NoopGatewayAllocator;
+import org.elasticsearch.test.VersionUtils;
+import org.elasticsearch.test.gateway.TestGatewayAllocator;
 
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
+import java.util.List;
 
 import static java.util.Collections.emptyMap;
+import static java.util.Collections.min;
 import static org.hamcrest.Matchers.endsWith;
 
 public class MetaDataCreateIndexServiceTests extends ESTestCase {
@@ -79,7 +82,7 @@ public void testValidateShrinkIndex() {
             Settings.builder().put("index.blocks.write", true).build());
 
         assertEquals("index [source] already exists",
-            expectThrows(IndexAlreadyExistsException.class, () ->
+            expectThrows(ResourceAlreadyExistsException.class, () ->
                 MetaDataCreateIndexService.validateShrinkIndex(state, "target", Collections.emptySet(), "source", Settings.EMPTY)
             ).getMessage());
 
@@ -133,7 +136,7 @@ public void testValidateShrinkIndex() {
             .build();
         AllocationService service = new AllocationService(Settings.builder().build(), new AllocationDeciders(Settings.EMPTY,
             Collections.singleton(new MaxRetryAllocationDecider(Settings.EMPTY))),
-            NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
+            new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
 
         RoutingTable routingTable = service.reroute(clusterState, "reroute").routingTable();
         clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
@@ -150,18 +153,27 @@ public void testValidateShrinkIndex() {
     }
 
     public void testShrinkIndexSettings() {
-        String indexName = randomAsciiOfLength(10);
+        String indexName = randomAlphaOfLength(10);
+        List<Version> versions = Arrays.asList(VersionUtils.randomVersion(random()), VersionUtils.randomVersion(random()),
+            VersionUtils.randomVersion(random()));
+        versions.sort((l, r) -> Long.compare(l.id, r.id));
+        Version version = versions.get(0);
+        Version minCompat = versions.get(1);
+        Version upgraded = versions.get(2);
         // create one that won't fail
         ClusterState clusterState = ClusterState.builder(createClusterState(indexName, randomIntBetween(2, 10), 0,
             Settings.builder()
                 .put("index.blocks.write", true)
                 .put("index.similarity.default.type", "BM25")
+                .put("index.version.created", version)
+                .put("index.version.upgraded", upgraded)
+                .put("index.version.minimum_compatible", minCompat.luceneVersion)
                 .put("index.analysis.analyzer.my_analyzer.tokenizer", "keyword")
                 .build())).nodes(DiscoveryNodes.builder().add(newNode("node1")))
             .build();
         AllocationService service = new AllocationService(Settings.builder().build(), new AllocationDeciders(Settings.EMPTY,
             Collections.singleton(new MaxRetryAllocationDecider(Settings.EMPTY))),
-            NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
+            new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
 
         RoutingTable routingTable = service.reroute(clusterState, "reroute").routingTable();
         clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
@@ -178,6 +190,8 @@ public void testShrinkIndexSettings() {
             "keyword", builder.build().get("index.analysis.analyzer.my_analyzer.tokenizer"));
         assertEquals("node1", builder.build().get("index.routing.allocation.initial_recovery._id"));
         assertEquals("1", builder.build().get("index.allocation.max_retries"));
+        assertEquals(version, builder.build().getAsVersion("index.version.created", null));
+        assertEquals(upgraded, builder.build().getAsVersion("index.version.upgraded", null));
     }
 
     private DiscoveryNode newNode(String nodeId) {
@@ -207,17 +221,4 @@ private void validateIndexName(String indexName, String errorMessage) {
                 .getDefault(Settings.EMPTY)).build()));
         assertThat(e.getMessage(), endsWith(errorMessage));
     }
-
-    private MetaDataCreateIndexService getCreateIndexService() {
-        return new MetaDataCreateIndexService(
-            Settings.EMPTY,
-            null,
-            null,
-            null,
-            null,
-            null,
-            null,
-            null,
-            null);
-    }
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataDeleteIndexServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataDeleteIndexServiceTests.java
index f172b7ffdfc37..d65a35b8c26c6 100644
--- a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataDeleteIndexServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataDeleteIndexServiceTests.java
@@ -25,7 +25,6 @@
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
-import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
@@ -56,11 +55,11 @@ public void testDeleteMissing() {
     }
 
     public void testDeleteSnapshotting() {
-        String index = randomAsciiOfLength(5);
+        String index = randomAlphaOfLength(5);
         Snapshot snapshot = new Snapshot("doesn't matter", new SnapshotId("snapshot name", "snapshot uuid"));
         SnapshotsInProgress snaps = new SnapshotsInProgress(new SnapshotsInProgress.Entry(snapshot, true, false,
                 SnapshotsInProgress.State.INIT, singletonList(new IndexId(index, "doesn't matter")),
-                System.currentTimeMillis(), ImmutableOpenMap.of()));
+                System.currentTimeMillis(), (long) randomIntBetween(0, 1000), ImmutableOpenMap.of()));
         ClusterState state = ClusterState.builder(clusterState(index))
                 .putCustom(SnapshotsInProgress.TYPE, snaps)
                 .build();
@@ -72,12 +71,11 @@ SnapshotsInProgress.State.INIT, singletonList(new IndexId(index, "doesn't matter
 
     public void testDeleteUnassigned() {
         // Create an unassigned index
-        String index = randomAsciiOfLength(5);
+        String index = randomAlphaOfLength(5);
         ClusterState before = clusterState(index);
 
         // Mock the built reroute
-        when(allocationService.reroute(any(ClusterState.class), any(String.class))).then(
-                i -> RoutingAllocation.Result.unchanged((ClusterState) i.getArguments()[0]));
+        when(allocationService.reroute(any(ClusterState.class), any(String.class))).then(i -> i.getArguments()[0]);
 
         // Remove it
         ClusterState after = service.deleteIndices(before, singleton(before.metaData().getIndices().get(index).getIndex()));
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesServiceTests.java
index 03f62830ba720..e5b52d8cf52bf 100644
--- a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesServiceTests.java
@@ -33,7 +33,7 @@
 import static java.util.Collections.singletonList;
 import static org.hamcrest.Matchers.contains;
 import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyCollectionOf;
+import static org.mockito.Matchers.anySetOf;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -41,11 +41,11 @@ public class MetaDataIndexAliasesServiceTests extends ESTestCase {
     private final AliasValidator aliasValidator = new AliasValidator(Settings.EMPTY);
     private final MetaDataDeleteIndexService deleteIndexService = mock(MetaDataDeleteIndexService.class);
     private final MetaDataIndexAliasesService service = new MetaDataIndexAliasesService(Settings.EMPTY, null, null, aliasValidator,
-            null, deleteIndexService);
+            deleteIndexService, xContentRegistry());
 
     public MetaDataIndexAliasesServiceTests() {
         // Mock any deletes so we don't need to worry about how MetaDataDeleteIndexService does its job
-        when(deleteIndexService.deleteIndices(any(ClusterState.class), anyCollectionOf(Index.class))).then(i -> {
+        when(deleteIndexService.deleteIndices(any(ClusterState.class), anySetOf(Index.class))).then(i -> {
             ClusterState state = (ClusterState) i.getArguments()[0];
             @SuppressWarnings("unchecked")
             Collection<Index> indices = (Collection<Index>) i.getArguments()[1];
@@ -60,7 +60,7 @@ public MetaDataIndexAliasesServiceTests() {
 
     public void testAddAndRemove() {
         // Create a state with a single index
-        String index = randomAsciiOfLength(5);
+        String index = randomAlphaOfLength(5);
         ClusterState before = createIndex(ClusterState.builder(ClusterName.DEFAULT).build(), index);
 
         // Add an alias to it
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeServiceTests.java
index 52c52242c0f6e..5e26672244ae4 100644
--- a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeServiceTests.java
@@ -30,7 +30,9 @@
 public class MetaDataIndexUpgradeServiceTests extends ESTestCase {
 
     public void testArchiveBrokenIndexSettings() {
-        MetaDataIndexUpgradeService service = new MetaDataIndexUpgradeService(Settings.EMPTY, new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS);
+        MetaDataIndexUpgradeService service = new MetaDataIndexUpgradeService(Settings.EMPTY, xContentRegistry(),
+            new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
+            Collections.emptyList());
         IndexMetaData src = newIndexMeta("foo", Settings.EMPTY);
         IndexMetaData indexMetaData = service.archiveBrokenIndexSettings(src);
         assertSame(indexMetaData, src);
@@ -56,18 +58,22 @@ public void testArchiveBrokenIndexSettings() {
     }
 
     public void testUpgrade() {
-        MetaDataIndexUpgradeService service = new MetaDataIndexUpgradeService(Settings.EMPTY, new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS);
+        MetaDataIndexUpgradeService service = new MetaDataIndexUpgradeService(Settings.EMPTY, xContentRegistry(),
+            new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
+            Collections.emptyList());
         IndexMetaData src = newIndexMeta("foo", Settings.builder().put("index.refresh_interval", "-200").build());
         assertFalse(service.isUpgraded(src));
-        src = service.upgradeIndexMetaData(src);
+        src = service.upgradeIndexMetaData(src, Version.CURRENT.minimumIndexCompatibilityVersion());
         assertTrue(service.isUpgraded(src));
         assertEquals("-200", src.getSettings().get("archived.index.refresh_interval"));
         assertNull(src.getSettings().get("index.refresh_interval"));
-        assertSame(src, service.upgradeIndexMetaData(src)); // no double upgrade
+        assertSame(src, service.upgradeIndexMetaData(src, Version.CURRENT.minimumIndexCompatibilityVersion())); // no double upgrade
     }
 
     public void testIsUpgraded() {
-        MetaDataIndexUpgradeService service = new MetaDataIndexUpgradeService(Settings.EMPTY, new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS);
+        MetaDataIndexUpgradeService service = new MetaDataIndexUpgradeService(Settings.EMPTY, xContentRegistry(),
+            new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
+            Collections.emptyList());
         IndexMetaData src = newIndexMeta("foo", Settings.builder().put("index.refresh_interval", "-200").build());
         assertFalse(service.isUpgraded(src));
         Version version = VersionUtils.randomVersionBetween(random(), VersionUtils.getFirstVersion(), VersionUtils.getPreviousVersion());
@@ -77,6 +83,56 @@ public void testIsUpgraded() {
         assertTrue(service.isUpgraded(src));
     }
 
+    public void testFailUpgrade() {
+        MetaDataIndexUpgradeService service = new MetaDataIndexUpgradeService(Settings.EMPTY, xContentRegistry(),
+            new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
+            Collections.emptyList());
+        final IndexMetaData metaData = newIndexMeta("foo", Settings.builder()
+            .put(IndexMetaData.SETTING_VERSION_UPGRADED, Version.V_2_0_0_beta1)
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.fromString("1.7.0")).build());
+        String message = expectThrows(IllegalStateException.class, () -> service.upgradeIndexMetaData(metaData,
+            Version.CURRENT.minimumIndexCompatibilityVersion())).getMessage();
+        assertEquals(message, "The index [[foo/BOOM]] was created with version [1.7.0] but the minimum compatible version is" +
+            " [2.0.0-beta1]. It should be re-indexed in Elasticsearch 2.x before upgrading to " + Version.CURRENT.toString() + ".");
+
+        IndexMetaData goodMeta = newIndexMeta("foo", Settings.builder()
+            .put(IndexMetaData.SETTING_VERSION_UPGRADED, Version.V_2_0_0_beta1)
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.fromString("2.1.0")).build());
+        service.upgradeIndexMetaData(goodMeta, Version.CURRENT.minimumIndexCompatibilityVersion());
+    }
+
+    public void testPluginUpgrade() {
+        MetaDataIndexUpgradeService service = new MetaDataIndexUpgradeService(Settings.EMPTY, xContentRegistry(),
+            new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
+            Collections.singletonList(
+                indexMetaData -> IndexMetaData.builder(indexMetaData)
+                    .settings(
+                        Settings.builder()
+                            .put(indexMetaData.getSettings())
+                            .put("index.refresh_interval", "10s")
+                    ).build()));
+        IndexMetaData src = newIndexMeta("foo", Settings.builder().put("index.refresh_interval", "200s").build());
+        assertFalse(service.isUpgraded(src));
+        src = service.upgradeIndexMetaData(src, Version.CURRENT.minimumIndexCompatibilityVersion());
+        assertTrue(service.isUpgraded(src));
+        assertEquals("10s", src.getSettings().get("index.refresh_interval"));
+        assertSame(src, service.upgradeIndexMetaData(src, Version.CURRENT.minimumIndexCompatibilityVersion())); // no double upgrade
+    }
+
+    public void testPluginUpgradeFailure() {
+        MetaDataIndexUpgradeService service = new MetaDataIndexUpgradeService(Settings.EMPTY, xContentRegistry(),
+            new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
+            Collections.singletonList(
+                indexMetaData -> {
+                    throw new IllegalStateException("Cannot upgrade index " + indexMetaData.getIndex().getName());
+                }
+            ));
+        IndexMetaData src = newIndexMeta("foo", Settings.EMPTY);
+        String message = expectThrows(IllegalStateException.class, () -> service.upgradeIndexMetaData(src,
+            Version.CURRENT.minimumIndexCompatibilityVersion())).getMessage();
+        assertEquals(message, "Cannot upgrade index foo");
+    }
+
     public static IndexMetaData newIndexMeta(String name, Settings indexSettings) {
         Settings build = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
             .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
@@ -89,5 +145,4 @@ public static IndexMetaData newIndexMeta(String name, Settings indexSettings) {
         IndexMetaData metaData = IndexMetaData.builder(name).settings(build).build();
         return metaData;
     }
-
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataMappingServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataMappingServiceTests.java
index c6ce30e2a52dd..c3e544f9b2b84 100644
--- a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataMappingServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataMappingServiceTests.java
@@ -18,11 +18,18 @@
  */
 package org.elasticsearch.cluster.metadata;
 
+import org.elasticsearch.action.admin.indices.mapping.put.PutMappingClusterStateUpdateRequest;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.mapper.DocumentMapper;
+import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
+import java.util.Collections;
+
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 
@@ -63,4 +70,43 @@ public void testAddExtraChildTypePointingToAlreadyParentExistingType() throws Ex
         assertThat(documentMapper.parentFieldMapper().active(), is(true));
     }
 
+    public void testParentIsAString() throws Exception {
+        // Shouldn't be able the add the _parent field pointing to an already existing type, which isn't a parent type
+        Exception e = expectThrows(MapperParsingException.class, () -> client().admin().indices().prepareCreate("test")
+                .addMapping("parent", "{\"properties\":{}}")
+                .addMapping("child", "{\"_parent\": \"parent\",\"properties\":{}}")
+                .get());
+        assertEquals("Failed to parse mapping [child]: [_parent] must be an object containing [type]", e.getMessage());
+    }
+
+    public void testMappingClusterStateUpdateDoesntChangeExistingIndices() throws Exception {
+        final IndexService indexService = createIndex("test", client().admin().indices().prepareCreate("test").addMapping("type"));
+        final CompressedXContent currentMapping = indexService.mapperService().documentMapper("type").mappingSource();
+
+        final MetaDataMappingService mappingService = getInstanceFromNode(MetaDataMappingService.class);
+        final ClusterService clusterService = getInstanceFromNode(ClusterService.class);
+        // TODO - it will be nice to get a random mapping generator
+        final PutMappingClusterStateUpdateRequest request = new PutMappingClusterStateUpdateRequest().type("type");
+        request.source("{ \"properties\" { \"field\": { \"type\": \"string\" }}}");
+        mappingService.putMappingExecutor.execute(clusterService.state(), Collections.singletonList(request));
+        assertThat(indexService.mapperService().documentMapper("type").mappingSource(), equalTo(currentMapping));
+    }
+
+    public void testClusterStateIsNotChangedWithIdenticalMappings() throws Exception {
+        createIndex("test", client().admin().indices().prepareCreate("test").addMapping("type"));
+
+        final MetaDataMappingService mappingService = getInstanceFromNode(MetaDataMappingService.class);
+        final ClusterService clusterService = getInstanceFromNode(ClusterService.class);
+        final PutMappingClusterStateUpdateRequest request = new PutMappingClusterStateUpdateRequest().type("type");
+        request.source("{ \"properties\" { \"field\": { \"type\": \"string\" }}}");
+        ClusterState result = mappingService.putMappingExecutor.execute(clusterService.state(), Collections.singletonList(request))
+            .resultingState;
+
+        assertFalse(result != clusterService.state());
+
+        ClusterState result2 = mappingService.putMappingExecutor.execute(result, Collections.singletonList(request))
+            .resultingState;
+
+        assertSame(result, result2);
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java
index cf040fb3c7f7f..dd7683c1de213 100644
--- a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java
@@ -20,22 +20,29 @@
 package org.elasticsearch.cluster.metadata;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.startsWith;
 
 public class MetaDataTests extends ESTestCase {
 
@@ -49,7 +56,42 @@ public void testIndexAndAliasWithSameName() {
             MetaData.builder().put(builder).build();
             fail("exception should have been thrown");
         } catch (IllegalStateException e) {
-            assertThat(e.getMessage(), equalTo("index and alias names need to be unique, but alias [index] and index [index] have the same name"));
+            assertThat(e.getMessage(), equalTo("index and alias names need to be unique, but the following duplicates were found [index (alias of [index])]"));
+        }
+    }
+
+    public void testAliasCollidingWithAnExistingIndex() {
+        int indexCount = randomIntBetween(10, 100);
+        Set<String> indices = new HashSet<>(indexCount);
+        for (int i = 0; i < indexCount; i++) {
+            indices.add(randomAlphaOfLength(10));
+        }
+        Map<String, Set<String>> aliasToIndices = new HashMap<>();
+        for (String alias: randomSubsetOf(randomIntBetween(1, 10), indices)) {
+            aliasToIndices.put(alias, new HashSet<>(randomSubsetOf(randomIntBetween(1, 3), indices)));
+        }
+        int properAliases = randomIntBetween(0, 3);
+        for (int i = 0; i < properAliases; i++) {
+            aliasToIndices.put(randomAlphaOfLength(5), new HashSet<>(randomSubsetOf(randomIntBetween(1, 3), indices)));
+        }
+        MetaData.Builder metaDataBuilder = MetaData.builder();
+        for (String index : indices) {
+            IndexMetaData.Builder indexBuilder = IndexMetaData.builder(index)
+                .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT))
+                .numberOfShards(1)
+                .numberOfReplicas(0);
+            aliasToIndices.forEach((key, value) -> {
+                if (value.contains(index)) {
+                    indexBuilder.putAlias(AliasMetaData.builder(key).build());
+                }
+            });
+            metaDataBuilder.put(indexBuilder);
+        }
+        try {
+            metaDataBuilder.build();
+            fail("exception should have been thrown");
+        } catch (IllegalStateException e) {
+            assertThat(e.getMessage(), startsWith("index and alias names need to be unique"));
         }
     }
 
@@ -129,7 +171,7 @@ public void testUnknownFieldClusterMetaData() throws IOException {
                     .field("random", "value")
                 .endObject()
             .endObject().bytes();
-        XContentParser parser = JsonXContent.jsonXContent.createParser(metadata);
+        XContentParser parser = createParser(JsonXContent.jsonXContent, metadata);
         try {
             MetaData.Builder.fromXContent(parser);
             fail();
@@ -145,7 +187,7 @@ public void testUnknownFieldIndexMetaData() throws IOException {
                     .field("random", "value")
                 .endObject()
             .endObject().bytes();
-        XContentParser parser = JsonXContent.jsonXContent.createParser(metadata);
+        XContentParser parser = createParser(JsonXContent.jsonXContent, metadata);
         try {
             IndexMetaData.Builder.fromXContent(parser);
             fail();
@@ -173,8 +215,8 @@ public void testXContentWithIndexGraveyard() throws IOException {
         builder.startObject();
         originalMeta.toXContent(builder, ToXContent.EMPTY_PARAMS);
         builder.endObject();
-        XContentParser parser = XContentType.JSON.xContent().createParser(builder.bytes());
-        final MetaData fromXContentMeta = MetaData.PROTO.fromXContent(parser, null);
+        XContentParser parser = createParser(JsonXContent.jsonXContent, builder.bytes());
+        final MetaData fromXContentMeta = MetaData.fromXContent(parser);
         assertThat(fromXContentMeta.indexGraveyard(), equalTo(originalMeta.indexGraveyard()));
     }
 
@@ -183,7 +225,10 @@ public void testSerializationWithIndexGraveyard() throws IOException {
         final MetaData originalMeta = MetaData.builder().indexGraveyard(graveyard).build();
         final BytesStreamOutput out = new BytesStreamOutput();
         originalMeta.writeTo(out);
-        final MetaData fromStreamMeta = MetaData.PROTO.readFrom(out.bytes().streamInput());
+        NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(ClusterModule.getNamedWriteables());
+        final MetaData fromStreamMeta = MetaData.readFrom(
+            new NamedWriteableAwareStreamInput(out.bytes().streamInput(), namedWriteableRegistry)
+        );
         assertThat(fromStreamMeta.indexGraveyard(), equalTo(fromStreamMeta.indexGraveyard()));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceIT.java b/core/src/test/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceIT.java
new file mode 100644
index 0000000000000..a40c1fa02e5d3
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceIT.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.metadata;
+
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.watcher.ResourceWatcherService;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.UnaryOperator;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST)
+public class TemplateUpgradeServiceIT extends ESIntegTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Collections.singletonList(TestPlugin.class);
+    }
+
+    public static class TestPlugin extends Plugin {
+        // This setting is used to simulate cluster state updates
+        static final Setting<Integer> UPDATE_TEMPLATE_DUMMY_SETTING =
+            Setting.intSetting("tests.update_template_count", 0, Setting.Property.NodeScope, Setting.Property.Dynamic);
+
+        protected final Logger logger;
+        protected final Settings settings;
+
+        public TestPlugin(Settings settings) {
+            this.logger = Loggers.getLogger(getClass(), settings);
+            this.settings = settings;
+        }
+
+        @Override
+        public Collection<Object> createComponents(Client client, ClusterService clusterService, ThreadPool threadPool,
+                                                   ResourceWatcherService resourceWatcherService, ScriptService scriptService,
+                                                   NamedXContentRegistry xContentRegistry) {
+            clusterService.getClusterSettings().addSettingsUpdateConsumer(UPDATE_TEMPLATE_DUMMY_SETTING, integer -> {
+                logger.debug("the template dummy setting was updated to {}", integer);
+            });
+            return super.createComponents(client, clusterService, threadPool, resourceWatcherService, scriptService, xContentRegistry);
+        }
+
+        @Override
+        public UnaryOperator<Map<String, IndexTemplateMetaData>> getIndexTemplateMetaDataUpgrader() {
+            return templates -> {
+                templates.put("test_added_template", IndexTemplateMetaData.builder("test_added_template")
+                    .template("*").build());
+                templates.remove("test_removed_template");
+                templates.put("test_changed_template", IndexTemplateMetaData.builder("test_changed_template").order(10)
+                    .template("*").build());
+                return templates;
+            };
+        }
+
+        @Override
+        public List<Setting<?>> getSettings() {
+            return Collections.singletonList(UPDATE_TEMPLATE_DUMMY_SETTING);
+        }
+    }
+
+
+    public void testTemplateUpdate() throws Exception {
+        assertTemplates();
+
+        // Change some templates
+        assertAcked(client().admin().indices().preparePutTemplate("test_dummy_template").setOrder(0)
+            .setTemplate("*").get());
+        assertAcked(client().admin().indices().preparePutTemplate("test_changed_template").setOrder(0)
+            .setTemplate("*").get());
+        assertAcked(client().admin().indices().preparePutTemplate("test_removed_template").setOrder(1)
+            .setTemplate("*").get());
+
+        AtomicInteger updateCount = new AtomicInteger();
+        // Wait for the templates to be updated back to normal
+        assertBusy(() -> {
+            // the updates only happen on cluster state updates, so we need to make sure that the cluster state updates are happening
+            // so we need to simulate updates to make sure the template upgrade kicks in
+            assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(
+                Settings.builder().put(TestPlugin.UPDATE_TEMPLATE_DUMMY_SETTING.getKey(), updateCount.incrementAndGet())
+            ).get());
+            List<IndexTemplateMetaData> templates = client().admin().indices().prepareGetTemplates("test_*").get().getIndexTemplates();
+            assertThat(templates, hasSize(3));
+            boolean addedFound = false;
+            boolean changedFound = false;
+            boolean dummyFound = false;
+            for (int i = 0; i < 3; i++) {
+                IndexTemplateMetaData templateMetaData = templates.get(i);
+                switch (templateMetaData.getName()) {
+                    case "test_added_template":
+                        assertFalse(addedFound);
+                        addedFound = true;
+                        break;
+                    case "test_changed_template":
+                        assertFalse(changedFound);
+                        changedFound = true;
+                        assertThat(templateMetaData.getOrder(), equalTo(10));
+                        break;
+                    case "test_dummy_template":
+                        assertFalse(dummyFound);
+                        dummyFound = true;
+                        break;
+                    default:
+                        fail("unexpected template " + templateMetaData.getName());
+                        break;
+                }
+            }
+            assertTrue(addedFound);
+            assertTrue(changedFound);
+            assertTrue(dummyFound);
+        });
+
+        // Wipe out all templates
+        assertAcked(client().admin().indices().prepareDeleteTemplate("test_*").get());
+
+        assertTemplates();
+
+    }
+
+    private void assertTemplates() throws Exception {
+        AtomicInteger updateCount = new AtomicInteger();
+        // Make sure all templates are recreated correctly
+        assertBusy(() -> {
+            // the updates only happen on cluster state updates, so we need to make sure that the cluster state updates are happening
+            // so we need to simulate updates to make sure the template upgrade kicks in
+            assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(
+                Settings.builder().put(TestPlugin.UPDATE_TEMPLATE_DUMMY_SETTING.getKey(), updateCount.incrementAndGet())
+            ).get());
+
+            List<IndexTemplateMetaData> templates = client().admin().indices().prepareGetTemplates("test_*").get().getIndexTemplates();
+            assertThat(templates, hasSize(2));
+            boolean addedFound = false;
+            boolean changedFound = false;
+            for (int i = 0; i < 2; i++) {
+                IndexTemplateMetaData templateMetaData = templates.get(i);
+                switch (templateMetaData.getName()) {
+                    case "test_added_template":
+                        assertFalse(addedFound);
+                        addedFound = true;
+                        break;
+                    case "test_changed_template":
+                        assertFalse(changedFound);
+                        changedFound = true;
+                        assertThat(templateMetaData.getOrder(), equalTo(10));
+                        break;
+                    default:
+                        fail("unexpected template " + templateMetaData.getName());
+                        break;
+                }
+            }
+
+            assertTrue(addedFound);
+            assertTrue(changedFound);
+        });
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceTests.java
new file mode 100644
index 0000000000000..f492f14395595
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceTests.java
@@ -0,0 +1,447 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.metadata;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest;
+import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateResponse;
+import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
+import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
+import org.elasticsearch.client.AdminClient;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.IndicesAdminClient;
+import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.LocalTransportAddress;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.ThreadPool;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static java.util.Collections.emptyMap;
+import static org.elasticsearch.test.VersionUtils.randomVersion;
+import static org.hamcrest.CoreMatchers.nullValue;
+import static org.hamcrest.CoreMatchers.startsWith;
+import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.lessThan;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+
+public class TemplateUpgradeServiceTests extends ESTestCase {
+
+    private final ClusterService clusterService = new ClusterService(Settings.EMPTY, new ClusterSettings(Settings.EMPTY,
+        ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null, () ->
+        new DiscoveryNode(UUIDs.randomBase64UUID(), LocalTransportAddress.buildUnique(), Version.CURRENT));
+
+    public void testCalculateChangesAddChangeAndDelete() {
+
+        boolean shouldAdd = randomBoolean();
+        boolean shouldRemove = randomBoolean();
+        boolean shouldChange = randomBoolean();
+
+        MetaData metaData = randomMetaData(
+            IndexTemplateMetaData.builder("user_template").build(),
+            IndexTemplateMetaData.builder("removed_test_template").build(),
+            IndexTemplateMetaData.builder("changed_test_template").build()
+        );
+
+        TemplateUpgradeService service = new TemplateUpgradeService(Settings.EMPTY, null, clusterService, null,
+            Arrays.asList(
+                templates -> {
+                    if (shouldAdd) {
+                        assertNull(templates.put("added_test_template", IndexTemplateMetaData.builder("added_test_template").build()));
+                    }
+                    return templates;
+                },
+                templates -> {
+                    if (shouldRemove) {
+                        assertNotNull(templates.remove("removed_test_template"));
+                    }
+                    return templates;
+                },
+                templates -> {
+                    if (shouldChange) {
+                        assertNotNull(templates.put("changed_test_template",
+                            IndexTemplateMetaData.builder("changed_test_template").order(10).build()));
+                    }
+                    return templates;
+                }
+            ));
+
+        Optional<Tuple<Map<String, BytesReference>, Set<String>>> optChanges =
+            service.calculateTemplateChanges(metaData.templates());
+
+        if (shouldAdd || shouldRemove || shouldChange) {
+            Tuple<Map<String, BytesReference>, Set<String>> changes = optChanges.orElseThrow(() ->
+                new AssertionError("Should have non empty changes"));
+            if (shouldAdd) {
+                assertThat(changes.v1().get("added_test_template"), notNullValue());
+                if (shouldChange) {
+                    assertThat(changes.v1().keySet(), hasSize(2));
+                    assertThat(changes.v1().get("changed_test_template"), notNullValue());
+                } else {
+                    assertThat(changes.v1().keySet(), hasSize(1));
+                }
+            } else {
+                if (shouldChange) {
+                    assertThat(changes.v1().get("changed_test_template"), notNullValue());
+                    assertThat(changes.v1().keySet(), hasSize(1));
+                } else {
+                    assertThat(changes.v1().keySet(), empty());
+                }
+            }
+
+            if (shouldRemove) {
+                assertThat(changes.v2(), hasSize(1));
+                assertThat(changes.v2().contains("removed_test_template"), equalTo(true));
+            } else {
+                assertThat(changes.v2(), empty());
+            }
+        } else {
+            assertThat(optChanges.isPresent(), equalTo(false));
+        }
+    }
+
+
+    @SuppressWarnings("unchecked")
+    public void testUpdateTemplates() {
+        int additionsCount = randomIntBetween(0, 5);
+        int deletionsCount = randomIntBetween(0, 3);
+
+        List<ActionListener<PutIndexTemplateResponse>> putTemplateListeners = new ArrayList<>();
+        List<ActionListener<DeleteIndexTemplateResponse>> deleteTemplateListeners = new ArrayList<>();
+
+        Client mockClient = mock(Client.class);
+        AdminClient mockAdminClient = mock(AdminClient.class);
+        IndicesAdminClient mockIndicesAdminClient = mock(IndicesAdminClient.class);
+        when(mockClient.admin()).thenReturn(mockAdminClient);
+        when(mockAdminClient.indices()).thenReturn(mockIndicesAdminClient);
+
+        doAnswer(invocation -> {
+            Object[] args = invocation.getArguments();
+            assert args.length == 2;
+            PutIndexTemplateRequest request = (PutIndexTemplateRequest) args[0];
+            assertThat(request.name(), equalTo("add_template_" + request.order()));
+            putTemplateListeners.add((ActionListener) args[1]);
+            return null;
+        }).when(mockIndicesAdminClient).putTemplate(any(PutIndexTemplateRequest.class), any(ActionListener.class));
+
+        doAnswer(invocation -> {
+            Object[] args = invocation.getArguments();
+            assert args.length == 2;
+            DeleteIndexTemplateRequest request = (DeleteIndexTemplateRequest) args[0];
+            assertThat(request.name(), startsWith("remove_template_"));
+            deleteTemplateListeners.add((ActionListener) args[1]);
+            return null;
+        }).when(mockIndicesAdminClient).deleteTemplate(any(DeleteIndexTemplateRequest.class), any(ActionListener.class));
+
+        Set<String> deletions = new HashSet<>(deletionsCount);
+        for (int i = 0; i < deletionsCount; i++) {
+            deletions.add("remove_template_" + i);
+        }
+        Map<String, BytesReference> additions = new HashMap<>(additionsCount);
+        for (int i = 0; i < additionsCount; i++) {
+            additions.put("add_template_" + i, new BytesArray("{\"index_patterns\" : \"*\", \"order\" : " + i + "}"));
+        }
+
+        TemplateUpgradeService service = new TemplateUpgradeService(Settings.EMPTY, mockClient, clusterService, null,
+            Collections.emptyList());
+
+        service.updateTemplates(additions, deletions);
+        int updatesInProgress = service.getUpdatesInProgress();
+
+        assertThat(putTemplateListeners, hasSize(additionsCount));
+        assertThat(deleteTemplateListeners, hasSize(deletionsCount));
+
+        for (int i = 0; i < additionsCount; i++) {
+            if (randomBoolean()) {
+                putTemplateListeners.get(i).onFailure(new RuntimeException("test - ignore"));
+            } else {
+                putTemplateListeners.get(i).onResponse(new PutIndexTemplateResponse(randomBoolean()) {
+
+                });
+            }
+        }
+
+        for (int i = 0; i < deletionsCount; i++) {
+            if (randomBoolean()) {
+                int prevUpdatesInProgress = service.getUpdatesInProgress();
+                deleteTemplateListeners.get(i).onFailure(new RuntimeException("test - ignore"));
+                assertThat(prevUpdatesInProgress - service.getUpdatesInProgress(), equalTo(1));
+            } else {
+                int prevUpdatesInProgress = service.getUpdatesInProgress();
+                deleteTemplateListeners.get(i).onResponse(new DeleteIndexTemplateResponse(randomBoolean()) {
+
+                });
+                assertThat(prevUpdatesInProgress - service.getUpdatesInProgress(), equalTo(1));
+            }
+        }
+        assertThat(updatesInProgress - service.getUpdatesInProgress(), equalTo(additionsCount + deletionsCount));
+    }
+
+    private static final Set<DiscoveryNode.Role> MASTER_DATA_ROLES =
+        Collections.unmodifiableSet(EnumSet.of(DiscoveryNode.Role.MASTER, DiscoveryNode.Role.DATA));
+
+    @SuppressWarnings("unchecked")
+    public void testClusterStateUpdate() {
+
+        AtomicReference<ActionListener<PutIndexTemplateResponse>> addedListener = new AtomicReference<>();
+        AtomicReference<ActionListener<PutIndexTemplateResponse>> changedListener = new AtomicReference<>();
+        AtomicReference<ActionListener<DeleteIndexTemplateResponse>> removedListener = new AtomicReference<>();
+        AtomicInteger updateInvocation = new AtomicInteger();
+
+        MetaData metaData = randomMetaData(
+            IndexTemplateMetaData.builder("user_template").build(),
+            IndexTemplateMetaData.builder("removed_test_template").build(),
+            IndexTemplateMetaData.builder("changed_test_template").build()
+        );
+
+        ThreadPool threadPool = mock(ThreadPool.class);
+        ExecutorService executorService = mock(ExecutorService.class);
+        when(threadPool.generic()).thenReturn(executorService);
+        doAnswer(invocation -> {
+            Object[] args = invocation.getArguments();
+            assert args.length == 1;
+            Runnable runnable = (Runnable) args[0];
+            runnable.run();
+            updateInvocation.incrementAndGet();
+            return null;
+        }).when(executorService).execute(any(Runnable.class));
+
+        Client mockClient = mock(Client.class);
+        AdminClient mockAdminClient = mock(AdminClient.class);
+        IndicesAdminClient mockIndicesAdminClient = mock(IndicesAdminClient.class);
+        when(mockClient.admin()).thenReturn(mockAdminClient);
+        when(mockAdminClient.indices()).thenReturn(mockIndicesAdminClient);
+
+        doAnswer(invocation -> {
+            Object[] args = invocation.getArguments();
+            assert args.length == 2;
+            PutIndexTemplateRequest request = (PutIndexTemplateRequest) args[0];
+            if (request.name().equals("added_test_template")) {
+                assertThat(addedListener.getAndSet((ActionListener) args[1]), nullValue());
+            } else if (request.name().equals("changed_test_template")) {
+                assertThat(changedListener.getAndSet((ActionListener) args[1]), nullValue());
+            } else {
+                fail("unexpected put template call for " + request.name());
+            }
+            return null;
+        }).when(mockIndicesAdminClient).putTemplate(any(PutIndexTemplateRequest.class), any(ActionListener.class));
+
+        doAnswer(invocation -> {
+            Object[] args = invocation.getArguments();
+            assert args.length == 2;
+            DeleteIndexTemplateRequest request = (DeleteIndexTemplateRequest) args[0];
+            assertThat(request.name(), startsWith("removed_test_template"));
+            assertThat(removedListener.getAndSet((ActionListener) args[1]), nullValue());
+            return null;
+        }).when(mockIndicesAdminClient).deleteTemplate(any(DeleteIndexTemplateRequest.class), any(ActionListener.class));
+
+        TemplateUpgradeService service = new TemplateUpgradeService(Settings.EMPTY, mockClient, clusterService, threadPool,
+            Arrays.asList(
+                templates -> {
+                    assertNull(templates.put("added_test_template", IndexTemplateMetaData.builder("added_test_template")
+                        .template("*").build()));
+                    return templates;
+                },
+                templates -> {
+                    assertNotNull(templates.remove("removed_test_template"));
+                    return templates;
+                },
+                templates -> {
+                    assertNotNull(templates.put("changed_test_template", IndexTemplateMetaData.builder("changed_test_template")
+                        .template("*").order(10).build()));
+                    return templates;
+                }
+            ));
+
+        ClusterState prevState = ClusterState.EMPTY_STATE;
+        ClusterState state = ClusterState.builder(prevState).nodes(DiscoveryNodes.builder()
+            .add(new DiscoveryNode("node1", "node1", buildNewFakeTransportAddress(), emptyMap(), MASTER_DATA_ROLES, Version.CURRENT)
+            ).localNodeId("node1").masterNodeId("node1").build()
+        ).metaData(metaData).build();
+        service.clusterChanged(new ClusterChangedEvent("test", state, prevState));
+
+        assertThat(updateInvocation.get(), equalTo(1));
+        assertThat(addedListener.get(), notNullValue());
+        assertThat(changedListener.get(), notNullValue());
+        assertThat(removedListener.get(), notNullValue());
+
+        prevState = state;
+        state = ClusterState.builder(prevState).metaData(MetaData.builder(state.metaData()).removeTemplate("user_template")).build();
+        service.clusterChanged(new ClusterChangedEvent("test 2", state, prevState));
+
+        // Make sure that update wasn't invoked since we are still running
+        assertThat(updateInvocation.get(), equalTo(1));
+
+        addedListener.getAndSet(null).onResponse(new PutIndexTemplateResponse(true) {
+        });
+        changedListener.getAndSet(null).onResponse(new PutIndexTemplateResponse(true) {
+        });
+        removedListener.getAndSet(null).onResponse(new DeleteIndexTemplateResponse(true) {
+        });
+
+        service.clusterChanged(new ClusterChangedEvent("test 3", state, prevState));
+
+        // Make sure that update was called this time since we are no longer running
+        assertThat(updateInvocation.get(), equalTo(2));
+
+        addedListener.getAndSet(null).onFailure(new RuntimeException("test - ignore"));
+        changedListener.getAndSet(null).onFailure(new RuntimeException("test - ignore"));
+        removedListener.getAndSet(null).onFailure(new RuntimeException("test - ignore"));
+
+        service.clusterChanged(new ClusterChangedEvent("test 3", state, prevState));
+
+        // Make sure that update wasn't called this time since the index template metadata didn't change
+        assertThat(updateInvocation.get(), equalTo(2));
+    }
+
+    private static final int NODE_TEST_ITERS = 100;
+
+    public void testOnlyOneNodeRunsTemplateUpdates() {
+        TemplateUpgradeService service = new TemplateUpgradeService(Settings.EMPTY, null, clusterService, null, Collections.emptyList());
+        for (int i = 0; i < NODE_TEST_ITERS; i++) {
+            int nodesCount = randomIntBetween(1, 10);
+            int clientNodesCount = randomIntBetween(0, 4);
+            DiscoveryNodes nodes = randomNodes(nodesCount, clientNodesCount);
+            int updaterNode = -1;
+            for (int j = 0; j < nodesCount; j++) {
+                DiscoveryNodes localNodes = DiscoveryNodes.builder(nodes).localNodeId(nodes.resolveNode("node_" + j).getId()).build();
+                if (service.shouldLocalNodeUpdateTemplates(localNodes)) {
+                    assertThat("Expected only one node to update template, found " + updaterNode + " and " + j, updaterNode, lessThan(0));
+                    updaterNode = j;
+                }
+            }
+            assertThat("Expected one node to update template", updaterNode, greaterThanOrEqualTo(0));
+        }
+    }
+
+    public void testIfMasterHasTheHighestVersionItShouldRunsTemplateUpdates() {
+        for (int i = 0; i < NODE_TEST_ITERS; i++) {
+            int nodesCount = randomIntBetween(1, 10);
+            int clientNodesCount = randomIntBetween(0, 4);
+            DiscoveryNodes nodes = randomNodes(nodesCount, clientNodesCount);
+            DiscoveryNodes.Builder builder = DiscoveryNodes.builder(nodes).localNodeId(nodes.resolveNode("_master").getId());
+            nodes = builder.build();
+            TemplateUpgradeService service = new TemplateUpgradeService(Settings.EMPTY, null, clusterService, null,
+                Collections.emptyList());
+            assertThat(service.shouldLocalNodeUpdateTemplates(nodes),
+                equalTo(nodes.getLargestNonClientNodeVersion().equals(nodes.getMasterNode().getVersion())));
+        }
+    }
+
+    public void testClientNodeDontRunTemplateUpdates() {
+        for (int i = 0; i < NODE_TEST_ITERS; i++) {
+            int nodesCount = randomIntBetween(1, 10);
+            int clientNodesCount = randomIntBetween(1, 4);
+            DiscoveryNodes nodes = randomNodes(nodesCount, clientNodesCount);
+            int testClient = randomIntBetween(0, clientNodesCount - 1);
+            DiscoveryNodes.Builder builder = DiscoveryNodes.builder(nodes).localNodeId(nodes.resolveNode("client_" + testClient).getId());
+            TemplateUpgradeService service = new TemplateUpgradeService(Settings.EMPTY, null, clusterService, null,
+                Collections.emptyList());
+            assertThat(service.shouldLocalNodeUpdateTemplates(builder.build()), equalTo(false));
+        }
+    }
+
+    private DiscoveryNodes randomNodes(int dataAndMasterNodes, int clientNodes) {
+        DiscoveryNodes.Builder builder = DiscoveryNodes.builder();
+        String masterNodeId = null;
+        for (int i = 0; i < dataAndMasterNodes; i++) {
+            String id = randomAlphaOfLength(10) + "_" + i;
+            Set<DiscoveryNode.Role> roles;
+            if (i == 0) {
+                masterNodeId = id;
+                // The first node has to be master node
+                if (randomBoolean()) {
+                    roles = EnumSet.of(DiscoveryNode.Role.MASTER, DiscoveryNode.Role.DATA);
+                } else {
+                    roles = EnumSet.of(DiscoveryNode.Role.MASTER);
+                }
+            } else {
+                if (randomBoolean()) {
+                    roles = EnumSet.of(DiscoveryNode.Role.DATA);
+                } else {
+                    roles = EnumSet.of(DiscoveryNode.Role.MASTER);
+                }
+            }
+            String node = "node_" + i;
+            builder.add(new DiscoveryNode(node, id, buildNewFakeTransportAddress(), emptyMap(), roles, randomVersion(random())));
+        }
+        builder.masterNodeId(masterNodeId);  // Node 0 is always a master node
+
+        for (int i = 0; i < clientNodes; i++) {
+            String node = "client_" + i;
+            builder.add(new DiscoveryNode(node, randomAlphaOfLength(10) + "__" + i, buildNewFakeTransportAddress(), emptyMap(),
+                EnumSet.noneOf(DiscoveryNode.Role.class), randomVersion(random())));
+        }
+        return builder.build();
+    }
+
+    public static MetaData randomMetaData(IndexTemplateMetaData... templates) {
+        MetaData.Builder builder = MetaData.builder();
+        for (IndexTemplateMetaData template : templates) {
+            builder.put(template);
+        }
+        for (int i = 0; i < randomIntBetween(1, 5); i++) {
+            builder.put(
+                IndexMetaData.builder(randomAlphaOfLength(10))
+                    .settings(settings(Version.CURRENT))
+                    .numberOfReplicas(randomIntBetween(0, 3))
+                    .numberOfShards(randomIntBetween(1, 5))
+            );
+        }
+        return builder.build();
+    }
+
+    public static TransportAddress buildNewFakeTransportAddress() {
+        return LocalTransportAddress.buildUnique();
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java
index 8ad86818614a3..89b639a750aff 100644
--- a/core/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java
@@ -21,8 +21,7 @@
 
 import org.elasticsearch.Version;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
@@ -150,14 +149,14 @@ public void testSimpleJsonFromAndTo() throws IOException {
         String metaDataSource = MetaData.Builder.toXContent(metaData);
 //        System.out.println("ToJson: " + metaDataSource);
 
-        MetaData parsedMetaData = MetaData.Builder.fromXContent(XContentFactory.xContent(XContentType.JSON).createParser(metaDataSource));
+        MetaData parsedMetaData = MetaData.Builder.fromXContent(createParser(JsonXContent.jsonXContent, metaDataSource));
 
         IndexMetaData indexMetaData = parsedMetaData.index("test1");
         assertThat(indexMetaData.primaryTerm(0), equalTo(1L));
         assertThat(indexMetaData.getNumberOfShards(), equalTo(1));
         assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2));
         assertThat(indexMetaData.getCreationDate(), equalTo(-1L));
-        assertThat(indexMetaData.getSettings().getAsMap().size(), equalTo(3));
+        assertThat(indexMetaData.getSettings().size(), equalTo(3));
         assertThat(indexMetaData.getMappings().size(), equalTo(0));
 
         indexMetaData = parsedMetaData.index("test2");
@@ -166,7 +165,7 @@ public void testSimpleJsonFromAndTo() throws IOException {
         assertThat(indexMetaData.primaryTerm(0), equalTo(2L));
         assertThat(indexMetaData.primaryTerm(1), equalTo(2L));
         assertThat(indexMetaData.getCreationDate(), equalTo(-1L));
-        assertThat(indexMetaData.getSettings().getAsMap().size(), equalTo(5));
+        assertThat(indexMetaData.getSettings().size(), equalTo(5));
         assertThat(indexMetaData.getSettings().get("setting1"), equalTo("value1"));
         assertThat(indexMetaData.getSettings().get("setting2"), equalTo("value2"));
         assertThat(indexMetaData.getMappings().size(), equalTo(0));
@@ -175,7 +174,7 @@ public void testSimpleJsonFromAndTo() throws IOException {
         assertThat(indexMetaData.getNumberOfShards(), equalTo(1));
         assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2));
         assertThat(indexMetaData.getCreationDate(), equalTo(-1L));
-        assertThat(indexMetaData.getSettings().getAsMap().size(), equalTo(3));
+        assertThat(indexMetaData.getSettings().size(), equalTo(3));
         assertThat(indexMetaData.getMappings().size(), equalTo(1));
         assertThat(indexMetaData.getMappings().get("mapping1").source().string(), equalTo(MAPPING_SOURCE1));
 
@@ -183,14 +182,14 @@ public void testSimpleJsonFromAndTo() throws IOException {
         assertThat(indexMetaData.getCreationDate(), equalTo(2L));
         assertThat(indexMetaData.getNumberOfShards(), equalTo(1));
         assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2));
-        assertThat(indexMetaData.getSettings().getAsMap().size(), equalTo(4));
+        assertThat(indexMetaData.getSettings().size(), equalTo(4));
         assertThat(indexMetaData.getMappings().size(), equalTo(0));
 
         indexMetaData = parsedMetaData.index("test5");
         assertThat(indexMetaData.getNumberOfShards(), equalTo(1));
         assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2));
         assertThat(indexMetaData.getCreationDate(), equalTo(-1L));
-        assertThat(indexMetaData.getSettings().getAsMap().size(), equalTo(5));
+        assertThat(indexMetaData.getSettings().size(), equalTo(5));
         assertThat(indexMetaData.getSettings().get("setting1"), equalTo("value1"));
         assertThat(indexMetaData.getSettings().get("setting2"), equalTo("value2"));
         assertThat(indexMetaData.getMappings().size(), equalTo(2));
@@ -201,7 +200,7 @@ public void testSimpleJsonFromAndTo() throws IOException {
         assertThat(indexMetaData.getNumberOfShards(), equalTo(1));
         assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2));
         assertThat(indexMetaData.getCreationDate(), equalTo(2L));
-        assertThat(indexMetaData.getSettings().getAsMap().size(), equalTo(6));
+        assertThat(indexMetaData.getSettings().size(), equalTo(6));
         assertThat(indexMetaData.getSettings().get("setting1"), equalTo("value1"));
         assertThat(indexMetaData.getSettings().get("setting2"), equalTo("value2"));
         assertThat(indexMetaData.getMappings().size(), equalTo(0));
@@ -210,7 +209,7 @@ public void testSimpleJsonFromAndTo() throws IOException {
         assertThat(indexMetaData.getNumberOfShards(), equalTo(1));
         assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2));
         assertThat(indexMetaData.getCreationDate(), equalTo(2L));
-        assertThat(indexMetaData.getSettings().getAsMap().size(), equalTo(4));
+        assertThat(indexMetaData.getSettings().size(), equalTo(4));
         assertThat(indexMetaData.getMappings().size(), equalTo(2));
         assertThat(indexMetaData.getMappings().get("mapping1").source().string(), equalTo(MAPPING_SOURCE1));
         assertThat(indexMetaData.getMappings().get("mapping2").source().string(), equalTo(MAPPING_SOURCE2));
@@ -219,7 +218,7 @@ public void testSimpleJsonFromAndTo() throws IOException {
         assertThat(indexMetaData.getNumberOfShards(), equalTo(1));
         assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2));
         assertThat(indexMetaData.getCreationDate(), equalTo(-1L));
-        assertThat(indexMetaData.getSettings().getAsMap().size(), equalTo(5));
+        assertThat(indexMetaData.getSettings().size(), equalTo(5));
         assertThat(indexMetaData.getSettings().get("setting1"), equalTo("value1"));
         assertThat(indexMetaData.getSettings().get("setting2"), equalTo("value2"));
         assertThat(indexMetaData.getMappings().size(), equalTo(2));
@@ -233,7 +232,7 @@ public void testSimpleJsonFromAndTo() throws IOException {
         assertThat(indexMetaData.getNumberOfShards(), equalTo(1));
         assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2));
         assertThat(indexMetaData.getCreationDate(), equalTo(2L));
-        assertThat(indexMetaData.getSettings().getAsMap().size(), equalTo(6));
+        assertThat(indexMetaData.getSettings().size(), equalTo(6));
         assertThat(indexMetaData.getSettings().get("setting1"), equalTo("value1"));
         assertThat(indexMetaData.getSettings().get("setting2"), equalTo("value2"));
         assertThat(indexMetaData.getMappings().size(), equalTo(2));
@@ -247,7 +246,7 @@ public void testSimpleJsonFromAndTo() throws IOException {
         assertThat(indexMetaData.getNumberOfShards(), equalTo(1));
         assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2));
         assertThat(indexMetaData.getCreationDate(), equalTo(-1L));
-        assertThat(indexMetaData.getSettings().getAsMap().size(), equalTo(5));
+        assertThat(indexMetaData.getSettings().size(), equalTo(5));
         assertThat(indexMetaData.getSettings().get("setting1"), equalTo("value1"));
         assertThat(indexMetaData.getSettings().get("setting2"), equalTo("value2"));
         assertThat(indexMetaData.getMappings().size(), equalTo(2));
@@ -261,7 +260,7 @@ public void testSimpleJsonFromAndTo() throws IOException {
         assertThat(indexMetaData.getNumberOfShards(), equalTo(1));
         assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2));
         assertThat(indexMetaData.getCreationDate(), equalTo(-1L));
-        assertThat(indexMetaData.getSettings().getAsMap().size(), equalTo(5));
+        assertThat(indexMetaData.getSettings().size(), equalTo(5));
         assertThat(indexMetaData.getSettings().get("setting1"), equalTo("value1"));
         assertThat(indexMetaData.getSettings().get("setting2"), equalTo("value2"));
         assertThat(indexMetaData.getMappings().size(), equalTo(2));
@@ -279,7 +278,7 @@ public void testSimpleJsonFromAndTo() throws IOException {
         assertThat(indexMetaData.getNumberOfShards(), equalTo(1));
         assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2));
         assertThat(indexMetaData.getCreationDate(), equalTo(2L));
-        assertThat(indexMetaData.getSettings().getAsMap().size(), equalTo(6));
+        assertThat(indexMetaData.getSettings().size(), equalTo(6));
         assertThat(indexMetaData.getSettings().get("setting1"), equalTo("value1"));
         assertThat(indexMetaData.getSettings().get("setting2"), equalTo("value2"));
         assertThat(indexMetaData.getMappings().size(), equalTo(2));
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java
index 744477d6722e1..2778525f7dad9 100644
--- a/core/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java
@@ -49,6 +49,10 @@ public void testConvertWildcardsJustIndicesTests() {
         assertThat(newHashSet(resolver.resolve(context, Arrays.asList("testX*", "kuku"))), equalTo(newHashSet("testXXX", "testXYY", "kuku")));
         assertThat(newHashSet(resolver.resolve(context, Arrays.asList("*"))), equalTo(newHashSet("testXXX", "testXYY", "testYYY", "kuku")));
         assertThat(newHashSet(resolver.resolve(context, Arrays.asList("*", "-kuku"))), equalTo(newHashSet("testXXX", "testXYY", "testYYY")));
+        assertThat(newHashSet(resolver.resolve(context, Arrays.asList("testXXX", "testYYY"))), equalTo(newHashSet("testXXX", "testYYY")));
+        assertThat(newHashSet(resolver.resolve(context, Arrays.asList("testXXX", "-testXXX"))), equalTo(newHashSet("testXXX", "-testXXX")));
+        assertThat(newHashSet(resolver.resolve(context, Arrays.asList("testXXX", "testY*"))), equalTo(newHashSet("testXXX", "testYYY")));
+        assertThat(newHashSet(resolver.resolve(context, Arrays.asList("testXXX", "-testX*"))), equalTo(newHashSet("testXXX")));
     }
 
     public void testConvertWildcardsTests() {
@@ -62,10 +66,10 @@ public void testConvertWildcardsTests() {
 
         IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, IndicesOptions.lenientExpandOpen());
         assertThat(newHashSet(resolver.resolve(context, Arrays.asList("testYY*", "alias*"))), equalTo(newHashSet("testXXX", "testXYY", "testYYY")));
-        assertThat(newHashSet(resolver.resolve(context, Arrays.asList("-kuku"))), equalTo(newHashSet("testXXX", "testXYY", "testYYY")));
-        assertThat(newHashSet(resolver.resolve(context, Arrays.asList("+test*", "-testYYY"))), equalTo(newHashSet("testXXX", "testXYY")));
-        assertThat(newHashSet(resolver.resolve(context, Arrays.asList("+testX*", "+testYYY"))), equalTo(newHashSet("testXXX", "testXYY", "testYYY")));
-        assertThat(newHashSet(resolver.resolve(context, Arrays.asList("+testYYY", "+testX*"))), equalTo(newHashSet("testXXX", "testXYY", "testYYY")));
+        assertThat(newHashSet(resolver.resolve(context, Arrays.asList("-kuku"))), equalTo(newHashSet("-kuku")));
+        assertThat(newHashSet(resolver.resolve(context, Arrays.asList("test*", "-testYYY"))), equalTo(newHashSet("testXXX", "testXYY")));
+        assertThat(newHashSet(resolver.resolve(context, Arrays.asList("testX*", "testYYY"))), equalTo(newHashSet("testXXX", "testXYY", "testYYY")));
+        assertThat(newHashSet(resolver.resolve(context, Arrays.asList("testYYY", "testX*"))), equalTo(newHashSet("testXXX", "testXYY", "testYYY")));
     }
 
     public void testConvertWildcardsOpenClosedIndicesTests() {
@@ -125,4 +129,19 @@ private IndexMetaData.Builder indexBuilder(String index) {
         return IndexMetaData.builder(index).settings(settings(Version.CURRENT).put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0));
     }
 
+    public void testForDeprecatedPlusPattern() {
+        MetaData.Builder mdBuilder = MetaData.builder()
+            .put(indexBuilder("testXXX").state(IndexMetaData.State.OPEN))
+            .put(indexBuilder("testXYY").state(IndexMetaData.State.OPEN))
+            .put(indexBuilder("testYYY").state(IndexMetaData.State.OPEN));
+        ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build();
+        IndexNameExpressionResolver.WildcardExpressionResolver resolver = new IndexNameExpressionResolver.WildcardExpressionResolver();
+
+        IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, IndicesOptions.fromOptions(true, true, true, true));
+        assertThat(newHashSet(resolver.resolve(context, Arrays.asList("+testX*", "-testYYY"))), equalTo(newHashSet("testXXX", "testXYY")));
+        assertThat(newHashSet(resolver.resolve(context, Arrays.asList("+testYYY", "+testXY*"))), equalTo(newHashSet("testYYY", "testXYY")));
+        assertThat(newHashSet(resolver.resolve(context, Arrays.asList("testYYY", "+testXX*"))), equalTo(newHashSet("testXXX", "testYYY")));
+        assertWarnings("support for '+' as part of index expressions is deprecated");
+    }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeFiltersTests.java b/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeFiltersTests.java
index 59f058a95fb86..56b6a24c97774 100644
--- a/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeFiltersTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeFiltersTests.java
@@ -248,6 +248,28 @@ public void testIpPublishFilteringNotMatchingOr() {
         assertThat(filters.match(node), equalTo(true));
     }
 
+    public void testIpPublishFilteringMatchingWildcard() {
+        boolean matches = randomBoolean();
+        Settings settings = shuffleSettings(Settings.builder()
+            .put("xxx._publish_ip", matches ? "192.1.*" : "192.2.*")
+            .build());
+        DiscoveryNodeFilters filters = DiscoveryNodeFilters.buildFromSettings(OR, "xxx.", settings);
+
+        DiscoveryNode node = new DiscoveryNode("", "", "", "", "192.1.1.54", localAddress, emptyMap(), emptySet(), null);
+        assertThat(filters.match(node), equalTo(matches));
+    }
+
+    public void testCommaSeparatedValuesTrimmed() {
+        DiscoveryNode node = new DiscoveryNode("", "", "", "", "192.1.1.54", localAddress, singletonMap("tag", "B"), emptySet(), null);
+
+        Settings settings = shuffleSettings(Settings.builder()
+            .put("xxx." + randomFrom("_ip", "_host_ip", "_publish_ip"), "192.1.1.1, 192.1.1.54")
+            .put("xxx.tag", "A, B")
+            .build());
+        DiscoveryNodeFilters filters = DiscoveryNodeFilters.buildFromSettings(OR, "xxx.", settings);
+        assertTrue(filters.match(node));
+    }
+
     private Settings shuffleSettings(Settings source) {
         Settings.Builder settings = Settings.builder();
         List<String> keys = new ArrayList<>(source.getAsMap().keySet());
diff --git a/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeTests.java b/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeTests.java
new file mode 100644
index 0000000000000..2a5f4e3a8ad9c
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeTests.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.node;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.test.ESTestCase;
+
+import java.net.InetAddress;
+
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.emptySet;
+
+public class DiscoveryNodeTests extends ESTestCase {
+
+    public void testDiscoveryNodeIsCreatedWithHostFromInetAddress() throws Exception {
+        InetAddress inetAddress = randomBoolean() ? InetAddress.getByName("192.0.2.1") :
+            InetAddress.getByAddress("name1", new byte[] { (byte) 192, (byte) 168, (byte) 0, (byte) 1});
+        TransportAddress transportAddress = new InetSocketTransportAddress(inetAddress, randomIntBetween(0, 65535));
+        DiscoveryNode node = new DiscoveryNode("name1", "id1", transportAddress, emptyMap(), emptySet(), Version.CURRENT);
+        assertEquals(transportAddress.getHost(), node.getHostName());
+        assertEquals(transportAddress.getAddress(), node.getHostAddress());
+    }
+
+    public void testDiscoveryNodeSerializationKeepsHost() throws Exception {
+        InetAddress inetAddress = InetAddress.getByAddress("name1", new byte[] { (byte) 192, (byte) 168, (byte) 0, (byte) 1});
+        TransportAddress transportAddress = new InetSocketTransportAddress(inetAddress, randomIntBetween(0, 65535));
+        DiscoveryNode node = new DiscoveryNode("name1", "id1", transportAddress, emptyMap(), emptySet(), Version.CURRENT);
+
+        BytesStreamOutput streamOutput = new BytesStreamOutput();
+        streamOutput.setVersion(Version.CURRENT);
+        node.writeTo(streamOutput);
+
+        StreamInput in = StreamInput.wrap(streamOutput.bytes().toBytesRef().bytes);
+        DiscoveryNode serialized = new DiscoveryNode(in);
+        assertEquals(transportAddress.getHost(), serialized.getHostName());
+        assertEquals(transportAddress.getHost(), serialized.getAddress().getHost());
+        assertEquals(transportAddress.getAddress(), serialized.getHostAddress());
+        assertEquals(transportAddress.getAddress(), serialized.getAddress().getAddress());
+        assertEquals(transportAddress.getPort(), serialized.getAddress().getPort());
+    }
+
+    public void testDiscoveryNodeSerializationToOldVersion() throws Exception {
+        InetAddress inetAddress = InetAddress.getByAddress("name1", new byte[] { (byte) 192, (byte) 168, (byte) 0, (byte) 1});
+        TransportAddress transportAddress = new InetSocketTransportAddress(inetAddress, randomIntBetween(0, 65535));
+        DiscoveryNode node = new DiscoveryNode("name1", "id1", transportAddress, emptyMap(), emptySet(), Version.CURRENT);
+
+        BytesStreamOutput streamOutput = new BytesStreamOutput();
+        streamOutput.setVersion(Version.V_5_0_0);
+        node.writeTo(streamOutput);
+
+        StreamInput in = StreamInput.wrap(streamOutput.bytes().toBytesRef().bytes);
+        in.setVersion(Version.V_5_0_0);
+        DiscoveryNode serialized = new DiscoveryNode(in);
+
+        assertEquals(transportAddress.getHost(), serialized.getHostName());
+        assertEquals(transportAddress.getHost(), serialized.getAddress().getHost());
+        assertEquals(transportAddress.getAddress(), serialized.getHostAddress());
+        assertEquals(transportAddress.getAddress(), serialized.getAddress().getAddress());
+        assertEquals(transportAddress.getPort(), serialized.getAddress().getPort());
+        assertFalse("if the minimum compatibility version moves past 5.0.3, remove the special casing in DiscoverNode(StreamInput) and " +
+                "the TransportAddress(StreamInput, String) constructor",
+            Version.CURRENT.minimumCompatibilityVersion().after(Version.V_5_0_2));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodesTests.java b/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodesTests.java
index d6a83108d0f77..a677d2b964f0a 100644
--- a/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodesTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodesTests.java
@@ -136,8 +136,8 @@ public void testDeltas() {
 
         final DiscoveryNodes discoNodesA = builderA.build();
         final DiscoveryNodes discoNodesB = builderB.build();
-        logger.info("nodes A: {}", discoNodesA.prettyPrint());
-        logger.info("nodes B: {}", discoNodesB.prettyPrint());
+        logger.info("nodes A: {}", discoNodesA);
+        logger.info("nodes B: {}", discoNodesB);
 
         DiscoveryNodes.Delta delta = discoNodesB.delta(discoNodesA);
 
@@ -172,7 +172,7 @@ private static List<DiscoveryNode> randomNodes(final int numNodes) {
         for (int i = 0; i < numNodes; i++) {
             Map<String, String> attributes = new HashMap<>();
             if (frequently()) {
-                attributes.put("custom", randomBoolean() ? "match" : randomAsciiOfLengthBetween(3, 5));
+                attributes.put("custom", randomBoolean() ? "match" : randomAlphaOfLengthBetween(3, 5));
             }
             final DiscoveryNode node = newNode(idGenerator.getAndIncrement(), attributes,
                 new HashSet<>(randomSubsetOf(Arrays.asList(DiscoveryNode.Role.values()))));
@@ -251,4 +251,22 @@ Set<String> matchingNodeIds(DiscoveryNodes nodes) {
 
         abstract Set<String> matchingNodeIds(DiscoveryNodes nodes);
     }
+
+    public void testMaxMinNodeVersion() {
+        DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder();
+        discoBuilder.add(new DiscoveryNode("name_" + 1, "node_" + 1, new LocalTransportAddress("0"), Collections.emptyMap(),
+            new HashSet<>(randomSubsetOf(Arrays.asList(DiscoveryNode.Role.values()))),
+            Version.fromString("5.1.0")));
+        discoBuilder.add(new DiscoveryNode("name_" + 2, "node_" + 2, new LocalTransportAddress("1"), Collections.emptyMap(),
+            new HashSet<>(randomSubsetOf(Arrays.asList(DiscoveryNode.Role.values()))),
+            Version.fromString("6.3.0")));
+        discoBuilder.add(new DiscoveryNode("name_" + 3, "node_" + 3, new LocalTransportAddress("2"), Collections.emptyMap(),
+            new HashSet<>(randomSubsetOf(Arrays.asList(DiscoveryNode.Role.values()))),
+            Version.fromString("1.1.0")));
+        discoBuilder.localNodeId("name_1");
+        discoBuilder.masterNodeId("name_2");
+        DiscoveryNodes build = discoBuilder.build();
+        assertEquals( Version.fromString("6.3.0"), build.getMaxNodeVersion());
+        assertEquals( Version.fromString("1.1.0"), build.getMinNodeVersion());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/AllocationIdTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/AllocationIdTests.java
index 036c168eee82c..23939e1da7658 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/AllocationIdTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/AllocationIdTests.java
@@ -23,7 +23,7 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESTestCase;
 
@@ -122,7 +122,7 @@ public void testReinitializing() {
         shard = shard.reinitializePrimaryShard();
         assertThat(shard.allocationId().getId(), notNullValue());
         assertThat(shard.allocationId().getRelocationId(), nullValue());
-        assertThat(shard.allocationId().getId(), not(equalTo(allocationId.getId())));
+        assertThat(shard.allocationId().getId(), equalTo(allocationId.getId()));
     }
 
     public void testSerialization() throws IOException {
@@ -131,7 +131,7 @@ public void testSerialization() throws IOException {
             allocationId = AllocationId.newRelocation(allocationId);
         }
         BytesReference bytes = allocationId.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS).bytes();
-        AllocationId parsedAllocationId = AllocationId.fromXContent(XContentFactory.xContent(XContentType.JSON).createParser(bytes));
+        AllocationId parsedAllocationId = AllocationId.fromXContent(createParser(JsonXContent.jsonXContent, bytes));
         assertEquals(allocationId, parsedAllocationId);
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationIT.java b/core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationIT.java
index 88f09a55c2eb4..778394520a5ac 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationIT.java
@@ -26,7 +26,6 @@
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.InternalTestCluster;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 
 import java.util.Collections;
 import java.util.List;
@@ -44,7 +43,7 @@ public class DelayedAllocationIT extends ESIntegTestCase {
      * get allocated to a free node when the node hosting it leaves the cluster.
      */
     public void testNoDelayedTimeout() throws Exception {
-        internalCluster().startNodesAsync(3).get();
+        internalCluster().startNodes(3);
         prepareCreate("test").setSettings(Settings.builder()
                 .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
                 .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
@@ -63,7 +62,7 @@ public void testNoDelayedTimeout() throws Exception {
      * on it before.
      */
     public void testDelayedAllocationNodeLeavesAndComesBack() throws Exception {
-        internalCluster().startNodesAsync(3).get();
+        internalCluster().startNodes(3);
         prepareCreate("test").setSettings(Settings.builder()
                 .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
                 .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
@@ -71,12 +70,7 @@ public void testDelayedAllocationNodeLeavesAndComesBack() throws Exception {
         ensureGreen("test");
         indexRandomData();
         internalCluster().stopRandomNode(InternalTestCluster.nameFilter(findNodeWithShard()));
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                assertThat(client().admin().cluster().prepareState().all().get().getState().getRoutingNodes().unassigned().size() > 0, equalTo(true));
-            }
-        });
+        assertBusy(() -> assertThat(client().admin().cluster().prepareState().all().get().getState().getRoutingNodes().unassigned().size() > 0, equalTo(true)));
         assertThat(client().admin().cluster().prepareHealth().get().getDelayedUnassignedShards(), equalTo(1));
         internalCluster().startNode(); // this will use the same data location as the stopped node
         ensureGreen("test");
@@ -87,7 +81,7 @@ public void run() {
      * though the node hosting the shard is not coming back.
      */
     public void testDelayedAllocationTimesOut() throws Exception {
-        internalCluster().startNodesAsync(3).get();
+        internalCluster().startNodes(3);
         prepareCreate("test").setSettings(Settings.builder()
                 .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
                 .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
@@ -109,7 +103,7 @@ public void testDelayedAllocationTimesOut() throws Exception {
      * even though the node it was hosted on will not come back.
      */
     public void testDelayedAllocationChangeWithSettingTo100ms() throws Exception {
-        internalCluster().startNodesAsync(3).get();
+        internalCluster().startNodes(3);
         prepareCreate("test").setSettings(Settings.builder()
                 .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
                 .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
@@ -117,12 +111,7 @@ public void testDelayedAllocationChangeWithSettingTo100ms() throws Exception {
         ensureGreen("test");
         indexRandomData();
         internalCluster().stopRandomNode(InternalTestCluster.nameFilter(findNodeWithShard()));
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                assertThat(client().admin().cluster().prepareState().all().get().getState().getRoutingNodes().unassigned().size() > 0, equalTo(true));
-            }
-        });
+        assertBusy(() -> assertThat(client().admin().cluster().prepareState().all().get().getState().getRoutingNodes().unassigned().size() > 0, equalTo(true)));
         assertThat(client().admin().cluster().prepareHealth().get().getDelayedUnassignedShards(), equalTo(1));
         assertAcked(client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder().put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), TimeValue.timeValueMillis(100))).get());
         ensureGreen("test");
@@ -135,7 +124,7 @@ public void run() {
      * even though the node it was hosted on will not come back.
      */
     public void testDelayedAllocationChangeWithSettingTo0() throws Exception {
-        internalCluster().startNodesAsync(3).get();
+        internalCluster().startNodes(3);
         prepareCreate("test").setSettings(Settings.builder()
                 .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
                 .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationServiceTests.java
index 3dc7e7d221558..f053bd7845467 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationServiceTests.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -32,7 +33,6 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.junit.After;
@@ -72,7 +72,7 @@ public void createDelayedAllocationService() {
         clusterService = mock(ClusterService.class);
         allocationService = createAllocationService(Settings.EMPTY, new DelayedShardsMockGatewayAllocator());
         delayedAllocationService = new TestDelayAllocationService(Settings.EMPTY, threadPool, clusterService, allocationService);
-        verify(clusterService).addFirst(delayedAllocationService);
+        verify(clusterService).addListener(delayedAllocationService);
     }
 
     @After
@@ -92,15 +92,11 @@ public void testNoDelayedUnassigned() throws Exception {
         clusterState = ClusterState.builder(clusterState)
             .nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2")).localNodeId("node1").masterNodeId("node1"))
             .build();
-        clusterState = ClusterState.builder(clusterState).routingResult(allocationService.reroute(clusterState, "reroute")).build();
+        clusterState = allocationService.reroute(clusterState, "reroute");
         // starting primaries
-        clusterState = ClusterState.builder(clusterState)
-            .routingResult(allocationService.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING)))
-            .build();
+        clusterState = allocationService.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
         // starting replicas
-        clusterState = ClusterState.builder(clusterState)
-            .routingResult(allocationService.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING)))
-            .build();
+        clusterState = allocationService.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
         assertThat(clusterState.getRoutingNodes().unassigned().size() > 0, equalTo(false));
         ClusterState prevState = clusterState;
         // remove node2 and reroute
@@ -110,8 +106,7 @@ public void testNoDelayedUnassigned() throws Exception {
             nodes.add(newNode("node3"));
         }
         clusterState = ClusterState.builder(clusterState).nodes(nodes).build();
-        clusterState = ClusterState.builder(clusterState).routingResult(
-            allocationService.deassociateDeadNodes(clusterState, true, "reroute")).build();
+        clusterState = allocationService.deassociateDeadNodes(clusterState, true, "reroute");
         ClusterState newState = clusterState;
         List<ShardRouting> unassignedShards = newState.getRoutingTable().shardsWithState(ShardRoutingState.UNASSIGNED);
         if (nodeAvailableForAllocation) {
@@ -141,15 +136,11 @@ public void testDelayedUnassignedScheduleReroute() throws Exception {
             .build();
         final long baseTimestampNanos = System.nanoTime();
         allocationService.setNanoTimeOverride(baseTimestampNanos);
-        clusterState = ClusterState.builder(clusterState).routingResult(allocationService.reroute(clusterState, "reroute")).build();
+        clusterState = allocationService.reroute(clusterState, "reroute");
         // starting primaries
-        clusterState = ClusterState.builder(clusterState)
-            .routingResult(allocationService.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING)))
-            .build();
+        clusterState = allocationService.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
         // starting replicas
-        clusterState = ClusterState.builder(clusterState)
-            .routingResult(allocationService.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING)))
-            .build();
+        clusterState = allocationService.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
         assertFalse("no shards should be unassigned", clusterState.getRoutingNodes().unassigned().size() > 0);
         String nodeId = null;
         final List<ShardRouting> allShards = clusterState.getRoutingTable().allShards("test");
@@ -164,8 +155,7 @@ public void testDelayedUnassignedScheduleReroute() throws Exception {
 
         // remove node that has replica and reroute
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).remove(nodeId)).build();
-        clusterState = ClusterState.builder(clusterState).routingResult(
-            allocationService.deassociateDeadNodes(clusterState, true, "reroute")).build();
+        clusterState = allocationService.deassociateDeadNodes(clusterState, true, "reroute");
         ClusterState stateWithDelayedShard = clusterState;
         // make sure the replica is marked as delayed (i.e. not reallocated)
         assertEquals(1, UnassignedInfo.getNumberOfDelayedUnassigned(stateWithDelayedShard));
@@ -238,15 +228,11 @@ public void testDelayedUnassignedScheduleRerouteAfterDelayedReroute() throws Exc
                 .add(newNode("node0", singleton(DiscoveryNode.Role.MASTER))).localNodeId("node0").masterNodeId("node0")
                 .add(newNode("node1")).add(newNode("node2")).add(newNode("node3")).add(newNode("node4"))).build();
         // allocate shards
-        clusterState = ClusterState.builder(clusterState).routingResult(allocationService.reroute(clusterState, "reroute")).build();
+        clusterState = allocationService.reroute(clusterState, "reroute");
         // start primaries
-        clusterState = ClusterState.builder(clusterState)
-            .routingResult(allocationService.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING)))
-            .build();
+        clusterState = allocationService.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
         // start replicas
-        clusterState = ClusterState.builder(clusterState)
-            .routingResult(allocationService.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING)))
-            .build();
+        clusterState = allocationService.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
         assertThat("all shards should be started", clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(4));
 
         // find replica of short_delay
@@ -280,8 +266,7 @@ public void testDelayedUnassignedScheduleRerouteAfterDelayedReroute() throws Exc
             .build();
         // make sure both replicas are marked as delayed (i.e. not reallocated)
         allocationService.setNanoTimeOverride(baseTimestampNanos);
-        clusterState = ClusterState.builder(clusterState).routingResult(
-            allocationService.deassociateDeadNodes(clusterState, true, "reroute")).build();
+        clusterState = allocationService.deassociateDeadNodes(clusterState, true, "reroute");
         final ClusterState stateWithDelayedShards = clusterState;
         assertEquals(2, UnassignedInfo.getNumberOfDelayedUnassigned(stateWithDelayedShards));
         RoutingNodes.UnassignedShards.UnassignedIterator iter = stateWithDelayedShards.getRoutingNodes().unassigned().iterator();
@@ -400,15 +385,11 @@ public void testDelayedUnassignedScheduleRerouteRescheduledOnShorterDelay() thro
             .build();
         final long nodeLeftTimestampNanos = System.nanoTime();
         allocationService.setNanoTimeOverride(nodeLeftTimestampNanos);
-        clusterState = ClusterState.builder(clusterState).routingResult(allocationService.reroute(clusterState, "reroute")).build();
+        clusterState = allocationService.reroute(clusterState, "reroute");
         // starting primaries
-        clusterState = ClusterState.builder(clusterState)
-            .routingResult(allocationService.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING)))
-            .build();
+        clusterState = allocationService.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
         // starting replicas
-        clusterState = ClusterState.builder(clusterState)
-            .routingResult(allocationService.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING)))
-            .build();
+        clusterState = allocationService.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
         assertFalse("no shards should be unassigned", clusterState.getRoutingNodes().unassigned().size() > 0);
         String nodeIdOfFooReplica = null;
         for (ShardRouting shardRouting : clusterState.getRoutingTable().allShards("foo")) {
@@ -422,8 +403,7 @@ public void testDelayedUnassignedScheduleRerouteRescheduledOnShorterDelay() thro
         // remove node that has replica and reroute
         clusterState = ClusterState.builder(clusterState).nodes(
             DiscoveryNodes.builder(clusterState.nodes()).remove(nodeIdOfFooReplica)).build();
-        clusterState = ClusterState.builder(clusterState).routingResult(
-            allocationService.deassociateDeadNodes(clusterState, true, "fake node left")).build();
+        clusterState = allocationService.deassociateDeadNodes(clusterState, true, "fake node left");
         ClusterState stateWithDelayedShard = clusterState;
         // make sure the replica is marked as delayed (i.e. not reallocated)
         assertEquals(1, UnassignedInfo.getNumberOfDelayedUnassigned(stateWithDelayedShard));
@@ -466,8 +446,7 @@ public void testDelayedUnassignedScheduleRerouteRescheduledOnShorterDelay() thro
             // remove node that has replica and reroute
             clusterState = ClusterState.builder(stateWithDelayedShard).nodes(
                 DiscoveryNodes.builder(stateWithDelayedShard.nodes()).remove(nodeIdOfBarReplica)).build();
-            ClusterState stateWithShorterDelay = ClusterState.builder(clusterState).routingResult(
-                allocationService.deassociateDeadNodes(clusterState, true, "fake node left")).build();
+            ClusterState stateWithShorterDelay = allocationService.deassociateDeadNodes(clusterState, true, "fake node left");
             delayedAllocationService.setNanoTimeOverride(clusterChangeEventTimestampNanos);
             delayedAllocationService.clusterChanged(
                 new ClusterChangedEvent("fake node left", stateWithShorterDelay, stateWithDelayedShard));
@@ -487,7 +466,7 @@ public void testDelayedUnassignedScheduleRerouteRescheduledOnShorterDelay() thro
     private static class TestDelayAllocationService extends DelayedAllocationService {
         private volatile long nanoTimeOverride = -1L;
 
-        public TestDelayAllocationService(Settings settings, ThreadPool threadPool, ClusterService clusterService,
+        TestDelayAllocationService(Settings settings, ThreadPool threadPool, ClusterService clusterService,
                                           AllocationService allocationService) {
             super(settings, threadPool, clusterService, allocationService);
         }
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/GroupShardsIteratorTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/GroupShardsIteratorTests.java
new file mode 100644
index 0000000000000..f2571fce3391d
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/GroupShardsIteratorTests.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.routing;
+
+import org.apache.lucene.util.CollectionUtil;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+public class GroupShardsIteratorTests extends ESTestCase {
+
+    public void testSize() {
+        List<ShardIterator> list = new ArrayList<>();
+        Index index = new Index("foo", "na");
+
+        list.add(new PlainShardIterator(new ShardId(index, 0), Arrays.asList(newRouting(index, 0, true), newRouting(index, 0, true),
+            newRouting(index, 0, true))));
+        list.add(new PlainShardIterator(new ShardId(index, 1), Collections.emptyList()));
+        list.add(new PlainShardIterator(new ShardId(index, 2), Arrays.asList(newRouting(index, 2, true))));
+        index = new Index("foo_1", "na");
+
+        list.add(new PlainShardIterator(new ShardId(index, 0), Arrays.asList(newRouting(index, 0, true))));
+        list.add(new PlainShardIterator(new ShardId(index, 1), Arrays.asList(newRouting(index, 1, true))));
+        GroupShardsIterator iter = new GroupShardsIterator<>(list);
+        assertEquals(7, iter.totalSizeWith1ForEmpty());
+        assertEquals(5, iter.size());
+        assertEquals(6, iter.totalSize());
+    }
+
+    public void testIterate() {
+        List<ShardIterator> list = new ArrayList<>();
+        Index index = new Index("foo", "na");
+
+        list.add(new PlainShardIterator(new ShardId(index, 0), Arrays.asList(newRouting(index, 0, true), newRouting(index, 0, true),
+            newRouting(index, 0, true))));
+        list.add(new PlainShardIterator(new ShardId(index, 1), Collections.emptyList()));
+        list.add(new PlainShardIterator(new ShardId(index, 2), Arrays.asList(newRouting(index, 2, true))));
+
+        list.add(new PlainShardIterator(new ShardId(index, 0), Arrays.asList(newRouting(index, 0, true))));
+        list.add(new PlainShardIterator(new ShardId(index, 1), Arrays.asList(newRouting(index, 1, true))));
+
+        index = new Index("foo_2", "na");
+        list.add(new PlainShardIterator(new ShardId(index, 0), Arrays.asList(newRouting(index, 0, true))));
+        list.add(new PlainShardIterator(new ShardId(index, 1), Arrays.asList(newRouting(index, 1, true))));
+
+        Collections.shuffle(list, random());
+        ArrayList<ShardIterator> actualIterators = new ArrayList<>();
+        GroupShardsIterator<ShardIterator> iter = new GroupShardsIterator<>(list);
+        for (ShardIterator shardsIterator : iter) {
+            actualIterators.add(shardsIterator);
+        }
+        CollectionUtil.timSort(actualIterators);
+        assertEquals(actualIterators, list);
+    }
+
+    public ShardRouting newRouting(Index index, int id, boolean started) {
+        ShardRouting shardRouting = ShardRouting.newUnassigned(new ShardId(index, id), true,
+            RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
+        shardRouting = ShardRoutingHelper.initialize(shardRouting, "some node");
+        if (started) {
+            shardRouting = ShardRoutingHelper.moveToStarted(shardRouting);
+        }
+        return shardRouting;
+    };
+}
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java
index 3118692b34149..d8cc4c0938677 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java
@@ -22,7 +22,6 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.action.support.replication.ClusterStateCreationUtils;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.cluster.routing.allocation.decider.AwarenessAllocationDecider;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
@@ -35,6 +34,7 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -43,8 +43,8 @@
 
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.Matchers.containsInAnyOrder;
-import static org.hamcrest.object.HasToString.hasToString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.object.HasToString.hasToString;
 
 public class OperationRoutingTests extends ESTestCase{
 
@@ -58,7 +58,7 @@ public void testGenerateShardId() {
             assertEquals(shardSplits[1], (shardSplits[1] / shardSplits[2]) * shardSplits[2]);
             IndexMetaData metaData = IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(shardSplits[0])
                 .numberOfReplicas(1).build();
-            String term = randomAsciiOfLength(10);
+            String term = randomAlphaOfLength(10);
             final int shard = OperationRouting.generateShardId(metaData, term, null);
             IndexMetaData shrunk = IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(shardSplits[1])
                 .numberOfReplicas(1)
@@ -75,6 +75,142 @@ public void testGenerateShardId() {
         }
     }
 
+    public void testPartitionedIndex() {
+        // make sure the same routing value always has each _id fall within the configured partition size
+        for (int shards = 1; shards < 5; shards++) {
+            for (int partitionSize = 1; partitionSize == 1 || partitionSize < shards; partitionSize++) {
+                IndexMetaData metaData = IndexMetaData.builder("test")
+                    .settings(settings(Version.CURRENT))
+                    .numberOfShards(shards)
+                    .routingPartitionSize(partitionSize)
+                    .numberOfReplicas(1)
+                    .build();
+
+                for (int i = 0; i < 20; i++) {
+                    String routing = randomUnicodeOfLengthBetween(1, 50);
+                    Set<Integer> shardSet = new HashSet<>();
+
+                    for (int k = 0; k < 150; k++) {
+                        String id = randomUnicodeOfLengthBetween(1, 50);
+
+                        shardSet.add(OperationRouting.generateShardId(metaData, id, routing));
+                    }
+
+                    assertEquals(partitionSize, shardSet.size());
+                }
+            }
+        }
+    }
+
+    public void testPartitionedIndexShrunk() {
+        Map<String, Map<String, Integer>> routingIdToShard = new HashMap<>();
+
+        Map<String, Integer> routingA = new HashMap<>();
+        routingA.put("a_0", 1);
+        routingA.put("a_1", 2);
+        routingA.put("a_2", 2);
+        routingA.put("a_3", 2);
+        routingA.put("a_4", 1);
+        routingA.put("a_5", 2);
+        routingIdToShard.put("a", routingA);
+
+        Map<String, Integer> routingB = new HashMap<>();
+        routingB.put("b_0", 0);
+        routingB.put("b_1", 0);
+        routingB.put("b_2", 0);
+        routingB.put("b_3", 0);
+        routingB.put("b_4", 3);
+        routingB.put("b_5", 3);
+        routingIdToShard.put("b", routingB);
+
+        Map<String, Integer> routingC = new HashMap<>();
+        routingC.put("c_0", 1);
+        routingC.put("c_1", 1);
+        routingC.put("c_2", 0);
+        routingC.put("c_3", 0);
+        routingC.put("c_4", 0);
+        routingC.put("c_5", 1);
+        routingIdToShard.put("c", routingC);
+
+        Map<String, Integer> routingD = new HashMap<>();
+        routingD.put("d_0", 2);
+        routingD.put("d_1", 2);
+        routingD.put("d_2", 3);
+        routingD.put("d_3", 3);
+        routingD.put("d_4", 3);
+        routingD.put("d_5", 3);
+        routingIdToShard.put("d", routingD);
+
+        IndexMetaData metaData = IndexMetaData.builder("test")
+                .settings(settings(Version.CURRENT))
+                .setRoutingNumShards(8)
+                .numberOfShards(4)
+                .routingPartitionSize(3)
+                .numberOfReplicas(1)
+                .build();
+
+        for (Map.Entry<String, Map<String, Integer>> routingIdEntry : routingIdToShard.entrySet()) {
+            String routing = routingIdEntry.getKey();
+
+            for (Map.Entry<String, Integer> idEntry : routingIdEntry.getValue().entrySet()) {
+                String id = idEntry.getKey();
+                int shard = idEntry.getValue();
+
+                assertEquals(shard, OperationRouting.generateShardId(metaData, id, routing));
+            }
+        }
+    }
+
+    public void testPartitionedIndexBWC() {
+        Map<String, Map<String, Integer>> routingIdToShard = new HashMap<>();
+
+        Map<String, Integer> routingA = new HashMap<>();
+        routingA.put("a_0", 3);
+        routingA.put("a_1", 2);
+        routingA.put("a_2", 2);
+        routingA.put("a_3", 3);
+        routingIdToShard.put("a", routingA);
+
+        Map<String, Integer> routingB = new HashMap<>();
+        routingB.put("b_0", 5);
+        routingB.put("b_1", 0);
+        routingB.put("b_2", 0);
+        routingB.put("b_3", 0);
+        routingIdToShard.put("b", routingB);
+
+        Map<String, Integer> routingC = new HashMap<>();
+        routingC.put("c_0", 4);
+        routingC.put("c_1", 4);
+        routingC.put("c_2", 3);
+        routingC.put("c_3", 4);
+        routingIdToShard.put("c", routingC);
+
+        Map<String, Integer> routingD = new HashMap<>();
+        routingD.put("d_0", 3);
+        routingD.put("d_1", 4);
+        routingD.put("d_2", 4);
+        routingD.put("d_3", 4);
+        routingIdToShard.put("d", routingD);
+
+        IndexMetaData metaData = IndexMetaData.builder("test")
+                .settings(settings(Version.CURRENT))
+                .numberOfShards(6)
+                .routingPartitionSize(2)
+                .numberOfReplicas(1)
+                .build();
+
+        for (Map.Entry<String, Map<String, Integer>> routingIdEntry : routingIdToShard.entrySet()) {
+            String routing = routingIdEntry.getKey();
+
+            for (Map.Entry<String, Integer> idEntry : routingIdEntry.getValue().entrySet()) {
+                String id = idEntry.getKey();
+                int shard = idEntry.getValue();
+
+                assertEquals(shard, OperationRouting.generateShardId(metaData, id, routing));
+            }
+        }
+    }
+
     /**
      * Ensures that all changes to the hash-function / shard selection are BWC
      */
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java b/core/src/test/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java
index 8bc9c29bb3797..6a1309f39da9e 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java
@@ -27,7 +27,6 @@
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.routing.allocation.command.AllocateEmptyPrimaryAllocationCommand;
 import org.elasticsearch.cluster.routing.allocation.command.AllocateStalePrimaryAllocationCommand;
-import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.collect.ImmutableOpenIntMap;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
@@ -35,10 +34,10 @@
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.InternalTestCluster;
+import org.elasticsearch.test.discovery.TestZenDiscovery;
 import org.elasticsearch.test.disruption.NetworkDisruption;
 import org.elasticsearch.test.disruption.NetworkDisruption.NetworkDisconnect;
 import org.elasticsearch.test.disruption.NetworkDisruption.TwoPartitions;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.test.transport.MockTransportService;
 
 import java.util.Arrays;
@@ -64,10 +63,16 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
         return Arrays.asList(MockTransportService.TestPlugin.class);
     }
 
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        return Settings.builder().put(super.nodeSettings(nodeOrdinal))
+            .put(TestZenDiscovery.USE_MOCK_PINGS.getKey(), false).build();
+    }
+
     private void createStaleReplicaScenario() throws Exception {
         logger.info("--> starting 3 nodes, 1 master, 2 data");
         String master = internalCluster().startMasterOnlyNode(Settings.EMPTY);
-        internalCluster().startDataOnlyNodesAsync(2).get();
+        internalCluster().startDataOnlyNodes(2);
 
         assertAcked(client().admin().indices().prepareCreate("test").setSettings(Settings.builder()
             .put("index.number_of_shards", 1).put("index.number_of_replicas", 1)).get());
@@ -115,12 +120,7 @@ private void createStaleReplicaScenario() throws Exception {
         logger.info("--> check that old primary shard does not get promoted to primary again");
         // kick reroute and wait for all shard states to be fetched
         client(master).admin().cluster().prepareReroute().get();
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                assertThat(internalCluster().getInstance(GatewayAllocator.class, master).getNumberOfInFlightFetch(), equalTo(0));
-            }
-        });
+        assertBusy(() -> assertThat(internalCluster().getInstance(GatewayAllocator.class, master).getNumberOfInFlightFetch(), equalTo(0)));
         // kick reroute a second time and check that all shards are unassigned
         assertThat(client(master).admin().cluster().prepareReroute().get().getState().getRoutingNodes().unassigned().size(), equalTo(2));
     }
@@ -155,7 +155,9 @@ public void testFailedAllocationOfStalePrimaryToDataNodeWithNoData() throws Exce
         client().admin().cluster().prepareReroute().add(new AllocateStalePrimaryAllocationCommand("test", 0, dataNodeWithNoShardCopy, true)).get();
 
         logger.info("--> wait until shard is failed and becomes unassigned again");
-        assertBusy(() -> assertTrue(client().admin().cluster().prepareState().get().getState().prettyPrint(), client().admin().cluster().prepareState().get().getState().getRoutingTable().index("test").allPrimaryShardsUnassigned()));
+        assertBusy(() ->
+            assertTrue(client().admin().cluster().prepareState().get().getState().toString(),
+                client().admin().cluster().prepareState().get().getState().getRoutingTable().index("test").allPrimaryShardsUnassigned()));
         assertThat(client().admin().cluster().prepareState().get().getState().getRoutingTable().index("test").getShards().get(0).primaryShard().unassignedInfo().getReason(), equalTo(UnassignedInfo.Reason.ALLOCATION_FAILED));
     }
 
@@ -266,7 +268,7 @@ public boolean clearData(String nodeName) {
 
     public void testNotWaitForQuorumCopies() throws Exception {
         logger.info("--> starting 3 nodes");
-        internalCluster().startNodesAsync(3).get();
+        internalCluster().startNodes(3);
         logger.info("--> creating index with 1 primary and 2 replicas");
         assertAcked(client().admin().indices().prepareCreate("test").setSettings(Settings.builder()
             .put("index.number_of_shards", randomIntBetween(1, 3)).put("index.number_of_replicas", 2)).get());
@@ -288,7 +290,7 @@ public void testNotWaitForQuorumCopies() throws Exception {
      */
     public void testForceAllocatePrimaryOnNoDecision() throws Exception {
         logger.info("--> starting 1 node");
-        final String node = internalCluster().startNodeAsync().get();
+        final String node = internalCluster().startNode();
         logger.info("--> creating index with 1 primary and 0 replicas");
         final String indexName = "test-idx";
         assertAcked(client().admin().indices()
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/PrimaryTermsTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/PrimaryTermsTests.java
index 388d454813e02..863a33b132773 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/PrimaryTermsTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/PrimaryTermsTests.java
@@ -21,16 +21,15 @@
 
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.health.ClusterStateHealth;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.node.DiscoveryNodes.Builder;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
-import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation;
-import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
+import org.elasticsearch.cluster.routing.allocation.FailedShard;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.cluster.ESAllocationTestCase;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -42,13 +41,12 @@
 
 import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.not;
 
 public class PrimaryTermsTests extends ESAllocationTestCase {
 
     private static final String TEST_INDEX_1 = "test1";
     private static final String TEST_INDEX_2 = "test2";
-    private RoutingTable testRoutingTable;
     private int numberOfShards;
     private int numberOfReplicas;
     private static final Settings DEFAULT_SETTINGS = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
@@ -65,7 +63,7 @@ public void setUp() throws Exception {
                 .put("cluster.routing.allocation.node_initial_primaries_recoveries", Integer.MAX_VALUE)
                 .build());
         this.numberOfShards = randomIntBetween(1, 5);
-        this.numberOfReplicas = randomIntBetween(1, 5);
+        this.numberOfReplicas = randomIntBetween(0, 5);
         logger.info("Setup test with {} shards and {} replicas.", this.numberOfShards, this.numberOfReplicas);
         this.primaryTermsPerIndex.clear();
         MetaData metaData = MetaData.builder()
@@ -73,7 +71,7 @@ public void setUp() throws Exception {
                 .put(createIndexMetaData(TEST_INDEX_2))
                 .build();
 
-        this.testRoutingTable = new RoutingTable.Builder()
+        RoutingTable routingTable = new RoutingTable.Builder()
                 .add(new IndexRoutingTable.Builder(metaData.index(TEST_INDEX_1).getIndex()).initializeAsNew(metaData.index(TEST_INDEX_1))
                         .build())
                 .add(new IndexRoutingTable.Builder(metaData.index(TEST_INDEX_2).getIndex()).initializeAsNew(metaData.index(TEST_INDEX_2))
@@ -81,7 +79,7 @@ public void setUp() throws Exception {
                 .build();
 
         this.clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
-            .metaData(metaData).routingTable(testRoutingTable).build();
+            .metaData(metaData).routingTable(routingTable).build();
     }
 
     /**
@@ -94,9 +92,8 @@ private void initPrimaries() {
             discoBuilder = discoBuilder.add(newNode("node" + i));
         }
         this.clusterState = ClusterState.builder(clusterState).nodes(discoBuilder).build();
-        RoutingAllocation.Result rerouteResult = allocationService.reroute(clusterState, "reroute");
-        this.testRoutingTable = rerouteResult.routingTable();
-        assertThat(rerouteResult.changed(), is(true));
+        ClusterState rerouteResult = allocationService.reroute(clusterState, "reroute");
+        assertThat(rerouteResult, not(equalTo(this.clusterState)));
         applyRerouteResult(rerouteResult);
         primaryTermsPerIndex.keySet().forEach(this::incrementPrimaryTerm);
     }
@@ -113,17 +110,16 @@ private void incrementPrimaryTerm(String index, int shard) {
     }
 
     private boolean startInitializingShards(String index) {
-        this.clusterState = ClusterState.builder(clusterState).routingTable(this.testRoutingTable).build();
         final List<ShardRouting> startedShards = this.clusterState.getRoutingNodes().shardsWithState(index, INITIALIZING);
         logger.info("start primary shards for index [{}]: {} ", index, startedShards);
-        RoutingAllocation.Result rerouteResult = allocationService.applyStartedShards(this.clusterState, startedShards);
+        ClusterState rerouteResult = allocationService.applyStartedShards(this.clusterState, startedShards);
+        boolean changed = rerouteResult.equals(this.clusterState) == false;
         applyRerouteResult(rerouteResult);
-        return rerouteResult.changed();
+        return changed;
     }
 
-    private void applyRerouteResult(RoutingAllocation.Result rerouteResult) {
+    private void applyRerouteResult(ClusterState newClusterState) {
         ClusterState previousClusterState = this.clusterState;
-        ClusterState newClusterState = ClusterState.builder(previousClusterState).routingResult(rerouteResult).build();
         ClusterState.Builder builder = ClusterState.builder(newClusterState).incrementVersion();
         if (previousClusterState.routingTable() != newClusterState.routingTable()) {
             builder.routingTable(RoutingTable.builder(newClusterState.routingTable()).version(newClusterState.routingTable().version() + 1)
@@ -133,7 +129,6 @@ private void applyRerouteResult(RoutingAllocation.Result rerouteResult) {
             builder.metaData(MetaData.builder(newClusterState.metaData()).version(newClusterState.metaData().version() + 1));
         }
         this.clusterState = builder.build();
-        this.testRoutingTable = rerouteResult.routingTable();
         final ClusterStateHealth clusterHealth = new ClusterStateHealth(clusterState);
         logger.info("applied reroute. active shards: p [{}], t [{}], init shards: [{}], relocating: [{}]",
                 clusterHealth.getActivePrimaryShards(), clusterHealth.getActiveShards(),
@@ -141,21 +136,18 @@ private void applyRerouteResult(RoutingAllocation.Result rerouteResult) {
     }
 
     private void failSomePrimaries(String index) {
-        this.clusterState = ClusterState.builder(clusterState).routingTable(this.testRoutingTable).build();
-        final IndexRoutingTable indexShardRoutingTable = testRoutingTable.index(index);
+        final IndexRoutingTable indexShardRoutingTable = clusterState.routingTable().index(index);
         Set<Integer> shardIdsToFail = new HashSet<>();
         for (int i = 1 + randomInt(numberOfShards - 1); i > 0; i--) {
             shardIdsToFail.add(randomInt(numberOfShards - 1));
         }
         logger.info("failing primary shards {} for index [{}]", shardIdsToFail, index);
-        List<FailedRerouteAllocation.FailedShard> failedShards = new ArrayList<>();
+        List<FailedShard> failedShards = new ArrayList<>();
         for (int shard : shardIdsToFail) {
-            failedShards.add(new FailedRerouteAllocation.FailedShard(indexShardRoutingTable.shard(shard).primaryShard(), "test", null));
+            failedShards.add(new FailedShard(indexShardRoutingTable.shard(shard).primaryShard(), "test", null));
             incrementPrimaryTerm(index, shard); // the primary failure should increment the primary term;
         }
-        RoutingAllocation.Result rerouteResult = allocationService.applyFailedShards(this.clusterState, failedShards,
-            Collections.emptyList());
-        applyRerouteResult(rerouteResult);
+        applyRerouteResult(allocationService.applyFailedShards(this.clusterState, failedShards,Collections.emptyList()));
     }
 
     private void addNodes() {
@@ -166,8 +158,7 @@ private void addNodes() {
             nodesBuilder.add(newNode("extra_" + i));
         }
         this.clusterState = ClusterState.builder(clusterState).nodes(nodesBuilder).build();
-        RoutingAllocation.Result rerouteResult = allocationService.reroute(this.clusterState, "nodes added");
-        applyRerouteResult(rerouteResult);
+        applyRerouteResult(allocationService.reroute(this.clusterState, "nodes added"));
 
     }
 
@@ -191,7 +182,7 @@ private void assertAllPrimaryTerm() {
     private void assertPrimaryTerm(String index) {
         final long[] terms = primaryTermsPerIndex.get(index);
         final IndexMetaData indexMetaData = clusterState.metaData().index(index);
-        for (IndexShardRoutingTable shardRoutingTable : this.testRoutingTable.index(index)) {
+        for (IndexShardRoutingTable shardRoutingTable : this.clusterState.routingTable().index(index)) {
             final int shard = shardRoutingTable.shardId().id();
             assertThat("primary term mismatch between indexMetaData of [" + index + "] and shard [" + shard + "]'s routing",
                     indexMetaData.primaryTerm(shard), equalTo(terms[shard]));
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/RandomShardRoutingMutator.java b/core/src/test/java/org/elasticsearch/cluster/routing/RandomShardRoutingMutator.java
index c3064c7fa9dc2..932d24f2da1f7 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/RandomShardRoutingMutator.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/RandomShardRoutingMutator.java
@@ -19,7 +19,9 @@
 
 package org.elasticsearch.cluster.routing;
 
-import static org.elasticsearch.test.ESTestCase.randomAsciiOfLength;
+import java.util.Set;
+
+import static org.elasticsearch.test.ESTestCase.randomAlphaOfLength;
 import static org.elasticsearch.test.ESTestCase.randomFrom;
 import static org.elasticsearch.test.ESTestCase.randomInt;
 
@@ -31,18 +33,18 @@ private RandomShardRoutingMutator() {
 
     }
 
-    public static ShardRouting randomChange(ShardRouting shardRouting, String[] nodes) {
+    public static ShardRouting randomChange(ShardRouting shardRouting, Set<String> nodes) {
         switch (randomInt(2)) {
             case 0:
                 if (shardRouting.unassigned() == false && shardRouting.primary() == false) {
-                    shardRouting = shardRouting.moveToUnassigned(new UnassignedInfo(randomReason(), randomAsciiOfLength(10)));
+                    shardRouting = shardRouting.moveToUnassigned(new UnassignedInfo(randomReason(), randomAlphaOfLength(10)));
                 } else if (shardRouting.unassignedInfo() != null) {
-                    shardRouting = shardRouting.updateUnassigned(new UnassignedInfo(randomReason(), randomAsciiOfLength(10)),
+                    shardRouting = shardRouting.updateUnassigned(new UnassignedInfo(randomReason(), randomAlphaOfLength(10)),
                         shardRouting.recoverySource());
                 }
                 break;
             case 1:
-                if (shardRouting.unassigned()) {
+                if (shardRouting.unassigned() && nodes.isEmpty() == false) {
                     shardRouting = shardRouting.initialize(randomFrom(nodes), null, -1);
                 }
                 break;
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingBackwardCompatibilityTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingBackwardCompatibilityTests.java
index c673a6fd2265d..0a39168c853b6 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingBackwardCompatibilityTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingBackwardCompatibilityTests.java
@@ -55,7 +55,7 @@ public void testBackwardCompatibility() throws Exception {
 
                 OperationRouting operationRouting = new OperationRouting(Settings.EMPTY, new ClusterSettings(Settings.EMPTY,
                     ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
-                for (Version version : VersionUtils.allVersions()) {
+                for (Version version : VersionUtils.allReleasedVersions()) {
                     if (version.onOrAfter(Version.V_2_0_0) == false) {
                         // unsupported version, no need to test
                         continue;
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingServiceTests.java
index 04277ba1eb484..3496d9281ed83 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingServiceTests.java
@@ -48,7 +48,7 @@ private class TestRoutingService extends RoutingService {
 
         private AtomicBoolean rerouted = new AtomicBoolean();
 
-        public TestRoutingService() {
+        TestRoutingService() {
             super(Settings.EMPTY, null, null);
         }
 
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java
index 2ac427374e161..055adbaebbce5 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java
@@ -21,23 +21,27 @@
 
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.node.DiscoveryNodes.Builder;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
-import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
-import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.index.shard.ShardId;
 import org.junit.Before;
 
+import java.util.Arrays;
 import java.util.Set;
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.nullValue;
 
 public class RoutingTableTests extends ESAllocationTestCase {
@@ -45,7 +49,6 @@ public class RoutingTableTests extends ESAllocationTestCase {
     private static final String TEST_INDEX_1 = "test1";
     private static final String TEST_INDEX_2 = "test2";
     private RoutingTable emptyRoutingTable;
-    private RoutingTable testRoutingTable;
     private int numberOfShards;
     private int numberOfReplicas;
     private int shardsPerIndex;
@@ -72,7 +75,7 @@ public void setUp() throws Exception {
                 .put(createIndexMetaData(TEST_INDEX_2))
                 .build();
 
-        this.testRoutingTable = new RoutingTable.Builder()
+        RoutingTable testRoutingTable = new RoutingTable.Builder()
                 .add(new IndexRoutingTable.Builder(metaData.index(TEST_INDEX_1).getIndex()).initializeAsNew(metaData.index(TEST_INDEX_1)).build())
                 .add(new IndexRoutingTable.Builder(metaData.index(TEST_INDEX_2).getIndex()).initializeAsNew(metaData.index(TEST_INDEX_2)).build())
                 .build();
@@ -89,18 +92,14 @@ private void initPrimaries() {
             discoBuilder = discoBuilder.add(newNode("node" + i));
         }
         this.clusterState = ClusterState.builder(clusterState).nodes(discoBuilder).build();
-        RoutingAllocation.Result rerouteResult = ALLOCATION_SERVICE.reroute(clusterState, "reroute");
-        this.testRoutingTable = rerouteResult.routingTable();
-        assertThat(rerouteResult.changed(), is(true));
-        this.clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        ClusterState rerouteResult = ALLOCATION_SERVICE.reroute(clusterState, "reroute");
+        assertThat(rerouteResult, not(equalTo(this.clusterState)));
+        this.clusterState = rerouteResult;
     }
 
     private void startInitializingShards(String index) {
-        this.clusterState = ClusterState.builder(clusterState).routingTable(this.testRoutingTable).build();
         logger.info("start primary shards for index {}", index);
-        RoutingAllocation.Result rerouteResult = ALLOCATION_SERVICE.applyStartedShards(this.clusterState, this.clusterState.getRoutingNodes().shardsWithState(index, INITIALIZING));
-        this.clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
-        this.testRoutingTable = rerouteResult.routingTable();
+        this.clusterState = ALLOCATION_SERVICE.applyStartedShards(this.clusterState, this.clusterState.getRoutingNodes().shardsWithState(index, INITIALIZING));
     }
 
     private IndexMetaData.Builder createIndexMetaData(String indexName) {
@@ -112,11 +111,11 @@ private IndexMetaData.Builder createIndexMetaData(String indexName) {
 
     public void testAllShards() {
         assertThat(this.emptyRoutingTable.allShards().size(), is(0));
-        assertThat(this.testRoutingTable.allShards().size(), is(this.totalNumberOfShards));
+        assertThat(this.clusterState.routingTable().allShards().size(), is(this.totalNumberOfShards));
 
-        assertThat(this.testRoutingTable.allShards(TEST_INDEX_1).size(), is(this.shardsPerIndex));
+        assertThat(this.clusterState.routingTable().allShards(TEST_INDEX_1).size(), is(this.shardsPerIndex));
         try {
-            assertThat(this.testRoutingTable.allShards("not_existing").size(), is(0));
+            assertThat(this.clusterState.routingTable().allShards("not_existing").size(), is(0));
             fail("Exception expected when calling allShards() with non existing index name");
         } catch (IndexNotFoundException e) {
             // expected
@@ -124,69 +123,69 @@ public void testAllShards() {
     }
 
     public void testHasIndex() {
-        assertThat(this.testRoutingTable.hasIndex(TEST_INDEX_1), is(true));
-        assertThat(this.testRoutingTable.hasIndex("foobar"), is(false));
+        assertThat(clusterState.routingTable().hasIndex(TEST_INDEX_1), is(true));
+        assertThat(clusterState.routingTable().hasIndex("foobar"), is(false));
     }
 
     public void testIndex() {
-        assertThat(this.testRoutingTable.index(TEST_INDEX_1).getIndex().getName(), is(TEST_INDEX_1));
-        assertThat(this.testRoutingTable.index("foobar"), is(nullValue()));
+        assertThat(clusterState.routingTable().index(TEST_INDEX_1).getIndex().getName(), is(TEST_INDEX_1));
+        assertThat(clusterState.routingTable().index("foobar"), is(nullValue()));
     }
 
     public void testIndicesRouting() {
-        assertThat(this.testRoutingTable.indicesRouting().size(), is(2));
-        assertThat(this.testRoutingTable.getIndicesRouting().size(), is(2));
-        assertSame(this.testRoutingTable.getIndicesRouting(), this.testRoutingTable.indicesRouting());
+        assertThat(clusterState.routingTable().indicesRouting().size(), is(2));
+        assertThat(clusterState.routingTable().getIndicesRouting().size(), is(2));
+        assertSame(clusterState.routingTable().getIndicesRouting(), clusterState.routingTable().indicesRouting());
     }
 
     public void testShardsWithState() {
-        assertThat(this.testRoutingTable.shardsWithState(ShardRoutingState.UNASSIGNED).size(), is(this.totalNumberOfShards));
+        assertThat(clusterState.routingTable().shardsWithState(ShardRoutingState.UNASSIGNED).size(), is(this.totalNumberOfShards));
 
         initPrimaries();
-        assertThat(this.testRoutingTable.shardsWithState(ShardRoutingState.UNASSIGNED).size(), is(this.totalNumberOfShards - 2 * this.numberOfShards));
-        assertThat(this.testRoutingTable.shardsWithState(ShardRoutingState.INITIALIZING).size(), is(2 * this.numberOfShards));
+        assertThat(clusterState.routingTable().shardsWithState(ShardRoutingState.UNASSIGNED).size(), is(this.totalNumberOfShards - 2 * this.numberOfShards));
+        assertThat(clusterState.routingTable().shardsWithState(ShardRoutingState.INITIALIZING).size(), is(2 * this.numberOfShards));
 
         startInitializingShards(TEST_INDEX_1);
-        assertThat(this.testRoutingTable.shardsWithState(ShardRoutingState.STARTED).size(), is(this.numberOfShards));
+        assertThat(clusterState.routingTable().shardsWithState(ShardRoutingState.STARTED).size(), is(this.numberOfShards));
         int initializingExpected = this.numberOfShards + this.numberOfShards * this.numberOfReplicas;
-        assertThat(this.testRoutingTable.shardsWithState(ShardRoutingState.INITIALIZING).size(), is(initializingExpected));
-        assertThat(this.testRoutingTable.shardsWithState(ShardRoutingState.UNASSIGNED).size(), is(this.totalNumberOfShards - initializingExpected - this.numberOfShards));
+        assertThat(clusterState.routingTable().shardsWithState(ShardRoutingState.INITIALIZING).size(), is(initializingExpected));
+        assertThat(clusterState.routingTable().shardsWithState(ShardRoutingState.UNASSIGNED).size(), is(this.totalNumberOfShards - initializingExpected - this.numberOfShards));
 
         startInitializingShards(TEST_INDEX_2);
-        assertThat(this.testRoutingTable.shardsWithState(ShardRoutingState.STARTED).size(), is(2 * this.numberOfShards));
+        assertThat(clusterState.routingTable().shardsWithState(ShardRoutingState.STARTED).size(), is(2 * this.numberOfShards));
         initializingExpected = 2 * this.numberOfShards * this.numberOfReplicas;
-        assertThat(this.testRoutingTable.shardsWithState(ShardRoutingState.INITIALIZING).size(), is(initializingExpected));
-        assertThat(this.testRoutingTable.shardsWithState(ShardRoutingState.UNASSIGNED).size(), is(this.totalNumberOfShards - initializingExpected - 2 * this.numberOfShards));
+        assertThat(clusterState.routingTable().shardsWithState(ShardRoutingState.INITIALIZING).size(), is(initializingExpected));
+        assertThat(clusterState.routingTable().shardsWithState(ShardRoutingState.UNASSIGNED).size(), is(this.totalNumberOfShards - initializingExpected - 2 * this.numberOfShards));
 
         // now start all replicas too
         startInitializingShards(TEST_INDEX_1);
         startInitializingShards(TEST_INDEX_2);
-        assertThat(this.testRoutingTable.shardsWithState(ShardRoutingState.STARTED).size(), is(this.totalNumberOfShards));
+        assertThat(clusterState.routingTable().shardsWithState(ShardRoutingState.STARTED).size(), is(this.totalNumberOfShards));
     }
 
     public void testActivePrimaryShardsGrouped() {
         assertThat(this.emptyRoutingTable.activePrimaryShardsGrouped(new String[0], true).size(), is(0));
         assertThat(this.emptyRoutingTable.activePrimaryShardsGrouped(new String[0], false).size(), is(0));
 
-        assertThat(this.testRoutingTable.activePrimaryShardsGrouped(new String[]{TEST_INDEX_1}, false).size(), is(0));
-        assertThat(this.testRoutingTable.activePrimaryShardsGrouped(new String[]{TEST_INDEX_1}, true).size(), is(this.numberOfShards));
+        assertThat(clusterState.routingTable().activePrimaryShardsGrouped(new String[]{TEST_INDEX_1}, false).size(), is(0));
+        assertThat(clusterState.routingTable().activePrimaryShardsGrouped(new String[]{TEST_INDEX_1}, true).size(), is(this.numberOfShards));
 
         initPrimaries();
-        assertThat(this.testRoutingTable.activePrimaryShardsGrouped(new String[]{TEST_INDEX_1}, false).size(), is(0));
-        assertThat(this.testRoutingTable.activePrimaryShardsGrouped(new String[]{TEST_INDEX_1}, true).size(), is(this.numberOfShards));
+        assertThat(clusterState.routingTable().activePrimaryShardsGrouped(new String[]{TEST_INDEX_1}, false).size(), is(0));
+        assertThat(clusterState.routingTable().activePrimaryShardsGrouped(new String[]{TEST_INDEX_1}, true).size(), is(this.numberOfShards));
 
         startInitializingShards(TEST_INDEX_1);
-        assertThat(this.testRoutingTable.activePrimaryShardsGrouped(new String[]{TEST_INDEX_1}, false).size(), is(this.numberOfShards));
-        assertThat(this.testRoutingTable.activePrimaryShardsGrouped(new String[]{TEST_INDEX_1, TEST_INDEX_2}, false).size(), is(this.numberOfShards));
-        assertThat(this.testRoutingTable.activePrimaryShardsGrouped(new String[]{TEST_INDEX_1}, true).size(), is(this.numberOfShards));
+        assertThat(clusterState.routingTable().activePrimaryShardsGrouped(new String[]{TEST_INDEX_1}, false).size(), is(this.numberOfShards));
+        assertThat(clusterState.routingTable().activePrimaryShardsGrouped(new String[]{TEST_INDEX_1, TEST_INDEX_2}, false).size(), is(this.numberOfShards));
+        assertThat(clusterState.routingTable().activePrimaryShardsGrouped(new String[]{TEST_INDEX_1}, true).size(), is(this.numberOfShards));
 
         startInitializingShards(TEST_INDEX_2);
-        assertThat(this.testRoutingTable.activePrimaryShardsGrouped(new String[]{TEST_INDEX_2}, false).size(), is(this.numberOfShards));
-        assertThat(this.testRoutingTable.activePrimaryShardsGrouped(new String[]{TEST_INDEX_1, TEST_INDEX_2}, false).size(), is(2 * this.numberOfShards));
-        assertThat(this.testRoutingTable.activePrimaryShardsGrouped(new String[]{TEST_INDEX_1, TEST_INDEX_2}, true).size(), is(2 * this.numberOfShards));
+        assertThat(clusterState.routingTable().activePrimaryShardsGrouped(new String[]{TEST_INDEX_2}, false).size(), is(this.numberOfShards));
+        assertThat(clusterState.routingTable().activePrimaryShardsGrouped(new String[]{TEST_INDEX_1, TEST_INDEX_2}, false).size(), is(2 * this.numberOfShards));
+        assertThat(clusterState.routingTable().activePrimaryShardsGrouped(new String[]{TEST_INDEX_1, TEST_INDEX_2}, true).size(), is(2 * this.numberOfShards));
 
         try {
-            this.testRoutingTable.activePrimaryShardsGrouped(new String[]{TEST_INDEX_1, "not_exists"}, true);
+            clusterState.routingTable().activePrimaryShardsGrouped(new String[]{TEST_INDEX_1, "not_exists"}, true);
             fail("Calling with non-existing index name should raise IndexMissingException");
         } catch (IndexNotFoundException e) {
             // expected
@@ -197,43 +196,43 @@ public void testAllActiveShardsGrouped() {
         assertThat(this.emptyRoutingTable.allActiveShardsGrouped(new String[0], true).size(), is(0));
         assertThat(this.emptyRoutingTable.allActiveShardsGrouped(new String[0], false).size(), is(0));
 
-        assertThat(this.testRoutingTable.allActiveShardsGrouped(new String[]{TEST_INDEX_1}, false).size(), is(0));
-        assertThat(this.testRoutingTable.allActiveShardsGrouped(new String[]{TEST_INDEX_1}, true).size(), is(this.shardsPerIndex));
+        assertThat(clusterState.routingTable().allActiveShardsGrouped(new String[]{TEST_INDEX_1}, false).size(), is(0));
+        assertThat(clusterState.routingTable().allActiveShardsGrouped(new String[]{TEST_INDEX_1}, true).size(), is(this.shardsPerIndex));
 
         initPrimaries();
-        assertThat(this.testRoutingTable.allActiveShardsGrouped(new String[]{TEST_INDEX_1}, false).size(), is(0));
-        assertThat(this.testRoutingTable.allActiveShardsGrouped(new String[]{TEST_INDEX_1}, true).size(), is(this.shardsPerIndex));
+        assertThat(clusterState.routingTable().allActiveShardsGrouped(new String[]{TEST_INDEX_1}, false).size(), is(0));
+        assertThat(clusterState.routingTable().allActiveShardsGrouped(new String[]{TEST_INDEX_1}, true).size(), is(this.shardsPerIndex));
 
         startInitializingShards(TEST_INDEX_1);
-        assertThat(this.testRoutingTable.allActiveShardsGrouped(new String[]{TEST_INDEX_1}, false).size(), is(this.numberOfShards));
-        assertThat(this.testRoutingTable.allActiveShardsGrouped(new String[]{TEST_INDEX_1, TEST_INDEX_2}, false).size(), is(this.numberOfShards));
-        assertThat(this.testRoutingTable.allActiveShardsGrouped(new String[]{TEST_INDEX_1}, true).size(), is(this.shardsPerIndex));
+        assertThat(clusterState.routingTable().allActiveShardsGrouped(new String[]{TEST_INDEX_1}, false).size(), is(this.numberOfShards));
+        assertThat(clusterState.routingTable().allActiveShardsGrouped(new String[]{TEST_INDEX_1, TEST_INDEX_2}, false).size(), is(this.numberOfShards));
+        assertThat(clusterState.routingTable().allActiveShardsGrouped(new String[]{TEST_INDEX_1}, true).size(), is(this.shardsPerIndex));
 
         startInitializingShards(TEST_INDEX_2);
-        assertThat(this.testRoutingTable.allActiveShardsGrouped(new String[]{TEST_INDEX_2}, false).size(), is(this.numberOfShards));
-        assertThat(this.testRoutingTable.allActiveShardsGrouped(new String[]{TEST_INDEX_1, TEST_INDEX_2}, false).size(), is(2 * this.numberOfShards));
-        assertThat(this.testRoutingTable.allActiveShardsGrouped(new String[]{TEST_INDEX_1, TEST_INDEX_2}, true).size(), is(this.totalNumberOfShards));
+        assertThat(clusterState.routingTable().allActiveShardsGrouped(new String[]{TEST_INDEX_2}, false).size(), is(this.numberOfShards));
+        assertThat(clusterState.routingTable().allActiveShardsGrouped(new String[]{TEST_INDEX_1, TEST_INDEX_2}, false).size(), is(2 * this.numberOfShards));
+        assertThat(clusterState.routingTable().allActiveShardsGrouped(new String[]{TEST_INDEX_1, TEST_INDEX_2}, true).size(), is(this.totalNumberOfShards));
 
         try {
-            this.testRoutingTable.allActiveShardsGrouped(new String[]{TEST_INDEX_1, "not_exists"}, true);
+            clusterState.routingTable().allActiveShardsGrouped(new String[]{TEST_INDEX_1, "not_exists"}, true);
         } catch (IndexNotFoundException e) {
             fail("Calling with non-existing index should be ignored at the moment");
         }
     }
 
     public void testAllAssignedShardsGrouped() {
-        assertThat(this.testRoutingTable.allAssignedShardsGrouped(new String[]{TEST_INDEX_1}, false).size(), is(0));
-        assertThat(this.testRoutingTable.allAssignedShardsGrouped(new String[]{TEST_INDEX_1}, true).size(), is(this.shardsPerIndex));
+        assertThat(clusterState.routingTable().allAssignedShardsGrouped(new String[]{TEST_INDEX_1}, false).size(), is(0));
+        assertThat(clusterState.routingTable().allAssignedShardsGrouped(new String[]{TEST_INDEX_1}, true).size(), is(this.shardsPerIndex));
 
         initPrimaries();
-        assertThat(this.testRoutingTable.allAssignedShardsGrouped(new String[]{TEST_INDEX_1}, false).size(), is(this.numberOfShards));
-        assertThat(this.testRoutingTable.allAssignedShardsGrouped(new String[]{TEST_INDEX_1}, true).size(), is(this.shardsPerIndex));
+        assertThat(clusterState.routingTable().allAssignedShardsGrouped(new String[]{TEST_INDEX_1}, false).size(), is(this.numberOfShards));
+        assertThat(clusterState.routingTable().allAssignedShardsGrouped(new String[]{TEST_INDEX_1}, true).size(), is(this.shardsPerIndex));
 
-        assertThat(this.testRoutingTable.allAssignedShardsGrouped(new String[]{TEST_INDEX_1, TEST_INDEX_2}, false).size(), is(2 * this.numberOfShards));
-        assertThat(this.testRoutingTable.allAssignedShardsGrouped(new String[]{TEST_INDEX_1, TEST_INDEX_2}, true).size(), is(this.totalNumberOfShards));
+        assertThat(clusterState.routingTable().allAssignedShardsGrouped(new String[]{TEST_INDEX_1, TEST_INDEX_2}, false).size(), is(2 * this.numberOfShards));
+        assertThat(clusterState.routingTable().allAssignedShardsGrouped(new String[]{TEST_INDEX_1, TEST_INDEX_2}, true).size(), is(this.totalNumberOfShards));
 
         try {
-            this.testRoutingTable.allAssignedShardsGrouped(new String[]{TEST_INDEX_1, "not_exists"}, false);
+            clusterState.routingTable().allAssignedShardsGrouped(new String[]{TEST_INDEX_1, "not_exists"}, false);
         } catch (IndexNotFoundException e) {
             fail("Calling with non-existing index should be ignored at the moment");
         }
@@ -242,19 +241,19 @@ public void testAllAssignedShardsGrouped() {
     public void testAllShardsForMultipleIndices() {
         assertThat(this.emptyRoutingTable.allShards(new String[0]).size(), is(0));
 
-        assertThat(this.testRoutingTable.allShards(new String[]{TEST_INDEX_1}).size(), is(this.shardsPerIndex));
+        assertThat(clusterState.routingTable().allShards(new String[]{TEST_INDEX_1}).size(), is(this.shardsPerIndex));
 
         initPrimaries();
-        assertThat(this.testRoutingTable.allShards(new String[]{TEST_INDEX_1}).size(), is(this.shardsPerIndex));
+        assertThat(clusterState.routingTable().allShards(new String[]{TEST_INDEX_1}).size(), is(this.shardsPerIndex));
 
         startInitializingShards(TEST_INDEX_1);
-        assertThat(this.testRoutingTable.allShards(new String[]{TEST_INDEX_1}).size(), is(this.shardsPerIndex));
+        assertThat(clusterState.routingTable().allShards(new String[]{TEST_INDEX_1}).size(), is(this.shardsPerIndex));
 
         startInitializingShards(TEST_INDEX_2);
-        assertThat(this.testRoutingTable.allShards(new String[]{TEST_INDEX_1, TEST_INDEX_2}).size(), is(this.totalNumberOfShards));
+        assertThat(clusterState.routingTable().allShards(new String[]{TEST_INDEX_1, TEST_INDEX_2}).size(), is(this.totalNumberOfShards));
 
         try {
-            this.testRoutingTable.allShards(new String[]{TEST_INDEX_1, "not_exists"});
+            clusterState.routingTable().allShards(new String[]{TEST_INDEX_1, "not_exists"});
         } catch (IndexNotFoundException e) {
             fail("Calling with non-existing index should be ignored at the moment");
         }
@@ -332,13 +331,32 @@ public void testValidations() {
         expectThrows(IllegalStateException.class, () -> indexRoutingTable.validate(metaData4));
     }
 
+    public void testDistinctNodes() {
+        ShardId shardId = new ShardId(new Index("index", "uuid"), 0);
+        ShardRouting routing1 = TestShardRouting.newShardRouting(shardId, "node1", randomBoolean(), ShardRoutingState.STARTED);
+        ShardRouting routing2 = TestShardRouting.newShardRouting(shardId, "node2", randomBoolean(), ShardRoutingState.STARTED);
+        ShardRouting routing3 = TestShardRouting.newShardRouting(shardId, "node1", randomBoolean(), ShardRoutingState.STARTED);
+        ShardRouting routing4 = TestShardRouting.newShardRouting(shardId, "node3", "node2", randomBoolean(), ShardRoutingState.RELOCATING);
+        assertTrue(IndexShardRoutingTable.Builder.distinctNodes(Arrays.asList(routing1, routing2)));
+        assertFalse(IndexShardRoutingTable.Builder.distinctNodes(Arrays.asList(routing1, routing3)));
+        assertFalse(IndexShardRoutingTable.Builder.distinctNodes(Arrays.asList(routing1, routing2, routing3)));
+        assertTrue(IndexShardRoutingTable.Builder.distinctNodes(Arrays.asList(routing1, routing4)));
+        assertFalse(IndexShardRoutingTable.Builder.distinctNodes(Arrays.asList(routing2, routing4)));
+    }
+
+    /** reverse engineer the in sync aid based on the given indexRoutingTable **/
     public static IndexMetaData updateActiveAllocations(IndexRoutingTable indexRoutingTable, IndexMetaData indexMetaData) {
         IndexMetaData.Builder imdBuilder = IndexMetaData.builder(indexMetaData);
         for (IndexShardRoutingTable shardTable : indexRoutingTable) {
             for (ShardRouting shardRouting : shardTable) {
-                Set<String> activeAllocations = shardTable.activeShards().stream().map(
+                Set<String> insyncAids = shardTable.activeShards().stream().map(
                     shr -> shr.allocationId().getId()).collect(Collectors.toSet());
-                imdBuilder.putInSyncAllocationIds(shardRouting.id(), activeAllocations);
+                final ShardRouting primaryShard = shardTable.primaryShard();
+                if (primaryShard.initializing() && primaryShard.recoverySource().getType() == RecoverySource.Type.EXISTING_STORE) {
+                    // simulate a primary was initialized based on aid
+                    insyncAids.add(primaryShard.allocationId().getId());
+                }
+                imdBuilder.putInSyncAllocationIds(shardRouting.id(), insyncAids);
             }
         }
         return imdBuilder.build();
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java
index a4c8d0e424717..0b0f6ae273d8e 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java
@@ -118,7 +118,7 @@ public void testEqualsIgnoringVersion() {
             switch (changeId) {
                 case 0:
                     // change index
-                    ShardId shardId = new ShardId(new Index("blubb", randomAsciiOfLength(10)), otherRouting.id());
+                    ShardId shardId = new ShardId(new Index("blubb", randomAlphaOfLength(10)), otherRouting.id());
                     otherRouting = new ShardRouting(shardId, otherRouting.currentNodeId(), otherRouting.relocatingNodeId(),
                             otherRouting.primary(), otherRouting.state(), otherRouting.recoverySource(), otherRouting.unassignedInfo(),
                             otherRouting.allocationId(), otherRouting.getExpectedShardSize());
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java
index f5cfedc9c7df5..23f8f7f89f0c2 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java
@@ -31,7 +31,7 @@
 import org.elasticsearch.cluster.routing.RecoverySource.SnapshotRecoverySource;
 import org.elasticsearch.cluster.routing.UnassignedInfo.AllocationStatus;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
-import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation;
+import org.elasticsearch.cluster.routing.allocation.FailedShard;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.io.stream.ByteBufferStreamInput;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
@@ -81,9 +81,9 @@ public void testReasonOrdinalOrder() {
     public void testSerialization() throws Exception {
         UnassignedInfo.Reason reason = RandomPicks.randomFrom(random(), UnassignedInfo.Reason.values());
         UnassignedInfo meta = reason == UnassignedInfo.Reason.ALLOCATION_FAILED ?
-            new UnassignedInfo(reason, randomBoolean() ? randomAsciiOfLength(4) : null, null, randomIntBetween(1, 100), System.nanoTime(),
+            new UnassignedInfo(reason, randomBoolean() ? randomAlphaOfLength(4) : null, null, randomIntBetween(1, 100), System.nanoTime(),
                                System.currentTimeMillis(), false, AllocationStatus.NO_ATTEMPT):
-            new UnassignedInfo(reason, randomBoolean() ? randomAsciiOfLength(4) : null);
+            new UnassignedInfo(reason, randomBoolean() ? randomAlphaOfLength(4) : null);
         BytesStreamOutput out = new BytesStreamOutput();
         meta.writeTo(out);
         out.close();
@@ -178,9 +178,9 @@ public void testReplicaAdded() {
                 .metaData(metaData)
                 .routingTable(RoutingTable.builder().addAsNew(metaData.index(index)).build()).build();
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1"))).build();
-        clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState, "reroute")).build();
+        clusterState = allocation.reroute(clusterState, "reroute");
         // starting primaries
-        clusterState = ClusterState.builder(clusterState).routingResult(allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING))).build();
+        clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
         IndexRoutingTable.Builder builder = IndexRoutingTable.builder(index);
         for (IndexShardRoutingTable indexShardRoutingTable : clusterState.routingTable().index(index)) {
             builder.addIndexShard(indexShardRoutingTable);
@@ -218,15 +218,15 @@ public void testNodeLeave() {
                 .metaData(metaData)
                 .routingTable(RoutingTable.builder().addAsNew(metaData.index("test")).build()).build();
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))).build();
-        clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState, "reroute")).build();
+        clusterState = allocation.reroute(clusterState, "reroute");
         // starting primaries
-        clusterState = ClusterState.builder(clusterState).routingResult(allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING))).build();
+        clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
         // starting replicas
-        clusterState = ClusterState.builder(clusterState).routingResult(allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING))).build();
+        clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
         assertThat(clusterState.getRoutingNodes().unassigned().size() > 0, equalTo(false));
         // remove node2 and reroute
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).remove("node2")).build();
-        clusterState = ClusterState.builder(clusterState).routingResult(allocation.deassociateDeadNodes(clusterState, true, "reroute")).build();
+        clusterState = allocation.deassociateDeadNodes(clusterState, true, "reroute");
         // verify that NODE_LEAVE is the reason for meta
         assertThat(clusterState.getRoutingNodes().unassigned().size() > 0, equalTo(true));
         assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).size(), equalTo(1));
@@ -247,15 +247,15 @@ public void testFailedShard() {
                 .metaData(metaData)
                 .routingTable(RoutingTable.builder().addAsNew(metaData.index("test")).build()).build();
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))).build();
-        clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState, "reroute")).build();
+        clusterState = allocation.reroute(clusterState, "reroute");
         // starting primaries
-        clusterState = ClusterState.builder(clusterState).routingResult(allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING))).build();
+        clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
         // starting replicas
-        clusterState = ClusterState.builder(clusterState).routingResult(allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING))).build();
+        clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
         assertThat(clusterState.getRoutingNodes().unassigned().size() > 0, equalTo(false));
         // fail shard
         ShardRouting shardToFail = clusterState.getRoutingNodes().shardsWithState(STARTED).get(0);
-        clusterState = ClusterState.builder(clusterState).routingResult(allocation.applyFailedShards(clusterState, Collections.singletonList(new FailedRerouteAllocation.FailedShard(shardToFail, "test fail", null)))).build();
+        clusterState = allocation.applyFailedShards(clusterState, Collections.singletonList(new FailedShard(shardToFail, "test fail", null)));
         // verify the reason and details
         assertThat(clusterState.getRoutingNodes().unassigned().size() > 0, equalTo(true));
         assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).size(), equalTo(1));
@@ -297,18 +297,18 @@ public void testNumberOfDelayedUnassigned() throws Exception {
                 .metaData(metaData)
                 .routingTable(RoutingTable.builder().addAsNew(metaData.index("test1")).addAsNew(metaData.index("test2")).build()).build();
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))).build();
-        clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState, "reroute")).build();
+        clusterState = allocation.reroute(clusterState, "reroute");
         assertThat(UnassignedInfo.getNumberOfDelayedUnassigned(clusterState), equalTo(0));
         // starting primaries
-        clusterState = ClusterState.builder(clusterState).routingResult(allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING))).build();
+        clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
         // starting replicas
-        clusterState = ClusterState.builder(clusterState).routingResult(allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING))).build();
+        clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
         assertThat(clusterState.getRoutingNodes().unassigned().size() > 0, equalTo(false));
         // remove node2 and reroute
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).remove("node2")).build();
         // make sure both replicas are marked as delayed (i.e. not reallocated)
-        clusterState = ClusterState.builder(clusterState).routingResult(allocation.deassociateDeadNodes(clusterState, true, "reroute")).build();
-        assertThat(clusterState.prettyPrint(), UnassignedInfo.getNumberOfDelayedUnassigned(clusterState), equalTo(2));
+        clusterState = allocation.deassociateDeadNodes(clusterState, true, "reroute");
+        assertThat(clusterState.toString(), UnassignedInfo.getNumberOfDelayedUnassigned(clusterState), equalTo(2));
     }
 
     public void testFindNextDelayedAllocation() {
@@ -325,24 +325,24 @@ public void testFindNextDelayedAllocation() {
                 .metaData(metaData)
                 .routingTable(RoutingTable.builder().addAsNew(metaData.index("test1")).addAsNew(metaData.index("test2")).build()).build();
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))).build();
-        clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState, "reroute")).build();
+        clusterState = allocation.reroute(clusterState, "reroute");
         assertThat(UnassignedInfo.getNumberOfDelayedUnassigned(clusterState), equalTo(0));
         // starting primaries
-        clusterState = ClusterState.builder(clusterState).routingResult(allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING))).build();
+        clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
         // starting replicas
-        clusterState = ClusterState.builder(clusterState).routingResult(allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING))).build();
+        clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
         assertThat(clusterState.getRoutingNodes().unassigned().size() > 0, equalTo(false));
         // remove node2 and reroute
         final long baseTime = System.nanoTime();
         allocation.setNanoTimeOverride(baseTime);
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).remove("node2")).build();
-        clusterState = ClusterState.builder(clusterState).routingResult(allocation.deassociateDeadNodes(clusterState, true, "reroute")).build();
+        clusterState = allocation.deassociateDeadNodes(clusterState, true, "reroute");
 
         final long delta = randomBoolean() ? 0 : randomInt((int) expectMinDelaySettingsNanos - 1);
 
         if (delta > 0) {
             allocation.setNanoTimeOverride(baseTime + delta);
-            clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState, "time moved")).build();
+            clusterState = allocation.reroute(clusterState, "time moved");
         }
 
         assertThat(UnassignedInfo.findNextDelayedAllocation(baseTime + delta, clusterState), equalTo(expectMinDelaySettingsNanos - delta));
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AddIncrementallyTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AddIncrementallyTests.java
index 3803f54ba767c..e658ff03a186d 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AddIncrementallyTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AddIncrementallyTests.java
@@ -23,6 +23,7 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -34,7 +35,6 @@
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.CollectionUtils;
-import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.hamcrest.Matcher;
 import org.hamcrest.Matchers;
 
@@ -43,6 +43,8 @@
 
 import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
 
 public class AddIncrementallyTests extends ESAllocationTestCase {
     private final Logger logger = Loggers.getLogger(AddIncrementallyTests.class);
@@ -53,41 +55,41 @@ public void testAddNodesAndIndices() {
         AllocationService service = createAllocationService(settings.build());
 
         ClusterState clusterState = initCluster(service, 1, 3, 3, 1);
-        assertThat(clusterState.getRoutingNodes().node("node0").shardsWithState(STARTED).size(), Matchers.equalTo(9));
-        assertThat(clusterState.getRoutingNodes().unassigned().size(), Matchers.equalTo(9));
+        assertThat(clusterState.getRoutingNodes().node("node0").shardsWithState(STARTED).size(), equalTo(9));
+        assertThat(clusterState.getRoutingNodes().unassigned().size(), equalTo(9));
         int nodeOffset = 1;
         clusterState = addNodes(clusterState, service, 1, nodeOffset++);
-        assertThat(clusterState.getRoutingNodes().node("node0").shardsWithState(STARTED).size(), Matchers.equalTo(9));
-        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(STARTED).size(), Matchers.equalTo(9));
-        assertThat(clusterState.getRoutingNodes().unassigned().size(), Matchers.equalTo(0));
-        assertNumIndexShardsPerNode(clusterState, Matchers.equalTo(3));
+        assertThat(clusterState.getRoutingNodes().node("node0").shardsWithState(STARTED).size(), equalTo(9));
+        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(STARTED).size(), equalTo(9));
+        assertThat(clusterState.getRoutingNodes().unassigned().size(), equalTo(0));
+        assertNumIndexShardsPerNode(clusterState, equalTo(3));
         clusterState = addNodes(clusterState, service, 1, nodeOffset++);
-        assertNumIndexShardsPerNode(clusterState, Matchers.equalTo(2));
+        assertNumIndexShardsPerNode(clusterState, equalTo(2));
         clusterState = addNodes(clusterState, service, 1, nodeOffset++);
         assertNumIndexShardsPerNode(clusterState, Matchers.lessThanOrEqualTo(2));
         assertAtLeastOneIndexShardPerNode(clusterState);
         clusterState = removeNodes(clusterState, service, 1);
-        assertNumIndexShardsPerNode(clusterState, Matchers.equalTo(2));
+        assertNumIndexShardsPerNode(clusterState, equalTo(2));
 
         clusterState = addIndex(clusterState, service, 3, 2, 3);
-        assertThat(clusterState.getRoutingNodes().unassigned().size(), Matchers.equalTo(2));
-        assertNumIndexShardsPerNode(clusterState, "test3", Matchers.equalTo(2));
+        assertThat(clusterState.getRoutingNodes().unassigned().size(), equalTo(2));
+        assertNumIndexShardsPerNode(clusterState, "test3", equalTo(2));
         assertNumIndexShardsPerNode(clusterState, Matchers.lessThanOrEqualTo(2));
 
         clusterState = addIndex(clusterState, service, 4, 2, 3);
-        assertThat(clusterState.getRoutingNodes().unassigned().size(), Matchers.equalTo(4));
-        assertNumIndexShardsPerNode(clusterState, "test4", Matchers.equalTo(2));
+        assertThat(clusterState.getRoutingNodes().unassigned().size(), equalTo(4));
+        assertNumIndexShardsPerNode(clusterState, "test4", equalTo(2));
         assertNumIndexShardsPerNode(clusterState, Matchers.lessThanOrEqualTo(2));
         clusterState = addNodes(clusterState, service, 1, nodeOffset++);
         assertNumIndexShardsPerNode(clusterState, Matchers.lessThanOrEqualTo(2));
-        assertThat(clusterState.getRoutingNodes().unassigned().size(), Matchers.equalTo(0));
+        assertThat(clusterState.getRoutingNodes().unassigned().size(), equalTo(0));
         clusterState = removeNodes(clusterState, service, 1);
-        assertThat(clusterState.getRoutingNodes().unassigned().size(), Matchers.equalTo(4));
+        assertThat(clusterState.getRoutingNodes().unassigned().size(), equalTo(4));
         assertNumIndexShardsPerNode(clusterState, Matchers.lessThanOrEqualTo(2));
         clusterState = addNodes(clusterState, service, 1, nodeOffset++);
         assertNumIndexShardsPerNode(clusterState, Matchers.lessThanOrEqualTo(2));
-        assertThat(clusterState.getRoutingNodes().unassigned().size(), Matchers.equalTo(0));
-        logger.debug("ClusterState: {}", clusterState.getRoutingNodes().prettyPrint());
+        assertThat(clusterState.getRoutingNodes().unassigned().size(), equalTo(0));
+        logger.debug("ClusterState: {}", clusterState.getRoutingNodes());
     }
 
     public void testMinimalRelocations() {
@@ -97,60 +99,58 @@ public void testMinimalRelocations() {
         AllocationService service = createAllocationService(settings.build());
 
         ClusterState clusterState = initCluster(service, 1, 3, 3, 1);
-        assertThat(clusterState.getRoutingNodes().node("node0").shardsWithState(STARTED).size(), Matchers.equalTo(9));
-        assertThat(clusterState.getRoutingNodes().unassigned().size(), Matchers.equalTo(9));
+        assertThat(clusterState.getRoutingNodes().node("node0").shardsWithState(STARTED).size(), equalTo(9));
+        assertThat(clusterState.getRoutingNodes().unassigned().size(), equalTo(9));
         int nodeOffset = 1;
         clusterState = addNodes(clusterState, service, 1, nodeOffset++);
-        assertThat(clusterState.getRoutingNodes().node("node0").shardsWithState(STARTED).size(), Matchers.equalTo(9));
-        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(STARTED).size(), Matchers.equalTo(9));
-        assertThat(clusterState.getRoutingNodes().unassigned().size(), Matchers.equalTo(0));
-        assertNumIndexShardsPerNode(clusterState, Matchers.equalTo(3));
+        assertThat(clusterState.getRoutingNodes().node("node0").shardsWithState(STARTED).size(), equalTo(9));
+        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(STARTED).size(), equalTo(9));
+        assertThat(clusterState.getRoutingNodes().unassigned().size(), equalTo(0));
+        assertNumIndexShardsPerNode(clusterState, equalTo(3));
 
         logger.info("now, start one more node, check that rebalancing will happen because we set it to always");
         DiscoveryNodes.Builder nodes = DiscoveryNodes.builder(clusterState.nodes());
         nodes.add(newNode("node2"));
         clusterState = ClusterState.builder(clusterState).nodes(nodes.build()).build();
 
-        RoutingAllocation.Result routingResult = service.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = service.reroute(clusterState, "reroute");
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
 
-        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(INITIALIZING).size(), Matchers.equalTo(2));
-        assertThat(clusterState.getRoutingNodes().node("node0").shardsWithState(INITIALIZING).size(), Matchers.equalTo(0));
-        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(INITIALIZING).size(), Matchers.equalTo(0));
+        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(INITIALIZING).size(), equalTo(2));
+        assertThat(clusterState.getRoutingNodes().node("node0").shardsWithState(INITIALIZING).size(), equalTo(0));
+        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(INITIALIZING).size(), equalTo(0));
 
-        routingResult = service.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        ClusterState newState = service.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
         routingNodes = clusterState.getRoutingNodes();
-        assertTrue(routingResult.changed());
-        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(STARTED).size(), Matchers.equalTo(2));
-        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(INITIALIZING).size(), Matchers.equalTo(2));
-        assertThat(clusterState.getRoutingNodes().node("node0").shardsWithState(INITIALIZING).size(), Matchers.equalTo(0));
-        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(INITIALIZING).size(), Matchers.equalTo(0));
-
-        routingResult = service.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(STARTED).size(), equalTo(2));
+        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(INITIALIZING).size(), equalTo(2));
+        assertThat(clusterState.getRoutingNodes().node("node0").shardsWithState(INITIALIZING).size(), equalTo(0));
+        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(INITIALIZING).size(), equalTo(0));
+
+        newState = service.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
         routingNodes = clusterState.getRoutingNodes();
-        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(STARTED).size(), Matchers.equalTo(4));
-        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(INITIALIZING).size(), Matchers.equalTo(2));
-        assertThat(clusterState.getRoutingNodes().node("node0").shardsWithState(INITIALIZING).size(), Matchers.equalTo(0));
-        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(INITIALIZING).size(), Matchers.equalTo(0));
-        assertTrue(routingResult.changed());
-
-        routingResult = service.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(STARTED).size(), equalTo(4));
+        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(INITIALIZING).size(), equalTo(2));
+        assertThat(clusterState.getRoutingNodes().node("node0").shardsWithState(INITIALIZING).size(), equalTo(0));
+        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(INITIALIZING).size(), equalTo(0));
+
+        newState  = service.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
         routingNodes = clusterState.getRoutingNodes();
-        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(STARTED).size(), Matchers.equalTo(6));
-        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(INITIALIZING).size(), Matchers.equalTo(0));
-        assertThat(clusterState.getRoutingNodes().node("node0").shardsWithState(INITIALIZING).size(), Matchers.equalTo(0));
-        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(INITIALIZING).size(), Matchers.equalTo(0));
-        assertTrue(routingResult.changed());
-
-        routingResult = service.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
-        assertFalse(routingResult.changed());
-        assertNumIndexShardsPerNode(clusterState, Matchers.equalTo(2));
-        logger.debug("ClusterState: {}", clusterState.getRoutingNodes().prettyPrint());
+        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(STARTED).size(), equalTo(6));
+        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(INITIALIZING).size(), equalTo(0));
+        assertThat(clusterState.getRoutingNodes().node("node0").shardsWithState(INITIALIZING).size(), equalTo(0));
+        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(INITIALIZING).size(), equalTo(0));
+
+        newState = service.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
+        assertThat(newState, equalTo(clusterState));
+        assertNumIndexShardsPerNode(clusterState, equalTo(2));
+        logger.debug("ClusterState: {}", clusterState.getRoutingNodes());
     }
 
     public void testMinimalRelocationsNoLimit() {
@@ -161,60 +161,58 @@ public void testMinimalRelocationsNoLimit() {
         AllocationService service = createAllocationService(settings.build());
 
         ClusterState clusterState = initCluster(service, 1, 3, 3, 1);
-        assertThat(clusterState.getRoutingNodes().node("node0").shardsWithState(STARTED).size(), Matchers.equalTo(9));
-        assertThat(clusterState.getRoutingNodes().unassigned().size(), Matchers.equalTo(9));
+        assertThat(clusterState.getRoutingNodes().node("node0").shardsWithState(STARTED).size(), equalTo(9));
+        assertThat(clusterState.getRoutingNodes().unassigned().size(), equalTo(9));
         int nodeOffset = 1;
         clusterState = addNodes(clusterState, service, 1, nodeOffset++);
-        assertThat(clusterState.getRoutingNodes().node("node0").shardsWithState(STARTED).size(), Matchers.equalTo(9));
-        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(STARTED).size(), Matchers.equalTo(9));
-        assertThat(clusterState.getRoutingNodes().unassigned().size(), Matchers.equalTo(0));
-        assertNumIndexShardsPerNode(clusterState, Matchers.equalTo(3));
+        assertThat(clusterState.getRoutingNodes().node("node0").shardsWithState(STARTED).size(), equalTo(9));
+        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(STARTED).size(), equalTo(9));
+        assertThat(clusterState.getRoutingNodes().unassigned().size(), equalTo(0));
+        assertNumIndexShardsPerNode(clusterState, equalTo(3));
 
         logger.info("now, start one more node, check that rebalancing will happen because we set it to always");
         DiscoveryNodes.Builder nodes = DiscoveryNodes.builder(clusterState.nodes());
         nodes.add(newNode("node2"));
         clusterState = ClusterState.builder(clusterState).nodes(nodes.build()).build();
 
-        RoutingAllocation.Result routingResult = service.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = service.reroute(clusterState, "reroute");
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
 
-        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(INITIALIZING).size(), Matchers.equalTo(2));
-        assertThat(clusterState.getRoutingNodes().node("node0").shardsWithState(INITIALIZING).size(), Matchers.equalTo(0));
-        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(INITIALIZING).size(), Matchers.equalTo(0));
+        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(INITIALIZING).size(), equalTo(2));
+        assertThat(clusterState.getRoutingNodes().node("node0").shardsWithState(INITIALIZING).size(), equalTo(0));
+        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(INITIALIZING).size(), equalTo(0));
 
-        routingResult = service.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        ClusterState newState = service.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
         routingNodes = clusterState.getRoutingNodes();
-        assertTrue(routingResult.changed());
-        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(STARTED).size(), Matchers.equalTo(2));
-        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(INITIALIZING).size(), Matchers.equalTo(2));
-        assertThat(clusterState.getRoutingNodes().node("node0").shardsWithState(INITIALIZING).size(), Matchers.equalTo(0));
-        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(INITIALIZING).size(), Matchers.equalTo(0));
-
-        routingResult = service.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(STARTED).size(), equalTo(2));
+        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(INITIALIZING).size(), equalTo(2));
+        assertThat(clusterState.getRoutingNodes().node("node0").shardsWithState(INITIALIZING).size(), equalTo(0));
+        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(INITIALIZING).size(), equalTo(0));
+
+        newState = service.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
         routingNodes = clusterState.getRoutingNodes();
-        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(STARTED).size(), Matchers.equalTo(4));
-        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(INITIALIZING).size(), Matchers.equalTo(2));
-        assertThat(clusterState.getRoutingNodes().node("node0").shardsWithState(INITIALIZING).size(), Matchers.equalTo(0));
-        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(INITIALIZING).size(), Matchers.equalTo(0));
-        assertTrue(routingResult.changed());
-
-        routingResult = service.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(STARTED).size(), equalTo(4));
+        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(INITIALIZING).size(), equalTo(2));
+        assertThat(clusterState.getRoutingNodes().node("node0").shardsWithState(INITIALIZING).size(), equalTo(0));
+        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(INITIALIZING).size(), equalTo(0));
+
+        newState = service.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
         routingNodes = clusterState.getRoutingNodes();
-        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(STARTED).size(), Matchers.equalTo(6));
-        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(INITIALIZING).size(), Matchers.equalTo(0));
-        assertThat(clusterState.getRoutingNodes().node("node0").shardsWithState(INITIALIZING).size(), Matchers.equalTo(0));
-        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(INITIALIZING).size(), Matchers.equalTo(0));
-        assertTrue(routingResult.changed());
-
-        routingResult = service.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
-        assertFalse(routingResult.changed());
-        assertNumIndexShardsPerNode(clusterState, Matchers.equalTo(2));
-        logger.debug("ClusterState: {}", clusterState.getRoutingNodes().prettyPrint());
+        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(STARTED).size(), equalTo(6));
+        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(INITIALIZING).size(), equalTo(0));
+        assertThat(clusterState.getRoutingNodes().node("node0").shardsWithState(INITIALIZING).size(), equalTo(0));
+        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(INITIALIZING).size(), equalTo(0));
+
+        newState = service.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
+        assertThat(newState, equalTo(clusterState));
+        assertNumIndexShardsPerNode(clusterState, equalTo(2));
+        logger.debug("ClusterState: {}", clusterState.getRoutingNodes());
     }
 
 
@@ -250,8 +248,7 @@ private ClusterState addNodes(ClusterState clusterState, AllocationService servi
 
         clusterState = ClusterState.builder(clusterState).nodes(nodes.build()).build();
 
-        RoutingAllocation.Result routingResult = service.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = service.reroute(clusterState, "reroute");
 
         // move initializing to started
         return applyStartedShardsUntilNoChange(clusterState, service);
@@ -282,18 +279,15 @@ private ClusterState initCluster(AllocationService service, int numberOfNodes, i
             nodes.add(newNode("node" + i));
         }
         ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).nodes(nodes).metaData(metaData).routingTable(initialRoutingTable).build();
-        RoutingAllocation.Result routingResult = service.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = service.reroute(clusterState, "reroute");
 
         logger.info("restart all the primary shards, replicas will start initializing");
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
-        routingResult = service.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = service.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
 
         logger.info("start the replica shards");
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = service.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = service.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
         routingNodes = clusterState.getRoutingNodes();
 
         logger.info("complete rebalancing");
@@ -313,18 +307,15 @@ private ClusterState addIndex(ClusterState clusterState, AllocationService servi
 
         MetaData metaData = metaDataBuilder.build();
         clusterState = ClusterState.builder(clusterState).metaData(metaData).routingTable(routingTableBuilder.build()).build();
-        RoutingAllocation.Result routingResult = service.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = service.reroute(clusterState, "reroute");
 
         logger.info("restart all the primary shards, replicas will start initializing");
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
-        routingResult = service.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = service.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
 
         logger.info("start the replica shards");
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = service.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = service.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
 
         logger.info("complete rebalancing");
         return applyStartedShardsUntilNoChange(clusterState, service);
@@ -344,22 +335,18 @@ private ClusterState removeNodes(ClusterState clusterState, AllocationService se
         }
 
         clusterState = ClusterState.builder(clusterState).nodes(nodes.build()).build();
-        clusterState = ClusterState.builder(clusterState)
-            .routingResult(service.deassociateDeadNodes(clusterState, true, "reroute")).build();
+        clusterState = service.deassociateDeadNodes(clusterState, true, "reroute");
 
         logger.info("start all the primary shards, replicas will start initializing");
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
-        RoutingAllocation.Result routingResult = service.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = service.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
 
         logger.info("start the replica shards");
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = service.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = service.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
 
         logger.info("rebalancing");
-        routingResult = service.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = service.reroute(clusterState, "reroute");
 
         logger.info("complete rebalancing");
         clusterState = applyStartedShardsUntilNoChange(clusterState, service);
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocateUnassignedDecisionTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocateUnassignedDecisionTests.java
new file mode 100644
index 0000000000000..096576a2bb4db
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocateUnassignedDecisionTests.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.routing.allocation;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.UnassignedInfo.AllocationStatus;
+import org.elasticsearch.cluster.routing.allocation.decider.Decision;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.transport.LocalTransportAddress;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static java.util.Collections.emptyList;
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.emptySet;
+import static org.hamcrest.Matchers.startsWith;
+
+/**
+ * Unit tests for the {@link AllocateUnassignedDecision} class.
+ */
+public class AllocateUnassignedDecisionTests extends ESTestCase {
+
+    private DiscoveryNode node1 = new DiscoveryNode("node1", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
+    private DiscoveryNode node2 = new DiscoveryNode("node2", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
+
+    public void testDecisionNotTaken() {
+        AllocateUnassignedDecision allocateUnassignedDecision = AllocateUnassignedDecision.NOT_TAKEN;
+        assertFalse(allocateUnassignedDecision.isDecisionTaken());
+        expectThrows(IllegalStateException.class, () -> allocateUnassignedDecision.getAllocationDecision());
+        expectThrows(IllegalStateException.class, () -> allocateUnassignedDecision.getAllocationStatus());
+        expectThrows(IllegalStateException.class, () -> allocateUnassignedDecision.getAllocationId());
+        expectThrows(IllegalStateException.class, () -> allocateUnassignedDecision.getTargetNode());
+        expectThrows(IllegalStateException.class, () -> allocateUnassignedDecision.getNodeDecisions());
+        expectThrows(IllegalStateException.class, () -> allocateUnassignedDecision.getExplanation());
+    }
+
+    public void testNoDecision() {
+        final AllocationStatus allocationStatus = randomFrom(
+            AllocationStatus.DELAYED_ALLOCATION, AllocationStatus.NO_VALID_SHARD_COPY, AllocationStatus.FETCHING_SHARD_DATA
+        );
+        AllocateUnassignedDecision noDecision = AllocateUnassignedDecision.no(allocationStatus, null);
+        assertTrue(noDecision.isDecisionTaken());
+        assertEquals(AllocationDecision.fromAllocationStatus(allocationStatus), noDecision.getAllocationDecision());
+        assertEquals(allocationStatus, noDecision.getAllocationStatus());
+        if (allocationStatus == AllocationStatus.FETCHING_SHARD_DATA) {
+            assertEquals("cannot allocate because information about existing shard data is still being retrieved from " +
+                             "some of the nodes", noDecision.getExplanation());
+        } else if (allocationStatus == AllocationStatus.DELAYED_ALLOCATION) {
+            assertThat(noDecision.getExplanation(), startsWith("cannot allocate because the cluster is still waiting"));
+        } else {
+            assertThat(noDecision.getExplanation(),
+                startsWith("cannot allocate because a previous copy of the primary shard existed"));
+        }
+        assertNull(noDecision.getNodeDecisions());
+        assertNull(noDecision.getTargetNode());
+        assertNull(noDecision.getAllocationId());
+
+        List<NodeAllocationResult> nodeDecisions = new ArrayList<>();
+        nodeDecisions.add(new NodeAllocationResult(node1, Decision.NO, 1));
+        nodeDecisions.add(new NodeAllocationResult(node2, Decision.NO, 2));
+        final boolean reuseStore = randomBoolean();
+        noDecision = AllocateUnassignedDecision.no(AllocationStatus.DECIDERS_NO, nodeDecisions, reuseStore);
+        assertTrue(noDecision.isDecisionTaken());
+        assertEquals(AllocationDecision.NO, noDecision.getAllocationDecision());
+        assertEquals(AllocationStatus.DECIDERS_NO, noDecision.getAllocationStatus());
+        if (reuseStore) {
+            assertEquals("cannot allocate because allocation is not permitted to any of the nodes that hold an in-sync shard copy",
+                noDecision.getExplanation());
+        } else {
+            assertEquals("cannot allocate because allocation is not permitted to any of the nodes", noDecision.getExplanation());
+        }
+        assertEquals(nodeDecisions.stream().sorted().collect(Collectors.toList()), noDecision.getNodeDecisions());
+        // node1 should be sorted first b/c of better weight ranking
+        assertEquals("node1", noDecision.getNodeDecisions().iterator().next().getNode().getId());
+        assertNull(noDecision.getTargetNode());
+        assertNull(noDecision.getAllocationId());
+
+        // test bad values
+        expectThrows(NullPointerException.class, () -> AllocateUnassignedDecision.no(null, null));
+    }
+
+    public void testThrottleDecision() {
+        List<NodeAllocationResult> nodeDecisions = new ArrayList<>();
+        nodeDecisions.add(new NodeAllocationResult(node1, Decision.NO, 1));
+        nodeDecisions.add(new NodeAllocationResult(node2, Decision.THROTTLE, 2));
+        AllocateUnassignedDecision throttleDecision = AllocateUnassignedDecision.throttle(nodeDecisions);
+        assertTrue(throttleDecision.isDecisionTaken());
+        assertEquals(AllocationDecision.THROTTLED, throttleDecision.getAllocationDecision());
+        assertEquals(AllocationStatus.DECIDERS_THROTTLED, throttleDecision.getAllocationStatus());
+        assertThat(throttleDecision.getExplanation(), startsWith("allocation temporarily throttled"));
+        assertEquals(nodeDecisions.stream().sorted().collect(Collectors.toList()), throttleDecision.getNodeDecisions());
+        // node2 should be sorted first b/c a THROTTLE is higher than a NO decision
+        assertEquals("node2", throttleDecision.getNodeDecisions().iterator().next().getNode().getId());
+        assertNull(throttleDecision.getTargetNode());
+        assertNull(throttleDecision.getAllocationId());
+    }
+
+    public void testYesDecision() {
+        List<NodeAllocationResult> nodeDecisions = new ArrayList<>();
+        nodeDecisions.add(new NodeAllocationResult(node1, Decision.NO, 1));
+        nodeDecisions.add(new NodeAllocationResult(node2, Decision.YES, 2));
+        String allocId = randomBoolean() ? "allocId" : null;
+        AllocateUnassignedDecision yesDecision = AllocateUnassignedDecision.yes(
+            node2, allocId, nodeDecisions, randomBoolean());
+        assertTrue(yesDecision.isDecisionTaken());
+        assertEquals(AllocationDecision.YES, yesDecision.getAllocationDecision());
+        assertNull(yesDecision.getAllocationStatus());
+        assertEquals("can allocate the shard", yesDecision.getExplanation());
+        assertEquals(nodeDecisions.stream().sorted().collect(Collectors.toList()), yesDecision.getNodeDecisions());
+        assertEquals("node2", yesDecision.getTargetNode().getId());
+        assertEquals(allocId, yesDecision.getAllocationId());
+        // node1 should be sorted first b/c YES decisions are the highest
+        assertEquals("node2", yesDecision.getNodeDecisions().iterator().next().getNode().getId());
+    }
+
+    public void testCachedDecisions() {
+        List<AllocationStatus> cachableStatuses = Arrays.asList(AllocationStatus.DECIDERS_NO, AllocationStatus.DECIDERS_THROTTLED,
+            AllocationStatus.NO_VALID_SHARD_COPY, AllocationStatus.FETCHING_SHARD_DATA, AllocationStatus.DELAYED_ALLOCATION);
+        for (AllocationStatus allocationStatus : cachableStatuses) {
+            if (allocationStatus == AllocationStatus.DECIDERS_THROTTLED) {
+                AllocateUnassignedDecision cached = AllocateUnassignedDecision.throttle(null);
+                AllocateUnassignedDecision another = AllocateUnassignedDecision.throttle(null);
+                assertSame(cached, another);
+                AllocateUnassignedDecision notCached = AllocateUnassignedDecision.throttle(new ArrayList<>());
+                another = AllocateUnassignedDecision.throttle(new ArrayList<>());
+                assertNotSame(notCached, another);
+            } else {
+                AllocateUnassignedDecision cached = AllocateUnassignedDecision.no(allocationStatus, null);
+                AllocateUnassignedDecision another = AllocateUnassignedDecision.no(allocationStatus, null);
+                assertSame(cached, another);
+                AllocateUnassignedDecision notCached = AllocateUnassignedDecision.no(allocationStatus, new ArrayList<>());
+                another = AllocateUnassignedDecision.no(allocationStatus, new ArrayList<>());
+                assertNotSame(notCached, another);
+            }
+        }
+
+        // yes decisions are not precomputed and cached
+        AllocateUnassignedDecision first = AllocateUnassignedDecision.yes(node1, "abc", emptyList(), randomBoolean());
+        AllocateUnassignedDecision second = AllocateUnassignedDecision.yes(node1, "abc", emptyList(), randomBoolean());
+        // same fields for the ShardAllocationDecision, but should be different instances
+        assertNotSame(first, second);
+    }
+
+    public void testSerialization() throws IOException {
+        DiscoveryNode node1 = new DiscoveryNode("node1", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
+        DiscoveryNode node2 = new DiscoveryNode("node2", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
+        Decision.Type finalDecision = randomFrom(Decision.Type.values());
+        DiscoveryNode assignedNode = finalDecision == Decision.Type.YES ? node1 : null;
+        List<NodeAllocationResult> nodeDecisions = new ArrayList<>();
+        nodeDecisions.add(new NodeAllocationResult(node1, Decision.NO, 2));
+        nodeDecisions.add(new NodeAllocationResult(node2, finalDecision == Decision.Type.YES ? Decision.YES :
+                                                              randomFrom(Decision.NO, Decision.THROTTLE, Decision.YES), 1));
+        AllocateUnassignedDecision decision;
+        if (finalDecision == Decision.Type.YES) {
+            decision = AllocateUnassignedDecision.yes(assignedNode, randomBoolean() ? randomAlphaOfLength(5) : null,
+                nodeDecisions, randomBoolean());
+        } else {
+            decision = AllocateUnassignedDecision.no(randomFrom(
+                AllocationStatus.DELAYED_ALLOCATION, AllocationStatus.NO_VALID_SHARD_COPY, AllocationStatus.FETCHING_SHARD_DATA
+            ), nodeDecisions, randomBoolean());
+        }
+        BytesStreamOutput output = new BytesStreamOutput();
+        decision.writeTo(output);
+        AllocateUnassignedDecision readDecision = new AllocateUnassignedDecision(output.bytes().streamInput());
+        assertEquals(decision.getTargetNode(), readDecision.getTargetNode());
+        assertEquals(decision.getAllocationStatus(), readDecision.getAllocationStatus());
+        assertEquals(decision.getExplanation(), readDecision.getExplanation());
+        assertEquals(decision.getNodeDecisions().size(), readDecision.getNodeDecisions().size());
+        assertEquals(decision.getAllocationId(), readDecision.getAllocationId());
+        assertEquals(decision.getAllocationDecision(), readDecision.getAllocationDecision());
+        // node2 should have the highest sort order
+        assertEquals("node2", readDecision.getNodeDecisions().iterator().next().getNode().getId());
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java
index 24faf9c45aef3..90e9d156a8fe0 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java
@@ -22,6 +22,7 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -32,12 +33,10 @@
 import org.elasticsearch.cluster.routing.allocation.command.AllocateEmptyPrimaryAllocationCommand;
 import org.elasticsearch.cluster.routing.allocation.command.AllocateReplicaAllocationCommand;
 import org.elasticsearch.cluster.routing.allocation.command.AllocateStalePrimaryAllocationCommand;
-import org.elasticsearch.cluster.routing.allocation.command.AllocationCommandRegistry;
 import org.elasticsearch.cluster.routing.allocation.command.AllocationCommands;
 import org.elasticsearch.cluster.routing.allocation.command.CancelAllocationCommand;
 import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand;
 import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -45,13 +44,12 @@
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.ShardNotFoundException;
-import org.elasticsearch.cluster.ESAllocationTestCase;
 
 import java.util.Collections;
 
@@ -61,6 +59,7 @@
 import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.nullValue;
 
 /**
@@ -82,12 +81,10 @@ public void testMoveShardCommand() {
 
         logger.info("adding two nodes and performing rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))).build();
-        RoutingAllocation.Result rerouteResult = allocation.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.reroute(clusterState, "reroute");
 
         logger.info("start primary shard");
-        rerouteResult = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         logger.info("move the shard");
         String existingNodeId = clusterState.routingTable().index("test").shard(0).primaryShard().currentNodeId();
@@ -97,15 +94,15 @@ public void testMoveShardCommand() {
         } else {
             toNodeId = "node1";
         }
-        rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new MoveAllocationCommand("test", 0, existingNodeId, toNodeId)), false, false);
-        assertThat(rerouteResult.changed(), equalTo(true));
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        ClusterState newState = allocation.reroute(clusterState,
+            new AllocationCommands(new MoveAllocationCommand("test", 0, existingNodeId, toNodeId)), false, false).getClusterState();
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
         assertThat(clusterState.getRoutingNodes().node(existingNodeId).iterator().next().state(), equalTo(ShardRoutingState.RELOCATING));
         assertThat(clusterState.getRoutingNodes().node(toNodeId).iterator().next().state(), equalTo(ShardRoutingState.INITIALIZING));
 
         logger.info("finish moving the shard");
-        rerouteResult = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         assertThat(clusterState.getRoutingNodes().node(existingNodeId).isEmpty(), equalTo(true));
         assertThat(clusterState.getRoutingNodes().node(toNodeId).iterator().next().state(), equalTo(ShardRoutingState.STARTED));
@@ -146,8 +143,7 @@ public void testAllocateCommand() {
                 .add(newNode("node3"))
                 .add(newNode("node4", singleton(DiscoveryNode.Role.MASTER)))
         ).build();
-        RoutingAllocation.Result rerouteResult = allocation.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.reroute(clusterState, "reroute");
         assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(0));
 
         logger.info("--> allocating to non-existent node, should fail");
@@ -199,16 +195,16 @@ public void testAllocateCommand() {
         }
 
         logger.info("--> allocating empty primary with acceptDataLoss flag set to true");
-        rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new AllocateEmptyPrimaryAllocationCommand("test", 0, "node1", true)), false, false);
-        assertThat(rerouteResult.changed(), equalTo(true));
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        ClusterState newState = allocation.reroute(clusterState,
+            new AllocationCommands(new AllocateEmptyPrimaryAllocationCommand("test", 0, "node1", true)), false, false).getClusterState();
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(INITIALIZING).size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(0));
 
         logger.info("--> start the primary shard");
-        rerouteResult = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(STARTED).size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(0));
@@ -221,9 +217,10 @@ public void testAllocateCommand() {
         }
 
         logger.info("--> allocate the replica shard on on the second node");
-        rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new AllocateReplicaAllocationCommand("test", 0, "node2")), false, false);
-        assertThat(rerouteResult.changed(), equalTo(true));
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        newState = allocation.reroute(clusterState,
+            new AllocationCommands(new AllocateReplicaAllocationCommand("test", 0, "node2")), false, false).getClusterState();
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(STARTED).size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(1));
@@ -231,8 +228,7 @@ public void testAllocateCommand() {
 
 
         logger.info("--> start the replica shard");
-        rerouteResult = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(STARTED).size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(1));
@@ -267,14 +263,14 @@ public void testCancelCommand() {
                 .add(newNode("node2"))
                 .add(newNode("node3"))
         ).build();
-        RoutingAllocation.Result rerouteResult = allocation.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.reroute(clusterState, "reroute");
         assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(0));
 
         logger.info("--> allocating empty primary shard with accept_data_loss flag set to true");
-        rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new AllocateEmptyPrimaryAllocationCommand("test", 0, "node1", true)), false, false);
-        assertThat(rerouteResult.changed(), equalTo(true));
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        ClusterState newState = allocation.reroute(clusterState,
+            new AllocationCommands(new AllocateEmptyPrimaryAllocationCommand("test", 0, "node1", true)), false, false).getClusterState();
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(INITIALIZING).size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(0));
@@ -287,8 +283,7 @@ public void testCancelCommand() {
         }
 
         logger.info("--> start the primary shard");
-        rerouteResult = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(STARTED).size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(0));
@@ -301,27 +296,30 @@ public void testCancelCommand() {
         }
 
         logger.info("--> allocate the replica shard on on the second node");
-        rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new AllocateReplicaAllocationCommand("test", 0, "node2")), false, false);
-        assertThat(rerouteResult.changed(), equalTo(true));
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        newState = allocation.reroute(clusterState,
+            new AllocationCommands(new AllocateReplicaAllocationCommand("test", 0, "node2")), false, false).getClusterState();
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(STARTED).size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(INITIALIZING).size(), equalTo(1));
 
         logger.info("--> cancel the relocation allocation");
-        rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new CancelAllocationCommand("test", 0, "node2", false)), false, false);
-        assertThat(rerouteResult.changed(), equalTo(true));
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        newState = allocation.reroute(clusterState,
+            new AllocationCommands(new CancelAllocationCommand("test", 0, "node2", false)), false, false).getClusterState();
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(STARTED).size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(0));
         assertThat(clusterState.getRoutingNodes().node("node3").size(), equalTo(0));
 
         logger.info("--> allocate the replica shard on on the second node");
-        rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new AllocateReplicaAllocationCommand("test", 0, "node2")), false, false);
-        assertThat(rerouteResult.changed(), equalTo(true));
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        newState = allocation.reroute(clusterState,
+            new AllocationCommands(new AllocateReplicaAllocationCommand("test", 0, "node2")), false, false).getClusterState();
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(STARTED).size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(1));
@@ -335,41 +333,41 @@ public void testCancelCommand() {
         }
 
         logger.info("--> start the replica shard");
-        rerouteResult = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(STARTED).size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(STARTED).size(), equalTo(1));
 
         logger.info("--> cancel allocation of the replica shard");
-        rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new CancelAllocationCommand("test", 0, "node2", false)), false, false);
-        assertThat(rerouteResult.changed(), equalTo(true));
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        newState = allocation.reroute(clusterState,
+            new AllocationCommands(new CancelAllocationCommand("test", 0, "node2", false)), false, false).getClusterState();
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(STARTED).size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(0));
         assertThat(clusterState.getRoutingNodes().node("node3").size(), equalTo(0));
 
         logger.info("--> allocate the replica shard on on the second node");
-        rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new AllocateReplicaAllocationCommand("test", 0, "node2")), false, false);
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
-        assertThat(rerouteResult.changed(), equalTo(true));
+        newState = allocation.reroute(clusterState,
+            new AllocationCommands(new AllocateReplicaAllocationCommand("test", 0, "node2")), false, false).getClusterState();
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(STARTED).size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(INITIALIZING).size(), equalTo(1));
         logger.info("--> start the replica shard");
-        rerouteResult = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(STARTED).size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(STARTED).size(), equalTo(1));
 
         logger.info("--> move the replica shard");
-        rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new MoveAllocationCommand("test", 0, "node2", "node3")), false, false);
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.reroute(clusterState,
+            new AllocationCommands(new MoveAllocationCommand("test", 0, "node2", "node3")), false, false).getClusterState();
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(STARTED).size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(1));
@@ -379,24 +377,25 @@ public void testCancelCommand() {
 
         if (randomBoolean()) {
             logger.info("--> cancel the primary allocation (with allow_primary set to true)");
-            rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new CancelAllocationCommand("test", 0, "node1", true)), false, false);
-            clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
-            assertThat(rerouteResult.changed(), equalTo(true));
+            newState = allocation.reroute(clusterState,
+                new AllocationCommands(new CancelAllocationCommand("test", 0, "node1", true)), false, false).getClusterState();
+            assertThat(newState, not(equalTo(clusterState)));
+            clusterState = newState;
             assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(0));
             assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(STARTED).iterator().next().primary(), equalTo(true));
             assertThat(clusterState.getRoutingNodes().node("node3").size(), equalTo(0));
         } else {
             logger.info("--> cancel the move of the replica shard");
-            rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new CancelAllocationCommand("test", 0, "node3", false)), false, false);
-            clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+            clusterState = allocation.reroute(clusterState,
+                new AllocationCommands(new CancelAllocationCommand("test", 0, "node3", false)), false, false).getClusterState();
             assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
             assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(STARTED).size(), equalTo(1));
             assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(1));
             assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(STARTED).size(), equalTo(1));
 
             logger.info("--> move the replica shard again");
-            rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new MoveAllocationCommand("test", 0, "node2", "node3")), false, false);
-            clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+            clusterState = allocation.reroute(clusterState,
+                new AllocationCommands(new MoveAllocationCommand("test", 0, "node2", "node3")), false, false).getClusterState();
             assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
             assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(STARTED).size(), equalTo(1));
             assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(1));
@@ -405,8 +404,8 @@ public void testCancelCommand() {
             assertThat(clusterState.getRoutingNodes().node("node3").shardsWithState(INITIALIZING).size(), equalTo(1));
 
             logger.info("--> cancel the source replica shard");
-            rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new CancelAllocationCommand("test", 0, "node2", false)), false, false);
-            clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+            clusterState = allocation.reroute(clusterState,
+                new AllocationCommands(new CancelAllocationCommand("test", 0, "node2", false)), false, false).getClusterState();
             assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
             assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(STARTED).size(), equalTo(1));
             assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(0));
@@ -415,17 +414,17 @@ public void testCancelCommand() {
             assertThat(clusterState.getRoutingNodes().node("node3").shardsWithState(INITIALIZING).get(0).relocatingNodeId(), nullValue());
 
             logger.info("--> start the former target replica shard");
-            rerouteResult = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-            clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
-            assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
+            clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
+                assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
             assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(STARTED).size(), equalTo(1));
             assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(0));
             assertThat(clusterState.getRoutingNodes().node("node3").shardsWithState(STARTED).size(), equalTo(1));
 
             logger.info("--> cancel the primary allocation (with allow_primary set to true)");
-            rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new CancelAllocationCommand("test", 0, "node1", true)), false, false);
-            clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
-            assertThat(rerouteResult.changed(), equalTo(true));
+            newState = allocation.reroute(clusterState,
+                new AllocationCommands(new CancelAllocationCommand("test", 0, "node1", true)), false, false).getClusterState();
+            assertThat(newState, not(equalTo(clusterState)));
+            clusterState = newState;
             assertThat(clusterState.getRoutingNodes().node("node3").shardsWithState(STARTED).iterator().next().primary(), equalTo(true));
             assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(0));
             assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(0));
@@ -445,8 +444,7 @@ public void testSerialization() throws Exception {
         StreamInput in = bytes.bytes().streamInput();
 
         // Since the commands are named writeable we need to register them and wrap the input stream
-        NetworkModule networkModule = new NetworkModule(null, Settings.EMPTY, true);
-        NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(networkModule.getNamedWriteables());
+        NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(NetworkModule.getNamedWriteables());
         in = new NamedWriteableAwareStreamInput(in, namedWriteableRegistry);
 
         // Now we can read them!
@@ -488,12 +486,11 @@ public void testXContent() throws Exception {
                 "       ,{\"cancel\" : {\"index\" : \"test\", \"shard\" : 4, \"node\" : \"node5\", \"allow_primary\" : true}} \n" +
                 "    ]\n" +
                 "}\n";
-        XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(commands);
+        XContentParser parser = createParser(JsonXContent.jsonXContent, commands);
         // move two tokens, parser expected to be "on" `commands` field
         parser.nextToken();
         parser.nextToken();
-        AllocationCommandRegistry registry = new NetworkModule(null, Settings.EMPTY, true).getAllocationCommandRegistry();
-        AllocationCommands sCommands = AllocationCommands.fromXContent(parser, ParseFieldMatcher.STRICT, registry);
+        AllocationCommands sCommands = AllocationCommands.fromXContent(parser);
 
         assertThat(sCommands.commands().size(), equalTo(5));
         assertThat(((AllocateEmptyPrimaryAllocationCommand) (sCommands.commands().get(0))).shardId(), equalTo(1));
@@ -520,4 +517,9 @@ public void testXContent() throws Exception {
         assertThat(((CancelAllocationCommand) (sCommands.commands().get(4))).node(), equalTo("node5"));
         assertThat(((CancelAllocationCommand) (sCommands.commands().get(4))).allowPrimary(), equalTo(true));
     }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return new NamedXContentRegistry(NetworkModule.getNamedXContents());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationDecisionTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationDecisionTests.java
new file mode 100644
index 0000000000000..0c76338cefc5f
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationDecisionTests.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.routing.allocation;
+
+import org.elasticsearch.cluster.routing.UnassignedInfo.AllocationStatus;
+import org.elasticsearch.cluster.routing.allocation.decider.Decision.Type;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+/**
+ * Tests for the {@link AllocationDecision} enum.
+ */
+public class AllocationDecisionTests extends ESTestCase {
+
+    /**
+     * Tests serialization and deserialization.
+     */
+    public void testSerialization() throws IOException {
+        AllocationDecision allocationDecision = randomFrom(AllocationDecision.values());
+        BytesStreamOutput output = new BytesStreamOutput();
+        allocationDecision.writeTo(output);
+        assertEquals(allocationDecision, AllocationDecision.readFrom(output.bytes().streamInput()));
+    }
+
+    /**
+     * Tests the order of values in the enum, because we depend on the natural enum sort order for sorting node decisions.
+     * See {@link AbstractAllocationDecision#getNodeDecisions()}.
+     */
+    public void testValuesOrder() {
+        assertEquals(0, AllocationDecision.YES.ordinal());
+        assertEquals(1, AllocationDecision.THROTTLED.ordinal());
+        assertEquals(2, AllocationDecision.NO.ordinal());
+        assertEquals(3, AllocationDecision.WORSE_BALANCE.ordinal());
+        assertEquals(4, AllocationDecision.AWAITING_INFO.ordinal());
+        assertEquals(5, AllocationDecision.ALLOCATION_DELAYED.ordinal());
+        assertEquals(6, AllocationDecision.NO_VALID_SHARD_COPY.ordinal());
+        assertEquals(7, AllocationDecision.NO_ATTEMPT.ordinal());
+        AllocationDecision[] decisions = AllocationDecision.values();
+        Arrays.sort(decisions);
+        assertEquals(AllocationDecision.YES, decisions[0]);
+        assertEquals(AllocationDecision.THROTTLED, decisions[1]);
+        assertEquals(AllocationDecision.NO, decisions[2]);
+        assertEquals(AllocationDecision.WORSE_BALANCE, decisions[3]);
+        assertEquals(AllocationDecision.AWAITING_INFO, decisions[4]);
+        assertEquals(AllocationDecision.ALLOCATION_DELAYED, decisions[5]);
+        assertEquals(AllocationDecision.NO_VALID_SHARD_COPY, decisions[6]);
+        assertEquals(AllocationDecision.NO_ATTEMPT, decisions[7]);
+    }
+
+    /**
+     * Tests getting a {@link AllocationDecision} from {@link Type}.
+     */
+    public void testFromDecisionType() {
+        Type type = randomFrom(Type.values());
+        AllocationDecision allocationDecision = AllocationDecision.fromDecisionType(type);
+        AllocationDecision expected = type == Type.NO ? AllocationDecision.NO :
+                                          type == Type.THROTTLE ? AllocationDecision.THROTTLED : AllocationDecision.YES;
+        assertEquals(expected, allocationDecision);
+    }
+
+    /**
+     * Tests getting a {@link AllocationDecision} from {@link AllocationStatus}.
+     */
+    public void testFromAllocationStatus() {
+        AllocationStatus allocationStatus = rarely() ? null : randomFrom(AllocationStatus.values());
+        AllocationDecision allocationDecision = AllocationDecision.fromAllocationStatus(allocationStatus);
+        AllocationDecision expected;
+        if (allocationStatus == null) {
+            expected = AllocationDecision.YES;
+        } else if (allocationStatus == AllocationStatus.DECIDERS_THROTTLED) {
+            expected = AllocationDecision.THROTTLED;
+        } else if (allocationStatus == AllocationStatus.FETCHING_SHARD_DATA) {
+            expected = AllocationDecision.AWAITING_INFO;
+        } else if (allocationStatus == AllocationStatus.DELAYED_ALLOCATION) {
+            expected = AllocationDecision.ALLOCATION_DELAYED;
+        } else if (allocationStatus == AllocationStatus.NO_VALID_SHARD_COPY) {
+            expected = AllocationDecision.NO_VALID_SHARD_COPY;
+        } else if (allocationStatus == AllocationStatus.NO_ATTEMPT) {
+            expected = AllocationDecision.NO_ATTEMPT;
+        } else {
+            expected = AllocationDecision.NO;
+        }
+        assertEquals(expected, allocationDecision);
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationPriorityTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationPriorityTests.java
index 7e93fa040d277..684985801c4d1 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationPriorityTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationPriorityTests.java
@@ -20,13 +20,13 @@
 
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.cluster.ESAllocationTestCase;
 
 import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
 
@@ -68,29 +68,24 @@ public void testPrioritizedIndicesAllocatedFirst() {
         ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(initialRoutingTable).build();
 
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))).build();
-        RoutingAllocation.Result rerouteResult = allocation.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
+        clusterState = allocation.reroute(clusterState, "reroute");
 
-        RoutingAllocation.Result routingResult = allocation.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = allocation.reroute(clusterState, "reroute");
         assertEquals(2, clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size());
         assertEquals(highPriorityName, clusterState.getRoutingNodes().shardsWithState(INITIALIZING).get(0).getIndexName());
         assertEquals(highPriorityName, clusterState.getRoutingNodes().shardsWithState(INITIALIZING).get(1).getIndexName());
 
-        routingResult = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
         assertEquals(2, clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size());
         assertEquals(lowPriorityName, clusterState.getRoutingNodes().shardsWithState(INITIALIZING).get(0).getIndexName());
         assertEquals(lowPriorityName, clusterState.getRoutingNodes().shardsWithState(INITIALIZING).get(1).getIndexName());
 
-        routingResult = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
         assertEquals(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).toString(),2, clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size());
         assertEquals(highPriorityName, clusterState.getRoutingNodes().shardsWithState(INITIALIZING).get(0).getIndexName());
         assertEquals(highPriorityName, clusterState.getRoutingNodes().shardsWithState(INITIALIZING).get(1).getIndexName());
 
-        routingResult = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
         assertEquals(2, clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size());
         assertEquals(lowPriorityName, clusterState.getRoutingNodes().shardsWithState(INITIALIZING).get(0).getIndexName());
         assertEquals(lowPriorityName, clusterState.getRoutingNodes().shardsWithState(INITIALIZING).get(1).getIndexName());
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AwarenessAllocationTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AwarenessAllocationTests.java
index f85cc97560e5a..2f23129e68a97 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AwarenessAllocationTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AwarenessAllocationTests.java
@@ -22,6 +22,7 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -31,10 +32,13 @@
 import org.elasticsearch.cluster.routing.allocation.command.AllocationCommands;
 import org.elasticsearch.cluster.routing.allocation.command.CancelAllocationCommand;
 import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand;
+import org.elasticsearch.cluster.routing.allocation.decider.AwarenessAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.cluster.ESAllocationTestCase;
+
+import java.util.HashMap;
+import java.util.Map;
 
 import static java.util.Collections.singletonMap;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
@@ -73,17 +77,14 @@ public void testMoveShardOnceNewNodeWithAttributeAdded1() {
                 .add(newNode("node1", singletonMap("rack_id", "1")))
                 .add(newNode("node2", singletonMap("rack_id", "1")))
         ).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
         assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(1));
 
         logger.info("--> start the shards (primaries)");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         logger.info("--> start the shards (replicas)");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(2));
 
@@ -91,16 +92,14 @@ public void testMoveShardOnceNewNodeWithAttributeAdded1() {
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
                 .add(newNode("node3", singletonMap("rack_id", "2")))
         ).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.RELOCATING).size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.RELOCATING).get(0).relocatingNodeId(), equalTo("node3"));
 
         logger.info("--> complete relocation");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(2));
 
@@ -111,9 +110,8 @@ public void testMoveShardOnceNewNodeWithAttributeAdded1() {
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
                 .add(newNode("node4", singletonMap("rack_id", "3")))
         ).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        assertFalse(routingResult.changed());
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        ClusterState newState = strategy.reroute(clusterState, "reroute");
+        assertThat(newState, equalTo(clusterState));
         assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(2));
     }
 
@@ -140,17 +138,14 @@ public void testMoveShardOnceNewNodeWithAttributeAdded2() {
                 .add(newNode("node2", singletonMap("rack_id", "1")))
                 .add(newNode("node3", singletonMap("rack_id", "1")))
         ).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
         assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(1));
 
         logger.info("--> start the shards (primaries)");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         logger.info("--> start the shards (replicas)");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(2));
 
@@ -158,16 +153,14 @@ public void testMoveShardOnceNewNodeWithAttributeAdded2() {
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
                 .add(newNode("node4", singletonMap("rack_id", "2")))
         ).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.RELOCATING).size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.RELOCATING).get(0).relocatingNodeId(), equalTo("node4"));
 
         logger.info("--> complete relocation");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(2));
 
@@ -178,9 +171,8 @@ public void testMoveShardOnceNewNodeWithAttributeAdded2() {
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
                 .add(newNode("node5", singletonMap("rack_id", "3")))
         ).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        assertFalse(routingResult.changed());
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        ClusterState newState = strategy.reroute(clusterState, "reroute");
+        assertThat(newState, equalTo(clusterState));
         assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(2));
     }
 
@@ -213,8 +205,7 @@ public void testMoveShardOnceNewNodeWithAttributeAdded3() {
                 .add(newNode("node1", singletonMap("rack_id", "1")))
                 .add(newNode("node2", singletonMap("rack_id", "1")))
         ).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         logger.info("Initializing shards: {}", clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
         logger.info("Started shards: {}", clusterState.getRoutingNodes().shardsWithState(STARTED));
@@ -224,12 +215,10 @@ public void testMoveShardOnceNewNodeWithAttributeAdded3() {
         assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(5));
 
         logger.info("--> start the shards (primaries)");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         logger.info("--> start the shards (replicas)");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(10));
 
@@ -237,8 +226,7 @@ public void testMoveShardOnceNewNodeWithAttributeAdded3() {
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
                 .add(newNode("node3", singletonMap("rack_id", "2")))
         ).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(5));
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.RELOCATING).size(), equalTo(5));
@@ -246,12 +234,10 @@ public void testMoveShardOnceNewNodeWithAttributeAdded3() {
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.RELOCATING).get(0).relocatingNodeId(), equalTo("node3"));
 
         logger.info("--> complete initializing");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         logger.info("--> run it again, since we still might have relocation");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(10));
 
@@ -262,13 +248,11 @@ public void testMoveShardOnceNewNodeWithAttributeAdded3() {
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
                 .add(newNode("node4", singletonMap("rack_id", "3")))
         ).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
         assertThat(clusterState.getRoutingNodes().shardsWithState(RELOCATING).size(), greaterThan(0));
 
         logger.info("--> complete relocation");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(10));
 
@@ -304,17 +288,14 @@ public void testMoveShardOnceNewNodeWithAttributeAdded4() {
                 .add(newNode("node1", singletonMap("rack_id", "1")))
                 .add(newNode("node2", singletonMap("rack_id", "1")))
         ).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
         assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(10));
 
         logger.info("--> start the shards (primaries)");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         logger.info("--> start the shards (replicas)");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(20));
 
@@ -322,8 +303,7 @@ public void testMoveShardOnceNewNodeWithAttributeAdded4() {
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
                 .add(newNode("node3", singletonMap("rack_id", "2")))
         ).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(10));
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.RELOCATING).size(), equalTo(10));
@@ -333,11 +313,9 @@ public void testMoveShardOnceNewNodeWithAttributeAdded4() {
         logger.info("--> complete initializing");
         for (int i = 0; i < 2; i++) {
             logger.info("--> complete initializing round: [{}]", i);
-            routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-            clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
-        }
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+            clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
+            }
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(20));
         assertThat(clusterState.getRoutingNodes().node("node3").size(), equalTo(10));
@@ -351,16 +329,14 @@ public void testMoveShardOnceNewNodeWithAttributeAdded4() {
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
                 .add(newNode("node4", singletonMap("rack_id", "3")))
         ).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
         assertThat(clusterState.getRoutingNodes().shardsWithState(RELOCATING).size(), greaterThan(0));
 
         logger.info("--> complete relocation");
         for (int i = 0; i < 2; i++) {
             logger.info("--> complete initializing round: [{}]", i);
-            routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-            clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
-        }
+            clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
+            }
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(20));
         assertThat(clusterState.getRoutingNodes().node("node3").size(), equalTo(5));
         assertThat(clusterState.getRoutingNodes().node("node4").size(), equalTo(5));
@@ -395,17 +371,14 @@ public void testMoveShardOnceNewNodeWithAttributeAdded5() {
                 .add(newNode("node1", singletonMap("rack_id", "1")))
                 .add(newNode("node2", singletonMap("rack_id", "1")))
         ).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
         assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(1));
 
         logger.info("--> start the shards (primaries)");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         logger.info("--> start the shards (replicas)");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(2));
 
@@ -413,16 +386,14 @@ public void testMoveShardOnceNewNodeWithAttributeAdded5() {
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
                 .add(newNode("node3", singletonMap("rack_id", "2")))
         ).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(2));
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).currentNodeId(), equalTo("node3"));
 
         logger.info("--> complete relocation");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(3));
 
@@ -433,15 +404,13 @@ public void testMoveShardOnceNewNodeWithAttributeAdded5() {
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
                 .add(newNode("node4", singletonMap("rack_id", "3")))
         ).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
         assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(2));
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.RELOCATING).size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.RELOCATING).get(0).relocatingNodeId(), equalTo("node4"));
 
         logger.info("--> complete relocation");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(3));
 
@@ -475,17 +444,14 @@ public void testMoveShardOnceNewNodeWithAttributeAdded6() {
                 .add(newNode("node3", singletonMap("rack_id", "1")))
                 .add(newNode("node4", singletonMap("rack_id", "1")))
         ).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
         assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(1));
 
         logger.info("--> start the shards (primaries)");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         logger.info("--> start the shards (replicas)");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(4));
 
@@ -493,16 +459,14 @@ public void testMoveShardOnceNewNodeWithAttributeAdded6() {
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
                 .add(newNode("node5", singletonMap("rack_id", "2")))
         ).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(3));
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.RELOCATING).size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.RELOCATING).get(0).relocatingNodeId(), equalTo("node5"));
 
         logger.info("--> complete relocation");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(4));
 
@@ -513,15 +477,13 @@ public void testMoveShardOnceNewNodeWithAttributeAdded6() {
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
                 .add(newNode("node6", singletonMap("rack_id", "3")))
         ).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
         assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(3));
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.RELOCATING).size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.RELOCATING).get(0).relocatingNodeId(), equalTo("node6"));
 
         logger.info("--> complete relocation");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(4));
 
@@ -554,13 +516,11 @@ public void testFullAwareness1() {
                 .add(newNode("node1", singletonMap("rack_id", "1")))
                 .add(newNode("node2", singletonMap("rack_id", "1")))
         ).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
         assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(1));
 
         logger.info("--> start the shards (primaries)");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         logger.info("--> replica will not start because we have only one rack value");
         assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(1));
@@ -570,16 +530,14 @@ public void testFullAwareness1() {
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
                 .add(newNode("node3", singletonMap("rack_id", "2")))
         ).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).currentNodeId(), equalTo("node3"));
 
         logger.info("--> complete relocation");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(2));
 
@@ -590,9 +548,8 @@ public void testFullAwareness1() {
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
                 .add(newNode("node4", singletonMap("rack_id", "3")))
         ).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        assertFalse(routingResult.changed());
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        ClusterState newState = strategy.reroute(clusterState, "reroute");
+        assertThat(newState, equalTo(clusterState));
         assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(2));
     }
 
@@ -622,13 +579,11 @@ public void testFullAwareness2() {
                 .add(newNode("node2", singletonMap("rack_id", "1")))
                 .add(newNode("node3", singletonMap("rack_id", "1")))
         ).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
         assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(1));
 
         logger.info("--> start the shards (primaries)");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         logger.info("--> replica will not start because we have only one rack value");
         assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(1));
@@ -638,16 +593,14 @@ public void testFullAwareness2() {
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
                 .add(newNode("node4", singletonMap("rack_id", "2")))
         ).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).currentNodeId(), equalTo("node4"));
 
         logger.info("--> complete relocation");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(2));
 
@@ -658,9 +611,8 @@ public void testFullAwareness2() {
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
                 .add(newNode("node5", singletonMap("rack_id", "3")))
         ).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        assertFalse(routingResult.changed());
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        ClusterState newState = strategy.reroute(clusterState, "reroute");
+        assertThat(newState, equalTo(clusterState));
         assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(2));
     }
 
@@ -696,13 +648,11 @@ public void testFullAwareness3() {
                 .add(newNode("node1", singletonMap("rack_id", "1")))
                 .add(newNode("node2", singletonMap("rack_id", "1")))
         ).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
         assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(10));
 
         logger.info("--> start the shards (primaries)");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(10));
 
@@ -710,20 +660,17 @@ public void testFullAwareness3() {
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
                 .add(newNode("node3", singletonMap("rack_id", "2")))
         ).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(10));
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(10));
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).currentNodeId(), equalTo("node3"));
 
         logger.info("--> complete initializing");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         logger.info("--> run it again, since we still might have relocation");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(20));
 
@@ -734,13 +681,11 @@ public void testFullAwareness3() {
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
                 .add(newNode("node4", singletonMap("rack_id", "3")))
         ).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
         assertThat(clusterState.getRoutingNodes().shardsWithState(RELOCATING).size(), greaterThan(0));
 
         logger.info("--> complete relocation");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(20));
 
@@ -775,19 +720,16 @@ public void testUnbalancedZones() {
                 .add(newNode("A-0", singletonMap("zone", "a")))
                 .add(newNode("B-0", singletonMap("zone", "b")))
         ).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
         assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(0));
         assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(5));
 
         logger.info("--> start the shards (primaries)");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
         assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(5));
         assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(5));
 
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
         logger.info("--> all replicas are allocated and started since we have on node in each zone");
         assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(10));
         assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(0));
@@ -796,15 +738,13 @@ public void testUnbalancedZones() {
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
                 .add(newNode("A-1", singletonMap("zone", "a")))
         ).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(8));
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(2));
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).currentNodeId(), equalTo("A-1"));
         logger.info("--> starting initializing shards on the new node");
 
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(10));
         assertThat(clusterState.getRoutingNodes().node("A-1").size(), equalTo(2));
@@ -840,14 +780,12 @@ public void testUnassignedShardsWithUnbalancedZones() {
                         .add(newNode("A-4", singletonMap("zone", "a")))
                         .add(newNode("B-0", singletonMap("zone", "b")))
         ).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
         assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(0));
         assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(1));
 
         logger.info("--> start the shard (primary)");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
         assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(3));
         assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).size(), equalTo(1)); // Unassigned shard is expected.
@@ -864,12 +802,57 @@ public void testUnassignedShardsWithUnbalancedZones() {
         }
         commands.add(new MoveAllocationCommand("test", 0, primaryNode, "A-4"));
 
-        routingResult = strategy.reroute(clusterState, commands, false, false);
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, commands, false, false).getClusterState();
 
         assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(0));
         assertThat(clusterState.getRoutingNodes().shardsWithState(RELOCATING).size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(4)); // +1 for relocating shard.
         assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).size(), equalTo(1)); // Still 1 unassigned.
     }
+
+    public void testMultipleAwarenessAttributes() {
+        AllocationService strategy = createAllocationService(Settings.builder()
+            .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.getKey(), "zone, rack")
+            .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING.getKey() + "zone.values", "a, b")
+            .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING.getKey() + "rack.values", "c, d")
+            .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always")
+            .build());
+
+        logger.info("Building initial routing table for 'testUnbalancedZones'");
+
+        MetaData metaData = MetaData.builder()
+            .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
+            .build();
+
+        RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metaData.index("test")).build();
+
+        ClusterState clusterState = ClusterState.builder(
+            org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
+        ).metaData(metaData).routingTable(initialRoutingTable).build();
+
+        logger.info("--> adding two nodes in different zones and do rerouting");
+        Map<String, String> nodeAAttributes = new HashMap<>();
+        nodeAAttributes.put("zone", "a");
+        nodeAAttributes.put("rack", "c");
+        Map<String, String> nodeBAttributes = new HashMap<>();
+        nodeBAttributes.put("zone", "b");
+        nodeBAttributes.put("rack", "d");
+        clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder()
+            .add(newNode("A-0", nodeAAttributes))
+            .add(newNode("B-0", nodeBAttributes))
+        ).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
+        assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(0));
+        assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(1));
+
+        logger.info("--> start the shards (primaries)");
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
+        assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(1));
+        assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(1));
+
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
+        logger.info("--> all replicas are allocated and started since we have one node in each zone and rack");
+        assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(2));
+        assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(0));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceConfigurationTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceConfigurationTests.java
index b426220614e6b..cfca80b78cac7 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceConfigurationTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceConfigurationTests.java
@@ -24,6 +24,7 @@
 import org.apache.lucene.util.ArrayUtil;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.EmptyClusterInfoService;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
@@ -40,12 +41,11 @@
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.cluster.ESAllocationTestCase;
-import org.elasticsearch.test.gateway.NoopGatewayAllocator;
+import org.elasticsearch.gateway.GatewayAllocator;
+import org.elasticsearch.test.gateway.TestGatewayAllocator;
 import org.hamcrest.Matchers;
 
-import java.util.HashMap;
-import java.util.Map;
+import java.util.List;
 
 import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED;
@@ -71,7 +71,7 @@ public void testIndexBalance() {
         settings.put(BalancedShardsAllocator.SHARD_BALANCE_FACTOR_SETTING.getKey(), replicaBalance);
         settings.put(BalancedShardsAllocator.THRESHOLD_SETTING.getKey(), balanceTreshold);
 
-        AllocationService strategy = createAllocationService(settings.build());
+        AllocationService strategy = createAllocationService(settings.build(), new NoopGatewayAllocator());
 
         ClusterState clusterState = initCluster(strategy);
         assertIndexBalance(clusterState.getRoutingTable(), clusterState.getRoutingNodes(), numberOfNodes, numberOfIndices, numberOfReplicas, numberOfShards, balanceTreshold);
@@ -95,7 +95,7 @@ public void testReplicaBalance() {
         settings.put(BalancedShardsAllocator.SHARD_BALANCE_FACTOR_SETTING.getKey(), replicaBalance);
         settings.put(BalancedShardsAllocator.THRESHOLD_SETTING.getKey(), balanceTreshold);
 
-        AllocationService strategy = createAllocationService(settings.build());
+        AllocationService strategy = createAllocationService(settings.build(), new NoopGatewayAllocator());
 
         ClusterState clusterState = initCluster(strategy);
         assertReplicaBalance(logger, clusterState.getRoutingNodes(), numberOfNodes, numberOfIndices, numberOfReplicas, numberOfShards, balanceTreshold);
@@ -132,18 +132,15 @@ private ClusterState initCluster(AllocationService strategy) {
             nodes.add(newNode("node" + i));
         }
         ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).nodes(nodes).metaData(metaData).routingTable(initialRoutingTable).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         logger.info("restart all the primary shards, replicas will start initializing");
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
 
         logger.info("start the replica shards");
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
 
         logger.info("complete rebalancing");
         return applyStartedShardsUntilNoChange(clusterState, strategy);
@@ -174,24 +171,19 @@ private ClusterState removeNodes(ClusterState clusterState, AllocationService st
 
         clusterState = ClusterState.builder(clusterState).nodes(nodes.build()).build();
         if (removed) {
-            clusterState = ClusterState.builder(clusterState).routingResult(
-                strategy.deassociateDeadNodes(clusterState, randomBoolean(), "removed nodes")
-            ).build();
+            clusterState = strategy.deassociateDeadNodes(clusterState, randomBoolean(), "removed nodes");
         }
 
         logger.info("start all the primary shards, replicas will start initializing");
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
-        RoutingAllocation.Result routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
 
         logger.info("start the replica shards");
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
 
         logger.info("rebalancing");
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         logger.info("complete rebalancing");
         return applyStartedShardsUntilNoChange(clusterState, strategy);
@@ -262,11 +254,7 @@ public void testNoRebalanceOnPrimaryOverload() {
         Settings.Builder settings = Settings.builder();
         AllocationService strategy = new AllocationService(settings.build(), randomAllocationDeciders(settings.build(),
                 new ClusterSettings(Settings.Builder.EMPTY_SETTINGS, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), random()),
-                NoopGatewayAllocator.INSTANCE, new ShardsAllocator() {
-
-                    public Map<DiscoveryNode, Float> weighShard(RoutingAllocation allocation, ShardRouting shard) {
-                        return new HashMap<DiscoveryNode, Float>();
-                    }
+                new TestGatewayAllocator(), new ShardsAllocator() {
             /*
              *  // this allocator tries to rebuild this scenario where a rebalance is
              *  // triggered solely by the primary overload on node [1] where a shard
@@ -333,6 +321,11 @@ public void allocate(RoutingAllocation allocation) {
 
                 }
             }
+
+            @Override
+            public ShardAllocationDecision decideShardAllocation(ShardRouting shard, RoutingAllocation allocation) {
+                throw new UnsupportedOperationException("explain not supported");
+            }
         }, EmptyClusterInfoService.INSTANCE);
         MetaData.Builder metaDataBuilder = MetaData.builder();
         RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
@@ -359,7 +352,7 @@ public void allocate(RoutingAllocation allocation) {
                 assertThat(shardRouting.state(), Matchers.equalTo(ShardRoutingState.INITIALIZING));
             }
         }
-        strategy = createAllocationService(settings.build());
+        strategy = createAllocationService(settings.build(), new NoopGatewayAllocator());
 
         logger.info("use the new allocator and check if it moves shards");
         routingNodes = clusterState.getRoutingNodes();
@@ -393,7 +386,26 @@ public void allocate(RoutingAllocation allocation) {
                 assertThat(shardRouting.state(), Matchers.equalTo(ShardRoutingState.STARTED));
             }
         }
-
     }
 
+    private class NoopGatewayAllocator extends GatewayAllocator {
+
+        NoopGatewayAllocator() {
+            super(Settings.EMPTY, null, null);
+        }
+
+        @Override
+        public void applyStartedShards(RoutingAllocation allocation, List<ShardRouting> startedShards) {
+            // noop
+        }
+
+        @Override
+        public void applyFailedShards(RoutingAllocation allocation, List<FailedShard> failedShards) {
+            // noop
+        }
+        @Override
+        public void allocateUnassigned(RoutingAllocation allocation) {
+            // noop
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceUnbalancedClusterTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceUnbalancedClusterTests.java
index 3890be6a4bac6..254ba81f93ce6 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceUnbalancedClusterTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceUnbalancedClusterTests.java
@@ -65,14 +65,12 @@ protected ClusterState allocateNew(ClusterState state) {
                 .build();
 
         ClusterState clusterState = ClusterState.builder(state).metaData(metaData).routingTable(initialRoutingTable).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
         while (true) {
             if (clusterState.routingTable().shardsWithState(INITIALIZING).isEmpty()) {
                 break;
             }
-            routingResult = strategy.applyStartedShards(clusterState, clusterState.routingTable().shardsWithState(INITIALIZING));
-            clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+            clusterState = strategy.applyStartedShards(clusterState, clusterState.routingTable().shardsWithState(INITIALIZING));
         }
         Map<String, Integer> counts = new HashMap<>();
         for (IndexShardRoutingTable table : clusterState.routingTable().index(index)) {
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/BalancedSingleShardTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/BalancedSingleShardTests.java
new file mode 100644
index 0000000000000..a63447e845b18
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/BalancedSingleShardTests.java
@@ -0,0 +1,384 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.routing.allocation;
+
+import org.elasticsearch.action.support.replication.ClusterStateCreationUtils;
+import org.elasticsearch.cluster.ClusterInfo;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.RoutingNode;
+import org.elasticsearch.cluster.routing.RoutingNodes;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
+import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator.Balancer;
+import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider;
+import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.decider.Decision;
+import org.elasticsearch.cluster.routing.allocation.decider.Decision.Type;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.settings.Settings;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import static java.util.Collections.emptySet;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.lessThan;
+import static org.hamcrest.Matchers.startsWith;
+
+/**
+ * Tests for balancing a single shard, see {@link Balancer#decideRebalance(ShardRouting)}.
+ */
+public class BalancedSingleShardTests extends ESAllocationTestCase {
+
+    public void testRebalanceNonStartedShardNotAllowed() {
+        BalancedShardsAllocator allocator = new BalancedShardsAllocator(Settings.EMPTY);
+        ClusterState clusterState = ClusterStateCreationUtils.state("idx", randomBoolean(),
+            randomFrom(ShardRoutingState.INITIALIZING, ShardRoutingState.UNASSIGNED, ShardRoutingState.RELOCATING));
+        ShardRouting shard = clusterState.routingTable().index("idx").shard(0).primaryShard();
+        MoveDecision rebalanceDecision = allocator.decideShardAllocation(shard, newRoutingAllocation(
+            new AllocationDeciders(Settings.EMPTY, Collections.emptyList()), clusterState)).getMoveDecision();
+        assertSame(MoveDecision.NOT_TAKEN, rebalanceDecision);
+    }
+
+    public void testRebalanceNotAllowedDuringPendingAsyncFetch() {
+        BalancedShardsAllocator allocator = new BalancedShardsAllocator(Settings.EMPTY);
+        ClusterState clusterState = ClusterStateCreationUtils.state("idx", randomBoolean(), ShardRoutingState.STARTED);
+        ShardRouting shard = clusterState.routingTable().index("idx").shard(0).primaryShard();
+        RoutingAllocation routingAllocation = newRoutingAllocation(
+            new AllocationDeciders(Settings.EMPTY, Collections.emptyList()), clusterState);
+        routingAllocation.setHasPendingAsyncFetch();
+        MoveDecision rebalanceDecision = allocator.decideShardAllocation(shard, routingAllocation).getMoveDecision();
+        assertNotNull(rebalanceDecision.getClusterRebalanceDecision());
+        assertEquals(AllocationDecision.AWAITING_INFO, rebalanceDecision.getAllocationDecision());
+        assertThat(rebalanceDecision.getExplanation(),
+            startsWith("cannot rebalance as information about existing copies of this shard in the cluster is still being gathered"));
+        assertEquals(clusterState.nodes().getSize() - 1, rebalanceDecision.getNodeDecisions().size());
+        assertNull(rebalanceDecision.getTargetNode());
+
+        assertAssignedNodeRemainsSame(allocator, routingAllocation, shard);
+    }
+
+    public void testRebalancingNotAllowedDueToCanRebalance() {
+        final Decision canRebalanceDecision = randomFrom(Decision.NO, Decision.THROTTLE);
+        AllocationDecider noRebalanceDecider = new AllocationDecider(Settings.EMPTY) {
+            @Override
+            public Decision canRebalance(ShardRouting shardRouting, RoutingAllocation allocation) {
+                return allocation.decision(canRebalanceDecision, "TEST", "foobar");
+            }
+        };
+        BalancedShardsAllocator allocator = new BalancedShardsAllocator(Settings.EMPTY);
+        ClusterState clusterState = ClusterStateCreationUtils.state("idx", randomBoolean(), ShardRoutingState.STARTED);
+        ShardRouting shard = clusterState.routingTable().index("idx").shard(0).primaryShard();
+        RoutingAllocation routingAllocation =  newRoutingAllocation(
+            new AllocationDeciders(Settings.EMPTY, Collections.singleton(noRebalanceDecider)), clusterState);
+        MoveDecision rebalanceDecision = allocator.decideShardAllocation(shard, routingAllocation).getMoveDecision();
+        assertEquals(canRebalanceDecision.type(), rebalanceDecision.getClusterRebalanceDecision().type());
+        assertEquals(AllocationDecision.fromDecisionType(canRebalanceDecision.type()), rebalanceDecision.getAllocationDecision());
+        assertThat(rebalanceDecision.getExplanation(), containsString(canRebalanceDecision.type() == Type.THROTTLE ?
+            "rebalancing is throttled" : "rebalancing is not allowed"));
+        assertNotNull(rebalanceDecision.getNodeDecisions());
+        assertNull(rebalanceDecision.getTargetNode());
+        assertEquals(1, rebalanceDecision.getClusterRebalanceDecision().getDecisions().size());
+        for (Decision subDecision : rebalanceDecision.getClusterRebalanceDecision().getDecisions()) {
+            assertEquals("foobar", ((Decision.Single) subDecision).getExplanation());
+        }
+
+        assertAssignedNodeRemainsSame(allocator, routingAllocation, shard);
+    }
+
+    public void testRebalancePossible() {
+        AllocationDecider canAllocateDecider = new AllocationDecider(Settings.EMPTY) {
+            @Override
+            public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
+                return Decision.YES;
+            }
+        };
+        Tuple<ClusterState, MoveDecision> rebalance = setupStateAndRebalance(canAllocateDecider, Settings.EMPTY, true);
+        ClusterState clusterState = rebalance.v1();
+        MoveDecision rebalanceDecision = rebalance.v2();
+        assertEquals(Type.YES, rebalanceDecision.getClusterRebalanceDecision().type());
+        assertNotNull(rebalanceDecision.getExplanation());
+        assertEquals(clusterState.nodes().getSize() - 1, rebalanceDecision.getNodeDecisions().size());
+    }
+
+    public void testRebalancingNotAllowedDueToCanAllocate() {
+        AllocationDecider canAllocateDecider = new AllocationDecider(Settings.EMPTY) {
+            @Override
+            public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
+                return Decision.NO;
+            }
+        };
+        Tuple<ClusterState, MoveDecision> rebalance = setupStateAndRebalance(canAllocateDecider, Settings.EMPTY, false);
+        ClusterState clusterState = rebalance.v1();
+        MoveDecision rebalanceDecision = rebalance.v2();
+        assertEquals(Type.YES, rebalanceDecision.getClusterRebalanceDecision().type());
+        assertEquals(AllocationDecision.NO, rebalanceDecision.getAllocationDecision());
+        assertThat(rebalanceDecision.getExplanation(), startsWith(
+            "cannot rebalance as no target node exists that can both allocate this shard and improve the cluster balance"));
+        assertEquals(clusterState.nodes().getSize() - 1, rebalanceDecision.getNodeDecisions().size());
+        assertNull(rebalanceDecision.getTargetNode());
+        int prevRanking = 0;
+        for (NodeAllocationResult result : rebalanceDecision.getNodeDecisions()) {
+            assertThat(result.getWeightRanking(), greaterThanOrEqualTo(prevRanking));
+            prevRanking = result.getWeightRanking();
+        }
+    }
+
+    public void testDontBalanceShardWhenThresholdNotMet() {
+        AllocationDecider canAllocateDecider = new AllocationDecider(Settings.EMPTY) {
+            @Override
+            public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
+                return Decision.YES;
+            }
+        };
+        // ridiculously high threshold setting so we won't rebalance
+        Settings balancerSettings = Settings.builder().put(BalancedShardsAllocator.THRESHOLD_SETTING.getKey(), 1000f).build();
+        Tuple<ClusterState, MoveDecision> rebalance = setupStateAndRebalance(canAllocateDecider, balancerSettings, false);
+        ClusterState clusterState = rebalance.v1();
+        MoveDecision rebalanceDecision = rebalance.v2();
+        assertEquals(Type.YES, rebalanceDecision.getClusterRebalanceDecision().type());
+        assertEquals(AllocationDecision.NO, rebalanceDecision.getAllocationDecision());
+        assertNotNull(rebalanceDecision.getExplanation());
+        assertEquals(clusterState.nodes().getSize() - 1, rebalanceDecision.getNodeDecisions().size());
+        assertNull(rebalanceDecision.getTargetNode());
+        int prevRanking = 0;
+        for (NodeAllocationResult result : rebalanceDecision.getNodeDecisions()) {
+            assertThat(result.getWeightRanking(), greaterThanOrEqualTo(prevRanking));
+            prevRanking = result.getWeightRanking();
+        }
+    }
+
+    public void testSingleShardBalanceProducesSameResultsAsBalanceStep() {
+        final String[] indices = { "idx1", "idx2" };
+        // Create a cluster state with 2 indices, each with 1 started primary shard, and only
+        // one node initially so that all primary shards get allocated to the same node.  We are only
+        // using 2 indices (i.e. 2 total primary shards) because if we have any more than 2 started shards
+        // in the routing table, then we have no guarantees about the order in which the 3 or more shards
+        // are selected to be rebalanced to the new node, and hence the node to which they are rebalanced
+        // is not deterministic.  Using only two shards guarantees that only one of those two shards will
+        // be rebalanced, and so we pick the one that was chosen to be rebalanced and execute the single-shard
+        // rebalance step on it to make sure it gets assigned to the same node.
+        ClusterState clusterState = ClusterStateCreationUtils.state(1, indices, 1);
+        // add new nodes so one of the primaries can be rebalanced
+        DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(clusterState.nodes());
+        int numAddedNodes = randomIntBetween(1, 5);
+        // randomly select a subset of the newly added nodes to set filter allocation on (but not all)
+        int excludeNodesSize = randomIntBetween(0, numAddedNodes - 1);
+        final Set<String> excludeNodes = new HashSet<>();
+        for (int i = 0; i < numAddedNodes; i++) {
+            DiscoveryNode discoveryNode = newNode(randomAlphaOfLength(7));
+            nodesBuilder.add(discoveryNode);
+            if (i < excludeNodesSize) {
+                excludeNodes.add(discoveryNode.getId());
+            }
+        }
+        clusterState = ClusterState.builder(clusterState).nodes(nodesBuilder).build();
+
+        AllocationDecider allocationDecider = new AllocationDecider(Settings.EMPTY) {
+            @Override
+            public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
+                if (excludeNodes.contains(node.nodeId())) {
+                    return Decision.NO;
+                }
+                return Decision.YES;
+            }
+        };
+        AllocationDecider rebalanceDecider = new AllocationDecider(Settings.EMPTY) {
+            @Override
+            public Decision canRebalance(ShardRouting shardRouting, RoutingAllocation allocation) {
+                return Decision.YES;
+            }
+        };
+        List<AllocationDecider> allocationDeciders = Arrays.asList(rebalanceDecider, allocationDecider);
+        RoutingAllocation routingAllocation = newRoutingAllocation(
+            new AllocationDeciders(Settings.EMPTY, allocationDeciders), clusterState);
+        // allocate and get the node that is now relocating
+        BalancedShardsAllocator allocator = new BalancedShardsAllocator(Settings.EMPTY);
+        allocator.allocate(routingAllocation);
+        ShardRouting shardToRebalance = null;
+        for (RoutingNode routingNode : routingAllocation.routingNodes()) {
+            List<ShardRouting> relocatingShards = routingNode.shardsWithState(ShardRoutingState.RELOCATING);
+            if (relocatingShards.size() > 0) {
+                shardToRebalance = randomFrom(relocatingShards);
+                break;
+            }
+        }
+
+        routingAllocation = newRoutingAllocation(new AllocationDeciders(Settings.EMPTY, allocationDeciders), clusterState);
+        routingAllocation.debugDecision(true);
+        ShardRouting shard = clusterState.getRoutingNodes().activePrimary(shardToRebalance.shardId());
+        MoveDecision rebalanceDecision = allocator.decideShardAllocation(shard, routingAllocation).getMoveDecision();
+        assertEquals(shardToRebalance.relocatingNodeId(), rebalanceDecision.getTargetNode().getId());
+        // make sure all excluded nodes returned a NO decision
+        for (NodeAllocationResult nodeResult : rebalanceDecision.getNodeDecisions()) {
+            if (excludeNodes.contains(nodeResult.getNode().getId())) {
+                assertEquals(Type.NO, nodeResult.getCanAllocateDecision().type());
+            }
+        }
+    }
+
+    public void testNodeDecisionsRanking() {
+        // only one shard, so moving it will not create a better balance anywhere, so all node decisions should
+        // return the same ranking as the current node
+        ClusterState clusterState = ClusterStateCreationUtils.state(randomIntBetween(1, 10), new String[] { "idx" }, 1);
+        ShardRouting shardToRebalance = clusterState.routingTable().index("idx").shardsWithState(ShardRoutingState.STARTED).get(0);
+        MoveDecision decision = executeRebalanceFor(shardToRebalance, clusterState, emptySet(), -1);
+        int currentRanking = decision.getCurrentNodeRanking();
+        assertEquals(1, currentRanking);
+        for (NodeAllocationResult result : decision.getNodeDecisions()) {
+            assertEquals(1, result.getWeightRanking());
+        }
+
+        // start off with one node and several shards assigned to that node, then add a few nodes to the cluster,
+        // each of these new nodes should have a better ranking than the current, given a low enough threshold
+        clusterState = ClusterStateCreationUtils.state(1, new String[] { "idx" }, randomIntBetween(2, 10));
+        shardToRebalance = clusterState.routingTable().index("idx").shardsWithState(ShardRoutingState.STARTED).get(0);
+        clusterState = addNodesToClusterState(clusterState, randomIntBetween(1, 10));
+        decision = executeRebalanceFor(shardToRebalance, clusterState, emptySet(), 0.01f);
+        for (NodeAllocationResult result : decision.getNodeDecisions()) {
+            assertThat(result.getWeightRanking(), lessThan(decision.getCurrentNodeRanking()));
+        }
+
+        // start off with 3 nodes and 7 shards, so that one of the 3 nodes will have 3 shards assigned, the remaining 2
+        // nodes will have 2 shard each.  then, add another node.  pick a shard on one of the nodes that has only 2 shard
+        // to rebalance.  the new node should have the best ranking (because it has no shards), followed by the node currently
+        // holding the shard as well as the other node with only 2 shards (they should have the same ranking), followed by the
+        // node with 3 shards which will have the lowest ranking.
+        clusterState = ClusterStateCreationUtils.state(3, new String[] { "idx" }, 7);
+        shardToRebalance = null;
+        Set<String> nodesWithTwoShards = new HashSet<>();
+        String nodeWithThreeShards = null;
+        for (RoutingNode node : clusterState.getRoutingNodes()) {
+            if (node.numberOfShardsWithState(ShardRoutingState.STARTED) == 2) {
+                nodesWithTwoShards.add(node.nodeId());
+                if (shardToRebalance == null) {
+                    shardToRebalance = node.shardsWithState(ShardRoutingState.STARTED).get(0);
+                }
+            } else {
+                assertEquals(3, node.numberOfShardsWithState(ShardRoutingState.STARTED));
+                assertNull(nodeWithThreeShards); // should only have one of these
+                nodeWithThreeShards = node.nodeId();
+            }
+        }
+        clusterState = addNodesToClusterState(clusterState, 1);
+        decision = executeRebalanceFor(shardToRebalance, clusterState, emptySet(), 0.01f);
+        for (NodeAllocationResult result : decision.getNodeDecisions()) {
+            if (result.getWeightRanking() < decision.getCurrentNodeRanking()) {
+                // highest ranked node should not be any of the initial nodes
+                assertFalse(nodesWithTwoShards.contains(result.getNode().getId()));
+                assertNotEquals(nodeWithThreeShards, result.getNode().getId());
+            } else if (result.getWeightRanking() > decision.getCurrentNodeRanking()) {
+                // worst ranked should be the node with two shards
+                assertEquals(nodeWithThreeShards, result.getNode().getId());
+            } else {
+                assertTrue(nodesWithTwoShards.contains(result.getNode().getId()));
+            }
+        }
+    }
+
+    private MoveDecision executeRebalanceFor(final ShardRouting shardRouting, final ClusterState clusterState,
+                                             final Set<String> noDecisionNodes, final float threshold) {
+        Settings settings = Settings.EMPTY;
+        if (Float.compare(-1.0f, threshold) != 0) {
+            settings = Settings.builder().put(BalancedShardsAllocator.THRESHOLD_SETTING.getKey(), threshold).build();
+        }
+        AllocationDecider allocationDecider = new AllocationDecider(Settings.EMPTY) {
+            @Override
+            public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
+                if (noDecisionNodes.contains(node.nodeId())) {
+                    return Decision.NO;
+                }
+                return Decision.YES;
+            }
+        };
+        AllocationDecider rebalanceDecider = new AllocationDecider(Settings.EMPTY) {
+            @Override
+            public Decision canRebalance(ShardRouting shardRouting, RoutingAllocation allocation) {
+                return Decision.YES;
+            }
+        };
+        BalancedShardsAllocator allocator = new BalancedShardsAllocator(settings);
+        RoutingAllocation routingAllocation = newRoutingAllocation(
+            new AllocationDeciders(Settings.EMPTY, Arrays.asList(allocationDecider, rebalanceDecider)), clusterState);
+        return allocator.decideShardAllocation(shardRouting, routingAllocation).getMoveDecision();
+    }
+
+    private ClusterState addNodesToClusterState(ClusterState clusterState, int numNodesToAdd) {
+        DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(clusterState.nodes());
+        for (int i = 0; i < numNodesToAdd; i++) {
+            DiscoveryNode discoveryNode = newNode(randomAlphaOfLength(7));
+            nodesBuilder.add(discoveryNode);
+        }
+        return ClusterState.builder(clusterState).nodes(nodesBuilder).build();
+    }
+
+    private Tuple<ClusterState, MoveDecision> setupStateAndRebalance(AllocationDecider allocationDecider,
+                                                                     Settings balancerSettings,
+                                                                     boolean rebalanceExpected) {
+        AllocationDecider rebalanceDecider = new AllocationDecider(Settings.EMPTY) {
+            @Override
+            public Decision canRebalance(ShardRouting shardRouting, RoutingAllocation allocation) {
+                return Decision.YES;
+            }
+        };
+        List<AllocationDecider> allocationDeciders = Arrays.asList(rebalanceDecider, allocationDecider);
+        final int numShards = randomIntBetween(8, 13);
+        BalancedShardsAllocator allocator = new BalancedShardsAllocator(balancerSettings);
+        ClusterState clusterState = ClusterStateCreationUtils.state("idx", 2, numShards);
+        // add a new node so shards can be rebalanced there
+        DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(clusterState.nodes());
+        nodesBuilder.add(newNode(randomAlphaOfLength(7)));
+        clusterState = ClusterState.builder(clusterState).nodes(nodesBuilder).build();
+        ShardRouting shard = clusterState.routingTable().index("idx").shard(0).primaryShard();
+        RoutingAllocation routingAllocation = newRoutingAllocation(
+            new AllocationDeciders(Settings.EMPTY, allocationDeciders), clusterState);
+        MoveDecision rebalanceDecision = allocator.decideShardAllocation(shard, routingAllocation).getMoveDecision();
+
+        if (rebalanceExpected == false) {
+            assertAssignedNodeRemainsSame(allocator, routingAllocation, shard);
+        }
+
+        return Tuple.tuple(clusterState, rebalanceDecision);
+    }
+
+    private RoutingAllocation newRoutingAllocation(AllocationDeciders deciders, ClusterState state) {
+        RoutingAllocation allocation = new RoutingAllocation(
+            deciders, new RoutingNodes(state, false), state, ClusterInfo.EMPTY, System.nanoTime(), false
+        );
+        allocation.debugDecision(true);
+        return allocation;
+    }
+
+    private void assertAssignedNodeRemainsSame(BalancedShardsAllocator allocator, RoutingAllocation routingAllocation,
+                                               ShardRouting originalRouting) {
+        allocator.allocate(routingAllocation);
+        RoutingNodes routingNodes = routingAllocation.routingNodes();
+        // make sure the previous node id is the same as the current one after rerouting
+        assertEquals(originalRouting.currentNodeId(), routingNodes.activePrimary(originalRouting.shardId()).currentNodeId());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/CatAllocationTestCase.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/CatAllocationTestCase.java
index 7adc7a4917316..db707c5478eff 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/CatAllocationTestCase.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/CatAllocationTestCase.java
@@ -21,6 +21,7 @@
 
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -31,7 +32,6 @@
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.cluster.ESAllocationTestCase;
 
 import java.io.BufferedReader;
 import java.io.IOException;
@@ -149,10 +149,9 @@ protected boolean balanceFirst() {
     }
 
     private ClusterState rebalance(ClusterState clusterState) {
-        RoutingTable routingTable;AllocationService strategy = createAllocationService(Settings.builder()
+        AllocationService strategy = createAllocationService(Settings.builder()
                 .build());
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
         int numRelocations = 0;
         while (true) {
             List<ShardRouting> initializing = clusterState.routingTable().shardsWithState(INITIALIZING);
@@ -161,8 +160,7 @@ private ClusterState rebalance(ClusterState clusterState) {
             }
             logger.debug("Initializing shards: {}", initializing);
             numRelocations += initializing.size();
-            routingResult = strategy.applyStartedShards(clusterState, initializing);
-            clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+            clusterState = strategy.applyStartedShards(clusterState, initializing);
         }
         logger.debug("--> num relocations to get balance: {}", numRelocations);
         return clusterState;
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ClusterRebalanceRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ClusterRebalanceRoutingTests.java
index 7d7db4ef2169d..8cccdb08fb5e9 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ClusterRebalanceRoutingTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ClusterRebalanceRoutingTests.java
@@ -22,6 +22,7 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -32,8 +33,7 @@
 import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.cluster.ESAllocationTestCase;
-import org.elasticsearch.test.gateway.NoopGatewayAllocator;
+import org.elasticsearch.test.gateway.TestGatewayAllocator;
 
 import java.util.concurrent.atomic.AtomicBoolean;
 
@@ -43,6 +43,7 @@
 import static org.elasticsearch.cluster.routing.ShardRoutingState.UNASSIGNED;
 import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
 
 public class ClusterRebalanceRoutingTests extends ESAllocationTestCase {
     private final Logger logger = Loggers.getLogger(ClusterRebalanceRoutingTests.class);
@@ -65,8 +66,7 @@ public void testAlways() {
 
         logger.info("start two nodes");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         for (int i = 0; i < clusterState.routingTable().index("test1").shards().size(); i++) {
             assertThat(clusterState.routingTable().index("test1").shard(i).shards().size(), equalTo(2));
@@ -82,9 +82,7 @@ public void testAlways() {
 
         logger.info("start all the primary shards for test1, replicas will start initializing");
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState("test1", INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
-        routingNodes = clusterState.getRoutingNodes();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState("test1", INITIALIZING));
 
         for (int i = 0; i < clusterState.routingTable().index("test1").shards().size(); i++) {
             assertThat(clusterState.routingTable().index("test1").shard(i).shards().size(), equalTo(2));
@@ -100,8 +98,7 @@ public void testAlways() {
 
         logger.info("start the test1 replica shards");
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState("test1", INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState("test1", INITIALIZING));
         routingNodes = clusterState.getRoutingNodes();
 
         for (int i = 0; i < clusterState.routingTable().index("test1").shards().size(); i++) {
@@ -120,8 +117,7 @@ public void testAlways() {
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
                 .add(newNode("node3")))
                 .build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
         routingNodes = clusterState.getRoutingNodes();
 
         assertThat(routingNodes.node("node3").size(), equalTo(1));
@@ -147,9 +143,7 @@ public void testClusterPrimariesActive1() {
 
         logger.info("start two nodes");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
-
+        clusterState = strategy.reroute(clusterState, "reroute");
         for (int i = 0; i < clusterState.routingTable().index("test1").shards().size(); i++) {
             assertThat(clusterState.routingTable().index("test1").shard(i).size(), equalTo(2));
             assertThat(clusterState.routingTable().index("test1").shard(i).primaryShard().state(), equalTo(INITIALIZING));
@@ -164,8 +158,7 @@ public void testClusterPrimariesActive1() {
 
         logger.info("start all the primary shards for test1, replicas will start initializing");
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState("test1", INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState("test1", INITIALIZING));
         routingNodes = clusterState.getRoutingNodes();
 
         for (int i = 0; i < clusterState.routingTable().index("test1").shards().size(); i++) {
@@ -182,8 +175,7 @@ public void testClusterPrimariesActive1() {
 
         logger.info("start the test1 replica shards");
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState("test1", INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState("test1", INITIALIZING));
         routingNodes = clusterState.getRoutingNodes();
 
         for (int i = 0; i < clusterState.routingTable().index("test1").shards().size(); i++) {
@@ -200,8 +192,7 @@ public void testClusterPrimariesActive1() {
 
         logger.info("start all the primary shards for test2, replicas will start initializing");
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState("test2", INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState("test2", INITIALIZING));
         routingNodes = clusterState.getRoutingNodes();
 
         for (int i = 0; i < clusterState.routingTable().index("test1").shards().size(); i++) {
@@ -220,8 +211,7 @@ public void testClusterPrimariesActive1() {
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
                 .add(newNode("node3")))
                 .build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
         routingNodes = clusterState.getRoutingNodes();
 
         assertThat(routingNodes.node("node3").size(), equalTo(1));
@@ -246,8 +236,7 @@ public void testClusterPrimariesActive2() {
 
         logger.info("start two nodes");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         for (int i = 0; i < clusterState.routingTable().index("test1").shards().size(); i++) {
             assertThat(clusterState.routingTable().index("test1").shard(i).shards().size(), equalTo(2));
@@ -263,8 +252,7 @@ public void testClusterPrimariesActive2() {
 
         logger.info("start all the primary shards for test1, replicas will start initializing");
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState("test1", INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState("test1", INITIALIZING));
         routingNodes = clusterState.getRoutingNodes();
 
         for (int i = 0; i < clusterState.routingTable().index("test1").shards().size(); i++) {
@@ -281,8 +269,7 @@ public void testClusterPrimariesActive2() {
 
         logger.info("start the test1 replica shards");
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState("test1", INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState("test1", INITIALIZING));
         routingNodes = clusterState.getRoutingNodes();
 
         for (int i = 0; i < clusterState.routingTable().index("test1").shards().size(); i++) {
@@ -301,8 +288,7 @@ public void testClusterPrimariesActive2() {
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
                 .add(newNode("node3")))
                 .build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
         routingNodes = clusterState.getRoutingNodes();
 
         assertThat(routingNodes.node("node3").isEmpty(), equalTo(true));
@@ -326,8 +312,7 @@ public void testClusterAllActive1() {
 
         logger.info("start two nodes");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         for (int i = 0; i < clusterState.routingTable().index("test1").shards().size(); i++) {
             assertThat(clusterState.routingTable().index("test1").shard(i).shards().size(), equalTo(2));
@@ -343,8 +328,7 @@ public void testClusterAllActive1() {
 
         logger.info("start all the primary shards for test1, replicas will start initializing");
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState("test1", INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState("test1", INITIALIZING));
         routingNodes = clusterState.getRoutingNodes();
 
         for (int i = 0; i < clusterState.routingTable().index("test1").shards().size(); i++) {
@@ -361,8 +345,7 @@ public void testClusterAllActive1() {
 
         logger.info("start the test1 replica shards");
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState("test1", INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState("test1", INITIALIZING));
         routingNodes = clusterState.getRoutingNodes();
 
         for (int i = 0; i < clusterState.routingTable().index("test1").shards().size(); i++) {
@@ -379,8 +362,7 @@ public void testClusterAllActive1() {
 
         logger.info("start all the primary shards for test2, replicas will start initializing");
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState("test2", INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState("test2", INITIALIZING));
         routingNodes = clusterState.getRoutingNodes();
 
         for (int i = 0; i < clusterState.routingTable().index("test1").shards().size(); i++) {
@@ -397,8 +379,7 @@ public void testClusterAllActive1() {
 
         logger.info("start the test2 replica shards");
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState("test2", INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState("test2", INITIALIZING));
         routingNodes = clusterState.getRoutingNodes();
 
         for (int i = 0; i < clusterState.routingTable().index("test1").shards().size(); i++) {
@@ -417,8 +398,7 @@ public void testClusterAllActive1() {
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
                 .add(newNode("node3")))
                 .build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
         routingNodes = clusterState.getRoutingNodes();
 
         assertThat(routingNodes.node("node3").size(), equalTo(1));
@@ -443,8 +423,7 @@ public void testClusterAllActive2() {
 
         logger.info("start two nodes");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         for (int i = 0; i < clusterState.routingTable().index("test1").shards().size(); i++) {
             assertThat(clusterState.routingTable().index("test1").shard(i).shards().size(), equalTo(2));
@@ -460,8 +439,7 @@ public void testClusterAllActive2() {
 
         logger.info("start all the primary shards for test1, replicas will start initializing");
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState("test1", INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState("test1", INITIALIZING));
         routingNodes = clusterState.getRoutingNodes();
 
         for (int i = 0; i < clusterState.routingTable().index("test1").shards().size(); i++) {
@@ -478,8 +456,7 @@ public void testClusterAllActive2() {
 
         logger.info("start the test1 replica shards");
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState("test1", INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState("test1", INITIALIZING));
         routingNodes = clusterState.getRoutingNodes();
 
         for (int i = 0; i < clusterState.routingTable().index("test1").shards().size(); i++) {
@@ -498,8 +475,7 @@ public void testClusterAllActive2() {
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
                 .add(newNode("node3")))
                 .build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
         routingNodes = clusterState.getRoutingNodes();
 
         assertThat(routingNodes.node("node3").isEmpty(), equalTo(true));
@@ -523,8 +499,7 @@ public void testClusterAllActive3() {
 
         logger.info("start two nodes");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         for (int i = 0; i < clusterState.routingTable().index("test1").shards().size(); i++) {
             assertThat(clusterState.routingTable().index("test1").shard(i).shards().size(), equalTo(2));
@@ -540,8 +515,7 @@ public void testClusterAllActive3() {
 
         logger.info("start all the primary shards for test1, replicas will start initializing");
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState("test1", INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState("test1", INITIALIZING));
         routingNodes = clusterState.getRoutingNodes();
 
         for (int i = 0; i < clusterState.routingTable().index("test1").shards().size(); i++) {
@@ -558,8 +532,7 @@ public void testClusterAllActive3() {
 
         logger.info("start the test1 replica shards");
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState("test1", INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState("test1", INITIALIZING));
         routingNodes = clusterState.getRoutingNodes();
 
         for (int i = 0; i < clusterState.routingTable().index("test1").shards().size(); i++) {
@@ -576,8 +549,7 @@ public void testClusterAllActive3() {
 
         logger.info("start all the primary shards for test2, replicas will start initializing");
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState("test2", INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState("test2", INITIALIZING));
         routingNodes = clusterState.getRoutingNodes();
 
         for (int i = 0; i < clusterState.routingTable().index("test1").shards().size(); i++) {
@@ -596,8 +568,7 @@ public void testClusterAllActive3() {
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
                 .add(newNode("node3")))
                 .build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
         routingNodes = clusterState.getRoutingNodes();
 
         assertThat(routingNodes.node("node3").isEmpty(), equalTo(true));
@@ -606,7 +577,7 @@ public void testClusterAllActive3() {
     public void testRebalanceWithIgnoredUnassignedShards() {
         final AtomicBoolean allocateTest1 = new AtomicBoolean(false);
 
-        AllocationService strategy = createAllocationService(Settings.EMPTY, new NoopGatewayAllocator() {
+        AllocationService strategy = createAllocationService(Settings.EMPTY, new TestGatewayAllocator() {
             @Override
             public void allocateUnassigned(RoutingAllocation allocation) {
                 if (allocateTest1.get() == false) {
@@ -638,8 +609,7 @@ public void allocateUnassigned(RoutingAllocation allocation) {
 
         logger.info("start two nodes");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1"))).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
             assertThat(clusterState.routingTable().index("test").shard(i).shards().size(), equalTo(1));
@@ -648,9 +618,7 @@ public void allocateUnassigned(RoutingAllocation allocation) {
 
         logger.debug("start all the primary shards for test");
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState("test", INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
-
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState("test", INITIALIZING));
         for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
             assertThat(clusterState.routingTable().index("test").shard(i).shards().size(), equalTo(1));
             assertThat(clusterState.routingTable().index("test").shard(i).primaryShard().state(), equalTo(STARTED));
@@ -661,10 +629,8 @@ public void allocateUnassigned(RoutingAllocation allocation) {
                 .add(newNode("node2")))
                 .build();
         logger.debug("reroute and check that nothing has changed");
-        RoutingAllocation.Result reroute = strategy.reroute(clusterState, "reroute");
-        assertFalse(reroute.changed());
-        routingResult = reroute;
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        ClusterState resultingState = strategy.reroute(clusterState, "reroute");
+        assertThat(resultingState, equalTo(clusterState));
 
         for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
             assertThat(clusterState.routingTable().index("test").shard(i).shards().size(), equalTo(1));
@@ -676,10 +642,9 @@ public void allocateUnassigned(RoutingAllocation allocation) {
         }
         logger.debug("now set allocateTest1 to true and reroute we should see the [test1] index initializing");
         allocateTest1.set(true);
-        reroute = strategy.reroute(clusterState, "reroute");
-        assertTrue(reroute.changed());
-        routingResult = reroute;
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        resultingState = strategy.reroute(clusterState, "reroute");
+        assertThat(resultingState, not(equalTo(clusterState)));
+        clusterState = resultingState;
         for (int i = 0; i < clusterState.routingTable().index("test1").shards().size(); i++) {
             assertThat(clusterState.routingTable().index("test1").shard(i).shards().size(), equalTo(1));
             assertThat(clusterState.routingTable().index("test1").shard(i).primaryShard().state(), equalTo(INITIALIZING));
@@ -688,8 +653,7 @@ public void allocateUnassigned(RoutingAllocation allocation) {
         logger.debug("now start initializing shards and expect exactly one rebalance from node1 to node 2 since index [test] is all on node1");
 
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState("test1", INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState("test1", INITIALIZING));
 
         for (int i = 0; i < clusterState.routingTable().index("test1").shards().size(); i++) {
             assertThat(clusterState.routingTable().index("test1").shard(i).shards().size(), equalTo(1));
@@ -713,7 +677,7 @@ public void allocateUnassigned(RoutingAllocation allocation) {
     public void testRebalanceWhileShardFetching() {
         final AtomicBoolean hasFetches = new AtomicBoolean(true);
         AllocationService strategy = createAllocationService(Settings.builder().put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(),
-                ClusterRebalanceAllocationDecider.ClusterRebalanceType.ALWAYS.toString()).build(), new NoopGatewayAllocator() {
+                ClusterRebalanceAllocationDecider.ClusterRebalanceType.ALWAYS.toString()).build(), new TestGatewayAllocator() {
             @Override
             public void allocateUnassigned(RoutingAllocation allocation) {
                 if (hasFetches.get()) {
@@ -738,8 +702,7 @@ public void allocateUnassigned(RoutingAllocation allocation) {
 
         logger.info("start two nodes");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1"))).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
             assertThat(clusterState.routingTable().index("test").shard(i).shards().size(), equalTo(1));
@@ -748,9 +711,7 @@ public void allocateUnassigned(RoutingAllocation allocation) {
 
         logger.debug("start all the primary shards for test");
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState("test", INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
-
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState("test", INITIALIZING));
         for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
             assertThat(clusterState.routingTable().index("test").shard(i).shards().size(), equalTo(1));
             assertThat(clusterState.routingTable().index("test").shard(i).primaryShard().state(), equalTo(STARTED));
@@ -761,8 +722,8 @@ public void allocateUnassigned(RoutingAllocation allocation) {
                 .add(newNode("node2")))
                 .build();
         logger.debug("reroute and check that nothing has changed");
-        routingResult = strategy.reroute(clusterState, "reroute");
-        assertFalse(routingResult.changed());
+        ClusterState resultState = strategy.reroute(clusterState, "reroute");
+        assertThat(resultState, equalTo(clusterState));
 
         for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
             assertThat(clusterState.routingTable().index("test").shard(i).shards().size(), equalTo(1));
@@ -774,9 +735,9 @@ public void allocateUnassigned(RoutingAllocation allocation) {
         }
         logger.debug("now set hasFetches to true and reroute we should now see exactly one relocating shard");
         hasFetches.set(false);
-        routingResult = strategy.reroute(clusterState, "reroute");
-        assertTrue(routingResult.changed());
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        resultState = strategy.reroute(clusterState, "reroute");
+        assertThat(resultState, not(equalTo(clusterState)));
+        clusterState = resultState;
         int numStarted = 0;
         int numRelocating = 0;
         for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ConcurrentRebalanceRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ConcurrentRebalanceRoutingTests.java
index d7b56c6508e55..aa7be906a63dc 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ConcurrentRebalanceRoutingTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ConcurrentRebalanceRoutingTests.java
@@ -22,6 +22,7 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -29,7 +30,6 @@
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.cluster.ESAllocationTestCase;
 
 import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.RELOCATING;
@@ -70,8 +70,7 @@ public void testClusterConcurrentRebalance() {
 
         logger.info("start two nodes and fully start the shards");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
             assertThat(clusterState.routingTable().index("test").shard(i).shards().size(), equalTo(2));
@@ -81,8 +80,7 @@ public void testClusterConcurrentRebalance() {
 
         logger.info("start all the primary shards, replicas will start initializing");
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
 
         for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
             assertThat(clusterState.routingTable().index("test").shard(i).shards().size(), equalTo(2));
@@ -94,8 +92,7 @@ public void testClusterConcurrentRebalance() {
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
                 .add(newNode("node3")).add(newNode("node4")).add(newNode("node5")).add(newNode("node6")).add(newNode("node7")).add(newNode("node8")).add(newNode("node9")).add(newNode("node10")))
                 .build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
             assertThat(clusterState.routingTable().index("test").shard(i).shards().size(), equalTo(2));
@@ -105,8 +102,7 @@ public void testClusterConcurrentRebalance() {
 
         logger.info("start the replica shards, rebalancing should start, but, only 3 should be rebalancing");
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
 
         // we only allow one relocation at a time
         assertThat(clusterState.routingTable().shardsWithState(STARTED).size(), equalTo(7));
@@ -114,8 +110,7 @@ public void testClusterConcurrentRebalance() {
 
         logger.info("finalize this session relocation, 3 more should relocate now");
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
 
         // we only allow one relocation at a time
         assertThat(clusterState.routingTable().shardsWithState(STARTED).size(), equalTo(7));
@@ -123,8 +118,7 @@ public void testClusterConcurrentRebalance() {
 
         logger.info("finalize this session relocation, 2 more should relocate now");
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
 
         // we only allow one relocation at a time
         assertThat(clusterState.routingTable().shardsWithState(STARTED).size(), equalTo(8));
@@ -132,8 +126,7 @@ public void testClusterConcurrentRebalance() {
 
         logger.info("finalize this session relocation, no more relocation");
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
 
         // we only allow one relocation at a time
         assertThat(clusterState.routingTable().shardsWithState(STARTED).size(), equalTo(10));
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/DeadNodesAllocationTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/DeadNodesAllocationTests.java
index c4d8545e97ccb..9076dde19fc23 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/DeadNodesAllocationTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/DeadNodesAllocationTests.java
@@ -22,6 +22,7 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -31,12 +32,12 @@
 import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.cluster.ESAllocationTestCase;
 
 import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.RELOCATING;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
 
 /**
  */
@@ -64,15 +65,12 @@ public void testSimpleDeadNodeOnStartedPrimaryShard() {
                 .add(newNode("node2"))
         ).build();
 
-        RoutingAllocation.Result rerouteResult = allocation.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.reroute(clusterState, "reroute");
 
         // starting primaries
-        rerouteResult = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
         // starting replicas
-        rerouteResult = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         logger.info("--> verifying all is allocated");
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
@@ -87,8 +85,7 @@ public void testSimpleDeadNodeOnStartedPrimaryShard() {
                 .add(newNode(nodeIdRemaining))
         ).build();
 
-        rerouteResult = allocation.deassociateDeadNodes(clusterState, true, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.deassociateDeadNodes(clusterState, true, "reroute");
 
         assertThat(clusterState.getRoutingNodes().node(nodeIdRemaining).iterator().next().primary(), equalTo(true));
         assertThat(clusterState.getRoutingNodes().node(nodeIdRemaining).iterator().next().state(), equalTo(STARTED));
@@ -115,15 +112,12 @@ public void testDeadNodeWhileRelocatingOnToNode() {
                 .add(newNode("node2"))
         ).build();
 
-        RoutingAllocation.Result rerouteResult = allocation.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.reroute(clusterState, "reroute");
 
         // starting primaries
-        rerouteResult = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
         // starting replicas
-        rerouteResult = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         logger.info("--> verifying all is allocated");
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
@@ -135,8 +129,7 @@ public void testDeadNodeWhileRelocatingOnToNode() {
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
                 .add(newNode("node3"))
         ).build();
-        rerouteResult = allocation.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.reroute(clusterState, "reroute");
 
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node1").iterator().next().state(), equalTo(STARTED));
@@ -148,11 +141,11 @@ public void testDeadNodeWhileRelocatingOnToNode() {
         String origReplicaNodeId = clusterState.routingTable().index("test").shard(0).replicaShards().get(0).currentNodeId();
 
         logger.info("--> moving primary shard to node3");
-        rerouteResult = allocation.reroute(clusterState, new AllocationCommands(
+        AllocationService.CommandsResult commandsResult = allocation.reroute(clusterState, new AllocationCommands(
                 new MoveAllocationCommand("test", 0, clusterState.routingTable().index("test").shard(0).primaryShard().currentNodeId(), "node3")),
             false, false);
-        assertThat(rerouteResult.changed(), equalTo(true));
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        assertThat(commandsResult.getClusterState(), not(equalTo(clusterState)));
+        clusterState = commandsResult.getClusterState();
         assertThat(clusterState.getRoutingNodes().node(origPrimaryNodeId).iterator().next().state(), equalTo(RELOCATING));
         assertThat(clusterState.getRoutingNodes().node("node3").iterator().next().state(), equalTo(INITIALIZING));
 
@@ -161,8 +154,7 @@ public void testDeadNodeWhileRelocatingOnToNode() {
                 .add(newNode(origPrimaryNodeId))
                 .add(newNode(origReplicaNodeId))
         ).build();
-        rerouteResult = allocation.deassociateDeadNodes(clusterState, true, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.deassociateDeadNodes(clusterState, true, "reroute");
 
         assertThat(clusterState.getRoutingNodes().node(origPrimaryNodeId).iterator().next().state(), equalTo(STARTED));
         assertThat(clusterState.getRoutingNodes().node(origReplicaNodeId).iterator().next().state(), equalTo(STARTED));
@@ -189,15 +181,12 @@ public void testDeadNodeWhileRelocatingOnFromNode() {
                 .add(newNode("node2"))
         ).build();
 
-        RoutingAllocation.Result rerouteResult = allocation.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.reroute(clusterState, "reroute");
 
         // starting primaries
-        rerouteResult = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
         // starting replicas
-        rerouteResult = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         logger.info("--> verifying all is allocated");
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
@@ -209,8 +198,7 @@ public void testDeadNodeWhileRelocatingOnFromNode() {
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
                 .add(newNode("node3"))
         ).build();
-        rerouteResult = allocation.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.reroute(clusterState, "reroute");
 
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node1").iterator().next().state(), equalTo(STARTED));
@@ -222,11 +210,11 @@ public void testDeadNodeWhileRelocatingOnFromNode() {
         String origReplicaNodeId = clusterState.routingTable().index("test").shard(0).replicaShards().get(0).currentNodeId();
 
         logger.info("--> moving primary shard to node3");
-        rerouteResult = allocation.reroute(clusterState, new AllocationCommands(
+        AllocationService.CommandsResult commandsResult = allocation.reroute(clusterState, new AllocationCommands(
                 new MoveAllocationCommand("test",0 , clusterState.routingTable().index("test").shard(0).primaryShard().currentNodeId(), "node3")),
             false, false);
-        assertThat(rerouteResult.changed(), equalTo(true));
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        assertThat(commandsResult.getClusterState(), not(equalTo(clusterState)));
+        clusterState = commandsResult.getClusterState();
         assertThat(clusterState.getRoutingNodes().node(origPrimaryNodeId).iterator().next().state(), equalTo(RELOCATING));
         assertThat(clusterState.getRoutingNodes().node("node3").iterator().next().state(), equalTo(INITIALIZING));
 
@@ -235,8 +223,7 @@ public void testDeadNodeWhileRelocatingOnFromNode() {
                 .add(newNode("node3"))
                 .add(newNode(origReplicaNodeId))
         ).build();
-        rerouteResult = allocation.deassociateDeadNodes(clusterState, true, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.deassociateDeadNodes(clusterState, true, "reroute");
 
         assertThat(clusterState.getRoutingNodes().node(origReplicaNodeId).iterator().next().state(), equalTo(STARTED));
         assertThat(clusterState.getRoutingNodes().node("node3").iterator().next().state(), equalTo(INITIALIZING));
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/DecisionsImpactOnClusterHealthTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/DecisionsImpactOnClusterHealthTests.java
index a6897d972c480..bee2275743bac 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/DecisionsImpactOnClusterHealthTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/DecisionsImpactOnClusterHealthTests.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.EmptyClusterInfoService;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.health.ClusterStateHealth;
@@ -38,8 +39,7 @@
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
-import org.elasticsearch.cluster.ESAllocationTestCase;
-import org.elasticsearch.test.gateway.NoopGatewayAllocator;
+import org.elasticsearch.test.gateway.TestGatewayAllocator;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -161,7 +161,7 @@ private ClusterState runAllocationTest(final Settings settings,
     private static AllocationService newAllocationService(Settings settings, Set<AllocationDecider> deciders) {
         return new AllocationService(settings,
                                      new AllocationDeciders(settings, deciders),
-                                     NoopGatewayAllocator.INSTANCE,
+                                     new TestGatewayAllocator(),
                                      new BalancedShardsAllocator(settings),
                                      EmptyClusterInfoService.INSTANCE);
     }
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ElectReplicaAsPrimaryDuringRelocationTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ElectReplicaAsPrimaryDuringRelocationTests.java
index 5707a229da3d2..77e83fd66544c 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ElectReplicaAsPrimaryDuringRelocationTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ElectReplicaAsPrimaryDuringRelocationTests.java
@@ -22,6 +22,7 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -30,11 +31,11 @@
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.cluster.ESAllocationTestCase;
 
 import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
 
 /**
  *
@@ -59,29 +60,26 @@ public void testElectReplicaAsPrimaryDuringRelocation() {
 
         logger.info("Adding two nodes and performing rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         logger.info("Start the primary shards");
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
 
         logger.info("Start the replica shards");
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
-        routingNodes = clusterState.getRoutingNodes();
+        ClusterState resultingState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
+        assertThat(resultingState, not(equalTo(clusterState)));
+        clusterState = resultingState;
 
-        assertTrue(routingResult.changed());
+        routingNodes = clusterState.getRoutingNodes();
         assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(2));
         assertThat(routingNodes.node("node1").numberOfShardsWithState(STARTED), equalTo(2));
         assertThat(routingNodes.node("node2").numberOfShardsWithState(STARTED), equalTo(2));
 
         logger.info("Start another node and perform rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node3"))).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         logger.info("find the replica shard that gets relocated");
         IndexShardRoutingTable indexShardRoutingTable = null;
@@ -95,8 +93,7 @@ public void testElectReplicaAsPrimaryDuringRelocation() {
         if (indexShardRoutingTable != null) {
             logger.info("kill the node [{}] of the primary shard for the relocating replica", indexShardRoutingTable.primaryShard().currentNodeId());
             clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).remove(indexShardRoutingTable.primaryShard().currentNodeId())).build();
-            routingResult = strategy.deassociateDeadNodes(clusterState, true, "reroute");
-            clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+            clusterState = strategy.deassociateDeadNodes(clusterState, true, "reroute");
 
             logger.info("make sure all the primary shards are active");
             assertThat(clusterState.routingTable().index("test").shard(0).primaryShard().active(), equalTo(true));
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ExpectedShardSizeAllocationTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ExpectedShardSizeAllocationTests.java
index d76b666549ed7..873c71f19b550 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ExpectedShardSizeAllocationTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ExpectedShardSizeAllocationTests.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.cluster.ClusterInfo;
 import org.elasticsearch.cluster.ClusterInfoService;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -35,10 +36,10 @@
 import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.cluster.ESAllocationTestCase;
 
 import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
 
 /**
  */
@@ -81,23 +82,20 @@ public void addListener(Listener listener) {
         ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
         logger.info("Adding one node and performing rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1"))).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         assertEquals(1, clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(ShardRoutingState.INITIALIZING));
         assertEquals(byteSize, clusterState.getRoutingTable().shardsWithState(ShardRoutingState.INITIALIZING).get(0).getExpectedShardSize());
         logger.info("Start the primary shard");
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
 
         assertEquals(1, clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(ShardRoutingState.STARTED));
         assertEquals(1, clusterState.getRoutingNodes().unassigned().size());
 
         logger.info("Add another one node and reroute");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node2"))).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         assertEquals(1, clusterState.getRoutingNodes().node("node2").numberOfShardsWithState(ShardRoutingState.INITIALIZING));
         assertEquals(byteSize, clusterState.getRoutingTable().shardsWithState(ShardRoutingState.INITIALIZING).get(0).getExpectedShardSize());
@@ -134,12 +132,10 @@ public void addListener(Listener listener) {
 
         logger.info("adding two nodes and performing rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))).build();
-        RoutingAllocation.Result rerouteResult = allocation.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.reroute(clusterState, "reroute");
 
         logger.info("start primary shard");
-        rerouteResult = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         logger.info("move the shard");
         String existingNodeId = clusterState.routingTable().index("test").shard(0).primaryShard().currentNodeId();
@@ -149,9 +145,10 @@ public void addListener(Listener listener) {
         } else {
             toNodeId = "node1";
         }
-        rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new MoveAllocationCommand("test", 0, existingNodeId, toNodeId)), false, false);
-        assertThat(rerouteResult.changed(), equalTo(true));
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        AllocationService.CommandsResult commandsResult =
+            allocation.reroute(clusterState, new AllocationCommands(new MoveAllocationCommand("test", 0, existingNodeId, toNodeId)), false, false);
+        assertThat(commandsResult.getClusterState(), not(equalTo(clusterState)));
+        clusterState = commandsResult.getClusterState();
         assertEquals(clusterState.getRoutingNodes().node(existingNodeId).iterator().next().state(), ShardRoutingState.RELOCATING);
         assertEquals(clusterState.getRoutingNodes().node(toNodeId).iterator().next().state(),ShardRoutingState.INITIALIZING);
 
@@ -159,8 +156,7 @@ public void addListener(Listener listener) {
         assertEquals(clusterState.getRoutingNodes().node(toNodeId).iterator().next().getExpectedShardSize(), byteSize);
 
         logger.info("finish moving the shard");
-        rerouteResult = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         assertThat(clusterState.getRoutingNodes().node(existingNodeId).isEmpty(), equalTo(true));
         assertThat(clusterState.getRoutingNodes().node(toNodeId).iterator().next().state(), equalTo(ShardRoutingState.STARTED));
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedNodeRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedNodeRoutingTests.java
index 6dd5c3a7b3020..866833f08d974 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedNodeRoutingTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedNodeRoutingTests.java
@@ -19,20 +19,51 @@
 
 package org.elasticsearch.cluster.routing.allocation;
 
+import com.carrotsearch.hppc.cursors.ObjectCursor;
+import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Version;
+import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteRequest;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
+import org.elasticsearch.action.support.ActiveShardCount;
+import org.elasticsearch.action.support.replication.ClusterStateCreationUtils;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
+import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.cluster.ESAllocationTestCase;
-
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.indices.cluster.AbstractIndicesClusterStateServiceTestCase;
+import org.elasticsearch.indices.cluster.ClusterStateChanges;
+import org.elasticsearch.indices.cluster.IndicesClusterStateService;
+import org.elasticsearch.test.VersionUtils;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.threadpool.ThreadPool;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
+import static org.elasticsearch.cluster.metadata.TemplateUpgradeServiceTests.buildNewFakeTransportAddress;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED;
 import static org.hamcrest.Matchers.equalTo;
@@ -58,19 +89,16 @@ public void testSimpleFailedNodeTest() {
 
         logger.info("start 4 nodes");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2")).add(newNode("node3")).add(newNode("node4"))).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         logger.info("start all the primary shards, replicas will start initializing");
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
         routingNodes = clusterState.getRoutingNodes();
 
         logger.info("start the replica shards");
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
         routingNodes = clusterState.getRoutingNodes();
 
         assertThat(routingNodes.node("node1").numberOfShardsWithState(STARTED), equalTo(1));
@@ -86,8 +114,7 @@ public void testSimpleFailedNodeTest() {
                 .remove(clusterState.routingTable().index("test2").shard(0).primaryShard().currentNodeId())
         )
                 .build();
-        routingResult = strategy.deassociateDeadNodes(clusterState, true, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.deassociateDeadNodes(clusterState, true, "reroute");
         routingNodes = clusterState.getRoutingNodes();
 
         for (RoutingNode routingNode : routingNodes) {
@@ -95,4 +122,120 @@ public void testSimpleFailedNodeTest() {
             assertThat(routingNode.numberOfShardsWithState(INITIALIZING), equalTo(1));
         }
     }
+
+    public void testRandomClusterPromotesNewestReplica() throws InterruptedException {
+
+        ThreadPool threadPool = new TestThreadPool(getClass().getName());
+        ClusterStateChanges cluster = new ClusterStateChanges(xContentRegistry(), threadPool);
+        ClusterState state = randomInitialClusterState();
+
+        // randomly add nodes of mixed versions
+        logger.info("--> adding random nodes");
+        for (int i = 0; i < randomIntBetween(4, 8); i++) {
+            DiscoveryNodes newNodes = DiscoveryNodes.builder(state.nodes())
+                .add(createNode()).build();
+            state = ClusterState.builder(state).nodes(newNodes).build();
+            state = cluster.reroute(state, new ClusterRerouteRequest()); // always reroute after adding node
+        }
+
+        // Log the node versions (for debugging if necessary)
+        for (ObjectCursor<DiscoveryNode> cursor : state.nodes().getDataNodes().values()) {
+            Version nodeVer = cursor.value.getVersion();
+            logger.info("--> node [{}] has version [{}]", cursor.value.getId(), nodeVer);
+        }
+
+        // randomly create some indices
+        logger.info("--> creating some indices");
+        for (int i = 0; i < randomIntBetween(2, 5); i++) {
+            String name = "index_" + randomAlphaOfLength(8).toLowerCase(Locale.ROOT);
+            Settings.Builder settingsBuilder = Settings.builder()
+                .put(SETTING_NUMBER_OF_SHARDS, randomIntBetween(1, 4))
+                .put(SETTING_NUMBER_OF_REPLICAS, randomIntBetween(2, 4));
+            CreateIndexRequest request = new CreateIndexRequest(name, settingsBuilder.build()).waitForActiveShards(ActiveShardCount.NONE);
+            state = cluster.createIndex(state, request);
+            assertTrue(state.metaData().hasIndex(name));
+        }
+
+        ClusterState previousState = state;
+
+        logger.info("--> starting shards");
+        state = cluster.applyStartedShards(state, state.getRoutingNodes().shardsWithState(INITIALIZING));
+        logger.info("--> starting replicas a random number of times");
+        for (int i = 0; i < randomIntBetween(1,10); i++) {
+            state = cluster.applyStartedShards(state, state.getRoutingNodes().shardsWithState(INITIALIZING));
+        }
+
+        boolean keepGoing = true;
+        while (keepGoing) {
+            List<ShardRouting> primaries = state.getRoutingNodes().shardsWithState(STARTED)
+                .stream().filter(ShardRouting::primary).collect(Collectors.toList());
+
+            // Pick a random subset of primaries to fail
+            List<FailedShard> shardsToFail = new ArrayList<>();
+            List<ShardRouting> failedPrimaries = randomSubsetOf(primaries);
+            failedPrimaries.stream().forEach(sr -> {
+                shardsToFail.add(new  FailedShard(randomFrom(sr), "failed primary", new Exception()));
+            });
+
+            logger.info("--> state before failing shards: {}", state);
+            state = cluster.applyFailedShards(state, shardsToFail);
+
+            final ClusterState compareState = state;
+            failedPrimaries.forEach(shardRouting -> {
+                logger.info("--> verifying version for {}", shardRouting);
+
+                ShardRouting newPrimary = compareState.routingTable().index(shardRouting.index())
+                    .shard(shardRouting.id()).primaryShard();
+                Version newPrimaryVersion = getNodeVersion(newPrimary, compareState);
+
+                logger.info("--> new primary is on version {}: {}", newPrimaryVersion, newPrimary);
+                compareState.routingTable().shardRoutingTable(newPrimary.shardId()).shardsWithState(STARTED)
+                    .stream()
+                    .forEach(sr -> {
+                        Version candidateVer = getNodeVersion(sr, compareState);
+                        if (candidateVer != null) {
+                            logger.info("--> candidate on {} node; shard routing: {}", candidateVer, sr);
+                            assertTrue("candidate was not on the newest version, new primary is on " +
+                                    newPrimaryVersion + " and there is a candidate on " + candidateVer,
+                                candidateVer.onOrBefore(newPrimaryVersion));
+                        }
+                    });
+            });
+
+            keepGoing = randomBoolean();
+        }
+        terminate(threadPool);
+    }
+
+    private static Version getNodeVersion(ShardRouting shardRouting, ClusterState state) {
+        return Optional.ofNullable(state.getNodes().get(shardRouting.currentNodeId())).map(DiscoveryNode::getVersion).orElse(null);
+    }
+
+    private static final AtomicInteger nodeIdGenerator = new AtomicInteger();
+
+    public ClusterState randomInitialClusterState() {
+        List<DiscoveryNode> allNodes = new ArrayList<>();
+        DiscoveryNode localNode = createNode(DiscoveryNode.Role.MASTER); // local node is the master
+        allNodes.add(localNode);
+        // at least two nodes that have the data role so that we can allocate shards
+        allNodes.add(createNode(DiscoveryNode.Role.DATA));
+        allNodes.add(createNode(DiscoveryNode.Role.DATA));
+        for (int i = 0; i < randomIntBetween(2, 5); i++) {
+            allNodes.add(createNode());
+        }
+        ClusterState state = ClusterStateCreationUtils.state(localNode, localNode, allNodes.toArray(new DiscoveryNode[allNodes.size()]));
+        return state;
+    }
+
+
+    protected DiscoveryNode createNode(DiscoveryNode.Role... mustHaveRoles) {
+        Set<DiscoveryNode.Role> roles = new HashSet<>(randomSubsetOf(Sets.newHashSet(DiscoveryNode.Role.values())));
+        for (DiscoveryNode.Role mustHaveRole : mustHaveRoles) {
+            roles.add(mustHaveRole);
+        }
+        final String id = String.format(Locale.ROOT, "node_%03d", nodeIdGenerator.incrementAndGet());
+        return new DiscoveryNode(id, id, buildNewFakeTransportAddress(), Collections.emptyMap(), roles,
+            VersionUtils.randomVersionBetween(random(), Version.V_5_6_0, null));
+    }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedShardsRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedShardsRoutingTests.java
index 852fbd1944b8d..a55b2771c0fbc 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedShardsRoutingTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedShardsRoutingTests.java
@@ -19,11 +19,14 @@
 
 package org.elasticsearch.cluster.routing.allocation;
 
+import com.carrotsearch.hppc.cursors.ObjectCursor;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
@@ -33,13 +36,15 @@
 import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.test.VersionUtils;
 
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
 
+import static org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.RELOCATING;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED;
@@ -69,7 +74,7 @@ public void testFailedShardPrimaryRelocatingToAndFrom() {
         RoutingTable routingTable = RoutingTable.builder()
                 .addAsNew(metaData.index("test"))
                 .build();
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("--> adding 2 nodes on same rack and do rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder()
@@ -77,15 +82,12 @@ public void testFailedShardPrimaryRelocatingToAndFrom() {
                         .add(newNode("node2"))
         ).build();
 
-        RoutingAllocation.Result rerouteResult = allocation.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.reroute(clusterState, "reroute");
 
         // starting primaries
-        rerouteResult = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
         // starting replicas
-        rerouteResult = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         logger.info("--> verifying all is allocated");
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
@@ -97,8 +99,7 @@ public void testFailedShardPrimaryRelocatingToAndFrom() {
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
                         .add(newNode("node3"))
         ).build();
-        rerouteResult = allocation.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.reroute(clusterState, "reroute");
 
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node1").iterator().next().state(), equalTo(STARTED));
@@ -110,33 +111,31 @@ public void testFailedShardPrimaryRelocatingToAndFrom() {
         String origReplicaNodeId = clusterState.routingTable().index("test").shard(0).replicaShards().get(0).currentNodeId();
 
         logger.info("--> moving primary shard to node3");
-        rerouteResult = allocation.reroute(clusterState, new AllocationCommands(
+        AllocationService.CommandsResult commandsResult = allocation.reroute(clusterState, new AllocationCommands(
                         new MoveAllocationCommand("test", 0, clusterState.routingTable().index("test").shard(0).primaryShard().currentNodeId(), "node3")),
             false, false);
-        assertThat(rerouteResult.changed(), equalTo(true));
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        assertThat(commandsResult.getClusterState(), not(equalTo(clusterState)));
+        clusterState = commandsResult.getClusterState();
         assertThat(clusterState.getRoutingNodes().node(origPrimaryNodeId).iterator().next().state(), equalTo(RELOCATING));
         assertThat(clusterState.getRoutingNodes().node("node3").iterator().next().state(), equalTo(INITIALIZING));
 
         logger.info("--> fail primary shard recovering instance on node3 being initialized");
-        rerouteResult = allocation.applyFailedShard(clusterState, clusterState.getRoutingNodes().node("node3").iterator().next());
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.applyFailedShard(clusterState, clusterState.getRoutingNodes().node("node3").iterator().next());
 
         assertThat(clusterState.getRoutingNodes().node(origPrimaryNodeId).iterator().next().state(), equalTo(STARTED));
         assertThat(clusterState.getRoutingNodes().node("node3").size(), equalTo(0));
 
         logger.info("--> moving primary shard to node3");
-        rerouteResult = allocation.reroute(clusterState, new AllocationCommands(
+        commandsResult = allocation.reroute(clusterState, new AllocationCommands(
                         new MoveAllocationCommand("test", 0, clusterState.routingTable().index("test").shard(0).primaryShard().currentNodeId(), "node3")),
             false, false);
-        assertThat(rerouteResult.changed(), equalTo(true));
-        clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
+        assertThat(commandsResult.getClusterState(), not(equalTo(clusterState)));
+        clusterState = commandsResult.getClusterState();
         assertThat(clusterState.getRoutingNodes().node(origPrimaryNodeId).iterator().next().state(), equalTo(RELOCATING));
         assertThat(clusterState.getRoutingNodes().node("node3").iterator().next().state(), equalTo(INITIALIZING));
 
         logger.info("--> fail primary shard recovering instance on node1 being relocated");
-        rerouteResult = allocation.applyFailedShard(clusterState, clusterState.getRoutingNodes().node(origPrimaryNodeId).iterator().next());
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.applyFailedShard(clusterState, clusterState.getRoutingNodes().node(origPrimaryNodeId).iterator().next());
 
         // check promotion of replica to primary
         assertThat(clusterState.getRoutingNodes().node(origReplicaNodeId).iterator().next().state(), equalTo(STARTED));
@@ -160,20 +159,20 @@ public void testFailPrimaryStartedCheckReplicaElected() {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(initialRoutingTable).build();
+        ClusterState clusterState = ClusterState.builder(CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(initialRoutingTable).build();
 
         logger.info("Adding two nodes and performing rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         logger.info("Start the shards (primaries)");
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
 
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
 
-        assertTrue(routingResult.changed());
+        ClusterState newState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
+
         assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
         for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
             assertThat(clusterState.routingTable().index("test").shard(i).size(), equalTo(2));
@@ -187,10 +186,10 @@ public void testFailPrimaryStartedCheckReplicaElected() {
 
         logger.info("Start the shards (backups)");
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        newState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
 
-        assertTrue(routingResult.changed());
         assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
         for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
             assertThat(clusterState.routingTable().index("test").shard(i).size(), equalTo(2));
@@ -204,11 +203,10 @@ public void testFailPrimaryStartedCheckReplicaElected() {
 
         logger.info("fail the primary shard, will have no place to be rerouted to (single node), so stays unassigned");
         ShardRouting shardToFail = clusterState.routingTable().index("test").shard(0).primaryShard();
-        routingResult = strategy.applyFailedShard(clusterState, shardToFail);
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
-        routingNodes = clusterState.getRoutingNodes();
+        newState = strategy.applyFailedShard(clusterState, shardToFail);
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
 
-        assertTrue(routingResult.changed());
         assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(2));
         assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(2));
         assertThat(clusterState.routingTable().index("test").shard(0).primaryShard().currentNodeId(), not(equalTo(shardToFail.currentNodeId())));
@@ -234,14 +232,14 @@ public void testFirstAllocationFailureSingleNode() {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(initialRoutingTable).build();
+        ClusterState clusterState = ClusterState.builder(CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(initialRoutingTable).build();
 
         logger.info("Adding single node and performing rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1"))).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        ClusterState newState = strategy.reroute(clusterState, "reroute");
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
 
-        assertTrue(routingResult.changed());
         assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
         for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
             assertThat(clusterState.routingTable().index("test").shard(i).size(), equalTo(2));
@@ -254,10 +252,10 @@ public void testFirstAllocationFailureSingleNode() {
 
         logger.info("fail the first shard, will have no place to be rerouted to (single node), so stays unassigned");
         ShardRouting firstShard = clusterState.getRoutingNodes().node("node1").iterator().next();
-        routingResult = strategy.applyFailedShard(clusterState, firstShard);
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        newState = strategy.applyFailedShard(clusterState, firstShard);
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
 
-        assertTrue(routingResult.changed());
         assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
         for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
             assertThat(clusterState.routingTable().index("test").shard(i).size(), equalTo(2));
@@ -285,7 +283,7 @@ public void testSingleShardMultipleAllocationFailures() {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(initialRoutingTable).build();
+        ClusterState clusterState = ClusterState.builder(CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(initialRoutingTable).build();
 
         logger.info("Adding {} nodes and performing rerouting", numberOfReplicas + 1);
         DiscoveryNodes.Builder nodeBuilder = DiscoveryNodes.builder();
@@ -295,17 +293,13 @@ public void testSingleShardMultipleAllocationFailures() {
         clusterState = ClusterState.builder(clusterState).nodes(nodeBuilder).build();
         while (!clusterState.routingTable().shardsWithState(UNASSIGNED).isEmpty()) {
             // start all initializing
-            clusterState = ClusterState.builder(clusterState)
-                    .routingResult(strategy
-                                    .applyStartedShards(clusterState, clusterState.routingTable().shardsWithState(INITIALIZING))
-                    )
-                    .build();
+            clusterState = strategy.applyStartedShards(clusterState, clusterState.routingTable().shardsWithState(INITIALIZING));
             // and assign more unassigned
-            clusterState = ClusterState.builder(clusterState).routingResult(strategy.reroute(clusterState, "reroute")).build();
+            clusterState = strategy.reroute(clusterState, "reroute");
         }
 
         int shardsToFail = randomIntBetween(1, numberOfReplicas);
-        ArrayList<FailedRerouteAllocation.FailedShard> failedShards = new ArrayList<>();
+        ArrayList<FailedShard> failedShards = new ArrayList<>();
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
         Set<String> failedNodes = new HashSet<>();
         Set<ShardRouting> shardRoutingsToFail = new HashSet<>();
@@ -314,18 +308,17 @@ public void testSingleShardMultipleAllocationFailures() {
             logger.info("failing shard on node [{}]", failedNode);
             ShardRouting shardToFail = routingNodes.node(failedNode).iterator().next();
             if (shardRoutingsToFail.contains(shardToFail) == false) {
-                failedShards.add(new FailedRerouteAllocation.FailedShard(shardToFail, null, null));
+                failedShards.add(new FailedShard(shardToFail, null, null));
                 failedNodes.add(failedNode);
                 shardRoutingsToFail.add(shardToFail);
             }
         }
 
-        RoutingAllocation.Result routingResult = strategy.applyFailedShards(clusterState, failedShards);
-
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyFailedShards(clusterState, failedShards);
         routingNodes = clusterState.getRoutingNodes();
-        for (FailedRerouteAllocation.FailedShard failedShard : failedShards) {
-            if (routingNodes.getByAllocationId(failedShard.routingEntry.shardId(), failedShard.routingEntry.allocationId().getId()) != null) {
+        for (FailedShard failedShard : failedShards) {
+            if (routingNodes.getByAllocationId(failedShard.getRoutingEntry().shardId(),
+                                               failedShard.getRoutingEntry().allocationId().getId()) != null) {
                 fail("shard " + failedShard + " was not failed");
             }
         }
@@ -353,15 +346,15 @@ public void testFirstAllocationFailureTwoNodes() {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(initialRoutingTable).build();
+        ClusterState clusterState = ClusterState.builder(CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(initialRoutingTable).build();
 
         logger.info("Adding two nodes and performing rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        ClusterState newState = strategy.reroute(clusterState, "reroute");
+        assertThat(newState, not(clusterState));
+        clusterState = newState;
         final String nodeHoldingPrimary = clusterState.routingTable().index("test").shard(0).primaryShard().currentNodeId();
 
-        assertTrue(routingResult.changed());
         assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
         for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
             assertThat(clusterState.routingTable().index("test").shard(i).size(), equalTo(2));
@@ -374,9 +367,9 @@ public void testFirstAllocationFailureTwoNodes() {
 
         logger.info("fail the first shard, will start INITIALIZING on the second node");
         final ShardRouting firstShard = clusterState.getRoutingNodes().node(nodeHoldingPrimary).iterator().next();
-        routingResult = strategy.applyFailedShard(clusterState, firstShard);
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
-        assertTrue(routingResult.changed());
+        newState = strategy.applyFailedShard(clusterState, firstShard);
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
 
         final String nodeHoldingPrimary2 = clusterState.routingTable().index("test").shard(0).primaryShard().currentNodeId();
         assertThat(nodeHoldingPrimary2, not(equalTo(nodeHoldingPrimary)));
@@ -408,19 +401,18 @@ public void testRebalanceFailure() {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(initialRoutingTable).build();
+        ClusterState clusterState = ClusterState.builder(CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(initialRoutingTable).build();
 
         logger.info("Adding two nodes and performing rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         logger.info("Start the shards (primaries)");
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        ClusterState newState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
 
-        assertTrue(routingResult.changed());
         assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(2));
         for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
             assertThat(clusterState.routingTable().index("test").shard(i).size(), equalTo(2));
@@ -434,10 +426,10 @@ public void testRebalanceFailure() {
 
         logger.info("Start the shards (backups)");
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        newState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
 
-        assertTrue(routingResult.changed());
         assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(2));
         for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
             assertThat(clusterState.routingTable().index("test").shard(i).size(), equalTo(2));
@@ -451,11 +443,11 @@ public void testRebalanceFailure() {
 
         logger.info("Adding third node and reroute");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node3"))).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        newState = strategy.reroute(clusterState, "reroute");
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
         routingNodes = clusterState.getRoutingNodes();
 
-        assertTrue(routingResult.changed());
         assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(2));
         assertThat(routingNodes.node("node1").numberOfShardsWithState(STARTED, RELOCATING), equalTo(2));
         assertThat(routingNodes.node("node1").numberOfShardsWithState(STARTED), lessThan(3));
@@ -466,12 +458,11 @@ public void testRebalanceFailure() {
 
         logger.info("Fail the shards on node 3");
         ShardRouting shardToFail = routingNodes.node("node3").iterator().next();
-        routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyFailedShard(clusterState, shardToFail);
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        newState = strategy.applyFailedShard(clusterState, shardToFail);
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
         routingNodes = clusterState.getRoutingNodes();
 
-        assertTrue(routingResult.changed());
         assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(2));
         assertThat(routingNodes.node("node1").numberOfShardsWithState(STARTED, RELOCATING), equalTo(2));
         assertThat(routingNodes.node("node1").numberOfShardsWithState(STARTED), lessThan(3));
@@ -494,7 +485,10 @@ public void testFailAllReplicasInitializingOnPrimaryFail() {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
+            .metaData(metaData).routingTable(routingTable).build();
+
+        ShardId shardId = new ShardId(metaData.index("test").getIndex(), 0);
 
         // add 4 nodes
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2")).add(newNode("node3")).add(newNode("node4"))).build();
@@ -502,26 +496,30 @@ public void testFailAllReplicasInitializingOnPrimaryFail() {
         assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).size(), equalTo(2));
         // start primary shards
-        clusterState = ClusterState.builder(clusterState).routingTable(allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING)).routingTable()).build();
+        clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
         assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(2));
 
-        // fail the primary shard, check replicas get removed as well...
-        ShardRouting primaryShardToFail = clusterState.routingTable().index("test").shard(0).primaryShard();
-        RoutingAllocation.Result routingResult = allocation.applyFailedShard(clusterState, primaryShardToFail);
-        assertThat(routingResult.changed(), equalTo(true));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
-        // the primary gets allocated on another node, replicas are unassigned
+        // start one replica so it can take over.
+        clusterState = allocation.applyStartedShards(clusterState,
+            Collections.singletonList(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).get(0)));
+        assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(2));
         assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(1));
-        assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).size(), equalTo(2));
+        ShardRouting startedReplica = clusterState.getRoutingNodes().activeReplicaWithHighestVersion(shardId);
 
-        ShardRouting newPrimaryShard = clusterState.routingTable().index("test").shard(0).primaryShard();
-        assertThat(newPrimaryShard, not(equalTo(primaryShardToFail)));
 
-        // start the primary shard
-        clusterState = ClusterState.builder(clusterState).routingResult(allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING))).build();
+        // fail the primary shard, check replicas get removed as well...
+        ShardRouting primaryShardToFail = clusterState.routingTable().index("test").shard(0).primaryShard();
+        ClusterState newState = allocation.applyFailedShard(clusterState, primaryShardToFail);
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
+        // the primary gets allocated on another node, replicas are initializing
         assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(2));
+
+        ShardRouting newPrimaryShard = clusterState.routingTable().index("test").shard(0).primaryShard();
+        assertThat(newPrimaryShard, not(equalTo(primaryShardToFail)));
+        assertThat(newPrimaryShard.allocationId(), equalTo(startedReplica.allocationId()));
     }
 
     public void testFailAllReplicasInitializingOnPrimaryFailWhileHavingAReplicaToElect() {
@@ -536,32 +534,147 @@ public void testFailAllReplicasInitializingOnPrimaryFailWhileHavingAReplicaToEle
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         // add 4 nodes
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2")).add(newNode("node3")).add(newNode("node4"))).build();
-        clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState, "reroute")).build();
+        clusterState = allocation.reroute(clusterState, "reroute");
         assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).size(), equalTo(2));
         // start primary shards
-        clusterState = ClusterState.builder(clusterState).routingResult(allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING))).build();
+        clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
         assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(2));
 
         // start another replica shard, while keep one initializing
-        clusterState = ClusterState.builder(clusterState).routingResult(allocation.applyStartedShards(clusterState, Collections.singletonList(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).get(0)))).build();
+        clusterState = allocation.applyStartedShards(clusterState, Collections.singletonList(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).get(0)));
         assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(2));
         assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(1));
 
         // fail the primary shard, check one replica gets elected to primary, others become INITIALIZING (from it)
         ShardRouting primaryShardToFail = clusterState.routingTable().index("test").shard(0).primaryShard();
-        RoutingAllocation.Result routingResult = allocation.applyFailedShard(clusterState, primaryShardToFail);
-        assertThat(routingResult.changed(), equalTo(true));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        ClusterState newState = allocation.applyFailedShard(clusterState, primaryShardToFail);
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
+        assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(1));
+        assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(2));
+
+        ShardRouting newPrimaryShard = clusterState.routingTable().index("test").shard(0).primaryShard();
+        assertThat(newPrimaryShard, not(equalTo(primaryShardToFail)));
+    }
+
+    public void testReplicaOnNewestVersionIsPromoted() {
+        AllocationService allocation = createAllocationService(Settings.builder().build());
+
+        MetaData metaData = MetaData.builder().put(IndexMetaData.builder("test")
+                .settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(3)) .build();
+
+        RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metaData.index("test")).build();
+
+        ClusterState clusterState = ClusterState.builder(CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
+                .metaData(metaData).routingTable(initialRoutingTable).build();
+
+        ShardId shardId = new ShardId(metaData.index("test").getIndex(), 0);
+
+        // add a single node
+        clusterState = ClusterState.builder(clusterState).nodes(
+                DiscoveryNodes.builder()
+                .add(newNode("node1-5.x", Version.V_5_4_0)))
+                .build();
+        clusterState = ClusterState.builder(clusterState).routingTable(allocation.reroute(clusterState, "reroute").routingTable()).build();
+        assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(1));
+        assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).size(), equalTo(3));
+
+        // start primary shard
+        clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
+        assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(1));
+        assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).size(), equalTo(3));
+
+        // add another 5.6 node
+        clusterState = ClusterState.builder(clusterState).nodes(
+                DiscoveryNodes.builder(clusterState.nodes())
+                .add(newNode("node2-5.x", Version.V_5_4_0)))
+                .build();
+
+        // start the shards, should have 1 primary and 1 replica available
+        clusterState = allocation.reroute(clusterState, "reroute");
         assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(1));
+        assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(1));
+        clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
+        assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(2));
+        assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).size(), equalTo(2));
+
+        clusterState = ClusterState.builder(clusterState).nodes(
+                DiscoveryNodes.builder(clusterState.nodes())
+                .add(newNode("node3-6.x", VersionUtils.randomVersionBetween(random(), Version.V_5_6_0, null)))
+                .add(newNode("node4-6.x", VersionUtils.randomVersionBetween(random(), Version.V_5_6_0, null))))
+                .build();
+
+        // start all the replicas
+        clusterState = allocation.reroute(clusterState, "reroute");
+        assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(2));
         assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(2));
+        clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
+        assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(4));
+        assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).size(), equalTo(0));
+
+        ShardRouting startedReplica = clusterState.getRoutingNodes().activeReplicaWithHighestVersion(shardId);
+        logger.info("--> all shards allocated, replica that should be promoted: {}", startedReplica);
+
+        // fail the primary shard again and make sure the correct replica is promoted
+        ShardRouting primaryShardToFail = clusterState.routingTable().index("test").shard(0).primaryShard();
+        ClusterState newState = allocation.applyFailedShard(clusterState, primaryShardToFail);
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
+        // the primary gets allocated on another node
+        assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(3));
 
         ShardRouting newPrimaryShard = clusterState.routingTable().index("test").shard(0).primaryShard();
         assertThat(newPrimaryShard, not(equalTo(primaryShardToFail)));
+        assertThat(newPrimaryShard.allocationId(), equalTo(startedReplica.allocationId()));
+
+        Version replicaNodeVersion = clusterState.nodes().getDataNodes().get(startedReplica.currentNodeId()).getVersion();
+        assertNotNull(replicaNodeVersion);
+        logger.info("--> shard {} got assigned to node with version {}", startedReplica, replicaNodeVersion);
+
+        for (ObjectCursor<DiscoveryNode> cursor : clusterState.nodes().getDataNodes().values()) {
+            if ("node1".equals(cursor.value.getId())) {
+                // Skip the node that the primary was on, it doesn't have a replica so doesn't need a version check
+                continue;
+            }
+            Version nodeVer = cursor.value.getVersion();
+            assertTrue("expected node [" + cursor.value.getId() + "] with version " + nodeVer + " to be before " + replicaNodeVersion,
+                    replicaNodeVersion.onOrAfter(nodeVer));
+        }
+
+        startedReplica = clusterState.getRoutingNodes().activeReplicaWithHighestVersion(shardId);
+        logger.info("--> failing primary shard a second time, should select: {}", startedReplica);
+
+        // fail the primary shard again, and ensure the same thing happens
+        ShardRouting secondPrimaryShardToFail = clusterState.routingTable().index("test").shard(0).primaryShard();
+        newState = allocation.applyFailedShard(clusterState, secondPrimaryShardToFail);
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
+        // the primary gets allocated on another node
+        assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(2));
+
+        newPrimaryShard = clusterState.routingTable().index("test").shard(0).primaryShard();
+        assertThat(newPrimaryShard, not(equalTo(secondPrimaryShardToFail)));
+        assertThat(newPrimaryShard.allocationId(), equalTo(startedReplica.allocationId()));
+
+        replicaNodeVersion = clusterState.nodes().getDataNodes().get(startedReplica.currentNodeId()).getVersion();
+        assertNotNull(replicaNodeVersion);
+        logger.info("--> shard {} got assigned to node with version {}", startedReplica, replicaNodeVersion);
+
+        for (ObjectCursor<DiscoveryNode> cursor : clusterState.nodes().getDataNodes().values()) {
+            if (primaryShardToFail.currentNodeId().equals(cursor.value.getId()) ||
+                    secondPrimaryShardToFail.currentNodeId().equals(cursor.value.getId())) {
+                // Skip the node that the primary was on, it doesn't have a replica so doesn't need a version check
+                continue;
+            }
+            Version nodeVer = cursor.value.getVersion();
+            assertTrue("expected node [" + cursor.value.getId() + "] with version " + nodeVer + " to be before " + replicaNodeVersion,
+                    replicaNodeVersion.onOrAfter(nodeVer));
+        }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FilterAllocationDeciderTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FilterAllocationDeciderTests.java
deleted file mode 100644
index 0d067830d7cda..0000000000000
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FilterAllocationDeciderTests.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.cluster.routing.allocation;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.EmptyClusterInfoService;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.RoutingTable;
-import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
-import org.elasticsearch.cluster.routing.allocation.decider.Decision;
-import org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider;
-import org.elasticsearch.common.settings.ClusterSettings;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.cluster.ESAllocationTestCase;
-import org.elasticsearch.test.gateway.NoopGatewayAllocator;
-
-import java.util.Collections;
-
-import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
-import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED;
-import static org.elasticsearch.cluster.routing.ShardRoutingState.UNASSIGNED;
-
-public class FilterAllocationDeciderTests extends ESAllocationTestCase {
-
-    public void testFilterInitialAllocation() {
-        FilterAllocationDecider filterAllocationDecider = new FilterAllocationDecider(Settings.EMPTY,
-            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
-        AllocationDeciders allocationDeciders = new AllocationDeciders(Settings.EMPTY,
-            Collections.singleton(filterAllocationDecider));
-        AllocationService service = new AllocationService(Settings.builder().build(), allocationDeciders,
-            NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
-        ClusterState state = createInitialClusterState(service, Settings.builder().put("index.routing.allocation.initial_recovery._id",
-            "node2").build());
-        RoutingTable routingTable = state.routingTable();
-
-        // we can initally only allocate on node2
-        assertEquals(routingTable.index("idx").shard(0).shards().get(0).state(), INITIALIZING);
-        assertEquals(routingTable.index("idx").shard(0).shards().get(0).currentNodeId(), "node2");
-        routingTable = service.applyFailedShard(state, routingTable.index("idx").shard(0).shards().get(0)).routingTable();
-        state = ClusterState.builder(state).routingTable(routingTable).build();
-        assertEquals(routingTable.index("idx").shard(0).shards().get(0).state(), UNASSIGNED);
-        assertNull(routingTable.index("idx").shard(0).shards().get(0).currentNodeId());
-
-        RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, state.getRoutingNodes(), state,
-            null, 0, false);
-        assertEquals(filterAllocationDecider.canAllocate(routingTable.index("idx").shard(0).shards().get(0),
-            state.getRoutingNodes().node("node2")
-            ,allocation), Decision.YES);
-        assertEquals(filterAllocationDecider.canAllocate(routingTable.index("idx").shard(0).shards().get(0),
-            state.getRoutingNodes().node("node1")
-            ,allocation), Decision.NO);
-
-        // after failing the shard we are unassigned since the node is blacklisted and we can't initialize on the other node
-        state = stateFromResult(state, service.reroute(state, "try allocate again"));
-        routingTable = state.routingTable();
-        assertEquals(routingTable.index("idx").shard(0).shards().get(0).state(), INITIALIZING);
-        assertEquals(routingTable.index("idx").shard(0).shards().get(0).currentNodeId(), "node2");
-
-        state = stateFromResult(state, service.applyStartedShards(state, routingTable.index("idx").shard(0).shards()));
-        routingTable = state.routingTable();
-
-        // ok now we are started and can be allocated anywhere!! lets see...
-        assertEquals(routingTable.index("idx").shard(0).shards().get(0).state(), STARTED);
-        assertEquals(routingTable.index("idx").shard(0).shards().get(0).currentNodeId(), "node2");
-
-        // we fail it again to check if we are initializing immediately on the other node
-        state = stateFromResult(state, service.applyFailedShard(state, routingTable.index("idx").shard(0).shards().get(0)));
-        routingTable = state.routingTable();
-        assertEquals(routingTable.index("idx").shard(0).shards().get(0).state(), INITIALIZING);
-        assertEquals(routingTable.index("idx").shard(0).shards().get(0).currentNodeId(), "node1");
-
-        allocation = new RoutingAllocation(allocationDeciders, state.getRoutingNodes(), state,
-            null, 0, false);
-        assertEquals(filterAllocationDecider.canAllocate(routingTable.index("idx").shard(0).shards().get(0),
-            state.getRoutingNodes().node("node2")
-            ,allocation), Decision.YES);
-        assertEquals(filterAllocationDecider.canAllocate(routingTable.index("idx").shard(0).shards().get(0),
-            state.getRoutingNodes().node("node1")
-            ,allocation), Decision.YES);
-    }
-
-    private ClusterState stateFromResult(ClusterState previousState, RoutingAllocation.Result result) {
-        return ClusterState.builder(previousState).routingTable(result.routingTable()).metaData(result.metaData()).build();
-    }
-
-    private ClusterState createInitialClusterState(AllocationService service, Settings settings) {
-        MetaData.Builder metaBuilder = MetaData.builder();
-        metaBuilder.put(IndexMetaData.builder("idx").settings(settings(Version.CURRENT).put(settings))
-            .numberOfShards(1).numberOfReplicas(0));
-        MetaData metaData = metaBuilder.build();
-        RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
-        routingTableBuilder.addAsNew(metaData.index("idx"));
-
-        RoutingTable routingTable = routingTableBuilder.build();
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING
-            .getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
-        clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2")))
-            .build();
-        routingTable = service.reroute(clusterState, "reroute", false).routingTable();
-        clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
-        return clusterState;
-    }
-}
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FilterRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FilterRoutingTests.java
index 6d0542b95cd9c..430809e6726fe 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FilterRoutingTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FilterRoutingTests.java
@@ -22,6 +22,7 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -31,7 +32,6 @@
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.hamcrest.Matchers;
 
 import java.util.List;
@@ -71,17 +71,14 @@ public void testClusterFilters() {
                 .add(newNode("node3", singletonMap("tag1", "value3")))
                 .add(newNode("node4", singletonMap("tag1", "value4")))
         ).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
         assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(2));
 
         logger.info("--> start the shards (primaries)");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         logger.info("--> start the shards (replicas)");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         logger.info("--> make sure shards are only allocated on tag1 with value1 and value2");
         List<ShardRouting> startedShards = clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED);
@@ -119,17 +116,14 @@ public void testIndexFilters() {
                 .add(newNode("node3", singletonMap("tag1", "value3")))
                 .add(newNode("node4", singletonMap("tag1", "value4")))
         ).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
         assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(2));
 
         logger.info("--> start the shards (primaries)");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         logger.info("--> start the shards (replicas)");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         logger.info("--> make sure shards are only allocated on tag1 with value1 and value2");
         List<ShardRouting> startedShards = clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED);
@@ -148,15 +142,13 @@ public void testIndexFilters() {
                 .build()))
             .build();
         clusterState = ClusterState.builder(clusterState).metaData(updatedMetaData).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(2));
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.RELOCATING).size(), equalTo(2));
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(2));
 
         logger.info("--> finish relocation");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         startedShards = clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED);
         assertThat(startedShards.size(), equalTo(4));
@@ -185,14 +177,12 @@ public void testConcurrentRecoveriesAfterShardsCannotRemainOnNode() {
         DiscoveryNode node1 = newNode("node1", singletonMap("tag1", "value1"));
         DiscoveryNode node2 = newNode("node2", singletonMap("tag1", "value2"));
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(node1).add(node2)).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
         assertThat(clusterState.getRoutingNodes().node(node1.getId()).numberOfShardsWithState(INITIALIZING), equalTo(2));
         assertThat(clusterState.getRoutingNodes().node(node2.getId()).numberOfShardsWithState(INITIALIZING), equalTo(2));
 
         logger.info("--> start the shards (only primaries)");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         logger.info("--> make sure all shards are started");
         assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(4));
@@ -204,29 +194,24 @@ public void testConcurrentRecoveriesAfterShardsCannotRemainOnNode() {
                 .build());
 
         logger.info("--> move shards from node1 to node2");
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
         logger.info("--> check that concurrent recoveries only allows 1 shard to move");
         assertThat(clusterState.getRoutingNodes().node(node1.getId()).numberOfShardsWithState(STARTED), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node(node2.getId()).numberOfShardsWithState(INITIALIZING), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node(node2.getId()).numberOfShardsWithState(STARTED), equalTo(2));
 
         logger.info("--> start the shards (only primaries)");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         logger.info("--> move second shard from node1 to node2");
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
         assertThat(clusterState.getRoutingNodes().node(node2.getId()).numberOfShardsWithState(INITIALIZING), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node(node2.getId()).numberOfShardsWithState(STARTED), equalTo(3));
 
         logger.info("--> start the shards (only primaries)");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
         assertThat(clusterState.getRoutingNodes().node(node2.getId()).numberOfShardsWithState(STARTED), equalTo(4));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/InSyncAllocationIdTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/InSyncAllocationIdTests.java
index f27b7e7889d69..616eff4381caf 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/InSyncAllocationIdTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/InSyncAllocationIdTests.java
@@ -78,15 +78,13 @@ public void testInSyncAllocationIdsUpdated() {
         logger.info("adding three nodes and performing rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(
                 newNode("node1")).add(newNode("node2")).add(newNode("node3"))).build();
-        RoutingAllocation.Result rerouteResult = allocation.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.reroute(clusterState, "reroute");
 
         assertThat(clusterState.metaData().index("test").inSyncAllocationIds(0).size(), equalTo(0));
         assertThat(clusterState.metaData().index("test-old").inSyncAllocationIds(0), equalTo(new HashSet<>(Arrays.asList("x", "y"))));
 
         logger.info("start primary shard");
-        rerouteResult = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         assertThat(clusterState.getRoutingTable().shardsWithState(STARTED).size(), equalTo(1));
         assertThat(clusterState.metaData().index("test").inSyncAllocationIds(0).size(), equalTo(1));
@@ -95,8 +93,7 @@ public void testInSyncAllocationIdsUpdated() {
         assertThat(clusterState.metaData().index("test-old").inSyncAllocationIds(0), equalTo(new HashSet<>(Arrays.asList("x", "y"))));
 
         logger.info("start replica shards");
-        rerouteResult = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         assertThat(clusterState.metaData().index("test").inSyncAllocationIds(0).size(), equalTo(3));
 
@@ -104,8 +101,7 @@ public void testInSyncAllocationIdsUpdated() {
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
                 .remove("node1"))
                 .build();
-        rerouteResult = allocation.deassociateDeadNodes(clusterState, true, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.deassociateDeadNodes(clusterState, true, "reroute");
 
         // in-sync allocation ids should not be updated
         assertThat(clusterState.metaData().index("test").inSyncAllocationIds(0).size(), equalTo(3));
@@ -114,8 +110,7 @@ public void testInSyncAllocationIdsUpdated() {
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
                 .remove("node2").remove("node3"))
                 .build();
-        rerouteResult = allocation.deassociateDeadNodes(clusterState, true, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.deassociateDeadNodes(clusterState, true, "reroute");
 
         // in-sync allocation ids should not be updated
         assertThat(clusterState.getRoutingTable().shardsWithState(UNASSIGNED).size(), equalTo(3));
@@ -125,24 +120,22 @@ public void testInSyncAllocationIdsUpdated() {
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
             .add(newNode("node1")))
             .build();
-        rerouteResult = allocation.reroute(clusterState,
-            new AllocationCommands(new AllocateEmptyPrimaryAllocationCommand("test", 0, "node1", true)), false, false);
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.reroute(clusterState,
+            new AllocationCommands(new AllocateEmptyPrimaryAllocationCommand("test", 0, "node1", true)), false, false)
+            .getClusterState();
 
         // check that in-sync allocation ids are reset by forcing an empty primary
         assertThat(clusterState.metaData().index("test").inSyncAllocationIds(0).size(), equalTo(0));
 
         logger.info("start primary shard");
-        rerouteResult = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         assertThat(clusterState.getRoutingTable().shardsWithState(STARTED).size(), equalTo(1));
         assertThat(clusterState.metaData().index("test").inSyncAllocationIds(0).size(), equalTo(1));
 
         logger.info("fail primary shard");
         ShardRouting startedPrimary = clusterState.getRoutingNodes().shardsWithState(STARTED).get(0);
-        rerouteResult = allocation.applyFailedShard(clusterState, startedPrimary);
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.applyFailedShard(clusterState, startedPrimary);
 
         assertThat(clusterState.getRoutingTable().shardsWithState(STARTED).size(), equalTo(0));
         assertEquals(Collections.singleton(startedPrimary.allocationId().getId()),
@@ -165,8 +158,7 @@ public void testDeadNodesBeforeReplicaFailed() throws Exception {
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
             .remove(replicaShard.currentNodeId()))
             .build();
-        RoutingAllocation.Result rerouteResult = allocation.deassociateDeadNodes(clusterState, true, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.deassociateDeadNodes(clusterState, true, "reroute");
         assertThat(clusterState.metaData().index("test").inSyncAllocationIds(0).size(), equalTo(2));
 
         logger.info("fail replica (for which there is no shard routing in the CS anymore)");
@@ -236,8 +228,7 @@ public void testInSyncIdsNotGrowingWithoutBounds() throws Exception {
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
             .remove(replicaShard.currentNodeId()))
             .build();
-        RoutingAllocation.Result rerouteResult = allocation.deassociateDeadNodes(clusterState, true, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.deassociateDeadNodes(clusterState, true, "reroute");
 
         // in-sync allocation ids should not be updated
         assertEquals(inSyncSet, clusterState.metaData().index("test").inSyncAllocationIds(0));
@@ -248,20 +239,17 @@ public void testInSyncIdsNotGrowingWithoutBounds() throws Exception {
             clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
                 .add(newNode(replicaShard.currentNodeId())))
                 .build();
-            rerouteResult = allocation.reroute(clusterState, "reroute");
-            clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+            clusterState = allocation.reroute(clusterState, "reroute");
 
             logger.info("start replica shards");
-            rerouteResult = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-            clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+            clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
             logger.info("remove the node");
             clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
                 .remove(replicaShard.currentNodeId()))
                 .build();
-            rerouteResult = allocation.deassociateDeadNodes(clusterState, true, "reroute");
-            clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
-        }
+            clusterState = allocation.deassociateDeadNodes(clusterState, true, "reroute");
+            }
 
         // in-sync allocation set is bounded
         Set<String> newInSyncSet = clusterState.metaData().index("test").inSyncAllocationIds(0);
@@ -271,6 +259,88 @@ public void testInSyncIdsNotGrowingWithoutBounds() throws Exception {
         assertThat(newInSyncSet, hasItem(primaryShard.allocationId().getId()));
     }
 
+    /**
+     * Only trim set of allocation ids when the set grows
+     */
+    public void testInSyncIdsNotTrimmedWhenNotGrowing() throws Exception {
+        ClusterState clusterState = createOnePrimaryOneReplicaClusterState(allocation);
+
+        Set<String> inSyncSet = clusterState.metaData().index("test").inSyncAllocationIds(0);
+        assertThat(inSyncSet.size(), equalTo(2));
+
+        IndexShardRoutingTable shardRoutingTable = clusterState.routingTable().index("test").shard(0);
+        ShardRouting primaryShard = shardRoutingTable.primaryShard();
+        ShardRouting replicaShard = shardRoutingTable.replicaShards().get(0);
+
+        logger.info("remove replica node");
+        clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
+            .remove(replicaShard.currentNodeId()))
+            .build();
+        clusterState = allocation.deassociateDeadNodes(clusterState, true, "reroute");
+
+        // in-sync allocation ids should not be updated
+        assertEquals(inSyncSet, clusterState.metaData().index("test").inSyncAllocationIds(0));
+
+        logger.info("remove primary node");
+        clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
+            .remove(primaryShard.currentNodeId()))
+            .build();
+        clusterState = allocation.deassociateDeadNodes(clusterState, true, "reroute");
+
+        // in-sync allocation ids should not be updated
+        assertEquals(inSyncSet, clusterState.metaData().index("test").inSyncAllocationIds(0));
+
+        logger.info("decrease number of replicas to 0");
+        clusterState = ClusterState.builder(clusterState)
+            .routingTable(RoutingTable.builder(clusterState.routingTable()).updateNumberOfReplicas(0, "test").build())
+            .metaData(MetaData.builder(clusterState.metaData()).updateNumberOfReplicas(0, "test")).build();
+
+        logger.info("add back node 1");
+        clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(
+            newNode("node1"))).build();
+        clusterState = allocation.reroute(clusterState, "reroute");
+
+        assertThat(clusterState.routingTable().index("test").shard(0).assignedShards().size(), equalTo(1));
+        // in-sync allocation ids should not be updated
+        assertEquals(inSyncSet, clusterState.metaData().index("test").inSyncAllocationIds(0));
+
+        logger.info("start primary shard");
+        clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
+        // in-sync allocation ids should not be updated
+        assertEquals(inSyncSet, clusterState.metaData().index("test").inSyncAllocationIds(0));
+    }
+
+    /**
+     * Don't remove allocation id of failed active primary if there is no replica to promote as primary.
+     */
+    public void testPrimaryAllocationIdNotRemovedFromInSyncSetWhenNoFailOver() throws Exception {
+        ClusterState clusterState = createOnePrimaryOneReplicaClusterState(allocation);
+
+        Set<String> inSyncSet = clusterState.metaData().index("test").inSyncAllocationIds(0);
+        assertThat(inSyncSet.size(), equalTo(2));
+
+        IndexShardRoutingTable shardRoutingTable = clusterState.routingTable().index("test").shard(0);
+        ShardRouting primaryShard = shardRoutingTable.primaryShard();
+        ShardRouting replicaShard = shardRoutingTable.replicaShards().get(0);
+
+        logger.info("remove replica node");
+        clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
+            .remove(replicaShard.currentNodeId()))
+            .build();
+        clusterState = allocation.deassociateDeadNodes(clusterState, true, "reroute");
+
+        // in-sync allocation ids should not be updated
+        assertEquals(inSyncSet, clusterState.metaData().index("test").inSyncAllocationIds(0));
+
+        logger.info("fail primary shard");
+        clusterState = failedClusterStateTaskExecutor.execute(clusterState, Collections.singletonList(new ShardEntry(
+            shardRoutingTable.shardId(), primaryShard.allocationId().getId(), 0L, "dummy", null))).resultingState;
+
+        assertThat(clusterState.routingTable().index("test").shard(0).assignedShards().size(), equalTo(0));
+        // in-sync allocation ids should not be updated
+        assertEquals(inSyncSet, clusterState.metaData().index("test").inSyncAllocationIds(0));
+    }
+
     private ClusterState createOnePrimaryOneReplicaClusterState(AllocationService allocation) {
         logger.info("creating an index with 1 shard, 1 replica");
         MetaData metaData = MetaData.builder()
@@ -285,14 +355,12 @@ private ClusterState createOnePrimaryOneReplicaClusterState(AllocationService al
         logger.info("adding two nodes and performing rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(
             newNode("node1")).add(newNode("node2"))).build();
-        RoutingAllocation.Result rerouteResult = allocation.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.reroute(clusterState, "reroute");
 
         assertThat(clusterState.metaData().index("test").inSyncAllocationIds(0).size(), equalTo(0));
 
         logger.info("start primary shard");
-        rerouteResult = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         assertThat(clusterState.getRoutingTable().shardsWithState(STARTED).size(), equalTo(1));
         assertThat(clusterState.metaData().index("test").inSyncAllocationIds(0).size(), equalTo(1));
@@ -300,8 +368,7 @@ private ClusterState createOnePrimaryOneReplicaClusterState(AllocationService al
             equalTo(clusterState.metaData().index("test").inSyncAllocationIds(0).iterator().next()));
 
         logger.info("start replica shard");
-        rerouteResult = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
         assertThat(clusterState.metaData().index("test").inSyncAllocationIds(0).size(), equalTo(2));
         return clusterState;
     }
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/IndexBalanceTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/IndexBalanceTests.java
index cd35669f04847..986d08843ada3 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/IndexBalanceTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/IndexBalanceTests.java
@@ -22,6 +22,7 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -30,12 +31,12 @@
 import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.cluster.ESAllocationTestCase;
 
 import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.UNASSIGNED;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.nullValue;
 
 /**
@@ -84,10 +85,10 @@ public void testBalanceAllNodesStarted() {
         clusterState = ClusterState.builder(clusterState)
                 .nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2")).add(newNode("node3"))).build();
 
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        ClusterState newState = strategy.reroute(clusterState, "reroute");
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
 
-        assertTrue(routingResult.changed());
         assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(3));
         for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
             assertThat(clusterState.routingTable().index("test").shard(i).size(), equalTo(2));
@@ -100,16 +101,13 @@ public void testBalanceAllNodesStarted() {
 
         logger.info("Another round of rebalancing");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
-
-        assertFalse(routingResult.changed());
+        newState = strategy.reroute(clusterState, "reroute");
+        assertThat(newState, equalTo(clusterState));
 
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
-
-        assertTrue(routingResult.changed());
+        newState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
         assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(3));
         for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
             assertThat(clusterState.routingTable().index("test").shard(i).size(), equalTo(2));
@@ -123,16 +121,16 @@ public void testBalanceAllNodesStarted() {
         }
 
         logger.info("Reroute, nothing should change");
-        routingResult = strategy.reroute(clusterState, "reroute");
-        assertFalse(routingResult.changed());
+        newState = strategy.reroute(clusterState, "reroute");
+        assertThat(newState, equalTo(clusterState));
 
         logger.info("Start the more shards");
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        newState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
         routingNodes = clusterState.getRoutingNodes();
 
-        assertTrue(routingResult.changed());
         assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(3));
         for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
             assertThat(clusterState.routingTable().index("test").shard(i).size(), equalTo(2));
@@ -202,10 +200,9 @@ public void testBalanceIncrementallyStartNodes() {
         logger.info("Adding one node and performing rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1"))).build();
 
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
-
-        assertTrue(routingResult.changed());
+        ClusterState newState = strategy.reroute(clusterState, "reroute");
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
         assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(3));
         for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
             assertThat(clusterState.routingTable().index("test").shard(i).size(), equalTo(2));
@@ -219,17 +216,15 @@ public void testBalanceIncrementallyStartNodes() {
         logger.info("Add another node and perform rerouting, nothing will happen since primary not started");
         clusterState = ClusterState.builder(clusterState)
                 .nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node2"))).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        newState = strategy.reroute(clusterState, "reroute");
+        assertThat(newState, equalTo(clusterState));
 
-        assertFalse(routingResult.changed());
 
         logger.info("Start the primary shard");
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
-
-        assertTrue(routingResult.changed());
+        newState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
         assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(3));
         for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
             assertThat(clusterState.routingTable().index("test").shard(i).size(), equalTo(2));
@@ -243,17 +238,17 @@ public void testBalanceIncrementallyStartNodes() {
         }
 
         logger.info("Reroute, nothing should change");
-        routingResult = strategy.reroute(clusterState, "reroute");
-        assertFalse(routingResult.changed());
+        newState = strategy.reroute(clusterState, "reroute");
+        assertThat(newState, equalTo(clusterState));
 
         logger.info("Start the backup shard");
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        newState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
         routingNodes = clusterState.getRoutingNodes();
 
-        assertTrue(routingResult.changed());
-        assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(3));
+                assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(3));
         for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
             assertThat(clusterState.routingTable().index("test").shard(i).size(), equalTo(2));
             assertThat(clusterState.routingTable().index("test").shard(i).shards().size(), equalTo(2));
@@ -273,30 +268,29 @@ public void testBalanceIncrementallyStartNodes() {
         logger.info("Add another node and perform rerouting, nothing will happen since primary not started");
         clusterState = ClusterState.builder(clusterState)
                 .nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node3"))).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
-
-        assertTrue(routingResult.changed());
+        newState = strategy.reroute(clusterState, "reroute");
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
 
         logger.info("Reroute, nothing should change");
-        routingResult = strategy.reroute(clusterState, "reroute");
-        assertFalse(routingResult.changed());
+        newState = strategy.reroute(clusterState, "reroute");
+        assertThat(newState, equalTo(clusterState));
 
         logger.info("Start the backup shard");
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        newState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
         routingNodes = clusterState.getRoutingNodes();
 
-        assertTrue(routingResult.changed());
-        assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(3));
+                assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(3));
 
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        newState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
         routingNodes = clusterState.getRoutingNodes();
 
-        assertTrue(routingResult.changed());
-        assertThat(clusterState.routingTable().index("test1").shards().size(), equalTo(3));
+                assertThat(clusterState.routingTable().index("test1").shards().size(), equalTo(3));
 
         assertThat(routingNodes.node("node1").numberOfShardsWithState(STARTED), equalTo(4));
         assertThat(routingNodes.node("node2").numberOfShardsWithState(STARTED), equalTo(4));
@@ -340,10 +334,9 @@ public void testBalanceAllNodesStartedAddIndex() {
         clusterState = ClusterState.builder(clusterState)
                 .nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2")).add(newNode("node3"))).build();
 
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
-
-        assertTrue(routingResult.changed());
+        ClusterState newState = strategy.reroute(clusterState, "reroute");
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
         assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(3));
         for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
             assertThat(clusterState.routingTable().index("test").shard(i).size(), equalTo(2));
@@ -356,16 +349,14 @@ public void testBalanceAllNodesStartedAddIndex() {
 
         logger.info("Another round of rebalancing");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        newState = strategy.reroute(clusterState, "reroute");
+        assertThat(newState, equalTo(clusterState));
 
-        assertFalse(routingResult.changed());
 
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
-
-        assertTrue(routingResult.changed());
+        newState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
         assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(3));
         for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
             assertThat(clusterState.routingTable().index("test").shard(i).size(), equalTo(2));
@@ -379,15 +370,15 @@ public void testBalanceAllNodesStartedAddIndex() {
         }
 
         logger.info("Reroute, nothing should change");
-        routingResult = strategy.reroute(clusterState, "reroute");
-        assertFalse(routingResult.changed());
+        newState = strategy.reroute(clusterState, "reroute");
+        assertThat(newState, equalTo(clusterState));
 
         logger.info("Start the more shards");
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        newState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
         routingNodes = clusterState.getRoutingNodes();
-        assertTrue(routingResult.changed());
         assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(3));
         for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
             assertThat(clusterState.routingTable().index("test").shard(i).size(), equalTo(2));
@@ -419,10 +410,9 @@ public void testBalanceAllNodesStartedAddIndex() {
 
         assertThat(clusterState.routingTable().index("test1").shards().size(), equalTo(3));
 
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
-
-        assertTrue(routingResult.changed());
+        newState = strategy.reroute(clusterState, "reroute");
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
         assertThat(clusterState.routingTable().index("test1").shards().size(), equalTo(3));
         for (int i = 0; i < clusterState.routingTable().index("test1").shards().size(); i++) {
             assertThat(clusterState.routingTable().index("test1").shard(i).size(), equalTo(2));
@@ -435,16 +425,14 @@ public void testBalanceAllNodesStartedAddIndex() {
 
         logger.info("Another round of rebalancing");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        newState = strategy.reroute(clusterState, "reroute");
+        assertThat(newState, equalTo(clusterState));
 
-        assertFalse(routingResult.changed());
 
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
-
-        assertTrue(routingResult.changed());
+        newState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
         assertThat(clusterState.routingTable().index("test1").shards().size(), equalTo(3));
         for (int i = 0; i < clusterState.routingTable().index("test1").shards().size(); i++) {
             assertThat(clusterState.routingTable().index("test1").shard(i).size(), equalTo(2));
@@ -458,15 +446,15 @@ public void testBalanceAllNodesStartedAddIndex() {
         }
 
         logger.info("Reroute, nothing should change");
-        routingResult = strategy.reroute(clusterState, "reroute");
-        assertFalse(routingResult.changed());
+        newState = strategy.reroute(clusterState, "reroute");
+        assertThat(newState, equalTo(clusterState));
 
         logger.info("Start the more shards");
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        newState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
         routingNodes = clusterState.getRoutingNodes();
-        assertTrue(routingResult.changed());
         assertThat(clusterState.routingTable().index("test1").shards().size(), equalTo(3));
         for (int i = 0; i < clusterState.routingTable().index("test1").shards().size(); i++) {
             assertThat(clusterState.routingTable().index("test1").shard(i).size(), equalTo(2));
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/MaxRetryAllocationDeciderTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/MaxRetryAllocationDeciderTests.java
index d84afc7f0b388..31e2330a600e8 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/MaxRetryAllocationDeciderTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/MaxRetryAllocationDeciderTests.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.EmptyClusterInfoService;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
@@ -34,8 +35,7 @@
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.cluster.ESAllocationTestCase;
-import org.elasticsearch.test.gateway.NoopGatewayAllocator;
+import org.elasticsearch.test.gateway.TestGatewayAllocator;
 
 import java.util.Collections;
 import java.util.List;
@@ -43,6 +43,8 @@
 import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.UNASSIGNED;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
 
 public class MaxRetryAllocationDeciderTests extends ESAllocationTestCase {
 
@@ -53,7 +55,7 @@ public void setUp() throws Exception {
         super.setUp();
         strategy = new AllocationService(Settings.builder().build(), new AllocationDeciders(Settings.EMPTY,
             Collections.singleton(new MaxRetryAllocationDecider(Settings.EMPTY))),
-            NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
+            new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
     }
 
     private ClusterState createInitialClusterState() {
@@ -86,34 +88,37 @@ public void testSingleRetryOnIgnore() {
         final int retries = MaxRetryAllocationDecider.SETTING_ALLOCATION_MAX_RETRY.get(Settings.EMPTY);
         // now fail it N-1 times
         for (int i = 0; i < retries-1; i++) {
-            List<FailedRerouteAllocation.FailedShard> failedShards = Collections.singletonList(
-                new FailedRerouteAllocation.FailedShard(routingTable.index("idx").shard(0).shards().get(0), "boom" + i,
+            List<FailedShard> failedShards = Collections.singletonList(
+                new FailedShard(routingTable.index("idx").shard(0).shards().get(0), "boom" + i,
                     new UnsupportedOperationException()));
-            RoutingAllocation.Result result = strategy.applyFailedShards(clusterState, failedShards);
-            assertTrue(result.changed());
-            routingTable = result.routingTable();
-            clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
+            ClusterState newState = strategy.applyFailedShards(clusterState, failedShards);
+            assertThat(newState, not(equalTo(clusterState)));
+            clusterState = newState;
+            routingTable = newState.routingTable();
             assertEquals(routingTable.index("idx").shards().size(), 1);
             assertEquals(routingTable.index("idx").shard(0).shards().get(0).state(), INITIALIZING);
             assertEquals(routingTable.index("idx").shard(0).shards().get(0).unassignedInfo().getNumFailedAllocations(), i+1);
             assertEquals(routingTable.index("idx").shard(0).shards().get(0).unassignedInfo().getMessage(), "boom" + i);
         }
         // now we go and check that we are actually stick to unassigned on the next failure
-        List<FailedRerouteAllocation.FailedShard> failedShards = Collections.singletonList(
-            new FailedRerouteAllocation.FailedShard(routingTable.index("idx").shard(0).shards().get(0), "boom",
+        List<FailedShard> failedShards = Collections.singletonList(
+            new FailedShard(routingTable.index("idx").shard(0).shards().get(0), "boom",
                 new UnsupportedOperationException()));
-        RoutingAllocation.Result result = strategy.applyFailedShards(clusterState, failedShards);
-        assertTrue(result.changed());
-        routingTable = result.routingTable();
-        clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
+        ClusterState newState = strategy.applyFailedShards(clusterState, failedShards);
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
+        routingTable = newState.routingTable();
         assertEquals(routingTable.index("idx").shards().size(), 1);
         assertEquals(routingTable.index("idx").shard(0).shards().get(0).unassignedInfo().getNumFailedAllocations(), retries);
         assertEquals(routingTable.index("idx").shard(0).shards().get(0).state(), UNASSIGNED);
         assertEquals(routingTable.index("idx").shard(0).shards().get(0).unassignedInfo().getMessage(), "boom");
 
-        result = strategy.reroute(clusterState, new AllocationCommands(), false, true); // manual reroute should retry once
-        assertTrue(result.changed());
-        routingTable = result.routingTable();
+        // manual reroute should retry once
+        newState = strategy.reroute(clusterState, new AllocationCommands(), false, true).getClusterState();
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
+        routingTable = newState.routingTable();
+
         clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
         assertEquals(routingTable.index("idx").shards().size(), 1);
         assertEquals(routingTable.index("idx").shard(0).shards().get(0).unassignedInfo().getNumFailedAllocations(), retries);
@@ -122,12 +127,13 @@ public void testSingleRetryOnIgnore() {
 
         // now we go and check that we are actually stick to unassigned on the next failure ie. no retry
         failedShards = Collections.singletonList(
-            new FailedRerouteAllocation.FailedShard(routingTable.index("idx").shard(0).shards().get(0), "boom",
+            new FailedShard(routingTable.index("idx").shard(0).shards().get(0), "boom",
                 new UnsupportedOperationException()));
-        result = strategy.applyFailedShards(clusterState, failedShards);
-        assertTrue(result.changed());
-        routingTable = result.routingTable();
-        clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
+
+        newState = strategy.applyFailedShards(clusterState, failedShards);
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
+        routingTable = newState.routingTable();
         assertEquals(routingTable.index("idx").shards().size(), 1);
         assertEquals(routingTable.index("idx").shard(0).shards().get(0).unassignedInfo().getNumFailedAllocations(), retries+1);
         assertEquals(routingTable.index("idx").shard(0).shards().get(0).state(), UNASSIGNED);
@@ -141,13 +147,13 @@ public void testFailedAllocation() {
         final int retries = MaxRetryAllocationDecider.SETTING_ALLOCATION_MAX_RETRY.get(Settings.EMPTY);
         // now fail it N-1 times
         for (int i = 0; i < retries-1; i++) {
-            List<FailedRerouteAllocation.FailedShard> failedShards = Collections.singletonList(
-                new FailedRerouteAllocation.FailedShard(routingTable.index("idx").shard(0).shards().get(0), "boom" + i,
+            List<FailedShard> failedShards = Collections.singletonList(
+                new FailedShard(routingTable.index("idx").shard(0).shards().get(0), "boom" + i,
                     new UnsupportedOperationException()));
-            RoutingAllocation.Result result = strategy.applyFailedShards(clusterState, failedShards);
-            assertTrue(result.changed());
-            routingTable = result.routingTable();
-            clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
+            ClusterState newState = strategy.applyFailedShards(clusterState, failedShards);
+            assertThat(newState, not(equalTo(clusterState)));
+            clusterState = newState;
+            routingTable = newState.routingTable();
             assertEquals(routingTable.index("idx").shards().size(), 1);
             ShardRouting unassignedPrimary = routingTable.index("idx").shard(0).shards().get(0);
             assertEquals(unassignedPrimary.state(), INITIALIZING);
@@ -159,13 +165,13 @@ public void testFailedAllocation() {
         }
         // now we go and check that we are actually stick to unassigned on the next failure
         {
-            List<FailedRerouteAllocation.FailedShard> failedShards = Collections.singletonList(
-                new FailedRerouteAllocation.FailedShard(routingTable.index("idx").shard(0).shards().get(0), "boom",
+            List<FailedShard> failedShards = Collections.singletonList(
+                new FailedShard(routingTable.index("idx").shard(0).shards().get(0), "boom",
                     new UnsupportedOperationException()));
-            RoutingAllocation.Result result = strategy.applyFailedShards(clusterState, failedShards);
-            assertTrue(result.changed());
-            routingTable = result.routingTable();
-            clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
+            ClusterState newState = strategy.applyFailedShards(clusterState, failedShards);
+            assertThat(newState, not(equalTo(clusterState)));
+            clusterState = newState;
+            routingTable = newState.routingTable();
             assertEquals(routingTable.index("idx").shards().size(), 1);
             ShardRouting unassignedPrimary = routingTable.index("idx").shard(0).shards().get(0);
             assertEquals(unassignedPrimary.unassignedInfo().getNumFailedAllocations(), retries);
@@ -183,10 +189,10 @@ public void testFailedAllocation() {
                     Settings.builder().put(clusterState.metaData().index("idx").getSettings()).put("index.allocation.max_retries",
                         retries+1).build()
                 ).build(), true).build()).build();
-        RoutingAllocation.Result result = strategy.reroute(clusterState, "settings changed", false);
-        assertTrue(result.changed());
-        routingTable = result.routingTable();
-        clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
+        ClusterState newState = strategy.reroute(clusterState, "settings changed", false);
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
+        routingTable = newState.routingTable();
         // good we are initializing and we are maintaining failure information
         assertEquals(routingTable.index("idx").shards().size(), 1);
         ShardRouting unassignedPrimary = routingTable.index("idx").shard(0).shards().get(0);
@@ -198,9 +204,9 @@ public void testFailedAllocation() {
             routingTable.index("idx").shard(0).shards().get(0), null, new RoutingAllocation(null, null, clusterState, null, 0, false)));
 
         // now we start the shard
-        routingTable = strategy.applyStartedShards(clusterState, Collections.singletonList(
-            routingTable.index("idx").shard(0).shards().get(0))).routingTable();
-        clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
+        clusterState = strategy.applyStartedShards(clusterState, Collections.singletonList(
+            routingTable.index("idx").shard(0).shards().get(0)));
+        routingTable = clusterState.routingTable();
 
         // all counters have been reset to 0 ie. no unassigned info
         assertEquals(routingTable.index("idx").shards().size(), 1);
@@ -208,17 +214,17 @@ public void testFailedAllocation() {
         assertEquals(routingTable.index("idx").shard(0).shards().get(0).state(), STARTED);
 
         // now fail again and see if it has a new counter
-        List<FailedRerouteAllocation.FailedShard> failedShards = Collections.singletonList(
-            new FailedRerouteAllocation.FailedShard(routingTable.index("idx").shard(0).shards().get(0), "ZOOOMG",
+        List<FailedShard> failedShards = Collections.singletonList(
+            new FailedShard(routingTable.index("idx").shard(0).shards().get(0), "ZOOOMG",
                 new UnsupportedOperationException()));
-        result = strategy.applyFailedShards(clusterState, failedShards);
-        assertTrue(result.changed());
-        routingTable = result.routingTable();
-        clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
+        newState = strategy.applyFailedShards(clusterState, failedShards);
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
+        routingTable = newState.routingTable();
         assertEquals(routingTable.index("idx").shards().size(), 1);
         unassignedPrimary = routingTable.index("idx").shard(0).shards().get(0);
         assertEquals(unassignedPrimary.unassignedInfo().getNumFailedAllocations(), 1);
-        assertEquals(unassignedPrimary.state(), INITIALIZING);
+        assertEquals(unassignedPrimary.state(), UNASSIGNED);
         assertEquals(unassignedPrimary.unassignedInfo().getMessage(), "ZOOOMG");
         // Counter reset, so MaxRetryAllocationDecider#canForceAllocatePrimary should return a YES decision
         assertEquals(Decision.YES, new MaxRetryAllocationDecider(Settings.EMPTY).canForceAllocatePrimary(
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/MoveDecisionTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/MoveDecisionTests.java
new file mode 100644
index 0000000000000..06e574ae56c1b
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/MoveDecisionTests.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.routing.allocation;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.allocation.decider.Decision;
+import org.elasticsearch.cluster.routing.allocation.decider.Decision.Type;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.transport.LocalTransportAddress;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.emptySet;
+
+/**
+ * Unit tests for the {@link MoveDecision} class.
+ */
+public class MoveDecisionTests extends ESTestCase {
+
+    public void testCachedDecisions() {
+        // cached stay decision
+        MoveDecision stay1 = MoveDecision.stay(null);
+        MoveDecision stay2 = MoveDecision.stay(null);
+        assertSame(stay1, stay2); // not in explain mode, so should use cached decision
+        stay1 = MoveDecision.stay(Decision.YES);
+        stay2 = MoveDecision.stay(Decision.YES);
+        assertNotSame(stay1, stay2);
+
+        // cached cannot move decision
+        stay1 = MoveDecision.cannotRemain(Decision.NO, AllocationDecision.NO, null, null);
+        stay2 = MoveDecision.cannotRemain(Decision.NO, AllocationDecision.NO, null, null);
+        assertSame(stay1, stay2);
+        // final decision is YES, so shouldn't use cached decision
+        DiscoveryNode node1 = new DiscoveryNode("node1", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
+        stay1 = MoveDecision.cannotRemain(Decision.NO, AllocationDecision.YES, node1, null);
+        stay2 = MoveDecision.cannotRemain(Decision.NO, AllocationDecision.YES, node1, null);
+        assertNotSame(stay1, stay2);
+        assertEquals(stay1.getTargetNode(), stay2.getTargetNode());
+        // final decision is NO, but in explain mode, so shouldn't use cached decision
+        stay1 = MoveDecision.cannotRemain(Decision.NO, AllocationDecision.NO, null, new ArrayList<>());
+        stay2 = MoveDecision.cannotRemain(Decision.NO, AllocationDecision.NO, null, new ArrayList<>());
+        assertNotSame(stay1, stay2);
+        assertSame(stay1.getAllocationDecision(), stay2.getAllocationDecision());
+        assertNotNull(stay1.getExplanation());
+        assertEquals(stay1.getExplanation(), stay2.getExplanation());
+    }
+
+    public void testStayDecision() {
+        MoveDecision stay = MoveDecision.stay(Decision.YES);
+        assertTrue(stay.canRemain());
+        assertFalse(stay.forceMove());
+        assertTrue(stay.isDecisionTaken());
+        assertNull(stay.getNodeDecisions());
+        assertEquals(AllocationDecision.NO_ATTEMPT, stay.getAllocationDecision());
+
+        stay = MoveDecision.stay(Decision.YES);
+        assertTrue(stay.canRemain());
+        assertFalse(stay.forceMove());
+        assertTrue(stay.isDecisionTaken());
+        assertNull(stay.getNodeDecisions());
+        assertEquals(AllocationDecision.NO_ATTEMPT, stay.getAllocationDecision());
+    }
+
+    public void testDecisionWithNodeExplanations() {
+        DiscoveryNode node1 = new DiscoveryNode("node1", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
+        DiscoveryNode node2 = new DiscoveryNode("node2", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
+        Decision nodeDecision = randomFrom(Decision.NO, Decision.THROTTLE, Decision.YES);
+        List<NodeAllocationResult> nodeDecisions = new ArrayList<>();
+        nodeDecisions.add(new NodeAllocationResult(node1, nodeDecision, 2));
+        nodeDecisions.add(new NodeAllocationResult(node2, nodeDecision, 1));
+        MoveDecision decision = MoveDecision.cannotRemain(Decision.NO, AllocationDecision.NO, null, nodeDecisions);
+        assertNotNull(decision.getAllocationDecision());
+        assertNotNull(decision.getExplanation());
+        assertNotNull(decision.getNodeDecisions());
+        assertEquals(2, decision.getNodeDecisions().size());
+        // both nodes have the same decision type but node2 has a higher weight ranking, so node2 comes first
+        assertEquals("node2", decision.getNodeDecisions().iterator().next().getNode().getId());
+
+        decision = MoveDecision.cannotRemain(Decision.NO, AllocationDecision.YES, node2, null);
+        assertEquals("node2", decision.getTargetNode().getId());
+    }
+
+    public void testSerialization() throws IOException {
+        List<NodeAllocationResult> nodeDecisions = new ArrayList<>();
+        DiscoveryNode node1 = new DiscoveryNode("node1", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
+        DiscoveryNode node2 = new DiscoveryNode("node2", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
+        Type finalDecision = randomFrom(Type.values());
+        DiscoveryNode assignedNode = finalDecision == Type.YES ? node1 : null;
+        nodeDecisions.add(new NodeAllocationResult(node1, Decision.NO, 2));
+        nodeDecisions.add(new NodeAllocationResult(node2, finalDecision == Type.YES ? Decision.YES :
+                                                              randomFrom(Decision.NO, Decision.THROTTLE, Decision.YES), 1));
+        MoveDecision moveDecision = MoveDecision.cannotRemain(Decision.NO, AllocationDecision.fromDecisionType(finalDecision),
+            assignedNode, nodeDecisions);
+        BytesStreamOutput output = new BytesStreamOutput();
+        moveDecision.writeTo(output);
+        MoveDecision readDecision = new MoveDecision(output.bytes().streamInput());
+        assertEquals(moveDecision.canRemain(), readDecision.canRemain());
+        assertEquals(moveDecision.getExplanation(), readDecision.getExplanation());
+        assertEquals(moveDecision.forceMove(), readDecision.forceMove());
+        assertEquals(moveDecision.getNodeDecisions().size(), readDecision.getNodeDecisions().size());
+        assertEquals(moveDecision.getTargetNode(), readDecision.getTargetNode());
+        assertEquals(moveDecision.getAllocationDecision(), readDecision.getAllocationDecision());
+        // node2 should have the highest sort order
+        assertEquals("node2", readDecision.getNodeDecisions().iterator().next().getNode().getId());
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeAllocationResultTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeAllocationResultTests.java
new file mode 100644
index 0000000000000..c149aba6e4367
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeAllocationResultTests.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.routing.allocation;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.allocation.NodeAllocationResult.ShardStoreInfo;
+import org.elasticsearch.cluster.routing.allocation.decider.Decision;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.transport.LocalTransportAddress;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.emptySet;
+
+/**
+ * Unit tests for the {@link NodeAllocationResult} class.
+ */
+public class NodeAllocationResultTests extends ESTestCase {
+
+    public void testSerialization() throws IOException {
+        DiscoveryNode node = new DiscoveryNode("node1", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
+        Decision decision = randomFrom(Decision.YES, Decision.THROTTLE, Decision.NO);
+        NodeAllocationResult explanation = new NodeAllocationResult(node, decision, 1);
+        BytesStreamOutput output = new BytesStreamOutput();
+        explanation.writeTo(output);
+        NodeAllocationResult readExplanation = new NodeAllocationResult(output.bytes().streamInput());
+        assertNodeExplanationEquals(explanation, readExplanation);
+    }
+
+    public void testShardStore() throws IOException {
+        DiscoveryNode node = new DiscoveryNode("node1", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
+        Decision decision = randomFrom(Decision.YES, Decision.THROTTLE, Decision.NO);
+        long matchingBytes = (long) randomIntBetween(1, 1000);
+        ShardStoreInfo shardStoreInfo = new ShardStoreInfo(matchingBytes);
+        NodeAllocationResult explanation = new NodeAllocationResult(node, shardStoreInfo, decision);
+        BytesStreamOutput output = new BytesStreamOutput();
+        explanation.writeTo(output);
+        NodeAllocationResult readExplanation = new NodeAllocationResult(output.bytes().streamInput());
+        assertNodeExplanationEquals(explanation, readExplanation);
+        assertEquals(matchingBytes, explanation.getShardStoreInfo().getMatchingBytes());
+        assertNull(explanation.getShardStoreInfo().getAllocationId());
+        assertFalse(explanation.getShardStoreInfo().isInSync());
+        assertFalse(explanation.getShardStoreInfo().hasMatchingSyncId());
+
+        String allocId = randomAlphaOfLength(5);
+        boolean inSync = randomBoolean();
+        shardStoreInfo = new ShardStoreInfo(allocId, inSync, randomBoolean() ? new Exception("bad stuff") : null);
+        explanation = new NodeAllocationResult(node, shardStoreInfo, decision);
+        output = new BytesStreamOutput();
+        explanation.writeTo(output);
+        readExplanation = new NodeAllocationResult(output.bytes().streamInput());
+        assertNodeExplanationEquals(explanation, readExplanation);
+        assertEquals(inSync, explanation.getShardStoreInfo().isInSync());
+        assertEquals(-1, explanation.getShardStoreInfo().getMatchingBytes());
+        assertFalse(explanation.getShardStoreInfo().hasMatchingSyncId());
+        assertEquals(allocId, explanation.getShardStoreInfo().getAllocationId());
+    }
+
+    private void assertNodeExplanationEquals(NodeAllocationResult expl1, NodeAllocationResult expl2) {
+        assertEquals(expl1.getNode(), expl2.getNode());
+        assertEquals(expl1.getCanAllocateDecision(), expl2.getCanAllocateDecision());
+        assertEquals(0, Float.compare(expl1.getWeightRanking(), expl2.getWeightRanking()));
+        if (expl1.getShardStoreInfo() != null) {
+            assertEquals(expl1.getShardStoreInfo().isInSync(), expl2.getShardStoreInfo().isInSync());
+            assertEquals(expl1.getShardStoreInfo().getAllocationId(), expl2.getShardStoreInfo().getAllocationId());
+            assertEquals(expl1.getShardStoreInfo().getMatchingBytes(), expl2.getShardStoreInfo().getMatchingBytes());
+            assertEquals(expl1.getShardStoreInfo().hasMatchingSyncId(), expl2.getShardStoreInfo().hasMatchingSyncId());
+        } else {
+            assertNull(expl2.getShardStoreInfo());
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java
index eab7d469dad9c..e65eb8db0bf9c 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.EmptyClusterInfoService;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
@@ -39,7 +40,6 @@
 import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
 import org.elasticsearch.cluster.routing.allocation.command.AllocationCommands;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.NodeVersionAllocationDecider;
@@ -52,9 +52,8 @@
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.snapshots.SnapshotId;
-import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.test.VersionUtils;
-import org.elasticsearch.test.gateway.NoopGatewayAllocator;
+import org.elasticsearch.test.gateway.TestGatewayAllocator;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -68,7 +67,6 @@
 import static org.elasticsearch.cluster.routing.ShardRoutingState.UNASSIGNED;
 import static org.elasticsearch.test.VersionUtils.randomVersion;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
@@ -110,8 +108,7 @@ public void testDoNotAllocateFromPrimary() {
 
         logger.info("start two nodes and fully start the shards");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
             assertThat(clusterState.routingTable().index("test").shard(i).shards().size(), equalTo(3));
@@ -122,9 +119,7 @@ public void testDoNotAllocateFromPrimary() {
 
         logger.info("start all the primary shards, replicas will start initializing");
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
-        routingNodes = clusterState.getRoutingNodes();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
 
         for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
             assertThat(clusterState.routingTable().index("test").shard(i).shards().size(), equalTo(3));
@@ -134,9 +129,7 @@ public void testDoNotAllocateFromPrimary() {
         }
 
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
-        routingNodes = clusterState.getRoutingNodes();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
 
         for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
             assertThat(clusterState.routingTable().index("test").shard(i).shards().size(), equalTo(3));
@@ -148,9 +141,7 @@ public void testDoNotAllocateFromPrimary() {
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
                 .add(newNode("node3", VersionUtils.getPreviousVersion())))
                 .build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
-        routingNodes = clusterState.getRoutingNodes();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
             assertThat(clusterState.routingTable().index("test").shard(i).shards().size(), equalTo(3));
@@ -163,8 +154,7 @@ public void testDoNotAllocateFromPrimary() {
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
                 .add(newNode("node4")))
                 .build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
             assertThat(clusterState.routingTable().index("test").shard(i).shards().size(), equalTo(3));
@@ -174,8 +164,7 @@ public void testDoNotAllocateFromPrimary() {
         }
 
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
 
         for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
             assertThat(clusterState.routingTable().index("test").shard(i).shards().size(), equalTo(3));
@@ -337,12 +326,11 @@ public void testRebalanceDoesNotAllocatePrimaryAndReplicasOnDifferentVersionNode
         AllocationDeciders allocationDeciders = new AllocationDeciders(Settings.EMPTY, Collections.singleton(new NodeVersionAllocationDecider(Settings.EMPTY)));
         AllocationService strategy = new MockAllocationService(Settings.EMPTY,
             allocationDeciders,
-            NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
-        RoutingAllocation.Result result = strategy.reroute(state, new AllocationCommands(), true, false);
+            new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
+        state = strategy.reroute(state, new AllocationCommands(), true, false).getClusterState();
         // the two indices must stay as is, the replicas cannot move to oldNode2 because versions don't match
-        state = ClusterState.builder(state).routingResult(result).build();
-        assertThat(result.routingTable().index(shard2.getIndex()).shardsWithState(ShardRoutingState.RELOCATING).size(), equalTo(0));
-        assertThat(result.routingTable().index(shard1.getIndex()).shardsWithState(ShardRoutingState.RELOCATING).size(), equalTo(0));
+        assertThat(state.routingTable().index(shard2.getIndex()).shardsWithState(ShardRoutingState.RELOCATING).size(), equalTo(0));
+        assertThat(state.routingTable().index(shard1.getIndex()).shardsWithState(ShardRoutingState.RELOCATING).size(), equalTo(0));
     }
 
     public void testRestoreDoesNotAllocateSnapshotOnOlderNodes() {
@@ -354,10 +342,12 @@ public void testRestoreDoesNotAllocateSnapshotOnOlderNodes() {
                 MASTER_DATA_ROLES, VersionUtils.getPreviousVersion());
 
         int numberOfShards = randomIntBetween(1, 3);
-        MetaData metaData = MetaData.builder()
-            .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(numberOfShards).numberOfReplicas
-                (randomIntBetween(0, 3)))
-            .build();
+        final IndexMetaData.Builder indexMetaData = IndexMetaData.builder("test").settings(settings(Version.CURRENT))
+            .numberOfShards(numberOfShards).numberOfReplicas(randomIntBetween(0, 3));
+        for (int i = 0; i < numberOfShards; i++) {
+            indexMetaData.putInSyncAllocationIds(i, Collections.singleton("_test_"));
+        }
+        MetaData metaData = MetaData.builder().put(indexMetaData).build();
 
         ClusterState state = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metaData(metaData)
@@ -370,41 +360,37 @@ public void testRestoreDoesNotAllocateSnapshotOnOlderNodes() {
             new NodeVersionAllocationDecider(Settings.EMPTY)));
         AllocationService strategy = new MockAllocationService(Settings.EMPTY,
             allocationDeciders,
-            NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
-        RoutingAllocation.Result result = strategy.reroute(state, new AllocationCommands(), true, false);
+            new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
+        state = strategy.reroute(state, new AllocationCommands(), true, false).getClusterState();
 
         // Make sure that primary shards are only allocated on the new node
         for (int i = 0; i < numberOfShards; i++) {
-            assertEquals("newNode", result.routingTable().index("test").getShards().get(i).primaryShard().currentNodeId());
+            assertEquals("newNode", state.routingTable().index("test").getShards().get(i).primaryShard().currentNodeId());
         }
     }
 
     private ClusterState stabilize(ClusterState clusterState, AllocationService service) {
-        logger.trace("RoutingNodes: {}", clusterState.getRoutingNodes().prettyPrint());
+        logger.trace("RoutingNodes: {}", clusterState.getRoutingNodes());
 
-        RoutingAllocation.Result routingResult = service.deassociateDeadNodes(clusterState, true, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = service.deassociateDeadNodes(clusterState, true, "reroute");
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
         assertRecoveryNodeVersions(routingNodes);
 
         logger.info("complete rebalancing");
-        for (int i = 0; i < 1000; i++) {   // at most 200 iters - this should be enough for all tests
-            logger.trace("RoutingNodes: {}", clusterState.getRoutingNodes().prettyPrint());
-            routingResult = service.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-            clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        boolean changed;
+        do {
+            logger.trace("RoutingNodes: {}", clusterState.getRoutingNodes());
+            ClusterState newState = service.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
+            changed = newState.equals(clusterState) == false;
+            clusterState = newState;
             routingNodes = clusterState.getRoutingNodes();
-            if (routingResult.changed() == false) {
-                break;
-            }
             assertRecoveryNodeVersions(routingNodes);
-        }
-        logger.info("stabilized success [{}]", routingResult.changed() == false);
-        assertFalse(routingResult.changed());
+        } while (changed);
         return clusterState;
     }
 
     private void assertRecoveryNodeVersions(RoutingNodes routingNodes) {
-        logger.trace("RoutingNodes: {}", routingNodes.prettyPrint());
+        logger.trace("RoutingNodes: {}", routingNodes);
 
         List<ShardRouting> mutableShardRoutings = routingNodes.shardsWithState(ShardRoutingState.RELOCATING);
         for (ShardRouting r : mutableShardRoutings) {
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PreferLocalPrimariesToRelocatingPrimariesTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PreferLocalPrimariesToRelocatingPrimariesTests.java
index 838cd5777efb3..7e528e601d317 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PreferLocalPrimariesToRelocatingPrimariesTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PreferLocalPrimariesToRelocatingPrimariesTests.java
@@ -20,13 +20,13 @@
 
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.cluster.ESAllocationTestCase;
 
 import static java.util.Collections.singletonMap;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
@@ -38,6 +38,7 @@
 /**
  */
 public class PreferLocalPrimariesToRelocatingPrimariesTests extends ESAllocationTestCase {
+
     public void testPreferLocalPrimaryAllocationOverFiltered() {
         int concurrentRecoveries = randomIntBetween(1, 10);
         int primaryRecoveries = randomIntBetween(1, 10);
@@ -66,15 +67,13 @@ public void testPreferLocalPrimaryAllocationOverFiltered() {
 
         logger.info("adding two nodes and performing rerouting till all are allocated");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder()
-                .add(newNode("node1", singletonMap("tag1", "value1")))
-                .add(newNode("node2", singletonMap("tag1", "value2")))).build();
+                .add(newNode("node1")).add(newNode("node2"))).build();
+
+        clusterState = strategy.reroute(clusterState, "reroute");
 
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
 
         while (!clusterState.getRoutingNodes().shardsWithState(INITIALIZING).isEmpty()) {
-            routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-            clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+            clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
         }
 
         logger.info("remove one of the nodes and apply filter to move everything from another node");
@@ -83,17 +82,16 @@ public void testPreferLocalPrimaryAllocationOverFiltered() {
                 .put(IndexMetaData.builder(clusterState.metaData().index("test1")).settings(settings(Version.CURRENT)
                         .put("index.number_of_shards", numberOfShards)
                         .put("index.number_of_replicas", 0)
-                        .put("index.routing.allocation.exclude.tag1", "value2")
+                        .put("index.routing.allocation.exclude._name", "node2")
                         .build()))
                 .put(IndexMetaData.builder(clusterState.metaData().index("test2")).settings(settings(Version.CURRENT)
                         .put("index.number_of_shards", numberOfShards)
                         .put("index.number_of_replicas", 0)
-                        .put("index.routing.allocation.exclude.tag1", "value2")
+                        .put("index.routing.allocation.exclude._name", "node2")
                         .build()))
                 .build();
         clusterState = ClusterState.builder(clusterState).metaData(metaData).nodes(DiscoveryNodes.builder(clusterState.nodes()).remove("node1")).build();
-        routingResult = strategy.deassociateDeadNodes(clusterState, true, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.deassociateDeadNodes(clusterState, true, "reroute");
 
         logger.info("[{}] primaries should be still started but [{}] other primaries should be unassigned", numberOfShards, numberOfShards);
         assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(numberOfShards));
@@ -103,8 +101,7 @@ public void testPreferLocalPrimaryAllocationOverFiltered() {
         logger.info("start node back up");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
                 .add(newNode("node1", singletonMap("tag1", "value1")))).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         while (clusterState.getRoutingNodes().shardsWithState(STARTED).size() < totalNumberOfShards) {
             int localInitializations = 0;
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PreferPrimaryAllocationTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PreferPrimaryAllocationTests.java
index 7badada5d5879..1c209157b13d6 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PreferPrimaryAllocationTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PreferPrimaryAllocationTests.java
@@ -22,6 +22,7 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -29,7 +30,6 @@
 import org.elasticsearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.cluster.ESAllocationTestCase;
 
 import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
 import static org.hamcrest.Matchers.equalTo;
@@ -63,12 +63,10 @@ public void testPreferPrimaryAllocationOverReplicas() {
 
         logger.info("adding two nodes and performing rerouting till all are allocated");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         while (!clusterState.getRoutingNodes().shardsWithState(INITIALIZING).isEmpty()) {
-            routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-            clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+            clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
         }
 
         logger.info("increasing the number of replicas to 1, and perform a reroute (to get the replicas allocation going)");
@@ -76,8 +74,7 @@ public void testPreferPrimaryAllocationOverReplicas() {
         metaData = MetaData.builder(clusterState.metaData()).updateNumberOfReplicas(1).build();
         clusterState = ClusterState.builder(clusterState).routingTable(updatedRoutingTable).metaData(metaData).build();
 
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         logger.info("2 replicas should be initializing now for the existing indices (we throttle to 1)");
         assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(2));
@@ -94,8 +91,7 @@ public void testPreferPrimaryAllocationOverReplicas() {
         clusterState = ClusterState.builder(clusterState).metaData(metaData).routingTable(updatedRoutingTable).build();
 
         logger.info("reroute, verify that primaries for the new index primary shards are allocated");
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         assertThat(clusterState.routingTable().index("new_index").shardsWithState(INITIALIZING).size(), equalTo(2));
     }
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryElectionRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryElectionRoutingTests.java
index ccdc4deb0278b..d789e6c4ec64e 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryElectionRoutingTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryElectionRoutingTests.java
@@ -22,6 +22,7 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -29,10 +30,10 @@
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.cluster.ESAllocationTestCase;
 
 import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED;
+import static org.elasticsearch.cluster.routing.ShardRoutingState.UNASSIGNED;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.nullValue;
 
@@ -59,28 +60,23 @@ public void testBackupElectionToPrimaryWhenPrimaryCanBeAllocatedToAnotherNode()
 
         logger.info("Adding two nodes and performing rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1"))).build();
-        RoutingAllocation.Result result = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(result).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node2"))).build();
-        result = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(result).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         logger.info("Start the primary shard (on node1)");
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
-        result = strategy.applyStartedShards(clusterState, routingNodes.node("node1").shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(result).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.node("node1").shardsWithState(INITIALIZING));
 
         logger.info("Start the backup shard (on node2)");
         routingNodes = clusterState.getRoutingNodes();
-        result = strategy.applyStartedShards(clusterState, routingNodes.node("node2").shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(result).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.node("node2").shardsWithState(INITIALIZING));
 
         logger.info("Adding third node and reroute and kill first node");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node3")).remove("node1")).build();
         RoutingTable prevRoutingTable = clusterState.routingTable();
-        result = strategy.deassociateDeadNodes(clusterState, true, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(result).build();
+        clusterState = strategy.deassociateDeadNodes(clusterState, true, "reroute");
         routingNodes = clusterState.getRoutingNodes();
         routingTable = clusterState.routingTable();
 
@@ -112,13 +108,11 @@ public void testRemovingInitializingReplicasIfPrimariesFails() {
 
         logger.info("Adding two nodes and performing rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))).build();
-        RoutingAllocation.Result rerouteResult = allocation.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.reroute(clusterState, "reroute");
 
         logger.info("Start the primary shards");
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
-        rerouteResult = allocation.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = allocation.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
         routingNodes = clusterState.getRoutingNodes();
 
         assertThat(routingNodes.shardsWithState(STARTED).size(), equalTo(2));
@@ -130,16 +124,14 @@ public void testRemovingInitializingReplicasIfPrimariesFails() {
         logger.info("--> fail node with primary");
         String nodeIdToFail = clusterState.routingTable().index("test").shard(0).primaryShard().currentNodeId();
         String nodeIdRemaining = nodeIdToFail.equals("node1") ? "node2" : "node1";
-        clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder()
-                .add(newNode(nodeIdRemaining))
-        ).build();
-        rerouteResult = allocation.deassociateDeadNodes(clusterState, true, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode(nodeIdRemaining))).build();
+        clusterState = allocation.deassociateDeadNodes(clusterState, true, "reroute");
         routingNodes = clusterState.getRoutingNodes();
 
         assertThat(routingNodes.shardsWithState(STARTED).size(), equalTo(1));
-        assertThat(routingNodes.shardsWithState(INITIALIZING).size(), equalTo(1));
-        assertThat(routingNodes.node(nodeIdRemaining).shardsWithState(INITIALIZING).get(0).primary(), equalTo(true));
+        assertThat(routingNodes.shardsWithState(INITIALIZING).size(), equalTo(0));
+        assertThat(routingNodes.shardsWithState(UNASSIGNED).size(), equalTo(3)); // 2 replicas and one primary
+        assertThat(routingNodes.node(nodeIdRemaining).shardsWithState(STARTED).get(0).primary(), equalTo(true));
         assertThat(clusterState.metaData().index("test").primaryTerm(0), equalTo(2L));
 
     }
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryNotRelocatedWhileBeingRecoveredTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryNotRelocatedWhileBeingRecoveredTests.java
index c1d4a168e868c..f2673805fa115 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryNotRelocatedWhileBeingRecoveredTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryNotRelocatedWhileBeingRecoveredTests.java
@@ -22,6 +22,7 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -29,7 +30,6 @@
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.cluster.ESAllocationTestCase;
 
 import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED;
@@ -62,28 +62,24 @@ public void testPrimaryNotRelocatedWhileBeingRecoveredFrom() {
 
         logger.info("Adding two nodes and performing rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1"))).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         logger.info("Start the primary shard (on node1)");
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.node("node1").shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.node("node1").shardsWithState(INITIALIZING));
 
         assertThat(clusterState.routingTable().shardsWithState(STARTED).size(), equalTo(5));
 
         logger.info("start another node, replica will start recovering form primary");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node2"))).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         assertThat(clusterState.routingTable().shardsWithState(STARTED).size(), equalTo(5));
         assertThat(clusterState.routingTable().shardsWithState(INITIALIZING).size(), equalTo(5));
 
         logger.info("start another node, make sure the primary is not relocated");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node3"))).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         assertThat(clusterState.routingTable().shardsWithState(STARTED).size(), equalTo(5));
         assertThat(clusterState.routingTable().shardsWithState(INITIALIZING).size(), equalTo(5));
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java
index ea618bb8b6da8..d226af26f8157 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java
@@ -21,6 +21,7 @@
 
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.EmptyClusterInfoService;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
@@ -36,12 +37,13 @@
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.cluster.routing.allocation.decider.ReplicaAfterPrimaryActiveAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.SameShardAllocationDecider;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.cluster.ESAllocationTestCase;
-import org.elasticsearch.test.gateway.NoopGatewayAllocator;
+import org.elasticsearch.test.gateway.TestGatewayAllocator;
 import org.hamcrest.Matchers;
 
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.Random;
 
@@ -57,8 +59,9 @@ public class RandomAllocationDeciderTests extends ESAllocationTestCase {
     public void testRandomDecisions() {
         RandomAllocationDecider randomAllocationDecider = new RandomAllocationDecider(random());
         AllocationService strategy = new AllocationService(Settings.builder().build(), new AllocationDeciders(Settings.EMPTY,
-                new HashSet<>(Arrays.asList(new SameShardAllocationDecider(Settings.EMPTY), new ReplicaAfterPrimaryActiveAllocationDecider(Settings.EMPTY),
-                        randomAllocationDecider))), NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
+                new HashSet<>(Arrays.asList(new SameShardAllocationDecider(Settings.EMPTY,
+                        new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)), new ReplicaAfterPrimaryActiveAllocationDecider(Settings.EMPTY),
+                    randomAllocationDecider))), new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
         int indices = scaledRandomIntBetween(1, 20);
         Builder metaBuilder = MetaData.builder();
         int maxNumReplicas = 1;
@@ -83,7 +86,7 @@ public void testRandomDecisions() {
         int nodeIdCounter = 0;
         int atMostNodes = scaledRandomIntBetween(Math.max(1, maxNumReplicas), 15);
         final boolean frequentNodes = randomBoolean();
-        RoutingAllocation.Result routingResult;
+        AllocationService.CommandsResult routingResult;
         for (int i = 0; i < numIters; i++) {
             logger.info("Start iteration [{}]", i);
             ClusterState.Builder stateBuilder = ClusterState.builder(clusterState);
@@ -101,22 +104,36 @@ public void testRandomDecisions() {
             boolean nodesRemoved = false;
             if (nodeIdCounter > 1 && rarely()) {
                 int nodeId = scaledRandomIntBetween(0, nodeIdCounter - 2);
-                logger.info("removing node [{}]", nodeId);
-                newNodesBuilder.remove("NODE_" + nodeId);
-                nodesRemoved = true;
+                final String node = "NODE_" + nodeId;
+                boolean safeToRemove = true;
+                RoutingNode routingNode = clusterState.getRoutingNodes().node(node);
+                for (ShardRouting shard: routingNode != null ? routingNode : Collections.<ShardRouting>emptyList()) {
+                    if (shard.active() && shard.primary()) {
+                        // make sure there is an active replica to prevent from going red
+                        if (clusterState.routingTable().shardRoutingTable(shard.shardId()).activeShards().size() <= 1) {
+                            safeToRemove = false;
+                            break;
+                        }
+                    }
+                }
+                if (safeToRemove) {
+                    logger.info("removing node [{}]", nodeId);
+                    newNodesBuilder.remove(node);
+                    nodesRemoved = true;
+                } else {
+                    logger.debug("not removing node [{}] as it holds a primary with no replacement", nodeId);
+                }
             }
 
             stateBuilder.nodes(newNodesBuilder.build());
             clusterState = stateBuilder.build();
             if (nodesRemoved) {
-                routingResult = strategy.deassociateDeadNodes(clusterState, true, "reroute");
+                clusterState = strategy.deassociateDeadNodes(clusterState, true, "reroute");
             } else {
-                routingResult = strategy.reroute(clusterState, "reroute");
+                clusterState = strategy.reroute(clusterState, "reroute");
             }
-            clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
             if (clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size() > 0) {
-                routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-                clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+                clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
             }
         }
         logger.info("Fill up nodes such that every shard can be allocated");
@@ -137,16 +154,14 @@ public void testRandomDecisions() {
         int iterations = 0;
         do {
             iterations++;
-            routingResult = strategy.reroute(clusterState, "reroute");
-            clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+            clusterState = strategy.reroute(clusterState, "reroute");
             if (clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size() > 0) {
-                routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-                clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+                clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
             }
 
         } while (clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size() != 0 ||
                 clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.UNASSIGNED).size() != 0 && iterations < 200);
-        logger.info("Done Balancing after [{}] iterations", iterations);
+        logger.info("Done Balancing after [{}] iterations. State:\n{}", iterations, clusterState);
         // we stop after 200 iterations if it didn't stabelize by then something is likely to be wrong
         assertThat("max num iteration exceeded", iterations, Matchers.lessThan(200));
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(0));
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RebalanceAfterActiveTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RebalanceAfterActiveTests.java
index a6929b651eb8f..ff2020d684435 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RebalanceAfterActiveTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RebalanceAfterActiveTests.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.cluster.ClusterInfo;
 import org.elasticsearch.cluster.ClusterInfoService;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -34,7 +35,6 @@
 import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.cluster.ESAllocationTestCase;
 
 import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.RELOCATING;
@@ -100,8 +100,7 @@ public void addListener(Listener listener) {
 
         logger.info("start two nodes and fully start the shards");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
             assertThat(clusterState.routingTable().index("test").shard(i).shards().size(), equalTo(2));
@@ -111,8 +110,7 @@ public void addListener(Listener listener) {
 
         logger.info("start all the primary shards, replicas will start initializing");
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
         routingNodes = clusterState.getRoutingNodes();
 
         for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
@@ -126,8 +124,7 @@ public void addListener(Listener listener) {
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
                 .add(newNode("node3")).add(newNode("node4")).add(newNode("node5")).add(newNode("node6")).add(newNode("node7")).add(newNode("node8")).add(newNode("node9")).add(newNode("node10")))
                 .build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
         routingNodes = clusterState.getRoutingNodes();
 
         for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
@@ -140,8 +137,7 @@ public void addListener(Listener listener) {
 
         logger.info("start the replica shards, rebalancing should start");
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
         routingNodes = clusterState.getRoutingNodes();
 
         // we only allow one relocation at a time
@@ -160,8 +156,7 @@ public void addListener(Listener listener) {
 
         logger.info("complete relocation, other half of relocation should happen");
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
         routingNodes = clusterState.getRoutingNodes();
 
         // we now only relocate 3, since 2 remain where they are!
@@ -178,8 +173,7 @@ public void addListener(Listener listener) {
 
         logger.info("complete relocation, that's it!");
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
         routingNodes = clusterState.getRoutingNodes();
 
         assertThat(clusterState.routingTable().shardsWithState(STARTED).size(), equalTo(10));
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RoutingNodesIntegrityTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RoutingNodesIntegrityTests.java
index c5998b0b73a67..6c837ed2b2038 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RoutingNodesIntegrityTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RoutingNodesIntegrityTests.java
@@ -22,6 +22,7 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -32,11 +33,11 @@
 import org.elasticsearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.cluster.ESAllocationTestCase;
 
 import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
 
 /**
  *
@@ -71,8 +72,7 @@ public void testBalanceAllNodesStarted() {
         assertThat(routingNodes.hasInactivePrimaries(), equalTo(false));
         assertThat(routingNodes.hasUnassignedPrimaries(), equalTo(true));
 
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
         routingNodes = clusterState.getRoutingNodes();
 
         assertThat(assertShardStats(routingNodes), equalTo(true));
@@ -82,21 +82,18 @@ public void testBalanceAllNodesStarted() {
 
         logger.info("Another round of rebalancing");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
 
         logger.info("Reroute, nothing should change");
-        routingResult = strategy.reroute(clusterState, "reroute");
-        assertFalse(routingResult.changed());
+        ClusterState newState = strategy.reroute(clusterState, "reroute");
+        assertThat(newState, equalTo(clusterState));
 
         logger.info("Start the more shards");
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
         routingNodes = clusterState.getRoutingNodes();
 
         assertThat(assertShardStats(routingNodes), equalTo(true));
@@ -104,8 +101,7 @@ public void testBalanceAllNodesStarted() {
         assertThat(routingNodes.hasInactivePrimaries(), equalTo(false));
         assertThat(routingNodes.hasUnassignedPrimaries(), equalTo(false));
 
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
 
     }
 
@@ -128,57 +124,51 @@ public void testBalanceIncrementallyStartNodes() {
         logger.info("Adding one node and performing rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1"))).build();
 
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         logger.info("Add another node and perform rerouting, nothing will happen since primary not started");
         clusterState = ClusterState.builder(clusterState)
                 .nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node2"))).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         logger.info("Start the primary shard");
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
 
         logger.info("Reroute, nothing should change");
-        routingResult = strategy.reroute(clusterState, "reroute");
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         logger.info("Start the backup shard");
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
         routingNodes = clusterState.getRoutingNodes();
 
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
         routingNodes = clusterState.getRoutingNodes();
 
         logger.info("Add another node and perform rerouting, nothing will happen since primary not started");
         clusterState = ClusterState.builder(clusterState)
                 .nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node3"))).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         logger.info("Reroute, nothing should change");
-        routingResult = strategy.reroute(clusterState, "reroute");
-        assertFalse(routingResult.changed());
+        ClusterState newState = strategy.reroute(clusterState, "reroute");
+        assertThat(newState, equalTo(clusterState));
 
         logger.info("Start the backup shard");
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        newState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
         routingNodes = clusterState.getRoutingNodes();
 
-        assertTrue(routingResult.changed());
         assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(3));
 
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        newState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
         routingNodes = clusterState.getRoutingNodes();
 
-        assertTrue(routingResult.changed());
         assertThat(clusterState.routingTable().index("test1").shards().size(), equalTo(3));
 
         assertThat(routingNodes.node("node1").numberOfShardsWithState(STARTED), equalTo(4));
@@ -220,8 +210,7 @@ public void testBalanceAllNodesStartedAddIndex() {
         assertThat(routingNodes.hasInactivePrimaries(), equalTo(false));
         assertThat(routingNodes.hasUnassignedPrimaries(), equalTo(true));
 
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
         routingNodes = clusterState.getRoutingNodes();
 
         assertThat(assertShardStats(routingNodes), equalTo(true));
@@ -231,18 +220,15 @@ public void testBalanceAllNodesStartedAddIndex() {
 
         logger.info("Another round of rebalancing");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
-
-        assertFalse(routingResult.changed());
+        ClusterState newState = strategy.reroute(clusterState, "reroute");
+        assertThat(newState, equalTo(clusterState));
 
         routingNodes = clusterState.getRoutingNodes();
         assertThat(routingNodes.node("node1").numberOfShardsWithState(INITIALIZING), equalTo(1));
         assertThat(routingNodes.node("node2").numberOfShardsWithState(INITIALIZING), equalTo(1));
         assertThat(routingNodes.node("node3").numberOfShardsWithState(INITIALIZING), equalTo(1));
 
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
         routingNodes = clusterState.getRoutingNodes();
 
         assertThat(assertShardStats(routingNodes), equalTo(true));
@@ -254,13 +240,12 @@ public void testBalanceAllNodesStartedAddIndex() {
         assertThat(routingNodes.node("node3").numberOfShardsWithState(STARTED), equalTo(1));
 
         logger.info("Reroute, nothing should change");
-        routingResult = strategy.reroute(clusterState, "reroute");
-        assertFalse(routingResult.changed());
+        newState = strategy.reroute(clusterState, "reroute");
+        assertThat(newState, equalTo(clusterState));
 
         logger.info("Start the more shards");
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
         routingNodes = clusterState.getRoutingNodes();
 
         assertThat(assertShardStats(routingNodes), equalTo(true));
@@ -297,13 +282,12 @@ public void testBalanceAllNodesStartedAddIndex() {
 
         assertThat(clusterState.routingTable().index("test1").shards().size(), equalTo(3));
 
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         logger.info("Reroute, assign");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        newState = strategy.reroute(clusterState, "reroute");
+        assertThat(newState, equalTo(clusterState));
         routingNodes = clusterState.getRoutingNodes();
 
         assertThat(assertShardStats(routingNodes), equalTo(true));
@@ -311,12 +295,10 @@ public void testBalanceAllNodesStartedAddIndex() {
         assertThat(routingNodes.hasInactivePrimaries(), equalTo(true));
         assertThat(routingNodes.hasUnassignedPrimaries(), equalTo(false));
 
-        assertFalse(routingResult.changed());
 
         logger.info("Reroute, start the primaries");
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
         routingNodes = clusterState.getRoutingNodes();
 
         assertThat(assertShardStats(routingNodes), equalTo(true));
@@ -326,8 +308,7 @@ public void testBalanceAllNodesStartedAddIndex() {
 
         logger.info("Reroute, start the replicas");
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
         routingNodes = clusterState.getRoutingNodes();
 
         assertThat(assertShardStats(routingNodes), equalTo(true));
@@ -347,8 +328,7 @@ public void testBalanceAllNodesStartedAddIndex() {
         logger.info("kill one node");
         IndexShardRoutingTable indexShardRoutingTable = clusterState.routingTable().index("test").shard(0);
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).remove(indexShardRoutingTable.primaryShard().currentNodeId())).build();
-        routingResult = strategy.deassociateDeadNodes(clusterState, true, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.deassociateDeadNodes(clusterState, true, "reroute");
         routingNodes = clusterState.getRoutingNodes();
 
         assertThat(assertShardStats(routingNodes), equalTo(true));
@@ -359,8 +339,7 @@ public void testBalanceAllNodesStartedAddIndex() {
 
         logger.info("Start Recovering shards round 1");
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
         routingNodes = clusterState.getRoutingNodes();
 
         assertThat(assertShardStats(routingNodes), equalTo(true));
@@ -370,8 +349,7 @@ public void testBalanceAllNodesStartedAddIndex() {
 
         logger.info("Start Recovering shards round 2");
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
         routingNodes = clusterState.getRoutingNodes();
 
         assertThat(assertShardStats(routingNodes), equalTo(true));
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/SameShardRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/SameShardRoutingTests.java
index 69c2efe9b0bb7..a5a88c1120cf5 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/SameShardRoutingTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/SameShardRoutingTests.java
@@ -21,20 +21,31 @@
 
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Version;
+import org.elasticsearch.action.support.replication.ClusterStateCreationUtils;
+import org.elasticsearch.cluster.ClusterInfo;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.RoutingNode;
+import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.TestShardRouting;
+import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.cluster.routing.allocation.decider.SameShardAllocationDecider;
 import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.LocalTransportAddress;
-import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.index.Index;
+
+import java.util.Collections;
 
 import static java.util.Collections.emptyMap;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
@@ -67,14 +78,12 @@ public void testSameHost() {
                         MASTER_DATA_ROLES, Version.CURRENT))
                 .add(new DiscoveryNode("node2", "node2", "node2", "test1", "test1", LocalTransportAddress.buildUnique(), emptyMap(),
                         MASTER_DATA_ROLES, Version.CURRENT))).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         assertThat(numberOfShardsOfType(clusterState.getRoutingNodes(), ShardRoutingState.INITIALIZING), equalTo(2));
 
         logger.info("--> start all primary shards, no replica will be started since its on the same host");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         assertThat(numberOfShardsOfType(clusterState.getRoutingNodes(), ShardRoutingState.STARTED), equalTo(2));
         assertThat(numberOfShardsOfType(clusterState.getRoutingNodes(), ShardRoutingState.INITIALIZING), equalTo(0));
@@ -83,8 +92,7 @@ public void testSameHost() {
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
                 .add(new DiscoveryNode("node3", "node3", "node3", "test2", "test2", LocalTransportAddress.buildUnique(), emptyMap(),
                         MASTER_DATA_ROLES, Version.CURRENT))).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         assertThat(numberOfShardsOfType(clusterState.getRoutingNodes(), ShardRoutingState.STARTED), equalTo(2));
         assertThat(numberOfShardsOfType(clusterState.getRoutingNodes(), ShardRoutingState.INITIALIZING), equalTo(2));
@@ -92,4 +100,32 @@ public void testSameHost() {
             assertThat(shardRouting.currentNodeId(), equalTo("node3"));
         }
     }
+
+    public void testForceAllocatePrimaryOnSameNodeNotAllowed() {
+        SameShardAllocationDecider decider = new SameShardAllocationDecider(
+            Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
+        ClusterState clusterState = ClusterStateCreationUtils.state("idx", randomIntBetween(2, 4), 1);
+        Index index = clusterState.getMetaData().index("idx").getIndex();
+        ShardRouting primaryShard = clusterState.routingTable().index(index).shard(0).primaryShard();
+        RoutingNode routingNode = clusterState.getRoutingNodes().node(primaryShard.currentNodeId());
+        RoutingAllocation routingAllocation = new RoutingAllocation(new AllocationDeciders(Settings.EMPTY, Collections.emptyList()),
+            new RoutingNodes(clusterState, false), clusterState, ClusterInfo.EMPTY, System.nanoTime(), false
+        );
+
+        // can't force allocate same shard copy to the same node
+        ShardRouting newPrimary = TestShardRouting.newShardRouting(primaryShard.shardId(), null, true, ShardRoutingState.UNASSIGNED);
+        Decision decision = decider.canForceAllocatePrimary(newPrimary, routingNode, routingAllocation);
+        assertEquals(Decision.Type.NO, decision.type());
+
+        // can force allocate to a different node
+        RoutingNode unassignedNode = null;
+        for (RoutingNode node : clusterState.getRoutingNodes()) {
+            if (node.isEmpty()) {
+                unassignedNode = node;
+                break;
+            }
+        }
+        decision = decider.canForceAllocatePrimary(newPrimary, unassignedNode, routingAllocation);
+        assertEquals(Decision.Type.YES, decision.type());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardStateIT.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardStateIT.java
index 7e05448bf91ab..7aeab5c940f18 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardStateIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardStateIT.java
@@ -21,6 +21,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.CheckedRunnable;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.indices.IndicesService;
@@ -50,7 +51,7 @@ public void testPrimaryFailureIncreasesTerm() throws Exception {
         logger.info("--> waiting for a yellow index");
         // JDK 9 type inference gets confused, so we have to help the
         // type inference
-        assertBusy(((Runnable) () -> assertThat(client().admin().cluster().prepareHealth().get().getStatus(),
+        assertBusy(((CheckedRunnable<Exception>) () -> assertThat(client().admin().cluster().prepareHealth().get().getStatus(),
                 equalTo(ClusterHealthStatus.YELLOW))));
 
         final long term0 = shard == 0 ? 2 : 1;
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardsLimitAllocationTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardsLimitAllocationTests.java
index 930c099f95625..85948f3c52cff 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardsLimitAllocationTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardsLimitAllocationTests.java
@@ -22,6 +22,7 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -32,7 +33,6 @@
 import org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.cluster.ESAllocationTestCase;
 
 import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.RELOCATING;
@@ -64,16 +64,14 @@ public void testIndexLevelShardsLimitAllocate() {
         ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
         logger.info("Adding two nodes and performing rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(ShardRoutingState.INITIALIZING), equalTo(2));
         assertThat(clusterState.getRoutingNodes().node("node2").numberOfShardsWithState(ShardRoutingState.INITIALIZING), equalTo(2));
 
         logger.info("Start the primary shards");
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
 
         assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(ShardRoutingState.STARTED), equalTo(2));
         assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(ShardRoutingState.INITIALIZING), equalTo(0));
@@ -83,8 +81,7 @@ public void testIndexLevelShardsLimitAllocate() {
 
         logger.info("Do another reroute, make sure its still not allocated");
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
     }
 
     public void testClusterLevelShardsLimitAllocate() {
@@ -108,16 +105,14 @@ public void testClusterLevelShardsLimitAllocate() {
         ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
         logger.info("Adding two nodes and performing rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(ShardRoutingState.INITIALIZING), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node2").numberOfShardsWithState(ShardRoutingState.INITIALIZING), equalTo(1));
 
         logger.info("Start the primary shards");
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
 
         assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(ShardRoutingState.STARTED), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node2").numberOfShardsWithState(ShardRoutingState.STARTED), equalTo(1));
@@ -130,15 +125,13 @@ public void testClusterLevelShardsLimitAllocate() {
                 .build());
 
         logger.info("Do another reroute, make sure shards are now allocated");
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(ShardRoutingState.INITIALIZING), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node2").numberOfShardsWithState(ShardRoutingState.INITIALIZING), equalTo(1));
 
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
 
         assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(ShardRoutingState.STARTED), equalTo(2));
         assertThat(clusterState.getRoutingNodes().node("node2").numberOfShardsWithState(ShardRoutingState.STARTED), equalTo(2));
@@ -171,13 +164,11 @@ public void testIndexLevelShardsLimitRemain() {
         ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(initialRoutingTable).build();
         logger.info("Adding one node and reroute");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1"))).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         logger.info("Start the primary shards");
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
 
         assertThat(numberOfShardsOfType(clusterState.getRoutingNodes(), STARTED), equalTo(5));
 
@@ -196,12 +187,10 @@ public void testIndexLevelShardsLimitRemain() {
 
         logger.info("Add another one node and reroute");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node2"))).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
 
         assertThat(numberOfShardsOfType(clusterState.getRoutingNodes(), STARTED), equalTo(10));
 
@@ -225,8 +214,7 @@ public void testIndexLevelShardsLimitRemain() {
         clusterState = ClusterState.builder(clusterState).metaData(metaData).build();
 
         logger.info("reroute after setting");
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(STARTED), equalTo(3));
         assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(RELOCATING), equalTo(2));
@@ -235,8 +223,7 @@ public void testIndexLevelShardsLimitRemain() {
         // the first move will destroy the balance and the balancer will move 2 shards from node2 to node one right after
         // moving the nodes to node2 since we consider INITIALIZING nodes during rebalance
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
         // now we are done compared to EvenShardCountAllocator since the Balancer is not soely based on the average
         assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(STARTED), equalTo(5));
         assertThat(clusterState.getRoutingNodes().node("node2").numberOfShardsWithState(STARTED), equalTo(5));
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/SingleShardNoReplicasRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/SingleShardNoReplicasRoutingTests.java
index c30ceef08b91b..dd89d6b6a529c 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/SingleShardNoReplicasRoutingTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/SingleShardNoReplicasRoutingTests.java
@@ -22,6 +22,7 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -33,7 +34,6 @@
 import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.cluster.ESAllocationTestCase;
 
 import java.util.ArrayList;
 import java.util.HashSet;
@@ -79,8 +79,7 @@ public void testSingleIndexStartedShard() {
 
         logger.info("Adding one node and performing rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1"))).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
         assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(1));
@@ -90,16 +89,16 @@ public void testSingleIndexStartedShard() {
 
         logger.info("Rerouting again, nothing should change");
         clusterState = ClusterState.builder(clusterState).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        assertThat(routingResult.changed(), equalTo(false));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        ClusterState newState = strategy.reroute(clusterState, "reroute");
+        assertThat(newState, equalTo(clusterState));
+        clusterState = newState;
 
         logger.info("Marking the shard as started");
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.node("node1").shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        newState = strategy.applyStartedShards(clusterState, routingNodes.node("node1").shardsWithState(INITIALIZING));
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
 
-        assertThat(routingResult.changed(), equalTo(true));
         assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
         assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(1));
         assertThat(clusterState.routingTable().index("test").shard(0).shards().size(), equalTo(1));
@@ -108,46 +107,59 @@ public void testSingleIndexStartedShard() {
 
         logger.info("Starting another node and making sure nothing changed");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node2"))).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        newState = strategy.reroute(clusterState, "reroute");
+        assertThat(newState, equalTo(clusterState));
+        clusterState = newState;
 
-        assertThat(routingResult.changed(), equalTo(false));
         assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
         assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(1));
         assertThat(clusterState.routingTable().index("test").shard(0).shards().size(), equalTo(1));
         assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).state(), equalTo(STARTED));
         assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).currentNodeId(), equalTo("node1"));
 
-        logger.info("Killing node1 where the shard is, checking the shard is relocated");
+        logger.info("Killing node1 where the shard is, checking the shard is unassigned");
 
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).remove("node1")).build();
-        routingResult = strategy.deassociateDeadNodes(clusterState, true, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        newState = strategy.deassociateDeadNodes(clusterState, true, "reroute");
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
+
+        assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
+        assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(1));
+        assertThat(clusterState.routingTable().index("test").shard(0).shards().size(), equalTo(1));
+        assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).state(), equalTo(UNASSIGNED));
+        assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).currentNodeId(), nullValue());
+
+        logger.info("Bring node1 back, and see it's assinged");
+
+        clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node1"))).build();
+        newState = strategy.reroute(clusterState, "reroute");
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
 
-        assertThat(routingResult.changed(), equalTo(true));
         assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
         assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(1));
         assertThat(clusterState.routingTable().index("test").shard(0).shards().size(), equalTo(1));
         assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).state(), equalTo(INITIALIZING));
-        assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).currentNodeId(), equalTo("node2"));
+        assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).currentNodeId(), equalTo("node1"));
+
 
         logger.info("Start another node, make sure that things remain the same (shard is in node2 and initializing)");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node3"))).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
-        assertThat(routingResult.changed(), equalTo(false));
+        newState = strategy.reroute(clusterState, "reroute");
+        assertThat(newState, equalTo(clusterState));
 
-        logger.info("Start the shard on node 2");
+        logger.info("Start the shard on node 1");
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.node("node2").shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        newState = strategy.applyStartedShards(clusterState, routingNodes.node("node1").shardsWithState(INITIALIZING));
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
 
-        assertThat(routingResult.changed(), equalTo(true));
         assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
         assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(1));
         assertThat(clusterState.routingTable().index("test").shard(0).shards().size(), equalTo(1));
         assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).state(), equalTo(STARTED));
-        assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).currentNodeId(), equalTo("node2"));
+        assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).currentNodeId(), equalTo("node1"));
     }
 
     public void testSingleIndexShardFailed() {
@@ -172,10 +184,10 @@ public void testSingleIndexShardFailed() {
 
         logger.info("Adding one node and rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1"))).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        ClusterState newState = strategy.reroute(clusterState, "reroute");
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
 
-        assertThat(routingResult.changed(), equalTo(true));
         assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
         assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(1));
         assertThat(clusterState.routingTable().index("test").shard(0).shards().size(), equalTo(1));
@@ -185,10 +197,10 @@ public void testSingleIndexShardFailed() {
 
         logger.info("Marking the shard as failed");
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyFailedShard(clusterState, routingNodes.node("node1").shardsWithState(INITIALIZING).get(0));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        newState = strategy.applyFailedShard(clusterState, routingNodes.node("node1").shardsWithState(INITIALIZING).get(0));
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
 
-        assertThat(routingResult.changed(), equalTo(true));
         assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
         assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(1));
         assertThat(clusterState.routingTable().index("test").shard(0).shards().size(), equalTo(1));
@@ -234,10 +246,10 @@ public void testMultiIndexEvenDistribution() {
             nodesBuilder.add(newNode("node" + i));
         }
         clusterState = ClusterState.builder(clusterState).nodes(nodesBuilder).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        ClusterState newState = strategy.reroute(clusterState, "reroute");
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
 
-        assertThat(routingResult.changed(), equalTo(true));
         for (int i = 0; i < numberOfIndices; i++) {
             assertThat(clusterState.routingTable().index("test" + i).shards().size(), equalTo(1));
             assertThat(clusterState.routingTable().index("test" + i).shard(0).size(), equalTo(1));
@@ -271,16 +283,14 @@ public void testMultiIndexEvenDistribution() {
             nodesBuilder.add(newNode("node" + i));
         }
         clusterState = ClusterState.builder(clusterState).nodes(nodesBuilder).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
-
-        assertThat(routingResult.changed(), equalTo(false));
+        newState = strategy.reroute(clusterState, "reroute");
+        assertThat(newState, equalTo(clusterState));
 
         logger.info("Marking the shard as started");
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        newState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
+        assertThat(newState, not(equalTo(clusterState)));
 
-        assertThat(routingResult.changed(), equalTo(true));
+        clusterState = newState;
         int numberOfRelocatingShards = 0;
         int numberOfStartedShards = 0;
         for (int i = 0; i < numberOfIndices; i++) {
@@ -333,10 +343,10 @@ public void testMultiIndexUnevenNodes() {
         clusterState = ClusterState.builder(clusterState)
                 .nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2")).add(newNode("node3")))
                 .build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        ClusterState newState = strategy.reroute(clusterState, "reroute");
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
 
-        assertThat(routingResult.changed(), equalTo(true));
         for (int i = 0; i < numberOfIndices; i++) {
             assertThat(clusterState.routingTable().index("test" + i).shards().size(), equalTo(1));
             assertThat(clusterState.routingTable().index("test" + i).shard(0).size(), equalTo(1));
@@ -353,15 +363,15 @@ public void testMultiIndexUnevenNodes() {
         clusterState = ClusterState.builder(clusterState)
                 .nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node4")).add(newNode("node5")))
                 .build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-
-        assertThat(routingResult.changed(), equalTo(false));
+        newState = strategy.reroute(clusterState, "reroute");
+        assertThat(newState, equalTo(clusterState));
+        clusterState = newState;
 
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        newState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
 
-        assertThat(routingResult.changed(), equalTo(true));
         for (int i = 0; i < numberOfIndices; i++) {
             assertThat(clusterState.routingTable().index("test" + i).shards().size(), equalTo(1));
             assertThat(clusterState.routingTable().index("test" + i).shard(0).size(), equalTo(1));
@@ -373,11 +383,11 @@ public void testMultiIndexUnevenNodes() {
         assertThat("4 target shard routing are initializing", numberOfShardsOfType(routingNodes, INITIALIZING), equalTo(4));
 
         logger.info("Now, mark the relocated as started");
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        newState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
 //        routingTable = strategy.reroute(new RoutingStrategyInfo(metaData, routingTable), nodes);
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
 
-        assertThat(routingResult.changed(), equalTo(true));
         for (int i = 0; i < numberOfIndices; i++) {
             assertThat(clusterState.routingTable().index("test" + i).shards().size(), equalTo(1));
             assertThat(clusterState.routingTable().index("test" + i).shard(0).size(), equalTo(1));
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/SingleShardOneReplicaRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/SingleShardOneReplicaRoutingTests.java
index cad5654eaa43e..0990850acae52 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/SingleShardOneReplicaRoutingTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/SingleShardOneReplicaRoutingTests.java
@@ -22,6 +22,7 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -29,12 +30,12 @@
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.cluster.ESAllocationTestCase;
 
 import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.UNASSIGNED;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.nullValue;
 
 /**
@@ -69,10 +70,10 @@ public void testSingleIndexFirstStartPrimaryThenBackups() {
         logger.info("Adding one node and performing rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1"))).build();
 
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        ClusterState newState = strategy.reroute(clusterState, "reroute");
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
 
-        assertTrue(routingResult.changed());
         assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
         assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(2));
         assertThat(clusterState.routingTable().index("test").shard(0).shards().size(), equalTo(2));
@@ -84,17 +85,15 @@ public void testSingleIndexFirstStartPrimaryThenBackups() {
 
         logger.info("Add another node and perform rerouting, nothing will happen since primary shards not started");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node2"))).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
-
-        assertFalse(routingResult.changed());
+        newState = strategy.reroute(clusterState, "reroute");
+        assertThat(newState, equalTo(clusterState));
 
         logger.info("Start the primary shard (on node1)");
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.node("node1").shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        newState = strategy.applyStartedShards(clusterState, routingNodes.node("node1").shardsWithState(INITIALIZING));
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
 
-        assertTrue(routingResult.changed());
         assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
         assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(2));
         assertThat(clusterState.routingTable().index("test").shard(0).shards().size(), equalTo(2));
@@ -107,15 +106,15 @@ public void testSingleIndexFirstStartPrimaryThenBackups() {
 
 
         logger.info("Reroute, nothing should change");
-        routingResult = strategy.reroute(clusterState, "reroute");
-        assertFalse(routingResult.changed());
+        newState = strategy.reroute(clusterState, "reroute");
+        assertThat(newState, equalTo(clusterState));
 
         logger.info("Start the backup shard");
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.node("node2").shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        newState = strategy.applyStartedShards(clusterState, routingNodes.node("node2").shardsWithState(INITIALIZING));
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
 
-        assertTrue(routingResult.changed());
         assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
         assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(2));
         assertThat(clusterState.routingTable().index("test").shard(0).shards().size(), equalTo(2));
@@ -128,10 +127,10 @@ public void testSingleIndexFirstStartPrimaryThenBackups() {
         logger.info("Kill node1, backup shard should become primary");
 
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).remove("node1")).build();
-        routingResult = strategy.deassociateDeadNodes(clusterState, true, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        newState = strategy.deassociateDeadNodes(clusterState, true, "reroute");
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
 
-        assertTrue(routingResult.changed());
         assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
         assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(2));
         assertThat(clusterState.routingTable().index("test").shard(0).shards().size(), equalTo(2));
@@ -145,10 +144,10 @@ public void testSingleIndexFirstStartPrimaryThenBackups() {
         logger.info("Start another node, backup shard should start initializing");
 
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node3"))).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        newState = strategy.reroute(clusterState, "reroute");
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
 
-        assertTrue(routingResult.changed());
         assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
         assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(2));
         assertThat(clusterState.routingTable().index("test").shard(0).shards().size(), equalTo(2));
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/StartedShardsRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/StartedShardsRoutingTests.java
index 76db92435fecf..ee9d69a220593 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/StartedShardsRoutingTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/StartedShardsRoutingTests.java
@@ -21,6 +21,7 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -32,14 +33,17 @@
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.cluster.ESAllocationTestCase;
 
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.List;
 
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.nullValue;
 
 public class StartedShardsRoutingTests extends ESAllocationTestCase {
@@ -68,18 +72,84 @@ public void testStartedShardsMatching() {
 
         logger.info("--> test starting of shard");
 
-        RoutingAllocation.Result result = allocation.applyStartedShards(state, Arrays.asList(initShard), false);
-        assertTrue("failed to start " + initShard + "\ncurrent routing table:" + result.routingTable().prettyPrint(), result.changed());
-        assertTrue(initShard + "isn't started \ncurrent routing table:" + result.routingTable().prettyPrint(),
-                result.routingTable().index("test").shard(initShard.id()).allShardsStarted());
-
+        ClusterState newState = allocation.applyStartedShards(state, Arrays.asList(initShard));
+        assertThat("failed to start " + initShard + "\ncurrent routing table:" + newState.routingTable(), newState, not(equalTo(state)));
+        assertTrue(initShard + "isn't started \ncurrent routing table:" + newState.routingTable(),
+                newState.routingTable().index("test").shard(initShard.id()).allShardsStarted());
+        state = newState;
 
         logger.info("--> testing starting of relocating shards");
-        result = allocation.applyStartedShards(state, Arrays.asList(relocatingShard.getTargetRelocatingShard()), false);
-        assertTrue("failed to start " + relocatingShard + "\ncurrent routing table:" + result.routingTable().prettyPrint(), result.changed());
-        ShardRouting shardRouting = result.routingTable().index("test").shard(relocatingShard.id()).getShards().get(0);
+        newState = allocation.applyStartedShards(state, Arrays.asList(relocatingShard.getTargetRelocatingShard()));
+        assertThat("failed to start " + relocatingShard + "\ncurrent routing table:" + newState.routingTable(),
+            newState, not(equalTo(state)));
+        ShardRouting shardRouting = newState.routingTable().index("test").shard(relocatingShard.id()).getShards().get(0);
         assertThat(shardRouting.state(), equalTo(ShardRoutingState.STARTED));
         assertThat(shardRouting.currentNodeId(), equalTo("node2"));
         assertThat(shardRouting.relocatingNodeId(), nullValue());
     }
+
+    public void testRelocatingPrimariesWithInitializingReplicas() {
+        AllocationService allocation = createAllocationService();
+
+        logger.info("--> building initial cluster state");
+        AllocationId primaryId = AllocationId.newRelocation(AllocationId.newInitializing());
+        AllocationId replicaId = AllocationId.newInitializing();
+        boolean relocatingReplica = randomBoolean();
+        if (relocatingReplica) {
+            replicaId = AllocationId.newRelocation(replicaId);
+        }
+
+        final IndexMetaData indexMetaData = IndexMetaData.builder("test")
+            .settings(settings(Version.CURRENT))
+            .numberOfShards(1).numberOfReplicas(1)
+            .putInSyncAllocationIds(0,
+                relocatingReplica ? Sets.newHashSet(primaryId.getId(), replicaId.getId()) : Sets.newHashSet(primaryId.getId()))
+            .build();
+        final Index index = indexMetaData.getIndex();
+        ClusterState.Builder stateBuilder = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
+            .nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2")).add(newNode("node3")).add(newNode("node4")))
+            .metaData(MetaData.builder().put(indexMetaData, false));
+
+        final ShardRouting relocatingPrimary = TestShardRouting.newShardRouting(
+            new ShardId(index, 0), "node1", "node2", true, ShardRoutingState.RELOCATING, primaryId);
+        final ShardRouting replica = TestShardRouting.newShardRouting(
+            new ShardId(index, 0), "node3", relocatingReplica ? "node4" : null, false,
+            relocatingReplica ? ShardRoutingState.RELOCATING : ShardRoutingState.INITIALIZING, replicaId);
+
+        stateBuilder.routingTable(RoutingTable.builder().add(IndexRoutingTable.builder(index)
+            .addIndexShard(new IndexShardRoutingTable.Builder(relocatingPrimary.shardId())
+                .addShard(relocatingPrimary)
+                .addShard(replica)
+                .build()))
+            .build());
+
+
+        ClusterState state = stateBuilder.build();
+
+        logger.info("--> test starting of relocating primary shard with initializing / relocating replica");
+        ClusterState newState = allocation.applyStartedShards(state, Arrays.asList(relocatingPrimary.getTargetRelocatingShard()));
+        assertNotEquals(newState, state);
+        assertTrue(newState.routingTable().index("test").allPrimaryShardsActive());
+        ShardRouting startedReplica = newState.routingTable().index("test").shard(0).replicaShards().get(0);
+        if (relocatingReplica) {
+            assertTrue(startedReplica.relocating());
+            assertEquals(replica.currentNodeId(), startedReplica.currentNodeId());
+            assertEquals(replica.relocatingNodeId(), startedReplica.relocatingNodeId());
+            assertEquals(replica.allocationId().getId(), startedReplica.allocationId().getId());
+            assertNotEquals(replica.allocationId().getRelocationId(), startedReplica.allocationId().getRelocationId());
+        } else {
+            assertTrue(startedReplica.initializing());
+            assertEquals(replica.currentNodeId(), startedReplica.currentNodeId());
+            assertNotEquals(replica.allocationId().getId(), startedReplica.allocationId().getId());
+        }
+
+        logger.info("--> test starting of relocating primary shard together with initializing / relocating replica");
+        List<ShardRouting> startedShards = new ArrayList<>();
+        startedShards.add(relocatingPrimary.getTargetRelocatingShard());
+        startedShards.add(relocatingReplica ? replica.getTargetRelocatingShard() : replica);
+        Collections.shuffle(startedShards, random());
+        newState = allocation.applyStartedShards(state, startedShards);
+        assertNotEquals(newState, state);
+        assertTrue(newState.routingTable().index("test").shard(0).allShardsStarted());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/TenShardsOneReplicaRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/TenShardsOneReplicaRoutingTests.java
index a6a7e6c06511d..5907232b5f282 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/TenShardsOneReplicaRoutingTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/TenShardsOneReplicaRoutingTests.java
@@ -22,6 +22,7 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -30,7 +31,6 @@
 import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.cluster.ESAllocationTestCase;
 
 import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.RELOCATING;
@@ -38,6 +38,7 @@
 import static org.elasticsearch.cluster.routing.ShardRoutingState.UNASSIGNED;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.lessThan;
+import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.nullValue;
 
 /**
@@ -82,10 +83,10 @@ public void testSingleIndexFirstStartPrimaryThenBackups() {
         logger.info("Adding one node and performing rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1"))).build();
 
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        ClusterState newState = strategy.reroute(clusterState, "reroute");
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
 
-        assertTrue(routingResult.changed());
         assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(10));
         for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
             assertThat(clusterState.routingTable().index("test").shard(i).size(), equalTo(2));
@@ -99,17 +100,15 @@ public void testSingleIndexFirstStartPrimaryThenBackups() {
 
         logger.info("Add another node and perform rerouting, nothing will happen since primary not started");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node2"))).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
-
-        assertFalse(routingResult.changed());
+        newState = strategy.reroute(clusterState, "reroute");
+        assertThat(newState, equalTo(clusterState));
 
         logger.info("Start the primary shard (on node1)");
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.node("node1").shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        newState = strategy.applyStartedShards(clusterState, routingNodes.node("node1").shardsWithState(INITIALIZING));
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
 
-        assertTrue(routingResult.changed());
         assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(10));
         for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
             assertThat(clusterState.routingTable().index("test").shard(i).size(), equalTo(2));
@@ -123,16 +122,16 @@ public void testSingleIndexFirstStartPrimaryThenBackups() {
         }
 
         logger.info("Reroute, nothing should change");
-        routingResult = strategy.reroute(clusterState, "reroute");
-        assertFalse(routingResult.changed());
+        newState = strategy.reroute(clusterState, "reroute");
+        assertThat(newState, equalTo(clusterState));
 
         logger.info("Start the backup shard");
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.node("node2").shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        newState = strategy.applyStartedShards(clusterState, routingNodes.node("node2").shardsWithState(INITIALIZING));
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
         routingNodes = clusterState.getRoutingNodes();
 
-        assertTrue(routingResult.changed());
         assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(10));
         for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
             assertThat(clusterState.routingTable().index("test").shard(i).size(), equalTo(2));
@@ -148,11 +147,11 @@ public void testSingleIndexFirstStartPrimaryThenBackups() {
 
         logger.info("Add another node and perform rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node3"))).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        newState = strategy.reroute(clusterState, "reroute");
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
         routingNodes = clusterState.getRoutingNodes();
 
-        assertTrue(routingResult.changed());
         assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(10));
         assertThat(routingNodes.node("node1").numberOfShardsWithState(STARTED, RELOCATING), equalTo(10));
         assertThat(routingNodes.node("node1").numberOfShardsWithState(STARTED), lessThan(10));
@@ -162,11 +161,11 @@ public void testSingleIndexFirstStartPrimaryThenBackups() {
 
         logger.info("Start the shards on node 3");
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.node("node3").shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        newState = strategy.applyStartedShards(clusterState, routingNodes.node("node3").shardsWithState(INITIALIZING));
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
         routingNodes = clusterState.getRoutingNodes();
 
-        assertTrue(routingResult.changed());
         assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(10));
         assertThat(routingNodes.node("node1").numberOfShardsWithState(STARTED), equalTo(7));
         assertThat(routingNodes.node("node2").numberOfShardsWithState(STARTED), equalTo(7));
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java
index 9083085b5c7c6..894b5b42f0cae 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java
@@ -20,23 +20,35 @@
 package org.elasticsearch.cluster.routing.allocation;
 
 import com.carrotsearch.hppc.IntHashSet;
+import com.carrotsearch.hppc.cursors.ObjectCursor;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.RecoverySource.SnapshotRecoverySource;
 import org.elasticsearch.cluster.routing.RoutingTable;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.ShardRoutingHelper;
+import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.routing.allocation.command.AllocationCommands;
 import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.snapshots.SnapshotId;
-import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.test.gateway.TestGatewayAllocator;
 
+import java.util.Collections;
+
+import static org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.RELOCATING;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED;
@@ -50,10 +62,12 @@ public class ThrottlingAllocationTests extends ESAllocationTestCase {
     private final Logger logger = Loggers.getLogger(ThrottlingAllocationTests.class);
 
     public void testPrimaryRecoveryThrottling() {
+
+        TestGatewayAllocator gatewayAllocator = new TestGatewayAllocator();
         AllocationService strategy = createAllocationService(Settings.builder()
                 .put("cluster.routing.allocation.node_concurrent_recoveries", 3)
                 .put("cluster.routing.allocation.node_initial_primaries_recoveries", 3)
-                .build());
+                .build(), gatewayAllocator);
 
         logger.info("Building initial routing table");
 
@@ -61,46 +75,39 @@ public void testPrimaryRecoveryThrottling() {
                 .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(10).numberOfReplicas(1))
                 .build();
 
-        RoutingTable initialRoutingTable = createRecoveryRoutingTable(metaData.index("test"));
-
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(initialRoutingTable).build();
+        ClusterState clusterState = createRecoveryStateAndInitalizeAllocations(metaData, gatewayAllocator);
 
         logger.info("start one node, do reroute, only 3 should initialize");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1"))).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         assertThat(clusterState.routingTable().shardsWithState(STARTED).size(), equalTo(0));
         assertThat(clusterState.routingTable().shardsWithState(INITIALIZING).size(), equalTo(3));
         assertThat(clusterState.routingTable().shardsWithState(UNASSIGNED).size(), equalTo(17));
 
         logger.info("start initializing, another 3 should initialize");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.routingTable().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.routingTable().shardsWithState(INITIALIZING));
 
         assertThat(clusterState.routingTable().shardsWithState(STARTED).size(), equalTo(3));
         assertThat(clusterState.routingTable().shardsWithState(INITIALIZING).size(), equalTo(3));
         assertThat(clusterState.routingTable().shardsWithState(UNASSIGNED).size(), equalTo(14));
 
         logger.info("start initializing, another 3 should initialize");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.routingTable().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.routingTable().shardsWithState(INITIALIZING));
 
         assertThat(clusterState.routingTable().shardsWithState(STARTED).size(), equalTo(6));
         assertThat(clusterState.routingTable().shardsWithState(INITIALIZING).size(), equalTo(3));
         assertThat(clusterState.routingTable().shardsWithState(UNASSIGNED).size(), equalTo(11));
 
         logger.info("start initializing, another 1 should initialize");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.routingTable().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.routingTable().shardsWithState(INITIALIZING));
 
         assertThat(clusterState.routingTable().shardsWithState(STARTED).size(), equalTo(9));
         assertThat(clusterState.routingTable().shardsWithState(INITIALIZING).size(), equalTo(1));
         assertThat(clusterState.routingTable().shardsWithState(UNASSIGNED).size(), equalTo(10));
 
         logger.info("start initializing, all primaries should be started");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.routingTable().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.routingTable().shardsWithState(INITIALIZING));
 
         assertThat(clusterState.routingTable().shardsWithState(STARTED).size(), equalTo(10));
         assertThat(clusterState.routingTable().shardsWithState(INITIALIZING).size(), equalTo(0));
@@ -108,11 +115,13 @@ public void testPrimaryRecoveryThrottling() {
     }
 
     public void testReplicaAndPrimaryRecoveryThrottling() {
+        TestGatewayAllocator gatewayAllocator = new TestGatewayAllocator();
         AllocationService strategy = createAllocationService(Settings.builder()
                 .put("cluster.routing.allocation.node_concurrent_recoveries", 3)
                 .put("cluster.routing.allocation.concurrent_source_recoveries", 3)
                 .put("cluster.routing.allocation.node_initial_primaries_recoveries", 3)
-                .build());
+                .build(),
+            gatewayAllocator);
 
         logger.info("Building initial routing table");
 
@@ -120,30 +129,24 @@ public void testReplicaAndPrimaryRecoveryThrottling() {
                 .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(5).numberOfReplicas(1))
                 .build();
 
-        RoutingTable initialRoutingTable = createRecoveryRoutingTable(metaData.index("test"));
-
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(initialRoutingTable).build();
+        ClusterState clusterState = createRecoveryStateAndInitalizeAllocations(metaData, gatewayAllocator);
 
-        logger.info("start one node, do reroute, only 3 should initialize");
-        clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1"))).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        logger.info("with one node, do reroute, only 3 should initialize");
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         assertThat(clusterState.routingTable().shardsWithState(STARTED).size(), equalTo(0));
         assertThat(clusterState.routingTable().shardsWithState(INITIALIZING).size(), equalTo(3));
         assertThat(clusterState.routingTable().shardsWithState(UNASSIGNED).size(), equalTo(7));
 
         logger.info("start initializing, another 2 should initialize");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.routingTable().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.routingTable().shardsWithState(INITIALIZING));
 
         assertThat(clusterState.routingTable().shardsWithState(STARTED).size(), equalTo(3));
         assertThat(clusterState.routingTable().shardsWithState(INITIALIZING).size(), equalTo(2));
         assertThat(clusterState.routingTable().shardsWithState(UNASSIGNED).size(), equalTo(5));
 
         logger.info("start initializing, all primaries should be started");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.routingTable().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.routingTable().shardsWithState(INITIALIZING));
 
         assertThat(clusterState.routingTable().shardsWithState(STARTED).size(), equalTo(5));
         assertThat(clusterState.routingTable().shardsWithState(INITIALIZING).size(), equalTo(0));
@@ -151,24 +154,21 @@ public void testReplicaAndPrimaryRecoveryThrottling() {
 
         logger.info("start another node, replicas should start being allocated");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node2"))).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         assertThat(clusterState.routingTable().shardsWithState(STARTED).size(), equalTo(5));
         assertThat(clusterState.routingTable().shardsWithState(INITIALIZING).size(), equalTo(3));
         assertThat(clusterState.routingTable().shardsWithState(UNASSIGNED).size(), equalTo(2));
 
         logger.info("start initializing replicas");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.routingTable().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.routingTable().shardsWithState(INITIALIZING));
 
         assertThat(clusterState.routingTable().shardsWithState(STARTED).size(), equalTo(8));
         assertThat(clusterState.routingTable().shardsWithState(INITIALIZING).size(), equalTo(2));
         assertThat(clusterState.routingTable().shardsWithState(UNASSIGNED).size(), equalTo(0));
 
         logger.info("start initializing replicas, all should be started");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.routingTable().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.routingTable().shardsWithState(INITIALIZING));
 
         assertThat(clusterState.routingTable().shardsWithState(STARTED).size(), equalTo(10));
         assertThat(clusterState.routingTable().shardsWithState(INITIALIZING).size(), equalTo(0));
@@ -176,46 +176,40 @@ public void testReplicaAndPrimaryRecoveryThrottling() {
     }
 
     public void testThrottleIncomingAndOutgoing() {
+        TestGatewayAllocator gatewayAllocator = new TestGatewayAllocator();
         Settings settings = Settings.builder()
             .put("cluster.routing.allocation.node_concurrent_recoveries", 5)
             .put("cluster.routing.allocation.node_initial_primaries_recoveries", 5)
             .put("cluster.routing.allocation.cluster_concurrent_rebalance", 5)
             .build();
-        AllocationService strategy = createAllocationService(settings);
+        AllocationService strategy = createAllocationService(settings, gatewayAllocator);
         logger.info("Building initial routing table");
 
         MetaData metaData = MetaData.builder()
             .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(9).numberOfReplicas(0))
             .build();
 
-        RoutingTable initialRoutingTable = createRecoveryRoutingTable(metaData.index("test"));
+        ClusterState clusterState = createRecoveryStateAndInitalizeAllocations(metaData, gatewayAllocator);
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(initialRoutingTable).build();
-
-        logger.info("start one node, do reroute, only 5 should initialize");
-        clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1"))).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        logger.info("with one node, do reroute, only 5 should initialize");
+        clusterState = strategy.reroute(clusterState, "reroute");
         assertThat(clusterState.routingTable().shardsWithState(STARTED).size(), equalTo(0));
         assertThat(clusterState.routingTable().shardsWithState(INITIALIZING).size(), equalTo(5));
         assertThat(clusterState.routingTable().shardsWithState(UNASSIGNED).size(), equalTo(4));
         assertEquals(clusterState.getRoutingNodes().getIncomingRecoveries("node1"), 5);
 
         logger.info("start initializing, all primaries should be started");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.routingTable().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.routingTable().shardsWithState(INITIALIZING));
 
         assertThat(clusterState.routingTable().shardsWithState(STARTED).size(), equalTo(5));
         assertThat(clusterState.routingTable().shardsWithState(INITIALIZING).size(), equalTo(4));
         assertThat(clusterState.routingTable().shardsWithState(UNASSIGNED).size(), equalTo(0));
 
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.routingTable().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.routingTable().shardsWithState(INITIALIZING));
 
         logger.info("start another 2 nodes, 5 shards should be relocating - at most 5 are allowed per node");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node2")).add(newNode("node3"))).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         assertThat(clusterState.routingTable().shardsWithState(STARTED).size(), equalTo(4));
         assertThat(clusterState.routingTable().shardsWithState(RELOCATING).size(), equalTo(5));
@@ -226,8 +220,7 @@ public void testThrottleIncomingAndOutgoing() {
         assertEquals(clusterState.getRoutingNodes().getIncomingRecoveries("node1"), 0);
         assertEquals(clusterState.getRoutingNodes().getOutgoingRecoveries("node1"), 5);
 
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.routingTable().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.routingTable().shardsWithState(INITIALIZING));
 
         logger.info("start the relocating shards, one more shard should relocate away from node1");
         assertThat(clusterState.routingTable().shardsWithState(STARTED).size(), equalTo(8));
@@ -241,9 +234,10 @@ public void testThrottleIncomingAndOutgoing() {
     }
 
     public void testOutgoingThrottlesAllocation() {
+        TestGatewayAllocator gatewayAllocator = new TestGatewayAllocator();
         AllocationService strategy = createAllocationService(Settings.builder()
             .put("cluster.routing.allocation.node_concurrent_outgoing_recoveries", 1)
-            .build());
+            .build(), gatewayAllocator);
 
         logger.info("Building initial routing table");
 
@@ -251,22 +245,17 @@ public void testOutgoingThrottlesAllocation() {
             .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(2))
             .build();
 
-        RoutingTable initialRoutingTable = createRecoveryRoutingTable(metaData.index("test"));
+        ClusterState clusterState = createRecoveryStateAndInitalizeAllocations(metaData, gatewayAllocator);
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(initialRoutingTable).build();
-
-        logger.info("start one node, do reroute, only 1 should initialize");
-        clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1"))).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        logger.info("with one node, do reroute, only 1 should initialize");
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         assertThat(clusterState.routingTable().shardsWithState(STARTED).size(), equalTo(0));
         assertThat(clusterState.routingTable().shardsWithState(INITIALIZING).size(), equalTo(1));
         assertThat(clusterState.routingTable().shardsWithState(UNASSIGNED).size(), equalTo(2));
 
         logger.info("start initializing");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.routingTable().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.routingTable().shardsWithState(INITIALIZING));
 
         assertThat(clusterState.routingTable().shardsWithState(STARTED).size(), equalTo(1));
         assertThat(clusterState.routingTable().shardsWithState(INITIALIZING).size(), equalTo(0));
@@ -274,8 +263,7 @@ public void testOutgoingThrottlesAllocation() {
 
         logger.info("start one more node, first non-primary should start being allocated");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node2"))).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         assertThat(clusterState.routingTable().shardsWithState(STARTED).size(), equalTo(1));
         assertThat(clusterState.routingTable().shardsWithState(INITIALIZING).size(), equalTo(1));
@@ -283,8 +271,7 @@ public void testOutgoingThrottlesAllocation() {
         assertEquals(clusterState.getRoutingNodes().getOutgoingRecoveries("node1"), 1);
 
         logger.info("start initializing non-primary");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.routingTable().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.routingTable().shardsWithState(INITIALIZING));
         assertThat(clusterState.routingTable().shardsWithState(STARTED).size(), equalTo(2));
         assertThat(clusterState.routingTable().shardsWithState(INITIALIZING).size(), equalTo(0));
         assertThat(clusterState.routingTable().shardsWithState(UNASSIGNED).size(), equalTo(1));
@@ -292,8 +279,7 @@ public void testOutgoingThrottlesAllocation() {
 
         logger.info("start one more node, initializing second non-primary");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node3"))).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         assertThat(clusterState.routingTable().shardsWithState(STARTED).size(), equalTo(2));
         assertThat(clusterState.routingTable().shardsWithState(INITIALIZING).size(), equalTo(1));
@@ -302,19 +288,18 @@ public void testOutgoingThrottlesAllocation() {
 
         logger.info("start one more node");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node4"))).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         assertEquals(clusterState.getRoutingNodes().getOutgoingRecoveries("node1"), 1);
 
         logger.info("move started non-primary to new node");
-        RoutingAllocation.Result reroute = strategy.reroute(clusterState, new AllocationCommands(
+        AllocationService.CommandsResult commandsResult = strategy.reroute(clusterState, new AllocationCommands(
             new MoveAllocationCommand("test", 0, "node2", "node4")), true, false);
-        assertEquals(reroute.explanations().explanations().size(), 1);
-        assertEquals(reroute.explanations().explanations().get(0).decisions().type(), Decision.Type.THROTTLE);
+        assertEquals(commandsResult.explanations().explanations().size(), 1);
+        assertEquals(commandsResult.explanations().explanations().get(0).decisions().type(), Decision.Type.THROTTLE);
         // even though it is throttled, move command still forces allocation
 
-        clusterState = ClusterState.builder(clusterState).routingResult(reroute).build();
+        clusterState = commandsResult.getClusterState();
         assertThat(clusterState.routingTable().shardsWithState(STARTED).size(), equalTo(1));
         assertThat(clusterState.routingTable().shardsWithState(RELOCATING).size(), equalTo(1));
         assertThat(clusterState.routingTable().shardsWithState(INITIALIZING).size(), equalTo(2));
@@ -323,23 +308,66 @@ public void testOutgoingThrottlesAllocation() {
         assertEquals(clusterState.getRoutingNodes().getOutgoingRecoveries("node2"), 0);
     }
 
-    private RoutingTable createRecoveryRoutingTable(IndexMetaData indexMetaData) {
+    private ClusterState createRecoveryStateAndInitalizeAllocations(MetaData metaData, TestGatewayAllocator gatewayAllocator) {
+        DiscoveryNode node1 = newNode("node1");
+        MetaData.Builder metaDataBuilder = new MetaData.Builder(metaData);
         RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
-        switch (randomInt(5)) {
-            case 0: routingTableBuilder.addAsRecovery(indexMetaData); break;
-            case 1: routingTableBuilder.addAsFromCloseToOpen(indexMetaData); break;
-            case 2: routingTableBuilder.addAsFromDangling(indexMetaData); break;
-            case 3: routingTableBuilder.addAsNewRestore(indexMetaData,
-                new SnapshotRecoverySource(new Snapshot("repo", new SnapshotId("snap", "randomId")), Version.CURRENT,
-                indexMetaData.getIndex().getName()), new IntHashSet()); break;
-            case 4: routingTableBuilder.addAsRestore(indexMetaData,
-                new SnapshotRecoverySource(new Snapshot("repo", new SnapshotId("snap", "randomId")), Version.CURRENT,
-                indexMetaData.getIndex().getName())); break;
-            case 5: routingTableBuilder.addAsNew(indexMetaData); break;
-            default: throw new IndexOutOfBoundsException();
+        for (ObjectCursor<IndexMetaData> cursor: metaData.indices().values()) {
+            Index index = cursor.value.getIndex();
+            IndexMetaData.Builder indexMetaDataBuilder = IndexMetaData.builder(cursor.value);
+            final int recoveryType = randomInt(5);
+            if (recoveryType <= 4) {
+                addInSyncAllocationIds(index, indexMetaDataBuilder, gatewayAllocator, node1);
+            }
+            IndexMetaData indexMetaData = indexMetaDataBuilder.build();
+            metaDataBuilder.put(indexMetaData, false);
+            switch (recoveryType) {
+                case 0:
+                    routingTableBuilder.addAsRecovery(indexMetaData);
+                    break;
+                case 1:
+                    routingTableBuilder.addAsFromCloseToOpen(indexMetaData);
+                    break;
+                case 2:
+                    routingTableBuilder.addAsFromDangling(indexMetaData);
+                    break;
+                case 3:
+                    routingTableBuilder.addAsNewRestore(indexMetaData,
+                        new SnapshotRecoverySource(new Snapshot("repo", new SnapshotId("snap", "randomId")), Version.CURRENT,
+                            indexMetaData.getIndex().getName()), new IntHashSet());
+                    break;
+                case 4:
+                    routingTableBuilder.addAsRestore(indexMetaData,
+                        new SnapshotRecoverySource(new Snapshot("repo", new SnapshotId("snap", "randomId")), Version.CURRENT,
+                            indexMetaData.getIndex().getName()));
+                    break;
+                case 5:
+                    routingTableBuilder.addAsNew(indexMetaData);
+                    break;
+                default:
+                    throw new IndexOutOfBoundsException();
+            }
         }
-
-        return routingTableBuilder.build();
+        return ClusterState.builder(CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
+            .nodes(DiscoveryNodes.builder().add(node1))
+            .metaData(metaDataBuilder.build())
+            .routingTable(routingTableBuilder.build()).build();
     }
 
+    private void addInSyncAllocationIds(Index index, IndexMetaData.Builder indexMetaData,
+                                        TestGatewayAllocator gatewayAllocator, DiscoveryNode node1) {
+        for (int shard = 0; shard < indexMetaData.numberOfShards(); shard++) {
+
+            final boolean primary = randomBoolean();
+            final ShardRouting unassigned = ShardRouting.newUnassigned(new ShardId(index, shard), primary,
+                primary ?
+                    RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE :
+                    RecoverySource.PeerRecoverySource.INSTANCE,
+                new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "test")
+            );
+            ShardRouting started = ShardRoutingHelper.moveToStarted(ShardRoutingHelper.initialize(unassigned, node1.getId()));
+            indexMetaData.putInSyncAllocationIds(shard, Collections.singleton(started.allocationId().getId()));
+            gatewayAllocator.addKnownAllocation(started);
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/UpdateNumberOfReplicasTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/UpdateNumberOfReplicasTests.java
index 4857086e24bd9..e711354b18116 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/UpdateNumberOfReplicasTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/UpdateNumberOfReplicasTests.java
@@ -22,6 +22,7 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -29,7 +30,6 @@
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.cluster.ESAllocationTestCase;
 
 import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED;
@@ -72,22 +72,21 @@ public void testUpdateNumberOfReplicas() {
         logger.info("Adding two nodes and performing rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))).build();
 
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         logger.info("Start all the primary shards");
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
 
         logger.info("Start all the replica shards");
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        ClusterState newState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
+
         final String nodeHoldingPrimary = clusterState.routingTable().index("test").shard(0).primaryShard().currentNodeId();
         final String nodeHoldingReplica = clusterState.routingTable().index("test").shard(0).replicaShards().get(0).currentNodeId();
         assertThat(nodeHoldingPrimary, not(equalTo(nodeHoldingReplica)));
-        assertTrue(routingResult.changed());
         assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
         assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(2));
         assertThat(clusterState.routingTable().index("test").shard(0).shards().size(), equalTo(2));
@@ -117,10 +116,10 @@ public void testUpdateNumberOfReplicas() {
 
         logger.info("Add another node and start the added replica");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node3"))).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        newState = strategy.reroute(clusterState, "reroute");
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
 
-        assertTrue(routingResult.changed());
         assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
         assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(3));
         assertThat(clusterState.routingTable().index("test").shard(0).primaryShard().state(), equalTo(STARTED));
@@ -132,10 +131,10 @@ public void testUpdateNumberOfReplicas() {
         assertThat(clusterState.routingTable().index("test").shard(0).replicaShardsWithState(INITIALIZING).get(0).currentNodeId(), equalTo("node3"));
 
         routingNodes = clusterState.getRoutingNodes();
-        routingResult = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        newState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
+        assertThat(newState, not(equalTo(clusterState)));
+        clusterState = newState;
 
-        assertTrue(routingResult.changed());
         assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
         assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(3));
         assertThat(clusterState.routingTable().index("test").shard(0).primaryShard().state(), equalTo(STARTED));
@@ -153,7 +152,6 @@ public void testUpdateNumberOfReplicas() {
 
         assertThat(clusterState.metaData().index("test").getNumberOfReplicas(), equalTo(1));
 
-        assertTrue(routingResult.changed());
         assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
         assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(2));
         assertThat(clusterState.routingTable().index("test").shard(0).primaryShard().state(), equalTo(STARTED));
@@ -163,9 +161,7 @@ public void testUpdateNumberOfReplicas() {
         assertThat(clusterState.routingTable().index("test").shard(0).replicaShards().get(0).currentNodeId(), anyOf(equalTo(nodeHoldingReplica), equalTo("node3")));
 
         logger.info("do a reroute, should remain the same");
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
-
-        assertFalse(routingResult.changed());
+        newState = strategy.reroute(clusterState, "reroute");
+        assertThat(newState, equalTo(clusterState));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DecisionTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DecisionTests.java
new file mode 100644
index 0000000000000..3774992643d83
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DecisionTests.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.routing.allocation.decider;
+
+import org.elasticsearch.cluster.routing.allocation.decider.Decision.Type;
+import org.elasticsearch.test.ESTestCase;
+
+import static org.elasticsearch.cluster.routing.allocation.decider.Decision.Type.NO;
+import static org.elasticsearch.cluster.routing.allocation.decider.Decision.Type.THROTTLE;
+import static org.elasticsearch.cluster.routing.allocation.decider.Decision.Type.YES;
+
+/**
+ * A class for unit testing the {@link Decision} class.
+ */
+public class DecisionTests extends ESTestCase {
+
+    /**
+     * Tests {@link Type#higherThan(Type)}
+     */
+    public void testHigherThan() {
+        // test YES type
+        assertTrue(YES.higherThan(NO));
+        assertTrue(YES.higherThan(THROTTLE));
+        assertFalse(YES.higherThan(YES));
+
+        // test THROTTLE type
+        assertTrue(THROTTLE.higherThan(NO));
+        assertFalse(THROTTLE.higherThan(THROTTLE));
+        assertFalse(THROTTLE.higherThan(YES));
+
+        // test NO type
+        assertFalse(NO.higherThan(NO));
+        assertFalse(NO.higherThan(THROTTLE));
+        assertFalse(NO.higherThan(YES));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java
index 98268e409fc45..10f1a13f48d6d 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.DiskUsage;
+import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.MockInternalClusterInfoService.DevNullClusterInfo;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
@@ -32,6 +33,7 @@
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
+import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
@@ -49,14 +51,12 @@
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.LocalTransportAddress;
-import org.elasticsearch.cluster.ESAllocationTestCase;
-import org.elasticsearch.test.gateway.NoopGatewayAllocator;
+import org.elasticsearch.test.gateway.TestGatewayAllocator;
 
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
-import java.util.concurrent.atomic.AtomicReference;
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.singleton;
@@ -66,11 +66,10 @@
 import static org.elasticsearch.cluster.routing.ShardRoutingState.UNASSIGNED;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
 
 public class DiskThresholdDeciderTests extends ESAllocationTestCase {
-    
+
     DiskThresholdDecider makeDecider(Settings settings) {
         return new DiskThresholdDecider(settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
     }
@@ -94,9 +93,10 @@ public void testDiskThreshold() {
         ImmutableOpenMap<String, Long> shardSizes = shardSizesBuilder.build();
         final ClusterInfo clusterInfo = new DevNullClusterInfo(usages, usages, shardSizes);
 
+        ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
         AllocationDeciders deciders = new AllocationDeciders(Settings.EMPTY,
                 new HashSet<>(Arrays.asList(
-                        new SameShardAllocationDecider(Settings.EMPTY),
+                        new SameShardAllocationDecider(Settings.EMPTY, clusterSettings),
                         makeDecider(diskSettings))));
 
         ClusterInfoService cis = new ClusterInfoService() {
@@ -115,7 +115,7 @@ public void addListener(Listener listener) {
                 .put("cluster.routing.allocation.node_concurrent_recoveries", 10)
                 .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always")
                 .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1)
-                .build(), deciders, NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(Settings.EMPTY), cis);
+                .build(), deciders, new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), cis);
 
         MetaData metaData = MetaData.builder()
                 .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
@@ -133,16 +133,14 @@ public void addListener(Listener listener) {
                 .add(newNode("node1"))
                 .add(newNode("node2"))
         ).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
         logShardStates(clusterState);
 
         // Primary shard should be initializing, replica should not
         assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(1));
 
         logger.info("--> start the shards (primaries)");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         logShardStates(clusterState);
         // Assert that we're able to start the primary
@@ -151,8 +149,7 @@ public void addListener(Listener listener) {
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(0));
 
         logger.info("--> start the shards (replicas)");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         logShardStates(clusterState);
         // Assert that the replica couldn't be started since node1 doesn't have enough space
@@ -163,8 +160,7 @@ public void addListener(Listener listener) {
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
                 .add(newNode("node3"))
         ).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         logShardStates(clusterState);
         // Assert that the replica is initialized now that node3 is available with enough space
@@ -172,8 +168,7 @@ public void addListener(Listener listener) {
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(1));
 
         logger.info("--> start the shards (replicas)");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         logShardStates(clusterState);
         // Assert that the replica couldn't be started since node1 doesn't have enough space
@@ -194,17 +189,16 @@ public void addListener(Listener listener) {
 
         deciders = new AllocationDeciders(Settings.EMPTY,
                 new HashSet<>(Arrays.asList(
-                        new SameShardAllocationDecider(Settings.EMPTY),
+                        new SameShardAllocationDecider(Settings.EMPTY, clusterSettings),
                         makeDecider(diskSettings))));
 
         strategy = new AllocationService(Settings.builder()
                 .put("cluster.routing.allocation.node_concurrent_recoveries", 10)
                 .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always")
                 .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1)
-                .build(), deciders, NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(Settings.EMPTY), cis);
+                .build(), deciders, new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), cis);
 
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
         logShardStates(clusterState);
 
         // Shards remain started
@@ -225,17 +219,16 @@ public void addListener(Listener listener) {
 
         deciders = new AllocationDeciders(Settings.EMPTY,
                 new HashSet<>(Arrays.asList(
-                        new SameShardAllocationDecider(Settings.EMPTY),
+                        new SameShardAllocationDecider(Settings.EMPTY, clusterSettings),
                         makeDecider(diskSettings))));
 
         strategy = new AllocationService(Settings.builder()
                 .put("cluster.routing.allocation.node_concurrent_recoveries", 10)
                 .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always")
                 .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1)
-                .build(), deciders, NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(Settings.EMPTY), cis);
+                .build(), deciders, new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), cis);
 
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         logShardStates(clusterState);
         // Shards remain started
@@ -250,8 +243,7 @@ public void addListener(Listener listener) {
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
                 .add(newNode("node4"))
         ).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         logShardStates(clusterState);
         // Shards remain started
@@ -259,8 +251,7 @@ public void addListener(Listener listener) {
         assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(1));
 
         logger.info("--> apply INITIALIZING shards");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         logShardStates(clusterState);
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(0));
@@ -290,9 +281,10 @@ public void testDiskThresholdWithAbsoluteSizes() {
         ImmutableOpenMap<String, Long> shardSizes = shardSizesBuilder.build();
         final ClusterInfo clusterInfo = new DevNullClusterInfo(usages, usages, shardSizes);
 
+        ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
         AllocationDeciders deciders = new AllocationDeciders(Settings.EMPTY,
                 new HashSet<>(Arrays.asList(
-                        new SameShardAllocationDecider(Settings.EMPTY),
+                        new SameShardAllocationDecider(Settings.EMPTY, clusterSettings),
                         makeDecider(diskSettings))));
 
         ClusterInfoService cis = new ClusterInfoService() {
@@ -312,7 +304,7 @@ public void addListener(Listener listener) {
                 .put("cluster.routing.allocation.node_concurrent_recoveries", 10)
                 .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always")
                 .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1)
-                .build(), deciders, NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(Settings.EMPTY), cis);
+                .build(), deciders, new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), cis);
 
         MetaData metaData = MetaData.builder()
                 .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(2))
@@ -331,8 +323,7 @@ public void addListener(Listener listener) {
                 .add(newNode("node2"))
         ).build();
 
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
         logShardStates(clusterState);
 
         // Primary should initialize, even though both nodes are over the limit initialize
@@ -370,18 +361,16 @@ public void addListener(Listener listener) {
                 .put("cluster.routing.allocation.node_concurrent_recoveries", 10)
                 .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always")
                 .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1)
-                .build(), deciders, NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(Settings.EMPTY), cis);
+                .build(), deciders, new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), cis);
 
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
         logShardStates(clusterState);
 
         // Now the replica should be able to initialize
         assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(2));
 
         logger.info("--> start the shards (primaries)");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         logShardStates(clusterState);
         // Assert that we're able to start the primary and replica, since they were both initializing
@@ -399,8 +388,7 @@ public void addListener(Listener listener) {
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
                 .add(newNode("node3"))
         ).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         logShardStates(clusterState);
         // Assert that the replica is initialized now that node3 is available with enough space
@@ -408,8 +396,7 @@ public void addListener(Listener listener) {
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(1));
 
         logger.info("--> start the shards (replicas)");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         logShardStates(clusterState);
         // Assert that all replicas could be started
@@ -430,17 +417,16 @@ public void addListener(Listener listener) {
 
         deciders = new AllocationDeciders(Settings.EMPTY,
                 new HashSet<>(Arrays.asList(
-                        new SameShardAllocationDecider(Settings.EMPTY),
+                        new SameShardAllocationDecider(Settings.EMPTY, clusterSettings),
                         makeDecider(diskSettings))));
 
         strategy = new AllocationService(Settings.builder()
                 .put("cluster.routing.allocation.node_concurrent_recoveries", 10)
                 .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always")
                 .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1)
-                .build(), deciders, NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(Settings.EMPTY), cis);
+                .build(), deciders, new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), cis);
 
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
         logShardStates(clusterState);
 
         // Shards remain started
@@ -461,17 +447,16 @@ public void addListener(Listener listener) {
 
         deciders = new AllocationDeciders(Settings.EMPTY,
                 new HashSet<>(Arrays.asList(
-                        new SameShardAllocationDecider(Settings.EMPTY),
+                        new SameShardAllocationDecider(Settings.EMPTY, clusterSettings),
                         makeDecider(diskSettings))));
 
         strategy = new AllocationService(Settings.builder()
                 .put("cluster.routing.allocation.node_concurrent_recoveries", 10)
                 .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always")
                 .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1)
-                .build(), deciders, NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(Settings.EMPTY), cis);
+                .build(), deciders, new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), cis);
 
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         logShardStates(clusterState);
         // Shards remain started
@@ -486,8 +471,7 @@ public void addListener(Listener listener) {
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
                 .add(newNode("node4"))
         ).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         logShardStates(clusterState);
         // Shards remain started
@@ -497,8 +481,7 @@ public void addListener(Listener listener) {
         assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(1));
 
         logger.info("--> apply INITIALIZING shards");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         logShardStates(clusterState);
         // primary shard already has been relocated away
@@ -513,8 +496,7 @@ public void addListener(Listener listener) {
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
                 .add(newNode("node5"))
         ).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
         logShardStates(clusterState);
         // Shards remain started on node3 and node4
@@ -525,8 +507,7 @@ public void addListener(Listener listener) {
         assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(1));
 
         logger.info("--> apply INITIALIZING shards");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         logger.info("--> final cluster state:");
         logShardStates(clusterState);
@@ -557,7 +538,9 @@ public void testDiskThresholdWithShardSizes() {
 
         AllocationDeciders deciders = new AllocationDeciders(Settings.EMPTY,
                 new HashSet<>(Arrays.asList(
-                        new SameShardAllocationDecider(Settings.EMPTY),
+                        new SameShardAllocationDecider(
+                            Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
+                        ),
                         makeDecider(diskSettings))));
 
         ClusterInfoService cis = new ClusterInfoService() {
@@ -577,7 +560,7 @@ public void addListener(Listener listener) {
                 .put("cluster.routing.allocation.node_concurrent_recoveries", 10)
                 .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always")
                 .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1)
-                .build(), deciders, NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(Settings.EMPTY), cis);
+                .build(), deciders, new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), cis);
 
         MetaData metaData = MetaData.builder()
                 .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0))
@@ -627,7 +610,9 @@ public void testUnknownDiskUsage() {
 
         AllocationDeciders deciders = new AllocationDeciders(Settings.EMPTY,
                 new HashSet<>(Arrays.asList(
-                        new SameShardAllocationDecider(Settings.EMPTY),
+                        new SameShardAllocationDecider(
+                            Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
+                        ),
                         makeDecider(diskSettings))));
 
         ClusterInfoService cis = new ClusterInfoService() {
@@ -647,7 +632,7 @@ public void addListener(Listener listener) {
                 .put("cluster.routing.allocation.node_concurrent_recoveries", 10)
                 .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always")
                 .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1)
-                .build(), deciders, NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(Settings.EMPTY), cis);
+                .build(), deciders, new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), cis);
 
         MetaData metaData = MetaData.builder()
                 .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0))
@@ -732,7 +717,9 @@ public void testShardRelocationsTakenIntoAccount() {
 
         DiskThresholdDecider decider = makeDecider(diskSettings);
         AllocationDeciders deciders = new AllocationDeciders(Settings.EMPTY,
-                new HashSet<>(Arrays.asList(new SameShardAllocationDecider(Settings.EMPTY), decider)));
+                new HashSet<>(Arrays.asList(new SameShardAllocationDecider(
+                    Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
+                ), decider)));
 
         ClusterInfoService cis = new ClusterInfoService() {
             @Override
@@ -751,7 +738,7 @@ public void addListener(Listener listener) {
                 .put("cluster.routing.allocation.node_concurrent_recoveries", 10)
                 .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always")
                 .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1)
-                .build(), deciders, NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(Settings.EMPTY), cis);
+                .build(), deciders, new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), cis);
 
         MetaData metaData = MetaData.builder()
                 .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
@@ -771,16 +758,14 @@ public void addListener(Listener listener) {
                 .add(newNode("node1"))
                 .add(newNode("node2"))
         ).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
         logShardStates(clusterState);
 
         // shards should be initializing
         assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(4));
 
         logger.info("--> start the shards");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         logShardStates(clusterState);
         // Assert that we're able to start the primary and replicas
@@ -794,8 +779,7 @@ public void addListener(Listener listener) {
         AllocationCommand relocate1 = new MoveAllocationCommand("test", 0, "node2", "node3");
         AllocationCommands cmds = new AllocationCommands(relocate1);
 
-        routingResult = strategy.reroute(clusterState, cmds, false, false);
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, cmds, false, false).getClusterState();
         logShardStates(clusterState);
 
         AllocationCommand relocate2 = new MoveAllocationCommand("test2", 0, "node2", "node3");
@@ -806,12 +790,14 @@ public void addListener(Listener listener) {
             // node3, which will put it over the low watermark when it
             // completes, with shard relocations taken into account this should
             // throw an exception about not being able to complete
-            strategy.reroute(clusterState, cmds, false, false).routingTable();
+            strategy.reroute(clusterState, cmds, false, false);
             fail("should not have been able to reroute the shard");
         } catch (IllegalArgumentException e) {
-            assertThat("can't allocated because there isn't enough room: " + e.getMessage(),
+            assertThat("can't be allocated because there isn't enough room: " + e.getMessage(),
                     e.getMessage(),
-                    containsString("the node is above the low watermark and has more than allowed [70.0%] used disk, free: [26.0%]"));
+                    containsString("the node is above the low watermark cluster setting " +
+                                   "[cluster.routing.allocation.disk.watermark.low=0.7], using more disk space than the maximum " +
+                                   "allowed [70.0%], actual free: [26.0%]"));
         }
 
     }
@@ -878,8 +864,13 @@ public void testCanRemainWithShardRelocatingAway() {
         ClusterState clusterState = ClusterState.builder(baseClusterState).routingTable(builder.build()).build();
         RoutingAllocation routingAllocation = new RoutingAllocation(null, new RoutingNodes(clusterState), clusterState, clusterInfo,
                 System.nanoTime(), false);
+        routingAllocation.debugDecision(true);
         Decision decision = diskThresholdDecider.canRemain(firstRouting, firstRoutingNode, routingAllocation);
         assertThat(decision.type(), equalTo(Decision.Type.NO));
+        assertThat(((Decision.Single) decision).getExplanation(), containsString(
+            "the shard cannot remain on this node because it is above the high watermark cluster setting " +
+            "[cluster.routing.allocation.disk.watermark.high=70%] and there is less than the required [30.0%] free disk on node, " +
+            "actual free: [20.0%]"));
 
         // Two shards consuming each 80% of disk space while 70% is allowed, but one is relocating, so shard 0 can stay
         firstRouting = TestShardRouting.newShardRouting("test", 0, "node1", null, true, ShardRoutingState.STARTED);
@@ -900,10 +891,22 @@ public void testCanRemainWithShardRelocatingAway() {
         clusterState = ClusterState.builder(baseClusterState).routingTable(builder.build()).build();
         routingAllocation = new RoutingAllocation(null, new RoutingNodes(clusterState), clusterState, clusterInfo, System.nanoTime(),
             false);
+        routingAllocation.debugDecision(true);
         decision = diskThresholdDecider.canRemain(firstRouting, firstRoutingNode, routingAllocation);
         assertThat(decision.type(), equalTo(Decision.Type.YES));
+        assertEquals("there is enough disk on this node for the shard to remain, free: [60b]",
+            ((Decision.Single) decision).getExplanation());
         decision = diskThresholdDecider.canAllocate(fooRouting, firstRoutingNode, routingAllocation);
         assertThat(decision.type(), equalTo(Decision.Type.NO));
+        if (fooRouting.recoverySource().getType() == RecoverySource.Type.EMPTY_STORE) {
+            assertThat(((Decision.Single) decision).getExplanation(), containsString(
+                "the node is above the high watermark cluster setting [cluster.routing.allocation.disk.watermark.high=70%], using " +
+                "more disk space than the maximum allowed [70.0%], actual free: [20.0%]"));
+        } else {
+            assertThat(((Decision.Single) decision).getExplanation(), containsString(
+                "the node is above the low watermark cluster setting [cluster.routing.allocation.disk.watermark.low=60%], using more " +
+                "disk space than the maximum allowed [60.0%], actual free: [20.0%]"));
+        }
 
         // Creating AllocationService instance and the services it depends on...
         ClusterInfoService cis = new ClusterInfoService() {
@@ -919,17 +922,20 @@ public void addListener(Listener listener) {
             }
         };
         AllocationDeciders deciders = new AllocationDeciders(Settings.EMPTY, new HashSet<>(Arrays.asList(
-                new SameShardAllocationDecider(Settings.EMPTY), diskThresholdDecider
+                new SameShardAllocationDecider(
+                    Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
+                ),
+                diskThresholdDecider
         )));
         AllocationService strategy = new AllocationService(Settings.builder()
                 .put("cluster.routing.allocation.node_concurrent_recoveries", 10)
                 .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always")
                 .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1)
-                .build(), deciders, NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(Settings.EMPTY), cis);
+                .build(), deciders, new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), cis);
         // Ensure that the reroute call doesn't alter the routing table, since the first primary is relocating away
         // and therefor we will have sufficient disk space on node1.
-        RoutingAllocation.Result result = strategy.reroute(clusterState, "reroute");
-        assertThat(result.changed(), is(false));
+        ClusterState result = strategy.reroute(clusterState, "reroute");
+        assertThat(result, equalTo(clusterState));
         assertThat(result.routingTable().index("test").getShards().get(0).primaryShard().state(), equalTo(STARTED));
         assertThat(result.routingTable().index("test").getShards().get(0).primaryShard().currentNodeId(), equalTo("node1"));
         assertThat(result.routingTable().index("test").getShards().get(0).primaryShard().relocatingNodeId(), nullValue());
@@ -998,10 +1004,12 @@ public void testForSingleDataNode() {
         ClusterState clusterState = ClusterState.builder(baseClusterState).routingTable(builder.build()).build();
         RoutingAllocation routingAllocation = new RoutingAllocation(null, new RoutingNodes(clusterState), clusterState, clusterInfo,
                 System.nanoTime(), false);
+        routingAllocation.debugDecision(true);
         Decision decision = diskThresholdDecider.canRemain(firstRouting, firstRoutingNode, routingAllocation);
 
         // Two shards should start happily
         assertThat(decision.type(), equalTo(Decision.Type.YES));
+        assertThat(((Decision.Single) decision).getExplanation(), containsString("there is only a single data node present"));
         ClusterInfoService cis = new ClusterInfoService() {
             @Override
             public ClusterInfo getClusterInfo() {
@@ -1015,7 +1023,10 @@ public void addListener(Listener listener) {
         };
 
         AllocationDeciders deciders = new AllocationDeciders(Settings.EMPTY, new HashSet<>(Arrays.asList(
-                new SameShardAllocationDecider(Settings.EMPTY), diskThresholdDecider
+                new SameShardAllocationDecider(
+                    Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
+                ),
+                diskThresholdDecider
         )));
 
         AllocationService strategy = new AllocationService(Settings.builder()
@@ -1023,8 +1034,8 @@ public void addListener(Listener listener) {
                 .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always")
 
                 .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1)
-                .build(), deciders, NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(Settings.EMPTY), cis);
-        RoutingAllocation.Result result = strategy.reroute(clusterState, "reroute");
+                .build(), deciders, new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), cis);
+        ClusterState result = strategy.reroute(clusterState, "reroute");
 
         assertThat(result.routingTable().index("test").getShards().get(0).primaryShard().state(), equalTo(STARTED));
         assertThat(result.routingTable().index("test").getShards().get(0).primaryShard().currentNodeId(), equalTo("node2"));
@@ -1058,8 +1069,11 @@ public void addListener(Listener listener) {
         clusterState = ClusterState.builder(updateClusterState).routingTable(builder.build()).build();
         routingAllocation = new RoutingAllocation(null, new RoutingNodes(clusterState), clusterState, clusterInfo, System.nanoTime(),
             false);
+        routingAllocation.debugDecision(true);
         decision = diskThresholdDecider.canRemain(firstRouting, firstRoutingNode, routingAllocation);
         assertThat(decision.type(), equalTo(Decision.Type.YES));
+        assertThat(((Decision.Single) decision).getExplanation(), containsString(
+            "there is enough disk on this node for the shard to remain, free: [60b]"));
 
         result = strategy.reroute(clusterState, "reroute");
         assertThat(result.routingTable().index("test").getShards().get(0).primaryShard().state(), equalTo(STARTED));
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java
index 4649bfb42c9cd..6e88ba7593c01 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.cluster.ClusterInfo;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.DiskUsage;
+import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.MockInternalClusterInfoService.DevNullClusterInfo;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
@@ -45,7 +46,6 @@
 import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.cluster.ESAllocationTestCase;
 
 import java.util.Arrays;
 import java.util.Collections;
@@ -53,6 +53,7 @@
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
+import static org.hamcrest.Matchers.containsString;
 
 /**
  * Unit tests for the DiskThresholdDecider
@@ -99,8 +100,15 @@ public void testCanAllocateUsesMaxAvailableSpace() {
         shardSizes.put("[test][0][p]", 10L); // 10 bytes
         final ClusterInfo clusterInfo = new ClusterInfo(leastAvailableUsages.build(), mostAvailableUsage.build(), shardSizes.build(), ImmutableOpenMap.of());
         RoutingAllocation allocation = new RoutingAllocation(new AllocationDeciders(Settings.EMPTY, Collections.singleton(decider)), clusterState.getRoutingNodes(), clusterState, clusterInfo, System.nanoTime(), false);
-        assertEquals(mostAvailableUsage.toString(), Decision.YES, decider.canAllocate(test_0, new RoutingNode("node_0", node_0), allocation));
-        assertEquals(mostAvailableUsage.toString(), Decision.NO, decider.canAllocate(test_0, new RoutingNode("node_1", node_1), allocation));
+        allocation.debugDecision(true);
+        Decision decision = decider.canAllocate(test_0, new RoutingNode("node_0", node_0), allocation);
+        assertEquals(mostAvailableUsage.toString(), Decision.Type.YES, decision.type());
+        assertThat(((Decision.Single) decision).getExplanation(), containsString("enough disk for shard on node"));
+        decision = decider.canAllocate(test_0, new RoutingNode("node_1", node_1), allocation);
+        assertEquals(mostAvailableUsage.toString(), Decision.Type.NO, decision.type());
+        assertThat(((Decision.Single) decision).getExplanation(), containsString(
+            "the node is above the high watermark cluster setting [cluster.routing.allocation.disk.watermark.high=90%], using more " +
+            "disk space than the maximum allowed [90.0%]"));
     }
 
     public void testCanRemainUsesLeastAvailableSpace() {
@@ -166,8 +174,16 @@ public void testCanRemainUsesLeastAvailableSpace() {
 
         final ClusterInfo clusterInfo = new ClusterInfo(leastAvailableUsages.build(), mostAvailableUsage.build(), shardSizes.build(), shardRoutingMap.build());
         RoutingAllocation allocation = new RoutingAllocation(new AllocationDeciders(Settings.EMPTY, Collections.singleton(decider)), clusterState.getRoutingNodes(), clusterState, clusterInfo, System.nanoTime(), false);
-        assertEquals(Decision.YES, decider.canRemain(test_0, new RoutingNode("node_0", node_0), allocation));
-        assertEquals(Decision.NO, decider.canRemain(test_1, new RoutingNode("node_1", node_1), allocation));
+        allocation.debugDecision(true);
+        Decision decision = decider.canRemain(test_0, new RoutingNode("node_0", node_0), allocation);
+        assertEquals(Decision.Type.YES, decision.type());
+        assertThat(((Decision.Single) decision).getExplanation(), containsString(
+            "there is enough disk on this node for the shard to remain, free: [10b]"));
+        decision = decider.canRemain(test_1, new RoutingNode("node_1", node_1), allocation);
+        assertEquals(Decision.Type.NO, decision.type());
+        assertThat(((Decision.Single) decision).getExplanation(), containsString("the shard cannot remain on this node because it is " +
+            "above the high watermark cluster setting [cluster.routing.allocation.disk.watermark.high=90%] and there is less than " +
+            "the required [10.0%] free disk on node, actual free: [9.0%]"));
         try {
             decider.canRemain(test_0, new RoutingNode("node_1", node_1), allocation);
             fail("not allocated on this node");
@@ -181,9 +197,15 @@ public void testCanRemainUsesLeastAvailableSpace() {
             // not allocated on that node
         }
 
-        assertEquals("can stay since allocated on a different path with enough space", Decision.YES, decider.canRemain(test_2, new RoutingNode("node_1", node_1), allocation));
+        decision = decider.canRemain(test_2, new RoutingNode("node_1", node_1), allocation);
+        assertEquals("can stay since allocated on a different path with enough space", Decision.Type.YES, decision.type());
+        assertThat(((Decision.Single) decision).getExplanation(), containsString(
+            "this shard is not allocated on the most utilized disk and can remain"));
 
-        assertEquals("can stay since we don't have information about this shard", Decision.YES, decider.canRemain(test_2, new RoutingNode("node_1", node_1), allocation));
+        decision = decider.canRemain(test_2, new RoutingNode("node_1", node_1), allocation);
+        assertEquals("can stay since we don't have information about this shard", Decision.Type.YES, decision.type());
+        assertThat(((Decision.Single) decision).getExplanation(), containsString(
+            "this shard is not allocated on the most utilized disk and can remain"));
     }
 
 
@@ -279,12 +301,10 @@ public void testSizeShrinkIndex() {
         AllocationService allocationService = createAllocationService();
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")))
             .build();
-        RoutingAllocation.Result result = allocationService.reroute(clusterState, "foo");
-        clusterState = ClusterState.builder(clusterState).routingTable(result.routingTable()).build();
+        clusterState = allocationService.reroute(clusterState, "foo");
 
-        result = allocationService.applyStartedShards(clusterState,
+        clusterState = allocationService.applyStartedShards(clusterState,
             clusterState.getRoutingTable().index("test").shardsWithState(ShardRoutingState.UNASSIGNED));
-        clusterState = ClusterState.builder(clusterState).routingTable(result.routingTable()).build();
 
         RoutingAllocation allocation = new RoutingAllocation(null, clusterState.getRoutingNodes(), clusterState, info, 0, false);
 
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationDeciderIT.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationDeciderIT.java
deleted file mode 100644
index 66391dc7b31b8..0000000000000
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationDeciderIT.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.cluster.routing.allocation.decider;
-
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.test.ESIntegTestCase;
-
-import java.util.Set;
-
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.hamcrest.Matchers.equalTo;
-
-/**
- * Simple integration for {@link EnableAllocationDecider} there is a more exhaustive unittest in
- * {@link EnableAllocationTests} this test is meant to check if the actual update of the settings
- * works as expected.
- */
-@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0)
-public class EnableAllocationDeciderIT extends ESIntegTestCase {
-
-    public void testEnableRebalance() throws InterruptedException {
-        final String firstNode = internalCluster().startNode();
-        client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), EnableAllocationDecider.Rebalance.NONE)).get();
-        // we test with 2 shards since otherwise it's pretty fragile if there are difference in the num or shards such that
-        // all shards are relocated to the second node which is not what we want here. It's solely a test for the settings to take effect
-        final int numShards = 2;
-        assertAcked(prepareCreate("test").setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, numShards)));
-        assertAcked(prepareCreate("test_1").setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, numShards)));
-        ensureGreen();
-        assertAllShardsOnNodes("test", firstNode);
-        assertAllShardsOnNodes("test_1", firstNode);
-
-        final String secondNode = internalCluster().startNode();
-        // prevent via index setting but only on index test
-        client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder().put(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), EnableAllocationDecider.Rebalance.NONE)).get();
-        client().admin().cluster().prepareReroute().get();
-        ensureGreen();
-        assertAllShardsOnNodes("test", firstNode);
-        assertAllShardsOnNodes("test_1", firstNode);
-
-        // now enable the index test to relocate since index settings override cluster settings
-        client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder().put(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), randomBoolean() ? EnableAllocationDecider.Rebalance.PRIMARIES : EnableAllocationDecider.Rebalance.ALL)).get();
-        logger.info("--> balance index [test]");
-        client().admin().cluster().prepareReroute().get();
-        ensureGreen("test");
-        Set<String> test = assertAllShardsOnNodes("test", firstNode, secondNode);
-        assertThat("index: [test] expected to be rebalanced on both nodes", test.size(), equalTo(2));
-
-        // flip the cluster wide setting such that we can also balance for index test_1 eventually we should have one shard of each index on each node
-        client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), randomBoolean() ? EnableAllocationDecider.Rebalance.PRIMARIES : EnableAllocationDecider.Rebalance.ALL)).get();
-        logger.info("--> balance index [test_1]");
-        client().admin().cluster().prepareReroute().get();
-        ensureGreen("test_1");
-        Set<String> test_1 = assertAllShardsOnNodes("test_1", firstNode, secondNode);
-        assertThat("index: [test_1] expected to be rebalanced on both nodes", test_1.size(), equalTo(2));
-
-        test = assertAllShardsOnNodes("test", firstNode, secondNode);
-        assertThat("index: [test] expected to be rebalanced on both nodes", test.size(), equalTo(2));
-    }
-}
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationTests.java
index a9c15de61dcce..4230504f538a5 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationTests.java
@@ -23,19 +23,18 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
-import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider.Allocation;
 import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider.Rebalance;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.cluster.ESAllocationTestCase;
 
 import java.util.EnumSet;
 import java.util.List;
@@ -138,15 +137,12 @@ public void testIndexEnableNone() {
                 .add(newNode("node1"))
                 .add(newNode("node2"))
         ).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
         assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(1));
         logger.info("--> start the shards (primaries)");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
         logger.info("--> start the shards (replicas)");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         logger.info("--> verify only enabled index has been routed");
         assertThat(clusterState.getRoutingNodes().shardsWithState("enabled", STARTED).size(), equalTo(2));
@@ -183,17 +179,14 @@ public void testEnableClusterBalance() {
                 .add(newNode("node1"))
                 .add(newNode("node2"))
         ).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
         assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(4));
         logger.info("--> start the shards (primaries)");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
         assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(4));
         assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(4));
 
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
         assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(8));
         assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(0));
 
@@ -203,8 +196,7 @@ public void testEnableClusterBalance() {
                 .add(newNode("node2"))
                 .add(newNode("node3"))
         ).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
         assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(8));
         assertThat(clusterState.getRoutingNodes().shardsWithState(RELOCATING).size(), equalTo(0));
 
@@ -221,8 +213,7 @@ public void testEnableClusterBalance() {
 
         }
         clusterSettings.applySettings(clusterState.metaData().settings());
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
         assertThat("expected 6 shards to be started 2 to relocate useClusterSettings: " + useClusterSetting, clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(6));
         assertThat("expected 2 shards to relocate useClusterSettings: " + useClusterSetting, clusterState.getRoutingNodes().shardsWithState(RELOCATING).size(), equalTo(2));
         List<ShardRouting> mutableShardRoutings = clusterState.getRoutingNodes().shardsWithState(RELOCATING);
@@ -247,8 +238,7 @@ public void testEnableClusterBalance() {
             default:
                 fail("only replicas, primaries or all are allowed");
         }
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
         assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(8));
         assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(0));
 
@@ -280,12 +270,10 @@ public void testEnableClusterBalanceNoReplicas() {
                 .add(newNode("node1"))
                 .add(newNode("node2"))
         ).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
         assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(6));
         logger.info("--> start the shards (primaries)");
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
         assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(6));
         assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(0));
 
@@ -295,8 +283,7 @@ public void testEnableClusterBalanceNoReplicas() {
                 .add(newNode("node2"))
                 .add(newNode("node3"))
         ).build();
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
         assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(6));
         assertThat(clusterState.getRoutingNodes().shardsWithState(RELOCATING).size(), equalTo(0));
         metaData = clusterState.metaData();
@@ -310,8 +297,7 @@ public void testEnableClusterBalanceNoReplicas() {
                     .put(IndexMetaData.builder(meta).settings(Settings.builder().put(meta.getSettings()).put(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), randomBoolean() ? Rebalance.PRIMARIES : Rebalance.ALL).build()))).build();
         }
         clusterSettings.applySettings(clusterState.metaData().settings());
-        routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
         assertThat("expected 4 primaries to be started and 2 to relocate useClusterSettings: " + useClusterSetting, clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(4));
         assertThat("expected 2 primaries to relocate useClusterSettings: " + useClusterSetting, clusterState.getRoutingNodes().shardsWithState(RELOCATING).size(), equalTo(2));
 
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDeciderTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDeciderTests.java
new file mode 100644
index 0000000000000..711e8af13db2a
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDeciderTests.java
@@ -0,0 +1,206 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.cluster.routing.allocation.decider;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.EmptyClusterInfoService;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.RecoverySource;
+import org.elasticsearch.cluster.routing.RoutingTable;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
+import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
+import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
+import org.elasticsearch.cluster.routing.allocation.decider.Decision.Type;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.IndexScopedSettings;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.gateway.TestGatewayAllocator;
+
+import java.util.Arrays;
+import java.util.Collections;
+
+import static org.elasticsearch.cluster.metadata.IndexMetaData.INDEX_SHRINK_SOURCE_NAME;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.INDEX_SHRINK_SOURCE_UUID;
+import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
+import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED;
+import static org.elasticsearch.cluster.routing.ShardRoutingState.UNASSIGNED;
+
+public class FilterAllocationDeciderTests extends ESAllocationTestCase {
+
+    public void testFilterInitialRecovery() {
+        ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        FilterAllocationDecider filterAllocationDecider = new FilterAllocationDecider(Settings.EMPTY, clusterSettings);
+        AllocationDeciders allocationDeciders = new AllocationDeciders(Settings.EMPTY,
+            Arrays.asList(filterAllocationDecider,
+                new SameShardAllocationDecider(Settings.EMPTY, clusterSettings),
+                new ReplicaAfterPrimaryActiveAllocationDecider(Settings.EMPTY)));
+        AllocationService service = new AllocationService(Settings.builder().build(), allocationDeciders,
+            new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
+        ClusterState state = createInitialClusterState(service, Settings.builder().put("index.routing.allocation.initial_recovery._id",
+            "node2").build());
+        RoutingTable routingTable = state.routingTable();
+
+        // we can initally only allocate on node2
+        assertEquals(routingTable.index("idx").shard(0).shards().get(0).state(), INITIALIZING);
+        assertEquals(routingTable.index("idx").shard(0).shards().get(0).currentNodeId(), "node2");
+        routingTable = service.applyFailedShard(state, routingTable.index("idx").shard(0).shards().get(0)).routingTable();
+        state = ClusterState.builder(state).routingTable(routingTable).build();
+        assertEquals(routingTable.index("idx").shard(0).shards().get(0).state(), UNASSIGNED);
+        assertNull(routingTable.index("idx").shard(0).shards().get(0).currentNodeId());
+
+        // after failing the shard we are unassigned since the node is blacklisted and we can't initialize on the other node
+        RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, state.getRoutingNodes(), state,
+            null, 0, false);
+        allocation.debugDecision(true);
+        Decision.Single decision = (Decision.Single) filterAllocationDecider.canAllocate(
+            routingTable.index("idx").shard(0).primaryShard(),
+            state.getRoutingNodes().node("node2"), allocation);
+        assertEquals(Type.YES, decision.type());
+        assertEquals("node passes include/exclude/require filters", decision.getExplanation());
+        ShardRouting primaryShard = routingTable.index("idx").shard(0).primaryShard();
+        decision = (Decision.Single) filterAllocationDecider.canAllocate(
+            routingTable.index("idx").shard(0).primaryShard(),
+            state.getRoutingNodes().node("node1"), allocation);
+        assertEquals(Type.NO, decision.type());
+        if (primaryShard.recoverySource().getType() == RecoverySource.Type.LOCAL_SHARDS) {
+            assertEquals("initial allocation of the shrunken index is only allowed on nodes [_id:\"node2\"] that " +
+                         "hold a copy of every shard in the index", decision.getExplanation());
+        } else {
+            assertEquals("initial allocation of the index is only allowed on nodes [_id:\"node2\"]", decision.getExplanation());
+        }
+
+        state = service.reroute(state, "try allocate again");
+        routingTable = state.routingTable();
+        assertEquals(routingTable.index("idx").shard(0).primaryShard().state(), INITIALIZING);
+        assertEquals(routingTable.index("idx").shard(0).primaryShard().currentNodeId(), "node2");
+
+        state = service.applyStartedShards(state, routingTable.index("idx").shard(0).shardsWithState(INITIALIZING));
+        routingTable = state.routingTable();
+
+        // ok now we are started and can be allocated anywhere!! lets see...
+        // first create another copy
+        assertEquals(routingTable.index("idx").shard(0).replicaShards().get(0).state(), INITIALIZING);
+        assertEquals(routingTable.index("idx").shard(0).replicaShards().get(0).currentNodeId(), "node1");
+        state = service.applyStartedShards(state, routingTable.index("idx").shard(0).replicaShardsWithState(INITIALIZING));
+        routingTable = state.routingTable();
+        assertEquals(routingTable.index("idx").shard(0).replicaShards().get(0).state(), STARTED);
+        assertEquals(routingTable.index("idx").shard(0).replicaShards().get(0).currentNodeId(), "node1");
+
+        // now remove the node of the other copy and fail the current
+        DiscoveryNode node1 = state.nodes().resolveNode("node1");
+        state = service.deassociateDeadNodes(
+            ClusterState.builder(state).nodes(DiscoveryNodes.builder(state.nodes()).remove("node1")).build(),
+            true, "test");
+        state = service.applyFailedShard(state, routingTable.index("idx").shard(0).primaryShard());
+
+        // now bring back node1 and see it's assigned
+        state = service.reroute(
+            ClusterState.builder(state).nodes(DiscoveryNodes.builder(state.nodes()).add(node1)).build(), "test");
+        routingTable = state.routingTable();
+        assertEquals(routingTable.index("idx").shard(0).primaryShard().state(), INITIALIZING);
+        assertEquals(routingTable.index("idx").shard(0).primaryShard().currentNodeId(), "node1");
+
+        allocation = new RoutingAllocation(allocationDeciders, state.getRoutingNodes(), state,
+            null, 0, false);
+        allocation.debugDecision(true);
+        decision = (Decision.Single) filterAllocationDecider.canAllocate(
+            routingTable.index("idx").shard(0).shards().get(0),
+            state.getRoutingNodes().node("node2"), allocation);
+        assertEquals(Type.YES, decision.type());
+        assertEquals("node passes include/exclude/require filters", decision.getExplanation());
+        decision = (Decision.Single) filterAllocationDecider.canAllocate(
+            routingTable.index("idx").shard(0).shards().get(0),
+            state.getRoutingNodes().node("node1"), allocation);
+        assertEquals(Type.YES, decision.type());
+        assertEquals("node passes include/exclude/require filters", decision.getExplanation());
+    }
+
+    private ClusterState createInitialClusterState(AllocationService service, Settings settings) {
+        RecoverySource.Type recoveryType = randomFrom(FilterAllocationDecider.INITIAL_RECOVERY_TYPES);
+        MetaData.Builder metaData = MetaData.builder();
+        final Settings.Builder indexSettings = settings(Version.CURRENT).put(settings);
+        final IndexMetaData sourceIndex;
+        if (recoveryType == RecoverySource.Type.LOCAL_SHARDS) {
+            //put a fake closed source index
+            sourceIndex = IndexMetaData.builder("sourceIndex")
+                .settings(settings(Version.CURRENT)).numberOfShards(2).numberOfReplicas(0)
+                .putInSyncAllocationIds(0, Collections.singleton("aid0"))
+                .putInSyncAllocationIds(1, Collections.singleton("aid1"))
+                .build();
+            metaData.put(sourceIndex, false);
+            indexSettings.put(INDEX_SHRINK_SOURCE_UUID.getKey(), sourceIndex.getIndexUUID());
+            indexSettings.put(INDEX_SHRINK_SOURCE_NAME.getKey(), sourceIndex.getIndex().getName());
+        } else {
+            sourceIndex = null;
+        }
+        final IndexMetaData.Builder indexMetaDataBuilder = IndexMetaData.builder("idx").settings(indexSettings)
+            .numberOfShards(1).numberOfReplicas(1);
+        final IndexMetaData indexMetaData = indexMetaDataBuilder.build();
+        metaData.put(indexMetaData, false);
+        RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
+        switch (recoveryType) {
+            case EMPTY_STORE:
+                routingTableBuilder.addAsNew(indexMetaData);
+                break;
+            case LOCAL_SHARDS:
+                routingTableBuilder.addAsFromCloseToOpen(sourceIndex);
+                routingTableBuilder.addAsNew(indexMetaData);
+                break;
+            default:
+                throw new UnsupportedOperationException(recoveryType + " is not supported");
+        }
+
+        RoutingTable routingTable = routingTableBuilder.build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING
+            .getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
+        clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2")))
+            .build();
+        return service.reroute(clusterState, "reroute");
+    }
+
+    public void testInvalidIPFilter() {
+        String ipKey = randomFrom("_ip", "_host_ip", "_publish_ip");
+        Setting<Settings> filterSetting = randomFrom(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING,
+            IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_SETTING, IndexMetaData.INDEX_ROUTING_EXCLUDE_GROUP_SETTING);
+        String invalidIP = randomFrom("192..168.1.1", "192.300.1.1");
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {
+            IndexScopedSettings indexScopedSettings = new IndexScopedSettings(Settings.EMPTY, IndexScopedSettings.BUILT_IN_INDEX_SETTINGS);
+            indexScopedSettings.updateDynamicSettings(Settings.builder().put(filterSetting.getKey() + ipKey, invalidIP).build(),
+                Settings.builder().put(Settings.EMPTY), Settings.builder(), "test ip validation");
+        });
+        assertEquals("invalid IP address [" + invalidIP + "] for [" + ipKey + "]", e.getMessage());
+    }
+
+    public void testWildcardIPFilter() {
+        String ipKey = randomFrom("_ip", "_host_ip", "_publish_ip");
+        Setting<Settings> filterSetting = randomFrom(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING,
+            IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_SETTING, IndexMetaData.INDEX_ROUTING_EXCLUDE_GROUP_SETTING);
+        String wildcardIP = randomFrom("192.168.*", "192.*.1.1");
+        IndexScopedSettings indexScopedSettings = new IndexScopedSettings(Settings.EMPTY, IndexScopedSettings.BUILT_IN_INDEX_SETTINGS);
+        indexScopedSettings.updateDynamicSettings(Settings.builder().put(filterSetting.getKey() + ipKey, wildcardIP).build(),
+            Settings.builder().put(Settings.EMPTY), Settings.builder(), "test ip validation");
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java
index 68e8fc9c94ef9..93ac2878abc0f 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java
@@ -54,15 +54,12 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
     }
 
     public void testRerouteOccursOnDiskPassingHighWatermark() throws Exception {
-        List<String> nodes = internalCluster().startNodesAsync(3).get();
+        List<String> nodes = internalCluster().startNodes(3);
 
         // Wait for all 3 nodes to be up
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                NodesStatsResponse resp = client().admin().cluster().prepareNodesStats().get();
-                assertThat(resp.getNodes().size(), equalTo(3));
-            }
+        assertBusy(() -> {
+            NodesStatsResponse resp = client().admin().cluster().prepareNodesStats().get();
+            assertThat(resp.getNodes().size(), equalTo(3));
         });
 
         // Start with all nodes at 50% usage
@@ -86,13 +83,10 @@ public void run() {
         ensureGreen("test");
 
         // Block until the "fake" cluster info is retrieved at least once
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                ClusterInfo info = cis.getClusterInfo();
-                logger.info("--> got: {} nodes", info.getNodeLeastAvailableDiskUsages().size());
-                assertThat(info.getNodeLeastAvailableDiskUsages().size(), greaterThan(0));
-            }
+        assertBusy(() -> {
+            ClusterInfo info = cis.getClusterInfo();
+            logger.info("--> got: {} nodes", info.getNodeLeastAvailableDiskUsages().size());
+            assertThat(info.getNodeLeastAvailableDiskUsages().size(), greaterThan(0));
         });
 
         final List<String> realNodeNames = new ArrayList<>();
@@ -113,21 +107,18 @@ public void run() {
         // Retrieve the count of shards on each node
         final Map<String, Integer> nodesToShardCount = new HashMap<>();
 
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                ClusterStateResponse resp = client().admin().cluster().prepareState().get();
-                Iterator<RoutingNode> iter = resp.getState().getRoutingNodes().iterator();
-                while (iter.hasNext()) {
-                    RoutingNode node = iter.next();
-                    logger.info("--> node {} has {} shards",
-                            node.nodeId(), resp.getState().getRoutingNodes().node(node.nodeId()).numberOfOwningShards());
-                    nodesToShardCount.put(node.nodeId(), resp.getState().getRoutingNodes().node(node.nodeId()).numberOfOwningShards());
-                }
-                assertThat("node1 has 5 shards", nodesToShardCount.get(realNodeNames.get(0)), equalTo(5));
-                assertThat("node2 has 5 shards", nodesToShardCount.get(realNodeNames.get(1)), equalTo(5));
-                assertThat("node3 has 0 shards", nodesToShardCount.get(realNodeNames.get(2)), equalTo(0));
+        assertBusy(() -> {
+            ClusterStateResponse resp12 = client().admin().cluster().prepareState().get();
+            Iterator<RoutingNode> iter12 = resp12.getState().getRoutingNodes().iterator();
+            while (iter12.hasNext()) {
+                RoutingNode node = iter12.next();
+                logger.info("--> node {} has {} shards",
+                        node.nodeId(), resp12.getState().getRoutingNodes().node(node.nodeId()).numberOfOwningShards());
+                nodesToShardCount.put(node.nodeId(), resp12.getState().getRoutingNodes().node(node.nodeId()).numberOfOwningShards());
             }
+            assertThat("node1 has 5 shards", nodesToShardCount.get(realNodeNames.get(0)), equalTo(5));
+            assertThat("node2 has 5 shards", nodesToShardCount.get(realNodeNames.get(1)), equalTo(5));
+            assertThat("node3 has 0 shards", nodesToShardCount.get(realNodeNames.get(2)), equalTo(0));
         });
 
         // Update the disk usages so one node is now back under the high watermark
@@ -138,21 +129,18 @@ public void run() {
         // Retrieve the count of shards on each node
         nodesToShardCount.clear();
 
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                ClusterStateResponse resp = client().admin().cluster().prepareState().get();
-                Iterator<RoutingNode> iter = resp.getState().getRoutingNodes().iterator();
-                while (iter.hasNext()) {
-                    RoutingNode node = iter.next();
-                    logger.info("--> node {} has {} shards",
-                            node.nodeId(), resp.getState().getRoutingNodes().node(node.nodeId()).numberOfOwningShards());
-                    nodesToShardCount.put(node.nodeId(), resp.getState().getRoutingNodes().node(node.nodeId()).numberOfOwningShards());
-                }
-                assertThat("node1 has at least 3 shards", nodesToShardCount.get(realNodeNames.get(0)), greaterThanOrEqualTo(3));
-                assertThat("node2 has at least 3 shards", nodesToShardCount.get(realNodeNames.get(1)), greaterThanOrEqualTo(3));
-                assertThat("node3 has at least 3 shards", nodesToShardCount.get(realNodeNames.get(2)), greaterThanOrEqualTo(3));
+        assertBusy(() -> {
+            ClusterStateResponse resp1 = client().admin().cluster().prepareState().get();
+            Iterator<RoutingNode> iter1 = resp1.getState().getRoutingNodes().iterator();
+            while (iter1.hasNext()) {
+                RoutingNode node = iter1.next();
+                logger.info("--> node {} has {} shards",
+                        node.nodeId(), resp1.getState().getRoutingNodes().node(node.nodeId()).numberOfOwningShards());
+                nodesToShardCount.put(node.nodeId(), resp1.getState().getRoutingNodes().node(node.nodeId()).numberOfOwningShards());
             }
+            assertThat("node1 has at least 3 shards", nodesToShardCount.get(realNodeNames.get(0)), greaterThanOrEqualTo(3));
+            assertThat("node2 has at least 3 shards", nodesToShardCount.get(realNodeNames.get(1)), greaterThanOrEqualTo(3));
+            assertThat("node3 has at least 3 shards", nodesToShardCount.get(realNodeNames.get(2)), greaterThanOrEqualTo(3));
         });
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/UpdateShardAllocationSettingsIT.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/UpdateShardAllocationSettingsIT.java
new file mode 100644
index 0000000000000..cd2fdfbe01491
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/UpdateShardAllocationSettingsIT.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.cluster.routing.allocation.decider;
+
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ESIntegTestCase;
+
+import java.util.Set;
+
+import static org.elasticsearch.cluster.routing.allocation.decider.SameShardAllocationDecider.CLUSTER_ROUTING_ALLOCATION_SAME_HOST_SETTING;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.equalTo;
+
+/**
+ * An integration test for testing updating shard allocation/routing settings and
+ * ensuring the updated settings take effect as expected.
+ */
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0)
+public class UpdateShardAllocationSettingsIT extends ESIntegTestCase {
+
+    /**
+     * Tests that updating the {@link EnableAllocationDecider} related settings works as expected.
+     */
+    public void testEnableRebalance() throws InterruptedException {
+        final String firstNode = internalCluster().startNode();
+        client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder()
+                .put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), EnableAllocationDecider.Rebalance.NONE))
+                .get();
+        // we test with 2 shards since otherwise it's pretty fragile if there are difference in the num or shards such that
+        // all shards are relocated to the second node which is not what we want here. It's solely a test for the settings to take effect
+        final int numShards = 2;
+        assertAcked(prepareCreate("test").setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, numShards)));
+        assertAcked(prepareCreate("test_1").setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, numShards)));
+        ensureGreen();
+        assertAllShardsOnNodes("test", firstNode);
+        assertAllShardsOnNodes("test_1", firstNode);
+
+        final String secondNode = internalCluster().startNode();
+        // prevent via index setting but only on index test
+        client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder()
+                .put(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), EnableAllocationDecider.Rebalance.NONE))
+                .get();
+        client().admin().cluster().prepareReroute().get();
+        ensureGreen();
+        assertAllShardsOnNodes("test", firstNode);
+        assertAllShardsOnNodes("test_1", firstNode);
+
+        // now enable the index test to relocate since index settings override cluster settings
+        client().admin().indices().prepareUpdateSettings("test")
+                .setSettings(Settings.builder().put(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING.getKey(),
+                        randomBoolean() ? EnableAllocationDecider.Rebalance.PRIMARIES : EnableAllocationDecider.Rebalance.ALL))
+                .get();
+        logger.info("--> balance index [test]");
+        client().admin().cluster().prepareReroute().get();
+        ensureGreen("test");
+        Set<String> test = assertAllShardsOnNodes("test", firstNode, secondNode);
+        assertThat("index: [test] expected to be rebalanced on both nodes", test.size(), equalTo(2));
+
+        // flip the cluster wide setting such that we can also balance for index
+        // test_1 eventually we should have one shard of each index on each node
+        client().admin().cluster().prepareUpdateSettings()
+                .setTransientSettings(Settings.builder().put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(),
+                        randomBoolean() ? EnableAllocationDecider.Rebalance.PRIMARIES : EnableAllocationDecider.Rebalance.ALL))
+                .get();
+        logger.info("--> balance index [test_1]");
+        client().admin().cluster().prepareReroute().get();
+        ensureGreen("test_1");
+        Set<String> test_1 = assertAllShardsOnNodes("test_1", firstNode, secondNode);
+        assertThat("index: [test_1] expected to be rebalanced on both nodes", test_1.size(), equalTo(2));
+
+        test = assertAllShardsOnNodes("test", firstNode, secondNode);
+        assertThat("index: [test] expected to be rebalanced on both nodes", test.size(), equalTo(2));
+    }
+
+    /**
+     * Tests that updating the {@link SameShardAllocationDecider#CLUSTER_ROUTING_ALLOCATION_SAME_HOST_SETTING} setting works as expected.
+     */
+    public void testUpdateSameHostSetting() {
+        internalCluster().startNodes(2);
+        // same same_host to true, since 2 nodes are started on the same host,
+        // only primaries should be assigned
+        client().admin().cluster().prepareUpdateSettings().setTransientSettings(
+            Settings.builder().put(CLUSTER_ROUTING_ALLOCATION_SAME_HOST_SETTING.getKey(), true)
+        ).get();
+        final String indexName = "idx";
+        client().admin().indices().prepareCreate(indexName).setSettings(
+            Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
+        ).get();
+        ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
+        assertFalse("replica should be unassigned",
+            clusterState.getRoutingTable().index(indexName).shardsWithState(ShardRoutingState.UNASSIGNED).isEmpty());
+        // now, update the same_host setting to allow shards to be allocated to multiple nodes on
+        // the same host - the replica should get assigned
+        client().admin().cluster().prepareUpdateSettings().setTransientSettings(
+            Settings.builder().put(CLUSTER_ROUTING_ALLOCATION_SAME_HOST_SETTING.getKey(), false)
+        ).get();
+        clusterState = client().admin().cluster().prepareState().get().getState();
+        assertTrue("all shards should be assigned",
+            clusterState.getRoutingTable().index(indexName).shardsWithState(ShardRoutingState.UNASSIGNED).isEmpty());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java b/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java
index 3259598f694f6..ae46e2993d293 100644
--- a/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java
@@ -20,19 +20,33 @@
 package org.elasticsearch.cluster.serialization;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.Diff;
+import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.RestoreInProgress;
+import org.elasticsearch.cluster.SnapshotDeletionsInProgress;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.snapshots.Snapshot;
+import org.elasticsearch.snapshots.SnapshotId;
+
+import java.util.Collections;
 
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
 
 /**
  *
@@ -54,11 +68,12 @@ public void testClusterStateSerialization() throws Exception {
         AllocationService strategy = createAllocationService();
         clusterState = ClusterState.builder(clusterState).routingTable(strategy.reroute(clusterState, "reroute").routingTable()).build();
 
-        ClusterState serializedClusterState = ClusterState.Builder.fromBytes(ClusterState.Builder.toBytes(clusterState), newNode("node1"));
+        ClusterState serializedClusterState = ClusterState.Builder.fromBytes(ClusterState.Builder.toBytes(clusterState), newNode("node1"),
+            new NamedWriteableRegistry(ClusterModule.getNamedWriteables()));
 
         assertThat(serializedClusterState.getClusterName().value(), equalTo(clusterState.getClusterName().value()));
 
-        assertThat(serializedClusterState.routingTable().prettyPrint(), equalTo(clusterState.routingTable().prettyPrint()));
+        assertThat(serializedClusterState.routingTable().toString(), equalTo(clusterState.routingTable().toString()));
     }
 
     public void testRoutingTableSerialization() throws Exception {
@@ -81,9 +96,69 @@ public void testRoutingTableSerialization() throws Exception {
         BytesStreamOutput outStream = new BytesStreamOutput();
         source.writeTo(outStream);
         StreamInput inStream = outStream.bytes().streamInput();
-        RoutingTable target = RoutingTable.Builder.readFrom(inStream);
+        RoutingTable target = RoutingTable.readFrom(inStream);
 
-        assertThat(target.prettyPrint(), equalTo(source.prettyPrint()));
+        assertThat(target.toString(), equalTo(source.toString()));
+    }
+
+    public void testSnapshotDeletionsInProgressSerialization() throws Exception {
+
+        boolean includeRestore = randomBoolean();
+
+        ClusterState.Builder builder = ClusterState.builder(ClusterState.EMPTY_STATE)
+            .putCustom(SnapshotDeletionsInProgress.TYPE,
+                SnapshotDeletionsInProgress.newInstance(
+                    new SnapshotDeletionsInProgress.Entry(
+                        new Snapshot("repo1", new SnapshotId("snap1", UUIDs.randomBase64UUID())),
+                        randomNonNegativeLong(), randomNonNegativeLong())
+                ));
+        if (includeRestore) {
+            builder.putCustom(RestoreInProgress.TYPE,
+                new RestoreInProgress(
+                    new RestoreInProgress.Entry(
+                        new Snapshot("repo2", new SnapshotId("snap2", UUIDs.randomBase64UUID())),
+                        RestoreInProgress.State.STARTED,
+                        Collections.singletonList("index_name"),
+                        ImmutableOpenMap.of()
+                    )
+                ));
+        }
+
+        ClusterState clusterState = builder.incrementVersion().build();
+
+        Diff<ClusterState> diffs = clusterState.diff(ClusterState.EMPTY_STATE);
+
+        // serialize with current version
+        BytesStreamOutput outStream = new BytesStreamOutput();
+        diffs.writeTo(outStream);
+        StreamInput inStream = outStream.bytes().streamInput();
+        inStream = new NamedWriteableAwareStreamInput(inStream, new NamedWriteableRegistry(ClusterModule.getNamedWriteables()));
+        Diff<ClusterState> serializedDiffs = ClusterState.readDiffFrom(inStream, clusterState.nodes().getLocalNode());
+        ClusterState stateAfterDiffs = serializedDiffs.apply(ClusterState.EMPTY_STATE);
+        assertThat(stateAfterDiffs.custom(RestoreInProgress.TYPE), includeRestore ? notNullValue() : nullValue());
+        assertThat(stateAfterDiffs.custom(SnapshotDeletionsInProgress.TYPE), notNullValue());
+
+        // serialize with old version
+        outStream = new BytesStreamOutput();
+        outStream.setVersion(Version.CURRENT.minimumCompatibilityVersion());
+        diffs.writeTo(outStream);
+        inStream = outStream.bytes().streamInput();
+        inStream = new NamedWriteableAwareStreamInput(inStream, new NamedWriteableRegistry(ClusterModule.getNamedWriteables()));
+        serializedDiffs = ClusterState.readDiffFrom(inStream, clusterState.nodes().getLocalNode());
+        stateAfterDiffs = serializedDiffs.apply(ClusterState.EMPTY_STATE);
+        assertThat(stateAfterDiffs.custom(RestoreInProgress.TYPE), includeRestore ? notNullValue() : nullValue());
+        assertThat(stateAfterDiffs.custom(SnapshotDeletionsInProgress.TYPE), nullValue());
+
+        // remove the custom and try serializing again with old version
+        clusterState = ClusterState.builder(clusterState).removeCustom(SnapshotDeletionsInProgress.TYPE).incrementVersion().build();
+        outStream = new BytesStreamOutput();
+        diffs.writeTo(outStream);
+        inStream = outStream.bytes().streamInput();
+        inStream = new NamedWriteableAwareStreamInput(inStream, new NamedWriteableRegistry(ClusterModule.getNamedWriteables()));
+        serializedDiffs = ClusterState.readDiffFrom(inStream, clusterState.nodes().getLocalNode());
+        stateAfterDiffs = serializedDiffs.apply(stateAfterDiffs);
+        assertThat(stateAfterDiffs.custom(RestoreInProgress.TYPE), includeRestore ? notNullValue() : nullValue());
+        assertThat(stateAfterDiffs.custom(SnapshotDeletionsInProgress.TYPE), nullValue());
     }
 
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterStateToStringTests.java b/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterStateToStringTests.java
index 7c11e2b8c239f..c726310d9217d 100644
--- a/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterStateToStringTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterStateToStringTests.java
@@ -21,6 +21,7 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
@@ -28,9 +29,9 @@
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.LocalTransportAddress;
-import org.elasticsearch.cluster.ESAllocationTestCase;
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
@@ -59,7 +60,7 @@ public void testClusterStateSerialization() throws Exception {
         AllocationService strategy = createAllocationService();
         clusterState = ClusterState.builder(clusterState).routingTable(strategy.reroute(clusterState, "reroute").routingTable()).build();
 
-        String clusterStateString = clusterState.toString();
+        String clusterStateString = Strings.toString(clusterState);
         assertNotNull(clusterStateString);
 
         assertThat(clusterStateString, containsString("test_idx"));
diff --git a/core/src/test/java/org/elasticsearch/cluster/serialization/DiffableTests.java b/core/src/test/java/org/elasticsearch/cluster/serialization/DiffableTests.java
index 611c261e33463..4475aed7dc398 100644
--- a/core/src/test/java/org/elasticsearch/cluster/serialization/DiffableTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/serialization/DiffableTests.java
@@ -65,7 +65,7 @@ protected MapDiff diff(Map<Integer, TestDiffable> before, Map<Integer, TestDiffa
             @Override
             protected MapDiff readDiff(StreamInput in) throws IOException {
                 return useProtoForDiffableSerialization
-                        ? DiffableUtils.readJdkMapDiff(in, keySerializer, TestDiffable.PROTO)
+                        ? DiffableUtils.readJdkMapDiff(in, keySerializer, TestDiffable::readFrom, TestDiffable::readDiffFrom)
                         : DiffableUtils.readJdkMapDiff(in, keySerializer, diffableValueSerializer());
             }
         }.execute();
@@ -113,7 +113,7 @@ protected MapDiff diff(ImmutableOpenMap<Integer, TestDiffable> before, Immutable
             @Override
             protected MapDiff readDiff(StreamInput in) throws IOException {
                 return useProtoForDiffableSerialization
-                        ? DiffableUtils.readImmutableOpenMapDiff(in, keySerializer, TestDiffable.PROTO)
+                        ? DiffableUtils.readImmutableOpenMapDiff(in, keySerializer, TestDiffable::readFrom, TestDiffable::readDiffFrom)
                         : DiffableUtils.readImmutableOpenMapDiff(in, keySerializer, diffableValueSerializer());
             }
         }.execute();
@@ -161,7 +161,7 @@ protected MapDiff diff(ImmutableOpenIntMap<TestDiffable> before, ImmutableOpenIn
             @Override
             protected MapDiff readDiff(StreamInput in) throws IOException {
                 return useProtoForDiffableSerialization
-                        ? DiffableUtils.readImmutableOpenIntMapDiff(in, keySerializer, TestDiffable.PROTO)
+                        ? DiffableUtils.readImmutableOpenIntMapDiff(in, keySerializer, TestDiffable::readFrom, TestDiffable::readDiffFrom)
                         : DiffableUtils.readImmutableOpenIntMapDiff(in, keySerializer, diffableValueSerializer());
             }
         }.execute();
@@ -398,7 +398,7 @@ public TestDiffable read(StreamInput in, K key) throws IOException {
 
             @Override
             public Diff<TestDiffable> readDiff(StreamInput in, K key) throws IOException {
-                return AbstractDiffable.readDiffFrom(TestDiffable.PROTO, in);
+                return AbstractDiffable.readDiffFrom(TestDiffable::readFrom, in);
             }
         };
     }
@@ -419,8 +419,6 @@ public String read(StreamInput in, K key) throws IOException {
 
     public static class TestDiffable extends AbstractDiffable<TestDiffable> {
 
-        public static final TestDiffable PROTO = new TestDiffable("");
-
         private final String value;
 
         public TestDiffable(String value) {
@@ -431,11 +429,14 @@ public String value() {
             return value;
         }
 
-        @Override
-        public TestDiffable readFrom(StreamInput in) throws IOException {
+        public static TestDiffable readFrom(StreamInput in) throws IOException {
             return new TestDiffable(in.readString());
         }
 
+        public static Diff<TestDiffable> readDiffFrom(StreamInput in) throws IOException {
+            return readDiffFrom(TestDiffable::readFrom, in);
+        }
+
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             out.writeString(value);
diff --git a/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceIT.java b/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceIT.java
index cf80e8378abb6..9df462db2b26f 100644
--- a/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceIT.java
@@ -20,32 +20,20 @@
 
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
-import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksResponse;
 import org.elasticsearch.cluster.AckedClusterStateUpdateTask;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
-import org.elasticsearch.cluster.LocalNodeMasterListener;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.Priority;
-import org.elasticsearch.common.component.AbstractLifecycleComponent;
-import org.elasticsearch.common.component.LifecycleComponent;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.inject.Singleton;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.discovery.zen.ZenDiscovery;
-import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
 import org.elasticsearch.test.junit.annotations.TestLogging;
-import org.elasticsearch.threadpool.ThreadPool;
 
-import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
@@ -56,23 +44,13 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.notNullValue;
 
 @ClusterScope(scope = Scope.TEST, numDataNodes = 0)
 @ESIntegTestCase.SuppressLocalMode
 public class ClusterServiceIT extends ESIntegTestCase {
 
-    @Override
-    protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Arrays.asList(TestPlugin.class);
-    }
-
     public void testAckedUpdateTask() throws Exception {
-        Settings settings = Settings.builder()
-                .put("discovery.type", "local")
-                .build();
-        internalCluster().startNode(settings);
+        internalCluster().startNode();
         ClusterService clusterService = internalCluster().getInstance(ClusterService.class);
 
         final AtomicBoolean allNodesAcked = new AtomicBoolean(false);
@@ -145,10 +123,7 @@ public void onFailure(String source, Exception e) {
     }
 
     public void testAckedUpdateTaskSameClusterState() throws Exception {
-        Settings settings = Settings.builder()
-                .put("discovery.type", "local")
-                .build();
-        internalCluster().startNode(settings);
+        internalCluster().startNode();
         ClusterService clusterService = internalCluster().getInstance(ClusterService.class);
 
         final AtomicBoolean allNodesAcked = new AtomicBoolean(false);
@@ -216,10 +191,7 @@ public void onFailure(String source, Exception e) {
     }
 
     public void testAckedUpdateTaskNoAckExpected() throws Exception {
-        Settings settings = Settings.builder()
-                .put("discovery.type", "local")
-                .build();
-        internalCluster().startNode(settings);
+        internalCluster().startNode();
         ClusterService clusterService = internalCluster().getInstance(ClusterService.class);
 
         final AtomicBoolean allNodesAcked = new AtomicBoolean(false);
@@ -288,10 +260,7 @@ public void onFailure(String source, Exception e) {
     }
 
     public void testAckedUpdateTaskTimeoutZero() throws Exception {
-        Settings settings = Settings.builder()
-                .put("discovery.type", "local")
-                .build();
-        internalCluster().startNode(settings);
+        internalCluster().startNode();
         ClusterService clusterService = internalCluster().getInstance(ClusterService.class);
 
         final AtomicBoolean allNodesAcked = new AtomicBoolean(false);
@@ -363,13 +332,10 @@ public void onFailure(String source, Exception e) {
         assertThat(processedLatch.await(1, TimeUnit.SECONDS), equalTo(true));
     }
 
-    @TestLogging("_root:debug,action.admin.cluster.tasks:trace")
+    @TestLogging("_root:debug,org.elasticsearch.action.admin.cluster.tasks:trace")
     public void testPendingUpdateTask() throws Exception {
-        Settings settings = Settings.builder()
-                .put("discovery.type", "local")
-                .build();
-        String node_0 = internalCluster().startNode(settings);
-        internalCluster().startCoordinatingOnlyNode(settings);
+        String node_0 = internalCluster().startNode();
+        internalCluster().startCoordinatingOnlyNode(Settings.EMPTY);
 
         final ClusterService clusterService = internalCluster().getInstance(ClusterService.class, node_0);
         final CountDownLatch block1 = new CountDownLatch(1);
@@ -498,142 +464,4 @@ public void onFailure(String source, Exception e) {
         assertTrue(controlSources.isEmpty());
         block2.countDown();
     }
-
-    public void testLocalNodeMasterListenerCallbacks() throws Exception {
-        Settings settings = Settings.builder()
-                .put("discovery.type", "zen")
-                .put("discovery.zen.minimum_master_nodes", 1)
-                .put(ZenDiscovery.PING_TIMEOUT_SETTING.getKey(), "400ms")
-                .put("discovery.initial_state_timeout", "500ms")
-                .build();
-
-        String node_0 = internalCluster().startNode(settings);
-        ClusterService clusterService = internalCluster().getInstance(ClusterService.class);
-        MasterAwareService testService = internalCluster().getInstance(MasterAwareService.class);
-
-        ClusterHealthResponse clusterHealth = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID)
-                .setWaitForNodes("1").get();
-        assertThat(clusterHealth.isTimedOut(), equalTo(false));
-
-        // the first node should be a master as the minimum required is 1
-        assertThat(clusterService.state().nodes().getMasterNode(), notNullValue());
-        assertThat(clusterService.state().nodes().isLocalNodeElectedMaster(), is(true));
-        assertThat(testService.master(), is(true));
-        String node_1 = internalCluster().startNode(settings);
-        final ClusterService clusterService1 = internalCluster().getInstance(ClusterService.class, node_1);
-        MasterAwareService testService1 = internalCluster().getInstance(MasterAwareService.class, node_1);
-
-        clusterHealth = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForNodes("2").get();
-        assertThat(clusterHealth.isTimedOut(), equalTo(false));
-
-        // the second node should not be the master as node1 is already the master.
-        assertThat(clusterService1.state().nodes().isLocalNodeElectedMaster(), is(false));
-        assertThat(testService1.master(), is(false));
-
-        internalCluster().stopCurrentMasterNode();
-        clusterHealth = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForNodes("1").get();
-        assertThat(clusterHealth.isTimedOut(), equalTo(false));
-
-        // now that node0 is closed, node1 should be elected as master
-        assertThat(clusterService1.state().nodes().isLocalNodeElectedMaster(), is(true));
-        assertThat(testService1.master(), is(true));
-
-        // start another node and set min_master_node
-        internalCluster().startNode(Settings.builder().put(settings));
-        assertFalse(client().admin().cluster().prepareHealth().setWaitForNodes("2").get().isTimedOut());
-
-        Settings transientSettings = Settings.builder()
-                .put("discovery.zen.minimum_master_nodes", 2)
-                .build();
-        client().admin().cluster().prepareUpdateSettings().setTransientSettings(transientSettings).get();
-
-        // and shutdown the second node
-        internalCluster().stopRandomNonMasterNode();
-
-        // there should not be any master as the minimum number of required eligible masters is not met
-        awaitBusy(() -> clusterService1.state().nodes().getMasterNode() == null &&
-                clusterService1.state().status() == ClusterState.ClusterStateStatus.APPLIED);
-        assertThat(testService1.master(), is(false));
-
-        // bring the node back up
-        String node_2 = internalCluster().startNode(Settings.builder().put(settings).put(transientSettings));
-        ClusterService clusterService2 = internalCluster().getInstance(ClusterService.class, node_2);
-        MasterAwareService testService2 = internalCluster().getInstance(MasterAwareService.class, node_2);
-
-        // make sure both nodes see each other otherwise the masternode below could be null if node 2 is master and node 1 did'r receive
-        // the updated cluster state...
-        assertThat(internalCluster().client(node_1).admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setLocal(true)
-                .setWaitForNodes("2").get().isTimedOut(), is(false));
-        assertThat(internalCluster().client(node_2).admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setLocal(true)
-                .setWaitForNodes("2").get().isTimedOut(), is(false));
-
-        // now that we started node1 again, a new master should be elected
-        assertThat(clusterService2.state().nodes().getMasterNode(), is(notNullValue()));
-        if (node_2.equals(clusterService2.state().nodes().getMasterNode().getName())) {
-            assertThat(testService1.master(), is(false));
-            assertThat(testService2.master(), is(true));
-        } else {
-            assertThat(testService1.master(), is(true));
-            assertThat(testService2.master(), is(false));
-        }
-    }
-
-    public static class TestPlugin extends Plugin {
-
-        @Override
-        public Collection<Class<? extends LifecycleComponent>> getGuiceServiceClasses() {
-            List<Class<? extends LifecycleComponent>> services = new ArrayList<>(1);
-            services.add(MasterAwareService.class);
-            return services;
-        }
-    }
-
-    @Singleton
-    public static class MasterAwareService extends AbstractLifecycleComponent implements LocalNodeMasterListener {
-
-        private final ClusterService clusterService;
-        private volatile boolean master;
-
-        @Inject
-        public MasterAwareService(Settings settings, ClusterService clusterService) {
-            super(settings);
-            clusterService.add(this);
-            this.clusterService = clusterService;
-            logger.info("initialized test service");
-        }
-
-        @Override
-        public void onMaster() {
-            logger.info("on master [{}]", clusterService.localNode());
-            master = true;
-        }
-
-        @Override
-        public void offMaster() {
-            logger.info("off master [{}]", clusterService.localNode());
-            master = false;
-        }
-
-        public boolean master() {
-            return master;
-        }
-
-        @Override
-        protected void doStart() {
-        }
-
-        @Override
-        protected void doStop() {
-        }
-
-        @Override
-        protected void doClose() {
-        }
-
-        @Override
-        public String executorName() {
-            return ThreadPool.Names.SAME;
-        }
-
-    }
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceTests.java
index c4c09275e8063..b8f85f733c394 100644
--- a/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceTests.java
@@ -19,30 +19,34 @@
 package org.elasticsearch.cluster.service;
 
 import org.apache.logging.log4j.Level;
-import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateObserver;
 import org.elasticsearch.cluster.ClusterStateTaskConfig;
 import org.elasticsearch.cluster.ClusterStateTaskExecutor;
 import org.elasticsearch.cluster.ClusterStateTaskListener;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.LocalClusterUpdateTask;
+import org.elasticsearch.cluster.LocalNodeMasterListener;
 import org.elasticsearch.cluster.NodeConnectionsService;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.lease.Releasable;
-import org.elasticsearch.common.logging.TestLoggers;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.BaseFuture;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.discovery.Discovery;
+import org.elasticsearch.discovery.DiscoverySettings;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.MockLogAppender;
 import org.elasticsearch.test.junit.annotations.TestLogging;
@@ -62,23 +66,21 @@
 import java.util.concurrent.BrokenBarrierException;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.CyclicBarrier;
 import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
-import java.util.stream.Collectors;
+import java.util.concurrent.atomic.AtomicReference;
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
 import static org.elasticsearch.test.ClusterServiceUtils.setState;
 import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasKey;
-import static org.hamcrest.Matchers.hasToString;
 import static org.hamcrest.Matchers.is;
 
 public class ClusterServiceTests extends ESTestCase {
@@ -114,22 +116,22 @@ public void tearDown() throws Exception {
     TimedClusterService createTimedClusterService(boolean makeMaster) throws InterruptedException {
         TimedClusterService timedClusterService = new TimedClusterService(Settings.builder().put("cluster.name",
             "ClusterServiceTests").build(), new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
-            threadPool);
-        timedClusterService.setLocalNode(new DiscoveryNode("node1", LocalTransportAddress.buildUnique(), emptyMap(),
-            emptySet(), Version.CURRENT));
+            threadPool, () -> new DiscoveryNode("node1",  LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT));
         timedClusterService.setNodeConnectionsService(new NodeConnectionsService(Settings.EMPTY, null, null) {
             @Override
-            public void connectToAddedNodes(ClusterChangedEvent event) {
+            public void connectToNodes(DiscoveryNodes discoveryNodes) {
                 // skip
             }
 
             @Override
-            public void disconnectFromRemovedNodes(ClusterChangedEvent event) {
+            public void disconnectFromNodesExcept(DiscoveryNodes nodesToKeep) {
                 // skip
             }
         });
         timedClusterService.setClusterStatePublisher((event, ackListener) -> {
         });
+        timedClusterService.setDiscoverySettings(new DiscoverySettings(Settings.EMPTY,
+            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)));
         timedClusterService.start();
         ClusterState state = timedClusterService.state();
         final DiscoveryNodes nodes = state.nodes();
@@ -141,70 +143,6 @@ public void disconnectFromRemovedNodes(ClusterChangedEvent event) {
         return timedClusterService;
     }
 
-    public void testTimeoutUpdateTask() throws Exception {
-        final CountDownLatch block = new CountDownLatch(1);
-        clusterService.submitStateUpdateTask("test1", new ClusterStateUpdateTask() {
-            @Override
-            public ClusterState execute(ClusterState currentState) {
-                try {
-                    block.await();
-                } catch (InterruptedException e) {
-                    throw new RuntimeException(e);
-                }
-                return currentState;
-            }
-
-            @Override
-            public void onFailure(String source, Exception e) {
-                throw new RuntimeException(e);
-            }
-        });
-
-        final CountDownLatch timedOut = new CountDownLatch(1);
-        final AtomicBoolean executeCalled = new AtomicBoolean();
-        clusterService.submitStateUpdateTask("test2", new ClusterStateUpdateTask() {
-            @Override
-            public TimeValue timeout() {
-                return TimeValue.timeValueMillis(2);
-            }
-
-            @Override
-            public void onFailure(String source, Exception e) {
-                timedOut.countDown();
-            }
-
-            @Override
-            public ClusterState execute(ClusterState currentState) {
-                executeCalled.set(true);
-                return currentState;
-            }
-
-            @Override
-            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
-            }
-        });
-
-        timedOut.await();
-        block.countDown();
-        final CountDownLatch allProcessed = new CountDownLatch(1);
-        clusterService.submitStateUpdateTask("test3", new ClusterStateUpdateTask() {
-            @Override
-            public void onFailure(String source, Exception e) {
-                throw new RuntimeException(e);
-            }
-
-            @Override
-            public ClusterState execute(ClusterState currentState) {
-                allProcessed.countDown();
-                return currentState;
-            }
-
-        });
-        allProcessed.await(); // executed another task to double check that execute on the timed out update task is not called...
-        assertThat(executeCalled.get(), equalTo(false));
-    }
-
-
     public void testMasterAwareExecution() throws Exception {
         ClusterService nonMaster = createTimedClusterService(false);
 
@@ -229,17 +167,12 @@ public void onFailure(String source, Exception e) {
 
         taskFailed[0] = true;
         final CountDownLatch latch2 = new CountDownLatch(1);
-        nonMaster.submitStateUpdateTask("test", new ClusterStateUpdateTask() {
+        nonMaster.submitStateUpdateTask("test", new LocalClusterUpdateTask() {
             @Override
-            public boolean runOnlyOnMaster() {
-                return false;
-            }
-
-            @Override
-            public ClusterState execute(ClusterState currentState) throws Exception {
+            public ClusterTasksResult<LocalClusterUpdateTask> execute(ClusterState currentState) throws Exception {
                 taskFailed[0] = false;
                 latch2.countDown();
-                return currentState;
+                return unchanged();
             }
 
             @Override
@@ -269,14 +202,9 @@ public void testClusterStateTaskListenerThrowingExceptionIsOkay() throws Interru
             ClusterStateTaskConfig.build(Priority.NORMAL),
             new ClusterStateTaskExecutor<Object>() {
                 @Override
-                public boolean runOnlyOnMaster() {
-                    return false;
-                }
-
-                @Override
-                public BatchResult<Object> execute(ClusterState currentState, List<Object> tasks) throws Exception {
+                public ClusterTasksResult<Object> execute(ClusterState currentState, List<Object> tasks) throws Exception {
                     ClusterState newClusterState = ClusterState.builder(currentState).build();
-                    return BatchResult.builder().successes(tasks).build(newClusterState);
+                    return ClusterTasksResult.builder().successes(tasks).build(newClusterState);
                 }
 
                 @Override
@@ -301,112 +229,49 @@ public void onFailure(String source, Exception e) {
         assertTrue(published.get());
     }
 
-    // test that for a single thread, tasks are executed in the order
-    // that they are submitted
-    public void testClusterStateUpdateTasksAreExecutedInOrder() throws BrokenBarrierException, InterruptedException {
-        class TaskExecutor implements ClusterStateTaskExecutor<Integer> {
-            List<Integer> tasks = new ArrayList<>();
-
-            @Override
-            public BatchResult<Integer> execute(ClusterState currentState, List<Integer> tasks) throws Exception {
-                this.tasks.addAll(tasks);
-                return BatchResult.<Integer>builder().successes(tasks).build(ClusterState.builder(currentState).build());
-            }
-
-            @Override
-            public boolean runOnlyOnMaster() {
-                return false;
-            }
-        }
-
-        int numberOfThreads = randomIntBetween(2, 8);
-        TaskExecutor[] executors = new TaskExecutor[numberOfThreads];
-        for (int i = 0; i < numberOfThreads; i++) {
-            executors[i] = new TaskExecutor();
-        }
-
-        int tasksSubmittedPerThread = randomIntBetween(2, 1024);
-
-        CopyOnWriteArrayList<Tuple<String, Throwable>> failures = new CopyOnWriteArrayList<>();
-        CountDownLatch updateLatch = new CountDownLatch(numberOfThreads * tasksSubmittedPerThread);
-
-        ClusterStateTaskListener listener = new ClusterStateTaskListener() {
-            @Override
-            public void onFailure(String source, Exception e) {
-                logger.error((Supplier<?>) () -> new ParameterizedMessage("unexpected failure: [{}]", source), e);
-                failures.add(new Tuple<>(source, e));
-                updateLatch.countDown();
-            }
-
-            @Override
-            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
-                updateLatch.countDown();
-            }
-        };
-
-        CyclicBarrier barrier = new CyclicBarrier(1 + numberOfThreads);
+    public void testBlockingCallInClusterStateTaskListenerFails() throws InterruptedException {
+        assumeTrue("assertions must be enabled for this test to work", BaseFuture.class.desiredAssertionStatus());
+        final CountDownLatch latch = new CountDownLatch(1);
+        final AtomicReference<AssertionError> assertionRef = new AtomicReference<>();
 
-        for (int i = 0; i < numberOfThreads; i++) {
-            final int index = i;
-            Thread thread = new Thread(() -> {
-                try {
-                    barrier.await();
-                    for (int j = 0; j < tasksSubmittedPerThread; j++) {
-                        clusterService.submitStateUpdateTask("[" + index + "][" + j + "]", j,
-                            ClusterStateTaskConfig.build(randomFrom(Priority.values())), executors[index], listener);
-                    }
-                    barrier.await();
-                } catch (InterruptedException | BrokenBarrierException e) {
-                    throw new AssertionError(e);
+        clusterService.submitStateUpdateTask(
+            "testBlockingCallInClusterStateTaskListenerFails",
+            new Object(),
+            ClusterStateTaskConfig.build(Priority.NORMAL),
+            new ClusterStateTaskExecutor<Object>() {
+                @Override
+                public ClusterTasksResult<Object> execute(ClusterState currentState, List<Object> tasks) throws Exception {
+                    ClusterState newClusterState = ClusterState.builder(currentState).build();
+                    return ClusterTasksResult.builder().successes(tasks).build(newClusterState);
                 }
-            });
-            thread.start();
-        }
-
-        // wait for all threads to be ready
-        barrier.await();
-        // wait for all threads to finish
-        barrier.await();
-
-        updateLatch.await();
-
-        assertThat(failures, empty());
-
-        for (int i = 0; i < numberOfThreads; i++) {
-            assertEquals(tasksSubmittedPerThread, executors[i].tasks.size());
-            for (int j = 0; j < tasksSubmittedPerThread; j++) {
-                assertNotNull(executors[i].tasks.get(j));
-                assertEquals("cluster state update task executed out of order", j, (int) executors[i].tasks.get(j));
-            }
-        }
-    }
-
-    public void testSingleBatchSubmission() throws InterruptedException {
-        Map<Integer, ClusterStateTaskListener> tasks = new HashMap<>();
-        final int numOfTasks = randomInt(10);
-        final CountDownLatch latch = new CountDownLatch(numOfTasks);
-        for (int i = 0; i < numOfTasks; i++) {
-            while (null != tasks.put(randomInt(1024), new ClusterStateTaskListener() {
+            },
+            new ClusterStateTaskListener() {
                 @Override
                 public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
-                    latch.countDown();
+                    BaseFuture<Void> future = new BaseFuture<Void>() {};
+                    try {
+                        if (randomBoolean()) {
+                            future.get(1L, TimeUnit.SECONDS);
+                        } else {
+                            future.get();
+                        }
+                    } catch (Exception e) {
+                        throw new RuntimeException(e);
+                    } catch (AssertionError e) {
+                        assertionRef.set(e);
+                        latch.countDown();
+                    }
                 }
 
                 @Override
                 public void onFailure(String source, Exception e) {
-                    fail(ExceptionsHelper.detailedMessage(e));
                 }
-            })) ;
-        }
-
-        clusterService.submitStateUpdateTasks("test", tasks, ClusterStateTaskConfig.build(Priority.LANGUID),
-            (currentState, taskList) -> {
-                assertThat(taskList.size(), equalTo(tasks.size()));
-                assertThat(taskList.stream().collect(Collectors.toSet()), equalTo(tasks.keySet()));
-                return ClusterStateTaskExecutor.BatchResult.<Integer>builder().successes(taskList).build(currentState);
-            });
+            }
+        );
 
         latch.await();
+        assertNotNull(assertionRef.get());
+        assertThat(assertionRef.get().getMessage(), containsString("not be the cluster state update thread. Reason: [Blocking operation]"));
     }
 
     public void testClusterStateBatchedUpdates() throws BrokenBarrierException, InterruptedException {
@@ -462,12 +327,12 @@ class TaskExecutor implements ClusterStateTaskExecutor<Task> {
             private AtomicInteger batches = new AtomicInteger();
             private AtomicInteger published = new AtomicInteger();
 
-            public TaskExecutor(List<Set<Task>> taskGroups) {
+            TaskExecutor(List<Set<Task>> taskGroups) {
                 this.taskGroups = taskGroups;
             }
 
             @Override
-            public BatchResult<Task> execute(ClusterState currentState, List<Task> tasks) throws Exception {
+            public ClusterTasksResult<Task> execute(ClusterState currentState, List<Task> tasks) throws Exception {
                 for (Set<Task> expectedSet : taskGroups) {
                     long count = tasks.stream().filter(expectedSet::contains).count();
                     assertThat("batched set should be executed together or not at all. Expected " + expectedSet + "s. Executing " + tasks,
@@ -481,12 +346,7 @@ public BatchResult<Task> execute(ClusterState currentState, List<Task> tasks) th
                     batches.incrementAndGet();
                     semaphore.acquire();
                 }
-                return BatchResult.<Task>builder().successes(tasks).build(maybeUpdatedClusterState);
-            }
-
-            @Override
-            public boolean runOnlyOnMaster() {
-                return false;
+                return ClusterTasksResult.<Task>builder().successes(tasks).build(maybeUpdatedClusterState);
             }
 
             @Override
@@ -607,88 +467,31 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS
         }
     }
 
-    /**
-     * Note, this test can only work as long as we have a single thread executor executing the state update tasks!
-     */
-    public void testPrioritizedTasks() throws Exception {
-        BlockingTask block = new BlockingTask(Priority.IMMEDIATE);
-        clusterService.submitStateUpdateTask("test", block);
-        int taskCount = randomIntBetween(5, 20);
-
-        // will hold all the tasks in the order in which they were executed
-        List<PrioritizedTask> tasks = new ArrayList<>(taskCount);
-        CountDownLatch latch = new CountDownLatch(taskCount);
-        for (int i = 0; i < taskCount; i++) {
-            Priority priority = randomFrom(Priority.values());
-            clusterService.submitStateUpdateTask("test", new PrioritizedTask(priority, latch, tasks));
-        }
-
-        block.close();
-        latch.await();
-
-        Priority prevPriority = null;
-        for (PrioritizedTask task : tasks) {
-            if (prevPriority == null) {
-                prevPriority = task.priority();
-            } else {
-                assertThat(task.priority().sameOrAfter(prevPriority), is(true));
-            }
-        }
-    }
-
-    public void testDuplicateSubmission() throws InterruptedException {
-        final CountDownLatch latch = new CountDownLatch(2);
-        try (BlockingTask blockingTask = new BlockingTask(Priority.IMMEDIATE)) {
-            clusterService.submitStateUpdateTask("blocking", blockingTask);
-
-            ClusterStateTaskExecutor<SimpleTask> executor = (currentState, tasks) ->
-                ClusterStateTaskExecutor.BatchResult.<SimpleTask>builder().successes(tasks).build(currentState);
-
-            SimpleTask task = new SimpleTask(1);
-            ClusterStateTaskListener listener = new ClusterStateTaskListener() {
-                @Override
-                public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
-                    latch.countDown();
-                }
-
-                @Override
-                public void onFailure(String source, Exception e) {
-                    fail(ExceptionsHelper.detailedMessage(e));
-                }
-            };
-
-            clusterService.submitStateUpdateTask("first time", task, ClusterStateTaskConfig.build(Priority.NORMAL), executor, listener);
-
-            final IllegalStateException e =
-                    expectThrows(
-                            IllegalStateException.class,
-                            () -> clusterService.submitStateUpdateTask(
-                                    "second time",
-                                    task,
-                                    ClusterStateTaskConfig.build(Priority.NORMAL),
-                                    executor, listener));
-            assertThat(e, hasToString(containsString("task [1] with source [second time] is already queued")));
-
-            clusterService.submitStateUpdateTask("third time a charm", new SimpleTask(1),
-                ClusterStateTaskConfig.build(Priority.NORMAL), executor, listener);
-
-            assertThat(latch.getCount(), equalTo(2L));
-        }
-        latch.await();
-    }
-
-    @TestLogging("cluster:TRACE") // To ensure that we log cluster state events on TRACE level
+    @TestLogging("org.elasticsearch.cluster.service:TRACE") // To ensure that we log cluster state events on TRACE level
     public void testClusterStateUpdateLogging() throws Exception {
         MockLogAppender mockAppender = new MockLogAppender();
-        mockAppender.addExpectation(new MockLogAppender.SeenEventExpectation("test1", "cluster.service", Level.DEBUG,
-            "*processing [test1]: took [1s] no change in cluster_state"));
-        mockAppender.addExpectation(new MockLogAppender.SeenEventExpectation("test2", "cluster.service", Level.TRACE,
-            "*failed to execute cluster state update in [2s]*"));
-        mockAppender.addExpectation(new MockLogAppender.SeenEventExpectation("test3", "cluster.service", Level.DEBUG,
-            "*processing [test3]: took [3s] done applying updated cluster_state (version: *, uuid: *)"));
-
-        Logger rootLogger = LogManager.getRootLogger();
-        TestLoggers.addAppender(rootLogger, mockAppender);
+        mockAppender.start();
+        mockAppender.addExpectation(
+                new MockLogAppender.SeenEventExpectation(
+                        "test1",
+                        "org.elasticsearch.cluster.service.ClusterServiceTests$TimedClusterService",
+                        Level.DEBUG,
+                        "*processing [test1]: took [1s] no change in cluster_state"));
+        mockAppender.addExpectation(
+                new MockLogAppender.SeenEventExpectation(
+                        "test2",
+                        "org.elasticsearch.cluster.service.ClusterServiceTests$TimedClusterService",
+                        Level.TRACE,
+                        "*failed to execute cluster state update in [2s]*"));
+        mockAppender.addExpectation(
+                new MockLogAppender.SeenEventExpectation(
+                        "test3",
+                        "org.elasticsearch.cluster.service.ClusterServiceTests$TimedClusterService",
+                        Level.DEBUG,
+                        "*processing [test3]: took [3s] done applying updated cluster_state (version: *, uuid: *)"));
+
+        Logger clusterLogger = Loggers.getLogger("org.elasticsearch.cluster.service");
+        Loggers.addAppender(clusterLogger, mockAppender);
         try {
             final CountDownLatch latch = new CountDownLatch(4);
             clusterService.currentTimeOverride = System.nanoTime();
@@ -743,7 +546,7 @@ public void onFailure(String source, Exception e) {
                     fail();
                 }
             });
-            // Additional update task to make sure all previous logging made it to the logger
+            // Additional update task to make sure all previous logging made it to the loggerName
             // We don't check logging for this on since there is no guarantee that it will occur before our check
             clusterService.submitStateUpdateTask("test4", new ClusterStateUpdateTask() {
                 @Override
@@ -763,25 +566,43 @@ public void onFailure(String source, Exception e) {
             });
             latch.await();
         } finally {
-            TestLoggers.removeAppender(rootLogger, mockAppender);
+            Loggers.removeAppender(clusterLogger, mockAppender);
+            mockAppender.stop();
         }
         mockAppender.assertAllExpectationsMatched();
     }
 
-    @TestLogging("cluster:WARN") // To ensure that we log cluster state events on WARN level
+    @TestLogging("org.elasticsearch.cluster.service:WARN") // To ensure that we log cluster state events on WARN level
     public void testLongClusterStateUpdateLogging() throws Exception {
         MockLogAppender mockAppender = new MockLogAppender();
-        mockAppender.addExpectation(new MockLogAppender.UnseenEventExpectation("test1 shouldn't see because setting is too low",
-            "cluster.service", Level.WARN, "*cluster state update task [test1] took [*] above the warn threshold of *"));
-        mockAppender.addExpectation(new MockLogAppender.SeenEventExpectation("test2", "cluster.service", Level.WARN,
-            "*cluster state update task [test2] took [32s] above the warn threshold of *"));
-        mockAppender.addExpectation(new MockLogAppender.SeenEventExpectation("test3", "cluster.service", Level.WARN,
-            "*cluster state update task [test3] took [33s] above the warn threshold of *"));
-        mockAppender.addExpectation(new MockLogAppender.SeenEventExpectation("test4", "cluster.service", Level.WARN,
-            "*cluster state update task [test4] took [34s] above the warn threshold of *"));
-
-        Logger rootLogger = LogManager.getRootLogger();
-        TestLoggers.addAppender(rootLogger, mockAppender);
+        mockAppender.start();
+        mockAppender.addExpectation(
+                new MockLogAppender.UnseenEventExpectation(
+                        "test1 shouldn't see because setting is too low",
+                        "org.elasticsearch.cluster.service.ClusterServiceTests$TimedClusterService",
+                        Level.WARN,
+                        "*cluster state update task [test1] took [*] above the warn threshold of *"));
+        mockAppender.addExpectation(
+                new MockLogAppender.SeenEventExpectation(
+                        "test2",
+                        "org.elasticsearch.cluster.service.ClusterServiceTests$TimedClusterService",
+                        Level.WARN,
+                        "*cluster state update task [test2] took [32s] above the warn threshold of *"));
+        mockAppender.addExpectation(
+                new MockLogAppender.SeenEventExpectation(
+                        "test3",
+                        "org.elasticsearch.cluster.service.ClusterServiceTests$TimedClusterService",
+                        Level.WARN,
+                        "*cluster state update task [test3] took [33s] above the warn threshold of *"));
+        mockAppender.addExpectation(
+                new MockLogAppender.SeenEventExpectation(
+                        "test4",
+                        "org.elasticsearch.cluster.service.ClusterServiceTests$TimedClusterService",
+                        Level.WARN,
+                        "*cluster state update task [test4] took [34s] above the warn threshold of *"));
+
+        Logger clusterLogger = Loggers.getLogger("org.elasticsearch.cluster.service");
+        Loggers.addAppender(clusterLogger, mockAppender);
         try {
             final CountDownLatch latch = new CountDownLatch(5);
             final CountDownLatch processedFirstTask = new CountDownLatch(1);
@@ -857,7 +678,7 @@ public void onFailure(String source, Exception e) {
                     fail();
                 }
             });
-            // Additional update task to make sure all previous logging made it to the logger
+            // Additional update task to make sure all previous logging made it to the loggerName
             // We don't check logging for this on since there is no guarantee that it will occur before our check
             clusterService.submitStateUpdateTask("test5", new ClusterStateUpdateTask() {
                 @Override
@@ -877,87 +698,216 @@ public void onFailure(String source, Exception e) {
             });
             latch.await();
         } finally {
-            TestLoggers.removeAppender(rootLogger, mockAppender);
+            Loggers.removeAppender(clusterLogger, mockAppender);
+            mockAppender.stop();
         }
         mockAppender.assertAllExpectationsMatched();
     }
 
-    private static class SimpleTask {
-        private final int id;
+    public void testDisconnectFromNewlyAddedNodesIfClusterStatePublishingFails() throws InterruptedException {
+        TimedClusterService timedClusterService = new TimedClusterService(Settings.builder().put("cluster.name",
+            "ClusterServiceTests").build(), new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
+            threadPool, () -> new DiscoveryNode("node1", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT));
+        Set<DiscoveryNode> currentNodes = new HashSet<>();
+        timedClusterService.setNodeConnectionsService(new NodeConnectionsService(Settings.EMPTY, null, null) {
+            @Override
+            public void connectToNodes(DiscoveryNodes discoveryNodes) {
+                discoveryNodes.forEach(currentNodes::add);
+            }
 
-        private SimpleTask(int id) {
-            this.id = id;
-        }
+            @Override
+            public void disconnectFromNodesExcept(DiscoveryNodes nodesToKeep) {
+                Set<DiscoveryNode> nodeSet = new HashSet<>();
+                nodesToKeep.iterator().forEachRemaining(nodeSet::add);
+                currentNodes.removeIf(node -> nodeSet.contains(node) == false);
+            }
+        });
+        AtomicBoolean failToCommit = new AtomicBoolean();
+        timedClusterService.setClusterStatePublisher((event, ackListener) -> {
+            if (failToCommit.get()) {
+                throw new Discovery.FailedToCommitClusterStateException("just to test this");
+            }
+        });
+        timedClusterService.setDiscoverySettings(new DiscoverySettings(Settings.EMPTY,
+            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)));
+        timedClusterService.start();
+        ClusterState state = timedClusterService.state();
+        final DiscoveryNodes nodes = state.nodes();
+        final DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(nodes)
+            .masterNodeId(nodes.getLocalNodeId());
+        state = ClusterState.builder(state).blocks(ClusterBlocks.EMPTY_CLUSTER_BLOCK)
+            .nodes(nodesBuilder).build();
+        setState(timedClusterService, state);
 
-        @Override
-        public int hashCode() {
-            return super.hashCode();
-        }
+        assertThat(currentNodes, equalTo(Sets.newHashSet(timedClusterService.state().getNodes())));
 
-        @Override
-        public boolean equals(Object obj) {
-            return super.equals(obj);
-        }
+        final CountDownLatch latch = new CountDownLatch(1);
 
-        @Override
-        public String toString() {
-            return Integer.toString(id);
-        }
+        // try to add node when cluster state publishing fails
+        failToCommit.set(true);
+        timedClusterService.submitStateUpdateTask("test", new ClusterStateUpdateTask() {
+            @Override
+            public ClusterState execute(ClusterState currentState) throws Exception {
+                DiscoveryNode newNode = new DiscoveryNode("node2", LocalTransportAddress.buildUnique(), emptyMap(),
+                    emptySet(), Version.CURRENT);
+                return ClusterState.builder(currentState).nodes(DiscoveryNodes.builder(currentState.nodes()).add(newNode)).build();
+            }
+
+            @Override
+            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                latch.countDown();
+            }
+
+            @Override
+            public void onFailure(String source, Exception e) {
+                latch.countDown();
+            }
+        });
+
+        latch.await();
+        assertThat(currentNodes, equalTo(Sets.newHashSet(timedClusterService.state().getNodes())));
+        timedClusterService.close();
     }
 
-    private static class BlockingTask extends ClusterStateUpdateTask implements Releasable {
-        private final CountDownLatch latch = new CountDownLatch(1);
+    public void testLocalNodeMasterListenerCallbacks() throws Exception {
+        TimedClusterService timedClusterService = createTimedClusterService(false);
 
-        public BlockingTask(Priority priority) {
-            super(priority);
-        }
+        AtomicBoolean isMaster = new AtomicBoolean();
+        timedClusterService.addLocalNodeMasterListener(new LocalNodeMasterListener() {
+            @Override
+            public void onMaster() {
+                isMaster.set(true);
+            }
 
-        @Override
-        public ClusterState execute(ClusterState currentState) throws Exception {
-            latch.await();
-            return currentState;
-        }
+            @Override
+            public void offMaster() {
+                isMaster.set(false);
+            }
 
-        @Override
-        public void onFailure(String source, Exception e) {
-        }
+            @Override
+            public String executorName() {
+                return ThreadPool.Names.SAME;
+            }
+        });
 
-        public void close() {
-            latch.countDown();
-        }
+        ClusterState state = timedClusterService.state();
+        DiscoveryNodes nodes = state.nodes();
+        DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(nodes).masterNodeId(nodes.getLocalNodeId());
+        state = ClusterState.builder(state).blocks(ClusterBlocks.EMPTY_CLUSTER_BLOCK).nodes(nodesBuilder).build();
+        setState(timedClusterService, state);
+        assertThat(isMaster.get(), is(true));
+
+        nodes = state.nodes();
+        nodesBuilder = DiscoveryNodes.builder(nodes).masterNodeId(null);
+        state = ClusterState.builder(state).blocks(ClusterBlocks.builder().addGlobalBlock(DiscoverySettings.NO_MASTER_BLOCK_WRITES))
+            .nodes(nodesBuilder).build();
+        setState(timedClusterService, state);
+        assertThat(isMaster.get(), is(false));
+        nodesBuilder = DiscoveryNodes.builder(nodes).masterNodeId(nodes.getLocalNodeId());
+        state = ClusterState.builder(state).blocks(ClusterBlocks.EMPTY_CLUSTER_BLOCK).nodes(nodesBuilder).build();
+        setState(timedClusterService, state);
+        assertThat(isMaster.get(), is(true));
 
+        timedClusterService.close();
     }
 
-    private static class PrioritizedTask extends ClusterStateUpdateTask {
+    public void testClusterStateApplierCantSampleClusterState() throws InterruptedException {
+        AtomicReference<Throwable> error = new AtomicReference<>();
+        AtomicBoolean applierCalled = new AtomicBoolean();
+        clusterService.addStateApplier(event -> {
+            try {
+                applierCalled.set(true);
+                clusterService.state();
+                error.set(new AssertionError("successfully sampled state"));
+            } catch (AssertionError e) {
+                if (e.getMessage().contains("should not be called by a cluster state applier") == false) {
+                    error.set(e);
+                }
+            }
+        });
 
-        private final CountDownLatch latch;
-        private final List<PrioritizedTask> tasks;
+        CountDownLatch latch = new CountDownLatch(1);
+        clusterService.submitStateUpdateTask("test", new ClusterStateUpdateTask() {
+            @Override
+            public ClusterState execute(ClusterState currentState) throws Exception {
+                return ClusterState.builder(currentState).build();
+            }
 
-        private PrioritizedTask(Priority priority, CountDownLatch latch, List<PrioritizedTask> tasks) {
-            super(priority);
-            this.latch = latch;
-            this.tasks = tasks;
-        }
+            @Override
+            public void onFailure(String source, Exception e) {
+                error.compareAndSet(null, e);
+            }
 
-        @Override
-        public ClusterState execute(ClusterState currentState) throws Exception {
-            tasks.add(this);
-            latch.countDown();
-            return currentState;
-        }
+            @Override
+            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                latch.countDown();
+            }
+        });
 
-        @Override
-        public void onFailure(String source, Exception e) {
-            latch.countDown();
-        }
+        latch.await();
+        assertNull(error.get());
+        assertTrue(applierCalled.get());
+    }
+
+    public void testClusterStateApplierCanCreateAnObserver() throws InterruptedException {
+        AtomicReference<Throwable> error = new AtomicReference<>();
+        AtomicBoolean applierCalled = new AtomicBoolean();
+        clusterService.addStateApplier(event -> {
+            try {
+                applierCalled.set(true);
+                ClusterStateObserver observer = new ClusterStateObserver(event.state(),
+                    clusterService, null, logger, threadPool.getThreadContext());
+                observer.waitForNextChange(new ClusterStateObserver.Listener() {
+                    @Override
+                    public void onNewClusterState(ClusterState state) {
+
+                    }
+
+                    @Override
+                    public void onClusterServiceClose() {
+
+                    }
+
+                    @Override
+                    public void onTimeout(TimeValue timeout) {
+
+                    }
+                });
+            } catch (AssertionError e) {
+                    error.set(e);
+            }
+        });
+
+        CountDownLatch latch = new CountDownLatch(1);
+        clusterService.submitStateUpdateTask("test", new ClusterStateUpdateTask() {
+            @Override
+            public ClusterState execute(ClusterState currentState) throws Exception {
+                return ClusterState.builder(currentState).build();
+            }
+
+            @Override
+            public void onFailure(String source, Exception e) {
+                error.compareAndSet(null, e);
+            }
+
+            @Override
+            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                latch.countDown();
+            }
+        });
+
+        latch.await();
+        assertNull(error.get());
+        assertTrue(applierCalled.get());
     }
 
     static class TimedClusterService extends ClusterService {
 
         public volatile Long currentTimeOverride = null;
 
-        public TimedClusterService(Settings settings, ClusterSettings clusterSettings, ThreadPool threadPool) {
-            super(settings, clusterSettings, threadPool);
+        TimedClusterService(Settings settings, ClusterSettings clusterSettings, ThreadPool threadPool,
+                                   Supplier<DiscoveryNode> localNodeSupplier) {
+            super(settings, clusterSettings, threadPool, localNodeSupplier);
         }
 
         @Override
diff --git a/core/src/test/java/org/elasticsearch/cluster/service/TaskBatcherTests.java b/core/src/test/java/org/elasticsearch/cluster/service/TaskBatcherTests.java
new file mode 100644
index 0000000000000..d5af9dd558155
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/cluster/service/TaskBatcherTests.java
@@ -0,0 +1,350 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.service;
+
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.apache.logging.log4j.util.Supplier;
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.cluster.ClusterStateTaskConfig;
+import org.elasticsearch.cluster.metadata.ProcessClusterEventTimeoutException;
+import org.elasticsearch.common.Priority;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor;
+import org.junit.Before;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.BrokenBarrierException;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.Semaphore;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasToString;
+
+public class TaskBatcherTests extends TaskExecutorTests {
+
+    protected TestTaskBatcher taskBatcher;
+
+    @Before
+    public void setUpBatchingTaskExecutor() throws Exception {
+        taskBatcher = new TestTaskBatcher(logger, threadExecutor);
+    }
+
+    class TestTaskBatcher extends TaskBatcher {
+
+        TestTaskBatcher(Logger logger, PrioritizedEsThreadPoolExecutor threadExecutor) {
+            super(logger, threadExecutor);
+        }
+
+        @Override
+        protected void run(Object batchingKey, List<? extends BatchedTask> tasks, String tasksSummary) {
+            List<UpdateTask> updateTasks = (List) tasks;
+            ((TestExecutor) batchingKey).execute(updateTasks.stream().map(t -> t.task).collect(Collectors.toList()));
+            updateTasks.forEach(updateTask -> updateTask.listener.processed(updateTask.source));
+        }
+
+        @Override
+        protected void onTimeout(List<? extends BatchedTask> tasks, TimeValue timeout) {
+            threadPool.generic().execute(
+                () -> tasks.forEach(
+                    task -> ((UpdateTask) task).listener.onFailure(task.source,
+                        new ProcessClusterEventTimeoutException(timeout, task.source))));
+        }
+
+        class UpdateTask extends BatchedTask {
+            final TestListener listener;
+
+            UpdateTask(Priority priority, String source, Object task, TestListener listener, TestExecutor<?> executor) {
+                super(priority, source, executor, task);
+                this.listener = listener;
+            }
+
+            @Override
+            public String describeTasks(List<? extends BatchedTask> tasks) {
+                return ((TestExecutor<Object>) batchingKey).describeTasks(
+                    tasks.stream().map(BatchedTask::getTask).collect(Collectors.toList()));
+            }
+        }
+
+    }
+
+    @Override
+    protected void submitTask(String source, TestTask testTask) {
+        submitTask(source, testTask, testTask, testTask, testTask);
+    }
+
+    private <T> void submitTask(String source, T task, ClusterStateTaskConfig config, TestExecutor<T> executor,
+                               TestListener listener) {
+        submitTasks(source, Collections.singletonMap(task, listener), config, executor);
+    }
+
+    private <T> void submitTasks(final String source,
+                                final Map<T, TestListener> tasks, final ClusterStateTaskConfig config,
+                                final TestExecutor<T> executor) {
+        List<TestTaskBatcher.UpdateTask> safeTasks = tasks.entrySet().stream()
+            .map(e -> taskBatcher.new UpdateTask(config.priority(), source, e.getKey(), e.getValue(), executor))
+            .collect(Collectors.toList());
+        taskBatcher.submitTasks(safeTasks, config.timeout());
+    }
+
+    @Override
+    public void testTimedOutTaskCleanedUp() throws Exception {
+        super.testTimedOutTaskCleanedUp();
+        synchronized (taskBatcher.tasksPerBatchingKey) {
+            assertTrue("expected empty map but was " + taskBatcher.tasksPerBatchingKey,
+                taskBatcher.tasksPerBatchingKey.isEmpty());
+        }
+    }
+
+    public void testOneExecutorDoesntStarveAnother() throws InterruptedException {
+        final List<String> executionOrder = Collections.synchronizedList(new ArrayList<>());
+        final Semaphore allowProcessing = new Semaphore(0);
+        final Semaphore startedProcessing = new Semaphore(0);
+
+        class TaskExecutor implements TestExecutor<String> {
+
+            @Override
+            public void execute(List<String> tasks) {
+                executionOrder.addAll(tasks); // do this first, so startedProcessing can be used as a notification that this is done.
+                startedProcessing.release(tasks.size());
+                try {
+                    allowProcessing.acquire(tasks.size());
+                } catch (InterruptedException e) {
+                    throw new RuntimeException(e);
+                }
+            }
+        }
+
+        TaskExecutor executorA = new TaskExecutor();
+        TaskExecutor executorB = new TaskExecutor();
+
+        final ClusterStateTaskConfig config = ClusterStateTaskConfig.build(Priority.NORMAL);
+        final TestListener noopListener = (source, e) -> {
+            throw new AssertionError(e);
+        };
+        // this blocks the cluster state queue, so we can set it up right
+        submitTask("0", "A0", config, executorA, noopListener);
+        // wait to be processed
+        startedProcessing.acquire(1);
+        assertThat(executionOrder, equalTo(Arrays.asList("A0")));
+
+
+        // these will be the first batch
+        submitTask("1", "A1", config, executorA, noopListener);
+        submitTask("2", "A2", config, executorA, noopListener);
+
+        // release the first 0 task, but not the second
+        allowProcessing.release(1);
+        startedProcessing.acquire(2);
+        assertThat(executionOrder, equalTo(Arrays.asList("A0", "A1", "A2")));
+
+        // setup the queue with pending tasks for another executor same priority
+        submitTask("3", "B3", config, executorB, noopListener);
+        submitTask("4", "B4", config, executorB, noopListener);
+
+
+        submitTask("5", "A5", config, executorA, noopListener);
+        submitTask("6", "A6", config, executorA, noopListener);
+
+        // now release the processing
+        allowProcessing.release(6);
+
+        // wait for last task to be processed
+        startedProcessing.acquire(4);
+
+        assertThat(executionOrder, equalTo(Arrays.asList("A0", "A1", "A2", "B3", "B4", "A5", "A6")));
+    }
+
+    static class TaskExecutor implements TestExecutor<Integer> {
+        List<Integer> tasks = new ArrayList<>();
+
+        @Override
+        public void execute(List<Integer> tasks) {
+            this.tasks.addAll(tasks);
+        }
+    }
+
+    // test that for a single thread, tasks are executed in the order
+    // that they are submitted
+    public void testTasksAreExecutedInOrder() throws BrokenBarrierException, InterruptedException {
+        int numberOfThreads = randomIntBetween(2, 8);
+        TaskExecutor[] executors = new TaskExecutor[numberOfThreads];
+        for (int i = 0; i < numberOfThreads; i++) {
+            executors[i] = new TaskExecutor();
+        }
+
+        int tasksSubmittedPerThread = randomIntBetween(2, 1024);
+
+        CopyOnWriteArrayList<Tuple<String, Throwable>> failures = new CopyOnWriteArrayList<>();
+        CountDownLatch updateLatch = new CountDownLatch(numberOfThreads * tasksSubmittedPerThread);
+
+        final TestListener listener = new TestListener() {
+            @Override
+            public void onFailure(String source, Exception e) {
+                logger.error((Supplier<?>) () -> new ParameterizedMessage("unexpected failure: [{}]", source), e);
+                failures.add(new Tuple<>(source, e));
+                updateLatch.countDown();
+            }
+
+            @Override
+            public void processed(String source) {
+                updateLatch.countDown();
+            }
+        };
+
+        CyclicBarrier barrier = new CyclicBarrier(1 + numberOfThreads);
+
+        for (int i = 0; i < numberOfThreads; i++) {
+            final int index = i;
+            Thread thread = new Thread(() -> {
+                try {
+                    barrier.await();
+                    for (int j = 0; j < tasksSubmittedPerThread; j++) {
+                        submitTask("[" + index + "][" + j + "]", j,
+                            ClusterStateTaskConfig.build(randomFrom(Priority.values())), executors[index], listener);
+                    }
+                    barrier.await();
+                } catch (InterruptedException | BrokenBarrierException e) {
+                    throw new AssertionError(e);
+                }
+            });
+            thread.start();
+        }
+
+        // wait for all threads to be ready
+        barrier.await();
+        // wait for all threads to finish
+        barrier.await();
+
+        updateLatch.await();
+
+        assertThat(failures, empty());
+
+        for (int i = 0; i < numberOfThreads; i++) {
+            assertEquals(tasksSubmittedPerThread, executors[i].tasks.size());
+            for (int j = 0; j < tasksSubmittedPerThread; j++) {
+                assertNotNull(executors[i].tasks.get(j));
+                assertEquals("cluster state update task executed out of order", j, (int) executors[i].tasks.get(j));
+            }
+        }
+    }
+
+    public void testSingleBatchSubmission() throws InterruptedException {
+        Map<Integer, TestListener> tasks = new HashMap<>();
+        final int numOfTasks = randomInt(10);
+        final CountDownLatch latch = new CountDownLatch(numOfTasks);
+        for (int i = 0; i < numOfTasks; i++) {
+            while (null != tasks.put(randomInt(1024), new TestListener() {
+                @Override
+                public void processed(String source) {
+                    latch.countDown();
+                }
+
+                @Override
+                public void onFailure(String source, Exception e) {
+                    fail(ExceptionsHelper.detailedMessage(e));
+                }
+            })) ;
+        }
+
+        TestExecutor<Integer> executor = taskList -> {
+            assertThat(taskList.size(), equalTo(tasks.size()));
+            assertThat(taskList.stream().collect(Collectors.toSet()), equalTo(tasks.keySet()));
+        };
+        submitTasks("test", tasks, ClusterStateTaskConfig.build(Priority.LANGUID), executor);
+
+        latch.await();
+    }
+
+    public void testDuplicateSubmission() throws InterruptedException {
+        final CountDownLatch latch = new CountDownLatch(2);
+        try (BlockingTask blockingTask = new BlockingTask(Priority.IMMEDIATE)) {
+            submitTask("blocking", blockingTask);
+
+            TestExecutor<SimpleTask> executor = tasks -> {};
+            SimpleTask task = new SimpleTask(1);
+            TestListener listener = new TestListener() {
+                @Override
+                public void processed(String source) {
+                    latch.countDown();
+                }
+
+                @Override
+                public void onFailure(String source, Exception e) {
+                    fail(ExceptionsHelper.detailedMessage(e));
+                }
+            };
+
+            submitTask("first time", task, ClusterStateTaskConfig.build(Priority.NORMAL), executor,
+                listener);
+
+            final IllegalStateException e =
+                expectThrows(
+                    IllegalStateException.class,
+                    () -> submitTask(
+                        "second time",
+                        task,
+                        ClusterStateTaskConfig.build(Priority.NORMAL),
+                        executor, listener));
+            assertThat(e, hasToString(containsString("task [1] with source [second time] is already queued")));
+
+            submitTask("third time a charm", new SimpleTask(1),
+                ClusterStateTaskConfig.build(Priority.NORMAL), executor, listener);
+
+            assertThat(latch.getCount(), equalTo(2L));
+        }
+        latch.await();
+    }
+
+    private static class SimpleTask {
+        private final int id;
+
+        private SimpleTask(int id) {
+            this.id = id;
+        }
+
+        @Override
+        public int hashCode() {
+            return super.hashCode();
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            return super.equals(obj);
+        }
+
+        @Override
+        public String toString() {
+            return Integer.toString(id);
+        }
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/cluster/service/TaskExecutorTests.java b/core/src/test/java/org/elasticsearch/cluster/service/TaskExecutorTests.java
new file mode 100644
index 0000000000000..fe426fdd42a9d
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/cluster/service/TaskExecutorTests.java
@@ -0,0 +1,365 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.cluster.service;
+
+import org.elasticsearch.cluster.ClusterStateTaskConfig;
+import org.elasticsearch.cluster.metadata.ProcessClusterEventTimeoutException;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Priority;
+import org.elasticsearch.common.lease.Releasable;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.elasticsearch.common.util.concurrent.EsExecutors.daemonThreadFactory;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.core.Is.is;
+
+public class TaskExecutorTests extends ESTestCase {
+
+    protected static ThreadPool threadPool;
+    protected PrioritizedEsThreadPoolExecutor threadExecutor;
+
+    @BeforeClass
+    public static void createThreadPool() {
+        threadPool = new TestThreadPool(getTestClass().getName());
+    }
+
+    @AfterClass
+    public static void stopThreadPool() {
+        if (threadPool != null) {
+            threadPool.shutdownNow();
+            threadPool = null;
+        }
+    }
+
+    @Before
+    public void setUpExecutor() {
+        threadExecutor = EsExecutors.newSinglePrioritizing("test_thread",
+            daemonThreadFactory(Settings.EMPTY, "test_thread"), threadPool.getThreadContext(), threadPool.scheduler());
+    }
+
+    @After
+    public void shutDownThreadExecutor() {
+        ThreadPool.terminate(threadExecutor, 10, TimeUnit.SECONDS);
+    }
+
+    protected interface TestListener {
+        void onFailure(String source, Exception e);
+
+        default void processed(String source) {
+            // do nothing by default
+        }
+    }
+
+    protected interface TestExecutor<T> {
+        void execute(List<T> tasks);
+
+        default String describeTasks(List<T> tasks) {
+            return tasks.stream().map(T::toString).reduce((s1,s2) -> {
+                if (s1.isEmpty()) {
+                    return s2;
+                } else if (s2.isEmpty()) {
+                    return s1;
+                } else {
+                    return s1 + ", " + s2;
+                }
+            }).orElse("");
+        }
+    }
+
+    /**
+     * Task class that works for single tasks as well as batching (see {@link TaskBatcherTests})
+     */
+    protected abstract static class TestTask implements TestExecutor<TestTask>, TestListener, ClusterStateTaskConfig {
+
+        @Override
+        public void execute(List<TestTask> tasks) {
+            tasks.forEach(TestTask::run);
+        }
+
+        @Nullable
+        @Override
+        public TimeValue timeout() {
+            return null;
+        }
+
+        @Override
+        public Priority priority() {
+            return Priority.NORMAL;
+        }
+
+        public abstract void run();
+    }
+
+    class UpdateTask extends SourcePrioritizedRunnable {
+        final TestTask testTask;
+
+        UpdateTask(String source, TestTask testTask) {
+            super(testTask.priority(), source);
+            this.testTask = testTask;
+        }
+
+        @Override
+        public void run() {
+            logger.trace("will process {}", source);
+            testTask.execute(Collections.singletonList(testTask));
+            testTask.processed(source);
+        }
+    }
+
+    // can be overridden by TaskBatcherTests
+    protected void submitTask(String source, TestTask testTask) {
+        SourcePrioritizedRunnable task = new UpdateTask(source, testTask);
+        TimeValue timeout = testTask.timeout();
+        if (timeout != null) {
+            threadExecutor.execute(task, timeout, () -> threadPool.generic().execute(() -> {
+                logger.debug("task [{}] timed out after [{}]", task, timeout);
+                testTask.onFailure(source, new ProcessClusterEventTimeoutException(timeout, source));
+            }));
+        } else {
+            threadExecutor.execute(task);
+        }
+    }
+
+
+    public void testTimedOutTaskCleanedUp() throws Exception {
+        final CountDownLatch block = new CountDownLatch(1);
+        final CountDownLatch blockCompleted = new CountDownLatch(1);
+        TestTask blockTask = new TestTask() {
+
+            @Override
+            public void run() {
+                try {
+                    block.await();
+                } catch (InterruptedException e) {
+                    throw new RuntimeException(e);
+                }
+                blockCompleted.countDown();
+            }
+
+            @Override
+            public void onFailure(String source, Exception e) {
+                throw new RuntimeException(e);
+            }
+        };
+        submitTask("block-task", blockTask);
+
+        final CountDownLatch block2 = new CountDownLatch(1);
+        TestTask unblockTask = new TestTask() {
+
+            @Override
+            public void run() {
+                block2.countDown();
+            }
+
+            @Override
+            public void onFailure(String source, Exception e) {
+                block2.countDown();
+            }
+
+            @Override
+            public TimeValue timeout() {
+                return TimeValue.ZERO;
+            }
+        };
+        submitTask("unblock-task", unblockTask);
+
+        block.countDown();
+        block2.await();
+        blockCompleted.await();
+    }
+
+    public void testTimeoutTask() throws Exception {
+        final CountDownLatch block = new CountDownLatch(1);
+        TestTask test1 = new TestTask() {
+            @Override
+            public void run() {
+                try {
+                    block.await();
+                } catch (InterruptedException e) {
+                    throw new RuntimeException(e);
+                }
+            }
+
+            @Override
+            public void onFailure(String source, Exception e) {
+                throw new RuntimeException(e);
+            }
+        };
+        submitTask("block-task", test1);
+
+        final CountDownLatch timedOut = new CountDownLatch(1);
+        final AtomicBoolean executeCalled = new AtomicBoolean();
+        TestTask test2 = new TestTask() {
+
+            @Override
+            public TimeValue timeout() {
+                return TimeValue.timeValueMillis(2);
+            }
+
+            @Override
+            public void run() {
+                executeCalled.set(true);
+            }
+
+            @Override
+            public void onFailure(String source, Exception e) {
+                timedOut.countDown();
+            }
+        };
+        submitTask("block-task", test2);
+
+        timedOut.await();
+        block.countDown();
+        final CountDownLatch allProcessed = new CountDownLatch(1);
+        TestTask test3 = new TestTask() {
+
+            @Override
+            public void run() {
+                allProcessed.countDown();
+            }
+
+            @Override
+            public void onFailure(String source, Exception e) {
+                throw new RuntimeException(e);
+            }
+        };
+        submitTask("block-task", test3);
+        allProcessed.await(); // executed another task to double check that execute on the timed out update task is not called...
+        assertThat(executeCalled.get(), equalTo(false));
+    }
+
+    static class TaskExecutor implements TestExecutor<Integer> {
+        List<Integer> tasks = new ArrayList<>();
+
+        @Override
+        public void execute(List<Integer> tasks) {
+            this.tasks.addAll(tasks);
+        }
+    }
+
+    /**
+     * Note, this test can only work as long as we have a single thread executor executing the state update tasks!
+     */
+    public void testPrioritizedTasks() throws Exception {
+        BlockingTask block = new BlockingTask(Priority.IMMEDIATE);
+        submitTask("test", block);
+        int taskCount = randomIntBetween(5, 20);
+
+        // will hold all the tasks in the order in which they were executed
+        List<PrioritizedTask> tasks = new ArrayList<>(taskCount);
+        CountDownLatch latch = new CountDownLatch(taskCount);
+        for (int i = 0; i < taskCount; i++) {
+            Priority priority = randomFrom(Priority.values());
+            PrioritizedTask task = new PrioritizedTask(priority, latch, tasks);
+            submitTask("test", task);
+        }
+
+        block.close();
+        latch.await();
+
+        Priority prevPriority = null;
+        for (PrioritizedTask task : tasks) {
+            if (prevPriority == null) {
+                prevPriority = task.priority();
+            } else {
+                assertThat(task.priority().sameOrAfter(prevPriority), is(true));
+            }
+        }
+    }
+
+    protected static class BlockingTask extends TestTask implements Releasable {
+        private final CountDownLatch latch = new CountDownLatch(1);
+        private final Priority priority;
+
+        BlockingTask(Priority priority) {
+            super();
+            this.priority = priority;
+        }
+
+        @Override
+        public void run() {
+            try {
+                latch.await();
+            } catch (InterruptedException e) {
+                throw new RuntimeException(e);
+            }
+        }
+
+        @Override
+        public void onFailure(String source, Exception e) {
+        }
+
+        @Override
+        public Priority priority() {
+            return priority;
+        }
+
+        public void close() {
+            latch.countDown();
+        }
+
+    }
+
+    protected static class PrioritizedTask extends TestTask {
+        private final CountDownLatch latch;
+        private final List<PrioritizedTask> tasks;
+        private final Priority priority;
+
+        private PrioritizedTask(Priority priority, CountDownLatch latch, List<PrioritizedTask> tasks) {
+            super();
+            this.latch = latch;
+            this.tasks = tasks;
+            this.priority = priority;
+        }
+
+        @Override
+        public void run() {
+            tasks.add(this);
+            latch.countDown();
+        }
+
+        @Override
+        public Priority priority() {
+            return priority;
+        }
+
+        @Override
+        public void onFailure(String source, Exception e) {
+            latch.countDown();
+        }
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/cluster/settings/ClusterSettingsIT.java b/core/src/test/java/org/elasticsearch/cluster/settings/ClusterSettingsIT.java
index 26bb97fcc0483..a74078d6ee7a6 100644
--- a/core/src/test/java/org/elasticsearch/cluster/settings/ClusterSettingsIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/settings/ClusterSettingsIT.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
 import org.elasticsearch.common.logging.ESLoggerFactory;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.discovery.Discovery;
@@ -32,6 +33,7 @@
 import org.elasticsearch.index.store.IndexStoreConfig;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
+import org.junit.After;
 
 import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@@ -41,8 +43,15 @@
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
-@ClusterScope(scope = TEST)
 public class ClusterSettingsIT extends ESIntegTestCase {
+
+    @After
+    public void cleanup() throws Exception {
+        assertAcked(client().admin().cluster().prepareUpdateSettings()
+            .setPersistentSettings(Settings.builder().putNull("*"))
+            .setTransientSettings(Settings.builder().putNull("*")));
+    }
+
     public void testClusterNonExistingSettingsUpdate() {
         String key1 = "no_idea_what_you_are_talking_about";
         int value1 = 10;
@@ -301,13 +310,25 @@ public void testClusterUpdateSettingsWithBlocks() {
             assertBlocked(request, MetaData.CLUSTER_READ_ONLY_BLOCK);
 
             // But it's possible to update the settings to update the "cluster.blocks.read_only" setting
-            Settings settings = Settings.builder().put(MetaData.SETTING_READ_ONLY_SETTING.getKey(), false).build();
+            Settings settings = Settings.builder().putNull(MetaData.SETTING_READ_ONLY_SETTING.getKey()).build();
             assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(settings).get());
 
         } finally {
             setClusterReadOnly(false);
         }
 
+        // Cluster settings updates are blocked when the cluster is read only
+        try {
+            // But it's possible to update the settings to update the "cluster.blocks.read_only" setting
+            Settings settings = Settings.builder().put(MetaData.SETTING_READ_ONLY_ALLOW_DELETE_SETTING.getKey(), true).build();
+            assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(settings).get());
+            assertBlocked(request, MetaData.CLUSTER_READ_ONLY_ALLOW_DELETE_BLOCK);
+        } finally {
+            // But it's possible to update the settings to update the "cluster.blocks.read_only" setting
+            Settings s = Settings.builder().putNull(MetaData.SETTING_READ_ONLY_ALLOW_DELETE_SETTING.getKey()).build();
+            assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(s).get());
+        }
+
         // It should work now
         ClusterUpdateSettingsResponse response = request.execute().actionGet();
 
@@ -330,9 +351,11 @@ public void testMissingUnits() {
         }
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/20318")
     public void testLoggerLevelUpdate() {
         assertAcked(prepareCreate("test"));
+
+        final Level level = ESLoggerFactory.getRootLogger().getLevel();
+
         final IllegalArgumentException e =
             expectThrows(
                 IllegalArgumentException.class,
@@ -352,8 +375,8 @@ public void testLoggerLevelUpdate() {
                 final Settings.Builder defaultSettings = Settings.builder().putNull("logger.*");
                 client().admin().cluster().prepareUpdateSettings().setTransientSettings(defaultSettings).execute().actionGet();
             }
-            assertEquals(ESLoggerFactory.LOG_DEFAULT_LEVEL_SETTING.get(Settings.EMPTY), ESLoggerFactory.getLogger("test").getLevel());
-            assertEquals(ESLoggerFactory.LOG_DEFAULT_LEVEL_SETTING.get(Settings.EMPTY), ESLoggerFactory.getRootLogger().getLevel());
+            assertEquals(level, ESLoggerFactory.getLogger("test").getLevel());
+            assertEquals(level, ESLoggerFactory.getRootLogger().getLevel());
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/cluster/shards/ClusterSearchShardsIT.java b/core/src/test/java/org/elasticsearch/cluster/shards/ClusterSearchShardsIT.java
index a88c24873f9ca..acb2e823f4453 100644
--- a/core/src/test/java/org/elasticsearch/cluster/shards/ClusterSearchShardsIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/shards/ClusterSearchShardsIT.java
@@ -33,6 +33,7 @@
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_READ;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_WRITE;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY_ALLOW_DELETE;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBlocked;
 import static org.hamcrest.Matchers.equalTo;
 
@@ -58,16 +59,16 @@ public void testSingleShardAllocation() throws Exception {
         ensureGreen();
         ClusterSearchShardsResponse response = client().admin().cluster().prepareSearchShards("test").execute().actionGet();
         assertThat(response.getGroups().length, equalTo(1));
-        assertThat(response.getGroups()[0].getIndex(), equalTo("test"));
-        assertThat(response.getGroups()[0].getShardId(), equalTo(0));
+        assertThat(response.getGroups()[0].getShardId().getIndexName(), equalTo("test"));
+        assertThat(response.getGroups()[0].getShardId().getId(), equalTo(0));
         assertThat(response.getGroups()[0].getShards().length, equalTo(1));
         assertThat(response.getNodes().length, equalTo(1));
         assertThat(response.getGroups()[0].getShards()[0].currentNodeId(), equalTo(response.getNodes()[0].getId()));
 
         response = client().admin().cluster().prepareSearchShards("test").setRouting("A").execute().actionGet();
         assertThat(response.getGroups().length, equalTo(1));
-        assertThat(response.getGroups()[0].getIndex(), equalTo("test"));
-        assertThat(response.getGroups()[0].getShardId(), equalTo(0));
+        assertThat(response.getGroups()[0].getShardId().getIndexName(), equalTo("test"));
+        assertThat(response.getGroups()[0].getShardId().getId(), equalTo(0));
         assertThat(response.getGroups()[0].getShards().length, equalTo(1));
         assertThat(response.getNodes().length, equalTo(1));
         assertThat(response.getGroups()[0].getShards()[0].currentNodeId(), equalTo(response.getNodes()[0].getId()));
@@ -81,7 +82,7 @@ public void testMultipleShardsSingleNodeAllocation() throws Exception {
 
         ClusterSearchShardsResponse response = client().admin().cluster().prepareSearchShards("test").execute().actionGet();
         assertThat(response.getGroups().length, equalTo(4));
-        assertThat(response.getGroups()[0].getIndex(), equalTo("test"));
+        assertThat(response.getGroups()[0].getShardId().getIndexName(), equalTo("test"));
         assertThat(response.getNodes().length, equalTo(1));
         assertThat(response.getGroups()[0].getShards()[0].currentNodeId(), equalTo(response.getNodes()[0].getId()));
 
@@ -90,7 +91,7 @@ public void testMultipleShardsSingleNodeAllocation() throws Exception {
 
         response = client().admin().cluster().prepareSearchShards("test").setPreference("_shards:2").execute().actionGet();
         assertThat(response.getGroups().length, equalTo(1));
-        assertThat(response.getGroups()[0].getShardId(), equalTo(2));
+        assertThat(response.getGroups()[0].getShardId().getId(), equalTo(2));
     }
 
     public void testMultipleIndicesAllocation() throws Exception {
@@ -111,10 +112,10 @@ public void testMultipleIndicesAllocation() throws Exception {
         boolean seenTest1 = false;
         boolean seenTest2 = false;
         for (ClusterSearchShardsGroup group : response.getGroups()) {
-            if (group.getIndex().equals("test1")) {
+            if (group.getShardId().getIndexName().equals("test1")) {
                 seenTest1 = true;
                 assertThat(group.getShards().length, equalTo(2));
-            } else if (group.getIndex().equals("test2")) {
+            } else if (group.getShardId().getIndexName().equals("test2")) {
                 seenTest2 = true;
                 assertThat(group.getShards().length, equalTo(2));
             } else {
@@ -138,7 +139,8 @@ public void testClusterSearchShardsWithBlocks() {
         ensureGreen("test-blocks");
 
         // Request is not blocked
-        for (String blockSetting : Arrays.asList(SETTING_BLOCKS_READ, SETTING_BLOCKS_WRITE, SETTING_READ_ONLY)) {
+        for (String blockSetting : Arrays.asList(SETTING_BLOCKS_READ, SETTING_BLOCKS_WRITE, SETTING_READ_ONLY,
+            SETTING_READ_ONLY_ALLOW_DELETE)) {
             try {
                 enableIndexBlock("test-blocks", blockSetting);
                 ClusterSearchShardsResponse response = client().admin().cluster().prepareSearchShards("test-blocks").execute().actionGet();
diff --git a/core/src/test/java/org/elasticsearch/cluster/structure/RoutingIteratorTests.java b/core/src/test/java/org/elasticsearch/cluster/structure/RoutingIteratorTests.java
index 8ebf2c3eb9479..172bcd6bd558b 100644
--- a/core/src/test/java/org/elasticsearch/cluster/structure/RoutingIteratorTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/structure/RoutingIteratorTests.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -35,12 +36,10 @@
 import org.elasticsearch.cluster.routing.ShardShuffler;
 import org.elasticsearch.cluster.routing.ShardsIterator;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
-import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.cluster.ESAllocationTestCase;
 
 import java.util.Collections;
 import java.util.HashMap;
@@ -251,14 +250,11 @@ public void testAttributePreferenceRouting() {
                 .add(newNode("node2", unmodifiableMap(node2Attributes)))
                 .localNodeId("node1")
         ).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         // after all are started, check routing iteration
         ShardIterator shardIterator = clusterState.routingTable().index("test").shard(0).preferAttributesActiveInitializingShardsIt(new String[]{"rack_id"}, clusterState.nodes());
@@ -300,11 +296,9 @@ public void testNodeSelectorRouting(){
                         .localNodeId("node1")
         ).build();
 
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         ShardsIterator shardsIterator = clusterState.routingTable().index("test").shard(0).onlyNodeSelectorActiveInitializingShardsIt("disk:ebs",clusterState.nodes());
         assertThat(shardsIterator.size(), equalTo(1));
@@ -373,19 +367,16 @@ public void testShardsAndPreferNodeRouting() {
                 .add(newNode("node2"))
                 .localNodeId("node1")
         ).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         OperationRouting operationRouting = new OperationRouting(Settings.EMPTY, new ClusterSettings(Settings.EMPTY,
             ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
 
-        GroupShardsIterator shardIterators = operationRouting.searchShards(clusterState, new String[]{"test"}, null, "_shards:0");
+        GroupShardsIterator<ShardIterator> shardIterators = operationRouting.searchShards(clusterState, new String[]{"test"}, null, "_shards:0");
         assertThat(shardIterators.size(), equalTo(1));
         assertThat(shardIterators.iterator().next().shardId().id(), equalTo(0));
 
@@ -394,7 +385,7 @@ public void testShardsAndPreferNodeRouting() {
         assertThat(shardIterators.iterator().next().shardId().id(), equalTo(1));
 
         //check node preference, first without preference to see they switch
-        shardIterators = operationRouting.searchShards(clusterState, new String[]{"test"}, null, "_shards:0;");
+        shardIterators = operationRouting.searchShards(clusterState, new String[]{"test"}, null, "_shards:0|");
         assertThat(shardIterators.size(), equalTo(1));
         assertThat(shardIterators.iterator().next().shardId().id(), equalTo(0));
         String firstRoundNodeId = shardIterators.iterator().next().nextOrNull().currentNodeId();
@@ -404,12 +395,12 @@ public void testShardsAndPreferNodeRouting() {
         assertThat(shardIterators.iterator().next().shardId().id(), equalTo(0));
         assertThat(shardIterators.iterator().next().nextOrNull().currentNodeId(), not(equalTo(firstRoundNodeId)));
 
-        shardIterators = operationRouting.searchShards(clusterState, new String[]{"test"}, null, "_shards:0;_prefer_nodes:node1");
+        shardIterators = operationRouting.searchShards(clusterState, new String[]{"test"}, null, "_shards:0|_prefer_nodes:node1");
         assertThat(shardIterators.size(), equalTo(1));
         assertThat(shardIterators.iterator().next().shardId().id(), equalTo(0));
         assertThat(shardIterators.iterator().next().nextOrNull().currentNodeId(), equalTo("node1"));
 
-        shardIterators = operationRouting.searchShards(clusterState, new String[]{"test"}, null, "_shards:0;_prefer_nodes:node1,node2");
+        shardIterators = operationRouting.searchShards(clusterState, new String[]{"test"}, null, "_shards:0|_prefer_nodes:node1,node2");
         assertThat(shardIterators.size(), equalTo(1));
         Iterator<ShardIterator> iterator = shardIterators.iterator();
         final ShardIterator it = iterator.next();
@@ -447,14 +438,12 @@ public void testReplicaShardPreferenceIters() throws Exception {
                         .add(newNode("node3"))
                         .localNodeId("node1")
         ).build();
-        RoutingAllocation.Result routingResult = strategy.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
 
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
         // When replicas haven't initialized, it comes back with the primary first, then initializing replicas
-        GroupShardsIterator shardIterators = operationRouting.searchShards(clusterState, new String[]{"test"}, null, "_replica_first");
+        GroupShardsIterator<ShardIterator> shardIterators = operationRouting.searchShards(clusterState, new String[]{"test"}, null, "_replica_first");
         assertThat(shardIterators.size(), equalTo(2)); // two potential shards
         ShardIterator iter = shardIterators.iterator().next();
         assertThat(iter.size(), equalTo(3)); // three potential candidates for the shard
@@ -472,11 +461,9 @@ public void testReplicaShardPreferenceIters() throws Exception {
         assertFalse(routing.primary());
         assertTrue(routing.initializing());
 
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
-        routingResult = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
 
 
         shardIterators = operationRouting.searchShards(clusterState, new String[]{"test"}, null, "_replica");
diff --git a/core/src/test/java/org/elasticsearch/codecs/CodecTests.java b/core/src/test/java/org/elasticsearch/codecs/CodecTests.java
index 1a2108fa73ca6..dac83d1dee823 100644
--- a/core/src/test/java/org/elasticsearch/codecs/CodecTests.java
+++ b/core/src/test/java/org/elasticsearch/codecs/CodecTests.java
@@ -48,7 +48,7 @@ protected Collection<Class<? extends Plugin>> getPlugins() {
 
     public void testAcceptPostingsFormat() throws IOException {
         int i = 0;
-        for (Version v : VersionUtils.allVersions()) {
+        for (Version v : VersionUtils.allReleasedVersions()) {
             if (v.onOrAfter(Version.V_2_0_0) == false) {
                 // no need to test, we don't support upgrading from these versions
                 continue;
@@ -82,7 +82,7 @@ public void testAcceptPostingsFormat() throws IOException {
 
     public void testAcceptDocValuesFormat() throws IOException {
         int i = 0;
-        for (Version v : VersionUtils.allVersions()) {
+        for (Version v : VersionUtils.allReleasedVersions()) {
             if (v.onOrAfter(Version.V_2_0_0) == false) {
                 // no need to test, we don't support upgrading from these versions
                 continue;
diff --git a/core/src/test/java/org/elasticsearch/common/BooleansTests.java b/core/src/test/java/org/elasticsearch/common/BooleansTests.java
index 176c4c75dc7a6..97724b7e99f86 100644
--- a/core/src/test/java/org/elasticsearch/common/BooleansTests.java
+++ b/core/src/test/java/org/elasticsearch/common/BooleansTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.common;
 
 import org.elasticsearch.test.ESTestCase;
-import org.hamcrest.Matchers;
 
 import java.util.Locale;
 
@@ -28,59 +27,88 @@
 import static org.hamcrest.Matchers.nullValue;
 
 public class BooleansTests extends ESTestCase {
+    private static final String[] NON_BOOLEANS = new String[]{"11", "00", "sdfsdfsf", "F", "T", "True", "False"};
+    private static final String[] BOOLEANS = new String[]{"true", "false", "on", "off", "yes", "no", "0", "1"};
+    private static final String[] TRUTHY = new String[]{"true", "on", "yes", "1"};
+    private static final String[] FALSY = new String[]{"false", "off", "no", "0"};
+
+    public void testIsNonBoolean() {
+        assertThat(Booleans.isBoolean(null, 0, 1), is(false));
+
+        for (String nb : NON_BOOLEANS) {
+            String t = "prefix" + nb + "suffix";
+            assertFalse("recognized [" + nb + "] as boolean", Booleans.isBoolean(t.toCharArray(), "prefix".length(), nb.length()));
+            assertFalse("recognized [" + nb + "] as boolean", Booleans.isStrictlyBoolean(t));
+        }
+    }
+
+    public void testParseBooleanWithFallback() {
+        assertFalse(Booleans.parseBoolean(null, false));
+        assertTrue(Booleans.parseBoolean(null, true));
+        assertNull(Booleans.parseBoolean(null, null));
+        assertFalse(Booleans.parseBoolean(null, Boolean.FALSE));
+        assertTrue(Booleans.parseBoolean(null, Boolean.TRUE));
+
+        assertTrue(Booleans.parseBoolean("true", randomFrom(Boolean.TRUE, Boolean.FALSE, null)));
+        assertFalse(Booleans.parseBoolean("false", randomFrom(Boolean.TRUE, Boolean.FALSE, null)));
+    }
+
+    public void testParseBooleanExact() {
+        assertTrue(Booleans.parseBooleanExact(randomFrom(TRUTHY)));
+        assertFalse(Booleans.parseBooleanExact(randomFrom(FALSY)));
+    }
+
+    public void testParseNonBooleanExact() {
+        expectThrows(IllegalArgumentException.class, () -> Booleans.parseBooleanExact(null));
+        for (String nonBoolean : NON_BOOLEANS) {
+            expectThrows(IllegalArgumentException.class, () -> Booleans.parseBooleanExact(nonBoolean));
+        }
+    }
+
+    public void testParseNonBooleanExactWithFallback() {
+        for (String nonBoolean : NON_BOOLEANS) {
+            boolean defaultValue = randomFrom(Boolean.TRUE, Boolean.FALSE);
+
+            expectThrows(IllegalArgumentException.class,
+                () -> Booleans.parseBooleanExact(nonBoolean, defaultValue));
+        }
+    }
+
     public void testIsBoolean() {
-        String[] booleans = new String[]{"true", "false", "on", "off", "yes", "no", "0", "1"};
-        String[] notBooleans = new String[]{"11", "00", "sdfsdfsf", "F", "T"};
         assertThat(Booleans.isBoolean(null, 0, 1), is(false));
 
-        for (String b : booleans) {
+        for (String b : BOOLEANS) {
             String t = "prefix" + b + "suffix";
-            assertThat("failed to recognize [" + b + "] as boolean", Booleans.isBoolean(t.toCharArray(), "prefix".length(), b.length()), Matchers.equalTo(true));
+            assertTrue("failed to recognize [" + b + "] as boolean",
+                Booleans.isBoolean(t.toCharArray(), "prefix".length(), b.length()));
+            assertTrue("failed to recognize [" + b + "] as boolean",
+                Booleans.isBoolean(new String(t.toCharArray(), "prefix".length(), b.length())));
         }
 
-        for (String nb : notBooleans) {
+        for (String nb : NON_BOOLEANS) {
             String t = "prefix" + nb + "suffix";
-            assertThat("recognized [" + nb + "] as boolean", Booleans.isBoolean(t.toCharArray(), "prefix".length(), nb.length()), Matchers.equalTo(false));
+            assertFalse("recognized [" + nb + "] as boolean",
+                Booleans.isBoolean(t.toCharArray(), "prefix".length(), nb.length()));
+            assertFalse("recognized [" + nb + "] as boolean",
+                Booleans.isBoolean(new String(t.toCharArray(), "prefix".length(), nb.length())));
         }
     }
 
     public void testParseBoolean() {
-        assertThat(Booleans.parseBoolean(randomFrom("true", "on", "yes", "1"), randomBoolean()), is(true));
-        assertThat(Booleans.parseBoolean(randomFrom("false", "off", "no", "0"), randomBoolean()), is(false));
+        assertThat(Booleans.parseBoolean(randomFrom(TRUTHY), randomBoolean()), is(true));
+        assertThat(Booleans.parseBoolean(randomFrom(FALSY), randomBoolean()), is(false));
         assertThat(Booleans.parseBoolean(randomFrom("true", "on", "yes").toUpperCase(Locale.ROOT), randomBoolean()), is(true));
         assertThat(Booleans.parseBoolean(null, false), is(false));
         assertThat(Booleans.parseBoolean(null, true), is(true));
 
-        assertThat(Booleans.parseBoolean(randomFrom("true", "on", "yes", "1"), randomFrom(Boolean.TRUE, Boolean.FALSE, null)), is(true));
-        assertThat(Booleans.parseBoolean(randomFrom("false", "off", "no", "0"), randomFrom(Boolean.TRUE, Boolean.FALSE, null)), is(false));
-        assertThat(Booleans.parseBoolean(randomFrom("true", "on", "yes").toUpperCase(Locale.ROOT),randomFrom(Boolean.TRUE, Boolean.FALSE, null)), is(true));
+        assertThat(Booleans.parseBoolean(
+            randomFrom("true", "on", "yes", "1"), randomFrom(Boolean.TRUE, Boolean.FALSE, null)), is(true));
+        assertThat(Booleans.parseBoolean(
+            randomFrom("false", "off", "no", "0"), randomFrom(Boolean.TRUE, Boolean.FALSE, null)), is(false));
+        assertThat(Booleans.parseBoolean(
+            randomFrom("true", "on", "yes").toUpperCase(Locale.ROOT),randomFrom(Boolean.TRUE, Boolean.FALSE, null)), is(true));
         assertThat(Booleans.parseBoolean(null, Boolean.FALSE), is(false));
         assertThat(Booleans.parseBoolean(null, Boolean.TRUE), is(true));
         assertThat(Booleans.parseBoolean(null, null), nullValue());
-
-        char[] chars = randomFrom("true", "on", "yes", "1").toCharArray();
-        assertThat(Booleans.parseBoolean(chars, 0, chars.length, randomBoolean()), is(true));
-        chars = randomFrom("false", "off", "no", "0").toCharArray();
-        assertThat(Booleans.parseBoolean(chars,0, chars.length, randomBoolean()), is(false));
-        chars = randomFrom("true", "on", "yes").toUpperCase(Locale.ROOT).toCharArray();
-        assertThat(Booleans.parseBoolean(chars,0, chars.length, randomBoolean()), is(true));
-    }
-
-    public void testParseBooleanExact() {
-        assertThat(Booleans.parseBooleanExact(randomFrom("true", "on", "yes", "1")), is(true));
-        assertThat(Booleans.parseBooleanExact(randomFrom("false", "off", "no", "0")), is(false));
-        try {
-            Booleans.parseBooleanExact(randomFrom("fred", "foo", "barney", null));
-            fail("Expected exception while parsing invalid boolean value ");
-        } catch (Exception ex) {
-            assertTrue(ex instanceof IllegalArgumentException);
-        }
-    }
-
-    public void testIsExplicit() {
-        assertThat(Booleans.isExplicitFalse(randomFrom("true", "on", "yes", "1", "foo", null)), is(false));
-        assertThat(Booleans.isExplicitFalse(randomFrom("false", "off", "no", "0")), is(true));
-        assertThat(Booleans.isExplicitTrue(randomFrom("true", "on", "yes", "1")), is(true));
-        assertThat(Booleans.isExplicitTrue(randomFrom("false", "off", "no", "0", "foo", null)), is(false));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/common/ChannelsTests.java b/core/src/test/java/org/elasticsearch/common/ChannelsTests.java
index c0cb3482b0e31..aeb86c73470f3 100644
--- a/core/src/test/java/org/elasticsearch/common/ChannelsTests.java
+++ b/core/src/test/java/org/elasticsearch/common/ChannelsTests.java
@@ -165,7 +165,7 @@ class MockFileChannel extends FileChannel {
 
         FileChannel delegate;
 
-        public MockFileChannel(FileChannel delegate) {
+        MockFileChannel(FileChannel delegate) {
             this.delegate = delegate;
         }
 
diff --git a/core/src/test/java/org/elasticsearch/common/FieldMemoryStatsTests.java b/core/src/test/java/org/elasticsearch/common/FieldMemoryStatsTests.java
new file mode 100644
index 0000000000000..6369ce2d6ea46
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/common/FieldMemoryStatsTests.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.common;
+
+import com.carrotsearch.hppc.ObjectLongHashMap;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+public class FieldMemoryStatsTests extends ESTestCase {
+
+    public void testSerialize() throws IOException {
+        FieldMemoryStats stats = randomFieldMemoryStats();
+        BytesStreamOutput out = new BytesStreamOutput();
+        stats.writeTo(out);
+        StreamInput input = out.bytes().streamInput();
+        FieldMemoryStats read = new FieldMemoryStats(input);
+        assertEquals(-1, input.read());
+        assertEquals(stats, read);
+    }
+
+    public void testHashCodeEquals() {
+        FieldMemoryStats stats = randomFieldMemoryStats();
+        assertEquals(stats, stats);
+        assertEquals(stats.hashCode(), stats.hashCode());
+        ObjectLongHashMap<String> map1 = new ObjectLongHashMap<>();
+        map1.put("bar", 1);
+        FieldMemoryStats stats1 = new FieldMemoryStats(map1);
+        ObjectLongHashMap<String> map2 = new ObjectLongHashMap<>();
+        map2.put("foo", 2);
+        FieldMemoryStats stats2 = new FieldMemoryStats(map2);
+
+        ObjectLongHashMap<String> map3 = new ObjectLongHashMap<>();
+        map3.put("foo", 2);
+        map3.put("bar", 1);
+        FieldMemoryStats stats3 = new FieldMemoryStats(map3);
+
+        ObjectLongHashMap<String> map4 = new ObjectLongHashMap<>();
+        map4.put("foo", 2);
+        map4.put("bar", 1);
+        FieldMemoryStats stats4 = new FieldMemoryStats(map4);
+
+        assertNotEquals(stats1, stats2);
+        assertNotEquals(stats1, stats3);
+        assertNotEquals(stats2, stats3);
+        assertEquals(stats4, stats3);
+
+        stats1.add(stats2);
+        assertEquals(stats1, stats3);
+        assertEquals(stats1, stats4);
+        assertEquals(stats1.hashCode(), stats3.hashCode());
+    }
+
+    public void testAdd() {
+        ObjectLongHashMap<String> map1 = new ObjectLongHashMap<>();
+        map1.put("bar", 1);
+        FieldMemoryStats stats1 = new FieldMemoryStats(map1);
+        ObjectLongHashMap<String> map2 = new ObjectLongHashMap<>();
+        map2.put("foo", 2);
+        FieldMemoryStats stats2 = new FieldMemoryStats(map2);
+
+        ObjectLongHashMap<String> map3 = new ObjectLongHashMap<>();
+        map3.put("bar", 1);
+        FieldMemoryStats stats3 = new FieldMemoryStats(map3);
+        stats3.add(stats1);
+
+        ObjectLongHashMap<String> map4 = new ObjectLongHashMap<>();
+        map4.put("foo", 2);
+        map4.put("bar", 2);
+        FieldMemoryStats stats4 = new FieldMemoryStats(map4);
+        assertNotEquals(stats3, stats4);
+        stats3.add(stats2);
+        assertEquals(stats3, stats4);
+    }
+
+    public static FieldMemoryStats randomFieldMemoryStats() {
+        ObjectLongHashMap<String> map = new ObjectLongHashMap<>();
+        int keys = randomIntBetween(1, 1000);
+        for (int i = 0; i < keys; i++) {
+            map.put(randomRealisticUnicodeOfCodepointLengthBetween(1, 10), randomNonNegativeLong());
+        }
+        return new FieldMemoryStats(map);
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/common/NumbersTests.java b/core/src/test/java/org/elasticsearch/common/NumbersTests.java
new file mode 100644
index 0000000000000..e5563993ad5fd
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/common/NumbersTests.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common;
+
+import org.elasticsearch.test.ESTestCase;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class NumbersTests extends ESTestCase {
+
+    public void testToLongExact() {
+        assertEquals(3L, Numbers.toLongExact(Long.valueOf(3L)));
+        assertEquals(3L, Numbers.toLongExact(Integer.valueOf(3)));
+        assertEquals(3L, Numbers.toLongExact(Short.valueOf((short) 3)));
+        assertEquals(3L, Numbers.toLongExact(Byte.valueOf((byte) 3)));
+        assertEquals(3L, Numbers.toLongExact(3d));
+        assertEquals(3L, Numbers.toLongExact(3f));
+        assertEquals(3L, Numbers.toLongExact(BigInteger.valueOf(3L)));
+        assertEquals(3L, Numbers.toLongExact(BigDecimal.valueOf(3L)));
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> Numbers.toLongExact(3.1d));
+        assertEquals("3.1 is not an integer value", e.getMessage());
+        e = expectThrows(IllegalArgumentException.class,
+                () -> Numbers.toLongExact(Double.NaN));
+        assertEquals("NaN is not an integer value", e.getMessage());
+        e = expectThrows(IllegalArgumentException.class,
+                () -> Numbers.toLongExact(Double.POSITIVE_INFINITY));
+        assertEquals("Infinity is not an integer value", e.getMessage());
+        e = expectThrows(IllegalArgumentException.class,
+                () -> Numbers.toLongExact(3.1f));
+        assertEquals("3.1 is not an integer value", e.getMessage());
+        e = expectThrows(IllegalArgumentException.class,
+                () -> Numbers.toLongExact(new AtomicInteger(3))); // not supported
+        assertEquals("Cannot check whether [3] of class [java.util.concurrent.atomic.AtomicInteger] is actually a long", e.getMessage());
+    }
+
+    public void testToIntExact() {
+        assertEquals(3L, Numbers.toIntExact(Long.valueOf(3L)));
+        assertEquals(3L, Numbers.toIntExact(Integer.valueOf(3)));
+        assertEquals(3L, Numbers.toIntExact(Short.valueOf((short) 3)));
+        assertEquals(3L, Numbers.toIntExact(Byte.valueOf((byte) 3)));
+        assertEquals(3L, Numbers.toIntExact(3d));
+        assertEquals(3L, Numbers.toIntExact(3f));
+        assertEquals(3L, Numbers.toIntExact(BigInteger.valueOf(3L)));
+        assertEquals(3L, Numbers.toIntExact(BigDecimal.valueOf(3L)));
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> Numbers.toIntExact(3.1d));
+        assertEquals("3.1 is not an integer value", e.getMessage());
+        e = expectThrows(IllegalArgumentException.class,
+                () -> Numbers.toLongExact(Double.NaN));
+        assertEquals("NaN is not an integer value", e.getMessage());
+        e = expectThrows(IllegalArgumentException.class,
+                () -> Numbers.toLongExact(Double.POSITIVE_INFINITY));
+        assertEquals("Infinity is not an integer value", e.getMessage());
+        e = expectThrows(IllegalArgumentException.class,
+                () -> Numbers.toIntExact(3.1f));
+        assertEquals("3.1 is not an integer value", e.getMessage());
+        ArithmeticException ae = expectThrows(ArithmeticException.class,
+                () -> Numbers.toIntExact(1L << 40));
+        assertEquals("integer overflow", ae.getMessage());
+        e = expectThrows(IllegalArgumentException.class,
+                () -> Numbers.toIntExact(new AtomicInteger(3))); // not supported
+        assertEquals("Cannot check whether [3] of class [java.util.concurrent.atomic.AtomicInteger] is actually a long", e.getMessage());
+    }
+
+    public void testToShortExact() {
+        assertEquals(3L, Numbers.toShortExact(Long.valueOf(3L)));
+        assertEquals(3L, Numbers.toShortExact(Integer.valueOf(3)));
+        assertEquals(3L, Numbers.toShortExact(Short.valueOf((short) 3)));
+        assertEquals(3L, Numbers.toShortExact(Byte.valueOf((byte) 3)));
+        assertEquals(3L, Numbers.toShortExact(3d));
+        assertEquals(3L, Numbers.toShortExact(3f));
+        assertEquals(3L, Numbers.toShortExact(BigInteger.valueOf(3L)));
+        assertEquals(3L, Numbers.toShortExact(BigDecimal.valueOf(3L)));
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> Numbers.toShortExact(3.1d));
+        assertEquals("3.1 is not an integer value", e.getMessage());
+        e = expectThrows(IllegalArgumentException.class,
+                () -> Numbers.toLongExact(Double.NaN));
+        assertEquals("NaN is not an integer value", e.getMessage());
+        e = expectThrows(IllegalArgumentException.class,
+                () -> Numbers.toLongExact(Double.POSITIVE_INFINITY));
+        assertEquals("Infinity is not an integer value", e.getMessage());
+        e = expectThrows(IllegalArgumentException.class,
+                () -> Numbers.toShortExact(3.1f));
+        assertEquals("3.1 is not an integer value", e.getMessage());
+        ArithmeticException ae = expectThrows(ArithmeticException.class,
+                () -> Numbers.toShortExact(100000));
+        assertEquals("short overflow: " + 100000, ae.getMessage());
+        e = expectThrows(IllegalArgumentException.class,
+                () -> Numbers.toShortExact(new AtomicInteger(3))); // not supported
+        assertEquals("Cannot check whether [3] of class [java.util.concurrent.atomic.AtomicInteger] is actually a long", e.getMessage());
+    }
+
+    public void testToByteExact() {
+        assertEquals(3L, Numbers.toByteExact(Long.valueOf(3L)));
+        assertEquals(3L, Numbers.toByteExact(Integer.valueOf(3)));
+        assertEquals(3L, Numbers.toByteExact(Short.valueOf((short) 3)));
+        assertEquals(3L, Numbers.toByteExact(Byte.valueOf((byte) 3)));
+        assertEquals(3L, Numbers.toByteExact(3d));
+        assertEquals(3L, Numbers.toByteExact(3f));
+        assertEquals(3L, Numbers.toByteExact(BigInteger.valueOf(3L)));
+        assertEquals(3L, Numbers.toByteExact(BigDecimal.valueOf(3L)));
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> Numbers.toByteExact(3.1d));
+        assertEquals("3.1 is not an integer value", e.getMessage());
+        e = expectThrows(IllegalArgumentException.class,
+                () -> Numbers.toLongExact(Double.NaN));
+        assertEquals("NaN is not an integer value", e.getMessage());
+        e = expectThrows(IllegalArgumentException.class,
+                () -> Numbers.toLongExact(Double.POSITIVE_INFINITY));
+        assertEquals("Infinity is not an integer value", e.getMessage());
+        e = expectThrows(IllegalArgumentException.class,
+                () -> Numbers.toByteExact(3.1f));
+        assertEquals("3.1 is not an integer value", e.getMessage());
+        ArithmeticException ae = expectThrows(ArithmeticException.class,
+                () -> Numbers.toByteExact(300));
+        assertEquals("byte overflow: " + 300, ae.getMessage());
+        e = expectThrows(IllegalArgumentException.class,
+                () -> Numbers.toByteExact(new AtomicInteger(3))); // not supported
+        assertEquals("Cannot check whether [3] of class [java.util.concurrent.atomic.AtomicInteger] is actually a long", e.getMessage());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/common/ParseFieldTests.java b/core/src/test/java/org/elasticsearch/common/ParseFieldTests.java
index 1f348000ee4cc..ab70bd6ecaea2 100644
--- a/core/src/test/java/org/elasticsearch/common/ParseFieldTests.java
+++ b/core/src/test/java/org/elasticsearch/common/ParseFieldTests.java
@@ -20,7 +20,6 @@
 
 import org.elasticsearch.test.ESTestCase;
 
-import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.CoreMatchers.not;
 import static org.hamcrest.CoreMatchers.sameInstance;
@@ -33,64 +32,31 @@ public void testParse() {
         String[] deprecated = new String[]{"barFoo", "bar_foo", "Foobar"};
         ParseField withDeprecations = field.withDeprecation(deprecated);
         assertThat(field, not(sameInstance(withDeprecations)));
-        assertThat(field.match(name, false), is(true));
-        assertThat(field.match("foo bar", false), is(false));
+        assertThat(field.match(name), is(true));
+        assertThat(field.match("foo bar"), is(false));
         for (String deprecatedName : deprecated) {
-            assertThat(field.match(deprecatedName, false), is(false));
+            assertThat(field.match(deprecatedName), is(false));
         }
 
-        assertThat(withDeprecations.match(name, false), is(true));
-        assertThat(withDeprecations.match("foo bar", false), is(false));
+        assertThat(withDeprecations.match(name), is(true));
+        assertThat(withDeprecations.match("foo bar"), is(false));
         for (String deprecatedName : deprecated) {
-            assertThat(withDeprecations.match(deprecatedName, false), is(true));
-        }
-
-        // now with strict mode
-        assertThat(field.match(name, true), is(true));
-        assertThat(field.match("foo bar", true), is(false));
-        for (String deprecatedName : deprecated) {
-            assertThat(field.match(deprecatedName, true), is(false));
-        }
-
-        assertThat(withDeprecations.match(name, true), is(true));
-        assertThat(withDeprecations.match("foo bar", true), is(false));
-        for (String deprecatedName : deprecated) {
-            IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {
-                withDeprecations.match(deprecatedName, true);
-            });
-            assertThat(e.getMessage(), containsString("used, expected [foo_bar] instead"));
+            assertThat(withDeprecations.match(deprecatedName), is(true));
+            assertWarnings("Deprecated field [" + deprecatedName + "] used, expected [foo_bar] instead");
         }
     }
 
     public void testAllDeprecated() {
         String name = "like_text";
-
-        boolean withDeprecatedNames = randomBoolean();
         String[] deprecated = new String[]{"text", "same_as_text"};
-        String[] allValues;
-        if (withDeprecatedNames) {
-            String[] newArray = new String[1 + deprecated.length];
-            newArray[0] = name;
-            System.arraycopy(deprecated, 0, newArray, 1, deprecated.length);
-            allValues = newArray;
-        } else {
-            allValues = new String[] {name};
-        }
-
-        ParseField field;
-        if (withDeprecatedNames) {
-            field = new ParseField(name).withDeprecation(deprecated).withAllDeprecated("like");
-        } else {
-            field = new ParseField(name).withAllDeprecated("like");
-        }
-
-        // strict mode off
-        assertThat(field.match(randomFrom(allValues), false), is(true));
-        assertThat(field.match("not a field name", false), is(false));
-
-        // now with strict mode
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> field.match(randomFrom(allValues), true));
-        assertThat(e.getMessage(), containsString(" used, replaced by [like]"));
+        ParseField field = new ParseField(name).withDeprecation(deprecated).withAllDeprecated("like");
+        assertFalse(field.match("not a field name"));
+        assertTrue(field.match("text"));
+        assertWarnings("Deprecated field [text] used, replaced by [like]");
+        assertTrue(field.match("same_as_text"));
+        assertWarnings("Deprecated field [same_as_text] used, replaced by [like]");
+        assertTrue(field.match("like_text"));
+        assertWarnings("Deprecated field [like_text] used, replaced by [like]");
     }
 
     public void testGetAllNamesIncludedDeprecated() {
diff --git a/core/src/test/java/org/elasticsearch/common/ReleasablesTests.java b/core/src/test/java/org/elasticsearch/common/ReleasablesTests.java
new file mode 100644
index 0000000000000..62686354913fd
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/common/ReleasablesTests.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.common;
+
+import org.elasticsearch.common.lease.Releasable;
+import org.elasticsearch.common.lease.Releasables;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class ReleasablesTests extends ESTestCase {
+
+    public void testReleaseOnce() {
+        AtomicInteger count = new AtomicInteger(0);
+        Releasable releasable = Releasables.releaseOnce(count::incrementAndGet, count::incrementAndGet);
+        assertEquals(0, count.get());
+        releasable.close();
+        assertEquals(2, count.get());
+        releasable.close();
+        assertEquals(2, count.get());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/common/StringsTests.java b/core/src/test/java/org/elasticsearch/common/StringsTests.java
index 1b987d47796ea..28dd134adf026 100644
--- a/core/src/test/java/org/elasticsearch/common/StringsTests.java
+++ b/core/src/test/java/org/elasticsearch/common/StringsTests.java
@@ -21,11 +21,9 @@
 
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.test.ESTestCase;
 
-import java.io.IOException;
-
 import static org.hamcrest.Matchers.containsString;
 
 public class StringsTests extends ESTestCase {
@@ -58,21 +56,36 @@ public void testCleanTruncate() {
         assertEquals("", Strings.cleanTruncate("foo", 0));
     }
 
-    public void testEvilToString() {
-        ToXContent needsEnclosingObject = new ToXContent() {
-            @Override
-            public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-                return builder.field("ok", "here").field("catastrophe", "");
+    public void testToStringToXContent() {
+        final ToXContent toXContent;
+        final boolean error;
+        if (randomBoolean()) {
+            if (randomBoolean()) {
+                error = false;
+                toXContent  = (builder, params) -> builder.field("ok", "here").field("catastrophe", "");
+            } else {
+                error = true;
+                toXContent  = (builder, params) ->
+                        builder.startObject().field("ok", "here").field("catastrophe", "").endObject();
+            }
+        } else {
+            if (randomBoolean()) {
+                error = false;
+                toXContent = (ToXContentObject) (builder, params) ->
+                        builder.startObject().field("ok", "here").field("catastrophe", "").endObject();
+            } else {
+                error = true;
+                toXContent = (ToXContentObject) (builder, params) -> builder.field("ok", "here").field("catastrophe", "");
             }
-        };
-        String toString = Strings.toString(needsEnclosingObject);
-        assertThat(toString, containsString("Error building toString out of XContent"));
-        assertThat(toString, containsString("Can not write a field name, expecting a value"));
+        }
 
-        // We can salvage it!
-        toString = Strings.toString(needsEnclosingObject, true);
-        assertThat(toString, containsString("\"ok\":\"here\""));
-        assertThat(toString, containsString("\"catastrophe\":\"\""));
+        String toString = Strings.toString(toXContent);
+        if (error) {
+            assertThat(toString, containsString("Error building toString out of XContent"));
+        } else {
+            assertThat(toString, containsString("\"ok\":\"here\""));
+            assertThat(toString, containsString("\"catastrophe\":\"\""));
+        }
     }
 
     public void testSplitStringToSet() {
diff --git a/core/src/test/java/org/elasticsearch/common/TableTests.java b/core/src/test/java/org/elasticsearch/common/TableTests.java
index 7e624cb749d15..c5e3c34910b9b 100644
--- a/core/src/test/java/org/elasticsearch/common/TableTests.java
+++ b/core/src/test/java/org/elasticsearch/common/TableTests.java
@@ -28,71 +28,43 @@
 import static org.hamcrest.Matchers.is;
 
 public class TableTests extends ESTestCase {
+
     public void testFailOnStartRowWithoutHeader() {
         Table table = new Table();
-        try {
-            table.startRow();
-            fail("Expected IllegalStateException");
-        } catch (IllegalStateException e) {
-            assertThat(e.getMessage(), is("no headers added..."));
-        }
+        Exception e = expectThrows(IllegalStateException.class, () -> table.startRow());
+        assertThat(e.getMessage(), is("no headers added..."));
     }
 
     public void testFailOnEndHeadersWithoutStart() {
         Table table = new Table();
-        try {
-            table.endHeaders();
-            fail("Expected IllegalStateException");
-        } catch (IllegalStateException e) {
-            assertThat(e.getMessage(), is("no headers added..."));
-        }
-
+        Exception e = expectThrows(IllegalStateException.class, () -> table.endHeaders());
+        assertThat(e.getMessage(), is("no headers added..."));
     }
 
     public void testFailOnAddCellWithoutHeader() {
         Table table = new Table();
-        try {
-            table.addCell("error");
-            fail("Expected IllegalStateException");
-        } catch (IllegalStateException e) {
-            assertThat(e.getMessage(), is("no block started..."));
-        }
-
+        Exception e = expectThrows(IllegalStateException.class, () -> table.addCell("error"));
+        assertThat(e.getMessage(), is("no block started..."));
     }
 
     public void testFailOnAddCellWithoutRow() {
         Table table = this.getTableWithHeaders();
-        try {
-            table.addCell("error");
-            fail("Expected IllegalStateException");
-        } catch (IllegalStateException e) {
-            assertThat(e.getMessage(), is("no block started..."));
-        }
-
+        Exception e = expectThrows(IllegalStateException.class, () -> table.addCell("error"));
+        assertThat(e.getMessage(), is("no block started..."));
     }
 
     public void testFailOnEndRowWithoutStart() {
         Table table = this.getTableWithHeaders();
-        try {
-            table.endRow();
-            fail("Expected IllegalStateException");
-        } catch (IllegalStateException e) {
-            assertThat(e.getMessage(), is("no row started..."));
-        }
-
+        Exception e = expectThrows(IllegalStateException.class, () -> table.endRow());
+        assertThat(e.getMessage(), is("no row started..."));
     }
 
     public void testFailOnLessCellsThanDeclared() {
         Table table = this.getTableWithHeaders();
         table.startRow();
         table.addCell("foo");
-        try {
-            table.endRow(true);
-            fail("Expected IllegalStateException");
-        } catch (IllegalStateException e) {
-            assertThat(e.getMessage(), is("mismatch on number of cells 1 in a row compared to header 2"));
-        }
-
+        Exception e = expectThrows(IllegalStateException.class, () -> table.endRow());
+        assertThat(e.getMessage(), is("mismatch on number of cells 1 in a row compared to header 2"));
     }
 
     public void testOnLessCellsThanDeclaredUnchecked() {
@@ -107,13 +79,8 @@ public void testFailOnMoreCellsThanDeclared() {
         table.startRow();
         table.addCell("foo");
         table.addCell("bar");
-        try {
-            table.addCell("foobar");
-            fail("Expected IllegalStateException");
-        } catch (IllegalStateException e) {
-            assertThat(e.getMessage(), is("can't add more cells to a row than the header"));
-        }
-
+        Exception e = expectThrows(IllegalStateException.class, () -> table.addCell("foobar"));
+        assertThat(e.getMessage(), is("can't add more cells to a row than the header"));
     }
 
     public void testSimple() {
@@ -200,6 +167,19 @@ public void testWithTimestamp() {
 
     }
 
+    public void testAliasMap() {
+        Table table = new Table();
+        table.startHeaders();
+        table.addCell("asdf", "alias:a");
+        table.addCell("ghij", "alias:g,h");
+        table.endHeaders();
+        Map<String, String> aliasMap = table.getAliasMap();
+        assertEquals(5, aliasMap.size());
+        assertEquals("asdf", aliasMap.get("a"));
+        assertEquals("ghij", aliasMap.get("g"));
+        assertEquals("ghij", aliasMap.get("h"));
+    }
+
     private Table getTableWithHeaders() {
         Table table = new Table();
         table.startHeaders();
diff --git a/core/src/test/java/org/elasticsearch/common/UUIDTests.java b/core/src/test/java/org/elasticsearch/common/UUIDTests.java
index f82e1a464d91f..62ad47868a65a 100644
--- a/core/src/test/java/org/elasticsearch/common/UUIDTests.java
+++ b/core/src/test/java/org/elasticsearch/common/UUIDTests.java
@@ -58,7 +58,7 @@ class UUIDGenRunner implements Runnable {
         public Set<String> uuidSet = null;
         UUIDGenerator uuidSource;
 
-        public UUIDGenRunner(int count, UUIDGenerator uuidSource) {
+        UUIDGenRunner(int count, UUIDGenerator uuidSource) {
             this.count = count;
             this.uuidSource = uuidSource;
         }
diff --git a/core/src/test/java/org/elasticsearch/common/blobstore/BlobPathTests.java b/core/src/test/java/org/elasticsearch/common/blobstore/BlobPathTests.java
index 09225914644e0..ec846ca30690e 100644
--- a/core/src/test/java/org/elasticsearch/common/blobstore/BlobPathTests.java
+++ b/core/src/test/java/org/elasticsearch/common/blobstore/BlobPathTests.java
@@ -35,5 +35,7 @@ public void testBuildAsString() {
         path = path.add("b").add("c");
         assertThat(path.buildAsString(), is("a/b/c/"));
 
+        path = path.add("d/");
+        assertThat(path.buildAsString(), is("a/b/c/d/"));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/common/breaker/MemoryCircuitBreakerTests.java b/core/src/test/java/org/elasticsearch/common/breaker/MemoryCircuitBreakerTests.java
index f10a0da30299e..ff93e04bf9259 100644
--- a/core/src/test/java/org/elasticsearch/common/breaker/MemoryCircuitBreakerTests.java
+++ b/core/src/test/java/org/elasticsearch/common/breaker/MemoryCircuitBreakerTests.java
@@ -235,6 +235,9 @@ public void testConstantFactor() throws Exception {
             fail("should never reach this");
         } catch (CircuitBreakingException cbe) {
             assertThat("breaker was tripped exactly twice", breaker.getTrippedCount(), equalTo(2L));
+
+            long newUsed = (long)(breaker.getUsed() * breaker.getOverhead());
+            assertThat(cbe.getMessage().contains("would be [" + newUsed + "/"), equalTo(true));
             assertThat(cbe.getMessage().contains("field [" + field + "]"), equalTo(true));
         }
     }
diff --git a/core/src/test/java/org/elasticsearch/common/bytes/BytesArrayTests.java b/core/src/test/java/org/elasticsearch/common/bytes/BytesArrayTests.java
index fff030200b7d2..16d69f829a82f 100644
--- a/core/src/test/java/org/elasticsearch/common/bytes/BytesArrayTests.java
+++ b/core/src/test/java/org/elasticsearch/common/bytes/BytesArrayTests.java
@@ -24,15 +24,25 @@
 import java.io.IOException;
 
 public class BytesArrayTests extends AbstractBytesReferenceTestCase {
+
     @Override
     protected BytesReference newBytesReference(int length) throws IOException {
+        return newBytesReference(length, randomInt(length));
+    }
+
+    @Override
+    protected BytesReference newBytesReferenceWithOffsetOfZero(int length) throws IOException {
+        return newBytesReference(length, 0);
+    }
+
+    private BytesReference newBytesReference(int length, int offset) throws IOException {
         // we know bytes stream output always creates a paged bytes reference, we use it to create randomized content
-        final BytesStreamOutput out = new BytesStreamOutput(length);
-        for (int i = 0; i < length; i++) {
+        final BytesStreamOutput out = new BytesStreamOutput(length + offset);
+        for (int i = 0; i < length + offset; i++) {
             out.writeByte((byte) random().nextInt(1 << 8));
         }
-        assertEquals(length, out.size());
-        BytesArray ref = new BytesArray(out.bytes().toBytesRef());
+        assertEquals(length + offset, out.size());
+        BytesArray ref = new BytesArray(out.bytes().toBytesRef().bytes, offset, length);
         assertEquals(length, ref.length());
         assertTrue(ref instanceof BytesArray);
         assertThat(ref.length(), Matchers.equalTo(length));
@@ -46,14 +56,14 @@ public void testArray() throws IOException {
             BytesArray pbr = (BytesArray) newBytesReference(sizes[i]);
             byte[] array = pbr.array();
             assertNotNull(array);
-            assertEquals(sizes[i], array.length);
+            assertEquals(sizes[i], array.length - pbr.offset());
             assertSame(array, pbr.array());
         }
     }
 
     public void testArrayOffset() throws IOException {
         int length = randomInt(PAGE_SIZE * randomIntBetween(2, 5));
-        BytesArray pbr = (BytesArray) newBytesReference(length);
+        BytesArray pbr = (BytesArray) newBytesReferenceWithOffsetOfZero(length);
         assertEquals(0, pbr.offset());
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/common/bytes/CompositeBytesReferenceTests.java b/core/src/test/java/org/elasticsearch/common/bytes/CompositeBytesReferenceTests.java
index aec957aba68be..05d7b8dea6a5f 100644
--- a/core/src/test/java/org/elasticsearch/common/bytes/CompositeBytesReferenceTests.java
+++ b/core/src/test/java/org/elasticsearch/common/bytes/CompositeBytesReferenceTests.java
@@ -29,8 +29,14 @@
 import java.util.List;
 
 public class CompositeBytesReferenceTests extends AbstractBytesReferenceTestCase {
+
     @Override
     protected BytesReference newBytesReference(int length) throws IOException {
+        return newBytesReferenceWithOffsetOfZero(length);
+    }
+
+    @Override
+    protected BytesReference newBytesReferenceWithOffsetOfZero(int length) throws IOException {
         // we know bytes stream output always creates a paged bytes reference, we use it to create randomized content
         List<BytesReference> referenceList = newRefList(length);
         BytesReference ref = new CompositeBytesReference(referenceList.toArray(new BytesReference[0]));
diff --git a/core/src/test/java/org/elasticsearch/common/bytes/PagedBytesReferenceTests.java b/core/src/test/java/org/elasticsearch/common/bytes/PagedBytesReferenceTests.java
index 6ae2b3cf943b8..ea592b12e3b1f 100644
--- a/core/src/test/java/org/elasticsearch/common/bytes/PagedBytesReferenceTests.java
+++ b/core/src/test/java/org/elasticsearch/common/bytes/PagedBytesReferenceTests.java
@@ -37,7 +37,13 @@
 
 public class PagedBytesReferenceTests extends AbstractBytesReferenceTestCase {
 
+    @Override
     protected BytesReference newBytesReference(int length) throws IOException {
+        return newBytesReferenceWithOffsetOfZero(length);
+    }
+
+    @Override
+    protected BytesReference newBytesReferenceWithOffsetOfZero(int length) throws IOException {
         // we know bytes stream output always creates a paged bytes reference, we use it to create randomized content
         ReleasableBytesStreamOutput out = new ReleasableBytesStreamOutput(length, bigarrays);
         for (int i = 0; i < length; i++) {
diff --git a/core/src/test/java/org/elasticsearch/common/cache/CacheBuilderTests.java b/core/src/test/java/org/elasticsearch/common/cache/CacheBuilderTests.java
new file mode 100644
index 0000000000000..e0a5786e18475
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/common/cache/CacheBuilderTests.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.cache;
+
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.test.ESTestCase;
+
+import static org.hamcrest.Matchers.containsString;
+
+public class CacheBuilderTests extends ESTestCase {
+
+    public void testSettingExpireAfterAccess() {
+        IllegalArgumentException iae =
+            expectThrows(IllegalArgumentException.class, () -> CacheBuilder.builder().setExpireAfterAccess(TimeValue.MINUS_ONE));
+        assertThat(iae.getMessage(), containsString("expireAfterAccess <="));
+        iae = expectThrows(IllegalArgumentException.class, () -> CacheBuilder.builder().setExpireAfterAccess(TimeValue.ZERO));
+        assertThat(iae.getMessage(), containsString("expireAfterAccess <="));
+        final TimeValue timeValue = TimeValue.parseTimeValue(randomPositiveTimeValue(), "");
+        Cache<Object, Object> cache = CacheBuilder.builder().setExpireAfterAccess(timeValue).build();
+        assertEquals(timeValue.getNanos(), cache.getExpireAfterAccessNanos());
+    }
+
+    public void testSettingExpireAfterWrite() {
+        IllegalArgumentException iae =
+            expectThrows(IllegalArgumentException.class, () -> CacheBuilder.builder().setExpireAfterWrite(TimeValue.MINUS_ONE));
+        assertThat(iae.getMessage(), containsString("expireAfterWrite <="));
+        iae = expectThrows(IllegalArgumentException.class, () -> CacheBuilder.builder().setExpireAfterWrite(TimeValue.ZERO));
+        assertThat(iae.getMessage(), containsString("expireAfterWrite <="));
+        final TimeValue timeValue = TimeValue.parseTimeValue(randomPositiveTimeValue(), "");
+        Cache<Object, Object> cache = CacheBuilder.builder().setExpireAfterWrite(timeValue).build();
+        assertEquals(timeValue.getNanos(), cache.getExpireAfterWriteNanos());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/common/cache/CacheTests.java b/core/src/test/java/org/elasticsearch/common/cache/CacheTests.java
index 3b88a3bdcfe9b..5675a7b524bd3 100644
--- a/core/src/test/java/org/elasticsearch/common/cache/CacheTests.java
+++ b/core/src/test/java/org/elasticsearch/common/cache/CacheTests.java
@@ -228,7 +228,7 @@ protected long now() {
                 return now.get();
             }
         };
-        cache.setExpireAfterAccess(1);
+        cache.setExpireAfterAccessNanos(1);
         List<Integer> evictedKeys = new ArrayList<>();
         cache.setRemovalListener(notification -> {
             assertEquals(RemovalNotification.RemovalReason.EVICTED, notification.getRemovalReason());
@@ -257,6 +257,28 @@ protected long now() {
         }
     }
 
+    public void testSimpleExpireAfterAccess() {
+        AtomicLong now = new AtomicLong();
+        Cache<Integer, String> cache = new Cache<Integer, String>() {
+            @Override
+            protected long now() {
+                return now.get();
+            }
+        };
+        cache.setExpireAfterAccessNanos(1);
+        now.set(0);
+        for (int i = 0; i < numberOfEntries; i++) {
+            cache.put(i, Integer.toString(i));
+        }
+        for (int i = 0; i < numberOfEntries; i++) {
+            assertEquals(cache.get(i), Integer.toString(i));
+        }
+        now.set(2);
+        for(int i = 0; i < numberOfEntries; i++) {
+            assertNull(cache.get(i));
+        }
+    }
+
     public void testExpirationAfterWrite() {
         AtomicLong now = new AtomicLong();
         Cache<Integer, String> cache = new Cache<Integer, String>() {
@@ -265,7 +287,7 @@ protected long now() {
                 return now.get();
             }
         };
-        cache.setExpireAfterWrite(1);
+        cache.setExpireAfterWriteNanos(1);
         List<Integer> evictedKeys = new ArrayList<>();
         cache.setRemovalListener(notification -> {
             assertEquals(RemovalNotification.RemovalReason.EVICTED, notification.getRemovalReason());
@@ -297,6 +319,29 @@ protected long now() {
         }
     }
 
+    public void testComputeIfAbsentAfterExpiration() throws ExecutionException {
+        AtomicLong now = new AtomicLong();
+        Cache<Integer, String> cache = new Cache<Integer, String>() {
+            @Override
+            protected long now() {
+                return now.get();
+            }
+        };
+        cache.setExpireAfterAccessNanos(1);
+        now.set(0);
+        for (int i = 0; i < numberOfEntries; i++) {
+            cache.put(i, Integer.toString(i) + "-first");
+        }
+        now.set(2);
+        for (int i = 0; i < numberOfEntries; i++) {
+            cache.computeIfAbsent(i, k -> Integer.toString(k) + "-second");
+        }
+        for (int i = 0; i < numberOfEntries; i++) {
+            assertEquals(i + "-second", cache.get(i));
+        }
+        assertEquals(numberOfEntries, cache.stats().getEvictions());
+    }
+
     // randomly promote some entries, step the clock forward, then check that the promoted entries remain and the
     // non-promoted entries were removed
     public void testPromotion() {
@@ -307,7 +352,7 @@ protected long now() {
                 return now.get();
             }
         };
-        cache.setExpireAfterAccess(1);
+        cache.setExpireAfterAccessNanos(1);
         now.set(0);
         for (int i = 0; i < numberOfEntries; i++) {
             cache.put(i, Integer.toString(i));
@@ -416,7 +461,7 @@ class Value {
             private String value;
             private long weight;
 
-            public Value(String value, long weight) {
+            Value(String value, long weight) {
                 this.value = value;
                 this.weight = weight;
             }
@@ -552,7 +597,7 @@ public void testDependentKeyDeadlock() throws BrokenBarrierException, Interrupte
         class Key {
             private final int key;
 
-            public Key(int key) {
+            Key(int key) {
                 this.key = key;
             }
 
@@ -706,6 +751,43 @@ public void testCachePollution() throws BrokenBarrierException, InterruptedExcep
         barrier.await();
     }
 
+    public void testExceptionThrownDuringConcurrentComputeIfAbsent() throws BrokenBarrierException, InterruptedException {
+        int numberOfThreads = randomIntBetween(2, 32);
+        final Cache<String, String> cache = CacheBuilder.<String, String>builder().build();
+
+        CyclicBarrier barrier = new CyclicBarrier(1 + numberOfThreads);
+
+        final String key = randomAlphaOfLengthBetween(2, 32);
+        for (int i = 0; i < numberOfThreads; i++) {
+            Thread thread = new Thread(() -> {
+                try {
+                    barrier.await();
+                    for (int j = 0; j < numberOfEntries; j++) {
+                        try {
+                            String value = cache.computeIfAbsent(key, k -> {
+                                throw new RuntimeException("failed to load");
+                            });
+                            fail("expected exception but got: " + value);
+                        } catch (ExecutionException e) {
+                            assertNotNull(e.getCause());
+                            assertThat(e.getCause(), instanceOf(RuntimeException.class));
+                            assertEquals(e.getCause().getMessage(), "failed to load");
+                        }
+                    }
+                    barrier.await();
+                } catch (BrokenBarrierException | InterruptedException e) {
+                    throw new AssertionError(e);
+                }
+            });
+            thread.start();
+        }
+
+        // wait for all threads to be ready
+        barrier.await();
+        // wait for all threads to finish
+        barrier.await();
+    }
+
     // test that the cache is not corrupted under lots of concurrent modifications, even hitting the same key
     // here be dragons: this test did catch one subtle bug during development; do not remove lightly
     public void testTorture() throws BrokenBarrierException, InterruptedException {
diff --git a/core/src/test/java/org/elasticsearch/common/geo/GeoDistanceTests.java b/core/src/test/java/org/elasticsearch/common/geo/GeoDistanceTests.java
index 416299f8e7ee5..ae9042237ac73 100644
--- a/core/src/test/java/org/elasticsearch/common/geo/GeoDistanceTests.java
+++ b/core/src/test/java/org/elasticsearch/common/geo/GeoDistanceTests.java
@@ -18,16 +18,20 @@
  */
 package org.elasticsearch.common.geo;
 
+import org.apache.lucene.geo.Rectangle;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.unit.DistanceUnit;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.VersionUtils;
 
 import java.io.IOException;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.isOneOf;
 import static org.hamcrest.Matchers.lessThan;
 
 /**
@@ -38,12 +42,10 @@ public class GeoDistanceTests extends ESTestCase {
     public void testGeoDistanceSerialization() throws IOException  {
         // make sure that ordinals don't change, because we rely on then in serialization
         assertThat(GeoDistance.PLANE.ordinal(), equalTo(0));
-        assertThat(GeoDistance.FACTOR.ordinal(), equalTo(1));
-        assertThat(GeoDistance.ARC.ordinal(), equalTo(2));
-        assertThat(GeoDistance.SLOPPY_ARC.ordinal(), equalTo(3));
-        assertThat(GeoDistance.values().length, equalTo(4));
+        assertThat(GeoDistance.ARC.ordinal(), equalTo(1));
+        assertThat(GeoDistance.values().length, equalTo(2));
 
-        GeoDistance geoDistance = randomFrom(GeoDistance.PLANE, GeoDistance.FACTOR, GeoDistance.ARC, GeoDistance.SLOPPY_ARC);
+        GeoDistance geoDistance = randomFrom(GeoDistance.PLANE, GeoDistance.ARC);
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             geoDistance.writeTo(out);
             try (StreamInput in = out.bytes().streamInput()) {;
@@ -68,18 +70,57 @@ public void testInvalidReadFrom() throws Exception {
         }
     }
 
+    public void testReadFromSerializationBWC() throws Exception {
+        int ordinal = randomInt(3);
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            out.writeVInt(ordinal);
+            try (StreamInput in = out.bytes().streamInput()) {
+                // set client version (should this be done in .streamInput()?)
+                in.setVersion(VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.V_5_3_2));
+                GeoDistance copy = GeoDistance.readFromStream(in);
+                assertThat(copy, isOneOf(GeoDistance.PLANE, GeoDistance.ARC));
+                if (ordinal == 1) {
+                    assertWarnings("[factor] is deprecated. Using [plane] instead.");
+                } else if (ordinal == 3) {
+                    assertWarnings("[sloppy_arc] is deprecated. Using [arc] instead.");
+                }
+            }
+        }
+    }
+
+    public void testWriteToSerializationBWC() throws Exception {
+        GeoDistance geoDistance = randomFrom(GeoDistance.PLANE, GeoDistance.ARC);
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            out.setVersion(VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.V_5_3_2));
+            geoDistance.writeTo(out);
+            try (StreamInput in = out.bytes().streamInput()) {
+                in.setVersion(out.getVersion());
+                GeoDistance copy = GeoDistance.readFromStream(in);
+                assertThat(copy, isOneOf(GeoDistance.PLANE, GeoDistance.ARC));
+            }
+        }
+    }
+
     public void testDistanceCheck() {
         // Note, is within is an approximation, so, even though 0.52 is outside 50mi, we still get "true"
-        GeoDistance.DistanceBoundingCheck check = GeoDistance.distanceBoundingCheck(0, 0, 50, DistanceUnit.MILES);
-        assertThat(check.isWithin(0.5, 0.5), equalTo(true));
-        assertThat(check.isWithin(0.52, 0.52), equalTo(true));
-        assertThat(check.isWithin(1, 1), equalTo(false));
+        double radius = DistanceUnit.convert(50, DistanceUnit.MILES, DistanceUnit.METERS);
+        Rectangle box = Rectangle.fromPointDistance(0, 0, radius);
+        assertThat(GeoUtils.rectangleContainsPoint(box, 0.5, 0.5), equalTo(true));
+        assertThat(GeoUtils.rectangleContainsPoint(box, 0.52, 0.52), equalTo(true));
+        assertThat(GeoUtils.rectangleContainsPoint(box, 1, 1), equalTo(false));
 
-        check = GeoDistance.distanceBoundingCheck(0, 179, 200, DistanceUnit.MILES);
-        assertThat(check.isWithin(0, -179), equalTo(true));
-        assertThat(check.isWithin(0, -178), equalTo(false));
+        radius = DistanceUnit.convert(200, DistanceUnit.MILES, DistanceUnit.METERS);
+        box = Rectangle.fromPointDistance(0, 179, radius);
+        assertThat(GeoUtils.rectangleContainsPoint(box, 0, -179), equalTo(true));
+        assertThat(GeoUtils.rectangleContainsPoint(box, 0, -178), equalTo(false));
     }
 
+    /**
+     * The old plane calculation in 1.x/2.x incorrectly computed the plane distance in decimal degrees. This test is
+     * well intended but bogus. todo: fix w/ new plane distance calculation
+     * note: plane distance error varies by latitude so the test will need to correctly estimate expected error
+     */
+    @AwaitsFix(bugUrl = "old plane calculation incorrectly computed everything in degrees. fix this bogus test")
     public void testArcDistanceVsPlaneInEllipsis() {
         GeoPoint centre = new GeoPoint(48.8534100, 2.3488000);
         GeoPoint northernPoint = new GeoPoint(48.8801108681, 2.35152032666);
diff --git a/core/src/test/java/org/elasticsearch/common/geo/GeoJSONShapeParserTests.java b/core/src/test/java/org/elasticsearch/common/geo/GeoJSONShapeParserTests.java
index 76376a4d30d32..5ac50b91bd3f5 100644
--- a/core/src/test/java/org/elasticsearch/common/geo/GeoJSONShapeParserTests.java
+++ b/core/src/test/java/org/elasticsearch/common/geo/GeoJSONShapeParserTests.java
@@ -19,13 +19,6 @@
 
 package org.elasticsearch.common.geo;
 
-import org.locationtech.spatial4j.exception.InvalidShapeException;
-import org.locationtech.spatial4j.shape.Circle;
-import org.locationtech.spatial4j.shape.Rectangle;
-import org.locationtech.spatial4j.shape.Shape;
-import org.locationtech.spatial4j.shape.ShapeCollection;
-import org.locationtech.spatial4j.shape.jts.JtsGeometry;
-import org.locationtech.spatial4j.shape.jts.JtsPoint;
 import com.vividsolutions.jts.geom.Coordinate;
 import com.vividsolutions.jts.geom.Geometry;
 import com.vividsolutions.jts.geom.GeometryFactory;
@@ -37,11 +30,19 @@
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.geo.builders.ShapeBuilder;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.hamcrest.ElasticsearchGeoAssertions;
+import org.locationtech.spatial4j.exception.InvalidShapeException;
+import org.locationtech.spatial4j.shape.Circle;
+import org.locationtech.spatial4j.shape.Rectangle;
+import org.locationtech.spatial4j.shape.Shape;
+import org.locationtech.spatial4j.shape.ShapeCollection;
+import org.locationtech.spatial4j.shape.jts.JtsGeometry;
+import org.locationtech.spatial4j.shape.jts.JtsPoint;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -58,22 +59,26 @@ public class GeoJSONShapeParserTests extends ESTestCase {
 
     private static final GeometryFactory GEOMETRY_FACTORY = SPATIAL_CONTEXT.getGeometryFactory();
 
-    public void testParse_simplePoint() throws IOException {
-        String pointGeoJson = XContentFactory.jsonBuilder().startObject().field("type", "Point")
-                .startArray("coordinates").value(100.0).value(0.0).endArray()
-                .endObject().string();
+    public void testParseSimplePoint() throws IOException {
+        XContentBuilder pointGeoJson = XContentFactory.jsonBuilder()
+                .startObject()
+                    .field("type", "Point")
+                    .startArray("coordinates").value(100.0).value(0.0).endArray()
+                .endObject();
 
         Point expected = GEOMETRY_FACTORY.createPoint(new Coordinate(100.0, 0.0));
         assertGeometryEquals(new JtsPoint(expected, SPATIAL_CONTEXT), pointGeoJson);
     }
 
-    public void testParse_lineString() throws IOException {
-        String lineGeoJson = XContentFactory.jsonBuilder().startObject().field("type", "LineString")
-                .startArray("coordinates")
-                .startArray().value(100.0).value(0.0).endArray()
-                .startArray().value(101.0).value(1.0).endArray()
-                .endArray()
-                .endObject().string();
+    public void testParseLineString() throws IOException {
+        XContentBuilder lineGeoJson = XContentFactory.jsonBuilder()
+                .startObject()
+                    .field("type", "LineString")
+                    .startArray("coordinates")
+                    .startArray().value(100.0).value(0.0).endArray()
+                    .startArray().value(101.0).value(1.0).endArray()
+                    .endArray()
+                .endObject();
 
         List<Coordinate> lineCoordinates = new ArrayList<>();
         lineCoordinates.add(new Coordinate(100, 0));
@@ -84,19 +89,21 @@ public void testParse_lineString() throws IOException {
         assertGeometryEquals(jtsGeom(expected), lineGeoJson);
     }
 
-    public void testParse_multiLineString() throws IOException {
-        String multilinesGeoJson = XContentFactory.jsonBuilder().startObject().field("type", "MultiLineString")
-                .startArray("coordinates")
-                .startArray()
-                .startArray().value(100.0).value(0.0).endArray()
-                .startArray().value(101.0).value(1.0).endArray()
-                .endArray()
-                .startArray()
-                .startArray().value(102.0).value(2.0).endArray()
-                .startArray().value(103.0).value(3.0).endArray()
-                .endArray()
-                .endArray()
-                .endObject().string();
+    public void testParseMultiLineString() throws IOException {
+        XContentBuilder multilinesGeoJson = XContentFactory.jsonBuilder()
+                .startObject()
+                    .field("type", "MultiLineString")
+                    .startArray("coordinates")
+                        .startArray()
+                            .startArray().value(100.0).value(0.0).endArray()
+                            .startArray().value(101.0).value(1.0).endArray()
+                        .endArray()
+                        .startArray()
+                            .startArray().value(102.0).value(2.0).endArray()
+                            .startArray().value(103.0).value(3.0).endArray()
+                        .endArray()
+                    .endArray()
+                .endObject();
 
         MultiLineString expected = GEOMETRY_FACTORY.createMultiLineString(new LineString[]{
                 GEOMETRY_FACTORY.createLineString(new Coordinate[]{
@@ -111,32 +118,38 @@ public void testParse_multiLineString() throws IOException {
         assertGeometryEquals(jtsGeom(expected), multilinesGeoJson);
     }
 
-    public void testParse_circle() throws IOException {
-        String multilinesGeoJson = XContentFactory.jsonBuilder().startObject().field("type", "circle")
-                .startArray("coordinates").value(100.0).value(0.0).endArray()
-                .field("radius", "100m")
-                .endObject().string();
+    public void testParseCircle() throws IOException {
+        XContentBuilder multilinesGeoJson = XContentFactory.jsonBuilder()
+                .startObject()
+                    .field("type", "circle")
+                    .startArray("coordinates").value(100.0).value(0.0).endArray()
+                    .field("radius", "100m")
+                .endObject();
 
         Circle expected = SPATIAL_CONTEXT.makeCircle(100.0, 0.0, 360 * 100 / GeoUtils.EARTH_EQUATOR);
         assertGeometryEquals(expected, multilinesGeoJson);
     }
 
-    public void testParse_multiDimensionShapes() throws IOException {
+    public void testParseMultiDimensionShapes() throws IOException {
         // multi dimension point
-        String pointGeoJson = XContentFactory.jsonBuilder().startObject().field("type", "Point")
-                .startArray("coordinates").value(100.0).value(0.0).value(15.0).value(18.0).endArray()
-                .endObject().string();
+        XContentBuilder pointGeoJson = XContentFactory.jsonBuilder()
+                .startObject()
+                    .field("type", "Point")
+                    .startArray("coordinates").value(100.0).value(0.0).value(15.0).value(18.0).endArray()
+                .endObject();
 
         Point expectedPt = GEOMETRY_FACTORY.createPoint(new Coordinate(100.0, 0.0));
         assertGeometryEquals(new JtsPoint(expectedPt, SPATIAL_CONTEXT), pointGeoJson);
 
         // multi dimension linestring
-        String lineGeoJson = XContentFactory.jsonBuilder().startObject().field("type", "LineString")
-                .startArray("coordinates")
-                .startArray().value(100.0).value(0.0).value(15.0).endArray()
-                .startArray().value(101.0).value(1.0).value(18.0).value(19.0).endArray()
-                .endArray()
-                .endObject().string();
+        XContentBuilder lineGeoJson = XContentFactory.jsonBuilder()
+                .startObject()
+                    .field("type", "LineString")
+                    .startArray("coordinates")
+                        .startArray().value(100.0).value(0.0).value(15.0).endArray()
+                        .startArray().value(101.0).value(1.0).value(18.0).value(19.0).endArray()
+                    .endArray()
+                .endObject();
 
         List<Coordinate> lineCoordinates = new ArrayList<>();
         lineCoordinates.add(new Coordinate(100, 0));
@@ -147,14 +160,14 @@ public void testParse_multiDimensionShapes() throws IOException {
         assertGeometryEquals(jtsGeom(expectedLS), lineGeoJson);
     }
 
-    public void testParse_envelope() throws IOException {
+    public void testParseEnvelope() throws IOException {
         // test #1: envelope with expected coordinate order (TopLeft, BottomRight)
-        String multilinesGeoJson = XContentFactory.jsonBuilder().startObject().field("type", "envelope")
+        XContentBuilder multilinesGeoJson = XContentFactory.jsonBuilder().startObject().field("type", "envelope")
                 .startArray("coordinates")
                 .startArray().value(-50).value(30).endArray()
                 .startArray().value(50).value(-30).endArray()
                 .endArray()
-                .endObject().string();
+                .endObject();
 
         Rectangle expected = SPATIAL_CONTEXT.makeRectangle(-50, 50, -30, 30);
         assertGeometryEquals(expected, multilinesGeoJson);
@@ -165,7 +178,7 @@ public void testParse_envelope() throws IOException {
                 .startArray().value(50).value(30).endArray()
                 .startArray().value(-50).value(-30).endArray()
                 .endArray()
-                .endObject().string();
+                .endObject();
 
         expected = SPATIAL_CONTEXT.makeRectangle(-50, 50, -30, 30);
         assertGeometryEquals(expected, multilinesGeoJson);
@@ -177,8 +190,8 @@ public void testParse_envelope() throws IOException {
                 .startArray().value(-50).value(-30).endArray()
                 .startArray().value(50).value(-39).endArray()
                 .endArray()
-                .endObject().string();
-        XContentParser parser = JsonXContent.jsonXContent.createParser(multilinesGeoJson);
+                .endObject();
+        XContentParser parser = createParser(multilinesGeoJson);
         parser.nextToken();
         ElasticsearchGeoAssertions.assertValidException(parser, ElasticsearchParseException.class);
 
@@ -186,24 +199,26 @@ public void testParse_envelope() throws IOException {
         multilinesGeoJson = XContentFactory.jsonBuilder().startObject().field("type", "envelope")
                 .startArray("coordinates")
                 .endArray()
-                .endObject().string();
-        parser = JsonXContent.jsonXContent.createParser(multilinesGeoJson);
+                .endObject();
+        parser = createParser(multilinesGeoJson);
         parser.nextToken();
         ElasticsearchGeoAssertions.assertValidException(parser, ElasticsearchParseException.class);
     }
 
-    public void testParse_polygonNoHoles() throws IOException {
-        String polygonGeoJson = XContentFactory.jsonBuilder().startObject().field("type", "Polygon")
-                .startArray("coordinates")
-                .startArray()
-                .startArray().value(100.0).value(1.0).endArray()
-                .startArray().value(101.0).value(1.0).endArray()
-                .startArray().value(101.0).value(0.0).endArray()
-                .startArray().value(100.0).value(0.0).endArray()
-                .startArray().value(100.0).value(1.0).endArray()
-                .endArray()
-                .endArray()
-                .endObject().string();
+    public void testParsePolygonNoHoles() throws IOException {
+        XContentBuilder polygonGeoJson = XContentFactory.jsonBuilder()
+                .startObject()
+                    .field("type", "Polygon")
+                    .startArray("coordinates")
+                        .startArray()
+                            .startArray().value(100.0).value(1.0).endArray()
+                            .startArray().value(101.0).value(1.0).endArray()
+                            .startArray().value(101.0).value(0.0).endArray()
+                            .startArray().value(100.0).value(0.0).endArray()
+                            .startArray().value(100.0).value(1.0).endArray()
+                        .endArray()
+                    .endArray()
+                .endObject();
 
         List<Coordinate> shellCoordinates = new ArrayList<>();
         shellCoordinates.add(new Coordinate(100, 0));
@@ -217,57 +232,67 @@ public void testParse_polygonNoHoles() throws IOException {
         assertGeometryEquals(jtsGeom(expected), polygonGeoJson);
     }
 
-    public void testParse_invalidPoint() throws IOException {
+    public void testParseInvalidPoint() throws IOException {
         // test case 1: create an invalid point object with multipoint data format
-        String invalidPoint1 = XContentFactory.jsonBuilder().startObject().field("type", "point")
-                .startArray("coordinates")
-                .startArray().value(-74.011).value(40.753).endArray()
-                .endArray()
-                .endObject().string();
-        XContentParser parser = JsonXContent.jsonXContent.createParser(invalidPoint1);
+        XContentBuilder invalidPoint1 = XContentFactory.jsonBuilder()
+                .startObject()
+                    .field("type", "point")
+                    .startArray("coordinates")
+                        .startArray().value(-74.011).value(40.753).endArray()
+                    .endArray()
+                .endObject();
+        XContentParser parser = createParser(invalidPoint1);
         parser.nextToken();
         ElasticsearchGeoAssertions.assertValidException(parser, ElasticsearchParseException.class);
 
         // test case 2: create an invalid point object with an empty number of coordinates
-        String invalidPoint2 = XContentFactory.jsonBuilder().startObject().field("type", "point")
-                .startArray("coordinates")
-                .endArray()
-                .endObject().string();
-        parser = JsonXContent.jsonXContent.createParser(invalidPoint2);
+        XContentBuilder invalidPoint2 = XContentFactory.jsonBuilder()
+                .startObject()
+                    .field("type", "point")
+                    .startArray("coordinates")
+                    .endArray()
+                .endObject();
+        parser = createParser(invalidPoint2);
         parser.nextToken();
         ElasticsearchGeoAssertions.assertValidException(parser, ElasticsearchParseException.class);
     }
 
-    public void testParse_invalidMultipoint() throws IOException {
+    public void testParseInvalidMultipoint() throws IOException {
         // test case 1: create an invalid multipoint object with single coordinate
-        String invalidMultipoint1 = XContentFactory.jsonBuilder().startObject().field("type", "multipoint")
-                .startArray("coordinates").value(-74.011).value(40.753).endArray()
-                .endObject().string();
-        XContentParser parser = JsonXContent.jsonXContent.createParser(invalidMultipoint1);
+        XContentBuilder invalidMultipoint1 = XContentFactory.jsonBuilder()
+                .startObject()
+                    .field("type", "multipoint")
+                    .startArray("coordinates").value(-74.011).value(40.753).endArray()
+                .endObject();
+        XContentParser parser = createParser(invalidMultipoint1);
         parser.nextToken();
         ElasticsearchGeoAssertions.assertValidException(parser, ElasticsearchParseException.class);
 
         // test case 2: create an invalid multipoint object with null coordinate
-        String invalidMultipoint2 = XContentFactory.jsonBuilder().startObject().field("type", "multipoint")
-                .startArray("coordinates")
-                .endArray()
-                .endObject().string();
-        parser = JsonXContent.jsonXContent.createParser(invalidMultipoint2);
+        XContentBuilder invalidMultipoint2 = XContentFactory.jsonBuilder()
+                .startObject()
+                    .field("type", "multipoint")
+                    .startArray("coordinates")
+                    .endArray()
+                .endObject();
+        parser = createParser(invalidMultipoint2);
         parser.nextToken();
         ElasticsearchGeoAssertions.assertValidException(parser, ElasticsearchParseException.class);
 
         // test case 3: create a valid formatted multipoint object with invalid number (0) of coordinates
-        String invalidMultipoint3 = XContentFactory.jsonBuilder().startObject().field("type", "multipoint")
-                .startArray("coordinates")
-                .startArray().endArray()
-                .endArray()
-                .endObject().string();
-        parser = JsonXContent.jsonXContent.createParser(invalidMultipoint3);
+        XContentBuilder invalidMultipoint3 = XContentFactory.jsonBuilder()
+                .startObject()
+                    .field("type", "multipoint")
+                    .startArray("coordinates")
+                        .startArray().endArray()
+                    .endArray()
+                .endObject();
+        parser = createParser(invalidMultipoint3);
         parser.nextToken();
         ElasticsearchGeoAssertions.assertValidException(parser, ElasticsearchParseException.class);
     }
 
-    public void testParse_invalidMultiPolygon() throws IOException {
+    public void testParseInvalidMultiPolygon() throws IOException {
         // test invalid multipolygon (an "accidental" polygon with inner rings outside outer ring)
         String multiPolygonGeoJson = XContentFactory.jsonBuilder().startObject().field("type", "MultiPolygon")
                 .startArray("coordinates")
@@ -297,12 +322,12 @@ public void testParse_invalidMultiPolygon() throws IOException {
                 .endArray()
                 .endObject().string();
 
-        XContentParser parser = JsonXContent.jsonXContent.createParser(multiPolygonGeoJson);
+        XContentParser parser = createParser(JsonXContent.jsonXContent, multiPolygonGeoJson);
         parser.nextToken();
         ElasticsearchGeoAssertions.assertValidException(parser, InvalidShapeException.class);
     }
 
-    public void testParse_OGCPolygonWithoutHoles() throws IOException {
+    public void testParseOGCPolygonWithoutHoles() throws IOException {
         // test 1: ccw poly not crossing dateline
         String polygonGeoJson = XContentFactory.jsonBuilder().startObject().field("type", "Polygon")
                 .startArray("coordinates")
@@ -317,7 +342,7 @@ public void testParse_OGCPolygonWithoutHoles() throws IOException {
                 .endArray()
                 .endObject().string();
 
-        XContentParser parser = JsonXContent.jsonXContent.createParser(polygonGeoJson);
+        XContentParser parser = createParser(JsonXContent.jsonXContent, polygonGeoJson);
         parser.nextToken();
         Shape shape = ShapeBuilder.parse(parser).build();
 
@@ -337,7 +362,7 @@ public void testParse_OGCPolygonWithoutHoles() throws IOException {
                 .endArray()
                 .endObject().string();
 
-        parser = JsonXContent.jsonXContent.createParser(polygonGeoJson);
+        parser = createParser(JsonXContent.jsonXContent, polygonGeoJson);
         parser.nextToken();
         shape = ShapeBuilder.parse(parser).build();
 
@@ -357,7 +382,7 @@ public void testParse_OGCPolygonWithoutHoles() throws IOException {
                 .endArray()
                 .endObject().string();
 
-        parser = JsonXContent.jsonXContent.createParser(polygonGeoJson);
+        parser = createParser(JsonXContent.jsonXContent, polygonGeoJson);
         parser.nextToken();
         shape = ShapeBuilder.parse(parser).build();
 
@@ -377,14 +402,14 @@ public void testParse_OGCPolygonWithoutHoles() throws IOException {
                 .endArray()
                 .endObject().string();
 
-        parser = JsonXContent.jsonXContent.createParser(polygonGeoJson);
+        parser = createParser(JsonXContent.jsonXContent, polygonGeoJson);
         parser.nextToken();
         shape = ShapeBuilder.parse(parser).build();
 
         ElasticsearchGeoAssertions.assertMultiPolygon(shape);
     }
 
-    public void testParse_OGCPolygonWithHoles() throws IOException {
+    public void testParseOGCPolygonWithHoles() throws IOException {
         // test 1: ccw poly not crossing dateline
         String polygonGeoJson = XContentFactory.jsonBuilder().startObject().field("type", "Polygon")
                 .startArray("coordinates")
@@ -405,7 +430,7 @@ public void testParse_OGCPolygonWithHoles() throws IOException {
                 .endArray()
                 .endObject().string();
 
-        XContentParser parser = JsonXContent.jsonXContent.createParser(polygonGeoJson);
+        XContentParser parser = createParser(JsonXContent.jsonXContent, polygonGeoJson);
         parser.nextToken();
         Shape shape = ShapeBuilder.parse(parser).build();
 
@@ -431,7 +456,7 @@ public void testParse_OGCPolygonWithHoles() throws IOException {
                 .endArray()
                 .endObject().string();
 
-        parser = JsonXContent.jsonXContent.createParser(polygonGeoJson);
+        parser = createParser(JsonXContent.jsonXContent, polygonGeoJson);
         parser.nextToken();
         shape = ShapeBuilder.parse(parser).build();
 
@@ -457,7 +482,7 @@ public void testParse_OGCPolygonWithHoles() throws IOException {
                 .endArray()
                 .endObject().string();
 
-        parser = JsonXContent.jsonXContent.createParser(polygonGeoJson);
+        parser = createParser(JsonXContent.jsonXContent, polygonGeoJson);
         parser.nextToken();
         shape = ShapeBuilder.parse(parser).build();
 
@@ -483,14 +508,14 @@ public void testParse_OGCPolygonWithHoles() throws IOException {
                 .endArray()
                 .endObject().string();
 
-        parser = JsonXContent.jsonXContent.createParser(polygonGeoJson);
+        parser = createParser(JsonXContent.jsonXContent, polygonGeoJson);
         parser.nextToken();
         shape = ShapeBuilder.parse(parser).build();
 
         ElasticsearchGeoAssertions.assertMultiPolygon(shape);
     }
 
-    public void testParse_invalidPolygon() throws IOException {
+    public void testParseInvalidPolygon() throws IOException {
         /**
          * The following 3 test cases ensure proper error handling of invalid polygons
          * per the GeoJSON specification
@@ -504,7 +529,7 @@ public void testParse_invalidPolygon() throws IOException {
                 .endArray()
                 .endArray()
                 .endObject().string();
-        XContentParser parser = JsonXContent.jsonXContent.createParser(invalidPoly);
+        XContentParser parser = createParser(JsonXContent.jsonXContent, invalidPoly);
         parser.nextToken();
         ElasticsearchGeoAssertions.assertValidException(parser, ElasticsearchParseException.class);
 
@@ -517,7 +542,7 @@ public void testParse_invalidPolygon() throws IOException {
                 .endArray()
                 .endObject().string();
 
-        parser = JsonXContent.jsonXContent.createParser(invalidPoly);
+        parser = createParser(JsonXContent.jsonXContent, invalidPoly);
         parser.nextToken();
         ElasticsearchGeoAssertions.assertValidException(parser, ElasticsearchParseException.class);
 
@@ -530,7 +555,7 @@ public void testParse_invalidPolygon() throws IOException {
                 .endArray()
                 .endObject().string();
 
-        parser = JsonXContent.jsonXContent.createParser(invalidPoly);
+        parser = createParser(JsonXContent.jsonXContent, invalidPoly);
         parser.nextToken();
         ElasticsearchGeoAssertions.assertValidException(parser, ElasticsearchParseException.class);
 
@@ -543,7 +568,7 @@ public void testParse_invalidPolygon() throws IOException {
                 .endArray()
                 .endObject().string();
 
-        parser = JsonXContent.jsonXContent.createParser(invalidPoly);
+        parser = createParser(JsonXContent.jsonXContent, invalidPoly);
         parser.nextToken();
         ElasticsearchGeoAssertions.assertValidException(parser, IllegalArgumentException.class);
 
@@ -554,7 +579,7 @@ public void testParse_invalidPolygon() throws IOException {
                 .endArray()
                 .endObject().string();
 
-        parser = JsonXContent.jsonXContent.createParser(invalidPoly);
+        parser = createParser(JsonXContent.jsonXContent, invalidPoly);
         parser.nextToken();
         ElasticsearchGeoAssertions.assertValidException(parser, IllegalArgumentException.class);
 
@@ -563,7 +588,7 @@ public void testParse_invalidPolygon() throws IOException {
                 .startArray("coordinates").endArray()
                 .endObject().string();
 
-        parser = JsonXContent.jsonXContent.createParser(invalidPoly);
+        parser = createParser(JsonXContent.jsonXContent, invalidPoly);
         parser.nextToken();
         ElasticsearchGeoAssertions.assertValidException(parser, ElasticsearchParseException.class);
 
@@ -574,30 +599,32 @@ public void testParse_invalidPolygon() throws IOException {
                 .endArray()
                 .endObject().string();
 
-        parser = JsonXContent.jsonXContent.createParser(invalidPoly);
+        parser = createParser(JsonXContent.jsonXContent, invalidPoly);
         parser.nextToken();
         ElasticsearchGeoAssertions.assertValidException(parser, ElasticsearchParseException.class);
     }
 
-    public void testParse_polygonWithHole() throws IOException {
-        String polygonGeoJson = XContentFactory.jsonBuilder().startObject().field("type", "Polygon")
-                .startArray("coordinates")
-                .startArray()
-                .startArray().value(100.0).value(1.0).endArray()
-                .startArray().value(101.0).value(1.0).endArray()
-                .startArray().value(101.0).value(0.0).endArray()
-                .startArray().value(100.0).value(0.0).endArray()
-                .startArray().value(100.0).value(1.0).endArray()
-                .endArray()
-                .startArray()
-                .startArray().value(100.2).value(0.8).endArray()
-                .startArray().value(100.2).value(0.2).endArray()
-                .startArray().value(100.8).value(0.2).endArray()
-                .startArray().value(100.8).value(0.8).endArray()
-                .startArray().value(100.2).value(0.8).endArray()
-                .endArray()
-                .endArray()
-                .endObject().string();
+    public void testParsePolygonWithHole() throws IOException {
+        XContentBuilder polygonGeoJson = XContentFactory.jsonBuilder()
+                .startObject()
+                    .field("type", "Polygon")
+                    .startArray("coordinates")
+                        .startArray()
+                            .startArray().value(100.0).value(1.0).endArray()
+                            .startArray().value(101.0).value(1.0).endArray()
+                            .startArray().value(101.0).value(0.0).endArray()
+                            .startArray().value(100.0).value(0.0).endArray()
+                            .startArray().value(100.0).value(1.0).endArray()
+                        .endArray()
+                        .startArray()
+                            .startArray().value(100.2).value(0.8).endArray()
+                            .startArray().value(100.2).value(0.2).endArray()
+                            .startArray().value(100.8).value(0.2).endArray()
+                            .startArray().value(100.8).value(0.8).endArray()
+                            .startArray().value(100.2).value(0.8).endArray()
+                        .endArray()
+                    .endArray()
+                .endObject();
 
         // add 3d point to test ISSUE #10501
         List<Coordinate> shellCoordinates = new ArrayList<>();
@@ -623,7 +650,7 @@ public void testParse_polygonWithHole() throws IOException {
         assertGeometryEquals(jtsGeom(expected), polygonGeoJson);
     }
 
-    public void testParse_selfCrossingPolygon() throws IOException {
+    public void testParseSelfCrossingPolygon() throws IOException {
         // test self crossing ccw poly not crossing dateline
         String polygonGeoJson = XContentFactory.jsonBuilder().startObject().field("type", "Polygon")
                 .startArray("coordinates")
@@ -639,18 +666,20 @@ public void testParse_selfCrossingPolygon() throws IOException {
                 .endArray()
                 .endObject().string();
 
-        XContentParser parser = JsonXContent.jsonXContent.createParser(polygonGeoJson);
+        XContentParser parser = createParser(JsonXContent.jsonXContent, polygonGeoJson);
         parser.nextToken();
         ElasticsearchGeoAssertions.assertValidException(parser, InvalidShapeException.class);
     }
 
-    public void testParse_multiPoint() throws IOException {
-        String multiPointGeoJson = XContentFactory.jsonBuilder().startObject().field("type", "MultiPoint")
-                .startArray("coordinates")
-                .startArray().value(100.0).value(0.0).endArray()
-                .startArray().value(101.0).value(1.0).endArray()
-                .endArray()
-                .endObject().string();
+    public void testParseMultiPoint() throws IOException {
+        XContentBuilder multiPointGeoJson = XContentFactory.jsonBuilder()
+                .startObject()
+                    .field("type", "MultiPoint")
+                    .startArray("coordinates")
+                        .startArray().value(100.0).value(0.0).endArray()
+                        .startArray().value(101.0).value(1.0).endArray()
+                    .endArray()
+                .endObject();
 
         ShapeCollection expected = shapeCollection(
                 SPATIAL_CONTEXT.makePoint(100, 0),
@@ -658,37 +687,39 @@ public void testParse_multiPoint() throws IOException {
         assertGeometryEquals(expected, multiPointGeoJson);
     }
 
-    public void testParse_multiPolygon() throws IOException {
+    public void testParseMultiPolygon() throws IOException {
         // test #1: two polygons; one without hole, one with hole
-        String multiPolygonGeoJson = XContentFactory.jsonBuilder().startObject().field("type", "MultiPolygon")
-                .startArray("coordinates")
-                .startArray()//first poly (without holes)
-                .startArray()
-                .startArray().value(102.0).value(2.0).endArray()
-                .startArray().value(103.0).value(2.0).endArray()
-                .startArray().value(103.0).value(3.0).endArray()
-                .startArray().value(102.0).value(3.0).endArray()
-                .startArray().value(102.0).value(2.0).endArray()
-                .endArray()
-                .endArray()
-                .startArray()//second poly (with hole)
-                .startArray()
-                .startArray().value(100.0).value(0.0).endArray()
-                .startArray().value(101.0).value(0.0).endArray()
-                .startArray().value(101.0).value(1.0).endArray()
-                .startArray().value(100.0).value(1.0).endArray()
-                .startArray().value(100.0).value(0.0).endArray()
-                .endArray()
-                .startArray()//hole
-                .startArray().value(100.2).value(0.8).endArray()
-                .startArray().value(100.2).value(0.2).endArray()
-                .startArray().value(100.8).value(0.2).endArray()
-                .startArray().value(100.8).value(0.8).endArray()
-                .startArray().value(100.2).value(0.8).endArray()
-                .endArray()
-                .endArray()
-                .endArray()
-                .endObject().string();
+        XContentBuilder multiPolygonGeoJson = XContentFactory.jsonBuilder()
+                .startObject()
+                    .field("type", "MultiPolygon")
+                    .startArray("coordinates")
+                        .startArray()//first poly (without holes)
+                            .startArray()
+                                .startArray().value(102.0).value(2.0).endArray()
+                                .startArray().value(103.0).value(2.0).endArray()
+                                .startArray().value(103.0).value(3.0).endArray()
+                                .startArray().value(102.0).value(3.0).endArray()
+                                .startArray().value(102.0).value(2.0).endArray()
+                            .endArray()
+                        .endArray()
+                        .startArray()//second poly (with hole)
+                            .startArray()
+                                .startArray().value(100.0).value(0.0).endArray()
+                                .startArray().value(101.0).value(0.0).endArray()
+                                .startArray().value(101.0).value(1.0).endArray()
+                                .startArray().value(100.0).value(1.0).endArray()
+                                .startArray().value(100.0).value(0.0).endArray()
+                            .endArray()
+                            .startArray()//hole
+                                .startArray().value(100.2).value(0.8).endArray()
+                                .startArray().value(100.2).value(0.2).endArray()
+                                .startArray().value(100.8).value(0.2).endArray()
+                                .startArray().value(100.8).value(0.8).endArray()
+                                .startArray().value(100.2).value(0.8).endArray()
+                            .endArray()
+                        .endArray()
+                    .endArray()
+                .endObject();
 
         List<Coordinate> shellCoordinates = new ArrayList<>();
         shellCoordinates.add(new Coordinate(100, 0));
@@ -727,26 +758,28 @@ public void testParse_multiPolygon() throws IOException {
         // test #2: multipolygon; one polygon with one hole
         // this test converting the multipolygon from a ShapeCollection type
         // to a simple polygon (jtsGeom)
-        multiPolygonGeoJson = XContentFactory.jsonBuilder().startObject().field("type", "MultiPolygon")
-                .startArray("coordinates")
-                .startArray()
-                .startArray()
-                .startArray().value(100.0).value(1.0).endArray()
-                .startArray().value(101.0).value(1.0).endArray()
-                .startArray().value(101.0).value(0.0).endArray()
-                .startArray().value(100.0).value(0.0).endArray()
-                .startArray().value(100.0).value(1.0).endArray()
-                .endArray()
-                .startArray()// hole
-                .startArray().value(100.2).value(0.8).endArray()
-                .startArray().value(100.2).value(0.2).endArray()
-                .startArray().value(100.8).value(0.2).endArray()
-                .startArray().value(100.8).value(0.8).endArray()
-                .startArray().value(100.2).value(0.8).endArray()
-                .endArray()
-                .endArray()
-                .endArray()
-                .endObject().string();
+        multiPolygonGeoJson = XContentFactory.jsonBuilder()
+                .startObject()
+                .field("type", "MultiPolygon")
+                    .startArray("coordinates")
+                        .startArray()
+                            .startArray()
+                                .startArray().value(100.0).value(1.0).endArray()
+                                .startArray().value(101.0).value(1.0).endArray()
+                                .startArray().value(101.0).value(0.0).endArray()
+                                .startArray().value(100.0).value(0.0).endArray()
+                                .startArray().value(100.0).value(1.0).endArray()
+                            .endArray()
+                            .startArray() // hole
+                                .startArray().value(100.2).value(0.8).endArray()
+                                .startArray().value(100.2).value(0.2).endArray()
+                                .startArray().value(100.8).value(0.2).endArray()
+                                .startArray().value(100.8).value(0.8).endArray()
+                                .startArray().value(100.2).value(0.8).endArray()
+                            .endArray()
+                        .endArray()
+                    .endArray()
+                .endObject();
 
         shellCoordinates = new ArrayList<>();
         shellCoordinates.add(new Coordinate(100, 1));
@@ -770,24 +803,24 @@ public void testParse_multiPolygon() throws IOException {
         assertGeometryEquals(jtsGeom(withHoles), multiPolygonGeoJson);
     }
 
-    public void testParse_geometryCollection() throws IOException {
-        String geometryCollectionGeoJson = XContentFactory.jsonBuilder().startObject()
-                .field("type", "GeometryCollection")
-                .startArray("geometries")
-                    .startObject()
-                        .field("type", "LineString")
-                        .startArray("coordinates")
-                            .startArray().value(100.0).value(0.0).endArray()
-                            .startArray().value(101.0).value(1.0).endArray()
-                        .endArray()
-                    .endObject()
-                    .startObject()
-                        .field("type", "Point")
-                        .startArray("coordinates").value(102.0).value(2.0).endArray()
-                    .endObject()
-                .endArray()
-                .endObject()
-                .string();
+    public void testParseGeometryCollection() throws IOException {
+        XContentBuilder geometryCollectionGeoJson = XContentFactory.jsonBuilder()
+                .startObject()
+                    .field("type", "GeometryCollection")
+                    .startArray("geometries")
+                        .startObject()
+                            .field("type", "LineString")
+                            .startArray("coordinates")
+                                .startArray().value(100.0).value(0.0).endArray()
+                                .startArray().value(101.0).value(1.0).endArray()
+                            .endArray()
+                        .endObject()
+                        .startObject()
+                            .field("type", "Point")
+                            .startArray("coordinates").value(102.0).value(2.0).endArray()
+                        .endObject()
+                    .endArray()
+                .endObject();
 
         Shape[] expected = new Shape[2];
         LineString expectedLineString = GEOMETRY_FACTORY.createLineString(new Coordinate[]{
@@ -803,191 +836,204 @@ public void testParse_geometryCollection() throws IOException {
     }
 
     public void testThatParserExtractsCorrectTypeAndCoordinatesFromArbitraryJson() throws IOException {
-        String pointGeoJson = XContentFactory.jsonBuilder().startObject()
-                .startObject("crs")
-                    .field("type", "name")
-                    .startObject("properties")
-                        .field("name", "urn:ogc:def:crs:OGC:1.3:CRS84")
+        XContentBuilder pointGeoJson = XContentFactory.jsonBuilder()
+                .startObject()
+                    .startObject("crs")
+                        .field("type", "name")
+                        .startObject("properties")
+                            .field("name", "urn:ogc:def:crs:OGC:1.3:CRS84")
+                        .endObject()
                     .endObject()
-                .endObject()
-                .field("bbox", "foobar")
-                .field("type", "point")
-                .field("bubu", "foobar")
-                .startArray("coordinates").value(100.0).value(0.0).endArray()
-                .startObject("nested").startArray("coordinates").value(200.0).value(0.0).endArray().endObject()
-                .startObject("lala").field("type", "NotAPoint").endObject()
-                .endObject().string();
+                    .field("bbox", "foobar")
+                    .field("type", "point")
+                    .field("bubu", "foobar")
+                    .startArray("coordinates").value(100.0).value(0.0).endArray()
+                    .startObject("nested").startArray("coordinates").value(200.0).value(0.0).endArray().endObject()
+                    .startObject("lala").field("type", "NotAPoint").endObject()
+                .endObject();
 
         Point expected = GEOMETRY_FACTORY.createPoint(new Coordinate(100.0, 0.0));
         assertGeometryEquals(new JtsPoint(expected, SPATIAL_CONTEXT), pointGeoJson);
     }
 
-    public void testParse_orientationOption() throws IOException {
+    public void testParseOrientationOption() throws IOException {
         // test 1: valid ccw (right handed system) poly not crossing dateline (with 'right' field)
-        String polygonGeoJson = XContentFactory.jsonBuilder().startObject().field("type", "Polygon")
-                .field("orientation", "right")
-                .startArray("coordinates")
-                .startArray()
-                .startArray().value(176.0).value(15.0).endArray()
-                .startArray().value(-177.0).value(10.0).endArray()
-                .startArray().value(-177.0).value(-10.0).endArray()
-                .startArray().value(176.0).value(-15.0).endArray()
-                .startArray().value(172.0).value(0.0).endArray()
-                .startArray().value(176.0).value(15.0).endArray()
-                .endArray()
-                .startArray()
-                .startArray().value(-172.0).value(8.0).endArray()
-                .startArray().value(174.0).value(10.0).endArray()
-                .startArray().value(-172.0).value(-8.0).endArray()
-                .startArray().value(-172.0).value(8.0).endArray()
-                .endArray()
-                .endArray()
-                .endObject().string();
+        XContentBuilder polygonGeoJson = XContentFactory.jsonBuilder()
+                .startObject()
+                    .field("type", "Polygon")
+                    .field("orientation", "right")
+                    .startArray("coordinates")
+                        .startArray()
+                            .startArray().value(176.0).value(15.0).endArray()
+                            .startArray().value(-177.0).value(10.0).endArray()
+                            .startArray().value(-177.0).value(-10.0).endArray()
+                            .startArray().value(176.0).value(-15.0).endArray()
+                            .startArray().value(172.0).value(0.0).endArray()
+                            .startArray().value(176.0).value(15.0).endArray()
+                        .endArray()
+                        .startArray()
+                            .startArray().value(-172.0).value(8.0).endArray()
+                            .startArray().value(174.0).value(10.0).endArray()
+                            .startArray().value(-172.0).value(-8.0).endArray()
+                            .startArray().value(-172.0).value(8.0).endArray()
+                        .endArray()
+                    .endArray()
+                .endObject();
 
-        XContentParser parser = JsonXContent.jsonXContent.createParser(polygonGeoJson);
+        XContentParser parser = createParser(polygonGeoJson);
         parser.nextToken();
         Shape shape = ShapeBuilder.parse(parser).build();
 
         ElasticsearchGeoAssertions.assertPolygon(shape);
 
         // test 2: valid ccw (right handed system) poly not crossing dateline (with 'ccw' field)
-        polygonGeoJson = XContentFactory.jsonBuilder().startObject().field("type", "Polygon")
-                .field("orientation", "ccw")
-                .startArray("coordinates")
-                .startArray()
-                .startArray().value(176.0).value(15.0).endArray()
-                .startArray().value(-177.0).value(10.0).endArray()
-                .startArray().value(-177.0).value(-10.0).endArray()
-                .startArray().value(176.0).value(-15.0).endArray()
-                .startArray().value(172.0).value(0.0).endArray()
-                .startArray().value(176.0).value(15.0).endArray()
-                .endArray()
-                .startArray()
-                .startArray().value(-172.0).value(8.0).endArray()
-                .startArray().value(174.0).value(10.0).endArray()
-                .startArray().value(-172.0).value(-8.0).endArray()
-                .startArray().value(-172.0).value(8.0).endArray()
-                .endArray()
-                .endArray()
-                .endObject().string();
+        polygonGeoJson = XContentFactory.jsonBuilder()
+                .startObject()
+                    .field("type", "Polygon")
+                    .field("orientation", "ccw")
+                    .startArray("coordinates")
+                        .startArray()
+                            .startArray().value(176.0).value(15.0).endArray()
+                            .startArray().value(-177.0).value(10.0).endArray()
+                            .startArray().value(-177.0).value(-10.0).endArray()
+                            .startArray().value(176.0).value(-15.0).endArray()
+                            .startArray().value(172.0).value(0.0).endArray()
+                            .startArray().value(176.0).value(15.0).endArray()
+                        .endArray()
+                        .startArray()
+                            .startArray().value(-172.0).value(8.0).endArray()
+                            .startArray().value(174.0).value(10.0).endArray()
+                            .startArray().value(-172.0).value(-8.0).endArray()
+                            .startArray().value(-172.0).value(8.0).endArray()
+                        .endArray()
+                    .endArray()
+                .endObject();
 
-        parser = JsonXContent.jsonXContent.createParser(polygonGeoJson);
+        parser = createParser(polygonGeoJson);
         parser.nextToken();
         shape = ShapeBuilder.parse(parser).build();
 
         ElasticsearchGeoAssertions.assertPolygon(shape);
 
         // test 3: valid ccw (right handed system) poly not crossing dateline (with 'counterclockwise' field)
-        polygonGeoJson = XContentFactory.jsonBuilder().startObject().field("type", "Polygon")
-                .field("orientation", "counterclockwise")
-                .startArray("coordinates")
-                .startArray()
-                .startArray().value(176.0).value(15.0).endArray()
-                .startArray().value(-177.0).value(10.0).endArray()
-                .startArray().value(-177.0).value(-10.0).endArray()
-                .startArray().value(176.0).value(-15.0).endArray()
-                .startArray().value(172.0).value(0.0).endArray()
-                .startArray().value(176.0).value(15.0).endArray()
-                .endArray()
-                .startArray()
-                .startArray().value(-172.0).value(8.0).endArray()
-                .startArray().value(174.0).value(10.0).endArray()
-                .startArray().value(-172.0).value(-8.0).endArray()
-                .startArray().value(-172.0).value(8.0).endArray()
-                .endArray()
-                .endArray()
-                .endObject().string();
+        polygonGeoJson = XContentFactory.jsonBuilder()
+                .startObject()
+                    .field("type", "Polygon")
+                    .field("orientation", "counterclockwise")
+                    .startArray("coordinates")
+                        .startArray()
+                            .startArray().value(176.0).value(15.0).endArray()
+                            .startArray().value(-177.0).value(10.0).endArray()
+                            .startArray().value(-177.0).value(-10.0).endArray()
+                            .startArray().value(176.0).value(-15.0).endArray()
+                            .startArray().value(172.0).value(0.0).endArray()
+                            .startArray().value(176.0).value(15.0).endArray()
+                        .endArray()
+                        .startArray()
+                            .startArray().value(-172.0).value(8.0).endArray()
+                            .startArray().value(174.0).value(10.0).endArray()
+                            .startArray().value(-172.0).value(-8.0).endArray()
+                            .startArray().value(-172.0).value(8.0).endArray()
+                        .endArray()
+                    .endArray()
+                .endObject();
 
-        parser = JsonXContent.jsonXContent.createParser(polygonGeoJson);
+        parser = createParser(polygonGeoJson);
         parser.nextToken();
         shape = ShapeBuilder.parse(parser).build();
 
         ElasticsearchGeoAssertions.assertPolygon(shape);
 
         // test 4: valid cw (left handed system) poly crossing dateline (with 'left' field)
-        polygonGeoJson = XContentFactory.jsonBuilder().startObject().field("type", "Polygon")
-                .field("orientation", "left")
-                .startArray("coordinates")
-                .startArray()
-                .startArray().value(176.0).value(15.0).endArray()
-                .startArray().value(-177.0).value(10.0).endArray()
-                .startArray().value(-177.0).value(-10.0).endArray()
-                .startArray().value(176.0).value(-15.0).endArray()
-                .startArray().value(172.0).value(0.0).endArray()
-                .startArray().value(176.0).value(15.0).endArray()
-                .endArray()
-                .startArray()
-                .startArray().value(-178.0).value(8.0).endArray()
-                .startArray().value(178.0).value(8.0).endArray()
-                .startArray().value(180.0).value(-8.0).endArray()
-                .startArray().value(-178.0).value(8.0).endArray()
-                .endArray()
-                .endArray()
-                .endObject().string();
+        polygonGeoJson = XContentFactory.jsonBuilder()
+                .startObject()
+                    .field("type", "Polygon")
+                    .field("orientation", "left")
+                    .startArray("coordinates")
+                        .startArray()
+                            .startArray().value(176.0).value(15.0).endArray()
+                            .startArray().value(-177.0).value(10.0).endArray()
+                            .startArray().value(-177.0).value(-10.0).endArray()
+                            .startArray().value(176.0).value(-15.0).endArray()
+                            .startArray().value(172.0).value(0.0).endArray()
+                            .startArray().value(176.0).value(15.0).endArray()
+                        .endArray()
+                        .startArray()
+                            .startArray().value(-178.0).value(8.0).endArray()
+                            .startArray().value(178.0).value(8.0).endArray()
+                            .startArray().value(180.0).value(-8.0).endArray()
+                            .startArray().value(-178.0).value(8.0).endArray()
+                        .endArray()
+                    .endArray()
+                .endObject();
 
-        parser = JsonXContent.jsonXContent.createParser(polygonGeoJson);
+        parser = createParser(polygonGeoJson);
         parser.nextToken();
         shape = ShapeBuilder.parse(parser).build();
 
         ElasticsearchGeoAssertions.assertMultiPolygon(shape);
 
         // test 5: valid cw multipoly (left handed system) poly crossing dateline (with 'cw' field)
-        polygonGeoJson = XContentFactory.jsonBuilder().startObject().field("type", "Polygon")
-                .field("orientation", "cw")
-                .startArray("coordinates")
-                .startArray()
-                .startArray().value(176.0).value(15.0).endArray()
-                .startArray().value(-177.0).value(10.0).endArray()
-                .startArray().value(-177.0).value(-10.0).endArray()
-                .startArray().value(176.0).value(-15.0).endArray()
-                .startArray().value(172.0).value(0.0).endArray()
-                .startArray().value(176.0).value(15.0).endArray()
-                .endArray()
-                .startArray()
-                .startArray().value(-178.0).value(8.0).endArray()
-                .startArray().value(178.0).value(8.0).endArray()
-                .startArray().value(180.0).value(-8.0).endArray()
-                .startArray().value(-178.0).value(8.0).endArray()
-                .endArray()
-                .endArray()
-                .endObject().string();
+        polygonGeoJson = XContentFactory.jsonBuilder()
+                .startObject()
+                    .field("type", "Polygon")
+                    .field("orientation", "cw")
+                    .startArray("coordinates")
+                        .startArray()
+                            .startArray().value(176.0).value(15.0).endArray()
+                            .startArray().value(-177.0).value(10.0).endArray()
+                            .startArray().value(-177.0).value(-10.0).endArray()
+                            .startArray().value(176.0).value(-15.0).endArray()
+                            .startArray().value(172.0).value(0.0).endArray()
+                            .startArray().value(176.0).value(15.0).endArray()
+                        .endArray()
+                        .startArray()
+                            .startArray().value(-178.0).value(8.0).endArray()
+                            .startArray().value(178.0).value(8.0).endArray()
+                            .startArray().value(180.0).value(-8.0).endArray()
+                            .startArray().value(-178.0).value(8.0).endArray()
+                        .endArray()
+                    .endArray()
+                .endObject();
 
-        parser = JsonXContent.jsonXContent.createParser(polygonGeoJson);
+        parser = createParser(polygonGeoJson);
         parser.nextToken();
         shape = ShapeBuilder.parse(parser).build();
 
         ElasticsearchGeoAssertions.assertMultiPolygon(shape);
 
         // test 6: valid cw multipoly (left handed system) poly crossing dateline (with 'clockwise' field)
-        polygonGeoJson = XContentFactory.jsonBuilder().startObject().field("type", "Polygon")
-                .field("orientation", "clockwise")
-                .startArray("coordinates")
-                .startArray()
-                .startArray().value(176.0).value(15.0).endArray()
-                .startArray().value(-177.0).value(10.0).endArray()
-                .startArray().value(-177.0).value(-10.0).endArray()
-                .startArray().value(176.0).value(-15.0).endArray()
-                .startArray().value(172.0).value(0.0).endArray()
-                .startArray().value(176.0).value(15.0).endArray()
-                .endArray()
-                .startArray()
-                .startArray().value(-178.0).value(8.0).endArray()
-                .startArray().value(178.0).value(8.0).endArray()
-                .startArray().value(180.0).value(-8.0).endArray()
-                .startArray().value(-178.0).value(8.0).endArray()
-                .endArray()
-                .endArray()
-                .endObject().string();
+        polygonGeoJson = XContentFactory.jsonBuilder()
+                .startObject()
+                    .field("type", "Polygon")
+                    .field("orientation", "clockwise")
+                    .startArray("coordinates")
+                        .startArray()
+                            .startArray().value(176.0).value(15.0).endArray()
+                            .startArray().value(-177.0).value(10.0).endArray()
+                            .startArray().value(-177.0).value(-10.0).endArray()
+                            .startArray().value(176.0).value(-15.0).endArray()
+                            .startArray().value(172.0).value(0.0).endArray()
+                            .startArray().value(176.0).value(15.0).endArray()
+                        .endArray()
+                        .startArray()
+                            .startArray().value(-178.0).value(8.0).endArray()
+                            .startArray().value(178.0).value(8.0).endArray()
+                            .startArray().value(180.0).value(-8.0).endArray()
+                            .startArray().value(-178.0).value(8.0).endArray()
+                        .endArray()
+                    .endArray()
+                .endObject();
 
-        parser = JsonXContent.jsonXContent.createParser(polygonGeoJson);
+        parser = createParser(polygonGeoJson);
         parser.nextToken();
         shape = ShapeBuilder.parse(parser).build();
 
         ElasticsearchGeoAssertions.assertMultiPolygon(shape);
     }
 
-    private void assertGeometryEquals(Shape expected, String geoJson) throws IOException {
-        XContentParser parser = JsonXContent.jsonXContent.createParser(geoJson);
+    private void assertGeometryEquals(Shape expected, XContentBuilder geoJson) throws IOException {
+        XContentParser parser = createParser(geoJson);
         parser.nextToken();
         ElasticsearchGeoAssertions.assertEquals(expected, ShapeBuilder.parse(parser).build());
     }
diff --git a/core/src/test/java/org/elasticsearch/common/geo/ShapeBuilderTests.java b/core/src/test/java/org/elasticsearch/common/geo/ShapeBuilderTests.java
index abbd6ce40aad7..dc01a39cb8145 100644
--- a/core/src/test/java/org/elasticsearch/common/geo/ShapeBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/common/geo/ShapeBuilderTests.java
@@ -19,12 +19,6 @@
 
 package org.elasticsearch.common.geo;
 
-import org.locationtech.spatial4j.exception.InvalidShapeException;
-import org.locationtech.spatial4j.shape.Circle;
-import org.locationtech.spatial4j.shape.Point;
-import org.locationtech.spatial4j.shape.Rectangle;
-import org.locationtech.spatial4j.shape.Shape;
-import org.locationtech.spatial4j.shape.impl.PointImpl;
 import com.vividsolutions.jts.geom.Coordinate;
 import com.vividsolutions.jts.geom.LineString;
 import com.vividsolutions.jts.geom.Polygon;
@@ -35,6 +29,12 @@
 import org.elasticsearch.common.geo.builders.ShapeBuilder;
 import org.elasticsearch.common.geo.builders.ShapeBuilders;
 import org.elasticsearch.test.ESTestCase;
+import org.locationtech.spatial4j.exception.InvalidShapeException;
+import org.locationtech.spatial4j.shape.Circle;
+import org.locationtech.spatial4j.shape.Point;
+import org.locationtech.spatial4j.shape.Rectangle;
+import org.locationtech.spatial4j.shape.Shape;
+import org.locationtech.spatial4j.shape.impl.PointImpl;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchGeoAssertions.assertMultiLineString;
 import static org.elasticsearch.test.hamcrest.ElasticsearchGeoAssertions.assertMultiPolygon;
@@ -183,17 +183,13 @@ public void testMultiLineString() {
     }
 
     public void testPolygonSelfIntersection() {
-        try {
-            ShapeBuilders.newPolygon(new CoordinatesBuilder()
+            PolygonBuilder newPolygon = ShapeBuilders.newPolygon(new CoordinatesBuilder()
                     .coordinate(-40.0, 50.0)
                     .coordinate(40.0, 50.0)
                     .coordinate(-40.0, -50.0)
-                    .coordinate(40.0, -50.0).close())
-                    .build();
-            fail("Expected InvalidShapeException");
-        } catch (InvalidShapeException e) {
-            assertThat(e.getMessage(), containsString("Self-intersection at or near point (0.0"));
-        }
+                    .coordinate(40.0, -50.0).close());
+        Exception e = expectThrows(InvalidShapeException.class, () -> newPolygon.build());
+        assertThat(e.getMessage(), containsString("Self-intersection at or near point (0.0"));
     }
 
     public void testGeoCircle() {
@@ -550,12 +546,8 @@ public void testShapeWithInvalidTangentialHole() {
                 .coordinate(179, -10)
                 .coordinate(164, 0)
                 ));
-        try {
-            builder.close().build();
-            fail("Expected InvalidShapeException");
-        } catch (InvalidShapeException e) {
-            assertThat(e.getMessage(), containsString("interior cannot share more than one point with the exterior"));
-        }
+        Exception e = expectThrows(InvalidShapeException.class, () -> builder.close().build());
+        assertThat(e.getMessage(), containsString("interior cannot share more than one point with the exterior"));
     }
 
     public void testBoundaryShapeWithTangentialHole() {
@@ -602,12 +594,8 @@ public void testBoundaryShapeWithInvalidTangentialHole() {
                 .coordinate(176, -10)
                 .coordinate(-177, 10)
                 ));
-        try {
-            builder.close().build();
-            fail("Expected InvalidShapeException");
-        } catch (InvalidShapeException e) {
-            assertThat(e.getMessage(), containsString("interior cannot share more than one point with the exterior"));
-        }
+        Exception e = expectThrows(InvalidShapeException.class, () -> builder.close().build());
+        assertThat(e.getMessage(), containsString("interior cannot share more than one point with the exterior"));
     }
 
     /**
@@ -659,11 +647,7 @@ public void testInvalidShapeWithConsecutiveDuplicatePoints() {
                 .coordinate(-176, 4)
                 .coordinate(180, 0)
                 );
-        try {
-            builder.close().build();
-            fail("Expected InvalidShapeException");
-        } catch (InvalidShapeException e) {
-            assertThat(e.getMessage(), containsString("duplicate consecutive coordinates at: ("));
-        }
+        Exception e = expectThrows(InvalidShapeException.class, () -> builder.close().build());
+        assertThat(e.getMessage(), containsString("duplicate consecutive coordinates at: ("));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/common/geo/builders/AbstractShapeBuilderTestCase.java b/core/src/test/java/org/elasticsearch/common/geo/builders/AbstractShapeBuilderTestCase.java
index 00acef88b89ea..c3e4781b3d27c 100644
--- a/core/src/test/java/org/elasticsearch/common/geo/builders/AbstractShapeBuilderTestCase.java
+++ b/core/src/test/java/org/elasticsearch/common/geo/builders/AbstractShapeBuilderTestCase.java
@@ -19,10 +19,9 @@
 
 package org.elasticsearch.common.geo.builders;
 
-import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.Writeable.Reader;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -35,11 +34,9 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
 
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.not;
+import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
 
 public abstract class AbstractShapeBuilderTestCase<SB extends ShapeBuilder> extends ESTestCase {
 
@@ -85,7 +82,7 @@ public void testFromXContent() throws IOException {
             }
             XContentBuilder builder = testShape.toXContent(contentBuilder, ToXContent.EMPTY_PARAMS);
             XContentBuilder shuffled = shuffleXContent(builder);
-            XContentParser shapeParser = XContentHelper.createParser(shuffled.bytes());
+            XContentParser shapeParser = createParser(shuffled);
             shapeParser.nextToken();
             ShapeBuilder parsedShape = ShapeBuilder.parse(shapeParser);
             assertNotSame(testShape, parsedShape);
@@ -97,11 +94,10 @@ public void testFromXContent() throws IOException {
     /**
      * Test serialization and deserialization of the test shape.
      */
-    @SuppressWarnings("unchecked")
     public void testSerialization() throws IOException {
         for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) {
             SB testShape = createTestShapeBuilder();
-            SB deserializedShape = (SB) copyShape(testShape);
+            SB deserializedShape = copyShape(testShape);
             assertEquals(testShape, deserializedShape);
             assertEquals(testShape.hashCode(), deserializedShape.hashCode());
             assertNotSame(testShape, deserializedShape);
@@ -111,40 +107,15 @@ public void testSerialization() throws IOException {
     /**
      * Test equality and hashCode properties
      */
-    @SuppressWarnings("unchecked")
     public void testEqualsAndHashcode() throws IOException {
         for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) {
-            SB firstShape = createTestShapeBuilder();
-            assertFalse("shape is equal to null", firstShape.equals(null));
-            assertFalse("shape is equal to incompatible type", firstShape.equals(""));
-            assertTrue("shape is not equal to self", firstShape.equals(firstShape));
-            assertThat("same shape's hashcode returns different values if called multiple times", firstShape.hashCode(),
-                    equalTo(firstShape.hashCode()));
-            assertThat("different shapes should not be equal", createMutation(firstShape), not(equalTo(firstShape)));
-
-            SB secondShape = (SB) copyShape(firstShape);
-            assertTrue("shape is not equal to self", secondShape.equals(secondShape));
-            assertTrue("shape is not equal to its copy", firstShape.equals(secondShape));
-            assertTrue("equals is not symmetric", secondShape.equals(firstShape));
-            assertThat("shape copy's hashcode is different from original hashcode", secondShape.hashCode(), equalTo(firstShape.hashCode()));
-
-            SB thirdShape = (SB) copyShape(secondShape);
-            assertTrue("shape is not equal to self", thirdShape.equals(thirdShape));
-            assertTrue("shape is not equal to its copy", secondShape.equals(thirdShape));
-            assertThat("shape copy's hashcode is different from original hashcode", secondShape.hashCode(), equalTo(thirdShape.hashCode()));
-            assertTrue("equals is not transitive", firstShape.equals(thirdShape));
-            assertThat("shape copy's hashcode is different from original hashcode", firstShape.hashCode(), equalTo(thirdShape.hashCode()));
-            assertTrue("equals is not symmetric", thirdShape.equals(secondShape));
-            assertTrue("equals is not symmetric", thirdShape.equals(firstShape));
+            checkEqualsAndHashCode(createTestShapeBuilder(), AbstractShapeBuilderTestCase::copyShape, this::createMutation);
         }
     }
 
-    static ShapeBuilder copyShape(ShapeBuilder original) throws IOException {
-        try (BytesStreamOutput output = new BytesStreamOutput()) {
-            original.writeTo(output);
-            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
-                return namedWriteableRegistry.getReader(ShapeBuilder.class, original.getWriteableName()).read(in);
-            }
-        }
+    protected static <T extends NamedWriteable> T copyShape(T original) throws IOException {
+        @SuppressWarnings("unchecked")
+        Reader<T> reader = (Reader<T>) namedWriteableRegistry.getReader(ShapeBuilder.class, original.getWriteableName());
+        return ESTestCase.copyWriteable(original, namedWriteableRegistry, reader);
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/common/geo/builders/CircleBuilderTests.java b/core/src/test/java/org/elasticsearch/common/geo/builders/CircleBuilderTests.java
index 3c9ca34c6eabc..348ac049f28d8 100644
--- a/core/src/test/java/org/elasticsearch/common/geo/builders/CircleBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/common/geo/builders/CircleBuilderTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.common.geo.builders;
 
 import com.vividsolutions.jts.geom.Coordinate;
+
 import org.elasticsearch.common.unit.DistanceUnit;
 
 import java.io.IOException;
@@ -37,7 +38,7 @@ protected CircleBuilder createMutation(CircleBuilder original) throws IOExceptio
     }
 
     static CircleBuilder mutate(CircleBuilder original) throws IOException {
-        CircleBuilder mutation = (CircleBuilder) copyShape(original);
+        CircleBuilder mutation = copyShape(original);
         double radius = original.radius();
         DistanceUnit unit = original.unit();
 
diff --git a/core/src/test/java/org/elasticsearch/common/hppc/HppcMapsTests.java b/core/src/test/java/org/elasticsearch/common/hppc/HppcMapsTests.java
index 0b23002a89018..1d340ad65dd9d 100644
--- a/core/src/test/java/org/elasticsearch/common/hppc/HppcMapsTests.java
+++ b/core/src/test/java/org/elasticsearch/common/hppc/HppcMapsTests.java
@@ -19,6 +19,7 @@
 package org.elasticsearch.common.hppc;
 
 import com.carrotsearch.hppc.ObjectHashSet;
+import org.elasticsearch.Assertions;
 import org.elasticsearch.common.collect.HppcMaps;
 import org.elasticsearch.test.ESTestCase;
 
@@ -29,9 +30,7 @@
 
 public class HppcMapsTests extends ESTestCase {
     public void testIntersection() throws Exception {
-        boolean enabled = false;
-        assert enabled = true;
-        assumeTrue("assertions enabled", enabled);
+        assumeTrue("assertions enabled", Assertions.ENABLED);
         ObjectHashSet<String> set1 = ObjectHashSet.from("1", "2", "3");
         ObjectHashSet<String> set2 = ObjectHashSet.from("1", "2", "3");
         List<String> values = toList(HppcMaps.intersection(set1, set2));
diff --git a/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java b/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java
index f1e6c1ba67e0a..1d5b6102385f1 100644
--- a/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java
+++ b/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java
@@ -19,7 +19,9 @@
 
 package org.elasticsearch.common.io.stream;
 
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.Constants;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.geo.GeoPoint;
@@ -28,8 +30,10 @@
 import org.elasticsearch.test.ESTestCase;
 import org.joda.time.DateTimeZone;
 
+import java.io.EOFException;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Base64;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
@@ -97,14 +101,7 @@ public void testIllegalBulkWrite() throws Exception {
         BytesStreamOutput out = new BytesStreamOutput();
 
         // bulk-write with wrong args
-        try {
-            out.writeBytes(new byte[]{}, 0, 1);
-            fail("expected IllegalArgumentException: length > (size-offset)");
-        }
-        catch (IllegalArgumentException iax1) {
-            // expected
-        }
-
+        expectThrows(IllegalArgumentException.class, () -> out.writeBytes(new byte[]{}, 0, 1));
         out.close();
     }
 
@@ -245,6 +242,9 @@ public void testSeek() throws Exception {
         assertEquals(position, out.position());
         assertEquals(position, BytesReference.toBytes(out.bytes()).length);
 
+        IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> out.seek(Integer.MAX_VALUE + 1L));
+        assertEquals("BytesStreamOutput cannot hold more than 2GB of data", iae.getMessage());
+
         out.close();
     }
 
@@ -258,6 +258,9 @@ public void testSkip() throws Exception {
         out.skip(forward);
         assertEquals(position + forward, out.position());
 
+        IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> out.skip(Integer.MAX_VALUE - 50));
+        assertEquals("BytesStreamOutput cannot hold more than 2GB of data", iae.getMessage());
+
         out.close();
     }
 
@@ -333,18 +336,21 @@ public void testSimpleStreams() throws Exception {
     }
 
     public void testNamedWriteable() throws IOException {
-        BytesStreamOutput out = new BytesStreamOutput();
-        NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(Collections.singletonList(
-            new NamedWriteableRegistry.Entry(BaseNamedWriteable.class, TestNamedWriteable.NAME, TestNamedWriteable::new)
-        ));
-        TestNamedWriteable namedWriteableIn = new TestNamedWriteable(randomAsciiOfLengthBetween(1, 10), randomAsciiOfLengthBetween(1, 10));
-        out.writeNamedWriteable(namedWriteableIn);
-        byte[] bytes = BytesReference.toBytes(out.bytes());
-        StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(bytes), namedWriteableRegistry);
-        assertEquals(in.available(), bytes.length);
-        BaseNamedWriteable namedWriteableOut = in.readNamedWriteable(BaseNamedWriteable.class);
-        assertEquals(namedWriteableIn, namedWriteableOut);
-        assertEquals(0, in.available());
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(Collections.singletonList(
+                    new NamedWriteableRegistry.Entry(BaseNamedWriteable.class, TestNamedWriteable.NAME, TestNamedWriteable::new)));
+            TestNamedWriteable namedWriteableIn = new TestNamedWriteable(randomAlphaOfLengthBetween(1, 10),
+                    randomAlphaOfLengthBetween(1, 10));
+            out.writeNamedWriteable(namedWriteableIn);
+            byte[] bytes = BytesReference.toBytes(out.bytes());
+
+            try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(bytes), namedWriteableRegistry)) {
+                assertEquals(in.available(), bytes.length);
+                BaseNamedWriteable namedWriteableOut = in.readNamedWriteable(BaseNamedWriteable.class);
+                assertEquals(namedWriteableIn, namedWriteableOut);
+                assertEquals(0, in.available());
+            }
+        }
     }
 
     public void testNamedWriteableList() throws IOException {
@@ -354,7 +360,7 @@ public void testNamedWriteableList() throws IOException {
         int size = between(0, 100);
         List<BaseNamedWriteable> expected = new ArrayList<>(size);
         for (int i = 0; i < size; i++) {
-            expected.add(new TestNamedWriteable(randomAsciiOfLengthBetween(1, 10), randomAsciiOfLengthBetween(1, 10)));
+            expected.add(new TestNamedWriteable(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10)));
         }
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
@@ -367,59 +373,61 @@ public void testNamedWriteableList() throws IOException {
     }
 
     public void testNamedWriteableNotSupportedWithoutWrapping() throws IOException {
-        BytesStreamOutput out = new BytesStreamOutput();
-        TestNamedWriteable testNamedWriteable = new TestNamedWriteable("test1", "test2");
-        out.writeNamedWriteable(testNamedWriteable);
-        StreamInput in = StreamInput.wrap(BytesReference.toBytes(out.bytes()));
-        try {
-            in.readNamedWriteable(BaseNamedWriteable.class);
-            fail("Expected UnsupportedOperationException");
-        } catch (UnsupportedOperationException e) {
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            TestNamedWriteable testNamedWriteable = new TestNamedWriteable("test1", "test2");
+            out.writeNamedWriteable(testNamedWriteable);
+            StreamInput in = StreamInput.wrap(BytesReference.toBytes(out.bytes()));
+            Exception e = expectThrows(UnsupportedOperationException.class, () -> in.readNamedWriteable(BaseNamedWriteable.class));
             assertThat(e.getMessage(), is("can't read named writeable from StreamInput"));
         }
     }
 
     public void testNamedWriteableReaderReturnsNull() throws IOException {
-        BytesStreamOutput out = new BytesStreamOutput();
-        NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(Collections.singletonList(
-            new NamedWriteableRegistry.Entry(BaseNamedWriteable.class, TestNamedWriteable.NAME, (StreamInput in) -> null)
-        ));
-        TestNamedWriteable namedWriteableIn = new TestNamedWriteable(randomAsciiOfLengthBetween(1, 10), randomAsciiOfLengthBetween(1, 10));
-        out.writeNamedWriteable(namedWriteableIn);
-        byte[] bytes = BytesReference.toBytes(out.bytes());
-        StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(bytes), namedWriteableRegistry);
-        assertEquals(in.available(), bytes.length);
-        IOException e = expectThrows(IOException.class, () -> in.readNamedWriteable(BaseNamedWriteable.class));
-        assertThat(e.getMessage(), endsWith("] returned null which is not allowed and probably means it screwed up the stream."));
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(Collections.singletonList(
+                    new NamedWriteableRegistry.Entry(BaseNamedWriteable.class, TestNamedWriteable.NAME, (StreamInput in) -> null)));
+            TestNamedWriteable namedWriteableIn = new TestNamedWriteable(randomAlphaOfLengthBetween(1, 10),
+                    randomAlphaOfLengthBetween(1, 10));
+            out.writeNamedWriteable(namedWriteableIn);
+            byte[] bytes = BytesReference.toBytes(out.bytes());
+            try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(bytes), namedWriteableRegistry)) {
+                assertEquals(in.available(), bytes.length);
+                IOException e = expectThrows(IOException.class, () -> in.readNamedWriteable(BaseNamedWriteable.class));
+                assertThat(e.getMessage(), endsWith("] returned null which is not allowed and probably means it screwed up the stream."));
+            }
+        }
     }
 
     public void testOptionalWriteableReaderReturnsNull() throws IOException {
-        BytesStreamOutput out = new BytesStreamOutput();
-        out.writeOptionalWriteable(new TestNamedWriteable(randomAsciiOfLengthBetween(1, 10), randomAsciiOfLengthBetween(1, 10)));
-        StreamInput in = StreamInput.wrap(BytesReference.toBytes(out.bytes()));
-        IOException e = expectThrows(IOException.class, () -> in.readOptionalWriteable((StreamInput ignored) -> null));
-        assertThat(e.getMessage(), endsWith("] returned null which is not allowed and probably means it screwed up the stream."));
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            out.writeOptionalWriteable(new TestNamedWriteable(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10)));
+            StreamInput in = StreamInput.wrap(BytesReference.toBytes(out.bytes()));
+            IOException e = expectThrows(IOException.class, () -> in.readOptionalWriteable((StreamInput ignored) -> null));
+            assertThat(e.getMessage(), endsWith("] returned null which is not allowed and probably means it screwed up the stream."));
+        }
     }
 
     public void testWriteableReaderReturnsWrongName() throws IOException {
-        BytesStreamOutput out = new BytesStreamOutput();
-        NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(Collections.singletonList(
-            new NamedWriteableRegistry.Entry(BaseNamedWriteable.class, TestNamedWriteable.NAME, (StreamInput in) ->
-                new TestNamedWriteable(in) {
-                    @Override
-                    public String getWriteableName() {
-                    return "intentionally-broken";
-                }
-                })
-        ));
-        TestNamedWriteable namedWriteableIn = new TestNamedWriteable(randomAsciiOfLengthBetween(1, 10), randomAsciiOfLengthBetween(1, 10));
-        out.writeNamedWriteable(namedWriteableIn);
-        byte[] bytes = BytesReference.toBytes(out.bytes());
-        StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(bytes), namedWriteableRegistry);
-        assertEquals(in.available(), bytes.length);
-        AssertionError e = expectThrows(AssertionError.class, () -> in.readNamedWriteable(BaseNamedWriteable.class));
-        assertThat(e.getMessage(),
-                endsWith(" claims to have a different name [intentionally-broken] than it was read from [test-named-writeable]."));
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(
+                    Collections.singletonList(new NamedWriteableRegistry.Entry(BaseNamedWriteable.class, TestNamedWriteable.NAME,
+                            (StreamInput in) -> new TestNamedWriteable(in) {
+                                @Override
+                                public String getWriteableName() {
+                                    return "intentionally-broken";
+                                }
+                            })));
+            TestNamedWriteable namedWriteableIn = new TestNamedWriteable(randomAlphaOfLengthBetween(1, 10),
+                    randomAlphaOfLengthBetween(1, 10));
+            out.writeNamedWriteable(namedWriteableIn);
+            byte[] bytes = BytesReference.toBytes(out.bytes());
+            try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(bytes), namedWriteableRegistry)) {
+                assertEquals(in.available(), bytes.length);
+                AssertionError e = expectThrows(AssertionError.class, () -> in.readNamedWriteable(BaseNamedWriteable.class));
+                assertThat(e.getMessage(),
+                        endsWith(" claims to have a different name [intentionally-broken] than it was read from [test-named-writeable]."));
+            }
+        }
     }
 
     public void testWriteStreamableList() throws IOException {
@@ -449,6 +457,22 @@ public void testWriteStreamableList() throws IOException {
         out.close();
     }
 
+    public void testWriteMap() throws IOException {
+        final int size = randomIntBetween(0, 100);
+        final Map<String, String> expected = new HashMap<>(randomIntBetween(0, 100));
+        for (int i = 0; i < size; ++i) {
+            expected.put(randomAlphaOfLength(2), randomAlphaOfLength(5));
+        }
+
+        final BytesStreamOutput out = new BytesStreamOutput();
+        out.writeMap(expected, StreamOutput::writeString, StreamOutput::writeString);
+        final StreamInput in = StreamInput.wrap(BytesReference.toBytes(out.bytes()));
+        final Map<String, String> loaded = in.readMap(StreamInput::readString, StreamInput::readString);
+
+        assertThat(loaded.size(), equalTo(expected.size()));
+        assertThat(expected, equalTo(loaded));
+    }
+
     public void testWriteMapOfLists() throws IOException {
         final int size = randomIntBetween(0, 5);
         final Map<String, List<String>> expected = new HashMap<>(size);
@@ -458,10 +482,10 @@ public void testWriteMapOfLists() throws IOException {
             List<String> list = new ArrayList<>(listSize);
 
             for (int j = 0; j < listSize; ++j) {
-                list.add(randomAsciiOfLength(5));
+                list.add(randomAlphaOfLength(5));
             }
 
-            expected.put(randomAsciiOfLength(2), list);
+            expected.put(randomAlphaOfLength(2), list);
         }
 
         final BytesStreamOutput out = new BytesStreamOutput();
@@ -507,7 +531,7 @@ private static class TestNamedWriteable extends BaseNamedWriteable {
             this.field2 = field2;
         }
 
-        public TestNamedWriteable(StreamInput in) throws IOException {
+        TestNamedWriteable(StreamInput in) throws IOException {
             field1 = in.readString();
             field2 = in.readString();
         }
@@ -551,32 +575,13 @@ public void testAccessAfterClose() throws Exception {
         assertEquals(-1, out.position());
 
         // writing a single byte must fail
-        try {
-            out.writeByte((byte)0);
-            fail("expected IllegalStateException: stream closed");
-        }
-        catch (IllegalStateException iex1) {
-            // expected
-        }
+        expectThrows(IllegalArgumentException.class, () -> out.writeByte((byte)0));
 
         // writing in bulk must fail
-        try {
-            out.writeBytes(new byte[0], 0, 0);
-            fail("expected IllegalStateException: stream closed");
-        }
-        catch (IllegalStateException iex1) {
-            // expected
-        }
+        expectThrows(IllegalArgumentException.class, () -> out.writeBytes(new byte[0], 0, 0));
 
         // toByteArray() must fail
-        try {
-            BytesReference.toBytes(out.bytes());
-            fail("expected IllegalStateException: stream closed");
-        }
-        catch (IllegalStateException iex1) {
-            // expected
-        }
-
+        expectThrows(IllegalArgumentException.class, () -> BytesReference.toBytes(out.bytes()));
     }
 
     // create & fill byte[] with randomized data
@@ -587,16 +592,15 @@ protected byte[] randomizedByteArrayWithSize(int size) {
     }
 
     public void testReadWriteGeoPoint() throws IOException {
-        {
-            BytesStreamOutput out = new BytesStreamOutput();
+        try (BytesStreamOutput out = new BytesStreamOutput()) {;
             GeoPoint geoPoint = new GeoPoint(randomDouble(), randomDouble());
             out.writeGenericValue(geoPoint);
             StreamInput wrap = out.bytes().streamInput();
             GeoPoint point = (GeoPoint) wrap.readGenericValue();
             assertEquals(point, geoPoint);
         }
-        {
-            BytesStreamOutput out = new BytesStreamOutput();
+
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
             GeoPoint geoPoint = new GeoPoint(randomDouble(), randomDouble());
             out.writeGeoPoint(geoPoint);
             StreamInput wrap = out.bytes().streamInput();
@@ -609,9 +613,9 @@ private static class TestStreamable implements Streamable {
 
         private boolean value;
 
-        public TestStreamable() { }
+        TestStreamable() { }
 
-        public TestStreamable(boolean value) {
+        TestStreamable(boolean value) {
             this.value = value;
         }
 
@@ -629,8 +633,8 @@ public void writeTo(StreamOutput out) throws IOException {
     public void testWriteMapWithConsistentOrder() throws IOException {
         Map<String, String> map =
             randomMap(new TreeMap<>(), randomIntBetween(2, 20),
-                () -> randomAsciiOfLength(5),
-                () -> randomAsciiOfLength(5));
+                () -> randomAlphaOfLength(5),
+                () -> randomAlphaOfLength(5));
 
         Map<String, Object> reverseMap = new TreeMap<>(Collections.reverseOrder());
         reverseMap.putAll(map);
@@ -640,35 +644,202 @@ public void testWriteMapWithConsistentOrder() throws IOException {
 
         assertNotEquals(mapKeys, reverseMapKeys);
 
-        BytesStreamOutput output = new BytesStreamOutput();
-        BytesStreamOutput reverseMapOutput = new BytesStreamOutput();
-        output.writeMapWithConsistentOrder(map);
-        reverseMapOutput.writeMapWithConsistentOrder(reverseMap);
+        try (BytesStreamOutput output = new BytesStreamOutput(); BytesStreamOutput reverseMapOutput = new BytesStreamOutput()) {
+            output.writeMapWithConsistentOrder(map);
+            reverseMapOutput.writeMapWithConsistentOrder(reverseMap);
 
-        assertEquals(output.bytes(), reverseMapOutput.bytes());
+            assertEquals(output.bytes(), reverseMapOutput.bytes());
+        }
     }
 
     public void testReadMapByUsingWriteMapWithConsistentOrder() throws IOException {
         Map<String, String> streamOutMap =
             randomMap(new HashMap<>(), randomIntBetween(2, 20),
-                () -> randomAsciiOfLength(5),
-                () -> randomAsciiOfLength(5));
-        BytesStreamOutput streamOut = new BytesStreamOutput();
-        streamOut.writeMapWithConsistentOrder(streamOutMap);
-        StreamInput in = StreamInput.wrap(BytesReference.toBytes(streamOut.bytes()));
-        Map<String, Object> streamInMap = in.readMap();
-        assertEquals(streamOutMap, streamInMap);
+                () -> randomAlphaOfLength(5),
+                () -> randomAlphaOfLength(5));
+        try (BytesStreamOutput streamOut = new BytesStreamOutput()) {
+            streamOut.writeMapWithConsistentOrder(streamOutMap);
+            StreamInput in = StreamInput.wrap(BytesReference.toBytes(streamOut.bytes()));
+            Map<String, Object> streamInMap = in.readMap();
+            assertEquals(streamOutMap, streamInMap);
+        }
     }
 
     public void testWriteMapWithConsistentOrderWithLinkedHashMapShouldThrowAssertError() throws IOException {
-        BytesStreamOutput output = new BytesStreamOutput();
-        Map<String, Object> map = new LinkedHashMap<>();
-        Throwable e = expectThrows(AssertionError.class, () -> output.writeMapWithConsistentOrder(map));
-        assertEquals(AssertionError.class, e.getClass());
+        try (BytesStreamOutput output = new BytesStreamOutput()) {
+            Map<String, Object> map = new LinkedHashMap<>();
+            Throwable e = expectThrows(AssertionError.class, () -> output.writeMapWithConsistentOrder(map));
+            assertEquals(AssertionError.class, e.getClass());
+        }
     }
 
     private static <K, V> Map<K, V> randomMap(Map<K, V> map, int size, Supplier<K> keyGenerator, Supplier<V> valueGenerator) {
         IntStream.range(0, size).forEach(i -> map.put(keyGenerator.get(), valueGenerator.get()));
         return map;
     }
+
+    public void testWriteRandomStrings() throws IOException {
+        final int iters = scaledRandomIntBetween(5, 20);
+        for (int iter = 0; iter < iters; iter++) {
+            List<String> strings = new ArrayList<>();
+            int numStrings = randomIntBetween(100, 1000);
+            BytesStreamOutput output = new BytesStreamOutput(0);
+            for (int i = 0; i < numStrings; i++) {
+                String s = randomRealisticUnicodeOfLengthBetween(0, 2048);
+                strings.add(s);
+                output.writeString(s);
+            }
+
+            try (StreamInput streamInput = output.bytes().streamInput()) {
+                for (int i = 0; i < numStrings; i++) {
+                    String s = streamInput.readString();
+                    assertEquals(strings.get(i), s);
+                }
+            }
+        }
+    }
+
+    /*
+     * tests the extreme case where characters use more than 2 bytes
+     */
+    public void testWriteLargeSurrogateOnlyString() throws IOException {
+        String deseretLetter = "\uD801\uDC00";
+        assertEquals(2, deseretLetter.length());
+        String largeString = IntStream.range(0, 2048).mapToObj(s -> deseretLetter).collect(Collectors.joining("")).trim();
+        assertEquals("expands to 4 bytes", 4, new BytesRef(deseretLetter).length);
+        try (BytesStreamOutput output = new BytesStreamOutput(0)) {
+            output.writeString(largeString);
+            try (StreamInput streamInput = output.bytes().streamInput()) {
+                assertEquals(largeString, streamInput.readString());
+            }
+        }
+    }
+
+    public void testReadTooLargeArraySize() throws IOException {
+        try (BytesStreamOutput output = new BytesStreamOutput(0)) {
+            output.writeVInt(10);
+            for (int i = 0; i < 10; i ++) {
+                output.writeInt(i);
+            }
+
+            output.writeVInt(Integer.MAX_VALUE);
+            for (int i = 0; i < 10; i ++) {
+                output.writeInt(i);
+            }
+            try (StreamInput streamInput = output.bytes().streamInput()) {
+                int[] ints = streamInput.readIntArray();
+                for (int i = 0; i < 10; i ++) {
+                    assertEquals(i, ints[i]);
+                }
+                expectThrows(IllegalStateException.class, () -> streamInput.readIntArray());
+            }
+        }
+    }
+
+    public void testReadCorruptedArraySize() throws IOException {
+        try (BytesStreamOutput output = new BytesStreamOutput(0)) {
+            output.writeVInt(10);
+            for (int i = 0; i < 10; i ++) {
+                output.writeInt(i);
+            }
+
+            output.writeVInt(100);
+            for (int i = 0; i < 10; i ++) {
+                output.writeInt(i);
+            }
+            try (StreamInput streamInput = output.bytes().streamInput()) {
+                int[] ints = streamInput.readIntArray();
+                for (int i = 0; i < 10; i ++) {
+                    assertEquals(i, ints[i]);
+                }
+                EOFException eofException = expectThrows(EOFException.class, () -> streamInput.readIntArray());
+                assertEquals("tried to read: 100 bytes but only 40 remaining", eofException.getMessage());
+            }
+        }
+    }
+
+    public void testReadNegativeArraySize() throws IOException {
+        try (BytesStreamOutput output = new BytesStreamOutput(0)) {
+            output.writeVInt(10);
+            for (int i = 0; i < 10; i ++) {
+                output.writeInt(i);
+            }
+
+            output.writeVInt(Integer.MIN_VALUE);
+            for (int i = 0; i < 10; i ++) {
+                output.writeInt(i);
+            }
+            try (StreamInput streamInput = output.bytes().streamInput()) {
+                int[] ints = streamInput.readIntArray();
+                for (int i = 0; i < 10; i ++) {
+                    assertEquals(i, ints[i]);
+                }
+                NegativeArraySizeException exception = expectThrows(NegativeArraySizeException.class, () -> streamInput.readIntArray());
+                assertEquals("array size must be positive but was: -2147483648", exception.getMessage());
+            }
+        }
+    }
+
+    public void testVInt() throws IOException {
+        final int value = randomInt();
+        BytesStreamOutput output = new BytesStreamOutput();
+        output.writeVInt(value);
+        StreamInput input = output.bytes().streamInput();
+        assertEquals(value, input.readVInt());
+    }
+
+    public void testVLong() throws IOException {
+        final long value = randomLong();
+        {
+            // Read works for positive and negative numbers
+            BytesStreamOutput output = new BytesStreamOutput();
+            output.writeVLongNoCheck(value); // Use NoCheck variant so we can write negative numbers
+            StreamInput input = output.bytes().streamInput();
+            assertEquals(value, input.readVLong());
+        }
+        if (value < 0) {
+            // Write doesn't work for negative numbers
+            BytesStreamOutput output = new BytesStreamOutput();
+            Exception e = expectThrows(IllegalStateException.class, () -> output.writeVLong(value));
+            assertEquals("Negative longs unsupported, use writeLong or writeZLong for negative numbers [" + value + "]", e.getMessage());
+        }
+
+        assertTrue("If we're not compatible with 5.1.1 we can drop the assertion below",
+                Version.CURRENT.minimumCompatibilityVersion().onOrBefore(Version.V_5_1_1));
+        /* Read -1 as serialized by a version of Elasticsearch that supported writing negative numbers with writeVLong. Note that this
+         * should be the same test as the first case (when value is negative) but we've kept some bytes so no matter what we do to
+         * writeVLong in the future we can be sure we can read bytes as written by Elasticsearch before 5.1.2 */
+        StreamInput in = new BytesArray(Base64.getDecoder().decode("////////////AQAAAAAAAA==")).streamInput();
+        assertEquals(-1, in.readVLong());
+    }
+
+    public enum TestEnum {
+        ONE,
+        TWO,
+        THREE
+    }
+
+    public void testEnum() throws IOException {
+        TestEnum value = randomFrom(TestEnum.values());
+        BytesStreamOutput output = new BytesStreamOutput();
+        output.writeEnum(value);
+        StreamInput input = output.bytes().streamInput();
+        assertEquals(value, input.readEnum(TestEnum.class));
+        assertEquals(0, input.available());
+    }
+
+    public void testInvalidEnum() throws IOException {
+        BytesStreamOutput output = new BytesStreamOutput();
+        int randomNumber = randomInt();
+        boolean validEnum = randomNumber >= 0 && randomNumber < TestEnum.values().length;
+        output.writeVInt(randomNumber);
+        StreamInput input = output.bytes().streamInput();
+        if (validEnum) {
+            assertEquals(TestEnum.values()[randomNumber], input.readEnum(TestEnum.class));
+        } else {
+            IOException ex = expectThrows(IOException.class, () -> input.readEnum(TestEnum.class));
+            assertEquals("Unknown TestEnum ordinal [" + randomNumber + "]", ex.getMessage());
+        }
+        assertEquals(0, input.available());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/common/io/stream/NamedWriteableRegistryTests.java b/core/src/test/java/org/elasticsearch/common/io/stream/NamedWriteableRegistryTests.java
index 8442e793a92b2..682615d6f1a17 100644
--- a/core/src/test/java/org/elasticsearch/common/io/stream/NamedWriteableRegistryTests.java
+++ b/core/src/test/java/org/elasticsearch/common/io/stream/NamedWriteableRegistryTests.java
@@ -28,7 +28,7 @@
 public class NamedWriteableRegistryTests extends ESTestCase {
 
     private static class DummyNamedWriteable implements NamedWriteable {
-        public DummyNamedWriteable(StreamInput in) {}
+        DummyNamedWriteable(StreamInput in) {}
         @Override
         public String getWriteableName() {
             return "test";
diff --git a/core/src/test/java/org/elasticsearch/common/io/stream/ReleasableBytesStreamOutputTests.java b/core/src/test/java/org/elasticsearch/common/io/stream/ReleasableBytesStreamOutputTests.java
new file mode 100644
index 0000000000000..557721a0241ab
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/common/io/stream/ReleasableBytesStreamOutputTests.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.io.stream;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.MockBigArrays;
+import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+public class ReleasableBytesStreamOutputTests extends ESTestCase {
+
+    public void testRelease() throws Exception {
+        MockBigArrays mockBigArrays =
+            new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService());
+        try (ReleasableBytesStreamOutput output =
+                 getRandomReleasableBytesStreamOutput(mockBigArrays)) {
+            output.writeBoolean(randomBoolean());
+        }
+        MockBigArrays.ensureAllArraysAreReleased();
+    }
+
+    private ReleasableBytesStreamOutput getRandomReleasableBytesStreamOutput(
+                                                MockBigArrays mockBigArrays) throws IOException {
+        ReleasableBytesStreamOutput output = new ReleasableBytesStreamOutput(mockBigArrays);
+        if (randomBoolean()) {
+            for (int i = 0; i < scaledRandomIntBetween(1, 32); i++) {
+                output.write(randomByte());
+            }
+        }
+        return output;
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java b/core/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java
index 06d39398c8e2b..9d885fe131c7a 100644
--- a/core/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java
+++ b/core/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.common.io.stream;
 
+import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.Tuple;
@@ -31,11 +32,80 @@
 import java.util.Collections;
 import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasToString;
 
 public class StreamTests extends ESTestCase {
+
+    public void testBooleanSerialization() throws IOException {
+        final BytesStreamOutput output = new BytesStreamOutput();
+        output.writeBoolean(false);
+        output.writeBoolean(true);
+
+        final BytesReference bytesReference = output.bytes();
+
+        final BytesRef bytesRef = bytesReference.toBytesRef();
+        assertThat(bytesRef.length, equalTo(2));
+        final byte[] bytes = bytesRef.bytes;
+        assertThat(bytes[0], equalTo((byte) 0));
+        assertThat(bytes[1], equalTo((byte) 1));
+
+        final StreamInput input = bytesReference.streamInput();
+        assertFalse(input.readBoolean());
+        assertTrue(input.readBoolean());
+
+        final Set<Byte> set = IntStream.range(Byte.MIN_VALUE, Byte.MAX_VALUE).mapToObj(v -> (byte) v).collect(Collectors.toSet());
+        set.remove((byte) 0);
+        set.remove((byte) 1);
+        final byte[] corruptBytes = new byte[] { randomFrom(set) };
+        final BytesReference corrupt = new BytesArray(corruptBytes);
+        final IllegalStateException e = expectThrows(IllegalStateException.class, () -> corrupt.streamInput().readBoolean());
+        final String message = String.format(Locale.ROOT, "unexpected byte [0x%02x]", corruptBytes[0]);
+        assertThat(e, hasToString(containsString(message)));
+    }
+
+    public void testOptionalBooleanSerialization() throws IOException {
+        final BytesStreamOutput output = new BytesStreamOutput();
+        output.writeOptionalBoolean(false);
+        output.writeOptionalBoolean(true);
+        output.writeOptionalBoolean(null);
+
+        final BytesReference bytesReference = output.bytes();
+
+        final BytesRef bytesRef = bytesReference.toBytesRef();
+        assertThat(bytesRef.length, equalTo(3));
+        final byte[] bytes = bytesRef.bytes;
+        assertThat(bytes[0], equalTo((byte) 0));
+        assertThat(bytes[1], equalTo((byte) 1));
+        assertThat(bytes[2], equalTo((byte) 2));
+
+        final StreamInput input = bytesReference.streamInput();
+        final Boolean maybeFalse = input.readOptionalBoolean();
+        assertNotNull(maybeFalse);
+        assertFalse(maybeFalse);
+        final Boolean maybeTrue = input.readOptionalBoolean();
+        assertNotNull(maybeTrue);
+        assertTrue(maybeTrue);
+        assertNull(input.readOptionalBoolean());
+
+        final Set<Byte> set = IntStream.range(Byte.MIN_VALUE, Byte.MAX_VALUE).mapToObj(v -> (byte) v).collect(Collectors.toSet());
+        set.remove((byte) 0);
+        set.remove((byte) 1);
+        set.remove((byte) 2);
+        final byte[] corruptBytes = new byte[] { randomFrom(set) };
+        final BytesReference corrupt = new BytesArray(corruptBytes);
+        final IllegalStateException e = expectThrows(IllegalStateException.class, () -> corrupt.streamInput().readOptionalBoolean());
+        final String message = String.format(Locale.ROOT, "unexpected byte [0x%02x]", corruptBytes[0]);
+        assertThat(e, hasToString(containsString(message)));
+    }
+
     public void testRandomVLongSerialization() throws IOException {
         for (int i = 0; i < 1024; i++) {
             long write = randomLong();
@@ -146,11 +216,11 @@ public void testWritableArrays() throws IOException {
     static final class WriteableString implements Writeable {
         final String string;
 
-        public WriteableString(String string) {
+        WriteableString(String string) {
             this.string = string;
         }
 
-        public WriteableString(StreamInput in) throws IOException {
+        WriteableString(StreamInput in) throws IOException {
             this(in.readString());
         }
 
@@ -179,4 +249,5 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeString(string);
         }
     }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/common/joda/DateMathParserTests.java b/core/src/test/java/org/elasticsearch/common/joda/DateMathParserTests.java
index b1d0abc96cf8b..ca9a6b3a1abf8 100644
--- a/core/src/test/java/org/elasticsearch/common/joda/DateMathParserTests.java
+++ b/core/src/test/java/org/elasticsearch/common/joda/DateMathParserTests.java
@@ -25,8 +25,8 @@
 import org.joda.time.DateTimeZone;
 
 import java.util.TimeZone;
-import java.util.concurrent.Callable;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.LongSupplier;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -36,26 +36,17 @@ public class DateMathParserTests extends ESTestCase {
     FormatDateTimeFormatter formatter = Joda.forPattern("dateOptionalTime||epoch_millis");
     DateMathParser parser = new DateMathParser(formatter);
 
-    private static Callable<Long> callable(final long value) {
-        return new Callable<Long>() {
-            @Override
-            public Long call() throws Exception {
-                return value;
-            }
-        };
-    }
-
     void assertDateMathEquals(String toTest, String expected) {
         assertDateMathEquals(toTest, expected, 0, false, null);
     }
 
     void assertDateMathEquals(String toTest, String expected, final long now, boolean roundUp, DateTimeZone timeZone) {
-        long gotMillis = parser.parse(toTest, callable(now), roundUp, timeZone);
+        long gotMillis = parser.parse(toTest, () -> now, roundUp, timeZone);
         assertDateEquals(gotMillis, toTest, expected);
     }
 
     void assertDateEquals(long gotMillis, String original, String expected) {
-        long expectedMillis = parser.parse(expected, callable(0));
+        long expectedMillis = parser.parse(expected, () -> 0);
         if (gotMillis != expectedMillis) {
             fail("Date math not equal\n" +
                 "Original              : " + original + "\n" +
@@ -77,16 +68,38 @@ public void testBasicDates() {
     }
 
     public void testRoundingDoesNotAffectExactDate() {
-        assertDateMathEquals("2014-11-12T22:55:00Z", "2014-11-12T22:55:00Z", 0, true, null);
-        assertDateMathEquals("2014-11-12T22:55:00Z", "2014-11-12T22:55:00Z", 0, false, null);
+        assertDateMathEquals("2014-11-12T22:55:00.000Z", "2014-11-12T22:55:00.000Z", 0, true, null);
+        assertDateMathEquals("2014-11-12T22:55:00.000Z", "2014-11-12T22:55:00.000Z", 0, false, null);
+
+        assertDateMathEquals("2014-11-12T22:55:00.000", "2014-11-12T21:55:00.000Z", 0, true, DateTimeZone.forID("+01:00"));
+        assertDateMathEquals("2014-11-12T22:55:00.000", "2014-11-12T21:55:00.000Z", 0, false, DateTimeZone.forID("+01:00"));
+
+        assertDateMathEquals("2014-11-12T22:55:00.000+01:00", "2014-11-12T21:55:00.000Z", 0, true, null);
+        assertDateMathEquals("2014-11-12T22:55:00.000+01:00", "2014-11-12T21:55:00.000Z", 0, false, null);
     }
 
     public void testTimezone() {
         // timezone works within date format
         assertDateMathEquals("2014-05-30T20:21+02:00", "2014-05-30T18:21:00.000");
 
+        // test alternative ways of writing zero offsets, according to ISO 8601 +00:00, +00, +0000 should work.
+        // joda also seems to allow for -00:00, -00, -0000
+        assertDateMathEquals("2014-05-30T18:21+00:00", "2014-05-30T18:21:00.000");
+        assertDateMathEquals("2014-05-30T18:21+00", "2014-05-30T18:21:00.000");
+        assertDateMathEquals("2014-05-30T18:21+0000", "2014-05-30T18:21:00.000");
+        assertDateMathEquals("2014-05-30T18:21-00:00", "2014-05-30T18:21:00.000");
+        assertDateMathEquals("2014-05-30T18:21-00", "2014-05-30T18:21:00.000");
+        assertDateMathEquals("2014-05-30T18:21-0000", "2014-05-30T18:21:00.000");
+
         // but also externally
         assertDateMathEquals("2014-05-30T20:21", "2014-05-30T18:21:00.000", 0, false, DateTimeZone.forID("+02:00"));
+        assertDateMathEquals("2014-05-30T18:21", "2014-05-30T18:21:00.000", 0, false, DateTimeZone.forID("+00:00"));
+        assertDateMathEquals("2014-05-30T18:21", "2014-05-30T18:21:00.000", 0, false, DateTimeZone.forID("+00:00"));
+        assertDateMathEquals("2014-05-30T18:21", "2014-05-30T18:21:00.000", 0, false, DateTimeZone.forID("+00"));
+        assertDateMathEquals("2014-05-30T18:21", "2014-05-30T18:21:00.000", 0, false, DateTimeZone.forID("+0000"));
+        assertDateMathEquals("2014-05-30T18:21", "2014-05-30T18:21:00.000", 0, false, DateTimeZone.forID("-00:00"));
+        assertDateMathEquals("2014-05-30T18:21", "2014-05-30T18:21:00.000", 0, false, DateTimeZone.forID("-00"));
+        assertDateMathEquals("2014-05-30T18:21", "2014-05-30T18:21:00.000", 0, false, DateTimeZone.forID("-0000"));
 
         // and timezone in the date has priority
         assertDateMathEquals("2014-05-30T20:21+03:00", "2014-05-30T17:21:00.000", 0, false, DateTimeZone.forID("-08:00"));
@@ -132,7 +145,7 @@ public void testMultipleAdjustments() {
 
 
     public void testNow() {
-        final long now = parser.parse("2014-11-18T14:27:32", callable(0), false, null);
+        final long now = parser.parse("2014-11-18T14:27:32", () -> 0, false, null);
 
         assertDateMathEquals("now", "2014-11-18T14:27:32", now, false, null);
         assertDateMathEquals("now+M", "2014-12-18T14:27:32", now, false, null);
@@ -143,7 +156,43 @@ public void testNow() {
         assertDateMathEquals("now/m", "2014-11-18T14:27", now, false, DateTimeZone.forID("+02:00"));
     }
 
-    public void testRounding() {
+    public void testRoundingPreservesEpochAsBaseDate() {
+        // If a user only specifies times, then the date needs to always be 1970-01-01 regardless of rounding
+        FormatDateTimeFormatter formatter = Joda.forPattern("HH:mm:ss");
+        DateMathParser parser = new DateMathParser(formatter);
+        assertEquals(
+                this.formatter.parser().parseMillis("1970-01-01T04:52:20.000Z"),
+                parser.parse("04:52:20", () -> 0, false, null));
+        assertEquals(
+                this.formatter.parser().parseMillis("1970-01-01T04:52:20.999Z"),
+                parser.parse("04:52:20", () -> 0, true, null));
+    }
+
+    // Implicit rounding happening when parts of the date are not specified
+    public void testImplicitRounding() {
+        assertDateMathEquals("2014-11-18", "2014-11-18", 0, false, null);
+        assertDateMathEquals("2014-11-18", "2014-11-18T23:59:59.999Z", 0, true, null);
+
+        assertDateMathEquals("2014-11-18T09:20", "2014-11-18T09:20", 0, false, null);
+        assertDateMathEquals("2014-11-18T09:20", "2014-11-18T09:20:59.999Z", 0, true, null);
+
+        assertDateMathEquals("2014-11-18", "2014-11-17T23:00:00.000Z", 0, false, DateTimeZone.forID("CET"));
+        assertDateMathEquals("2014-11-18", "2014-11-18T22:59:59.999Z", 0, true, DateTimeZone.forID("CET"));
+
+        assertDateMathEquals("2014-11-18T09:20", "2014-11-18T08:20:00.000Z", 0, false, DateTimeZone.forID("CET"));
+        assertDateMathEquals("2014-11-18T09:20", "2014-11-18T08:20:59.999Z", 0, true, DateTimeZone.forID("CET"));
+
+        // implicit rounding with explicit timezone in the date format
+        FormatDateTimeFormatter formatter = Joda.forPattern("YYYY-MM-ddZ");
+        DateMathParser parser = new DateMathParser(formatter);
+        long time = parser.parse("2011-10-09+01:00", () -> 0, false, null);
+        assertEquals(this.parser.parse("2011-10-09T00:00:00.000+01:00", () -> 0), time);
+        time = parser.parse("2011-10-09+01:00", () -> 0, true, null);
+        assertEquals(this.parser.parse("2011-10-09T23:59:59.999+01:00", () -> 0), time);
+    }
+
+    // Explicit rounding using the || separator
+    public void testExplicitRounding() {
         assertDateMathEquals("2014-11-18||/y", "2014-01-01", 0, false, null);
         assertDateMathEquals("2014-11-18||/y", "2014-12-31T23:59:59.999", 0, true, null);
         assertDateMathEquals("2014||/y", "2014-01-01", 0, false, null);
@@ -204,7 +253,7 @@ public void testTimestamps() {
 
         // also check other time units
         DateMathParser parser = new DateMathParser(Joda.forPattern("epoch_second||dateOptionalTime"));
-        long datetime = parser.parse("1418248078", callable(0));
+        long datetime = parser.parse("1418248078", () -> 0);
         assertDateEquals(datetime, "1418248078", "2014-12-10T21:47:58.000");
 
         // a timestamp before 10000 is a year
@@ -217,7 +266,7 @@ public void testTimestamps() {
 
     void assertParseException(String msg, String date, String exc) {
         try {
-            parser.parse(date, callable(0));
+            parser.parse(date, () -> 0);
             fail("Date: " + date + "\n" + msg);
         } catch (ElasticsearchParseException e) {
             assertThat(ExceptionsHelper.detailedMessage(e).contains(exc), equalTo(true));
@@ -239,12 +288,9 @@ public void testIllegalDateFormat() {
 
     public void testOnlyCallsNowIfNecessary() {
         final AtomicBoolean called = new AtomicBoolean();
-        final Callable<Long> now = new Callable<Long>() {
-            @Override
-            public Long call() throws Exception {
-                called.set(true);
-                return 42L;
-            }
+        final LongSupplier now = () -> {
+            called.set(true);
+            return 42L;
         };
         parser.parse("2014-11-18T14:27:32", now, false, null);
         assertFalse(called.get());
@@ -255,7 +301,7 @@ public Long call() throws Exception {
     public void testThatUnixTimestampMayNotHaveTimeZone() {
         DateMathParser parser = new DateMathParser(Joda.forPattern("epoch_millis"));
         try {
-            parser.parse("1234567890123", callable(42), false, DateTimeZone.forTimeZone(TimeZone.getTimeZone("CET")));
+            parser.parse("1234567890123", () -> 42, false, DateTimeZone.forTimeZone(TimeZone.getTimeZone("CET")));
             fail("Expected ElasticsearchParseException");
         } catch(ElasticsearchParseException e) {
             assertThat(e.getMessage(), containsString("failed to parse date field"));
diff --git a/core/src/test/java/org/elasticsearch/common/logging/DeprecationLoggerTests.java b/core/src/test/java/org/elasticsearch/common/logging/DeprecationLoggerTests.java
index f75e73ced2c8e..3f2274321a249 100644
--- a/core/src/test/java/org/elasticsearch/common/logging/DeprecationLoggerTests.java
+++ b/core/src/test/java/org/elasticsearch/common/logging/DeprecationLoggerTests.java
@@ -18,9 +18,11 @@
  */
 package org.elasticsearch.common.logging;
 
+import com.carrotsearch.randomizedtesting.generators.CodepointSetGenerator;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.hamcrest.RegexMatcher;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -28,57 +30,67 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.IntStream;
 
-import static org.hamcrest.Matchers.not;
+import static org.elasticsearch.common.logging.DeprecationLogger.WARNING_HEADER_PATTERN;
+import static org.elasticsearch.test.hamcrest.RegexMatcher.matches;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.not;
 
 /**
  * Tests {@link DeprecationLogger}
  */
 public class DeprecationLoggerTests extends ESTestCase {
 
+    private static final RegexMatcher warningValueMatcher = matches(WARNING_HEADER_PATTERN.pattern());
+
     private final DeprecationLogger logger = new DeprecationLogger(Loggers.getLogger(getClass()));
 
-    public void testAddsHeaderWithThreadContext() throws IOException {
-        String msg = "A simple message [{}]";
-        String param = randomAsciiOfLengthBetween(1, 5);
-        String formatted = LoggerMessageFormat.format(msg, (Object)param);
+    @Override
+    protected boolean enableWarningsCheck() {
+        //this is a low level test for the deprecation logger, setup and checks are done manually
+        return false;
+    }
 
+    public void testAddsHeaderWithThreadContext() throws IOException {
         try (ThreadContext threadContext = new ThreadContext(Settings.EMPTY)) {
-            Set<ThreadContext> threadContexts = Collections.singleton(threadContext);
+            final Set<ThreadContext> threadContexts = Collections.singleton(threadContext);
 
-            logger.deprecated(threadContexts, msg, param);
+            final String param = randomAlphaOfLengthBetween(1, 5);
+            logger.deprecated(threadContexts, "A simple message [{}]", param);
 
-            Map<String, List<String>> responseHeaders = threadContext.getResponseHeaders();
+            final Map<String, List<String>> responseHeaders = threadContext.getResponseHeaders();
 
-            assertEquals(1, responseHeaders.size());
-            assertEquals(formatted, responseHeaders.get(DeprecationLogger.DEPRECATION_HEADER).get(0));
+            assertThat(responseHeaders.size(), equalTo(1));
+            final List<String> responses = responseHeaders.get("Warning");
+            assertThat(responses, hasSize(1));
+            assertThat(responses.get(0), warningValueMatcher);
+            assertThat(responses.get(0), containsString("\"A simple message [" + param + "]\""));
         }
     }
 
     public void testAddsCombinedHeaderWithThreadContext() throws IOException {
-        String msg = "A simple message [{}]";
-        String param = randomAsciiOfLengthBetween(1, 5);
-        String formatted = LoggerMessageFormat.format(msg, (Object)param);
-        String formatted2 = randomAsciiOfLengthBetween(1, 10);
-
         try (ThreadContext threadContext = new ThreadContext(Settings.EMPTY)) {
-            Set<ThreadContext> threadContexts = Collections.singleton(threadContext);
+            final Set<ThreadContext> threadContexts = Collections.singleton(threadContext);
 
-            logger.deprecated(threadContexts, msg, param);
-            logger.deprecated(threadContexts, formatted2);
+            final String param = randomAlphaOfLengthBetween(1, 5);
+            logger.deprecated(threadContexts, "A simple message [{}]", param);
+            final String second = randomAlphaOfLengthBetween(1, 10);
+            logger.deprecated(threadContexts, second);
 
-            Map<String, List<String>> responseHeaders = threadContext.getResponseHeaders();
+            final Map<String, List<String>> responseHeaders = threadContext.getResponseHeaders();
 
             assertEquals(1, responseHeaders.size());
 
-            List<String> responses = responseHeaders.get(DeprecationLogger.DEPRECATION_HEADER);
+            final List<String> responses = responseHeaders.get("Warning");
 
             assertEquals(2, responses.size());
-            assertEquals(formatted, responses.get(0));
-            assertEquals(formatted2, responses.get(1));
+            assertThat(responses.get(0), warningValueMatcher);
+            assertThat(responses.get(0), containsString("\"A simple message [" + param + "]\""));
+            assertThat(responses.get(1), warningValueMatcher);
+            assertThat(responses.get(1), containsString("\"" + second + "\""));
         }
     }
 
@@ -87,28 +99,30 @@ public void testCanRemoveThreadContext() throws IOException {
         final String unexpected = "testCannotRemoveThreadContext";
 
         try (ThreadContext threadContext = new ThreadContext(Settings.EMPTY)) {
-            // NOTE: by adding it to the logger, we allow any concurrent test to write to it (from their own threads)
             DeprecationLogger.setThreadContext(threadContext);
-
             logger.deprecated(expected);
 
-            Map<String, List<String>> responseHeaders = threadContext.getResponseHeaders();
-            List<String> responses = responseHeaders.get(DeprecationLogger.DEPRECATION_HEADER);
+            {
+                final Map<String, List<String>> responseHeaders = threadContext.getResponseHeaders();
+                final List<String> responses = responseHeaders.get("Warning");
 
-            // ensure it works (note: concurrent tests may be adding to it, but in different threads, so it should have no impact)
-            assertThat(responses, hasSize(atLeast(1)));
-            assertThat(responses, hasItem(equalTo(expected)));
+                assertThat(responses, hasSize(1));
+                assertThat(responses.get(0), warningValueMatcher);
+                assertThat(responses.get(0), containsString(expected));
+            }
 
             DeprecationLogger.removeThreadContext(threadContext);
-
             logger.deprecated(unexpected);
 
-            responseHeaders = threadContext.getResponseHeaders();
-            responses = responseHeaders.get(DeprecationLogger.DEPRECATION_HEADER);
+            {
+                final Map<String, List<String>> responseHeaders = threadContext.getResponseHeaders();
+                final List<String> responses = responseHeaders.get("Warning");
 
-            assertThat(responses, hasSize(atLeast(1)));
-            assertThat(responses, hasItem(expected));
-            assertThat(responses, not(hasItem(unexpected)));
+                assertThat(responses, hasSize(1));
+                assertThat(responses.get(0), warningValueMatcher);
+                assertThat(responses.get(0), containsString(expected));
+                assertThat(responses.get(0), not(containsString(unexpected)));
+            }
         }
     }
 
@@ -152,4 +166,28 @@ public void testFailsWhenRemovingUnknownThreadContext() throws IOException {
         }
     }
 
+    public void testWarningValueFromWarningHeader() throws InterruptedException {
+        final String s = randomAlphaOfLength(16);
+        final String first = DeprecationLogger.formatWarning(s);
+        assertThat(DeprecationLogger.extractWarningValueFromWarningHeader(first), equalTo(s));
+    }
+
+    public void testEscape() {
+        assertThat(DeprecationLogger.escape("\\"), equalTo("\\\\"));
+        assertThat(DeprecationLogger.escape("\""), equalTo("\\\""));
+        assertThat(DeprecationLogger.escape("\\\""), equalTo("\\\\\\\""));
+        assertThat(DeprecationLogger.escape("\"foo\\bar\""),equalTo("\\\"foo\\\\bar\\\""));
+        // test that characters other than '\' and '"' are left unchanged
+        String chars = "\t !" + range(0x23, 0x5b) + range(0x5d, 0x73) + range(0x80, 0xff);
+        final String s = new CodepointSetGenerator(chars.toCharArray()).ofCodePointsLength(random(), 16, 16);
+        assertThat(DeprecationLogger.escape(s), equalTo(s));
+    }
+
+    private String range(int lowerInclusive, int upperInclusive) {
+        return IntStream
+                .range(lowerInclusive, upperInclusive + 1)
+                .collect(StringBuilder::new, StringBuilder::appendCodePoint, StringBuilder::append)
+                .toString();
+    }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java b/core/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java
index 33a0b855a7a66..4b4ea2856aaa5 100644
--- a/core/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java
+++ b/core/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java
@@ -378,7 +378,7 @@ public void testAsSequentialAccessBits() throws Exception {
             Weight termWeight = new TermQuery(new Term("foo", "bar")).createWeight(searcher, false);
             assertEquals(1, reader.leaves().size());
             LeafReaderContext leafReaderContext = searcher.getIndexReader().leaves().get(0);
-            Bits bits = Lucene.asSequentialAccessBits(leafReaderContext.reader().maxDoc(), termWeight.scorer(leafReaderContext));
+            Bits bits = Lucene.asSequentialAccessBits(leafReaderContext.reader().maxDoc(), termWeight.scorerSupplier(leafReaderContext));
 
             expectThrows(IndexOutOfBoundsException.class, () -> bits.get(-1));
             expectThrows(IndexOutOfBoundsException.class, () -> bits.get(leafReaderContext.reader().maxDoc()));
diff --git a/core/src/test/java/org/elasticsearch/common/lucene/index/FreqTermsEnumTests.java b/core/src/test/java/org/elasticsearch/common/lucene/index/FreqTermsEnumTests.java
index 3d1b0fdb84293..ea5bc64a4eaa0 100644
--- a/core/src/test/java/org/elasticsearch/common/lucene/index/FreqTermsEnumTests.java
+++ b/core/src/test/java/org/elasticsearch/common/lucene/index/FreqTermsEnumTests.java
@@ -31,7 +31,7 @@
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.NoMergePolicy;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.queries.TermsQuery;
+import org.apache.lucene.search.TermInSetQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.BytesRef;
@@ -90,7 +90,7 @@ public void setUp() throws Exception {
         iw = new IndexWriter(dir, conf);
         terms = new String[scaledRandomIntBetween(10, 300)];
         for (int i = 0; i < terms.length; i++) {
-            terms[i] = randomAsciiOfLength(5);
+            terms[i] = randomAlphaOfLength(5);
         }
 
         int numberOfDocs = scaledRandomIntBetween(30, 300);
@@ -138,19 +138,19 @@ public void setUp() throws Exception {
 
         // now go over each doc, build the relevant references and filter
         reader = DirectoryReader.open(iw);
-        List<Term> filterTerms = new ArrayList<>();
+        List<BytesRef> filterTerms = new ArrayList<>();
         for (int docId = 0; docId < reader.maxDoc(); docId++) {
             Document doc = reader.document(docId);
             addFreqs(doc, referenceAll);
             if (!deletedIds.contains(doc.getField("id").stringValue())) {
                 addFreqs(doc, referenceNotDeleted);
                 if (randomBoolean()) {
-                    filterTerms.add(new Term("id", doc.getField("id").stringValue()));
+                    filterTerms.add(new BytesRef(doc.getField("id").stringValue()));
                     addFreqs(doc, referenceFilter);
                 }
             }
         }
-        filter = new TermsQuery(filterTerms);
+        filter = new TermInSetQuery("id",filterTerms);
     }
 
     private void addFreqs(Document doc, Map<String, FreqHolder> reference) {
diff --git a/core/src/test/java/org/elasticsearch/common/lucene/search/MatchNoDocsQueryTests.java b/core/src/test/java/org/elasticsearch/common/lucene/search/MatchNoDocsQueryTests.java
deleted file mode 100644
index b328fd2ee9b9e..0000000000000
--- a/core/src/test/java/org/elasticsearch/common/lucene/search/MatchNoDocsQueryTests.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.common.lucene.search;
-
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.store.RAMDirectory;
-import org.elasticsearch.common.lucene.Lucene;
-import org.elasticsearch.test.ESTestCase;
-
-import static org.hamcrest.Matchers.equalTo;
-
-public class MatchNoDocsQueryTests extends ESTestCase {
-    public void testSimple() throws Exception {
-        MatchNoDocsQuery query = new MatchNoDocsQuery("field 'title' not found");
-        assertThat(query.toString(), equalTo("MatchNoDocsQuery[\"field 'title' not found\"]"));
-        Query rewrite = query.rewrite(null);
-        assertTrue(rewrite instanceof MatchNoDocsQuery);
-        assertThat(rewrite.toString(), equalTo("MatchNoDocsQuery[\"field 'title' not found\"]"));
-    }
-
-    public void testSearch() throws Exception {
-        IndexWriter writer = new IndexWriter(new RAMDirectory(), new IndexWriterConfig(Lucene.STANDARD_ANALYZER));
-        Document doc = new Document();
-        doc.add(new Field("field", "aaa bbb ccc ddd", TextField.TYPE_NOT_STORED));
-        writer.addDocument(doc);
-        IndexReader reader = DirectoryReader.open(writer);
-        IndexSearcher searcher = new IndexSearcher(reader);
-
-        Query query = new MatchNoDocsQuery("field not found");
-        assertThat(searcher.count(query), equalTo(0));
-
-        BooleanQuery.Builder bq = new BooleanQuery.Builder();
-        bq.add(new BooleanClause(new TermQuery(new Term("field", "aaa")), BooleanClause.Occur.SHOULD));
-        bq.add(new BooleanClause(new MatchNoDocsQuery("field not found"), BooleanClause.Occur.MUST));
-        query = bq.build();
-        assertThat(searcher.count(query), equalTo(0));
-        assertThat(query.toString(), equalTo("field:aaa +MatchNoDocsQuery[\"field not found\"]"));
-
-
-        bq = new BooleanQuery.Builder();
-        bq.add(new BooleanClause(new TermQuery(new Term("field", "aaa")), BooleanClause.Occur.SHOULD));
-        bq.add(new BooleanClause(new MatchNoDocsQuery("field not found"), BooleanClause.Occur.SHOULD));
-        query = bq.build();
-        assertThat(query.toString(), equalTo("field:aaa MatchNoDocsQuery[\"field not found\"]"));
-        assertThat(searcher.count(query), equalTo(1));
-        Query rewrite = query.rewrite(reader);
-        assertThat(rewrite.toString(), equalTo("field:aaa MatchNoDocsQuery[\"field not found\"]"));
-    }
-}
diff --git a/core/src/test/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunctionTests.java b/core/src/test/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunctionTests.java
index 21fbf43a133a8..d7ee7629c92d9 100644
--- a/core/src/test/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunctionTests.java
+++ b/core/src/test/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunctionTests.java
@@ -21,9 +21,8 @@
 
 import org.apache.lucene.index.LeafReaderContext;
 import org.elasticsearch.script.AbstractDoubleSearchScript;
-import org.elasticsearch.script.LeafSearchScript;
-import org.elasticsearch.script.Script;
 import org.elasticsearch.script.GeneralScriptException;
+import org.elasticsearch.script.LeafSearchScript;
 import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.test.ESTestCase;
 
@@ -35,7 +34,7 @@ public class ScriptScoreFunctionTests extends ESTestCase {
      */
     public void testScriptScoresReturnsNaN() throws IOException {
         // script that always returns NaN
-        ScoreFunction scoreFunction = new ScriptScoreFunction(new Script("Double.NaN"), new SearchScript() {
+        ScoreFunction scoreFunction = new ScriptScoreFunction(mockScript("Double.NaN"), new SearchScript() {
             @Override
             public LeafSearchScript getLeafSearchScript(LeafReaderContext context) throws IOException {
                 return new AbstractDoubleSearchScript() {
diff --git a/core/src/test/java/org/elasticsearch/common/lucene/uid/VersionLookupTests.java b/core/src/test/java/org/elasticsearch/common/lucene/uid/VersionLookupTests.java
index a67a728edccc8..c94de46605b35 100644
--- a/core/src/test/java/org/elasticsearch/common/lucene/uid/VersionLookupTests.java
+++ b/core/src/test/java/org/elasticsearch/common/lucene/uid/VersionLookupTests.java
@@ -31,78 +31,78 @@
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.FixedBitSet;
 import org.elasticsearch.common.lucene.Lucene;
-import org.elasticsearch.common.lucene.uid.Versions.DocIdAndVersion;
-import org.elasticsearch.index.mapper.UidFieldMapper;
+import org.elasticsearch.common.lucene.uid.VersionsResolver.DocIdAndVersion;
+import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.VersionFieldMapper;
 import org.elasticsearch.test.ESTestCase;
 
-/** 
- * test per-segment lookup of version-related datastructures
+/**
+ * test per-segment lookup of version-related data structures
  */
 public class VersionLookupTests extends ESTestCase {
 
-    /** 
+    /**
      * test version lookup actually works
      */
     public void testSimple() throws Exception {
         Directory dir = newDirectory();
         IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER));
         Document doc = new Document();
-        doc.add(new Field(UidFieldMapper.NAME, "6", UidFieldMapper.Defaults.FIELD_TYPE));
+        doc.add(new Field(IdFieldMapper.NAME, "6", IdFieldMapper.Defaults.FIELD_TYPE));
         doc.add(new NumericDocValuesField(VersionFieldMapper.NAME, 87));
         writer.addDocument(doc);
         DirectoryReader reader = DirectoryReader.open(writer);
         LeafReaderContext segment = reader.leaves().get(0);
-        PerThreadIDAndVersionLookup lookup = new PerThreadIDAndVersionLookup(segment.reader());
+        PerThreadIDAndVersionLookup lookup = new PerThreadIDAndVersionLookup(segment.reader(), IdFieldMapper.NAME);
         // found doc
-        DocIdAndVersion result = lookup.lookup(new BytesRef("6"), null, segment);
+        DocIdAndVersion result = lookup.lookupVersion(new BytesRef("6"), null, segment);
         assertNotNull(result);
         assertEquals(87, result.version);
         assertEquals(0, result.docId);
         // not found doc
-        assertNull(lookup.lookup(new BytesRef("7"), null, segment));
+        assertNull(lookup.lookupVersion(new BytesRef("7"), null, segment));
         // deleted doc
-        assertNull(lookup.lookup(new BytesRef("6"), new Bits.MatchNoBits(1), segment));
+        assertNull(lookup.lookupVersion(new BytesRef("6"), new Bits.MatchNoBits(1), segment));
         reader.close();
         writer.close();
         dir.close();
     }
-    
-    /** 
+
+    /**
      * test version lookup with two documents matching the ID
      */
     public void testTwoDocuments() throws Exception {
         Directory dir = newDirectory();
         IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER));
         Document doc = new Document();
-        doc.add(new Field(UidFieldMapper.NAME, "6", UidFieldMapper.Defaults.FIELD_TYPE));
+        doc.add(new Field(IdFieldMapper.NAME, "6", IdFieldMapper.Defaults.FIELD_TYPE));
         doc.add(new NumericDocValuesField(VersionFieldMapper.NAME, 87));
         writer.addDocument(doc);
         writer.addDocument(doc);
         DirectoryReader reader = DirectoryReader.open(writer);
         LeafReaderContext segment = reader.leaves().get(0);
-        PerThreadIDAndVersionLookup lookup = new PerThreadIDAndVersionLookup(segment.reader());
+        PerThreadIDAndVersionLookup lookup = new PerThreadIDAndVersionLookup(segment.reader(), IdFieldMapper.NAME);
         // return the last doc when there are duplicates
-        DocIdAndVersion result = lookup.lookup(new BytesRef("6"), null, segment);
+        DocIdAndVersion result = lookup.lookupVersion(new BytesRef("6"), null, segment);
         assertNotNull(result);
         assertEquals(87, result.version);
         assertEquals(1, result.docId);
         // delete the first doc only
         FixedBitSet live = new FixedBitSet(2);
         live.set(1);
-        result = lookup.lookup(new BytesRef("6"), live, segment);
+        result = lookup.lookupVersion(new BytesRef("6"), live, segment);
         assertNotNull(result);
         assertEquals(87, result.version);
         assertEquals(1, result.docId);
         // delete the second doc only
         live.clear(1);
         live.set(0);
-        result = lookup.lookup(new BytesRef("6"), live, segment);
+        result = lookup.lookupVersion(new BytesRef("6"), live, segment);
         assertNotNull(result);
         assertEquals(87, result.version);
         assertEquals(0, result.docId);
         // delete both docs
-        assertNull(lookup.lookup(new BytesRef("6"), new Bits.MatchNoBits(2), segment));
+        assertNull(lookup.lookupVersion(new BytesRef("6"), new Bits.MatchNoBits(2), segment));
         reader.close();
         writer.close();
         dir.close();
diff --git a/core/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java b/core/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java
index 45693e101d462..6b9960294e41d 100644
--- a/core/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java
+++ b/core/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java
@@ -38,6 +38,8 @@
 import java.util.ArrayList;
 import java.util.List;
 
+import static org.elasticsearch.common.lucene.uid.VersionsResolver.loadDocIdAndVersion;
+import static org.elasticsearch.common.lucene.uid.VersionsResolver.loadVersion;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.nullValue;
 
@@ -61,15 +63,15 @@ public void testVersions() throws Exception {
         Directory dir = newDirectory();
         IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER));
         DirectoryReader directoryReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "_na_", 1));
-        MatcherAssert.assertThat(Versions.loadVersion(directoryReader, new Term(UidFieldMapper.NAME, "1")), equalTo(Versions.NOT_FOUND));
+        MatcherAssert.assertThat(loadVersion(directoryReader, new Term(UidFieldMapper.NAME, "1")), equalTo(Versions.NOT_FOUND));
 
         Document doc = new Document();
         doc.add(new Field(UidFieldMapper.NAME, "1", UidFieldMapper.Defaults.FIELD_TYPE));
         doc.add(new NumericDocValuesField(VersionFieldMapper.NAME, 1));
         writer.updateDocument(new Term(UidFieldMapper.NAME, "1"), doc);
         directoryReader = reopen(directoryReader);
-        assertThat(Versions.loadVersion(directoryReader, new Term(UidFieldMapper.NAME, "1")), equalTo(1L));
-        assertThat(Versions.loadDocIdAndVersion(directoryReader, new Term(UidFieldMapper.NAME, "1")).version, equalTo(1L));
+        assertThat(loadVersion(directoryReader, new Term(UidFieldMapper.NAME, "1")), equalTo(1L));
+        assertThat(loadDocIdAndVersion(directoryReader, new Term(UidFieldMapper.NAME, "1")).version, equalTo(1L));
 
         doc = new Document();
         Field uid = new Field(UidFieldMapper.NAME, "1", UidFieldMapper.Defaults.FIELD_TYPE);
@@ -78,8 +80,8 @@ public void testVersions() throws Exception {
         doc.add(version);
         writer.updateDocument(new Term(UidFieldMapper.NAME, "1"), doc);
         directoryReader = reopen(directoryReader);
-        assertThat(Versions.loadVersion(directoryReader, new Term(UidFieldMapper.NAME, "1")), equalTo(2L));
-        assertThat(Versions.loadDocIdAndVersion(directoryReader, new Term(UidFieldMapper.NAME, "1")).version, equalTo(2L));
+        assertThat(loadVersion(directoryReader, new Term(UidFieldMapper.NAME, "1")), equalTo(2L));
+        assertThat(loadDocIdAndVersion(directoryReader, new Term(UidFieldMapper.NAME, "1")).version, equalTo(2L));
 
         // test reuse of uid field
         doc = new Document();
@@ -89,13 +91,13 @@ public void testVersions() throws Exception {
         writer.updateDocument(new Term(UidFieldMapper.NAME, "1"), doc);
 
         directoryReader = reopen(directoryReader);
-        assertThat(Versions.loadVersion(directoryReader, new Term(UidFieldMapper.NAME, "1")), equalTo(3L));
-        assertThat(Versions.loadDocIdAndVersion(directoryReader, new Term(UidFieldMapper.NAME, "1")).version, equalTo(3L));
+        assertThat(loadVersion(directoryReader, new Term(UidFieldMapper.NAME, "1")), equalTo(3L));
+        assertThat(loadDocIdAndVersion(directoryReader, new Term(UidFieldMapper.NAME, "1")).version, equalTo(3L));
 
         writer.deleteDocuments(new Term(UidFieldMapper.NAME, "1"));
         directoryReader = reopen(directoryReader);
-        assertThat(Versions.loadVersion(directoryReader, new Term(UidFieldMapper.NAME, "1")), equalTo(Versions.NOT_FOUND));
-        assertThat(Versions.loadDocIdAndVersion(directoryReader, new Term(UidFieldMapper.NAME, "1")), nullValue());
+        assertThat(loadVersion(directoryReader, new Term(UidFieldMapper.NAME, "1")), equalTo(Versions.NOT_FOUND));
+        assertThat(loadDocIdAndVersion(directoryReader, new Term(UidFieldMapper.NAME, "1")), nullValue());
         directoryReader.close();
         writer.close();
         dir.close();
@@ -121,21 +123,21 @@ public void testNestedDocuments() throws IOException {
 
         writer.updateDocuments(new Term(UidFieldMapper.NAME, "1"), docs);
         DirectoryReader directoryReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "_na_", 1));
-        assertThat(Versions.loadVersion(directoryReader, new Term(UidFieldMapper.NAME, "1")), equalTo(5L));
-        assertThat(Versions.loadDocIdAndVersion(directoryReader, new Term(UidFieldMapper.NAME, "1")).version, equalTo(5L));
+        assertThat(loadVersion(directoryReader, new Term(UidFieldMapper.NAME, "1")), equalTo(5L));
+        assertThat(loadDocIdAndVersion(directoryReader, new Term(UidFieldMapper.NAME, "1")).version, equalTo(5L));
 
         version.setLongValue(6L);
         writer.updateDocuments(new Term(UidFieldMapper.NAME, "1"), docs);
         version.setLongValue(7L);
         writer.updateDocuments(new Term(UidFieldMapper.NAME, "1"), docs);
         directoryReader = reopen(directoryReader);
-        assertThat(Versions.loadVersion(directoryReader, new Term(UidFieldMapper.NAME, "1")), equalTo(7L));
-        assertThat(Versions.loadDocIdAndVersion(directoryReader, new Term(UidFieldMapper.NAME, "1")).version, equalTo(7L));
+        assertThat(loadVersion(directoryReader, new Term(UidFieldMapper.NAME, "1")), equalTo(7L));
+        assertThat(loadDocIdAndVersion(directoryReader, new Term(UidFieldMapper.NAME, "1")).version, equalTo(7L));
 
         writer.deleteDocuments(new Term(UidFieldMapper.NAME, "1"));
         directoryReader = reopen(directoryReader);
-        assertThat(Versions.loadVersion(directoryReader, new Term(UidFieldMapper.NAME, "1")), equalTo(Versions.NOT_FOUND));
-        assertThat(Versions.loadDocIdAndVersion(directoryReader, new Term(UidFieldMapper.NAME, "1")), nullValue());
+        assertThat(loadVersion(directoryReader, new Term(UidFieldMapper.NAME, "1")), equalTo(Versions.NOT_FOUND));
+        assertThat(loadDocIdAndVersion(directoryReader, new Term(UidFieldMapper.NAME, "1")), nullValue());
         directoryReader.close();
         writer.close();
         dir.close();
@@ -143,7 +145,7 @@ public void testNestedDocuments() throws IOException {
 
     /** Test that version map cache works, is evicted on close, etc */
     public void testCache() throws Exception {
-        int size = Versions.lookupStates.size();
+        int size = VersionsResolver.lookupStates.size();
 
         Directory dir = newDirectory();
         IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER));
@@ -153,22 +155,22 @@ public void testCache() throws Exception {
         writer.addDocument(doc);
         DirectoryReader reader = DirectoryReader.open(writer);
         // should increase cache size by 1
-        assertEquals(87, Versions.loadVersion(reader, new Term(UidFieldMapper.NAME, "6")));
-        assertEquals(size+1, Versions.lookupStates.size());
+        assertEquals(87, loadVersion(reader, new Term(UidFieldMapper.NAME, "6")));
+        assertEquals(size+1, VersionsResolver.lookupStates.size());
         // should be cache hit
-        assertEquals(87, Versions.loadVersion(reader, new Term(UidFieldMapper.NAME, "6")));
-        assertEquals(size+1, Versions.lookupStates.size());
+        assertEquals(87, loadVersion(reader, new Term(UidFieldMapper.NAME, "6")));
+        assertEquals(size+1, VersionsResolver.lookupStates.size());
 
         reader.close();
         writer.close();
         // core should be evicted from the map
-        assertEquals(size, Versions.lookupStates.size());
+        assertEquals(size, VersionsResolver.lookupStates.size());
         dir.close();
     }
 
     /** Test that version map cache behaves properly with a filtered reader */
     public void testCacheFilterReader() throws Exception {
-        int size = Versions.lookupStates.size();
+        int size = VersionsResolver.lookupStates.size();
 
         Directory dir = newDirectory();
         IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER));
@@ -177,18 +179,18 @@ public void testCacheFilterReader() throws Exception {
         doc.add(new NumericDocValuesField(VersionFieldMapper.NAME, 87));
         writer.addDocument(doc);
         DirectoryReader reader = DirectoryReader.open(writer);
-        assertEquals(87, Versions.loadVersion(reader, new Term(UidFieldMapper.NAME, "6")));
-        assertEquals(size+1, Versions.lookupStates.size());
+        assertEquals(87, loadVersion(reader, new Term(UidFieldMapper.NAME, "6")));
+        assertEquals(size+1, VersionsResolver.lookupStates.size());
         // now wrap the reader
         DirectoryReader wrapped = ElasticsearchDirectoryReader.wrap(reader, new ShardId("bogus", "_na_", 5));
-        assertEquals(87, Versions.loadVersion(wrapped, new Term(UidFieldMapper.NAME, "6")));
+        assertEquals(87, loadVersion(wrapped, new Term(UidFieldMapper.NAME, "6")));
         // same size map: core cache key is shared
-        assertEquals(size+1, Versions.lookupStates.size());
+        assertEquals(size+1, VersionsResolver.lookupStates.size());
 
         reader.close();
         writer.close();
         // core should be evicted from the map
-        assertEquals(size, Versions.lookupStates.size());
+        assertEquals(size, VersionsResolver.lookupStates.size());
         dir.close();
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java b/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java
index 734068347b3e9..d6b0f08a777c3 100644
--- a/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java
@@ -23,38 +23,62 @@
 import org.elasticsearch.common.Table;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.inject.ModuleTestCase;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.BoundTransportAddress;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.http.HttpInfo;
-import org.elasticsearch.http.HttpServerAdapter;
 import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.http.HttpStats;
+import org.elasticsearch.http.NullDispatcher;
+import org.elasticsearch.indices.breaker.CircuitBreakerService;
+import org.elasticsearch.plugins.NetworkPlugin;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.cat.AbstractCatAction;
 import org.elasticsearch.test.transport.AssertingLocalTransport;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.Transport;
-import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.transport.TransportInterceptor;
+import org.elasticsearch.transport.TransportRequest;
+import org.elasticsearch.transport.TransportRequestHandler;
 
+import java.io.IOException;
+import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Supplier;
 
 public class NetworkModuleTests extends ModuleTestCase {
+    private ThreadPool threadPool;
 
-    static class FakeTransportService extends TransportService {
-        public FakeTransportService() {
-            super(null, null, null);
-        }
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        threadPool = new TestThreadPool(NetworkModuleTests.class.getName());
+    }
+
+    @Override
+    public void tearDown() throws Exception {
+        super.tearDown();
+        ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS);
     }
 
     static class FakeTransport extends AssertingLocalTransport {
-        public FakeTransport() {
+        FakeTransport() {
             super(null, null, null, null);
         }
     }
 
     static class FakeHttpTransport extends AbstractLifecycleComponent implements HttpServerTransport {
-        public FakeHttpTransport() {
+        FakeHttpTransport() {
             super(null);
         }
         @Override
@@ -75,24 +99,23 @@ public HttpInfo info() {
         public HttpStats stats() {
             return null;
         }
-        @Override
-        public void httpServerAdapter(HttpServerAdapter httpServerAdapter) {}
     }
 
+
     static class FakeRestHandler extends BaseRestHandler {
-        public FakeRestHandler() {
+        FakeRestHandler() {
             super(null);
         }
         @Override
-        public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception {}
+        public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { return channel -> {}; }
     }
 
     static class FakeCatRestHandler extends AbstractCatAction {
-        public FakeCatRestHandler() {
+        FakeCatRestHandler() {
             super(null);
         }
         @Override
-        protected void doRequest(RestRequest request, RestChannel channel, NodeClient client) {}
+        protected RestChannelConsumer doCatRequest(RestRequest request, NodeClient client) { return channel -> {}; }
         @Override
         protected void documentation(StringBuilder sb) {}
         @Override
@@ -101,64 +124,186 @@ protected Table getTableWithHeader(RestRequest request) {
         }
     }
 
-    public void testRegisterTransportService() {
-        Settings settings = Settings.builder().put(NetworkModule.TRANSPORT_SERVICE_TYPE_KEY, "custom")
-            .put(NetworkModule.HTTP_ENABLED.getKey(), false)
-            .put(NetworkModule.TRANSPORT_TYPE_KEY, "local")
-            .build();
-        NetworkModule module = new NetworkModule(new NetworkService(settings, Collections.emptyList()), settings, false);
-        module.registerTransportService("custom", FakeTransportService.class);
-        assertBinding(module, TransportService.class, FakeTransportService.class);
-        assertFalse(module.isTransportClient());
-
-        // check it works with transport only as well
-        module = new NetworkModule(new NetworkService(settings, Collections.emptyList()), settings, true);
-        module.registerTransportService("custom", FakeTransportService.class);
-        assertBinding(module, TransportService.class, FakeTransportService.class);
-        assertTrue(module.isTransportClient());
-    }
-
     public void testRegisterTransport() {
         Settings settings = Settings.builder().put(NetworkModule.TRANSPORT_TYPE_KEY, "custom")
             .put(NetworkModule.HTTP_ENABLED.getKey(), false)
             .build();
-        NetworkModule module = new NetworkModule(new NetworkService(settings, Collections.emptyList()), settings, false);
-        module.registerTransport("custom", FakeTransport.class);
-        assertBinding(module, Transport.class, FakeTransport.class);
+        Supplier<Transport> custom = FakeTransport::new;
+        NetworkPlugin plugin = new NetworkPlugin() {
+            @Override
+            public Map<String, Supplier<Transport>> getTransports(Settings settings, ThreadPool threadPool, BigArrays bigArrays,
+                                                                  CircuitBreakerService circuitBreakerService,
+                                                                  NamedWriteableRegistry namedWriteableRegistry,
+                                                                  NetworkService networkService) {
+                return Collections.singletonMap("custom", custom);
+            }
+        };
+        NetworkModule module = newNetworkModule(settings, false, plugin);
         assertFalse(module.isTransportClient());
+        assertFalse(module.isHttpEnabled());
+        assertSame(custom, module.getTransportSupplier());
 
         // check it works with transport only as well
-        module = new NetworkModule(new NetworkService(settings, Collections.emptyList()), settings, true);
-        module.registerTransport("custom", FakeTransport.class);
-        assertBinding(module, Transport.class, FakeTransport.class);
+        module = newNetworkModule(settings, true, plugin);
+        assertSame(custom, module.getTransportSupplier());
         assertTrue(module.isTransportClient());
+        assertFalse(module.isHttpEnabled());
     }
 
     public void testRegisterHttpTransport() {
         Settings settings = Settings.builder()
             .put(NetworkModule.HTTP_TYPE_SETTING.getKey(), "custom")
             .put(NetworkModule.TRANSPORT_TYPE_KEY, "local").build();
-        NetworkModule module = new NetworkModule(new NetworkService(settings, Collections.emptyList()), settings, false);
-        module.registerHttpTransport("custom", FakeHttpTransport.class);
-        assertBinding(module, HttpServerTransport.class, FakeHttpTransport.class);
-        assertFalse(module.isTransportClient());
+        Supplier<HttpServerTransport> custom = FakeHttpTransport::new;
 
-        // check registration not allowed for transport only
-        module = new NetworkModule(new NetworkService(settings, Collections.emptyList()), settings, true);
-        assertTrue(module.isTransportClient());
-        try {
-            module.registerHttpTransport("custom", FakeHttpTransport.class);
-            fail();
-        } catch (IllegalArgumentException e) {
-            assertTrue(e.getMessage().contains("Cannot register http transport"));
-            assertTrue(e.getMessage().contains("for transport client"));
-        }
+        NetworkModule module = newNetworkModule(settings, false, new NetworkPlugin() {
+            @Override
+            public Map<String, Supplier<HttpServerTransport>> getHttpTransports(Settings settings, ThreadPool threadPool,
+                                                                                BigArrays bigArrays,
+                                                                                CircuitBreakerService circuitBreakerService,
+                                                                                NamedWriteableRegistry namedWriteableRegistry,
+                                                                                NamedXContentRegistry xContentRegistry,
+                                                                                NetworkService networkService,
+                                                                                HttpServerTransport.Dispatcher requestDispatcher) {
+                return Collections.singletonMap("custom", custom);
+            }
+        });
+        assertSame(custom, module.getHttpServerTransportSupplier());
+        assertFalse(module.isTransportClient());
+        assertTrue(module.isHttpEnabled());
 
-        // not added if http is disabled
         settings = Settings.builder().put(NetworkModule.HTTP_ENABLED.getKey(), false)
             .put(NetworkModule.TRANSPORT_TYPE_KEY, "local").build();
-        module = new NetworkModule(new NetworkService(settings, Collections.emptyList()), settings, false);
-        assertNotBound(module, HttpServerTransport.class);
-        assertFalse(module.isTransportClient());
+        NetworkModule newModule = newNetworkModule(settings, false);
+        assertFalse(newModule.isTransportClient());
+        assertFalse(newModule.isHttpEnabled());
+        expectThrows(IllegalStateException.class, () -> newModule.getHttpServerTransportSupplier());
+    }
+
+    public void testOverrideDefault() {
+        Settings settings = Settings.builder()
+            .put(NetworkModule.HTTP_TYPE_SETTING.getKey(), "custom")
+            .put(NetworkModule.HTTP_DEFAULT_TYPE_SETTING.getKey(), "default_custom")
+            .put(NetworkModule.TRANSPORT_DEFAULT_TYPE_SETTING.getKey(), "local")
+            .put(NetworkModule.TRANSPORT_TYPE_KEY, "default_custom").build();
+        Supplier<Transport> customTransport = FakeTransport::new;
+        Supplier<HttpServerTransport> custom = FakeHttpTransport::new;
+        Supplier<HttpServerTransport> def = FakeHttpTransport::new;
+        NetworkModule module = newNetworkModule(settings, false, new NetworkPlugin() {
+            @Override
+            public Map<String, Supplier<Transport>> getTransports(Settings settings, ThreadPool threadPool, BigArrays bigArrays,
+                                                                  CircuitBreakerService circuitBreakerService,
+                                                                  NamedWriteableRegistry namedWriteableRegistry,
+                                                                  NetworkService networkService) {
+                return Collections.singletonMap("default_custom", customTransport);
+            }
+
+            @Override
+            public Map<String, Supplier<HttpServerTransport>> getHttpTransports(Settings settings, ThreadPool threadPool,
+                                                                                BigArrays bigArrays,
+                                                                                CircuitBreakerService circuitBreakerService,
+                                                                                NamedWriteableRegistry namedWriteableRegistry,
+                                                                                NamedXContentRegistry xContentRegistry,
+                                                                                NetworkService networkService,
+                                                                                HttpServerTransport.Dispatcher requestDispatcher) {
+                Map<String, Supplier<HttpServerTransport>> supplierMap = new HashMap<>();
+                supplierMap.put("custom", custom);
+                supplierMap.put("default_custom", def);
+                return supplierMap;
+            }
+        });
+        assertSame(custom, module.getHttpServerTransportSupplier());
+        assertSame(customTransport, module.getTransportSupplier());
+    }
+
+    public void testDefaultKeys() {
+        Settings settings = Settings.builder()
+            .put(NetworkModule.HTTP_DEFAULT_TYPE_SETTING.getKey(), "default_custom")
+            .put(NetworkModule.TRANSPORT_DEFAULT_TYPE_SETTING.getKey(), "default_custom").build();
+        Supplier<HttpServerTransport> custom = FakeHttpTransport::new;
+        Supplier<HttpServerTransport> def = FakeHttpTransport::new;
+        Supplier<Transport> customTransport = FakeTransport::new;
+        NetworkModule module = newNetworkModule(settings, false, new NetworkPlugin() {
+            @Override
+            public Map<String, Supplier<Transport>> getTransports(Settings settings, ThreadPool threadPool, BigArrays bigArrays,
+                                                                  CircuitBreakerService circuitBreakerService,
+                                                                  NamedWriteableRegistry namedWriteableRegistry,
+                                                                  NetworkService networkService) {
+                return Collections.singletonMap("default_custom", customTransport);
+            }
+
+            @Override
+            public Map<String, Supplier<HttpServerTransport>> getHttpTransports(Settings settings, ThreadPool threadPool,
+                                                                                BigArrays bigArrays,
+                                                                                CircuitBreakerService circuitBreakerService,
+                                                                                NamedWriteableRegistry namedWriteableRegistry,
+                                                                                NamedXContentRegistry xContentRegistry,
+                                                                                NetworkService networkService,
+                                                                                HttpServerTransport.Dispatcher requestDispatcher) {
+                Map<String, Supplier<HttpServerTransport>> supplierMap = new HashMap<>();
+                supplierMap.put("custom", custom);
+                supplierMap.put("default_custom", def);
+                return supplierMap;
+            }
+        });
+
+        assertSame(def, module.getHttpServerTransportSupplier());
+        assertSame(customTransport, module.getTransportSupplier());
+    }
+
+    public void testRegisterInterceptor() {
+        Settings settings = Settings.builder()
+            .put(NetworkModule.HTTP_ENABLED.getKey(), false)
+            .put(NetworkModule.TRANSPORT_TYPE_KEY, "local").build();
+        AtomicInteger called = new AtomicInteger(0);
+
+        TransportInterceptor interceptor = new TransportInterceptor() {
+            @Override
+            public <T extends TransportRequest> TransportRequestHandler<T> interceptHandler(String action, String executor,
+                                                                                            boolean forceExecution,
+                                                                                            TransportRequestHandler<T> actualHandler) {
+                called.incrementAndGet();
+                if ("foo/bar/boom".equals(action)) {
+                    assertTrue(forceExecution);
+                } else {
+                    assertFalse(forceExecution);
+                }
+                return actualHandler;
+            }
+        };
+        NetworkModule module = newNetworkModule(settings, false, new NetworkPlugin() {
+                @Override
+                public List<TransportInterceptor> getTransportInterceptors(ThreadContext threadContext) {
+                    assertNotNull(threadContext);
+                    return Collections.singletonList(interceptor);
+                }
+            });
+
+        TransportInterceptor transportInterceptor = module.getTransportInterceptor();
+        assertEquals(0, called.get());
+        transportInterceptor.interceptHandler("foo/bar/boom", null, true, null);
+        assertEquals(1, called.get());
+        transportInterceptor.interceptHandler("foo/baz/boom", null, false, null);
+        assertEquals(2, called.get());
+        assertTrue(transportInterceptor instanceof  NetworkModule.CompositeTransportInterceptor);
+        assertEquals(((NetworkModule.CompositeTransportInterceptor)transportInterceptor).transportInterceptors.size(), 1);
+        assertSame(((NetworkModule.CompositeTransportInterceptor)transportInterceptor).transportInterceptors.get(0), interceptor);
+
+        NullPointerException nullPointerException = expectThrows(NullPointerException.class, () -> {
+            newNetworkModule(settings, false, new NetworkPlugin() {
+                @Override
+                public List<TransportInterceptor> getTransportInterceptors(ThreadContext threadContext) {
+                    assertNotNull(threadContext);
+                    return Collections.singletonList(null);
+                }
+            });
+        });
+        assertEquals("interceptor must not be null", nullPointerException.getMessage());
+
+    }
+
+    private NetworkModule newNetworkModule(Settings settings, boolean transportClient, NetworkPlugin... plugins) {
+        return new NetworkModule(settings, transportClient, Arrays.asList(plugins), threadPool, null, null, null, xContentRegistry(), null,
+            new NullDispatcher());
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/common/network/NetworkUtilsTests.java b/core/src/test/java/org/elasticsearch/common/network/NetworkUtilsTests.java
index e5b95f258a3e2..85a7472416166 100644
--- a/core/src/test/java/org/elasticsearch/common/network/NetworkUtilsTests.java
+++ b/core/src/test/java/org/elasticsearch/common/network/NetworkUtilsTests.java
@@ -22,6 +22,10 @@
 import org.elasticsearch.test.ESTestCase;
 
 import java.net.InetAddress;
+import java.net.NetworkInterface;
+import java.util.Collections;
+
+import static org.hamcrest.Matchers.containsString;
 
 /**
  * Tests for network utils. Please avoid using any methods that cause DNS lookups!
@@ -74,4 +78,30 @@ public void testFilter() throws Exception {
         assertArrayEquals(new InetAddress[] { InetAddress.getByName("127.0.0.1") }, NetworkUtils.filterIPV4(addresses));
         assertArrayEquals(new InetAddress[] { InetAddress.getByName("::1") }, NetworkUtils.filterIPV6(addresses));
     }
+
+    /**
+     * Test that selecting by name is possible and properly matches the addresses on all interfaces and virtual
+     * interfaces.
+     *
+     * Note that to avoid that this test fails when interfaces are down or they do not have addresses assigned to them,
+     * they are ignored.
+     */
+    public void testAddressInterfaceLookup() throws Exception {
+        for (NetworkInterface netIf : NetworkUtils.getInterfaces()) {
+            if (!netIf.isUp() || Collections.list(netIf.getInetAddresses()).isEmpty()) {
+                continue;
+            }
+
+            String name = netIf.getName();
+            InetAddress[] expectedAddresses = Collections.list(netIf.getInetAddresses()).toArray(new InetAddress[0]);
+            InetAddress[] foundAddresses = NetworkUtils.getAddressesForInterface(name);
+            assertArrayEquals(expectedAddresses, foundAddresses);
+        }
+    }
+
+    public void testNonExistingInterface() throws Exception {
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
+                () -> NetworkUtils.getAddressesForInterface("non-existing"));
+        assertThat(exception.getMessage(), containsString("No interface named 'non-existing' found"));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/common/rounding/TimeZoneRoundingTests.java b/core/src/test/java/org/elasticsearch/common/rounding/TimeZoneRoundingTests.java
index ff83ddfa57d15..e9ea72753beba 100644
--- a/core/src/test/java/org/elasticsearch/common/rounding/TimeZoneRoundingTests.java
+++ b/core/src/test/java/org/elasticsearch/common/rounding/TimeZoneRoundingTests.java
@@ -30,6 +30,8 @@
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeConstants;
 import org.joda.time.DateTimeZone;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
 import org.joda.time.format.ISODateTimeFormat;
 
 import java.util.ArrayList;
@@ -41,6 +43,7 @@
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.lessThan;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
+import static org.hamcrest.Matchers.startsWith;
 
 /**
  */
@@ -513,6 +516,63 @@ public void testEdgeCasesTransition() {
         }
     }
 
+    /**
+     * tests for dst transition with overlaps and day roundings.
+     */
+    public void testDST_END_Edgecases() {
+        // First case, dst happens at 1am local time, switching back one hour.
+        // We want the overlapping hour to count for the next day, making it a 25h interval
+
+        DateTimeUnit timeUnit = DateTimeUnit.DAY_OF_MONTH;
+        DateTimeZone tz = DateTimeZone.forID("Atlantic/Azores");
+        Rounding rounding = new Rounding.TimeUnitRounding(timeUnit, tz);
+
+        // Sunday, 29 October 2000, 01:00:00 clocks were turned backward 1 hour
+        // to Sunday, 29 October 2000, 00:00:00 local standard time instead
+
+        long midnightBeforeTransition = time("2000-10-29T00:00:00", tz);
+        long nextMidnight = time("2000-10-30T00:00:00", tz);
+
+        assertInterval(midnightBeforeTransition, nextMidnight, rounding, 25 * 60, tz);
+
+        // Second case, dst happens at 0am local time, switching back one hour to 23pm local time.
+        // We want the overlapping hour to count for the previous day here
+
+        tz = DateTimeZone.forID("America/Lima");
+        rounding = new Rounding.TimeUnitRounding(timeUnit, tz);
+
+        // Sunday, 1 April 1990, 00:00:00 clocks were turned backward 1 hour to
+        // Saturday, 31 March 1990, 23:00:00 local standard time instead
+
+        midnightBeforeTransition = time("1990-03-31T00:00:00.000-04:00");
+        nextMidnight = time("1990-04-01T00:00:00.000-05:00");
+        assertInterval(midnightBeforeTransition, nextMidnight, rounding, 25 * 60, tz);
+
+        // make sure the next interval is 24h long again
+        long midnightAfterTransition = time("1990-04-01T00:00:00.000-05:00");
+        nextMidnight = time("1990-04-02T00:00:00.000-05:00");
+        assertInterval(midnightAfterTransition, nextMidnight, rounding, 24 * 60, tz);
+    }
+
+    /**
+     * Test that time zones are correctly parsed. There is a bug with
+     * Joda 2.9.4 (see https://github.com/JodaOrg/joda-time/issues/373)
+     */
+    public void testsTimeZoneParsing() {
+        final DateTime expected = new DateTime(2016, 11, 10, 5, 37, 59, randomDateTimeZone());
+
+        // Formatter used to print and parse the sample date.
+        // Printing the date works but parsing it back fails
+        // with Joda 2.9.4
+        DateTimeFormatter formatter = DateTimeFormat.forPattern("YYYY-MM-dd'T'HH:mm:ss " + randomFrom("ZZZ", "[ZZZ]", "'['ZZZ']'"));
+
+        String dateTimeAsString = formatter.print(expected);
+        assertThat(dateTimeAsString, startsWith("2016-11-10T05:37:59 "));
+
+        DateTime parsedDateTime = formatter.parseDateTime(dateTimeAsString);
+        assertThat(parsedDateTime.getZone(), equalTo(expected.getZone()));
+    }
+
     private static void assertInterval(long rounded, long nextRoundingValue, Rounding rounding, int minutes,
             DateTimeZone tz) {
         assertInterval(rounded, dateBetween(rounded, nextRoundingValue), nextRoundingValue, rounding, tz);
diff --git a/core/src/test/java/org/elasticsearch/common/settings/AddFileKeyStoreCommandTests.java b/core/src/test/java/org/elasticsearch/common/settings/AddFileKeyStoreCommandTests.java
new file mode 100644
index 0000000000000..9044103e43b2d
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/common/settings/AddFileKeyStoreCommandTests.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Map;
+
+import org.elasticsearch.cli.Command;
+import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cli.UserException;
+import org.elasticsearch.env.Environment;
+
+import static org.hamcrest.Matchers.containsString;
+
+public class AddFileKeyStoreCommandTests extends KeyStoreCommandTestCase {
+    @Override
+    protected Command newCommand() {
+        return new AddFileKeyStoreCommand() {
+            @Override
+            protected Environment createEnv(Terminal terminal, Map<String, String> settings) {
+                return env;
+            }
+        };
+    }
+
+    private Path createRandomFile() throws IOException {
+        int length = randomIntBetween(10, 20);
+        byte[] bytes = new byte[length];
+        for (int i = 0; i < length; ++i) {
+            bytes[i] = randomByte();
+        }
+        Path file = env.configFile().resolve("randomfile");
+        Files.write(file, bytes);
+        return file;
+    }
+
+    private void addFile(KeyStoreWrapper keystore, String setting, Path file) throws Exception {
+        keystore.setFile(setting, Files.readAllBytes(file));
+        keystore.save(env.configFile());
+    }
+
+    public void testMissing() throws Exception {
+        UserException e = expectThrows(UserException.class, this::execute);
+        assertEquals(ExitCodes.DATA_ERROR, e.exitCode);
+        assertThat(e.getMessage(), containsString("keystore not found"));
+    }
+
+    public void testOverwritePromptDefault() throws Exception {
+        Path file = createRandomFile();
+        KeyStoreWrapper keystore = createKeystore("");
+        addFile(keystore, "foo", file);
+        terminal.addTextInput("");
+        execute("foo", "path/dne");
+        assertSecureFile("foo", file);
+    }
+
+    public void testOverwritePromptExplicitNo() throws Exception {
+        Path file = createRandomFile();
+        KeyStoreWrapper keystore = createKeystore("");
+        addFile(keystore, "foo", file);
+        terminal.addTextInput("n"); // explicit no
+        execute("foo", "path/dne");
+        assertSecureFile("foo", file);
+    }
+
+    public void testOverwritePromptExplicitYes() throws Exception {
+        Path file1 = createRandomFile();
+        KeyStoreWrapper keystore = createKeystore("");
+        addFile(keystore, "foo", file1);
+        terminal.addTextInput("y");
+        Path file2 = createRandomFile();
+        execute("foo", file2.toString());
+        assertSecureFile("foo", file2);
+    }
+
+    public void testOverwriteForceShort() throws Exception {
+        Path file1 = createRandomFile();
+        KeyStoreWrapper keystore = createKeystore("");
+        addFile(keystore, "foo", file1);
+        Path file2 = createRandomFile();
+        execute("-f", "foo", file2.toString());
+        assertSecureFile("foo", file2);
+    }
+
+    public void testOverwriteForceLong() throws Exception {
+        Path file1 = createRandomFile();
+        KeyStoreWrapper keystore = createKeystore("");
+        addFile(keystore, "foo", file1);
+        Path file2 = createRandomFile();
+        execute("--force", "foo", file2.toString());
+        assertSecureFile("foo", file2);
+    }
+
+    public void testForceNonExistent() throws Exception {
+        createKeystore("");
+        Path file = createRandomFile();
+        execute("--force", "foo", file.toString());
+        assertSecureFile("foo", file);
+    }
+
+    public void testMissingSettingName() throws Exception {
+        createKeystore("");
+        UserException e = expectThrows(UserException.class, this::execute);
+        assertEquals(ExitCodes.USAGE, e.exitCode);
+        assertThat(e.getMessage(), containsString("Missing setting name"));
+    }
+
+    public void testMissingFileName() throws Exception {
+        createKeystore("");
+        UserException e = expectThrows(UserException.class, () -> execute("foo"));
+        assertEquals(ExitCodes.USAGE, e.exitCode);
+        assertThat(e.getMessage(), containsString("Missing file name"));
+    }
+
+    public void testFileDNE() throws Exception {
+        createKeystore("");
+        UserException e = expectThrows(UserException.class, () -> execute("foo", "path/dne"));
+        assertEquals(ExitCodes.IO_ERROR, e.exitCode);
+        assertThat(e.getMessage(), containsString("File [path/dne] does not exist"));
+    }
+
+    public void testExtraArguments() throws Exception {
+        createKeystore("");
+        Path file = createRandomFile();
+        UserException e = expectThrows(UserException.class, () -> execute("foo", file.toString(), "bar"));
+        assertEquals(e.getMessage(), ExitCodes.USAGE, e.exitCode);
+        assertThat(e.getMessage(), containsString("Unrecognized extra arguments [bar]"));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/common/settings/AddStringKeyStoreCommandTests.java b/core/src/test/java/org/elasticsearch/common/settings/AddStringKeyStoreCommandTests.java
new file mode 100644
index 0000000000000..11c3f107fe768
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/common/settings/AddStringKeyStoreCommandTests.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.Map;
+
+import org.elasticsearch.cli.Command;
+import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cli.UserException;
+import org.elasticsearch.env.Environment;
+
+import static org.hamcrest.Matchers.containsString;
+
+public class AddStringKeyStoreCommandTests extends KeyStoreCommandTestCase {
+    InputStream input;
+
+    @Override
+    protected Command newCommand() {
+        return new AddStringKeyStoreCommand() {
+            @Override
+            protected Environment createEnv(Terminal terminal, Map<String, String> settings) {
+                return env;
+            }
+            @Override
+            InputStream getStdin() {
+                return input;
+            }
+        };
+    }
+
+    public void testMissing() throws Exception {
+        UserException e = expectThrows(UserException.class, this::execute);
+        assertEquals(ExitCodes.DATA_ERROR, e.exitCode);
+        assertThat(e.getMessage(), containsString("keystore not found"));
+    }
+
+    public void testOverwritePromptDefault() throws Exception {
+        createKeystore("", "foo", "bar");
+        terminal.addTextInput("");
+        execute("foo");
+        assertSecureString("foo", "bar");
+    }
+
+    public void testOverwritePromptExplicitNo() throws Exception {
+        createKeystore("", "foo", "bar");
+        terminal.addTextInput("n"); // explicit no
+        execute("foo");
+        assertSecureString("foo", "bar");
+    }
+
+    public void testOverwritePromptExplicitYes() throws Exception {
+        createKeystore("", "foo", "bar");
+        terminal.addTextInput("y");
+        terminal.addSecretInput("newvalue");
+        execute("foo");
+        assertSecureString("foo", "newvalue");
+    }
+
+    public void testOverwriteForceShort() throws Exception {
+        createKeystore("", "foo", "bar");
+        terminal.addSecretInput("newvalue");
+        execute("-f", "foo"); // force
+        assertSecureString("foo", "newvalue");
+    }
+
+    public void testOverwriteForceLong() throws Exception {
+        createKeystore("", "foo", "bar");
+        terminal.addSecretInput("and yet another secret value");
+        execute("--force", "foo"); // force
+        assertSecureString("foo", "and yet another secret value");
+    }
+
+    public void testForceNonExistent() throws Exception {
+        createKeystore("");
+        terminal.addSecretInput("value");
+        execute("--force", "foo"); // force
+        assertSecureString("foo", "value");
+    }
+
+    public void testPromptForValue() throws Exception {
+        KeyStoreWrapper.create(new char[0]).save(env.configFile());
+        terminal.addSecretInput("secret value");
+        execute("foo");
+        assertSecureString("foo", "secret value");
+    }
+
+    public void testStdinShort() throws Exception {
+        KeyStoreWrapper.create(new char[0]).save(env.configFile());
+        setInput("secret value 1");
+        execute("-x", "foo");
+        assertSecureString("foo", "secret value 1");
+    }
+
+    public void testStdinLong() throws Exception {
+        KeyStoreWrapper.create(new char[0]).save(env.configFile());
+        setInput("secret value 2");
+        execute("--stdin", "foo");
+        assertSecureString("foo", "secret value 2");
+    }
+
+    public void testNonAsciiValue() throws Exception {
+        KeyStoreWrapper.create(new char[0]).save(env.configFile());
+        terminal.addSecretInput("non-äsčîï");
+        UserException e = expectThrows(UserException.class, () -> execute("foo"));
+        assertEquals(ExitCodes.DATA_ERROR, e.exitCode);
+        assertEquals("String value must contain only ASCII", e.getMessage());
+    }
+
+    public void testMissingSettingName() throws Exception {
+        createKeystore("");
+        terminal.addTextInput("");
+        UserException e = expectThrows(UserException.class, this::execute);
+        assertEquals(ExitCodes.USAGE, e.exitCode);
+        assertThat(e.getMessage(), containsString("The setting name can not be null"));
+    }
+
+    void setInput(String inputStr) {
+        input = new ByteArrayInputStream(inputStr.getBytes(StandardCharsets.UTF_8));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/common/settings/CreateKeyStoreCommandTests.java b/core/src/test/java/org/elasticsearch/common/settings/CreateKeyStoreCommandTests.java
new file mode 100644
index 0000000000000..5d4741c72912c
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/common/settings/CreateKeyStoreCommandTests.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Map;
+
+import org.elasticsearch.cli.Command;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.env.Environment;
+
+public class CreateKeyStoreCommandTests extends KeyStoreCommandTestCase {
+
+    @Override
+    protected Command newCommand() {
+        return new CreateKeyStoreCommand() {
+            @Override
+            protected Environment createEnv(Terminal terminal, Map<String, String> settings) {
+                return env;
+            }
+        };
+    }
+
+    public void testPosix() throws Exception {
+        execute();
+        Path configDir = env.configFile();
+        assertNotNull(KeyStoreWrapper.load(configDir));
+    }
+
+    public void testNotPosix() throws Exception {
+        env = setupEnv(false, fileSystems);
+        execute();
+        Path configDir = env.configFile();
+        assertNotNull(KeyStoreWrapper.load(configDir));
+    }
+
+    public void testOverwrite() throws Exception {
+        Path keystoreFile = KeyStoreWrapper.keystorePath(env.configFile());
+        byte[] content = "not a keystore".getBytes(StandardCharsets.UTF_8);
+        Files.write(keystoreFile, content);
+
+        terminal.addTextInput(""); // default is no
+        execute();
+        assertArrayEquals(content, Files.readAllBytes(keystoreFile));
+
+        terminal.addTextInput("n"); // explicit no
+        execute();
+        assertArrayEquals(content, Files.readAllBytes(keystoreFile));
+
+        terminal.addTextInput("y");
+        execute();
+        assertNotNull(KeyStoreWrapper.load(env.configFile()));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/common/settings/KeyStoreCommandTestCase.java b/core/src/test/java/org/elasticsearch/common/settings/KeyStoreCommandTestCase.java
new file mode 100644
index 0000000000000..500b7b627b840
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/common/settings/KeyStoreCommandTestCase.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+
+import com.google.common.jimfs.Configuration;
+import com.google.common.jimfs.Jimfs;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.LuceneTestCase;
+import org.elasticsearch.cli.CommandTestCase;
+import org.elasticsearch.common.io.PathUtilsForTesting;
+import org.elasticsearch.env.Environment;
+import org.junit.After;
+import org.junit.Before;
+
+/**
+ * Base test case for manipulating the ES keystore.
+ */
+@LuceneTestCase.SuppressFileSystems("*") // we do our own mocking
+public abstract class KeyStoreCommandTestCase extends CommandTestCase {
+
+    Environment env;
+
+    List<FileSystem> fileSystems = new ArrayList<>();
+
+    @After
+    public void closeMockFileSystems() throws IOException {
+        IOUtils.close(fileSystems);
+    }
+
+    @Before
+    public void setupEnv() throws IOException {
+        env = setupEnv(true, fileSystems); // default to posix, but tests may call setupEnv(false) to overwrite
+    }
+
+    static Environment setupEnv(boolean posix, List<FileSystem> fileSystems) throws IOException {
+        final Configuration configuration;
+        if (posix) {
+            configuration = Configuration.unix().toBuilder().setAttributeViews("basic", "owner", "posix", "unix").build();
+        } else {
+            configuration = Configuration.unix();
+        }
+        FileSystem fs = Jimfs.newFileSystem(configuration);
+        fileSystems.add(fs);
+        PathUtilsForTesting.installMock(fs); // restored by restoreFileSystem in ESTestCase
+        Path home = fs.getPath("/", "test-home");
+        Files.createDirectories(home.resolve("config"));
+        return new Environment(Settings.builder().put("path.home", home).build());
+    }
+
+    KeyStoreWrapper createKeystore(String password, String... settings) throws Exception {
+        KeyStoreWrapper keystore = KeyStoreWrapper.create(password.toCharArray());
+        assertEquals(0, settings.length % 2);
+        for (int i = 0; i < settings.length; i += 2) {
+            keystore.setString(settings[i], settings[i + 1].toCharArray());
+        }
+        keystore.save(env.configFile());
+        return keystore;
+    }
+
+    KeyStoreWrapper loadKeystore(String password) throws Exception {
+        KeyStoreWrapper keystore = KeyStoreWrapper.load(env.configFile());
+        keystore.decrypt(password.toCharArray());
+        return keystore;
+    }
+
+    void assertSecureString(String setting, String value) throws Exception {
+        assertSecureString(loadKeystore(""), setting, value);
+    }
+
+    void assertSecureString(KeyStoreWrapper keystore, String setting, String value) throws Exception {
+        assertEquals(value, keystore.getString(setting).toString());
+    }
+
+    void assertSecureFile(String setting, Path file) throws Exception {
+        assertSecureFile(loadKeystore(""), setting, file);
+    }
+
+    void assertSecureFile(KeyStoreWrapper keystore, String setting, Path file) throws Exception {
+        byte[] expectedBytes = Files.readAllBytes(file);
+        try (InputStream input = keystore.getFile(setting)) {
+            for (int i = 0; i < expectedBytes.length; ++i) {
+                int got = input.read();
+                int expected = Byte.toUnsignedInt(expectedBytes[i]);
+                if (got < 0) {
+                    fail("Got EOF from keystore stream at position " + i + " but expected 0x" + Integer.toHexString(expected));
+                }
+                assertEquals("Byte " + i, expected, got);
+            }
+            int eof = input.read();
+            if (eof != -1) {
+                fail("Found extra bytes in file stream from keystore, expected " + expectedBytes.length +
+                     " bytes but found 0x" + Integer.toHexString(eof));
+            }
+        }
+
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java b/core/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java
new file mode 100644
index 0000000000000..0b42eb59f827f
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.FileSystem;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.After;
+import org.junit.Before;
+
+public class KeyStoreWrapperTests extends ESTestCase {
+
+    Environment env;
+    List<FileSystem> fileSystems = new ArrayList<>();
+
+    @After
+    public void closeMockFileSystems() throws IOException {
+        IOUtils.close(fileSystems);
+    }
+
+    @Before
+    public void setupEnv() throws IOException {
+        env = KeyStoreCommandTestCase.setupEnv(true, fileSystems);
+    }
+
+    public void testFileSettingExhaustiveBytes() throws Exception {
+        KeyStoreWrapper keystore = KeyStoreWrapper.create(new char[0]);
+        byte[] bytes = new byte[256];
+        for (int i = 0; i < 256; ++i) {
+            bytes[i] = (byte)i;
+        }
+        keystore.setFile("foo", bytes);
+        keystore.save(env.configFile());
+        keystore = KeyStoreWrapper.load(env.configFile());
+        keystore.decrypt(new char[0]);
+        try (InputStream stream = keystore.getFile("foo")) {
+            for (int i = 0; i < 256; ++i) {
+                int got = stream.read();
+                if (got < 0) {
+                    fail("Expected 256 bytes but read " + i);
+                }
+                assertEquals(i, got);
+            }
+            assertEquals(-1, stream.read()); // nothing left
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/common/settings/ListKeyStoreCommandTests.java b/core/src/test/java/org/elasticsearch/common/settings/ListKeyStoreCommandTests.java
new file mode 100644
index 0000000000000..1a8bdfc077d8f
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/common/settings/ListKeyStoreCommandTests.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import java.util.Map;
+
+import org.elasticsearch.cli.Command;
+import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cli.UserException;
+import org.elasticsearch.env.Environment;
+
+import static org.hamcrest.Matchers.containsString;
+
+public class ListKeyStoreCommandTests extends KeyStoreCommandTestCase {
+
+    @Override
+    protected Command newCommand() {
+        return new ListKeyStoreCommand() {
+            @Override
+            protected Environment createEnv(Terminal terminal, Map<String, String> settings) {
+                return env;
+            }
+        };
+    }
+
+    public void testMissing() throws Exception {
+        UserException e = expectThrows(UserException.class, this::execute);
+        assertEquals(ExitCodes.DATA_ERROR, e.exitCode);
+        assertThat(e.getMessage(), containsString("keystore not found"));
+    }
+
+    public void testEmpty() throws Exception {
+        createKeystore("");
+        execute();
+        assertTrue(terminal.getOutput(), terminal.getOutput().isEmpty());
+    }
+
+    public void testOne() throws Exception {
+        createKeystore("", "foo", "bar");
+        execute();
+        assertEquals("foo\n", terminal.getOutput());
+    }
+
+    public void testMultiple() throws Exception {
+        createKeystore("", "foo", "1", "baz", "2", "bar", "3");
+        execute();
+        assertEquals("bar\nbaz\nfoo\n", terminal.getOutput()); // sorted
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/common/settings/MemorySizeSettingsTests.java b/core/src/test/java/org/elasticsearch/common/settings/MemorySizeSettingsTests.java
index f6d411c0df35a..24ce166d15844 100644
--- a/core/src/test/java/org/elasticsearch/common/settings/MemorySizeSettingsTests.java
+++ b/core/src/test/java/org/elasticsearch/common/settings/MemorySizeSettingsTests.java
@@ -38,33 +38,33 @@ public class MemorySizeSettingsTests extends ESTestCase {
 
     public void testPageCacheLimitHeapSetting() {
         assertMemorySizeSetting(PageCacheRecycler.LIMIT_HEAP_SETTING, "cache.recycler.page.limit.heap",
-                new ByteSizeValue((long) (JvmInfo.jvmInfo().getMem().getHeapMax().bytes() * 0.1)));
+                new ByteSizeValue((long) (JvmInfo.jvmInfo().getMem().getHeapMax().getBytes() * 0.1)));
     }
 
     public void testIndexBufferSizeSetting() {
         assertMemorySizeSetting(IndexingMemoryController.INDEX_BUFFER_SIZE_SETTING, "indices.memory.index_buffer_size",
-                new ByteSizeValue((long) (JvmInfo.jvmInfo().getMem().getHeapMax().bytes() * 0.1)));
+                new ByteSizeValue((long) (JvmInfo.jvmInfo().getMem().getHeapMax().getBytes() * 0.1)));
     }
 
     public void testQueryCacheSizeSetting() {
         assertMemorySizeSetting(IndicesQueryCache.INDICES_CACHE_QUERY_SIZE_SETTING, "indices.queries.cache.size",
-                new ByteSizeValue((long) (JvmInfo.jvmInfo().getMem().getHeapMax().bytes() * 0.1)));
+                new ByteSizeValue((long) (JvmInfo.jvmInfo().getMem().getHeapMax().getBytes() * 0.1)));
     }
 
     public void testIndicesRequestCacheSetting() {
         assertMemorySizeSetting(IndicesRequestCache.INDICES_CACHE_QUERY_SIZE, "indices.requests.cache.size",
-                new ByteSizeValue((long) (JvmInfo.jvmInfo().getMem().getHeapMax().bytes() * 0.01)));
+                new ByteSizeValue((long) (JvmInfo.jvmInfo().getMem().getHeapMax().getBytes() * 0.01)));
     }
 
     public void testCircuitBreakerSettings() {
         assertMemorySizeSetting(HierarchyCircuitBreakerService.TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING, "indices.breaker.total.limit",
-                new ByteSizeValue((long) (JvmInfo.jvmInfo().getMem().getHeapMax().bytes() * 0.7)));
+                new ByteSizeValue((long) (JvmInfo.jvmInfo().getMem().getHeapMax().getBytes() * 0.7)));
         assertMemorySizeSetting(HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_LIMIT_SETTING, "indices.breaker.fielddata.limit",
-                new ByteSizeValue((long) (JvmInfo.jvmInfo().getMem().getHeapMax().bytes() * 0.6)));
+                new ByteSizeValue((long) (JvmInfo.jvmInfo().getMem().getHeapMax().getBytes() * 0.6)));
         assertMemorySizeSetting(HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING, "indices.breaker.request.limit",
-                new ByteSizeValue((long) (JvmInfo.jvmInfo().getMem().getHeapMax().bytes() * 0.6)));
+                new ByteSizeValue((long) (JvmInfo.jvmInfo().getMem().getHeapMax().getBytes() * 0.6)));
         assertMemorySizeSetting(HierarchyCircuitBreakerService.IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_LIMIT_SETTING,
-                "network.breaker.inflight_requests.limit", new ByteSizeValue((JvmInfo.jvmInfo().getMem().getHeapMax().bytes())));
+                "network.breaker.inflight_requests.limit", new ByteSizeValue((JvmInfo.jvmInfo().getMem().getHeapMax().getBytes())));
     }
 
     public void testIndicesFieldDataCacheSetting() {
@@ -80,7 +80,7 @@ private void assertMemorySizeSetting(Setting<ByteSizeValue> setting, String sett
                 equalTo(defaultValue));
         Settings settingWithPercentage = Settings.builder().put(settingKey, "25%").build();
         assertThat(setting.get(settingWithPercentage),
-                equalTo(new ByteSizeValue((long) (JvmInfo.jvmInfo().getMem().getHeapMax().bytes() * 0.25))));
+                equalTo(new ByteSizeValue((long) (JvmInfo.jvmInfo().getMem().getHeapMax().getBytes() * 0.25))));
         Settings settingWithBytesValue = Settings.builder().put(settingKey, "1024b").build();
         assertThat(setting.get(settingWithBytesValue), equalTo(new ByteSizeValue(1024)));
     }
diff --git a/core/src/test/java/org/elasticsearch/common/settings/RemoveSettingKeyStoreCommandTests.java b/core/src/test/java/org/elasticsearch/common/settings/RemoveSettingKeyStoreCommandTests.java
new file mode 100644
index 0000000000000..b74382d8cf563
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/common/settings/RemoveSettingKeyStoreCommandTests.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import javax.crypto.SecretKeyFactory;
+import java.security.Provider;
+import java.security.Security;
+import java.util.Map;
+import java.util.Set;
+
+import org.elasticsearch.cli.Command;
+import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cli.UserException;
+import org.elasticsearch.env.Environment;
+
+import static org.hamcrest.Matchers.containsString;
+
+public class RemoveSettingKeyStoreCommandTests extends KeyStoreCommandTestCase {
+
+    @Override
+    protected Command newCommand() {
+        return new RemoveSettingKeyStoreCommand() {
+            @Override
+            protected Environment createEnv(Terminal terminal, Map<String, String> settings) {
+                return env;
+            }
+        };
+    }
+
+    public void testMissing() throws Exception {
+        UserException e = expectThrows(UserException.class, () -> execute("foo"));
+        assertEquals(ExitCodes.DATA_ERROR, e.exitCode);
+        assertThat(e.getMessage(), containsString("keystore not found"));
+    }
+
+    public void testNoSettings() throws Exception {
+        createKeystore("");
+        UserException e = expectThrows(UserException.class, this::execute);
+        assertEquals(ExitCodes.USAGE, e.exitCode);
+        assertThat(e.getMessage(), containsString("Must supply at least one setting"));
+    }
+
+    public void testNonExistentSetting() throws Exception {
+        createKeystore("");
+        UserException e = expectThrows(UserException.class, () -> execute("foo"));
+        assertEquals(ExitCodes.CONFIG, e.exitCode);
+        assertThat(e.getMessage(), containsString("[foo] does not exist"));
+    }
+
+    public void testOne() throws Exception {
+        createKeystore("", "foo", "bar");
+        execute("foo");
+        assertFalse(loadKeystore("").getSettingNames().contains("foo"));
+    }
+
+    public void testMany() throws Exception {
+        createKeystore("", "foo", "1", "bar", "2", "baz", "3");
+        execute("foo", "baz");
+        Set<String> settings = loadKeystore("").getSettingNames();
+        assertFalse(settings.contains("foo"));
+        assertFalse(settings.contains("baz"));
+        assertTrue(settings.contains("bar"));
+        assertEquals(1, settings.size());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java b/core/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java
index bd6cb530b2bc8..dd96acdd6c75d 100644
--- a/core/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java
+++ b/core/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java
@@ -33,16 +33,100 @@
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedHashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BiConsumer;
 import java.util.function.Function;
 
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.startsWith;
+
 public class ScopedSettingsTests extends ESTestCase {
 
+    public void testResetSetting() {
+        Setting<Integer> dynamicSetting = Setting.intSetting("some.dyn.setting", 1, Property.Dynamic, Property.NodeScope);
+        Setting<Integer> staticSetting = Setting.intSetting("some.static.setting", 1, Property.NodeScope);
+        Settings currentSettings = Settings.builder().put("some.dyn.setting", 5).put("some.static.setting", 6).put("archived.foo.bar", 9)
+            .build();
+        ClusterSettings service = new ClusterSettings(currentSettings
+            , new HashSet<>(Arrays.asList(dynamicSetting, staticSetting)));
+
+        expectThrows(IllegalArgumentException.class, () ->
+        service.updateDynamicSettings(Settings.builder().put("some.dyn.setting", 8).putNull("some.static.setting").build(),
+            Settings.builder().put(currentSettings), Settings.builder(), "node"));
+
+        Settings.Builder target = Settings.builder().put(currentSettings);
+        Settings.Builder update = Settings.builder();
+        assertTrue(service.updateDynamicSettings(Settings.builder().put("some.dyn.setting", 8).build(),
+            target, update, "node"));
+        assertEquals(8, dynamicSetting.get(target.build()).intValue());
+        assertEquals(6, staticSetting.get(target.build()).intValue());
+        assertEquals(9, target.build().getAsInt("archived.foo.bar", null).intValue());
+
+        target = Settings.builder().put(currentSettings);
+        update = Settings.builder();
+        assertTrue(service.updateDynamicSettings(Settings.builder().putNull("some.dyn.setting").build(),
+            target, update, "node"));
+        assertEquals(1, dynamicSetting.get(target.build()).intValue());
+        assertEquals(6, staticSetting.get(target.build()).intValue());
+        assertEquals(9, target.build().getAsInt("archived.foo.bar", null).intValue());
+
+        target = Settings.builder().put(currentSettings);
+        update = Settings.builder();
+        assertTrue(service.updateDynamicSettings(Settings.builder().putNull("archived.foo.bar").build(),
+            target, update, "node"));
+        assertEquals(5, dynamicSetting.get(target.build()).intValue());
+        assertEquals(6, staticSetting.get(target.build()).intValue());
+        assertNull(target.build().getAsInt("archived.foo.bar", null));
+
+        target = Settings.builder().put(currentSettings);
+        update = Settings.builder();
+        assertTrue(service.updateDynamicSettings(Settings.builder().putNull("some.*").build(),
+            target, update, "node"));
+        assertEquals(1, dynamicSetting.get(target.build()).intValue());
+        assertEquals(6, staticSetting.get(target.build()).intValue());
+        assertEquals(9, target.build().getAsInt("archived.foo.bar", null).intValue());
+
+        target = Settings.builder().put(currentSettings);
+        update = Settings.builder();
+        assertTrue(service.updateDynamicSettings(Settings.builder().putNull("*").build(),
+            target, update, "node"));
+        assertEquals(1, dynamicSetting.get(target.build()).intValue());
+        assertEquals(6, staticSetting.get(target.build()).intValue());
+        assertNull(target.build().getAsInt("archived.foo.bar", null));
+    }
+
+    public void testResetSettingWithIPValidator() {
+        Settings currentSettings = Settings.builder().put("index.routing.allocation.require._ip", "192.168.0.1,127.0.0.1")
+            .put("index.some.dyn.setting", 1)
+            .build();
+        Setting<Integer> dynamicSetting = Setting.intSetting("index.some.dyn.setting", 1, Property.Dynamic, Property.IndexScope);
+
+        IndexScopedSettings settings = new IndexScopedSettings(currentSettings,
+            new HashSet<>(Arrays.asList(dynamicSetting, IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING)));
+        Settings s = IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.get(currentSettings);
+        assertEquals(1, s.size());
+        assertEquals("192.168.0.1,127.0.0.1", s.get("_ip"));
+        Settings.Builder builder = Settings.builder();
+        Settings updates = Settings.builder().putNull("index.routing.allocation.require._ip")
+            .put("index.some.dyn.setting", 1).build();
+        settings.validate(updates);
+        settings.updateDynamicSettings(updates,
+            Settings.builder().put(currentSettings), builder, "node");
+        currentSettings = builder.build();
+        s = IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.get(currentSettings);
+        assertEquals(0, s.size());
+        assertEquals(1, dynamicSetting.get(currentSettings).intValue());
+        assertEquals(1, currentSettings.size());
+    }
+
     public void testAddConsumer() {
         Setting<Integer> testSetting = Setting.intSetting("foo.bar", 1, Property.Dynamic, Property.NodeScope);
         Setting<Integer> testSetting2 = Setting.intSetting("foo.bar.baz", 1, Property.Dynamic, Property.NodeScope);
@@ -71,6 +155,52 @@ public void testAddConsumer() {
         assertEquals(0, consumer2.get());
     }
 
+    public void testAddConsumerAffix() {
+        Setting.AffixSetting<Integer> intSetting = Setting.affixKeySetting("foo.", "bar",
+            (k) ->  Setting.intSetting(k, 1, Property.Dynamic, Property.NodeScope));
+        Setting.AffixSetting<List<Integer>> listSetting = Setting.affixKeySetting("foo.", "list",
+            (k) -> Setting.listSetting(k, Arrays.asList("1"), Integer::parseInt, Property.Dynamic, Property.NodeScope));
+        AbstractScopedSettings service = new ClusterSettings(Settings.EMPTY,new HashSet<>(Arrays.asList(intSetting, listSetting)));
+        Map<String, List<Integer>> listResults = new HashMap<>();
+        Map<String, Integer> intResults = new HashMap<>();
+
+        BiConsumer<String, Integer> intConsumer = intResults::put;
+        BiConsumer<String, List<Integer>> listConsumer = listResults::put;
+
+        service.addAffixUpdateConsumer(listSetting, listConsumer, (s, k) -> {});
+        service.addAffixUpdateConsumer(intSetting, intConsumer, (s, k) -> {});
+        assertEquals(0, listResults.size());
+        assertEquals(0, intResults.size());
+        service.applySettings(Settings.builder()
+            .put("foo.test.bar", 2)
+            .put("foo.test_1.bar", 7)
+            .putArray("foo.test_list.list", "16", "17")
+            .putArray("foo.test_list_1.list", "18", "19", "20")
+            .build());
+        assertEquals(2, intResults.get("test").intValue());
+        assertEquals(7, intResults.get("test_1").intValue());
+        assertEquals(Arrays.asList(16, 17), listResults.get("test_list"));
+        assertEquals(Arrays.asList(18, 19, 20), listResults.get("test_list_1"));
+        assertEquals(2, listResults.size());
+        assertEquals(2, intResults.size());
+
+        listResults.clear();
+        intResults.clear();
+
+        service.applySettings(Settings.builder()
+            .put("foo.test.bar", 2)
+            .put("foo.test_1.bar", 8)
+            .putArray("foo.test_list.list", "16", "17")
+            .putNull("foo.test_list_1.list")
+            .build());
+        assertNull("test wasn't changed", intResults.get("test"));
+        assertEquals(8, intResults.get("test_1").intValue());
+        assertNull("test_list wasn't changed", listResults.get("test_list"));
+        assertEquals(Arrays.asList(1), listResults.get("test_list_1")); // reset to default
+        assertEquals(1, listResults.size());
+        assertEquals(1, intResults.size());
+    }
+
     public void testApply() {
         Setting<Integer> testSetting = Setting.intSetting("foo.bar", 1, Property.Dynamic, Property.NodeScope);
         Setting<Integer> testSetting2 = Setting.intSetting("foo.bar.baz", 1, Property.Dynamic, Property.NodeScope);
@@ -146,26 +276,135 @@ public void testIsDynamic(){
             new ClusterSettings(Settings.EMPTY,
                 new HashSet<>(Arrays.asList(Setting.intSetting("foo.bar", 1, Property.Dynamic, Property.NodeScope),
                     Setting.intSetting("foo.bar.baz", 1, Property.NodeScope))));
-        assertFalse(settings.hasDynamicSetting("foo.bar.baz"));
-        assertTrue(settings.hasDynamicSetting("foo.bar"));
+        assertFalse(settings.isDynamicSetting("foo.bar.baz"));
+        assertTrue(settings.isDynamicSetting("foo.bar"));
         assertNotNull(settings.get("foo.bar.baz"));
         settings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-        assertTrue(settings.hasDynamicSetting("transport.tracer.include." + randomIntBetween(1, 100)));
-        assertFalse(settings.hasDynamicSetting("transport.tracer.include.BOOM"));
-        assertTrue(settings.hasDynamicSetting("cluster.routing.allocation.require.value"));
+        assertTrue(settings.isDynamicSetting("transport.tracer.include." + randomIntBetween(1, 100)));
+        assertFalse(settings.isDynamicSetting("transport.tracer.include.BOOM"));
+        assertTrue(settings.isDynamicSetting("cluster.routing.allocation.require.value"));
+    }
+
+    public void testIsFinal() {
+        ClusterSettings settings =
+            new ClusterSettings(Settings.EMPTY,
+                new HashSet<>(Arrays.asList(Setting.intSetting("foo.int", 1, Property.Final, Property.NodeScope),
+                    Setting.groupSetting("foo.group.",  Property.Final, Property.NodeScope),
+                    Setting.groupSetting("foo.list.",  Property.Final, Property.NodeScope),
+                    Setting.intSetting("foo.int.baz", 1, Property.NodeScope))));
+
+        assertFalse(settings.isFinalSetting("foo.int.baz"));
+        assertTrue(settings.isFinalSetting("foo.int"));
+
+        assertFalse(settings.isFinalSetting("foo.list"));
+        assertTrue(settings.isFinalSetting("foo.list.0.key"));
+        assertTrue(settings.isFinalSetting("foo.list.key"));
+
+        assertFalse(settings.isFinalSetting("foo.group"));
+        assertTrue(settings.isFinalSetting("foo.group.key"));
     }
 
     public void testDiff() throws IOException {
-        Setting<Integer> foobarbaz = Setting.intSetting("foo.bar.baz", 1, Property.NodeScope);
-        Setting<Integer> foobar = Setting.intSetting("foo.bar", 1, Property.Dynamic, Property.NodeScope);
-        ClusterSettings settings = new ClusterSettings(Settings.EMPTY, new HashSet<>(Arrays.asList(foobar, foobarbaz)));
+        Setting<Integer> fooBarBaz = Setting.intSetting("foo.bar.baz", 1, Property.NodeScope);
+        Setting<Integer> fooBar = Setting.intSetting("foo.bar", 1, Property.Dynamic, Property.NodeScope);
+        Setting<Settings> someGroup = Setting.groupSetting("some.group.", Property.Dynamic, Property.NodeScope);
+        Setting<Boolean> someAffix = Setting.affixKeySetting("some.prefix.", "somekey", (key) -> Setting.boolSetting(key, true,
+            Property.NodeScope));
+        Setting<List<String>> foorBarQuux =
+                Setting.listSetting("foo.bar.quux", Arrays.asList("a", "b", "c"), Function.identity(), Property.NodeScope);
+        ClusterSettings settings = new ClusterSettings(Settings.EMPTY, new HashSet<>(Arrays.asList(fooBar, fooBarBaz, foorBarQuux,
+            someGroup, someAffix)));
         Settings diff = settings.diff(Settings.builder().put("foo.bar", 5).build(), Settings.EMPTY);
-        assertEquals(diff.getAsMap().size(), 1);
-        assertEquals(diff.getAsInt("foo.bar.baz", null), Integer.valueOf(1));
+        assertEquals(4, diff.size()); // 4 since foo.bar.quux has 3 values essentially
+        assertThat(diff.getAsInt("foo.bar.baz", null), equalTo(1));
+        assertArrayEquals(diff.getAsArray("foo.bar.quux", null), new String[] {"a", "b", "c"});
+
+        diff = settings.diff(
+                Settings.builder().put("foo.bar", 5).build(),
+                Settings.builder().put("foo.bar.baz", 17).putArray("foo.bar.quux", "d", "e", "f").build());
+        assertEquals(4, diff.size()); // 4 since foo.bar.quux has 3 values essentially
+        assertThat(diff.getAsInt("foo.bar.baz", null), equalTo(17));
+        assertArrayEquals(diff.getAsArray("foo.bar.quux", null), new String[] {"d", "e", "f"});
+
+        diff = settings.diff(
+            Settings.builder().put("some.group.foo", 5).build(),
+            Settings.builder().put("some.group.foobar", 17, "some.group.foo", 25).build());
+        assertEquals(6, diff.size()); // 6 since foo.bar.quux has 3 values essentially
+        assertThat(diff.getAsInt("some.group.foobar", null), equalTo(17));
+        assertNull(diff.get("some.group.foo"));
+        assertArrayEquals(diff.getAsArray("foo.bar.quux", null), new String[] {"a", "b", "c"});
+        assertThat(diff.getAsInt("foo.bar.baz", null), equalTo(1));
+        assertThat(diff.getAsInt("foo.bar", null), equalTo(1));
+
+        diff = settings.diff(
+            Settings.builder().put("some.prefix.foo.somekey", 5).build(),
+            Settings.builder().put("some.prefix.foobar.somekey", 17,
+                "some.prefix.foo.somekey", 18).build());
+        assertEquals(6, diff.size()); // 6 since foo.bar.quux has 3 values essentially
+        assertThat(diff.getAsInt("some.prefix.foobar.somekey", null), equalTo(17));
+        assertNull(diff.get("some.prefix.foo.somekey"));
+        assertArrayEquals(diff.getAsArray("foo.bar.quux", null), new String[] {"a", "b", "c"});
+        assertThat(diff.getAsInt("foo.bar.baz", null), equalTo(1));
+        assertThat(diff.getAsInt("foo.bar", null), equalTo(1));
+    }
 
-        diff = settings.diff(Settings.builder().put("foo.bar", 5).build(), Settings.builder().put("foo.bar.baz", 17).build());
-        assertEquals(diff.getAsMap().size(), 1);
-        assertEquals(diff.getAsInt("foo.bar.baz", null), Integer.valueOf(17));
+    public void testDiffWithAffixAndComplexMatcher() {
+        Setting<Integer> fooBarBaz = Setting.intSetting("foo.bar.baz", 1, Property.NodeScope);
+        Setting<Integer> fooBar = Setting.intSetting("foo.bar", 1, Property.Dynamic, Property.NodeScope);
+        Setting<Settings> someGroup = Setting.groupSetting("some.group.", Property.Dynamic, Property.NodeScope);
+        Setting<Boolean> someAffix = Setting.affixKeySetting("some.prefix.", "somekey", (key) -> Setting.boolSetting(key, true,
+            Property.NodeScope));
+        Setting<List<String>> foorBarQuux = Setting.affixKeySetting("foo.", "quux",
+            (key) -> Setting.listSetting(key,  Arrays.asList("a", "b", "c"), Function.identity(), Property.NodeScope));
+        ClusterSettings settings = new ClusterSettings(Settings.EMPTY, new HashSet<>(Arrays.asList(fooBar, fooBarBaz, foorBarQuux,
+            someGroup, someAffix)));
+        Settings diff = settings.diff(Settings.builder().put("foo.bar", 5).build(), Settings.EMPTY);
+        assertEquals(1, diff.size());
+        assertThat(diff.getAsInt("foo.bar.baz", null), equalTo(1));
+        assertNull(diff.getAsArray("foo.bar.quux", null)); // affix settings don't know their concrete keys
+
+        diff = settings.diff(
+            Settings.builder().put("foo.bar", 5).build(),
+            Settings.builder().put("foo.bar.baz", 17).putArray("foo.bar.quux", "d", "e", "f").build());
+        assertEquals(4, diff.size());
+        assertThat(diff.getAsInt("foo.bar.baz", null), equalTo(17));
+        assertArrayEquals(diff.getAsArray("foo.bar.quux", null), new String[] {"d", "e", "f"});
+
+        diff = settings.diff(
+            Settings.builder().put("some.group.foo", 5).build(),
+            Settings.builder().put("some.group.foobar", 17, "some.group.foo", 25).build());
+        assertEquals(3, diff.size());
+        assertThat(diff.getAsInt("some.group.foobar", null), equalTo(17));
+        assertNull(diff.get("some.group.foo"));
+        assertNull(diff.getAsArray("foo.bar.quux", null)); // affix settings don't know their concrete keys
+        assertThat(diff.getAsInt("foo.bar.baz", null), equalTo(1));
+        assertThat(diff.getAsInt("foo.bar", null), equalTo(1));
+
+        diff = settings.diff(
+            Settings.builder().put("some.prefix.foo.somekey", 5).build(),
+            Settings.builder().put("some.prefix.foobar.somekey", 17,
+                "some.prefix.foo.somekey", 18).build());
+        assertEquals(3, diff.size());
+        assertThat(diff.getAsInt("some.prefix.foobar.somekey", null), equalTo(17));
+        assertNull(diff.get("some.prefix.foo.somekey"));
+        assertNull(diff.getAsArray("foo.bar.quux", null)); // affix settings don't know their concrete keys
+        assertThat(diff.getAsInt("foo.bar.baz", null), equalTo(1));
+        assertThat(diff.getAsInt("foo.bar", null), equalTo(1));
+
+        diff = settings.diff(
+            Settings.builder().put("some.prefix.foo.somekey", 5).build(),
+            Settings.builder().put("some.prefix.foobar.somekey", 17,
+                "some.prefix.foo.somekey", 18)
+            .putArray("foo.bar.quux", "x", "y", "z")
+            .putArray("foo.baz.quux", "d", "e", "f")
+                .build());
+        assertEquals(9, diff.size());
+        assertThat(diff.getAsInt("some.prefix.foobar.somekey", null), equalTo(17));
+        assertNull(diff.get("some.prefix.foo.somekey"));
+        assertArrayEquals(diff.getAsArray("foo.bar.quux", null), new String[] {"x", "y", "z"});
+        assertArrayEquals(diff.getAsArray("foo.baz.quux", null), new String[] {"d", "e", "f"});
+        assertThat(diff.getAsInt("foo.bar.baz", null), equalTo(1));
+        assertThat(diff.getAsInt("foo.bar", null), equalTo(1));
     }
 
     public void testUpdateTracer() {
@@ -208,42 +447,51 @@ public void testValidate() {
             " removed settings";
         settings.validate(Settings.builder().put("index.store.type", "boom"));
         settings.validate(Settings.builder().put("index.store.type", "boom").build());
-        try {
-            settings.validate(Settings.builder().put("index.store.type", "boom", "i.am.not.a.setting", true));
-            fail();
-        } catch (IllegalArgumentException e) {
-            assertEquals("unknown setting [i.am.not.a.setting]" + unknownMsgSuffix, e.getMessage());
-        }
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () ->
+            settings.validate(Settings.builder().put("index.store.type", "boom", "i.am.not.a.setting", true)));
+        assertEquals("unknown setting [i.am.not.a.setting]" + unknownMsgSuffix, e.getMessage());
 
-        try {
-            settings.validate(Settings.builder().put("index.store.type", "boom", "i.am.not.a.setting", true).build());
-            fail();
-        } catch (IllegalArgumentException e) {
-            assertEquals("unknown setting [i.am.not.a.setting]" + unknownMsgSuffix, e.getMessage());
-        }
+        e = expectThrows(IllegalArgumentException.class, () ->
+            settings.validate(Settings.builder().put("index.store.type", "boom", "i.am.not.a.setting", true).build()));
+        assertEquals("unknown setting [i.am.not.a.setting]" + unknownMsgSuffix, e.getMessage());
 
-        try {
-            settings.validate(Settings.builder().put("index.store.type", "boom", "index.number_of_replicas", true).build());
-            fail();
-        } catch (IllegalArgumentException e) {
-            assertEquals("Failed to parse value [true] for setting [index.number_of_replicas]", e.getMessage());
-        }
+        e = expectThrows(IllegalArgumentException.class, () ->
+            settings.validate(Settings.builder().put("index.store.type", "boom", "index.number_of_replicas", true).build()));
+        assertEquals("Failed to parse value [true] for setting [index.number_of_replicas]", e.getMessage());
 
-        try {
-            settings.validate("index.number_of_replicas", Settings.builder().put("index.number_of_replicas", "true").build());
-            fail();
-        } catch (IllegalArgumentException e) {
-            assertEquals("Failed to parse value [true] for setting [index.number_of_replicas]", e.getMessage());
-        }
+        e = expectThrows(IllegalArgumentException.class, () ->
+            settings.validate("index.number_of_replicas", Settings.builder().put("index.number_of_replicas", "true").build()));
+        assertEquals("Failed to parse value [true] for setting [index.number_of_replicas]", e.getMessage());
 
-        try {
-            settings.validate("index.similarity.classic.type", Settings.builder().put("index.similarity.classic.type", "mine").build());
-            fail();
-        } catch (IllegalArgumentException e) {
-            assertEquals("illegal value for [index.similarity.classic] cannot redefine built-in similarity", e.getMessage());
-        }
+        e = expectThrows(IllegalArgumentException.class, () ->
+            settings.validate("index.similarity.classic.type", Settings.builder().put("index.similarity.classic.type", "mine").build()));
+        assertEquals("illegal value for [index.similarity.classic] cannot redefine built-in similarity", e.getMessage());
+    }
+
+    public void testValidateSecureSettings() {
+        MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("some.secure.setting", "secret");
+        Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
+        final ClusterSettings clusterSettings = new ClusterSettings(settings, Collections.emptySet());
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> clusterSettings.validate(settings));
+        assertThat(e.getMessage(), startsWith("unknown secure setting [some.secure.setting]"));
+
+        ClusterSettings clusterSettings2 = new ClusterSettings(settings,
+            Collections.singleton(SecureSetting.secureString("some.secure.setting", null)));
+        clusterSettings2.validate(settings);
     }
 
+    public void testDiffSecureSettings() {
+        MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("some.secure.setting", "secret");
+        Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
+        ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY,
+            Collections.singleton(SecureSetting.secureString("some.secure.setting", null)));
+
+        Settings diffed = clusterSettings.diff(Settings.EMPTY, settings);
+        assertTrue(diffed.isEmpty());
+    }
 
     public static IndexMetaData newIndexMeta(String name, Settings indexSettings) {
         Settings build = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
@@ -303,6 +551,17 @@ public void testKeyPattern() {
             Settings.EMPTY, Collections.singleton(Setting.boolSetting("index.boo", true, Property.NodeScope)));
     }
 
+    public void testAffixKeyPattern() {
+        assertTrue(AbstractScopedSettings.isValidAffixKey("prefix.*.suffix"));
+        assertTrue(AbstractScopedSettings.isValidAffixKey("prefix.*.split.suffix"));
+        assertTrue(AbstractScopedSettings.isValidAffixKey("split.prefix.*.split.suffix"));
+        assertFalse(AbstractScopedSettings.isValidAffixKey("prefix.*.suffix."));
+        assertFalse(AbstractScopedSettings.isValidAffixKey("prefix.*"));
+        assertFalse(AbstractScopedSettings.isValidAffixKey("*.suffix"));
+        assertFalse(AbstractScopedSettings.isValidAffixKey("*"));
+        assertFalse(AbstractScopedSettings.isValidAffixKey(""));
+    }
+
     public void testLoggingUpdates() {
         final Level level = ESLoggerFactory.getRootLogger().getLevel();
         final Level testLevel = ESLoggerFactory.getLogger("test").getLevel();
@@ -365,4 +624,45 @@ public void testOverlappingComplexMatchSettings() {
             }
         }
     }
+
+    public void testUpdateNumberOfShardsFail() {
+        IndexScopedSettings settings = new IndexScopedSettings(Settings.EMPTY,
+            IndexScopedSettings.BUILT_IN_INDEX_SETTINGS);
+        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
+            () -> settings.updateSettings(Settings.builder().put("index.number_of_shards", 8).build(),
+                Settings.builder(), Settings.builder(), "index"));
+        assertThat(ex.getMessage(),
+            containsString("final index setting [index.number_of_shards], not updateable"));
+    }
+
+    public void testFinalSettingUpdateFail() {
+        Setting<Integer> finalSetting = Setting.intSetting("some.final.setting", 1, Property.Final, Property.NodeScope);
+        Setting<Settings> finalGroupSetting = Setting.groupSetting("some.final.group.", Property.Final, Property.NodeScope);
+        Settings currentSettings = Settings.builder()
+            .put("some.final.setting", 9)
+            .put("some.final.group.foo", 7)
+            .build();
+        ClusterSettings service = new ClusterSettings(currentSettings
+            , new HashSet<>(Arrays.asList(finalSetting, finalGroupSetting)));
+
+        IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () ->
+            service.updateDynamicSettings(Settings.builder().put("some.final.setting", 8).build(),
+                Settings.builder().put(currentSettings), Settings.builder(), "node"));
+        assertThat(exc.getMessage(), containsString("final node setting [some.final.setting]"));
+
+        exc = expectThrows(IllegalArgumentException.class, () ->
+            service.updateDynamicSettings(Settings.builder().putNull("some.final.setting").build(),
+                Settings.builder().put(currentSettings), Settings.builder(), "node"));
+        assertThat(exc.getMessage(), containsString("final node setting [some.final.setting]"));
+
+        exc = expectThrows(IllegalArgumentException.class, () ->
+            service.updateSettings(Settings.builder().put("some.final.group.new", 8).build(),
+                Settings.builder().put(currentSettings), Settings.builder(), "node"));
+        assertThat(exc.getMessage(), containsString("final node setting [some.final.group.new]"));
+
+        exc = expectThrows(IllegalArgumentException.class, () ->
+            service.updateSettings(Settings.builder().put("some.final.group.foo", 5).build(),
+                Settings.builder().put(currentSettings), Settings.builder(), "node"));
+        assertThat(exc.getMessage(), containsString("final node setting [some.final.group.foo]"));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/common/settings/SecureStringTests.java b/core/src/test/java/org/elasticsearch/common/settings/SecureStringTests.java
new file mode 100644
index 0000000000000..50bdab1d4f9bd
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/common/settings/SecureStringTests.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Arrays;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.sameInstance;
+
+public class SecureStringTests extends ESTestCase {
+
+    public void testCloseableCharsDoesNotModifySecureString() {
+        final char[] password = randomAlphaOfLengthBetween(1, 32).toCharArray();
+        SecureString secureString = new SecureString(password);
+        assertSecureStringEqualToChars(password, secureString);
+        try (SecureString copy = secureString.clone()) {
+            assertArrayEquals(password, copy.getChars());
+            assertThat(copy.getChars(), not(sameInstance(password)));
+        }
+        assertSecureStringEqualToChars(password, secureString);
+    }
+
+    public void testClosingSecureStringDoesNotModifyCloseableChars() {
+        final char[] password = randomAlphaOfLengthBetween(1, 32).toCharArray();
+        SecureString secureString = new SecureString(password);
+        assertSecureStringEqualToChars(password, secureString);
+        SecureString copy = secureString.clone();
+        assertArrayEquals(password, copy.getChars());
+        assertThat(copy.getChars(), not(sameInstance(password)));
+        final char[] passwordCopy = Arrays.copyOf(password, password.length);
+        assertArrayEquals(password, passwordCopy);
+        secureString.close();
+        assertNotEquals(password[0], passwordCopy[0]);
+        assertArrayEquals(passwordCopy, copy.getChars());
+    }
+
+    public void testClosingChars() {
+        final char[] password = randomAlphaOfLengthBetween(1, 32).toCharArray();
+        SecureString secureString = new SecureString(password);
+        assertSecureStringEqualToChars(password, secureString);
+        SecureString copy = secureString.clone();
+        assertArrayEquals(password, copy.getChars());
+        assertThat(copy.getChars(), not(sameInstance(password)));
+        copy.close();
+        if (randomBoolean()) {
+            // close another time and no exception is thrown
+            copy.close();
+        }
+        IllegalStateException e = expectThrows(IllegalStateException.class, copy::getChars);
+        assertThat(e.getMessage(), containsString("already been closed"));
+    }
+
+    public void testGetCloseableCharsAfterSecureStringClosed() {
+        final char[] password = randomAlphaOfLengthBetween(1, 32).toCharArray();
+        SecureString secureString = new SecureString(password);
+        assertSecureStringEqualToChars(password, secureString);
+        secureString.close();
+        if (randomBoolean()) {
+            // close another time and no exception is thrown
+            secureString.close();
+        }
+        IllegalStateException e = expectThrows(IllegalStateException.class, secureString::clone);
+        assertThat(e.getMessage(), containsString("already been closed"));
+    }
+
+    private void assertSecureStringEqualToChars(char[] expected, SecureString secureString) {
+        int pos = 0;
+        for (int i : secureString.chars().toArray()) {
+            if (pos >= expected.length) {
+                fail("Index " + i + " greated than or equal to array length " + expected.length);
+            } else {
+                assertEquals(expected[pos++], (char) i);
+            }
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/common/settings/SettingTests.java b/core/src/test/java/org/elasticsearch/common/settings/SettingTests.java
index 17bd0d93d7233..5bc003c4de96a 100644
--- a/core/src/test/java/org/elasticsearch/common/settings/SettingTests.java
+++ b/core/src/test/java/org/elasticsearch/common/settings/SettingTests.java
@@ -31,8 +31,11 @@
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 
 public class SettingTests extends ESTestCase {
@@ -48,11 +51,11 @@ public void testByteSize() {
             Setting.byteSizeSetting("a.byte.size", new ByteSizeValue(1024), Property.Dynamic, Property.NodeScope);
         assertFalse(byteSizeValueSetting.isGroupSetting());
         ByteSizeValue byteSizeValue = byteSizeValueSetting.get(Settings.EMPTY);
-        assertEquals(byteSizeValue.bytes(), 1024);
+        assertEquals(byteSizeValue.getBytes(), 1024);
 
         byteSizeValueSetting = Setting.byteSizeSetting("a.byte.size", s -> "2048b", Property.Dynamic, Property.NodeScope);
         byteSizeValue = byteSizeValueSetting.get(Settings.EMPTY);
-        assertEquals(byteSizeValue.bytes(), 2048);
+        assertEquals(byteSizeValue.getBytes(), 2048);
 
 
         AtomicReference<ByteSizeValue> value = new AtomicReference<>(null);
@@ -75,20 +78,20 @@ public void testMemorySize() {
 
         assertFalse(memorySizeValueSetting.isGroupSetting());
         ByteSizeValue memorySizeValue = memorySizeValueSetting.get(Settings.EMPTY);
-        assertEquals(memorySizeValue.bytes(), 1024);
+        assertEquals(memorySizeValue.getBytes(), 1024);
 
         memorySizeValueSetting = Setting.memorySizeSetting("a.byte.size", s -> "2048b", Property.Dynamic, Property.NodeScope);
         memorySizeValue = memorySizeValueSetting.get(Settings.EMPTY);
-        assertEquals(memorySizeValue.bytes(), 2048);
+        assertEquals(memorySizeValue.getBytes(), 2048);
 
         memorySizeValueSetting = Setting.memorySizeSetting("a.byte.size", "50%", Property.Dynamic, Property.NodeScope);
         assertFalse(memorySizeValueSetting.isGroupSetting());
         memorySizeValue = memorySizeValueSetting.get(Settings.EMPTY);
-        assertEquals(memorySizeValue.bytes(), JvmInfo.jvmInfo().getMem().getHeapMax().bytes() * 0.5, 1.0);
+        assertEquals(memorySizeValue.getBytes(), JvmInfo.jvmInfo().getMem().getHeapMax().getBytes() * 0.5, 1.0);
 
         memorySizeValueSetting = Setting.memorySizeSetting("a.byte.size", s -> "25%", Property.Dynamic, Property.NodeScope);
         memorySizeValue = memorySizeValueSetting.get(Settings.EMPTY);
-        assertEquals(memorySizeValue.bytes(), JvmInfo.jvmInfo().getMem().getHeapMax().bytes() * 0.25, 1.0);
+        assertEquals(memorySizeValue.getBytes(), JvmInfo.jvmInfo().getMem().getHeapMax().getBytes() * 0.25, 1.0);
 
         AtomicReference<ByteSizeValue> value = new AtomicReference<>(null);
         ClusterSettings.SettingUpdater<ByteSizeValue> settingUpdater = memorySizeValueSetting.newUpdater(value::set, logger);
@@ -104,7 +107,7 @@ public void testMemorySize() {
         assertEquals(new ByteSizeValue(12), value.get());
 
         assertTrue(settingUpdater.apply(Settings.builder().put("a.byte.size", "20%").build(), Settings.EMPTY));
-        assertEquals(new ByteSizeValue((int) (JvmInfo.jvmInfo().getMem().getHeapMax().bytes() * 0.2)), value.get());
+        assertEquals(new ByteSizeValue((int) (JvmInfo.jvmInfo().getMem().getHeapMax().getBytes() * 0.2)), value.get());
     }
 
     public void testSimpleUpdate() {
@@ -425,7 +428,7 @@ public void testListSettingAcceptsNumberSyntax() {
     }
 
     public void testDynamicKeySetting() {
-        Setting<Boolean> setting = Setting.prefixKeySetting("foo.", "false", Boolean::parseBoolean, Property.NodeScope);
+        Setting<Boolean> setting = Setting.prefixKeySetting("foo.", (key) -> Setting.boolSetting(key, false, Property.NodeScope));
         assertTrue(setting.hasComplexMatcher());
         assertTrue(setting.match("foo.bar"));
         assertFalse(setting.match("foo"));
@@ -441,13 +444,13 @@ public void testDynamicKeySetting() {
         }
     }
 
-    public void testAdfixKeySetting() {
+    public void testAffixKeySetting() {
         Setting<Boolean> setting =
-            Setting.adfixKeySetting("foo", "enable", "false", Boolean::parseBoolean, Property.NodeScope);
+            Setting.affixKeySetting("foo.", "enable", (key) -> Setting.boolSetting(key, false, Property.NodeScope));
         assertTrue(setting.hasComplexMatcher());
         assertTrue(setting.match("foo.bar.enable"));
         assertTrue(setting.match("foo.baz.enable"));
-        assertTrue(setting.match("foo.bar.baz.enable"));
+        assertFalse(setting.match("foo.bar.baz.enable"));
         assertFalse(setting.match("foo.bar"));
         assertFalse(setting.match("foo.bar.baz.enabled"));
         assertFalse(setting.match("foo"));
@@ -455,12 +458,51 @@ public void testAdfixKeySetting() {
         assertTrue(concreteSetting.get(Settings.builder().put("foo.bar.enable", "true").build()));
         assertFalse(concreteSetting.get(Settings.builder().put("foo.baz.enable", "true").build()));
 
-        try {
-            setting.getConcreteSetting("foo");
-            fail();
-        } catch (IllegalArgumentException ex) {
-            assertEquals("key [foo] must match [foo*enable.] but didn't.", ex.getMessage());
-        }
+        IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> setting.getConcreteSetting("foo"));
+        assertEquals("key [foo] must match [foo.*.enable] but didn't.", exc.getMessage());
+
+        exc = expectThrows(IllegalArgumentException.class, () -> Setting.affixKeySetting("foo", "enable",
+            (key) -> Setting.boolSetting(key, false, Property.NodeScope)));
+        assertEquals("prefix must end with a '.'", exc.getMessage());
+
+        Setting<List<String>> listAffixSetting = Setting.affixKeySetting("foo.", "bar",
+            (key) -> Setting.listSetting(key, Collections.emptyList(), Function.identity(), Property.NodeScope));
+
+        assertTrue(listAffixSetting.hasComplexMatcher());
+        assertTrue(listAffixSetting.match("foo.test.bar"));
+        assertTrue(listAffixSetting.match("foo.test_1.bar"));
+        assertFalse(listAffixSetting.match("foo.buzz.baz.bar"));
+        assertFalse(listAffixSetting.match("foo.bar"));
+        assertFalse(listAffixSetting.match("foo.baz"));
+        assertFalse(listAffixSetting.match("foo"));
+    }
+
+    public void testGetAllConcreteSettings() {
+        Setting.AffixSetting<List<String>> listAffixSetting = Setting.affixKeySetting("foo.", "bar",
+            (key) -> Setting.listSetting(key, Collections.emptyList(), Function.identity(), Property.NodeScope));
+
+        Settings settings = Settings.builder()
+            .putArray("foo.1.bar", "1", "2")
+            .putArray("foo.2.bar", "3", "4", "5")
+            .putArray("foo.bar", "6")
+            .putArray("some.other", "6")
+            .putArray("foo.3.bar", "6")
+            .build();
+        Stream<Setting<List<String>>> allConcreteSettings = listAffixSetting.getAllConcreteSettings(settings);
+        Map<String, List<String>> collect = allConcreteSettings.collect(Collectors.toMap(Setting::getKey, (s) -> s.get(settings)));
+        assertEquals(3, collect.size());
+        assertEquals(Arrays.asList("1", "2"), collect.get("foo.1.bar"));
+        assertEquals(Arrays.asList("3", "4", "5"), collect.get("foo.2.bar"));
+        assertEquals(Arrays.asList("6"), collect.get("foo.3.bar"));
+    }
+
+    public void testAffixSettingsFailOnGet() {
+        Setting.AffixSetting<List<String>> listAffixSetting = Setting.affixKeySetting("foo.", "bar",
+            (key) -> Setting.listSetting(key, Collections.singletonList("testelement"), Function.identity(), Property.NodeScope));
+        expectThrows(UnsupportedOperationException.class, () -> listAffixSetting.get(Settings.EMPTY));
+        expectThrows(UnsupportedOperationException.class, () -> listAffixSetting.getRaw(Settings.EMPTY));
+        assertEquals(Collections.singletonList("testelement"), listAffixSetting.getDefault(Settings.EMPTY));
+        assertEquals("[\"testelement\"]", listAffixSetting.getDefaultRaw(Settings.EMPTY));
     }
 
     public void testMinMaxInt() {
@@ -510,11 +552,27 @@ public void testMutuallyExclusiveScopes() {
      * We can't have Null properties
      */
     public void testRejectNullProperties() {
-        try {
-            Setting.simpleString("foo.bar", (Property[]) null);
-            fail();
-        } catch (IllegalArgumentException ex) {
-            assertThat(ex.getMessage(), containsString("properties cannot be null for setting"));
-        }
+        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
+            () -> Setting.simpleString("foo.bar", (Property[]) null));
+        assertThat(ex.getMessage(), containsString("properties cannot be null for setting"));
+    }
+
+    public void testRejectConflictProperties() {
+        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
+            () -> Setting.simpleString("foo.bar", Property.Final, Property.Dynamic));
+        assertThat(ex.getMessage(), containsString("final setting [foo.bar] cannot be dynamic"));
     }
+
+    public void testTimeValue() {
+        final TimeValue random = TimeValue.parseTimeValue(randomTimeValue(), "test");
+
+        Setting<TimeValue> setting = Setting.timeSetting("foo", random);
+        assertThat(setting.get(Settings.EMPTY), equalTo(random));
+
+        final int factor = randomIntBetween(1, 10);
+        setting = Setting.timeSetting("foo", (s) -> TimeValue.timeValueMillis(random.getMillis() * factor), TimeValue.ZERO);
+        assertThat(setting.get(Settings.builder().put("foo", "12h").build()), equalTo(TimeValue.timeValueHours(12)));
+        assertThat(setting.get(Settings.EMPTY).getMillis(), equalTo(random.getMillis() * factor));
+    }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/common/settings/SettingsFilterTests.java b/core/src/test/java/org/elasticsearch/common/settings/SettingsFilterTests.java
index 14e93a3bbc35b..93f745d290e5c 100644
--- a/core/src/test/java/org/elasticsearch/common/settings/SettingsFilterTests.java
+++ b/core/src/test/java/org/elasticsearch/common/settings/SettingsFilterTests.java
@@ -118,7 +118,7 @@ private void testFiltering(Settings source, Settings filtered, String... pattern
         source.toXContent(xContentBuilder, request);
         xContentBuilder.endObject();
         String filteredSettingsString = xContentBuilder.string();
-        filteredSettings = Settings.builder().loadFromSource(filteredSettingsString).build();
+        filteredSettings = Settings.builder().loadFromSource(filteredSettingsString, xContentBuilder.contentType()).build();
         assertThat(filteredSettings.getAsMap().entrySet(), equalTo(filtered.getAsMap().entrySet()));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java b/core/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java
index dc0545624d662..e0a6a37f4cb03 100644
--- a/core/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java
@@ -24,8 +24,11 @@
 import org.joda.time.MonthDay;
 
 import java.util.Arrays;
-import java.util.Collections;
+import java.util.List;
+import java.util.Set;
 
+import static java.util.Collections.emptyList;
+import static java.util.Collections.singletonList;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.is;
 
@@ -163,12 +166,37 @@ public void testRegisterSettingsFilter() {
             Setting.boolSetting("bar.foo", true, Property.NodeScope, Property.Filtered),
             Setting.boolSetting("bar.baz", true, Property.NodeScope)), Arrays.asList("foo.*"));
         assertInstanceBinding(module, Settings.class, (s) -> s == settings);
-        assertInstanceBinding(module, SettingsFilter.class, (s) -> s.filter(settings).getAsMap().size() == 1);
+        assertInstanceBinding(module, SettingsFilter.class, (s) -> s.filter(settings).size() == 1);
         assertInstanceBinding(module, SettingsFilter.class, (s) -> s.filter(settings).getAsMap().containsKey("bar.baz"));
         assertInstanceBinding(module, SettingsFilter.class, (s) -> s.filter(settings).getAsMap().get("bar.baz").equals("false"));
 
     }
 
+    public void testRegisterListSettingsFilter() {
+        final Settings settings = Settings.builder()
+                                          .put("foo.bar", true)
+                                          .put("foo.baz", "one")
+                                          .putArray("foo.qux", Arrays.asList("two", "three", "four"))
+                                          .build();
+
+        final List<Setting<?>> additionalSettings = Arrays.asList(
+            Setting.simpleString("foo.bar", Property.NodeScope),
+            Setting.simpleString("foo.baz", Property.NodeScope, Property.Filtered),
+            Setting.listSetting("foo.qux", singletonList("default"), String::toString, Property.NodeScope, Property.Filtered)
+        );
+
+        final SettingsModule module = new SettingsModule(settings, additionalSettings, emptyList());
+
+        final Set<String> filters = module.getSettingsFilter().getPatterns();
+        assertEquals(3, filters.size());
+        assertTrue(filters.contains("foo.baz"));
+        assertTrue(filters.contains("foo.qux"));
+        assertTrue(filters.contains("foo.qux.*"));
+
+        assertInstanceBinding(module, SettingsFilter.class, (s) -> s.filter(settings).size() == 1);
+        assertInstanceBinding(module, SettingsFilter.class, (s) -> s.filter(settings).getAsMap().containsKey("foo.bar"));
+    }
+
     public void testMutuallyExclusiveScopes() {
         new SettingsModule(Settings.EMPTY, Setting.simpleString("foo.bar", Property.NodeScope));
         new SettingsModule(Settings.EMPTY, Setting.simpleString("index.foo.bar", Property.IndexScope));
diff --git a/core/src/test/java/org/elasticsearch/common/settings/SettingsTests.java b/core/src/test/java/org/elasticsearch/common/settings/SettingsTests.java
index 346c5bc60de34..4c2ebd5b81093 100644
--- a/core/src/test/java/org/elasticsearch/common/settings/SettingsTests.java
+++ b/core/src/test/java/org/elasticsearch/common/settings/SettingsTests.java
@@ -19,18 +19,23 @@
 
 package org.elasticsearch.common.settings;
 
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.loader.YamlSettingsLoader;
 import org.elasticsearch.test.ESTestCase;
 import org.hamcrest.Matchers;
 
 import java.io.IOException;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.NoSuchElementException;
 import java.util.Set;
 
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.arrayContaining;
 import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasToString;
@@ -62,7 +67,7 @@ public void testReplacePropertiesPlaceholderSystemVariablesHaveNoEffect() {
     }
 
     public void testReplacePropertiesPlaceholderByEnvironmentVariables() {
-        final String hostname = randomAsciiOfLength(16);
+        final String hostname = randomAlphaOfLength(16);
         final Settings implicitEnvSettings = Settings.builder()
             .put("setting1", "${HOSTNAME}")
             .replacePropertyPlaceholders(name -> "HOSTNAME".equals(name) ? hostname : null)
@@ -131,15 +136,49 @@ public void testFallbackToFlattenedSettings() {
 
     public void testGetAsSettings() {
         Settings settings = Settings.builder()
+                .put("bar", "hello world")
                 .put("foo", "abc")
                 .put("foo.bar", "def")
                 .put("foo.baz", "ghi").build();
 
         Settings fooSettings = settings.getAsSettings("foo");
+        assertFalse(fooSettings.isEmpty());
+        assertEquals(2, fooSettings.size());
         assertThat(fooSettings.get("bar"), equalTo("def"));
         assertThat(fooSettings.get("baz"), equalTo("ghi"));
     }
 
+    public void testMultLevelGetPrefix() {
+        Settings settings = Settings.builder()
+            .put("1.2.3", "hello world")
+            .put("1.2.3.4", "abc")
+            .put("2.3.4", "def")
+            .put("3.4", "ghi").build();
+
+        Settings firstLevelSettings = settings.getByPrefix("1.");
+        assertFalse(firstLevelSettings.isEmpty());
+        assertEquals(2, firstLevelSettings.size());
+        assertThat(firstLevelSettings.get("2.3.4"), equalTo("abc"));
+        assertThat(firstLevelSettings.get("2.3"), equalTo("hello world"));
+
+        Settings secondLevelSetting = firstLevelSettings.getByPrefix("2.");
+        assertFalse(secondLevelSetting.isEmpty());
+        assertEquals(2, secondLevelSetting.size());
+        assertNull(secondLevelSetting.get("2.3.4"));
+        assertNull(secondLevelSetting.get("1.2.3.4"));
+        assertNull(secondLevelSetting.get("1.2.3"));
+        assertThat(secondLevelSetting.get("3.4"), equalTo("abc"));
+        assertThat(secondLevelSetting.get("3"), equalTo("hello world"));
+
+        Settings thirdLevelSetting = secondLevelSetting.getByPrefix("3.");
+        assertFalse(thirdLevelSetting.isEmpty());
+        assertEquals(1, thirdLevelSetting.size());
+        assertNull(thirdLevelSetting.get("2.3.4"));
+        assertNull(thirdLevelSetting.get("3.4"));
+        assertNull(thirdLevelSetting.get("1.2.3"));
+        assertThat(thirdLevelSetting.get("4"), equalTo("abc"));
+    }
+
     public void testNames() {
         Settings settings = Settings.builder()
                 .put("bar", "baz")
@@ -273,7 +312,7 @@ public void testPrefixNormalization() {
                 .normalizePrefix("foo.")
                 .build();
 
-        assertThat(settings.getAsMap().size(), equalTo(1));
+        assertThat(settings.size(), equalTo(1));
         assertThat(settings.get("bar"), nullValue());
         assertThat(settings.get("foo.bar"), equalTo("baz"));
 
@@ -284,7 +323,7 @@ public void testPrefixNormalization() {
                 .normalizePrefix("foo.")
                 .build();
 
-        assertThat(settings.getAsMap().size(), equalTo(2));
+        assertThat(settings.size(), equalTo(2));
         assertThat(settings.get("bar"), nullValue());
         assertThat(settings.get("foo.bar"), equalTo("baz"));
         assertThat(settings.get("foo.test"), equalTo("test"));
@@ -295,7 +334,226 @@ public void testPrefixNormalization() {
                 .build();
 
 
-        assertThat(settings.getAsMap().size(), equalTo(1));
+        assertThat(settings.size(), equalTo(1));
         assertThat(settings.get("foo.test"), equalTo("test"));
     }
+
+    public void testFilteredMap() {
+        Settings.Builder builder = Settings.builder();
+        builder.put("a", "a1");
+        builder.put("a.b", "ab1");
+        builder.put("a.b.c", "ab2");
+        builder.put("a.c", "ac1");
+        builder.put("a.b.c.d", "ab3");
+
+
+        Map<String, String> fiteredMap = builder.build().filter((k) -> k.startsWith("a.b")).getAsMap();
+        assertEquals(3, fiteredMap.size());
+        int numKeys = 0;
+        for (String k : fiteredMap.keySet()) {
+            numKeys++;
+            assertTrue(k.startsWith("a.b"));
+        }
+
+        assertEquals(3, numKeys);
+        int numValues = 0;
+
+        for (String v : fiteredMap.values()) {
+            numValues++;
+            assertTrue(v.startsWith("ab"));
+        }
+        assertEquals(3, numValues);
+        assertFalse(fiteredMap.containsKey("a.c"));
+        assertFalse(fiteredMap.containsKey("a"));
+        assertTrue(fiteredMap.containsKey("a.b"));
+        assertTrue(fiteredMap.containsKey("a.b.c"));
+        assertTrue(fiteredMap.containsKey("a.b.c.d"));
+        expectThrows(UnsupportedOperationException.class, () ->
+            fiteredMap.remove("a.b"));
+        assertEquals("ab1", fiteredMap.get("a.b"));
+        assertEquals("ab2", fiteredMap.get("a.b.c"));
+        assertEquals("ab3", fiteredMap.get("a.b.c.d"));
+
+        Iterator<String> iterator = fiteredMap.keySet().iterator();
+        for (int i = 0; i < 10; i++) {
+            assertTrue(iterator.hasNext());
+        }
+        assertEquals("a.b", iterator.next());
+        if (randomBoolean()) {
+            assertTrue(iterator.hasNext());
+        }
+        assertEquals("a.b.c", iterator.next());
+        if (randomBoolean()) {
+            assertTrue(iterator.hasNext());
+        }
+        assertEquals("a.b.c.d", iterator.next());
+        assertFalse(iterator.hasNext());
+        expectThrows(NoSuchElementException.class, () -> iterator.next());
+
+    }
+
+    public void testPrefixMap() {
+        Settings.Builder builder = Settings.builder();
+        builder.put("a", "a1");
+        builder.put("a.b", "ab1");
+        builder.put("a.b.c", "ab2");
+        builder.put("a.c", "ac1");
+        builder.put("a.b.c.d", "ab3");
+
+        Map<String, String> prefixMap = builder.build().getByPrefix("a.").getAsMap();
+        assertEquals(4, prefixMap.size());
+        int numKeys = 0;
+        for (String k : prefixMap.keySet()) {
+            numKeys++;
+            assertTrue(k, k.startsWith("b") || k.startsWith("c"));
+        }
+
+        assertEquals(4, numKeys);
+        int numValues = 0;
+
+        for (String v : prefixMap.values()) {
+            numValues++;
+            assertTrue(v, v.startsWith("ab") || v.startsWith("ac"));
+        }
+        assertEquals(4, numValues);
+        assertFalse(prefixMap.containsKey("a"));
+        assertTrue(prefixMap.containsKey("c"));
+        assertTrue(prefixMap.containsKey("b"));
+        assertTrue(prefixMap.containsKey("b.c"));
+        assertTrue(prefixMap.containsKey("b.c.d"));
+        expectThrows(UnsupportedOperationException.class, () ->
+            prefixMap.remove("a.b"));
+        assertEquals("ab1", prefixMap.get("b"));
+        assertEquals("ab2", prefixMap.get("b.c"));
+        assertEquals("ab3", prefixMap.get("b.c.d"));
+        Iterator<String> prefixIterator = prefixMap.keySet().iterator();
+        for (int i = 0; i < 10; i++) {
+            assertTrue(prefixIterator.hasNext());
+        }
+        assertEquals("b", prefixIterator.next());
+        if (randomBoolean()) {
+            assertTrue(prefixIterator.hasNext());
+        }
+        assertEquals("b.c", prefixIterator.next());
+        if (randomBoolean()) {
+            assertTrue(prefixIterator.hasNext());
+        }
+        assertEquals("b.c.d", prefixIterator.next());
+        if (randomBoolean()) {
+            assertTrue(prefixIterator.hasNext());
+        }
+        assertEquals("c", prefixIterator.next());
+        assertFalse(prefixIterator.hasNext());
+        expectThrows(NoSuchElementException.class, () -> prefixIterator.next());
+    }
+
+    public void testSecureSettingsPrefix() {
+        MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("test.prefix.foo", "somethingsecure");
+        Settings.Builder builder = Settings.builder();
+        builder.setSecureSettings(secureSettings);
+        Settings settings = builder.build();
+        Settings prefixSettings = settings.getByPrefix("test.prefix.");
+        assertTrue(prefixSettings.names().contains("foo"));
+    }
+
+    public void testGroupPrefix() {
+        MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("test.key1.foo", "somethingsecure");
+        secureSettings.setString("test.key1.bar", "somethingsecure");
+        secureSettings.setString("test.key2.foo", "somethingsecure");
+        secureSettings.setString("test.key2.bog", "somethingsecure");
+        Settings.Builder builder = Settings.builder();
+        builder.put("test.key1.baz", "blah1");
+        builder.put("test.key1.other", "blah2");
+        builder.put("test.key2.baz", "blah3");
+        builder.put("test.key2.else", "blah4");
+        builder.setSecureSettings(secureSettings);
+        Settings settings = builder.build();
+        Map<String, Settings> groups = settings.getGroups("test");
+        assertEquals(2, groups.size());
+        Settings key1 = groups.get("key1");
+        assertNotNull(key1);
+        assertThat(key1.names(), containsInAnyOrder("foo", "bar", "baz", "other"));
+        Settings key2 = groups.get("key2");
+        assertNotNull(key2);
+        assertThat(key2.names(), containsInAnyOrder("foo", "bog", "baz", "else"));
+    }
+
+    public void testEmptyFilterMap() {
+        Settings.Builder builder = Settings.builder();
+        builder.put("a", "a1");
+        builder.put("a.b", "ab1");
+        builder.put("a.b.c", "ab2");
+        builder.put("a.c", "ac1");
+        builder.put("a.b.c.d", "ab3");
+
+        Map<String, String> fiteredMap = builder.build().filter((k) -> false).getAsMap();
+        assertEquals(0, fiteredMap.size());
+        for (String k : fiteredMap.keySet()) {
+            fail("no element");
+
+        }
+        for (String v : fiteredMap.values()) {
+            fail("no element");
+        }
+        assertFalse(fiteredMap.containsKey("a.c"));
+        assertFalse(fiteredMap.containsKey("a"));
+        assertFalse(fiteredMap.containsKey("a.b"));
+        assertFalse(fiteredMap.containsKey("a.b.c"));
+        assertFalse(fiteredMap.containsKey("a.b.c.d"));
+        expectThrows(UnsupportedOperationException.class, () ->
+            fiteredMap.remove("a.b"));
+        assertNull(fiteredMap.get("a.b"));
+        assertNull(fiteredMap.get("a.b.c"));
+        assertNull(fiteredMap.get("a.b.c.d"));
+
+        Iterator<String> iterator = fiteredMap.keySet().iterator();
+        for (int i = 0; i < 10; i++) {
+            assertFalse(iterator.hasNext());
+        }
+        expectThrows(NoSuchElementException.class, () -> iterator.next());
+    }
+
+    public void testEmpty() {
+        assertTrue(Settings.EMPTY.isEmpty());
+        MockSecureSettings secureSettings = new MockSecureSettings();
+        assertTrue(Settings.builder().setSecureSettings(secureSettings).build().isEmpty());
+    }
+
+    public void testWriteSettingsToStream() throws IOException {
+        BytesStreamOutput out = new BytesStreamOutput();
+        MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("test.key1.foo", "somethingsecure");
+        secureSettings.setString("test.key1.bar", "somethingsecure");
+        secureSettings.setString("test.key2.foo", "somethingsecure");
+        secureSettings.setString("test.key2.bog", "somethingsecure");
+        Settings.Builder builder = Settings.builder();
+        builder.put("test.key1.baz", "blah1");
+        builder.setSecureSettings(secureSettings);
+        assertEquals(5, builder.build().size());
+        Settings.writeSettingsToStream(builder.build(), out);
+        StreamInput in = StreamInput.wrap(out.bytes().toBytesRef().bytes);
+        Settings settings = Settings.readSettingsFromStream(in);
+        assertEquals(1, settings.size());
+        assertEquals("blah1", settings.get("test.key1.baz"));
+    }
+
+    public void testSecureSettingConflict() {
+        Setting<SecureString> setting = SecureSetting.secureString("something.secure", null);
+        Settings settings = Settings.builder().put("something.secure", "notreallysecure").build();
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> setting.get(settings));
+        assertTrue(e.getMessage().contains("must be stored inside the Elasticsearch keystore"));
+    }
+
+    public void testGetAsArrayFailsOnDuplicates() {
+        final Settings settings =
+                Settings.builder()
+                        .put("foobar.0", "bar")
+                        .put("foobar.1", "baz")
+                        .put("foobar", "foo")
+                        .build();
+        final IllegalStateException e = expectThrows(IllegalStateException.class, () -> settings.getAsArray("foobar"));
+        assertThat(e, hasToString(containsString("settings object contains values for [foobar=foo] and [foobar.0=bar]")));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/common/settings/loader/JsonSettingsLoaderTests.java b/core/src/test/java/org/elasticsearch/common/settings/loader/JsonSettingsLoaderTests.java
index e6c134ca5f6d0..aec12a446d584 100644
--- a/core/src/test/java/org/elasticsearch/common/settings/loader/JsonSettingsLoaderTests.java
+++ b/core/src/test/java/org/elasticsearch/common/settings/loader/JsonSettingsLoaderTests.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsException;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESTestCase;
 
 import static org.hamcrest.CoreMatchers.containsString;
@@ -49,7 +50,8 @@ public void testSimpleJsonSettings() throws Exception {
 
     public void testDuplicateKeysThrowsException() {
         final String json = "{\"foo\":\"bar\",\"foo\":\"baz\"}";
-        final SettingsException e = expectThrows(SettingsException.class, () -> Settings.builder().loadFromSource(json).build());
+        final SettingsException e = expectThrows(SettingsException.class,
+            () -> Settings.builder().loadFromSource(json, XContentType.JSON).build());
         assertEquals(e.getCause().getClass(), ElasticsearchParseException.class);
         assertThat(
                 e.toString(),
diff --git a/core/src/test/java/org/elasticsearch/common/settings/loader/YamlSettingsLoaderTests.java b/core/src/test/java/org/elasticsearch/common/settings/loader/YamlSettingsLoaderTests.java
index 7c956de8f9a76..803014d35ec7e 100644
--- a/core/src/test/java/org/elasticsearch/common/settings/loader/YamlSettingsLoaderTests.java
+++ b/core/src/test/java/org/elasticsearch/common/settings/loader/YamlSettingsLoaderTests.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsException;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESTestCase;
 
 import java.nio.charset.StandardCharsets;
@@ -70,7 +71,7 @@ public void testIndentationWithExplicitDocumentStart() throws Exception {
     public void testDuplicateKeysThrowsException() {
         String yaml = "foo: bar\nfoo: baz";
         SettingsException e = expectThrows(SettingsException.class, () -> {
-            Settings.builder().loadFromSource(yaml);
+            Settings.builder().loadFromSource(yaml, XContentType.YAML);
         });
         assertEquals(e.getCause().getClass(), ElasticsearchParseException.class);
         String msg = e.getCause().getMessage();
diff --git a/core/src/test/java/org/elasticsearch/common/unit/ByteSizeUnitTests.java b/core/src/test/java/org/elasticsearch/common/unit/ByteSizeUnitTests.java
index ab6d281894635..719313d1c8640 100644
--- a/core/src/test/java/org/elasticsearch/common/unit/ByteSizeUnitTests.java
+++ b/core/src/test/java/org/elasticsearch/common/unit/ByteSizeUnitTests.java
@@ -19,20 +19,23 @@
 
 package org.elasticsearch.common.unit;
 
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.test.ESTestCase;
 
+import java.io.IOException;
+
 import static org.elasticsearch.common.unit.ByteSizeUnit.BYTES;
 import static org.elasticsearch.common.unit.ByteSizeUnit.GB;
 import static org.elasticsearch.common.unit.ByteSizeUnit.KB;
 import static org.elasticsearch.common.unit.ByteSizeUnit.MB;
 import static org.elasticsearch.common.unit.ByteSizeUnit.PB;
 import static org.elasticsearch.common.unit.ByteSizeUnit.TB;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 
-/**
- *
- */
 public class ByteSizeUnitTests extends ESTestCase {
+
     public void testBytes() {
         assertThat(BYTES.toBytes(1), equalTo(1L));
         assertThat(BYTES.toKB(1024), equalTo(1L));
@@ -77,4 +80,23 @@ public void testPB() {
         assertThat(PB.toTB(1), equalTo(1024L));
         assertThat(PB.toPB(1), equalTo(1L));
     }
+
+    public void testSerialization() throws IOException {
+        for (ByteSizeUnit unit : ByteSizeUnit.values()) {
+            try (BytesStreamOutput out = new BytesStreamOutput()) {
+                unit.writeTo(out);
+
+                try (StreamInput in = out.bytes().streamInput()) {
+                    ByteSizeUnit deserialized = ByteSizeUnit.readFrom(in);
+                    assertEquals(unit, deserialized);
+                }
+            }
+        }
+    }
+
+    public void testFromUnknownId() throws IOException {
+        final byte randomId = (byte) randomIntBetween(ByteSizeUnit.values().length + 1, 100);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> ByteSizeUnit.fromId(randomId));
+        assertThat(e.getMessage(), containsString("No byte size unit found for id [" + String.valueOf(randomId) + "]"));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/common/unit/ByteSizeValueTests.java b/core/src/test/java/org/elasticsearch/common/unit/ByteSizeValueTests.java
index 5104e56cb352d..2cc418bc27343 100644
--- a/core/src/test/java/org/elasticsearch/common/unit/ByteSizeValueTests.java
+++ b/core/src/test/java/org/elasticsearch/common/unit/ByteSizeValueTests.java
@@ -36,24 +36,24 @@
  */
 public class ByteSizeValueTests extends ESTestCase {
     public void testActualPeta() {
-        MatcherAssert.assertThat(new ByteSizeValue(4, ByteSizeUnit.PB).bytes(), equalTo(4503599627370496L));
+        MatcherAssert.assertThat(new ByteSizeValue(4, ByteSizeUnit.PB).getBytes(), equalTo(4503599627370496L));
     }
 
     public void testActualTera() {
-        MatcherAssert.assertThat(new ByteSizeValue(4, ByteSizeUnit.TB).bytes(), equalTo(4398046511104L));
+        MatcherAssert.assertThat(new ByteSizeValue(4, ByteSizeUnit.TB).getBytes(), equalTo(4398046511104L));
     }
 
     public void testActual() {
-        MatcherAssert.assertThat(new ByteSizeValue(4, ByteSizeUnit.GB).bytes(), equalTo(4294967296L));
+        MatcherAssert.assertThat(new ByteSizeValue(4, ByteSizeUnit.GB).getBytes(), equalTo(4294967296L));
     }
 
     public void testSimple() {
-        assertThat(ByteSizeUnit.BYTES.toBytes(10), is(new ByteSizeValue(10, ByteSizeUnit.BYTES).bytes()));
-        assertThat(ByteSizeUnit.KB.toKB(10), is(new ByteSizeValue(10, ByteSizeUnit.KB).kb()));
-        assertThat(ByteSizeUnit.MB.toMB(10), is(new ByteSizeValue(10, ByteSizeUnit.MB).mb()));
-        assertThat(ByteSizeUnit.GB.toGB(10), is(new ByteSizeValue(10, ByteSizeUnit.GB).gb()));
-        assertThat(ByteSizeUnit.TB.toTB(10), is(new ByteSizeValue(10, ByteSizeUnit.TB).tb()));
-        assertThat(ByteSizeUnit.PB.toPB(10), is(new ByteSizeValue(10, ByteSizeUnit.PB).pb()));
+        assertThat(ByteSizeUnit.BYTES.toBytes(10), is(new ByteSizeValue(10, ByteSizeUnit.BYTES).getBytes()));
+        assertThat(ByteSizeUnit.KB.toKB(10), is(new ByteSizeValue(10, ByteSizeUnit.KB).getKb()));
+        assertThat(ByteSizeUnit.MB.toMB(10), is(new ByteSizeValue(10, ByteSizeUnit.MB).getMb()));
+        assertThat(ByteSizeUnit.GB.toGB(10), is(new ByteSizeValue(10, ByteSizeUnit.GB).getGb()));
+        assertThat(ByteSizeUnit.TB.toTB(10), is(new ByteSizeValue(10, ByteSizeUnit.TB).getTb()));
+        assertThat(ByteSizeUnit.PB.toPB(10), is(new ByteSizeValue(10, ByteSizeUnit.PB).getPb()));
     }
 
     public void testEquality() {
@@ -126,57 +126,78 @@ public void testParsing() {
     }
 
     public void testFailOnMissingUnits() {
-        try {
-            ByteSizeValue.parseBytesSizeValue("23", "test");
-            fail("Expected ElasticsearchParseException");
-        } catch (ElasticsearchParseException e) {
-            assertThat(e.getMessage(), containsString("failed to parse setting [test]"));
-        }
+        Exception e = expectThrows(ElasticsearchParseException.class, () -> ByteSizeValue.parseBytesSizeValue("23", "test"));
+        assertThat(e.getMessage(), containsString("failed to parse setting [test]"));
     }
 
     public void testFailOnUnknownUnits() {
-        try {
-            ByteSizeValue.parseBytesSizeValue("23jw", "test");
-            fail("Expected ElasticsearchParseException");
-        } catch (ElasticsearchParseException e) {
-            assertThat(e.getMessage(), containsString("failed to parse setting [test]"));
-        }
+        Exception e = expectThrows(ElasticsearchParseException.class, () -> ByteSizeValue.parseBytesSizeValue("23jw", "test"));
+        assertThat(e.getMessage(), containsString("failed to parse setting [test]"));
     }
 
     public void testFailOnEmptyParsing() {
-        try {
-            assertThat(ByteSizeValue.parseBytesSizeValue("", "emptyParsing").toString(), is("23kb"));
-            fail("Expected ElasticsearchParseException");
-        } catch (ElasticsearchParseException e) {
-            assertThat(e.getMessage(), containsString("failed to parse setting [emptyParsing]"));
-        }
+        Exception e = expectThrows(ElasticsearchParseException.class,
+                () -> assertThat(ByteSizeValue.parseBytesSizeValue("", "emptyParsing").toString(), is("23kb")));
+        assertThat(e.getMessage(), containsString("failed to parse setting [emptyParsing]"));
     }
 
     public void testFailOnEmptyNumberParsing() {
-        try {
-            assertThat(ByteSizeValue.parseBytesSizeValue("g", "emptyNumberParsing").toString(), is("23b"));
-            fail("Expected ElasticsearchParseException");
-        } catch (ElasticsearchParseException e) {
-            assertThat(e.getMessage(), containsString("failed to parse [g]"));
-        }
+        Exception e = expectThrows(ElasticsearchParseException.class,
+                () -> assertThat(ByteSizeValue.parseBytesSizeValue("g", "emptyNumberParsing").toString(), is("23b")));
+        assertThat(e.getMessage(), containsString("failed to parse [g]"));
     }
 
     public void testNoDotsAllowed() {
-        try {
-            ByteSizeValue.parseBytesSizeValue("42b.", null, "test");
-            fail("Expected ElasticsearchParseException");
-        } catch (ElasticsearchParseException e) {
-            assertThat(e.getMessage(), containsString("failed to parse setting [test]"));
-        }
+        Exception e = expectThrows(ElasticsearchParseException.class, () -> ByteSizeValue.parseBytesSizeValue("42b.", null, "test"));
+        assertThat(e.getMessage(), containsString("failed to parse setting [test]"));
+    }
+
+    public void testCompareEquality() {
+        long firstRandom = randomNonNegativeLong();
+        ByteSizeUnit randomUnit = randomFrom(ByteSizeUnit.values());
+        ByteSizeValue firstByteValue = new ByteSizeValue(firstRandom, randomUnit);
+        ByteSizeValue secondByteValue = new ByteSizeValue(firstRandom, randomUnit);
+        assertEquals(0, firstByteValue.compareTo(secondByteValue));
+    }
+
+    public void testCompareValue() {
+        long firstRandom = randomNonNegativeLong();
+        long secondRandom = randomValueOtherThan(firstRandom, ESTestCase::randomNonNegativeLong);
+        ByteSizeUnit unit = randomFrom(ByteSizeUnit.values());
+        ByteSizeValue firstByteValue = new ByteSizeValue(firstRandom, unit);
+        ByteSizeValue secondByteValue = new ByteSizeValue(secondRandom, unit);
+        assertEquals(firstRandom > secondRandom, firstByteValue.compareTo(secondByteValue) > 0);
+        assertEquals(secondRandom > firstRandom, secondByteValue.compareTo(firstByteValue) > 0);
+    }
+
+    public void testCompareUnits() {
+        long number = randomNonNegativeLong();
+        ByteSizeUnit randomUnit = randomValueOtherThan(ByteSizeUnit.PB, ()->randomFrom(ByteSizeUnit.values()));
+        ByteSizeValue firstByteValue = new ByteSizeValue(number, randomUnit);
+        ByteSizeValue secondByteValue = new ByteSizeValue(number, ByteSizeUnit.PB);
+        assertTrue(firstByteValue.compareTo(secondByteValue) < 0);
+        assertTrue(secondByteValue.compareTo(firstByteValue) > 0);
+    }
+
+    public void testEdgeCompare() {
+        ByteSizeValue maxLongValuePB = new ByteSizeValue(Long.MAX_VALUE, ByteSizeUnit.PB);
+        ByteSizeValue maxLongValueB = new ByteSizeValue(Long.MAX_VALUE, ByteSizeUnit.BYTES);
+        assertTrue(maxLongValuePB.compareTo(maxLongValueB) > 0);
+    }
+
+    public void testConversionHashCode() {
+        ByteSizeValue firstValue = new ByteSizeValue(randomIntBetween(0, Integer.MAX_VALUE), ByteSizeUnit.GB);
+        ByteSizeValue secondValue = new ByteSizeValue(firstValue.getBytes(), ByteSizeUnit.BYTES);
+        assertEquals(firstValue.hashCode(), secondValue.hashCode());
     }
 
     public void testSerialization() throws IOException {
-        ByteSizeValue byteSizeValue = new ByteSizeValue(randomPositiveLong(), randomFrom(ByteSizeUnit.values()));
+        ByteSizeValue byteSizeValue = new ByteSizeValue(randomNonNegativeLong(), randomFrom(ByteSizeUnit.values()));
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             byteSizeValue.writeTo(out);
             try (StreamInput in = out.bytes().streamInput()) {
                 ByteSizeValue deserializedByteSizeValue = new ByteSizeValue(in);
-                assertEquals(byteSizeValue, deserializedByteSizeValue);
+                assertEquals(byteSizeValue.getBytes(), deserializedByteSizeValue.getBytes());
             }
         }
     }
diff --git a/core/src/test/java/org/elasticsearch/common/unit/FuzzinessTests.java b/core/src/test/java/org/elasticsearch/common/unit/FuzzinessTests.java
index 3f6f1848fd809..b370250bf9dad 100644
--- a/core/src/test/java/org/elasticsearch/common/unit/FuzzinessTests.java
+++ b/core/src/test/java/org/elasticsearch/common/unit/FuzzinessTests.java
@@ -20,9 +20,8 @@
 
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.xcontent.XContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
@@ -31,7 +30,6 @@
 import static org.hamcrest.CoreMatchers.anyOf;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.sameInstance;
-import static org.hamcrest.number.IsCloseTo.closeTo;
 
 public class FuzzinessTests extends ESTestCase {
     public void testNumerics() {
@@ -43,12 +41,11 @@ public void testParseFromXContent() throws IOException {
         final int iters = randomIntBetween(10, 50);
         for (int i = 0; i < iters; i++) {
             {
-                XContent xcontent = XContentType.JSON.xContent();
                 float floatValue = randomFloat();
-                String json = jsonBuilder().startObject()
+                XContentBuilder json = jsonBuilder().startObject()
                         .field(Fuzziness.X_FIELD_NAME, floatValue)
-                        .endObject().string();
-                XContentParser parser = xcontent.createParser(json);
+                        .endObject();
+                XContentParser parser = createParser(json);
                 assertThat(parser.nextToken(), equalTo(XContentParser.Token.START_OBJECT));
                 assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME));
                 assertThat(parser.nextToken(), equalTo(XContentParser.Token.VALUE_NUMBER));
@@ -57,17 +54,16 @@ public void testParseFromXContent() throws IOException {
                 assertThat(parser.nextToken(), equalTo(XContentParser.Token.END_OBJECT));
             }
             {
-                XContent xcontent = XContentType.JSON.xContent();
                 Integer intValue = frequently() ? randomIntBetween(0, 2) : randomIntBetween(0, 100);
                 Float floatRep = randomFloat();
                 Number value = intValue;
                 if (randomBoolean()) {
                     value = new Float(floatRep += intValue);
                 }
-                String json = jsonBuilder().startObject()
+                XContentBuilder json = jsonBuilder().startObject()
                         .field(Fuzziness.X_FIELD_NAME, randomBoolean() ? value.toString() : value)
-                        .endObject().string();
-                XContentParser parser = xcontent.createParser(json);
+                        .endObject();
+                XContentParser parser = createParser(json);
                 assertThat(parser.nextToken(), equalTo(XContentParser.Token.START_OBJECT));
                 assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME));
                 assertThat(parser.nextToken(), anyOf(equalTo(XContentParser.Token.VALUE_NUMBER), equalTo(XContentParser.Token.VALUE_STRING)));
@@ -93,14 +89,15 @@ public void testParseFromXContent() throws IOException {
                 }
             }
             {
-                XContent xcontent = XContentType.JSON.xContent();
-                String json = jsonBuilder().startObject()
-                        .field(Fuzziness.X_FIELD_NAME, randomBoolean() ? "AUTO" : "auto")
-                        .endObject().string();
+                XContentBuilder json;
                 if (randomBoolean()) {
-                    json = Fuzziness.AUTO.toXContent(jsonBuilder().startObject(), null).endObject().string();
+                    json = Fuzziness.AUTO.toXContent(jsonBuilder().startObject(), null).endObject();
+                } else {
+                    json = jsonBuilder().startObject()
+                            .field(Fuzziness.X_FIELD_NAME, randomBoolean() ? "AUTO" : "auto")
+                            .endObject();
                 }
-                XContentParser parser = xcontent.createParser(json);
+                XContentParser parser = createParser(json);
                 assertThat(parser.nextToken(), equalTo(XContentParser.Token.START_OBJECT));
                 assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME));
                 assertThat(parser.nextToken(), equalTo(XContentParser.Token.VALUE_STRING));
diff --git a/core/src/test/java/org/elasticsearch/common/unit/SizeValueTests.java b/core/src/test/java/org/elasticsearch/common/unit/SizeValueTests.java
index b5fc54de7d06b..b9c4ba8a1c596 100644
--- a/core/src/test/java/org/elasticsearch/common/unit/SizeValueTests.java
+++ b/core/src/test/java/org/elasticsearch/common/unit/SizeValueTests.java
@@ -67,4 +67,37 @@ public void testThatNegativeValuesThrowException() {
             assertThat(e.getMessage(), containsString("may not be negative"));
         }
     }
+
+    public void testCompareEquality() {
+        long randomValue = randomNonNegativeLong();
+        SizeUnit randomUnit = randomFrom(SizeUnit.values());
+        SizeValue firstValue = new SizeValue(randomValue, randomUnit);
+        SizeValue secondValue = new SizeValue(randomValue, randomUnit);
+        assertEquals(0, firstValue.compareTo(secondValue));
+    }
+
+    public void testCompareValue() {
+        long firstRandom = randomNonNegativeLong();
+        long secondRandom = randomValueOtherThan(firstRandom, ESTestCase::randomNonNegativeLong);
+        SizeUnit unit = randomFrom(SizeUnit.values());
+        SizeValue firstSizeValue = new SizeValue(firstRandom, unit);
+        SizeValue secondSizeValue = new SizeValue(secondRandom, unit);
+        assertEquals(firstRandom > secondRandom, firstSizeValue.compareTo(secondSizeValue) > 0);
+        assertEquals(secondRandom > firstRandom, secondSizeValue.compareTo(firstSizeValue) > 0);
+    }
+
+    public void testCompareUnits() {
+        long number = randomNonNegativeLong();
+        SizeUnit randomUnit = randomValueOtherThan(SizeUnit.PETA, ()->randomFrom(SizeUnit.values()));
+        SizeValue firstValue = new SizeValue(number, randomUnit);
+        SizeValue secondValue = new SizeValue(number, SizeUnit.PETA);
+        assertTrue(firstValue.compareTo(secondValue) < 0);
+        assertTrue(secondValue.compareTo(firstValue) > 0);
+    }
+
+    public void testConversionHashCode() {
+        SizeValue firstValue = new SizeValue(randomIntBetween(0, Integer.MAX_VALUE), SizeUnit.GIGA);
+        SizeValue secondValue = new SizeValue(firstValue.getSingles(), SizeUnit.SINGLE);
+        assertEquals(firstValue.hashCode(), secondValue.hashCode());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java b/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java
index 4d0ac5257a31e..02394df3867e8 100644
--- a/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java
+++ b/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java
@@ -138,7 +138,7 @@ public void testRoundTrip() {
     private static final String FRACTIONAL_TIME_VALUES_ARE_NOT_SUPPORTED = "fractional time values are not supported";
 
     public void testNonFractionalTimeValues() {
-        final String s = randomAsciiOfLength(10) + randomTimeUnit();
+        final String s = randomAlphaOfLength(10) + randomTimeUnit();
         final ElasticsearchParseException e =
             expectThrows(ElasticsearchParseException.class, () -> TimeValue.parseTimeValue(s, null, "test"));
         assertThat(e, hasToString(containsString("failed to parse [" + s + "]")));
@@ -224,4 +224,37 @@ public void testToStringRep() {
         assertEquals("36h", new TimeValue(36, TimeUnit.HOURS).getStringRep());
         assertEquals("1000d", new TimeValue(1000, TimeUnit.DAYS).getStringRep());
     }
+
+    public void testCompareEquality() {
+        long randomLong = randomNonNegativeLong();
+        TimeUnit randomUnit = randomFrom(TimeUnit.values());
+        TimeValue firstValue = new TimeValue(randomLong, randomUnit);
+        TimeValue secondValue = new TimeValue(randomLong, randomUnit);
+        assertEquals(0, firstValue.compareTo(secondValue));
+    }
+
+    public void testCompareValue() {
+        long firstRandom = randomNonNegativeLong();
+        long secondRandom = randomValueOtherThan(firstRandom, ESTestCase::randomNonNegativeLong);
+        TimeUnit unit = randomFrom(TimeUnit.values());
+        TimeValue firstValue = new TimeValue(firstRandom, unit);
+        TimeValue secondValue = new TimeValue(secondRandom, unit);
+        assertEquals(firstRandom > secondRandom, firstValue.compareTo(secondValue) > 0);
+        assertEquals(secondRandom > firstRandom, secondValue.compareTo(firstValue) > 0);
+    }
+
+    public void testCompareUnits() {
+        long number = randomNonNegativeLong();
+        TimeUnit randomUnit = randomValueOtherThan(TimeUnit.DAYS, ()->randomFrom(TimeUnit.values()));
+        TimeValue firstValue = new TimeValue(number, randomUnit);
+        TimeValue secondValue = new TimeValue(number, TimeUnit.DAYS);
+        assertTrue(firstValue.compareTo(secondValue) < 0);
+        assertTrue(secondValue.compareTo(firstValue) > 0);
+    }
+
+    public void testConversionHashCode() {
+        TimeValue firstValue = new TimeValue(randomIntBetween(0, Integer.MAX_VALUE), TimeUnit.MINUTES);
+        TimeValue secondValue = new TimeValue(firstValue.getSeconds(), TimeUnit.SECONDS);
+        assertEquals(firstValue.hashCode(), secondValue.hashCode());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/common/util/BigArraysTests.java b/core/src/test/java/org/elasticsearch/common/util/BigArraysTests.java
index 6fadb6e550662..945dda446ce3a 100644
--- a/core/src/test/java/org/elasticsearch/common/util/BigArraysTests.java
+++ b/core/src/test/java/org/elasticsearch/common/util/BigArraysTests.java
@@ -27,14 +27,19 @@
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
-import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.util.Arrays;
+import java.util.List;
+import java.util.function.Function;
 
-public class BigArraysTests extends ESSingleNodeTestCase {
+import static org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+
+public class BigArraysTests extends ESTestCase {
 
     private BigArrays randombigArrays() {
         return new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService());
@@ -330,22 +335,17 @@ private ByteArray byteArrayWithBytes(byte[] bytes) {
     }
 
     public void testMaxSizeExceededOnNew() throws Exception {
-        final int size = scaledRandomIntBetween(5, 1 << 22);
-        for (String type : Arrays.asList("Byte", "Int", "Long", "Float", "Double", "Object")) {
-            HierarchyCircuitBreakerService hcbs = new HierarchyCircuitBreakerService(
-                    Settings.builder()
-                            .put(HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), size - 1, ByteSizeUnit.BYTES)
-                            .build(),
-                    new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
-            BigArrays bigArrays = new BigArrays(null, hcbs, false).withCircuitBreaking();
-            Method create = BigArrays.class.getMethod("new" + type + "Array", long.class);
+        final long size = scaledRandomIntBetween(5, 1 << 22);
+        final long maxSize = size - 1;
+        for (BigArraysHelper bigArraysHelper : bigArrayCreators(maxSize, true)) {
             try {
-                create.invoke(bigArrays, size);
-                fail("expected an exception on " + create);
-            } catch (InvocationTargetException e) {
-                assertTrue(e.getCause() instanceof CircuitBreakingException);
+                bigArraysHelper.arrayAllocator.apply(size);
+                fail("circuit breaker should trip");
+            } catch (CircuitBreakingException e) {
+                assertEquals(maxSize, e.getByteLimit());
+                assertThat(e.getBytesWanted(), greaterThanOrEqualTo(size));
             }
-            assertEquals(0, hcbs.getBreaker(CircuitBreaker.REQUEST).getUsed());
+            assertEquals(0, bigArraysHelper.bigArrays.breakerService().getBreaker(CircuitBreaker.REQUEST).getUsed());
         }
     }
 
@@ -354,7 +354,7 @@ public void testMaxSizeExceededOnResize() throws Exception {
             final long maxSize = randomIntBetween(1 << 10, 1 << 22);
             HierarchyCircuitBreakerService hcbs = new HierarchyCircuitBreakerService(
                     Settings.builder()
-                            .put(HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), maxSize, ByteSizeUnit.BYTES)
+                            .put(REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), maxSize, ByteSizeUnit.BYTES)
                             .build(),
                     new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
             BigArrays bigArrays = new BigArrays(null, hcbs, false).withCircuitBreaking();
@@ -377,4 +377,63 @@ public void testMaxSizeExceededOnResize() throws Exception {
         }
     }
 
+    public void testEstimatedBytesSameAsActualBytes() throws Exception {
+        final int maxSize = 1 << scaledRandomIntBetween(15, 22);
+        final long size = randomIntBetween((1 << 14) + 1, maxSize);
+        for (final BigArraysHelper bigArraysHelper : bigArrayCreators(maxSize, false)) {
+            final BigArray bigArray = bigArraysHelper.arrayAllocator.apply(size);
+            assertEquals(bigArraysHelper.ramEstimator.apply(size).longValue(), bigArray.ramBytesUsed());
+        }
+    }
+
+    private List<BigArraysHelper> bigArrayCreators(final long maxSize, final boolean withBreaking) {
+        final BigArrays byteBigArrays = newBigArraysInstance(maxSize, withBreaking);
+        BigArraysHelper byteHelper = new BigArraysHelper(byteBigArrays,
+            (Long size) -> byteBigArrays.newByteArray(size),
+            (Long size) -> BigByteArray.estimateRamBytes(size));
+        final BigArrays intBigArrays = newBigArraysInstance(maxSize, withBreaking);
+        BigArraysHelper intHelper = new BigArraysHelper(intBigArrays,
+            (Long size) -> intBigArrays.newIntArray(size),
+            (Long size) -> BigIntArray.estimateRamBytes(size));
+        final BigArrays longBigArrays = newBigArraysInstance(maxSize, withBreaking);
+        BigArraysHelper longHelper = new BigArraysHelper(longBigArrays,
+            (Long size) -> longBigArrays.newLongArray(size),
+            (Long size) -> BigLongArray.estimateRamBytes(size));
+        final BigArrays floatBigArrays = newBigArraysInstance(maxSize, withBreaking);
+        BigArraysHelper floatHelper = new BigArraysHelper(floatBigArrays,
+            (Long size) -> floatBigArrays.newFloatArray(size),
+            (Long size) -> BigFloatArray.estimateRamBytes(size));
+        final BigArrays doubleBigArrays = newBigArraysInstance(maxSize, withBreaking);
+        BigArraysHelper doubleHelper = new BigArraysHelper(doubleBigArrays,
+            (Long size) -> doubleBigArrays.newDoubleArray(size),
+            (Long size) -> BigDoubleArray.estimateRamBytes(size));
+        final BigArrays objectBigArrays = newBigArraysInstance(maxSize, withBreaking);
+        BigArraysHelper objectHelper = new BigArraysHelper(objectBigArrays,
+            (Long size) -> objectBigArrays.newObjectArray(size),
+            (Long size) -> BigObjectArray.estimateRamBytes(size));
+        return Arrays.asList(byteHelper, intHelper, longHelper, floatHelper, doubleHelper, objectHelper);
+    }
+
+    private BigArrays newBigArraysInstance(final long maxSize, final boolean withBreaking) {
+        HierarchyCircuitBreakerService hcbs = new HierarchyCircuitBreakerService(
+            Settings.builder()
+                .put(REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), maxSize, ByteSizeUnit.BYTES)
+                .build(),
+            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
+        BigArrays bigArrays = new BigArrays(null, hcbs, false);
+        return (withBreaking ? bigArrays.withCircuitBreaking() : bigArrays);
+    }
+
+    private static class BigArraysHelper {
+        final BigArrays bigArrays;
+        final Function<Long, BigArray> arrayAllocator;
+        final Function<Long, Long> ramEstimator;
+
+        BigArraysHelper(BigArrays bigArrays, Function<Long, BigArray> arrayAllocator, Function<Long, Long> ramEstimator) {
+            this.bigArrays = bigArrays;
+            this.arrayAllocator = arrayAllocator;
+            this.ramEstimator = ramEstimator;
+        }
+    }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/common/util/BytesRefHashTests.java b/core/src/test/java/org/elasticsearch/common/util/BytesRefHashTests.java
index 9f90fcf151d84..78abed0b32050 100644
--- a/core/src/test/java/org/elasticsearch/common/util/BytesRefHashTests.java
+++ b/core/src/test/java/org/elasticsearch/common/util/BytesRefHashTests.java
@@ -63,7 +63,7 @@ public void testDuell() {
         final int len = randomIntBetween(1, 100000);
         final BytesRef[] values = new BytesRef[len];
         for (int i = 0; i < values.length; ++i) {
-            values[i] = new BytesRef(randomAsciiOfLength(5));
+            values[i] = new BytesRef(randomAlphaOfLength(5));
         }
         final ObjectLongMap<BytesRef> valueToId = new ObjectLongHashMap<>();
         final BytesRef[] idToValue = new BytesRef[values.length];
diff --git a/core/src/test/java/org/elasticsearch/common/util/ExtensionPointTests.java b/core/src/test/java/org/elasticsearch/common/util/ExtensionPointTests.java
deleted file mode 100644
index 8fabbcc60ae8d..0000000000000
--- a/core/src/test/java/org/elasticsearch/common/util/ExtensionPointTests.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.common.util;
-
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.elasticsearch.common.inject.Binder;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.inject.Injector;
-import org.elasticsearch.common.inject.Module;
-import org.elasticsearch.common.inject.ModulesBuilder;
-import org.elasticsearch.test.ESTestCase;
-
-public class ExtensionPointTests extends ESTestCase {
-
-    public void testClassSet() {
-        final ExtensionPoint.ClassSet<TestBaseClass> allocationDeciders = new ExtensionPoint.ClassSet<>("test_class", TestBaseClass.class, Consumer.class);
-        allocationDeciders.registerExtension(TestImpl.class);
-        Injector injector = new ModulesBuilder().add(new Module() {
-            @Override
-            public void configure(Binder binder) {
-                allocationDeciders.bind(binder);
-            }
-        }).createInjector();
-        assertEquals(1, TestImpl.instances.get());
-
-    }
-
-    public static class TestBaseClass {}
-
-    public static class Consumer {
-        @Inject
-        public Consumer(Set<TestBaseClass> deciders, TestImpl other) {
-            // we require the TestImpl more than once to ensure it's bound as a singleton
-        }
-    }
-
-    public static class TestImpl extends TestBaseClass {
-        static final AtomicInteger instances = new AtomicInteger(0);
-
-        @Inject
-        public TestImpl() {
-            instances.incrementAndGet();
-        }
-    }
-}
diff --git a/core/src/test/java/org/elasticsearch/common/util/IndexFolderUpgraderTests.java b/core/src/test/java/org/elasticsearch/common/util/IndexFolderUpgraderTests.java
index 94fc0d8875229..bf10c117b1336 100644
--- a/core/src/test/java/org/elasticsearch/common/util/IndexFolderUpgraderTests.java
+++ b/core/src/test/java/org/elasticsearch/common/util/IndexFolderUpgraderTests.java
@@ -29,6 +29,7 @@
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.gateway.MetaDataStateFormat;
@@ -70,7 +71,7 @@ public void testUpgradeCustomDataPath() throws IOException {
             .put(NodeEnvironment.ADD_NODE_LOCK_ID_TO_CUSTOM_PATH.getKey(), randomBoolean())
             .put(Environment.PATH_SHARED_DATA_SETTING.getKey(), customPath.toAbsolutePath().toString()).build();
         try (NodeEnvironment nodeEnv = newNodeEnvironment(nodeSettings)) {
-            final Index index = new Index(randomAsciiOfLength(10), UUIDs.randomBase64UUID());
+            final Index index = new Index(randomAlphaOfLength(10), UUIDs.randomBase64UUID());
             Settings settings = Settings.builder()
                 .put(nodeSettings)
                 .put(IndexMetaData.SETTING_INDEX_UUID, index.getUUID())
@@ -99,7 +100,7 @@ public void testPartialUpgradeCustomDataPath() throws IOException {
             .put(NodeEnvironment.ADD_NODE_LOCK_ID_TO_CUSTOM_PATH.getKey(), randomBoolean())
             .put(Environment.PATH_SHARED_DATA_SETTING.getKey(), customPath.toAbsolutePath().toString()).build();
         try (NodeEnvironment nodeEnv = newNodeEnvironment(nodeSettings)) {
-            final Index index = new Index(randomAsciiOfLength(10), UUIDs.randomBase64UUID());
+            final Index index = new Index(randomAlphaOfLength(10), UUIDs.randomBase64UUID());
             Settings settings = Settings.builder()
                 .put(nodeSettings)
                 .put(IndexMetaData.SETTING_INDEX_UUID, index.getUUID())
@@ -138,7 +139,7 @@ public void testUpgrade() throws IOException {
         final Settings nodeSettings = Settings.builder()
             .put(NodeEnvironment.ADD_NODE_LOCK_ID_TO_CUSTOM_PATH.getKey(), randomBoolean()).build();
         try (NodeEnvironment nodeEnv = newNodeEnvironment(nodeSettings)) {
-            final Index index = new Index(randomAsciiOfLength(10), UUIDs.randomBase64UUID());
+            final Index index = new Index(randomAlphaOfLength(10), UUIDs.randomBase64UUID());
             Settings settings = Settings.builder()
                 .put(nodeSettings)
                 .put(IndexMetaData.SETTING_INDEX_UUID, index.getUUID())
@@ -163,7 +164,7 @@ public void testUpgradeIndices() throws IOException {
         try (NodeEnvironment nodeEnv = newNodeEnvironment(nodeSettings)) {
             Map<IndexSettings, Tuple<Integer, Integer>>  indexSettingsMap = new HashMap<>();
             for (int i = 0; i < randomIntBetween(2, 5); i++) {
-                final Index index = new Index(randomAsciiOfLength(10), UUIDs.randomBase64UUID());
+                final Index index = new Index(randomAlphaOfLength(10), UUIDs.randomBase64UUID());
                 Settings settings = Settings.builder()
                     .put(nodeSettings)
                     .put(IndexMetaData.SETTING_INDEX_UUID, index.getUUID())
@@ -211,11 +212,11 @@ public void testUpgradeRealIndex() throws IOException, URISyntaxException {
                 throw new IllegalStateException("Backwards index must contain exactly one cluster but was " + list.length);
             }
             // the bwc scripts packs the indices under this path
-            Path src = list[0].resolve("nodes/0/indices/" + indexName);
+            Path src = OldIndexUtils.getIndexDir(logger, indexName, path.getFileName().toString(), list[0]);
             assertTrue("[" + path + "] missing index dir: " + src.toString(), Files.exists(src));
             final Path indicesPath = randomFrom(nodeEnvironment.nodePaths()).indicesPath;
             logger.info("--> injecting index [{}] into [{}]", indexName, indicesPath);
-            OldIndexUtils.copyIndex(logger, src, indexName, indicesPath);
+            OldIndexUtils.copyIndex(logger, src, src.getFileName().toString(), indicesPath);
             IndexFolderUpgrader.upgradeIndicesIfNeeded(Settings.EMPTY, nodeEnvironment);
 
             // ensure old index folder is deleted
@@ -223,7 +224,7 @@ public void testUpgradeRealIndex() throws IOException, URISyntaxException {
             assertEquals(indexFolders.size(), 1);
 
             // ensure index metadata is moved
-            IndexMetaData indexMetaData = IndexMetaData.FORMAT.loadLatestState(logger,
+            IndexMetaData indexMetaData = IndexMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY,
                 nodeEnvironment.resolveIndexFolder(indexFolders.iterator().next()));
             assertNotNull(indexMetaData);
             Index index = indexMetaData.getIndex();
@@ -265,7 +266,8 @@ private void checkIndex(NodeEnvironment nodeEnv, IndexSettings indexSettings,
                             int numIdxFiles, int numTranslogFiles) throws IOException {
         final Index index = indexSettings.getIndex();
         // ensure index state can be loaded
-        IndexMetaData loadLatestState = IndexMetaData.FORMAT.loadLatestState(logger, nodeEnv.indexPaths(index));
+        IndexMetaData loadLatestState = IndexMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY,
+            nodeEnv.indexPaths(index));
         assertNotNull(loadLatestState);
         assertEquals(loadLatestState.getIndex(), index);
         for (int shardId = 0; shardId < indexSettings.getNumberOfShards(); shardId++) {
diff --git a/core/src/test/java/org/elasticsearch/common/util/concurrent/EsExecutorsTests.java b/core/src/test/java/org/elasticsearch/common/util/concurrent/EsExecutorsTests.java
index 72db2911fc023..142123bb48349 100644
--- a/core/src/test/java/org/elasticsearch/common/util/concurrent/EsExecutorsTests.java
+++ b/core/src/test/java/org/elasticsearch/common/util/concurrent/EsExecutorsTests.java
@@ -229,12 +229,9 @@ public void run() {
         assertThat("wrong pool size", pool.getPoolSize(), equalTo(max));
         assertThat("wrong active size", pool.getActiveCount(), equalTo(max));
         barrier.await();
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                assertThat("wrong active count", pool.getActiveCount(), equalTo(0));
-                assertThat("idle threads didn't shrink below max. (" + pool.getPoolSize() + ")", pool.getPoolSize(), lessThan(max));
-            }
+        assertBusy(() -> {
+            assertThat("wrong active count", pool.getActiveCount(), equalTo(0));
+            assertThat("idle threads didn't shrink below max. (" + pool.getPoolSize() + ")", pool.getPoolSize(), lessThan(max));
         });
         terminate(pool);
     }
diff --git a/core/src/test/java/org/elasticsearch/common/util/concurrent/PrioritizedExecutorsTests.java b/core/src/test/java/org/elasticsearch/common/util/concurrent/PrioritizedExecutorsTests.java
index 933a46de510dc..17b43a079dc53 100644
--- a/core/src/test/java/org/elasticsearch/common/util/concurrent/PrioritizedExecutorsTests.java
+++ b/core/src/test/java/org/elasticsearch/common/util/concurrent/PrioritizedExecutorsTests.java
@@ -65,7 +65,7 @@ public void testPriorityQueue() throws Exception {
     }
 
     public void testSubmitPrioritizedExecutorWithRunnables() throws Exception {
-        ExecutorService executor = EsExecutors.newSinglePrioritizing(getTestName(), EsExecutors.daemonThreadFactory(getTestName()), holder);
+        ExecutorService executor = EsExecutors.newSinglePrioritizing(getTestName(), EsExecutors.daemonThreadFactory(getTestName()), holder, null);
         List<Integer> results = new ArrayList<>(8);
         CountDownLatch awaitingLatch = new CountDownLatch(1);
         CountDownLatch finishedLatch = new CountDownLatch(8);
@@ -94,7 +94,7 @@ public void testSubmitPrioritizedExecutorWithRunnables() throws Exception {
     }
 
     public void testExecutePrioritizedExecutorWithRunnables() throws Exception {
-        ExecutorService executor = EsExecutors.newSinglePrioritizing(getTestName(), EsExecutors.daemonThreadFactory(getTestName()), holder);
+        ExecutorService executor = EsExecutors.newSinglePrioritizing(getTestName(), EsExecutors.daemonThreadFactory(getTestName()), holder, null);
         List<Integer> results = new ArrayList<>(8);
         CountDownLatch awaitingLatch = new CountDownLatch(1);
         CountDownLatch finishedLatch = new CountDownLatch(8);
@@ -123,7 +123,7 @@ public void testExecutePrioritizedExecutorWithRunnables() throws Exception {
     }
 
     public void testSubmitPrioritizedExecutorWithCallables() throws Exception {
-        ExecutorService executor = EsExecutors.newSinglePrioritizing(getTestName(), EsExecutors.daemonThreadFactory(getTestName()), holder);
+        ExecutorService executor = EsExecutors.newSinglePrioritizing(getTestName(), EsExecutors.daemonThreadFactory(getTestName()), holder, null);
         List<Integer> results = new ArrayList<>(8);
         CountDownLatch awaitingLatch = new CountDownLatch(1);
         CountDownLatch finishedLatch = new CountDownLatch(8);
@@ -152,7 +152,7 @@ public void testSubmitPrioritizedExecutorWithCallables() throws Exception {
     }
 
     public void testSubmitPrioritizedExecutorWithMixed() throws Exception {
-        ExecutorService executor = EsExecutors.newSinglePrioritizing(getTestName(), EsExecutors.daemonThreadFactory(getTestName()), holder);
+        ExecutorService executor = EsExecutors.newSinglePrioritizing(getTestName(), EsExecutors.daemonThreadFactory(getTestName()), holder, null);
         List<Integer> results = new ArrayList<>(8);
         CountDownLatch awaitingLatch = new CountDownLatch(1);
         CountDownLatch finishedLatch = new CountDownLatch(8);
@@ -182,7 +182,7 @@ public void testSubmitPrioritizedExecutorWithMixed() throws Exception {
 
     public void testTimeout() throws Exception {
         ScheduledExecutorService timer = Executors.newSingleThreadScheduledExecutor(EsExecutors.daemonThreadFactory(getTestName()));
-        PrioritizedEsThreadPoolExecutor executor = EsExecutors.newSinglePrioritizing(getTestName(), EsExecutors.daemonThreadFactory(getTestName()), holder);
+        PrioritizedEsThreadPoolExecutor executor = EsExecutors.newSinglePrioritizing(getTestName(), EsExecutors.daemonThreadFactory(getTestName()), holder, timer);
         final CountDownLatch invoked = new CountDownLatch(1);
         final CountDownLatch block = new CountDownLatch(1);
         executor.execute(new Runnable() {
@@ -219,7 +219,7 @@ public void run() {
                              public String toString() {
                                  return "the waiting";
                              }
-                         }, timer, TimeValue.timeValueMillis(100) /* enough timeout to catch them in the pending list... */, new Runnable() {
+                         }, TimeValue.timeValueMillis(100) /* enough timeout to catch them in the pending list... */, new Runnable() {
                     @Override
                     public void run() {
                         timedOut.countDown();
@@ -245,14 +245,14 @@ public void testTimeoutCleanup() throws Exception {
         ThreadPool threadPool = new TestThreadPool("test");
         final ScheduledThreadPoolExecutor timer = (ScheduledThreadPoolExecutor) threadPool.scheduler();
         final AtomicBoolean timeoutCalled = new AtomicBoolean();
-        PrioritizedEsThreadPoolExecutor executor = EsExecutors.newSinglePrioritizing(getTestName(), EsExecutors.daemonThreadFactory(getTestName()), holder);
+        PrioritizedEsThreadPoolExecutor executor = EsExecutors.newSinglePrioritizing(getTestName(), EsExecutors.daemonThreadFactory(getTestName()), holder, timer);
         final CountDownLatch invoked = new CountDownLatch(1);
         executor.execute(new Runnable() {
                              @Override
                              public void run() {
                                  invoked.countDown();
                              }
-                         }, timer, TimeValue.timeValueHours(1), new Runnable() {
+                         }, TimeValue.timeValueHours(1), new Runnable() {
                     @Override
                     public void run() {
                         // We should never get here
@@ -264,12 +264,7 @@ public void run() {
 
         // the timeout handler is added post execution (and quickly cancelled). We have allow for this
         // and use assert busy
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                assertThat(timer.getQueue().size(), equalTo(0));
-            }
-        }, 5, TimeUnit.SECONDS);
+        assertBusy(() -> assertThat(timer.getQueue().size(), equalTo(0)), 5, TimeUnit.SECONDS);
         assertThat(timeoutCalled.get(), equalTo(false));
         assertTrue(terminate(executor));
         assertTrue(terminate(threadPool));
diff --git a/core/src/test/java/org/elasticsearch/common/util/concurrent/RefCountedTests.java b/core/src/test/java/org/elasticsearch/common/util/concurrent/RefCountedTests.java
index c5d0ec4257ee6..30182adb932b6 100644
--- a/core/src/test/java/org/elasticsearch/common/util/concurrent/RefCountedTests.java
+++ b/core/src/test/java/org/elasticsearch/common/util/concurrent/RefCountedTests.java
@@ -130,7 +130,7 @@ private final class MyRefCounted extends AbstractRefCounted {
 
         private final AtomicBoolean closed = new AtomicBoolean(false);
 
-        public MyRefCounted() {
+        MyRefCounted() {
             super("test");
         }
 
diff --git a/core/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java b/core/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java
index d402f09f07ddd..bee56c229c02a 100644
--- a/core/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java
+++ b/core/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java
@@ -19,6 +19,7 @@
 package org.elasticsearch.common.util.concurrent;
 
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
 
@@ -27,6 +28,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Supplier;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasItem;
@@ -86,7 +88,7 @@ public void testStoreContext() {
         assertEquals("bar", threadContext.getHeader("foo"));
         assertEquals(Integer.valueOf(1), threadContext.getTransient("ctx.foo"));
         assertEquals("1", threadContext.getHeader("default"));
-        ThreadContext.StoredContext storedContext = threadContext.newStoredContext();
+        ThreadContext.StoredContext storedContext = threadContext.newStoredContext(false);
         threadContext.putHeader("foo.bar", "baz");
         try (ThreadContext.StoredContext ctx = threadContext.stashContext()) {
             assertNull(threadContext.getHeader("foo"));
@@ -109,6 +111,63 @@ public void testStoreContext() {
         assertNull(threadContext.getHeader("foo.bar"));
     }
 
+    public void testRestorableContext() {
+        Settings build = Settings.builder().put("request.headers.default", "1").build();
+        ThreadContext threadContext = new ThreadContext(build);
+        threadContext.putHeader("foo", "bar");
+        threadContext.putTransient("ctx.foo", 1);
+        threadContext.addResponseHeader("resp.header", "baaaam");
+        Supplier<ThreadContext.StoredContext> contextSupplier = threadContext.newRestorableContext(true);
+
+        try (ThreadContext.StoredContext ctx = threadContext.stashContext()) {
+            assertNull(threadContext.getHeader("foo"));
+            assertEquals("1", threadContext.getHeader("default"));
+            threadContext.addResponseHeader("resp.header", "boom");
+            try (ThreadContext.StoredContext tmp = contextSupplier.get()) {
+                assertEquals("bar", threadContext.getHeader("foo"));
+                assertEquals(Integer.valueOf(1), threadContext.getTransient("ctx.foo"));
+                assertEquals("1", threadContext.getHeader("default"));
+                assertEquals(2, threadContext.getResponseHeaders().get("resp.header").size());
+                assertEquals("boom", threadContext.getResponseHeaders().get("resp.header").get(0));
+                assertEquals("baaaam", threadContext.getResponseHeaders().get("resp.header").get(1));
+            }
+            assertNull(threadContext.getHeader("foo"));
+            assertNull(threadContext.getTransient("ctx.foo"));
+            assertEquals(1, threadContext.getResponseHeaders().get("resp.header").size());
+            assertEquals("boom", threadContext.getResponseHeaders().get("resp.header").get(0));
+        }
+        assertEquals("bar", threadContext.getHeader("foo"));
+        assertEquals(Integer.valueOf(1), threadContext.getTransient("ctx.foo"));
+        assertEquals("1", threadContext.getHeader("default"));
+        assertEquals(1, threadContext.getResponseHeaders().get("resp.header").size());
+        assertEquals("baaaam", threadContext.getResponseHeaders().get("resp.header").get(0));
+
+        contextSupplier = threadContext.newRestorableContext(false);
+
+        try (ThreadContext.StoredContext ctx = threadContext.stashContext()) {
+            assertNull(threadContext.getHeader("foo"));
+            assertEquals("1", threadContext.getHeader("default"));
+            threadContext.addResponseHeader("resp.header", "boom");
+            try (ThreadContext.StoredContext tmp = contextSupplier.get()) {
+                assertEquals("bar", threadContext.getHeader("foo"));
+                assertEquals(Integer.valueOf(1), threadContext.getTransient("ctx.foo"));
+                assertEquals("1", threadContext.getHeader("default"));
+                assertEquals(1, threadContext.getResponseHeaders().get("resp.header").size());
+                assertEquals("baaaam", threadContext.getResponseHeaders().get("resp.header").get(0));
+            }
+            assertNull(threadContext.getHeader("foo"));
+            assertNull(threadContext.getTransient("ctx.foo"));
+            assertEquals(1, threadContext.getResponseHeaders().get("resp.header").size());
+            assertEquals("boom", threadContext.getResponseHeaders().get("resp.header").get(0));
+        }
+
+        assertEquals("bar", threadContext.getHeader("foo"));
+        assertEquals(Integer.valueOf(1), threadContext.getTransient("ctx.foo"));
+        assertEquals("1", threadContext.getHeader("default"));
+        assertEquals(1, threadContext.getResponseHeaders().get("resp.header").size());
+        assertEquals("baaaam", threadContext.getResponseHeaders().get("resp.header").get(0));
+    }
+
     public void testResponseHeaders() {
         final boolean expectThird = randomBoolean();
 
@@ -120,6 +179,14 @@ public void testResponseHeaders() {
             threadContext.addResponseHeader("foo", "bar");
         }
 
+        final String value = DeprecationLogger.formatWarning("qux");
+        threadContext.addResponseHeader("baz", value, DeprecationLogger::extractWarningValueFromWarningHeader);
+        // pretend that another thread created the same response at a different time
+        if (randomBoolean()) {
+            final String duplicateValue = DeprecationLogger.formatWarning("qux");
+            threadContext.addResponseHeader("baz", duplicateValue, DeprecationLogger::extractWarningValueFromWarningHeader);
+        }
+
         threadContext.addResponseHeader("Warning", "One is the loneliest number");
         threadContext.addResponseHeader("Warning", "Two can be as bad as one");
         if (expectThird) {
@@ -128,11 +195,14 @@ public void testResponseHeaders() {
 
         final Map<String, List<String>> responseHeaders = threadContext.getResponseHeaders();
         final List<String> foo = responseHeaders.get("foo");
+        final List<String> baz = responseHeaders.get("baz");
         final List<String> warnings = responseHeaders.get("Warning");
         final int expectedWarnings = expectThird ? 3 : 2;
 
         assertThat(foo, hasSize(1));
+        assertThat(baz, hasSize(1));
         assertEquals("bar", foo.get(0));
+        assertEquals(value, baz.get(0));
         assertThat(warnings, hasSize(expectedWarnings));
         assertThat(warnings, hasItem(equalTo("One is the loneliest number")));
         assertThat(warnings, hasItem(equalTo("Two can be as bad as one")));
@@ -319,6 +389,9 @@ public void testPreserveContext() throws IOException {
 
             // But we do inside of it
             withContext.run();
+
+            // but not after
+            assertNull(threadContext.getHeader("foo"));
         }
     }
 
@@ -350,6 +423,196 @@ public void testPreserveContextKeepsOriginalContextWhenCalledTwice() throws IOEx
         }
     }
 
+    public void testPreservesThreadsOriginalContextOnRunException() throws IOException {
+        try (ThreadContext threadContext = new ThreadContext(Settings.EMPTY)) {
+            Runnable withContext;
+
+            // create a abstract runnable, add headers and transient objects and verify in the methods
+            try (ThreadContext.StoredContext ignored = threadContext.stashContext()) {
+                threadContext.putHeader("foo", "bar");
+                boolean systemContext = randomBoolean();
+                if (systemContext) {
+                    threadContext.markAsSystemContext();
+                }
+                threadContext.putTransient("foo", "bar_transient");
+                withContext = threadContext.preserveContext(new AbstractRunnable() {
+
+                    @Override
+                    public void onAfter() {
+                        assertEquals(systemContext, threadContext.isSystemContext());
+                        assertEquals("bar", threadContext.getHeader("foo"));
+                        assertEquals("bar_transient", threadContext.getTransient("foo"));
+                        assertNotNull(threadContext.getTransient("failure"));
+                        assertEquals("exception from doRun", ((RuntimeException)threadContext.getTransient("failure")).getMessage());
+                        assertFalse(threadContext.isDefaultContext());
+                        threadContext.putTransient("after", "after");
+                    }
+
+                    @Override
+                    public void onFailure(Exception e) {
+                        assertEquals(systemContext, threadContext.isSystemContext());
+                        assertEquals("exception from doRun", e.getMessage());
+                        assertEquals("bar", threadContext.getHeader("foo"));
+                        assertEquals("bar_transient", threadContext.getTransient("foo"));
+                        assertFalse(threadContext.isDefaultContext());
+                        threadContext.putTransient("failure", e);
+                    }
+
+                    @Override
+                    protected void doRun() throws Exception {
+                        assertEquals(systemContext, threadContext.isSystemContext());
+                        assertEquals("bar", threadContext.getHeader("foo"));
+                        assertEquals("bar_transient", threadContext.getTransient("foo"));
+                        assertFalse(threadContext.isDefaultContext());
+                        throw new RuntimeException("exception from doRun");
+                    }
+                });
+            }
+
+            // We don't see the header outside of the runnable
+            assertNull(threadContext.getHeader("foo"));
+            assertNull(threadContext.getTransient("foo"));
+            assertNull(threadContext.getTransient("failure"));
+            assertNull(threadContext.getTransient("after"));
+            assertTrue(threadContext.isDefaultContext());
+
+            // But we do inside of it
+            withContext.run();
+
+            // verify not seen after
+            assertNull(threadContext.getHeader("foo"));
+            assertNull(threadContext.getTransient("foo"));
+            assertNull(threadContext.getTransient("failure"));
+            assertNull(threadContext.getTransient("after"));
+            assertTrue(threadContext.isDefaultContext());
+
+            // repeat with regular runnable
+            try (ThreadContext.StoredContext ignored = threadContext.stashContext()) {
+                threadContext.putHeader("foo", "bar");
+                threadContext.putTransient("foo", "bar_transient");
+                withContext = threadContext.preserveContext(() -> {
+                    assertEquals("bar", threadContext.getHeader("foo"));
+                    assertEquals("bar_transient", threadContext.getTransient("foo"));
+                    assertFalse(threadContext.isDefaultContext());
+                    threadContext.putTransient("run", true);
+                    throw new RuntimeException("exception from run");
+                });
+            }
+
+            assertNull(threadContext.getHeader("foo"));
+            assertNull(threadContext.getTransient("foo"));
+            assertNull(threadContext.getTransient("run"));
+            assertTrue(threadContext.isDefaultContext());
+
+            final Runnable runnable = withContext;
+            RuntimeException e = expectThrows(RuntimeException.class, runnable::run);
+            assertEquals("exception from run", e.getMessage());
+            assertNull(threadContext.getHeader("foo"));
+            assertNull(threadContext.getTransient("foo"));
+            assertNull(threadContext.getTransient("run"));
+            assertTrue(threadContext.isDefaultContext());
+        }
+    }
+
+    public void testPreservesThreadsOriginalContextOnFailureException() throws IOException {
+        try (ThreadContext threadContext = new ThreadContext(Settings.EMPTY)) {
+            Runnable withContext;
+
+            // a runnable that throws from onFailure
+            try (ThreadContext.StoredContext ignored = threadContext.stashContext()) {
+                threadContext.putHeader("foo", "bar");
+                threadContext.putTransient("foo", "bar_transient");
+                withContext = threadContext.preserveContext(new AbstractRunnable() {
+                    @Override
+                    public void onFailure(Exception e) {
+                        throw new RuntimeException("from onFailure", e);
+                    }
+
+                    @Override
+                    protected void doRun() throws Exception {
+                        assertEquals("bar", threadContext.getHeader("foo"));
+                        assertEquals("bar_transient", threadContext.getTransient("foo"));
+                        assertFalse(threadContext.isDefaultContext());
+                        throw new RuntimeException("from doRun");
+                    }
+                });
+            }
+
+            // We don't see the header outside of the runnable
+            assertNull(threadContext.getHeader("foo"));
+            assertNull(threadContext.getTransient("foo"));
+            assertTrue(threadContext.isDefaultContext());
+
+            // But we do inside of it
+            RuntimeException e = expectThrows(RuntimeException.class, withContext::run);
+            assertEquals("from onFailure", e.getMessage());
+            assertEquals("from doRun", e.getCause().getMessage());
+
+            // but not after
+            assertNull(threadContext.getHeader("foo"));
+            assertNull(threadContext.getTransient("foo"));
+            assertTrue(threadContext.isDefaultContext());
+        }
+    }
+
+    public void testPreservesThreadsOriginalContextOnAfterException() throws IOException {
+        try (ThreadContext threadContext = new ThreadContext(Settings.EMPTY)) {
+            Runnable withContext;
+
+            // a runnable that throws from onAfter
+            try (ThreadContext.StoredContext ignored = threadContext.stashContext()) {
+                threadContext.putHeader("foo", "bar");
+                threadContext.putTransient("foo", "bar_transient");
+                withContext = threadContext.preserveContext(new AbstractRunnable() {
+
+                    @Override
+                    public void onAfter() {
+                        throw new RuntimeException("from onAfter");
+                    }
+
+                    @Override
+                    public void onFailure(Exception e) {
+                        throw new RuntimeException("from onFailure", e);
+                    }
+
+                    @Override
+                    protected void doRun() throws Exception {
+                        assertEquals("bar", threadContext.getHeader("foo"));
+                        assertEquals("bar_transient", threadContext.getTransient("foo"));
+                        assertFalse(threadContext.isDefaultContext());
+                    }
+                });
+            }
+
+            // We don't see the header outside of the runnable
+            assertNull(threadContext.getHeader("foo"));
+            assertNull(threadContext.getTransient("foo"));
+            assertTrue(threadContext.isDefaultContext());
+
+            // But we do inside of it
+            RuntimeException e = expectThrows(RuntimeException.class, withContext::run);
+            assertEquals("from onAfter", e.getMessage());
+            assertNull(e.getCause());
+
+            // but not after
+            assertNull(threadContext.getHeader("foo"));
+            assertNull(threadContext.getTransient("foo"));
+            assertTrue(threadContext.isDefaultContext());
+        }
+    }
+
+    public void testMarkAsSystemContext() throws IOException {
+        try (ThreadContext threadContext = new ThreadContext(Settings.EMPTY)) {
+            assertFalse(threadContext.isSystemContext());
+            try(ThreadContext.StoredContext context = threadContext.stashContext()){
+                assertFalse(threadContext.isSystemContext());
+                threadContext.markAsSystemContext();
+                assertTrue(threadContext.isSystemContext());
+            }
+            assertFalse(threadContext.isSystemContext());
+        }
+    }
+
     /**
      * Sometimes wraps a Runnable in an AbstractRunnable.
      */
diff --git a/core/src/test/java/org/elasticsearch/common/util/set/SetsTests.java b/core/src/test/java/org/elasticsearch/common/util/set/SetsTests.java
new file mode 100644
index 0000000000000..0c1869a6b4086
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/common/util/set/SetsTests.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.util.set;
+
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+
+public class SetsTests extends ESTestCase {
+
+    public void testDifference() {
+        final int endExclusive = randomIntBetween(0, 256);
+        final Tuple<Set<Integer>, Set<Integer>> sets = randomSets(endExclusive);
+        final Set<Integer> difference = Sets.difference(sets.v1(), sets.v2());
+        assertDifference(endExclusive, sets, difference);
+    }
+
+    public void testSortedDifference() {
+        final int endExclusive = randomIntBetween(0, 256);
+        final Tuple<Set<Integer>, Set<Integer>> sets = randomSets(endExclusive);
+        final Set<Integer> difference = Sets.sortedDifference(sets.v1(), sets.v2());
+        assertDifference(endExclusive, sets, difference);
+        final Iterator<Integer> it = difference.iterator();
+        if (it.hasNext()) {
+            int current = it.next();
+            while (it.hasNext()) {
+                final int next = it.next();
+                assertThat(next, greaterThan(current));
+                current = next;
+            }
+        }
+    }
+
+    /**
+     * Assert the difference between two sets is as expected.
+     *
+     * @param endExclusive the exclusive upper bound of the elements of either set
+     * @param sets         a pair of sets with elements from {@code [0, endExclusive)}
+     * @param difference   the difference between the two sets
+     */
+    private void assertDifference(
+            final int endExclusive, final Tuple<Set<Integer>, Set<Integer>> sets, final Set<Integer> difference) {
+        for (int i = 0; i < endExclusive; i++) {
+            assertThat(difference.contains(i), equalTo(sets.v1().contains(i) && !sets.v2().contains(i)));
+        }
+    }
+
+    /**
+     * Produces two random sets consisting of elements from {@code [0, endExclusive)}.
+     *
+     * @param endExclusive the exclusive upper bound of the elements of the sets
+     * @return a pair of sets
+     */
+    private Tuple<Set<Integer>, Set<Integer>> randomSets(final int endExclusive) {
+        final Set<Integer> left = new HashSet<>(randomSubsetOf(IntStream.range(0, endExclusive).boxed().collect(Collectors.toSet())));
+        final Set<Integer> right = new HashSet<>(randomSubsetOf(IntStream.range(0, endExclusive).boxed().collect(Collectors.toSet())));
+        return Tuple.tuple(left, right);
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java b/core/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java
index a602e81b85400..eb0adc1990c8f 100644
--- a/core/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java
+++ b/core/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java
@@ -19,23 +19,675 @@
 
 package org.elasticsearch.common.xcontent;
 
+import com.fasterxml.jackson.core.JsonGenerationException;
 import com.fasterxml.jackson.core.JsonGenerator;
 
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.Constants;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.geo.GeoPoint;
+import org.elasticsearch.common.io.PathUtils;
+import org.elasticsearch.common.text.Text;
+import org.elasticsearch.common.unit.DistanceUnit;
 import org.elasticsearch.common.xcontent.XContentParser.Token;
 import org.elasticsearch.test.ESTestCase;
+import org.hamcrest.Matcher;
+import org.hamcrest.Matchers;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+import org.joda.time.Instant;
+import org.joda.time.ReadableInstant;
+import org.joda.time.format.DateTimeFormatter;
+import org.joda.time.format.ISODateTimeFormat;
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
-import java.math.BigDecimal;
 import java.math.BigInteger;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Calendar;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.singletonMap;
+import static org.hamcrest.Matchers.allOf;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.notNullValue;
 
 public abstract class BaseXContentTestCase extends ESTestCase {
 
-    public abstract XContentType xcontentType();
+    protected abstract XContentType xcontentType();
+
+    private XContentBuilder builder() throws IOException {
+        return XContentBuilder.builder(xcontentType().xContent());
+    }
+
+    public void testContentType() throws IOException {
+        assertThat(builder().contentType(), equalTo(xcontentType()));
+    }
+
+    public void testStartEndObject() throws IOException {
+        expectUnclosedException(() -> builder().startObject().bytes());
+        expectUnclosedException(() -> builder().startObject().close());
+        expectUnclosedException(() -> builder().startObject().string());
+
+        expectObjectException(() -> builder().endObject().bytes());
+        expectObjectException(() -> builder().endObject().close());
+        expectObjectException(() -> builder().endObject().string());
+
+        expectValueException(() -> builder().startObject("foo").endObject());
+        expectNonNullFieldException(() -> builder().startObject().startObject(null));
+
+        assertResult("{}", () -> builder().startObject().endObject());
+        assertResult("{'foo':{}}", () -> builder().startObject().startObject("foo").endObject().endObject());
+
+        assertResult("{'foo':{'bar':{}}}", () -> builder()
+                .startObject()
+                    .startObject("foo")
+                        .startObject("bar")
+                        .endObject()
+                    .endObject()
+                .endObject());
+    }
+
+    public void testStartEndArray() throws IOException {
+        expectUnclosedException(() -> builder().startArray().bytes());
+        expectUnclosedException(() -> builder().startArray().close());
+        expectUnclosedException(() -> builder().startArray().string());
+
+        expectArrayException(() -> builder().endArray().bytes());
+        expectArrayException(() -> builder().endArray().close());
+        expectArrayException(() -> builder().endArray().string());
+
+        expectValueException(() -> builder().startArray("foo").endObject());
+        expectFieldException(() -> builder().startObject().startArray().endArray().endObject());
+        expectNonNullFieldException(() -> builder().startObject().startArray(null).endArray().endObject());
+
+        assertResult("{'foo':[]}", () -> builder().startObject().startArray("foo").endArray().endObject());
+        assertResult("{'foo':[1,2,3]}", () -> builder()
+                .startObject()
+                    .startArray("foo")
+                        .value(1)
+                        .value(2)
+                        .value(3)
+                    .endArray()
+                .endObject());
+    }
+
+    public void testField() throws IOException {
+        expectValueException(() -> builder().field("foo").bytes());
+        expectNonNullFieldException(() -> builder().field(null).bytes());
+        expectUnclosedException(() -> builder().startObject().field("foo").bytes());
+
+        assertResult("{'foo':'bar'}", () -> builder().startObject().field("foo").value("bar").endObject());
+    }
+
+    public void testNullField() throws IOException {
+        expectValueException(() -> builder().nullField("foo").bytes());
+        expectNonNullFieldException(() -> builder().nullField(null).bytes());
+        expectUnclosedException(() -> builder().startObject().nullField("foo").bytes());
+
+        assertResult("{'foo':null}", () -> builder().startObject().nullField("foo").endObject());
+    }
+
+    public void testNullValue() throws IOException {
+        assertResult("{'foo':null}", () -> builder().startObject().field("foo").nullValue().endObject());
+    }
+
+    public void testBooleans() throws IOException {
+        assertResult("{'boolean':null}", () -> builder().startObject().field("boolean", (Boolean) null).endObject());
+        assertResult("{'boolean':true}", () -> builder().startObject().field("boolean", Boolean.TRUE).endObject());
+        assertResult("{'boolean':false}", () -> builder().startObject().field("boolean", Boolean.FALSE).endObject());
+        assertResult("{'boolean':[true,false,true]}", () -> builder().startObject().array("boolean", true, false, true).endObject());
+        assertResult("{'boolean':[false,true]}", () -> builder().startObject().array("boolean", new boolean[]{false, true}).endObject());
+        assertResult("{'boolean':null}", () -> builder().startObject().array("boolean", (boolean[]) null).endObject());
+        assertResult("{'boolean':[]}", () -> builder().startObject().array("boolean", new boolean[]{}).endObject());
+        assertResult("{'boolean':null}", () -> builder().startObject().field("boolean").value((Boolean) null).endObject());
+        assertResult("{'boolean':true}", () -> builder().startObject().field("boolean").value(Boolean.TRUE).endObject());
+        assertResult("{'boolean':false}", () -> builder().startObject().field("boolean").value(Boolean.FALSE).endObject());
+    }
+
+    public void testBytes() throws IOException {
+        assertResult("{'byte':null}", () -> builder().startObject().field("byte", (Byte) null).endObject());
+        assertResult("{'byte':0}", () -> builder().startObject().field("byte", (byte) 0).endObject());
+        assertResult("{'byte':1}", () -> builder().startObject().field("byte", (byte) 1).endObject());
+        assertResult("{'byte':null}", () -> builder().startObject().field("byte").value((Byte) null).endObject());
+        assertResult("{'byte':0}", () -> builder().startObject().field("byte").value((byte) 0).endObject());
+        assertResult("{'byte':1}", () -> builder().startObject().field("byte").value((byte) 1).endObject());
+    }
+
+    public void testDoubles() throws IOException {
+        assertResult("{'double':null}", () -> builder().startObject().field("double", (Double) null).endObject());
+        assertResult("{'double':42.5}", () -> builder().startObject().field("double", Double.valueOf(42.5)).endObject());
+        assertResult("{'double':1.2}", () -> builder().startObject().field("double", 1.2).endObject());
+        assertResult("{'double':[42.0,43.0,45]}", () -> builder().startObject().array("double", 42.0, 43.0, 45).endObject());
+        assertResult("{'double':null}", () -> builder().startObject().array("double", (double[]) null).endObject());
+        assertResult("{'double':[]}", () -> builder().startObject().array("double", new double[]{}).endObject());
+        assertResult("{'double':null}", () -> builder().startObject().field("double").value((Double) null).endObject());
+        assertResult("{'double':0.001}", () -> builder().startObject().field("double").value(0.001).endObject());
+        assertResult("{'double':[1.7976931348623157E308,4.9E-324]}", () -> builder()
+                .startObject()
+                .array("double", new double[]{Double.MAX_VALUE, Double.MIN_VALUE})
+                .endObject());
+    }
+
+    public void testFloats() throws IOException {
+        assertResult("{'float':null}", () -> builder().startObject().field("float", (Float) null).endObject());
+        assertResult("{'float':42.5}", () -> builder().startObject().field("float", Float.valueOf(42.5f)).endObject());
+        assertResult("{'float':1.2}", () -> builder().startObject().field("float", 1.2f).endObject());
+        assertResult("{'float':null}", () -> builder().startObject().array("float", (float[]) null).endObject());
+        assertResult("{'float':[]}", () -> builder().startObject().array("float", new float[]{}).endObject());
+        assertResult("{'float':null}", () -> builder().startObject().field("float").value((Float) null).endObject());
+        assertResult("{'float':9.9E-7}", () -> builder().startObject().field("float").value(0.00000099f).endObject());
+        assertResult("{'float':[42.0,43.0,45.666668]}", () -> builder()
+                .startObject()
+                    .array("float", 42.0f, 43.0f, 45.66666667f)
+                .endObject());
+        assertResult("{'float':[3.4028235E38,1.4E-45]}", () -> builder()
+                .startObject()
+                    .array("float", new float[]{Float.MAX_VALUE, Float.MIN_VALUE})
+                .endObject());
+    }
+
+    public void testIntegers() throws IOException {
+        assertResult("{'integer':null}", () -> builder().startObject().field("integer", (Integer) null).endObject());
+        assertResult("{'integer':42}", () -> builder().startObject().field("integer", Integer.valueOf(42)).endObject());
+        assertResult("{'integer':3}", () -> builder().startObject().field("integer", 3).endObject());
+        assertResult("{'integer':[1,3,5,7,11]}", () -> builder().startObject().array("integer", 1, 3, 5, 7, 11).endObject());
+        assertResult("{'integer':null}", () -> builder().startObject().array("integer", (int[]) null).endObject());
+        assertResult("{'integer':[]}", () -> builder().startObject().array("integer", new int[]{}).endObject());
+        assertResult("{'integer':null}", () -> builder().startObject().field("integer").value((Integer) null).endObject());
+        assertResult("{'integer':42}", () -> builder().startObject().field("integer").value(42).endObject());
+        assertResult("{'integer':[2147483647,-2147483648]}", () -> builder()
+                .startObject()
+                    .array("integer", new int[]{Integer.MAX_VALUE, Integer.MIN_VALUE})
+                .endObject());
+    }
+
+    public void testLongs() throws IOException {
+        assertResult("{'long':null}", () -> builder().startObject().field("long", (Long) null).endObject());
+        assertResult("{'long':42}", () -> builder().startObject().field("long", Long.valueOf(42L)).endObject());
+        assertResult("{'long':9223372036854775807}", () -> builder().startObject().field("long", 9_223_372_036_854_775_807L).endObject());
+        assertResult("{'long':[1,3,5,7,11]}", () -> builder().startObject().array("long", 1L, 3L, 5L, 7L, 11L).endObject());
+        assertResult("{'long':null}", () -> builder().startObject().array("long", (long[]) null).endObject());
+        assertResult("{'long':[]}", () -> builder().startObject().array("long", new long[]{}).endObject());
+        assertResult("{'long':null}", () -> builder().startObject().field("long").value((Long) null).endObject());
+        assertResult("{'long':42}", () -> builder().startObject().field("long").value(42).endObject());
+        assertResult("{'long':[2147483647,-2147483648]}", () -> builder()
+                .startObject()
+                    .array("long", new long[]{Integer.MAX_VALUE, Integer.MIN_VALUE})
+                .endObject());
+    }
+
+    public void testShorts() throws IOException {
+        assertResult("{'short':null}", () -> builder().startObject().field("short", (Short) null).endObject());
+        assertResult("{'short':5000}", () -> builder().startObject().field("short", Short.valueOf((short) 5000)).endObject());
+        assertResult("{'short':null}", () -> builder().startObject().array("short", (short[]) null).endObject());
+        assertResult("{'short':[]}", () -> builder().startObject().array("short", new short[]{}).endObject());
+        assertResult("{'short':null}", () -> builder().startObject().field("short").value((Short) null).endObject());
+        assertResult("{'short':42}", () -> builder().startObject().field("short").value((short) 42).endObject());
+        assertResult("{'short':[1,3,5,7,11]}", () -> builder()
+                .startObject()
+                    .array("short", (short) 1, (short) 3, (short) 5, (short) 7, (short) 11)
+                .endObject());
+        assertResult("{'short':[32767,-32768]}", () -> builder()
+                .startObject()
+                    .array("short", new short[]{Short.MAX_VALUE, Short.MIN_VALUE})
+                .endObject());
+    }
+
+    public void testStrings() throws IOException {
+        assertResult("{'string':null}", () -> builder().startObject().field("string", (String) null).endObject());
+        assertResult("{'string':'value'}", () -> builder().startObject().field("string", "value").endObject());
+        assertResult("{'string':''}", () -> builder().startObject().field("string", "").endObject());
+        assertResult("{'string':null}", () -> builder().startObject().array("string", (String[]) null).endObject());
+        assertResult("{'string':[]}", () -> builder().startObject().array("string", Strings.EMPTY_ARRAY).endObject());
+        assertResult("{'string':null}", () -> builder().startObject().field("string").value((String) null).endObject());
+        assertResult("{'string':'42'}", () -> builder().startObject().field("string").value("42").endObject());
+        assertResult("{'string':['a','b','c','d']}", () -> builder()
+                .startObject()
+                    .array("string", "a", "b", "c", "d")
+                .endObject());
+    }
+
+    public void testBinaryField() throws Exception {
+        assertResult("{'binary':null}", () -> builder().startObject().field("binary", (byte[]) null).endObject());
+
+        final byte[] randomBytes = randomBytes();
+        BytesReference bytes = builder().startObject().field("binary", randomBytes).endObject().bytes();
+
+        XContentParser parser = createParser(xcontentType().xContent(), bytes);
+        assertSame(parser.nextToken(), Token.START_OBJECT);
+        assertSame(parser.nextToken(), Token.FIELD_NAME);
+        assertEquals(parser.currentName(), "binary");
+        assertTrue(parser.nextToken().isValue());
+        assertArrayEquals(randomBytes, parser.binaryValue());
+        assertSame(parser.nextToken(), Token.END_OBJECT);
+        assertNull(parser.nextToken());
+    }
+
+    public void testBinaryValue() throws Exception {
+        assertResult("{'binary':null}", () -> builder().startObject().field("binary").value((byte[]) null).endObject());
+
+        final byte[] randomBytes = randomBytes();
+        BytesReference bytes = builder().startObject().field("binary").value(randomBytes).endObject().bytes();
+
+        XContentParser parser = createParser(xcontentType().xContent(), bytes);
+        assertSame(parser.nextToken(), Token.START_OBJECT);
+        assertSame(parser.nextToken(), Token.FIELD_NAME);
+        assertEquals(parser.currentName(), "binary");
+        assertTrue(parser.nextToken().isValue());
+        assertArrayEquals(randomBytes, parser.binaryValue());
+        assertSame(parser.nextToken(), Token.END_OBJECT);
+        assertNull(parser.nextToken());
+    }
+
+    public void testBinaryValueWithOffsetLength() throws Exception {
+        assertResult("{'binary':null}", () -> builder().startObject().field("binary").value(null, 0, 0).endObject());
+
+        final byte[] randomBytes = randomBytes();
+        final int offset = randomIntBetween(0, randomBytes.length - 1);
+        final int length = randomIntBetween(1, Math.max(1, randomBytes.length - offset - 1));
+
+        XContentBuilder builder = builder().startObject();
+        if (randomBoolean()) {
+            builder.field("bin", randomBytes, offset, length);
+        } else {
+            builder.field("bin").value(randomBytes, offset, length);
+        }
+        builder.endObject();
+
+        XContentParser parser = createParser(xcontentType().xContent(), builder.bytes());
+        assertSame(parser.nextToken(), Token.START_OBJECT);
+        assertSame(parser.nextToken(), Token.FIELD_NAME);
+        assertEquals(parser.currentName(), "bin");
+        assertTrue(parser.nextToken().isValue());
+        assertArrayEquals(Arrays.copyOfRange(randomBytes, offset, offset + length), parser.binaryValue());
+        assertSame(parser.nextToken(), Token.END_OBJECT);
+        assertNull(parser.nextToken());
+    }
+
+    public void testBinaryUTF8() throws Exception {
+        assertResult("{'utf8':null}", () -> builder().startObject().utf8Field("utf8", null).endObject());
+
+        final BytesRef randomBytesRef = new BytesRef(randomBytes());
+        XContentBuilder builder = builder().startObject();
+        if (randomBoolean()) {
+            builder.utf8Field("utf8", randomBytesRef);
+        } else {
+            builder.field("utf8").utf8Value(randomBytesRef);
+        }
+        builder.endObject();
+
+        XContentParser parser = createParser(xcontentType().xContent(), builder.bytes());
+        assertSame(parser.nextToken(), Token.START_OBJECT);
+        assertSame(parser.nextToken(), Token.FIELD_NAME);
+        assertEquals(parser.currentName(), "utf8");
+        assertTrue(parser.nextToken().isValue());
+        assertThat(parser.utf8Bytes().utf8ToString(), equalTo(randomBytesRef.utf8ToString()));
+        assertSame(parser.nextToken(), Token.END_OBJECT);
+        assertNull(parser.nextToken());
+    }
+
+    public void testText() throws Exception {
+        assertResult("{'text':null}", () -> builder().startObject().field("text", (Text) null).endObject());
+        assertResult("{'text':''}", () -> builder().startObject().field("text", new Text("")).endObject());
+        assertResult("{'text':'foo bar'}", () -> builder().startObject().field("text", new Text("foo bar")).endObject());
+
+        final BytesReference random = new BytesArray(randomBytes());
+        XContentBuilder builder = builder().startObject().field("text", new Text(random)).endObject();
+
+        XContentParser parser = createParser(xcontentType().xContent(), builder.bytes());
+        assertSame(parser.nextToken(), Token.START_OBJECT);
+        assertSame(parser.nextToken(), Token.FIELD_NAME);
+        assertEquals(parser.currentName(), "text");
+        assertTrue(parser.nextToken().isValue());
+        assertThat(parser.utf8Bytes().utf8ToString(), equalTo(random.utf8ToString()));
+        assertSame(parser.nextToken(), Token.END_OBJECT);
+        assertNull(parser.nextToken());
+    }
+
+    public void testReadableInstant() throws Exception {
+        assertResult("{'instant':null}", () -> builder().startObject().field("instant", (ReadableInstant) null).endObject());
+        assertResult("{'instant':null}", () -> builder().startObject().field("instant").value((ReadableInstant) null).endObject());
+
+        final DateTime t1 = new DateTime(2016, 1, 1, 0, 0, DateTimeZone.UTC);
+
+        String expected = "{'t1':'2016-01-01T00:00:00.000Z'}";
+        assertResult(expected, () -> builder().startObject().field("t1", t1).endObject());
+        assertResult(expected, () -> builder().startObject().field("t1").value(t1).endObject());
+
+        final DateTime t2 = new DateTime(2016, 12, 25, 7, 59, 42, 213, DateTimeZone.UTC);
+
+        expected = "{'t2':'2016-12-25T07:59:42.213Z'}";
+        assertResult(expected, () -> builder().startObject().field("t2", t2).endObject());
+        assertResult(expected, () -> builder().startObject().field("t2").value(t2).endObject());
+
+        final DateTimeFormatter formatter = randomFrom(ISODateTimeFormat.basicDate(), ISODateTimeFormat.dateTimeNoMillis());
+        final DateTime t3 = DateTime.now();
+
+        expected = "{'t3':'" + formatter.print(t3) + "'}";
+        assertResult(expected, () -> builder().startObject().field("t3", t3, formatter).endObject());
+        assertResult(expected, () -> builder().startObject().field("t3").value(t3, formatter).endObject());
+
+        final DateTime t4 = new DateTime(randomDateTimeZone());
+
+        expected = "{'t4':'" + formatter.print(t4) + "'}";
+        assertResult(expected, () -> builder().startObject().field("t4", t4, formatter).endObject());
+        assertResult(expected, () -> builder().startObject().field("t4").value(t4, formatter).endObject());
+
+        long date = Math.abs(randomLong() % (2 * (long) 10e11)); // 1970-01-01T00:00:00Z - 2033-05-18T05:33:20.000+02:00
+        final DateTime t5 = new DateTime(date, randomDateTimeZone());
+
+        expected = "{'t5':'" + XContentBuilder.DEFAULT_DATE_PRINTER.print(t5) + "'}";
+        assertResult(expected, () -> builder().startObject().field("t5", t5).endObject());
+        assertResult(expected, () -> builder().startObject().field("t5").value(t5).endObject());
+
+        expected = "{'t5':'" + formatter.print(t5) + "'}";
+        assertResult(expected, () -> builder().startObject().field("t5", t5, formatter).endObject());
+        assertResult(expected, () -> builder().startObject().field("t5").value(t5, formatter).endObject());
+
+        Instant i1 = new Instant(1451606400000L); // 2016-01-01T00:00:00.000Z
+        expected = "{'i1':'2016-01-01T00:00:00.000Z'}";
+        assertResult(expected, () -> builder().startObject().field("i1", i1).endObject());
+        assertResult(expected, () -> builder().startObject().field("i1").value(i1).endObject());
+
+        Instant i2 = new Instant(1482652782213L); // 2016-12-25T07:59:42.213Z
+        expected = "{'i2':'" + formatter.print(i2) + "'}";
+        assertResult(expected, () -> builder().startObject().field("i2", i2, formatter).endObject());
+        assertResult(expected, () -> builder().startObject().field("i2").value(i2, formatter).endObject());
+
+        expectNonNullFormatterException(() -> builder().startObject().field("t3", t3, null).endObject());
+        expectNonNullFormatterException(() -> builder().startObject().field("t3").value(t3, null).endObject());
+    }
+
+    public void testDate() throws Exception {
+        assertResult("{'date':null}", () -> builder().startObject().field("date", (Date) null).endObject());
+        assertResult("{'date':null}", () -> builder().startObject().field("date").value((Date) null).endObject());
+
+        final Date d1 = new DateTime(2016, 1, 1, 0, 0, DateTimeZone.UTC).toDate();
+        assertResult("{'d1':'2016-01-01T00:00:00.000Z'}", () -> builder().startObject().field("d1", d1).endObject());
+        assertResult("{'d1':'2016-01-01T00:00:00.000Z'}", () -> builder().startObject().field("d1").value(d1).endObject());
+
+        final Date d2 = new DateTime(2016, 12, 25, 7, 59, 42, 213, DateTimeZone.UTC).toDate();
+        assertResult("{'d2':'2016-12-25T07:59:42.213Z'}", () -> builder().startObject().field("d2", d2).endObject());
+        assertResult("{'d2':'2016-12-25T07:59:42.213Z'}", () -> builder().startObject().field("d2").value(d2).endObject());
+
+        final DateTimeFormatter formatter = randomFrom(ISODateTimeFormat.basicDate(), ISODateTimeFormat.dateTimeNoMillis());
+        final Date d3 = DateTime.now().toDate();
+
+        String expected = "{'d3':'" + formatter.print(d3.getTime()) + "'}";
+        assertResult(expected, () -> builder().startObject().field("d3", d3, formatter).endObject());
+        assertResult(expected, () -> builder().startObject().field("d3").value(d3, formatter).endObject());
+
+        expectNonNullFormatterException(() -> builder().startObject().field("d3", d3, null).endObject());
+        expectNonNullFormatterException(() -> builder().startObject().field("d3").value(d3, null).endObject());
+        expectNonNullFormatterException(() -> builder().value(null, 1L));
+    }
+
+    public void testDateField() throws Exception {
+        final Date d = new DateTime(2016, 1, 1, 0, 0, DateTimeZone.UTC).toDate();
+
+        assertResult("{'date_in_millis':1451606400000}", () -> builder()
+                .startObject()
+                    .dateField("date_in_millis", "date", d.getTime())
+                .endObject());
+        assertResult("{'date':'2016-01-01T00:00:00.000Z','date_in_millis':1451606400000}", () -> builder()
+                .humanReadable(true)
+                .startObject
+                        ().dateField("date_in_millis", "date", d.getTime())
+                .endObject());
+    }
+
+    public void testCalendar() throws Exception {
+        Calendar calendar = new DateTime(2016, 1, 1, 0, 0, DateTimeZone.UTC).toCalendar(Locale.ROOT);
+        assertResult("{'calendar':'2016-01-01T00:00:00.000Z'}", () -> builder()
+                .startObject()
+                    .field("calendar")
+                    .value(calendar)
+                .endObject());
+    }
+
+    public void testGeoPoint() throws Exception {
+        assertResult("{'geo':null}", () -> builder().startObject().field("geo", (GeoPoint) null).endObject());
+        assertResult("{'geo':{'lat':52.4267578125,'lon':13.271484375}}", () -> builder()
+                .startObject()
+                .   field("geo", GeoPoint.fromGeohash("u336q"))
+                .endObject());
+        assertResult("{'geo':{'lat':52.5201416015625,'lon':13.4033203125}}", () -> builder()
+                .startObject()
+                    .field("geo")
+                    .value(GeoPoint.fromGeohash("u33dc1"))
+                .endObject());
+    }
+
+    public void testLatLon() throws Exception {
+        final String expected = "{'latlon':{'lat':13.271484375,'lon':52.4267578125}}";
+        assertResult(expected, () -> builder().startObject().latlon("latlon", 13.271484375, 52.4267578125).endObject());
+        assertResult(expected, () -> builder().startObject().field("latlon").latlon(13.271484375, 52.4267578125).endObject());
+    }
+
+    public void testPath() throws Exception {
+        assertResult("{'path':null}", () -> builder().startObject().field("path", (Path) null).endObject());
+
+        final Path path = PathUtils.get("first", "second", "third");
+        final String expected = Constants.WINDOWS ? "{'path':'first\\\\second\\\\third'}" : "{'path':'first/second/third'}";
+        assertResult(expected, () -> builder().startObject().field("path", path).endObject());
+    }
+
+    public void testObjects() throws Exception {
+        Map<String, Object[]> objects = new HashMap<>();
+        objects.put("{'objects':[false,true,false]}", new Object[]{false, true, false});
+        objects.put("{'objects':[1,1,2,3,5,8,13]}", new Object[]{(byte) 1, (byte) 1, (byte) 2, (byte) 3, (byte) 5, (byte) 8, (byte) 13});
+        objects.put("{'objects':[1.0,1.0,2.0,3.0,5.0,8.0,13.0]}", new Object[]{1.0d, 1.0d, 2.0d, 3.0d, 5.0d, 8.0d, 13.0d});
+        objects.put("{'objects':[1.0,1.0,2.0,3.0,5.0,8.0,13.0]}", new Object[]{1.0f, 1.0f, 2.0f, 3.0f, 5.0f, 8.0f, 13.0f});
+        objects.put("{'objects':[{'lat':45.759429931640625,'lon':4.8394775390625}]}", new Object[]{GeoPoint.fromGeohash("u05kq4k")});
+        objects.put("{'objects':[1,1,2,3,5,8,13]}", new Object[]{1, 1, 2, 3, 5, 8, 13});
+        objects.put("{'objects':[1,1,2,3,5,8,13]}", new Object[]{1L, 1L, 2L, 3L, 5L, 8L, 13L});
+        objects.put("{'objects':[1,1,2,3,5,8]}", new Object[]{(short) 1, (short) 1, (short) 2, (short) 3, (short) 5, (short) 8});
+        objects.put("{'objects':['a','b','c']}", new Object[]{"a", "b", "c"});
+        objects.put("{'objects':['a','b','c']}", new Object[]{new Text("a"), new Text(new BytesArray("b")), new Text("c")});
+        objects.put("{'objects':null}", null);
+        objects.put("{'objects':[null,null,null]}", new Object[]{null, null, null});
+        objects.put("{'objects':['OPEN','CLOSE']}", IndexMetaData.State.values());
+        objects.put("{'objects':[{'f1':'v1'},{'f2':'v2'}]}", new Object[]{singletonMap("f1", "v1"), singletonMap("f2", "v2")});
+        objects.put("{'objects':[[1,2,3],[4,5]]}", new Object[]{Arrays.asList(1, 2, 3), Arrays.asList(4, 5)});
+
+        final String paths = Constants.WINDOWS ? "{'objects':['a\\\\b\\\\c','d\\\\e']}" : "{'objects':['a/b/c','d/e']}";
+        objects.put(paths, new Object[]{PathUtils.get("a", "b", "c"), PathUtils.get("d", "e")});
+
+        final DateTimeFormatter formatter = XContentBuilder.DEFAULT_DATE_PRINTER;
+        final Date d1 = new DateTime(2016, 1, 1, 0, 0, DateTimeZone.UTC).toDate();
+        final Date d2 = new DateTime(2015, 1, 1, 0, 0, DateTimeZone.UTC).toDate();
+        objects.put("{'objects':['" + formatter.print(d1.getTime()) + "','" + formatter.print(d2.getTime()) + "']}", new Object[]{d1, d2});
+
+        final DateTime dt1 = DateTime.now();
+        final DateTime dt2 = new DateTime(2016, 12, 25, 7, 59, 42, 213, DateTimeZone.UTC);
+        objects.put("{'objects':['" + formatter.print(dt1) + "','2016-12-25T07:59:42.213Z']}", new Object[]{dt1, dt2});
+
+        final Calendar c1 = new DateTime(2012, 7, 7, 10, 23, DateTimeZone.UTC).toCalendar(Locale.ROOT);
+        final Calendar c2 = new DateTime(2014, 11, 16, 19, 36, DateTimeZone.UTC).toCalendar(Locale.ROOT);
+        objects.put("{'objects':['2012-07-07T10:23:00.000Z','2014-11-16T19:36:00.000Z']}", new Object[]{c1, c2});
+
+        final ToXContent x1 = (builder, params) -> builder.startObject().field("f1", "v1").field("f2", 2).array("f3", 3, 4, 5).endObject();
+        final ToXContent x2 = (builder, params) -> builder.startObject().field("f1", "v1").field("f2", x1).endObject();
+        objects.put("{'objects':[{'f1':'v1','f2':2,'f3':[3,4,5]},{'f1':'v1','f2':{'f1':'v1','f2':2,'f3':[3,4,5]}}]}", new Object[]{x1, x2});
+
+        for (Map.Entry<String, Object[]> o : objects.entrySet()) {
+            final String expected = o.getKey();
+            assertResult(expected, () -> builder().startObject().field("objects", o.getValue()).endObject());
+            assertResult(expected, () -> builder().startObject().field("objects").value(o.getValue()).endObject());
+            assertResult(expected, () -> builder().startObject().field("objects").values(o.getValue()).endObject());
+            assertResult(expected, () -> builder().startObject().array("objects", o.getValue()).endObject());
+        }
+    }
+
+    public void testObject() throws Exception {
+        Map<String, Object> object = new HashMap<>();
+        object.put("{'object':false}", Boolean.FALSE);
+        object.put("{'object':13}", (byte) 13);
+        object.put("{'object':5.0}", 5.0d);
+        object.put("{'object':8.0}", 8.0f);
+        object.put("{'object':{'lat':45.759429931640625,'lon':4.8394775390625}}", GeoPoint.fromGeohash("u05kq4k"));
+        object.put("{'object':3}", 3);
+        object.put("{'object':2}", 2L);
+        object.put("{'object':1}", (short) 1);
+        object.put("{'object':'string'}", "string");
+        object.put("{'object':'a'}", new Text("a"));
+        object.put("{'object':'b'}", new Text(new BytesArray("b")));
+        object.put("{'object':null}", null);
+        object.put("{'object':'OPEN'}", IndexMetaData.State.OPEN);
+        object.put("{'object':'NM'}", DistanceUnit.NAUTICALMILES);
+        object.put("{'object':{'f1':'v1'}}", singletonMap("f1", "v1"));
+        object.put("{'object':{'f1':{'f2':'v2'}}}", singletonMap("f1", singletonMap("f2", "v2")));
+        object.put("{'object':[1,2,3]}", Arrays.asList(1, 2, 3));
+
+        final String path = Constants.WINDOWS ? "{'object':'a\\\\b\\\\c'}" : "{'object':'a/b/c'}";
+        object.put(path, PathUtils.get("a", "b", "c"));
+
+        final DateTimeFormatter formatter = XContentBuilder.DEFAULT_DATE_PRINTER;
+        final Date d1 = new DateTime(2016, 1, 1, 0, 0, DateTimeZone.UTC).toDate();
+        object.put("{'object':'" + formatter.print(d1.getTime()) + "'}", d1);
+
+        final DateTime d2 = DateTime.now();
+        object.put("{'object':'" + formatter.print(d2) + "'}", d2);
+
+        final Calendar c1 = new DateTime(2010, 1, 1, 0, 0, DateTimeZone.UTC).toCalendar(Locale.ROOT);
+        object.put("{'object':'2010-01-01T00:00:00.000Z'}", c1);
+
+        final ToXContent x1 = (builder, params) -> builder.startObject().field("f1", "v1").field("f2", 2).array("f3", 3, 4, 5).endObject();
+        final ToXContent x2 = (builder, params) -> builder.startObject().field("f1", "v1").field("f2", x1).endObject();
+        object.put("{'object':{'f1':'v1','f2':{'f1':'v1','f2':2,'f3':[3,4,5]}}}", x2);
+
+        for (Map.Entry<String, Object> o : object.entrySet()) {
+            final String expected = o.getKey();
+            assertResult(expected, () -> builder().humanReadable(true).startObject().field("object", o.getValue()).endObject());
+            assertResult(expected, () -> builder().humanReadable(true).startObject().field("object").value(o.getValue()).endObject());
+        }
+
+        assertResult("{'objects':[null,null,null]}", () -> builder().startObject().array("objects", null, null, null).endObject());
+    }
+
+    public void testToXContent() throws Exception {
+        assertResult("{'xcontent':null}", () -> builder().startObject().field("xcontent", (ToXContent) null).endObject());
+        assertResult("{'xcontent':null}", () -> builder().startObject().field("xcontent").value((ToXContent) null).endObject());
+
+        ToXContent xcontent0 = (builder, params) -> {
+            builder.startObject();
+            builder.field("field", "value");
+            builder.array("array", "1", "2", "3");
+            builder.startObject("foo");
+            builder.field("bar", "baz");
+            builder.endObject();
+            builder.endObject();
+            return builder;
+        };
+
+        assertResult("{'field':'value','array':['1','2','3'],'foo':{'bar':'baz'}}", () -> builder().value(xcontent0));
+        assertResult("{'xcontent':{'field':'value','array':['1','2','3'],'foo':{'bar':'baz'}}}", () -> builder()
+                .startObject()
+                    .field("xcontent", xcontent0)
+                .endObject());
+
+        ToXContent xcontent1 = (builder, params) -> {
+            builder.startObject();
+            builder.field("field", "value");
+            builder.startObject("foo");
+            builder.field("bar", "baz");
+            builder.endObject();
+            builder.endObject();
+            return builder;
+        };
+
+        ToXContent xcontent2 = (builder, params) -> {
+            builder.startObject();
+            builder.field("root", xcontent0);
+            builder.array("childs", xcontent0, xcontent1);
+            builder.endObject();
+            return builder;
+        };
+        assertResult("{'root':{" +
+                            "'field':'value'," +
+                            "'array':['1','2','3']," +
+                            "'foo':{'bar':'baz'}" +
+                        "}," +
+                        "'childs':[" +
+                            "{'field':'value','array':['1','2','3'],'foo':{'bar':'baz'}}," +
+                            "{'field':'value','foo':{'bar':'baz'}}" +
+                        "]}", () -> builder().value(xcontent2));
+    }
+
+    public void testMap() throws Exception {
+        Map<String, Map<String, ?>> maps = new HashMap<>();
+        maps.put("{'map':null}", (Map) null);
+        maps.put("{'map':{}}", Collections.emptyMap());
+        maps.put("{'map':{'key':'value'}}", singletonMap("key", "value"));
+
+        Map<String, Object> innerMap = new HashMap<>();
+        innerMap.put("string", "value");
+        innerMap.put("int", 42);
+        innerMap.put("long", 42L);
+        innerMap.put("long[]", new long[]{1L, 3L});
+        innerMap.put("path", PathUtils.get("path", "to", "file"));
+        innerMap.put("object", singletonMap("key", "value"));
+
+        final String path = Constants.WINDOWS ? "path\\\\to\\\\file" : "path/to/file";
+        maps.put("{'map':{'path':'" + path + "','string':'value','long[]':[1,3],'int':42,'long':42,'object':{'key':'value'}}}", innerMap);
+
+        for (Map.Entry<String, Map<String, ?>> m : maps.entrySet()) {
+            final String expected = m.getKey();
+            assertResult(expected, () -> builder().startObject().field("map", m.getValue()).endObject());
+            assertResult(expected, () -> builder().startObject().field("map").value(m.getValue()).endObject());
+            assertResult(expected, () -> builder().startObject().field("map").map(m.getValue()).endObject());
+        }
+    }
+
+    public void testIterable() throws Exception {
+        Map<String, Iterable<?>> iterables = new HashMap<>();
+        iterables.put("{'iter':null}", (Iterable) null);
+        iterables.put("{'iter':[]}", Collections.emptyList());
+        iterables.put("{'iter':['a','b']}", Arrays.asList("a", "b"));
+
+        final String path = Constants.WINDOWS ? "{'iter':'path\\\\to\\\\file'}" : "{'iter':'path/to/file'}";
+        iterables.put(path, PathUtils.get("path", "to", "file"));
+
+        final String paths = Constants.WINDOWS ? "{'iter':['a\\\\b\\\\c','c\\\\d']}" : "{'iter':['a/b/c','c/d']}";
+        iterables.put(paths, Arrays.asList(PathUtils.get("a", "b", "c"), PathUtils.get("c", "d")));
+
+        for (Map.Entry<String, Iterable<?>> i : iterables.entrySet()) {
+            final String expected = i.getKey();
+            assertResult(expected, () -> builder().startObject().field("iter", i.getValue()).endObject());
+            assertResult(expected, () -> builder().startObject().field("iter").value(i.getValue()).endObject());
+        }
+    }
+
+    public void testUnknownObject() throws Exception {
+        Map<String, Object> objects = new HashMap<>();
+        objects.put("{'obj':50.63}", DistanceUnit.METERS.fromMeters(50.63));
+        objects.put("{'obj':'MINUTES'}", TimeUnit.MINUTES);
+        objects.put("{'obj':'class org.elasticsearch.common.xcontent.BaseXContentTestCase'}", BaseXContentTestCase.class);
+
+        for (Map.Entry<String, ?> o : objects.entrySet()) {
+            final String expected = o.getKey();
+            assertResult(expected, () -> builder().startObject().field("obj", o.getValue()).endObject());
+            assertResult(expected, () -> builder().startObject().field("obj").value(o.getValue()).endObject());
+        }
+    }
 
     public void testBasics() throws IOException {
         ByteArrayOutputStream os = new ByteArrayOutputStream();
@@ -56,7 +708,7 @@ public void testMissingEndObject() throws IOException {
                 generator.writeNumber(2L);
             }
         });
-        assertEquals(e.getMessage(), "unclosed object or array found");
+        assertEquals(e.getMessage(), "Unclosed object or array found");
     }
 
     public void testMissingEndArray() throws IOException {
@@ -67,11 +719,11 @@ public void testMissingEndArray() throws IOException {
                 generator.writeNumber(2L);
             }
         });
-        assertEquals(e.getMessage(), "unclosed object or array found");
+        assertEquals(e.getMessage(), "Unclosed object or array found");
     }
 
     public void testRawField() throws Exception {
-        for (boolean useStream : new boolean[] {false, true}) {
+        for (boolean useStream : new boolean[]{false, true}) {
             for (XContentType xcontentType : XContentType.values()) {
                 doTestRawField(xcontentType.xContent(), useStream);
             }
@@ -99,7 +751,7 @@ void doTestRawField(XContent source, boolean useStream) throws Exception {
             generator.writeEndObject();
         }
 
-        XContentParser parser = xcontentType().xContent().createParser(os.toByteArray());
+        XContentParser parser = xcontentType().xContent().createParser(NamedXContentRegistry.EMPTY, os.toByteArray());
         assertEquals(Token.START_OBJECT, parser.nextToken());
         assertEquals(Token.FIELD_NAME, parser.nextToken());
         assertEquals("bar", parser.currentName());
@@ -133,7 +785,7 @@ void doTestRawValue(XContent source) throws Exception {
             generator.writeRawValue(new BytesArray(rawData));
         }
 
-        XContentParser parser = xcontentType().xContent().createParser(os.toByteArray());
+        XContentParser parser = xcontentType().xContent().createParser(NamedXContentRegistry.EMPTY, os.toByteArray());
         assertEquals(Token.START_OBJECT, parser.nextToken());
         assertEquals(Token.FIELD_NAME, parser.nextToken());
         assertEquals("foo", parser.currentName());
@@ -149,7 +801,7 @@ void doTestRawValue(XContent source) throws Exception {
             generator.writeEndObject();
         }
 
-        parser = xcontentType().xContent().createParser(os.toByteArray());
+        parser = xcontentType().xContent().createParser(NamedXContentRegistry.EMPTY, os.toByteArray());
         assertEquals(Token.START_OBJECT, parser.nextToken());
         assertEquals(Token.FIELD_NAME, parser.nextToken());
         assertEquals("test", parser.currentName());
@@ -177,9 +829,251 @@ protected void doTestBigInteger(JsonGenerator generator, ByteArrayOutputStream o
         generator.flush();
         byte[] serialized = os.toByteArray();
 
-        XContentParser parser = xcontentType().xContent().createParser(serialized);
+        XContentParser parser = xcontentType().xContent().createParser(NamedXContentRegistry.EMPTY, serialized);
         Map<String, Object> map = parser.map();
         assertEquals("bar", map.get("foo"));
         assertEquals(bigInteger, map.get("bigint"));
     }
+
+    public void testEnsureNameNotNull() {
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> XContentBuilder.ensureNameNotNull(null));
+        assertThat(e.getMessage(), containsString("Field name cannot be null"));
+    }
+
+    public void testFormatterNameNotNull() {
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> XContentBuilder.ensureFormatterNotNull(null));
+        assertThat(e.getMessage(), containsString("DateTimeFormatter cannot be null"));
+    }
+
+    public void testEnsureNotNull() {
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> XContentBuilder.ensureNotNull(null, "message"));
+        assertThat(e.getMessage(), containsString("message"));
+
+        XContentBuilder.ensureNotNull("foo", "No exception must be thrown");
+    }
+
+    public void testEnsureNoSelfReferences() throws IOException {
+        XContentBuilder.ensureNoSelfReferences(emptyMap());
+        XContentBuilder.ensureNoSelfReferences(null);
+
+        Map<String, Object> map = new HashMap<>();
+        map.put("field", map);
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> builder().map(map));
+        assertThat(e.getMessage(), containsString("Object has already been built and is self-referencing itself"));
+    }
+
+    /**
+     * Test that the same map written multiple times do not trigger the self-reference check in
+     * {@link XContentBuilder#ensureNoSelfReferences(Object)}
+     */
+    public void testRepeatedMapsAndNoSelfReferences() throws Exception {
+        Map<String, Object> mapB = singletonMap("b", "B");
+        Map<String, Object> mapC = singletonMap("c", "C");
+        Map<String, Object> mapD = singletonMap("d", "D");
+        Map<String, Object> mapA = new HashMap<>();
+        mapA.put("a", 0);
+        mapA.put("b1", mapB);
+        mapA.put("b2", mapB);
+        mapA.put("c", Arrays.asList(mapC, mapC));
+        mapA.put("d1", mapD);
+        mapA.put("d2", singletonMap("d3", mapD));
+
+        final String expected =
+                "{'map':{'b2':{'b':'B'},'a':0,'c':[{'c':'C'},{'c':'C'}],'d1':{'d':'D'},'d2':{'d3':{'d':'D'}},'b1':{'b':'B'}}}";
+
+        assertResult(expected, () -> builder().startObject().field("map", mapA).endObject());
+        assertResult(expected, () -> builder().startObject().field("map").value(mapA).endObject());
+        assertResult(expected, () -> builder().startObject().field("map").map(mapA).endObject());
+    }
+
+    public void testSelfReferencingMapsOneLevel() throws IOException {
+        Map<String, Object> map0 = new HashMap<>();
+        Map<String, Object> map1 = new HashMap<>();
+
+        map0.put("foo", 0);
+        map0.put("map1", map1); // map 0 -> map 1
+
+        map1.put("bar", 1);
+        map1.put("map0", map0); // map 1 -> map 0 loop
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> builder().map(map0));
+        assertThat(e.getMessage(), containsString("Object has already been built and is self-referencing itself"));
+    }
+
+    public void testSelfReferencingMapsTwoLevels() throws IOException {
+        Map<String, Object> map0 = new HashMap<>();
+        Map<String, Object> map1 = new HashMap<>();
+        Map<String, Object> map2 = new HashMap<>();
+
+        map0.put("foo", 0);
+        map0.put("map1", map1); // map 0 -> map 1
+
+        map1.put("bar", 1);
+        map1.put("map2", map2); // map 1 -> map 2
+
+        map2.put("baz", 2);
+        map2.put("map0", map0); // map 2 -> map 0 loop
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> builder().map(map0));
+        assertThat(e.getMessage(), containsString("Object has already been built and is self-referencing itself"));
+    }
+
+    public void testSelfReferencingObjectsArray() throws IOException {
+        Object[] values = new Object[3];
+        values[0] = 0;
+        values[1] = 1;
+        values[2] = values;
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> builder()
+                .startObject()
+                .field("field", values)
+                .endObject());
+        assertThat(e.getMessage(), containsString("Object has already been built and is self-referencing itself"));
+
+        e = expectThrows(IllegalArgumentException.class, () -> builder()
+                .startObject()
+                .array("field", values)
+                .endObject());
+        assertThat(e.getMessage(), containsString("Object has already been built and is self-referencing itself"));
+    }
+
+    public void testSelfReferencingIterable() throws IOException {
+        List<Object> values = new ArrayList<>();
+        values.add("foo");
+        values.add("bar");
+        values.add(values);
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> builder()
+                .startObject()
+                .field("field", (Iterable) values)
+                .endObject());
+        assertThat(e.getMessage(), containsString("Object has already been built and is self-referencing itself"));
+    }
+
+    public void testSelfReferencingIterableOneLevel() throws IOException {
+        Map<String, Object> map = new HashMap<>();
+        map.put("foo", 0);
+        map.put("bar", 1);
+
+        Iterable<Object> values = Arrays.asList("one", "two", map);
+        map.put("baz", values);
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> builder()
+                .startObject()
+                .field("field", (Iterable) values)
+                .endObject());
+        assertThat(e.getMessage(), containsString("Object has already been built and is self-referencing itself"));
+    }
+
+    public void testSelfReferencingIterableTwoLevels() throws IOException {
+        Map<String, Object> map0 = new HashMap<>();
+        Map<String, Object> map1 = new HashMap<>();
+        Map<String, Object> map2 = new HashMap<>();
+
+        List<Object> it1 = new ArrayList<>();
+
+        map0.put("foo", 0);
+        map0.put("it1", (Iterable<?>) it1); // map 0 -> it1
+
+        it1.add(map1);
+        it1.add(map2); // it 1 -> map 1, map 2
+
+        map2.put("baz", 2);
+        map2.put("map0", map0); // map 2 -> map 0 loop
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> builder().map(map0));
+        assertThat(e.getMessage(), containsString("Object has already been built and is self-referencing itself"));
+    }
+
+    public void testNamedObject() throws IOException {
+        Object test1 = new Object();
+        Object test2 = new Object();
+        NamedXContentRegistry registry = new NamedXContentRegistry(Arrays.asList(
+                new NamedXContentRegistry.Entry(Object.class, new ParseField("test1"), p -> test1),
+                new NamedXContentRegistry.Entry(Object.class, new ParseField("test2", "deprecated"), p -> test2),
+                new NamedXContentRegistry.Entry(Object.class, new ParseField("str"), p -> p.text())));
+        XContentBuilder b = XContentBuilder.builder(xcontentType().xContent());
+        b.value("test");
+        XContentParser p = xcontentType().xContent().createParser(registry, b.bytes());
+        assertEquals(test1, p.namedObject(Object.class, "test1", null));
+        assertEquals(test2, p.namedObject(Object.class, "test2", null));
+        assertEquals(test2, p.namedObject(Object.class, "deprecated", null));
+
+        assertWarnings("Deprecated field [deprecated] used, expected [test2] instead");
+        {
+            p.nextToken();
+            assertEquals("test", p.namedObject(Object.class, "str", null));
+            NamedXContentRegistry.UnknownNamedObjectException e = expectThrows(NamedXContentRegistry.UnknownNamedObjectException.class,
+                    () -> p.namedObject(Object.class, "unknown", null));
+            assertEquals("Unknown Object [unknown]", e.getMessage());
+            assertEquals("java.lang.Object", e.getCategoryClass());
+            assertEquals("unknown", e.getName());
+        }
+        {
+            Exception e = expectThrows(ElasticsearchException.class, () -> p.namedObject(String.class, "doesn't matter", null));
+            assertEquals("Unknown namedObject category [java.lang.String]", e.getMessage());
+        }
+        {
+            XContentParser emptyRegistryParser = xcontentType().xContent().createParser(NamedXContentRegistry.EMPTY, new byte[] {});
+            Exception e = expectThrows(ElasticsearchException.class,
+                    () -> emptyRegistryParser.namedObject(String.class, "doesn't matter", null));
+            assertEquals("namedObject is not supported for this parser", e.getMessage());
+        }
+    }
+
+    private static void expectUnclosedException(ThrowingRunnable runnable) {
+        IllegalStateException e = expectThrows(IllegalStateException.class, runnable);
+        assertThat(e.getMessage(), containsString("Failed to close the XContentBuilder"));
+        assertThat(e.getCause(), allOf(notNullValue(), instanceOf(IOException.class)));
+        assertThat(e.getCause().getMessage(), containsString("Unclosed object or array found"));
+    }
+
+    private static void expectValueException(ThrowingRunnable runnable) {
+        JsonGenerationException e = expectThrows(JsonGenerationException.class, runnable);
+        assertThat(e.getMessage(), containsString("expecting a value"));
+    }
+
+    private static void expectFieldException(ThrowingRunnable runnable) {
+        JsonGenerationException e = expectThrows(JsonGenerationException.class, runnable);
+        assertThat(e.getMessage(), containsString("expecting field name"));
+    }
+
+    private static void expectNonNullFieldException(ThrowingRunnable runnable) {
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, runnable);
+        assertThat(e.getMessage(), containsString("Field name cannot be null"));
+    }
+
+    private static void expectNonNullFormatterException(ThrowingRunnable runnable) {
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, runnable);
+        assertThat(e.getMessage(), containsString("DateTimeFormatter cannot be null"));
+    }
+
+    private static void expectObjectException(ThrowingRunnable runnable) {
+        JsonGenerationException e = expectThrows(JsonGenerationException.class, runnable);
+        assertThat(e.getMessage(), containsString("Current context not Object"));
+    }
+
+    private static void expectArrayException(ThrowingRunnable runnable) {
+        JsonGenerationException e = expectThrows(JsonGenerationException.class, runnable);
+        assertThat(e.getMessage(), containsString("Current context not Array"));
+    }
+
+    public static Matcher<String> equalToJson(String json) {
+        return Matchers.equalTo(json.replace("'", "\""));
+    }
+
+    private static void assertResult(String expected, Builder builder) throws IOException {
+        // Build the XContentBuilder, convert its bytes to JSON and check it matches
+        assertThat(XContentHelper.convertToJson(builder.build().bytes(), randomBoolean()), equalToJson(expected));
+    }
+
+    private static byte[] randomBytes() throws Exception {
+        return randomUnicodeOfLength(scaledRandomIntBetween(10, 1000)).getBytes("UTF-8");
+    }
+
+    @FunctionalInterface
+    private interface Builder {
+        XContentBuilder build() throws IOException;
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/ConstructingObjectParserTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/ConstructingObjectParserTests.java
index bef4a047ef57e..31088bb2d23f8 100644
--- a/core/src/test/java/org/elasticsearch/common/xcontent/ConstructingObjectParserTests.java
+++ b/core/src/test/java/org/elasticsearch/common/xcontent/ConstructingObjectParserTests.java
@@ -19,12 +19,12 @@
 
 package org.elasticsearch.common.xcontent;
 
+import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.test.ESTestCase;
 import org.hamcrest.Matcher;
 
@@ -41,25 +41,44 @@
 import static org.hamcrest.Matchers.nullValue;
 
 public class ConstructingObjectParserTests extends ESTestCase {
-    private static final ParseFieldMatcherSupplier MATCHER = () -> ParseFieldMatcher.STRICT;
+    public void testNullDeclares() {
+        ConstructingObjectParser<Void, Void> objectParser = new ConstructingObjectParser<>("foo", a -> null);
+        Exception e = expectThrows(IllegalArgumentException.class,
+                () -> objectParser.declareField(null, (r, c) -> null, new ParseField("test"), ObjectParser.ValueType.STRING));
+        assertEquals("[consumer] is required", e.getMessage());
+        e = expectThrows(IllegalArgumentException.class, () -> objectParser.declareField(
+                (o, v) -> {}, (ContextParser<Void, Object>) null,
+                new ParseField("test"), ObjectParser.ValueType.STRING));
+        assertEquals("[parser] is required", e.getMessage());
+        e = expectThrows(IllegalArgumentException.class, () -> objectParser.declareField(
+                (o, v) -> {}, (CheckedFunction<XContentParser, Object, IOException>) null,
+                new ParseField("test"), ObjectParser.ValueType.STRING));
+        assertEquals("[parser] is required", e.getMessage());
+        e = expectThrows(IllegalArgumentException.class, () -> objectParser.declareField(
+                (o, v) -> {}, (r, c) -> null, null, ObjectParser.ValueType.STRING));
+        assertEquals("[parseField] is required", e.getMessage());
+        e = expectThrows(IllegalArgumentException.class, () -> objectParser.declareField(
+                (o, v) -> {}, (r, c) -> null, new ParseField("test"), null));
+        assertEquals("[type] is required", e.getMessage());
+    }
 
     /**
      * Builds the object in random order and parses it.
      */
     public void testRandomOrder() throws Exception {
-        HasCtorArguments expected = new HasCtorArguments(randomAsciiOfLength(5), randomInt());
+        HasCtorArguments expected = new HasCtorArguments(randomAlphaOfLength(5), randomInt());
         expected.setMineral(randomInt());
         expected.setFruit(randomInt());
-        expected.setA(randomBoolean() ? null : randomAsciiOfLength(5));
-        expected.setB(randomBoolean() ? null : randomAsciiOfLength(5));
-        expected.setC(randomBoolean() ? null : randomAsciiOfLength(5));
+        expected.setA(randomBoolean() ? null : randomAlphaOfLength(5));
+        expected.setB(randomBoolean() ? null : randomAlphaOfLength(5));
+        expected.setC(randomBoolean() ? null : randomAlphaOfLength(5));
         expected.setD(randomBoolean());
         XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint();
         expected.toXContent(builder, ToXContent.EMPTY_PARAMS);
         builder = shuffleXContent(builder);
         BytesReference bytes = builder.bytes();
-        try (XContentParser parser = XContentFactory.xContent(bytes).createParser(bytes)) {
-            HasCtorArguments parsed = randomFrom(HasCtorArguments.ALL_PARSERS).apply(parser, MATCHER);
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, bytes)) {
+            HasCtorArguments parsed = randomFrom(HasCtorArguments.ALL_PARSERS).apply(parser, null);
             assertEquals(expected.animal, parsed.animal);
             assertEquals(expected.vegetable, parsed.vegetable);
             assertEquals(expected.mineral, parsed.mineral);
@@ -75,13 +94,13 @@ public void testRandomOrder() throws Exception {
     }
 
     public void testMissingAllConstructorArgs() throws IOException {
-        XContentParser parser = XContentType.JSON.xContent().createParser(
+        XContentParser parser = createParser(JsonXContent.jsonXContent, 
                   "{\n"
                 + "  \"mineral\": 1\n"
                 + "}");
-        ConstructingObjectParser<HasCtorArguments, ParseFieldMatcherSupplier> objectParser = randomBoolean() ? HasCtorArguments.PARSER
+        ConstructingObjectParser<HasCtorArguments, Void> objectParser = randomBoolean() ? HasCtorArguments.PARSER
                 : HasCtorArguments.PARSER_VEGETABLE_OPTIONAL;
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> objectParser.apply(parser, MATCHER));
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> objectParser.apply(parser, null));
         if (objectParser == HasCtorArguments.PARSER) {
             assertEquals("Required [animal, vegetable]", e.getMessage());
         } else {
@@ -90,68 +109,68 @@ public void testMissingAllConstructorArgs() throws IOException {
     }
 
     public void testMissingAllConstructorArgsButNotRequired() throws IOException {
-        XContentParser parser = XContentType.JSON.xContent().createParser(
+        XContentParser parser = createParser(JsonXContent.jsonXContent, 
                 "{\n"
               + "  \"mineral\": 1\n"
               + "}");
-        HasCtorArguments parsed = HasCtorArguments.PARSER_ALL_OPTIONAL.apply(parser, MATCHER);
+        HasCtorArguments parsed = HasCtorArguments.PARSER_ALL_OPTIONAL.apply(parser, null);
         assertEquals(1, parsed.mineral);
     }
 
     public void testMissingSecondConstructorArg() throws IOException {
-        XContentParser parser = XContentType.JSON.xContent().createParser(
+        XContentParser parser = createParser(JsonXContent.jsonXContent, 
                   "{\n"
                 + "  \"mineral\": 1,\n"
                 + "  \"animal\": \"cat\"\n"
                 + "}");
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-                () -> HasCtorArguments.PARSER.apply(parser, MATCHER));
+                () -> HasCtorArguments.PARSER.apply(parser, null));
         assertEquals("Required [vegetable]", e.getMessage());
     }
 
     public void testMissingSecondConstructorArgButNotRequired() throws IOException {
-        XContentParser parser = XContentType.JSON.xContent().createParser(
+        XContentParser parser = createParser(JsonXContent.jsonXContent, 
                 "{\n"
               + "  \"mineral\": 1,\n"
               + "  \"animal\": \"cat\"\n"
               + "}");
         @SuppressWarnings("unchecked")
         HasCtorArguments parsed = randomFrom(HasCtorArguments.PARSER_VEGETABLE_OPTIONAL, HasCtorArguments.PARSER_ALL_OPTIONAL).apply(parser,
-                MATCHER);
+                null);
         assertEquals(1, parsed.mineral);
         assertEquals("cat", parsed.animal);
     }
 
     public void testMissingFirstConstructorArg() throws IOException {
-        XContentParser parser = XContentType.JSON.xContent().createParser(
+        XContentParser parser = createParser(JsonXContent.jsonXContent, 
                   "{\n"
                 + "  \"mineral\": 1,\n"
                 + "  \"vegetable\": 2\n"
                 + "}");
         @SuppressWarnings("unchecked")
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-                () -> randomFrom(HasCtorArguments.PARSER, HasCtorArguments.PARSER_VEGETABLE_OPTIONAL).apply(parser, MATCHER));
+                () -> randomFrom(HasCtorArguments.PARSER, HasCtorArguments.PARSER_VEGETABLE_OPTIONAL).apply(parser, null));
         assertEquals("Required [animal]", e.getMessage());
     }
 
     public void testMissingFirstConstructorArgButNotRequired() throws IOException {
-        XContentParser parser = XContentType.JSON.xContent().createParser(
+        XContentParser parser = createParser(JsonXContent.jsonXContent, 
                 "{\n"
               + "  \"mineral\": 1,\n"
               + "  \"vegetable\": 2\n"
               + "}");
-        HasCtorArguments parsed = HasCtorArguments.PARSER_ALL_OPTIONAL.apply(parser, MATCHER);
+        HasCtorArguments parsed = HasCtorArguments.PARSER_ALL_OPTIONAL.apply(parser, null);
         assertEquals(1, parsed.mineral);
         assertEquals((Integer) 2, parsed.vegetable);
     }
 
     public void testRepeatedConstructorParam() throws IOException {
-        XContentParser parser = XContentType.JSON.xContent().createParser(
+        XContentParser parser = createParser(JsonXContent.jsonXContent, 
                   "{\n"
                 + "  \"vegetable\": 1,\n"
                 + "  \"vegetable\": 2\n"
                 + "}");
-        Throwable e = expectThrows(ParsingException.class, () -> randomFrom(HasCtorArguments.ALL_PARSERS).apply(parser, MATCHER));
+        Throwable e = expectThrows(ParsingException.class, () -> randomFrom(HasCtorArguments.ALL_PARSERS).apply(parser, null));
         assertEquals("[has_required_arguments] failed to parse field [vegetable]", e.getMessage());
         e = e.getCause();
         assertThat(e, instanceOf(IllegalArgumentException.class));
@@ -159,13 +178,13 @@ public void testRepeatedConstructorParam() throws IOException {
     }
 
     public void testBadParam() throws IOException {
-        XContentParser parser = XContentType.JSON.xContent().createParser(
+        XContentParser parser = createParser(JsonXContent.jsonXContent, 
                   "{\n"
                 + "  \"animal\": \"cat\",\n"
                 + "  \"vegetable\": 2,\n"
                 + "  \"a\": \"supercalifragilisticexpialidocious\"\n"
                 + "}");
-        ParsingException e = expectThrows(ParsingException.class, () -> randomFrom(HasCtorArguments.ALL_PARSERS).apply(parser, MATCHER));
+        ParsingException e = expectThrows(ParsingException.class, () -> randomFrom(HasCtorArguments.ALL_PARSERS).apply(parser, null));
         assertEquals("[has_required_arguments] failed to parse field [a]", e.getMessage());
         assertEquals(4, e.getLineNumber());
         assertEquals("[a] must be less than 10 characters in length but was [supercalifragilisticexpialidocious]",
@@ -173,13 +192,13 @@ public void testBadParam() throws IOException {
     }
 
     public void testBadParamBeforeObjectBuilt() throws IOException {
-        XContentParser parser = XContentType.JSON.xContent().createParser(
+        XContentParser parser = createParser(JsonXContent.jsonXContent, 
                   "{\n"
                 + "  \"a\": \"supercalifragilisticexpialidocious\",\n"
                 + "  \"animal\": \"cat\"\n,"
                 + "  \"vegetable\": 2\n"
                 + "}");
-        ParsingException e = expectThrows(ParsingException.class, () -> randomFrom(HasCtorArguments.ALL_PARSERS).apply(parser, MATCHER));
+        ParsingException e = expectThrows(ParsingException.class, () -> randomFrom(HasCtorArguments.ALL_PARSERS).apply(parser, null));
         assertEquals("[has_required_arguments] failed to parse field [vegetable]", e.getMessage());
         assertEquals(4, e.getLineNumber());
         e = (ParsingException) e.getCause();
@@ -195,10 +214,10 @@ public void testBadParamBeforeObjectBuilt() throws IOException {
     public void testConstructorArgsMustBeConfigured() throws IOException {
         class NoConstructorArgs {
         }
-        ConstructingObjectParser<NoConstructorArgs, ParseFieldMatcherSupplier> parser = new ConstructingObjectParser<>(
+        ConstructingObjectParser<NoConstructorArgs, Void> parser = new ConstructingObjectParser<>(
                 "constructor_args_required", (a) -> new NoConstructorArgs());
         try {
-            parser.apply(XContentType.JSON.xContent().createParser("{}"), null);
+            parser.apply(createParser(JsonXContent.jsonXContent, "{}"), null);
             fail("Expected AssertionError");
         } catch (AssertionError e) {
             assertEquals("[constructor_args_required] must configure at least on constructor argument. If it doesn't have any it should "
@@ -212,7 +231,7 @@ class NoConstructorArgs {
     public void testCalledOneTime() throws IOException {
         boolean ctorArgOptional = randomBoolean();
         class CalledOneTime {
-            public CalledOneTime(String yeah) {
+            CalledOneTime(String yeah) {
                 Matcher<String> yeahMatcher = equalTo("!");
                 if (ctorArgOptional) {
                     // either(yeahMatcher).or(nullValue) is broken by https://github.com/hamcrest/JavaHamcrest/issues/49
@@ -227,40 +246,71 @@ void setFoo(String foo) {
                 fooSet = true;
             }
         }
-        ConstructingObjectParser<CalledOneTime, ParseFieldMatcherSupplier> parser = new ConstructingObjectParser<>("one_time_test",
+        ConstructingObjectParser<CalledOneTime, Void> parser = new ConstructingObjectParser<>("one_time_test",
                 (a) -> new CalledOneTime((String) a[0]));
         parser.declareString(CalledOneTime::setFoo, new ParseField("foo"));
         parser.declareString(ctorArgOptional ? optionalConstructorArg() : constructorArg(), new ParseField("yeah"));
 
         // ctor arg first so we can test for the bug we found one time
-        XContentParser xcontent = XContentType.JSON.xContent().createParser(
+        XContentParser xcontent = createParser(JsonXContent.jsonXContent, 
                 "{\n"
               + "  \"yeah\": \"!\",\n"
               + "  \"foo\": \"foo\"\n"
               + "}");
-        CalledOneTime result = parser.apply(xcontent, MATCHER);
+        CalledOneTime result = parser.apply(xcontent, null);
         assertTrue(result.fooSet);
 
         // and ctor arg second just in case
-        xcontent = XContentType.JSON.xContent().createParser(
+        xcontent = createParser(JsonXContent.jsonXContent, 
                 "{\n"
               + "  \"foo\": \"foo\",\n"
               + "  \"yeah\": \"!\"\n"
               + "}");
-        result = parser.apply(xcontent, MATCHER);
+        result = parser.apply(xcontent, null);
         assertTrue(result.fooSet);
 
         if (ctorArgOptional) {
             // and without the constructor arg if we've made it optional
-            xcontent = XContentType.JSON.xContent().createParser(
+            xcontent = createParser(JsonXContent.jsonXContent, 
                     "{\n"
                   + "  \"foo\": \"foo\"\n"
                   + "}");
-            result = parser.apply(xcontent, MATCHER);
+            result = parser.apply(xcontent, null);
         }
         assertTrue(result.fooSet);
     }
 
+    public void testIgnoreUnknownFields() throws IOException {
+        XContentParser parser = createParser(JsonXContent.jsonXContent, 
+                  "{\n"
+                + "  \"test\" : \"foo\",\n"
+                + "  \"junk\" : 2\n"
+                + "}");
+        class TestStruct {
+            public final String test;
+            TestStruct(String test) {
+                this.test = test;
+            }
+        }
+        ConstructingObjectParser<TestStruct, Void> objectParser = new ConstructingObjectParser<>("foo", true, a ->
+                new TestStruct((String) a[0]));
+        objectParser.declareString(constructorArg(), new ParseField("test"));
+        TestStruct s = objectParser.apply(parser, null);
+        assertEquals(s.test, "foo");
+    }
+
+    public void testConstructObjectUsingContext() throws IOException {
+        XContentParser parser = createParser(JsonXContent.jsonXContent,
+            "{\n"
+                + "  \"animal\": \"dropbear\",\n"
+                + "  \"mineral\": -8\n"
+                + "}");
+        HasCtorArguments parsed = HasCtorArguments.PARSER_INT_CONTEXT.apply(parser, 42);
+        assertEquals(Integer.valueOf(42), parsed.vegetable);
+        assertEquals("dropbear", parsed.animal);
+        assertEquals(-8, parsed.mineral);
+    }
+
     private static class HasCtorArguments implements ToXContent {
         @Nullable
         final String animal;
@@ -273,7 +323,7 @@ private static class HasCtorArguments implements ToXContent {
         String c;
         boolean d;
 
-        public HasCtorArguments(@Nullable String animal, @Nullable Integer vegetable) {
+        HasCtorArguments(@Nullable String animal, @Nullable Integer vegetable) {
             this.animal = animal;
             this.vegetable = vegetable;
         }
@@ -336,27 +386,40 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
          * It is normal just to declare a single PARSER but we use a couple of different parsers for testing so we have all of these. Don't
          * this this style is normal just because it is in the test.
          */
-        public static final ConstructingObjectParser<HasCtorArguments, ParseFieldMatcherSupplier> PARSER = buildParser(true, true);
-        public static final ConstructingObjectParser<HasCtorArguments, ParseFieldMatcherSupplier> PARSER_VEGETABLE_OPTIONAL = buildParser(
-                true, false);
-        public static final ConstructingObjectParser<HasCtorArguments, ParseFieldMatcherSupplier> PARSER_ALL_OPTIONAL = buildParser(false,
-                false);
-        public static final List<ConstructingObjectParser<HasCtorArguments, ParseFieldMatcherSupplier>> ALL_PARSERS = unmodifiableList(
+        public static final ConstructingObjectParser<HasCtorArguments, Void> PARSER = buildParser(true, true);
+        public static final ConstructingObjectParser<HasCtorArguments, Void> PARSER_VEGETABLE_OPTIONAL = buildParser(true, false);
+        public static final ConstructingObjectParser<HasCtorArguments, Void> PARSER_ALL_OPTIONAL = buildParser(false, false);
+
+        public static final List<ConstructingObjectParser<HasCtorArguments, Void>> ALL_PARSERS = unmodifiableList(
                 Arrays.asList(PARSER, PARSER_VEGETABLE_OPTIONAL, PARSER_ALL_OPTIONAL));
 
-        private static ConstructingObjectParser<HasCtorArguments, ParseFieldMatcherSupplier> buildParser(boolean animalRequired,
+        public static final ConstructingObjectParser<HasCtorArguments, Integer> PARSER_INT_CONTEXT = buildContextParser();
+
+        private static ConstructingObjectParser<HasCtorArguments, Void> buildParser(boolean animalRequired,
                 boolean vegetableRequired) {
-            ConstructingObjectParser<HasCtorArguments, ParseFieldMatcherSupplier> parser = new ConstructingObjectParser<>(
+            ConstructingObjectParser<HasCtorArguments, Void> parser = new ConstructingObjectParser<>(
                     "has_required_arguments", a -> new HasCtorArguments((String) a[0], (Integer) a[1]));
             parser.declareString(animalRequired ? constructorArg() : optionalConstructorArg(), new ParseField("animal"));
             parser.declareInt(vegetableRequired ? constructorArg() : optionalConstructorArg(), new ParseField("vegetable"));
+            declareSetters(parser);
+            return parser;
+        }
+
+        private static ConstructingObjectParser<HasCtorArguments, Integer> buildContextParser() {
+            ConstructingObjectParser<HasCtorArguments, Integer> parser = new ConstructingObjectParser<>(
+                    "has_required_arguments", false, (args, ctx) -> new HasCtorArguments((String) args[0], ctx));
+            parser.declareString(constructorArg(), new ParseField("animal"));
+            declareSetters(parser);
+            return parser;
+        }
+
+        private static void declareSetters(ConstructingObjectParser<HasCtorArguments, ?> parser) {
             parser.declareInt(HasCtorArguments::setMineral, new ParseField("mineral"));
             parser.declareInt(HasCtorArguments::setFruit, new ParseField("fruit"));
             parser.declareString(HasCtorArguments::setA, new ParseField("a"));
             parser.declareString(HasCtorArguments::setB, new ParseField("b"));
             parser.declareString(HasCtorArguments::setC, new ParseField("c"));
             parser.declareBoolean(HasCtorArguments::setD, new ParseField("d"));
-            return parser;
         }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java
index a8d26e87ecf22..0d879e4813116 100644
--- a/core/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java
+++ b/core/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java
@@ -18,27 +18,28 @@
  */
 package org.elasticsearch.common.xcontent;
 
-import static org.hamcrest.Matchers.hasSize;
+import org.elasticsearch.common.CheckedFunction;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.xcontent.ObjectParser.NamedObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.net.URI;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
-import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.xcontent.ObjectParser.NamedObjectParser;
-import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
-import org.elasticsearch.test.ESTestCase;
+import static org.hamcrest.Matchers.hasSize;
 
 public class ObjectParserTests extends ESTestCase {
 
-    private static final ParseFieldMatcherSupplier STRICT_PARSING = () -> ParseFieldMatcher.STRICT;
-
     public void testBasics() throws IOException {
-        XContentParser parser = XContentType.JSON.xContent().createParser(
+        XContentParser parser = createParser(JsonXContent.jsonXContent, 
                   "{\n"
                 + "  \"test\" : \"foo\",\n"
                 + "  \"test_number\" : 2,\n"
@@ -56,13 +57,13 @@ public void setInts(List<Integer> ints) {
                 this.ints = ints;
             }
         }
-        ObjectParser<TestStruct, ParseFieldMatcherSupplier> objectParser = new ObjectParser<>("foo");
+        ObjectParser<TestStruct, Void> objectParser = new ObjectParser<>("foo");
         TestStruct s = new TestStruct();
 
         objectParser.declareField((i, c, x) -> c.test = i.text(), new ParseField("test"), ObjectParser.ValueType.STRING);
         objectParser.declareInt(TestStruct::setTestNumber, new ParseField("test_number"));
         objectParser.declareIntArray(TestStruct::setInts, new ParseField("test_array"));
-        objectParser.parse(parser, s, STRICT_PARSING);
+        objectParser.parse(parser, s, null);
         assertEquals(s.test, "foo");
         assertEquals(s.testNumber, 2);
         assertEquals(s.ints, Arrays.asList(1, 2, 3, 4));
@@ -72,85 +73,166 @@ public void setInts(List<Integer> ints) {
                 + "FieldParser{preferred_name=test_number, supportedTokens=[VALUE_STRING, VALUE_NUMBER], type=INT}]}");
     }
 
+    public void testNullDeclares() {
+        ObjectParser<Void, Void> objectParser = new ObjectParser<>("foo");
+        Exception e = expectThrows(IllegalArgumentException.class,
+                () -> objectParser.declareField(null, (r, c) -> null, new ParseField("test"), ObjectParser.ValueType.STRING));
+        assertEquals("[consumer] is required", e.getMessage());
+        e = expectThrows(IllegalArgumentException.class, () -> objectParser.declareField(
+                (o, v) -> {}, (ContextParser<Void, Object>) null,
+                new ParseField("test"), ObjectParser.ValueType.STRING));
+        assertEquals("[parser] is required", e.getMessage());
+        e = expectThrows(IllegalArgumentException.class, () -> objectParser.declareField(
+                (o, v) -> {}, (CheckedFunction<XContentParser, Object, IOException>) null,
+                new ParseField("test"), ObjectParser.ValueType.STRING));
+        assertEquals("[parser] is required", e.getMessage());
+        e = expectThrows(IllegalArgumentException.class, () -> objectParser.declareField(
+                (o, v) -> {}, (r, c) -> null, null, ObjectParser.ValueType.STRING));
+        assertEquals("[parseField] is required", e.getMessage());
+        e = expectThrows(IllegalArgumentException.class, () -> objectParser.declareField(
+                (o, v) -> {}, (r, c) -> null, new ParseField("test"), null));
+        assertEquals("[type] is required", e.getMessage());
+    }
+
     public void testObjectOrDefault() throws IOException {
-        XContentParser parser = XContentType.JSON.xContent().createParser("{\"object\" : { \"test\": 2}}");
-        ObjectParser<StaticTestStruct, ParseFieldMatcherSupplier> objectParser = new ObjectParser<>("foo", StaticTestStruct::new);
+        XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"object\" : { \"test\": 2}}");
+        ObjectParser<StaticTestStruct, Void> objectParser = new ObjectParser<>("foo", StaticTestStruct::new);
         objectParser.declareInt(StaticTestStruct::setTest, new ParseField("test"));
         objectParser.declareObjectOrDefault(StaticTestStruct::setObject, objectParser, StaticTestStruct::new, new ParseField("object"));
-        StaticTestStruct s = objectParser.parse(parser, STRICT_PARSING);
+        StaticTestStruct s = objectParser.parse(parser, null);
         assertEquals(s.object.test, 2);
-        parser = XContentType.JSON.xContent().createParser("{\"object\" : false }");
-        s = objectParser.parse(parser, STRICT_PARSING);
+        parser = createParser(JsonXContent.jsonXContent, "{\"object\" : false }");
+        s = objectParser.parse(parser, null);
         assertNull(s.object);
 
-        parser = XContentType.JSON.xContent().createParser("{\"object\" : true }");
-        s = objectParser.parse(parser, STRICT_PARSING);
+        parser = createParser(JsonXContent.jsonXContent, "{\"object\" : true }");
+        s = objectParser.parse(parser, null);
         assertNotNull(s.object);
         assertEquals(s.object.test, 0);
+    }
 
+    /**
+     * This test ensures we can use a classic pull-parsing parser
+     * together with the object parser
+     */
+    public void testUseClassicPullParsingSubParser() throws IOException {
+        class ClassicParser {
+            URI parseURI(XContentParser parser) throws IOException {
+                String fieldName = null;
+                String host = "";
+                int port = 0;
+                XContentParser.Token token;
+                while (( token = parser.currentToken()) != XContentParser.Token.END_OBJECT) {
+                    if (token == XContentParser.Token.FIELD_NAME) {
+                        fieldName = parser.currentName();
+                    } else if (token == XContentParser.Token.VALUE_STRING){
+                        if (fieldName.equals("host")) {
+                            host = parser.text();
+                        } else {
+                            throw new IllegalStateException("boom");
+                        }
+                    } else if (token == XContentParser.Token.VALUE_NUMBER){
+                        if (fieldName.equals("port")) {
+                            port = parser.intValue();
+                        } else {
+                            throw new IllegalStateException("boom");
+                        }
+                    }
+                    parser.nextToken();
+                }
+                return URI.create(host + ":" + port);
+            }
+        }
+        class Foo {
+            public String name;
+            public URI uri;
+            public void setName(String name) {
+                this.name = name;
+            }
+
+            public void setURI(URI uri) {
+                this.uri = uri;
+            }
+        }
+
+        class CustomParseContext {
+
+            public final ClassicParser parser;
+
+            CustomParseContext(ClassicParser parser) {
+                this.parser = parser;
+            }
+
+            public URI parseURI(XContentParser parser) {
+                try {
+                    return this.parser.parseURI(parser);
+                } catch (IOException e) {
+                    throw new UncheckedIOException(e);
+                }
+            }
+        }
+        XContentParser parser = createParser(JsonXContent.jsonXContent,
+                "{\"url\" : { \"host\": \"http://foobar\", \"port\" : 80}, \"name\" : \"foobarbaz\"}");
+        ObjectParser<Foo, CustomParseContext> objectParser = new ObjectParser<>("foo");
+        objectParser.declareString(Foo::setName, new ParseField("name"));
+        objectParser.declareObjectOrDefault(Foo::setURI, (p, s) -> s.parseURI(p), () -> null, new ParseField("url"));
+        Foo s = objectParser.parse(parser, new Foo(), new CustomParseContext(new ClassicParser()));
+        assertEquals(s.uri.getHost(),  "foobar");
+        assertEquals(s.uri.getPort(),  80);
+        assertEquals(s.name, "foobarbaz");
     }
 
     public void testExceptions() throws IOException {
-        XContentParser parser = XContentType.JSON.xContent().createParser("{\"test\" : \"foo\"}");
+        XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"test\" : \"foo\"}");
         class TestStruct {
             public void setTest(int test) {
             }
         }
-        ObjectParser<TestStruct, ParseFieldMatcherSupplier> objectParser = new ObjectParser<>("the_parser");
+        ObjectParser<TestStruct, Void> objectParser = new ObjectParser<>("the_parser");
         TestStruct s = new TestStruct();
         objectParser.declareInt(TestStruct::setTest, new ParseField("test"));
 
         try {
-            objectParser.parse(parser, s, STRICT_PARSING);
+            objectParser.parse(parser, s, null);
             fail("numeric value expected");
         } catch (ParsingException ex) {
             assertEquals(ex.getMessage(), "[the_parser] failed to parse field [test]");
             assertTrue(ex.getCause() instanceof NumberFormatException);
         }
 
-        parser = XContentType.JSON.xContent().createParser("{\"not_supported_field\" : \"foo\"}");
+        parser = createParser(JsonXContent.jsonXContent, "{\"not_supported_field\" : \"foo\"}");
         try {
-            objectParser.parse(parser, s, STRICT_PARSING);
+            objectParser.parse(parser, s, null);
             fail("field not supported");
         } catch (IllegalArgumentException ex) {
             assertEquals(ex.getMessage(), "[the_parser] unknown field [not_supported_field], parser not found");
         }
     }
 
-    public void testDeprecationFail() throws IOException {
-        XContentParser parser = XContentType.JSON.xContent().createParser("{\"old_test\" : \"foo\"}");
+    public void testDeprecationWarnings() throws IOException {
         class TestStruct {
             public String test;
         }
-        ObjectParser<TestStruct, ParseFieldMatcherSupplier> objectParser = new ObjectParser<>("foo");
+        ObjectParser<TestStruct, Void> objectParser = new ObjectParser<>("foo");
         TestStruct s = new TestStruct();
-
+        XContentParser parser = createParser(XContentType.JSON.xContent(), "{\"old_test\" : \"foo\"}");
         objectParser.declareField((i, v, c) -> v.test = i.text(), new ParseField("test", "old_test"), ObjectParser.ValueType.STRING);
-
-        try {
-            objectParser.parse(parser, s, STRICT_PARSING);
-            fail("deprecated value");
-        } catch (IllegalArgumentException ex) {
-            assertEquals(ex.getMessage(), "Deprecated field [old_test] used, expected [test] instead");
-
-        }
-        assertNull(s.test);
-        parser = XContentType.JSON.xContent().createParser("{\"old_test\" : \"foo\"}");
-        objectParser.parse(parser, s, () -> ParseFieldMatcher.EMPTY);
+        objectParser.parse(parser, s, null);
         assertEquals("foo", s.test);
+        assertWarnings("Deprecated field [old_test] used, expected [test] instead");
     }
 
     public void testFailOnValueType() throws IOException {
-        XContentParser parser = XContentType.JSON.xContent().createParser("{\"numeric_value\" : false}");
+        XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"numeric_value\" : false}");
         class TestStruct {
             public String test;
         }
-        ObjectParser<TestStruct, ParseFieldMatcherSupplier> objectParser = new ObjectParser<>("foo");
+        ObjectParser<TestStruct, Void> objectParser = new ObjectParser<>("foo");
         TestStruct s = new TestStruct();
 
         objectParser.declareField((i, c, x) -> c.test = i.text(), new ParseField("numeric_value"), ObjectParser.ValueType.FLOAT);
         try {
-            objectParser.parse(parser, s, STRICT_PARSING);
+            objectParser.parse(parser, s, null);
             fail("wrong type - must be number");
         } catch (IllegalArgumentException ex) {
             assertEquals(ex.getMessage(), "[foo] numeric_value doesn't support values of type: VALUE_BOOLEAN");
@@ -158,45 +240,45 @@ class TestStruct {
     }
 
     public void testParseNested() throws IOException {
-        XContentParser parser = XContentType.JSON.xContent().createParser("{ \"test\" : 1, \"object\" : { \"test\": 2}}");
+        XContentParser parser = createParser(JsonXContent.jsonXContent, "{ \"test\" : 1, \"object\" : { \"test\": 2}}");
         class TestStruct {
             public int test;
             TestStruct object;
         }
-        ObjectParser<TestStruct, ParseFieldMatcherSupplier> objectParser = new ObjectParser<>("foo");
+        ObjectParser<TestStruct, Void> objectParser = new ObjectParser<>("foo");
         TestStruct s = new TestStruct();
         s.object = new TestStruct();
         objectParser.declareField((i, c, x) -> c.test = i.intValue(), new ParseField("test"), ValueType.INT);
-        objectParser.declareField((i, c, x) -> objectParser.parse(parser, c.object, STRICT_PARSING), new ParseField("object"),
+        objectParser.declareField((i, c, x) -> objectParser.parse(parser, c.object, null), new ParseField("object"),
                 ValueType.OBJECT);
-        objectParser.parse(parser, s, STRICT_PARSING);
+        objectParser.parse(parser, s, null);
         assertEquals(s.test, 1);
         assertEquals(s.object.test, 2);
     }
 
     public void testParseNestedShortcut() throws IOException {
-        XContentParser parser = XContentType.JSON.xContent().createParser("{ \"test\" : 1, \"object\" : { \"test\": 2}}");
-        ObjectParser<StaticTestStruct, ParseFieldMatcherSupplier> objectParser = new ObjectParser<>("foo", StaticTestStruct::new);
+        XContentParser parser = createParser(JsonXContent.jsonXContent, "{ \"test\" : 1, \"object\" : { \"test\": 2}}");
+        ObjectParser<StaticTestStruct, Void> objectParser = new ObjectParser<>("foo", StaticTestStruct::new);
         objectParser.declareInt(StaticTestStruct::setTest, new ParseField("test"));
         objectParser.declareObject(StaticTestStruct::setObject, objectParser, new ParseField("object"));
-        StaticTestStruct s = objectParser.parse(parser, STRICT_PARSING);
+        StaticTestStruct s = objectParser.parse(parser, null);
         assertEquals(s.test, 1);
         assertEquals(s.object.test, 2);
     }
 
     public void testEmptyObject() throws IOException {
-        XContentParser parser = XContentType.JSON.xContent().createParser("{\"object\" : {}}");
-        ObjectParser<StaticTestStruct, ParseFieldMatcherSupplier> objectParser = new ObjectParser<>("foo", StaticTestStruct::new);
+        XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"object\" : {}}");
+        ObjectParser<StaticTestStruct, Void> objectParser = new ObjectParser<>("foo", StaticTestStruct::new);
         objectParser.declareObject(StaticTestStruct::setObject, objectParser, new ParseField("object"));
-        StaticTestStruct s = objectParser.parse(parser, STRICT_PARSING);
+        StaticTestStruct s = objectParser.parse(parser, null);
         assertNotNull(s.object);
     }
 
     public void testEmptyObjectInArray() throws IOException {
-        XContentParser parser = XContentType.JSON.xContent().createParser("{\"object_array\" : [{}]}");
-        ObjectParser<StaticTestStruct, ParseFieldMatcherSupplier> objectParser = new ObjectParser<>("foo", StaticTestStruct::new);
+        XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"object_array\" : [{}]}");
+        ObjectParser<StaticTestStruct, Void> objectParser = new ObjectParser<>("foo", StaticTestStruct::new);
         objectParser.declareObjectArray(StaticTestStruct::setObjectArray, objectParser, new ParseField("object_array"));
-        StaticTestStruct s = objectParser.parse(parser, STRICT_PARSING);
+        StaticTestStruct s = objectParser.parse(parser, null);
         assertNotNull(s.objectArray);
     }
 
@@ -230,10 +312,10 @@ public void set(TestEnum value) {
                 test = value;
             }
         }
-        XContentParser parser = XContentType.JSON.xContent().createParser("{ \"test\" : \"FOO\" }");
-        ObjectParser<TestStruct, ParseFieldMatcherSupplier> objectParser = new ObjectParser<>("foo");
+        XContentParser parser = createParser(JsonXContent.jsonXContent, "{ \"test\" : \"FOO\" }");
+        ObjectParser<TestStruct, Void> objectParser = new ObjectParser<>("foo");
         objectParser.declareString((struct, value) -> struct.set(TestEnum.valueOf(value)), new ParseField("test"));
-        TestStruct s = objectParser.parse(parser, new TestStruct(), STRICT_PARSING);
+        TestStruct s = objectParser.parse(parser, new TestStruct(), null);
         assertEquals(s.test, TestEnum.FOO);
     }
 
@@ -271,10 +353,14 @@ public void testAllVariants() throws IOException {
             builder.field("string_array_field", "5");
         }
         boolean nullValue = randomBoolean();
-        builder.field("boolean_field", nullValue);
+        if (randomBoolean()) {
+            builder.field("boolean_field", nullValue);
+        } else {
+            builder.field("boolean_field", Boolean.toString(nullValue));
+        }
         builder.field("string_or_null", nullValue ? null : "5");
         builder.endObject();
-        XContentParser parser = XContentType.JSON.xContent().createParser(builder.string());
+        XContentParser parser = createParser(JsonXContent.jsonXContent, builder.string());
         class TestStruct {
             int int_field;
             long long_field;
@@ -327,7 +413,7 @@ public void setString_or_null(String string_or_null) {
                 this.string_or_null = string_or_null;
             }
         }
-        ObjectParser<TestStruct, ParseFieldMatcherSupplier> objectParser = new ObjectParser<>("foo");
+        ObjectParser<TestStruct, Void> objectParser = new ObjectParser<>("foo");
         objectParser.declareInt(TestStruct::setInt_field, new ParseField("int_field"));
         objectParser.declareIntArray(TestStruct::setInt_array_field, new ParseField("int_array_field"));
         objectParser.declareLong(TestStruct::setLong_field, new ParseField("long_field"));
@@ -341,20 +427,20 @@ public void setString_or_null(String string_or_null) {
 
         objectParser.declareStringOrNull(TestStruct::setString_or_null, new ParseField("string_or_null"));
         objectParser.declareBoolean(TestStruct::setNull_value, new ParseField("boolean_field"));
-        TestStruct parse = objectParser.parse(parser, new TestStruct(), STRICT_PARSING);
-        assertArrayEquals(parse.double_array_field.toArray(), Arrays.asList(2.1d).toArray());
+        TestStruct parse = objectParser.parse(parser, new TestStruct(), null);
+        assertArrayEquals(parse.double_array_field.toArray(), Collections.singletonList(2.1d).toArray());
         assertEquals(parse.double_field, 2.1d, 0.0d);
 
-        assertArrayEquals(parse.long_array_field.toArray(), Arrays.asList(4L).toArray());
+        assertArrayEquals(parse.long_array_field.toArray(), Collections.singletonList(4L).toArray());
         assertEquals(parse.long_field, 4L);
 
-        assertArrayEquals(parse.string_array_field.toArray(), Arrays.asList("5").toArray());
+        assertArrayEquals(parse.string_array_field.toArray(), Collections.singletonList("5").toArray());
         assertEquals(parse.string_field, "5");
 
-        assertArrayEquals(parse.int_array_field.toArray(), Arrays.asList(1).toArray());
+        assertArrayEquals(parse.int_array_field.toArray(), Collections.singletonList(1).toArray());
         assertEquals(parse.int_field, 1);
 
-        assertArrayEquals(parse.float_array_field.toArray(), Arrays.asList(3.1f).toArray());
+        assertArrayEquals(parse.float_array_field.toArray(), Collections.singletonList(3.1f).toArray());
         assertEquals(parse.float_field, 3.1f, 0.0f);
 
         assertEquals(nullValue, parse.null_value);
@@ -366,33 +452,33 @@ public void setString_or_null(String string_or_null) {
     }
 
     public void testParseNamedObject() throws IOException {
-        XContentParser parser = XContentType.JSON.xContent().createParser(
+        XContentParser parser = createParser(JsonXContent.jsonXContent, 
                   "{\"named\": {\n"
                 + "  \"a\": {}"
                 + "}}");
-        NamedObjectHolder h = NamedObjectHolder.PARSER.apply(parser, STRICT_PARSING);
+        NamedObjectHolder h = NamedObjectHolder.PARSER.apply(parser, null);
         assertThat(h.named, hasSize(1));
         assertEquals("a", h.named.get(0).name);
         assertFalse(h.namedSuppliedInOrder);
     }
 
     public void testParseNamedObjectInOrder() throws IOException {
-        XContentParser parser = XContentType.JSON.xContent().createParser(
+        XContentParser parser = createParser(JsonXContent.jsonXContent, 
                   "{\"named\": [\n"
                 + "  {\"a\": {}}"
                 + "]}");
-        NamedObjectHolder h = NamedObjectHolder.PARSER.apply(parser, STRICT_PARSING);
+        NamedObjectHolder h = NamedObjectHolder.PARSER.apply(parser, null);
         assertThat(h.named, hasSize(1));
         assertEquals("a", h.named.get(0).name);
         assertTrue(h.namedSuppliedInOrder);
     }
 
     public void testParseNamedObjectTwoFieldsInArray() throws IOException {
-        XContentParser parser = XContentType.JSON.xContent().createParser(
+        XContentParser parser = createParser(JsonXContent.jsonXContent, 
                   "{\"named\": [\n"
                 + "  {\"a\": {}, \"b\": {}}"
                 + "]}");
-        ParsingException e = expectThrows(ParsingException.class, () -> NamedObjectHolder.PARSER.apply(parser, STRICT_PARSING));
+        ParsingException e = expectThrows(ParsingException.class, () -> NamedObjectHolder.PARSER.apply(parser, null));
         assertEquals("[named_object_holder] failed to parse field [named]", e.getMessage());
         assertEquals(
                 "[named] can be a single object with any number of fields or an array where each entry is an object with a single field",
@@ -400,11 +486,11 @@ public void testParseNamedObjectTwoFieldsInArray() throws IOException {
     }
 
     public void testParseNamedObjectNoFieldsInArray() throws IOException {
-        XContentParser parser = XContentType.JSON.xContent().createParser(
+        XContentParser parser = createParser(JsonXContent.jsonXContent, 
                   "{\"named\": [\n"
                 + "  {}"
                 + "]}");
-        ParsingException e = expectThrows(ParsingException.class, () -> NamedObjectHolder.PARSER.apply(parser, STRICT_PARSING));
+        ParsingException e = expectThrows(ParsingException.class, () -> NamedObjectHolder.PARSER.apply(parser, null));
         assertEquals("[named_object_holder] failed to parse field [named]", e.getMessage());
         assertEquals(
                 "[named] can be a single object with any number of fields or an array where each entry is an object with a single field",
@@ -412,11 +498,11 @@ public void testParseNamedObjectNoFieldsInArray() throws IOException {
     }
 
     public void testParseNamedObjectJunkInArray() throws IOException {
-        XContentParser parser = XContentType.JSON.xContent().createParser(
+        XContentParser parser = createParser(JsonXContent.jsonXContent, 
                   "{\"named\": [\n"
                 + "  \"junk\""
                 + "]}");
-        ParsingException e = expectThrows(ParsingException.class, () -> NamedObjectHolder.PARSER.apply(parser, STRICT_PARSING));
+        ParsingException e = expectThrows(ParsingException.class, () -> NamedObjectHolder.PARSER.apply(parser, null));
         assertEquals("[named_object_holder] failed to parse field [named]", e.getMessage());
         assertEquals(
                 "[named] can be a single object with any number of fields or an array where each entry is an object with a single field",
@@ -424,24 +510,95 @@ public void testParseNamedObjectJunkInArray() throws IOException {
     }
 
     public void testParseNamedObjectInOrderNotSupported() throws IOException {
-        XContentParser parser = XContentType.JSON.xContent().createParser(
+        XContentParser parser = createParser(JsonXContent.jsonXContent,
                   "{\"named\": [\n"
                 + "  {\"a\": {}}"
                 + "]}");
 
         // Create our own parser for this test so we can disable support for the "ordered" mode specified by the array above
-        ObjectParser<NamedObjectHolder, ParseFieldMatcherSupplier> objectParser = new ObjectParser<>("named_object_holder",
+        ObjectParser<NamedObjectHolder, Void> objectParser = new ObjectParser<>("named_object_holder",
                 NamedObjectHolder::new);
         objectParser.declareNamedObjects(NamedObjectHolder::setNamed, NamedObject.PARSER, new ParseField("named"));
 
         // Now firing the xml through it fails
-        ParsingException e = expectThrows(ParsingException.class, () -> objectParser.apply(parser, STRICT_PARSING));
+        ParsingException e = expectThrows(ParsingException.class, () -> objectParser.apply(parser, null));
         assertEquals("[named_object_holder] failed to parse field [named]", e.getMessage());
         assertEquals("[named] doesn't support arrays. Use a single object with multiple fields.", e.getCause().getMessage());
     }
 
+    public void testIgnoreUnknownFields() throws IOException {
+        XContentBuilder b = XContentBuilder.builder(XContentType.JSON.xContent());
+        b.startObject();
+        {
+            b.field("test", "foo");
+            b.field("junk", 2);
+        }
+        b.endObject();
+        b = shuffleXContent(b);
+        XContentParser parser = createParser(JsonXContent.jsonXContent, b.bytes());
+
+        class TestStruct {
+            public String test;
+        }
+        ObjectParser<TestStruct, Void> objectParser = new ObjectParser<>("foo", true, null);
+        objectParser.declareField((i, c, x) -> c.test = i.text(), new ParseField("test"), ObjectParser.ValueType.STRING);
+        TestStruct s = objectParser.parse(parser, new TestStruct(), null);
+        assertEquals(s.test, "foo");
+    }
+
+    public void testIgnoreUnknownObjects() throws IOException {
+        XContentBuilder b = XContentBuilder.builder(XContentType.JSON.xContent());
+        b.startObject();
+        {
+            b.field("test", "foo");
+            b.startObject("junk");
+            {
+                b.field("really", "junk");
+            }
+            b.endObject();
+        }
+        b.endObject();
+        b = shuffleXContent(b);
+        XContentParser parser = createParser(JsonXContent.jsonXContent, b.bytes());
+
+        class TestStruct {
+            public String test;
+        }
+        ObjectParser<TestStruct, Void> objectParser = new ObjectParser<>("foo", true, null);
+        objectParser.declareField((i, c, x) -> c.test = i.text(), new ParseField("test"), ObjectParser.ValueType.STRING);
+        TestStruct s = objectParser.parse(parser, new TestStruct(), null);
+        assertEquals(s.test, "foo");
+    }
+
+    public void testIgnoreUnknownArrays() throws IOException {
+        XContentBuilder b = XContentBuilder.builder(XContentType.JSON.xContent());
+        b.startObject();
+        {
+            b.field("test", "foo");
+            b.startArray("junk");
+            {
+                b.startObject();
+                {
+                    b.field("really", "junk");
+                }
+                b.endObject();
+            }
+            b.endArray();
+        }
+        b.endObject();
+        b = shuffleXContent(b);
+        XContentParser parser = createParser(JsonXContent.jsonXContent, b.bytes());
+        class TestStruct {
+            public String test;
+        }
+        ObjectParser<TestStruct, Void> objectParser = new ObjectParser<>("foo", true, null);
+        objectParser.declareField((i, c, x) -> c.test = i.text(), new ParseField("test"), ObjectParser.ValueType.STRING);
+        TestStruct s = objectParser.parse(parser, new TestStruct(), null);
+        assertEquals(s.test, "foo");
+    }
+
     static class NamedObjectHolder {
-        public static final ObjectParser<NamedObjectHolder, ParseFieldMatcherSupplier> PARSER = new ObjectParser<>("named_object_holder",
+        public static final ObjectParser<NamedObjectHolder, Void> PARSER = new ObjectParser<>("named_object_holder",
                 NamedObjectHolder::new);
         static {
             PARSER.declareNamedObjects(NamedObjectHolder::setNamed, NamedObject.PARSER, NamedObjectHolder::keepNamedInOrder,
@@ -461,11 +618,11 @@ public void keepNamedInOrder() {
     }
 
     public static class NamedObject {
-        public static final NamedObjectParser<NamedObject, ParseFieldMatcherSupplier> PARSER;
+        public static final NamedObjectParser<NamedObject, Void> PARSER;
         static {
-            ObjectParser<NamedObject, ParseFieldMatcherSupplier> parser = new ObjectParser<>("named");
+            ObjectParser<NamedObject, Void> parser = new ObjectParser<>("named");
             parser.declareInt(NamedObject::setFoo, new ParseField("foo"));
-            PARSER = (XContentParser p, ParseFieldMatcherSupplier v, String name) -> parser.parse(p, new NamedObject(name), STRICT_PARSING);
+            PARSER = (XContentParser p, Void v, String name) -> parser.parse(p, new NamedObject(name), null);
         }
 
         final String name;
diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/UnknownNamedObjectExceptionTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/UnknownNamedObjectExceptionTests.java
new file mode 100644
index 0000000000000..4fcc16416b56f
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/common/xcontent/UnknownNamedObjectExceptionTests.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.xcontent;
+
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry.UnknownNamedObjectException;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+public class UnknownNamedObjectExceptionTests extends ESTestCase {
+    public void testRoundTrip() throws IOException {
+        XContentLocation location = new XContentLocation(between(1, 1000), between(1, 1000));
+        UnknownNamedObjectException created = new UnknownNamedObjectException(location, UnknownNamedObjectExceptionTests.class,
+                randomAlphaOfLength(5));
+        UnknownNamedObjectException roundTripped;
+
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            created.writeTo(out);
+            try (StreamInput in = out.bytes().streamInput()) {
+                roundTripped = new UnknownNamedObjectException(in);
+            }
+        }
+        assertEquals(created.getMessage(), roundTripped.getMessage());
+        assertEquals(created.getLineNumber(), roundTripped.getLineNumber());
+        assertEquals(created.getColumnNumber(), roundTripped.getColumnNumber());
+        assertEquals(created.getCategoryClass(), roundTripped.getCategoryClass());
+        assertEquals(created.getName(), roundTripped.getName());
+    }
+
+    public void testStatusCode() {
+        XContentLocation location = new XContentLocation(between(1, 1000), between(1, 1000));
+        UnknownNamedObjectException e = new UnknownNamedObjectException(location, UnknownNamedObjectExceptionTests.class,
+                randomAlphaOfLength(5));
+        assertEquals(RestStatus.BAD_REQUEST, e.status());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/XContentParserTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/XContentParserTests.java
index cce349f417c85..a83bfc9294684 100644
--- a/core/src/test/java/org/elasticsearch/common/xcontent/XContentParserTests.java
+++ b/core/src/test/java/org/elasticsearch/common/xcontent/XContentParserTests.java
@@ -19,17 +19,25 @@
 
 package org.elasticsearch.common.xcontent;
 
+import com.fasterxml.jackson.core.JsonParseException;
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.singletonMap;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.isIn;
 import static org.hamcrest.Matchers.nullValue;
 
 public class XContentParserTests extends ESTestCase {
@@ -55,8 +63,8 @@ public void testReadListThrowsException() throws IOException {
     }
 
     @SuppressWarnings("unchecked")
-    private static <T> List<T> readList(String source) throws IOException {
-        try (XContentParser parser = XContentType.JSON.xContent().createParser(source)) {
+    private <T> List<T> readList(String source) throws IOException {
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, source)) {
             XContentParser.Token token = parser.nextToken();
             assertThat(token, equalTo(XContentParser.Token.START_OBJECT));
             token = parser.nextToken();
@@ -75,4 +83,134 @@ private void assertReadListThrowsException(String source) {
             assertThat(e.getMessage(), containsString("Failed to parse list"));
         }
     }
+
+    public void testReadMapStrings() throws IOException {
+        Map<String, String> map = readMapStrings("{\"foo\": {\"kbar\":\"vbar\"}}");
+        assertThat(map.get("kbar"), equalTo("vbar"));
+        assertThat(map.size(), equalTo(1));
+        map = readMapStrings("{\"foo\": {\"kbar\":\"vbar\", \"kbaz\":\"vbaz\"}}");
+        assertThat(map.get("kbar"), equalTo("vbar"));
+        assertThat(map.get("kbaz"), equalTo("vbaz"));
+        assertThat(map.size(), equalTo(2));
+        map = readMapStrings("{\"foo\": {}}");
+        assertThat(map.size(), equalTo(0));
+    }
+
+    private Map<String, String> readMapStrings(String source) throws IOException {
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, source)) {
+            XContentParser.Token token = parser.nextToken();
+            assertThat(token, equalTo(XContentParser.Token.START_OBJECT));
+            token = parser.nextToken();
+            assertThat(token, equalTo(XContentParser.Token.FIELD_NAME));
+            assertThat(parser.currentName(), equalTo("foo"));
+            token = parser.nextToken();
+            assertThat(token, equalTo(XContentParser.Token.START_OBJECT));
+            return randomBoolean() ? parser.mapStringsOrdered() : parser.mapStrings();
+        }
+    }
+
+    public void testReadBooleans() throws IOException {
+        String falsy = randomFrom("\"false\"", "false");
+        String truthy = randomFrom("\"true\"", "true");
+
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"foo\": " + falsy + ", \"bar\": " + truthy + "}")) {
+            XContentParser.Token token = parser.nextToken();
+            assertThat(token, equalTo(XContentParser.Token.START_OBJECT));
+            token = parser.nextToken();
+            assertThat(token, equalTo(XContentParser.Token.FIELD_NAME));
+            assertThat(parser.currentName(), equalTo("foo"));
+            token = parser.nextToken();
+            assertThat(token, isIn(Arrays.asList(XContentParser.Token.VALUE_STRING, XContentParser.Token.VALUE_BOOLEAN)));
+            assertTrue(parser.isBooleanValue());
+            assertFalse(parser.booleanValue());
+
+            token = parser.nextToken();
+            assertThat(token, equalTo(XContentParser.Token.FIELD_NAME));
+            assertThat(parser.currentName(), equalTo("bar"));
+            token = parser.nextToken();
+            assertThat(token, isIn(Arrays.asList(XContentParser.Token.VALUE_STRING, XContentParser.Token.VALUE_BOOLEAN)));
+            assertTrue(parser.isBooleanValue());
+            assertTrue(parser.booleanValue());
+        }
+    }
+
+    public void testEmptyList() throws IOException {
+        XContentBuilder builder = XContentFactory.jsonBuilder().startObject()
+                .startArray("some_array")
+                .endArray().endObject();
+
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, builder.string())) {
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
+            assertEquals("some_array", parser.currentName());
+            if (random().nextBoolean()) {
+                // sometimes read the start array token, sometimes not
+                assertEquals(XContentParser.Token.START_ARRAY, parser.nextToken());
+            }
+            assertEquals(Collections.emptyList(), parser.list());
+        }
+    }
+
+    public void testSimpleList() throws IOException {
+        XContentBuilder builder = XContentFactory.jsonBuilder().startObject()
+                .startArray("some_array")
+                .value(1)
+                .value(3)
+                .value(0)
+                .endArray().endObject();
+
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, builder.string())) {
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
+            assertEquals("some_array", parser.currentName());
+            if (random().nextBoolean()) {
+                // sometimes read the start array token, sometimes not
+                assertEquals(XContentParser.Token.START_ARRAY, parser.nextToken());
+            }
+            assertEquals(Arrays.asList(1, 3, 0), parser.list());
+        }
+    }
+
+    public void testNestedList() throws IOException {
+        XContentBuilder builder = XContentFactory.jsonBuilder().startObject()
+                .startArray("some_array")
+                .startArray().endArray()
+                .startArray().value(1).value(3).endArray()
+                .startArray().value(2).endArray()
+                .endArray().endObject();
+
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, builder.string())) {
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
+            assertEquals("some_array", parser.currentName());
+            if (random().nextBoolean()) {
+                // sometimes read the start array token, sometimes not
+                assertEquals(XContentParser.Token.START_ARRAY, parser.nextToken());
+            }
+            assertEquals(
+                    Arrays.asList(Collections.<Integer>emptyList(), Arrays.asList(1, 3), Arrays.asList(2)),
+                    parser.list());
+        }
+    }
+
+    public void testNestedMapInList() throws IOException {
+        XContentBuilder builder = XContentFactory.jsonBuilder().startObject()
+                .startArray("some_array")
+                .startObject().field("foo", "bar").endObject()
+                .startObject().endObject()
+                .endArray().endObject();
+
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, builder.string())) {
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
+            assertEquals("some_array", parser.currentName());
+            if (random().nextBoolean()) {
+                // sometimes read the start array token, sometimes not
+                assertEquals(XContentParser.Token.START_ARRAY, parser.nextToken());
+            }
+            assertEquals(
+                    Arrays.asList(singletonMap("foo", "bar"), emptyMap()),
+                    parser.list());
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/XContentParserUtilsTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/XContentParserUtilsTests.java
new file mode 100644
index 0000000000000..26bf83d7d56b1
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/common/xcontent/XContentParserUtilsTests.java
@@ -0,0 +1,226 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.xcontent;
+
+import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.common.CheckedBiConsumer;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Base64;
+import java.util.List;
+
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureFieldName;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.parseTypedKeysObject;
+import static org.hamcrest.Matchers.containsString;
+
+public class XContentParserUtilsTests extends ESTestCase {
+
+    public void testEnsureExpectedToken() throws IOException {
+        final XContentParser.Token randomToken = randomFrom(XContentParser.Token.values());
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, "{}")) {
+            // Parser current token is null
+            assertNull(parser.currentToken());
+            ParsingException e = expectThrows(ParsingException.class,
+                    () -> ensureExpectedToken(randomToken, parser.currentToken(), parser::getTokenLocation));
+            assertEquals("Failed to parse object: expecting token of type [" + randomToken + "] but found [null]", e.getMessage());
+            ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
+            ensureExpectedToken(XContentParser.Token.END_OBJECT, parser.nextToken(), parser::getTokenLocation);
+        }
+    }
+
+    public void testParseStoredFieldsValueString() throws IOException {
+        final String value = randomAlphaOfLengthBetween(0, 10);
+        assertParseStoredFieldsValue(value, (xcontentType, result) -> assertEquals(value, result));
+    }
+
+    public void testParseStoredFieldsValueInt() throws IOException {
+        final Integer value = randomInt();
+        assertParseStoredFieldsValue(value, (xcontentType, result) -> assertEquals(value, result));
+    }
+
+    public void testParseStoredFieldsValueLong() throws IOException {
+        final Long value = randomLong();
+        assertParseStoredFieldsValue(value, (xcontentType, result) -> assertEquals(value, result));
+    }
+
+    public void testParseStoredFieldsValueDouble() throws IOException {
+        final Double value = randomDouble();
+        assertParseStoredFieldsValue(value, (xcontentType, result) -> assertEquals(value, ((Number) result).doubleValue(), 0.0d));
+    }
+
+    public void testParseStoredFieldsValueFloat() throws IOException {
+        final Float value = randomFloat();
+        assertParseStoredFieldsValue(value, (xcontentType, result) -> assertEquals(value, ((Number) result).floatValue(), 0.0f));
+    }
+
+    public void testParseStoredFieldsValueBoolean() throws IOException {
+        final Boolean value = randomBoolean();
+        assertParseStoredFieldsValue(value, (xcontentType, result) -> assertEquals(value, result));
+    }
+
+    public void testParseStoredFieldsValueBinary() throws IOException {
+        final byte[] value = randomUnicodeOfLength(scaledRandomIntBetween(10, 1000)).getBytes("UTF-8");
+        assertParseStoredFieldsValue(value, (xcontentType, result) -> {
+            if (xcontentType == XContentType.JSON || xcontentType == XContentType.YAML) {
+                //binary values will be parsed back and returned as base64 strings when reading from json and yaml
+                assertArrayEquals(value, Base64.getDecoder().decode((String) result));
+            } else {
+                //binary values will be parsed back and returned as BytesArray when reading from cbor and smile
+                assertArrayEquals(value, ((BytesArray) result).array());
+            }
+        });
+    }
+
+    public void testParseStoredFieldsValueUnknown() throws IOException {
+        ParsingException e = expectThrows(ParsingException.class, () ->
+                assertParseStoredFieldsValue(null, (x, r) -> fail("Should have thrown a parsing exception")));
+        assertThat(e.getMessage(), containsString("unexpected token"));
+    }
+
+    private void assertParseStoredFieldsValue(final Object value, final CheckedBiConsumer<XContentType, Object, IOException> consumer)
+            throws IOException {
+        final XContentType xContentType = randomFrom(XContentType.values());
+        try (XContentBuilder builder = XContentBuilder.builder(xContentType.xContent())) {
+            final String fieldName = randomAlphaOfLengthBetween(0, 10);
+
+            builder.startObject();
+            builder.field(fieldName, value);
+            builder.endObject();
+
+            try (XContentParser parser = createParser(builder)) {
+                ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
+                ensureFieldName(parser, parser.nextToken(), fieldName);
+                assertNotNull(parser.nextToken());
+                consumer.accept(xContentType, XContentParserUtils.parseStoredFieldsValue(parser));
+                ensureExpectedToken(XContentParser.Token.END_OBJECT, parser.nextToken(), parser::getTokenLocation);
+                assertNull(parser.nextToken());
+            }
+        }
+    }
+
+    public void testParseTypedKeysObject() throws IOException {
+        final String delimiter = randomFrom("#", ":", "/", "-", "_", "|", "_delim_");
+        final XContentType xContentType = randomFrom(XContentType.values());
+
+        final ObjectParser<SetOnce<Boolean>, Void> BOOLPARSER = new ObjectParser<>("bool", () -> new SetOnce<>());
+        BOOLPARSER.declareBoolean(SetOnce::set, new ParseField("field"));
+        final ObjectParser<SetOnce<Long>, Void> LONGPARSER = new ObjectParser<>("long", () -> new SetOnce<>());
+        LONGPARSER.declareLong(SetOnce::set, new ParseField("field"));
+
+        List<NamedXContentRegistry.Entry> namedXContents = new ArrayList<>();
+        namedXContents.add(new NamedXContentRegistry.Entry(Boolean.class, new ParseField("bool"), p -> BOOLPARSER.parse(p, null).get()));
+        namedXContents.add(new NamedXContentRegistry.Entry(Long.class, new ParseField("long"), p -> LONGPARSER.parse(p, null).get()));
+        final NamedXContentRegistry namedXContentRegistry = new NamedXContentRegistry(namedXContents);
+
+        BytesReference bytes = toXContent((builder, params) -> builder.startObject("name").field("field", 0).endObject(), xContentType,
+                randomBoolean());
+        try (XContentParser parser = xContentType.xContent().createParser(namedXContentRegistry, bytes)) {
+            ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
+            ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.nextToken(), parser::getTokenLocation);
+            ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
+            SetOnce<Boolean> booleanConsumer = new SetOnce<>();
+            parseTypedKeysObject(parser, delimiter, Boolean.class, booleanConsumer::set);
+            // because of the missing type to identify the parser, we expect no return value, but also no exception
+            assertNull(booleanConsumer.get());
+            ensureExpectedToken(XContentParser.Token.END_OBJECT, parser.currentToken(), parser::getTokenLocation);
+            ensureExpectedToken(XContentParser.Token.END_OBJECT, parser.nextToken(), parser::getTokenLocation);
+            assertNull(parser.nextToken());
+        }
+
+        bytes = toXContent((builder, params) -> builder.startObject("type" + delimiter + "name").field("bool", true).endObject(),
+                xContentType, randomBoolean());
+        try (XContentParser parser = xContentType.xContent().createParser(namedXContentRegistry, bytes)) {
+            ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
+            ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.nextToken(), parser::getTokenLocation);
+            ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
+            NamedXContentRegistry.UnknownNamedObjectException e = expectThrows(NamedXContentRegistry.UnknownNamedObjectException.class,
+                    () -> parseTypedKeysObject(parser, delimiter, Boolean.class, a -> {}));
+            assertEquals("Unknown Boolean [type]", e.getMessage());
+            assertEquals("type", e.getName());
+            assertEquals("java.lang.Boolean", e.getCategoryClass());
+        }
+
+        final long longValue = randomLong();
+        final boolean boolValue = randomBoolean();
+        bytes = toXContent((builder, params) -> {
+            builder.startObject("long" + delimiter + "l").field("field", longValue).endObject();
+            builder.startObject("bool" + delimiter + "l").field("field", boolValue).endObject();
+            return builder;
+        }, xContentType, randomBoolean());
+
+        try (XContentParser parser = xContentType.xContent().createParser(namedXContentRegistry, bytes)) {
+            ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
+
+            ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.nextToken(), parser::getTokenLocation);
+            ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
+            SetOnce<Long> parsedLong = new SetOnce<>();
+            parseTypedKeysObject(parser, delimiter, Long.class, parsedLong::set);
+            assertNotNull(parsedLong);
+            assertEquals(longValue, parsedLong.get().longValue());
+
+            ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.nextToken(), parser::getTokenLocation);
+            ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
+            SetOnce<Boolean> parsedBoolean = new SetOnce<>();
+            parseTypedKeysObject(parser, delimiter, Boolean.class, parsedBoolean::set);
+            assertNotNull(parsedBoolean);
+            assertEquals(boolValue, parsedBoolean.get());
+
+            ensureExpectedToken(XContentParser.Token.END_OBJECT, parser.nextToken(), parser::getTokenLocation);
+        }
+    }
+
+    public void testParseTypedKeysObjectErrors() throws IOException {
+        final XContentType xContentType = randomFrom(XContentType.values());
+        {
+            BytesReference bytes = toXContent((builder, params) -> builder.startObject("name").field("field", 0).endObject(), xContentType,
+                    randomBoolean());
+            try (XContentParser parser = xContentType.xContent().createParser(NamedXContentRegistry.EMPTY, bytes)) {
+                ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
+                ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.nextToken(), parser::getTokenLocation);
+                ParsingException exception = expectThrows(ParsingException.class,
+                        () -> parseTypedKeysObject(parser, "#", Boolean.class, o -> {
+                        }));
+                assertEquals("Failed to parse object: unexpected token [FIELD_NAME] found", exception.getMessage());
+            }
+        }
+        {
+            BytesReference bytes = toXContent((builder, params) -> builder.startObject("").field("field", 0).endObject(), xContentType,
+                    randomBoolean());
+            try (XContentParser parser = xContentType.xContent().createParser(NamedXContentRegistry.EMPTY, bytes)) {
+                ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
+                ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.nextToken(), parser::getTokenLocation);
+                ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
+                ParsingException exception = expectThrows(ParsingException.class,
+                        () -> parseTypedKeysObject(parser, "#", Boolean.class, o -> {
+                        }));
+                assertEquals("Failed to parse object: empty key", exception.getMessage());
+            }
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/builder/XContentBuilderTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/builder/XContentBuilderTests.java
index d0e095e8c65c9..d3a5e44a89efd 100644
--- a/core/src/test/java/org/elasticsearch/common/xcontent/builder/XContentBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/common/xcontent/builder/XContentBuilderTests.java
@@ -29,6 +29,7 @@
 import org.elasticsearch.common.xcontent.XContentGenerator;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.ByteArrayOutputStream;
@@ -173,9 +174,9 @@ public void testByteConversion() throws Exception {
 
     public void testDateTypesConversion() throws Exception {
         Date date = new Date();
-        String expectedDate = XContentBuilder.defaultDatePrinter.print(date.getTime());
+        String expectedDate = XContentBuilder.DEFAULT_DATE_PRINTER.print(date.getTime());
         Calendar calendar = new GregorianCalendar(TimeZone.getTimeZone("UTC"), Locale.ROOT);
-        String expectedCalendar = XContentBuilder.defaultDatePrinter.print(calendar.getTimeInMillis());
+        String expectedCalendar = XContentBuilder.DEFAULT_DATE_PRINTER.print(calendar.getTimeInMillis());
         XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
         builder.startObject().field("date", date).endObject();
         assertThat(builder.string(), equalTo("{\"date\":\"" + expectedDate + "\"}"));
@@ -213,7 +214,7 @@ public void testCopyCurrentStructure() throws Exception {
 
         builder.field("fakefield", terms).endObject().endObject().endObject();
 
-        XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.bytes());
+        XContentParser parser = createParser(JsonXContent.jsonXContent, builder.bytes());
 
         XContentBuilder filterBuilder = null;
         XContentParser.Token token;
@@ -235,7 +236,7 @@ public void testCopyCurrentStructure() throws Exception {
         }
 
         assertNotNull(filterBuilder);
-        parser = XContentFactory.xContent(XContentType.JSON).createParser(filterBuilder.bytes());
+        parser = createParser(JsonXContent.jsonXContent, filterBuilder.bytes());
         assertThat(parser.nextToken(), equalTo(XContentParser.Token.START_OBJECT));
         assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME));
         assertThat(parser.currentName(), equalTo("terms"));
@@ -339,17 +340,17 @@ public void testWriteMapWithNullKeys() throws IOException {
             builder.map(Collections.singletonMap(null, "test"));
             fail("write map should have failed");
         } catch(IllegalArgumentException e) {
-            assertThat(e.getMessage(), equalTo("field name cannot be null"));
+            assertThat(e.getMessage(), equalTo("Field name cannot be null"));
         }
     }
 
     public void testWriteMapValueWithNullKeys() throws IOException {
         XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
         try {
-            builder.value(Collections.singletonMap(null, "test"));
+            builder.map(Collections.singletonMap(null, "test"));
             fail("write map should have failed");
         } catch(IllegalArgumentException e) {
-            assertThat(e.getMessage(), equalTo("field name cannot be null"));
+            assertThat(e.getMessage(), equalTo("Field name cannot be null"));
         }
     }
 
@@ -360,7 +361,7 @@ public void testWriteFieldMapWithNullKeys() throws IOException {
             builder.field("map", Collections.singletonMap(null, "test"));
             fail("write map should have failed");
         } catch(IllegalArgumentException e) {
-            assertThat(e.getMessage(), equalTo("field name cannot be null"));
+            assertThat(e.getMessage(), equalTo("Field name cannot be null"));
         }
     }
 
@@ -371,8 +372,8 @@ public void testMissingEndObject() throws IOException {
                 builder.field("foo", true);
             }
         });
-        assertThat(e.getMessage(), equalTo("failed to close the XContentBuilder"));
-        assertThat(e.getCause().getMessage(), equalTo("unclosed object or array found"));
+        assertThat(e.getMessage(), equalTo("Failed to close the XContentBuilder"));
+        assertThat(e.getCause().getMessage(), equalTo("Unclosed object or array found"));
     }
 
     public void testMissingEndArray() throws IOException {
@@ -384,7 +385,7 @@ public void testMissingEndArray() throws IOException {
                 builder.value(1);
             }
         });
-        assertThat(e.getMessage(), equalTo("failed to close the XContentBuilder"));
-        assertThat(e.getCause().getMessage(), equalTo("unclosed object or array found"));
+        assertThat(e.getMessage(), equalTo("Failed to close the XContentBuilder"));
+        assertThat(e.getCause().getMessage(), equalTo("Unclosed object or array found"));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/cbor/CborXContentParserTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/cbor/CborXContentParserTests.java
index 173a58cff8924..1a66987510152 100644
--- a/core/src/test/java/org/elasticsearch/common/xcontent/cbor/CborXContentParserTests.java
+++ b/core/src/test/java/org/elasticsearch/common/xcontent/cbor/CborXContentParserTests.java
@@ -22,7 +22,6 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
@@ -34,7 +33,7 @@ public void testEmptyValue() throws IOException {
         for (int i = 0; i < 2; i++) {
             // Running this part twice triggers the issue.
             // See https://github.com/elastic/elasticsearch/issues/8629
-            XContentParser parser = XContentFactory.xContent(XContentType.CBOR).createParser(ref);
+            XContentParser parser = createParser(CborXContent.cborXContent, ref);
             while (parser.nextToken() != null) {
                 parser.utf8Bytes();
             }
diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/cbor/JsonVsCborTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/cbor/JsonVsCborTests.java
index efbca114aacbc..cabd63b6c1b5d 100644
--- a/core/src/test/java/org/elasticsearch/common/xcontent/cbor/JsonVsCborTests.java
+++ b/core/src/test/java/org/elasticsearch/common/xcontent/cbor/JsonVsCborTests.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.common.xcontent.XContentGenerator;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
@@ -48,8 +49,10 @@ public void testCompareParsingTokens() throws IOException {
         xsonGen.writeStringField("test", "value");
         jsonGen.writeStringField("test", "value");
 
-        xsonGen.writeArrayFieldStart("arr");
-        jsonGen.writeArrayFieldStart("arr");
+        xsonGen.writeFieldName("arr");
+        xsonGen.writeStartArray();
+        jsonGen.writeFieldName("arr");
+        jsonGen.writeStartArray();
         xsonGen.writeNumber(1);
         jsonGen.writeNumber(1);
         xsonGen.writeNull();
@@ -63,8 +66,7 @@ public void testCompareParsingTokens() throws IOException {
         xsonGen.close();
         jsonGen.close();
 
-        verifySameTokens(XContentFactory.xContent(XContentType.JSON).createParser(jsonOs.bytes()),
-            XContentFactory.xContent(XContentType.CBOR).createParser(xsonOs.bytes()));
+        verifySameTokens(createParser(JsonXContent.jsonXContent, jsonOs.bytes()), createParser(CborXContent.cborXContent, xsonOs.bytes()));
     }
 
     private void verifySameTokens(XContentParser parser1, XContentParser parser2) throws IOException {
diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/smile/JsonVsSmileTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/smile/JsonVsSmileTests.java
index 63b19a63822e1..5b8a03df5b92c 100644
--- a/core/src/test/java/org/elasticsearch/common/xcontent/smile/JsonVsSmileTests.java
+++ b/core/src/test/java/org/elasticsearch/common/xcontent/smile/JsonVsSmileTests.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.common.xcontent.XContentGenerator;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
@@ -48,8 +49,10 @@ public void testCompareParsingTokens() throws IOException {
         xsonGen.writeStringField("test", "value");
         jsonGen.writeStringField("test", "value");
 
-        xsonGen.writeArrayFieldStart("arr");
-        jsonGen.writeArrayFieldStart("arr");
+        xsonGen.writeFieldName("arr");
+        xsonGen.writeStartArray();
+        jsonGen.writeFieldName("arr");
+        jsonGen.writeStartArray();
         xsonGen.writeNumber(1);
         jsonGen.writeNumber(1);
         xsonGen.writeNull();
@@ -63,8 +66,8 @@ public void testCompareParsingTokens() throws IOException {
         xsonGen.close();
         jsonGen.close();
 
-        verifySameTokens(XContentFactory.xContent(XContentType.JSON).createParser(jsonOs.bytes()),
-            XContentFactory.xContent(XContentType.SMILE).createParser(xsonOs.bytes()));
+        verifySameTokens(createParser(JsonXContent.jsonXContent, jsonOs.bytes()),
+                createParser(SmileXContent.smileXContent, xsonOs.bytes()));
     }
 
     private void verifySameTokens(XContentParser parser1, XContentParser parser2) throws IOException {
diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/support/AbstractFilteringTestCase.java b/core/src/test/java/org/elasticsearch/common/xcontent/support/AbstractFilteringTestCase.java
new file mode 100644
index 0000000000000..9d95ea6013f3a
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/common/xcontent/support/AbstractFilteringTestCase.java
@@ -0,0 +1,1430 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.xcontent.support;
+
+import org.elasticsearch.common.CheckedFunction;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.common.xcontent.XContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.Set;
+
+import static java.util.Collections.emptySet;
+import static java.util.Collections.singleton;
+
+/**
+ * Tests for {@link XContent} filtering.
+ */
+public abstract class AbstractFilteringTestCase extends ESTestCase {
+
+    @FunctionalInterface
+    protected interface Builder extends CheckedFunction<XContentBuilder, XContentBuilder, IOException> {
+    }
+
+    protected abstract void testFilter(Builder expected, Builder actual, Set<String> includes, Set<String> excludes) throws IOException;
+
+    /** Sample test case **/
+    private static final Builder SAMPLE = builder -> builder.startObject()
+            .field("title", "My awesome book")
+                .field("pages", 456)
+                .field("price", 27.99)
+                .field("timestamp", 1428582942867L)
+                .nullField("default")
+                .startArray("tags")
+                    .value("elasticsearch")
+                    .value("java")
+                .endArray()
+                .startArray("authors")
+                    .startObject()
+                        .field("name", "John Doe")
+                        .field("lastname", "John")
+                        .field("firstname", "Doe")
+                    .endObject()
+                    .startObject()
+                        .field("name", "William Smith")
+                        .field("lastname", "William")
+                        .field("firstname", "Smith")
+                    .endObject()
+                .endArray()
+                .startObject("properties")
+                    .field("weight", 0.8d)
+                    .startObject("language")
+                        .startObject("en")
+                            .field("lang", "English")
+                            .field("available", true)
+                            .startArray("distributors")
+                                .startObject()
+                                    .field("name", "The Book Shop")
+                                    .startArray("addresses")
+                                        .startObject()
+                                            .field("name", "address #1")
+                                            .field("street", "Hampton St")
+                                            .field("city", "London")
+                                        .endObject()
+                                        .startObject()
+                                            .field("name", "address #2")
+                                            .field("street", "Queen St")
+                                            .field("city", "Stornoway")
+                                        .endObject()
+                                    .endArray()
+                                .endObject()
+                                .startObject()
+                                    .field("name", "Sussex Books House")
+                                .endObject()
+                            .endArray()
+                        .endObject()
+                        .startObject("fr")
+                            .field("lang", "French")
+                            .field("available", false)
+                            .startArray("distributors")
+                                .startObject()
+                                    .field("name", "La Maison du Livre")
+                                    .startArray("addresses")
+                                        .startObject()
+                                            .field("name", "address #1")
+                                            .field("street", "Rue Mouffetard")
+                                            .field("city", "Paris")
+                                        .endObject()
+                                    .endArray()
+                                .endObject()
+                                .startObject()
+                                    .field("name", "Thetra")
+                                .endObject()
+                            .endArray()
+                        .endObject()
+                    .endObject()
+                .endObject()
+            .endObject();
+
+    public void testNoFiltering() throws Exception {
+        final Builder expected = SAMPLE;
+
+        testFilter(expected, SAMPLE, emptySet(), emptySet());
+        testFilter(expected, SAMPLE, singleton("*"), emptySet());
+        testFilter(expected, SAMPLE, singleton("**"), emptySet());
+        testFilter(expected, SAMPLE, emptySet(), singleton("xyz"));
+    }
+
+    public void testNoMatch() throws Exception {
+        final Builder expected = builder -> builder.startObject().endObject();
+
+        testFilter(expected, SAMPLE, singleton("xyz"), emptySet());
+        testFilter(expected, SAMPLE, emptySet(), singleton("*"));
+        testFilter(expected, SAMPLE, emptySet(), singleton("**"));
+    }
+
+    public void testSimpleFieldInclusive() throws Exception {
+        final Builder expected = builder -> builder.startObject().field("title", "My awesome book").endObject();
+
+        testFilter(expected, SAMPLE, singleton("title"), emptySet());
+    }
+
+    public void testSimpleFieldExclusive() throws Exception {
+        final Builder expected = builder -> builder.startObject()
+                                                            .field("pages", 456)
+                                                            .field("price", 27.99)
+                                                            .field("timestamp", 1428582942867L)
+                                                            .nullField("default")
+                                                            .startArray("tags")
+                                                                .value("elasticsearch")
+                                                                .value("java")
+                                                            .endArray()
+                                                            .startArray("authors")
+                                                                .startObject()
+                                                                    .field("name", "John Doe")
+                                                                    .field("lastname", "John")
+                                                                    .field("firstname", "Doe")
+                                                                .endObject()
+                                                                .startObject()
+                                                                    .field("name", "William Smith")
+                                                                    .field("lastname", "William")
+                                                                    .field("firstname", "Smith")
+                                                                .endObject()
+                                                            .endArray()
+                                                            .startObject("properties")
+                                                                .field("weight", 0.8d)
+                                                                .startObject("language")
+                                                                    .startObject("en")
+                                                                        .field("lang", "English")
+                                                                        .field("available", true)
+                                                                        .startArray("distributors")
+                                                                            .startObject()
+                                                                                .field("name", "The Book Shop")
+                                                                                .startArray("addresses")
+                                                                                    .startObject()
+                                                                                        .field("name", "address #1")
+                                                                                        .field("street", "Hampton St")
+                                                                                        .field("city", "London")
+                                                                                    .endObject()
+                                                                                    .startObject()
+                                                                                        .field("name", "address #2")
+                                                                                        .field("street", "Queen St")
+                                                                                        .field("city", "Stornoway")
+                                                                                    .endObject()
+                                                                                .endArray()
+                                                                            .endObject()
+                                                                            .startObject()
+                                                                                .field("name", "Sussex Books House")
+                                                                            .endObject()
+                                                                        .endArray()
+                                                                    .endObject()
+                                                                    .startObject("fr")
+                                                                        .field("lang", "French")
+                                                                        .field("available", false)
+                                                                        .startArray("distributors")
+                                                                            .startObject()
+                                                                                .field("name", "La Maison du Livre")
+                                                                                .startArray("addresses")
+                                                                                    .startObject()
+                                                                                        .field("name", "address #1")
+                                                                                        .field("street", "Rue Mouffetard")
+                                                                                        .field("city", "Paris")
+                                                                                    .endObject()
+                                                                                .endArray()
+                                                                            .endObject()
+                                                                            .startObject()
+                                                                                .field("name", "Thetra")
+                                                                            .endObject()
+                                                                        .endArray()
+                                                                    .endObject()
+                                                                .endObject()
+                                                            .endObject()
+                                                        .endObject();
+
+        testFilter(expected, SAMPLE, emptySet(), singleton("title"));
+    }
+
+    public void testSimpleFieldWithWildcardInclusive() throws Exception {
+        final Builder expected = builder -> builder.startObject()
+                                                            .field("price", 27.99)
+                                                            .startObject("properties")
+                                                                .field("weight", 0.8d)
+                                                                .startObject("language")
+                                                                    .startObject("en")
+                                                                        .field("lang", "English")
+                                                                        .field("available", true)
+                                                                        .startArray("distributors")
+                                                                            .startObject()
+                                                                                .field("name", "The Book Shop")
+                                                                                .startArray("addresses")
+                                                                                    .startObject()
+                                                                                        .field("name", "address #1")
+                                                                                        .field("street", "Hampton St")
+                                                                                        .field("city", "London")
+                                                                                    .endObject()
+                                                                                    .startObject()
+                                                                                        .field("name", "address #2")
+                                                                                        .field("street", "Queen St")
+                                                                                        .field("city", "Stornoway")
+                                                                                    .endObject()
+                                                                                .endArray()
+                                                                            .endObject()
+                                                                            .startObject()
+                                                                                .field("name", "Sussex Books House")
+                                                                            .endObject()
+                                                                        .endArray()
+                                                                    .endObject()
+                                                                    .startObject("fr")
+                                                                        .field("lang", "French")
+                                                                        .field("available", false)
+                                                                        .startArray("distributors")
+                                                                            .startObject()
+                                                                                .field("name", "La Maison du Livre")
+                                                                                .startArray("addresses")
+                                                                                    .startObject()
+                                                                                        .field("name", "address #1")
+                                                                                        .field("street", "Rue Mouffetard")
+                                                                                        .field("city", "Paris")
+                                                                                    .endObject()
+                                                                                .endArray()
+                                                                            .endObject()
+                                                                            .startObject()
+                                                                                .field("name", "Thetra")
+                                                                            .endObject()
+                                                                        .endArray()
+                                                                    .endObject()
+                                                                .endObject()
+                                                            .endObject()
+                                                        .endObject();
+
+        testFilter(expected, SAMPLE, singleton("pr*"), emptySet());
+    }
+
+    public void testSimpleFieldWithWildcardExclusive() throws Exception {
+        final Builder expected = builder -> builder.startObject()
+                                                            .field("title", "My awesome book")
+                                                            .field("pages", 456)
+                                                            .field("timestamp", 1428582942867L)
+                                                            .nullField("default")
+                                                            .startArray("tags")
+                                                                .value("elasticsearch")
+                                                                .value("java")
+                                                            .endArray()
+                                                            .startArray("authors")
+                                                                .startObject()
+                                                                    .field("name", "John Doe")
+                                                                    .field("lastname", "John")
+                                                                    .field("firstname", "Doe")
+                                                                .endObject()
+                                                                .startObject()
+                                                                    .field("name", "William Smith")
+                                                                    .field("lastname", "William")
+                                                                    .field("firstname", "Smith")
+                                                                .endObject()
+                                                            .endArray()
+                                                        .endObject();
+
+        testFilter(expected, SAMPLE, emptySet(), singleton("pr*"));
+    }
+
+    public void testMultipleFieldsInclusive() throws Exception {
+        final Builder expected = builder -> builder.startObject()
+                                                            .field("title", "My awesome book")
+                                                            .field("pages", 456)
+                                                        .endObject();
+
+        testFilter(expected, SAMPLE, Sets.newHashSet("title", "pages"), emptySet());
+    }
+
+    public void testMultipleFieldsExclusive() throws Exception {
+        final Builder expected = builder -> builder.startObject()
+                                                            .field("price", 27.99)
+                                                            .field("timestamp", 1428582942867L)
+                                                            .nullField("default")
+                                                            .startArray("tags")
+                                                                .value("elasticsearch")
+                                                                .value("java")
+                                                            .endArray()
+                                                            .startArray("authors")
+                                                                .startObject()
+                                                                    .field("name", "John Doe")
+                                                                    .field("lastname", "John")
+                                                                    .field("firstname", "Doe")
+                                                                .endObject()
+                                                                .startObject()
+                                                                    .field("name", "William Smith")
+                                                                    .field("lastname", "William")
+                                                                    .field("firstname", "Smith")
+                                                                .endObject()
+                                                            .endArray()
+                                                            .startObject("properties")
+                                                                .field("weight", 0.8d)
+                                                                .startObject("language")
+                                                                    .startObject("en")
+                                                                        .field("lang", "English")
+                                                                        .field("available", true)
+                                                                        .startArray("distributors")
+                                                                            .startObject()
+                                                                                .field("name", "The Book Shop")
+                                                                                .startArray("addresses")
+                                                                                    .startObject()
+                                                                                        .field("name", "address #1")
+                                                                                        .field("street", "Hampton St")
+                                                                                        .field("city", "London")
+                                                                                    .endObject()
+                                                                                    .startObject()
+                                                                                        .field("name", "address #2")
+                                                                                        .field("street", "Queen St")
+                                                                                        .field("city", "Stornoway")
+                                                                                    .endObject()
+                                                                                .endArray()
+                                                                            .endObject()
+                                                                            .startObject()
+                                                                                .field("name", "Sussex Books House")
+                                                                            .endObject()
+                                                                        .endArray()
+                                                                    .endObject()
+                                                                    .startObject("fr")
+                                                                        .field("lang", "French")
+                                                                        .field("available", false)
+                                                                        .startArray("distributors")
+                                                                            .startObject()
+                                                                                .field("name", "La Maison du Livre")
+                                                                                .startArray("addresses")
+                                                                                    .startObject()
+                                                                                        .field("name", "address #1")
+                                                                                        .field("street", "Rue Mouffetard")
+                                                                                        .field("city", "Paris")
+                                                                                    .endObject()
+                                                                                .endArray()
+                                                                            .endObject()
+                                                                            .startObject()
+                                                                                .field("name", "Thetra")
+                                                                            .endObject()
+                                                                        .endArray()
+                                                                    .endObject()
+                                                                .endObject()
+                                                            .endObject()
+                                                        .endObject();
+
+        testFilter(expected, SAMPLE, emptySet(), Sets.newHashSet("title", "pages"));
+    }
+
+    public void testSimpleArrayInclusive() throws Exception {
+        final Builder expected = builder -> builder.startObject()
+                                                            .startArray("tags")
+                                                                .value("elasticsearch")
+                                                                .value("java")
+                                                            .endArray()
+                                                    .endObject();
+
+        testFilter(expected, SAMPLE, singleton("tags"), emptySet());
+    }
+
+    public void testSimpleArrayExclusive() throws Exception {
+        final Builder expected = builder -> builder.startObject()
+                                                            .field("title", "My awesome book")
+                                                            .field("pages", 456)
+                                                            .field("price", 27.99)
+                                                            .field("timestamp", 1428582942867L)
+                                                            .nullField("default")
+                                                            .startArray("authors")
+                                                                .startObject()
+                                                                    .field("name", "John Doe")
+                                                                    .field("lastname", "John")
+                                                                    .field("firstname", "Doe")
+                                                                .endObject()
+                                                                .startObject()
+                                                                    .field("name", "William Smith")
+                                                                    .field("lastname", "William")
+                                                                    .field("firstname", "Smith")
+                                                                .endObject()
+                                                            .endArray()
+                                                            .startObject("properties")
+                                                                .field("weight", 0.8d)
+                                                                .startObject("language")
+                                                                    .startObject("en")
+                                                                        .field("lang", "English")
+                                                                        .field("available", true)
+                                                                        .startArray("distributors")
+                                                                            .startObject()
+                                                                                .field("name", "The Book Shop")
+                                                                                .startArray("addresses")
+                                                                                    .startObject()
+                                                                                        .field("name", "address #1")
+                                                                                        .field("street", "Hampton St")
+                                                                                        .field("city", "London")
+                                                                                    .endObject()
+                                                                                    .startObject()
+                                                                                        .field("name", "address #2")
+                                                                                        .field("street", "Queen St")
+                                                                                        .field("city", "Stornoway")
+                                                                                    .endObject()
+                                                                                .endArray()
+                                                                            .endObject()
+                                                                            .startObject()
+                                                                                .field("name", "Sussex Books House")
+                                                                            .endObject()
+                                                                        .endArray()
+                                                                    .endObject()
+                                                                    .startObject("fr")
+                                                                        .field("lang", "French")
+                                                                        .field("available", false)
+                                                                        .startArray("distributors")
+                                                                            .startObject()
+                                                                                .field("name", "La Maison du Livre")
+                                                                                .startArray("addresses")
+                                                                                    .startObject()
+                                                                                        .field("name", "address #1")
+                                                                                        .field("street", "Rue Mouffetard")
+                                                                                        .field("city", "Paris")
+                                                                                    .endObject()
+                                                                                .endArray()
+                                                                            .endObject()
+                                                                            .startObject()
+                                                                                .field("name", "Thetra")
+                                                                            .endObject()
+                                                                        .endArray()
+                                                                    .endObject()
+                                                                .endObject()
+                                                            .endObject()
+                                                        .endObject();
+
+        testFilter(expected, SAMPLE, emptySet(), singleton("tags"));
+    }
+
+    public void testSimpleArrayOfObjectsInclusive() throws Exception {
+        final Builder expected = builder -> builder.startObject()
+                                                            .startArray("authors")
+                                                                .startObject()
+                                                                    .field("name", "John Doe")
+                                                                    .field("lastname", "John")
+                                                                    .field("firstname", "Doe")
+                                                                .endObject()
+                                                                .startObject()
+                                                                    .field("name", "William Smith")
+                                                                    .field("lastname", "William")
+                                                                    .field("firstname", "Smith")
+                                                                .endObject()
+                                                            .endArray()
+                                                        .endObject();
+
+        testFilter(expected, SAMPLE, singleton("authors"), emptySet());
+        testFilter(expected, SAMPLE, singleton("authors.*"), emptySet());
+        testFilter(expected, SAMPLE, singleton("authors.*name"), emptySet());
+    }
+
+    public void testSimpleArrayOfObjectsExclusive() throws Exception {
+        final Builder expected = builder -> builder.startObject()
+                                                            .field("title", "My awesome book")
+                                                            .field("pages", 456)
+                                                            .field("price", 27.99)
+                                                            .field("timestamp", 1428582942867L)
+                                                            .nullField("default")
+                                                            .startArray("tags")
+                                                                .value("elasticsearch")
+                                                                .value("java")
+                                                            .endArray()
+                                                            .startObject("properties")
+                                                                .field("weight", 0.8d)
+                                                                .startObject("language")
+                                                                    .startObject("en")
+                                                                        .field("lang", "English")
+                                                                        .field("available", true)
+                                                                        .startArray("distributors")
+                                                                            .startObject()
+                                                                                .field("name", "The Book Shop")
+                                                                                .startArray("addresses")
+                                                                                    .startObject()
+                                                                                        .field("name", "address #1")
+                                                                                        .field("street", "Hampton St")
+                                                                                        .field("city", "London")
+                                                                                    .endObject()
+                                                                                    .startObject()
+                                                                                        .field("name", "address #2")
+                                                                                        .field("street", "Queen St")
+                                                                                        .field("city", "Stornoway")
+                                                                                    .endObject()
+                                                                                .endArray()
+                                                                            .endObject()
+                                                                            .startObject()
+                                                                                .field("name", "Sussex Books House")
+                                                                            .endObject()
+                                                                        .endArray()
+                                                                    .endObject()
+                                                                    .startObject("fr")
+                                                                        .field("lang", "French")
+                                                                        .field("available", false)
+                                                                        .startArray("distributors")
+                                                                            .startObject()
+                                                                                .field("name", "La Maison du Livre")
+                                                                                .startArray("addresses")
+                                                                                    .startObject()
+                                                                                        .field("name", "address #1")
+                                                                                        .field("street", "Rue Mouffetard")
+                                                                                        .field("city", "Paris")
+                                                                                    .endObject()
+                                                                                .endArray()
+                                                                            .endObject()
+                                                                            .startObject()
+                                                                                .field("name", "Thetra")
+                                                                            .endObject()
+                                                                        .endArray()
+                                                                    .endObject()
+                                                                .endObject()
+                                                            .endObject()
+                                                       .endObject();
+
+        testFilter(expected, SAMPLE, emptySet(), singleton("authors"));
+        testFilter(expected, SAMPLE, emptySet(), singleton("authors.*"));
+        testFilter(expected, SAMPLE, emptySet(), singleton("authors.*name"));
+    }
+
+    public void testSimpleArrayOfObjectsPropertyInclusive() throws Exception {
+        final Builder expected = builder -> builder.startObject()
+                                                            .startArray("authors")
+                                                                .startObject()
+                                                                    .field("lastname", "John")
+                                                                .endObject()
+                                                                .startObject()
+                                                                    .field("lastname", "William")
+                                                                .endObject()
+                                                            .endArray()
+                                                        .endObject();
+
+        testFilter(expected, SAMPLE, singleton("authors.lastname"), emptySet());
+        testFilter(expected, SAMPLE, singleton("authors.l*"), emptySet());
+    }
+
+    public void testSimpleArrayOfObjectsPropertyExclusive() throws Exception {
+        final Builder expected = builder -> builder.startObject()
+                                                            .field("title", "My awesome book")
+                                                            .field("pages", 456)
+                                                            .field("price", 27.99)
+                                                            .field("timestamp", 1428582942867L)
+                                                            .nullField("default")
+                                                            .startArray("tags")
+                                                                .value("elasticsearch")
+                                                                .value("java")
+                                                            .endArray()
+                                                            .startArray("authors")
+                                                                .startObject()
+                                                                    .field("name", "John Doe")
+                                                                    .field("firstname", "Doe")
+                                                                .endObject()
+                                                                .startObject()
+                                                                    .field("name", "William Smith")
+                                                                    .field("firstname", "Smith")
+                                                                .endObject()
+                                                            .endArray()
+                                                            .startObject("properties")
+                                                                .field("weight", 0.8d)
+                                                                .startObject("language")
+                                                                    .startObject("en")
+                                                                        .field("lang", "English")
+                                                                        .field("available", true)
+                                                                        .startArray("distributors")
+                                                                            .startObject()
+                                                                                .field("name", "The Book Shop")
+                                                                                .startArray("addresses")
+                                                                                    .startObject()
+                                                                                        .field("name", "address #1")
+                                                                                        .field("street", "Hampton St")
+                                                                                        .field("city", "London")
+                                                                                    .endObject()
+                                                                                    .startObject()
+                                                                                        .field("name", "address #2")
+                                                                                        .field("street", "Queen St")
+                                                                                        .field("city", "Stornoway")
+                                                                                    .endObject()
+                                                                                .endArray()
+                                                                            .endObject()
+                                                                            .startObject()
+                                                                                .field("name", "Sussex Books House")
+                                                                            .endObject()
+                                                                        .endArray()
+                                                                    .endObject()
+                                                                    .startObject("fr")
+                                                                        .field("lang", "French")
+                                                                        .field("available", false)
+                                                                        .startArray("distributors")
+                                                                            .startObject()
+                                                                                .field("name", "La Maison du Livre")
+                                                                                .startArray("addresses")
+                                                                                    .startObject()
+                                                                                        .field("name", "address #1")
+                                                                                        .field("street", "Rue Mouffetard")
+                                                                                        .field("city", "Paris")
+                                                                                    .endObject()
+                                                                                .endArray()
+                                                                            .endObject()
+                                                                            .startObject()
+                                                                                .field("name", "Thetra")
+                                                                            .endObject()
+                                                                        .endArray()
+                                                                    .endObject()
+                                                                .endObject()
+                                                            .endObject()
+                                                       .endObject();
+
+        testFilter(expected, SAMPLE, emptySet(), singleton("authors.lastname"));
+        testFilter(expected, SAMPLE, emptySet(), singleton("authors.l*"));
+    }
+
+    public void testRecurseField1Inclusive() throws Exception {
+        final Builder expected = builder -> builder.startObject()
+                                                            .startArray("authors")
+                                                                .startObject()
+                                                                    .field("name", "John Doe")
+                                                                .endObject()
+                                                                .startObject()
+                                                                    .field("name", "William Smith")
+                                                            .   endObject()
+                                                            .endArray()
+                                                            .startObject("properties")
+                                                                .startObject("language")
+                                                                    .startObject("en")
+                                                                        .startArray("distributors")
+                                                                            .startObject()
+                                                                                .field("name", "The Book Shop")
+                                                                                .startArray("addresses")
+                                                                                    .startObject()
+                                                                                        .field("name", "address #1")
+                                                                                    .endObject()
+                                                                                    .startObject()
+                                                                                        .field("name", "address #2")
+                                                                                    .endObject()
+                                                                                .endArray()
+                                                                            .endObject()
+                                                                            .startObject()
+                                                                                .field("name", "Sussex Books House")
+                                                                            .endObject()
+                                                                        .endArray()
+                                                                    .endObject()
+                                                                    .startObject("fr")
+                                                                        .startArray("distributors")
+                                                                            .startObject()
+                                                                                .field("name", "La Maison du Livre")
+                                                                                .startArray("addresses")
+                                                                                    .startObject()
+                                                                                        .field("name", "address #1")
+                                                                                    .endObject()
+                                                                                .endArray()
+                                                                            .endObject()
+                                                                            .startObject()
+                                                                                .field("name", "Thetra")
+                                                                            .endObject()
+                                                                        .endArray()
+                                                                    .endObject()
+                                                                .endObject()
+                                                            .endObject()
+                                                        .endObject();
+
+        testFilter(expected, SAMPLE, singleton("**.name"), emptySet());
+    }
+
+    public void testRecurseField1Exclusive() throws Exception {
+        final Builder expected = builder -> builder.startObject()
+                                                            .field("title", "My awesome book")
+                                                            .field("pages", 456)
+                                                            .field("price", 27.99)
+                                                            .field("timestamp", 1428582942867L)
+                                                            .nullField("default")
+                                                            .startArray("tags")
+                                                                .value("elasticsearch")
+                                                                .value("java")
+                                                            .endArray()
+                                                            .startArray("authors")
+                                                                .startObject()
+                                                                    .field("lastname", "John")
+                                                                    .field("firstname", "Doe")
+                                                                .endObject()
+                                                                .startObject()
+                                                                    .field("lastname", "William")
+                                                                    .field("firstname", "Smith")
+                                                                .endObject()
+                                                            .endArray()
+                                                            .startObject("properties")
+                                                                .field("weight", 0.8d)
+                                                                .startObject("language")
+                                                                    .startObject("en")
+                                                                        .field("lang", "English")
+                                                                        .field("available", true)
+                                                                        .startArray("distributors")
+                                                                            .startObject()
+                                                                                .startArray("addresses")
+                                                                                    .startObject()
+                                                                                        .field("street", "Hampton St")
+                                                                                        .field("city", "London")
+                                                                                    .endObject()
+                                                                                    .startObject()
+                                                                                        .field("street", "Queen St")
+                                                                                        .field("city", "Stornoway")
+                                                                                    .endObject()
+                                                                                .endArray()
+                                                                            .endObject()
+                                                                        .endArray()
+                                                                    .endObject()
+                                                                    .startObject("fr")
+                                                                        .field("lang", "French")
+                                                                        .field("available", false)
+                                                                        .startArray("distributors")
+                                                                            .startObject()
+                                                                                .startArray("addresses")
+                                                                                    .startObject()
+                                                                                        .field("street", "Rue Mouffetard")
+                                                                                        .field("city", "Paris")
+                                                                                    .endObject()
+                                                                                .endArray()
+                                                                            .endObject()
+                                                                        .endArray()
+                                                                    .endObject()
+                                                                .endObject()
+                                                            .endObject()
+                                                       .endObject();
+
+        testFilter(expected, SAMPLE, emptySet(), singleton("**.name"));
+    }
+
+    public void testRecurseField2Inclusive() throws Exception {
+        final Builder expected = builder -> builder.startObject()
+                                                            .startObject("properties")
+                                                                .startObject("language")
+                                                                    .startObject("en")
+                                                                        .startArray("distributors")
+                                                                            .startObject()
+                                                                                .field("name", "The Book Shop")
+                                                                                .startArray("addresses")
+                                                                                    .startObject()
+                                                                                        .field("name", "address #1")
+                                                                                    .endObject()
+                                                                                    .startObject()
+                                                                                        .field("name", "address #2")
+                                                                                    .endObject()
+                                                                                .endArray()
+                                                                            .endObject()
+                                                                            .startObject()
+                                                                                .field("name", "Sussex Books House")
+                                                                            .endObject()
+                                                                        .endArray()
+                                                                    .endObject()
+                                                                    .startObject("fr")
+                                                                        .startArray("distributors")
+                                                                            .startObject()
+                                                                                .field("name", "La Maison du Livre")
+                                                                                .startArray("addresses")
+                                                                                    .startObject()
+                                                                                        .field("name", "address #1")
+                                                                                    .endObject()
+                                                                                .endArray()
+                                                                            .endObject()
+                                                                            .startObject()
+                                                                                .field("name", "Thetra")
+                                                                            .endObject()
+                                                                        .endArray()
+                                                                    .endObject()
+                                                                .endObject()
+                                                            .endObject()
+                                                        .endObject();
+
+        testFilter(expected, SAMPLE, singleton("properties.**.name"), emptySet());
+    }
+
+    public void testRecurseField2Exclusive() throws Exception {
+        final Builder expected = builder -> builder.startObject()
+                                                            .field("title", "My awesome book")
+                                                            .field("pages", 456)
+                                                            .field("price", 27.99)
+                                                            .field("timestamp", 1428582942867L)
+                                                            .nullField("default")
+                                                            .startArray("tags")
+                                                                .value("elasticsearch")
+                                                                .value("java")
+                                                            .endArray()
+                                                            .startArray("authors")
+                                                                .startObject()
+                                                                    .field("name", "John Doe")
+                                                                    .field("lastname", "John")
+                                                                    .field("firstname", "Doe")
+                                                                .endObject()
+                                                                .startObject()
+                                                                    .field("name", "William Smith")
+                                                                    .field("lastname", "William")
+                                                                    .field("firstname", "Smith")
+                                                                .endObject()
+                                                            .endArray()
+                                                            .startObject("properties")
+                                                                .field("weight", 0.8d)
+                                                                .startObject("language")
+                                                                    .startObject("en")
+                                                                        .field("lang", "English")
+                                                                        .field("available", true)
+                                                                        .startArray("distributors")
+                                                                            .startObject()
+                                                                                .startArray("addresses")
+                                                                                    .startObject()
+                                                                                        .field("street", "Hampton St")
+                                                                                        .field("city", "London")
+                                                                                    .endObject()
+                                                                                    .startObject()
+                                                                                        .field("street", "Queen St")
+                                                                                        .field("city", "Stornoway")
+                                                                                    .endObject()
+                                                                                .endArray()
+                                                                            .endObject()
+                                                                        .endArray()
+                                                                    .endObject()
+                                                                    .startObject("fr")
+                                                                        .field("lang", "French")
+                                                                        .field("available", false)
+                                                                        .startArray("distributors")
+                                                                            .startObject()
+                                                                                .startArray("addresses")
+                                                                                    .startObject()
+                                                                                        .field("street", "Rue Mouffetard")
+                                                                                        .field("city", "Paris")
+                                                                                    .endObject()
+                                                                                .endArray()
+                                                                            .endObject()
+                                                                        .endArray()
+                                                                    .endObject()
+                                                                .endObject()
+                                                            .endObject()
+                                                       .endObject();
+
+        testFilter(expected, SAMPLE, emptySet(), singleton("properties.**.name"));
+    }
+
+    public void testRecurseField3Inclusive() throws Exception {
+        final Builder expected = builder -> builder.startObject()
+                                                            .startObject("properties")
+                                                                .startObject("language")
+                                                                    .startObject("en")
+                                                                        .startArray("distributors")
+                                                                            .startObject()
+                                                                                .field("name", "The Book Shop")
+                                                                                .startArray("addresses")
+                                                                                    .startObject()
+                                                                                        .field("name", "address #1")
+                                                                                    .endObject()
+                                                                                    .startObject()
+                                                                                        .field("name", "address #2")
+                                                                                    .endObject()
+                                                                                .endArray()
+                                                                            .endObject()
+                                                                            .startObject()
+                                                                                .field("name", "Sussex Books House")
+                                                                            .endObject()
+                                                                        .endArray()
+                                                                    .endObject()
+                                                                .endObject()
+                                                            .endObject()
+                                                        .endObject();
+
+        testFilter(expected, SAMPLE, singleton("properties.*.en.**.name"), emptySet());
+    }
+
+    public void testRecurseField3Exclusive() throws Exception {
+        final Builder expected = builder -> builder.startObject()
+                                                            .field("title", "My awesome book")
+                                                            .field("pages", 456)
+                                                            .field("price", 27.99)
+                                                            .field("timestamp", 1428582942867L)
+                                                            .nullField("default")
+                                                            .startArray("tags")
+                                                                .value("elasticsearch")
+                                                                .value("java")
+                                                            .endArray()
+                                                            .startArray("authors")
+                                                                .startObject()
+                                                                    .field("name", "John Doe")
+                                                                    .field("lastname", "John")
+                                                                    .field("firstname", "Doe")
+                                                                .endObject()
+                                                                .startObject()
+                                                                    .field("name", "William Smith")
+                                                                    .field("lastname", "William")
+                                                                    .field("firstname", "Smith")
+                                                                .endObject()
+                                                            .endArray()
+                                                            .startObject("properties")
+                                                                .field("weight", 0.8d)
+                                                                .startObject("language")
+                                                                    .startObject("en")
+                                                                        .field("lang", "English")
+                                                                        .field("available", true)
+                                                                        .startArray("distributors")
+                                                                            .startObject()
+                                                                                .startArray("addresses")
+                                                                                    .startObject()
+                                                                                        .field("street", "Hampton St")
+                                                                                        .field("city", "London")
+                                                                                    .endObject()
+                                                                                    .startObject()
+                                                                                        .field("street", "Queen St")
+                                                                                        .field("city", "Stornoway")
+                                                                                    .endObject()
+                                                                                .endArray()
+                                                                            .endObject()
+                                                                        .endArray()
+                                                                    .endObject()
+                                                                    .startObject("fr")
+                                                                        .field("lang", "French")
+                                                                        .field("available", false)
+                                                                        .startArray("distributors")
+                                                                            .startObject()
+                                                                                .field("name", "La Maison du Livre")
+                                                                                .startArray("addresses")
+                                                                                    .startObject()
+                                                                                        .field("name", "address #1")
+                                                                                        .field("street", "Rue Mouffetard")
+                                                                                        .field("city", "Paris")
+                                                                                    .endObject()
+                                                                                .endArray()
+                                                                            .endObject()
+                                                                            .startObject()
+                                                                                .field("name", "Thetra")
+                                                                            .endObject()
+                                                                        .endArray()
+                                                                    .endObject()
+                                                                .endObject()
+                                                            .endObject()
+                                                       .endObject();
+
+        testFilter(expected, SAMPLE, emptySet(), singleton("properties.*.en.**.name"));
+    }
+
+    public void testRecurseField4Inclusive() throws Exception {
+        final Builder expected = builder -> builder.startObject()
+                                                            .startObject("properties")
+                                                                .startObject("language")
+                                                                    .startObject("en")
+                                                                        .startArray("distributors")
+                                                                            .startObject()
+                                                                                .field("name", "The Book Shop")
+                                                                            .endObject()
+                                                                            .startObject()
+                                                                                .field("name", "Sussex Books House")
+                                                                            .endObject()
+                                                                        .endArray()
+                                                                    .endObject()
+                                                                    .startObject("fr")
+                                                                        .startArray("distributors")
+                                                                            .startObject()
+                                                                                .field("name", "La Maison du Livre")
+                                                                            .endObject()
+                                                                            .startObject()
+                                                                                .field("name", "Thetra")
+                                                                            .endObject()
+                                                                        .endArray()
+                                                                    .endObject()
+                                                                .endObject()
+                                                            .endObject()
+                                                        .endObject();
+
+        testFilter(expected, SAMPLE, singleton("properties.**.distributors.name"), emptySet());
+    }
+
+    public void testRecurseField4Exclusive() throws Exception {
+        final Builder expected = builder -> builder.startObject()
+                                                            .field("title", "My awesome book")
+                                                            .field("pages", 456)
+                                                            .field("price", 27.99)
+                                                            .field("timestamp", 1428582942867L)
+                                                            .nullField("default")
+                                                            .startArray("tags")
+                                                                .value("elasticsearch")
+                                                                .value("java")
+                                                            .endArray()
+                                                            .startArray("authors")
+                                                                .startObject()
+                                                                    .field("name", "John Doe")
+                                                                    .field("lastname", "John")
+                                                                    .field("firstname", "Doe")
+                                                                .endObject()
+                                                                .startObject()
+                                                                    .field("name", "William Smith")
+                                                                    .field("lastname", "William")
+                                                                    .field("firstname", "Smith")
+                                                                .endObject()
+                                                            .endArray()
+                                                            .startObject("properties")
+                                                                .field("weight", 0.8d)
+                                                                .startObject("language")
+                                                                    .startObject("en")
+                                                                        .field("lang", "English")
+                                                                        .field("available", true)
+                                                                        .startArray("distributors")
+                                                                            .startObject()
+                                                                                .startArray("addresses")
+                                                                                    .startObject()
+                                                                                        .field("name", "address #1")
+                                                                                        .field("street", "Hampton St")
+                                                                                        .field("city", "London")
+                                                                                    .endObject()
+                                                                                    .startObject()
+                                                                                        .field("name", "address #2")
+                                                                                        .field("street", "Queen St")
+                                                                                        .field("city", "Stornoway")
+                                                                                    .endObject()
+                                                                                .endArray()
+                                                                            .endObject()
+                                                                        .endArray()
+                                                                    .endObject()
+                                                                    .startObject("fr")
+                                                                        .field("lang", "French")
+                                                                        .field("available", false)
+                                                                        .startArray("distributors")
+                                                                            .startObject()
+                                                                                .startArray("addresses")
+                                                                                    .startObject()
+                                                                                        .field("name", "address #1")
+                                                                                        .field("street", "Rue Mouffetard")
+                                                                                        .field("city", "Paris")
+                                                                                    .endObject()
+                                                                                .endArray()
+                                                                            .endObject()
+                                                                        .endArray()
+                                                                    .endObject()
+                                                                .endObject()
+                                                            .endObject()
+                                                       .endObject();
+
+        testFilter(expected, SAMPLE, emptySet(), singleton("properties.**.distributors.name"));
+    }
+
+    public void testRawField() throws Exception {
+        final Builder expectedRawField = builder -> builder
+                .startObject()
+                    .field("foo", 0)
+                    .startObject("raw")
+                        .field("content", "hello world!")
+                    .endObject()
+                .endObject();
+
+        final Builder expectedRawFieldFiltered = builder -> builder
+                .startObject()
+                    .field("foo", 0)
+                .endObject();
+
+        final Builder expectedRawFieldNotFiltered = builder -> builder
+                .startObject()
+                    .startObject("raw")
+                        .field("content", "hello world!")
+                    .endObject()
+                .endObject();
+
+        Builder sampleWithRaw = builder -> {
+            BytesReference raw = XContentBuilder.builder(builder.contentType().xContent())
+                    .startObject()
+                        .field("content", "hello world!")
+                    .endObject()
+                    .bytes();
+            return builder.startObject().field("foo", 0).rawField("raw", raw).endObject();
+        };
+
+        // Test method: rawField(String fieldName, BytesReference content)
+        testFilter(expectedRawField, sampleWithRaw, emptySet(), emptySet());
+        testFilter(expectedRawFieldFiltered, sampleWithRaw, singleton("f*"), emptySet());
+        testFilter(expectedRawFieldFiltered, sampleWithRaw, emptySet(), singleton("r*"));
+        testFilter(expectedRawFieldNotFiltered, sampleWithRaw, singleton("r*"), emptySet());
+        testFilter(expectedRawFieldNotFiltered, sampleWithRaw, emptySet(), singleton("f*"));
+
+        sampleWithRaw = builder -> {
+            BytesReference raw = XContentBuilder.builder(builder.contentType().xContent())
+                    .startObject()
+                    .   field("content", "hello world!")
+                    .endObject()
+                    .bytes();
+            return builder.startObject().field("foo", 0).rawField("raw", raw.streamInput()).endObject();
+        };
+
+        // Test method: rawField(String fieldName, InputStream content)
+        testFilter(expectedRawField, sampleWithRaw, emptySet(), emptySet());
+        testFilter(expectedRawFieldFiltered, sampleWithRaw, singleton("f*"), emptySet());
+        testFilter(expectedRawFieldFiltered, sampleWithRaw, emptySet(), singleton("r*"));
+        testFilter(expectedRawFieldNotFiltered, sampleWithRaw, singleton("r*"), emptySet());
+        testFilter(expectedRawFieldNotFiltered, sampleWithRaw, emptySet(), singleton("f*"));
+    }
+
+    public void testArrays() throws Exception {
+        // Test: Array of values (no filtering)
+        final Builder sampleArrayOfValues = builder -> builder
+                .startObject()
+                    .startArray("tags")
+                        .value("lorem").value("ipsum").value("dolor")
+                    .endArray()
+                .endObject();
+        testFilter(sampleArrayOfValues, sampleArrayOfValues, singleton("t*"), emptySet());
+        testFilter(sampleArrayOfValues, sampleArrayOfValues, singleton("tags"), emptySet());
+        testFilter(sampleArrayOfValues, sampleArrayOfValues, emptySet(), singleton("a"));
+
+        // Test: Array of values (with filtering)
+        Builder expected = builder -> builder.startObject().endObject();
+        testFilter(expected, sampleArrayOfValues, singleton("foo"), emptySet());
+        testFilter(expected, sampleArrayOfValues, emptySet(), singleton("t*"));
+        testFilter(expected, sampleArrayOfValues, emptySet(), singleton("tags"));
+
+        // Test: Array of objects (no filtering)
+        final Builder sampleArrayOfObjects = builder -> builder
+                .startObject()
+                    .startArray("tags")
+                        .startObject()
+                            .field("lastname", "lorem")
+                        .endObject()
+                        .startObject()
+                            .field("firstname", "ipsum")
+                        .endObject()
+                    .endArray()
+                .endObject();
+        testFilter(sampleArrayOfObjects, sampleArrayOfObjects, singleton("t*"), emptySet());
+        testFilter(sampleArrayOfObjects, sampleArrayOfObjects, singleton("tags"), emptySet());
+        testFilter(sampleArrayOfObjects, sampleArrayOfObjects, emptySet(), singleton("a"));
+
+        // Test: Array of objects (with filtering)
+        testFilter(expected, sampleArrayOfObjects, singleton("foo"), emptySet());
+        testFilter(expected, sampleArrayOfObjects, emptySet(), singleton("t*"));
+        testFilter(expected, sampleArrayOfObjects, emptySet(), singleton("tags"));
+
+        // Test: Array of objects (with partial filtering)
+        expected = builder -> builder
+                .startObject()
+                    .startArray("tags")
+                        .startObject()
+                            .field("firstname", "ipsum")
+                        .endObject()
+                    .endArray()
+                .endObject();
+        testFilter(expected, sampleArrayOfObjects, singleton("t*.firstname"), emptySet());
+        testFilter(expected, sampleArrayOfObjects, emptySet(), singleton("t*.lastname"));
+    }
+
+    public void testEmptyObject() throws IOException {
+        final Builder sample = builder -> builder.startObject().startObject("foo").endObject().endObject();
+
+        Builder expected = builder -> builder.startObject().startObject("foo").endObject().endObject();
+        testFilter(expected, sample, singleton("foo"), emptySet());
+        testFilter(expected, sample, emptySet(), singleton("bar"));
+        testFilter(expected, sample, singleton("f*"), singleton("baz"));
+
+        expected = builder -> builder.startObject().endObject();
+        testFilter(expected, sample, emptySet(), singleton("foo"));
+        testFilter(expected, sample, singleton("bar"), emptySet());
+        testFilter(expected, sample, singleton("f*"), singleton("foo"));
+    }
+
+    public void testSingleFieldWithBothExcludesIncludes() throws IOException {
+        final Builder expected = builder -> builder
+            .startObject()
+                .field("pages", 456)
+                .field("price", 27.99)
+            .endObject();
+
+        testFilter(expected, SAMPLE, singleton("p*"), singleton("properties"));
+    }
+
+    public void testObjectsInArrayWithBothExcludesIncludes() throws IOException {
+        Set<String> includes = Sets.newHashSet("tags", "authors");
+        Set<String> excludes = singleton("authors.name");
+
+        final Builder expected = builder -> builder
+            .startObject()
+                .startArray("tags")
+                    .value("elasticsearch")
+                    .value("java")
+                .endArray()
+                .startArray("authors")
+                    .startObject()
+                        .field("lastname", "John")
+                        .field("firstname", "Doe")
+                    .endObject()
+                    .startObject()
+                        .field("lastname", "William")
+                        .field("firstname", "Smith")
+                    .endObject()
+                .endArray()
+            .endObject();
+
+        testFilter(expected, SAMPLE, includes, excludes);
+    }
+
+    public void testRecursiveObjectsInArrayWithBothExcludesIncludes() throws IOException {
+        Set<String> includes = Sets.newHashSet("**.language", "properties.weight");
+        Set<String> excludes = singleton("**.distributors");
+
+        final Builder expected = builder -> builder
+            .startObject()
+                .startObject("properties")
+                    .field("weight", 0.8d)
+                    .startObject("language")
+                        .startObject("en")
+                            .field("lang", "English")
+                            .field("available", true)
+                        .endObject()
+                        .startObject("fr")
+                            .field("lang", "French")
+                            .field("available", false)
+                        .endObject()
+                    .endObject()
+                .endObject()
+            .endObject();
+
+        testFilter(expected, SAMPLE, includes, excludes);
+    }
+
+    public void testRecursiveSameObjectWithBothExcludesIncludes() throws IOException {
+        Set<String> includes = singleton("**.distributors");
+        Set<String> excludes = singleton("**.distributors");
+
+        final Builder expected = builder -> builder.startObject().endObject();
+        testFilter(expected, SAMPLE, includes, excludes);
+    }
+
+    public void testRecursiveObjectsPropertiesWithBothExcludesIncludes() throws IOException {
+        Set<String> includes = singleton("**.en.*");
+        Set<String> excludes = Sets.newHashSet("**.distributors.*.name", "**.street");
+
+        final Builder expected = builder -> builder
+            .startObject()
+                .startObject("properties")
+                    .startObject("language")
+                        .startObject("en")
+                            .field("lang", "English")
+                            .field("available", true)
+                            .startArray("distributors")
+                                .startObject()
+                                    .field("name", "The Book Shop")
+                                    .startArray("addresses")
+                                        .startObject()
+                                            .field("city", "London")
+                                        .endObject()
+                                        .startObject()
+                                            .field("city", "Stornoway")
+                                        .endObject()
+                                    .endArray()
+                                .endObject()
+                                .startObject()
+                                    .field("name", "Sussex Books House")
+                                .endObject()
+                            .endArray()
+                        .endObject()
+                    .endObject()
+                .endObject()
+            .endObject();
+
+        testFilter(expected, SAMPLE, includes, excludes);
+    }
+
+    public void testWithLfAtEnd() throws IOException {
+        final Builder sample = builder -> builder
+                .startObject()
+                    .startObject("foo")
+                        .field("bar", "baz")
+                    .endObject()
+                .endObject()
+                .prettyPrint()
+                .lfAtEnd();
+
+        testFilter(sample, sample, singleton("foo"), emptySet());
+        testFilter(sample, sample, emptySet(), singleton("bar"));
+        testFilter(sample, sample, singleton("f*"), singleton("baz"));
+
+        final Builder expected = builder -> builder.startObject().endObject().prettyPrint().lfAtEnd();
+        testFilter(expected, sample, emptySet(), singleton("foo"));
+        testFilter(expected, sample, singleton("bar"), emptySet());
+        testFilter(expected, sample, singleton("f*"), singleton("foo"));
+    }
+
+    public void testBasics() throws Exception {
+        final Builder sample = builder -> builder
+                .startObject()
+                    .field("test1", "value1")
+                    .field("test2", "value2")
+                    .field("something_else", "value3")
+                .endObject();
+
+        Builder expected = builder -> builder
+                .startObject()
+                    .field("test1", "value1")
+                .endObject();
+        testFilter(expected, sample, singleton("test1"), emptySet());
+
+        expected = builder -> builder
+                .startObject()
+                    .field("test1", "value1")
+                    .field("test2", "value2")
+                .endObject();
+        testFilter(expected, sample, singleton("test*"), emptySet());
+
+        expected = builder -> builder
+                .startObject()
+                    .field("test2", "value2")
+                    .field("something_else", "value3")
+                .endObject();
+        testFilter(expected, sample, emptySet(), singleton("test1"));
+
+        // more complex object...
+        final Builder complex = builder -> builder
+                .startObject()
+                    .startObject("path1")
+                        .startArray("path2")
+                            .startObject().field("test", "value1").endObject()
+                            .startObject().field("test", "value2").endObject()
+                        .endArray()
+                    .endObject()
+                    .field("test1", "value1")
+                .endObject();
+
+        expected = builder -> builder
+                .startObject()
+                    .startObject("path1")
+                        .startArray("path2")
+                            .startObject().field("test", "value1").endObject()
+                            .startObject().field("test", "value2").endObject()
+                        .endArray()
+                    .endObject()
+                .endObject();
+        testFilter(expected, complex, singleton("path1"), emptySet());
+        testFilter(expected, complex, singleton("path1*"), emptySet());
+        testFilter(expected, complex, singleton("path1.path2.*"), emptySet());
+
+        expected = builder -> builder
+                .startObject()
+                    .field("test1", "value1")
+                .endObject();
+        testFilter(expected, complex, singleton("test1*"), emptySet());
+    }
+
+    /**
+     * Generalization of {@link XContentMapValuesTests#testSupplementaryCharactersInPaths()}
+     */
+    public void testFilterSupplementaryCharactersInPaths() throws IOException {
+        final Builder sample = builder -> builder
+                .startObject()
+                    .field("搜索", 2)
+                    .field("指数", 3)
+                .endObject();
+
+        Builder expected = builder -> builder
+                .startObject()
+                    .field("搜索", 2)
+                .endObject();
+        testFilter(expected, sample, singleton("搜索"), emptySet());
+
+        expected = builder -> builder
+                .startObject()
+                    .field("指数", 3)
+                .endObject();
+        testFilter(expected, sample, emptySet(), singleton("搜索"));
+    }
+
+    /**
+     * Generalization of {@link XContentMapValuesTests#testSharedPrefixes()}
+     */
+    public void testFilterSharedPrefixes() throws IOException {
+        final Builder sample = builder -> builder
+                .startObject()
+                    .field("foobar", 2)
+                    .field("foobaz", 3)
+                .endObject();
+
+        Builder expected = builder -> builder
+                .startObject()
+                    .field("foobar", 2)
+                .endObject();
+        testFilter(expected, sample, singleton("foobar"), emptySet());
+
+        expected = builder -> builder
+                .startObject()
+                    .field("foobaz", 3)
+                .endObject();
+        testFilter(expected, sample, emptySet(), singleton("foobar"));
+    }
+
+    /**
+     * Generalization of {@link XContentMapValuesTests#testPrefix()}
+     */
+    public void testFilterPrefix() throws IOException {
+        final Builder sample = builder -> builder
+                .startObject()
+                    .array("photos", "foo", "bar")
+                    .field("photosCount", 2)
+                .endObject();
+
+        Builder expected = builder -> builder
+                .startObject()
+                    .field("photosCount", 2)
+                .endObject();
+        testFilter(expected, sample, singleton("photosCount"), emptySet());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/support/XContentHelperTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/support/XContentHelperTests.java
index a8091fc112273..219bd12daeaea 100644
--- a/core/src/test/java/org/elasticsearch/common/xcontent/support/XContentHelperTests.java
+++ b/core/src/test/java/org/elasticsearch/common/xcontent/support/XContentHelperTests.java
@@ -19,10 +19,17 @@
 
 package org.elasticsearch.common.xcontent.support;
 
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESTestCase;
 import org.hamcrest.Matchers;
 
+import java.io.IOException;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
@@ -61,5 +68,39 @@ public void testMergingListValuesAreMapsOfOne() {
         assertThat(content, Matchers.equalTo(expected));
     }
 
-
+    public void testToXContent() throws IOException {
+        final XContentType xContentType = randomFrom(XContentType.values());
+        final ToXContent toXContent;
+        final boolean error;
+        if (randomBoolean()) {
+            if (randomBoolean()) {
+                error = false;
+                toXContent = (builder, params) -> builder.field("field", "value");
+            } else {
+                error = true;
+                toXContent = (builder, params) -> builder.startObject().field("field", "value").endObject();
+            }
+        } else {
+            if (randomBoolean()) {
+                error = false;
+                toXContent = (ToXContentObject) (builder, params) -> builder.startObject().field("field", "value").endObject();
+            } else {
+                error = true;
+                toXContent = (ToXContentObject) (builder, params) -> builder.field("field", "value");
+            }
+        }
+        if (error) {
+            expectThrows(IOException.class, () -> XContentHelper.toXContent(toXContent, xContentType, randomBoolean()));
+        } else {
+            BytesReference bytes = XContentHelper.toXContent(toXContent, xContentType, randomBoolean());
+            try (XContentParser parser = xContentType.xContent().createParser(NamedXContentRegistry.EMPTY, bytes)) {
+                assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+                assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
+                assertTrue(parser.nextToken().isValue());
+                assertEquals("value", parser.text());
+                assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
+                assertNull(parser.nextToken());
+            }
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java
index 1c4ff9874a5aa..ce092e6f2123d 100644
--- a/core/src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java
+++ b/core/src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java
@@ -21,13 +21,12 @@
 
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.test.ESTestCase;
-import org.hamcrest.Matchers;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -35,71 +34,40 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
-import static java.util.Collections.emptyMap;
-import static java.util.Collections.singletonMap;
+import static org.elasticsearch.common.xcontent.XContentHelper.convertToMap;
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
 import static org.hamcrest.Matchers.hasEntry;
 import static org.hamcrest.Matchers.hasKey;
+import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.core.IsEqual.equalTo;
 
-/**
- */
-public class XContentMapValuesTests extends ESTestCase {
-    public void testFilter() throws Exception {
-        XContentBuilder builder = XContentFactory.jsonBuilder().startObject()
-                .field("test1", "value1")
-                .field("test2", "value2")
-                .field("something_else", "value3")
-                .endObject();
-
-        Map<String, Object> source;
-        try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.string())) {
-            source = parser.map();
-        }
-        Map<String, Object> filter = XContentMapValues.filter(source, new String[]{"test1"}, Strings.EMPTY_ARRAY);
-        assertThat(filter.size(), equalTo(1));
-        assertThat(filter.get("test1").toString(), equalTo("value1"));
-
-        filter = XContentMapValues.filter(source, new String[]{"test*"}, Strings.EMPTY_ARRAY);
-        assertThat(filter.size(), equalTo(2));
-        assertThat(filter.get("test1").toString(), equalTo("value1"));
-        assertThat(filter.get("test2").toString(), equalTo("value2"));
+public class XContentMapValuesTests extends AbstractFilteringTestCase {
 
-        filter = XContentMapValues.filter(source, Strings.EMPTY_ARRAY, new String[]{"test1"});
-        assertThat(filter.size(), equalTo(2));
-        assertThat(filter.get("test2").toString(), equalTo("value2"));
-        assertThat(filter.get("something_else").toString(), equalTo("value3"));
+    @Override
+    protected void testFilter(Builder expected, Builder actual, Set<String> includes, Set<String> excludes) throws IOException {
+        final XContentType xContentType = randomFrom(XContentType.values());
+        final boolean humanReadable = randomBoolean();
 
-        // more complex object...
-        builder = XContentFactory.jsonBuilder().startObject()
-                .startObject("path1")
-                .startArray("path2")
-                .startObject().field("test", "value1").endObject()
-                .startObject().field("test", "value2").endObject()
-                .endArray()
-                .endObject()
-                .field("test1", "value1")
-                .endObject();
-
-        try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.string())) {
-            source = parser.map();
+        String[] sourceIncludes;
+        if (includes == null) {
+            sourceIncludes = randomBoolean() ? Strings.EMPTY_ARRAY : null;
+        } else {
+            sourceIncludes = includes.toArray(new String[includes.size()]);
+        }
+        String[] sourceExcludes;
+        if (excludes == null) {
+            sourceExcludes = randomBoolean() ? Strings.EMPTY_ARRAY : null;
+        } else {
+            sourceExcludes = excludes.toArray(new String[excludes.size()]);
         }
-        filter = XContentMapValues.filter(source, new String[]{"path1"}, Strings.EMPTY_ARRAY);
-        assertThat(filter.size(), equalTo(1));
-
-        filter = XContentMapValues.filter(source, new String[]{"path1*"}, Strings.EMPTY_ARRAY);
-        assertThat(filter.get("path1"), equalTo(source.get("path1")));
-        assertThat(filter.containsKey("test1"), equalTo(false));
-
-        filter = XContentMapValues.filter(source, new String[]{"test1*"}, Strings.EMPTY_ARRAY);
-        assertThat(filter.get("test1"), equalTo(source.get("test1")));
-        assertThat(filter.containsKey("path1"), equalTo(false));
 
-        filter = XContentMapValues.filter(source, new String[]{"path1.path2.*"}, Strings.EMPTY_ARRAY);
-        assertThat(filter.get("path1"), equalTo(source.get("path1")));
-        assertThat(filter.containsKey("test1"), equalTo(false));
+        assertEquals("Filtered map must be equal to the expected map",
+                toMap(expected, xContentType, humanReadable),
+                XContentMapValues.filter(toMap(actual, xContentType, humanReadable), sourceIncludes, sourceExcludes));
     }
 
     @SuppressWarnings({"unchecked"})
@@ -109,7 +77,7 @@ public void testExtractValue() throws Exception {
                 .endObject();
 
         Map<String, Object> map;
-        try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.string())) {
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, builder.string())) {
             map = parser.map();
         }
         assertThat(XContentMapValues.extractValue("test", map).toString(), equalTo("value"));
@@ -120,7 +88,7 @@ public void testExtractValue() throws Exception {
                 .startObject("path1").startObject("path2").field("test", "value").endObject().endObject()
                 .endObject();
 
-        try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.string())) {
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, builder.string())) {
             map = parser.map();
         }
         assertThat(XContentMapValues.extractValue("path1.path2.test", map).toString(), equalTo("value"));
@@ -139,18 +107,18 @@ public void testExtractValue() throws Exception {
 
         // lists
         builder = XContentFactory.jsonBuilder().startObject()
-                .startObject("path1").field("test", "value1", "value2").endObject()
+                .startObject("path1").array("test", "value1", "value2").endObject()
                 .endObject();
 
-        try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.string())) {
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, builder.string())) {
             map = parser.map();
         }
 
         extValue = XContentMapValues.extractValue("path1.test", map);
         assertThat(extValue, instanceOf(List.class));
 
-        List extListValue = (List) extValue;
-        assertThat(extListValue.size(), equalTo(2));
+        List<?> extListValue = (List) extValue;
+        assertThat(extListValue, hasSize(2));
 
         builder = XContentFactory.jsonBuilder().startObject()
                 .startObject("path1")
@@ -161,7 +129,7 @@ public void testExtractValue() throws Exception {
                 .endObject()
                 .endObject();
 
-        try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.string())) {
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, builder.string())) {
             map = parser.map();
         }
 
@@ -169,7 +137,7 @@ public void testExtractValue() throws Exception {
         assertThat(extValue, instanceOf(List.class));
 
         extListValue = (List) extValue;
-        assertThat(extListValue.size(), equalTo(2));
+        assertThat(extListValue, hasSize(2));
         assertThat(extListValue.get(0).toString(), equalTo("value1"));
         assertThat(extListValue.get(1).toString(), equalTo("value2"));
 
@@ -177,7 +145,7 @@ public void testExtractValue() throws Exception {
         builder = XContentFactory.jsonBuilder().startObject()
                 .field("xxx.yyy", "value")
                 .endObject();
-        try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.string())) {
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, builder.string())) {
             map = parser.map();
         }
         assertThat(XContentMapValues.extractValue("xxx.yyy", map).toString(), equalTo("value"));
@@ -186,7 +154,7 @@ public void testExtractValue() throws Exception {
                 .startObject("path1.xxx").startObject("path2.yyy").field("test", "value").endObject().endObject()
                 .endObject();
 
-        try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.string())) {
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, builder.string())) {
             map = parser.map();
         }
         assertThat(XContentMapValues.extractValue("path1.xxx.path2.yyy.test", map).toString(), equalTo("value"));
@@ -198,7 +166,7 @@ public void testExtractRawValue() throws Exception {
                 .endObject();
 
         Map<String, Object> map;
-        try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.string())) {
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, builder.string())) {
             map = parser.map();
         }
         assertThat(XContentMapValues.extractRawValues("test", map).get(0).toString(), equalTo("value"));
@@ -207,7 +175,7 @@ public void testExtractRawValue() throws Exception {
                 .field("test.me", "value")
                 .endObject();
 
-        try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.string())) {
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, builder.string())) {
             map = parser.map();
         }
         assertThat(XContentMapValues.extractRawValues("test.me", map).get(0).toString(), equalTo("value"));
@@ -216,7 +184,7 @@ public void testExtractRawValue() throws Exception {
                 .startObject("path1").startObject("path2").field("test", "value").endObject().endObject()
                 .endObject();
 
-        try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.string())) {
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, builder.string())) {
             map = parser.map();
         }
         assertThat(XContentMapValues.extractRawValues("path1.path2.test", map).get(0).toString(), equalTo("value"));
@@ -225,7 +193,7 @@ public void testExtractRawValue() throws Exception {
                 .startObject("path1.xxx").startObject("path2.yyy").field("test", "value").endObject().endObject()
                 .endObject();
 
-        try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.string())) {
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, builder.string())) {
             map = parser.map();
         }
         assertThat(XContentMapValues.extractRawValues("path1.xxx.path2.yyy.test", map).get(0).toString(), equalTo("value"));
@@ -235,9 +203,9 @@ public void testPrefixedNamesFilteringTest() {
         Map<String, Object> map = new HashMap<>();
         map.put("obj", "value");
         map.put("obj_name", "value_name");
-        Map<String, Object> filterdMap = XContentMapValues.filter(map, new String[]{"obj_name"}, Strings.EMPTY_ARRAY);
-        assertThat(filterdMap.size(), equalTo(1));
-        assertThat((String) filterdMap.get("obj_name"), equalTo("value_name"));
+        Map<String, Object> filteredMap = XContentMapValues.filter(map, new String[]{"obj_name"}, Strings.EMPTY_ARRAY);
+        assertThat(filteredMap.size(), equalTo(1));
+        assertThat((String) filteredMap.get("obj_name"), equalTo("value_name"));
     }
 
 
@@ -252,19 +220,17 @@ public void testNestedFiltering() {
                             put("nested", 2);
                             put("nested_2", 3);
                         }}));
-        Map<String, Object> falteredMap = XContentMapValues.filter(map, new String[]{"array.nested"}, Strings.EMPTY_ARRAY);
-        assertThat(falteredMap.size(), equalTo(1));
+        Map<String, Object> filteredMap = XContentMapValues.filter(map, new String[]{"array.nested"}, Strings.EMPTY_ARRAY);
+        assertThat(filteredMap.size(), equalTo(1));
 
-        // Selecting members of objects within arrays (ex. [ 1, { nested: "value"} ])  always returns all values in the array (1 in the ex)
-        // this is expected behavior as this types of objects are not supported in ES
-        assertThat((Integer) ((List) falteredMap.get("array")).get(0), equalTo(1));
-        assertThat(((Map<String, Object>) ((List) falteredMap.get("array")).get(1)).size(), equalTo(1));
-        assertThat((Integer) ((Map<String, Object>) ((List) falteredMap.get("array")).get(1)).get("nested"), equalTo(2));
+        assertThat(((List<?>) filteredMap.get("array")), hasSize(1));
+        assertThat(((Map<String, Object>) ((List) filteredMap.get("array")).get(0)).size(), equalTo(1));
+        assertThat((Integer) ((Map<String, Object>) ((List) filteredMap.get("array")).get(0)).get("nested"), equalTo(2));
 
-        falteredMap = XContentMapValues.filter(map, new String[]{"array.*"}, Strings.EMPTY_ARRAY);
-        assertThat(falteredMap.size(), equalTo(1));
-        assertThat((Integer) ((List) falteredMap.get("array")).get(0), equalTo(1));
-        assertThat(((Map<String, Object>) ((List) falteredMap.get("array")).get(1)).size(), equalTo(2));
+        filteredMap = XContentMapValues.filter(map, new String[]{"array.*"}, Strings.EMPTY_ARRAY);
+        assertThat(filteredMap.size(), equalTo(1));
+        assertThat(((List<?>) filteredMap.get("array")), hasSize(1));
+        assertThat(((Map<String, Object>) ((List) filteredMap.get("array")).get(0)).size(), equalTo(2));
 
         map.clear();
         map.put("field", "value");
@@ -273,16 +239,16 @@ public void testNestedFiltering() {
                     put("field", "value");
                     put("field2", "value2");
                 }});
-        falteredMap = XContentMapValues.filter(map, new String[]{"obj.field"}, Strings.EMPTY_ARRAY);
-        assertThat(falteredMap.size(), equalTo(1));
-        assertThat(((Map<String, Object>) falteredMap.get("obj")).size(), equalTo(1));
-        assertThat((String) ((Map<String, Object>) falteredMap.get("obj")).get("field"), equalTo("value"));
+        filteredMap = XContentMapValues.filter(map, new String[]{"obj.field"}, Strings.EMPTY_ARRAY);
+        assertThat(filteredMap.size(), equalTo(1));
+        assertThat(((Map<String, Object>) filteredMap.get("obj")).size(), equalTo(1));
+        assertThat((String) ((Map<String, Object>) filteredMap.get("obj")).get("field"), equalTo("value"));
 
-        falteredMap = XContentMapValues.filter(map, new String[]{"obj.*"}, Strings.EMPTY_ARRAY);
-        assertThat(falteredMap.size(), equalTo(1));
-        assertThat(((Map<String, Object>) falteredMap.get("obj")).size(), equalTo(2));
-        assertThat((String) ((Map<String, Object>) falteredMap.get("obj")).get("field"), equalTo("value"));
-        assertThat((String) ((Map<String, Object>) falteredMap.get("obj")).get("field2"), equalTo("value2"));
+        filteredMap = XContentMapValues.filter(map, new String[]{"obj.*"}, Strings.EMPTY_ARRAY);
+        assertThat(filteredMap.size(), equalTo(1));
+        assertThat(((Map<String, Object>) filteredMap.get("obj")).size(), equalTo(2));
+        assertThat((String) ((Map<String, Object>) filteredMap.get("obj")).get("field"), equalTo("value"));
+        assertThat((String) ((Map<String, Object>) filteredMap.get("obj")).get("field2"), equalTo("value2"));
 
     }
 
@@ -324,7 +290,7 @@ public void testCompleteObjectFiltering() {
 
         filteredMap = XContentMapValues.filter(map, new String[]{"array"}, new String[]{"*.field2"});
         assertThat(filteredMap.size(), equalTo(1));
-        assertThat(((List) filteredMap.get("array")).size(), equalTo(2));
+        assertThat(((List<?>) filteredMap.get("array")), hasSize(2));
         assertThat((Integer) ((List) filteredMap.get("array")).get(0), equalTo(1));
         assertThat(((Map<String, Object>) ((List) filteredMap.get("array")).get(1)).size(), equalTo(1));
         assertThat(((Map<String, Object>) ((List) filteredMap.get("array")).get(1)).get("field").toString(), equalTo("value"));
@@ -378,7 +344,6 @@ public void testFilterWithEmptyIncludesExcludes() {
         Map<String, Object> filteredMap = XContentMapValues.filter(map, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY);
         assertThat(filteredMap.size(), equalTo(1));
         assertThat(filteredMap.get("field").toString(), equalTo("value"));
-
     }
 
     public void testThatFilterIncludesEmptyObjectWhenUsingIncludes() throws Exception {
@@ -387,7 +352,7 @@ public void testThatFilterIncludesEmptyObjectWhenUsingIncludes() throws Exceptio
                 .endObject()
                 .endObject();
 
-        Tuple<XContentType, Map<String, Object>> mapTuple = XContentHelper.convertToMap(builder.bytes(), true);
+        Tuple<XContentType, Map<String, Object>> mapTuple = convertToMap(builder.bytes(), true, builder.contentType());
         Map<String, Object> filteredSource = XContentMapValues.filter(mapTuple.v2(), new String[]{"obj"}, Strings.EMPTY_ARRAY);
 
         assertThat(mapTuple.v2(), equalTo(filteredSource));
@@ -399,7 +364,7 @@ public void testThatFilterIncludesEmptyObjectWhenUsingExcludes() throws Exceptio
                 .endObject()
                 .endObject();
 
-        Tuple<XContentType, Map<String, Object>> mapTuple = XContentHelper.convertToMap(builder.bytes(), true);
+        Tuple<XContentType, Map<String, Object>> mapTuple = convertToMap(builder.bytes(), true, builder.contentType());
         Map<String, Object> filteredSource = XContentMapValues.filter(mapTuple.v2(), Strings.EMPTY_ARRAY, new String[]{"nonExistingField"});
 
         assertThat(mapTuple.v2(), equalTo(filteredSource));
@@ -412,7 +377,7 @@ public void testNotOmittingObjectsWithExcludedProperties() throws Exception {
                 .endObject()
                 .endObject();
 
-        Tuple<XContentType, Map<String, Object>> mapTuple = XContentHelper.convertToMap(builder.bytes(), true);
+        Tuple<XContentType, Map<String, Object>> mapTuple = convertToMap(builder.bytes(), true, builder.contentType());
         Map<String, Object> filteredSource = XContentMapValues.filter(mapTuple.v2(), Strings.EMPTY_ARRAY, new String[]{"obj.f1"});
 
         assertThat(filteredSource.size(), equalTo(1));
@@ -432,25 +397,25 @@ public void testNotOmittingObjectWithNestedExcludedObject() throws Exception {
                 .endObject();
 
         // implicit include
-        Tuple<XContentType, Map<String, Object>> mapTuple = XContentHelper.convertToMap(builder.bytes(), true);
+        Tuple<XContentType, Map<String, Object>> mapTuple = convertToMap(builder.bytes(), true, builder.contentType());
         Map<String, Object> filteredSource = XContentMapValues.filter(mapTuple.v2(), Strings.EMPTY_ARRAY, new String[]{"*.obj2"});
 
         assertThat(filteredSource.size(), equalTo(1));
         assertThat(filteredSource, hasKey("obj1"));
-        assertThat(((Map) filteredSource.get("obj1")).size(), Matchers.equalTo(0));
+        assertThat(((Map) filteredSource.get("obj1")).size(), equalTo(0));
 
         // explicit include
         filteredSource = XContentMapValues.filter(mapTuple.v2(), new String[]{"obj1"}, new String[]{"*.obj2"});
         assertThat(filteredSource.size(), equalTo(1));
         assertThat(filteredSource, hasKey("obj1"));
-        assertThat(((Map) filteredSource.get("obj1")).size(), Matchers.equalTo(0));
+        assertThat(((Map) filteredSource.get("obj1")).size(), equalTo(0));
 
         // wild card include
         filteredSource = XContentMapValues.filter(mapTuple.v2(), new String[]{"*.obj2"}, new String[]{"*.obj3"});
         assertThat(filteredSource.size(), equalTo(1));
         assertThat(filteredSource, hasKey("obj1"));
         assertThat(((Map<String, Object>) filteredSource.get("obj1")), hasKey("obj2"));
-        assertThat(((Map) ((Map) filteredSource.get("obj1")).get("obj2")).size(), Matchers.equalTo(0));
+        assertThat(((Map) ((Map) filteredSource.get("obj1")).get("obj2")).size(), equalTo(0));
     }
 
     @SuppressWarnings({"unchecked"})
@@ -462,7 +427,7 @@ public void testIncludingObjectWithNestedIncludedObject() throws Exception {
                 .endObject()
                 .endObject();
 
-        Tuple<XContentType, Map<String, Object>> mapTuple = XContentHelper.convertToMap(builder.bytes(), true);
+        Tuple<XContentType, Map<String, Object>> mapTuple = convertToMap(builder.bytes(), true, builder.contentType());
         Map<String, Object> filteredSource = XContentMapValues.filter(mapTuple.v2(), new String[]{"*.obj2"}, Strings.EMPTY_ARRAY);
 
         assertThat(filteredSource.size(), equalTo(1));
@@ -472,83 +437,59 @@ public void testIncludingObjectWithNestedIncludedObject() throws Exception {
         assertThat(((Map) ((Map) filteredSource.get("obj1")).get("obj2")).size(), equalTo(0));
     }
 
-    public void testEmptyList() throws IOException {
-        XContentBuilder builder = XContentFactory.jsonBuilder().startObject()
-                .startArray("some_array")
-                .endArray().endObject();
-
-        try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.string())) {
-            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
-            assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
-            assertEquals("some_array", parser.currentName());
-            if (random().nextBoolean()) {
-                // sometimes read the start array token, sometimes not
-                assertEquals(XContentParser.Token.START_ARRAY, parser.nextToken());
-            }
-            assertEquals(Collections.emptyList(), parser.list());
-        }
+
+    public void testDotsInFieldNames() {
+        Map<String, Object> map = new HashMap<>();
+        map.put("foo.bar", 2);
+        Map<String, Object> sub = new HashMap<>();
+        sub.put("baz", 3);
+        map.put("foo", sub);
+        map.put("quux", 5);
+
+        // dots in field names in includes
+        Map<String, Object> filtered = XContentMapValues.filter(map, new String[] {"foo"}, new String[0]);
+        Map<String, Object> expected = new HashMap<>(map);
+        expected.remove("quux");
+        assertEquals(expected, filtered);
+
+        // dots in field names in excludes
+        filtered = XContentMapValues.filter(map, new String[0], new String[] {"foo"});
+        expected = new HashMap<>(map);
+        expected.keySet().retainAll(Collections.singleton("quux"));
+        assertEquals(expected, filtered);
     }
 
-    public void testSimpleList() throws IOException {
-        XContentBuilder builder = XContentFactory.jsonBuilder().startObject()
-                .startArray("some_array")
-                    .value(1)
-                    .value(3)
-                    .value(0)
-                .endArray().endObject();
-
-        try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.string())) {
-            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
-            assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
-            assertEquals("some_array", parser.currentName());
-            if (random().nextBoolean()) {
-                // sometimes read the start array token, sometimes not
-                assertEquals(XContentParser.Token.START_ARRAY, parser.nextToken());
-            }
-            assertEquals(Arrays.asList(1, 3, 0), parser.list());
-        }
+    public void testSupplementaryCharactersInPaths() {
+        Map<String, Object> map = new HashMap<>();
+        map.put("搜索", 2);
+        map.put("指数", 3);
+
+        assertEquals(Collections.singletonMap("搜索", 2), XContentMapValues.filter(map, new String[] {"搜索"}, new String[0]));
+        assertEquals(Collections.singletonMap("指数", 3), XContentMapValues.filter(map, new String[0], new String[] {"搜索"}));
     }
 
-    public void testNestedList() throws IOException {
-        XContentBuilder builder = XContentFactory.jsonBuilder().startObject()
-                .startArray("some_array")
-                    .startArray().endArray()
-                    .startArray().value(1).value(3).endArray()
-                    .startArray().value(2).endArray()
-                .endArray().endObject();
-
-        try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.string())) {
-            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
-            assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
-            assertEquals("some_array", parser.currentName());
-            if (random().nextBoolean()) {
-                // sometimes read the start array token, sometimes not
-                assertEquals(XContentParser.Token.START_ARRAY, parser.nextToken());
-            }
-            assertEquals(
-                    Arrays.asList(Collections.<Integer>emptyList(), Arrays.asList(1, 3), Arrays.asList(2)),
-                    parser.list());
-        }
+    public void testSharedPrefixes() {
+        Map<String, Object> map = new HashMap<>();
+        map.put("foobar", 2);
+        map.put("foobaz", 3);
+
+        assertEquals(Collections.singletonMap("foobar", 2), XContentMapValues.filter(map, new String[] {"foobar"}, new String[0]));
+        assertEquals(Collections.singletonMap("foobaz", 3), XContentMapValues.filter(map, new String[0], new String[] {"foobar"}));
     }
 
-    public void testNestedMapInList() throws IOException {
-        XContentBuilder builder = XContentFactory.jsonBuilder().startObject()
-                .startArray("some_array")
-                    .startObject().field("foo", "bar").endObject()
-                    .startObject().endObject()
-                .endArray().endObject();
-
-        try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.string())) {
-            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
-            assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
-            assertEquals("some_array", parser.currentName());
-            if (random().nextBoolean()) {
-                // sometimes read the start array token, sometimes not
-                assertEquals(XContentParser.Token.START_ARRAY, parser.nextToken());
-            }
-            assertEquals(
-                    Arrays.asList(singletonMap("foo", "bar"), emptyMap()),
-                    parser.list());
-        }
+    public void testPrefix() {
+        Map<String, Object> map = new HashMap<>();
+        map.put("photos", Arrays.asList(new String[] {"foo", "bar"}));
+        map.put("photosCount", 2);
+
+        Map<String, Object> filtered = XContentMapValues.filter(map, new String[] {"photosCount"}, new String[0]);
+        Map<String, Object> expected = new HashMap<>();
+        expected.put("photosCount", 2);
+        assertEquals(expected, filtered);
+    }
+
+    private static Map<String, Object> toMap(Builder test, XContentType xContentType, boolean humanReadable) throws IOException {
+        ToXContentObject toXContent = (builder, params) -> test.apply(builder);
+        return convertToMap(toXContent(toXContent, xContentType, humanReadable), true, xContentType).v2();
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractFilteringJsonGeneratorTestCase.java b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractFilteringJsonGeneratorTestCase.java
deleted file mode 100644
index f7ffcac32b33e..0000000000000
--- a/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractFilteringJsonGeneratorTestCase.java
+++ /dev/null
@@ -1,1394 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.common.xcontent.support.filtering;
-
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.util.set.Sets;
-import org.elasticsearch.common.xcontent.XContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.test.ESTestCase;
-
-import java.io.IOException;
-import java.util.Set;
-import java.util.function.Function;
-
-import static java.util.Collections.emptySet;
-import static java.util.Collections.singleton;
-import static org.hamcrest.CoreMatchers.is;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.nullValue;
-
-public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase {
-
-    protected abstract XContentType getXContentType();
-
-    protected abstract void assertXContentBuilder(XContentBuilder expected, XContentBuilder builder);
-
-    protected void assertString(XContentBuilder expected, XContentBuilder builder) {
-        assertNotNull(builder);
-        assertNotNull(expected);
-
-        // Verify that the result is equal to the expected string
-        assertThat(builder.bytes().utf8ToString(), is(expected.bytes().utf8ToString()));
-    }
-
-    protected void assertBinary(XContentBuilder expected, XContentBuilder builder) {
-        assertNotNull(builder);
-        assertNotNull(expected);
-
-        try {
-            XContent xContent = XContentFactory.xContent(builder.contentType());
-            XContentParser jsonParser = xContent.createParser(expected.bytes());
-            XContentParser testParser = xContent.createParser(builder.bytes());
-
-            while (true) {
-                XContentParser.Token token1 = jsonParser.nextToken();
-                XContentParser.Token token2 = testParser.nextToken();
-                if (token1 == null) {
-                    assertThat(token2, nullValue());
-                    return;
-                }
-                assertThat(token1, equalTo(token2));
-                switch (token1) {
-                    case FIELD_NAME:
-                        assertThat(jsonParser.currentName(), equalTo(testParser.currentName()));
-                        break;
-                    case VALUE_STRING:
-                        assertThat(jsonParser.text(), equalTo(testParser.text()));
-                        break;
-                    case VALUE_NUMBER:
-                        assertThat(jsonParser.numberType(), equalTo(testParser.numberType()));
-                        assertThat(jsonParser.numberValue(), equalTo(testParser.numberValue()));
-                        break;
-                }
-            }
-        } catch (Exception e) {
-            fail("Fail to verify the result of the XContentBuilder: " + e.getMessage());
-        }
-    }
-
-    private XContentBuilder newXContentBuilder() throws IOException {
-        return XContentBuilder.builder(getXContentType().xContent());
-    }
-
-    private XContentBuilder newXContentBuilderWithIncludes(String filter) throws IOException {
-        return newXContentBuilder(singleton(filter), emptySet());
-    }
-
-    private XContentBuilder newXContentBuilderWithExcludes(String filter) throws IOException {
-        return newXContentBuilder(emptySet(), singleton(filter));
-    }
-
-    private XContentBuilder newXContentBuilder(Set<String> includes, Set<String> excludes) throws IOException {
-        return XContentBuilder.builder(getXContentType().xContent(), includes, excludes);
-    }
-
-    /**
-     * Build a sample using a given XContentBuilder
-     */
-    private XContentBuilder sample(XContentBuilder builder) throws IOException {
-        assertNotNull(builder);
-        builder.startObject()
-                .field("title", "My awesome book")
-                .field("pages", 456)
-                .field("price", 27.99)
-                .field("timestamp", 1428582942867L)
-                .nullField("default")
-                .startArray("tags")
-                    .value("elasticsearch")
-                    .value("java")
-                .endArray()
-                .startArray("authors")
-                    .startObject()
-                        .field("name", "John Doe")
-                        .field("lastname", "John")
-                        .field("firstname", "Doe")
-                    .endObject()
-                    .startObject()
-                        .field("name", "William Smith")
-                        .field("lastname", "William")
-                        .field("firstname", "Smith")
-                    .endObject()
-                .endArray()
-                .startObject("properties")
-                    .field("weight", 0.8d)
-                    .startObject("language")
-                        .startObject("en")
-                            .field("lang", "English")
-                            .field("available", true)
-                            .startArray("distributors")
-                                .startObject()
-                                    .field("name", "The Book Shop")
-                                    .startArray("addresses")
-                                        .startObject()
-                                            .field("name", "address #1")
-                                            .field("street", "Hampton St")
-                                            .field("city", "London")
-                                        .endObject()
-                                        .startObject()
-                                            .field("name", "address #2")
-                                            .field("street", "Queen St")
-                                            .field("city", "Stornoway")
-                                        .endObject()
-                                    .endArray()
-                                .endObject()
-                                .startObject()
-                                    .field("name", "Sussex Books House")
-                                .endObject()
-                            .endArray()
-                        .endObject()
-                        .startObject("fr")
-                            .field("lang", "French")
-                            .field("available", false)
-                            .startArray("distributors")
-                                .startObject()
-                                    .field("name", "La Maison du Livre")
-                                    .startArray("addresses")
-                                        .startObject()
-                                            .field("name", "address #1")
-                                            .field("street", "Rue Mouffetard")
-                                            .field("city", "Paris")
-                                        .endObject()
-                                    .endArray()
-                                .endObject()
-                                .startObject()
-                                    .field("name", "Thetra")
-                                .endObject()
-                            .endArray()
-                        .endObject()
-                    .endObject()
-                .endObject()
-            .endObject();
-        return builder;
-    }
-
-    /** Create a new {@link XContentBuilder} and use it to build the sample using the given inclusive filter **/
-    private XContentBuilder sampleWithIncludes(String filter) throws IOException {
-        return sample(newXContentBuilderWithIncludes(filter));
-    }
-
-    /** Create a new {@link XContentBuilder} and use it to build the sample using the given exclusive filter **/
-    private XContentBuilder sampleWithExcludes(String filter) throws IOException {
-        return sample(newXContentBuilderWithExcludes(filter));
-    }
-
-    /** Create a new {@link XContentBuilder} and use it to build the sample using the given includes and exclusive filters **/
-    private XContentBuilder sampleWithFilters(Set<String> includes, Set<String> excludes) throws IOException {
-        return sample(newXContentBuilder(includes, excludes));
-    }
-
-    /** Create a new {@link XContentBuilder} and use it to build the sample **/
-    private XContentBuilder sample() throws IOException {
-        return sample(newXContentBuilder());
-    }
-
-    public void testNoFiltering() throws Exception {
-        XContentBuilder expected = sample();
-
-        assertXContentBuilder(expected, sample());
-        assertXContentBuilder(expected, sampleWithIncludes("*"));
-        assertXContentBuilder(expected, sampleWithIncludes("**"));
-        assertXContentBuilder(expected, sampleWithExcludes("xyz"));
-    }
-
-    public void testNoMatch() throws Exception {
-        XContentBuilder expected = newXContentBuilder().startObject().endObject();
-
-        assertXContentBuilder(expected, sampleWithIncludes("xyz"));
-        assertXContentBuilder(expected, sampleWithExcludes("*"));
-        assertXContentBuilder(expected, sampleWithExcludes("**"));
-    }
-
-    public void testSimpleFieldInclusive() throws Exception {
-        XContentBuilder expected = newXContentBuilder().startObject().field("title", "My awesome book").endObject();
-
-        assertXContentBuilder(expected, sampleWithIncludes("title"));
-    }
-
-    public void testSimpleFieldExclusive() throws Exception {
-        XContentBuilder expected = newXContentBuilder().startObject()
-                                                            .field("pages", 456)
-                                                            .field("price", 27.99)
-                                                            .field("timestamp", 1428582942867L)
-                                                            .nullField("default")
-                                                            .startArray("tags")
-                                                                .value("elasticsearch")
-                                                                .value("java")
-                                                            .endArray()
-                                                            .startArray("authors")
-                                                                .startObject()
-                                                                    .field("name", "John Doe")
-                                                                    .field("lastname", "John")
-                                                                    .field("firstname", "Doe")
-                                                                .endObject()
-                                                                .startObject()
-                                                                    .field("name", "William Smith")
-                                                                    .field("lastname", "William")
-                                                                    .field("firstname", "Smith")
-                                                                .endObject()
-                                                            .endArray()
-                                                            .startObject("properties")
-                                                                .field("weight", 0.8d)
-                                                                .startObject("language")
-                                                                    .startObject("en")
-                                                                        .field("lang", "English")
-                                                                        .field("available", true)
-                                                                        .startArray("distributors")
-                                                                            .startObject()
-                                                                                .field("name", "The Book Shop")
-                                                                                .startArray("addresses")
-                                                                                    .startObject()
-                                                                                        .field("name", "address #1")
-                                                                                        .field("street", "Hampton St")
-                                                                                        .field("city", "London")
-                                                                                    .endObject()
-                                                                                    .startObject()
-                                                                                        .field("name", "address #2")
-                                                                                        .field("street", "Queen St")
-                                                                                        .field("city", "Stornoway")
-                                                                                    .endObject()
-                                                                                .endArray()
-                                                                            .endObject()
-                                                                            .startObject()
-                                                                                .field("name", "Sussex Books House")
-                                                                            .endObject()
-                                                                        .endArray()
-                                                                    .endObject()
-                                                                    .startObject("fr")
-                                                                        .field("lang", "French")
-                                                                        .field("available", false)
-                                                                        .startArray("distributors")
-                                                                            .startObject()
-                                                                                .field("name", "La Maison du Livre")
-                                                                                .startArray("addresses")
-                                                                                    .startObject()
-                                                                                        .field("name", "address #1")
-                                                                                        .field("street", "Rue Mouffetard")
-                                                                                        .field("city", "Paris")
-                                                                                    .endObject()
-                                                                                .endArray()
-                                                                            .endObject()
-                                                                            .startObject()
-                                                                                .field("name", "Thetra")
-                                                                            .endObject()
-                                                                        .endArray()
-                                                                    .endObject()
-                                                                .endObject()
-                                                            .endObject()
-                                                        .endObject();
-
-        assertXContentBuilder(expected, sampleWithExcludes("title"));
-    }
-
-    public void testSimpleFieldWithWildcardInclusive() throws Exception {
-        XContentBuilder expected = newXContentBuilder().startObject()
-                                                            .field("price", 27.99)
-                                                            .startObject("properties")
-                                                                .field("weight", 0.8d)
-                                                                .startObject("language")
-                                                                    .startObject("en")
-                                                                        .field("lang", "English")
-                                                                        .field("available", true)
-                                                                        .startArray("distributors")
-                                                                            .startObject()
-                                                                                .field("name", "The Book Shop")
-                                                                                .startArray("addresses")
-                                                                                    .startObject()
-                                                                                        .field("name", "address #1")
-                                                                                        .field("street", "Hampton St")
-                                                                                        .field("city", "London")
-                                                                                    .endObject()
-                                                                                    .startObject()
-                                                                                        .field("name", "address #2")
-                                                                                        .field("street", "Queen St")
-                                                                                        .field("city", "Stornoway")
-                                                                                    .endObject()
-                                                                                .endArray()
-                                                                            .endObject()
-                                                                            .startObject()
-                                                                                .field("name", "Sussex Books House")
-                                                                            .endObject()
-                                                                        .endArray()
-                                                                    .endObject()
-                                                                    .startObject("fr")
-                                                                        .field("lang", "French")
-                                                                        .field("available", false)
-                                                                        .startArray("distributors")
-                                                                            .startObject()
-                                                                                .field("name", "La Maison du Livre")
-                                                                                .startArray("addresses")
-                                                                                    .startObject()
-                                                                                        .field("name", "address #1")
-                                                                                        .field("street", "Rue Mouffetard")
-                                                                                        .field("city", "Paris")
-                                                                                    .endObject()
-                                                                                .endArray()
-                                                                            .endObject()
-                                                                            .startObject()
-                                                                                .field("name", "Thetra")
-                                                                            .endObject()
-                                                                        .endArray()
-                                                                    .endObject()
-                                                                .endObject()
-                                                            .endObject()
-                                                        .endObject();
-
-        assertXContentBuilder(expected, sampleWithIncludes("pr*"));
-    }
-
-    public void testSimpleFieldWithWildcardExclusive() throws Exception {
-        XContentBuilder expected = newXContentBuilder().startObject()
-                                                            .field("title", "My awesome book")
-                                                            .field("pages", 456)
-                                                            .field("timestamp", 1428582942867L)
-                                                            .nullField("default")
-                                                            .startArray("tags")
-                                                                .value("elasticsearch")
-                                                                .value("java")
-                                                            .endArray()
-                                                            .startArray("authors")
-                                                                .startObject()
-                                                                    .field("name", "John Doe")
-                                                                    .field("lastname", "John")
-                                                                    .field("firstname", "Doe")
-                                                                .endObject()
-                                                                .startObject()
-                                                                    .field("name", "William Smith")
-                                                                    .field("lastname", "William")
-                                                                    .field("firstname", "Smith")
-                                                                .endObject()
-                                                            .endArray()
-                                                        .endObject();
-
-        assertXContentBuilder(expected, sampleWithExcludes("pr*"));
-    }
-
-    public void testMultipleFieldsInclusive() throws Exception {
-        XContentBuilder expected = newXContentBuilder().startObject()
-                                                            .field("title", "My awesome book")
-                                                            .field("pages", 456)
-                                                        .endObject();
-
-        assertXContentBuilder(expected, sampleWithFilters(Sets.newHashSet("title", "pages"), emptySet()));
-    }
-
-    public void testMultipleFieldsExclusive() throws Exception {
-        XContentBuilder expected = newXContentBuilder().startObject()
-                                                            .field("price", 27.99)
-                                                            .field("timestamp", 1428582942867L)
-                                                            .nullField("default")
-                                                            .startArray("tags")
-                                                                .value("elasticsearch")
-                                                                .value("java")
-                                                            .endArray()
-                                                            .startArray("authors")
-                                                                .startObject()
-                                                                    .field("name", "John Doe")
-                                                                    .field("lastname", "John")
-                                                                    .field("firstname", "Doe")
-                                                                .endObject()
-                                                                .startObject()
-                                                                    .field("name", "William Smith")
-                                                                    .field("lastname", "William")
-                                                                    .field("firstname", "Smith")
-                                                                .endObject()
-                                                            .endArray()
-                                                            .startObject("properties")
-                                                                .field("weight", 0.8d)
-                                                                .startObject("language")
-                                                                    .startObject("en")
-                                                                        .field("lang", "English")
-                                                                        .field("available", true)
-                                                                        .startArray("distributors")
-                                                                            .startObject()
-                                                                                .field("name", "The Book Shop")
-                                                                                .startArray("addresses")
-                                                                                    .startObject()
-                                                                                        .field("name", "address #1")
-                                                                                        .field("street", "Hampton St")
-                                                                                        .field("city", "London")
-                                                                                    .endObject()
-                                                                                    .startObject()
-                                                                                        .field("name", "address #2")
-                                                                                        .field("street", "Queen St")
-                                                                                        .field("city", "Stornoway")
-                                                                                    .endObject()
-                                                                                .endArray()
-                                                                            .endObject()
-                                                                            .startObject()
-                                                                                .field("name", "Sussex Books House")
-                                                                            .endObject()
-                                                                        .endArray()
-                                                                    .endObject()
-                                                                    .startObject("fr")
-                                                                        .field("lang", "French")
-                                                                        .field("available", false)
-                                                                        .startArray("distributors")
-                                                                            .startObject()
-                                                                                .field("name", "La Maison du Livre")
-                                                                                .startArray("addresses")
-                                                                                    .startObject()
-                                                                                        .field("name", "address #1")
-                                                                                        .field("street", "Rue Mouffetard")
-                                                                                        .field("city", "Paris")
-                                                                                    .endObject()
-                                                                                .endArray()
-                                                                            .endObject()
-                                                                            .startObject()
-                                                                                .field("name", "Thetra")
-                                                                            .endObject()
-                                                                        .endArray()
-                                                                    .endObject()
-                                                                .endObject()
-                                                            .endObject()
-                                                        .endObject();
-
-        assertXContentBuilder(expected, sample(newXContentBuilder(emptySet(), Sets.newHashSet("title", "pages"))));
-    }
-
-    public void testSimpleArrayInclusive() throws Exception {
-        XContentBuilder expected = newXContentBuilder().startObject()
-                                                            .startArray("tags")
-                                                                .value("elasticsearch")
-                                                                .value("java")
-                                                            .endArray()
-                                                    .endObject();
-
-        assertXContentBuilder(expected, sampleWithIncludes("tags"));
-    }
-
-    public void testSimpleArrayExclusive() throws Exception {
-        XContentBuilder expected = newXContentBuilder().startObject()
-                                                            .field("title", "My awesome book")
-                                                            .field("pages", 456)
-                                                            .field("price", 27.99)
-                                                            .field("timestamp", 1428582942867L)
-                                                            .nullField("default")
-                                                            .startArray("authors")
-                                                                .startObject()
-                                                                    .field("name", "John Doe")
-                                                                    .field("lastname", "John")
-                                                                    .field("firstname", "Doe")
-                                                                .endObject()
-                                                                .startObject()
-                                                                    .field("name", "William Smith")
-                                                                    .field("lastname", "William")
-                                                                    .field("firstname", "Smith")
-                                                                .endObject()
-                                                            .endArray()
-                                                            .startObject("properties")
-                                                                .field("weight", 0.8d)
-                                                                .startObject("language")
-                                                                    .startObject("en")
-                                                                        .field("lang", "English")
-                                                                        .field("available", true)
-                                                                        .startArray("distributors")
-                                                                            .startObject()
-                                                                                .field("name", "The Book Shop")
-                                                                                .startArray("addresses")
-                                                                                    .startObject()
-                                                                                        .field("name", "address #1")
-                                                                                        .field("street", "Hampton St")
-                                                                                        .field("city", "London")
-                                                                                    .endObject()
-                                                                                    .startObject()
-                                                                                        .field("name", "address #2")
-                                                                                        .field("street", "Queen St")
-                                                                                        .field("city", "Stornoway")
-                                                                                    .endObject()
-                                                                                .endArray()
-                                                                            .endObject()
-                                                                            .startObject()
-                                                                                .field("name", "Sussex Books House")
-                                                                            .endObject()
-                                                                        .endArray()
-                                                                    .endObject()
-                                                                    .startObject("fr")
-                                                                        .field("lang", "French")
-                                                                        .field("available", false)
-                                                                        .startArray("distributors")
-                                                                            .startObject()
-                                                                                .field("name", "La Maison du Livre")
-                                                                                .startArray("addresses")
-                                                                                    .startObject()
-                                                                                        .field("name", "address #1")
-                                                                                        .field("street", "Rue Mouffetard")
-                                                                                        .field("city", "Paris")
-                                                                                    .endObject()
-                                                                                .endArray()
-                                                                            .endObject()
-                                                                            .startObject()
-                                                                                .field("name", "Thetra")
-                                                                            .endObject()
-                                                                        .endArray()
-                                                                    .endObject()
-                                                                .endObject()
-                                                            .endObject()
-                                                        .endObject();
-
-        assertXContentBuilder(expected, sampleWithExcludes("tags"));
-    }
-
-    public void testSimpleArrayOfObjectsInclusive() throws Exception {
-        XContentBuilder expected = newXContentBuilder().startObject()
-                                                            .startArray("authors")
-                                                                .startObject()
-                                                                    .field("name", "John Doe")
-                                                                    .field("lastname", "John")
-                                                                    .field("firstname", "Doe")
-                                                                .endObject()
-                                                                .startObject()
-                                                                    .field("name", "William Smith")
-                                                                    .field("lastname", "William")
-                                                                    .field("firstname", "Smith")
-                                                                .endObject()
-                                                            .endArray()
-                                                        .endObject();
-
-        assertXContentBuilder(expected, sampleWithIncludes("authors"));
-        assertXContentBuilder(expected, sampleWithIncludes("authors.*"));
-        assertXContentBuilder(expected, sampleWithIncludes("authors.*name"));
-    }
-
-    public void testSimpleArrayOfObjectsExclusive() throws Exception {
-        XContentBuilder expected = newXContentBuilder().startObject()
-                                                            .field("title", "My awesome book")
-                                                            .field("pages", 456)
-                                                            .field("price", 27.99)
-                                                            .field("timestamp", 1428582942867L)
-                                                            .nullField("default")
-                                                            .startArray("tags")
-                                                                .value("elasticsearch")
-                                                                .value("java")
-                                                            .endArray()
-                                                            .startObject("properties")
-                                                                .field("weight", 0.8d)
-                                                                .startObject("language")
-                                                                    .startObject("en")
-                                                                        .field("lang", "English")
-                                                                        .field("available", true)
-                                                                        .startArray("distributors")
-                                                                            .startObject()
-                                                                                .field("name", "The Book Shop")
-                                                                                .startArray("addresses")
-                                                                                    .startObject()
-                                                                                        .field("name", "address #1")
-                                                                                        .field("street", "Hampton St")
-                                                                                        .field("city", "London")
-                                                                                    .endObject()
-                                                                                    .startObject()
-                                                                                        .field("name", "address #2")
-                                                                                        .field("street", "Queen St")
-                                                                                        .field("city", "Stornoway")
-                                                                                    .endObject()
-                                                                                .endArray()
-                                                                            .endObject()
-                                                                            .startObject()
-                                                                                .field("name", "Sussex Books House")
-                                                                            .endObject()
-                                                                        .endArray()
-                                                                    .endObject()
-                                                                    .startObject("fr")
-                                                                        .field("lang", "French")
-                                                                        .field("available", false)
-                                                                        .startArray("distributors")
-                                                                            .startObject()
-                                                                                .field("name", "La Maison du Livre")
-                                                                                .startArray("addresses")
-                                                                                    .startObject()
-                                                                                        .field("name", "address #1")
-                                                                                        .field("street", "Rue Mouffetard")
-                                                                                        .field("city", "Paris")
-                                                                                    .endObject()
-                                                                                .endArray()
-                                                                            .endObject()
-                                                                            .startObject()
-                                                                                .field("name", "Thetra")
-                                                                            .endObject()
-                                                                        .endArray()
-                                                                    .endObject()
-                                                                .endObject()
-                                                            .endObject()
-                                                       .endObject();
-
-        assertXContentBuilder(expected, sampleWithExcludes("authors"));
-        assertXContentBuilder(expected, sampleWithExcludes("authors.*"));
-        assertXContentBuilder(expected, sampleWithExcludes("authors.*name"));
-    }
-
-    public void testSimpleArrayOfObjectsPropertyInclusive() throws Exception {
-        XContentBuilder expected = newXContentBuilder().startObject()
-                                                            .startArray("authors")
-                                                                .startObject()
-                                                                    .field("lastname", "John")
-                                                                .endObject()
-                                                                .startObject()
-                                                                    .field("lastname", "William")
-                                                                .endObject()
-                                                            .endArray()
-                                                        .endObject();
-
-        assertXContentBuilder(expected, sampleWithIncludes("authors.lastname"));
-        assertXContentBuilder(expected, sampleWithIncludes("authors.l*"));
-    }
-
-    public void testSimpleArrayOfObjectsPropertyExclusive() throws Exception {
-        XContentBuilder expected = newXContentBuilder().startObject()
-                                                            .field("title", "My awesome book")
-                                                            .field("pages", 456)
-                                                            .field("price", 27.99)
-                                                            .field("timestamp", 1428582942867L)
-                                                            .nullField("default")
-                                                            .startArray("tags")
-                                                                .value("elasticsearch")
-                                                                .value("java")
-                                                            .endArray()
-                                                            .startArray("authors")
-                                                                .startObject()
-                                                                    .field("name", "John Doe")
-                                                                    .field("firstname", "Doe")
-                                                                .endObject()
-                                                                .startObject()
-                                                                    .field("name", "William Smith")
-                                                                    .field("firstname", "Smith")
-                                                                .endObject()
-                                                            .endArray()
-                                                            .startObject("properties")
-                                                                .field("weight", 0.8d)
-                                                                .startObject("language")
-                                                                    .startObject("en")
-                                                                        .field("lang", "English")
-                                                                        .field("available", true)
-                                                                        .startArray("distributors")
-                                                                            .startObject()
-                                                                                .field("name", "The Book Shop")
-                                                                                .startArray("addresses")
-                                                                                    .startObject()
-                                                                                        .field("name", "address #1")
-                                                                                        .field("street", "Hampton St")
-                                                                                        .field("city", "London")
-                                                                                    .endObject()
-                                                                                    .startObject()
-                                                                                        .field("name", "address #2")
-                                                                                        .field("street", "Queen St")
-                                                                                        .field("city", "Stornoway")
-                                                                                    .endObject()
-                                                                                .endArray()
-                                                                            .endObject()
-                                                                            .startObject()
-                                                                                .field("name", "Sussex Books House")
-                                                                            .endObject()
-                                                                        .endArray()
-                                                                    .endObject()
-                                                                    .startObject("fr")
-                                                                        .field("lang", "French")
-                                                                        .field("available", false)
-                                                                        .startArray("distributors")
-                                                                            .startObject()
-                                                                                .field("name", "La Maison du Livre")
-                                                                                .startArray("addresses")
-                                                                                    .startObject()
-                                                                                        .field("name", "address #1")
-                                                                                        .field("street", "Rue Mouffetard")
-                                                                                        .field("city", "Paris")
-                                                                                    .endObject()
-                                                                                .endArray()
-                                                                            .endObject()
-                                                                            .startObject()
-                                                                                .field("name", "Thetra")
-                                                                            .endObject()
-                                                                        .endArray()
-                                                                    .endObject()
-                                                                .endObject()
-                                                            .endObject()
-                                                       .endObject();
-
-        assertXContentBuilder(expected, sampleWithExcludes("authors.lastname"));
-        assertXContentBuilder(expected, sampleWithExcludes("authors.l*"));
-    }
-
-    public void testRecurseField1Inclusive() throws Exception {
-        XContentBuilder expected = newXContentBuilder().startObject()
-                                                            .startArray("authors")
-                                                                .startObject()
-                                                                    .field("name", "John Doe")
-                                                                .endObject()
-                                                                .startObject()
-                                                                    .field("name", "William Smith")
-                                                            .   endObject()
-                                                            .endArray()
-                                                            .startObject("properties")
-                                                                .startObject("language")
-                                                                    .startObject("en")
-                                                                        .startArray("distributors")
-                                                                            .startObject()
-                                                                                .field("name", "The Book Shop")
-                                                                                .startArray("addresses")
-                                                                                    .startObject()
-                                                                                        .field("name", "address #1")
-                                                                                    .endObject()
-                                                                                    .startObject()
-                                                                                        .field("name", "address #2")
-                                                                                    .endObject()
-                                                                                .endArray()
-                                                                            .endObject()
-                                                                            .startObject()
-                                                                                .field("name", "Sussex Books House")
-                                                                            .endObject()
-                                                                        .endArray()
-                                                                    .endObject()
-                                                                    .startObject("fr")
-                                                                        .startArray("distributors")
-                                                                            .startObject()
-                                                                                .field("name", "La Maison du Livre")
-                                                                                .startArray("addresses")
-                                                                                    .startObject()
-                                                                                        .field("name", "address #1")
-                                                                                    .endObject()
-                                                                                .endArray()
-                                                                            .endObject()
-                                                                            .startObject()
-                                                                                .field("name", "Thetra")
-                                                                            .endObject()
-                                                                        .endArray()
-                                                                    .endObject()
-                                                                .endObject()
-                                                            .endObject()
-                                                        .endObject();
-
-        assertXContentBuilder(expected, sampleWithIncludes("**.name"));
-    }
-
-    public void testRecurseField1Exclusive() throws Exception {
-        XContentBuilder expected = newXContentBuilder().startObject()
-                                                            .field("title", "My awesome book")
-                                                            .field("pages", 456)
-                                                            .field("price", 27.99)
-                                                            .field("timestamp", 1428582942867L)
-                                                            .nullField("default")
-                                                            .startArray("tags")
-                                                                .value("elasticsearch")
-                                                                .value("java")
-                                                            .endArray()
-                                                            .startArray("authors")
-                                                                .startObject()
-                                                                    .field("lastname", "John")
-                                                                    .field("firstname", "Doe")
-                                                                .endObject()
-                                                                .startObject()
-                                                                    .field("lastname", "William")
-                                                                    .field("firstname", "Smith")
-                                                                .endObject()
-                                                            .endArray()
-                                                            .startObject("properties")
-                                                                .field("weight", 0.8d)
-                                                                .startObject("language")
-                                                                    .startObject("en")
-                                                                        .field("lang", "English")
-                                                                        .field("available", true)
-                                                                        .startArray("distributors")
-                                                                            .startObject()
-                                                                                .startArray("addresses")
-                                                                                    .startObject()
-                                                                                        .field("street", "Hampton St")
-                                                                                        .field("city", "London")
-                                                                                    .endObject()
-                                                                                    .startObject()
-                                                                                        .field("street", "Queen St")
-                                                                                        .field("city", "Stornoway")
-                                                                                    .endObject()
-                                                                                .endArray()
-                                                                            .endObject()
-                                                                        .endArray()
-                                                                    .endObject()
-                                                                    .startObject("fr")
-                                                                        .field("lang", "French")
-                                                                        .field("available", false)
-                                                                        .startArray("distributors")
-                                                                            .startObject()
-                                                                                .startArray("addresses")
-                                                                                    .startObject()
-                                                                                        .field("street", "Rue Mouffetard")
-                                                                                        .field("city", "Paris")
-                                                                                    .endObject()
-                                                                                .endArray()
-                                                                            .endObject()
-                                                                        .endArray()
-                                                                    .endObject()
-                                                                .endObject()
-                                                            .endObject()
-                                                       .endObject();
-
-        assertXContentBuilder(expected, sampleWithExcludes("**.name"));
-    }
-
-    public void testRecurseField2Inclusive() throws Exception {
-        XContentBuilder expected = newXContentBuilder().startObject()
-                                                            .startObject("properties")
-                                                                .startObject("language")
-                                                                    .startObject("en")
-                                                                        .startArray("distributors")
-                                                                            .startObject()
-                                                                                .field("name", "The Book Shop")
-                                                                                .startArray("addresses")
-                                                                                    .startObject()
-                                                                                        .field("name", "address #1")
-                                                                                    .endObject()
-                                                                                    .startObject()
-                                                                                        .field("name", "address #2")
-                                                                                    .endObject()
-                                                                                .endArray()
-                                                                            .endObject()
-                                                                            .startObject()
-                                                                                .field("name", "Sussex Books House")
-                                                                            .endObject()
-                                                                        .endArray()
-                                                                    .endObject()
-                                                                    .startObject("fr")
-                                                                        .startArray("distributors")
-                                                                            .startObject()
-                                                                                .field("name", "La Maison du Livre")
-                                                                                .startArray("addresses")
-                                                                                    .startObject()
-                                                                                        .field("name", "address #1")
-                                                                                    .endObject()
-                                                                                .endArray()
-                                                                            .endObject()
-                                                                            .startObject()
-                                                                                .field("name", "Thetra")
-                                                                            .endObject()
-                                                                        .endArray()
-                                                                    .endObject()
-                                                                .endObject()
-                                                            .endObject()
-                                                        .endObject();
-
-        assertXContentBuilder(expected, sampleWithIncludes("properties.**.name"));
-    }
-
-    public void testRecurseField2Exclusive() throws Exception {
-        XContentBuilder expected = newXContentBuilder().startObject()
-                                                            .field("title", "My awesome book")
-                                                            .field("pages", 456)
-                                                            .field("price", 27.99)
-                                                            .field("timestamp", 1428582942867L)
-                                                            .nullField("default")
-                                                            .startArray("tags")
-                                                                .value("elasticsearch")
-                                                                .value("java")
-                                                            .endArray()
-                                                            .startArray("authors")
-                                                                .startObject()
-                                                                    .field("name", "John Doe")
-                                                                    .field("lastname", "John")
-                                                                    .field("firstname", "Doe")
-                                                                .endObject()
-                                                                .startObject()
-                                                                    .field("name", "William Smith")
-                                                                    .field("lastname", "William")
-                                                                    .field("firstname", "Smith")
-                                                                .endObject()
-                                                            .endArray()
-                                                            .startObject("properties")
-                                                                .field("weight", 0.8d)
-                                                                .startObject("language")
-                                                                    .startObject("en")
-                                                                        .field("lang", "English")
-                                                                        .field("available", true)
-                                                                        .startArray("distributors")
-                                                                            .startObject()
-                                                                                .startArray("addresses")
-                                                                                    .startObject()
-                                                                                        .field("street", "Hampton St")
-                                                                                        .field("city", "London")
-                                                                                    .endObject()
-                                                                                    .startObject()
-                                                                                        .field("street", "Queen St")
-                                                                                        .field("city", "Stornoway")
-                                                                                    .endObject()
-                                                                                .endArray()
-                                                                            .endObject()
-                                                                        .endArray()
-                                                                    .endObject()
-                                                                    .startObject("fr")
-                                                                        .field("lang", "French")
-                                                                        .field("available", false)
-                                                                        .startArray("distributors")
-                                                                            .startObject()
-                                                                                .startArray("addresses")
-                                                                                    .startObject()
-                                                                                        .field("street", "Rue Mouffetard")
-                                                                                        .field("city", "Paris")
-                                                                                    .endObject()
-                                                                                .endArray()
-                                                                            .endObject()
-                                                                        .endArray()
-                                                                    .endObject()
-                                                                .endObject()
-                                                            .endObject()
-                                                       .endObject();
-
-        assertXContentBuilder(expected, sampleWithExcludes("properties.**.name"));
-    }
-
-    public void testRecurseField3Inclusive() throws Exception {
-        XContentBuilder expected = newXContentBuilder().startObject()
-                                                            .startObject("properties")
-                                                                .startObject("language")
-                                                                    .startObject("en")
-                                                                        .startArray("distributors")
-                                                                            .startObject()
-                                                                                .field("name", "The Book Shop")
-                                                                                .startArray("addresses")
-                                                                                    .startObject()
-                                                                                        .field("name", "address #1")
-                                                                                    .endObject()
-                                                                                    .startObject()
-                                                                                        .field("name", "address #2")
-                                                                                    .endObject()
-                                                                                .endArray()
-                                                                            .endObject()
-                                                                            .startObject()
-                                                                                .field("name", "Sussex Books House")
-                                                                            .endObject()
-                                                                        .endArray()
-                                                                    .endObject()
-                                                                .endObject()
-                                                            .endObject()
-                                                        .endObject();
-
-        assertXContentBuilder(expected, sampleWithIncludes("properties.*.en.**.name"));
-    }
-
-    public void testRecurseField3Exclusive() throws Exception {
-        XContentBuilder expected = newXContentBuilder().startObject()
-                                                            .field("title", "My awesome book")
-                                                            .field("pages", 456)
-                                                            .field("price", 27.99)
-                                                            .field("timestamp", 1428582942867L)
-                                                            .nullField("default")
-                                                            .startArray("tags")
-                                                                .value("elasticsearch")
-                                                                .value("java")
-                                                            .endArray()
-                                                            .startArray("authors")
-                                                                .startObject()
-                                                                    .field("name", "John Doe")
-                                                                    .field("lastname", "John")
-                                                                    .field("firstname", "Doe")
-                                                                .endObject()
-                                                                .startObject()
-                                                                    .field("name", "William Smith")
-                                                                    .field("lastname", "William")
-                                                                    .field("firstname", "Smith")
-                                                                .endObject()
-                                                            .endArray()
-                                                            .startObject("properties")
-                                                                .field("weight", 0.8d)
-                                                                .startObject("language")
-                                                                    .startObject("en")
-                                                                        .field("lang", "English")
-                                                                        .field("available", true)
-                                                                        .startArray("distributors")
-                                                                            .startObject()
-                                                                                .startArray("addresses")
-                                                                                    .startObject()
-                                                                                        .field("street", "Hampton St")
-                                                                                        .field("city", "London")
-                                                                                    .endObject()
-                                                                                    .startObject()
-                                                                                        .field("street", "Queen St")
-                                                                                        .field("city", "Stornoway")
-                                                                                    .endObject()
-                                                                                .endArray()
-                                                                            .endObject()
-                                                                        .endArray()
-                                                                    .endObject()
-                                                                    .startObject("fr")
-                                                                        .field("lang", "French")
-                                                                        .field("available", false)
-                                                                        .startArray("distributors")
-                                                                            .startObject()
-                                                                                .field("name", "La Maison du Livre")
-                                                                                .startArray("addresses")
-                                                                                    .startObject()
-                                                                                        .field("name", "address #1")
-                                                                                        .field("street", "Rue Mouffetard")
-                                                                                        .field("city", "Paris")
-                                                                                    .endObject()
-                                                                                .endArray()
-                                                                            .endObject()
-                                                                            .startObject()
-                                                                                .field("name", "Thetra")
-                                                                            .endObject()
-                                                                        .endArray()
-                                                                    .endObject()
-                                                                .endObject()
-                                                            .endObject()
-                                                       .endObject();
-
-        assertXContentBuilder(expected, sampleWithExcludes("properties.*.en.**.name"));
-    }
-
-    public void testRecurseField4Inclusive() throws Exception {
-        XContentBuilder expected = newXContentBuilder().startObject()
-                                                            .startObject("properties")
-                                                                .startObject("language")
-                                                                    .startObject("en")
-                                                                        .startArray("distributors")
-                                                                            .startObject()
-                                                                                .field("name", "The Book Shop")
-                                                                            .endObject()
-                                                                            .startObject()
-                                                                                .field("name", "Sussex Books House")
-                                                                            .endObject()
-                                                                        .endArray()
-                                                                    .endObject()
-                                                                    .startObject("fr")
-                                                                        .startArray("distributors")
-                                                                            .startObject()
-                                                                                .field("name", "La Maison du Livre")
-                                                                            .endObject()
-                                                                            .startObject()
-                                                                                .field("name", "Thetra")
-                                                                            .endObject()
-                                                                        .endArray()
-                                                                    .endObject()
-                                                                .endObject()
-                                                            .endObject()
-                                                        .endObject();
-
-        assertXContentBuilder(expected, sampleWithIncludes("properties.**.distributors.name"));
-    }
-
-    public void testRecurseField4Exclusive() throws Exception {
-        XContentBuilder expected = newXContentBuilder().startObject()
-                                                            .field("title", "My awesome book")
-                                                            .field("pages", 456)
-                                                            .field("price", 27.99)
-                                                            .field("timestamp", 1428582942867L)
-                                                            .nullField("default")
-                                                            .startArray("tags")
-                                                                .value("elasticsearch")
-                                                                .value("java")
-                                                            .endArray()
-                                                            .startArray("authors")
-                                                                .startObject()
-                                                                    .field("name", "John Doe")
-                                                                    .field("lastname", "John")
-                                                                    .field("firstname", "Doe")
-                                                                .endObject()
-                                                                .startObject()
-                                                                    .field("name", "William Smith")
-                                                                    .field("lastname", "William")
-                                                                    .field("firstname", "Smith")
-                                                                .endObject()
-                                                            .endArray()
-                                                            .startObject("properties")
-                                                                .field("weight", 0.8d)
-                                                                .startObject("language")
-                                                                    .startObject("en")
-                                                                        .field("lang", "English")
-                                                                        .field("available", true)
-                                                                        .startArray("distributors")
-                                                                            .startObject()
-                                                                                .startArray("addresses")
-                                                                                    .startObject()
-                                                                                        .field("name", "address #1")
-                                                                                        .field("street", "Hampton St")
-                                                                                        .field("city", "London")
-                                                                                    .endObject()
-                                                                                    .startObject()
-                                                                                        .field("name", "address #2")
-                                                                                        .field("street", "Queen St")
-                                                                                        .field("city", "Stornoway")
-                                                                                    .endObject()
-                                                                                .endArray()
-                                                                            .endObject()
-                                                                        .endArray()
-                                                                    .endObject()
-                                                                    .startObject("fr")
-                                                                        .field("lang", "French")
-                                                                        .field("available", false)
-                                                                        .startArray("distributors")
-                                                                            .startObject()
-                                                                                .startArray("addresses")
-                                                                                    .startObject()
-                                                                                        .field("name", "address #1")
-                                                                                        .field("street", "Rue Mouffetard")
-                                                                                        .field("city", "Paris")
-                                                                                    .endObject()
-                                                                                .endArray()
-                                                                            .endObject()
-                                                                        .endArray()
-                                                                    .endObject()
-                                                                .endObject()
-                                                            .endObject()
-                                                       .endObject();
-
-        assertXContentBuilder(expected, sampleWithExcludes("properties.**.distributors.name"));
-    }
-
-    public void testRawField() throws Exception {
-        XContentBuilder expectedRawField = newXContentBuilder().startObject().field("foo", 0).startObject("raw")
-                .field("content", "hello world!").endObject().endObject();
-        XContentBuilder expectedRawFieldFiltered = newXContentBuilder().startObject().field("foo", 0).endObject();
-        XContentBuilder expectedRawFieldNotFiltered = newXContentBuilder().startObject().startObject("raw").field("content", "hello world!")
-                .endObject().endObject();
-
-        BytesReference raw = newXContentBuilder().startObject().field("content", "hello world!").endObject().bytes();
-
-        // Test method: rawField(String fieldName, BytesReference content)
-        assertXContentBuilder(expectedRawField, newXContentBuilder().startObject().field("foo", 0).rawField("raw", raw).endObject());
-        assertXContentBuilder(expectedRawFieldFiltered,
-                newXContentBuilderWithIncludes("f*").startObject().field("foo", 0).rawField("raw", raw).endObject());
-        assertXContentBuilder(expectedRawFieldFiltered,
-                newXContentBuilderWithExcludes("r*").startObject().field("foo", 0).rawField("raw", raw).endObject());
-        assertXContentBuilder(expectedRawFieldNotFiltered,
-                newXContentBuilderWithIncludes("r*").startObject().field("foo", 0).rawField("raw", raw).endObject());
-        assertXContentBuilder(expectedRawFieldNotFiltered,
-                newXContentBuilderWithExcludes("f*").startObject().field("foo", 0).rawField("raw", raw).endObject());
-
-        // Test method: rawField(String fieldName, InputStream content)
-        assertXContentBuilder(expectedRawField,
-                newXContentBuilder().startObject().field("foo", 0).rawField("raw", raw.streamInput()).endObject());
-        assertXContentBuilder(expectedRawFieldFiltered, newXContentBuilderWithIncludes("f*").startObject().field("foo", 0)
-                .rawField("raw", raw.streamInput()).endObject());
-        assertXContentBuilder(expectedRawFieldFiltered, newXContentBuilderWithExcludes("r*").startObject().field("foo", 0)
-                .rawField("raw", raw.streamInput()).endObject());
-        assertXContentBuilder(expectedRawFieldNotFiltered, newXContentBuilderWithIncludes("r*").startObject().field("foo", 0)
-                .rawField("raw", raw.streamInput()).endObject());
-        assertXContentBuilder(expectedRawFieldNotFiltered, newXContentBuilderWithExcludes("f*").startObject().field("foo", 0)
-                .rawField("raw", raw.streamInput()).endObject());
-    }
-
-    public void testArrays() throws Exception {
-        // Test: Array of values (no filtering)
-        XContentBuilder expected = newXContentBuilder().startObject().startArray("tags").value("lorem").value("ipsum").value("dolor")
-                .endArray().endObject();
-        assertXContentBuilder(expected, newXContentBuilderWithIncludes("t*").startObject().startArray("tags").value("lorem").value("ipsum")
-                .value("dolor").endArray().endObject());
-        assertXContentBuilder(expected, newXContentBuilderWithIncludes("tags").startObject().startArray("tags").value("lorem")
-                .value("ipsum").value("dolor").endArray().endObject());
-        assertXContentBuilder(expected, newXContentBuilderWithExcludes("a").startObject().startArray("tags").value("lorem").value("ipsum")
-                .value("dolor").endArray().endObject());
-
-        // Test: Array of values (with filtering)
-        assertXContentBuilder(newXContentBuilder().startObject().endObject(), newXContentBuilderWithIncludes("foo").startObject()
-                .startArray("tags").value("lorem").value("ipsum").value("dolor").endArray().endObject());
-        assertXContentBuilder(newXContentBuilder().startObject().endObject(), newXContentBuilderWithExcludes("t*").startObject()
-                .startArray("tags").value("lorem").value("ipsum").value("dolor").endArray().endObject());
-        assertXContentBuilder(newXContentBuilder().startObject().endObject(), newXContentBuilderWithExcludes("tags").startObject()
-                .startArray("tags").value("lorem").value("ipsum").value("dolor").endArray().endObject());
-
-        // Test: Array of objects (no filtering)
-        expected = newXContentBuilder().startObject().startArray("tags").startObject().field("lastname", "lorem").endObject().startObject()
-                .field("firstname", "ipsum").endObject().endArray().endObject();
-        assertXContentBuilder(expected, newXContentBuilderWithIncludes("t*").startObject().startArray("tags").startObject()
-                .field("lastname", "lorem").endObject().startObject().field("firstname", "ipsum").endObject().endArray().endObject());
-        assertXContentBuilder(expected, newXContentBuilderWithIncludes("tags").startObject().startArray("tags").startObject()
-                .field("lastname", "lorem").endObject().startObject().field("firstname", "ipsum").endObject().endArray().endObject());
-        assertXContentBuilder(expected, newXContentBuilderWithExcludes("a").startObject().startArray("tags").startObject()
-                .field("lastname", "lorem").endObject().startObject().field("firstname", "ipsum").endObject().endArray().endObject());
-
-        // Test: Array of objects (with filtering)
-        assertXContentBuilder(newXContentBuilder().startObject().endObject(),
-                newXContentBuilderWithIncludes("foo").startObject().startArray("tags").startObject().field("lastname", "lorem").endObject()
-                        .startObject().field("firstname", "ipsum").endObject().endArray().endObject());
-        assertXContentBuilder(newXContentBuilder().startObject().endObject(),
-                newXContentBuilderWithExcludes("t*").startObject().startArray("tags").startObject().field("lastname", "lorem").endObject()
-                        .startObject().field("firstname", "ipsum").endObject().endArray().endObject());
-        assertXContentBuilder(newXContentBuilder().startObject().endObject(),
-                newXContentBuilderWithExcludes("tags").startObject().startArray("tags").startObject().field("lastname", "lorem").endObject()
-                        .startObject().field("firstname", "ipsum").endObject().endArray().endObject());
-
-        // Test: Array of objects (with partial filtering)
-        expected = newXContentBuilder().startObject().startArray("tags").startObject().field("firstname", "ipsum").endObject().endArray()
-                .endObject();
-        assertXContentBuilder(expected, newXContentBuilderWithIncludes("t*.firstname").startObject().startArray("tags").startObject()
-                .field("lastname", "lorem").endObject().startObject().field("firstname", "ipsum").endObject().endArray().endObject());
-        assertXContentBuilder(expected, newXContentBuilderWithExcludes("t*.lastname").startObject().startArray("tags").startObject()
-                .field("lastname", "lorem").endObject().startObject().field("firstname", "ipsum").endObject().endArray().endObject());
-    }
-
-    public void testEmptyObject() throws IOException {
-        final Function<XContentBuilder, XContentBuilder> build = builder -> {
-            try {
-                return builder.startObject().startObject("foo").endObject().endObject();
-            } catch (IOException e) {
-                throw new RuntimeException(e);
-            }
-        };
-
-        XContentBuilder expected = build.apply(newXContentBuilder());
-        assertXContentBuilder(expected, build.apply(newXContentBuilderWithIncludes("foo")));
-        assertXContentBuilder(expected, build.apply(newXContentBuilderWithExcludes("bar")));
-        assertXContentBuilder(expected, build.apply(newXContentBuilder(singleton("f*"), singleton("baz"))));
-
-        expected = newXContentBuilder().startObject().endObject();
-        assertXContentBuilder(expected, build.apply(newXContentBuilderWithExcludes("foo")));
-        assertXContentBuilder(expected, build.apply(newXContentBuilderWithIncludes("bar")));
-        assertXContentBuilder(expected, build.apply(newXContentBuilder(singleton("f*"), singleton("foo"))));
-    }
-
-    public void testSingleFieldObject() throws IOException {
-        final Function<XContentBuilder, XContentBuilder> build = builder -> {
-            try {
-                return builder.startObject().startObject("foo").field("bar", "test").endObject().endObject();
-            } catch (IOException e) {
-                throw new RuntimeException(e);
-            }
-        };
-
-        XContentBuilder expected = build.apply(newXContentBuilder());
-        assertXContentBuilder(expected, build.apply(newXContentBuilderWithIncludes("foo.bar")));
-        assertXContentBuilder(expected, build.apply(newXContentBuilderWithExcludes("foo.baz")));
-        assertXContentBuilder(expected, build.apply(newXContentBuilder(singleton("foo"), singleton("foo.baz"))));
-
-        expected = newXContentBuilder().startObject().endObject();
-        assertXContentBuilder(expected, build.apply(newXContentBuilderWithExcludes("foo.bar")));
-        assertXContentBuilder(expected, build.apply(newXContentBuilder(singleton("foo"), singleton("foo.b*"))));
-    }
-
-    public void testSingleFieldWithBothExcludesIncludes() throws IOException {
-        XContentBuilder expected = newXContentBuilder()
-            .startObject()
-                .field("pages", 456)
-                .field("price", 27.99)
-            .endObject();
-
-        assertXContentBuilder(expected, sampleWithFilters(singleton("p*"), singleton("properties")));
-    }
-
-    public void testObjectsInArrayWithBothExcludesIncludes() throws IOException {
-        Set<String> includes = Sets.newHashSet("tags", "authors");
-        Set<String> excludes = singleton("authors.name");
-
-        XContentBuilder expected = newXContentBuilder()
-            .startObject()
-                .startArray("tags")
-                    .value("elasticsearch")
-                    .value("java")
-                .endArray()
-                .startArray("authors")
-                    .startObject()
-                        .field("lastname", "John")
-                        .field("firstname", "Doe")
-                    .endObject()
-                    .startObject()
-                        .field("lastname", "William")
-                        .field("firstname", "Smith")
-                    .endObject()
-                .endArray()
-            .endObject();
-
-        assertXContentBuilder(expected, sampleWithFilters(includes, excludes));
-    }
-
-    public void testRecursiveObjectsInArrayWithBothExcludesIncludes() throws IOException {
-        Set<String> includes = Sets.newHashSet("**.language", "properties.weight");
-        Set<String> excludes = singleton("**.distributors");
-
-        XContentBuilder expected = newXContentBuilder()
-            .startObject()
-                .startObject("properties")
-                    .field("weight", 0.8d)
-                    .startObject("language")
-                        .startObject("en")
-                            .field("lang", "English")
-                            .field("available", true)
-                        .endObject()
-                        .startObject("fr")
-                            .field("lang", "French")
-                            .field("available", false)
-                        .endObject()
-                    .endObject()
-                .endObject()
-            .endObject();
-
-        assertXContentBuilder(expected, sampleWithFilters(includes, excludes));
-    }
-
-    public void testRecursiveSameObjectWithBothExcludesIncludes() throws IOException {
-        Set<String> includes = singleton("**.distributors");
-        Set<String> excludes = singleton("**.distributors");
-
-        XContentBuilder expected = newXContentBuilder().startObject().endObject();
-        assertXContentBuilder(expected, sampleWithFilters(includes, excludes));
-    }
-
-    public void testRecursiveObjectsPropertiesWithBothExcludesIncludes() throws IOException {
-        Set<String> includes = singleton("**.en.*");
-        Set<String> excludes = Sets.newHashSet("**.distributors.*.name", "**.street");
-
-        XContentBuilder expected = newXContentBuilder()
-            .startObject()
-                .startObject("properties")
-                    .startObject("language")
-                        .startObject("en")
-                            .field("lang", "English")
-                            .field("available", true)
-                            .startArray("distributors")
-                                .startObject()
-                                    .field("name", "The Book Shop")
-                                    .startArray("addresses")
-                                        .startObject()
-                                            .field("city", "London")
-                                        .endObject()
-                                        .startObject()
-                                            .field("city", "Stornoway")
-                                        .endObject()
-                                    .endArray()
-                                .endObject()
-                                .startObject()
-                                    .field("name", "Sussex Books House")
-                                .endObject()
-                            .endArray()
-                        .endObject()
-                    .endObject()
-                .endObject()
-            .endObject();
-
-        assertXContentBuilder(expected, sampleWithFilters(includes, excludes));
-    }
-
-    public void testWithLfAtEnd() throws IOException {
-        final Function<XContentBuilder, XContentBuilder> build = builder -> {
-            try {
-                return builder.startObject().startObject("foo").field("bar", "baz").endObject().endObject().prettyPrint().lfAtEnd();
-            } catch (IOException e) {
-                throw new RuntimeException(e);
-            }
-        };
-
-        XContentBuilder expected = build.apply(newXContentBuilder());
-        assertXContentBuilder(expected, build.apply(newXContentBuilderWithIncludes("foo")));
-        assertXContentBuilder(expected, build.apply(newXContentBuilderWithExcludes("bar")));
-        assertXContentBuilder(expected, build.apply(newXContentBuilder(singleton("f*"), singleton("baz"))));
-
-        expected = newXContentBuilder().startObject().endObject().prettyPrint().lfAtEnd();
-        assertXContentBuilder(expected, build.apply(newXContentBuilderWithExcludes("foo")));
-        assertXContentBuilder(expected, build.apply(newXContentBuilderWithIncludes("bar")));
-        assertXContentBuilder(expected, build.apply(newXContentBuilder(singleton("f*"), singleton("foo"))));
-    }
-}
diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractXContentFilteringTestCase.java b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractXContentFilteringTestCase.java
new file mode 100644
index 0000000000000..b7ab56f452827
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractXContentFilteringTestCase.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.xcontent.support.filtering;
+
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.support.AbstractFilteringTestCase;
+
+import java.io.IOException;
+import java.util.Set;
+
+import static java.util.Collections.emptySet;
+import static java.util.Collections.singleton;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+
+public abstract class AbstractXContentFilteringTestCase extends AbstractFilteringTestCase {
+
+    protected final void testFilter(Builder expected, Builder actual, Set<String> includes, Set<String> excludes) throws IOException {
+        assertFilterResult(expected.apply(createBuilder()), actual.apply(createBuilder(includes, excludes)));
+    }
+
+    protected abstract void assertFilterResult(XContentBuilder expected, XContentBuilder actual);
+
+    protected abstract XContentType getXContentType();
+
+    private XContentBuilder createBuilder() throws IOException {
+        return XContentBuilder.builder(getXContentType().xContent());
+    }
+
+    private XContentBuilder createBuilder(Set<String> includes, Set<String> excludes) throws IOException {
+        return XContentBuilder.builder(getXContentType().xContent(), includes, excludes);
+    }
+
+    public void testSingleFieldObject() throws IOException {
+        final Builder sample = builder -> builder.startObject().startObject("foo").field("bar", "test").endObject().endObject();
+
+        Builder expected = builder -> builder.startObject().startObject("foo").field("bar", "test").endObject().endObject();
+        testFilter(expected, sample, singleton("foo.bar"), emptySet());
+        testFilter(expected, sample, emptySet(), singleton("foo.baz"));
+        testFilter(expected, sample, singleton("foo"), singleton("foo.baz"));
+
+        expected = builder -> builder.startObject().endObject();
+        testFilter(expected, sample, emptySet(), singleton("foo.bar"));
+        testFilter(expected, sample, singleton("foo"), singleton("foo.b*"));
+    }
+
+    static void assertXContentBuilderAsString(final XContentBuilder expected, final XContentBuilder actual) {
+        assertThat(actual.bytes().utf8ToString(), is(expected.bytes().utf8ToString()));
+    }
+
+    static void assertXContentBuilderAsBytes(final XContentBuilder expected, final XContentBuilder actual) {
+        try {
+            XContent xContent = XContentFactory.xContent(actual.contentType());
+            XContentParser jsonParser = xContent.createParser(NamedXContentRegistry.EMPTY, expected.bytes());
+            XContentParser testParser = xContent.createParser(NamedXContentRegistry.EMPTY, actual.bytes());
+
+            while (true) {
+                XContentParser.Token token1 = jsonParser.nextToken();
+                XContentParser.Token token2 = testParser.nextToken();
+                if (token1 == null) {
+                    assertThat(token2, nullValue());
+                    return;
+                }
+                assertThat(token1, equalTo(token2));
+                switch (token1) {
+                    case FIELD_NAME:
+                        assertThat(jsonParser.currentName(), equalTo(testParser.currentName()));
+                        break;
+                    case VALUE_STRING:
+                        assertThat(jsonParser.text(), equalTo(testParser.text()));
+                        break;
+                    case VALUE_NUMBER:
+                        assertThat(jsonParser.numberType(), equalTo(testParser.numberType()));
+                        assertThat(jsonParser.numberValue(), equalTo(testParser.numberValue()));
+                        break;
+                }
+            }
+        } catch (Exception e) {
+            fail("Fail to verify the result of the XContentBuilder: " + e.getMessage());
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/CborFilteringGeneratorTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/CborFilteringGeneratorTests.java
deleted file mode 100644
index fab77a26be740..0000000000000
--- a/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/CborFilteringGeneratorTests.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.common.xcontent.support.filtering;
-
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentType;
-
-public class CborFilteringGeneratorTests extends JsonFilteringGeneratorTests {
-
-    @Override
-    protected XContentType getXContentType() {
-        return XContentType.CBOR;
-    }
-
-    @Override
-    protected void assertXContentBuilder(XContentBuilder expected, XContentBuilder builder) {
-        assertBinary(expected, builder);
-    }
-}
diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/CborXContentFilteringTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/CborXContentFilteringTests.java
new file mode 100644
index 0000000000000..db63987c17d36
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/CborXContentFilteringTests.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.xcontent.support.filtering;
+
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentType;
+
+public class CborXContentFilteringTests extends AbstractXContentFilteringTestCase {
+
+    @Override
+    protected XContentType getXContentType() {
+        return XContentType.CBOR;
+    }
+
+    @Override
+    protected void assertFilterResult(XContentBuilder expected, XContentBuilder actual) {
+        assertXContentBuilderAsBytes(expected, actual);
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathTests.java
index 4eec46d9b274e..e151ac67a332f 100644
--- a/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathTests.java
+++ b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathTests.java
@@ -202,7 +202,7 @@ public void testSimpleWildcardFilterPath() {
         assertThat(filterPath.isSimpleWildcard(), is(true));
         assertThat(filterPath.getSegment(), equalTo("*"));
 
-        FilterPath next = filterPath.matchProperty(randomAsciiOfLength(2));
+        FilterPath next = filterPath.matchProperty(randomAlphaOfLength(2));
         assertNotNull(next);
         assertSame(next, FilterPath.EMPTY);
     }
@@ -246,7 +246,7 @@ public void testDoubleWildcardFilterPath() {
         assertThat(filterPath.isDoubleWildcard(), is(true));
         assertThat(filterPath.getSegment(), equalTo("**"));
 
-        FilterPath next = filterPath.matchProperty(randomAsciiOfLength(2));
+        FilterPath next = filterPath.matchProperty(randomAlphaOfLength(2));
         assertNotNull(next);
         assertSame(next, FilterPath.EMPTY);
     }
@@ -263,7 +263,7 @@ public void testStartsWithDoubleWildcardFilterPath() {
         assertThat(filterPath.matches(), is(false));
         assertThat(filterPath.getSegment(), equalTo("**"));
 
-        FilterPath next = filterPath.matchProperty(randomAsciiOfLength(2));
+        FilterPath next = filterPath.matchProperty(randomAlphaOfLength(2));
         assertNotNull(next);
         assertThat(next.matches(), is(false));
         assertThat(next.getSegment(), equalTo("bar"));
diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/JsonFilteringGeneratorTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/JsonFilteringGeneratorTests.java
deleted file mode 100644
index a5188842fdcf3..0000000000000
--- a/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/JsonFilteringGeneratorTests.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.common.xcontent.support.filtering;
-
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentType;
-
-public class JsonFilteringGeneratorTests extends AbstractFilteringJsonGeneratorTestCase {
-
-    @Override
-    protected XContentType getXContentType() {
-        return XContentType.JSON;
-    }
-
-    @Override
-    protected void assertXContentBuilder(XContentBuilder expected, XContentBuilder builder) {
-        assertString(expected, builder);
-    }
-}
diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/JsonXContentFilteringTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/JsonXContentFilteringTests.java
new file mode 100644
index 0000000000000..2a473029aaacb
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/JsonXContentFilteringTests.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.xcontent.support.filtering;
+
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentType;
+
+public class JsonXContentFilteringTests extends AbstractXContentFilteringTestCase {
+
+    @Override
+    protected XContentType getXContentType() {
+        return XContentType.JSON;
+    }
+
+    @Override
+    protected void assertFilterResult(XContentBuilder expected, XContentBuilder actual) {
+        if (randomBoolean()) {
+            assertXContentBuilderAsString(expected, actual);
+        } else {
+            assertXContentBuilderAsBytes(expected, actual);
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/SmileFilteringGeneratorTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/SmileFilteringGeneratorTests.java
index a12e12be17202..b4f35350d69b9 100644
--- a/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/SmileFilteringGeneratorTests.java
+++ b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/SmileFilteringGeneratorTests.java
@@ -22,7 +22,7 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentType;
 
-public class SmileFilteringGeneratorTests extends JsonFilteringGeneratorTests {
+public class SmileFilteringGeneratorTests extends AbstractXContentFilteringTestCase {
 
     @Override
     protected XContentType getXContentType() {
@@ -30,7 +30,7 @@ protected XContentType getXContentType() {
     }
 
     @Override
-    protected void assertXContentBuilder(XContentBuilder expected, XContentBuilder builder) {
-        assertBinary(expected, builder);
+    protected void assertFilterResult(XContentBuilder expected, XContentBuilder actual) {
+        assertXContentBuilderAsBytes(expected, actual);
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/YamlFilteringGeneratorTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/YamlFilteringGeneratorTests.java
index c85fbc922538a..a101da0e4b1b0 100644
--- a/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/YamlFilteringGeneratorTests.java
+++ b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/YamlFilteringGeneratorTests.java
@@ -22,7 +22,7 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentType;
 
-public class YamlFilteringGeneratorTests extends AbstractFilteringJsonGeneratorTestCase {
+public class YamlFilteringGeneratorTests extends AbstractXContentFilteringTestCase {
 
     @Override
     protected XContentType getXContentType() {
@@ -30,7 +30,11 @@ protected XContentType getXContentType() {
     }
 
     @Override
-    protected void assertXContentBuilder(XContentBuilder expected, XContentBuilder builder) {
-        assertString(expected, builder);
+    protected void assertFilterResult(XContentBuilder expected, XContentBuilder actual) {
+        if (randomBoolean()) {
+            assertXContentBuilderAsString(expected, actual);
+        } else {
+            assertXContentBuilderAsBytes(expected, actual);
+        }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/deps/joda/SimpleJodaTests.java b/core/src/test/java/org/elasticsearch/deps/joda/SimpleJodaTests.java
index 442a566a77e8b..91c9814b45fc8 100644
--- a/core/src/test/java/org/elasticsearch/deps/joda/SimpleJodaTests.java
+++ b/core/src/test/java/org/elasticsearch/deps/joda/SimpleJodaTests.java
@@ -317,6 +317,36 @@ public void testForInvalidDatesInEpochMillis() {
         }
     }
 
+    public void testForInvalidTimeZoneWithEpochSeconds() {
+        DateTimeFormatter dateTimeFormatter = new DateTimeFormatterBuilder()
+            .append(new Joda.EpochTimeParser(false))
+            .toFormatter()
+            .withZone(DateTimeZone.forOffsetHours(1));
+        FormatDateTimeFormatter formatter =
+            new FormatDateTimeFormatter("epoch_seconds", dateTimeFormatter, Locale.ROOT);
+        try {
+            formatter.parser().parseDateTime("1433144433655");
+            fail("Expected IllegalArgumentException");
+        } catch (IllegalArgumentException e) {
+            assertThat(e.getMessage(), containsString("time_zone must be UTC"));
+        }
+    }
+
+    public void testForInvalidTimeZoneWithEpochMillis() {
+        DateTimeFormatter dateTimeFormatter = new DateTimeFormatterBuilder()
+            .append(new Joda.EpochTimeParser(true))
+            .toFormatter()
+            .withZone(DateTimeZone.forOffsetHours(1));
+        FormatDateTimeFormatter formatter =
+            new FormatDateTimeFormatter("epoch_millis", dateTimeFormatter, Locale.ROOT);
+        try {
+            formatter.parser().parseDateTime("1433144433");
+            fail("Expected IllegalArgumentException");
+        } catch (IllegalArgumentException e) {
+            assertThat(e.getMessage(), containsString("time_zone must be UTC"));
+        }
+    }
+
     public void testThatEpochParserIsPrinter() {
         FormatDateTimeFormatter formatter = Joda.forPattern("epoch_millis");
         assertThat(formatter.parser().isPrinter(), is(true));
diff --git a/core/src/test/java/org/elasticsearch/discovery/BlockingClusterStatePublishResponseHandlerTests.java b/core/src/test/java/org/elasticsearch/discovery/BlockingClusterStatePublishResponseHandlerTests.java
index 481d15020fc46..ca587fe28e5be 100644
--- a/core/src/test/java/org/elasticsearch/discovery/BlockingClusterStatePublishResponseHandlerTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/BlockingClusterStatePublishResponseHandlerTests.java
@@ -48,7 +48,7 @@ private static class PublishResponder extends AbstractRunnable {
         final Logger logger;
         final BlockingClusterStatePublishResponseHandler handler;
 
-        public PublishResponder(boolean fail, DiscoveryNode node, CyclicBarrier barrier, Logger logger, BlockingClusterStatePublishResponseHandler handler) {
+        PublishResponder(boolean fail, DiscoveryNode node, CyclicBarrier barrier, Logger logger, BlockingClusterStatePublishResponseHandler handler) {
             this.fail = fail;
 
             this.node = node;
diff --git a/core/src/test/java/org/elasticsearch/discovery/DiscoveryModuleTests.java b/core/src/test/java/org/elasticsearch/discovery/DiscoveryModuleTests.java
index 057b54c7a078f..59043398c44b1 100644
--- a/core/src/test/java/org/elasticsearch/discovery/DiscoveryModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/DiscoveryModuleTests.java
@@ -18,53 +18,144 @@
  */
 package org.elasticsearch.discovery;
 
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Supplier;
+
+import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.Version;
-import org.elasticsearch.common.inject.ModuleTestCase;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.network.NetworkService;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.discovery.local.LocalDiscovery;
+import org.elasticsearch.discovery.zen.UnicastHostsProvider;
 import org.elasticsearch.discovery.zen.ZenDiscovery;
-import org.elasticsearch.discovery.zen.elect.ElectMasterService;
-import org.elasticsearch.node.Node;
+import org.elasticsearch.plugins.DiscoveryPlugin;
+import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.NoopDiscovery;
+import org.elasticsearch.test.transport.MockTransportService;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.junit.After;
+import org.junit.Before;
 
-/**
- */
-public class DiscoveryModuleTests extends ModuleTestCase {
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class DiscoveryModuleTests extends ESTestCase {
 
-    public static class DummyMasterElectionService extends ElectMasterService {
+    private TransportService transportService;
+    private NamedWriteableRegistry namedWriteableRegistry;
+    private ClusterService clusterService;
+    private ThreadPool threadPool;
+
+    public interface DummyHostsProviderPlugin extends DiscoveryPlugin {
+        Map<String, Supplier<UnicastHostsProvider>> impl();
+        @Override
+        default Map<String, Supplier<UnicastHostsProvider>> getZenHostsProviders(TransportService transportService,
+                                                                                 NetworkService networkService) {
+            return impl();
+        }
+    }
 
-        public DummyMasterElectionService(Settings settings) {
-            super(settings);
+    public interface DummyDiscoveryPlugin extends DiscoveryPlugin {
+        Map<String, Supplier<Discovery>> impl();
+        @Override
+        default Map<String, Supplier<Discovery>> getDiscoveryTypes(ThreadPool threadPool, TransportService transportService,
+                                                                   NamedWriteableRegistry namedWriteableRegistry,
+                                                                   ClusterService clusterService, UnicastHostsProvider hostsProvider) {
+            return impl();
         }
     }
 
-    public void testRegisterMasterElectionService() {
-        Settings settings = Settings.builder().put(DiscoveryModule.ZEN_MASTER_SERVICE_TYPE_SETTING.getKey(), "custom").build();
-        DiscoveryModule module = new DiscoveryModule(settings);
-        module.addElectMasterService("custom", DummyMasterElectionService.class);
-        assertBinding(module, ElectMasterService.class, DummyMasterElectionService.class);
-        assertBinding(module, Discovery.class, ZenDiscovery.class);
+    @Before
+    public void setupDummyServices() {
+        threadPool = mock(ThreadPool.class);
+        transportService = MockTransportService.local(Settings.EMPTY, Version.CURRENT, threadPool, null);
+        clusterService = mock(ClusterService.class);
+        namedWriteableRegistry = new NamedWriteableRegistry(Collections.emptyList());
+        ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        when(clusterService.getClusterSettings()).thenReturn(clusterSettings);
+
+    }
+
+    @After
+    public void clearDummyServices() throws IOException {
+        IOUtils.close(transportService);
+    }
+
+    private DiscoveryModule newModule(Settings settings, List<DiscoveryPlugin> plugins) {
+        return new DiscoveryModule(settings, threadPool, transportService, namedWriteableRegistry, null, clusterService, plugins);
     }
 
-    public void testLoadUnregisteredMasterElectionService() {
-        Settings settings = Settings.builder().put(DiscoveryModule.ZEN_MASTER_SERVICE_TYPE_SETTING.getKey(), "foobar").build();
-        DiscoveryModule module = new DiscoveryModule(settings);
-        module.addElectMasterService("custom", DummyMasterElectionService.class);
-        assertBindingFailure(module, "Unknown master service type [foobar]");
+    public void testDefaults() {
+        DiscoveryModule module = newModule(Settings.EMPTY, Collections.emptyList());
+        assertTrue(module.getDiscovery() instanceof ZenDiscovery);
     }
 
-    public void testRegisterDefaults() {
-        Settings settings = Settings.EMPTY;
-        DiscoveryModule module = new DiscoveryModule(settings);
-        assertBinding(module, Discovery.class, ZenDiscovery.class);
+    public void testLazyConstructionDiscovery() {
+        DummyDiscoveryPlugin plugin = () -> Collections.singletonMap("custom",
+            () -> { throw new AssertionError("created discovery type which was not selected"); });
+        newModule(Settings.EMPTY, Collections.singletonList(plugin));
     }
 
     public void testRegisterDiscovery() {
         Settings settings = Settings.builder().put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), "custom").build();
-        DiscoveryModule module = new DiscoveryModule(settings);
-        module.addDiscoveryType("custom", NoopDiscovery.class);
-        assertBinding(module, Discovery.class, NoopDiscovery.class);
+        DummyDiscoveryPlugin plugin = () -> Collections.singletonMap("custom", NoopDiscovery::new);
+        DiscoveryModule module = newModule(settings, Collections.singletonList(plugin));
+        assertTrue(module.getDiscovery() instanceof NoopDiscovery);
     }
 
+    public void testUnknownDiscovery() {
+        Settings settings = Settings.builder().put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), "dne").build();
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () ->
+            newModule(settings, Collections.emptyList()));
+        assertEquals("Unknown discovery type [dne]", e.getMessage());
+    }
+
+    public void testDuplicateDiscovery() {
+        DummyDiscoveryPlugin plugin1 = () -> Collections.singletonMap("dup", () -> null);
+        DummyDiscoveryPlugin plugin2 = () -> Collections.singletonMap("dup", () -> null);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () ->
+            newModule(Settings.EMPTY, Arrays.asList(plugin1, plugin2)));
+        assertEquals("Cannot register discovery type [dup] twice", e.getMessage());
+    }
+
+    public void testHostsProvider() {
+        Settings settings = Settings.builder().put(DiscoveryModule.DISCOVERY_HOSTS_PROVIDER_SETTING.getKey(), "custom").build();
+        final UnicastHostsProvider provider = Collections::emptyList;
+        AtomicBoolean created = new AtomicBoolean(false);
+        DummyHostsProviderPlugin plugin = () -> Collections.singletonMap("custom", () -> {
+            created.set(true);
+            return Collections::emptyList;
+        });
+        newModule(settings, Collections.singletonList(plugin));
+        assertTrue(created.get());
+    }
 
+    public void testUnknownHostsProvider() {
+        Settings settings = Settings.builder().put(DiscoveryModule.DISCOVERY_HOSTS_PROVIDER_SETTING.getKey(), "dne").build();
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () ->
+            newModule(settings, Collections.emptyList()));
+        assertEquals("Unknown zen hosts provider [dne]", e.getMessage());
+    }
+
+    public void testDuplicateHostsProvider() {
+        DummyHostsProviderPlugin plugin1 = () -> Collections.singletonMap("dup", () -> null);
+        DummyHostsProviderPlugin plugin2 = () -> Collections.singletonMap("dup", () -> null);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () ->
+            newModule(Settings.EMPTY, Arrays.asList(plugin1, plugin2)));
+        assertEquals("Cannot register zen hosts provider [dup] twice", e.getMessage());
+    }
+
+    public void testLazyConstructionHostsProvider() {
+        DummyHostsProviderPlugin plugin = () -> Collections.singletonMap("custom",
+            () -> { throw new AssertionError("created hosts provider which was not selected"); });
+        newModule(Settings.EMPTY, Collections.singletonList(plugin));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java b/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java
index fe06f9a85d4a3..ff77a512f9007 100644
--- a/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java
+++ b/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java
@@ -28,9 +28,7 @@
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
 import org.elasticsearch.cluster.block.ClusterBlock;
@@ -41,7 +39,6 @@
 import org.elasticsearch.cluster.routing.Murmur3HashFunction;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
-import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Priority;
@@ -49,14 +46,14 @@
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.discovery.zen.ElectMasterService;
+import org.elasticsearch.discovery.zen.FaultDetection;
+import org.elasticsearch.discovery.zen.MembershipAction;
+import org.elasticsearch.discovery.zen.PublishClusterStateAction;
+import org.elasticsearch.discovery.zen.UnicastZenPing;
 import org.elasticsearch.discovery.zen.ZenDiscovery;
-import org.elasticsearch.discovery.zen.elect.ElectMasterService;
-import org.elasticsearch.discovery.zen.fd.FaultDetection;
-import org.elasticsearch.discovery.zen.membership.MembershipAction;
-import org.elasticsearch.discovery.zen.ping.ZenPing;
-import org.elasticsearch.discovery.zen.ping.ZenPingService;
-import org.elasticsearch.discovery.zen.ping.unicast.UnicastZenPing;
-import org.elasticsearch.discovery.zen.publish.PublishClusterStateAction;
+import org.elasticsearch.discovery.zen.ZenPing;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.indices.store.IndicesStoreIntegrationIT;
 import org.elasticsearch.monitor.jvm.HotThreads;
@@ -66,7 +63,7 @@
 import org.elasticsearch.test.ESIntegTestCase.Scope;
 import org.elasticsearch.test.InternalTestCluster;
 import org.elasticsearch.test.discovery.ClusterDiscoveryConfiguration;
-import org.elasticsearch.test.disruption.BlockClusterStateProcessing;
+import org.elasticsearch.test.discovery.TestZenDiscovery;
 import org.elasticsearch.test.disruption.IntermittentLongGCDisruption;
 import org.elasticsearch.test.disruption.LongGCDisruption;
 import org.elasticsearch.test.disruption.NetworkDisruption;
@@ -82,7 +79,8 @@
 import org.elasticsearch.test.disruption.SlowClusterStateProcessing;
 import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.test.transport.MockTransportService;
-import org.elasticsearch.transport.TransportException;
+import org.elasticsearch.transport.ConnectionProfile;
+import org.elasticsearch.transport.TcpTransport;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportService;
@@ -109,26 +107,29 @@
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.cluster.metadata.IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.nullValue;
 
-@ClusterScope(scope = Scope.TEST, numDataNodes = 0, transportClientRatio = 0)
+@ClusterScope(scope = Scope.TEST, numDataNodes = 0, transportClientRatio = 0, autoMinMasterNodes = false)
 @ESIntegTestCase.SuppressLocalMode
-@TestLogging("_root:DEBUG,cluster.service:TRACE")
+@TestLogging("_root:DEBUG,org.elasticsearch.cluster.service:TRACE")
 public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase {
 
     private static final TimeValue DISRUPTION_HEALING_OVERHEAD = TimeValue.timeValueSeconds(40); // we use 30s as timeout in many places.
 
     private ClusterDiscoveryConfiguration discoveryConfig;
 
-
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {
-        return discoveryConfig.nodeSettings(nodeOrdinal);
+        return Settings.builder().put(discoveryConfig.nodeSettings(nodeOrdinal))
+            .put(TestZenDiscovery.USE_MOCK_PINGS.getKey(), false).build();
     }
 
     @Before
@@ -146,11 +147,31 @@ protected int numberOfReplicas() {
         return 1;
     }
 
+    private boolean disableBeforeIndexDeletion;
+
+    @Before
+    public void setUp() throws Exception {
+        super.setUp();
+        disableBeforeIndexDeletion = false;
+    }
+
+    @Override
+    public void setDisruptionScheme(ServiceDisruptionScheme scheme) {
+        if (scheme instanceof NetworkDisruption &&
+            ((NetworkDisruption) scheme).getNetworkLinkDisruptionType() instanceof NetworkUnresponsive) {
+            // the network unresponsive disruption may leave operations in flight
+            // this is because this disruption scheme swallows requests by design
+            // as such, these operations will never be marked as finished
+            disableBeforeIndexDeletion = true;
+        }
+        super.setDisruptionScheme(scheme);
+    }
+
     @Override
-    protected void beforeIndexDeletion() {
-        // some test may leave operations in flight
-        // this is because the disruption schemes swallow requests by design
-        // as such, these operations will never be marked as finished
+    protected void beforeIndexDeletion() throws Exception {
+        if (disableBeforeIndexDeletion == false) {
+            super.beforeIndexDeletion();
+        }
     }
 
     private List<String> startCluster(int numberOfNodes) throws ExecutionException, InterruptedException {
@@ -162,43 +183,45 @@ private List<String> startCluster(int numberOfNodes, int minimumMasterNode) thro
     }
 
     private List<String> startCluster(int numberOfNodes, int minimumMasterNode, @Nullable int[] unicastHostsOrdinals) throws
-            ExecutionException, InterruptedException {
-        configureUnicastCluster(numberOfNodes, unicastHostsOrdinals, minimumMasterNode);
-        List<String> nodes = internalCluster().startNodesAsync(numberOfNodes).get();
+        ExecutionException, InterruptedException {
+        configureCluster(numberOfNodes, unicastHostsOrdinals, minimumMasterNode);
+        List<String> nodes = internalCluster().startNodes(numberOfNodes);
         ensureStableCluster(numberOfNodes);
 
         // TODO: this is a temporary solution so that nodes will not base their reaction to a partition based on previous successful results
-        for (ZenPingService pingService : internalCluster().getInstances(ZenPingService.class)) {
-            for (ZenPing zenPing : pingService.zenPings()) {
-                if (zenPing instanceof UnicastZenPing) {
-                    ((UnicastZenPing) zenPing).clearTemporalResponses();
-                }
-            }
+        ZenPing zenPing = ((TestZenDiscovery) internalCluster().getInstance(Discovery.class)).getZenPing();
+        if (zenPing instanceof UnicastZenPing) {
+            ((UnicastZenPing) zenPing).clearTemporalResponses();
         }
         return nodes;
     }
 
     static final Settings DEFAULT_SETTINGS = Settings.builder()
-            .put(FaultDetection.PING_TIMEOUT_SETTING.getKey(), "1s") // for hitting simulated network failures quickly
-            .put(FaultDetection.PING_RETRIES_SETTING.getKey(), "1") // for hitting simulated network failures quickly
-            .put("discovery.zen.join_timeout", "10s")  // still long to induce failures but to long so test won't time out
-            .put(DiscoverySettings.PUBLISH_TIMEOUT_SETTING.getKey(), "1s") // <-- for hitting simulated network failures quickly
-            .build();
+        .put(FaultDetection.PING_TIMEOUT_SETTING.getKey(), "1s") // for hitting simulated network failures quickly
+        .put(FaultDetection.PING_RETRIES_SETTING.getKey(), "1") // for hitting simulated network failures quickly
+        .put("discovery.zen.join_timeout", "10s")  // still long to induce failures but to long so test won't time out
+        .put(DiscoverySettings.PUBLISH_TIMEOUT_SETTING.getKey(), "1s") // <-- for hitting simulated network failures quickly
+        .put(TcpTransport.TCP_CONNECT_TIMEOUT.getKey(), "10s") // Network delay disruption waits for the min between this
+        // value and the time of disruption and does not recover immediately
+        // when disruption is stop. We should make sure we recover faster
+        // then the default of 30s, causing ensureGreen and friends to time out
+
+        .build();
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
         return Arrays.asList(MockTransportService.TestPlugin.class);
     }
 
-    private void configureUnicastCluster(
+    private void configureCluster(
         int numberOfNodes,
         @Nullable int[] unicastHostsOrdinals,
         int minimumMasterNode
     ) throws ExecutionException, InterruptedException {
-        configureUnicastCluster(DEFAULT_SETTINGS, numberOfNodes, unicastHostsOrdinals, minimumMasterNode);
+        configureCluster(DEFAULT_SETTINGS, numberOfNodes, unicastHostsOrdinals, minimumMasterNode);
     }
 
-    private void configureUnicastCluster(
+    private void configureCluster(
         Settings settings,
         int numberOfNodes,
         @Nullable int[] unicastHostsOrdinals,
@@ -210,10 +233,10 @@ private void configureUnicastCluster(
         logger.info("---> configured unicast");
         // TODO: Rarely use default settings form some of these
         Settings nodeSettings = Settings.builder()
-                .put(settings)
-                .put(NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.getKey(), numberOfNodes)
-                .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), minimumMasterNode)
-                .build();
+            .put(settings)
+            .put(NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.getKey(), numberOfNodes)
+            .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), minimumMasterNode)
+            .build();
 
         if (discoveryConfig == null) {
             if (unicastHostsOrdinals == null) {
@@ -279,8 +302,8 @@ public void testNodesFDAfterMasterReelection() throws Exception {
 
         logger.info("--> reducing min master nodes to 2");
         assertAcked(client().admin().cluster().prepareUpdateSettings()
-                .setTransientSettings(Settings.builder().put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), 2))
-                .get());
+            .setTransientSettings(Settings.builder().put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), 2))
+            .get());
 
         String master = internalCluster().getMasterName();
         String nonMaster = null;
@@ -307,8 +330,8 @@ public void testVerifyApiBlocksDuringPartition() throws Exception {
 
         // Makes sure that the get request can be executed on each node locally:
         assertAcked(prepareCreate("test").setSettings(Settings.builder()
-                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
-                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 2)
+            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 2)
         ));
 
         // Everything is stable now, it is now time to simulate evil...
@@ -349,7 +372,7 @@ public void testVerifyApiBlocksDuringPartition() throws Exception {
             }
             if (!success) {
                 fail("node [" + node + "] has no master or has blocks, despite of being on the right side of the partition. State dump:\n"
-                        + nodeState.prettyPrint());
+                    + nodeState);
             }
         }
 
@@ -361,8 +384,8 @@ public void testVerifyApiBlocksDuringPartition() throws Exception {
 
         logger.info("Verify no master block with {} set to {}", DiscoverySettings.NO_MASTER_BLOCK_SETTING.getKey(), "all");
         client().admin().cluster().prepareUpdateSettings()
-                .setTransientSettings(Settings.builder().put(DiscoverySettings.NO_MASTER_BLOCK_SETTING.getKey(), "all"))
-                .get();
+            .setTransientSettings(Settings.builder().put(DiscoverySettings.NO_MASTER_BLOCK_SETTING.getKey(), "all"))
+            .get();
 
         networkDisruption.startDisrupting();
 
@@ -384,15 +407,15 @@ public void testVerifyApiBlocksDuringPartition() throws Exception {
      * This test isolates the master from rest of the cluster, waits for a new master to be elected, restores the partition
      * and verifies that all node agree on the new cluster state
      */
-    @TestLogging("_root:DEBUG,cluster.service:TRACE,gateway:TRACE,indices.store:TRACE")
+    @TestLogging("_root:DEBUG,org.elasticsearch.cluster.service:TRACE,org.elasticsearch.gateway:TRACE,org.elasticsearch.indices.store:TRACE")
     public void testIsolateMasterAndVerifyClusterStateConsensus() throws Exception {
         final List<String> nodes = startCluster(3);
 
         assertAcked(prepareCreate("test")
-                .setSettings(Settings.builder()
-                        .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1 + randomInt(2))
-                        .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, randomInt(2))
-                ));
+            .setSettings(Settings.builder()
+                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1 + randomInt(2))
+                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, randomInt(2))
+            ));
 
         ensureGreen();
         String isolatedNode = internalCluster().getMasterName();
@@ -413,7 +436,7 @@ public void testIsolateMasterAndVerifyClusterStateConsensus() throws Exception {
 
         for (String node : nodes) {
             ensureStableCluster(3, new TimeValue(DISRUPTION_HEALING_OVERHEAD.millis() + networkDisruption.expectedTimeToHeal().millis()),
-                    true, node);
+                true, node);
         }
 
         logger.info("issue a reroute");
@@ -436,13 +459,13 @@ public void testIsolateMasterAndVerifyClusterStateConsensus() throws Exception {
                 assertEquals("unequal node count", state.nodes().getSize(), nodeState.nodes().getSize());
                 assertEquals("different masters ", state.nodes().getMasterNodeId(), nodeState.nodes().getMasterNodeId());
                 assertEquals("different meta data version", state.metaData().version(), nodeState.metaData().version());
-                if (!state.routingTable().prettyPrint().equals(nodeState.routingTable().prettyPrint())) {
+                if (!state.routingTable().toString().equals(nodeState.routingTable().toString())) {
                     fail("different routing");
                 }
             } catch (AssertionError t) {
                 fail("failed comparing cluster state: " + t.getMessage() + "\n" +
-                        "--- cluster state of node [" + nodes.get(0) + "]: ---\n" + state.prettyPrint() +
-                        "\n--- cluster state [" + node + "]: ---\n" + nodeState.prettyPrint());
+                    "--- cluster state of node [" + nodes.get(0) + "]: ---\n" + state +
+                    "\n--- cluster state [" + node + "]: ---\n" + nodeState);
             }
 
         }
@@ -454,8 +477,8 @@ public void testIsolateMasterAndVerifyClusterStateConsensus() throws Exception {
      * <p>
      * This test is a superset of tests run in the Jepsen test suite, with the exception of versioned updates
      */
-    @TestLogging("_root:DEBUG,action.index:TRACE,action.get:TRACE,discovery:TRACE,cluster.service:TRACE,"
-            + "indices.recovery:TRACE,indices.cluster:TRACE")
+    @TestLogging("_root:DEBUG,org.elasticsearch.action.index:TRACE,org.elasticsearch.action.get:TRACE,discovery:TRACE,org.elasticsearch.cluster.service:TRACE,"
+        + "org.elasticsearch.indices.recovery:TRACE,org.elasticsearch.indices.cluster:TRACE")
     public void testAckedIndexing() throws Exception {
 
         final int seconds = !(TEST_NIGHTLY && rarely()) ? 1 : 5;
@@ -464,10 +487,10 @@ public void testAckedIndexing() throws Exception {
         final List<String> nodes = startCluster(rarely() ? 5 : 3);
 
         assertAcked(prepareCreate("test")
-                .setSettings(Settings.builder()
-                        .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1 + randomInt(2))
-                        .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, randomInt(2))
-                ));
+            .setSettings(Settings.builder()
+                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1 + randomInt(2))
+                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, randomInt(2))
+            ));
         ensureGreen();
 
         ServiceDisruptionScheme disruptionScheme = addRandomDisruptionScheme();
@@ -503,7 +526,7 @@ public void testAckedIndexing() throws Exception {
                                 int shard = Math.floorMod(Murmur3HashFunction.hash(id), numPrimaries);
                                 logger.trace("[{}] indexing id [{}] through node [{}] targeting shard [{}]", name, id, node, shard);
                                 IndexResponse response =
-                                        client.prepareIndex("test", "type", id).setSource("{}").setTimeout(timeout).get(timeout);
+                                    client.prepareIndex("test", "type", id).setSource("{}", XContentType.JSON).setTimeout(timeout).get(timeout);
                                 assertEquals(DocWriteResponse.Result.CREATED, response.getResult());
                                 ackedDocs.put(id, node);
                                 logger.trace("[{}] indexed id [{}] through node [{}]", name, id, node);
@@ -520,7 +543,7 @@ public void testAckedIndexing() throws Exception {
                         } catch (InterruptedException e) {
                             // fine - semaphore interrupt
                         } catch (AssertionError | Exception e) {
-                            logger.info("unexpected exception in background thread of [{}]", e, node);
+                            logger.info((Supplier<?>) () -> new ParameterizedMessage("unexpected exception in background thread of [{}]", node), e);
                         }
                     }
                 });
@@ -557,17 +580,33 @@ public void testAckedIndexing() throws Exception {
                 disruptionScheme.stopDisrupting();
                 for (String node : internalCluster().getNodeNames()) {
                     ensureStableCluster(nodes.size(), TimeValue.timeValueMillis(disruptionScheme.expectedTimeToHeal().millis() +
-                            DISRUPTION_HEALING_OVERHEAD.millis()), true, node);
+                        DISRUPTION_HEALING_OVERHEAD.millis()), true, node);
+                }
+                // in case of a bridge partition, shard allocation can fail "index.allocation.max_retries" times if the master
+                // is the super-connected node and recovery source and target are on opposite sides of the bridge
+                if (disruptionScheme instanceof NetworkDisruption &&
+                    ((NetworkDisruption) disruptionScheme).getDisruptedLinks() instanceof Bridge) {
+                    assertAcked(client().admin().cluster().prepareReroute().setRetryFailed(true));
                 }
                 ensureGreen("test");
 
+                // make sure all nodes have the updated cluster state with the latest routing table
+                final long clusterStateVersionOnMaster = internalCluster().clusterService(internalCluster().getMasterName())
+                                                             .state().getVersion();
+                assertBusy(() -> {
+                    for (String node : nodes) {
+                        assertThat(internalCluster().clusterService(node).state().getVersion(),
+                            greaterThanOrEqualTo(clusterStateVersionOnMaster));
+                    }
+                });
+
                 logger.info("validating successful docs");
                 for (String node : nodes) {
                     try {
                         logger.debug("validating through node [{}] ([{}] acked docs)", node, ackedDocs.size());
                         for (String id : ackedDocs.keySet()) {
                             assertTrue("doc [" + id + "] indexed via node [" + ackedDocs.get(id) + "] not found",
-                                    client(node).prepareGet("test", "type", id).setPreference("_local").get().isExists());
+                                client(node).prepareGet("test", "type", id).setPreference("_local").get().isExists());
                         }
                     } catch (AssertionError e) {
                         throw new AssertionError(e.getMessage() + " (checked via node [" + node + "]", e);
@@ -601,7 +640,7 @@ public void testMasterNodeGCs() throws Exception {
 
         String oldMasterNode = internalCluster().getMasterName();
         // a very long GC, but it's OK as we remove the disruption when it has had an effect
-        SingleNodeDisruption masterNodeDisruption = new IntermittentLongGCDisruption(oldMasterNode, random(), 100, 200, 30000, 60000);
+        SingleNodeDisruption masterNodeDisruption = new IntermittentLongGCDisruption(random(), oldMasterNode, 100, 200, 30000, 60000);
         internalCluster().setDisruptionScheme(masterNodeDisruption);
         masterNodeDisruption.startDisrupting();
 
@@ -636,7 +675,7 @@ public void testMasterNodeGCs() throws Exception {
      * that already are following another elected master node. These nodes should reject this cluster state and prevent
      * them from following the stale master.
      */
-    @TestLogging("_root:DEBUG,cluster.service:TRACE,test.disruption:TRACE")
+    @TestLogging("_root:DEBUG,org.elasticsearch.cluster.service:TRACE,org.elasticsearch.test.disruption:TRACE")
     public void testStaleMasterNotHijackingMajority() throws Exception {
         // 3 node cluster with unicast discovery and minimum_master_nodes set to 2:
         final List<String> nodes = startCluster(3, 2);
@@ -657,32 +696,26 @@ public void testStaleMasterNotHijackingMajority() throws Exception {
 
         // Keeps track of the previous and current master when a master node transition took place on each node on the majority side:
         final Map<String, List<Tuple<String, String>>> masters = Collections.synchronizedMap(new HashMap<String, List<Tuple<String,
-                String>>>());
+            String>>>());
         for (final String node : majoritySide) {
             masters.put(node, new ArrayList<Tuple<String, String>>());
-            internalCluster().getInstance(ClusterService.class, node).add(new ClusterStateListener() {
-                @Override
-                public void clusterChanged(ClusterChangedEvent event) {
-                    DiscoveryNode previousMaster = event.previousState().nodes().getMasterNode();
-                    DiscoveryNode currentMaster = event.state().nodes().getMasterNode();
-                    if (!Objects.equals(previousMaster, currentMaster)) {
-                        logger.info("node {} received new cluster state: {} \n and had previous cluster state: {}", node, event.state(),
-                                event.previousState());
-                        String previousMasterNodeName = previousMaster != null ? previousMaster.getName() : null;
-                        String currentMasterNodeName = currentMaster != null ? currentMaster.getName() : null;
-                        masters.get(node).add(new Tuple<>(previousMasterNodeName, currentMasterNodeName));
-                    }
+            internalCluster().getInstance(ClusterService.class, node).addListener(event -> {
+                DiscoveryNode previousMaster = event.previousState().nodes().getMasterNode();
+                DiscoveryNode currentMaster = event.state().nodes().getMasterNode();
+                if (!Objects.equals(previousMaster, currentMaster)) {
+                    logger.info("node {} received new cluster state: {} \n and had previous cluster state: {}", node, event.state(),
+                        event.previousState());
+                    String previousMasterNodeName = previousMaster != null ? previousMaster.getName() : null;
+                    String currentMasterNodeName = currentMaster != null ? currentMaster.getName() : null;
+                    masters.get(node).add(new Tuple<>(previousMasterNodeName, currentMasterNodeName));
                 }
             });
         }
 
         final CountDownLatch oldMasterNodeSteppedDown = new CountDownLatch(1);
-        internalCluster().getInstance(ClusterService.class, oldMasterNode).add(new ClusterStateListener() {
-            @Override
-            public void clusterChanged(ClusterChangedEvent event) {
-                if (event.state().nodes().getMasterNodeId() == null) {
-                    oldMasterNodeSteppedDown.countDown();
-                }
+        internalCluster().getInstance(ClusterService.class, oldMasterNode).addListener(event -> {
+            if (event.state().nodes().getMasterNodeId() == null) {
+                oldMasterNodeSteppedDown.countDown();
             }
         });
 
@@ -712,17 +745,17 @@ public void clusterChanged(ClusterChangedEvent event) {
         // The old master node will send this update + the cluster state where he is flagged as master to the other
         // nodes that follow the new master. These nodes should ignore this update.
         internalCluster().getInstance(ClusterService.class, oldMasterNode).submitStateUpdateTask("sneaky-update", new
-                ClusterStateUpdateTask(Priority.IMMEDIATE) {
-            @Override
-            public ClusterState execute(ClusterState currentState) throws Exception {
-                return ClusterState.builder(currentState).build();
-            }
+            ClusterStateUpdateTask(Priority.IMMEDIATE) {
+                @Override
+                public ClusterState execute(ClusterState currentState) throws Exception {
+                    return ClusterState.builder(currentState).build();
+                }
 
-            @Override
-            public void onFailure(String source, Exception e) {
-                logger.warn((Supplier<?>) () -> new ParameterizedMessage("failure [{}]", source), e);
-            }
-        });
+                @Override
+                public void onFailure(String source, Exception e) {
+                    logger.warn((Supplier<?>) () -> new ParameterizedMessage("failure [{}]", source), e);
+                }
+            });
 
         // Save the new elected master node
         final String newMasterNode = internalCluster().getMasterName(majoritySide.get(0));
@@ -735,30 +768,22 @@ public void onFailure(String source, Exception e) {
         oldMasterNodeSteppedDown.await(30, TimeUnit.SECONDS);
         // Make sure that the end state is consistent on all nodes:
         assertDiscoveryCompleted(nodes);
-        // Use assertBusy(...) because the unfrozen node may take a while to actually join the cluster.
-        // The assertDiscoveryCompleted(...) can't know if all nodes have the old master node in all of the local cluster states
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                assertMaster(newMasterNode, nodes);
-            }
-        });
-
+        assertMaster(newMasterNode, nodes);
 
         assertThat(masters.size(), equalTo(2));
         for (Map.Entry<String, List<Tuple<String, String>>> entry : masters.entrySet()) {
             String nodeName = entry.getKey();
             List<Tuple<String, String>> recordedMasterTransition = entry.getValue();
             assertThat("[" + nodeName + "] Each node should only record two master node transitions", recordedMasterTransition.size(),
-                    equalTo(2));
+                equalTo(2));
             assertThat("[" + nodeName + "] First transition's previous master should be [null]", recordedMasterTransition.get(0).v1(),
-                    equalTo(oldMasterNode));
+                equalTo(oldMasterNode));
             assertThat("[" + nodeName + "] First transition's current master should be [" + newMasterNode + "]", recordedMasterTransition
-                    .get(0).v2(), nullValue());
+                .get(0).v2(), nullValue());
             assertThat("[" + nodeName + "] Second transition's previous master should be [null]", recordedMasterTransition.get(1).v1(),
-                    nullValue());
+                nullValue());
             assertThat("[" + nodeName + "] Second transition's current master should be [" + newMasterNode + "]",
-                    recordedMasterTransition.get(1).v2(), equalTo(newMasterNode));
+                recordedMasterTransition.get(1).v2(), equalTo(newMasterNode));
         }
     }
 
@@ -770,11 +795,11 @@ public void testRejoinDocumentExistsInAllShardCopies() throws Exception {
         List<String> nodes = startCluster(3);
 
         assertAcked(prepareCreate("test")
-                .setSettings(Settings.builder()
-                        .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
-                        .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 2)
-                )
-                .get());
+            .setSettings(Settings.builder()
+                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 2)
+            )
+            .get());
         ensureGreen("test");
 
         nodes = new ArrayList<>(nodes);
@@ -790,13 +815,13 @@ public void testRejoinDocumentExistsInAllShardCopies() throws Exception {
 
 
         IndexResponse indexResponse = internalCluster().client(notIsolatedNode).prepareIndex("test", "type").setSource("field", "value")
-                .get();
+            .get();
         assertThat(indexResponse.getVersion(), equalTo(1L));
 
         logger.info("Verifying if document exists via node[{}]", notIsolatedNode);
         GetResponse getResponse = internalCluster().client(notIsolatedNode).prepareGet("test", "type", indexResponse.getId())
-                .setPreference("_local")
-                .get();
+            .setPreference("_local")
+            .get();
         assertThat(getResponse.isExists(), is(true));
         assertThat(getResponse.getVersion(), equalTo(1L));
         assertThat(getResponse.getId(), equalTo(indexResponse.getId()));
@@ -809,8 +834,8 @@ public void testRejoinDocumentExistsInAllShardCopies() throws Exception {
         for (String node : nodes) {
             logger.info("Verifying if document exists after isolating node[{}] via node[{}]", isolatedNode, node);
             getResponse = internalCluster().client(node).prepareGet("test", "type", indexResponse.getId())
-                    .setPreference("_local")
-                    .get();
+                .setPreference("_local")
+                .get();
             assertThat(getResponse.isExists(), is(true));
             assertThat(getResponse.getVersion(), equalTo(1L));
             assertThat(getResponse.getId(), equalTo(indexResponse.getId()));
@@ -834,10 +859,9 @@ public void testUnicastSinglePingResponseContainsMaster() throws Exception {
 
         // Forcefully clean temporal response lists on all nodes. Otherwise the node in the unicast host list
         // includes all the other nodes that have pinged it and the issue doesn't manifest
-        for (ZenPingService pingService : internalCluster().getInstances(ZenPingService.class)) {
-            for (ZenPing zenPing : pingService.zenPings()) {
-                ((UnicastZenPing) zenPing).clearTemporalResponses();
-            }
+        ZenPing zenPing = ((TestZenDiscovery) internalCluster().getInstance(Discovery.class)).getZenPing();
+        if (zenPing instanceof UnicastZenPing) {
+            ((UnicastZenPing) zenPing).clearTemporalResponses();
         }
 
         // Simulate a network issue between the unlucky node and elected master node in both directions.
@@ -872,10 +896,9 @@ public void testIsolatedUnicastNodes() throws Exception {
 
         // Forcefully clean temporal response lists on all nodes. Otherwise the node in the unicast host list
         // includes all the other nodes that have pinged it and the issue doesn't manifest
-        for (ZenPingService pingService : internalCluster().getInstances(ZenPingService.class)) {
-            for (ZenPing zenPing : pingService.zenPings()) {
-                ((UnicastZenPing) zenPing).clearTemporalResponses();
-            }
+        ZenPing zenPing = ((TestZenDiscovery) internalCluster().getInstance(Discovery.class)).getZenPing();
+        if (zenPing instanceof UnicastZenPing) {
+            ((UnicastZenPing) zenPing).clearTemporalResponses();
         }
 
         // Simulate a network issue between the unicast target node and the rest of the cluster
@@ -911,11 +934,11 @@ public void testClusterJoinDespiteOfPublishingIssues() throws Exception {
         DiscoveryNodes discoveryNodes = internalCluster().getInstance(ClusterService.class, nonMasterNode).state().nodes();
 
         TransportService masterTranspotService =
-                internalCluster().getInstance(TransportService.class, discoveryNodes.getMasterNode().getName());
+            internalCluster().getInstance(TransportService.class, discoveryNodes.getMasterNode().getName());
 
         logger.info("blocking requests from non master [{}] to master [{}]", nonMasterNode, masterNode);
         MockTransportService nonMasterTransportService = (MockTransportService) internalCluster().getInstance(TransportService.class,
-                nonMasterNode);
+            nonMasterNode);
         nonMasterTransportService.addFailToSendNoConnectRule(masterTranspotService);
 
         assertNoMaster(nonMasterNode);
@@ -934,15 +957,21 @@ public void testClusterJoinDespiteOfPublishingIssues() throws Exception {
         logger.info("allowing requests from non master [{}] to master [{}], waiting for two join request", nonMasterNode, masterNode);
         final CountDownLatch countDownLatch = new CountDownLatch(2);
         nonMasterTransportService.addDelegate(masterTranspotService, new MockTransportService.DelegateTransport(nonMasterTransportService
-                .original()) {
+            .original()) {
             @Override
-            public void sendRequest(DiscoveryNode node, long requestId, String action, TransportRequest request, TransportRequestOptions
-                    options) throws IOException, TransportException {
+            protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request,
+                                       TransportRequestOptions options) throws IOException {
                 if (action.equals(MembershipAction.DISCOVERY_JOIN_ACTION_NAME)) {
                     countDownLatch.countDown();
                 }
-                super.sendRequest(node, requestId, action, request, options);
+                super.sendRequest(connection, requestId, action, request, options);
+            }
+
+            @Override
+            public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) throws IOException {
+                return super.openConnection(node, profile);
             }
+
         });
 
         countDownLatch.await();
@@ -965,40 +994,44 @@ public void testSendingShardFailure() throws Exception {
         List<String> nonMasterNodes = nodes.stream().filter(node -> !node.equals(masterNode)).collect(Collectors.toList());
         String nonMasterNode = randomFrom(nonMasterNodes);
         assertAcked(prepareCreate("test")
-                .setSettings(Settings.builder()
-                        .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 3)
-                        .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 2)
-                ));
+            .setSettings(Settings.builder()
+                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 3)
+                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 2)
+            ));
         ensureGreen();
         String nonMasterNodeId = internalCluster().clusterService(nonMasterNode).localNode().getId();
 
         // fail a random shard
         ShardRouting failedShard =
-                randomFrom(clusterService().state().getRoutingNodes().node(nonMasterNodeId).shardsWithState(ShardRoutingState.STARTED));
+            randomFrom(clusterService().state().getRoutingNodes().node(nonMasterNodeId).shardsWithState(ShardRoutingState.STARTED));
         ShardStateAction service = internalCluster().getInstance(ShardStateAction.class, nonMasterNode);
         CountDownLatch latch = new CountDownLatch(1);
         AtomicBoolean success = new AtomicBoolean();
 
         String isolatedNode = randomBoolean() ? masterNode : nonMasterNode;
         TwoPartitions partitions = isolateNode(isolatedNode);
-        NetworkDisruption networkDisruption = addRandomDisruptionType(partitions);
+        // we cannot use the NetworkUnresponsive disruption type here as it will swallow the "shard failed" request, calling neither
+        // onSuccess nor onFailure on the provided listener.
+        NetworkLinkDisruptionType disruptionType = new NetworkDisconnect();
+        NetworkDisruption networkDisruption = new NetworkDisruption(partitions, disruptionType);
+        setDisruptionScheme(networkDisruption);
         networkDisruption.startDisrupting();
 
         service.localShardFailed(failedShard, "simulated", new CorruptIndexException("simulated", (String) null), new
-                ShardStateAction.Listener() {
-            @Override
-            public void onSuccess() {
-                success.set(true);
-                latch.countDown();
-            }
+            ShardStateAction.Listener() {
+                @Override
+                public void onSuccess() {
+                    success.set(true);
+                    latch.countDown();
+                }
 
-            @Override
-            public void onFailure(Exception e) {
-                success.set(false);
-                latch.countDown();
-                assert false;
-            }
-        });
+                @Override
+                public void onFailure(Exception e) {
+                    success.set(false);
+                    latch.countDown();
+                    assert false;
+                }
+            });
 
         if (isolatedNode.equals(nonMasterNode)) {
             assertNoMaster(nonMasterNode);
@@ -1025,16 +1058,12 @@ public void onFailure(Exception e) {
     }
 
     public void testClusterFormingWithASlowNode() throws Exception {
-        configureUnicastCluster(3, null, 2);
+        configureCluster(3, null, 2);
 
         SlowClusterStateProcessing disruption = new SlowClusterStateProcessing(random(), 0, 0, 1000, 2000);
 
         // don't wait for initial state, wat want to add the disruption while the cluster is forming..
-        internalCluster().startNodesAsync(3,
-                Settings.builder()
-                        .put(DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING.getKey(), "1ms")
-                        .put(DiscoverySettings.PUBLISH_TIMEOUT_SETTING.getKey(), "3s")
-                        .build()).get();
+        internalCluster().startNodes(3, Settings.builder().put(DiscoverySettings.PUBLISH_TIMEOUT_SETTING.getKey(), "3s").build());
 
         logger.info("applying disruption while cluster is forming ...");
 
@@ -1063,7 +1092,7 @@ public void testNodeNotReachableFromMaster() throws Exception {
 
         logger.info("blocking request from master [{}] to [{}]", masterNode, nonMasterNode);
         MockTransportService masterTransportService = (MockTransportService) internalCluster().getInstance(TransportService.class,
-                masterNode);
+            masterNode);
         if (randomBoolean()) {
             masterTransportService.addUnresponsiveRule(internalCluster().getInstance(TransportService.class, nonMasterNode));
         } else {
@@ -1088,48 +1117,34 @@ public void testNodeNotReachableFromMaster() throws Exception {
      */
     public void testSearchWithRelocationAndSlowClusterStateProcessing() throws Exception {
         // don't use DEFAULT settings (which can cause node disconnects on a slow CI machine)
-        configureUnicastCluster(Settings.EMPTY, 3, null, 1);
-        InternalTestCluster.Async<String> masterNodeFuture = internalCluster().startMasterOnlyNodeAsync();
-        InternalTestCluster.Async<String> node_1Future = internalCluster().startDataOnlyNodeAsync();
+        configureCluster(Settings.EMPTY, 3, null, 1);
+        final String masterNode = internalCluster().startMasterOnlyNode();
+        final String node_1 = internalCluster().startDataOnlyNode();
 
-        final String node_1 = node_1Future.get();
-        final String masterNode = masterNodeFuture.get();
         logger.info("--> creating index [test] with one shard and on replica");
         assertAcked(prepareCreate("test").setSettings(
-                Settings.builder().put(indexSettings())
-                        .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
-                        .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0))
+            Settings.builder().put(indexSettings())
+                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0))
         );
         ensureGreen("test");
 
-        InternalTestCluster.Async<String> node_2Future = internalCluster().startDataOnlyNodeAsync();
-        final String node_2 = node_2Future.get();
+        final String node_2 = internalCluster().startDataOnlyNode();
         List<IndexRequestBuilder> indexRequestBuilderList = new ArrayList<>();
         for (int i = 0; i < 100; i++) {
-            indexRequestBuilderList.add(client().prepareIndex().setIndex("test").setType("doc").setSource("{\"int_field\":1}"));
+            indexRequestBuilderList.add(client().prepareIndex().setIndex("test").setType("doc")
+                .setSource("{\"int_field\":1}", XContentType.JSON));
         }
         indexRandom(true, indexRequestBuilderList);
-        SingleNodeDisruption disruption = new BlockClusterStateProcessing(node_2, random());
 
-        internalCluster().setDisruptionScheme(disruption);
-        MockTransportService transportServiceNode2 = (MockTransportService) internalCluster().getInstance(TransportService.class, node_2);
-        CountDownLatch beginRelocationLatch = new CountDownLatch(1);
-        CountDownLatch endRelocationLatch = new CountDownLatch(1);
-        transportServiceNode2.addTracer(new IndicesStoreIntegrationIT.ReclocationStartEndTracer(logger, beginRelocationLatch,
-                endRelocationLatch));
-        internalCluster().client().admin().cluster().prepareReroute().add(new MoveAllocationCommand("test", 0, node_1, node_2)).get();
-        // wait for relocation to start
-        beginRelocationLatch.await();
-        disruption.startDisrupting();
-        // wait for relocation to finish
-        endRelocationLatch.await();
+        IndicesStoreIntegrationIT.relocateAndBlockCompletion(logger, "test", 0, node_1, node_2);
         // now search for the documents and see if we get a reply
         assertThat(client().prepareSearch().setSize(0).get().getHits().totalHits(), equalTo(100L));
     }
 
     public void testIndexImportedFromDataOnlyNodesIfMasterLostDataFolder() throws Exception {
         // test for https://github.com/elastic/elasticsearch/issues/8823
-        configureUnicastCluster(2, null, 1);
+        configureCluster(2, null, 1);
         String masterNode = internalCluster().startMasterOnlyNode(Settings.EMPTY);
         internalCluster().startDataOnlyNode(Settings.EMPTY);
 
@@ -1155,21 +1170,19 @@ public boolean clearData(String nodeName) {
      */
     public void testIndicesDeleted() throws Exception {
         final Settings settings = Settings.builder()
-                                      .put(DEFAULT_SETTINGS)
-                                      .put(DiscoverySettings.PUBLISH_TIMEOUT_SETTING.getKey(), "0s") // don't wait on isolated data node
-                                      .put(DiscoverySettings.COMMIT_TIMEOUT_SETTING.getKey(), "30s") // wait till cluster state is committed
-                                      .build();
+            .put(DEFAULT_SETTINGS)
+            .put(DiscoverySettings.PUBLISH_TIMEOUT_SETTING.getKey(), "0s") // don't wait on isolated data node
+            .put(DiscoverySettings.COMMIT_TIMEOUT_SETTING.getKey(), "30s") // wait till cluster state is committed
+            .build();
         final String idxName = "test";
-        configureUnicastCluster(settings, 3, null, 2);
-        InternalTestCluster.Async<List<String>> masterNodes = internalCluster().startMasterOnlyNodesAsync(2);
-        InternalTestCluster.Async<String> dataNode = internalCluster().startDataOnlyNodeAsync();
-        dataNode.get();
-        final List<String> allMasterEligibleNodes = masterNodes.get();
+        configureCluster(settings, 3, null, 2);
+        final List<String> allMasterEligibleNodes = internalCluster().startMasterOnlyNodes(2);
+        final String dataNode = internalCluster().startDataOnlyNode();
         ensureStableCluster(3);
         assertAcked(prepareCreate("test"));
 
         final String masterNode1 = internalCluster().getMasterName();
-        NetworkDisruption networkDisruption = new NetworkDisruption(new TwoPartitions(masterNode1, dataNode.get()),
+        NetworkDisruption networkDisruption = new NetworkDisruption(new TwoPartitions(masterNode1, dataNode),
             new NetworkUnresponsive());
         internalCluster().setDisruptionScheme(networkDisruption);
         networkDisruption.startDisrupting();
@@ -1180,8 +1193,7 @@ public void testIndicesDeleted() throws Exception {
         assertBusy(() -> {
             for (String masterNode : allMasterEligibleNodes) {
                 final ClusterState masterState = internalCluster().clusterService(masterNode).state();
-                assertTrue("index not deleted on " + masterNode, masterState.metaData().hasIndex(idxName) == false &&
-                                                                 masterState.status() == ClusterState.ClusterStateStatus.APPLIED);
+                assertTrue("index not deleted on " + masterNode, masterState.metaData().hasIndex(idxName) == false);
             }
         });
         internalCluster().restartNode(masterNode1, InternalTestCluster.EMPTY_CALLBACK);
@@ -1189,6 +1201,61 @@ public void testIndicesDeleted() throws Exception {
         assertFalse(client().admin().indices().prepareExists(idxName).get().isExists());
     }
 
+    public void testElectMasterWithLatestVersion() throws Exception {
+        configureCluster(3, null, 2);
+        final Set<String> nodes = new HashSet<>(internalCluster().startNodes(3));
+        ensureStableCluster(3);
+        ServiceDisruptionScheme isolateAllNodes = new NetworkDisruption(new NetworkDisruption.IsolateAllNodes(nodes), new NetworkDisconnect());
+        internalCluster().setDisruptionScheme(isolateAllNodes);
+
+        logger.info("--> forcing a complete election to make sure \"preferred\" master is elected");
+        isolateAllNodes.startDisrupting();
+        for (String node : nodes) {
+            assertNoMaster(node);
+        }
+        internalCluster().clearDisruptionScheme();
+        ensureStableCluster(3);
+        final String preferredMasterName = internalCluster().getMasterName();
+        final DiscoveryNode preferredMaster = internalCluster().clusterService(preferredMasterName).localNode();
+        for (String node : nodes) {
+            DiscoveryNode discoveryNode = internalCluster().clusterService(node).localNode();
+            assertThat(discoveryNode.getId(), greaterThanOrEqualTo(preferredMaster.getId()));
+        }
+
+        logger.info("--> preferred master is {}", preferredMaster);
+        final Set<String> nonPreferredNodes = new HashSet<>(nodes);
+        nonPreferredNodes.remove(preferredMasterName);
+        final ServiceDisruptionScheme isolatePreferredMaster =
+            new NetworkDisruption(
+                new NetworkDisruption.TwoPartitions(
+                    Collections.singleton(preferredMasterName), nonPreferredNodes),
+                new NetworkDisconnect());
+        internalCluster().setDisruptionScheme(isolatePreferredMaster);
+        isolatePreferredMaster.startDisrupting();
+
+        assertAcked(client(randomFrom(nonPreferredNodes)).admin().indices().prepareCreate("test").setSettings(
+            INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1,
+            INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0
+        ));
+
+        internalCluster().clearDisruptionScheme(false);
+        internalCluster().setDisruptionScheme(isolateAllNodes);
+
+        logger.info("--> forcing a complete election again");
+        isolateAllNodes.startDisrupting();
+        for (String node : nodes) {
+            assertNoMaster(node);
+        }
+
+        isolateAllNodes.stopDisrupting();
+
+        final ClusterState state = client().admin().cluster().prepareState().get().getState();
+        if (state.metaData().hasIndex("test") == false) {
+            fail("index 'test' was lost. current cluster state: " + state);
+        }
+
+    }
+
     protected NetworkDisruption addRandomDisruptionType(TwoPartitions partitions) {
         final NetworkLinkDisruptionType disruptionType;
         if (randomBoolean()) {
@@ -1222,10 +1289,17 @@ private ServiceDisruptionScheme addRandomDisruptionScheme() {
         }
         final NetworkLinkDisruptionType disruptionType;
         switch (randomInt(2)) {
-            case 0: disruptionType = new NetworkUnresponsive(); break;
-            case 1: disruptionType = new NetworkDisconnect(); break;
-            case 2: disruptionType = NetworkDelay.random(random()); break;
-            default: throw new IllegalArgumentException();
+            case 0:
+                disruptionType = new NetworkUnresponsive();
+                break;
+            case 1:
+                disruptionType = new NetworkDisconnect();
+                break;
+            case 2:
+                disruptionType = NetworkDelay.random(random());
+                break;
+            default:
+                throw new IllegalArgumentException();
         }
         final ServiceDisruptionScheme scheme;
         if (rarely()) {
@@ -1250,56 +1324,52 @@ private void assertNoMaster(final String node, TimeValue maxWaitTime) throws Exc
     }
 
     private void assertNoMaster(final String node, @Nullable final ClusterBlock expectedBlocks, TimeValue maxWaitTime) throws Exception {
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                ClusterState state = getNodeClusterState(node);
-                assertNull("node [" + node + "] still has [" + state.nodes().getMasterNode() + "] as master", state.nodes().getMasterNode());
-                if (expectedBlocks != null) {
-                    for (ClusterBlockLevel level : expectedBlocks.levels()) {
-                        assertTrue("node [" + node + "] does have level [" + level + "] in it's blocks", state.getBlocks().hasGlobalBlock
-                                (level));
-                    }
+        assertBusy(() -> {
+            ClusterState state = getNodeClusterState(node);
+            assertNull("node [" + node + "] still has [" + state.nodes().getMasterNode() + "] as master", state.nodes().getMasterNode());
+            if (expectedBlocks != null) {
+                for (ClusterBlockLevel level : expectedBlocks.levels()) {
+                    assertTrue("node [" + node + "] does have level [" + level + "] in it's blocks", state.getBlocks().hasGlobalBlock
+                        (level));
                 }
             }
         }, maxWaitTime.getMillis(), TimeUnit.MILLISECONDS);
     }
 
     private void assertDifferentMaster(final String node, final String oldMasterNode) throws Exception {
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                ClusterState state = getNodeClusterState(node);
-                String masterNode = null;
-                if (state.nodes().getMasterNode() != null) {
-                    masterNode = state.nodes().getMasterNode().getName();
-                }
-                logger.trace("[{}] master is [{}]", node, state.nodes().getMasterNode());
-                assertThat("node [" + node + "] still has [" + masterNode + "] as master",
-                        oldMasterNode, not(equalTo(masterNode)));
+        assertBusy(() -> {
+            ClusterState state = getNodeClusterState(node);
+            String masterNode = null;
+            if (state.nodes().getMasterNode() != null) {
+                masterNode = state.nodes().getMasterNode().getName();
             }
+            logger.trace("[{}] master is [{}]", node, state.nodes().getMasterNode());
+            assertThat("node [" + node + "] still has [" + masterNode + "] as master",
+                oldMasterNode, not(equalTo(masterNode)));
         }, 10, TimeUnit.SECONDS);
     }
 
-    private void assertMaster(String masterNode, List<String> nodes) {
-        for (String node : nodes) {
-            ClusterState state = getNodeClusterState(node);
-            String failMsgSuffix = "cluster_state:\n" + state.prettyPrint();
-            assertThat("wrong node count on [" + node + "]. " + failMsgSuffix, state.nodes().getSize(), equalTo(nodes.size()));
-            String otherMasterNodeName = state.nodes().getMasterNode() != null ? state.nodes().getMasterNode().getName() : null;
-            assertThat("wrong master on node [" + node + "]. " + failMsgSuffix, otherMasterNodeName, equalTo(masterNode));
-        }
+    private void assertMaster(String masterNode, List<String> nodes) throws Exception {
+        assertBusy(() -> {
+            for (String node : nodes) {
+                ClusterState state = getNodeClusterState(node);
+                String failMsgSuffix = "cluster_state:\n" + state;
+                assertThat("wrong node count on [" + node + "]. " + failMsgSuffix, state.nodes().getSize(), equalTo(nodes.size()));
+                String otherMasterNodeName = state.nodes().getMasterNode() != null ? state.nodes().getMasterNode().getName() : null;
+                assertThat("wrong master on node [" + node + "]. " + failMsgSuffix, otherMasterNodeName, equalTo(masterNode));
+            }
+        });
     }
 
     private void assertDiscoveryCompleted(List<String> nodes) throws InterruptedException {
         for (final String node : nodes) {
             assertTrue(
-                    "node [" + node + "] is still joining master",
-                    awaitBusy(
-                            () -> !((ZenDiscovery) internalCluster().getInstance(Discovery.class, node)).joiningCluster(),
-                            30,
-                            TimeUnit.SECONDS
-                    )
+                "node [" + node + "] is still joining master",
+                awaitBusy(
+                    () -> !((ZenDiscovery) internalCluster().getInstance(Discovery.class, node)).joiningCluster(),
+                    30,
+                    TimeUnit.SECONDS
+                )
             );
         }
     }
diff --git a/core/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java b/core/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java
index b1b7749d88c26..11bd8e8ea6b4a 100644
--- a/core/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java
@@ -32,11 +32,12 @@
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.discovery.zen.fd.FaultDetection;
-import org.elasticsearch.discovery.zen.fd.MasterFaultDetection;
-import org.elasticsearch.discovery.zen.fd.NodesFaultDetection;
+import org.elasticsearch.discovery.zen.FaultDetection;
+import org.elasticsearch.discovery.zen.MasterFaultDetection;
+import org.elasticsearch.discovery.zen.NodesFaultDetection;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService;
+import org.elasticsearch.node.Node;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.threadpool.TestThreadPool;
@@ -56,8 +57,6 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
-import static java.util.Collections.emptyMap;
-import static java.util.Collections.emptySet;
 import static java.util.Collections.singleton;
 import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
 import static org.elasticsearch.test.ClusterServiceUtils.setState;
@@ -73,10 +72,12 @@ public class ZenFaultDetectionTests extends ESTestCase {
     protected static final Version version0 = Version.fromId(/*0*/99);
     protected DiscoveryNode nodeA;
     protected MockTransportService serviceA;
+    private Settings settingsA;
 
     protected static final Version version1 = Version.fromId(199);
     protected DiscoveryNode nodeB;
     protected MockTransportService serviceB;
+    private Settings settingsB;
 
     @Override
     @Before
@@ -87,17 +88,19 @@ public void setUp() throws Exception {
             .build();
         ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
         threadPool = new TestThreadPool(getClass().getName());
-        clusterServiceA = createClusterService(threadPool);
-        clusterServiceB = createClusterService(threadPool);
         circuitBreakerService = new HierarchyCircuitBreakerService(settings, clusterSettings);
-        serviceA = build(Settings.builder().put("name", "TS_A").build(), version0);
-        nodeA = new DiscoveryNode("TS_A", "TS_A", serviceA.boundAddress().publishAddress(), emptyMap(), emptySet(), version0);
-        serviceB = build(Settings.builder().put("name", "TS_B").build(), version1);
-        nodeB = new DiscoveryNode("TS_B", "TS_B", serviceB.boundAddress().publishAddress(), emptyMap(), emptySet(), version1);
+        settingsA = Settings.builder().put("node.name", "TS_A").put(settings).build();
+        serviceA = build(settingsA, version0);
+        nodeA = serviceA.getLocalDiscoNode();
+        settingsB = Settings.builder().put("node.name", "TS_B").put(settings).build();
+        serviceB = build(settingsB, version1);
+        nodeB = serviceB.getLocalDiscoNode();
+        clusterServiceA = createClusterService(settingsA, threadPool, nodeA);
+        clusterServiceB = createClusterService(settingsB, threadPool, nodeB);
 
         // wait till all nodes are properly connected and the event has been sent, so tests in this class
         // will not get this callback called on the connections done in this setup
-        final CountDownLatch latch = new CountDownLatch(4);
+        final CountDownLatch latch = new CountDownLatch(2);
         TransportConnectionListener waitForConnection = new TransportConnectionListener() {
             @Override
             public void onNodeConnected(DiscoveryNode node) {
@@ -136,18 +139,22 @@ public void tearDown() throws Exception {
     protected MockTransportService build(Settings settings, Version version) {
         NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(Collections.emptyList());
         MockTransportService transportService =
-                new MockTransportService(
-                        Settings.builder()
-                             // trace zenfd actions but keep the default otherwise
-                            .put(TransportService.TRACE_LOG_EXCLUDE_SETTING.getKey(), singleton(TransportLivenessAction.NAME))
-                            .build(),
-                        new LocalTransport(settings, threadPool, namedWriteableRegistry, circuitBreakerService) {
-                            @Override
-                            protected Version getVersion() {
-                                return version;
-                            }
-                        },
-                        threadPool);
+            new MockTransportService(
+                Settings.builder()
+                    // trace zenfd actions but keep the default otherwise
+                    .put(TransportService.TRACE_LOG_EXCLUDE_SETTING.getKey(), singleton(TransportLivenessAction.NAME))
+                    .build(),
+                new LocalTransport(settings, threadPool, namedWriteableRegistry, circuitBreakerService) {
+                    @Override
+                    protected Version getVersion() {
+                        return version;
+                    }
+                },
+                threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR,
+                (boundAddress) ->
+                    new DiscoveryNode(Node.NODE_NAME_SETTING.get(settings), boundAddress.publishAddress(),
+                        Node.NODE_ATTRIBUTES.get(settings).getAsMap(), DiscoveryNode.getRolesFromSettings(settings), version),
+                null);
         transportService.start();
         transportService.acceptIncomingRequests();
         return transportService;
@@ -172,15 +179,17 @@ private DiscoveryNodes buildNodesForB(boolean master) {
     }
 
     public void testNodesFaultDetectionConnectOnDisconnect() throws InterruptedException {
-        Settings.Builder settings = Settings.builder();
         boolean shouldRetry = randomBoolean();
         // make sure we don't ping again after the initial ping
-        settings.put(FaultDetection.CONNECT_ON_NETWORK_DISCONNECT_SETTING.getKey(), shouldRetry)
-                .put(FaultDetection.PING_INTERVAL_SETTING.getKey(), "5m");
+        final Settings pingSettings = Settings.builder()
+            .put(FaultDetection.CONNECT_ON_NETWORK_DISCONNECT_SETTING.getKey(), shouldRetry)
+            .put(FaultDetection.PING_INTERVAL_SETTING.getKey(), "5m").build();
         ClusterState clusterState = ClusterState.builder(new ClusterName("test")).nodes(buildNodesForA(true)).build();
-        NodesFaultDetection nodesFDA = new NodesFaultDetection(settings.build(), threadPool, serviceA, clusterState.getClusterName());
+        NodesFaultDetection nodesFDA = new NodesFaultDetection(Settings.builder().put(settingsA).put(pingSettings).build(),
+            threadPool, serviceA, clusterState.getClusterName());
         nodesFDA.setLocalNode(nodeA);
-        NodesFaultDetection nodesFDB = new NodesFaultDetection(settings.build(), threadPool, serviceB, clusterState.getClusterName());
+        NodesFaultDetection nodesFDB = new NodesFaultDetection(Settings.builder().put(settingsB).put(pingSettings).build(),
+            threadPool, serviceB, clusterState.getClusterName());
         nodesFDB.setLocalNode(nodeB);
         final CountDownLatch pingSent = new CountDownLatch(1);
         nodesFDB.addListener(new NodesFaultDetection.Listener() {
@@ -225,7 +234,7 @@ public void onNodeFailure(DiscoveryNode node, String reason) {
     public void testMasterFaultDetectionConnectOnDisconnect() throws InterruptedException {
         Settings.Builder settings = Settings.builder();
         boolean shouldRetry = randomBoolean();
-        ClusterName clusterName = new ClusterName(randomAsciiOfLengthBetween(3, 20));
+        ClusterName clusterName = new ClusterName(randomAlphaOfLengthBetween(3, 20));
 
         // make sure we don't ping
         settings.put(FaultDetection.CONNECT_ON_NETWORK_DISCONNECT_SETTING.getKey(), shouldRetry)
@@ -235,7 +244,7 @@ public void testMasterFaultDetectionConnectOnDisconnect() throws InterruptedExce
         setState(clusterServiceA, state);
         MasterFaultDetection masterFD = new MasterFaultDetection(settings.build(), threadPool, serviceA,
             clusterServiceA);
-        masterFD.start(nodeB, "test");
+        masterFD.restart(nodeB, "test");
 
         final String[] failureReason = new String[1];
         final DiscoveryNode[] failureNode = new DiscoveryNode[1];
@@ -262,13 +271,12 @@ public void testMasterFaultDetectionConnectOnDisconnect() throws InterruptedExce
     }
 
     public void testMasterFaultDetectionNotSizeLimited() throws InterruptedException {
-        Settings.Builder settings = Settings.builder();
         boolean shouldRetry = randomBoolean();
-        ClusterName clusterName = new ClusterName(randomAsciiOfLengthBetween(3, 20));
-        settings
+        ClusterName clusterName = new ClusterName(randomAlphaOfLengthBetween(3, 20));
+        final Settings settings = Settings.builder()
             .put(FaultDetection.CONNECT_ON_NETWORK_DISCONNECT_SETTING.getKey(), shouldRetry)
             .put(FaultDetection.PING_INTERVAL_SETTING.getKey(), "1s")
-        .put("cluster.name", clusterName.value());
+            .put("cluster.name", clusterName.value()).build();
         final ClusterState stateNodeA = ClusterState.builder(clusterName).nodes(buildNodesForA(false)).build();
         setState(clusterServiceA, stateNodeA);
 
@@ -280,16 +288,16 @@ public void testMasterFaultDetectionNotSizeLimited() throws InterruptedException
         serviceA.addTracer(pingProbeA);
         serviceB.addTracer(pingProbeB);
 
-        MasterFaultDetection masterFDNodeA = new MasterFaultDetection(settings.build(), threadPool, serviceA,
-            clusterServiceA);
-        masterFDNodeA.start(nodeB, "test");
+        MasterFaultDetection masterFDNodeA = new MasterFaultDetection(Settings.builder().put(settingsA).put(settings).build(),
+            threadPool, serviceA, clusterServiceA);
+        masterFDNodeA.restart(nodeB, "test");
 
         final ClusterState stateNodeB = ClusterState.builder(clusterName).nodes(buildNodesForB(true)).build();
         setState(clusterServiceB, stateNodeB);
 
-        MasterFaultDetection masterFDNodeB = new MasterFaultDetection(settings.build(), threadPool, serviceB,
-            clusterServiceB);
-        masterFDNodeB.start(nodeB, "test");
+        MasterFaultDetection masterFDNodeB = new MasterFaultDetection(Settings.builder().put(settingsB).put(settings).build(),
+            threadPool, serviceB, clusterServiceB);
+        masterFDNodeB.restart(nodeB, "test");
 
         // let's do a few pings
         pingProbeA.awaitMinCompletedPings();
@@ -306,7 +314,7 @@ private static class PingProbe extends MockTransportService.Tracer {
         private final Set<Tuple<DiscoveryNode, Long>> completedPings = Collections.newSetFromMap(new ConcurrentHashMap<>());
         private final CountDownLatch waitForPings;
 
-        public PingProbe(int minCompletedPings) {
+        PingProbe(int minCompletedPings) {
             this.waitForPings = new CountDownLatch(minCompletedPings);
         }
 
diff --git a/core/src/test/java/org/elasticsearch/discovery/ZenUnicastDiscoveryIT.java b/core/src/test/java/org/elasticsearch/discovery/ZenUnicastDiscoveryIT.java
deleted file mode 100644
index 95ad650cd1656..0000000000000
--- a/core/src/test/java/org/elasticsearch/discovery/ZenUnicastDiscoveryIT.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.discovery;
-
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
-import org.elasticsearch.test.ESIntegTestCase.Scope;
-import org.elasticsearch.test.discovery.ClusterDiscoveryConfiguration;
-import org.junit.Before;
-
-import java.util.List;
-import java.util.concurrent.ExecutionException;
-
-import static org.hamcrest.Matchers.equalTo;
-
-@ClusterScope(scope = Scope.TEST, numDataNodes = 0)
-@ESIntegTestCase.SuppressLocalMode
-public class ZenUnicastDiscoveryIT extends ESIntegTestCase {
-
-    private ClusterDiscoveryConfiguration discoveryConfig;
-
-    @Override
-    protected Settings nodeSettings(int nodeOrdinal) {
-        return discoveryConfig.nodeSettings(nodeOrdinal);
-    }
-
-    @Before
-    public void clearConfig() {
-        discoveryConfig = null;
-    }
-
-    public void testNormalClusterForming() throws ExecutionException, InterruptedException {
-        int currentNumNodes = randomIntBetween(3, 5);
-
-        // use explicit unicast hosts so we can start those first
-        int[] unicastHostOrdinals = new int[randomIntBetween(1, currentNumNodes)];
-        for (int i = 0; i < unicastHostOrdinals.length; i++) {
-            unicastHostOrdinals[i] = i;
-        }
-        discoveryConfig = new ClusterDiscoveryConfiguration.UnicastZen(currentNumNodes, unicastHostOrdinals);
-
-        // start the unicast hosts
-        internalCluster().startNodesAsync(unicastHostOrdinals.length).get();
-
-        // start the rest of the cluster
-        internalCluster().startNodesAsync(currentNumNodes - unicastHostOrdinals.length).get();
-
-        if (client().admin().cluster().prepareHealth().setWaitForNodes("" + currentNumNodes).get().isTimedOut()) {
-            logger.info("cluster forming timed out, cluster state:\n{}", client().admin().cluster().prepareState().get().getState().prettyPrint());
-            fail("timed out waiting for cluster to form with [" + currentNumNodes + "] nodes");
-        }
-    }
-
-    // Without the 'include temporalResponses responses to nodesToConnect' improvement in UnicastZenPing#sendPings this
-    // test fails, because 2 nodes elect themselves as master and the health request times out b/c waiting_for_nodes=N
-    // can't be satisfied.
-    public void testMinimumMasterNodes() throws Exception {
-        int currentNumNodes = randomIntBetween(3, 5);
-        final int min_master_nodes = currentNumNodes / 2 + 1;
-        int currentNumOfUnicastHosts = randomIntBetween(min_master_nodes, currentNumNodes);
-        final Settings settings = Settings.builder()
-                .put("discovery.zen.join_timeout", TimeValue.timeValueSeconds(10))
-                .put("discovery.zen.minimum_master_nodes", min_master_nodes)
-                .build();
-        discoveryConfig = new ClusterDiscoveryConfiguration.UnicastZen(currentNumNodes, currentNumOfUnicastHosts, settings);
-
-        List<String> nodes = internalCluster().startNodesAsync(currentNumNodes).get();
-
-        ensureStableCluster(currentNumNodes);
-
-        DiscoveryNode masterDiscoNode = null;
-        for (String node : nodes) {
-            ClusterState state = internalCluster().client(node).admin().cluster().prepareState().setLocal(true).execute().actionGet().getState();
-            assertThat(state.nodes().getSize(), equalTo(currentNumNodes));
-            if (masterDiscoNode == null) {
-                masterDiscoNode = state.nodes().getMasterNode();
-            } else {
-                assertThat(masterDiscoNode.equals(state.nodes().getMasterNode()), equalTo(true));
-            }
-        }
-    }
-}
diff --git a/core/src/test/java/org/elasticsearch/discovery/single/SingleNodeDiscoveryIT.java b/core/src/test/java/org/elasticsearch/discovery/single/SingleNodeDiscoveryIT.java
new file mode 100644
index 0000000000000..4d22dc3e78e66
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/discovery/single/SingleNodeDiscoveryIT.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.discovery.single;
+
+import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.discovery.zen.UnicastHostsProvider;
+import org.elasticsearch.discovery.zen.UnicastZenPing;
+import org.elasticsearch.discovery.zen.ZenPing;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.InternalTestCluster;
+import org.elasticsearch.test.NodeConfigurationSource;
+import org.elasticsearch.test.transport.MockTransportService;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.transport.MockTcpTransportPlugin;
+import org.elasticsearch.transport.TransportService;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Stack;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CountDownLatch;
+import java.util.function.Function;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
+
+@ESIntegTestCase.ClusterScope(
+        scope = ESIntegTestCase.Scope.TEST,
+        numDataNodes = 1,
+        numClientNodes = 0,
+        supportsDedicatedMasters = false,
+        autoMinMasterNodes = false)
+public class SingleNodeDiscoveryIT extends ESIntegTestCase {
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        return Settings
+                .builder()
+                .put(super.nodeSettings(nodeOrdinal))
+                .put("discovery.type", "single-node")
+                .put("transport.tcp.port", "0")
+                .build();
+    }
+
+    public void testDoesNotRespondToZenPings() throws Exception {
+        final Settings settings =
+                Settings.builder().put("cluster.name", internalCluster().getClusterName()).build();
+        final Version version = Version.CURRENT;
+        final Stack<Closeable> closeables = new Stack<>();
+        final TestThreadPool threadPool = new TestThreadPool(getClass().getName());
+        try {
+            final MockTransportService pingTransport =
+                    MockTransportService.createNewService(settings, version, threadPool, null);
+            pingTransport.start();
+            closeables.push(pingTransport);
+            final TransportService nodeTransport =
+                    internalCluster().getInstance(TransportService.class);
+            // try to ping the single node directly
+            final UnicastHostsProvider provider =
+                    () -> Collections.singletonList(nodeTransport.getLocalNode());
+            final CountDownLatch latch = new CountDownLatch(1);
+            final UnicastZenPing unicastZenPing =
+                    new UnicastZenPing(settings, threadPool, pingTransport, provider) {
+                        @Override
+                        protected void finishPingingRound(PingingRound pingingRound) {
+                            latch.countDown();
+                            super.finishPingingRound(pingingRound);
+                        }
+                    };
+            final DiscoveryNodes nodes = DiscoveryNodes.builder()
+                    .add(nodeTransport.getLocalNode())
+                    .add(pingTransport.getLocalNode())
+                    .localNodeId(pingTransport.getLocalNode().getId())
+                    .build();
+            final ClusterName clusterName = new ClusterName(internalCluster().getClusterName());
+            final ClusterState state = ClusterState.builder(clusterName).nodes(nodes).build();
+            unicastZenPing.start(() -> state);
+            closeables.push(unicastZenPing);
+            final CompletableFuture<ZenPing.PingCollection> responses = new CompletableFuture<>();
+            unicastZenPing.ping(responses::complete, TimeValue.timeValueSeconds(3));
+            latch.await();
+            responses.get();
+            assertThat(responses.get().size(), equalTo(0));
+        } finally {
+            while (!closeables.isEmpty()) {
+                IOUtils.closeWhileHandlingException(closeables.pop());
+            }
+            terminate(threadPool);
+        }
+    }
+
+    public void testSingleNodesDoNotDiscoverEachOther() throws IOException, InterruptedException {
+        final TransportService service = internalCluster().getInstance(TransportService.class);
+        final int port = service.boundAddress().publishAddress().getPort();
+        final NodeConfigurationSource configurationSource = new NodeConfigurationSource() {
+            @Override
+            public Settings nodeSettings(int nodeOrdinal) {
+                return Settings
+                        .builder()
+                        .put("discovery.type", "single-node")
+                        .put("http.enabled", false)
+                        .put("transport.type", "mock-socket-network")
+                        /*
+                         * We align the port ranges of the two as then with zen discovery these two
+                         * nodes would find each other.
+                         */
+                        .put("transport.tcp.port", port + "-" + (port + 5 - 1))
+                        .build();
+            }
+        };
+        try (InternalTestCluster other =
+                new InternalTestCluster(
+                        randomLong(),
+                        createTempDir(),
+                        false,
+                        false,
+                        1,
+                        1,
+                        internalCluster().getClusterName(),
+                        configurationSource,
+                        0,
+                        false,
+                        "other",
+                        Collections.singletonList(MockTcpTransportPlugin.class),
+                        Function.identity())) {
+            other.beforeTest(random(), 0);
+            final ClusterState first = internalCluster().getInstance(ClusterService.class).state();
+            final ClusterState second = other.getInstance(ClusterService.class).state();
+            assertThat(first.nodes().getSize(), equalTo(1));
+            assertThat(second.nodes().getSize(), equalTo(1));
+            assertThat(
+                    first.nodes().getMasterNodeId(),
+                    not(equalTo(second.nodes().getMasterNodeId())));
+            assertThat(
+                    first.metaData().clusterUUID(),
+                    not(equalTo(second.metaData().clusterUUID())));
+        }
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/discovery/single/SingleNodeDiscoveryTests.java b/core/src/test/java/org/elasticsearch/discovery/single/SingleNodeDiscoveryTests.java
new file mode 100644
index 0000000000000..a0e0b699d78db
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/discovery/single/SingleNodeDiscoveryTests.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.discovery.single;
+
+import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateObserver;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.transport.MockTransportService;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.threadpool.ThreadPool;
+
+import java.io.Closeable;
+import java.util.Stack;
+import java.util.concurrent.CountDownLatch;
+
+import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
+import static org.hamcrest.Matchers.equalTo;
+
+public class SingleNodeDiscoveryTests extends ESTestCase {
+
+    public void testInitialJoin() throws Exception {
+        final Settings settings = Settings.EMPTY;
+        final Version version = Version.CURRENT;
+        final ThreadPool threadPool = new TestThreadPool(getClass().getName());
+        final Stack<Closeable> stack = new Stack<>();
+        try {
+            final MockTransportService transportService =
+                    MockTransportService.createNewService(settings, version, threadPool, null);
+            stack.push(transportService);
+            transportService.start();
+            final DiscoveryNode node = transportService.getLocalNode();
+            final ClusterService clusterService = createClusterService(threadPool, node);
+            stack.push(clusterService);
+            final SingleNodeDiscovery discovery =
+                    new SingleNodeDiscovery(Settings.EMPTY, clusterService);
+            discovery.startInitialJoin();
+
+            // we are racing against the initial join which is asynchronous so we use an observer
+            final ClusterState state = clusterService.state();
+            final ThreadContext threadContext = threadPool.getThreadContext();
+            final ClusterStateObserver observer =
+                    new ClusterStateObserver(state, clusterService, null, logger, threadContext);
+            if (state.nodes().getMasterNodeId() == null) {
+                final CountDownLatch latch = new CountDownLatch(1);
+                observer.waitForNextChange(new ClusterStateObserver.Listener() {
+                    @Override
+                    public void onNewClusterState(ClusterState state) {
+                        latch.countDown();
+                    }
+
+                    @Override
+                    public void onClusterServiceClose() {
+                        latch.countDown();
+                    }
+
+                    @Override
+                    public void onTimeout(TimeValue timeout) {
+                        assert false;
+                    }
+                }, s -> s.nodes().getMasterNodeId() != null);
+
+                latch.await();
+            }
+
+            final DiscoveryNodes nodes = clusterService.state().nodes();
+            assertThat(nodes.getSize(), equalTo(1));
+            assertThat(nodes.getMasterNode().getId(), equalTo(node.getId()));
+        } finally {
+            while (!stack.isEmpty()) {
+                IOUtils.closeWhileHandlingException(stack.pop());
+            }
+            terminate(threadPool);
+        }
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/ElectMasterServiceTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/ElectMasterServiceTests.java
index b31b0cbaa55e0..079153aa44549 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/ElectMasterServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/ElectMasterServiceTests.java
@@ -23,7 +23,7 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.LocalTransportAddress;
-import org.elasticsearch.discovery.zen.elect.ElectMasterService;
+import org.elasticsearch.discovery.zen.ElectMasterService.MasterCandidate;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.ArrayList;
@@ -31,6 +31,10 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
 
 public class ElectMasterServiceTests extends ESTestCase {
 
@@ -55,9 +59,25 @@ List<DiscoveryNode> generateRandomNodes() {
         return nodes;
     }
 
+    List<MasterCandidate> generateRandomCandidates() {
+        int count = scaledRandomIntBetween(1, 100);
+        ArrayList<MasterCandidate> candidates = new ArrayList<>(count);
+        for (int i = 0; i < count; i++) {
+            Set<DiscoveryNode.Role> roles = new HashSet<>();
+            roles.add(DiscoveryNode.Role.MASTER);
+            DiscoveryNode node = new DiscoveryNode("n_" + i, "n_" + i, LocalTransportAddress.buildUnique(), Collections.emptyMap(),
+                roles, Version.CURRENT);
+            candidates.add(new MasterCandidate(node, randomBoolean() ? MasterCandidate.UNRECOVERED_CLUSTER_VERSION : randomNonNegativeLong()));
+        }
+
+        Collections.shuffle(candidates, random());
+        return candidates;
+    }
+
+
     public void testSortByMasterLikelihood() {
         List<DiscoveryNode> nodes = generateRandomNodes();
-        List<DiscoveryNode> sortedNodes = electMasterService().sortByMasterLikelihood(nodes);
+        List<DiscoveryNode> sortedNodes = ElectMasterService.sortByMasterLikelihood(nodes);
         assertEquals(nodes.size(), sortedNodes.size());
         DiscoveryNode prevNode = sortedNodes.get(0);
         for (int i = 1; i < sortedNodes.size(); i++) {
@@ -69,37 +89,69 @@ public void testSortByMasterLikelihood() {
             }
             prevNode = node;
         }
+    }
+
+    public void testTieBreakActiveMasters() {
+        List<DiscoveryNode> nodes = generateRandomCandidates().stream().map(MasterCandidate::getNode).collect(Collectors.toList());
+        DiscoveryNode bestMaster = electMasterService().tieBreakActiveMasters(nodes);
+        for (DiscoveryNode node: nodes) {
+            if (node.equals(bestMaster) == false) {
+                assertTrue(bestMaster.getId().compareTo(node.getId()) < 0);
+            }
+        }
+    }
 
+    public void testHasEnoughNodes() {
+        List<DiscoveryNode> nodes = rarely() ? Collections.emptyList() : generateRandomNodes();
+        ElectMasterService service = electMasterService();
+        int masterNodes = (int) nodes.stream().filter(DiscoveryNode::isMasterNode).count();
+        service.minimumMasterNodes(randomIntBetween(-1, masterNodes));
+        assertThat(service.hasEnoughMasterNodes(nodes), equalTo(masterNodes > 0));
+        service.minimumMasterNodes(masterNodes + 1 + randomIntBetween(0, nodes.size()));
+        assertFalse(service.hasEnoughMasterNodes(nodes));
+    }
+
+    public void testHasEnoughCandidates() {
+        List<MasterCandidate> candidates = rarely() ? Collections.emptyList() : generateRandomCandidates();
+        ElectMasterService service = electMasterService();
+        service.minimumMasterNodes(randomIntBetween(-1, candidates.size()));
+        assertThat(service.hasEnoughCandidates(candidates), equalTo(candidates.size() > 0));
+        service.minimumMasterNodes(candidates.size() + 1 + randomIntBetween(0, candidates.size()));
+        assertFalse(service.hasEnoughCandidates(candidates));
     }
 
     public void testElectMaster() {
+        List<MasterCandidate> candidates = generateRandomCandidates();
+        ElectMasterService service = electMasterService();
+        int minMasterNodes = randomIntBetween(0, candidates.size());
+        service.minimumMasterNodes(minMasterNodes);
+        MasterCandidate master = service.electMaster(candidates);
+        assertNotNull(master);
+        for (MasterCandidate candidate : candidates) {
+            if (candidate.getNode().equals(master.getNode())) {
+                // nothing much to test here
+            } else if (candidate.getClusterStateVersion() == master.getClusterStateVersion()) {
+                assertThat("candidate " + candidate + " has a lower or equal id than master " + master, candidate.getNode().getId(),
+                    greaterThan(master.getNode().getId()));
+            } else {
+                assertThat("candidate " + master + " has a higher cluster state version than candidate " + candidate,
+                    master.getClusterStateVersion(), greaterThan(candidate.getClusterStateVersion()));
+            }
+        }
+    }
+
+    public void testCountMasterNodes() {
         List<DiscoveryNode> nodes = generateRandomNodes();
         ElectMasterService service = electMasterService();
-        int min_master_nodes = randomIntBetween(0, nodes.size());
-        service.minimumMasterNodes(min_master_nodes);
 
-        int master_nodes = 0;
+        int masterNodes = 0;
+
         for (DiscoveryNode node : nodes) {
             if (node.isMasterNode()) {
-                master_nodes++;
+                masterNodes++;
             }
         }
-        DiscoveryNode master = null;
-        if (service.hasEnoughMasterNodes(nodes)) {
-            master = service.electMaster(nodes);
-        }
 
-        if (master_nodes == 0) {
-            assertNull(master);
-        } else if (min_master_nodes > 0 && master_nodes < min_master_nodes) {
-            assertNull(master);
-        } else {
-            assertNotNull(master);
-            for (DiscoveryNode node : nodes) {
-                if (node.isMasterNode()) {
-                    assertTrue(master.getId().compareTo(node.getId()) <= 0);
-                }
-            }
-        }
+        assertEquals(masterNodes, service.countMasterNodes(nodes));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/MembershipActionTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/MembershipActionTests.java
new file mode 100644
index 0000000000000..1bed2ed746e84
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/MembershipActionTests.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.discovery.zen;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.LocalTransportAddress;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.VersionUtils;
+
+import static org.elasticsearch.test.VersionUtils.getPreviousVersion;
+import static org.elasticsearch.test.VersionUtils.incompatibleFutureVersion;
+import static org.elasticsearch.test.VersionUtils.maxCompatibleVersion;
+import static org.elasticsearch.test.VersionUtils.randomCompatibleVersion;
+import static org.elasticsearch.test.VersionUtils.randomVersion;
+import static org.elasticsearch.test.VersionUtils.randomVersionBetween;
+
+public class MembershipActionTests extends ESTestCase {
+
+    public void testPreventJoinClusterWithNewerIndices() {
+        Settings.builder().build();
+        MetaData.Builder metaBuilder = MetaData.builder();
+        IndexMetaData indexMetaData = IndexMetaData.builder("test")
+            .settings(settings(Version.CURRENT))
+            .numberOfShards(1)
+            .numberOfReplicas(1).build();
+        metaBuilder.put(indexMetaData, false);
+        MetaData metaData = metaBuilder.build();
+        MembershipAction.ensureIndexCompatibility(Version.CURRENT, metaData);
+
+        expectThrows(IllegalStateException.class, () ->
+        MembershipAction.ensureIndexCompatibility(VersionUtils.getPreviousVersion(Version.CURRENT),
+            metaData));
+    }
+
+    public void testPreventJoinClusterWithUnsupportedIndices() {
+        Settings.builder().build();
+        MetaData.Builder metaBuilder = MetaData.builder();
+        IndexMetaData indexMetaData = IndexMetaData.builder("test")
+            .settings(settings(Version.fromId(Version.CURRENT
+                .minimumIndexCompatibilityVersion().id - 1)))
+            .numberOfShards(1)
+            .numberOfReplicas(1).build();
+        metaBuilder.put(indexMetaData, false);
+        MetaData metaData = metaBuilder.build();
+        expectThrows(IllegalStateException.class, () ->
+            MembershipAction.ensureIndexCompatibility(Version.CURRENT,
+                metaData));
+    }
+
+    public void testPreventJoinClusterWithUnsupportedNodeVersions() {
+        DiscoveryNodes.Builder builder = DiscoveryNodes.builder();
+        final Version version = randomVersion(random());
+        builder.add(new DiscoveryNode(UUIDs.base64UUID(), LocalTransportAddress.buildUnique(), version));
+        builder.add(new DiscoveryNode(UUIDs.base64UUID(), LocalTransportAddress.buildUnique(), randomCompatibleVersion(random(), version)));
+        DiscoveryNodes nodes = builder.build();
+
+        final Version maxNodeVersion = nodes.getMaxNodeVersion();
+        final Version minNodeVersion = nodes.getMinNodeVersion();
+        if (maxNodeVersion.onOrAfter(Version.V_5_0_0)) {
+            final Version tooLow = getPreviousVersion(maxNodeVersion.minimumCompatibilityVersion());
+            expectThrows(IllegalStateException.class, () -> {
+                if (randomBoolean()) {
+                    MembershipAction.ensureNodesCompatibility(tooLow, nodes);
+                } else {
+                    MembershipAction.ensureNodesCompatibility(tooLow, minNodeVersion, maxNodeVersion);
+                }
+            });
+        }
+
+        if (minNodeVersion.before(Version.V_5_0_0_alpha1)) {
+            Version tooHigh = incompatibleFutureVersion(minNodeVersion);
+            expectThrows(IllegalStateException.class, () -> {
+                if (randomBoolean()) {
+                    MembershipAction.ensureNodesCompatibility(tooHigh, nodes);
+                } else {
+                    MembershipAction.ensureNodesCompatibility(tooHigh, minNodeVersion, maxNodeVersion);
+                }
+            });
+        }
+
+        final Version minGoodVersion = maxNodeVersion.major == minNodeVersion.major ?
+            // we have to stick with the same major
+            minNodeVersion :
+            maxNodeVersion.minimumCompatibilityVersion();
+        final Version justGood = randomVersionBetween(random(), minGoodVersion, maxCompatibleVersion(minNodeVersion));
+
+        if (randomBoolean()) {
+            MembershipAction.ensureNodesCompatibility(justGood, nodes);
+        } else {
+            MembershipAction.ensureNodesCompatibility(justGood, minNodeVersion, maxNodeVersion);
+        }
+    }
+
+    public void testSuccess() {
+        Settings.builder().build();
+        MetaData.Builder metaBuilder = MetaData.builder();
+        IndexMetaData indexMetaData = IndexMetaData.builder("test")
+            .settings(settings(VersionUtils.randomVersionBetween(random(),
+                Version.CURRENT.minimumIndexCompatibilityVersion(), Version.CURRENT)))
+            .numberOfShards(1)
+            .numberOfReplicas(1).build();
+        metaBuilder.put(indexMetaData, false);
+        indexMetaData = IndexMetaData.builder("test1")
+            .settings(settings(VersionUtils.randomVersionBetween(random(),
+                Version.CURRENT.minimumIndexCompatibilityVersion(), Version.CURRENT)))
+            .numberOfShards(1)
+            .numberOfReplicas(1).build();
+        metaBuilder.put(indexMetaData, false);
+        MetaData metaData = metaBuilder.build();
+            MembershipAction.ensureIndexCompatibility(Version.CURRENT,
+                metaData);
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
index 9a111152a1a67..61a4900481c1c 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
@@ -23,7 +23,9 @@
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.NotMasterException;
+import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -36,18 +38,14 @@
 import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.BaseFuture;
-import org.elasticsearch.discovery.DiscoverySettings;
-import org.elasticsearch.discovery.zen.elect.ElectMasterService;
-import org.elasticsearch.discovery.zen.membership.MembershipAction;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -84,14 +82,19 @@
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_VERSION_CREATED;
 import static org.elasticsearch.cluster.routing.RoutingTableTests.updateActiveAllocations;
+import static org.elasticsearch.gateway.GatewayService.STATE_NOT_RECOVERED_BLOCK;
 import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
 import static org.elasticsearch.test.ClusterServiceUtils.setState;
+import static org.elasticsearch.test.VersionUtils.allVersions;
+import static org.elasticsearch.test.VersionUtils.getPreviousVersion;
+import static org.elasticsearch.test.VersionUtils.randomCompatibleVersion;
+import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 
-@TestLogging("discovery.zen:TRACE")
+@TestLogging("org.elasticsearch.discovery.zen:TRACE,org.elasticsearch.cluster.service:TRACE")
 public class NodeJoinControllerTests extends ESTestCase {
 
     private static ThreadPool threadPool;
@@ -120,8 +123,7 @@ public void setUp() throws Exception {
         setState(clusterService, ClusterState.builder(clusterService.state()).nodes(
             DiscoveryNodes.builder(initialNodes).masterNodeId(localNode.getId())));
         nodeJoinController = new NodeJoinController(clusterService, createAllocationService(Settings.EMPTY),
-            new ElectMasterService(Settings.EMPTY), new DiscoverySettings(Settings.EMPTY, new ClusterSettings(Settings.EMPTY,
-            ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)), Settings.EMPTY);
+            new ElectMasterService(Settings.EMPTY), Settings.EMPTY);
     }
 
     @After
@@ -561,7 +563,7 @@ public void testRejectingJoinWithSameIdButDifferentNode() throws InterruptedExce
             randomBoolean() ? existing.getAddress() : LocalTransportAddress.buildUnique(),
             randomBoolean() ? existing.getAttributes() : Collections.singletonMap("attr", "other"),
             randomBoolean() ? existing.getRoles() : new HashSet<>(randomSubsetOf(Arrays.asList(DiscoveryNode.Role.values()))),
-            randomBoolean() ? existing.getVersion() : VersionUtils.randomVersion(random()));
+            existing.getVersion());
 
         ExecutionException e = expectThrows(ExecutionException.class, () -> joinNode(other_node));
         assertThat(e.getMessage(), containsString("found existing node"));
@@ -580,6 +582,96 @@ public void testRejectingRestartedNodeJoinsBeforeProcessingNodeLeft() throws Int
         assertThat(e.getMessage(), containsString("found existing node"));
     }
 
+    public void testRejectingJoinWithIncompatibleVersion() throws InterruptedException, ExecutionException {
+        addNodes(randomInt(5));
+        final Version badVersion = getPreviousVersion(Version.CURRENT.minimumCompatibilityVersion());
+        final DiscoveryNode badNode = new DiscoveryNode("badNode", LocalTransportAddress.buildUnique(), emptyMap(),
+            new HashSet<>(randomSubsetOf(Arrays.asList(DiscoveryNode.Role.values()))), badVersion);
+
+        final Version goodVersion =
+            randomFrom(allVersions().stream().filter(Version.CURRENT::isCompatible).collect(Collectors.toList()));
+        final DiscoveryNode goodNode = new DiscoveryNode("goodNode", LocalTransportAddress.buildUnique(), emptyMap(),
+            new HashSet<>(randomSubsetOf(Arrays.asList(DiscoveryNode.Role.values()))), goodVersion);
+
+        CountDownLatch latch = new CountDownLatch(1);
+        // block cluster state
+        clusterService.submitStateUpdateTask("test", new ClusterStateUpdateTask(Priority.IMMEDIATE) {
+            @Override
+            public ClusterState execute(ClusterState currentState) throws Exception {
+                latch.await();
+                return currentState;
+            }
+
+            @Override
+            public void onFailure(String source, Exception e) {
+                throw new AssertionError(e);
+            }
+        });
+
+        final SimpleFuture badJoin;
+        final SimpleFuture goodJoin;
+        if (randomBoolean()) {
+            badJoin = joinNodeAsync(badNode);
+            goodJoin = joinNodeAsync(goodNode);
+        } else {
+            goodJoin = joinNodeAsync(goodNode);
+            badJoin = joinNodeAsync(badNode);
+        }
+        assert goodJoin.isDone() == false;
+        assert badJoin.isDone() == false;
+        latch.countDown();
+        goodJoin.get();
+        ExecutionException e = expectThrows(ExecutionException.class, badJoin::get);
+        assertThat(e.getCause(), instanceOf(IllegalStateException.class));
+        assertThat(e.getCause().getMessage(), allOf(containsString("node version"), containsString("not supported")));
+    }
+
+    public void testRejectingJoinWithIncompatibleVersionWithUnrecoveredState() throws InterruptedException, ExecutionException {
+        addNodes(randomInt(5));
+        ClusterState.Builder builder = ClusterState.builder(clusterService.state());
+        builder.blocks(ClusterBlocks.builder().addGlobalBlock(STATE_NOT_RECOVERED_BLOCK));
+        setState(clusterService, builder.build());
+        final Version badVersion = getPreviousVersion(Version.CURRENT.minimumCompatibilityVersion());
+        final DiscoveryNode badNode = new DiscoveryNode("badNode", LocalTransportAddress.buildUnique(), emptyMap(),
+            new HashSet<>(randomSubsetOf(Arrays.asList(DiscoveryNode.Role.values()))), badVersion);
+
+        final Version goodVersion = randomFrom(randomCompatibleVersion(random(), Version.CURRENT));
+        final DiscoveryNode goodNode = new DiscoveryNode("goodNode", LocalTransportAddress.buildUnique(), emptyMap(),
+            new HashSet<>(randomSubsetOf(Arrays.asList(DiscoveryNode.Role.values()))), goodVersion);
+
+        CountDownLatch latch = new CountDownLatch(1);
+        // block cluster state
+        clusterService.submitStateUpdateTask("test", new ClusterStateUpdateTask(Priority.IMMEDIATE) {
+            @Override
+            public ClusterState execute(ClusterState currentState) throws Exception {
+                latch.await();
+                return currentState;
+            }
+
+            @Override
+            public void onFailure(String source, Exception e) {
+                throw new AssertionError(e);
+            }
+        });
+
+        final SimpleFuture badJoin;
+        final SimpleFuture goodJoin;
+        if (randomBoolean()) {
+            badJoin = joinNodeAsync(badNode);
+            goodJoin = joinNodeAsync(goodNode);
+        } else {
+            goodJoin = joinNodeAsync(goodNode);
+            badJoin = joinNodeAsync(badNode);
+        }
+        assert goodJoin.isDone() == false;
+        assert badJoin.isDone() == false;
+        latch.countDown();
+        goodJoin.get();
+        ExecutionException e = expectThrows(ExecutionException.class, badJoin::get);
+        assertThat(e.getCause(), instanceOf(IllegalStateException.class));
+        assertThat(e.getCause().getMessage(), allOf(containsString("node version"), containsString("not supported")));
+    }
+
     /**
      * Tests tha node can become a master, even though the last cluster state it knows contains
      * nodes that conflict with the joins it got and needs to become a master
@@ -629,12 +721,15 @@ public void testElectionBasedOnConflictingNodes() throws InterruptedException, E
 
         setState(clusterService, stateBuilder.build());
 
-        final DiscoveryNode restartedNode = new DiscoveryNode(otherNode.getId(),
-            randomBoolean() ? otherNode.getAddress() : LocalTransportAddress.buildUnique(), otherNode.getAttributes(),
-            otherNode.getRoles(), Version.CURRENT);
+        // conflict on node id or address
+        final DiscoveryNode conflictingNode = randomBoolean() ?
+            new DiscoveryNode(otherNode.getId(), randomBoolean() ? otherNode.getAddress() : LocalTransportAddress.buildUnique(),
+                otherNode.getAttributes(), otherNode.getRoles(), Version.CURRENT) :
+            new DiscoveryNode("conflicting_address_node", otherNode.getAddress(), otherNode.getAttributes(), otherNode.getRoles(),
+                Version.CURRENT);
 
         nodeJoinController.startElectionContext();
-        final SimpleFuture joinFuture = joinNodeAsync(restartedNode);
+        final SimpleFuture joinFuture = joinNodeAsync(conflictingNode);
         final CountDownLatch elected = new CountDownLatch(1);
         nodeJoinController.waitToBeElectedAsMaster(1, TimeValue.timeValueHours(5), new NodeJoinController.ElectionCallback() {
             @Override
@@ -658,9 +753,9 @@ public void onFailure(Throwable t) {
         assertTrue(finalNodes.isLocalNodeElectedMaster());
         assertThat(finalNodes.getLocalNode(), equalTo(masterNode));
         assertThat(finalNodes.getSize(), equalTo(2));
-        assertThat(finalNodes.get(restartedNode.getId()), equalTo(restartedNode));
+        assertThat(finalNodes.get(conflictingNode.getId()), equalTo(conflictingNode));
         List<ShardRouting> activeShardsOnRestartedNode =
-            StreamSupport.stream(finalState.getRoutingNodes().node(restartedNode.getId()).spliterator(), false)
+            StreamSupport.stream(finalState.getRoutingNodes().node(conflictingNode.getId()).spliterator(), false)
                 .filter(ShardRouting::active).collect(Collectors.toList());
         assertThat(activeShardsOnRestartedNode, empty());
     }
@@ -679,10 +774,10 @@ private void addNodes(int count) {
 
     protected void assertNodesInCurrentState(List<DiscoveryNode> expectedNodes) {
         final ClusterState state = clusterService.state();
-        logger.info("assert for [{}] in:\n{}", expectedNodes, state.prettyPrint());
+        logger.info("assert for [{}] in:\n{}", expectedNodes, state);
         DiscoveryNodes discoveryNodes = state.nodes();
         for (DiscoveryNode node : expectedNodes) {
-            assertThat("missing " + node + "\n" + discoveryNodes.prettyPrint(), discoveryNodes.get(node.getId()), equalTo(node));
+            assertThat("missing " + node + "\n" + discoveryNodes, discoveryNodes.get(node.getId()), equalTo(node));
         }
         assertThat(discoveryNodes.getSize(), equalTo(expectedNodes.size()));
     }
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/NodeRemovalClusterStateTaskExecutorTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/NodeRemovalClusterStateTaskExecutorTests.java
index 35335a8ede4ce..31c9ec4dcb737 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/NodeRemovalClusterStateTaskExecutorTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/NodeRemovalClusterStateTaskExecutorTests.java
@@ -26,16 +26,14 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
-import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.common.transport.LocalTransportAddress;
-import org.elasticsearch.discovery.zen.elect.ElectMasterService;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
-import java.util.function.BiFunction;
+import java.util.function.Consumer;
 import java.util.stream.Collectors;
 import java.util.stream.StreamSupport;
 
@@ -69,7 +67,7 @@ public void testRemovingNonExistentNodes() throws Exception {
                         .map(node -> new ZenDiscovery.NodeRemovalClusterStateTaskExecutor.Task(node, randomBoolean() ? "left" : "failed"))
                         .collect(Collectors.toList());
 
-        final ClusterStateTaskExecutor.BatchResult<ZenDiscovery.NodeRemovalClusterStateTaskExecutor.Task> result
+        final ClusterStateTaskExecutor.ClusterTasksResult<ZenDiscovery.NodeRemovalClusterStateTaskExecutor.Task> result
                 = executor.execute(clusterState, tasks);
         assertThat(result.resultingState, equalTo(clusterState));
     }
@@ -80,17 +78,12 @@ public void testNotEnoughMasterNodesAfterRemove() throws Exception {
 
         final AllocationService allocationService = mock(AllocationService.class);
 
-        final AtomicBoolean rejoined = new AtomicBoolean();
-        final AtomicReference<ClusterState> rejoinedClusterState = new AtomicReference<>();
-        final BiFunction<ClusterState, String, ClusterState> rejoin = (cs, r) -> {
-            rejoined.set(true);
-            rejoinedClusterState.set(ClusterState.builder(cs).build());
-            return rejoinedClusterState.get();
-        };
+        final AtomicBoolean rejoinCalled = new AtomicBoolean();
+        final Consumer<String> submitRejoin = source -> rejoinCalled.set(true);
 
         final AtomicReference<ClusterState> remainingNodesClusterState = new AtomicReference<>();
         final ZenDiscovery.NodeRemovalClusterStateTaskExecutor executor =
-                new ZenDiscovery.NodeRemovalClusterStateTaskExecutor(allocationService, electMasterService, rejoin, logger) {
+                new ZenDiscovery.NodeRemovalClusterStateTaskExecutor(allocationService, electMasterService, submitRejoin, logger) {
                     @Override
                     ClusterState remainingNodesClusterState(ClusterState currentState, DiscoveryNodes.Builder remainingNodesBuilder) {
                         remainingNodesClusterState.set(super.remainingNodesClusterState(currentState, remainingNodesBuilder));
@@ -113,18 +106,20 @@ ClusterState remainingNodesClusterState(ClusterState currentState, DiscoveryNode
         }
         final ClusterState clusterState = ClusterState.builder(new ClusterName("test")).nodes(builder).build();
 
-        final ClusterStateTaskExecutor.BatchResult<ZenDiscovery.NodeRemovalClusterStateTaskExecutor.Task> result =
+        final ClusterStateTaskExecutor.ClusterTasksResult<ZenDiscovery.NodeRemovalClusterStateTaskExecutor.Task> result =
                 executor.execute(clusterState, tasks);
         verify(electMasterService).hasEnoughMasterNodes(eq(remainingNodesClusterState.get().nodes()));
+        verify(electMasterService).countMasterNodes(eq(remainingNodesClusterState.get().nodes()));
+        verify(electMasterService).minimumMasterNodes();
         verifyNoMoreInteractions(electMasterService);
 
         // ensure that we did not reroute
         verifyNoMoreInteractions(allocationService);
-        assertTrue(rejoined.get());
-        assertThat(result.resultingState, equalTo(rejoinedClusterState.get()));
+        assertTrue(rejoinCalled.get());
+        assertThat(result.resultingState, equalTo(clusterState));
 
         for (final ZenDiscovery.NodeRemovalClusterStateTaskExecutor.Task task : tasks) {
-            assertNull(result.resultingState.nodes().get(task.node().getId()));
+            assertNotNull(result.resultingState.nodes().get(task.node().getId()));
         }
     }
 
@@ -134,16 +129,13 @@ public void testRerouteAfterRemovingNodes() throws Exception {
 
         final AllocationService allocationService = mock(AllocationService.class);
         when(allocationService.deassociateDeadNodes(any(ClusterState.class), eq(true), any(String.class)))
-            .thenReturn(mock(RoutingAllocation.Result.class));
+            .thenAnswer(im -> im.getArguments()[0]);
 
-        final BiFunction<ClusterState, String, ClusterState> rejoin = (cs, r) -> {
-            fail("rejoin should not be invoked");
-            return cs;
-        };
+        final Consumer<String> submitRejoin = source -> fail("rejoin should not be invoked");
 
         final AtomicReference<ClusterState> remainingNodesClusterState = new AtomicReference<>();
         final ZenDiscovery.NodeRemovalClusterStateTaskExecutor executor =
-                new ZenDiscovery.NodeRemovalClusterStateTaskExecutor(allocationService, electMasterService, rejoin, logger) {
+                new ZenDiscovery.NodeRemovalClusterStateTaskExecutor(allocationService, electMasterService, submitRejoin, logger) {
                     @Override
                     ClusterState remainingNodesClusterState(ClusterState currentState, DiscoveryNodes.Builder remainingNodesBuilder) {
                         remainingNodesClusterState.set(super.remainingNodesClusterState(currentState, remainingNodesBuilder));
@@ -166,7 +158,7 @@ ClusterState remainingNodesClusterState(ClusterState currentState, DiscoveryNode
         }
         final ClusterState clusterState = ClusterState.builder(new ClusterName("test")).nodes(builder).build();
 
-        final ClusterStateTaskExecutor.BatchResult<ZenDiscovery.NodeRemovalClusterStateTaskExecutor.Task> result =
+        final ClusterStateTaskExecutor.ClusterTasksResult<ZenDiscovery.NodeRemovalClusterStateTaskExecutor.Task> result =
                 executor.execute(clusterState, tasks);
         verify(electMasterService).hasEnoughMasterNodes(eq(remainingNodesClusterState.get().nodes()));
         verifyNoMoreInteractions(electMasterService);
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/publish/PendingClusterStatesQueueTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/PendingClusterStatesQueueTests.java
similarity index 98%
rename from core/src/test/java/org/elasticsearch/discovery/zen/publish/PendingClusterStatesQueueTests.java
rename to core/src/test/java/org/elasticsearch/discovery/zen/PendingClusterStatesQueueTests.java
index 9bb8bf801f165..14b63762f106d 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/publish/PendingClusterStatesQueueTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/PendingClusterStatesQueueTests.java
@@ -16,7 +16,8 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.discovery.zen.publish;
+
+package org.elasticsearch.discovery.zen;
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
@@ -26,7 +27,7 @@
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.LocalTransportAddress;
-import org.elasticsearch.discovery.zen.publish.PendingClusterStatesQueue.ClusterStateContext;
+import org.elasticsearch.discovery.zen.PendingClusterStatesQueue.ClusterStateContext;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.ArrayList;
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateActionTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/PublishClusterStateActionTests.java
similarity index 88%
rename from core/src/test/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateActionTests.java
rename to core/src/test/java/org/elasticsearch/discovery/zen/PublishClusterStateActionTests.java
index 31c828ec30fd5..025d8ab3a436d 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateActionTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/PublishClusterStateActionTests.java
@@ -17,12 +17,13 @@
  * under the License.
  */
 
-package org.elasticsearch.discovery.zen.publish;
+package org.elasticsearch.discovery.zen;
 
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateListener;
@@ -36,13 +37,12 @@
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.discovery.Discovery;
 import org.elasticsearch.discovery.DiscoverySettings;
-import org.elasticsearch.discovery.zen.DiscoveryNodesProvider;
-import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.junit.annotations.TestLogging;
@@ -82,7 +82,7 @@
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
-@TestLogging("discovery.zen.publish:TRACE")
+@TestLogging("org.elasticsearch.discovery.zen.publish:TRACE")
 public class PublishClusterStateActionTests extends ESTestCase {
 
     private static final ClusterName CLUSTER_NAME = ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY);
@@ -90,7 +90,7 @@ public class PublishClusterStateActionTests extends ESTestCase {
     protected ThreadPool threadPool;
     protected Map<String, MockNode> nodes = new HashMap<>();
 
-    public static class MockNode implements PublishClusterStateAction.NewPendingClusterStateListener, DiscoveryNodesProvider {
+    public static class MockNode implements PublishClusterStateAction.NewPendingClusterStateListener {
         public final DiscoveryNode discoveryNode;
         public final MockTransportService service;
         public MockPublishAction action;
@@ -100,21 +100,25 @@ public static class MockNode implements PublishClusterStateAction.NewPendingClus
 
         private final Logger logger;
 
-        public MockNode(DiscoveryNode discoveryNode, MockTransportService service, @Nullable ClusterStateListener listener, Logger logger) {
+        public MockNode(DiscoveryNode discoveryNode, MockTransportService service,
+                        @Nullable ClusterStateListener listener, Logger logger) {
             this.discoveryNode = discoveryNode;
             this.service = service;
             this.listener = listener;
             this.logger = logger;
-            this.clusterState = ClusterState.builder(CLUSTER_NAME).nodes(DiscoveryNodes.builder().add(discoveryNode).localNodeId(discoveryNode.getId()).build()).build();
+            this.clusterState = ClusterState.builder(CLUSTER_NAME).nodes(DiscoveryNodes.builder()
+                .add(discoveryNode).localNodeId(discoveryNode.getId()).build()).build();
         }
 
         public MockNode setAsMaster() {
-            this.clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).masterNodeId(discoveryNode.getId())).build();
+            this.clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
+                .masterNodeId(discoveryNode.getId())).build();
             return this;
         }
 
         public MockNode resetMasterId() {
-            this.clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).masterNodeId(null)).build();
+            this.clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
+                .masterNodeId(null)).build();
             return this;
         }
 
@@ -126,7 +130,8 @@ public void connectTo(DiscoveryNode node) {
         @Override
         public void onNewClusterState(String reason) {
             ClusterState newClusterState = action.pendingStatesQueue().getNextClusterStateToProcess();
-            logger.debug("[{}] received version [{}], uuid [{}]", discoveryNode.getName(), newClusterState.version(), newClusterState.stateUUID());
+            logger.debug("[{}] received version [{}], uuid [{}]",
+                discoveryNode.getName(), newClusterState.version(), newClusterState.stateUUID());
             if (listener != null) {
                 ClusterChangedEvent event = new ClusterChangedEvent("", newClusterState, clusterState);
                 listener.clusterChanged(event);
@@ -137,7 +142,6 @@ public void onNewClusterState(String reason) {
             action.pendingStatesQueue().markAsProcessed(newClusterState);
         }
 
-        @Override
         public DiscoveryNodes nodes() {
             return clusterState.nodes();
         }
@@ -145,26 +149,27 @@ public DiscoveryNodes nodes() {
     }
 
     public MockNode createMockNode(final String name) throws Exception {
-        return createMockNode(name, Settings.EMPTY);
+        return createMockNode(name, Settings.EMPTY, null);
     }
 
-    public MockNode createMockNode(String name, Settings settings) throws Exception {
-        return createMockNode(name, settings, null);
+    public MockNode createMockNode(String name, final Settings basSettings, @Nullable ClusterStateListener listener) throws Exception {
+        return createMockNode(name, basSettings, listener, threadPool, logger, nodes);
     }
 
-    public MockNode createMockNode(String name, final Settings basSettings, @Nullable ClusterStateListener listener) throws Exception {
+    public static MockNode createMockNode(String name, final Settings basSettings, @Nullable ClusterStateListener listener,
+                                          ThreadPool threadPool, Logger logger, Map<String, MockNode> nodes) throws Exception {
         final Settings settings = Settings.builder()
                 .put("name", name)
-                .put(TransportService.TRACE_LOG_INCLUDE_SETTING.getKey(), "", TransportService.TRACE_LOG_EXCLUDE_SETTING.getKey(), "NOTHING")
+                .put(TransportService.TRACE_LOG_INCLUDE_SETTING.getKey(), "",
+                     TransportService.TRACE_LOG_EXCLUDE_SETTING.getKey(), "NOTHING")
                 .put(basSettings)
                 .build();
 
-        MockTransportService service = buildTransportService(settings);
-        DiscoveryNode discoveryNode = DiscoveryNode.createLocal(settings, service.boundAddress().publishAddress(),
-            NodeEnvironment.generateNodeId(settings));
+        MockTransportService service = buildTransportService(settings, threadPool);
+        DiscoveryNode discoveryNode = service.getLocalDiscoNode();
         MockNode node = new MockNode(discoveryNode, service, listener, logger);
         node.action = buildPublishClusterStateAction(settings, service, () -> node.clusterState, node);
-        final CountDownLatch latch = new CountDownLatch(nodes.size() * 2 + 1);
+        final CountDownLatch latch = new CountDownLatch(nodes.size() * 2);
         TransportConnectionListener waitForConnection = new TransportConnectionListener() {
             @Override
             public void onNodeConnected(DiscoveryNode node) {
@@ -182,7 +187,6 @@ public void onNodeDisconnected(DiscoveryNode node) {
             curNode.connectTo(node.discoveryNode);
             node.connectTo(curNode.discoveryNode);
         }
-        node.connectTo(node.discoveryNode);
         assertThat("failed to wait for all nodes to connect", latch.await(5, TimeUnit.SECONDS), equalTo(true));
         for (MockNode curNode : nodes.values()) {
             curNode.service.removeConnectionListener(waitForConnection);
@@ -222,20 +226,19 @@ public void setUp() throws Exception {
     public void tearDown() throws Exception {
         super.tearDown();
         for (MockNode curNode : nodes.values()) {
-            curNode.action.close();
             curNode.service.close();
         }
         terminate(threadPool);
     }
 
-    protected MockTransportService buildTransportService(Settings settings) {
-        MockTransportService transportService = MockTransportService.local(Settings.EMPTY, Version.CURRENT, threadPool);
+    private static MockTransportService buildTransportService(Settings settings, ThreadPool threadPool) {
+        MockTransportService transportService = MockTransportService.local(settings, Version.CURRENT, threadPool, null);
         transportService.start();
         transportService.acceptIncomingRequests();
         return transportService;
     }
 
-    protected MockPublishAction buildPublishClusterStateAction(
+    private static MockPublishAction buildPublishClusterStateAction(
             Settings settings,
             MockTransportService transportService,
             Supplier<ClusterState> clusterStateSupplier,
@@ -243,9 +246,11 @@ protected MockPublishAction buildPublishClusterStateAction(
     ) {
         DiscoverySettings discoverySettings =
                 new DiscoverySettings(settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
+        NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(ClusterModule.getNamedWriteables());
         return new MockPublishAction(
                 settings,
                 transportService,
+                namedWriteableRegistry,
                 clusterStateSupplier,
                 listener,
                 discoverySettings,
@@ -253,8 +258,8 @@ protected MockPublishAction buildPublishClusterStateAction(
     }
 
     public void testSimpleClusterStatePublishing() throws Exception {
-        MockNode nodeA = createMockNode("nodeA", Settings.EMPTY).setAsMaster();
-        MockNode nodeB = createMockNode("nodeB", Settings.EMPTY);
+        MockNode nodeA = createMockNode("nodeA").setAsMaster();
+        MockNode nodeB = createMockNode("nodeB");
 
         // Initial cluster state
         ClusterState clusterState = nodeA.clusterState;
@@ -268,7 +273,8 @@ public void testSimpleClusterStatePublishing() throws Exception {
 
         // cluster state update - add block
         previousClusterState = clusterState;
-        clusterState = ClusterState.builder(clusterState).blocks(ClusterBlocks.builder().addGlobalBlock(MetaData.CLUSTER_READ_ONLY_BLOCK)).incrementVersion().build();
+        clusterState = ClusterState.builder(clusterState).blocks(ClusterBlocks.builder()
+            .addGlobalBlock(MetaData.CLUSTER_READ_ONLY_BLOCK)).incrementVersion().build();
         publishStateAndWait(nodeA.action, clusterState, previousClusterState);
         assertSameStateFromDiff(nodeB.clusterState, clusterState);
         assertThat(nodeB.clusterState.blocks().global().size(), equalTo(1));
@@ -282,7 +288,7 @@ public void testSimpleClusterStatePublishing() throws Exception {
 
         // Adding new node - this node should get full cluster state while nodeB should still be getting diffs
 
-        MockNode nodeC = createMockNode("nodeC", Settings.EMPTY);
+        MockNode nodeC = createMockNode("nodeC");
 
         // cluster state update 3 - register node C
         previousClusterState = clusterState;
@@ -295,7 +301,8 @@ public void testSimpleClusterStatePublishing() throws Exception {
 
         // cluster state update 4 - update settings
         previousClusterState = clusterState;
-        MetaData metaData = MetaData.builder(clusterState.metaData()).transientSettings(Settings.builder().put("foo", "bar").build()).build();
+        MetaData metaData = MetaData.builder(clusterState.metaData())
+            .transientSettings(Settings.builder().put("foo", "bar").build()).build();
         clusterState = ClusterState.builder(clusterState).metaData(metaData).incrementVersion().build();
         publishStateAndWait(nodeA.action, clusterState, previousClusterState);
         assertSameStateFromDiff(nodeB.clusterState, clusterState);
@@ -336,9 +343,10 @@ public void testUnexpectedDiffPublishing() throws Exception {
             fail("Shouldn't send cluster state to myself");
         }).setAsMaster();
 
-        MockNode nodeB = createMockNode("nodeB", Settings.EMPTY);
+        MockNode nodeB = createMockNode("nodeB");
 
-        // Initial cluster state with both states - the second node still shouldn't get diff even though it's present in the previous cluster state
+        // Initial cluster state with both states - the second node still shouldn't
+        // get diff even though it's present in the previous cluster state
         DiscoveryNodes discoveryNodes = DiscoveryNodes.builder(nodeA.nodes()).add(nodeB.discoveryNode).build();
         ClusterState previousClusterState = ClusterState.builder(CLUSTER_NAME).nodes(discoveryNodes).build();
         ClusterState clusterState = ClusterState.builder(previousClusterState).incrementVersion().build();
@@ -347,7 +355,8 @@ public void testUnexpectedDiffPublishing() throws Exception {
 
         // cluster state update - add block
         previousClusterState = clusterState;
-        clusterState = ClusterState.builder(clusterState).blocks(ClusterBlocks.builder().addGlobalBlock(MetaData.CLUSTER_READ_ONLY_BLOCK)).incrementVersion().build();
+        clusterState = ClusterState.builder(clusterState).blocks(ClusterBlocks.builder()
+            .addGlobalBlock(MetaData.CLUSTER_READ_ONLY_BLOCK)).incrementVersion().build();
         publishStateAndWait(nodeA.action, clusterState, previousClusterState);
         assertSameStateFromDiff(nodeB.clusterState, clusterState);
     }
@@ -370,7 +379,8 @@ public void clusterChanged(ClusterChangedEvent event) {
         });
 
         // Initial cluster state
-        DiscoveryNodes discoveryNodes = DiscoveryNodes.builder().add(nodeA.discoveryNode).localNodeId(nodeA.discoveryNode.getId()).masterNodeId(nodeA.discoveryNode.getId()).build();
+        DiscoveryNodes discoveryNodes = DiscoveryNodes.builder()
+            .add(nodeA.discoveryNode).localNodeId(nodeA.discoveryNode.getId()).masterNodeId(nodeA.discoveryNode.getId()).build();
         ClusterState clusterState = ClusterState.builder(CLUSTER_NAME).nodes(discoveryNodes).build();
 
         // cluster state update - add nodeB
@@ -381,7 +391,8 @@ public void clusterChanged(ClusterChangedEvent event) {
 
         // cluster state update - add block
         previousClusterState = clusterState;
-        clusterState = ClusterState.builder(clusterState).blocks(ClusterBlocks.builder().addGlobalBlock(MetaData.CLUSTER_READ_ONLY_BLOCK)).incrementVersion().build();
+        clusterState = ClusterState.builder(clusterState).blocks(ClusterBlocks.builder()
+            .addGlobalBlock(MetaData.CLUSTER_READ_ONLY_BLOCK)).incrementVersion().build();
         publishStateAndWait(nodeA.action, clusterState, previousClusterState);
     }
 
@@ -444,9 +455,10 @@ public void clusterChanged(ClusterChangedEvent event) {
             }
         }).setAsMaster();
 
-        MockNode nodeB = createMockNode("nodeB", Settings.EMPTY);
+        MockNode nodeB = createMockNode("nodeB");
 
-        // Initial cluster state with both states - the second node still shouldn't get diff even though it's present in the previous cluster state
+        // Initial cluster state with both states - the second node still shouldn't get
+        // diff even though it's present in the previous cluster state
         DiscoveryNodes discoveryNodes = DiscoveryNodes.builder(nodeA.nodes()).add(nodeB.discoveryNode).build();
         ClusterState previousClusterState = ClusterState.builder(CLUSTER_NAME).nodes(discoveryNodes).build();
         ClusterState clusterState = ClusterState.builder(previousClusterState).incrementVersion().build();
@@ -455,7 +467,8 @@ public void clusterChanged(ClusterChangedEvent event) {
 
         // cluster state update - add block
         previousClusterState = clusterState;
-        clusterState = ClusterState.builder(clusterState).blocks(ClusterBlocks.builder().addGlobalBlock(MetaData.CLUSTER_READ_ONLY_BLOCK)).incrementVersion().build();
+        clusterState = ClusterState.builder(clusterState).blocks(ClusterBlocks.builder()
+            .addGlobalBlock(MetaData.CLUSTER_READ_ONLY_BLOCK)).incrementVersion().build();
 
         ClusterState unserializableClusterState = new ClusterState(clusterState.version(), clusterState.stateUUID(), clusterState) {
             @Override
@@ -495,7 +508,7 @@ public void testFailToPublishWithLessThanMinMasterNodes() throws Exception {
         final int dataNodes = randomIntBetween(0, 5);
         final Settings dataSettings = Settings.builder().put(Node.NODE_MASTER_SETTING.getKey(), false).build();
         for (int i = 0; i < dataNodes; i++) {
-            discoveryNodesBuilder.add(createMockNode("data_" + i, dataSettings).discoveryNode);
+            discoveryNodesBuilder.add(createMockNode("data_" + i, dataSettings, null).discoveryNode);
         }
         discoveryNodesBuilder.localNodeId(master.discoveryNode.getId()).masterNodeId(master.discoveryNode.getId());
         DiscoveryNodes discoveryNodes = discoveryNodesBuilder.build();
@@ -521,7 +534,7 @@ public void testPublishingWithSendingErrors() throws Exception {
         settings.put(DiscoverySettings.COMMIT_TIMEOUT_SETTING.getKey(), expectingToCommit == false && timeOutNodes > 0 ? "100ms" : "1h")
                 .put(DiscoverySettings.PUBLISH_TIMEOUT_SETTING.getKey(), "5ms"); // test is about committing
 
-        MockNode master = createMockNode("master", settings.build());
+        MockNode master = createMockNode("master", settings.build(), null);
 
         // randomize things a bit
         int[] nodeTypes = new int[goodNodes + errorNodes + timeOutNodes];
@@ -551,7 +564,8 @@ public void testPublishingWithSendingErrors() throws Exception {
         }
         final int dataNodes = randomIntBetween(0, 3); // data nodes don't matter
         for (int i = 0; i < dataNodes; i++) {
-            final MockNode mockNode = createMockNode("data_" + i, Settings.builder().put(Node.NODE_MASTER_SETTING.getKey(), false).build());
+            final MockNode mockNode = createMockNode("data_" + i,
+                Settings.builder().put(Node.NODE_MASTER_SETTING.getKey(), false).build(), null);
             discoveryNodesBuilder.add(mockNode.discoveryNode);
             if (randomBoolean()) {
                 // we really don't care - just chaos monkey
@@ -598,11 +612,12 @@ public void testIncomingClusterStateValidation() throws Exception {
 
         logger.info("--> testing acceptances of any master when having no master");
         ClusterState state = ClusterState.builder(node.clusterState)
-                .nodes(DiscoveryNodes.builder(node.nodes()).masterNodeId(randomAsciiOfLength(10))).incrementVersion().build();
+                .nodes(DiscoveryNodes.builder(node.nodes()).masterNodeId(randomAlphaOfLength(10))).incrementVersion().build();
         node.action.validateIncomingState(state, null);
 
         // now set a master node
-        node.clusterState = ClusterState.builder(node.clusterState).nodes(DiscoveryNodes.builder(node.nodes()).masterNodeId("master")).build();
+        node.clusterState = ClusterState.builder(node.clusterState)
+            .nodes(DiscoveryNodes.builder(node.nodes()).masterNodeId("master")).build();
         logger.info("--> testing rejection of another master");
         try {
             node.action.validateIncomingState(state, node.clusterState);
@@ -618,7 +633,8 @@ public void testIncomingClusterStateValidation() throws Exception {
 
         logger.info("--> testing rejection of another cluster name");
         try {
-            node.action.validateIncomingState(ClusterState.builder(new ClusterName(randomAsciiOfLength(10))).nodes(node.nodes()).build(), node.clusterState);
+            node.action.validateIncomingState(ClusterState.builder(new ClusterName(randomAlphaOfLength(10)))
+                .nodes(node.nodes()).build(), node.clusterState);
             fail("node accepted state with another cluster name");
         } catch (IllegalStateException OK) {
             assertThat(OK.toString(), containsString("received state from a node that is not part of the cluster"));
@@ -686,8 +702,8 @@ public void testOutOfOrderCommitMessages() throws Throwable {
         logger.info("--> publishing states");
         for (ClusterState state : states) {
             node.action.handleIncomingClusterStateRequest(
-                    new BytesTransportRequest(PublishClusterStateAction.serializeFullClusterState(state, Version.CURRENT), Version.CURRENT),
-                    channel);
+                new BytesTransportRequest(PublishClusterStateAction.serializeFullClusterState(state, Version.CURRENT), Version.CURRENT),
+                channel);
             assertThat(channel.response.get(), equalTo((TransportResponse) TransportResponse.Empty.INSTANCE));
             assertThat(channel.error.get(), nullValue());
             channel.clear();
@@ -724,12 +740,14 @@ public void testOutOfOrderCommitMessages() throws Throwable {
      */
     public void testTimeoutOrCommit() throws Exception {
         Settings settings = Settings.builder()
-                .put(DiscoverySettings.COMMIT_TIMEOUT_SETTING.getKey(), "1ms").build(); // short but so we will sometime commit sometime timeout
+            // short but so we will sometime commit sometime timeout
+            .put(DiscoverySettings.COMMIT_TIMEOUT_SETTING.getKey(), "1ms").build();
 
-        MockNode master = createMockNode("master", settings);
-        MockNode node = createMockNode("node", settings);
+        MockNode master = createMockNode("master", settings, null);
+        MockNode node = createMockNode("node", settings, null);
         ClusterState state = ClusterState.builder(master.clusterState)
-                .nodes(DiscoveryNodes.builder(master.clusterState.nodes()).add(node.discoveryNode).masterNodeId(master.discoveryNode.getId())).build();
+                .nodes(DiscoveryNodes.builder(master.clusterState.nodes())
+                    .add(node.discoveryNode).masterNodeId(master.discoveryNode.getId())).build();
 
         for (int i = 0; i < 10; i++) {
             state = ClusterState.builder(state).incrementVersion().build();
@@ -754,7 +772,8 @@ public void testTimeoutOrCommit() throws Exception {
 
     private MetaData buildMetaDataForVersion(MetaData metaData, long version) {
         ImmutableOpenMap.Builder<String, IndexMetaData> indices = ImmutableOpenMap.builder(metaData.indices());
-        indices.put("test" + version, IndexMetaData.builder("test" + version).settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT))
+        indices.put("test" + version, IndexMetaData.builder("test" + version)
+                .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT))
                 .numberOfShards((int) version).numberOfReplicas(0).build());
         return MetaData.builder(metaData)
                 .transientSettings(Settings.builder().put("test", version).build())
@@ -771,16 +790,19 @@ private void assertProperMetaDataForVersion(MetaData metaData, long version) {
         assertThat(metaData.transientSettings().get("test"), equalTo(Long.toString(version)));
     }
 
-    public void publishStateAndWait(PublishClusterStateAction action, ClusterState state, ClusterState previousState) throws InterruptedException {
+    public void publishStateAndWait(PublishClusterStateAction action, ClusterState state,
+                                    ClusterState previousState) throws InterruptedException {
         publishState(action, state, previousState).await(1, TimeUnit.SECONDS);
     }
 
-    public AssertingAckListener publishState(PublishClusterStateAction action, ClusterState state, ClusterState previousState) throws InterruptedException {
+    public AssertingAckListener publishState(PublishClusterStateAction action, ClusterState state,
+                                             ClusterState previousState) throws InterruptedException {
         final int minimumMasterNodes = randomIntBetween(-1, state.nodes().getMasterNodes().size());
         return publishState(action, state, previousState, minimumMasterNodes);
     }
 
-    public AssertingAckListener publishState(PublishClusterStateAction action, ClusterState state, ClusterState previousState, int minMasterNodes) throws InterruptedException {
+    public AssertingAckListener publishState(PublishClusterStateAction action, ClusterState state,
+                                             ClusterState previousState, int minMasterNodes) throws InterruptedException {
         AssertingAckListener assertingAckListener = new AssertingAckListener(state.nodes().getSize() - 1);
         ClusterChangedEvent changedEvent = new ClusterChangedEvent("test update", state, previousState);
         action.publish(changedEvent, minMasterNodes, assertingAckListener);
@@ -828,7 +850,8 @@ public List<Tuple<DiscoveryNode, Throwable>> awaitErrors(long timeout, TimeUnit
 
     void assertSameState(ClusterState actual, ClusterState expected) {
         assertThat(actual, notNullValue());
-        final String reason = "\n--> actual ClusterState: " + actual.prettyPrint() + "\n--> expected ClusterState:" + expected.prettyPrint();
+        final String reason = "\n--> actual ClusterState: " + actual + "\n" +
+                                "--> expected ClusterState:" + expected;
         assertThat("unequal UUIDs" + reason, actual.stateUUID(), equalTo(expected.stateUUID()));
         assertThat("unequal versions" + reason, actual.version(), equalTo(expected.version()));
     }
@@ -843,15 +866,17 @@ void assertSameStateFromFull(ClusterState actual, ClusterState expected) {
         assertFalse(actual.wasReadFromDiff());
     }
 
-    static class MockPublishAction extends PublishClusterStateAction {
+    public static class MockPublishAction extends PublishClusterStateAction {
 
         AtomicBoolean timeoutOnSend = new AtomicBoolean();
         AtomicBoolean errorOnSend = new AtomicBoolean();
         AtomicBoolean timeoutOnCommit = new AtomicBoolean();
         AtomicBoolean errorOnCommit = new AtomicBoolean();
 
-        public MockPublishAction(Settings settings, TransportService transportService, Supplier<ClusterState> clusterStateSupplier, NewPendingClusterStateListener listener, DiscoverySettings discoverySettings, ClusterName clusterName) {
-            super(settings, transportService, clusterStateSupplier, listener, discoverySettings, clusterName);
+        public MockPublishAction(Settings settings, TransportService transportService, NamedWriteableRegistry namedWriteableRegistry,
+                                 Supplier<ClusterState> clusterStateSupplier, NewPendingClusterStateListener listener,
+                                 DiscoverySettings discoverySettings, ClusterName clusterName) {
+            super(settings, transportService, namedWriteableRegistry, clusterStateSupplier, listener, discoverySettings, clusterName);
         }
 
         @Override
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/UnicastZenPingTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/UnicastZenPingTests.java
new file mode 100644
index 0000000000000..2ae14ca92e9bd
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/UnicastZenPingTests.java
@@ -0,0 +1,917 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.discovery.zen;
+
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.block.ClusterBlocks;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNode.Role;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.CheckedBiConsumer;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.network.NetworkAddress;
+import org.elasticsearch.common.network.NetworkService;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.BoundTransportAddress;
+import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
+import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.VersionUtils;
+import org.elasticsearch.test.transport.MockTransportService;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.ConnectTransportException;
+import org.elasticsearch.transport.ConnectionProfile;
+import org.elasticsearch.transport.MockTcpTransport;
+import org.elasticsearch.transport.Transport;
+import org.elasticsearch.transport.TransportConnectionListener;
+import org.elasticsearch.transport.TransportException;
+import org.elasticsearch.transport.TransportRequestOptions;
+import org.elasticsearch.transport.TransportResponseHandler;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.transport.TransportSettings;
+import org.junit.After;
+import org.junit.Before;
+import org.mockito.Matchers;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.Stack;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BiFunction;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.emptySet;
+import static org.elasticsearch.gateway.GatewayService.STATE_NOT_RECOVERED_BLOCK;
+import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+
+public class UnicastZenPingTests extends ESTestCase {
+
+    private ThreadPool threadPool;
+    private ExecutorService executorService;
+    // close in reverse order as opened
+    private Stack<Closeable> closeables;
+
+    @Before
+    public void setUp() throws Exception {
+        super.setUp();
+        threadPool = new TestThreadPool(getClass().getName());
+        final ThreadFactory threadFactory = EsExecutors.daemonThreadFactory("[" + getClass().getName() + "]");
+        executorService =
+            EsExecutors.newScaling(getClass().getName(), 0, 2, 60, TimeUnit.SECONDS, threadFactory, threadPool.getThreadContext());
+        closeables = new Stack<>();
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        try {
+            logger.info("shutting down...");
+            // JDK stack is broken, it does not iterate in the expected order (http://bugs.java.com/bugdatabase/view_bug.do?bug_id=4475301)
+            final List<Closeable> reverse = new ArrayList<>();
+            while (!closeables.isEmpty()) {
+                reverse.add(closeables.pop());
+            }
+            IOUtils.close(reverse);
+        } finally {
+            terminate(executorService);
+            terminate(threadPool);
+            super.tearDown();
+        }
+    }
+
+    private static final UnicastHostsProvider EMPTY_HOSTS_PROVIDER = Collections::emptyList;
+
+    public void testSimplePings() throws IOException, InterruptedException, ExecutionException {
+        // use ephemeral ports
+        final Settings settings = Settings.builder().put("cluster.name", "test").put(TransportSettings.PORT.getKey(), 0).build();
+        final Settings settingsMismatch =
+            Settings.builder().put(settings).put("cluster.name", "mismatch").put(TransportSettings.PORT.getKey(), 0).build();
+
+        NetworkService networkService = new NetworkService(settings, Collections.emptyList());
+
+        final BiFunction<Settings, Version, Transport> supplier = (s, v) -> new MockTcpTransport(
+            s,
+            threadPool,
+            BigArrays.NON_RECYCLING_INSTANCE,
+            new NoneCircuitBreakerService(),
+            new NamedWriteableRegistry(Collections.emptyList()),
+            networkService,
+            v) {
+            @Override
+            public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile,
+                                      CheckedBiConsumer<Connection, ConnectionProfile, IOException> connectionValidator)
+                throws ConnectTransportException {
+                throw new AssertionError("zen pings should never connect to node (got [" + node + "])");
+            }
+        };
+
+        NetworkHandle handleA = startServices(settings, threadPool, "UZP_A", Version.CURRENT, supplier);
+        closeables.push(handleA.transportService);
+        NetworkHandle handleB = startServices(settings, threadPool, "UZP_B", Version.CURRENT, supplier);
+        closeables.push(handleB.transportService);
+        NetworkHandle handleC = startServices(settingsMismatch, threadPool, "UZP_C", Version.CURRENT, supplier);
+        closeables.push(handleC.transportService);
+        final Version versionD;
+        if (randomBoolean()) {
+            versionD = VersionUtils.randomVersionBetween(random(), Version.CURRENT.minimumCompatibilityVersion(), Version.CURRENT);
+        } else {
+            versionD = Version.CURRENT;
+        }
+        logger.info("UZP_D version set to [{}]", versionD);
+        NetworkHandle handleD = startServices(settingsMismatch, threadPool, "UZP_D", versionD, supplier);
+        closeables.push(handleD.transportService);
+
+        final ClusterState state = ClusterState.builder(new ClusterName("test")).version(randomNonNegativeLong()).build();
+        final ClusterState stateMismatch = ClusterState.builder(new ClusterName("mismatch")).version(randomNonNegativeLong()).build();
+
+        Settings hostsSettings = Settings.builder()
+            .putArray("discovery.zen.ping.unicast.hosts",
+                NetworkAddress.format(new InetSocketAddress(handleA.address.address().getAddress(), handleA.address.address().getPort())),
+                NetworkAddress.format(new InetSocketAddress(handleB.address.address().getAddress(), handleB.address.address().getPort())),
+                NetworkAddress.format(new InetSocketAddress(handleC.address.address().getAddress(), handleC.address.address().getPort())),
+                NetworkAddress.format(new InetSocketAddress(handleD.address.address().getAddress(), handleD.address.address().getPort())))
+            .put("cluster.name", "test")
+            .build();
+
+        Settings hostsSettingsMismatch = Settings.builder().put(hostsSettings).put(settingsMismatch).build();
+        TestUnicastZenPing zenPingA = new TestUnicastZenPing(hostsSettings, threadPool, handleA, EMPTY_HOSTS_PROVIDER);
+        ClusterState stateA = ClusterState.builder(state)
+            .blocks(ClusterBlocks.builder().addGlobalBlock(STATE_NOT_RECOVERED_BLOCK))
+            .nodes(DiscoveryNodes.builder().add(handleA.node).localNodeId("UZP_A"))
+            .build();
+        zenPingA.start(() -> stateA);
+        closeables.push(zenPingA);
+
+        TestUnicastZenPing zenPingB = new TestUnicastZenPing(hostsSettings, threadPool, handleB, EMPTY_HOSTS_PROVIDER);
+        ClusterState stateB = ClusterState.builder(state)
+            .nodes(DiscoveryNodes.builder().add(handleB.node).localNodeId("UZP_B"))
+            .build();
+        zenPingB.start(() -> stateB);
+        closeables.push(zenPingB);
+
+        TestUnicastZenPing zenPingC = new TestUnicastZenPing(hostsSettingsMismatch, threadPool, handleC,
+            EMPTY_HOSTS_PROVIDER) {
+            @Override
+            protected Version getVersion() {
+                return versionD;
+            }
+        };
+        ClusterState stateC = ClusterState.builder(stateMismatch)
+            .nodes(DiscoveryNodes.builder().add(handleC.node).localNodeId("UZP_C"))
+            .build();
+        zenPingC.start(() -> stateC);
+        closeables.push(zenPingC);
+
+        TestUnicastZenPing zenPingD = new TestUnicastZenPing(hostsSettingsMismatch, threadPool, handleD,
+            EMPTY_HOSTS_PROVIDER);
+        ClusterState stateD = ClusterState.builder(stateMismatch)
+            .nodes(DiscoveryNodes.builder().add(handleD.node).localNodeId("UZP_D"))
+            .build();
+        zenPingD.start(() -> stateD);
+        closeables.push(zenPingD);
+
+        logger.info("ping from UZP_A");
+        Collection<ZenPing.PingResponse> pingResponses = zenPingA.pingAndWait().toList();
+        assertThat(pingResponses.size(), equalTo(1));
+        ZenPing.PingResponse ping = pingResponses.iterator().next();
+        assertThat(ping.node().getId(), equalTo("UZP_B"));
+        assertThat(ping.getClusterStateVersion(), equalTo(state.version()));
+        assertPingCount(handleA, handleB, 3);
+        assertPingCount(handleA, handleC, 0); // mismatch, shouldn't ping
+        assertPingCount(handleA, handleD, 0); // mismatch, shouldn't ping
+
+        // ping again, this time from B,
+        logger.info("ping from UZP_B");
+        pingResponses = zenPingB.pingAndWait().toList();
+        assertThat(pingResponses.size(), equalTo(1));
+        ping = pingResponses.iterator().next();
+        assertThat(ping.node().getId(), equalTo("UZP_A"));
+        assertThat(ping.getClusterStateVersion(), equalTo(ElectMasterService.MasterCandidate.UNRECOVERED_CLUSTER_VERSION));
+        assertPingCount(handleB, handleA, 3);
+        assertPingCount(handleB, handleC, 0); // mismatch, shouldn't ping
+        assertPingCount(handleB, handleD, 0); // mismatch, shouldn't ping
+
+        logger.info("ping from UZP_C");
+        pingResponses = zenPingC.pingAndWait().toList();
+        assertThat(pingResponses.size(), equalTo(1));
+        assertPingCount(handleC, handleA, 0);
+        assertPingCount(handleC, handleB, 0);
+        assertPingCount(handleC, handleD, 3);
+
+        logger.info("ping from UZP_D");
+        pingResponses = zenPingD.pingAndWait().toList();
+        assertThat(pingResponses.size(), equalTo(1));
+        assertPingCount(handleD, handleA, 0);
+        assertPingCount(handleD, handleB, 0);
+        assertPingCount(handleD, handleC, 3);
+    }
+
+    public void testUnknownHostNotCached() throws ExecutionException, InterruptedException {
+        // use ephemeral ports
+        final Settings settings = Settings.builder().put("cluster.name", "test").put(TransportSettings.PORT.getKey(), 0).build();
+
+        final NetworkService networkService = new NetworkService(settings, Collections.emptyList());
+
+        final Map<String, TransportAddress[]> addresses = new HashMap<>();
+        final BiFunction<Settings, Version, Transport> supplier = (s, v) -> new MockTcpTransport(
+            s,
+            threadPool,
+            BigArrays.NON_RECYCLING_INSTANCE,
+            new NoneCircuitBreakerService(),
+            new NamedWriteableRegistry(Collections.emptyList()),
+            networkService,
+            v) {
+            @Override
+            public TransportAddress[] addressesFromString(String address, int perAddressLimit) throws UnknownHostException {
+                final TransportAddress[] transportAddresses = addresses.get(address);
+                if (transportAddresses == null) {
+                    throw new UnknownHostException(address);
+                } else {
+                    return transportAddresses;
+                }
+            }
+        };
+
+        final NetworkHandle handleA = startServices(settings, threadPool, "UZP_A", Version.CURRENT, supplier);
+        closeables.push(handleA.transportService);
+        final NetworkHandle handleB = startServices(settings, threadPool, "UZP_B", Version.CURRENT, supplier);
+        closeables.push(handleB.transportService);
+        final NetworkHandle handleC = startServices(settings, threadPool, "UZP_C", Version.CURRENT, supplier);
+        closeables.push(handleC.transportService);
+
+        addresses.put(
+            "UZP_A",
+            new TransportAddress[]{
+                new InetSocketTransportAddress(
+                    new InetSocketAddress(handleA.address.address().getAddress(), handleA.address.address().getPort()))});
+        addresses.put(
+            "UZP_C",
+            new TransportAddress[]{
+                new InetSocketTransportAddress(
+                    new InetSocketAddress(handleC.address.address().getAddress(), handleC.address.address().getPort()))});
+
+        final Settings hostsSettings = Settings.builder()
+            .putArray("discovery.zen.ping.unicast.hosts", "UZP_A", "UZP_B", "UZP_C")
+            .put("cluster.name", "test")
+            .build();
+
+        final ClusterState state = ClusterState.builder(new ClusterName("test")).version(randomNonNegativeLong()).build();
+
+        final TestUnicastZenPing zenPingA = new TestUnicastZenPing(hostsSettings, threadPool, handleA, EMPTY_HOSTS_PROVIDER);
+        ClusterState stateA = ClusterState.builder(state)
+            .blocks(ClusterBlocks.builder().addGlobalBlock(STATE_NOT_RECOVERED_BLOCK))
+            .nodes(DiscoveryNodes.builder().add(handleA.node).localNodeId("UZP_A"))
+            .build();
+        zenPingA.start(() -> stateA);
+        closeables.push(zenPingA);
+
+        TestUnicastZenPing zenPingB = new TestUnicastZenPing(hostsSettings, threadPool, handleB, EMPTY_HOSTS_PROVIDER);
+        ClusterState stateB = ClusterState.builder(state)
+            .nodes(DiscoveryNodes.builder().add(handleB.node).localNodeId("UZP_B"))
+            .build();
+        zenPingB.start(() -> stateB);
+        closeables.push(zenPingB);
+
+        TestUnicastZenPing zenPingC = new TestUnicastZenPing(hostsSettings, threadPool, handleC, EMPTY_HOSTS_PROVIDER);
+        ClusterState stateC = ClusterState.builder(state)
+            .nodes(DiscoveryNodes.builder().add(handleC.node).localNodeId("UZP_C"))
+            .build();
+        zenPingC.start(() -> stateC);
+        closeables.push(zenPingC);
+
+        // the presence of an unresolvable host should not prevent resolvable hosts from being pinged
+        {
+            final Collection<ZenPing.PingResponse> pingResponses = zenPingA.pingAndWait().toList();
+            assertThat(pingResponses.size(), equalTo(1));
+            ZenPing.PingResponse ping = pingResponses.iterator().next();
+            assertThat(ping.node().getId(), equalTo("UZP_C"));
+            assertThat(ping.getClusterStateVersion(), equalTo(state.version()));
+            assertPingCount(handleA, handleB, 0);
+            assertPingCount(handleA, handleC, 3);
+            assertNull(handleA.counters.get(handleB.address));
+        }
+
+        final HashMap<TransportAddress, Integer> moreThan = new HashMap<>();
+        // we should see at least one ping to UZP_B, and one more ping than we have already seen to UZP_C
+        moreThan.put(handleB.address, 0);
+        moreThan.put(handleC.address, handleA.counters.get(handleC.address).intValue());
+
+        // now allow UZP_B to be resolvable
+        addresses.put(
+            "UZP_B",
+            new TransportAddress[]{
+                new InetSocketTransportAddress(
+                    new InetSocketAddress(handleB.address.address().getAddress(), handleB.address.address().getPort()))});
+
+        // now we should see pings to UZP_B; this establishes that host resolutions are not cached
+        {
+            handleA.counters.clear();
+            final Collection<ZenPing.PingResponse> secondPingResponses = zenPingA.pingAndWait().toList();
+            assertThat(secondPingResponses.size(), equalTo(2));
+            final Set<String> ids = new HashSet<>(secondPingResponses.stream().map(p -> p.node().getId()).collect(Collectors.toList()));
+            assertThat(ids, equalTo(new HashSet<>(Arrays.asList("UZP_B", "UZP_C"))));
+            assertPingCount(handleA, handleB, 3);
+            assertPingCount(handleA, handleC, 3);
+        }
+    }
+
+    public void testPortLimit() throws InterruptedException {
+        final NetworkService networkService = new NetworkService(Settings.EMPTY, Collections.emptyList());
+        final Transport transport = new MockTcpTransport(
+            Settings.EMPTY,
+            threadPool,
+            BigArrays.NON_RECYCLING_INSTANCE,
+            new NoneCircuitBreakerService(),
+            new NamedWriteableRegistry(Collections.emptyList()),
+            networkService,
+            Version.CURRENT) {
+
+            @Override
+            public BoundTransportAddress boundAddress() {
+                return new BoundTransportAddress(
+                    new TransportAddress[]{new InetSocketTransportAddress(InetAddress.getLoopbackAddress(), 9500)},
+                    new InetSocketTransportAddress(InetAddress.getLoopbackAddress(), 9500)
+                );
+            }
+        };
+        closeables.push(transport);
+        final TransportService transportService =
+            new TransportService(Settings.EMPTY, transport, threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> null, null);
+        closeables.push(transportService);
+        final int limitPortCounts = randomIntBetween(1, 10);
+        final List<DiscoveryNode> discoveryNodes = TestUnicastZenPing.resolveHostsLists(
+            executorService,
+            logger,
+            Collections.singletonList("127.0.0.1"),
+            limitPortCounts,
+            transportService,
+            "test_",
+            TimeValue.timeValueSeconds(1));
+        assertThat(discoveryNodes, hasSize(limitPortCounts));
+        final Set<Integer> ports = new HashSet<>();
+        for (final DiscoveryNode discoveryNode : discoveryNodes) {
+            assertTrue(discoveryNode.getAddress().isLoopbackOrLinkLocalAddress());
+            ports.add(discoveryNode.getAddress().getPort());
+        }
+        assertThat(ports, equalTo(IntStream.range(9300, 9300 + limitPortCounts).mapToObj(m -> m).collect(Collectors.toSet())));
+    }
+
+    public void testRemovingLocalAddresses() throws InterruptedException {
+        final NetworkService networkService = new NetworkService(Settings.EMPTY, Collections.emptyList());
+        final InetAddress loopbackAddress = InetAddress.getLoopbackAddress();
+        final Transport transport = new MockTcpTransport(
+            Settings.EMPTY,
+            threadPool,
+            BigArrays.NON_RECYCLING_INSTANCE,
+            new NoneCircuitBreakerService(),
+            new NamedWriteableRegistry(Collections.emptyList()),
+            networkService,
+            Version.CURRENT) {
+
+            @Override
+            public BoundTransportAddress boundAddress() {
+                return new BoundTransportAddress(
+                    new TransportAddress[]{
+                        new InetSocketTransportAddress(loopbackAddress, 9300),
+                        new InetSocketTransportAddress(loopbackAddress, 9301)
+                    },
+                    new InetSocketTransportAddress(loopbackAddress, 9302)
+                );
+            }
+        };
+        closeables.push(transport);
+        final TransportService transportService =
+            new TransportService(Settings.EMPTY, transport, threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> null, null);
+        closeables.push(transportService);
+        final List<DiscoveryNode> discoveryNodes = TestUnicastZenPing.resolveHostsLists(
+            executorService,
+            logger,
+            Collections.singletonList(NetworkAddress.format(loopbackAddress)),
+            10,
+            transportService,
+            "test_",
+            TimeValue.timeValueSeconds(1));
+        assertThat(discoveryNodes, hasSize(7));
+        final Set<Integer> ports = new HashSet<>();
+        for (final DiscoveryNode discoveryNode : discoveryNodes) {
+            assertTrue(discoveryNode.getAddress().isLoopbackOrLinkLocalAddress());
+            ports.add(discoveryNode.getAddress().getPort());
+        }
+        assertThat(ports, equalTo(IntStream.range(9303, 9310).mapToObj(m -> m).collect(Collectors.toSet())));
+    }
+
+    public void testUnknownHost() throws InterruptedException {
+        final Logger logger = mock(Logger.class);
+        final NetworkService networkService = new NetworkService(Settings.EMPTY, Collections.emptyList());
+        final String hostname = randomAlphaOfLength(8);
+        final UnknownHostException unknownHostException = new UnknownHostException(hostname);
+        final Transport transport = new MockTcpTransport(
+            Settings.EMPTY,
+            threadPool,
+            BigArrays.NON_RECYCLING_INSTANCE,
+            new NoneCircuitBreakerService(),
+            new NamedWriteableRegistry(Collections.emptyList()),
+            networkService,
+            Version.CURRENT) {
+
+            @Override
+            public BoundTransportAddress boundAddress() {
+                return new BoundTransportAddress(
+                    new TransportAddress[]{new InetSocketTransportAddress(InetAddress.getLoopbackAddress(), 9300)},
+                    new InetSocketTransportAddress(InetAddress.getLoopbackAddress(), 9300)
+                );
+            }
+
+            @Override
+            public TransportAddress[] addressesFromString(String address, int perAddressLimit) throws UnknownHostException {
+                throw unknownHostException;
+            }
+
+        };
+        closeables.push(transport);
+
+        final TransportService transportService =
+            new TransportService(Settings.EMPTY, transport, threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> null, null);
+        closeables.push(transportService);
+
+        final List<DiscoveryNode> discoveryNodes = TestUnicastZenPing.resolveHostsLists(
+            executorService,
+            logger,
+            Arrays.asList(hostname),
+            1,
+            transportService,
+            "test_",
+            TimeValue.timeValueSeconds(1)
+        );
+
+        assertThat(discoveryNodes, empty());
+        verify(logger).warn("failed to resolve host [" + hostname + "]", unknownHostException);
+    }
+
+    public void testResolveTimeout() throws InterruptedException {
+        final Logger logger = mock(Logger.class);
+        final NetworkService networkService = new NetworkService(Settings.EMPTY, Collections.emptyList());
+        final CountDownLatch latch = new CountDownLatch(1);
+        final Transport transport = new MockTcpTransport(
+            Settings.EMPTY,
+            threadPool,
+            BigArrays.NON_RECYCLING_INSTANCE,
+            new NoneCircuitBreakerService(),
+            new NamedWriteableRegistry(Collections.emptyList()),
+            networkService,
+            Version.CURRENT) {
+
+            @Override
+            public BoundTransportAddress boundAddress() {
+                return new BoundTransportAddress(
+                    new TransportAddress[]{new InetSocketTransportAddress(InetAddress.getLoopbackAddress(), 9500)},
+                    new InetSocketTransportAddress(InetAddress.getLoopbackAddress(), 9500)
+                );
+            }
+
+            @Override
+            public TransportAddress[] addressesFromString(String address, int perAddressLimit) throws UnknownHostException {
+                if ("hostname1".equals(address)) {
+                    return new TransportAddress[]{new InetSocketTransportAddress(InetAddress.getLoopbackAddress(), 9300)};
+                } else if ("hostname2".equals(address)) {
+                    try {
+                        latch.await();
+                        return new TransportAddress[]{new InetSocketTransportAddress(InetAddress.getLoopbackAddress(), 9300)};
+                    } catch (InterruptedException e) {
+                        throw new RuntimeException(e);
+                    }
+                } else {
+                    throw new UnknownHostException(address);
+                }
+            }
+
+        };
+        closeables.push(transport);
+
+        final TransportService transportService =
+            new TransportService(Settings.EMPTY, transport, threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> null, null);
+        closeables.push(transportService);
+        final TimeValue resolveTimeout = TimeValue.timeValueSeconds(randomIntBetween(1, 3));
+        try {
+            final List<DiscoveryNode> discoveryNodes = TestUnicastZenPing.resolveHostsLists(
+                executorService,
+                logger,
+                Arrays.asList("hostname1", "hostname2"),
+                1,
+                transportService,
+                "test+",
+                resolveTimeout);
+
+            assertThat(discoveryNodes, hasSize(1));
+            verify(logger).trace(
+                "resolved host [{}] to {}", "hostname1",
+                new TransportAddress[]{new InetSocketTransportAddress(InetAddress.getLoopbackAddress(), 9300)});
+            verify(logger).warn("timed out after [{}] resolving host [{}]", resolveTimeout, "hostname2");
+            verifyNoMoreInteractions(logger);
+        } finally {
+            latch.countDown();
+        }
+    }
+
+    public void testResolveReuseExistingNodeConnections() throws ExecutionException, InterruptedException {
+        final Settings settings = Settings.builder().put("cluster.name", "test").put(TransportSettings.PORT.getKey(), 0).build();
+
+        NetworkService networkService = new NetworkService(settings, Collections.emptyList());
+
+        final BiFunction<Settings, Version, Transport> supplier = (s, v) -> new MockTcpTransport(
+            s,
+            threadPool,
+            BigArrays.NON_RECYCLING_INSTANCE,
+            new NoneCircuitBreakerService(),
+            new NamedWriteableRegistry(Collections.emptyList()),
+            networkService,
+            v);
+
+        NetworkHandle handleA = startServices(settings, threadPool, "UZP_A", Version.CURRENT, supplier, EnumSet.allOf(Role.class));
+        closeables.push(handleA.transportService);
+        NetworkHandle handleB = startServices(settings, threadPool, "UZP_B", Version.CURRENT, supplier, EnumSet.allOf(Role.class));
+        closeables.push(handleB.transportService);
+
+        final boolean useHosts = randomBoolean();
+        final Settings.Builder hostsSettingsBuilder = Settings.builder().put("cluster.name", "test");
+        if (useHosts) {
+            hostsSettingsBuilder.putArray("discovery.zen.ping.unicast.hosts",
+                NetworkAddress.format(new InetSocketAddress(handleB.address.address().getAddress(), handleB.address.address().getPort()))
+            );
+        } else {
+            hostsSettingsBuilder.put("discovery.zen.ping.unicast.hosts", (String) null);
+        }
+        final Settings hostsSettings = hostsSettingsBuilder.build();
+        final ClusterState state = ClusterState.builder(new ClusterName("test")).version(randomNonNegativeLong()).build();
+
+        // connection to reuse
+        handleA.transportService.connectToNode(handleB.node);
+
+        // install a listener to check that no new connections are made
+        handleA.transportService.addConnectionListener(new TransportConnectionListener() {
+            @Override
+            public void onConnectionOpened(Transport.Connection connection) {
+                fail("should not open any connections. got [" + connection.getNode() + "]");
+            }
+        });
+
+        final TestUnicastZenPing zenPingA = new TestUnicastZenPing(hostsSettings, threadPool, handleA, EMPTY_HOSTS_PROVIDER);
+        final ClusterState stateA = ClusterState.builder(state)
+            .blocks(ClusterBlocks.builder().addGlobalBlock(STATE_NOT_RECOVERED_BLOCK))
+            .nodes(DiscoveryNodes.builder().add(handleA.node).add(handleB.node).localNodeId("UZP_A"))
+            .build();
+        zenPingA.start(() -> stateA);
+        closeables.push(zenPingA);
+
+        TestUnicastZenPing zenPingB = new TestUnicastZenPing(hostsSettings, threadPool, handleB, EMPTY_HOSTS_PROVIDER);
+        final ClusterState stateB = ClusterState.builder(state)
+            .nodes(DiscoveryNodes.builder().add(handleB.node).localNodeId("UZP_B"))
+            .build();
+        zenPingB.start(() -> stateB);
+        closeables.push(zenPingB);
+
+        Collection<ZenPing.PingResponse> pingResponses = zenPingA.pingAndWait().toList();
+        assertThat(pingResponses.size(), equalTo(1));
+        ZenPing.PingResponse ping = pingResponses.iterator().next();
+        assertThat(ping.node().getId(), equalTo("UZP_B"));
+        assertThat(ping.getClusterStateVersion(), equalTo(state.version()));
+
+    }
+
+    public void testPingingTemporalPings() throws ExecutionException, InterruptedException {
+        final Settings settings = Settings.builder().put("cluster.name", "test").put(TransportSettings.PORT.getKey(), 0).build();
+
+        NetworkService networkService = new NetworkService(settings, Collections.emptyList());
+
+        final BiFunction<Settings, Version, Transport> supplier = (s, v) -> new MockTcpTransport(
+            s,
+            threadPool,
+            BigArrays.NON_RECYCLING_INSTANCE,
+            new NoneCircuitBreakerService(),
+            new NamedWriteableRegistry(Collections.emptyList()),
+            networkService,
+            v);
+
+        NetworkHandle handleA = startServices(settings, threadPool, "UZP_A", Version.CURRENT, supplier, EnumSet.allOf(Role.class));
+        closeables.push(handleA.transportService);
+        NetworkHandle handleB = startServices(settings, threadPool, "UZP_B", Version.CURRENT, supplier, EnumSet.allOf(Role.class));
+        closeables.push(handleB.transportService);
+
+        final Settings hostsSettings = Settings.builder()
+            .put("cluster.name", "test")
+            .put("discovery.zen.ping.unicast.hosts", (String) null) // use nodes for simplicity
+            .build();
+        final ClusterState state = ClusterState.builder(new ClusterName("test")).version(randomNonNegativeLong()).build();
+        final ClusterState stateA = ClusterState.builder(state)
+            .blocks(ClusterBlocks.builder().addGlobalBlock(STATE_NOT_RECOVERED_BLOCK))
+            .nodes(DiscoveryNodes.builder().add(handleA.node).add(handleB.node).localNodeId("UZP_A")).build();
+
+        final TestUnicastZenPing zenPingA = new TestUnicastZenPing(hostsSettings, threadPool, handleA, EMPTY_HOSTS_PROVIDER);
+        zenPingA.start(() -> stateA);
+        closeables.push(zenPingA);
+
+        // Node B doesn't know about A!
+        final ClusterState stateB = ClusterState.builder(state).nodes(
+            DiscoveryNodes.builder().add(handleB.node).localNodeId("UZP_B")).build();
+        TestUnicastZenPing zenPingB = new TestUnicastZenPing(hostsSettings, threadPool, handleB, EMPTY_HOSTS_PROVIDER);
+        zenPingB.start(() -> stateB);
+        closeables.push(zenPingB);
+
+        {
+            logger.info("pinging from UZP_A so UZP_B will learn about it");
+            Collection<ZenPing.PingResponse> pingResponses = zenPingA.pingAndWait().toList();
+            assertThat(pingResponses.size(), equalTo(1));
+            ZenPing.PingResponse ping = pingResponses.iterator().next();
+            assertThat(ping.node().getId(), equalTo("UZP_B"));
+            assertThat(ping.getClusterStateVersion(), equalTo(state.version()));
+        }
+        {
+            logger.info("pinging from UZP_B");
+            Collection<ZenPing.PingResponse> pingResponses = zenPingB.pingAndWait().toList();
+            assertThat(pingResponses.size(), equalTo(1));
+            ZenPing.PingResponse ping = pingResponses.iterator().next();
+            assertThat(ping.node().getId(), equalTo("UZP_A"));
+            assertThat(ping.getClusterStateVersion(), equalTo(-1L)); // A has a block
+        }
+    }
+
+    public void testInvalidHosts() throws InterruptedException {
+        final Logger logger = mock(Logger.class);
+        final NetworkService networkService = new NetworkService(Settings.EMPTY, Collections.emptyList());
+        final Transport transport = new MockTcpTransport(
+            Settings.EMPTY,
+            threadPool,
+            BigArrays.NON_RECYCLING_INSTANCE,
+            new NoneCircuitBreakerService(),
+            new NamedWriteableRegistry(Collections.emptyList()),
+            networkService,
+            Version.CURRENT) {
+            @Override
+            public BoundTransportAddress boundAddress() {
+                return new BoundTransportAddress(
+                    new TransportAddress[]{new InetSocketTransportAddress(InetAddress.getLoopbackAddress(), 9300)},
+                    new InetSocketTransportAddress(InetAddress.getLoopbackAddress(), 9300)
+                );
+            }
+        };
+        closeables.push(transport);
+
+        final TransportService transportService =
+            new TransportService(Settings.EMPTY, transport, threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> null, null);
+        closeables.push(transportService);
+        final List<DiscoveryNode> discoveryNodes = TestUnicastZenPing.resolveHostsLists(
+            executorService,
+            logger,
+            Arrays.asList("127.0.0.1:9300:9300", "127.0.0.1:9301"),
+            1,
+            transportService,
+            "test_",
+            TimeValue.timeValueSeconds(1));
+        assertThat(discoveryNodes, hasSize(1)); // only one of the two is valid and will be used
+        assertThat(discoveryNodes.get(0).getAddress().getAddress(), equalTo("127.0.0.1"));
+        assertThat(discoveryNodes.get(0).getAddress().getPort(), equalTo(9301));
+        verify(logger).warn(eq("failed to resolve host [127.0.0.1:9300:9300]"), Matchers.any(ExecutionException.class));
+    }
+
+    private void assertPingCount(final NetworkHandle fromNode, final NetworkHandle toNode, int expectedCount) {
+        final AtomicInteger counter = fromNode.counters.getOrDefault(toNode.address, new AtomicInteger());
+        final String onNodeName = fromNode.node.getName();
+        assertNotNull("handle for [" + onNodeName + "] has no 'expected' counter", counter);
+        final String forNodeName = toNode.node.getName();
+        assertThat("node [" + onNodeName + "] ping count to [" + forNodeName + "] is unexpected",
+            counter.get(), equalTo(expectedCount));
+    }
+
+    private NetworkHandle startServices(
+        final Settings settings,
+        final ThreadPool threadPool,
+        final String nodeId,
+        final Version version,
+        final BiFunction<Settings, Version, Transport> supplier) {
+        return startServices(settings, threadPool, nodeId, version, supplier, emptySet());
+
+    }
+
+    private NetworkHandle startServices(
+        final Settings settings,
+        final ThreadPool threadPool,
+        final String nodeId,
+        final Version version,
+        final BiFunction<Settings, Version, Transport> supplier,
+        final Set<Role> nodeRoles) {
+        final Settings nodeSettings = Settings.builder().put(settings)
+            .put("node.name", nodeId)
+            .put(TransportService.TRACE_LOG_INCLUDE_SETTING.getKey(), "internal:discovery/zen/unicast")
+            .build();
+        final Transport transport = supplier.apply(nodeSettings, version);
+        final MockTransportService transportService =
+            new MockTransportService(nodeSettings, transport, threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR,  boundAddress ->
+                new DiscoveryNode(nodeId, nodeId, boundAddress.publishAddress(), emptyMap(), nodeRoles, version), null);
+        transportService.start();
+        transportService.acceptIncomingRequests();
+        final ConcurrentMap<TransportAddress, AtomicInteger> counters = ConcurrentCollections.newConcurrentMap();
+        transportService.addTracer(new MockTransportService.Tracer() {
+            @Override
+            public void requestSent(DiscoveryNode node, long requestId, String action, TransportRequestOptions options) {
+                counters.computeIfAbsent(node.getAddress(), k -> new AtomicInteger());
+                counters.get(node.getAddress()).incrementAndGet();
+            }
+        });
+        return new NetworkHandle((InetSocketTransportAddress)transport.boundAddress().publishAddress(), transportService,
+            transportService.getLocalNode(), counters);
+    }
+
+    private static class NetworkHandle {
+
+        public final InetSocketTransportAddress address;
+        public final TransportService transportService;
+        public final DiscoveryNode node;
+        public final ConcurrentMap<TransportAddress, AtomicInteger> counters;
+
+        NetworkHandle(
+            final InetSocketTransportAddress address,
+            final TransportService transportService,
+            final DiscoveryNode discoveryNode,
+            final ConcurrentMap<TransportAddress, AtomicInteger> counters) {
+            this.address = address;
+            this.transportService = transportService;
+            this.node = discoveryNode;
+            this.counters = counters;
+        }
+    }
+
+    private static class TestUnicastZenPing extends UnicastZenPing {
+
+        TestUnicastZenPing(Settings settings, ThreadPool threadPool, NetworkHandle networkHandle,
+                                  UnicastHostsProvider unicastHostsProvider) {
+            super(Settings.builder().put("node.name", networkHandle.node.getName()).put(settings).build(),
+                threadPool, networkHandle.transportService, unicastHostsProvider);
+        }
+
+        volatile CountDownLatch allTasksCompleted;
+        volatile AtomicInteger pendingTasks;
+        volatile CountDownLatch pingingRoundClosed;
+
+        PingCollection pingAndWait() throws ExecutionException, InterruptedException {
+            allTasksCompleted = new CountDownLatch(1);
+            pingingRoundClosed = new CountDownLatch(1);
+            pendingTasks = new AtomicInteger();
+            // mark the three sending rounds as ongoing
+            markTaskAsStarted("send pings");
+            markTaskAsStarted("send pings");
+            markTaskAsStarted("send pings");
+            final AtomicReference<PingCollection> response = new AtomicReference<>();
+            ping(response::set, TimeValue.timeValueMillis(1), TimeValue.timeValueSeconds(1));
+            pingingRoundClosed.await();
+            final PingCollection result = response.get();
+            assertNotNull("pinging didn't complete",  result);
+            return result;
+        }
+
+        @Override
+        protected void finishPingingRound(PingingRound pingingRound) {
+            // wait for all activity to finish before closing
+            try {
+                allTasksCompleted.await();
+            } catch (InterruptedException e) {
+                // ok, finish anyway
+            }
+            super.finishPingingRound(pingingRound);
+            pingingRoundClosed.countDown();
+        }
+
+        @Override
+        protected void sendPings(TimeValue timeout, PingingRound pingingRound) {
+            super.sendPings(timeout, pingingRound);
+            markTaskAsCompleted("send pings");
+        }
+
+        @Override
+        protected void submitToExecutor(AbstractRunnable abstractRunnable) {
+            markTaskAsStarted("executor runnable");
+            super.submitToExecutor(new AbstractRunnable() {
+                @Override
+                public void onRejection(Exception e) {
+                    try {
+                        super.onRejection(e);
+                    } finally {
+                        markTaskAsCompleted("executor runnable (rejected)");
+                    }
+                }
+
+                @Override
+                public void onAfter() {
+                    markTaskAsCompleted("executor runnable");
+                }
+
+                @Override
+                protected void doRun() throws Exception {
+                    abstractRunnable.run();
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    // we shouldn't really end up here.
+                    throw new AssertionError("unexpected error", e);
+                }
+            });
+        }
+
+        private void markTaskAsStarted(String task) {
+            logger.trace("task [{}] started. count [{}]", task, pendingTasks.incrementAndGet());
+        }
+
+        private void markTaskAsCompleted(String task) {
+            final int left = pendingTasks.decrementAndGet();
+            logger.trace("task [{}] completed. count [{}]", task, left);
+            if (left == 0) {
+                allTasksCompleted.countDown();
+            }
+        }
+
+        @Override
+        protected TransportResponseHandler<UnicastPingResponse> getPingResponseHandler(PingingRound pingingRound, DiscoveryNode node) {
+            markTaskAsStarted("ping [" + node + "]");
+            TransportResponseHandler<UnicastPingResponse> original = super.getPingResponseHandler(pingingRound, node);
+            return new TransportResponseHandler<UnicastPingResponse>() {
+                @Override
+                public UnicastPingResponse newInstance() {
+                    return original.newInstance();
+                }
+
+                @Override
+                public void handleResponse(UnicastPingResponse response) {
+                    original.handleResponse(response);
+                    markTaskAsCompleted("ping [" + node + "]");
+                }
+
+                @Override
+                public void handleException(TransportException exp) {
+                    original.handleException(exp);
+                    markTaskAsCompleted("ping [" + node + "] (error)");
+                }
+
+                @Override
+                public String executor() {
+                    return original.executor();
+                }
+            };
+        }
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java b/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java
index b7aaf2795824f..9968ce0a824ef 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java
@@ -24,9 +24,7 @@
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
 import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
-import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -34,21 +32,15 @@
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.InetSocketTransportAddress;
 import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.discovery.Discovery;
 import org.elasticsearch.discovery.DiscoveryStats;
-import org.elasticsearch.discovery.zen.elect.ElectMasterService;
-import org.elasticsearch.discovery.zen.fd.FaultDetection;
-import org.elasticsearch.discovery.zen.membership.MembershipAction;
-import org.elasticsearch.discovery.zen.publish.PublishClusterStateAction;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.TestCustomMetaData;
-import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.BytesTransportRequest;
@@ -57,66 +49,47 @@
 import org.elasticsearch.transport.TransportResponse;
 import org.elasticsearch.transport.TransportService;
 import org.hamcrest.Matchers;
-import org.junit.Before;
 
 import java.io.IOException;
-import java.net.InetAddress;
 import java.net.UnknownHostException;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.EnumSet;
 import java.util.List;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.either;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
-import static org.hamcrest.Matchers.nullValue;
-import static org.hamcrest.Matchers.sameInstance;
 
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0)
 @ESIntegTestCase.SuppressLocalMode
 @TestLogging("_root:DEBUG")
 public class ZenDiscoveryIT extends ESIntegTestCase {
 
-    private Version previousMajorVersion;
-
-    @Before
-    public void computePrevMajorVersion() {
-        Version previousMajor;
-        // find a GA build whose major version is <N
-        do {
-            previousMajor = VersionUtils.randomVersion(random());
-        } while (previousMajor.onOrAfter(Version.CURRENT.minimumCompatibilityVersion())
-                || previousMajor.isAlpha()
-                || previousMajor.isBeta()
-                || previousMajor.isRC());
-        previousMajorVersion = previousMajor;
-    }
-
     public void testNoShardRelocationsOccurWhenElectedMasterNodeFails() throws Exception {
         Settings defaultSettings = Settings.builder()
                 .put(FaultDetection.PING_TIMEOUT_SETTING.getKey(), "1s")
                 .put(FaultDetection.PING_RETRIES_SETTING.getKey(), "1")
-                .put("discovery.type", "zen")
                 .build();
 
         Settings masterNodeSettings = Settings.builder()
                 .put(Node.NODE_DATA_SETTING.getKey(), false)
                 .put(defaultSettings)
                 .build();
-        internalCluster().startNodesAsync(2, masterNodeSettings).get();
+        internalCluster().startNodes(2, masterNodeSettings);
         Settings dateNodeSettings = Settings.builder()
                 .put(Node.NODE_MASTER_SETTING.getKey(), false)
                 .put(defaultSettings)
                 .build();
-        internalCluster().startNodesAsync(2, dateNodeSettings).get();
+        internalCluster().startNodes(2, dateNodeSettings);
         ClusterHealthResponse clusterHealthResponse = client().admin().cluster().prepareHealth()
                 .setWaitForEvents(Priority.LANGUID)
                 .setWaitForNodes("4")
@@ -131,13 +104,10 @@ public void testNoShardRelocationsOccurWhenElectedMasterNodeFails() throws Excep
 
         final String oldMaster = internalCluster().getMasterName();
         internalCluster().stopCurrentMasterNode();
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                String current = internalCluster().getMasterName();
-                assertThat(current, notNullValue());
-                assertThat(current, not(equalTo(oldMaster)));
-            }
+        assertBusy(() -> {
+            String current = internalCluster().getMasterName();
+            assertThat(current, notNullValue());
+            assertThat(current, not(equalTo(oldMaster)));
         });
         ensureSearchable("test");
 
@@ -150,7 +120,6 @@ public void testNodeFailuresAreProcessedOnce() throws ExecutionException, Interr
         Settings defaultSettings = Settings.builder()
                 .put(FaultDetection.PING_TIMEOUT_SETTING.getKey(), "1s")
                 .put(FaultDetection.PING_RETRIES_SETTING.getKey(), "1")
-                .put("discovery.type", "zen")
                 .build();
 
         Settings masterNodeSettings = Settings.builder()
@@ -162,22 +131,19 @@ public void testNodeFailuresAreProcessedOnce() throws ExecutionException, Interr
                 .put(Node.NODE_MASTER_SETTING.getKey(), false)
                 .put(defaultSettings)
                 .build();
-        internalCluster().startNodesAsync(2, dateNodeSettings).get();
+        internalCluster().startNodes(2, dateNodeSettings);
         client().admin().cluster().prepareHealth().setWaitForNodes("3").get();
 
         ClusterService clusterService = internalCluster().getInstance(ClusterService.class, master);
-        final ArrayList<ClusterState> statesFound = new ArrayList<>();
+        final AtomicInteger numUpdates = new AtomicInteger();
         final CountDownLatch nodesStopped = new CountDownLatch(1);
-        clusterService.add(new ClusterStateListener() {
-            @Override
-            public void clusterChanged(ClusterChangedEvent event) {
-                statesFound.add(event.state());
-                try {
-                    // block until both nodes have stopped to accumulate node failures
-                    nodesStopped.await();
-                } catch (InterruptedException e) {
-                    //meh
-                }
+        clusterService.addStateApplier(event -> {
+            numUpdates.incrementAndGet();
+            try {
+                // block until both nodes have stopped to accumulate node failures
+                nodesStopped.await();
+            } catch (InterruptedException e) {
+                //meh
             }
         });
 
@@ -186,15 +152,11 @@ public void clusterChanged(ClusterChangedEvent event) {
         nodesStopped.countDown();
 
         client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).get(); // wait for all to be processed
-        assertThat(statesFound, Matchers.hasSize(2));
+        assertThat(numUpdates.get(), either(equalTo(1)).or(equalTo(2))); // due to batching, both nodes can be handled in same CS update
     }
 
     public void testNodeRejectsClusterStateWithWrongMasterNode() throws Exception {
-        Settings settings = Settings.builder()
-                .put("discovery.type", "zen")
-                .build();
-        List<String> nodeNames = internalCluster().startNodesAsync(2, settings).get();
-        client().admin().cluster().prepareHealth().setWaitForNodes("2").get();
+        List<String> nodeNames = internalCluster().startNodes(2);
 
         List<String> nonMasterNodes = new ArrayList<>(nodeNames);
         nonMasterNodes.remove(internalCluster().getMasterName());
@@ -278,12 +240,7 @@ public static class CustomMetaData extends TestCustomMetaData {
         }
 
         @Override
-        protected TestCustomMetaData newTestCustomMetaData(String data) {
-            return new CustomMetaData(data);
-        }
-
-        @Override
-        public String type() {
+        public String getWriteableName() {
             return TYPE;
         }
 
@@ -293,44 +250,6 @@ public EnumSet<MetaData.XContentContext> context() {
         }
     }
 
-    public void testHandleNodeJoin_incompatibleMinVersion() throws UnknownHostException {
-        Settings nodeSettings = Settings.builder()
-                .put("discovery.type", "zen") // <-- To override the local setting if set externally
-                .build();
-        String nodeName = internalCluster().startNode(nodeSettings);
-        ZenDiscovery zenDiscovery = (ZenDiscovery) internalCluster().getInstance(Discovery.class, nodeName);
-        ClusterService clusterService = internalCluster().getInstance(ClusterService.class, nodeName);
-        DiscoveryNode node = new DiscoveryNode("_node_id", new InetSocketTransportAddress(InetAddress.getByName("0.0.0.0"), 0),
-                emptyMap(), emptySet(), previousMajorVersion);
-        final AtomicReference<IllegalStateException> holder = new AtomicReference<>();
-        zenDiscovery.handleJoinRequest(node, clusterService.state(), new MembershipAction.JoinCallback() {
-            @Override
-            public void onSuccess() {
-            }
-
-            @Override
-            public void onFailure(Exception e) {
-                holder.set((IllegalStateException) e);
-            }
-        });
-
-        assertThat(holder.get(), notNullValue());
-        assertThat(holder.get().getMessage(), equalTo("Can't handle join request from a node with a version [" + previousMajorVersion
-                + "] that is lower than the minimum compatible version [" + Version.CURRENT.minimumCompatibilityVersion() + "]"));
-    }
-
-    public void testJoinElectedMaster_incompatibleMinVersion() {
-        ElectMasterService electMasterService = new ElectMasterService(Settings.EMPTY);
-
-        DiscoveryNode node = new DiscoveryNode("_node_id", new LocalTransportAddress("_id"), emptyMap(),
-                Collections.singleton(DiscoveryNode.Role.MASTER), Version.CURRENT);
-        assertThat(electMasterService.electMaster(Collections.singletonList(node)), sameInstance(node));
-        node = new DiscoveryNode("_node_id", new LocalTransportAddress("_id"), emptyMap(), emptySet(), previousMajorVersion);
-        assertThat("Can't join master because version " + previousMajorVersion
-                + " is lower than the minimum compatable version " + Version.CURRENT + " can support",
-                electMasterService.electMaster(Collections.singletonList(node)), nullValue());
-    }
-
     public void testDiscoveryStats() throws IOException {
         String expectedStatsJsonResponse = "{\n" +
                 "  \"discovery\" : {\n" +
@@ -342,10 +261,7 @@ public void testDiscoveryStats() throws IOException {
                 "  }\n" +
                 "}";
 
-        Settings nodeSettings = Settings.builder()
-                .put("discovery.type", "zen") // <-- To override the local setting if set externally
-                .build();
-        internalCluster().startNode(nodeSettings);
+        internalCluster().startNode();
 
         logger.info("--> request node discovery stats");
         NodesStatsResponse statsResponse = client().admin().cluster().prepareNodesStats().clear().setDiscovery(true).get();
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryUnitTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryUnitTests.java
index ba4c14c205820..c515c2b74ebc2 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryUnitTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryUnitTests.java
@@ -19,31 +19,71 @@
 
 package org.elasticsearch.discovery.zen;
 
+import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.Version;
+import org.elasticsearch.action.support.replication.ClusterStateCreationUtils;
+import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNode.Role;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.IndexRoutingTable;
+import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
+import org.elasticsearch.cluster.routing.RoutingTable;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.TestShardRouting;
+import org.elasticsearch.cluster.routing.UnassignedInfo;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.LocalTransportAddress;
-import org.elasticsearch.discovery.zen.ping.ZenPing;
+import org.elasticsearch.discovery.Discovery;
+import org.elasticsearch.discovery.zen.PublishClusterStateActionTests.AssertingAckListener;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.VersionUtils;
+import org.elasticsearch.test.transport.MockTransportService;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportChannel;
+import org.elasticsearch.transport.TransportResponse;
+import org.elasticsearch.transport.TransportResponseOptions;
+import org.elasticsearch.transport.TransportService;
 
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayDeque;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.EnumSet;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.stream.Collectors;
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_CREATION_DATE;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_VERSION_CREATED;
+import static org.elasticsearch.cluster.routing.RoutingTableTests.updateActiveAllocations;
+import static org.elasticsearch.discovery.zen.ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING;
 import static org.elasticsearch.discovery.zen.ZenDiscovery.shouldIgnoreOrRejectNewClusterState;
+import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
+import static org.elasticsearch.test.ClusterServiceUtils.setState;
+import static org.hamcrest.Matchers.arrayWithSize;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.emptyArray;
 import static org.hamcrest.Matchers.equalTo;
 
-/**
- */
 public class ZenDiscoveryUnitTests extends ESTestCase {
 
     public void testShouldIgnoreNewClusterState() {
@@ -71,7 +111,7 @@ public void testShouldIgnoreNewClusterState() {
 
         currentNodes = DiscoveryNodes.builder();
         currentNodes.masterNodeId("b").add(new DiscoveryNode("b", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT));
-        ;
+
         // version isn't taken into account, so randomize it to ensure this.
         if (randomBoolean()) {
             currentState.version(2);
@@ -107,10 +147,10 @@ public void testFilterNonMasterPingResponse() {
         ArrayList<DiscoveryNode> masterNodes = new ArrayList<>();
         ArrayList<DiscoveryNode> allNodes = new ArrayList<>();
         for (int i = randomIntBetween(10, 20); i >= 0; i--) {
-            Set<DiscoveryNode.Role> roles = new HashSet<>(randomSubsetOf(Arrays.asList(DiscoveryNode.Role.values())));
+            Set<Role> roles = new HashSet<>(randomSubsetOf(Arrays.asList(Role.values())));
             DiscoveryNode node = new DiscoveryNode("node_" + i, "id_" + i, LocalTransportAddress.buildUnique(), Collections.emptyMap(),
                     roles, Version.CURRENT);
-            responses.add(new ZenPing.PingResponse(node, randomBoolean() ? null : node, new ClusterName("test"), randomBoolean()));
+            responses.add(new ZenPing.PingResponse(node, randomBoolean() ? null : node, new ClusterName("test"), randomLong()));
             allNodes.add(node);
             if (node.isMasterNode()) {
                 masterNodes.add(node);
@@ -118,8 +158,7 @@ public void testFilterNonMasterPingResponse() {
         }
 
         boolean ignore = randomBoolean();
-        List<ZenPing.PingResponse> filtered = ZenDiscovery.filterPingResponses(
-                responses.toArray(new ZenPing.PingResponse[responses.size()]), ignore, logger);
+        List<ZenPing.PingResponse> filtered = ZenDiscovery.filterPingResponses(responses, ignore, logger);
         final List<DiscoveryNode> filteredNodes = filtered.stream().map(ZenPing.PingResponse::node).collect(Collectors.toList());
         if (ignore) {
             assertThat(filteredNodes, equalTo(masterNodes));
@@ -127,4 +166,221 @@ public void testFilterNonMasterPingResponse() {
             assertThat(filteredNodes, equalTo(allNodes));
         }
     }
+
+    public void testNodesUpdatedAfterClusterStatePublished() throws Exception {
+        ThreadPool threadPool = new TestThreadPool(getClass().getName());
+        // randomly make minimum_master_nodes a value higher than we have nodes for, so it will force failure
+        int minMasterNodes = randomBoolean() ? 3 : 1;
+        Settings settings = Settings.builder()
+                                .put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.toString(minMasterNodes)).build();
+
+        ArrayDeque<Closeable> toClose = new ArrayDeque<>();
+        try {
+            Set<DiscoveryNode> expectedFDNodes = null;
+
+            final MockTransportService masterTransport = MockTransportService.local(settings, Version.CURRENT, threadPool, null);
+            masterTransport.start();
+            DiscoveryNode masterNode = masterTransport.getLocalNode();
+            toClose.addFirst(masterTransport);
+            ClusterState state = ClusterStateCreationUtils.state(masterNode, masterNode, masterNode);
+            // build the zen discovery and cluster service
+            ClusterService masterClusterService = createClusterService(threadPool, masterNode);
+            toClose.addFirst(masterClusterService);
+            // TODO: clustername shouldn't be stored twice in cluster service, but for now, work around it
+            state = ClusterState.builder(masterClusterService.getClusterName()).nodes(state.nodes()).build();
+            setState(masterClusterService, state);
+            ZenDiscovery masterZen = buildZenDiscovery(settings, masterTransport, masterClusterService, threadPool);
+            toClose.addFirst(masterZen);
+            masterTransport.acceptIncomingRequests();
+
+            final MockTransportService otherTransport = MockTransportService.local(settings, Version.CURRENT, threadPool, null);
+            otherTransport.start();
+            toClose.addFirst(otherTransport);
+            DiscoveryNode otherNode = otherTransport.getLocalNode();
+            final ClusterState otherState = ClusterState.builder(masterClusterService.getClusterName())
+                .nodes(DiscoveryNodes.builder().add(otherNode).localNodeId(otherNode.getId())).build();
+            ClusterService otherClusterService = createClusterService(threadPool, masterNode);
+            toClose.addFirst(otherClusterService);
+            setState(otherClusterService, otherState);
+            ZenDiscovery otherZen = buildZenDiscovery(settings, otherTransport, otherClusterService, threadPool);
+            toClose.addFirst(otherZen);
+            otherTransport.acceptIncomingRequests();
+
+            masterTransport.connectToNode(otherNode);
+            otherTransport.connectToNode(masterNode);
+
+            // a new cluster state with a new discovery node (we will test if the cluster state
+            // was updated by the presence of this node in NodesFaultDetection)
+            ClusterState newState = ClusterState.builder(masterClusterService.state()).incrementVersion().nodes(
+                DiscoveryNodes.builder(state.nodes()).add(otherNode).masterNodeId(masterNode.getId())
+            ).build();
+
+            try {
+                // publishing a new cluster state
+                ClusterChangedEvent clusterChangedEvent = new ClusterChangedEvent("testing", newState, state);
+                AssertingAckListener listener = new AssertingAckListener(newState.nodes().getSize() - 1);
+                expectedFDNodes = masterZen.getFaultDetectionNodes();
+                masterZen.publish(clusterChangedEvent, listener);
+                listener.await(1, TimeUnit.HOURS);
+                // publish was a success, update expected FD nodes based on new cluster state
+                expectedFDNodes = fdNodesForState(newState, masterNode);
+            } catch (Discovery.FailedToCommitClusterStateException e) {
+                // not successful, so expectedFDNodes above should remain what it was originally assigned
+                assertEquals(3, minMasterNodes); // ensure min master nodes is the higher value, otherwise we shouldn't fail
+            }
+
+            assertEquals(expectedFDNodes, masterZen.getFaultDetectionNodes());
+        } finally {
+            IOUtils.close(toClose);
+            terminate(threadPool);
+        }
+    }
+
+    public void testPendingCSQueueIsClearedWhenClusterStatePublished() throws Exception {
+        ThreadPool threadPool = new TestThreadPool(getClass().getName());
+        // randomly make minimum_master_nodes a value higher than we have nodes for, so it will force failure
+        int minMasterNodes =  randomBoolean() ? 3 : 1;
+        Settings settings = Settings.builder()
+            .put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.toString(minMasterNodes)).build();
+
+        ArrayDeque<Closeable> toClose = new ArrayDeque<>();
+        try {
+            final MockTransportService masterTransport = MockTransportService.local(settings, Version.CURRENT, threadPool, null);
+            masterTransport.start();
+            DiscoveryNode masterNode = masterTransport.getLocalNode();
+            toClose.addFirst(masterTransport);
+            ClusterState state = ClusterStateCreationUtils.state(masterNode, null, masterNode);
+            // build the zen discovery and cluster service
+            ClusterService masterClusterService = createClusterService(threadPool, masterNode);
+            toClose.addFirst(masterClusterService);
+            state = ClusterState.builder(masterClusterService.getClusterName()).nodes(state.nodes()).build();
+            setState(masterClusterService, state);
+            ZenDiscovery masterZen = buildZenDiscovery(settings, masterTransport, masterClusterService, threadPool);
+            toClose.addFirst(masterZen);
+            masterTransport.acceptIncomingRequests();
+
+            // inject a pending cluster state
+            masterZen.pendingClusterStatesQueue().addPending(ClusterState.builder(new ClusterName("foreign")).build());
+
+            // a new cluster state with a new discovery node (we will test if the cluster state
+            // was updated by the presence of this node in NodesFaultDetection)
+            ClusterState newState = ClusterState.builder(masterClusterService.state()).incrementVersion().nodes(
+                DiscoveryNodes.builder(masterClusterService.state().nodes()).masterNodeId(masterNode.getId())
+            ).build();
+
+
+            try {
+                // publishing a new cluster state
+                ClusterChangedEvent clusterChangedEvent = new ClusterChangedEvent("testing", newState, state);
+                AssertingAckListener listener = new AssertingAckListener(newState.nodes().getSize() - 1);
+                masterZen.publish(clusterChangedEvent, listener);
+                listener.await(1, TimeUnit.HOURS);
+                // publish was a success, check that queue as cleared
+                assertThat(masterZen.pendingClusterStates(), emptyArray());
+            } catch (Discovery.FailedToCommitClusterStateException e) {
+                // not successful, so the pending queue should stay
+                assertThat(masterZen.pendingClusterStates(), arrayWithSize(1));
+                assertThat(masterZen.pendingClusterStates()[0].getClusterName().value(), equalTo("foreign"));
+            }
+        } finally {
+            IOUtils.close(toClose);
+            terminate(threadPool);
+        }
+    }
+
+    private ZenDiscovery buildZenDiscovery(Settings settings, TransportService service, ClusterService clusterService, ThreadPool threadPool) {
+        ZenDiscovery zenDiscovery = new ZenDiscovery(settings, threadPool, service, new NamedWriteableRegistry(ClusterModule.getNamedWriteables()),
+            clusterService, Collections::emptyList);
+        zenDiscovery.start();
+        return zenDiscovery;
+    }
+
+    private Set<DiscoveryNode> fdNodesForState(ClusterState clusterState, DiscoveryNode localNode) {
+        final Set<DiscoveryNode> discoveryNodes = new HashSet<>();
+        clusterState.getNodes().getNodes().valuesIt().forEachRemaining(discoveryNode -> {
+            // the local node isn't part of the nodes that are pinged (don't ping ourselves)
+            if (discoveryNode.getId().equals(localNode.getId()) == false) {
+                discoveryNodes.add(discoveryNode);
+            }
+        });
+        return discoveryNodes;
+    }
+
+    public void testValidateOnUnsupportedIndexVersionCreated() throws Exception {
+        final int iters = randomIntBetween(3, 10);
+        for (int i = 0; i < iters; i++) {
+            ClusterState.Builder stateBuilder = ClusterState.builder(ClusterName.DEFAULT);
+            final DiscoveryNode otherNode = new DiscoveryNode("other_node", new LocalTransportAddress("0"), emptyMap(),
+                EnumSet.allOf(DiscoveryNode.Role.class), Version.CURRENT);
+            MembershipAction.ValidateJoinRequestRequestHandler request = new MembershipAction.ValidateJoinRequestRequestHandler();
+            final boolean incompatible = randomBoolean();
+            IndexMetaData indexMetaData = IndexMetaData.builder("test").settings(Settings.builder()
+                .put(SETTING_VERSION_CREATED, incompatible ? Version.fromString("1.7.5")
+                    : VersionUtils.randomVersionBetween(random(), Version.CURRENT.minimumIndexCompatibilityVersion(), Version.CURRENT))
+                .put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 0)
+                .put(SETTING_CREATION_DATE, System.currentTimeMillis()))
+                .state(IndexMetaData.State.OPEN)
+                .build();
+            IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(indexMetaData.getIndex());
+            RoutingTable.Builder routing = new RoutingTable.Builder();
+            routing.addAsNew(indexMetaData);
+            final ShardId shardId = new ShardId("test", "_na_", 0);
+            IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(shardId);
+
+            final DiscoveryNode primaryNode = otherNode;
+            indexShardRoutingBuilder.addShard(TestShardRouting.newShardRouting("test", 0, primaryNode.getId(), null, true,
+                ShardRoutingState.INITIALIZING, new UnassignedInfo(UnassignedInfo.Reason.INDEX_REOPENED, "getting there")));
+            indexRoutingTableBuilder.addIndexShard(indexShardRoutingBuilder.build());
+            IndexRoutingTable indexRoutingTable = indexRoutingTableBuilder.build();
+            IndexMetaData updatedIndexMetaData = updateActiveAllocations(indexRoutingTable, indexMetaData);
+            stateBuilder.metaData(MetaData.builder().put(updatedIndexMetaData, false).generateClusterUuidIfNeeded())
+                .routingTable(RoutingTable.builder().add(indexRoutingTable).build());
+            if (incompatible) {
+                IllegalStateException ex = expectThrows(IllegalStateException.class, () ->
+                    request.messageReceived(new MembershipAction.ValidateJoinRequest(stateBuilder.build()), null));
+                assertEquals("index [test] version not supported: "
+                    + Version.fromString("1.7.5")
+                    + " minimum compatible index version is: " + Version.CURRENT.minimumIndexCompatibilityVersion(), ex.getMessage());
+            } else {
+                AtomicBoolean sendResponse = new AtomicBoolean(false);
+                request.messageReceived(new MembershipAction.ValidateJoinRequest(stateBuilder.build()), new TransportChannel() {
+                    @Override
+                    public String action() {
+                        return null;
+                    }
+
+                    @Override
+                    public String getProfileName() {
+                        return null;
+                    }
+
+                    @Override
+                    public long getRequestId() {
+                        return 0;
+                    }
+
+                    @Override
+                    public String getChannelType() {
+                        return null;
+                    }
+
+                    @Override
+                    public void sendResponse(TransportResponse response) throws IOException {
+                        sendResponse.set(true);
+                    }
+
+                    @Override
+                    public void sendResponse(TransportResponse response, TransportResponseOptions options) throws IOException {
+
+                    }
+
+                    @Override
+                    public void sendResponse(Exception exception) throws IOException {
+
+                    }
+                });
+                assertTrue(sendResponse.get());
+            }
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/ZenPingTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/ZenPingTests.java
index 72674f44e3dc9..2013d22db2dd5 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/ZenPingTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/ZenPingTests.java
@@ -24,11 +24,11 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.LocalTransportAddress;
-import org.elasticsearch.discovery.zen.ping.ZenPing;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.List;
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
@@ -39,7 +39,7 @@ public void testPingCollection() {
         DiscoveryNode[] nodes = new DiscoveryNode[randomIntBetween(1, 30)];
         long maxIdPerNode[] = new long[nodes.length];
         DiscoveryNode masterPerNode[] = new DiscoveryNode[nodes.length];
-        boolean hasJoinedOncePerNode[] = new boolean[nodes.length];
+        long clusterStateVersionPerNode[] = new long[nodes.length];
         ArrayList<ZenPing.PingResponse> pings = new ArrayList<>();
         for (int i = 0; i < nodes.length; i++) {
             nodes[i] = new DiscoveryNode("" + i, LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
@@ -51,9 +51,9 @@ public void testPingCollection() {
             if (randomBoolean()) {
                 masterNode = nodes[randomInt(nodes.length - 1)];
             }
-            boolean hasJoinedOnce = randomBoolean();
+            long clusterStateVersion = randomLong();
             ZenPing.PingResponse ping = new ZenPing.PingResponse(nodes[node], masterNode, ClusterName.CLUSTER_NAME_SETTING.
-                getDefault(Settings.EMPTY), hasJoinedOnce);
+                getDefault(Settings.EMPTY), clusterStateVersion);
             if (rarely()) {
                 // ignore some pings
                 continue;
@@ -61,7 +61,7 @@ public void testPingCollection() {
             // update max ping info
             maxIdPerNode[node] = ping.id();
             masterPerNode[node] = masterNode;
-            hasJoinedOncePerNode[node] = hasJoinedOnce;
+            clusterStateVersionPerNode[node] = clusterStateVersion;
             pings.add(ping);
         }
 
@@ -69,15 +69,15 @@ public void testPingCollection() {
         Collections.shuffle(pings, random());
 
         ZenPing.PingCollection collection = new ZenPing.PingCollection();
-        collection.addPings(pings.toArray(new ZenPing.PingResponse[pings.size()]));
+        pings.forEach(collection::addPing);
 
-        ZenPing.PingResponse[] aggregate = collection.toArray();
+        List<ZenPing.PingResponse> aggregate = collection.toList();
 
         for (ZenPing.PingResponse ping : aggregate) {
             int nodeId = Integer.parseInt(ping.node().getId());
             assertThat(maxIdPerNode[nodeId], equalTo(ping.id()));
             assertThat(masterPerNode[nodeId], equalTo(ping.master()));
-            assertThat(hasJoinedOncePerNode[nodeId], equalTo(ping.hasJoinedOnce()));
+            assertThat(clusterStateVersionPerNode[nodeId], equalTo(ping.getClusterStateVersion()));
 
             maxIdPerNode[nodeId] = -1; // mark as seen
         }
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPingIT.java b/core/src/test/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPingIT.java
deleted file mode 100644
index ea5779c33bbab..0000000000000
--- a/core/src/test/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPingIT.java
+++ /dev/null
@@ -1,239 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.discovery.zen.ping.unicast;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.network.NetworkAddress;
-import org.elasticsearch.common.network.NetworkService;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.InetSocketTransportAddress;
-import org.elasticsearch.common.transport.TransportAddress;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.util.BigArrays;
-import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
-import org.elasticsearch.discovery.zen.elect.ElectMasterService;
-import org.elasticsearch.discovery.zen.ping.PingContextProvider;
-import org.elasticsearch.discovery.zen.ping.ZenPing;
-import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.VersionUtils;
-import org.elasticsearch.threadpool.TestThreadPool;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.MockTcpTransport;
-import org.elasticsearch.transport.TransportConnectionListener;
-import org.elasticsearch.transport.TransportService;
-import org.elasticsearch.transport.TransportSettings;
-
-import java.net.InetSocketAddress;
-import java.util.Collections;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import static java.util.Collections.emptyMap;
-import static java.util.Collections.emptySet;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.greaterThan;
-
-public class UnicastZenPingIT extends ESTestCase {
-    public void testSimplePings() throws InterruptedException {
-        int startPort = 11000 + randomIntBetween(0, 1000);
-        int endPort = startPort + 10;
-        Settings settings = Settings.builder()
-            .put("cluster.name", "test")
-            .put(TransportSettings.PORT.getKey(), startPort + "-" + endPort).build();
-
-        Settings settingsMismatch = Settings.builder().put(settings)
-            .put("cluster.name", "mismatch")
-            .put(TransportSettings.PORT.getKey(), startPort + "-" + endPort).build();
-
-        ThreadPool threadPool = new TestThreadPool(getClass().getName());
-        NetworkService networkService = new NetworkService(settings, Collections.emptyList());
-        ElectMasterService electMasterService = new ElectMasterService(settings);
-
-        NetworkHandle handleA = startServices(settings, threadPool, networkService, "UZP_A", Version.CURRENT);
-        NetworkHandle handleB = startServices(settings, threadPool, networkService, "UZP_B", Version.CURRENT);
-        NetworkHandle handleC = startServices(settingsMismatch, threadPool, networkService, "UZP_C", Version.CURRENT);
-        // just fake that no versions are compatible with this node
-        Version previousVersion = VersionUtils.getPreviousVersion(Version.CURRENT.minimumCompatibilityVersion());
-        Version versionD = VersionUtils.randomVersionBetween(random(), previousVersion.minimumCompatibilityVersion(), previousVersion);
-        NetworkHandle handleD = startServices(settingsMismatch, threadPool, networkService, "UZP_D", versionD);
-
-        Settings hostsSettings = Settings.builder()
-                .putArray("discovery.zen.ping.unicast.hosts",
-                NetworkAddress.format(new InetSocketAddress(handleA.address.address().getAddress(), handleA.address.address().getPort())),
-                NetworkAddress.format(new InetSocketAddress(handleB.address.address().getAddress(), handleB.address.address().getPort())),
-                NetworkAddress.format(new InetSocketAddress(handleC.address.address().getAddress(), handleC.address.address().getPort())),
-                NetworkAddress.format(new InetSocketAddress(handleD.address.address().getAddress(), handleD.address.address().getPort())))
-                .put("cluster.name", "test")
-                .build();
-
-        Settings hostsSettingsMismatch = Settings.builder().put(hostsSettings).put(settingsMismatch).build();
-        UnicastZenPing zenPingA = new UnicastZenPing(hostsSettings, threadPool, handleA.transportService, electMasterService, null);
-        zenPingA.setPingContextProvider(new PingContextProvider() {
-            @Override
-            public DiscoveryNodes nodes() {
-                return DiscoveryNodes.builder().add(handleA.node).localNodeId("UZP_A").build();
-            }
-
-            @Override
-            public boolean nodeHasJoinedClusterOnce() {
-                return false;
-            }
-        });
-        zenPingA.start();
-
-        UnicastZenPing zenPingB = new UnicastZenPing(hostsSettings, threadPool, handleB.transportService, electMasterService, null);
-        zenPingB.setPingContextProvider(new PingContextProvider() {
-            @Override
-            public DiscoveryNodes nodes() {
-                return DiscoveryNodes.builder().add(handleB.node).localNodeId("UZP_B").build();
-            }
-
-            @Override
-            public boolean nodeHasJoinedClusterOnce() {
-                return true;
-            }
-        });
-        zenPingB.start();
-
-        UnicastZenPing zenPingC = new UnicastZenPing(hostsSettingsMismatch, threadPool, handleC.transportService, electMasterService,
-            null) {
-            @Override
-            protected Version getVersion() {
-                return versionD;
-            }
-        };
-        zenPingC.setPingContextProvider(new PingContextProvider() {
-            @Override
-            public DiscoveryNodes nodes() {
-                return DiscoveryNodes.builder().add(handleC.node).localNodeId("UZP_C").build();
-            }
-
-            @Override
-            public boolean nodeHasJoinedClusterOnce() {
-                return false;
-            }
-        });
-        zenPingC.start();
-
-        UnicastZenPing zenPingD = new UnicastZenPing(hostsSettingsMismatch, threadPool, handleD.transportService, electMasterService, null);
-        zenPingD.setPingContextProvider(new PingContextProvider() {
-            @Override
-            public DiscoveryNodes nodes() {
-                return DiscoveryNodes.builder().add(handleD.node).localNodeId("UZP_D").build();
-            }
-
-            @Override
-            public boolean nodeHasJoinedClusterOnce() {
-                return false;
-            }
-        });
-        zenPingD.start();
-
-        try {
-            logger.info("ping from UZP_A");
-            ZenPing.PingResponse[] pingResponses = zenPingA.pingAndWait(TimeValue.timeValueSeconds(1));
-            assertThat(pingResponses.length, equalTo(1));
-            assertThat(pingResponses[0].node().getId(), equalTo("UZP_B"));
-            assertTrue(pingResponses[0].hasJoinedOnce());
-            assertCounters(handleA, handleA, handleB, handleC, handleD);
-
-            // ping again, this time from B,
-            logger.info("ping from UZP_B");
-            pingResponses = zenPingB.pingAndWait(TimeValue.timeValueSeconds(1));
-            assertThat(pingResponses.length, equalTo(1));
-            assertThat(pingResponses[0].node().getId(), equalTo("UZP_A"));
-            assertFalse(pingResponses[0].hasJoinedOnce());
-            assertCounters(handleB, handleA, handleB, handleC, handleD);
-
-            logger.info("ping from UZP_C");
-            pingResponses = zenPingC.pingAndWait(TimeValue.timeValueSeconds(1));
-            assertThat(pingResponses.length, equalTo(0));
-            assertCounters(handleC, handleA, handleB, handleC, handleD);
-
-            logger.info("ping from UZP_D");
-            pingResponses = zenPingD.pingAndWait(TimeValue.timeValueSeconds(1));
-            assertThat(pingResponses.length, equalTo(0));
-            assertCounters(handleD, handleA, handleB, handleC, handleD);
-        } finally {
-            zenPingA.close();
-            zenPingB.close();
-            zenPingC.close();
-            zenPingD.close();
-            handleA.transportService.close();
-            handleB.transportService.close();
-            handleC.transportService.close();
-            handleD.transportService.close();
-            terminate(threadPool);
-        }
-    }
-
-    // assert that we tried to ping each of the configured nodes at least once
-    private void assertCounters(NetworkHandle that, NetworkHandle...handles) {
-        for (NetworkHandle handle : handles) {
-            if (handle != that) {
-                assertThat(that.counters.get(handle.address).get(), greaterThan(0));
-            }
-        }
-    }
-
-    private NetworkHandle startServices(Settings settings, ThreadPool threadPool, NetworkService networkService, String nodeId,
-                                        Version version) {
-        MockTcpTransport transport = new MockTcpTransport(settings, threadPool, BigArrays.NON_RECYCLING_INSTANCE,
-            new NoneCircuitBreakerService(), new NamedWriteableRegistry(Collections.emptyList()), networkService, version);
-        final TransportService transportService = new TransportService(settings, transport, threadPool);
-        transportService.start();
-        transportService.acceptIncomingRequests();
-        ConcurrentMap<TransportAddress, AtomicInteger> counters = ConcurrentCollections.newConcurrentMap();
-        transportService.addConnectionListener(new TransportConnectionListener() {
-            @Override
-            public void onNodeConnected(DiscoveryNode node) {
-                counters.computeIfAbsent(node.getAddress(), k -> new AtomicInteger());
-                counters.get(node.getAddress()).incrementAndGet();
-            }
-
-            @Override
-            public void onNodeDisconnected(DiscoveryNode node) {
-            }
-        });
-        final DiscoveryNode node = new DiscoveryNode(nodeId, transportService.boundAddress().publishAddress(), emptyMap(), emptySet(),
-            version);
-        transportService.setLocalNode(node);
-        return new NetworkHandle((InetSocketTransportAddress)transport.boundAddress().publishAddress(), transportService, node, counters);
-    }
-
-    private static class NetworkHandle {
-        public final InetSocketTransportAddress address;
-        public final TransportService transportService;
-        public final DiscoveryNode node;
-        public final ConcurrentMap<TransportAddress, AtomicInteger> counters;
-
-        public NetworkHandle(InetSocketTransportAddress address, TransportService transportService, DiscoveryNode discoveryNode,
-                             ConcurrentMap<TransportAddress, AtomicInteger> counters) {
-            this.address = address;
-            this.transportService = transportService;
-            this.node = discoveryNode;
-            this.counters = counters;
-        }
-    }
-}
diff --git a/core/src/test/java/org/elasticsearch/document/DocumentActionsIT.java b/core/src/test/java/org/elasticsearch/document/DocumentActionsIT.java
index 9196a16060ce8..102c2d1545623 100644
--- a/core/src/test/java/org/elasticsearch/document/DocumentActionsIT.java
+++ b/core/src/test/java/org/elasticsearch/document/DocumentActionsIT.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.document;
 
+import org.elasticsearch.action.DocWriteRequest.OpType;
 import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheResponse;
 import org.elasticsearch.action.admin.indices.flush.FlushResponse;
 import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse;
@@ -32,6 +33,7 @@
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
 
@@ -103,7 +105,7 @@ public void testIndexActions() throws Exception {
 
         logger.info("Get [type1/1] with script");
         for (int i = 0; i < 5; i++) {
-            getResult = client().prepareGet("test", "type1", "1").setFields("name").execute().actionGet();
+            getResult = client().prepareGet("test", "type1", "1").setStoredFields("name").execute().actionGet();
             assertThat(getResult.getIndex(), equalTo(getConcreteIndexName()));
             assertThat(getResult.isExists(), equalTo(true));
             assertThat(getResult.getSourceAsBytes(), nullValue());
@@ -184,38 +186,38 @@ public void testBulk() throws Exception {
                 .add(client().prepareIndex().setIndex("test").setType("type1").setId("2").setSource(source("2", "test")).setCreate(true))
                 .add(client().prepareIndex().setIndex("test").setType("type1").setSource(source("3", "test")))
                 .add(client().prepareDelete().setIndex("test").setType("type1").setId("1"))
-                .add(client().prepareIndex().setIndex("test").setType("type1").setSource("{ xxx }")) // failure
+                .add(client().prepareIndex().setIndex("test").setType("type1").setSource("{ xxx }", XContentType.JSON)) // failure
                 .execute().actionGet();
 
         assertThat(bulkResponse.hasFailures(), equalTo(true));
         assertThat(bulkResponse.getItems().length, equalTo(5));
 
         assertThat(bulkResponse.getItems()[0].isFailed(), equalTo(false));
-        assertThat(bulkResponse.getItems()[0].getOpType(), equalTo("index"));
+        assertThat(bulkResponse.getItems()[0].getOpType(), equalTo(OpType.INDEX));
         assertThat(bulkResponse.getItems()[0].getIndex(), equalTo(getConcreteIndexName()));
         assertThat(bulkResponse.getItems()[0].getType(), equalTo("type1"));
         assertThat(bulkResponse.getItems()[0].getId(), equalTo("1"));
 
         assertThat(bulkResponse.getItems()[1].isFailed(), equalTo(false));
-        assertThat(bulkResponse.getItems()[1].getOpType(), equalTo("create"));
+        assertThat(bulkResponse.getItems()[1].getOpType(), equalTo(OpType.CREATE));
         assertThat(bulkResponse.getItems()[1].getIndex(), equalTo(getConcreteIndexName()));
         assertThat(bulkResponse.getItems()[1].getType(), equalTo("type1"));
         assertThat(bulkResponse.getItems()[1].getId(), equalTo("2"));
 
         assertThat(bulkResponse.getItems()[2].isFailed(), equalTo(false));
-        assertThat(bulkResponse.getItems()[2].getOpType(), equalTo("index"));
+        assertThat(bulkResponse.getItems()[2].getOpType(), equalTo(OpType.INDEX));
         assertThat(bulkResponse.getItems()[2].getIndex(), equalTo(getConcreteIndexName()));
         assertThat(bulkResponse.getItems()[2].getType(), equalTo("type1"));
         String generatedId3 = bulkResponse.getItems()[2].getId();
 
         assertThat(bulkResponse.getItems()[3].isFailed(), equalTo(false));
-        assertThat(bulkResponse.getItems()[3].getOpType(), equalTo("delete"));
+        assertThat(bulkResponse.getItems()[3].getOpType(), equalTo(OpType.DELETE));
         assertThat(bulkResponse.getItems()[3].getIndex(), equalTo(getConcreteIndexName()));
         assertThat(bulkResponse.getItems()[3].getType(), equalTo("type1"));
         assertThat(bulkResponse.getItems()[3].getId(), equalTo("1"));
 
         assertThat(bulkResponse.getItems()[4].isFailed(), equalTo(true));
-        assertThat(bulkResponse.getItems()[4].getOpType(), equalTo("index"));
+        assertThat(bulkResponse.getItems()[4].getOpType(), equalTo(OpType.INDEX));
         assertThat(bulkResponse.getItems()[4].getIndex(), equalTo(getConcreteIndexName()));
         assertThat(bulkResponse.getItems()[4].getType(), equalTo("type1"));
 
diff --git a/core/src/test/java/org/elasticsearch/document/ShardInfoIT.java b/core/src/test/java/org/elasticsearch/document/ShardInfoIT.java
index 0ba97bee899f8..8bb81f8a976e1 100644
--- a/core/src/test/java/org/elasticsearch/document/ShardInfoIT.java
+++ b/core/src/test/java/org/elasticsearch/document/ShardInfoIT.java
@@ -31,6 +31,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESIntegTestCase;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@@ -47,7 +48,7 @@ public class ShardInfoIT extends ESIntegTestCase {
 
     public void testIndexAndDelete() throws Exception {
         prepareIndex(1);
-        IndexResponse indexResponse = client().prepareIndex("idx", "type").setSource("{}").get();
+        IndexResponse indexResponse = client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON).get();
         assertShardInfo(indexResponse);
         DeleteResponse deleteResponse = client().prepareDelete("idx", "type", indexResponse.getId()).get();
         assertShardInfo(deleteResponse);
@@ -55,7 +56,8 @@ public void testIndexAndDelete() throws Exception {
 
     public void testUpdate() throws Exception {
         prepareIndex(1);
-        UpdateResponse updateResponse = client().prepareUpdate("idx", "type", "1").setDoc("{}").setDocAsUpsert(true).get();
+        UpdateResponse updateResponse = client().prepareUpdate("idx", "type", "1").setDoc("{}", XContentType.JSON).setDocAsUpsert(true)
+            .get();
         assertShardInfo(updateResponse);
     }
 
@@ -63,7 +65,7 @@ public void testBulkWithIndexAndDeleteItems() throws Exception {
         prepareIndex(1);
         BulkRequestBuilder bulkRequestBuilder = client().prepareBulk();
         for (int i = 0; i < 10; i++) {
-            bulkRequestBuilder.add(client().prepareIndex("idx", "type").setSource("{}"));
+            bulkRequestBuilder.add(client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON));
         }
 
         BulkResponse bulkResponse = bulkRequestBuilder.get();
@@ -85,7 +87,8 @@ public void testBulkWithUpdateItems() throws Exception {
         prepareIndex(1);
         BulkRequestBuilder bulkRequestBuilder = client().prepareBulk();
         for (int i = 0; i < 10; i++) {
-            bulkRequestBuilder.add(client().prepareUpdate("idx", "type", Integer.toString(i)).setDoc("{}").setDocAsUpsert(true));
+            bulkRequestBuilder.add(client().prepareUpdate("idx", "type", Integer.toString(i)).setDoc("{}", XContentType.JSON)
+                .setDocAsUpsert(true));
         }
 
         BulkResponse bulkResponse = bulkRequestBuilder.get();
@@ -127,24 +130,21 @@ private void assertShardInfo(ReplicationResponse response, int expectedTotal, in
     }
 
     private void ensureActiveShardCopies(final int shardId, final int copyCount) throws Exception {
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                ClusterState state = client().admin().cluster().prepareState().get().getState();
-                assertThat(state.routingTable().index("idx"), not(nullValue()));
-                assertThat(state.routingTable().index("idx").shard(shardId), not(nullValue()));
-                assertThat(state.routingTable().index("idx").shard(shardId).activeShards().size(), equalTo(copyCount));
-
-                ClusterHealthResponse healthResponse = client().admin().cluster().prepareHealth("idx")
-                        .setWaitForNoRelocatingShards(true)
-                        .get();
-                assertThat(healthResponse.isTimedOut(), equalTo(false));
-
-                RecoveryResponse recoveryResponse = client().admin().indices().prepareRecoveries("idx")
-                        .setActiveOnly(true)
-                        .get();
-                assertThat(recoveryResponse.shardRecoveryStates().get("idx").size(), equalTo(0));
-            }
+        assertBusy(() -> {
+            ClusterState state = client().admin().cluster().prepareState().get().getState();
+            assertThat(state.routingTable().index("idx"), not(nullValue()));
+            assertThat(state.routingTable().index("idx").shard(shardId), not(nullValue()));
+            assertThat(state.routingTable().index("idx").shard(shardId).activeShards().size(), equalTo(copyCount));
+
+            ClusterHealthResponse healthResponse = client().admin().cluster().prepareHealth("idx")
+                    .setWaitForNoRelocatingShards(true)
+                    .get();
+            assertThat(healthResponse.isTimedOut(), equalTo(false));
+
+            RecoveryResponse recoveryResponse = client().admin().indices().prepareRecoveries("idx")
+                    .setActiveOnly(true)
+                    .get();
+            assertThat(recoveryResponse.shardRecoveryStates().get("idx").size(), equalTo(0));
         });
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/env/EnvironmentTests.java b/core/src/test/java/org/elasticsearch/env/EnvironmentTests.java
index 855ce83343b6c..083e2ad5cc078 100644
--- a/core/src/test/java/org/elasticsearch/env/EnvironmentTests.java
+++ b/core/src/test/java/org/elasticsearch/env/EnvironmentTests.java
@@ -23,10 +23,12 @@
 
 import java.io.IOException;
 import java.net.URL;
+import java.nio.file.Path;
 
 import static org.hamcrest.CoreMatchers.endsWith;
 import static org.hamcrest.CoreMatchers.notNullValue;
 import static org.hamcrest.CoreMatchers.nullValue;
+import static org.hamcrest.Matchers.equalTo;
 
 /**
  * Simple unit-tests for Environment.java
@@ -71,4 +73,104 @@ public void testRepositoryResolution() throws IOException {
         assertThat(environment.resolveRepoURL(new URL("jar:http://localhost/test/../repo1?blah!/repo/")), nullValue());
     }
 
+    public void testDefaultPathData() {
+        final Path defaultPathData = createTempDir().toAbsolutePath();
+        final Settings settings = Settings.builder()
+                .put("path.home", createTempDir().toAbsolutePath())
+                .put("default.path.data", defaultPathData)
+                .build();
+        final Environment environment = new Environment(settings);
+        assertThat(environment.dataFiles(), equalTo(new Path[] { defaultPathData }));
+    }
+
+    public void testPathDataOverrideDefaultPathData() {
+        final Path pathData = createTempDir().toAbsolutePath();
+        final Path defaultPathData = createTempDir().toAbsolutePath();
+        final Settings settings = Settings.builder()
+                .put("path.home", createTempDir().toAbsolutePath())
+                .put("path.data", pathData)
+                .put("default.path.data", defaultPathData)
+                .build();
+        final Environment environment = new Environment(settings);
+        assertThat(environment.dataFiles(), equalTo(new Path[] { pathData }));
+    }
+
+    public void testPathDataWhenNotSet() {
+        final Path pathHome = createTempDir().toAbsolutePath();
+        final Settings settings = Settings.builder().put("path.home", pathHome).build();
+        final Environment environment = new Environment(settings);
+        assertThat(environment.dataFiles(), equalTo(new Path[]{pathHome.resolve("data")}));
+    }
+
+    public void testPathDataNotSetInEnvironmentIfNotSet() {
+        final Path defaultPathData = createTempDir().toAbsolutePath();
+        final Settings settings = Settings.builder()
+                .put("path.home", createTempDir().toAbsolutePath())
+                .put("default.path.data", defaultPathData)
+                .build();
+        assertFalse(Environment.PATH_DATA_SETTING.exists(settings));
+        assertTrue(Environment.DEFAULT_PATH_DATA_SETTING.exists(settings));
+        final Environment environment = new Environment(settings);
+        assertFalse(Environment.PATH_DATA_SETTING.exists(environment.settings()));
+        assertTrue(Environment.DEFAULT_PATH_DATA_SETTING.exists(environment.settings()));
+    }
+
+    public void testDefaultPathLogs() {
+        final Path defaultPathLogs = createTempDir().toAbsolutePath();
+        final Settings settings = Settings.builder()
+                .put("path.home", createTempDir().toAbsolutePath())
+                .put("default.path.logs", defaultPathLogs)
+                .build();
+        final Environment environment = new Environment(settings);
+        assertThat(environment.logsFile(), equalTo(defaultPathLogs));
+    }
+
+    public void testPathLogsOverrideDefaultPathLogs() {
+        final Path pathLogs = createTempDir().toAbsolutePath();
+        final Path defaultPathLogs = createTempDir().toAbsolutePath();
+        final Settings settings = Settings.builder()
+                .put("path.home", createTempDir().toAbsolutePath())
+                .put("path.logs", pathLogs)
+                .put("default.path.logs", defaultPathLogs)
+                .build();
+        final Environment environment = new Environment(settings);
+        assertThat(environment.logsFile(), equalTo(pathLogs));
+    }
+
+    public void testPathLogsWhenNotSet() {
+        final Path pathHome = createTempDir().toAbsolutePath();
+        final Settings settings = Settings.builder().put("path.home", pathHome).build();
+        final Environment environment = new Environment(settings);
+        assertThat(environment.logsFile(), equalTo(pathHome.resolve("logs")));
+    }
+
+    public void testDefaultPathConf() {
+        final Path defaultPathConf = createTempDir().toAbsolutePath();
+        final Settings settings = Settings.builder()
+                .put("path.home", createTempDir().toAbsolutePath())
+                .put("default.path.conf", defaultPathConf)
+                .build();
+        final Environment environment = new Environment(settings);
+        assertThat(environment.configFile(), equalTo(defaultPathConf));
+    }
+
+    public void testPathConfOverrideDefaultPathConf() {
+        final Path pathConf = createTempDir().toAbsolutePath();
+        final Path defaultPathConf = createTempDir().toAbsolutePath();
+        final Settings settings = Settings.builder()
+                .put("path.home", createTempDir().toAbsolutePath())
+                .put("path.conf", pathConf)
+                .put("default.path.conf", defaultPathConf)
+                .build();
+        final Environment environment = new Environment(settings);
+        assertThat(environment.configFile(), equalTo(pathConf));
+    }
+
+    public void testPathConfWhenNotSet() {
+        final Path pathHome = createTempDir().toAbsolutePath();
+        final Settings settings = Settings.builder().put("path.home", pathHome).build();
+        final Environment environment = new Environment(settings);
+        assertThat(environment.configFile(), equalTo(pathHome.resolve("config")));
+    }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java b/core/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java
index e5c1c53dad77f..b261fd6864c3c 100644
--- a/core/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java
+++ b/core/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java
@@ -34,6 +34,7 @@
 import org.elasticsearch.test.IndexSettingsModule;
 
 import java.io.IOException;
+import java.nio.file.AtomicMoveNotSupportedException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
@@ -460,6 +461,39 @@ public void testPersistentNodeId() throws IOException {
         env.close();
     }
 
+    public void testExistingTempFiles() throws IOException {
+        String[] paths = tmpPaths();
+        // simulate some previous left over temp files
+        for (String path : randomSubsetOf(randomIntBetween(1, paths.length), paths)) {
+            final Path nodePath = NodeEnvironment.resolveNodePath(PathUtils.get(path), 0);
+            Files.createDirectories(nodePath);
+            Files.createFile(nodePath.resolve(NodeEnvironment.TEMP_FILE_NAME));
+            if (randomBoolean()) {
+                Files.createFile(nodePath.resolve(NodeEnvironment.TEMP_FILE_NAME + ".tmp"));
+            }
+            if (randomBoolean()) {
+                Files.createFile(nodePath.resolve(NodeEnvironment.TEMP_FILE_NAME + ".final"));
+            }
+        }
+        NodeEnvironment env = newNodeEnvironment(paths, Settings.EMPTY);
+        try {
+            env.ensureAtomicMoveSupported();
+        } catch (AtomicMoveNotSupportedException e) {
+            // that's OK :)
+        }
+        env.close();
+        // check we clean up
+        for (String path: paths) {
+            final Path nodePath = NodeEnvironment.resolveNodePath(PathUtils.get(path), 0);
+            final Path tempFile = nodePath.resolve(NodeEnvironment.TEMP_FILE_NAME);
+            assertFalse(tempFile + " should have been cleaned", Files.exists(tempFile));
+            final Path srcTempFile = nodePath.resolve(NodeEnvironment.TEMP_FILE_NAME + ".src");
+            assertFalse(srcTempFile + " should have been cleaned", Files.exists(srcTempFile));
+            final Path targetTempFile = nodePath.resolve(NodeEnvironment.TEMP_FILE_NAME + ".target");
+            assertFalse(targetTempFile + " should have been cleaned", Files.exists(targetTempFile));
+        }
+    }
+
     /** Converts an array of Strings to an array of Paths, adding an additional child if specified */
     private Path[] stringsToPaths(String[] strings, String additional) {
         Path[] locations = new Path[strings.length];
diff --git a/core/src/test/java/org/elasticsearch/explain/ExplainActionIT.java b/core/src/test/java/org/elasticsearch/explain/ExplainActionIT.java
index 4aacb67299964..528b03bc831b7 100644
--- a/core/src/test/java/org/elasticsearch/explain/ExplainActionIT.java
+++ b/core/src/test/java/org/elasticsearch/explain/ExplainActionIT.java
@@ -131,7 +131,7 @@ public void testExplainWithFields() throws Exception {
         refresh();
         ExplainResponse response = client().prepareExplain(indexOrAlias(), "test", "1")
                 .setQuery(QueryBuilders.matchAllQuery())
-                .setFields("obj1.field1").get();
+                .setStoredFields("obj1.field1").get();
         assertNotNull(response);
         assertTrue(response.isMatch());
         assertNotNull(response.getExplanation());
@@ -148,7 +148,7 @@ public void testExplainWithFields() throws Exception {
         refresh();
         response = client().prepareExplain(indexOrAlias(), "test", "1")
                 .setQuery(QueryBuilders.matchAllQuery())
-                .setFields("obj1.field1").setFetchSource(true).get();
+                .setStoredFields("obj1.field1").setFetchSource(true).get();
         assertNotNull(response);
         assertTrue(response.isMatch());
         assertNotNull(response.getExplanation());
@@ -164,7 +164,7 @@ public void testExplainWithFields() throws Exception {
 
         response = client().prepareExplain(indexOrAlias(), "test", "1")
                 .setQuery(QueryBuilders.matchAllQuery())
-                .setFields("obj1.field1", "obj1.field2").get();
+                .setStoredFields("obj1.field1", "obj1.field2").get();
         assertNotNull(response);
         assertTrue(response.isMatch());
         String v1 = (String) response.getGetResult().field("obj1.field1").getValue();
diff --git a/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsIntegrationIT.java b/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsIntegrationIT.java
index 6240a13a0bde9..ef92115272fe2 100644
--- a/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsIntegrationIT.java
+++ b/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsIntegrationIT.java
@@ -27,8 +27,10 @@
 import org.elasticsearch.action.fieldstats.FieldStatsResponse;
 import org.elasticsearch.action.fieldstats.IndexConstraint;
 import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.index.cache.request.RequestCacheStats;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.geo.RandomGeoGenerator;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -40,6 +42,7 @@
 import static org.elasticsearch.action.fieldstats.IndexConstraint.Property.MIN;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllSuccessful;
+import static org.hamcrest.Matchers.closeTo;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.nullValue;
@@ -60,7 +63,8 @@ public void testRandom() throws Exception {
                     "long", "type=long",
                     "integer", "type=integer",
                     "short", "type=short",
-                    "byte", "type=byte"));
+                    "byte", "type=byte",
+                    "location", "type=geo_point"));
         ensureGreen("test");
 
         // index=false
@@ -74,7 +78,8 @@ public void testRandom() throws Exception {
             "long", "type=long,index=false",
             "integer", "type=integer,index=false",
             "short", "type=short,index=false",
-            "byte", "type=byte,index=false"
+            "byte", "type=byte,index=false",
+            "location", "type=geo_point,index=false"
         ));
         ensureGreen("test1");
 
@@ -89,7 +94,8 @@ public void testRandom() throws Exception {
             "long", "type=long,index=false",
             "integer", "type=integer,index=false",
             "short", "type=short,index=false",
-            "byte", "type=byte,index=false"
+            "byte", "type=byte,index=false",
+            "location", "type=geo_point,index=false"
         ));
         ensureGreen("test3");
 
@@ -107,6 +113,8 @@ public void testRandom() throws Exception {
         double maxFloat = Double.NEGATIVE_INFINITY;
         double minDouble = Double.POSITIVE_INFINITY;
         double maxDouble = Double.NEGATIVE_INFINITY;
+        GeoPoint minLoc = new GeoPoint(90, 180);
+        GeoPoint maxLoc = new GeoPoint(-90, -180);
         String minString = new String(Character.toChars(1114111));
         String maxString = "0";
 
@@ -135,6 +143,9 @@ public void testRandom() throws Exception {
             double d = randomDouble();
             minDouble = Math.min(minDouble, d);
             maxDouble = Math.max(maxDouble, d);
+            GeoPoint loc = RandomGeoGenerator.randomPoint(random());
+            minLoc.reset(Math.min(loc.lat(), minLoc.lat()), Math.min(loc.lon(), minLoc.lon()));
+            maxLoc.reset(Math.max(loc.lat(), maxLoc.lat()), Math.max(loc.lon(), maxLoc.lon()));
             String str = randomRealisticUnicodeOfLength(3);
             if (str.compareTo(minString) < 0) {
                 minString = str;
@@ -151,6 +162,7 @@ public void testRandom() throws Exception {
                                 "half_float", hf,
                                 "float", f,
                                 "double", d,
+                                "location", loc,
                                 "string", str)
             );
         }
@@ -158,7 +170,7 @@ public void testRandom() throws Exception {
 
         FieldStatsResponse response = client()
             .prepareFieldStats()
-            .setFields("byte", "short", "integer", "long", "half_float", "float", "double", "string").get();
+            .setFields("byte", "short", "integer", "long", "half_float", "float", "double", "location", "string").get();
         assertAllSuccessful(response);
 
         for (FieldStats<?> stats : response.getAllFieldStats().values()) {
@@ -188,6 +200,11 @@ public void testRandom() throws Exception {
         assertThat(response.getAllFieldStats().get("double").getMinValue(), equalTo(minDouble));
         assertThat(response.getAllFieldStats().get("double").getMaxValue(), equalTo(maxDouble));
         assertThat(response.getAllFieldStats().get("double").getDisplayType(), equalTo("float"));
+        assertThat(((GeoPoint)response.getAllFieldStats().get("location").getMinValue()).lat(), closeTo(minLoc.lat(), 1E-5));
+        assertThat(((GeoPoint)response.getAllFieldStats().get("location").getMinValue()).lon(), closeTo(minLoc.lon(), 1E-5));
+        assertThat(((GeoPoint)response.getAllFieldStats().get("location").getMaxValue()).lat(), closeTo(maxLoc.lat(), 1E-5));
+        assertThat(((GeoPoint)response.getAllFieldStats().get("location").getMaxValue()).lon(), closeTo(maxLoc.lon(), 1E-5));
+        assertThat(response.getAllFieldStats().get("location").getDisplayType(), equalTo("geo_point"));
     }
 
     public void testFieldStatsIndexLevel() throws Exception {
@@ -522,6 +539,25 @@ public void testCached() throws Exception {
         assertEquals(oldHitCount, indexStats.getHitCount());
     }
 
+    public void testGeoPointNotIndexed() throws Exception {
+        assertAcked(prepareCreate("test").addMapping("test", "value", "type=long", "location", "type=geo_point,index=no"));
+        ensureGreen("test");
+        client().prepareIndex("test", "test").setSource("value", 1L, "location", new GeoPoint(32, -132)).get();
+        client().prepareIndex("test", "test").setSource("value", 2L).get();
+        client().prepareIndex("test", "test").setSource("value", 3L).get();
+        client().prepareIndex("test", "test").setSource("value", 4L).get();
+        refresh();
+
+        FieldStatsResponse response = client().prepareFieldStats().setFields("value", "location").get();
+        assertAllSuccessful(response);
+        assertThat(response.getIndicesMergedFieldStats().size(), equalTo(1));
+        assertThat(response.getAllFieldStats().get("location").getMinValue(), equalTo(null));
+        assertThat(response.getAllFieldStats().get("location").getMaxValue(), equalTo(null));
+        assertThat(response.getAllFieldStats().get("location").isAggregatable(), equalTo(true));
+        assertThat(response.getAllFieldStats().get("location").isSearchable(), equalTo(false));
+
+    }
+
     private void indexRange(String index, long from, long to) throws Exception {
         indexRange(index, "value", from, to);
     }
diff --git a/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsTests.java b/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsTests.java
index 4a5f79a12a827..d47f04a56fd7e 100644
--- a/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsTests.java
+++ b/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsTests.java
@@ -19,17 +19,31 @@
 
 package org.elasticsearch.fieldstats;
 
+import org.apache.lucene.geo.GeoTestUtil;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.fieldstats.FieldStats;
 import org.elasticsearch.action.fieldstats.FieldStatsResponse;
 import org.elasticsearch.action.fieldstats.IndexConstraint;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.joda.Joda;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.mapper.DateFieldMapper;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.elasticsearch.test.InternalSettingsPlugin;
+import org.elasticsearch.test.VersionUtils;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
 import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Date;
 import java.util.List;
 import java.util.Locale;
 
@@ -40,11 +54,15 @@
 import static org.elasticsearch.action.fieldstats.IndexConstraint.Property.MAX;
 import static org.elasticsearch.action.fieldstats.IndexConstraint.Property.MIN;
 import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.equalTo;
 
 /**
  */
 public class FieldStatsTests extends ESSingleNodeTestCase {
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return pluginList(InternalSettingsPlugin.class);
+    }
+
     public void testByte() {
         testNumberRange("field1", "byte", 12, 18);
         testNumberRange("field1", "byte", -5, 5);
@@ -69,83 +87,157 @@ public void testLong() {
         testNumberRange("field1", "long", -312321312312422L, -312321312312412L);
     }
 
+    private static String makeType(String type, boolean indexed, boolean docValues, boolean stored) {
+        return new StringBuilder()
+            .append("type=").append(type)
+            .append(",index=").append(indexed)
+            .append(",doc_values=").append(docValues)
+            .append(",store=").append(stored).toString();
+    }
+
     public void testString() {
-        createIndex("test", Settings.EMPTY, "test", "field", "type=text");
+        createIndex("test", Settings.EMPTY, "test",
+            "field_index", makeType("keyword", true, false, false),
+            "field_dv", makeType("keyword", false, true, false),
+            "field_stored", makeType("keyword", false, true, true),
+            "field_source", makeType("keyword", false, false, false));
         for (int value = 0; value <= 10; value++) {
-            client().prepareIndex("test", "test").setSource("field",
-                String.format(Locale.ENGLISH, "%03d", value)).get();
+            String keyword =  String.format(Locale.ENGLISH, "%03d", value);
+            client().prepareIndex("test", "test")
+                .setSource("field_index", keyword,
+                    "field_dv", keyword,
+                    "field_stored", keyword,
+                    "field_source", keyword).get();
         }
         client().admin().indices().prepareRefresh().get();
 
-        FieldStatsResponse result = client().prepareFieldStats().setFields("field").get();
-        assertThat(result.getAllFieldStats().get("field").getMaxDoc(), equalTo(11L));
-        assertThat(result.getAllFieldStats().get("field").getDocCount(), equalTo(11L));
-        assertThat(result.getAllFieldStats().get("field").getDensity(), equalTo(100));
-        assertThat(result.getAllFieldStats().get("field").getMinValue(),
-            equalTo(new BytesRef(String.format(Locale.ENGLISH, "%03d", 0))));
-        assertThat(result.getAllFieldStats().get("field").getMaxValue(),
-            equalTo(new BytesRef(String.format(Locale.ENGLISH, "%03d", 10))));
-        assertThat(result.getAllFieldStats().get("field").getMinValueAsString(),
-            equalTo(String.format(Locale.ENGLISH, "%03d", 0)));
-        assertThat(result.getAllFieldStats().get("field").getMaxValueAsString(),
-            equalTo(String.format(Locale.ENGLISH, "%03d", 10)));
-        assertThat(result.getAllFieldStats().get("field").getDisplayType(),
-            equalTo("string"));
+        FieldStatsResponse result = client().prepareFieldStats()
+            .setFields("field_index", "field_dv", "field_stored", "field_source").get();
+        assertEquals(result.getAllFieldStats().size(), 3);
+        for (String field : new String[] {"field_index", "field_dv", "field_stored"}) {
+            FieldStats stats = result.getAllFieldStats().get(field);
+            assertEquals(stats.getMaxDoc(), 11L);
+            assertEquals(stats.getDisplayType(),
+                "string");
+            if ("field_index".equals(field)) {
+                assertEquals(stats.getMinValue(),
+                    new BytesRef(String.format(Locale.ENGLISH, "%03d", 0)));
+                assertEquals(stats.getMaxValue(),
+                    new BytesRef(String.format(Locale.ENGLISH, "%03d", 10)));
+                assertEquals(stats.getMinValueAsString(),
+                    String.format(Locale.ENGLISH, "%03d", 0));
+                assertEquals(stats.getMaxValueAsString(),
+                    String.format(Locale.ENGLISH, "%03d", 10));
+                assertEquals(stats.getDocCount(), 11L);
+                assertEquals(stats.getDensity(), 100);
+            } else {
+                assertEquals(stats.getDocCount(), 0L);
+                assertNull(stats.getMinValue());
+                assertNull(stats.getMaxValue());
+                assertEquals(stats.getDensity(), 0);
+            }
+        }
     }
 
     public void testDouble() {
-        String fieldName = "field";
-        createIndex("test", Settings.EMPTY, "test", fieldName, "type=double");
+        createIndex("test", Settings.EMPTY, "test",
+            "field_index", makeType("double", true, false, false),
+            "field_dv", makeType("double", false, true, false),
+            "field_stored", makeType("double", false, true, true),
+            "field_source", makeType("double", false, false, false));
         for (double value = -1; value <= 9; value++) {
-            client().prepareIndex("test", "test").setSource(fieldName, value).get();
+            client().prepareIndex("test", "test")
+                .setSource("field_index", value, "field_dv", value, "field_stored", value, "field_source", value).get();
         }
         client().admin().indices().prepareRefresh().get();
-
-        FieldStatsResponse result = client().prepareFieldStats().setFields(fieldName).get();
-        assertThat(result.getAllFieldStats().get(fieldName).getMaxDoc(), equalTo(11L));
-        assertThat(result.getAllFieldStats().get(fieldName).getDocCount(), equalTo(11L));
-        assertThat(result.getAllFieldStats().get(fieldName).getDensity(), equalTo(100));
-        assertThat(result.getAllFieldStats().get(fieldName).getMinValue(), equalTo(-1d));
-        assertThat(result.getAllFieldStats().get(fieldName).getMaxValue(), equalTo(9d));
-        assertThat(result.getAllFieldStats().get(fieldName).getMinValueAsString(), equalTo(Double.toString(-1)));
-        assertThat(result.getAllFieldStats().get(fieldName).getDisplayType(), equalTo("float"));
+        FieldStatsResponse result = client().prepareFieldStats()
+            .setFields("field_index", "field_dv", "field_stored", "field_source").get();
+        for (String field : new String[] {"field_index", "field_dv", "field_stored"}) {
+            FieldStats stats = result.getAllFieldStats().get(field);
+            assertEquals(stats.getMaxDoc(), 11L);
+            assertEquals(stats.getDisplayType(), "float");
+            if ("field_index".equals(field)) {
+                assertEquals(stats.getDocCount(), 11L);
+                assertEquals(stats.getDensity(), 100);
+                assertEquals(stats.getMinValue(), -1d);
+                assertEquals(stats.getMaxValue(), 9d);
+                assertEquals(stats.getMinValueAsString(), Double.toString(-1));
+            } else {
+                assertEquals(stats.getDocCount(), 0L);
+                assertNull(stats.getMinValue());
+                assertNull(stats.getMaxValue());
+                assertEquals(stats.getDensity(), 0);
+            }
+        }
     }
 
     public void testHalfFloat() {
-        String fieldName = "field";
-        createIndex("test", Settings.EMPTY, "test", fieldName, "type=half_float");
+        createIndex("test", Settings.EMPTY, "test",
+            "field_index", makeType("half_float", true, false, false),
+            "field_dv", makeType("half_float", false, true, false),
+            "field_stored", makeType("half_float", false, true, true),
+            "field_source", makeType("half_float", false, false, false));
         for (float value = -1; value <= 9; value++) {
-            client().prepareIndex("test", "test").setSource(fieldName, value).get();
+            client().prepareIndex("test", "test")
+                .setSource("field_index", value, "field_dv", value, "field_stored", value, "field_source", value).get();
         }
         client().admin().indices().prepareRefresh().get();
 
-        FieldStatsResponse result = client().prepareFieldStats().setFields(fieldName).get();
-        assertThat(result.getAllFieldStats().get(fieldName).getMaxDoc(), equalTo(11L));
-        assertThat(result.getAllFieldStats().get(fieldName).getDocCount(), equalTo(11L));
-        assertThat(result.getAllFieldStats().get(fieldName).getDensity(), equalTo(100));
-        assertThat(result.getAllFieldStats().get(fieldName).getMinValue(), equalTo(-1d));
-        assertThat(result.getAllFieldStats().get(fieldName).getMaxValue(), equalTo(9d));
-        assertThat(result.getAllFieldStats().get(fieldName).getMinValueAsString(), equalTo(Float.toString(-1)));
-        assertThat(result.getAllFieldStats().get(fieldName).getMaxValueAsString(), equalTo(Float.toString(9)));
-        assertThat(result.getAllFieldStats().get(fieldName).getDisplayType(), equalTo("float"));
+        FieldStatsResponse result = client().prepareFieldStats()
+            .setFields("field_index", "field_dv", "field_stored", "field_source").get();
+        for (String field : new String[] {"field_index", "field_dv", "field_stored"}) {
+            FieldStats stats = result.getAllFieldStats().get(field);
+            assertEquals(stats.getMaxDoc(), 11L);
+            assertEquals(stats.getDisplayType(), "float");
+            if (field.equals("field_index")) {
+                assertEquals(stats.getDocCount(), 11L);
+                assertEquals(stats.getDensity(), 100);
+                assertEquals(stats.getMinValue(), -1d);
+                assertEquals(stats.getMaxValue(), 9d);
+                assertEquals(stats.getMinValueAsString(), Float.toString(-1));
+                assertEquals(stats.getMaxValueAsString(), Float.toString(9));
+            } else {
+                assertEquals(stats.getDocCount(), 0L);
+                assertNull(stats.getMinValue());
+                assertNull(stats.getMaxValue());
+                assertEquals(stats.getDensity(), 0);
+            }
+        }
     }
 
     public void testFloat() {
         String fieldName = "field";
-        createIndex("test", Settings.EMPTY, "test", fieldName, "type=float");
+        createIndex("test", Settings.EMPTY, "test",
+            "field_index", makeType("float", true, false, false),
+            "field_dv", makeType("float", false, true, false),
+            "field_stored", makeType("float", false, true, true),
+            "field_source", makeType("float", false, false, false));
         for (float value = -1; value <= 9; value++) {
-            client().prepareIndex("test", "test").setSource(fieldName, value).get();
+            client().prepareIndex("test", "test")
+                .setSource("field_index", value, "field_dv", value, "field_stored", value, "field_source", value).get();
         }
         client().admin().indices().prepareRefresh().get();
 
-        FieldStatsResponse result = client().prepareFieldStats().setFields(fieldName).get();
-        assertThat(result.getAllFieldStats().get(fieldName).getMaxDoc(), equalTo(11L));
-        assertThat(result.getAllFieldStats().get(fieldName).getDocCount(), equalTo(11L));
-        assertThat(result.getAllFieldStats().get(fieldName).getDensity(), equalTo(100));
-        assertThat(result.getAllFieldStats().get(fieldName).getMinValue(), equalTo(-1d));
-        assertThat(result.getAllFieldStats().get(fieldName).getMaxValue(), equalTo(9d));
-        assertThat(result.getAllFieldStats().get(fieldName).getMinValueAsString(), equalTo(Float.toString(-1)));
-        assertThat(result.getAllFieldStats().get(fieldName).getMaxValueAsString(), equalTo(Float.toString(9)));
+        FieldStatsResponse result = client().prepareFieldStats()
+            .setFields("field_index", "field_dv", "field_stored", "field_source").get();
+        for (String field : new String[]{"field_index", "field_dv", "field_stored"}) {
+            FieldStats stats = result.getAllFieldStats().get(field);
+            assertEquals(stats.getMaxDoc(), 11L);
+            assertEquals(stats.getDisplayType(), "float");
+            if (field.equals("field_index")) {
+                assertEquals(stats.getDocCount(), 11L);
+                assertEquals(stats.getDensity(), 100);
+                assertEquals(stats.getMinValue(), -1d);
+                assertEquals(stats.getMaxValue(), 9d);
+                assertEquals(stats.getMinValueAsString(), Float.toString(-1));
+                assertEquals(stats.getMaxValueAsString(), Float.toString(9));
+            } else {
+                assertEquals(stats.getDocCount(), 0L);
+                assertNull(stats.getMinValue());
+                assertNull(stats.getMaxValue());
+                assertEquals(stats.getDensity(), 0);
+            }
+        }
     }
 
     private void testNumberRange(String fieldName, String fieldType, long min, long max) {
@@ -162,21 +254,21 @@ private void testNumberRange(String fieldName, String fieldType, long min, long
 
         FieldStatsResponse result = client().prepareFieldStats().setFields(fieldName).get();
         long numDocs = max - min + 1;
-        assertThat(result.getAllFieldStats().get(fieldName).getMaxDoc(), equalTo(numDocs));
-        assertThat(result.getAllFieldStats().get(fieldName).getDocCount(), equalTo(numDocs));
-        assertThat(result.getAllFieldStats().get(fieldName).getDensity(), equalTo(100));
-        assertThat(result.getAllFieldStats().get(fieldName).getMinValue(), equalTo(min));
-        assertThat(result.getAllFieldStats().get(fieldName).getMaxValue(), equalTo(max));
-        assertThat(result.getAllFieldStats().get(fieldName).getMinValueAsString(),
-            equalTo(java.lang.Long.toString(min)));
-        assertThat(result.getAllFieldStats().get(fieldName).getMaxValueAsString(),
-            equalTo(java.lang.Long.toString(max)));
-        assertThat(result.getAllFieldStats().get(fieldName).isSearchable(), equalTo(true));
-        assertThat(result.getAllFieldStats().get(fieldName).isAggregatable(), equalTo(true));
+        assertEquals(result.getAllFieldStats().get(fieldName).getMaxDoc(), numDocs);
+        assertEquals(result.getAllFieldStats().get(fieldName).getDocCount(), numDocs);
+        assertEquals(result.getAllFieldStats().get(fieldName).getDensity(), 100);
+        assertEquals(result.getAllFieldStats().get(fieldName).getMinValue(), min);
+        assertEquals(result.getAllFieldStats().get(fieldName).getMaxValue(), max);
+        assertEquals(result.getAllFieldStats().get(fieldName).getMinValueAsString(),
+            java.lang.Long.toString(min));
+        assertEquals(result.getAllFieldStats().get(fieldName).getMaxValueAsString(),
+            java.lang.Long.toString(max));
+        assertEquals(result.getAllFieldStats().get(fieldName).isSearchable(), true);
+        assertEquals(result.getAllFieldStats().get(fieldName).isAggregatable(), true);
         if (fieldType.equals("float") || fieldType.equals("double") || fieldType.equals("half-float")) {
-            assertThat(result.getAllFieldStats().get(fieldName).getDisplayType(), equalTo("float"));
+            assertEquals(result.getAllFieldStats().get(fieldName).getDisplayType(), "float");
         } else {
-            assertThat(result.getAllFieldStats().get(fieldName).getDisplayType(), equalTo("integer"));
+            assertEquals(result.getAllFieldStats().get(fieldName).getDisplayType(), "integer");
         }
 
         client().admin().indices().prepareDelete("test").get();
@@ -189,18 +281,19 @@ public void testMerge() {
         stats.add(new FieldStats.Long(1, 1L, 1L, 1L, true, false, 1L, 1L));
         stats.add(new FieldStats.Long(1, 1L, 1L, 1L, true, false, 1L, 1L));
         stats.add(new FieldStats.Long(1, 1L, 1L, 1L, true, false, 1L, 1L));
+        stats.add(new FieldStats.Long(0, 0, 0, 0, false, false));
 
         FieldStats stat = new FieldStats.Long(1, 1L, 1L, 1L, true, false, 1L, 1L);
         for (FieldStats otherStat : stats) {
             stat.accumulate(otherStat);
         }
-        assertThat(stat.getMaxDoc(), equalTo(4L));
-        assertThat(stat.getDocCount(), equalTo(4L));
-        assertThat(stat.getSumDocFreq(), equalTo(4L));
-        assertThat(stat.getSumTotalTermFreq(), equalTo(4L));
-        assertThat(stat.isSearchable(), equalTo(true));
-        assertThat(stat.isAggregatable(), equalTo(false));
-        assertThat(stat.getDisplayType(), equalTo("integer"));
+        assertEquals(stat.getMaxDoc(), 4L);
+        assertEquals(stat.getDocCount(), 4L);
+        assertEquals(stat.getSumDocFreq(), 4L);
+        assertEquals(stat.getSumTotalTermFreq(), 4L);
+        assertEquals(stat.isSearchable(), true);
+        assertEquals(stat.isAggregatable(), false);
+        assertEquals(stat.getDisplayType(), "integer");
     }
 
     public void testMerge_notAvailable() {
@@ -213,26 +306,28 @@ public void testMerge_notAvailable() {
         for (FieldStats otherStat : stats) {
             stat.accumulate(otherStat);
         }
-        assertThat(stat.getMaxDoc(), equalTo(4L));
-        assertThat(stat.getDocCount(), equalTo(-1L));
-        assertThat(stat.getSumDocFreq(), equalTo(-1L));
-        assertThat(stat.getSumTotalTermFreq(), equalTo(-1L));
-        assertThat(stat.isSearchable(), equalTo(true));
-        assertThat(stat.isAggregatable(), equalTo(true));
-        assertThat(stat.getDisplayType(), equalTo("integer"));
-
-        stats.add(new FieldStats.Long(1, -1L, -1L, -1L, true, true, 1L, 1L));
+        assertEquals(stat.getMaxDoc(), 4L);
+        assertEquals(stat.getDocCount(), -1L);
+        assertEquals(stat.getSumDocFreq(), -1L);
+        assertEquals(stat.getSumTotalTermFreq(), -1L);
+        assertEquals(stat.isSearchable(), true);
+        assertEquals(stat.isAggregatable(), true);
+        assertEquals(stat.getDisplayType(), "integer");
+
+        stats.add(new FieldStats.Long(1, -1L, -1L, -1L, false, true));
         stat = stats.remove(0);
         for (FieldStats otherStat : stats) {
             stat.accumulate(otherStat);
         }
-        assertThat(stat.getMaxDoc(), equalTo(4L));
-        assertThat(stat.getDocCount(), equalTo(-1L));
-        assertThat(stat.getSumDocFreq(), equalTo(-1L));
-        assertThat(stat.getSumTotalTermFreq(), equalTo(-1L));
-        assertThat(stat.isSearchable(), equalTo(true));
-        assertThat(stat.isAggregatable(), equalTo(true));
-        assertThat(stat.getDisplayType(), equalTo("integer"));
+        assertEquals(stat.getMaxDoc(), 4L);
+        assertEquals(stat.getDocCount(), -1L);
+        assertEquals(stat.getSumDocFreq(), -1L);
+        assertEquals(stat.getSumTotalTermFreq(), -1L);
+        assertEquals(stat.isSearchable(), true);
+        assertEquals(stat.isAggregatable(), true);
+        assertEquals(stat.getDisplayType(), "integer");
+        assertNull(stat.getMaxValue());
+        assertNull(stat.getMinValue());
     }
 
     public void testNumberFiltering() {
@@ -246,9 +341,9 @@ public void testNumberFiltering() {
                 .setFields("value")
                 .setLevel("indices")
                 .get();
-        assertThat(response.getIndicesMergedFieldStats().size(), equalTo(2));
-        assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(), equalTo(1L));
-        assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), equalTo(3L));
+        assertEquals(response.getIndicesMergedFieldStats().size(), 2);
+        assertEquals(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(), 1L);
+        assertEquals(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), 3L);
 
         response = client().prepareFieldStats()
                 .setFields("value")
@@ -256,7 +351,7 @@ public void testNumberFiltering() {
                     new IndexConstraint("value", MAX, LTE, "0"))
                 .setLevel("indices")
                 .get();
-        assertThat(response.getIndicesMergedFieldStats().size(), equalTo(0));
+        assertEquals(response.getIndicesMergedFieldStats().size(), 0);
 
         response = client().prepareFieldStats()
                 .setFields("value")
@@ -264,7 +359,7 @@ public void testNumberFiltering() {
                     new IndexConstraint("value", MAX, LT, "1"))
                 .setLevel("indices")
                 .get();
-        assertThat(response.getIndicesMergedFieldStats().size(), equalTo(0));
+        assertEquals(response.getIndicesMergedFieldStats().size(), 0);
 
         response = client().prepareFieldStats()
                 .setFields("value")
@@ -272,8 +367,8 @@ public void testNumberFiltering() {
                     new IndexConstraint("value", MAX, LTE, "1"))
                 .setLevel("indices")
                 .get();
-        assertThat(response.getIndicesMergedFieldStats().size(), equalTo(1));
-        assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(), equalTo(1L));
+        assertEquals(response.getIndicesMergedFieldStats().size(), 1);
+        assertEquals(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(), 1L);
 
         response = client().prepareFieldStats()
                 .setFields("value")
@@ -281,8 +376,8 @@ public void testNumberFiltering() {
                     new IndexConstraint("value", MAX, LTE,  "2"))
                 .setLevel("indices")
                 .get();
-        assertThat(response.getIndicesMergedFieldStats().size(), equalTo(1));
-        assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(), equalTo(1L));
+        assertEquals(response.getIndicesMergedFieldStats().size(), 1);
+        assertEquals(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(), 1L);
 
         response = client().prepareFieldStats()
                 .setFields("value")
@@ -290,7 +385,7 @@ public void testNumberFiltering() {
                     new IndexConstraint("value", MAX, LTE, "2"))
                 .setLevel("indices")
                 .get();
-        assertThat(response.getIndicesMergedFieldStats().size(), equalTo(0));
+        assertEquals(response.getIndicesMergedFieldStats().size(), 0);
 
         response = client().prepareFieldStats()
                 .setFields("value")
@@ -298,8 +393,8 @@ public void testNumberFiltering() {
                     new IndexConstraint("value", MAX, LTE, "3"))
                 .setLevel("indices")
                 .get();
-        assertThat(response.getIndicesMergedFieldStats().size(), equalTo(1));
-        assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), equalTo(3L));
+        assertEquals(response.getIndicesMergedFieldStats().size(), 1);
+        assertEquals(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), 3L);
 
         response = client().prepareFieldStats()
                 .setFields("value")
@@ -307,8 +402,8 @@ public void testNumberFiltering() {
                     new IndexConstraint("value", MAX, LTE, "4"))
                 .setLevel("indices")
                 .get();
-        assertThat(response.getIndicesMergedFieldStats().size(), equalTo(1));
-        assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), equalTo(3L));
+        assertEquals(response.getIndicesMergedFieldStats().size(), 1);
+        assertEquals(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), 3L);
 
         response = client().prepareFieldStats()
                 .setFields("value")
@@ -316,7 +411,7 @@ public void testNumberFiltering() {
                     new IndexConstraint("value", MAX, LTE, "4"))
                 .setLevel("indices")
                 .get();
-        assertThat(response.getIndicesMergedFieldStats().size(), equalTo(0));
+        assertEquals(response.getIndicesMergedFieldStats().size(), 0);
 
         response = client().prepareFieldStats()
                 .setFields("value")
@@ -324,9 +419,9 @@ public void testNumberFiltering() {
                     new IndexConstraint("value", MAX, LTE, "3"))
                 .setLevel("indices")
                 .get();
-        assertThat(response.getIndicesMergedFieldStats().size(), equalTo(2));
-        assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(), equalTo(1L));
-        assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), equalTo(3L));
+        assertEquals(response.getIndicesMergedFieldStats().size(), 2);
+        assertEquals(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(), 1L);
+        assertEquals(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), 3L);
 
         response = client().prepareFieldStats()
                 .setFields("value")
@@ -334,7 +429,7 @@ public void testNumberFiltering() {
                     new IndexConstraint("value", MAX, LT, "3"))
                 .setLevel("indices")
                 .get();
-        assertThat(response.getIndicesMergedFieldStats().size(), equalTo(0));
+        assertEquals(response.getIndicesMergedFieldStats().size(), 0);
     }
 
     public void testDateFiltering() {
@@ -343,8 +438,9 @@ public void testDateFiltering() {
         DateTime dateTime2 = new DateTime(2014, 1, 2, 0, 0, 0, 0, DateTimeZone.UTC);
         String dateTime2Str = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parser().print(dateTime2);
 
-        createIndex("test1", Settings.EMPTY, "type", "value", "type=date");
-        client().prepareIndex("test1", "test").setSource("value", dateTime1Str).get();
+        createIndex("test1", Settings.EMPTY, "type", "value", "type=date", "value2", "type=date,index=false");
+        client().prepareIndex("test1", "test")
+            .setSource("value", dateTime1Str, "value2", dateTime1Str).get();
         createIndex("test2", Settings.EMPTY, "type", "value", "type=date");
         client().prepareIndex("test2", "test").setSource("value", dateTime2Str).get();
         client().admin().indices().prepareRefresh().get();
@@ -353,17 +449,17 @@ public void testDateFiltering() {
                 .setFields("value")
                 .setLevel("indices")
                 .get();
-        assertThat(response.getIndicesMergedFieldStats().size(), equalTo(2));
-        assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(),
-            equalTo(dateTime1.getMillis()));
-        assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(),
-            equalTo(dateTime2.getMillis()));
-        assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValueAsString(),
-            equalTo(dateTime1Str));
-        assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValueAsString(),
-            equalTo(dateTime2Str));
-        assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getDisplayType(),
-            equalTo("date"));
+        assertEquals(response.getIndicesMergedFieldStats().size(), 2);
+        assertEquals(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(),
+            dateTime1.getMillis());
+        assertEquals(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(),
+            dateTime2.getMillis());
+        assertEquals(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValueAsString(),
+            dateTime1Str);
+        assertEquals(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValueAsString(),
+            dateTime2Str);
+        assertEquals(response.getIndicesMergedFieldStats().get("test2").get("value").getDisplayType(),
+            "date");
 
         response = client().prepareFieldStats()
                 .setFields("value")
@@ -371,7 +467,7 @@ public void testDateFiltering() {
                     new IndexConstraint("value", MAX, LTE, "2013-12-31T00:00:00.000Z"))
                 .setLevel("indices")
                 .get();
-        assertThat(response.getIndicesMergedFieldStats().size(), equalTo(0));
+        assertEquals(response.getIndicesMergedFieldStats().size(), 0);
 
         response = client().prepareFieldStats()
                 .setFields("value")
@@ -379,13 +475,13 @@ public void testDateFiltering() {
                     new IndexConstraint("value", MAX, LTE, "2014-01-01T00:00:00.000Z"))
                 .setLevel("indices")
                 .get();
-        assertThat(response.getIndicesMergedFieldStats().size(), equalTo(1));
-        assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(),
-            equalTo(dateTime1.getMillis()));
-        assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValueAsString(),
-            equalTo(dateTime1Str));
-        assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getDisplayType(),
-            equalTo("date"));
+        assertEquals(response.getIndicesMergedFieldStats().size(), 1);
+        assertEquals(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(),
+            dateTime1.getMillis());
+        assertEquals(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValueAsString(),
+            dateTime1Str);
+        assertEquals(response.getIndicesMergedFieldStats().get("test1").get("value").getDisplayType(),
+            "date");
 
         response = client().prepareFieldStats()
                 .setFields("value")
@@ -393,11 +489,11 @@ public void testDateFiltering() {
                     new IndexConstraint("value", MAX, LTE, "2014-01-02T00:00:00.000Z"))
                 .setLevel("indices")
                 .get();
-        assertThat(response.getIndicesMergedFieldStats().size(), equalTo(1));
-        assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(),
-            equalTo(dateTime2.getMillis()));
-        assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValueAsString(),
-            equalTo(dateTime2Str));
+        assertEquals(response.getIndicesMergedFieldStats().size(), 1);
+        assertEquals(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(),
+            dateTime2.getMillis());
+        assertEquals(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValueAsString(),
+            dateTime2Str);
 
         response = client().prepareFieldStats()
                 .setFields("value")
@@ -405,7 +501,7 @@ public void testDateFiltering() {
                     new IndexConstraint("value", MAX, LTE, "2014-01-03T00:00:00.000Z"))
                 .setLevel("indices")
                 .get();
-        assertThat(response.getIndicesMergedFieldStats().size(), equalTo(0));
+        assertEquals(response.getIndicesMergedFieldStats().size(), 0);
 
         response = client().prepareFieldStats()
                 .setFields("value")
@@ -413,47 +509,53 @@ public void testDateFiltering() {
                     new IndexConstraint("value", MAX, LTE, "2014-01-02T01:00:00.000Z"))
                 .setLevel("indices")
                 .get();
-        assertThat(response.getIndicesMergedFieldStats().size(), equalTo(1));
-        assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(),
-            equalTo(dateTime2.getMillis()));
-        assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValueAsString(),
-            equalTo(dateTime2Str));
-        assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getDisplayType(),
-            equalTo("date"));
+        assertEquals(response.getIndicesMergedFieldStats().size(), 1);
+        assertEquals(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(),
+            dateTime2.getMillis());
+        assertEquals(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValueAsString(),
+            dateTime2Str);
+        assertEquals(response.getIndicesMergedFieldStats().get("test2").get("value").getDisplayType(),
+            "date");
 
         response = client().prepareFieldStats()
                 .setFields("value")
                 .setIndexContraints(new IndexConstraint("value", MIN, GTE, "2014-01-01T00:00:00.000Z"))
                 .setLevel("indices")
                 .get();
-        assertThat(response.getIndicesMergedFieldStats().size(), equalTo(2));
-        assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(),
-            equalTo(dateTime1.getMillis()));
-        assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(),
-            equalTo(dateTime2.getMillis()));
-        assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValueAsString(),
-            equalTo(dateTime1Str));
-        assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValueAsString(),
-            equalTo(dateTime2Str));
-        assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getDisplayType(),
-            equalTo("date"));
+        assertEquals(response.getIndicesMergedFieldStats().size(), 2);
+        assertEquals(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(),
+            dateTime1.getMillis());
+        assertEquals(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(),
+            dateTime2.getMillis());
+        assertEquals(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValueAsString(),
+            dateTime1Str);
+        assertEquals(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValueAsString(),
+            dateTime2Str);
+        assertEquals(response.getIndicesMergedFieldStats().get("test2").get("value").getDisplayType(),
+            "date");
 
         response = client().prepareFieldStats()
                 .setFields("value")
                 .setIndexContraints(new IndexConstraint("value", MAX, LTE, "2014-01-02T00:00:00.000Z"))
                 .setLevel("indices")
                 .get();
-        assertThat(response.getIndicesMergedFieldStats().size(), equalTo(2));
-        assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(),
-            equalTo(dateTime1.getMillis()));
-        assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(),
-            equalTo(dateTime2.getMillis()));
-        assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValueAsString(),
-            equalTo(dateTime1Str));
-        assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValueAsString(),
-            equalTo(dateTime2Str));
-        assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getDisplayType(),
-            equalTo("date"));
+        assertEquals(response.getIndicesMergedFieldStats().size(), 2);
+        assertEquals(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(),
+            dateTime1.getMillis());
+        assertEquals(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(),
+            dateTime2.getMillis());
+        assertEquals(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValueAsString(),
+            dateTime1Str);
+        assertEquals(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValueAsString(),
+            dateTime2Str);
+        assertEquals(response.getIndicesMergedFieldStats().get("test2").get("value").getDisplayType(), "date");
+
+        response = client().prepareFieldStats()
+            .setFields("value2")
+            .setIndexContraints(new IndexConstraint("value2", MAX, LTE, "2014-01-02T00:00:00.000Z"))
+            .setLevel("indices")
+            .get();
+        assertEquals(response.getIndicesMergedFieldStats().size(), 0);
     }
 
     public void testDateFiltering_optionalFormat() {
@@ -472,11 +574,11 @@ public void testDateFiltering_optionalFormat() {
                     new IndexConstraint("value", MAX, LTE, String.valueOf(dateTime2.getMillis()), "epoch_millis"))
                 .setLevel("indices")
                 .get();
-        assertThat(response.getIndicesMergedFieldStats().size(), equalTo(1));
-        assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValueAsString(),
-            equalTo("2014-01-02T00:00:00.000Z"));
-        assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getDisplayType(),
-            equalTo("date"));
+        assertEquals(response.getIndicesMergedFieldStats().size(), 1);
+        assertEquals(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValueAsString(),
+            "2014-01-02T00:00:00.000Z");
+        assertEquals(response.getIndicesMergedFieldStats().get("test2").get("value").getDisplayType(),
+            "date");
 
         try {
             client().prepareFieldStats()
@@ -497,8 +599,8 @@ public void testEmptyIndex() {
             .setFields("*")
             .setLevel("indices")
             .get();
-        assertThat(response.getIndicesMergedFieldStats().size(), equalTo(1));
-        assertThat(response.getIndicesMergedFieldStats().get("test1").size(), equalTo(0));
+        assertEquals(response.getIndicesMergedFieldStats().size(), 1);
+        assertEquals(response.getIndicesMergedFieldStats().get("test1").size(), 0);
     }
 
     public void testMetaFieldsNotIndexed() {
@@ -509,8 +611,132 @@ public void testMetaFieldsNotIndexed() {
         FieldStatsResponse response = client().prepareFieldStats()
             .setFields("_id", "_type")
             .get();
-        assertThat(response.getAllFieldStats().size(), equalTo(1));
-        assertThat(response.getAllFieldStats().get("_type").isSearchable(), equalTo(true));
-        assertThat(response.getAllFieldStats().get("_type").isAggregatable(), equalTo(true));
+        assertEquals(response.getAllFieldStats().size(), 1);
+        assertEquals(response.getAllFieldStats().get("_type").isSearchable(), true);
+        assertEquals(response.getAllFieldStats().get("_type").isAggregatable(), true);
+    }
+
+    public void testSerialization() throws IOException {
+        for (Version version : new Version[] {Version.CURRENT, Version.V_5_0_1}){
+            for (int i = 0; i < 20; i++) {
+                assertSerialization(randomFieldStats(version.onOrAfter(Version.V_5_2_0)), version);
+            }
+        }
+    }
+
+    /**
+     * creates a random field stats which does not guarantee that {@link FieldStats#maxValue} is greater than {@link FieldStats#minValue}
+     **/
+    public static FieldStats randomFieldStats(boolean withNullMinMax) throws UnknownHostException {
+        int type = randomInt(5);
+        switch (type) {
+            case 0:
+                if (withNullMinMax && randomBoolean()) {
+                    return new FieldStats.Long(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
+                        randomNonNegativeLong(), randomBoolean(), randomBoolean());
+                } else {
+                    return new FieldStats.Long(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
+                        randomNonNegativeLong(), randomBoolean(), randomBoolean(), randomLong(), randomLong());
+                }
+            case 1:
+                if (withNullMinMax && randomBoolean()) {
+                    return new FieldStats.Double(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
+                        randomNonNegativeLong(), randomBoolean(), randomBoolean());
+                } else {
+                    return new FieldStats.Double(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
+                        randomNonNegativeLong(), randomBoolean(), randomBoolean(), randomDouble(), randomDouble());
+                }
+            case 2:
+                if (withNullMinMax && randomBoolean()) {
+                    return new FieldStats.Date(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
+                        randomNonNegativeLong(), randomBoolean(), randomBoolean());
+                } else {
+                    return new FieldStats.Date(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
+                        randomNonNegativeLong(), randomBoolean(), randomBoolean(), Joda.forPattern("basicDate"),
+                        new Date().getTime(), new Date().getTime());
+                }
+            case 3:
+                if (withNullMinMax && randomBoolean()) {
+                    return new FieldStats.Text(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
+                        randomNonNegativeLong(), randomBoolean(), randomBoolean());
+                } else {
+                    return new FieldStats.Text(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
+                        randomNonNegativeLong(), randomBoolean(), randomBoolean(),
+                        new BytesRef(randomAlphaOfLength(10)), new BytesRef(randomAlphaOfLength(20)));
+                }
+            case 4:
+                if (withNullMinMax && randomBoolean()) {
+                    return new FieldStats.Ip(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
+                        randomNonNegativeLong(), randomBoolean(), randomBoolean());
+                } else {
+                    return new FieldStats.Ip(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
+                        randomNonNegativeLong(), randomBoolean(), randomBoolean(),
+                        InetAddress.getByName("::1"), InetAddress.getByName("::1"));
+                }
+            case 5:
+                if (withNullMinMax && randomBoolean()) {
+                    return new FieldStats.Ip(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
+                        randomNonNegativeLong(), randomBoolean(), randomBoolean());
+                } else {
+                    return new FieldStats.Ip(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
+                        randomNonNegativeLong(), randomBoolean(), randomBoolean(),
+                        InetAddress.getByName("1.2.3.4"), InetAddress.getByName("1.2.3.4"));
+                }
+            default:
+                throw new IllegalArgumentException("Invalid type");
+        }
+    }
+
+    public void testGeopoint2x() {
+        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.V_2_4_5);
+        Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
+        createIndex("test", settings, "test",
+            "field_index", makeType("geo_point", true, false, false));
+        int numDocs = random().nextInt(20);
+        for (int i = 0; i <= numDocs; ++i) {
+            double lat = GeoTestUtil.nextLatitude();
+            double lon = GeoTestUtil.nextLongitude();
+            final String src = lat + "," + lon;
+            client().prepareIndex("test", "test").setSource("field_index", src).get();
+        }
+
+        client().admin().indices().prepareRefresh().get();
+        FieldStatsResponse result = client().prepareFieldStats().setFields("field_index").get();
+        FieldStats stats = result.getAllFieldStats().get("field_index");
+        assertEquals(stats.getDisplayType(), "geo_point");
+        // min/max random testing is not straightforward; there are 3 different encodings since V_2_0
+        // e.g., before V2_3 used legacy numeric encoding which is wildly different from V_2_3 which is morton encoded
+        // which is wildly different from V_5_0 which is point encoded. Skipping min/max in favor of testing
+    }
+
+    public void testGeopoint5x() {
+        Version version = VersionUtils.randomVersionBetween(random(), Version.V_5_0_0, Version.CURRENT);
+        Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
+        createIndex("test", settings, "test",
+            "field_index", makeType("geo_point", true, false, false));
+        int numDocs = random().nextInt(20);
+        for (int i = 0; i <= numDocs; ++i) {
+            double lat = GeoTestUtil.nextLatitude();
+            double lon = GeoTestUtil.nextLongitude();
+            final String src = lat + "," + lon;
+            client().prepareIndex("test", "test").setSource("field_index", src).get();
+        }
+
+        client().admin().indices().prepareRefresh().get();
+        FieldStatsResponse result = client().prepareFieldStats().setFields("field_index").get();
+        FieldStats stats = result.getAllFieldStats().get("field_index");
+        assertEquals(stats.getDisplayType(), "geo_point");
+    }
+
+    private void assertSerialization(FieldStats stats, Version version) throws IOException {
+        BytesStreamOutput output = new BytesStreamOutput();
+        output.setVersion(version);
+        stats.writeTo(output);
+        output.flush();
+        StreamInput input = output.bytes().streamInput();
+        input.setVersion(version);
+        FieldStats deserializedStats = FieldStats.readFrom(input);
+        assertEquals(stats, deserializedStats);
+        assertEquals(stats.hashCode(), deserializedStats.hashCode());
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java b/core/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java
index 043eaa2708fbd..bda00f09c3d49 100644
--- a/core/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java
+++ b/core/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java
@@ -143,6 +143,55 @@ public void testFullCircleSingleNodeFailure() throws Exception {
         assertThat(fetchData.getData().get(node1), sameInstance(response1));
     }
 
+    public void testIgnoreResponseFromDifferentRound() throws Exception {
+        DiscoveryNodes nodes = DiscoveryNodes.builder().add(node1).build();
+        test.addSimulation(node1.getId(), response1);
+
+        // first fetch, no data, still on going
+        AsyncShardFetch.FetchResult<Response> fetchData = test.fetchData(nodes, emptySet());
+        assertThat(fetchData.hasData(), equalTo(false));
+        assertThat(test.reroute.get(), equalTo(0));
+
+        // handle a response with incorrect round id, wait on reroute incrementing
+        test.processAsyncFetch(Collections.singletonList(response1), Collections.emptyList(), 0);
+        assertThat(fetchData.hasData(), equalTo(false));
+        assertThat(test.reroute.get(), equalTo(1));
+
+        // fire a response (with correct round id), wait on reroute incrementing
+        test.fireSimulationAndWait(node1.getId());
+        // verify we get back the data node
+        assertThat(test.reroute.get(), equalTo(2));
+        fetchData = test.fetchData(nodes, emptySet());
+        assertThat(fetchData.hasData(), equalTo(true));
+        assertThat(fetchData.getData().size(), equalTo(1));
+        assertThat(fetchData.getData().get(node1), sameInstance(response1));
+    }
+
+    public void testIgnoreFailureFromDifferentRound() throws Exception {
+        DiscoveryNodes nodes = DiscoveryNodes.builder().add(node1).build();
+        // add a failed response for node1
+        test.addSimulation(node1.getId(), failure1);
+
+        // first fetch, no data, still on going
+        AsyncShardFetch.FetchResult<Response> fetchData = test.fetchData(nodes, emptySet());
+        assertThat(fetchData.hasData(), equalTo(false));
+        assertThat(test.reroute.get(), equalTo(0));
+
+        // handle a failure with incorrect round id, wait on reroute incrementing
+        test.processAsyncFetch(Collections.emptyList(), Collections.singletonList(
+            new FailedNodeException(node1.getId(), "dummy failure", failure1)), 0);
+        assertThat(fetchData.hasData(), equalTo(false));
+        assertThat(test.reroute.get(), equalTo(1));
+
+        // fire a response, wait on reroute incrementing
+        test.fireSimulationAndWait(node1.getId());
+        // failure, fetched data exists, but has no data
+        assertThat(test.reroute.get(), equalTo(2));
+        fetchData = test.fetchData(nodes, emptySet());
+        assertThat(fetchData.hasData(), equalTo(true));
+        assertThat(fetchData.getData().size(), equalTo(0));
+    }
+
     public void testTwoNodesOnSetup() throws Exception {
         DiscoveryNodes nodes = DiscoveryNodes.builder().add(node1).add(node2).build();
         test.addSimulation(node1.getId(), response1);
@@ -235,7 +284,7 @@ static class Entry {
             private final CountDownLatch executeLatch = new CountDownLatch(1);
             private final CountDownLatch waitLatch = new CountDownLatch(1);
 
-            public Entry(Response response, Throwable failure) {
+            Entry(Response response, Throwable failure) {
                 this.response = response;
                 this.failure = failure;
             }
@@ -245,7 +294,7 @@ public Entry(Response response, Throwable failure) {
         private final Map<String, Entry> simulations = new ConcurrentHashMap<>();
         private AtomicInteger reroute = new AtomicInteger();
 
-        public TestFetch(ThreadPool threadPool) {
+        TestFetch(ThreadPool threadPool) {
             super(Loggers.getLogger(TestFetch.class), "test", new ShardId("test", "_na_", 1), null);
             this.threadPool = threadPool;
         }
@@ -270,7 +319,7 @@ protected void reroute(ShardId shardId, String reason) {
         }
 
         @Override
-        protected void asyncFetch(final ShardId shardId, DiscoveryNode[] nodes) {
+        protected void asyncFetch(DiscoveryNode[] nodes, long fetchingRound) {
             for (final DiscoveryNode node : nodes) {
                 final String nodeId = node.getId();
                 threadPool.generic().execute(new Runnable() {
@@ -286,11 +335,10 @@ public void run() {
                             assert entry != null;
                             entry.executeLatch.await();
                             if (entry.failure != null) {
-                                processAsyncFetch(shardId, null, Collections.singletonList(new FailedNodeException(nodeId,
-                                                                                                                   "unexpected",
-                                                                                                                   entry.failure)));
+                                processAsyncFetch(null,
+                                    Collections.singletonList(new FailedNodeException(nodeId, "unexpected", entry.failure)), fetchingRound);
                             } else {
-                                processAsyncFetch(shardId, Collections.singletonList(entry.response), null);
+                                processAsyncFetch(Collections.singletonList(entry.response), null, fetchingRound);
                             }
                         } catch (Exception e) {
                             logger.error("unexpected failure", e);
@@ -308,7 +356,7 @@ public void run() {
 
     static class Response extends BaseNodeResponse {
 
-        public Response(DiscoveryNode node) {
+        Response(DiscoveryNode node) {
             super(node);
         }
     }
diff --git a/core/src/test/java/org/elasticsearch/gateway/DanglingIndicesStateTests.java b/core/src/test/java/org/elasticsearch/gateway/DanglingIndicesStateTests.java
index 74df2f33fc096..7618ad0e95c07 100644
--- a/core/src/test/java/org/elasticsearch/gateway/DanglingIndicesStateTests.java
+++ b/core/src/test/java/org/elasticsearch/gateway/DanglingIndicesStateTests.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.cluster.metadata.IndexGraveyard;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.Index;
@@ -34,6 +35,7 @@
 import java.util.Map;
 
 import static org.hamcrest.Matchers.equalTo;
+import static org.mockito.Mockito.mock;
 
 /**
  */
@@ -47,8 +49,8 @@ public class DanglingIndicesStateTests extends ESTestCase {
 
     public void testCleanupWhenEmpty() throws Exception {
         try (NodeEnvironment env = newNodeEnvironment()) {
-            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env);
-            DanglingIndicesState danglingState = new DanglingIndicesState(Settings.EMPTY, env, metaStateService, null);
+            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env, xContentRegistry());
+            DanglingIndicesState danglingState = createDanglingIndicesState(env, metaStateService);
 
             assertTrue(danglingState.getDanglingIndices().isEmpty());
             MetaData metaData = MetaData.builder().build();
@@ -58,8 +60,8 @@ public void testCleanupWhenEmpty() throws Exception {
     }
     public void testDanglingIndicesDiscovery() throws Exception {
         try (NodeEnvironment env = newNodeEnvironment()) {
-            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env);
-            DanglingIndicesState danglingState = new DanglingIndicesState(Settings.EMPTY, env, metaStateService, null);
+            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env, xContentRegistry());
+            DanglingIndicesState danglingState = createDanglingIndicesState(env, metaStateService);
 
             assertTrue(danglingState.getDanglingIndices().isEmpty());
             MetaData metaData = MetaData.builder().build();
@@ -76,8 +78,8 @@ public void testDanglingIndicesDiscovery() throws Exception {
 
     public void testInvalidIndexFolder() throws Exception {
         try (NodeEnvironment env = newNodeEnvironment()) {
-            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env);
-            DanglingIndicesState danglingState = new DanglingIndicesState(Settings.EMPTY, env, metaStateService, null);
+            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env, xContentRegistry());
+            DanglingIndicesState danglingState = createDanglingIndicesState(env, metaStateService);
 
             MetaData metaData = MetaData.builder().build();
             final String uuid = "test1UUID";
@@ -100,8 +102,8 @@ public void testInvalidIndexFolder() throws Exception {
 
     public void testDanglingProcessing() throws Exception {
         try (NodeEnvironment env = newNodeEnvironment()) {
-            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env);
-            DanglingIndicesState danglingState = new DanglingIndicesState(Settings.EMPTY, env, metaStateService, null);
+            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env, xContentRegistry());
+            DanglingIndicesState danglingState = createDanglingIndicesState(env, metaStateService);
 
             MetaData metaData = MetaData.builder().build();
 
@@ -144,8 +146,8 @@ public void testDanglingProcessing() throws Exception {
 
     public void testDanglingIndicesNotImportedWhenTombstonePresent() throws Exception {
         try (NodeEnvironment env = newNodeEnvironment()) {
-            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env);
-            DanglingIndicesState danglingState = new DanglingIndicesState(Settings.EMPTY, env, metaStateService, null);
+            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env, xContentRegistry());
+            DanglingIndicesState danglingState = createDanglingIndicesState(env, metaStateService);
 
             final Settings.Builder settings = Settings.builder().put(indexSettings).put(IndexMetaData.SETTING_INDEX_UUID, "test1UUID");
             IndexMetaData dangledIndex = IndexMetaData.builder("test1").settings(settings).build();
@@ -157,4 +159,9 @@ public void testDanglingIndicesNotImportedWhenTombstonePresent() throws Exceptio
 
         }
     }
+
+    private DanglingIndicesState createDanglingIndicesState(NodeEnvironment env, MetaStateService metaStateService) {
+        return new DanglingIndicesState(Settings.EMPTY, env, metaStateService, null,
+            mock(ClusterService.class));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java b/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java
index 9b340fd863a96..b962135f3f674 100644
--- a/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java
+++ b/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java
@@ -29,6 +29,7 @@
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexGraveyard;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
@@ -38,8 +39,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.discovery.zen.elect.ElectMasterService;
-import org.elasticsearch.env.Environment;
+import org.elasticsearch.discovery.zen.ElectMasterService;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.indices.IndexClosedException;
@@ -58,6 +58,7 @@
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.nullValue;
 
@@ -94,7 +95,7 @@ public void testMappingMetaDataParsed() throws Exception {
 
     public void testSimpleOpenClose() throws Exception {
         logger.info("--> starting 2 nodes");
-        internalCluster().startNodesAsync(2).get();
+        internalCluster().startNodes(2);
 
         logger.info("--> creating test index");
         createIndex("test");
@@ -237,7 +238,7 @@ public void testTwoNodesSingleDoc() throws Exception {
         logger.info("--> cleaning nodes");
 
         logger.info("--> starting 2 nodes");
-        internalCluster().startNodesAsync(2).get();
+        internalCluster().startNodes(2);
 
         logger.info("--> indexing a simple document");
         client().prepareIndex("test", "type1", "1").setSource("field1", "value1").setRefreshPolicy(IMMEDIATE).get();
@@ -277,7 +278,7 @@ public void testTwoNodesSingleDoc() throws Exception {
     public void testDanglingIndices() throws Exception {
         logger.info("--> starting two nodes");
 
-        final String node_1 = internalCluster().startNodesAsync(2).get().get(0);
+        final String node_1 = internalCluster().startNodes(2).get(0);
 
         logger.info("--> indexing a simple document");
         client().prepareIndex("test", "type1", "1").setSource("field1", "value1").setRefreshPolicy(IMMEDIATE).get();
@@ -327,27 +328,13 @@ public void testIndexDeletionWhenNodeRejoins() throws Exception {
         final String indexName = "test-index-del-on-node-rejoin-idx";
         final int numNodes = 2;
 
-        final List<String> nodes;
-        if (randomBoolean()) {
-            // test with a regular index
-            logger.info("--> starting a cluster with " + numNodes + " nodes");
-            nodes = internalCluster().startNodesAsync(numNodes).get();
-            logger.info("--> create an index");
-            createIndex(indexName);
-        } else {
-            // test with a shadow replica index
-            final Path dataPath = createTempDir();
-            logger.info("--> created temp data path for shadow replicas [{}]", dataPath);
-            logger.info("--> starting a cluster with " + numNodes + " nodes");
-            final Settings nodeSettings = Settings.builder()
-                                                  .put("node.add_lock_id_to_custom_path", false)
-                                                  .put(Environment.PATH_SHARED_DATA_SETTING.getKey(), dataPath.toString())
-                                                  .put("index.store.fs.fs_lock", randomFrom("native", "simple"))
-                                                  .build();
-            nodes = internalCluster().startNodesAsync(numNodes, nodeSettings).get();
-            logger.info("--> create a shadow replica index");
-            createShadowReplicaIndex(indexName, dataPath, numNodes - 1);
-        }
+        // test with a regular index, shadow replicas are removed in 6.x and test failures with
+        // them are too hard to debug
+        logger.info("--> starting a cluster with " + numNodes + " nodes");
+        final List<String> nodes = internalCluster().startNodes(numNodes,
+            Settings.builder().put(IndexGraveyard.SETTING_MAX_TOMBSTONES.getKey(), randomIntBetween(10, 100)).build());
+        logger.info("--> create an index");
+        createIndex(indexName);
 
         logger.info("--> waiting for green status");
         ensureGreen();
@@ -414,7 +401,7 @@ public void testRecoverBrokenIndexMetadata() throws Exception {
         IndexMetaData metaData = state.getMetaData().index("test");
         for (NodeEnvironment services : internalCluster().getInstances(NodeEnvironment.class)) {
             IndexMetaData brokenMeta = IndexMetaData.builder(metaData).settings(Settings.builder().put(metaData.getSettings())
-                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_0_0_beta1.id)
+                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT.minimumIndexCompatibilityVersion().id)
                  // this is invalid but should be archived
                 .put("index.similarity.BM25.type", "classic")
                  // this one is not validated ahead of time and breaks allocation
@@ -434,7 +421,7 @@ public void testRecoverBrokenIndexMetadata() throws Exception {
         assertEquals(ex.getMessage(), "Failed to verify index " + metaData.getIndex());
         assertNotNull(ex.getCause());
         assertEquals(IllegalArgumentException.class, ex.getCause().getClass());
-        assertEquals(ex.getCause().getMessage(), "Unknown tokenfilter type [icu_collation] for [myCollator]");
+        assertEquals(ex.getCause().getMessage(), "Unknown filter type [icu_collation] for [myCollator]");
     }
 
     /**
@@ -491,7 +478,7 @@ public void testRecoverMissingAnalyzer() throws Exception {
         assertEquals(ex.getMessage(), "Failed to verify index " + metaData.getIndex());
         assertNotNull(ex.getCause());
         assertEquals(MapperParsingException.class, ex.getCause().getClass());
-        assertEquals(ex.getCause().getMessage(), "analyzer [test] not found for field [field1]");
+        assertThat(ex.getCause().getMessage(), containsString("analyzer [test] not found for field [field1]"));
     }
 
     public void testArchiveBrokenClusterSettings() throws Exception {
diff --git a/core/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java b/core/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java
index 2a37a7f0a6004..e7daa9a791d43 100644
--- a/core/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java
+++ b/core/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java
@@ -22,7 +22,9 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.metadata.MetaDataIndexUpgradeService;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -33,9 +35,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.plugins.MetaDataUpgrader;
-import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.test.TestCustomMetaData;
-import org.junit.Before;
 
 import java.util.Arrays;
 import java.util.Collections;
@@ -61,12 +61,6 @@
  */
 public class GatewayMetaStateTests extends ESAllocationTestCase {
 
-    @Before
-    public void setup() {
-        MetaData.registerPrototype(CustomMetaData1.TYPE, new CustomMetaData1(""));
-        MetaData.registerPrototype(CustomMetaData2.TYPE, new CustomMetaData2(""));
-    }
-
     ClusterChangedEvent generateEvent(boolean initializing, boolean versionChanged, boolean masterEligible) {
         //ridiculous settings to make sure we don't run into uninitialized because fo default
         AllocationService strategy = createAllocationService(Settings.builder()
@@ -265,7 +259,8 @@ public void testAddCustomMetaDataOnUpgrade() throws Exception {
             Collections.singletonList(customs -> {
                 customs.put(CustomMetaData1.TYPE, new CustomMetaData1("modified_data1"));
                 return customs;
-            })
+            }),
+            Collections.emptyList()
         );
         MetaData upgrade = GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader);
         assertTrue(upgrade != metaData);
@@ -280,7 +275,8 @@ public void testRemoveCustomMetaDataOnUpgrade() throws Exception {
             Collections.singletonList(customs -> {
                 customs.remove(CustomMetaData1.TYPE);
                 return customs;
-            })
+            }),
+            Collections.emptyList()
         );
         MetaData upgrade = GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader);
         assertTrue(upgrade != metaData);
@@ -294,7 +290,8 @@ public void testUpdateCustomMetaDataOnUpgrade() throws Exception {
             Collections.singletonList(customs -> {
                 customs.put(CustomMetaData1.TYPE, new CustomMetaData1("modified_data1"));
                 return customs;
-            })
+            }),
+            Collections.emptyList()
         );
 
         MetaData upgrade = GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader);
@@ -304,9 +301,27 @@ public void testUpdateCustomMetaDataOnUpgrade() throws Exception {
         assertThat(((TestCustomMetaData) upgrade.custom(CustomMetaData1.TYPE)).getData(), equalTo("modified_data1"));
     }
 
+
+    public void testUpdateTemplateMetaDataOnUpgrade() throws Exception {
+        MetaData metaData = randomMetaData();
+        MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(
+            Collections.emptyList(),
+            Collections.singletonList(
+                templates -> {
+                    templates.put("added_test_template", IndexTemplateMetaData.builder("added_test_template").build());
+                    return templates;
+                }
+            ));
+
+        MetaData upgrade = GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader);
+        assertTrue(upgrade != metaData);
+        assertFalse(MetaData.isGlobalStateEquals(upgrade, metaData));
+        assertTrue(upgrade.templates().containsKey("added_test_template"));
+    }
+
     public void testNoMetaDataUpgrade() throws Exception {
         MetaData metaData = randomMetaData(new CustomMetaData1("data"));
-        MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(Collections.emptyList());
+        MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(Collections.emptyList(), Collections.emptyList());
         MetaData upgrade = GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader);
         assertTrue(upgrade == metaData);
         assertTrue(MetaData.isGlobalStateEquals(upgrade, metaData));
@@ -321,7 +336,7 @@ public void testCustomMetaDataValidation() throws Exception {
             customs -> {
                 throw new IllegalStateException("custom meta data too old");
             }
-        ));
+        ), Collections.emptyList());
         try {
             GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader);
         } catch (IllegalStateException e) {
@@ -341,7 +356,8 @@ public void testMultipleCustomMetaDataUpgrade() throws Exception {
             case 2:
                 metaData = randomMetaData();
                 break;
-            default: throw new IllegalStateException("should never happen");
+            default:
+                throw new IllegalStateException("should never happen");
         }
         MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(
             Arrays.asList(
@@ -352,8 +368,8 @@ public void testMultipleCustomMetaDataUpgrade() throws Exception {
                 customs -> {
                     customs.put(CustomMetaData2.TYPE, new CustomMetaData1("modified_data2"));
                     return customs;
-                })
-        );
+                }
+            ), Collections.emptyList());
         MetaData upgrade = GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader);
         assertTrue(upgrade != metaData);
         assertFalse(MetaData.isGlobalStateEquals(upgrade, metaData));
@@ -368,7 +384,7 @@ public void testMultipleCustomMetaDataUpgrade() throws Exception {
 
     public void testIndexMetaDataUpgrade() throws Exception {
         MetaData metaData = randomMetaData();
-        MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(Collections.emptyList());
+        MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(Collections.emptyList(), Collections.emptyList());
         MetaData upgrade = GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(true), metaDataUpgrader);
         assertTrue(upgrade != metaData);
         assertTrue(MetaData.isGlobalStateEquals(upgrade, metaData));
@@ -379,7 +395,8 @@ public void testIndexMetaDataUpgrade() throws Exception {
 
     public void testCustomMetaDataNoChange() throws Exception {
         MetaData metaData = randomMetaData(new CustomMetaData1("data"));
-        MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(Collections.singletonList(HashMap::new));
+        MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(Collections.singletonList(HashMap::new),
+            Collections.singletonList(HashMap::new));
         MetaData upgrade = GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader);
         assertTrue(upgrade == metaData);
         assertTrue(MetaData.isGlobalStateEquals(upgrade, metaData));
@@ -388,15 +405,73 @@ public void testCustomMetaDataNoChange() throws Exception {
         }
     }
 
+    public void testIndexTemplateValidation() throws Exception {
+        MetaData metaData = randomMetaData();
+        MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(
+            Collections.emptyList(),
+            Collections.singletonList(
+                customs -> {
+                    throw new IllegalStateException("template is incompatible");
+                }));
+        String message = expectThrows(IllegalStateException.class,
+            () -> GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader)).getMessage();
+        assertThat(message, equalTo("template is incompatible"));
+    }
+
+
+    public void testMultipleIndexTemplateUpgrade() throws Exception {
+        final MetaData metaData;
+        switch (randomIntBetween(0, 2)) {
+            case 0:
+                metaData = randomMetaDataWithIndexTemplates("template1", "template2");
+                break;
+            case 1:
+                metaData = randomMetaDataWithIndexTemplates(randomBoolean() ? "template1" : "template2");
+                break;
+            case 2:
+                metaData = randomMetaData();
+                break;
+            default:
+                throw new IllegalStateException("should never happen");
+        }
+        MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(
+            Collections.emptyList(),
+            Arrays.asList(
+                indexTemplateMetaDatas -> {
+                    indexTemplateMetaDatas.put("template1", IndexTemplateMetaData.builder("template1").settings(
+                        Settings.builder().put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 20).build()).build());
+                    return indexTemplateMetaDatas;
+
+                },
+                indexTemplateMetaDatas -> {
+                    indexTemplateMetaDatas.put("template2", IndexTemplateMetaData.builder("template2").settings(
+                        Settings.builder().put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 10).build()).build());
+                    return indexTemplateMetaDatas;
+
+                }
+            ));
+        MetaData upgrade = GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader);
+        assertTrue(upgrade != metaData);
+        assertFalse(MetaData.isGlobalStateEquals(upgrade, metaData));
+        assertNotNull(upgrade.templates().get("template1"));
+        assertThat(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(upgrade.templates().get("template1").settings()), equalTo(20));
+        assertNotNull(upgrade.templates().get("template2"));
+        assertThat(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.get(upgrade.templates().get("template2").settings()), equalTo(10));
+        for (IndexMetaData indexMetaData : upgrade) {
+            assertTrue(metaData.hasIndexMetaData(indexMetaData));
+        }
+    }
+
     private static class MockMetaDataIndexUpgradeService extends MetaDataIndexUpgradeService {
         private final boolean upgrade;
 
-        public MockMetaDataIndexUpgradeService(boolean upgrade) {
-            super(Settings.EMPTY, null, null);
+        MockMetaDataIndexUpgradeService(boolean upgrade) {
+            super(Settings.EMPTY, null, null, null, null);
             this.upgrade = upgrade;
         }
+
         @Override
-        public IndexMetaData upgradeIndexMetaData(IndexMetaData indexMetaData) {
+        public IndexMetaData upgradeIndexMetaData(IndexMetaData indexMetaData, Version minimumIndexCompatibilityVersion) {
             return upgrade ? IndexMetaData.builder(indexMetaData).build() : indexMetaData;
         }
     }
@@ -409,12 +484,7 @@ protected CustomMetaData1(String data) {
         }
 
         @Override
-        protected TestCustomMetaData newTestCustomMetaData(String data) {
-            return new CustomMetaData1(data);
-        }
-
-        @Override
-        public String type() {
+        public String getWriteableName() {
             return TYPE;
         }
 
@@ -432,12 +502,7 @@ protected CustomMetaData2(String data) {
         }
 
         @Override
-        protected TestCustomMetaData newTestCustomMetaData(String data) {
-            return new CustomMetaData2(data);
-        }
-
-        @Override
-        public String type() {
+        public String getWriteableName() {
             return TYPE;
         }
 
@@ -450,11 +515,32 @@ public EnumSet<MetaData.XContentContext> context() {
     private static MetaData randomMetaData(TestCustomMetaData... customMetaDatas) {
         MetaData.Builder builder = MetaData.builder();
         for (TestCustomMetaData customMetaData : customMetaDatas) {
-            builder.putCustom(customMetaData.type(), customMetaData);
+            builder.putCustom(customMetaData.getWriteableName(), customMetaData);
+        }
+        for (int i = 0; i < randomIntBetween(1, 5); i++) {
+            builder.put(
+                IndexMetaData.builder(randomAlphaOfLength(10))
+                    .settings(settings(Version.CURRENT))
+                    .numberOfReplicas(randomIntBetween(0, 3))
+                    .numberOfShards(randomIntBetween(1, 5))
+            );
+        }
+        return builder.build();
+    }
+
+    private static MetaData randomMetaDataWithIndexTemplates(String... templates) {
+        MetaData.Builder builder = MetaData.builder();
+        for (String template : templates) {
+            IndexTemplateMetaData templateMetaData = IndexTemplateMetaData.builder(template)
+                .settings(settings(Version.CURRENT)
+                    .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), randomIntBetween(0, 3))
+                    .put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), randomIntBetween(1, 5)))
+                .build();
+            builder.put(templateMetaData);
         }
         for (int i = 0; i < randomIntBetween(1, 5); i++) {
             builder.put(
-                IndexMetaData.builder(randomAsciiOfLength(10))
+                IndexMetaData.builder(randomAlphaOfLength(10))
                     .settings(settings(Version.CURRENT))
                     .numberOfReplicas(randomIntBetween(0, 3))
                     .numberOfShards(randomIntBetween(1, 5))
diff --git a/core/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java b/core/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java
index d2085ab9147e0..5c5aab847459b 100644
--- a/core/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java
@@ -19,9 +19,13 @@
 
 package org.elasticsearch.gateway;
 
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.NoopDiscovery;
@@ -34,9 +38,9 @@ public class GatewayServiceTests extends ESTestCase {
     private GatewayService createService(Settings.Builder settings) {
         ClusterService clusterService = new ClusterService(Settings.builder().put("cluster.name", "GatewayServiceTests").build(),
                 new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
-                null);
+                null, () -> new DiscoveryNode(UUIDs.randomBase64UUID(), LocalTransportAddress.buildUnique(), Version.CURRENT));
         return new GatewayService(settings.build(),
-                null, clusterService, null, null, null, new NoopDiscovery(), null, null);
+                null, clusterService, null, null, null, new NoopDiscovery(), null);
     }
 
     public void testDefaultRecoverAfterTime() throws IOException {
diff --git a/core/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java b/core/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java
index 0f0e69b26437c..1b443a33da589 100644
--- a/core/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java
+++ b/core/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java
@@ -35,6 +35,7 @@
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -89,7 +90,7 @@ public MetaData fromXContent(XContentParser parser) throws IOException {
         assertThat(resource, notNullValue());
         Path dst = tmp.resolve("global-3.st");
         Files.copy(resource, dst);
-        MetaData read = format.read(dst);
+        MetaData read = format.read(xContentRegistry(), dst);
         assertThat(read, notNullValue());
         assertThat(read.clusterUUID(), equalTo("3O1tDF1IRB6fSJ-GrTMUtg"));
         // indices are empty since they are serialized separately
@@ -114,7 +115,7 @@ public void testReadWriteState() throws IOException {
             list = content("foo-*", stateDir);
             assertEquals(list.length, 1);
             assertThat(list[0].getFileName().toString(), equalTo("foo-" + id + ".st"));
-            DummyState read = format.read(list[0]);
+            DummyState read = format.read(NamedXContentRegistry.EMPTY, list[0]);
             assertThat(read, equalTo(state));
         }
         final int version2 = between(version, Integer.MAX_VALUE);
@@ -130,7 +131,7 @@ public void testReadWriteState() throws IOException {
             list = content("foo-*", stateDir);
             assertEquals(list.length,1);
             assertThat(list[0].getFileName().toString(), equalTo("foo-"+ (id+1) + ".st"));
-            DummyState read = format.read(list[0]);
+            DummyState read = format.read(NamedXContentRegistry.EMPTY, list[0]);
             assertThat(read, equalTo(state2));
 
         }
@@ -156,7 +157,7 @@ public void testVersionMismatch() throws IOException {
             list = content("foo-*", stateDir);
             assertEquals(list.length, 1);
             assertThat(list[0].getFileName().toString(), equalTo("foo-" + id + ".st"));
-            DummyState read = format.read(list[0]);
+            DummyState read = format.read(NamedXContentRegistry.EMPTY, list[0]);
             assertThat(read, equalTo(state));
         }
     }
@@ -180,12 +181,12 @@ public void testCorruption() throws IOException {
             list = content("foo-*", stateDir);
             assertEquals(list.length, 1);
             assertThat(list[0].getFileName().toString(), equalTo("foo-" + id + ".st"));
-            DummyState read = format.read(list[0]);
+            DummyState read = format.read(NamedXContentRegistry.EMPTY, list[0]);
             assertThat(read, equalTo(state));
             // now corrupt it
             corruptFile(list[0], logger);
             try {
-                format.read(list[0]);
+                format.read(NamedXContentRegistry.EMPTY, list[0]);
                 fail("corrupted file");
             } catch (CorruptStateException ex) {
                 // expected
@@ -195,7 +196,7 @@ public void testCorruption() throws IOException {
 
     public static void corruptFile(Path file, Logger logger) throws IOException {
         Path fileToCorrupt = file;
-        try (final SimpleFSDirectory dir = new SimpleFSDirectory(fileToCorrupt.getParent())) {
+        try (SimpleFSDirectory dir = new SimpleFSDirectory(fileToCorrupt.getParent())) {
             long checksumBeforeCorruption;
             try (IndexInput input = dir.openInput(fileToCorrupt.getFileName().toString(), IOContext.DEFAULT)) {
                 checksumBeforeCorruption = CodecUtil.retrieveChecksum(input);
@@ -272,7 +273,7 @@ public void testLoadState() throws IOException {
         }
         List<Path> dirList = Arrays.asList(dirs);
         Collections.shuffle(dirList, random());
-        MetaData loadedMetaData = format.loadLatestState(logger, dirList.toArray(new Path[0]));
+        MetaData loadedMetaData = format.loadLatestState(logger, xContentRegistry(), dirList.toArray(new Path[0]));
         MetaData latestMetaData = meta.get(numStates-1);
         assertThat(loadedMetaData.clusterUUID(), not(equalTo("_na_")));
         assertThat(loadedMetaData.clusterUUID(), equalTo(latestMetaData.clusterUUID()));
@@ -299,7 +300,7 @@ public void testLoadState() throws IOException {
                 MetaDataStateFormatTests.corruptFile(file, logger);
             }
             try {
-                format.loadLatestState(logger, dirList.toArray(new Path[0]));
+                format.loadLatestState(logger, xContentRegistry(), dirList.toArray(new Path[0]));
                 fail("latest version can not be read");
             } catch (ElasticsearchException ex) {
                 assertThat(ExceptionsHelper.unwrap(ex, CorruptStateException.class), notNullValue());
@@ -326,12 +327,12 @@ private MetaData randomMeta() throws IOException {
         MetaData.Builder mdBuilder = MetaData.builder();
         mdBuilder.generateClusterUuidIfNeeded();
         for (int i = 0; i < numIndices; i++) {
-            mdBuilder.put(indexBuilder(randomAsciiOfLength(10) + "idx-"+i));
+            mdBuilder.put(indexBuilder(randomAlphaOfLength(10) + "idx-"+i));
         }
         int numDelIndices = randomIntBetween(0, 5);
         final IndexGraveyard.Builder graveyard = IndexGraveyard.builder();
         for (int i = 0; i < numDelIndices; i++) {
-            graveyard.addTombstone(new Index(randomAsciiOfLength(10) + "del-idx-" + i, UUIDs.randomBase64UUID()));
+            graveyard.addTombstone(new Index(randomAlphaOfLength(10) + "del-idx-" + i, UUIDs.randomBase64UUID()));
         }
         mdBuilder.indexGraveyard(graveyard.build());
         return mdBuilder.build();
@@ -385,7 +386,7 @@ public String toString() {
                     '}';
         }
 
-        public DummyState(String string, int aInt, long aLong, double aDouble, boolean aBoolean) {
+        DummyState(String string, int aInt, long aLong, double aDouble, boolean aBoolean) {
             this.string = string;
             this.aInt = aInt;
             this.aLong = aLong;
@@ -393,7 +394,7 @@ public DummyState(String string, int aInt, long aLong, double aDouble, boolean a
             this.aBoolean = aBoolean;
         }
 
-        public DummyState() {
+        DummyState() {
 
         }
 
diff --git a/core/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesIT.java b/core/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesIT.java
index 3c2917f38e87d..3dfeca3053ed7 100644
--- a/core/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesIT.java
+++ b/core/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesIT.java
@@ -36,6 +36,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.LinkedHashMap;
+import java.util.List;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@@ -57,10 +58,9 @@ public void testMetaWrittenAlsoOnDataNode() throws Exception {
     public void testMetaIsRemovedIfAllShardsFromIndexRemoved() throws Exception {
         // this test checks that the index state is removed from a data only node once all shards have been allocated away from it
         String masterNode = internalCluster().startMasterOnlyNode(Settings.EMPTY);
-        InternalTestCluster.Async<String> nodeName1 = internalCluster().startDataOnlyNodeAsync();
-        InternalTestCluster.Async<String> nodeName2 = internalCluster().startDataOnlyNodeAsync();
-        String node1 = nodeName1.get();
-        String node2 = nodeName2.get();
+        List<String> nodeNames= internalCluster().startDataOnlyNodes(2);
+        String node1 = nodeNames.get(0);
+        String node2 = nodeNames.get(1);
 
         String index = "index";
         assertAcked(prepareCreate(index).setSettings(Settings.builder().put("index.number_of_replicas", 0).put(IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + "_name", node1)));
diff --git a/core/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java b/core/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java
index 82c38748a4886..f1e4675f4698a 100644
--- a/core/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java
@@ -41,7 +41,7 @@ public class MetaStateServiceTests extends ESTestCase {
 
     public void testWriteLoadIndex() throws Exception {
         try (NodeEnvironment env = newNodeEnvironment()) {
-            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env);
+            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env, xContentRegistry());
 
             IndexMetaData index = IndexMetaData.builder("test1").settings(indexSettings).build();
             metaStateService.writeIndex("test_write", index);
@@ -51,14 +51,14 @@ public void testWriteLoadIndex() throws Exception {
 
     public void testLoadMissingIndex() throws Exception {
         try (NodeEnvironment env = newNodeEnvironment()) {
-            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env);
+            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env, xContentRegistry());
             assertThat(metaStateService.loadIndexState(new Index("test1", "test1UUID")), nullValue());
         }
     }
 
     public void testWriteLoadGlobal() throws Exception {
         try (NodeEnvironment env = newNodeEnvironment()) {
-            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env);
+            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env, xContentRegistry());
 
             MetaData metaData = MetaData.builder()
                     .persistentSettings(Settings.builder().put("test1", "value1").build())
@@ -70,7 +70,7 @@ public void testWriteLoadGlobal() throws Exception {
 
     public void testWriteGlobalStateWithIndexAndNoIndexIsLoaded() throws Exception {
         try (NodeEnvironment env = newNodeEnvironment()) {
-            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env);
+            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env, xContentRegistry());
 
             MetaData metaData = MetaData.builder()
                     .persistentSettings(Settings.builder().put("test1", "value1").build())
@@ -86,7 +86,7 @@ public void testWriteGlobalStateWithIndexAndNoIndexIsLoaded() throws Exception {
 
     public void testLoadGlobal() throws Exception {
         try (NodeEnvironment env = newNodeEnvironment()) {
-            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env);
+            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env, xContentRegistry());
 
             IndexMetaData index = IndexMetaData.builder("test1").settings(indexSettings).build();
             MetaData metaData = MetaData.builder()
diff --git a/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java b/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java
index 080b1f0a00e0c..c3b496ea5d300 100644
--- a/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java
+++ b/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java
@@ -23,18 +23,20 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.health.ClusterStateHealth;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.RecoverySource.SnapshotRecoverySource;
+import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.routing.UnassignedInfo.AllocationStatus;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider;
@@ -44,11 +46,11 @@
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.env.ShardLockObtainFailedException;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.ShardStateMetaData;
 import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.snapshots.SnapshotId;
-import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.junit.Before;
 
 import java.util.Arrays;
@@ -57,6 +59,9 @@
 import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.cluster.routing.UnassignedInfo.Reason.CLUSTER_RECOVERED;
+import static org.elasticsearch.cluster.routing.UnassignedInfo.Reason.INDEX_CREATED;
+import static org.elasticsearch.cluster.routing.UnassignedInfo.Reason.INDEX_REOPENED;
 import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
@@ -71,6 +76,16 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase {
     private final DiscoveryNode node3 = newNode("node3");
     private TestAllocator testAllocator;
 
+
+    /**
+     * needed due to random usage of {@link IndexMetaData#INDEX_SHARED_FS_ALLOW_RECOVERY_ON_ANY_NODE_SETTING}. removed once
+     * shadow replicas are removed.
+     */
+    @Override
+    protected boolean enableWarningsCheck() {
+        return false;
+    }
+
     @Before
     public void buildTestAllocator() {
         this.testAllocator = new TestAllocator();
@@ -78,11 +93,9 @@ public void buildTestAllocator() {
 
     public void testNoProcessPrimaryNotAllocatedBefore() {
         final RoutingAllocation allocation;
-        if (randomBoolean()) {
-            allocation = routingAllocationWithOnePrimaryNoReplicas(yesAllocationDeciders(), randomBoolean(), Version.CURRENT);
-        } else {
-            allocation = routingAllocationWithOnePrimaryNoReplicas(yesAllocationDeciders(), true, Version.V_2_1_0);
-        }
+        // with old version, we can't know if a shard was allocated before or not
+        allocation = routingAllocationWithOnePrimaryNoReplicas(yesAllocationDeciders(),
+            randomFrom(INDEX_CREATED, CLUSTER_RECOVERED, INDEX_REOPENED), Version.CURRENT);
         testAllocator.allocateUnassigned(allocation);
         assertThat(allocation.routingNodesChanged(), equalTo(false));
         assertThat(allocation.routingNodes().unassigned().size(), equalTo(1));
@@ -96,9 +109,9 @@ public void testNoProcessPrimaryNotAllocatedBefore() {
     public void testNoAsyncFetchData() {
         final RoutingAllocation allocation;
         if (randomBoolean()) {
-            allocation = routingAllocationWithOnePrimaryNoReplicas(yesAllocationDeciders(), false, Version.CURRENT, "allocId");
+            allocation = routingAllocationWithOnePrimaryNoReplicas(yesAllocationDeciders(), CLUSTER_RECOVERED, Version.CURRENT, "allocId");
         } else {
-            allocation = routingAllocationWithOnePrimaryNoReplicas(yesAllocationDeciders(), false, Version.V_2_1_0);
+            allocation = routingAllocationWithOnePrimaryNoReplicas(yesAllocationDeciders(), CLUSTER_RECOVERED, Version.V_2_1_0);
         }
         testAllocator.allocateUnassigned(allocation);
         assertThat(allocation.routingNodesChanged(), equalTo(true));
@@ -114,9 +127,9 @@ public void testNoAsyncFetchData() {
     public void testNoAllocationFound() {
         final RoutingAllocation allocation;
         if (randomBoolean()) {
-            allocation = routingAllocationWithOnePrimaryNoReplicas(yesAllocationDeciders(), false, Version.CURRENT, "allocId");
+            allocation = routingAllocationWithOnePrimaryNoReplicas(yesAllocationDeciders(), CLUSTER_RECOVERED, Version.CURRENT, "allocId");
         } else {
-            allocation = routingAllocationWithOnePrimaryNoReplicas(yesAllocationDeciders(), false, Version.V_2_1_0);
+            allocation = routingAllocationWithOnePrimaryNoReplicas(yesAllocationDeciders(), CLUSTER_RECOVERED, Version.V_2_1_0);
         }
         testAllocator.addData(node1, ShardStateMetaData.NO_VERSION, null, randomBoolean());
         testAllocator.allocateUnassigned(allocation);
@@ -130,7 +143,7 @@ public void testNoAllocationFound() {
      * Tests when the node returns data with a shard allocation id that does not match active allocation ids, it will be moved to ignore unassigned.
      */
     public void testNoMatchingAllocationIdFound() {
-        RoutingAllocation allocation = routingAllocationWithOnePrimaryNoReplicas(yesAllocationDeciders(), false, Version.CURRENT, "id2");
+        RoutingAllocation allocation = routingAllocationWithOnePrimaryNoReplicas(yesAllocationDeciders(), CLUSTER_RECOVERED, Version.CURRENT, "id2");
         testAllocator.addData(node1, ShardStateMetaData.NO_VERSION, "id1", randomBoolean());
         testAllocator.allocateUnassigned(allocation);
         assertThat(allocation.routingNodesChanged(), equalTo(true));
@@ -144,7 +157,7 @@ public void testNoMatchingAllocationIdFound() {
      * This is the case when we have old shards from pre-3.0 days.
      */
     public void testNoActiveAllocationIds() {
-        RoutingAllocation allocation = routingAllocationWithOnePrimaryNoReplicas(yesAllocationDeciders(), false, Version.V_2_1_1);
+        RoutingAllocation allocation = routingAllocationWithOnePrimaryNoReplicas(yesAllocationDeciders(), CLUSTER_RECOVERED, Version.V_2_1_1);
         testAllocator.addData(node1, 1, null, randomBoolean());
         testAllocator.allocateUnassigned(allocation);
         assertThat(allocation.routingNodesChanged(), equalTo(true));
@@ -160,10 +173,11 @@ public void testNoActiveAllocationIds() {
     public void testStoreException() {
         final RoutingAllocation allocation;
         if (randomBoolean()) {
-            allocation = routingAllocationWithOnePrimaryNoReplicas(yesAllocationDeciders(), false, randomFrom(Version.V_2_0_0, Version.CURRENT), "allocId1");
+            allocation = routingAllocationWithOnePrimaryNoReplicas(yesAllocationDeciders(), CLUSTER_RECOVERED,
+                randomFrom(Version.V_2_0_0, Version.CURRENT), "allocId1");
             testAllocator.addData(node1, ShardStateMetaData.NO_VERSION, "allocId1", randomBoolean(), new CorruptIndexException("test", "test"));
         } else {
-            allocation = routingAllocationWithOnePrimaryNoReplicas(yesAllocationDeciders(), false, Version.V_2_1_1);
+            allocation = routingAllocationWithOnePrimaryNoReplicas(yesAllocationDeciders(), CLUSTER_RECOVERED, Version.V_2_1_1);
             testAllocator.addData(node1, 3, null, randomBoolean(), new CorruptIndexException("test", "test"));
         }
         testAllocator.allocateUnassigned(allocation);
@@ -173,6 +187,69 @@ public void testStoreException() {
         assertClusterHealthStatus(allocation, ClusterHealthStatus.YELLOW);
     }
 
+    /**
+     * Tests that when the node returns a ShardLockObtainFailedException, it will be considered as a valid shard copy
+     */
+    public void testShardLockObtainFailedException() {
+        final RoutingAllocation allocation;
+        boolean useAllocationIds = randomBoolean();
+        if (useAllocationIds) {
+            allocation = routingAllocationWithOnePrimaryNoReplicas(yesAllocationDeciders(), CLUSTER_RECOVERED,
+                randomFrom(Version.V_2_0_0, Version.CURRENT), "allocId1");
+            testAllocator.addData(node1, ShardStateMetaData.NO_VERSION, "allocId1", randomBoolean(),
+                new ShardLockObtainFailedException(shardId, "test"));
+        } else {
+            allocation = routingAllocationWithOnePrimaryNoReplicas(yesAllocationDeciders(), CLUSTER_RECOVERED, Version.V_2_1_1);
+            testAllocator.addData(node1, 3, null, randomBoolean(), new ShardLockObtainFailedException(shardId, "test"));
+        }
+        testAllocator.allocateUnassigned(allocation);
+        assertThat(allocation.routingNodesChanged(), equalTo(true));
+        assertThat(allocation.routingNodes().unassigned().ignored().isEmpty(), equalTo(true));
+        assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(1));
+        assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).currentNodeId(), equalTo(node1.getId()));
+        if (useAllocationIds) {
+            // check that allocation id is reused
+            assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).allocationId().getId(), equalTo("allocId1"));
+        }
+        assertClusterHealthStatus(allocation, ClusterHealthStatus.YELLOW);
+    }
+
+    /**
+     * Tests that when one node returns a ShardLockObtainFailedException and another properly loads the store, it will
+     * select the second node as target
+     */
+    public void testShardLockObtainFailedExceptionPreferOtherValidCopies() {
+        final RoutingAllocation allocation;
+        boolean useAllocationIds = randomBoolean();
+        String allocId1 = randomAlphaOfLength(10);
+        String allocId2 = randomAlphaOfLength(10);
+        if (useAllocationIds) {
+            allocation = routingAllocationWithOnePrimaryNoReplicas(yesAllocationDeciders(), CLUSTER_RECOVERED,
+                randomFrom(Version.V_2_0_0, Version.CURRENT), allocId1, allocId2);
+            testAllocator.addData(node1, ShardStateMetaData.NO_VERSION, allocId1, randomBoolean(),
+                new ShardLockObtainFailedException(shardId, "test"));
+            testAllocator.addData(node2, ShardStateMetaData.NO_VERSION, allocId2, randomBoolean(), null);
+        } else {
+            allocation = routingAllocationWithOnePrimaryNoReplicas(yesAllocationDeciders(), CLUSTER_RECOVERED, Version.V_2_1_1);
+            testAllocator.addData(node1, 3, null, randomBoolean(), new ShardLockObtainFailedException(shardId, "test"));
+            if (randomBoolean()) {
+                testAllocator.addData(node2, randomIntBetween(2, 4), null, randomBoolean(), null);
+            } else {
+                testAllocator.addData(node2, ShardStateMetaData.NO_VERSION, "some alloc id", randomBoolean(), null);
+            }
+        }
+        testAllocator.allocateUnassigned(allocation);
+        assertThat(allocation.routingNodesChanged(), equalTo(true));
+        assertThat(allocation.routingNodes().unassigned().ignored().isEmpty(), equalTo(true));
+        assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(1));
+        assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).currentNodeId(), equalTo(node2.getId()));
+        if (useAllocationIds) {
+            // check that allocation id is reused
+            assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).allocationId().getId(), equalTo(allocId2));
+        }
+        assertClusterHealthStatus(allocation, ClusterHealthStatus.YELLOW);
+    }
+
     /**
      * Tests that when there is a node to allocate the shard to, it will be allocated to it.
      */
@@ -180,10 +257,12 @@ public void testFoundAllocationAndAllocating() {
         final RoutingAllocation allocation;
         boolean useAllocationIds = randomBoolean();
         if (useAllocationIds) {
-            allocation = routingAllocationWithOnePrimaryNoReplicas(yesAllocationDeciders(), false, randomFrom(Version.V_2_0_0, Version.CURRENT), "allocId1");
+            allocation = routingAllocationWithOnePrimaryNoReplicas(yesAllocationDeciders(), randomFrom(CLUSTER_RECOVERED, INDEX_REOPENED),
+                randomFrom(Version.V_2_0_0, Version.CURRENT), "allocId1");
             testAllocator.addData(node1, ShardStateMetaData.NO_VERSION, "allocId1", randomBoolean());
         } else {
-            allocation = routingAllocationWithOnePrimaryNoReplicas(yesAllocationDeciders(), false, Version.V_2_2_0);
+            allocation = routingAllocationWithOnePrimaryNoReplicas(yesAllocationDeciders(), randomFrom(CLUSTER_RECOVERED, INDEX_REOPENED),
+                Version.V_2_2_0);
             testAllocator.addData(node1, 3, null, randomBoolean());
         }
         testAllocator.allocateUnassigned(allocation);
@@ -210,7 +289,7 @@ public void testForceAllocatePrimary() {
             // the allocator will see if it can force assign the primary, where the decision will be YES
             new TestAllocateDecision(randomBoolean() ? Decision.YES : Decision.NO), getNoDeciderThatAllowsForceAllocate()
         ));
-        RoutingAllocation allocation = routingAllocationWithOnePrimaryNoReplicas(deciders, false, Version.CURRENT, "allocId1");
+        RoutingAllocation allocation = routingAllocationWithOnePrimaryNoReplicas(deciders, CLUSTER_RECOVERED, Version.CURRENT, "allocId1");
         testAllocator.allocateUnassigned(allocation);
         assertThat(allocation.routingNodesChanged(), equalTo(true));
         assertTrue(allocation.routingNodes().unassigned().ignored().isEmpty());
@@ -233,7 +312,7 @@ public void testDontAllocateOnNoOrThrottleForceAllocationDecision() {
             new TestAllocateDecision(Decision.NO), forceDecisionNo ? getNoDeciderThatDeniesForceAllocate() :
                                                                      getNoDeciderThatThrottlesForceAllocate()
         ));
-        RoutingAllocation allocation = routingAllocationWithOnePrimaryNoReplicas(deciders, false, Version.CURRENT, "allocId1");
+        RoutingAllocation allocation = routingAllocationWithOnePrimaryNoReplicas(deciders, CLUSTER_RECOVERED, Version.CURRENT, "allocId1");
         testAllocator.allocateUnassigned(allocation);
         assertThat(allocation.routingNodesChanged(), equalTo(true));
         List<ShardRouting> ignored = allocation.routingNodes().unassigned().ignored();
@@ -257,7 +336,7 @@ public void testDontForceAllocateOnThrottleDecision() {
             // force allocating the shard, we still THROTTLE due to the decision from TestAllocateDecision
             new TestAllocateDecision(Decision.THROTTLE), getNoDeciderThatAllowsForceAllocate()
         ));
-        RoutingAllocation allocation = routingAllocationWithOnePrimaryNoReplicas(deciders, false, Version.CURRENT, "allocId1");
+        RoutingAllocation allocation = routingAllocationWithOnePrimaryNoReplicas(deciders, CLUSTER_RECOVERED, Version.CURRENT, "allocId1");
         testAllocator.allocateUnassigned(allocation);
         assertThat(allocation.routingNodesChanged(), equalTo(true));
         List<ShardRouting> ignored = allocation.routingNodes().unassigned().ignored();
@@ -272,7 +351,8 @@ public void testDontForceAllocateOnThrottleDecision() {
     public void testPreferAllocatingPreviousPrimary() {
         String primaryAllocId = UUIDs.randomBase64UUID();
         String replicaAllocId = UUIDs.randomBase64UUID();
-        RoutingAllocation allocation = routingAllocationWithOnePrimaryNoReplicas(yesAllocationDeciders(), false, randomFrom(Version.V_2_0_0, Version.CURRENT), primaryAllocId, replicaAllocId);
+        RoutingAllocation allocation = routingAllocationWithOnePrimaryNoReplicas(yesAllocationDeciders(),
+            randomFrom(CLUSTER_RECOVERED, INDEX_REOPENED), randomFrom(Version.V_2_0_0, Version.CURRENT), primaryAllocId, replicaAllocId);
         boolean node1HasPrimaryShard = randomBoolean();
         testAllocator.addData(node1, ShardStateMetaData.NO_VERSION, node1HasPrimaryShard ? primaryAllocId : replicaAllocId, node1HasPrimaryShard);
         testAllocator.addData(node2, ShardStateMetaData.NO_VERSION, node1HasPrimaryShard ? replicaAllocId : primaryAllocId, !node1HasPrimaryShard);
@@ -292,10 +372,11 @@ public void testPreferAllocatingPreviousPrimary() {
     public void testFoundAllocationButThrottlingDecider() {
         final RoutingAllocation allocation;
         if (randomBoolean()) {
-            allocation = routingAllocationWithOnePrimaryNoReplicas(throttleAllocationDeciders(), false, randomFrom(Version.V_2_0_0, Version.CURRENT), "allocId1");
+            allocation = routingAllocationWithOnePrimaryNoReplicas(throttleAllocationDeciders(), CLUSTER_RECOVERED,
+                randomFrom(Version.V_2_0_0, Version.CURRENT), "allocId1");
             testAllocator.addData(node1, ShardStateMetaData.NO_VERSION, "allocId1", randomBoolean());
         } else {
-            allocation = routingAllocationWithOnePrimaryNoReplicas(throttleAllocationDeciders(), false, Version.V_2_2_0);
+            allocation = routingAllocationWithOnePrimaryNoReplicas(throttleAllocationDeciders(), CLUSTER_RECOVERED, Version.V_2_2_0);
             testAllocator.addData(node1, 3, null, randomBoolean());
         }
         testAllocator.allocateUnassigned(allocation);
@@ -312,10 +393,11 @@ public void testFoundAllocationButThrottlingDecider() {
     public void testFoundAllocationButNoDecider() {
         final RoutingAllocation allocation;
         if (randomBoolean()) {
-            allocation = routingAllocationWithOnePrimaryNoReplicas(noAllocationDeciders(), false, randomFrom(Version.V_2_0_0, Version.CURRENT), "allocId1");
+            allocation = routingAllocationWithOnePrimaryNoReplicas(noAllocationDeciders(), CLUSTER_RECOVERED,
+                randomFrom(Version.V_2_0_0, Version.CURRENT), "allocId1");
             testAllocator.addData(node1, ShardStateMetaData.NO_VERSION, "allocId1", randomBoolean());
         } else {
-            allocation = routingAllocationWithOnePrimaryNoReplicas(noAllocationDeciders(), false, Version.V_2_0_0);
+            allocation = routingAllocationWithOnePrimaryNoReplicas(noAllocationDeciders(), CLUSTER_RECOVERED, Version.V_2_0_0);
             testAllocator.addData(node1, 3, null, randomBoolean());
         }
         testAllocator.allocateUnassigned(allocation);
@@ -330,7 +412,7 @@ public void testFoundAllocationButNoDecider() {
      * Tests that the highest version node is chosen for allocation.
      */
     public void testAllocateToTheHighestVersionOnLegacyIndex() {
-        RoutingAllocation allocation = routingAllocationWithOnePrimaryNoReplicas(yesAllocationDeciders(), false, Version.V_2_0_0);
+        RoutingAllocation allocation = routingAllocationWithOnePrimaryNoReplicas(yesAllocationDeciders(), CLUSTER_RECOVERED, Version.V_2_0_0);
         testAllocator.addData(node1, 10, null, randomBoolean()).addData(node2, 12, null, randomBoolean());
         testAllocator.allocateUnassigned(allocation);
         assertThat(allocation.routingNodesChanged(), equalTo(true));
@@ -347,7 +429,7 @@ public void testAllocateToTheHighestVersionOnLegacyIndex() {
      * allocation mode would be chosen).
      */
     public void testVersionBasedAllocationPrefersShardWithAllocationId() {
-        RoutingAllocation allocation = routingAllocationWithOnePrimaryNoReplicas(yesAllocationDeciders(), false, Version.V_2_0_0);
+        RoutingAllocation allocation = routingAllocationWithOnePrimaryNoReplicas(yesAllocationDeciders(), CLUSTER_RECOVERED, Version.V_2_0_0);
         testAllocator.addData(node1, 10, null, randomBoolean());
         testAllocator.addData(node2, ShardStateMetaData.NO_VERSION, "some allocId", randomBoolean());
         testAllocator.addData(node3, 12, null, randomBoolean());
@@ -616,17 +698,27 @@ public void testEnoughCopiesFoundForAllocationOnLegacyIndexWithDifferentVersion(
         assertClusterHealthStatus(allocation, ClusterHealthStatus.RED);
     }
 
-    private RoutingAllocation routingAllocationWithOnePrimaryNoReplicas(AllocationDeciders deciders, boolean asNew, Version version,
+    private RoutingAllocation routingAllocationWithOnePrimaryNoReplicas(AllocationDeciders deciders,
+                                                                        UnassignedInfo.Reason reason, Version version,
                                                                         String... activeAllocationIds) {
         MetaData metaData = MetaData.builder()
                 .put(IndexMetaData.builder(shardId.getIndexName()).settings(settings(version))
                     .numberOfShards(1).numberOfReplicas(0).putInSyncAllocationIds(shardId.id(), Sets.newHashSet(activeAllocationIds)))
                 .build();
         RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
-        if (asNew) {
-            routingTableBuilder.addAsNew(metaData.index(shardId.getIndex()));
-        } else {
-            routingTableBuilder.addAsRecovery(metaData.index(shardId.getIndex()));
+        switch (reason) {
+
+            case INDEX_CREATED:
+                routingTableBuilder.addAsNew(metaData.index(shardId.getIndex()));
+                break;
+            case CLUSTER_RECOVERED:
+                routingTableBuilder.addAsRecovery(metaData.index(shardId.getIndex()));
+                break;
+            case INDEX_REOPENED:
+                routingTableBuilder.addAsFromCloseToOpen(metaData.index(shardId.getIndex()));
+                break;
+            default:
+                throw new IllegalArgumentException("can't do " + reason + " for you. teach me");
         }
         ClusterState state = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
                 .metaData(metaData)
@@ -674,7 +766,7 @@ class TestAllocator extends PrimaryShardAllocator {
 
         private Map<DiscoveryNode, TransportNodesListGatewayStartedShards.NodeGatewayStartedShards> data;
 
-        public TestAllocator() {
+        TestAllocator() {
             super(Settings.EMPTY);
         }
 
@@ -697,7 +789,7 @@ public TestAllocator addData(DiscoveryNode node, long version, String allocation
 
         @Override
         protected AsyncShardFetch.FetchResult<TransportNodesListGatewayStartedShards.NodeGatewayStartedShards> fetchData(ShardRouting shard, RoutingAllocation allocation) {
-            return new AsyncShardFetch.FetchResult<>(shardId, data, Collections.<String>emptySet(), Collections.<String>emptySet());
+            return new AsyncShardFetch.FetchResult<>(shardId, data, Collections.<String>emptySet());
         }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/gateway/QuorumGatewayIT.java b/core/src/test/java/org/elasticsearch/gateway/QuorumGatewayIT.java
index a1d16bfd884fd..3abaff3295924 100644
--- a/core/src/test/java/org/elasticsearch/gateway/QuorumGatewayIT.java
+++ b/core/src/test/java/org/elasticsearch/gateway/QuorumGatewayIT.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.discovery.zen.elect.ElectMasterService;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
@@ -37,10 +36,7 @@
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
 
-/**
- *
- */
-@ClusterScope(numDataNodes =0, scope= Scope.TEST)
+@ClusterScope(numDataNodes = 0, scope = Scope.TEST)
 public class QuorumGatewayIT extends ESIntegTestCase {
     @Override
     protected int numberOfReplicas() {
@@ -50,9 +46,7 @@ protected int numberOfReplicas() {
     public void testQuorumRecovery() throws Exception {
         logger.info("--> starting 3 nodes");
         // we are shutting down nodes - make sure we don't have 2 clusters if we test network
-        internalCluster().startNodesAsync(3,
-                Settings.builder().put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), 2).build()).get();
-
+        internalCluster().startNodes(3);
 
         createIndex("test");
         ensureGreen();
diff --git a/core/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java b/core/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java
index 1e35bcdd46939..973c9e87b0644 100644
--- a/core/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java
+++ b/core/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.discovery.zen.ElectMasterService;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
@@ -34,7 +35,7 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasItem;
 
-@ClusterScope(scope = Scope.TEST, numDataNodes = 0)
+@ClusterScope(scope = Scope.TEST, numDataNodes = 0, autoMinMasterNodes = false)
 public class RecoverAfterNodesIT extends ESIntegTestCase {
     private static final TimeValue BLOCK_WAIT_TIMEOUT = TimeValue.timeValueSeconds(10);
 
@@ -49,20 +50,22 @@ public Set<ClusterBlock> waitForNoBlocksOnNode(TimeValue timeout, Client nodeCli
         return blocks;
     }
 
-    public Client startNode(Settings.Builder settings) {
-        String name = internalCluster().startNode(settings);
+    public Client startNode(Settings.Builder settings, int minMasterNodes) {
+        String name = internalCluster().startNode(
+            Settings.builder().put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), minMasterNodes)
+                .put(settings.build()));
         return internalCluster().client(name);
     }
 
     public void testRecoverAfterNodes() throws Exception {
         logger.info("--> start node (1)");
-        Client clientNode1 = startNode(Settings.builder().put("gateway.recover_after_nodes", 3));
+        Client clientNode1 = startNode(Settings.builder().put("gateway.recover_after_nodes", 3), 1);
         assertThat(clientNode1.admin().cluster().prepareState().setLocal(true).execute().actionGet()
                 .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE),
                 hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK));
 
         logger.info("--> start node (2)");
-        Client clientNode2 = startNode(Settings.builder().put("gateway.recover_after_nodes", 3));
+        Client clientNode2 = startNode(Settings.builder().put("gateway.recover_after_nodes", 3), 1);
         Thread.sleep(BLOCK_WAIT_TIMEOUT.millis());
         assertThat(clientNode1.admin().cluster().prepareState().setLocal(true).execute().actionGet()
                 .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE),
@@ -72,7 +75,7 @@ public void testRecoverAfterNodes() throws Exception {
                 hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK));
 
         logger.info("--> start node (3)");
-        Client clientNode3 = startNode(Settings.builder().put("gateway.recover_after_nodes", 3));
+        Client clientNode3 = startNode(Settings.builder().put("gateway.recover_after_nodes", 3), 1);
 
         assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, clientNode1).isEmpty(), equalTo(true));
         assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, clientNode2).isEmpty(), equalTo(true));
@@ -81,13 +84,17 @@ public void testRecoverAfterNodes() throws Exception {
 
     public void testRecoverAfterMasterNodes() throws Exception {
         logger.info("--> start master_node (1)");
-        Client master1 = startNode(Settings.builder().put("gateway.recover_after_master_nodes", 2).put(Node.NODE_DATA_SETTING.getKey(), false).put(Node.NODE_MASTER_SETTING.getKey(), true));
+        Client master1 = startNode(Settings.builder()
+            .put("gateway.recover_after_master_nodes", 2).put(Node.NODE_DATA_SETTING.getKey(), false)
+            .put(Node.NODE_MASTER_SETTING.getKey(), true), 1);
         assertThat(master1.admin().cluster().prepareState().setLocal(true).execute().actionGet()
                 .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE),
                 hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK));
 
         logger.info("--> start data_node (1)");
-        Client data1 = startNode(Settings.builder().put("gateway.recover_after_master_nodes", 2).put(Node.NODE_DATA_SETTING.getKey(), true).put(Node.NODE_MASTER_SETTING.getKey(), false));
+        Client data1 = startNode(Settings.builder()
+            .put("gateway.recover_after_master_nodes", 2)
+            .put(Node.NODE_DATA_SETTING.getKey(), true).put(Node.NODE_MASTER_SETTING.getKey(), false), 1);
         assertThat(master1.admin().cluster().prepareState().setLocal(true).execute().actionGet()
                 .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE),
                 hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK));
@@ -96,7 +103,9 @@ public void testRecoverAfterMasterNodes() throws Exception {
                 hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK));
 
         logger.info("--> start data_node (2)");
-        Client data2 = startNode(Settings.builder().put("gateway.recover_after_master_nodes", 2).put(Node.NODE_DATA_SETTING.getKey(), true).put(Node.NODE_MASTER_SETTING.getKey(), false));
+        Client data2 = startNode(Settings.builder()
+            .put("gateway.recover_after_master_nodes", 2).put(Node.NODE_DATA_SETTING.getKey(), true)
+            .put(Node.NODE_MASTER_SETTING.getKey(), false), 1);
         assertThat(master1.admin().cluster().prepareState().setLocal(true).execute().actionGet()
                 .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE),
                 hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK));
@@ -108,7 +117,10 @@ public void testRecoverAfterMasterNodes() throws Exception {
                 hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK));
 
         logger.info("--> start master_node (2)");
-        Client master2 = startNode(Settings.builder().put("gateway.recover_after_master_nodes", 2).put(Node.NODE_DATA_SETTING.getKey(), false).put(Node.NODE_MASTER_SETTING.getKey(), true));
+        Client master2 = startNode(Settings.builder()
+            .put("gateway.recover_after_master_nodes", 2)
+            .put(Node.NODE_DATA_SETTING.getKey(), false)
+            .put(Node.NODE_MASTER_SETTING.getKey(), true), 1);
         assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, master1).isEmpty(), equalTo(true));
         assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, master2).isEmpty(), equalTo(true));
         assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, data1).isEmpty(), equalTo(true));
@@ -117,13 +129,19 @@ public void testRecoverAfterMasterNodes() throws Exception {
 
     public void testRecoverAfterDataNodes() throws Exception {
         logger.info("--> start master_node (1)");
-        Client master1 = startNode(Settings.builder().put("gateway.recover_after_data_nodes", 2).put(Node.NODE_DATA_SETTING.getKey(), false).put(Node.NODE_MASTER_SETTING.getKey(), true));
+        Client master1 = startNode(Settings.builder()
+            .put("gateway.recover_after_data_nodes", 2)
+            .put(Node.NODE_DATA_SETTING.getKey(), false)
+            .put(Node.NODE_MASTER_SETTING.getKey(), true), 1);
         assertThat(master1.admin().cluster().prepareState().setLocal(true).execute().actionGet()
                 .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE),
                 hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK));
 
         logger.info("--> start data_node (1)");
-        Client data1 = startNode(Settings.builder().put("gateway.recover_after_data_nodes", 2).put(Node.NODE_DATA_SETTING.getKey(), true).put(Node.NODE_MASTER_SETTING.getKey(), false));
+        Client data1 = startNode(Settings.builder()
+            .put("gateway.recover_after_data_nodes", 2)
+            .put(Node.NODE_DATA_SETTING.getKey(), true)
+            .put(Node.NODE_MASTER_SETTING.getKey(), false), 1);
         assertThat(master1.admin().cluster().prepareState().setLocal(true).execute().actionGet()
                 .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE),
                 hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK));
@@ -132,7 +150,10 @@ public void testRecoverAfterDataNodes() throws Exception {
                 hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK));
 
         logger.info("--> start master_node (2)");
-        Client master2 = startNode(Settings.builder().put("gateway.recover_after_data_nodes", 2).put(Node.NODE_DATA_SETTING.getKey(), false).put(Node.NODE_MASTER_SETTING.getKey(), true));
+        Client master2 = startNode(Settings.builder()
+            .put("gateway.recover_after_data_nodes", 2)
+            .put(Node.NODE_DATA_SETTING.getKey(), false)
+            .put(Node.NODE_MASTER_SETTING.getKey(), true), 1);
         assertThat(master2.admin().cluster().prepareState().setLocal(true).execute().actionGet()
                 .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE),
                 hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK));
@@ -144,7 +165,10 @@ public void testRecoverAfterDataNodes() throws Exception {
                 hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK));
 
         logger.info("--> start data_node (2)");
-        Client data2 = startNode(Settings.builder().put("gateway.recover_after_data_nodes", 2).put(Node.NODE_DATA_SETTING.getKey(), true).put(Node.NODE_MASTER_SETTING.getKey(), false));
+        Client data2 = startNode(Settings.builder()
+            .put("gateway.recover_after_data_nodes", 2)
+            .put(Node.NODE_DATA_SETTING.getKey(), true)
+            .put(Node.NODE_MASTER_SETTING.getKey(), false), 1);
         assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, master1).isEmpty(), equalTo(true));
         assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, master2).isEmpty(), equalTo(true));
         assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, data1).isEmpty(), equalTo(true));
diff --git a/core/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java b/core/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java
index 824a6bbaf3924..b30ffa70ab702 100644
--- a/core/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java
+++ b/core/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
 import org.elasticsearch.action.admin.indices.stats.IndexStats;
 import org.elasticsearch.action.admin.indices.stats.ShardStats;
-import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -44,6 +43,7 @@
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
+import org.elasticsearch.test.InternalTestCluster;
 import org.elasticsearch.test.InternalTestCluster.RestartCallback;
 import org.elasticsearch.test.store.MockFSDirectoryService;
 import org.elasticsearch.test.store.MockFSIndexStore;
@@ -315,7 +315,7 @@ public boolean clearData(String nodeName) {
 
     public void testLatestVersionLoaded() throws Exception {
         // clean two nodes
-        internalCluster().startNodesAsync(2, Settings.builder().put("gateway.recover_after_nodes", 2).build()).get();
+        internalCluster().startNodes(2, Settings.builder().put("gateway.recover_after_nodes", 2).build());
 
         client().prepareIndex("test", "type1", "1").setSource(jsonBuilder().startObject().field("field", "value1").endObject()).execute().actionGet();
         client().admin().indices().prepareFlush().execute().actionGet();
@@ -332,48 +332,43 @@ public void testLatestVersionLoaded() throws Exception {
         String metaDataUuid = client().admin().cluster().prepareState().execute().get().getState().getMetaData().clusterUUID();
         assertThat(metaDataUuid, not(equalTo("_na_")));
 
-        Map<String, long[]> primaryTerms = assertAndCapturePrimaryTerms(null);
-
         logger.info("--> closing first node, and indexing more data to the second node");
-        internalCluster().fullRestart(new RestartCallback() {
+        internalCluster().stopRandomDataNode();
 
-            @Override
-            public void doAfterNodes(int numNodes, Client client) throws Exception {
-                if (numNodes == 1) {
-                    logger.info("--> one node is closed - start indexing data into the second one");
-                    client.prepareIndex("test", "type1", "3").setSource(jsonBuilder().startObject().field("field", "value3").endObject()).execute().actionGet();
-                    // TODO: remove once refresh doesn't fail immediately if there a master block:
-                    // https://github.com/elastic/elasticsearch/issues/9997
-                    client.admin().cluster().prepareHealth("test").setWaitForYellowStatus().get();
-                    client.admin().indices().prepareRefresh().execute().actionGet();
-
-                    for (int i = 0; i < 10; i++) {
-                        assertHitCount(client.prepareSearch().setSize(0).setQuery(matchAllQuery()).execute().actionGet(), 3);
-                    }
+        logger.info("--> one node is closed - start indexing data into the second one");
+        client().prepareIndex("test", "type1", "3").setSource(jsonBuilder().startObject().field("field", "value3").endObject()).execute().actionGet();
+        // TODO: remove once refresh doesn't fail immediately if there a master block:
+        // https://github.com/elastic/elasticsearch/issues/9997
+        client().admin().cluster().prepareHealth("test").setWaitForYellowStatus().get();
+        client().admin().indices().prepareRefresh().execute().actionGet();
 
-                    logger.info("--> add some metadata, additional type and template");
-                    client.admin().indices().preparePutMapping("test").setType("type2")
-                        .setSource(jsonBuilder().startObject().startObject("type2").endObject().endObject())
-                        .execute().actionGet();
-                    client.admin().indices().preparePutTemplate("template_1")
-                        .setTemplate("te*")
-                        .setOrder(0)
-                        .addMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties")
-                            .startObject("field1").field("type", "text").field("store", true).endObject()
-                            .startObject("field2").field("type", "keyword").field("store", true).endObject()
-                            .endObject().endObject().endObject())
-                        .execute().actionGet();
-                    client.admin().indices().prepareAliases().addAlias("test", "test_alias", QueryBuilders.termQuery("field", "value")).execute().actionGet();
-                    logger.info("--> starting two nodes back, verifying we got the latest version");
-                }
+        for (int i = 0; i < 10; i++) {
+            assertHitCount(client().prepareSearch().setSize(0).setQuery(matchAllQuery()).execute().actionGet(), 3);
+        }
 
-            }
+        logger.info("--> add some metadata, additional type and template");
+        client().admin().indices().preparePutMapping("test").setType("type2")
+            .setSource(jsonBuilder().startObject().startObject("type2").endObject().endObject())
+            .execute().actionGet();
+        client().admin().indices().preparePutTemplate("template_1")
+            .setTemplate("te*")
+            .setOrder(0)
+            .addMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties")
+                .startObject("field1").field("type", "text").field("store", true).endObject()
+                .startObject("field2").field("type", "keyword").field("store", true).endObject()
+                .endObject().endObject().endObject())
+            .execute().actionGet();
+        client().admin().indices().prepareAliases().addAlias("test", "test_alias", QueryBuilders.termQuery("field", "value")).execute().actionGet();
 
-        });
+        logger.info("--> stopping the second node");
+        internalCluster().stopRandomDataNode();
+
+        logger.info("--> starting the two nodes back");
+
+        internalCluster().startNodes(2, Settings.builder().put("gateway.recover_after_nodes", 2).build());
 
         logger.info("--> running cluster_health (wait for the shards to startup)");
         ensureGreen();
-        primaryTerms = assertAndCapturePrimaryTerms(primaryTerms);
 
         assertThat(client().admin().cluster().prepareState().execute().get().getState().getMetaData().clusterUUID(), equalTo(metaDataUuid));
 
@@ -396,7 +391,7 @@ public void testReusePeerRecovery() throws Exception {
             .put(ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_OUTGOING_RECOVERIES_SETTING.getKey(), 4)
             .put(MockFSDirectoryService.CRASH_INDEX_SETTING.getKey(), false).build();
 
-        internalCluster().startNodesAsync(4, settings).get();
+        internalCluster().startNodes(4, settings);
         // prevent any rebalance actions during the peer recovery
         // if we run into a relocation the reuse count will be 0 and this fails the test. We are testing here if
         // we reuse the files on disk after full restarts for replicas.
@@ -417,7 +412,7 @@ public void testReusePeerRecovery() throws Exception {
         logger.info("Running Cluster Health");
         ensureGreen();
         client().admin().indices().prepareForceMerge("test").setMaxNumSegments(100).get(); // just wait for merges
-        client().admin().indices().prepareFlush().setWaitIfOngoing(true).setForce(true).get();
+        client().admin().indices().prepareFlush().setForce(true).get();
 
         boolean useSyncIds = randomBoolean();
         if (useSyncIds == false) {
@@ -501,27 +496,28 @@ public void assertSyncIdsNotNull() {
     public void testRecoveryDifferentNodeOrderStartup() throws Exception {
         // we need different data paths so we make sure we start the second node fresh
 
-        final String node_1 = internalCluster().startNode(Settings.builder().put(Environment.PATH_DATA_SETTING.getKey(), createTempDir()).build());
+        final Path pathNode1 = createTempDir();
+        final String node_1 = internalCluster().startNode(Settings.builder().put(Environment.PATH_DATA_SETTING.getKey(), pathNode1).build());
 
         client().prepareIndex("test", "type1", "1").setSource("field", "value").execute().actionGet();
 
-        internalCluster().startNode(Settings.builder().put(Environment.PATH_DATA_SETTING.getKey(), createTempDir()).build());
+        final Path pathNode2 = createTempDir();
+        final String node_2 = internalCluster().startNode(Settings.builder().put(Environment.PATH_DATA_SETTING.getKey(), pathNode2).build());
 
         ensureGreen();
         Map<String, long[]> primaryTerms = assertAndCapturePrimaryTerms(null);
 
-
-        internalCluster().fullRestart(new RestartCallback() {
-
-            @Override
-            public boolean doRestart(String nodeName) {
-                return !node_1.equals(nodeName);
-            }
-        });
-
+        if (randomBoolean()) {
+            internalCluster().stopRandomNode(InternalTestCluster.nameFilter(node_1));
+            internalCluster().stopRandomNode(InternalTestCluster.nameFilter(node_2));
+        } else {
+            internalCluster().stopRandomNode(InternalTestCluster.nameFilter(node_2));
+            internalCluster().stopRandomNode(InternalTestCluster.nameFilter(node_1));
+        }
+        // start the second node again
+        internalCluster().startNode(Settings.builder().put(Environment.PATH_DATA_SETTING.getKey(), pathNode2).build());
         ensureYellow();
         primaryTerms = assertAndCapturePrimaryTerms(primaryTerms);
-
         assertThat(client().admin().indices().prepareExists("test").execute().actionGet().isExists(), equalTo(true));
         assertHitCount(client().prepareSearch("test").setSize(0).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet(), 1);
     }
diff --git a/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java b/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java
index 188f10c588e57..c991e420f39ab 100644
--- a/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java
+++ b/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java
@@ -42,6 +42,7 @@
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.cluster.routing.allocation.decider.SameShardAllocationDecider;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.set.Sets;
@@ -211,7 +212,9 @@ public void testNoOrThrottleDecidersRemainsInUnassigned() {
      */
     public void testThrottleWhenAllocatingToMatchingNode() {
         RoutingAllocation allocation = onePrimaryOnNode1And1Replica(new AllocationDeciders(Settings.EMPTY,
-            Arrays.asList(new TestAllocateDecision(Decision.YES), new SameShardAllocationDecider(Settings.EMPTY),
+            Arrays.asList(new TestAllocateDecision(Decision.YES),
+                new SameShardAllocationDecider(
+                    Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)),
                 new AllocationDecider(Settings.EMPTY) {
                     @Override
                     public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
@@ -341,7 +344,7 @@ class TestAllocator extends ReplicaShardAllocator {
         private Map<DiscoveryNode, TransportNodesListShardStoreMetaData.StoreFilesMetaData> data = null;
         private AtomicBoolean fetchDataCalled = new AtomicBoolean(false);
 
-        public TestAllocator() {
+        TestAllocator() {
             super(Settings.EMPTY);
         }
 
@@ -384,7 +387,12 @@ protected AsyncShardFetch.FetchResult<TransportNodesListShardStoreMetaData.NodeS
                     tData.put(entry.getKey(), new TransportNodesListShardStoreMetaData.NodeStoreFilesMetaData(entry.getKey(), entry.getValue()));
                 }
             }
-            return new AsyncShardFetch.FetchResult<>(shardId, tData, Collections.<String>emptySet(), Collections.<String>emptySet());
+            return new AsyncShardFetch.FetchResult<>(shardId, tData, Collections.emptySet());
+        }
+
+        @Override
+        protected boolean hasInitiatedFetching(ShardRouting shard) {
+            return fetchDataCalled.get();
         }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/gateway/ReusePeerRecoverySharedTest.java b/core/src/test/java/org/elasticsearch/gateway/ReusePeerRecoverySharedTest.java
index 6454f8a22091d..81be3057b0161 100644
--- a/core/src/test/java/org/elasticsearch/gateway/ReusePeerRecoverySharedTest.java
+++ b/core/src/test/java/org/elasticsearch/gateway/ReusePeerRecoverySharedTest.java
@@ -80,7 +80,7 @@ public static void testCase(Settings indexSettings, Runnable restartCluster, Log
         client().admin().cluster().prepareHealth().setWaitForGreenStatus().setTimeout("30s").get();
         // just wait for merges
         client().admin().indices().prepareForceMerge("test").setMaxNumSegments(100).get();
-        client().admin().indices().prepareFlush().setWaitIfOngoing(true).setForce(true).get();
+        client().admin().indices().prepareFlush().setForce(true).get();
 
         if (useSyncIds == false) {
             logger.info("--> disabling allocation while the cluster is shut down");
diff --git a/core/src/test/java/org/elasticsearch/get/GetActionIT.java b/core/src/test/java/org/elasticsearch/get/GetActionIT.java
index 57de501d93da2..1134c6b28ae73 100644
--- a/core/src/test/java/org/elasticsearch/get/GetActionIT.java
+++ b/core/src/test/java/org/elasticsearch/get/GetActionIT.java
@@ -37,6 +37,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.engine.VersionConflictEngineException;
 import org.elasticsearch.index.mapper.TimestampFieldMapper;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -84,7 +85,7 @@ public void testSimpleGet() {
         assertThat(response.getSourceAsMap().get("field2").toString(), equalTo("value2"));
 
         logger.info("--> realtime get 1 (no source, implicit)");
-        response = client().prepareGet(indexOrAlias(), "type1", "1").setFields(Strings.EMPTY_ARRAY).get();
+        response = client().prepareGet(indexOrAlias(), "type1", "1").setStoredFields(Strings.EMPTY_ARRAY).get();
         assertThat(response.isExists(), equalTo(true));
         assertThat(response.getIndex(), equalTo("test"));
         Set<String> fields = new HashSet<>(response.getFields().keySet());
@@ -109,7 +110,7 @@ public void testSimpleGet() {
         assertThat(response.getSourceAsMap().get("field2").toString(), equalTo("value2"));
 
         logger.info("--> realtime fetch of field");
-        response = client().prepareGet(indexOrAlias(), "type1", "1").setFields("field1").get();
+        response = client().prepareGet(indexOrAlias(), "type1", "1").setStoredFields("field1").get();
         assertThat(response.isExists(), equalTo(true));
         assertThat(response.getIndex(), equalTo("test"));
         assertThat(response.getSourceAsBytes(), nullValue());
@@ -117,7 +118,8 @@ public void testSimpleGet() {
         assertThat(response.getField("field2"), nullValue());
 
         logger.info("--> realtime fetch of field & source");
-        response = client().prepareGet(indexOrAlias(), "type1", "1").setFields("field1").setFetchSource("field1", null).get();
+        response = client().prepareGet(indexOrAlias(), "type1", "1")
+            .setStoredFields("field1").setFetchSource("field1", null).get();
         assertThat(response.isExists(), equalTo(true));
         assertThat(response.getIndex(), equalTo("test"));
         assertThat(response.getSourceAsMap(), hasKey("field1"));
@@ -143,7 +145,7 @@ public void testSimpleGet() {
         assertThat(response.getSourceAsMap().get("field2").toString(), equalTo("value2"));
 
         logger.info("--> realtime fetch of field (loaded from index)");
-        response = client().prepareGet(indexOrAlias(), "type1", "1").setFields("field1").get();
+        response = client().prepareGet(indexOrAlias(), "type1", "1").setStoredFields("field1").get();
         assertThat(response.isExists(), equalTo(true));
         assertThat(response.getIndex(), equalTo("test"));
         assertThat(response.getSourceAsBytes(), nullValue());
@@ -151,7 +153,8 @@ public void testSimpleGet() {
         assertThat(response.getField("field2"), nullValue());
 
         logger.info("--> realtime fetch of field & source (loaded from index)");
-        response = client().prepareGet(indexOrAlias(), "type1", "1").setFields("field1").setFetchSource(true).get();
+        response = client().prepareGet(indexOrAlias(), "type1", "1")
+            .setStoredFields("field1").setFetchSource(true).get();
         assertThat(response.isExists(), equalTo(true));
         assertThat(response.getIndex(), equalTo("test"));
         assertThat(response.getSourceAsBytes(), not(nullValue()));
@@ -232,8 +235,8 @@ public void testSimpleMultiGet() throws Exception {
 
         // multi get with specific field
         response = client().prepareMultiGet()
-                .add(new MultiGetRequest.Item(indexOrAlias(), "type1", "1").fields("field"))
-                .add(new MultiGetRequest.Item(indexOrAlias(), "type1", "3").fields("field"))
+                .add(new MultiGetRequest.Item(indexOrAlias(), "type1", "1").storedFields("field"))
+                .add(new MultiGetRequest.Item(indexOrAlias(), "type1", "3").storedFields("field"))
                 .get();
 
         assertThat(response.getResponses().length, equalTo(2));
@@ -264,12 +267,12 @@ public void testGetDocWithMultivaluedFields() throws Exception {
         assertThat(response.isExists(), equalTo(false));
 
         client().prepareIndex("test", "type1", "1")
-                .setSource(jsonBuilder().startObject().field("field", "1", "2").endObject()).get();
+                .setSource(jsonBuilder().startObject().array("field", "1", "2").endObject()).get();
 
         client().prepareIndex("test", "type2", "1")
-                .setSource(jsonBuilder().startObject().field("field", "1", "2").endObject()).get();
+                .setSource(jsonBuilder().startObject().array("field", "1", "2").endObject()).get();
 
-        response = client().prepareGet("test", "type1", "1").setFields("field").get();
+        response = client().prepareGet("test", "type1", "1").setStoredFields("field").get();
         assertThat(response.isExists(), equalTo(true));
         assertThat(response.getId(), equalTo("1"));
         assertThat(response.getType(), equalTo("type1"));
@@ -281,7 +284,7 @@ public void testGetDocWithMultivaluedFields() throws Exception {
         assertThat(response.getFields().get("field").getValues().get(1).toString(), equalTo("2"));
 
 
-        response = client().prepareGet("test", "type2", "1").setFields("field").get();
+        response = client().prepareGet("test", "type2", "1").setStoredFields("field").get();
         assertThat(response.isExists(), equalTo(true));
         assertThat(response.getType(), equalTo("type2"));
         assertThat(response.getId(), equalTo("1"));
@@ -294,7 +297,7 @@ public void testGetDocWithMultivaluedFields() throws Exception {
 
         // Now test values being fetched from stored fields.
         refresh();
-        response = client().prepareGet("test", "type1", "1").setFields("field").get();
+        response = client().prepareGet("test", "type1", "1").setStoredFields("field").get();
         assertThat(response.isExists(), equalTo(true));
         assertThat(response.getId(), equalTo("1"));
         fields = new HashSet<>(response.getFields().keySet());
@@ -304,7 +307,7 @@ public void testGetDocWithMultivaluedFields() throws Exception {
         assertThat(response.getFields().get("field").getValues().get(0).toString(), equalTo("1"));
         assertThat(response.getFields().get("field").getValues().get(1).toString(), equalTo("2"));
 
-        response = client().prepareGet("test", "type2", "1").setFields("field").get();
+        response = client().prepareGet("test", "type2", "1").setStoredFields("field").get();
         assertThat(response.isExists(), equalTo(true));
         assertThat(response.getId(), equalTo("1"));
         fields = new HashSet<>(response.getFields().keySet());
@@ -546,7 +549,7 @@ public void testGetFieldsMetaData() throws Exception {
 
         GetResponse getResponse = client().prepareGet(indexOrAlias(), "my-type1", "1")
                 .setRouting("1")
-                .setFields("field1")
+                .setStoredFields("field1")
                 .get();
         assertThat(getResponse.isExists(), equalTo(true));
         assertThat(getResponse.getField("field1").isMetadataField(), equalTo(false));
@@ -559,7 +562,7 @@ public void testGetFieldsMetaData() throws Exception {
         flush();
 
         getResponse = client().prepareGet(indexOrAlias(), "my-type1", "1")
-                .setFields("field1")
+                .setStoredFields("field1")
                 .setRouting("1")
                 .get();
         assertThat(getResponse.isExists(), equalTo(true));
@@ -584,21 +587,18 @@ public void testGetFieldsNonLeafField() throws Exception {
                 .setSource(jsonBuilder().startObject().startObject("field1").field("field2", "value1").endObject().endObject())
                 .get();
 
-        try {
-            client().prepareGet(indexOrAlias(), "my-type1", "1").setFields("field1").get();
-            fail();
-        } catch (IllegalArgumentException e) {
-            //all well
-        }
+
+        IllegalArgumentException exc =
+            expectThrows(IllegalArgumentException.class,
+                () -> client().prepareGet(indexOrAlias(), "my-type1", "1").setStoredFields("field1").get());
+        assertThat(exc.getMessage(), equalTo("field [field1] isn't a leaf field"));
 
         flush();
 
-        try {
-            client().prepareGet(indexOrAlias(), "my-type1", "1").setFields("field1").get();
-            fail();
-        } catch (IllegalArgumentException e) {
-            //all well
-        }
+        exc =
+            expectThrows(IllegalArgumentException.class,
+                () -> client().prepareGet(indexOrAlias(), "my-type1", "1").setStoredFields("field1").get());
+        assertThat(exc.getMessage(), equalTo("field [field1] isn't a leaf field"));
     }
 
     public void testGetFieldsComplexField() throws Exception {
@@ -639,20 +639,20 @@ public void testGetFieldsComplexField() throws Exception {
 
         logger.info("indexing documents");
 
-        client().prepareIndex("my-index", "my-type1", "1").setSource(source).get();
-        client().prepareIndex("my-index", "my-type2", "1").setSource(source).get();
+        client().prepareIndex("my-index", "my-type1", "1").setSource(source, XContentType.JSON).get();
+        client().prepareIndex("my-index", "my-type2", "1").setSource(source, XContentType.JSON).get();
 
         logger.info("checking real time retrieval");
 
         String field = "field1.field2.field3.field4";
-        GetResponse getResponse = client().prepareGet("my-index", "my-type1", "1").setFields(field).get();
+        GetResponse getResponse = client().prepareGet("my-index", "my-type1", "1").setStoredFields(field).get();
         assertThat(getResponse.isExists(), equalTo(true));
         assertThat(getResponse.getField(field).isMetadataField(), equalTo(false));
         assertThat(getResponse.getField(field).getValues().size(), equalTo(2));
         assertThat(getResponse.getField(field).getValues().get(0).toString(), equalTo("value1"));
         assertThat(getResponse.getField(field).getValues().get(1).toString(), equalTo("value2"));
 
-        getResponse = client().prepareGet("my-index", "my-type2", "1").setFields(field).get();
+        getResponse = client().prepareGet("my-index", "my-type2", "1").setStoredFields(field).get();
         assertThat(getResponse.isExists(), equalTo(true));
         assertThat(getResponse.getField(field).isMetadataField(), equalTo(false));
         assertThat(getResponse.getField(field).getValues().size(), equalTo(2));
@@ -677,14 +677,14 @@ public void testGetFieldsComplexField() throws Exception {
 
         logger.info("checking post-flush retrieval");
 
-        getResponse = client().prepareGet("my-index", "my-type1", "1").setFields(field).get();
+        getResponse = client().prepareGet("my-index", "my-type1", "1").setStoredFields(field).get();
         assertThat(getResponse.isExists(), equalTo(true));
         assertThat(getResponse.getField(field).isMetadataField(), equalTo(false));
         assertThat(getResponse.getField(field).getValues().size(), equalTo(2));
         assertThat(getResponse.getField(field).getValues().get(0).toString(), equalTo("value1"));
         assertThat(getResponse.getField(field).getValues().get(1).toString(), equalTo("value2"));
 
-        getResponse = client().prepareGet("my-index", "my-type2", "1").setFields(field).get();
+        getResponse = client().prepareGet("my-index", "my-type2", "1").setStoredFields(field).get();
         assertThat(getResponse.isExists(), equalTo(true));
         assertThat(getResponse.getField(field).isMetadataField(), equalTo(false));
         assertThat(getResponse.getField(field).getValues().size(), equalTo(2));
@@ -711,7 +711,7 @@ public void testGetAllField() throws Exception {
         index("test", "my-type1", "1", "some_field", "some text");
         refresh();
 
-        GetResponse getResponse = client().prepareGet(indexOrAlias(), "my-type1", "1").setFields("_all").get();
+        GetResponse getResponse = client().prepareGet(indexOrAlias(), "my-type1", "1").setStoredFields("_all").get();
         assertNotNull(getResponse.getField("_all").getValue());
         assertThat(getResponse.getField("_all").getValue().toString(), equalTo("some text"));
     }
@@ -732,7 +732,7 @@ public void testUngeneratedFieldsThatAreNeverStored() throws IOException {
                 "    }\n" +
                 "  }\n" +
                 "}";
-        assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setSource(createIndexSource));
+        assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setSource(createIndexSource, XContentType.JSON));
         ensureGreen();
         String doc = "{\n" +
                 "  \"suggest\": {\n" +
@@ -771,10 +771,11 @@ public void testUngeneratedFieldsThatAreAlwaysStored() throws IOException {
                 "    }\n" +
                 "  }\n" +
                 "}";
-        assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setSource(createIndexSource));
+        assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setSource(createIndexSource, XContentType.JSON));
         ensureGreen();
 
-        client().prepareIndex("test", "doc").setId("1").setSource("{}").setParent("1").setTTL(TimeValue.timeValueHours(1).getMillis()).get();
+        client().prepareIndex("test", "doc").setId("1").setSource("{}", XContentType.JSON).setParent("1")
+            .setTTL(TimeValue.timeValueHours(1).getMillis()).get();
 
         String[] fieldsList = {"_parent"};
         // before refresh - document is only in translog
@@ -795,12 +796,12 @@ public void testUngeneratedFieldsNotPartOfSourceStored() throws IOException {
             "  }\n" +
             "}";
 
-        assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setSource(createIndexSource));
+        assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setSource(createIndexSource, XContentType.JSON));
         ensureGreen();
         String doc = "{\n" +
             "  \"text\": \"some text.\"\n" +
             "}\n";
-        client().prepareIndex("test", "doc").setId("1").setSource(doc).setRouting("1").get();
+        client().prepareIndex("test", "doc").setId("1").setSource(doc, XContentType.JSON).setRouting("1").get();
         String[] fieldsList = {"_routing"};
         // before refresh - document is only in translog
         assertGetFieldsAlwaysWorks(indexOrAlias(), "doc", "1", fieldsList, "1");
@@ -853,7 +854,7 @@ void indexSingleDocumentWithStringFieldsGeneratedFromText(boolean stored, boolea
                 "  }\n" +
                 "}";
 
-        assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setSource(createIndexSource));
+        assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setSource(createIndexSource, XContentType.JSON));
         ensureGreen();
         String doc = "{\n" +
                 "  \"text1\": \"some text.\"\n," +
@@ -915,7 +916,7 @@ void indexSingleDocumentWithNumericFieldsGeneratedFromText(boolean stored, boole
                 "  }\n" +
                 "}";
 
-        assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setSource(createIndexSource));
+        assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setSource(createIndexSource, XContentType.JSON));
         ensureGreen();
         String doc = "{\n" +
                 "  \"token_count\": \"A text with five words.\",\n" +
@@ -930,36 +931,30 @@ private void assertGetFieldsAlwaysWorks(String index, String type, String docId,
 
     private void assertGetFieldsAlwaysWorks(String index, String type, String docId, String[] fields, @Nullable String routing) {
         for (String field : fields) {
-            assertGetFieldWorks(index, type, docId, field, false, routing);
-            assertGetFieldWorks(index, type, docId, field, true, routing);
+            assertGetFieldWorks(index, type, docId, field, routing);
+            assertGetFieldWorks(index, type, docId, field, routing);
         }
     }
 
-    private void assertGetFieldWorks(String index, String type, String docId, String field, boolean ignoreErrors, @Nullable String routing) {
-        GetResponse response = getDocument(index, type, docId, field, ignoreErrors, routing);
+    private void assertGetFieldWorks(String index, String type, String docId, String field, @Nullable String routing) {
+        GetResponse response = getDocument(index, type, docId, field, routing);
         assertThat(response.getId(), equalTo(docId));
         assertTrue(response.isExists());
         assertNotNull(response.getField(field));
-        response = multiGetDocument(index, type, docId, field, ignoreErrors, routing);
+        response = multiGetDocument(index, type, docId, field, routing);
         assertThat(response.getId(), equalTo(docId));
         assertTrue(response.isExists());
         assertNotNull(response.getField(field));
     }
 
-    protected void assertGetFieldsException(String index, String type, String docId, String[] fields) {
-        for (String field : fields) {
-            assertGetFieldException(index, type, docId, field);
-        }
-    }
-
     private void assertGetFieldException(String index, String type, String docId, String field) {
         try {
-            client().prepareGet().setIndex(index).setType(type).setId(docId).setFields(field).setIgnoreErrorsOnGeneratedFields(false).get();
+            client().prepareGet().setIndex(index).setType(type).setId(docId).setStoredFields(field);
             fail();
         } catch (ElasticsearchException e) {
             assertTrue(e.getMessage().contains("You can only get this field after refresh() has been called."));
         }
-        MultiGetResponse multiGetResponse = client().prepareMultiGet().add(new MultiGetRequest.Item(index, type, docId).fields(field)).setIgnoreErrorsOnGeneratedFields(false).get();
+        MultiGetResponse multiGetResponse = client().prepareMultiGet().add(new MultiGetRequest.Item(index, type, docId).storedFields(field)).get();
         assertNull(multiGetResponse.getResponses()[0].getResponse());
         assertTrue(multiGetResponse.getResponses()[0].getFailure().getMessage().contains("You can only get this field after refresh() has been called."));
     }
@@ -970,7 +965,7 @@ protected void assertGetFieldsNull(String index, String type, String docId, Stri
 
     protected void assertGetFieldsNull(String index, String type, String docId, String[] fields, @Nullable String routing) {
         for (String field : fields) {
-            assertGetFieldNull(index, type, docId, field, true, routing);
+            assertGetFieldNull(index, type, docId, field, routing);
         }
     }
 
@@ -980,37 +975,37 @@ protected void assertGetFieldsAlwaysNull(String index, String type, String docId
 
     protected void assertGetFieldsAlwaysNull(String index, String type, String docId, String[] fields, @Nullable String routing) {
         for (String field : fields) {
-            assertGetFieldNull(index, type, docId, field, true, routing);
-            assertGetFieldNull(index, type, docId, field, false, routing);
+            assertGetFieldNull(index, type, docId, field, routing);
+            assertGetFieldNull(index, type, docId, field, routing);
         }
     }
 
-    protected void assertGetFieldNull(String index, String type, String docId, String field, boolean ignoreErrors, @Nullable String routing) {
+    protected void assertGetFieldNull(String index, String type, String docId, String field, @Nullable String routing) {
         //for get
-        GetResponse response = getDocument(index, type, docId, field, ignoreErrors, routing);
+        GetResponse response = getDocument(index, type, docId, field, routing);
         assertTrue(response.isExists());
         assertNull(response.getField(field));
         assertThat(response.getId(), equalTo(docId));
         //same for multi get
-        response = multiGetDocument(index, type, docId, field, ignoreErrors, routing);
+        response = multiGetDocument(index, type, docId, field, routing);
         assertNull(response.getField(field));
         assertThat(response.getId(), equalTo(docId));
         assertTrue(response.isExists());
     }
 
-    private GetResponse multiGetDocument(String index, String type, String docId, String field, boolean ignoreErrors, @Nullable String routing) {
-        MultiGetRequest.Item getItem = new MultiGetRequest.Item(index, type, docId).fields(field);
+    private GetResponse multiGetDocument(String index, String type, String docId, String field, @Nullable String routing) {
+      MultiGetRequest.Item getItem = new MultiGetRequest.Item(index, type, docId).storedFields(field);
         if (routing != null) {
             getItem.routing(routing);
         }
-        MultiGetRequestBuilder multiGetRequestBuilder = client().prepareMultiGet().add(getItem).setIgnoreErrorsOnGeneratedFields(ignoreErrors);
+        MultiGetRequestBuilder multiGetRequestBuilder = client().prepareMultiGet().add(getItem);
         MultiGetResponse multiGetResponse = multiGetRequestBuilder.get();
         assertThat(multiGetResponse.getResponses().length, equalTo(1));
         return multiGetResponse.getResponses()[0].getResponse();
     }
 
-    private GetResponse getDocument(String index, String type, String docId, String field, boolean ignoreErrors, @Nullable String routing) {
-        GetRequestBuilder getRequestBuilder = client().prepareGet().setIndex(index).setType(type).setId(docId).setFields(field).setIgnoreErrorsOnGeneratedFields(ignoreErrors);
+    private GetResponse getDocument(String index, String type, String docId, String field, @Nullable String routing) {
+        GetRequestBuilder getRequestBuilder = client().prepareGet().setIndex(index).setType(type).setId(docId).setStoredFields(field);
         if (routing != null) {
             getRequestBuilder.setRouting(routing);
         }
diff --git a/core/src/test/java/org/elasticsearch/http/HttpServerTests.java b/core/src/test/java/org/elasticsearch/http/HttpServerTests.java
deleted file mode 100644
index 87167cdb73345..0000000000000
--- a/core/src/test/java/org/elasticsearch/http/HttpServerTests.java
+++ /dev/null
@@ -1,231 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.http;
-
-import java.util.Collections;
-import java.util.Map;
-
-import org.elasticsearch.common.breaker.CircuitBreaker;
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.component.AbstractLifecycleComponent;
-import org.elasticsearch.common.settings.ClusterSettings;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.BoundTransportAddress;
-import org.elasticsearch.common.transport.LocalTransportAddress;
-import org.elasticsearch.common.transport.TransportAddress;
-import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.indices.breaker.CircuitBreakerService;
-import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService;
-import org.elasticsearch.rest.AbstractRestChannel;
-import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.RestController;
-import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.RestResponse;
-import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.test.ESTestCase;
-import org.junit.Before;
-
-public class HttpServerTests extends ESTestCase {
-    private static final ByteSizeValue BREAKER_LIMIT = new ByteSizeValue(20);
-    private HttpServer httpServer;
-    private CircuitBreaker inFlightRequestsBreaker;
-
-    @Before
-    public void setup() {
-        Settings settings = Settings.EMPTY;
-        CircuitBreakerService circuitBreakerService = new HierarchyCircuitBreakerService(
-            Settings.builder()
-                .put(HierarchyCircuitBreakerService.IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), BREAKER_LIMIT)
-                .build(),
-            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
-        // we can do this here only because we know that we don't adjust breaker settings dynamically in the test
-        inFlightRequestsBreaker = circuitBreakerService.getBreaker(CircuitBreaker.IN_FLIGHT_REQUESTS);
-
-        HttpServerTransport httpServerTransport = new TestHttpServerTransport();
-        RestController restController = new RestController(settings, Collections.emptySet());
-        restController.registerHandler(RestRequest.Method.GET, "/",
-            (request, channel, client) -> channel.sendResponse(
-                new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY)));
-        restController.registerHandler(RestRequest.Method.GET, "/error", (request, channel, client) -> {
-                throw new IllegalArgumentException("test error");
-            });
-
-        httpServer = new HttpServer(settings, httpServerTransport, restController, null, circuitBreakerService);
-        httpServer.start();
-    }
-
-    public void testDispatchRequestAddsAndFreesBytesOnSuccess() {
-        int contentLength = BREAKER_LIMIT.bytesAsInt();
-        String content = randomAsciiOfLength(contentLength);
-        TestRestRequest request = new TestRestRequest("/", content);
-        AssertingChannel channel = new AssertingChannel(request, true, RestStatus.OK);
-
-        httpServer.dispatchRequest(request, channel, new ThreadContext(Settings.EMPTY));
-
-        assertEquals(0, inFlightRequestsBreaker.getTrippedCount());
-        assertEquals(0, inFlightRequestsBreaker.getUsed());
-    }
-
-    public void testDispatchRequestAddsAndFreesBytesOnError() {
-        int contentLength = BREAKER_LIMIT.bytesAsInt();
-        String content = randomAsciiOfLength(contentLength);
-        TestRestRequest request = new TestRestRequest("/error", content);
-        AssertingChannel channel = new AssertingChannel(request, true, RestStatus.BAD_REQUEST);
-
-        httpServer.dispatchRequest(request, channel, new ThreadContext(Settings.EMPTY));
-
-        assertEquals(0, inFlightRequestsBreaker.getTrippedCount());
-        assertEquals(0, inFlightRequestsBreaker.getUsed());
-    }
-
-    public void testDispatchRequestAddsAndFreesBytesOnlyOnceOnError() {
-        int contentLength = BREAKER_LIMIT.bytesAsInt();
-        String content = randomAsciiOfLength(contentLength);
-        // we will produce an error in the rest handler and one more when sending the error response
-        TestRestRequest request = new TestRestRequest("/error", content);
-        ExceptionThrowingChannel channel = new ExceptionThrowingChannel(request, true);
-
-        httpServer.dispatchRequest(request, channel, new ThreadContext(Settings.EMPTY));
-
-        assertEquals(0, inFlightRequestsBreaker.getTrippedCount());
-        assertEquals(0, inFlightRequestsBreaker.getUsed());
-    }
-
-    public void testDispatchRequestLimitsBytes() {
-        int contentLength = BREAKER_LIMIT.bytesAsInt() + 1;
-        String content = randomAsciiOfLength(contentLength);
-        TestRestRequest request = new TestRestRequest("/", content);
-        AssertingChannel channel = new AssertingChannel(request, true, RestStatus.SERVICE_UNAVAILABLE);
-
-        httpServer.dispatchRequest(request, channel, new ThreadContext(Settings.EMPTY));
-
-        assertEquals(1, inFlightRequestsBreaker.getTrippedCount());
-        assertEquals(0, inFlightRequestsBreaker.getUsed());
-    }
-
-    private static final class TestHttpServerTransport extends AbstractLifecycleComponent implements
-        HttpServerTransport {
-
-        public TestHttpServerTransport() {
-            super(Settings.EMPTY);
-        }
-
-        @Override
-        protected void doStart() {
-        }
-
-        @Override
-        protected void doStop() {
-        }
-
-        @Override
-        protected void doClose() {
-        }
-
-        @Override
-        public BoundTransportAddress boundAddress() {
-            LocalTransportAddress transportAddress = new LocalTransportAddress("1");
-            return new BoundTransportAddress(new TransportAddress[] {transportAddress} ,transportAddress);
-        }
-
-        @Override
-        public HttpInfo info() {
-            return null;
-        }
-
-        @Override
-        public HttpStats stats() {
-            return null;
-        }
-
-        @Override
-        public void httpServerAdapter(HttpServerAdapter httpServerAdapter) {
-
-        }
-    }
-
-    private static final class AssertingChannel extends AbstractRestChannel {
-        private final RestStatus expectedStatus;
-
-        protected AssertingChannel(RestRequest request, boolean detailedErrorsEnabled, RestStatus expectedStatus) {
-            super(request, detailedErrorsEnabled);
-            this.expectedStatus = expectedStatus;
-        }
-
-        @Override
-        public void sendResponse(RestResponse response) {
-            assertEquals(expectedStatus, response.status());
-        }
-    }
-
-    private static final class ExceptionThrowingChannel extends AbstractRestChannel {
-
-        protected ExceptionThrowingChannel(RestRequest request, boolean detailedErrorsEnabled) {
-            super(request, detailedErrorsEnabled);
-        }
-
-        @Override
-        public void sendResponse(RestResponse response) {
-            throw new IllegalStateException("always throwing an exception for testing");
-        }
-    }
-
-    private static final class TestRestRequest extends RestRequest {
-
-        private final BytesReference content;
-
-        private TestRestRequest(String path, String content) {
-            super(Collections.emptyMap(), path);
-            this.content = new BytesArray(content);
-        }
-
-        @Override
-        public Method method() {
-            return Method.GET;
-        }
-
-        @Override
-        public String uri() {
-            return null;
-        }
-
-        @Override
-        public boolean hasContent() {
-            return true;
-        }
-
-        @Override
-        public BytesReference content() {
-            return content;
-        }
-
-        @Override
-        public String header(String name) {
-            return null;
-        }
-
-        @Override
-        public Iterable<Map.Entry<String, String>> headers() {
-            return null;
-        }
-
-    }
-}
diff --git a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java
index e2d9770e1c2fb..569fc88c1a4ed 100644
--- a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java
@@ -30,12 +30,11 @@
 import org.apache.lucene.search.Weight;
 import org.apache.lucene.search.similarities.BM25Similarity;
 import org.apache.lucene.search.similarities.Similarity;
+import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.SetOnce.AlreadySetException;
 import org.elasticsearch.Version;
-import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
@@ -49,7 +48,10 @@
 import org.elasticsearch.index.cache.query.QueryCache;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.engine.EngineException;
+import org.elasticsearch.index.engine.InternalEngineTests;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
+import org.elasticsearch.index.mapper.ParsedDocument;
+import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.index.shard.IndexEventListener;
 import org.elasticsearch.index.shard.IndexSearcherWrapper;
 import org.elasticsearch.index.shard.IndexingOperationListener;
@@ -63,12 +65,11 @@
 import org.elasticsearch.indices.IndicesQueryCache;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
+import org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason;
 import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
 import org.elasticsearch.indices.mapper.MapperRegistry;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.script.ScriptContextRegistry;
 import org.elasticsearch.script.ScriptEngineRegistry;
-import org.elasticsearch.script.ScriptEngineService;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.ScriptSettings;
 import org.elasticsearch.search.internal.SearchContext;
@@ -82,12 +83,11 @@
 import org.elasticsearch.watcher.ResourceWatcherService;
 
 import java.io.IOException;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.function.Consumer;
 
+import static java.util.Collections.emptyList;
 import static java.util.Collections.emptyMap;
 
 public class IndexModuleTests extends ESTestCase {
@@ -96,7 +96,6 @@ public class IndexModuleTests extends ESTestCase {
     private IndexSettings indexSettings;
     private Environment environment;
     private NodeEnvironment nodeEnvironment;
-    private NodeServicesProvider nodeServicesProvider;
     private IndicesQueryCache indicesQueryCache;
 
     private IndexService.ShardStoreDeleter deleter = new IndexService.ShardStoreDeleter() {
@@ -110,30 +109,30 @@ public void addPendingDelete(ShardId shardId, IndexSettings indexSettings) {
 
     private final IndexFieldDataCache.Listener listener = new IndexFieldDataCache.Listener() {};
     private MapperRegistry mapperRegistry;
-
-    static NodeServicesProvider newNodeServiceProvider(Settings settings, Environment environment, Client client, ScriptEngineService... scriptEngineServices) throws IOException {
-        // TODO this can be used in other place too - lets first refactor the IndicesQueriesRegistry
-        ThreadPool threadPool = new TestThreadPool("test");
-        CircuitBreakerService circuitBreakerService = new NoneCircuitBreakerService();
-        BigArrays bigArrays = new BigArrays(settings, circuitBreakerService);
-        ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Arrays.asList(scriptEngineServices));
-        ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList());
-        ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
-        ScriptService scriptService = new ScriptService(settings, environment, new ResourceWatcherService(settings, threadPool), scriptEngineRegistry, scriptContextRegistry, scriptSettings);
-        IndicesQueriesRegistry indicesQueriesRegistry = new IndicesQueriesRegistry();
-        ClusterService clusterService = ClusterServiceUtils.createClusterService(threadPool);
-        return new NodeServicesProvider(threadPool, bigArrays, client, scriptService, indicesQueriesRegistry, circuitBreakerService, clusterService);
-    }
+    private ThreadPool threadPool;
+    private CircuitBreakerService circuitBreakerService;
+    private BigArrays bigArrays;
+    private ScriptService scriptService;
+    private ClusterService clusterService;
 
     @Override
     public void setUp() throws Exception {
         super.setUp();
-        settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build();
+        settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build();
         indicesQueryCache = new IndicesQueryCache(settings);
         indexSettings = IndexSettingsModule.newIndexSettings("foo", settings);
         index = indexSettings.getIndex();
         environment = new Environment(settings);
-        nodeServicesProvider = newNodeServiceProvider(settings, environment, null);
+        threadPool = new TestThreadPool("test");
+        circuitBreakerService = new NoneCircuitBreakerService();
+        bigArrays = new BigArrays(settings, circuitBreakerService);
+        ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(emptyList());
+        ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList());
+        ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
+        scriptService = new ScriptService(settings, environment, new ResourceWatcherService(settings, threadPool), scriptEngineRegistry,
+                scriptContextRegistry, scriptSettings);
+        clusterService = ClusterServiceUtils.createClusterService(threadPool);
         nodeEnvironment = new NodeEnvironment(settings, environment);
         mapperRegistry = new IndicesModule(Collections.emptyList()).getMapperRegistry();
     }
@@ -141,18 +140,22 @@ public void setUp() throws Exception {
     @Override
     public void tearDown() throws Exception {
         super.tearDown();
-        nodeEnvironment.close();
-        indicesQueryCache.close();
-        nodeServicesProvider.getClusterService().close();
-        ThreadPool.terminate(nodeServicesProvider.getThreadPool(), 10, TimeUnit.SECONDS);
+        IOUtils.close(nodeEnvironment, indicesQueryCache, clusterService);
+        ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS);
+    }
+
+    private IndexService newIndexService(IndexModule module) throws IOException {
+        return module.newIndexService(nodeEnvironment, xContentRegistry(), deleter, circuitBreakerService, bigArrays, threadPool,
+                scriptService, clusterService, null, indicesQueryCache, mapperRegistry,
+                new IndicesFieldDataCache(settings, listener));
     }
 
     public void testWrapperIsBound() throws IOException {
         IndexModule module = new IndexModule(indexSettings, null,
-                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
+                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap()));
         module.setSearcherWrapper((s) -> new Wrapper());
         module.engineFactory.set(new MockEngineFactory(AssertingDirectoryReader.class));
-        IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, indicesQueryCache, mapperRegistry, new IndicesFieldDataCache(settings, listener));
+        IndexService indexService = newIndexService(module);
         assertTrue(indexService.getSearcherWrapper() instanceof Wrapper);
         assertSame(indexService.getEngineFactory(), module.engineFactory.get());
         indexService.close("simon says", false);
@@ -168,7 +171,7 @@ public void testRegisterIndexStore() throws IOException {
             .build();
         IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(index, settings);
         IndexModule module = new IndexModule(indexSettings, null,
-                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
+                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap()));
         module.addIndexStore("foo_store", FooStore::new);
         try {
             module.addIndexStore("foo_store", FooStore::new);
@@ -176,7 +179,7 @@ public void testRegisterIndexStore() throws IOException {
         } catch (IllegalArgumentException ex) {
             // fine
         }
-        IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, indicesQueryCache, mapperRegistry, new IndicesFieldDataCache(settings, listener));
+        IndexService indexService = newIndexService(module);
         assertTrue(indexService.getIndexStore() instanceof FooStore);
 
         indexService.close("simon says", false);
@@ -186,20 +189,19 @@ public void testOtherServiceBound() throws IOException {
         final AtomicBoolean atomicBoolean = new AtomicBoolean(false);
         final IndexEventListener eventListener = new IndexEventListener() {
             @Override
-            public void beforeIndexDeleted(IndexService indexService) {
+            public void beforeIndexRemoved(IndexService indexService, IndexRemovalReason reason) {
                 atomicBoolean.set(true);
             }
         };
         IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(index, settings);
         IndexModule module = new IndexModule(indexSettings, null,
-                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
+                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap()));
         module.addIndexEventListener(eventListener);
-        IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, indicesQueryCache, mapperRegistry,
-            new IndicesFieldDataCache(settings, this.listener));
+        IndexService indexService = newIndexService(module);
         IndexSettings x = indexService.getIndexSettings();
         assertEquals(x.getSettings().getAsMap(), indexSettings.getSettings().getAsMap());
         assertEquals(x.getIndex(), index);
-        indexService.getIndexEventListener().beforeIndexDeleted(null);
+        indexService.getIndexEventListener().beforeIndexRemoved(null, null);
         assertTrue(atomicBoolean.get());
         indexService.close("simon says", false);
     }
@@ -208,7 +210,7 @@ public void beforeIndexDeleted(IndexService indexService) {
     public void testListener() throws IOException {
         Setting<Boolean> booleanSetting = Setting.boolSetting("index.foo.bar", false, Property.Dynamic, Property.IndexScope);
         IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings(index, settings, booleanSetting), null,
-                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
+                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap()));
         Setting<Boolean> booleanSetting2 = Setting.boolSetting("index.foo.bar.baz", false, Property.Dynamic, Property.IndexScope);
         AtomicBoolean atomicBoolean = new AtomicBoolean(false);
         module.addSettingsUpdateConsumer(booleanSetting, atomicBoolean::set);
@@ -220,8 +222,7 @@ public void testListener() throws IOException {
 
         }
 
-        IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, indicesQueryCache, mapperRegistry,
-            new IndicesFieldDataCache(settings, listener));
+        IndexService indexService = newIndexService(module);
         assertSame(booleanSetting, indexService.getIndexSettings().getScopedSettings().get(booleanSetting.getKey()));
 
         indexService.close("simon says", false);
@@ -229,11 +230,11 @@ public void testListener() throws IOException {
 
     public void testAddIndexOperationListener() throws IOException {
         IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings(index, settings), null,
-                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
+                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap()));
         AtomicBoolean executed = new AtomicBoolean(false);
         IndexingOperationListener listener = new IndexingOperationListener() {
             @Override
-            public Engine.Index preIndex(Engine.Index operation) {
+            public Engine.Index preIndex(ShardId shardId, Engine.Index operation) {
                 executed.set(true);
                 return operation;
             }
@@ -244,15 +245,16 @@ public Engine.Index preIndex(Engine.Index operation) {
         expectThrows(IllegalArgumentException.class, () -> module.addIndexOperationListener(null));
 
 
-        IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, indicesQueryCache, mapperRegistry,
-            new IndicesFieldDataCache(settings, this.listener));
+        IndexService indexService = newIndexService(module);
         assertEquals(2, indexService.getIndexOperationListeners().size());
         assertEquals(IndexingSlowLog.class, indexService.getIndexOperationListeners().get(0).getClass());
         assertSame(listener, indexService.getIndexOperationListeners().get(1));
 
-        Engine.Index index = new Engine.Index(new Term("_uid", "1"), null);
+        ParsedDocument doc = InternalEngineTests.createParsedDoc("1", "test", null);
+        Engine.Index index = new Engine.Index(new Term("_uid",  Uid.createUidAsBytes(doc.type(), doc.id())), doc);
+        ShardId shardId = new ShardId(new Index("foo", "bar"), 0);
         for (IndexingOperationListener l : indexService.getIndexOperationListeners()) {
-            l.preIndex(index);
+            l.preIndex(shardId, index);
         }
         assertTrue(executed.get());
         indexService.close("simon says", false);
@@ -260,7 +262,7 @@ public Engine.Index preIndex(Engine.Index operation) {
 
     public void testAddSearchOperationListener() throws IOException {
         IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings(index, settings), null,
-                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
+                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap()));
         AtomicBoolean executed = new AtomicBoolean(false);
         SearchOperationListener listener = new SearchOperationListener() {
 
@@ -275,8 +277,7 @@ public void onNewContext(SearchContext context) {
         expectThrows(IllegalArgumentException.class, () -> module.addSearchOperationListener(null));
 
 
-        IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, indicesQueryCache, mapperRegistry,
-            new IndicesFieldDataCache(settings, this.listener));
+        IndexService indexService = newIndexService(module);
         assertEquals(2, indexService.getSearchOperationListener().size());
         assertEquals(SearchSlowLog.class, indexService.getSearchOperationListener().get(0).getClass());
         assertSame(listener, indexService.getSearchOperationListener().get(1));
@@ -296,7 +297,7 @@ public void testAddSimilarity() throws IOException {
                 .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
                 .build();
         IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings), null,
-                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
+                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap()));
         module.addSimilarity("test_similarity", (string, settings) -> new SimilarityProvider() {
             @Override
             public String name() {
@@ -309,8 +310,7 @@ public Similarity get() {
             }
         });
 
-        IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, indicesQueryCache, mapperRegistry,
-            new IndicesFieldDataCache(settings, listener));
+        IndexService indexService = newIndexService(module);
         SimilarityService similarityService = indexService.similarityService();
         assertNotNull(similarityService.getSimilarity("my_similarity"));
         assertTrue(similarityService.getSimilarity("my_similarity").get() instanceof TestSimilarity);
@@ -321,7 +321,7 @@ public Similarity get() {
 
     public void testFrozen() {
         IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings(index, settings), null,
-                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
+                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap()));
         module.freeze();
         String msg = "Can't modify IndexModule once the index service has been created";
         assertEquals(msg, expectThrows(IllegalStateException.class, () -> module.addSearchOperationListener(null)).getMessage());
@@ -340,13 +340,9 @@ public void testSetupUnknownSimilarity() throws IOException {
                 .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
                 .build();
         IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings), null,
-                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
-        try {
-            module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, indicesQueryCache, mapperRegistry,
-                new IndicesFieldDataCache(settings, listener));
-        } catch (IllegalArgumentException ex) {
-            assertEquals("Unknown Similarity type [test_similarity] for [my_similarity]", ex.getMessage());
-        }
+                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap()));
+        Exception ex = expectThrows(IllegalArgumentException.class, () -> newIndexService(module));
+        assertEquals("Unknown Similarity type [test_similarity] for [my_similarity]", ex.getMessage());
     }
 
     public void testSetupWithoutType() throws IOException {
@@ -356,13 +352,9 @@ public void testSetupWithoutType() throws IOException {
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
                 .build();
         IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings), null,
-                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
-        try {
-            module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, indicesQueryCache, mapperRegistry,
-                new IndicesFieldDataCache(settings, listener));
-        } catch (IllegalArgumentException ex) {
-            assertEquals("Similarity [my_similarity] must have an associated type", ex.getMessage());
-        }
+                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap()));
+        Exception ex = expectThrows(IllegalArgumentException.class, () -> newIndexService(module));
+        assertEquals("Similarity [my_similarity] must have an associated type", ex.getMessage());
     }
 
     public void testForceCustomQueryCache() throws IOException {
@@ -370,11 +362,10 @@ public void testForceCustomQueryCache() throws IOException {
                 .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
         IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings), null,
-                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
+                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap()));
         module.forceQueryCacheProvider((a, b) -> new CustomQueryCache());
         expectThrows(AlreadySetException.class, () -> module.forceQueryCacheProvider((a, b) -> new CustomQueryCache()));
-        IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, indicesQueryCache, mapperRegistry,
-            new IndicesFieldDataCache(settings, listener));
+        IndexService indexService = newIndexService(module);
         assertTrue(indexService.cache().query() instanceof CustomQueryCache);
         indexService.close("simon says", false);
     }
@@ -384,9 +375,8 @@ public void testDefaultQueryCacheImplIsSelected() throws IOException {
                 .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
         IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings), null,
-                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
-        IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, indicesQueryCache, mapperRegistry,
-            new IndicesFieldDataCache(settings, listener));
+                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap()));
+        IndexService indexService = newIndexService(module);
         assertTrue(indexService.cache().query() instanceof IndexQueryCache);
         indexService.close("simon says", false);
     }
@@ -397,10 +387,9 @@ public void testDisableQueryCacheHasPrecedenceOverForceQueryCache() throws IOExc
             .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
             .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
         IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings), null,
-                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
+                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap()));
         module.forceQueryCacheProvider((a, b) -> new CustomQueryCache());
-        IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, indicesQueryCache, mapperRegistry,
-            new IndicesFieldDataCache(settings, listener));
+        IndexService indexService = newIndexService(module);
         assertTrue(indexService.cache().query() instanceof DisabledQueryCache);
         indexService.close("simon says", false);
     }
@@ -431,7 +420,7 @@ private static class TestSimilarity extends Similarity {
         private final String key;
 
 
-        public TestSimilarity(String key) {
+        TestSimilarity(String key) {
             if (key == null) {
                 throw new AssertionError("key is null");
             }
diff --git a/core/src/test/java/org/elasticsearch/index/IndexRequestBuilderIT.java b/core/src/test/java/org/elasticsearch/index/IndexRequestBuilderIT.java
index 84a9fe2970bd0..7fa769b2613eb 100644
--- a/core/src/test/java/org/elasticsearch/index/IndexRequestBuilderIT.java
+++ b/core/src/test/java/org/elasticsearch/index/IndexRequestBuilderIT.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
@@ -40,10 +41,11 @@ public void testSetSource() throws InterruptedException, ExecutionException {
         map.put("test_field", "foobar");
         IndexRequestBuilder[] builders = new IndexRequestBuilder[] {
                 client().prepareIndex("test", "test").setSource((Object)"test_field", (Object)"foobar"),
-                client().prepareIndex("test", "test").setSource("{\"test_field\" : \"foobar\"}"),
-                client().prepareIndex("test", "test").setSource(new BytesArray("{\"test_field\" : \"foobar\"}")),
-                client().prepareIndex("test", "test").setSource(new BytesArray("{\"test_field\" : \"foobar\"}")),
-                client().prepareIndex("test", "test").setSource(BytesReference.toBytes(new BytesArray("{\"test_field\" : \"foobar\"}"))),
+                client().prepareIndex("test", "test").setSource("{\"test_field\" : \"foobar\"}", XContentType.JSON),
+                client().prepareIndex("test", "test").setSource(new BytesArray("{\"test_field\" : \"foobar\"}"), XContentType.JSON),
+                client().prepareIndex("test", "test").setSource(new BytesArray("{\"test_field\" : \"foobar\"}"), XContentType.JSON),
+                client().prepareIndex("test", "test")
+                    .setSource(BytesReference.toBytes(new BytesArray("{\"test_field\" : \"foobar\"}")), XContentType.JSON),
                 client().prepareIndex("test", "test").setSource(map)
         };
         indexRandom(true, builders);
diff --git a/core/src/test/java/org/elasticsearch/index/IndexServiceTests.java b/core/src/test/java/org/elasticsearch/index/IndexServiceTests.java
index 22324e1ff2b5c..89b1875f7d3df 100644
--- a/core/src/test/java/org/elasticsearch/index/IndexServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/index/IndexServiceTests.java
@@ -21,20 +21,19 @@
 
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.TopDocs;
-import org.elasticsearch.cluster.metadata.AliasMetaData;
+import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.translog.Translog;
-import org.elasticsearch.indices.InvalidAliasNameException;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
 
@@ -43,24 +42,34 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 
-import static org.elasticsearch.index.query.QueryBuilders.termQuery;
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.nullValue;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
 
 /** Unit test(s) for IndexService */
 public class IndexServiceTests extends ESSingleNodeTestCase {
     public void testDetermineShadowEngineShouldBeUsed() {
-        Settings regularSettings = Settings.builder()
-                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 2)
-                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
-                .build();
-
-        Settings shadowSettings = Settings.builder()
-                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 2)
-                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
-                .put(IndexMetaData.SETTING_SHADOW_REPLICAS, true)
-                .build();
+        IndexSettings regularSettings = new IndexSettings(
+            IndexMetaData
+                .builder("regular")
+                .settings(Settings.builder()
+                    .put(SETTING_NUMBER_OF_SHARDS, 2)
+                    .put(SETTING_NUMBER_OF_REPLICAS, 1)
+                    .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                    .build())
+                .build(),
+            Settings.EMPTY);
+
+        IndexSettings shadowSettings = new IndexSettings(
+            IndexMetaData
+                .builder("shadow")
+                .settings(Settings.builder()
+                    .put(SETTING_NUMBER_OF_SHARDS, 2)
+                    .put(SETTING_NUMBER_OF_REPLICAS, 1)
+                    .put(IndexMetaData.SETTING_SHADOW_REPLICAS, true)
+                    .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                    .build())
+                .build(),
+            Settings.EMPTY);
 
         assertFalse("no shadow replicas for normal settings", IndexService.useShadowEngine(true, regularSettings));
         assertFalse("no shadow replicas for normal settings", IndexService.useShadowEngine(false, regularSettings));
@@ -75,82 +84,6 @@ public static CompressedXContent filter(QueryBuilder filterBuilder) throws IOExc
         return new CompressedXContent(builder.string());
     }
 
-    public void testFilteringAliases() throws Exception {
-        IndexService indexService = createIndex("test", Settings.EMPTY);
-        add(indexService, "cats", filter(termQuery("animal", "cat")));
-        add(indexService, "dogs", filter(termQuery("animal", "dog")));
-        add(indexService, "all", null);
-
-        assertThat(indexService.getMetaData().getAliases().containsKey("cats"), equalTo(true));
-        assertThat(indexService.getMetaData().getAliases().containsKey("dogs"), equalTo(true));
-        assertThat(indexService.getMetaData().getAliases().containsKey("turtles"), equalTo(false));
-
-        assertThat(indexService.aliasFilter(indexService.newQueryShardContext(), "cats").toString(), equalTo("animal:cat"));
-        assertThat(indexService.aliasFilter(indexService.newQueryShardContext(), "cats", "dogs").toString(), equalTo("animal:cat animal:dog"));
-
-        // Non-filtering alias should turn off all filters because filters are ORed
-        assertThat(indexService.aliasFilter(indexService.newQueryShardContext(), "all"), nullValue());
-        assertThat(indexService.aliasFilter(indexService.newQueryShardContext(), "cats", "all"), nullValue());
-        assertThat(indexService.aliasFilter(indexService.newQueryShardContext(), "all", "cats"), nullValue());
-
-        add(indexService, "cats", filter(termQuery("animal", "feline")));
-        add(indexService, "dogs", filter(termQuery("animal", "canine")));
-        assertThat(indexService.aliasFilter(indexService.newQueryShardContext(), "dogs", "cats").toString(), equalTo("animal:canine animal:feline"));
-    }
-
-    public void testAliasFilters() throws Exception {
-        IndexService indexService = createIndex("test", Settings.EMPTY);
-
-        add(indexService, "cats", filter(termQuery("animal", "cat")));
-        add(indexService, "dogs", filter(termQuery("animal", "dog")));
-
-        assertThat(indexService.aliasFilter(indexService.newQueryShardContext()), nullValue());
-        assertThat(indexService.aliasFilter(indexService.newQueryShardContext(), "dogs").toString(), equalTo("animal:dog"));
-        assertThat(indexService.aliasFilter(indexService.newQueryShardContext(), "dogs", "cats").toString(), equalTo("animal:dog animal:cat"));
-
-        add(indexService, "cats", filter(termQuery("animal", "feline")));
-        add(indexService, "dogs", filter(termQuery("animal", "canine")));
-
-        assertThat(indexService.aliasFilter(indexService.newQueryShardContext(), "dogs", "cats").toString(), equalTo("animal:canine animal:feline"));
-    }
-
-    public void testRemovedAliasFilter() throws Exception {
-        IndexService indexService = createIndex("test", Settings.EMPTY);
-
-        add(indexService, "cats", filter(termQuery("animal", "cat")));
-        remove(indexService, "cats");
-        try {
-            indexService.aliasFilter(indexService.newQueryShardContext(), "cats");
-            fail("Expected InvalidAliasNameException");
-        } catch (InvalidAliasNameException e) {
-            assertThat(e.getMessage(), containsString("Invalid alias name [cats]"));
-        }
-    }
-
-    public void testUnknownAliasFilter() throws Exception {
-        IndexService indexService = createIndex("test", Settings.EMPTY);
-
-        add(indexService, "cats", filter(termQuery("animal", "cat")));
-        add(indexService, "dogs", filter(termQuery("animal", "dog")));
-
-        try {
-            indexService.aliasFilter(indexService.newQueryShardContext(), "unknown");
-            fail();
-        } catch (InvalidAliasNameException e) {
-            // all is well
-        }
-    }
-
-    private void remove(IndexService service, String alias) {
-        IndexMetaData build = IndexMetaData.builder(service.getMetaData()).removeAlias(alias).build();
-        service.updateMetaData(build);
-    }
-
-    private void add(IndexService service, String alias, @Nullable CompressedXContent filter) {
-        IndexMetaData build = IndexMetaData.builder(service.getMetaData()).putAlias(AliasMetaData.builder(alias).filter(filter).build()).build();
-        service.updateMetaData(build);
-    }
-
     public void testBaseAsyncTask() throws InterruptedException, IOException {
         IndexService indexService = createIndex("test", Settings.EMPTY);
         AtomicReference<CountDownLatch> latch = new AtomicReference<>(new CountDownLatch(1));
@@ -286,7 +219,7 @@ public void testRefreshActuallyWorks() throws Exception {
         // now disable
         IndexMetaData metaData = IndexMetaData.builder(indexService.getMetaData()).settings(Settings.builder().put(indexService.getMetaData().getSettings()).put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), -1)).build();
         indexService.updateMetaData(metaData);
-        client().prepareIndex("test", "test", "1").setSource("{\"foo\": \"bar\"}").get();
+        client().prepareIndex("test", "test", "1").setSource("{\"foo\": \"bar\"}", XContentType.JSON).get();
         IndexShard shard = indexService.getShard(0);
         try (Engine.Searcher searcher = shard.acquireSearcher("test")) {
             TopDocs search = searcher.searcher().search(new MatchAllDocsQuery(), 10);
@@ -313,7 +246,7 @@ public void testAsyncFsyncActuallyWorks() throws Exception {
         IndexService indexService = createIndex("test", settings);
         ensureGreen("test");
         assertTrue(indexService.getRefreshTask().mustReschedule());
-        client().prepareIndex("test", "test", "1").setSource("{\"foo\": \"bar\"}").get();
+        client().prepareIndex("test", "test", "1").setSource("{\"foo\": \"bar\"}", XContentType.JSON).get();
         IndexShard shard = indexService.getShard(0);
         assertBusy(() -> {
             assertFalse(shard.getTranslog().syncNeeded());
@@ -332,7 +265,7 @@ public void testRescheduleAsyncFsync() throws Exception {
         indexService.updateMetaData(metaData);
         assertNotNull(indexService.getFsyncTask());
         assertTrue(indexService.getRefreshTask().mustReschedule());
-        client().prepareIndex("test", "test", "1").setSource("{\"foo\": \"bar\"}").get();
+        client().prepareIndex("test", "test", "1").setSource("{\"foo\": \"bar\"}", XContentType.JSON).get();
         IndexShard shard = indexService.getShard(0);
         assertBusy(() -> {
             assertFalse(shard.getTranslog().syncNeeded());
diff --git a/core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java b/core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java
index 3909354c98952..49e7680656971 100644
--- a/core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java
+++ b/core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java
@@ -16,11 +16,11 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+
 package org.elasticsearch.index;
 
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.cluster.metadata.MetaDataIndexUpgradeService;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Setting;
@@ -29,18 +29,20 @@
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.translog.Translog;
-import org.elasticsearch.indices.mapper.MapperRegistry;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.VersionUtils;
 
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Function;
 
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.core.StringContains.containsString;
+import static org.hamcrest.object.HasToString.hasToString;
+
 public class IndexSettingsTests extends ESTestCase {
 
     public void testRunListener() {
@@ -289,6 +291,29 @@ public void testMaxResultWindow() {
         settings = new IndexSettings(metaData, Settings.EMPTY);
         assertEquals(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY).intValue(), settings.getMaxResultWindow());
     }
+    
+    public void testMaxAdjacencyMatrixFiltersSetting() {
+        IndexMetaData metaData = newIndexMeta("index", Settings.builder()
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+            .put(IndexSettings.MAX_ADJACENCY_MATRIX_FILTERS_SETTING.getKey(), 15)
+            .build());
+        IndexSettings settings = new IndexSettings(metaData, Settings.EMPTY);
+        assertEquals(15, settings.getMaxAdjacencyMatrixFilters());
+        settings.updateIndexMetaData(newIndexMeta("index",
+            Settings.builder().put(IndexSettings.MAX_ADJACENCY_MATRIX_FILTERS_SETTING.getKey(),
+            42).build()));
+        assertEquals(42, settings.getMaxAdjacencyMatrixFilters());
+        settings.updateIndexMetaData(newIndexMeta("index", Settings.EMPTY));
+        assertEquals(IndexSettings.MAX_ADJACENCY_MATRIX_FILTERS_SETTING.get(Settings.EMPTY).intValue(), 
+                settings.getMaxAdjacencyMatrixFilters());
+
+        metaData = newIndexMeta("index", Settings.builder()
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+            .build());
+        settings = new IndexSettings(metaData, Settings.EMPTY);
+        assertEquals(IndexSettings.MAX_ADJACENCY_MATRIX_FILTERS_SETTING.get(Settings.EMPTY).intValue(), 
+                settings.getMaxAdjacencyMatrixFilters());
+    }    
 
     public void testGCDeletesSetting() {
         TimeValue gcDeleteSetting = new TimeValue(Math.abs(randomInt()), TimeUnit.MILLISECONDS);
@@ -348,26 +373,48 @@ public void testTranslogFlushSizeThreshold() {
         assertEquals(actualNewTranslogFlushThresholdSize, settings.getFlushThresholdSize());
     }
 
-
     public void testArchiveBrokenIndexSettings() {
-        Settings settings = IndexScopedSettings.DEFAULT_SCOPED_SETTINGS.archiveUnknownOrBrokenSettings(Settings.EMPTY);
+        Settings settings =
+            IndexScopedSettings.DEFAULT_SCOPED_SETTINGS.archiveUnknownOrInvalidSettings(
+                Settings.EMPTY,
+                e -> { assert false : "should not have been invoked, no unknown settings"; },
+                (e, ex) -> { assert false : "should not have been invoked, no invalid settings"; });
         assertSame(settings, Settings.EMPTY);
-        settings = IndexScopedSettings.DEFAULT_SCOPED_SETTINGS.archiveUnknownOrBrokenSettings(Settings.builder()
-            .put("index.refresh_interval", "-200").build());
+        settings =
+            IndexScopedSettings.DEFAULT_SCOPED_SETTINGS.archiveUnknownOrInvalidSettings(
+                Settings.builder().put("index.refresh_interval", "-200").build(),
+                e -> { assert false : "should not have been invoked, no invalid settings"; },
+                (e, ex) -> {
+                    assertThat(e.getKey(), equalTo("index.refresh_interval"));
+                    assertThat(e.getValue(), equalTo("-200"));
+                    assertThat(ex, hasToString(containsString("failed to parse setting [index.refresh_interval] with value [-200]")));
+                });
         assertEquals("-200", settings.get("archived.index.refresh_interval"));
         assertNull(settings.get("index.refresh_interval"));
 
         Settings prevSettings = settings; // no double archive
-        settings = IndexScopedSettings.DEFAULT_SCOPED_SETTINGS.archiveUnknownOrBrokenSettings(prevSettings);
+        settings =
+            IndexScopedSettings.DEFAULT_SCOPED_SETTINGS.archiveUnknownOrInvalidSettings(
+                prevSettings,
+                e -> { assert false : "should not have been invoked, no unknown settings"; },
+                (e, ex) -> { assert false : "should not have been invoked, no invalid settings"; });
         assertSame(prevSettings, settings);
 
-        settings = IndexScopedSettings.DEFAULT_SCOPED_SETTINGS.archiveUnknownOrBrokenSettings(Settings.builder()
-            .put("index.version.created", Version.CURRENT.id) // private setting
-            .put("index.unknown", "foo")
-            .put("index.refresh_interval", "2s").build());
+        settings =
+            IndexScopedSettings.DEFAULT_SCOPED_SETTINGS.archiveUnknownOrInvalidSettings(
+                Settings.builder()
+                    .put("index.version.created", Version.CURRENT.id) // private setting
+                    .put("index.unknown", "foo")
+                    .put("index.refresh_interval", "2s").build(),
+                e -> {
+                    assertThat(e.getKey(), equalTo("index.unknown"));
+                    assertThat(e.getValue(), equalTo("foo"));
+                },
+                (e, ex) -> { assert false : "should not have been invoked, no invalid settings"; });
 
         assertEquals("foo", settings.get("archived.index.unknown"));
         assertEquals(Integer.toString(Version.CURRENT.id), settings.get("index.version.created"));
         assertEquals("2s", settings.get("index.refresh_interval"));
     }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/index/IndexTests.java b/core/src/test/java/org/elasticsearch/index/IndexTests.java
index ffa12a1943df2..c39a43e849060 100644
--- a/core/src/test/java/org/elasticsearch/index/IndexTests.java
+++ b/core/src/test/java/org/elasticsearch/index/IndexTests.java
@@ -24,7 +24,6 @@
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.test.ESTestCase;
 
@@ -51,12 +50,12 @@ public void testToString() {
     }
 
     public void testXContent() throws IOException {
-        final String name = randomAsciiOfLengthBetween(4, 15);
+        final String name = randomAlphaOfLengthBetween(4, 15);
         final String uuid = UUIDs.randomBase64UUID();
         final Index original = new Index(name, uuid);
         final XContentBuilder builder = JsonXContent.contentBuilder();
         original.toXContent(builder, ToXContent.EMPTY_PARAMS);
-        XContentParser parser = XContentType.JSON.xContent().createParser(builder.bytes());
+        XContentParser parser = createParser(JsonXContent.jsonXContent, builder.bytes());
         parser.nextToken(); // the beginning of the parser
         assertThat(Index.fromXContent(parser), equalTo(original));
     }
diff --git a/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java b/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java
index 989c60fc91620..7c3d4c1dce620 100644
--- a/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java
+++ b/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java
@@ -33,7 +33,6 @@
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.common.Priority;
@@ -57,7 +56,6 @@
 import org.elasticsearch.test.InternalTestCluster;
 import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.test.transport.MockTransportService;
-import org.elasticsearch.transport.TransportException;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportService;
@@ -110,7 +108,7 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
 
     public void testCannotCreateWithBadPath() throws Exception {
         Settings nodeSettings = nodeSettings("/badpath");
-        internalCluster().startNodesAsync(1, nodeSettings).get();
+        internalCluster().startNodes(1, nodeSettings);
         Settings idxSettings = Settings.builder()
                 .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
                 .put(IndexMetaData.SETTING_DATA_PATH, "/etc/foo")
@@ -132,7 +130,7 @@ public void testRestoreToShadow() throws ExecutionException, InterruptedExceptio
         final Path dataPath = createTempDir();
         Settings nodeSettings = nodeSettings(dataPath);
 
-        internalCluster().startNodesAsync(3, nodeSettings).get();
+        internalCluster().startNodes(3, nodeSettings);
         Settings idxSettings = Settings.builder()
                 .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
                 .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).build();
@@ -142,7 +140,7 @@ public void testRestoreToShadow() throws ExecutionException, InterruptedExceptio
         for (int i = 0; i < numDocs; i++) {
             client().prepareIndex("foo", "doc", ""+i).setSource("foo", "bar").get();
         }
-        assertNoFailures(client().admin().indices().prepareFlush().setForce(true).setWaitIfOngoing(true).execute().actionGet());
+        assertNoFailures(client().admin().indices().prepareFlush().setForce(true).execute().actionGet());
 
         assertAcked(client().admin().cluster().preparePutRepository("test-repo")
                 .setType("fs").setSettings(Settings.builder()
@@ -184,12 +182,12 @@ public void testRestoreToShadow() throws ExecutionException, InterruptedExceptio
 
     }
 
-    @TestLogging("gateway:TRACE")
+    @TestLogging("org.elasticsearch.gateway:TRACE")
     public void testIndexWithFewDocuments() throws Exception {
         final Path dataPath = createTempDir();
         Settings nodeSettings = nodeSettings(dataPath);
 
-        internalCluster().startNodesAsync(3, nodeSettings).get();
+        internalCluster().startNodes(3, nodeSettings);
         final String IDX = "test";
 
         Settings idxSettings = Settings.builder()
@@ -359,6 +357,9 @@ public void testPrimaryRelocation() throws Exception {
         client().admin().indices().prepareUpdateSettings(IDX).setSettings(build).execute().actionGet();
 
         ensureGreen(IDX);
+        // check if primary has relocated to node3
+        assertEquals(internalCluster().clusterService(node3).localNode().getId(),
+            client().admin().cluster().prepareState().get().getState().routingTable().index(IDX).shard(0).primaryShard().currentNodeId());
         logger.info("--> performing query");
         SearchResponse resp = client().prepareSearch(IDX).setQuery(matchAllQuery()).get();
         assertHitCount(resp, 2);
@@ -492,14 +493,13 @@ public void testPrimaryRelocationWhereRecoveryFails() throws Exception {
                 new MockTransportService.DelegateTransport(mockTransportService.original()) {
 
                     @Override
-                    public void sendRequest(DiscoveryNode node, long requestId, String action,
-                                            TransportRequest request, TransportRequestOptions options)
-                            throws IOException, TransportException {
+                    protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request,
+                                               TransportRequestOptions options) throws IOException {
                         if (keepFailing.get() && action.equals(PeerRecoveryTargetService.Actions.TRANSLOG_OPS)) {
                             logger.info("--> failing translog ops");
                             throw new ElasticsearchException("failing on purpose");
                         }
-                        super.sendRequest(node, requestId, action, request, options);
+                        super.sendRequest(connection, requestId, action, request, options);
                     }
                 });
 
@@ -551,7 +551,7 @@ public void testIndexWithShadowReplicasCleansUp() throws Exception {
 
         final int nodeCount = randomIntBetween(2, 5);
         logger.info("--> starting {} nodes", nodeCount);
-        final List<String> nodes = internalCluster().startNodesAsync(nodeCount, nodeSettings).get();
+        final List<String> nodes = internalCluster().startNodes(nodeCount, nodeSettings);
         final String IDX = "test";
         final Tuple<Integer, Integer> numPrimariesAndReplicas = randomPrimariesAndReplicas(nodeCount);
         final int numPrimaries = numPrimariesAndReplicas.v1();
@@ -599,12 +599,12 @@ public void testIndexWithShadowReplicasCleansUp() throws Exception {
     // deleting the index and hence, deleting all the shard data for the index, the test
     // failure still showed some Lucene files in the data directory for that index. Not sure
     // why that is, so turning on more logging here.
-    @TestLogging("indices:TRACE,env:TRACE")
+    @TestLogging("org.elasticsearch.indices:TRACE,org.elasticsearch.env:TRACE")
     public void testShadowReplicaNaturalRelocation() throws Exception {
         Path dataPath = createTempDir();
         Settings nodeSettings = nodeSettings(dataPath);
 
-        final List<String> nodes = internalCluster().startNodesAsync(2, nodeSettings).get();
+        final List<String> nodes = internalCluster().startNodes(2, nodeSettings);
         String IDX = "test";
 
         Settings idxSettings = Settings.builder()
@@ -631,16 +631,13 @@ public void testShadowReplicaNaturalRelocation() throws Exception {
         final String node3 = internalCluster().startNode(nodeSettings);
         nodes.add(node3);
 
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                client().admin().cluster().prepareHealth().setWaitForNodes("3").get();
-                ClusterStateResponse resp = client().admin().cluster().prepareState().get();
-                RoutingNodes nodes = resp.getState().getRoutingNodes();
-                for (RoutingNode node : nodes) {
-                    logger.info("--> node has {} shards (needs at least 2)", node.numberOfOwningShards());
-                    assertThat("at least 2 shards on node", node.numberOfOwningShards(), greaterThanOrEqualTo(2));
-                }
+        assertBusy(() -> {
+            client().admin().cluster().prepareHealth().setWaitForNodes("3").get();
+            ClusterStateResponse resp = client().admin().cluster().prepareState().get();
+            RoutingNodes nodes1 = resp.getState().getRoutingNodes();
+            for (RoutingNode node : nodes1) {
+                logger.info("--> node has {} shards (needs at least 2)", node.numberOfOwningShards());
+                assertThat("at least 2 shards on node", node.numberOfOwningShards(), greaterThanOrEqualTo(2));
             }
         });
         ensureYellow(IDX);
@@ -660,7 +657,7 @@ public void testShadowReplicasUsingFieldData() throws Exception {
         Path dataPath = createTempDir();
         Settings nodeSettings = nodeSettings(dataPath);
 
-        internalCluster().startNodesAsync(3, nodeSettings).get();
+        internalCluster().startNodes(3, nodeSettings);
         String IDX = "test";
 
         Settings idxSettings = Settings.builder()
@@ -692,16 +689,13 @@ public void testShadowReplicasUsingFieldData() throws Exception {
 
     /** wait until none of the nodes have shards allocated on them */
     private void assertNoShardsOn(final List<String> nodeList) throws Exception {
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                ClusterStateResponse resp = client().admin().cluster().prepareState().get();
-                RoutingNodes nodes = resp.getState().getRoutingNodes();
-                for (RoutingNode node : nodes) {
-                    logger.info("--> node {} has {} shards", node.node().getName(), node.numberOfOwningShards());
-                    if (nodeList.contains(node.node().getName())) {
-                        assertThat("no shards on node", node.numberOfOwningShards(), equalTo(0));
-                    }
+        assertBusy(() -> {
+            ClusterStateResponse resp = client().admin().cluster().prepareState().get();
+            RoutingNodes nodes = resp.getState().getRoutingNodes();
+            for (RoutingNode node : nodes) {
+                logger.info("--> node {} has {} shards", node.node().getName(), node.numberOfOwningShards());
+                if (nodeList.contains(node.node().getName())) {
+                    assertThat("no shards on node", node.numberOfOwningShards(), equalTo(0));
                 }
             }
         }, 1, TimeUnit.MINUTES);
@@ -709,16 +703,13 @@ public void run() {
 
     /** wait until the node has the specified number of shards allocated on it */
     private void assertShardCountOn(final String nodeName, final int shardCount) throws Exception {
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                ClusterStateResponse resp = client().admin().cluster().prepareState().get();
-                RoutingNodes nodes = resp.getState().getRoutingNodes();
-                for (RoutingNode node : nodes) {
-                    logger.info("--> node {} has {} shards", node.node().getName(), node.numberOfOwningShards());
-                    if (nodeName.equals(node.node().getName())) {
-                        assertThat(node.numberOfOwningShards(), equalTo(shardCount));
-                    }
+        assertBusy(() -> {
+            ClusterStateResponse resp = client().admin().cluster().prepareState().get();
+            RoutingNodes nodes = resp.getState().getRoutingNodes();
+            for (RoutingNode node : nodes) {
+                logger.info("--> node {} has {} shards", node.node().getName(), node.numberOfOwningShards());
+                if (nodeName.equals(node.node().getName())) {
+                    assertThat(node.numberOfOwningShards(), equalTo(shardCount));
                 }
             }
         }, 1, TimeUnit.MINUTES);
@@ -730,10 +721,9 @@ public void testIndexOnSharedFSRecoversToAnyNode() throws Exception {
         Settings fooSettings = Settings.builder().put(nodeSettings).put("node.attr.affinity", "foo").build();
         Settings barSettings = Settings.builder().put(nodeSettings).put("node.attr.affinity", "bar").build();
 
-        final InternalTestCluster.Async<List<String>> fooNodes = internalCluster().startNodesAsync(2, fooSettings);
-        final InternalTestCluster.Async<List<String>> barNodes = internalCluster().startNodesAsync(2, barSettings);
-        fooNodes.get();
-        barNodes.get();
+        List<String> allNodes = internalCluster().startNodes(fooSettings, fooSettings, barSettings, barSettings);
+        List<String> fooNodes = allNodes.subList(0, 2);
+        List<String> barNodes = allNodes.subList(2, 4);
         String IDX = "test";
 
         Settings includeFoo = Settings.builder()
@@ -767,27 +757,27 @@ public void testIndexOnSharedFSRecoversToAnyNode() throws Exception {
         client().admin().indices().prepareUpdateSettings(IDX).setSettings(includeBar).get();
 
         // wait for the shards to move from "foo" nodes to "bar" nodes
-        assertNoShardsOn(fooNodes.get());
+        assertNoShardsOn(fooNodes);
 
         // put shards back on "foo"
         client().admin().indices().prepareUpdateSettings(IDX).setSettings(includeFoo).get();
 
         // wait for the shards to move from "bar" nodes to "foo" nodes
-        assertNoShardsOn(barNodes.get());
+        assertNoShardsOn(barNodes);
 
         // Stop a foo node
         logger.info("--> stopping first 'foo' node");
-        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(fooNodes.get().get(0)));
+        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(fooNodes.get(0)));
 
         // Ensure that the other foo node has all the shards now
-        assertShardCountOn(fooNodes.get().get(1), 5);
+        assertShardCountOn(fooNodes.get(1), 5);
 
         // Assert no shards on the "bar" nodes
-        assertNoShardsOn(barNodes.get());
+        assertNoShardsOn(barNodes);
 
         // Stop the second "foo" node
         logger.info("--> stopping second 'foo' node");
-        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(fooNodes.get().get(1)));
+        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(fooNodes.get(1)));
 
         // The index should still be able to be allocated (on the "bar" nodes),
         // all the "foo" nodes are gone
@@ -798,7 +788,7 @@ public void testIndexOnSharedFSRecoversToAnyNode() throws Exception {
         String newFooNode = internalCluster().startNode(fooSettings);
 
         assertShardCountOn(newFooNode, 5);
-        assertNoShardsOn(barNodes.get());
+        assertNoShardsOn(barNodes);
     }
 
     public void testDeletingClosedIndexRemovesFiles() throws Exception {
@@ -807,7 +797,7 @@ public void testDeletingClosedIndexRemovesFiles() throws Exception {
 
         final int numNodes = randomIntBetween(2, 5);
         logger.info("--> starting {} nodes", numNodes);
-        final List<String> nodes = internalCluster().startNodesAsync(numNodes, nodeSettings).get();
+        final List<String> nodes = internalCluster().startNodes(numNodes, nodeSettings);
         final String IDX = "test";
         final Tuple<Integer, Integer> numPrimariesAndReplicas = randomPrimariesAndReplicas(numNodes);
         final int numPrimaries = numPrimariesAndReplicas.v1();
@@ -850,7 +840,7 @@ public void testNodeJoinsWithoutShadowReplicaConfigured() throws Exception {
         Path dataPath = createTempDir();
         Settings nodeSettings = nodeSettings(dataPath);
 
-        internalCluster().startNodesAsync(2, nodeSettings).get();
+        internalCluster().startNodes(2, nodeSettings);
         String IDX = "test";
 
         Settings idxSettings = Settings.builder()
@@ -867,7 +857,7 @@ public void testNodeJoinsWithoutShadowReplicaConfigured() throws Exception {
         client().prepareIndex(IDX, "doc", "2").setSource("foo", "bar").get();
         flushAndRefresh(IDX);
 
-        internalCluster().startNodesAsync(1).get();
+        internalCluster().startNodes(1);
         ensureYellow(IDX);
 
         final ClusterHealthResponse clusterHealth = client().admin().cluster()
diff --git a/core/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java b/core/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java
index 8951123d0daf6..04439758ae0f6 100644
--- a/core/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java
+++ b/core/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.IndexingSlowLog.SlowLogParsedDocumentPrinter;
 import org.elasticsearch.index.mapper.ParsedDocument;
@@ -40,7 +41,7 @@ public class IndexingSlowLogTests extends ESTestCase {
     public void testSlowLogParsedDocumentPrinterSourceToLog() throws IOException {
         BytesReference source = JsonXContent.contentBuilder().startObject().field("foo", "bar").endObject().bytes();
         ParsedDocument pd = new ParsedDocument(new NumericDocValuesField("version", 1), "id",
-                "test", null, 0, -1, null, source, null);
+                "test", null, 0, -1, null, source, XContentType.JSON, null);
         Index index = new Index("foo", "123");
         // Turning off document logging doesn't log source[]
         SlowLogParsedDocumentPrinter p = new SlowLogParsedDocumentPrinter(index, pd, 10, true, 0);
diff --git a/core/src/test/java/org/elasticsearch/index/MergePolicySettingsTests.java b/core/src/test/java/org/elasticsearch/index/MergePolicySettingsTests.java
index 9caf9790c7a9c..ff60a206229d6 100644
--- a/core/src/test/java/org/elasticsearch/index/MergePolicySettingsTests.java
+++ b/core/src/test/java/org/elasticsearch/index/MergePolicySettingsTests.java
@@ -20,13 +20,9 @@
 
 import org.apache.lucene.index.NoMergePolicy;
 import org.apache.lucene.index.TieredMergePolicy;
-import org.elasticsearch.Version;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.index.MergePolicyConfig;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESTestCase;
 
@@ -85,9 +81,9 @@ public void testTieredMergePolicySettingsUpdate() throws IOException {
         indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_EXPUNGE_DELETES_ALLOWED_SETTING.getKey(), MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED + 1.0d).build()));
         assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getForceMergeDeletesPctAllowed(), MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED + 1.0d, 0.0d);
 
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(), MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.mbFrac(), 0);
-        indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_FLOOR_SEGMENT_SETTING.getKey(), new ByteSizeValue(MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.mb() + 1, ByteSizeUnit.MB)).build()));
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.mb() + 1, ByteSizeUnit.MB).mbFrac(), 0.001);
+        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(), MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMbFrac(), 0);
+        indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_FLOOR_SEGMENT_SETTING.getKey(), new ByteSizeValue(MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMb() + 1, ByteSizeUnit.MB)).build()));
+        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMb() + 1, ByteSizeUnit.MB).getMbFrac(), 0.001);
 
         assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE);
         indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING.getKey(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE - 1).build()));
@@ -97,9 +93,9 @@ public void testTieredMergePolicySettingsUpdate() throws IOException {
         indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_EXPLICIT_SETTING.getKey(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT - 1).build()));
         assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT-1);
 
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.mbFrac(), 0.0001);
-        indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGED_SEGMENT_SETTING.getKey(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.bytes() + 1)).build()));
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.bytes() + 1).mbFrac(), 0.0001);
+        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getMbFrac(), 0.0001);
+        indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGED_SEGMENT_SETTING.getKey(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1)).build()));
+        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(), 0.0001);
 
         assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT, 0);
         indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_RECLAIM_DELETES_WEIGHT_SETTING.getKey(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT + 1).build()));
@@ -111,10 +107,10 @@ public void testTieredMergePolicySettingsUpdate() throws IOException {
 
         indexSettings.updateIndexMetaData(newIndexMeta("index", EMPTY_SETTINGS)); // see if defaults are restored
         assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getForceMergeDeletesPctAllowed(), MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED, 0.0d);
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.mb(), ByteSizeUnit.MB).mbFrac(), 0.00);
+        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMb(), ByteSizeUnit.MB).getMbFrac(), 0.00);
         assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE);
         assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT);
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.bytes() + 1).mbFrac(), 0.0001);
+        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(), 0.0001);
         assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT, 0);
         assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER, 0);
     }
diff --git a/core/src/test/java/org/elasticsearch/index/MergeSchedulerSettingsTests.java b/core/src/test/java/org/elasticsearch/index/MergeSchedulerSettingsTests.java
new file mode 100644
index 0000000000000..e9eb5d8b83d2e
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/MergeSchedulerSettingsTests.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index;
+
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.core.LogEvent;
+import org.apache.logging.log4j.core.appender.AbstractAppender;
+import org.apache.logging.log4j.core.filter.RegexFilter;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ESTestCase;
+
+import static org.elasticsearch.common.util.concurrent.EsExecutors.PROCESSORS_SETTING;
+import static org.elasticsearch.index.IndexSettingsTests.newIndexMeta;
+import static org.elasticsearch.index.MergeSchedulerConfig.MAX_MERGE_COUNT_SETTING;
+import static org.elasticsearch.index.MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING;
+import static org.hamcrest.core.StringContains.containsString;
+
+public class MergeSchedulerSettingsTests extends ESTestCase {
+    private static class MockAppender extends AbstractAppender {
+        public boolean sawUpdateMaxThreadCount;
+        public boolean sawUpdateAutoThrottle;
+
+        MockAppender(final String name) throws IllegalAccessException {
+            super(name, RegexFilter.createFilter(".*(\n.*)*", new String[0], false, null, null), null);
+        }
+
+        @Override
+        public void append(LogEvent event) {
+            String message = event.getMessage().getFormattedMessage();
+            if (event.getLevel() == Level.TRACE && event.getLoggerName().endsWith("lucene.iw")) {
+            }
+            if (event.getLevel() == Level.INFO
+                && message.contains("updating [index.merge.scheduler.max_thread_count] from [10000] to [1]")) {
+                sawUpdateMaxThreadCount = true;
+            }
+            if (event.getLevel() == Level.INFO
+                && message.contains("updating [index.merge.scheduler.auto_throttle] from [true] to [false]")) {
+                sawUpdateAutoThrottle = true;
+            }
+        }
+
+        @Override
+        public boolean ignoreExceptions() {
+            return false;
+        }
+
+    }
+
+    public void testUpdateAutoThrottleSettings() throws Exception {
+        MockAppender mockAppender = new MockAppender("testUpdateAutoThrottleSettings");
+        mockAppender.start();
+        final Logger settingsLogger = Loggers.getLogger("org.elasticsearch.common.settings.IndexScopedSettings");
+        Loggers.addAppender(settingsLogger, mockAppender);
+        Loggers.setLevel(settingsLogger, Level.TRACE);
+        try {
+            Settings.Builder builder = Settings.builder()
+                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1")
+                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0")
+                .put(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING.getKey(), "2")
+                .put(MergePolicyConfig.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING.getKey(), "2")
+                .put(MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING.getKey(), "1")
+                .put(MergeSchedulerConfig.MAX_MERGE_COUNT_SETTING.getKey(), "2")
+                .put(MergeSchedulerConfig.AUTO_THROTTLE_SETTING.getKey(), "true");
+            IndexSettings settings = new IndexSettings(newIndexMeta("index", builder.build()), Settings.EMPTY);
+            assertEquals(settings.getMergeSchedulerConfig().isAutoThrottle(), true);
+            builder.put(MergeSchedulerConfig.AUTO_THROTTLE_SETTING.getKey(), "false");
+            settings.updateIndexMetaData(newIndexMeta("index", builder.build()));
+            // Make sure we log the change:
+            assertTrue(mockAppender.sawUpdateAutoThrottle);
+            assertEquals(settings.getMergeSchedulerConfig().isAutoThrottle(), false);
+        } finally {
+            Loggers.removeAppender(settingsLogger, mockAppender);
+            mockAppender.stop();
+            Loggers.setLevel(settingsLogger, (Level) null);
+        }
+    }
+
+    // #6882: make sure we can change index.merge.scheduler.max_thread_count live
+    public void testUpdateMergeMaxThreadCount() throws Exception {
+        MockAppender mockAppender = new MockAppender("testUpdateAutoThrottleSettings");
+        mockAppender.start();
+        final Logger settingsLogger = Loggers.getLogger("org.elasticsearch.common.settings.IndexScopedSettings");
+        Loggers.addAppender(settingsLogger, mockAppender);
+        Loggers.setLevel(settingsLogger, Level.TRACE);
+        try {
+            Settings.Builder builder = Settings.builder()
+                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1")
+                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0")
+                .put(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING.getKey(), "2")
+                .put(MergePolicyConfig.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING.getKey(), "2")
+                .put(MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING.getKey(), "10000")
+                .put(MergeSchedulerConfig.MAX_MERGE_COUNT_SETTING.getKey(), "10000");
+            IndexSettings settings = new IndexSettings(newIndexMeta("index", builder.build()), Settings.EMPTY);
+            assertEquals(settings.getMergeSchedulerConfig().getMaxMergeCount(), 10000);
+            assertEquals(settings.getMergeSchedulerConfig().getMaxThreadCount(), 10000);
+            settings.updateIndexMetaData(newIndexMeta("index", builder.build()));
+            assertFalse(mockAppender.sawUpdateMaxThreadCount);
+            builder.put(MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING.getKey(), "1");
+            settings.updateIndexMetaData(newIndexMeta("index", builder.build()));
+            // Make sure we log the change:
+            assertTrue(mockAppender.sawUpdateMaxThreadCount);
+        } finally {
+            Loggers.removeAppender(settingsLogger, mockAppender);
+            mockAppender.stop();
+            Loggers.setLevel(settingsLogger, (Level) null);
+        }
+    }
+
+    private static IndexMetaData createMetaData(int maxThreadCount, int maxMergeCount, int numProc) {
+        Settings.Builder builder = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT);
+        if (maxThreadCount != -1) {
+            builder.put(MAX_THREAD_COUNT_SETTING.getKey(), maxThreadCount);
+        }
+        if (maxMergeCount != -1) {
+            builder.put(MAX_MERGE_COUNT_SETTING.getKey(), maxMergeCount);
+        }
+        if (numProc != -1) {
+            builder.put(PROCESSORS_SETTING.getKey(), numProc);
+        }
+        return newIndexMeta("index", builder.build());
+    }
+
+    public void testMaxThreadAndMergeCount() {
+        IllegalArgumentException exc =
+            expectThrows(IllegalArgumentException.class,
+                () -> new MergeSchedulerConfig(new IndexSettings(createMetaData(10, 4, -1), Settings.EMPTY)));
+        assertThat(exc.getMessage(), containsString("maxThreadCount (= 10) should be <= maxMergeCount (= 4)"));
+
+        IndexSettings settings = new IndexSettings(createMetaData(-1, -1, 2), Settings.EMPTY);
+        assertEquals(1, settings.getMergeSchedulerConfig().getMaxThreadCount());
+        assertEquals(6, settings.getMergeSchedulerConfig().getMaxMergeCount());
+
+        settings = new IndexSettings(createMetaData(4, 10, -1), Settings.EMPTY);
+        assertEquals(4, settings.getMergeSchedulerConfig().getMaxThreadCount());
+        assertEquals(10, settings.getMergeSchedulerConfig().getMaxMergeCount());
+        IndexMetaData newMetaData = createMetaData(15, 20, -1);
+
+        settings.updateIndexMetaData(newMetaData);
+        assertEquals(15, settings.getMergeSchedulerConfig().getMaxThreadCount());
+        assertEquals(20, settings.getMergeSchedulerConfig().getMaxMergeCount());
+
+        settings.updateIndexMetaData(createMetaData(40, 50, -1));
+        assertEquals(40, settings.getMergeSchedulerConfig().getMaxThreadCount());
+        assertEquals(50, settings.getMergeSchedulerConfig().getMaxMergeCount());
+
+        settings.updateIndexMetaData(createMetaData(40, -1, -1));
+        assertEquals(40, settings.getMergeSchedulerConfig().getMaxThreadCount());
+        assertEquals(45, settings.getMergeSchedulerConfig().getMaxMergeCount());
+
+        final IndexSettings finalSettings = settings;
+        exc = expectThrows(IllegalArgumentException.class,
+            () -> finalSettings.updateIndexMetaData(createMetaData(40, 30, -1)));
+        assertThat(exc.getMessage(), containsString("maxThreadCount (= 40) should be <= maxMergeCount (= 30)"));
+
+        exc = expectThrows(IllegalArgumentException.class,
+            () -> finalSettings.updateIndexMetaData(createMetaData(-1, 3, 8)));
+        assertThat(exc.getMessage(), containsString("maxThreadCount (= 4) should be <= maxMergeCount (= 3)"));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java b/core/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java
index 062f8c01f425c..ffc038a40f487 100644
--- a/core/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java
+++ b/core/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java
@@ -26,9 +26,10 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.internal.SearchContext;
@@ -39,6 +40,8 @@
 
 import java.io.IOException;
 
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.startsWith;
 
 public class SearchSlowLogTests extends ESSingleNodeTestCase {
@@ -46,122 +49,104 @@ public class SearchSlowLogTests extends ESSingleNodeTestCase {
     protected SearchContext createSearchContext(IndexService indexService) {
         BigArrays bigArrays = indexService.getBigArrays();
         ThreadPool threadPool = indexService.getThreadPool();
-        ScriptService scriptService = node().injector().getInstance(ScriptService.class);
-        return new TestSearchContext(threadPool, bigArrays, scriptService, indexService) {
+        return new TestSearchContext(threadPool, bigArrays, indexService) {
+            final ShardSearchRequest request = new ShardSearchRequest() {
+                private SearchSourceBuilder searchSourceBuilder;
+                @Override
+                public ShardId shardId() {
+                    return new ShardId(indexService.index(), 0);
+                }
+
+                @Override
+                public String[] types() {
+                    return new String[0];
+                }
+
+                @Override
+                public SearchSourceBuilder source() {
+                    return searchSourceBuilder;
+                }
+
+                @Override
+                public void source(SearchSourceBuilder source) {
+                    searchSourceBuilder = source;
+                }
+
+                @Override
+                public int numberOfShards() {
+                    return 0;
+                }
+
+                @Override
+                public SearchType searchType() {
+                    return null;
+                }
+
+                @Override
+                public QueryBuilder filteringAliases() {
+                    return null;
+                }
+
+                @Override
+                public float indexBoost() {
+                    return 1.0f;
+                }
+
+                @Override
+                public long nowInMillis() {
+                    return 0;
+                }
+
+                @Override
+                public Boolean requestCache() {
+                    return null;
+                }
+
+                @Override
+                public Scroll scroll() {
+                    return null;
+                }
+
+                @Override
+                public void setProfile(boolean profile) {
+
+                }
+
+                @Override
+                public boolean isProfile() {
+                    return false;
+                }
+
+                @Override
+                public BytesReference cacheKey() throws IOException {
+                    return null;
+                }
+
+                @Override
+                public void rewrite(QueryShardContext context) throws IOException {
+                }
+
+                @Override
+                public String getClusterAlias() {
+                    return null;
+                }
+            };
             @Override
             public ShardSearchRequest request() {
-                return new ShardSearchRequest() {
-                    @Override
-                    public ShardId shardId() {
-                        return new ShardId(indexService.index(), 0);
-                    }
-
-                    @Override
-                    public String[] types() {
-                        return new String[0];
-                    }
-
-                    @Override
-                    public SearchSourceBuilder source() {
-                        return null;
-                    }
-
-                    @Override
-                    public void source(SearchSourceBuilder source) {
-
-                    }
-
-                    @Override
-                    public int numberOfShards() {
-                        return 0;
-                    }
-
-                    @Override
-                    public SearchType searchType() {
-                        return null;
-                    }
-
-                    @Override
-                    public String[] filteringAliases() {
-                        return new String[0];
-                    }
-
-                    @Override
-                    public long nowInMillis() {
-                        return 0;
-                    }
-
-                    @Override
-                    public Boolean requestCache() {
-                        return null;
-                    }
-
-                    @Override
-                    public Scroll scroll() {
-                        return null;
-                    }
-
-                    @Override
-                    public void setProfile(boolean profile) {
-
-                    }
-
-                    @Override
-                    public boolean isProfile() {
-                        return false;
-                    }
-
-                    @Override
-                    public BytesReference cacheKey() throws IOException {
-                        return null;
-                    }
-
-                    @Override
-                    public void rewrite(QueryShardContext context) throws IOException {
-                    }
-                };
+                return request;
             }
         };
     }
 
     public void testSlowLogSearchContextPrinterToLog() throws IOException {
         IndexService index = createIndex("foo");
-        // Turning off document logging doesn't log source[]
         SearchContext searchContext = createSearchContext(index);
-        SearchSlowLog.SlowLogSearchContextPrinter p = new SearchSlowLog.SlowLogSearchContextPrinter(searchContext, 10, true);
+        SearchSourceBuilder source = SearchSourceBuilder.searchSource().query(QueryBuilders.matchAllQuery());
+        searchContext.request().source(source);
+        SearchSlowLog.SlowLogSearchContextPrinter p = new SearchSlowLog.SlowLogSearchContextPrinter(searchContext, 10);
         assertThat(p.toString(), startsWith("[foo][0]"));
-    }
-
-    public void testReformatSetting() {
-        IndexMetaData metaData = newIndexMeta("index", Settings.builder()
-            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
-            .put(SearchSlowLog.INDEX_SEARCH_SLOWLOG_REFORMAT.getKey(), false)
-            .build());
-        IndexSettings settings = new IndexSettings(metaData, Settings.EMPTY);
-        SearchSlowLog log = new SearchSlowLog(settings);
-        assertFalse(log.isReformat());
-        settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(SearchSlowLog.INDEX_SEARCH_SLOWLOG_REFORMAT.getKey(), "true").build()));
-        assertTrue(log.isReformat());
-
-        settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(SearchSlowLog.INDEX_SEARCH_SLOWLOG_REFORMAT.getKey(), "false").build()));
-        assertFalse(log.isReformat());
-
-        settings.updateIndexMetaData(newIndexMeta("index", Settings.EMPTY));
-        assertTrue(log.isReformat());
-
-        metaData = newIndexMeta("index", Settings.builder()
-            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
-            .build());
-        settings = new IndexSettings(metaData, Settings.EMPTY);
-        log = new SearchSlowLog(settings);
-        assertTrue(log.isReformat());
-        try {
-            settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(SearchSlowLog.INDEX_SEARCH_SLOWLOG_REFORMAT.getKey(), "NOT A BOOLEAN").build()));
-            fail();
-        } catch (IllegalArgumentException ex) {
-            assertEquals(ex.getMessage(), "Failed to parse value [NOT A BOOLEAN] cannot be parsed to boolean [ true/1/on/yes OR false/0/off/no ]");
-        }
-        assertTrue(log.isReformat());
+        // Makes sure that output doesn't contain any new lines
+        assertThat(p.toString(), not(containsString("\n")));
     }
 
     public void testLevelSetting() {
@@ -192,7 +177,6 @@ public void testLevelSetting() {
             .build());
         settings = new IndexSettings(metaData, Settings.EMPTY);
         log = new SearchSlowLog(settings);
-        assertTrue(log.isReformat());
         try {
             settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(SearchSlowLog.INDEX_SEARCH_SLOWLOG_LEVEL.getKey(), "NOT A LEVEL").build()));
             fail();
diff --git a/core/src/test/java/org/elasticsearch/index/TransportIndexFailuresIT.java b/core/src/test/java/org/elasticsearch/index/TransportIndexFailuresIT.java
deleted file mode 100644
index 1ead4554a1143..0000000000000
--- a/core/src/test/java/org/elasticsearch/index/TransportIndexFailuresIT.java
+++ /dev/null
@@ -1,161 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index;
-
-import org.elasticsearch.action.index.IndexAction;
-import org.elasticsearch.action.index.IndexResponse;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.health.ClusterHealthStatus;
-import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
-import org.elasticsearch.cluster.routing.RoutingNodes;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.discovery.DiscoverySettings;
-import org.elasticsearch.discovery.zen.fd.FaultDetection;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.transport.MockTransportService;
-import org.elasticsearch.transport.TransportService;
-
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-
-import static java.util.Collections.singleton;
-import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
-import static org.elasticsearch.cluster.routing.ShardRoutingState.RELOCATING;
-import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED;
-import static org.elasticsearch.cluster.routing.ShardRoutingState.UNASSIGNED;
-import static org.hamcrest.Matchers.equalTo;
-
-/**
- * Test failure when index replication actions fail mid-flight
- */
-@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, transportClientRatio = 0)
-@ESIntegTestCase.SuppressLocalMode
-public class TransportIndexFailuresIT extends ESIntegTestCase {
-
-    private static final Settings nodeSettings = Settings.builder()
-            .put("discovery.type", "zen") // <-- To override the local setting if set externally
-            .put(FaultDetection.PING_TIMEOUT_SETTING.getKey(), "1s") // <-- for hitting simulated network failures quickly
-            .put(FaultDetection.PING_RETRIES_SETTING.getKey(), "1") // <-- for hitting simulated network failures quickly
-            .put(DiscoverySettings.PUBLISH_TIMEOUT_SETTING.getKey(), "1s") // <-- for hitting simulated network failures quickly
-            .put("discovery.zen.minimum_master_nodes", 1)
-            .build();
-
-    @Override
-    protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Arrays.asList(MockTransportService.TestPlugin.class);
-    }
-
-    @Override
-    protected int numberOfShards() {
-        return 1;
-    }
-
-    @Override
-    protected int numberOfReplicas() {
-        return 1;
-    }
-
-    public void testNetworkPartitionDuringReplicaIndexOp() throws Exception {
-        final String INDEX = "testidx";
-
-        List<String> nodes = internalCluster().startNodesAsync(2, nodeSettings).get();
-
-        // Create index test with 1 shard, 1 replica and ensure it is green
-        createIndex(INDEX);
-        ensureGreen(INDEX);
-
-        // Disable allocation so the replica cannot be reallocated when it fails
-        Settings s = Settings.builder().put("cluster.routing.allocation.enable", "none").build();
-        client().admin().cluster().prepareUpdateSettings().setTransientSettings(s).get();
-
-        // Determine which node holds the primary shard
-        ClusterState state = getNodeClusterState(nodes.get(0));
-        IndexShardRoutingTable shard = state.getRoutingTable().index(INDEX).shard(0);
-        String primaryNode;
-        String replicaNode;
-        if (shard.getShards().get(0).primary()) {
-            primaryNode = nodes.get(0);
-            replicaNode = nodes.get(1);
-        } else {
-            primaryNode = nodes.get(1);
-            replicaNode = nodes.get(0);
-        }
-        logger.info("--> primary shard is on {}", primaryNode);
-
-        // Index a document to make sure everything works well
-        IndexResponse resp = internalCluster().client(primaryNode).prepareIndex(INDEX, "doc").setSource("foo", "bar").get();
-        assertThat("document exists on primary node",
-                internalCluster().client(primaryNode).prepareGet(INDEX, "doc", resp.getId()).setPreference("_only_local").get().isExists(),
-                equalTo(true));
-        assertThat("document exists on replica node",
-                internalCluster().client(replicaNode).prepareGet(INDEX, "doc", resp.getId()).setPreference("_only_local").get().isExists(),
-                equalTo(true));
-
-        // Disrupt the network so indexing requests fail to replicate
-        logger.info("--> preventing index/replica operations");
-        TransportService mockTransportService = internalCluster().getInstance(TransportService.class, primaryNode);
-        ((MockTransportService) mockTransportService).addFailToSendNoConnectRule(
-                internalCluster().getInstance(TransportService.class, replicaNode),
-                singleton(IndexAction.NAME + "[r]")
-        );
-        mockTransportService = internalCluster().getInstance(TransportService.class, replicaNode);
-        ((MockTransportService) mockTransportService).addFailToSendNoConnectRule(
-                internalCluster().getInstance(TransportService.class, primaryNode),
-                singleton(IndexAction.NAME + "[r]")
-        );
-
-        logger.info("--> indexing into primary");
-        // the replica shard should now be marked as failed because the replication operation will fail
-        resp = internalCluster().client(primaryNode).prepareIndex(INDEX, "doc").setSource("foo", "baz").get();
-        // wait until the cluster reaches an exact yellow state, meaning replica has failed
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                assertThat(client().admin().cluster().prepareHealth().get().getStatus(), equalTo(ClusterHealthStatus.YELLOW));
-            }
-        });
-        assertThat("document should still be indexed and available",
-                client().prepareGet(INDEX, "doc", resp.getId()).get().isExists(), equalTo(true));
-
-        state = getNodeClusterState(randomFrom(nodes.toArray(Strings.EMPTY_ARRAY)));
-        RoutingNodes rn = state.getRoutingNodes();
-        logger.info("--> counts: total: {}, unassigned: {}, initializing: {}, relocating: {}, started: {}",
-                rn.shards(input -> true).size(),
-                rn.shardsWithState(UNASSIGNED).size(),
-                rn.shardsWithState(INITIALIZING).size(),
-                rn.shardsWithState(RELOCATING).size(),
-                rn.shardsWithState(STARTED).size());
-        logger.info("--> unassigned: {}, initializing: {}, relocating: {}, started: {}",
-                rn.shardsWithState(UNASSIGNED),
-                rn.shardsWithState(INITIALIZING),
-                rn.shardsWithState(RELOCATING),
-                rn.shardsWithState(STARTED));
-
-        assertThat("only a single shard is now active (replica should be failed and not reallocated)",
-                rn.shardsWithState(STARTED).size(), equalTo(1));
-    }
-
-    private ClusterState getNodeClusterState(String node) {
-        return internalCluster().client(node).admin().cluster().prepareState().setLocal(true).get().getState();
-    }
-}
diff --git a/core/src/test/java/org/elasticsearch/index/VersionTypeTests.java b/core/src/test/java/org/elasticsearch/index/VersionTypeTests.java
index 2afe0b7feacfd..4c79ce1b495f0 100644
--- a/core/src/test/java/org/elasticsearch/index/VersionTypeTests.java
+++ b/core/src/test/java/org/elasticsearch/index/VersionTypeTests.java
@@ -77,6 +77,13 @@ public void testVersionValidation() {
         assertTrue(VersionType.EXTERNAL_GTE.validateVersionForReads(randomIntBetween(1, Integer.MAX_VALUE)));
         assertFalse(VersionType.EXTERNAL_GTE.validateVersionForReads(randomIntBetween(Integer.MIN_VALUE, -1)));
 
+        assertTrue(VersionType.FORCE.validateVersionForWrites(randomIntBetween(1, Integer.MAX_VALUE)));
+        assertFalse(VersionType.FORCE.validateVersionForWrites(Versions.MATCH_ANY));
+        assertFalse(VersionType.FORCE.validateVersionForWrites(randomIntBetween(Integer.MIN_VALUE, 0)));
+        assertTrue(VersionType.FORCE.validateVersionForReads(Versions.MATCH_ANY));
+        assertTrue(VersionType.FORCE.validateVersionForReads(randomIntBetween(1, Integer.MAX_VALUE)));
+        assertFalse(VersionType.FORCE.validateVersionForReads(randomIntBetween(Integer.MIN_VALUE, -1)));
+
         assertTrue(VersionType.INTERNAL.validateVersionForWrites(randomIntBetween(1, Integer.MAX_VALUE)));
         assertTrue(VersionType.INTERNAL.validateVersionForWrites(Versions.MATCH_ANY));
         assertFalse(VersionType.INTERNAL.validateVersionForWrites(randomIntBetween(Integer.MIN_VALUE, 0)));
@@ -146,6 +153,36 @@ public void testExternalGTEVersionConflict() throws Exception {
 
     }
 
+    public void testForceVersionConflict() throws Exception {
+        assertFalse(VersionType.FORCE.isVersionConflictForWrites(Versions.NOT_FOUND, 10, randomBoolean()));
+
+        // MATCH_ANY must throw an exception in the case of force version, as the version must be set! it used as the new value
+        try {
+            VersionType.FORCE.isVersionConflictForWrites(10, Versions.MATCH_ANY, randomBoolean());
+            fail();
+        } catch (IllegalStateException e) {
+            //yes!!
+        }
+
+        // if we didn't find a version (but the index does support it), we always accept
+        assertFalse(VersionType.FORCE.isVersionConflictForWrites(Versions.NOT_FOUND, Versions.NOT_FOUND, randomBoolean()));
+        assertFalse(VersionType.FORCE.isVersionConflictForWrites(Versions.NOT_FOUND, 10, randomBoolean()));
+
+        assertFalse(VersionType.FORCE.isVersionConflictForReads(Versions.NOT_FOUND, Versions.NOT_FOUND));
+        assertFalse(VersionType.FORCE.isVersionConflictForReads(Versions.NOT_FOUND, 10));
+        assertFalse(VersionType.FORCE.isVersionConflictForReads(Versions.NOT_FOUND, Versions.MATCH_ANY));
+
+
+        // and the standard behavior
+        assertFalse(VersionType.FORCE.isVersionConflictForWrites(10, 10, randomBoolean()));
+        assertFalse(VersionType.FORCE.isVersionConflictForWrites(9, 10, randomBoolean()));
+        assertFalse(VersionType.FORCE.isVersionConflictForWrites(10, 9, randomBoolean()));
+        assertFalse(VersionType.FORCE.isVersionConflictForReads(10, 10));
+        assertFalse(VersionType.FORCE.isVersionConflictForReads(9, 10));
+        assertFalse(VersionType.FORCE.isVersionConflictForReads(10, 9));
+        assertFalse(VersionType.FORCE.isVersionConflictForReads(10, Versions.MATCH_ANY));
+    }
+
     public void testUpdateVersion() {
         assertThat(VersionType.INTERNAL.updateVersion(Versions.NOT_FOUND, 10), equalTo(1L));
         assertThat(VersionType.INTERNAL.updateVersion(1, 1), equalTo(2L));
@@ -159,6 +196,9 @@ public void testUpdateVersion() {
         assertThat(VersionType.EXTERNAL_GTE.updateVersion(1, 10), equalTo(10L));
         assertThat(VersionType.EXTERNAL_GTE.updateVersion(10, 10), equalTo(10L));
 
+        assertThat(VersionType.FORCE.updateVersion(Versions.NOT_FOUND, 10), equalTo(10L));
+        assertThat(VersionType.FORCE.updateVersion(11, 10), equalTo(10L));
+
 // Old indexing code
 //        if (index.versionType() == VersionType.INTERNAL) { // internal version type
 //            updatedVersion = (currentVersion == Versions.NOT_SET || currentVersion == Versions.NOT_FOUND) ? 1 : currentVersion + 1;
diff --git a/core/src/test/java/org/elasticsearch/index/WaitUntilRefreshIT.java b/core/src/test/java/org/elasticsearch/index/WaitUntilRefreshIT.java
index 378947ec345e5..ca089e6eb83ba 100644
--- a/core/src/test/java/org/elasticsearch/index/WaitUntilRefreshIT.java
+++ b/core/src/test/java/org/elasticsearch/index/WaitUntilRefreshIT.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
 import org.elasticsearch.action.update.UpdateResponse;
+import org.elasticsearch.client.Requests;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.ScriptPlugin;
@@ -35,7 +36,7 @@
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.NativeScriptFactory;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.junit.Before;
 
@@ -95,21 +96,22 @@ public void testUpdate() throws InterruptedException, ExecutionException {
         assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "bar")).get(), "1");
 
         // Update with RefreshPolicy.WAIT_UNTIL
-        UpdateResponse update = client().prepareUpdate("test", "test", "1").setDoc("foo", "baz").setRefreshPolicy(RefreshPolicy.WAIT_UNTIL)
+        UpdateResponse update = client().prepareUpdate("test", "test", "1")
+            .setDoc(Requests.INDEX_CONTENT_TYPE, "foo", "baz").setRefreshPolicy(RefreshPolicy.WAIT_UNTIL)
                 .get();
         assertEquals(2, update.getVersion());
         assertFalse("request shouldn't have forced a refresh", update.forcedRefresh());
         assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "baz")).get(), "1");
 
         // Upsert with RefreshPolicy.WAIT_UNTIL
-        update = client().prepareUpdate("test", "test", "2").setDocAsUpsert(true).setDoc("foo", "cat")
+        update = client().prepareUpdate("test", "test", "2").setDocAsUpsert(true).setDoc(Requests.INDEX_CONTENT_TYPE, "foo", "cat")
                 .setRefreshPolicy(RefreshPolicy.WAIT_UNTIL).get();
         assertEquals(1, update.getVersion());
         assertFalse("request shouldn't have forced a refresh", update.forcedRefresh());
         assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "cat")).get(), "2");
 
         // Update-becomes-delete with RefreshPolicy.WAIT_UNTIL
-        update = client().prepareUpdate("test", "test", "2").setScript(new Script("delete_plz", ScriptType.INLINE, "native", emptyMap()))
+        update = client().prepareUpdate("test", "test", "2").setScript(new Script(ScriptType.INLINE, "native", "delete_plz", emptyMap()))
                 .setRefreshPolicy(RefreshPolicy.WAIT_UNTIL).get();
         assertEquals(2, update.getVersion());
         assertFalse("request shouldn't have forced a refresh", update.forcedRefresh());
@@ -125,7 +127,7 @@ public void testBulk() {
 
         // Update by bulk with RefreshPolicy.WAIT_UNTIL
         bulk = client().prepareBulk().setRefreshPolicy(RefreshPolicy.WAIT_UNTIL);
-        bulk.add(client().prepareUpdate("test", "test", "1").setDoc("foo", "baz"));
+        bulk.add(client().prepareUpdate("test", "test", "1").setDoc(Requests.INDEX_CONTENT_TYPE, "foo", "baz"));
         assertBulkSuccess(bulk.get());
         assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "baz")).get(), "1");
 
diff --git a/core/src/test/java/org/elasticsearch/index/analysis/ASCIIFoldingTokenFilterFactoryTests.java b/core/src/test/java/org/elasticsearch/index/analysis/ASCIIFoldingTokenFilterFactoryTests.java
index 7b6d74a418cc7..973225df180f3 100644
--- a/core/src/test/java/org/elasticsearch/index/analysis/ASCIIFoldingTokenFilterFactoryTests.java
+++ b/core/src/test/java/org/elasticsearch/index/analysis/ASCIIFoldingTokenFilterFactoryTests.java
@@ -23,6 +23,7 @@
 import org.apache.lucene.analysis.core.WhitespaceTokenizer;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.ESTokenStreamTestCase;
 
 import java.io.IOException;
@@ -30,11 +31,11 @@
 
 public class ASCIIFoldingTokenFilterFactoryTests extends ESTokenStreamTestCase {
     public void testDefault() throws IOException {
-        AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(Settings.builder()
+        ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(Settings.builder()
                 .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
                 .put("index.analysis.filter.my_ascii_folding.type", "asciifolding")
                 .build());
-        TokenFilterFactory tokenFilter = analysisService.tokenFilter("my_ascii_folding");
+        TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_ascii_folding");
         String source = "Ansprüche";
         String[] expected = new String[]{"Anspruche"};
         Tokenizer tokenizer = new WhitespaceTokenizer();
@@ -43,16 +44,23 @@ public void testDefault() throws IOException {
     }
 
     public void testPreserveOriginal() throws IOException {
-        AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(Settings.builder()
+        ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(Settings.builder()
                 .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
                 .put("index.analysis.filter.my_ascii_folding.type", "asciifolding")
                 .put("index.analysis.filter.my_ascii_folding.preserve_original", true)
                 .build());
-        TokenFilterFactory tokenFilter = analysisService.tokenFilter("my_ascii_folding");
+        TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_ascii_folding");
         String source = "Ansprüche";
         String[] expected = new String[]{"Anspruche", "Ansprüche"};
         Tokenizer tokenizer = new WhitespaceTokenizer();
         tokenizer.setReader(new StringReader(source));
         assertTokenStreamContents(tokenFilter.create(tokenizer), expected);
+
+        // but the multi-term aware component still emits a single token
+        tokenFilter = (TokenFilterFactory) ((MultiTermAwareComponent) tokenFilter).getMultiTermComponent();
+        tokenizer = new WhitespaceTokenizer();
+        tokenizer.setReader(new StringReader(source));
+        expected = new String[]{"Anspruche"};
+        assertTokenStreamContents(tokenFilter.create(tokenizer), expected);
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java
new file mode 100644
index 0000000000000..9d9631e1b006c
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java
@@ -0,0 +1,240 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.analysis;
+
+import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.en.EnglishAnalyzer;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.indices.analysis.AnalysisModule;
+import org.elasticsearch.indices.analysis.PreBuiltAnalyzers;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.IndexSettingsModule;
+import org.elasticsearch.test.VersionUtils;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static java.util.Collections.emptyList;
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.singletonMap;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+
+public class AnalysisRegistryTests extends ESTestCase {
+
+    private AnalysisRegistry registry;
+
+    private static AnalyzerProvider<?> analyzerProvider(final String name) {
+        return new PreBuiltAnalyzerProvider(name, AnalyzerScope.INDEX, new EnglishAnalyzer());
+    }
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        Settings settings = Settings
+            .builder()
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+            .build();
+        registry = new AnalysisRegistry(new Environment(settings),
+            emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap());
+    }
+
+    public void testDefaultAnalyzers() throws IOException {
+        Version version = VersionUtils.randomVersion(random());
+        Settings settings = Settings
+            .builder()
+            .put(IndexMetaData.SETTING_VERSION_CREATED, version)
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+            .build();
+        IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
+        IndexAnalyzers indexAnalyzers = new AnalysisRegistry(new Environment(settings),
+                emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap())
+            .build(idxSettings);
+        assertThat(indexAnalyzers.getDefaultIndexAnalyzer().analyzer(), instanceOf(StandardAnalyzer.class));
+        assertThat(indexAnalyzers.getDefaultSearchAnalyzer().analyzer(), instanceOf(StandardAnalyzer.class));
+        assertThat(indexAnalyzers.getDefaultSearchQuoteAnalyzer().analyzer(), instanceOf(StandardAnalyzer.class));
+    }
+
+    public void testOverrideDefaultAnalyzer() throws IOException {
+        Version version = VersionUtils.randomVersion(random());
+        Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
+        IndexAnalyzers indexAnalyzers = registry.build(IndexSettingsModule.newIndexSettings("index", settings),
+            singletonMap("default", analyzerProvider("default"))
+                , emptyMap(), emptyMap(), emptyMap(), emptyMap());
+        assertThat(indexAnalyzers.getDefaultIndexAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class));
+        assertThat(indexAnalyzers.getDefaultSearchAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class));
+        assertThat(indexAnalyzers.getDefaultSearchQuoteAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class));
+    }
+
+    public void testOverrideDefaultIndexAnalyzerIsUnsupported() {
+        Version version = VersionUtils.randomVersionBetween(random(), Version.V_5_0_0_alpha1, Version.CURRENT);
+        Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
+        AnalyzerProvider<?> defaultIndex = new PreBuiltAnalyzerProvider("default_index", AnalyzerScope.INDEX, new EnglishAnalyzer());
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> registry.build(IndexSettingsModule.newIndexSettings("index", settings),
+                        singletonMap("default_index", defaultIndex), emptyMap(), emptyMap(), emptyMap(), emptyMap()));
+        assertTrue(e.getMessage().contains("[index.analysis.analyzer.default_index] is not supported"));
+    }
+
+    public void testBackCompatOverrideDefaultIndexAnalyzer() {
+        Version version = VersionUtils.randomVersionBetween(random(), VersionUtils.getFirstVersion(),
+                VersionUtils.getPreviousVersion(Version.V_5_0_0_alpha1));
+        Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
+        IndexAnalyzers indexAnalyzers = registry.build(IndexSettingsModule.newIndexSettings("index", settings),
+                singletonMap("default_index", analyzerProvider("default_index")), emptyMap(), emptyMap(), emptyMap(), emptyMap());
+        assertThat(indexAnalyzers.getDefaultIndexAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class));
+        assertThat(indexAnalyzers.getDefaultSearchAnalyzer().analyzer(), instanceOf(StandardAnalyzer.class));
+        assertThat(indexAnalyzers.getDefaultSearchQuoteAnalyzer().analyzer(), instanceOf(StandardAnalyzer.class));
+        assertWarnings("setting [index.analysis.analyzer.default_index] is deprecated, use [index.analysis.analyzer.default] " +
+                "instead for index [index]");
+    }
+
+    public void testOverrideDefaultSearchAnalyzer() {
+        Version version = VersionUtils.randomVersion(random());
+        Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
+        IndexAnalyzers indexAnalyzers = registry.build(IndexSettingsModule.newIndexSettings("index", settings),
+                singletonMap("default_search", analyzerProvider("default_search")), emptyMap(), emptyMap(), emptyMap(), emptyMap());
+        assertThat(indexAnalyzers.getDefaultIndexAnalyzer().analyzer(), instanceOf(StandardAnalyzer.class));
+        assertThat(indexAnalyzers.getDefaultSearchAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class));
+        assertThat(indexAnalyzers.getDefaultSearchQuoteAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class));
+    }
+
+    public void testBackCompatOverrideDefaultIndexAndSearchAnalyzer() {
+        Version version = VersionUtils.randomVersionBetween(random(), VersionUtils.getFirstVersion(),
+                VersionUtils.getPreviousVersion(Version.V_5_0_0_alpha1));
+        Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
+        Map<String, AnalyzerProvider<?>> analyzers = new HashMap<>();
+        analyzers.put("default_index", analyzerProvider("default_index"));
+        analyzers.put("default_search", analyzerProvider("default_search"));
+        IndexAnalyzers indexAnalyzers = registry.build(IndexSettingsModule.newIndexSettings("index", settings),
+                analyzers, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
+        assertThat(indexAnalyzers.getDefaultIndexAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class));
+        assertThat(indexAnalyzers.getDefaultSearchAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class));
+        assertThat(indexAnalyzers.getDefaultSearchQuoteAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class));
+        assertWarnings("setting [index.analysis.analyzer.default_index] is deprecated, use [index.analysis.analyzer.default] " +
+                "instead for index [index]");
+    }
+
+    public void testConfigureCamelCaseTokenFilter() throws IOException {
+        Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build();
+        Settings indexSettings = Settings.builder()
+                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                .put("index.analysis.filter.wordDelimiter.type", "word_delimiter")
+                .put("index.analysis.filter.wordDelimiter.split_on_numerics", false)
+                .put("index.analysis.analyzer.custom_analyzer.tokenizer", "whitespace")
+                .putArray("index.analysis.analyzer.custom_analyzer.filter", "lowercase", "wordDelimiter")
+                .put("index.analysis.analyzer.custom_analyzer_1.tokenizer", "whitespace")
+                .putArray("index.analysis.analyzer.custom_analyzer_1.filter", "lowercase", "word_delimiter").build();
+
+        IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings);
+
+        IndexAnalyzers indexAnalyzers = new AnalysisModule(new Environment(settings), emptyList()).getAnalysisRegistry()
+                .build(idxSettings);
+        try (NamedAnalyzer custom_analyser = indexAnalyzers.get("custom_analyzer")) {
+            assertNotNull(custom_analyser);
+            TokenStream tokenStream = custom_analyser.tokenStream("foo", "J2SE j2ee");
+            tokenStream.reset();
+            CharTermAttribute charTermAttribute = tokenStream.addAttribute(CharTermAttribute.class);
+            List<String> token = new ArrayList<>();
+            while(tokenStream.incrementToken()) {
+                token.add(charTermAttribute.toString());
+            }
+            assertEquals(token.toString(), 2, token.size());
+            assertEquals("j2se", token.get(0));
+            assertEquals("j2ee", token.get(1));
+        }
+
+        try (NamedAnalyzer custom_analyser = indexAnalyzers.get("custom_analyzer_1")) {
+            assertNotNull(custom_analyser);
+            TokenStream tokenStream = custom_analyser.tokenStream("foo", "J2SE j2ee");
+            tokenStream.reset();
+            CharTermAttribute charTermAttribute = tokenStream.addAttribute(CharTermAttribute.class);
+            List<String> token = new ArrayList<>();
+            while(tokenStream.incrementToken()) {
+                token.add(charTermAttribute.toString());
+            }
+            assertEquals(token.toString(), 6, token.size());
+            assertEquals("j", token.get(0));
+            assertEquals("2", token.get(1));
+            assertEquals("se", token.get(2));
+            assertEquals("j", token.get(3));
+            assertEquals("2", token.get(4));
+            assertEquals("ee", token.get(5));
+        }
+    }
+
+    public void testBuiltInAnalyzersAreCached() throws IOException {
+        Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build();
+        Settings indexSettings = Settings.builder()
+                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
+        IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings);
+        IndexAnalyzers indexAnalyzers = new AnalysisRegistry(new Environment(settings),
+                emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap())
+                .build(idxSettings);
+        IndexAnalyzers otherIndexAnalyzers = new AnalysisRegistry(new Environment(settings), emptyMap(), emptyMap(), emptyMap(),
+                emptyMap(), emptyMap()).build(idxSettings);
+        final int numIters = randomIntBetween(5, 20);
+        for (int i = 0; i < numIters; i++) {
+            PreBuiltAnalyzers preBuiltAnalyzers = RandomPicks.randomFrom(random(), PreBuiltAnalyzers.values());
+            assertSame(indexAnalyzers.get(preBuiltAnalyzers.name()), otherIndexAnalyzers.get(preBuiltAnalyzers.name()));
+        }
+    }
+
+    public void testNoTypeOrTokenizerErrorMessage() throws IOException {
+        Version version = VersionUtils.randomVersion(random());
+        Settings settings = Settings
+            .builder()
+            .put(IndexMetaData.SETTING_VERSION_CREATED, version)
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+            .putArray("index.analysis.analyzer.test_analyzer.filter", new String[] {"lowercase", "stop", "shingle"})
+            .putArray("index.analysis.analyzer.test_analyzer.char_filter", new String[] {"html_strip"})
+            .build();
+        IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> new AnalysisRegistry(new Environment(settings),
+                        emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap()).build(idxSettings));
+        assertThat(e.getMessage(), equalTo("analyzer [test_analyzer] must specify either an analyzer type, or a tokenizer"));
+    }
+
+    public void testCloseIndexAnalyzersMultipleTimes() throws IOException {
+        Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build();
+        Settings indexSettings = Settings.builder()
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
+        IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings);
+        IndexAnalyzers indexAnalyzers = new AnalysisRegistry(new Environment(settings),
+                emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap())
+            .build(idxSettings);
+        indexAnalyzers.close();
+        indexAnalyzers.close();
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisServiceTests.java b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisServiceTests.java
deleted file mode 100644
index 52fcdd4bb2ea5..0000000000000
--- a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisServiceTests.java
+++ /dev/null
@@ -1,207 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.analysis;
-
-import com.carrotsearch.randomizedtesting.generators.RandomPicks;
-
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.en.EnglishAnalyzer;
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.elasticsearch.Version;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.indices.analysis.AnalysisModule;
-import org.elasticsearch.indices.analysis.PreBuiltAnalyzers;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.IndexSettingsModule;
-import org.elasticsearch.test.VersionUtils;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static java.util.Collections.emptyList;
-import static java.util.Collections.emptyMap;
-import static java.util.Collections.singletonMap;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.instanceOf;
-
-public class AnalysisServiceTests extends ESTestCase {
-
-    private static AnalyzerProvider<?> analyzerProvider(final String name) {
-        return new PreBuiltAnalyzerProvider(name, AnalyzerScope.INDEX, new EnglishAnalyzer());
-    }
-
-    public void testDefaultAnalyzers() throws IOException {
-        Version version = VersionUtils.randomVersion(random());
-        Settings settings = Settings
-            .builder()
-            .put(IndexMetaData.SETTING_VERSION_CREATED, version)
-            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
-            .build();
-        IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
-        AnalysisService analysisService = new AnalysisRegistry(new Environment(settings), emptyMap(), emptyMap(), emptyMap(), emptyMap())
-                .build(idxSettings);
-        assertThat(analysisService.defaultIndexAnalyzer().analyzer(), instanceOf(StandardAnalyzer.class));
-        assertThat(analysisService.defaultSearchAnalyzer().analyzer(), instanceOf(StandardAnalyzer.class));
-        assertThat(analysisService.defaultSearchQuoteAnalyzer().analyzer(), instanceOf(StandardAnalyzer.class));
-    }
-
-    public void testOverrideDefaultAnalyzer() throws IOException {
-        Version version = VersionUtils.randomVersion(random());
-        Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
-        AnalysisService analysisService = new AnalysisService(IndexSettingsModule.newIndexSettings("index", settings),
-                singletonMap("default", analyzerProvider("default")), emptyMap(), emptyMap(), emptyMap());
-        assertThat(analysisService.defaultIndexAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class));
-        assertThat(analysisService.defaultSearchAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class));
-        assertThat(analysisService.defaultSearchQuoteAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class));
-    }
-
-    public void testOverrideDefaultIndexAnalyzerIsUnsupported() {
-        Version version = VersionUtils.randomVersionBetween(random(), Version.V_5_0_0_alpha1, Version.CURRENT);
-        Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
-        AnalyzerProvider<?> defaultIndex = new PreBuiltAnalyzerProvider("default_index", AnalyzerScope.INDEX, new EnglishAnalyzer()); 
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-                () -> new AnalysisService(IndexSettingsModule.newIndexSettings("index", settings),
-                        singletonMap("default_index", defaultIndex), emptyMap(), emptyMap(), emptyMap()));
-        assertTrue(e.getMessage().contains("[index.analysis.analyzer.default_index] is not supported"));
-    }
-
-    public void testBackCompatOverrideDefaultIndexAnalyzer() {
-        Version version = VersionUtils.randomVersionBetween(random(), VersionUtils.getFirstVersion(),
-                VersionUtils.getPreviousVersion(Version.V_5_0_0_alpha1));
-        Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
-        AnalysisService analysisService = new AnalysisService(IndexSettingsModule.newIndexSettings("index", settings),
-                singletonMap("default_index", analyzerProvider("default_index")), emptyMap(), emptyMap(), emptyMap());
-        assertThat(analysisService.defaultIndexAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class));
-        assertThat(analysisService.defaultSearchAnalyzer().analyzer(), instanceOf(StandardAnalyzer.class));
-        assertThat(analysisService.defaultSearchQuoteAnalyzer().analyzer(), instanceOf(StandardAnalyzer.class));
-    }
-
-    public void testOverrideDefaultSearchAnalyzer() {
-        Version version = VersionUtils.randomVersion(random());
-        Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
-        AnalysisService analysisService = new AnalysisService(IndexSettingsModule.newIndexSettings("index", settings),
-                singletonMap("default_search", analyzerProvider("default_search")), emptyMap(), emptyMap(), emptyMap());
-        assertThat(analysisService.defaultIndexAnalyzer().analyzer(), instanceOf(StandardAnalyzer.class));
-        assertThat(analysisService.defaultSearchAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class));
-        assertThat(analysisService.defaultSearchQuoteAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class));
-    }
-
-    public void testBackCompatOverrideDefaultIndexAndSearchAnalyzer() {
-        Version version = VersionUtils.randomVersionBetween(random(), VersionUtils.getFirstVersion(),
-                VersionUtils.getPreviousVersion(Version.V_5_0_0_alpha1));
-        Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
-        Map<String, AnalyzerProvider<?>> analyzers = new HashMap<>();
-        analyzers.put("default_index", analyzerProvider("default_index"));
-        analyzers.put("default_search", analyzerProvider("default_search"));
-        AnalysisService analysisService = new AnalysisService(IndexSettingsModule.newIndexSettings("index", settings),
-                analyzers, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
-        assertThat(analysisService.defaultIndexAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class));
-        assertThat(analysisService.defaultSearchAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class));
-        assertThat(analysisService.defaultSearchQuoteAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class));
-    }
-
-    public void testConfigureCamelCaseTokenFilter() throws IOException {
-        Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build();
-        Settings indexSettings = Settings.builder()
-                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
-                .put("index.analysis.filter.wordDelimiter.type", "word_delimiter")
-                .put("index.analysis.filter.wordDelimiter.split_on_numerics", false)
-                .put("index.analysis.analyzer.custom_analyzer.tokenizer", "whitespace")
-                .putArray("index.analysis.analyzer.custom_analyzer.filter", "lowercase", "wordDelimiter")
-                .put("index.analysis.analyzer.custom_analyzer_1.tokenizer", "whitespace")
-                .putArray("index.analysis.analyzer.custom_analyzer_1.filter", "lowercase", "word_delimiter").build();
-
-        IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings);
-        
-        AnalysisService analysisService = new AnalysisModule(new Environment(settings), emptyList()).getAnalysisRegistry()
-                .build(idxSettings);
-        try (NamedAnalyzer custom_analyser = analysisService.analyzer("custom_analyzer")) {
-            assertNotNull(custom_analyser);
-            TokenStream tokenStream = custom_analyser.tokenStream("foo", "J2SE j2ee");
-            tokenStream.reset();
-            CharTermAttribute charTermAttribute = tokenStream.addAttribute(CharTermAttribute.class);
-            List<String> token = new ArrayList<>();
-            while(tokenStream.incrementToken()) {
-                token.add(charTermAttribute.toString());
-            }
-            assertEquals(token.toString(), 2, token.size());
-            assertEquals("j2se", token.get(0));
-            assertEquals("j2ee", token.get(1));
-        }
-
-        try (NamedAnalyzer custom_analyser = analysisService.analyzer("custom_analyzer_1")) {
-            assertNotNull(custom_analyser);
-            TokenStream tokenStream = custom_analyser.tokenStream("foo", "J2SE j2ee");
-            tokenStream.reset();
-            CharTermAttribute charTermAttribute = tokenStream.addAttribute(CharTermAttribute.class);
-            List<String> token = new ArrayList<>();
-            while(tokenStream.incrementToken()) {
-                token.add(charTermAttribute.toString());
-            }
-            assertEquals(token.toString(), 6, token.size());
-            assertEquals("j", token.get(0));
-            assertEquals("2", token.get(1));
-            assertEquals("se", token.get(2));
-            assertEquals("j", token.get(3));
-            assertEquals("2", token.get(4));
-            assertEquals("ee", token.get(5));
-        }
-    }
-
-    public void testBuiltInAnalyzersAreCached() throws IOException {
-        Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build();
-        Settings indexSettings = Settings.builder()
-                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
-        IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings);
-        AnalysisService analysisService = new AnalysisRegistry(new Environment(settings), emptyMap(), emptyMap(), emptyMap(), emptyMap())
-                .build(idxSettings);
-        AnalysisService otherAnalysisSergice = new AnalysisRegistry(new Environment(settings), emptyMap(), emptyMap(), emptyMap(),
-                emptyMap()).build(idxSettings);
-        final int numIters = randomIntBetween(5, 20);
-        for (int i = 0; i < numIters; i++) {
-            PreBuiltAnalyzers preBuiltAnalyzers = RandomPicks.randomFrom(random(), PreBuiltAnalyzers.values());
-            assertSame(analysisService.analyzer(preBuiltAnalyzers.name()), otherAnalysisSergice.analyzer(preBuiltAnalyzers.name()));
-        }
-    }
-
-    public void testNoTypeOrTokenizerErrorMessage() throws IOException {
-        Version version = VersionUtils.randomVersion(random());
-        Settings settings = Settings
-            .builder()
-            .put(IndexMetaData.SETTING_VERSION_CREATED, version)
-            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
-            .putArray("index.analysis.analyzer.test_analyzer.filter", new String[] {"lowercase", "stop", "shingle"})
-            .putArray("index.analysis.analyzer.test_analyzer.char_filter", new String[] {"html_strip"})
-            .build();
-        IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
-
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-                () -> new AnalysisRegistry(new Environment(settings), emptyMap(), emptyMap(), emptyMap(), emptyMap()).build(idxSettings));
-        assertThat(e.getMessage(), equalTo("analyzer [test_analyzer] must specify either an analyzer type, or a tokenizer"));
-    }
-}
diff --git a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisTestsHelper.java b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisTestsHelper.java
index 40ec2b412ff15..a60c21c1a7e6d 100644
--- a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisTestsHelper.java
+++ b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisTestsHelper.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.indices.analysis.AnalysisModule;
+import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 
 import java.io.IOException;
@@ -34,21 +35,25 @@
 
 public class AnalysisTestsHelper {
 
-    public static AnalysisService createAnalysisServiceFromClassPath(Path baseDir, String resource) throws IOException {
+    public static ESTestCase.TestAnalysis createTestAnalysisFromClassPath(Path baseDir, String resource) throws IOException {
         Settings settings = Settings.builder()
                 .loadFromStream(resource, AnalysisTestsHelper.class.getResourceAsStream(resource))
                 .put(Environment.PATH_HOME_SETTING.getKey(), baseDir.toString())
                 .build();
 
-        return createAnalysisServiceFromSettings(settings);
+        return createTestAnalysisFromSettings(settings);
     }
 
-    public static AnalysisService createAnalysisServiceFromSettings(
+    public static ESTestCase.TestAnalysis createTestAnalysisFromSettings(
             Settings settings) throws IOException {
         if (settings.get(IndexMetaData.SETTING_VERSION_CREATED) == null) {
             settings = Settings.builder().put(settings).put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
         }
-        IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("test", settings);
-        return new AnalysisModule(new Environment(settings), emptyList()).getAnalysisRegistry().build(idxSettings);
+        IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("test", settings);
+        AnalysisRegistry analysisRegistry = new AnalysisModule(new Environment(settings), emptyList()).getAnalysisRegistry();
+        return new ESTestCase.TestAnalysis(analysisRegistry.build(indexSettings),
+            analysisRegistry.buildTokenFilterFactories(indexSettings),
+            analysisRegistry.buildTokenizerFactories(indexSettings),
+            analysisRegistry.buildCharFilterFactories(indexSettings));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/analysis/BaseWordDelimiterTokenFilterFactoryTestCase.java b/core/src/test/java/org/elasticsearch/index/analysis/BaseWordDelimiterTokenFilterFactoryTestCase.java
new file mode 100644
index 0000000000000..713e9424759a0
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/analysis/BaseWordDelimiterTokenFilterFactoryTestCase.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.index.analysis;
+
+
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.core.WhitespaceTokenizer;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.ESTokenStreamTestCase;
+
+import java.io.IOException;
+import java.io.StringReader;
+
+/**
+ * Base class to test {@link WordDelimiterTokenFilterFactory}  and {@link WordDelimiterGraphTokenFilterFactory}
+ */
+public abstract class BaseWordDelimiterTokenFilterFactoryTestCase extends ESTokenStreamTestCase {
+    final String type;
+
+    public BaseWordDelimiterTokenFilterFactoryTestCase(String type) {
+        this.type = type;
+    }
+
+    public void testDefault() throws IOException {
+        ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(Settings.builder()
+                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+                .put("index.analysis.filter.my_word_delimiter.type", type)
+                .build());
+        TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_word_delimiter");
+        String source = "PowerShot 500-42 wi-fi wi-fi-4000 j2se O'Neil's";
+        String[] expected = new String[]{"Power", "Shot", "500", "42", "wi", "fi", "wi",
+            "fi", "4000", "j", "2", "se", "O", "Neil"};
+        Tokenizer tokenizer = new WhitespaceTokenizer();
+        tokenizer.setReader(new StringReader(source));
+        assertTokenStreamContents(tokenFilter.create(tokenizer), expected);
+    }
+
+    public void testCatenateWords() throws IOException {
+        ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(Settings.builder()
+                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+                .put("index.analysis.filter.my_word_delimiter.type", type)
+                .put("index.analysis.filter.my_word_delimiter.catenate_words", "true")
+                .put("index.analysis.filter.my_word_delimiter.generate_word_parts", "false")
+                .build());
+        TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_word_delimiter");
+        String source = "PowerShot 500-42 wi-fi wi-fi-4000 j2se O'Neil's";
+        String[] expected = new String[]{"PowerShot", "500", "42", "wifi", "wifi", "4000", "j", "2", "se", "ONeil"};
+        Tokenizer tokenizer = new WhitespaceTokenizer();
+        tokenizer.setReader(new StringReader(source));
+        assertTokenStreamContents(tokenFilter.create(tokenizer), expected);
+    }
+
+    public void testCatenateNumbers() throws IOException {
+        ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(Settings.builder()
+                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+                .put("index.analysis.filter.my_word_delimiter.type", type)
+                .put("index.analysis.filter.my_word_delimiter.generate_number_parts", "false")
+                .put("index.analysis.filter.my_word_delimiter.catenate_numbers", "true")
+                .build());
+        TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_word_delimiter");
+        String source = "PowerShot 500-42 wi-fi wi-fi-4000 j2se O'Neil's";
+        String[] expected = new String[]{"Power", "Shot", "50042", "wi", "fi", "wi", "fi", "4000", "j", "2",
+            "se", "O", "Neil"};
+        Tokenizer tokenizer = new WhitespaceTokenizer();
+        tokenizer.setReader(new StringReader(source));
+        assertTokenStreamContents(tokenFilter.create(tokenizer), expected);
+    }
+
+    public void testCatenateAll() throws IOException {
+        ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(Settings.builder()
+                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+                .put("index.analysis.filter.my_word_delimiter.type", type)
+                .put("index.analysis.filter.my_word_delimiter.generate_word_parts", "false")
+                .put("index.analysis.filter.my_word_delimiter.generate_number_parts", "false")
+                .put("index.analysis.filter.my_word_delimiter.catenate_all", "true")
+                .build());
+        TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_word_delimiter");
+        String source = "PowerShot 500-42 wi-fi wi-fi-4000 j2se O'Neil's";
+        String[] expected = new String[]{"PowerShot", "50042", "wifi", "wifi4000", "j2se", "ONeil"};
+        Tokenizer tokenizer = new WhitespaceTokenizer();
+        tokenizer.setReader(new StringReader(source));
+        assertTokenStreamContents(tokenFilter.create(tokenizer), expected);
+    }
+
+    public void testSplitOnCaseChange() throws IOException {
+        ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(Settings.builder()
+                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+                .put("index.analysis.filter.my_word_delimiter.type", type)
+                .put("index.analysis.filter.my_word_delimiter.split_on_case_change", "false")
+                .build());
+        TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_word_delimiter");
+        String source = "PowerShot";
+        String[] expected = new String[]{"PowerShot"};
+        Tokenizer tokenizer = new WhitespaceTokenizer();
+        tokenizer.setReader(new StringReader(source));
+        assertTokenStreamContents(tokenFilter.create(tokenizer), expected);
+    }
+
+    public void testPreserveOriginal() throws IOException {
+        ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(Settings.builder()
+                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+                .put("index.analysis.filter.my_word_delimiter.type", type)
+                .put("index.analysis.filter.my_word_delimiter.preserve_original", "true")
+                .build());
+        TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_word_delimiter");
+        String source = "PowerShot 500-42 wi-fi wi-fi-4000 j2se O'Neil's";
+        String[] expected = new String[]{"PowerShot", "Power", "Shot", "500-42", "500", "42", "wi-fi", "wi", "fi",
+            "wi-fi-4000", "wi", "fi", "4000", "j2se", "j", "2", "se", "O'Neil's", "O", "Neil"};
+        Tokenizer tokenizer = new WhitespaceTokenizer();
+        tokenizer.setReader(new StringReader(source));
+        assertTokenStreamContents(tokenFilter.create(tokenizer), expected);
+    }
+
+    public void testStemEnglishPossessive() throws IOException {
+        ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(Settings.builder()
+                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+                .put("index.analysis.filter.my_word_delimiter.type", type)
+                .put("index.analysis.filter.my_word_delimiter.stem_english_possessive", "false")
+                .build());
+        TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_word_delimiter");
+        String source = "PowerShot 500-42 wi-fi wi-fi-4000 j2se O'Neil's";
+        String[] expected = new String[]{"Power", "Shot", "500", "42", "wi", "fi", "wi", "fi", "4000", "j", "2",
+            "se", "O", "Neil", "s"};
+        Tokenizer tokenizer = new WhitespaceTokenizer();
+        tokenizer.setReader(new StringReader(source));
+        assertTokenStreamContents(tokenFilter.create(tokenizer), expected);
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/analysis/CJKFilterFactoryTests.java b/core/src/test/java/org/elasticsearch/index/analysis/CJKFilterFactoryTests.java
index d2e2d4cc6e25d..846f70addafd6 100644
--- a/core/src/test/java/org/elasticsearch/index/analysis/CJKFilterFactoryTests.java
+++ b/core/src/test/java/org/elasticsearch/index/analysis/CJKFilterFactoryTests.java
@@ -19,8 +19,11 @@
 
 package org.elasticsearch.index.analysis;
 
+import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.miscellaneous.DisableGraphAttribute;
 import org.apache.lucene.analysis.standard.StandardTokenizer;
+import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.ESTokenStreamTestCase;
 
 import java.io.IOException;
@@ -30,8 +33,8 @@ public class CJKFilterFactoryTests extends ESTokenStreamTestCase {
     private static final String RESOURCE = "/org/elasticsearch/index/analysis/cjk_analysis.json";
 
     public void testDefault() throws IOException {
-        AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromClassPath(createTempDir(), RESOURCE);
-        TokenFilterFactory tokenFilter = analysisService.tokenFilter("cjk_bigram");
+        ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromClassPath(createTempDir(), RESOURCE);
+        TokenFilterFactory tokenFilter = analysis.tokenFilter.get("cjk_bigram");
         String source = "多くの学生が試験に落ちた。";
         String[] expected = new String[]{"多く", "くの", "の学", "学生", "生が", "が試", "試験", "験に", "に落", "落ち", "ちた" };
         Tokenizer tokenizer = new StandardTokenizer();
@@ -40,8 +43,8 @@ public void testDefault() throws IOException {
     }
 
     public void testNoFlags() throws IOException {
-        AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromClassPath(createTempDir(), RESOURCE);
-        TokenFilterFactory tokenFilter = analysisService.tokenFilter("cjk_no_flags");
+        ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromClassPath(createTempDir(), RESOURCE);
+        TokenFilterFactory tokenFilter = analysis.tokenFilter.get("cjk_no_flags");
         String source = "多くの学生が試験に落ちた。";
         String[] expected = new String[]{"多く", "くの", "の学", "学生", "生が", "が試", "試験", "験に", "に落", "落ち", "ちた" };
         Tokenizer tokenizer = new StandardTokenizer();
@@ -50,8 +53,8 @@ public void testNoFlags() throws IOException {
     }
 
     public void testHanOnly() throws IOException {
-        AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromClassPath(createTempDir(), RESOURCE);
-        TokenFilterFactory tokenFilter = analysisService.tokenFilter("cjk_han_only");
+        ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromClassPath(createTempDir(), RESOURCE);
+        TokenFilterFactory tokenFilter = analysis.tokenFilter.get("cjk_han_only");
         String source = "多くの学生が試験に落ちた。";
         String[] expected = new String[]{"多", "く", "の",  "学生", "が",  "試験", "に",  "落", "ち", "た"  };
         Tokenizer tokenizer = new StandardTokenizer();
@@ -60,12 +63,33 @@ public void testHanOnly() throws IOException {
     }
 
     public void testHanUnigramOnly() throws IOException {
-        AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromClassPath(createTempDir(), RESOURCE);
-        TokenFilterFactory tokenFilter = analysisService.tokenFilter("cjk_han_unigram_only");
+        ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromClassPath(createTempDir(), RESOURCE);
+        TokenFilterFactory tokenFilter = analysis.tokenFilter.get("cjk_han_unigram_only");
         String source = "多くの学生が試験に落ちた。";
         String[] expected = new String[]{"多", "く", "の",  "学", "学生", "生", "が",  "試", "試験", "験", "に",  "落", "ち", "た"  };
         Tokenizer tokenizer = new StandardTokenizer();
         tokenizer.setReader(new StringReader(source));
         assertTokenStreamContents(tokenFilter.create(tokenizer), expected);
     }
+
+    public void testDisableGraph() throws IOException {
+        ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromClassPath(createTempDir(), RESOURCE);
+        TokenFilterFactory allFlagsFactory = analysis.tokenFilter.get("cjk_all_flags");
+        TokenFilterFactory hanOnlyFactory = analysis.tokenFilter.get("cjk_han_only");
+
+        String source = "多くの学生が試験に落ちた。";
+        Tokenizer tokenizer = new StandardTokenizer();
+        tokenizer.setReader(new StringReader(source));
+        try (TokenStream tokenStream = allFlagsFactory.create(tokenizer)) {
+            // This config outputs different size of ngrams so graph analysis is disabled
+            assertTrue(tokenStream.hasAttribute(DisableGraphAttribute.class));
+        }
+
+        tokenizer = new StandardTokenizer();
+        tokenizer.setReader(new StringReader(source));
+        try (TokenStream tokenStream = hanOnlyFactory.create(tokenizer)) {
+            // This config uses only bigrams so graph analysis is enabled
+            assertFalse(tokenStream.hasAttribute(DisableGraphAttribute.class));
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/analysis/CharFilterTests.java b/core/src/test/java/org/elasticsearch/index/analysis/CharFilterTests.java
index 3f2b1461ef36c..206dffd0fb774 100644
--- a/core/src/test/java/org/elasticsearch/index/analysis/CharFilterTests.java
+++ b/core/src/test/java/org/elasticsearch/index/analysis/CharFilterTests.java
@@ -26,7 +26,7 @@
 import org.elasticsearch.test.ESTokenStreamTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 
-import static org.elasticsearch.test.ESTestCase.createAnalysisService;
+import static org.elasticsearch.test.ESTestCase.createTestAnalysis;
 
 /**
  */
@@ -41,8 +41,8 @@ public void testMappingCharFilter() throws Exception {
                 .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
                 .build();
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("test", settings);
-        AnalysisService analysisService = createAnalysisService(idxSettings, settings);
-        NamedAnalyzer analyzer1 = analysisService.analyzer("custom_with_char_filter");
+        IndexAnalyzers indexAnalyzers = createTestAnalysis(idxSettings, settings).indexAnalyzers;
+        NamedAnalyzer analyzer1 = indexAnalyzers.get("custom_with_char_filter");
 
         assertTokenStreamContents(analyzer1.tokenStream("test", "jeff quit phish"), new String[]{"jeff", "qit", "fish"});
 
@@ -58,9 +58,9 @@ public void testHtmlStripCharFilter() throws Exception {
                 .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
                 .build();
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("test", settings);
-        AnalysisService analysisService = createAnalysisService(idxSettings, settings);
 
-        NamedAnalyzer analyzer1 = analysisService.analyzer("custom_with_char_filter");
+        IndexAnalyzers indexAnalyzers = createTestAnalysis(idxSettings, settings).indexAnalyzers;
+        NamedAnalyzer analyzer1 = indexAnalyzers.get("custom_with_char_filter");
 
         assertTokenStreamContents(analyzer1.tokenStream("test", "<b>hello</b>!"), new String[]{"hello"});
 
@@ -80,8 +80,8 @@ public void testPatternReplaceCharFilter() throws Exception {
             .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
             .build();
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("test", settings);
-        AnalysisService analysisService = createAnalysisService(idxSettings, settings);
-        NamedAnalyzer analyzer1 = analysisService.analyzer("custom_with_char_filter");
+        IndexAnalyzers indexAnalyzers = createTestAnalysis(idxSettings, settings).indexAnalyzers;
+        NamedAnalyzer analyzer1 = indexAnalyzers.get("custom_with_char_filter");
 
         assertTokenStreamContents(analyzer1.tokenStream("test", "faBBbBB aBbbbBf"), new String[]{"foo", "oof"});
     }
diff --git a/core/src/test/java/org/elasticsearch/index/analysis/CompoundAnalysisTests.java b/core/src/test/java/org/elasticsearch/index/analysis/CompoundAnalysisTests.java
index 0c9010b2c9b03..ede4240455848 100644
--- a/core/src/test/java/org/elasticsearch/index/analysis/CompoundAnalysisTests.java
+++ b/core/src/test/java/org/elasticsearch/index/analysis/CompoundAnalysisTests.java
@@ -61,9 +61,7 @@ public Map<String, AnalysisProvider<TokenFilterFactory>> getTokenFilters() {
                 return singletonMap("myfilter", MyFilterTokenFilterFactory::new);
             }
         }));
-        AnalysisService analysisService = analysisModule.getAnalysisRegistry().build(idxSettings);
-
-        TokenFilterFactory filterFactory = analysisService.tokenFilter("dict_dec");
+        TokenFilterFactory filterFactory = analysisModule.getAnalysisRegistry().buildTokenFilterFactories(idxSettings).get("dict_dec");
         MatcherAssert.assertThat(filterFactory, instanceOf(DictionaryCompoundWordTokenFilterFactory.class));
     }
 
@@ -85,9 +83,8 @@ public Map<String, AnalysisProvider<TokenFilterFactory>> getTokenFilters() {
                 return singletonMap("myfilter", MyFilterTokenFilterFactory::new);
             }
         }));
-        AnalysisService analysisService = analysisModule.getAnalysisRegistry().build(idxSettings);
-
-        Analyzer analyzer = analysisService.analyzer(analyzerName).analyzer();
+        IndexAnalyzers indexAnalyzers = analysisModule.getAnalysisRegistry().build(idxSettings);
+        Analyzer analyzer = indexAnalyzers.get(analyzerName).analyzer();
 
         AllEntries allEntries = new AllEntries();
         allEntries.addText("field1", text, 1.0f);
diff --git a/core/src/test/java/org/elasticsearch/index/analysis/CustomNormalizerTests.java b/core/src/test/java/org/elasticsearch/index/analysis/CustomNormalizerTests.java
new file mode 100644
index 0000000000000..3e71a60973724
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/analysis/CustomNormalizerTests.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.analysis;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.ESTokenStreamTestCase;
+
+import java.io.IOException;
+
+public class CustomNormalizerTests extends ESTokenStreamTestCase {
+
+    public void testBasics() throws IOException {
+        Settings settings = Settings.builder()
+                .putArray("index.analysis.normalizer.my_normalizer.filter", "lowercase", "asciifolding")
+                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+                .build();
+        ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(settings);
+        assertNull(analysis.indexAnalyzers.get("my_normalizer"));
+        NamedAnalyzer normalizer = analysis.indexAnalyzers.getNormalizer("my_normalizer");
+        assertNotNull(normalizer);
+        assertEquals("my_normalizer", normalizer.name());
+        assertTokenStreamContents(normalizer.tokenStream("foo", "Cet été-là"), new String[] {"cet ete-la"});
+        assertEquals(new BytesRef("cet ete-la"), normalizer.normalize("foo", "Cet été-là"));
+    }
+
+    public void testUnknownType() {
+        Settings settings = Settings.builder()
+                .put("index.analysis.normalizer.my_normalizer.type", "foobar")
+                .putArray("index.analysis.normalizer.my_normalizer.filter", "lowercase", "asciifolding")
+                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+                .build();
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> AnalysisTestsHelper.createTestAnalysisFromSettings(settings));
+        assertEquals("Unknown normalizer type [foobar] for [my_normalizer]", e.getMessage());
+    }
+
+    public void testTokenizer() throws IOException {
+        Settings settings = Settings.builder()
+                .put("index.analysis.normalizer.my_normalizer.tokenizer", "keyword")
+                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+                .build();
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> AnalysisTestsHelper.createTestAnalysisFromSettings(settings));
+        assertEquals("Custom normalizer [my_normalizer] cannot configure a tokenizer", e.getMessage());
+    }
+
+    public void testCharFilters() throws IOException {
+        Settings settings = Settings.builder()
+                .put("index.analysis.char_filter.my_mapping.type", "mapping")
+                .putArray("index.analysis.char_filter.my_mapping.mappings", "a => z")
+                .putArray("index.analysis.normalizer.my_normalizer.char_filter", "my_mapping")
+                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+                .build();
+        ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(settings);
+        assertNull(analysis.indexAnalyzers.get("my_normalizer"));
+        NamedAnalyzer normalizer = analysis.indexAnalyzers.getNormalizer("my_normalizer");
+        assertNotNull(normalizer);
+        assertEquals("my_normalizer", normalizer.name());
+        assertTokenStreamContents(normalizer.tokenStream("foo", "abc"), new String[] {"zbc"});
+        assertEquals(new BytesRef("zbc"), normalizer.normalize("foo", "abc"));
+    }
+
+    public void testIllegalFilters() throws IOException {
+        Settings settings = Settings.builder()
+                .putArray("index.analysis.normalizer.my_normalizer.filter", "porter_stem")
+                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+                .build();
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> AnalysisTestsHelper.createTestAnalysisFromSettings(settings));
+        assertEquals("Custom normalizer [my_normalizer] may not use filter [porter_stem]", e.getMessage());
+    }
+
+    public void testIllegalCharFilters() throws IOException {
+        Settings settings = Settings.builder()
+                .putArray("index.analysis.normalizer.my_normalizer.char_filter", "html_strip")
+                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+                .build();
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> AnalysisTestsHelper.createTestAnalysisFromSettings(settings));
+        assertEquals("Custom normalizer [my_normalizer] may not use char filter [html_strip]", e.getMessage());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/analysis/FlattenGraphTokenFilterFactoryTests.java b/core/src/test/java/org/elasticsearch/index/analysis/FlattenGraphTokenFilterFactoryTests.java
new file mode 100644
index 0000000000000..259da010daaf5
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/analysis/FlattenGraphTokenFilterFactoryTests.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.analysis;
+
+import java.io.IOException;
+
+import org.apache.lucene.analysis.CannedTokenStream;
+import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.TokenStream;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.test.ESTokenStreamTestCase;
+import org.elasticsearch.test.IndexSettingsModule;
+
+public class FlattenGraphTokenFilterFactoryTests extends ESTokenStreamTestCase {
+
+    public void testBasic() throws IOException {
+
+        Index index = new Index("test", "_na_");
+        String name = "ngr";
+        Settings indexSettings = newAnalysisSettingsBuilder().build();
+        IndexSettings indexProperties = IndexSettingsModule.newIndexSettings(index, indexSettings);
+        Settings settings = newAnalysisSettingsBuilder().build();
+
+        // "wow that's funny" and "what the fudge" are separate side paths, in parallel with "wtf", on input:
+        TokenStream in = new CannedTokenStream(0, 12, new Token[] {
+                    token("wtf", 1, 5, 0, 3),
+                    token("what", 0, 1, 0, 3),
+                    token("wow", 0, 3, 0, 3),
+                    token("the", 1, 1, 0, 3),
+                    token("fudge", 1, 3, 0, 3),
+                    token("that's", 1, 1, 0, 3),
+                    token("funny", 1, 1, 0, 3),
+                    token("happened", 1, 1, 4, 12)
+                });
+
+        TokenStream tokens = new FlattenGraphTokenFilterFactory(indexProperties, null, name, settings).create(in);
+
+        // ... but on output, it's flattened to wtf/what/wow that's/the fudge/funny happened:
+        assertTokenStreamContents(tokens,
+                new String[] {"wtf", "what", "wow", "the", "that's", "fudge", "funny", "happened"},
+                new int[] {0, 0, 0, 0, 0, 0, 0, 4},
+                new int[] {3, 3, 3, 3, 3, 3, 3, 12},
+                new int[] {1, 0, 0, 1, 0, 1, 0, 1},
+                new int[] {3, 1, 1, 1, 1, 1, 1, 1},
+                12);
+    }
+
+    private static Token token(String term, int posInc, int posLength, int startOffset, int endOffset) {
+        final Token t = new Token(term, startOffset, endOffset);
+        t.setPositionIncrement(posInc);
+        t.setPositionLength(posLength);
+        return t;
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/analysis/HunspellTokenFilterFactoryTests.java b/core/src/test/java/org/elasticsearch/index/analysis/HunspellTokenFilterFactoryTests.java
index 45e4dd24f94b1..2708387da12de 100644
--- a/core/src/test/java/org/elasticsearch/index/analysis/HunspellTokenFilterFactoryTests.java
+++ b/core/src/test/java/org/elasticsearch/index/analysis/HunspellTokenFilterFactoryTests.java
@@ -36,8 +36,8 @@ public void testDedup() throws IOException {
                 .put("index.analysis.filter.en_US.locale", "en_US")
                 .build();
 
-        AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(settings);
-        TokenFilterFactory tokenFilter = analysisService.tokenFilter("en_US");
+        TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(settings);
+        TokenFilterFactory tokenFilter = analysis.tokenFilter.get("en_US");
         assertThat(tokenFilter, instanceOf(HunspellTokenFilterFactory.class));
         HunspellTokenFilterFactory hunspellTokenFilter = (HunspellTokenFilterFactory) tokenFilter;
         assertThat(hunspellTokenFilter.dedup(), is(true));
@@ -50,8 +50,8 @@ public void testDedup() throws IOException {
                 .put("index.analysis.filter.en_US.locale", "en_US")
                 .build();
 
-        analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(settings);
-        tokenFilter = analysisService.tokenFilter("en_US");
+        analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(settings);
+        tokenFilter = analysis.tokenFilter.get("en_US");
         assertThat(tokenFilter, instanceOf(HunspellTokenFilterFactory.class));
         hunspellTokenFilter = (HunspellTokenFilterFactory) tokenFilter;
         assertThat(hunspellTokenFilter.dedup(), is(false));
diff --git a/core/src/test/java/org/elasticsearch/index/analysis/KeepFilterFactoryTests.java b/core/src/test/java/org/elasticsearch/index/analysis/KeepFilterFactoryTests.java
index 35148874e18be..d5a6a590e78ae 100644
--- a/core/src/test/java/org/elasticsearch/index/analysis/KeepFilterFactoryTests.java
+++ b/core/src/test/java/org/elasticsearch/index/analysis/KeepFilterFactoryTests.java
@@ -23,6 +23,7 @@
 import org.apache.lucene.analysis.core.WhitespaceTokenizer;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.ESTokenStreamTestCase;
 import org.junit.Assert;
 
@@ -35,8 +36,8 @@ public class KeepFilterFactoryTests extends ESTokenStreamTestCase {
     private static final String RESOURCE = "/org/elasticsearch/index/analysis/keep_analysis.json";
 
     public void testLoadWithoutSettings() throws IOException {
-        AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromClassPath(createTempDir(), RESOURCE);
-        TokenFilterFactory tokenFilter = analysisService.tokenFilter("keep");
+        ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromClassPath(createTempDir(), RESOURCE);
+        TokenFilterFactory tokenFilter = analysis.tokenFilter.get("keep");
         Assert.assertNull(tokenFilter);
     }
 
@@ -48,7 +49,7 @@ public void testLoadOverConfiguredSettings() {
                 .put("index.analysis.filter.broken_keep_filter.keep_words", "[\"Hello\", \"worlD\"]")
                 .build();
         try {
-            AnalysisTestsHelper.createAnalysisServiceFromSettings(settings);
+            AnalysisTestsHelper.createTestAnalysisFromSettings(settings);
             Assert.fail("path and array are configured");
         } catch (IllegalArgumentException e) {
         } catch (IOException e) {
@@ -64,7 +65,7 @@ public void testKeepWordsPathSettings() {
                 .build();
         try {
             // test our none existing setup is picked up
-            AnalysisTestsHelper.createAnalysisServiceFromSettings(settings);
+            AnalysisTestsHelper.createTestAnalysisFromSettings(settings);
             fail("expected an exception due to non existent keep_words_path");
         } catch (IllegalArgumentException e) {
         } catch (IOException e) {
@@ -76,7 +77,7 @@ public void testKeepWordsPathSettings() {
                 .build();
         try {
             // test our none existing setup is picked up
-            AnalysisTestsHelper.createAnalysisServiceFromSettings(settings);
+            AnalysisTestsHelper.createTestAnalysisFromSettings(settings);
             fail("expected an exception indicating that you can't use [keep_words_path] with [keep_words] ");
         } catch (IllegalArgumentException e) {
         } catch (IOException e) {
@@ -86,8 +87,8 @@ public void testKeepWordsPathSettings() {
     }
 
     public void testCaseInsensitiveMapping() throws IOException {
-        AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromClassPath(createTempDir(), RESOURCE);
-        TokenFilterFactory tokenFilter = analysisService.tokenFilter("my_keep_filter");
+        ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromClassPath(createTempDir(), RESOURCE);
+        TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_keep_filter");
         assertThat(tokenFilter, instanceOf(KeepWordFilterFactory.class));
         String source = "hello small world";
         String[] expected = new String[]{"hello", "world"};
@@ -97,8 +98,8 @@ public void testCaseInsensitiveMapping() throws IOException {
     }
 
     public void testCaseSensitiveMapping() throws IOException {
-        AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromClassPath(createTempDir(), RESOURCE);
-        TokenFilterFactory tokenFilter = analysisService.tokenFilter("my_case_sensitive_keep_filter");
+        ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromClassPath(createTempDir(), RESOURCE);
+        TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_case_sensitive_keep_filter");
         assertThat(tokenFilter, instanceOf(KeepWordFilterFactory.class));
         String source = "Hello small world";
         String[] expected = new String[]{"Hello"};
diff --git a/core/src/test/java/org/elasticsearch/index/analysis/KeepTypesFilterFactoryTests.java b/core/src/test/java/org/elasticsearch/index/analysis/KeepTypesFilterFactoryTests.java
index 986b79fad20d4..48ce1139d8f20 100644
--- a/core/src/test/java/org/elasticsearch/index/analysis/KeepTypesFilterFactoryTests.java
+++ b/core/src/test/java/org/elasticsearch/index/analysis/KeepTypesFilterFactoryTests.java
@@ -23,6 +23,7 @@
 import org.apache.lucene.analysis.standard.StandardTokenizer;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.ESTokenStreamTestCase;
 
 import java.io.IOException;
@@ -37,8 +38,8 @@ public void testKeepTypes() throws IOException {
                 .put("index.analysis.filter.keep_numbers.type", "keep_types")
                 .putArray("index.analysis.filter.keep_numbers.types", new String[] {"<NUM>", "<SOMETHINGELSE>"})
                 .build();
-        AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(settings);
-        TokenFilterFactory tokenFilter = analysisService.tokenFilter("keep_numbers");
+        ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(settings);
+        TokenFilterFactory tokenFilter = analysis.tokenFilter.get("keep_numbers");
         assertThat(tokenFilter, instanceOf(KeepTypesFilterFactory.class));
         String source = "Hello 123 world";
         String[] expected = new String[]{"123"};
diff --git a/core/src/test/java/org/elasticsearch/index/analysis/KeywordMarkerFilterFactoryTests.java b/core/src/test/java/org/elasticsearch/index/analysis/KeywordMarkerFilterFactoryTests.java
new file mode 100644
index 0000000000000..3298537af97aa
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/analysis/KeywordMarkerFilterFactoryTests.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.analysis;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.core.WhitespaceTokenizer;
+import org.apache.lucene.analysis.miscellaneous.PatternKeywordMarkerFilter;
+import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.test.ESTestCase.TestAnalysis;
+import org.elasticsearch.test.ESTokenStreamTestCase;
+
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.instanceOf;
+
+/**
+ * Tests for the {@link KeywordMarkerTokenFilterFactory} class.
+ */
+public class KeywordMarkerFilterFactoryTests extends ESTokenStreamTestCase {
+
+    /**
+     * Tests using a keyword set for the keyword marker filter.
+     */
+    public void testKeywordSet() throws IOException {
+        Settings settings = Settings.builder()
+            .put("index.analysis.filter.my_keyword.type", "keyword_marker")
+            .put("index.analysis.filter.my_keyword.keywords", "running, sleeping")
+            .put("index.analysis.analyzer.my_keyword.type", "custom")
+            .put("index.analysis.analyzer.my_keyword.tokenizer", "standard")
+            .put("index.analysis.analyzer.my_keyword.filter", "my_keyword, porter_stem")
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+            .build();
+        TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(settings);
+        TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_keyword");
+        assertThat(tokenFilter, instanceOf(KeywordMarkerTokenFilterFactory.class));
+        TokenStream filter = tokenFilter.create(new WhitespaceTokenizer());
+        assertThat(filter, instanceOf(SetKeywordMarkerFilter.class));
+        NamedAnalyzer analyzer = analysis.indexAnalyzers.get("my_keyword");
+        // jogging is not part of the keywords set, so verify that its the only stemmed word
+        assertAnalyzesTo(analyzer, "running jogging sleeping",
+            new String[] { "running", "jog", "sleeping" });
+    }
+
+    /**
+     * Tests using a regular expression pattern for the keyword marker filter.
+     */
+    public void testKeywordPattern() throws IOException {
+        Settings settings = Settings.builder()
+            .put("index.analysis.filter.my_keyword.type", "keyword_marker")
+            .put("index.analysis.filter.my_keyword.keywords_pattern", "run[a-z]ing")
+            .put("index.analysis.analyzer.my_keyword.type", "custom")
+            .put("index.analysis.analyzer.my_keyword.tokenizer", "standard")
+            .put("index.analysis.analyzer.my_keyword.filter", "my_keyword, porter_stem")
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+            .build();
+        TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(settings);
+        TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_keyword");
+        assertThat(tokenFilter, instanceOf(KeywordMarkerTokenFilterFactory.class));
+        TokenStream filter = tokenFilter.create(new WhitespaceTokenizer());
+        assertThat(filter, instanceOf(PatternKeywordMarkerFilter.class));
+        NamedAnalyzer analyzer = analysis.indexAnalyzers.get("my_keyword");
+        // running should match the pattern, so it should not be stemmed but sleeping should
+        assertAnalyzesTo(analyzer, "running sleeping", new String[] { "running", "sleep" });
+    }
+
+    /**
+     * Verifies that both keywords and patterns cannot be specified together.
+     */
+    public void testCannotSpecifyBothKeywordsAndPattern() throws IOException {
+        Settings settings = Settings.builder()
+            .put("index.analysis.filter.my_keyword.type", "keyword_marker")
+            .put("index.analysis.filter.my_keyword.keywords", "running")
+            .put("index.analysis.filter.my_keyword.keywords_pattern", "run[a-z]ing")
+            .put("index.analysis.analyzer.my_keyword.type", "custom")
+            .put("index.analysis.analyzer.my_keyword.tokenizer", "standard")
+            .put("index.analysis.analyzer.my_keyword.filter", "my_keyword, porter_stem")
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+            .build();
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> AnalysisTestsHelper.createTestAnalysisFromSettings(settings));
+        assertEquals("cannot specify both `keywords_pattern` and `keywords` or `keywords_path`",
+            e.getMessage());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/analysis/LimitTokenCountFilterFactoryTests.java b/core/src/test/java/org/elasticsearch/index/analysis/LimitTokenCountFilterFactoryTests.java
index bf17e5c7bf037..f1d810505bf7a 100644
--- a/core/src/test/java/org/elasticsearch/index/analysis/LimitTokenCountFilterFactoryTests.java
+++ b/core/src/test/java/org/elasticsearch/index/analysis/LimitTokenCountFilterFactoryTests.java
@@ -23,6 +23,7 @@
 import org.apache.lucene.analysis.core.WhitespaceTokenizer;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.ESTokenStreamTestCase;
 
 import java.io.IOException;
@@ -34,9 +35,9 @@ public void testDefault() throws IOException {
                 .put("index.analysis.filter.limit_default.type", "limit")
                 .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
                 .build();
-        AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(settings);
+        ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(settings);
         {
-            TokenFilterFactory tokenFilter = analysisService.tokenFilter("limit_default");
+            TokenFilterFactory tokenFilter = analysis.tokenFilter.get("limit_default");
             String source = "the quick brown fox";
             String[] expected = new String[] { "the" };
             Tokenizer tokenizer = new WhitespaceTokenizer();
@@ -44,7 +45,7 @@ public void testDefault() throws IOException {
             assertTokenStreamContents(tokenFilter.create(tokenizer), expected);
         }
         {
-            TokenFilterFactory tokenFilter = analysisService.tokenFilter("limit");
+            TokenFilterFactory tokenFilter = analysis.tokenFilter.get("limit");
             String source = "the quick brown fox";
             String[] expected = new String[] { "the" };
             Tokenizer tokenizer = new WhitespaceTokenizer();
@@ -61,8 +62,8 @@ public void testSettings() throws IOException {
                     .put("index.analysis.filter.limit_1.consume_all_tokens", true)
                     .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
                     .build();
-            AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(settings);
-            TokenFilterFactory tokenFilter = analysisService.tokenFilter("limit_1");
+            ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(settings);
+            TokenFilterFactory tokenFilter = analysis.tokenFilter.get("limit_1");
             String source = "the quick brown fox";
             String[] expected = new String[] { "the", "quick", "brown" };
             Tokenizer tokenizer = new WhitespaceTokenizer();
@@ -76,8 +77,8 @@ public void testSettings() throws IOException {
                     .put("index.analysis.filter.limit_1.consume_all_tokens", false)
                     .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
                     .build();
-            AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(settings);
-            TokenFilterFactory tokenFilter = analysisService.tokenFilter("limit_1");
+            ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(settings);
+            TokenFilterFactory tokenFilter = analysis.tokenFilter.get("limit_1");
             String source = "the quick brown fox";
             String[] expected = new String[] { "the", "quick", "brown" };
             Tokenizer tokenizer = new WhitespaceTokenizer();
@@ -92,8 +93,8 @@ public void testSettings() throws IOException {
                     .put("index.analysis.filter.limit_1.consume_all_tokens", true)
                     .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
                     .build();
-            AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(settings);
-            TokenFilterFactory tokenFilter = analysisService.tokenFilter("limit_1");
+            ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(settings);
+            TokenFilterFactory tokenFilter = analysis.tokenFilter.get("limit_1");
             String source = "the quick brown fox";
             String[] expected = new String[] { "the", "quick", "brown", "fox" };
             Tokenizer tokenizer = new WhitespaceTokenizer();
diff --git a/core/src/test/java/org/elasticsearch/index/analysis/MinHashFilterFactoryTests.java b/core/src/test/java/org/elasticsearch/index/analysis/MinHashFilterFactoryTests.java
index 60f01cac70051..fc78afa7ab9cc 100644
--- a/core/src/test/java/org/elasticsearch/index/analysis/MinHashFilterFactoryTests.java
+++ b/core/src/test/java/org/elasticsearch/index/analysis/MinHashFilterFactoryTests.java
@@ -23,6 +23,7 @@
 import org.apache.lucene.analysis.core.WhitespaceTokenizer;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.ESTokenStreamTestCase;
 
 import java.io.IOException;
@@ -36,8 +37,8 @@ public void testDefault() throws IOException {
         Settings settings = Settings.builder()
             .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
             .build();
-        AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(settings);
-        TokenFilterFactory tokenFilter = analysisService.tokenFilter("min_hash");
+        ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(settings);
+        TokenFilterFactory tokenFilter = analysis.tokenFilter.get("min_hash");
         String source = "the quick brown fox";
         Tokenizer tokenizer = new WhitespaceTokenizer();
         tokenizer.setReader(new StringReader(source));
@@ -57,8 +58,8 @@ public void testSettings() throws IOException {
             .put("index.analysis.filter.test_min_hash.with_rotation", false)
             .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
             .build();
-        AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(settings);
-        TokenFilterFactory tokenFilter = analysisService.tokenFilter("test_min_hash");
+        ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(settings);
+        TokenFilterFactory tokenFilter = analysis.tokenFilter.get("test_min_hash");
         String source = "sushi";
         Tokenizer tokenizer = new WhitespaceTokenizer();
         tokenizer.setReader(new StringReader(source));
diff --git a/core/src/test/java/org/elasticsearch/index/analysis/PatternAnalyzerTests.java b/core/src/test/java/org/elasticsearch/index/analysis/PatternAnalyzerTests.java
index ab0a24d9dd82e..d80cbf66c34d0 100644
--- a/core/src/test/java/org/elasticsearch/index/analysis/PatternAnalyzerTests.java
+++ b/core/src/test/java/org/elasticsearch/index/analysis/PatternAnalyzerTests.java
@@ -21,6 +21,7 @@
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.core.StopAnalyzer;
+import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.test.ESTokenStreamTestCase;
 
 import java.io.IOException;
@@ -111,4 +112,11 @@ public void testRandomStrings() throws Exception {
     Analyzer a = new PatternAnalyzer(Pattern.compile(","), true, StopAnalyzer.ENGLISH_STOP_WORDS_SET);
     checkRandomData(random(), a, 10000*RANDOM_MULTIPLIER);
   }
+
+  public void testNormalize() {
+      PatternAnalyzer a = new PatternAnalyzer(Pattern.compile("\\s+"), false, null);
+      assertEquals(new BytesRef("FooBar"), a.normalize("dummy", "FooBar"));
+      a = new PatternAnalyzer(Pattern.compile("\\s+"), true, null);
+      assertEquals(new BytesRef("foobar"), a.normalize("dummy", "FooBar"));
+  }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/analysis/PatternCaptureTokenFilterTests.java b/core/src/test/java/org/elasticsearch/index/analysis/PatternCaptureTokenFilterTests.java
index caefb1039c296..126bbe2ab93ce 100644
--- a/core/src/test/java/org/elasticsearch/index/analysis/PatternCaptureTokenFilterTests.java
+++ b/core/src/test/java/org/elasticsearch/index/analysis/PatternCaptureTokenFilterTests.java
@@ -27,7 +27,7 @@
 import org.elasticsearch.test.ESTokenStreamTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 
-import static org.elasticsearch.test.ESTestCase.createAnalysisService;
+import static org.elasticsearch.test.ESTestCase.createTestAnalysis;
 import static org.hamcrest.Matchers.containsString;
 
 public class PatternCaptureTokenFilterTests extends ESTokenStreamTestCase {
@@ -40,17 +40,16 @@ public void testPatternCaptureTokenFilter() throws Exception {
                 .build();
 
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
-        AnalysisService analysisService = createAnalysisService(idxSettings, settings);
-
-        NamedAnalyzer analyzer1 = analysisService.analyzer("single");
+        IndexAnalyzers indexAnalyzers = createTestAnalysis(idxSettings, settings).indexAnalyzers;
+        NamedAnalyzer analyzer1 = indexAnalyzers.get("single");
 
         assertTokenStreamContents(analyzer1.tokenStream("test", "foobarbaz"), new String[]{"foobarbaz","foobar","foo"});
 
-        NamedAnalyzer analyzer2 = analysisService.analyzer("multi");
+        NamedAnalyzer analyzer2 = indexAnalyzers.get("multi");
 
         assertTokenStreamContents(analyzer2.tokenStream("test", "abc123def"), new String[]{"abc123def","abc","123","def"});
 
-        NamedAnalyzer analyzer3 = analysisService.analyzer("preserve");
+        NamedAnalyzer analyzer3 = indexAnalyzers.get("preserve");
 
         assertTokenStreamContents(analyzer3.tokenStream("test", "foobarbaz"), new String[]{"foobar","foo"});
     }
diff --git a/core/src/test/java/org/elasticsearch/index/analysis/ShingleTokenFilterFactoryTests.java b/core/src/test/java/org/elasticsearch/index/analysis/ShingleTokenFilterFactoryTests.java
index f33ddc88cf4e5..3a66f417c4713 100644
--- a/core/src/test/java/org/elasticsearch/index/analysis/ShingleTokenFilterFactoryTests.java
+++ b/core/src/test/java/org/elasticsearch/index/analysis/ShingleTokenFilterFactoryTests.java
@@ -26,6 +26,8 @@
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.core.WhitespaceTokenizer;
+import org.apache.lucene.analysis.miscellaneous.DisableGraphAttribute;
+import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.ESTokenStreamTestCase;
 
 import java.io.IOException;
@@ -38,8 +40,8 @@ public class ShingleTokenFilterFactoryTests extends ESTokenStreamTestCase {
     private static final String RESOURCE = "/org/elasticsearch/index/analysis/shingle_analysis.json";
 
     public void testDefault() throws IOException {
-        AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromClassPath(createTempDir(), RESOURCE);
-        TokenFilterFactory tokenFilter = analysisService.tokenFilter("shingle");
+        ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromClassPath(createTempDir(), RESOURCE);
+        TokenFilterFactory tokenFilter = analysis.tokenFilter.get("shingle");
         String source = "the quick brown fox";
         String[] expected = new String[]{"the", "the quick", "quick", "quick brown", "brown", "brown fox", "fox"};
         Tokenizer tokenizer = new WhitespaceTokenizer();
@@ -48,8 +50,8 @@ public void testDefault() throws IOException {
     }
 
     public void testInverseMapping() throws IOException {
-        AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromClassPath(createTempDir(), RESOURCE);
-        TokenFilterFactory tokenFilter = analysisService.tokenFilter("shingle_inverse");
+        ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromClassPath(createTempDir(), RESOURCE);
+        TokenFilterFactory tokenFilter = analysis.tokenFilter.get("shingle_inverse");
         assertThat(tokenFilter, instanceOf(ShingleTokenFilterFactory.class));
         String source = "the quick brown fox";
         String[] expected = new String[]{"the_quick_brown", "quick_brown_fox"};
@@ -59,8 +61,8 @@ public void testInverseMapping() throws IOException {
     }
 
     public void testInverseMappingNoShingles() throws IOException {
-        AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromClassPath(createTempDir(), RESOURCE);
-        TokenFilterFactory tokenFilter = analysisService.tokenFilter("shingle_inverse");
+        ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromClassPath(createTempDir(), RESOURCE);
+        TokenFilterFactory tokenFilter = analysis.tokenFilter.get("shingle_inverse");
         assertThat(tokenFilter, instanceOf(ShingleTokenFilterFactory.class));
         String source = "the quick";
         String[] expected = new String[]{"the", "quick"};
@@ -70,8 +72,8 @@ public void testInverseMappingNoShingles() throws IOException {
     }
 
     public void testFillerToken() throws IOException {
-        AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromClassPath(createTempDir(), RESOURCE);
-        TokenFilterFactory tokenFilter = analysisService.tokenFilter("shingle_filler");
+        ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromClassPath(createTempDir(), RESOURCE);
+        TokenFilterFactory tokenFilter = analysis.tokenFilter.get("shingle_filler");
         String source = "simon the sorcerer";
         String[] expected = new String[]{"simon FILLER", "simon FILLER sorcerer", "FILLER sorcerer"};
         Tokenizer tokenizer = new WhitespaceTokenizer();
@@ -79,4 +81,33 @@ public void testFillerToken() throws IOException {
         TokenStream stream = new StopFilter(tokenizer, StopFilter.makeStopSet("the"));
         assertTokenStreamContents(tokenFilter.create(stream), expected);
     }
+
+    public void testDisableGraph() throws IOException {
+        ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromClassPath(createTempDir(), RESOURCE);
+        TokenFilterFactory shingle = analysis.tokenFilter.get("shingle");
+        TokenFilterFactory shingleFiller = analysis.tokenFilter.get("shingle_filler");
+        TokenFilterFactory shingleInverse = analysis.tokenFilter.get("shingle_inverse");
+
+        String source = "hello world";
+        Tokenizer tokenizer = new WhitespaceTokenizer();
+        tokenizer.setReader(new StringReader(source));
+        try (TokenStream stream = shingle.create(tokenizer)) {
+            // The default config outputs unigram so graph analysis is disabled
+            assertTrue(stream.hasAttribute(DisableGraphAttribute.class));
+        }
+
+        tokenizer = new WhitespaceTokenizer();
+        tokenizer.setReader(new StringReader(source));
+        try (TokenStream stream = shingleInverse.create(tokenizer)) {
+            // This config uses a single size of shingles so graph analysis is enabled
+            assertFalse(stream.hasAttribute(DisableGraphAttribute.class));
+        }
+
+        tokenizer = new WhitespaceTokenizer();
+        tokenizer.setReader(new StringReader(source));
+        try (TokenStream stream = shingleFiller.create(tokenizer)) {
+            // This config uses a single size of shingles so graph analysis is enabled
+            assertTrue(stream.hasAttribute(DisableGraphAttribute.class));
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/analysis/StemmerTokenFilterFactoryTests.java b/core/src/test/java/org/elasticsearch/index/analysis/StemmerTokenFilterFactoryTests.java
index a414f412767e4..7c4818c63b32a 100644
--- a/core/src/test/java/org/elasticsearch/index/analysis/StemmerTokenFilterFactoryTests.java
+++ b/core/src/test/java/org/elasticsearch/index/analysis/StemmerTokenFilterFactoryTests.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.ESTokenStreamTestCase;
 import org.elasticsearch.test.VersionUtils;
 
@@ -53,13 +54,14 @@ public void testEnglishFilterFactory() throws IOException {
                     .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
                     .build();
 
-            AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(settings);
-            TokenFilterFactory tokenFilter = analysisService.tokenFilter("my_english");
+            ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(settings);
+            TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_english");
             assertThat(tokenFilter, instanceOf(StemmerTokenFilterFactory.class));
             Tokenizer tokenizer = new WhitespaceTokenizer();
             tokenizer.setReader(new StringReader("foo bar"));
             TokenStream create = tokenFilter.create(tokenizer);
-            NamedAnalyzer analyzer = analysisService.analyzer("my_english");
+            IndexAnalyzers indexAnalyzers = analysis.indexAnalyzers;
+            NamedAnalyzer analyzer = indexAnalyzers.get("my_english");
             assertThat(create, instanceOf(PorterStemFilter.class));
             assertAnalyzesTo(analyzer, "consolingly", new String[]{"consolingli"});
         }
@@ -80,13 +82,14 @@ public void testPorter2FilterFactory() throws IOException {
                     .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
                     .build();
 
-            AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(settings);
-            TokenFilterFactory tokenFilter = analysisService.tokenFilter("my_porter2");
+            ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(settings);
+            TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_porter2");
             assertThat(tokenFilter, instanceOf(StemmerTokenFilterFactory.class));
             Tokenizer tokenizer = new WhitespaceTokenizer();
             tokenizer.setReader(new StringReader("foo bar"));
             TokenStream create = tokenFilter.create(tokenizer);
-            NamedAnalyzer analyzer = analysisService.analyzer("my_porter2");
+            IndexAnalyzers indexAnalyzers = analysis.indexAnalyzers;
+            NamedAnalyzer analyzer = indexAnalyzers.get("my_porter2");
             assertThat(create, instanceOf(SnowballFilter.class));
             assertAnalyzesTo(analyzer, "possibly", new String[]{"possibl"});
         }
diff --git a/core/src/test/java/org/elasticsearch/index/analysis/StopAnalyzerTests.java b/core/src/test/java/org/elasticsearch/index/analysis/StopAnalyzerTests.java
index 88c5fe692d61f..e166f4b7b9e30 100644
--- a/core/src/test/java/org/elasticsearch/index/analysis/StopAnalyzerTests.java
+++ b/core/src/test/java/org/elasticsearch/index/analysis/StopAnalyzerTests.java
@@ -27,7 +27,7 @@
 import org.elasticsearch.test.ESTokenStreamTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 
-import static org.elasticsearch.test.ESTestCase.createAnalysisService;
+import static org.elasticsearch.test.ESTestCase.createTestAnalysis;
 
 public class StopAnalyzerTests extends ESTokenStreamTestCase {
     public void testDefaultsCompoundAnalysis() throws Exception {
@@ -38,13 +38,12 @@ public void testDefaultsCompoundAnalysis() throws Exception {
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
                 .build();
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
-        AnalysisService analysisService = createAnalysisService(idxSettings, settings);
-
-        NamedAnalyzer analyzer1 = analysisService.analyzer("analyzer1");
+        IndexAnalyzers indexAnalyzers = createTestAnalysis(idxSettings, settings).indexAnalyzers;
+        NamedAnalyzer analyzer1 = indexAnalyzers.get("analyzer1");
 
         assertTokenStreamContents(analyzer1.tokenStream("test", "to be or not to be"), new String[0]);
 
-        NamedAnalyzer analyzer2 = analysisService.analyzer("analyzer2");
+        NamedAnalyzer analyzer2 = indexAnalyzers.get("analyzer2");
 
         assertTokenStreamContents(analyzer2.tokenStream("test", "to be or not to be"), new String[0]);
     }
diff --git a/core/src/test/java/org/elasticsearch/index/analysis/StopTokenFilterTests.java b/core/src/test/java/org/elasticsearch/index/analysis/StopTokenFilterTests.java
index 07b00f30e2073..7d91619075b3a 100644
--- a/core/src/test/java/org/elasticsearch/index/analysis/StopTokenFilterTests.java
+++ b/core/src/test/java/org/elasticsearch/index/analysis/StopTokenFilterTests.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.common.settings.Settings.Builder;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.ESTokenStreamTestCase;
 
 import java.io.IOException;
@@ -47,7 +48,7 @@ public void testPositionIncrementSetting() throws IOException {
         builder.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString());
         Settings settings = builder.build();
         try {
-            AnalysisTestsHelper.createAnalysisServiceFromSettings(settings);
+            AnalysisTestsHelper.createTestAnalysisFromSettings(settings);
             fail("Expected IllegalArgumentException");
         } catch (IllegalArgumentException e) {
             assertThat(e.getMessage(), containsString("enable_position_increments is not supported anymore"));
@@ -62,8 +63,8 @@ public void testCorrectPositionIncrementSetting() throws IOException {
             // don't specify
         }
         builder.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString());
-        AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(builder.build());
-        TokenFilterFactory tokenFilter = analysisService.tokenFilter("my_stop");
+        ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(builder.build());
+        TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_stop");
         assertThat(tokenFilter, instanceOf(StopTokenFilterFactory.class));
         Tokenizer tokenizer = new WhitespaceTokenizer();
         tokenizer.setReader(new StringReader("foo bar"));
@@ -77,8 +78,8 @@ public void testThatSuggestStopFilterWorks() throws Exception {
                 .put("index.analysis.filter.my_stop.remove_trailing", false)
                 .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
                 .build();
-        AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(settings);
-        TokenFilterFactory tokenFilter = analysisService.tokenFilter("my_stop");
+        ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(settings);
+        TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_stop");
         assertThat(tokenFilter, instanceOf(StopTokenFilterFactory.class));
         Tokenizer tokenizer = new WhitespaceTokenizer();
         tokenizer.setReader(new StringReader("foo an"));
diff --git a/core/src/test/java/org/elasticsearch/index/analysis/WordDelimiterGraphTokenFilterFactoryTests.java b/core/src/test/java/org/elasticsearch/index/analysis/WordDelimiterGraphTokenFilterFactoryTests.java
new file mode 100644
index 0000000000000..2ae4267104a05
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/analysis/WordDelimiterGraphTokenFilterFactoryTests.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.index.analysis;
+
+
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.core.WhitespaceTokenizer;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.io.StringReader;
+
+public class WordDelimiterGraphTokenFilterFactoryTests extends BaseWordDelimiterTokenFilterFactoryTestCase {
+    public WordDelimiterGraphTokenFilterFactoryTests() {
+        super("word_delimiter_graph");
+    }
+
+    public void testMultiTerms() throws IOException {
+        ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(Settings.builder()
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+            .put("index.analysis.filter.my_word_delimiter.type", type)
+            .put("index.analysis.filter.my_word_delimiter.catenate_all", "true")
+            .put("index.analysis.filter.my_word_delimiter.preserve_original", "true")
+            .build());
+
+        TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_word_delimiter");
+        String source = "PowerShot 500-42 wi-fi wi-fi-4000 j2se O'Neil's";
+        String[] expected = new String[]{"PowerShot", "PowerShot", "Power", "Shot", "50042", "500-42", "500", "42",
+            "wifi", "wi-fi", "wi", "fi", "wifi4000", "wi-fi-4000", "wi", "fi", "4000", "j2se", "j2se", "j", "2", "se",
+            "ONeil", "O'Neil's", "O", "Neil" };
+        Tokenizer tokenizer = new WhitespaceTokenizer();
+        tokenizer.setReader(new StringReader(source));
+        int[] expectedIncr = new int[]{1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1};
+        int[] expectedPosLen = new int[]{2, 2, 1, 1, 2, 2, 1, 1, 2, 2, 1, 1, 3, 3, 1, 1, 1, 3, 3, 1, 1, 1, 2, 2, 1, 1};
+        assertTokenStreamContents(tokenFilter.create(tokenizer), expected, null, null, null,
+                expectedIncr, expectedPosLen, null);
+    }
+
+    /** Correct offset order when doing both parts and concatenation: PowerShot is a synonym of Power */
+    public void testPartsAndCatenate() throws IOException {
+        ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(Settings.builder()
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+            .put("index.analysis.filter.my_word_delimiter.type", type)
+            .put("index.analysis.filter.my_word_delimiter.catenate_words", "true")
+            .put("index.analysis.filter.my_word_delimiter.generate_word_parts", "true")
+            .build());
+        TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_word_delimiter");
+        String source = "PowerShot";
+        int[] expectedIncr = new int[]{1, 0, 1};
+        int[] expectedPosLen = new int[]{2, 1, 1};
+        String[] expected = new String[]{"PowerShot", "Power", "Shot" };
+        Tokenizer tokenizer = new WhitespaceTokenizer();
+        tokenizer.setReader(new StringReader(source));
+        assertTokenStreamContents(tokenFilter.create(tokenizer), expected, null, null, null,
+            expectedIncr, expectedPosLen, null);
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/analysis/WordDelimiterTokenFilterFactoryTests.java b/core/src/test/java/org/elasticsearch/index/analysis/WordDelimiterTokenFilterFactoryTests.java
index f94252caba42e..1e919e00bbb29 100644
--- a/core/src/test/java/org/elasticsearch/index/analysis/WordDelimiterTokenFilterFactoryTests.java
+++ b/core/src/test/java/org/elasticsearch/index/analysis/WordDelimiterTokenFilterFactoryTests.java
@@ -23,122 +23,25 @@
 import org.apache.lucene.analysis.core.WhitespaceTokenizer;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
-import org.elasticsearch.test.ESTokenStreamTestCase;
+import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
 import java.io.StringReader;
 
-public class WordDelimiterTokenFilterFactoryTests extends ESTokenStreamTestCase {
-    public void testDefault() throws IOException {
-        AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(Settings.builder()
-                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
-                .put("index.analysis.filter.my_word_delimiter.type", "word_delimiter")
-                .build());
-        TokenFilterFactory tokenFilter = analysisService.tokenFilter("my_word_delimiter");
-        String source = "PowerShot 500-42 wi-fi wi-fi-4000 j2se O'Neil's";
-        String[] expected = new String[]{"Power", "Shot", "500", "42", "wi", "fi", "wi", "fi", "4000", "j", "2", "se", "O", "Neil"};
-        Tokenizer tokenizer = new WhitespaceTokenizer();
-        tokenizer.setReader(new StringReader(source));
-        assertTokenStreamContents(tokenFilter.create(tokenizer), expected);
-    }
-
-    public void testCatenateWords() throws IOException {
-        AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(Settings.builder()
-                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
-                .put("index.analysis.filter.my_word_delimiter.type", "word_delimiter")
-                .put("index.analysis.filter.my_word_delimiter.catenate_words", "true")
-                .put("index.analysis.filter.my_word_delimiter.generate_word_parts", "false")
-                .build());
-        TokenFilterFactory tokenFilter = analysisService.tokenFilter("my_word_delimiter");
-        String source = "PowerShot 500-42 wi-fi wi-fi-4000 j2se O'Neil's";
-        String[] expected = new String[]{"PowerShot", "500", "42", "wifi", "wifi", "4000", "j", "2", "se", "ONeil"};
-        Tokenizer tokenizer = new WhitespaceTokenizer();
-        tokenizer.setReader(new StringReader(source));
-        assertTokenStreamContents(tokenFilter.create(tokenizer), expected);
-    }
-
-    public void testCatenateNumbers() throws IOException {
-        AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(Settings.builder()
-                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
-                .put("index.analysis.filter.my_word_delimiter.type", "word_delimiter")
-                .put("index.analysis.filter.my_word_delimiter.generate_number_parts", "false")
-                .put("index.analysis.filter.my_word_delimiter.catenate_numbers", "true")
-                .build());
-        TokenFilterFactory tokenFilter = analysisService.tokenFilter("my_word_delimiter");
-        String source = "PowerShot 500-42 wi-fi wi-fi-4000 j2se O'Neil's";
-        String[] expected = new String[]{"Power", "Shot", "50042", "wi", "fi", "wi", "fi", "4000", "j", "2", "se", "O", "Neil"};
-        Tokenizer tokenizer = new WhitespaceTokenizer();
-        tokenizer.setReader(new StringReader(source));
-        assertTokenStreamContents(tokenFilter.create(tokenizer), expected);
-    }
-
-    public void testCatenateAll() throws IOException {
-        AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(Settings.builder()
-                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
-                .put("index.analysis.filter.my_word_delimiter.type", "word_delimiter")
-                .put("index.analysis.filter.my_word_delimiter.generate_word_parts", "false")
-                .put("index.analysis.filter.my_word_delimiter.generate_number_parts", "false")
-                .put("index.analysis.filter.my_word_delimiter.catenate_all", "true")
-                .build());
-        TokenFilterFactory tokenFilter = analysisService.tokenFilter("my_word_delimiter");
-        String source = "PowerShot 500-42 wi-fi wi-fi-4000 j2se O'Neil's";
-        String[] expected = new String[]{"PowerShot", "50042", "wifi", "wifi4000", "j2se", "ONeil"};
-        Tokenizer tokenizer = new WhitespaceTokenizer();
-        tokenizer.setReader(new StringReader(source));
-        assertTokenStreamContents(tokenFilter.create(tokenizer), expected);
-    }
-
-    public void testSplitOnCaseChange() throws IOException {
-        AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(Settings.builder()
-                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
-                .put("index.analysis.filter.my_word_delimiter.type", "word_delimiter")
-                .put("index.analysis.filter.my_word_delimiter.split_on_case_change", "false")
-                .build());
-        TokenFilterFactory tokenFilter = analysisService.tokenFilter("my_word_delimiter");
-        String source = "PowerShot";
-        String[] expected = new String[]{"PowerShot"};
-        Tokenizer tokenizer = new WhitespaceTokenizer();
-        tokenizer.setReader(new StringReader(source));
-        assertTokenStreamContents(tokenFilter.create(tokenizer), expected);
-    }
-
-    public void testPreserveOriginal() throws IOException {
-        AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(Settings.builder()
-                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
-                .put("index.analysis.filter.my_word_delimiter.type", "word_delimiter")
-                .put("index.analysis.filter.my_word_delimiter.preserve_original", "true")
-                .build());
-        TokenFilterFactory tokenFilter = analysisService.tokenFilter("my_word_delimiter");
-        String source = "PowerShot 500-42 wi-fi wi-fi-4000 j2se O'Neil's";
-        String[] expected = new String[]{"PowerShot", "Power", "Shot", "500-42", "500", "42", "wi-fi", "wi", "fi", "wi-fi-4000", "wi", "fi", "4000", "j2se", "j", "2", "se", "O'Neil's", "O", "Neil"};
-        Tokenizer tokenizer = new WhitespaceTokenizer();
-        tokenizer.setReader(new StringReader(source));
-        assertTokenStreamContents(tokenFilter.create(tokenizer), expected);
-    }
-
-    public void testStemEnglishPossessive() throws IOException {
-        AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(Settings.builder()
-                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
-                .put("index.analysis.filter.my_word_delimiter.type", "word_delimiter")
-                .put("index.analysis.filter.my_word_delimiter.stem_english_possessive", "false")
-                .build());
-        TokenFilterFactory tokenFilter = analysisService.tokenFilter("my_word_delimiter");
-        String source = "PowerShot 500-42 wi-fi wi-fi-4000 j2se O'Neil's";
-        String[] expected = new String[]{"Power", "Shot", "500", "42", "wi", "fi", "wi", "fi", "4000", "j", "2", "se", "O", "Neil", "s"};
-        Tokenizer tokenizer = new WhitespaceTokenizer();
-        tokenizer.setReader(new StringReader(source));
-        assertTokenStreamContents(tokenFilter.create(tokenizer), expected);
+public class WordDelimiterTokenFilterFactoryTests extends BaseWordDelimiterTokenFilterFactoryTestCase {
+    public WordDelimiterTokenFilterFactoryTests() {
+        super("word_delimiter");
     }
 
     /** Correct offset order when doing both parts and concatenation: PowerShot is a synonym of Power */
     public void testPartsAndCatenate() throws IOException {
-        AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(Settings.builder()
-                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
-                .put("index.analysis.filter.my_word_delimiter.type", "word_delimiter")
-                .put("index.analysis.filter.my_word_delimiter.catenate_words", "true")
-                .put("index.analysis.filter.my_word_delimiter.generate_word_parts", "true")
-                .build());
-        TokenFilterFactory tokenFilter = analysisService.tokenFilter("my_word_delimiter");
+        ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(Settings.builder()
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+            .put("index.analysis.filter.my_word_delimiter.type", type)
+            .put("index.analysis.filter.my_word_delimiter.catenate_words", "true")
+            .put("index.analysis.filter.my_word_delimiter.generate_word_parts", "true")
+            .build());
+        TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_word_delimiter");
         String source = "PowerShot";
         String[] expected = new String[]{"Power", "PowerShot", "Shot" };
         Tokenizer tokenizer = new WhitespaceTokenizer();
diff --git a/core/src/test/java/org/elasticsearch/index/analysis/commongrams/CommonGramsTokenFilterFactoryTests.java b/core/src/test/java/org/elasticsearch/index/analysis/commongrams/CommonGramsTokenFilterFactoryTests.java
index 3c192052db51b..ed6866e6a81f3 100644
--- a/core/src/test/java/org/elasticsearch/index/analysis/commongrams/CommonGramsTokenFilterFactoryTests.java
+++ b/core/src/test/java/org/elasticsearch/index/analysis/commongrams/CommonGramsTokenFilterFactoryTests.java
@@ -24,9 +24,10 @@
 import org.apache.lucene.analysis.core.WhitespaceTokenizer;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
-import org.elasticsearch.index.analysis.AnalysisService;
 import org.elasticsearch.index.analysis.AnalysisTestsHelper;
+import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.analysis.TokenFilterFactory;
+import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.ESTokenStreamTestCase;
 import org.junit.Assert;
 
@@ -43,7 +44,7 @@ public void testDefault() throws IOException {
                                 .build();
 
         try {
-            AnalysisTestsHelper.createAnalysisServiceFromSettings(settings);
+            AnalysisTestsHelper.createTestAnalysisFromSettings(settings);
             Assert.fail("[common_words] or [common_words_path] is set");
         } catch (IllegalArgumentException e) {
         } catch (IOException e) {
@@ -58,9 +59,9 @@ public void testWithoutCommonWordsMatch() throws IOException {
                      .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
                      .build();
 
-            AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(settings);
+            ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(settings);
             {
-                TokenFilterFactory tokenFilter = analysisService.tokenFilter("common_grams_default");
+                TokenFilterFactory tokenFilter = analysis.tokenFilter.get("common_grams_default");
                 String source = "the quick brown is a fox Or noT";
                 String[] expected = new String[] { "the", "quick", "brown", "is", "a", "fox", "Or", "noT" };
                 Tokenizer tokenizer = new WhitespaceTokenizer();
@@ -75,9 +76,9 @@ public void testWithoutCommonWordsMatch() throws IOException {
                      .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
                      .putArray("index.analysis.filter.common_grams_default.common_words", "chromosome", "protein")
                      .build();
-            AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(settings);
+            ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(settings);
             {
-                TokenFilterFactory tokenFilter = analysisService.tokenFilter("common_grams_default");
+                TokenFilterFactory tokenFilter = analysis.tokenFilter.get("common_grams_default");
                 String source = "the quick brown is a fox Or noT";
                 String[] expected = new String[] { "the", "quick", "brown", "is", "a", "fox", "Or", "noT" };
                 Tokenizer tokenizer = new WhitespaceTokenizer();
@@ -94,8 +95,8 @@ public void testSettings() throws IOException {
                     .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
                     .putArray("index.analysis.filter.common_grams_1.common_words", "the", "Or", "Not", "a", "is", "an", "they", "are")
                     .build();
-            AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(settings);
-            TokenFilterFactory tokenFilter = analysisService.tokenFilter("common_grams_1");
+            ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(settings);
+            TokenFilterFactory tokenFilter = analysis.tokenFilter.get("common_grams_1");
             String source = "the quick brown is a fox or noT";
             String[] expected = new String[] { "the", "the_quick", "quick", "brown", "brown_is", "is", "is_a", "a", "a_fox", "fox", "fox_or", "or", "or_noT", "noT" };
             Tokenizer tokenizer = new WhitespaceTokenizer();
@@ -108,8 +109,8 @@ public void testSettings() throws IOException {
                     .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
                     .putArray("index.analysis.filter.common_grams_2.common_words", "the", "Or", "noT", "a", "is", "an", "they", "are")
                     .build();
-            AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(settings);
-            TokenFilterFactory tokenFilter = analysisService.tokenFilter("common_grams_2");
+            ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(settings);
+            TokenFilterFactory tokenFilter = analysis.tokenFilter.get("common_grams_2");
             String source = "the quick brown is a fox or why noT";
             String[] expected = new String[] { "the", "the_quick", "quick", "brown", "brown_is", "is", "is_a", "a", "a_fox", "fox", "or", "why", "why_noT", "noT" };
             Tokenizer tokenizer = new WhitespaceTokenizer();
@@ -121,8 +122,8 @@ public void testSettings() throws IOException {
                     .putArray("index.analysis.filter.common_grams_3.common_words", "the", "or", "not", "a", "is", "an", "they", "are")
                     .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
                     .build();
-            AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(settings);
-            TokenFilterFactory tokenFilter = analysisService.tokenFilter("common_grams_3");
+            ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(settings);
+            TokenFilterFactory tokenFilter = analysis.tokenFilter.get("common_grams_3");
             String source = "the quick brown is a fox Or noT";
             String[] expected = new String[] { "the", "the_quick", "quick", "brown", "brown_is", "is", "is_a", "a", "a_fox", "fox", "Or", "noT" };
             Tokenizer tokenizer = new WhitespaceTokenizer();
@@ -138,15 +139,17 @@ public void testCommonGramsAnalysis() throws IOException {
                      .put(Environment.PATH_HOME_SETTING.getKey(), createHome())
                      .build();
         {
-            AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(settings);
-            Analyzer analyzer = analysisService.analyzer("commongramsAnalyzer").analyzer();
+            IndexAnalyzers indexAnalyzers = AnalysisTestsHelper.createTestAnalysisFromSettings(settings)
+                .indexAnalyzers;
+            Analyzer analyzer = indexAnalyzers.get("commongramsAnalyzer").analyzer();
             String source = "the quick brown is a fox or not";
             String[] expected = new String[] { "the", "quick", "quick_brown", "brown", "brown_is", "is", "a", "a_fox", "fox", "fox_or", "or", "not" };
             assertTokenStreamContents(analyzer.tokenStream("test", source), expected);
         }
         {
-            AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(settings);
-            Analyzer analyzer = analysisService.analyzer("commongramsAnalyzer_file").analyzer();
+            IndexAnalyzers indexAnalyzers = AnalysisTestsHelper.createTestAnalysisFromSettings(settings)
+                .indexAnalyzers;
+            Analyzer analyzer = indexAnalyzers.get("commongramsAnalyzer_file").analyzer();
             String source = "the quick brown is a fox or not";
             String[] expected = new String[] { "the", "quick", "quick_brown", "brown", "brown_is", "is", "a", "a_fox", "fox", "fox_or", "or", "not" };
             assertTokenStreamContents(analyzer.tokenStream("test", source), expected);
@@ -161,8 +164,8 @@ public void testQueryModeSettings() throws IOException {
                     .put("index.analysis.filter.common_grams_1.ignore_case", true)
                     .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
                     .build();
-            AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(settings);
-            TokenFilterFactory tokenFilter = analysisService.tokenFilter("common_grams_1");
+            ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(settings);
+            TokenFilterFactory tokenFilter = analysis.tokenFilter.get("common_grams_1");
             String source = "the quick brown is a fox or noT";
             String[] expected = new String[] { "the_quick", "quick", "brown_is", "is_a", "a_fox", "fox_or", "or_noT" };
             Tokenizer tokenizer = new WhitespaceTokenizer();
@@ -176,8 +179,8 @@ public void testQueryModeSettings() throws IOException {
                     .put("index.analysis.filter.common_grams_2.ignore_case", false)
                     .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
                     .build();
-            AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(settings);
-            TokenFilterFactory tokenFilter = analysisService.tokenFilter("common_grams_2");
+            ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(settings);
+            TokenFilterFactory tokenFilter = analysis.tokenFilter.get("common_grams_2");
             String source = "the quick brown is a fox or why noT";
             String[] expected = new String[] { "the_quick", "quick", "brown_is", "is_a", "a_fox", "fox", "or", "why_noT" };
             Tokenizer tokenizer = new WhitespaceTokenizer();
@@ -190,8 +193,8 @@ public void testQueryModeSettings() throws IOException {
                     .putArray("index.analysis.filter.common_grams_3.common_words", "the", "Or", "noT", "a", "is", "an", "they", "are")
                     .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
                     .build();
-            AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(settings);
-            TokenFilterFactory tokenFilter = analysisService.tokenFilter("common_grams_3");
+            ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(settings);
+            TokenFilterFactory tokenFilter = analysis.tokenFilter.get("common_grams_3");
             String source = "the quick brown is a fox or why noT";
             String[] expected = new String[] { "the_quick", "quick", "brown_is", "is_a", "a_fox", "fox", "or", "why_noT" };
             Tokenizer tokenizer = new WhitespaceTokenizer();
@@ -204,8 +207,8 @@ public void testQueryModeSettings() throws IOException {
                     .putArray("index.analysis.filter.common_grams_4.common_words", "the", "or", "not", "a", "is", "an", "they", "are")
                     .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
                     .build();
-            AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(settings);
-            TokenFilterFactory tokenFilter = analysisService.tokenFilter("common_grams_4");
+            ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(settings);
+            TokenFilterFactory tokenFilter = analysis.tokenFilter.get("common_grams_4");
             String source = "the quick brown is a fox Or noT";
             String[] expected = new String[] { "the_quick", "quick", "brown_is", "is_a", "a_fox", "fox", "Or", "noT" };
             Tokenizer tokenizer = new WhitespaceTokenizer();
@@ -221,15 +224,17 @@ public void testQueryModeCommonGramsAnalysis() throws IOException {
             .put(Environment.PATH_HOME_SETTING.getKey(), createHome())
                 .build();
         {
-            AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(settings);
-            Analyzer analyzer = analysisService.analyzer("commongramsAnalyzer").analyzer();
+            IndexAnalyzers indexAnalyzers = AnalysisTestsHelper.createTestAnalysisFromSettings(settings)
+                .indexAnalyzers;
+            Analyzer analyzer = indexAnalyzers.get("commongramsAnalyzer").analyzer();
             String source = "the quick brown is a fox or not";
             String[] expected = new String[] { "the", "quick_brown", "brown_is", "is", "a_fox", "fox_or", "or", "not" };
             assertTokenStreamContents(analyzer.tokenStream("test", source), expected);
         }
         {
-            AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(settings);
-            Analyzer analyzer = analysisService.analyzer("commongramsAnalyzer_file").analyzer();
+            IndexAnalyzers indexAnalyzers = AnalysisTestsHelper.createTestAnalysisFromSettings(settings)
+                .indexAnalyzers;
+            Analyzer analyzer = indexAnalyzers.get("commongramsAnalyzer_file").analyzer();
             String source = "the quick brown is a fox or not";
             String[] expected = new String[] { "the", "quick_brown", "brown_is", "is", "a_fox", "fox_or", "or", "not" };
             assertTokenStreamContents(analyzer.tokenStream("test", source), expected);
diff --git a/core/src/test/java/org/elasticsearch/index/analysis/synonyms/SynonymsAnalysisTests.java b/core/src/test/java/org/elasticsearch/index/analysis/synonyms/SynonymsAnalysisTests.java
index de271d720c3a1..c6dfdc1a41366 100644
--- a/core/src/test/java/org/elasticsearch/index/analysis/synonyms/SynonymsAnalysisTests.java
+++ b/core/src/test/java/org/elasticsearch/index/analysis/synonyms/SynonymsAnalysisTests.java
@@ -30,7 +30,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.analysis.AnalysisService;
+import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 import org.hamcrest.MatcherAssert;
@@ -46,7 +46,7 @@
  */
 public class SynonymsAnalysisTests extends ESTestCase {
     protected final Logger logger = Loggers.getLogger(getClass());
-    private AnalysisService analysisService;
+    private IndexAnalyzers indexAnalyzers;
 
     public void testSynonymsAnalysis() throws IOException {
         InputStream synonyms = getClass().getResourceAsStream("synonyms.txt");
@@ -64,7 +64,7 @@ public void testSynonymsAnalysis() throws IOException {
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
 
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
-        analysisService = createAnalysisService(idxSettings, settings);
+        indexAnalyzers = createTestAnalysis(idxSettings, settings).indexAnalyzers;
 
         match("synonymAnalyzer", "kimchy is the dude abides", "shay is the elasticsearch man!");
         match("synonymAnalyzer_file", "kimchy is the dude abides", "shay is the elasticsearch man!");
@@ -74,8 +74,7 @@ public void testSynonymsAnalysis() throws IOException {
     }
 
     private void match(String analyzerName, String source, String target) throws IOException {
-
-        Analyzer analyzer = analysisService.analyzer(analyzerName).analyzer();
+        Analyzer analyzer = indexAnalyzers.get(analyzerName).analyzer();
 
         TokenStream stream = AllTokenStream.allTokenStream("_all", source, 1.0f, analyzer);
         stream.reset();
diff --git a/core/src/test/java/org/elasticsearch/index/codec/CodecTests.java b/core/src/test/java/org/elasticsearch/index/codec/CodecTests.java
index cf706f33cc259..e49e21bd2ff99 100644
--- a/core/src/test/java/org/elasticsearch/index/codec/CodecTests.java
+++ b/core/src/test/java/org/elasticsearch/index/codec/CodecTests.java
@@ -37,7 +37,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.analysis.AnalysisService;
+import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.indices.mapper.MapperRegistry;
@@ -89,15 +89,16 @@ private void assertCompressionEquals(Mode expected, Codec actual) throws Excepti
         dir.close();
     }
 
-    private static CodecService createCodecService() throws IOException {
+    private CodecService createCodecService() throws IOException {
         Settings nodeSettings = Settings.builder()
                 .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
                 .build();
         IndexSettings settings = IndexSettingsModule.newIndexSettings("_na", nodeSettings);
         SimilarityService similarityService = new SimilarityService(settings, Collections.emptyMap());
-        AnalysisService analysisService = createAnalysisService(settings, nodeSettings);
+        IndexAnalyzers indexAnalyzers = createTestAnalysis(settings, nodeSettings).indexAnalyzers;
         MapperRegistry mapperRegistry = new MapperRegistry(Collections.emptyMap(), Collections.emptyMap());
-        MapperService service = new MapperService(settings, analysisService, similarityService, mapperRegistry, () -> null);
+        MapperService service = new MapperService(settings, indexAnalyzers, xContentRegistry(), similarityService, mapperRegistry,
+                () -> null);
         return new CodecService(service, ESLoggerFactory.getLogger("test"));
     }
 
diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
index 7b62966515d1c..6946179ed42bc 100644
--- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
+++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
@@ -26,12 +26,18 @@
 import org.apache.logging.log4j.core.LogEvent;
 import org.apache.logging.log4j.core.appender.AbstractAppender;
 import org.apache.logging.log4j.core.filter.RegexFilter;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.StoredField;
 import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.KeepOnlyLastCommitDeletionPolicy;
 import org.apache.lucene.index.LiveIndexWriterConfig;
 import org.apache.lucene.index.LogByteSizeMergePolicy;
@@ -43,12 +49,15 @@
 import org.apache.lucene.index.TieredMergePolicy;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.ReferenceManager;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.search.TotalHitCountCollector;
 import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.Lock;
 import org.apache.lucene.store.MockDirectoryWrapper;
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.TestUtil;
 import org.elasticsearch.ElasticsearchException;
@@ -57,23 +66,29 @@
 import org.elasticsearch.action.support.TransportActions;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.logging.Loggers;
-import org.elasticsearch.common.logging.PrefixMessageFactory;
-import org.elasticsearch.common.logging.TestLoggers;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
+import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.VersionType;
-import org.elasticsearch.index.analysis.AnalysisService;
+import org.elasticsearch.index.analysis.AnalyzerScope;
+import org.elasticsearch.index.analysis.IndexAnalyzers;
+import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.codec.CodecService;
 import org.elasticsearch.index.engine.Engine.Searcher;
+import org.elasticsearch.index.fieldvisitor.FieldsVisitor;
 import org.elasticsearch.index.mapper.ContentPath;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.DocumentMapperForType;
@@ -85,8 +100,8 @@
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.RootObjectMapper;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
+import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.index.mapper.UidFieldMapper;
-import org.elasticsearch.index.shard.DocsStats;
 import org.elasticsearch.index.shard.IndexSearcherWrapper;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.ShardUtils;
@@ -102,6 +117,7 @@
 import org.elasticsearch.test.DummyShardLock;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
+import org.elasticsearch.test.OldIndexUtils;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.hamcrest.MatcherAssert;
@@ -110,6 +126,7 @@
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.UncheckedIOException;
 import java.nio.charset.Charset;
 import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
@@ -118,23 +135,34 @@
 import java.util.Arrays;
 import java.util.Base64;
 import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Queue;
+import java.util.Set;
 import java.util.concurrent.BrokenBarrierException;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.CyclicBarrier;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+import java.util.function.Supplier;
 
+import static java.util.Collections.emptyList;
 import static java.util.Collections.emptyMap;
+import static java.util.Collections.shuffle;
 import static org.elasticsearch.index.engine.Engine.Operation.Origin.PRIMARY;
 import static org.elasticsearch.index.engine.Engine.Operation.Origin.REPLICA;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.everyItem;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.hasKey;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
@@ -204,8 +232,12 @@ public void setUp() throws Exception {
     }
 
     public EngineConfig copy(EngineConfig config, EngineConfig.OpenMode openMode) {
+        return copy(config, openMode, config.getAnalyzer());
+    }
+
+    public EngineConfig copy(EngineConfig config, EngineConfig.OpenMode openMode, Analyzer analyzer) {
         return new EngineConfig(openMode, config.getShardId(), config.getThreadPool(), config.getIndexSettings(), config.getWarmer(),
-            config.getStore(), config.getDeletionPolicy(), config.getMergePolicy(), config.getAnalyzer(), config.getSimilarity(),
+            config.getStore(), config.getDeletionPolicy(), config.getMergePolicy(), analyzer, config.getSimilarity(),
             new CodecService(null, logger), config.getEventListener(), config.getTranslogRecoveryPerformer(), config.getQueryCache(),
             config.getQueryCachingPolicy(), config.getTranslogConfig(), config.getFlushMergesAfter(), config.getRefreshListeners(),
             config.getMaxUnsafeAutoIdTimestamp());
@@ -222,23 +254,34 @@ public void tearDown() throws Exception {
     }
 
 
-    private Document testDocumentWithTextField() {
+    private static Document testDocumentWithTextField() {
+       return testDocumentWithTextField("test");
+    }
+
+    private static Document testDocumentWithTextField(String value) {
         Document document = testDocument();
-        document.add(new TextField("value", "test", Field.Store.YES));
+        document.add(new TextField("value", value, Field.Store.YES));
         return document;
     }
 
-    private Document testDocument() {
+
+    private static Document testDocument() {
         return new Document();
     }
 
+    public static ParsedDocument createParsedDoc(String id, String type, String routing) {
+        return testParsedDocument(id, type, routing, -1, -1, testDocumentWithTextField(), new BytesArray("{ \"value\" : \"test\" }"), null);
+    }
 
-    private ParsedDocument testParsedDocument(String uid, String id, String type, String routing, long timestamp, long ttl, Document document, BytesReference source, Mapping mappingUpdate) {
-        Field uidField = new Field("_uid", uid, UidFieldMapper.Defaults.FIELD_TYPE);
+    private static ParsedDocument testParsedDocument(String id, String type, String routing, long timestamp, long ttl, Document document, BytesReference source, Mapping mappingUpdate) {
+        Field uidField = new Field("_uid", Uid.createUid(type, id), UidFieldMapper.Defaults.FIELD_TYPE);
         Field versionField = new NumericDocValuesField("_version", 0);
         document.add(uidField);
         document.add(versionField);
-        return new ParsedDocument(versionField, id, type, routing, timestamp, ttl, Arrays.asList(document), source, mappingUpdate);
+        BytesRef ref = source.toBytesRef();
+        document.add(new StoredField(SourceFieldMapper.NAME, ref.bytes, ref.offset, ref.length));
+        return new ParsedDocument(versionField, id, type, routing, timestamp, ttl, Arrays.asList(document), source, XContentType.JSON,
+            mappingUpdate);
     }
 
     protected Store createStore() throws IOException {
@@ -278,15 +321,31 @@ protected InternalEngine createEngine(Store store, Path translogPath) throws IOE
     }
 
     protected InternalEngine createEngine(IndexSettings indexSettings, Store store, Path translogPath, MergePolicy mergePolicy) throws IOException {
-        EngineConfig config = config(indexSettings, store, translogPath, mergePolicy, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP);
-        InternalEngine internalEngine = new InternalEngine(config);
+        return createEngine(indexSettings, store, translogPath, mergePolicy, null);
+    }
+
+    protected InternalEngine createEngine(
+            IndexSettings indexSettings,
+            Store store,
+            Path translogPath,
+            MergePolicy mergePolicy,
+            Supplier<IndexWriter> indexWriterSupplier) throws IOException {
+        EngineConfig config = config(indexSettings, store, translogPath, mergePolicy, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, null);
+        InternalEngine internalEngine = new InternalEngine(config) {
+            @Override
+            IndexWriter createWriter(boolean create) throws IOException {
+                return (indexWriterSupplier != null) ? indexWriterSupplier.get() : super.createWriter(create);
+            }
+
+        };
         if (config.getOpenMode() == EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG) {
             internalEngine.recoverFromTranslog();
         }
         return internalEngine;
     }
 
-    public EngineConfig config(IndexSettings indexSettings, Store store, Path translogPath, MergePolicy mergePolicy, long maxUnsafeAutoIdTimestamp) {
+    public EngineConfig config(IndexSettings indexSettings, Store store, Path translogPath, MergePolicy mergePolicy,
+                               long maxUnsafeAutoIdTimestamp, ReferenceManager.RefreshListener refreshListener) {
         IndexWriterConfig iwc = newIndexWriterConfig();
         TranslogConfig translogConfig = new TranslogConfig(shardId, translogPath, indexSettings, BigArrays.NON_RECYCLING_INSTANCE);
         final EngineConfig.OpenMode openMode;
@@ -305,17 +364,25 @@ public void onFailedEngine(String reason, @Nullable Exception e) {
                 // we don't need to notify anybody in this test
             }
         };
+        final List<ReferenceManager.RefreshListener> refreshListenerList =
+            refreshListener == null ? emptyList() : Collections.singletonList(refreshListener);
         EngineConfig config = new EngineConfig(openMode, shardId, threadPool, indexSettings, null, store, createSnapshotDeletionPolicy(),
                 mergePolicy, iwc.getAnalyzer(), iwc.getSimilarity(), new CodecService(null, logger), listener,
-                new TranslogHandler(shardId.getIndexName(), logger), IndexSearcher.getDefaultQueryCache(),
-                IndexSearcher.getDefaultQueryCachingPolicy(), translogConfig, TimeValue.timeValueMinutes(5), null, maxUnsafeAutoIdTimestamp);
+                new TranslogHandler(xContentRegistry(), shardId.getIndexName(), logger), IndexSearcher.getDefaultQueryCache(),
+                IndexSearcher.getDefaultQueryCachingPolicy(), translogConfig, TimeValue.timeValueMinutes(5), refreshListenerList,
+            maxUnsafeAutoIdTimestamp);
 
         return config;
     }
 
-    protected static final BytesReference B_1 = new BytesArray(new byte[]{1});
-    protected static final BytesReference B_2 = new BytesArray(new byte[]{2});
-    protected static final BytesReference B_3 = new BytesArray(new byte[]{3});
+    private static final BytesReference B_1 = new BytesArray(new byte[]{1});
+    private static final BytesReference B_2 = new BytesArray(new byte[]{2});
+    private static final BytesReference B_3 = new BytesArray(new byte[]{3});
+    private static final BytesArray SOURCE = bytesArray("{}");
+
+    private static BytesArray bytesArray(String string) {
+        return new BytesArray(string.getBytes(Charset.defaultCharset()));
+    }
 
     public void testSegments() throws Exception {
         try (Store store = createStore();
@@ -326,13 +393,13 @@ public void testSegments() throws Exception {
             assertThat(engine.segmentsStats(false).getMemoryInBytes(), equalTo(0L));
 
             // create two docs and refresh
-            ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null);
-            Engine.Index first = new Engine.Index(newUid("1"), doc);
-            engine.index(first);
-            ParsedDocument doc2 = testParsedDocument("2", "2", "test", null, -1, -1, testDocumentWithTextField(), B_2, null);
-            Engine.Index second = new Engine.Index(newUid("2"), doc2);
-            engine.index(second);
-            assertThat(second.getTranslogLocation(), greaterThan(first.getTranslogLocation()));
+            ParsedDocument doc = testParsedDocument("1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null);
+            Engine.Index first = indexForDoc(doc);
+            Engine.IndexResult firstResult = engine.index(first);
+            ParsedDocument doc2 = testParsedDocument("2", "test", null, -1, -1, testDocumentWithTextField(), B_2, null);
+            Engine.Index second = indexForDoc(doc2);
+            Engine.IndexResult secondResult = engine.index(second);
+            assertThat(secondResult.getTranslogLocation(), greaterThan(firstResult.getTranslogLocation()));
             engine.refresh("test");
 
             segments = engine.segments(false);
@@ -362,8 +429,8 @@ public void testSegments() throws Exception {
             assertThat(segments.get(0).getDeletedDocs(), equalTo(0));
             assertThat(segments.get(0).isCompound(), equalTo(true));
 
-            ParsedDocument doc3 = testParsedDocument("3", "3", "test", null, -1, -1, testDocumentWithTextField(), B_3, null);
-            engine.index(new Engine.Index(newUid("3"), doc3));
+            ParsedDocument doc3 = testParsedDocument("3", "test", null, -1, -1, testDocumentWithTextField(), B_3, null);
+            engine.index(indexForDoc(doc3));
             engine.refresh("test");
 
             segments = engine.segments(false);
@@ -389,7 +456,7 @@ public void testSegments() throws Exception {
             assertThat(segments.get(1).isCompound(), equalTo(true));
 
 
-            engine.delete(new Engine.Delete("test", "1", newUid("1")));
+            engine.delete(new Engine.Delete("test", "1", newUid(doc)));
             engine.refresh("test");
 
             segments = engine.segments(false);
@@ -409,8 +476,8 @@ public void testSegments() throws Exception {
             assertThat(segments.get(1).isCompound(), equalTo(true));
 
             engine.onSettingsChanged();
-            ParsedDocument doc4 = testParsedDocument("4", "4", "test", null, -1, -1, testDocumentWithTextField(), B_3, null);
-            engine.index(new Engine.Index(newUid("4"), doc4));
+            ParsedDocument doc4 = testParsedDocument("4", "test", null, -1, -1, testDocumentWithTextField(), B_3, null);
+            engine.index(indexForDoc(doc4));
             engine.refresh("test");
 
             segments = engine.segments(false);
@@ -443,19 +510,19 @@ public void testVerboseSegments() throws Exception {
             List<Segment> segments = engine.segments(true);
             assertThat(segments.isEmpty(), equalTo(true));
 
-            ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null);
-            engine.index(new Engine.Index(newUid("1"), doc));
+            ParsedDocument doc = testParsedDocument("1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null);
+            engine.index(indexForDoc(doc));
             engine.refresh("test");
 
             segments = engine.segments(true);
             assertThat(segments.size(), equalTo(1));
             assertThat(segments.get(0).ramTree, notNullValue());
 
-            ParsedDocument doc2 = testParsedDocument("2", "2", "test", null, -1, -1, testDocumentWithTextField(), B_2, null);
-            engine.index(new Engine.Index(newUid("2"), doc2));
+            ParsedDocument doc2 = testParsedDocument("2", "test", null, -1, -1, testDocumentWithTextField(), B_2, null);
+            engine.index(indexForDoc(doc2));
             engine.refresh("test");
-            ParsedDocument doc3 = testParsedDocument("3", "3", "test", null, -1, -1, testDocumentWithTextField(), B_3, null);
-            engine.index(new Engine.Index(newUid("3"), doc3));
+            ParsedDocument doc3 = testParsedDocument("3", "test", null, -1, -1, testDocumentWithTextField(), B_3, null);
+            engine.index(indexForDoc(doc3));
             engine.refresh("test");
 
             segments = engine.segments(true);
@@ -469,12 +536,12 @@ public void testVerboseSegments() throws Exception {
     public void testSegmentsWithMergeFlag() throws Exception {
         try (Store store = createStore();
             Engine engine = createEngine(defaultSettings, store, createTempDir(), new TieredMergePolicy())) {
-            ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null);
-            Engine.Index index = new Engine.Index(newUid("1"), doc);
+            ParsedDocument doc = testParsedDocument("1", "test", null, -1, -1, testDocument(), B_1, null);
+            Engine.Index index = indexForDoc(doc);
             engine.index(index);
             engine.flush();
             assertThat(engine.segments(false).size(), equalTo(1));
-            index = new Engine.Index(newUid("2"), doc);
+            index = indexForDoc(testParsedDocument("2", "test", null, -1, -1, testDocument(), B_1, null));
             engine.index(index);
             engine.flush();
             List<Segment> segments = engine.segments(false);
@@ -482,7 +549,7 @@ public void testSegmentsWithMergeFlag() throws Exception {
             for (Segment segment : segments) {
                 assertThat(segment.getMergeId(), nullValue());
             }
-            index = new Engine.Index(newUid("3"), doc);
+            index = indexForDoc(testParsedDocument("3", "test", null, -1, -1, testDocument(), B_1, null));
             engine.index(index);
             engine.flush();
             segments = engine.segments(false);
@@ -491,7 +558,7 @@ public void testSegmentsWithMergeFlag() throws Exception {
                 assertThat(segment.getMergeId(), nullValue());
             }
 
-            index = new Engine.Index(newUid("4"), doc);
+            index = indexForDoc(doc);
             engine.index(index);
             engine.flush();
             final long gen1 = store.readLastCommittedSegmentsInfo().getGeneration();
@@ -523,8 +590,8 @@ public void testSegmentsStatsIncludingFileSizes() throws Exception {
             Engine engine = createEngine(defaultSettings, store, createTempDir(), NoMergePolicy.INSTANCE)) {
             assertThat(engine.segmentsStats(true).getFileSizes().size(), equalTo(0));
 
-            ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null);
-            engine.index(new Engine.Index(newUid("1"), doc));
+            ParsedDocument doc = testParsedDocument("1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null);
+            engine.index(indexForDoc(doc));
             engine.refresh("test");
 
             SegmentsStats stats = engine.segmentsStats(true);
@@ -533,19 +600,19 @@ public void testSegmentsStatsIncludingFileSizes() throws Exception {
 
             ObjectObjectCursor<String, Long> firstEntry = stats.getFileSizes().iterator().next();
 
-            ParsedDocument doc2 = testParsedDocument("2", "2", "test", null, -1, -1, testDocumentWithTextField(), B_2, null);
-            engine.index(new Engine.Index(newUid("2"), doc2));
+            ParsedDocument doc2 = testParsedDocument("2", "test", null, -1, -1, testDocumentWithTextField(), B_2, null);
+            engine.index(indexForDoc(doc2));
             engine.refresh("test");
 
             assertThat(engine.segmentsStats(true).getFileSizes().get(firstEntry.key), greaterThan(firstEntry.value));
         }
     }
 
-    public void testCommitStats() {
+    public void testCommitStats() throws Exception {
         Document document = testDocumentWithTextField();
         document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE));
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_1, null);
-        engine.index(new Engine.Index(newUid("1"), doc));
+        ParsedDocument doc = testParsedDocument("1", "test", null, -1, -1, document, B_1, null);
+        engine.index(indexForDoc(doc));
 
         CommitStats stats1 = engine.commitStats();
         assertThat(stats1.getGeneration(), greaterThan(0L));
@@ -585,6 +652,7 @@ public IndexSearcher wrap(IndexSearcher searcher) throws EngineException {
         engine.close();
 
         engine = new InternalEngine(copy(engine.config(), EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG));
+        assertTrue(engine.isRecovering());
         engine.recoverFromTranslog();
         Engine.Searcher searcher = wrapper.wrap(engine.acquireSearcher("test"));
         assertThat(counter.get(), equalTo(2));
@@ -593,15 +661,18 @@ public IndexSearcher wrap(IndexSearcher searcher) throws EngineException {
     }
 
     public void testFlushIsDisabledDuringTranslogRecovery() throws IOException {
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null);
-        engine.index(new Engine.Index(newUid("1"), doc));
+        assertFalse(engine.isRecovering());
+        ParsedDocument doc = testParsedDocument("1", "test", null, -1, -1, testDocumentWithTextField(), SOURCE, null);
+        engine.index(indexForDoc(doc));
         engine.close();
 
         engine = new InternalEngine(copy(engine.config(), EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG));
-        expectThrows(FlushNotAllowedEngineException.class, () -> engine.flush(true, true));
+        expectThrows(IllegalStateException.class, () -> engine.flush(true, true));
+        assertTrue(engine.isRecovering());
         engine.recoverFromTranslog();
-        doc = testParsedDocument("2", "2", "test", null, -1, -1, testDocumentWithTextField(), B_1, null);
-        engine.index(new Engine.Index(newUid("2"), doc));
+        assertFalse(engine.isRecovering());
+        doc = testParsedDocument("2", "test", null, -1, -1, testDocumentWithTextField(), SOURCE, null);
+        engine.index(indexForDoc(doc));
         engine.flush();
     }
 
@@ -612,13 +683,13 @@ public void testTranslogMultipleOperationsSameDocument() throws IOException {
         try {
             initialEngine = engine;
             for (int i = 0; i < ops; i++) {
-                final ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null);
+                final ParsedDocument doc = testParsedDocument("1", "test", null, -1, -1, testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null);
                 if (randomBoolean()) {
-                    final Engine.Index operation = new Engine.Index(newUid("test#1"), doc, i, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), -1, false);
+                    final Engine.Index operation = new Engine.Index(newUid(doc), doc, i, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), -1, false);
                     operations.add(operation);
                     initialEngine.index(operation);
                 } else {
-                    final Engine.Delete operation = new Engine.Delete("test", "1", newUid("test#1"), i, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), false);
+                    final Engine.Delete operation = new Engine.Delete("test", "1", newUid(doc), i, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime());
                     operations.add(operation);
                     initialEngine.delete(operation);
                 }
@@ -648,8 +719,8 @@ public void testTranslogRecoveryDoesNotReplayIntoTranslog() throws IOException {
             initialEngine = engine;
             for (int i = 0; i < docs; i++) {
                 final String id = Integer.toString(i);
-                final ParsedDocument doc = testParsedDocument(id, id, "test", null, -1, -1, testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null);
-                initialEngine.index(new Engine.Index(newUid(id), doc));
+                final ParsedDocument doc = testParsedDocument(id, "test", null, -1, -1, testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null);
+                initialEngine.index(indexForDoc(doc));
             }
         } finally {
             IOUtils.close(initialEngine);
@@ -677,33 +748,31 @@ public CommitId flush(boolean force, boolean waitIfOngoing) throws EngineExcepti
     }
 
     public void testConcurrentGetAndFlush() throws Exception {
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null);
-        engine.index(new Engine.Index(newUid("1"), doc));
+        ParsedDocument doc = testParsedDocument("1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null);
+        engine.index(indexForDoc(doc));
 
         final AtomicReference<Engine.GetResult> latestGetResult = new AtomicReference<>();
-        latestGetResult.set(engine.get(new Engine.Get(true, newUid("1"))));
+        final Function<String, Searcher> searcherFactory = engine::acquireSearcher;
+        latestGetResult.set(engine.get(new Engine.Get(true, doc.type(), doc.id(), newUid(doc)), searcherFactory));
         final AtomicBoolean flushFinished = new AtomicBoolean(false);
         final CyclicBarrier barrier = new CyclicBarrier(2);
-        Thread getThread = new Thread() {
-            @Override
-            public void run() {
-                try {
-                    barrier.await();
-                } catch (InterruptedException | BrokenBarrierException e) {
-                    throw new RuntimeException(e);
+        Thread getThread = new Thread(() -> {
+            try {
+                barrier.await();
+            } catch (InterruptedException | BrokenBarrierException e) {
+                throw new RuntimeException(e);
+            }
+            while (flushFinished.get() == false) {
+                Engine.GetResult previousGetResult = latestGetResult.get();
+                if (previousGetResult != null) {
+                    previousGetResult.release();
                 }
-                while (flushFinished.get() == false) {
-                    Engine.GetResult previousGetResult = latestGetResult.get();
-                    if (previousGetResult != null) {
-                        previousGetResult.release();
-                    }
-                    latestGetResult.set(engine.get(new Engine.Get(true, newUid("1"))));
-                    if (latestGetResult.get().exists() == false) {
-                        break;
-                    }
+                latestGetResult.set(engine.get(new Engine.Get(true, doc.type(), doc.id(), newUid(doc)), searcherFactory));
+                if (latestGetResult.get().exists() == false) {
+                    break;
                 }
             }
-        };
+        });
         getThread.start();
         barrier.await();
         engine.flush();
@@ -718,11 +787,13 @@ public void testSimpleOperations() throws Exception {
         MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0));
         searchResult.close();
 
+        final Function<String, Searcher> searcherFactory = engine::acquireSearcher;
+
         // create a document
         Document document = testDocumentWithTextField();
         document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE));
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_1, null);
-        engine.index(new Engine.Index(newUid("1"), doc));
+        ParsedDocument doc = testParsedDocument("1", "test", null, -1, -1, document, B_1, null);
+        engine.index(indexForDoc(doc));
 
         // its not there...
         searchResult = engine.acquireSearcher("test");
@@ -731,12 +802,12 @@ public void testSimpleOperations() throws Exception {
         searchResult.close();
 
         // but, not there non realtime
-        Engine.GetResult getResult = engine.get(new Engine.Get(false, newUid("1")));
+        Engine.GetResult getResult = engine.get(new Engine.Get(false, doc.type(), doc.id(), newUid(doc)), searcherFactory);
         assertThat(getResult.exists(), equalTo(false));
         getResult.release();
 
         // but, we can still get it (in realtime)
-        getResult = engine.get(new Engine.Get(true, newUid("1")));
+        getResult = engine.get(new Engine.Get(true, doc.type(), doc.id(), newUid(doc)), searcherFactory);
         assertThat(getResult.exists(), equalTo(true));
         assertThat(getResult.docIdAndVersion(), notNullValue());
         getResult.release();
@@ -751,7 +822,7 @@ public void testSimpleOperations() throws Exception {
         searchResult.close();
 
         // also in non realtime
-        getResult = engine.get(new Engine.Get(false, newUid("1")));
+        getResult = engine.get(new Engine.Get(false, doc.type(), doc.id(), newUid(doc)), searcherFactory);
         assertThat(getResult.exists(), equalTo(true));
         assertThat(getResult.docIdAndVersion(), notNullValue());
         getResult.release();
@@ -760,8 +831,8 @@ public void testSimpleOperations() throws Exception {
         document = testDocument();
         document.add(new TextField("value", "test1", Field.Store.YES));
         document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_2), SourceFieldMapper.Defaults.FIELD_TYPE));
-        doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_2, null);
-        engine.index(new Engine.Index(newUid("1"), doc));
+        doc = testParsedDocument("1", "test", null, -1, -1, document, B_2, null);
+        engine.index(indexForDoc(doc));
 
         // its not updated yet...
         searchResult = engine.acquireSearcher("test");
@@ -771,7 +842,7 @@ public void testSimpleOperations() throws Exception {
         searchResult.close();
 
         // but, we can still get it (in realtime)
-        getResult = engine.get(new Engine.Get(true, newUid("1")));
+        getResult = engine.get(new Engine.Get(true, doc.type(), doc.id(), newUid(doc)), searcherFactory);
         assertThat(getResult.exists(), equalTo(true));
         assertThat(getResult.docIdAndVersion(), notNullValue());
         getResult.release();
@@ -786,7 +857,7 @@ public void testSimpleOperations() throws Exception {
         searchResult.close();
 
         // now delete
-        engine.delete(new Engine.Delete("test", "1", newUid("1")));
+        engine.delete(new Engine.Delete("test", "1", newUid(doc)));
 
         // its not deleted yet
         searchResult = engine.acquireSearcher("test");
@@ -796,7 +867,7 @@ public void testSimpleOperations() throws Exception {
         searchResult.close();
 
         // but, get should not see it (in realtime)
-        getResult = engine.get(new Engine.Get(true, newUid("1")));
+        getResult = engine.get(new Engine.Get(true, doc.type(), doc.id(), newUid(doc)), searcherFactory);
         assertThat(getResult.exists(), equalTo(false));
         getResult.release();
 
@@ -812,8 +883,8 @@ public void testSimpleOperations() throws Exception {
         // add it back
         document = testDocumentWithTextField();
         document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE));
-        doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_1, null);
-        engine.index(new Engine.Index(newUid("1"), doc, Versions.MATCH_DELETED));
+        doc = testParsedDocument("1", "test", null, -1, -1, document, B_1, null);
+        engine.index(new Engine.Index(newUid(doc), doc, Versions.MATCH_DELETED));
 
         // its not there...
         searchResult = engine.acquireSearcher("test");
@@ -836,7 +907,7 @@ public void testSimpleOperations() throws Exception {
         engine.flush();
 
         // and, verify get (in real time)
-        getResult = engine.get(new Engine.Get(true, newUid("1")));
+        getResult = engine.get(new Engine.Get(true, doc.type(), doc.id(), newUid(doc)), searcherFactory);
         assertThat(getResult.exists(), equalTo(true));
         assertThat(getResult.docIdAndVersion(), notNullValue());
         getResult.release();
@@ -845,8 +916,8 @@ public void testSimpleOperations() throws Exception {
         // now do an update
         document = testDocument();
         document.add(new TextField("value", "test1", Field.Store.YES));
-        doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_1, null);
-        engine.index(new Engine.Index(newUid("1"), doc));
+        doc = testParsedDocument("1", "test", null, -1, -1, document, B_1, null);
+        engine.index(indexForDoc(doc));
 
         // its not updated yet...
         searchResult = engine.acquireSearcher("test");
@@ -871,8 +942,8 @@ public void testSearchResultRelease() throws Exception {
         searchResult.close();
 
         // create a document
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null);
-        engine.index(new Engine.Index(newUid("1"), doc));
+        ParsedDocument doc = testParsedDocument("1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null);
+        engine.index(indexForDoc(doc));
 
         // its not there...
         searchResult = engine.acquireSearcher("test");
@@ -890,7 +961,7 @@ public void testSearchResultRelease() throws Exception {
         // don't release the search result yet...
 
         // delete, refresh and do a new search, it should not be there
-        engine.delete(new Engine.Delete("test", "1", newUid("1")));
+        engine.delete(new Engine.Delete("test", "1", newUid(doc)));
         engine.refresh("test");
         Engine.Searcher updateSearchResult = engine.acquireSearcher("test");
         MatcherAssert.assertThat(updateSearchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0));
@@ -905,10 +976,10 @@ public void testSearchResultRelease() throws Exception {
     public void testSyncedFlush() throws IOException {
         try (Store store = createStore();
             Engine engine = new InternalEngine(config(defaultSettings, store, createTempDir(),
-                     new LogByteSizeMergePolicy(), IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP))) {
+                     new LogByteSizeMergePolicy(), IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, null))) {
             final String syncId = randomUnicodeOfCodepointLengthBetween(10, 20);
-            ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null);
-            engine.index(new Engine.Index(newUid("1"), doc));
+            ParsedDocument doc = testParsedDocument("1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null);
+            engine.index(indexForDoc(doc));
             Engine.CommitId commitID = engine.flush();
             assertThat(commitID, equalTo(new Engine.CommitId(store.readLastCommittedSegmentsInfo().getId())));
             byte[] wrongBytes = Base64.getDecoder().decode(commitID.toString());
@@ -916,7 +987,7 @@ public void testSyncedFlush() throws IOException {
             Engine.CommitId wrongId = new Engine.CommitId(wrongBytes);
             assertEquals("should fail to sync flush with wrong id (but no docs)", engine.syncFlush(syncId + "1", wrongId),
                     Engine.SyncedFlushResult.COMMIT_MISMATCH);
-            engine.index(new Engine.Index(newUid("2"), doc));
+            engine.index(indexForDoc(doc));
             assertEquals("should fail to sync flush with right id but pending doc", engine.syncFlush(syncId + "2", commitID),
                     Engine.SyncedFlushResult.PENDING_OPERATIONS);
             commitID = engine.flush();
@@ -932,22 +1003,23 @@ public void testRenewSyncFlush() throws Exception {
         for (int i = 0; i < iters; i++) {
             try (Store store = createStore();
                  InternalEngine engine = new InternalEngine(config(defaultSettings, store, createTempDir(),
-                         new LogDocMergePolicy(), IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP))) {
+                         new LogDocMergePolicy(), IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, null))) {
                 final String syncId = randomUnicodeOfCodepointLengthBetween(10, 20);
-                ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null);
-                Engine.Index doc1 = new Engine.Index(newUid("1"), doc);
+                ParsedDocument doc = testParsedDocument("1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null);
+                Engine.Index doc1 = indexForDoc(doc);
                 engine.index(doc1);
                 assertEquals(engine.getLastWriteNanos(), doc1.startTime());
                 engine.flush();
-                Engine.Index doc2 = new Engine.Index(newUid("2"), doc);
+                Engine.Index doc2 = indexForDoc(testParsedDocument("2", "test", null, -1, -1, testDocumentWithTextField(), B_1, null));
                 engine.index(doc2);
                 assertEquals(engine.getLastWriteNanos(), doc2.startTime());
                 engine.flush();
                 final boolean forceMergeFlushes = randomBoolean();
+                final ParsedDocument parsedDoc3 = testParsedDocument("3", "test", null, -1, -1, testDocumentWithTextField(), B_1, null);
                 if (forceMergeFlushes) {
-                    engine.index(new Engine.Index(newUid("3"), doc, Versions.MATCH_ANY, VersionType.INTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime() - engine.engineConfig.getFlushMergesAfter().nanos(), -1, false));
+                    engine.index(new Engine.Index(newUid(parsedDoc3), parsedDoc3, Versions.MATCH_ANY, VersionType.INTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime() - engine.engineConfig.getFlushMergesAfter().nanos(), -1, false));
                 } else {
-                    engine.index(new Engine.Index(newUid("3"), doc));
+                    engine.index(indexForDoc(parsedDoc3));
                 }
                 Engine.CommitId commitID = engine.flush();
                 assertEquals("should succeed to flush commit with right id and no pending doc", engine.syncFlush(syncId, commitID),
@@ -969,7 +1041,7 @@ public void testRenewSyncFlush() throws Exception {
                 assertEquals(engine.getLastCommittedSegmentInfos().getUserData().get(Engine.SYNC_COMMIT_ID), syncId);
 
                 if (randomBoolean()) {
-                    Engine.Index doc4 = new Engine.Index(newUid("4"), doc);
+                    Engine.Index doc4 = indexForDoc(testParsedDocument("4", "test", null, -1, -1, testDocumentWithTextField(), B_1, null));
                     engine.index(doc4);
                     assertEquals(engine.getLastWriteNanos(), doc4.startTime());
                 } else {
@@ -987,8 +1059,8 @@ public void testRenewSyncFlush() throws Exception {
 
     public void testSycnedFlushSurvivesEngineRestart() throws IOException {
         final String syncId = randomUnicodeOfCodepointLengthBetween(10, 20);
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null);
-        engine.index(new Engine.Index(newUid("1"), doc));
+        ParsedDocument doc = testParsedDocument("1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null);
+        engine.index(indexForDoc(doc));
         final Engine.CommitId commitID = engine.flush();
         assertEquals("should succeed to flush commit with right id and no pending doc", engine.syncFlush(syncId, commitID),
                 Engine.SyncedFlushResult.SUCCESS);
@@ -1010,15 +1082,15 @@ public void testSycnedFlushSurvivesEngineRestart() throws IOException {
 
     public void testSycnedFlushVanishesOnReplay() throws IOException {
         final String syncId = randomUnicodeOfCodepointLengthBetween(10, 20);
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null);
-        engine.index(new Engine.Index(newUid("1"), doc));
+        ParsedDocument doc = testParsedDocument("1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null);
+        engine.index(indexForDoc(doc));
         final Engine.CommitId commitID = engine.flush();
         assertEquals("should succeed to flush commit with right id and no pending doc", engine.syncFlush(syncId, commitID),
                 Engine.SyncedFlushResult.SUCCESS);
         assertEquals(store.readLastCommittedSegmentsInfo().getUserData().get(Engine.SYNC_COMMIT_ID), syncId);
         assertEquals(engine.getLastCommittedSegmentInfos().getUserData().get(Engine.SYNC_COMMIT_ID), syncId);
-        doc = testParsedDocument("2", "2", "test", null, -1, -1, testDocumentWithTextField(), new BytesArray("{}"), null);
-        engine.index(new Engine.Index(newUid("2"), doc));
+        doc = testParsedDocument("2", "test", null, -1, -1, testDocumentWithTextField(), new BytesArray("{}"), null);
+        engine.index(indexForDoc(doc));
         EngineConfig config = engine.config();
         engine.close();
         engine = new InternalEngine(copy(config, EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG));
@@ -1026,145 +1098,36 @@ public void testSycnedFlushVanishesOnReplay() throws IOException {
         assertNull("Sync ID must be gone since we have a document to replay", engine.getLastCommittedSegmentInfos().getUserData().get(Engine.SYNC_COMMIT_ID));
     }
 
-    public void testVersioningNewCreate() {
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null);
-        Engine.Index create = new Engine.Index(newUid("1"), doc, Versions.MATCH_DELETED);
-        engine.index(create);
-        assertThat(create.version(), equalTo(1L));
-
-        create = new Engine.Index(newUid("1"), doc, create.version(), create.versionType().versionTypeForReplicationAndRecovery(), REPLICA, 0, -1, false);
-        replicaEngine.index(create);
-        assertThat(create.version(), equalTo(1L));
-    }
-
-    public void testVersioningNewIndex() {
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null);
-        Engine.Index index = new Engine.Index(newUid("1"), doc);
-        engine.index(index);
-        assertThat(index.version(), equalTo(1L));
-
-        index = new Engine.Index(newUid("1"), doc, index.version(), index.versionType().versionTypeForReplicationAndRecovery(), REPLICA, 0, -1, false);
-        replicaEngine.index(index);
-        assertThat(index.version(), equalTo(1L));
-    }
-
-    public void testExternalVersioningNewIndex() {
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null);
-        Engine.Index index = new Engine.Index(newUid("1"), doc, 12, VersionType.EXTERNAL, PRIMARY, 0, -1, false);
-        engine.index(index);
-        assertThat(index.version(), equalTo(12L));
-
-        index = new Engine.Index(newUid("1"), doc, index.version(), index.versionType().versionTypeForReplicationAndRecovery(), REPLICA, 0, -1, false);
-        replicaEngine.index(index);
-        assertThat(index.version(), equalTo(12L));
-    }
-
-    public void testVersioningIndexConflict() {
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null);
-        Engine.Index index = new Engine.Index(newUid("1"), doc);
-        engine.index(index);
-        assertThat(index.version(), equalTo(1L));
-
-        index = new Engine.Index(newUid("1"), doc);
-        engine.index(index);
-        assertThat(index.version(), equalTo(2L));
-
-        index = new Engine.Index(newUid("1"), doc, 1L, VersionType.INTERNAL, Engine.Operation.Origin.PRIMARY, 0, -1, false);
-        try {
-            engine.index(index);
-            fail();
-        } catch (VersionConflictEngineException e) {
-            // all is well
-        }
-
-        // future versions should not work as well
-        index = new Engine.Index(newUid("1"), doc, 3L, VersionType.INTERNAL, PRIMARY, 0, -1, false);
-        try {
-            engine.index(index);
-            fail();
-        } catch (VersionConflictEngineException e) {
-            // all is well
-        }
-    }
-
-    public void testExternalVersioningIndexConflict() {
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null);
-        Engine.Index index = new Engine.Index(newUid("1"), doc, 12, VersionType.EXTERNAL, PRIMARY, 0, -1, false);
-        engine.index(index);
-        assertThat(index.version(), equalTo(12L));
-
-        index = new Engine.Index(newUid("1"), doc, 14, VersionType.EXTERNAL, PRIMARY, 0, -1, false);
-        engine.index(index);
-        assertThat(index.version(), equalTo(14L));
-
-        index = new Engine.Index(newUid("1"), doc, 13, VersionType.EXTERNAL, PRIMARY, 0, -1, false);
-        try {
-            engine.index(index);
-            fail();
-        } catch (VersionConflictEngineException e) {
-            // all is well
-        }
-    }
-
-    public void testVersioningIndexConflictWithFlush() {
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null);
-        Engine.Index index = new Engine.Index(newUid("1"), doc);
-        engine.index(index);
-        assertThat(index.version(), equalTo(1L));
+    public void testVersioningNewCreate() throws IOException {
+        ParsedDocument doc = testParsedDocument("1", "test", null, -1, -1, testDocument(), B_1, null);
+        Engine.Index create = new Engine.Index(newUid(doc), doc, Versions.MATCH_DELETED);
+        Engine.IndexResult indexResult = engine.index(create);
+        assertThat(indexResult.getVersion(), equalTo(1L));
 
-        index = new Engine.Index(newUid("1"), doc);
-        engine.index(index);
-        assertThat(index.version(), equalTo(2L));
-
-        engine.flush();
-
-        index = new Engine.Index(newUid("1"), doc, 1L, VersionType.INTERNAL, PRIMARY, 0, -1, false);
-        try {
-            engine.index(index);
-            fail();
-        } catch (VersionConflictEngineException e) {
-            // all is well
-        }
-
-        // future versions should not work as well
-        index = new Engine.Index(newUid("1"), doc, 3L, VersionType.INTERNAL, PRIMARY, 0, -1, false);
-        try {
-            engine.index(index);
-            fail();
-        } catch (VersionConflictEngineException e) {
-            // all is well
-        }
+        create = new Engine.Index(newUid(doc), doc, indexResult.getVersion(), create.versionType().versionTypeForReplicationAndRecovery(), REPLICA, 0, -1, false);
+        Engine.IndexResult replicaResult = replicaEngine.index(create);
+        assertThat(replicaResult.getVersion(), equalTo(1L));
     }
 
-    public void testExternalVersioningIndexConflictWithFlush() {
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null);
-        Engine.Index index = new Engine.Index(newUid("1"), doc, 12, VersionType.EXTERNAL, PRIMARY, 0, -1, false);
-        engine.index(index);
-        assertThat(index.version(), equalTo(12L));
-
-        index = new Engine.Index(newUid("1"), doc, 14, VersionType.EXTERNAL, PRIMARY, 0, -1, false);
-        engine.index(index);
-        assertThat(index.version(), equalTo(14L));
+    public void testVersioningNewIndex() throws IOException {
+        ParsedDocument doc = testParsedDocument("1", "test", null, -1, -1, testDocument(), B_1, null);
+        Engine.Index index = indexForDoc(doc);
+        Engine.IndexResult indexResult = engine.index(index);
+        assertThat(indexResult.getVersion(), equalTo(1L));
 
-        engine.flush();
-
-        index = new Engine.Index(newUid("1"), doc, 13, VersionType.EXTERNAL, PRIMARY, 0, -1, false);
-        try {
-            engine.index(index);
-            fail();
-        } catch (VersionConflictEngineException e) {
-            // all is well
-        }
+        index = new Engine.Index(newUid(doc), doc, indexResult.getVersion(), index.versionType().versionTypeForReplicationAndRecovery(), REPLICA, 0, -1, false);
+        Engine.IndexResult replicaResult = replicaEngine.index(index);
+        assertThat(replicaResult.getVersion(), equalTo(1L));
     }
 
     public void testForceMerge() throws IOException {
         try (Store store = createStore();
             Engine engine = new InternalEngine(config(defaultSettings, store, createTempDir(),
-                     new LogByteSizeMergePolicy(), IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP))) { // use log MP here we test some behavior in ESMP
+                     new LogByteSizeMergePolicy(), IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, null))) { // use log MP here we test some behavior in ESMP
             int numDocs = randomIntBetween(10, 100);
             for (int i = 0; i < numDocs; i++) {
-                ParsedDocument doc = testParsedDocument(Integer.toString(i), Integer.toString(i), "test", null, -1, -1, testDocument(), B_1, null);
-                Engine.Index index = new Engine.Index(newUid(Integer.toString(i)), doc);
+                ParsedDocument doc = testParsedDocument(Integer.toString(i), "test", null, -1, -1, testDocument(), B_1, null);
+                Engine.Index index = indexForDoc(doc);
                 engine.index(index);
                 engine.refresh("test");
             }
@@ -1174,8 +1137,8 @@ public void testForceMerge() throws IOException {
             engine.forceMerge(true, 1, false, false, false);
             assertEquals(engine.segments(true).size(), 1);
 
-            ParsedDocument doc = testParsedDocument(Integer.toString(0), Integer.toString(0), "test", null, -1, -1, testDocument(), B_1, null);
-            Engine.Index index = new Engine.Index(newUid(Integer.toString(0)), doc);
+            ParsedDocument doc = testParsedDocument(Integer.toString(0), "test", null, -1, -1, testDocument(), B_1, null);
+            Engine.Index index = indexForDoc(doc);
             engine.delete(new Engine.Delete(index.type(), index.id(), index.uid()));
             engine.forceMerge(true, 10, true, false, false); //expunge deletes
 
@@ -1185,8 +1148,8 @@ public void testForceMerge() throws IOException {
                 assertEquals(engine.config().getMergePolicy().toString(), numDocs - 1, test.reader().maxDoc());
             }
 
-            doc = testParsedDocument(Integer.toString(1), Integer.toString(1), "test", null, -1, -1, testDocument(), B_1, null);
-            index = new Engine.Index(newUid(Integer.toString(1)), doc);
+            doc = testParsedDocument(Integer.toString(1), "test", null, -1, -1, testDocument(), B_1, null);
+            index = indexForDoc(doc);
             engine.delete(new Engine.Delete(index.type(), index.id(), index.uid()));
             engine.forceMerge(true, 10, false, false, false); //expunge deletes
 
@@ -1220,8 +1183,8 @@ public void run() {
                                 int numDocs = randomIntBetween(1, 20);
                                 for (int j = 0; j < numDocs; j++) {
                                     i++;
-                                    ParsedDocument doc = testParsedDocument(Integer.toString(i), Integer.toString(i), "test", null, -1, -1, testDocument(), B_1, null);
-                                    Engine.Index index = new Engine.Index(newUid(Integer.toString(i)), doc);
+                                    ParsedDocument doc = testParsedDocument(Integer.toString(i), "test", null, -1, -1, testDocument(), B_1, null);
+                                    Engine.Index index = indexForDoc(doc);
                                     engine.index(index);
                                 }
                                 engine.refresh("test");
@@ -1232,8 +1195,10 @@ public void run() {
                                     return;
                                 }
                             }
-                        } catch (AlreadyClosedException | EngineClosedException ex) {
+                        } catch (AlreadyClosedException ex) {
                             // fine
+                        } catch (IOException e) {
+                            throw new AssertionError(e);
                         }
                     }
                 };
@@ -1252,257 +1217,531 @@ public void run() {
 
     }
 
-    public void testVersioningDeleteConflict() {
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null);
-        Engine.Index index = new Engine.Index(newUid("1"), doc);
-        engine.index(index);
-        assertThat(index.version(), equalTo(1L));
-
-        index = new Engine.Index(newUid("1"), doc);
-        engine.index(index);
-        assertThat(index.version(), equalTo(2L));
-
-        Engine.Delete delete = new Engine.Delete("test", "1", newUid("1"), 1L, VersionType.INTERNAL, PRIMARY, 0, false);
-        try {
-            engine.delete(delete);
-            fail();
-        } catch (VersionConflictEngineException e) {
-            // all is well
-        }
-
-        // future versions should not work as well
-        delete = new Engine.Delete("test", "1", newUid("1"), 3L, VersionType.INTERNAL, PRIMARY, 0, false);
-        try {
-            engine.delete(delete);
-            fail();
-        } catch (VersionConflictEngineException e) {
-            // all is well
-        }
+    public void testVersioningCreateExistsException() throws IOException {
+        ParsedDocument doc = testParsedDocument("1", "test", null, System.currentTimeMillis(), -1, testDocument(), B_1, null);
+        Engine.Index create = new Engine.Index(newUid(doc), doc, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, 0, -1, false);
+        Engine.IndexResult indexResult = engine.index(create);
+        assertThat(indexResult.getVersion(), equalTo(1L));
 
-        // now actually delete
-        delete = new Engine.Delete("test", "1", newUid("1"), 2L, VersionType.INTERNAL, PRIMARY, 0, false);
-        engine.delete(delete);
-        assertThat(delete.version(), equalTo(3L));
+        create = new Engine.Index(newUid(doc), doc, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, 0, -1, false);
+        indexResult = engine.index(create);
+        assertTrue(indexResult.hasFailure());
+        assertThat(indexResult.getFailure(), instanceOf(VersionConflictEngineException.class));
 
-        // now check if we can index to a delete doc with version
-        index = new Engine.Index(newUid("1"), doc, 2L, VersionType.INTERNAL, PRIMARY, 0, -1, false);
-        try {
-            engine.index(index);
-            fail();
-        } catch (VersionConflictEngineException e) {
-            // all is well
-        }
+    }
 
-        // we shouldn't be able to create as well
-        Engine.Index create = new Engine.Index(newUid("1"), doc, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, 0, -1, false);
-        try {
-            engine.index(create);
-        } catch (VersionConflictEngineException e) {
-            // all is well
+    protected List<Engine.Operation> generateSingleDocHistory(boolean forReplica, VersionType versionType, int minOpCount, int maxOpCount) {
+        final int numOfOps = randomIntBetween(minOpCount, maxOpCount);
+        final List<Engine.Operation> ops = new ArrayList<>();
+        final Term id = newUid(Uid.createUid("test", "1"));
+        final String valuePrefix = forReplica ? "r_" : "p_";
+        for (int i = 0; i < numOfOps; i++) {
+            final Engine.Operation op;
+            final long version;
+            switch (versionType) {
+                case INTERNAL:
+                    version = forReplica ? i : Versions.MATCH_ANY;
+                    break;
+                case EXTERNAL:
+                    version = i;
+                    break;
+                case EXTERNAL_GTE:
+                    version = randomBoolean() ? Math.max(i - 1, 0) : i;
+                    break;
+                case FORCE:
+                    version = randomNonNegativeLong();
+                    break;
+                default:
+                    throw new UnsupportedOperationException("unknown version type: " + versionType);
+            }
+            if (randomBoolean()) {
+                op = new Engine.Index(id, testParsedDocument("1", "test", null, System.currentTimeMillis(), -1L,
+                    testDocumentWithTextField(valuePrefix + i), B_1, null),
+                    version,
+                    forReplica ? versionType.versionTypeForReplicationAndRecovery() : versionType,
+                    forReplica ? REPLICA : PRIMARY,
+                    System.currentTimeMillis(), -1, false
+                );
+            } else {
+                op = new Engine.Delete("test", "1", id,
+                    version,
+                    forReplica ? versionType.versionTypeForReplicationAndRecovery() : versionType,
+                    forReplica ? REPLICA : PRIMARY,
+                    System.currentTimeMillis());
+            }
+            ops.add(op);
         }
+        return ops;
     }
 
-    public void testVersioningDeleteConflictWithFlush() {
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null);
-        Engine.Index index = new Engine.Index(newUid("1"), doc);
-        engine.index(index);
-        assertThat(index.version(), equalTo(1L));
+    public void testOutOfOrderDocsOnReplica() throws IOException {
+        final List<Engine.Operation> ops = generateSingleDocHistory(true,
+            randomFrom(VersionType.INTERNAL, VersionType.EXTERNAL), 2, 20);
+        assertOpsOnReplica(ops, replicaEngine, true);
+    }
 
-        index = new Engine.Index(newUid("1"), doc);
-        engine.index(index);
-        assertThat(index.version(), equalTo(2L));
+    public void testNonStandardVersioningOnReplica() throws IOException {
+        final List<Engine.Operation> ops = generateSingleDocHistory(true, randomFrom(VersionType.EXTERNAL_GTE, VersionType.FORCE), 2, 20);
+        assertOpsOnReplica(ops, replicaEngine, false);
+    }
 
-        engine.flush();
 
-        Engine.Delete delete = new Engine.Delete("test", "1", newUid("1"), 1L, VersionType.INTERNAL, PRIMARY, 0, false);
-        try {
-            engine.delete(delete);
-            fail();
-        } catch (VersionConflictEngineException e) {
-            // all is well
-        }
+    public void testOutOfOrderDocsOnReplicaOldPrimary() throws IOException {
+        IndexSettings oldSettings = IndexSettingsModule.newIndexSettings("testOld", Settings.builder()
+            .put(IndexSettings.INDEX_GC_DELETES_SETTING.getKey(), "1h") // make sure this doesn't kick in on us
+            .put(EngineConfig.INDEX_CODEC_SETTING.getKey(), codecName)
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_5_4_0)
+            .put(IndexSettings.MAX_REFRESH_LISTENERS_PER_SHARD.getKey(),
+                between(10, 10 * IndexSettings.MAX_REFRESH_LISTENERS_PER_SHARD.get(Settings.EMPTY)))
+            .build());
 
-        // future versions should not work as well
-        delete = new Engine.Delete("test", "1", newUid("1"), 3L, VersionType.INTERNAL, PRIMARY, 0, false);
-        try {
-            engine.delete(delete);
-            fail();
-        } catch (VersionConflictEngineException e) {
-            // all is well
+        try (Store oldReplicaStore = createStore();
+             InternalEngine replicaEngine =
+                 createEngine(oldSettings, oldReplicaStore, createTempDir("translog-old-replica"), newMergePolicy())) {
+            final List<Engine.Operation> ops = generateSingleDocHistory(true, randomFrom(VersionType.INTERNAL, VersionType.EXTERNAL), 2, 20);
+            assertOpsOnReplica(ops, replicaEngine, true);
         }
+    }
 
-        engine.flush();
-
-        // now actually delete
-        delete = new Engine.Delete("test", "1", newUid("1"), 2L, VersionType.INTERNAL, PRIMARY, 0, false);
-        engine.delete(delete);
-        assertThat(delete.version(), equalTo(3L));
-
-        engine.flush();
+    private void assertOpsOnReplica(List<Engine.Operation> ops, InternalEngine replicaEngine, boolean shuffleOps) throws IOException {
+        final Engine.Operation lastOp = ops.get(ops.size() - 1);
+        final String lastFieldValue;
+        if (lastOp instanceof Engine.Index) {
+            Engine.Index index = (Engine.Index) lastOp;
+            lastFieldValue = index.docs().get(0).get("value");
+        } else {
+            // delete
+            lastFieldValue = null;
+        }
+        if (shuffleOps) {
+            shuffle(ops, random());
+        }
+        boolean firstOp = true;
+        for (Engine.Operation op : ops) {
+            logger.info("performing [{}], v [{}]", op.operationType().name().charAt(0), op.version());
+            if (op instanceof Engine.Index) {
+                Engine.IndexResult result = replicaEngine.index((Engine.Index) op);
+                // replicas don't really care to about creation status of documents
+                // this allows to ignore the case where a document was found in the live version maps in
+                // a delete state and return false for the created flag in favor of code simplicity
+                // as deleted or not. This check is just signal regression so a decision can be made if it's
+                // intentional
+                assertThat(result.isCreated(), equalTo(firstOp));
+                assertThat(result.getVersion(), equalTo(op.version()));
+                assertThat(result.hasFailure(), equalTo(false));
 
-        // now check if we can index to a delete doc with version
-        index = new Engine.Index(newUid("1"), doc, 2L, VersionType.INTERNAL, PRIMARY, 0, -1, false);
-        try {
-            engine.index(index);
-            fail();
-        } catch (VersionConflictEngineException e) {
-            // all is well
+            } else {
+                Engine.DeleteResult result = replicaEngine.delete((Engine.Delete) op);
+                // Replicas don't really care to about found status of documents
+                // this allows to ignore the case where a document was found in the live version maps in
+                // a delete state and return true for the found flag in favor of code simplicity
+                // his check is just signal regression so a decision can be made if it's
+                // intentional
+                assertThat(result.isFound(), equalTo(firstOp == false));
+                assertThat(result.getVersion(), equalTo(op.version()));
+                assertThat(result.hasFailure(), equalTo(false));
+            }
+            if (randomBoolean()) {
+                engine.refresh("test");
+            } if (randomBoolean()) {
+                engine.flush();
+            }
+            firstOp = false;
         }
 
-        // we shouldn't be able to create as well
-        Engine.Index create = new Engine.Index(newUid("1"), doc, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, 0, -1, false);
-        try {
-            engine.index(create);
-        } catch (VersionConflictEngineException e) {
-            // all is well
+        assertVisibleCount(replicaEngine, lastFieldValue == null ? 0 : 1);
+        if (lastFieldValue != null) {
+            try (Searcher searcher = replicaEngine.acquireSearcher("test")) {
+                final TotalHitCountCollector collector = new TotalHitCountCollector();
+                searcher.searcher().search(new TermQuery(new Term("value", lastFieldValue)), collector);
+                assertThat(collector.getTotalHits(), equalTo(1));
+            }
         }
     }
 
-    public void testVersioningCreateExistsException() {
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null);
-        Engine.Index create = new Engine.Index(newUid("1"), doc, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, 0, -1, false);
-        engine.index(create);
-        assertThat(create.version(), equalTo(1L));
+    public void testConcurrentOutOfDocsOnReplica() throws IOException, InterruptedException {
+        final List<Engine.Operation> ops = generateSingleDocHistory(true, randomFrom(VersionType.INTERNAL, VersionType.EXTERNAL), 100, 300);
+        final Engine.Operation lastOp = ops.get(ops.size() - 1);
+        final String lastFieldValue;
+        if (lastOp instanceof Engine.Index) {
+            Engine.Index index = (Engine.Index) lastOp;
+            lastFieldValue = index.docs().get(0).get("value");
+        } else {
+            // delete
+            lastFieldValue = null;
+        }
+        shuffle(ops, random());
+        concurrentlyApplyOps(ops, engine);
 
-        create = new Engine.Index(newUid("1"), doc, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, 0, -1, false);
-        try {
-            engine.index(create);
-            fail();
-        } catch (VersionConflictEngineException e) {
-            // all is well
+        assertVisibleCount(engine, lastFieldValue == null ? 0 : 1);
+        if (lastFieldValue != null) {
+            try (Searcher searcher = engine.acquireSearcher("test")) {
+                final TotalHitCountCollector collector = new TotalHitCountCollector();
+                searcher.searcher().search(new TermQuery(new Term("value", lastFieldValue)), collector);
+                assertThat(collector.getTotalHits(), equalTo(1));
+            }
         }
     }
 
-    public void testVersioningCreateExistsExceptionWithFlush() {
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null);
-        Engine.Index create = new Engine.Index(newUid("1"), doc, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, 0, -1, false);
-        engine.index(create);
-        assertThat(create.version(), equalTo(1L));
-
-        engine.flush();
-
-        create = new Engine.Index(newUid("1"), doc, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, 0, -1, false);
-        try {
-            engine.index(create);
-            fail();
-        } catch (VersionConflictEngineException e) {
-            // all is well
+    private void concurrentlyApplyOps(List<Engine.Operation> ops, InternalEngine engine) throws InterruptedException {
+        Thread[] thread = new Thread[randomIntBetween(3, 5)];
+        CountDownLatch startGun = new CountDownLatch(thread.length);
+        AtomicInteger offset = new AtomicInteger(-1);
+        for (int i = 0; i < thread.length; i++) {
+            thread[i] = new Thread(() -> {
+                startGun.countDown();
+                try {
+                    startGun.await();
+                } catch (InterruptedException e) {
+                    throw new AssertionError(e);
+                }
+                int docOffset;
+                while ((docOffset = offset.incrementAndGet()) < ops.size()) {
+                    try {
+                        final Engine.Operation op = ops.get(docOffset);
+                        if (op instanceof Engine.Index) {
+                            engine.index((Engine.Index)op);
+                        } else {
+                            engine.delete((Engine.Delete)op);
+                        }
+                        if ((docOffset + 1) % 4 == 0) {
+                            engine.refresh("test");
+                        }
+                    } catch (IOException e) {
+                        throw new AssertionError(e);
+                    }
+                }
+            });
+            thread[i].start();
+        }
+        for (int i = 0; i < thread.length; i++) {
+            thread[i].join();
         }
     }
 
-    public void testVersioningReplicaConflict1() {
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null);
-        Engine.Index index = new Engine.Index(newUid("1"), doc);
-        engine.index(index);
-        assertThat(index.version(), equalTo(1L));
+    public void testInternalVersioningOnPrimary() throws IOException {
+        final List<Engine.Operation> ops = generateSingleDocHistory(false, VersionType.INTERNAL, 2, 20);
+        assertOpsOnPrimary(ops, Versions.NOT_FOUND, true, engine);
+    }
+
+    private int assertOpsOnPrimary(List<Engine.Operation> ops, long currentOpVersion, boolean docDeleted, InternalEngine engine)
+        throws IOException {
+        String lastFieldValue = null;
+        int opsPerformed = 0;
+        long lastOpVersion = currentOpVersion;
+        BiFunction<Long, Engine.Index, Engine.Index> indexWithVersion = (version, index) -> new Engine.Index(index.uid(), index.parsedDoc(),
+            version, index.versionType(), index.origin(), index.startTime(), index.getAutoGeneratedIdTimestamp(), index.isRetry());
+        BiFunction<Long, Engine.Delete, Engine.Delete> delWithVersion = (version, delete) -> new Engine.Delete(delete.type(), delete.id(),
+            delete.uid(), version, delete.versionType(), delete.origin(), delete.startTime());
+        for (Engine.Operation op : ops) {
+            final boolean versionConflict = rarely();
+            final boolean versionedOp = versionConflict || randomBoolean();
+            final long conflictingVersion = docDeleted || randomBoolean() ?
+                lastOpVersion + (randomBoolean() ? 1 : -1) :
+                Versions.MATCH_DELETED;
+            final long correctVersion = docDeleted && randomBoolean() ? Versions.MATCH_DELETED : lastOpVersion;
+            logger.info("performing [{}]{}{}",
+                op.operationType().name().charAt(0),
+                versionConflict ? " (conflict " + conflictingVersion +")" : "",
+                versionedOp ? " (versioned " + correctVersion + ")" : "");
+            if (op instanceof Engine.Index) {
+                final Engine.Index index = (Engine.Index) op;
+                if (versionConflict) {
+                    // generate a conflict
+                    Engine.IndexResult result = engine.index(indexWithVersion.apply(conflictingVersion, index));
+                    assertThat(result.isCreated(), equalTo(false));
+                    assertThat(result.getVersion(), equalTo(lastOpVersion));
+                    assertThat(result.hasFailure(), equalTo(true));
+                    assertThat(result.getFailure(), instanceOf(VersionConflictEngineException.class));
+                } else {
+                    Engine.IndexResult result = engine.index(versionedOp ? indexWithVersion.apply(correctVersion, index) : index);
+                    assertThat(result.isCreated(), equalTo(docDeleted));
+                    assertThat(result.getVersion(), equalTo(Math.max(lastOpVersion + 1, 1)));
+                    assertThat(result.hasFailure(), equalTo(false));
+                    assertThat(result.getFailure(), nullValue());
+                    lastFieldValue = index.docs().get(0).get("value");
+                    docDeleted = false;
+                    lastOpVersion = result.getVersion();
+                    opsPerformed++;
+                }
+            } else {
+                final Engine.Delete delete = (Engine.Delete) op;
+                if (versionConflict) {
+                    // generate a conflict
+                    Engine.DeleteResult result = engine.delete(delWithVersion.apply(conflictingVersion, delete));
+                    assertThat(result.isFound(), equalTo(docDeleted == false));
+                    assertThat(result.getVersion(), equalTo(lastOpVersion));
+                    assertThat(result.hasFailure(), equalTo(true));
+                    assertThat(result.getFailure(), instanceOf(VersionConflictEngineException.class));
+                } else {
+                    Engine.DeleteResult result = engine.delete(versionedOp ? delWithVersion.apply(correctVersion, delete) : delete);
+                    assertThat(result.isFound(), equalTo(docDeleted == false));
+                    assertThat(result.getVersion(), equalTo(Math.max(lastOpVersion + 1, 1)));
+                    assertThat(result.hasFailure(), equalTo(false));
+                    assertThat(result.getFailure(), nullValue());
+                    docDeleted = true;
+                    lastOpVersion = result.getVersion();
+                    opsPerformed++;
+                }
+            }
+            if (randomBoolean()) {
+                // refresh and take the chance to check everything is ok so far
+                assertVisibleCount(engine, docDeleted ? 0 : 1);
+                // even if doc is not not deleted, lastFieldValue can still be null if this is the
+                // first op and it failed.
+                if (docDeleted == false && lastFieldValue != null) {
+                    try (Searcher searcher = engine.acquireSearcher("test")) {
+                        final TotalHitCountCollector collector = new TotalHitCountCollector();
+                        searcher.searcher().search(new TermQuery(new Term("value", lastFieldValue)), collector);
+                        assertThat(collector.getTotalHits(), equalTo(1));
+                    }
+                }
+            }
+            if (randomBoolean()) {
+                engine.flush();
+            }
 
-        index = new Engine.Index(newUid("1"), doc);
-        engine.index(index);
-        assertThat(index.version(), equalTo(2L));
+            if (rarely()) {
+                // simulate GC deletes
+                engine.refresh("gc_simulation");
+                engine.clearDeletedTombstones();
+                if (docDeleted) {
+                    lastOpVersion = Versions.NOT_FOUND;
+                }
+            }
+        }
 
-        // apply the second index to the replica, should work fine
-        index = new Engine.Index(newUid("1"), doc, index.version(), VersionType.INTERNAL.versionTypeForReplicationAndRecovery(), REPLICA,  0, -1, false);
-        replicaEngine.index(index);
-        assertThat(index.version(), equalTo(2L));
+        assertVisibleCount(engine, docDeleted ? 0 : 1);
+        if (docDeleted == false) {
+            try (Searcher searcher = engine.acquireSearcher("test")) {
+                final TotalHitCountCollector collector = new TotalHitCountCollector();
+                searcher.searcher().search(new TermQuery(new Term("value", lastFieldValue)), collector);
+                assertThat(collector.getTotalHits(), equalTo(1));
+            }
+        }
+        return opsPerformed;
+    }
 
-        // now, the old one should not work
-        index = new Engine.Index(newUid("1"), doc, 1L, VersionType.INTERNAL.versionTypeForReplicationAndRecovery(), REPLICA, 0, -1, false);
-        try {
-            replicaEngine.index(index);
-            fail();
-        } catch (VersionConflictEngineException e) {
-            // all is well
+    public void testNonInternalVersioningOnPrimary() throws IOException {
+        final Set<VersionType> nonInternalVersioning = new HashSet<>(Arrays.asList(VersionType.values()));
+        nonInternalVersioning.remove(VersionType.INTERNAL);
+        final VersionType versionType = randomFrom(nonInternalVersioning);
+        final List<Engine.Operation> ops = generateSingleDocHistory(false, versionType, 2, 20);
+        final Engine.Operation lastOp = ops.get(ops.size() - 1);
+        final String lastFieldValue;
+        if (lastOp instanceof Engine.Index) {
+            Engine.Index index = (Engine.Index) lastOp;
+            lastFieldValue = index.docs().get(0).get("value");
+        } else {
+            // delete
+            lastFieldValue = null;
+        }
+        // other version types don't support out of order processing.
+        if (versionType == VersionType.EXTERNAL) {
+            shuffle(ops, random());
+        }
+        long highestOpVersion = Versions.NOT_FOUND;
+        long seqNo = -1;
+        boolean docDeleted = true;
+        for (Engine.Operation op : ops) {
+            logger.info("performing [{}], v [{}]", op.operationType().name().charAt(0), op.version());
+            if (op instanceof Engine.Index) {
+                final Engine.Index index = (Engine.Index) op;
+                Engine.IndexResult result = engine.index(index);
+                if (op.versionType().isVersionConflictForWrites(highestOpVersion, op.version(), docDeleted) == false) {
+                    seqNo++;
+                    assertThat(result.isCreated(), equalTo(docDeleted));
+                    assertThat(result.getVersion(), equalTo(op.version()));
+                    assertThat(result.hasFailure(), equalTo(false));
+                    assertThat(result.getFailure(), nullValue());
+                    docDeleted = false;
+                    highestOpVersion = op.version();
+                } else {
+                    assertThat(result.isCreated(), equalTo(false));
+                    assertThat(result.getVersion(), equalTo(highestOpVersion));
+                    assertThat(result.hasFailure(), equalTo(true));
+                    assertThat(result.getFailure(), instanceOf(VersionConflictEngineException.class));
+                }
+            } else {
+                final Engine.Delete delete = (Engine.Delete) op;
+                Engine.DeleteResult result = engine.delete(delete);
+                if (op.versionType().isVersionConflictForWrites(highestOpVersion, op.version(), docDeleted) == false) {
+                    seqNo++;
+                    assertThat(result.isFound(), equalTo(docDeleted == false));
+                    assertThat(result.getVersion(), equalTo(op.version()));
+                    assertThat(result.hasFailure(), equalTo(false));
+                    assertThat(result.getFailure(), nullValue());
+                    docDeleted = true;
+                    highestOpVersion = op.version();
+                } else {
+                    assertThat(result.isFound(), equalTo(docDeleted == false));
+                    assertThat(result.getVersion(), equalTo(highestOpVersion));
+                    assertThat(result.hasFailure(), equalTo(true));
+                    assertThat(result.getFailure(), instanceOf(VersionConflictEngineException.class));
+                }
+            }
+            if (randomBoolean()) {
+                engine.refresh("test");
+            }
+            if (randomBoolean()) {
+                engine.flush();
+            }
         }
 
-        // second version on replica should fail as well
-        try {
-            index = new Engine.Index(newUid("1"), doc, 2L
-                    , VersionType.INTERNAL.versionTypeForReplicationAndRecovery(), REPLICA, 0, -1, false);
-            replicaEngine.index(index);
-            assertThat(index.version(), equalTo(2L));
-        } catch (VersionConflictEngineException e) {
-            // all is well
+        assertVisibleCount(engine, docDeleted ? 0 : 1);
+        if (docDeleted == false) {
+            logger.info("searching for [{}]", lastFieldValue);
+            try (Searcher searcher = engine.acquireSearcher("test")) {
+                final TotalHitCountCollector collector = new TotalHitCountCollector();
+                searcher.searcher().search(new TermQuery(new Term("value", lastFieldValue)), collector);
+                assertThat(collector.getTotalHits(), equalTo(1));
+            }
         }
     }
 
-    public void testVersioningReplicaConflict2() {
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null);
-        Engine.Index index = new Engine.Index(newUid("1"), doc);
-        engine.index(index);
-        assertThat(index.version(), equalTo(1L));
-
-        // apply the first index to the replica, should work fine
-        index = new Engine.Index(newUid("1"), doc, 1L
-                , VersionType.INTERNAL.versionTypeForReplicationAndRecovery(), REPLICA, 0, -1, false);
-        replicaEngine.index(index);
-        assertThat(index.version(), equalTo(1L));
+    public void testVersioningPromotedReplica() throws IOException {
+        final List<Engine.Operation> replicaOps = generateSingleDocHistory(true, VersionType.INTERNAL, 2, 20);
+        List<Engine.Operation> primaryOps = generateSingleDocHistory(false, VersionType.INTERNAL, 2, 20);
+        Engine.Operation lastReplicaOp = replicaOps.get(replicaOps.size() - 1);
+        final boolean deletedOnReplica = lastReplicaOp instanceof Engine.Delete;
+        final long finalReplicaVersion = lastReplicaOp.version();
+        assertOpsOnReplica(replicaOps, replicaEngine, true);
+        assertOpsOnPrimary(primaryOps, finalReplicaVersion, deletedOnReplica, replicaEngine);
+    }
 
-        // index it again
-        index = new Engine.Index(newUid("1"), doc);
-        engine.index(index);
-        assertThat(index.version(), equalTo(2L));
+    public void testConcurrentExternalVersioningOnPrimary() throws IOException, InterruptedException {
+        final List<Engine.Operation> ops = generateSingleDocHistory(false, VersionType.EXTERNAL, 100, 300);
+        final Engine.Operation lastOp = ops.get(ops.size() - 1);
+        final String lastFieldValue;
+        if (lastOp instanceof Engine.Index) {
+            Engine.Index index = (Engine.Index) lastOp;
+            lastFieldValue = index.docs().get(0).get("value");
+        } else {
+            // delete
+            lastFieldValue = null;
+        }
+        shuffle(ops, random());
+        concurrentlyApplyOps(ops, engine);
 
-        // now delete it
-        Engine.Delete delete = new Engine.Delete("test", "1", newUid("1"));
-        engine.delete(delete);
-        assertThat(delete.version(), equalTo(3L));
+        assertVisibleCount(engine, lastFieldValue == null ? 0 : 1);
+        if (lastFieldValue != null) {
+            try (Searcher searcher = engine.acquireSearcher("test")) {
+                final TotalHitCountCollector collector = new TotalHitCountCollector();
+                searcher.searcher().search(new TermQuery(new Term("value", lastFieldValue)), collector);
+                assertThat(collector.getTotalHits(), equalTo(1));
+            }
+        }
+    }
 
-        // apply the delete on the replica (skipping the second index)
-        delete = new Engine.Delete("test", "1", newUid("1"), 3L
-                , VersionType.INTERNAL.versionTypeForReplicationAndRecovery(), REPLICA, 0, false);
-        replicaEngine.delete(delete);
-        assertThat(delete.version(), equalTo(3L));
+    public void testConcurrentGetAndSetOnPrimary() throws IOException, InterruptedException {
+        Thread[] thread = new Thread[randomIntBetween(3, 5)];
+        CountDownLatch startGun = new CountDownLatch(thread.length);
+        final int opsPerThread = randomIntBetween(10, 20);
+        class OpAndVersion {
+            final long version;
+            final String removed;
+            final String added;
+
+            OpAndVersion(long version, String removed, String added) {
+                this.version = version;
+                this.removed = removed;
+                this.added = added;
+            }
+        }
+        final AtomicInteger idGenerator = new AtomicInteger();
+        final Queue<OpAndVersion> history = ConcurrentCollections.newQueue();
+        ParsedDocument doc = testParsedDocument("1", "test", null, System.currentTimeMillis(), -1L, testDocument(), bytesArray(""), null);
+        final Term uidTerm = newUid(doc);
+        engine.index(indexForDoc(doc));
+        final Function<String, Searcher> searcherFactory = engine::acquireSearcher;
+        for (int i = 0; i < thread.length; i++) {
+            thread[i] = new Thread(() -> {
+                startGun.countDown();
+                try {
+                    startGun.await();
+                } catch (InterruptedException e) {
+                    throw new AssertionError(e);
+                }
+                for (int op = 0; op < opsPerThread; op++) {
+                    try (Engine.GetResult get = engine.get(new Engine.Get(true, doc.type(), doc.id(), uidTerm), searcherFactory)) {
+                        FieldsVisitor visitor = new FieldsVisitor(true);
+                        get.docIdAndVersion().context.reader().document(get.docIdAndVersion().docId, visitor);
+                        List<String> values = new ArrayList<>(Strings.commaDelimitedListToSet(visitor.source().utf8ToString()));
+                        String removed = op % 3 == 0 && values.size() > 0 ? values.remove(0) : null;
+                        String added = "v_" + idGenerator.incrementAndGet();
+                        values.add(added);
+                        Engine.Index index = new Engine.Index(uidTerm,
+                            testParsedDocument("1", "test", null, System.currentTimeMillis(), -1L, testDocument(),
+                                bytesArray(Strings.collectionToCommaDelimitedString(values)), null),
+                            get.version(), VersionType.INTERNAL,
+                            PRIMARY, System.currentTimeMillis(), -1, false);
+                        Engine.IndexResult indexResult = engine.index(index);
+                        if (indexResult.hasFailure() == false) {
+                            history.add(new OpAndVersion(indexResult.getVersion(), removed, added));
+                        }
 
-        // second time delete with same version should fail
-        try {
-            delete = new Engine.Delete("test", "1", newUid("1"), 3L
-                    , VersionType.INTERNAL.versionTypeForReplicationAndRecovery(), REPLICA, 0, false);
-            replicaEngine.delete(delete);
-            fail("excepted VersionConflictEngineException to be thrown");
-        } catch (VersionConflictEngineException e) {
-            // all is well
+                    } catch (IOException e) {
+                        throw new AssertionError(e);
+                    }
+                }
+            });
+            thread[i].start();
+        }
+        for (int i = 0; i < thread.length; i++) {
+            thread[i].join();
+        }
+        List<OpAndVersion> sortedHistory = new ArrayList<>(history);
+        sortedHistory.sort(Comparator.comparing(o -> o.version));
+        Set<String> currentValues = new HashSet<>();
+        for (int i = 0; i < sortedHistory.size(); i++) {
+            OpAndVersion op = sortedHistory.get(i);
+            if (i > 0) {
+                assertThat("duplicate version", op.version, not(equalTo(sortedHistory.get(i - 1).version)));
+            }
+            boolean exists = op.removed == null ? true : currentValues.remove(op.removed);
+            assertTrue(op.removed + " should exist", exists);
+            exists = currentValues.add(op.added);
+            assertTrue(op.added + " should not exist", exists);
         }
 
-        // now do the second index on the replica, it should fail
-        try {
-            index = new Engine.Index(newUid("1"), doc, 2L, VersionType.INTERNAL.versionTypeForReplicationAndRecovery(), REPLICA, 0, -1, false);
-            replicaEngine.index(index);
-            fail("excepted VersionConflictEngineException to be thrown");
-        } catch (VersionConflictEngineException e) {
-            // all is well
+        try (Engine.GetResult get = engine.get(new Engine.Get(true, doc.type(), doc.id(), uidTerm), searcherFactory)) {
+            FieldsVisitor visitor = new FieldsVisitor(true);
+            get.docIdAndVersion().context.reader().document(get.docIdAndVersion().docId, visitor);
+            List<String> values = Arrays.asList(Strings.commaDelimitedListToStringArray(visitor.source().utf8ToString()));
+            assertThat(currentValues, equalTo(new HashSet<>(values)));
         }
     }
 
-    public void testBasicCreatedFlag() {
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null);
-        Engine.Index index = new Engine.Index(newUid("1"), doc);
-        engine.index(index);
-        assertTrue(index.isCreated());
+    public void testBasicCreatedFlag() throws IOException {
+        ParsedDocument doc = testParsedDocument("1", "test", null, -1, -1, testDocument(), B_1, null);
+        Engine.Index index = indexForDoc(doc);
+        Engine.IndexResult indexResult = engine.index(index);
+        assertTrue(indexResult.isCreated());
 
-        index = new Engine.Index(newUid("1"), doc);
-        engine.index(index);
-        assertFalse(index.isCreated());
+        index = indexForDoc(doc);
+        indexResult = engine.index(index);
+        assertFalse(indexResult.isCreated());
 
-        engine.delete(new Engine.Delete(null, "1", newUid("1")));
+        engine.delete(new Engine.Delete("doc", "1", newUid(doc)));
 
-        index = new Engine.Index(newUid("1"), doc);
-        engine.index(index);
-        assertTrue(index.isCreated());
+        index = indexForDoc(doc);
+        indexResult = engine.index(index);
+        assertTrue(indexResult.isCreated());
     }
 
-    public void testCreatedFlagAfterFlush() {
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null);
-        Engine.Index index = new Engine.Index(newUid("1"), doc);
-        engine.index(index);
-        assertTrue(index.isCreated());
+    public void testCreatedFlagAfterFlush() throws IOException {
+        ParsedDocument doc = testParsedDocument("1", "doc", null, -1, -1, testDocument(), B_1, null);
+        Engine.Index index = indexForDoc(doc);
+        Engine.IndexResult indexResult = engine.index(index);
+        assertTrue(indexResult.isCreated());
 
-        engine.delete(new Engine.Delete(null, "1", newUid("1")));
+        engine.delete(new Engine.Delete("doc", "1", newUid(doc)));
 
         engine.flush();
 
-        index = new Engine.Index(newUid("1"), doc);
-        engine.index(index);
-        assertTrue(index.isCreated());
+        index = indexForDoc(doc);
+        indexResult = engine.index(index);
+        assertTrue(indexResult.isCreated());
     }
 
     private static class MockAppender extends AbstractAppender {
@@ -1510,19 +1749,19 @@ private static class MockAppender extends AbstractAppender {
 
         public boolean sawIndexWriterIFDMessage;
 
-        public MockAppender(final String name) throws IllegalAccessException {
-            super(name, RegexFilter.createFilter(".*(\n.*)*", new String[0], true, null, null), null);
+        MockAppender(final String name) throws IllegalAccessException {
+            super(name, RegexFilter.createFilter(".*(\n.*)*", new String[0], false, null, null), null);
         }
 
         @Override
         public void append(LogEvent event) {
             final String formattedMessage = event.getMessage().getFormattedMessage();
-            if (event.getLevel() == Level.TRACE && formattedMessage.contains("[index][1] ")) {
-                if (event.getLoggerName().endsWith("lucene.iw") &&
+            if (event.getLevel() == Level.TRACE && event.getMarker().getName().contains("[index][1] ")) {
+                if (event.getLoggerName().endsWith(".IW") &&
                     formattedMessage.contains("IW: apply all deletes during flush")) {
                     sawIndexWriterMessage = true;
                 }
-                if (event.getLoggerName().endsWith("lucene.iw.ifd")) {
+                if (event.getLoggerName().endsWith(".IFD")) {
                     sawIndexWriterIFDMessage = true;
                 }
             }
@@ -1532,92 +1771,89 @@ public void append(LogEvent event) {
     // #5891: make sure IndexWriter's infoStream output is
     // sent to lucene.iw with log level TRACE:
 
-    public void testIndexWriterInfoStream() throws IllegalAccessException {
+    public void testIndexWriterInfoStream() throws IllegalAccessException, IOException {
         assumeFalse("who tests the tester?", VERBOSE);
         MockAppender mockAppender = new MockAppender("testIndexWriterInfoStream");
+        mockAppender.start();
 
         Logger rootLogger = LogManager.getRootLogger();
         Level savedLevel = rootLogger.getLevel();
-        TestLoggers.addAppender(rootLogger, mockAppender);
+        Loggers.addAppender(rootLogger, mockAppender);
         Loggers.setLevel(rootLogger, Level.DEBUG);
         rootLogger = LogManager.getRootLogger();
 
         try {
             // First, with DEBUG, which should NOT log IndexWriter output:
-            ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null);
-            engine.index(new Engine.Index(newUid("1"), doc));
+            ParsedDocument doc = testParsedDocument("1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null);
+            engine.index(indexForDoc(doc));
             engine.flush();
             assertFalse(mockAppender.sawIndexWriterMessage);
 
             // Again, with TRACE, which should log IndexWriter output:
             Loggers.setLevel(rootLogger, Level.TRACE);
-            engine.index(new Engine.Index(newUid("2"), doc));
+            engine.index(indexForDoc(doc));
             engine.flush();
             assertTrue(mockAppender.sawIndexWriterMessage);
 
         } finally {
-            TestLoggers.removeAppender(rootLogger, mockAppender);
-            Loggers.setLevel(rootLogger, savedLevel.toString());
+            Loggers.removeAppender(rootLogger, mockAppender);
+            mockAppender.stop();
+            Loggers.setLevel(rootLogger, savedLevel);
         }
     }
 
     // #8603: make sure we can separately log IFD's messages
-    public void testIndexWriterIFDInfoStream() throws IllegalAccessException {
+    public void testIndexWriterIFDInfoStream() throws IllegalAccessException, IOException {
         assumeFalse("who tests the tester?", VERBOSE);
         MockAppender mockAppender = new MockAppender("testIndexWriterIFDInfoStream");
+        mockAppender.start();
 
-        final Logger iwIFDLogger;
-        if (LogManager.getContext(false).hasLogger("org.elasticsearch.index.engine.lucene.iw.ifd", new PrefixMessageFactory())) {
-            // Works when running this test inside Intellij:
-            iwIFDLogger = LogManager.getLogger("org.elasticsearch.index.engine.lucene.iw.ifd");
-            assertNotNull(iwIFDLogger);
-        } else {
-            // Works when running this test from command line:
-            assertTrue(LogManager.getContext(false).hasLogger("index.engine.lucene.iw.ifd", new PrefixMessageFactory()));
-            iwIFDLogger = LogManager.getLogger("index.engine.lucene.iw.ifd");
-        }
+        final Logger iwIFDLogger = Loggers.getLogger("org.elasticsearch.index.engine.Engine.IFD");
 
-        TestLoggers.addAppender(iwIFDLogger, mockAppender);
+        Loggers.addAppender(iwIFDLogger, mockAppender);
         Loggers.setLevel(iwIFDLogger, Level.DEBUG);
 
         try {
             // First, with DEBUG, which should NOT log IndexWriter output:
-            ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null);
-            engine.index(new Engine.Index(newUid("1"), doc));
+            ParsedDocument doc = testParsedDocument("1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null);
+            engine.index(indexForDoc(doc));
             engine.flush();
             assertFalse(mockAppender.sawIndexWriterMessage);
             assertFalse(mockAppender.sawIndexWriterIFDMessage);
 
             // Again, with TRACE, which should only log IndexWriter IFD output:
             Loggers.setLevel(iwIFDLogger, Level.TRACE);
-            engine.index(new Engine.Index(newUid("2"), doc));
+            engine.index(indexForDoc(doc));
             engine.flush();
             assertFalse(mockAppender.sawIndexWriterMessage);
             assertTrue(mockAppender.sawIndexWriterIFDMessage);
 
         } finally {
-            TestLoggers.removeAppender(iwIFDLogger, mockAppender);
+            Loggers.removeAppender(iwIFDLogger, mockAppender);
+            mockAppender.stop();
             Loggers.setLevel(iwIFDLogger, (Level) null);
         }
     }
 
     public void testEnableGcDeletes() throws Exception {
         try (Store store = createStore();
-            Engine engine = new InternalEngine(config(defaultSettings, store, createTempDir(), newMergePolicy(), IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP))) {
+            Engine engine = new InternalEngine(config(defaultSettings, store, createTempDir(), newMergePolicy(), IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, null))) {
             engine.config().setEnableGcDeletes(false);
 
+            final Function<String, Searcher> searcherFactory = engine::acquireSearcher;
+
             // Add document
             Document document = testDocument();
             document.add(new TextField("value", "test1", Field.Store.YES));
 
-            ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_2, null);
-            engine.index(new Engine.Index(newUid("1"), doc, 1, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), -1, false));
+            ParsedDocument doc = testParsedDocument("1", "test", null, -1, -1, document, B_2, null);
+            engine.index(new Engine.Index(newUid(doc), doc, 1, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), -1, false));
 
             // Delete document we just added:
-            engine.delete(new Engine.Delete("test", "1", newUid("1"), 10, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), false));
+            engine.delete(new Engine.Delete("test", "1", newUid(doc), 10, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime()));
 
             // Get should not find the document
-            Engine.GetResult getResult = engine.get(new Engine.Get(true, newUid("1")));
+            Engine.GetResult getResult = engine.get(new Engine.Get(true, doc.type(), doc.id(), newUid(doc)), searcherFactory);
             assertThat(getResult.exists(), equalTo(false));
 
             // Give the gc pruning logic a chance to kick in
@@ -1628,34 +1864,29 @@ public void testEnableGcDeletes() throws Exception {
             }
 
             // Delete non-existent document
-            engine.delete(new Engine.Delete("test", "2", newUid("2"), 10, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), false));
+            ParsedDocument doc2 = testParsedDocument("2", "test", null, -1, -1, document, B_2, null);
+            engine.delete(new Engine.Delete("test", "2", newUid(doc2), 10, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime()));
 
             // Get should not find the document (we never indexed uid=2):
-            getResult = engine.get(new Engine.Get(true, newUid("2")));
+            getResult = engine.get(new Engine.Get(true, doc.type(), doc.id(), newUid(doc2)), searcherFactory);
             assertThat(getResult.exists(), equalTo(false));
 
             // Try to index uid=1 with a too-old version, should fail:
-            try {
-                engine.index(new Engine.Index(newUid("1"), doc, 2, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), -1, false));
-                fail("did not hit expected exception");
-            } catch (VersionConflictEngineException vcee) {
-                // expected
-            }
+            Engine.IndexResult indexResult = engine.index(new Engine.Index(newUid(doc), doc, 2, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), -1, false));
+            assertTrue(indexResult.hasFailure());
+            assertThat(indexResult.getFailure(), instanceOf(VersionConflictEngineException.class));
 
             // Get should still not find the document
-            getResult = engine.get(new Engine.Get(true, newUid("1")));
+            getResult = engine.get(new Engine.Get(true, doc.type(), doc.id(), newUid(doc)), searcherFactory);
             assertThat(getResult.exists(), equalTo(false));
 
             // Try to index uid=2 with a too-old version, should fail:
-            try {
-                engine.index(new Engine.Index(newUid("2"), doc, 2, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), -1, false));
-                fail("did not hit expected exception");
-            } catch (VersionConflictEngineException vcee) {
-                // expected
-            }
+            indexResult = engine.index(new Engine.Index(newUid(doc2), doc2, 2, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), -1, false));
+            assertTrue(indexResult.hasFailure());
+            assertThat(indexResult.getFailure(), instanceOf(VersionConflictEngineException.class));
 
             // Get should not find the document
-            getResult = engine.get(new Engine.Get(true, newUid("2")));
+            getResult = engine.get(new Engine.Get(true, doc.type(), doc.id(), newUid(doc)), searcherFactory);
             assertThat(getResult.exists(), equalTo(false));
         }
     }
@@ -1664,6 +1895,20 @@ protected Term newUid(String id) {
         return new Term("_uid", id);
     }
 
+    protected Term newUid(ParsedDocument doc) {
+        return new Term("_uid", Uid.createUid(doc.type(), doc.id()));
+    }
+
+    private Engine.Index indexForDoc(ParsedDocument doc) {
+        return new Engine.Index(newUid(doc), doc);
+    }
+
+    private Engine.Index replicaIndexForDoc(ParsedDocument doc, long version, boolean isRetry) {
+        return  new Engine.Index(newUid(doc), doc, version,  VersionType.EXTERNAL,
+            Engine.Operation.Origin.REPLICA, System.nanoTime(),
+            IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, isRetry);
+    }
+
     public void testExtractShardId() {
         try (Engine.Searcher test = this.engine.acquireSearcher("test")) {
             ShardId shardId = ShardUtils.extractShardId(test.getDirectoryReader());
@@ -1739,23 +1984,19 @@ public void testMissingTranslog() throws IOException {
             // expected
         }
         // now it should be OK.
-        EngineConfig config = copy(config(defaultSettings, store, primaryTranslogDir, newMergePolicy(), IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP), EngineConfig.OpenMode.OPEN_INDEX_CREATE_TRANSLOG);
+        EngineConfig config = copy(config(defaultSettings, store, primaryTranslogDir, newMergePolicy(), IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, null), EngineConfig.OpenMode.OPEN_INDEX_CREATE_TRANSLOG);
         engine = new InternalEngine(config);
     }
 
     public void testTranslogReplayWithFailure() throws IOException {
         final int numDocs = randomIntBetween(1, 10);
         for (int i = 0; i < numDocs; i++) {
-            ParsedDocument doc = testParsedDocument(Integer.toString(i), Integer.toString(i), "test", null, -1, -1, testDocument(), new BytesArray("{}"), null);
-            Engine.Index firstIndexRequest = new Engine.Index(newUid(Integer.toString(i)), doc, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false);
-            engine.index(firstIndexRequest);
-            assertThat(firstIndexRequest.version(), equalTo(1L));
-        }
-        engine.refresh("test");
-        try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
-            TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), randomIntBetween(numDocs, numDocs + 10));
-            assertThat(topDocs.totalHits, equalTo(numDocs));
+            ParsedDocument doc = testParsedDocument(Integer.toString(i), "test", null, -1, -1, testDocument(), SOURCE, null);
+            Engine.Index firstIndexRequest = new Engine.Index(newUid(doc), doc, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false);
+            Engine.IndexResult indexResult = engine.index(firstIndexRequest);
+            assertThat(indexResult.getVersion(), equalTo(1L));
         }
+        assertVisibleCount(engine, numDocs);
         engine.close();
         final MockDirectoryWrapper directory = DirectoryUtils.getLeaf(store.directory(), MockDirectoryWrapper.class);
         if (directory != null) {
@@ -1787,25 +2028,33 @@ public void testTranslogReplayWithFailure() throws IOException {
             // no mock directory, no fun.
             engine = createEngine(store, primaryTranslogDir);
         }
-        try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
-            TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), randomIntBetween(numDocs, numDocs + 10));
-            assertThat(topDocs.totalHits, equalTo(numDocs));
+        assertVisibleCount(engine, numDocs, false);
+    }
+
+    private static void assertVisibleCount(InternalEngine engine, int numDocs) throws IOException {
+        assertVisibleCount(engine, numDocs, true);
+    }
+
+    private static void assertVisibleCount(InternalEngine engine, int numDocs, boolean refresh) throws IOException {
+        if (refresh) {
+            engine.refresh("test");
+        }
+        try (Searcher searcher = engine.acquireSearcher("test")) {
+            final TotalHitCountCollector collector = new TotalHitCountCollector();
+            searcher.searcher().search(new MatchAllDocsQuery(), collector);
+            assertThat(collector.getTotalHits(), equalTo(numDocs));
         }
     }
 
     public void testSkipTranslogReplay() throws IOException {
         final int numDocs = randomIntBetween(1, 10);
         for (int i = 0; i < numDocs; i++) {
-            ParsedDocument doc = testParsedDocument(Integer.toString(i), Integer.toString(i), "test", null, -1, -1, testDocument(), new BytesArray("{}"), null);
-            Engine.Index firstIndexRequest = new Engine.Index(newUid(Integer.toString(i)), doc, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false);
-            engine.index(firstIndexRequest);
-            assertThat(firstIndexRequest.version(), equalTo(1L));
-        }
-        engine.refresh("test");
-        try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
-            TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), randomIntBetween(numDocs, numDocs + 10));
-            assertThat(topDocs.totalHits, equalTo(numDocs));
+            ParsedDocument doc = testParsedDocument(Integer.toString(i), "test", null, -1, -1, testDocument(), SOURCE, null);
+            Engine.Index firstIndexRequest = new Engine.Index(newUid(doc), doc, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false);
+            Engine.IndexResult firstIndexResult = engine.index(firstIndexRequest);
+            assertThat(firstIndexResult.getVersion(), equalTo(1L));
         }
+        assertVisibleCount(engine, numDocs);
         engine.close();
         engine = new InternalEngine(engine.config());
 
@@ -1844,17 +2093,18 @@ public void testUpgradeOldIndex() throws IOException {
             Path[] list = filterExtraFSFiles(FileSystemUtils.files(unzipDataDir));
 
             if (list.length != 1) {
-                throw new IllegalStateException("Backwards index must contain exactly one cluster but was " + list.length + " " + Arrays.toString(list));
+                throw new IllegalStateException("Backwards index must contain exactly one cluster but was " + list.length
+                    + " " + Arrays.toString(list));
             }
+
             // the bwc scripts packs the indices under this path
-            Path src = list[0].resolve("nodes/0/indices/" + indexName);
-            Path translog = list[0].resolve("nodes/0/indices/" + indexName).resolve("0").resolve("translog");
-            assertTrue("[" + indexFile + "] missing index dir: " + src.toString(), Files.exists(src));
+            Path src = OldIndexUtils.getIndexDir(logger, indexName, indexFile.toString(), list[0]);
+            Path translog = src.resolve("0").resolve("translog");
             assertTrue("[" + indexFile + "] missing translog dir: " + translog.toString(), Files.exists(translog));
             Path[] tlogFiles = filterExtraFSFiles(FileSystemUtils.files(translog));
             assertEquals(Arrays.toString(tlogFiles), tlogFiles.length, 2); // ckp & tlog
             Path tlogFile = tlogFiles[0].getFileName().toString().endsWith("tlog") ? tlogFiles[0] : tlogFiles[1];
-            final long size = Files.size(tlogFiles[0]);
+            final long size = Files.size(tlogFile);
             logger.debug("upgrading index {} file: {} size: {}", indexName, tlogFiles[0].getFileName(), size);
             Directory directory = newFSDirectory(src.resolve("0").resolve("index"));
             Store store = createStore(directory);
@@ -1886,10 +2136,10 @@ public void testUpgradeOldIndex() throws IOException {
                 }
                 final int numExtraDocs = randomIntBetween(1, 10);
                 for (int i = 0; i < numExtraDocs; i++) {
-                    ParsedDocument doc = testParsedDocument("extra" + Integer.toString(i), "extra" + Integer.toString(i), "test", null, -1, -1, testDocument(), new BytesArray("{}"), null);
-                    Engine.Index firstIndexRequest = new Engine.Index(newUid(Integer.toString(i)), doc, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false);
-                    engine.index(firstIndexRequest);
-                    assertThat(firstIndexRequest.version(), equalTo(1L));
+                    ParsedDocument doc = testParsedDocument("extra" + Integer.toString(i), "test", null, -1, -1, testDocument(), new BytesArray("{}"), null);
+                    Engine.Index firstIndexRequest = new Engine.Index(newUid(doc), doc, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false);
+                    Engine.IndexResult firstIndexResult = engine.index(firstIndexRequest);
+                    assertThat(firstIndexResult.getVersion(), equalTo(1L));
                 }
                 engine.refresh("test");
                 try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
@@ -1915,16 +2165,12 @@ private Path[] filterExtraFSFiles(Path[] files) {
     public void testTranslogReplay() throws IOException {
         final int numDocs = randomIntBetween(1, 10);
         for (int i = 0; i < numDocs; i++) {
-            ParsedDocument doc = testParsedDocument(Integer.toString(i), Integer.toString(i), "test", null, -1, -1, testDocument(), new BytesArray("{}"), null);
-            Engine.Index firstIndexRequest = new Engine.Index(newUid(Integer.toString(i)), doc, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false);
-            engine.index(firstIndexRequest);
-            assertThat(firstIndexRequest.version(), equalTo(1L));
-        }
-        engine.refresh("test");
-        try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
-            TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), randomIntBetween(numDocs, numDocs + 10));
-            assertThat(topDocs.totalHits, equalTo(numDocs));
+            ParsedDocument doc = testParsedDocument(Integer.toString(i), "test", null, -1, -1, testDocument(), SOURCE, null);
+            Engine.Index firstIndexRequest = new Engine.Index(newUid(doc), doc, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false);
+            Engine.IndexResult firstIndexResult = engine.index(firstIndexRequest);
+            assertThat(firstIndexResult.getVersion(), equalTo(1L));
         }
+        assertVisibleCount(engine, numDocs);
 
         TranslogHandler parser = (TranslogHandler) engine.config().getTranslogRecoveryPerformer();
         parser.mappingUpdate = dynamicUpdate();
@@ -1933,10 +2179,7 @@ public void testTranslogReplay() throws IOException {
         engine = new InternalEngine(copy(engine.config(), EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG)); // we need to reuse the engine config unless the parser.mappingModified won't work
         engine.recoverFromTranslog();
 
-        try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
-            TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), randomIntBetween(numDocs, numDocs + 10));
-            assertThat(topDocs.totalHits, equalTo(numDocs));
-        }
+        assertVisibleCount(engine, numDocs, false);
         parser = (TranslogHandler) engine.config().getTranslogRecoveryPerformer();
         assertEquals(numDocs, parser.recoveredOps.get());
         if (parser.mappingUpdate != null) {
@@ -1948,29 +2191,25 @@ public void testTranslogReplay() throws IOException {
 
         engine.close();
         engine = createEngine(store, primaryTranslogDir);
-        try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
-            TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), randomIntBetween(numDocs, numDocs + 10));
-            assertThat(topDocs.totalHits, equalTo(numDocs));
-        }
+        assertVisibleCount(engine, numDocs, false);
         parser = (TranslogHandler) engine.config().getTranslogRecoveryPerformer();
         assertEquals(0, parser.recoveredOps.get());
 
         final boolean flush = randomBoolean();
         int randomId = randomIntBetween(numDocs + 1, numDocs + 10);
-        String uuidValue = "test#" + Integer.toString(randomId);
-        ParsedDocument doc = testParsedDocument(uuidValue, Integer.toString(randomId), "test", null, -1, -1, testDocument(), new BytesArray("{}"), null);
-        Engine.Index firstIndexRequest = new Engine.Index(newUid(uuidValue), doc, 1, VersionType.EXTERNAL, PRIMARY, System.nanoTime(), -1, false);
-        engine.index(firstIndexRequest);
-        assertThat(firstIndexRequest.version(), equalTo(1L));
+        ParsedDocument doc = testParsedDocument(Integer.toString(randomId), "test", null, -1, -1, testDocument(), new BytesArray("{}"), null);
+        Engine.Index firstIndexRequest = new Engine.Index(newUid(doc), doc, 1, VersionType.EXTERNAL, PRIMARY, System.nanoTime(), -1, false);
+        Engine.IndexResult firstIndexResult = engine.index(firstIndexRequest);
+        assertThat(firstIndexResult.getVersion(), equalTo(1L));
         if (flush) {
             engine.flush();
         }
 
-        doc = testParsedDocument(uuidValue, Integer.toString(randomId), "test", null, -1, -1, testDocument(), new BytesArray("{}"), null);
-        Engine.Index idxRequest = new Engine.Index(newUid(uuidValue), doc, 2, VersionType.EXTERNAL, PRIMARY, System.nanoTime(), -1, false);
-        engine.index(idxRequest);
+        doc = testParsedDocument(Integer.toString(randomId), "test", null, -1, -1, testDocument(), new BytesArray("{}"), null);
+        Engine.Index idxRequest = new Engine.Index(newUid(doc), doc, 2, VersionType.EXTERNAL, PRIMARY, System.nanoTime(), -1, false);
+        Engine.IndexResult indexResult = engine.index(idxRequest);
         engine.refresh("test");
-        assertThat(idxRequest.version(), equalTo(2L));
+        assertThat(indexResult.getVersion(), equalTo(2L));
         try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
             TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), numDocs + 1);
             assertThat(topDocs.totalHits, equalTo(numDocs + 1));
@@ -1984,7 +2223,7 @@ public void testTranslogReplay() throws IOException {
         }
         parser = (TranslogHandler) engine.config().getTranslogRecoveryPerformer();
         assertEquals(flush ? 1 : 2, parser.recoveredOps.get());
-        engine.delete(new Engine.Delete("test", Integer.toString(randomId), newUid(uuidValue)));
+        engine.delete(new Engine.Delete("test", Integer.toString(randomId), newUid(doc)));
         if (randomBoolean()) {
             engine.refresh("test");
         } else {
@@ -1999,28 +2238,30 @@ public void testTranslogReplay() throws IOException {
 
     public static class TranslogHandler extends TranslogRecoveryPerformer {
 
-        private final DocumentMapper docMapper;
+        private final MapperService mapperService;
         public Mapping mappingUpdate = null;
 
         public final AtomicInteger recoveredOps = new AtomicInteger(0);
 
-        public TranslogHandler(String indexName, Logger logger) {
+        public TranslogHandler(NamedXContentRegistry xContentRegistry, String indexName, Logger logger) {
             super(new ShardId("test", "_na_", 0), null, logger);
             Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
-            RootObjectMapper.Builder rootBuilder = new RootObjectMapper.Builder("test");
             Index index = new Index(indexName, "_na_");
             IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(index, settings);
-            AnalysisService analysisService = new AnalysisService(indexSettings, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
+            IndexAnalyzers indexAnalyzers = null;
+            NamedAnalyzer defaultAnalyzer = new NamedAnalyzer("default", AnalyzerScope.INDEX, new StandardAnalyzer());
+            indexAnalyzers = new IndexAnalyzers(indexSettings, defaultAnalyzer, defaultAnalyzer, defaultAnalyzer, Collections.emptyMap(), Collections.emptyMap());
             SimilarityService similarityService = new SimilarityService(indexSettings, Collections.emptyMap());
             MapperRegistry mapperRegistry = new IndicesModule(Collections.emptyList()).getMapperRegistry();
-            MapperService mapperService = new MapperService(indexSettings, analysisService, similarityService, mapperRegistry, () -> null);
-            DocumentMapper.Builder b = new DocumentMapper.Builder(rootBuilder, mapperService);
-            this.docMapper = b.build(mapperService);
+            mapperService = new MapperService(indexSettings, indexAnalyzers, xContentRegistry, similarityService, mapperRegistry,
+                    () -> null);
         }
 
         @Override
         protected DocumentMapperForType docMapper(String type) {
-            return new DocumentMapperForType(docMapper, mappingUpdate);
+            RootObjectMapper.Builder rootBuilder = new RootObjectMapper.Builder(type);
+            DocumentMapper.Builder b = new DocumentMapper.Builder(rootBuilder, mapperService);
+            return new DocumentMapperForType(b.build(mapperService), mappingUpdate);
         }
 
         @Override
@@ -2032,16 +2273,12 @@ protected void operationProcessed() {
     public void testRecoverFromForeignTranslog() throws IOException {
         final int numDocs = randomIntBetween(1, 10);
         for (int i = 0; i < numDocs; i++) {
-            ParsedDocument doc = testParsedDocument(Integer.toString(i), Integer.toString(i), "test", null, -1, -1, testDocument(), new BytesArray("{}"), null);
-            Engine.Index firstIndexRequest = new Engine.Index(newUid(Integer.toString(i)), doc, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false);
-            engine.index(firstIndexRequest);
-            assertThat(firstIndexRequest.version(), equalTo(1L));
-        }
-        engine.refresh("test");
-        try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
-            TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), randomIntBetween(numDocs, numDocs + 10));
-            assertThat(topDocs.totalHits, equalTo(numDocs));
+            ParsedDocument doc = testParsedDocument(Integer.toString(i), "test", null, -1, -1, testDocument(), new BytesArray("{}"), null);
+            Engine.Index firstIndexRequest = new Engine.Index(newUid(doc), doc, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false);
+            Engine.IndexResult indexResult = engine.index(firstIndexRequest);
+            assertThat(indexResult.getVersion(), equalTo(1L));
         }
+        assertVisibleCount(engine, numDocs);
         Translog.TranslogGeneration generation = engine.getTranslog().getGeneration();
         engine.close();
 
@@ -2068,10 +2305,7 @@ public void testRecoverFromForeignTranslog() throws IOException {
         }
 
         engine = createEngine(store, primaryTranslogDir); // and recover again!
-        try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
-            TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), randomIntBetween(numDocs, numDocs + 10));
-            assertThat(topDocs.totalHits, equalTo(numDocs));
-        }
+        assertVisibleCount(engine, numDocs, false);
     }
 
     public void testShardNotAvailableExceptionWhenEngineClosedConcurrently() throws IOException, InterruptedException {
@@ -2112,16 +2346,71 @@ public void run() {
         assertTrue("expected an Exception that signals shard is not available", TransportActions.isShardNotAvailableException(exception.get()));
     }
 
+    /**
+     * Tests that when the the close method returns the engine is actually guaranteed to have cleaned up and that resources are closed
+     */
+    public void testConcurrentEngineClosed() throws BrokenBarrierException, InterruptedException {
+        Thread[] closingThreads = new Thread[3];
+        CyclicBarrier barrier = new CyclicBarrier(1 + closingThreads.length + 1);
+        Thread failEngine = new Thread(new AbstractRunnable() {
+            @Override
+            public void onFailure(Exception e) {
+                throw new AssertionError(e);
+            }
+
+            @Override
+            protected void doRun() throws Exception {
+                barrier.await();
+                engine.failEngine("test", new RuntimeException("test"));
+            }
+        });
+        failEngine.start();
+        for (int i = 0;i < closingThreads.length ; i++) {
+            boolean flushAndClose = randomBoolean();
+            closingThreads[i] = new Thread(new AbstractRunnable() {
+                @Override
+                public void onFailure(Exception e) {
+                    throw new AssertionError(e);
+                }
+
+                @Override
+                protected void doRun() throws Exception {
+                    barrier.await();
+                    if (flushAndClose) {
+                        engine.flushAndClose();
+                    } else {
+                        engine.close();
+                    }
+                    // try to acquire the writer lock - i.e., everything is closed, we need to synchronize
+                    // to avoid races between closing threads
+                    synchronized (closingThreads) {
+                        try (Lock ignored = store.directory().obtainLock(IndexWriter.WRITE_LOCK_NAME)) {
+                            // all good.
+                        }
+                    }
+                }
+            });
+            closingThreads[i].setName("closingThread_" + i);
+            closingThreads[i].start();
+        }
+        barrier.await();
+        failEngine.join();
+        for (Thread t : closingThreads) {
+            t.join();
+        }
+    }
+
     public void testCurrentTranslogIDisCommitted() throws IOException {
         try (Store store = createStore()) {
-            EngineConfig config = config(defaultSettings, store, createTempDir(), newMergePolicy(), IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP);
+            EngineConfig config = config(defaultSettings, store, createTempDir(), newMergePolicy(), IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, null);
 
             // create
             {
-                ParsedDocument doc = testParsedDocument(Integer.toString(0), Integer.toString(0), "test", null, -1, -1, testDocument(), new BytesArray("{}"), null);
-                Engine.Index firstIndexRequest = new Engine.Index(newUid(Integer.toString(0)), doc, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false);
+                ParsedDocument doc = testParsedDocument(Integer.toString(0), "test", null, -1, -1, testDocument(), new BytesArray("{}"), null);
+                Engine.Index firstIndexRequest = new Engine.Index(newUid(doc), doc, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false);
 
                 try (InternalEngine engine = new InternalEngine(copy(config, EngineConfig.OpenMode.CREATE_INDEX_AND_TRANSLOG))){
+                    assertFalse(engine.isRecovering());
                     engine.index(firstIndexRequest);
 
                     expectThrows(IllegalStateException.class, () -> engine.recoverFromTranslog());
@@ -2134,6 +2423,7 @@ public void testCurrentTranslogIDisCommitted() throws IOException {
             {
                 for (int i = 0; i < 2; i++) {
                     try (InternalEngine engine = new InternalEngine(copy(config, EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG))) {
+                        assertTrue(engine.isRecovering());
                         Map<String, String> userData = engine.getLastCommittedSegmentInfos().getUserData();
                         if (i == 0) {
                             assertEquals("1", userData.get(Translog.TRANSLOG_GENERATION_KEY));
@@ -2175,57 +2465,192 @@ public void testCurrentTranslogIDisCommitted() throws IOException {
         }
     }
 
-    public void testDocStats() throws IOException {
-        final int numDocs = randomIntBetween(2, 10); // at least 2 documents otherwise we don't see any deletes below
-        for (int i = 0; i < numDocs; i++) {
-            ParsedDocument doc = testParsedDocument(Integer.toString(i), Integer.toString(i), "test", null, -1, -1, testDocument(), new BytesArray("{}"), null);
-            Engine.Index firstIndexRequest = new Engine.Index(newUid(Integer.toString(i)), doc, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false);
-            engine.index(firstIndexRequest);
-            assertThat(firstIndexRequest.version(), equalTo(1L));
-        }
-        DocsStats docStats = engine.getDocStats();
-        assertEquals(numDocs, docStats.getCount());
-        assertEquals(0, docStats.getDeleted());
-        engine.forceMerge(randomBoolean(), 1, false, false, false);
-
-        ParsedDocument doc = testParsedDocument(Integer.toString(0), Integer.toString(0), "test", null, -1, -1, testDocument(), new BytesArray("{}"), null);
-        Engine.Index firstIndexRequest = new Engine.Index(newUid(Integer.toString(0)), doc, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false);
-        engine.index(firstIndexRequest);
-        assertThat(firstIndexRequest.version(), equalTo(2L));
-        engine.flush(); // flush - buffered deletes are not counted
-        docStats = engine.getDocStats();
-        assertEquals(1, docStats.getDeleted());
-        assertEquals(numDocs, docStats.getCount());
-        engine.forceMerge(randomBoolean(), 1, false, false, false);
-        docStats = engine.getDocStats();
-        assertEquals(0, docStats.getDeleted());
-        assertEquals(numDocs, docStats.getCount());
-    }
-
-    public void testDoubleDelivery() throws IOException {
-        final ParsedDocument doc = testParsedDocument("1", "1", "test", null, 100, -1, testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null);
-        Engine.Index operation = randomAppendOnly(1, doc, false);
-        Engine.Index retry = randomAppendOnly(1, doc, true);
+    private static class ThrowingIndexWriter extends IndexWriter {
+        private AtomicReference<Supplier<Exception>> failureToThrow = new AtomicReference<>();
+
+        ThrowingIndexWriter(Directory d, IndexWriterConfig conf) throws IOException {
+            super(d, conf);
+        }
+
+        @Override
+        public long addDocument(Iterable<? extends IndexableField> doc) throws IOException {
+            maybeThrowFailure();
+            return super.addDocument(doc);
+        }
+
+        private void maybeThrowFailure() throws IOException {
+            if (failureToThrow.get() != null) {
+                Exception failure = failureToThrow.get().get();
+                if (failure instanceof RuntimeException) {
+                    throw (RuntimeException) failure;
+                } else if (failure instanceof IOException) {
+                    throw (IOException) failure;
+                } else {
+                    assert false: "unsupported failure class: " + failure.getClass().getCanonicalName();
+                }
+            }
+        }
+
+        @Override
+        public long deleteDocuments(Term... terms) throws IOException {
+            maybeThrowFailure();
+            return super.deleteDocuments(terms);
+        }
+
+        public void setThrowFailure(Supplier<Exception> failureSupplier) {
+            failureToThrow.set(failureSupplier);
+        }
+
+        public void clearFailure() {
+            failureToThrow.set(null);
+        }
+    }
+
+    public void testHandleDocumentFailure() throws Exception {
+        try (Store store = createStore()) {
+            final ParsedDocument doc1 = testParsedDocument("1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null);
+            final ParsedDocument doc2 = testParsedDocument("2", "test", null, -1, -1, testDocumentWithTextField(), B_1, null);
+            final ParsedDocument doc3 = testParsedDocument("3", "test", null, -1, -1, testDocumentWithTextField(), B_1, null);
+
+            ThrowingIndexWriter throwingIndexWriter = new ThrowingIndexWriter(store.directory(), new IndexWriterConfig());
+            try (Engine engine = createEngine(defaultSettings, store, createTempDir(), NoMergePolicy.INSTANCE, () -> throwingIndexWriter)) {
+                // test document failure while indexing
+                if (randomBoolean()) {
+                    throwingIndexWriter.setThrowFailure(() -> new IOException("simulated"));
+                } else {
+                    throwingIndexWriter.setThrowFailure(() -> new IllegalArgumentException("simulated max token length"));
+                }
+                Engine.IndexResult indexResult = engine.index(indexForDoc(doc1));
+                assertNotNull(indexResult.getFailure());
+
+                throwingIndexWriter.clearFailure();
+                indexResult = engine.index(indexForDoc(doc1));
+                assertNull(indexResult.getFailure());
+                engine.index(indexForDoc(doc2));
+
+                // test failure while deleting
+                // all these simulated exceptions are not fatal to the IW so we treat them as document failures
+                if (randomBoolean()) {
+                    throwingIndexWriter.setThrowFailure(() -> new IOException("simulated"));
+                    assertThat(engine.delete(new Engine.Delete("test", "1", newUid(doc1))).getFailure(), instanceOf(IOException.class));
+                } else {
+                    throwingIndexWriter.setThrowFailure(() -> new IllegalArgumentException("simulated max token length"));
+                    assertThat(engine.delete(new Engine.Delete("test", "1", newUid(doc1))).getFailure(),
+                        instanceOf(IllegalArgumentException.class));
+                }
+
+                // test non document level failure is thrown
+                if (randomBoolean()) {
+                    // simulate close by corruption
+                    throwingIndexWriter.setThrowFailure(null);
+                    UncheckedIOException uncheckedIOException = expectThrows(UncheckedIOException.class, () -> {
+                        Engine.Index index = indexForDoc(doc3);
+                        index.parsedDoc().rootDoc().add(new StoredField("foo", "bar") {
+                            // this is a hack to add a failure during store document which triggers a tragic event
+                            // and in turn fails the engine
+                            @Override
+                            public BytesRef binaryValue() {
+                                throw new UncheckedIOException(new MockDirectoryWrapper.FakeIOException());
+                            }
+                        });
+                        engine.index(index);
+                    });
+                    assertTrue(uncheckedIOException.getCause() instanceof MockDirectoryWrapper.FakeIOException);
+                } else {
+                    // normal close
+                    engine.close();
+                }
+                // now the engine is closed check we respond correctly
+                try {
+                    if (randomBoolean()) {
+                        engine.index(indexForDoc(doc1));
+                    } else {
+                        engine.delete(new Engine.Delete("test", "", newUid(doc1)));
+                    }
+                    fail("engine should be closed");
+                } catch (Exception e) {
+                    assertThat(e, instanceOf(AlreadyClosedException.class));
+                }
+            }
+        }
+    }
+
+    public void testDoubleDeliveryPrimary() throws IOException {
+        final ParsedDocument doc = testParsedDocument("1", "test", null, System.currentTimeMillis(), -1L,
+            testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null);
+        final Engine.Index operation = appendOnlyPrimary(doc, false, 1);
+        final Engine.Index retry = appendOnlyPrimary(doc, true, 1);
         if (randomBoolean()) {
-            engine.index(operation);
+            Engine.IndexResult indexResult = engine.index(operation);
             assertFalse(engine.indexWriterHasDeletions());
             assertEquals(0, engine.getNumVersionLookups());
-            assertNotNull(operation.getTranslogLocation());
-            engine.index(retry);
+            assertNotNull(indexResult.getTranslogLocation());
+            Engine.IndexResult retryResult = engine.index(retry);
             assertTrue(engine.indexWriterHasDeletions());
             assertEquals(0, engine.getNumVersionLookups());
-            assertNotNull(retry.getTranslogLocation());
-            assertTrue(retry.getTranslogLocation().compareTo(operation.getTranslogLocation()) > 0);
+            assertNotNull(retryResult.getTranslogLocation());
+            assertTrue(retryResult.getTranslogLocation().compareTo(indexResult.getTranslogLocation()) > 0);
         } else {
-            engine.index(retry);
+            Engine.IndexResult retryResult = engine.index(retry);
             assertTrue(engine.indexWriterHasDeletions());
             assertEquals(0, engine.getNumVersionLookups());
-            assertNotNull(retry.getTranslogLocation());
-            engine.index(operation);
+            assertNotNull(retryResult.getTranslogLocation());
+            Engine.IndexResult indexResult = engine.index(operation);
             assertTrue(engine.indexWriterHasDeletions());
             assertEquals(0, engine.getNumVersionLookups());
-            assertNotNull(retry.getTranslogLocation());
-            assertTrue(retry.getTranslogLocation().compareTo(operation.getTranslogLocation()) < 0);
+            assertNotNull(retryResult.getTranslogLocation());
+            assertTrue(retryResult.getTranslogLocation().compareTo(indexResult.getTranslogLocation()) < 0);
+        }
+
+        engine.refresh("test");
+        try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
+            TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), 10);
+            assertEquals(1, topDocs.totalHits);
+        }
+        if (randomBoolean()) {
+            Engine.IndexResult indexResult = engine.index(operation);
+            assertNotNull(indexResult.getTranslogLocation());
+            Engine.IndexResult retryResult = engine.index(retry);
+            assertNotNull(retryResult.getTranslogLocation());
+            assertTrue(retryResult.getTranslogLocation().compareTo(indexResult.getTranslogLocation()) > 0);
+        } else {
+            Engine.IndexResult retryResult = engine.index(retry);
+            assertNotNull(retryResult.getTranslogLocation());
+            Engine.IndexResult indexResult = engine.index(operation);
+            assertNotNull(indexResult.getTranslogLocation());
+            assertTrue(retryResult.getTranslogLocation().compareTo(indexResult.getTranslogLocation()) < 0);
+        }
+
+        engine.refresh("test");
+        try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
+            TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), 10);
+            assertEquals(1, topDocs.totalHits);
+        }
+    }
+
+    public void testDoubleDeliveryReplicaAppendingOnly() throws IOException {
+        final ParsedDocument doc = testParsedDocument("1", "test", null, System.currentTimeMillis(), -1,
+            testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null);
+        final Engine.Index operation = appendOnlyReplica(doc, false, 1);
+        final Engine.Index retry = appendOnlyReplica(doc, true, 1);
+        if (randomBoolean()) {
+            Engine.IndexResult indexResult = engine.index(operation);
+            assertFalse(engine.indexWriterHasDeletions());
+            assertEquals(0, engine.getNumVersionLookups());
+            assertNotNull(indexResult.getTranslogLocation());
+            Engine.IndexResult retryResult = engine.index(retry);
+            assertFalse(engine.indexWriterHasDeletions());
+            assertEquals(1, engine.getNumVersionLookups());
+            assertNull(retryResult.getTranslogLocation()); // we didn't index it nor put it in the translog
+        } else {
+            Engine.IndexResult retryResult = engine.index(retry);
+            assertFalse(engine.indexWriterHasDeletions());
+            assertEquals(1, engine.getNumVersionLookups());
+            assertNotNull(retryResult.getTranslogLocation());
+            Engine.IndexResult indexResult = engine.index(operation);
+            assertFalse(engine.indexWriterHasDeletions());
+            assertEquals(2, engine.getNumVersionLookups());
+            assertNull(indexResult.getTranslogLocation()); // we didn't index it nor put it in the translog
         }
 
         engine.refresh("test");
@@ -2233,20 +2658,16 @@ public void testDoubleDelivery() throws IOException {
             TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), 10);
             assertEquals(1, topDocs.totalHits);
         }
-        operation = randomAppendOnly(1, doc, false);
-        retry = randomAppendOnly(1, doc, true);
         if (randomBoolean()) {
-            engine.index(operation);
-            assertNotNull(operation.getTranslogLocation());
-            engine.index(retry);
-            assertNotNull(retry.getTranslogLocation());
-            assertTrue(retry.getTranslogLocation().compareTo(operation.getTranslogLocation()) > 0);
+            Engine.IndexResult indexResult = engine.index(operation);
+            assertNull(indexResult.getTranslogLocation()); // we don't index because a retry has already been processed.
+            Engine.IndexResult retryResult = engine.index(retry);
+            assertNull(retryResult.getTranslogLocation());
         } else {
-            engine.index(retry);
-            assertNotNull(retry.getTranslogLocation());
-            engine.index(operation);
-            assertNotNull(retry.getTranslogLocation());
-            assertTrue(retry.getTranslogLocation().compareTo(operation.getTranslogLocation()) < 0);
+            Engine.IndexResult retryResult = engine.index(retry);
+            assertNull(retryResult.getTranslogLocation());
+            Engine.IndexResult indexResult = engine.index(operation);
+            assertNull(indexResult.getTranslogLocation());
         }
 
         engine.refresh("test");
@@ -2256,32 +2677,74 @@ public void testDoubleDelivery() throws IOException {
         }
     }
 
+    public void testDoubleDeliveryReplica() throws IOException {
+        final ParsedDocument doc = testParsedDocument("1", "test", null, System.currentTimeMillis(), -1L,
+            testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null);
+        Engine.Index operation = replicaIndexForDoc(doc, 20, false);
+        Engine.Index duplicate = replicaIndexForDoc(doc, 20, true);
+        if (randomBoolean()) {
+            Engine.IndexResult indexResult = engine.index(operation);
+            assertFalse(engine.indexWriterHasDeletions());
+            assertEquals(1, engine.getNumVersionLookups());
+            assertNotNull(indexResult.getTranslogLocation());
+            if (randomBoolean()) {
+                engine.refresh("test");
+            }
+            Engine.IndexResult retryResult = engine.index(duplicate);
+            assertFalse(engine.indexWriterHasDeletions());
+            assertEquals(2, engine.getNumVersionLookups());
+            assertNull(retryResult.getTranslogLocation());
+        } else {
+            Engine.IndexResult retryResult = engine.index(duplicate);
+            assertFalse(engine.indexWriterHasDeletions());
+            assertEquals(1, engine.getNumVersionLookups());
+            assertNotNull(retryResult.getTranslogLocation());
+            if (randomBoolean()) {
+                engine.refresh("test");
+            }
+            Engine.IndexResult indexResult = engine.index(operation);
+            assertFalse(engine.indexWriterHasDeletions());
+            assertEquals(2, engine.getNumVersionLookups());
+            assertNull(indexResult.getTranslogLocation()); // we didn't index, no need to put in translog
+        }
+
+        engine.refresh("test");
+        try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
+            TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), 10);
+            assertEquals(1, topDocs.totalHits);
+        }
+        engine.refresh("test");
+        try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
+            TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), 10);
+            assertEquals(1, topDocs.totalHits);
+        }
+    }
 
     public void testRetryWithAutogeneratedIdWorksAndNoDuplicateDocs() throws IOException {
 
-        final ParsedDocument doc = testParsedDocument("1", "1", "test", null, 100, -1, testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null);
+        final ParsedDocument doc = testParsedDocument("1", "test", null, 100, -1, testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null);
         boolean isRetry = false;
         long autoGeneratedIdTimestamp = 0;
 
-        Engine.Index index = new Engine.Index(newUid("1"), doc, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime(), autoGeneratedIdTimestamp, isRetry);
-        engine.index(index);
-        assertThat(index.version(), equalTo(1L));
+        Engine.Index index = new Engine.Index(newUid(doc), doc, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime(), autoGeneratedIdTimestamp, isRetry);
+        Engine.IndexResult indexResult = engine.index(index);
+        assertThat(indexResult.getVersion(), equalTo(1L));
 
-        index = new Engine.Index(newUid("1"), doc, index.version(), index.versionType().versionTypeForReplicationAndRecovery(), REPLICA, System.nanoTime(), autoGeneratedIdTimestamp, isRetry);
-        replicaEngine.index(index);
-        assertThat(index.version(), equalTo(1L));
+        index = new Engine.Index(newUid(doc), doc, indexResult.getVersion(), index.versionType().versionTypeForReplicationAndRecovery(), REPLICA, System.nanoTime(), autoGeneratedIdTimestamp, isRetry);
+        Engine.IndexResult replicaResult = replicaEngine.index(index);
+        assertThat(replicaResult.getVersion(), equalTo(1L));
 
         isRetry = true;
-        index = new Engine.Index(newUid("1"), doc, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime(), autoGeneratedIdTimestamp, isRetry);
-        engine.index(index);
-        assertThat(index.version(), equalTo(1L));
+        index = new Engine.Index(newUid(doc), doc, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime(), autoGeneratedIdTimestamp, isRetry);
+        Engine.IndexResult indexResult1 = engine.index(index);
+        assertThat(indexResult1.getVersion(), equalTo(1L));
         engine.refresh("test");
         try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
             TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), 10);
             assertEquals(1, topDocs.totalHits);
         }
 
-        index = new Engine.Index(newUid("1"), doc, index.version(), index.versionType().versionTypeForReplicationAndRecovery(), REPLICA, System.nanoTime(), autoGeneratedIdTimestamp, isRetry);
+        index = new Engine.Index(newUid(doc), doc, indexResult.getVersion(), index.versionType().versionTypeForReplicationAndRecovery(), REPLICA, System.nanoTime(), autoGeneratedIdTimestamp, isRetry);
         replicaEngine.index(index);
         replicaEngine.refresh("test");
         try (Engine.Searcher searcher = replicaEngine.acquireSearcher("test")) {
@@ -2292,30 +2755,30 @@ public void testRetryWithAutogeneratedIdWorksAndNoDuplicateDocs() throws IOExcep
 
     public void testRetryWithAutogeneratedIdsAndWrongOrderWorksAndNoDuplicateDocs() throws IOException {
 
-        final ParsedDocument doc = testParsedDocument("1", "1", "test", null, 100, -1, testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null);
+        final ParsedDocument doc = testParsedDocument("1", "test", null, 100, -1, testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null);
         boolean isRetry = true;
         long autoGeneratedIdTimestamp = 0;
 
 
-        Engine.Index firstIndexRequest = new Engine.Index(newUid("1"), doc, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime(), autoGeneratedIdTimestamp, isRetry);
-        engine.index(firstIndexRequest);
-        assertThat(firstIndexRequest.version(), equalTo(1L));
+        Engine.Index firstIndexRequest = new Engine.Index(newUid(doc), doc, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime(), autoGeneratedIdTimestamp, isRetry);
+        Engine.IndexResult firstIndexResult = engine.index(firstIndexRequest);
+        assertThat(firstIndexResult.getVersion(), equalTo(1L));
 
-        Engine.Index firstIndexRequestReplica = new Engine.Index(newUid("1"), doc, firstIndexRequest.version(), firstIndexRequest.versionType().versionTypeForReplicationAndRecovery(), REPLICA, System.nanoTime(), autoGeneratedIdTimestamp, isRetry);
-        replicaEngine.index(firstIndexRequestReplica);
-        assertThat(firstIndexRequestReplica.version(), equalTo(1L));
+        Engine.Index firstIndexRequestReplica = new Engine.Index(newUid(doc), doc, firstIndexResult.getVersion(), firstIndexRequest.versionType().versionTypeForReplicationAndRecovery(), REPLICA, System.nanoTime(), autoGeneratedIdTimestamp, isRetry);
+        Engine.IndexResult firstIndexReplicaResult = replicaEngine.index(firstIndexRequestReplica);
+        assertThat(firstIndexReplicaResult.getVersion(), equalTo(1L));
 
         isRetry = false;
-        Engine.Index secondIndexRequest = new Engine.Index(newUid("1"), doc, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime(), autoGeneratedIdTimestamp, isRetry);
-        engine.index(secondIndexRequest);
-        assertTrue(secondIndexRequest.isCreated());
+        Engine.Index secondIndexRequest = new Engine.Index(newUid(doc), doc, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime(), autoGeneratedIdTimestamp, isRetry);
+        Engine.IndexResult indexResult = engine.index(secondIndexRequest);
+        assertTrue(indexResult.isCreated());
         engine.refresh("test");
         try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
             TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), 10);
             assertEquals(1, topDocs.totalHits);
         }
 
-        Engine.Index secondIndexRequestReplica = new Engine.Index(newUid("1"), doc, firstIndexRequest.version(), firstIndexRequest.versionType().versionTypeForReplicationAndRecovery(), REPLICA, System.nanoTime(), autoGeneratedIdTimestamp, isRetry);
+        Engine.Index secondIndexRequestReplica = new Engine.Index(newUid(doc), doc, firstIndexResult.getVersion(), firstIndexRequest.versionType().versionTypeForReplicationAndRecovery(), REPLICA, System.nanoTime(), autoGeneratedIdTimestamp, isRetry);
         replicaEngine.index(secondIndexRequestReplica);
         replicaEngine.refresh("test");
         try (Engine.Searcher searcher = replicaEngine.acquireSearcher("test")) {
@@ -2324,21 +2787,42 @@ public void testRetryWithAutogeneratedIdsAndWrongOrderWorksAndNoDuplicateDocs()
         }
     }
 
-    public Engine.Index randomAppendOnly(int docId, ParsedDocument doc, boolean retry) {
+    public Engine.Index randomAppendOnly(ParsedDocument doc, boolean retry, final long autoGeneratedIdTimestamp) {
         if (randomBoolean()) {
-            return new Engine.Index(newUid(Integer.toString(docId)), doc, Versions.MATCH_ANY, VersionType.INTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), docId, retry);
+            return appendOnlyPrimary(doc, retry, autoGeneratedIdTimestamp);
+        } else {
+            return appendOnlyReplica(doc, retry, autoGeneratedIdTimestamp);
         }
-        return new Engine.Index(newUid(Integer.toString(docId)), doc, 1, VersionType.EXTERNAL, Engine.Operation.Origin.REPLICA, System.nanoTime(), docId, retry);
+    }
+
+    public Engine.Index appendOnlyPrimary(ParsedDocument doc, boolean retry, final long autoGeneratedIdTimestamp) {
+        return new Engine.Index(newUid(doc), doc, Versions.MATCH_ANY,
+            VersionType.INTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), autoGeneratedIdTimestamp, retry);
+    }
+
+    public Engine.Index appendOnlyReplica(ParsedDocument doc, boolean retry, final long autoGeneratedIdTimestamp) {
+        return new Engine.Index(newUid(doc), doc, 1, VersionType.EXTERNAL,
+            Engine.Operation.Origin.REPLICA, System.nanoTime(), autoGeneratedIdTimestamp, retry);
     }
 
     public void testRetryConcurrently() throws InterruptedException, IOException {
         Thread[] thread = new Thread[randomIntBetween(3, 5)];
         int numDocs = randomIntBetween(1000, 10000);
         List<Engine.Index> docs = new ArrayList<>();
+        final boolean primary = randomBoolean();
         for (int i = 0; i < numDocs; i++) {
-            final ParsedDocument doc = testParsedDocument(Integer.toString(i), Integer.toString(i), "test", null, i, -1, testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null);
-            Engine.Index originalIndex = randomAppendOnly(i, doc, false);
-            Engine.Index retryIndex = randomAppendOnly(i, doc, true);
+            final ParsedDocument doc = testParsedDocument(Integer.toString(i), "test", null,
+                System.currentTimeMillis(), -1L, testDocumentWithTextField(),
+                new BytesArray("{}".getBytes(Charset.defaultCharset())), null);
+            final Engine.Index originalIndex;
+            final Engine.Index retryIndex;
+            if (primary) {
+               originalIndex = appendOnlyPrimary(doc, false, i);
+               retryIndex = appendOnlyPrimary(doc, true, i);
+            } else {
+                originalIndex = appendOnlyReplica(doc, false, i);
+                retryIndex = appendOnlyReplica(doc, true, i);
+            }
             docs.add(originalIndex);
             docs.add(retryIndex);
         }
@@ -2346,49 +2830,91 @@ public void testRetryConcurrently() throws InterruptedException, IOException {
         CountDownLatch startGun = new CountDownLatch(thread.length);
         AtomicInteger offset = new AtomicInteger(-1);
         for (int i = 0; i < thread.length; i++) {
-            thread[i] = new Thread() {
-                @Override
-                public void run() {
-                    startGun.countDown();
+            thread[i] = new Thread(() -> {
+                startGun.countDown();
+                try {
+                    startGun.await();
+                } catch (InterruptedException e) {
+                    throw new AssertionError(e);
+                }
+                int docOffset;
+                while ((docOffset = offset.incrementAndGet()) < docs.size()) {
                     try {
-                        startGun.await();
-                    } catch (InterruptedException e) {
-                        throw new AssertionError(e);
-                    }
-                    int docOffset;
-                    while ((docOffset = offset.incrementAndGet()) < docs.size()) {
                         engine.index(docs.get(docOffset));
+                    } catch (IOException e) {
+                        throw new AssertionError(e);
                     }
                 }
-            };
+            });
             thread[i].start();
         }
         for (int i = 0; i < thread.length; i++) {
             thread[i].join();
         }
-        assertEquals(0, engine.getNumVersionLookups());
-        assertEquals(0, engine.getNumIndexVersionsLookups());
+        if (primary) {
+            assertEquals(0, engine.getNumVersionLookups());
+            assertEquals(0, engine.getNumIndexVersionsLookups());
+        } else {
+            // we don't really know what order the operations will arrive and thus can't predict how many
+            // version lookups will be needed
+            assertThat(engine.getNumIndexVersionsLookups(), lessThanOrEqualTo(engine.getNumVersionLookups()));
+        }
         engine.refresh("test");
         try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
             TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), 10);
             assertEquals(numDocs, topDocs.totalHits);
         }
-        assertTrue(engine.indexWriterHasDeletions());
+        if (primary) {
+            // primaries rely on lucene dedup and may index the same document twice
+            assertTrue(engine.indexWriterHasDeletions());
+        } else {
+            // replicas rely on seq# based dedup and in this setup (same seq#) should never rely on lucene
+            assertFalse(engine.indexWriterHasDeletions());
+        }
     }
 
     public void testEngineMaxTimestampIsInitialized() throws IOException {
         try (Store store = createStore();
              Engine engine = new InternalEngine(config(defaultSettings, store, createTempDir(), NoMergePolicy.INSTANCE,
-                 IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP))) {
+                 IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, null))) {
             assertEquals(IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, engine.segmentsStats(false).getMaxUnsafeAutoIdTimestamp());
 
         }
 
-        long maxTimestamp = Math.abs(randomLong());
-        try (Store store = createStore();
-             Engine engine = new InternalEngine(config(defaultSettings, store, createTempDir(), NoMergePolicy.INSTANCE,
-                 maxTimestamp))) {
-            assertEquals(maxTimestamp, engine.segmentsStats(false).getMaxUnsafeAutoIdTimestamp());
+        final long timestamp1 = Math.abs(randomLong());
+        final Path storeDir = createTempDir();
+        final Path translogDir = createTempDir();
+        try (Store store = createStore(newFSDirectory(storeDir));
+             Engine engine = new InternalEngine(
+                 config(defaultSettings, store, translogDir, NoMergePolicy.INSTANCE, timestamp1, null) )) {
+            assertEquals(timestamp1, engine.segmentsStats(false).getMaxUnsafeAutoIdTimestamp());
+        }
+        final long timestamp2 = randomNonNegativeLong();
+        final long timestamp3 = randomNonNegativeLong();
+        final long maxTimestamp12 = Math.max(timestamp1, timestamp2);
+        final long maxTimestamp123 = Math.max(maxTimestamp12, timestamp3);
+        try (Store store = createStore(newFSDirectory(storeDir));
+             Engine engine = new InternalEngine(
+                 copy(config(defaultSettings, store, translogDir, NoMergePolicy.INSTANCE, timestamp2, null),
+                     randomFrom(EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG, EngineConfig.OpenMode.OPEN_INDEX_CREATE_TRANSLOG)))) {
+            assertEquals(maxTimestamp12, engine.segmentsStats(false).getMaxUnsafeAutoIdTimestamp());
+            if (engine.config().getOpenMode() == EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG) {
+                // recover from translog and commit maxTimestamp12
+                engine.recoverFromTranslog();
+                // force flush as the were no ops performed
+                engine.flush(true, false);
+            }
+            final ParsedDocument doc = testParsedDocument("1", "test", null, 1, -1, testDocumentWithTextField(),
+                new BytesArray("{}".getBytes(Charset.defaultCharset())), null);
+            engine.index(appendOnlyPrimary(doc, true, timestamp3));
+            assertEquals(maxTimestamp123, engine.segmentsStats(false).getMaxUnsafeAutoIdTimestamp());
+        }
+        try (Store store = createStore(newFSDirectory(storeDir));
+             Engine engine = new InternalEngine(
+                 config(defaultSettings, store, translogDir, NoMergePolicy.INSTANCE, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, null))) {
+            assertEquals(maxTimestamp12, engine.segmentsStats(false).getMaxUnsafeAutoIdTimestamp());
+            engine.recoverFromTranslog();
+            assertEquals(maxTimestamp123, engine.segmentsStats(false).getMaxUnsafeAutoIdTimestamp());
         }
     }
 
@@ -2399,8 +2925,8 @@ public void testAppendConcurrently() throws InterruptedException, IOException {
         assertEquals(0, engine.getNumIndexVersionsLookups());
         List<Engine.Index> docs = new ArrayList<>();
         for (int i = 0; i < numDocs; i++) {
-            final ParsedDocument doc = testParsedDocument(Integer.toString(i), Integer.toString(i), "test", null, i, -1, testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null);
-            Engine.Index index = randomAppendOnly(i, doc, false);
+            final ParsedDocument doc = testParsedDocument(Integer.toString(i), "test", null, i, -1, testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null);
+            Engine.Index index = randomAppendOnly(doc, false, i);
             docs.add(index);
         }
         Collections.shuffle(docs, random());
@@ -2418,7 +2944,11 @@ public void run() {
                     }
                     int docOffset;
                     while ((docOffset = offset.incrementAndGet()) < docs.size()) {
-                        engine.index(docs.get(docOffset));
+                        try {
+                            engine.index(docs.get(docOffset));
+                        } catch (IOException e) {
+                            throw new AssertionError(e);
+                        }
                     }
                 }
             };
@@ -2446,4 +2976,106 @@ public static long getNumVersionLookups(InternalEngine engine) { // for other te
     public static long getNumIndexVersionsLookups(InternalEngine engine) { // for other tests to access this
         return engine.getNumIndexVersionsLookups();
     }
+
+    public void testFailEngineOnRandomIO() throws IOException, InterruptedException {
+        MockDirectoryWrapper wrapper = newMockDirectory();
+        final Path translogPath = createTempDir("testFailEngineOnRandomIO");
+        try (Store store = createStore(wrapper)) {
+            CyclicBarrier join = new CyclicBarrier(2);
+            CountDownLatch start = new CountDownLatch(1);
+            AtomicInteger controller = new AtomicInteger(0);
+            EngineConfig config = config(defaultSettings, store, translogPath, newMergePolicy(),
+                IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, new ReferenceManager.RefreshListener() {
+                    @Override
+                    public void beforeRefresh() throws IOException {
+                    }
+
+                    @Override
+                    public void afterRefresh(boolean didRefresh) throws IOException {
+                        int i = controller.incrementAndGet();
+                        if (i == 1) {
+                            throw new MockDirectoryWrapper.FakeIOException();
+                        } else if (i == 2) {
+                            try {
+                                start.await();
+                            } catch (InterruptedException e) {
+                                throw new AssertionError(e);
+                            }
+                            throw new ElasticsearchException("something completely different");
+                        }
+                    }
+                });
+            InternalEngine internalEngine = new InternalEngine(config);
+            int docId = 0;
+            final ParsedDocument doc = testParsedDocument(Integer.toString(docId), "test", null, docId, -1,
+                testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null);
+
+            Engine.Index index = randomBoolean() ? indexForDoc(doc) : randomAppendOnly(doc, false, docId);
+            internalEngine.index(index);
+            Runnable r = () ->  {
+                try {
+                    join.await();
+                } catch (Exception e) {
+                    throw new AssertionError(e);
+                }
+                try {
+                    internalEngine.refresh("test");
+                    fail();
+                } catch (AlreadyClosedException ex) {
+                    if (ex.getCause() != null) {
+                        assertTrue(ex.toString(), ex.getCause() instanceof MockDirectoryWrapper.FakeIOException);
+                    }
+                } catch (RefreshFailedEngineException ex) {
+                    // fine
+                } finally {
+                    start.countDown();
+                }
+
+            };
+            Thread t = new Thread(r);
+            Thread t1 = new Thread(r);
+            t.start();
+            t1.start();
+            t.join();
+            t1.join();
+            assertTrue(internalEngine.isClosed.get());
+            assertTrue(internalEngine.failedEngine.get() instanceof MockDirectoryWrapper.FakeIOException);
+        }
+    }
+
+    public void testTragicEventErrorBubblesUp() throws IOException {
+        engine.close();
+        final AtomicBoolean failWithFatalError = new AtomicBoolean(true);
+        final VirtualMachineError error = randomFrom(
+            new InternalError(),
+            new OutOfMemoryError(),
+            new StackOverflowError(),
+            new UnknownError());
+        engine = new InternalEngine(copy(engine.config(), EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG, new Analyzer() {
+            @Override
+            protected TokenStreamComponents createComponents(String fieldName) {
+                return new TokenStreamComponents(new Tokenizer() {
+                    @Override
+                    public boolean incrementToken() throws IOException {
+                        if (failWithFatalError.get()) {
+                            throw error;
+                        } else {
+                            throw new AssertionError("should not get to this point");
+                        }
+                    }
+                });
+            }
+        }));
+        final Document document = testDocument();
+        document.add(new TextField("value", "test", Field.Store.YES));
+        final ParsedDocument firstDoc = testParsedDocument("1", "test", null, -1, -1, document, B_1, null);
+        final Engine.Index first = indexForDoc(firstDoc);
+        expectThrows(error.getClass(), () -> engine.index(first));
+        failWithFatalError.set(false);
+        final ParsedDocument secondDoc = testParsedDocument("2", "test", null, -1, -1, document, B_1, null);
+        final Engine.Index second = indexForDoc(secondDoc);
+        expectThrows(error.getClass(), () -> engine.index(second));
+        assertNull(engine.failedEngine.get());
+    }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/index/engine/LiveVersionMapTests.java b/core/src/test/java/org/elasticsearch/index/engine/LiveVersionMapTests.java
index ed80b98c7f933..9161bc413c86c 100644
--- a/core/src/test/java/org/elasticsearch/index/engine/LiveVersionMapTests.java
+++ b/core/src/test/java/org/elasticsearch/index/engine/LiveVersionMapTests.java
@@ -22,11 +22,13 @@
 import org.apache.lucene.util.BytesRefBuilder;
 import org.apache.lucene.util.RamUsageTester;
 import org.apache.lucene.util.TestUtil;
+import org.elasticsearch.bootstrap.JavaVersion;
 import org.elasticsearch.test.ESTestCase;
 
 public class LiveVersionMapTests extends ESTestCase {
 
     public void testRamBytesUsed() throws Exception {
+        assumeTrue("Test disabled for JDK 9", JavaVersion.current().compareTo(JavaVersion.parse("9")) < 0);
         LiveVersionMap map = new LiveVersionMap();
         for (int i = 0; i < 100000; ++i) {
             BytesRefBuilder uid = new BytesRefBuilder();
@@ -36,8 +38,8 @@ public void testRamBytesUsed() throws Exception {
         }
         long actualRamBytesUsed = RamUsageTester.sizeOf(map);
         long estimatedRamBytesUsed = map.ramBytesUsed();
-        // less than 25% off
-        assertEquals(actualRamBytesUsed, estimatedRamBytesUsed, actualRamBytesUsed / 4);
+        // less than 50% off
+        assertEquals(actualRamBytesUsed, estimatedRamBytesUsed, actualRamBytesUsed / 2);
 
         // now refresh
         map.beforeRefresh();
diff --git a/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java
index 6dea774f2589a..c101acf4090b1 100644
--- a/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java
+++ b/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java
@@ -32,7 +32,9 @@
 import org.apache.lucene.index.SnapshotDeletionPolicy;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.ReferenceManager;
 import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.util.IOUtils;
@@ -44,20 +46,19 @@
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.lucene.Lucene;
-import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.codec.CodecService;
 import org.elasticsearch.index.mapper.Mapping;
 import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
+import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.index.mapper.UidFieldMapper;
 import org.elasticsearch.index.shard.RefreshListeners;
-import org.elasticsearch.index.shard.DocsStats;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.ShardUtils;
 import org.elasticsearch.index.store.DirectoryService;
@@ -77,11 +78,12 @@
 import java.io.IOException;
 import java.nio.file.Path;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicBoolean;
 
-import static org.elasticsearch.index.engine.Engine.Operation.Origin.PRIMARY;
+import static java.util.Collections.emptyList;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.hasKey;
@@ -170,13 +172,14 @@ private ParseContext.Document testDocument() {
     }
 
 
-    private ParsedDocument testParsedDocument(String uid, String id, String type, String routing, long timestamp, long ttl, ParseContext.Document document, BytesReference source, Mapping mappingsUpdate) {
-        Field uidField = new Field("_uid", uid, UidFieldMapper.Defaults.FIELD_TYPE);
+    private ParsedDocument testParsedDocument(String id, String type, String routing, long timestamp, long ttl, ParseContext.Document document, BytesReference source, Mapping mappingsUpdate) {
+        Field uidField = new Field("_uid", Uid.createUid(type, id), UidFieldMapper.Defaults.FIELD_TYPE);
         Field versionField = new NumericDocValuesField("_version", 0);
         document.add(uidField);
         document.add(versionField);
         document.add(new LongPoint("point_field", 42)); // so that points report memory/disk usage
-        return new ParsedDocument(versionField, id, type, routing, timestamp, ttl, Arrays.asList(document), source, mappingsUpdate);
+        return new ParsedDocument(versionField, id, type, routing, timestamp, ttl, Arrays.asList(document), source, XContentType.JSON,
+            mappingsUpdate);
     }
 
     protected Store createStore(Path p) throws IOException {
@@ -245,26 +248,36 @@ public void onFailedEngine(String reason, @Nullable Exception e) {
             }
         };
         TranslogConfig translogConfig = new TranslogConfig(shardId, translogPath, indexSettings, BigArrays.NON_RECYCLING_INSTANCE);
+        final List<ReferenceManager.RefreshListener> refreshListenerList =
+            refreshListeners == null ? emptyList() : Collections.singletonList(refreshListeners);
         EngineConfig config = new EngineConfig(openMode, shardId, threadPool, indexSettings, null, store, createSnapshotDeletionPolicy(),
                 mergePolicy, iwc.getAnalyzer(), iwc.getSimilarity(), new CodecService(null, logger), eventListener, null,
                 IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), translogConfig,
-                TimeValue.timeValueMinutes(5), refreshListeners, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP);
+                TimeValue.timeValueMinutes(5), refreshListenerList, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP);
 
         return config;
     }
 
-    protected Term newUid(String id) {
-        return new Term("_uid", id);
+//    protected Term newUid(String id) {
+//        return new Term("_uid", id);
+//    }
+
+    protected Term newUid(ParsedDocument doc) {
+        return new Term("_uid", Uid.createUid(doc.type(), doc.id()));
+    }
+
+    private Engine.Index indexForDoc(ParsedDocument doc) {
+        return new Engine.Index(newUid(doc), doc);
     }
 
     protected static final BytesReference B_1 = new BytesArray(new byte[]{1});
     protected static final BytesReference B_2 = new BytesArray(new byte[]{2});
     protected static final BytesReference B_3 = new BytesArray(new byte[]{3});
 
-    public void testCommitStats() {
+    public void testCommitStats() throws IOException {
         // create a doc and refresh
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null);
-        primaryEngine.index(new Engine.Index(newUid("1"), doc));
+        ParsedDocument doc = testParsedDocument("1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null);
+        primaryEngine.index(indexForDoc(doc));
 
         CommitStats stats1 = replicaEngine.commitStats();
         assertThat(stats1.getGeneration(), greaterThan(0L));
@@ -295,11 +308,11 @@ public void testSegments() throws Exception {
         assertThat(primaryEngine.segmentsStats(false).getMemoryInBytes(), equalTo(0L));
 
         // create a doc and refresh
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null);
-        primaryEngine.index(new Engine.Index(newUid("1"), doc));
+        ParsedDocument doc = testParsedDocument("1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null);
+        primaryEngine.index(indexForDoc(doc));
 
-        ParsedDocument doc2 = testParsedDocument("2", "2", "test", null, -1, -1, testDocumentWithTextField(), B_2, null);
-        primaryEngine.index(new Engine.Index(newUid("2"), doc2));
+        ParsedDocument doc2 = testParsedDocument("2", "test", null, -1, -1, testDocumentWithTextField(), B_2, null);
+        primaryEngine.index(indexForDoc(doc2));
         primaryEngine.refresh("test");
 
         segments = primaryEngine.segments(false);
@@ -357,8 +370,8 @@ public void testSegments() throws Exception {
         assertThat(segments.get(0).isCompound(), equalTo(true));
 
 
-        ParsedDocument doc3 = testParsedDocument("3", "3", "test", null, -1, -1, testDocumentWithTextField(), B_3, null);
-        primaryEngine.index(new Engine.Index(newUid("3"), doc3));
+        ParsedDocument doc3 = testParsedDocument("3", "test", null, -1, -1, testDocumentWithTextField(), B_3, null);
+        primaryEngine.index(indexForDoc(doc3));
         primaryEngine.refresh("test");
 
         segments = primaryEngine.segments(false);
@@ -407,7 +420,7 @@ public void testSegments() throws Exception {
         assertThat(segments.get(1).getDeletedDocs(), equalTo(0));
         assertThat(segments.get(1).isCompound(), equalTo(true));
 
-        primaryEngine.delete(new Engine.Delete("test", "1", newUid("1")));
+        primaryEngine.delete(new Engine.Delete("test", "1", newUid(doc)));
         primaryEngine.refresh("test");
 
         segments = primaryEngine.segments(false);
@@ -429,8 +442,8 @@ public void testSegments() throws Exception {
         primaryEngine.flush();
         replicaEngine.refresh("test");
 
-        ParsedDocument doc4 = testParsedDocument("4", "4", "test", null, -1, -1, testDocumentWithTextField(), B_3, null);
-        primaryEngine.index(new Engine.Index(newUid("4"), doc4));
+        ParsedDocument doc4 = testParsedDocument("4", "test", null, -1, -1, testDocumentWithTextField(), B_3, null);
+        primaryEngine.index(indexForDoc(doc4));
         primaryEngine.refresh("test");
 
         segments = primaryEngine.segments(false);
@@ -462,19 +475,19 @@ public void testVerboseSegments() throws Exception {
         List<Segment> segments = primaryEngine.segments(true);
         assertThat(segments.isEmpty(), equalTo(true));
 
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null);
-        primaryEngine.index(new Engine.Index(newUid("1"), doc));
+        ParsedDocument doc = testParsedDocument("1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null);
+        primaryEngine.index(indexForDoc(doc));
         primaryEngine.refresh("test");
 
         segments = primaryEngine.segments(true);
         assertThat(segments.size(), equalTo(1));
         assertThat(segments.get(0).ramTree, notNullValue());
 
-        ParsedDocument doc2 = testParsedDocument("2", "2", "test", null, -1, -1, testDocumentWithTextField(), B_2, null);
-        primaryEngine.index(new Engine.Index(newUid("2"), doc2));
+        ParsedDocument doc2 = testParsedDocument("2", "test", null, -1,-1, testDocumentWithTextField(), B_2, null);
+        primaryEngine.index(indexForDoc(doc2));
         primaryEngine.refresh("test");
-        ParsedDocument doc3 = testParsedDocument("3", "3", "test", null, -1, -1, testDocumentWithTextField(), B_3, null);
-        primaryEngine.index(new Engine.Index(newUid("3"), doc3));
+        ParsedDocument doc3 = testParsedDocument("3", "test", null, -1,-1, testDocumentWithTextField(), B_3, null);
+        primaryEngine.index(indexForDoc(doc3));
         primaryEngine.refresh("test");
 
         segments = primaryEngine.segments(true);
@@ -499,9 +512,9 @@ public void testShadowEngineIgnoresWriteOperations() throws Exception {
         // create a document
         ParseContext.Document document = testDocumentWithTextField();
         document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE));
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_1, null);
+        ParsedDocument doc = testParsedDocument("1", "test", null, -1, -1, document, B_1, null);
         try {
-            replicaEngine.index(new Engine.Index(newUid("1"), doc));
+            replicaEngine.index(indexForDoc(doc));
             fail("should have thrown an exception");
         } catch (UnsupportedOperationException e) {}
         replicaEngine.refresh("test");
@@ -511,16 +524,16 @@ public void testShadowEngineIgnoresWriteOperations() throws Exception {
         MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0));
         MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 0));
         searchResult.close();
-        Engine.GetResult getResult = replicaEngine.get(new Engine.Get(true, newUid("1")));
+        Engine.GetResult getResult = replicaEngine.get(new Engine.Get(true, doc.type(), doc.id(), newUid(doc)), replicaEngine::acquireSearcher);
         assertThat(getResult.exists(), equalTo(false));
         getResult.release();
 
         // index a document
         document = testDocument();
         document.add(new TextField("value", "test1", Field.Store.YES));
-        doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_1, null);
+        doc = testParsedDocument("1", "test", null, -1, -1, document, B_1, null);
         try {
-            replicaEngine.index(new Engine.Index(newUid("1"), doc));
+            replicaEngine.index(indexForDoc(doc));
             fail("should have thrown an exception");
         } catch (UnsupportedOperationException e) {}
         replicaEngine.refresh("test");
@@ -530,15 +543,15 @@ public void testShadowEngineIgnoresWriteOperations() throws Exception {
         MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0));
         MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 0));
         searchResult.close();
-        getResult = replicaEngine.get(new Engine.Get(true, newUid("1")));
+        getResult = replicaEngine.get(new Engine.Get(true, doc.type(), doc.id(), newUid(doc)), replicaEngine::acquireSearcher);
         assertThat(getResult.exists(), equalTo(false));
         getResult.release();
 
         // Now, add a document to the primary so we can test shadow engine deletes
         document = testDocumentWithTextField();
         document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE));
-        doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_1, null);
-        primaryEngine.index(new Engine.Index(newUid("1"), doc));
+        doc = testParsedDocument("1", "test", null, -1,-1, document, B_1, null);
+        primaryEngine.index(indexForDoc(doc));
         primaryEngine.flush();
         replicaEngine.refresh("test");
 
@@ -549,14 +562,14 @@ public void testShadowEngineIgnoresWriteOperations() throws Exception {
         searchResult.close();
 
         // And the replica can retrieve it
-        getResult = replicaEngine.get(new Engine.Get(false, newUid("1")));
+        getResult = replicaEngine.get(new Engine.Get(false, doc.type(), doc.id(), newUid(doc)), replicaEngine::acquireSearcher);
         assertThat(getResult.exists(), equalTo(true));
         assertThat(getResult.docIdAndVersion(), notNullValue());
         getResult.release();
 
         // try to delete it on the replica
         try {
-            replicaEngine.delete(new Engine.Delete("test", "1", newUid("1")));
+            replicaEngine.delete(new Engine.Delete("test", "1", newUid(doc)));
             fail("should have thrown an exception");
         } catch (UnsupportedOperationException e) {}
         replicaEngine.flush();
@@ -568,7 +581,7 @@ public void testShadowEngineIgnoresWriteOperations() throws Exception {
         MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
         MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 1));
         searchResult.close();
-        getResult = replicaEngine.get(new Engine.Get(false, newUid("1")));
+        getResult = replicaEngine.get(new Engine.Get(false, doc.type(), doc.id(), newUid(doc)), replicaEngine::acquireSearcher);
         assertThat(getResult.exists(), equalTo(true));
         assertThat(getResult.docIdAndVersion(), notNullValue());
         getResult.release();
@@ -578,7 +591,7 @@ public void testShadowEngineIgnoresWriteOperations() throws Exception {
         MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
         MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 1));
         searchResult.close();
-        getResult = primaryEngine.get(new Engine.Get(false, newUid("1")));
+        getResult = primaryEngine.get(new Engine.Get(false, doc.type(), doc.id(), newUid(doc)), primaryEngine::acquireSearcher);
         assertThat(getResult.exists(), equalTo(true));
         assertThat(getResult.docIdAndVersion(), notNullValue());
         getResult.release();
@@ -592,8 +605,8 @@ public void testSimpleOperations() throws Exception {
         // create a document
         ParseContext.Document document = testDocumentWithTextField();
         document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE));
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_1, null);
-        primaryEngine.index(new Engine.Index(newUid("1"), doc));
+        ParsedDocument doc = testParsedDocument("1", "test", null, -1,-1, document, B_1, null);
+        primaryEngine.index(indexForDoc(doc));
 
         // its not there...
         searchResult = primaryEngine.acquireSearcher("test");
@@ -608,18 +621,18 @@ public void testSimpleOperations() throws Exception {
         searchResult.close();
 
         // but, we can still get it (in realtime)
-        Engine.GetResult getResult = primaryEngine.get(new Engine.Get(true, newUid("1")));
+        Engine.GetResult getResult = primaryEngine.get(new Engine.Get(true, doc.type(), doc.id(), newUid(doc)), primaryEngine::acquireSearcher);
         assertThat(getResult.exists(), equalTo(true));
         assertThat(getResult.docIdAndVersion(), notNullValue());
         getResult.release();
 
         // can't get it from the replica, because it's not in the translog for a shadow replica
-        getResult = replicaEngine.get(new Engine.Get(true, newUid("1")));
+        getResult = replicaEngine.get(new Engine.Get(true, doc.type(), doc.id(), newUid(doc)), replicaEngine::acquireSearcher);
         assertThat(getResult.exists(), equalTo(false));
         getResult.release();
 
         // but, not there non realtime
-        getResult = primaryEngine.get(new Engine.Get(false, newUid("1")));
+        getResult = primaryEngine.get(new Engine.Get(false, doc.type(), doc.id(), newUid(doc)), primaryEngine::acquireSearcher);
         assertThat(getResult.exists(), equalTo(true));
         getResult.release();
 
@@ -630,7 +643,7 @@ public void testSimpleOperations() throws Exception {
         searchResult.close();
 
         // also in non realtime
-        getResult = primaryEngine.get(new Engine.Get(false, newUid("1")));
+        getResult = primaryEngine.get(new Engine.Get(false, doc.type(), doc.id(), newUid(doc)), primaryEngine::acquireSearcher);
         assertThat(getResult.exists(), equalTo(true));
         assertThat(getResult.docIdAndVersion(), notNullValue());
         getResult.release();
@@ -645,8 +658,8 @@ public void testSimpleOperations() throws Exception {
         document = testDocument();
         document.add(new TextField("value", "test1", Field.Store.YES));
         document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_2), SourceFieldMapper.Defaults.FIELD_TYPE));
-        doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_2, null);
-        primaryEngine.index(new Engine.Index(newUid("1"), doc));
+        doc = testParsedDocument("1", "test", null, -1,-1, document, B_2, null);
+        primaryEngine.index(indexForDoc(doc));
 
         // its not updated yet...
         searchResult = primaryEngine.acquireSearcher("test");
@@ -656,7 +669,7 @@ public void testSimpleOperations() throws Exception {
         searchResult.close();
 
         // but, we can still get it (in realtime)
-        getResult = primaryEngine.get(new Engine.Get(true, newUid("1")));
+        getResult = primaryEngine.get(new Engine.Get(true, doc.type(), doc.id(), newUid(doc)), primaryEngine::acquireSearcher);
         assertThat(getResult.exists(), equalTo(true));
         assertThat(getResult.docIdAndVersion(), notNullValue());
         getResult.release();
@@ -689,7 +702,7 @@ public void testSimpleOperations() throws Exception {
         searchResult.close();
 
         // now delete
-        primaryEngine.delete(new Engine.Delete("test", "1", newUid("1")));
+        primaryEngine.delete(new Engine.Delete("test", "1", newUid(doc)));
 
         // its not deleted yet
         searchResult = primaryEngine.acquireSearcher("test");
@@ -699,7 +712,7 @@ public void testSimpleOperations() throws Exception {
         searchResult.close();
 
         // but, get should not see it (in realtime)
-        getResult = primaryEngine.get(new Engine.Get(true, newUid("1")));
+        getResult = primaryEngine.get(new Engine.Get(true, doc.type(), doc.id(), newUid(doc)), primaryEngine::acquireSearcher);
         assertThat(getResult.exists(), equalTo(false));
         getResult.release();
 
@@ -715,8 +728,8 @@ public void testSimpleOperations() throws Exception {
         // add it back
         document = testDocumentWithTextField();
         document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE));
-        doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_1, null);
-        primaryEngine.index(new Engine.Index(newUid("1"), doc));
+        doc = testParsedDocument("1", "test", null, -1,-1, document, B_1, null);
+        primaryEngine.index(indexForDoc(doc));
 
         // its not there...
         searchResult = primaryEngine.acquireSearcher("test");
@@ -739,7 +752,7 @@ public void testSimpleOperations() throws Exception {
         primaryEngine.flush();
 
         // and, verify get (in real time)
-        getResult = primaryEngine.get(new Engine.Get(true, newUid("1")));
+        getResult = primaryEngine.get(new Engine.Get(true, doc.type(), doc.id(), newUid(doc)), primaryEngine::acquireSearcher);
         assertThat(getResult.exists(), equalTo(true));
         assertThat(getResult.docIdAndVersion(), notNullValue());
         getResult.release();
@@ -751,7 +764,7 @@ public void testSimpleOperations() throws Exception {
         MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 1));
         MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test1")), 0));
         searchResult.close();
-        getResult = replicaEngine.get(new Engine.Get(true, newUid("1")));
+        getResult = replicaEngine.get(new Engine.Get(true, doc.type(), doc.id(), newUid(doc)), primaryEngine::acquireSearcher);
         assertThat(getResult.exists(), equalTo(true));
         assertThat(getResult.docIdAndVersion(), notNullValue());
         getResult.release();
@@ -760,8 +773,8 @@ public void testSimpleOperations() throws Exception {
         // now do an update
         document = testDocument();
         document.add(new TextField("value", "test1", Field.Store.YES));
-        doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_1, null);
-        primaryEngine.index(new Engine.Index(newUid("1"), doc));
+        doc = testParsedDocument("1", "test", null, -1,-1, document, B_1, null);
+        primaryEngine.index(indexForDoc(doc));
 
         // its not updated yet...
         searchResult = primaryEngine.acquireSearcher("test");
@@ -796,8 +809,8 @@ public void testSearchResultRelease() throws Exception {
         searchResult.close();
 
         // create a document
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null);
-        primaryEngine.index(new Engine.Index(newUid("1"), doc));
+        ParsedDocument doc = testParsedDocument("1", "test", null, -1,-1, testDocumentWithTextField(), B_1, null);
+        primaryEngine.index(indexForDoc(doc));
 
         // its not there...
         searchResult = primaryEngine.acquireSearcher("test");
@@ -826,7 +839,7 @@ public void testSearchResultRelease() throws Exception {
         // don't release the replica search result yet...
 
         // delete, refresh and do a new search, it should not be there
-        primaryEngine.delete(new Engine.Delete("test", "1", newUid("1")));
+        primaryEngine.delete(new Engine.Delete(doc.type(), doc.id(), newUid(doc)));
         primaryEngine.flush();
         primaryEngine.refresh("test");
         replicaEngine.refresh("test");
@@ -840,9 +853,9 @@ public void testSearchResultRelease() throws Exception {
         searchResult.close();
     }
 
-    public void testFailEngineOnCorruption() {
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null);
-        primaryEngine.index(new Engine.Index(newUid("1"), doc));
+    public void testFailEngineOnCorruption() throws IOException {
+        ParsedDocument doc = testParsedDocument("1", "test", null, -1,-1, testDocumentWithTextField(), B_1, null);
+        primaryEngine.index(indexForDoc(doc));
         primaryEngine.flush();
         MockDirectoryWrapper leaf = DirectoryUtils.getLeaf(replicaEngine.config().getStore().directory(), MockDirectoryWrapper.class);
         leaf.setRandomIOExceptionRate(1.0);
@@ -859,7 +872,7 @@ public void testFailEngineOnCorruption() {
             MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 1));
             searchResult.close();
             fail("exception expected");
-        } catch (EngineClosedException ex) {
+        } catch (AlreadyClosedException ex) {
             // all is well
         }
     }
@@ -878,8 +891,8 @@ public void testExtractShardId() {
      */
     public void testFailStart() throws IOException {
         // Need a commit point for this
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null);
-        primaryEngine.index(new Engine.Index(newUid("1"), doc));
+        ParsedDocument doc = testParsedDocument("1", "test", null, -1,-1, testDocumentWithTextField(), B_1, null);
+        primaryEngine.index(indexForDoc(doc));
         primaryEngine.flush();
 
         // this test fails if any reader, searcher or directory is not closed - MDW FTW
@@ -964,8 +977,8 @@ public void run() {
         // create a document
         ParseContext.Document document = testDocumentWithTextField();
         document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE));
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_1, null);
-        pEngine.index(new Engine.Index(newUid("1"), doc));
+        ParsedDocument doc = testParsedDocument("1", "test", null, -1,-1, document, B_1, null);
+        pEngine.index(indexForDoc(doc));
         pEngine.flush(true, true);
 
         t.join();
@@ -982,38 +995,4 @@ public void testNoTranslog() {
             // all good
         }
     }
-
-    public void testDocStats() throws IOException {
-        final int numDocs = randomIntBetween(2, 10); // at least 2 documents otherwise we don't see any deletes below
-        for (int i = 0; i < numDocs; i++) {
-            ParsedDocument doc = testParsedDocument(Integer.toString(i), Integer.toString(i), "test", null, -1, -1, testDocument(), new BytesArray("{}"), null);
-            Engine.Index firstIndexRequest = new Engine.Index(newUid(Integer.toString(i)), doc, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false);
-            primaryEngine.index(firstIndexRequest);
-            assertThat(firstIndexRequest.version(), equalTo(1L));
-        }
-        DocsStats docStats = primaryEngine.getDocStats();
-        assertEquals(numDocs, docStats.getCount());
-        assertEquals(0, docStats.getDeleted());
-
-        docStats = replicaEngine.getDocStats();
-        assertEquals(0, docStats.getCount());
-        assertEquals(0, docStats.getDeleted());
-        primaryEngine.flush();
-
-        docStats = replicaEngine.getDocStats();
-        assertEquals(0, docStats.getCount());
-        assertEquals(0, docStats.getDeleted());
-        replicaEngine.refresh("test");
-        docStats = replicaEngine.getDocStats();
-        assertEquals(numDocs, docStats.getCount());
-        assertEquals(0, docStats.getDeleted());
-        primaryEngine.forceMerge(randomBoolean(), 1, false, false, false);
-    }
-
-    public void testRefreshListenersFails() throws IOException {
-        EngineConfig config = config(defaultSettings, store, createTempDir(), newMergePolicy(),
-                new RefreshListeners(null, null, null, logger));
-        Exception e = expectThrows(IllegalArgumentException.class, () -> new ShadowEngine(config));
-        assertEquals("ShadowEngine doesn't support RefreshListeners", e.getMessage());
-    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/engine/VersionValueTests.java b/core/src/test/java/org/elasticsearch/index/engine/VersionValueTests.java
index e66f55ff676c4..7af8ebc7580f9 100644
--- a/core/src/test/java/org/elasticsearch/index/engine/VersionValueTests.java
+++ b/core/src/test/java/org/elasticsearch/index/engine/VersionValueTests.java
@@ -20,8 +20,6 @@
 package org.elasticsearch.index.engine;
 
 import org.apache.lucene.util.RamUsageTester;
-import org.elasticsearch.index.translog.Translog;
-import org.elasticsearch.index.translog.TranslogTests;
 import org.elasticsearch.test.ESTestCase;
 
 public class VersionValueTests extends ESTestCase {
diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataImplTestCase.java b/core/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataImplTestCase.java
index b6ae9948675ac..114ec291bb0b6 100644
--- a/core/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataImplTestCase.java
+++ b/core/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataImplTestCase.java
@@ -83,7 +83,7 @@ public void testDeletedDocs() throws Exception {
     public void testSingleValueAllSet() throws Exception {
         fillSingleValueAllSet();
         IndexFieldData indexFieldData = getForField("value");
-        LeafReaderContext readerContext = refreshReader();
+        LeafReaderContext readerContexts = refreshReader();
         AtomicFieldData fieldData = indexFieldData.load(readerContext);
         assertThat(fieldData.ramBytesUsed(), greaterThanOrEqualTo(minRamBytesUsed()));
 
@@ -105,9 +105,9 @@ public void testSingleValueAllSet() throws Exception {
 
         IndexSearcher searcher = new IndexSearcher(readerContext.reader());
         TopFieldDocs topDocs;
-
+        SortField sortField = indexFieldData.sortField(null, MultiValueMode.MIN, null, false);
         topDocs = searcher.search(new MatchAllDocsQuery(), 10,
-                new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MIN, null))));
+            new Sort(sortField));
         assertThat(topDocs.totalHits, equalTo(3));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(1));
         assertThat(toString(((FieldDoc) topDocs.scoreDocs[0]).fields[0]), equalTo(one()));
@@ -116,8 +116,9 @@ public void testSingleValueAllSet() throws Exception {
         assertThat(topDocs.scoreDocs[2].doc, equalTo(2));
         assertThat(toString(((FieldDoc) topDocs.scoreDocs[2]).fields[0]), equalTo(three()));
 
+        sortField = indexFieldData.sortField(null, MultiValueMode.MAX, null, true);
         topDocs = searcher.search(new MatchAllDocsQuery(), 10,
-                new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MAX, null), true)));
+            new Sort(sortField));
         assertThat(topDocs.totalHits, equalTo(3));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(2));
         assertThat(topDocs.scoreDocs[1].doc, equalTo(0));
@@ -161,24 +162,25 @@ public void testSingleValueWithMissing() throws Exception {
     public void testMultiValueAllSet() throws Exception {
         fillMultiValueAllSet();
         IndexFieldData indexFieldData = getForField("value");
-        AtomicFieldData fieldData = indexFieldData.load(refreshReader());
+        LeafReaderContext readerContexts = refreshReader();
+        AtomicFieldData fieldData = indexFieldData.load(readerContext);
         assertThat(fieldData.ramBytesUsed(), greaterThanOrEqualTo(minRamBytesUsed()));
 
         SortedBinaryDocValues bytesValues = fieldData.getBytesValues();
-
         assertValues(bytesValues, 0, two(), four());
         assertValues(bytesValues, 1, one());
         assertValues(bytesValues, 2, three());
 
         IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer));
-        TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MIN, null))));
+        SortField sortField = indexFieldData.sortField(null, MultiValueMode.MIN, null, false);
+        TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(sortField));
         assertThat(topDocs.totalHits, equalTo(3));
         assertThat(topDocs.scoreDocs.length, equalTo(3));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(1));
         assertThat(topDocs.scoreDocs[1].doc, equalTo(0));
         assertThat(topDocs.scoreDocs[2].doc, equalTo(2));
-
-        topDocs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MAX, null), true)));
+        sortField = indexFieldData.sortField(null, MultiValueMode.MAX, null, true);
+        topDocs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(sortField));
         assertThat(topDocs.totalHits, equalTo(3));
         assertThat(topDocs.scoreDocs.length, equalTo(3));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(0));
@@ -227,8 +229,10 @@ public void testSortMultiValuesFields() throws Exception {
         IndexFieldData indexFieldData = getForField("value");
 
         IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer));
+        SortField sortField =
+            indexFieldData.sortField(null, MultiValueMode.MIN, null, false);
         TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10,
-                new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MIN, null))));
+                new Sort(sortField));
         assertThat(topDocs.totalHits, equalTo(8));
         assertThat(topDocs.scoreDocs.length, equalTo(8));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(7));
@@ -248,8 +252,9 @@ public void testSortMultiValuesFields() throws Exception {
         assertThat(topDocs.scoreDocs[7].doc, equalTo(5));
         assertThat((BytesRef) ((FieldDoc) topDocs.scoreDocs[7]).fields[0], equalTo(null));
 
+        sortField = indexFieldData.sortField(null, MultiValueMode.MAX, null, true);
         topDocs = searcher.search(new MatchAllDocsQuery(), 10,
-                new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MAX, null), true)));
+                new Sort(sortField));
         assertThat(topDocs.totalHits, equalTo(8));
         assertThat(topDocs.scoreDocs.length, equalTo(8));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(6));
diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java b/core/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java
index 741a1b545268b..fbb03f34fcd6f 100644
--- a/core/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java
+++ b/core/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java
@@ -266,8 +266,8 @@ public void testActualMissingValue(boolean reverse) throws IOException {
         final IndexFieldData indexFieldData = getForField("value");
         final String missingValue = values[1];
         IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer));
-        XFieldComparatorSource comparator = indexFieldData.comparatorSource(missingValue, MultiValueMode.MIN, null);
-        TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), randomBoolean() ? numDocs : randomIntBetween(10, numDocs), new Sort(new SortField("value", comparator, reverse)));
+        SortField sortField = indexFieldData.sortField(missingValue, MultiValueMode.MIN, null, reverse);
+        TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), randomBoolean() ? numDocs : randomIntBetween(10, numDocs), new Sort(sortField));
         assertEquals(numDocs, topDocs.totalHits);
         BytesRef previousValue = reverse ? UnicodeUtil.BIG_TERM : new BytesRef();
         for (int i = 0; i < topDocs.scoreDocs.length; ++i) {
@@ -320,8 +320,8 @@ public void testSortMissing(boolean first, boolean reverse) throws IOException {
         }
         final IndexFieldData indexFieldData = getForField("value");
         IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer));
-        XFieldComparatorSource comparator = indexFieldData.comparatorSource(first ? "_first" : "_last", MultiValueMode.MIN, null);
-        TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), randomBoolean() ? numDocs : randomIntBetween(10, numDocs), new Sort(new SortField("value", comparator, reverse)));
+        SortField sortField = indexFieldData.sortField(first ? "_first" : "_last", MultiValueMode.MIN, null, reverse);
+        TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), randomBoolean() ? numDocs : randomIntBetween(10, numDocs), new Sort(sortField));
         assertEquals(numDocs, topDocs.totalHits);
         BytesRef previousValue = first ? null : reverse ? UnicodeUtil.BIG_TERM : new BytesRef();
         for (int i = 0; i < topDocs.scoreDocs.length; ++i) {
@@ -473,6 +473,7 @@ public void testGlobalOrdinals() throws Exception {
         refreshReader();
         IndexOrdinalsFieldData ifd = getForField("string", "value", hasDocValues());
         IndexOrdinalsFieldData globalOrdinals = ifd.loadGlobal(topLevelReader);
+        assertNotNull(globalOrdinals.getOrdinalMap());
         assertThat(topLevelReader.leaves().size(), equalTo(3));
 
         // First segment
@@ -597,6 +598,7 @@ public void testGlobalOrdinalsGetRemovedOnceIndexReaderCloses() throws Exception
         refreshReader();
         IndexOrdinalsFieldData ifd = getForField("string", "value", hasDocValues());
         IndexOrdinalsFieldData globalOrdinals = ifd.loadGlobal(topLevelReader);
+        assertNotNull(globalOrdinals.getOrdinalMap());
         assertThat(ifd.loadGlobal(topLevelReader), sameInstance(globalOrdinals));
         // 3 b/c 1 segment level caches and 1 top level cache
         // in case of doc values, we don't cache atomic FD, so only the top-level cache is there
diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/BinaryDVFieldDataTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/BinaryDVFieldDataTests.java
index 5f8beb12424c9..9a1eef0dfa443 100644
--- a/core/src/test/java/org/elasticsearch/index/fielddata/BinaryDVFieldDataTests.java
+++ b/core/src/test/java/org/elasticsearch/index/fielddata/BinaryDVFieldDataTests.java
@@ -19,22 +19,19 @@
 
 package org.elasticsearch.index.fielddata;
 
-import com.carrotsearch.hppc.ObjectArrayList;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.compress.CompressedXContent;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.ParsedDocument;
+import org.elasticsearch.index.mapper.SourceToParse;
 
-import static org.hamcrest.Matchers.equalTo;
+import java.util.ArrayList;
+import java.util.List;
 
-/**
- *
- */
 public class BinaryDVFieldDataTests extends AbstractFieldDataTestCase {
     @Override
     protected boolean hasDocValues() {
@@ -54,61 +51,98 @@ public void testDocValue() throws Exception {
         final DocumentMapper mapper = mapperService.documentMapperParser().parse("test", new CompressedXContent(mapping));
 
 
-        ObjectArrayList<byte[]> bytesList1 = new ObjectArrayList<>(2);
+        List<BytesRef> bytesList1 = new ArrayList<>(2);
         bytesList1.add(randomBytes());
         bytesList1.add(randomBytes());
-        XContentBuilder doc = XContentFactory.jsonBuilder().startObject().startArray("field").value(bytesList1.get(0)).value(bytesList1.get(1)).endArray().endObject();
+        XContentBuilder doc = XContentFactory.jsonBuilder().startObject();
+        {
+            doc.startArray("field");
+            doc.value(bytesList1.get(0));
+            doc.value(bytesList1.get(1));
+            doc.endArray();
+        }
+        doc.endObject();
         ParsedDocument d = mapper.parse("test", "test", "1", doc.bytes());
         writer.addDocument(d.rootDoc());
 
-        byte[] bytes1 = randomBytes();
+        BytesRef bytes1 = randomBytes();
         doc = XContentFactory.jsonBuilder().startObject().field("field", bytes1).endObject();
-        d = mapper.parse("test", "test", "2", doc.bytes());
+        d = mapper.parse(SourceToParse.source("test", "test", "2", doc.bytes(), XContentType.JSON));
         writer.addDocument(d.rootDoc());
 
         doc = XContentFactory.jsonBuilder().startObject().endObject();
-        d = mapper.parse("test", "test", "3", doc.bytes());
+        d = mapper.parse(SourceToParse.source("test", "test", "3", doc.bytes(), XContentType.JSON));
         writer.addDocument(d.rootDoc());
 
         // test remove duplicate value
-        ObjectArrayList<byte[]> bytesList2 = new ObjectArrayList<>(2);
+        List<BytesRef> bytesList2 = new ArrayList<>(2);
         bytesList2.add(randomBytes());
         bytesList2.add(randomBytes());
-        doc = XContentFactory.jsonBuilder().startObject().startArray("field").value(bytesList2.get(0)).value(bytesList2.get(1)).value(bytesList2.get(0)).endArray().endObject();
+        doc = XContentFactory.jsonBuilder().startObject();
+        {
+            doc.startArray("field");
+            doc.value(bytesList2.get(0));
+            doc.value(bytesList2.get(1));
+            doc.value(bytesList2.get(0));
+            doc.endArray();
+        }
+        doc.endObject();
         d = mapper.parse("test", "test", "4", doc.bytes());
         writer.addDocument(d.rootDoc());
 
-        LeafReaderContext reader = refreshReader();
         IndexFieldData<?> indexFieldData = getForField("field");
-        AtomicFieldData fieldData = indexFieldData.load(reader);
+        LeafReaderContext reader = refreshReader();
+
+        bytesList1.sort(null);
+        bytesList2.sort(null);
 
+        // Test SortedBinaryDocValues's decoding:
+        AtomicFieldData fieldData = indexFieldData.load(reader);
         SortedBinaryDocValues bytesValues = fieldData.getBytesValues();
 
-        CollectionUtils.sortAndDedup(bytesList1);
         bytesValues.setDocument(0);
-        assertThat(bytesValues.count(), equalTo(2));
-        assertThat(bytesValues.valueAt(0), equalTo(new BytesRef(bytesList1.get(0))));
-        assertThat(bytesValues.valueAt(1), equalTo(new BytesRef(bytesList1.get(1))));
+        assertEquals(2, bytesValues.count());
+        assertEquals(bytesList1.get(0), bytesValues.valueAt(0));
+        assertEquals(bytesList1.get(1), bytesValues.valueAt(1));
 
         bytesValues.setDocument(1);
-        assertThat(bytesValues.count(), equalTo(1));
-        assertThat(bytesValues.valueAt(0), equalTo(new BytesRef(bytes1)));
+        assertEquals(1,bytesValues.count());
+        assertEquals(bytes1,bytesValues.valueAt(0));
 
         bytesValues.setDocument(2);
-        assertThat(bytesValues.count(), equalTo(0));
+        assertEquals(0, bytesValues.count());
 
-        CollectionUtils.sortAndDedup(bytesList2);
         bytesValues.setDocument(3);
-        assertThat(bytesValues.count(), equalTo(2));
-        assertThat(bytesValues.valueAt(0), equalTo(new BytesRef(bytesList2.get(0))));
-        assertThat(bytesValues.valueAt(1), equalTo(new BytesRef(bytesList2.get(1))));
+        assertEquals(2, bytesValues.count());
+        assertEquals(bytesList2.get(0), bytesValues.valueAt(0));
+        assertEquals(bytesList2.get(1), bytesValues.valueAt(1));
+
+        // Test whether ScriptDocValues.BytesRefs makes a deepcopy
+        fieldData = indexFieldData.load(reader);
+        ScriptDocValues<?> scriptValues = fieldData.getScriptValues();
+        scriptValues.setNextDocId(0);
+        assertEquals(2, scriptValues.size());
+        assertEquals(bytesList1.get(0), scriptValues.get(0));
+        assertEquals(bytesList1.get(1), scriptValues.get(1));
+
+        scriptValues.setNextDocId(1);
+        assertEquals(1, scriptValues.size());
+        assertEquals(bytes1, scriptValues.get(0));
+
+        scriptValues.setNextDocId(2);
+        assertEquals(0, scriptValues.size());
+
+        scriptValues.setNextDocId(3);
+        assertEquals(2, scriptValues.size());
+        assertEquals(bytesList2.get(0), scriptValues.get(0));
+        assertEquals(bytesList2.get(1), scriptValues.get(1));
     }
 
-    private byte[] randomBytes() {
+    private static BytesRef randomBytes() {
         int size = randomIntBetween(10, 1000);
         byte[] bytes = new byte[size];
         random().nextBytes(bytes);
-        return bytes;
+        return new BytesRef(bytes);
     }
 
     @Override
diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataCacheTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataCacheTests.java
index e9b510592e5c3..375b10e262766 100644
--- a/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataCacheTests.java
+++ b/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataCacheTests.java
@@ -35,6 +35,7 @@
 import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.fielddata.plain.AbstractAtomicOrdinalsFieldData;
 import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData;
 import org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData;
 import org.elasticsearch.index.mapper.TextFieldMapper;
@@ -86,7 +87,8 @@ public void testLoadGlobal_neverCacheIfFieldIsMissing() throws Exception {
     }
 
     private SortedSetDVOrdinalsIndexFieldData createSortedDV(String fieldName, IndexFieldDataCache indexFieldDataCache) {
-        return new SortedSetDVOrdinalsIndexFieldData(createIndexSettings(), indexFieldDataCache, fieldName, new NoneCircuitBreakerService());
+        return new SortedSetDVOrdinalsIndexFieldData(createIndexSettings(), indexFieldDataCache, fieldName, new NoneCircuitBreakerService(),
+                AbstractAtomicOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION);
     }
 
     private PagedBytesIndexFieldData createPagedBytes(String fieldName, IndexFieldDataCache indexFieldDataCache) {
diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataStatsTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataStatsTests.java
new file mode 100644
index 0000000000000..90995bec34d34
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataStatsTests.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.index.fielddata;
+
+import org.elasticsearch.common.FieldMemoryStats;
+import org.elasticsearch.common.FieldMemoryStatsTests;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+public class FieldDataStatsTests extends ESTestCase {
+
+    public void testSerialize() throws IOException {
+        FieldMemoryStats map = randomBoolean() ? null : FieldMemoryStatsTests.randomFieldMemoryStats();
+        FieldDataStats stats = new FieldDataStats(randomNonNegativeLong(), randomNonNegativeLong(), map == null ? null :
+            map);
+        BytesStreamOutput out = new BytesStreamOutput();
+        stats.writeTo(out);
+        FieldDataStats read = new FieldDataStats();
+        StreamInput input = out.bytes().streamInput();
+        read.readFrom(input);
+        assertEquals(-1, input.read());
+        assertEquals(stats.evictions, read.evictions);
+        assertEquals(stats.memorySize, read.memorySize);
+        assertEquals(stats.getFields(), read.getFields());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/NoOrdinalsStringFieldDataTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/NoOrdinalsStringFieldDataTests.java
index 9fdede24d5cd7..33170eb39ece1 100644
--- a/core/src/test/java/org/elasticsearch/index/fielddata/NoOrdinalsStringFieldDataTests.java
+++ b/core/src/test/java/org/elasticsearch/index/fielddata/NoOrdinalsStringFieldDataTests.java
@@ -20,6 +20,8 @@
 package org.elasticsearch.index.fielddata;
 
 import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.SortField;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
@@ -53,8 +55,9 @@ public AtomicFieldData loadDirect(LeafReaderContext context) throws Exception {
             }
 
             @Override
-            public XFieldComparatorSource comparatorSource(Object missingValue, MultiValueMode sortMode, Nested nested) {
-                return new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested);
+            public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse) {
+                XFieldComparatorSource source = new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested);
+                return new SortField(getFieldName(), source, reverse);
             }
 
             @Override
diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/ParentChildFieldDataTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/ParentChildFieldDataTests.java
deleted file mode 100644
index e722e29bc423d..0000000000000
--- a/core/src/test/java/org/elasticsearch/index/fielddata/ParentChildFieldDataTests.java
+++ /dev/null
@@ -1,276 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.fielddata;
-
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.SortedDocValuesField;
-import org.apache.lucene.document.StringField;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.SortedDocValues;
-import org.apache.lucene.search.FieldDoc;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.MatchAllDocsQuery;
-import org.apache.lucene.search.Sort;
-import org.apache.lucene.search.SortField;
-import org.apache.lucene.search.TopFieldDocs;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
-import org.elasticsearch.common.compress.CompressedXContent;
-import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.ParentFieldMapper;
-import org.elasticsearch.index.mapper.Uid;
-import org.elasticsearch.index.mapper.UidFieldMapper;
-import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.search.MultiValueMode;
-import org.junit.Before;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.atomic.AtomicReference;
-
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.nullValue;
-
-/**
- */
-public class ParentChildFieldDataTests extends AbstractFieldDataTestCase {
-    private final String parentType = "parent";
-    private final String childType = "child";
-    private final String grandChildType = "grand-child";
-
-    @Before
-    public void before() throws Exception {
-        mapperService.merge(
-                childType, new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef(childType, "_parent", "type=" + parentType).string()), MapperService.MergeReason.MAPPING_UPDATE, false
-        );
-        mapperService.merge(
-                grandChildType, new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef(grandChildType, "_parent", "type=" + childType).string()), MapperService.MergeReason.MAPPING_UPDATE, false
-        );
-
-        Document d = new Document();
-        d.add(new StringField(UidFieldMapper.NAME, Uid.createUid(parentType, "1"), Field.Store.NO));
-        d.add(createJoinField(parentType, "1"));
-        writer.addDocument(d);
-
-        d = new Document();
-        d.add(new StringField(UidFieldMapper.NAME, Uid.createUid(childType, "2"), Field.Store.NO));
-        d.add(new StringField(ParentFieldMapper.NAME, Uid.createUid(parentType, "1"), Field.Store.NO));
-        d.add(createJoinField(parentType, "1"));
-        d.add(createJoinField(childType, "2"));
-        writer.addDocument(d);
-        writer.commit();
-
-        d = new Document();
-        d.add(new StringField(UidFieldMapper.NAME, Uid.createUid(childType, "3"), Field.Store.NO));
-        d.add(new StringField(ParentFieldMapper.NAME, Uid.createUid(parentType, "1"), Field.Store.NO));
-        d.add(createJoinField(parentType, "1"));
-        d.add(createJoinField(childType, "3"));
-        writer.addDocument(d);
-
-        d = new Document();
-        d.add(new StringField(UidFieldMapper.NAME, Uid.createUid(parentType, "2"), Field.Store.NO));
-        d.add(createJoinField(parentType, "2"));
-        writer.addDocument(d);
-
-        d = new Document();
-        d.add(new StringField(UidFieldMapper.NAME, Uid.createUid(childType, "4"), Field.Store.NO));
-        d.add(new StringField(ParentFieldMapper.NAME, Uid.createUid(parentType, "2"), Field.Store.NO));
-        d.add(createJoinField(parentType, "2"));
-        d.add(createJoinField(childType, "4"));
-        writer.addDocument(d);
-
-        d = new Document();
-        d.add(new StringField(UidFieldMapper.NAME, Uid.createUid(childType, "5"), Field.Store.NO));
-        d.add(new StringField(ParentFieldMapper.NAME, Uid.createUid(parentType, "1"), Field.Store.NO));
-        d.add(createJoinField(parentType, "1"));
-        d.add(createJoinField(childType, "5"));
-        writer.addDocument(d);
-        writer.commit();
-
-        d = new Document();
-        d.add(new StringField(UidFieldMapper.NAME, Uid.createUid(grandChildType, "6"), Field.Store.NO));
-        d.add(new StringField(ParentFieldMapper.NAME, Uid.createUid(childType, "2"), Field.Store.NO));
-        d.add(createJoinField(childType, "2"));
-        writer.addDocument(d);
-
-        d = new Document();
-        d.add(new StringField(UidFieldMapper.NAME, Uid.createUid("other-type", "1"), Field.Store.NO));
-        writer.addDocument(d);
-    }
-
-    private SortedDocValuesField createJoinField(String parentType, String id) {
-        return new SortedDocValuesField(ParentFieldMapper.joinField(parentType), new BytesRef(id));
-    }
-
-    public void testGetBytesValues() throws Exception {
-        IndexFieldData indexFieldData = getForField(childType);
-        AtomicFieldData fieldData = indexFieldData.load(refreshReader());
-
-        SortedBinaryDocValues bytesValues = fieldData.getBytesValues();
-        bytesValues.setDocument(0);
-        assertThat(bytesValues.count(), equalTo(1));
-        assertThat(bytesValues.valueAt(0).utf8ToString(), equalTo("1"));
-
-        bytesValues.setDocument(1);
-        assertThat(bytesValues.count(), equalTo(2));
-        assertThat(bytesValues.valueAt(0).utf8ToString(), equalTo("1"));
-        assertThat(bytesValues.valueAt(1).utf8ToString(), equalTo("2"));
-
-        bytesValues.setDocument(2);
-        assertThat(bytesValues.count(), equalTo(2));
-        assertThat(bytesValues.valueAt(0).utf8ToString(), equalTo("1"));
-        assertThat(bytesValues.valueAt(1).utf8ToString(), equalTo("3"));
-
-        bytesValues.setDocument(3);
-        assertThat(bytesValues.count(), equalTo(1));
-        assertThat(bytesValues.valueAt(0).utf8ToString(), equalTo("2"));
-
-        bytesValues.setDocument(4);
-        assertThat(bytesValues.count(), equalTo(2));
-        assertThat(bytesValues.valueAt(0).utf8ToString(), equalTo("2"));
-        assertThat(bytesValues.valueAt(1).utf8ToString(), equalTo("4"));
-
-        bytesValues.setDocument(5);
-        assertThat(bytesValues.count(), equalTo(2));
-        assertThat(bytesValues.valueAt(0).utf8ToString(), equalTo("1"));
-        assertThat(bytesValues.valueAt(1).utf8ToString(), equalTo("5"));
-
-        bytesValues.setDocument(6);
-        assertThat(bytesValues.count(), equalTo(1));
-        assertThat(bytesValues.valueAt(0).utf8ToString(), equalTo("2"));
-
-        bytesValues.setDocument(7);
-        assertThat(bytesValues.count(), equalTo(0));
-    }
-
-    public void testSorting() throws Exception {
-        IndexFieldData indexFieldData = getForField(parentType);
-        IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer));
-        IndexFieldData.XFieldComparatorSource comparator = indexFieldData.comparatorSource("_last", MultiValueMode.MIN, null);
-
-        TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(new SortField(ParentFieldMapper.joinField(parentType), comparator, false)));
-        assertThat(topDocs.totalHits, equalTo(8));
-        assertThat(topDocs.scoreDocs.length, equalTo(8));
-        assertThat(topDocs.scoreDocs[0].doc, equalTo(0));
-        assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[0]).fields[0]).utf8ToString(), equalTo("1"));
-        assertThat(topDocs.scoreDocs[1].doc, equalTo(1));
-        assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[1]).fields[0]).utf8ToString(), equalTo("1"));
-        assertThat(topDocs.scoreDocs[2].doc, equalTo(2));
-        assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[2]).fields[0]).utf8ToString(), equalTo("1"));
-        assertThat(topDocs.scoreDocs[3].doc, equalTo(5));
-        assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[3]).fields[0]).utf8ToString(), equalTo("1"));
-        assertThat(topDocs.scoreDocs[4].doc, equalTo(3));
-        assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).utf8ToString(), equalTo("2"));
-        assertThat(topDocs.scoreDocs[5].doc, equalTo(4));
-        assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[5]).fields[0]).utf8ToString(), equalTo("2"));
-        assertThat(topDocs.scoreDocs[6].doc, equalTo(6));
-        assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[6]).fields[0]).utf8ToString(), equalTo("2"));
-        assertThat(topDocs.scoreDocs[7].doc, equalTo(7));
-        assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[7]).fields[0]), equalTo(null));
-
-        topDocs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(new SortField(ParentFieldMapper.joinField(parentType), comparator, true)));
-        assertThat(topDocs.totalHits, equalTo(8));
-        assertThat(topDocs.scoreDocs.length, equalTo(8));
-        assertThat(topDocs.scoreDocs[0].doc, equalTo(3));
-        assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[0]).fields[0]).utf8ToString(), equalTo("2"));
-        assertThat(topDocs.scoreDocs[1].doc, equalTo(4));
-        assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[1]).fields[0]).utf8ToString(), equalTo("2"));
-        assertThat(topDocs.scoreDocs[2].doc, equalTo(6));
-        assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[2]).fields[0]).utf8ToString(), equalTo("2"));
-        assertThat(topDocs.scoreDocs[3].doc, equalTo(0));
-        assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[3]).fields[0]).utf8ToString(), equalTo("1"));
-        assertThat(topDocs.scoreDocs[4].doc, equalTo(1));
-        assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).utf8ToString(), equalTo("1"));
-        assertThat(topDocs.scoreDocs[5].doc, equalTo(2));
-        assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[5]).fields[0]).utf8ToString(), equalTo("1"));
-        assertThat(topDocs.scoreDocs[6].doc, equalTo(5));
-        assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[6]).fields[0]).utf8ToString(), equalTo("1"));
-        assertThat(topDocs.scoreDocs[7].doc, equalTo(7));
-        assertThat(((FieldDoc) topDocs.scoreDocs[7]).fields[0], nullValue());
-    }
-
-    public void testThreads() throws Exception {
-        final ParentChildIndexFieldData indexFieldData = getForField(childType);
-        final DirectoryReader reader = ElasticsearchDirectoryReader.wrap(
-                DirectoryReader.open(writer), new ShardId(new Index("test", ""), 0));
-        final IndexParentChildFieldData global = indexFieldData.loadGlobal(reader);
-        final AtomicReference<Exception> error = new AtomicReference<>();
-        final int numThreads = scaledRandomIntBetween(3, 8);
-        final Thread[] threads = new Thread[numThreads];
-        final CountDownLatch latch = new CountDownLatch(1);
-
-        final Map<Object, BytesRef[]> expected = new HashMap<>();
-        for (LeafReaderContext context : reader.leaves()) {
-            AtomicParentChildFieldData leafData = global.load(context);
-            SortedDocValues parentIds = leafData.getOrdinalsValues(parentType);
-            final BytesRef[] ids = new BytesRef[parentIds.getValueCount()];
-            for (int j = 0; j < parentIds.getValueCount(); ++j) {
-                final BytesRef id = parentIds.lookupOrd(j);
-                if (id != null) {
-                    ids[j] = BytesRef.deepCopyOf(id);
-                }
-            }
-            expected.put(context.reader().getCoreCacheKey(), ids);
-        }
-
-        for (int i = 0; i < numThreads; ++i) {
-            threads[i] = new Thread() {
-                @Override
-                public void run() {
-                    try {
-                        latch.await();
-                        for (int i = 0; i < 100000; ++i) {
-                            for (LeafReaderContext context : reader.leaves()) {
-                                AtomicParentChildFieldData leafData = global.load(context);
-                                SortedDocValues parentIds = leafData.getOrdinalsValues(parentType);
-                                final BytesRef[] expectedIds = expected.get(context.reader().getCoreCacheKey());
-                                for (int j = 0; j < parentIds.getValueCount(); ++j) {
-                                    final BytesRef id = parentIds.lookupOrd(j);
-                                    assertEquals(expectedIds[j], id);
-                                }
-                            }
-                        }
-                    } catch (Exception e) {
-                        error.compareAndSet(null, e);
-                    }
-                }
-            };
-            threads[i].start();
-        }
-        latch.countDown();
-        for (Thread thread : threads) {
-            thread.join();
-        }
-        if (error.get() != null) {
-            throw error.get();
-        }
-    }
-
-    @Override
-    protected String getFieldDataType() {
-        return "_parent";
-    }
-}
diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesDatesTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesDatesTests.java
new file mode 100644
index 0000000000000..f8579efae737a
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesDatesTests.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import org.apache.lucene.index.SortedNumericDocValues;
+import org.elasticsearch.index.fielddata.ScriptDocValues.Dates;
+import org.elasticsearch.test.ESTestCase;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+import org.joda.time.ReadableDateTime;
+
+public class ScriptDocValuesDatesTests extends ESTestCase {
+    public void test() {
+        long[][] values = new long[between(3, 10)][];
+        ReadableDateTime[][] expectedDates = new ReadableDateTime[values.length][];
+        for (int d = 0; d < values.length; d++) {
+            values[d] = new long[randomBoolean() ? randomBoolean() ? 0 : 1 : between(2, 100)];
+            expectedDates[d] = new ReadableDateTime[values[d].length];
+            for (int i = 0; i < values[d].length; i++) {
+                expectedDates[d][i] = new DateTime(randomNonNegativeLong(), DateTimeZone.UTC);
+                values[d][i] = expectedDates[d][i].getMillis();
+            }
+        }
+        Dates dates = wrap(values);
+
+        for (int round = 0; round < 10; round++) {
+            int d = between(0, values.length - 1);
+            dates.setNextDocId(d);
+            assertEquals(expectedDates[d].length > 0 ? expectedDates[d][0] : new DateTime(0, DateTimeZone.UTC), dates.getValue());
+
+            assertEquals(values[d].length, dates.size());
+            for (int i = 0; i < values[d].length; i++) {
+                assertEquals(expectedDates[d][i], dates.get(i));
+            }
+
+            Exception e = expectThrows(UnsupportedOperationException.class, () -> dates.add(new DateTime()));
+            assertEquals("doc values are unmodifiable", e.getMessage());
+        }
+    }
+
+    private Dates wrap(long[][] values) {
+        return new Dates(new SortedNumericDocValues() {
+            long[] current;
+
+            @Override
+            public void setDocument(int doc) {
+                current = values[doc];
+            }
+            @Override
+            public int count() {
+                return current.length;
+            }
+            @Override
+            public long valueAt(int index) {
+                return current[index];
+            }
+        });
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesGeoPointsTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesGeoPointsTests.java
new file mode 100644
index 0000000000000..466ca0b6991a6
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesGeoPointsTests.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import org.elasticsearch.common.geo.GeoPoint;
+import org.elasticsearch.common.geo.GeoUtils;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Arrays;
+
+public class ScriptDocValuesGeoPointsTests extends ESTestCase {
+
+    private static MultiGeoPointValues wrap(final GeoPoint... points) {
+        return new MultiGeoPointValues() {
+            int docID = -1;
+
+            @Override
+            public GeoPoint valueAt(int i) {
+                if (docID != 0) {
+                    fail();
+                }
+                return points[i];
+            }
+
+            @Override
+            public void setDocument(int docId) {
+                this.docID = docId;
+            }
+
+            @Override
+            public int count() {
+                if (docID != 0) {
+                    return 0;
+                }
+                return points.length;
+            }
+        };
+    }
+
+    private static double randomLat() {
+        return randomDouble() * 180 - 90;
+    }
+
+    private static double randomLon() {
+        return randomDouble() * 360 - 180;
+    }
+
+    public void testGeoGetLatLon() {
+        final double lat1 = randomLat();
+        final double lat2 = randomLat();
+        final double lon1 = randomLon();
+        final double lon2 = randomLon();
+        final MultiGeoPointValues values = wrap(new GeoPoint(lat1, lon1), new GeoPoint(lat2, lon2));
+        final ScriptDocValues.GeoPoints script = new ScriptDocValues.GeoPoints(values);
+        script.setNextDocId(1);
+        assertEquals(true, script.isEmpty());
+        script.setNextDocId(0);
+        assertEquals(false, script.isEmpty());
+        assertEquals(new GeoPoint(lat1, lon1), script.getValue());
+        assertEquals(Arrays.asList(new GeoPoint(lat1, lon1), new GeoPoint(lat2, lon2)), script.getValues());
+        assertEquals(lat1, script.getLat(), 0);
+        assertEquals(lon1, script.getLon(), 0);
+        assertTrue(Arrays.equals(new double[] {lat1, lat2}, script.getLats()));
+        assertTrue(Arrays.equals(new double[] {lon1, lon2}, script.getLons()));
+    }
+
+    public void testGeoDistance() {
+        final double lat = randomLat();
+        final double lon = randomLon();
+        final MultiGeoPointValues values = wrap(new GeoPoint(lat, lon));
+        final ScriptDocValues.GeoPoints script = new ScriptDocValues.GeoPoints(values);
+        script.setNextDocId(0);
+
+        final ScriptDocValues.GeoPoints emptyScript = new ScriptDocValues.GeoPoints(wrap());
+        emptyScript.setNextDocId(0);
+
+        final double otherLat = randomLat();
+        final double otherLon = randomLon();
+
+        assertEquals(GeoUtils.arcDistance(lat, lon, otherLat, otherLon) / 1000d,
+                script.arcDistance(otherLat, otherLon) / 1000d, 0.01);
+        assertEquals(GeoUtils.arcDistance(lat, lon, otherLat, otherLon) / 1000d,
+                script.arcDistanceWithDefault(otherLat, otherLon, 42) / 1000d, 0.01);
+        assertEquals(42, emptyScript.arcDistanceWithDefault(otherLat, otherLon, 42), 0);
+
+        assertEquals(GeoUtils.planeDistance(lat, lon, otherLat, otherLon) / 1000d,
+                script.planeDistance(otherLat, otherLon) / 1000d, 0.01);
+        assertEquals(GeoUtils.planeDistance(lat, lon, otherLat, otherLon) / 1000d,
+                script.planeDistanceWithDefault(otherLat, otherLon, 42) / 1000d, 0.01);
+        assertEquals(42, emptyScript.planeDistanceWithDefault(otherLat, otherLon, 42), 0);
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesLongsTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesLongsTests.java
new file mode 100644
index 0000000000000..934fa628d4734
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesLongsTests.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import org.apache.lucene.index.SortedNumericDocValues;
+import org.elasticsearch.index.fielddata.ScriptDocValues.Longs;
+import org.elasticsearch.test.ESTestCase;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+import org.joda.time.ReadableDateTime;
+
+public class ScriptDocValuesLongsTests extends ESTestCase {
+    public void testLongs() {
+        long[][] values = new long[between(3, 10)][];
+        for (int d = 0; d < values.length; d++) {
+            values[d] = new long[randomBoolean() ? randomBoolean() ? 0 : 1 : between(2, 100)];
+            for (int i = 0; i < values[d].length; i++) {
+                values[d][i] = randomLong();
+            }
+        }
+        Longs longs = wrap(values);
+
+        for (int round = 0; round < 10; round++) {
+            int d = between(0, values.length - 1);
+            longs.setNextDocId(d);
+            assertEquals(values[d].length > 0 ? values[d][0] : 0, longs.getValue());
+
+            assertEquals(values[d].length, longs.size());
+            assertEquals(values[d].length, longs.getValues().size());
+            for (int i = 0; i < values[d].length; i++) {
+                assertEquals(values[d][i], longs.get(i).longValue());
+                assertEquals(values[d][i], longs.getValues().get(i).longValue());
+            }
+
+            Exception e = expectThrows(UnsupportedOperationException.class, () -> longs.getValues().add(100L));
+            assertEquals("doc values are unmodifiable", e.getMessage());
+        }
+    }
+
+    public void testDates() {
+        long[][] values = new long[between(3, 10)][];
+        ReadableDateTime[][] dates = new ReadableDateTime[values.length][];
+        for (int d = 0; d < values.length; d++) {
+            values[d] = new long[randomBoolean() ? randomBoolean() ? 0 : 1 : between(2, 100)];
+            dates[d] = new ReadableDateTime[values[d].length];
+            for (int i = 0; i < values[d].length; i++) {
+                dates[d][i] = new DateTime(randomNonNegativeLong(), DateTimeZone.UTC);
+                values[d][i] = dates[d][i].getMillis();
+            }
+        }
+        Longs longs = wrap(values);
+
+        for (int round = 0; round < 10; round++) {
+            int d = between(0, values.length - 1);
+            longs.setNextDocId(d);
+            assertEquals(dates[d].length > 0 ? dates[d][0] : new DateTime(0, DateTimeZone.UTC), longs.getDate());
+
+            assertEquals(values[d].length, longs.getDates().size());
+            for (int i = 0; i < values[d].length; i++) {
+                assertEquals(dates[d][i], longs.getDates().get(i));
+            }
+
+            Exception e = expectThrows(UnsupportedOperationException.class, () -> longs.getDates().add(new DateTime()));
+            assertEquals("doc values are unmodifiable", e.getMessage());
+        }
+    }
+
+    private Longs wrap(long[][] values) {
+        return new Longs(new SortedNumericDocValues() {
+            long[] current;
+
+            @Override
+            public void setDocument(int doc) {
+                current = values[doc];
+            }
+            @Override
+            public int count() {
+                return current.length;
+            }
+            @Override
+            public long valueAt(int index) {
+                return current[index];
+            }
+        });
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesTests.java
deleted file mode 100644
index 49f92e63c95e6..0000000000000
--- a/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesTests.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.fielddata;
-
-import org.elasticsearch.common.geo.GeoPoint;
-import org.elasticsearch.common.geo.GeoUtils;
-import org.elasticsearch.test.ESTestCase;
-
-import java.util.Arrays;
-
-public class ScriptDocValuesTests extends ESTestCase {
-
-    private static MultiGeoPointValues wrap(final GeoPoint... points) {
-        return new MultiGeoPointValues() {
-            int docID = -1;
-
-            @Override
-            public GeoPoint valueAt(int i) {
-                if (docID != 0) {
-                    fail();
-                }
-                return points[i];
-            }
-
-            @Override
-            public void setDocument(int docId) {
-                this.docID = docId;
-            }
-
-            @Override
-            public int count() {
-                if (docID != 0) {
-                    return 0;
-                }
-                return points.length;
-            }
-        };
-    }
-
-    private static double randomLat() {
-        return randomDouble() * 180 - 90;
-    }
-
-    private static double randomLon() {
-        return randomDouble() * 360 - 180;
-    }
-
-    public void testGeoGetLatLon() {
-        final double lat1 = randomLat();
-        final double lat2 = randomLat();
-        final double lon1 = randomLon();
-        final double lon2 = randomLon();
-        final MultiGeoPointValues values = wrap(new GeoPoint(lat1, lon1), new GeoPoint(lat2, lon2));
-        final ScriptDocValues.GeoPoints script = new ScriptDocValues.GeoPoints(values);
-        script.setNextDocId(1);
-        assertEquals(true, script.isEmpty());
-        script.setNextDocId(0);
-        assertEquals(false, script.isEmpty());
-        assertEquals(new GeoPoint(lat1, lon1), script.getValue());
-        assertEquals(Arrays.asList(new GeoPoint(lat1, lon1), new GeoPoint(lat2, lon2)), script.getValues());
-        assertEquals(lat1, script.getLat(), 0);
-        assertEquals(lon1, script.getLon(), 0);
-        assertTrue(Arrays.equals(new double[] {lat1, lat2}, script.getLats()));
-        assertTrue(Arrays.equals(new double[] {lon1, lon2}, script.getLons()));
-    }
-
-    public void testGeoDistance() {
-        final double lat = randomLat();
-        final double lon = randomLon();
-        final MultiGeoPointValues values = wrap(new GeoPoint(lat, lon));
-        final ScriptDocValues.GeoPoints script = new ScriptDocValues.GeoPoints(values);
-        script.setNextDocId(0);
-
-        final ScriptDocValues.GeoPoints emptyScript = new ScriptDocValues.GeoPoints(wrap());
-        emptyScript.setNextDocId(0);
-
-        final double otherLat = randomLat();
-        final double otherLon = randomLon();
-
-        assertEquals(GeoUtils.arcDistance(lat, lon, otherLat, otherLon) / 1000d,
-                script.arcDistance(otherLat, otherLon) / 1000d, 0.01);
-        assertEquals(GeoUtils.arcDistance(lat, lon, otherLat, otherLon) / 1000d,
-                script.arcDistanceWithDefault(otherLat, otherLon, 42) / 1000d, 0.01);
-        assertEquals(42, emptyScript.arcDistanceWithDefault(otherLat, otherLon, 42), 0);
-
-        assertEquals(GeoUtils.planeDistance(lat, lon, otherLat, otherLon) / 1000d,
-                script.planeDistance(otherLat, otherLon) / 1000d, 0.01);
-        assertEquals(GeoUtils.planeDistance(lat, lon, otherLat, otherLon) / 1000d,
-                script.planeDistanceWithDefault(otherLat, otherLon, 42) / 1000d, 0.01);
-        assertEquals(42, emptyScript.planeDistanceWithDefault(otherLat, otherLon, 42), 0);
-    }
-
-}
diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/UidFieldDataTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/UidFieldDataTests.java
new file mode 100644
index 0000000000000..4a1a9d711a0cd
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/fielddata/UidFieldDataTests.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.RandomAccessOrds;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Locale;
+
+public class UidFieldDataTests extends ESTestCase {
+
+    private static class DummySortedDocValues extends SortedDocValues {
+
+        @Override
+        public int getOrd(int docID) {
+            return docID;
+        }
+
+        @Override
+        public BytesRef lookupOrd(int ord) {
+            return new BytesRef(String.format(Locale.ENGLISH, "%03d", ord));
+        }
+
+        @Override
+        public int getValueCount() {
+            return 100;
+        }
+
+    }
+
+    private static class DummyAtomicOrdinalsFieldData implements AtomicOrdinalsFieldData {
+
+        @Override
+        public ScriptDocValues<?> getScriptValues() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public SortedBinaryDocValues getBytesValues() {
+            return FieldData.toString(getOrdinalsValues());
+        }
+
+        @Override
+        public long ramBytesUsed() {
+            return 0;
+        }
+
+        @Override
+        public void close() {
+        }
+
+        @Override
+        public RandomAccessOrds getOrdinalsValues() {
+            return DocValues.singleton(new DummySortedDocValues());
+        }
+
+    }
+
+    public void testSortedSetValues() throws Exception {
+        AtomicOrdinalsFieldData fd = new UidIndexFieldData.UidAtomicFieldData(new BytesRef("type#"), new DummyAtomicOrdinalsFieldData());
+        SortedSetDocValues dv = fd.getOrdinalsValues();
+        dv.setDocument(30);
+        assertEquals(30, dv.nextOrd());
+        assertEquals(SortedSetDocValues.NO_MORE_ORDS, dv.nextOrd());
+        assertEquals(new BytesRef("type#030"), dv.lookupOrd(30));
+        assertEquals(30, dv.lookupTerm(new BytesRef("type#030")));
+        assertEquals(-1 - 31, dv.lookupTerm(new BytesRef("type#0305")));
+    }
+
+    public void testScriptValues() throws IOException {
+        AtomicOrdinalsFieldData fd = new UidIndexFieldData.UidAtomicFieldData(new BytesRef("type#"), new DummyAtomicOrdinalsFieldData());
+        ScriptDocValues<?> values = fd.getScriptValues();
+        values.setNextDocId(30);
+        assertEquals(Collections.singletonList("type#030"), values);
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/index/get/GetFieldTests.java b/core/src/test/java/org/elasticsearch/index/get/GetFieldTests.java
new file mode 100644
index 0000000000000..62cd45508d810
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/get/GetFieldTests.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.get;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.mapper.ParentFieldMapper;
+import org.elasticsearch.index.mapper.RoutingFieldMapper;
+import org.elasticsearch.index.mapper.UidFieldMapper;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.RandomObjects;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.function.Supplier;
+
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
+
+public class GetFieldTests extends ESTestCase {
+
+    public void testToXContent() {
+        GetField getField = new GetField("field", Arrays.asList("value1", "value2"));
+        String output = Strings.toString(getField);
+        assertEquals("{\"field\":[\"value1\",\"value2\"]}", output);
+    }
+
+    public void testEqualsAndHashcode() {
+        checkEqualsAndHashCode(randomGetField(XContentType.JSON).v1(), GetFieldTests::copyGetField, GetFieldTests::mutateGetField);
+    }
+
+    public void testToAndFromXContent() throws Exception {
+        XContentType xContentType = randomFrom(XContentType.values());
+        Tuple<GetField, GetField> tuple = randomGetField(xContentType);
+        GetField getField = tuple.v1();
+        GetField expectedGetField = tuple.v2();
+        boolean humanReadable = randomBoolean();
+        BytesReference originalBytes = toShuffledXContent(getField, xContentType, ToXContent.EMPTY_PARAMS, humanReadable);
+        //test that we can parse what we print out
+        GetField parsedGetField;
+        try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) {
+            //we need to move to the next token, the start object one that we manually added is not expected
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
+            parsedGetField = GetField.fromXContent(parser);
+            assertEquals(XContentParser.Token.END_ARRAY, parser.currentToken());
+            assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
+            assertNull(parser.nextToken());
+        }
+        assertEquals(expectedGetField, parsedGetField);
+        BytesReference finalBytes = toXContent(parsedGetField, xContentType, humanReadable);
+        assertToXContentEquivalent(originalBytes, finalBytes, xContentType);
+    }
+
+    private static GetField copyGetField(GetField getField) {
+        return new GetField(getField.getName(), getField.getValues());
+    }
+
+    private static GetField mutateGetField(GetField getField) {
+        List<Supplier<GetField>> mutations = new ArrayList<>();
+        mutations.add(() -> new GetField(randomUnicodeOfCodepointLength(15), getField.getValues()));
+        mutations.add(() -> new GetField(getField.getName(), randomGetField(XContentType.JSON).v1().getValues()));
+        return randomFrom(mutations).get();
+    }
+
+    public static Tuple<GetField, GetField> randomGetField(XContentType xContentType) {
+        if (randomBoolean()) {
+            String fieldName = randomFrom(ParentFieldMapper.NAME, RoutingFieldMapper.NAME, UidFieldMapper.NAME);
+            GetField getField = new GetField(fieldName, Collections.singletonList(randomAlphaOfLengthBetween(3, 10)));
+            return Tuple.tuple(getField, getField);
+        }
+        String fieldName = randomAlphaOfLengthBetween(3, 10);
+        Tuple<List<Object>, List<Object>> tuple = RandomObjects.randomStoredFieldValues(random(), xContentType);
+        GetField input = new GetField(fieldName, tuple.v1());
+        GetField expected = new GetField(fieldName, tuple.v2());
+        return Tuple.tuple(input, expected);
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/get/GetResultTests.java b/core/src/test/java/org/elasticsearch/index/get/GetResultTests.java
new file mode 100644
index 0000000000000..c23a648aff9ba
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/get/GetResultTests.java
@@ -0,0 +1,223 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.get;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.RandomObjects;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Supplier;
+
+import static java.util.Collections.singletonList;
+import static java.util.Collections.singletonMap;
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.elasticsearch.index.get.GetFieldTests.randomGetField;
+import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
+
+public class GetResultTests extends ESTestCase {
+
+    public void testToAndFromXContent() throws Exception {
+        XContentType xContentType = randomFrom(XContentType.values());
+        Tuple<GetResult, GetResult> tuple = randomGetResult(xContentType);
+        GetResult getResult = tuple.v1();
+        GetResult expectedGetResult = tuple.v2();
+        boolean humanReadable = randomBoolean();
+        BytesReference originalBytes = toShuffledXContent(getResult, xContentType, ToXContent.EMPTY_PARAMS, humanReadable, "_source");
+        //test that we can parse what we print out
+        GetResult parsedGetResult;
+        try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) {
+            parsedGetResult = GetResult.fromXContent(parser);
+            assertNull(parser.nextToken());
+        }
+        assertEquals(expectedGetResult, parsedGetResult);
+        //print the parsed object out and test that the output is the same as the original output
+        BytesReference finalBytes = toXContent(parsedGetResult, xContentType, humanReadable);
+        assertToXContentEquivalent(originalBytes, finalBytes, xContentType);
+        //check that the source stays unchanged, no shuffling of keys nor anything like that
+        assertEquals(expectedGetResult.sourceAsString(), parsedGetResult.sourceAsString());
+    }
+
+    public void testToXContent() throws IOException {
+        {
+            GetResult getResult = new GetResult("index", "type", "id", 1, true, new BytesArray("{ \"field1\" : " +
+                    "\"value1\", \"field2\":\"value2\"}"), singletonMap("field1", new GetField("field1",
+                    singletonList("value1"))));
+            String output = Strings.toString(getResult);
+            assertEquals("{\"_index\":\"index\",\"_type\":\"type\",\"_id\":\"id\",\"_version\":1,\"found\":true,\"_source\":{ \"field1\" " +
+                    ": \"value1\", \"field2\":\"value2\"},\"fields\":{\"field1\":[\"value1\"]}}", output);
+        }
+        {
+            GetResult getResult = new GetResult("index", "type", "id", 1, false, null, null);
+            String output = Strings.toString(getResult);
+            assertEquals("{\"_index\":\"index\",\"_type\":\"type\",\"_id\":\"id\",\"found\":false}", output);
+        }
+    }
+
+    public void testToAndFromXContentEmbedded() throws Exception {
+        XContentType xContentType = randomFrom(XContentType.values());
+        Tuple<GetResult, GetResult> tuple = randomGetResult(xContentType);
+        GetResult getResult = tuple.v1();
+
+        // We don't expect to retrieve the index/type/id of the GetResult because they are not rendered
+        // by the toXContentEmbedded method.
+        GetResult expectedGetResult = new GetResult(null, null, null, -1,
+                tuple.v2().isExists(), tuple.v2().sourceRef(), tuple.v2().getFields());
+
+        boolean humanReadable = randomBoolean();
+        BytesReference originalBytes = toXContentEmbedded(getResult, xContentType, humanReadable);
+
+        // Test that we can parse the result of toXContentEmbedded()
+        GetResult parsedEmbeddedGetResult;
+        try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) {
+            ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
+            parsedEmbeddedGetResult = GetResult.fromXContentEmbedded(parser);
+            assertNull(parser.nextToken());
+        }
+
+        assertEquals(expectedGetResult, parsedEmbeddedGetResult);
+        //print the parsed object out and test that the output is the same as the original output
+        BytesReference finalBytes = toXContentEmbedded(parsedEmbeddedGetResult, xContentType, humanReadable);
+        assertToXContentEquivalent(originalBytes, finalBytes, xContentType);
+        //check that the source stays unchanged, no shuffling of keys nor anything like that
+        assertEquals(expectedGetResult.sourceAsString(), parsedEmbeddedGetResult.sourceAsString());
+    }
+
+    public void testToXContentEmbedded() throws IOException {
+        Map<String, GetField> fields = new HashMap<>();
+        fields.put("foo", new GetField("foo", singletonList("bar")));
+        fields.put("baz", new GetField("baz", Arrays.asList("baz_0", "baz_1")));
+
+        GetResult getResult = new GetResult("index", "type", "id", 2, true,
+                new BytesArray("{\"foo\":\"bar\",\"baz\":[\"baz_0\",\"baz_1\"]}"), fields);
+
+        BytesReference originalBytes = toXContentEmbedded(getResult, XContentType.JSON, false);
+        assertEquals("{\"found\":true,\"_source\":{\"foo\":\"bar\",\"baz\":[\"baz_0\",\"baz_1\"]}," +
+                "\"fields\":{\"foo\":[\"bar\"],\"baz\":[\"baz_0\",\"baz_1\"]}}", originalBytes.utf8ToString());
+    }
+
+    public void testToXContentEmbeddedNotFound() throws IOException {
+        GetResult getResult = new GetResult("index", "type", "id", 1, false, null, null);
+
+        BytesReference originalBytes = toXContentEmbedded(getResult, XContentType.JSON, false);
+        assertEquals("{\"found\":false}", originalBytes.utf8ToString());
+    }
+
+    public void testGetSourceAsBytes() {
+        XContentType xContentType = randomFrom(XContentType.values());
+        Tuple<GetResult, GetResult> tuple = randomGetResult(xContentType);
+        GetResult getResult = tuple.v1();
+        if (getResult.isExists() && getResult.isSourceEmpty() == false) {
+            assertNotNull(getResult.sourceRef());
+        } else {
+            assertNull(getResult.sourceRef());
+        }
+    }
+
+    public void testEqualsAndHashcode() {
+        checkEqualsAndHashCode(randomGetResult(XContentType.JSON).v1(), GetResultTests::copyGetResult, GetResultTests::mutateGetResult);
+    }
+
+    public static GetResult copyGetResult(GetResult getResult) {
+        return new GetResult(getResult.getIndex(), getResult.getType(), getResult.getId(), getResult.getVersion(),
+                getResult.isExists(), getResult.internalSourceRef(), getResult.getFields());
+    }
+
+    public static GetResult mutateGetResult(GetResult getResult) {
+        List<Supplier<GetResult>> mutations = new ArrayList<>();
+        mutations.add(() -> new GetResult(randomUnicodeOfLength(15), getResult.getType(), getResult.getId(), getResult.getVersion(),
+                getResult.isExists(), getResult.internalSourceRef(), getResult.getFields()));
+        mutations.add(() -> new GetResult(getResult.getIndex(), randomUnicodeOfLength(15), getResult.getId(), getResult.getVersion(),
+                getResult.isExists(), getResult.internalSourceRef(), getResult.getFields()));
+        mutations.add(() -> new GetResult(getResult.getIndex(), getResult.getType(), randomUnicodeOfLength(15), getResult.getVersion(),
+                getResult.isExists(), getResult.internalSourceRef(), getResult.getFields()));
+        mutations.add(() -> new GetResult(getResult.getIndex(), getResult.getType(), getResult.getId(), randomNonNegativeLong(),
+                getResult.isExists(), getResult.internalSourceRef(), getResult.getFields()));
+        mutations.add(() -> new GetResult(getResult.getIndex(), getResult.getType(), getResult.getId(), getResult.getVersion(),
+                getResult.isExists() == false, getResult.internalSourceRef(), getResult.getFields()));
+        mutations.add(() -> new GetResult(getResult.getIndex(), getResult.getType(), getResult.getId(), getResult.getVersion(),
+                getResult.isExists(), RandomObjects.randomSource(random()), getResult.getFields()));
+        mutations.add(() -> new GetResult(getResult.getIndex(), getResult.getType(), getResult.getId(), getResult.getVersion(),
+                getResult.isExists(), getResult.internalSourceRef(), randomGetFields(XContentType.JSON).v1()));
+        return randomFrom(mutations).get();
+    }
+
+    public static Tuple<GetResult, GetResult> randomGetResult(XContentType xContentType) {
+        final String index = randomAlphaOfLengthBetween(3, 10);
+        final String type = randomAlphaOfLengthBetween(3, 10);
+        final String id = randomAlphaOfLengthBetween(3, 10);
+        final long version;
+        final boolean exists;
+        BytesReference source = null;
+        Map<String, GetField> fields = null;
+        Map<String, GetField> expectedFields = null;
+        if (frequently()) {
+            version = randomNonNegativeLong();
+            exists = true;
+            if (frequently()) {
+                source = RandomObjects.randomSource(random());
+            }
+            if (randomBoolean()) {
+                Tuple<Map<String, GetField>, Map<String, GetField>> tuple = randomGetFields(xContentType);
+                fields = tuple.v1();
+                expectedFields = tuple.v2();
+            }
+        } else {
+            version = -1;
+            exists = false;
+        }
+        GetResult getResult = new GetResult(index, type, id, version, exists, source, fields);
+        GetResult expectedGetResult = new GetResult(index, type, id, version, exists, source, expectedFields);
+        return Tuple.tuple(getResult, expectedGetResult);
+    }
+
+    private static Tuple<Map<String, GetField>,Map<String, GetField>> randomGetFields(XContentType xContentType) {
+        int numFields = randomIntBetween(2, 10);
+        Map<String, GetField> fields = new HashMap<>(numFields);
+        Map<String, GetField> expectedFields = new HashMap<>(numFields);
+        for (int i = 0; i < numFields; i++) {
+            Tuple<GetField, GetField> tuple = randomGetField(xContentType);
+            GetField getField = tuple.v1();
+            GetField expectedGetField = tuple.v2();
+            fields.put(getField.getName(), getField);
+            expectedFields.put(expectedGetField.getName(), expectedGetField);
+        }
+        return Tuple.tuple(fields, expectedFields);
+    }
+
+    private static BytesReference toXContentEmbedded(GetResult getResult, XContentType xContentType, boolean humanReadable)
+            throws IOException {
+        return XContentHelper.toXContent(getResult::toXContentEmbedded, xContentType, humanReadable);
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/AbstractNumericFieldMapperTestCase.java b/core/src/test/java/org/elasticsearch/index/mapper/AbstractNumericFieldMapperTestCase.java
new file mode 100644
index 0000000000000..15c2e8207e91e
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/mapper/AbstractNumericFieldMapperTestCase.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.index.mapper;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.IndexService;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.elasticsearch.test.InternalSettingsPlugin;
+import org.elasticsearch.test.VersionUtils;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Set;
+
+import static com.carrotsearch.randomizedtesting.RandomizedTest.getRandom;
+import static org.hamcrest.Matchers.containsString;
+
+public abstract class AbstractNumericFieldMapperTestCase extends ESSingleNodeTestCase {
+    protected Set<String> TYPES;
+    protected Set<String> WHOLE_TYPES;
+    protected IndexService indexService;
+    protected DocumentMapperParser parser;
+
+    @Before
+    public void setup() {
+        indexService = createIndex("test");
+        parser = indexService.mapperService().documentMapperParser();
+        setTypeList();
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return pluginList(InternalSettingsPlugin.class);
+    }
+
+
+    protected abstract void setTypeList();
+
+    public void testDefaults() throws Exception {
+        for (String type : TYPES) {
+            doTestDefaults(type);
+        }
+    }
+
+    protected abstract void doTestDefaults(String type) throws Exception;
+
+    public void testNotIndexed() throws Exception {
+        for (String type : TYPES) {
+            doTestNotIndexed(type);
+        }
+    }
+
+    protected abstract void doTestNotIndexed(String type) throws Exception;
+
+    public void testNoDocValues() throws Exception {
+        for (String type : TYPES) {
+            doTestNoDocValues(type);
+        }
+    }
+
+    protected abstract void doTestNoDocValues(String type) throws Exception;
+
+    public void testStore() throws Exception {
+        for (String type : TYPES) {
+            doTestStore(type);
+        }
+    }
+
+    protected abstract void doTestStore(String type) throws Exception;
+
+    public void testCoerce() throws Exception {
+        for (String type : TYPES) {
+            doTestCoerce(type);
+        }
+    }
+
+    protected abstract void doTestCoerce(String type) throws IOException;
+
+    public void testIncludeInAll() throws Exception {
+        for (String type : TYPES) {
+            doTestIncludeInAll(type);
+        }
+    }
+
+    protected abstract void doTestIncludeInAll(String type) throws Exception;
+
+    public void testDecimalCoerce() throws Exception {
+        for (String type : WHOLE_TYPES) {
+            doTestDecimalCoerce(type);
+        }
+    }
+
+    protected abstract void doTestDecimalCoerce(String type) throws IOException;
+
+    public void testNullValue() throws IOException {
+        for (String type : TYPES) {
+            doTestNullValue(type);
+        }
+    }
+
+    protected abstract void doTestNullValue(String type) throws IOException;
+
+    public void testEmptyName() throws IOException {
+        // after version 5
+        for (String type : TYPES) {
+            String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties").startObject("").field("type", type).endObject().endObject()
+                .endObject().endObject().string();
+
+            IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> parser.parse("type", new CompressedXContent(mapping))
+            );
+            assertThat(e.getMessage(), containsString("name cannot be empty string"));
+        }
+
+        // before 5.x
+        Version oldVersion = VersionUtils.randomVersionBetween(getRandom(), Version.V_2_0_0, Version.V_2_3_5);
+        Settings oldIndexSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, oldVersion).build();
+        indexService = createIndex("test_old", oldIndexSettings);
+        parser = indexService.mapperService().documentMapperParser();
+        for (String type : TYPES) {
+            String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties").startObject("").field("type", type).endObject().endObject()
+                .endObject().endObject().string();
+
+            DocumentMapper defaultMapper = parser.parse("type", new CompressedXContent(mapping));
+            assertEquals(mapping, defaultMapper.mappingSource().string());
+        }
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/AllFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/AllFieldMapperTests.java
index 091aa6003c93d..06a02e0efadd0 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/AllFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/AllFieldMapperTests.java
@@ -30,6 +30,7 @@
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.lucene.all.AllTermQuery;
 import org.elasticsearch.common.lucene.all.AllTokenStream;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -90,6 +91,8 @@ public void testSimpleAllMappers() throws Exception {
         AllFieldMapper mapper = docMapper.allFieldMapper();
         assertThat(mapper.fieldType().queryStringTermQuery(new Term("_all", "foobar")),
             Matchers.instanceOf(AllTermQuery.class));
+        assertWarnings("field [include_in_all] is deprecated, as [_all] is deprecated, and will be disallowed" +
+                        " in 6.0, use [copy_to] instead.");
     }
 
     public void testAllMappersNoBoost() throws Exception {
@@ -110,6 +113,8 @@ public void testAllMappersNoBoost() throws Exception {
         AllFieldMapper mapper = docMapper.allFieldMapper();
         assertThat(mapper.fieldType().queryStringTermQuery(new Term("_all", "foobar")),
             Matchers.instanceOf(AllTermQuery.class));
+        assertWarnings("field [include_in_all] is deprecated, as [_all] is deprecated, and will be disallowed" +
+                        " in 6.0, use [copy_to] instead.");
     }
 
     public void testAllMappersTermQuery() throws Exception {
@@ -129,6 +134,8 @@ public void testAllMappersTermQuery() throws Exception {
         AllFieldMapper mapper = docMapper.allFieldMapper();
         assertThat(mapper.fieldType().queryStringTermQuery(new Term("_all", "foobar")),
             Matchers.instanceOf(AllTermQuery.class));
+        assertWarnings("field [include_in_all] is deprecated, as [_all] is deprecated, and will be disallowed" +
+                        " in 6.0, use [copy_to] instead.");
     }
 
     // #6187: make sure we see AllTermQuery even when offsets are indexed in the _all field:
@@ -148,6 +155,8 @@ public void testAllMappersWithOffsetsTermQuery() throws Exception {
         AllFieldMapper mapper = docMapper.allFieldMapper();
         assertThat(mapper.fieldType().queryStringTermQuery(new Term("_all", "foobar")),
             Matchers.instanceOf(AllTermQuery.class));
+        assertWarnings("field [include_in_all] is deprecated, as [_all] is deprecated, and will be disallowed" +
+                        " in 6.0, use [copy_to] instead.");
     }
 
     // #6187: if _all doesn't index positions then we never use AllTokenStream, even if some fields have boost
@@ -165,6 +174,8 @@ public void testBoostWithOmitPositions() throws Exception {
             assertThat(field.tokenStream(docMapper.mappers().indexAnalyzer(), null),
                 Matchers.not(Matchers.instanceOf(AllTokenStream.class)));
         }
+        assertWarnings("field [include_in_all] is deprecated, as [_all] is deprecated, and will be disallowed" +
+                        " in 6.0, use [copy_to] instead.");
     }
 
     // #6187: if no fields were boosted, we shouldn't use AllTokenStream
@@ -181,6 +192,8 @@ public void testNoBoost() throws Exception {
             assertThat(field.tokenStream(docMapper.mappers().indexAnalyzer(), null),
                 Matchers.not(Matchers.instanceOf(AllTokenStream.class)));
         }
+        assertWarnings("field [include_in_all] is deprecated, as [_all] is deprecated, and will be disallowed" +
+                        " in 6.0, use [copy_to] instead.");
     }
 
     public void testSimpleAllMappersWithReparse() throws Exception {
@@ -207,6 +220,8 @@ public void testSimpleAllMappersWithReparse() throws Exception {
                     Matchers.not(Matchers.instanceOf(AllTokenStream.class)));
             }
         }
+        assertWarnings("field [include_in_all] is deprecated, as [_all] is deprecated, and will be disallowed" +
+                        " in 6.0, use [copy_to] instead.");
     }
 
     public void testSimpleAllMappersWithStore() throws Exception {
@@ -222,6 +237,8 @@ public void testSimpleAllMappersWithStore() throws Exception {
             assertThat(fields[i].stringValue(), equalTo(expected[i]));
             assertThat(fields[i].fieldType().omitNorms(), equalTo(false));
         }
+        assertWarnings("field [include_in_all] is deprecated, as [_all] is deprecated, and will be disallowed" +
+                        " in 6.0, use [copy_to] instead.");
     }
 
 
@@ -242,6 +259,8 @@ public void testSimpleAllMappersWithReparseWithStore() throws Exception {
             assertThat(fields[i].stringValue(), equalTo(expected[i]));
             assertThat(fields[i].fieldType().omitNorms(), equalTo(false));
         }
+        assertWarnings("field [include_in_all] is deprecated, as [_all] is deprecated, and will be disallowed" +
+                        " in 6.0, use [copy_to] instead.");
     }
 
     public void testRandom() throws Exception {
@@ -354,6 +373,8 @@ public void testMultiField_includeInAllSetToFalse() throws IOException {
         Document doc = docMapper.parse("test", "test", "1", builder.bytes()).rootDoc();
         IndexableField[] fields = doc.getFields("_all");
         assertThat(fields.length, equalTo(0));
+        assertWarnings("field [include_in_all] is deprecated, as [_all] is deprecated, and will be disallowed" +
+                        " in 6.0, use [copy_to] instead.");
     }
 
     public void testMultiField_defaults() throws IOException {
@@ -476,4 +497,14 @@ public void testIncludeInObjectNotAllowed() throws Exception {
                 e.getMessage().contains("Field [_all] is a metadata field and cannot be added inside a document"));
         }
     }
+
+    public void testAllDefaults() {
+        // We use to have a bug with the default mapping having null analyzers because
+        // it was not fully constructed and was in particular lacking analyzers
+        IndexService index = createIndex("index", Settings.EMPTY, "type");
+        AllFieldMapper all = index.mapperService().documentMapper("type").allFieldMapper();
+        assertNotNull(all.fieldType().indexAnalyzer());
+        assertNotNull(all.fieldType().searchAnalyzer());
+        assertNotNull(all.fieldType().searchQuoteAnalyzer());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java
index 044f74e271218..a294a291a6a7f 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java
@@ -30,10 +30,6 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.index.mapper.BinaryFieldMapper;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.FieldMapper;
-import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
@@ -102,7 +98,7 @@ public void testStoredValue() throws IOException {
             BytesRef indexedValue = doc.rootDoc().getBinaryValue("field");
             assertEquals(new BytesRef(value), indexedValue);
             FieldMapper fieldMapper = mapper.mappers().smartNameFieldMapper("field");
-            Object originalValue = fieldMapper.fieldType().valueForSearch(indexedValue);
+            Object originalValue = fieldMapper.fieldType().valueForDisplay(indexedValue);
             assertEquals(new BytesArray(value), originalValue);
         }
     }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java
index b07f3b43ff6a5..00562b4c4f361 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java
@@ -31,6 +31,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
@@ -38,11 +39,6 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.IndexService;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.DocumentMapperParser;
-import org.elasticsearch.index.mapper.FieldMapper;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.ParseContext.Document;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
@@ -51,18 +47,19 @@
 import org.junit.Before;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Collection;
+import java.util.List;
 
 import static com.carrotsearch.randomizedtesting.RandomizedTest.getRandom;
 import static org.hamcrest.Matchers.containsString;
 
 public class BooleanFieldMapperTests extends ESSingleNodeTestCase {
-
-    IndexService indexService;
-    DocumentMapperParser parser;
+    private IndexService indexService;
+    private DocumentMapperParser parser;
 
     @Before
-    public void before() {
+    public void setup() {
         indexService = createIndex("test");
         parser = indexService.mapperService().documentMapperParser();
     }
@@ -130,6 +127,48 @@ public void testSerialization() throws IOException {
         assertEquals("{\"field\":{\"type\":\"boolean\",\"doc_values\":false,\"null_value\":true}}", builder.string());
     }
 
+    public void testParsesBooleansLenient() throws IOException {
+        String mapping = XContentFactory.jsonBuilder()
+            .startObject()
+                .startObject("type")
+                    .startObject("properties")
+                        .startObject("field1")
+                            .field("type", "boolean")
+                        .endObject()
+                        .startObject("field2")
+                            .field("type", "boolean")
+                        .endObject()
+                    .endObject()
+                .endObject()
+            .endObject().string();
+        DocumentMapper defaultMapper = parser.parse("type", new CompressedXContent(mapping));
+
+        String falsy = randomFrom("false", "off", "no", "0");
+        String truthy = randomFrom("true", "on", "yes", "1");
+
+        ParsedDocument parsedDoc = defaultMapper.parse("legacy", "type", "1", XContentFactory.jsonBuilder()
+            .startObject()
+                .field("field1", falsy)
+                .field("field2", truthy)
+            .endObject()
+            .bytes());
+        Document doc = parsedDoc.rootDoc();
+        assertEquals("F", doc.getField("field1").stringValue());
+        assertEquals("T", doc.getField("field2").stringValue());
+
+        List<String> expectedDeprecationWarnings = new ArrayList<>();
+        if (Booleans.isStrictlyBoolean(falsy) == false) {
+            expectedDeprecationWarnings.add("Expected a boolean [true/false] for property [field1] but got ["+ falsy + "]");
+        }
+        if (Booleans.isStrictlyBoolean(truthy) == false) {
+            expectedDeprecationWarnings.add("Expected a boolean [true/false] for property [field2] but got [" + truthy + "]");
+        }
+
+        if (expectedDeprecationWarnings.isEmpty() == false) {
+            assertWarnings(expectedDeprecationWarnings.toArray(new String[1]));
+        }
+    }
+
     public void testMultiFields() throws IOException {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("properties")
@@ -142,7 +181,8 @@ public void testMultiFields() throws IOException {
                         .endObject()
                     .endObject().endObject()
                 .endObject().endObject().string();
-        DocumentMapper mapper = indexService.mapperService().merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE, false);
+        DocumentMapper mapper = indexService.mapperService()
+            .merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE, false);
         assertEquals(mapping, mapper.mappingSource().toString());
         BytesReference source = XContentFactory.jsonBuilder()
                 .startObject()
@@ -217,6 +257,8 @@ public void testBwCompatDocValues() throws Exception {
         assertEquals(DocValuesType.SORTED_NUMERIC, LegacyStringMappingTests.docValuesType(doc, "bool1"));
         assertEquals(DocValuesType.NONE, LegacyStringMappingTests.docValuesType(doc, "bool2"));
         assertEquals(DocValuesType.SORTED_NUMERIC, LegacyStringMappingTests.docValuesType(doc, "bool3"));
+        assertWarnings("Expected a boolean [true/false] for property [index] but got [no]",
+                "Expected a boolean [true/false] for property [index] but got [not_analyzed]");
     }
 
     public void testEmptyName() throws IOException {
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/BooleanFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/BooleanFieldTypeTests.java
index 4f52fcd187def..d119a27f22eb5 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/BooleanFieldTypeTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/BooleanFieldTypeTests.java
@@ -21,8 +21,6 @@
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.TermQuery;
-import org.elasticsearch.index.mapper.BooleanFieldMapper;
-import org.elasticsearch.index.mapper.MappedFieldType;
 import org.junit.Before;
 
 public class BooleanFieldTypeTests extends FieldTypeTestCase {
@@ -44,11 +42,11 @@ public void testValueFormat() {
 
     public void testValueForSearch() {
         MappedFieldType ft = createDefaultFieldType();
-        assertEquals(true, ft.valueForSearch("T"));
-        assertEquals(false, ft.valueForSearch("F"));
-        expectThrows(IllegalArgumentException.class, () -> ft.valueForSearch(0));
-        expectThrows(IllegalArgumentException.class, () -> ft.valueForSearch("true"));
-        expectThrows(IllegalArgumentException.class, () -> ft.valueForSearch("G"));
+        assertEquals(true, ft.valueForDisplay("T"));
+        assertEquals(false, ft.valueForDisplay("F"));
+        expectThrows(IllegalArgumentException.class, () -> ft.valueForDisplay(0));
+        expectThrows(IllegalArgumentException.class, () -> ft.valueForDisplay("true"));
+        expectThrows(IllegalArgumentException.class, () -> ft.valueForDisplay("G"));
     }
 
     public void testTermQuery() {
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapper2xTests.java b/core/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapper2xTests.java
index a44941a19d88c..188e9cd5f9beb 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapper2xTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapper2xTests.java
@@ -98,7 +98,7 @@ public void testThatSerializationIncludesAllElements() throws Exception {
         completionFieldMapper.toXContent(builder, null).endObject();
         builder.close();
         Map<String, Object> serializedMap;
-        try (XContentParser parser = JsonXContent.jsonXContent.createParser(builder.bytes())) {
+        try (XContentParser parser = createParser(builder)) {
             serializedMap = parser.map();
         }
         Map<String, Object> configMap = (Map<String, Object>) serializedMap.get("completion");
@@ -131,7 +131,7 @@ public void testThatSerializationCombinesToOneAnalyzerFieldIfBothAreEqual() thro
         completionFieldMapper.toXContent(builder, null).endObject();
         builder.close();
         Map<String, Object> serializedMap;
-        try (XContentParser parser = JsonXContent.jsonXContent.createParser(builder.bytes())) {
+        try (XContentParser parser = createParser(builder)) {
             serializedMap = parser.map();
         }
         Map<String, Object> configMap = (Map<String, Object>) serializedMap.get("completion");
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java
index 902b9f2f4068a..802aee7f48274 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java
@@ -37,13 +37,6 @@
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
-import org.elasticsearch.index.mapper.CompletionFieldMapper;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.DocumentMapperParser;
-import org.elasticsearch.index.mapper.FieldMapper;
-import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.mapper.MapperParsingException;
-import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
 import java.io.IOException;
@@ -140,7 +133,7 @@ public void testTypeParsing() throws Exception {
         XContentBuilder builder = jsonBuilder().startObject();
         completionFieldMapper.toXContent(builder, ToXContent.EMPTY_PARAMS).endObject();
         builder.close();
-        Map<String, Object> serializedMap = JsonXContent.jsonXContent.createParser(builder.bytes()).map();
+        Map<String, Object> serializedMap = createParser(JsonXContent.jsonXContent, builder.bytes()).map();
         Map<String, Object> configMap = (Map<String, Object>) serializedMap.get("completion");
         assertThat(configMap.get("analyzer").toString(), is("simple"));
         assertThat(configMap.get("search_analyzer").toString(), is("standard"));
@@ -285,7 +278,7 @@ public void testParsingMixed() throws Exception {
                 .field("weight", 4)
                 .endObject()
                 .startObject()
-                .field("input", "suggestion4", "suggestion5", "suggestion6")
+                .array("input", "suggestion4", "suggestion5", "suggestion6")
                 .field("weight", 5)
                 .endObject()
                 .endArray()
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/CopyToMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/CopyToMapperTests.java
index e71ac5b492cee..dc23f50333f6b 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/CopyToMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/CopyToMapperTests.java
@@ -28,16 +28,7 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.IndexService;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.DocumentMapperParser;
-import org.elasticsearch.index.mapper.FieldMapper;
-import org.elasticsearch.index.mapper.LegacyLongFieldMapper;
-import org.elasticsearch.index.mapper.MapperParsingException;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.ParseContext.Document;
-import org.elasticsearch.index.mapper.ParsedDocument;
-import org.elasticsearch.index.mapper.TextFieldMapper;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
 import java.util.Arrays;
@@ -46,7 +37,6 @@
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.startsWith;
 
@@ -89,7 +79,7 @@ public void testCopyToFieldsParsing() throws Exception {
         stringFieldMapper.toXContent(builder, ToXContent.EMPTY_PARAMS).endObject();
         builder.close();
         Map<String, Object> serializedMap;
-        try (XContentParser parser = JsonXContent.jsonXContent.createParser(builder.bytes())) {
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, builder.bytes())) {
             serializedMap = parser.map();
         }
         Map<String, Object> copyTestMap = (Map<String, Object>) serializedMap.get("copy_test");
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/CustomBoostMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/CustomBoostMappingTests.java
index 9bc87e874f976..c115012eb48d0 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/CustomBoostMappingTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/CustomBoostMappingTests.java
@@ -27,9 +27,6 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.IndexService;
-import org.elasticsearch.index.mapper.DocumentFieldMappers;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
@@ -82,6 +79,7 @@ public void testBackCompatCustomBoostValues() throws Exception {
         assertThat(doc.rootDoc().getField("d_field").boost(), equalTo(7.0f));
         assertThat(doc.rootDoc().getField("f_field").boost(), equalTo(8.0f));
         assertThat(doc.rootDoc().getField("date_field").boost(), equalTo(9.0f));
+        assertWarnings("The [norms{enabled:true/false}] way of specifying norms is deprecated, please use [norms:true/false] instead");
     }
 
     public void testBackCompatFieldMappingBoostValues() throws Exception {
@@ -97,7 +95,7 @@ public void testBackCompatFieldMappingBoostValues() throws Exception {
                     .startObject("date_field").field("type", "date").field("boost", 9.0f).endObject()
                     .endObject().endObject().endObject().string();
             IndexService indexService = createIndex("test", BW_SETTINGS);
-            QueryShardContext context = indexService.newQueryShardContext();
+            QueryShardContext context = indexService.newQueryShardContext(0, null, () -> 0L);
             DocumentMapper mapper = indexService.mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
             DocumentFieldMappers fieldMappers = mapper.mappers();
             assertThat(fieldMappers.getMapper("s_field").fieldType().termQuery("0", context), instanceOf(TermQuery.class));
@@ -150,7 +148,7 @@ public void testBackCompatFieldMappingBoostValues() throws Exception {
                     .startObject("date_field").field("type", "date").field("boost", 9.0f).endObject()
                     .endObject().endObject().endObject().string();
             IndexService indexService = createIndex("text");
-            QueryShardContext context = indexService.newQueryShardContext();
+            QueryShardContext context = indexService.newQueryShardContext(0, null, () -> 0L);
             DocumentMapper mapper = indexService.mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
             DocumentFieldMappers fieldMappers = mapper.mappers();
             assertThat(fieldMappers.getMapper("s_field").fieldType().termQuery("0", context), instanceOf(BoostQuery.class));
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java
index a80f94845d110..1094743efdaef 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java
@@ -25,19 +25,15 @@
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.IndexService;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.DocumentMapperParser;
-import org.elasticsearch.index.mapper.FieldMapper;
-import org.elasticsearch.index.mapper.MapperParsingException;
-import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
 import org.elasticsearch.test.VersionUtils;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+import org.joda.time.format.DateTimeFormat;
 import org.junit.Before;
 
 import java.io.IOException;
@@ -45,6 +41,7 @@
 
 import static com.carrotsearch.randomizedtesting.RandomizedTest.getRandom;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.notNullValue;
 
 public class DateFieldMapperTests extends ESSingleNodeTestCase {
 
@@ -52,7 +49,7 @@ public class DateFieldMapperTests extends ESSingleNodeTestCase {
     DocumentMapperParser parser;
 
     @Before
-    public void before() {
+    public void setup() {
         indexService = createIndex("test");
         parser = indexService.mapperService().documentMapperParser();
     }
@@ -228,6 +225,8 @@ public void testIncludeInAll() throws Exception {
 
         fields = doc.rootDoc().getFields("_all");
         assertEquals(0, fields.length);
+        assertWarnings("field [include_in_all] is deprecated, as [_all] is deprecated, and will be disallowed" +
+                        " in 6.0, use [copy_to] instead.");
     }
 
     public void testChangeFormat() throws IOException {
@@ -354,4 +353,61 @@ public void testEmptyName() throws IOException {
         DocumentMapper defaultMapper = parser.parse("type", new CompressedXContent(mapping));
         assertEquals(mapping, defaultMapper.mappingSource().toString());
     }
+
+    /**
+     * Test that time zones are correctly parsed by the {@link DateFieldMapper}.
+     * There is a known bug with Joda 2.9.4 reported in https://github.com/JodaOrg/joda-time/issues/373.
+     */
+    public void testTimeZoneParsing() throws Exception {
+        final String timeZonePattern = "yyyy-MM-dd" + randomFrom("ZZZ", "[ZZZ]", "'['ZZZ']'");
+
+        String mapping = XContentFactory.jsonBuilder().startObject()
+                .startObject("type")
+                    .startObject("properties")
+                        .startObject("field")
+                            .field("type", "date")
+                            .field("format", timeZonePattern)
+                        .endObject()
+                    .endObject()
+                .endObject().endObject().string();
+
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+        assertEquals(mapping, mapper.mappingSource().toString());
+
+        final DateTimeZone randomTimeZone = randomBoolean() ? DateTimeZone.forID(randomFrom("UTC", "CET")) : randomDateTimeZone();
+        final DateTime randomDate = new DateTime(2016, 03, 11, 0, 0, 0, randomTimeZone);
+
+        ParsedDocument doc = mapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
+                .startObject()
+                    .field("field", DateTimeFormat.forPattern(timeZonePattern).print(randomDate))
+                .endObject()
+                .bytes());
+
+        IndexableField[] fields = doc.rootDoc().getFields("field");
+        assertEquals(2, fields.length);
+
+        assertEquals(randomDate.withZone(DateTimeZone.UTC).getMillis(), fields[0].numericValue().longValue());
+    }
+
+    public void testMergeDate() throws IOException {
+        String initMapping = XContentFactory.jsonBuilder().startObject().startObject("movie")
+            .startObject("properties")
+            .startObject("release_date").field("type", "date").field("format", "yyyy/MM/dd").endObject()
+            .endObject().endObject().endObject().string();
+        DocumentMapper initMapper = indexService.mapperService().merge("movie", new CompressedXContent(initMapping),
+            MapperService.MergeReason.MAPPING_UPDATE, randomBoolean());
+
+        assertThat(initMapper.mappers().getMapper("release_date"), notNullValue());
+        assertFalse(initMapper.mappers().getMapper("release_date").fieldType().stored());
+
+        String updateFormatMapping = XContentFactory.jsonBuilder().startObject().startObject("movie")
+            .startObject("properties")
+            .startObject("release_date").field("type", "date").field("format", "epoch_millis").endObject()
+            .endObject().endObject().endObject().string();
+
+        Exception e = expectThrows(IllegalArgumentException.class,
+            () -> indexService.mapperService().merge("movie", new CompressedXContent(updateFormatMapping),
+                MapperService.MergeReason.MAPPING_UPDATE, randomBoolean()));
+        assertThat(e.getMessage(), containsString("[mapper [release_date] has different [format] values]"));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java
index bdeaa6921de35..925ae210b08f9 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java
@@ -18,91 +18,102 @@
  */
 package org.elasticsearch.index.mapper;
 
-import java.io.IOException;
-import java.util.Locale;
-
 import org.apache.lucene.document.LongPoint;
+import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.MultiReader;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.IndexOrDocValuesQuery;
+import org.apache.lucene.search.Query;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.joda.DateMathParser;
 import org.elasticsearch.common.joda.Joda;
-import org.elasticsearch.index.mapper.DateFieldMapper;
-import org.elasticsearch.index.mapper.LegacyDateFieldMapper;
-import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.DateFieldMapper.DateFieldType;
 import org.elasticsearch.index.mapper.MappedFieldType.Relation;
 import org.elasticsearch.index.mapper.ParseContext.Document;
+import org.elasticsearch.index.query.QueryRewriteContext;
+import org.elasticsearch.index.query.QueryShardContext;
 import org.joda.time.DateTimeZone;
 import org.junit.Before;
 
+import java.io.IOException;
+import java.util.Locale;
+
 public class DateFieldTypeTests extends FieldTypeTestCase {
     @Override
     protected MappedFieldType createDefaultFieldType() {
         return new DateFieldMapper.DateFieldType();
     }
 
+    private static long nowInMillis;
+
     @Before
     public void setupProperties() {
         setDummyNullValue(10);
-        addModifier(new Modifier("format", true) {
+        addModifier(new Modifier("format", false) {
             @Override
             public void modify(MappedFieldType ft) {
                 ((DateFieldType) ft).setDateTimeFormatter(Joda.forPattern("basic_week_date", Locale.ROOT));
             }
         });
-        addModifier(new Modifier("locale", true) {
+        addModifier(new Modifier("locale", false) {
             @Override
             public void modify(MappedFieldType ft) {
                 ((DateFieldType) ft).setDateTimeFormatter(Joda.forPattern("date_optional_time", Locale.CANADA));
             }
         });
+        nowInMillis = randomNonNegativeLong();
     }
 
     public void testIsFieldWithinQueryEmptyReader() throws IOException {
+        QueryRewriteContext context = new QueryRewriteContext(null, null, null, xContentRegistry(), null, null,
+                () -> nowInMillis);
         IndexReader reader = new MultiReader();
         DateFieldType ft = new DateFieldType();
         ft.setName("my_date");
         assertEquals(Relation.DISJOINT, ft.isFieldWithinQuery(reader, "2015-10-12", "2016-04-03",
-                randomBoolean(), randomBoolean(), null, null));
+                randomBoolean(), randomBoolean(), null, null, context));
     }
 
     private void doTestIsFieldWithinQuery(DateFieldType ft, DirectoryReader reader,
             DateTimeZone zone, DateMathParser alternateFormat) throws IOException {
+        QueryRewriteContext context = new QueryRewriteContext(null, null, null, xContentRegistry(), null, null,
+                () -> nowInMillis);
         assertEquals(Relation.INTERSECTS, ft.isFieldWithinQuery(reader, "2015-10-09", "2016-01-02",
-                randomBoolean(), randomBoolean(), null, null));
+                randomBoolean(), randomBoolean(), null, null, context));
         assertEquals(Relation.INTERSECTS, ft.isFieldWithinQuery(reader, "2016-01-02", "2016-06-20",
-                randomBoolean(), randomBoolean(), null, null));
+                randomBoolean(), randomBoolean(), null, null, context));
         assertEquals(Relation.INTERSECTS, ft.isFieldWithinQuery(reader, "2016-01-02", "2016-02-12",
-                randomBoolean(), randomBoolean(), null, null));
+                randomBoolean(), randomBoolean(), null, null, context));
         assertEquals(Relation.DISJOINT, ft.isFieldWithinQuery(reader, "2014-01-02", "2015-02-12",
-                randomBoolean(), randomBoolean(), null, null));
+                randomBoolean(), randomBoolean(), null, null, context));
         assertEquals(Relation.DISJOINT, ft.isFieldWithinQuery(reader, "2016-05-11", "2016-08-30",
-                randomBoolean(), randomBoolean(), null, null));
+                randomBoolean(), randomBoolean(), null, null, context));
         assertEquals(Relation.WITHIN, ft.isFieldWithinQuery(reader, "2015-09-25", "2016-05-29",
-                randomBoolean(), randomBoolean(), null, null));
+                randomBoolean(), randomBoolean(), null, null, context));
         assertEquals(Relation.WITHIN, ft.isFieldWithinQuery(reader, "2015-10-12", "2016-04-03",
-                true, true, null, null));
+                true, true, null, null, context));
         assertEquals(Relation.INTERSECTS, ft.isFieldWithinQuery(reader, "2015-10-12", "2016-04-03",
-                false, false, null, null));
+                false, false, null, null, context));
         assertEquals(Relation.INTERSECTS, ft.isFieldWithinQuery(reader, "2015-10-12", "2016-04-03",
-                false, true, null, null));
+                false, true, null, null, context));
         assertEquals(Relation.INTERSECTS, ft.isFieldWithinQuery(reader, "2015-10-12", "2016-04-03",
-                true, false, null, null));
+                true, false, null, null, context));
     }
 
     public void testIsFieldWithinQuery() throws IOException {
         Directory dir = newDirectory();
         IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null));
-        long instant1 = LegacyDateFieldMapper.Defaults.DATE_TIME_FORMATTER.parser().parseDateTime("2015-10-12").getMillis();
-        long instant2 = LegacyDateFieldMapper.Defaults.DATE_TIME_FORMATTER.parser().parseDateTime("2016-04-03").getMillis();
+        long instant1 = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parser().parseDateTime("2015-10-12").getMillis();
+        long instant2 = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parser().parseDateTime("2016-04-03").getMillis();
         Document doc = new Document();
         LongPoint field = new LongPoint("my_date", instant1);
         doc.add(field);
@@ -112,7 +123,7 @@ public void testIsFieldWithinQuery() throws IOException {
         DirectoryReader reader = DirectoryReader.open(w);
         DateFieldType ft = new DateFieldType();
         ft.setName("my_date");
-        DateMathParser alternateFormat = new DateMathParser(LegacyDateFieldMapper.Defaults.DATE_TIME_FORMATTER);
+        DateMathParser alternateFormat = new DateMathParser(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER);
         doTestIsFieldWithinQuery(ft, reader, null, null);
         doTestIsFieldWithinQuery(ft, reader, null, alternateFormat);
         doTestIsFieldWithinQuery(ft, reader, DateTimeZone.UTC, null);
@@ -121,13 +132,16 @@ public void testIsFieldWithinQuery() throws IOException {
         // Fields with no value indexed.
         DateFieldType ft2 = new DateFieldType();
         ft2.setName("my_date2");
-        assertEquals(Relation.DISJOINT, ft2.isFieldWithinQuery(reader, "2015-10-09", "2016-01-02", false, false, null, null));
+
+        QueryRewriteContext context = new QueryRewriteContext(null, null, null, xContentRegistry(), null, null,
+                () -> nowInMillis);
+        assertEquals(Relation.DISJOINT, ft2.isFieldWithinQuery(reader, "2015-10-09", "2016-01-02", false, false, null, null, context));
         IOUtils.close(reader, w, dir);
     }
 
     public void testValueFormat() {
         MappedFieldType ft = createDefaultFieldType();
-        long instant = LegacyDateFieldMapper.Defaults.DATE_TIME_FORMATTER.parser().parseDateTime("2015-10-12T14:10:55").getMillis();
+        long instant = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parser().parseDateTime("2015-10-12T14:10:55").getMillis();
         assertEquals("2015-10-12T14:10:55.000Z",
                 ft.docValueFormat(null, DateTimeZone.UTC).format(instant));
         assertEquals("2015-10-12T15:10:55.000+01:00",
@@ -136,47 +150,64 @@ public void testValueFormat() {
                 createDefaultFieldType().docValueFormat("YYYY", DateTimeZone.UTC).format(instant));
         assertEquals(instant,
                 ft.docValueFormat(null, DateTimeZone.UTC).parseLong("2015-10-12T14:10:55", false, null));
-        assertEquals(instant,
+        assertEquals(instant + 999,
                 ft.docValueFormat(null, DateTimeZone.UTC).parseLong("2015-10-12T14:10:55", true, null));
-        assertEquals(LegacyDateFieldMapper.Defaults.DATE_TIME_FORMATTER.parser().parseDateTime("2015-10-13").getMillis() - 1,
+        assertEquals(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parser().parseDateTime("2015-10-13").getMillis() - 1,
                 ft.docValueFormat(null, DateTimeZone.UTC).parseLong("2015-10-12||/d", true, null));
     }
 
     public void testValueForSearch() {
         MappedFieldType ft = createDefaultFieldType();
         String date = "2015-10-12T12:09:55.000Z";
-        long instant = LegacyDateFieldMapper.Defaults.DATE_TIME_FORMATTER.parser().parseDateTime(date).getMillis();
-        assertEquals(date, ft.valueForSearch(instant));
+        long instant = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parser().parseDateTime(date).getMillis();
+        assertEquals(date, ft.valueForDisplay(instant));
     }
 
     public void testTermQuery() {
+        Settings indexSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1).build();
+        QueryShardContext context = new QueryShardContext(0,
+                new IndexSettings(IndexMetaData.builder("foo").settings(indexSettings).build(),
+                        indexSettings),
+                null, null, null, null, null, xContentRegistry(), null, null, () -> nowInMillis);
         MappedFieldType ft = createDefaultFieldType();
         ft.setName("field");
         String date = "2015-10-12T14:10:55";
-        long instant = LegacyDateFieldMapper.Defaults.DATE_TIME_FORMATTER.parser().parseDateTime(date).getMillis();
+        long instant = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parser().parseDateTime(date).getMillis();
         ft.setIndexOptions(IndexOptions.DOCS);
-        assertEquals(LongPoint.newExactQuery("field", instant), ft.termQuery(date, null));
+        Query expected = new IndexOrDocValuesQuery(
+                LongPoint.newRangeQuery("field", instant, instant + 999),
+                SortedNumericDocValuesField.newRangeQuery("field", instant, instant + 999));
+        assertEquals(expected, ft.termQuery(date, context));
 
         ft.setIndexOptions(IndexOptions.NONE);
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-                () -> ft.termQuery(date, null));
+                () -> ft.termQuery(date, context));
         assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage());
     }
 
     public void testRangeQuery() throws IOException {
+        Settings indexSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1).build();
+        QueryShardContext context = new QueryShardContext(0,
+                new IndexSettings(IndexMetaData.builder("foo").settings(indexSettings).build(), indexSettings),
+                null, null, null, null, null, xContentRegistry(), null, null, () -> nowInMillis);
         MappedFieldType ft = createDefaultFieldType();
         ft.setName("field");
         String date1 = "2015-10-12T14:10:55";
         String date2 = "2016-04-28T11:33:52";
-        long instant1 = LegacyDateFieldMapper.Defaults.DATE_TIME_FORMATTER.parser().parseDateTime(date1).getMillis();
-        long instant2 = LegacyDateFieldMapper.Defaults.DATE_TIME_FORMATTER.parser().parseDateTime(date2).getMillis();
+        long instant1 = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parser().parseDateTime(date1).getMillis();
+        long instant2 = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parser().parseDateTime(date2).getMillis() + 999;
         ft.setIndexOptions(IndexOptions.DOCS);
-        assertEquals(LongPoint.newRangeQuery("field", instant1, instant2),
-                ft.rangeQuery(date1, date2, true, true).rewrite(new MultiReader()));
+        Query expected = new IndexOrDocValuesQuery(
+                LongPoint.newRangeQuery("field", instant1, instant2),
+                SortedNumericDocValuesField.newRangeQuery("field", instant1, instant2));
+        assertEquals(expected,
+                ft.rangeQuery(date1, date2, true, true, context).rewrite(new MultiReader()));
 
         ft.setIndexOptions(IndexOptions.NONE);
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-                () -> ft.rangeQuery(date1, date2, true, true));
+                () -> ft.rangeQuery(date1, date2, true, true, context));
         assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage());
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java
index 43fc1d5c82ac8..27b90801ecc6d 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java
@@ -24,10 +24,12 @@
 import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.analysis.AnalyzerScope;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 
 import java.io.IOException;
@@ -40,8 +42,8 @@ public class DocumentFieldMapperTests extends LuceneTestCase {
     private static class FakeAnalyzer extends Analyzer {
 
         private final String output;
-        
-        public FakeAnalyzer(String output) {
+
+        FakeAnalyzer(String output) {
             this.output = output;
         }
 
@@ -63,19 +65,19 @@ public boolean incrementToken() throws IOException {
             };
             return new TokenStreamComponents(tokenizer);
         }
-        
+
     }
 
     static class FakeFieldType extends TermBasedFieldType {
 
-        public FakeFieldType() {
+        FakeFieldType() {
             super();
         }
-        
+
         FakeFieldType(FakeFieldType other) {
             super(other);
         }
-        
+
         @Override
         public MappedFieldType clone() {
             return new FakeFieldType(this);
@@ -85,34 +87,34 @@ public MappedFieldType clone() {
         public String typeName() {
             return "fake";
         }
-        
+
     }
 
     static class FakeFieldMapper extends FieldMapper {
 
         private static final Settings SETTINGS = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
 
-        public FakeFieldMapper(String simpleName, MappedFieldType fieldType) {
+        FakeFieldMapper(String simpleName, MappedFieldType fieldType) {
             super(simpleName, fieldType.clone(), fieldType.clone(), SETTINGS, null, null);
         }
 
         @Override
-        protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
+        protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
         }
 
         @Override
         protected String contentType() {
             return null;
         }
-        
+
     }
 
     public void testAnalyzers() throws IOException {
         FakeFieldType fieldType1 = new FakeFieldType();
         fieldType1.setName("field1");
-        fieldType1.setIndexAnalyzer(new NamedAnalyzer("foo", new FakeAnalyzer("index")));
-        fieldType1.setSearchAnalyzer(new NamedAnalyzer("bar", new FakeAnalyzer("search")));
-        fieldType1.setSearchQuoteAnalyzer(new NamedAnalyzer("baz", new FakeAnalyzer("search_quote")));
+        fieldType1.setIndexAnalyzer(new NamedAnalyzer("foo", AnalyzerScope.INDEX, new FakeAnalyzer("index")));
+        fieldType1.setSearchAnalyzer(new NamedAnalyzer("bar", AnalyzerScope.INDEX, new FakeAnalyzer("search")));
+        fieldType1.setSearchQuoteAnalyzer(new NamedAnalyzer("baz", AnalyzerScope.INDEX, new FakeAnalyzer("search_quote")));
         FieldMapper fieldMapper1 = new FakeFieldMapper("field1", fieldType1);
 
         FakeFieldType fieldType2 = new FakeFieldType();
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DocumentMapperMergeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DocumentMapperMergeTests.java
index a9b1753ba1c8a..b48d065113917 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/DocumentMapperMergeTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/DocumentMapperMergeTests.java
@@ -24,13 +24,6 @@
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
-import org.elasticsearch.index.mapper.DocumentFieldMappers;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.DocumentMapperParser;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.Mapping;
-import org.elasticsearch.index.mapper.ObjectMapper;
-import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
 import java.io.IOException;
@@ -167,7 +160,10 @@ public void run() {
                     barrier.await();
                     for (int i = 0; i < 200 && stopped.get() == false; i++) {
                         final String fieldName = Integer.toString(i);
-                        ParsedDocument doc = documentMapper.parse("test", "test", fieldName, new BytesArray("{ \"" + fieldName + "\" : \"test\" }"));
+                        ParsedDocument doc = documentMapper.parse("test",
+                                "test",
+                                fieldName,
+                                new BytesArray("{ \"" + fieldName + "\" : \"test\" }"));
                         Mapping update = doc.dynamicMappingsUpdate();
                         assert update != null;
                         lastIntroducedFieldName.set(fieldName);
@@ -186,7 +182,10 @@ public void run() {
             while(stopped.get() == false) {
                 final String fieldName = lastIntroducedFieldName.get();
                 final BytesReference source = new BytesArray("{ \"" + fieldName + "\" : \"test\" }");
-                ParsedDocument parsedDoc = documentMapper.parse("test", "test", "random", source);
+                ParsedDocument parsedDoc = documentMapper.parse("test",
+                        "test",
+                        "random",
+                        source);
                 if (parsedDoc.dynamicMappingsUpdate() != null) {
                     // not in the mapping yet, try again
                     continue;
@@ -226,4 +225,65 @@ public void testDoNotRepeatOriginalMapping() throws IOException {
         assertNotNull(mapper.mappers().getMapper("foo"));
         assertFalse(mapper.sourceMapper().enabled());
     }
+
+    public void testMergeChildType() throws IOException {
+        DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();
+
+        String initMapping = XContentFactory.jsonBuilder().startObject().startObject("child")
+            .startObject("_parent").field("type", "parent").endObject()
+            .endObject().endObject().string();
+        DocumentMapper initMapper = parser.parse("child", new CompressedXContent(initMapping));
+
+        assertThat(initMapper.mappers().getMapper("_parent#parent"), notNullValue());
+
+        String updatedMapping1 = XContentFactory.jsonBuilder().startObject().startObject("child")
+            .startObject("properties")
+            .startObject("name").field("type", "text").endObject()
+            .endObject().endObject().endObject().string();
+        DocumentMapper updatedMapper1 = parser.parse("child", new CompressedXContent(updatedMapping1));
+        DocumentMapper mergedMapper1 = initMapper.merge(updatedMapper1.mapping(), false);
+
+        assertThat(mergedMapper1.mappers().getMapper("_parent#parent"), notNullValue());
+        assertThat(mergedMapper1.mappers().getMapper("name"), notNullValue());
+
+        String updatedMapping2 = XContentFactory.jsonBuilder().startObject().startObject("child")
+            .startObject("_parent").field("type", "parent").endObject()
+            .startObject("properties")
+            .startObject("age").field("type", "byte").endObject()
+            .endObject().endObject().endObject().string();
+        DocumentMapper updatedMapper2 = parser.parse("child", new CompressedXContent(updatedMapping2));
+        DocumentMapper mergedMapper2 = mergedMapper1.merge(updatedMapper2.mapping(), false);
+
+        assertThat(mergedMapper2.mappers().getMapper("_parent#parent"), notNullValue());
+        assertThat(mergedMapper2.mappers().getMapper("name"), notNullValue());
+        assertThat(mergedMapper2.mappers().getMapper("age"), notNullValue());
+
+        String modParentMapping = XContentFactory.jsonBuilder().startObject().startObject("child")
+            .startObject("_parent").field("type", "new_parent").endObject()
+            .endObject().endObject().string();
+        DocumentMapper modParentMapper = parser.parse("child", new CompressedXContent(modParentMapping));
+        Exception e = expectThrows(IllegalArgumentException.class, () -> initMapper.merge(modParentMapper.mapping(), false));
+        assertThat(e.getMessage(), containsString("The _parent field's type option can't be changed: [parent]->[new_parent]"));
+    }
+
+    public void testMergeAddingParent() throws IOException {
+        DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();
+
+        String initMapping = XContentFactory.jsonBuilder().startObject().startObject("cowboy")
+            .startObject("properties")
+            .startObject("name").field("type", "text").endObject()
+            .endObject().endObject().endObject().string();
+        DocumentMapper initMapper = parser.parse("cowboy", new CompressedXContent(initMapping));
+
+        assertThat(initMapper.mappers().getMapper("name"), notNullValue());
+
+        String updatedMapping = XContentFactory.jsonBuilder().startObject().startObject("cowboy")
+            .startObject("_parent").field("type", "parent").endObject()
+            .startObject("properties")
+            .startObject("age").field("type", "byte").endObject()
+            .endObject().endObject().endObject().string();
+        DocumentMapper updatedMapper = parser.parse("cowboy", new CompressedXContent(updatedMapping));
+        Exception e = expectThrows(IllegalArgumentException.class, () -> initMapper.merge(updatedMapper.mapping(), false));
+        assertThat(e.getMessage(), containsString("The _parent field's type option can't be changed: [null]->[parent]"));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
index a36f6188713a1..a6a130ad9d4e3 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
@@ -19,34 +19,36 @@
 
 package org.elasticsearch.index.mapper;
 
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
 import org.apache.lucene.index.IndexableField;
-import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressedXContent;
-import org.elasticsearch.common.lucene.all.AllField;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.mapper.ParseContext.Document;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
 
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath;
 import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 
@@ -120,6 +122,137 @@ public void testDotsWithExistingMapper() throws Exception {
         assertEquals("789", values[2]);
     }
 
+    public void testDotsWithExistingNestedMapper() throws Exception {
+        DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
+            .startObject("foo").field("type", "nested").startObject("properties")
+            .startObject("bar").field("type", "integer")
+            .endObject().endObject().endObject().endObject().endObject().endObject().string();
+        DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
+
+        BytesReference bytes = XContentFactory.jsonBuilder()
+            .startObject()
+            .field("foo.bar", 123)
+            .endObject().bytes();
+        MapperParsingException e = expectThrows(MapperParsingException.class,
+                () -> mapper.parse("test", "type", "1", bytes));
+        assertEquals(
+                "Cannot add a value for field [foo.bar] since one of the intermediate objects is mapped as a nested object: [foo]",
+                e.getMessage());
+    }
+
+    public void testDotsWithDynamicNestedMapper() throws Exception {
+        DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startArray("dynamic_templates")
+                    .startObject()
+                        .startObject("objects_as_nested")
+                            .field("match_mapping_type", "object")
+                            .startObject("mapping")
+                                .field("type", "nested")
+                            .endObject()
+                        .endObject()
+                    .endObject()
+                .endArray().endObject().endObject().string();
+        DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
+
+        BytesReference bytes = XContentFactory.jsonBuilder()
+            .startObject()
+            .field("foo.bar",42)
+            .endObject().bytes();
+        MapperParsingException e = expectThrows(MapperParsingException.class,
+                () -> mapper.parse("test", "type", "1", bytes));
+        assertEquals(
+                "It is forbidden to create dynamic nested objects ([foo]) through `copy_to` or dots in field names",
+                e.getMessage());
+    }
+
+    public void testNestedHaveIdAndTypeFields() throws Exception {
+        DocumentMapperParser mapperParser1 = createIndex("index1", Settings.builder()
+            .put("index.mapping.single_type", false).build()
+        ).mapperService().documentMapperParser();
+        DocumentMapperParser mapperParser2 = createIndex("index2", Settings.builder()
+            .put("index.mapping.single_type", true).build()
+        ).mapperService().documentMapperParser();
+
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties");
+        {
+            mapping.startObject("foo");
+            mapping.field("type", "nested");
+            {
+                mapping.startObject("properties");
+                {
+
+                    mapping.startObject("bar");
+                    mapping.field("type", "keyword");
+                    mapping.endObject();
+                }
+                mapping.endObject();
+            }
+            mapping.endObject();
+        }
+        {
+            mapping.startObject("baz");
+            mapping.field("type", "keyword");
+            mapping.endObject();
+        }
+        mapping.endObject().endObject().endObject();
+        DocumentMapper mapper1 = mapperParser1.parse("type", new CompressedXContent(mapping.string()));
+        DocumentMapper mapper2 = mapperParser2.parse("type", new CompressedXContent(mapping.string()));
+
+        XContentBuilder doc = XContentFactory.jsonBuilder().startObject();
+        {
+            doc.startArray("foo");
+            {
+                doc.startObject();
+                doc.field("bar", "value1");
+                doc.endObject();
+            }
+            doc.endArray();
+            doc.field("baz", "value2");
+        }
+        doc.endObject();
+
+        // Verify in the case where multiple types are allowed that the _uid field is added to nested documents:
+        ParsedDocument result = mapper1.parse(SourceToParse.source("index1", "type", "1", doc.bytes(), XContentType.JSON));
+        assertEquals(2, result.docs().size());
+        // Nested document:
+        assertNull(result.docs().get(0).getField(IdFieldMapper.NAME));
+        assertNotNull(result.docs().get(0).getField(UidFieldMapper.NAME));
+        assertEquals("type#1", result.docs().get(0).getField(UidFieldMapper.NAME).stringValue());
+        assertEquals(UidFieldMapper.Defaults.NESTED_FIELD_TYPE, result.docs().get(0).getField(UidFieldMapper.NAME).fieldType());
+        assertNotNull(result.docs().get(0).getField(TypeFieldMapper.NAME));
+        assertEquals("__foo", result.docs().get(0).getField(TypeFieldMapper.NAME).stringValue());
+        assertEquals("value1", result.docs().get(0).getField("foo.bar").binaryValue().utf8ToString());
+        // Root document:
+        assertNull(result.docs().get(1).getField(IdFieldMapper.NAME));
+        assertNotNull(result.docs().get(1).getField(UidFieldMapper.NAME));
+        assertEquals("type#1", result.docs().get(1).getField(UidFieldMapper.NAME).stringValue());
+        assertEquals(UidFieldMapper.Defaults.FIELD_TYPE, result.docs().get(1).getField(UidFieldMapper.NAME).fieldType());
+        assertNotNull(result.docs().get(1).getField(TypeFieldMapper.NAME));
+        assertEquals("type", result.docs().get(1).getField(TypeFieldMapper.NAME).stringValue());
+        assertEquals("value2", result.docs().get(1).getField("baz").binaryValue().utf8ToString());
+
+        // Verify in the case where only a single type is allowed that the _id field is added to nested documents:
+        result = mapper2.parse(SourceToParse.source("index2", "type", "1", doc.bytes(), XContentType.JSON));
+        assertEquals(2, result.docs().size());
+        // Nested document:
+        assertNull(result.docs().get(0).getField(UidFieldMapper.NAME));
+        assertNotNull(result.docs().get(0).getField(IdFieldMapper.NAME));
+        assertEquals("1", result.docs().get(0).getField(IdFieldMapper.NAME).stringValue());
+        assertEquals(IdFieldMapper.Defaults.NESTED_FIELD_TYPE, result.docs().get(0).getField(IdFieldMapper.NAME).fieldType());
+        assertNotNull(result.docs().get(0).getField(TypeFieldMapper.NAME));
+        assertEquals("__foo", result.docs().get(0).getField(TypeFieldMapper.NAME).stringValue());
+        assertEquals("value1", result.docs().get(0).getField("foo.bar").binaryValue().utf8ToString());
+        // Root document:
+        assertNull(result.docs().get(1).getField(UidFieldMapper.NAME));
+        assertNotNull(result.docs().get(1).getField(IdFieldMapper.NAME));
+        assertEquals("1", result.docs().get(1).getField(IdFieldMapper.NAME).stringValue());
+        assertEquals(IdFieldMapper.Defaults.FIELD_TYPE, result.docs().get(1).getField(IdFieldMapper.NAME).fieldType());
+        assertNull(result.docs().get(1).getField(TypeFieldMapper.NAME));
+        assertEquals("value2", result.docs().get(1).getField("baz").binaryValue().utf8ToString());
+    }
+
     public void testPropagateDynamicWithExistingMapper() throws Exception {
         DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
@@ -293,7 +426,7 @@ public void testDynamicGeoPointArrayWithTemplate() throws Exception {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
             .startArray("dynamic_templates").startObject().startObject("georule")
                 .field("match", "foo*")
-                .startObject("mapping").field("type", "geo_point").endObject()
+                .startObject("mapping").field("type", "geo_point").field("doc_values", false).endObject()
             .endObject().endObject().endArray().endObject().endObject().string();
         DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
 
@@ -373,7 +506,7 @@ public void testDynamicStrictLongArray() throws Exception {
     public void testMappedGeoPointArray() throws Exception {
         DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("properties").startObject("foo").field("type", "geo_point")
+                .startObject("properties").startObject("foo").field("type", "geo_point").field("doc_values", false)
                 .endObject().endObject().endObject().endObject().string();
         DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
 
@@ -611,7 +744,8 @@ public void testDynamicDottedFieldNameLongArrayWithExistingParentWrongType() thr
                 .value(0)
                 .value(1)
             .endArray().endObject().bytes();
-        MapperParsingException exception = expectThrows(MapperParsingException.class, () -> mapper.parse("test", "type", "1", bytes));
+        MapperParsingException exception = expectThrows(MapperParsingException.class,
+                () -> mapper.parse("test", "type", "1", bytes));
         assertEquals("Could not dynamically add mapping for field [foo.bar.baz]. "
                 + "Existing mapping for [foo] must be of type object but found [long].", exception.getMessage());
     }
@@ -729,7 +863,8 @@ public void testDynamicDottedFieldNameLongWithExistingParentWrongType() throws E
         BytesReference bytes = XContentFactory.jsonBuilder()
                 .startObject().field("foo.bar.baz", 0)
             .endObject().bytes();
-        MapperParsingException exception = expectThrows(MapperParsingException.class, () -> mapper.parse("test", "type", "1", bytes));
+        MapperParsingException exception = expectThrows(MapperParsingException.class,
+                () -> mapper.parse("test", "type", "1", bytes));
         assertEquals("Could not dynamically add mapping for field [foo.bar.baz]. "
                 + "Existing mapping for [foo] must be of type object but found [long].", exception.getMessage());
     }
@@ -850,7 +985,8 @@ public void testDynamicDottedFieldNameObjectWithExistingParentWrongType() throws
 
         BytesReference bytes = XContentFactory.jsonBuilder().startObject().startObject("foo.bar.baz").field("a", 0).endObject().endObject()
                 .bytes();
-        MapperParsingException exception = expectThrows(MapperParsingException.class, () -> mapper.parse("test", "type", "1", bytes));
+        MapperParsingException exception = expectThrows(MapperParsingException.class,
+                () -> mapper.parse("test", "type", "1", bytes));
         assertEquals("Could not dynamically add mapping for field [foo.bar.baz]. "
                 + "Existing mapping for [foo] must be of type object but found [long].", exception.getMessage());
     }
@@ -1215,4 +1351,36 @@ public void testIncludeInAllPropagation() throws IOException {
         }
         assertEquals(new HashSet<>(Arrays.asList("b", "d")), values);
     }
+
+    public void testDynamicFieldsStartingAndEndingWithDot() throws Exception {
+        BytesReference bytes = XContentFactory.jsonBuilder().startObject().startArray("top.")
+                .startObject().startArray("foo.")
+                .startObject()
+                .field("thing", "bah")
+                .endObject().endArray()
+                .endObject().endArray()
+                .endObject().bytes();
+
+        client().prepareIndex("idx", "type").setSource(bytes).get();
+
+        bytes = XContentFactory.jsonBuilder().startObject().startArray("top.")
+                .startObject().startArray("foo.")
+                .startObject()
+                .startObject("bar.")
+                .startObject("aoeu")
+                .field("a", 1).field("b", 2)
+                .endObject()
+                .endObject()
+                .endObject()
+                .endArray().endObject().endArray()
+                .endObject().bytes();
+
+        try {
+            client().prepareIndex("idx", "type").setSource(bytes).get();
+            fail("should have failed to dynamically introduce a double-dot field");
+        } catch (IllegalArgumentException e) {
+            assertThat(e.getMessage(),
+                    containsString("object field starting or ending with a [.] makes object resolution ambiguous: [top..foo..bar]"));
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DoubleIndexingDocTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DoubleIndexingDocTests.java
index 62189aaa87aae..0fa58bbd27e03 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/DoubleIndexingDocTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/DoubleIndexingDocTests.java
@@ -29,6 +29,7 @@
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.ParsedDocument;
+import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
 import static org.hamcrest.Matchers.equalTo;
@@ -47,6 +48,7 @@ public void testDoubleIndexingSameDoc() throws Exception {
         IndexService index = createIndex("test");
         client().admin().indices().preparePutMapping("test").setType("type").setSource(mapping).get();
         DocumentMapper mapper = index.mapperService().documentMapper("type");
+        QueryShardContext context = index.newQueryShardContext(0, null, () -> 0L);
 
         ParsedDocument doc = mapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
                 .startObject()
@@ -67,25 +69,25 @@ public void testDoubleIndexingSameDoc() throws Exception {
         IndexReader reader = DirectoryReader.open(writer);
         IndexSearcher searcher = new IndexSearcher(reader);
 
-        TopDocs topDocs = searcher.search(mapper.mappers().smartNameFieldMapper("field1").fieldType().termQuery("value1", null), 10);
+        TopDocs topDocs = searcher.search(mapper.mappers().smartNameFieldMapper("field1").fieldType().termQuery("value1", context), 10);
         assertThat(topDocs.totalHits, equalTo(2));
 
-        topDocs = searcher.search(mapper.mappers().smartNameFieldMapper("field2").fieldType().termQuery("1", null), 10);
+        topDocs = searcher.search(mapper.mappers().smartNameFieldMapper("field2").fieldType().termQuery("1", context), 10);
         assertThat(topDocs.totalHits, equalTo(2));
 
-        topDocs = searcher.search(mapper.mappers().smartNameFieldMapper("field3").fieldType().termQuery("1.1", null), 10);
+        topDocs = searcher.search(mapper.mappers().smartNameFieldMapper("field3").fieldType().termQuery("1.1", context), 10);
         assertThat(topDocs.totalHits, equalTo(2));
 
-        topDocs = searcher.search(mapper.mappers().smartNameFieldMapper("field4").fieldType().termQuery("2010-01-01", null), 10);
+        topDocs = searcher.search(mapper.mappers().smartNameFieldMapper("field4").fieldType().termQuery("2010-01-01", context), 10);
         assertThat(topDocs.totalHits, equalTo(2));
 
-        topDocs = searcher.search(mapper.mappers().smartNameFieldMapper("field5").fieldType().termQuery("1", null), 10);
+        topDocs = searcher.search(mapper.mappers().smartNameFieldMapper("field5").fieldType().termQuery("1", context), 10);
         assertThat(topDocs.totalHits, equalTo(2));
 
-        topDocs = searcher.search(mapper.mappers().smartNameFieldMapper("field5").fieldType().termQuery("2", null), 10);
+        topDocs = searcher.search(mapper.mappers().smartNameFieldMapper("field5").fieldType().termQuery("2", context), 10);
         assertThat(topDocs.totalHits, equalTo(2));
 
-        topDocs = searcher.search(mapper.mappers().smartNameFieldMapper("field5").fieldType().termQuery("3", null), 10);
+        topDocs = searcher.search(mapper.mappers().smartNameFieldMapper("field5").fieldType().termQuery("3", context), 10);
         assertThat(topDocs.totalHits, equalTo(2));
         writer.close();
         reader.close();
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java
index 111f4b470d432..d3cad421b0651 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java
@@ -20,23 +20,33 @@
 package org.elasticsearch.index.mapper;
 
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.bulk.BulkItemResponse;
+import org.elasticsearch.action.bulk.BulkRequest;
+import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.bulk.TransportBulkAction;
+import org.elasticsearch.action.bulk.TransportShardBulkAction;
 import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.action.index.TransportIndexAction;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.AutoCreateIndex;
+import org.elasticsearch.action.update.UpdateHelper;
+import org.elasticsearch.client.Requests;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.MockTcpTransport;
+import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.transport.local.LocalTransport;
 import org.junit.After;
@@ -52,38 +62,39 @@
 
 public class DynamicMappingDisabledTests extends ESSingleNodeTestCase {
 
-    private static ThreadPool THREAD_POOL;
+    private static ThreadPool threadPool;
     private ClusterService clusterService;
     private LocalTransport transport;
     private TransportService transportService;
-    private IndicesService indicesService;
-    private ShardStateAction shardStateAction;
-    private ActionFilters actionFilters;
-    private IndexNameExpressionResolver indexNameExpressionResolver;
-    private AutoCreateIndex autoCreateIndex;
-    private Settings settings;
+    private TransportBulkAction transportBulkAction;
 
     @BeforeClass
     public static void createThreadPool() {
-        THREAD_POOL = new TestThreadPool("DynamicMappingDisabledTests");
+        threadPool = new TestThreadPool("DynamicMappingDisabledTests");
     }
 
     @Override
     public void setUp() throws Exception {
         super.setUp();
-        settings = Settings.builder()
+        Settings settings = Settings.builder()
                 .put(MapperService.INDEX_MAPPER_DYNAMIC_SETTING.getKey(), false)
                 .build();
-        clusterService = createClusterService(THREAD_POOL);
-        transport = new LocalTransport(settings, THREAD_POOL, new NamedWriteableRegistry(Collections.emptyList()),
+        clusterService = createClusterService(threadPool);
+        transport = new LocalTransport(settings, threadPool, new NamedWriteableRegistry(Collections.emptyList()),
                     new NoneCircuitBreakerService());
-        transportService = new TransportService(clusterService.getSettings(), transport, THREAD_POOL);
-        indicesService = getInstanceFromNode(IndicesService.class);
-        shardStateAction = new ShardStateAction(settings, clusterService, transportService, null, null, THREAD_POOL);
-        actionFilters = new ActionFilters(Collections.emptySet());
-        indexNameExpressionResolver = new IndexNameExpressionResolver(settings);
-        autoCreateIndex = new AutoCreateIndex(settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
-                indexNameExpressionResolver);
+        transportService = new TransportService(clusterService.getSettings(), transport, threadPool,
+            TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null);
+        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
+        ShardStateAction shardStateAction = new ShardStateAction(settings, clusterService, transportService, null, null, threadPool);
+        ActionFilters actionFilters = new ActionFilters(Collections.emptySet());
+        IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(settings);
+        AutoCreateIndex autoCreateIndex = new AutoCreateIndex(settings, new ClusterSettings(settings,
+                ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), indexNameExpressionResolver);
+        UpdateHelper updateHelper = new UpdateHelper(settings, null);
+        TransportShardBulkAction shardBulkAction = new TransportShardBulkAction(settings, transportService, clusterService,
+                indicesService, threadPool, shardStateAction, null, updateHelper, actionFilters, indexNameExpressionResolver);
+        transportBulkAction = new TransportBulkAction(settings, threadPool, transportService, clusterService,
+                null, shardBulkAction, null, actionFilters, indexNameExpressionResolver, autoCreateIndex, System::currentTimeMillis);
     }
 
     @After
@@ -96,34 +107,34 @@ public void tearDown() throws Exception {
 
     @AfterClass
     public static void destroyThreadPool() {
-        ThreadPool.terminate(THREAD_POOL, 30, TimeUnit.SECONDS);
+        ThreadPool.terminate(threadPool, 30, TimeUnit.SECONDS);
         // since static must set to null to be eligible for collection
-        THREAD_POOL = null;
+        threadPool = null;
     }
 
     public void testDynamicDisabled() {
-        TransportIndexAction action = new TransportIndexAction(settings, transportService, clusterService,
-                indicesService, THREAD_POOL, shardStateAction, null, null, actionFilters, indexNameExpressionResolver,
-                autoCreateIndex);
-
         IndexRequest request = new IndexRequest("index", "type", "1");
-        request.source("foo", 3);
-        final AtomicBoolean onFailureCalled = new AtomicBoolean();
+        request.source(Requests.INDEX_CONTENT_TYPE, "foo", 3);
+        BulkRequest bulkRequest = new BulkRequest();
+        bulkRequest.add(request);
+        final AtomicBoolean gotResponse = new AtomicBoolean();
 
-        action.execute(request, new ActionListener<IndexResponse>() {
+        transportBulkAction.execute(bulkRequest, new ActionListener<BulkResponse>() {
             @Override
-            public void onResponse(IndexResponse indexResponse) {
-                fail("Indexing request should have failed");
+            public void onResponse(BulkResponse bulkResponse) {
+                BulkItemResponse itemResponse = bulkResponse.getItems()[0];
+                assertTrue(itemResponse.isFailed());
+                assertThat(itemResponse.getFailure().getCause(), instanceOf(IndexNotFoundException.class));
+                assertEquals("no such index and [index.mapper.dynamic] is [false]", itemResponse.getFailure().getCause().getMessage());
+                gotResponse.set(true);
             }
 
             @Override
             public void onFailure(Exception e) {
-                onFailureCalled.set(true);
-                assertThat(e, instanceOf(IndexNotFoundException.class));
-                assertEquals(e.getMessage(), "no such index");
+                fail("unexpected failure in bulk action, expected failed bulk item");
             }
         });
 
-        assertTrue(onFailureCalled.get());
+        assertTrue(gotResponse.get());
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java
index 4d593318f54ac..41256113cd6bf 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java
@@ -28,8 +28,8 @@
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.mapper.BooleanFieldMapper.BooleanFieldType;
 import org.elasticsearch.index.mapper.DateFieldMapper.DateFieldType;
@@ -197,13 +197,15 @@ private String serialize(ToXContent mapper) throws Exception {
 
     private Mapper parse(DocumentMapper mapper, DocumentMapperParser parser, XContentBuilder builder) throws Exception {
         Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
-        SourceToParse source = SourceToParse.source("test", mapper.type(), "some_id", builder.bytes());
-        ParseContext.InternalParseContext ctx = new ParseContext.InternalParseContext(settings, parser, mapper, source, XContentHelper.createParser(source.source()));
-        assertEquals(XContentParser.Token.START_OBJECT, ctx.parser().nextToken());
-        ctx.parser().nextToken();
-        DocumentParser.parseObjectOrNested(ctx, mapper.root(), true);
-        Mapping mapping = DocumentParser.createDynamicUpdate(mapper.mapping(), mapper, ctx.getDynamicMappers());
-        return mapping == null ? null : mapping.root();
+        SourceToParse source = SourceToParse.source("test", mapper.type(), "some_id", builder.bytes(), builder.contentType());
+        try (XContentParser xContentParser = createParser(JsonXContent.jsonXContent, source.source())) {
+            ParseContext.InternalParseContext ctx = new ParseContext.InternalParseContext(settings, parser, mapper, source, xContentParser);
+            assertEquals(XContentParser.Token.START_OBJECT, ctx.parser().nextToken());
+            ctx.parser().nextToken();
+            DocumentParser.parseObjectOrNested(ctx, mapper.root(), true);
+            Mapping mapping = DocumentParser.createDynamicUpdate(mapper.mapping(), mapper, ctx.getDynamicMappers());
+            return mapping == null ? null : mapping.root();
+        }
     }
 
     public void testDynamicMappingsNotNeeded() throws Exception {
@@ -548,7 +550,7 @@ public void testMixTemplateMultiFieldAndMappingReuse() throws Exception {
         XContentBuilder json = XContentFactory.jsonBuilder().startObject()
                     .field("field", "foo")
                 .endObject();
-        SourceToParse source = SourceToParse.source("test", "type1", "1", json.bytes());
+        SourceToParse source = SourceToParse.source("test", "type1", "1", json.bytes(), json.contentType());
         DocumentMapper mapper = indexService.mapperService().documentMapper("type1");
         assertNull(mapper.mappers().getMapper("field.raw"));
         ParsedDocument parsed = mapper.parse(source);
@@ -642,6 +644,59 @@ public void testNumericDetectionDefault() throws Exception {
         assertThat(mapper, instanceOf(TextFieldMapper.class));
     }
 
+    public void testDateDetectionInheritsFormat() throws Exception {
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startArray("dynamic_date_formats")
+                    .value("yyyy-MM-dd")
+                .endArray()
+                .startArray("dynamic_templates")
+                    .startObject()
+                        .startObject("dates")
+                            .field("match_mapping_type", "date")
+                            .field("match", "*2")
+                            .startObject("mapping")
+                            .endObject()
+                        .endObject()
+                    .endObject()
+                    .startObject()
+                        .startObject("dates")
+                            .field("match_mapping_type", "date")
+                            .field("match", "*3")
+                            .startObject("mapping")
+                                .field("format", "yyyy-MM-dd||epoch_millis")
+                            .endObject()
+                        .endObject()
+                    .endObject()
+                .endArray()
+                .endObject().endObject().string();
+
+        IndexService index = createIndex("test");
+        client().admin().indices().preparePutMapping("test").setType("type").setSource(mapping).get();
+        DocumentMapper defaultMapper = index.mapperService().documentMapper("type");
+
+        ParsedDocument doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
+                .startObject()
+                    .field("date1", "2016-11-20")
+                    .field("date2", "2016-11-20")
+                    .field("date3", "2016-11-20")
+                .endObject()
+                .bytes());
+        assertNotNull(doc.dynamicMappingsUpdate());
+        assertAcked(client().admin().indices().preparePutMapping("test").setType("type").setSource(doc.dynamicMappingsUpdate().toString()).get());
+
+        defaultMapper = index.mapperService().documentMapper("type");
+
+        DateFieldMapper dateMapper1 = (DateFieldMapper) defaultMapper.mappers().smartNameFieldMapper("date1");
+        DateFieldMapper dateMapper2 = (DateFieldMapper) defaultMapper.mappers().smartNameFieldMapper("date2");
+        DateFieldMapper dateMapper3 = (DateFieldMapper) defaultMapper.mappers().smartNameFieldMapper("date3");
+        // inherited from dynamic date format
+        assertEquals("yyyy-MM-dd", dateMapper1.fieldType().dateTimeFormatter().format());
+        // inherited from dynamic date format since the mapping in the template did not specify a format
+        assertEquals("yyyy-MM-dd", dateMapper2.fieldType().dateTimeFormatter().format());
+        // not inherited from the dynamic date format since the template defined an explicit format
+        assertEquals("yyyy-MM-dd||epoch_millis", dateMapper3.fieldType().dateTimeFormatter().format());
+    }
+
     public void testDynamicTemplateOrder() throws IOException {
         // https://github.com/elastic/elasticsearch/issues/18625
         // elasticsearch used to apply templates that do not have a match_mapping_type first
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateTests.java
index 9830488989ce7..f9c94aad18450 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateTests.java
@@ -55,6 +55,8 @@ public void testParseUnknownMatchType() {
         templateDef.put("mapping", Collections.singletonMap("store", true));
         // if a wrong match type is specified, we ignore the template
         assertNull(DynamicTemplate.parse("my_template", templateDef, Version.V_5_0_0_alpha5));
+        assertWarnings("match_mapping_type [short] is invalid and will be ignored: No field type matched on [short], " +
+                "possible values are [object, string, long, double, boolean, date, binary]");
     }
 
     public void testMatchAllTemplate() {
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/ExternalFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/ExternalFieldMapperTests.java
index 6f76b1c59b885..664bfcd3fcc7f 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/ExternalFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/ExternalFieldMapperTests.java
@@ -25,9 +25,11 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.indices.mapper.MapperRegistry;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
@@ -38,7 +40,9 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.function.Supplier;
 
+import static org.hamcrest.Matchers.closeTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
 
@@ -59,8 +63,12 @@ public void testExternalValues() throws Exception {
                 Collections.singletonMap(ExternalMapperPlugin.EXTERNAL, new ExternalMapper.TypeParser(ExternalMapperPlugin.EXTERNAL, "foo")),
                 Collections.singletonMap(ExternalMetadataMapper.CONTENT_TYPE, new ExternalMetadataMapper.TypeParser()));
 
+        Supplier<QueryShardContext> queryShardContext = () -> {
+            return indexService.newQueryShardContext(0, null, () -> { throw new UnsupportedOperationException(); });
+        };
         DocumentMapperParser parser = new DocumentMapperParser(indexService.getIndexSettings(), indexService.mapperService(),
-                indexService.analysisService(), indexService.similarityService(), mapperRegistry, indexService::newQueryShardContext);
+                indexService.getIndexAnalyzers(), indexService.xContentRegistry(), indexService.similarityService(), mapperRegistry,
+                queryShardContext);
         DocumentMapper documentMapper = parser.parse("type", new CompressedXContent(
                 XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject(ExternalMetadataMapper.CONTENT_TYPE)
@@ -83,8 +91,10 @@ public void testExternalValues() throws Exception {
         assertThat(doc.rootDoc().getField("field.point"), notNullValue());
         if (version.before(Version.V_2_2_0)) {
             assertThat(doc.rootDoc().getField("field.point").stringValue(), is("42.0,51.0"));
-        } else {
-            assertThat(Long.parseLong(doc.rootDoc().getField("field.point").stringValue()), is(GeoPointField.encodeLatLon(42.0, 51.0)));
+        } else if (version.onOrAfter(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            GeoPoint point = new GeoPoint().resetFromIndexableField(doc.rootDoc().getField("field.point"));
+            assertThat(point.lat(), closeTo(42.0, 1e-5));
+            assertThat(point.lon(), closeTo(51.0, 1e-5));
         }
 
         assertThat(doc.rootDoc().getField("field.shape"), notNullValue());
@@ -106,8 +116,12 @@ public void testExternalValuesWithMultifield() throws Exception {
         mapperParsers.put(KeywordFieldMapper.CONTENT_TYPE, new KeywordFieldMapper.TypeParser());
         MapperRegistry mapperRegistry = new MapperRegistry(mapperParsers, Collections.emptyMap());
 
+        Supplier<QueryShardContext> queryShardContext = () -> {
+            return indexService.newQueryShardContext(0, null, () -> { throw new UnsupportedOperationException(); });
+        };
         DocumentMapperParser parser = new DocumentMapperParser(indexService.getIndexSettings(), indexService.mapperService(),
-                indexService.analysisService(), indexService.similarityService(), mapperRegistry, indexService::newQueryShardContext);
+                indexService.getIndexAnalyzers(), indexService.xContentRegistry(), indexService.similarityService(), mapperRegistry,
+                queryShardContext);
 
         DocumentMapper documentMapper = parser.parse("type", new CompressedXContent(
                 XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
@@ -141,8 +155,12 @@ public void testExternalValuesWithMultifield() throws Exception {
         assertThat(doc.rootDoc().getField("field.point"), notNullValue());
         if (version.before(Version.V_2_2_0)) {
             assertThat(doc.rootDoc().getField("field.point").stringValue(), is("42.0,51.0"));
-        } else {
+        } else if (version.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
             assertThat(Long.parseLong(doc.rootDoc().getField("field.point").stringValue()), is(GeoPointField.encodeLatLon(42.0, 51.0)));
+        } else {
+            GeoPoint point = new GeoPoint().resetFromIndexableField(doc.rootDoc().getField("field.point"));
+            assertThat(point.lat(), closeTo(42.0, 1E-5));
+            assertThat(point.lon(), closeTo(51.0, 1E-5));
         }
 
         IndexableField shape = doc.rootDoc().getField("field.shape");
@@ -172,8 +190,12 @@ public void testExternalValuesWithMultifieldTwoLevels() throws Exception {
         mapperParsers.put(TextFieldMapper.CONTENT_TYPE, new TextFieldMapper.TypeParser());
         MapperRegistry mapperRegistry = new MapperRegistry(mapperParsers, Collections.emptyMap());
 
+        Supplier<QueryShardContext> queryShardContext = () -> {
+            return indexService.newQueryShardContext(0, null, () -> { throw new UnsupportedOperationException(); });
+        };
         DocumentMapperParser parser = new DocumentMapperParser(indexService.getIndexSettings(), indexService.mapperService(),
-                indexService.analysisService(), indexService.similarityService(), mapperRegistry, indexService::newQueryShardContext);
+                indexService.getIndexAnalyzers(), indexService.xContentRegistry(), indexService.similarityService(), mapperRegistry,
+                queryShardContext);
 
         DocumentMapper documentMapper = parser.parse("type", new CompressedXContent(
                 XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
@@ -211,7 +233,7 @@ public void testExternalValuesWithMultifieldTwoLevels() throws Exception {
         assertThat(doc.rootDoc().getField("field.point"), notNullValue());
         if (version.before(Version.V_2_2_0)) {
             assertThat(doc.rootDoc().getField("field.point").stringValue(), is("42.0,51.0"));
-        } else {
+        } else if (version.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
             assertThat(Long.parseLong(doc.rootDoc().getField("field.point").stringValue()), is(GeoPointField.encodeLatLon(42.0, 51.0)));
         }
 
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java b/core/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java
index 2e75124c29af2..b21cc6be5a2f2 100755
--- a/core/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.apache.lucene.index.IndexableField;
 import org.locationtech.spatial4j.shape.Point;
 import org.apache.lucene.document.Field;
 import org.elasticsearch.Version;
@@ -59,6 +60,7 @@ public static class Builder extends FieldMapper.Builder<Builder, ExternalMapper>
         private BooleanFieldMapper.Builder boolBuilder = new BooleanFieldMapper.Builder(Names.FIELD_BOOL);
         private GeoPointFieldMapper.Builder pointBuilder = new GeoPointFieldMapper.Builder(Names.FIELD_POINT);
         private LegacyGeoPointFieldMapper.Builder legacyPointBuilder = new LegacyGeoPointFieldMapper.Builder(Names.FIELD_POINT);
+        private LatLonPointFieldMapper.Builder latLonPointBuilder = new LatLonPointFieldMapper.Builder(Names.FIELD_POINT);
         private GeoShapeFieldMapper.Builder shapeBuilder = new GeoShapeFieldMapper.Builder(Names.FIELD_SHAPE);
         private Mapper.Builder stringBuilder;
         private String generatedValue;
@@ -82,8 +84,14 @@ public ExternalMapper build(BuilderContext context) {
             context.path().add(name);
             BinaryFieldMapper binMapper = binBuilder.build(context);
             BooleanFieldMapper boolMapper = boolBuilder.build(context);
-            BaseGeoPointFieldMapper pointMapper = (context.indexCreatedVersion().before(Version.V_2_2_0)) ?
-                    legacyPointBuilder.build(context) : pointBuilder.build(context);
+            BaseGeoPointFieldMapper pointMapper;
+            if (context.indexCreatedVersion().before(Version.V_2_2_0)) {
+                pointMapper = legacyPointBuilder.build(context);
+            } else if (context.indexCreatedVersion().onOrAfter(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+                pointMapper = latLonPointBuilder.build(context);
+            } else {
+                pointMapper = pointBuilder.build(context);
+            }
             GeoShapeFieldMapper shapeMapper = shapeBuilder.build(context);
             FieldMapper stringMapper = (FieldMapper)stringBuilder.build(context);
             context.path().remove();
@@ -116,7 +124,7 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
 
     static class ExternalFieldType extends TermBasedFieldType {
 
-        public ExternalFieldType() {}
+        ExternalFieldType() {}
 
         protected ExternalFieldType(ExternalFieldType ref) {
             super(ref);
@@ -183,7 +191,7 @@ public Mapper parse(ParseContext context) throws IOException {
     }
 
     @Override
-    protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
+    protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
         throw new UnsupportedOperationException();
     }
 
@@ -198,7 +206,7 @@ public FieldMapper updateFieldType(Map<String, MappedFieldType> fullNameToFieldT
         MultiFields multiFieldsUpdate = multiFields.updateFieldType(fullNameToFieldType);
         BinaryFieldMapper binMapperUpdate = (BinaryFieldMapper) binMapper.updateFieldType(fullNameToFieldType);
         BooleanFieldMapper boolMapperUpdate = (BooleanFieldMapper) boolMapper.updateFieldType(fullNameToFieldType);
-        GeoPointFieldMapper pointMapperUpdate = (GeoPointFieldMapper) pointMapper.updateFieldType(fullNameToFieldType);
+        BaseGeoPointFieldMapper pointMapperUpdate = (BaseGeoPointFieldMapper) pointMapper.updateFieldType(fullNameToFieldType);
         GeoShapeFieldMapper shapeMapperUpdate = (GeoShapeFieldMapper) shapeMapper.updateFieldType(fullNameToFieldType);
         TextFieldMapper stringMapperUpdate = (TextFieldMapper) stringMapper.updateFieldType(fullNameToFieldType);
         if (update == this
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/ExternalMetadataMapper.java b/core/src/test/java/org/elasticsearch/index/mapper/ExternalMetadataMapper.java
index 6d585b7e7e760..9f815a7fdea9e 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/ExternalMetadataMapper.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/ExternalMetadataMapper.java
@@ -22,6 +22,7 @@
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Field.Store;
 import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.IndexableField;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.mapper.BooleanFieldMapper;
@@ -30,6 +31,7 @@
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MetadataFieldMapper;
 import org.elasticsearch.index.mapper.ParseContext;
+import org.elasticsearch.index.mapper.Mapper.TypeParser.ParserContext;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -54,7 +56,7 @@ protected ExternalMetadataMapper(Settings indexSettings) {
     }
 
     @Override
-    protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
+    protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
         // handled in post parse
     }
 
@@ -103,7 +105,8 @@ public static class TypeParser implements MetadataFieldMapper.TypeParser {
         }
 
         @Override
-        public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName) {
+        public MetadataFieldMapper getDefault(MappedFieldType fieldType, ParserContext context) {
+            final Settings indexSettings = context.mapperService().getIndexSettings().getSettings();
             return new ExternalMetadataMapper(indexSettings);
         }
 
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/ExternalValuesMapperIntegrationIT.java b/core/src/test/java/org/elasticsearch/index/mapper/ExternalValuesMapperIntegrationIT.java
index 342fa247d73f6..c75871e43b387 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/ExternalValuesMapperIntegrationIT.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/ExternalValuesMapperIntegrationIT.java
@@ -112,7 +112,7 @@ public void testExternalValues() throws Exception {
         assertThat(response.getHits().totalHits(), equalTo((long) 1));
 
         response = client().prepareSearch("test-idx")
-                .setPostFilter(QueryBuilders.geoDistanceRangeQuery("field.point", 42.0, 51.0).to("1km"))
+                .setPostFilter(QueryBuilders.geoDistanceQuery("field.point").point(42.0, 51.0).distance("1km"))
                 .execute().actionGet();
 
         assertThat(response.getHits().totalHits(), equalTo((long) 1));
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java b/core/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java
index 5a7c38ac05d0e..4c6418ee7ba51 100755
--- a/core/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java
@@ -22,6 +22,7 @@
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.SortedSetDocValuesField;
 import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.settings.Settings;
@@ -133,7 +134,7 @@ protected boolean customBoost() {
     }
 
     @Override
-    protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
+    protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
         StringFieldMapper.ValueAndBoost valueAndBoost = parseCreateFieldForString(context, fieldType().boost());
         if (valueAndBoost.value() == null) {
             return;
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/FieldLevelBoostTests.java b/core/src/test/java/org/elasticsearch/index/mapper/FieldLevelBoostTests.java
index 2d451a36df8d4..5786cfbe4e83a 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/FieldLevelBoostTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/FieldLevelBoostTests.java
@@ -26,8 +26,6 @@
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.ParseContext.Document;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
@@ -100,6 +98,8 @@ public void testBackCompatFieldLevelBoost() throws Exception {
 
         f = doc.getField("short_field");
         assertThat((double) f.boost(), closeTo(9.0, 0.001));
+
+        assertWarnings("The [norms{enabled:true/false}] way of specifying norms is deprecated, please use [norms:true/false] instead");
     }
 
     public void testBackCompatFieldLevelMappingBoost() throws Exception {
@@ -281,7 +281,7 @@ public void testBackCompatInvalidFieldLevelBoost() throws Exception {
         } catch (Exception ex) {
             assertThat(ex, instanceOf(MapperParsingException.class));
         }
-
+        assertWarnings("The [norms{enabled:true/false}] way of specifying norms is deprecated, please use [norms:true/false] instead");
     }
 
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldMapperTests.java
index baa9f72801987..374c22aaba7b7 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldMapperTests.java
@@ -24,24 +24,12 @@
 import org.apache.lucene.index.IndexableField;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.compress.CompressedXContent;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.IndexService;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.DocumentMapperParser;
-import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
-import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.mapper.Mapper;
-import org.elasticsearch.index.mapper.MapperParsingException;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.MetadataFieldMapper;
-import org.elasticsearch.index.mapper.ParseContext;
-import org.elasticsearch.index.mapper.ParsedDocument;
-import org.elasticsearch.index.mapper.TermBasedFieldType;
+import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.indices.IndicesModule;
 import org.elasticsearch.indices.mapper.MapperRegistry;
-import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
 import java.io.IOException;
@@ -51,6 +39,7 @@
 import java.util.Map;
 import java.util.SortedSet;
 import java.util.TreeSet;
+import java.util.function.Supplier;
 
 public class FieldNamesFieldMapperTests extends ESSingleNodeTestCase {
 
@@ -177,7 +166,8 @@ public DummyMetadataFieldMapper build(BuilderContext context) {
             }
 
             @Override
-            public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName) {
+            public MetadataFieldMapper getDefault(MappedFieldType fieldType, ParserContext context) {
+                final Settings indexSettings = context.mapperService().getIndexSettings().getSettings();
                 return new DummyMetadataFieldMapper(indexSettings);
             }
 
@@ -185,7 +175,7 @@ public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fi
 
         private static class DummyFieldType extends TermBasedFieldType {
 
-            public DummyFieldType() {
+            DummyFieldType() {
                 super();
             }
 
@@ -227,7 +217,7 @@ public void postParse(ParseContext context) throws IOException {
         }
 
         @Override
-        protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
+        protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
         }
 
         @Override
@@ -244,9 +234,14 @@ public void testSeesFieldsFromPlugins() throws IOException {
             Collections.singletonMap("_dummy", new DummyMetadataFieldMapper.TypeParser())
         );
         final MapperRegistry mapperRegistry = indicesModule.getMapperRegistry();
-        MapperService mapperService = new MapperService(indexService.getIndexSettings(), indexService.analysisService(), indexService.similarityService(), mapperRegistry, indexService::newQueryShardContext);
+        Supplier<QueryShardContext> queryShardContext = () -> {
+            return indexService.newQueryShardContext(0, null, () -> { throw new UnsupportedOperationException(); });
+        };
+        MapperService mapperService = new MapperService(indexService.getIndexSettings(), indexService.getIndexAnalyzers(),
+                indexService.xContentRegistry(), indexService.similarityService(), mapperRegistry, queryShardContext);
         DocumentMapperParser parser = new DocumentMapperParser(indexService.getIndexSettings(), mapperService,
-                indexService.analysisService(), indexService.similarityService(), mapperRegistry, indexService::newQueryShardContext);
+                indexService.getIndexAnalyzers(), indexService.xContentRegistry(), indexService.similarityService(), mapperRegistry,
+                queryShardContext);
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string();
         DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
         ParsedDocument parsedDocument = mapper.parse("index", "type", "id", new BytesArray("{}"));
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java b/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java
index 80e5df4081c85..4ae9b004413c3 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java
@@ -72,11 +72,12 @@ public void testAddExistingField() {
         MockFieldMapper f = new MockFieldMapper("foo");
         MockFieldMapper f2 = new MockFieldMapper("foo");
         FieldTypeLookup lookup = new FieldTypeLookup();
-        lookup = lookup.copyAndAddAll("type1", newList(f), randomBoolean());
-        FieldTypeLookup lookup2 = lookup.copyAndAddAll("type2", newList(f2), randomBoolean());
+        lookup = lookup.copyAndAddAll("type1", newList(f), true);
+        FieldTypeLookup lookup2 = lookup.copyAndAddAll("type2", newList(f2), true);
 
-        assertSame(f2.fieldType(), lookup2.get("foo"));
         assertEquals(1, size(lookup2.iterator()));
+        assertSame(f.fieldType(), lookup2.get("foo"));
+        assertEquals(f2.fieldType(), lookup2.get("foo"));
     }
 
     public void testAddExistingIndexName() {
@@ -206,7 +207,7 @@ private int size(Iterator<MappedFieldType> iterator) {
     }
 
     static class OtherFakeFieldType extends TermBasedFieldType {
-        public OtherFakeFieldType() {
+        OtherFakeFieldType() {
         }
 
         protected OtherFakeFieldType(OtherFakeFieldType ref) {
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java
index b5256761f6ef9..7843f378573e8 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java
@@ -18,7 +18,6 @@
  */
 package org.elasticsearch.index.mapper;
 
-import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.spatial.geopoint.document.GeoPointField;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.Version;
@@ -31,22 +30,17 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.DocumentMapperParser;
-import org.elasticsearch.index.mapper.MapperParsingException;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.SearchHitField;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
 import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.test.geo.RandomGeoGenerator;
+import org.hamcrest.CoreMatchers;
 
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
-import java.lang.NumberFormatException;
 
 import static com.carrotsearch.randomizedtesting.RandomizedTest.getRandom;
 import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
@@ -67,12 +61,12 @@ protected Collection<Class<? extends Plugin>> getPlugins() {
         return pluginList(InternalSettingsPlugin.class);
     }
 
-    public void testLatLonValues() throws Exception {
+    public void testLegacyLatLonValues() throws Exception {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true).endObject().endObject()
                 .endObject().endObject().string();
 
-        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT);
+        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.V_5_0_0_alpha5);
         Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
         DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
 
@@ -89,20 +83,23 @@ public void testLatLonValues() throws Exception {
         assertThat(doc.rootDoc().getField("point.lon"), notNullValue());
         assertThat(doc.rootDoc().getField("point.lon").fieldType().stored(), is(stored));
         assertThat(doc.rootDoc().getField("point.geohash"), nullValue());
-        if (indexCreatedBefore22 == true) {
+        if (indexCreatedBefore22) {
             assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3"));
         } else {
             assertThat(Long.parseLong(doc.rootDoc().get("point")), equalTo(GeoPointField.encodeLatLon(1.2, 1.3)));
         }
+        if (version.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            assertWarnings("geo_point lat_lon parameter is deprecated and will be removed in the next major release");
+        }
     }
 
-    public void testLatLonValuesWithGeohash() throws Exception {
+    public void testLegacyLatLonValuesWithGeohash() throws Exception {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true)
                 .field("geohash", true).endObject().endObject()
                 .endObject().endObject().string();
 
-        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT);
+        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.V_5_0_0_alpha5);
         Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
         DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
 
@@ -119,14 +116,18 @@ public void testLatLonValuesWithGeohash() throws Exception {
         } else {
             assertThat(doc.rootDoc().get("point.geohash"), equalTo(stringEncode(1.3, 1.2)));
         }
+        if (version.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            assertWarnings("geo_point lat_lon parameter is deprecated and will be removed in the next major release",
+                    "geo_point geohash parameter is deprecated and will be removed in the next major release");
+        }
     }
 
-    public void testLatLonInOneValueWithGeohash() throws Exception {
+    public void testLegacyLatLonInOneValueWithGeohash() throws Exception {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true)
                 .field("geohash", true).endObject().endObject().endObject().endObject().string();
 
-        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT);
+        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.V_5_0_0_alpha5);
         Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
         DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
 
@@ -143,14 +144,18 @@ public void testLatLonInOneValueWithGeohash() throws Exception {
         } else {
             assertThat(doc.rootDoc().get("point.geohash"), equalTo(stringEncode(1.3, 1.2)));
         }
+        if (version.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            assertWarnings("geo_point lat_lon parameter is deprecated and will be removed in the next major release",
+                    "geo_point geohash parameter is deprecated and will be removed in the next major release");
+        }
     }
 
-    public void testGeoHashIndexValue() throws Exception {
+    public void testLegacyGeoHashIndexValue() throws Exception {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true)
                 .field("geohash", true).endObject().endObject().endObject().endObject().string();
 
-        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT);
+        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.V_5_0_0_alpha5);
         Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
         DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
 
@@ -167,14 +172,20 @@ public void testGeoHashIndexValue() throws Exception {
         } else {
             assertThat(doc.rootDoc().get("point.geohash"), equalTo(stringEncode(1.3, 1.2)));
         }
+        if (version.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            assertWarnings("geo_point lat_lon parameter is deprecated and will be removed in the next major release",
+                    "geo_point geohash parameter is deprecated and will be removed in the next major release");
+        }
     }
 
     public void testGeoHashValue() throws Exception {
-        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true)
-                .endObject().endObject().endObject().endObject().string();
-
         Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT);
+        XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startObject("properties").startObject("point").field("type", "geo_point");
+        if (version.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            xContentBuilder = xContentBuilder.field("lat_lon", true);
+        }
+        String mapping = xContentBuilder.endObject().endObject().endObject().endObject().string();
         Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
         DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
 
@@ -184,13 +195,16 @@ public void testGeoHashValue() throws Exception {
                 .endObject()
                 .bytes());
 
-        assertThat(doc.rootDoc().getField("point.lat"), notNullValue());
-        assertThat(doc.rootDoc().getField("point.lon"), notNullValue());
-        assertThat(doc.rootDoc().get("point"), notNullValue());
+        assertThat(doc.rootDoc().getField("point"), notNullValue());
+        if (version.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            assertThat(doc.rootDoc().getField("point.lat"), notNullValue());
+            assertThat(doc.rootDoc().getField("point.lon"), notNullValue());
+            assertWarnings("geo_point lat_lon parameter is deprecated and will be removed in the next major release");
+        }
     }
 
-    public void testNormalizeLatLonValuesDefault() throws Exception {
-        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT);
+    public void testNormalizeLegacyLatLonValuesDefault() throws Exception {
+        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.V_5_0_0_alpha5);
         // default to normalize
         XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("properties").startObject("point").field("type", "geo_point");
@@ -239,8 +253,8 @@ public void testNormalizeLatLonValuesDefault() throws Exception {
         }
     }
 
-    public void testValidateLatLonValues() throws Exception {
-        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT);
+    public void testLegacyValidateLatLonValues() throws Exception {
+        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.V_5_0_0_alpha5);
         XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true);
         if (version.before(Version.V_2_2_0)) {
@@ -257,89 +271,67 @@ public void testValidateLatLonValues() throws Exception {
                 .endObject()
                 .bytes());
 
-        try {
+        expectThrows(MapperParsingException.class, () ->
             defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
                     .startObject()
                     .startObject("point").field("lat", -91).field("lon", 1.3).endObject()
                     .endObject()
-                    .bytes());
-            fail();
-        } catch (MapperParsingException e) {
+                    .bytes()));
 
-        }
-
-        try {
+        expectThrows(MapperParsingException.class, () ->
             defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
                     .startObject()
                     .startObject("point").field("lat", 91).field("lon", 1.3).endObject()
                     .endObject()
-                    .bytes());
-            fail();
-        } catch (MapperParsingException e) {
-
-        }
+                    .bytes()));
 
-        try {
+        expectThrows(MapperParsingException.class, () ->
             defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
                     .startObject()
                     .startObject("point").field("lat", 1.2).field("lon", -181).endObject()
                     .endObject()
-                    .bytes());
-            fail();
-        } catch (MapperParsingException e) {
-
-        }
+                    .bytes()));
 
-        try {
+        expectThrows(MapperParsingException.class, () ->
             defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
                     .startObject()
                     .startObject("point").field("lat", 1.2).field("lon", 181).endObject()
                     .endObject()
-                    .bytes());
-            fail();
-        } catch (MapperParsingException e) {
+                    .bytes()));
 
-        }
-
-        try {
+        MapperParsingException e = expectThrows(MapperParsingException.class, () ->
             defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
                     .startObject()
                     .startObject("point").field("lat", "-").field("lon", 1.3).endObject()
                     .endObject()
-                    .bytes());
-            fail();
-        } catch (MapperParsingException e) {
-            assertThat(e.getRootCause(), instanceOf(NumberFormatException.class));
-            assertThat(e.getRootCause().toString(), containsString("java.lang.NumberFormatException: For input string: \"-\""));
-        }
+                    .bytes()));
+        assertThat(e.getRootCause(), instanceOf(NumberFormatException.class));
+        assertThat(e.getRootCause().toString(), containsString("java.lang.NumberFormatException: For input string: \"-\""));
 
-        try {
+        e = expectThrows(MapperParsingException.class, () ->
             defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
                     .startObject()
                     .startObject("point").field("lat", 1.2).field("lon", "-").endObject()
                     .endObject()
-                    .bytes());
-            fail();
-        } catch (MapperParsingException e) {
-            assertThat(e.getRootCause(), instanceOf(NumberFormatException.class));
-            assertThat(e.getRootCause().toString(), containsString("java.lang.NumberFormatException: For input string: \"-\""));
-        }
+                    .bytes()));
+        assertThat(e.getRootCause(), instanceOf(NumberFormatException.class));
+        assertThat(e.getRootCause().toString(), containsString("java.lang.NumberFormatException: For input string: \"-\""));
 
-        try {
+        e = expectThrows(MapperParsingException.class, () ->
             defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
                     .startObject()
                     .startObject("point").field("lat", "-").field("lon", "-").endObject()
                     .endObject()
-                    .bytes());
-            fail();
-        } catch (MapperParsingException e) {
-            assertThat(e.getRootCause(), instanceOf(NumberFormatException.class));
-            assertThat(e.getRootCause().toString(), containsString("java.lang.NumberFormatException: For input string: \"-\""));
+                    .bytes()));
+        assertThat(e.getRootCause(), instanceOf(NumberFormatException.class));
+        assertThat(e.getRootCause().toString(), containsString("java.lang.NumberFormatException: For input string: \"-\""));
+        if (version.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            assertWarnings("geo_point lat_lon parameter is deprecated and will be removed in the next major release");
         }
     }
 
-    public void testNoValidateLatLonValues() throws Exception {
-        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT);
+    public void testNoValidateLegacyLatLonValues() throws Exception {
+        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.V_5_0_0_alpha5);
         XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true);
         if (version.before(Version.V_2_2_0)) {
@@ -397,14 +389,20 @@ public void testNoValidateLatLonValues() throws Exception {
                 .startObject("point").field("lat", "-").field("lon", "-").endObject()
                 .endObject()
                 .bytes());
+
+        if (version.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            assertWarnings("geo_point lat_lon parameter is deprecated and will be removed in the next major release");
+        }
     }
 
     public void testLatLonValuesStored() throws Exception {
-        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true)
-                .field("store", true).endObject().endObject().endObject().endObject().string();
-
-        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT);
+        Version version = VersionUtils.randomVersionBetween(random(), Version.CURRENT, Version.CURRENT);
+        XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startObject("properties").startObject("point").field("type", "geo_point");
+        if (version.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            xContentBuilder = xContentBuilder.field("lat_lon", true);
+        }
+        String mapping = xContentBuilder.field("store", true).endObject().endObject().endObject().endObject().string();
         Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
         DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
 
@@ -414,24 +412,29 @@ public void testLatLonValuesStored() throws Exception {
                 .endObject()
                 .bytes());
 
-        assertThat(doc.rootDoc().getField("point.lat"), notNullValue());
-        assertThat(doc.rootDoc().getField("point.lat").numericValue().doubleValue(), equalTo(1.2));
-        assertThat(doc.rootDoc().getField("point.lon"), notNullValue());
-        assertThat(doc.rootDoc().getField("point.lon").numericValue().doubleValue(), equalTo(1.3));
-        assertThat(doc.rootDoc().getField("point.geohash"), nullValue());
-        if (version.before(Version.V_2_2_0)) {
-            assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3"));
-        } else {
-            assertThat(Long.parseLong(doc.rootDoc().get("point")), equalTo(GeoPointField.encodeLatLon(1.2, 1.3)));
+        assertThat(doc.rootDoc().getField("point"), notNullValue());
+        if (version.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            assertThat(doc.rootDoc().getField("point.lat"), notNullValue());
+            assertThat(doc.rootDoc().getField("point.lat").numericValue().doubleValue(), equalTo(1.2));
+            assertThat(doc.rootDoc().getField("point.lon"), notNullValue());
+            assertThat(doc.rootDoc().getField("point.lon").numericValue().doubleValue(), equalTo(1.3));
+            assertThat(doc.rootDoc().getField("point.geohash"), nullValue());
+            if (version.before(Version.V_2_2_0)) {
+                assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3"));
+            } else {
+                assertThat(Long.parseLong(doc.rootDoc().get("point")), equalTo(GeoPointField.encodeLatLon(1.2, 1.3)));
+            }
         }
     }
 
     public void testArrayLatLonValues() throws Exception {
-        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true)
-                .field("store", true).endObject().endObject().endObject().endObject().string();
-
         Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT);
+        XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startObject("properties").startObject("point").field("type", "geo_point").field("doc_values", false);
+        if (version.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            xContentBuilder = xContentBuilder.field("lat_lon", true);
+        }
+        String mapping = xContentBuilder.field("store", true).endObject().endObject().endObject().endObject().string();
         Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
         DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
 
@@ -444,28 +447,14 @@ public void testArrayLatLonValues() throws Exception {
                 .endObject()
                 .bytes());
 
-        if (version.onOrAfter(Version.V_5_0_0_alpha2)) {
-            assertThat(doc.rootDoc().getFields("point.lat").length, equalTo(4));
-            assertThat(doc.rootDoc().getFields("point.lon").length, equalTo(4));
-
-            // point field for 1st value
-            assertThat(doc.rootDoc().getFields("point.lat")[0].numericValue().doubleValue(), equalTo(1.2));
-            assertThat(doc.rootDoc().getFields("point.lon")[0].numericValue().doubleValue(), equalTo(1.3));
-            // stored field for 1st value
-            assertThat(doc.rootDoc().getFields("point.lat")[1].numericValue().doubleValue(), equalTo(1.2));
-            assertThat(doc.rootDoc().getFields("point.lon")[1].numericValue().doubleValue(), equalTo(1.3));
-            // indexed hash
-            assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(GeoPointField.encodeLatLon(1.2, 1.3)));
-
-            // point field for 2nd value
-            assertThat(doc.rootDoc().getFields("point.lat")[2].numericValue().doubleValue(), equalTo(1.4));
-            assertThat(doc.rootDoc().getFields("point.lon")[2].numericValue().doubleValue(), equalTo(1.5));
-            // stored field for 2nd value
-            assertThat(doc.rootDoc().getFields("point.lat")[3].numericValue().doubleValue(), equalTo(1.4));
-            assertThat(doc.rootDoc().getFields("point.lon")[3].numericValue().doubleValue(), equalTo(1.5));
-            // indexed hash
-            assertThat(Long.parseLong(doc.rootDoc().getFields("point")[1].stringValue()), equalTo(GeoPointField.encodeLatLon(1.4, 1.5)));
+        // doc values are enabled by default, but in this test we disable them; we should only have 2 points
+        assertThat(doc.rootDoc().getFields("point"), notNullValue());
+        if (version.onOrAfter(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            assertThat(doc.rootDoc().getFields("point").length, equalTo(4));
         } else {
+            assertThat(doc.rootDoc().getFields("point").length, equalTo(2));
+        }
+        if (version.before(Version.V_5_0_0_alpha2)) {
             assertThat(doc.rootDoc().getFields("point.lat").length, equalTo(2));
             assertThat(doc.rootDoc().getFields("point.lon").length, equalTo(2));
             assertThat(doc.rootDoc().getFields("point.lat")[0].numericValue().doubleValue(), equalTo(1.2));
@@ -483,16 +472,22 @@ public void testArrayLatLonValues() throws Exception {
                 assertThat(Long.parseLong(doc.rootDoc().getFields("point")[1].stringValue()), equalTo(GeoPointField.encodeLatLon(1.4, 1.5)));
             }
         }
+        if (version.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            assertWarnings("geo_point lat_lon parameter is deprecated and will be removed in the next major release");
+        }
     }
 
     public void testLatLonInOneValue() throws Exception {
-        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true)
-                .endObject().endObject().endObject().endObject().string();
-
         Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT);
+        XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startObject("properties").startObject("point").field("type", "geo_point");
+        if (version.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            xContentBuilder = xContentBuilder.field("lat_lon", true);
+        }
+        String mapping = xContentBuilder.endObject().endObject().endObject().endObject().string();
         Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
-        DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser().parse("type",
+            new CompressedXContent(mapping));
 
         ParsedDocument doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
                 .startObject()
@@ -500,49 +495,67 @@ public void testLatLonInOneValue() throws Exception {
                 .endObject()
                 .bytes());
 
-        assertThat(doc.rootDoc().getField("point.lat"), notNullValue());
-        assertThat(doc.rootDoc().getField("point.lon"), notNullValue());
-        if (version.before(Version.V_2_2_0)) {
-            assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3"));
-        } else {
-            assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(GeoPointField.encodeLatLon(1.2, 1.3)));
+        assertThat(doc.rootDoc().getField("point"), notNullValue());
+        if (version.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            assertThat(doc.rootDoc().getField("point.lat"), notNullValue());
+            assertThat(doc.rootDoc().getField("point.lon"), notNullValue());
+            if (version.before(Version.V_2_2_0)) {
+                assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3"));
+            } else {
+                assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(GeoPointField.encodeLatLon(1.2, 1.3)));
+            }
+        }
+        if (version.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            assertWarnings("geo_point lat_lon parameter is deprecated and will be removed in the next major release");
         }
     }
 
     public void testLatLonInOneValueStored() throws Exception {
-        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true).field("store", true).endObject().endObject()
-                .endObject().endObject().string();
-
         Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT);
+        XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startObject("properties").startObject("point").field("type", "geo_point");
+        if (version.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            xContentBuilder = xContentBuilder.field("lat_lon", true);
+        }
+        String mapping = xContentBuilder.field("store", true).endObject().endObject().endObject().endObject().string();
         Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
-        DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser().parse("type",
+            new CompressedXContent(mapping));
 
         ParsedDocument doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
                 .startObject()
                 .field("point", "1.2,1.3")
                 .endObject()
                 .bytes());
-
-        assertThat(doc.rootDoc().getField("point.lat"), notNullValue());
-        assertThat(doc.rootDoc().getField("point.lat").numericValue().doubleValue(), equalTo(1.2));
-        assertThat(doc.rootDoc().getField("point.lon"), notNullValue());
-        assertThat(doc.rootDoc().getField("point.lon").numericValue().doubleValue(), equalTo(1.3));
-        if (version.before(Version.V_2_2_0)) {
-            assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3"));
-        } else {
-            assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(GeoPointField.encodeLatLon(1.2, 1.3)));
+        assertThat(doc.rootDoc().getField("point"), notNullValue());
+        if (version.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            assertThat(doc.rootDoc().getField("point.lat"), notNullValue());
+            assertThat(doc.rootDoc().getField("point.lat").numericValue().doubleValue(), equalTo(1.2));
+            assertThat(doc.rootDoc().getField("point.lon"), notNullValue());
+            assertThat(doc.rootDoc().getField("point.lon").numericValue().doubleValue(), equalTo(1.3));
+            if (version.before(Version.V_2_2_0)) {
+                assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3"));
+            } else {
+                assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()),
+                    equalTo(GeoPointField.encodeLatLon(1.2, 1.3)));
+            }
+        }
+        if (version.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            assertWarnings("geo_point lat_lon parameter is deprecated and will be removed in the next major release");
         }
     }
 
     public void testLatLonInOneValueArray() throws Exception {
-        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true)
-                .field("store", true).endObject().endObject().endObject().endObject().string();
-
         Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT);
+        XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startObject("properties").startObject("point").field("type", "geo_point").field("doc_values", false);
+        if (version.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            xContentBuilder = xContentBuilder.field("lat_lon", true);
+        }
+        String mapping = xContentBuilder.field("store", true).endObject().endObject().endObject().endObject().string();
         Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
-        DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser().parse("type",
+            new CompressedXContent(mapping));
 
         ParsedDocument doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
                 .startObject()
@@ -553,41 +566,42 @@ public void testLatLonInOneValueArray() throws Exception {
                 .endObject()
                 .bytes());
 
+        // doc values are enabled by default, but in this test we disable them; we should only have 2 points
+        assertThat(doc.rootDoc().getFields("point"), notNullValue());
         if (version.before(Version.V_5_0_0_alpha2)) {
+            assertThat(doc.rootDoc().getFields("point").length, equalTo(2));
             assertThat(doc.rootDoc().getFields("point.lat").length, equalTo(2));
             assertThat(doc.rootDoc().getFields("point.lon").length, equalTo(2));
             assertThat(doc.rootDoc().getFields("point.lat")[0].numericValue().doubleValue(), equalTo(1.2));
             assertThat(doc.rootDoc().getFields("point.lon")[0].numericValue().doubleValue(), equalTo(1.3));
             assertThat(doc.rootDoc().getFields("point.lat")[1].numericValue().doubleValue(), equalTo(1.4));
             assertThat(doc.rootDoc().getFields("point.lon")[1].numericValue().doubleValue(), equalTo(1.5));
-        } else {
-            IndexableField[] latPoints = doc.rootDoc().getFields("point.lat");
-            IndexableField[] lonPoints = doc.rootDoc().getFields("point.lon");
-            assertThat(latPoints.length, equalTo(4));
-            assertThat(lonPoints.length, equalTo(4));
-            assertThat(latPoints[0].numericValue().doubleValue(), equalTo(1.2));
-            assertThat(lonPoints[0].numericValue().doubleValue(), equalTo(1.3));
-            assertThat(latPoints[2].numericValue().doubleValue(), equalTo(1.4));
-            assertThat(lonPoints[2].numericValue().doubleValue(), equalTo(1.5));
+        } else if (version.onOrAfter(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            assertThat(doc.rootDoc().getFields("point").length, equalTo(4));
         }
         if (version.before(Version.V_2_2_0)) {
             assertThat(doc.rootDoc().getFields("point")[0].stringValue(), equalTo("1.2,1.3"));
-        } else {
+        } else if (version.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
             assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(GeoPointField.encodeLatLon(1.2, 1.3)));
         }
         if (version.before(Version.V_2_2_0)) {
             assertThat(doc.rootDoc().getFields("point")[1].stringValue(), equalTo("1.4,1.5"));
-        } else {
+        } else if (version.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
             assertThat(Long.parseLong(doc.rootDoc().getFields("point")[1].stringValue()), equalTo(GeoPointField.encodeLatLon(1.4, 1.5)));
         }
+        if (version.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            assertWarnings("geo_point lat_lon parameter is deprecated and will be removed in the next major release");
+        }
     }
 
     public void testLonLatArray() throws Exception {
-        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true).endObject().endObject()
-                .endObject().endObject().string();
-
         Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT);
+        XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startObject("properties").startObject("point").field("type", "geo_point");
+        if (version.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            xContentBuilder = xContentBuilder.field("lat_lon", true);
+        }
+        String mapping = xContentBuilder.endObject().endObject().endObject().endObject().string();
         Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
         DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
 
@@ -597,22 +611,28 @@ public void testLonLatArray() throws Exception {
                 .endObject()
                 .bytes());
 
-        assertThat(doc.rootDoc().getField("point.lat"), notNullValue());
-        assertThat(doc.rootDoc().getField("point.lon"), notNullValue());
-        if (version.before(Version.V_2_2_0)) {
-            assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3"));
-        } else {
-            assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(GeoPointField.encodeLatLon(1.2, 1.3)));
+        assertThat(doc.rootDoc().getField("point"), notNullValue());
+        if (version.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            assertThat(doc.rootDoc().getField("point.lat"), notNullValue());
+            assertThat(doc.rootDoc().getField("point.lon"), notNullValue());
+            if (version.before(Version.V_2_2_0)) {
+                assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3"));
+            } else {
+                assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(GeoPointField.encodeLatLon(1.2, 1.3)));
+            }
+            assertWarnings("geo_point lat_lon parameter is deprecated and will be removed in the next major release");
         }
     }
 
     public void testLonLatArrayDynamic() throws Exception {
-        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startArray("dynamic_templates").startObject()
-                .startObject("point").field("match", "point*").startObject("mapping").field("type", "geo_point")
-                .field("lat_lon", true).endObject().endObject().endObject().endArray().endObject().endObject().string();
-
-        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT);
+        Version version = VersionUtils.randomVersionBetween(random(), Version.CURRENT, Version.CURRENT);
+        XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startArray("dynamic_templates").startObject().startObject("point").field("match", "point*")
+            .startObject("mapping").field("type", "geo_point");
+        if (version.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            xContentBuilder = xContentBuilder.field("lat_lon", true);
+        }
+        String mapping = xContentBuilder.endObject().endObject().endObject().endArray().endObject().endObject().string();
         Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
         DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
 
@@ -622,47 +642,64 @@ public void testLonLatArrayDynamic() throws Exception {
                 .endObject()
                 .bytes());
 
-        assertThat(doc.rootDoc().getField("point.lat"), notNullValue());
-        assertThat(doc.rootDoc().getField("point.lon"), notNullValue());
-        if (version.before(Version.V_2_2_0)) {
-            assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3"));
-        } else {
-            assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(GeoPointField.encodeLatLon(1.2, 1.3)));
+        assertThat(doc.rootDoc().getField("point"), notNullValue());
+        if (version.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            assertThat(doc.rootDoc().getField("point.lat"), notNullValue());
+            assertThat(doc.rootDoc().getField("point.lon"), notNullValue());
+            if (version.before(Version.V_2_2_0)) {
+                assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3"));
+            } else {
+                assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(GeoPointField.encodeLatLon(1.2, 1.3)));
+            }
         }
     }
 
     public void testLonLatArrayStored() throws Exception {
-        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true)
-                .field("store", true).endObject().endObject().endObject().endObject().string();
-
         Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT);
+        XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startObject("properties").startObject("point").field("type", "geo_point");
+        if (version.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            xContentBuilder = xContentBuilder.field("lat_lon", true);
+        }
+        String mapping = xContentBuilder.field("store", true).endObject().endObject().endObject().endObject().string();
         Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
         DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
 
+        if (version.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            assertWarnings("geo_point lat_lon parameter is deprecated and will be removed in the next major release");
+        }
+
         ParsedDocument doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
                 .startObject()
                 .startArray("point").value(1.3).value(1.2).endArray()
                 .endObject()
                 .bytes());
 
-        assertThat(doc.rootDoc().getField("point.lat"), notNullValue());
-        assertThat(doc.rootDoc().getField("point.lat").numericValue().doubleValue(), equalTo(1.2));
-        assertThat(doc.rootDoc().getField("point.lon"), notNullValue());
-        assertThat(doc.rootDoc().getField("point.lon").numericValue().doubleValue(), equalTo(1.3));
-        if (version.before(Version.V_2_2_0)) {
-            assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3"));
+        assertThat(doc.rootDoc().getField("point"), notNullValue());
+        if (version.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            assertThat(doc.rootDoc().getField("point.lat"), notNullValue());
+            assertThat(doc.rootDoc().getField("point.lat").numericValue().doubleValue(), equalTo(1.2));
+            assertThat(doc.rootDoc().getField("point.lon"), notNullValue());
+            assertThat(doc.rootDoc().getField("point.lon").numericValue().doubleValue(), equalTo(1.3));
+            if (version.before(Version.V_2_2_0)) {
+                assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3"));
+            } else {
+                assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(GeoPointField.encodeLatLon(1.2, 1.3)));
+            }
         } else {
-            assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(GeoPointField.encodeLatLon(1.2, 1.3)));
+            assertThat(doc.rootDoc().getFields("point").length, equalTo(3));
         }
     }
 
     public void testLonLatArrayArrayStored() throws Exception {
-        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true)
-                .field("store", true).endObject().endObject().endObject().endObject().string();
-
         Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT);
+        XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startObject("properties").startObject("point").field("type", "geo_point");
+        if (version.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            xContentBuilder = xContentBuilder.field("lat_lon", true);
+        }
+        String mapping = xContentBuilder.field("store", true).field("doc_values", false).endObject().endObject()
+            .endObject().endObject().string();
         Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
         DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
 
@@ -675,7 +712,9 @@ public void testLonLatArrayArrayStored() throws Exception {
                 .endObject()
                 .bytes());
 
+        assertThat(doc.rootDoc().getFields("point"), notNullValue());
         if (version.before(Version.V_5_0_0_alpha2)) {
+            assertThat(doc.rootDoc().getFields("point").length, CoreMatchers.equalTo(2));
             assertThat(doc.rootDoc().getFields("point.lat").length, equalTo(2));
             assertThat(doc.rootDoc().getFields("point.lon").length, equalTo(2));
             assertThat(doc.rootDoc().getFields("point.lat")[0].numericValue().doubleValue(), equalTo(1.2));
@@ -692,19 +731,11 @@ public void testLonLatArrayArrayStored() throws Exception {
             } else {
                 assertThat(Long.parseLong(doc.rootDoc().getFields("point")[1].stringValue()), equalTo(GeoPointField.encodeLatLon(1.4, 1.5)));
             }
-        } else {
-            assertThat(doc.rootDoc().getFields("point.lat").length, equalTo(4));
-            assertThat(doc.rootDoc().getFields("point.lon").length, equalTo(4));
-            assertThat(doc.rootDoc().getFields("point.lat")[0].numericValue().doubleValue(), equalTo(1.2));
-            assertThat(doc.rootDoc().getFields("point.lat")[1].numericValue().doubleValue(), equalTo(1.2));
-            assertThat(doc.rootDoc().getFields("point.lon")[0].numericValue().doubleValue(), equalTo(1.3));
-            assertThat(doc.rootDoc().getFields("point.lon")[1].numericValue().doubleValue(), equalTo(1.3));
-            assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(GeoPointField.encodeLatLon(1.2, 1.3)));
-            assertThat(doc.rootDoc().getFields("point.lat")[2].numericValue().doubleValue(), equalTo(1.4));
-            assertThat(doc.rootDoc().getFields("point.lat")[3].numericValue().doubleValue(), equalTo(1.4));
-            assertThat(doc.rootDoc().getFields("point.lon")[2].numericValue().doubleValue(), equalTo(1.5));
-            assertThat(doc.rootDoc().getFields("point.lon")[3].numericValue().doubleValue(), equalTo(1.5));
-            assertThat(Long.parseLong(doc.rootDoc().getFields("point")[1].stringValue()), equalTo(GeoPointField.encodeLatLon(1.4, 1.5)));
+        } else if (version.onOrAfter(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            assertThat(doc.rootDoc().getFields("point").length, CoreMatchers.equalTo(4));
+        }
+        if (version.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            assertWarnings("geo_point lat_lon parameter is deprecated and will be removed in the next major release");
         }
     }
 
@@ -717,76 +748,107 @@ public void testOptionDeprecation() throws Exception {
         Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
         DocumentMapperParser parser = createIndex("test", settings).mapperService().documentMapperParser();
         // test deprecation exceptions on newly created indexes
-        try {
-            String validateMapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                    .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true).field("geohash", true)
-                    .field("validate", true).endObject().endObject()
+        if (version.onOrAfter(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            try {
+                String normalizeMapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                    .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true).endObject().endObject()
                     .endObject().endObject().string();
-            parser.parse("type", new CompressedXContent(validateMapping));
-            fail("process completed successfully when " + MapperParsingException.class.getName() + " expected");
-        } catch (MapperParsingException e) {
+                parser.parse("type", new CompressedXContent(normalizeMapping));
+            } catch (MapperParsingException e) {
+                assertEquals(e.getMessage(), "Mapping definition for [point] has unsupported parameters:  [lat_lon : true]");
+            }
+        }
+
+        if (version.onOrAfter(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            String normalizeMapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
+                    .startObject("point").field("type", "geo_point").field("geohash", true).endObject().endObject().endObject().endObject()
+                    .string();
+            Exception e = expectThrows(MapperParsingException.class, () ->
+                parser.parse("type", new CompressedXContent(normalizeMapping)));
+            assertEquals(e.getMessage(), "Mapping definition for [point] has unsupported parameters:  [geohash : true]");
+        }
+
+        {
+            XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
+                    .startObject("point").field("type", "geo_point");
+            if (version.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+                xContentBuilder = xContentBuilder.field("lat_lon", true).field("geohash", true);
+            }
+            String validateMapping = xContentBuilder.field("validate", true).endObject().endObject().endObject().endObject().string();
+            Exception e = expectThrows(MapperParsingException.class, () ->
+                parser.parse("type", new CompressedXContent(validateMapping)));
             assertEquals(e.getMessage(), "Mapping definition for [point] has unsupported parameters:  [validate : true]");
         }
 
-        try {
-            String validateMapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                    .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true).field("geohash", true)
-                    .field("validate_lat", true).endObject().endObject()
-                    .endObject().endObject().string();
-            parser.parse("type", new CompressedXContent(validateMapping));
-            fail("process completed successfully when " + MapperParsingException.class.getName() + " expected");
-        } catch (MapperParsingException e) {
+        {
+            XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties").startObject("point").field("type", "geo_point");
+            if (version.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+                xContentBuilder = xContentBuilder.field("lat_lon", true).field("geohash", true);
+            }
+            String validateMapping = xContentBuilder.field("validate_lat", true).endObject().endObject().endObject().endObject().string();
+            Exception e = expectThrows(MapperParsingException.class, () ->
+                parser.parse("type", new CompressedXContent(validateMapping)));
             assertEquals(e.getMessage(), "Mapping definition for [point] has unsupported parameters:  [validate_lat : true]");
         }
 
-        try {
-            String validateMapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                    .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true).field("geohash", true)
-                    .field("validate_lon", true).endObject().endObject()
-                    .endObject().endObject().string();
-            parser.parse("type", new CompressedXContent(validateMapping));
-            fail("process completed successfully when " + MapperParsingException.class.getName() + " expected");
-        } catch (MapperParsingException e) {
+        {
+            XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties").startObject("point").field("type", "geo_point");
+            if (version.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+                xContentBuilder = xContentBuilder.field("lat_lon", true).field("geohash", true);
+            }
+            String validateMapping = xContentBuilder.field("validate_lon", true).endObject().endObject().endObject().endObject().string();
+            Exception e = expectThrows(MapperParsingException.class, () ->
+                parser.parse("type", new CompressedXContent(validateMapping)));
             assertEquals(e.getMessage(), "Mapping definition for [point] has unsupported parameters:  [validate_lon : true]");
         }
 
         // test deprecated normalize
-        try {
-            String normalizeMapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                    .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true).field("geohash", true)
-                    .field("normalize", true).endObject().endObject()
-                    .endObject().endObject().string();
-            parser.parse("type", new CompressedXContent(normalizeMapping));
-            fail("process completed successfully when " + MapperParsingException.class.getName() + " expected");
-        } catch (MapperParsingException e) {
+        {
+            XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties").startObject("point").field("type", "geo_point");
+            if (version.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+                xContentBuilder = xContentBuilder.field("lat_lon", true).field("geohash", true);
+            }
+            String normalizeMapping = xContentBuilder.field("normalize", true).endObject().endObject().endObject().endObject().string();
+            Exception e = expectThrows(MapperParsingException.class, () ->
+                parser.parse("type", new CompressedXContent(normalizeMapping)));
             assertEquals(e.getMessage(), "Mapping definition for [point] has unsupported parameters:  [normalize : true]");
         }
 
-        try {
-            String normalizeMapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                    .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true).field("geohash", true)
-                    .field("normalize_lat", true).endObject().endObject()
-                    .endObject().endObject().string();
-            parser.parse("type", new CompressedXContent(normalizeMapping));
-            fail("process completed successfully when " + MapperParsingException.class.getName() + " expected");
-        } catch (MapperParsingException e) {
+        {
+            XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties").startObject("point").field("type", "geo_point");
+            if (version.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+                xContentBuilder = xContentBuilder.field("lat_lon", true).field("geohash", true);
+            }
+            String normalizeMapping = xContentBuilder.field("normalize_lat", true).endObject().endObject().endObject().endObject().string();
+            Exception e = expectThrows(MapperParsingException.class, () ->
+                parser.parse("type", new CompressedXContent(normalizeMapping)));
             assertEquals(e.getMessage(), "Mapping definition for [point] has unsupported parameters:  [normalize_lat : true]");
         }
 
-        try {
-            String normalizeMapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                    .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true).field("geohash", true)
-                    .field("normalize_lon", true).endObject().endObject()
-                    .endObject().endObject().string();
-            parser.parse("type", new CompressedXContent(normalizeMapping));
-            fail("process completed successfully when " + MapperParsingException.class.getName() + " expected");
-        } catch (MapperParsingException e) {
+        {
+            XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties").startObject("point").field("type", "geo_point");
+            if (version.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+                xContentBuilder = xContentBuilder.field("lat_lon", true).field("geohash", true);
+            }
+            String normalizeMapping = xContentBuilder.field("normalize_lon", true).endObject().endObject().endObject().endObject().string();
+            Exception e = expectThrows(MapperParsingException.class, () ->
+                parser.parse("type", new CompressedXContent(normalizeMapping)));
             assertEquals(e.getMessage(), "Mapping definition for [point] has unsupported parameters:  [normalize_lon : true]");
         }
+
+        if (version.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            assertWarnings("geo_point lat_lon parameter is deprecated and will be removed in the next major release",
+                    "geo_point geohash parameter is deprecated and will be removed in the next major release");
+        }
     }
 
-    public void testGeoPointMapperMerge() throws Exception {
-        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT);
+    public void testLegacyGeoPointMapperMerge() throws Exception {
+        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.V_5_0_0_alpha5);
         Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
         String stage1Mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true)
@@ -796,23 +858,24 @@ public void testGeoPointMapperMerge() throws Exception {
         String stage2Mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", false)
                 .field("geohash", false).endObject().endObject().endObject().endObject().string();
-        try {
-            mapperService.merge("type", new CompressedXContent(stage2Mapping), MapperService.MergeReason.MAPPING_UPDATE, false);
-            fail();
-        } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), containsString("mapper [point] has different [lat_lon]"));
-            assertThat(e.getMessage(), containsString("mapper [point] has different [geohash]"));
-            assertThat(e.getMessage(), containsString("mapper [point] has different [geohash_precision]"));
-        }
+        Exception e = expectThrows(IllegalArgumentException.class, () ->
+            mapperService.merge("type", new CompressedXContent(stage2Mapping), MapperService.MergeReason.MAPPING_UPDATE, false));
+        assertThat(e.getMessage(), containsString("mapper [point] has different [lat_lon]"));
+        assertThat(e.getMessage(), containsString("mapper [point] has different [geohash]"));
+        assertThat(e.getMessage(), containsString("mapper [point] has different [geohash_precision]"));
 
         // correct mapping and ensure no failures
-        stage2Mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+        String stage2MappingCorrect = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true)
                 .field("geohash", true).endObject().endObject().endObject().endObject().string();
-        mapperService.merge("type", new CompressedXContent(stage2Mapping), MapperService.MergeReason.MAPPING_UPDATE, false);
+        mapperService.merge("type", new CompressedXContent(stage2MappingCorrect), MapperService.MergeReason.MAPPING_UPDATE, false);
+        if (version.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            assertWarnings("geo_point lat_lon parameter is deprecated and will be removed in the next major release",
+                    "geo_point geohash parameter is deprecated and will be removed in the next major release");
+        }
     }
 
-    public void testGeoHashSearch() throws Exception {
+    public void testLegacyGeoHashSearch() throws Exception {
         // create a geo_point mapping with geohash enabled and random (between 1 and 12) geohash precision
         int precision = randomIntBetween(1, 12);
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("pin").startObject("properties").startObject("location")
@@ -820,7 +883,7 @@ public void testGeoHashSearch() throws Exception {
                 .endObject().endObject().endObject().string();
 
         // create index and add a test point (dr5regy6rc6z)
-        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT);
+        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.V_5_0_0_alpha1);
         Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
         CreateIndexRequestBuilder mappingRequest = client().admin().indices().prepareCreate("test").setSettings(settings)
                 .addMapping("pin", mapping);
@@ -837,7 +900,7 @@ public void testGeoHashSearch() throws Exception {
         assertEquals("dr5regy6rc6y".substring(0, precision), m.get("location.geohash").value());
     }
 
-    public void testGeoHashSearchWithPrefix() throws Exception {
+    public void testLegacyGeoHashSearchWithPrefix() throws Exception {
         // create a geo_point mapping with geohash enabled and random (between 1 and 12) geohash precision
         int precision = randomIntBetween(1, 12);
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("pin").startObject("properties").startObject("location")
@@ -845,7 +908,7 @@ public void testGeoHashSearchWithPrefix() throws Exception {
                 .endObject().endObject().endObject().endObject().string();
 
         // create index and add a test point (dr5regy6rc6z)
-        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT);
+        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.V_5_0_0_alpha5);
         Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
         CreateIndexRequestBuilder mappingRequest = client().admin().indices().prepareCreate("test").setSettings(settings)
                 .addMapping("pin", mapping);
@@ -870,9 +933,11 @@ public void testGeoHashSearchWithPrefix() throws Exception {
     public void testMultiField() throws Exception {
         int numDocs = randomIntBetween(10, 100);
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("pin").startObject("properties").startObject("location")
-            .field("type", "geo_point").startObject("fields")
-            .startObject("geohash").field("type", "geo_point").field("geohash_precision", 12).field("geohash_prefix", true).endObject()
-            .startObject("latlon").field("type", "geo_point").field("lat_lon", true).endObject().endObject()
+            .field("type", "geo_point")
+            .startObject("fields")
+            .startObject("geohash").field("type", "keyword").endObject()  // test geohash as keyword
+            .startObject("latlon").field("type", "string").endObject()  // test geohash as string
+            .endObject()
             .endObject().endObject().endObject().endObject().string();
         CreateIndexRequestBuilder mappingRequest = client().admin().indices().prepareCreate("test")
             .addMapping("pin", mapping);
@@ -886,6 +951,7 @@ public void testMultiField() throws Exception {
                 .field("lon", pt.lon()).endObject().endObject()).setRefreshPolicy(IMMEDIATE).get();
         }
 
+        // TODO these tests are bogus and need to be Fix
         // query by geohash subfield
         SearchResponse searchResponse = client().prepareSearch().addStoredField("location.geohash").setQuery(matchAllQuery()).execute().actionGet();
         assertEquals(numDocs, searchResponse.getHits().totalHits());
@@ -899,7 +965,7 @@ public void testMultiField() throws Exception {
     public void testEmptyName() throws Exception {
         // after 5.x
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-            .startObject("properties").startObject("").field("type", "geo_point").field("lat_lon", true).endObject().endObject()
+            .startObject("properties").startObject("").field("type", "geo_point").endObject().endObject()
             .endObject().endObject().string();
 
         Version version = Version.CURRENT;
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldTypeTests.java
index 6e6222ac87143..fac30002fbb98 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldTypeTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldTypeTests.java
@@ -18,16 +18,13 @@
  */
 package org.elasticsearch.index.mapper;
 
-import org.elasticsearch.index.mapper.BaseGeoPointFieldMapper;
-import org.elasticsearch.index.mapper.LegacyDoubleFieldMapper;
-import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.mapper.StringFieldMapper;
+import org.elasticsearch.index.mapper.BaseGeoPointFieldMapper.LegacyGeoPointFieldType;
 import org.junit.Before;
 
 public class GeoPointFieldTypeTests extends FieldTypeTestCase {
     @Override
     protected MappedFieldType createDefaultFieldType() {
-        return new BaseGeoPointFieldMapper.GeoPointFieldType();
+        return new LegacyGeoPointFieldType();
     }
 
     @Before
@@ -35,13 +32,14 @@ public void setupProperties() {
         addModifier(new Modifier("geohash", false) {
             @Override
             public void modify(MappedFieldType ft) {
-                ((BaseGeoPointFieldMapper.GeoPointFieldType)ft).setGeoHashEnabled(new StringFieldMapper.StringFieldType(), 1, true);
+                ((LegacyGeoPointFieldType)ft).setGeoHashEnabled(new StringFieldMapper.StringFieldType(), 1, true);
             }
         });
         addModifier(new Modifier("lat_lon", false) {
             @Override
             public void modify(MappedFieldType ft) {
-                ((BaseGeoPointFieldMapper.GeoPointFieldType)ft).setLatLonEnabled(new LegacyDoubleFieldMapper.DoubleFieldType(), new LegacyDoubleFieldMapper.DoubleFieldType());
+                ((LegacyGeoPointFieldType)ft).setLatLonEnabled(new LegacyDoubleFieldMapper.DoubleFieldType(),
+                    new LegacyDoubleFieldMapper.DoubleFieldType());
             }
         });
     }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java
index 4a22d56e8a941..5f258c1439632 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java
@@ -29,11 +29,6 @@
 import org.elasticsearch.common.geo.builders.ShapeBuilder;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.DocumentMapperParser;
-import org.elasticsearch.index.mapper.FieldMapper;
-import org.elasticsearch.index.mapper.GeoShapeFieldMapper;
-import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/GeohashMappingGeoPointTests.java b/core/src/test/java/org/elasticsearch/index/mapper/GeohashMappingGeoPointTests.java
deleted file mode 100644
index 05581e7902113..0000000000000
--- a/core/src/test/java/org/elasticsearch/index/mapper/GeohashMappingGeoPointTests.java
+++ /dev/null
@@ -1,171 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.mapper;
-
-import org.apache.lucene.spatial.geopoint.document.GeoPointField;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.Version;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.compress.CompressedXContent;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.index.mapper.BaseGeoPointFieldMapper;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.FieldMapper;
-import org.elasticsearch.index.mapper.ParsedDocument;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.test.ESSingleNodeTestCase;
-import org.elasticsearch.test.InternalSettingsPlugin;
-import org.elasticsearch.test.VersionUtils;
-
-import java.util.Collection;
-
-import static org.elasticsearch.common.geo.GeoHashUtils.stringEncode;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.notNullValue;
-import static org.hamcrest.Matchers.nullValue;
-
-/**
- *
- */
-public class GeohashMappingGeoPointTests extends ESSingleNodeTestCase {
-
-    @Override
-    protected Collection<Class<? extends Plugin>> getPlugins() {
-        return pluginList(InternalSettingsPlugin.class);
-    }
-
-    public void testLatLonValues() throws Exception {
-        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", false)
-                .endObject().endObject().endObject().endObject().string();
-
-        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT);
-        Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
-        DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
-
-        ParsedDocument doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
-                .startObject()
-                .startObject("point").field("lat", 1.2).field("lon", 1.3).endObject()
-                .endObject()
-                .bytes());
-
-        assertThat(doc.rootDoc().getField("point.lat"), nullValue());
-        assertThat(doc.rootDoc().getField("point.lon"), nullValue());
-        if (version.before(Version.V_2_2_0)) {
-            assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3"));
-        } else {
-            assertThat(Long.parseLong(doc.rootDoc().get("point")), equalTo(GeoPointField.encodeLatLon(1.2, 1.3)));
-        }
-    }
-
-    public void testLatLonInOneValue() throws Exception {
-        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", false).endObject().endObject()
-                .endObject().endObject().string();
-
-        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT);
-        Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
-        DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
-
-        ParsedDocument doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
-                .startObject()
-                .field("point", "1.2,1.3")
-                .endObject()
-                .bytes());
-
-        assertThat(doc.rootDoc().getField("point.lat"), nullValue());
-        assertThat(doc.rootDoc().getField("point.lon"), nullValue());
-        if (version.before(Version.V_2_2_0)) {
-            assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3"));
-        } else {
-            assertThat(Long.parseLong(doc.rootDoc().get("point")), equalTo(GeoPointField.encodeLatLon(1.2, 1.3)));
-        }
-    }
-
-    public void testGeoHashValue() throws Exception {
-        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("properties").startObject("point").field("type", "geo_point").field("geohash", true)
-                .endObject().endObject().endObject().endObject().string();
-
-        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT);
-        Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
-        DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
-
-        ParsedDocument doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
-                .startObject()
-                .field("point", stringEncode(1.3, 1.2))
-                .endObject()
-                .bytes());
-
-        assertThat(doc.rootDoc().getField("point.lat"), nullValue());
-        assertThat(doc.rootDoc().getField("point.lon"), nullValue());
-        assertThat(doc.rootDoc().getBinaryValue("point.geohash"), equalTo(new BytesRef(stringEncode(1.3, 1.2))));
-        assertThat(doc.rootDoc().get("point"), notNullValue());
-    }
-
-    public void testGeoHashPrecisionAsInteger() throws Exception {
-        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("properties").startObject("point").field("type", "geo_point").field("geohash", true)
-                .field("geohash_precision", 10).endObject().endObject().endObject().endObject().string();
-
-        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT);
-        Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
-        DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
-        FieldMapper mapper = defaultMapper.mappers().smartNameFieldMapper("point");
-        assertThat(mapper, instanceOf(BaseGeoPointFieldMapper.class));
-        BaseGeoPointFieldMapper geoPointFieldMapper = (BaseGeoPointFieldMapper) mapper;
-        assertThat(geoPointFieldMapper.fieldType().geoHashPrecision(), is(10));
-    }
-
-    public void testGeoHashPrecisionAsLength() throws Exception {
-        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("properties").startObject("point").field("type", "geo_point").field("geohash", true).field("geohash_precision", "5m").endObject().endObject()
-                .endObject().endObject().string();
-
-        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT);
-        Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
-        DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
-        FieldMapper mapper = defaultMapper.mappers().smartNameFieldMapper("point");
-        assertThat(mapper, instanceOf(BaseGeoPointFieldMapper.class));
-        BaseGeoPointFieldMapper geoPointFieldMapper = (BaseGeoPointFieldMapper) mapper;
-        assertThat(geoPointFieldMapper.fieldType().geoHashPrecision(), is(10));
-    }
-
-    public void testNullValue() throws Exception {
-        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("properties").startObject("point").field("type", "geo_point").endObject().endObject()
-                .endObject().endObject().string();
-
-        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT);
-        Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
-        DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
-
-        ParsedDocument doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
-                .startObject()
-                .field("point", (Object) null)
-                .endObject()
-                .bytes());
-
-        assertThat(doc.rootDoc().get("point"), nullValue());
-    }
-}
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java
index 55b6e7bf71c30..608f25db4cc48 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java
@@ -19,34 +19,21 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexableField;
+import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.IdFieldMapper;
-import org.elasticsearch.index.mapper.MapperParsingException;
-import org.elasticsearch.index.mapper.ParsedDocument;
-import org.elasticsearch.index.mapper.SourceToParse;
-import org.elasticsearch.index.mapper.UidFieldMapper;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.mapper.MapperService.MergeReason;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
-import static org.hamcrest.Matchers.notNullValue;
-import static org.hamcrest.Matchers.nullValue;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
 
 public class IdFieldMapperTests extends ESSingleNodeTestCase {
-    
-    public void testId() throws Exception {
-        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .endObject().endObject().string();
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
-
-        ParsedDocument doc = docMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
-                .startObject()
-                .endObject()
-                .bytes());
-
-        assertThat(doc.rootDoc().get(UidFieldMapper.NAME), notNullValue());
-        assertThat(doc.rootDoc().get(IdFieldMapper.NAME), nullValue());
-    }
 
     public void testIncludeInObjectNotAllowed() throws Exception {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string();
@@ -54,10 +41,34 @@ public void testIncludeInObjectNotAllowed() throws Exception {
 
         try {
             docMapper.parse(SourceToParse.source("test", "type", "1", XContentFactory.jsonBuilder()
-                .startObject().field("_id", "1").endObject().bytes()));
+                .startObject().field("_id", "1").endObject().bytes(), XContentType.JSON));
             fail("Expected failure to parse metadata field");
         } catch (MapperParsingException e) {
             assertTrue(e.getMessage(), e.getMessage().contains("Field [_id] is a metadata field and cannot be added inside a document"));
         }
     }
+
+    public void testDefaultsMultipleTypes() throws IOException {
+        Settings indexSettings = Settings.builder()
+                .put("index.mapping.single_type", false)
+                .build();
+        MapperService mapperService = createIndex("test", indexSettings).mapperService();
+        DocumentMapper mapper = mapperService.merge("type", new CompressedXContent("{\"type\":{}}"), MergeReason.MAPPING_UPDATE, false);
+        ParsedDocument document = mapper.parse(SourceToParse.source("index", "type", "id", new BytesArray("{}"), XContentType.JSON));
+        assertEquals(Collections.<IndexableField>emptyList(), Arrays.asList(document.rootDoc().getFields(IdFieldMapper.NAME)));
+    }
+
+    public void testDefaultsSingleType() throws IOException {
+        Settings indexSettings = Settings.builder()
+                .put("index.mapping.single_type", true)
+                .build();
+        MapperService mapperService = createIndex("test", indexSettings).mapperService();
+        DocumentMapper mapper = mapperService.merge("type", new CompressedXContent("{\"type\":{}}"), MergeReason.MAPPING_UPDATE, false);
+        ParsedDocument document = mapper.parse(SourceToParse.source("index", "type", "id", new BytesArray("{}"), XContentType.JSON));
+        IndexableField[] fields = document.rootDoc().getFields(IdFieldMapper.NAME);
+        assertEquals(1, fields.length);
+        assertEquals(IndexOptions.DOCS, fields[0].fieldType().indexOptions());
+        assertTrue(fields[0].fieldType().stored());
+        assertEquals("id", fields[0].stringValue());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/IdFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/IdFieldTypeTests.java
index 1cd9063da5fd2..2209027c12f37 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/IdFieldTypeTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/IdFieldTypeTests.java
@@ -18,8 +18,19 @@
  */
 package org.elasticsearch.index.mapper;
 
-import org.elasticsearch.index.mapper.IdFieldMapper;
-import org.elasticsearch.index.mapper.MappedFieldType;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermInSetQuery;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.mockito.Mockito;
+
+import java.util.Collection;
+import java.util.Collections;
 
 public class IdFieldTypeTests extends FieldTypeTestCase {
     @Override
@@ -31,7 +42,63 @@ public void testRangeQuery() {
         MappedFieldType ft = createDefaultFieldType();
         ft.setName("_id");
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-                () -> ft.rangeQuery(null, null, randomBoolean(), randomBoolean()));
+                () -> ft.rangeQuery(null, null, randomBoolean(), randomBoolean(), null));
         assertEquals("Field [_id] of type [_id] does not support range queries", e.getMessage());
     }
+
+    public void testTermsQueryWhenTypesAreEnabled() throws Exception {
+        QueryShardContext context = Mockito.mock(QueryShardContext.class);
+        Settings indexSettings = Settings.builder()
+                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
+                .put("index.mapping.single_type", false).build();
+        IndexMetaData indexMetaData = IndexMetaData.builder(IndexMetaData.INDEX_UUID_NA_VALUE).settings(indexSettings).build();
+        IndexSettings mockSettings = new IndexSettings(indexMetaData, Settings.EMPTY);
+        Mockito.when(context.getIndexSettings()).thenReturn(mockSettings);
+
+        MapperService mapperService = Mockito.mock(MapperService.class);
+        Collection<String> types = Collections.emptySet();
+        Mockito.when(context.queryTypes()).thenReturn(types);
+        Mockito.when(context.getMapperService()).thenReturn(mapperService);
+
+        MappedFieldType ft = IdFieldMapper.defaultFieldType(mockSettings);
+        ft.setName(IdFieldMapper.NAME);
+        Query query = ft.termQuery("id", context);
+        assertEquals(new TermInSetQuery("_uid"), query);
+
+        types = Collections.singleton("type");
+        Mockito.when(context.queryTypes()).thenReturn(types);
+        query = ft.termQuery("id", context);
+        assertEquals(new TermInSetQuery("_uid", new BytesRef("type#id")), query);
+    }
+
+    public void testTermsQueryWhenTypesAreDisabled() throws Exception {
+        QueryShardContext context = Mockito.mock(QueryShardContext.class);
+        Settings indexSettings = Settings.builder()
+                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
+                .put("index.mapping.single_type", true).build();
+        IndexMetaData indexMetaData = IndexMetaData.builder(IndexMetaData.INDEX_UUID_NA_VALUE).settings(indexSettings).build();
+        IndexSettings mockSettings = new IndexSettings(indexMetaData, Settings.EMPTY);
+        Mockito.when(context.getIndexSettings()).thenReturn(mockSettings);
+
+        MapperService mapperService = Mockito.mock(MapperService.class);
+        Collection<String> types = Collections.emptySet();
+        Mockito.when(context.queryTypes()).thenReturn(types);
+        Mockito.when(context.getMapperService()).thenReturn(mapperService);
+
+        MappedFieldType ft = IdFieldMapper.defaultFieldType(mockSettings);
+        ft.setName(IdFieldMapper.NAME);
+        Query query = ft.termQuery("id", context);
+        assertEquals(new TermInSetQuery("_id", new BytesRef("id")), query);
+
+        types = Collections.singleton("type");
+        Mockito.when(context.queryTypes()).thenReturn(types);
+        query = ft.termQuery("id", context);
+        assertEquals(new TermInSetQuery("_id", new BytesRef("id")), query);
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java
index 3d3a69ea80095..cbade6ced355f 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java
@@ -32,31 +32,26 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.IndexService;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.DocumentMapperParser;
-import org.elasticsearch.index.mapper.IpFieldMapper;
-import org.elasticsearch.index.mapper.MapperParsingException;
-import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
 import org.elasticsearch.test.VersionUtils;
 import org.junit.Before;
 
-import static com.carrotsearch.randomizedtesting.RandomizedTest.getRandom;
-import static org.hamcrest.Matchers.containsString;
-
 import java.io.IOException;
 import java.net.InetAddress;
 import java.util.Collection;
 
+import static com.carrotsearch.randomizedtesting.RandomizedTest.getRandom;
+import static org.hamcrest.Matchers.containsString;
+
 public class IpFieldMapperTests extends ESSingleNodeTestCase {
 
     IndexService indexService;
     DocumentMapperParser parser;
 
     @Before
-    public void before() {
+    public void setup() {
         indexService = createIndex("test");
         parser = indexService.mapperService().documentMapperParser();
     }
@@ -233,6 +228,8 @@ public void testIncludeInAll() throws Exception {
 
         fields = doc.rootDoc().getFields("_all");
         assertEquals(0, fields.length);
+        assertWarnings("field [include_in_all] is deprecated, as [_all] is deprecated, and will be disallowed" +
+                        " in 6.0, use [copy_to] instead.");
     }
 
     public void testNullValue() throws IOException {
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/IpFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/IpFieldTypeTests.java
index a5a1423d210c1..5c65aa5a09de7 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/IpFieldTypeTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/IpFieldTypeTests.java
@@ -19,14 +19,16 @@
 package org.elasticsearch.index.mapper;
 
 import java.net.InetAddress;
+import java.util.Arrays;
 
 import org.apache.lucene.document.InetAddressPoint;
 import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
+import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.network.InetAddresses;
-import org.elasticsearch.index.mapper.IpFieldMapper;
-import org.elasticsearch.index.mapper.MappedFieldType;
 
 public class IpFieldTypeTests extends FieldTypeTestCase {
     @Override
@@ -49,11 +51,11 @@ public void testValueForSearch() throws Exception {
         MappedFieldType ft = createDefaultFieldType();
         String ip = "2001:db8::2:1";
         BytesRef asBytes = new BytesRef(InetAddressPoint.encode(InetAddresses.forString(ip)));
-        assertEquals(ip, ft.valueForSearch(asBytes));
+        assertEquals(ip, ft.valueForDisplay(asBytes));
 
         ip = "192.168.1.7";
         asBytes = new BytesRef(InetAddressPoint.encode(InetAddresses.forString(ip)));
-        assertEquals(ip, ft.valueForSearch(asBytes));
+        assertEquals(ip, ft.valueForDisplay(asBytes));
     }
 
     public void testTermQuery() {
@@ -80,6 +82,22 @@ public void testTermQuery() {
         assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage());
     }
 
+    public void testTermsQuery() {
+        MappedFieldType ft = createDefaultFieldType();
+        ft.setName("field");
+
+        assertEquals(InetAddressPoint.newSetQuery("field", InetAddresses.forString("::2"), InetAddresses.forString("::5")),
+                ft.termsQuery(Arrays.asList(InetAddresses.forString("::2"), InetAddresses.forString("::5")), null));
+        assertEquals(InetAddressPoint.newSetQuery("field", InetAddresses.forString("::2"), InetAddresses.forString("::5")),
+                ft.termsQuery(Arrays.asList("::2", "::5"), null));
+
+        // if the list includes a prefix query we fallback to a bool query
+        assertEquals(new ConstantScoreQuery(new BooleanQuery.Builder()
+                .add(ft.termQuery("::42", null), Occur.SHOULD)
+                .add(ft.termQuery("::2/16", null), Occur.SHOULD).build()),
+                ft.termsQuery(Arrays.asList("::42", "::2/16"), null));
+    }
+
     public void testRangeQuery() {
         MappedFieldType ft = createDefaultFieldType();
         ft.setName("field");
@@ -88,83 +106,83 @@ public void testRangeQuery() {
                 InetAddressPoint.newRangeQuery("field",
                         InetAddresses.forString("::"),
                         InetAddressPoint.MAX_VALUE),
-                ft.rangeQuery(null, null, randomBoolean(), randomBoolean()));
+                ft.rangeQuery(null, null, randomBoolean(), randomBoolean(), null));
 
         assertEquals(
                 InetAddressPoint.newRangeQuery("field",
                         InetAddresses.forString("::"),
                         InetAddresses.forString("192.168.2.0")),
-                ft.rangeQuery(null, "192.168.2.0", randomBoolean(), true));
+                ft.rangeQuery(null, "192.168.2.0", randomBoolean(), true, null));
 
         assertEquals(
                 InetAddressPoint.newRangeQuery("field",
                         InetAddresses.forString("::"),
                         InetAddresses.forString("192.168.1.255")),
-                ft.rangeQuery(null, "192.168.2.0", randomBoolean(), false));
+                ft.rangeQuery(null, "192.168.2.0", randomBoolean(), false, null));
 
         assertEquals(
                 InetAddressPoint.newRangeQuery("field",
                         InetAddresses.forString("2001:db8::"),
                         InetAddressPoint.MAX_VALUE),
-                ft.rangeQuery("2001:db8::", null, true, randomBoolean()));
+                ft.rangeQuery("2001:db8::", null, true, randomBoolean(), null));
 
         assertEquals(
                 InetAddressPoint.newRangeQuery("field",
                         InetAddresses.forString("2001:db8::1"),
                         InetAddressPoint.MAX_VALUE),
-                ft.rangeQuery("2001:db8::", null, false, randomBoolean()));
+                ft.rangeQuery("2001:db8::", null, false, randomBoolean(), null));
 
         assertEquals(
                 InetAddressPoint.newRangeQuery("field",
                         InetAddresses.forString("2001:db8::"),
                         InetAddresses.forString("2001:db8::ffff")),
-                ft.rangeQuery("2001:db8::", "2001:db8::ffff", true, true));
+                ft.rangeQuery("2001:db8::", "2001:db8::ffff", true, true, null));
 
         assertEquals(
                 InetAddressPoint.newRangeQuery("field",
                         InetAddresses.forString("2001:db8::1"),
                         InetAddresses.forString("2001:db8::fffe")),
-                ft.rangeQuery("2001:db8::", "2001:db8::ffff", false, false));
+                ft.rangeQuery("2001:db8::", "2001:db8::ffff", false, false, null));
 
         assertEquals(
                 InetAddressPoint.newRangeQuery("field",
                         InetAddresses.forString("2001:db8::2"),
                         InetAddresses.forString("2001:db8::")),
                 // same lo/hi values but inclusive=false so this won't match anything
-                ft.rangeQuery("2001:db8::1", "2001:db8::1", false, false));
+                ft.rangeQuery("2001:db8::1", "2001:db8::1", false, false, null));
 
         // Upper bound is the min IP and is not inclusive
         assertEquals(new MatchNoDocsQuery(),
-                ft.rangeQuery("::", "::", true, false));
+                ft.rangeQuery("::", "::", true, false, null));
 
         // Lower bound is the max IP and is not inclusive
         assertEquals(new MatchNoDocsQuery(),
-                ft.rangeQuery("ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", false, true));
+                ft.rangeQuery("ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", false, true, null));
 
         assertEquals(
                 InetAddressPoint.newRangeQuery("field",
                         InetAddresses.forString("::"),
                         InetAddresses.forString("::fffe:ffff:ffff")),
                 // same lo/hi values but inclusive=false so this won't match anything
-                ft.rangeQuery("::", "0.0.0.0", true, false));
+                ft.rangeQuery("::", "0.0.0.0", true, false, null));
 
         assertEquals(
                 InetAddressPoint.newRangeQuery("field",
                         InetAddresses.forString("::1:0:0:0"),
                         InetAddressPoint.MAX_VALUE),
                 // same lo/hi values but inclusive=false so this won't match anything
-                ft.rangeQuery("255.255.255.255", "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", false, true));
+                ft.rangeQuery("255.255.255.255", "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", false, true, null));
 
         assertEquals(
                 // lower bound is ipv4, upper bound is ipv6
                 InetAddressPoint.newRangeQuery("field",
                         InetAddresses.forString("192.168.1.7"),
                         InetAddresses.forString("2001:db8::")),
-                ft.rangeQuery("::ffff:c0a8:107", "2001:db8::", true, true));
+                ft.rangeQuery("::ffff:c0a8:107", "2001:db8::", true, true, null));
 
         ft.setIndexOptions(IndexOptions.NONE);
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-                () -> ft.rangeQuery("::1", "2001::", true, true));
+                () -> ft.rangeQuery("::1", "2001::", true, true, null));
         assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage());
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java
index 396cbe49ee9d0..e1b692b2eac21 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java
@@ -30,6 +30,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.mapper.MapperService.MergeReason;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
@@ -55,8 +56,12 @@ protected Collection<Class<? extends Plugin>> getPlugins() {
     DocumentMapperParser parser;
 
     @Before
-    public void before() {
-        indexService = createIndex("test");
+    public void setup() {
+        indexService = createIndex("test", Settings.builder()
+                .put("index.analysis.normalizer.my_lowercase.type", "custom")
+                .putArray("index.analysis.normalizer.my_lowercase.filter", "lowercase")
+                .put("index.analysis.normalizer.my_asciifolding.type", "custom")
+                .putArray("index.analysis.normalizer.my_asciifolding.filter", "asciifolding").build());
         parser = indexService.mapperService().documentMapperParser();
     }
 
@@ -292,7 +297,6 @@ public void testBoostImplicitlyEnablesNormsOnOldIndex() throws IOException {
                             .field("boost", 2f)
                             .field("index", "not_analyzed")
                             .field("norms", true)
-                            .field("fielddata", false)
                         .endObject()
                     .endObject()
                 .endObject().endObject().string();
@@ -319,6 +323,62 @@ public void testEnableNorms() throws IOException {
         assertFalse(fields[0].fieldType().omitNorms());
     }
 
+    public void testNormalizer() throws IOException {
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties").startObject("field")
+                .field("type", "keyword").field("normalizer", "my_lowercase").endObject().endObject()
+                .endObject().endObject().string();
+
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+
+        assertEquals(mapping, mapper.mappingSource().toString());
+
+        ParsedDocument doc = mapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
+                .startObject()
+                .field("field", "AbC")
+                .endObject()
+                .bytes());
+
+        IndexableField[] fields = doc.rootDoc().getFields("field");
+        assertEquals(2, fields.length);
+
+        assertEquals(new BytesRef("abc"), fields[0].binaryValue());
+        IndexableFieldType fieldType = fields[0].fieldType();
+        assertThat(fieldType.omitNorms(), equalTo(true));
+        assertFalse(fieldType.tokenized());
+        assertFalse(fieldType.stored());
+        assertThat(fieldType.indexOptions(), equalTo(IndexOptions.DOCS));
+        assertThat(fieldType.storeTermVectors(), equalTo(false));
+        assertThat(fieldType.storeTermVectorOffsets(), equalTo(false));
+        assertThat(fieldType.storeTermVectorPositions(), equalTo(false));
+        assertThat(fieldType.storeTermVectorPayloads(), equalTo(false));
+        assertEquals(DocValuesType.NONE, fieldType.docValuesType());
+
+        assertEquals(new BytesRef("abc"), fields[1].binaryValue());
+        fieldType = fields[1].fieldType();
+        assertThat(fieldType.indexOptions(), equalTo(IndexOptions.NONE));
+        assertEquals(DocValuesType.SORTED_SET, fieldType.docValuesType());
+    }
+
+    public void testUpdateNormalizer() throws IOException {
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties").startObject("field")
+                .field("type", "keyword").field("normalizer", "my_lowercase").endObject().endObject()
+                .endObject().endObject().string();
+        indexService.mapperService().merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE, randomBoolean());
+
+        String mapping2 = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties").startObject("field")
+                .field("type", "keyword").field("normalizer", "my_asciifolding").endObject().endObject()
+                .endObject().endObject().string();
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> indexService.mapperService().merge("type",
+                        new CompressedXContent(mapping2), MergeReason.MAPPING_UPDATE, randomBoolean()));
+        assertEquals(
+                "Mapper for [field] conflicts with existing mapping in other types:\n[mapper [field] has different [normalizer]]",
+                e.getMessage());
+    }
+
     public void testEmptyName() throws IOException {
         String mapping = XContentFactory.jsonBuilder().startObject()
                 .startObject("type")
@@ -348,7 +408,6 @@ public void testEmptyName() throws IOException {
                         .startObject("")
                             .field("type", "string")
                             .field("index", "not_analyzed")
-                            .field("fielddata", false)
                         .endObject()
                     .endObject()
                 .endObject().endObject().string();
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java
index 2f54af2601b45..809ceb5831004 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java
@@ -20,22 +20,44 @@
 
 import com.carrotsearch.randomizedtesting.generators.RandomStrings;
 
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.LowerCaseFilter;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.core.WhitespaceTokenizer;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.queries.TermsQuery;
+import org.apache.lucene.search.TermInSetQuery;
 import org.apache.lucene.search.FuzzyQuery;
 import org.apache.lucene.search.RegexpQuery;
 import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.unit.Fuzziness;
-import org.elasticsearch.index.mapper.KeywordFieldMapper;
-import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.analysis.AnalyzerScope;
+import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.mapper.KeywordFieldMapper.KeywordFieldType;
 import org.elasticsearch.index.mapper.MappedFieldType.Relation;
+import org.junit.Before;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.List;
 
 public class KeywordFieldTypeTests extends FieldTypeTestCase {
+
+    @Before
+    public void setupProperties() {
+        addModifier(new Modifier("normalizer", false) {
+            @Override
+            public void modify(MappedFieldType ft) {
+                ((KeywordFieldType) ft).setNormalizer(Lucene.KEYWORD_ANALYZER);
+            }
+        });
+    }
+
     @Override
     protected MappedFieldType createDefaultFieldType() {
         return new KeywordFieldMapper.KeywordFieldType();
@@ -47,7 +69,7 @@ public void testIsFieldWithinQuery() throws IOException {
         assertEquals(Relation.INTERSECTS, ft.isFieldWithinQuery(null,
                 RandomStrings.randomAsciiOfLengthBetween(random(), 0, 5),
                 RandomStrings.randomAsciiOfLengthBetween(random(), 0, 5),
-                randomBoolean(), randomBoolean(), null, null));
+                randomBoolean(), randomBoolean(), null, null, null));
     }
 
     public void testTermQuery() {
@@ -62,11 +84,39 @@ public void testTermQuery() {
         assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage());
     }
 
+    public void testTermQueryWithNormalizer() {
+        MappedFieldType ft = createDefaultFieldType();
+        ft.setName("field");
+        ft.setIndexOptions(IndexOptions.DOCS);
+        Analyzer normalizer = new Analyzer() {
+            @Override
+            protected TokenStreamComponents createComponents(String fieldName) {
+                Tokenizer in = new WhitespaceTokenizer();
+                TokenFilter out = new LowerCaseFilter(in);
+                return new TokenStreamComponents(in, out);
+            }
+            @Override
+            protected TokenStream normalize(String fieldName, TokenStream in) {
+                return new LowerCaseFilter(in);
+            }
+        };
+        ft.setSearchAnalyzer(new NamedAnalyzer("my_normalizer", AnalyzerScope.INDEX, normalizer));
+        assertEquals(new TermQuery(new Term("field", "foo bar")), ft.termQuery("fOo BaR", null));
+
+        ft.setIndexOptions(IndexOptions.NONE);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> ft.termQuery("bar", null));
+        assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage());
+    }
+
     public void testTermsQuery() {
         MappedFieldType ft = createDefaultFieldType();
         ft.setName("field");
         ft.setIndexOptions(IndexOptions.DOCS);
-        assertEquals(new TermsQuery(new Term("field", "foo"), new Term("field", "bar")),
+        List<BytesRef> terms = new ArrayList<>();
+        terms.add(new BytesRef("foo"));
+        terms.add(new BytesRef("bar"));
+        assertEquals(new TermInSetQuery("field", terms),
                 ft.termsQuery(Arrays.asList("foo", "bar"), null));
 
         ft.setIndexOptions(IndexOptions.NONE);
@@ -100,4 +150,13 @@ public void testFuzzyQuery() {
                 () -> ft.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true));
         assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage());
     }
+
+    public void testNormalizeQueries() {
+        MappedFieldType ft = createDefaultFieldType();
+        ft.setName("field");
+        ft.setSearchAnalyzer(Lucene.KEYWORD_ANALYZER);
+        assertEquals(new TermQuery(new Term("field", new BytesRef("FOO"))), ft.termQuery("FOO", null));
+        ft.setSearchAnalyzer(Lucene.STANDARD_ANALYZER);
+        assertEquals(new TermQuery(new Term("field", new BytesRef("foo"))), ft.termQuery("FOO", null));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/LegacyByteFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/LegacyByteFieldTypeTests.java
index 1f24acd4c207a..2f3a4ca6fec9c 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/LegacyByteFieldTypeTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/LegacyByteFieldTypeTests.java
@@ -18,8 +18,6 @@
  */
 package org.elasticsearch.index.mapper;
 
-import org.elasticsearch.index.mapper.LegacyByteFieldMapper;
-import org.elasticsearch.index.mapper.MappedFieldType;
 import org.junit.Before;
 
 public class LegacyByteFieldTypeTests extends FieldTypeTestCase {
@@ -36,6 +34,6 @@ public void setupProperties() {
     public void testValueForSearch() {
         MappedFieldType ft = createDefaultFieldType();
         // bytes are stored as ints
-        assertEquals(Byte.valueOf((byte) 3), ft.valueForSearch(Integer.valueOf(3)));
+        assertEquals(Byte.valueOf((byte) 3), ft.valueForDisplay(Integer.valueOf(3)));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/LegacyDateFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/LegacyDateFieldMapperTests.java
index 79703986c83c8..113064239e687 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/LegacyDateFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/LegacyDateFieldMapperTests.java
@@ -40,12 +40,12 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.ParseContext.Document;
+import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
-import org.elasticsearch.test.TestSearchContext;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.junit.Before;
@@ -235,6 +235,12 @@ public void testDateDetection() throws Exception {
     }
 
     public void testHourFormat() throws Exception {
+        long nowInMillis = randomNonNegativeLong();
+        Settings indexSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1).build();
+        QueryShardContext context = new QueryShardContext(0,
+                new IndexSettings(IndexMetaData.builder("foo").settings(indexSettings).build(), indexSettings), null, null, null, null,
+                null, xContentRegistry(), null, null, () -> nowInMillis);
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .field("date_detection", false)
                 .startObject("properties").startObject("date_field").field("type", "date").field("format", "HH:mm:ss").endObject().endObject()
@@ -249,18 +255,19 @@ public void testHourFormat() throws Exception {
             .bytes());
         assertThat(((LegacyLongFieldMapper.CustomLongNumericField) doc.rootDoc().getField("date_field")).numericAsString(), equalTo(Long.toString(new DateTime(TimeValue.timeValueHours(10).millis(), DateTimeZone.UTC).getMillis())));
 
-        LegacyNumericRangeQuery<Long> rangeQuery;
-        try {
-            SearchContext.setCurrent(new TestSearchContext(null));
-            rangeQuery = (LegacyNumericRangeQuery<Long>) defaultMapper.mappers().smartNameFieldMapper("date_field").fieldType().rangeQuery("10:00:00", "11:00:00", true, true).rewrite(null);
-        } finally {
-            SearchContext.removeCurrent();
-        }
-        assertThat(rangeQuery.getMax(), equalTo(new DateTime(TimeValue.timeValueHours(11).millis(), DateTimeZone.UTC).getMillis()));
+        LegacyNumericRangeQuery<Long> rangeQuery = (LegacyNumericRangeQuery<Long>) defaultMapper.mappers().smartNameFieldMapper("date_field").fieldType()
+                .rangeQuery("10:00:00", "11:00:00", true, true, context);
+        assertThat(rangeQuery.getMax(), equalTo(new DateTime(TimeValue.timeValueHours(11).millis(), DateTimeZone.UTC).getMillis() + 999));
         assertThat(rangeQuery.getMin(), equalTo(new DateTime(TimeValue.timeValueHours(10).millis(), DateTimeZone.UTC).getMillis()));
     }
 
     public void testDayWithoutYearFormat() throws Exception {
+        long nowInMillis = randomNonNegativeLong();
+        Settings indexSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1).build();
+        QueryShardContext context = new QueryShardContext(0,
+                new IndexSettings(IndexMetaData.builder("foo").settings(indexSettings).build(), indexSettings), null, null, null, null,
+                null, xContentRegistry(), null, null, () -> nowInMillis);
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .field("date_detection", false)
                 .startObject("properties").startObject("date_field").field("type", "date").field("format", "MMM dd HH:mm:ss").endObject().endObject()
@@ -275,14 +282,9 @@ public void testDayWithoutYearFormat() throws Exception {
                 .bytes());
         assertThat(((LegacyLongFieldMapper.CustomLongNumericField) doc.rootDoc().getField("date_field")).numericAsString(), equalTo(Long.toString(new DateTime(TimeValue.timeValueHours(34).millis(), DateTimeZone.UTC).getMillis())));
 
-        LegacyNumericRangeQuery<Long> rangeQuery;
-        try {
-            SearchContext.setCurrent(new TestSearchContext(null));
-            rangeQuery = (LegacyNumericRangeQuery<Long>) defaultMapper.mappers().smartNameFieldMapper("date_field").fieldType().rangeQuery("Jan 02 10:00:00", "Jan 02 11:00:00", true, true).rewrite(null);
-        } finally {
-            SearchContext.removeCurrent();
-        }
-        assertThat(rangeQuery.getMax(), equalTo(new DateTime(TimeValue.timeValueHours(35).millis(), DateTimeZone.UTC).getMillis()));
+        LegacyNumericRangeQuery<Long> rangeQuery = (LegacyNumericRangeQuery<Long>) defaultMapper.mappers().smartNameFieldMapper("date_field").fieldType()
+                .rangeQuery("Jan 02 10:00:00", "Jan 02 11:00:00", true, true, context);
+        assertThat(rangeQuery.getMax(), equalTo(new DateTime(TimeValue.timeValueHours(35).millis() + 999, DateTimeZone.UTC).getMillis()));
         assertThat(rangeQuery.getMin(), equalTo(new DateTime(TimeValue.timeValueHours(34).millis(), DateTimeZone.UTC).getMillis()));
     }
 
@@ -410,7 +412,7 @@ private Map<String, String> getConfigurationViaXContent(LegacyDateFieldMapper da
         XContentBuilder builder = JsonXContent.contentBuilder().startObject();
         dateFieldMapper.toXContent(builder, ToXContent.EMPTY_PARAMS).endObject();
         Map<String, Object> dateFieldMapperMap;
-        try (XContentParser parser = JsonXContent.jsonXContent.createParser(builder.bytes())) {
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, builder.bytes())) {
             dateFieldMapperMap = parser.map();
         }
         assertThat(dateFieldMapperMap, hasKey("field"));
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/LegacyDateFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/LegacyDateFieldTypeTests.java
index 03137ceffd136..48e08d8715127 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/LegacyDateFieldTypeTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/LegacyDateFieldTypeTests.java
@@ -29,11 +29,10 @@
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.common.joda.DateMathParser;
 import org.elasticsearch.common.joda.Joda;
-import org.elasticsearch.index.mapper.LegacyDateFieldMapper;
-import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.LegacyDateFieldMapper.DateFieldType;
 import org.elasticsearch.index.mapper.MappedFieldType.Relation;
 import org.elasticsearch.index.mapper.ParseContext.Document;
+import org.elasticsearch.index.query.QueryRewriteContext;
 import org.joda.time.DateTimeZone;
 import org.junit.Before;
 
@@ -47,6 +46,8 @@ protected MappedFieldType createDefaultFieldType() {
         return new LegacyDateFieldMapper.DateFieldType();
     }
 
+    private static long nowInMillis;
+
     @Before
     public void setupProperties() {
         setDummyNullValue(10);
@@ -68,6 +69,7 @@ public void modify(MappedFieldType ft) {
                 ((LegacyDateFieldMapper.DateFieldType)ft).setTimeUnit(TimeUnit.HOURS);
             }
         });
+        nowInMillis = randomNonNegativeLong();
     }
 
     public void testIsFieldWithinQueryEmptyReader() throws IOException {
@@ -75,31 +77,32 @@ public void testIsFieldWithinQueryEmptyReader() throws IOException {
         DateFieldType ft = new DateFieldType();
         ft.setName("my_date");
         assertEquals(Relation.DISJOINT, ft.isFieldWithinQuery(reader, "2015-10-12", "2016-04-03",
-                randomBoolean(), randomBoolean(), null, null));
+                randomBoolean(), randomBoolean(), null, null, null));
     }
 
     private void doTestIsFieldWithinQuery(DateFieldType ft, DirectoryReader reader,
             DateTimeZone zone, DateMathParser alternateFormat) throws IOException {
+        QueryRewriteContext context = new QueryRewriteContext(null, null, null, xContentRegistry(), null, null, () -> nowInMillis);
         assertEquals(Relation.INTERSECTS, ft.isFieldWithinQuery(reader, "2015-10-09", "2016-01-02",
-                randomBoolean(), randomBoolean(), null, null));
+                randomBoolean(), randomBoolean(), null, null, context));
         assertEquals(Relation.INTERSECTS, ft.isFieldWithinQuery(reader, "2016-01-02", "2016-06-20",
-                randomBoolean(), randomBoolean(), null, null));
+                randomBoolean(), randomBoolean(), null, null, context));
         assertEquals(Relation.INTERSECTS, ft.isFieldWithinQuery(reader, "2016-01-02", "2016-02-12",
-                randomBoolean(), randomBoolean(), null, null));
+                randomBoolean(), randomBoolean(), null, null, context));
         assertEquals(Relation.DISJOINT, ft.isFieldWithinQuery(reader, "2014-01-02", "2015-02-12",
-                randomBoolean(), randomBoolean(), null, null));
+                randomBoolean(), randomBoolean(), null, null, context));
         assertEquals(Relation.DISJOINT, ft.isFieldWithinQuery(reader, "2016-05-11", "2016-08-30",
-                randomBoolean(), randomBoolean(), null, null));
+                randomBoolean(), randomBoolean(), null, null, context));
         assertEquals(Relation.WITHIN, ft.isFieldWithinQuery(reader, "2015-09-25", "2016-05-29",
-                randomBoolean(), randomBoolean(), null, null));
+                randomBoolean(), randomBoolean(), null, null, context));
         assertEquals(Relation.WITHIN, ft.isFieldWithinQuery(reader, "2015-10-12", "2016-04-03",
-                true, true, null, null));
+                true, true, null, null, context));
         assertEquals(Relation.INTERSECTS, ft.isFieldWithinQuery(reader, "2015-10-12", "2016-04-03",
-                false, false, null, null));
+                false, false, null, null, context));
         assertEquals(Relation.INTERSECTS, ft.isFieldWithinQuery(reader, "2015-10-12", "2016-04-03",
-                false, true, null, null));
+                false, true, null, null, context));
         assertEquals(Relation.INTERSECTS, ft.isFieldWithinQuery(reader, "2015-10-12", "2016-04-03",
-                true, false, null, null));
+                true, false, null, null, context));
     }
 
     public void testIsFieldWithinQuery() throws IOException {
@@ -135,7 +138,7 @@ public void testValueFormat() {
                 createDefaultFieldType().docValueFormat("YYYY", DateTimeZone.UTC).format(instant));
         assertEquals(instant,
                 ft.docValueFormat(null, DateTimeZone.UTC).parseLong("2015-10-12T14:10:55", false, null));
-        assertEquals(instant,
+        assertEquals(instant + 999,
                 ft.docValueFormat(null, DateTimeZone.UTC).parseLong("2015-10-12T14:10:55", true, null));
         assertEquals(LegacyDateFieldMapper.Defaults.DATE_TIME_FORMATTER.parser().parseDateTime("2015-10-13").getMillis() - 1,
                 ft.docValueFormat(null, DateTimeZone.UTC).parseLong("2015-10-12||/d", true, null));
@@ -145,6 +148,6 @@ public void testValueForSearch() {
         MappedFieldType ft = createDefaultFieldType();
         String date = "2015-10-12T12:09:55.000Z";
         long instant = LegacyDateFieldMapper.Defaults.DATE_TIME_FORMATTER.parser().parseDateTime(date).getMillis();
-        assertEquals(date, ft.valueForSearch(instant));
+        assertEquals(date, ft.valueForDisplay(instant));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/LegacyDoubleFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/LegacyDoubleFieldTypeTests.java
index 87feb39648399..93ea0eb35fc06 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/LegacyDoubleFieldTypeTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/LegacyDoubleFieldTypeTests.java
@@ -18,8 +18,6 @@
  */
 package org.elasticsearch.index.mapper;
 
-import org.elasticsearch.index.mapper.LegacyDoubleFieldMapper;
-import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.LegacyDoubleFieldMapper.DoubleFieldType;
 import org.elasticsearch.index.mapper.MappedFieldType.Relation;
 import org.junit.Before;
@@ -41,11 +39,11 @@ public void testIsFieldWithinQuery() throws IOException {
         DoubleFieldType ft = new DoubleFieldType();
         // current impl ignores args and shourd always return INTERSECTS
         assertEquals(Relation.INTERSECTS, ft.isFieldWithinQuery(null, randomDouble(), randomDouble(),
-                randomBoolean(), randomBoolean(), null, null));
+                randomBoolean(), randomBoolean(), null, null, null));
     }
 
     public void testValueForSearch() {
         MappedFieldType ft = createDefaultFieldType();
-        assertEquals(Double.valueOf(1.2), ft.valueForSearch(1.2));
+        assertEquals(Double.valueOf(1.2), ft.valueForDisplay(1.2));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/LegacyFloatFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/LegacyFloatFieldTypeTests.java
index 17da3c88ea475..a476c81fb4736 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/LegacyFloatFieldTypeTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/LegacyFloatFieldTypeTests.java
@@ -18,8 +18,6 @@
  */
 package org.elasticsearch.index.mapper;
 
-import org.elasticsearch.index.mapper.LegacyFloatFieldMapper;
-import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.LegacyFloatFieldMapper.FloatFieldType;
 import org.elasticsearch.index.mapper.MappedFieldType.Relation;
 import org.junit.Before;
@@ -41,11 +39,11 @@ public void testIsFieldWithinQuery() throws IOException {
         FloatFieldType ft = new FloatFieldType();
         // current impl ignores args and shourd always return INTERSECTS
         assertEquals(Relation.INTERSECTS, ft.isFieldWithinQuery(null, randomFloat(), randomFloat(),
-                randomBoolean(), randomBoolean(), null, null));
+                randomBoolean(), randomBoolean(), null, null, null));
     }
 
     public void testValueForSearch() {
         MappedFieldType ft = createDefaultFieldType();
-        assertEquals(Float.valueOf(1.2f), ft.valueForSearch(1.2f));
+        assertEquals(Float.valueOf(1.2f), ft.valueForDisplay(1.2f));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/LegacyGeohashMappingGeoPointTests.java b/core/src/test/java/org/elasticsearch/index/mapper/LegacyGeohashMappingGeoPointTests.java
new file mode 100644
index 0000000000000..9362bb24df453
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/mapper/LegacyGeohashMappingGeoPointTests.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.mapper;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.mapper.BaseGeoPointFieldMapper.LegacyGeoPointFieldType;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.elasticsearch.test.InternalSettingsPlugin;
+import org.elasticsearch.test.VersionUtils;
+
+import java.util.Collection;
+
+import static org.elasticsearch.common.geo.GeoHashUtils.stringEncode;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+
+/**
+ *
+ */
+public class LegacyGeohashMappingGeoPointTests extends ESSingleNodeTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return pluginList(InternalSettingsPlugin.class);
+    }
+
+    public void testGeoHashValue() throws Exception {
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties").startObject("point").field("type", "geo_point").field("geohash", true)
+                .endObject().endObject().endObject().endObject().string();
+
+        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.V_2_4_0);
+        Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
+        DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(mapping));
+
+        ParsedDocument doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
+                .startObject()
+                .field("point", stringEncode(1.3, 1.2))
+                .endObject()
+                .bytes());
+
+        assertThat(doc.rootDoc().getField("point.lat"), nullValue());
+        assertThat(doc.rootDoc().getField("point.lon"), nullValue());
+        assertThat(doc.rootDoc().getField("point.geohash").stringValue(), equalTo(stringEncode(1.3, 1.2)));
+        assertThat(doc.rootDoc().get("point"), notNullValue());
+        assertWarnings("geo_point geohash parameter is deprecated and will be removed in the next major release");
+    }
+
+    public void testGeoHashPrecisionAsInteger() throws Exception {
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties").startObject("point").field("type", "geo_point").field("geohash", true)
+                .field("geohash_precision", 10).endObject().endObject().endObject().endObject().string();
+
+        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.V_2_4_0);
+        Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
+        DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(mapping));
+        FieldMapper mapper = defaultMapper.mappers().smartNameFieldMapper("point");
+        assertThat(mapper, instanceOf(BaseGeoPointFieldMapper.class));
+        BaseGeoPointFieldMapper geoPointFieldMapper = (BaseGeoPointFieldMapper) mapper;
+        assertThat(((LegacyGeoPointFieldType)geoPointFieldMapper.fieldType()).geoHashPrecision(), is(10));
+        assertWarnings("geo_point geohash parameter is deprecated and will be removed in the next major release",
+                "geo_point geohash_precision parameter is deprecated and will be removed in the next major release");
+    }
+
+    public void testGeoHashPrecisionAsLength() throws Exception {
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties").startObject("point").field("type", "geo_point").field("geohash", true)
+            .field("geohash_precision", "5m").endObject().endObject()
+                .endObject().endObject().string();
+
+        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.V_2_4_0);
+        Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
+        DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(mapping));
+        FieldMapper mapper = defaultMapper.mappers().smartNameFieldMapper("point");
+        assertThat(mapper, instanceOf(BaseGeoPointFieldMapper.class));
+        BaseGeoPointFieldMapper geoPointFieldMapper = (BaseGeoPointFieldMapper) mapper;
+        assertThat(((LegacyGeoPointFieldType)geoPointFieldMapper.fieldType()).geoHashPrecision(), is(10));
+        assertWarnings("geo_point geohash parameter is deprecated and will be removed in the next major release",
+                "geo_point geohash_precision parameter is deprecated and will be removed in the next major release");
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/LegacyIntegerFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/LegacyIntegerFieldTypeTests.java
index e0acbcdd127f0..7fd6cfcfba8bf 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/LegacyIntegerFieldTypeTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/LegacyIntegerFieldTypeTests.java
@@ -18,8 +18,6 @@
  */
 package org.elasticsearch.index.mapper;
 
-import org.elasticsearch.index.mapper.LegacyIntegerFieldMapper;
-import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.LegacyIntegerFieldMapper.IntegerFieldType;
 import org.elasticsearch.index.mapper.MappedFieldType.Relation;
 import org.junit.Before;
@@ -41,11 +39,11 @@ public void testIsFieldWithinQuery() throws IOException {
         IntegerFieldType ft = new IntegerFieldType();
         // current impl ignores args and shourd always return INTERSECTS
         assertEquals(Relation.INTERSECTS, ft.isFieldWithinQuery(null, randomInt(), randomInt(),
-                randomBoolean(), randomBoolean(), null, null));
+                randomBoolean(), randomBoolean(), null, null, null));
     }
 
     public void testValueForSearch() {
         MappedFieldType ft = createDefaultFieldType();
-        assertEquals(Integer.valueOf(3), ft.valueForSearch(Integer.valueOf(3)));
+        assertEquals(Integer.valueOf(3), ft.valueForDisplay(Integer.valueOf(3)));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/LegacyIpFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/LegacyIpFieldMapperTests.java
index af525318640c3..faa09043d7d64 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/LegacyIpFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/LegacyIpFieldMapperTests.java
@@ -87,7 +87,7 @@ public void testThatIpv6AddressThrowsException() throws Exception {
             LegacyIpFieldMapper.ipToLong("2001:db8:0:8d3:0:8a2e:70:7344");
             fail("Expected ip address parsing to fail but did not happen");
         } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), containsString("not a valid ipv4 address"));
+            assertThat(e.getMessage(), containsString("Reindex into a new index to get IPv6 support"));
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/LegacyLongFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/LegacyLongFieldTypeTests.java
index 78d30462d22d3..2177bcff67556 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/LegacyLongFieldTypeTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/LegacyLongFieldTypeTests.java
@@ -18,8 +18,6 @@
  */
 package org.elasticsearch.index.mapper;
 
-import org.elasticsearch.index.mapper.LegacyLongFieldMapper;
-import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.LegacyLongFieldMapper.LongFieldType;
 import org.elasticsearch.index.mapper.MappedFieldType.Relation;
 import org.junit.Before;
@@ -41,11 +39,11 @@ public void testIsFieldWithinQuery() throws IOException {
         LongFieldType ft = new LongFieldType();
         // current impl ignores args and shourd always return INTERSECTS
         assertEquals(Relation.INTERSECTS, ft.isFieldWithinQuery(null, randomLong(), randomLong(),
-                randomBoolean(), randomBoolean(), null, null));
+                randomBoolean(), randomBoolean(), null, null, null));
     }
 
     public void testValueForSearch() {
         MappedFieldType ft = createDefaultFieldType();
-        assertEquals(Long.valueOf(3), ft.valueForSearch(Long.valueOf(3)));
+        assertEquals(Long.valueOf(3), ft.valueForDisplay(Long.valueOf(3)));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/LegacyNumberFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/LegacyNumberFieldMapperTests.java
index 46bd89f83e05b..45ca9e3448ec6 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/LegacyNumberFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/LegacyNumberFieldMapperTests.java
@@ -30,17 +30,7 @@
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.index.IndexService;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.DocumentMapperParser;
-import org.elasticsearch.index.mapper.FieldMapper;
-import org.elasticsearch.index.mapper.LegacyFloatFieldMapper;
-import org.elasticsearch.index.mapper.LegacyLongFieldMapper;
-import org.elasticsearch.index.mapper.LegacyNumberFieldMapper;
-import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.ParseContext.Document;
-import org.elasticsearch.index.mapper.ParsedDocument;
-import org.elasticsearch.index.mapper.TextFieldMapper;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
@@ -49,7 +39,6 @@
 import java.util.Arrays;
 import java.util.Collection;
 
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.notNullValue;
@@ -264,6 +253,7 @@ public void testDocValues() throws Exception {
         assertEquals(DocValuesType.SORTED_NUMERIC, LegacyStringMappingTests.docValuesType(doc, "double1"));
         assertEquals(DocValuesType.NONE, LegacyStringMappingTests.docValuesType(doc, "int2"));
         assertEquals(DocValuesType.NONE, LegacyStringMappingTests.docValuesType(doc, "double2"));
+        assertWarnings("Expected a boolean [true/false] for property [index] but got [no]");
     }
 
     public void testUnIndex() throws IOException {
@@ -318,6 +308,8 @@ public void testBwCompatIndex() throws IOException {
         DocumentMapper defaultMapper = createIndex("test", oldSettings).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
         assertEquals("{\"type\":{\"properties\":{\"double\":{\"type\":\"double\"},\"int\":{\"type\":\"integer\",\"index\":false}}}}",
                 defaultMapper.mapping().toString());
+        assertWarnings("Expected a boolean [true/false] for property [index] but got [no]",
+                "Expected a boolean [true/false] for property [index] but got [not_analyzed]");
     }
 
     public void testDocValuesOnNested() throws Exception {
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/LegacyShortFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/LegacyShortFieldTypeTests.java
index be5a6a4e0c351..2e22bac6e9582 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/LegacyShortFieldTypeTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/LegacyShortFieldTypeTests.java
@@ -18,8 +18,6 @@
  */
 package org.elasticsearch.index.mapper;
 
-import org.elasticsearch.index.mapper.LegacyShortFieldMapper;
-import org.elasticsearch.index.mapper.MappedFieldType;
 import org.junit.Before;
 
 public class LegacyShortFieldTypeTests extends FieldTypeTestCase {
@@ -36,6 +34,6 @@ public void setupProperties() {
     public void testValueForSearch() {
         MappedFieldType ft = createDefaultFieldType();
         // shorts are stored as ints
-        assertEquals(Short.valueOf((short) 3), ft.valueForSearch(Integer.valueOf(3)));
+        assertEquals(Short.valueOf((short) 3), ft.valueForDisplay(Integer.valueOf(3)));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/LegacyStringMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/LegacyStringMappingTests.java
index 3c07ec4b90c8d..c07ea90f73aa8 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/LegacyStringMappingTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/LegacyStringMappingTests.java
@@ -73,7 +73,7 @@ protected Collection<Class<? extends Plugin>> getPlugins() {
     DocumentMapperParser parser;
 
     @Before
-    public void before() {
+    public void setup() {
         indexService = createIndex("test",
                 // we need 2.x since string is deprecated in 5.0
                 Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0).build());
@@ -188,6 +188,8 @@ public void testDefaultsForNotAnalyzed() throws Exception {
 
         defaultMapper = parser.parse("type", new CompressedXContent(mapping));
 
+        assertWarnings("The [norms{enabled:true/false}] way of specifying norms is deprecated, please use [norms:true/false] instead");
+
         doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
                 .startObject()
                 .field("field", "1234")
@@ -211,6 +213,8 @@ public void testDefaultsForNotAnalyzed() throws Exception {
 
         defaultMapper = parser.parse("type", new CompressedXContent(mapping));
 
+        assertWarnings("[omit_norms] is deprecated, please use [norms] instead with the opposite boolean value");
+
         doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
                 .startObject()
                 .field("field", "1234")
@@ -354,7 +358,7 @@ private Map<String, Object> getSerializedMap(String fieldName, DocumentMapper ma
         builder.close();
 
         Map<String, Object> fieldMap;
-        try (XContentParser parser = JsonXContent.jsonXContent.createParser(builder.bytes())) {
+        try (XContentParser parser = createParser(builder)) {
             fieldMap = parser.map();
         }
         @SuppressWarnings("unchecked")
@@ -593,6 +597,8 @@ public void testDisableNorms() throws Exception {
                 .endObject().endObject().endObject().endObject().string();
         defaultMapper = mapperService.merge("type", new CompressedXContent(updatedMapping), MapperService.MergeReason.MAPPING_UPDATE, false);
 
+        assertWarnings("The [norms{enabled:true/false}] way of specifying norms is deprecated, please use [norms:true/false] instead");
+
         doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
                 .startObject()
                 .field("field", "1234")
@@ -602,15 +608,14 @@ public void testDisableNorms() throws Exception {
         fieldType = doc.rootDoc().getField("field").fieldType();
         assertEquals(true, fieldType.omitNorms());
 
-        updatedMapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+        String updatedMapping2 = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("properties").startObject("field").field("type", "string").startObject("norms").field("enabled", true).endObject()
                 .endObject().endObject().endObject().endObject().string();
-        try {
-            mapperService.merge("type", new CompressedXContent(updatedMapping), MapperService.MergeReason.MAPPING_UPDATE, false);
-            fail();
-        } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), containsString("different [norms]"));
-        }
+
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
+                () -> mapperService.merge("type", new CompressedXContent(updatedMapping2), MapperService.MergeReason.MAPPING_UPDATE, false));
+        assertThat(exception.getMessage(), containsString("different [norms]"));
+        assertWarnings("The [norms{enabled:true/false}] way of specifying norms is deprecated, please use [norms:true/false] instead");
     }
 
     /**
@@ -660,6 +665,34 @@ public void testFielddataLoading() throws IOException {
         assertTrue(mapper.mappers().getMapper("field").fieldType().eagerGlobalOrdinals());
     }
 
+    public void testFielddataOnAnalyzedString() throws Exception {
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties").startObject("field")
+                    .field("type", "string")
+                    .field("fielddata", false)
+                .endObject().endObject()
+                .endObject().endObject().string();
+
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+        assertEquals(mapping, mapper.mappingSource().toString());
+        assertFalse(((StringFieldType) mapper.mappers().getMapper("field").fieldType()).fielddata());
+    }
+
+    public void testFielddataOnNotAnalyzedString() throws Exception {
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties").startObject("field")
+                    .field("type", "string")
+                    .field("index", "not_analyzed")
+                    .field("doc_values", false)
+                    .field("fielddata", false)
+                .endObject().endObject()
+                .endObject().endObject().string();
+
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+        assertEquals(mapping, mapper.mappingSource().toString());
+        assertFalse(((StringFieldType) mapper.mappers().getMapper("field").fieldType()).fielddata());
+    }
+
     public void testFielddataFilter() throws IOException {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("properties").startObject("field")
@@ -731,9 +764,9 @@ public void testNonAnalyzedFieldPositionIncrement() throws IOException {
                 .field("position_increment_gap", 10)
                 .endObject().endObject().endObject().endObject().string();
 
-            IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-                () -> parser.parse("type", new CompressedXContent(mapping)));
-            assertEquals("Cannot set position_increment_gap on field [field] without positions enabled", e.getMessage());
+            // allowed in index created before 5.0
+            parser.parse("type", new CompressedXContent(mapping));
+            assertWarnings("setting position_increment_gap on field [field] without positions enabled is deprecated and will be ignored");
         }
     }
 
@@ -746,9 +779,9 @@ public void testAnalyzedFieldPositionIncrementWithoutPositions() throws IOExcept
                 .field("position_increment_gap", 10)
                 .endObject().endObject().endObject().endObject().string();
 
-            IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-                () -> parser.parse("type", new CompressedXContent(mapping)));
-            assertEquals("Cannot set position_increment_gap on field [field] without positions enabled", e.getMessage());
+            // allowed in index created before 5.0
+            parser.parse("type", new CompressedXContent(mapping));
+            assertWarnings("setting position_increment_gap on field [field] without positions enabled is deprecated and will be ignored");
         }
     }
 
@@ -763,7 +796,6 @@ public void testMergeKeywordIntoString() throws IOException {
         Map<String, Object> expectedMapping = new HashMap<>();
         expectedMapping.put("type", "string");
         expectedMapping.put("index", "not_analyzed");
-        expectedMapping.put("fielddata", false);
         mergeMappingStep(expectedMapping, b -> b.field("type", "string").field("index", "not_analyzed"));
         mergeMappingStep(expectedMapping, b -> b.field("type", "keyword"));
     }
@@ -780,7 +812,6 @@ public void testMergeKeywordIntoStringWithStore() throws IOException {
         Map<String, Object> expectedMapping = new HashMap<>();
         expectedMapping.put("type", "string");
         expectedMapping.put("index", "not_analyzed");
-        expectedMapping.put("fielddata", false);
         expectedMapping.put("store", true);
         mergeMappingStep(expectedMapping, b -> b.field("type", "string").field("index", "not_analyzed").field("store", true));
         mergeMappingStep(expectedMapping, b -> b.field("type", "keyword").field("store", true));
@@ -799,7 +830,6 @@ public void testMergeKeywordIntoStringWithNorms() throws IOException {
         Map<String, Object> expectedMapping = new HashMap<>();
         expectedMapping.put("type", "string");
         expectedMapping.put("index", "not_analyzed");
-        expectedMapping.put("fielddata", false);
         expectedMapping.put("norms", true);
         mergeMappingStep(expectedMapping, b -> b.field("type", "string").field("index", "not_analyzed").field("norms", true));
         mergeMappingStep(expectedMapping, b -> b.field("type", "keyword").field("norms", true));
@@ -807,12 +837,12 @@ public void testMergeKeywordIntoStringWithNorms() throws IOException {
         mergeMappingStep(expectedMapping, b -> b.field("type", "string").field("index", "not_analyzed")
                 .startObject("norms")
                     .field("enabled", true)
-                    .field("loading", randomAsciiOfLength(5)) // Totally ignored even though it used to be eager/lazy
+                    .field("loading", randomAlphaOfLength(5)) // Totally ignored even though it used to be eager/lazy
                 .endObject());
         mergeMappingStep(expectedMapping, b -> b.field("type", "keyword")
                 .startObject("norms")
                     .field("enabled", true)
-                    .field("loading", randomAsciiOfLength(5))
+                    .field("loading", randomAlphaOfLength(5))
                 .endObject());
     }
 
@@ -820,7 +850,6 @@ public void testMergeKeywordIntoStringWithBoost() throws IOException {
         Map<String, Object> expectedMapping = new HashMap<>();
         expectedMapping.put("type", "string");
         expectedMapping.put("index", "not_analyzed");
-        expectedMapping.put("fielddata", false);
         expectedMapping.put("boost", 1.5);
         expectedMapping.put("norms", true); // Implied by having a boost
         mergeMappingStep(expectedMapping, b -> b.field("type", "string").field("index", "not_analyzed").field("boost", 1.5));
@@ -833,7 +862,6 @@ public void testMergeKeywordIntoStringWithFields() throws IOException {
         Map<String, Object> expectedMapping = new HashMap<>();
         expectedMapping.put("type", "string");
         expectedMapping.put("index", "not_analyzed");
-        expectedMapping.put("fielddata", false);
         Map<String, Object> expectedFields = new HashMap<>();
         expectedMapping.put("fields", expectedFields);
         Map<String, Object> expectedFoo = new HashMap<>();
@@ -887,7 +915,6 @@ public void testMergeKeywordIntoStringWithCopyTo() throws IOException {
         Map<String, Object> expectedMapping = new HashMap<>();
         expectedMapping.put("type", "string");
         expectedMapping.put("index", "not_analyzed");
-        expectedMapping.put("fielddata", false);
         expectedMapping.put("copy_to", singletonList("another_field"));
         mergeMappingStep(expectedMapping, b -> b.field("type", "string").field("index", "not_analyzed").field("copy_to", "another_field"));
         mergeMappingStep(expectedMapping, b -> b.field("type", "keyword").field("copy_to", "another_field"));
@@ -897,7 +924,6 @@ public void testMergeKeywordIntoStringWithIncludeInAll() throws IOException {
         Map<String, Object> expectedMapping = new HashMap<>();
         expectedMapping.put("type", "string");
         expectedMapping.put("index", "not_analyzed");
-        expectedMapping.put("fielddata", false);
         expectedMapping.put("include_in_all", false);
         mergeMappingStep(expectedMapping, b -> b.field("type", "string").field("index", "not_analyzed").field("include_in_all", false));
         mergeMappingStep(expectedMapping, b -> b.field("type", "keyword").field("include_in_all", false));
@@ -907,7 +933,6 @@ public void testMergeKeywordIntoStringWithIgnoreAbove() throws IOException {
         Map<String, Object> expectedMapping = new HashMap<>();
         expectedMapping.put("type", "string");
         expectedMapping.put("index", "not_analyzed");
-        expectedMapping.put("fielddata", false);
         expectedMapping.put("ignore_above", 128);
         mergeMappingStep(expectedMapping, b -> b.field("type", "string").field("index", "not_analyzed").field("ignore_above", 128));
         mergeMappingStep(expectedMapping, b -> b.field("type", "keyword").field("ignore_above", 128));
@@ -917,7 +942,6 @@ public void testMergeKeywordIntoStringWithIndexOptions() throws IOException {
         Map<String, Object> expectedMapping = new HashMap<>();
         expectedMapping.put("type", "string");
         expectedMapping.put("index", "not_analyzed");
-        expectedMapping.put("fielddata", false);
         expectedMapping.put("index_options", "freqs");
         mergeMappingStep(expectedMapping, b -> b.field("type", "string").field("index", "not_analyzed").field("index_options", "freqs"));
         mergeMappingStep(expectedMapping, b -> b.field("type", "keyword").field("index_options", "freqs"));
@@ -927,7 +951,6 @@ public void testMergeKeywordIntoStringWithSimilarity() throws IOException {
         Map<String, Object> expectedMapping = new HashMap<>();
         expectedMapping.put("type", "string");
         expectedMapping.put("index", "not_analyzed");
-        expectedMapping.put("fielddata", false);
         expectedMapping.put("similarity", "BM25");
         mergeMappingStep(expectedMapping, b -> b.field("type", "string").field("index", "not_analyzed").field("similarity", "BM25"));
         mergeMappingStep(expectedMapping, b -> b.field("type", "keyword").field("similarity", "BM25"));
@@ -1195,4 +1218,18 @@ private String randomIndexOptions() {
             throw new IllegalArgumentException("Unknown options [" + options + "]");
         }
     }
+
+    public void testOldMappingsAreMinimal() throws Exception {
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties").startObject("text").field("type", "string")
+                .endObject().endObject().endObject().endObject().string();
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+        assertEquals(mapping, mapper.mapping().toString());
+
+        mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties").startObject("text").field("type", "string").field("index", "not_analyzed")
+                .endObject().endObject().endObject().endObject().string();
+        mapper = parser.parse("type", new CompressedXContent(mapping));
+        assertEquals(mapping, mapper.mapping().toString());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java b/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java
index 4eaf58a602f2c..cd18e2966916d 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java
@@ -19,38 +19,50 @@
 
 package org.elasticsearch.index.mapper;
 
-import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.concurrent.ExecutionException;
-import java.util.function.Function;
-
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.mapper.KeywordFieldMapper.KeywordFieldType;
 import org.elasticsearch.index.mapper.MapperService.MergeReason;
 import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.elasticsearch.test.InternalSettingsPlugin;
+import org.elasticsearch.test.VersionUtils;
+import org.hamcrest.Matchers;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.function.Function;
 
 import static org.hamcrest.CoreMatchers.containsString;
-import static org.hamcrest.Matchers.hasToString;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.startsWith;
 
 public class MapperServiceTests extends ESSingleNodeTestCase {
 
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return Collections.singletonList(InternalSettingsPlugin.class);
+    }
+
     public void testTypeNameStartsWithIllegalDot() {
         String index = "test-index";
         String type = ".test-type";
         String field = "field";
-        MapperParsingException e = expectThrows(MapperParsingException.class, () -> {
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {
             client().admin().indices().prepareCreate(index)
                     .addMapping(type, field, "type=text")
                     .execute().actionGet();
@@ -63,7 +75,7 @@ public void testTypeNameTooLong() {
         String field = "field";
         String type = new String(new char[256]).replace("\0", "a");
 
-        MapperParsingException e = expectThrows(MapperParsingException.class, () -> {
+        MapperException e = expectThrows(MapperException.class, () -> {
             client().admin().indices().prepareCreate(index)
                     .addMapping(type, field, "type=text")
                     .execute().actionGet();
@@ -92,7 +104,7 @@ public void testTypes() throws Exception {
     public void testIndexIntoDefaultMapping() throws Throwable {
         // 1. test implicit index creation
         ExecutionException e = expectThrows(ExecutionException.class, () -> {
-            client().prepareIndex("index1", MapperService.DEFAULT_MAPPING, "1").setSource("{}").execute().get();
+            client().prepareIndex("index1", MapperService.DEFAULT_MAPPING, "1").setSource("{}", XContentType.JSON).execute().get();
         });
         Throwable throwable = ExceptionsHelper.unwrapCause(e.getCause());
         if (throwable instanceof IllegalArgumentException) {
@@ -132,6 +144,7 @@ public void testTotalFieldsExceedsLimit() throws Throwable {
                 .mapperService().merge("type", new CompressedXContent(mapping.apply("type")), MergeReason.MAPPING_UPDATE, false);
         });
         assertTrue(e.getMessage(), e.getMessage().contains("Limit of total fields [1] in index [test2] has been exceeded"));
+        assertWarnings("The [string] field is deprecated, please use [text] or [keyword] instead on [field1]");
     }
 
     public void testMappingDepthExceedsLimit() throws Throwable {
@@ -162,29 +175,166 @@ public void testMappingDepthExceedsLimit() throws Throwable {
     }
 
     public void testUnmappedFieldType() {
-        MapperService mapperService = createIndex("index").mapperService();
-        assertThat(mapperService.unmappedFieldType("keyword"), instanceOf(KeywordFieldType.class));
-        assertThat(mapperService.unmappedFieldType("long"), instanceOf(NumberFieldType.class));
-        // back compat
-        assertThat(mapperService.unmappedFieldType("string"), instanceOf(KeywordFieldType.class));
+        assertUnmappedFieldType(Version.CURRENT);
     }
 
+    public void testUnmappedFieldTypeBWC() {
+        // test BWC with indices created in 2.x
+        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.V_2_4_6);
+        assertUnmappedFieldType(version);
+    }
+
+    private void assertUnmappedFieldType(Version version) {
+        MapperService mapperService =
+            createIndex("index", Settings.builder().put("index.version.created", version).build()).mapperService();
+        if (version.after(Version.V_2_4_6)) {
+            assertThat(mapperService.unmappedFieldType("keyword"), instanceOf(KeywordFieldType.class));
+        } else {
+            assertThat(mapperService.unmappedFieldType("keyword"), instanceOf(StringFieldType.class));
+        }
+        if (version.after(Version.V_2_4_6)) {
+            assertThat(mapperService.unmappedFieldType("long"), instanceOf(NumberFieldType.class));
+        } else {
+            assertThat(mapperService.unmappedFieldType("long"), instanceOf(LegacyLongFieldMapper.LongFieldType.class));
+        }
+        if (version.after(Version.V_2_4_6)) {
+            assertThat(mapperService.unmappedFieldType("string"), instanceOf(KeywordFieldType.class));
+        } else {
+            assertThat(mapperService.unmappedFieldType("string"), instanceOf(StringFieldType.class));
+        }
+        assertWarnings("[unmapped_type:string] should be replaced with [unmapped_type:keyword]");
+    }
 
     public void testMergeWithMap() throws Throwable {
         IndexService indexService1 = createIndex("index1");
         MapperService mapperService = indexService1.mapperService();
         Map<String, Map<String, Object>> mappings = new HashMap<>();
 
-        mappings.put(MapperService.DEFAULT_MAPPING, MapperService.parseMapping("{}"));
+        mappings.put(MapperService.DEFAULT_MAPPING, MapperService.parseMapping(xContentRegistry(), "{}"));
         MapperException e = expectThrows(MapperParsingException.class,
-            () -> mapperService.merge(mappings, false));
+            () -> mapperService.merge(mappings, MergeReason.MAPPING_UPDATE, false));
         assertThat(e.getMessage(), startsWith("Failed to parse mapping [" + MapperService.DEFAULT_MAPPING + "]: "));
 
         mappings.clear();
-        mappings.put("type1", MapperService.parseMapping("{}"));
+        mappings.put("type1", MapperService.parseMapping(xContentRegistry(), "{}"));
 
         e = expectThrows( MapperParsingException.class,
-            () -> mapperService.merge(mappings, false));
+            () -> mapperService.merge(mappings, MergeReason.MAPPING_UPDATE, false));
         assertThat(e.getMessage(), startsWith("Failed to parse mapping [type1]: "));
     }
+
+    public void testMergeParentTypesSame() {
+        // Verifies that a merge (absent a DocumentMapper change)
+        // doesn't change the parentTypes reference.
+        // The collection was being rewrapped with each merge
+        // in v5.2 resulting in eventual StackOverflowErrors.
+        // https://github.com/elastic/elasticsearch/issues/23604
+
+        IndexService indexService1 = createIndex("index1");
+        MapperService mapperService = indexService1.mapperService();
+        Set<String> parentTypes = mapperService.getParentTypes();
+
+        Map<String, Map<String, Object>> mappings = new HashMap<>();
+        mapperService.merge(mappings, MergeReason.MAPPING_UPDATE, false);
+        assertSame(parentTypes, mapperService.getParentTypes());
+    }
+
+    public void testOtherDocumentMappersOnlyUpdatedWhenChangingFieldType() throws IOException {
+        IndexService indexService = createIndex("test");
+
+        CompressedXContent simpleMapping = new CompressedXContent(XContentFactory.jsonBuilder().startObject()
+            .startObject("properties")
+            .startObject("field")
+            .field("type", "text")
+            .endObject()
+            .endObject().endObject().bytes());
+
+        indexService.mapperService().merge("type1", simpleMapping, MergeReason.MAPPING_UPDATE, true);
+        DocumentMapper documentMapper = indexService.mapperService().documentMapper("type1");
+
+        indexService.mapperService().merge("type2", simpleMapping, MergeReason.MAPPING_UPDATE, true);
+        assertSame(indexService.mapperService().documentMapper("type1"), documentMapper);
+
+        CompressedXContent normsDisabledMapping = new CompressedXContent(XContentFactory.jsonBuilder().startObject()
+            .startObject("properties")
+            .startObject("field")
+            .field("type", "text")
+            .field("norms", false)
+            .endObject()
+            .endObject().endObject().bytes());
+
+        indexService.mapperService().merge("type3", normsDisabledMapping, MergeReason.MAPPING_UPDATE, true);
+        assertNotSame(indexService.mapperService().documentMapper("type1"), documentMapper);
+    }
+
+    public void testAllEnabled() throws Exception {
+        IndexService indexService = createIndex("test");
+        assertFalse(indexService.mapperService().allEnabled());
+
+        CompressedXContent enabledAll = new CompressedXContent(XContentFactory.jsonBuilder().startObject()
+                .startObject("_all")
+                    .field("enabled", true)
+                .endObject().endObject().bytes());
+
+        CompressedXContent disabledAll = new CompressedXContent(XContentFactory.jsonBuilder().startObject()
+                .startObject("_all")
+                    .field("enabled", false)
+                .endObject().endObject().bytes());
+
+        indexService.mapperService().merge(MapperService.DEFAULT_MAPPING, enabledAll,
+                MergeReason.MAPPING_UPDATE, random().nextBoolean());
+        assertFalse(indexService.mapperService().allEnabled()); // _default_ does not count
+
+        indexService.mapperService().merge("some_type", enabledAll,
+                MergeReason.MAPPING_UPDATE, random().nextBoolean());
+        assertTrue(indexService.mapperService().allEnabled());
+
+        indexService.mapperService().merge("other_type", disabledAll,
+                MergeReason.MAPPING_UPDATE, random().nextBoolean());
+        assertTrue(indexService.mapperService().allEnabled()); // this returns true if any of the types has _all enabled
+    }
+
+     public void testPartitionedConstraints() {
+        // partitioned index must have routing
+         IllegalArgumentException noRoutingException = expectThrows(IllegalArgumentException.class, () -> {
+            client().admin().indices().prepareCreate("test-index")
+                    .addMapping("type", "{\"type\":{}}")
+                    .setSettings(Settings.builder()
+                        .put("index.number_of_shards", 4)
+                        .put("index.routing_partition_size", 2))
+                    .execute().actionGet();
+        });
+        assertTrue(noRoutingException.getMessage(), noRoutingException.getMessage().contains("must have routing"));
+
+        // partitioned index cannot have parent/child relationships
+        IllegalArgumentException parentException = expectThrows(IllegalArgumentException.class, () -> {
+            client().admin().indices().prepareCreate("test-index")
+                    .addMapping("parent", "{\"parent\":{\"_routing\":{\"required\":true}}}")
+                    .addMapping("child", "{\"child\": {\"_routing\":{\"required\":true}, \"_parent\": {\"type\": \"parent\"}}}")
+                    .setSettings(Settings.builder()
+                        .put("index.number_of_shards", 4)
+                        .put("index.routing_partition_size", 2))
+                    .execute().actionGet();
+        });
+        assertTrue(parentException.getMessage(), parentException.getMessage().contains("cannot have a _parent field"));
+
+        // valid partitioned index
+        assertTrue(client().admin().indices().prepareCreate("test-index")
+            .addMapping("type", "{\"type\":{\"_routing\":{\"required\":true}}}")
+            .setSettings(Settings.builder()
+                .put("index.number_of_shards", 4)
+                .put("index.routing_partition_size", 2))
+            .execute().actionGet().isAcknowledged());
+    }
+
+    public void testForbidMultipleTypes() throws IOException {
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string();
+        MapperService mapperService = createIndex("test", Settings.builder().put("index.mapping.single_type", true).build()).mapperService();
+        mapperService.merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE, randomBoolean());
+
+        String mapping2 = XContentFactory.jsonBuilder().startObject().startObject("type2").endObject().endObject().string();
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> mapperService.merge("type2", new CompressedXContent(mapping2), MergeReason.MAPPING_UPDATE, randomBoolean()));
+        assertThat(e.getMessage(), Matchers.startsWith("Rejecting mapping update to [test] as the final mapping would have more than 1 type: "));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldCopyToMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldCopyToMapperTests.java
index f2b65f0f976d4..3c6a42ebd3753 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldCopyToMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldCopyToMapperTests.java
@@ -27,9 +27,6 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.MapperTestUtils;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.MapperParsingException;
-import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.VersionUtils;
 
@@ -48,7 +45,8 @@ public void testExceptionForCopyToInMultiFields() throws IOException {
 
         // first check that for newer versions we throw exception if copy_to is found withing multi field
         Version indexVersion = randomFrom(versionsWithAndWithoutExpectedExceptions.v1());
-        MapperService mapperService = MapperTestUtils.newMapperService(createTempDir(), Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, indexVersion).build());
+        MapperService mapperService = MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(),
+                Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, indexVersion).build());
         try {
             mapperService.parse("type", new CompressedXContent(mapping.string()), true);
             fail("Parsing should throw an exception because the mapping contains a copy_to in a multi field");
@@ -58,7 +56,8 @@ public void testExceptionForCopyToInMultiFields() throws IOException {
 
         // now test that with an older version the parsing just works
         indexVersion = randomFrom(versionsWithAndWithoutExpectedExceptions.v2());
-        mapperService = MapperTestUtils.newMapperService(createTempDir(), Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, indexVersion).build());
+        mapperService = MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(),
+                Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, indexVersion).build());
         DocumentMapper documentMapper = mapperService.parse("type", new CompressedXContent(mapping.string()), true);
         assertFalse(documentMapper.mapping().toString().contains("copy_to"));
     }
@@ -92,7 +91,7 @@ private static XContentBuilder createMappinmgWithCopyToInMultiField() throws IOE
     private static Tuple<List<Version>, List<Version>> versionsWithAndWithoutExpectedExceptions() {
         List<Version> versionsWithException = new ArrayList<>();
         List<Version> versionsWithoutException = new ArrayList<>();
-        for (Version version : VersionUtils.allVersions()) {
+        for (Version version : VersionUtils.allReleasedVersions()) {
             if (version.after(Version.V_2_1_0) ||
                 (version.after(Version.V_2_0_1) && version.before(Version.V_2_1_0))) {
                 versionsWithException.add(version);
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldIncludeInAllMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldIncludeInAllMapperTests.java
new file mode 100644
index 0000000000000..1a86db319d1ae
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldIncludeInAllMapperTests.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.index.mapper;
+
+import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.MapperTestUtils;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+
+
+public class MultiFieldIncludeInAllMapperTests extends ESTestCase {
+    public void testExceptionForIncludeInAllInMultiFields() throws IOException {
+        XContentBuilder mapping = createMappingWithIncludeInAllInMultiField();
+
+        // first check that for newer versions we throw exception if include_in_all is found within multi field
+        MapperService mapperService = MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), Settings.EMPTY);
+        mapperService.parse("type", new CompressedXContent(mapping.string()), true);
+        assertWarnings("include_in_all in multi fields is deprecated because it doesn't do "
+                + "anything. Found the include_in_all in field [c] which is within a multi field.");
+    }
+
+    private static XContentBuilder createMappingWithIncludeInAllInMultiField() throws IOException {
+        XContentBuilder mapping = jsonBuilder();
+        mapping.startObject()
+                .startObject("type")
+                .startObject("properties")
+                .startObject("a")
+                .field("type", "text")
+                .endObject()
+                .startObject("b")
+                .field("type", "text")
+                .startObject("fields")
+                .startObject("c")
+                .field("type", "text")
+                .field("include_in_all", false)
+                .endObject()
+                .endObject()
+                .endObject()
+                .endObject()
+                .endObject()
+                .endObject();
+        return mapping;
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldTests.java b/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldTests.java
index 4bf1995722b6e..6c31aa7d906ba 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldTests.java
@@ -29,15 +29,6 @@
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.index.IndexService;
-import org.elasticsearch.index.mapper.DateFieldMapper;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.DocumentMapperParser;
-import org.elasticsearch.index.mapper.KeywordFieldMapper;
-import org.elasticsearch.index.mapper.MapperParsingException;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.RootObjectMapper;
-import org.elasticsearch.index.mapper.TextFieldMapper;
-import org.elasticsearch.index.mapper.TokenCountFieldMapper;
 import org.elasticsearch.index.mapper.ParseContext.Document;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
@@ -176,7 +167,7 @@ public void testBuildThenParse() throws Exception {
     public void testMultiFieldsInConsistentOrder() throws Exception {
         String[] multiFieldNames = new String[randomIntBetween(2, 10)];
         for (int i = 0; i < multiFieldNames.length; i++) {
-            multiFieldNames[i] = randomAsciiOfLength(4);
+            multiFieldNames[i] = randomAlphaOfLength(4);
         }
 
         XContentBuilder builder = jsonBuilder().startObject().startObject("type").startObject("properties")
@@ -189,7 +180,8 @@ public void testMultiFieldsInConsistentOrder() throws Exception {
         DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
         Arrays.sort(multiFieldNames);
 
-        Map<String, Object> sourceAsMap = XContentHelper.convertToMap(docMapper.mappingSource().compressedReference(), true).v2();
+        Map<String, Object> sourceAsMap =
+            XContentHelper.convertToMap(docMapper.mappingSource().compressedReference(), true, builder.contentType()).v2();
         @SuppressWarnings("unchecked")
         Map<String, Object> multiFields = (Map<String, Object>) XContentMapValues.extractValue("type.properties.my_field.fields", sourceAsMap);
         assertThat(multiFields.size(), equalTo(multiFieldNames.length));
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldsIntegrationIT.java b/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldsIntegrationIT.java
index e11f0b90e85e6..8711ead6edf0d 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldsIntegrationIT.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldsIntegrationIT.java
@@ -128,7 +128,7 @@ public void testGeoPointMultiField() throws Exception {
                 .setQuery(constantScoreQuery(geoDistanceQuery("a").point(51, 19).distance(50, DistanceUnit.KILOMETERS)))
                 .get();
         assertThat(countResponse.getHits().totalHits(), equalTo(1L));
-        countResponse = client().prepareSearch("my-index").setSize(0).setQuery(matchQuery("a.b", point.toString())).get();
+        countResponse = client().prepareSearch("my-index").setSize(0).setQuery(matchQuery("a.b", point.geohash())).get();
         assertThat(countResponse.getHits().totalHits(), equalTo(1L));
     }
 
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java
index 89a42a884c63d..06f638d6b5316 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java
@@ -22,19 +22,15 @@
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.ObjectMapper;
 import org.elasticsearch.index.mapper.MapperService.MergeReason;
 import org.elasticsearch.index.mapper.ObjectMapper.Dynamic;
-import org.elasticsearch.index.mapper.ParsedDocument;
-import org.elasticsearch.index.mapper.TypeFieldMapper;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.util.function.Function;
 
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.nullValue;
@@ -366,22 +362,16 @@ public void testLimitOfNestedFieldsPerIndex() throws Exception {
         createIndex("test1").mapperService().merge("type", new CompressedXContent(mapping.apply("type")), MergeReason.MAPPING_UPDATE, false);
 
         // explicitly setting limit to 0 prevents nested fields
-        try {
+        Exception e = expectThrows(IllegalArgumentException.class, () ->
             createIndex("test2", Settings.builder().put(MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING.getKey(), 0).build())
-                .mapperService().merge("type", new CompressedXContent(mapping.apply("type")), MergeReason.MAPPING_UPDATE, false);
-            fail("Expected IllegalArgumentException");
-        } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), containsString("Limit of nested fields [0] in index [test2] has been exceeded"));
-        }
+                .mapperService().merge("type", new CompressedXContent(mapping.apply("type")), MergeReason.MAPPING_UPDATE, false));
+        assertThat(e.getMessage(), containsString("Limit of nested fields [0] in index [test2] has been exceeded"));
 
         // setting limit to 1 with 2 nested fields fails
-        try {
+        e = expectThrows(IllegalArgumentException.class, () ->
             createIndex("test3", Settings.builder().put(MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING.getKey(), 1).build())
-                .mapperService().merge("type", new CompressedXContent(mapping.apply("type")), MergeReason.MAPPING_UPDATE, false);
-            fail("Expected IllegalArgumentException");
-        } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), containsString("Limit of nested fields [1] in index [test3] has been exceeded"));
-        }
+                .mapperService().merge("type", new CompressedXContent(mapping.apply("type")), MergeReason.MAPPING_UPDATE, false));
+        assertThat(e.getMessage(), containsString("Limit of nested fields [1] in index [test3] has been exceeded"));
 
         MapperService mapperService = createIndex("test4", Settings.builder().put(MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING.getKey(), 2)
             .build()).mapperService();
@@ -391,15 +381,43 @@ public void testLimitOfNestedFieldsPerIndex() throws Exception {
         // adding new fields from different type is not ok
         String mapping2 = XContentFactory.jsonBuilder().startObject().startObject("type3").startObject("properties").startObject("nested3")
             .field("type", "nested").startObject("properties").endObject().endObject().endObject().endObject().endObject().string();
-        try {
-            mapperService.merge("type3", new CompressedXContent(mapping2), MergeReason.MAPPING_UPDATE, false);
-            fail("Expected IllegalArgumentException");
-        } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), containsString("Limit of nested fields [2] in index [test4] has been exceeded"));
-        }
+        e = expectThrows(IllegalArgumentException.class, () ->
+            mapperService.merge("type3", new CompressedXContent(mapping2), MergeReason.MAPPING_UPDATE, false));
+        assertThat(e.getMessage(), containsString("Limit of nested fields [2] in index [test4] has been exceeded"));
 
         // do not check nested fields limit if mapping is not updated
         createIndex("test5", Settings.builder().put(MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING.getKey(), 0).build())
             .mapperService().merge("type", new CompressedXContent(mapping.apply("type")), MergeReason.MAPPING_RECOVERY, false);
     }
+
+    public void testParentObjectMapperAreNested() throws Exception {
+        MapperService mapperService = createIndex("index1", Settings.EMPTY, "doc", jsonBuilder().startObject()
+                .startObject("properties")
+                    .startObject("comments")
+                        .field("type", "nested")
+                        .startObject("properties")
+                            .startObject("messages")
+                                .field("type", "nested").endObject()
+                            .endObject()
+                        .endObject()
+                    .endObject()
+                .endObject()).mapperService();
+        ObjectMapper objectMapper = mapperService.getObjectMapper("comments.messages");
+        assertTrue(objectMapper.parentObjectMapperAreNested(mapperService));
+
+        mapperService = createIndex("index2", Settings.EMPTY, "doc", jsonBuilder().startObject()
+            .startObject("properties")
+                .startObject("comments")
+                    .field("type", "object")
+                        .startObject("properties")
+                            .startObject("messages")
+                                .field("type", "nested").endObject()
+                            .endObject()
+                    .endObject()
+                .endObject()
+            .endObject()).mapperService();
+        objectMapper = mapperService.getObjectMapper("comments.messages");
+        assertFalse(objectMapper.parentObjectMapperAreNested(mapperService));
+    }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java
index 875cec3ebca7a..6913719853cb8 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java
@@ -21,58 +21,25 @@
 
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.IndexableField;
-import org.elasticsearch.Version;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.compress.CompressedXContent;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.index.IndexService;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.DocumentMapperParser;
-import org.elasticsearch.index.mapper.FieldMapper;
-import org.elasticsearch.index.mapper.MapperParsingException;
-import org.elasticsearch.index.mapper.ParsedDocument;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.test.ESSingleNodeTestCase;
-import org.elasticsearch.test.InternalSettingsPlugin;
-import org.elasticsearch.test.VersionUtils;
-import org.junit.Before;
+import org.elasticsearch.common.xcontent.XContentType;
 
 import java.io.IOException;
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.HashSet;
-import java.util.Set;
 
-import static com.carrotsearch.randomizedtesting.RandomizedTest.getRandom;
 import static org.hamcrest.Matchers.containsString;
 
-public class NumberFieldMapperTests extends ESSingleNodeTestCase {
-
-    private static final Set<String> TYPES = new HashSet<>(Arrays.asList("byte", "short", "integer", "long", "float", "double"));
-
-    IndexService indexService;
-    DocumentMapperParser parser;
-
-    @Before
-    public void before() {
-        indexService = createIndex("test");
-        parser = indexService.mapperService().documentMapperParser();
-    }
-
-    public void testDefaults() throws Exception {
-        for (String type : TYPES) {
-            doTestDefaults(type);
-        }
-    }
+public class NumberFieldMapperTests extends AbstractNumericFieldMapperTestCase {
 
     @Override
-    protected Collection<Class<? extends Plugin>> getPlugins() {
-        return pluginList(InternalSettingsPlugin.class);
+    protected void setTypeList() {
+        TYPES = new HashSet<>(Arrays.asList("byte", "short", "integer", "long", "float", "double"));
+        WHOLE_TYPES = new HashSet<>(Arrays.asList("byte", "short", "integer", "long"));
     }
 
+    @Override
     public void doTestDefaults(String type) throws Exception {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("properties").startObject("field").field("type", type).endObject().endObject()
@@ -99,12 +66,7 @@ public void doTestDefaults(String type) throws Exception {
         assertFalse(dvField.fieldType().stored());
     }
 
-    public void testNotIndexed() throws Exception {
-        for (String type : TYPES) {
-            doTestNotIndexed(type);
-        }
-    }
-
+    @Override
     public void doTestNotIndexed(String type) throws Exception {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("properties").startObject("field").field("type", type).field("index", false).endObject().endObject()
@@ -126,12 +88,7 @@ public void doTestNotIndexed(String type) throws Exception {
         assertEquals(DocValuesType.SORTED_NUMERIC, dvField.fieldType().docValuesType());
     }
 
-    public void testNoDocValues() throws Exception {
-        for (String type : TYPES) {
-            doTestNotIndexed(type);
-        }
-    }
-
+    @Override
     public void doTestNoDocValues(String type) throws Exception {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("properties").startObject("field").field("type", type).field("doc_values", false).endObject().endObject()
@@ -154,12 +111,7 @@ public void doTestNoDocValues(String type) throws Exception {
         assertEquals(123, pointField.numericValue().doubleValue(), 0d);
     }
 
-    public void testStore() throws Exception {
-        for (String type : TYPES) {
-            doTestStore(type);
-        }
-    }
-
+    @Override
     public void doTestStore(String type) throws Exception {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("properties").startObject("field").field("type", type).field("store", true).endObject().endObject()
@@ -187,12 +139,7 @@ public void doTestStore(String type) throws Exception {
         assertEquals(123, storedField.numericValue().doubleValue(), 0d);
     }
 
-    public void testCoerce() throws Exception {
-        for (String type : TYPES) {
-            doTestCoerce(type);
-        }
-    }
-
+    @Override
     public void doTestCoerce(String type) throws IOException {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("properties").startObject("field").field("type", type).endObject().endObject()
@@ -233,13 +180,35 @@ public void doTestCoerce(String type) throws IOException {
         assertThat(e.getCause().getMessage(), containsString("passed as String"));
     }
 
+    @Override
+    protected void doTestDecimalCoerce(String type) throws IOException {
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties").startObject("field").field("type", type).endObject().endObject()
+                .endObject().endObject().string();
+
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+
+        assertEquals(mapping, mapper.mappingSource().toString());
+
+        ParsedDocument doc = mapper.parse(SourceToParse.source("test", "type", "1", XContentFactory.jsonBuilder()
+                .startObject()
+                .field("field", "7.89")
+                .endObject()
+                .bytes(),
+                XContentType.JSON));
+
+        IndexableField[] fields = doc.rootDoc().getFields("field");
+        IndexableField pointField = fields[0];
+        assertEquals(7, pointField.numericValue().doubleValue(), 0d);
+    }
+
     public void testIgnoreMalformed() throws Exception {
         for (String type : TYPES) {
             doTestIgnoreMalformed(type);
         }
     }
 
-    public void doTestIgnoreMalformed(String type) throws IOException {
+    private void doTestIgnoreMalformed(String type) throws IOException {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("properties").startObject("field").field("type", type).endObject().endObject()
                 .endObject().endObject().string();
@@ -272,12 +241,7 @@ public void doTestIgnoreMalformed(String type) throws IOException {
         assertEquals(0, fields.length);
     }
 
-    public void testIncludeInAll() throws Exception {
-        for (String type : TYPES) {
-            doTestIncludeInAll(type);
-        }
-    }
-
+    @Override
     public void doTestIncludeInAll(String type) throws Exception {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("properties").startObject("field").field("type", type).endObject().endObject()
@@ -314,6 +278,8 @@ public void doTestIncludeInAll(String type) throws Exception {
 
         fields = doc.rootDoc().getFields("_all");
         assertEquals(0, fields.length);
+        assertWarnings("field [include_in_all] is deprecated, as [_all] is deprecated, and will be disallowed" +
+                        " in 6.0, use [copy_to] instead.");
     }
 
     public void testRejectNorms() throws IOException {
@@ -333,13 +299,8 @@ public void testRejectNorms() throws IOException {
         }
     }
 
-    public void testNullValue() throws IOException {
-        for (String type : TYPES) {
-            doTestNullValue(type);
-        }
-    }
-
-    private void doTestNullValue(String type) throws IOException {
+    @Override
+    protected void doTestNullValue(String type) throws IOException {
         String mapping = XContentFactory.jsonBuilder().startObject()
                 .startObject("type")
                     .startObject("properties")
@@ -393,32 +354,4 @@ private void doTestNullValue(String type) throws IOException {
         assertEquals(DocValuesType.SORTED_NUMERIC, dvField.fieldType().docValuesType());
         assertFalse(dvField.fieldType().stored());
     }
-
-    public void testEmptyName() throws IOException {
-        // after version 5
-        for (String type : TYPES) {
-            String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("properties").startObject("").field("type", type).endObject().endObject()
-                .endObject().endObject().string();
-
-            IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-                () -> parser.parse("type", new CompressedXContent(mapping))
-            );
-            assertThat(e.getMessage(), containsString("name cannot be empty string"));
-        }
-
-        // before 5.x
-        Version oldVersion = VersionUtils.randomVersionBetween(getRandom(), Version.V_2_0_0, Version.V_2_3_5);
-        Settings oldIndexSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, oldVersion).build();
-        indexService = createIndex("test_old", oldIndexSettings);
-        parser = indexService.mapperService().documentMapperParser();
-        for (String type : TYPES) {
-            String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("properties").startObject("").field("type", type).endObject().endObject()
-                .endObject().endObject().string();
-
-            DocumentMapper defaultMapper = parser.parse("type", new CompressedXContent(mapping));
-            assertEquals(mapping, defaultMapper.mappingSource().string());
-        }
-    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java
index c887565ae69c6..08be1900d6fdd 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java
@@ -24,22 +24,30 @@
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.FloatPoint;
 import org.apache.lucene.document.HalfFloatPoint;
+import org.apache.lucene.document.IntPoint;
 import org.apache.lucene.document.LongPoint;
+import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.search.IndexOrDocValuesQuery;
 import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.IOUtils;
-import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.mapper.NumberFieldMapper;
+import org.apache.lucene.util.TestUtil;
 import org.elasticsearch.index.mapper.MappedFieldType.Relation;
 import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType;
+import org.hamcrest.Matchers;
 import org.junit.Before;
 
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.function.Supplier;
 
 public class NumberFieldTypeTests extends FieldTypeTestCase {
 
@@ -59,7 +67,53 @@ public void testIsFieldWithinQuery() throws IOException {
         MappedFieldType ft = createDefaultFieldType();
         // current impl ignores args and should always return INTERSECTS
         assertEquals(Relation.INTERSECTS, ft.isFieldWithinQuery(null, randomDouble(), randomDouble(),
-                randomBoolean(), randomBoolean(), null, null));
+                randomBoolean(), randomBoolean(), null, null, null));
+    }
+
+    public void testIntegerTermsQueryWithDecimalPart() {
+        MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberType.INTEGER);
+        ft.setName("field");
+        ft.setIndexOptions(IndexOptions.DOCS);
+        assertEquals(IntPoint.newSetQuery("field", 1), ft.termsQuery(Arrays.asList(1, 2.1), null));
+        assertEquals(IntPoint.newSetQuery("field", 1), ft.termsQuery(Arrays.asList(1.0, 2.1), null));
+        assertTrue(ft.termsQuery(Arrays.asList(1.1, 2.1), null) instanceof MatchNoDocsQuery);
+    }
+
+    public void testLongTermsQueryWithDecimalPart() {
+        MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberType.LONG);
+        ft.setName("field");
+        ft.setIndexOptions(IndexOptions.DOCS);
+        assertEquals(LongPoint.newSetQuery("field", 1), ft.termsQuery(Arrays.asList(1, 2.1), null));
+        assertEquals(LongPoint.newSetQuery("field", 1), ft.termsQuery(Arrays.asList(1.0, 2.1), null));
+        assertTrue(ft.termsQuery(Arrays.asList(1.1, 2.1), null) instanceof MatchNoDocsQuery);
+    }
+
+    public void testByteTermQueryWithDecimalPart() {
+        MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberType.BYTE);
+        ft.setName("field");
+        ft.setIndexOptions(IndexOptions.DOCS);
+        assertTrue(ft.termQuery(42.1, null) instanceof MatchNoDocsQuery);
+    }
+
+    public void testShortTermQueryWithDecimalPart() {
+        MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberType.SHORT);
+        ft.setName("field");
+        ft.setIndexOptions(IndexOptions.DOCS);
+        assertTrue(ft.termQuery(42.1, null) instanceof MatchNoDocsQuery);
+    }
+
+    public void testIntegerTermQueryWithDecimalPart() {
+        MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberType.INTEGER);
+        ft.setName("field");
+        ft.setIndexOptions(IndexOptions.DOCS);
+        assertTrue(ft.termQuery(42.1, null) instanceof MatchNoDocsQuery);
+    }
+
+    public void testLongTermQueryWithDecimalPart() {
+        MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
+        ft.setName("field");
+        ft.setIndexOptions(IndexOptions.DOCS);
+        assertTrue(ft.termQuery(42.1, null) instanceof MatchNoDocsQuery);
     }
 
     public void testTermQuery() {
@@ -74,49 +128,156 @@ public void testTermQuery() {
         assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage());
     }
 
+    public void testRangeQueryWithNegativeBounds() {
+        MappedFieldType ftInt = new NumberFieldMapper.NumberFieldType(NumberType.INTEGER);
+        ftInt.setName("field");
+        ftInt.setIndexOptions(IndexOptions.DOCS);
+        assertEquals(ftInt.rangeQuery(-3, -3, true, true, null), ftInt.rangeQuery(-3.5, -2.5, true, true, null));
+        assertEquals(ftInt.rangeQuery(-3, -3, true, true, null), ftInt.rangeQuery(-3.5, -2.5, false, false, null));
+        assertEquals(ftInt.rangeQuery(0, 0, true, true, null), ftInt.rangeQuery(-0.5, 0.5, true, true, null));
+        assertEquals(ftInt.rangeQuery(0, 0, true, true, null), ftInt.rangeQuery(-0.5, 0.5, false, false, null));
+        assertEquals(ftInt.rangeQuery(1, 2, true, true, null), ftInt.rangeQuery(0.5, 2.5, true, true, null));
+        assertEquals(ftInt.rangeQuery(1, 2, true, true, null), ftInt.rangeQuery(0.5, 2.5, false, false, null));
+        assertEquals(ftInt.rangeQuery(0, 2, true, true, null), ftInt.rangeQuery(-0.5, 2.5, true, true, null));
+        assertEquals(ftInt.rangeQuery(0, 2, true, true, null), ftInt.rangeQuery(-0.5, 2.5, false, false, null));
+
+        assertEquals(ftInt.rangeQuery(-2, 0, true, true, null), ftInt.rangeQuery(-2.5, 0.5, true, true, null));
+        assertEquals(ftInt.rangeQuery(-2, 0, true, true, null), ftInt.rangeQuery(-2.5, 0.5, false, false, null));
+        assertEquals(ftInt.rangeQuery(-2, -1, true, true, null), ftInt.rangeQuery(-2.5, -0.5, true, true, null));
+        assertEquals(ftInt.rangeQuery(-2, -1, true, true, null), ftInt.rangeQuery(-2.5, -0.5, false, false, null));
+
+        MappedFieldType ftLong = new NumberFieldMapper.NumberFieldType(NumberType.LONG);
+        ftLong.setName("field");
+        ftLong.setIndexOptions(IndexOptions.DOCS);
+        assertEquals(ftLong.rangeQuery(-3, -3, true, true, null), ftLong.rangeQuery(-3.5, -2.5, true, true, null));
+        assertEquals(ftLong.rangeQuery(-3, -3, true, true, null), ftLong.rangeQuery(-3.5, -2.5, false, false, null));
+        assertEquals(ftLong.rangeQuery(0, 0, true, true, null), ftLong.rangeQuery(-0.5, 0.5, true, true, null));
+        assertEquals(ftLong.rangeQuery(0, 0, true, true, null), ftLong.rangeQuery(-0.5, 0.5, false, false, null));
+        assertEquals(ftLong.rangeQuery(1, 2, true, true, null), ftLong.rangeQuery(0.5, 2.5, true, true, null));
+        assertEquals(ftLong.rangeQuery(1, 2, true, true, null), ftLong.rangeQuery(0.5, 2.5, false, false, null));
+        assertEquals(ftLong.rangeQuery(0, 2, true, true, null), ftLong.rangeQuery(-0.5, 2.5, true, true, null));
+        assertEquals(ftLong.rangeQuery(0, 2, true, true, null), ftLong.rangeQuery(-0.5, 2.5, false, false, null));
+
+        assertEquals(ftLong.rangeQuery(-2, 0, true, true, null), ftLong.rangeQuery(-2.5, 0.5, true, true, null));
+        assertEquals(ftLong.rangeQuery(-2, 0, true, true, null), ftLong.rangeQuery(-2.5, 0.5, false, false, null));
+        assertEquals(ftLong.rangeQuery(-2, -1, true, true, null), ftLong.rangeQuery(-2.5, -0.5, true, true, null));
+        assertEquals(ftLong.rangeQuery(-2, -1, true, true, null), ftLong.rangeQuery(-2.5, -0.5, false, false, null));
+    }
+
+    public void testByteRangeQueryWithDecimalParts() {
+        MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberType.BYTE);
+        ft.setName("field");
+        ft.setIndexOptions(IndexOptions.DOCS);
+        assertEquals(ft.rangeQuery(2, 10, true, true, null), ft.rangeQuery(1.1, 10, true, true, null));
+        assertEquals(ft.rangeQuery(2, 10, true, true, null), ft.rangeQuery(1.1, 10, false, true, null));
+        assertEquals(ft.rangeQuery(1, 10, true, true, null), ft.rangeQuery(1, 10.1, true, true, null));
+        assertEquals(ft.rangeQuery(1, 10, true, true, null), ft.rangeQuery(1, 10.1, true, false, null));
+    }
+
+    public void testShortRangeQueryWithDecimalParts() {
+        MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberType.SHORT);
+        ft.setName("field");
+        ft.setIndexOptions(IndexOptions.DOCS);
+        assertEquals(ft.rangeQuery(2, 10, true, true, null), ft.rangeQuery(1.1, 10, true, true, null));
+        assertEquals(ft.rangeQuery(2, 10, true, true, null), ft.rangeQuery(1.1, 10, false, true, null));
+        assertEquals(ft.rangeQuery(1, 10, true, true, null), ft.rangeQuery(1, 10.1, true, true, null));
+        assertEquals(ft.rangeQuery(1, 10, true, true, null), ft.rangeQuery(1, 10.1, true, false, null));
+    }
+
+    public void testIntegerRangeQueryWithDecimalParts() {
+        MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberType.INTEGER);
+        ft.setName("field");
+        ft.setIndexOptions(IndexOptions.DOCS);
+        assertEquals(ft.rangeQuery(2, 10, true, true, null), ft.rangeQuery(1.1, 10, true, true, null));
+        assertEquals(ft.rangeQuery(2, 10, true, true, null), ft.rangeQuery(1.1, 10, false, true, null));
+        assertEquals(ft.rangeQuery(1, 10, true, true, null), ft.rangeQuery(1, 10.1, true, true, null));
+        assertEquals(ft.rangeQuery(1, 10, true, true, null), ft.rangeQuery(1, 10.1, true, false, null));
+    }
+
+    public void testLongRangeQueryWithDecimalParts() {
+        MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberType.LONG);
+        ft.setName("field");
+        ft.setIndexOptions(IndexOptions.DOCS);
+        assertEquals(ft.rangeQuery(2, 10, true, true, null), ft.rangeQuery(1.1, 10, true, true, null));
+        assertEquals(ft.rangeQuery(2, 10, true, true, null), ft.rangeQuery(1.1, 10, false, true, null));
+        assertEquals(ft.rangeQuery(1, 10, true, true, null), ft.rangeQuery(1, 10.1, true, true, null));
+        assertEquals(ft.rangeQuery(1, 10, true, true, null), ft.rangeQuery(1, 10.1, true, false, null));
+    }
+
     public void testRangeQuery() {
         MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
         ft.setName("field");
         ft.setIndexOptions(IndexOptions.DOCS);
-        assertEquals(LongPoint.newRangeQuery("field", 1, 3), ft.rangeQuery("1", "3", true, true));
+        Query expected = new IndexOrDocValuesQuery(
+                LongPoint.newRangeQuery("field", 1, 3),
+                SortedNumericDocValuesField.newRangeQuery("field", 1, 3));
+        assertEquals(expected, ft.rangeQuery("1", "3", true, true, null));
 
         ft.setIndexOptions(IndexOptions.NONE);
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-                () -> ft.rangeQuery("1", "3", true, true));
+                () -> ft.rangeQuery("1", "3", true, true, null));
         assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage());
     }
 
     public void testConversions() {
-        assertEquals((byte) 3, NumberType.BYTE.parse(3d));
-        assertEquals((short) 3, NumberType.SHORT.parse(3d));
-        assertEquals(3, NumberType.INTEGER.parse(3d));
-        assertEquals(3L, NumberType.LONG.parse(3d));
-        assertEquals(3f, NumberType.HALF_FLOAT.parse(3d));
-        assertEquals(3f, NumberType.FLOAT.parse(3d));
-        assertEquals(3d, NumberType.DOUBLE.parse(3d));
-
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> NumberType.BYTE.parse(3.5));
-        assertEquals("Value [3.5] has a decimal part", e.getMessage());
-        e = expectThrows(IllegalArgumentException.class, () -> NumberType.SHORT.parse(3.5));
-        assertEquals("Value [3.5] has a decimal part", e.getMessage());
-        e = expectThrows(IllegalArgumentException.class, () -> NumberType.INTEGER.parse(3.5));
-        assertEquals("Value [3.5] has a decimal part", e.getMessage());
-        e = expectThrows(IllegalArgumentException.class, () -> NumberType.LONG.parse(3.5));
-        assertEquals("Value [3.5] has a decimal part", e.getMessage());
-        assertEquals(3.5f, NumberType.FLOAT.parse(3.5));
-        assertEquals(3.5d, NumberType.DOUBLE.parse(3.5));
-
-        e = expectThrows(IllegalArgumentException.class, () -> NumberType.BYTE.parse(128));
+        assertEquals((byte) 3, NumberType.BYTE.parse(3d, true));
+        assertEquals((short) 3, NumberType.SHORT.parse(3d, true));
+        assertEquals(3, NumberType.INTEGER.parse(3d, true));
+        assertEquals(3L, NumberType.LONG.parse(3d, true));
+        assertEquals(3f, NumberType.HALF_FLOAT.parse(3d, true));
+        assertEquals(3f, NumberType.FLOAT.parse(3d, true));
+        assertEquals(3d, NumberType.DOUBLE.parse(3d, true));
+
+        assertEquals((byte) 3, NumberType.BYTE.parse(3.5, true));
+        assertEquals((short) 3, NumberType.SHORT.parse(3.5, true));
+        assertEquals(3, NumberType.INTEGER.parse(3.5, true));
+        assertEquals(3L, NumberType.LONG.parse(3.5, true));
+
+        assertEquals(3.5f, NumberType.FLOAT.parse(3.5, true));
+        assertEquals(3.5d, NumberType.DOUBLE.parse(3.5, true));
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> NumberType.BYTE.parse(128, true));
         assertEquals("Value [128] is out of range for a byte", e.getMessage());
-        e = expectThrows(IllegalArgumentException.class, () -> NumberType.SHORT.parse(65536));
+        e = expectThrows(IllegalArgumentException.class, () -> NumberType.SHORT.parse(65536, true));
         assertEquals("Value [65536] is out of range for a short", e.getMessage());
-        e = expectThrows(IllegalArgumentException.class, () -> NumberType.INTEGER.parse(2147483648L));
+        e = expectThrows(IllegalArgumentException.class, () -> NumberType.INTEGER.parse(2147483648L, true));
         assertEquals("Value [2147483648] is out of range for an integer", e.getMessage());
-        e = expectThrows(IllegalArgumentException.class, () -> NumberType.LONG.parse(10000000000000000000d));
+        e = expectThrows(IllegalArgumentException.class, () -> NumberType.LONG.parse(10000000000000000000d, true));
         assertEquals("Value [1.0E19] is out of range for a long", e.getMessage());
-        assertEquals(1.1f, NumberType.HALF_FLOAT.parse(1.1));
-        assertEquals(1.1f, NumberType.FLOAT.parse(1.1));
-        assertEquals(1.1d, NumberType.DOUBLE.parse(1.1));
+        assertEquals(1.1f, NumberType.HALF_FLOAT.parse(1.1, true));
+        assertEquals(1.1f, NumberType.FLOAT.parse(1.1, true));
+        assertEquals(1.1d, NumberType.DOUBLE.parse(1.1, true));
+    }
+
+    public void testCoercions() {
+        assertEquals((byte) 5, NumberType.BYTE.parse((short) 5, true));
+        assertEquals((byte) 5, NumberType.BYTE.parse("5", true));
+        assertEquals((byte) 5, NumberType.BYTE.parse("5.0", true));
+        assertEquals((byte) 5, NumberType.BYTE.parse("5.9", true));
+        assertEquals((byte) 5, NumberType.BYTE.parse(new BytesRef("5.3".getBytes(StandardCharsets.UTF_8)), true));
+
+        assertEquals((short) 5, NumberType.SHORT.parse((byte) 5, true));
+        assertEquals((short) 5, NumberType.SHORT.parse("5", true));
+        assertEquals((short) 5, NumberType.SHORT.parse("5.0", true));
+        assertEquals((short) 5, NumberType.SHORT.parse("5.9", true));
+        assertEquals((short) 5, NumberType.SHORT.parse(new BytesRef("5.3".getBytes(StandardCharsets.UTF_8)), true));
+
+        assertEquals(5, NumberType.INTEGER.parse((byte) 5, true));
+        assertEquals(5, NumberType.INTEGER.parse("5", true));
+        assertEquals(5, NumberType.INTEGER.parse("5.0", true));
+        assertEquals(5, NumberType.INTEGER.parse("5.9", true));
+        assertEquals(5, NumberType.INTEGER.parse(new BytesRef("5.3".getBytes(StandardCharsets.UTF_8)), true));
+        assertEquals(Integer.MAX_VALUE, NumberType.INTEGER.parse(Integer.MAX_VALUE, true));
+
+        assertEquals((long) 5, NumberType.LONG.parse((byte) 5, true));
+        assertEquals((long) 5, NumberType.LONG.parse("5", true));
+        assertEquals((long) 5, NumberType.LONG.parse("5.0", true));
+        assertEquals((long) 5, NumberType.LONG.parse("5.9", true));
+        assertEquals((long) 5, NumberType.LONG.parse(new BytesRef("5.3".getBytes(StandardCharsets.UTF_8)), true));
+
+        // these will lose precision if they get treated as a double
+        assertEquals(-4115420654264075766L, NumberType.LONG.parse("-4115420654264075766", true));
+        assertEquals(-4115420654264075766L, NumberType.LONG.parse(-4115420654264075766L, true));
     }
 
     public void testHalfFloatRange() throws IOException {
@@ -143,10 +304,82 @@ public void testHalfFloatRange() throws IOException {
             float u = (randomFloat() * 2 - 1) * 70000;
             boolean includeLower = randomBoolean();
             boolean includeUpper = randomBoolean();
-            Query floatQ = NumberFieldMapper.NumberType.FLOAT.rangeQuery("float", l, u, includeLower, includeUpper);
-            Query halfFloatQ = NumberFieldMapper.NumberType.HALF_FLOAT.rangeQuery("half_float", l, u, includeLower, includeUpper);
+            Query floatQ = NumberFieldMapper.NumberType.FLOAT.rangeQuery("float", l, u, includeLower, includeUpper, false);
+            Query halfFloatQ = NumberFieldMapper.NumberType.HALF_FLOAT.rangeQuery("half_float", l, u, includeLower, includeUpper, false);
             assertEquals(searcher.count(floatQ), searcher.count(halfFloatQ));
         }
         IOUtils.close(reader, dir);
     }
+
+    public void testNegativeZero() {
+        assertEquals(
+                NumberType.DOUBLE.rangeQuery("field", null, -0d, true, true, false),
+                NumberType.DOUBLE.rangeQuery("field", null, +0d, true, false, false));
+        assertEquals(
+                NumberType.FLOAT.rangeQuery("field", null, -0f, true, true, false),
+                NumberType.FLOAT.rangeQuery("field", null, +0f, true, false, false));
+        assertEquals(
+                NumberType.HALF_FLOAT.rangeQuery("field", null, -0f, true, true, false),
+                NumberType.HALF_FLOAT.rangeQuery("field", null, +0f, true, false, false));
+
+        assertFalse(NumberType.DOUBLE.termQuery("field", -0d).equals(NumberType.DOUBLE.termQuery("field", +0d)));
+        assertFalse(NumberType.FLOAT.termQuery("field", -0f).equals(NumberType.FLOAT.termQuery("field", +0f)));
+        assertFalse(NumberType.HALF_FLOAT.termQuery("field", -0f).equals(NumberType.HALF_FLOAT.termQuery("field", +0f)));
+    }
+
+    // Make sure we construct the IndexOrDocValuesQuery objects with queries that match
+    // the same ranges
+    public void testDocValueByteRange() throws Exception {
+        doTestDocValueRangeQueries(NumberType.BYTE, () -> (byte) random().nextInt(256));
+    }
+
+    public void testDocValueShortRange() throws Exception {
+        doTestDocValueRangeQueries(NumberType.SHORT, () -> (short) random().nextInt(65536));
+    }
+
+    public void testDocValueIntRange() throws Exception {
+        doTestDocValueRangeQueries(NumberType.INTEGER, random()::nextInt);
+    }
+
+    public void testDocValueLongRange() throws Exception {
+        doTestDocValueRangeQueries(NumberType.LONG, random()::nextLong);
+    }
+
+    public void testDocValueHalfFloatRange() throws Exception {
+        doTestDocValueRangeQueries(NumberType.HALF_FLOAT, random()::nextFloat);
+    }
+
+    public void testDocValueFloatRange() throws Exception {
+        doTestDocValueRangeQueries(NumberType.FLOAT, random()::nextFloat);
+    }
+
+    public void testDocValueDoubleRange() throws Exception {
+        doTestDocValueRangeQueries(NumberType.DOUBLE, random()::nextDouble);
+    }
+
+    public void doTestDocValueRangeQueries(NumberType type, Supplier<Number> valueSupplier) throws Exception {
+        Directory dir = newDirectory();
+        IndexWriter w = new IndexWriter(dir, newIndexWriterConfig());
+        final int numDocs = TestUtil.nextInt(random(), 100, 500);
+        for (int i = 0; i < numDocs; ++i) {
+            w.addDocument(type.createFields("foo", valueSupplier.get(), true, true, false));
+        }
+        DirectoryReader reader = DirectoryReader.open(w);
+        IndexSearcher searcher = newSearcher(reader);
+        w.close();
+        final int iters = 10;
+        for (int iter = 0; iter < iters; ++iter) {
+            Query query = type.rangeQuery("foo",
+                    random().nextBoolean() ? null : valueSupplier.get(),
+                    random().nextBoolean() ? null : valueSupplier.get(),
+                    randomBoolean(), randomBoolean(), true);
+            assertThat(query, Matchers.instanceOf(IndexOrDocValuesQuery.class));
+            IndexOrDocValuesQuery indexOrDvQuery = (IndexOrDocValuesQuery) query;
+            assertEquals(
+                    searcher.count(indexOrDvQuery.getIndexQuery()),
+                    searcher.count(indexOrDvQuery.getRandomAccessQuery()));
+        }
+        reader.close();
+        dir.close();
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/ParentFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/ParentFieldMapperTests.java
index 15738ada3bcc7..2f381a7b8d54d 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/ParentFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/ParentFieldMapperTests.java
@@ -18,30 +18,24 @@
  */
 package org.elasticsearch.index.mapper;
 
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.IndexableField;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.compress.CompressedXContent;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.analysis.AnalysisService;
-import org.elasticsearch.index.mapper.ContentPath;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.Mapper;
-import org.elasticsearch.index.mapper.MapperParsingException;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.ParentFieldMapper;
+import org.elasticsearch.index.analysis.AnalyzerScope;
+import org.elasticsearch.index.analysis.IndexAnalyzers;
+import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.mapper.MapperService.MergeReason;
-import org.elasticsearch.index.mapper.ParseContext;
-import org.elasticsearch.index.mapper.ParsedDocument;
-import org.elasticsearch.index.mapper.SourceToParse;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.indices.IndicesModule;
 import org.elasticsearch.test.ESSingleNodeTestCase;
@@ -65,7 +59,7 @@ public void testParentSetInDocNotAllowed() throws Exception {
 
         try {
             docMapper.parse(SourceToParse.source("test", "type", "1", XContentFactory.jsonBuilder()
-                .startObject().field("_parent", "1122").endObject().bytes()));
+                .startObject().field("_parent", "1122").endObject().bytes(), XContentType.JSON));
             fail("Expected failure to parse metadata field");
         } catch (MapperParsingException e) {
             assertTrue(e.getMessage(), e.getMessage().contains("Field [_parent] is a metadata field and cannot be added inside a document"));
@@ -84,13 +78,14 @@ public void testJoinFieldSet() throws Exception {
 
         // Indexing parent doc:
         DocumentMapper parentDocMapper = indexService.mapperService().documentMapper("parent_type");
-        ParsedDocument doc = parentDocMapper.parse(SourceToParse.source("test", "parent_type", "1122", new BytesArray("{}")));
+        ParsedDocument doc =
+            parentDocMapper.parse(SourceToParse.source("test", "parent_type", "1122", new BytesArray("{}"), XContentType.JSON));
         assertEquals(1, getNumberOfFieldWithParentPrefix(doc.rootDoc()));
         assertEquals("1122", doc.rootDoc().getBinaryValue("_parent#parent_type").utf8ToString());
 
         // Indexing child doc:
         DocumentMapper childDocMapper = indexService.mapperService().documentMapper("child_type");
-        doc = childDocMapper.parse(SourceToParse.source("test", "child_type", "1", new BytesArray("{}")).parent("1122"));
+        doc = childDocMapper.parse(SourceToParse.source("test", "child_type", "1", new BytesArray("{}"), XContentType.JSON).parent("1122"));
 
         assertEquals(1, getNumberOfFieldWithParentPrefix(doc.rootDoc()));
         assertEquals("1122", doc.rootDoc().getBinaryValue("_parent#parent_type").utf8ToString());
@@ -104,17 +99,18 @@ public void testJoinFieldNotSet() throws Exception {
                 .startObject()
                 .field("x_field", "x_value")
                 .endObject()
-                .bytes()));
+                .bytes(), XContentType.JSON));
         assertEquals(0, getNumberOfFieldWithParentPrefix(doc.rootDoc()));
     }
 
     public void testNoParentNullFieldCreatedIfNoParentSpecified() throws Exception {
         Index index = new Index("_index", "testUUID");
         IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(index, Settings.EMPTY);
-        AnalysisService analysisService = new AnalysisService(indexSettings, Collections.emptyMap(), Collections.emptyMap(),
+        NamedAnalyzer namedAnalyzer = new NamedAnalyzer("default", AnalyzerScope.INDEX, new StandardAnalyzer());
+        IndexAnalyzers indexAnalyzers = new IndexAnalyzers(indexSettings, namedAnalyzer, namedAnalyzer, namedAnalyzer,
             Collections.emptyMap(), Collections.emptyMap());
         SimilarityService similarityService = new SimilarityService(indexSettings, Collections.emptyMap());
-        MapperService mapperService = new MapperService(indexSettings, analysisService, similarityService,
+        MapperService mapperService = new MapperService(indexSettings, indexAnalyzers, xContentRegistry(), similarityService,
             new IndicesModule(emptyList()).getMapperRegistry(), () -> null);
         XContentBuilder mappingSource = jsonBuilder().startObject().startObject("some_type")
             .startObject("properties")
@@ -124,6 +120,8 @@ public void testNoParentNullFieldCreatedIfNoParentSpecified() throws Exception {
         Set<String> allFields = new HashSet<>(mapperService.simpleMatchToIndexNames("*"));
         assertTrue(allFields.contains("_parent"));
         assertFalse(allFields.contains("_parent#null"));
+        MappedFieldType fieldType = mapperService.fullName("_parent");
+        assertFalse(fieldType.eagerGlobalOrdinals());
     }
 
     private static int getNumberOfFieldWithParentPrefix(ParseContext.Document doc) {
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java
new file mode 100644
index 0000000000000..3ce5d26fe9faf
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java
@@ -0,0 +1,464 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.index.mapper;
+
+import org.apache.lucene.document.InetAddressPoint;
+import org.apache.lucene.index.IndexableField;
+import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.network.InetAddresses;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Locale;
+
+import static org.elasticsearch.index.query.RangeQueryBuilder.GT_FIELD;
+import static org.elasticsearch.index.query.RangeQueryBuilder.GTE_FIELD;
+import static org.elasticsearch.index.query.RangeQueryBuilder.LT_FIELD;
+import static org.elasticsearch.index.query.RangeQueryBuilder.LTE_FIELD;
+import static org.hamcrest.Matchers.anyOf;
+import static org.hamcrest.Matchers.containsString;
+
+public class RangeFieldMapperTests extends AbstractNumericFieldMapperTestCase {
+    private static String FROM_DATE = "2016-10-31";
+    private static String TO_DATE = "2016-11-01 20:00:00";
+    private static String FROM_IP = "::ffff:c0a8:107";
+    private static String TO_IP = "2001:db8::";
+    private static int FROM = 5;
+    private static String FROM_STR = FROM + "";
+    private static int TO = 10;
+    private static String TO_STR = TO + "";
+    private static String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis";
+
+    @Override
+    protected void setTypeList() {
+        TYPES = new HashSet<>(Arrays.asList("date_range", "ip_range", "float_range", "double_range", "integer_range", "long_range"));
+        WHOLE_TYPES = new HashSet<>(Arrays.asList("integer_range", "long_range"));
+    }
+
+    private Object getFrom(String type) {
+        if (type.equals("date_range")) {
+            return FROM_DATE;
+        } else if (type.equals("ip_range")) {
+            return FROM_IP;
+        }
+        return random().nextBoolean() ? FROM : FROM_STR;
+    }
+
+    private String getFromField() {
+        return random().nextBoolean() ? GT_FIELD.getPreferredName() : GTE_FIELD.getPreferredName();
+    }
+
+    private String getToField() {
+        return random().nextBoolean() ? LT_FIELD.getPreferredName() : LTE_FIELD.getPreferredName();
+    }
+
+    private Object getTo(String type) {
+        if (type.equals("date_range")) {
+            return TO_DATE;
+        } else if (type.equals("ip_range")) {
+            return TO_IP;
+        }
+        return random().nextBoolean() ? TO : TO_STR;
+    }
+
+    private Object getMax(String type) {
+        if (type.equals("date_range") || type.equals("long_range")) {
+            return Long.MAX_VALUE;
+        } else if (type.equals("ip_range")) {
+            return InetAddressPoint.MAX_VALUE;
+        } else if (type.equals("integer_range")) {
+            return Integer.MAX_VALUE;
+        } else if (type.equals("float_range")) {
+            return Float.POSITIVE_INFINITY;
+        }
+        return Double.POSITIVE_INFINITY;
+    }
+
+    @Override
+    public void doTestDefaults(String type) throws Exception {
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startObject("properties").startObject("field").field("type", type);
+        if (type.equals("date_range")) {
+            mapping = mapping.field("format", DATE_FORMAT);
+        }
+        mapping = mapping.endObject().endObject().endObject().endObject();
+
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping.string()));
+        assertEquals(mapping.string(), mapper.mappingSource().toString());
+
+        ParsedDocument doc = mapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
+            .startObject()
+            .startObject("field")
+            .field(getFromField(), getFrom(type))
+            .field(getToField(), getTo(type))
+            .endObject()
+            .endObject().bytes());
+
+        IndexableField[] fields = doc.rootDoc().getFields("field");
+        assertEquals(1, fields.length);
+        IndexableField pointField = fields[0];
+        assertEquals(2, pointField.fieldType().pointDimensionCount());
+        assertFalse(pointField.fieldType().stored());
+    }
+
+    @Override
+    protected void doTestNotIndexed(String type) throws Exception {
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startObject("properties").startObject("field").field("type", type).field("index", false);
+        if (type.equals("date_range")) {
+            mapping = mapping.field("format", DATE_FORMAT);
+        }
+        mapping = mapping.endObject().endObject().endObject().endObject();
+
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping.string()));
+        assertEquals(mapping.string(), mapper.mappingSource().toString());
+
+        ParsedDocument doc = mapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
+            .startObject()
+            .startObject("field")
+            .field(getFromField(), getFrom(type))
+            .field(getToField(), getTo(type))
+            .endObject()
+            .endObject().bytes());
+
+        IndexableField[] fields = doc.rootDoc().getFields("field");
+        assertEquals(0, fields.length);
+    }
+
+    @Override
+    protected void doTestNoDocValues(String type) throws Exception {
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startObject("properties").startObject("field").field("type", type).field("doc_values", false);
+        if (type.equals("date_range")) {
+            mapping = mapping.field("format", DATE_FORMAT);
+        }
+        mapping = mapping.endObject().endObject().endObject().endObject();
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping.string()));
+        assertEquals(mapping.string(), mapper.mappingSource().toString());
+
+        ParsedDocument doc = mapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
+            .startObject()
+            .startObject("field")
+            .field(getFromField(), getFrom(type))
+            .field(getToField(), getTo(type))
+            .endObject()
+            .endObject().bytes());
+
+        IndexableField[] fields = doc.rootDoc().getFields("field");
+        assertEquals(1, fields.length);
+        IndexableField pointField = fields[0];
+        assertEquals(2, pointField.fieldType().pointDimensionCount());
+    }
+
+    @Override
+    protected void doTestStore(String type) throws Exception {
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startObject("properties").startObject("field").field("type", type).field("store", true);
+        if (type.equals("date_range")) {
+            mapping = mapping.field("format", DATE_FORMAT);
+        }
+        mapping = mapping.endObject().endObject().endObject().endObject();
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping.string()));
+        assertEquals(mapping.string(), mapper.mappingSource().toString());
+
+        ParsedDocument doc = mapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
+            .startObject()
+            .startObject("field")
+            .field(getFromField(), getFrom(type))
+            .field(getToField(), getTo(type))
+            .endObject()
+            .endObject().bytes());
+
+        IndexableField[] fields = doc.rootDoc().getFields("field");
+        assertEquals(2, fields.length);
+        IndexableField pointField = fields[0];
+        assertEquals(2, pointField.fieldType().pointDimensionCount());
+        IndexableField storedField = fields[1];
+        assertTrue(storedField.fieldType().stored());
+        String strVal = "5";
+        if (type.equals("date_range")) {
+            strVal = "1477872000000";
+        } else if (type.equals("ip_range")) {
+            strVal = InetAddresses.toAddrString(InetAddresses.forString("192.168.1.7")) + " : "
+                + InetAddresses.toAddrString(InetAddresses.forString("2001:db8:0:0:0:0:0:0"));
+        }
+        assertThat(storedField.stringValue(), containsString(strVal));
+    }
+
+    @Override
+    public void doTestCoerce(String type) throws IOException {
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startObject("properties").startObject("field").field("type", type);
+        if (type.equals("date_range")) {
+            mapping = mapping.field("format", DATE_FORMAT);
+        }
+        mapping = mapping.endObject().endObject().endObject().endObject();
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping.string()));
+
+        assertEquals(mapping.string(), mapper.mappingSource().toString());
+
+        ParsedDocument doc = mapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
+            .startObject()
+            .startObject("field")
+            .field(getFromField(), getFrom(type))
+            .field(getToField(), getTo(type))
+            .endObject()
+            .endObject().bytes());
+
+        IndexableField[] fields = doc.rootDoc().getFields("field");
+        assertEquals(1, fields.length);
+        IndexableField pointField = fields[0];
+        assertEquals(2, pointField.fieldType().pointDimensionCount());
+
+        mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startObject("properties").startObject("field").field("type", type).field("coerce", false).endObject().endObject()
+            .endObject().endObject();
+        DocumentMapper mapper2 = parser.parse("type", new CompressedXContent(mapping.string()));
+
+        assertEquals(mapping.string(), mapper2.mappingSource().toString());
+
+        ThrowingRunnable runnable = () -> mapper2.parse("test", "type", "1", XContentFactory.jsonBuilder()
+            .startObject()
+            .startObject("field")
+            .field(getFromField(), "5.2")
+            .field(getToField(), "10")
+            .endObject()
+            .endObject().bytes());
+        MapperParsingException e = expectThrows(MapperParsingException.class, runnable);
+        assertThat(e.getCause().getMessage(), anyOf(containsString("passed as String"),
+            containsString("failed to parse date"), containsString("is not an IP string literal")));
+    }
+
+    @Override
+    protected void doTestIncludeInAll(String type) throws Exception {
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startObject("properties").startObject("field").field("type", type);
+        if (type.equals("date_range")) {
+            mapping = mapping.field("format", DATE_FORMAT);
+        }
+        mapping = mapping.endObject().endObject().endObject().endObject();
+
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping.string()));
+
+        assertEquals(mapping.string(), mapper.mappingSource().toString());
+
+        ParsedDocument doc = mapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
+            .startObject()
+            .startObject("field")
+            .field(getFromField(), getFrom(type))
+            .field(getToField(), getTo(type))
+            .endObject()
+            .endObject().bytes());
+
+        IndexableField[] fields = doc.rootDoc().getFields("_all");
+        assertEquals(1, fields.length);
+
+        String strVal = "5";
+        if (type.equals("date_range")) {
+            strVal = "1477872000000";
+        } else if (type.equals("ip_range")) {
+            strVal = InetAddresses.toAddrString(InetAddresses.forString("192.168.1.7")) + " : "
+                + InetAddresses.toAddrString(InetAddresses.forString("2001:db8:0:0:0:0:0:0"));
+        }
+        assertThat(fields[0].stringValue(), containsString(strVal));
+
+        mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startObject("properties").startObject("field").field("type", type);
+        if (type.equals("date_range")) {
+            mapping = mapping.field("format", DATE_FORMAT);
+        }
+        mapping = mapping.field("include_in_all", false).endObject().endObject()
+            .endObject().endObject();
+
+        mapper = parser.parse("type", new CompressedXContent(mapping.string()));
+
+        assertEquals(mapping.string(), mapper.mappingSource().toString());
+
+        doc = mapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
+            .startObject()
+            .startObject("field")
+            .field(getFromField(), getFrom(type))
+            .field(getToField(), getTo(type))
+            .endObject()
+            .endObject().bytes());
+
+        fields = doc.rootDoc().getFields("_all");
+        assertEquals(0, fields.length);
+        assertWarnings("field [include_in_all] is deprecated, as [_all] is deprecated, and will be disallowed" +
+                        " in 6.0, use [copy_to] instead.");
+    }
+
+     @Override
+    protected void doTestDecimalCoerce(String type) throws IOException {
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startObject("properties").startObject("field").field("type", type);
+
+        mapping = mapping.endObject().endObject().endObject().endObject();
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping.string()));
+
+        assertEquals(mapping.string(), mapper.mappingSource().toString());
+
+        ParsedDocument doc1 = mapper.parse(SourceToParse.source("test", "type", "1", XContentFactory.jsonBuilder()
+            .startObject()
+            .startObject("field")
+            .field(GT_FIELD.getPreferredName(), "2.34")
+            .field(LT_FIELD.getPreferredName(), "5.67")
+            .endObject()
+            .endObject().bytes(),
+            XContentType.JSON));
+
+        ParsedDocument doc2 = mapper.parse(SourceToParse.source("test", "type", "1", XContentFactory.jsonBuilder()
+            .startObject()
+            .startObject("field")
+            .field(GT_FIELD.getPreferredName(), "2")
+            .field(LT_FIELD.getPreferredName(), "5")
+            .endObject()
+            .endObject().bytes(),
+            XContentType.JSON));
+
+        IndexableField[] fields1 = doc1.rootDoc().getFields("field");
+        IndexableField[] fields2 = doc2.rootDoc().getFields("field");
+
+        assertNotNull(fields1[0].binaryValue());
+        assertEquals(fields1[0].binaryValue(), fields2[0].binaryValue());
+    }
+
+    @Override
+    protected void doTestNullValue(String type) throws IOException {
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startObject("properties").startObject("field").field("type", type).field("store", true);
+        if (type.equals("date_range")) {
+            mapping = mapping.field("format", DATE_FORMAT);
+        }
+        mapping = mapping.endObject().endObject().endObject().endObject();
+
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping.string()));
+        assertEquals(mapping.string(), mapper.mappingSource().toString());
+
+        // test null value for min and max
+        ParsedDocument doc = mapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
+            .startObject()
+            .startObject("field")
+            .nullField(getFromField())
+            .nullField(getToField())
+            .endObject()
+            .endObject().bytes());
+        assertEquals(2, doc.rootDoc().getFields("field").length);
+        IndexableField[] fields = doc.rootDoc().getFields("field");
+        IndexableField storedField = fields[1];
+        String expected = type.equals("ip_range") ? InetAddresses.toAddrString((InetAddress)getMax(type)) : getMax(type) +"";
+        assertThat(storedField.stringValue(), containsString(expected));
+
+        // test null max value
+        doc = mapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
+            .startObject()
+            .startObject("field")
+            .field(getFromField(), getFrom(type))
+            .nullField(getToField())
+            .endObject()
+            .endObject().bytes());
+
+        fields = doc.rootDoc().getFields("field");
+        assertEquals(2, fields.length);
+        IndexableField pointField = fields[0];
+        assertEquals(2, pointField.fieldType().pointDimensionCount());
+        assertFalse(pointField.fieldType().stored());
+        storedField = fields[1];
+        assertTrue(storedField.fieldType().stored());
+        String strVal = "5";
+        if (type.equals("date_range")) {
+            strVal = "1477872000000";
+        } else if (type.equals("ip_range")) {
+            strVal = InetAddresses.toAddrString(InetAddresses.forString("192.168.1.7")) + " : "
+                + InetAddresses.toAddrString(InetAddresses.forString("ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff"));
+        }
+        assertThat(storedField.stringValue(), containsString(strVal));
+    }
+
+    public void testNoBounds() throws Exception {
+        for (String type : TYPES) {
+            doTestNoBounds(type);
+        }
+    }
+
+    public void doTestNoBounds(String type) throws IOException {
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startObject("properties").startObject("field").field("type", type).field("store", true);
+        if (type.equals("date_range")) {
+            mapping = mapping.field("format", DATE_FORMAT);
+        }
+        mapping = mapping.endObject().endObject().endObject().endObject();
+
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping.string()));
+        assertEquals(mapping.string(), mapper.mappingSource().toString());
+
+        // test no bounds specified
+        ParsedDocument doc = mapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
+            .startObject()
+            .startObject("field")
+            .endObject()
+            .endObject().bytes());
+
+        IndexableField[] fields = doc.rootDoc().getFields("field");
+        assertEquals(2, fields.length);
+        IndexableField pointField = fields[0];
+        assertEquals(2, pointField.fieldType().pointDimensionCount());
+        assertFalse(pointField.fieldType().stored());
+        IndexableField storedField = fields[1];
+        assertTrue(storedField.fieldType().stored());
+        String expected = type.equals("ip_range") ? InetAddresses.toAddrString((InetAddress)getMax(type)) : getMax(type) +"";
+        assertThat(storedField.stringValue(), containsString(expected));
+    }
+
+    public void testIllegalArguments() throws Exception {
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startObject("properties").startObject("field").field("type", RangeFieldMapper.RangeType.INTEGER.name)
+            .field("format", DATE_FORMAT).endObject().endObject().endObject().endObject();
+
+        ThrowingRunnable runnable = () -> parser.parse("type", new CompressedXContent(mapping.string()));
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, runnable);
+        assertThat(e.getMessage(), containsString("should not define a dateTimeFormatter"));
+    }
+
+    public void testSerializeDefaults() throws Exception {
+        for (String type : TYPES) {
+            String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties").startObject("field").field("type", type).endObject().endObject()
+                .endObject().endObject().string();
+
+            DocumentMapper docMapper = parser.parse("type", new CompressedXContent(mapping));
+            RangeFieldMapper mapper = (RangeFieldMapper) docMapper.root().getMapper("field");
+            XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
+            mapper.doXContentBody(builder, true, ToXContent.EMPTY_PARAMS);
+            String got = builder.endObject().string();
+
+            // if type is date_range we check that the mapper contains the default format and locale
+            // otherwise it should not contain a locale or format
+            assertTrue(got, got.contains("\"format\":\"strict_date_optional_time||epoch_millis\"") == type.equals("date_range"));
+            assertTrue(got, got.contains("\"locale\":" + "\"" + Locale.ROOT + "\"") == type.equals("date_range"));
+        }
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java
new file mode 100644
index 0000000000000..9f3a3510189da
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java
@@ -0,0 +1,235 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.index.mapper;
+
+import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+
+import org.apache.lucene.document.DoubleRange;
+import org.apache.lucene.document.FloatRange;
+import org.apache.lucene.document.InetAddressPoint;
+import org.apache.lucene.document.InetAddressRange;
+import org.apache.lucene.document.IntRange;
+import org.apache.lucene.document.LongRange;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.geo.ShapeRelation;
+import org.elasticsearch.common.joda.Joda;
+import org.elasticsearch.common.network.InetAddresses;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.mapper.RangeFieldMapper.RangeType;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.test.IndexSettingsModule;
+import org.joda.time.DateTime;
+import org.junit.Before;
+
+import java.net.InetAddress;
+import java.util.Locale;
+
+public class RangeFieldTypeTests extends FieldTypeTestCase {
+    RangeType type;
+    protected static String FIELDNAME = "field";
+    protected static int DISTANCE = 10;
+    private static long nowInMillis;
+
+    @Before
+    public void setupProperties() {
+        type = RandomPicks.randomFrom(random(), RangeType.values());
+        nowInMillis = randomNonNegativeLong();
+        if (type == RangeType.DATE) {
+            addModifier(new Modifier("format", true) {
+                @Override
+                public void modify(MappedFieldType ft) {
+                    ((RangeFieldMapper.RangeFieldType) ft).setDateTimeFormatter(Joda.forPattern("basic_week_date", Locale.ROOT));
+                }
+            });
+            addModifier(new Modifier("locale", true) {
+                @Override
+                public void modify(MappedFieldType ft) {
+                    ((RangeFieldMapper.RangeFieldType) ft).setDateTimeFormatter(Joda.forPattern("date_optional_time", Locale.CANADA));
+                }
+            });
+        }
+    }
+
+    @Override
+    protected RangeFieldMapper.RangeFieldType createDefaultFieldType() {
+        return new RangeFieldMapper.RangeFieldType(type);
+    }
+
+    public void testRangeQuery() throws Exception {
+        Settings indexSettings = Settings.builder()
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
+        IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAlphaOfLengthBetween(1, 10), indexSettings);
+        QueryShardContext context = new QueryShardContext(0, idxSettings, null, null, null, null, null, xContentRegistry(),
+                null, null, () -> nowInMillis);
+        RangeFieldMapper.RangeFieldType ft = new RangeFieldMapper.RangeFieldType(type);
+        ft.setName(FIELDNAME);
+        ft.setIndexOptions(IndexOptions.DOCS);
+
+        ShapeRelation relation = RandomPicks.randomFrom(random(), ShapeRelation.values());
+        boolean includeLower = random().nextBoolean();
+        boolean includeUpper = random().nextBoolean();
+        Object from = nextFrom();
+        Object to = nextTo(from);
+
+        assertEquals(getExpectedRangeQuery(relation, from, to, includeLower, includeUpper),
+            ft.rangeQuery(from, to, includeLower, includeUpper, relation, context));
+    }
+
+    private Query getExpectedRangeQuery(ShapeRelation relation, Object from, Object to, boolean includeLower, boolean includeUpper) {
+        switch (type) {
+            case DATE:
+                return getDateRangeQuery(relation, (DateTime)from, (DateTime)to, includeLower, includeUpper);
+            case INTEGER:
+                return getIntRangeQuery(relation, (int)from, (int)to, includeLower, includeUpper);
+            case LONG:
+                return getLongRangeQuery(relation, (long)from, (long)to, includeLower, includeUpper);
+            case DOUBLE:
+                return getDoubleRangeQuery(relation, (double)from, (double)to, includeLower, includeUpper);
+            case IP:
+                return getInetAddressRangeQuery(relation, (InetAddress)from, (InetAddress)to, includeLower, includeUpper);
+            default:
+                return getFloatRangeQuery(relation, (float)from, (float)to, includeLower, includeUpper);
+        }
+    }
+
+    private Query getDateRangeQuery(ShapeRelation relation, DateTime from, DateTime to, boolean includeLower, boolean includeUpper) {
+        return getLongRangeQuery(relation, from.getMillis(), to.getMillis(), includeLower, includeUpper);
+    }
+
+    private Query getIntRangeQuery(ShapeRelation relation, int from, int to, boolean includeLower, boolean includeUpper) {
+        int[] lower = new int[] {from + (includeLower ? 0 : 1)};
+        int[] upper = new int[] {to - (includeUpper ? 0 : 1)};
+        if (relation == ShapeRelation.WITHIN) {
+            return IntRange.newWithinQuery(FIELDNAME, lower, upper);
+        } else if (relation == ShapeRelation.CONTAINS) {
+            return IntRange.newContainsQuery(FIELDNAME, lower, upper);
+        }
+        return IntRange.newIntersectsQuery(FIELDNAME, lower, upper);
+    }
+
+    private Query getLongRangeQuery(ShapeRelation relation, long from, long to, boolean includeLower, boolean includeUpper) {
+        long[] lower = new long[] {from + (includeLower ? 0 : 1)};
+        long[] upper = new long[] {to - (includeUpper ? 0 : 1)};
+        if (relation == ShapeRelation.WITHIN) {
+            return LongRange.newWithinQuery(FIELDNAME, lower, upper);
+        } else if (relation == ShapeRelation.CONTAINS) {
+            return LongRange.newContainsQuery(FIELDNAME, lower, upper);
+        }
+        return LongRange.newIntersectsQuery(FIELDNAME, lower, upper);
+    }
+
+    private Query getFloatRangeQuery(ShapeRelation relation, float from, float to, boolean includeLower, boolean includeUpper) {
+        float[] lower = new float[] {includeLower ? from : Math.nextUp(from)};
+        float[] upper = new float[] {includeUpper ? to : Math.nextDown(to)};
+        if (relation == ShapeRelation.WITHIN) {
+            return FloatRange.newWithinQuery(FIELDNAME, lower, upper);
+        } else if (relation == ShapeRelation.CONTAINS) {
+            return FloatRange.newContainsQuery(FIELDNAME, lower, upper);
+        }
+        return FloatRange.newIntersectsQuery(FIELDNAME, lower, upper);
+    }
+
+    private Query getDoubleRangeQuery(ShapeRelation relation, double from, double to, boolean includeLower,
+                                      boolean includeUpper) {
+        double[] lower = new double[] {includeLower ? from : Math.nextUp(from)};
+        double[] upper = new double[] {includeUpper ? to : Math.nextDown(to)};
+        if (relation == ShapeRelation.WITHIN) {
+            return DoubleRange.newWithinQuery(FIELDNAME, lower, upper);
+        } else if (relation == ShapeRelation.CONTAINS) {
+            return DoubleRange.newContainsQuery(FIELDNAME, lower, upper);
+        }
+        return DoubleRange.newIntersectsQuery(FIELDNAME, lower, upper);
+    }
+
+    private Query getInetAddressRangeQuery(ShapeRelation relation, InetAddress from, InetAddress to, boolean includeLower,
+                                           boolean includeUpper) {
+        InetAddress lower = includeLower ? from : InetAddressPoint.nextUp(from);
+        InetAddress upper = includeUpper ? to : InetAddressPoint.nextDown(to);
+        if (relation == ShapeRelation.WITHIN) {
+            return InetAddressRange.newWithinQuery(FIELDNAME, lower, upper);
+        } else if (relation == ShapeRelation.CONTAINS) {
+            return InetAddressRange.newContainsQuery(FIELDNAME, lower, upper);
+        }
+        return InetAddressRange.newIntersectsQuery(FIELDNAME, lower, upper);
+    }
+
+    private Object nextFrom() throws Exception {
+        switch (type) {
+            case INTEGER:
+                return (int)(random().nextInt() * 0.5 - DISTANCE);
+            case DATE:
+                return DateTime.now();
+            case LONG:
+                return (long)(random().nextLong() * 0.5 - DISTANCE);
+            case FLOAT:
+                return (float)(random().nextFloat() * 0.5 - DISTANCE);
+            case IP:
+                return InetAddress.getByName("::ffff:c0a8:107");
+            default:
+                return random().nextDouble() * 0.5 - DISTANCE;
+        }
+    }
+
+    private Object nextTo(Object from) throws Exception {
+        switch (type) {
+            case INTEGER:
+                return (Integer)from + DISTANCE;
+            case DATE:
+                return DateTime.now().plusDays(DISTANCE);
+            case LONG:
+                return (Long)from + DISTANCE;
+            case DOUBLE:
+                return (Double)from + DISTANCE;
+            case IP:
+                return InetAddress.getByName("2001:db8::");
+            default:
+                return (Float)from + DISTANCE;
+        }
+    }
+
+    public void testParseIp() {
+        assertEquals(InetAddresses.forString("::1"), RangeFieldMapper.RangeType.IP.parse(InetAddresses.forString("::1"), randomBoolean()));
+        assertEquals(InetAddresses.forString("::1"), RangeFieldMapper.RangeType.IP.parse("::1", randomBoolean()));
+        assertEquals(InetAddresses.forString("::1"), RangeFieldMapper.RangeType.IP.parse(new BytesRef("::1"), randomBoolean()));
+    }
+
+    public void testTermQuery() throws Exception, IllegalArgumentException {
+        // See https://github.com/elastic/elasticsearch/issues/25950
+        Settings indexSettings = Settings.builder()
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
+        IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAlphaOfLengthBetween(1, 10), indexSettings);
+        QueryShardContext context = new QueryShardContext(0, idxSettings, null, null, null, null, null, xContentRegistry(),
+                null, null, () -> nowInMillis);
+        RangeFieldMapper.RangeFieldType ft = new RangeFieldMapper.RangeFieldType(type);
+        ft.setName(FIELDNAME);
+        ft.setIndexOptions(IndexOptions.DOCS);
+
+        Object value = nextFrom();
+        ShapeRelation relation = ShapeRelation.INTERSECTS;
+        boolean includeLower = true;
+        boolean includeUpper = true;
+        assertEquals(getExpectedRangeQuery(relation, value, value, includeLower, includeUpper),
+            ft.termQuery(value, context));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java
index 92e93ede9aa7a..9c26a9806e331 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java
@@ -21,6 +21,7 @@
 
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.ParsedDocument;
@@ -40,7 +41,7 @@ public void testRoutingMapper() throws Exception {
             .startObject()
             .field("field", "value")
             .endObject()
-            .bytes()).routing("routing_value"));
+            .bytes(), XContentType.JSON).routing("routing_value"));
 
         assertThat(doc.rootDoc().get("_routing"), equalTo("routing_value"));
         assertThat(doc.rootDoc().get("field"), equalTo("value"));
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java
index 3556cea23adf8..ed29a41f4371e 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java
@@ -25,15 +25,8 @@
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.IndexService;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.DocumentMapperParser;
-import org.elasticsearch.index.mapper.FieldMapper;
-import org.elasticsearch.index.mapper.MapperParsingException;
-import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
@@ -41,7 +34,9 @@
 import org.junit.Before;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.Collection;
+import java.util.List;
 
 import static com.carrotsearch.randomizedtesting.RandomizedTest.getRandom;
 import static org.hamcrest.Matchers.containsString;
@@ -52,7 +47,7 @@ public class ScaledFloatFieldMapperTests extends ESSingleNodeTestCase {
     DocumentMapperParser parser;
 
     @Before
-    public void before() {
+    public void setup() {
         indexService = createIndex("test");
         parser = indexService.mapperService().documentMapperParser();
     }
@@ -228,10 +223,19 @@ public void testCoerce() throws Exception {
     }
 
     public void testIgnoreMalformed() throws Exception {
+        doTestIgnoreMalformed("a", "For input string: \"a\"");
+
+        List<String> values = Arrays.asList("NaN", "Infinity", "-Infinity");
+        for (String value : values) {
+            doTestIgnoreMalformed(value, "[scaled_float] only supports finite values, but got [" + value + "]");
+        }
+    }
+
+    private void doTestIgnoreMalformed(String value, String exceptionMessageContains) throws Exception {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("properties").startObject("field").field("type", "scaled_float")
-                .field("scaling_factor", 10.0).endObject().endObject()
-                .endObject().endObject().string();
+            .startObject("properties").startObject("field").field("type", "scaled_float")
+            .field("scaling_factor", 10.0).endObject().endObject()
+            .endObject().endObject().string();
 
         DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
 
@@ -239,22 +243,22 @@ public void testIgnoreMalformed() throws Exception {
 
         ThrowingRunnable runnable = () -> mapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
                 .startObject()
-                .field("field", "a")
+                .field("field", value)
                 .endObject()
                 .bytes());
         MapperParsingException e = expectThrows(MapperParsingException.class, runnable);
-        assertThat(e.getCause().getMessage(), containsString("For input string: \"a\""));
+        assertThat(e.getCause().getMessage(), containsString(exceptionMessageContains));
 
         mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("properties").startObject("field").field("type", "scaled_float")
-                .field("scaling_factor", 10.0).field("ignore_malformed", true).endObject().endObject()
-                .endObject().endObject().string();
+            .startObject("properties").startObject("field").field("type", "scaled_float")
+            .field("scaling_factor", 10.0).field("ignore_malformed", true).endObject().endObject()
+            .endObject().endObject().string();
 
         DocumentMapper mapper2 = parser.parse("type", new CompressedXContent(mapping));
 
         ParsedDocument doc = mapper2.parse("test", "type", "1", XContentFactory.jsonBuilder()
                 .startObject()
-                .field("field", "a")
+                .field("field", value)
                 .endObject()
                 .bytes());
 
@@ -300,6 +304,8 @@ public void testIncludeInAll() throws Exception {
 
         fields = doc.rootDoc().getFields("_all");
         assertEquals(0, fields.length);
+        assertWarnings("field [include_in_all] is deprecated, as [_all] is deprecated, and will be disallowed" +
+                        " in 6.0, use [copy_to] instead.");
     }
 
     public void testNullValue() throws IOException {
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldTypeTests.java
index 59571b702318d..dd66421986779 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldTypeTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldTypeTests.java
@@ -23,6 +23,7 @@
 import org.apache.lucene.document.DoublePoint;
 import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.document.SortedNumericDocValuesField;
+import org.apache.lucene.document.StoredField;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
@@ -38,9 +39,6 @@
 import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
-import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.mapper.NumberFieldMapper;
-import org.elasticsearch.index.mapper.ScaledFloatFieldMapper;
 import org.junit.Before;
 
 import java.io.IOException;
@@ -121,8 +119,8 @@ public void testRangeQuery() throws IOException {
             Double u = randomBoolean() ? null : (randomDouble() * 2 - 1) * 10000;
             boolean includeLower = randomBoolean();
             boolean includeUpper = randomBoolean();
-            Query doubleQ = NumberFieldMapper.NumberType.DOUBLE.rangeQuery("double", l, u, includeLower, includeUpper);
-            Query scaledFloatQ = ft.rangeQuery(l, u, includeLower, includeUpper);
+            Query doubleQ = NumberFieldMapper.NumberType.DOUBLE.rangeQuery("double", l, u, includeLower, includeUpper, false);
+            Query scaledFloatQ = ft.rangeQuery(l, u, includeLower, includeUpper, null);
             assertEquals(searcher.count(doubleQ), searcher.count(scaledFloatQ));
         }
         IOUtils.close(reader, dir);
@@ -132,8 +130,8 @@ public void testValueForSearch() {
         ScaledFloatFieldMapper.ScaledFloatFieldType ft = new ScaledFloatFieldMapper.ScaledFloatFieldType();
         ft.setName("scaled_float");
         ft.setScalingFactor(0.1 + randomDouble() * 100);
-        assertNull(ft.valueForSearch(null));
-        assertEquals(10/ft.getScalingFactor(), ft.valueForSearch(10L));
+        assertNull(ft.valueForDisplay(null));
+        assertEquals(10/ft.getScalingFactor(), ft.valueForDisplay(10L));
     }
 
     public void testStats() throws IOException {
@@ -146,6 +144,15 @@ public void testStats() throws IOException {
             assertNull(ft.stats(reader));
         }
         Document doc = new Document();
+        doc.add(new StoredField("scaled_float", -1));
+        w.addDocument(doc);
+        try (DirectoryReader reader = DirectoryReader.open(w)) {
+            // field exists, but has no point values
+            FieldStats<?> stats = ft.stats(reader);
+            assertFalse(stats.hasMinMax());
+            assertNull(stats.getMinValue());
+            assertNull(stats.getMaxValue());
+        }
         LongPoint point = new LongPoint("scaled_float", -1);
         doc.add(point);
         w.addDocument(doc);
@@ -155,7 +162,7 @@ public void testStats() throws IOException {
             FieldStats<?> stats = ft.stats(reader);
             assertEquals(-1/ft.getScalingFactor(), stats.getMinValue());
             assertEquals(10/ft.getScalingFactor(), stats.getMaxValue());
-            assertEquals(2, stats.getMaxDoc());
+            assertEquals(3, stats.getMaxDoc());
         }
         w.deleteAll();
         try (DirectoryReader reader = DirectoryReader.open(w)) {
@@ -185,6 +192,7 @@ public void testFieldData() throws IOException {
             // single-valued
             ft.setName("scaled_float1");
             IndexNumericFieldData fielddata = (IndexNumericFieldData) ft.fielddataBuilder().build(indexSettings, ft, null, null, null);
+            assertEquals(fielddata.getNumericType(), IndexNumericFieldData.NumericType.DOUBLE);
             AtomicNumericFieldData leafFieldData = fielddata.load(reader.leaves().get(0));
             SortedNumericDoubleValues values = leafFieldData.getDoubleValues();
             values.setDocument(0);
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java
index 169a7e1d90a47..d6d9cb927cd3d 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java
@@ -28,11 +28,7 @@
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.DocumentMapperParser;
-import org.elasticsearch.index.mapper.MapperParsingException;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.ParsedDocument;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
@@ -87,7 +83,7 @@ public void testFormatBackCompat() throws Exception {
 
     public void testIncludes() throws Exception {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-            .startObject("_source").field("includes", new String[]{"path1*"}).endObject()
+            .startObject("_source").array("includes", new String[]{"path1*"}).endObject()
             .endObject().endObject().string();
 
         DocumentMapper documentMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
@@ -99,7 +95,7 @@ public void testIncludes() throws Exception {
 
         IndexableField sourceField = doc.rootDoc().getField("_source");
         Map<String, Object> sourceAsMap;
-        try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(new BytesArray(sourceField.binaryValue()))) {
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, new BytesArray(sourceField.binaryValue()))) {
             sourceAsMap = parser.map();
         }
         assertThat(sourceAsMap.containsKey("path1"), equalTo(true));
@@ -108,7 +104,7 @@ public void testIncludes() throws Exception {
 
     public void testExcludes() throws Exception {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-            .startObject("_source").field("excludes", new String[]{"path1*"}).endObject()
+            .startObject("_source").array("excludes", new String[]{"path1*"}).endObject()
             .endObject().endObject().string();
 
         DocumentMapper documentMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
@@ -120,7 +116,7 @@ public void testExcludes() throws Exception {
 
         IndexableField sourceField = doc.rootDoc().getField("_source");
         Map<String, Object> sourceAsMap;
-        try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(new BytesArray(sourceField.binaryValue()))) {
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, new BytesArray(sourceField.binaryValue()))) {
             sourceAsMap = parser.map();
         }
         assertThat(sourceAsMap.containsKey("path1"), equalTo(false));
@@ -164,7 +160,8 @@ public void testDefaultMappingAndWithMappingOverride() throws Exception {
                 .startObject("_source").field("enabled", true).endObject()
                 .endObject().endObject().string();
 
-        DocumentMapper mapper = createIndex("test").mapperService().documentMapperParser().parse("my_type", new CompressedXContent(mapping), defaultMapping);
+        DocumentMapper mapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("my_type", new CompressedXContent(mapping), defaultMapping);
         assertThat(mapper.type(), equalTo("my_type"));
         assertThat(mapper.sourceMapper().enabled(), equalTo(true));
     }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/StringMappingUpgradeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/StringMappingUpgradeTests.java
index 311bf0205ed40..14dc9ca81828f 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/StringMappingUpgradeTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/StringMappingUpgradeTests.java
@@ -19,8 +19,6 @@
 
 package org.elasticsearch.index.mapper;
 
-import com.carrotsearch.randomizedtesting.generators.RandomPicks;
-
 import org.apache.lucene.index.IndexOptions;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
@@ -30,24 +28,17 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.IndexService;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.DocumentMapperParser;
-import org.elasticsearch.index.mapper.FieldMapper;
-import org.elasticsearch.index.mapper.KeywordFieldMapper;
-import org.elasticsearch.index.mapper.Mapper;
-import org.elasticsearch.index.mapper.ParsedDocument;
-import org.elasticsearch.index.mapper.StringFieldMapper;
-import org.elasticsearch.index.mapper.TextFieldMapper;
 import org.elasticsearch.index.mapper.TextFieldMapper.TextFieldType;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
 
 import java.io.IOException;
-import java.util.Arrays;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import static org.hamcrest.Matchers.containsString;
@@ -69,6 +60,7 @@ public void testUpgradeDefaults() throws IOException {
         DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
         FieldMapper field = mapper.mappers().getMapper("field");
         assertThat(field, instanceOf(TextFieldMapper.class));
+        assertWarnings("The [string] field is deprecated, please use [text] or [keyword] instead on [field]");
     }
 
     public void testUpgradeAnalyzedString() throws IOException {
@@ -80,6 +72,7 @@ public void testUpgradeAnalyzedString() throws IOException {
         DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
         FieldMapper field = mapper.mappers().getMapper("field");
         assertThat(field, instanceOf(TextFieldMapper.class));
+        assertWarnings("The [string] field is deprecated, please use [text] or [keyword] instead on [field]");
     }
 
     public void testUpgradeNotAnalyzedString() throws IOException {
@@ -92,6 +85,7 @@ public void testUpgradeNotAnalyzedString() throws IOException {
         DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
         FieldMapper field = mapper.mappers().getMapper("field");
         assertThat(field, instanceOf(KeywordFieldMapper.class));
+        assertWarnings("The [string] field is deprecated, please use [text] or [keyword] instead on [field]");
     }
 
     public void testUpgradeNotIndexedString() throws IOException {
@@ -104,6 +98,7 @@ public void testUpgradeNotIndexedString() throws IOException {
         FieldMapper field = mapper.mappers().getMapper("field");
         assertThat(field, instanceOf(KeywordFieldMapper.class));
         assertEquals(IndexOptions.NONE, field.fieldType().indexOptions());
+        assertWarnings("The [string] field is deprecated, please use [text] or [keyword] instead on [field]");
     }
 
     public void testUpgradeIndexOptions() throws IOException {
@@ -117,6 +112,7 @@ public void testUpgradeIndexOptions() throws IOException {
         FieldMapper field = mapper.mappers().getMapper("field");
         assertThat(field, instanceOf(TextFieldMapper.class));
         assertEquals(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, field.fieldType().indexOptions());
+        assertWarnings("The [string] field is deprecated, please use [text] or [keyword] instead on [field]");
     }
 
     public void testUpgradePositionGap() throws IOException {
@@ -130,6 +126,7 @@ public void testUpgradePositionGap() throws IOException {
         FieldMapper field = mapper.mappers().getMapper("field");
         assertThat(field, instanceOf(TextFieldMapper.class));
         assertEquals(42, field.fieldType().indexAnalyzer().getPositionIncrementGap("field"));
+        assertWarnings("The [string] field is deprecated, please use [text] or [keyword] instead on [field]");
     }
 
     public void testIllegalIndexValue() throws IOException {
@@ -195,6 +192,7 @@ public void testUpgradeFielddataSettings() throws IOException {
             assertEquals(Integer.MAX_VALUE, fieldType.fielddataMaxFrequency(), 0d);
         }
         assertEquals("eager_global_ordinals".equals(loading), field.fieldType().eagerGlobalOrdinals());
+        assertWarnings("The [string] field is deprecated, please use [text] or [keyword] instead on [field]");
     }
 
     public void testUpgradeIgnoreAbove() throws IOException {
@@ -208,6 +206,7 @@ public void testUpgradeIgnoreAbove() throws IOException {
         FieldMapper field = mapper.mappers().getMapper("field");
         assertThat(field, instanceOf(KeywordFieldMapper.class));
         assertEquals(200, ((KeywordFieldMapper) field).ignoreAbove());
+        assertWarnings("The [string] field is deprecated, please use [text] or [keyword] instead on [field]");
     }
 
     public void testUpgradeAnalyzer() throws IOException {
@@ -225,6 +224,7 @@ public void testUpgradeAnalyzer() throws IOException {
         assertEquals("standard", field.fieldType().indexAnalyzer().name());
         assertEquals("whitespace", field.fieldType().searchAnalyzer().name());
         assertEquals("keyword", field.fieldType().searchQuoteAnalyzer().name());
+        assertWarnings("The [string] field is deprecated, please use [text] or [keyword] instead on [field]");
     }
 
     public void testUpgradeTextIncludeInAll() throws IOException {
@@ -238,6 +238,9 @@ public void testUpgradeTextIncludeInAll() throws IOException {
         FieldMapper field = mapper.mappers().getMapper("field");
         assertThat(field, instanceOf(TextFieldMapper.class));
         assertFalse(((TextFieldMapper) field).includeInAll());
+        assertWarnings("The [string] field is deprecated, please use [text] or [keyword] instead on [field]",
+                "field [include_in_all] is deprecated, as [_all] is deprecated, and will be disallowed " +
+                        "in 6.0, use [copy_to] instead.");
     }
 
     public void testUpgradeKeywordIncludeInAll() throws IOException {
@@ -251,16 +254,23 @@ public void testUpgradeKeywordIncludeInAll() throws IOException {
         FieldMapper field = mapper.mappers().getMapper("field");
         assertThat(field, instanceOf(KeywordFieldMapper.class));
         assertTrue(((KeywordFieldMapper) field).includeInAll());
+        assertWarnings("The [string] field is deprecated, please use [text] or [keyword] instead on [field]",
+                "field [include_in_all] is deprecated, as [_all] is deprecated, and will be disallowed" +
+                        " in 6.0, use [copy_to] instead.");
     }
 
     public void testUpgradeRandomMapping() throws IOException {
         final int iters = 20;
         for (int i = 0; i < iters; ++i) {
-            doTestUpgradeRandomMapping(i);
+            List<String> warnings = doTestUpgradeRandomMapping(i);
+            if (warnings.isEmpty() == false) {
+                assertWarnings(warnings.toArray(new String[warnings.size()]));
+            }
         }
     }
 
-    private void doTestUpgradeRandomMapping(int iter) throws IOException {
+    private List<String> doTestUpgradeRandomMapping(int iter) throws IOException {
+        List<String> warnings = new ArrayList<>();
         IndexService indexService;
         boolean oldIndex = randomBoolean();
         String indexName = "test" + iter;
@@ -283,8 +293,17 @@ private void doTestUpgradeRandomMapping(int iter) throws IOException {
         } else if (randomBoolean()) {
             mapping.field("index", "analyzed");
         }
+        //TODO why don't we ever have warnings about index not being a boolean?
+        //e.g. "Expected a boolean for property [index] but got [no]" or "Expected a boolean for property [index] but got [not_analyzed]"
         if (randomBoolean()) {
-            mapping.field("store", RandomPicks.randomFrom(random(), Arrays.asList("yes", "no", true, false)));
+            Object store;
+            if (randomBoolean()) {
+                store = randomFrom("yes", "no");
+                warnings.add("Expected a boolean [true/false] for property [field.store] but got [" + store + "]");
+            } else {
+                store = randomFrom(true, false);
+            }
+            mapping.field("store", store);
         }
         if (keyword && randomBoolean()) {
             mapping.field("doc_values", randomBoolean());
@@ -293,14 +312,23 @@ private void doTestUpgradeRandomMapping(int iter) throws IOException {
             mapping.field("analyzer", "keyword");
         }
         if (randomBoolean()) {
+            //TODO does it make sense that the norms warnings are emitted only for old indices?
             hasNorms = randomBoolean();
             if (randomBoolean()) {
                 mapping.field("omit_norms", hasNorms == false);
+                if (oldIndex) {
+                    warnings.add("[omit_norms] is deprecated, please use [norms] instead with the opposite boolean value");
+                }
             } else {
                 mapping.field("norms", Collections.singletonMap("enabled", hasNorms));
+                if (oldIndex) {
+                    warnings.add("The [norms{enabled:true/false}] way of specifying norms is deprecated, " +
+                            "please use [norms:true/false] instead");
+                }
             }
         }
         if (randomBoolean()) {
+            //TODO fielddata and frequency filter are randomized but never used throughout the test
             Map<String, Object> fielddata = new HashMap<>();
             if (randomBoolean()) {
                 fielddata.put("format", randomFrom("paged_bytes", "disabled"));
@@ -336,20 +364,25 @@ private void doTestUpgradeRandomMapping(int iter) throws IOException {
             IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
                     () -> parser.parse("type", new CompressedXContent(mapping.string())));
             assertThat(e.getMessage(), containsString("The [string] type is removed in 5.0"));
+            //no deprecation warnings, we fail fast in this case, before any warning could be emitted.
+            return Collections.emptyList();
+        }
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping.string()));
+        FieldMapper field = mapper.mappers().getMapper("field");
+        if (oldIndex) {
+            assertThat(field, instanceOf(StringFieldMapper.class));
         } else {
-            DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping.string()));
-            FieldMapper field = mapper.mappers().getMapper("field");
-            if (oldIndex) {
-                assertThat(field, instanceOf(StringFieldMapper.class));
-            } else if (keyword) {
+            warnings.add("The [string] field is deprecated, please use [text] or [keyword] instead on [field]");
+            if (keyword) {
                 assertThat(field, instanceOf(KeywordFieldMapper.class));
             } else {
                 assertThat(field, instanceOf(TextFieldMapper.class));
             }
-            if (field.fieldType().indexOptions() != IndexOptions.NONE) {
-                assertEquals(hasNorms, field.fieldType().omitNorms() == false);
-            }
         }
+        if (field.fieldType().indexOptions() != IndexOptions.NONE) {
+            assertEquals(hasNorms, field.fieldType().omitNorms() == false);
+        }
+        return warnings;
     }
 
     public void testUpgradeTemplateWithDynamicType() throws IOException {
@@ -419,6 +452,7 @@ public void testUpgradeTemplateWithDynamicTypeKeyword() throws IOException {
         ParsedDocument doc = mapper.parse("test", "type", "id", source);
         Mapper fooMapper = doc.dynamicMappingsUpdate().root().getMapper("foo");
         assertThat(fooMapper, instanceOf(KeywordFieldMapper.class));
+        assertWarnings("Expected a boolean [true/false] for property [index] but got [not_analyzed]");
     }
 
     public void testUpgradeTemplateWithDynamicTypeKeyword2() throws IOException {
@@ -442,5 +476,7 @@ public void testUpgradeTemplateWithDynamicTypeKeyword2() throws IOException {
         ParsedDocument doc = mapper.parse("test", "type", "id", source);
         Mapper fooMapper = doc.dynamicMappingsUpdate().root().getMapper("foo");
         assertThat(fooMapper, instanceOf(KeywordFieldMapper.class));
+        assertWarnings("Expected a boolean [true/false] for property [index] but got [not_analyzed]");
     }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/TTLFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/TTLFieldMapperTests.java
index bf51be3c2d438..cff4be241ee7a 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/TTLFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/TTLFieldMapperTests.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.mapper.DocumentMapper;
@@ -67,9 +68,9 @@ public void testRejectedOn5x() throws IOException {
                     .endObject()
                 .endObject().endObject().string();
         IndexService index = createIndex("test");
-        IllegalArgumentException expected = expectThrows(IllegalArgumentException.class,
+        MapperParsingException expected = expectThrows(MapperParsingException.class,
                 () -> index.mapperService().merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE, false));
-        assertThat(expected.getMessage(), startsWith("[_ttl] is removed"));
+        assertThat(expected.getMessage(), containsString("[_ttl] is removed"));
     }
 
     public void testSimpleDisabled() throws Exception {
@@ -80,14 +81,14 @@ public void testSimpleDisabled() throws Exception {
                 .field("field", "value")
                 .endObject()
                 .bytes();
-        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", source).ttl(Long.MAX_VALUE));
+        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", source, XContentType.JSON).ttl(Long.MAX_VALUE));
 
         assertThat(doc.rootDoc().getField("_ttl"), equalTo(null));
     }
 
     public void testEnabled() throws Exception {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("_ttl").field("enabled", "yes").endObject()
+                .startObject("_ttl").field("enabled", "true").endObject()
                 .endObject().endObject().string();
         DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
         BytesReference source = XContentFactory.jsonBuilder()
@@ -95,7 +96,7 @@ public void testEnabled() throws Exception {
                 .field("field", "value")
                 .endObject()
                 .bytes();
-        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", source).ttl(Long.MAX_VALUE));
+        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", source, XContentType.JSON).ttl(Long.MAX_VALUE));
 
         assertThat(doc.rootDoc().getField("_ttl").fieldType().stored(), equalTo(true));
         assertNotSame(IndexOptions.NONE, doc.rootDoc().getField("_ttl").fieldType().indexOptions());
@@ -117,7 +118,7 @@ public void testThatEnablingTTLFieldOnMergeWorks() throws Exception {
 
         String mappingWithTtl = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("_ttl")
-                .field("enabled", "yes")
+                .field("enabled", "true")
                 .endObject()
                 .startObject("properties").field("field").startObject().field("type", "text").endObject().endObject()
                 .endObject().endObject().string();
@@ -133,7 +134,7 @@ public void testThatEnablingTTLFieldOnMergeWorks() throws Exception {
     public void testThatChangingTTLKeepsMapperEnabled() throws Exception {
         String mappingWithTtl = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("_ttl")
-                .field("enabled", "yes")
+                .field("enabled", "true")
                 .endObject()
                 .startObject("properties").field("field").startObject().field("type", "text").endObject().endObject()
                 .endObject().endObject().string();
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java
index af24e16b54b4f..83e00f8ffe042 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java
@@ -63,7 +63,7 @@ public class TextFieldMapperTests extends ESSingleNodeTestCase {
     DocumentMapperParser parser;
 
     @Before
-    public void before() {
+    public void setup() {
         indexService = createIndex("test");
         parser = indexService.mapperService().documentMapperParser();
     }
@@ -208,7 +208,7 @@ public void testDefaultPositionIncrementGap() throws IOException {
 
         ParsedDocument doc = mapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
                 .startObject()
-                .field("field", new String[] {"a", "b"})
+                .array("field", new String[] {"a", "b"})
                 .endObject()
                 .bytes());
 
@@ -219,7 +219,7 @@ public void testDefaultPositionIncrementGap() throws IOException {
         assertEquals("b", fields[1].stringValue());
 
         IndexShard shard = indexService.getShard(0);
-        shard.index(new Engine.Index(new Term("_uid", "1"), doc));
+        shard.index(new Engine.Index(new Term("_uid", Uid.createUid(doc.type(), doc.id())), doc));
         shard.refresh("test");
         try (Engine.Searcher searcher = shard.acquireSearcher("test")) {
             LeafReader leaf = searcher.getDirectoryReader().leaves().get(0).reader();
@@ -247,7 +247,7 @@ public void testPositionIncrementGap() throws IOException {
 
         ParsedDocument doc = mapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
                 .startObject()
-                .field("field", new String[] {"a", "b"})
+                .array("field", new String[] {"a", "b"})
                 .endObject()
                 .bytes());
 
@@ -258,7 +258,7 @@ public void testPositionIncrementGap() throws IOException {
         assertEquals("b", fields[1].stringValue());
 
         IndexShard shard = indexService.getShard(0);
-        shard.index(new Engine.Index(new Term("_uid", "1"), doc));
+        shard.index(new Engine.Index(new Term("_uid", Uid.createUid(doc.type(), doc.id())), doc));
         shard.refresh("test");
         try (Engine.Searcher searcher = shard.acquireSearcher("test")) {
             LeafReader leaf = searcher.getDirectoryReader().leaves().get(0).reader();
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java
index 70103111e5564..895bb97e16665 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java
@@ -18,15 +18,18 @@
  */
 package org.elasticsearch.index.mapper;
 
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.List;
 
 import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.queries.TermsQuery;
+import org.apache.lucene.search.TermInSetQuery;
 import org.apache.lucene.search.FuzzyQuery;
 import org.apache.lucene.search.RegexpQuery;
 import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.TextFieldMapper;
@@ -86,7 +89,10 @@ public void testTermsQuery() {
         MappedFieldType ft = createDefaultFieldType();
         ft.setName("field");
         ft.setIndexOptions(IndexOptions.DOCS);
-        assertEquals(new TermsQuery(new Term("field", "foo"), new Term("field", "bar")),
+        List<BytesRef> terms = new ArrayList<>();
+        terms.add(new BytesRef("foo"));
+        terms.add(new BytesRef("bar"));
+        assertEquals(new TermInSetQuery("field", terms),
                 ft.termsQuery(Arrays.asList("foo", "bar"), null));
 
         ft.setIndexOptions(IndexOptions.NONE);
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/TimestampFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/TimestampFieldMapperTests.java
index 2b87bb10b544b..b8cf7b2250562 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/TimestampFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/TimestampFieldMapperTests.java
@@ -30,20 +30,12 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.joda.Joda;
-import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexService;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.DocumentMapperParser;
-import org.elasticsearch.index.mapper.MapperParsingException;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.ParsedDocument;
-import org.elasticsearch.index.mapper.SourceToParse;
-import org.elasticsearch.index.mapper.TimestampFieldMapper;
 import org.elasticsearch.index.mapper.MapperService.MergeReason;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
@@ -57,7 +49,6 @@
 import java.util.LinkedHashMap;
 
 import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath;
-import static org.elasticsearch.test.VersionUtils.randomVersion;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -86,9 +77,9 @@ public void testRejectedOn5x() throws IOException {
                     .endObject()
                 .endObject().endObject().string();
         IndexService index = createIndex("test");
-        IllegalArgumentException expected = expectThrows(IllegalArgumentException.class,
+        MapperParsingException expected = expectThrows(MapperParsingException.class,
                 () -> index.mapperService().merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE, false));
-        assertThat(expected.getMessage(), startsWith("[_timestamp] is removed"));
+        assertThat(expected.getMessage(), containsString("[_timestamp] is removed"));
     }
 
     public void testSimpleDisabled() throws Exception {
@@ -99,14 +90,14 @@ public void testSimpleDisabled() throws Exception {
                 .field("field", "value")
                 .endObject()
                 .bytes();
-        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", source).timestamp(1));
+        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", source, XContentType.JSON).timestamp(1));
 
         assertThat(doc.rootDoc().getField("_timestamp"), equalTo(null));
     }
 
     public void testEnabled() throws Exception {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("_timestamp").field("enabled", "yes").endObject()
+                .startObject("_timestamp").field("enabled", "true").endObject()
                 .endObject().endObject().string();
         DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
         BytesReference source = XContentFactory.jsonBuilder()
@@ -114,7 +105,7 @@ public void testEnabled() throws Exception {
                 .field("field", "value")
                 .endObject()
                 .bytes();
-        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", source).timestamp(1));
+        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", source, XContentType.JSON).timestamp(1));
 
         assertThat(doc.rootDoc().getField("_timestamp").fieldType().stored(), equalTo(true));
         assertNotSame(IndexOptions.NONE, doc.rootDoc().getField("_timestamp").fieldType().indexOptions());
@@ -156,7 +147,7 @@ public void testThatDisablingDuringMergeIsWorking() throws Exception {
     public void testTimestampMissingDefaultToEpochValue() throws Exception {
         XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("_timestamp")
-                    .field("enabled", "yes")
+                    .field("enabled", "true")
                     .field("default", "1970-01-01")
                     .field("format", "YYYY-MM-dd")
                 .endObject()
@@ -181,7 +172,7 @@ public void testTimestampMissingDefaultToEpochValue() throws Exception {
     public void testTimestampMissingNowDefaultValue() throws Exception {
         XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("_timestamp")
-                    .field("enabled", "yes")
+                    .field("enabled", "true")
                     .field("default", "now")
                     .field("format", "YYYY-MM-dd")
                 .endObject()
@@ -209,59 +200,50 @@ public void testTimestampMissingNowDefaultValue() throws Exception {
     public void testPathMissingWithForcedNullDefaultShouldFail() throws Exception {
         XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("_timestamp")
-                    .field("enabled", "yes")
+                    .field("enabled", "true")
                     .field("path", "timestamp")
                     .field("default", (String) null)
                 .endObject()
                 .endObject().endObject();
-        try {
-            createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string()));
-            fail("we should reject the mapping with a TimestampParsingException: default timestamp can not be set to null");
-        } catch (TimestampParsingException e) {
-            assertThat(e.getDetailedMessage(), containsString("default timestamp can not be set to null"));
-        }
+        TimestampParsingException e = expectThrows(TimestampParsingException.class, () -> createIndex("test", BW_SETTINGS).mapperService()
+                .documentMapperParser().parse("type", new CompressedXContent(mapping.string())));
+        assertThat(e.getDetailedMessage(), containsString("default timestamp can not be set to null"));
     }
 
     // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null]
     public void testTimestampMissingWithForcedNullDefaultShouldFail() throws Exception {
         XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("_timestamp")
-                    .field("enabled", "yes")
+                    .field("enabled", "true")
                     .field("default", (String) null)
                 .endObject()
                 .endObject().endObject();
 
-        try {
-            createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string()));
-            fail("we should reject the mapping with a TimestampParsingException: default timestamp can not be set to null");
-        } catch (TimestampParsingException e) {
-            assertThat(e.getDetailedMessage(), containsString("default timestamp can not be set to null"));
-        }
+        TimestampParsingException e = expectThrows(TimestampParsingException.class, () -> createIndex("test", BW_SETTINGS).mapperService()
+                .documentMapperParser().parse("type", new CompressedXContent(mapping.string())));
+        assertThat(e.getDetailedMessage(), containsString("default timestamp can not be set to null"));
     }
 
     // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null]
     public void testTimestampDefaultAndIgnore() throws Exception {
         XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("_timestamp")
-                    .field("enabled", "yes")
+                    .field("enabled", "true")
                     .field("default", "1971-12-26")
                     .field("ignore_missing", false)
                 .endObject()
                 .endObject().endObject();
 
-        try {
-            createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string()));
-            fail("we should reject the mapping with a TimestampParsingException: default timestamp can not be set with ignore_missing set to false");
-        } catch (TimestampParsingException e) {
-            assertThat(e.getDetailedMessage(), containsString("default timestamp can not be set with ignore_missing set to false"));
-        }
+        TimestampParsingException e = expectThrows(TimestampParsingException.class, () -> createIndex("test", BW_SETTINGS).mapperService()
+                .documentMapperParser().parse("type", new CompressedXContent(mapping.string())));
+        assertThat(e.getDetailedMessage(), containsString("default timestamp can not be set with ignore_missing set to false"));
     }
 
     // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null]
     public void testTimestampMissingShouldNotFail() throws Exception {
         XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("_timestamp")
-                    .field("enabled", "yes")
+                    .field("enabled", "true")
                 .endObject()
                 .endObject().endObject();
         XContentBuilder doc = XContentFactory.jsonBuilder()
@@ -297,7 +279,7 @@ public void testDefaultTimestampStream() throws IOException {
             out.close();
             BytesReference bytes = out.bytes();
 
-            MappingMetaData metaData = MappingMetaData.PROTO.readFrom(bytes.streamInput());
+            MappingMetaData metaData = new MappingMetaData(bytes.streamInput());
 
             assertThat(metaData, is(expected));
         }
@@ -314,7 +296,7 @@ public void testDefaultTimestampStream() throws IOException {
             out.close();
             BytesReference bytes = out.bytes();
 
-            MappingMetaData metaData = MappingMetaData.PROTO.readFrom(bytes.streamInput());
+            MappingMetaData metaData = new MappingMetaData(bytes.streamInput());
 
             assertThat(metaData, is(expected));
         }
@@ -331,7 +313,7 @@ public void testDefaultTimestampStream() throws IOException {
             out.close();
             BytesReference bytes = out.bytes();
 
-            MappingMetaData metaData = MappingMetaData.PROTO.readFrom(bytes.streamInput());
+            MappingMetaData metaData = new MappingMetaData(bytes.streamInput());
 
             assertThat(metaData, is(expected));
         }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/TimestampFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/TimestampFieldTypeTests.java
index c17d60c8145db..53c0c89f8c392 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/TimestampFieldTypeTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/TimestampFieldTypeTests.java
@@ -18,10 +18,6 @@
  */
 package org.elasticsearch.index.mapper;
 
-import org.elasticsearch.index.mapper.DateFieldMapper;
-import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.mapper.TimestampFieldMapper;
-
 public class TimestampFieldTypeTests extends LegacyDateFieldTypeTests {
     @Override
     protected MappedFieldType createDefaultFieldType() {
@@ -33,6 +29,6 @@ public void testValueForSearch() {
         MappedFieldType ft = createDefaultFieldType();
         String date = "2015-10-12T12:09:55.000Z";
         long instant = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parser().parseDateTime(date).getMillis();
-        assertEquals(instant, ft.valueForSearch(instant));
+        assertEquals(instant, ft.valueForDisplay(instant));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/TokenCountFieldMapperIntegrationIT.java b/core/src/test/java/org/elasticsearch/index/mapper/TokenCountFieldMapperIntegrationIT.java
index a84a18d2aef19..825d99e4c16ab 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/TokenCountFieldMapperIntegrationIT.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/TokenCountFieldMapperIntegrationIT.java
@@ -131,6 +131,12 @@ private void init() throws IOException {
                                     .field("analyzer", "standard")
                                     .field("doc_values", true)
                                 .endObject()
+                                .startObject("token_count_without_position_increments")
+                                    .field("type", "token_count")
+                                    .field("analyzer", "english")
+                                    .field("enable_position_increments", false)
+                                    .field("store", true)
+                                .endObject()
                             .endObject()
                         .endObject()
                     .endObject()
@@ -169,6 +175,7 @@ private SearchRequestBuilder searchByNumericRange(int low, int high) {
     private SearchRequestBuilder prepareSearch() {
         SearchRequestBuilder request = client().prepareSearch("test").setTypes("test");
         request.addStoredField("foo.token_count");
+        request.addStoredField("foo.token_count_without_position_increments");
         if (loadCountedFields) {
             request.addStoredField("foo");
         }
@@ -186,32 +193,38 @@ private void assertSearchReturns(SearchResponse result, String... ids) {
         for (SearchHit hit : result.getHits()) {
             String id = hit.id();
             if (id.equals("single")) {
-                assertSearchHit(hit, 4);
+                assertSearchHit(hit, new int[]{4}, new int[]{4});
             } else if (id.equals("bulk1")) {
-                assertSearchHit(hit, 3);
+                assertSearchHit(hit, new int[]{3}, new int[]{3});
             } else if (id.equals("bulk2")) {
-                assertSearchHit(hit, 5);
+                assertSearchHit(hit, new int[]{5}, new int[]{4});
             } else if (id.equals("multi")) {
-                assertSearchHit(hit, 2, 7);
+                assertSearchHit(hit, new int[]{2, 7}, new int[]{2, 7});
             } else if (id.equals("multibulk1")) {
-                assertSearchHit(hit, 1, 8);
+                assertSearchHit(hit, new int[]{1, 8}, new int[]{1, 8});
             } else if (id.equals("multibulk2")) {
-                assertSearchHit(hit, 6, 10);
+                assertSearchHit(hit, new int[]{6, 10}, new int[]{3, 9});
             } else {
                 throw new ElasticsearchException("Unexpected response!");
             }
         }
     }
 
-    private void assertSearchHit(SearchHit hit, int... termCounts) {
+    private void assertSearchHit(SearchHit hit, int[] standardTermCounts, int[] englishTermCounts) {
         assertThat(hit.field("foo.token_count"), not(nullValue()));
-        assertThat(hit.field("foo.token_count").values().size(), equalTo(termCounts.length));
-        for (int i = 0; i < termCounts.length; i++) {
-            assertThat((Integer) hit.field("foo.token_count").values().get(i), equalTo(termCounts[i]));
+        assertThat(hit.field("foo.token_count").getValues().size(), equalTo(standardTermCounts.length));
+        for (int i = 0; i < standardTermCounts.length; i++) {
+            assertThat((Integer) hit.field("foo.token_count").getValues().get(i), equalTo(standardTermCounts[i]));
+        }
+
+        assertThat(hit.field("foo.token_count_without_position_increments"), not(nullValue()));
+        assertThat(hit.field("foo.token_count_without_position_increments").getValues().size(), equalTo(englishTermCounts.length));
+        for (int i = 0; i < englishTermCounts.length; i++) {
+            assertThat((Integer) hit.field("foo.token_count_without_position_increments").getValues().get(i), equalTo(englishTermCounts[i]));
         }
 
         if (loadCountedFields && storeCountedFields) {
-            assertThat(hit.field("foo").values().size(), equalTo(termCounts.length));
+            assertThat(hit.field("foo").getValues().size(), equalTo(standardTermCounts.length));
         }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/TokenCountFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/TokenCountFieldMapperTests.java
index 835295def4fe0..c87ab86eba113 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/TokenCountFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/TokenCountFieldMapperTests.java
@@ -26,9 +26,11 @@
 import org.apache.lucene.analysis.TokenStream;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
@@ -80,26 +82,49 @@ public void testMerge() throws IOException {
         assertThat(((TokenCountFieldMapper) stage2.mappers().smartNameFieldMapper("tc")).analyzer(), equalTo("standard"));
     }
 
-    public void testCountPositions() throws IOException {
-        // We're looking to make sure that we:
-        Token t1 = new Token();      // Don't count tokens without an increment
+    /**
+     *  When position increments are counted, we're looking to make sure that we:
+        - don't count tokens without an increment
+        - count normal tokens with one increment
+        - count funny tokens with more than one increment
+        - count the final token increments on the rare token streams that have them
+     */
+    public void testCountPositionsWithIncrements() throws IOException {
+        Analyzer analyzer = createMockAnalyzer();
+        assertThat(TokenCountFieldMapper.countPositions(analyzer, "", "", true), equalTo(7));
+    }
+
+    /**
+     *  When position increments are not counted (only positions are counted), we're looking to make sure that we:
+        - don't count tokens without an increment
+        - count normal tokens with one increment
+        - count funny tokens with more than one increment as only one
+        - don't count the final token increments on the rare token streams that have them
+     */
+    public void testCountPositionsWithoutIncrements() throws IOException {
+        Analyzer analyzer = createMockAnalyzer();
+        assertThat(TokenCountFieldMapper.countPositions(analyzer, "", "", false), equalTo(2));
+    }
+
+    private Analyzer createMockAnalyzer() {
+        Token t1 = new Token();      // Token without an increment
         t1.setPositionIncrement(0);
         Token t2 = new Token();
-        t2.setPositionIncrement(1);  // Count normal tokens with one increment
+        t2.setPositionIncrement(1);  // Normal token with one increment
         Token t3 = new Token();
-        t2.setPositionIncrement(2);  // Count funny tokens with more than one increment
-        int finalTokenIncrement = 4; // Count the final token increment on the rare token streams that have them
+        t2.setPositionIncrement(2);  // Funny token with more than one increment
+        int finalTokenIncrement = 4; // Final token increment
         Token[] tokens = new Token[] {t1, t2, t3};
         Collections.shuffle(Arrays.asList(tokens), random());
         final TokenStream tokenStream = new CannedTokenStream(finalTokenIncrement, 0, tokens);
         // TODO: we have no CannedAnalyzer?
         Analyzer analyzer = new Analyzer() {
-                @Override
-                public TokenStreamComponents createComponents(String fieldName) {
-                    return new TokenStreamComponents(new MockTokenizer(), tokenStream);
-                }
-            };
-        assertThat(TokenCountFieldMapper.countPositions(analyzer, "", ""), equalTo(7));
+            @Override
+            public TokenStreamComponents createComponents(String fieldName) {
+                return new TokenStreamComponents(new MockTokenizer(), tokenStream);
+            }
+        };
+        return analyzer;
     }
 
     @Override
@@ -135,4 +160,55 @@ public void testEmptyName() throws IOException {
         DocumentMapper defaultMapper = parser2x.parse("type", new CompressedXContent(mapping));
         assertEquals(mapping, defaultMapper.mappingSource().string());
     }
+
+    public void testParseNullValue() throws Exception {
+        DocumentMapper mapper = createIndexWithTokenCountField();
+        ParseContext.Document doc = parseDocument(mapper, createDocument(null));
+        assertNull(doc.getField("test.tc"));
+    }
+
+    public void testParseEmptyValue() throws Exception {
+        DocumentMapper mapper = createIndexWithTokenCountField();
+        ParseContext.Document doc = parseDocument(mapper, createDocument(""));
+        assertEquals(0, doc.getField("test.tc").numericValue());
+    }
+
+    public void testParseNotNullValue() throws Exception {
+        DocumentMapper mapper = createIndexWithTokenCountField();
+        ParseContext.Document doc = parseDocument(mapper, createDocument("three tokens string"));
+        assertEquals(3, doc.getField("test.tc").numericValue());
+    }
+
+    private DocumentMapper createIndexWithTokenCountField() throws IOException {
+        final String content = XContentFactory.jsonBuilder().startObject()
+            .startObject("person")
+                .startObject("properties")
+                    .startObject("test")
+                        .field("type", "text")
+                        .startObject("fields")
+                            .startObject("tc")
+                                .field("type", "token_count")
+                                .field("analyzer", "standard")
+                            .endObject()
+                        .endObject()
+                    .endObject()
+                .endObject()
+            .endObject().endObject().string();
+
+        return createIndex("test").mapperService().documentMapperParser().parse("person", new CompressedXContent(content));
+    }
+
+    private SourceToParse createDocument(String fieldValue) throws Exception {
+        BytesReference request = XContentFactory.jsonBuilder()
+            .startObject()
+                .field("test", fieldValue)
+            .endObject().bytes();
+
+        return SourceToParse.source("test", "person", "1", request, XContentType.JSON);
+    }
+
+    private ParseContext.Document parseDocument(DocumentMapper mapper, SourceToParse request) {
+        return mapper.parse(request)
+            .docs().stream().findFirst().orElseThrow(() -> new IllegalStateException("Test object not parsed"));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/TypeFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/TypeFieldMapperTests.java
index 3573cfd8b60fe..b5632e52587c3 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/TypeFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/TypeFieldMapperTests.java
@@ -19,22 +19,36 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.DocValuesType;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData;
+import org.elasticsearch.index.fielddata.IndexFieldDataCache;
+import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
 import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.TypeFieldMapper;
+import org.elasticsearch.index.mapper.MapperService.MergeReason;
+import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
+import org.hamcrest.Matchers;
 
+import java.io.IOException;
+import java.util.Arrays;
 import java.util.Collection;
-
-import static org.hamcrest.Matchers.instanceOf;
+import java.util.Collections;
 
 public class TypeFieldMapperTests extends ESSingleNodeTestCase {
 
@@ -43,13 +57,12 @@ protected Collection<Class<? extends Plugin>> getPlugins() {
         return pluginList(InternalSettingsPlugin.class);
     }
 
-    public void testDocValues() throws Exception {
-        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string();
+    public void testDocValuesMultipleTypes() throws Exception {
+        testDocValues(false);
+    }
 
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
-        TypeFieldMapper typeMapper = docMapper.metadataMapper(TypeFieldMapper.class);
-        assertTrue(typeMapper.fieldType().hasDocValues());
-        assertThat(typeMapper.fieldType().fielddataBuilder(), instanceOf(DocValuesIndexFieldData.Builder.class));
+    public void testDocValuesSingleType() throws Exception {
+        testDocValues(true);
     }
 
     public void testDocValuesPre21() throws Exception {
@@ -60,6 +73,55 @@ public void testDocValuesPre21() throws Exception {
         DocumentMapper docMapper = createIndex("test", bwcSettings).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
         TypeFieldMapper typeMapper = docMapper.metadataMapper(TypeFieldMapper.class);
         assertFalse(typeMapper.fieldType().hasDocValues());
-        assertThat(typeMapper.fieldType().fielddataBuilder(), instanceOf(PagedBytesIndexFieldData.Builder.class));
+        assertThat(typeMapper.fieldType().fielddataBuilder(), Matchers.instanceOf(PagedBytesIndexFieldData.Builder.class));
+    }
+
+    public void testDocValues(boolean singleType) throws IOException {
+        Settings indexSettings = Settings.builder()
+                .put("index.mapping.single_type", singleType)
+                .build();
+        MapperService mapperService = createIndex("test", indexSettings).mapperService();
+        DocumentMapper mapper = mapperService.merge("type", new CompressedXContent("{\"type\":{}}"), MergeReason.MAPPING_UPDATE, false);
+        ParsedDocument document = mapper.parse(SourceToParse.source("index", "type", "id", new BytesArray("{}"), XContentType.JSON));
+
+        Directory dir = newDirectory();
+        IndexWriter w = new IndexWriter(dir, newIndexWriterConfig());
+        w.addDocument(document.rootDoc());
+        DirectoryReader r = DirectoryReader.open(w);
+        w.close();
+
+        MappedFieldType ft = mapperService.fullName(TypeFieldMapper.NAME);
+        IndexOrdinalsFieldData fd = (IndexOrdinalsFieldData) ft.fielddataBuilder().build(mapperService.getIndexSettings(),
+                ft, new IndexFieldDataCache.None(), new NoneCircuitBreakerService(), mapperService);
+        AtomicOrdinalsFieldData afd = fd.load(r.leaves().get(0));
+        SortedSetDocValues values = afd.getOrdinalsValues();
+        values.setDocument(0);
+        assertEquals(0, values.nextOrd());
+        assertEquals(SortedSetDocValues.NO_MORE_ORDS, values.nextOrd());
+        assertEquals(new BytesRef("type"), values.lookupOrd(0));
+        r.close();
+        dir.close();
+    }
+
+    public void testDefaultsMultipleTypes() throws IOException {
+        Settings indexSettings = Settings.builder()
+                .put("index.mapping.single_type", false)
+                .build();
+        MapperService mapperService = createIndex("test", indexSettings).mapperService();
+        DocumentMapper mapper = mapperService.merge("type", new CompressedXContent("{\"type\":{}}"), MergeReason.MAPPING_UPDATE, false);
+        ParsedDocument document = mapper.parse(SourceToParse.source("index", "type", "id", new BytesArray("{}"), XContentType.JSON));
+        IndexableField[] fields = document.rootDoc().getFields(TypeFieldMapper.NAME);
+        assertEquals(IndexOptions.DOCS, fields[0].fieldType().indexOptions());
+        assertEquals(DocValuesType.SORTED_SET, fields[1].fieldType().docValuesType());
+    }
+
+    public void testDefaultsSingleType() throws IOException {
+        Settings indexSettings = Settings.builder()
+                .put("index.mapping.single_type", true)
+                .build();
+        MapperService mapperService = createIndex("test", indexSettings).mapperService();
+        DocumentMapper mapper = mapperService.merge("type", new CompressedXContent("{\"type\":{}}"), MergeReason.MAPPING_UPDATE, false);
+        ParsedDocument document = mapper.parse(SourceToParse.source("index", "type", "id", new BytesArray("{}"), XContentType.JSON));
+        assertEquals(Collections.<IndexableField>emptyList(), Arrays.asList(document.rootDoc().getFields(TypeFieldMapper.NAME)));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/TypeFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/TypeFieldTypeTests.java
index 54217ea94ae53..b8a2805efe9bb 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/TypeFieldTypeTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/TypeFieldTypeTests.java
@@ -30,14 +30,25 @@
 import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.PhraseQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.IOUtils;
-import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.mapper.TypeFieldMapper;
-import org.junit.Before;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.lucene.search.Queries;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.mockito.Mockito;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Set;
 
 public class TypeFieldTypeTests extends FieldTypeTestCase {
     @Override
@@ -45,31 +56,62 @@ protected MappedFieldType createDefaultFieldType() {
         return new TypeFieldMapper.TypeFieldType();
     }
 
-    @Before
-    public void setupProperties() {
-        addModifier(new Modifier("fielddata", true) {
-            @Override
-            public void modify(MappedFieldType ft) {
-                TypeFieldMapper.TypeFieldType tft = (TypeFieldMapper.TypeFieldType) ft;
-                tft.setFielddata(tft.fielddata() == false);
-            }
-        });
+    public void testTermsQueryWhenTypesAreDisabled() throws Exception {
+        QueryShardContext context = Mockito.mock(QueryShardContext.class);
+        Settings indexSettings = Settings.builder()
+                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
+                .put("index.mapping.single_type", true).build();
+        IndexMetaData indexMetaData = IndexMetaData.builder(IndexMetaData.INDEX_UUID_NA_VALUE).settings(indexSettings).build();
+        IndexSettings mockSettings = new IndexSettings(indexMetaData, Settings.EMPTY);
+        Mockito.when(context.getIndexSettings()).thenReturn(mockSettings);
+
+        MapperService mapperService = Mockito.mock(MapperService.class);
+        Set<String> types = Collections.emptySet();
+        Mockito.when(mapperService.types()).thenReturn(types);
+        Mockito.when(context.getMapperService()).thenReturn(mapperService);
+
+        TypeFieldMapper.TypeFieldType ft = new TypeFieldMapper.TypeFieldType();
+        ft.setName(TypeFieldMapper.NAME);
+        Query query = ft.termQuery("my_type", context);
+        assertEquals(new MatchNoDocsQuery(), query);
+
+        types = Collections.singleton("my_type");
+        Mockito.when(mapperService.types()).thenReturn(types);
+        query = ft.termQuery("my_type", context);
+        assertEquals(new MatchAllDocsQuery(), query);
+
+        Mockito.when(mapperService.hasNested()).thenReturn(true);
+        query = ft.termQuery("my_type", context);
+        assertEquals(Queries.newNonNestedFilter(), query);
+
+        types = Collections.singleton("other_type");
+        Mockito.when(mapperService.types()).thenReturn(types);
+        query = ft.termQuery("my_type", context);
+        assertEquals(new MatchNoDocsQuery(), query);
     }
 
-    public void testTermQuery() throws Exception {
+    public void testTermsQueryWhenTypesAreEnabled() throws Exception {
         Directory dir = newDirectory();
         IndexWriter w = new IndexWriter(dir, newIndexWriterConfig());
-        Document doc = new Document();
-        StringField type = new StringField(TypeFieldMapper.NAME, "my_type", Store.NO);
-        doc.add(type);
-        w.addDocument(doc);
-        w.addDocument(doc);
-        IndexReader reader = DirectoryReader.open(w);
+        IndexReader reader = openReaderWithNewType("my_type", w);
+
+        QueryShardContext context = Mockito.mock(QueryShardContext.class);
+        Settings indexSettings = Settings.builder()
+                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
+                .put("index.mapping.single_type", false).build();
+        IndexMetaData indexMetaData = IndexMetaData.builder(IndexMetaData.INDEX_UUID_NA_VALUE).settings(indexSettings).build();
+        IndexSettings mockSettings = new IndexSettings(indexMetaData, Settings.EMPTY);
+        Mockito.when(context.getIndexSettings()).thenReturn(mockSettings);
 
         TypeFieldMapper.TypeFieldType ft = new TypeFieldMapper.TypeFieldType();
         ft.setName(TypeFieldMapper.NAME);
-        Query query = ft.termQuery("my_type", null);
-
+        Query query = ft.termQuery("my_type", context);
         assertEquals(new MatchAllDocsQuery(), query.rewrite(reader));
 
         // Make sure that Lucene actually simplifies the query when there is a single type
@@ -78,13 +120,55 @@ public void testTermQuery() throws Exception {
         Query rewritten = new IndexSearcher(reader).rewrite(filteredQuery);
         assertEquals(userQuery, rewritten);
 
-        type.setStringValue("my_type2");
-        w.addDocument(doc);
+        // ... and does not rewrite it if there is more than one type
         reader.close();
-        reader = DirectoryReader.open(w);
+        reader = openReaderWithNewType("my_type2", w);
+        Query expected = new ConstantScoreQuery(
+            new BooleanQuery.Builder()
+                .add(new TermQuery(new Term(TypeFieldMapper.NAME, "my_type")), Occur.SHOULD)
+            .build()
+        );
+        assertEquals(expected, query.rewrite(reader));
 
-        assertEquals(new ConstantScoreQuery(new TermQuery(new Term(TypeFieldMapper.NAME, "my_type"))), query.rewrite(reader));
+        BytesRef[] types =
+            new BytesRef[] {new BytesRef("my_type"), new BytesRef("my_type2"), new BytesRef("my_type3")};
+        // the query should match all documents
+        query = new TypeFieldMapper.TypesQuery(types);
+        assertEquals(new MatchAllDocsQuery(), query.rewrite(reader));
+
+        reader.close();
+        reader = openReaderWithNewType("unknown_type", w);
+        // the query cannot rewrite to a match all docs sinc unknown_type is not queried.
+        query = new TypeFieldMapper.TypesQuery(types);
+        expected =
+            new ConstantScoreQuery(
+                new BooleanQuery.Builder()
+                    .add(new TermQuery(new Term(TypeFieldMapper.CONTENT_TYPE, types[0])), Occur.SHOULD)
+                    .add(new TermQuery(new Term(TypeFieldMapper.CONTENT_TYPE, types[1])), Occur.SHOULD)
+                .build()
+            );
+        rewritten = query.rewrite(reader);
+        assertEquals(expected, rewritten);
+
+        // make sure that redundant types does not rewrite to MatchAllDocsQuery
+        query = new TypeFieldMapper.TypesQuery(new BytesRef("my_type"), new BytesRef("my_type"), new BytesRef("my_type"));
+        expected =
+            new ConstantScoreQuery(
+                new BooleanQuery.Builder()
+                    .add(new TermQuery(new Term(TypeFieldMapper.CONTENT_TYPE, "my_type")), Occur.SHOULD)
+                    .build()
+            );
+        rewritten = query.rewrite(reader);
+        assertEquals(expected, rewritten);
 
         IOUtils.close(reader, w, dir);
     }
+
+    static DirectoryReader openReaderWithNewType(String type, IndexWriter writer) throws IOException {
+        Document doc = new Document();
+        StringField typeField = new StringField(TypeFieldMapper.NAME, type, Store.NO);
+        doc.add(typeField);
+        writer.addDocument(doc);
+        return DirectoryReader.open(writer);
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/UidFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/UidFieldMapperTests.java
new file mode 100644
index 0000000000000..e5503738f0686
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/mapper/UidFieldMapperTests.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.mapper;
+
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexableField;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.mapper.MapperService.MergeReason;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+
+public class UidFieldMapperTests extends ESSingleNodeTestCase {
+
+    public void testDefaultsMultipleTypes() throws IOException {
+        Settings indexSettings = Settings.builder()
+                .put("index.mapping.single_type", false)
+                .build();
+        MapperService mapperService = createIndex("test", indexSettings).mapperService();
+        DocumentMapper mapper = mapperService.merge("type", new CompressedXContent("{\"type\":{}}"), MergeReason.MAPPING_UPDATE, false);
+        ParsedDocument document = mapper.parse(SourceToParse.source("index", "type", "id", new BytesArray("{}"), XContentType.JSON));
+        IndexableField[] fields = document.rootDoc().getFields(UidFieldMapper.NAME);
+        assertEquals(1, fields.length);
+        assertEquals(IndexOptions.DOCS, fields[0].fieldType().indexOptions());
+        assertTrue(fields[0].fieldType().stored());
+        assertEquals("type#id", fields[0].stringValue());
+    }
+
+    public void testDefaultsSingleType() throws IOException {
+        Settings indexSettings = Settings.builder()
+                .put("index.mapping.single_type", true)
+                .build();
+        MapperService mapperService = createIndex("test", indexSettings).mapperService();
+        DocumentMapper mapper = mapperService.merge("type", new CompressedXContent("{\"type\":{}}"), MergeReason.MAPPING_UPDATE, false);
+        ParsedDocument document = mapper.parse(SourceToParse.source("index", "type", "id", new BytesArray("{}"), XContentType.JSON));
+        assertEquals(Collections.<IndexableField>emptyList(), Arrays.asList(document.rootDoc().getFields(UidFieldMapper.NAME)));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/UidFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/UidFieldTypeTests.java
index 4e7b2d5fa84cf..67a626156761e 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/UidFieldTypeTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/UidFieldTypeTests.java
@@ -18,8 +18,22 @@
  */
 package org.elasticsearch.index.mapper;
 
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.MatchNoDocsQuery;
+import org.apache.lucene.search.TermInSetQuery;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.UidFieldMapper;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.mockito.Mockito;
+
+import java.util.Collection;
+import java.util.Collections;
 
 public class UidFieldTypeTests extends FieldTypeTestCase {
     @Override
@@ -31,7 +45,60 @@ public void testRangeQuery() {
         MappedFieldType ft = createDefaultFieldType();
         ft.setName("_uid");
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-                () -> ft.rangeQuery(null, null, randomBoolean(), randomBoolean()));
+                () -> ft.rangeQuery(null, null, randomBoolean(), randomBoolean(), null));
         assertEquals("Field [_uid] of type [_uid] does not support range queries", e.getMessage());
     }
+
+    public void testTermsQueryWhenTypesAreEnabled() throws Exception {
+        QueryShardContext context = Mockito.mock(QueryShardContext.class);
+        Settings indexSettings = Settings.builder()
+                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
+                .put("index.mapping.single_type", false).build();
+        IndexMetaData indexMetaData = IndexMetaData.builder(IndexMetaData.INDEX_UUID_NA_VALUE).settings(indexSettings).build();
+        IndexSettings mockSettings = new IndexSettings(indexMetaData, Settings.EMPTY);
+        Mockito.when(context.getIndexSettings()).thenReturn(mockSettings);
+
+        MapperService mapperService = Mockito.mock(MapperService.class);
+        Collection<String> types = Collections.emptySet();
+        Mockito.when(context.queryTypes()).thenReturn(types);
+        Mockito.when(context.getMapperService()).thenReturn(mapperService);
+
+        MappedFieldType ft = UidFieldMapper.defaultFieldType(mockSettings);
+        ft.setName(UidFieldMapper.NAME);
+        Query query = ft.termQuery("type#id", context);
+        assertEquals(new TermInSetQuery("_uid", new BytesRef("type#id")), query);
+    }
+
+    public void testTermsQueryWhenTypesAreDisabled() throws Exception {
+        QueryShardContext context = Mockito.mock(QueryShardContext.class);
+        Settings indexSettings = Settings.builder()
+                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
+                .put("index.mapping.single_type", true).build();
+        IndexMetaData indexMetaData = IndexMetaData.builder(IndexMetaData.INDEX_UUID_NA_VALUE).settings(indexSettings).build();
+        IndexSettings mockSettings = new IndexSettings(indexMetaData, Settings.EMPTY);
+        Mockito.when(context.getIndexSettings()).thenReturn(mockSettings);
+
+        MapperService mapperService = Mockito.mock(MapperService.class);
+        Collection<String> types = Collections.emptySet();
+        Mockito.when(mapperService.types()).thenReturn(types);
+        Mockito.when(context.getMapperService()).thenReturn(mapperService);
+
+        MappedFieldType ft = UidFieldMapper.defaultFieldType(mockSettings);
+        ft.setName(UidFieldMapper.NAME);
+        Query query = ft.termQuery("type#id", context);
+        assertEquals(new MatchNoDocsQuery(""), query);
+
+        types = Collections.singleton("type");
+        Mockito.when(mapperService.types()).thenReturn(types);
+        query = ft.termQuery("type#id", context);
+        assertEquals(new TermInSetQuery("_id", new BytesRef("id")), query);
+        query = ft.termQuery("type2#id", context);
+        assertEquals(new TermInSetQuery("_id"), query);
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/UpdateMappingOnClusterIT.java b/core/src/test/java/org/elasticsearch/index/mapper/UpdateMappingOnClusterIT.java
index 9ae01b51a64f1..0661bd8772a05 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/UpdateMappingOnClusterIT.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/UpdateMappingOnClusterIT.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.test.ESIntegTestCase;
 
 import java.util.HashMap;
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/UpdateMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/UpdateMappingTests.java
index 73cf070cba7bf..9d46e30e4cb5b 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/UpdateMappingTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/UpdateMappingTests.java
@@ -20,16 +20,12 @@
 package org.elasticsearch.index.mapper;
 
 import org.elasticsearch.Version;
-import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.IndexService;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.FieldMapper;
-import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.MapperService.MergeReason;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
@@ -37,9 +33,7 @@
 
 import java.io.IOException;
 import java.util.Collection;
-import java.util.LinkedHashMap;
 
-import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.equalTo;
 
@@ -93,13 +87,29 @@ private void testNoConflictWhileMergingAndMappingChanged(XContentBuilder mapping
 
     public void testConflictFieldsMapping(String fieldName) throws Exception {
         //test store, ... all the parameters that are not to be changed just like in other fields
-        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject(fieldName).field("enabled", true).field("store", "no").endObject()
-                .endObject().endObject();
-        XContentBuilder mappingUpdate = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject(fieldName).field("enabled", true).field("store", "yes").endObject()
-                .startObject("properties").startObject("text").field("type", "text").endObject().endObject()
-                .endObject().endObject();
+        XContentBuilder mapping = XContentFactory.jsonBuilder()
+            .startObject()
+                .startObject("type")
+                    .startObject(fieldName)
+                        .field("enabled", true)
+                            .field("store", false)
+                        .endObject()
+                    .endObject()
+            .endObject();
+        XContentBuilder mappingUpdate = XContentFactory.jsonBuilder()
+            .startObject()
+                .startObject("type")
+                    .startObject(fieldName)
+                        .field("enabled", true)
+                        .field("store", true)
+                    .endObject()
+                    .startObject("properties")
+                        .startObject("text")
+                            .field("type", "text")
+                        .endObject()
+                    .endObject()
+                .endObject()
+            .endObject();
         testConflictWhileMergingAndMappingUnchanged(mapping, mappingUpdate);
     }
 
diff --git a/core/src/test/java/org/elasticsearch/index/query/AbstractTermQueryTestCase.java b/core/src/test/java/org/elasticsearch/index/query/AbstractTermQueryTestCase.java
index 60ba7aa3f7bf5..1f58e7852192b 100644
--- a/core/src/test/java/org/elasticsearch/index/query/AbstractTermQueryTestCase.java
+++ b/core/src/test/java/org/elasticsearch/index/query/AbstractTermQueryTestCase.java
@@ -30,7 +30,7 @@ public abstract class AbstractTermQueryTestCase<QB extends BaseTermQueryBuilder<
     protected abstract QB createQueryBuilder(String fieldName, Object value);
 
     public void testIllegalArguments() throws QueryShardException {
-        String term = randomAsciiOfLengthBetween(1, 30);
+        String term = randomAlphaOfLengthBetween(1, 30);
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> createQueryBuilder(null, term));
         assertEquals("field name is null or empty", e.getMessage());
         e = expectThrows(IllegalArgumentException.class, () -> createQueryBuilder("", term));
diff --git a/core/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java
index 5e9112611715b..93014a14098cd 100644
--- a/core/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java
@@ -24,12 +24,11 @@
 import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
 import org.hamcrest.Matchers;
 
@@ -45,7 +44,6 @@
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.instanceOf;
-import static org.hamcrest.CoreMatchers.startsWith;
 
 public class BoolQueryBuilderTests extends AbstractQueryTestCase<BoolQueryBuilder> {
     @Override
@@ -58,7 +56,7 @@ protected BoolQueryBuilder doCreateTestQueryBuilder() {
             query.disableCoord(randomBoolean());
         }
         if (randomBoolean()) {
-            query.minimumNumberShouldMatch(randomMinimumShouldMatch());
+            query.minimumShouldMatch(randomMinimumShouldMatch());
         }
         int mustClauses = randomIntBetween(0, 3);
         for (int i = 0; i < mustClauses; i++) {
@@ -80,10 +78,11 @@ protected BoolQueryBuilder doCreateTestQueryBuilder() {
     }
 
     @Override
-    protected void doAssertLuceneQuery(BoolQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
+    protected void doAssertLuceneQuery(BoolQueryBuilder queryBuilder, Query query, SearchContext searchContext) throws IOException {
         if (!queryBuilder.hasClauses()) {
             assertThat(query, instanceOf(MatchAllDocsQuery.class));
         } else {
+            QueryShardContext context = searchContext.getQueryShardContext();
             List<BooleanClause> clauses = new ArrayList<>();
             clauses.addAll(getBooleanClauses(queryBuilder.must(), BooleanClause.Occur.MUST, context));
             clauses.addAll(getBooleanClauses(queryBuilder.mustNot(), BooleanClause.Occur.MUST_NOT, context));
@@ -172,25 +171,25 @@ public void testIllegalArguments() {
     // https://github.com/elastic/elasticsearch/issues/7240
     public void testEmptyBooleanQuery() throws Exception {
         XContentBuilder contentBuilder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
-        BytesReference query = contentBuilder.startObject().startObject("bool").endObject().endObject().bytes();
-        Query parsedQuery = parseQuery(query).toQuery(createShardContext());
+        contentBuilder.startObject().startObject("bool").endObject().endObject();
+        Query parsedQuery = parseQuery(createParser(contentBuilder)).toQuery(createShardContext());
         assertThat(parsedQuery, Matchers.instanceOf(MatchAllDocsQuery.class));
     }
 
     public void testDefaultMinShouldMatch() throws Exception {
         // Queries have a minShouldMatch of 0
-        BooleanQuery bq = (BooleanQuery) parseQuery(boolQuery().must(termQuery("foo", "bar")).buildAsBytes()).toQuery(createShardContext());
+        BooleanQuery bq = (BooleanQuery) parseQuery(boolQuery().must(termQuery("foo", "bar"))).toQuery(createShardContext());
         assertEquals(0, bq.getMinimumNumberShouldMatch());
 
-        bq = (BooleanQuery) parseQuery(boolQuery().should(termQuery("foo", "bar")).buildAsBytes()).toQuery(createShardContext());
+        bq = (BooleanQuery) parseQuery(boolQuery().should(termQuery("foo", "bar"))).toQuery(createShardContext());
         assertEquals(0, bq.getMinimumNumberShouldMatch());
 
         // Filters have a minShouldMatch of 0/1
-        ConstantScoreQuery csq = (ConstantScoreQuery) parseQuery(constantScoreQuery(boolQuery().must(termQuery("foo", "bar"))).buildAsBytes()).toQuery(createShardContext());
+        ConstantScoreQuery csq = (ConstantScoreQuery) parseQuery(constantScoreQuery(boolQuery().must(termQuery("foo", "bar")))).toQuery(createShardContext());
         bq = (BooleanQuery) csq.getQuery();
         assertEquals(0, bq.getMinimumNumberShouldMatch());
 
-        csq = (ConstantScoreQuery) parseQuery(constantScoreQuery(boolQuery().should(termQuery("foo", "bar"))).buildAsBytes()).toQuery(createShardContext());
+        csq = (ConstantScoreQuery) parseQuery(constantScoreQuery(boolQuery().should(termQuery("foo", "bar")))).toQuery(createShardContext());
         bq = (BooleanQuery) csq.getQuery();
         assertEquals(1, bq.getMinimumNumberShouldMatch());
     }
@@ -243,16 +242,14 @@ public void testMinShouldMatchBiggerThanNumberOfShouldClauses() throws Exception
             boolQuery()
                 .should(termQuery("foo", "bar"))
                 .should(termQuery("foo2", "bar2"))
-                .minimumNumberShouldMatch("3")
-                .buildAsBytes()).toQuery(createShardContext());
+                .minimumShouldMatch("3")).toQuery(createShardContext());
         assertEquals(3, bq.getMinimumNumberShouldMatch());
 
         bq = (BooleanQuery) parseQuery(
             boolQuery()
                 .should(termQuery("foo", "bar"))
                 .should(termQuery("foo2", "bar2"))
-                .minimumNumberShouldMatch(3)
-                .buildAsBytes()).toQuery(createShardContext());
+                .minimumShouldMatch(3)).toQuery(createShardContext());
         assertEquals(3, bq.getMinimumNumberShouldMatch());
     }
 
@@ -261,9 +258,8 @@ public void testMinShouldMatchDisableCoord() throws Exception {
                 boolQuery()
                         .should(termQuery("foo", "bar"))
                         .should(termQuery("foo2", "bar2"))
-                        .minimumNumberShouldMatch("3")
-                        .disableCoord(true)
-                        .buildAsBytes()).toQuery(createShardContext());
+                        .minimumShouldMatch("3")
+                        .disableCoord(true)).toQuery(createShardContext());
         assertEquals(3, bq.getMinimumNumberShouldMatch());
     }
 
@@ -341,14 +337,40 @@ public void testFromJsonEmptyQueryBody() throws IOException {
                 "}" +
               "}";
 
-        BoolQueryBuilder queryBuilder = (BoolQueryBuilder) parseQuery(query, ParseFieldMatcher.EMPTY);
+        BoolQueryBuilder queryBuilder = (BoolQueryBuilder) parseQuery(query);
         assertEquals(query, 0, queryBuilder.must().size());
         assertEquals(query, 0, queryBuilder.filter().size());
         assertEquals(query, 0, queryBuilder.mustNot().size());
         assertEquals(query, 0, queryBuilder.should().size());
+        // we should have deprecation warning headers regardless of throwing an exception
+        assertWarnings("query malformed, empty clause found at [1:27]",
+                "query malformed, empty clause found at [1:46]",
+                "query malformed, empty clause found at [1:100]");
+    }
 
-        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> parseQuery(query, ParseFieldMatcher.STRICT));
-        assertThat(ex.getMessage(), startsWith("query malformed, empty clause found at"));
+    /**
+     * We deprecated `minimum_number_should_match`, it should still parse correctly but add a warning header
+     */
+    public void testMinimumNumberShouldMatchDeprecated() throws IOException {
+        String query =
+                "{" +
+                "\"bool\" : {" +
+                "  \"should\" : { " +
+                "    \"term\" : {" +
+                "      \"tag\" : {" +
+                "        \"value\" : \"wow\"," +
+                "        \"boost\" : 1.0" +
+                "      }" +
+                "    } " +
+                "   }," +
+                "  \"minimum_number_should_match\" : 1" +
+                "}}";
+
+        BoolQueryBuilder queryBuilder = (BoolQueryBuilder) parseQuery(query);
+        assertEquals(query, 1, queryBuilder.should().size());
+        assertEquals(query, "1", queryBuilder.minimumShouldMatch());
+        // we should have deprecation warning headers regardless of throwing an exception
+        assertWarnings("Deprecated field [minimum_number_should_match] used, expected [minimum_should_match] instead");
     }
 
     /**
@@ -357,7 +379,7 @@ public void testFromJsonEmptyQueryBody() throws IOException {
     public void testUnknownQueryName() throws IOException {
         String query = "{\"bool\" : {\"must\" : { \"unknown_query\" : { } } } }";
 
-        ParsingException ex = expectThrows(ParsingException.class, () -> parseQuery(query, ParseFieldMatcher.EMPTY));
+        ParsingException ex = expectThrows(ParsingException.class, () -> parseQuery(query));
         assertEquals("no [query] registered for [unknown_query]", ex.getMessage());
     }
 
@@ -365,12 +387,22 @@ public void testUnknownQueryName() throws IOException {
      * test that two queries in object throws error
      */
     public void testTooManyQueriesInObject() throws IOException {
-        String clauseType = randomFrom(new String[] {"must", "should", "must_not", "filter"});
+        String clauseType = randomFrom("must", "should", "must_not", "filter");
         // should also throw error if invalid query is preceded by a valid one
-        String query = "{\"bool\" : {\"" + clauseType
-                + "\" : { \"match\" : { \"foo\" : \"bar\" } , \"match\" : { \"baz\" : \"buzz\" } } } }";
-        ParsingException ex = expectThrows(ParsingException.class, () -> parseQuery(query, ParseFieldMatcher.EMPTY));
-        assertEquals("expected [END_OBJECT] but got [FIELD_NAME], possibly too many query clauses", ex.getMessage());
+        String query = "{\n" +
+                "  \"bool\": {\n" +
+                "    \"" + clauseType + "\": {\n" +
+                "      \"match\": {\n" +
+                "        \"foo\": \"bar\"\n" +
+                "      },\n" +
+                "      \"match\": {\n" +
+                "        \"baz\": \"buzz\"\n" +
+                "      }\n" +
+                "    }\n" +
+                "  }\n" +
+                "}";
+        ParsingException ex = expectThrows(ParsingException.class, () -> parseQuery(query));
+        assertEquals("[match] malformed query, expected [END_OBJECT] but found [FIELD_NAME]", ex.getMessage());
     }
 
     public void testRewrite() throws IOException {
@@ -429,7 +461,7 @@ public void testRewriteMultipleTimes() throws IOException {
         boolQueryBuilder.must(new WrapperQueryBuilder(new WrapperQueryBuilder(new MatchAllQueryBuilder().toString()).toString()));
         QueryBuilder rewritten = boolQueryBuilder.rewrite(createShardContext());
         BoolQueryBuilder expected = new BoolQueryBuilder();
-        expected.must(new WrapperQueryBuilder(new MatchAllQueryBuilder().toString()));
+        expected.must(new MatchAllQueryBuilder());
         assertEquals(expected, rewritten);
 
         expected = new BoolQueryBuilder();
@@ -438,4 +470,23 @@ public void testRewriteMultipleTimes() throws IOException {
         assertEquals(rewrittenAgain, expected);
         assertEquals(QueryBuilder.rewriteQuery(boolQueryBuilder, createShardContext()), expected);
     }
+
+    public void testRewriteWithMatchNone() throws IOException {
+        BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
+        boolQueryBuilder.must(new WrapperQueryBuilder(new WrapperQueryBuilder(new MatchNoneQueryBuilder().toString()).toString()));
+        QueryBuilder rewritten = boolQueryBuilder.rewrite(createShardContext());
+        assertEquals(new MatchNoneQueryBuilder(), rewritten);
+
+        boolQueryBuilder = new BoolQueryBuilder();
+        boolQueryBuilder.must(new TermQueryBuilder("foo","bar"));
+        boolQueryBuilder.filter(new WrapperQueryBuilder(new WrapperQueryBuilder(new MatchNoneQueryBuilder().toString()).toString()));
+        rewritten = boolQueryBuilder.rewrite(createShardContext());
+        assertEquals(new MatchNoneQueryBuilder(), rewritten);
+
+        boolQueryBuilder = new BoolQueryBuilder();
+        boolQueryBuilder.must(new TermQueryBuilder("foo","bar"));
+        boolQueryBuilder.filter(new BoolQueryBuilder().should(new TermQueryBuilder("foo","bar")).filter(new MatchNoneQueryBuilder()));
+        rewritten = QueryBuilder.rewriteQuery(boolQueryBuilder, createShardContext());
+        assertEquals(new MatchNoneQueryBuilder(), rewritten);
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/query/BoostingQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/BoostingQueryBuilderTests.java
index 86592847e9534..2c878996e6934 100644
--- a/core/src/test/java/org/elasticsearch/index/query/BoostingQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/BoostingQueryBuilderTests.java
@@ -21,9 +21,9 @@
 
 import org.apache.lucene.queries.BoostingQuery;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
@@ -31,7 +31,8 @@
 
 import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.hamcrest.CoreMatchers.nullValue;
-import static org.hamcrest.CoreMatchers.startsWith;;
+
+;
 
 public class BoostingQueryBuilderTests extends AbstractQueryTestCase<BoostingQueryBuilder> {
 
@@ -43,9 +44,9 @@ protected BoostingQueryBuilder doCreateTestQueryBuilder() {
     }
 
     @Override
-    protected void doAssertLuceneQuery(BoostingQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
-        Query positive = queryBuilder.positiveQuery().toQuery(context);
-        Query negative = queryBuilder.negativeQuery().toQuery(context);
+    protected void doAssertLuceneQuery(BoostingQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
+        Query positive = queryBuilder.positiveQuery().toQuery(context.getQueryShardContext());
+        Query negative = queryBuilder.negativeQuery().toQuery(context.getQueryShardContext());
         if (positive == null || negative == null) {
             assertThat(query, nullValue());
         } else {
@@ -104,27 +105,25 @@ public void testFromJsonEmptyQueryBody() throws IOException {
                 "    \"negative_boost\" : 23.0" +
                 "  }" +
                 "}";
-        XContentParser parser = XContentFactory.xContent(query).createParser(query);
-        QueryParseContext context = createParseContext(parser, ParseFieldMatcher.EMPTY);
+        XContentParser parser = createParser(JsonXContent.jsonXContent, query);
+        QueryParseContext context = createParseContext(parser);
         Optional<QueryBuilder> innerQueryBuilder = context.parseInnerQueryBuilder();
         assertTrue(innerQueryBuilder.isPresent() == false);
 
+        assertWarnings("query malformed, empty clause found at [1:36]");
+
         query =
-                "{ \"boosting\" : {" +
-                "    \"positive\" : { \"match_all\" : {} }, " +
-                "    \"negative\" : { }, " +
-                "    \"negative_boost\" : 23.0" +
-                "  }" +
+                "{ \"boosting\" : {\n" +
+                "    \"positive\" : { \"match_all\" : {} },\n" +
+                "    \"negative\" : { },\n" +
+                "    \"negative_boost\" : 23.0\n" +
+                "  }\n" +
                 "}";
-        parser = XContentFactory.xContent(query).createParser(query);
-        context = createParseContext(parser, ParseFieldMatcher.EMPTY);
+        parser = createParser(JsonXContent.jsonXContent, query);
+        context = createParseContext(parser);
         innerQueryBuilder = context.parseInnerQueryBuilder();
         assertTrue(innerQueryBuilder.isPresent() == false);
-
-        parser = XContentFactory.xContent(query).createParser(query);
-        QueryParseContext otherContext = createParseContext(parser, ParseFieldMatcher.STRICT);
-        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> otherContext.parseInnerQueryBuilder());
-        assertThat(ex.getMessage(), startsWith("query malformed, empty clause found at"));
+        assertWarnings("query malformed, empty clause found at [3:20]");
     }
 
     public void testRewrite() throws IOException {
diff --git a/core/src/test/java/org/elasticsearch/index/query/CommonTermsQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/CommonTermsQueryBuilderTests.java
index 057fcf36d72c0..66c36e0418ada 100644
--- a/core/src/test/java/org/elasticsearch/index/query/CommonTermsQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/CommonTermsQueryBuilderTests.java
@@ -22,6 +22,7 @@
 import org.apache.lucene.queries.ExtendedCommonTermsQuery;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
@@ -43,13 +44,13 @@ protected CommonTermsQueryBuilder doCreateTestQueryBuilder() {
         int numberOfTerms = randomIntBetween(0, 10);
         StringBuilder text = new StringBuilder("");
         for (int i = 0; i < numberOfTerms; i++) {
-            text.append(randomAsciiOfLengthBetween(1, 10)).append(" ");
+            text.append(randomAlphaOfLengthBetween(1, 10)).append(" ");
         }
         // mapped or unmapped field
         if (randomBoolean()) {
             query = new CommonTermsQueryBuilder(STRING_FIELD_NAME, text.toString());
         } else {
-            query = new CommonTermsQueryBuilder(randomAsciiOfLengthBetween(1, 10), text.toString());
+            query = new CommonTermsQueryBuilder(randomAlphaOfLengthBetween(1, 10), text.toString());
         }
 
         if (randomBoolean()) {
@@ -87,8 +88,8 @@ protected CommonTermsQueryBuilder doCreateTestQueryBuilder() {
     @Override
     protected Map<String, CommonTermsQueryBuilder> getAlternateVersions() {
         Map<String, CommonTermsQueryBuilder> alternateVersions = new HashMap<>();
-        CommonTermsQueryBuilder commonTermsQuery = new CommonTermsQueryBuilder(randomAsciiOfLengthBetween(1, 10),
-                randomAsciiOfLengthBetween(1, 10));
+        CommonTermsQueryBuilder commonTermsQuery = new CommonTermsQueryBuilder(randomAlphaOfLengthBetween(1, 10),
+                randomAlphaOfLengthBetween(1, 10));
         String contentString = "{\n" +
                 "    \"common\" : {\n" +
                 "        \"" + commonTermsQuery.fieldName() + "\" : \"" + commonTermsQuery.value() + "\"\n" +
@@ -99,7 +100,7 @@ protected Map<String, CommonTermsQueryBuilder> getAlternateVersions() {
     }
 
     @Override
-    protected void doAssertLuceneQuery(CommonTermsQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
+    protected void doAssertLuceneQuery(CommonTermsQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
         assertThat(query, instanceOf(ExtendedCommonTermsQuery.class));
         ExtendedCommonTermsQuery extendedCommonTermsQuery = (ExtendedCommonTermsQuery) query;
         assertThat(extendedCommonTermsQuery.getHighFreqMinimumNumberShouldMatchSpec(), equalTo(queryBuilder.highFreqMinimumShouldMatch()));
@@ -174,7 +175,7 @@ public void testCommonTermsQuery3() throws IOException {
     // see #11730
     public void testCommonTermsQuery4() throws IOException {
         boolean disableCoord = randomBoolean();
-        Query parsedQuery = parseQuery(commonTermsQuery("field", "text").disableCoord(disableCoord).buildAsBytes()).toQuery(createShardContext());
+        Query parsedQuery = parseQuery(commonTermsQuery("field", "text").disableCoord(disableCoord)).toQuery(createShardContext());
         assertThat(parsedQuery, instanceOf(ExtendedCommonTermsQuery.class));
         ExtendedCommonTermsQuery ectQuery = (ExtendedCommonTermsQuery) parsedQuery;
         assertThat(ectQuery.isCoordDisabled(), equalTo(disableCoord));
diff --git a/core/src/test/java/org/elasticsearch/index/query/ConstantScoreQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/ConstantScoreQueryBuilderTests.java
index d716a553a5af4..ccc2b188abdf4 100644
--- a/core/src/test/java/org/elasticsearch/index/query/ConstantScoreQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/ConstantScoreQueryBuilderTests.java
@@ -21,10 +21,10 @@
 
 import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
@@ -32,7 +32,6 @@
 
 import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.hamcrest.CoreMatchers.nullValue;
-import static org.hamcrest.CoreMatchers.startsWith;
 import static org.hamcrest.Matchers.containsString;
 
 public class ConstantScoreQueryBuilderTests extends AbstractQueryTestCase<ConstantScoreQueryBuilder> {
@@ -45,8 +44,8 @@ protected ConstantScoreQueryBuilder doCreateTestQueryBuilder() {
     }
 
     @Override
-    protected void doAssertLuceneQuery(ConstantScoreQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
-        Query innerQuery = queryBuilder.innerQuery().toQuery(context);
+    protected void doAssertLuceneQuery(ConstantScoreQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
+        Query innerQuery = queryBuilder.innerQuery().toQuery(context.getQueryShardContext());
         if (innerQuery == null) {
             assertThat(query, nullValue());
         } else {
@@ -94,7 +93,7 @@ public void testIllegalArguments() {
     }
 
     @Override
-    public void testUnknownField() throws IOException {
+    public void testUnknownField() {
         assumeTrue("test doesn't apply for query filter queries", false);
     }
 
@@ -128,15 +127,16 @@ public void testFromJsonEmptyQueryBody() throws IOException {
                 "    \"filter\" : { }" +
                 "  }" +
                 "}";
-        XContentParser parser = XContentFactory.xContent(query).createParser(query);
-        QueryParseContext context = createParseContext(parser, ParseFieldMatcher.EMPTY);
+        XContentParser parser = createParser(JsonXContent.jsonXContent, query);
+        QueryParseContext context = createParseContext(parser);
         Optional<QueryBuilder> innerQueryBuilder = context.parseInnerQueryBuilder();
         assertTrue(innerQueryBuilder.isPresent() == false);
-
-        parser = XContentFactory.xContent(query).createParser(query);
-        QueryParseContext otherContext = createParseContext(parser, ParseFieldMatcher.STRICT);
-        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> otherContext.parseInnerQueryBuilder());
-        assertThat(ex.getMessage(), startsWith("query malformed, empty clause found at"));
+        assertWarnings("query malformed, empty clause found at [1:40]");
     }
 
+    public void testRewriteToMatchNone() throws IOException {
+        ConstantScoreQueryBuilder constantScoreQueryBuilder = new ConstantScoreQueryBuilder(new MatchNoneQueryBuilder());
+        QueryBuilder rewrite = constantScoreQueryBuilder.rewrite(createShardContext());
+        assertEquals(rewrite, new MatchNoneQueryBuilder());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/query/DisMaxQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/DisMaxQueryBuilderTests.java
index 8fa8724ae3c8a..689e04af93095 100644
--- a/core/src/test/java/org/elasticsearch/index/query/DisMaxQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/DisMaxQueryBuilderTests.java
@@ -24,8 +24,8 @@
 import org.apache.lucene.search.DisjunctionMaxQuery;
 import org.apache.lucene.search.PrefixQuery;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
+import org.apache.lucene.search.MatchNoDocsQuery;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
@@ -57,8 +57,8 @@ protected DisMaxQueryBuilder doCreateTestQueryBuilder() {
     }
 
     @Override
-    protected void doAssertLuceneQuery(DisMaxQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
-        Collection<Query> queries = AbstractQueryBuilder.toQueries(queryBuilder.innerQueries(), context);
+    protected void doAssertLuceneQuery(DisMaxQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
+        Collection<Query> queries = AbstractQueryBuilder.toQueries(queryBuilder.innerQueries(), context.getQueryShardContext());
         assertThat(query, instanceOf(DisjunctionMaxQuery.class));
         DisjunctionMaxQuery disjunctionMaxQuery = (DisjunctionMaxQuery) query;
         assertThat(disjunctionMaxQuery.getTieBreakerMultiplier(), equalTo(queryBuilder.tieBreaker()));
@@ -93,11 +93,12 @@ public void testInnerQueryEmptyException() throws IOException {
                 + "             { \"queries\" : [ {\"" + ConstantScoreQueryBuilder.NAME + "\" : { \"filter\" : { } } } ] "
                 + "             }"
                 + "           }";
-        QueryBuilder queryBuilder = parseQuery(queryString, ParseFieldMatcher.EMPTY);
+        QueryBuilder queryBuilder = parseQuery(queryString);
         QueryShardContext context = createShardContext();
         Query luceneQuery = queryBuilder.toQuery(context);
         assertThat(luceneQuery, instanceOf(MatchNoDocsQuery.class));
-        assertThat(((MatchNoDocsQuery) luceneQuery).toString(), equalTo("MatchNoDocsQuery[\"no clauses for dismax query.\"]"));
+        assertThat(luceneQuery.toString(), equalTo("MatchNoDocsQuery(\"no clauses for dismax query.\")"));
+        assertWarnings("query malformed, empty clause found at [1:78]");
     }
 
     public void testIllegalArguments() {
diff --git a/core/src/test/java/org/elasticsearch/index/query/DisableGraphQueryTests.java b/core/src/test/java/org/elasticsearch/index/query/DisableGraphQueryTests.java
new file mode 100644
index 0000000000000..770405935e277
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/query/DisableGraphQueryTests.java
@@ -0,0 +1,251 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.query;
+
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.SynonymQuery;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.PhraseQuery;
+import org.apache.lucene.search.DisjunctionMaxQuery;
+import org.apache.lucene.search.MultiPhraseQuery;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.search.MatchQuery;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+
+/**
+ * Makes sure that graph analysis is disabled with shingle filters of different size
+ */
+public class DisableGraphQueryTests extends ESSingleNodeTestCase {
+    private static IndexService indexService;
+    private static QueryShardContext shardContext;
+    private static Query expectedQuery;
+    private static Query expectedPhraseQuery;
+    private static Query expectedQueryWithUnigram;
+    private static Query expectedPhraseQueryWithUnigram;
+
+    @Before
+    public void setup() {
+        Settings settings = Settings.builder()
+            .put("index.analysis.filter.shingle.type", "shingle")
+            .put("index.analysis.filter.shingle.output_unigrams", false)
+            .put("index.analysis.filter.shingle.min_size", 2)
+            .put("index.analysis.filter.shingle.max_size", 2)
+            .put("index.analysis.filter.shingle_unigram.type", "shingle")
+            .put("index.analysis.filter.shingle_unigram.output_unigrams", true)
+            .put("index.analysis.filter.shingle_unigram.min_size", 2)
+            .put("index.analysis.filter.shingle_unigram.max_size", 2)
+            .put("index.analysis.analyzer.text_shingle.tokenizer", "whitespace")
+            .put("index.analysis.analyzer.text_shingle.filter", "lowercase, shingle")
+            .put("index.analysis.analyzer.text_shingle_unigram.tokenizer", "whitespace")
+            .put("index.analysis.analyzer.text_shingle_unigram.filter",
+                "lowercase, shingle_unigram")
+            .build();
+        indexService = createIndex("test", settings, "t",
+            "text_shingle", "type=text,analyzer=text_shingle",
+            "text_shingle_unigram", "type=text,analyzer=text_shingle_unigram");
+        shardContext = indexService.newQueryShardContext(0, null, () -> 0L);
+
+        // parsed queries for "text_shingle_unigram:(foo bar baz)" with query parsers
+        // that ignores position length attribute
+         expectedQueryWithUnigram= new BooleanQuery.Builder()
+            .add(
+                new SynonymQuery(
+                    new Term("text_shingle_unigram", "foo"),
+                    new Term("text_shingle_unigram", "foo bar")
+                ), BooleanClause.Occur.SHOULD)
+            .add(
+                new SynonymQuery(
+                    new Term("text_shingle_unigram", "bar"),
+                    new Term("text_shingle_unigram", "bar baz")
+            ), BooleanClause.Occur.SHOULD)
+            .add(
+                new TermQuery(
+                    new Term("text_shingle_unigram", "baz")
+                ), BooleanClause.Occur.SHOULD)
+            .build();
+
+        // parsed query for "text_shingle_unigram:\"foo bar baz\" with query parsers
+        // that ignores position length attribute
+        expectedPhraseQueryWithUnigram = new MultiPhraseQuery.Builder()
+            .add(
+                new Term[] {
+                    new Term("text_shingle_unigram", "foo"),
+                    new Term("text_shingle_unigram", "foo bar")
+                }, 0)
+            .add(
+                new Term[] {
+                    new Term("text_shingle_unigram", "bar"),
+                    new Term("text_shingle_unigram", "bar baz")
+                }, 1)
+            .add(
+                new Term[] {
+                    new Term("text_shingle_unigram", "baz"),
+                }, 2)
+            .build();
+
+        // parsed query for "text_shingle:(foo bar baz)
+        expectedQuery = new BooleanQuery.Builder()
+            .add(
+                new TermQuery(new Term("text_shingle", "foo bar")),
+                BooleanClause.Occur.SHOULD
+            )
+            .add(
+                new TermQuery(new Term("text_shingle","bar baz")),
+                BooleanClause.Occur.SHOULD
+            )
+            .add(
+                new TermQuery(new Term("text_shingle","baz biz")),
+                BooleanClause.Occur.SHOULD
+            )
+            .build();
+
+        // parsed query for "text_shingle:"foo bar baz"
+        expectedPhraseQuery = new PhraseQuery.Builder()
+            .add(
+                new Term("text_shingle", "foo bar")
+            )
+            .add(
+                new Term("text_shingle","bar baz")
+            )
+            .add(
+                new Term("text_shingle","baz biz")
+            )
+            .build();
+    }
+
+    @After
+    public void cleanup() {
+        indexService = null;
+        shardContext = null;
+        expectedQuery = null;
+        expectedPhraseQuery = null;
+    }
+
+    public void testMatchPhraseQuery() throws IOException {
+        MatchPhraseQueryBuilder builder =
+            new MatchPhraseQueryBuilder("text_shingle_unigram", "foo bar baz");
+        Query query = builder.doToQuery(shardContext);
+        assertThat(expectedPhraseQueryWithUnigram, equalTo(query));
+
+        builder =
+            new MatchPhraseQueryBuilder("text_shingle", "foo bar baz biz");
+        query = builder.doToQuery(shardContext);
+        assertThat(expectedPhraseQuery, equalTo(query));
+    }
+
+    public void testMatchQuery() throws IOException {
+        MatchQueryBuilder builder =
+            new MatchQueryBuilder("text_shingle_unigram", "foo bar baz");
+        Query query = builder.doToQuery(shardContext);
+        assertThat(expectedQueryWithUnigram, equalTo(query));
+
+        builder = new MatchQueryBuilder("text_shingle", "foo bar baz biz");
+        query = builder.doToQuery(shardContext);
+        assertThat(expectedQuery, equalTo(query));
+    }
+
+    public void testMultiMatchQuery() throws IOException {
+        MultiMatchQueryBuilder builder = new MultiMatchQueryBuilder("foo bar baz",
+            "text_shingle_unigram");
+        Query query = builder.doToQuery(shardContext);
+        assertThat(expectedQueryWithUnigram, equalTo(query));
+
+        builder.type(MatchQuery.Type.PHRASE);
+        query = builder.doToQuery(shardContext);
+        assertThat(expectedPhraseQueryWithUnigram, equalTo(query));
+
+        builder = new MultiMatchQueryBuilder("foo bar baz biz", "text_shingle");
+        query = builder.doToQuery(shardContext);
+        assertThat(expectedQuery, equalTo(query));
+
+        builder.type(MatchQuery.Type.PHRASE);
+        query = builder.doToQuery(shardContext);
+        assertThat(expectedPhraseQuery, equalTo(query));
+    }
+
+    public void testSimpleQueryString() throws IOException {
+        SimpleQueryStringBuilder builder = new SimpleQueryStringBuilder("foo bar baz");
+        builder.field("text_shingle_unigram");
+        builder.flags(SimpleQueryStringFlag.NONE);
+        Query query = builder.doToQuery(shardContext);
+        assertThat(expectedQueryWithUnigram, equalTo(query));
+
+        builder = new SimpleQueryStringBuilder("\"foo bar baz\"");
+        builder.field("text_shingle_unigram");
+        builder.flags(SimpleQueryStringFlag.PHRASE);
+        query = builder.doToQuery(shardContext);
+        assertThat(expectedPhraseQueryWithUnigram, equalTo(query));
+
+        builder = new SimpleQueryStringBuilder("foo bar baz biz");
+        builder.field("text_shingle");
+        builder.flags(SimpleQueryStringFlag.NONE);
+        query = builder.doToQuery(shardContext);
+        assertThat(expectedQuery, equalTo(query));
+
+        builder = new SimpleQueryStringBuilder("\"foo bar baz biz\"");
+        builder.field("text_shingle");
+        builder.flags(SimpleQueryStringFlag.PHRASE);
+        query = builder.doToQuery(shardContext);
+        assertThat(expectedPhraseQuery, equalTo(query));
+    }
+
+    public void testQueryString() throws IOException {
+        QueryStringQueryBuilder builder = new QueryStringQueryBuilder("foo bar baz");
+        builder.field("text_shingle_unigram");
+        builder.splitOnWhitespace(false);
+        Query query = builder.doToQuery(shardContext);
+        assertThat(expectedQueryWithUnigram, equalTo(query));
+
+        builder = new QueryStringQueryBuilder("\"foo bar baz\"");
+        builder.field("text_shingle_unigram");
+        builder.splitOnWhitespace(false);
+        query = builder.doToQuery(shardContext);
+        assertThat(query, instanceOf(DisjunctionMaxQuery.class));
+        DisjunctionMaxQuery maxQuery = (DisjunctionMaxQuery) query;
+        assertThat(maxQuery.getDisjuncts().size(), equalTo(1));
+        assertThat(expectedPhraseQueryWithUnigram, equalTo(maxQuery.getDisjuncts().get(0)));
+
+        builder = new QueryStringQueryBuilder("foo bar baz biz");
+        builder.field("text_shingle");
+        builder.splitOnWhitespace(false);
+        query = builder.doToQuery(shardContext);
+        assertThat(expectedQuery, equalTo(query));
+
+        builder = new QueryStringQueryBuilder("\"foo bar baz biz\"");
+        builder.field("text_shingle");
+        builder.splitOnWhitespace(false);
+        query = builder.doToQuery(shardContext);
+        assertThat(query, instanceOf(DisjunctionMaxQuery.class));
+        maxQuery = (DisjunctionMaxQuery) query;
+        assertThat(maxQuery.getDisjuncts().size(), equalTo(1));
+        assertThat(expectedPhraseQuery, equalTo(maxQuery.getDisjuncts().get(0)));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/query/ExistsQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/ExistsQueryBuilderTests.java
index 89de6b102ac20..cfc2d78942050 100644
--- a/core/src/test/java/org/elasticsearch/index/query/ExistsQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/ExistsQueryBuilderTests.java
@@ -22,9 +22,11 @@
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
+import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
 import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
@@ -41,7 +43,7 @@ protected ExistsQueryBuilder doCreateTestQueryBuilder() {
         if (randomBoolean()) {
             fieldPattern = randomFrom(MAPPED_FIELD_NAMES);
         } else {
-            fieldPattern = randomAsciiOfLengthBetween(1, 10);
+            fieldPattern = randomAlphaOfLengthBetween(1, 10);
         }
         // also sometimes test wildcard patterns
         if (randomBoolean()) {
@@ -55,13 +57,19 @@ protected ExistsQueryBuilder doCreateTestQueryBuilder() {
     }
 
     @Override
-    protected void doAssertLuceneQuery(ExistsQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
+    protected void doAssertLuceneQuery(ExistsQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
         String fieldPattern = queryBuilder.fieldName();
-        Collection<String> fields = context.simpleMatchToIndexNames(fieldPattern);
+        Collection<String> fields = context.getQueryShardContext().simpleMatchToIndexNames(fieldPattern);
         if (getCurrentTypes().length == 0) {
             assertThat(query, instanceOf(MatchNoDocsQuery.class));
             MatchNoDocsQuery matchNoDocsQuery = (MatchNoDocsQuery) query;
             assertThat(matchNoDocsQuery.toString(null), containsString("Missing types in \"exists\" query."));
+        } else if (fields.size() == 1) {
+            assertThat(query, instanceOf(ConstantScoreQuery.class));
+            ConstantScoreQuery constantScoreQuery = (ConstantScoreQuery) query;
+            assertThat(constantScoreQuery.getQuery(), instanceOf(TermQuery.class));
+            TermQuery termQuery = (TermQuery) constantScoreQuery.getQuery();
+            assertEquals(fields.iterator().next(), termQuery.getTerm().text());
         } else {
             assertThat(query, instanceOf(ConstantScoreQuery.class));
             ConstantScoreQuery constantScoreQuery = (ConstantScoreQuery) query;
diff --git a/core/src/test/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilderTests.java
index e6cefdb67af16..d7fe00e730c91 100644
--- a/core/src/test/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilderTests.java
@@ -22,6 +22,7 @@
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.spans.FieldMaskingSpanQuery;
 import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
@@ -36,23 +37,23 @@ protected FieldMaskingSpanQueryBuilder doCreateTestQueryBuilder() {
         if (randomBoolean()) {
             fieldName = randomFrom(MAPPED_FIELD_NAMES);
         } else {
-            fieldName = randomAsciiOfLengthBetween(1, 10);
+            fieldName = randomAlphaOfLengthBetween(1, 10);
         }
         SpanTermQueryBuilder innerQuery = new SpanTermQueryBuilderTests().createTestQueryBuilder();
         return new FieldMaskingSpanQueryBuilder(innerQuery, fieldName);
     }
 
     @Override
-    protected void doAssertLuceneQuery(FieldMaskingSpanQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
+    protected void doAssertLuceneQuery(FieldMaskingSpanQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
         String fieldInQuery = queryBuilder.fieldName();
-        MappedFieldType fieldType = context.fieldMapper(fieldInQuery);
+        MappedFieldType fieldType = context.getQueryShardContext().fieldMapper(fieldInQuery);
         if (fieldType != null) {
             fieldInQuery = fieldType.name();
         }
         assertThat(query, instanceOf(FieldMaskingSpanQuery.class));
         FieldMaskingSpanQuery fieldMaskingSpanQuery = (FieldMaskingSpanQuery) query;
         assertThat(fieldMaskingSpanQuery.getField(), equalTo(fieldInQuery));
-        assertThat(fieldMaskingSpanQuery.getMaskedQuery(), equalTo(queryBuilder.innerQuery().toQuery(context)));
+        assertThat(fieldMaskingSpanQuery.getMaskedQuery(), equalTo(queryBuilder.innerQuery().toQuery(context.getQueryShardContext())));
     }
 
     public void testIllegalArguments() {
diff --git a/core/src/test/java/org/elasticsearch/index/query/FuzzyQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/FuzzyQueryBuilderTests.java
index 205aaa2448482..58c70fcfcb39b 100644
--- a/core/src/test/java/org/elasticsearch/index/query/FuzzyQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/FuzzyQueryBuilderTests.java
@@ -25,6 +25,7 @@
 import org.apache.lucene.search.Query;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.unit.Fuzziness;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
@@ -61,7 +62,7 @@ protected FuzzyQueryBuilder doCreateTestQueryBuilder() {
     @Override
     protected Map<String, FuzzyQueryBuilder> getAlternateVersions() {
         Map<String, FuzzyQueryBuilder> alternateVersions = new HashMap<>();
-        FuzzyQueryBuilder fuzzyQuery = new FuzzyQueryBuilder(randomAsciiOfLengthBetween(1, 10), randomAsciiOfLengthBetween(1, 10));
+        FuzzyQueryBuilder fuzzyQuery = new FuzzyQueryBuilder(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10));
         String contentString = "{\n" +
                 "    \"fuzzy\" : {\n" +
                 "        \"" + fuzzyQuery.fieldName() + "\" : \"" + fuzzyQuery.value() + "\"\n" +
@@ -72,7 +73,7 @@ protected Map<String, FuzzyQueryBuilder> getAlternateVersions() {
     }
 
     @Override
-    protected void doAssertLuceneQuery(FuzzyQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
+    protected void doAssertLuceneQuery(FuzzyQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
         assertThat(query, instanceOf(FuzzyQuery.class));
     }
 
@@ -156,7 +157,16 @@ public void testFromJson() throws IOException {
     }
 
     public void testParseFailsWithMultipleFields() throws IOException {
-        String json = "{\n" +
+        String json1 = "{\n" +
+                "  \"fuzzy\" : {\n" +
+                "    \"message1\" : {\n" +
+                "      \"value\" : \"this is a test\"\n" +
+                "    }\n" +
+                "  }\n" +
+                "}";
+        parseQuery(json1); // should be all good
+
+        String json2 = "{\n" +
                 "  \"fuzzy\" : {\n" +
                 "    \"message1\" : {\n" +
                 "      \"value\" : \"this is a test\"\n" +
@@ -167,7 +177,7 @@ public void testParseFailsWithMultipleFields() throws IOException {
                 "  }\n" +
                 "}";
 
-        ParsingException e = expectThrows(ParsingException.class, () -> parseQuery(json));
+        ParsingException e = expectThrows(ParsingException.class, () -> parseQuery(json2));
         assertEquals("[fuzzy] query doesn't support multiple fields, found [message1] and [message2]", e.getMessage());
 
         String shortJson = "{\n" +
@@ -180,4 +190,17 @@ public void testParseFailsWithMultipleFields() throws IOException {
         e = expectThrows(ParsingException.class, () -> parseQuery(shortJson));
         assertEquals("[fuzzy] query doesn't support multiple fields, found [message1] and [message2]", e.getMessage());
     }
+
+    public void testParseFailsWithValueArray() {
+        String query = "{\n" +
+                "  \"fuzzy\" : {\n" +
+                "    \"message1\" : {\n" +
+                "      \"value\" : [ \"one\", \"two\", \"three\"]\n" +
+                "    }\n" +
+                "  }\n" +
+                "}";
+
+        ParsingException e = expectThrows(ParsingException.class, () -> parseQuery(query));
+        assertEquals("[fuzzy] unexpected token [START_ARRAY] after [value]", e.getMessage());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderTests.java
index ea020b5f48731..6676c5a14f131 100644
--- a/core/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderTests.java
@@ -19,19 +19,22 @@
 
 package org.elasticsearch.index.query;
 
+import org.apache.lucene.document.LatLonDocValuesField;
+import org.apache.lucene.document.LatLonPoint;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
+import org.apache.lucene.search.IndexOrDocValuesQuery;
 import org.apache.lucene.search.LegacyNumericRangeQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.spatial.geopoint.search.GeoPointInBBoxQuery;
 import org.elasticsearch.Version;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.geo.GeoUtils;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.search.geo.LegacyInMemoryGeoBoundingBoxQuery;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
 import org.elasticsearch.test.geo.RandomShapeGenerator;
 import org.locationtech.spatial4j.io.GeohashUtils;
@@ -121,7 +124,7 @@ public void testToQuery() throws IOException {
 
     public void testExceptionOnMissingTypes() throws IOException {
         assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length == 0);
-        QueryShardException e = expectThrows(QueryShardException.class, () -> super.testToQuery());
+        QueryShardException e = expectThrows(QueryShardException.class, super::testToQuery);
         assertEquals("failed to find geo_point field [mapped_geo_point]", e.getMessage());
     }
 
@@ -254,8 +257,9 @@ public void testStrictnessDefault() {
     }
 
     @Override
-    protected void doAssertLuceneQuery(GeoBoundingBoxQueryBuilder queryBuilder, Query query, QueryShardContext context)
+    protected void doAssertLuceneQuery(GeoBoundingBoxQueryBuilder queryBuilder, Query query, SearchContext searchContext)
         throws IOException {
+        QueryShardContext context = searchContext.getQueryShardContext();
         MappedFieldType fieldType = context.fieldMapper(queryBuilder.fieldName());
         if (fieldType == null) {
             assertTrue("Found no indexed geo query.", query instanceof MatchNoDocsQuery);
@@ -266,8 +270,24 @@ protected void doAssertLuceneQuery(GeoBoundingBoxQueryBuilder queryBuilder, Quer
                 } else {
                     assertTrue("Found no indexed geo query.", query instanceof LegacyInMemoryGeoBoundingBoxQuery);
                 }
-            } else {
+            } else if (context.indexVersionCreated().before(Version.V_5_0_0_beta1)) {
                 assertTrue("Found no indexed geo query.", query instanceof GeoPointInBBoxQuery);
+            } else {
+                assertTrue("Found no indexed geo query.", query instanceof Query);
+                if (query instanceof IndexOrDocValuesQuery) {
+                    Query indexQuery = ((IndexOrDocValuesQuery) query).getIndexQuery();
+                    assertEquals(LatLonPoint.newBoxQuery(queryBuilder.fieldName(),
+                            queryBuilder.bottomRight().lat(),
+                            queryBuilder.topLeft().lat(),
+                            queryBuilder.topLeft().lon(),
+                            queryBuilder.bottomRight().lon()), indexQuery);
+                    Query dvQuery = ((IndexOrDocValuesQuery) query).getRandomAccessQuery();
+                    assertEquals(LatLonDocValuesField.newBoxQuery(queryBuilder.fieldName(),
+                            queryBuilder.bottomRight().lat(),
+                            queryBuilder.topLeft().lat(),
+                            queryBuilder.topLeft().lon(),
+                            queryBuilder.bottomRight().lon()), dvQuery);
+                }
             }
         }
     }
@@ -429,7 +449,7 @@ private void assertGeoBoundingBoxQuery(String query) throws IOException {
             assertThat(filter.topLeft().lon(), closeTo(-70, 1E-5));
             assertThat(filter.bottomRight().lat(), closeTo(30, 1E-5));
             assertThat(filter.bottomRight().lon(), closeTo(-80, 1E-5));
-        } else {
+        } else if (shardContext.indexVersionCreated().before(Version.V_5_0_0_beta1)) {
             GeoPointInBBoxQuery q = (GeoPointInBBoxQuery) parsedQuery;
             assertThat(q.getField(), equalTo(GEO_POINT_FIELD_NAME));
             assertThat(q.getMaxLat(), closeTo(40, 1E-5));
@@ -475,14 +495,12 @@ public void testFromJson() throws IOException {
                         "    \"boost\" : 1.0\n" +
                         "  }\n" +
                         "}";
-        QueryBuilder parsedGeoBboxShortcut = parseQuery(json, ParseFieldMatcher.EMPTY);
+        QueryBuilder parsedGeoBboxShortcut = parseQuery(deprecatedJson);
         assertThat(parsedGeoBboxShortcut, equalTo(parsed));
-
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseQuery(deprecatedJson));
-        assertEquals("Deprecated field [geo_bbox] used, expected [geo_bounding_box] instead", e.getMessage());
+        assertWarnings("Deprecated field [geo_bbox] used, expected [geo_bounding_box] instead");
     }
 
-    public void testFromJsonCoerceFails() throws IOException {
+    public void testFromJsonCoerceIsDeprecated() throws IOException {
         String json =
                 "{\n" +
                 "  \"geo_bounding_box\" : {\n" +
@@ -496,11 +514,12 @@ public void testFromJsonCoerceFails() throws IOException {
                 "    \"boost\" : 1.0\n" +
                 "  }\n" +
                 "}";
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseQuery(json));
-        assertTrue(e.getMessage().startsWith("Deprecated field "));
+
+        parseQuery(json);
+        assertWarnings("Deprecated field [coerce] used, replaced by [validation_method]");
     }
 
-    public void testFromJsonIgnoreMalformedFails() throws IOException {
+    public void testFromJsonIgnoreMalformedIsDeprecated() throws IOException {
         String json =
                 "{\n" +
                 "  \"geo_bounding_box\" : {\n" +
@@ -514,8 +533,8 @@ public void testFromJsonIgnoreMalformedFails() throws IOException {
                 "    \"boost\" : 1.0\n" +
                 "  }\n" +
                 "}";
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseQuery(json));
-        assertTrue(e.getMessage().startsWith("Deprecated field "));
+        parseQuery(json);
+        assertWarnings("Deprecated field [ignore_malformed] used, replaced by [validation_method]");
     }
 
     @Override
diff --git a/core/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderTests.java
index 676041fdd8cfd..b5ba3fc0e3c1c 100644
--- a/core/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderTests.java
@@ -19,6 +19,9 @@
 
 package org.elasticsearch.index.query;
 
+import org.apache.lucene.document.LatLonDocValuesField;
+import org.apache.lucene.document.LatLonPoint;
+import org.apache.lucene.search.IndexOrDocValuesQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.spatial.geopoint.search.GeoPointDistanceQuery;
@@ -28,7 +31,9 @@
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.geo.GeoUtils;
 import org.elasticsearch.common.unit.DistanceUnit;
-import org.elasticsearch.index.search.geo.GeoDistanceRangeQuery;
+import org.elasticsearch.index.mapper.LatLonPointFieldMapper;
+import org.elasticsearch.index.search.geo.LegacyGeoDistanceRangeQuery;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
 import org.elasticsearch.test.geo.RandomShapeGenerator;
 import org.locationtech.spatial4j.shape.Point;
@@ -128,8 +133,8 @@ public void testToQuery() throws IOException {
     }
 
     @Override
-    protected void doAssertLuceneQuery(GeoDistanceQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
-        Version version = context.indexVersionCreated();
+    protected void doAssertLuceneQuery(GeoDistanceQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
+        Version version = context.getQueryShardContext().indexVersionCreated();
         if (version.before(Version.V_2_2_0)) {
             assertLegacyQuery(queryBuilder, query);
         } else {
@@ -138,8 +143,8 @@ protected void doAssertLuceneQuery(GeoDistanceQueryBuilder queryBuilder, Query q
     }
 
     private void assertLegacyQuery(GeoDistanceQueryBuilder queryBuilder, Query query) throws IOException {
-        assertThat(query, instanceOf(GeoDistanceRangeQuery.class));
-        GeoDistanceRangeQuery geoQuery = (GeoDistanceRangeQuery) query;
+        assertThat(query, instanceOf(LegacyGeoDistanceRangeQuery.class));
+        LegacyGeoDistanceRangeQuery geoQuery = (LegacyGeoDistanceRangeQuery) query;
         assertThat(geoQuery.fieldName(), equalTo(queryBuilder.fieldName()));
         if (queryBuilder.point() != null) {
             assertThat(geoQuery.lat(), equalTo(queryBuilder.point().lat()));
@@ -147,26 +152,33 @@ private void assertLegacyQuery(GeoDistanceQueryBuilder queryBuilder, Query query
         }
         assertThat(geoQuery.geoDistance(), equalTo(queryBuilder.geoDistance()));
         assertThat(geoQuery.minInclusiveDistance(), equalTo(Double.NEGATIVE_INFINITY));
-        double distance = queryBuilder.distance();
-        if (queryBuilder.geoDistance() != null) {
-            distance = queryBuilder.geoDistance().normalize(distance, DistanceUnit.DEFAULT);
-        }
-        assertThat(geoQuery.maxInclusiveDistance(), closeTo(distance, Math.abs(distance) / 1000));
+        assertThat(geoQuery.maxInclusiveDistance(), closeTo(queryBuilder.distance(), Math.abs(queryBuilder.distance()) / 1000));
     }
 
     private void assertGeoPointQuery(GeoDistanceQueryBuilder queryBuilder, Query query) throws IOException {
-        assertThat(query, instanceOf(GeoPointDistanceQuery.class));
-        GeoPointDistanceQuery geoQuery = (GeoPointDistanceQuery) query;
-        assertThat(geoQuery.getField(), equalTo(queryBuilder.fieldName()));
-        if (queryBuilder.point() != null) {
-            assertThat(geoQuery.getCenterLat(), equalTo(queryBuilder.point().lat()));
-            assertThat(geoQuery.getCenterLon(), equalTo(queryBuilder.point().lon()));
-        }
-        double distance = queryBuilder.distance();
-        if (queryBuilder.geoDistance() != null) {
-            distance = queryBuilder.geoDistance().normalize(distance, DistanceUnit.DEFAULT);
-            distance = org.elasticsearch.common.geo.GeoUtils.maxRadialDistance(queryBuilder.point(), distance);
-            assertThat(geoQuery.getRadiusMeters(), closeTo(distance, GeoUtils.TOLERANCE));
+        Version version = createShardContext().indexVersionCreated();
+        if (version.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            assertThat(query, instanceOf(GeoPointDistanceQuery.class));
+            GeoPointDistanceQuery geoQuery = (GeoPointDistanceQuery) query;
+            assertThat(geoQuery.getField(), equalTo(queryBuilder.fieldName()));
+            if (queryBuilder.point() != null) {
+                assertThat(geoQuery.getCenterLat(), equalTo(queryBuilder.point().lat()));
+                assertThat(geoQuery.getCenterLon(), equalTo(queryBuilder.point().lon()));
+            }
+            assertThat(geoQuery.getRadiusMeters(), closeTo(queryBuilder.distance(), GeoUtils.TOLERANCE));
+        } else if (query instanceof IndexOrDocValuesQuery) {
+            Query indexQuery = ((IndexOrDocValuesQuery) query).getIndexQuery();
+            assertEquals(LatLonPoint.newDistanceQuery(queryBuilder.fieldName(),
+                    queryBuilder.point().lat(),
+                    queryBuilder.point().lon(),
+                    queryBuilder.distance()),
+                    indexQuery);
+            Query dvQuery = ((IndexOrDocValuesQuery) query).getRandomAccessQuery();
+            assertEquals(LatLonDocValuesField.newDistanceQuery(queryBuilder.fieldName(),
+                    queryBuilder.point().lat(),
+                    queryBuilder.point().lon(),
+                    queryBuilder.distance()),
+                    dvQuery);
         }
     }
 
@@ -339,13 +351,13 @@ private void assertGeoDistanceRangeQuery(String query, double lat, double lon, d
         Query parsedQuery = parseQuery(query).toQuery(createShardContext());
         Version version = createShardContext().indexVersionCreated();
         if (version.before(Version.V_2_2_0)) {
-            GeoDistanceRangeQuery q = (GeoDistanceRangeQuery) parsedQuery;
+            LegacyGeoDistanceRangeQuery q = (LegacyGeoDistanceRangeQuery) parsedQuery;
             assertThat(q.fieldName(), equalTo(GEO_POINT_FIELD_NAME));
             assertThat(q.lat(), closeTo(lat, 1E-5D));
             assertThat(q.lon(), closeTo(lon, 1E-5D));
             assertThat(q.minInclusiveDistance(), equalTo(Double.NEGATIVE_INFINITY));
             assertThat(q.maxInclusiveDistance(), closeTo(distanceUnit.convert(distance, DistanceUnit.MILES), 1E-5D));
-        } else {
+        } else if (version.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
             GeoPointDistanceQuery q = (GeoPointDistanceQuery) parsedQuery;
             assertThat(q.getField(), equalTo(GEO_POINT_FIELD_NAME));
             assertThat(q.getCenterLat(), closeTo(lat, 1E-5D));
@@ -360,7 +372,7 @@ public void testFromJson() throws IOException {
                 "  \"geo_distance\" : {\n" +
                 "    \"pin.location\" : [ -70.0, 40.0 ],\n" +
                 "    \"distance\" : 12000.0,\n" +
-                "    \"distance_type\" : \"sloppy_arc\",\n" +
+                "    \"distance_type\" : \"arc\",\n" +
                 "    \"validation_method\" : \"STRICT\",\n" +
                 "    \"ignore_unmapped\" : false,\n" +
                 "    \"boost\" : 1.0\n" +
@@ -373,53 +385,70 @@ public void testFromJson() throws IOException {
         assertEquals(json, 12000.0, parsed.distance(), 0.0001);
     }
 
-    public void testOptimizeBboxFails() throws IOException {
+    public void testSloppyArcIsDeprecated() throws IOException {
         String json =
             "{\n" +
                 "  \"geo_distance\" : {\n" +
                 "    \"pin.location\" : [ -70.0, 40.0 ],\n" +
                 "    \"distance\" : 12000.0,\n" +
                 "    \"distance_type\" : \"sloppy_arc\",\n" +
+                "    \"validation_method\" : \"STRICT\",\n" +
+                "    \"ignore_unmapped\" : false,\n" +
+                "    \"boost\" : 1.0\n" +
+                "  }\n" +
+                "}";
+        parseQuery(json);
+        assertWarnings("[sloppy_arc] is deprecated. Use [arc] instead.");
+    }
+
+    public void testOptimizeBboxIsDeprecated() throws IOException {
+        String json =
+            "{\n" +
+                "  \"geo_distance\" : {\n" +
+                "    \"pin.location\" : [ -70.0, 40.0 ],\n" +
+                "    \"distance\" : 12000.0,\n" +
+                "    \"distance_type\" : \"arc\",\n" +
                 "    \"optimize_bbox\" : \"memory\",\n" +
                 "    \"validation_method\" : \"STRICT\",\n" +
                 "    \"ignore_unmapped\" : false,\n" +
                 "    \"boost\" : 1.0\n" +
                 "  }\n" +
                 "}";
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseQuery(json));
-        assertTrue(e.getMessage().startsWith("Deprecated field "));
+        parseQuery(json);
+        assertWarnings("Deprecated field [optimize_bbox] used, replaced by [no replacement: " +
+                "`optimize_bbox` is no longer supported due to recent improvements]");
     }
 
-    public void testFromCoerceFails() throws IOException {
+    public void testFromCoerceIsDeprecated() throws IOException {
         String json =
                 "{\n" +
                 "  \"geo_distance\" : {\n" +
                 "    \"pin.location\" : [ -70.0, 40.0 ],\n" +
                 "    \"distance\" : 12000.0,\n" +
-                "    \"distance_type\" : \"sloppy_arc\",\n" +
+                "    \"distance_type\" : \"arc\",\n" +
                 "    \"coerce\" : true,\n" +
                 "    \"ignore_unmapped\" : false,\n" +
                 "    \"boost\" : 1.0\n" +
                 "  }\n" +
                 "}";
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseQuery(json));
-        assertTrue(e.getMessage().startsWith("Deprecated field "));
+        parseQuery(json);
+        assertWarnings("Deprecated field [coerce] used, replaced by [validation_method]");
     }
 
-    public void testFromJsonIgnoreMalformedFails() throws IOException {
+    public void testFromJsonIgnoreMalformedIsDeprecated() throws IOException {
         String json =
                 "{\n" +
                 "  \"geo_distance\" : {\n" +
                 "    \"pin.location\" : [ -70.0, 40.0 ],\n" +
                 "    \"distance\" : 12000.0,\n" +
-                "    \"distance_type\" : \"sloppy_arc\",\n" +
+                "    \"distance_type\" : \"arc\",\n" +
                 "    \"ignore_malformed\" : true,\n" +
                 "    \"ignore_unmapped\" : false,\n" +
                 "    \"boost\" : 1.0\n" +
                 "  }\n" +
                 "}";
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseQuery(json));
-        assertTrue(e.getMessage().startsWith("Deprecated field "));
+        parseQuery(json);
+        assertWarnings("Deprecated field [ignore_malformed] used, replaced by [validation_method]");
     }
 
     @Override
diff --git a/core/src/test/java/org/elasticsearch/index/query/GeoDistanceRangeQueryTests.java b/core/src/test/java/org/elasticsearch/index/query/GeoDistanceRangeQueryTests.java
index 7b165d8a37a90..49d2c05883c0c 100644
--- a/core/src/test/java/org/elasticsearch/index/query/GeoDistanceRangeQueryTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/GeoDistanceRangeQueryTests.java
@@ -29,8 +29,10 @@
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.geo.GeoUtils;
 import org.elasticsearch.common.unit.DistanceUnit;
+import org.elasticsearch.index.mapper.LatLonPointFieldMapper;
 import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.search.geo.GeoDistanceRangeQuery;
+import org.elasticsearch.index.search.geo.LegacyGeoDistanceRangeQuery;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
 import org.elasticsearch.test.geo.RandomGeoGenerator;
 
@@ -117,9 +119,9 @@ protected GeoDistanceRangeQueryBuilder doCreateTestQueryBuilder() {
     }
 
     @Override
-    protected void doAssertLuceneQuery(GeoDistanceRangeQueryBuilder queryBuilder, Query query, QueryShardContext context)
+    protected void doAssertLuceneQuery(GeoDistanceRangeQueryBuilder queryBuilder, Query query, SearchContext context)
             throws IOException {
-        Version version = context.indexVersionCreated();
+        Version version = context.getQueryShardContext().indexVersionCreated();
         if (version.before(Version.V_2_2_0)) {
             assertLegacyQuery(queryBuilder, query);
         } else {
@@ -128,8 +130,8 @@ protected void doAssertLuceneQuery(GeoDistanceRangeQueryBuilder queryBuilder, Qu
     }
 
     private void assertLegacyQuery(GeoDistanceRangeQueryBuilder queryBuilder, Query query) throws IOException {
-        assertThat(query, instanceOf(GeoDistanceRangeQuery.class));
-        GeoDistanceRangeQuery geoQuery = (GeoDistanceRangeQuery) query;
+        assertThat(query, instanceOf(LegacyGeoDistanceRangeQuery.class));
+        LegacyGeoDistanceRangeQuery geoQuery = (LegacyGeoDistanceRangeQuery) query;
         assertThat(geoQuery.fieldName(), equalTo(queryBuilder.fieldName()));
         if (queryBuilder.point() != null) {
             GeoPoint expectedPoint = new GeoPoint(queryBuilder.point());
@@ -145,9 +147,6 @@ private void assertLegacyQuery(GeoDistanceRangeQueryBuilder queryBuilder, Query
             if (queryBuilder.unit() != null) {
                 fromValue = queryBuilder.unit().toMeters(fromValue);
             }
-            if (queryBuilder.geoDistance() != null) {
-                fromValue = queryBuilder.geoDistance().normalize(fromValue, DistanceUnit.DEFAULT);
-            }
             double fromSlop = Math.abs(fromValue) / 1000;
             if (queryBuilder.includeLower() == false) {
                 fromSlop = NumericUtils.sortableLongToDouble((NumericUtils.doubleToSortableLong(Math.abs(fromValue)) + 1L)) / 1000.0;
@@ -159,9 +158,6 @@ private void assertLegacyQuery(GeoDistanceRangeQueryBuilder queryBuilder, Query
             if (queryBuilder.unit() != null) {
                 toValue = queryBuilder.unit().toMeters(toValue);
             }
-            if (queryBuilder.geoDistance() != null) {
-                toValue = queryBuilder.geoDistance().normalize(toValue, DistanceUnit.DEFAULT);
-            }
             double toSlop = Math.abs(toValue) / 1000;
             if (queryBuilder.includeUpper() == false) {
                 toSlop = NumericUtils.sortableLongToDouble((NumericUtils.doubleToSortableLong(Math.abs(toValue)) - 1L)) / 1000.0;
@@ -204,7 +200,10 @@ private void assertGeoPointQuery(GeoDistanceRangeQueryBuilder queryBuilder, Quer
     @Override
     public void testToQuery() throws IOException {
         assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
-        super.testToQuery();
+        if (createShardContext().indexVersionCreated().before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            super.testToQuery();
+            assertWarnings("geo_distance_range search is deprecated. Use geo_distance aggregation or sort instead.");
+        }
     }
 
     public void testNullFieldName() {
@@ -254,6 +253,11 @@ public void testInvalidDistanceUnit() {
     }
 
     public void testNestedRangeQuery() throws IOException {
+        // geo distance range queries are no longer supported in 5.0 they are replaced by using aggregations or sort
+        if (createShardContext().indexVersionCreated().onOrAfter(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            return;
+        }
+
         // create a nested geo_point type with a subfield named "geohash" (explicit testing for ISSUE #15179)
         MapperService mapperService = createShardContext().getMapperService();
         String nestedMapping =
@@ -281,6 +285,7 @@ public void testNestedRangeQuery() throws IOException {
         NestedQueryBuilder builder = (NestedQueryBuilder) parseQuery(queryJson);
         QueryShardContext context = createShardContext();
         builder.toQuery(context);
+        assertWarnings("geo_distance_range search is deprecated. Use geo_distance aggregation or sort instead.");
     }
 
     public void testFromJson() throws IOException {
@@ -293,7 +298,7 @@ public void testFromJson() throws IOException {
                 "    \"include_lower\" : true,\n" +
                 "    \"include_upper\" : true,\n" +
                 "    \"unit\" : \"m\",\n" +
-                "    \"distance_type\" : \"sloppy_arc\",\n" +
+                "    \"distance_type\" : \"arc\",\n" +
                 "    \"validation_method\" : \"STRICT\",\n" +
                 "    \"ignore_unmapped\" : false,\n" +
                 "    \"boost\" : 1.0\n" +
@@ -314,14 +319,15 @@ public void testFromJsonOptimizeBboxFails() throws IOException {
                 "    \"include_lower\" : true,\n" +
                 "    \"include_upper\" : true,\n" +
                 "    \"unit\" : \"m\",\n" +
-                "    \"distance_type\" : \"sloppy_arc\",\n" +
+                "    \"distance_type\" : \"arc\",\n" +
                 "    \"optimize_bbox\" : \"memory\",\n" +
                 "    \"ignore_unmapped\" : false,\n" +
                 "    \"boost\" : 1.0\n" +
                 "  }\n" +
                 "}";
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseQuery(json));
-        assertTrue(e.getMessage().startsWith("Deprecated field "));
+        parseQuery(json);
+        assertWarnings("Deprecated field [optimize_bbox] used, replaced by [no replacement: `optimize_bbox` is no longer " +
+                "supported due to recent improvements]");
     }
 
     public void testFromJsonCoerceFails() throws IOException {
@@ -334,14 +340,14 @@ public void testFromJsonCoerceFails() throws IOException {
                 "    \"include_lower\" : true,\n" +
                 "    \"include_upper\" : true,\n" +
                 "    \"unit\" : \"m\",\n" +
-                "    \"distance_type\" : \"sloppy_arc\",\n" +
+                "    \"distance_type\" : \"arc\",\n" +
                 "    \"coerce\" : true,\n" +
                 "    \"ignore_unmapped\" : false,\n" +
                 "    \"boost\" : 1.0\n" +
                 "  }\n" +
                 "}";
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseQuery(json));
-        assertTrue(e.getMessage().startsWith("Deprecated field "));
+        parseQuery(json);
+        assertWarnings("Deprecated field [coerce] used, replaced by [use validation_method instead]");
     }
 
     public void testFromJsonIgnoreMalformedFails() throws IOException {
@@ -354,20 +360,23 @@ public void testFromJsonIgnoreMalformedFails() throws IOException {
                 "    \"include_lower\" : true,\n" +
                 "    \"include_upper\" : true,\n" +
                 "    \"unit\" : \"m\",\n" +
-                "    \"distance_type\" : \"sloppy_arc\",\n" +
+                "    \"distance_type\" : \"arc\",\n" +
                 "    \"ignore_malformed\" : true,\n" +
                 "    \"ignore_unmapped\" : false,\n" +
                 "    \"boost\" : 1.0\n" +
                 "  }\n" +
                 "}";
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseQuery(json));
-        assertTrue(e.getMessage().startsWith("Deprecated field "));
+        parseQuery(json);
+        assertWarnings("Deprecated field [ignore_malformed] used, replaced by [use validation_method instead]");
     }
 
     @Override
     public void testMustRewrite() throws IOException {
         assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
-        super.testMustRewrite();
+        if (createShardContext().indexVersionCreated().before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            super.testMustRewrite();
+            assertWarnings("geo_distance_range search is deprecated. Use geo_distance aggregation or sort instead.");
+        }
     }
 
     public void testIgnoreUnmapped() throws IOException {
diff --git a/core/src/test/java/org/elasticsearch/index/query/GeoPolygonQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/GeoPolygonQueryBuilderTests.java
index dc518b9b76a81..b3fcaed5f2453 100644
--- a/core/src/test/java/org/elasticsearch/index/query/GeoPolygonQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/GeoPolygonQueryBuilderTests.java
@@ -30,7 +30,9 @@
 import org.elasticsearch.common.geo.builders.ShapeBuilder;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.mapper.LatLonPointFieldMapper;
 import org.elasticsearch.index.search.geo.GeoPolygonQuery;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
 import org.elasticsearch.test.geo.RandomShapeGenerator;
 import org.elasticsearch.test.geo.RandomShapeGenerator.ShapeType;
@@ -64,13 +66,14 @@ protected GeoPolygonQueryBuilder doCreateTestQueryBuilder() {
     }
 
     @Override
-    protected void doAssertLuceneQuery(GeoPolygonQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
-        Version version = context.indexVersionCreated();
+    protected void doAssertLuceneQuery(GeoPolygonQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
+        Version version = context.getQueryShardContext().indexVersionCreated();
         if (version.before(Version.V_2_2_0)) {
             assertLegacyQuery(queryBuilder, query);
-        } else {
+        } else if (version.onOrAfter(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
             assertGeoPointQuery(queryBuilder, query);
         }
+        // todo LatLonPointInPolygon is package private
     }
 
     private void assertLegacyQuery(GeoPolygonQueryBuilder queryBuilder, Query query) {
@@ -121,7 +124,9 @@ private void assertGeoPointQuery(GeoPolygonQueryBuilder queryBuilder, Query quer
     @Override
     public void testToQuery() throws IOException {
         assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
-        super.testToQuery();
+        if (createShardContext().indexVersionCreated().before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            super.testToQuery();
+        }
     }
 
     private static List<GeoPoint> randomPolygon() {
@@ -189,8 +194,8 @@ public void testDeprecatedXContent() throws IOException {
         builder.field("normalize", true); // deprecated
         builder.endObject();
         builder.endObject();
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseQuery(builder.string()));
-        assertEquals("Deprecated field [normalize] used, replaced by [use validation_method instead]", e.getMessage());
+        parseQuery(builder.string());
+        assertWarnings("Deprecated field [normalize] used, replaced by [validation_method]");
     }
 
     public void testParsingAndToQueryParsingExceptions() throws IOException {
@@ -294,7 +299,7 @@ private void assertGeoPolygonQuery(String query) throws IOException {
             assertThat(filter.points()[1].lon(), closeTo(-80, 0.00001));
             assertThat(filter.points()[2].lat(), closeTo(20, 0.00001));
             assertThat(filter.points()[2].lon(), closeTo(-90, 0.00001));
-        } else {
+        } else if (version.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
             GeoPointInPolygonQuery q = (GeoPointInPolygonQuery) parsedQuery;
             assertThat(q.getField(), equalTo(GEO_POINT_FIELD_NAME));
             assertEquals(1, q.getPolygons().length);
@@ -310,6 +315,9 @@ private void assertGeoPolygonQuery(String query) throws IOException {
             assertThat(lons[2], closeTo(-90, 1E-5));
             assertThat(lats[3], equalTo(lats[0]));
             assertThat(lons[3], equalTo(lons[0]));
+        } else {
+            // todo LatLonPointInPolygon is package private, need a closeTo check on the query
+            // since some points can be computed from the geohash
         }
     }
 
@@ -341,9 +349,8 @@ public void testFromJsonIgnoreMalformedDeprecated() throws IOException {
                 "    \"boost\" : 1.0\n" +
                 "  }\n" +
                 "}";
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseQuery(json));
-        assertTrue(e.getMessage().startsWith("Deprecated field "));
-
+        parseQuery(json);
+        assertWarnings("Deprecated field [ignore_malformed] used, replaced by [validation_method]");
     }
 
     public void testFromJsonCoerceDeprecated() throws IOException {
@@ -358,8 +365,8 @@ public void testFromJsonCoerceDeprecated() throws IOException {
                 "    \"boost\" : 1.0\n" +
                 "  }\n" +
                 "}";
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseQuery(json));
-        assertTrue(e.getMessage().startsWith("Deprecated field "));
+        parseQuery(json);
+        assertWarnings("Deprecated field [coerce] used, replaced by [validation_method]");
     }
 
     @Override
diff --git a/core/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java
index 993e75724de7a..ddc4810b871a7 100644
--- a/core/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java
@@ -37,6 +37,7 @@
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.get.GetResult;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
 import org.elasticsearch.test.geo.RandomShapeGenerator;
 import org.elasticsearch.test.geo.RandomShapeGenerator.ShapeType;
@@ -69,15 +70,15 @@ protected GeoShapeQueryBuilder doCreateTestQueryBuilder() {
             builder = new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, shape);
         } else {
             indexedShapeToReturn = shape;
-            indexedShapeId = randomAsciiOfLengthBetween(3, 20);
-            indexedShapeType = randomAsciiOfLengthBetween(3, 20);
+            indexedShapeId = randomAlphaOfLengthBetween(3, 20);
+            indexedShapeType = randomAlphaOfLengthBetween(3, 20);
             builder = new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, indexedShapeId, indexedShapeType);
             if (randomBoolean()) {
-                indexedShapeIndex = randomAsciiOfLengthBetween(3, 20);
+                indexedShapeIndex = randomAlphaOfLengthBetween(3, 20);
                 builder.indexedShapeIndex(indexedShapeIndex);
             }
             if (randomBoolean()) {
-                indexedShapePath = randomAsciiOfLengthBetween(3, 20);
+                indexedShapePath = randomAlphaOfLengthBetween(3, 20);
                 builder.indexedShapePath(indexedShapePath);
             }
         }
@@ -133,7 +134,7 @@ public void clearShapeFields() {
     }
 
     @Override
-    protected void doAssertLuceneQuery(GeoShapeQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
+    protected void doAssertLuceneQuery(GeoShapeQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
         // Logic for doToQuery is complex and is hard to test here. Need to rely
         // on Integration tests to determine if created query is correct
         // TODO improve GeoShapeQueryBuilder.doToQuery() method to make it
diff --git a/core/src/test/java/org/elasticsearch/index/query/GeohashCellQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/GeohashCellQueryBuilderTests.java
index e9e4a71190d38..e66b70be42fcc 100644
--- a/core/src/test/java/org/elasticsearch/index/query/GeohashCellQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/GeohashCellQueryBuilderTests.java
@@ -20,15 +20,18 @@
 package org.elasticsearch.index.query;
 
 import org.apache.lucene.index.Term;
-import org.apache.lucene.queries.TermsQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermInSetQuery;
 import org.apache.lucene.search.TermQuery;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.unit.DistanceUnit;
 import org.elasticsearch.index.mapper.BaseGeoPointFieldMapper;
 import org.elasticsearch.index.mapper.GeoPointFieldMapper;
+import org.elasticsearch.index.mapper.LatLonPointFieldMapper;
 import org.elasticsearch.index.query.GeohashCellQuery.Builder;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
 import org.elasticsearch.test.geo.RandomShapeGenerator;
 import org.locationtech.spatial4j.shape.Point;
@@ -62,9 +65,9 @@ protected Builder doCreateTestQueryBuilder() {
     }
 
     @Override
-    protected void doAssertLuceneQuery(Builder queryBuilder, Query query, QueryShardContext context) throws IOException {
+    protected void doAssertLuceneQuery(Builder queryBuilder, Query query, SearchContext context) throws IOException {
         if (queryBuilder.neighbors()) {
-            assertThat(query, instanceOf(TermsQuery.class));
+            assertThat(query, instanceOf(TermInSetQuery.class));
         } else {
             assertThat(query, instanceOf(TermQuery.class));
             TermQuery termQuery = (TermQuery) query;
@@ -87,7 +90,10 @@ protected void doAssertLuceneQuery(Builder queryBuilder, Query query, QueryShard
     @Override
     public void testToQuery() throws IOException {
         assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
-        super.testToQuery();
+        Version version = createShardContext().indexVersionCreated();
+        if (version.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            super.testToQuery();
+        }
     }
 
     public void testNullField() {
@@ -140,7 +146,10 @@ public void testFromJson() throws IOException {
     @Override
     public void testMustRewrite() throws IOException {
         assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
-        super.testMustRewrite();
+        Version version = createShardContext().indexVersionCreated();
+        if (version.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            super.testMustRewrite();
+        }
     }
 
     public void testIgnoreUnmapped() throws IOException {
diff --git a/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java
deleted file mode 100644
index d26dd477a85b3..0000000000000
--- a/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java
+++ /dev/null
@@ -1,361 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.query;
-
-import com.carrotsearch.randomizedtesting.generators.RandomPicks;
-import org.apache.lucene.queries.TermsQuery;
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.ConstantScoreQuery;
-import org.apache.lucene.search.MatchNoDocsQuery;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.join.ScoreMode;
-import org.apache.lucene.search.similarities.PerFieldSimilarityWrapper;
-import org.apache.lucene.search.similarities.Similarity;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.compress.CompressedXContent;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.TypeFieldMapper;
-import org.elasticsearch.index.mapper.Uid;
-import org.elasticsearch.index.mapper.UidFieldMapper;
-import org.elasticsearch.index.similarity.SimilarityService;
-import org.elasticsearch.search.fetch.subphase.InnerHitsContext;
-import org.elasticsearch.search.internal.SearchContext;
-import org.elasticsearch.search.sort.FieldSortBuilder;
-import org.elasticsearch.search.sort.SortOrder;
-import org.elasticsearch.test.AbstractQueryTestCase;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Optional;
-
-import static org.hamcrest.CoreMatchers.containsString;
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.CoreMatchers.instanceOf;
-import static org.hamcrest.CoreMatchers.notNullValue;
-import static org.hamcrest.CoreMatchers.startsWith;
-
-public class HasChildQueryBuilderTests extends AbstractQueryTestCase<HasChildQueryBuilder> {
-    protected static final String PARENT_TYPE = "parent";
-    protected static final String CHILD_TYPE = "child";
-
-    private static String similarity;
-
-    boolean requiresRewrite = false;
-
-    @Override
-    protected void initializeAdditionalMappings(MapperService mapperService) throws IOException {
-        similarity = randomFrom("classic", "BM25");
-        mapperService.merge(PARENT_TYPE, new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef(PARENT_TYPE,
-                STRING_FIELD_NAME, "type=text",
-                STRING_FIELD_NAME_2, "type=keyword",
-                INT_FIELD_NAME, "type=integer",
-                DOUBLE_FIELD_NAME, "type=double",
-                BOOLEAN_FIELD_NAME, "type=boolean",
-                DATE_FIELD_NAME, "type=date",
-                OBJECT_FIELD_NAME, "type=object"
-        ).string()), MapperService.MergeReason.MAPPING_UPDATE, false);
-        mapperService.merge(CHILD_TYPE, new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef(CHILD_TYPE,
-                "_parent", "type=" + PARENT_TYPE,
-                STRING_FIELD_NAME, "type=text",
-                "custom_string", "type=text,similarity=" + similarity,
-                INT_FIELD_NAME, "type=integer",
-                DOUBLE_FIELD_NAME, "type=double",
-                BOOLEAN_FIELD_NAME, "type=boolean",
-                DATE_FIELD_NAME, "type=date",
-                OBJECT_FIELD_NAME, "type=object"
-        ).string()), MapperService.MergeReason.MAPPING_UPDATE, false);
-    }
-
-    /**
-     * @return a {@link HasChildQueryBuilder} with random values all over the place
-     */
-    @Override
-    protected HasChildQueryBuilder doCreateTestQueryBuilder() {
-        int min = randomIntBetween(0, Integer.MAX_VALUE / 2);
-        int max = randomIntBetween(min, Integer.MAX_VALUE);
-
-        QueryBuilder innerQueryBuilder = RandomQueryBuilder.createQuery(random());
-        if (randomBoolean()) {
-            requiresRewrite = true;
-            innerQueryBuilder = new WrapperQueryBuilder(innerQueryBuilder.toString());
-        }
-
-        HasChildQueryBuilder hqb = new HasChildQueryBuilder(CHILD_TYPE, innerQueryBuilder,
-                RandomPicks.randomFrom(random(), ScoreMode.values()));
-        hqb.minMaxChildren(min, max);
-        if (randomBoolean()) {
-            hqb.innerHit(new InnerHitBuilder()
-                    .setName(randomAsciiOfLengthBetween(1, 10))
-                    .setSize(randomIntBetween(0, 100))
-                    .addSort(new FieldSortBuilder(STRING_FIELD_NAME_2).order(SortOrder.ASC)));
-        }
-        hqb.ignoreUnmapped(randomBoolean());
-        return hqb;
-    }
-
-    @Override
-    protected void doAssertLuceneQuery(HasChildQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
-        assertThat(query, instanceOf(HasChildQueryBuilder.LateParsingQuery.class));
-        HasChildQueryBuilder.LateParsingQuery lpq = (HasChildQueryBuilder.LateParsingQuery) query;
-        assertEquals(queryBuilder.minChildren(), lpq.getMinChildren());
-        assertEquals(queryBuilder.maxChildren(), lpq.getMaxChildren());
-        assertEquals(queryBuilder.scoreMode(), lpq.getScoreMode()); // WTF is this why do we have two?
-        if (queryBuilder.innerHit() != null) {
-            // have to rewrite again because the provided queryBuilder hasn't been rewritten (directly returned from
-            // doCreateTestQueryBuilder)
-            queryBuilder = (HasChildQueryBuilder) queryBuilder.rewrite(context);
-            SearchContext searchContext = SearchContext.current();
-            assertNotNull(searchContext);
-            Map<String, InnerHitBuilder> innerHitBuilders = new HashMap<>();
-            InnerHitBuilder.extractInnerHits(queryBuilder, innerHitBuilders);
-            for (InnerHitBuilder builder : innerHitBuilders.values()) {
-                builder.build(searchContext, searchContext.innerHits());
-            }
-            assertNotNull(searchContext.innerHits());
-            assertEquals(1, searchContext.innerHits().getInnerHits().size());
-            assertTrue(searchContext.innerHits().getInnerHits().containsKey(queryBuilder.innerHit().getName()));
-            InnerHitsContext.BaseInnerHits innerHits =
-                    searchContext.innerHits().getInnerHits().get(queryBuilder.innerHit().getName());
-            assertEquals(innerHits.size(), queryBuilder.innerHit().getSize());
-            assertEquals(innerHits.sort().sort.getSort().length, 1);
-            assertEquals(innerHits.sort().sort.getSort()[0].getField(), STRING_FIELD_NAME_2);
-        }
-    }
-
-    public void testIllegalValues() {
-        QueryBuilder query = RandomQueryBuilder.createQuery(random());
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-                () -> QueryBuilders.hasChildQuery(null, query, ScoreMode.None));
-        assertEquals("[has_child] requires 'type' field", e.getMessage());
-
-        e = expectThrows(IllegalArgumentException.class, () -> QueryBuilders.hasChildQuery("foo", null, ScoreMode.None));
-        assertEquals("[has_child] requires 'query' field", e.getMessage());
-
-        e = expectThrows(IllegalArgumentException.class, () -> QueryBuilders.hasChildQuery("foo", query, null));
-        assertEquals("[has_child] requires 'score_mode' field", e.getMessage());
-
-        int positiveValue = randomIntBetween(0, Integer.MAX_VALUE);
-        HasChildQueryBuilder foo = QueryBuilders.hasChildQuery("foo", query, ScoreMode.None); // all good
-        e = expectThrows(IllegalArgumentException.class, () -> foo.minMaxChildren(randomIntBetween(Integer.MIN_VALUE, -1), positiveValue));
-        assertEquals("[has_child] requires non-negative 'min_children' field", e.getMessage());
-
-        e = expectThrows(IllegalArgumentException.class, () -> foo.minMaxChildren(positiveValue, randomIntBetween(Integer.MIN_VALUE, -1)));
-        assertEquals("[has_child] requires non-negative 'max_children' field", e.getMessage());
-
-        e = expectThrows(IllegalArgumentException.class, () -> foo.minMaxChildren(positiveValue, positiveValue - 10));
-        assertEquals("[has_child] 'max_children' is less than 'min_children'", e.getMessage());
-    }
-
-    public void testFromJson() throws IOException {
-        String query =
-                "{\n" +
-                "  \"has_child\" : {\n" +
-                "    \"query\" : {\n" +
-                "      \"range\" : {\n" +
-                "        \"mapped_string\" : {\n" +
-                "          \"from\" : \"agJhRET\",\n" +
-                "          \"to\" : \"zvqIq\",\n" +
-                "          \"include_lower\" : true,\n" +
-                "          \"include_upper\" : true,\n" +
-                "          \"boost\" : 1.0\n" +
-                "        }\n" +
-                "      }\n" +
-                "    },\n" +
-                "    \"type\" : \"child\",\n" +
-                "    \"score_mode\" : \"avg\",\n" +
-                "    \"min_children\" : 883170873,\n" +
-                "    \"max_children\" : 1217235442,\n" +
-                "    \"ignore_unmapped\" : false,\n" +
-                "    \"boost\" : 2.0,\n" +
-                "    \"_name\" : \"WNzYMJKRwePuRBh\",\n" +
-                "    \"inner_hits\" : {\n" +
-                "      \"name\" : \"inner_hits_name\",\n" +
-                "      \"from\" : 0,\n" +
-                "      \"size\" : 100,\n" +
-                "      \"version\" : false,\n" +
-                "      \"explain\" : false,\n" +
-                "      \"track_scores\" : false,\n" +
-                "      \"sort\" : [ {\n" +
-                "        \"mapped_string\" : {\n" +
-                "          \"order\" : \"asc\"\n" +
-                "        }\n" +
-                "      } ]\n" +
-                "    }\n" +
-                "  }\n" +
-                "}";
-        HasChildQueryBuilder queryBuilder = (HasChildQueryBuilder) parseQuery(query);
-        checkGeneratedJson(query, queryBuilder);
-        assertEquals(query, queryBuilder.maxChildren(), 1217235442);
-        assertEquals(query, queryBuilder.minChildren(), 883170873);
-        assertEquals(query, queryBuilder.boost(), 2.0f, 0.0f);
-        assertEquals(query, queryBuilder.queryName(), "WNzYMJKRwePuRBh");
-        assertEquals(query, queryBuilder.childType(), "child");
-        assertEquals(query, queryBuilder.scoreMode(), ScoreMode.Avg);
-        assertNotNull(query, queryBuilder.innerHit());
-        InnerHitBuilder expected = new InnerHitBuilder(new InnerHitBuilder(), queryBuilder.query(), "child")
-                .setName("inner_hits_name")
-                .setSize(100)
-                .addSort(new FieldSortBuilder("mapped_string").order(SortOrder.ASC));
-        assertEquals(query, queryBuilder.innerHit(), expected);
-    }
-
-    /**
-     * we resolve empty inner clauses by representing this whole query as empty optional upstream
-     */
-    public void testFromJsonEmptyQueryBody() throws IOException {
-        String query =  "{\n" +
-                "  \"has_child\" : {\n" +
-                "    \"query\" : { },\n" +
-                "    \"type\" : \"child\"" +
-                "   }" +
-                "}";
-        XContentParser parser = XContentFactory.xContent(query).createParser(query);
-        QueryParseContext context = createParseContext(parser, ParseFieldMatcher.EMPTY);
-        Optional<QueryBuilder> innerQueryBuilder = context.parseInnerQueryBuilder();
-        assertTrue(innerQueryBuilder.isPresent() == false);
-
-        parser = XContentFactory.xContent(query).createParser(query);
-        QueryParseContext otherContext = createParseContext(parser, ParseFieldMatcher.STRICT);
-        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> otherContext.parseInnerQueryBuilder());
-        assertThat(ex.getMessage(), startsWith("query malformed, empty clause found at"));
-    }
-
-    public void testToQueryInnerQueryType() throws IOException {
-        String[] searchTypes = new String[]{PARENT_TYPE};
-        QueryShardContext shardContext = createShardContext();
-        shardContext.setTypes(searchTypes);
-        HasChildQueryBuilder hasChildQueryBuilder = QueryBuilders.hasChildQuery(CHILD_TYPE, new IdsQueryBuilder().addIds("id"), ScoreMode.None);
-        Query query = hasChildQueryBuilder.toQuery(shardContext);
-        //verify that the context types are still the same as the ones we previously set
-        assertThat(shardContext.getTypes(), equalTo(searchTypes));
-        assertLateParsingQuery(query, CHILD_TYPE, "id");
-    }
-
-    static void assertLateParsingQuery(Query query, String type, String id) throws IOException {
-        assertThat(query, instanceOf(HasChildQueryBuilder.LateParsingQuery.class));
-        HasChildQueryBuilder.LateParsingQuery lateParsingQuery = (HasChildQueryBuilder.LateParsingQuery) query;
-        assertThat(lateParsingQuery.getInnerQuery(), instanceOf(BooleanQuery.class));
-        BooleanQuery booleanQuery = (BooleanQuery) lateParsingQuery.getInnerQuery();
-        assertThat(booleanQuery.clauses().size(), equalTo(2));
-        //check the inner ids query, we have to call rewrite to get to check the type it's executed against
-        assertThat(booleanQuery.clauses().get(0).getOccur(), equalTo(BooleanClause.Occur.MUST));
-        assertThat(booleanQuery.clauses().get(0).getQuery(), instanceOf(TermsQuery.class));
-        TermsQuery termsQuery = (TermsQuery) booleanQuery.clauses().get(0).getQuery();
-        Query rewrittenTermsQuery = termsQuery.rewrite(null);
-        assertThat(rewrittenTermsQuery, instanceOf(ConstantScoreQuery.class));
-        ConstantScoreQuery constantScoreQuery = (ConstantScoreQuery) rewrittenTermsQuery;
-        assertThat(constantScoreQuery.getQuery(), instanceOf(BooleanQuery.class));
-        BooleanQuery booleanTermsQuery = (BooleanQuery) constantScoreQuery.getQuery();
-        assertThat(booleanTermsQuery.clauses().toString(), booleanTermsQuery.clauses().size(), equalTo(1));
-        assertThat(booleanTermsQuery.clauses().get(0).getOccur(), equalTo(BooleanClause.Occur.SHOULD));
-        assertThat(booleanTermsQuery.clauses().get(0).getQuery(), instanceOf(TermQuery.class));
-        TermQuery termQuery = (TermQuery) booleanTermsQuery.clauses().get(0).getQuery();
-        assertThat(termQuery.getTerm().field(), equalTo(UidFieldMapper.NAME));
-        //we want to make sure that the inner ids query gets executed against the child type rather than the main type we initially set to the context
-        BytesRef[] ids = Uid.createUidsForTypesAndIds(Collections.singletonList(type), Collections.singletonList(id));
-        assertThat(termQuery.getTerm().bytes(), equalTo(ids[0]));
-        //check the type filter
-        assertThat(booleanQuery.clauses().get(1).getOccur(), equalTo(BooleanClause.Occur.FILTER));
-        assertEquals(new TypeFieldMapper.TypeQuery(new BytesRef(type)), booleanQuery.clauses().get(1).getQuery());
-    }
-
-    @Override
-    public void testMustRewrite() throws IOException {
-        try {
-            super.testMustRewrite();
-        } catch (UnsupportedOperationException e) {
-            if (requiresRewrite == false) {
-                throw e;
-            }
-        }
-    }
-
-    public void testNonDefaultSimilarity() throws Exception {
-        QueryShardContext shardContext = createShardContext();
-        HasChildQueryBuilder hasChildQueryBuilder = QueryBuilders.hasChildQuery(CHILD_TYPE, new TermQueryBuilder("custom_string", "value"), ScoreMode.None);
-        HasChildQueryBuilder.LateParsingQuery query = (HasChildQueryBuilder.LateParsingQuery) hasChildQueryBuilder.toQuery(shardContext);
-        Similarity expected = SimilarityService.BUILT_IN.get(similarity).apply(similarity, Settings.EMPTY).get();
-        assertThat(((PerFieldSimilarityWrapper) query.getSimilarity()).get("custom_string"), instanceOf(expected.getClass()));
-    }
-
-    public void testMinFromString() {
-        assertThat("fromString(min) != MIN", ScoreMode.Min, equalTo(HasChildQueryBuilder.parseScoreMode("min")));
-        assertThat("min", equalTo(HasChildQueryBuilder.scoreModeAsString(ScoreMode.Min)));
-    }
-
-    public void testMaxFromString() {
-        assertThat("fromString(max) != MAX", ScoreMode.Max, equalTo(HasChildQueryBuilder.parseScoreMode("max")));
-        assertThat("max", equalTo(HasChildQueryBuilder.scoreModeAsString(ScoreMode.Max)));
-    }
-
-    public void testAvgFromString() {
-        assertThat("fromString(avg) != AVG", ScoreMode.Avg, equalTo(HasChildQueryBuilder.parseScoreMode("avg")));
-        assertThat("avg", equalTo(HasChildQueryBuilder.scoreModeAsString(ScoreMode.Avg)));
-    }
-
-    public void testSumFromString() {
-        assertThat("fromString(total) != SUM", ScoreMode.Total, equalTo(HasChildQueryBuilder.parseScoreMode("sum")));
-        assertThat("sum", equalTo(HasChildQueryBuilder.scoreModeAsString(ScoreMode.Total)));
-    }
-
-    public void testNoneFromString() {
-        assertThat("fromString(none) != NONE", ScoreMode.None, equalTo(HasChildQueryBuilder.parseScoreMode("none")));
-        assertThat("none", equalTo(HasChildQueryBuilder.scoreModeAsString(ScoreMode.None)));
-    }
-
-    /**
-     * Should throw {@link IllegalArgumentException} instead of NPE.
-     */
-    public void testThatNullFromStringThrowsException() {
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> HasChildQueryBuilder.parseScoreMode(null));
-        assertEquals("No score mode for child query [null] found", e.getMessage());
-    }
-
-    /**
-     * Failure should not change (and the value should never match anything...).
-     */
-    public void testThatUnrecognizedFromStringThrowsException() {
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-                () -> HasChildQueryBuilder.parseScoreMode("unrecognized value"));
-        assertEquals("No score mode for child query [unrecognized value] found", e.getMessage());
-    }
-
-    public void testIgnoreUnmapped() throws IOException {
-        final HasChildQueryBuilder queryBuilder = new HasChildQueryBuilder("unmapped", new MatchAllQueryBuilder(), ScoreMode.None);
-        queryBuilder.ignoreUnmapped(true);
-        Query query = queryBuilder.toQuery(createShardContext());
-        assertThat(query, notNullValue());
-        assertThat(query, instanceOf(MatchNoDocsQuery.class));
-
-        final HasChildQueryBuilder failingQueryBuilder = new HasChildQueryBuilder("unmapped", new MatchAllQueryBuilder(), ScoreMode.None);
-        failingQueryBuilder.ignoreUnmapped(false);
-        QueryShardException e = expectThrows(QueryShardException.class, () -> failingQueryBuilder.toQuery(createShardContext()));
-        assertThat(e.getMessage(), containsString("[" + HasChildQueryBuilder.NAME + "] no mapping found for type [unmapped]"));
-    }
-}
diff --git a/core/src/test/java/org/elasticsearch/index/query/HasParentQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/HasParentQueryBuilderTests.java
deleted file mode 100644
index 14dedabd4bd9a..0000000000000
--- a/core/src/test/java/org/elasticsearch/index/query/HasParentQueryBuilderTests.java
+++ /dev/null
@@ -1,244 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.query;
-
-import org.apache.lucene.search.MatchNoDocsQuery;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.join.ScoreMode;
-import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.compress.CompressedXContent;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.search.fetch.subphase.InnerHitsContext;
-import org.elasticsearch.search.internal.SearchContext;
-import org.elasticsearch.search.sort.FieldSortBuilder;
-import org.elasticsearch.search.sort.SortOrder;
-import org.elasticsearch.test.AbstractQueryTestCase;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Optional;
-
-import static org.hamcrest.CoreMatchers.containsString;
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.CoreMatchers.instanceOf;
-import static org.hamcrest.CoreMatchers.notNullValue;
-import static org.hamcrest.CoreMatchers.startsWith;
-
-public class HasParentQueryBuilderTests extends AbstractQueryTestCase<HasParentQueryBuilder> {
-    protected static final String PARENT_TYPE = "parent";
-    protected static final String CHILD_TYPE = "child";
-
-    boolean requiresRewrite = false;
-
-    @Override
-    protected void initializeAdditionalMappings(MapperService mapperService) throws IOException {
-        mapperService.merge(PARENT_TYPE, new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef(PARENT_TYPE,
-                STRING_FIELD_NAME, "type=text",
-                STRING_FIELD_NAME_2, "type=keyword",
-                INT_FIELD_NAME, "type=integer",
-                DOUBLE_FIELD_NAME, "type=double",
-                BOOLEAN_FIELD_NAME, "type=boolean",
-                DATE_FIELD_NAME, "type=date",
-                OBJECT_FIELD_NAME, "type=object"
-        ).string()), MapperService.MergeReason.MAPPING_UPDATE, false);
-        mapperService.merge(CHILD_TYPE, new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef(CHILD_TYPE,
-                "_parent", "type=" + PARENT_TYPE,
-                STRING_FIELD_NAME, "type=text",
-                STRING_FIELD_NAME_2, "type=keyword",
-                INT_FIELD_NAME, "type=integer",
-                DOUBLE_FIELD_NAME, "type=double",
-                BOOLEAN_FIELD_NAME, "type=boolean",
-                DATE_FIELD_NAME, "type=date",
-                OBJECT_FIELD_NAME, "type=object"
-        ).string()), MapperService.MergeReason.MAPPING_UPDATE, false);
-        mapperService.merge("just_a_type", new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef("just_a_type"
-        ).string()), MapperService.MergeReason.MAPPING_UPDATE, false);
-    }
-
-    /**
-     * @return a {@link HasChildQueryBuilder} with random values all over the place
-     */
-    @Override
-    protected HasParentQueryBuilder doCreateTestQueryBuilder() {
-        QueryBuilder innerQueryBuilder = RandomQueryBuilder.createQuery(random());
-        if (randomBoolean()) {
-            requiresRewrite = true;
-            innerQueryBuilder = new WrapperQueryBuilder(innerQueryBuilder.toString());
-        }
-        HasParentQueryBuilder hqb = new HasParentQueryBuilder(PARENT_TYPE, innerQueryBuilder, randomBoolean());
-        if (randomBoolean()) {
-            hqb.innerHit(new InnerHitBuilder()
-                    .setName(randomAsciiOfLengthBetween(1, 10))
-                    .setSize(randomIntBetween(0, 100))
-                    .addSort(new FieldSortBuilder(STRING_FIELD_NAME_2).order(SortOrder.ASC)));
-        }
-        hqb.ignoreUnmapped(randomBoolean());
-        return hqb;
-    }
-
-    @Override
-    protected void doAssertLuceneQuery(HasParentQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
-        assertThat(query, instanceOf(HasChildQueryBuilder.LateParsingQuery.class));
-        HasChildQueryBuilder.LateParsingQuery lpq = (HasChildQueryBuilder.LateParsingQuery) query;
-        assertEquals(queryBuilder.score() ? ScoreMode.Max : ScoreMode.None, lpq.getScoreMode());
-
-        if (queryBuilder.innerHit() != null) {
-            // have to rewrite again because the provided queryBuilder hasn't been rewritten (directly returned from
-            // doCreateTestQueryBuilder)
-            queryBuilder = (HasParentQueryBuilder) queryBuilder.rewrite(context);
-
-            SearchContext searchContext = SearchContext.current();
-            assertNotNull(searchContext);
-            Map<String, InnerHitBuilder> innerHitBuilders = new HashMap<>();
-            InnerHitBuilder.extractInnerHits(queryBuilder, innerHitBuilders);
-            for (InnerHitBuilder builder : innerHitBuilders.values()) {
-                builder.build(searchContext, searchContext.innerHits());
-            }
-            assertNotNull(searchContext.innerHits());
-            assertEquals(1, searchContext.innerHits().getInnerHits().size());
-            assertTrue(searchContext.innerHits().getInnerHits().containsKey(queryBuilder.innerHit().getName()));
-            InnerHitsContext.BaseInnerHits innerHits = searchContext.innerHits()
-                    .getInnerHits().get(queryBuilder.innerHit().getName());
-            assertEquals(innerHits.size(), queryBuilder.innerHit().getSize());
-            assertEquals(innerHits.sort().sort.getSort().length, 1);
-            assertEquals(innerHits.sort().sort.getSort()[0].getField(), STRING_FIELD_NAME_2);
-        }
-    }
-
-    public void testIllegalValues() throws IOException {
-        QueryBuilder query = RandomQueryBuilder.createQuery(random());
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-                () -> QueryBuilders.hasParentQuery(null, query, false));
-        assertThat(e.getMessage(), equalTo("[has_parent] requires 'type' field"));
-
-        e = expectThrows(IllegalArgumentException.class,
-                () -> QueryBuilders.hasParentQuery("foo", null, false));
-        assertThat(e.getMessage(), equalTo("[has_parent] requires 'query' field"));
-
-        QueryShardContext context = createShardContext();
-        HasParentQueryBuilder qb = QueryBuilders.hasParentQuery("just_a_type", new MatchAllQueryBuilder(), false);
-        QueryShardException qse = expectThrows(QueryShardException.class, () -> qb.doToQuery(context));
-        assertThat(qse.getMessage(), equalTo("[has_parent] no child types found for type [just_a_type]"));
-    }
-
-    public void testDeprecatedXContent() throws IOException {
-        XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint();
-        builder.startObject();
-        builder.startObject("has_parent");
-        builder.field("query");
-        new TermQueryBuilder("a", "a").toXContent(builder, ToXContent.EMPTY_PARAMS);
-        builder.field("type", "foo"); // deprecated
-        builder.endObject();
-        builder.endObject();
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseQuery(builder.string()));
-        assertEquals("Deprecated field [type] used, expected [parent_type] instead", e.getMessage());
-
-        HasParentQueryBuilder queryBuilder = (HasParentQueryBuilder) parseQuery(builder.string(), ParseFieldMatcher.EMPTY);
-        assertEquals("foo", queryBuilder.type());
-    }
-
-    public void testToQueryInnerQueryType() throws IOException {
-        String[] searchTypes = new String[]{CHILD_TYPE};
-        QueryShardContext shardContext = createShardContext();
-        shardContext.setTypes(searchTypes);
-        HasParentQueryBuilder hasParentQueryBuilder = new HasParentQueryBuilder(PARENT_TYPE, new IdsQueryBuilder().addIds("id"),
-                false);
-        Query query = hasParentQueryBuilder.toQuery(shardContext);
-        //verify that the context types are still the same as the ones we previously set
-        assertThat(shardContext.getTypes(), equalTo(searchTypes));
-        HasChildQueryBuilderTests.assertLateParsingQuery(query, PARENT_TYPE, "id");
-    }
-
-    @Override
-    public void testMustRewrite() throws IOException {
-        try {
-            super.testMustRewrite();
-        } catch (UnsupportedOperationException e) {
-            if (requiresRewrite == false) {
-                throw e;
-            }
-        }
-    }
-
-    public void testFromJson() throws IOException {
-        String json =
-                "{\n" +
-                "  \"has_parent\" : {\n" +
-                "    \"query\" : {\n" +
-                "      \"term\" : {\n" +
-                "        \"tag\" : {\n" +
-                "          \"value\" : \"something\",\n" +
-                "          \"boost\" : 1.0\n" +
-                "        }\n" +
-                "      }\n" +
-                "    },\n" +
-                "    \"parent_type\" : \"blog\",\n" +
-                "    \"score\" : true,\n" +
-                "    \"ignore_unmapped\" : false,\n" +
-                "    \"boost\" : 1.0\n" +
-                "  }\n" +
-                "}";
-        HasParentQueryBuilder parsed = (HasParentQueryBuilder) parseQuery(json);
-        checkGeneratedJson(json, parsed);
-        assertEquals(json, "blog", parsed.type());
-        assertEquals(json, "something", ((TermQueryBuilder) parsed.query()).value());
-    }
-
-    /**
-     * we resolve empty inner clauses by representing this whole query as empty optional upstream
-     */
-    public void testFromJsonEmptyQueryBody() throws IOException {
-        String query =  "{\n" +
-                "  \"has_parent\" : {\n" +
-                "    \"query\" : { },\n" +
-                "    \"parent_type\" : \"blog\"" +
-                "   }" +
-                "}";
-        XContentParser parser = XContentFactory.xContent(query).createParser(query);
-        QueryParseContext context = createParseContext(parser, ParseFieldMatcher.EMPTY);
-        Optional<QueryBuilder> innerQueryBuilder = context.parseInnerQueryBuilder();
-        assertTrue(innerQueryBuilder.isPresent() == false);
-
-        parser = XContentFactory.xContent(query).createParser(query);
-        QueryParseContext otherContext = createParseContext(parser, ParseFieldMatcher.STRICT);
-        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> otherContext.parseInnerQueryBuilder());
-        assertThat(ex.getMessage(), startsWith("query malformed, empty clause found at"));
-    }
-
-    public void testIgnoreUnmapped() throws IOException {
-        final HasParentQueryBuilder queryBuilder = new HasParentQueryBuilder("unmapped", new MatchAllQueryBuilder(), false);
-        queryBuilder.ignoreUnmapped(true);
-        Query query = queryBuilder.toQuery(createShardContext());
-        assertThat(query, notNullValue());
-        assertThat(query, instanceOf(MatchNoDocsQuery.class));
-
-        final HasParentQueryBuilder failingQueryBuilder = new HasParentQueryBuilder("unmapped", new MatchAllQueryBuilder(), false);
-        failingQueryBuilder.ignoreUnmapped(false);
-        QueryShardException e = expectThrows(QueryShardException.class, () -> failingQueryBuilder.toQuery(createShardContext()));
-        assertThat(e.getMessage(),
-                    containsString("[" + HasParentQueryBuilder.NAME + "] query configured 'parent_type' [unmapped] is not a valid type"));
-    }
-}
diff --git a/core/src/test/java/org/elasticsearch/index/query/IdsQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/IdsQueryBuilderTests.java
index 723509d775ff2..604c6e5c35aa6 100644
--- a/core/src/test/java/org/elasticsearch/index/query/IdsQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/IdsQueryBuilderTests.java
@@ -19,29 +19,21 @@
 
 package org.elasticsearch.index.query;
 
-
-import org.apache.lucene.queries.TermsQuery;
+import org.apache.lucene.search.TermInSetQuery;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.MatchNoDocsQuery;
 import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
+import org.elasticsearch.index.mapper.UidFieldMapper;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
 
 import static org.hamcrest.CoreMatchers.instanceOf;
-import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.contains;
 
 public class IdsQueryBuilderTests extends AbstractQueryTestCase<IdsQueryBuilder> {
-    /**
-     * Check that parser throws exception on missing values field.
-     */
-    public void testIdsNotProvided() throws IOException {
-        String noIdsFieldQuery = "{\"ids\" : { \"type\" : \"my_type\"  }";
-        ParsingException e = expectThrows(ParsingException.class, () -> parseQuery(noIdsFieldQuery));
-        assertThat(e.getMessage(), containsString("no ids values provided"));
-    }
 
     @Override
     protected IdsQueryBuilder doCreateTestQueryBuilder() {
@@ -53,7 +45,7 @@ protected IdsQueryBuilder doCreateTestQueryBuilder() {
                 if (frequently()) {
                     types[i] = randomFrom(getCurrentTypes());
                 } else {
-                    types[i] = randomAsciiOfLengthBetween(1, 10);
+                    types[i] = randomAlphaOfLengthBetween(1, 10);
                 }
             }
         } else {
@@ -66,11 +58,11 @@ protected IdsQueryBuilder doCreateTestQueryBuilder() {
         int numberOfIds = randomIntBetween(0, 10);
         String[] ids = new String[numberOfIds];
         for (int i = 0; i < numberOfIds; i++) {
-            ids[i] = randomAsciiOfLengthBetween(1, 10);
+            ids[i] = randomAlphaOfLengthBetween(1, 10);
         }
         IdsQueryBuilder query;
         if (types.length > 0 || randomBoolean()) {
-            query = new IdsQueryBuilder(types);
+            query = new IdsQueryBuilder().types(types);
             query.addIds(ids);
         } else {
             query = new IdsQueryBuilder();
@@ -80,20 +72,20 @@ protected IdsQueryBuilder doCreateTestQueryBuilder() {
     }
 
     @Override
-    protected void doAssertLuceneQuery(IdsQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
-        if (queryBuilder.ids().size() == 0) {
+    protected void doAssertLuceneQuery(IdsQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
+        if (queryBuilder.ids().size() == 0 || context.getQueryShardContext().fieldMapper(UidFieldMapper.NAME) == null) {
             assertThat(query, instanceOf(MatchNoDocsQuery.class));
         } else {
-            assertThat(query, instanceOf(TermsQuery.class));
+            assertThat(query, instanceOf(TermInSetQuery.class));
         }
     }
 
     public void testIllegalArguments() {
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new IdsQueryBuilder((String[]) null));
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new IdsQueryBuilder().types((String[]) null));
         assertEquals("[ids] types cannot be null", e.getMessage());
 
         IdsQueryBuilder idsQueryBuilder = new IdsQueryBuilder();
-        e = expectThrows(IllegalArgumentException.class, () -> idsQueryBuilder.addIds((String[])null));
+        e = expectThrows(IllegalArgumentException.class, () -> idsQueryBuilder.addIds((String[]) null));
         assertEquals("[ids] ids cannot be null", e.getMessage());
     }
 
@@ -101,7 +93,7 @@ public void testIllegalArguments() {
     public void testIdsQueryWithInvalidValues() throws Exception {
         String query = "{ \"ids\": { \"values\": [[1]] } }";
         ParsingException e = expectThrows(ParsingException.class, () -> parseQuery(query));
-        assertEquals("Illegal value for id, expecting a string or number, got: START_ARRAY", e.getMessage());
+        assertEquals("[ids] failed to parse field [values]", e.getMessage());
     }
 
     public void testFromJson() throws IOException {
@@ -115,42 +107,76 @@ public void testFromJson() throws IOException {
                 "}";
         IdsQueryBuilder parsed = (IdsQueryBuilder) parseQuery(json);
         checkGeneratedJson(json, parsed);
-        assertEquals(json, 3, parsed.ids().size());
+        assertThat(parsed.ids(), contains("1","100","4"));
+        assertEquals(json, "my_type", parsed.types()[0]);
+
+        // check that type that is not an array and also ids that are numbers are parsed
+        json =
+                "{\n" +
+                "  \"ids\" : {\n" +
+                "    \"type\" : \"my_type\",\n" +
+                "    \"values\" : [ 1, 100, 4 ],\n" +
+                "    \"boost\" : 1.0\n" +
+                "  }\n" +
+                "}";
+        parsed = (IdsQueryBuilder) parseQuery(json);
+        assertThat(parsed.ids(), contains("1","100","4"));
         assertEquals(json, "my_type", parsed.types()[0]);
+
+        // check with empty type array
+        json =
+                "{\n" +
+                "  \"ids\" : {\n" +
+                "    \"type\" : [ ],\n" +
+                "    \"values\" : [ \"1\", \"100\", \"4\" ],\n" +
+                "    \"boost\" : 1.0\n" +
+                "  }\n" +
+                "}";
+        parsed = (IdsQueryBuilder) parseQuery(json);
+        assertThat(parsed.ids(), contains("1","100","4"));
+        assertEquals(json, 0, parsed.types().length);
+
+        // check without type
+        json =
+                "{\n" +
+                "  \"ids\" : {\n" +
+                "    \"values\" : [ \"1\", \"100\", \"4\" ],\n" +
+                "    \"boost\" : 1.0\n" +
+                "  }\n" +
+                "}";
+        parsed = (IdsQueryBuilder) parseQuery(json);
+        assertThat(parsed.ids(), contains("1","100","4"));
+        assertEquals(json, 0, parsed.types().length);
     }
 
     public void testFromJsonDeprecatedSyntax() throws IOException {
-        IdsQueryBuilder tempQuery = createTestQueryBuilder();
-        assumeTrue("test requires at least one type", tempQuery.types() != null && tempQuery.types().length > 0);
-
-        String type = tempQuery.types()[0];
-        IdsQueryBuilder testQuery = new IdsQueryBuilder(type);
+        IdsQueryBuilder testQuery = new IdsQueryBuilder().types("my_type");
 
         //single value type can also be called _type
         final String contentString = "{\n" +
                 "    \"ids\" : {\n" +
-                "        \"_type\" : \"" + type + "\",\n" +
-                "        \"values\" : []\n" +
+                "        \"_type\" : \"my_type\",\n" +
+                "        \"values\" : [ ]\n" +
                 "    }\n" +
                 "}";
 
-        IdsQueryBuilder parsed = (IdsQueryBuilder) parseQuery(contentString, ParseFieldMatcher.EMPTY);
+        IdsQueryBuilder parsed = (IdsQueryBuilder) parseQuery(contentString);
         assertEquals(testQuery, parsed);
 
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseQuery(contentString));
-        assertEquals("Deprecated field [_type] used, expected [type] instead", e.getMessage());
+        parseQuery(contentString);
+        assertWarnings("Deprecated field [_type] used, expected [type] instead");
 
-        //array of types can also be called type rather than types
+        //array of types can also be called types rather than type
         final String contentString2 = "{\n" +
                 "    \"ids\" : {\n" +
-                "        \"types\" : [\"" + type + "\"],\n" +
-                "        \"values\" : []\n" +
+                "        \"types\" : [\"my_type\"],\n" +
+                "        \"values\" : [ ]\n" +
                 "    }\n" +
                 "}";
-        parsed = (IdsQueryBuilder) parseQuery(contentString, ParseFieldMatcher.EMPTY);
+        parsed = (IdsQueryBuilder) parseQuery(contentString2);
         assertEquals(testQuery, parsed);
 
-        e = expectThrows(IllegalArgumentException.class, () -> parseQuery(contentString2));
-        assertEquals("Deprecated field [types] used, expected [type] instead", e.getMessage());
+        parseQuery(contentString2);
+        assertWarnings("Deprecated field [types] used, expected [type] instead");
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/query/IndicesQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/IndicesQueryBuilderTests.java
index d1c2dcee90c38..5d2816e43eab7 100644
--- a/core/src/test/java/org/elasticsearch/index/query/IndicesQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/IndicesQueryBuilderTests.java
@@ -20,12 +20,23 @@
 package org.elasticsearch.index.query;
 
 import org.apache.lucene.search.Query;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
+import org.junit.After;
 
 import java.io.IOException;
 
 public class IndicesQueryBuilderTests extends AbstractQueryTestCase<IndicesQueryBuilder> {
 
+    /**
+     * All tests create deprecation warnings when an new {@link IndicesQueryBuilder} is created.
+     * Instead of having to check them once in every single test, this is done here after each test is run
+     */
+    @After
+    void assertWarnings() throws IOException {
+        assertWarnings("indices query is deprecated. Instead search on the '_index' field");
+    }
+
     @Override
     protected IndicesQueryBuilder doCreateTestQueryBuilder() {
         String[] indices;
@@ -50,12 +61,12 @@ protected IndicesQueryBuilder doCreateTestQueryBuilder() {
     }
 
     @Override
-    protected void doAssertLuceneQuery(IndicesQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
+    protected void doAssertLuceneQuery(IndicesQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
         Query expected;
         if (queryBuilder.indices().length == 1 && getIndex().getName().equals(queryBuilder.indices()[0])) {
-            expected = queryBuilder.innerQuery().toQuery(context);
+            expected = queryBuilder.innerQuery().toQuery(context.getQueryShardContext());
         } else {
-            expected = queryBuilder.noMatchQuery().toQuery(context);
+            expected = queryBuilder.noMatchQuery().toQuery(context.getQueryShardContext());
         }
         assertEquals(expected, query);
     }
diff --git a/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java
index 5f8c847e41ec2..20b0fc1e16d65 100644
--- a/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java
@@ -18,27 +18,23 @@
  */
 package org.elasticsearch.index.query;
 
-import org.apache.lucene.search.join.ScoreMode;
-import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilderTests;
+import org.elasticsearch.search.internal.ShardSearchLocalRequest;
 import org.elasticsearch.search.sort.SortBuilder;
 import org.elasticsearch.search.sort.SortBuilders;
 import org.elasticsearch.search.sort.SortOrder;
@@ -56,28 +52,33 @@
 import java.util.function.Supplier;
 
 import static java.util.Collections.emptyList;
+import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.not;
-import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.sameInstance;
 
 public class InnerHitBuilderTests extends ESTestCase {
 
     private static final int NUMBER_OF_TESTBUILDERS = 20;
     private static NamedWriteableRegistry namedWriteableRegistry;
-    private static IndicesQueriesRegistry indicesQueriesRegistry;
+    private static NamedXContentRegistry xContentRegistry;
 
     @BeforeClass
     public static void init() {
         SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList());
         namedWriteableRegistry = new NamedWriteableRegistry(searchModule.getNamedWriteables());
-        indicesQueriesRegistry = searchModule.getQueryParserRegistry();
+        xContentRegistry = new NamedXContentRegistry(searchModule.getNamedXContents());
     }
 
     @AfterClass
     public static void afterClass() throws Exception {
         namedWriteableRegistry = null;
-        indicesQueriesRegistry = null;
+        xContentRegistry = null;
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return xContentRegistry;
     }
 
     public void testSerialization() throws Exception {
@@ -90,18 +91,39 @@ public void testSerialization() throws Exception {
         }
     }
 
+    /**
+     * Test that if we serialize and deserialize an object, further
+     * serialization leads to identical bytes representation.
+     *
+     * This is necessary to ensure because we use the serialized BytesReference
+     * of this builder as part of the cacheKey in
+     * {@link ShardSearchLocalRequest} (via
+     * {@link SearchSourceBuilder#collapse(org.elasticsearch.search.collapse.CollapseBuilder)})
+     */
+    public void testSerializationOrder() throws Exception {
+        for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) {
+            InnerHitBuilder original = randomInnerHits();
+            InnerHitBuilder deserialized = serializedCopy(original);
+            assertEquals(deserialized, original);
+            assertEquals(deserialized.hashCode(), original.hashCode());
+            assertNotSame(deserialized, original);
+            BytesStreamOutput out1 = new BytesStreamOutput();
+            BytesStreamOutput out2 = new BytesStreamOutput();
+            original.writeTo(out1);
+            deserialized.writeTo(out2);
+            assertEquals(out1.bytes(), out2.bytes());
+        }
+    }
+
     public void testFromAndToXContent() throws Exception {
         for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) {
-            InnerHitBuilder innerHit = randomInnerHits(true, false);
+            InnerHitBuilder innerHit = randomInnerHits();
             XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
             innerHit.toXContent(builder, ToXContent.EMPTY_PARAMS);
-            XContentBuilder shuffled = shuffleXContent(builder);
-            if (randomBoolean()) {
-                shuffled.prettyPrint();
-            }
-
-            XContentParser parser = XContentHelper.createParser(shuffled.bytes());
-            QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.EMPTY);
+            //fields is printed out as an object but parsed into a List where order matters, we disable shuffling
+            XContentBuilder shuffled = shuffleXContent(builder, "fields");
+            XContentParser parser = createParser(shuffled);
+            QueryParseContext context = new QueryParseContext(parser);
             InnerHitBuilder secondInnerHits = InnerHitBuilder.fromXContent(context);
             assertThat(innerHit, not(sameInstance(secondInnerHits)));
             assertThat(innerHit, equalTo(secondInnerHits));
@@ -109,123 +131,24 @@ public void testFromAndToXContent() throws Exception {
         }
     }
 
-    public void testEqualsAndHashcode() throws IOException {
+    public void testEqualsAndHashcode() {
         for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) {
-            InnerHitBuilder firstInnerHit = randomInnerHits();
-            assertFalse("inner hit is equal to null", firstInnerHit.equals(null));
-            assertFalse("inner hit is equal to incompatible type", firstInnerHit.equals(""));
-            assertTrue("inner it is not equal to self", firstInnerHit.equals(firstInnerHit));
-            assertThat("same inner hit's hashcode returns different values if called multiple times", firstInnerHit.hashCode(),
-                    equalTo(firstInnerHit.hashCode()));
-            assertThat("different inner hits should not be equal", mutate(serializedCopy(firstInnerHit)), not(equalTo(firstInnerHit)));
-
-            InnerHitBuilder secondBuilder = serializedCopy(firstInnerHit);
-            assertTrue("inner hit is not equal to self", secondBuilder.equals(secondBuilder));
-            assertTrue("inner hit is not equal to its copy", firstInnerHit.equals(secondBuilder));
-            assertTrue("equals is not symmetric", secondBuilder.equals(firstInnerHit));
-            assertThat("inner hits copy's hashcode is different from original hashcode", secondBuilder.hashCode(),
-                    equalTo(firstInnerHit.hashCode()));
-
-            InnerHitBuilder thirdBuilder = serializedCopy(secondBuilder);
-            assertTrue("inner hit is not equal to self", thirdBuilder.equals(thirdBuilder));
-            assertTrue("inner hit is not equal to its copy", secondBuilder.equals(thirdBuilder));
-            assertThat("inner hit copy's hashcode is different from original hashcode", secondBuilder.hashCode(),
-                    equalTo(thirdBuilder.hashCode()));
-            assertTrue("equals is not transitive", firstInnerHit.equals(thirdBuilder));
-            assertThat("inner hit copy's hashcode is different from original hashcode", firstInnerHit.hashCode(),
-                    equalTo(thirdBuilder.hashCode()));
-            assertTrue("equals is not symmetric", thirdBuilder.equals(secondBuilder));
-            assertTrue("equals is not symmetric", thirdBuilder.equals(firstInnerHit));
+            checkEqualsAndHashCode(randomInnerHits(), InnerHitBuilderTests::serializedCopy, InnerHitBuilderTests::mutate);
         }
     }
 
-    public void testInlineLeafInnerHitsNestedQuery() throws Exception {
-        InnerHitBuilder leafInnerHits = randomInnerHits();
-        NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None);
-        nestedQueryBuilder.innerHit(leafInnerHits);
-        Map<String, InnerHitBuilder> innerHitBuilders = new HashMap<>();
-        nestedQueryBuilder.extractInnerHitBuilders(innerHitBuilders);
-        assertThat(innerHitBuilders.get(leafInnerHits.getName()), notNullValue());
-    }
-
-    public void testInlineLeafInnerHitsHasChildQuery() throws Exception {
-        InnerHitBuilder leafInnerHits = randomInnerHits();
-        HasChildQueryBuilder hasChildQueryBuilder = new HasChildQueryBuilder("type", new MatchAllQueryBuilder(), ScoreMode.None)
-                .innerHit(leafInnerHits);
-        Map<String, InnerHitBuilder> innerHitBuilders = new HashMap<>();
-        hasChildQueryBuilder.extractInnerHitBuilders(innerHitBuilders);
-        assertThat(innerHitBuilders.get(leafInnerHits.getName()), notNullValue());
-    }
-
-    public void testInlineLeafInnerHitsHasParentQuery() throws Exception {
-        InnerHitBuilder leafInnerHits = randomInnerHits();
-        HasParentQueryBuilder hasParentQueryBuilder = new HasParentQueryBuilder("type", new MatchAllQueryBuilder(), false)
-                .innerHit(leafInnerHits);
-        Map<String, InnerHitBuilder> innerHitBuilders = new HashMap<>();
-        hasParentQueryBuilder.extractInnerHitBuilders(innerHitBuilders);
-        assertThat(innerHitBuilders.get(leafInnerHits.getName()), notNullValue());
-    }
-
-    public void testInlineLeafInnerHitsNestedQueryViaBoolQuery() {
-        InnerHitBuilder leafInnerHits = randomInnerHits();
-        NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None)
-                .innerHit(leafInnerHits);
-        BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder().should(nestedQueryBuilder);
-        Map<String, InnerHitBuilder> innerHitBuilders = new HashMap<>();
-        boolQueryBuilder.extractInnerHitBuilders(innerHitBuilders);
-        assertThat(innerHitBuilders.get(leafInnerHits.getName()), notNullValue());
-    }
-
-    public void testInlineLeafInnerHitsNestedQueryViaConstantScoreQuery() {
-        InnerHitBuilder leafInnerHits = randomInnerHits();
-        NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None)
-                .innerHit(leafInnerHits);
-        ConstantScoreQueryBuilder constantScoreQueryBuilder = new ConstantScoreQueryBuilder(nestedQueryBuilder);
-        Map<String, InnerHitBuilder> innerHitBuilders = new HashMap<>();
-        constantScoreQueryBuilder.extractInnerHitBuilders(innerHitBuilders);
-        assertThat(innerHitBuilders.get(leafInnerHits.getName()), notNullValue());
-    }
-
-    public void testInlineLeafInnerHitsNestedQueryViaBoostingQuery() {
-        InnerHitBuilder leafInnerHits1 = randomInnerHits();
-        NestedQueryBuilder nestedQueryBuilder1 = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None)
-                .innerHit(leafInnerHits1);
-        InnerHitBuilder leafInnerHits2 = randomInnerHits();
-        NestedQueryBuilder nestedQueryBuilder2 = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None)
-                .innerHit(leafInnerHits2);
-        BoostingQueryBuilder constantScoreQueryBuilder = new BoostingQueryBuilder(nestedQueryBuilder1, nestedQueryBuilder2);
-        Map<String, InnerHitBuilder> innerHitBuilders = new HashMap<>();
-        constantScoreQueryBuilder.extractInnerHitBuilders(innerHitBuilders);
-        assertThat(innerHitBuilders.get(leafInnerHits1.getName()), notNullValue());
-        assertThat(innerHitBuilders.get(leafInnerHits2.getName()), notNullValue());
-    }
-
-    public void testInlineLeafInnerHitsNestedQueryViaFunctionScoreQuery() {
-        InnerHitBuilder leafInnerHits = randomInnerHits();
-        NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None)
-                .innerHit(leafInnerHits);
-        FunctionScoreQueryBuilder functionScoreQueryBuilder = new FunctionScoreQueryBuilder(nestedQueryBuilder);
-        Map<String, InnerHitBuilder> innerHitBuilders = new HashMap<>();
-        ((AbstractQueryBuilder<?>) functionScoreQueryBuilder).extractInnerHitBuilders(innerHitBuilders);
-        assertThat(innerHitBuilders.get(leafInnerHits.getName()), notNullValue());
-    }
-
     public static InnerHitBuilder randomInnerHits() {
-        return randomInnerHits(true, true);
-    }
-
-    public static InnerHitBuilder randomInnerHits(boolean recursive, boolean includeQueryTypeOrPath) {
         InnerHitBuilder innerHits = new InnerHitBuilder();
-        innerHits.setName(randomAsciiOfLengthBetween(1, 16));
+        innerHits.setName(randomAlphaOfLengthBetween(1, 16));
         innerHits.setFrom(randomIntBetween(0, 128));
         innerHits.setSize(randomIntBetween(0, 128));
         innerHits.setExplain(randomBoolean());
         innerHits.setVersion(randomBoolean());
         innerHits.setTrackScores(randomBoolean());
         if (randomBoolean()) {
-            innerHits.setStoredFieldNames(randomListStuff(16, () -> randomAsciiOfLengthBetween(1, 16)));
+            innerHits.setStoredFieldNames(randomListStuff(16, () -> randomAlphaOfLengthBetween(1, 16)));
         }
-        innerHits.setDocValueFields(randomListStuff(16, () -> randomAsciiOfLengthBetween(1, 16)));
+        innerHits.setDocValueFields(randomListStuff(16, () -> randomAlphaOfLengthBetween(1, 16)));
         // Random script fields deduped on their field name.
         Map<String, SearchSourceBuilder.ScriptField> scriptFields = new HashMap<>();
         for (SearchSourceBuilder.ScriptField field: randomListStuff(16, InnerHitBuilderTests::randomScript)) {
@@ -233,148 +156,106 @@ public static InnerHitBuilder randomInnerHits(boolean recursive, boolean include
         }
         innerHits.setScriptFields(new HashSet<>(scriptFields.values()));
         FetchSourceContext randomFetchSourceContext;
-        if (randomBoolean()) {
-            randomFetchSourceContext = new FetchSourceContext(randomBoolean());
-        } else {
-            randomFetchSourceContext = new FetchSourceContext(
+        int randomInt = randomIntBetween(0, 2);
+        if (randomInt == 0) {
+            randomFetchSourceContext = new FetchSourceContext(true, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY);
+        } else if (randomInt == 1) {
+            randomFetchSourceContext = new FetchSourceContext(true,
                     generateRandomStringArray(12, 16, false),
                     generateRandomStringArray(12, 16, false)
             );
+        } else {
+            randomFetchSourceContext = new FetchSourceContext(randomBoolean());
         }
         innerHits.setFetchSourceContext(randomFetchSourceContext);
         if (randomBoolean()) {
             innerHits.setSorts(randomListStuff(16,
-                    () -> SortBuilders.fieldSort(randomAsciiOfLengthBetween(5, 20)).order(randomFrom(SortOrder.values())))
+                    () -> SortBuilders.fieldSort(randomAlphaOfLengthBetween(5, 20)).order(randomFrom(SortOrder.values())))
             );
         }
         innerHits.setHighlightBuilder(HighlightBuilderTests.randomHighlighterBuilder());
-        if (recursive && randomBoolean()) {
-            int size = randomIntBetween(1, 16);
-            for (int i = 0; i < size; i++) {
-                innerHits.addChildInnerHit(randomInnerHits(false, includeQueryTypeOrPath));
-            }
-        }
+        return innerHits;
+    }
 
-        if (includeQueryTypeOrPath) {
-            QueryBuilder query = new MatchQueryBuilder(randomAsciiOfLengthBetween(1, 16), randomAsciiOfLengthBetween(1, 16));
+    static InnerHitBuilder mutate(InnerHitBuilder original) throws IOException {
+        final InnerHitBuilder copy = serializedCopy(original);
+        List<Runnable> modifiers = new ArrayList<>(12);
+        modifiers.add(() -> copy.setFrom(randomValueOtherThan(copy.getFrom(), () -> randomIntBetween(0, 128))));
+        modifiers.add(() -> copy.setSize(randomValueOtherThan(copy.getSize(), () -> randomIntBetween(0, 128))));
+        modifiers.add(() -> copy.setExplain(!copy.isExplain()));
+        modifiers.add(() -> copy.setVersion(!copy.isVersion()));
+        modifiers.add(() -> copy.setTrackScores(!copy.isTrackScores()));
+        modifiers.add(() -> copy.setName(randomValueOtherThan(copy.getName(), () -> randomAlphaOfLengthBetween(1, 16))));
+        modifiers.add(() -> {
             if (randomBoolean()) {
-                return new InnerHitBuilder(innerHits, randomAsciiOfLength(8), query);
+                copy.setDocValueFields(randomValueOtherThan(copy.getDocValueFields(), () -> {
+                    return randomListStuff(16, () -> randomAlphaOfLengthBetween(1, 16));
+                }));
             } else {
-                return new InnerHitBuilder(innerHits, query, randomAsciiOfLength(8));
+                copy.addDocValueField(randomAlphaOfLengthBetween(1, 16));
             }
-        } else {
-            return innerHits;
-        }
-    }
-
-    public void testCopyConstructor() throws Exception {
-        InnerHitBuilder original = randomInnerHits();
-        InnerHitBuilder copy = original.getNestedPath() != null ?
-                new InnerHitBuilder(original, original.getNestedPath(), original.getQuery()) :
-                new InnerHitBuilder(original, original.getQuery(), original.getParentChildType());
-        assertThat(copy, equalTo(original));
-        copy = mutate(copy);
-        assertThat(copy, not(equalTo(original)));
-    }
-
-    static InnerHitBuilder mutate(InnerHitBuilder instance) throws IOException {
-        int surprise = randomIntBetween(0, 11);
-        switch (surprise) {
-            case 0:
-                instance.setFrom(randomValueOtherThan(instance.getFrom(), () -> randomIntBetween(0, 128)));
-                break;
-            case 1:
-                instance.setSize(randomValueOtherThan(instance.getSize(), () -> randomIntBetween(0, 128)));
-                break;
-            case 2:
-                instance.setExplain(!instance.isExplain());
-                break;
-            case 3:
-                instance.setVersion(!instance.isVersion());
-                break;
-            case 4:
-                instance.setTrackScores(!instance.isTrackScores());
-                break;
-            case 5:
-                instance.setName(randomValueOtherThan(instance.getName(), () -> randomAsciiOfLengthBetween(1, 16)));
-                break;
-            case 6:
-                if (randomBoolean()) {
-                    instance.setDocValueFields(randomValueOtherThan(instance.getDocValueFields(), () -> {
-                        return randomListStuff(16, () -> randomAsciiOfLengthBetween(1, 16));
-                    }));
-                } else {
-                    instance.addDocValueField(randomAsciiOfLengthBetween(1, 16));
-                }
-                break;
-            case 7:
-                if (randomBoolean()) {
-                    instance.setScriptFields(randomValueOtherThan(instance.getScriptFields(), () -> {
-                        return new HashSet<>(randomListStuff(16, InnerHitBuilderTests::randomScript));}));
-                } else {
-                    SearchSourceBuilder.ScriptField script = randomScript();
-                    instance.addScriptField(script.fieldName(), script.script());
-                }
-                break;
-            case 8:
-                instance.setFetchSourceContext(randomValueOtherThan(instance.getFetchSourceContext(), () -> {
-                    FetchSourceContext randomFetchSourceContext;
-                    if (randomBoolean()) {
-                        randomFetchSourceContext = new FetchSourceContext(randomBoolean());
-                    } else {
-                        randomFetchSourceContext = new FetchSourceContext(
-                                generateRandomStringArray(12, 16, false),
-                                generateRandomStringArray(12, 16, false)
-                        );
-                    }
-                    return randomFetchSourceContext;
+        });
+        modifiers.add(() -> {
+            if (randomBoolean()) {
+                copy.setScriptFields(randomValueOtherThan(copy.getScriptFields(), () -> {
+                    return new HashSet<>(randomListStuff(16, InnerHitBuilderTests::randomScript));
                 }));
-                break;
-            case 9:
+            } else {
+                SearchSourceBuilder.ScriptField script = randomScript();
+                copy.addScriptField(script.fieldName(), script.script());
+            }
+        });
+        modifiers.add(() -> copy.setFetchSourceContext(randomValueOtherThan(copy.getFetchSourceContext(), () -> {
+            FetchSourceContext randomFetchSourceContext;
+            if (randomBoolean()) {
+                randomFetchSourceContext = new FetchSourceContext(randomBoolean());
+            } else {
+                randomFetchSourceContext = new FetchSourceContext(true, generateRandomStringArray(12, 16, false),
+                        generateRandomStringArray(12, 16, false));
+            }
+            return randomFetchSourceContext;
+        })));
+        modifiers.add(() -> {
                 if (randomBoolean()) {
-                    final List<SortBuilder<?>> sortBuilders = randomValueOtherThan(instance.getSorts(), () -> {
+                    final List<SortBuilder<?>> sortBuilders = randomValueOtherThan(copy.getSorts(), () -> {
                         List<SortBuilder<?>> builders = randomListStuff(16,
-                                () -> SortBuilders.fieldSort(randomAsciiOfLengthBetween(5, 20)).order(randomFrom(SortOrder.values())));
+                                () -> SortBuilders.fieldSort(randomAlphaOfLengthBetween(5, 20)).order(randomFrom(SortOrder.values())));
                         return builders;
                     });
-                    instance.setSorts(sortBuilders);
+                    copy.setSorts(sortBuilders);
                 } else {
-                    instance.addSort(SortBuilders.fieldSort(randomAsciiOfLengthBetween(5, 20)));
+                    copy.addSort(SortBuilders.fieldSort(randomAlphaOfLengthBetween(5, 20)));
                 }
-                break;
-            case 10:
-                instance.setHighlightBuilder(randomValueOtherThan(instance.getHighlightBuilder(),
-                        HighlightBuilderTests::randomHighlighterBuilder));
-                break;
-            case 11:
-                if (instance.getStoredFieldsContext() == null || randomBoolean()) {
-                    List<String> previous = instance.getStoredFieldsContext() == null ?
-                        Collections.emptyList() : instance.getStoredFieldsContext().fieldNames();
+        });
+        modifiers.add(() -> copy
+                .setHighlightBuilder(randomValueOtherThan(copy.getHighlightBuilder(), HighlightBuilderTests::randomHighlighterBuilder)));
+        modifiers.add(() -> {
+                if (copy.getStoredFieldsContext() == null || randomBoolean()) {
+                    List<String> previous = copy.getStoredFieldsContext() == null ?
+                        Collections.emptyList() : copy.getStoredFieldsContext().fieldNames();
                     List<String> newValues = randomValueOtherThan(previous,
-                            () -> randomListStuff(1, 16, () -> randomAsciiOfLengthBetween(1, 16)));
-                    instance.setStoredFieldNames(newValues);
+                            () -> randomListStuff(1, 16, () -> randomAlphaOfLengthBetween(1, 16)));
+                    copy.setStoredFieldNames(newValues);
                 } else {
-                    instance.getStoredFieldsContext().addFieldName(randomAsciiOfLengthBetween(1, 16));
+                    copy.getStoredFieldsContext().addFieldName(randomAlphaOfLengthBetween(1, 16));
                 }
-                break;
-            default:
-                throw new IllegalStateException("unexpected surprise [" + surprise + "]");
-        }
-        return instance;
+        });
+        randomFrom(modifiers).run();
+        return copy;
     }
 
     static SearchSourceBuilder.ScriptField randomScript() {
-        ScriptService.ScriptType randomScriptType = randomFrom(ScriptService.ScriptType.values());
-        Map<String, Object> randomMap = null;
+        ScriptType randomScriptType = randomFrom(ScriptType.values());
+        Map<String, Object> randomMap = new HashMap<>();
         if (randomBoolean()) {
-            randomMap = new HashMap<>();
             int numEntries = randomIntBetween(0, 32);
             for (int i = 0; i < numEntries; i++) {
-                randomMap.put(String.valueOf(i), randomAsciiOfLength(16));
+                randomMap.put(String.valueOf(i), randomAlphaOfLength(16));
             }
         }
-        Script script = new Script(randomAsciiOfLength(128), randomScriptType, randomAsciiOfLengthBetween(1, 4),randomMap);
-        return new SearchSourceBuilder.ScriptField(randomAsciiOfLengthBetween(1, 32), script, randomBoolean());
+        Script script = new Script(randomScriptType, randomScriptType == ScriptType.STORED ? null : randomAlphaOfLengthBetween(1, 4),
+            randomAlphaOfLength(128), randomMap);
+        return new SearchSourceBuilder.ScriptField(randomAlphaOfLengthBetween(1, 32), script, randomBoolean());
     }
 
     static <T> List<T> randomListStuff(int maxSize, Supplier<T> valueSupplier) {
@@ -391,12 +272,7 @@ static <T> List<T> randomListStuff(int minSize, int maxSize, Supplier<T> valueSu
     }
 
     private static InnerHitBuilder serializedCopy(InnerHitBuilder original) throws IOException {
-        try (BytesStreamOutput output = new BytesStreamOutput()) {
-            original.writeTo(output);
-            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
-                return new InnerHitBuilder(in);
-            }
-        }
+        return ESTestCase.copyWriteable(original, namedWriteableRegistry, InnerHitBuilder::new);
     }
 
 }
diff --git a/core/src/test/java/org/elasticsearch/index/query/MatchAllQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/MatchAllQueryBuilderTests.java
index 9195fc83ecc60..5a8597872b04e 100644
--- a/core/src/test/java/org/elasticsearch/index/query/MatchAllQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/MatchAllQueryBuilderTests.java
@@ -21,6 +21,7 @@
 
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Query;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
@@ -35,7 +36,7 @@ protected MatchAllQueryBuilder doCreateTestQueryBuilder() {
     }
 
     @Override
-    protected void doAssertLuceneQuery(MatchAllQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
+    protected void doAssertLuceneQuery(MatchAllQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
         assertThat(query, instanceOf(MatchAllDocsQuery.class));
     }
 
diff --git a/core/src/test/java/org/elasticsearch/index/query/MatchNoneQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/MatchNoneQueryBuilderTests.java
index 4806ec368a4f1..85e0ee72ce296 100644
--- a/core/src/test/java/org/elasticsearch/index/query/MatchNoneQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/MatchNoneQueryBuilderTests.java
@@ -20,7 +20,8 @@
 package org.elasticsearch.index.query;
 
 import org.apache.lucene.search.Query;
-import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
+import org.apache.lucene.search.MatchNoDocsQuery;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
@@ -35,7 +36,7 @@ protected MatchNoneQueryBuilder doCreateTestQueryBuilder() {
     }
 
     @Override
-    protected void doAssertLuceneQuery(MatchNoneQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
+    protected void doAssertLuceneQuery(MatchNoneQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
         assertThat(query, instanceOf(MatchNoDocsQuery.class));
     }
 
diff --git a/core/src/test/java/org/elasticsearch/index/query/MatchPhrasePrefixQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/MatchPhrasePrefixQueryBuilderTests.java
index 9162a1013de4d..59b7a8bd712c5 100644
--- a/core/src/test/java/org/elasticsearch/index/query/MatchPhrasePrefixQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/MatchPhrasePrefixQueryBuilderTests.java
@@ -20,12 +20,14 @@
 package org.elasticsearch.index.query;
 
 import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.IndexOrDocValuesQuery;
 import org.apache.lucene.search.PointRangeQuery;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.TermQuery;
 import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
 import org.elasticsearch.common.lucene.search.MultiPhrasePrefixQuery;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
@@ -50,7 +52,7 @@ protected MatchPhrasePrefixQueryBuilder doCreateTestQueryBuilder() {
             int terms = randomIntBetween(0, 3);
             StringBuilder builder = new StringBuilder();
             for (int i = 0; i < terms; i++) {
-                builder.append(randomAsciiOfLengthBetween(1, 10)).append(" ");
+                builder.append(randomAlphaOfLengthBetween(1, 10)).append(" ");
             }
             value = builder.toString().trim();
         } else {
@@ -76,8 +78,8 @@ protected MatchPhrasePrefixQueryBuilder doCreateTestQueryBuilder() {
     @Override
     protected Map<String, MatchPhrasePrefixQueryBuilder> getAlternateVersions() {
         Map<String, MatchPhrasePrefixQueryBuilder> alternateVersions = new HashMap<>();
-        MatchPhrasePrefixQueryBuilder matchPhrasePrefixQuery = new MatchPhrasePrefixQueryBuilder(randomAsciiOfLengthBetween(1, 10),
-                randomAsciiOfLengthBetween(1, 10));
+        MatchPhrasePrefixQueryBuilder matchPhrasePrefixQuery = new MatchPhrasePrefixQueryBuilder(randomAlphaOfLengthBetween(1, 10),
+                randomAlphaOfLengthBetween(1, 10));
         String contentString = "{\n" +
                 "    \"match_phrase_prefix\" : {\n" +
                 "        \"" + matchPhrasePrefixQuery.fieldName() + "\" : \"" + matchPhrasePrefixQuery.value() + "\"\n" +
@@ -88,12 +90,13 @@ protected Map<String, MatchPhrasePrefixQueryBuilder> getAlternateVersions() {
     }
 
     @Override
-    protected void doAssertLuceneQuery(MatchPhrasePrefixQueryBuilder queryBuilder, Query query, QueryShardContext context)
+    protected void doAssertLuceneQuery(MatchPhrasePrefixQueryBuilder queryBuilder, Query query, SearchContext context)
             throws IOException {
         assertThat(query, notNullValue());
         assertThat(query,
                 either(instanceOf(BooleanQuery.class)).or(instanceOf(MultiPhrasePrefixQuery.class))
-                .or(instanceOf(TermQuery.class)).or(instanceOf(PointRangeQuery.class)).or(instanceOf(MatchNoDocsQuery.class)));
+                .or(instanceOf(TermQuery.class)).or(instanceOf(PointRangeQuery.class))
+                .or(instanceOf(IndexOrDocValuesQuery.class)).or(instanceOf(MatchNoDocsQuery.class)));
     }
 
     public void testIllegalValues() {
diff --git a/core/src/test/java/org/elasticsearch/index/query/MatchPhraseQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/MatchPhraseQueryBuilderTests.java
index fd3711479b5c8..35c33fb71e855 100644
--- a/core/src/test/java/org/elasticsearch/index/query/MatchPhraseQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/MatchPhraseQueryBuilderTests.java
@@ -20,12 +20,14 @@
 package org.elasticsearch.index.query;
 
 import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.IndexOrDocValuesQuery;
 import org.apache.lucene.search.PhraseQuery;
 import org.apache.lucene.search.PointRangeQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.MatchNoDocsQuery;
 import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
@@ -50,7 +52,7 @@ protected MatchPhraseQueryBuilder doCreateTestQueryBuilder() {
             int terms = randomIntBetween(0, 3);
             StringBuilder builder = new StringBuilder();
             for (int i = 0; i < terms; i++) {
-                builder.append(randomAsciiOfLengthBetween(1, 10)).append(" ");
+                builder.append(randomAlphaOfLengthBetween(1, 10)).append(" ");
             }
             value = builder.toString().trim();
         } else {
@@ -72,8 +74,8 @@ protected MatchPhraseQueryBuilder doCreateTestQueryBuilder() {
     @Override
     protected Map<String, MatchPhraseQueryBuilder> getAlternateVersions() {
         Map<String, MatchPhraseQueryBuilder> alternateVersions = new HashMap<>();
-        MatchPhraseQueryBuilder matchPhraseQuery = new MatchPhraseQueryBuilder(randomAsciiOfLengthBetween(1, 10),
-                randomAsciiOfLengthBetween(1, 10));
+        MatchPhraseQueryBuilder matchPhraseQuery = new MatchPhraseQueryBuilder(randomAlphaOfLengthBetween(1, 10),
+                randomAlphaOfLengthBetween(1, 10));
         String contentString = "{\n" +
                 "    \"match_phrase\" : {\n" +
                 "        \"" + matchPhraseQuery.fieldName() + "\" : \"" + matchPhraseQuery.value() + "\"\n" +
@@ -84,10 +86,11 @@ protected Map<String, MatchPhraseQueryBuilder> getAlternateVersions() {
     }
 
     @Override
-    protected void doAssertLuceneQuery(MatchPhraseQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
+    protected void doAssertLuceneQuery(MatchPhraseQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
         assertThat(query, notNullValue());
         assertThat(query, either(instanceOf(BooleanQuery.class)).or(instanceOf(PhraseQuery.class))
-                .or(instanceOf(TermQuery.class)).or(instanceOf(PointRangeQuery.class)).or(instanceOf(MatchNoDocsQuery.class)));
+                .or(instanceOf(TermQuery.class)).or(instanceOf(PointRangeQuery.class))
+                .or(instanceOf(IndexOrDocValuesQuery.class)).or(instanceOf(MatchNoDocsQuery.class)));
     }
 
     public void testIllegalValues() {
diff --git a/core/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java
index ea6fa65dddba4..f6122e5a1f9c9 100644
--- a/core/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java
@@ -22,21 +22,27 @@
 import org.apache.lucene.queries.ExtendedCommonTermsQuery;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.FuzzyQuery;
+import org.apache.lucene.search.IndexOrDocValuesQuery;
 import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.PhraseQuery;
 import org.apache.lucene.search.PointRangeQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
-import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
 import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
+import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.lucene.search.MultiPhrasePrefixQuery;
 import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.search.MatchQuery;
 import org.elasticsearch.index.search.MatchQuery.Type;
 import org.elasticsearch.index.search.MatchQuery.ZeroTermsQuery;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
 import org.hamcrest.Matcher;
 
@@ -64,7 +70,7 @@ protected MatchQueryBuilder doCreateTestQueryBuilder() {
             int terms = randomIntBetween(0, 3);
             StringBuilder builder = new StringBuilder();
             for (int i = 0; i < terms; i++) {
-                builder.append(randomAsciiOfLengthBetween(1, 10)).append(" ");
+                builder.append(randomAlphaOfLengthBetween(1, 10)).append(" ");
             }
             value = builder.toString().trim();
         } else {
@@ -74,13 +80,8 @@ protected MatchQueryBuilder doCreateTestQueryBuilder() {
         MatchQueryBuilder matchQuery = new MatchQueryBuilder(fieldName, value);
         matchQuery.operator(randomFrom(Operator.values()));
 
-        if (randomBoolean()) {
-            if (fieldName.equals(DATE_FIELD_NAME)) {
-                // tokenized dates would trigger parse errors
-                matchQuery.analyzer(randomFrom("keyword", "whitespace"));
-            } else {
-                matchQuery.analyzer(randomFrom("simple", "keyword", "whitespace"));
-            }
+        if (randomBoolean() && fieldName.equals(STRING_FIELD_NAME)) {
+            matchQuery.analyzer(randomFrom("simple", "keyword", "whitespace"));
         }
 
         if (fieldName.equals(STRING_FIELD_NAME) && randomBoolean()) {
@@ -124,7 +125,7 @@ protected MatchQueryBuilder doCreateTestQueryBuilder() {
     @Override
     protected Map<String, MatchQueryBuilder> getAlternateVersions() {
         Map<String, MatchQueryBuilder> alternateVersions = new HashMap<>();
-        MatchQueryBuilder matchQuery = new MatchQueryBuilder(randomAsciiOfLengthBetween(1, 10), randomAsciiOfLengthBetween(1, 10));
+        MatchQueryBuilder matchQuery = new MatchQueryBuilder(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10));
         String contentString = "{\n" +
                 "    \"match\" : {\n" +
                 "        \"" + matchQuery.fieldName() + "\" : \"" + matchQuery.value() + "\"\n" +
@@ -135,7 +136,7 @@ protected Map<String, MatchQueryBuilder> getAlternateVersions() {
     }
 
     @Override
-    protected void doAssertLuceneQuery(MatchQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
+    protected void doAssertLuceneQuery(MatchQueryBuilder queryBuilder, Query query, SearchContext searchContext) throws IOException {
         assertThat(query, notNullValue());
 
         if (query instanceof MatchAllDocsQuery) {
@@ -147,20 +148,20 @@ protected void doAssertLuceneQuery(MatchQueryBuilder queryBuilder, Query query,
         case BOOLEAN:
             assertThat(query, either(instanceOf(BooleanQuery.class)).or(instanceOf(ExtendedCommonTermsQuery.class))
                     .or(instanceOf(TermQuery.class)).or(instanceOf(FuzzyQuery.class)).or(instanceOf(MatchNoDocsQuery.class))
-                    .or(instanceOf(PointRangeQuery.class)));
+                    .or(instanceOf(PointRangeQuery.class)).or(instanceOf(IndexOrDocValuesQuery.class)));
             break;
         case PHRASE:
             assertThat(query, either(instanceOf(BooleanQuery.class)).or(instanceOf(PhraseQuery.class))
                     .or(instanceOf(TermQuery.class)).or(instanceOf(FuzzyQuery.class))
-                    .or(instanceOf(PointRangeQuery.class)));
+                    .or(instanceOf(PointRangeQuery.class)).or(instanceOf(IndexOrDocValuesQuery.class)));
             break;
         case PHRASE_PREFIX:
             assertThat(query, either(instanceOf(BooleanQuery.class)).or(instanceOf(MultiPhrasePrefixQuery.class))
                     .or(instanceOf(TermQuery.class)).or(instanceOf(FuzzyQuery.class))
-                    .or(instanceOf(PointRangeQuery.class)));
+                    .or(instanceOf(PointRangeQuery.class)).or(instanceOf(IndexOrDocValuesQuery.class)));
             break;
         }
-
+        QueryShardContext context = searchContext.getQueryShardContext();
         MappedFieldType fieldType = context.fieldMapper(queryBuilder.fieldName());
         if (query instanceof TermQuery && fieldType != null) {
             String queryValue = queryBuilder.value().toString();
@@ -305,17 +306,15 @@ public void testLegacyMatchPhrasePrefixQuery() throws IOException {
                 "    }\n" +
                 "  }\n" +
                 "}";
-        MatchQueryBuilder qb = (MatchQueryBuilder) parseQuery(json, ParseFieldMatcher.EMPTY);
+        MatchQueryBuilder qb = (MatchQueryBuilder) parseQuery(json);
         checkGeneratedJson(json, qb);
 
         assertEquals(json, expectedQB, qb);
 
         assertSerialization(qb);
 
-        // Now check with strict parsing an exception is thrown
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseQuery(json, ParseFieldMatcher.STRICT));
-        assertThat(e.getMessage(),
-                containsString("Deprecated field [type] used, replaced by [match_phrase and match_phrase_prefix query]"));
+        assertWarnings("Deprecated field [type] used, replaced by [match_phrase and match_phrase_prefix query]",
+                "Deprecated field [slop] used, replaced by [match_phrase query]");
     }
 
     public void testLegacyMatchPhraseQuery() throws IOException {
@@ -338,17 +337,13 @@ public void testLegacyMatchPhraseQuery() throws IOException {
                 "    }\n" +
                 "  }\n" +
                 "}";
-        MatchQueryBuilder qb = (MatchQueryBuilder) parseQuery(json, ParseFieldMatcher.EMPTY);
+        MatchQueryBuilder qb = (MatchQueryBuilder) parseQuery(json);
         checkGeneratedJson(json, qb);
 
         assertEquals(json, expectedQB, qb);
-
         assertSerialization(qb);
-
-        // Now check with strict parsing an exception is thrown
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseQuery(json, ParseFieldMatcher.STRICT));
-        assertThat(e.getMessage(),
-                containsString("Deprecated field [type] used, replaced by [match_phrase and match_phrase_prefix query]"));
+        assertWarnings("Deprecated field [type] used, replaced by [match_phrase and match_phrase_prefix query]",
+                "Deprecated field [slop] used, replaced by [match_phrase query]");
     }
 
     public void testLegacyFuzzyMatchQuery() throws IOException {
@@ -369,12 +364,10 @@ public void testLegacyFuzzyMatchQuery() throws IOException {
                 "    }\n" +
                 "  }\n" +
                 "}";
-        MatchQueryBuilder qb = (MatchQueryBuilder) parseQuery(json, ParseFieldMatcher.EMPTY);
+        MatchQueryBuilder qb = (MatchQueryBuilder) parseQuery(json);
         assertThat(qb, equalTo(expectedQB));
-
-        // Now check with strict parsing an exception is thrown
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseQuery(json, ParseFieldMatcher.STRICT));
-        assertThat(e.getMessage(), containsString("Deprecated field [" + type + "] used, expected [match] instead"));
+        assertWarnings("Deprecated field [" + type + "] used, expected [match] instead",
+                "Deprecated field [slop] used, replaced by [match_phrase query]");
     }
 
     public void testFuzzinessOnNonStringField() throws Exception {
@@ -431,4 +424,62 @@ public void testParseFailsWithMultipleFields() throws IOException {
         e = expectThrows(ParsingException.class, () -> parseQuery(shortJson));
         assertEquals("[match] query doesn't support multiple fields, found [message1] and [message2]", e.getMessage());
     }
+
+    public void testParseFailsWithTermsArray() throws Exception {
+        String json1 = "{\n" +
+                "  \"match\" : {\n" +
+                "    \"message1\" : {\n" +
+                "      \"query\" : [\"term1\", \"term2\"]\n" +
+                "    }\n" +
+                "  }\n" +
+                "}";
+        expectThrows(ParsingException.class, () -> parseQuery(json1));
+
+        String json2 = "{\n" +
+                "  \"match\" : {\n" +
+                "    \"message1\" : [\"term1\", \"term2\"]\n" +
+                "  }\n" +
+                "}";
+        expectThrows(IllegalStateException.class, () -> parseQuery(json2));
+    }
+
+    public void testExceptionUsingAnalyzerOnNumericField() {
+        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
+        QueryShardContext shardContext = createShardContext();
+        MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder(DOUBLE_FIELD_NAME, 6.075210893508043E-4);
+        matchQueryBuilder.analyzer("simple");
+        NumberFormatException e = expectThrows(NumberFormatException.class, () -> matchQueryBuilder.toQuery(shardContext));
+        assertEquals("For input string: \"e\"", e.getMessage());
+    }
+
+    @Override
+    protected void initializeAdditionalMappings(MapperService mapperService) throws IOException {
+        mapperService.merge("doc", new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef("doc",
+            "string_boost", "type=text,boost=4").string()), MapperService.MergeReason.MAPPING_UPDATE, false);
+    }
+
+    public void testMatchPhrasePrefixWithBoost() throws Exception {
+        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
+        QueryShardContext context = createShardContext();
+        assumeTrue("test runs only when the index version is on or after V_5_0_0_alpha1",
+            context.indexVersionCreated().onOrAfter(Version.V_5_0_0_alpha1));
+
+        {
+            // field boost is applied on a single term query
+            MatchPhrasePrefixQueryBuilder builder = new MatchPhrasePrefixQueryBuilder("string_boost", "foo");
+            Query query = builder.toQuery(context);
+            assertThat(query, instanceOf(BoostQuery.class));
+            assertThat(((BoostQuery) query).getBoost(), equalTo(4f));
+            Query innerQuery = ((BoostQuery) query).getQuery();
+            assertThat(innerQuery, instanceOf(MultiPhrasePrefixQuery.class));
+        }
+
+        {
+            // field boost is ignored on phrase query
+            MatchPhrasePrefixQueryBuilder builder = new MatchPhrasePrefixQueryBuilder("string_boost", "foo bar");
+            Query query = builder.toQuery(context);
+            assertThat(query, instanceOf(MultiPhrasePrefixQuery.class));
+        }
+
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/query/MockRepeatAnalyzer.java b/core/src/test/java/org/elasticsearch/index/query/MockRepeatAnalyzer.java
index fc76a0d464c41..f4a0600bd94c9 100644
--- a/core/src/test/java/org/elasticsearch/index/query/MockRepeatAnalyzer.java
+++ b/core/src/test/java/org/elasticsearch/index/query/MockRepeatAnalyzer.java
@@ -35,7 +35,7 @@ private static class MockRepeatFilter extends TokenFilter {
         PositionIncrementAttribute posIncAtt = addAttribute(PositionIncrementAttribute.class);
         String repeat;
 
-        public MockRepeatFilter(TokenStream input) {
+        MockRepeatFilter(TokenStream input) {
             super(input);
         }
 
diff --git a/core/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java
index eaf224b666879..5529a64d57f64 100644
--- a/core/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java
@@ -23,28 +23,34 @@
 import org.apache.lucene.index.Fields;
 import org.apache.lucene.index.MultiFields;
 import org.apache.lucene.index.memory.MemoryIndex;
+import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.termvectors.MultiTermVectorsItemResponse;
 import org.elasticsearch.action.termvectors.MultiTermVectorsRequest;
 import org.elasticsearch.action.termvectors.MultiTermVectorsResponse;
 import org.elasticsearch.action.termvectors.TermVectorsRequest;
 import org.elasticsearch.action.termvectors.TermVectorsResponse;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.lucene.search.MoreLikeThisQuery;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.query.MoreLikeThisQueryBuilder.Item;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
 import org.junit.Before;
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Base64;
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
@@ -55,6 +61,7 @@
 import static org.elasticsearch.index.query.QueryBuilders.moreLikeThisQuery;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.instanceOf;
 
 public class MoreLikeThisQueryBuilderTests extends AbstractQueryTestCase<MoreLikeThisQueryBuilder> {
@@ -87,11 +94,11 @@ private static String[] randomStringFields() {
 
     private Item generateRandomItem() {
         String index = randomBoolean() ? getIndex().getName() : null;
-        String type = getRandomType();  // set to one type to avoid ambiguous types
+        String type = "doc";
         // indexed item or artificial document
         Item item;
         if (randomBoolean()) {
-            item = new Item(index, type, randomAsciiOfLength(10));
+            item = new Item(index, type, randomAlphaOfLength(10));
         } else {
             item = new Item(index, type, randomArtificialDoc());
         }
@@ -104,7 +111,7 @@ private Item generateRandomItem() {
             item.perFieldAnalyzer(randomPerFieldAnalyzer());
         }
         if (randomBoolean()) {
-            item.routing(randomAsciiOfLength(10));
+            item.routing(randomAlphaOfLength(10));
         }
         if (randomBoolean()) {
             item.version(randomInt(5));
@@ -120,7 +127,7 @@ private XContentBuilder randomArtificialDoc() {
         try {
             doc = XContentFactory.jsonBuilder().startObject();
             for (String field : randomFields) {
-                doc.field(field, randomAsciiOfLength(10));
+                doc.field(field, randomAlphaOfLength(10));
             }
             doc.endObject();
         } catch (IOException e) {
@@ -243,9 +250,16 @@ private static Fields generateFields(String[] fieldNames, String text) throws IO
     }
 
     @Override
-    protected void doAssertLuceneQuery(MoreLikeThisQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
+    protected void doAssertLuceneQuery(MoreLikeThisQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
         if (queryBuilder.likeItems() != null && queryBuilder.likeItems().length > 0) {
             assertThat(query, instanceOf(BooleanQuery.class));
+            BooleanQuery booleanQuery = (BooleanQuery) query;
+            for (BooleanClause booleanClause : booleanQuery) {
+                if (booleanClause.getQuery() instanceof MoreLikeThisQuery) {
+                    MoreLikeThisQuery moreLikeThisQuery = (MoreLikeThisQuery) booleanClause.getQuery();
+                    assertThat(moreLikeThisQuery.getLikeFields().length, greaterThan(0));
+                }
+            }
         } else {
             // we rely on integration tests for a deeper check here
             assertThat(query, instanceOf(MoreLikeThisQuery.class));
@@ -275,7 +289,7 @@ public void testUnsupportedFields() throws IOException {
     }
 
     public void testMoreLikeThisBuilder() throws Exception {
-        Query parsedQuery = parseQuery(moreLikeThisQuery(new String[]{"name.first", "name.last"}, new String[]{"something"}, null).minTermFreq(1).maxQueryTerms(12).buildAsBytes()).toQuery(createShardContext());
+        Query parsedQuery = parseQuery(moreLikeThisQuery(new String[]{"name.first", "name.last"}, new String[]{"something"}, null).minTermFreq(1).maxQueryTerms(12)).toQuery(createShardContext());
         assertThat(parsedQuery, instanceOf(MoreLikeThisQuery.class));
         MoreLikeThisQuery mltQuery = (MoreLikeThisQuery) parsedQuery;
         assertThat(mltQuery.getMoreLikeFields()[0], equalTo("name.first"));
@@ -292,14 +306,45 @@ public void testItemSerialization() throws IOException {
         assertEquals(expectedItem, newItem);
     }
 
+    public void testItemCopy() throws IOException {
+        Item expectedItem = generateRandomItem();
+        Item newItem = new Item(expectedItem);
+        assertEquals(expectedItem, newItem);
+    }
+
     public void testItemFromXContent() throws IOException {
         Item expectedItem = generateRandomItem();
         String json = expectedItem.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS).string();
-        XContentParser parser = XContentFactory.xContent(json).createParser(json);
-        Item newItem = Item.parse(parser, ParseFieldMatcher.STRICT, new Item());
+        XContentParser parser = createParser(JsonXContent.jsonXContent, json);
+        Item newItem = Item.parse(parser, new Item());
         assertEquals(expectedItem, newItem);
     }
 
+    public void testItemSerializationBwc() throws IOException {
+        final byte[] data = Base64.getDecoder().decode("AQVpbmRleAEEdHlwZQEODXsiZm9vIjoiYmFyIn0A/wD//////////QAAAAAAAAAA");
+        final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2,
+            Version.V_5_1_1, Version.V_5_1_2, Version.V_5_2_0);
+        try (StreamInput in = StreamInput.wrap(data)) {
+            in.setVersion(version);
+            Item item = new Item(in);
+            assertEquals(XContentType.JSON, item.xContentType());
+            assertEquals("{\"foo\":\"bar\"}", item.doc().utf8ToString());
+            assertEquals("index", item.index());
+            assertEquals("type", item.type());
+
+            try (BytesStreamOutput out = new BytesStreamOutput()) {
+                out.setVersion(version);
+                item.writeTo(out);
+                assertArrayEquals(data, out.bytes().toBytesRef().bytes);
+            }
+        }
+    }
+
+    @Override
+    protected boolean isCachable(MoreLikeThisQueryBuilder queryBuilder) {
+        return queryBuilder.likeItems().length == 0; // items are always fetched
+    }
+
     public void testFromJson() throws IOException {
         String json =
                 "{\n" +
@@ -361,10 +406,8 @@ public void testFromJson() throws IOException {
                         "  }\n" +
                         "}";
 
-        MoreLikeThisQueryBuilder parsedQueryMltShortcut = (MoreLikeThisQueryBuilder) parseQuery(deprecatedJson, ParseFieldMatcher.EMPTY);
+        MoreLikeThisQueryBuilder parsedQueryMltShortcut = (MoreLikeThisQueryBuilder) parseQuery(deprecatedJson);
         assertThat(parsedQueryMltShortcut, equalTo(parsed));
-
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseQuery(deprecatedJson));
-        assertEquals("Deprecated field [mlt] used, expected [more_like_this] instead", e.getMessage());
+        assertWarnings("Deprecated field [mlt] used, expected [more_like_this] instead");
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java
index e96c99bdcf6bf..22ff8caecfff8 100644
--- a/core/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java
@@ -25,18 +25,20 @@
 import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.DisjunctionMaxQuery;
 import org.apache.lucene.search.FuzzyQuery;
+import org.apache.lucene.search.IndexOrDocValuesQuery;
 import org.apache.lucene.search.LegacyNumericRangeQuery;
 import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.PhraseQuery;
 import org.apache.lucene.search.PointRangeQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.lucene.all.AllTermQuery;
-import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
 import org.elasticsearch.common.lucene.search.MultiPhrasePrefixQuery;
 import org.elasticsearch.index.query.MultiMatchQueryBuilder.Type;
 import org.elasticsearch.index.search.MatchQuery;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
@@ -83,13 +85,8 @@ protected MultiMatchQueryBuilder doCreateTestQueryBuilder() {
         if (randomBoolean()) {
             query.operator(randomFrom(Operator.values()));
         }
-        if (randomBoolean()) {
-            if (fieldName.equals(DATE_FIELD_NAME)) {
-                // tokenized dates would trigger parse errors
-                query.analyzer("keyword");
-            } else {
-                query.analyzer(randomAnalyzer());
-            }
+        if (randomBoolean() && fieldName.equals(STRING_FIELD_NAME)) {
+            query.analyzer(randomAnalyzer());
         }
         if (randomBoolean()) {
             query.slop(randomIntBetween(0, 5));
@@ -142,7 +139,7 @@ protected Map<String, MultiMatchQueryBuilder> getAlternateVersions() {
     }
 
     @Override
-    protected void doAssertLuceneQuery(MultiMatchQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
+    protected void doAssertLuceneQuery(MultiMatchQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
         // we rely on integration tests for deeper checks here
         assertThat(query, either(instanceOf(BoostQuery.class)).or(instanceOf(TermQuery.class)).or(instanceOf(AllTermQuery.class))
                 .or(instanceOf(BooleanQuery.class)).or(instanceOf(DisjunctionMaxQuery.class))
@@ -150,7 +147,7 @@ protected void doAssertLuceneQuery(MultiMatchQueryBuilder queryBuilder, Query qu
                 .or(instanceOf(MatchAllDocsQuery.class)).or(instanceOf(ExtendedCommonTermsQuery.class))
                 .or(instanceOf(MatchNoDocsQuery.class)).or(instanceOf(PhraseQuery.class))
                 .or(instanceOf(LegacyNumericRangeQuery.class))
-                .or(instanceOf(PointRangeQuery.class)));
+                .or(instanceOf(PointRangeQuery.class)).or(instanceOf(IndexOrDocValuesQuery.class)));
     }
 
     public void testIllegaArguments() {
@@ -274,7 +271,7 @@ public void testFuzzinessNotAllowedTypes() throws IOException {
         }
     }
 
-    public void testQueryParameterArrayException() throws IOException {
+    public void testQueryParameterArrayException() {
         String json =
                 "{\n" +
                 "  \"multi_match\" : {\n" +
@@ -287,6 +284,16 @@ public void testQueryParameterArrayException() throws IOException {
         assertEquals("[multi_match] unknown token [START_ARRAY] after [query]", e.getMessage());
     }
 
+    public void testExceptionUsingAnalyzerOnNumericField() {
+        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
+        QueryShardContext shardContext = createShardContext();
+        MultiMatchQueryBuilder multiMatchQueryBuilder = new MultiMatchQueryBuilder(6.075210893508043E-4);
+        multiMatchQueryBuilder.field(DOUBLE_FIELD_NAME);
+        multiMatchQueryBuilder.analyzer("simple");
+        NumberFormatException e = expectThrows(NumberFormatException.class, () -> multiMatchQueryBuilder.toQuery(shardContext));
+        assertEquals("For input string: \"e\"", e.getMessage());
+    }
+
     public void testFuzzinessOnNonStringField() throws Exception {
         assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         MultiMatchQueryBuilder query = new MultiMatchQueryBuilder(42).field(INT_FIELD_NAME).field(BOOLEAN_FIELD_NAME);
diff --git a/core/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java
index f122d66ebe340..065b8d20ff6e6 100644
--- a/core/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java
@@ -20,28 +20,35 @@
 package org.elasticsearch.index.query;
 
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
-
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.join.ScoreMode;
-import org.apache.lucene.search.join.ToParentBlockJoinQuery;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
 import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.index.mapper.LatLonPointFieldMapper;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
+import org.elasticsearch.index.search.ESToParentBlockJoinQuery;
 import org.elasticsearch.search.fetch.subphase.InnerHitsContext;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.sort.FieldSortBuilder;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.test.AbstractQueryTestCase;
+import org.elasticsearch.test.VersionUtils;
+import org.hamcrest.Matchers;
 
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 
+import static org.elasticsearch.index.query.InnerHitBuilderTests.randomInnerHits;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.hamcrest.CoreMatchers.notNullValue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 public class NestedQueryBuilderTests extends AbstractQueryTestCase<NestedQueryBuilder> {
 
@@ -49,20 +56,29 @@ public class NestedQueryBuilderTests extends AbstractQueryTestCase<NestedQueryBu
 
     @Override
     protected void initializeAdditionalMappings(MapperService mapperService) throws IOException {
-        mapperService.merge("nested_doc", new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef("nested_doc",
+        String geoFieldMapping = (mapperService.getIndexSettings().getIndexVersionCreated()
+            .before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) ?
+            LEGACY_GEO_POINT_FIELD_MAPPING : "type=geo_point";
+        mapperService.merge("doc", new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef("doc",
                 STRING_FIELD_NAME, "type=text",
                 INT_FIELD_NAME, "type=integer",
                 DOUBLE_FIELD_NAME, "type=double",
                 BOOLEAN_FIELD_NAME, "type=boolean",
                 DATE_FIELD_NAME, "type=date",
                 OBJECT_FIELD_NAME, "type=object",
-                GEO_POINT_FIELD_NAME, GEO_POINT_FIELD_MAPPING,
+                GEO_POINT_FIELD_NAME, geoFieldMapping,
                 "nested1", "type=nested"
         ).string()), MapperService.MergeReason.MAPPING_UPDATE, false);
+        if (mapperService.getIndexSettings().getIndexVersionCreated()
+                .before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+            assertWarnings("geo_point lat_lon parameter is deprecated and will be removed in the next major release",
+                    "geo_point geohash parameter is deprecated and will be removed in the next major release",
+                    "geo_point geohash_prefix parameter is deprecated and will be removed in the next major release");
+        }
     }
 
     /**
-     * @return a {@link HasChildQueryBuilder} with random values all over the place
+     * @return a {@link NestedQueryBuilder} with random values all over the place
      */
     @Override
     protected NestedQueryBuilder doCreateTestQueryBuilder() {
@@ -73,44 +89,55 @@ protected NestedQueryBuilder doCreateTestQueryBuilder() {
         }
         NestedQueryBuilder nqb = new NestedQueryBuilder("nested1", innerQueryBuilder,
                 RandomPicks.randomFrom(random(), ScoreMode.values()));
+        nqb.ignoreUnmapped(randomBoolean());
         if (randomBoolean()) {
-            nqb.innerHit(new InnerHitBuilder()
-                    .setName(randomAsciiOfLengthBetween(1, 10))
+            nqb.innerHit(new InnerHitBuilder(randomAlphaOfLengthBetween(1, 10))
                     .setSize(randomIntBetween(0, 100))
-                    .addSort(new FieldSortBuilder(INT_FIELD_NAME).order(SortOrder.ASC)));
+                    .addSort(new FieldSortBuilder(INT_FIELD_NAME).order(SortOrder.ASC))
+                    .setIgnoreUnmapped(nqb.ignoreUnmapped()));
         }
-        nqb.ignoreUnmapped(randomBoolean());
         return nqb;
     }
 
     @Override
-    protected void doAssertLuceneQuery(NestedQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
-        QueryBuilder innerQueryBuilder = queryBuilder.query();
-        assertThat(query, instanceOf(ToParentBlockJoinQuery.class));
-        ToParentBlockJoinQuery parentBlockJoinQuery = (ToParentBlockJoinQuery) query;
+    protected void doAssertLuceneQuery(NestedQueryBuilder queryBuilder, Query query, SearchContext searchContext) throws IOException {
+        assertThat(query, instanceOf(ESToParentBlockJoinQuery.class));
         // TODO how to assert this?
         if (queryBuilder.innerHit() != null) {
             // have to rewrite again because the provided queryBuilder hasn't been rewritten (directly returned from
             // doCreateTestQueryBuilder)
-            queryBuilder = (NestedQueryBuilder) queryBuilder.rewrite(context);
+            queryBuilder = (NestedQueryBuilder) queryBuilder.rewrite(searchContext.getQueryShardContext());
 
-            SearchContext searchContext = SearchContext.current();
             assertNotNull(searchContext);
-            Map<String, InnerHitBuilder> innerHitBuilders = new HashMap<>();
-            InnerHitBuilder.extractInnerHits(queryBuilder, innerHitBuilders);
-            for (InnerHitBuilder builder : innerHitBuilders.values()) {
+            Map<String, InnerHitContextBuilder> innerHitInternals = new HashMap<>();
+            InnerHitContextBuilder.extractInnerHits(queryBuilder, innerHitInternals);
+            for (InnerHitContextBuilder builder : innerHitInternals.values()) {
                 builder.build(searchContext, searchContext.innerHits());
             }
             assertNotNull(searchContext.innerHits());
             assertEquals(1, searchContext.innerHits().getInnerHits().size());
             assertTrue(searchContext.innerHits().getInnerHits().containsKey(queryBuilder.innerHit().getName()));
-            InnerHitsContext.BaseInnerHits innerHits = searchContext.innerHits().getInnerHits().get(queryBuilder.innerHit().getName());
+            InnerHitsContext.InnerHitSubContext innerHits = searchContext.innerHits().getInnerHits().get(queryBuilder.innerHit().getName());
             assertEquals(innerHits.size(), queryBuilder.innerHit().getSize());
             assertEquals(innerHits.sort().sort.getSort().length, 1);
             assertEquals(innerHits.sort().sort.getSort()[0].getField(), INT_FIELD_NAME);
         }
     }
 
+    /**
+     * Test (de)serialization on all previous released versions
+     */
+    public void testSerializationBWC() throws IOException {
+        for (Version version : VersionUtils.allReleasedVersions()) {
+            NestedQueryBuilder testQuery = createTestQueryBuilder();
+            if (version.before(Version.V_5_2_0) && testQuery.innerHit() != null) {
+                // ignore unmapped for inner_hits has been added on 5.2
+                testQuery.innerHit().setIgnoreUnmapped(false);
+            }
+            assertSerialization(testQuery, version);
+        }
+    }
+
     public void testValidate() {
         QueryBuilder innerQuery = RandomQueryBuilder.createQuery(random());
         IllegalArgumentException e =
@@ -195,4 +222,140 @@ public void testIgnoreUnmapped() throws IOException {
         IllegalStateException e = expectThrows(IllegalStateException.class, () -> failingQueryBuilder.toQuery(createShardContext()));
         assertThat(e.getMessage(), containsString("[" + NestedQueryBuilder.NAME + "] failed to find nested object under path [unmapped]"));
     }
+
+    public void testIgnoreUnmappedWithRewrite() throws IOException {
+        // WrapperQueryBuilder makes sure we always rewrite
+        final NestedQueryBuilder queryBuilder =
+            new NestedQueryBuilder("unmapped", new WrapperQueryBuilder(new MatchAllQueryBuilder().toString()), ScoreMode.None);
+        queryBuilder.ignoreUnmapped(true);
+        QueryShardContext queryShardContext = createShardContext();
+        Query query = queryBuilder.rewrite(queryShardContext).toQuery(queryShardContext);
+        assertThat(query, notNullValue());
+        assertThat(query, instanceOf(MatchNoDocsQuery.class));
+    }
+
+    public void testMinFromString() {
+        assertThat("fromString(min) != MIN", ScoreMode.Min, equalTo(NestedQueryBuilder.parseScoreMode("min")));
+        assertThat("min", equalTo(NestedQueryBuilder.scoreModeAsString(ScoreMode.Min)));
+    }
+
+    public void testMaxFromString() {
+        assertThat("fromString(max) != MAX", ScoreMode.Max, equalTo(NestedQueryBuilder.parseScoreMode("max")));
+        assertThat("max", equalTo(NestedQueryBuilder.scoreModeAsString(ScoreMode.Max)));
+    }
+
+    public void testAvgFromString() {
+        assertThat("fromString(avg) != AVG", ScoreMode.Avg, equalTo(NestedQueryBuilder.parseScoreMode("avg")));
+        assertThat("avg", equalTo(NestedQueryBuilder.scoreModeAsString(ScoreMode.Avg)));
+    }
+
+    public void testSumFromString() {
+        assertThat("fromString(total) != SUM", ScoreMode.Total, equalTo(NestedQueryBuilder.parseScoreMode("sum")));
+        assertThat("sum", equalTo(NestedQueryBuilder.scoreModeAsString(ScoreMode.Total)));
+    }
+
+    public void testNoneFromString() {
+        assertThat("fromString(none) != NONE", ScoreMode.None, equalTo(NestedQueryBuilder.parseScoreMode("none")));
+        assertThat("none", equalTo(NestedQueryBuilder.scoreModeAsString(ScoreMode.None)));
+    }
+
+    /**
+     * Should throw {@link IllegalArgumentException} instead of NPE.
+     */
+    public void testThatNullFromStringThrowsException() {
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> NestedQueryBuilder.parseScoreMode(null));
+        assertEquals("No score mode for child query [null] found", e.getMessage());
+    }
+
+    /**
+     * Failure should not change (and the value should never match anything...).
+     */
+    public void testThatUnrecognizedFromStringThrowsException() {
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> NestedQueryBuilder.parseScoreMode("unrecognized value"));
+        assertEquals("No score mode for child query [unrecognized value] found", e.getMessage());
+    }
+
+    public void testInlineLeafInnerHitsNestedQuery() throws Exception {
+        InnerHitBuilder leafInnerHits = randomInnerHits();
+        NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None);
+        nestedQueryBuilder.innerHit(leafInnerHits);
+        Map<String, InnerHitContextBuilder> innerHitBuilders = new HashMap<>();
+        nestedQueryBuilder.extractInnerHitBuilders(innerHitBuilders);
+        assertThat(innerHitBuilders.get(leafInnerHits.getName()), Matchers.notNullValue());
+    }
+
+    public void testInlineLeafInnerHitsNestedQueryViaBoolQuery() {
+        InnerHitBuilder leafInnerHits = randomInnerHits();
+        NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None)
+            .innerHit(leafInnerHits);
+        BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder().should(nestedQueryBuilder);
+        Map<String, InnerHitContextBuilder> innerHitBuilders = new HashMap<>();
+        boolQueryBuilder.extractInnerHitBuilders(innerHitBuilders);
+        assertThat(innerHitBuilders.get(leafInnerHits.getName()), Matchers.notNullValue());
+    }
+
+    public void testInlineLeafInnerHitsNestedQueryViaConstantScoreQuery() {
+        InnerHitBuilder leafInnerHits = randomInnerHits();
+        NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None)
+            .innerHit(leafInnerHits);
+        ConstantScoreQueryBuilder constantScoreQueryBuilder = new ConstantScoreQueryBuilder(nestedQueryBuilder);
+        Map<String, InnerHitContextBuilder> innerHitBuilders = new HashMap<>();
+        constantScoreQueryBuilder.extractInnerHitBuilders(innerHitBuilders);
+        assertThat(innerHitBuilders.get(leafInnerHits.getName()), Matchers.notNullValue());
+    }
+
+    public void testInlineLeafInnerHitsNestedQueryViaBoostingQuery() {
+        InnerHitBuilder leafInnerHits1 = randomInnerHits();
+        NestedQueryBuilder nestedQueryBuilder1 = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None)
+            .innerHit(leafInnerHits1);
+        InnerHitBuilder leafInnerHits2 = randomInnerHits();
+        NestedQueryBuilder nestedQueryBuilder2 = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None)
+            .innerHit(leafInnerHits2);
+        BoostingQueryBuilder constantScoreQueryBuilder = new BoostingQueryBuilder(nestedQueryBuilder1, nestedQueryBuilder2);
+        Map<String, InnerHitContextBuilder> innerHitBuilders = new HashMap<>();
+        constantScoreQueryBuilder.extractInnerHitBuilders(innerHitBuilders);
+        assertThat(innerHitBuilders.get(leafInnerHits1.getName()), Matchers.notNullValue());
+        assertThat(innerHitBuilders.get(leafInnerHits2.getName()), Matchers.notNullValue());
+    }
+
+    public void testInlineLeafInnerHitsNestedQueryViaFunctionScoreQuery() {
+        InnerHitBuilder leafInnerHits = randomInnerHits();
+        NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None)
+            .innerHit(leafInnerHits);
+        FunctionScoreQueryBuilder functionScoreQueryBuilder = new FunctionScoreQueryBuilder(nestedQueryBuilder);
+        Map<String, InnerHitContextBuilder> innerHitBuilders = new HashMap<>();
+        ((AbstractQueryBuilder<?>) functionScoreQueryBuilder).extractInnerHitBuilders(innerHitBuilders);
+        assertThat(innerHitBuilders.get(leafInnerHits.getName()), Matchers.notNullValue());
+    }
+
+    public void testBuildIgnoreUnmappedNestQuery() throws Exception {
+        QueryShardContext queryShardContext = mock(QueryShardContext.class);
+        when(queryShardContext.getObjectMapper("path")).thenReturn(null);
+        SearchContext searchContext = mock(SearchContext.class);
+        when(searchContext.getQueryShardContext()).thenReturn(queryShardContext);
+
+        InnerHitBuilder leafInnerHits = randomInnerHits();
+        NestedQueryBuilder query1 = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None);
+        query1.innerHit(leafInnerHits);
+        final Map<String, InnerHitContextBuilder> innerHitBuilders = new HashMap<>();
+        final InnerHitsContext innerHitsContext = new InnerHitsContext();
+        expectThrows(IllegalStateException.class, () -> {
+            query1.extractInnerHitBuilders(innerHitBuilders);
+            assertThat(innerHitBuilders.size(), Matchers.equalTo(1));
+            assertTrue(innerHitBuilders.containsKey(leafInnerHits.getName()));
+            innerHitBuilders.get(leafInnerHits.getName()).build(searchContext, innerHitsContext);
+        });
+        innerHitBuilders.clear();
+        NestedQueryBuilder query2 = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None);
+        query2.innerHit(leafInnerHits.setIgnoreUnmapped(true));
+        query2.extractInnerHitBuilders(innerHitBuilders);
+        assertThat(innerHitBuilders.size(), Matchers.equalTo(1));
+        assertTrue(innerHitBuilders.containsKey(leafInnerHits.getName()));
+        assertThat(innerHitBuilders.get(leafInnerHits.getName()), instanceOf(NestedQueryBuilder.NestedInnerHitContextBuilder.class));
+        NestedQueryBuilder.NestedInnerHitContextBuilder nestedContextBuilder =
+            (NestedQueryBuilder.NestedInnerHitContextBuilder) innerHitBuilders.get(leafInnerHits.getName());
+        nestedContextBuilder.build(searchContext, innerHitsContext);
+        assertThat(innerHitsContext.getInnerHits().size(), Matchers.equalTo(0));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/query/ParentIdQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/ParentIdQueryBuilderTests.java
deleted file mode 100644
index 750d9ce319fa9..0000000000000
--- a/core/src/test/java/org/elasticsearch/index/query/ParentIdQueryBuilderTests.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.query;
-
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.DocValuesTermsQuery;
-import org.apache.lucene.search.MatchNoDocsQuery;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TermQuery;
-import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
-import org.elasticsearch.common.compress.CompressedXContent;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.TypeFieldMapper;
-import org.elasticsearch.test.AbstractQueryTestCase;
-import org.hamcrest.Matchers;
-
-import java.io.IOException;
-
-import static org.hamcrest.CoreMatchers.containsString;
-import static org.hamcrest.CoreMatchers.instanceOf;
-import static org.hamcrest.CoreMatchers.notNullValue;
-
-public class ParentIdQueryBuilderTests extends AbstractQueryTestCase<ParentIdQueryBuilder> {
-
-    protected static final String PARENT_TYPE = "parent";
-    protected static final String CHILD_TYPE = "child";
-
-    @Override
-    protected void initializeAdditionalMappings(MapperService mapperService) throws IOException {
-        mapperService.merge(PARENT_TYPE, new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef(PARENT_TYPE,
-                STRING_FIELD_NAME, "type=text",
-                INT_FIELD_NAME, "type=integer",
-                DOUBLE_FIELD_NAME, "type=double",
-                BOOLEAN_FIELD_NAME, "type=boolean",
-                DATE_FIELD_NAME, "type=date",
-                OBJECT_FIELD_NAME, "type=object"
-        ).string()), MapperService.MergeReason.MAPPING_UPDATE, false);
-        mapperService.merge(CHILD_TYPE, new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef(CHILD_TYPE,
-                "_parent", "type=" + PARENT_TYPE,
-                STRING_FIELD_NAME, "type=text",
-                INT_FIELD_NAME, "type=integer",
-                DOUBLE_FIELD_NAME, "type=double",
-                BOOLEAN_FIELD_NAME, "type=boolean",
-                DATE_FIELD_NAME, "type=date",
-                OBJECT_FIELD_NAME, "type=object"
-        ).string()), MapperService.MergeReason.MAPPING_UPDATE, false);
-    }
-
-    @Override
-    protected ParentIdQueryBuilder doCreateTestQueryBuilder() {
-        return new ParentIdQueryBuilder(CHILD_TYPE, randomAsciiOfLength(4)).ignoreUnmapped(randomBoolean());
-    }
-
-    @Override
-    protected void doAssertLuceneQuery(ParentIdQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
-        assertThat(query, Matchers.instanceOf(BooleanQuery.class));
-        BooleanQuery booleanQuery = (BooleanQuery) query;
-        assertThat(booleanQuery.clauses().size(), Matchers.equalTo(2));
-        DocValuesTermsQuery idQuery = (DocValuesTermsQuery) booleanQuery.clauses().get(0).getQuery();
-        // there are no getters to get the field and terms on DocValuesTermsQuery, so lets validate by creating a
-        // new query based on the builder:
-        assertThat(idQuery, Matchers.equalTo(new DocValuesTermsQuery("_parent#" + PARENT_TYPE, queryBuilder.getId())));
-
-        TermQuery typeQuery = (TermQuery) booleanQuery.clauses().get(1).getQuery();
-        assertThat(typeQuery.getTerm().field(), Matchers.equalTo(TypeFieldMapper.NAME));
-        assertThat(typeQuery.getTerm().text(), Matchers.equalTo(queryBuilder.getType()));
-    }
-
-    public void testFromJson() throws IOException {
-        String query =
-            "{\n" +
-                "  \"parent_id\" : {\n" +
-                "    \"type\" : \"child\",\n" +
-                "    \"id\" : \"123\",\n" +
-                "    \"ignore_unmapped\" : false,\n" +
-                "    \"boost\" : 3.0,\n" +
-                "    \"_name\" : \"name\"" +
-                "  }\n" +
-                "}";
-        ParentIdQueryBuilder queryBuilder = (ParentIdQueryBuilder) parseQuery(query);
-        checkGeneratedJson(query, queryBuilder);
-        assertThat(queryBuilder.getType(), Matchers.equalTo("child"));
-        assertThat(queryBuilder.getId(), Matchers.equalTo("123"));
-        assertThat(queryBuilder.boost(), Matchers.equalTo(3f));
-        assertThat(queryBuilder.queryName(), Matchers.equalTo("name"));
-    }
-
-    public void testIgnoreUnmapped() throws IOException {
-        final ParentIdQueryBuilder queryBuilder = new ParentIdQueryBuilder("unmapped", "foo");
-        queryBuilder.ignoreUnmapped(true);
-        Query query = queryBuilder.toQuery(createShardContext());
-        assertThat(query, notNullValue());
-        assertThat(query, instanceOf(MatchNoDocsQuery.class));
-
-        final ParentIdQueryBuilder failingQueryBuilder = new ParentIdQueryBuilder("unmapped", "foo");
-        failingQueryBuilder.ignoreUnmapped(false);
-        QueryShardException e = expectThrows(QueryShardException.class, () -> failingQueryBuilder.toQuery(createShardContext()));
-        assertThat(e.getMessage(), containsString("[" + ParentIdQueryBuilder.NAME + "] no mapping found for type [unmapped]"));
-    }
-
-}
diff --git a/core/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java
index 0c4c59d11002d..11cf639974557 100644
--- a/core/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java
@@ -24,6 +24,7 @@
 import org.apache.lucene.search.PrefixQuery;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
@@ -59,13 +60,13 @@ protected Map<String, PrefixQueryBuilder> getAlternateVersions() {
     }
 
     private static PrefixQueryBuilder randomPrefixQuery() {
-        String fieldName = randomBoolean() ? STRING_FIELD_NAME : randomAsciiOfLengthBetween(1, 10);
-        String value = randomAsciiOfLengthBetween(1, 10);
+        String fieldName = randomBoolean() ? STRING_FIELD_NAME : randomAlphaOfLengthBetween(1, 10);
+        String value = randomAlphaOfLengthBetween(1, 10);
         return new PrefixQueryBuilder(fieldName, value);
     }
 
     @Override
-    protected void doAssertLuceneQuery(PrefixQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
+    protected void doAssertLuceneQuery(PrefixQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
         assertThat(query, instanceOf(PrefixQuery.class));
         PrefixQuery prefixQuery = (PrefixQuery) query;
         assertThat(prefixQuery.getPrefix().field(), equalTo(queryBuilder.fieldName()));
@@ -84,7 +85,7 @@ public void testIllegalArguments() {
 
     public void testBlendedRewriteMethod() throws IOException {
         String rewrite = "top_terms_blended_freqs_10";
-        Query parsedQuery = parseQuery(prefixQuery("field", "val").rewrite(rewrite).buildAsBytes()).toQuery(createShardContext());
+        Query parsedQuery = parseQuery(prefixQuery("field", "val").rewrite(rewrite)).toQuery(createShardContext());
         assertThat(parsedQuery, instanceOf(PrefixQuery.class));
         PrefixQuery prefixQuery = (PrefixQuery) parsedQuery;
         assertThat(prefixQuery.getPrefix(), equalTo(new Term("field", "val")));
diff --git a/core/src/test/java/org/elasticsearch/index/query/QueryDSLDocumentationTests.java b/core/src/test/java/org/elasticsearch/index/query/QueryDSLDocumentationTests.java
index 7f1d9aebbd954..61e19690d39d4 100644
--- a/core/src/test/java/org/elasticsearch/index/query/QueryDSLDocumentationTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/QueryDSLDocumentationTests.java
@@ -30,7 +30,7 @@
 import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
 import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder.FilterFunctionBuilder;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
@@ -53,8 +53,6 @@
 import static org.elasticsearch.index.query.QueryBuilders.geoHashCellQuery;
 import static org.elasticsearch.index.query.QueryBuilders.geoPolygonQuery;
 import static org.elasticsearch.index.query.QueryBuilders.geoShapeQuery;
-import static org.elasticsearch.index.query.QueryBuilders.hasChildQuery;
-import static org.elasticsearch.index.query.QueryBuilders.hasParentQuery;
 import static org.elasticsearch.index.query.QueryBuilders.idsQuery;
 import static org.elasticsearch.index.query.QueryBuilders.indicesQuery;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
@@ -200,20 +198,6 @@ public void testGeoHashCell() {
             .precision(3);
     }
 
-    public void testHasChild() {
-        hasChildQuery(
-                "blog_tag",
-                termQuery("tag","something"),
-                ScoreMode.None);
-    }
-
-    public void testHasParent() {
-        hasParentQuery(
-            "blog",
-            termQuery("tag","something"),
-                false);
-    }
-
     public void testIds() {
         idsQuery("my_type", "type2")
                 .addIds("1", "4", "100");
@@ -221,7 +205,7 @@ public void testIds() {
         idsQuery().addIds("1", "4", "100");
     }
 
-    public void testIndices() {
+    public void testIndices() throws IOException {
         indicesQuery(
                 termQuery("tag", "wow"),
                 "index1", "index2"
@@ -231,6 +215,7 @@ public void testIndices() {
                 termQuery("tag", "wow"),
                 "index1", "index2"
             ).noMatchQuery("all");
+        assertWarnings("indices query is deprecated. Instead search on the '_index' field");
     }
 
     public void testMatchAll() {
@@ -293,13 +278,11 @@ public void testScript() {
                 new Script("doc['num1'].value > 1")
             );
 
-        Map<String, Integer> parameters = new HashMap<>();
+        Map<String, Object> parameters = new HashMap<>();
         parameters.put("param1", 5);
         scriptQuery(
                 new Script(
-                    "mygroovyscript",
-                    ScriptType.FILE,
-                    "groovy",
+                    ScriptType.FILE, "groovy", "mygroovyscript",
                     parameters)
             );
 
diff --git a/core/src/test/java/org/elasticsearch/index/query/QueryParseContextTests.java b/core/src/test/java/org/elasticsearch/index/query/QueryParseContextTests.java
index 5568d2fa5a7d2..c7720f361fe45 100644
--- a/core/src/test/java/org/elasticsearch/index/query/QueryParseContextTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/QueryParseContextTests.java
@@ -19,16 +19,14 @@
 
 package org.elasticsearch.index.query;
 
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.test.ESTestCase;
+import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
 import java.io.IOException;
@@ -37,19 +35,23 @@
 import static java.util.Collections.emptyList;
 
 public class QueryParseContextTests extends ESTestCase {
-
-    private static IndicesQueriesRegistry indicesQueriesRegistry;
+    private static NamedXContentRegistry xContentRegistry;
 
     @BeforeClass
     public static void init() {
-        indicesQueriesRegistry = new SearchModule(Settings.EMPTY, false, emptyList()).getQueryParserRegistry();
+        xContentRegistry = new NamedXContentRegistry(new SearchModule(Settings.EMPTY, false, emptyList()).getNamedXContents());
+    }
+
+    @AfterClass
+    public static void cleanup() {
+        xContentRegistry = null;
     }
 
     public void testParseTopLevelBuilder() throws IOException {
         QueryBuilder query = new MatchQueryBuilder("foo", "bar");
         String requestBody = "{ \"query\" : " + query.toString() + "}";
-        try (XContentParser parser = XContentFactory.xContent(requestBody).createParser(requestBody)) {
-            QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.STRICT);
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, requestBody)) {
+            QueryParseContext context = new QueryParseContext(parser);
             QueryBuilder actual = context.parseTopLevelQueryBuilder();
             assertEquals(query, actual);
         }
@@ -57,8 +59,8 @@ public void testParseTopLevelBuilder() throws IOException {
 
     public void testParseTopLevelBuilderEmptyObject() throws IOException {
         String requestBody = "{}";
-        try (XContentParser parser = XContentFactory.xContent(requestBody).createParser(requestBody)) {
-            QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.STRICT);
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, requestBody)) {
+            QueryParseContext context = new QueryParseContext(parser);
             QueryBuilder query = context.parseTopLevelQueryBuilder();
             assertNull(query);
         }
@@ -66,8 +68,8 @@ public void testParseTopLevelBuilderEmptyObject() throws IOException {
 
     public void testParseTopLevelBuilderUnknownParameter() throws IOException {
         String requestBody = "{ \"foo\" : \"bar\"}";
-        try (XContentParser parser = XContentFactory.xContent(requestBody).createParser(requestBody)) {
-            QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.STRICT);
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, requestBody)) {
+            QueryParseContext context = new QueryParseContext(parser);
             ParsingException exception = expectThrows(ParsingException.class, () ->  context.parseTopLevelQueryBuilder());
             assertEquals("request does not support [foo]", exception.getMessage());
         }
@@ -76,8 +78,8 @@ public void testParseTopLevelBuilderUnknownParameter() throws IOException {
     public void testParseInnerQueryBuilder() throws IOException {
         QueryBuilder query = new MatchQueryBuilder("foo", "bar");
         String source = query.toString();
-        try (XContentParser parser = XContentFactory.xContent(source).createParser(source)) {
-            QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.STRICT);
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, source)) {
+            QueryParseContext context = new QueryParseContext(parser);
             Optional<QueryBuilder> actual = context.parseInnerQueryBuilder();
             assertEquals(query, actual.get());
         }
@@ -85,43 +87,48 @@ public void testParseInnerQueryBuilder() throws IOException {
 
     public void testParseInnerQueryBuilderEmptyBody() throws IOException {
         String source = "{}";
-        try (XContentParser parser = XContentFactory.xContent(source).createParser(source)) {
-            QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.EMPTY);
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, source)) {
+            QueryParseContext context = new QueryParseContext(parser);
             Optional<QueryBuilder> emptyQuery = context.parseInnerQueryBuilder();
             assertFalse(emptyQuery.isPresent());
+            assertWarnings("query malformed, empty clause found at [1:2]");
         }
     }
 
     public void testParseInnerQueryBuilderExceptions() throws IOException {
         String source = "{ \"foo\": \"bar\" }";
-        try (XContentParser parser = JsonXContent.jsonXContent.createParser(source)) {
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, source)) {
             parser.nextToken();
             parser.nextToken(); // don't start with START_OBJECT to provoke exception
-            QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.STRICT);
+            QueryParseContext context = new QueryParseContext(parser);
             ParsingException exception = expectThrows(ParsingException.class, () ->  context.parseInnerQueryBuilder());
             assertEquals("[_na] query malformed, must start with start_object", exception.getMessage());
         }
 
         source = "{}";
-        try (XContentParser parser = JsonXContent.jsonXContent.createParser(source)) {
-            QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.STRICT);
-            IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () ->  context.parseInnerQueryBuilder());
-            assertEquals("query malformed, empty clause found at [1:2]", exception.getMessage());
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, source)) {
+            QueryParseContext context = new QueryParseContext(parser);
+            context.parseInnerQueryBuilder();
+            assertWarnings("query malformed, empty clause found at [1:2]");
         }
 
         source = "{ \"foo\" : \"bar\" }";
-        try (XContentParser parser = JsonXContent.jsonXContent.createParser(source)) {
-            QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.STRICT);
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, source)) {
+            QueryParseContext context = new QueryParseContext(parser);
             ParsingException exception = expectThrows(ParsingException.class, () ->  context.parseInnerQueryBuilder());
             assertEquals("[foo] query malformed, no start_object after query name", exception.getMessage());
         }
 
         source = "{ \"foo\" : {} }";
-        try (XContentParser parser = JsonXContent.jsonXContent.createParser(source)) {
-            QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.STRICT);
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, source)) {
+            QueryParseContext context = new QueryParseContext(parser);
             ParsingException exception = expectThrows(ParsingException.class, () ->  context.parseInnerQueryBuilder());
             assertEquals("no [query] registered for [foo]", exception.getMessage());
         }
     }
 
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return xContentRegistry;
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/query/QueryRewriteContextTests.java b/core/src/test/java/org/elasticsearch/index/query/QueryRewriteContextTests.java
index a39fbae1764ce..d5817457a46c3 100644
--- a/core/src/test/java/org/elasticsearch/index/query/QueryRewriteContextTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/QueryRewriteContextTests.java
@@ -23,10 +23,10 @@
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.script.ScriptSettings;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.test.ESTestCase;
@@ -34,27 +34,32 @@
 import static java.util.Collections.emptyList;
 
 public class QueryRewriteContextTests extends ESTestCase {
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return new NamedXContentRegistry(new SearchModule(Settings.EMPTY, false, emptyList()).getNamedXContents());
+    }
 
     public void testNewParseContextWithLegacyScriptLanguage() throws Exception {
-        String defaultLegacyScriptLanguage = randomAsciiOfLength(4);
+        String defaultLegacyScriptLanguage = randomAlphaOfLength(4);
         IndexMetaData.Builder indexMetadata = new IndexMetaData.Builder("index");
         indexMetadata.settings(Settings.builder().put("index.version.created", Version.CURRENT)
                 .put("index.number_of_shards", 1)
                 .put("index.number_of_replicas", 1)
         );
-        IndicesQueriesRegistry indicesQueriesRegistry = new SearchModule(Settings.EMPTY, false, emptyList()).getQueryParserRegistry();
+        final long nowInMills = randomNonNegativeLong();
         IndexSettings indexSettings = new IndexSettings(indexMetadata.build(),
                 Settings.builder().put(ScriptSettings.LEGACY_SCRIPT_SETTING, defaultLegacyScriptLanguage).build());
-        QueryRewriteContext queryRewriteContext =
-                new QueryRewriteContext(indexSettings, null, null, indicesQueriesRegistry, null, null, null);;
+        QueryRewriteContext queryRewriteContext = new QueryRewriteContext(indexSettings, null, null, xContentRegistry(),
+                null, null, () -> nowInMills);
 
         // verify that the default script language in the query parse context is equal to defaultLegacyScriptLanguage variable:
-        QueryParseContext queryParseContext =
-                queryRewriteContext.newParseContextWithLegacyScriptLanguage(XContentHelper.createParser(new BytesArray("{}")));
+        QueryParseContext queryParseContext = queryRewriteContext
+                .newParseContextWithLegacyScriptLanguage(XContentHelper.createParser(xContentRegistry(), new BytesArray("{}")));
         assertEquals(defaultLegacyScriptLanguage, queryParseContext.getDefaultScriptLanguage());
 
         // verify that the script query's script language is equal to defaultLegacyScriptLanguage variable:
-        XContentParser parser = XContentHelper.createParser(new BytesArray("{\"script\" : {\"script\": \"return true\"}}"));
+        XContentParser parser = XContentHelper.createParser(xContentRegistry(),
+                new BytesArray("{\"script\" : {\"script\": \"return true\"}}"));
         queryParseContext = queryRewriteContext.newParseContextWithLegacyScriptLanguage(parser);
         ScriptQueryBuilder queryBuilder = (ScriptQueryBuilder) queryParseContext.parseInnerQueryBuilder().get();
         assertEquals(defaultLegacyScriptLanguage, queryBuilder.script().getLang());
diff --git a/core/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java b/core/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java
index 3b3bdf31b7577..b83c1b2897a3c 100644
--- a/core/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java
@@ -46,9 +46,10 @@ public void testFailIfFieldMappingNotFound() {
         IndexSettings indexSettings = new IndexSettings(indexMetadata.build(), Settings.EMPTY);
         MapperService mapperService = mock(MapperService.class);
         when(mapperService.getIndexSettings()).thenReturn(indexSettings);
+        final long nowInMillis = randomNonNegativeLong();
         QueryShardContext context = new QueryShardContext(
-            indexSettings, null, null, mapperService, null, null, null, null, null, null
-        );
+            0, indexSettings, null, null, mapperService, null, null, xContentRegistry(), null, null,
+            () -> nowInMillis);
 
         context.setAllowUnmappedFields(false);
         MappedFieldType fieldType = new TextFieldMapper.TextFieldType();
diff --git a/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java
index ce49f18ccfcd0..795189d7ded9a 100644
--- a/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java
@@ -19,12 +19,15 @@
 
 package org.elasticsearch.index.query;
 
+import org.apache.lucene.analysis.MockSynonymAnalyzer;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.queryparser.classic.MapperQueryParser;
 import org.apache.lucene.queryparser.classic.QueryParserSettings;
 import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.BoostQuery;
+import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.DisjunctionMaxQuery;
 import org.apache.lucene.search.FuzzyQuery;
 import org.apache.lucene.search.MatchAllDocsQuery;
@@ -36,21 +39,35 @@
 import org.apache.lucene.search.RegexpQuery;
 import org.apache.lucene.search.SynonymQuery;
 import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TermRangeQuery;
+import org.apache.lucene.search.WildcardQuery;
+import org.apache.lucene.search.spans.SpanNearQuery;
+import org.apache.lucene.search.spans.SpanOrQuery;
+import org.apache.lucene.search.spans.SpanTermQuery;
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.automaton.TooComplexToDeterminizeException;
+import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.lucene.all.AllTermQuery;
 import org.elasticsearch.common.unit.Fuzziness;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
 import org.hamcrest.Matchers;
 import org.joda.time.DateTimeZone;
 
 import java.io.IOException;
-import java.util.Collections;
+import java.util.ArrayList;
 import java.util.List;
 
 import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBooleanSubQuery;
 import static org.hamcrest.CoreMatchers.either;
 import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.instanceOf;
 
@@ -62,17 +79,16 @@ protected QueryStringQueryBuilder doCreateTestQueryBuilder() {
         String query = "";
         for (int i = 0; i < numTerms; i++) {
             //min length 4 makes sure that the text is not an operator (AND/OR) so toQuery won't break
-            query += (randomBoolean() ? STRING_FIELD_NAME + ":" : "") + randomAsciiOfLengthBetween(4, 10) + " ";
+            query += (randomBoolean() ? STRING_FIELD_NAME + ":" : "") + randomAlphaOfLengthBetween(4, 10) + " ";
         }
         QueryStringQueryBuilder queryStringQueryBuilder = new QueryStringQueryBuilder(query);
         if (randomBoolean()) {
             queryStringQueryBuilder.defaultField(randomBoolean() ?
-                STRING_FIELD_NAME : randomAsciiOfLengthBetween(1, 10));
-        }
-        if (randomBoolean()) {
+                STRING_FIELD_NAME : randomAlphaOfLengthBetween(1, 10));
+        } else {
             int numFields = randomIntBetween(1, 5);
             for (int i = 0; i < numFields; i++) {
-                String fieldName = randomBoolean() ? STRING_FIELD_NAME : randomAsciiOfLengthBetween(1, 10);
+                String fieldName = randomBoolean() ? STRING_FIELD_NAME : randomAlphaOfLengthBetween(1, 10);
                 if (randomBoolean()) {
                     queryStringQueryBuilder.field(fieldName);
                 } else {
@@ -102,9 +118,6 @@ protected QueryStringQueryBuilder doCreateTestQueryBuilder() {
         if (randomBoolean()) {
             queryStringQueryBuilder.maxDeterminizedStates(randomIntBetween(1, 100));
         }
-        if (randomBoolean()) {
-            queryStringQueryBuilder.lowercaseExpandedTerms(randomBoolean());
-        }
         if (randomBoolean()) {
             queryStringQueryBuilder.autoGeneratePhraseQueries(randomBoolean());
         }
@@ -133,7 +146,7 @@ protected QueryStringQueryBuilder doCreateTestQueryBuilder() {
             queryStringQueryBuilder.rewrite(getRandomRewriteMethod());
         }
         if (randomBoolean()) {
-            queryStringQueryBuilder.quoteFieldSuffix(randomAsciiOfLengthBetween(1, 3));
+            queryStringQueryBuilder.quoteFieldSuffix(randomAlphaOfLengthBetween(1, 3));
         }
         if (randomBoolean()) {
             queryStringQueryBuilder.tieBreaker(randomFloat());
@@ -144,23 +157,25 @@ protected QueryStringQueryBuilder doCreateTestQueryBuilder() {
         if (randomBoolean()) {
             queryStringQueryBuilder.useDisMax(randomBoolean());
         }
-        if (randomBoolean()) {
-            queryStringQueryBuilder.locale(randomLocale(random()));
-        }
         if (randomBoolean()) {
             queryStringQueryBuilder.timeZone(randomDateTimeZone().getID());
         }
+        if (queryStringQueryBuilder.autoGeneratePhraseQueries() == false) {
+            // setSplitOnWhitespace(false) is disallowed when getAutoGeneratePhraseQueries() == true
+            queryStringQueryBuilder.splitOnWhitespace(randomBoolean());
+        }
         return queryStringQueryBuilder;
     }
 
     @Override
     protected void doAssertLuceneQuery(QueryStringQueryBuilder queryBuilder,
-                                       Query query, QueryShardContext context) throws IOException {
+                                       Query query, SearchContext context) throws IOException {
         if ("".equals(queryBuilder.queryString())) {
             assertThat(query, instanceOf(MatchNoDocsQuery.class));
         } else {
             assertThat(query, either(instanceOf(TermQuery.class)).or(instanceOf(AllTermQuery.class))
-                    .or(instanceOf(BooleanQuery.class)).or(instanceOf(DisjunctionMaxQuery.class)));
+                    .or(instanceOf(BooleanQuery.class)).or(instanceOf(DisjunctionMaxQuery.class))
+                    .or(instanceOf(PhraseQuery.class)));
         }
     }
 
@@ -171,6 +186,9 @@ public void testIllegalArguments() {
     public void testToQueryMatchAllQuery() throws Exception {
         Query query = queryStringQuery("*:*").toQuery(createShardContext());
         assertThat(query, instanceOf(MatchAllDocsQuery.class));
+
+        query = queryStringQuery("*").toQuery(createShardContext());
+        assertThat(query, instanceOf(MatchAllDocsQuery.class));
     }
 
     public void testToQueryTermQuery() throws IOException {
@@ -311,7 +329,6 @@ public void testToQueryWildcarQuery() throws Exception {
             MapperQueryParser queryParser = new MapperQueryParser(createShardContext());
             QueryParserSettings settings = new QueryParserSettings("first foo-bar-foobar* last");
             settings.defaultField(STRING_FIELD_NAME);
-            settings.fieldsAndWeights(Collections.emptyMap());
             settings.analyzeWildcard(true);
             settings.fuzziness(Fuzziness.AUTO);
             settings.rewriteMethod(MultiTermQuery.CONSTANT_SCORE_REWRITE);
@@ -339,7 +356,6 @@ public void testToQueryWilcardQueryWithSynonyms() throws Exception {
             MapperQueryParser queryParser = new MapperQueryParser(createShardContext());
             QueryParserSettings settings = new QueryParserSettings("first foo-bar-foobar* last");
             settings.defaultField(STRING_FIELD_NAME);
-            settings.fieldsAndWeights(Collections.emptyMap());
             settings.analyzeWildcard(true);
             settings.fuzziness(Fuzziness.AUTO);
             settings.rewriteMethod(MultiTermQuery.CONSTANT_SCORE_REWRITE);
@@ -371,6 +387,113 @@ public void testToQueryWilcardQueryWithSynonyms() throws Exception {
         }
     }
 
+    public void testToQueryWithGraph() throws Exception {
+        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
+        for (Operator op : Operator.values()) {
+            BooleanClause.Occur defaultOp = op.toBooleanClauseOccur();
+            MapperQueryParser queryParser = new MapperQueryParser(createShardContext());
+            QueryParserSettings settings = new QueryParserSettings("");
+            settings.defaultField(STRING_FIELD_NAME);
+            settings.fuzziness(Fuzziness.AUTO);
+            settings.analyzeWildcard(true);
+            settings.rewriteMethod(MultiTermQuery.CONSTANT_SCORE_REWRITE);
+            settings.defaultOperator(op.toQueryParserOperator());
+            settings.forceAnalyzer(new MockSynonymAnalyzer());
+            settings.forceQuoteAnalyzer(new MockSynonymAnalyzer());
+            queryParser.reset(settings);
+
+            // simple multi-term
+            Query query = queryParser.parse("guinea pig");
+            Query expectedQuery = new BooleanQuery.Builder()
+                    .add(new BooleanQuery.Builder()
+                            .add(new TermQuery(new Term(STRING_FIELD_NAME, "guinea")), Occur.MUST)
+                            .add(new TermQuery(new Term(STRING_FIELD_NAME, "pig")), Occur.MUST).build(), Occur.SHOULD)
+                    .add(new TermQuery(new Term(STRING_FIELD_NAME, "cavy")), Occur.SHOULD)
+                    .build();
+            expectedQuery = new BooleanQuery.Builder()
+                    .add(expectedQuery, defaultOp)
+                    .build();
+            assertThat(query, Matchers.equalTo(expectedQuery));
+
+            // simple with additional tokens
+            query = queryParser.parse("that guinea pig smells");
+            expectedQuery = new BooleanQuery.Builder()
+                    .add(new TermQuery(new Term(STRING_FIELD_NAME, "that")), defaultOp)
+                    .add(new BooleanQuery.Builder()
+                         .add(new BooleanQuery.Builder()
+                            .add(new BooleanClause(new TermQuery(new Term(STRING_FIELD_NAME, "guinea")), Occur.MUST))
+                            .add(new BooleanClause(new TermQuery(new Term(STRING_FIELD_NAME, "pig")), Occur.MUST))
+                            .build(), Occur.SHOULD)
+                            .add(new TermQuery(new Term(STRING_FIELD_NAME, "cavy")), Occur.SHOULD).build(), defaultOp)
+                    .add(new TermQuery(new Term(STRING_FIELD_NAME, "smells")), defaultOp)
+                    .build();
+            assertThat(query, Matchers.equalTo(expectedQuery));
+
+            // complex
+            query = queryParser.parse("+that -(guinea pig) +smells");
+            expectedQuery = new BooleanQuery.Builder()
+                    .add(new TermQuery(new Term(STRING_FIELD_NAME, "that")), Occur.MUST)
+                    .add(new BooleanQuery.Builder()
+                            .add(new BooleanQuery.Builder()
+                                    .add(new BooleanQuery.Builder()
+                                            .add(new TermQuery(new Term(STRING_FIELD_NAME, "guinea")), Occur.MUST)
+                                            .add(new TermQuery(new Term(STRING_FIELD_NAME, "pig")), Occur.MUST)
+                                            .build(), Occur.SHOULD)
+                                    .add(new TermQuery(new Term(STRING_FIELD_NAME, "cavy")), Occur.SHOULD)
+                                    .build(), defaultOp)
+                            .build(), Occur.MUST_NOT)
+                    .add(new TermQuery(new Term(STRING_FIELD_NAME, "smells")), Occur.MUST)
+                    .build();
+
+            assertThat(query, Matchers.equalTo(expectedQuery));
+
+            // no parent should cause guinea and pig to be treated as separate tokens
+            query = queryParser.parse("+that -guinea pig +smells");
+            expectedQuery = new BooleanQuery.Builder()
+                .add(new TermQuery(new Term(STRING_FIELD_NAME, "that")), BooleanClause.Occur.MUST)
+                .add(new TermQuery(new Term(STRING_FIELD_NAME, "guinea")), BooleanClause.Occur.MUST_NOT)
+                .add(new TermQuery(new Term(STRING_FIELD_NAME, "pig")), defaultOp)
+                .add(new TermQuery(new Term(STRING_FIELD_NAME, "smells")), BooleanClause.Occur.MUST)
+                .build();
+
+            assertThat(query, Matchers.equalTo(expectedQuery));
+
+            // span query
+            query = queryParser.parse("\"that guinea pig smells\"");
+            expectedQuery = new BooleanQuery.Builder()
+                .add(new SpanNearQuery.Builder(STRING_FIELD_NAME, true)
+                    .addClause(new SpanTermQuery(new Term(STRING_FIELD_NAME, "that")))
+                    .addClause(new SpanOrQuery(
+                        new SpanNearQuery.Builder(STRING_FIELD_NAME, true)
+                            .addClause(new SpanTermQuery(new Term(STRING_FIELD_NAME, "guinea")))
+                            .addClause(new SpanTermQuery(new Term(STRING_FIELD_NAME, "pig"))).build(),
+                        new SpanTermQuery(new Term(STRING_FIELD_NAME, "cavy"))))
+                    .addClause(new SpanTermQuery(new Term(STRING_FIELD_NAME, "smells")))
+                    .build(), Occur.SHOULD)
+                .setDisableCoord(true)
+                .build();
+            assertThat(query, Matchers.equalTo(expectedQuery));
+
+            // span query with slop
+            query = queryParser.parse("\"that guinea pig smells\"~2");
+            expectedQuery = new BooleanQuery.Builder()
+                .add(new SpanNearQuery.Builder(STRING_FIELD_NAME, true)
+                    .addClause(new SpanTermQuery(new Term(STRING_FIELD_NAME, "that")))
+                    .addClause(new SpanOrQuery(
+                        new SpanNearQuery.Builder(STRING_FIELD_NAME, true)
+                            .addClause(new SpanTermQuery(new Term(STRING_FIELD_NAME, "guinea")))
+                            .addClause(new SpanTermQuery(new Term(STRING_FIELD_NAME, "pig"))).build(),
+                        new SpanTermQuery(new Term(STRING_FIELD_NAME, "cavy"))))
+                    .addClause(new SpanTermQuery(new Term(STRING_FIELD_NAME, "smells")))
+                    .setSlop(2)
+                    .build(),
+                    Occur.SHOULD)
+                .setDisableCoord(true)
+                .build();
+            assertThat(query, Matchers.equalTo(expectedQuery));
+        }
+    }
+
     public void testToQueryRegExpQuery() throws Exception {
         assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         Query query = queryStringQuery("/foo*bar/").defaultField(STRING_FIELD_NAME)
@@ -391,6 +514,79 @@ public void testToQueryRegExpQueryTooComplex() throws Exception {
         assertThat(e.getMessage(), containsString("would result in more than 10000 states"));
     }
 
+    /**
+     * Validates that {@code max_determinized_states} can be parsed and lowers the allowed number of determinized states.
+     */
+    public void testToQueryRegExpQueryMaxDeterminizedStatesParsing() throws Exception {
+        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
+        XContentBuilder builder = JsonXContent.contentBuilder();
+        builder.startObject(); {
+            builder.startObject("query_string"); {
+                builder.field("query", "/[ac]*a[ac]{1,10}/");
+                builder.field("default_field", STRING_FIELD_NAME);
+                builder.field("max_determinized_states", 10);
+            }
+            builder.endObject();
+        }
+        builder.endObject();
+
+        QueryBuilder queryBuilder = new QueryParseContext(createParser(builder)).parseInnerQueryBuilder().get();
+        TooComplexToDeterminizeException e = expectThrows(TooComplexToDeterminizeException.class,
+                () -> queryBuilder.toQuery(createShardContext()));
+        assertThat(e.getMessage(), containsString("Determinizing [ac]*"));
+        assertThat(e.getMessage(), containsString("would result in more than 10 states"));
+    }
+
+    /**
+     * Validates that {@code max_determinized_states} can be parsed and lowers the allowed number of determinized states.
+     */
+    public void testToQueryRegExpQueryMaxDeterminedStatesParsing() throws Exception {
+        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
+        XContentBuilder builder = JsonXContent.contentBuilder();
+        builder.startObject(); {
+            builder.startObject("query_string"); {
+                builder.field("query", "/[ac]*a[ac]{1,10}/");
+                builder.field("default_field", STRING_FIELD_NAME);
+                builder.field("max_determined_states", 10);
+            }
+            builder.endObject();
+        }
+        builder.endObject();
+
+        QueryBuilder queryBuilder = new QueryParseContext(createParser(builder)).parseInnerQueryBuilder().get();
+        TooComplexToDeterminizeException e = expectThrows(TooComplexToDeterminizeException.class,
+                () -> queryBuilder.toQuery(createShardContext()));
+        assertThat(e.getMessage(), containsString("Determinizing [ac]*"));
+        assertThat(e.getMessage(), containsString("would result in more than 10 states"));
+        assertWarnings("Deprecated field [max_determined_states] used, expected [max_determinized_states] instead");
+    }
+
+    /**
+     * Validates that {@code max_determinized_states} can be parsed and lowers the allowed number of determinized states.
+     */
+    public void testEnabledPositionIncrements() throws Exception {
+        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
+
+        boolean useDeprecatedName = randomBoolean();
+        XContentBuilder builder = JsonXContent.contentBuilder();
+        builder.startObject(); {
+            builder.startObject("query_string"); {
+                builder.field("query", "text");
+                builder.field("default_field", STRING_FIELD_NAME);
+                builder.field(useDeprecatedName ? "enable_position_increment" : "enable_position_increments", false);
+            }
+            builder.endObject();
+        }
+        builder.endObject();
+
+        QueryStringQueryBuilder queryBuilder = (QueryStringQueryBuilder) new QueryParseContext(createParser(builder))
+                .parseInnerQueryBuilder().get();
+        assertFalse(queryBuilder.enablePositionIncrements());
+        if (useDeprecatedName) {
+            assertWarnings("Deprecated field [enable_position_increment] used, expected [enable_position_increments] instead");
+        }
+    }
+
     public void testToQueryFuzzyQueryAutoFuziness() throws Exception {
         assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
 
@@ -531,6 +727,198 @@ public void testToQueryPhraseQueryBoostAndSlop() throws IOException {
         assertThat(phraseQuery.getTerms().length, equalTo(2));
     }
 
+    public void testToQueryWildcardNonExistingFields() throws IOException {
+        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
+        QueryStringQueryBuilder queryStringQueryBuilder =
+            new QueryStringQueryBuilder("foo bar").field("invalid*");
+        Query query = queryStringQueryBuilder.toQuery(createShardContext());
+        Query expectedQuery = new BooleanQuery.Builder()
+            .add(new MatchNoDocsQuery("empty fields"), Occur.SHOULD)
+            .add(new MatchNoDocsQuery("empty fields"), Occur.SHOULD)
+            .build();
+        assertThat(expectedQuery, equalTo(query));
+
+        queryStringQueryBuilder =
+            new QueryStringQueryBuilder("field:foo bar").field("invalid*");
+        query = queryStringQueryBuilder.toQuery(createShardContext());
+        expectedQuery = new BooleanQuery.Builder()
+            .add(new TermQuery(new Term("field", "foo")), Occur.SHOULD)
+            .add(new MatchNoDocsQuery("empty fields"), Occur.SHOULD)
+            .build();
+        assertThat(expectedQuery, equalTo(query));
+
+
+    }
+
+    public void testToQuerySplitOnWhitespace() throws IOException {
+        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
+        // splitOnWhitespace=false
+        {
+            QueryStringQueryBuilder queryBuilder =
+                new QueryStringQueryBuilder("foo bar")
+                    .field(STRING_FIELD_NAME).field(STRING_FIELD_NAME_2)
+                    .splitOnWhitespace(false);
+            Query query = queryBuilder.toQuery(createShardContext());
+            BooleanQuery bq1 =
+                new BooleanQuery.Builder()
+                    .add(new BooleanClause(new TermQuery(new Term(STRING_FIELD_NAME, "foo")), BooleanClause.Occur.SHOULD))
+                    .add(new BooleanClause(new TermQuery(new Term(STRING_FIELD_NAME, "bar")), BooleanClause.Occur.SHOULD))
+                    .build();
+            List<Query> disjuncts = new ArrayList<>();
+            disjuncts.add(bq1);
+            disjuncts.add(new TermQuery(new Term(STRING_FIELD_NAME_2, "foo bar")));
+            DisjunctionMaxQuery expectedQuery = new DisjunctionMaxQuery(disjuncts, 0.0f);
+            assertThat(query, equalTo(expectedQuery));
+        }
+
+        {
+            QueryStringQueryBuilder queryBuilder =
+                new QueryStringQueryBuilder("mapped_string:other foo bar")
+                    .field(STRING_FIELD_NAME).field(STRING_FIELD_NAME_2)
+                    .splitOnWhitespace(false);
+            Query query = queryBuilder.toQuery(createShardContext());
+            BooleanQuery bq1 =
+                new BooleanQuery.Builder()
+                    .add(new BooleanClause(new TermQuery(new Term(STRING_FIELD_NAME, "foo")), BooleanClause.Occur.SHOULD))
+                    .add(new BooleanClause(new TermQuery(new Term(STRING_FIELD_NAME, "bar")), BooleanClause.Occur.SHOULD))
+                    .build();
+            List<Query> disjuncts = new ArrayList<>();
+            disjuncts.add(bq1);
+            disjuncts.add(new TermQuery(new Term(STRING_FIELD_NAME_2, "foo bar")));
+            DisjunctionMaxQuery disjunctionMaxQuery = new DisjunctionMaxQuery(disjuncts, 0.0f);
+            BooleanQuery expectedQuery =
+                new BooleanQuery.Builder()
+                    .add(disjunctionMaxQuery, BooleanClause.Occur.SHOULD)
+                    .add(new TermQuery(new Term(STRING_FIELD_NAME, "other")), BooleanClause.Occur.SHOULD)
+                    .build();
+            assertThat(query, equalTo(expectedQuery));
+        }
+
+        {
+            QueryStringQueryBuilder queryBuilder =
+                new QueryStringQueryBuilder("foo OR bar")
+                    .field(STRING_FIELD_NAME).field(STRING_FIELD_NAME_2)
+                    .splitOnWhitespace(false);
+            Query query = queryBuilder.toQuery(createShardContext());
+
+            List<Query> disjuncts1 = new ArrayList<>();
+            disjuncts1.add(new TermQuery(new Term(STRING_FIELD_NAME, "foo")));
+            disjuncts1.add(new TermQuery(new Term(STRING_FIELD_NAME_2, "foo")));
+            DisjunctionMaxQuery maxQuery1 = new DisjunctionMaxQuery(disjuncts1, 0.0f);
+
+            List<Query> disjuncts2 = new ArrayList<>();
+            disjuncts2.add(new TermQuery(new Term(STRING_FIELD_NAME, "bar")));
+            disjuncts2.add(new TermQuery(new Term(STRING_FIELD_NAME_2, "bar")));
+            DisjunctionMaxQuery maxQuery2 = new DisjunctionMaxQuery(disjuncts2, 0.0f);
+
+            BooleanQuery expectedQuery =
+                new BooleanQuery.Builder()
+                    .add(new BooleanClause(maxQuery1, BooleanClause.Occur.SHOULD))
+                    .add(new BooleanClause(maxQuery2, BooleanClause.Occur.SHOULD))
+                    .build();
+            assertThat(query, equalTo(expectedQuery));
+        }
+
+        // split_on_whitespace=false breaks range query with simple syntax
+        {
+            // throws an exception when lenient is set to false
+            QueryStringQueryBuilder queryBuilder =
+                new QueryStringQueryBuilder(">10 foo")
+                    .field(INT_FIELD_NAME)
+                    .splitOnWhitespace(false);
+            IllegalArgumentException exc =
+                expectThrows(IllegalArgumentException.class, () -> queryBuilder.toQuery(createShardContext()));
+            assertThat(exc.getMessage(), equalTo("For input string: \"10 foo\""));
+        }
+
+        {
+            // returns an empty boolean query when lenient is set to true
+            QueryStringQueryBuilder queryBuilder =
+                new QueryStringQueryBuilder(">10 foo")
+                    .field(INT_FIELD_NAME)
+                    .splitOnWhitespace(false)
+                    .lenient(true);
+            Query query = queryBuilder.toQuery(createShardContext());
+            BooleanQuery bq = new BooleanQuery.Builder().build();
+            assertThat(bq, equalTo(query));
+        }
+
+        // splitOnWhitespace=true
+        {
+            QueryStringQueryBuilder queryBuilder =
+                new QueryStringQueryBuilder("foo bar")
+                    .field(STRING_FIELD_NAME).field(STRING_FIELD_NAME_2)
+                    .splitOnWhitespace(true);
+            Query query = queryBuilder.toQuery(createShardContext());
+
+            List<Query> disjuncts1 = new ArrayList<>();
+            disjuncts1.add(new TermQuery(new Term(STRING_FIELD_NAME, "foo")));
+            disjuncts1.add(new TermQuery(new Term(STRING_FIELD_NAME_2, "foo")));
+            DisjunctionMaxQuery maxQuery1 = new DisjunctionMaxQuery(disjuncts1, 0.0f);
+
+            List<Query> disjuncts2 = new ArrayList<>();
+            disjuncts2.add(new TermQuery(new Term(STRING_FIELD_NAME, "bar")));
+            disjuncts2.add(new TermQuery(new Term(STRING_FIELD_NAME_2, "bar")));
+            DisjunctionMaxQuery maxQuery2 = new DisjunctionMaxQuery(disjuncts2, 0.0f);
+
+            BooleanQuery expectedQuery =
+                new BooleanQuery.Builder()
+                    .add(new BooleanClause(maxQuery1, BooleanClause.Occur.SHOULD))
+                    .add(new BooleanClause(maxQuery2, BooleanClause.Occur.SHOULD))
+                    .build();
+            assertThat(query, equalTo(expectedQuery));
+        }
+    }
+
+    public void testExistsFieldQuery() throws Exception {
+        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
+        QueryShardContext context = createShardContext();
+        QueryStringQueryBuilder queryBuilder = new QueryStringQueryBuilder("foo:*");
+        Query query = queryBuilder.toQuery(context);
+        Query expected = new ConstantScoreQuery(new TermQuery(new Term("_field_names", "foo")));
+        assertThat(query, equalTo(expected));
+
+        queryBuilder = new QueryStringQueryBuilder("_all:*");
+        query = queryBuilder.toQuery(context);
+        expected = new MatchAllDocsQuery();
+        assertThat(query, equalTo(expected));
+
+        queryBuilder = new QueryStringQueryBuilder("*:*");
+        query = queryBuilder.toQuery(context);
+        expected = new MatchAllDocsQuery();
+        assertThat(query, equalTo(expected));
+
+        queryBuilder = new QueryStringQueryBuilder("*");
+        query = queryBuilder.toQuery(context);
+        List<Query> fieldQueries = new ArrayList<> ();
+        for (String type : QueryStringQueryBuilder.allQueryableDefaultFields(context).keySet()) {
+            fieldQueries.add(new ConstantScoreQuery(new TermQuery(new Term("_field_names", type))));
+        }
+        expected = new DisjunctionMaxQuery(fieldQueries, 0f);
+        Query expectedSingle = new MatchAllDocsQuery();
+        assertThat(query, anyOf(equalTo(expected), equalTo(expectedSingle)));
+    }
+
+    public void testDisabledFieldNamesField() throws Exception {
+        QueryShardContext context = createShardContext();
+        context.getMapperService().merge("doc",
+            new CompressedXContent(
+                PutMappingRequest.buildFromSimplifiedDef("doc",
+                    "foo", "type=text",
+                    "_field_names", "enabled=false").string()),
+            MapperService.MergeReason.MAPPING_UPDATE, true);
+        QueryStringQueryBuilder queryBuilder = new QueryStringQueryBuilder("foo:*");
+        Query query = queryBuilder.toQuery(context);
+        Query expected = new WildcardQuery(new Term("foo", "*"));
+        assertThat(query, equalTo(expected));
+        context.getMapperService().merge("doc",
+            new CompressedXContent(
+                PutMappingRequest.buildFromSimplifiedDef("doc",
+                    "foo", "type=text",
+                    "_field_names", "enabled=true").string()),
+            MapperService.MergeReason.MAPPING_UPDATE, true);
+    }
+
     public void testFromJson() throws IOException {
         String json =
                 "{\n" +
@@ -542,15 +930,14 @@ public void testFromJson() throws IOException {
                 "    \"tie_breaker\" : 0.0,\n" +
                 "    \"default_operator\" : \"or\",\n" +
                 "    \"auto_generate_phrase_queries\" : false,\n" +
-                "    \"max_determined_states\" : 10000,\n" +
-                "    \"lowercase_expanded_terms\" : true,\n" +
-                "    \"enable_position_increment\" : true,\n" +
+                "    \"max_determinized_states\" : 10000,\n" +
+                "    \"enable_position_increments\" : true,\n" +
                 "    \"fuzziness\" : \"AUTO\",\n" +
                 "    \"fuzzy_prefix_length\" : 0,\n" +
                 "    \"fuzzy_max_expansions\" : 50,\n" +
                 "    \"phrase_slop\" : 0,\n" +
-                "    \"locale\" : \"und\",\n" +
                 "    \"escape\" : false,\n" +
+                "    \"split_on_whitespace\" : true,\n" +
                 "    \"boost\" : 1.0\n" +
                 "  }\n" +
                 "}";
@@ -562,4 +949,93 @@ public void testFromJson() throws IOException {
         assertEquals(json, "content", parsed.defaultField());
     }
 
+    public void testExpandedTerms() throws Exception {
+        // Prefix
+        Query query = new QueryStringQueryBuilder("aBc*")
+                .field(STRING_FIELD_NAME)
+                .analyzer("whitespace")
+                .toQuery(createShardContext());
+        assertEquals(new PrefixQuery(new Term(STRING_FIELD_NAME, "aBc")), query);
+        query = new QueryStringQueryBuilder("aBc*")
+                .field(STRING_FIELD_NAME)
+                .analyzer("standard")
+                .toQuery(createShardContext());
+        assertEquals(new PrefixQuery(new Term(STRING_FIELD_NAME, "abc")), query);
+
+        // Wildcard
+        query = new QueryStringQueryBuilder("aBc*D")
+                .field(STRING_FIELD_NAME)
+                .analyzer("whitespace")
+                .toQuery(createShardContext());
+        assertEquals(new WildcardQuery(new Term(STRING_FIELD_NAME, "aBc*D")), query);
+        query = new QueryStringQueryBuilder("aBc*D")
+                .field(STRING_FIELD_NAME)
+                .analyzer("standard")
+                .toQuery(createShardContext());
+        assertEquals(new WildcardQuery(new Term(STRING_FIELD_NAME, "abc*d")), query);
+
+        // Fuzzy
+        query = new QueryStringQueryBuilder("aBc~1")
+                .field(STRING_FIELD_NAME)
+                .analyzer("whitespace")
+                .toQuery(createShardContext());
+        FuzzyQuery fuzzyQuery = (FuzzyQuery) query;
+        assertEquals(new Term(STRING_FIELD_NAME, "aBc"), fuzzyQuery.getTerm());
+        query = new QueryStringQueryBuilder("aBc~1")
+                .field(STRING_FIELD_NAME)
+                .analyzer("standard")
+                .toQuery(createShardContext());
+        fuzzyQuery = (FuzzyQuery) query;
+        assertEquals(new Term(STRING_FIELD_NAME, "abc"), fuzzyQuery.getTerm());
+
+        // Range
+        query = new QueryStringQueryBuilder("[aBc TO BcD]")
+                .field(STRING_FIELD_NAME)
+                .analyzer("whitespace")
+                .toQuery(createShardContext());
+        assertEquals(new TermRangeQuery(STRING_FIELD_NAME, new BytesRef("aBc"), new BytesRef("BcD"), true, true), query);
+        query = new QueryStringQueryBuilder("[aBc TO BcD]")
+                .field(STRING_FIELD_NAME)
+                .analyzer("standard")
+                .toQuery(createShardContext());
+        assertEquals(new TermRangeQuery(STRING_FIELD_NAME, new BytesRef("abc"), new BytesRef("bcd"), true, true), query);
+    }
+
+    public void testAllFieldsWithFields() throws IOException {
+        String json =
+                "{\n" +
+                "  \"query_string\" : {\n" +
+                "    \"query\" : \"this AND that OR thus\",\n" +
+                "    \"fields\" : [\"foo\"],\n" +
+                "    \"all_fields\" : true\n" +
+                "  }\n" +
+                "}";
+
+        ParsingException e = expectThrows(ParsingException.class, () -> parseQuery(json));
+        assertThat(e.getMessage(),
+                containsString("cannot use [all_fields] parameter in conjunction with [default_field] or [fields]"));
+
+        String json2 =
+                "{\n" +
+                "  \"query_string\" : {\n" +
+                "    \"query\" : \"this AND that OR thus\",\n" +
+                "    \"default_field\" : \"foo\",\n" +
+                "    \"all_fields\" : true\n" +
+                "  }\n" +
+                "}";
+
+        e = expectThrows(ParsingException.class, () -> parseQuery(json2));
+        assertThat(e.getMessage(),
+                containsString("cannot use [all_fields] parameter in conjunction with [default_field] or [fields]"));
+    }
+
+    public void testInvalidCombo() throws IOException {
+        QueryStringQueryBuilder builder = new QueryStringQueryBuilder("foo bar");
+        builder.autoGeneratePhraseQueries(true);
+        builder.splitOnWhitespace(false);
+        IllegalArgumentException exc =
+            expectThrows(IllegalArgumentException.class, () -> builder.toQuery(createShardContext()));
+        assertEquals(exc.getMessage(),
+            "it is disallowed to disable [split_on_whitespace] if [auto_generate_phrase_queries] is activated");
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/query/RandomQueryBuilder.java b/core/src/test/java/org/elasticsearch/index/query/RandomQueryBuilder.java
index c244273d13a9e..9b064485592a7 100644
--- a/core/src/test/java/org/elasticsearch/index/query/RandomQueryBuilder.java
+++ b/core/src/test/java/org/elasticsearch/index/query/RandomQueryBuilder.java
@@ -19,8 +19,9 @@
 
 package org.elasticsearch.index.query;
 
-import com.carrotsearch.randomizedtesting.generators.RandomInts;
+import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
 import com.carrotsearch.randomizedtesting.generators.RandomStrings;
+
 import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.util.Random;
@@ -38,7 +39,7 @@ public class RandomQueryBuilder {
      * @return a random {@link QueryBuilder}
      */
     public static QueryBuilder createQuery(Random r) {
-        switch (RandomInts.randomIntBetween(r, 0, 3)) {
+        switch (RandomNumbers.randomIntBetween(r, 0, 3)) {
             case 0:
                 return new MatchAllQueryBuilderTests().createTestQueryBuilder();
             case 1:
@@ -61,7 +62,7 @@ public static MultiTermQueryBuilder createMultiTermQuery(Random r) {
         // for now, only use String Rangequeries for MultiTerm test, numeric and date makes little sense
         // see issue #12123 for discussion
         MultiTermQueryBuilder multiTermQueryBuilder;
-        switch(RandomInts.randomIntBetween(r, 0, 3)) {
+        switch(RandomNumbers.randomIntBetween(r, 0, 3)) {
             case 0:
                 RangeQueryBuilder stringRangeQuery = new RangeQueryBuilder(AbstractQueryTestCase.STRING_FIELD_NAME);
                 stringRangeQuery.from("a" + RandomStrings.randomAsciiOfLengthBetween(r, 1, 10));
@@ -69,19 +70,20 @@ public static MultiTermQueryBuilder createMultiTermQuery(Random r) {
                 multiTermQueryBuilder = stringRangeQuery;
                 break;
             case 1:
-                multiTermQueryBuilder = new FuzzyQueryBuilder(AbstractQueryTestCase.STRING_FIELD_NAME, RandomStrings.randomAsciiOfLengthBetween(r, 1, 10));
+                multiTermQueryBuilder = new PrefixQueryBuilderTests().createTestQueryBuilder();
                 break;
             case 2:
-                multiTermQueryBuilder = new PrefixQueryBuilderTests().createTestQueryBuilder();
+                multiTermQueryBuilder = new WildcardQueryBuilderTests().createTestQueryBuilder();
                 break;
             case 3:
-                multiTermQueryBuilder = new WildcardQueryBuilderTests().createTestQueryBuilder();
+                multiTermQueryBuilder = new FuzzyQueryBuilder(AbstractQueryTestCase.STRING_FIELD_NAME,
+                        RandomStrings.randomAsciiOfLengthBetween(r, 1, 10));
                 break;
             default:
                 throw new UnsupportedOperationException();
         }
         if (r.nextBoolean()) {
-            multiTermQueryBuilder.boost(2.0f / RandomInts.randomIntBetween(r, 1, 20));
+            multiTermQueryBuilder.boost(2.0f / RandomNumbers.randomIntBetween(r, 1, 20));
         }
         return multiTermQueryBuilder;
     }
diff --git a/core/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java
index b4fda42177ef8..c1f212aaef8b9 100644
--- a/core/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java
@@ -19,18 +19,25 @@
 
 package org.elasticsearch.index.query;
 
+import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+
 import org.apache.lucene.document.IntPoint;
 import org.apache.lucene.document.LongPoint;
+import org.apache.lucene.search.IndexOrDocValuesQuery;
 import org.apache.lucene.search.LegacyNumericRangeQuery;
 import org.apache.lucene.search.PointRangeQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermRangeQuery;
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.geo.ShapeRelation;
 import org.elasticsearch.common.lucene.BytesRefs;
+import org.elasticsearch.index.mapper.DateFieldMapper;
+import org.elasticsearch.index.mapper.LegacyDateFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MappedFieldType.Relation;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
@@ -58,13 +65,13 @@ protected RangeQueryBuilder doCreateTestQueryBuilder() {
         switch (randomIntBetween(0, 2)) {
             case 0:
                 // use mapped integer field for numeric range queries
-                query = new RangeQueryBuilder(INT_FIELD_NAME);
+                query = new RangeQueryBuilder(randomBoolean() ? INT_FIELD_NAME : INT_RANGE_FIELD_NAME);
                 query.from(randomIntBetween(1, 100));
                 query.to(randomIntBetween(101, 200));
                 break;
             case 1:
                 // use mapped date field, using date string representation
-                query = new RangeQueryBuilder(DATE_FIELD_NAME);
+                query = new RangeQueryBuilder(randomBoolean() ? DATE_FIELD_NAME : DATE_RANGE_FIELD_NAME);
                 query.from(new DateTime(System.currentTimeMillis() - randomIntBetween(0, 1000000), DateTimeZone.UTC).toString());
                 query.to(new DateTime(System.currentTimeMillis() + randomIntBetween(0, 1000000), DateTimeZone.UTC).toString());
                 // Create timestamp option only then we have a date mapper,
@@ -81,8 +88,8 @@ protected RangeQueryBuilder doCreateTestQueryBuilder() {
             case 2:
             default:
                 query = new RangeQueryBuilder(STRING_FIELD_NAME);
-                query.from("a" + randomAsciiOfLengthBetween(1, 10));
-                query.to("z" + randomAsciiOfLengthBetween(1, 10));
+                query.from("a" + randomAlphaOfLengthBetween(1, 10));
+                query.to("z" + randomAlphaOfLengthBetween(1, 10));
                 break;
         }
         query.includeLower(randomBoolean()).includeUpper(randomBoolean());
@@ -92,6 +99,9 @@ protected RangeQueryBuilder doCreateTestQueryBuilder() {
         if (randomBoolean()) {
             query.to(null);
         }
+        if (query.fieldName().equals(INT_RANGE_FIELD_NAME) || query.fieldName().equals(DATE_RANGE_FIELD_NAME)) {
+            query.relation(RandomPicks.randomFrom(random(), ShapeRelation.values()).getRelationName());
+        }
         return query;
     }
 
@@ -116,8 +126,12 @@ protected Map<String, RangeQueryBuilder> getAlternateVersions() {
     }
 
     @Override
-    protected void doAssertLuceneQuery(RangeQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
-        if (getCurrentTypes().length == 0 || (queryBuilder.fieldName().equals(DATE_FIELD_NAME) == false && queryBuilder.fieldName().equals(INT_FIELD_NAME) == false)) {
+    protected void doAssertLuceneQuery(RangeQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
+        if (getCurrentTypes().length == 0 ||
+            (queryBuilder.fieldName().equals(DATE_FIELD_NAME) == false
+                && queryBuilder.fieldName().equals(INT_FIELD_NAME) == false
+                && queryBuilder.fieldName().equals(DATE_RANGE_FIELD_NAME) == false
+                && queryBuilder.fieldName().equals(INT_RANGE_FIELD_NAME) == false)) {
             assertThat(query, instanceOf(TermRangeQuery.class));
             TermRangeQuery termRangeQuery = (TermRangeQuery) query;
             assertThat(termRangeQuery.getField(), equalTo(queryBuilder.fieldName()));
@@ -126,9 +140,74 @@ protected void doAssertLuceneQuery(RangeQueryBuilder queryBuilder, Query query,
             assertThat(termRangeQuery.includesLower(), equalTo(queryBuilder.includeLower()));
             assertThat(termRangeQuery.includesUpper(), equalTo(queryBuilder.includeUpper()));
         } else if (queryBuilder.fieldName().equals(DATE_FIELD_NAME)) {
-            //we can't properly test unmapped dates because LateParsingQuery is package private
+            assertThat(query, either(instanceOf(LegacyNumericRangeQuery.class)).or(instanceOf(IndexOrDocValuesQuery.class)));
+            if (query instanceof IndexOrDocValuesQuery) {
+                query = ((IndexOrDocValuesQuery) query).getIndexQuery();
+                assertThat(query, instanceOf(PointRangeQuery.class));
+            }
+            MapperService mapperService = context.getQueryShardContext().getMapperService();
+            MappedFieldType mappedFieldType = mapperService.fullName(DATE_FIELD_NAME);
+            final Long fromInMillis;
+            final Long toInMillis;
+            // we have to normalize the incoming value into milliseconds since it could be literally anything
+            if (mappedFieldType instanceof  LegacyDateFieldMapper.DateFieldType) {
+                fromInMillis = queryBuilder.from() == null ?  null :
+                    ((LegacyDateFieldMapper.DateFieldType) mappedFieldType).parseToMilliseconds(queryBuilder.from(),
+                        queryBuilder.includeLower(),
+                        queryBuilder.getDateTimeZone(),
+                        queryBuilder.getForceDateParser(), context.getQueryShardContext());
+                toInMillis = queryBuilder.to() == null ? null :
+                    ((LegacyDateFieldMapper.DateFieldType) mappedFieldType).parseToMilliseconds(queryBuilder.to(),
+                        queryBuilder.includeUpper(),
+                        queryBuilder.getDateTimeZone(),
+                        queryBuilder.getForceDateParser(), context.getQueryShardContext());
+            } else if (mappedFieldType instanceof DateFieldMapper.DateFieldType) {
+                fromInMillis = queryBuilder.from() == null ? null :
+                    ((DateFieldMapper.DateFieldType) mappedFieldType).parseToMilliseconds(queryBuilder.from(),
+                        queryBuilder.includeLower(),
+                        queryBuilder.getDateTimeZone(),
+                        queryBuilder.getForceDateParser(), context.getQueryShardContext());
+                toInMillis = queryBuilder.to() == null ? null :
+                    ((DateFieldMapper.DateFieldType) mappedFieldType).parseToMilliseconds(queryBuilder.to(),
+                        queryBuilder.includeUpper(),
+                        queryBuilder.getDateTimeZone(),
+                        queryBuilder.getForceDateParser(), context.getQueryShardContext());
+            } else {
+                fromInMillis = toInMillis = null;
+                fail("unexpected mapped field type: [" + mappedFieldType.getClass() + "] " + mappedFieldType.toString());
+            }
+
+            if (query instanceof LegacyNumericRangeQuery) {
+                LegacyNumericRangeQuery numericRangeQuery = (LegacyNumericRangeQuery) query;
+                assertThat(numericRangeQuery.getField(), equalTo(queryBuilder.fieldName()));
+                assertThat(numericRangeQuery.getMin(), equalTo(fromInMillis));
+                assertThat(numericRangeQuery.getMax(), equalTo(toInMillis));
+                assertThat(numericRangeQuery.includesMin(), equalTo(queryBuilder.includeLower()));
+                assertThat(numericRangeQuery.includesMax(), equalTo(queryBuilder.includeUpper()));
+            } else {
+                Long min = fromInMillis;
+                Long max = toInMillis;
+                long minLong, maxLong;
+                if (min == null) {
+                    minLong = Long.MIN_VALUE;
+                } else {
+                    minLong = min.longValue();
+                    if (queryBuilder.includeLower() == false && minLong != Long.MAX_VALUE) {
+                        minLong++;
+                    }
+                }
+                if (max == null) {
+                    maxLong = Long.MAX_VALUE;
+                } else {
+                    maxLong = max.longValue();
+                    if (queryBuilder.includeUpper() == false && maxLong != Long.MIN_VALUE) {
+                        maxLong--;
+                    }
+                }
+                assertEquals(LongPoint.newRangeQuery(DATE_FIELD_NAME, minLong, maxLong), query);
+            }
         } else if (queryBuilder.fieldName().equals(INT_FIELD_NAME)) {
-            assertThat(query, either(instanceOf(LegacyNumericRangeQuery.class)).or(instanceOf(PointRangeQuery.class)));
+            assertThat(query, either(instanceOf(LegacyNumericRangeQuery.class)).or(instanceOf(IndexOrDocValuesQuery.class)));
             if (query instanceof LegacyNumericRangeQuery) {
                 LegacyNumericRangeQuery numericRangeQuery = (LegacyNumericRangeQuery) query;
                 assertThat(numericRangeQuery.getField(), equalTo(queryBuilder.fieldName()));
@@ -137,6 +216,8 @@ protected void doAssertLuceneQuery(RangeQueryBuilder queryBuilder, Query query,
                 assertThat(numericRangeQuery.includesMin(), equalTo(queryBuilder.includeLower()));
                 assertThat(numericRangeQuery.includesMax(), equalTo(queryBuilder.includeUpper()));
             } else {
+                query = ((IndexOrDocValuesQuery) query).getIndexQuery();
+                assertThat(query, instanceOf(PointRangeQuery.class));
                 Integer min = (Integer) queryBuilder.from();
                 Integer max = (Integer) queryBuilder.to();
                 int minInt, maxInt;
@@ -156,12 +237,11 @@ protected void doAssertLuceneQuery(RangeQueryBuilder queryBuilder, Query query,
                         maxInt--;
                     }
                 }
-                try {
                 assertEquals(IntPoint.newRangeQuery(INT_FIELD_NAME, minInt, maxInt), query);
-                }catch(AssertionError e) {
-                    throw e;
-                }
             }
+        } else if (queryBuilder.fieldName().equals(DATE_RANGE_FIELD_NAME)
+            || queryBuilder.fieldName().equals(INT_RANGE_FIELD_NAME)) {
+            // todo can't check RangeFieldQuery because its currently package private (this will change)
         } else {
             throw new UnsupportedOperationException();
         }
@@ -202,7 +282,7 @@ public void testToQueryNumericField() throws IOException {
         assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         Query parsedQuery = rangeQuery(INT_FIELD_NAME).from(23).to(54).includeLower(true).includeUpper(false).toQuery(createShardContext());
         // since age is automatically registered in data, we encode it as numeric
-        assertThat(parsedQuery, either(instanceOf(LegacyNumericRangeQuery.class)).or(instanceOf(PointRangeQuery.class)));
+        assertThat(parsedQuery, either(instanceOf(LegacyNumericRangeQuery.class)).or(instanceOf(IndexOrDocValuesQuery.class)));
         if (parsedQuery instanceof LegacyNumericRangeQuery) {
             LegacyNumericRangeQuery rangeQuery = (LegacyNumericRangeQuery) parsedQuery;
             assertThat(rangeQuery.getField(), equalTo(INT_FIELD_NAME));
@@ -211,6 +291,8 @@ public void testToQueryNumericField() throws IOException {
             assertThat(rangeQuery.includesMin(), equalTo(true));
             assertThat(rangeQuery.includesMax(), equalTo(false));
         } else {
+            parsedQuery = ((IndexOrDocValuesQuery) parsedQuery).getIndexQuery();
+            assertThat(parsedQuery, instanceOf(PointRangeQuery.class));
             assertEquals(IntPoint.newRangeQuery(INT_FIELD_NAME, 23, 53), parsedQuery);
         }
     }
@@ -227,8 +309,8 @@ public void testDateRangeQueryFormat() throws IOException {
                 "        }\n" +
                 "    }\n" +
                 "}";
-        Query parsedQuery = parseQuery(query).toQuery(createShardContext()).rewrite(null);
-        assertThat(parsedQuery, either(instanceOf(LegacyNumericRangeQuery.class)).or(instanceOf(PointRangeQuery.class)));
+        Query parsedQuery = parseQuery(query).toQuery(createShardContext());
+        assertThat(parsedQuery, either(instanceOf(LegacyNumericRangeQuery.class)).or(instanceOf(IndexOrDocValuesQuery.class)));
 
         if (parsedQuery instanceof LegacyNumericRangeQuery) {
             // Min value was 01/01/2012 (dd/MM/yyyy)
@@ -239,6 +321,8 @@ public void testDateRangeQueryFormat() throws IOException {
             DateTime max = DateTime.parse("2030-01-01T00:00:00.000+00");
             assertThat(((LegacyNumericRangeQuery) parsedQuery).getMax().longValue(), is(max.getMillis()));
         } else {
+            parsedQuery = ((IndexOrDocValuesQuery) parsedQuery).getIndexQuery();
+            assertThat(parsedQuery, instanceOf(PointRangeQuery.class));
             assertEquals(LongPoint.newRangeQuery(DATE_FIELD_NAME,
                     DateTime.parse("2012-01-01T00:00:00.000+00").getMillis(),
                     DateTime.parse("2030-01-01T00:00:00.000+00").getMillis() - 1),
@@ -255,8 +339,7 @@ public void testDateRangeQueryFormat() throws IOException {
                 "        }\n" +
                 "    }\n" +
                 "}";
-        Query rewrittenQuery = parseQuery(invalidQuery).toQuery(createShardContext());
-        expectThrows(ElasticsearchParseException.class, () -> rewrittenQuery.rewrite(null));
+        expectThrows(ElasticsearchParseException.class, () -> parseQuery(invalidQuery).toQuery(createShardContext()));
     }
 
     public void testDateRangeBoundaries() throws IOException {
@@ -269,8 +352,8 @@ public void testDateRangeBoundaries() throws IOException {
                 "        }\n" +
                 "    }\n" +
                 "}\n";
-        Query parsedQuery = parseQuery(query).toQuery(createShardContext()).rewrite(null);
-        assertThat(parsedQuery, either(instanceOf(LegacyNumericRangeQuery.class)).or(instanceOf(PointRangeQuery.class)));
+        Query parsedQuery = parseQuery(query).toQuery(createShardContext());
+        assertThat(parsedQuery, either(instanceOf(LegacyNumericRangeQuery.class)).or(instanceOf(IndexOrDocValuesQuery.class)));
         if (parsedQuery instanceof LegacyNumericRangeQuery) {
             LegacyNumericRangeQuery rangeQuery = (LegacyNumericRangeQuery) parsedQuery;
 
@@ -282,6 +365,8 @@ public void testDateRangeBoundaries() throws IOException {
             assertThat(rangeQuery.getMax().longValue(), is(max.getMillis()));
             assertTrue(rangeQuery.includesMax());
         } else {
+            parsedQuery = ((IndexOrDocValuesQuery) parsedQuery).getIndexQuery();
+            assertThat(parsedQuery, instanceOf(PointRangeQuery.class));
             assertEquals(LongPoint.newRangeQuery(DATE_FIELD_NAME,
                     DateTime.parse("2014-11-01T00:00:00.000+00").getMillis(),
                     DateTime.parse("2014-12-08T23:59:59.999+00").getMillis()),
@@ -296,8 +381,8 @@ public void testDateRangeBoundaries() throws IOException {
                 "        }\n" +
                 "    }\n" +
                 "}";
-        parsedQuery = parseQuery(query).toQuery(createShardContext()).rewrite(null);
-        assertThat(parsedQuery, either(instanceOf(LegacyNumericRangeQuery.class)).or(instanceOf(PointRangeQuery.class)));
+        parsedQuery = parseQuery(query).toQuery(createShardContext());
+        assertThat(parsedQuery, either(instanceOf(LegacyNumericRangeQuery.class)).or(instanceOf(IndexOrDocValuesQuery.class)));
         if (parsedQuery instanceof LegacyNumericRangeQuery) {
             LegacyNumericRangeQuery rangeQuery = (LegacyNumericRangeQuery) parsedQuery;
 
@@ -309,6 +394,8 @@ public void testDateRangeBoundaries() throws IOException {
             assertThat(rangeQuery.getMax().longValue(), is(max.getMillis()));
             assertFalse(rangeQuery.includesMax());
         } else {
+            parsedQuery = ((IndexOrDocValuesQuery) parsedQuery).getIndexQuery();
+            assertThat(parsedQuery, instanceOf(PointRangeQuery.class));
             assertEquals(LongPoint.newRangeQuery(DATE_FIELD_NAME,
                     DateTime.parse("2014-11-30T23:59:59.999+00").getMillis() + 1,
                     DateTime.parse("2014-12-08T00:00:00.000+00").getMillis() - 1),
@@ -328,21 +415,23 @@ public void testDateRangeQueryTimezone() throws IOException {
                 "        }\n" +
                 "    }\n" +
                 "}";
-        Query parsedQuery = parseQuery(query).toQuery(createShardContext()).rewrite(null);
-        if (parsedQuery instanceof PointRangeQuery) {
-            // TODO what can we assert
+        QueryShardContext context = createShardContext();
+        Query parsedQuery = parseQuery(query).toQuery(context);
+        if (parsedQuery instanceof IndexOrDocValuesQuery) {
+            parsedQuery = ((IndexOrDocValuesQuery) parsedQuery).getIndexQuery();
+            assertThat(parsedQuery, instanceOf(PointRangeQuery.class));
         } else {
             assertThat(parsedQuery, instanceOf(LegacyNumericRangeQuery.class));
 
             // Min value was 2012-01-01 (UTC) so we need to remove one hour
             DateTime min = DateTime.parse("2012-01-01T00:00:00.000+01:00");
-            // Max value is when we started the test. So it should be some ms from now
-            DateTime max = new DateTime(startDate, DateTimeZone.UTC);
+            // Max value is the nowInMillis set by the uery shard context
+            long max = context.nowInMillis();
 
             assertThat(((LegacyNumericRangeQuery) parsedQuery).getMin().longValue(), is(min.getMillis()));
 
             // We should not have a big difference here (should be some ms)
-            assertThat(((LegacyNumericRangeQuery) parsedQuery).getMax().longValue() - max.getMillis(), lessThanOrEqualTo(60000L));
+            assertThat(((LegacyNumericRangeQuery) parsedQuery).getMax().longValue() - max, lessThanOrEqualTo(60000L));
         }
 
         query = "{\n" +
@@ -406,18 +495,33 @@ public void testNamedQueryParsing() throws IOException {
                 "  }\n" +
                 "}";
 
-        // non strict parsing should accept "_name" on top level
-        assertNotNull(parseQuery(json, ParseFieldMatcher.EMPTY));
-
-        // with strict parsing, ParseField will throw exception
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-                () -> parseQuery(deprecatedJson, ParseFieldMatcher.STRICT));
-        assertEquals("Deprecated field [_name] used, replaced by [query name is not supported in short version of range query]",
-                e.getMessage());
+        assertNotNull(parseQuery(deprecatedJson));
+        assertWarnings("Deprecated field [_name] used, replaced by [query name is not supported in short version of range query]");
     }
 
     public void testRewriteDateToMatchAll() throws IOException {
-        String fieldName = randomAsciiOfLengthBetween(1, 20);
+        String fieldName = randomAlphaOfLengthBetween(1, 20);
+        RangeQueryBuilder query = new RangeQueryBuilder(fieldName) {
+            @Override
+            protected MappedFieldType.Relation getRelation(QueryRewriteContext queryRewriteContext) throws IOException {
+                return Relation.WITHIN;
+            }
+        };
+        DateTime queryFromValue = new DateTime(2015, 1, 1, 0, 0, 0, ISOChronology.getInstanceUTC());
+        DateTime queryToValue = new DateTime(2016, 1, 1, 0, 0, 0, ISOChronology.getInstanceUTC());
+        query.from(queryFromValue);
+        query.to(queryToValue);
+        QueryShardContext queryShardContext = createShardContext();
+        QueryBuilder rewritten = query.rewrite(queryShardContext);
+        assertThat(rewritten, instanceOf(RangeQueryBuilder.class));
+        RangeQueryBuilder rewrittenRange = (RangeQueryBuilder) rewritten;
+        assertThat(rewrittenRange.fieldName(), equalTo(fieldName));
+        assertThat(rewrittenRange.from(), equalTo(null));
+        assertThat(rewrittenRange.to(), equalTo(null));
+    }
+
+    public void testRewriteDateToMatchAllWithTimezoneAndFormat() throws IOException {
+        String fieldName = randomAlphaOfLengthBetween(1, 20);
         RangeQueryBuilder query = new RangeQueryBuilder(fieldName) {
             @Override
             protected MappedFieldType.Relation getRelation(QueryRewriteContext queryRewriteContext) throws IOException {
@@ -428,6 +532,8 @@ protected MappedFieldType.Relation getRelation(QueryRewriteContext queryRewriteC
         DateTime queryToValue = new DateTime(2016, 1, 1, 0, 0, 0, ISOChronology.getInstanceUTC());
         query.from(queryFromValue);
         query.to(queryToValue);
+        query.timeZone(randomFrom(DateTimeZone.getAvailableIDs()));
+        query.format("yyyy-MM-dd");
         QueryShardContext queryShardContext = createShardContext();
         QueryBuilder rewritten = query.rewrite(queryShardContext);
         assertThat(rewritten, instanceOf(RangeQueryBuilder.class));
@@ -435,10 +541,12 @@ protected MappedFieldType.Relation getRelation(QueryRewriteContext queryRewriteC
         assertThat(rewrittenRange.fieldName(), equalTo(fieldName));
         assertThat(rewrittenRange.from(), equalTo(null));
         assertThat(rewrittenRange.to(), equalTo(null));
+        assertThat(rewrittenRange.timeZone(), equalTo(null));
+        assertThat(rewrittenRange.format(), equalTo(null));
     }
 
     public void testRewriteDateToMatchNone() throws IOException {
-        String fieldName = randomAsciiOfLengthBetween(1, 20);
+        String fieldName = randomAlphaOfLengthBetween(1, 20);
         RangeQueryBuilder query = new RangeQueryBuilder(fieldName) {
             @Override
             protected MappedFieldType.Relation getRelation(QueryRewriteContext queryRewriteContext) throws IOException {
@@ -455,7 +563,7 @@ protected MappedFieldType.Relation getRelation(QueryRewriteContext queryRewriteC
     }
 
     public void testRewriteDateToSame() throws IOException {
-        String fieldName = randomAsciiOfLengthBetween(1, 20);
+        String fieldName = randomAlphaOfLengthBetween(1, 20);
         RangeQueryBuilder query = new RangeQueryBuilder(fieldName) {
             @Override
             protected MappedFieldType.Relation getRelation(QueryRewriteContext queryRewriteContext) throws IOException {
@@ -472,7 +580,7 @@ protected MappedFieldType.Relation getRelation(QueryRewriteContext queryRewriteC
     }
 
     public void testRewriteOpenBoundsToSame() throws IOException {
-        String fieldName = randomAsciiOfLengthBetween(1, 20);
+        String fieldName = randomAlphaOfLengthBetween(1, 20);
         RangeQueryBuilder query = new RangeQueryBuilder(fieldName) {
             @Override
             protected MappedFieldType.Relation getRelation(QueryRewriteContext queryRewriteContext) throws IOException {
diff --git a/core/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java b/core/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java
index d8e7ddf7640b1..d7ef534bcb2bd 100644
--- a/core/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java
@@ -36,8 +36,8 @@ public class RangeQueryRewriteTests extends ESSingleNodeTestCase {
     public void testRewriteMissingField() throws Exception {
         IndexService indexService = createIndex("test");
         IndexReader reader = new MultiReader();
-        QueryRewriteContext context = new QueryRewriteContext(indexService.getIndexSettings(),
-                indexService.mapperService(), null, null, null, reader, null);
+        QueryRewriteContext context = new QueryShardContext(0, indexService.getIndexSettings(), null, null, indexService.mapperService(),
+                null, null, xContentRegistry(), null, reader, null);
         RangeQueryBuilder range = new RangeQueryBuilder("foo");
         assertEquals(Relation.DISJOINT, range.getRelation(context));
     }
@@ -53,8 +53,8 @@ public void testRewriteMissingReader() throws Exception {
             .endObject().endObject().string();
         indexService.mapperService().merge("type",
                 new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE, false);
-        QueryRewriteContext context = new QueryRewriteContext(indexService.getIndexSettings(),
-                indexService.mapperService(), null, null, null, null, null);
+        QueryRewriteContext context = new QueryShardContext(0, indexService.getIndexSettings(), null, null, indexService.mapperService(),
+                null, null, xContentRegistry(), null, null, null);
         RangeQueryBuilder range = new RangeQueryBuilder("foo");
         // can't make assumptions on a missing reader, so it must return INTERSECT
         assertEquals(Relation.INTERSECTS, range.getRelation(context));
@@ -72,8 +72,8 @@ public void testRewriteEmptyReader() throws Exception {
         indexService.mapperService().merge("type",
                 new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE, false);
         IndexReader reader = new MultiReader();
-        QueryRewriteContext context = new QueryRewriteContext(indexService.getIndexSettings(),
-                indexService.mapperService(), null, null, null, reader, null);
+        QueryRewriteContext context = new QueryShardContext(0, indexService.getIndexSettings(), null, null, indexService.mapperService(),
+                null, null, xContentRegistry(), null, reader, null);
         RangeQueryBuilder range = new RangeQueryBuilder("foo");
         // no values -> DISJOINT
         assertEquals(Relation.DISJOINT, range.getRelation(context));
diff --git a/core/src/test/java/org/elasticsearch/index/query/RegexpQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/RegexpQueryBuilderTests.java
index 7daed416a058b..efbb84c3239dd 100644
--- a/core/src/test/java/org/elasticsearch/index/query/RegexpQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/RegexpQueryBuilderTests.java
@@ -22,6 +22,7 @@
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.RegexpQuery;
 import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
@@ -70,13 +71,13 @@ protected Map<String, RegexpQueryBuilder> getAlternateVersions() {
 
     private static RegexpQueryBuilder randomRegexpQuery() {
         // mapped or unmapped fields
-        String fieldName = randomBoolean() ? STRING_FIELD_NAME : randomAsciiOfLengthBetween(1, 10);
-        String value = randomAsciiOfLengthBetween(1, 10);
+        String fieldName = randomBoolean() ? STRING_FIELD_NAME : randomAlphaOfLengthBetween(1, 10);
+        String value = randomAlphaOfLengthBetween(1, 10);
         return new RegexpQueryBuilder(fieldName, value);
     }
 
     @Override
-    protected void doAssertLuceneQuery(RegexpQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
+    protected void doAssertLuceneQuery(RegexpQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
         assertThat(query, instanceOf(RegexpQuery.class));
         RegexpQuery regexpQuery = (RegexpQuery) query;
         assertThat(regexpQuery.getField(), equalTo(queryBuilder.fieldName()));
diff --git a/core/src/test/java/org/elasticsearch/index/query/ScriptQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/ScriptQueryBuilderTests.java
index 63b8101515e69..d273825f9794c 100644
--- a/core/src/test/java/org/elasticsearch/index/query/ScriptQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/ScriptQueryBuilderTests.java
@@ -20,9 +20,11 @@
 package org.elasticsearch.index.query;
 
 import org.apache.lucene.search.Query;
+import org.elasticsearch.index.query.ScriptQueryBuilder.ScriptQuery;
 import org.elasticsearch.script.MockScriptEngine;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.script.ScriptType;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
@@ -37,12 +39,22 @@ public class ScriptQueryBuilderTests extends AbstractQueryTestCase<ScriptQueryBu
     protected ScriptQueryBuilder doCreateTestQueryBuilder() {
         String script = "1";
         Map<String, Object> params = Collections.emptyMap();
-        return new ScriptQueryBuilder(new Script(script, ScriptType.INLINE, MockScriptEngine.NAME, params));
+        return new ScriptQueryBuilder(new Script(ScriptType.INLINE, MockScriptEngine.NAME, script, params));
     }
 
     @Override
-    protected void doAssertLuceneQuery(ScriptQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
+    protected boolean builderGeneratesCacheableQueries() {
+        return false;
+    }
+
+    @Override
+    protected void doAssertLuceneQuery(ScriptQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
         assertThat(query, instanceOf(ScriptQueryBuilder.ScriptQuery.class));
+        // make sure the query would not get cached
+        ScriptQuery sQuery = (ScriptQuery) query;
+        ScriptQuery clone = new ScriptQuery(sQuery.script, sQuery.searchScript);
+        assertFalse(sQuery.equals(clone));
+        assertFalse(sQuery.hashCode() == clone.hashCode());
     }
 
     public void testIllegalConstructorArg() {
@@ -51,12 +63,11 @@ public void testIllegalConstructorArg() {
 
     public void testFromJsonVerbose() throws IOException {
         String json =
-                "{\n" +
+            "{\n" +
                 "  \"script\" : {\n" +
                 "    \"script\" : {\n" +
-                "      \"inline\" : \"5\",\n" +
-                "      \"lang\" : \"mockscript\",\n" +
-                "      \"params\" : { }\n" +
+                "      \"source\" : \"5\",\n" +
+                "      \"lang\" : \"mockscript\"\n" +
                 "    },\n" +
                 "    \"boost\" : 1.0,\n" +
                 "    \"_name\" : \"PcKdEyPOmR\"\n" +
@@ -71,22 +82,27 @@ public void testFromJsonVerbose() throws IOException {
 
     public void testFromJson() throws IOException {
         String json =
-                "{\n" +
-                        "  \"script\" : {\n" +
-                        "    \"script\" : \"5\"," +
-                        "    \"boost\" : 1.0,\n" +
-                        "    \"_name\" : \"PcKdEyPOmR\"\n" +
-                        "  }\n" +
-                        "}";
+            "{\n" +
+                "  \"script\" : {\n" +
+                "    \"script\" : \"5\"," +
+                "    \"boost\" : 1.0,\n" +
+                "    \"_name\" : \"PcKdEyPOmR\"\n" +
+                "  }\n" +
+                "}";
 
         ScriptQueryBuilder parsed = (ScriptQueryBuilder) parseQuery(json);
-        assertEquals(json, "5", parsed.script().getScript());
+        assertEquals(json, "5", parsed.script().getIdOrCode());
     }
 
     @Override
     protected Set<String> getObjectsHoldingArbitraryContent() {
         //script_score.script.params can contain arbitrary parameters. no error is expected when
         //adding additional objects within the params object.
-        return Collections.singleton(Script.ScriptField.PARAMS.getPreferredName());
+        return Collections.singleton(Script.PARAMS_PARSE_FIELD.getPreferredName());
+    }
+
+    @Override
+    protected boolean isCachable(ScriptQueryBuilder queryBuilder) {
+        return false;
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/query/SimpleQueryParserTests.java b/core/src/test/java/org/elasticsearch/index/query/SimpleQueryParserTests.java
index 8511ad6d9c72e..de423db442a1b 100644
--- a/core/src/test/java/org/elasticsearch/index/query/SimpleQueryParserTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/SimpleQueryParserTests.java
@@ -20,16 +20,28 @@
 package org.elasticsearch.index.query;
 
 import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.MockSynonymAnalyzer;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.PrefixQuery;
 import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.SynonymQuery;
-import org.apache.lucene.search.PrefixQuery;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.spans.SpanNearQuery;
+import org.apache.lucene.search.spans.SpanOrQuery;
+import org.apache.lucene.search.spans.SpanQuery;
+import org.apache.lucene.search.spans.SpanTermQuery;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.MockFieldMapper;
 import org.elasticsearch.test.ESTestCase;
 
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -37,7 +49,7 @@
 
 public class SimpleQueryParserTests extends ESTestCase {
     private static class MockSimpleQueryParser extends SimpleQueryParser {
-        public MockSimpleQueryParser(Analyzer analyzer, Map<String, Float> weights, int flags, Settings settings) {
+        MockSimpleQueryParser(Analyzer analyzer, Map<String, Float> weights, int flags, Settings settings) {
             super(analyzer, weights, flags, settings, null);
         }
 
@@ -106,4 +118,92 @@ public void testAnalyzerWildcardWithSynonyms() {
         }
     }
 
+    public void testAnalyzerWithGraph() {
+        SimpleQueryParser.Settings settings = new SimpleQueryParser.Settings();
+        settings.analyzeWildcard(true);
+        Map<String, Float> weights = new HashMap<>();
+        weights.put("field1", 1.0f);
+        SimpleQueryParser parser = new MockSimpleQueryParser(new MockSynonymAnalyzer(), weights, -1, settings);
+
+        for (Operator op : Operator.values()) {
+            BooleanClause.Occur defaultOp = op.toBooleanClauseOccur();
+            parser.setDefaultOperator(defaultOp);
+
+            // non-phrase won't detect multi-word synonym because of whitespace splitting
+            Query query = parser.parse("guinea pig");
+
+            Query expectedQuery = new BooleanQuery.Builder()
+                .add(new BooleanClause(new TermQuery(new Term("field1", "guinea")), defaultOp))
+                .add(new BooleanClause(new TermQuery(new Term("field1", "pig")), defaultOp))
+                .build();
+            assertThat(query, equalTo(expectedQuery));
+
+            // phrase will pick it up
+            query = parser.parse("\"guinea pig\"");
+            SpanTermQuery span1 = new SpanTermQuery(new Term("field1", "guinea"));
+            SpanTermQuery span2 = new SpanTermQuery(new Term("field1", "pig"));
+            expectedQuery = new SpanOrQuery(
+                new SpanNearQuery(new SpanQuery[] { span1, span2 }, 0, true),
+                new SpanTermQuery(new Term("field1", "cavy")));
+
+            assertThat(query, equalTo(expectedQuery));
+
+            // phrase with slop
+            query = parser.parse("big \"tiny guinea pig\"~2");
+
+            expectedQuery = new BooleanQuery.Builder()
+                .add(new TermQuery(new Term("field1", "big")), defaultOp)
+                .add(new SpanNearQuery(new SpanQuery[] {
+                    new SpanTermQuery(new Term("field1", "tiny")),
+                    new SpanOrQuery(
+                        new SpanNearQuery(new SpanQuery[] { span1, span2 }, 0, true),
+                        new SpanTermQuery(new Term("field1", "cavy"))
+                    )
+                }, 2, true), defaultOp)
+                .build();
+            assertThat(query, equalTo(expectedQuery));
+        }
+    }
+
+    public void testQuoteFieldSuffix() {
+        SimpleQueryParser.Settings sqpSettings = new SimpleQueryParser.Settings();
+        sqpSettings.quoteFieldSuffix(".quote");
+
+        Settings indexSettings = Settings.builder()
+                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+                .put(IndexMetaData.SETTING_INDEX_UUID, "some_uuid")
+                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                .build();
+        IndexMetaData indexState = IndexMetaData.builder("index").settings(indexSettings).build();
+        IndexSettings settings = new IndexSettings(indexState, Settings.EMPTY);
+        QueryShardContext mockShardContext = new QueryShardContext(0, settings, null, null, null, null, null, xContentRegistry(),
+                null, null, System::currentTimeMillis) {
+            @Override
+            public MappedFieldType fieldMapper(String name) {
+                return new MockFieldMapper.FakeFieldType();
+            }
+        };
+
+        SimpleQueryParser parser = new SimpleQueryParser(new StandardAnalyzer(),
+                Collections.singletonMap("foo", 1f), -1, sqpSettings, mockShardContext);
+        assertEquals(new TermQuery(new Term("foo", "bar")), parser.parse("bar"));
+        assertEquals(new TermQuery(new Term("foo.quote", "bar")), parser.parse("\"bar\""));
+
+        // Now check what happens if foo.quote does not exist
+        mockShardContext = new QueryShardContext(0, settings, null, null, null, null, null, xContentRegistry(),
+                null, null, System::currentTimeMillis) {
+            @Override
+            public MappedFieldType fieldMapper(String name) {
+                if (name.equals("foo.quote")) {
+                    return null;
+                }
+                return new MockFieldMapper.FakeFieldType();
+            }
+        };
+        parser = new SimpleQueryParser(new StandardAnalyzer(),
+                Collections.singletonMap("foo", 1f), -1, sqpSettings, mockShardContext);
+        assertEquals(new TermQuery(new Term("foo", "bar")), parser.parse("bar"));
+        assertEquals(new TermQuery(new Term("foo", "bar")), parser.parse("\"bar\""));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java
index 93fbcfd930f3e..034c4fed64dcb 100644
--- a/core/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java
@@ -23,11 +23,17 @@
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.BoostQuery;
+import org.apache.lucene.search.FuzzyQuery;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
+import org.apache.lucene.search.PrefixQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.util.TestUtil;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
@@ -38,6 +44,7 @@
 import java.util.Map;
 import java.util.Set;
 
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.instanceOf;
@@ -48,19 +55,13 @@ public class SimpleQueryStringBuilderTests extends AbstractQueryTestCase<SimpleQ
 
     @Override
     protected SimpleQueryStringBuilder doCreateTestQueryBuilder() {
-        SimpleQueryStringBuilder result = new SimpleQueryStringBuilder(randomAsciiOfLengthBetween(1, 10));
+        SimpleQueryStringBuilder result = new SimpleQueryStringBuilder(randomAlphaOfLengthBetween(1, 10));
         if (randomBoolean()) {
             result.analyzeWildcard(randomBoolean());
         }
         if (randomBoolean()) {
             result.lenient(randomBoolean());
         }
-        if (randomBoolean()) {
-            result.lowercaseExpandedTerms(randomBoolean());
-        }
-        if (randomBoolean()) {
-            result.locale(randomLocale(random()));
-        }
         if (randomBoolean()) {
             result.minimumShouldMatch(randomMinimumShouldMatch());
         }
@@ -70,6 +71,9 @@ protected SimpleQueryStringBuilder doCreateTestQueryBuilder() {
         if (randomBoolean()) {
             result.defaultOperator(randomFrom(Operator.values()));
         }
+        if (randomBoolean()) {
+            result.quoteFieldSuffix(TestUtil.randomSimpleString(random()));
+        }
         if (randomBoolean()) {
             Set<SimpleQueryStringFlag> flagSet = new HashSet<>();
             int size = randomIntBetween(0, SimpleQueryStringFlag.values().length);
@@ -85,9 +89,9 @@ protected SimpleQueryStringBuilder doCreateTestQueryBuilder() {
         Map<String, Float> fields = new HashMap<>();
         for (int i = 0; i < fieldCount; i++) {
             if (randomBoolean()) {
-                fields.put(randomAsciiOfLengthBetween(1, 10), AbstractQueryBuilder.DEFAULT_BOOST);
+                fields.put(randomAlphaOfLengthBetween(1, 10), AbstractQueryBuilder.DEFAULT_BOOST);
             } else {
-                fields.put(randomBoolean() ? STRING_FIELD_NAME : randomAsciiOfLengthBetween(1, 10), 2.0f / randomIntBetween(1, 20));
+                fields.put(randomBoolean() ? STRING_FIELD_NAME : randomAlphaOfLengthBetween(1, 10), 2.0f / randomIntBetween(1, 20));
             }
         }
         result.fields(fields);
@@ -108,28 +112,11 @@ public void testDefaults() {
         assertEquals("Wrong default default operator.", Operator.OR, qb.defaultOperator());
         assertEquals("Wrong default default operator field.", Operator.OR, SimpleQueryStringBuilder.DEFAULT_OPERATOR);
 
-        assertEquals("Wrong default default locale.", Locale.ROOT, qb.locale());
-        assertEquals("Wrong default default locale field.", Locale.ROOT, SimpleQueryStringBuilder.DEFAULT_LOCALE);
-
         assertEquals("Wrong default default analyze_wildcard.", false, qb.analyzeWildcard());
         assertEquals("Wrong default default analyze_wildcard field.", false, SimpleQueryStringBuilder.DEFAULT_ANALYZE_WILDCARD);
 
-        assertEquals("Wrong default default lowercase_expanded_terms.", true, qb.lowercaseExpandedTerms());
-        assertEquals("Wrong default default lowercase_expanded_terms field.", true,
-                SimpleQueryStringBuilder.DEFAULT_LOWERCASE_EXPANDED_TERMS);
-
         assertEquals("Wrong default default lenient.", false, qb.lenient());
         assertEquals("Wrong default default lenient field.", false, SimpleQueryStringBuilder.DEFAULT_LENIENT);
-
-        assertEquals("Wrong default default locale.", Locale.ROOT, qb.locale());
-        assertEquals("Wrong default default locale field.", Locale.ROOT, SimpleQueryStringBuilder.DEFAULT_LOCALE);
-    }
-
-    public void testDefaultNullLocale() {
-        SimpleQueryStringBuilder qb = new SimpleQueryStringBuilder("The quick brown fox.");
-        qb.locale(null);
-        assertEquals("Setting locale to null should result in returning to default value.", SimpleQueryStringBuilder.DEFAULT_LOCALE,
-                qb.locale());
     }
 
     public void testDefaultNullComplainFlags() {
@@ -212,7 +199,7 @@ public void testFieldsCannotBeSetToNull() {
     }
 
     public void testDefaultFieldParsing() throws IOException {
-        String query = randomAsciiOfLengthBetween(1, 10).toLowerCase(Locale.ROOT);
+        String query = randomAlphaOfLengthBetween(1, 10).toLowerCase(Locale.ROOT);
         String contentString = "{\n" +
                 "    \"simple_query_string\" : {\n" +
                 "      \"query\" : \"" + query + "\"" +
@@ -241,7 +228,7 @@ public void testDefaultFieldParsing() throws IOException {
      * actual functionality of query parsing.
      */
     @Override
-    protected void doAssertLuceneQuery(SimpleQueryStringBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
+    protected void doAssertLuceneQuery(SimpleQueryStringBuilder queryBuilder, Query query, SearchContext context) throws IOException {
         assertThat(query, notNullValue());
 
         if ("".equals(queryBuilder.value())) {
@@ -249,12 +236,10 @@ protected void doAssertLuceneQuery(SimpleQueryStringBuilder queryBuilder, Query
         } else if (queryBuilder.fields().size() > 1) {
             assertThat(query, instanceOf(BooleanQuery.class));
             BooleanQuery boolQuery = (BooleanQuery) query;
-            if (queryBuilder.lowercaseExpandedTerms()) {
-                for (BooleanClause clause : boolQuery.clauses()) {
-                    if (clause.getQuery() instanceof TermQuery) {
-                        TermQuery inner = (TermQuery) clause.getQuery();
-                        assertThat(inner.getTerm().bytes().toString(), is(inner.getTerm().bytes().toString().toLowerCase(Locale.ROOT)));
-                    }
+            for (BooleanClause clause : boolQuery.clauses()) {
+                if (clause.getQuery() instanceof TermQuery) {
+                    TermQuery inner = (TermQuery) clause.getQuery();
+                    assertThat(inner.getTerm().bytes().toString(), is(inner.getTerm().bytes().toString().toLowerCase(Locale.ROOT)));
                 }
             }
             assertThat(boolQuery.clauses().size(), equalTo(queryBuilder.fields().size()));
@@ -270,7 +255,12 @@ protected void doAssertLuceneQuery(SimpleQueryStringBuilder queryBuilder, Query
             Map.Entry<String, Float> field = queryBuilder.fields().entrySet().iterator().next();
             assertTermOrBoostQuery(query, field.getKey(), queryBuilder.value(), field.getValue());
         } else if (queryBuilder.fields().size() == 0) {
-            assertTermQuery(query, MetaData.ALL, queryBuilder.value());
+            MapperService ms = context.mapperService();
+            if (ms.allEnabled()) {
+                assertTermQuery(query, MetaData.ALL, queryBuilder.value());
+            } else {
+                assertThat(query.getClass(), equalTo(MatchNoDocsQuery.class));
+            }
         } else {
             fail("Encountered lucene query type we do not have a validation implementation for in our "
                     + SimpleQueryStringBuilderTests.class.getSimpleName());
@@ -329,10 +319,9 @@ public void testFromJson() throws IOException {
                 "    \"analyzer\" : \"snowball\",\n" +
                 "    \"flags\" : -1,\n" +
                 "    \"default_operator\" : \"and\",\n" +
-                "    \"lowercase_expanded_terms\" : true,\n" +
                 "    \"lenient\" : false,\n" +
                 "    \"analyze_wildcard\" : false,\n" +
-                "    \"locale\" : \"und\",\n" +
+                "    \"quote_field_suffix\" : \".quote\",\n" +
                 "    \"boost\" : 1.0\n" +
                 "  }\n" +
                 "}";
@@ -343,6 +332,7 @@ public void testFromJson() throws IOException {
         assertEquals(json, "\"fried eggs\" +(eggplant | potato) -frittata", parsed.value());
         assertEquals(json, 2, parsed.fields().size());
         assertEquals(json, "snowball", parsed.analyzer());
+        assertEquals(json, ".quote", parsed.quoteFieldSuffix());
     }
 
     public void testMinimumShouldMatch() throws IOException {
@@ -388,4 +378,47 @@ public void testIndexMetaField() throws IOException {
             assertThat(query, instanceOf(MatchAllDocsQuery.class));
         }
     }
+
+    public void testExpandedTerms() throws Exception {
+        // Prefix
+        Query query = new SimpleQueryStringBuilder("aBc*")
+                .field(STRING_FIELD_NAME)
+                .analyzer("whitespace")
+                .toQuery(createShardContext());
+        assertEquals(new PrefixQuery(new Term(STRING_FIELD_NAME, "aBc")), query);
+        query = new SimpleQueryStringBuilder("aBc*")
+                .field(STRING_FIELD_NAME)
+                .analyzer("standard")
+                .toQuery(createShardContext());
+        assertEquals(new PrefixQuery(new Term(STRING_FIELD_NAME, "abc")), query);
+
+        // Fuzzy
+        query = new SimpleQueryStringBuilder("aBc~1")
+                .field(STRING_FIELD_NAME)
+                .analyzer("whitespace")
+                .toQuery(createShardContext());
+        FuzzyQuery expected = new FuzzyQuery(new Term(STRING_FIELD_NAME, "aBc"), 1);
+        assertEquals(expected, query);
+        query = new SimpleQueryStringBuilder("aBc~1")
+                .field(STRING_FIELD_NAME)
+                .analyzer("standard")
+                .toQuery(createShardContext());
+        expected = new FuzzyQuery(new Term(STRING_FIELD_NAME, "abc"), 1);
+        assertEquals(expected, query);
+    }
+
+    public void testAllFieldsWithFields() throws IOException {
+        String json =
+                "{\n" +
+                "  \"simple_query_string\" : {\n" +
+                "    \"query\" : \"this that thus\",\n" +
+                "    \"fields\" : [\"foo\"],\n" +
+                "    \"all_fields\" : true\n" +
+                "  }\n" +
+                "}";
+
+        ParsingException e = expectThrows(ParsingException.class, () -> parseQuery(json));
+        assertThat(e.getMessage(),
+                containsString("cannot use [all_fields] parameter in conjunction with [fields]"));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/query/SpanContainingQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/SpanContainingQueryBuilderTests.java
index abaa35818c90a..153958b9af3e0 100644
--- a/core/src/test/java/org/elasticsearch/index/query/SpanContainingQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/SpanContainingQueryBuilderTests.java
@@ -21,6 +21,7 @@
 
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.spans.SpanContainingQuery;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
@@ -35,7 +36,7 @@ protected SpanContainingQueryBuilder doCreateTestQueryBuilder() {
     }
 
     @Override
-    protected void doAssertLuceneQuery(SpanContainingQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
+    protected void doAssertLuceneQuery(SpanContainingQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
         assertThat(query, instanceOf(SpanContainingQuery.class));
     }
 
diff --git a/core/src/test/java/org/elasticsearch/index/query/SpanFirstQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/SpanFirstQueryBuilderTests.java
index d74ae56fe0feb..8fd75e346ce08 100644
--- a/core/src/test/java/org/elasticsearch/index/query/SpanFirstQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/SpanFirstQueryBuilderTests.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
@@ -39,7 +40,7 @@ protected SpanFirstQueryBuilder doCreateTestQueryBuilder() {
     }
 
     @Override
-    protected void doAssertLuceneQuery(SpanFirstQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
+    protected void doAssertLuceneQuery(SpanFirstQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
         assertThat(query, instanceOf(SpanFirstQuery.class));
     }
 
diff --git a/core/src/test/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilderTests.java
index 8abaee66725d5..50371ced0e02e 100644
--- a/core/src/test/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilderTests.java
@@ -19,12 +19,18 @@
 
 package org.elasticsearch.index.query;
 
+import org.apache.lucene.index.Term;
 import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.MultiTermQuery;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.spans.SpanBoostQuery;
 import org.apache.lucene.search.spans.SpanMultiTermQueryWrapper;
 import org.apache.lucene.search.spans.SpanQuery;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
@@ -41,7 +47,7 @@ protected SpanMultiTermQueryBuilder doCreateTestQueryBuilder() {
     }
 
     @Override
-    protected void doAssertLuceneQuery(SpanMultiTermQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
+    protected void doAssertLuceneQuery(SpanMultiTermQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
         if (queryBuilder.innerQuery().boost() != AbstractQueryBuilder.DEFAULT_BOOST) {
             assertThat(query, instanceOf(SpanBoostQuery.class));
             SpanBoostQuery boostQuery = (SpanBoostQuery) query;
@@ -50,7 +56,7 @@ protected void doAssertLuceneQuery(SpanMultiTermQueryBuilder queryBuilder, Query
         }
         assertThat(query, instanceOf(SpanMultiTermQueryWrapper.class));
         SpanMultiTermQueryWrapper spanMultiTermQueryWrapper = (SpanMultiTermQueryWrapper) query;
-        Query multiTermQuery = queryBuilder.innerQuery().toQuery(context);
+        Query multiTermQuery = queryBuilder.innerQuery().toQuery(context.getQueryShardContext());
         if (queryBuilder.innerQuery().boost() != AbstractQueryBuilder.DEFAULT_BOOST) {
             assertThat(multiTermQuery, instanceOf(BoostQuery.class));
             BoostQuery boostQuery = (BoostQuery) multiTermQuery;
@@ -71,12 +77,57 @@ public void testIllegalArgument() {
      * to a date.
      */
     public void testUnsupportedInnerQueryType() throws IOException {
-        QueryShardContext context = createShardContext();
-        // test makes only sense if we have at least one type registered with date field mapping
-        assumeTrue("test runs only if there is a registered type",
-                getCurrentTypes().length > 0 && context.fieldMapper(DATE_FIELD_NAME) != null);
+        MultiTermQueryBuilder query = new MultiTermQueryBuilder() {
+            @Override
+            public Query toQuery(QueryShardContext context) throws IOException {
+                return new TermQuery(new Term("foo", "bar"));
+            }
 
-        RangeQueryBuilder query = new RangeQueryBuilder(DATE_FIELD_NAME);
+            @Override
+            public Query toFilter(QueryShardContext context) throws IOException {
+                return toQuery(context);
+            }
+
+            @Override
+            public QueryBuilder queryName(String queryName) {
+                return this;
+            }
+
+            @Override
+            public String queryName() {
+                return "foo";
+            }
+
+            @Override
+            public float boost() {
+                return 1f;
+            }
+
+            @Override
+            public QueryBuilder boost(float boost) {
+                return this;
+            }
+
+            @Override
+            public String getName() {
+                return "foo";
+            }
+
+            @Override
+            public String getWriteableName() {
+                return "foo";
+            }
+
+            @Override
+            public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+                return builder;
+            }
+
+            @Override
+            public void writeTo(StreamOutput out) throws IOException {
+
+            }
+        };
         SpanMultiTermQueryBuilder spamMultiTermQuery = new SpanMultiTermQueryBuilder(query);
         UnsupportedOperationException e = expectThrows(UnsupportedOperationException.class,
                 () -> spamMultiTermQuery.toQuery(createShardContext()));
diff --git a/core/src/test/java/org/elasticsearch/index/query/SpanNearQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/SpanNearQueryBuilderTests.java
index f2f04d04bbf81..8d122021d09a7 100644
--- a/core/src/test/java/org/elasticsearch/index/query/SpanNearQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/SpanNearQueryBuilderTests.java
@@ -20,15 +20,18 @@
 package org.elasticsearch.index.query;
 
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.spans.SpanBoostQuery;
 import org.apache.lucene.search.spans.SpanNearQuery;
 import org.apache.lucene.search.spans.SpanQuery;
+import org.apache.lucene.search.spans.SpanTermQuery;
 import org.elasticsearch.Version;
-import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
 import java.util.Iterator;
 
+import static org.hamcrest.CoreMatchers.either;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.instanceOf;
 
@@ -45,25 +48,33 @@ protected SpanNearQueryBuilder doCreateTestQueryBuilder() {
     }
 
     @Override
-    protected void doAssertLuceneQuery(SpanNearQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
-        assertThat(query, instanceOf(SpanNearQuery.class));
-        SpanNearQuery spanNearQuery = (SpanNearQuery) query;
-        assertThat(spanNearQuery.getSlop(), equalTo(queryBuilder.slop()));
-        assertThat(spanNearQuery.isInOrder(), equalTo(queryBuilder.inOrder()));
-        assertThat(spanNearQuery.getClauses().length, equalTo(queryBuilder.clauses().size()));
-        Iterator<SpanQueryBuilder> spanQueryBuilderIterator = queryBuilder.clauses().iterator();
-        for (SpanQuery spanQuery : spanNearQuery.getClauses()) {
-            assertThat(spanQuery, equalTo(spanQueryBuilderIterator.next().toQuery(context)));
+    protected void doAssertLuceneQuery(SpanNearQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
+        assertThat(query, either(instanceOf(SpanNearQuery.class))
+            .or(instanceOf(SpanTermQuery.class))
+            .or(instanceOf(SpanBoostQuery.class))
+            .or(instanceOf(MatchAllQueryBuilder.class)));
+        if (query instanceof SpanNearQuery) {
+            SpanNearQuery spanNearQuery = (SpanNearQuery) query;
+            assertThat(spanNearQuery.getSlop(), equalTo(queryBuilder.slop()));
+            assertThat(spanNearQuery.isInOrder(), equalTo(queryBuilder.inOrder()));
+            assertThat(spanNearQuery.getClauses().length, equalTo(queryBuilder.clauses().size()));
+            Iterator<SpanQueryBuilder> spanQueryBuilderIterator = queryBuilder.clauses().iterator();
+            for (SpanQuery spanQuery : spanNearQuery.getClauses()) {
+                assertThat(spanQuery, equalTo(spanQueryBuilderIterator.next().toQuery(context.getQueryShardContext())));
+            }
+        } else if (query instanceof SpanTermQuery || query instanceof SpanBoostQuery) {
+            assertThat(queryBuilder.clauses().size(), equalTo(1));
+            assertThat(query, equalTo(queryBuilder.clauses().get(0).toQuery(context.getQueryShardContext())));
         }
     }
 
     public void testIllegalArguments() {
-            IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new SpanNearQueryBuilder(null, 1));
-            assertEquals("[span_near] must include at least one clause", e.getMessage());
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new SpanNearQueryBuilder(null, 1));
+        assertEquals("[span_near] must include at least one clause", e.getMessage());
 
-            SpanNearQueryBuilder spanNearQueryBuilder = new SpanNearQueryBuilder(new SpanTermQueryBuilder("field", "value"), 1);
-            e = expectThrows(IllegalArgumentException.class, () -> spanNearQueryBuilder.addClause(null));
-            assertEquals("[span_near]  clauses cannot be null", e.getMessage());
+        SpanNearQueryBuilder spanNearQueryBuilder = new SpanNearQueryBuilder(new SpanTermQueryBuilder("field", "value"), 1);
+        e = expectThrows(IllegalArgumentException.class, () -> spanNearQueryBuilder.addClause(null));
+        assertEquals("[span_near]  clauses cannot be null", e.getMessage());
     }
 
     public void testClausesUnmodifiable() {
@@ -112,6 +123,28 @@ public void testFromJson() throws IOException {
         assertEquals(json, false, parsed.inOrder());
     }
 
+    public void testParsingSlopDefault() throws IOException {
+        String json =
+                "{\n" +
+                "  \"span_near\" : {\n" +
+                "    \"clauses\" : [ {\n" +
+                "      \"span_term\" : {\n" +
+                "        \"field\" : {\n" +
+                "          \"value\" : \"value1\",\n" +
+                "          \"boost\" : 1.0\n" +
+                "        }\n" +
+                "      }\n" +
+                "    }]\n" +
+                "  }\n" +
+                "}";
+
+        SpanNearQueryBuilder parsed = (SpanNearQueryBuilder) parseQuery(json);
+        assertEquals(json, 1, parsed.clauses().size());
+        assertEquals(json, SpanNearQueryBuilder.DEFAULT_SLOP, parsed.slop());
+        assertEquals(json, SpanNearQueryBuilder.DEFAULT_BOOST, parsed.boost(), 0.0);
+        assertEquals(json, SpanNearQueryBuilder.DEFAULT_IN_ORDER, parsed.inOrder());
+    }
+
     public void testCollectPayloadsDeprecated() throws Exception {
         assertEquals("We can remove support for ignoring collect_payloads in 6.0.0", 5, Version.CURRENT.major);
         String json =
@@ -146,6 +179,8 @@ public void testCollectPayloadsDeprecated() throws Exception {
                 "  }\n" +
                 "}";
 
-        parseQuery(json, ParseFieldMatcher.EMPTY); // Just don't throw an error and we're fine
+        parseQuery(json);
+        // we should get warning headers
+        assertWarnings("Deprecated field [collect_payloads] used, replaced by [no longer supported]");
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/query/SpanNotQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/SpanNotQueryBuilderTests.java
index b8a9dcd2afcfa..961d6092d76e5 100644
--- a/core/src/test/java/org/elasticsearch/index/query/SpanNotQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/SpanNotQueryBuilderTests.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
@@ -54,11 +55,11 @@ protected SpanNotQueryBuilder doCreateTestQueryBuilder() {
     }
 
     @Override
-    protected void doAssertLuceneQuery(SpanNotQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
+    protected void doAssertLuceneQuery(SpanNotQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
         assertThat(query, instanceOf(SpanNotQuery.class));
         SpanNotQuery spanNotQuery = (SpanNotQuery) query;
-        assertThat(spanNotQuery.getExclude(), equalTo(queryBuilder.excludeQuery().toQuery(context)));
-        assertThat(spanNotQuery.getInclude(), equalTo(queryBuilder.includeQuery().toQuery(context)));
+        assertThat(spanNotQuery.getExclude(), equalTo(queryBuilder.excludeQuery().toQuery(context.getQueryShardContext())));
+        assertThat(spanNotQuery.getInclude(), equalTo(queryBuilder.includeQuery().toQuery(context.getQueryShardContext())));
     }
 
     public void testIllegalArgument() {
diff --git a/core/src/test/java/org/elasticsearch/index/query/SpanOrQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/SpanOrQueryBuilderTests.java
index 183bab7a8185e..a480681a5f456 100644
--- a/core/src/test/java/org/elasticsearch/index/query/SpanOrQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/SpanOrQueryBuilderTests.java
@@ -22,6 +22,7 @@
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.spans.SpanOrQuery;
 import org.apache.lucene.search.spans.SpanQuery;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
@@ -42,13 +43,13 @@ protected SpanOrQueryBuilder doCreateTestQueryBuilder() {
     }
 
     @Override
-    protected void doAssertLuceneQuery(SpanOrQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
+    protected void doAssertLuceneQuery(SpanOrQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
         assertThat(query, instanceOf(SpanOrQuery.class));
         SpanOrQuery spanOrQuery = (SpanOrQuery) query;
         assertThat(spanOrQuery.getClauses().length, equalTo(queryBuilder.clauses().size()));
         Iterator<SpanQueryBuilder> spanQueryBuilderIterator = queryBuilder.clauses().iterator();
         for (SpanQuery spanQuery : spanOrQuery.getClauses()) {
-            assertThat(spanQuery, equalTo(spanQueryBuilderIterator.next().toQuery(context)));
+            assertThat(spanQuery, equalTo(spanQueryBuilderIterator.next().toQuery(context.getQueryShardContext())));
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/index/query/SpanTermQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/SpanTermQueryBuilderTests.java
index cfe764615c44d..d21a18c083977 100644
--- a/core/src/test/java/org/elasticsearch/index/query/SpanTermQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/SpanTermQueryBuilderTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.index.query;
 
+import com.fasterxml.jackson.core.io.JsonStringEncoder;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
@@ -26,8 +27,7 @@
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.index.mapper.MappedFieldType;
-
-import com.fasterxml.jackson.core.io.JsonStringEncoder;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 
@@ -45,7 +45,7 @@ protected SpanTermQueryBuilder doCreateTestQueryBuilder() {
             fieldName = STRING_FIELD_NAME;
         }
         if (frequently()) {
-            value = randomAsciiOfLengthBetween(1, 10);
+            value = randomAlphaOfLengthBetween(1, 10);
         } else {
             // generate unicode string in 10% of cases
             JsonStringEncoder encoder = JsonStringEncoder.getInstance();
@@ -53,7 +53,7 @@ protected SpanTermQueryBuilder doCreateTestQueryBuilder() {
         }
 
         if (fieldName == null) {
-            fieldName = randomAsciiOfLengthBetween(1, 10);
+            fieldName = randomAlphaOfLengthBetween(1, 10);
         }
         return createQueryBuilder(fieldName, value);
     }
@@ -64,11 +64,11 @@ protected SpanTermQueryBuilder createQueryBuilder(String fieldName, Object value
     }
 
     @Override
-    protected void doAssertLuceneQuery(SpanTermQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
+    protected void doAssertLuceneQuery(SpanTermQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
         assertThat(query, instanceOf(SpanTermQuery.class));
         SpanTermQuery spanTermQuery = (SpanTermQuery) query;
         assertThat(spanTermQuery.getTerm().field(), equalTo(queryBuilder.fieldName()));
-        MappedFieldType mapper = context.fieldMapper(queryBuilder.fieldName());
+        MappedFieldType mapper = context.getQueryShardContext().fieldMapper(queryBuilder.fieldName());
         if (mapper != null) {
             Term term = ((TermQuery) mapper.termQuery(queryBuilder.value(), null)).getTerm();
             assertThat(spanTermQuery.getTerm(), equalTo(term));
@@ -92,7 +92,7 @@ public SpanTermQueryBuilder[] createSpanTermQueryBuilders(int amount) {
                 spanTermQuery.boost(2.0f / randomIntBetween(1, 20));
             }
             if (randomBoolean()) {
-                spanTermQuery.queryName(randomAsciiOfLengthBetween(1, 10));
+                spanTermQuery.queryName(randomAlphaOfLengthBetween(1, 10));
             }
             clauses[i] = spanTermQuery;
         }
@@ -131,4 +131,13 @@ public void testParseFailsWithMultipleFields() throws IOException {
         assertEquals("[span_term] query doesn't support multiple fields, found [message1] and [message2]", e.getMessage());
     }
 
+    public void testWithMetaDataField() throws IOException {
+        QueryShardContext context = createShardContext();
+        for (String field : new String[]{"field1", "field2"}) {
+            SpanTermQueryBuilder spanTermQueryBuilder = new SpanTermQueryBuilder(field, "toto");
+            Query query = spanTermQueryBuilder.toQuery(context);
+            Query expected = new SpanTermQuery(new Term(field, "toto"));
+            assertEquals(expected, query);
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/query/SpanWithinQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/SpanWithinQueryBuilderTests.java
index ef684412e5474..c897b5ee6a05a 100644
--- a/core/src/test/java/org/elasticsearch/index/query/SpanWithinQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/SpanWithinQueryBuilderTests.java
@@ -21,6 +21,7 @@
 
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.spans.SpanWithinQuery;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
@@ -35,7 +36,7 @@ protected SpanWithinQueryBuilder doCreateTestQueryBuilder() {
     }
 
     @Override
-    protected void doAssertLuceneQuery(SpanWithinQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
+    protected void doAssertLuceneQuery(SpanWithinQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
         assertThat(query, instanceOf(SpanWithinQuery.class));
     }
 
diff --git a/core/src/test/java/org/elasticsearch/index/query/TermQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/TermQueryBuilderTests.java
index 0e6542e4e3d6e..45cbd43864f90 100644
--- a/core/src/test/java/org/elasticsearch/index/query/TermQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/TermQueryBuilderTests.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 
@@ -52,7 +53,7 @@ protected TermQueryBuilder doCreateTestQueryBuilder() {
                     fieldName = STRING_FIELD_NAME;
                 }
                 if (frequently()) {
-                    value = randomAsciiOfLengthBetween(1, 10);
+                    value = randomAlphaOfLengthBetween(1, 10);
                 } else {
                     // generate unicode string in 10% of cases
                     JsonStringEncoder encoder = JsonStringEncoder.getInstance();
@@ -76,7 +77,7 @@ protected TermQueryBuilder doCreateTestQueryBuilder() {
         }
 
         if (fieldName == null) {
-            fieldName = randomAsciiOfLengthBetween(1, 10);
+            fieldName = randomAlphaOfLengthBetween(1, 10);
         }
         return createQueryBuilder(fieldName, value);
     }
@@ -90,9 +91,9 @@ protected TermQueryBuilder createQueryBuilder(String fieldName, Object value) {
     }
 
     @Override
-    protected void doAssertLuceneQuery(TermQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
+    protected void doAssertLuceneQuery(TermQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
         assertThat(query, either(instanceOf(TermQuery.class)).or(instanceOf(PointRangeQuery.class)));
-        MappedFieldType mapper = context.fieldMapper(queryBuilder.fieldName());
+        MappedFieldType mapper = context.getQueryShardContext().fieldMapper(queryBuilder.fieldName());
         if (query instanceof TermQuery) {
             TermQuery termQuery = (TermQuery) query;
             assertThat(termQuery.getTerm().field(), equalTo(queryBuilder.fieldName()));
@@ -119,13 +120,13 @@ public void testTermArray() throws IOException {
 
     public void testFromJson() throws IOException {
         String json =
-                "{\n" + 
-                "  \"term\" : {\n" + 
-                "    \"exact_value\" : {\n" + 
-                "      \"value\" : \"Quick Foxes!\",\n" + 
-                "      \"boost\" : 1.0\n" + 
-                "    }\n" + 
-                "  }\n" + 
+                "{\n" +
+                "  \"term\" : {\n" +
+                "    \"exact_value\" : {\n" +
+                "      \"value\" : \"Quick Foxes!\",\n" +
+                "      \"boost\" : 1.0\n" +
+                "    }\n" +
+                "  }\n" +
                 "}";
 
         TermQueryBuilder parsed = (TermQueryBuilder) parseQuery(json);
diff --git a/core/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java
index 74c55da15b502..57985af91b735 100644
--- a/core/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java
@@ -23,25 +23,27 @@
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.CollectionUtil;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.get.GetResponse;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.get.GetResult;
 import org.elasticsearch.indices.TermsLookup;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
 import org.junit.Before;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.stream.Collectors;
 
@@ -64,7 +66,7 @@ public void randomTerms() {
             }
         }
         this.randomTerms = randomTerms;
-        termsPath = randomAsciiOfLength(10).replace('.', '_');
+        termsPath = randomAlphaOfLength(10).replace('.', '_');
     }
 
     @Override
@@ -76,7 +78,8 @@ protected TermsQueryBuilder doCreateTestQueryBuilder() {
             String fieldName;
             do {
                 fieldName = getRandomFieldName();
-            } while (fieldName.equals(GEO_POINT_FIELD_NAME) || fieldName.equals(GEO_SHAPE_FIELD_NAME));
+            } while (fieldName.equals(GEO_POINT_FIELD_NAME) || fieldName.equals(GEO_SHAPE_FIELD_NAME)
+                || fieldName.equals(INT_RANGE_FIELD_NAME) || fieldName.equals(DATE_RANGE_FIELD_NAME));
             Object[] values = new Object[randomInt(5)];
             for (int i = 0; i < values.length; i++) {
                 values[i] = getRandomValueForFieldName(fieldName);
@@ -84,18 +87,18 @@ protected TermsQueryBuilder doCreateTestQueryBuilder() {
             query = new TermsQueryBuilder(fieldName, values);
         } else {
             // right now the mock service returns us a list of strings
-            query = new TermsQueryBuilder(randomBoolean() ? randomAsciiOfLengthBetween(1,10) : STRING_FIELD_NAME, randomTermsLookup());
+            query = new TermsQueryBuilder(randomBoolean() ? randomAlphaOfLengthBetween(1,10) : STRING_FIELD_NAME, randomTermsLookup());
         }
         return query;
     }
 
     private TermsLookup randomTermsLookup() {
-        return new TermsLookup(randomBoolean() ? randomAsciiOfLength(10) : null, randomAsciiOfLength(10), randomAsciiOfLength(10),
-                termsPath).routing(randomBoolean() ? randomAsciiOfLength(10) : null);
+        return new TermsLookup(randomAlphaOfLength(10), randomAlphaOfLength(10), randomAlphaOfLength(10),
+                termsPath).routing(randomBoolean() ? randomAlphaOfLength(10) : null);
     }
 
     @Override
-    protected void doAssertLuceneQuery(TermsQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
+    protected void doAssertLuceneQuery(TermsQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
         if (queryBuilder.termsLookup() == null && (queryBuilder.values() == null || queryBuilder.values().isEmpty())) {
             assertThat(query, instanceOf(MatchNoDocsQuery.class));
             MatchNoDocsQuery matchNoDocsQuery = (MatchNoDocsQuery) query;
@@ -212,7 +215,7 @@ public void testNumeric() throws IOException {
             TermsQueryBuilder builder = new TermsQueryBuilder("foo", new int[]{1, 3, 4});
             TermsQueryBuilder copy = (TermsQueryBuilder) assertSerialization(builder);
             List<Object> values = copy.values();
-            assertEquals(Arrays.asList(1, 3, 4), values);
+            assertEquals(Arrays.asList(1L, 3L, 4L), values);
         }
         {
             TermsQueryBuilder builder = new TermsQueryBuilder("foo", new double[]{1, 3, 4});
@@ -262,11 +265,9 @@ public void testFromJson() throws IOException {
                         "    \"boost\" : 1.0\n" +
                         "  }\n" +
                         "}";
-        QueryBuilder inShortcutParsed = parseQuery(json, ParseFieldMatcher.EMPTY);
+        QueryBuilder inShortcutParsed = parseQuery(deprecatedJson);
         assertThat(inShortcutParsed, equalTo(parsed));
-
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseQuery(deprecatedJson));
-        assertEquals("Deprecated field [in] used, expected [terms] instead", e.getMessage());
+        assertWarnings("Deprecated field [in] used, expected [terms] instead");
     }
 
     @Override
@@ -279,6 +280,12 @@ public void testMustRewrite() throws IOException {
             randomTerms.stream().filter(x -> x != null).collect(Collectors.toList()))); // terms lookup removes null values
     }
 
+    public void testNullIndexIsDeprecated() throws IOException {
+        TermsQueryBuilder termsQueryBuilder = new TermsQueryBuilder(STRING_FIELD_NAME, new TermsLookup(null, "foo", "bar", "baz"));
+        termsQueryBuilder.rewrite(createShardContext());
+        assertWarnings("Omitting the index in terms lookup is deprecated");
+    }
+
     public void testGeo() throws Exception {
         assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         TermsQueryBuilder query = new TermsQueryBuilder(GEO_POINT_FIELD_NAME, "2,3");
@@ -288,5 +295,34 @@ public void testGeo() throws Exception {
         assertEquals("Geo fields do not support exact searching, use dedicated geo queries instead: [mapped_geo_point]",
                 e.getMessage());
     }
+
+    @Override
+    protected boolean isCachable(TermsQueryBuilder queryBuilder) {
+        // even though we use a terms lookup here we do this during rewrite and that means we are cachable on toQuery
+        // that's why we return true here all the time
+        return super.isCachable(queryBuilder);
+    }
+
+    public void testConversion() {
+        List<Object> list = Arrays.asList();
+        assertSame(Collections.emptyList(), TermsQueryBuilder.convert(list));
+        assertEquals(list, TermsQueryBuilder.convertBack(TermsQueryBuilder.convert(list)));
+
+        list = Arrays.asList("abc");
+        assertEquals(Arrays.asList(new BytesRef("abc")), TermsQueryBuilder.convert(list));
+        assertEquals(list, TermsQueryBuilder.convertBack(TermsQueryBuilder.convert(list)));
+
+        list = Arrays.asList("abc", new BytesRef("def"));
+        assertEquals(Arrays.asList(new BytesRef("abc"), new BytesRef("def")), TermsQueryBuilder.convert(list));
+        assertEquals(Arrays.asList("abc", "def"), TermsQueryBuilder.convertBack(TermsQueryBuilder.convert(list)));
+
+        list = Arrays.asList(5, 42L);
+        assertEquals(Arrays.asList(5L, 42L), TermsQueryBuilder.convert(list));
+        assertEquals(Arrays.asList(5L, 42L), TermsQueryBuilder.convertBack(TermsQueryBuilder.convert(list)));
+
+        list = Arrays.asList(5, 42d);
+        assertEquals(Arrays.asList(5, 42d), TermsQueryBuilder.convert(list));
+        assertEquals(Arrays.asList(5, 42d), TermsQueryBuilder.convertBack(TermsQueryBuilder.convert(list)));
+    }
 }
 
diff --git a/core/src/test/java/org/elasticsearch/index/query/TypeQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/TypeQueryBuilderTests.java
index bcae7e399486c..d2aec9747653d 100644
--- a/core/src/test/java/org/elasticsearch/index/query/TypeQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/TypeQueryBuilderTests.java
@@ -19,27 +19,37 @@
 
 package org.elasticsearch.index.query;
 
+import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.index.mapper.TypeFieldMapper;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
 
+import static org.hamcrest.Matchers.anyOf;
+import static org.hamcrest.Matchers.equalTo;
+
+
 public class TypeQueryBuilderTests extends AbstractQueryTestCase<TypeQueryBuilder> {
 
     @Override
     protected TypeQueryBuilder doCreateTestQueryBuilder() {
-        return new TypeQueryBuilder(getRandomType());
+        return new TypeQueryBuilder("doc");
     }
 
     @Override
-    protected void doAssertLuceneQuery(TypeQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
+    protected void doAssertLuceneQuery(TypeQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
         if (createShardContext().getMapperService().documentMapper(queryBuilder.type()) == null) {
             assertEquals(new MatchNoDocsQuery(), query);
         } else {
-            assertEquals(new TypeFieldMapper.TypeQuery(new BytesRef(queryBuilder.type())), query);
+            assertThat(query,
+                anyOf(
+                    equalTo(new TypeFieldMapper.TypesQuery(new BytesRef(queryBuilder.type()))),
+                    equalTo(new MatchAllDocsQuery()))
+            );
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java
index de9487fca7458..8f24bfb4541b2 100644
--- a/core/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java
@@ -19,9 +19,11 @@
 
 package org.elasticsearch.index.query;
 
+import org.apache.lucene.index.Term;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.WildcardQuery;
 import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
@@ -57,16 +59,16 @@ protected Map<String, WildcardQueryBuilder> getAlternateVersions() {
 
     private static WildcardQueryBuilder randomWildcardQuery() {
         // mapped or unmapped field
-        String text = randomAsciiOfLengthBetween(1, 10);
+        String text = randomAlphaOfLengthBetween(1, 10);
         if (randomBoolean()) {
             return new WildcardQueryBuilder(STRING_FIELD_NAME, text);
         } else {
-            return new WildcardQueryBuilder(randomAsciiOfLengthBetween(1, 10), text);
+            return new WildcardQueryBuilder(randomAlphaOfLengthBetween(1, 10), text);
         }
     }
 
     @Override
-    protected void doAssertLuceneQuery(WildcardQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
+    protected void doAssertLuceneQuery(WildcardQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
         assertThat(query, instanceOf(WildcardQuery.class));
         WildcardQuery wildcardQuery = (WildcardQuery) query;
         assertThat(wildcardQuery.getField(), equalTo(queryBuilder.fieldName()));
@@ -87,7 +89,7 @@ public void testIllegalArguments() {
     public void testEmptyValue() throws IOException {
         QueryShardContext context = createShardContext();
         context.setAllowUnmappedFields(true);
-        WildcardQueryBuilder wildcardQueryBuilder = new WildcardQueryBuilder(getRandomType(), "");
+        WildcardQueryBuilder wildcardQueryBuilder = new WildcardQueryBuilder("doc", "");
         assertEquals(wildcardQueryBuilder.toQuery(context).getClass(), WildcardQuery.class);
     }
 
@@ -124,4 +126,14 @@ public void testParseFailsWithMultipleFields() throws IOException {
         e = expectThrows(ParsingException.class, () -> parseQuery(shortJson));
         assertEquals("[wildcard] query doesn't support multiple fields, found [user1] and [user2]", e.getMessage());
     }
+
+    public void testWithMetaDataField() throws IOException {
+        QueryShardContext context = createShardContext();
+        for (String field : new String[]{"field1", "field2"}) {
+            WildcardQueryBuilder wildcardQueryBuilder = new WildcardQueryBuilder(field, "toto");
+            Query query = wildcardQueryBuilder.toQuery(context);
+            Query expected = new WildcardQuery(new Term(field, "toto"));
+            assertEquals(expected, query);
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/query/WrapperQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/WrapperQueryBuilderTests.java
index ee507365c5668..4a5837d3f7c46 100644
--- a/core/src/test/java/org/elasticsearch/index/query/WrapperQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/WrapperQueryBuilderTests.java
@@ -19,13 +19,16 @@
 
 package org.elasticsearch.index.query;
 
+import org.apache.lucene.index.Term;
 import org.apache.lucene.index.memory.MemoryIndex;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
 import org.elasticsearch.action.support.ToXContentToBytes;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
@@ -54,9 +57,9 @@ protected WrapperQueryBuilder doCreateTestQueryBuilder() {
     }
 
     @Override
-    protected void doAssertLuceneQuery(WrapperQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
+    protected void doAssertLuceneQuery(WrapperQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
         QueryBuilder innerQuery = queryBuilder.rewrite(createShardContext());
-        Query expected = rewrite(innerQuery.toQuery(context));
+        Query expected = rewrite(innerQuery.toQuery(context.getQueryShardContext()));
         assertEquals(rewrite(query), expected);
     }
 
@@ -75,7 +78,7 @@ public void testIllegalArgument() {
      * anything else.
      */
     @Override
-    public void testUnknownField() throws IOException {
+    public void testUnknownField() {
         String json = "{ \"" + WrapperQueryBuilder.NAME + "\" : {\"bogusField\" : \"someValue\"} }";
         ParsingException e = expectThrows(ParsingException.class, () -> parseQuery(json));
         assertTrue(e.getMessage().contains("bogusField"));
@@ -128,6 +131,24 @@ public void testRewriteWithInnerBoost() throws IOException {
         assertEquals(new BoolQueryBuilder().must(query).boost(3), builder.rewrite(shardContext));
     }
 
+    public void testRewriteInnerQueryToo() throws IOException {
+        QueryShardContext shardContext = createShardContext();
+
+        QueryBuilder qb = new WrapperQueryBuilder(
+            new WrapperQueryBuilder(new TermQueryBuilder("foo", "bar").toString()).toString()
+        );
+        assertEquals(new TermQuery(new Term("foo", "bar")), qb.rewrite(shardContext).toQuery(shardContext));
+        qb = new WrapperQueryBuilder(
+            new WrapperQueryBuilder(
+                new WrapperQueryBuilder(new TermQueryBuilder("foo", "bar").toString()).toString()
+            ).toString()
+        );
+        assertEquals(new TermQuery(new Term("foo", "bar")), qb.rewrite(shardContext).toQuery(shardContext));
+
+        qb = new WrapperQueryBuilder(new BoolQueryBuilder().toString());
+        assertEquals(new MatchAllDocsQuery(), qb.rewrite(shardContext).toQuery(shardContext));
+    }
+
     @Override
     protected Query rewrite(Query query) throws IOException {
         // WrapperQueryBuilder adds some optimization if the wrapper and query builder have boosts / query names that wraps
diff --git a/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java
index c5b31eb6a9211..f1849f95b3690 100644
--- a/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java
@@ -24,8 +24,8 @@
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.lucene.search.function.CombineFunction;
@@ -34,12 +34,12 @@
 import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery;
 import org.elasticsearch.common.lucene.search.function.WeightFactorFunction;
 import org.elasticsearch.common.unit.DistanceUnit;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.AbstractQueryBuilder;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.query.RandomQueryBuilder;
 import org.elasticsearch.index.query.TermQueryBuilder;
 import org.elasticsearch.index.query.WrapperQueryBuilder;
@@ -48,8 +48,9 @@
 import org.elasticsearch.plugins.SearchPlugin;
 import org.elasticsearch.script.MockScriptEngine;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.MultiValueMode;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
 import org.hamcrest.Matcher;
 import org.joda.time.DateTime;
@@ -108,7 +109,7 @@ protected FunctionScoreQueryBuilder doCreateTestQueryBuilder() {
     protected Set<String> getObjectsHoldingArbitraryContent() {
         //script_score.script.params can contain arbitrary parameters. no error is expected when adding additional objects
         //within the params object. Score functions get parsed in the data nodes, so they are not validated in the coord node.
-        return new HashSet<>(Arrays.asList(Script.ScriptField.PARAMS.getPreferredName(), ExponentialDecayFunctionBuilder.NAME,
+        return new HashSet<>(Arrays.asList(Script.PARAMS_PARSE_FIELD.getPreferredName(), ExponentialDecayFunctionBuilder.NAME,
                 LinearDecayFunctionBuilder.NAME, GaussDecayFunctionBuilder.NAME));
     }
 
@@ -168,16 +169,18 @@ private static ScoreFunctionBuilder<?> randomScoreFunction() {
             String script = "1";
             Map<String, Object> params = Collections.emptyMap();
             functionBuilder = new ScriptScoreFunctionBuilder(
-                    new Script(script, ScriptService.ScriptType.INLINE, MockScriptEngine.NAME, params));
+                    new Script(ScriptType.INLINE, MockScriptEngine.NAME, script, params));
             break;
         case 3:
             RandomScoreFunctionBuilder randomScoreFunctionBuilder = new RandomScoreFunctionBuilderWithFixedSeed();
-            if (randomBoolean()) {
-                randomScoreFunctionBuilder.seed(randomLong());
-            } else if (randomBoolean()) {
-                randomScoreFunctionBuilder.seed(randomInt());
-            } else {
-                randomScoreFunctionBuilder.seed(randomAsciiOfLengthBetween(1, 10));
+            if (randomBoolean()) { // sometimes provide no seed
+                if (randomBoolean()) {
+                    randomScoreFunctionBuilder.seed(randomLong());
+                } else if (randomBoolean()) {
+                    randomScoreFunctionBuilder.seed(randomInt());
+                } else {
+                    randomScoreFunctionBuilder.seed(randomAlphaOfLengthBetween(1, 10));
+                }
             }
             functionBuilder = randomScoreFunctionBuilder;
             break;
@@ -214,7 +217,7 @@ private static DecayFunctionBuilder<?> createRandomDecayFunction() {
             break;
         case DATE_FIELD_NAME:
             origin = new DateTime(System.currentTimeMillis() - randomIntBetween(0, 1000000), DateTimeZone.UTC).toString();
-            scale = randomPositiveTimeValue();
+            scale = randomTimeValue(1, 1000, new String[]{"d", "h", "ms", "s", "m"});
             offset = randomPositiveTimeValue();
             break;
         default:
@@ -238,7 +241,7 @@ private static DecayFunctionBuilder<?> createRandomDecayFunction() {
     }
 
     @Override
-    protected void doAssertLuceneQuery(FunctionScoreQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
+    protected void doAssertLuceneQuery(FunctionScoreQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
         assertThat(query, either(instanceOf(FunctionScoreQuery.class)).or(instanceOf(FiltersFunctionScoreQuery.class)));
     }
 
@@ -357,7 +360,10 @@ public void testParseFunctionsArray() throws IOException {
             assertThat(functionScoreQueryBuilder.maxBoost(), equalTo(10f));
 
             if (i < XContentType.values().length) {
-                queryBuilder = parseQuery(((AbstractQueryBuilder) queryBuilder).buildAsBytes(XContentType.values()[i]));
+                BytesReference bytes = ((AbstractQueryBuilder) queryBuilder).buildAsBytes(XContentType.values()[i]);
+                try (XContentParser parser = createParser(XContentType.values()[i].xContent(), bytes)) {
+                    queryBuilder = parseQuery(parser);
+                }
             }
         }
     }
@@ -409,7 +415,10 @@ public void testParseSingleFunction() throws IOException {
             assertThat(functionScoreQueryBuilder.maxBoost(), equalTo(10f));
 
             if (i < XContentType.values().length) {
-                queryBuilder = parseQuery(((AbstractQueryBuilder) queryBuilder).buildAsBytes(XContentType.values()[i]));
+                BytesReference bytes = ((AbstractQueryBuilder) queryBuilder).buildAsBytes(XContentType.values()[i]);
+                try (XContentParser parser = createParser(XContentType.values()[i].xContent(), bytes)) {
+                    queryBuilder = parseQuery(parser);
+                }
             }
         }
     }
@@ -547,7 +556,7 @@ public void testMalformedThrowsException() throws IOException {
     }
 
     public void testCustomWeightFactorQueryBuilderWithFunctionScore() throws IOException {
-        Query parsedQuery = parseQuery(functionScoreQuery(termQuery("name.last", "banon"), weightFactorFunction(1.3f)).buildAsBytes())
+        Query parsedQuery = parseQuery(functionScoreQuery(termQuery("name.last", "banon"), weightFactorFunction(1.3f)))
                 .toQuery(createShardContext());
         assertThat(parsedQuery, instanceOf(FunctionScoreQuery.class));
         FunctionScoreQuery functionScoreQuery = (FunctionScoreQuery) parsedQuery;
@@ -556,7 +565,7 @@ public void testCustomWeightFactorQueryBuilderWithFunctionScore() throws IOExcep
     }
 
     public void testCustomWeightFactorQueryBuilderWithFunctionScoreWithoutQueryGiven() throws IOException {
-        Query parsedQuery = parseQuery(functionScoreQuery(weightFactorFunction(1.3f)).buildAsBytes()).toQuery(createShardContext());
+        Query parsedQuery = parseQuery(functionScoreQuery(weightFactorFunction(1.3f))).toQuery(createShardContext());
         assertThat(parsedQuery, instanceOf(FunctionScoreQuery.class));
         FunctionScoreQuery functionScoreQuery = (FunctionScoreQuery) parsedQuery;
         assertThat(functionScoreQuery.getSubQuery() instanceof MatchAllDocsQuery, equalTo(true));
@@ -587,13 +596,13 @@ public void testFromJson() throws IOException {
         String json =
             "{\n" +
                 "  \"function_score\" : {\n" +
-                "    \"query\" : { },\n" +
+                "    \"query\" : { \"match_all\" : {} },\n" +
                 "    \"functions\" : [ {\n" +
-                "      \"filter\" : { },\n" +
+                "      \"filter\" : { \"match_all\" : {}},\n" +
                 "      \"weight\" : 23.0,\n" +
                 "      \"random_score\" : { }\n" +
                 "    }, {\n" +
-                "      \"filter\" : { },\n" +
+                "      \"filter\" : { \"match_all\" : {}},\n" +
                 "      \"weight\" : 5.0\n" +
                 "    } ],\n" +
                 "    \"score_mode\" : \"multiply\",\n" +
@@ -604,18 +613,18 @@ public void testFromJson() throws IOException {
                 "  }\n" +
                 "}";
 
-        FunctionScoreQueryBuilder parsed = (FunctionScoreQueryBuilder) parseQuery(json, ParseFieldMatcher.EMPTY);
+        FunctionScoreQueryBuilder parsed = (FunctionScoreQueryBuilder) parseQuery(json);
         // this should be equivalent to the same with a match_all query
         String expected =
                 "{\n" +
                     "  \"function_score\" : {\n" +
                     "    \"query\" : { \"match_all\" : {} },\n" +
                     "    \"functions\" : [ {\n" +
-                    "      \"filter\" : { },\n" +
+                    "      \"filter\" : { \"match_all\" : {}},\n" +
                     "      \"weight\" : 23.0,\n" +
                     "      \"random_score\" : { }\n" +
                     "    }, {\n" +
-                    "      \"filter\" : { },\n" +
+                    "      \"filter\" : { \"match_all\" : {}},\n" +
                     "      \"weight\" : 5.0\n" +
                     "    } ],\n" +
                     "    \"score_mode\" : \"multiply\",\n" +
@@ -626,7 +635,7 @@ public void testFromJson() throws IOException {
                     "  }\n" +
                     "}";
 
-        FunctionScoreQueryBuilder expectedParsed = (FunctionScoreQueryBuilder) parseQuery(json, ParseFieldMatcher.EMPTY);
+        FunctionScoreQueryBuilder expectedParsed = (FunctionScoreQueryBuilder) parseQuery(json);
         assertEquals(expectedParsed, parsed);
 
         assertEquals(json, 2, parsed.filterFunctionBuilders().length);
@@ -642,11 +651,19 @@ public void testMustRewrite() throws IOException {
     }
 
     public void testRewrite() throws IOException {
-        FunctionScoreQueryBuilder functionScoreQueryBuilder = new FunctionScoreQueryBuilder(
-                new WrapperQueryBuilder(new TermQueryBuilder("foo", "bar").toString()));
+        FunctionScoreQueryBuilder functionScoreQueryBuilder =
+            new FunctionScoreQueryBuilder(new WrapperQueryBuilder(new TermQueryBuilder("foo", "bar").toString()))
+                .boostMode(CombineFunction.REPLACE)
+                .scoreMode(FiltersFunctionScoreQuery.ScoreMode.SUM)
+                .setMinScore(1)
+                .maxBoost(100);
         FunctionScoreQueryBuilder rewrite = (FunctionScoreQueryBuilder) functionScoreQueryBuilder.rewrite(createShardContext());
         assertNotSame(functionScoreQueryBuilder, rewrite);
         assertEquals(rewrite.query(), new TermQueryBuilder("foo", "bar"));
+        assertEquals(rewrite.boostMode(), CombineFunction.REPLACE);
+        assertEquals(rewrite.scoreMode(), FiltersFunctionScoreQuery.ScoreMode.SUM);
+        assertEquals(rewrite.getMinScore(), 1f, 0.0001);
+        assertEquals(rewrite.maxBoost(), 100f, 0.0001);
     }
 
     public void testRewriteWithFunction() throws IOException {
@@ -700,7 +717,7 @@ public void testMalformedQueryFunctionFieldNotSupported() throws IOException {
         expectParsingException(json, "field [not_supported] is not supported");
     }
 
-    public void testMalformedQuery() throws IOException {
+    public void testMalformedQueryMultipleQueryObjects() throws IOException {
         //verify that an error is thrown rather than setting the query twice (https://github.com/elastic/elasticsearch/issues/16583)
         String json = "{\n" +
                 "    \"function_score\":{\n" +
@@ -715,6 +732,25 @@ public void testMalformedQuery() throws IOException {
                 "        }\n" +
                 "    }\n" +
                 "}";
+        expectParsingException(json, equalTo("[bool] malformed query, expected [END_OBJECT] but found [FIELD_NAME]"));
+    }
+
+    public void testMalformedQueryMultipleQueryElements() throws IOException {
+        String json = "{\n" +
+                "    \"function_score\":{\n" +
+                "        \"query\":{\n" +
+                "            \"bool\":{\n" +
+                "                \"must\":{\"match\":{\"field\":\"value\"}}" +
+                "             }\n" +
+                "            },\n" +
+                "        \"query\":{\n" +
+                "            \"bool\":{\n" +
+                "                \"must\":{\"match\":{\"field\":\"value\"}}" +
+                "             }\n" +
+                "            }\n" +
+                "        }\n" +
+                "    }\n" +
+                "}";
         expectParsingException(json, "[query] is already defined.");
     }
 
@@ -733,22 +769,16 @@ private void expectParsingException(String json, String message) {
     static class RandomScoreFunctionBuilderWithFixedSeed extends RandomScoreFunctionBuilder {
         public static final String NAME = "random_with_fixed_seed";
 
-        public RandomScoreFunctionBuilderWithFixedSeed() {
+        RandomScoreFunctionBuilderWithFixedSeed() {
         }
 
         /**
          * Read from a stream.
          */
-        public RandomScoreFunctionBuilderWithFixedSeed(StreamInput in) throws IOException {
+        RandomScoreFunctionBuilderWithFixedSeed(StreamInput in) throws IOException {
             super(in);
         }
 
-        @Override
-        int getCurrentShardId() {
-            // We can't use the method that normal queries use during this test so we hack something instead
-            return 0;
-        }
-
         @Override
         public String getName() {
             return NAME;
@@ -758,7 +788,9 @@ public static RandomScoreFunctionBuilder fromXContent(QueryParseContext parseCon
                 throws IOException, ParsingException {
             RandomScoreFunctionBuilder builder = RandomScoreFunctionBuilder.fromXContent(parseContext);
             RandomScoreFunctionBuilderWithFixedSeed replacement = new RandomScoreFunctionBuilderWithFixedSeed();
-            replacement.seed(builder.getSeed());
+            if (builder.getSeed() != null) {
+                replacement.seed(builder.getSeed());
+            }
             return replacement;
         }
     }
@@ -770,4 +802,18 @@ public List<ScoreFunctionSpec<?>> getScoreFunctions() {
                     RandomScoreFunctionBuilderWithFixedSeed::new, RandomScoreFunctionBuilderWithFixedSeed::fromXContent));
         }
     }
+
+    @Override
+    protected boolean isCachable(FunctionScoreQueryBuilder queryBuilder) {
+        FilterFunctionBuilder[] filterFunctionBuilders = queryBuilder.filterFunctionBuilders();
+        for (FilterFunctionBuilder builder : filterFunctionBuilders) {
+            if (builder.getScoreFunction() instanceof ScriptScoreFunctionBuilder) {
+                return false;
+            } else if (builder.getScoreFunction() instanceof RandomScoreFunctionBuilder
+                && ((RandomScoreFunctionBuilder) builder.getScoreFunction()).getSeed() == null) {
+                return false;
+            }
+        }
+        return true;
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreTests.java b/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreTests.java
index 652c4b4aef30b..16e66f50e0d1c 100644
--- a/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreTests.java
@@ -37,6 +37,7 @@
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.search.Weight;
+import org.apache.lucene.search.SortField;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.Accountable;
 import org.apache.lucene.util.BytesRef;
@@ -135,8 +136,7 @@ public AtomicFieldData loadDirect(LeafReaderContext context) throws Exception {
         }
 
         @Override
-        public IndexFieldData.XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode,
-                IndexFieldData.XFieldComparatorSource.Nested nested) {
+        public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMode, XFieldComparatorSource.Nested nested, boolean reverse) {
             throw new UnsupportedOperationException(UNSUPPORTED);
         }
 
@@ -225,8 +225,7 @@ public AtomicNumericFieldData loadDirect(LeafReaderContext context) throws Excep
         }
 
         @Override
-        public XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode,
-                XFieldComparatorSource.Nested nested) {
+        public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMode, XFieldComparatorSource.Nested nested, boolean reverse) {
             throw new UnsupportedOperationException(UNSUPPORTED);
         }
 
@@ -751,6 +750,33 @@ public void testExplanationAndScoreEqualsEvenIfNoFunctionMatches() throws IOExce
         assertThat(searchResult.scoreDocs[0].score, equalTo(explanation.getValue()));
     }
 
+    public void testWeightFactorNeedsScore() {
+        for (boolean needsScore : new boolean[] {true, false}) {
+            WeightFactorFunction function = new WeightFactorFunction(10.0f, new ScoreFunction(CombineFunction.REPLACE) {
+                @Override
+                public LeafScoreFunction getLeafScoreFunction(LeafReaderContext ctx) throws IOException {
+                    return null;
+                }
+
+                @Override
+                public boolean needsScores() {
+                    return needsScore;
+                }
+
+                @Override
+                protected boolean doEquals(ScoreFunction other) {
+                    return false;
+                }
+
+                @Override
+                protected int doHashCode() {
+                    return 0;
+                }
+            });
+            assertEquals(needsScore, function.needsScores());
+        }
+    }
+
     private static class DummyScoreFunction extends ScoreFunction {
         protected DummyScoreFunction(CombineFunction scoreCombiner) {
             super(scoreCombiner);
diff --git a/core/src/test/java/org/elasticsearch/index/query/plugin/CustomQueryParserIT.java b/core/src/test/java/org/elasticsearch/index/query/plugin/CustomQueryParserIT.java
index 67619a3bbe53c..c2c39be93be5b 100644
--- a/core/src/test/java/org/elasticsearch/index/query/plugin/CustomQueryParserIT.java
+++ b/core/src/test/java/org/elasticsearch/index/query/plugin/CustomQueryParserIT.java
@@ -74,7 +74,8 @@ public void testCustomDummyQueryWithinBooleanQuery() {
 
     private static QueryShardContext queryShardContext() {
         IndicesService indicesService = internalCluster().getDataNodeInstance(IndicesService.class);
-        return indicesService.indexServiceSafe(resolveIndex("index")).newQueryShardContext();
+        return indicesService.indexServiceSafe(resolveIndex("index")).newQueryShardContext(
+                randomInt(20), null, () -> { throw new UnsupportedOperationException(); });
     }
 
     //see #11120
diff --git a/core/src/test/java/org/elasticsearch/index/refresh/RefreshStatsTests.java b/core/src/test/java/org/elasticsearch/index/refresh/RefreshStatsTests.java
new file mode 100644
index 0000000000000..e19c21e852418
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/refresh/RefreshStatsTests.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.refresh;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.test.AbstractStreamableTestCase;
+
+import java.io.IOException;
+
+public class RefreshStatsTests extends AbstractStreamableTestCase<RefreshStats> {
+    @Override
+    protected RefreshStats createTestInstance() {
+        return new RefreshStats(randomNonNegativeLong(), randomNonNegativeLong(), between(0, Integer.MAX_VALUE));
+    }
+
+    @Override
+    protected RefreshStats createBlankInstance() {
+        return new RefreshStats();
+    }
+
+    public void testPre5Dot2() throws IOException {
+        // We can drop the compatibility once the assertion just below this list fails
+        assertTrue(Version.CURRENT.minimumCompatibilityVersion().before(Version.V_5_2_0));
+
+        RefreshStats instance = createTestInstance();
+        RefreshStats copied = copyInstance(instance, Version.V_5_1_1);
+        assertEquals(instance.getTotal(), copied.getTotal());
+        assertEquals(instance.getTotalTimeInMillis(), copied.getTotalTimeInMillis());
+        assertEquals(0, copied.getListeners());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestTestCase.java b/core/src/test/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestTestCase.java
new file mode 100644
index 0000000000000..debeab52d979d
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestTestCase.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex;
+
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.support.ActiveShardCount;
+import org.elasticsearch.tasks.TaskId;
+import org.elasticsearch.test.ESTestCase;
+
+import static org.elasticsearch.common.unit.TimeValue.parseTimeValue;
+
+/**
+ * Shared superclass for testing reindex and friends. In particular it makes sure to test the slice features.
+ */
+public abstract class AbstractBulkByScrollRequestTestCase<R extends AbstractBulkByScrollRequest<R>> extends ESTestCase {
+    public void testForSlice() {
+        R original = newRequest();
+        original.setAbortOnVersionConflict(randomBoolean());
+        original.setRefresh(randomBoolean());
+        original.setTimeout(parseTimeValue(randomPositiveTimeValue(), "timeout"));
+        original.setWaitForActiveShards(
+                randomFrom(ActiveShardCount.ALL, ActiveShardCount.NONE, ActiveShardCount.ONE, ActiveShardCount.DEFAULT));
+        original.setRetryBackoffInitialTime(parseTimeValue(randomPositiveTimeValue(), "retry_backoff_initial_time"));
+        original.setMaxRetries(between(0, 1000));
+        original.setSlices(between(2, 1000));
+        original.setRequestsPerSecond(
+                randomBoolean() ? Float.POSITIVE_INFINITY : randomValueOtherThanMany(r -> r < 0, ESTestCase::randomFloat));
+        original.setSize(randomBoolean() ? AbstractBulkByScrollRequest.SIZE_ALL_MATCHES : between(0, Integer.MAX_VALUE));
+
+        TaskId slicingTask = new TaskId(randomAlphaOfLength(5), randomLong());
+        SearchRequest sliceRequest = new SearchRequest();
+        R forSliced = original.forSlice(slicingTask, sliceRequest);
+        assertEquals(original.isAbortOnVersionConflict(), forSliced.isAbortOnVersionConflict());
+        assertEquals(original.isRefresh(), forSliced.isRefresh());
+        assertEquals(original.getTimeout(), forSliced.getTimeout());
+        assertEquals(original.getWaitForActiveShards(), forSliced.getWaitForActiveShards());
+        assertEquals(original.getRetryBackoffInitialTime(), forSliced.getRetryBackoffInitialTime());
+        assertEquals(original.getMaxRetries(), forSliced.getMaxRetries());
+        assertEquals("only the parent task should store results", false, forSliced.getShouldStoreResult());
+        assertEquals("slice requests always have a single worker", 1, forSliced.getSlices());
+        assertEquals("requests_per_second is split between all workers", original.getRequestsPerSecond() / original.getSlices(),
+                forSliced.getRequestsPerSecond(), Float.MIN_NORMAL);
+        assertEquals("size is split evenly between all workers", original.getSize() == AbstractBulkByScrollRequest.SIZE_ALL_MATCHES
+                ? AbstractBulkByScrollRequest.SIZE_ALL_MATCHES : original.getSize() / original.getSlices(), forSliced.getSize());
+        assertEquals(slicingTask, forSliced.getParentTask());
+
+        extraForSliceAssertions(original, forSliced);
+    }
+
+    protected abstract R newRequest();
+    protected abstract void extraRandomizationForSlice(R original);
+    protected abstract void extraForSliceAssertions(R original, R forSliced);
+}
diff --git a/core/src/test/java/org/elasticsearch/index/reindex/BulkByScrollResponseTests.java b/core/src/test/java/org/elasticsearch/index/reindex/BulkByScrollResponseTests.java
new file mode 100644
index 0000000000000..a288328391a9d
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/reindex/BulkByScrollResponseTests.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.List;
+
+import static java.util.Collections.emptyList;
+import static java.util.Collections.singletonList;
+import static org.apache.lucene.util.TestUtil.randomSimpleString;
+import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
+
+public class BulkByScrollResponseTests extends ESTestCase {
+
+    public void testRountTrip() throws IOException {
+        BulkByScrollResponse response = new BulkByScrollResponse(timeValueMillis(randomNonNegativeLong()),
+                BulkByScrollTaskStatusTests.randomStatus(), randomIndexingFailures(), randomSearchFailures(), randomBoolean());
+        BulkByScrollResponse tripped = new BulkByScrollResponse();
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            response.writeTo(out);
+            try (StreamInput in = out.bytes().streamInput()) {
+                tripped.readFrom(in);
+            }
+        }
+        assertResponseEquals(response, tripped);
+    }
+
+    private List<Failure> randomIndexingFailures() {
+        return usually() ? emptyList()
+                : singletonList(new Failure(randomSimpleString(random()), randomSimpleString(random()),
+                        randomSimpleString(random()), new IllegalArgumentException("test")));
+    }
+
+    private List<ScrollableHitSource.SearchFailure> randomSearchFailures() {
+        if (randomBoolean()) {
+            return emptyList();
+        }
+        String index = null;
+        Integer shardId = null;
+        String nodeId = null;
+        if (randomBoolean()) {
+            index = randomAlphaOfLength(5);
+            shardId = randomInt();
+            nodeId = usually() ? randomAlphaOfLength(5) : null;
+        }
+        return singletonList(new ScrollableHitSource.SearchFailure(new ElasticsearchException("foo"), index, shardId, nodeId));
+    }
+
+    private void assertResponseEquals(BulkByScrollResponse expected, BulkByScrollResponse actual) {
+        assertEquals(expected.getTook(), actual.getTook());
+        BulkByScrollTaskStatusTests.assertTaskStatusEquals(Version.CURRENT, expected.getStatus(), actual.getStatus());
+        assertEquals(expected.getBulkFailures().size(), actual.getBulkFailures().size());
+        for (int i = 0; i < expected.getBulkFailures().size(); i++) {
+            Failure expectedFailure = expected.getBulkFailures().get(i);
+            Failure actualFailure = actual.getBulkFailures().get(i);
+            assertEquals(expectedFailure.getIndex(), actualFailure.getIndex());
+            assertEquals(expectedFailure.getType(), actualFailure.getType());
+            assertEquals(expectedFailure.getId(), actualFailure.getId());
+            assertEquals(expectedFailure.getMessage(), actualFailure.getMessage());
+            assertEquals(expectedFailure.getStatus(), actualFailure.getStatus());
+        }
+        assertEquals(expected.getSearchFailures().size(), actual.getSearchFailures().size());
+        for (int i = 0; i < expected.getSearchFailures().size(); i++) {
+            ScrollableHitSource.SearchFailure expectedFailure = expected.getSearchFailures().get(i);
+            ScrollableHitSource.SearchFailure actualFailure = actual.getSearchFailures().get(i);
+            assertEquals(expectedFailure.getIndex(), actualFailure.getIndex());
+            assertEquals(expectedFailure.getShardId(), actualFailure.getShardId());
+            assertEquals(expectedFailure.getNodeId(), actualFailure.getNodeId());
+            assertEquals(expectedFailure.getReason().getClass(), actualFailure.getReason().getClass());
+            assertEquals(expectedFailure.getReason().getMessage(), actualFailure.getReason().getMessage());
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskStatusTests.java b/core/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskStatusTests.java
new file mode 100644
index 0000000000000..e2bf25ce1b531
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskStatusTests.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex;
+
+import org.apache.lucene.util.LuceneTestCase;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
+import org.elasticsearch.common.Randomness;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.test.ESTestCase;
+import org.hamcrest.Matchers;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.stream.IntStream;
+
+import static java.lang.Math.abs;
+import static java.util.Collections.emptyList;
+import static java.util.stream.Collectors.toList;
+import static org.apache.lucene.util.TestUtil.randomSimpleString;
+import static org.elasticsearch.common.unit.TimeValue.parseTimeValue;
+import static org.hamcrest.Matchers.hasSize;
+
+public class BulkByScrollTaskStatusTests extends ESTestCase {
+    public void testBulkByTaskStatus() throws IOException {
+        BulkByScrollTask.Status status = randomStatus();
+        BytesStreamOutput out = new BytesStreamOutput();
+        status.writeTo(out);
+        BulkByScrollTask.Status tripped = new BulkByScrollTask.Status(out.bytes().streamInput());
+        assertTaskStatusEquals(out.getVersion(), status, tripped);
+
+        // Also check round tripping pre-5.1 which is the first version to support parallelized scroll
+        out = new BytesStreamOutput();
+        out.setVersion(Version.V_5_0_0_rc1); // This can be V_5_0_0
+        status.writeTo(out);
+        StreamInput in = out.bytes().streamInput();
+        in.setVersion(Version.V_5_0_0_rc1);
+        tripped = new BulkByScrollTask.Status(in);
+        assertTaskStatusEquals(Version.V_5_0_0_rc1, status, tripped);
+    }
+
+    /**
+     * Assert that two task statuses are equal after serialization.
+     * @param version the version at which expected was serialized
+     */
+    public static void assertTaskStatusEquals(Version version, BulkByScrollTask.Status expected, BulkByScrollTask.Status actual) {
+        assertEquals(expected.getTotal(), actual.getTotal());
+        assertEquals(expected.getUpdated(), actual.getUpdated());
+        assertEquals(expected.getCreated(), actual.getCreated());
+        assertEquals(expected.getDeleted(), actual.getDeleted());
+        assertEquals(expected.getBatches(), actual.getBatches());
+        assertEquals(expected.getVersionConflicts(), actual.getVersionConflicts());
+        assertEquals(expected.getNoops(), actual.getNoops());
+        assertEquals(expected.getBulkRetries(), actual.getBulkRetries());
+        assertEquals(expected.getSearchRetries(), actual.getSearchRetries());
+        assertEquals(expected.getThrottled(), actual.getThrottled());
+        assertEquals(expected.getRequestsPerSecond(), actual.getRequestsPerSecond(), 0f);
+        assertEquals(expected.getReasonCancelled(), actual.getReasonCancelled());
+        assertEquals(expected.getThrottledUntil(), actual.getThrottledUntil());
+        if (version.onOrAfter(Version.V_5_1_1)) {
+            assertThat(actual.getSliceStatuses(), Matchers.hasSize(expected.getSliceStatuses().size()));
+            for (int i = 0; i < expected.getSliceStatuses().size(); i++) {
+                BulkByScrollTask.StatusOrException sliceStatus = expected.getSliceStatuses().get(i);
+                if (sliceStatus == null) {
+                    assertNull(actual.getSliceStatuses().get(i));
+                } else if (sliceStatus.getException() == null) {
+                    assertNull(actual.getSliceStatuses().get(i).getException());
+                    assertTaskStatusEquals(version, sliceStatus.getStatus(), actual.getSliceStatuses().get(i).getStatus());
+                } else {
+                    assertNull(actual.getSliceStatuses().get(i).getStatus());
+                    // Just check the message because we're not testing exception serialization in general here.
+                    assertEquals(sliceStatus.getException().getMessage(), actual.getSliceStatuses().get(i).getException().getMessage());
+                }
+            }
+        } else {
+            assertEquals(emptyList(), actual.getSliceStatuses());
+        }
+    }
+
+    public static BulkByScrollTask.Status randomStatus() {
+        if (randomBoolean()) {
+            return randomWorkingStatus(null);
+        }
+        boolean canHaveNullStatues = randomBoolean();
+        List<BulkByScrollTask.StatusOrException> statuses = IntStream.range(0, between(0, 10))
+                .mapToObj(i -> {
+                    if (canHaveNullStatues && LuceneTestCase.rarely()) {
+                        return null;
+                    }
+                    if (randomBoolean()) {
+                        return new BulkByScrollTask.StatusOrException(new ElasticsearchException(randomAlphaOfLength(5)));
+                    }
+                    return new BulkByScrollTask.StatusOrException(randomWorkingStatus(i));
+                })
+                .collect(toList());
+        return new BulkByScrollTask.Status(statuses, randomBoolean() ? "test" : null);
+    }
+
+    private static BulkByScrollTask.Status randomWorkingStatus(Integer sliceId) {
+        // These all should be believably small because we sum them if we have multiple workers
+        int total = between(0, 10000000);
+        int updated = between(0, total);
+        int created = between(0, total - updated);
+        int deleted = between(0, total - updated - created);
+        int noops = total - updated - created - deleted;
+        int batches = between(0, 10000);
+        long versionConflicts = between(0, total);
+        long bulkRetries = between(0, 10000000);
+        long searchRetries = between(0, 100000);
+        return new BulkByScrollTask.Status(sliceId, total, updated, created, deleted, batches, versionConflicts, noops, bulkRetries,
+                searchRetries, parseTimeValue(randomPositiveTimeValue(), "test"), abs(Randomness.get().nextFloat()),
+                randomBoolean() ? null : randomSimpleString(Randomness.get()), parseTimeValue(randomPositiveTimeValue(), "test"));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskTests.java b/core/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskTests.java
new file mode 100644
index 0000000000000..9704202ef26d2
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskTests.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex;
+
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import static java.lang.Math.min;
+import static org.elasticsearch.common.unit.TimeValue.parseTimeValue;
+import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
+import static org.elasticsearch.common.unit.TimeValue.timeValueNanos;
+import static org.hamcrest.Matchers.containsString;
+
+public class BulkByScrollTaskTests extends ESTestCase {
+
+    public void testStatusHatesNegatives() {
+        checkStatusNegatives(-1  ,  0, 0, 0, 0, 0, 0, 0, 0, 0, "sliceId");
+        checkStatusNegatives(null, -1, 0, 0, 0, 0, 0, 0, 0, 0, "total");
+        checkStatusNegatives(null, 0, -1, 0, 0, 0, 0, 0, 0, 0, "updated");
+        checkStatusNegatives(null, 0, 0, -1, 0, 0, 0, 0, 0, 0, "created");
+        checkStatusNegatives(null, 0, 0, 0, -1, 0, 0, 0, 0, 0, "deleted");
+        checkStatusNegatives(null, 0, 0, 0, 0, -1, 0, 0, 0, 0, "batches");
+        checkStatusNegatives(null, 0, 0, 0, 0, 0, -1, 0, 0, 0, "versionConflicts");
+        checkStatusNegatives(null, 0, 0, 0, 0, 0, 0, -1, 0, 0, "noops");
+        checkStatusNegatives(null, 0, 0, 0, 0, 0, 0, 0, -1, 0, "bulkRetries");
+        checkStatusNegatives(null, 0, 0, 0, 0, 0, 0, 0, 0, -1, "searchRetries");
+    }
+
+    /**
+     * Build a task status with only some values. Used for testing negative values.
+     */
+    private void checkStatusNegatives(Integer sliceId, long total, long updated, long created, long deleted, int batches,
+            long versionConflicts, long noops, long bulkRetries, long searchRetries, String fieldName) {
+        TimeValue throttle = parseTimeValue(randomPositiveTimeValue(), "test");
+        TimeValue throttledUntil = parseTimeValue(randomPositiveTimeValue(), "test");
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new BulkByScrollTask.Status(sliceId, total, updated,
+                created, deleted, batches, versionConflicts, noops, bulkRetries, searchRetries, throttle, 0f, null, throttledUntil));
+        assertEquals(e.getMessage(), fieldName + " must be greater than 0 but was [-1]");
+    }
+
+    public void testXContentRepresentationOfUnlimitedRequestsPerSecond() throws IOException {
+        XContentBuilder builder = JsonXContent.contentBuilder();
+        BulkByScrollTask.Status status = new BulkByScrollTask.Status(null, 0, 0, 0, 0, 0, 0, 0, 0, 0, timeValueMillis(0),
+                Float.POSITIVE_INFINITY, null, timeValueMillis(0));
+        status.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        assertThat(builder.string(), containsString("\"requests_per_second\":-1"));
+    }
+
+    public void testXContentRepresentationOfUnfinishedSlices() throws IOException {
+        XContentBuilder builder = JsonXContent.contentBuilder();
+        BulkByScrollTask.Status completedStatus = new BulkByScrollTask.Status(2, 0, 0, 0, 0, 0, 0, 0, 0, 0, timeValueMillis(0),
+                Float.POSITIVE_INFINITY, null, timeValueMillis(0));
+        BulkByScrollTask.Status status = new BulkByScrollTask.Status(
+                Arrays.asList(null, null, new BulkByScrollTask.StatusOrException(completedStatus)), null);
+        status.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        assertThat(builder.string(), containsString("\"slices\":[null,null,{\"slice_id\":2"));
+    }
+
+    public void testXContentRepresentationOfSliceFailures() throws IOException {
+        XContentBuilder builder = JsonXContent.contentBuilder();
+        Exception e = new Exception();
+        BulkByScrollTask.Status status = new BulkByScrollTask.Status(Arrays.asList(null, null, new BulkByScrollTask.StatusOrException(e)),
+                null);
+        status.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        assertThat(builder.string(), containsString("\"slices\":[null,null,{\"type\":\"exception\""));
+    }
+
+    public void testMergeStatuses() {
+        BulkByScrollTask.StatusOrException[] statuses = new BulkByScrollTask.StatusOrException[between(2, 100)];
+        boolean containsNullStatuses = randomBoolean();
+        int mergedTotal = 0;
+        int mergedUpdated = 0;
+        int mergedCreated = 0;
+        int mergedDeleted = 0;
+        int mergedBatches = 0;
+        int mergedVersionConflicts = 0;
+        int mergedNoops = 0;
+        int mergedBulkRetries = 0;
+        int mergedSearchRetries = 0;
+        TimeValue mergedThrottled = timeValueNanos(0);
+        float mergedRequestsPerSecond = 0;
+        TimeValue mergedThrottledUntil = timeValueNanos(Integer.MAX_VALUE);
+        for (int i = 0; i < statuses.length; i++) {
+            if (containsNullStatuses && rarely()) {
+                continue;
+            }
+            int total = between(0, 10000);
+            int updated = between(0, total);
+            int created = between(0, total - updated);
+            int deleted = between(0, total - updated - created);
+            int batches = between(0, 10);
+            int versionConflicts = between(0, 100);
+            int noops = total - updated - created - deleted;
+            int bulkRetries = between(0, 100);
+            int searchRetries = between(0, 100);
+            TimeValue throttled = timeValueNanos(between(0, 10000));
+            float requestsPerSecond = randomValueOtherThanMany(r -> r <= 0, () -> randomFloat());
+            String reasonCancelled = randomBoolean() ? null : "test";
+            TimeValue throttledUntil = timeValueNanos(between(0, 1000));
+            statuses[i] = new BulkByScrollTask.StatusOrException(new BulkByScrollTask.Status(i, total, updated, created, deleted, batches,
+                    versionConflicts, noops, bulkRetries, searchRetries, throttled, requestsPerSecond, reasonCancelled, throttledUntil));
+            mergedTotal += total;
+            mergedUpdated += updated;
+            mergedCreated += created;
+            mergedDeleted += deleted;
+            mergedBatches += batches;
+            mergedVersionConflicts += versionConflicts;
+            mergedNoops += noops;
+            mergedBulkRetries += bulkRetries;
+            mergedSearchRetries += searchRetries;
+            mergedThrottled = timeValueNanos(mergedThrottled.nanos() + throttled.nanos());
+            mergedRequestsPerSecond += requestsPerSecond;
+            mergedThrottledUntil = timeValueNanos(min(mergedThrottledUntil.nanos(), throttledUntil.nanos()));
+        }
+        String reasonCancelled = randomBoolean() ? randomAlphaOfLength(10) : null;
+        BulkByScrollTask.Status merged = new BulkByScrollTask.Status(Arrays.asList(statuses), reasonCancelled);
+        assertEquals(mergedTotal, merged.getTotal());
+        assertEquals(mergedUpdated, merged.getUpdated());
+        assertEquals(mergedCreated, merged.getCreated());
+        assertEquals(mergedDeleted, merged.getDeleted());
+        assertEquals(mergedBatches, merged.getBatches());
+        assertEquals(mergedVersionConflicts, merged.getVersionConflicts());
+        assertEquals(mergedNoops, merged.getNoops());
+        assertEquals(mergedBulkRetries, merged.getBulkRetries());
+        assertEquals(mergedSearchRetries, merged.getSearchRetries());
+        assertEquals(mergedThrottled, merged.getThrottled());
+        assertEquals(mergedRequestsPerSecond, merged.getRequestsPerSecond(), 0.0001f);
+        assertEquals(mergedThrottledUntil, merged.getThrottledUntil());
+        assertEquals(reasonCancelled, merged.getReasonCancelled());
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryRequestTests.java b/core/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryRequestTests.java
new file mode 100644
index 0000000000000..511cb1d0de6bc
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryRequestTests.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex;
+
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.support.IndicesOptions;
+
+import static org.apache.lucene.util.TestUtil.randomSimpleString;
+
+public class DeleteByQueryRequestTests extends AbstractBulkByScrollRequestTestCase<DeleteByQueryRequest> {
+    public void testDeleteteByQueryRequestImplementsIndicesRequestReplaceable() {
+        int numIndices = between(1, 100);
+        String[] indices = new String[numIndices];
+        for (int i = 0; i < numIndices; i++) {
+            indices[i] = randomSimpleString(random(), 1, 30);
+        }
+
+        SearchRequest searchRequest = new SearchRequest(indices);
+        IndicesOptions indicesOptions = IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean());
+        searchRequest.indicesOptions(indicesOptions);
+
+        DeleteByQueryRequest request = new DeleteByQueryRequest(searchRequest);
+        for (int i = 0; i < numIndices; i++) {
+            assertEquals(indices[i], request.indices()[i]);
+        }
+
+        assertSame(indicesOptions, request.indicesOptions());
+        assertSame(request.indicesOptions(), request.getSearchRequest().indicesOptions());
+
+        int numNewIndices = between(1, 100);
+        String[] newIndices = new String[numNewIndices];
+        for (int i = 0; i < numNewIndices; i++) {
+            newIndices[i] = randomSimpleString(random(), 1, 30);
+        }
+        request.indices(newIndices);
+        for (int i = 0; i < numNewIndices; i++) {;
+            assertEquals(newIndices[i], request.indices()[i]);
+        }
+        for (int i = 0; i < numNewIndices; i++) {;
+            assertEquals(newIndices[i], request.getSearchRequest().indices()[i]);
+        }
+    }
+
+    @Override
+    protected DeleteByQueryRequest newRequest() {
+        return new DeleteByQueryRequest(new SearchRequest(randomAlphaOfLength(5)));
+    }
+
+    @Override
+    protected void extraRandomizationForSlice(DeleteByQueryRequest original) {
+        // Nothing else to randomize
+    }
+
+    @Override
+    protected void extraForSliceAssertions(DeleteByQueryRequest original, DeleteByQueryRequest forSliced) {
+        // No extra assertions needed
+    }
+
+    public void testTypesGetter() {
+        int numTypes = between(1, 50);
+        String[] types = new String[numTypes];
+        for (int i = 0; i < numTypes; i++) {
+            types[i] = randomSimpleString(random(), 1, 30);
+        }
+        SearchRequest searchRequest = new SearchRequest();
+        searchRequest.types(types);
+        DeleteByQueryRequest request = new DeleteByQueryRequest(searchRequest);
+        assertArrayEquals(request.types(), types);
+    }
+
+    public void testTypesSetter() {
+        int numTypes = between(1, 50);
+        String[] types = new String[numTypes];
+        for (int i = 0; i < numTypes; i++) {
+            types[i] = randomSimpleString(random(), 1, 30);
+        }
+        SearchRequest searchRequest = new SearchRequest();
+        DeleteByQueryRequest request = new DeleteByQueryRequest(searchRequest);
+        request.types(types);
+        assertArrayEquals(request.types(), types);
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/reindex/ParentBulkByScrollTaskTests.java b/core/src/test/java/org/elasticsearch/index/reindex/ParentBulkByScrollTaskTests.java
new file mode 100644
index 0000000000000..6e2d44abed53a
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/reindex/ParentBulkByScrollTaskTests.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.Before;
+import org.mockito.ArgumentCaptor;
+
+import java.util.Arrays;
+import java.util.List;
+
+import static java.util.Collections.emptyList;
+import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
+import static org.mockito.Mockito.atMost;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+
+public class ParentBulkByScrollTaskTests extends ESTestCase {
+    private int slices;
+    private ParentBulkByScrollTask task;
+
+    @Before
+    public void createTask() {
+        slices = between(2, 50);
+        task = new ParentBulkByScrollTask(1, "test_type", "test_action", "test", null, slices);
+    }
+
+    public void testBasicData() {
+        assertEquals(1, task.getId());
+        assertEquals("test_type", task.getType());
+        assertEquals("test_action", task.getAction());
+        assertEquals("test", task.getDescription());
+    }
+
+    public void testProgress() {
+        long total = 0;
+        long created = 0;
+        long updated = 0;
+        long deleted = 0;
+        long noops = 0;
+        long versionConflicts = 0;
+        int batches = 0;
+        List<BulkByScrollTask.StatusOrException> sliceStatuses = Arrays.asList(new BulkByScrollTask.StatusOrException[slices]);
+        BulkByScrollTask.Status status = task.getStatus();
+        assertEquals(total, status.getTotal());
+        assertEquals(created, status.getCreated());
+        assertEquals(updated, status.getUpdated());
+        assertEquals(deleted, status.getDeleted());
+        assertEquals(noops, status.getNoops());
+        assertEquals(versionConflicts, status.getVersionConflicts());
+        assertEquals(batches, status.getBatches());
+        assertEquals(sliceStatuses, status.getSliceStatuses());
+
+        for (int slice = 0; slice < slices; slice++) {
+            int thisTotal = between(10, 10000);
+            int thisCreated = between(0, thisTotal);
+            int thisUpdated = between(0, thisTotal - thisCreated);
+            int thisDeleted = between(0, thisTotal - thisCreated - thisUpdated);
+            int thisNoops = thisTotal - thisCreated - thisUpdated - thisDeleted;
+            int thisVersionConflicts = between(0, 1000);
+            int thisBatches = between(1, 100);
+            BulkByScrollTask.Status sliceStatus = new BulkByScrollTask.Status(slice, thisTotal, thisUpdated, thisCreated, thisDeleted,
+                    thisBatches, thisVersionConflicts, thisNoops, 0, 0, timeValueMillis(0), 0, null, timeValueMillis(0));
+            total += thisTotal;
+            created += thisCreated;
+            updated += thisUpdated;
+            deleted += thisDeleted;
+            noops += thisNoops;
+            versionConflicts += thisVersionConflicts;
+            batches += thisBatches;
+            sliceStatuses.set(slice, new BulkByScrollTask.StatusOrException(sliceStatus));
+
+            @SuppressWarnings("unchecked")
+            ActionListener<BulkByScrollResponse> listener = slice < slices - 1 ? neverCalled() : mock(ActionListener.class);
+            task.onSliceResponse(listener, slice,
+                    new BulkByScrollResponse(timeValueMillis(10), sliceStatus, emptyList(), emptyList(), false));
+
+            status = task.getStatus();
+            assertEquals(total, status.getTotal());
+            assertEquals(created, status.getCreated());
+            assertEquals(updated, status.getUpdated());
+            assertEquals(deleted, status.getDeleted());
+            assertEquals(versionConflicts, status.getVersionConflicts());
+            assertEquals(batches, status.getBatches());
+            assertEquals(noops, status.getNoops());
+            assertEquals(sliceStatuses, status.getSliceStatuses());
+
+            if (slice == slices - 1) {
+                // The whole thing succeeded so we should have got the success
+                status = captureResponse(BulkByScrollResponse.class, listener).getStatus();
+                assertEquals(total, status.getTotal());
+                assertEquals(created, status.getCreated());
+                assertEquals(updated, status.getUpdated());
+                assertEquals(deleted, status.getDeleted());
+                assertEquals(versionConflicts, status.getVersionConflicts());
+                assertEquals(batches, status.getBatches());
+                assertEquals(noops, status.getNoops());
+                assertEquals(sliceStatuses, status.getSliceStatuses());
+            }
+        }
+    }
+
+    private <T> ActionListener<T> neverCalled() {
+        return new ActionListener<T>() {
+            @Override
+            public void onResponse(T response) {
+                throw new RuntimeException("Expected no interactions but got [" + response + "]");
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                throw new RuntimeException("Expected no interations but was received a failure", e);
+            }
+        };
+    }
+
+
+    private <T> T captureResponse(Class<T> responseClass, ActionListener<T> listener) {
+        ArgumentCaptor<Exception> failure = ArgumentCaptor.forClass(Exception.class);
+        // Rethrow any failures just so we get a nice exception if there were any. We don't expect any though.
+        verify(listener, atMost(1)).onFailure(failure.capture());
+        if (false == failure.getAllValues().isEmpty()) {
+            throw new AssertionError(failure.getValue());
+        }
+        ArgumentCaptor<T> response = ArgumentCaptor.forClass(responseClass);
+        verify(listener).onResponse(response.capture());
+        return response.getValue();
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/index/reindex/ReindexRequestTests.java b/core/src/test/java/org/elasticsearch/index/reindex/ReindexRequestTests.java
new file mode 100644
index 0000000000000..a2349126078ba
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/reindex/ReindexRequestTests.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex;
+
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.search.slice.SliceBuilder;
+
+import static java.util.Collections.emptyMap;
+import static org.elasticsearch.common.unit.TimeValue.parseTimeValue;
+import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
+
+/**
+ * Tests some of the validation of {@linkplain ReindexRequest}. See reindex's rest tests for much more.
+ */
+public class ReindexRequestTests extends AbstractBulkByScrollRequestTestCase<ReindexRequest> {
+    public void testTimestampAndTtlNotAllowed() {
+        ReindexRequest reindex = newRequest();
+        reindex.getDestination().ttl("1s").timestamp("now");
+        ActionRequestValidationException e = reindex.validate();
+        assertEquals("Validation Failed: 1: setting ttl on destination isn't supported. use scripts instead.;"
+                + "2: setting timestamp on destination isn't supported. use scripts instead.;",
+                e.getMessage());
+    }
+
+    public void testReindexFromRemoteDoesNotSupportSearchQuery() {
+        ReindexRequest reindex = newRequest();
+        reindex.setRemoteInfo(
+                new RemoteInfo(randomAlphaOfLength(5), randomAlphaOfLength(5), between(1, Integer.MAX_VALUE), new BytesArray("real_query"),
+                        null, null, emptyMap(), RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT));
+        reindex.getSearchRequest().source().query(matchAllQuery()); // Unsupported place to put query
+        ActionRequestValidationException e = reindex.validate();
+        assertEquals("Validation Failed: 1: reindex from remote sources should use RemoteInfo's query instead of source's query;",
+                e.getMessage());
+    }
+
+    public void testReindexFromRemoteDoesNotSupportWorkers() {
+        ReindexRequest reindex = newRequest();
+        reindex.setRemoteInfo(
+                new RemoteInfo(randomAlphaOfLength(5), randomAlphaOfLength(5), between(1, Integer.MAX_VALUE), new BytesArray("real_query"),
+                        null, null, emptyMap(), RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT));
+        reindex.setSlices(between(2, Integer.MAX_VALUE));
+        ActionRequestValidationException e = reindex.validate();
+        assertEquals(
+                "Validation Failed: 1: reindex from remote sources doesn't support workers > 1 but was [" + reindex.getSlices() + "];",
+                e.getMessage());
+    }
+
+    public void testNoSliceWithWorkers() {
+        ReindexRequest reindex = newRequest();
+        reindex.getSearchRequest().source().slice(new SliceBuilder(0, 4));
+        reindex.setSlices(between(2, Integer.MAX_VALUE));
+        ActionRequestValidationException e = reindex.validate();
+        assertEquals("Validation Failed: 1: can't specify both slice and workers;", e.getMessage());
+    }
+
+    @Override
+    protected void extraRandomizationForSlice(ReindexRequest original) {
+        if (randomBoolean()) {
+            original.setScript(mockScript(randomAlphaOfLength(5)));
+        }
+        if (randomBoolean()) {
+            original.setRemoteInfo(new RemoteInfo(randomAlphaOfLength(5), randomAlphaOfLength(5), between(1, 10000),
+                    new BytesArray(randomAlphaOfLength(5)), null, null, emptyMap(),
+                    parseTimeValue(randomPositiveTimeValue(), "socket_timeout"),
+                    parseTimeValue(randomPositiveTimeValue(), "connect_timeout")));
+        }
+    }
+
+    @Override
+    protected void extraForSliceAssertions(ReindexRequest original, ReindexRequest forSliced) {
+        assertEquals(original.getScript(), forSliced.getScript());
+        assertEquals(original.getDestination(), forSliced.getDestination());
+        assertEquals(original.getRemoteInfo(), forSliced.getRemoteInfo());
+    }
+
+    @Override
+    protected ReindexRequest newRequest() {
+        ReindexRequest reindex = new ReindexRequest(new SearchRequest(), new IndexRequest());
+        reindex.getSearchRequest().indices("source");
+        reindex.getDestination().index("dest");
+        return reindex;
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryRequestTests.java b/core/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryRequestTests.java
new file mode 100644
index 0000000000000..b30968cf056b5
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryRequestTests.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex;
+
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.support.IndicesOptions;
+
+import static org.apache.lucene.util.TestUtil.randomSimpleString;
+
+public class UpdateByQueryRequestTests extends AbstractBulkByScrollRequestTestCase<UpdateByQueryRequest> {
+    public void testUpdateByQueryRequestImplementsIndicesRequestReplaceable() {
+        int numIndices = between(1, 100);
+        String[] indices = new String[numIndices];
+        for (int i = 0; i < numIndices; i++) {
+            indices[i] = randomSimpleString(random(), 1, 30);
+        }
+
+        SearchRequest searchRequest = new SearchRequest(indices);
+        IndicesOptions indicesOptions = IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean());
+        searchRequest.indicesOptions(indicesOptions);
+
+        UpdateByQueryRequest request = new UpdateByQueryRequest(searchRequest);
+        for (int i = 0; i < numIndices; i++) {
+            assertEquals(indices[i], request.indices()[i]);
+        }
+
+        assertSame(indicesOptions, request.indicesOptions());
+        assertSame(request.indicesOptions(), request.getSearchRequest().indicesOptions());
+
+        int numNewIndices = between(1, 100);
+        String[] newIndices = new String[numNewIndices];
+        for (int i = 0; i < numNewIndices; i++) {
+            newIndices[i] = randomSimpleString(random(), 1, 30);
+        }
+        request.indices(newIndices);
+        for (int i = 0; i < numNewIndices; i++) {;
+            assertEquals(newIndices[i], request.indices()[i]);
+        }
+        for (int i = 0; i < numNewIndices; i++) {;
+            assertEquals(newIndices[i], request.getSearchRequest().indices()[i]);
+        }
+    }
+
+    @Override
+    protected UpdateByQueryRequest newRequest() {
+        return new UpdateByQueryRequest(new SearchRequest(randomAlphaOfLength(5)));
+    }
+
+    @Override
+    protected void extraRandomizationForSlice(UpdateByQueryRequest original) {
+        if (randomBoolean()) {
+            original.setScript(mockScript(randomAlphaOfLength(5)));
+        }
+        if (randomBoolean()) {
+            original.setPipeline(randomAlphaOfLength(5));
+        }
+    }
+
+    @Override
+    protected void extraForSliceAssertions(UpdateByQueryRequest original, UpdateByQueryRequest forSliced) {
+        assertEquals(original.getScript(), forSliced.getScript());
+        assertEquals(original.getPipeline(), forSliced.getPipeline());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/reindex/WorkingBulkByScrollTaskTests.java b/core/src/test/java/org/elasticsearch/index/reindex/WorkingBulkByScrollTaskTests.java
new file mode 100644
index 0000000000000..5d594d080b8b0
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/reindex/WorkingBulkByScrollTaskTests.java
@@ -0,0 +1,265 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex;
+
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
+import org.elasticsearch.tasks.TaskId;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.Delayed;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.elasticsearch.common.unit.TimeValue.timeValueNanos;
+import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
+import static org.hamcrest.Matchers.both;
+import static org.hamcrest.Matchers.closeTo;
+import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
+
+public class WorkingBulkByScrollTaskTests extends ESTestCase {
+    private WorkingBulkByScrollTask task;
+
+    @Before
+    public void createTask() {
+        task = new WorkingBulkByScrollTask(1, "test_type", "test_action", "test", TaskId.EMPTY_TASK_ID, null, Float.POSITIVE_INFINITY);
+    }
+
+    public void testBasicData() {
+        assertEquals(1, task.getId());
+        assertEquals("test_type", task.getType());
+        assertEquals("test_action", task.getAction());
+        assertEquals("test", task.getDescription());
+    }
+
+    public void testProgress() {
+        long created = 0;
+        long updated = 0;
+        long deleted = 0;
+        long versionConflicts = 0;
+        long noops = 0;
+        int batch = 0;
+        BulkByScrollTask.Status status = task.getStatus();
+        assertEquals(0, status.getTotal());
+        assertEquals(created, status.getCreated());
+        assertEquals(updated, status.getUpdated());
+        assertEquals(deleted, status.getDeleted());
+        assertEquals(versionConflicts, status.getVersionConflicts());
+        assertEquals(batch, status.getBatches());
+        assertEquals(noops, status.getNoops());
+
+        long totalHits = randomIntBetween(10, 1000);
+        task.setTotal(totalHits);
+        for (long p = 0; p < totalHits; p++) {
+            status = task.getStatus();
+            assertEquals(totalHits, status.getTotal());
+            assertEquals(created, status.getCreated());
+            assertEquals(updated, status.getUpdated());
+            assertEquals(deleted, status.getDeleted());
+            assertEquals(versionConflicts, status.getVersionConflicts());
+            assertEquals(batch, status.getBatches());
+            assertEquals(noops, status.getNoops());
+
+            if (randomBoolean()) {
+                created++;
+                task.countCreated();
+            } else if (randomBoolean()) {
+                updated++;
+                task.countUpdated();
+            } else {
+                deleted++;
+                task.countDeleted();
+            }
+
+            if (rarely()) {
+                versionConflicts++;
+                task.countVersionConflict();
+            }
+
+            if (rarely()) {
+                batch++;
+                task.countBatch();
+            }
+
+            if (rarely()) {
+                noops++;
+                task.countNoop();
+            }
+        }
+        status = task.getStatus();
+        assertEquals(totalHits, status.getTotal());
+        assertEquals(created, status.getCreated());
+        assertEquals(updated, status.getUpdated());
+        assertEquals(deleted, status.getDeleted());
+        assertEquals(versionConflicts, status.getVersionConflicts());
+        assertEquals(batch, status.getBatches());
+        assertEquals(noops, status.getNoops());
+    }
+
+    /**
+     * Furiously rethrottles a delayed request to make sure that we never run it twice.
+     */
+    public void testDelayAndRethrottle() throws IOException, InterruptedException {
+        List<Throwable> errors = new CopyOnWriteArrayList<>();
+        AtomicBoolean done = new AtomicBoolean();
+        int threads = between(1, 10);
+        CyclicBarrier waitForShutdown = new CyclicBarrier(threads);
+
+        /*
+         * We never end up waiting this long because the test rethrottles over and over again, ratcheting down the delay a random amount
+         * each time.
+         */
+        float originalRequestsPerSecond = (float) randomDoubleBetween(1, 10000, true);
+        task.rethrottle(originalRequestsPerSecond);
+        TimeValue maxDelay = timeValueSeconds(between(1, 5));
+        assertThat(maxDelay.nanos(), greaterThanOrEqualTo(0L));
+        int batchSizeForMaxDelay = (int) (maxDelay.seconds() * originalRequestsPerSecond);
+        ThreadPool threadPool = new TestThreadPool(getTestName()) {
+            @Override
+            public ScheduledFuture<?> schedule(TimeValue delay, String name, Runnable command) {
+                assertThat(delay.nanos(), both(greaterThanOrEqualTo(0L)).and(lessThanOrEqualTo(maxDelay.nanos())));
+                return super.schedule(delay, name, command);
+            }
+        };
+        try {
+            task.delayPrepareBulkRequest(threadPool, timeValueNanos(System.nanoTime()), batchSizeForMaxDelay, new AbstractRunnable() {
+                @Override
+                protected void doRun() throws Exception {
+                    boolean oldValue = done.getAndSet(true);
+                    if (oldValue) {
+                        throw new RuntimeException("Ran twice oh no!");
+                    }
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    errors.add(e);
+                }
+            });
+
+            // Rethrottle on a random number of threads, one of which is this thread.
+            Runnable test = () -> {
+                try {
+                    int rethrottles = 0;
+                    while (false == done.get()) {
+                        float requestsPerSecond = (float) randomDoubleBetween(0, originalRequestsPerSecond * 2, true);
+                        task.rethrottle(requestsPerSecond);
+                        rethrottles += 1;
+                    }
+                    logger.info("Rethrottled [{}] times", rethrottles);
+                    waitForShutdown.await();
+                } catch (Exception e) {
+                    errors.add(e);
+                }
+            };
+            for (int i = 1; i < threads; i++) {
+                threadPool.generic().execute(test);
+            }
+            test.run();
+        } finally {
+            // Other threads should finish up quickly as they are checking the same AtomicBoolean.
+            threadPool.shutdown();
+            threadPool.awaitTermination(10, TimeUnit.SECONDS);
+        }
+        assertThat(errors, empty());
+    }
+
+    public void testDelayNeverNegative() throws IOException {
+        // Thread pool that returns a ScheduledFuture that claims to have a negative delay
+        ThreadPool threadPool = new TestThreadPool("test") {
+            public ScheduledFuture<?> schedule(TimeValue delay, String name, Runnable command) {
+                return new ScheduledFuture<Void>() {
+                    @Override
+                    public long getDelay(TimeUnit unit) {
+                        return -1;
+                    }
+
+                    @Override
+                    public int compareTo(Delayed o) {
+                        throw new UnsupportedOperationException();
+                    }
+
+                    @Override
+                    public boolean cancel(boolean mayInterruptIfRunning) {
+                        throw new UnsupportedOperationException();
+                    }
+
+                    @Override
+                    public boolean isCancelled() {
+                        throw new UnsupportedOperationException();
+                    }
+
+                    @Override
+                    public boolean isDone() {
+                        throw new UnsupportedOperationException();
+                    }
+
+                    @Override
+                    public Void get() throws InterruptedException, ExecutionException {
+                        throw new UnsupportedOperationException();
+                    }
+
+                    @Override
+                    public Void get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
+                        throw new UnsupportedOperationException();
+                    }
+                };
+            }
+        };
+        try {
+            // Have the task use the thread pool to delay a task that does nothing
+            task.delayPrepareBulkRequest(threadPool, timeValueSeconds(0), 1, new AbstractRunnable() {
+                @Override
+                protected void doRun() throws Exception {
+                }
+                @Override
+                public void onFailure(Exception e) {
+                    throw new UnsupportedOperationException();
+                }
+            });
+            // Even though the future returns a negative delay we just return 0 because the time is up.
+            assertEquals(timeValueSeconds(0), task.getStatus().getThrottledUntil());
+        } finally {
+            threadPool.shutdown();
+        }
+    }
+
+    public void testPerfectlyThrottledBatchTime() {
+        task.rethrottle(Float.POSITIVE_INFINITY);
+        assertThat((double) task.perfectlyThrottledBatchTime(randomInt()), closeTo(0f, 0f));
+
+        int total = between(0, 1000000);
+        task.rethrottle(1);
+        assertThat((double) task.perfectlyThrottledBatchTime(total),
+                closeTo(TimeUnit.SECONDS.toNanos(total), TimeUnit.SECONDS.toNanos(1)));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java b/core/src/test/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
index ec794091a42f5..5a4aeb5baec09 100644
--- a/core/src/test/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
+++ b/core/src/test/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
@@ -18,75 +18,37 @@
  */
 package org.elasticsearch.index.replication;
 
-import org.apache.lucene.document.Document;
-import org.apache.lucene.index.IndexNotFoundException;
-import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.store.AlreadyClosedException;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.IOUtils;
-import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.admin.indices.flush.FlushRequest;
+import org.elasticsearch.action.bulk.TransportShardBulkAction;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.index.IndexResponse;
-import org.elasticsearch.action.index.TransportIndexAction;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.action.support.replication.ReplicationOperation;
 import org.elasticsearch.action.support.replication.ReplicationResponse;
-import org.elasticsearch.action.support.replication.TransportWriteAction;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.routing.RecoverySource.PeerRecoverySource;
-import org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingHelper;
-import org.elasticsearch.cluster.routing.ShardRoutingState;
-import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.common.collect.Iterators;
-import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.LocalTransportAddress;
-import org.elasticsearch.common.unit.ByteSizeUnit;
-import org.elasticsearch.common.util.BigArrays;
-import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.Index;
-import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.MapperTestUtils;
-import org.elasticsearch.index.cache.IndexCache;
-import org.elasticsearch.index.cache.query.DisabledQueryCache;
 import org.elasticsearch.index.engine.Engine;
-import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.Uid;
-import org.elasticsearch.index.mapper.UidFieldMapper;
-import org.elasticsearch.index.shard.IndexEventListener;
 import org.elasticsearch.index.shard.IndexShard;
-import org.elasticsearch.index.shard.IndexShardState;
+import org.elasticsearch.index.shard.IndexShardTestCase;
 import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.index.shard.ShardPath;
-import org.elasticsearch.index.similarity.SimilarityService;
-import org.elasticsearch.index.store.DirectoryService;
-import org.elasticsearch.index.store.Store;
-import org.elasticsearch.indices.recovery.RecoveryFailedException;
-import org.elasticsearch.indices.recovery.RecoverySourceHandler;
 import org.elasticsearch.indices.recovery.RecoveryState;
 import org.elasticsearch.indices.recovery.RecoveryTarget;
-import org.elasticsearch.indices.recovery.PeerRecoveryTargetService;
-import org.elasticsearch.indices.recovery.StartRecoveryRequest;
-import org.elasticsearch.test.DummyShardLock;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.junit.annotations.TestLogging;
-import org.elasticsearch.threadpool.TestThreadPool;
-import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportResponse;
 
 import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
@@ -94,10 +56,8 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.Future;
 import java.util.concurrent.FutureTask;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.BiFunction;
 import java.util.function.Consumer;
@@ -107,98 +67,24 @@
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 
-public abstract class ESIndexLevelReplicationTestCase extends ESTestCase {
+public abstract class ESIndexLevelReplicationTestCase extends IndexShardTestCase {
 
-    protected ThreadPool threadPool;
     protected final Index index = new Index("test", "uuid");
     private final ShardId shardId = new ShardId(index, 0);
     private final Map<String, String> indexMapping = Collections.singletonMap("type", "{ \"type\": {} }");
-    protected static final PeerRecoveryTargetService.RecoveryListener recoveryListener = new PeerRecoveryTargetService.RecoveryListener() {
-        @Override
-        public void onRecoveryDone(RecoveryState state) {
-
-        }
-
-        @Override
-        public void onRecoveryFailure(RecoveryState state, RecoveryFailedException e, boolean sendShardFailure) {
-            fail(ExceptionsHelper.detailedMessage(e));
-        }
-    };
-
-
-    @TestLogging("index.shard:TRACE,index.replication:TRACE,indices.recovery:TRACE")
-    public void testIndexingDuringFileRecovery() throws Exception {
-        try (ReplicationGroup shards = createGroup(randomInt(1))) {
-            shards.startAll();
-            int docs = shards.indexDocs(randomInt(50));
-            shards.flush();
-            IndexShard replica = shards.addReplica();
-            final CountDownLatch recoveryBlocked = new CountDownLatch(1);
-            final CountDownLatch releaseRecovery = new CountDownLatch(1);
-            final Future<Void> recoveryFuture = shards.asyncRecoverReplica(replica,
-                new BiFunction<IndexShard, DiscoveryNode, RecoveryTarget>() {
-                @Override
-                public RecoveryTarget apply(IndexShard indexShard, DiscoveryNode node) {
-                    return new RecoveryTarget(indexShard, node, recoveryListener, version -> {}) {
-                        @Override
-                        public void renameAllTempFiles() throws IOException {
-                            super.renameAllTempFiles();
-                            recoveryBlocked.countDown();
-                            try {
-                                releaseRecovery.await();
-                            } catch (InterruptedException e) {
-                                throw new IOException("terminated by interrupt", e);
-                            }
-                        }
-                    };
-                }
-            });
-
-            recoveryBlocked.await();
-            docs += shards.indexDocs(randomInt(20));
-            releaseRecovery.countDown();
-            recoveryFuture.get();
-
-            shards.assertAllEqual(docs);
-        }
-    }
-
-    @Override
-    public void setUp() throws Exception {
-        super.setUp();
-        threadPool = new TestThreadPool(getClass().getName());
-    }
-
-    @Override
-    public void tearDown() throws Exception {
-        super.tearDown();
-        ThreadPool.terminate(threadPool, 30, TimeUnit.SECONDS);
-    }
-
-    private Store createStore(IndexSettings indexSettings, ShardPath shardPath) throws IOException {
-        final ShardId shardId = shardPath.getShardId();
-        final DirectoryService directoryService = new DirectoryService(shardId, indexSettings) {
-            @Override
-            public Directory newDirectory() throws IOException {
-                return newFSDirectory(shardPath.resolveIndex());
-            }
-
-            @Override
-            public long throttleTimeInNanos() {
-                return 0;
-            }
-        };
-        return new Store(shardId, indexSettings, directoryService, new DummyShardLock(shardId));
-    }
 
     protected ReplicationGroup createGroup(int replicas) throws IOException {
-        final Path homePath = createTempDir();
-        Settings build = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+        Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
             .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, replicas)
             .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
             .build();
-        IndexMetaData metaData = IndexMetaData.builder(index.getName()).settings(build).primaryTerm(0, 1).build();
-        return new ReplicationGroup(metaData, homePath);
+        IndexMetaData.Builder metaData = IndexMetaData.builder(index.getName())
+            .settings(settings)
+            .primaryTerm(0, 1);
+        for (Map.Entry<String, String> typeMapping: indexMapping.entrySet()) {
+            metaData.putMapping(typeMapping.getKey(), typeMapping.getValue());
+        }
+        return new ReplicationGroup(metaData.build());
     }
 
     protected DiscoveryNode getDiscoveryNode(String id) {
@@ -206,56 +92,28 @@ protected DiscoveryNode getDiscoveryNode(String id) {
             Collections.singleton(DiscoveryNode.Role.DATA), Version.CURRENT);
     }
 
-    private IndexShard newShard(boolean primary, DiscoveryNode node, IndexMetaData indexMetaData, Path homePath) throws IOException {
-        // add node name to settings for propper logging
-        final Settings nodeSettings = Settings.builder().put("node.name", node.getName()).build();
-        final IndexSettings indexSettings = new IndexSettings(indexMetaData, nodeSettings);
-        ShardRouting shardRouting = TestShardRouting.newShardRouting(shardId, node.getId(), primary, ShardRoutingState.INITIALIZING,
-            primary ? StoreRecoverySource.EMPTY_STORE_INSTANCE : PeerRecoverySource.INSTANCE);
-        final Path path = Files.createDirectories(homePath.resolve(node.getId()));
-        final NodeEnvironment.NodePath nodePath = new NodeEnvironment.NodePath(path);
-        ShardPath shardPath = new ShardPath(false, nodePath.resolve(shardId), nodePath.resolve(shardId), shardId);
-        Store store = createStore(indexSettings, shardPath);
-        IndexCache indexCache = new IndexCache(indexSettings, new DisabledQueryCache(indexSettings), null);
-        MapperService mapperService = MapperTestUtils.newMapperService(homePath, indexSettings.getSettings());
-        for (Map.Entry<String, String> type : indexMapping.entrySet()) {
-            mapperService.merge(type.getKey(), new CompressedXContent(type.getValue()), MapperService.MergeReason.MAPPING_RECOVERY, true);
-        }
-        SimilarityService similarityService = new SimilarityService(indexSettings, Collections.emptyMap());
-        final IndexEventListener indexEventListener = new IndexEventListener() {
-        };
-        final Engine.Warmer warmer = searcher -> {
-        };
-        return new IndexShard(shardRouting, indexSettings, shardPath, store, indexCache, mapperService, similarityService, null, null,
-            indexEventListener, null, threadPool, BigArrays.NON_RECYCLING_INSTANCE, warmer, Collections.emptyList(),
-            Collections.emptyList());
-    }
-
 
     protected class ReplicationGroup implements AutoCloseable, Iterable<IndexShard> {
         private final IndexShard primary;
         private final List<IndexShard> replicas;
         private final IndexMetaData indexMetaData;
-        private final Path homePath;
         private final AtomicInteger replicaId = new AtomicInteger();
         private final AtomicInteger docId = new AtomicInteger();
         boolean closed = false;
 
-        ReplicationGroup(final IndexMetaData indexMetaData, Path homePath) throws IOException {
-            primary = newShard(true, getDiscoveryNode("s0"), indexMetaData, homePath);
+        ReplicationGroup(final IndexMetaData indexMetaData) throws IOException {
+            primary = newShard(shardId, true, "s0", indexMetaData, null);
             replicas = new ArrayList<>();
             this.indexMetaData = indexMetaData;
-            this.homePath = homePath;
             for (int i = 0; i < indexMetaData.getNumberOfReplicas(); i++) {
                 addReplica();
             }
-
         }
 
         public int indexDocs(final int numOfDoc) throws Exception {
             for (int doc = 0; doc < numOfDoc; doc++) {
                 final IndexRequest indexRequest = new IndexRequest(index.getName(), "type", Integer.toString(docId.incrementAndGet()))
-                    .source("{}");
+                    .source("{}", XContentType.JSON);
                 final IndexResponse response = index(indexRequest);
                 assertEquals(DocWriteResponse.Result.CREATED, response.getResult());
             }
@@ -264,7 +122,7 @@ public int indexDocs(final int numOfDoc) throws Exception {
 
         public int appendDocs(final int numOfDoc) throws Exception {
             for (int doc = 0; doc < numOfDoc; doc++) {
-                final IndexRequest indexRequest = new IndexRequest(index.getName(), "type").source("{}");
+                final IndexRequest indexRequest = new IndexRequest(index.getName(), "type").source("{}", XContentType.JSON);
                 final IndexResponse response = index(indexRequest);
                 assertEquals(DocWriteResponse.Result.CREATED, response.getResult());
             }
@@ -289,7 +147,7 @@ public synchronized void startAll() throws IOException {
         }
 
         public synchronized IndexShard addReplica() throws IOException {
-            final IndexShard replica = newShard(false, getDiscoveryNode("s" + replicaId.incrementAndGet()), indexMetaData, homePath);
+            final IndexShard replica = newShard(shardId, false,"s" + replicaId.incrementAndGet(), indexMetaData, null);
             replicas.add(replica);
             return replica;
         }
@@ -304,39 +162,8 @@ public void recoverReplica(IndexShard replica, BiFunction<IndexShard, DiscoveryN
         }
 
         public void recoverReplica(IndexShard replica, BiFunction<IndexShard, DiscoveryNode, RecoveryTarget> targetSupplier,
-                                   boolean markAsRecovering)
-            throws IOException {
-            final DiscoveryNode pNode = getPrimaryNode();
-            final DiscoveryNode rNode = getDiscoveryNode(replica.routingEntry().currentNodeId());
-            if (markAsRecovering) {
-                replica.markAsRecovering("remote",
-                    new RecoveryState(replica.routingEntry(), pNode, rNode));
-            } else {
-                assertEquals(replica.state(), IndexShardState.RECOVERING);
-            }
-            replica.prepareForIndexRecovery();
-            RecoveryTarget recoveryTarget = targetSupplier.apply(replica, pNode);
-            StartRecoveryRequest request = new StartRecoveryRequest(replica.shardId(), pNode, rNode,
-                getMetadataSnapshotOrEmpty(replica), false, 0);
-            RecoverySourceHandler recovery = new RecoverySourceHandler(primary, recoveryTarget, request, () -> 0L, e -> () -> {},
-                (int) ByteSizeUnit.MB.toKB(1), logger);
-            recovery.recoverToTarget();
-            recoveryTarget.markAsDone();
-            replica.updateRoutingEntry(ShardRoutingHelper.moveToStarted(replica.routingEntry()));
-        }
-
-        private Store.MetadataSnapshot getMetadataSnapshotOrEmpty(IndexShard replica) throws IOException {
-            Store.MetadataSnapshot result;
-            try {
-                result = replica.snapshotStoreMetadata();
-            } catch (IndexNotFoundException e) {
-                // OK!
-                result = Store.MetadataSnapshot.EMPTY;
-            } catch (IOException e) {
-                logger.warn("failed read store, treating as empty", e);
-                result = Store.MetadataSnapshot.EMPTY;
-            }
-            return result;
+                                   boolean markAsRecovering) throws IOException {
+            ESIndexLevelReplicationTestCase.this.recoverReplica(replica, primary, targetSupplier, markAsRecovering);
         }
 
         public synchronized DiscoveryNode getPrimaryNode() {
@@ -367,24 +194,6 @@ public synchronized void assertAllEqual(int expectedCount) throws IOException {
             }
         }
 
-        private Set<Uid> getShardDocUIDs(final IndexShard shard) throws IOException {
-            shard.refresh("get_uids");
-            try (Engine.Searcher searcher = shard.acquireSearcher("test")) {
-                Set<Uid> ids = new HashSet<>();
-                for (LeafReaderContext leafContext : searcher.reader().leaves()) {
-                    LeafReader reader = leafContext.reader();
-                    Bits liveDocs = reader.getLiveDocs();
-                    for (int i = 0; i < reader.maxDoc(); i++) {
-                        if (liveDocs == null || liveDocs.get(i)) {
-                            Document uuid = reader.document(i, Collections.singleton(UidFieldMapper.NAME));
-                            ids.add(Uid.createUid(uuid.get(UidFieldMapper.NAME)));
-                        }
-                    }
-                }
-                return ids;
-            }
-        }
-
         public synchronized void refresh(String source) {
             for (IndexShard shard : this) {
                 shard.refresh(source);
@@ -406,10 +215,7 @@ public synchronized List<ShardRouting> shardRoutings() {
         public synchronized void close() throws Exception {
             if (closed == false) {
                 closed = true;
-                for (IndexShard shard : this) {
-                    shard.close("eol", false);
-                    IOUtils.close(shard.store());
-                }
+                closeShards(this);
             } else {
                 throw new AlreadyClosedException("too bad");
             }
@@ -429,7 +235,7 @@ class IndexingOp extends ReplicationOperation<IndexRequest, IndexRequest, Indexi
 
         private final ReplicationGroup replicationGroup;
 
-        public IndexingOp(IndexRequest request, ActionListener<IndexingResult> listener, ReplicationGroup replicationGroup) {
+        IndexingOp(IndexRequest request, ActionListener<IndexingResult> listener, ReplicationGroup replicationGroup) {
             super(request, new PrimaryRef(replicationGroup), listener, true, new ReplicasRef(replicationGroup),
                 () -> null, logger, "indexing");
             this.replicationGroup = replicationGroup;
@@ -472,10 +278,22 @@ public void failShard(String message, Exception exception) {
 
         @Override
         public IndexingResult perform(IndexRequest request) throws Exception {
-            TransportWriteAction.WriteResult<IndexResponse> result = TransportIndexAction.executeIndexRequestOnPrimary(request, primary,
-                null);
-            request.primaryTerm(primary.getPrimaryTerm());
-            return new IndexingResult(request, result.getResponse());
+            Engine.IndexResult indexResult = TransportShardBulkAction.executeIndexRequestOnPrimary(request, primary,
+                    null);
+            if (indexResult.hasFailure() == false) {
+                // update the version on request so it will happen on the replicas
+                final long version = indexResult.getVersion();
+                request.version(version);
+                request.versionType(request.versionType().versionTypeForReplicationAndRecovery());
+                assert request.versionType().validateVersionForWrites(request.version());
+            }
+            IndexResponse response = new IndexResponse(
+                    primary.shardId(),
+                    request.type(),
+                    request.id(),
+                    indexResult.getVersion(),
+                    indexResult.isCreated());
+            return new IndexingResult(request, response);
         }
 
     }
@@ -492,7 +310,7 @@ public void performOn(ShardRouting replicaRouting, IndexRequest request, ActionL
             try {
                 IndexShard replica = replicationGroup.replicas.stream()
                     .filter(s -> replicaRouting.isSameAllocation(s.routingEntry())).findFirst().get();
-                TransportIndexAction.executeIndexRequestOnReplica(request, replica);
+                TransportShardBulkAction.executeIndexRequestOnReplica(request, replica);
                 listener.onResponse(TransportResponse.Empty.INSTANCE);
             } catch (Exception t) {
                 listener.onFailure(t);
@@ -500,13 +318,13 @@ public void performOn(ShardRouting replicaRouting, IndexRequest request, ActionL
         }
 
         @Override
-        public void failShard(ShardRouting replica, long primaryTerm, String message, Exception exception, Runnable onSuccess,
+        public void failShard(ShardRouting replica, String message, Exception exception, Runnable onSuccess,
                               Consumer<Exception> onPrimaryDemoted, Consumer<Exception> onIgnoredFailure) {
             throw new UnsupportedOperationException();
         }
 
         @Override
-        public void markShardCopyAsStale(ShardId shardId, String allocationId, long primaryTerm, Runnable onSuccess,
+        public void markShardCopyAsStale(ShardId shardId, String allocationId, Runnable onSuccess,
                                          Consumer<Exception> onPrimaryDemoted, Consumer<Exception> onIgnoredFailure) {
             throw new UnsupportedOperationException();
         }
@@ -517,7 +335,7 @@ private static class IndexingResult implements ReplicationOperation.PrimaryResul
         final IndexRequest replicaRequest;
         final IndexResponse finalResponse;
 
-        public IndexingResult(IndexRequest replicaRequest, IndexResponse finalResponse) {
+        IndexingResult(IndexRequest replicaRequest, IndexResponse finalResponse) {
             this.replicaRequest = replicaRequest;
             this.finalResponse = finalResponse;
         }
diff --git a/core/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java b/core/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java
index 407b374a92e9f..23b627aab599c 100644
--- a/core/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java
+++ b/core/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java
@@ -21,6 +21,7 @@
 import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.engine.InternalEngine;
 import org.elasticsearch.index.engine.InternalEngineTests;
@@ -90,20 +91,16 @@ public void cleanFiles(int totalTranslogOps, Store.MetadataSnapshot sourceMetaDa
             future.get();
             thread.join();
             shards.assertAllEqual(numDocs);
-            Engine engine = IndexShardTests.getEngineFromShard(replica);
-            assertEquals("expected at no version lookups ", InternalEngineTests.getNumVersionLookups((InternalEngine) engine), 0);
-            for (IndexShard shard : shards) {
-                engine = IndexShardTests.getEngineFromShard(shard);
-                assertEquals(0, InternalEngineTests.getNumIndexVersionsLookups((InternalEngine) engine));
-                assertEquals(0, InternalEngineTests.getNumVersionLookups((InternalEngine) engine));
-            }
+            Engine engine = IndexShardTests.getEngineFromShard(shards.getPrimary());
+            assertEquals(0, InternalEngineTests.getNumIndexVersionsLookups((InternalEngine) engine));
+            assertEquals(0, InternalEngineTests.getNumVersionLookups((InternalEngine) engine));
         }
     }
 
     public void testInheritMaxValidAutoIDTimestampOnRecovery() throws Exception {
         try (ReplicationGroup shards = createGroup(0)) {
             shards.startAll();
-            final IndexRequest indexRequest = new IndexRequest(index.getName(), "type").source("{}");
+            final IndexRequest indexRequest = new IndexRequest(index.getName(), "type").source("{}", XContentType.JSON);
             indexRequest.onRetry(); // force an update of the timestamp
             final IndexResponse response = shards.index(indexRequest);
             assertEquals(DocWriteResponse.Result.CREATED, response.getResult());
diff --git a/core/src/test/java/org/elasticsearch/index/search/ESToParentBlockJoinQueryTests.java b/core/src/test/java/org/elasticsearch/index/search/ESToParentBlockJoinQueryTests.java
new file mode 100644
index 0000000000000..7f19b7fe6f729
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/search/ESToParentBlockJoinQueryTests.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.search;
+
+import org.apache.lucene.index.MultiReader;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.PhraseQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.join.QueryBitSetProducer;
+import org.apache.lucene.search.join.ScoreMode;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+public class ESToParentBlockJoinQueryTests extends ESTestCase {
+
+    public void testEquals() {
+        Query q1 = new ESToParentBlockJoinQuery(
+                new TermQuery(new Term("is", "child")),
+                new QueryBitSetProducer(new TermQuery(new Term("is", "parent"))),
+                ScoreMode.Avg, "nested");
+
+        Query q2 = new ESToParentBlockJoinQuery(
+                new TermQuery(new Term("is", "child")),
+                new QueryBitSetProducer(new TermQuery(new Term("is", "parent"))),
+                ScoreMode.Avg, "nested");
+        assertEquals(q1, q2);
+        assertEquals(q1.hashCode(), q2.hashCode());
+
+        Query q3 = new ESToParentBlockJoinQuery(
+                new TermQuery(new Term("is", "not_child")),
+                new QueryBitSetProducer(new TermQuery(new Term("is", "parent"))),
+                ScoreMode.Avg, "nested");
+        assertFalse(q1.equals(q3));
+        assertFalse(q1.hashCode() == q3.hashCode());
+
+        Query q4 = new ESToParentBlockJoinQuery(
+                new TermQuery(new Term("is", "child")),
+                new QueryBitSetProducer(new TermQuery(new Term("is", "other_parent"))),
+                ScoreMode.Avg, "nested");
+        assertFalse(q1.equals(q4));
+        assertFalse(q1.hashCode() == q4.hashCode());
+
+        Query q5 = new ESToParentBlockJoinQuery(
+                new TermQuery(new Term("is", "child")),
+                new QueryBitSetProducer(new TermQuery(new Term("is", "parent"))),
+                ScoreMode.Total, "nested");
+        assertFalse(q1.equals(q5));
+        assertFalse(q1.hashCode() == q5.hashCode());
+
+        Query q6 = new ESToParentBlockJoinQuery(
+                new TermQuery(new Term("is", "child")),
+                new QueryBitSetProducer(new TermQuery(new Term("is", "parent"))),
+                ScoreMode.Avg, "nested2");
+        assertFalse(q1.equals(q6));
+        assertFalse(q1.hashCode() == q6.hashCode());
+    }
+
+    public void testRewrite() throws IOException {
+        Query q = new ESToParentBlockJoinQuery(
+                new PhraseQuery("body", "term"), // rewrites to a TermQuery
+                new QueryBitSetProducer(new TermQuery(new Term("is", "parent"))),
+                ScoreMode.Avg, "nested");
+        Query expected = new ESToParentBlockJoinQuery(
+                 new TermQuery(new Term("body", "term")),
+                 new QueryBitSetProducer(new TermQuery(new Term("is", "parent"))),
+                 ScoreMode.Avg, "nested");
+         Query rewritten = q.rewrite(new MultiReader());
+         assertEquals(expected, rewritten);
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/search/MatchQueryIT.java b/core/src/test/java/org/elasticsearch/index/search/MatchQueryIT.java
new file mode 100644
index 0000000000000..0e43b90e366cb
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/search/MatchQueryIT.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.search;
+
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
+import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.query.Operator;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits;
+
+public class MatchQueryIT extends ESIntegTestCase {
+    private static final String INDEX = "test";
+
+    /**
+     * Test setup.
+     */
+    @Before
+    public void setUp() throws Exception {
+        super.setUp();
+        CreateIndexRequestBuilder builder = prepareCreate(INDEX).setSettings(
+            Settings.builder()
+                .put(indexSettings())
+                .put("index.analysis.filter.syns.type", "synonym")
+                .putArray("index.analysis.filter.syns.synonyms", "wtf, what the fudge", "foo, bar baz")
+                .put("index.analysis.analyzer.lower_syns.type", "custom")
+                .put("index.analysis.analyzer.lower_syns.tokenizer", "standard")
+                .putArray("index.analysis.analyzer.lower_syns.filter", "lowercase", "syns")
+                .put("index.analysis.filter.graphsyns.type", "synonym_graph")
+                .putArray("index.analysis.filter.graphsyns.synonyms", "wtf, what the fudge", "foo, bar baz")
+                .put("index.analysis.analyzer.lower_graphsyns.type", "custom")
+                .put("index.analysis.analyzer.lower_graphsyns.tokenizer", "standard")
+                .putArray("index.analysis.analyzer.lower_graphsyns.filter", "lowercase", "graphsyns")
+        );
+
+        assertAcked(builder.addMapping(INDEX, createMapping()));
+        ensureGreen();
+    }
+
+    private List<IndexRequestBuilder> getDocs() {
+        List<IndexRequestBuilder> builders = new ArrayList<>();
+        builders.add(client().prepareIndex("test", "test", "1").setSource("field", "say wtf happened foo"));
+        builders.add(client().prepareIndex("test", "test", "2").setSource("field", "bar baz what the fudge man"));
+        builders.add(client().prepareIndex("test", "test", "3").setSource("field", "wtf"));
+        builders.add(client().prepareIndex("test", "test", "4").setSource("field", "what is the name for fudge"));
+        builders.add(client().prepareIndex("test", "test", "5").setSource("field", "bar two three"));
+        builders.add(client().prepareIndex("test", "test", "6").setSource("field", "bar baz two three"));
+
+        return builders;
+    }
+
+    /**
+     * Setup the index mappings for the test index.
+     *
+     * @return the json builder with the index mappings
+     * @throws IOException on error creating mapping json
+     */
+    private XContentBuilder createMapping() throws IOException {
+        return XContentFactory.jsonBuilder()
+            .startObject()
+                .startObject(INDEX)
+                    .startObject("properties")
+                        .startObject("field")
+                            .field("type", "text")
+                        .endObject()
+                    .endObject()
+                .endObject()
+            .endObject();
+    }
+
+    public void testSimpleMultiTermPhrase() throws ExecutionException, InterruptedException {
+        indexRandom(true, false, getDocs());
+
+        // first search using regular synonym field using phrase
+        SearchResponse searchResponse = client().prepareSearch(INDEX)
+            .setQuery(QueryBuilders.matchPhraseQuery("field", "foo two three").analyzer("lower_syns")).get();
+
+        // because foo -> "bar baz" where "foo" and "bar" at position 0, "baz" and "two" at position 1.
+        // "bar two three", "bar baz three", "foo two three", "foo baz three"
+        assertHitCount(searchResponse, 1L);
+        assertSearchHits(searchResponse, "5"); // we should not match this but we do
+
+        // same query using graph should find correct result
+        searchResponse = client().prepareSearch(INDEX).setQuery(QueryBuilders.matchPhraseQuery("field", "foo two three")
+            .analyzer("lower_graphsyns")).get();
+
+        assertHitCount(searchResponse, 1L);
+        assertSearchHits(searchResponse, "6");
+    }
+
+    public void testSimpleMultiTermAnd() throws ExecutionException, InterruptedException {
+        indexRandom(true, false, getDocs());
+
+        // first search using regular synonym field using phrase
+        SearchResponse searchResponse = client().prepareSearch(INDEX).setQuery(QueryBuilders.matchQuery("field", "say what the fudge")
+            .operator(Operator.AND).analyzer("lower_syns")).get();
+
+        // Old synonyms work fine in that case, but it is coincidental
+        assertHitCount(searchResponse, 1L);
+        assertSearchHits(searchResponse, "1");
+
+        // same query using graph should find correct result
+        searchResponse = client().prepareSearch(INDEX).setQuery(QueryBuilders.matchQuery("field", "say what the fudge")
+            .operator(Operator.AND).analyzer("lower_graphsyns")).get();
+
+        assertHitCount(searchResponse, 1L);
+        assertSearchHits(searchResponse, "1");
+    }
+
+    public void testMinShouldMatch() throws ExecutionException, InterruptedException {
+        indexRandom(true, false, getDocs());
+
+        // no min should match
+        SearchResponse searchResponse = client().prepareSearch(INDEX).setQuery(QueryBuilders.matchQuery("field", "three what the fudge foo")
+            .operator(Operator.OR).analyzer("lower_graphsyns")).get();
+
+        assertHitCount(searchResponse, 6L);
+        assertSearchHits(searchResponse, "1", "2", "3", "4", "5", "6");
+
+        // same query, with min_should_match of 2
+        searchResponse = client().prepareSearch(INDEX).setQuery(QueryBuilders.matchQuery("field", "three what the fudge foo")
+            .operator(Operator.OR).analyzer("lower_graphsyns").minimumShouldMatch("80%")).get();
+
+        // three wtf foo = 2 terms, match #1
+        // three wtf bar baz = 3 terms, match #6
+        // three what the fudge foo = 4 terms, no match
+        // three what the fudge bar baz = 4 terms, match #2
+        assertHitCount(searchResponse, 3L);
+        assertSearchHits(searchResponse, "1", "2", "6");
+    }
+
+    public void testPhrasePrefix() throws ExecutionException, InterruptedException {
+        List<IndexRequestBuilder> builders = getDocs();
+        builders.add(client().prepareIndex("test", "test", "7").setSource("field", "WTFD!"));
+        builders.add(client().prepareIndex("test", "test", "8").setSource("field", "Weird Al's WHAT THE FUDGESICLE"));
+        indexRandom(true, false, builders);
+
+        SearchResponse searchResponse = client().prepareSearch(INDEX).setQuery(QueryBuilders.matchPhrasePrefixQuery("field", "wtf")
+            .analyzer("lower_graphsyns")).get();
+
+        assertHitCount(searchResponse, 5L);
+        assertSearchHits(searchResponse, "1", "2", "3", "7", "8");
+    }
+
+    public void testCommonTerms() throws ExecutionException, InterruptedException {
+        String route = "commonTermsTest";
+        List<IndexRequestBuilder> builders = getDocs();
+        for (IndexRequestBuilder indexRequet : builders) {
+            // route all docs to same shard for this test
+            indexRequet.setRouting(route);
+        }
+        indexRandom(true, false, builders);
+
+        // do a search with no cutoff frequency to show which docs should match
+        SearchResponse searchResponse = client().prepareSearch(INDEX)
+            .setRouting(route)
+            .setQuery(QueryBuilders.matchQuery("field", "bar three happened")
+                .operator(Operator.OR)).get();
+
+        assertHitCount(searchResponse, 4L);
+        assertSearchHits(searchResponse, "1", "2", "5", "6");
+
+        // do same search with cutoff and see less documents match
+        // in this case, essentially everything but "happened" gets excluded
+        searchResponse = client().prepareSearch(INDEX)
+            .setRouting(route)
+            .setQuery(QueryBuilders.matchQuery("field", "bar three happened")
+                .operator(Operator.OR).cutoffFrequency(1f)).get();
+
+        assertHitCount(searchResponse, 1L);
+        assertSearchHits(searchResponse, "1");
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java b/core/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java
index 884e1d4582928..992667a5056e8 100644
--- a/core/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java
+++ b/core/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java
@@ -22,15 +22,18 @@
 import org.apache.lucene.index.Term;
 import org.apache.lucene.queries.BlendedTermQuery;
 import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.DisjunctionMaxQuery;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.SynonymQuery;
 import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.lucene.search.MultiPhrasePrefixQuery;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.mapper.MapperService;
@@ -45,6 +48,8 @@
 import java.util.Arrays;
 
 import static org.elasticsearch.index.query.QueryBuilders.multiMatchQuery;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
 
 public class MultiMatchQueryTests extends ESSingleNodeTestCase {
 
@@ -52,7 +57,12 @@ public class MultiMatchQueryTests extends ESSingleNodeTestCase {
 
     @Before
     public void setup() throws IOException {
-        IndexService indexService = createIndex("test");
+        Settings settings = Settings.builder()
+            .put("index.analysis.filter.syns.type","synonym")
+            .putArray("index.analysis.filter.syns.synonyms","quick,fast")
+            .put("index.analysis.analyzer.syns.tokenizer","standard")
+            .put("index.analysis.analyzer.syns.filter","syns").build();
+        IndexService indexService = createIndex("test", settings);
         MapperService mapperService = indexService.mapperService();
         String mapping = "{\n" +
                 "    \"person\":{\n" +
@@ -60,10 +70,12 @@ public void setup() throws IOException {
                 "            \"name\":{\n" +
                 "                  \"properties\":{\n" +
                 "                        \"first\": {\n" +
-                "                            \"type\":\"text\"\n" +
+                "                            \"type\":\"text\",\n" +
+                "                            \"analyzer\":\"syns\"\n" +
                 "                        }," +
                 "                        \"last\": {\n" +
-                "                            \"type\":\"text\"\n" +
+                "                            \"type\":\"text\",\n" +
+                "                            \"analyzer\":\"syns\"\n" +
                 "                        }" +
                 "                   }" +
                 "            }\n" +
@@ -75,7 +87,8 @@ public void setup() throws IOException {
     }
 
     public void testCrossFieldMultiMatchQuery() throws IOException {
-        QueryShardContext queryShardContext = indexService.newQueryShardContext();
+        QueryShardContext queryShardContext = indexService.newQueryShardContext(
+                randomInt(20), null, () -> { throw new UnsupportedOperationException(); });
         queryShardContext.setAllowUnmappedFields(true);
         Query parsedQuery = multiMatchQuery("banon").field("name.first", 2).field("name.last", 3).field("foobar").type(MultiMatchQueryBuilder.Type.CROSS_FIELDS).toQuery(queryShardContext);
         try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) {
@@ -98,7 +111,9 @@ public void testBlendTerms() {
         Term[] terms = new Term[] { new Term("foo", "baz"), new Term("bar", "baz") };
         float[] boosts = new float[] {2, 3};
         Query expected = BlendedTermQuery.booleanBlendedQuery(terms, boosts, false);
-        Query actual = MultiMatchQuery.blendTerm(new BytesRef("baz"), null, 1f, new FieldAndFieldType(ft1, 2), new FieldAndFieldType(ft2, 3));
+        Query actual = MultiMatchQuery.blendTerm(
+                indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }),
+                new BytesRef("baz"), null, 1f, new FieldAndFieldType(ft1, 2), new FieldAndFieldType(ft2, 3));
         assertEquals(expected, actual);
     }
 
@@ -112,7 +127,9 @@ public void testBlendTermsWithFieldBoosts() {
         Term[] terms = new Term[] { new Term("foo", "baz"), new Term("bar", "baz") };
         float[] boosts = new float[] {200, 30};
         Query expected = BlendedTermQuery.booleanBlendedQuery(terms, boosts, false);
-        Query actual = MultiMatchQuery.blendTerm(new BytesRef("baz"), null, 1f, new FieldAndFieldType(ft1, 2), new FieldAndFieldType(ft2, 3));
+        Query actual = MultiMatchQuery.blendTerm(
+                indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }),
+                new BytesRef("baz"), null, 1f, new FieldAndFieldType(ft1, 2), new FieldAndFieldType(ft2, 3));
         assertEquals(expected, actual);
     }
 
@@ -129,7 +146,9 @@ public Query termQuery(Object value, QueryShardContext context) {
         Term[] terms = new Term[] { new Term("foo", "baz") };
         float[] boosts = new float[] {2};
         Query expected = BlendedTermQuery.booleanBlendedQuery(terms, boosts, false);
-        Query actual = MultiMatchQuery.blendTerm(new BytesRef("baz"), null, 1f, new FieldAndFieldType(ft1, 2), new FieldAndFieldType(ft2, 3));
+        Query actual = MultiMatchQuery.blendTerm(
+                indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }),
+                new BytesRef("baz"), null, 1f, new FieldAndFieldType(ft1, 2), new FieldAndFieldType(ft2, 3));
         assertEquals(expected, actual);
     }
 
@@ -151,7 +170,49 @@ public Query termQuery(Object value, QueryShardContext context) {
                 .add(expectedClause1, Occur.SHOULD)
                 .add(expectedClause2, Occur.SHOULD)
                 .build();
-        Query actual = MultiMatchQuery.blendTerm(new BytesRef("baz"), null, 1f, new FieldAndFieldType(ft1, 2), new FieldAndFieldType(ft2, 3));
+        Query actual = MultiMatchQuery.blendTerm(
+                indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }),
+                new BytesRef("baz"), null, 1f, new FieldAndFieldType(ft1, 2), new FieldAndFieldType(ft2, 3));
         assertEquals(expected, actual);
     }
+
+    public void testMultiMatchPrefixWithAllField() throws IOException {
+        QueryShardContext queryShardContext = indexService.newQueryShardContext(
+                randomInt(20), null, () -> { throw new UnsupportedOperationException(); });
+        queryShardContext.setAllowUnmappedFields(true);
+        Query parsedQuery =
+            multiMatchQuery("foo").field("_all").type(MultiMatchQueryBuilder.Type.PHRASE_PREFIX).toQuery(queryShardContext);
+        assertThat(parsedQuery, instanceOf(MultiPhrasePrefixQuery.class));
+        assertThat(parsedQuery.toString(), equalTo("_all:\"foo*\""));
+    }
+
+    public void testMultiMatchCrossFieldsWithSynonyms() throws IOException {
+        QueryShardContext queryShardContext = indexService.newQueryShardContext(
+            randomInt(20), null, () -> { throw new UnsupportedOperationException(); });
+
+        // check that synonym query is used for a single field
+        Query parsedQuery =
+            multiMatchQuery("quick").field("name.first")
+                .type(MultiMatchQueryBuilder.Type.CROSS_FIELDS).toQuery(queryShardContext);
+        Term[] terms = new Term[2];
+        terms[0] = new Term("name.first", "quick");
+        terms[1] = new Term("name.first", "fast");
+        Query expectedQuery = new SynonymQuery(terms);
+        assertThat(parsedQuery, equalTo(expectedQuery));
+
+        // check that blended term query is used for multiple fields
+        parsedQuery =
+            multiMatchQuery("quick").field("name.first").field("name.last")
+                .type(MultiMatchQueryBuilder.Type.CROSS_FIELDS).toQuery(queryShardContext);
+        terms = new Term[4];
+        terms[0] = new Term("name.first", "quick");
+        terms[1] = new Term("name.first", "fast");
+        terms[2] = new Term("name.last", "quick");
+        terms[3] = new Term("name.last", "fast");
+        float[] boosts = new float[4];
+        Arrays.fill(boosts, 1.0f);
+        expectedQuery = BlendedTermQuery.dismaxBlendedQuery(terms, boosts, 1.0f);
+        assertThat(parsedQuery, equalTo(expectedQuery));
+
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/search/NestedHelperTests.java b/core/src/test/java/org/elasticsearch/index/search/NestedHelperTests.java
new file mode 100644
index 0000000000000..2b43daa679b0f
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/search/NestedHelperTests.java
@@ -0,0 +1,362 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.search;
+
+import org.apache.lucene.index.MultiReader;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BooleanClause.Occur;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.MatchNoDocsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.join.ScoreMode;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
+import org.elasticsearch.index.query.NestedQueryBuilder;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.query.TermQueryBuilder;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+
+import java.io.IOException;
+
+public class NestedHelperTests extends ESSingleNodeTestCase {
+
+    IndexService indexService;
+    MapperService mapperService;
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject()
+                .startObject("type")
+                    .startObject("properties")
+                        .startObject("foo")
+                            .field("type", "keyword")
+                        .endObject()
+                        .startObject("foo2")
+                            .field("type", "long")
+                        .endObject()
+                        .startObject("nested1")
+                            .field("type", "nested")
+                            .startObject("properties")
+                                .startObject("foo")
+                                    .field("type", "keyword")
+                                .endObject()
+                                .startObject("foo2")
+                                    .field("type", "long")
+                                .endObject()
+                            .endObject()
+                        .endObject()
+                        .startObject("nested2")
+                            .field("type", "nested")
+                            .field("include_in_parent", true)
+                            .startObject("properties")
+                                .startObject("foo")
+                                    .field("type", "keyword")
+                                .endObject()
+                                .startObject("foo2")
+                                    .field("type", "long")
+                                .endObject()
+                            .endObject()
+                        .endObject()
+                        .startObject("nested3")
+                            .field("type", "nested")
+                            .field("include_in_root", true)
+                            .startObject("properties")
+                                .startObject("foo")
+                                    .field("type", "keyword")
+                                .endObject()
+                                .startObject("foo2")
+                                    .field("type", "long")
+                                .endObject()
+                            .endObject()
+                        .endObject()
+                    .endObject()
+                .endObject().endObject();
+        indexService = createIndex("index", Settings.EMPTY, "type", mapping);
+        mapperService = indexService.mapperService();
+    }
+
+    public void testMatchAll() {
+        assertTrue(new NestedHelper(mapperService).mightMatchNestedDocs(new MatchAllDocsQuery()));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(new MatchAllDocsQuery(), "nested1"));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(new MatchAllDocsQuery(), "nested2"));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(new MatchAllDocsQuery(), "nested3"));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(new MatchAllDocsQuery(), "nested_missing"));
+    }
+
+    public void testMatchNo() {
+        assertFalse(new NestedHelper(mapperService).mightMatchNestedDocs(new MatchNoDocsQuery()));
+        assertFalse(new NestedHelper(mapperService).mightMatchNonNestedDocs(new MatchNoDocsQuery(), "nested1"));
+        assertFalse(new NestedHelper(mapperService).mightMatchNonNestedDocs(new MatchNoDocsQuery(), "nested2"));
+        assertFalse(new NestedHelper(mapperService).mightMatchNonNestedDocs(new MatchNoDocsQuery(), "nested3"));
+        assertFalse(new NestedHelper(mapperService).mightMatchNonNestedDocs(new MatchNoDocsQuery(), "nested_missing"));
+    }
+
+    public void testTermQuery() {
+        Query termQuery = mapperService.fullName("foo").termQuery("bar", null);
+        assertFalse(new NestedHelper(mapperService).mightMatchNestedDocs(termQuery));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termQuery, "nested1"));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termQuery, "nested2"));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termQuery, "nested3"));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termQuery, "nested_missing"));
+
+        termQuery = mapperService.fullName("nested1.foo").termQuery("bar", null);
+        assertTrue(new NestedHelper(mapperService).mightMatchNestedDocs(termQuery));
+        assertFalse(new NestedHelper(mapperService).mightMatchNonNestedDocs(termQuery, "nested1"));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termQuery, "nested2"));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termQuery, "nested3"));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termQuery, "nested_missing"));
+
+        termQuery = mapperService.fullName("nested2.foo").termQuery("bar", null);
+        assertTrue(new NestedHelper(mapperService).mightMatchNestedDocs(termQuery));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termQuery, "nested1"));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termQuery, "nested2"));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termQuery, "nested3"));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termQuery, "nested_missing"));
+
+        termQuery = mapperService.fullName("nested3.foo").termQuery("bar", null);
+        assertTrue(new NestedHelper(mapperService).mightMatchNestedDocs(termQuery));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termQuery, "nested1"));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termQuery, "nested2"));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termQuery, "nested3"));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termQuery, "nested_missing"));
+    }
+
+    public void testRangeQuery() {
+        Query rangeQuery = mapperService.fullName("foo2").rangeQuery(2, 5, true, true, null);
+        assertFalse(new NestedHelper(mapperService).mightMatchNestedDocs(rangeQuery));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(rangeQuery, "nested1"));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(rangeQuery, "nested2"));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(rangeQuery, "nested3"));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(rangeQuery, "nested_missing"));
+
+        rangeQuery = mapperService.fullName("nested1.foo2").rangeQuery(2, 5, true, true, null);
+        assertTrue(new NestedHelper(mapperService).mightMatchNestedDocs(rangeQuery));
+        assertFalse(new NestedHelper(mapperService).mightMatchNonNestedDocs(rangeQuery, "nested1"));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(rangeQuery, "nested2"));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(rangeQuery, "nested3"));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(rangeQuery, "nested_missing"));
+
+        rangeQuery = mapperService.fullName("nested2.foo2").rangeQuery(2, 5, true, true, null);
+        assertTrue(new NestedHelper(mapperService).mightMatchNestedDocs(rangeQuery));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(rangeQuery, "nested1"));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(rangeQuery, "nested2"));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(rangeQuery, "nested3"));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(rangeQuery, "nested_missing"));
+
+        rangeQuery = mapperService.fullName("nested3.foo2").rangeQuery(2, 5, true, true, null);
+        assertTrue(new NestedHelper(mapperService).mightMatchNestedDocs(rangeQuery));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(rangeQuery, "nested1"));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(rangeQuery, "nested2"));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(rangeQuery, "nested3"));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(rangeQuery, "nested_missing"));
+    }
+
+    public void testDisjunction() {
+        BooleanQuery bq = new BooleanQuery.Builder()
+                .add(new TermQuery(new Term("foo", "bar")), Occur.SHOULD)
+                .add(new TermQuery(new Term("foo", "baz")), Occur.SHOULD)
+                .build();
+        assertFalse(new NestedHelper(mapperService).mightMatchNestedDocs(bq));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(bq, "nested1"));
+
+        bq = new BooleanQuery.Builder()
+                .add(new TermQuery(new Term("nested1.foo", "bar")), Occur.SHOULD)
+                .add(new TermQuery(new Term("nested1.foo", "baz")), Occur.SHOULD)
+                .build();
+        assertTrue(new NestedHelper(mapperService).mightMatchNestedDocs(bq));
+        assertFalse(new NestedHelper(mapperService).mightMatchNonNestedDocs(bq, "nested1"));
+
+        bq = new BooleanQuery.Builder()
+                .add(new TermQuery(new Term("nested2.foo", "bar")), Occur.SHOULD)
+                .add(new TermQuery(new Term("nested2.foo", "baz")), Occur.SHOULD)
+                .build();
+        assertTrue(new NestedHelper(mapperService).mightMatchNestedDocs(bq));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(bq, "nested2"));
+
+        bq = new BooleanQuery.Builder()
+                .add(new TermQuery(new Term("nested3.foo", "bar")), Occur.SHOULD)
+                .add(new TermQuery(new Term("nested3.foo", "baz")), Occur.SHOULD)
+                .build();
+        assertTrue(new NestedHelper(mapperService).mightMatchNestedDocs(bq));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(bq, "nested3"));
+
+        bq = new BooleanQuery.Builder()
+                .add(new TermQuery(new Term("foo", "bar")), Occur.SHOULD)
+                .add(new MatchAllDocsQuery(), Occur.SHOULD)
+                .build();
+        assertTrue(new NestedHelper(mapperService).mightMatchNestedDocs(bq));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(bq, "nested1"));
+
+        bq = new BooleanQuery.Builder()
+                .add(new TermQuery(new Term("nested1.foo", "bar")), Occur.SHOULD)
+                .add(new MatchAllDocsQuery(), Occur.SHOULD)
+                .build();
+        assertTrue(new NestedHelper(mapperService).mightMatchNestedDocs(bq));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(bq, "nested1"));
+
+        bq = new BooleanQuery.Builder()
+                .add(new TermQuery(new Term("nested2.foo", "bar")), Occur.SHOULD)
+                .add(new MatchAllDocsQuery(), Occur.SHOULD)
+                .build();
+        assertTrue(new NestedHelper(mapperService).mightMatchNestedDocs(bq));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(bq, "nested2"));
+
+        bq = new BooleanQuery.Builder()
+                .add(new TermQuery(new Term("nested3.foo", "bar")), Occur.SHOULD)
+                .add(new MatchAllDocsQuery(), Occur.SHOULD)
+                .build();
+        assertTrue(new NestedHelper(mapperService).mightMatchNestedDocs(bq));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(bq, "nested3"));
+    }
+
+    private static Occur requiredOccur() {
+        return random().nextBoolean() ? Occur.MUST : Occur.FILTER;
+    }
+
+    public void testConjunction() {
+        BooleanQuery bq = new BooleanQuery.Builder()
+                .add(new TermQuery(new Term("foo", "bar")), requiredOccur())
+                .add(new MatchAllDocsQuery(), requiredOccur())
+                .build();
+        assertFalse(new NestedHelper(mapperService).mightMatchNestedDocs(bq));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(bq, "nested1"));
+
+        bq = new BooleanQuery.Builder()
+                .add(new TermQuery(new Term("nested1.foo", "bar")), requiredOccur())
+                .add(new MatchAllDocsQuery(), requiredOccur())
+                .build();
+        assertTrue(new NestedHelper(mapperService).mightMatchNestedDocs(bq));
+        assertFalse(new NestedHelper(mapperService).mightMatchNonNestedDocs(bq, "nested1"));
+
+        bq = new BooleanQuery.Builder()
+                .add(new TermQuery(new Term("nested2.foo", "bar")), requiredOccur())
+                .add(new MatchAllDocsQuery(), requiredOccur())
+                .build();
+        assertTrue(new NestedHelper(mapperService).mightMatchNestedDocs(bq));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(bq, "nested2"));
+
+        bq = new BooleanQuery.Builder()
+                .add(new TermQuery(new Term("nested3.foo", "bar")), requiredOccur())
+                .add(new MatchAllDocsQuery(), requiredOccur())
+                .build();
+        assertTrue(new NestedHelper(mapperService).mightMatchNestedDocs(bq));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(bq, "nested3"));
+
+        bq = new BooleanQuery.Builder()
+                .add(new MatchAllDocsQuery(), requiredOccur())
+                .add(new MatchAllDocsQuery(), requiredOccur())
+                .build();
+        assertTrue(new NestedHelper(mapperService).mightMatchNestedDocs(bq));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(bq, "nested1"));
+
+        bq = new BooleanQuery.Builder()
+                .add(new MatchAllDocsQuery(), requiredOccur())
+                .add(new MatchAllDocsQuery(), requiredOccur())
+                .build();
+        assertTrue(new NestedHelper(mapperService).mightMatchNestedDocs(bq));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(bq, "nested1"));
+
+        bq = new BooleanQuery.Builder()
+                .add(new MatchAllDocsQuery(), requiredOccur())
+                .add(new MatchAllDocsQuery(), requiredOccur())
+                .build();
+        assertTrue(new NestedHelper(mapperService).mightMatchNestedDocs(bq));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(bq, "nested2"));
+
+        bq = new BooleanQuery.Builder()
+                .add(new MatchAllDocsQuery(), requiredOccur())
+                .add(new MatchAllDocsQuery(), requiredOccur())
+                .build();
+        assertTrue(new NestedHelper(mapperService).mightMatchNestedDocs(bq));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(bq, "nested3"));
+    }
+
+    public void testNested() throws IOException {
+        QueryShardContext context = indexService.newQueryShardContext(0, new MultiReader(), () -> 0);
+        NestedQueryBuilder queryBuilder = new NestedQueryBuilder("nested1", new MatchAllQueryBuilder(), ScoreMode.Avg);
+        ESToParentBlockJoinQuery query = (ESToParentBlockJoinQuery) queryBuilder.toQuery(context);
+
+        Query expectedChildQuery = new BooleanQuery.Builder()
+                .add(new MatchAllDocsQuery(), Occur.MUST)
+                // we automatically add a filter since the inner query might match non-nested docs
+                .add(new TermQuery(new Term("_type", "__nested1")), Occur.FILTER)
+                .build();
+        assertEquals(expectedChildQuery, query.getChildQuery());
+
+        assertFalse(new NestedHelper(mapperService).mightMatchNestedDocs(query));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(query, "nested1"));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(query, "nested2"));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(query, "nested3"));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(query, "nested_missing"));
+
+        queryBuilder = new NestedQueryBuilder("nested1", new TermQueryBuilder("nested1.foo", "bar"), ScoreMode.Avg);
+        query = (ESToParentBlockJoinQuery) queryBuilder.toQuery(context);
+
+        // this time we do not add a filter since the inner query only matches inner docs
+        expectedChildQuery = new TermQuery(new Term("nested1.foo", "bar"));
+        assertEquals(expectedChildQuery, query.getChildQuery());
+
+        assertFalse(new NestedHelper(mapperService).mightMatchNestedDocs(query));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(query, "nested1"));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(query, "nested2"));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(query, "nested3"));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(query, "nested_missing"));
+
+        queryBuilder = new NestedQueryBuilder("nested2", new TermQueryBuilder("nested2.foo", "bar"), ScoreMode.Avg);
+        query = (ESToParentBlockJoinQuery) queryBuilder.toQuery(context);
+
+        // we need to add the filter again because of include_in_parent
+        expectedChildQuery = new BooleanQuery.Builder()
+                .add(new TermQuery(new Term("nested2.foo", "bar")), Occur.MUST)
+                .add(new TermQuery(new Term("_type", "__nested2")), Occur.FILTER)
+                .build();
+        assertEquals(expectedChildQuery, query.getChildQuery());
+
+        assertFalse(new NestedHelper(mapperService).mightMatchNestedDocs(query));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(query, "nested1"));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(query, "nested2"));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(query, "nested3"));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(query, "nested_missing"));
+
+        queryBuilder = new NestedQueryBuilder("nested3", new TermQueryBuilder("nested3.foo", "bar"), ScoreMode.Avg);
+        query = (ESToParentBlockJoinQuery) queryBuilder.toQuery(context);
+
+        // we need to add the filter again because of include_in_root
+        expectedChildQuery = new BooleanQuery.Builder()
+                .add(new TermQuery(new Term("nested3.foo", "bar")), Occur.MUST)
+                .add(new TermQuery(new Term("_type", "__nested3")), Occur.FILTER)
+                .build();
+        assertEquals(expectedChildQuery, query.getChildQuery());
+
+        assertFalse(new NestedHelper(mapperService).mightMatchNestedDocs(query));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(query, "nested1"));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(query, "nested2"));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(query, "nested3"));
+        assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(query, "nested_missing"));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/search/geo/GeoPointParsingTests.java b/core/src/test/java/org/elasticsearch/index/search/geo/GeoPointParsingTests.java
index 1af1b55a6eda5..6cf3c7efeaf0d 100644
--- a/core/src/test/java/org/elasticsearch/index/search/geo/GeoPointParsingTests.java
+++ b/core/src/test/java/org/elasticsearch/index/search/geo/GeoPointParsingTests.java
@@ -30,11 +30,11 @@
 
 import java.io.IOException;
 
-import static org.hamcrest.Matchers.is;
 import static org.elasticsearch.common.geo.GeoHashUtils.stringEncode;
+import static org.hamcrest.Matchers.is;
 
 public class GeoPointParsingTests  extends ESTestCase {
-    static double TOLERANCE = 1E-5;
+    private static final double TOLERANCE = 1E-5;
 
     public void testGeoPointReset() throws IOException {
         double lat = 1 + randomDouble() * 89;
@@ -110,15 +110,10 @@ public void testInvalidPointEmbeddedObject() throws IOException {
         content.endObject();
         content.endObject();
 
-        XContentParser parser = JsonXContent.jsonXContent.createParser(content.bytes());
+        XContentParser parser = createParser(JsonXContent.jsonXContent, content.bytes());
         parser.nextToken();
-
-        try {
-            GeoUtils.parseGeoPoint(parser);
-            fail("Expected ElasticsearchParseException");
-        } catch (ElasticsearchParseException e) {
-            assertThat(e.getMessage(), is("field must be either [lat], [lon] or [geohash]"));
-        }
+        Exception e = expectThrows(ElasticsearchParseException.class, () -> GeoUtils.parseGeoPoint(parser));
+        assertThat(e.getMessage(), is("field must be either [lat], [lon] or [geohash]"));
     }
 
     public void testInvalidPointLatHashMix() throws IOException {
@@ -127,15 +122,11 @@ public void testInvalidPointLatHashMix() throws IOException {
         content.field("lat", 0).field("geohash", stringEncode(0d, 0d));
         content.endObject();
 
-        XContentParser parser = JsonXContent.jsonXContent.createParser(content.bytes());
+        XContentParser parser = createParser(JsonXContent.jsonXContent, content.bytes());
         parser.nextToken();
 
-        try {
-            GeoUtils.parseGeoPoint(parser);
-            fail("Expected ElasticsearchParseException");
-        } catch (ElasticsearchParseException e) {
-            assertThat(e.getMessage(), is("field must be either lat/lon or geohash"));
-        }
+        Exception e = expectThrows(ElasticsearchParseException.class, () -> GeoUtils.parseGeoPoint(parser));
+        assertThat(e.getMessage(), is("field must be either lat/lon or geohash"));
     }
 
     public void testInvalidPointLonHashMix() throws IOException {
@@ -144,15 +135,11 @@ public void testInvalidPointLonHashMix() throws IOException {
         content.field("lon", 0).field("geohash", stringEncode(0d, 0d));
         content.endObject();
 
-        XContentParser parser = JsonXContent.jsonXContent.createParser(content.bytes());
+        XContentParser parser = createParser(JsonXContent.jsonXContent, content.bytes());
         parser.nextToken();
 
-        try {
-            GeoUtils.parseGeoPoint(parser);
-            fail("Expected ElasticsearchParseException");
-        } catch (ElasticsearchParseException e) {
-            assertThat(e.getMessage(), is("field must be either lat/lon or geohash"));
-        }
+        Exception e = expectThrows(ElasticsearchParseException.class, () -> GeoUtils.parseGeoPoint(parser));
+        assertThat(e.getMessage(), is("field must be either lat/lon or geohash"));
     }
 
     public void testInvalidField() throws IOException {
@@ -161,47 +148,43 @@ public void testInvalidField() throws IOException {
         content.field("lon", 0).field("lat", 0).field("test", 0);
         content.endObject();
 
-        XContentParser parser = JsonXContent.jsonXContent.createParser(content.bytes());
+        XContentParser parser = createParser(JsonXContent.jsonXContent, content.bytes());
         parser.nextToken();
 
-        try {
-            GeoUtils.parseGeoPoint(parser);
-            fail("Expected ElasticsearchParseException");
-        } catch (ElasticsearchParseException e) {
-            assertThat(e.getMessage(), is("field must be either [lat], [lon] or [geohash]"));
-        }
+        Exception e = expectThrows(ElasticsearchParseException.class, () -> GeoUtils.parseGeoPoint(parser));
+        assertThat(e.getMessage(), is("field must be either [lat], [lon] or [geohash]"));
     }
 
-    private static XContentParser objectLatLon(double lat, double lon) throws IOException {
+    private XContentParser objectLatLon(double lat, double lon) throws IOException {
         XContentBuilder content = JsonXContent.contentBuilder();
         content.startObject();
         content.field("lat", lat).field("lon", lon);
         content.endObject();
-        XContentParser parser = JsonXContent.jsonXContent.createParser(content.bytes());
+        XContentParser parser = createParser(JsonXContent.jsonXContent, content.bytes());
         parser.nextToken();
         return parser;
     }
 
-    private static XContentParser arrayLatLon(double lat, double lon) throws IOException {
+    private XContentParser arrayLatLon(double lat, double lon) throws IOException {
         XContentBuilder content = JsonXContent.contentBuilder();
         content.startArray().value(lon).value(lat).endArray();
-        XContentParser parser = JsonXContent.jsonXContent.createParser(content.bytes());
+        XContentParser parser = createParser(JsonXContent.jsonXContent, content.bytes());
         parser.nextToken();
         return parser;
     }
 
-    private static XContentParser stringLatLon(double lat, double lon) throws IOException {
+    private XContentParser stringLatLon(double lat, double lon) throws IOException {
         XContentBuilder content = JsonXContent.contentBuilder();
         content.value(Double.toString(lat) + ", " + Double.toString(lon));
-        XContentParser parser = JsonXContent.jsonXContent.createParser(content.bytes());
+        XContentParser parser = createParser(JsonXContent.jsonXContent, content.bytes());
         parser.nextToken();
         return parser;
     }
 
-    private static XContentParser geohash(double lat, double lon) throws IOException {
+    private XContentParser geohash(double lat, double lon) throws IOException {
         XContentBuilder content = JsonXContent.contentBuilder();
         content.value(stringEncode(lon, lat));
-        XContentParser parser = JsonXContent.jsonXContent.createParser(content.bytes());
+        XContentParser parser = createParser(JsonXContent.jsonXContent, content.bytes());
         parser.nextToken();
         return parser;
     }
diff --git a/core/src/test/java/org/elasticsearch/index/search/geo/GeoUtilsTests.java b/core/src/test/java/org/elasticsearch/index/search/geo/GeoUtilsTests.java
index 59c0e4a3bfce6..204b71e82a192 100644
--- a/core/src/test/java/org/elasticsearch/index/search/geo/GeoUtilsTests.java
+++ b/core/src/test/java/org/elasticsearch/index/search/geo/GeoUtilsTests.java
@@ -19,20 +19,19 @@
 
 package org.elasticsearch.index.search.geo;
 
-import org.locationtech.spatial4j.context.SpatialContext;
-import org.locationtech.spatial4j.distance.DistanceUtils;
 import org.apache.lucene.spatial.prefix.tree.Cell;
 import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree;
 import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree;
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.geo.GeoHashUtils;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.geo.GeoUtils;
-import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentParser.Token;
 import org.elasticsearch.test.ESTestCase;
+import org.locationtech.spatial4j.context.SpatialContext;
+import org.locationtech.spatial4j.distance.DistanceUtils;
 
 import java.io.IOException;
 
@@ -384,25 +383,25 @@ public void testParseGeoPoint() throws IOException {
         for (int i = 0; i < 100; i++) {
             double lat = randomDouble() * 180 - 90 + randomIntBetween(-1000, 1000) * 180;
             double lon = randomDouble() * 360 - 180 + randomIntBetween(-1000, 1000) * 360;
-            BytesReference jsonBytes = jsonBuilder().startObject().field("lat", lat).field("lon", lon).endObject().bytes();
-            XContentParser parser = XContentHelper.createParser(jsonBytes);
+            XContentBuilder json = jsonBuilder().startObject().field("lat", lat).field("lon", lon).endObject();
+            XContentParser parser = createParser(json);
             parser.nextToken();
             GeoPoint point = GeoUtils.parseGeoPoint(parser);
             assertThat(point, equalTo(new GeoPoint(lat, lon)));
-            jsonBytes = jsonBuilder().startObject().field("lat", String.valueOf(lat)).field("lon", String.valueOf(lon)).endObject().bytes();
-            parser = XContentHelper.createParser(jsonBytes);
+            json = jsonBuilder().startObject().field("lat", String.valueOf(lat)).field("lon", String.valueOf(lon)).endObject();
+            parser = createParser(json);
             parser.nextToken();
             point = GeoUtils.parseGeoPoint(parser);
             assertThat(point, equalTo(new GeoPoint(lat, lon)));
-            jsonBytes = jsonBuilder().startObject().startArray("foo").value(lon).value(lat).endArray().endObject().bytes();
-            parser = XContentHelper.createParser(jsonBytes);
+            json = jsonBuilder().startObject().startArray("foo").value(lon).value(lat).endArray().endObject();
+            parser = createParser(json);
             while (parser.currentToken() != Token.START_ARRAY) {
                 parser.nextToken();
             }
             point = GeoUtils.parseGeoPoint(parser);
             assertThat(point, equalTo(new GeoPoint(lat, lon)));
-            jsonBytes = jsonBuilder().startObject().field("foo", lat + "," + lon).endObject().bytes();
-            parser = XContentHelper.createParser(jsonBytes);
+            json = jsonBuilder().startObject().field("foo", lat + "," + lon).endObject();
+            parser = createParser(json);
             while (parser.currentToken() != Token.VALUE_STRING) {
                 parser.nextToken();
             }
@@ -418,14 +417,14 @@ public void testParseGeoPointGeohash() throws IOException {
             for (int j = 0; j < geoHashLength; j++) {
                 geohashBuilder.append(BASE_32[randomInt(BASE_32.length - 1)]);
             }
-            BytesReference jsonBytes = jsonBuilder().startObject().field("geohash", geohashBuilder.toString()).endObject().bytes();
-            XContentParser parser = XContentHelper.createParser(jsonBytes);
+            XContentBuilder json = jsonBuilder().startObject().field("geohash", geohashBuilder.toString()).endObject();
+            XContentParser parser = createParser(json);
             parser.nextToken();
             GeoPoint point = GeoUtils.parseGeoPoint(parser);
             assertThat(point.lat(), allOf(lessThanOrEqualTo(90.0), greaterThanOrEqualTo(-90.0)));
             assertThat(point.lon(), allOf(lessThanOrEqualTo(180.0), greaterThanOrEqualTo(-180.0)));
-            jsonBytes = jsonBuilder().startObject().field("geohash", geohashBuilder.toString()).endObject().bytes();
-            parser = XContentHelper.createParser(jsonBytes);
+            json = jsonBuilder().startObject().field("geohash", geohashBuilder.toString()).endObject();
+            parser = createParser(json);
             while (parser.currentToken() != Token.VALUE_STRING) {
                 parser.nextToken();
             }
@@ -436,145 +435,103 @@ public void testParseGeoPointGeohash() throws IOException {
     }
 
     public void testParseGeoPointGeohashWrongType() throws IOException {
-        BytesReference jsonBytes = jsonBuilder().startObject().field("geohash", 1.0).endObject().bytes();
-        XContentParser parser = XContentHelper.createParser(jsonBytes);
+        XContentBuilder json = jsonBuilder().startObject().field("geohash", 1.0).endObject();
+        XContentParser parser = createParser(json);
         parser.nextToken();
-        try {
-            GeoUtils.parseGeoPoint(parser);
-            fail("Expected ElasticsearchParseException");
-        } catch (ElasticsearchParseException e) {
-            assertThat(e.getMessage(), containsString("geohash must be a string"));
-        }
+        Exception e = expectThrows(ElasticsearchParseException.class, () -> GeoUtils.parseGeoPoint(parser));
+        assertThat(e.getMessage(), containsString("geohash must be a string"));
     }
 
     public void testParseGeoPointLatNoLon() throws IOException {
         double lat = 0.0;
-        BytesReference jsonBytes = jsonBuilder().startObject().field("lat", lat).endObject().bytes();
-        XContentParser parser = XContentHelper.createParser(jsonBytes);
+        XContentBuilder json = jsonBuilder().startObject().field("lat", lat).endObject();
+        XContentParser parser = createParser(json);
         parser.nextToken();
-        try {
-            GeoUtils.parseGeoPoint(parser);
-            fail("Expected ElasticsearchParseException");
-        } catch (ElasticsearchParseException e) {
-            assertThat(e.getMessage(), is("field [lon] missing"));
-        }
+        Exception e = expectThrows(ElasticsearchParseException.class, () -> GeoUtils.parseGeoPoint(parser));
+        assertThat(e.getMessage(), is("field [lon] missing"));
     }
 
     public void testParseGeoPointLonNoLat() throws IOException {
         double lon = 0.0;
-        BytesReference jsonBytes = jsonBuilder().startObject().field("lon", lon).endObject().bytes();
-        XContentParser parser = XContentHelper.createParser(jsonBytes);
+        XContentBuilder json = jsonBuilder().startObject().field("lon", lon).endObject();
+        XContentParser parser = createParser(json);
         parser.nextToken();
-        try {
-            GeoUtils.parseGeoPoint(parser);
-            fail("Expected ElasticsearchParseException");
-        } catch (ElasticsearchParseException e) {
-            assertThat(e.getMessage(), is("field [lat] missing"));
-        }
+        Exception e = expectThrows(ElasticsearchParseException.class, () -> GeoUtils.parseGeoPoint(parser));
+        assertThat(e.getMessage(), is("field [lat] missing"));
     }
 
     public void testParseGeoPointLonWrongType() throws IOException {
         double lat = 0.0;
-        BytesReference jsonBytes = jsonBuilder().startObject().field("lat", lat).field("lon", false).endObject().bytes();
-        XContentParser parser = XContentHelper.createParser(jsonBytes);
+        XContentBuilder json = jsonBuilder().startObject().field("lat", lat).field("lon", false).endObject();
+        XContentParser parser = createParser(json);
         parser.nextToken();
-        try {
-            GeoUtils.parseGeoPoint(parser);
-            fail("Expected ElasticsearchParseException");
-        } catch (ElasticsearchParseException e) {
-            assertThat(e.getMessage(), is("longitude must be a number"));
-        }
+        Exception e = expectThrows(ElasticsearchParseException.class, () -> GeoUtils.parseGeoPoint(parser));
+        assertThat(e.getMessage(), is("longitude must be a number"));
     }
 
     public void testParseGeoPointLatWrongType() throws IOException {
         double lon = 0.0;
-        BytesReference jsonBytes = jsonBuilder().startObject().field("lat", false).field("lon", lon).endObject().bytes();
-        XContentParser parser = XContentHelper.createParser(jsonBytes);
+        XContentBuilder json = jsonBuilder().startObject().field("lat", false).field("lon", lon).endObject();
+        XContentParser parser = createParser(json);
         parser.nextToken();
-        try {
-            GeoUtils.parseGeoPoint(parser);
-            fail("Expected ElasticsearchParseException");
-        } catch (ElasticsearchParseException e) {
-            assertThat(e.getMessage(), is("latitude must be a number"));
-        }
+        Exception e = expectThrows(ElasticsearchParseException.class, () -> GeoUtils.parseGeoPoint(parser));
+        assertThat(e.getMessage(), is("latitude must be a number"));
     }
 
     public void testParseGeoPointExtraField() throws IOException {
         double lat = 0.0;
         double lon = 0.0;
-        BytesReference jsonBytes = jsonBuilder().startObject().field("lat", lat).field("lon", lon).field("foo", true).endObject().bytes();
-        XContentParser parser = XContentHelper.createParser(jsonBytes);
+        XContentBuilder json = jsonBuilder().startObject().field("lat", lat).field("lon", lon).field("foo", true).endObject();
+        XContentParser parser = createParser(json);
         parser.nextToken();
-        try {
-            GeoUtils.parseGeoPoint(parser);
-            fail("Expected ElasticsearchParseException");
-        } catch (ElasticsearchParseException e) {
-            assertThat(e.getMessage(), is("field must be either [lat], [lon] or [geohash]"));
-        }
+        Exception e = expectThrows(ElasticsearchParseException.class, () -> GeoUtils.parseGeoPoint(parser));
+        assertThat(e.getMessage(), is("field must be either [lat], [lon] or [geohash]"));
     }
 
     public void testParseGeoPointLonLatGeoHash() throws IOException {
         double lat = 0.0;
         double lon = 0.0;
         String geohash = "abcd";
-        BytesReference jsonBytes = jsonBuilder().startObject().field("lat", lat).field("lon", lon).field("geohash", geohash).endObject()
-                .bytes();
-        XContentParser parser = XContentHelper.createParser(jsonBytes);
+        XContentBuilder json = jsonBuilder().startObject().field("lat", lat).field("lon", lon).field("geohash", geohash).endObject();
+        XContentParser parser = createParser(json);
         parser.nextToken();
-        try {
-            GeoUtils.parseGeoPoint(parser);
-            fail("Expected ElasticsearchParseException");
-        } catch (ElasticsearchParseException e) {
-            assertThat(e.getMessage(), containsString("field must be either lat/lon or geohash"));
-        }
+        Exception e = expectThrows(ElasticsearchParseException.class, () -> GeoUtils.parseGeoPoint(parser));
+        assertThat(e.getMessage(), containsString("field must be either lat/lon or geohash"));
     }
 
     public void testParseGeoPointArrayTooManyValues() throws IOException {
         double lat = 0.0;
         double lon = 0.0;
         double elev = 0.0;
-        BytesReference jsonBytes = jsonBuilder().startObject().startArray("foo").value(lon).value(lat).value(elev).endArray().endObject()
-                .bytes();
-        XContentParser parser = XContentHelper.createParser(jsonBytes);
+        XContentBuilder json = jsonBuilder().startObject().startArray("foo").value(lon).value(lat).value(elev).endArray().endObject();
+        XContentParser parser = createParser(json);
         while (parser.currentToken() != Token.START_ARRAY) {
             parser.nextToken();
         }
-        try {
-            GeoUtils.parseGeoPoint(parser);
-            fail("Expected ElasticsearchParseException");
-        } catch (ElasticsearchParseException e) {
-            assertThat(e.getMessage(), is("only two values allowed"));
-        }
+        Exception e = expectThrows(ElasticsearchParseException.class, () -> GeoUtils.parseGeoPoint(parser));
+        assertThat(e.getMessage(), is("only two values allowed"));
     }
 
     public void testParseGeoPointArrayWrongType() throws IOException {
         double lat = 0.0;
         boolean lon = false;
-        BytesReference jsonBytes = jsonBuilder().startObject().startArray("foo").value(lon).value(lat).endArray().endObject().bytes();
-        XContentParser parser = XContentHelper.createParser(jsonBytes);
+        XContentBuilder json = jsonBuilder().startObject().startArray("foo").value(lon).value(lat).endArray().endObject();
+        XContentParser parser = createParser(json);
         while (parser.currentToken() != Token.START_ARRAY) {
             parser.nextToken();
         }
-        try {
-            GeoUtils.parseGeoPoint(parser);
-            fail("Expected ElasticsearchParseException");
-        } catch (ElasticsearchParseException e) {
-            assertThat(e.getMessage(), is("numeric value expected"));
-        }
+        Exception e = expectThrows(ElasticsearchParseException.class, () -> GeoUtils.parseGeoPoint(parser));
+        assertThat(e.getMessage(), is("numeric value expected"));
     }
 
     public void testParseGeoPointInvalidType() throws IOException {
-        BytesReference jsonBytes = jsonBuilder().startObject().field("foo", 5).endObject().bytes();
-        XContentParser parser = XContentHelper.createParser(jsonBytes);
+        XContentBuilder json = jsonBuilder().startObject().field("foo", 5).endObject();
+        XContentParser parser = createParser(json);
         while (parser.currentToken() != Token.VALUE_NUMBER) {
             parser.nextToken();
         }
-        try {
-            GeoUtils.parseGeoPoint(parser);
-            fail("Expected ElasticsearchParseException");
-        } catch (ElasticsearchParseException e) {
-            assertThat(e.getMessage(), is("geo_point expected"));
-        }
+        Exception e = expectThrows(ElasticsearchParseException.class, () -> GeoUtils.parseGeoPoint(parser));
+        assertThat(e.getMessage(), is("geo_point expected"));
     }
 
     public void testPrefixTreeCellSizes() {
diff --git a/core/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java b/core/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java
index dd46c4a61db39..d6b7acd5166c3 100644
--- a/core/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java
+++ b/core/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java
@@ -115,9 +115,9 @@ public void testDuel() throws Exception {
     private TopDocs getTopDocs(IndexSearcher searcher, IndexFieldData<?> indexFieldData, String missingValue, MultiValueMode sortMode, int n, boolean reverse) throws IOException {
         Query parentFilter = new TermQuery(new Term("__type", "parent"));
         Query childFilter = new TermQuery(new Term("__type", "child"));
-        XFieldComparatorSource nestedComparatorSource = indexFieldData.comparatorSource(missingValue, sortMode, createNested(searcher, parentFilter, childFilter));
+        SortField sortField = indexFieldData.sortField(missingValue, sortMode, createNested(searcher, parentFilter, childFilter), reverse);
         Query query = new ConstantScoreQuery(parentFilter);
-        Sort sort = new Sort(new SortField("f", nestedComparatorSource, reverse));
+        Sort sort = new Sort(sortField);
         return searcher.search(query, n, sort);
     }
 
diff --git a/core/src/test/java/org/elasticsearch/index/shard/ElasticsearchQueryCachingPolicyTests.java b/core/src/test/java/org/elasticsearch/index/shard/ElasticsearchQueryCachingPolicyTests.java
new file mode 100644
index 0000000000000..0344a15810f3b
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/shard/ElasticsearchQueryCachingPolicyTests.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.shard;
+
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.PhraseQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.QueryCachingPolicy;
+import org.apache.lucene.search.TermQuery;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+public class ElasticsearchQueryCachingPolicyTests extends ESTestCase {
+
+    public void testDoesNotCacheTermQueries() throws IOException {
+        QueryCachingPolicy policy = QueryCachingPolicy.ALWAYS_CACHE;
+        assertTrue(policy.shouldCache(new TermQuery(new Term("foo", "bar"))));
+        assertTrue(policy.shouldCache(new PhraseQuery("foo", "bar", "baz")));
+        policy = new ElasticsearchQueryCachingPolicy(policy);
+        assertFalse(policy.shouldCache(new TermQuery(new Term("foo", "bar"))));
+        assertTrue(policy.shouldCache(new PhraseQuery("foo", "bar", "baz")));
+    }
+
+    public void testDoesNotPutTermQueriesIntoTheHistory() {
+        boolean[] used = new boolean[1];
+        QueryCachingPolicy policy = new QueryCachingPolicy() {
+            @Override
+            public boolean shouldCache(Query query) throws IOException {
+                throw new UnsupportedOperationException();
+            }
+            @Override
+            public void onUse(Query query) {
+                used[0] = true;
+            }
+        };
+        policy = new ElasticsearchQueryCachingPolicy(policy);
+        policy.onUse(new TermQuery(new Term("foo", "bar")));
+        assertFalse(used[0]);
+        policy.onUse(new PhraseQuery("foo", "bar", "baz"));
+        assertTrue(used[0]);
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexSearcherWrapperTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexSearcherWrapperTests.java
index cfadab6efb8f8..114552b7d13da 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/IndexSearcherWrapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/IndexSearcherWrapperTests.java
@@ -124,7 +124,7 @@ public IndexSearcher wrap(IndexSearcher searcher) throws EngineException {
         };
         final ConcurrentHashMap<Object, TopDocs> cache = new ConcurrentHashMap<>();
         try (Engine.Searcher engineSearcher = new Engine.Searcher("foo", searcher)) {
-            try (final Engine.Searcher wrap = wrapper.wrap(engineSearcher)) {
+            try (Engine.Searcher wrap = wrapper.wrap(engineSearcher)) {
                 ElasticsearchDirectoryReader.addReaderCloseListener(wrap.getDirectoryReader(), reader -> {
                     cache.remove(reader.getCoreCacheKey());
                 });
@@ -207,7 +207,7 @@ private static class FieldMaskingReader extends FilterDirectoryReader {
         private final String field;
         private final AtomicInteger closeCalls;
 
-        public FieldMaskingReader(String field, DirectoryReader in, AtomicInteger closeCalls) throws IOException {
+        FieldMaskingReader(String field, DirectoryReader in, AtomicInteger closeCalls) throws IOException {
             super(in, new SubReaderWrapper() {
                 @Override
                 public LeafReader wrap(LeafReader reader) {
@@ -239,7 +239,7 @@ private static class BrokenWrapper extends FilterDirectoryReader {
 
         private final boolean hideDelegate;
 
-        public BrokenWrapper(DirectoryReader in, boolean hideDelegate) throws IOException {
+        BrokenWrapper(DirectoryReader in, boolean hideDelegate) throws IOException {
             super(in, new SubReaderWrapper() {
                 @Override
                 public LeafReader wrap(LeafReader reader) {
diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java
new file mode 100644
index 0000000000000..3446d4c06d757
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java
@@ -0,0 +1,481 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.index.shard;
+
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.store.LockObtainFailedException;
+import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.admin.indices.stats.IndexStats;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.cluster.ClusterInfoService;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.InternalClusterInfoService;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.RecoverySource;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.TestShardRouting;
+import org.elasticsearch.cluster.routing.UnassignedInfo;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.LocalTransportAddress;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.env.ShardLock;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.flush.FlushStats;
+import org.elasticsearch.index.mapper.Mapping;
+import org.elasticsearch.index.mapper.ParseContext;
+import org.elasticsearch.index.mapper.ParsedDocument;
+import org.elasticsearch.index.mapper.Uid;
+import org.elasticsearch.index.mapper.UidFieldMapper;
+import org.elasticsearch.index.translog.Translog;
+import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.indices.recovery.RecoveryState;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.DummyShardLock;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.elasticsearch.test.IndexSettingsModule;
+import org.elasticsearch.test.InternalSettingsPlugin;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.BrokenBarrierException;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.emptySet;
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.NONE;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
+import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
+import static org.hamcrest.Matchers.equalTo;
+
+public class IndexShardIT extends ESSingleNodeTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return pluginList(InternalSettingsPlugin.class);
+    }
+
+    private ParsedDocument testParsedDocument(String id, String type, String routing, long timestamp, long ttl,
+                                              ParseContext.Document document, BytesReference source, XContentType xContentType,
+                                              Mapping mappingUpdate) {
+        Field uidField = new Field("_uid", Uid.createUid(type, id), UidFieldMapper.Defaults.FIELD_TYPE);
+        Field versionField = new NumericDocValuesField("_version", 0);
+        document.add(uidField);
+        document.add(versionField);
+        return new ParsedDocument(versionField, id, type, routing, timestamp, ttl, Collections.singletonList(document), source,
+            xContentType, mappingUpdate);
+    }
+
+    public void testLockTryingToDelete() throws Exception {
+        createIndex("test");
+        ensureGreen();
+        NodeEnvironment env = getInstanceFromNode(NodeEnvironment.class);
+
+        ClusterService cs = getInstanceFromNode(ClusterService.class);
+        final Index index = cs.state().metaData().index("test").getIndex();
+        Path[] shardPaths = env.availableShardPaths(new ShardId(index, 0));
+        logger.info("--> paths: [{}]", (Object)shardPaths);
+        // Should not be able to acquire the lock because it's already open
+        try {
+            NodeEnvironment.acquireFSLockForPaths(IndexSettingsModule.newIndexSettings("test", Settings.EMPTY), shardPaths);
+            fail("should not have been able to acquire the lock");
+        } catch (LockObtainFailedException e) {
+            assertTrue("msg: " + e.getMessage(), e.getMessage().contains("unable to acquire write.lock"));
+        }
+        // Test without the regular shard lock to assume we can acquire it
+        // (worst case, meaning that the shard lock could be acquired and
+        // we're green to delete the shard's directory)
+        ShardLock sLock = new DummyShardLock(new ShardId(index, 0));
+        try {
+            env.deleteShardDirectoryUnderLock(sLock, IndexSettingsModule.newIndexSettings("test", Settings.EMPTY));
+            fail("should not have been able to delete the directory");
+        } catch (LockObtainFailedException e) {
+            assertTrue("msg: " + e.getMessage(), e.getMessage().contains("unable to acquire write.lock"));
+        }
+    }
+
+    public void testMarkAsInactiveTriggersSyncedFlush() throws Exception {
+        assertAcked(client().admin().indices().prepareCreate("test")
+            .setSettings(SETTING_NUMBER_OF_SHARDS, 1, SETTING_NUMBER_OF_REPLICAS, 0));
+        client().prepareIndex("test", "test").setSource("{}", XContentType.JSON).get();
+        ensureGreen("test");
+        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
+        indicesService.indexService(resolveIndex("test")).getShardOrNull(0).checkIdle(0);
+        assertBusy(() -> {
+                IndexStats indexStats = client().admin().indices().prepareStats("test").clear().get().getIndex("test");
+                assertNotNull(indexStats.getShards()[0].getCommitStats().getUserData().get(Engine.SYNC_COMMIT_ID));
+                indicesService.indexService(resolveIndex("test")).getShardOrNull(0).checkIdle(0);
+            }
+        );
+        IndexStats indexStats = client().admin().indices().prepareStats("test").get().getIndex("test");
+        assertNotNull(indexStats.getShards()[0].getCommitStats().getUserData().get(Engine.SYNC_COMMIT_ID));
+    }
+
+    public void testDurableFlagHasEffect() {
+        createIndex("test");
+        ensureGreen();
+        client().prepareIndex("test", "bar", "1").setSource("{}", XContentType.JSON).get();
+        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
+        IndexService test = indicesService.indexService(resolveIndex("test"));
+        IndexShard shard = test.getShardOrNull(0);
+        setDurability(shard, Translog.Durability.REQUEST);
+        assertFalse(ShardUtilsTests.getShardEngine(shard).getTranslog().syncNeeded());
+        setDurability(shard, Translog.Durability.ASYNC);
+        client().prepareIndex("test", "bar", "2").setSource("{}", XContentType.JSON).get();
+        assertTrue(ShardUtilsTests.getShardEngine(shard).getTranslog().syncNeeded());
+        setDurability(shard, Translog.Durability.REQUEST);
+        client().prepareDelete("test", "bar", "1").get();
+        assertFalse(ShardUtilsTests.getShardEngine(shard).getTranslog().syncNeeded());
+
+        setDurability(shard, Translog.Durability.ASYNC);
+        client().prepareDelete("test", "bar", "2").get();
+        assertTrue(ShardUtilsTests.getShardEngine(shard).getTranslog().syncNeeded());
+        setDurability(shard, Translog.Durability.REQUEST);
+        assertNoFailures(client().prepareBulk()
+            .add(client().prepareIndex("test", "bar", "3").setSource("{}", XContentType.JSON))
+            .add(client().prepareDelete("test", "bar", "1")).get());
+        assertFalse(ShardUtilsTests.getShardEngine(shard).getTranslog().syncNeeded());
+
+        setDurability(shard, Translog.Durability.ASYNC);
+        assertNoFailures(client().prepareBulk()
+            .add(client().prepareIndex("test", "bar", "4").setSource("{}", XContentType.JSON))
+            .add(client().prepareDelete("test", "bar", "3")).get());
+        setDurability(shard, Translog.Durability.REQUEST);
+        assertTrue(ShardUtilsTests.getShardEngine(shard).getTranslog().syncNeeded());
+    }
+
+    private void setDurability(IndexShard shard, Translog.Durability durability) {
+        client().admin().indices().prepareUpdateSettings(shard.shardId().getIndexName()).setSettings(
+            Settings.builder().put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), durability.name()).build()).get();
+        assertEquals(durability, shard.getTranslogDurability());
+    }
+
+    public void testUpdatePriority() {
+        assertAcked(client().admin().indices().prepareCreate("test")
+            .setSettings(IndexMetaData.SETTING_PRIORITY, 200));
+        IndexService indexService = getInstanceFromNode(IndicesService.class).indexService(resolveIndex("test"));
+        assertEquals(200, indexService.getIndexSettings().getSettings().getAsInt(IndexMetaData.SETTING_PRIORITY, 0).intValue());
+        client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder().put(IndexMetaData.SETTING_PRIORITY, 400)
+            .build()).get();
+        assertEquals(400, indexService.getIndexSettings().getSettings().getAsInt(IndexMetaData.SETTING_PRIORITY, 0).intValue());
+    }
+
+    public void testIndexDirIsDeletedWhenShardRemoved() throws Exception {
+        Environment env = getInstanceFromNode(Environment.class);
+        Path idxPath = env.sharedDataFile().resolve(randomAlphaOfLength(10));
+        logger.info("--> idxPath: [{}]", idxPath);
+        Settings idxSettings = Settings.builder()
+            .put(IndexMetaData.SETTING_DATA_PATH, idxPath)
+            .build();
+        createIndex("test", idxSettings);
+        ensureGreen("test");
+        client().prepareIndex("test", "bar", "1").setSource("{}", XContentType.JSON).setRefreshPolicy(IMMEDIATE).get();
+        SearchResponse response = client().prepareSearch("test").get();
+        assertHitCount(response, 1L);
+        client().admin().indices().prepareDelete("test").get();
+        assertAllIndicesRemovedAndDeletionCompleted(Collections.singleton(getInstanceFromNode(IndicesService.class)));
+        assertPathHasBeenCleared(idxPath);
+    }
+
+    public void testExpectedShardSizeIsPresent() throws InterruptedException {
+        assertAcked(client().admin().indices().prepareCreate("test")
+            .setSettings(SETTING_NUMBER_OF_SHARDS, 1, SETTING_NUMBER_OF_REPLICAS, 0));
+        for (int i = 0; i < 50; i++) {
+            client().prepareIndex("test", "test").setSource("{}", XContentType.JSON).get();
+        }
+        ensureGreen("test");
+        InternalClusterInfoService clusterInfoService = (InternalClusterInfoService) getInstanceFromNode(ClusterInfoService.class);
+        clusterInfoService.refresh();
+        ClusterState state = getInstanceFromNode(ClusterService.class).state();
+        Long test = clusterInfoService.getClusterInfo().getShardSize(state.getRoutingTable().index("test")
+            .getShards().get(0).primaryShard());
+        assertNotNull(test);
+        assertTrue(test > 0);
+    }
+
+    public void testIndexCanChangeCustomDataPath() throws Exception {
+        Environment env = getInstanceFromNode(Environment.class);
+        Path idxPath = env.sharedDataFile().resolve(randomAlphaOfLength(10));
+        final String INDEX = "idx";
+        Path startDir = idxPath.resolve("start-" + randomAlphaOfLength(10));
+        Path endDir = idxPath.resolve("end-" + randomAlphaOfLength(10));
+        logger.info("--> start dir: [{}]", startDir.toAbsolutePath().toString());
+        logger.info("-->   end dir: [{}]", endDir.toAbsolutePath().toString());
+        // temp dirs are automatically created, but the end dir is what
+        // startDir is going to be renamed as, so it needs to be deleted
+        // otherwise we get all sorts of errors about the directory
+        // already existing
+        IOUtils.rm(endDir);
+
+        Settings sb = Settings.builder()
+            .put(IndexMetaData.SETTING_DATA_PATH, startDir.toAbsolutePath().toString())
+            .build();
+        Settings sb2 = Settings.builder()
+            .put(IndexMetaData.SETTING_DATA_PATH, endDir.toAbsolutePath().toString())
+            .build();
+
+        logger.info("--> creating an index with data_path [{}]", startDir.toAbsolutePath().toString());
+        createIndex(INDEX, sb);
+        ensureGreen(INDEX);
+        client().prepareIndex(INDEX, "bar", "1").setSource("{}", XContentType.JSON).setRefreshPolicy(IMMEDIATE).get();
+
+        SearchResponse resp = client().prepareSearch(INDEX).setQuery(matchAllQuery()).get();
+        assertThat("found the hit", resp.getHits().getTotalHits(), equalTo(1L));
+
+        logger.info("--> closing the index [{}]", INDEX);
+        client().admin().indices().prepareClose(INDEX).get();
+        logger.info("--> index closed, re-opening...");
+        client().admin().indices().prepareOpen(INDEX).get();
+        logger.info("--> index re-opened");
+        ensureGreen(INDEX);
+
+        resp = client().prepareSearch(INDEX).setQuery(matchAllQuery()).get();
+        assertThat("found the hit", resp.getHits().getTotalHits(), equalTo(1L));
+
+        // Now, try closing and changing the settings
+
+        logger.info("--> closing the index [{}]", INDEX);
+        client().admin().indices().prepareClose(INDEX).get();
+
+        logger.info("--> moving data on disk [{}] to [{}]", startDir.getFileName(), endDir.getFileName());
+        assert Files.exists(endDir) == false : "end directory should not exist!";
+        Files.move(startDir, endDir, StandardCopyOption.REPLACE_EXISTING);
+
+        logger.info("--> updating settings...");
+        client().admin().indices().prepareUpdateSettings(INDEX)
+            .setSettings(sb2)
+            .setIndicesOptions(IndicesOptions.fromOptions(true, false, true, true))
+            .get();
+
+        assert Files.exists(startDir) == false : "start dir shouldn't exist";
+
+        logger.info("--> settings updated and files moved, re-opening index");
+        client().admin().indices().prepareOpen(INDEX).get();
+        logger.info("--> index re-opened");
+        ensureGreen(INDEX);
+
+        resp = client().prepareSearch(INDEX).setQuery(matchAllQuery()).get();
+        assertThat("found the hit", resp.getHits().getTotalHits(), equalTo(1L));
+
+        assertAcked(client().admin().indices().prepareDelete(INDEX));
+        assertAllIndicesRemovedAndDeletionCompleted(Collections.singleton(getInstanceFromNode(IndicesService.class)));
+        assertPathHasBeenCleared(startDir.toAbsolutePath());
+        assertPathHasBeenCleared(endDir.toAbsolutePath());
+    }
+
+    public void testMaybeFlush() throws Exception {
+        createIndex("test", Settings.builder().put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), Translog.Durability.REQUEST)
+            .build());
+        ensureGreen();
+        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
+        IndexService test = indicesService.indexService(resolveIndex("test"));
+        IndexShard shard = test.getShardOrNull(0);
+        assertFalse(shard.shouldFlush());
+        client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder()
+            .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(),
+                new ByteSizeValue(133 /* size of the operation + header&footer*/, ByteSizeUnit.BYTES)).build()).get();
+        client().prepareIndex("test", "test", "0").setSource("{}").setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get();
+        assertFalse(shard.shouldFlush());
+        ParsedDocument doc = testParsedDocument("1", "test", null, -1, -1, new ParseContext.Document(),
+            new BytesArray(new byte[]{1}), XContentType.JSON, null);
+        Engine.Index index = new Engine.Index(new Term("_uid", Uid.createUid(doc.type(), doc.id())), doc);
+        shard.index(index);
+        assertTrue(shard.shouldFlush());
+        assertEquals(2, shard.getEngine().getTranslog().totalOperations());
+        client().prepareIndex("test", "test", "2").setSource("{}", XContentType.JSON)
+            .setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get();
+        assertBusy(() -> { // this is async
+            assertFalse(shard.shouldFlush());
+        });
+        assertEquals(0, shard.getEngine().getTranslog().totalOperations());
+        shard.getEngine().getTranslog().sync();
+        long size = shard.getEngine().getTranslog().sizeInBytes();
+        logger.info("--> current translog size: [{}] num_ops [{}] generation [{}]", shard.getEngine().getTranslog().sizeInBytes(),
+            shard.getEngine().getTranslog().totalOperations(), shard.getEngine().getTranslog().getGeneration());
+        client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder().put(
+            IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(size, ByteSizeUnit.BYTES))
+            .build()).get();
+        client().prepareDelete("test", "test", "2").get();
+        logger.info("--> translog size after delete: [{}] num_ops [{}] generation [{}]", shard.getEngine().getTranslog().sizeInBytes(),
+            shard.getEngine().getTranslog().totalOperations(), shard.getEngine().getTranslog().getGeneration());
+        assertBusy(() -> { // this is async
+            logger.info("--> translog size on iter  : [{}] num_ops [{}] generation [{}]", shard.getEngine().getTranslog().sizeInBytes(),
+                shard.getEngine().getTranslog().totalOperations(), shard.getEngine().getTranslog().getGeneration());
+            assertFalse(shard.shouldFlush());
+        });
+        assertEquals(0, shard.getEngine().getTranslog().totalOperations());
+    }
+
+    public void testStressMaybeFlush() throws Exception {
+        createIndex("test");
+        ensureGreen();
+        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
+        IndexService test = indicesService.indexService(resolveIndex("test"));
+        final IndexShard shard = test.getShardOrNull(0);
+        assertFalse(shard.shouldFlush());
+        client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder().put(
+            IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(),
+            new ByteSizeValue(133/* size of the operation + header&footer*/, ByteSizeUnit.BYTES)).build()).get();
+        client().prepareIndex("test", "test", "0").setSource("{}", XContentType.JSON)
+            .setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get();
+        assertFalse(shard.shouldFlush());
+        final AtomicBoolean running = new AtomicBoolean(true);
+        final int numThreads = randomIntBetween(2, 4);
+        Thread[] threads = new Thread[numThreads];
+        CyclicBarrier barrier = new CyclicBarrier(numThreads + 1);
+        for (int i = 0; i < threads.length; i++) {
+            threads[i] = new Thread() {
+                @Override
+                public void run() {
+                    try {
+                        barrier.await();
+                    } catch (InterruptedException | BrokenBarrierException e) {
+                        throw new RuntimeException(e);
+                    }
+                    while (running.get()) {
+                        shard.maybeFlush();
+                    }
+                }
+            };
+            threads[i].start();
+        }
+        barrier.await();
+        FlushStats flushStats = shard.flushStats();
+        long total = flushStats.getTotal();
+        client().prepareIndex("test", "test", "1").setSource("{}", XContentType.JSON).get();
+        assertBusy(() -> assertEquals(total + 1, shard.flushStats().getTotal()));
+        running.set(false);
+        for (int i = 0; i < threads.length; i++) {
+            threads[i].join();
+        }
+        assertEquals(total + 1, shard.flushStats().getTotal());
+    }
+
+    public void testShardHasMemoryBufferOnTranslogRecover() throws Throwable {
+        createIndex("test");
+        ensureGreen();
+        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
+        IndexService indexService = indicesService.indexService(resolveIndex("test"));
+        IndexShard shard = indexService.getShardOrNull(0);
+        client().prepareIndex("test", "test", "0").setSource("{\"foo\" : \"bar\"}", XContentType.JSON).get();
+        client().prepareDelete("test", "test", "0").get();
+        client().prepareIndex("test", "test", "1").setSource("{\"foo\" : \"bar\"}", XContentType.JSON).setRefreshPolicy(IMMEDIATE).get();
+
+        IndexSearcherWrapper wrapper = new IndexSearcherWrapper() {};
+        shard.close("simon says", false);
+        AtomicReference<IndexShard> shardRef = new AtomicReference<>();
+        List<Exception> failures = new ArrayList<>();
+        IndexingOperationListener listener = new IndexingOperationListener() {
+
+            @Override
+            public void postIndex(ShardId shardId, Engine.Index index, Engine.IndexResult result) {
+                try {
+                    assertNotNull(shardRef.get());
+                    // this is all IMC needs to do - check current memory and refresh
+                    assertTrue(shardRef.get().getIndexBufferRAMBytesUsed() > 0);
+                    shardRef.get().refresh("test");
+                } catch (Exception e) {
+                    failures.add(e);
+                    throw e;
+                }
+            }
+
+
+            @Override
+            public void postDelete(ShardId shardId, Engine.Delete delete, Engine.DeleteResult result) {
+                try {
+                    assertNotNull(shardRef.get());
+                    // this is all IMC needs to do - check current memory and refresh
+                    assertTrue(shardRef.get().getIndexBufferRAMBytesUsed() > 0);
+                    shardRef.get().refresh("test");
+                } catch (Exception e) {
+                    failures.add(e);
+                    throw e;
+                }
+            }
+        };
+        final IndexShard newShard = newIndexShard(indexService, shard, wrapper, listener);
+        shardRef.set(newShard);
+        recoverShard(newShard);
+
+        try {
+            ExceptionsHelper.rethrowAndSuppress(failures);
+        } finally {
+            newShard.close("just do it", randomBoolean());
+        }
+    }
+
+
+    public  static final IndexShard recoverShard(IndexShard newShard) throws IOException {
+        DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
+        newShard.markAsRecovering("store", new RecoveryState(newShard.routingEntry(), localNode, null));
+        assertTrue(newShard.recoverFromStore());
+        newShard.updateRoutingEntry(newShard.routingEntry().moveToStarted());
+        return newShard;
+    }
+
+    public  static final IndexShard newIndexShard(IndexService indexService, IndexShard shard, IndexSearcherWrapper wrapper,
+                                                  IndexingOperationListener... listeners) throws IOException {
+        ShardRouting initializingShardRouting = getInitializingShardRouting(shard.routingEntry());
+        IndexShard newShard = new IndexShard(initializingShardRouting, indexService.getIndexSettings(), shard.shardPath(),
+            shard.store(), indexService.cache(), indexService.mapperService(), indexService.similarityService(),
+            indexService.fieldData(), shard.getEngineFactory(), indexService.getIndexEventListener(), wrapper,
+            indexService.getThreadPool(), indexService.getBigArrays(), null, Collections.emptyList(), Arrays.asList(listeners));
+        return newShard;
+    }
+
+    private static ShardRouting getInitializingShardRouting(ShardRouting existingShardRouting) {
+        ShardRouting shardRouting = TestShardRouting.newShardRouting(existingShardRouting.shardId(),
+            existingShardRouting.currentNodeId(), null, existingShardRouting.primary(), ShardRoutingState.INITIALIZING,
+            existingShardRouting.allocationId());
+        shardRouting = shardRouting.updateUnassigned(new UnassignedInfo(UnassignedInfo.Reason.INDEX_REOPENED, "fake recovery"),
+            RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE);
+        return shardRouting;
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardOperationsLockTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardOperationsLockTests.java
index c9bb9e1986629..e456508f8af63 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardOperationsLockTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardOperationsLockTests.java
@@ -18,9 +18,13 @@
  */
 package org.elasticsearch.index.shard;
 
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.PlainActionFuture;
-import org.elasticsearch.common.inject.internal.Nullable;
 import org.elasticsearch.common.lease.Releasable;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
+import org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -35,8 +39,10 @@
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Function;
+import java.util.function.Supplier;
 
+import static org.hamcrest.Matchers.either;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 
@@ -48,7 +54,18 @@ public class IndexShardOperationsLockTests extends ESTestCase {
 
     @BeforeClass
     public static void setupThreadPool() {
-        threadPool = new TestThreadPool("IndexShardOperationsLockTests");
+        int bulkThreadPoolSize = randomIntBetween(1, 2);
+        int bulkThreadPoolQueueSize = randomIntBetween(1, 2);
+        threadPool = new TestThreadPool("IndexShardOperationsLockTests",
+            Settings.builder()
+                .put("thread_pool." + ThreadPool.Names.BULK + ".size", bulkThreadPoolSize)
+                .put("thread_pool." + ThreadPool.Names.BULK + ".queue_size", bulkThreadPoolQueueSize)
+                .build());
+        assertThat(threadPool.executor(ThreadPool.Names.BULK), instanceOf(EsThreadPoolExecutor.class));
+        assertThat(((EsThreadPoolExecutor) threadPool.executor(ThreadPool.Names.BULK)).getCorePoolSize(), equalTo(bulkThreadPoolSize));
+        assertThat(((EsThreadPoolExecutor) threadPool.executor(ThreadPool.Names.BULK)).getMaximumPoolSize(), equalTo(bulkThreadPoolSize));
+        assertThat(((EsThreadPoolExecutor) threadPool.executor(ThreadPool.Names.BULK)).getQueue().remainingCapacity(),
+            equalTo(bulkThreadPoolQueueSize));
     }
 
     @AfterClass
@@ -71,33 +88,53 @@ public void checkNoInflightOperations() {
     public void testAllOperationsInvoked() throws InterruptedException, TimeoutException, ExecutionException {
         int numThreads = 10;
 
+        class DummyException extends RuntimeException {}
+
         List<PlainActionFuture<Releasable>> futures = new ArrayList<>();
         List<Thread> operationThreads = new ArrayList<>();
-        CountDownLatch latch = new CountDownLatch(numThreads / 2);
+        CountDownLatch latch = new CountDownLatch(numThreads / 4);
+        boolean forceExecution = randomBoolean();
         for (int i = 0; i < numThreads; i++) {
+            // the bulk thread pool uses a bounded size and can get rejections, see setupThreadPool
+            String threadPoolName = randomFrom(ThreadPool.Names.BULK, ThreadPool.Names.GENERIC);
+            boolean failingListener = randomBoolean();
             PlainActionFuture<Releasable> future = new PlainActionFuture<Releasable>() {
                 @Override
                 public void onResponse(Releasable releasable) {
                     releasable.close();
-                    super.onResponse(releasable);
+                    if (failingListener) {
+                        throw new DummyException();
+                    } else {
+                        super.onResponse(releasable);
+                    }
                 }
             };
             Thread thread = new Thread() {
                 public void run() {
                     latch.countDown();
-                    block.acquire(future, ThreadPool.Names.GENERIC, true);
+                    try {
+                        block.acquire(future, threadPoolName, forceExecution);
+                    } catch (DummyException dummyException) {
+                        // ok, notify future
+                        assertTrue(failingListener);
+                        future.onFailure(dummyException);
+                    }
                 }
             };
             futures.add(future);
             operationThreads.add(thread);
         }
 
+        boolean closeAfterBlocking = randomBoolean();
         CountDownLatch blockFinished = new CountDownLatch(1);
         threadPool.generic().execute(() -> {
             try {
                 latch.await();
                 blockAndWait().close();
                 blockFinished.countDown();
+                if (closeAfterBlocking) {
+                    block.close();
+                }
             } catch (InterruptedException e) {
                 throw new RuntimeException(e);
             }
@@ -108,7 +145,16 @@ public void run() {
         }
 
         for (PlainActionFuture<Releasable> future : futures) {
-            assertNotNull(future.get(1, TimeUnit.MINUTES));
+            try {
+                assertNotNull(future.get(1, TimeUnit.MINUTES));
+            } catch (ExecutionException e) {
+                if (closeAfterBlocking) {
+                    assertThat(e.getCause(), either(instanceOf(DummyException.class)).or(instanceOf(EsRejectedExecutionException.class))
+                        .or(instanceOf(IndexShardClosedException.class)));
+                } else {
+                    assertThat(e.getCause(), either(instanceOf(DummyException.class)).or(instanceOf(EsRejectedExecutionException.class)));
+                }
+            }
         }
 
         for (Thread thread : operationThreads) {
@@ -118,7 +164,6 @@ public void run() {
         blockFinished.await();
     }
 
-
     public void testOperationsInvokedImmediatelyIfNoBlock() throws ExecutionException, InterruptedException {
         PlainActionFuture<Releasable> future = new PlainActionFuture<>();
         block.acquire(future, ThreadPool.Names.GENERIC, true);
@@ -146,7 +191,57 @@ public void testOperationsDelayedIfBlock() throws ExecutionException, Interrupte
             block.acquire(future, ThreadPool.Names.GENERIC, true);
             assertFalse(future.isDone());
         }
-        future.get(1, TimeUnit.MINUTES).close();
+        future.get(1, TimeUnit.HOURS).close();
+    }
+
+    /**
+     * Tests that the ThreadContext is restored when a operation is executed after it has been delayed due to a block
+     */
+    public void testThreadContextPreservedIfBlock() throws ExecutionException, InterruptedException, TimeoutException {
+        final ThreadContext context = threadPool.getThreadContext();
+        final Function<ActionListener<Releasable>, Boolean> contextChecker = (listener) -> {
+            if ("bar".equals(context.getHeader("foo")) == false) {
+                listener.onFailure(new IllegalStateException("context did not have value [bar] for header [foo]. Actual value [" +
+                    context.getHeader("foo") + "]"));
+            } else if ("baz".equals(context.getTransient("bar")) == false) {
+                listener.onFailure(new IllegalStateException("context did not have value [baz] for transient [bar]. Actual value [" +
+                    context.getTransient("bar") + "]"));
+            } else {
+                return true;
+            }
+            return false;
+        };
+        PlainActionFuture<Releasable> future = new PlainActionFuture<Releasable>() {
+            @Override
+            public void onResponse(Releasable releasable) {
+                if (contextChecker.apply(this)) {
+                    super.onResponse(releasable);
+                }
+            }
+        };
+        PlainActionFuture<Releasable> future2 = new PlainActionFuture<Releasable>() {
+            @Override
+            public void onResponse(Releasable releasable) {
+                if (contextChecker.apply(this)) {
+                    super.onResponse(releasable);
+                }
+            }
+        };
+
+        try (Releasable releasable = blockAndWait()) {
+            // we preserve the thread context here so that we have a different context in the call to acquire than the context present
+            // when the releasable is closed
+            try (ThreadContext.StoredContext ignore = context.newStoredContext(false)) {
+                context.putHeader("foo", "bar");
+                context.putTransient("bar", "baz");
+                // test both with and without a executor name
+                block.acquire(future, ThreadPool.Names.GENERIC, true);
+                block.acquire(future2, null, true);
+            }
+            assertFalse(future.isDone());
+        }
+        future.get(1, TimeUnit.HOURS).close();
+        future2.get(1, TimeUnit.HOURS).close();
     }
 
     protected Releasable blockAndWait() throws InterruptedException {
diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
index b4725a8506d50..d27242f0532ef 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
@@ -29,81 +29,65 @@
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.LockObtainFailedException;
 import org.apache.lucene.util.Constants;
-import org.apache.lucene.util.IOUtils;
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.flush.FlushRequest;
-import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
+import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
 import org.elasticsearch.action.admin.indices.stats.CommonStats;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
-import org.elasticsearch.action.admin.indices.stats.IndexStats;
 import org.elasticsearch.action.admin.indices.stats.ShardStats;
 import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.index.TransportIndexAction;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.PlainActionFuture;
-import org.elasticsearch.cluster.ClusterInfoService;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.InternalClusterInfoService;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.metadata.RepositoryMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.AllocationId;
-import org.elasticsearch.cluster.routing.RecoverySource.LocalShardsRecoverySource;
-import org.elasticsearch.cluster.routing.RecoverySource.SnapshotRecoverySource;
-import org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource;
+import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingHelper;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
-import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.lease.Releasables;
+import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.LocalTransportAddress;
-import org.elasticsearch.common.unit.ByteSizeUnit;
-import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
+import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.env.Environment;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.env.NodeEnvironment;
-import org.elasticsearch.env.ShardLock;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.index.IndexService;
-import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.engine.EngineException;
 import org.elasticsearch.index.fielddata.FieldDataStats;
 import org.elasticsearch.index.fielddata.IndexFieldData;
-import org.elasticsearch.index.flush.FlushStats;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapping;
 import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.ParsedDocument;
+import org.elasticsearch.index.mapper.SourceToParse;
 import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.index.mapper.UidFieldMapper;
 import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.index.translog.TranslogTests;
-import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.indices.IndicesQueryCache;
 import org.elasticsearch.indices.recovery.RecoveryState;
-import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.indices.recovery.RecoveryTarget;
 import org.elasticsearch.repositories.IndexId;
 import org.elasticsearch.repositories.Repository;
 import org.elasticsearch.repositories.RepositoryData;
@@ -112,20 +96,14 @@
 import org.elasticsearch.snapshots.SnapshotInfo;
 import org.elasticsearch.snapshots.SnapshotShardFailure;
 import org.elasticsearch.test.DummyShardLock;
-import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.FieldMaskingReader;
-import org.elasticsearch.test.IndexSettingsModule;
-import org.elasticsearch.test.InternalSettingsPlugin;
 import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.IOException;
-import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.StandardCopyOption;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -142,35 +120,42 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.BiConsumer;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
-import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
-import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.NONE;
-import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
-import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
-import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_VERSION_CREATED;
+import static org.elasticsearch.cluster.routing.TestShardRouting.newShardRouting;
 import static org.elasticsearch.common.lucene.Lucene.cleanLuceneIndex;
 import static org.elasticsearch.common.xcontent.ToXContent.EMPTY_PARAMS;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits;
+import static org.elasticsearch.index.engine.Engine.Operation.Origin.PRIMARY;
+import static org.elasticsearch.repositories.RepositoryData.EMPTY_REPO_GEN;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.nullValue;
 
 /**
  * Simple unit-test IndexShard related operations.
  */
-public class IndexShardTests extends ESSingleNodeTestCase {
+public class IndexShardTests extends IndexShardTestCase {
 
-    @Override
-    protected Collection<Class<? extends Plugin>> getPlugins() {
-        return pluginList(InternalSettingsPlugin.class);
+    public static ShardStateMetaData load(Logger logger, Path... shardPaths) throws IOException {
+        return ShardStateMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, shardPaths);
+    }
+
+    public static void write(ShardStateMetaData shardStateMetaData,
+                             Path... shardPaths) throws IOException {
+        ShardStateMetaData.FORMAT.write(shardStateMetaData, shardPaths);
+    }
+
+    public static Engine getEngineFromShard(IndexShard shard) {
+        return shard.getEngineOrNull();
     }
 
     public void testWriteShardState() throws Exception {
@@ -197,73 +182,41 @@ public void testWriteShardState() throws Exception {
         }
     }
 
-    public void testLockTryingToDelete() throws Exception {
-        createIndex("test");
-        ensureGreen();
-        NodeEnvironment env = getInstanceFromNode(NodeEnvironment.class);
-
-        ClusterService cs = getInstanceFromNode(ClusterService.class);
-        final Index index = cs.state().metaData().index("test").getIndex();
-        Path[] shardPaths = env.availableShardPaths(new ShardId(index, 0));
-        logger.info("--> paths: [{}]", (Object)shardPaths);
-        // Should not be able to acquire the lock because it's already open
-        try {
-            NodeEnvironment.acquireFSLockForPaths(IndexSettingsModule.newIndexSettings("test", Settings.EMPTY), shardPaths);
-            fail("should not have been able to acquire the lock");
-        } catch (LockObtainFailedException e) {
-            assertTrue("msg: " + e.getMessage(), e.getMessage().contains("unable to acquire write.lock"));
-        }
-        // Test without the regular shard lock to assume we can acquire it
-        // (worst case, meaning that the shard lock could be acquired and
-        // we're green to delete the shard's directory)
-        ShardLock sLock = new DummyShardLock(new ShardId(index, 0));
-        try {
-            env.deleteShardDirectoryUnderLock(sLock, IndexSettingsModule.newIndexSettings("test", Settings.EMPTY));
-            fail("should not have been able to delete the directory");
-        } catch (LockObtainFailedException e) {
-            assertTrue("msg: " + e.getMessage(), e.getMessage().contains("unable to acquire write.lock"));
-        }
-    }
-
     public void testPersistenceStateMetadataPersistence() throws Exception {
-        createIndex("test");
-        ensureGreen();
-        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        NodeEnvironment env = getInstanceFromNode(NodeEnvironment.class);
-        IndexService test = indicesService.indexService(resolveIndex("test"));
-        IndexShard shard = test.getShardOrNull(0);
-        ShardStateMetaData shardStateMetaData = load(logger, env.availableShardPaths(shard.shardId));
+        IndexShard shard = newStartedShard();
+        final Path shardStatePath = shard.shardPath().getShardStatePath();
+        ShardStateMetaData shardStateMetaData = load(logger, shardStatePath);
         assertEquals(getShardStateMetadata(shard), shardStateMetaData);
         ShardRouting routing = shard.shardRouting;
         shard.updateRoutingEntry(routing);
 
-        shardStateMetaData = load(logger, env.availableShardPaths(shard.shardId));
+        shardStateMetaData = load(logger, shardStatePath);
         assertEquals(shardStateMetaData, getShardStateMetadata(shard));
-        assertEquals(shardStateMetaData, new ShardStateMetaData(routing.primary(), shard.indexSettings().getUUID(), routing.allocationId()));
+        assertEquals(shardStateMetaData,
+            new ShardStateMetaData(routing.primary(), shard.indexSettings().getUUID(), routing.allocationId()));
 
         routing = TestShardRouting.relocate(shard.shardRouting, "some node", 42L);
         shard.updateRoutingEntry(routing);
-        shardStateMetaData = load(logger, env.availableShardPaths(shard.shardId));
+        shardStateMetaData = load(logger, shardStatePath);
         assertEquals(shardStateMetaData, getShardStateMetadata(shard));
-        assertEquals(shardStateMetaData, new ShardStateMetaData(routing.primary(), shard.indexSettings().getUUID(), routing.allocationId()));
+        assertEquals(shardStateMetaData,
+            new ShardStateMetaData(routing.primary(), shard.indexSettings().getUUID(), routing.allocationId()));
+        closeShards(shard);
     }
 
     public void testFailShard() throws Exception {
-        createIndex("test");
-        ensureGreen();
-        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        NodeEnvironment env = getInstanceFromNode(NodeEnvironment.class);
-        IndexService test = indicesService.indexService(resolveIndex("test"));
-        IndexShard shard = test.getShardOrNull(0);
+        IndexShard shard = newStartedShard();
+        final ShardPath shardPath = shard.shardPath();
+        assertNotNull(shardPath);
         // fail shard
         shard.failShard("test shard fail", new CorruptIndexException("", ""));
+        closeShards(shard);
         // check state file still exists
-        ShardStateMetaData shardStateMetaData = load(logger, env.availableShardPaths(shard.shardId));
+        ShardStateMetaData shardStateMetaData = load(logger, shardPath.getShardStatePath());
         assertEquals(shardStateMetaData, getShardStateMetadata(shard));
-        ShardPath shardPath = ShardPath.loadShardPath(logger, env, shard.shardId(), test.getIndexSettings());
-        assertNotNull(shardPath);
         // but index can't be opened for a failed shard
-        assertThat("store index should be corrupted", Store.canOpenIndex(logger, shardPath.resolveIndex(), shard.shardId(), env::shardLock),
+        assertThat("store index should be corrupted", Store.canOpenIndex(logger, shardPath.resolveIndex(), shard.shardId(),
+            (shardId, lockTimeoutMS) -> new DummyShardLock(shardId)),
             equalTo(false));
     }
 
@@ -286,10 +239,12 @@ private AllocationId randomAllocationId() {
 
     public void testShardStateMetaHashCodeEquals() {
         AllocationId allocationId = randomBoolean() ? null : randomAllocationId();
-        ShardStateMetaData meta = new ShardStateMetaData(randomLong(), randomBoolean(), randomRealisticUnicodeOfCodepointLengthBetween(1, 10), allocationId);
+        ShardStateMetaData meta = new ShardStateMetaData(randomLong(), randomBoolean(),
+            randomRealisticUnicodeOfCodepointLengthBetween(1, 10), allocationId);
 
         assertEquals(meta, new ShardStateMetaData(meta.legacyVersion, meta.primary, meta.indexUUID, meta.allocationId));
-        assertEquals(meta.hashCode(), new ShardStateMetaData(meta.legacyVersion, meta.primary, meta.indexUUID, meta.allocationId).hashCode());
+        assertEquals(meta.hashCode(),
+            new ShardStateMetaData(meta.legacyVersion, meta.primary, meta.indexUUID, meta.allocationId).hashCode());
 
         assertFalse(meta.equals(new ShardStateMetaData(meta.legacyVersion, !meta.primary, meta.indexUUID, meta.allocationId)));
         assertFalse(meta.equals(new ShardStateMetaData(meta.legacyVersion + 1, meta.primary, meta.indexUUID, meta.allocationId)));
@@ -298,20 +253,17 @@ public void testShardStateMetaHashCodeEquals() {
         Set<Integer> hashCodes = new HashSet<>();
         for (int i = 0; i < 30; i++) { // just a sanity check that we impl hashcode
             allocationId = randomBoolean() ? null : randomAllocationId();
-            meta = new ShardStateMetaData(randomLong(), randomBoolean(), randomRealisticUnicodeOfCodepointLengthBetween(1, 10), allocationId);
+            meta = new ShardStateMetaData(randomLong(), randomBoolean(),
+                randomRealisticUnicodeOfCodepointLengthBetween(1, 10), allocationId);
             hashCodes.add(meta.hashCode());
         }
         assertTrue("more than one unique hashcode expected but got: " + hashCodes.size(), hashCodes.size() > 1);
 
     }
 
-    public void testDeleteIndexPreventsNewOperations() throws InterruptedException, ExecutionException, IOException {
-        assertAcked(client().admin().indices().prepareCreate("test").setSettings(Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0)).get());
-        ensureGreen("test");
-        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService indexService = indicesService.indexServiceSafe(resolveIndex("test"));
-        IndexShard indexShard = indexService.getShardOrNull(0);
-        client().admin().indices().prepareDelete("test").get();
+    public void testClosesPreventsNewOperations() throws InterruptedException, ExecutionException, IOException {
+        IndexShard indexShard = newStartedShard();
+        closeShards(indexShard);
         assertThat(indexShard.getActiveOperationsCount(), equalTo(0));
         try {
             indexShard.acquirePrimaryOperationLock(null, ThreadPool.Names.INDEX);
@@ -328,35 +280,27 @@ public void testDeleteIndexPreventsNewOperations() throws InterruptedException,
     }
 
     public void testOperationLocksOnPrimaryShards() throws InterruptedException, ExecutionException, IOException {
-        assertAcked(client().admin().indices().prepareCreate("test").setSettings(Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0)).get());
-        ensureGreen("test");
-        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService indexService = indicesService.indexServiceSafe(resolveIndex("test"));
-        IndexShard indexShard = indexService.getShardOrNull(0);
-        long primaryTerm = indexShard.getPrimaryTerm();
-
-        ShardRouting temp = indexShard.routingEntry();
-        final ShardRouting newPrimaryShardRouting;
+        final ShardId shardId = new ShardId("test", "_na_", 0);
+        final IndexShard indexShard;
+
         if (randomBoolean()) {
             // relocation target
-            newPrimaryShardRouting = TestShardRouting.newShardRouting(temp.shardId(), temp.currentNodeId(), "other node",
-                true, ShardRoutingState.INITIALIZING, AllocationId.newRelocation(temp.allocationId()));
+            indexShard = newShard(newShardRouting(shardId, "local_node", "other node",
+                true, ShardRoutingState.INITIALIZING, AllocationId.newRelocation(AllocationId.newInitializing())));
         } else if (randomBoolean()) {
             // simulate promotion
-            ShardRouting newReplicaShardRouting = TestShardRouting.newShardRouting(temp.shardId(), temp.currentNodeId(), null,
-                false, ShardRoutingState.STARTED, temp.allocationId());
-            indexShard.updateRoutingEntry(newReplicaShardRouting);
-            primaryTerm = primaryTerm + 1;
-            indexShard.updatePrimaryTerm(primaryTerm);
-            newPrimaryShardRouting = TestShardRouting.newShardRouting(temp.shardId(), temp.currentNodeId(), null,
-                true, ShardRoutingState.STARTED, temp.allocationId());
+            indexShard = newStartedShard(false);
+            ShardRouting replicaRouting = indexShard.routingEntry();
+            indexShard.updatePrimaryTerm(indexShard.getPrimaryTerm() + 1);
+            ShardRouting primaryRouting = newShardRouting(replicaRouting.shardId(), replicaRouting.currentNodeId(), null,
+                true, ShardRoutingState.STARTED, replicaRouting.allocationId());
+            indexShard.updateRoutingEntry(primaryRouting);
         } else {
-            newPrimaryShardRouting = temp;
+            indexShard = newStartedShard(true);
         }
-        indexShard.updateRoutingEntry(newPrimaryShardRouting);
-
+        final long primaryTerm = indexShard.getPrimaryTerm();
         assertEquals(0, indexShard.getActiveOperationsCount());
-        if (newPrimaryShardRouting.isRelocationTarget() == false) {
+        if (indexShard.routingEntry().isRelocationTarget() == false) {
             try {
                 indexShard.acquireReplicaOperationLock(primaryTerm, null, ThreadPool.Names.INDEX);
                 fail("shard shouldn't accept operations as replica");
@@ -371,6 +315,8 @@ public void testOperationLocksOnPrimaryShards() throws InterruptedException, Exe
 
         Releasables.close(operation1, operation2);
         assertEquals(0, indexShard.getActiveOperationsCount());
+
+        closeShards(indexShard);
     }
 
     private Releasable acquirePrimaryOperationLockBlockingly(IndexShard indexShard) throws ExecutionException, InterruptedException {
@@ -379,56 +325,52 @@ private Releasable acquirePrimaryOperationLockBlockingly(IndexShard indexShard)
         return fut.get();
     }
 
-    private Releasable acquireReplicaOperationLockBlockingly(IndexShard indexShard, long opPrimaryTerm) throws ExecutionException, InterruptedException {
+    private Releasable acquireReplicaOperationLockBlockingly(IndexShard indexShard, long opPrimaryTerm)
+        throws ExecutionException, InterruptedException {
         PlainActionFuture<Releasable> fut = new PlainActionFuture<>();
         indexShard.acquireReplicaOperationLock(opPrimaryTerm, fut, ThreadPool.Names.INDEX);
         return fut.get();
     }
 
     public void testOperationLocksOnReplicaShards() throws InterruptedException, ExecutionException, IOException {
-        assertAcked(client().admin().indices().prepareCreate("test").setSettings(Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0)).get());
-        ensureGreen("test");
-        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService indexService = indicesService.indexServiceSafe(resolveIndex("test"));
-        IndexShard indexShard = indexService.getShardOrNull(0);
-        long primaryTerm = indexShard.getPrimaryTerm();
-
-        // ugly hack to allow the shard to operated as a replica
-        final ShardRouting temp = indexShard.routingEntry();
-        final ShardRouting newShardRouting;
+        final ShardId shardId = new ShardId("test", "_na_", 0);
+        final IndexShard indexShard;
+
         switch (randomInt(2)) {
             case 0:
                 // started replica
-                newShardRouting = TestShardRouting.newShardRouting(temp.shardId(), temp.currentNodeId(), null,
-                    false, ShardRoutingState.STARTED, AllocationId.newRelocation(temp.allocationId()));
-
-                indexShard.updateRoutingEntry(newShardRouting);
+                indexShard = newStartedShard(false);
                 break;
-            case 1:
+            case 1: {
                 // initializing replica / primary
                 final boolean relocating = randomBoolean();
-                newShardRouting = TestShardRouting.newShardRouting(temp.shardId(), temp.currentNodeId(),
+                ShardRouting routing = newShardRouting(shardId, "local_node",
                     relocating ? "sourceNode" : null,
                     relocating ? randomBoolean() : false,
                     ShardRoutingState.INITIALIZING,
-                    relocating ? AllocationId.newRelocation(temp.allocationId()) : temp.allocationId());
-                indexShard.updateRoutingEntry(newShardRouting);
+                    relocating ? AllocationId.newRelocation(AllocationId.newInitializing()) : AllocationId.newInitializing());
+                indexShard = newShard(routing);
                 break;
-            case 2:
+            }
+            case 2: {
                 // relocation source
-                newShardRouting = TestShardRouting.newShardRouting(temp.shardId(), temp.currentNodeId(), "otherNode",
-                    false, ShardRoutingState.RELOCATING, AllocationId.newRelocation(temp.allocationId()));
-                indexShard.updateRoutingEntry(newShardRouting);
+                indexShard = newStartedShard(true);
+                ShardRouting routing = indexShard.routingEntry();
+                routing = newShardRouting(routing.shardId(), routing.currentNodeId(), "otherNode",
+                    true, ShardRoutingState.RELOCATING, AllocationId.newRelocation(routing.allocationId()));
+                indexShard.updateRoutingEntry(routing);
                 indexShard.relocated("test");
                 break;
+            }
             default:
                 throw new UnsupportedOperationException("get your numbers straight");
 
         }
-        logger.info("updated shard routing to {}", newShardRouting);
+        final ShardRouting shardRouting = indexShard.routingEntry();
+        logger.info("shard routing to {}", shardRouting);
 
         assertEquals(0, indexShard.getActiveOperationsCount());
-        if (newShardRouting.primary() == false) {
+        if (shardRouting.primary() == false) {
             try {
                 indexShard.acquirePrimaryOperationLock(null, ThreadPool.Names.INDEX);
                 fail("shard shouldn't accept primary ops");
@@ -437,6 +379,8 @@ public void testOperationLocksOnReplicaShards() throws InterruptedException, Exe
             }
         }
 
+        final long primaryTerm = indexShard.getPrimaryTerm();
+
         Releasable operation1 = acquireReplicaOperationLockBlockingly(indexShard, primaryTerm);
         assertEquals(1, indexShard.getActiveOperationsCount());
         Releasable operation2 = acquireReplicaOperationLockBlockingly(indexShard, primaryTerm);
@@ -454,76 +398,45 @@ public void testOperationLocksOnReplicaShards() throws InterruptedException, Exe
         acquireReplicaOperationLockBlockingly(indexShard, primaryTerm + 1 + randomInt(20)).close();
         Releasables.close(operation1, operation2);
         assertEquals(0, indexShard.getActiveOperationsCount());
-    }
 
-    public void testMarkAsInactiveTriggersSyncedFlush() throws Exception {
-        assertAcked(client().admin().indices().prepareCreate("test")
-            .setSettings(SETTING_NUMBER_OF_SHARDS, 1, SETTING_NUMBER_OF_REPLICAS, 0));
-        client().prepareIndex("test", "test").setSource("{}").get();
-        ensureGreen("test");
-        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        indicesService.indexService(resolveIndex("test")).getShardOrNull(0).checkIdle(0);
-        assertBusy(() -> {
-            IndexStats indexStats = client().admin().indices().prepareStats("test").clear().get().getIndex("test");
-            assertNotNull(indexStats.getShards()[0].getCommitStats().getUserData().get(Engine.SYNC_COMMIT_ID));
-            indicesService.indexService(resolveIndex("test")).getShardOrNull(0).checkIdle(0);
-            }
-        );
-        IndexStats indexStats = client().admin().indices().prepareStats("test").get().getIndex("test");
-        assertNotNull(indexStats.getShards()[0].getCommitStats().getUserData().get(Engine.SYNC_COMMIT_ID));
-    }
-
-    public static ShardStateMetaData load(Logger logger, Path... shardPaths) throws IOException {
-        return ShardStateMetaData.FORMAT.loadLatestState(logger, shardPaths);
-    }
-
-    public static void write(ShardStateMetaData shardStateMetaData,
-                             Path... shardPaths) throws IOException {
-        ShardStateMetaData.FORMAT.write(shardStateMetaData, shardPaths);
+        closeShards(indexShard);
     }
 
     public void testAcquireIndexCommit() throws IOException {
-        createIndex("test");
-        ensureGreen();
-        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService test = indicesService.indexService(resolveIndex("test"));
-        final IndexShard shard = test.getShardOrNull(0);
+        final IndexShard shard = newStartedShard();
         int numDocs = randomInt(20);
         for (int i = 0; i < numDocs; i++) {
-            client().prepareIndex("test", "type", "id_" + i).setSource("{}").get();
+            indexDoc(shard, "type", "id_" + i);
         }
         final boolean flushFirst = randomBoolean();
         IndexCommit commit = shard.acquireIndexCommit(flushFirst);
         int moreDocs = randomInt(20);
         for (int i = 0; i < moreDocs; i++) {
-            client().prepareIndex("test", "type", "id_" + numDocs + i).setSource("{}").get();
+            indexDoc(shard, "type", "id_" + numDocs + i);
         }
-        shard.flush(new FlushRequest("index"));
+        flushShard(shard);
         // check that we can still read the commit that we captured
         try (IndexReader reader = DirectoryReader.open(commit)) {
             assertThat(reader.numDocs(), equalTo(flushFirst ? numDocs : 0));
         }
         shard.releaseIndexCommit(commit);
-        shard.flush(new FlushRequest("index").force(true));
+        flushShard(shard, true);
+
         // check it's clean up
         assertThat(DirectoryReader.listCommits(shard.store().directory()), hasSize(1));
+
+        closeShards(shard);
     }
 
     /***
      * test one can snapshot the store at various lifecycle stages
      */
     public void testSnapshotStore() throws IOException {
-        createIndex("test");
-        ensureGreen();
-        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService test = indicesService.indexService(resolveIndex("test"));
-        final IndexShard shard = test.getShardOrNull(0);
-        client().prepareIndex("test", "test", "0").setSource("{}").setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get();
-        client().admin().indices().prepareFlush().get();
-        ShardRouting routing = shard.routingEntry();
-        test.removeShard(0, "b/c simon says so");
-        routing = routing.reinitializePrimaryShard();
-        IndexShard newShard = test.createShard(routing);
+        final IndexShard shard = newStartedShard(true);
+        indexDoc(shard, "test", "0");
+        flushShard(shard);
+
+        final IndexShard newShard = reinitShard(shard);
         DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
 
         Store.MetadataSnapshot snapshot = newShard.snapshotStoreMetadata();
@@ -539,7 +452,7 @@ public void testSnapshotStore() throws IOException {
         snapshot = newShard.snapshotStoreMetadata();
         assertThat(snapshot.getSegmentsFile().name(), equalTo("segments_2"));
 
-        newShard.updateRoutingEntry(getInitializingShardRouting(routing).moveToStarted());
+        newShard.updateRoutingEntry(newShard.routingEntry().moveToStarted());
 
         snapshot = newShard.snapshotStoreMetadata();
         assertThat(snapshot.getSegmentsFile().name(), equalTo("segments_2"));
@@ -548,48 +461,12 @@ public void testSnapshotStore() throws IOException {
 
         snapshot = newShard.snapshotStoreMetadata();
         assertThat(snapshot.getSegmentsFile().name(), equalTo("segments_2"));
-    }
 
-    public void testDurableFlagHasEffect() {
-        createIndex("test");
-        ensureGreen();
-        client().prepareIndex("test", "bar", "1").setSource("{}").get();
-        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService test = indicesService.indexService(resolveIndex("test"));
-        IndexShard shard = test.getShardOrNull(0);
-        setDurability(shard, Translog.Durability.REQUEST);
-        assertFalse(shard.getEngine().getTranslog().syncNeeded());
-        setDurability(shard, Translog.Durability.ASYNC);
-        client().prepareIndex("test", "bar", "2").setSource("{}").get();
-        assertTrue(shard.getEngine().getTranslog().syncNeeded());
-        setDurability(shard, Translog.Durability.REQUEST);
-        client().prepareDelete("test", "bar", "1").get();
-        assertFalse(shard.getEngine().getTranslog().syncNeeded());
-
-        setDurability(shard, Translog.Durability.ASYNC);
-        client().prepareDelete("test", "bar", "2").get();
-        assertTrue(shard.getEngine().getTranslog().syncNeeded());
-        setDurability(shard, Translog.Durability.REQUEST);
-        assertNoFailures(client().prepareBulk()
-            .add(client().prepareIndex("test", "bar", "3").setSource("{}"))
-            .add(client().prepareDelete("test", "bar", "1")).get());
-        assertFalse(shard.getEngine().getTranslog().syncNeeded());
-
-        setDurability(shard, Translog.Durability.ASYNC);
-        assertNoFailures(client().prepareBulk()
-            .add(client().prepareIndex("test", "bar", "4").setSource("{}"))
-            .add(client().prepareDelete("test", "bar", "3")).get());
-        setDurability(shard, Translog.Durability.REQUEST);
-        assertTrue(shard.getEngine().getTranslog().syncNeeded());
+        closeShards(newShard);
     }
 
-    public void testAsyncFsync() throws InterruptedException {
-        createIndex("test");
-        ensureGreen();
-        client().prepareIndex("test", "bar", "1").setSource("{}").get();
-        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService test = indicesService.indexService(resolveIndex("test"));
-        IndexShard shard = test.getShardOrNull(0);
+    public void testAsyncFsync() throws InterruptedException, IOException {
+        IndexShard shard = newStartedShard();
         Semaphore semaphore = new Semaphore(Integer.MAX_VALUE);
         Thread[] thread = new Thread[randomIntBetween(3, 5)];
         CountDownLatch latch = new CountDownLatch(thread.length);
@@ -607,7 +484,7 @@ public void run() {
                     } catch (Exception ex) {
                         throw new RuntimeException(ex);
                     }
-                };
+                }
             };
             thread[i].start();
         }
@@ -616,170 +493,36 @@ public void run() {
             thread[i].join();
         }
         assertTrue(semaphore.tryAcquire(Integer.MAX_VALUE, 10, TimeUnit.SECONDS));
-    }
 
-    private void setDurability(IndexShard shard, Translog.Durability durability) {
-        client().admin().indices().prepareUpdateSettings(shard.shardId.getIndexName()).setSettings(Settings.builder().put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), durability.name()).build()).get();
-        assertEquals(durability, shard.getTranslogDurability());
+        closeShards(shard);
     }
 
-    public void testMinimumCompatVersion() {
+    public void testMinimumCompatVersion() throws IOException {
         Version versionCreated = VersionUtils.randomVersion(random());
-        assertAcked(client().admin().indices().prepareCreate("test")
-            .setSettings(SETTING_NUMBER_OF_SHARDS, 1, SETTING_NUMBER_OF_REPLICAS, 0, SETTING_VERSION_CREATED, versionCreated.id));
-        client().prepareIndex("test", "test").setSource("{}").get();
-        ensureGreen("test");
-        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexShard test = indicesService.indexService(resolveIndex("test")).getShardOrNull(0);
+        Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, versionCreated.id)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+            .build();
+        IndexMetaData metaData = IndexMetaData.builder("test")
+            .settings(settings)
+            .primaryTerm(0, 1).build();
+        IndexShard test = newShard(new ShardId(metaData.getIndex(), 0), true, "n1", metaData, null);
+        recoveryShardFromStore(test);
+
+        indexDoc(test, "test", "test");
         assertEquals(versionCreated.luceneVersion, test.minimumCompatibleVersion());
-        client().prepareIndex("test", "test").setSource("{}").get();
+        indexDoc(test, "test", "test");
         assertEquals(versionCreated.luceneVersion, test.minimumCompatibleVersion());
         test.getEngine().flush();
         assertEquals(Version.CURRENT.luceneVersion, test.minimumCompatibleVersion());
-    }
-
-    public void testUpdatePriority() {
-        assertAcked(client().admin().indices().prepareCreate("test")
-            .setSettings(IndexMetaData.SETTING_PRIORITY, 200));
-        IndexService indexService = getInstanceFromNode(IndicesService.class).indexService(resolveIndex("test"));
-        assertEquals(200, indexService.getIndexSettings().getSettings().getAsInt(IndexMetaData.SETTING_PRIORITY, 0).intValue());
-        client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder().put(IndexMetaData.SETTING_PRIORITY, 400).build()).get();
-        assertEquals(400, indexService.getIndexSettings().getSettings().getAsInt(IndexMetaData.SETTING_PRIORITY, 0).intValue());
-    }
 
-    public void testRecoverIntoLeftover() throws IOException {
-        createIndex("test");
-        ensureGreen("test");
-        client().prepareIndex("test", "bar", "1").setSource("{}").setRefreshPolicy(IMMEDIATE).get();
-        client().admin().indices().prepareFlush("test").get();
-        SearchResponse response = client().prepareSearch("test").get();
-        assertHitCount(response, 1L);
-        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService test = indicesService.indexService(resolveIndex("test"));
-        IndexShard shard = test.getShardOrNull(0);
-        ShardPath shardPath = shard.shardPath();
-        Path dataPath = shardPath.getDataPath();
-        client().admin().indices().prepareClose("test").get();
-        Path tempDir = createTempDir();
-        Files.move(dataPath, tempDir.resolve("test"));
-        client().admin().indices().prepareDelete("test").get();
-        Files.createDirectories(dataPath.getParent());
-        Files.move(tempDir.resolve("test"), dataPath);
-        createIndex("test");
-        ensureGreen("test");
-        response = client().prepareSearch("test").get();
-        assertHitCount(response, 0L);
-    }
-
-    public void testIndexDirIsDeletedWhenShardRemoved() throws Exception {
-        Environment env = getInstanceFromNode(Environment.class);
-        Path idxPath = env.sharedDataFile().resolve(randomAsciiOfLength(10));
-        logger.info("--> idxPath: [{}]", idxPath);
-        Settings idxSettings = Settings.builder()
-            .put(IndexMetaData.SETTING_DATA_PATH, idxPath)
-            .build();
-        createIndex("test", idxSettings);
-        ensureGreen("test");
-        client().prepareIndex("test", "bar", "1").setSource("{}").setRefreshPolicy(IMMEDIATE).get();
-        SearchResponse response = client().prepareSearch("test").get();
-        assertHitCount(response, 1L);
-        client().admin().indices().prepareDelete("test").get();
-        assertAllIndicesRemovedAndDeletionCompleted(Collections.singleton(getInstanceFromNode(IndicesService.class)));
-        assertPathHasBeenCleared(idxPath);
-    }
-
-    public void testExpectedShardSizeIsPresent() throws InterruptedException {
-        assertAcked(client().admin().indices().prepareCreate("test")
-            .setSettings(SETTING_NUMBER_OF_SHARDS, 1, SETTING_NUMBER_OF_REPLICAS, 0));
-        for (int i = 0; i < 50; i++) {
-            client().prepareIndex("test", "test").setSource("{}").get();
-        }
-        ensureGreen("test");
-        InternalClusterInfoService clusterInfoService = (InternalClusterInfoService) getInstanceFromNode(ClusterInfoService.class);
-        clusterInfoService.refresh();
-        ClusterState state = getInstanceFromNode(ClusterService.class).state();
-        Long test = clusterInfoService.getClusterInfo().getShardSize(state.getRoutingTable().index("test").getShards().get(0).primaryShard());
-        assertNotNull(test);
-        assertTrue(test > 0);
-    }
-
-    public void testIndexCanChangeCustomDataPath() throws Exception {
-        Environment env = getInstanceFromNode(Environment.class);
-        Path idxPath = env.sharedDataFile().resolve(randomAsciiOfLength(10));
-        final String INDEX = "idx";
-        Path startDir = idxPath.resolve("start-" + randomAsciiOfLength(10));
-        Path endDir = idxPath.resolve("end-" + randomAsciiOfLength(10));
-        logger.info("--> start dir: [{}]", startDir.toAbsolutePath().toString());
-        logger.info("-->   end dir: [{}]", endDir.toAbsolutePath().toString());
-        // temp dirs are automatically created, but the end dir is what
-        // startDir is going to be renamed as, so it needs to be deleted
-        // otherwise we get all sorts of errors about the directory
-        // already existing
-        IOUtils.rm(endDir);
-
-        Settings sb = Settings.builder()
-            .put(IndexMetaData.SETTING_DATA_PATH, startDir.toAbsolutePath().toString())
-            .build();
-        Settings sb2 = Settings.builder()
-            .put(IndexMetaData.SETTING_DATA_PATH, endDir.toAbsolutePath().toString())
-            .build();
-
-        logger.info("--> creating an index with data_path [{}]", startDir.toAbsolutePath().toString());
-        createIndex(INDEX, sb);
-        ensureGreen(INDEX);
-        client().prepareIndex(INDEX, "bar", "1").setSource("{}").setRefreshPolicy(IMMEDIATE).get();
-
-        SearchResponse resp = client().prepareSearch(INDEX).setQuery(matchAllQuery()).get();
-        assertThat("found the hit", resp.getHits().getTotalHits(), equalTo(1L));
-
-        logger.info("--> closing the index [{}]", INDEX);
-        client().admin().indices().prepareClose(INDEX).get();
-        logger.info("--> index closed, re-opening...");
-        client().admin().indices().prepareOpen(INDEX).get();
-        logger.info("--> index re-opened");
-        ensureGreen(INDEX);
-
-        resp = client().prepareSearch(INDEX).setQuery(matchAllQuery()).get();
-        assertThat("found the hit", resp.getHits().getTotalHits(), equalTo(1L));
-
-        // Now, try closing and changing the settings
-
-        logger.info("--> closing the index [{}]", INDEX);
-        client().admin().indices().prepareClose(INDEX).get();
-
-        logger.info("--> moving data on disk [{}] to [{}]", startDir.getFileName(), endDir.getFileName());
-        assert Files.exists(endDir) == false : "end directory should not exist!";
-        Files.move(startDir, endDir, StandardCopyOption.REPLACE_EXISTING);
-
-        logger.info("--> updating settings...");
-        client().admin().indices().prepareUpdateSettings(INDEX)
-            .setSettings(sb2)
-            .setIndicesOptions(IndicesOptions.fromOptions(true, false, true, true))
-            .get();
-
-        assert Files.exists(startDir) == false : "start dir shouldn't exist";
-
-        logger.info("--> settings updated and files moved, re-opening index");
-        client().admin().indices().prepareOpen(INDEX).get();
-        logger.info("--> index re-opened");
-        ensureGreen(INDEX);
-
-        resp = client().prepareSearch(INDEX).setQuery(matchAllQuery()).get();
-        assertThat("found the hit", resp.getHits().getTotalHits(), equalTo(1L));
-
-        assertAcked(client().admin().indices().prepareDelete(INDEX));
-        assertAllIndicesRemovedAndDeletionCompleted(Collections.singleton(getInstanceFromNode(IndicesService.class)));
-        assertPathHasBeenCleared(startDir.toAbsolutePath());
-        assertPathHasBeenCleared(endDir.toAbsolutePath());
+        closeShards(test);
     }
 
     public void testShardStats() throws IOException {
-        createIndex("test");
-        ensureGreen();
-        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService test = indicesService.indexService(resolveIndex("test"));
-        IndexShard shard = test.getShardOrNull(0);
-        ShardStats stats = new ShardStats(shard.routingEntry(), shard.shardPath(), new CommonStats(indicesService.getIndicesQueryCache(), shard, new CommonStatsFlags()), shard.commitStats());
+        IndexShard shard = newStartedShard();
+        ShardStats stats = new ShardStats(shard.routingEntry(), shard.shardPath(),
+            new CommonStats(new IndicesQueryCache(Settings.EMPTY), shard, new CommonStatsFlags()), shard.commitStats());
         assertEquals(shard.shardPath().getRootDataPath().toString(), stats.getDataPath());
         assertEquals(shard.shardPath().getRootStatePath().toString(), stats.getStatePath());
         assertEquals(shard.shardPath().isCustomDataPath(), stats.isCustomDataPath());
@@ -790,7 +533,7 @@ public void testShardStats() throws IOException {
             StreamInput in = out.bytes().streamInput();
             stats = ShardStats.readShardStats(in);
         }
-        XContentBuilder builder = XContentFactory.jsonBuilder();
+        XContentBuilder builder = jsonBuilder();
         builder.startObject();
         stats.toXContent(builder, EMPTY_PARAMS);
         builder.endObject();
@@ -800,25 +543,48 @@ public void testShardStats() throws IOException {
         expectedSubSequence.append("\",\"data_path\":\"");
         expectedSubSequence.append(shard.shardPath().getRootDataPath().toString());
         expectedSubSequence.append("\",\"is_custom_data_path\":").append(shard.shardPath().isCustomDataPath()).append("}");
-        assumeFalse("Some path weirdness on windows", Constants.WINDOWS);
-        assertTrue(xContent.contains(expectedSubSequence));
+        if (Constants.WINDOWS) {
+            // Some path weirdness on windows
+        } else {
+            assertTrue(xContent.contains(expectedSubSequence));
+        }
+        closeShards(shard);
     }
 
-    private ParsedDocument testParsedDocument(String uid, String id, String type, String routing, long timestamp, long ttl, ParseContext.Document document, BytesReference source, Mapping mappingUpdate) {
-        Field uidField = new Field("_uid", uid, UidFieldMapper.Defaults.FIELD_TYPE);
+    public void testRefreshMetric() throws IOException {
+        IndexShard shard = newStartedShard();
+        assertThat(shard.refreshStats().getTotal(), equalTo(2L)); // one refresh on end of recovery, one on starting shard
+        long initialTotalTime = shard.refreshStats().getTotalTimeInMillis();
+        // check time advances
+        for (int i = 1; shard.refreshStats().getTotalTimeInMillis() == initialTotalTime; i++) {
+            indexDoc(shard, "test", "test" + i);
+            assertThat(shard.refreshStats().getTotal(), equalTo(2L + i - 1));
+            shard.refresh("test");
+            assertThat(shard.refreshStats().getTotal(), equalTo(2L + i));
+            assertThat(shard.refreshStats().getTotalTimeInMillis(), greaterThanOrEqualTo(initialTotalTime));
+        }
+        long refreshCount = shard.refreshStats().getTotal();
+        indexDoc(shard, "test", "last");
+        try (Engine.GetResult ignored = shard.get(new Engine.Get(true, "test", "last",
+            new Term(UidFieldMapper.NAME, Uid.createUid("test", "last"))))) {
+            assertThat(shard.refreshStats().getTotal(), equalTo(refreshCount + 1));
+        }
+        closeShards(shard);
+    }
+
+    private ParsedDocument testParsedDocument(String id, String type, String routing, long timestamp, long ttl,
+                                              ParseContext.Document document, BytesReference source, Mapping mappingUpdate) {
+        Field uidField = new Field("_uid", Uid.createUid(type, id), UidFieldMapper.Defaults.FIELD_TYPE);
         Field versionField = new NumericDocValuesField("_version", 0);
         document.add(uidField);
         document.add(versionField);
-        return new ParsedDocument(versionField, id, type, routing, timestamp, ttl, Arrays.asList(document), source, mappingUpdate);
+        return new ParsedDocument(versionField, id, type, routing, timestamp, ttl, Arrays.asList(document), source,
+            XContentType.JSON, mappingUpdate);
     }
 
     public void testIndexingOperationsListeners() throws IOException {
-        createIndex("test_iol");
-        ensureGreen();
-        client().prepareIndex("test_iol", "test", "0").setSource("{\"foo\" : \"bar\"}").setRefreshPolicy(IMMEDIATE).get();
-        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService test = indicesService.indexService(resolveIndex("test_iol"));
-        IndexShard shard = test.getShardOrNull(0);
+        IndexShard shard = newStartedShard(true);
+        indexDoc(shard, "test", "0", "{\"foo\" : \"bar\"}");
         AtomicInteger preIndex = new AtomicInteger();
         AtomicInteger postIndexCreate = new AtomicInteger();
         AtomicInteger postIndexUpdate = new AtomicInteger();
@@ -827,47 +593,57 @@ public void testIndexingOperationsListeners() throws IOException {
         AtomicInteger postDelete = new AtomicInteger();
         AtomicInteger postDeleteException = new AtomicInteger();
         shard.close("simon says", true);
-        shard = reinitWithWrapper(test, shard, null, new IndexingOperationListener() {
+        shard = reinitShard(shard, new IndexingOperationListener() {
             @Override
-            public Engine.Index preIndex(Engine.Index operation) {
+            public Engine.Index preIndex(ShardId shardId, Engine.Index operation) {
                 preIndex.incrementAndGet();
                 return operation;
             }
 
             @Override
-            public void postIndex(Engine.Index index, boolean created) {
-                if(created) {
-                    postIndexCreate.incrementAndGet();
+            public void postIndex(ShardId shardId, Engine.Index index, Engine.IndexResult result) {
+                if (result.hasFailure() == false) {
+                    if (result.isCreated()) {
+                        postIndexCreate.incrementAndGet();
+                    } else {
+                        postIndexUpdate.incrementAndGet();
+                    }
                 } else {
-                    postIndexUpdate.incrementAndGet();
+                    postIndex(shardId, index, result.getFailure());
                 }
             }
 
             @Override
-            public void postIndex(Engine.Index index, Exception ex) {
+            public void postIndex(ShardId shardId, Engine.Index index, Exception ex) {
                 postIndexException.incrementAndGet();
             }
 
             @Override
-            public Engine.Delete preDelete(Engine.Delete delete) {
+            public Engine.Delete preDelete(ShardId shardId, Engine.Delete delete) {
                 preDelete.incrementAndGet();
                 return delete;
             }
 
             @Override
-            public void postDelete(Engine.Delete delete) {
-                postDelete.incrementAndGet();
+            public void postDelete(ShardId shardId, Engine.Delete delete, Engine.DeleteResult result) {
+                if (result.hasFailure() == false) {
+                    postDelete.incrementAndGet();
+                } else {
+                    postDelete(shardId, delete, result.getFailure());
+                }
             }
 
             @Override
-            public void postDelete(Engine.Delete delete, Exception ex) {
+            public void postDelete(ShardId shardId, Engine.Delete delete, Exception ex) {
                 postDeleteException.incrementAndGet();
 
             }
         });
+        recoveryShardFromStore(shard);
 
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, new ParseContext.Document(), new BytesArray(new byte[]{1}), null);
-        Engine.Index index = new Engine.Index(new Term("_uid", "1"), doc);
+        ParsedDocument doc = testParsedDocument("1", "test", null, -1, -1, new ParseContext.Document(),
+            new BytesArray(new byte[]{1}), null);
+        Engine.Index index = new Engine.Index(new Term("_uid", Uid.createUid(doc.type(), doc.id())), doc);
         shard.index(index);
         assertEquals(1, preIndex.get());
         assertEquals(1, postIndexCreate.get());
@@ -886,7 +662,7 @@ public void postDelete(Engine.Delete delete, Exception ex) {
         assertEquals(0, postDelete.get());
         assertEquals(0, postDeleteException.get());
 
-        Engine.Delete delete = new Engine.Delete("test", "1", new Term("_uid", "1"));
+        Engine.Delete delete = new Engine.Delete("test", "1", new Term("_uid", Uid.createUid(doc.type(), doc.id())));
         shard.delete(delete);
 
         assertEquals(2, preIndex.get());
@@ -903,7 +679,7 @@ public void postDelete(Engine.Delete delete, Exception ex) {
         try {
             shard.index(index);
             fail();
-        } catch (IllegalIndexShardStateException e) {
+        } catch (AlreadyClosedException e) {
 
         }
 
@@ -917,7 +693,7 @@ public void postDelete(Engine.Delete delete, Exception ex) {
         try {
             shard.delete(delete);
             fail();
-        } catch (IllegalIndexShardStateException e) {
+        } catch (AlreadyClosedException e) {
 
         }
 
@@ -928,95 +704,13 @@ public void postDelete(Engine.Delete delete, Exception ex) {
         assertEquals(1, preDelete.get());
         assertEquals(1, postDelete.get());
         assertEquals(0, postDeleteException.get());
-    }
 
-    public void testMaybeFlush() throws Exception {
-        createIndex("test", Settings.builder().put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), Translog.Durability.REQUEST).build());
-        ensureGreen();
-        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService test = indicesService.indexService(resolveIndex("test"));
-        IndexShard shard = test.getShardOrNull(0);
-        assertFalse(shard.shouldFlush());
-        client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder().put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(133 /* size of the operation + header&footer*/, ByteSizeUnit.BYTES)).build()).get();
-        client().prepareIndex("test", "test", "0").setSource("{}").setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get();
-        assertFalse(shard.shouldFlush());
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, new ParseContext.Document(), new BytesArray(new byte[]{1}), null);
-        Engine.Index index = new Engine.Index(new Term("_uid", "1"), doc);
-        shard.index(index);
-        assertTrue(shard.shouldFlush());
-        assertEquals(2, shard.getEngine().getTranslog().totalOperations());
-        client().prepareIndex("test", "test", "2").setSource("{}").setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get();
-        assertBusy(() -> { // this is async
-            assertFalse(shard.shouldFlush());
-        });
-        assertEquals(0, shard.getEngine().getTranslog().totalOperations());
-        shard.getEngine().getTranslog().sync();
-        long size = shard.getEngine().getTranslog().sizeInBytes();
-        logger.info("--> current translog size: [{}] num_ops [{}] generation [{}]", shard.getEngine().getTranslog().sizeInBytes(), shard.getEngine().getTranslog().totalOperations(), shard.getEngine().getTranslog().getGeneration());
-        client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder().put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(size, ByteSizeUnit.BYTES))
-            .build()).get();
-        client().prepareDelete("test", "test", "2").get();
-        logger.info("--> translog size after delete: [{}] num_ops [{}] generation [{}]", shard.getEngine().getTranslog().sizeInBytes(), shard.getEngine().getTranslog().totalOperations(), shard.getEngine().getTranslog().getGeneration());
-        assertBusy(() -> { // this is async
-            logger.info("--> translog size on iter  : [{}] num_ops [{}] generation [{}]", shard.getEngine().getTranslog().sizeInBytes(), shard.getEngine().getTranslog().totalOperations(), shard.getEngine().getTranslog().getGeneration());
-            assertFalse(shard.shouldFlush());
-        });
-        assertEquals(0, shard.getEngine().getTranslog().totalOperations());
-    }
-
-    public void testStressMaybeFlush() throws Exception {
-        createIndex("test");
-        ensureGreen();
-        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService test = indicesService.indexService(resolveIndex("test"));
-        final IndexShard shard = test.getShardOrNull(0);
-        assertFalse(shard.shouldFlush());
-        client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder().put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(133/* size of the operation + header&footer*/, ByteSizeUnit.BYTES)).build()).get();
-        client().prepareIndex("test", "test", "0").setSource("{}").setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get();
-        assertFalse(shard.shouldFlush());
-        final AtomicBoolean running = new AtomicBoolean(true);
-        final int numThreads = randomIntBetween(2, 4);
-        Thread[] threads = new Thread[numThreads];
-        CyclicBarrier barrier = new CyclicBarrier(numThreads + 1);
-        for (int i = 0; i < threads.length; i++) {
-            threads[i] = new Thread() {
-                @Override
-                public void run() {
-                    try {
-                        barrier.await();
-                    } catch (InterruptedException | BrokenBarrierException e) {
-                        throw new RuntimeException(e);
-                    }
-                    while (running.get()) {
-                        shard.maybeFlush();
-                    }
-                }
-            };
-            threads[i].start();
-        }
-        barrier.await();
-        FlushStats flushStats = shard.flushStats();
-        long total = flushStats.getTotal();
-        client().prepareIndex("test", "test", "1").setSource("{}").get();
-        assertBusy(() -> {
-            assertEquals(total + 1, shard.flushStats().getTotal());
-        });
-        running.set(false);
-        for (int i = 0; i < threads.length; i++) {
-            threads[i].join();
-        }
-        assertEquals(total + 1, shard.flushStats().getTotal());
+        closeShards(shard);
     }
 
     public void testLockingBeforeAndAfterRelocated() throws Exception {
-        assertAcked(client().admin().indices().prepareCreate("test").setSettings(
-            Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0)
-        ).get());
-        ensureGreen();
-        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService test = indicesService.indexService(resolveIndex("test"));
-        final IndexShard shard = test.getShardOrNull(0);
-        assertBusy(() -> assertThat(shard.state(), equalTo(IndexShardState.STARTED)));
+        final IndexShard shard = newStartedShard(true);
+        shard.updateRoutingEntry(ShardRoutingHelper.relocate(shard.routingEntry(), "other_node"));
         CountDownLatch latch = new CountDownLatch(1);
         Thread recoveryThread = new Thread(() -> {
             latch.countDown();
@@ -1041,17 +735,13 @@ public void testLockingBeforeAndAfterRelocated() throws Exception {
             // lock can again be acquired
             assertThat(shard.state(), equalTo(IndexShardState.RELOCATED));
         }
+
+        closeShards(shard);
     }
 
     public void testDelayedOperationsBeforeAndAfterRelocated() throws Exception {
-        assertAcked(client().admin().indices().prepareCreate("test").setSettings(
-            Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0)
-        ).get());
-        ensureGreen();
-        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService test = indicesService.indexService(resolveIndex("test"));
-        final IndexShard shard = test.getShardOrNull(0);
-        assertBusy(() -> assertThat(shard.state(), equalTo(IndexShardState.STARTED)));
+        final IndexShard shard = newStartedShard(true);
+        shard.updateRoutingEntry(ShardRoutingHelper.relocate(shard.routingEntry(), "other_node"));
         Thread recoveryThread = new Thread(() -> {
             try {
                 shard.relocated("simulated recovery");
@@ -1079,16 +769,13 @@ public void onResponse(Releasable releasable) {
         }
 
         recoveryThread.join();
+
+        closeShards(shard);
     }
 
     public void testStressRelocated() throws Exception {
-        assertAcked(client().admin().indices().prepareCreate("test").setSettings(
-            Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0)
-        ).get());
-        ensureGreen();
-        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService test = indicesService.indexService(resolveIndex("test"));
-        final IndexShard shard = test.getShardOrNull(0);
+        final IndexShard shard = newStartedShard(true);
+        shard.updateRoutingEntry(ShardRoutingHelper.relocate(shard.routingEntry(), "other_node"));
         final int numThreads = randomIntBetween(2, 4);
         Thread[] indexThreads = new Thread[numThreads];
         CountDownLatch allPrimaryOperationLocksAcquired = new CountDownLatch(numThreads);
@@ -1136,85 +823,170 @@ public void run() {
         for (Thread indexThread : indexThreads) {
             indexThread.join();
         }
+
+        closeShards(shard);
+    }
+
+    public void testRelocatedShardCanNotBeRevived() throws IOException, InterruptedException {
+        final IndexShard shard = newStartedShard(true);
+        final ShardRouting originalRouting = shard.routingEntry();
+        shard.updateRoutingEntry(ShardRoutingHelper.relocate(originalRouting, "other_node"));
+        shard.relocated("test");
+        expectThrows(IllegalIndexShardStateException.class, () -> shard.updateRoutingEntry(originalRouting));
+        closeShards(shard);
+    }
+
+    public void testShardCanNotBeMarkedAsRelocatedIfRelocationCancelled() throws IOException, InterruptedException {
+        final IndexShard shard = newStartedShard(true);
+        final ShardRouting originalRouting = shard.routingEntry();
+        shard.updateRoutingEntry(ShardRoutingHelper.relocate(originalRouting, "other_node"));
+        shard.updateRoutingEntry(originalRouting);
+        expectThrows(IllegalIndexShardStateException.class, () ->  shard.relocated("test"));
+        closeShards(shard);
+    }
+
+    public void testRelocatedShardCanNotBeRevivedConcurrently() throws IOException, InterruptedException, BrokenBarrierException {
+        final IndexShard shard = newStartedShard(true);
+        final ShardRouting originalRouting = shard.routingEntry();
+        shard.updateRoutingEntry(ShardRoutingHelper.relocate(originalRouting, "other_node"));
+        CyclicBarrier cyclicBarrier = new CyclicBarrier(3);
+        AtomicReference<Exception> relocationException = new AtomicReference<>();
+        Thread relocationThread = new Thread(new AbstractRunnable() {
+            @Override
+            public void onFailure(Exception e) {
+                relocationException.set(e);
+            }
+
+            @Override
+            protected void doRun() throws Exception {
+                cyclicBarrier.await();
+                shard.relocated("test");
+            }
+        });
+        relocationThread.start();
+        AtomicReference<Exception> cancellingException = new AtomicReference<>();
+        Thread cancellingThread = new Thread(new AbstractRunnable() {
+            @Override
+            public void onFailure(Exception e) {
+                cancellingException.set(e);
+            }
+
+            @Override
+            protected void doRun() throws Exception {
+                cyclicBarrier.await();
+                shard.updateRoutingEntry(originalRouting);
+            }
+        });
+        cancellingThread.start();
+        cyclicBarrier.await();
+        relocationThread.join();
+        cancellingThread.join();
+        if (shard.state() == IndexShardState.RELOCATED) {
+            logger.debug("shard was relocated successfully");
+            assertThat(cancellingException.get(), instanceOf(IllegalIndexShardStateException.class));
+            assertThat("current routing:" + shard.routingEntry(), shard.routingEntry().relocating(), equalTo(true));
+            assertThat(relocationException.get(), nullValue());
+        } else {
+            logger.debug("shard relocation was cancelled");
+            assertThat(relocationException.get(), instanceOf(IllegalIndexShardStateException.class));
+            assertThat("current routing:" + shard.routingEntry(), shard.routingEntry().relocating(), equalTo(false));
+            assertThat(cancellingException.get(), nullValue());
+
+        }
+        closeShards(shard);
     }
 
     public void testRecoverFromStore() throws IOException {
-        createIndex("test");
-        ensureGreen();
-        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService test = indicesService.indexService(resolveIndex("test"));
-        final IndexShard shard = test.getShardOrNull(0);
+        final IndexShard shard = newStartedShard(true);
         int translogOps = 1;
-        client().prepareIndex("test", "test", "0").setSource("{}").setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get();
+        indexDoc(shard, "test", "0");
         if (randomBoolean()) {
-            client().admin().indices().prepareFlush().get();
+            flushShard(shard);
             translogOps = 0;
         }
-        ShardRouting routing = shard.routingEntry();
-        test.removeShard(0, "b/c simon says so");
-        routing = ShardRoutingHelper.reinitPrimary(routing);
-        IndexShard newShard = test.createShard(routing);
-        newShard.updateRoutingEntry(routing);
+        IndexShard newShard = reinitShard(shard);
         DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
-        newShard.markAsRecovering("store", new RecoveryState(routing, localNode, null));
+        newShard.markAsRecovering("store", new RecoveryState(newShard.routingEntry(), localNode, null));
         assertTrue(newShard.recoverFromStore());
         assertEquals(translogOps, newShard.recoveryState().getTranslog().recoveredOperations());
         assertEquals(translogOps, newShard.recoveryState().getTranslog().totalOperations());
         assertEquals(translogOps, newShard.recoveryState().getTranslog().totalOperationsOnStart());
         assertEquals(100.0f, newShard.recoveryState().getTranslog().recoveredPercent(), 0.01f);
-        newShard.updateRoutingEntry(routing.moveToStarted());
-        SearchResponse response = client().prepareSearch().get();
-        assertHitCount(response, 1);
+        newShard.updateRoutingEntry(newShard.routingEntry().moveToStarted());
+        assertDocCount(newShard, 1);
+        closeShards(newShard);
+    }
+
+    public void testRecoverFromStoreWithOutOfOrderDelete() throws IOException {
+        final IndexShard shard = newStartedShard(false);
+        final Engine.Index index = shard.prepareIndexOnReplica(
+            SourceToParse.source(SourceToParse.Origin.REPLICA, shard.shardId().getIndexName(), "type", "id", new BytesArray("{}"),
+                XContentType.JSON), 1, VersionType.EXTERNAL, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false);
+        final Engine.Delete delete = shard.prepareDeleteOnReplica("type", "id", 2, VersionType.EXTERNAL);
+        shard.delete(delete);
+        final int translogOps;
+        if (randomBoolean()) {
+            flushShard(shard, true); // lucene won't flush due to just one pending delete
+            translogOps = 0;
+        } else {
+            translogOps = 1;
+        }
+        final Engine.IndexResult result = shard.index(index);
+        assertThat(result.getTranslogLocation(), nullValue());
+        final ShardRouting replicaRouting = shard.routingEntry();
+        IndexShard newShard = reinitShard(shard,
+            newShardRouting(replicaRouting.shardId(), replicaRouting.currentNodeId(), true, ShardRoutingState.INITIALIZING,
+                RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE));
+        DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
+        newShard.markAsRecovering("store", new RecoveryState(newShard.routingEntry(), localNode, null));
+        assertTrue(newShard.recoverFromStore());
+        assertEquals(translogOps, newShard.recoveryState().getTranslog().recoveredOperations());
+        assertEquals(translogOps, newShard.recoveryState().getTranslog().totalOperations());
+        assertEquals(translogOps, newShard.recoveryState().getTranslog().totalOperationsOnStart());
+        assertEquals(100.0f, newShard.recoveryState().getTranslog().recoveredPercent(), 0.01f);
+        newShard.updateRoutingEntry(newShard.routingEntry().moveToStarted());
+        assertDocCount(newShard, 0);
+        closeShards(newShard);
     }
 
     public void testRecoverFromCleanStore() throws IOException {
-        createIndex("test");
-        ensureGreen();
-        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService test = indicesService.indexService(resolveIndex("test"));
-        final IndexShard shard = test.getShardOrNull(0);
-        client().prepareIndex("test", "test", "0").setSource("{}").setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get();
+        final IndexShard shard = newStartedShard(true);
+        indexDoc(shard, "test", "0");
         if (randomBoolean()) {
-            client().admin().indices().prepareFlush().get();
+            flushShard(shard);
         }
-        ShardRouting routing = shard.routingEntry();
-        test.removeShard(0, "b/c simon says so");
-        routing = ShardRoutingHelper.reinitPrimary(routing, UnassignedInfo.Reason.INDEX_CREATED, StoreRecoverySource.EMPTY_STORE_INSTANCE);
-        IndexShard newShard = test.createShard(routing);
-        newShard.updateRoutingEntry(routing);
+        final ShardRouting shardRouting = shard.routingEntry();
+        IndexShard newShard = reinitShard(shard,
+            ShardRoutingHelper.initWithSameId(shardRouting, RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE)
+        );
+
         DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
-        newShard.markAsRecovering("store", new RecoveryState(routing, localNode, null));
+        newShard.markAsRecovering("store", new RecoveryState(newShard.routingEntry(), localNode, null));
         assertTrue(newShard.recoverFromStore());
         assertEquals(0, newShard.recoveryState().getTranslog().recoveredOperations());
         assertEquals(0, newShard.recoveryState().getTranslog().totalOperations());
         assertEquals(0, newShard.recoveryState().getTranslog().totalOperationsOnStart());
         assertEquals(100.0f, newShard.recoveryState().getTranslog().recoveredPercent(), 0.01f);
-        newShard.updateRoutingEntry(getInitializingShardRouting(routing).moveToStarted());
-        SearchResponse response = client().prepareSearch().get();
-        assertHitCount(response, 0);
+        newShard.updateRoutingEntry(newShard.routingEntry().moveToStarted());
+        assertDocCount(newShard, 0);
+        closeShards(newShard);
     }
 
     public void testFailIfIndexNotPresentInRecoverFromStore() throws Exception {
-        createIndex("test");
-        ensureGreen();
-        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
-        IndexService test = indicesService.indexService(resolveIndex("test"));
-        final IndexShard shard = test.getShardOrNull(0);
-
-        client().prepareIndex("test", "test", "0").setSource("{}").setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get();
+        final IndexShard shard = newStartedShard(true);
+        indexDoc(shard, "test", "0");
         if (randomBoolean()) {
-            client().admin().indices().prepareFlush().get();
+            flushShard(shard);
         }
-        final ShardRouting origRouting = shard.routingEntry();
-        ShardRouting routing = origRouting;
+
         Store store = shard.store();
         store.incRef();
-        test.removeShard(0, "b/c simon says so");
+        closeShards(shard);
         cleanLuceneIndex(store.directory());
         store.decRef();
-        routing = ShardRoutingHelper.reinitPrimary(routing);
-        IndexShard newShard = test.createShard(routing);
+        IndexShard newShard = reinitShard(shard);
+        DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
+        ShardRouting routing = newShard.routingEntry();
         newShard.markAsRecovering("store", new RecoveryState(routing, localNode, null));
         try {
             newShard.recoverFromStore();
@@ -1224,7 +996,7 @@ public void testFailIfIndexNotPresentInRecoverFromStore() throws Exception {
         }
 
         routing = ShardRoutingHelper.moveToUnassigned(routing, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "because I say so"));
-        routing = ShardRoutingHelper.initialize(routing, origRouting.currentNodeId());
+        routing = ShardRoutingHelper.initialize(routing, newShard.routingEntry().currentNodeId());
         assertTrue("it's already recovering, we should ignore new ones", newShard.ignoreRecoveryAttempt());
         try {
             newShard.markAsRecovering("store", new RecoveryState(routing, localNode, null));
@@ -1232,30 +1004,25 @@ public void testFailIfIndexNotPresentInRecoverFromStore() throws Exception {
         } catch (IllegalIndexShardStateException e) {
             // OK!
         }
-        test.removeShard(0, "I broken it");
-        routing = routing.updateUnassigned(routing.unassignedInfo(), StoreRecoverySource.EMPTY_STORE_INSTANCE);
-        newShard = test.createShard(routing);
-        newShard.markAsRecovering("store", new RecoveryState(routing, localNode, null));
+
+        newShard = reinitShard(newShard,
+            ShardRoutingHelper.initWithSameId(routing, RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE));
+        newShard.markAsRecovering("store", new RecoveryState(newShard.routingEntry(), localNode, null));
         assertTrue("recover even if there is nothing to recover", newShard.recoverFromStore());
 
-        newShard.updateRoutingEntry(getInitializingShardRouting(routing).moveToStarted());
-        SearchResponse response = client().prepareSearch().get();
-        assertHitCount(response, 0);
+        newShard.updateRoutingEntry(newShard.routingEntry().moveToStarted());
+        assertDocCount(newShard, 0);
         // we can't issue this request through a client because of the inconsistencies we created with the cluster state
         // doing it directly instead
-        IndexRequest request = client().prepareIndex("test", "test", "0").setSource("{}").request();
-        request.process(null, false, "test");
-        TransportIndexAction.executeIndexRequestOnPrimary(request, newShard, null);
+        indexDoc(newShard, "test", "0");
         newShard.refresh("test");
-        assertHitCount(client().prepareSearch().get(), 1);
+        assertDocCount(newShard, 1);
+
+        closeShards(newShard);
     }
 
     public void testRecoveryFailsAfterMovingToRelocatedState() throws InterruptedException, IOException {
-        createIndex("test");
-        ensureGreen();
-        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService test = indicesService.indexService(resolveIndex("test"));
-        final IndexShard shard = test.getShardOrNull(0);
+        final IndexShard shard = newStartedShard(true);
         ShardRouting origRouting = shard.routingEntry();
         assertThat(shard.state(), equalTo(IndexShardState.STARTED));
         ShardRouting inRecoveryRouting = ShardRoutingHelper.relocate(origRouting, "some_node");
@@ -1267,37 +1034,37 @@ public void testRecoveryFailsAfterMovingToRelocatedState() throws InterruptedExc
             fail("Expected IndexShardRelocatedException");
         } catch (IndexShardRelocatedException expected) {
         }
+
+        closeShards(shard);
     }
 
     public void testRestoreShard() throws IOException {
-        createIndex("test");
-        createIndex("test_target");
-        ensureGreen();
-        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService test = indicesService.indexService(resolveIndex("test"));
-        IndexService test_target = indicesService.indexService(resolveIndex("test_target"));
-        final IndexShard test_shard = test.getShardOrNull(0);
-
-        client().prepareIndex("test", "test", "0").setSource("{}").setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get();
-        client().prepareIndex("test_target", "test", "1").setSource("{}").setRefreshPolicy(IMMEDIATE).get();
-        assertHitCount(client().prepareSearch("test_target").get(), 1);
-        assertSearchHits(client().prepareSearch("test_target").get(), "1");
-        client().admin().indices().prepareFlush("test").get(); // only flush test
-        final ShardRouting origRouting = test_target.getShardOrNull(0).routingEntry();
+        final IndexShard source = newStartedShard(true);
+        IndexShard target = newStartedShard(true);
+
+        indexDoc(source, "test", "0");
+        if (randomBoolean()) {
+            source.refresh("test");
+        }
+        indexDoc(target, "test", "1");
+        target.refresh("test");
+        assertDocs(target, new Uid("test", "1"));
+        flushShard(source); // only flush source
+        final ShardRouting origRouting = target.routingEntry();
         ShardRouting routing = ShardRoutingHelper.reinitPrimary(origRouting);
         final Snapshot snapshot = new Snapshot("foo", new SnapshotId("bar", UUIDs.randomBase64UUID()));
-        routing = ShardRoutingHelper.newWithRestoreSource(routing, new SnapshotRecoverySource(snapshot, Version.CURRENT, "test"));
-        test_target.removeShard(0, "just do it man!");
-        final IndexShard test_target_shard = test_target.createShard(routing);
-        Store sourceStore = test_shard.store();
-        Store targetStore = test_target_shard.store();
+        routing = ShardRoutingHelper.newWithRestoreSource(routing,
+            new RecoverySource.SnapshotRecoverySource(snapshot, Version.CURRENT, "test"));
+        target = reinitShard(target, routing);
+        Store sourceStore = source.store();
+        Store targetStore = target.store();
 
-        test_target_shard.updateRoutingEntry(routing);
         DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
-        test_target_shard.markAsRecovering("store", new RecoveryState(routing, localNode, null));
-        assertTrue(test_target_shard.restoreFromRepository(new RestoreOnlyRepository("test") {
+        target.markAsRecovering("store", new RecoveryState(routing, localNode, null));
+        assertTrue(target.restoreFromRepository(new RestoreOnlyRepository("test") {
             @Override
-            public void restoreShard(IndexShard shard, SnapshotId snapshotId, Version version, IndexId indexId, ShardId snapshotShardId, RecoveryState recoveryState) {
+            public void restoreShard(IndexShard shard, SnapshotId snapshotId, Version version, IndexId indexId, ShardId snapshotShardId,
+                                     RecoveryState recoveryState) {
                 try {
                     cleanLuceneIndex(targetStore.directory());
                     for (String file : sourceStore.directory().listAll()) {
@@ -1312,21 +1079,20 @@ public void restoreShard(IndexShard shard, SnapshotId snapshotId, Version versio
             }
         }));
 
-        test_target_shard.updateRoutingEntry(routing.moveToStarted());
-        assertHitCount(client().prepareSearch("test_target").get(), 1);
-        assertSearchHits(client().prepareSearch("test_target").get(), "0");
+        target.updateRoutingEntry(routing.moveToStarted());
+        assertDocs(target, new Uid("test", "0"));
+
+        closeShards(source, target);
     }
 
     public void testSearcherWrapperIsUsed() throws IOException {
-        createIndex("test");
-        ensureGreen();
-        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService indexService = indicesService.indexService(resolveIndex("test"));
-        IndexShard shard = indexService.getShardOrNull(0);
-        client().prepareIndex("test", "test", "0").setSource("{\"foo\" : \"bar\"}").setRefreshPolicy(IMMEDIATE).get();
-        client().prepareIndex("test", "test", "1").setSource("{\"foobar\" : \"bar\"}").setRefreshPolicy(IMMEDIATE).get();
-
-        Engine.GetResult getResult = shard.get(new Engine.Get(false, new Term(UidFieldMapper.NAME, Uid.createUid("test", "1"))));
+        IndexShard shard = newStartedShard(true);
+        indexDoc(shard, "test", "0", "{\"foo\" : \"bar\"}");
+        indexDoc(shard, "test", "1", "{\"foobar\" : \"bar\"}");
+        shard.refresh("test");
+
+        Engine.GetResult getResult = shard.get(new Engine.Get(
+                false, "test", "1", new Term(UidFieldMapper.NAME, Uid.createUid("test", "1"))));
         assertTrue(getResult.exists());
         assertNotNull(getResult.searcher());
         getResult.release();
@@ -1347,35 +1113,29 @@ public IndexSearcher wrap(IndexSearcher searcher) throws EngineException {
                 return searcher;
             }
         };
-        shard.close("simon says", true);
-        IndexShard newShard = reinitWithWrapper(indexService, shard, wrapper);
-        try {
-            try (Engine.Searcher searcher = newShard.acquireSearcher("test")) {
-                TopDocs search = searcher.searcher().search(new TermQuery(new Term("foo", "bar")), 10);
-                assertEquals(search.totalHits, 0);
-                search = searcher.searcher().search(new TermQuery(new Term("foobar", "bar")), 10);
-                assertEquals(search.totalHits, 1);
-            }
-            getResult = newShard.get(new Engine.Get(false, new Term(UidFieldMapper.NAME, Uid.createUid("test", "1"))));
-            assertTrue(getResult.exists());
-            assertNotNull(getResult.searcher()); // make sure get uses the wrapped reader
-            assertTrue(getResult.searcher().reader() instanceof FieldMaskingReader);
-            getResult.release();
-        } finally {
-            newShard.close("just do it", randomBoolean());
+        closeShards(shard);
+        IndexShard newShard = newShard(ShardRoutingHelper.reinitPrimary(shard.routingEntry()),
+            shard.shardPath(), shard.indexSettings().getIndexMetaData(), wrapper);
+
+        recoveryShardFromStore(newShard);
+
+        try (Engine.Searcher searcher = newShard.acquireSearcher("test")) {
+            TopDocs search = searcher.searcher().search(new TermQuery(new Term("foo", "bar")), 10);
+            assertEquals(search.totalHits, 0);
+            search = searcher.searcher().search(new TermQuery(new Term("foobar", "bar")), 10);
+            assertEquals(search.totalHits, 1);
         }
-    }
+        getResult = newShard.get(new Engine.Get(false, "test", "1",
+                new Term(UidFieldMapper.NAME, Uid.createUid("test", "1"))));
+        assertTrue(getResult.exists());
+        assertNotNull(getResult.searcher()); // make sure get uses the wrapped reader
+        assertTrue(getResult.searcher().reader() instanceof FieldMaskingReader);
+        getResult.release();
 
-    public void testSearcherWrapperWorksWithGlobaOrdinals() throws IOException {
-        createIndex("test");
-        ensureGreen();
-        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService indexService = indicesService.indexService(resolveIndex("test"));
-        IndexShard shard = indexService.getShardOrNull(0);
-        client().admin().indices().preparePutMapping("test").setType("test").setSource("foo", "type=text,fielddata=true").get();
-        client().prepareIndex("test", "test", "0").setSource("{\"foo\" : \"bar\"}").setRefreshPolicy(IMMEDIATE).get();
-        client().prepareIndex("test", "test", "1").setSource("{\"foobar\" : \"bar\"}").setRefreshPolicy(IMMEDIATE).get();
+        closeShards(newShard);
+    }
 
+    public void testSearcherWrapperWorksWithGlobalOrdinals() throws IOException {
         IndexSearcherWrapper wrapper = new IndexSearcherWrapper() {
             @Override
             public DirectoryReader wrap(DirectoryReader reader) throws IOException {
@@ -1388,338 +1148,252 @@ public IndexSearcher wrap(IndexSearcher searcher) throws EngineException {
             }
         };
 
-        shard.close("simon says", true);
-        IndexShard newShard = reinitWithWrapper(indexService, shard, wrapper);
-        try {
-            // test global ordinals are evicted
-            MappedFieldType foo = newShard.mapperService().fullName("foo");
-            IndexFieldData.Global ifd = shard.indexFieldDataService().getForField(foo);
-            FieldDataStats before = shard.fieldData().stats("foo");
-            assertThat(before.getMemorySizeInBytes(), equalTo(0L));
-            FieldDataStats after = null;
-            try (Engine.Searcher searcher = newShard.acquireSearcher("test")) {
-                assumeTrue("we have to have more than one segment", searcher.getDirectoryReader().leaves().size() > 1);
-                ifd.loadGlobal(searcher.getDirectoryReader());
-                after = shard.fieldData().stats("foo");
-                assertEquals(after.getEvictions(), before.getEvictions());
-                // If a field doesn't exist an empty IndexFieldData is returned and that isn't cached:
-                assertThat(after.getMemorySizeInBytes(), equalTo(0L));
-            }
-            assertEquals(shard.fieldData().stats("foo").getEvictions(), before.getEvictions());
-            assertEquals(shard.fieldData().stats("foo").getMemorySizeInBytes(), after.getMemorySizeInBytes());
-            newShard.flush(new FlushRequest().force(true).waitIfOngoing(true));
-            newShard.refresh("test");
-            assertEquals(shard.fieldData().stats("foo").getMemorySizeInBytes(), before.getMemorySizeInBytes());
-            assertEquals(shard.fieldData().stats("foo").getEvictions(), before.getEvictions());
-        } finally {
-            newShard.close("just do it", randomBoolean());
+        Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+            .build();
+        IndexMetaData metaData = IndexMetaData.builder("test")
+            .putMapping("test", "{ \"properties\": { \"foo\":  { \"type\": \"text\", \"fielddata\": true }}}")
+            .settings(settings)
+            .primaryTerm(0, 1).build();
+        IndexShard shard = newShard(new ShardId(metaData.getIndex(), 0), true, "n1", metaData, wrapper);
+        recoveryShardFromStore(shard);
+        indexDoc(shard, "test", "0", "{\"foo\" : \"bar\"}");
+        shard.refresh("created segment 1");
+        indexDoc(shard, "test", "1", "{\"foobar\" : \"bar\"}");
+        shard.refresh("created segment 2");
+
+        // test global ordinals are evicted
+        MappedFieldType foo = shard.mapperService().fullName("foo");
+        IndexFieldData.Global ifd = shard.indexFieldDataService().getForField(foo);
+        FieldDataStats before = shard.fieldData().stats("foo");
+        assertThat(before.getMemorySizeInBytes(), equalTo(0L));
+        FieldDataStats after = null;
+        try (Engine.Searcher searcher = shard.acquireSearcher("test")) {
+            assertThat("we have to have more than one segment", searcher.getDirectoryReader().leaves().size(), greaterThan(1));
+            ifd.loadGlobal(searcher.getDirectoryReader());
+            after = shard.fieldData().stats("foo");
+            assertEquals(after.getEvictions(), before.getEvictions());
+            // If a field doesn't exist an empty IndexFieldData is returned and that isn't cached:
+            assertThat(after.getMemorySizeInBytes(), equalTo(0L));
         }
+        assertEquals(shard.fieldData().stats("foo").getEvictions(), before.getEvictions());
+        assertEquals(shard.fieldData().stats("foo").getMemorySizeInBytes(), after.getMemorySizeInBytes());
+        shard.flush(new FlushRequest().force(true).waitIfOngoing(true));
+        shard.refresh("test");
+        assertEquals(shard.fieldData().stats("foo").getMemorySizeInBytes(), before.getMemorySizeInBytes());
+        assertEquals(shard.fieldData().stats("foo").getEvictions(), before.getEvictions());
+
+        closeShards(shard);
     }
 
-    public void testIndexingOperationListnenersIsInvokedOnRecovery() throws IOException {
-        createIndex("test");
-        ensureGreen();
-        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService indexService = indicesService.indexService(resolveIndex("test"));
-        IndexShard shard = indexService.getShardOrNull(0);
-        client().prepareIndex("test", "test", "0").setSource("{\"foo\" : \"bar\"}").get();
-        client().prepareDelete("test", "test", "0").get();
-        client().prepareIndex("test", "test", "1").setSource("{\"foo\" : \"bar\"}").setRefreshPolicy(IMMEDIATE).get();
-
-        IndexSearcherWrapper wrapper = new IndexSearcherWrapper() {};
-        shard.close("simon says", false);
+    public void testIndexingOperationListenersIsInvokedOnRecovery() throws IOException {
+        IndexShard shard = newStartedShard(true);
+        indexDoc(shard, "test", "0", "{\"foo\" : \"bar\"}");
+        deleteDoc(shard, "test", "0");
+        indexDoc(shard, "test", "1", "{\"foo\" : \"bar\"}");
+        shard.refresh("test");
+
         final AtomicInteger preIndex = new AtomicInteger();
         final AtomicInteger postIndex = new AtomicInteger();
         final AtomicInteger preDelete = new AtomicInteger();
         final AtomicInteger postDelete = new AtomicInteger();
         IndexingOperationListener listener = new IndexingOperationListener() {
             @Override
-            public Engine.Index preIndex(Engine.Index operation) {
+            public Engine.Index preIndex(ShardId shardId, Engine.Index index) {
                 preIndex.incrementAndGet();
-                return operation;
+                return index;
             }
 
             @Override
-            public void postIndex(Engine.Index index, boolean created) {
+            public void postIndex(ShardId shardId, Engine.Index index, Engine.IndexResult result) {
                 postIndex.incrementAndGet();
             }
 
             @Override
-            public Engine.Delete preDelete(Engine.Delete delete) {
+            public Engine.Delete preDelete(ShardId shardId, Engine.Delete delete) {
                 preDelete.incrementAndGet();
                 return delete;
             }
 
             @Override
-            public void postDelete(Engine.Delete delete) {
+            public void postDelete(ShardId shardId, Engine.Delete delete, Engine.DeleteResult result) {
                 postDelete.incrementAndGet();
-
-            }
-        };
-        final IndexShard newShard = reinitWithWrapper(indexService, shard, wrapper, listener);
-        try {
-            IndexingStats indexingStats = newShard.indexingStats();
-            // ensure we are not influencing the indexing stats
-            assertEquals(0, indexingStats.getTotal().getDeleteCount());
-            assertEquals(0, indexingStats.getTotal().getDeleteCurrent());
-            assertEquals(0, indexingStats.getTotal().getIndexCount());
-            assertEquals(0, indexingStats.getTotal().getIndexCurrent());
-            assertEquals(0, indexingStats.getTotal().getIndexFailedCount());
-            assertEquals(2, preIndex.get());
-            assertEquals(2, postIndex.get());
-            assertEquals(1, preDelete.get());
-            assertEquals(1, postDelete.get());
-        } finally {
-            newShard.close("just do it", randomBoolean());
-        }
-    }
-
-    public void testShardHasMemoryBufferOnTranslogRecover() throws Throwable {
-        createIndex("test");
-        ensureGreen();
-        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService indexService = indicesService.indexService(resolveIndex("test"));
-        IndexShard shard = indexService.getShardOrNull(0);
-        client().prepareIndex("test", "test", "0").setSource("{\"foo\" : \"bar\"}").get();
-        client().prepareDelete("test", "test", "0").get();
-        client().prepareIndex("test", "test", "1").setSource("{\"foo\" : \"bar\"}").setRefreshPolicy(IMMEDIATE).get();
-
-        IndexSearcherWrapper wrapper = new IndexSearcherWrapper() {};
-        shard.close("simon says", false);
-        AtomicReference<IndexShard> shardRef = new AtomicReference<>();
-        List<Exception> failures = new ArrayList<>();
-        IndexingOperationListener listener = new IndexingOperationListener() {
-
-            @Override
-            public void postIndex(Engine.Index index, boolean created) {
-                try {
-                    assertNotNull(shardRef.get());
-                    // this is all IMC needs to do - check current memory and refresh
-                    assertTrue(shardRef.get().getIndexBufferRAMBytesUsed() > 0);
-                    shardRef.get().refresh("test");
-                } catch (Exception e) {
-                    failures.add(e);
-                    throw e;
-                }
-            }
-
-
-            @Override
-            public void postDelete(Engine.Delete delete) {
-                try {
-                    assertNotNull(shardRef.get());
-                    // this is all IMC needs to do - check current memory and refresh
-                    assertTrue(shardRef.get().getIndexBufferRAMBytesUsed() > 0);
-                    shardRef.get().refresh("test");
-                } catch (Exception e) {
-                    failures.add(e);
-                    throw e;
-                }
             }
         };
-        final IndexShard newShard = newIndexShard(indexService, shard, wrapper, listener);
-        shardRef.set(newShard);
-        recoverShard(newShard);
+        final IndexShard newShard = reinitShard(shard, listener);
+        recoveryShardFromStore(newShard);
+        IndexingStats indexingStats = newShard.indexingStats();
+        // ensure we are not influencing the indexing stats
+        assertEquals(0, indexingStats.getTotal().getDeleteCount());
+        assertEquals(0, indexingStats.getTotal().getDeleteCurrent());
+        assertEquals(0, indexingStats.getTotal().getIndexCount());
+        assertEquals(0, indexingStats.getTotal().getIndexCurrent());
+        assertEquals(0, indexingStats.getTotal().getIndexFailedCount());
+        assertEquals(2, preIndex.get());
+        assertEquals(2, postIndex.get());
+        assertEquals(1, preDelete.get());
+        assertEquals(1, postDelete.get());
 
-        try {
-            ExceptionsHelper.rethrowAndSuppress(failures);
-        } finally {
-            newShard.close("just do it", randomBoolean());
-        }
+        closeShards(newShard);
     }
 
     public void testSearchIsReleaseIfWrapperFails() throws IOException {
-        createIndex("test");
-        ensureGreen();
-        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService indexService = indicesService.indexService(resolveIndex("test"));
-        IndexShard shard = indexService.getShardOrNull(0);
-        client().prepareIndex("test", "test", "0").setSource("{\"foo\" : \"bar\"}").setRefreshPolicy(IMMEDIATE).get();
+        IndexShard shard = newStartedShard(true);
+        indexDoc(shard, "test", "0", "{\"foo\" : \"bar\"}");
+        shard.refresh("test");
         IndexSearcherWrapper wrapper = new IndexSearcherWrapper() {
             @Override
             public DirectoryReader wrap(DirectoryReader reader) throws IOException {
                 throw new RuntimeException("boom");
             }
 
+            @Override
             public IndexSearcher wrap(IndexSearcher searcher) throws EngineException {
                 return searcher;
             }
         };
 
-        shard.close("simon says", true);
-        IndexShard newShard = reinitWithWrapper(indexService, shard, wrapper);
+        closeShards(shard);
+        IndexShard newShard = newShard(ShardRoutingHelper.reinitPrimary(shard.routingEntry()),
+            shard.shardPath(), shard.indexSettings().getIndexMetaData(), wrapper);
+
+        recoveryShardFromStore(newShard);
+
         try {
             newShard.acquireSearcher("test");
             fail("exception expected");
         } catch (RuntimeException ex) {
             //
-        } finally {
-            newShard.close("just do it", randomBoolean());
         }
-    }
-
-    public static final IndexShard reinitWithWrapper(IndexService indexService, IndexShard shard, IndexSearcherWrapper wrapper, IndexingOperationListener... listeners) throws IOException {
-        IndexShard newShard = newIndexShard(indexService, shard, wrapper, listeners);
-        return recoverShard(newShard);
-    }
-
-    public  static final IndexShard recoverShard(IndexShard newShard) throws IOException {
-        DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
-        newShard.markAsRecovering("store", new RecoveryState(newShard.routingEntry(), localNode, null));
-        assertTrue(newShard.recoverFromStore());
-        newShard.updateRoutingEntry(newShard.routingEntry().moveToStarted());
-        return newShard;
-    }
-
-    public  static final IndexShard newIndexShard(IndexService indexService, IndexShard shard, IndexSearcherWrapper wrapper, IndexingOperationListener... listeners) throws IOException {
-        ShardRouting initializingShardRouting = getInitializingShardRouting(shard.routingEntry());
-        IndexShard newShard = new IndexShard(initializingShardRouting, indexService.getIndexSettings(), shard.shardPath(),
-            shard.store(), indexService.cache(), indexService.mapperService(), indexService.similarityService(),
-            indexService.fieldData(), shard.getEngineFactory(), indexService.getIndexEventListener(), wrapper,
-            indexService.getThreadPool(), indexService.getBigArrays(), null, Collections.emptyList(), Arrays.asList(listeners));
-        return newShard;
-    }
-
-    private static ShardRouting getInitializingShardRouting(ShardRouting existingShardRouting) {
-        ShardRouting shardRouting = TestShardRouting.newShardRouting(existingShardRouting.shardId(),
-            existingShardRouting.currentNodeId(), null, existingShardRouting.primary(), ShardRoutingState.INITIALIZING,
-            existingShardRouting.allocationId());
-        shardRouting = shardRouting.updateUnassigned(new UnassignedInfo(UnassignedInfo.Reason.INDEX_REOPENED, "fake recovery"),
-            StoreRecoverySource.EXISTING_STORE_INSTANCE);
-        return shardRouting;
+        closeShards(newShard);
     }
 
     public void testTranslogRecoverySyncsTranslog() throws IOException {
-        createIndex("testindexfortranslogsync");
-        client().admin().indices().preparePutMapping("testindexfortranslogsync").setType("testtype").setSource(jsonBuilder().startObject()
-            .startObject("testtype")
-            .startObject("properties")
-            .startObject("foo")
-            .field("type", "text")
-            .endObject()
-            .endObject().endObject().endObject()).get();
-        ensureGreen();
-        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService test = indicesService.indexService(resolveIndex("testindexfortranslogsync"));
-        IndexShard shard = test.getShardOrNull(0);
-        ShardRouting routing = getInitializingShardRouting(shard.routingEntry());
-        test.removeShard(0, "b/c britta says so");
-        IndexShard newShard = test.createShard(routing);
-        DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
-        newShard.markAsRecovering("for testing", new RecoveryState(routing, localNode, null));
-        List<Translog.Operation> operations = new ArrayList<>();
-        operations.add(new Translog.Index("testtype", "1", BytesReference.toBytes(jsonBuilder().startObject().field("foo", "bar").endObject().bytes())));
-        newShard.prepareForIndexRecovery();
-        newShard.recoveryState().getTranslog().totalOperations(operations.size());
-        newShard.skipTranslogRecovery(IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP);
-        newShard.performBatchRecovery(operations);
-        assertFalse(newShard.getTranslog().syncNeeded());
+        Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
+            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+            .build();
+        IndexMetaData metaData = IndexMetaData.builder("test")
+            .putMapping("test", "{ \"properties\": { \"foo\":  { \"type\": \"text\"}}}")
+            .settings(settings)
+            .primaryTerm(0, 1).build();
+        IndexShard primary = newShard(new ShardId(metaData.getIndex(), 0), true, "n1", metaData, null);
+        recoveryShardFromStore(primary);
+
+        indexDoc(primary, "test", "0", "{\"foo\" : \"bar\"}");
+        IndexShard replica = newShard(primary.shardId(), false, "n2", metaData, null);
+        recoverReplica(replica, primary, (shard, discoveryNode) ->
+            new RecoveryTarget(shard, discoveryNode, recoveryListener, aLong -> {
+            }) {
+                @Override
+                public void indexTranslogOperations(List<Translog.Operation> operations, int totalTranslogOps) {
+                    super.indexTranslogOperations(operations, totalTranslogOps);
+                    assertFalse(replica.getTranslog().syncNeeded());
+                }
+            }, true);
+
+        closeShards(primary, replica);
     }
 
-    public void testIndexingBufferDuringInternalRecovery() throws IOException {
-        createIndex("index");
-        client().admin().indices().preparePutMapping("index").setType("testtype").setSource(jsonBuilder().startObject()
-            .startObject("testtype")
-            .startObject("properties")
-            .startObject("foo")
-            .field("type", "text")
-            .endObject()
-            .endObject().endObject().endObject()).get();
-        ensureGreen();
-        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService test = indicesService.indexService(resolveIndex("index"));
-        IndexShard shard = test.getShardOrNull(0);
-        ShardRouting routing = getInitializingShardRouting(shard.routingEntry());
-        test.removeShard(0, "b/c britta says so");
-        IndexShard newShard = test.createShard(routing);
-        newShard.shardRouting = routing;
+    public void testShardActiveDuringInternalRecovery() throws IOException {
+        IndexShard shard = newStartedShard(true);
+        indexDoc(shard, "type", "0");
+        shard = reinitShard(shard);
         DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
-        newShard.markAsRecovering("for testing", new RecoveryState(routing, localNode, null));
+        shard.markAsRecovering("for testing", new RecoveryState(shard.routingEntry(), localNode, null));
         // Shard is still inactive since we haven't started recovering yet
-        assertFalse(newShard.isActive());
-        newShard.prepareForIndexRecovery();
+        assertFalse(shard.isActive());
+        shard.prepareForIndexRecovery();
         // Shard is still inactive since we haven't started recovering yet
-        assertFalse(newShard.isActive());
-        newShard.performTranslogRecovery(true);
+        assertFalse(shard.isActive());
+        shard.performTranslogRecovery(true);
         // Shard should now be active since we did recover:
-        assertTrue(newShard.isActive());
+        assertTrue(shard.isActive());
+        closeShards(shard);
     }
 
-    public void testIndexingBufferDuringPeerRecovery() throws IOException {
-        createIndex("index");
-        client().admin().indices().preparePutMapping("index").setType("testtype").setSource(jsonBuilder().startObject()
-            .startObject("testtype")
-            .startObject("properties")
-            .startObject("foo")
-            .field("type", "text")
-            .endObject()
-            .endObject().endObject().endObject()).get();
-        ensureGreen();
-        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService test = indicesService.indexService(resolveIndex("index"));
-        IndexShard shard = test.getShardOrNull(0);
-        ShardRouting routing = getInitializingShardRouting(shard.routingEntry());
-        test.removeShard(0, "b/c britta says so");
-        IndexShard newShard = test.createShard(routing);
+    public void testShardActiveDuringPeerRecovery() throws IOException {
+        Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
+            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+            .build();
+        IndexMetaData metaData = IndexMetaData.builder("test")
+            .putMapping("test", "{ \"properties\": { \"foo\":  { \"type\": \"text\"}}}")
+            .settings(settings)
+            .primaryTerm(0, 1).build();
+        IndexShard primary = newShard(new ShardId(metaData.getIndex(), 0), true, "n1", metaData, null);
+        recoveryShardFromStore(primary);
+
+        indexDoc(primary, "test", "0", "{\"foo\" : \"bar\"}");
+        IndexShard replica = newShard(primary.shardId(), false, "n2", metaData, null);
         DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
-        newShard.markAsRecovering("for testing", new RecoveryState(routing, localNode, null));
+        replica.markAsRecovering("for testing", new RecoveryState(replica.routingEntry(), localNode, localNode));
         // Shard is still inactive since we haven't started recovering yet
-        assertFalse(newShard.isActive());
-        List<Translog.Operation> operations = new ArrayList<>();
-        operations.add(new Translog.Index("testtype", "1", BytesReference.toBytes(jsonBuilder().startObject().field("foo", "bar").endObject().bytes())));
-        newShard.prepareForIndexRecovery();
-        newShard.skipTranslogRecovery(IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP);
-        // Shard is still inactive since we haven't started recovering yet
-        assertFalse(newShard.isActive());
-        newShard.performBatchRecovery(operations);
-        // Shard should now be active since we did recover:
-        assertTrue(newShard.isActive());
+        assertFalse(replica.isActive());
+        recoverReplica(replica, primary, (shard, discoveryNode) ->
+            new RecoveryTarget(shard, discoveryNode, recoveryListener, aLong -> {
+            }) {
+                @Override
+                public void prepareForTranslogOperations(int totalTranslogOps, long maxUnsafeAutoIdTimestamp) throws IOException {
+                    super.prepareForTranslogOperations(totalTranslogOps, maxUnsafeAutoIdTimestamp);
+                    // Shard is still inactive since we haven't started recovering yet
+                    assertFalse(replica.isActive());
+
+                }
+
+                @Override
+                public void indexTranslogOperations(List<Translog.Operation> operations, int totalTranslogOps) {
+                    super.indexTranslogOperations(operations, totalTranslogOps);
+                    // Shard should now be active since we did recover:
+                    assertTrue(replica.isActive());
+                }
+            }, false);
+
+        closeShards(primary, replica);
     }
 
     public void testRecoverFromLocalShard() throws IOException {
-        createIndex("index");
-        createIndex("index_1");
-        createIndex("index_2");
-        client().admin().indices().preparePutMapping("index").setType("test").setSource(jsonBuilder().startObject()
-            .startObject("test")
-            .startObject("properties")
-            .startObject("foo")
-            .field("type", "text")
-            .endObject()
-            .endObject().endObject().endObject()).get();
-        client().prepareIndex("index", "test", "0").setSource("{\"foo\" : \"bar\"}").setRefreshPolicy(IMMEDIATE).get();
-        client().prepareIndex("index", "test", "1").setSource("{\"foo\" : \"bar\"}").setRefreshPolicy(IMMEDIATE).get();
-
-
-        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService test = indicesService.indexService(resolveIndex("index_1"));
-        IndexShard shard = test.getShardOrNull(0);
-        ShardRouting routing = ShardRoutingHelper.initWithSameId(shard.routingEntry(), LocalShardsRecoverySource.INSTANCE);
-        test.removeShard(0, "b/c simon says so");
+        Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
+            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+            .build();
+        IndexMetaData metaData = IndexMetaData.builder("source")
+            .putMapping("test", "{ \"properties\": { \"foo\":  { \"type\": \"text\"}}}")
+            .settings(settings)
+            .primaryTerm(0, 1).build();
+
+        IndexShard sourceShard = newShard(new ShardId(metaData.getIndex(), 0), true, "n1", metaData, null);
+        recoveryShardFromStore(sourceShard);
+
+        indexDoc(sourceShard, "test", "0", "{\"foo\" : \"bar\"}");
+        indexDoc(sourceShard, "test", "1", "{\"foo\" : \"bar\"}");
+        sourceShard.refresh("test");
+
+
+        ShardRouting targetRouting = newShardRouting(new ShardId("index_1", "index_1", 0), "n1", true,
+            ShardRoutingState.INITIALIZING, RecoverySource.LocalShardsRecoverySource.INSTANCE);
+
+        final IndexShard targetShard;
         DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
+        Map<String, MappingMetaData> requestedMappingUpdates = ConcurrentCollections.newConcurrentMap();
         {
-            final IndexShard newShard = test.createShard(routing);
-            newShard.markAsRecovering("store", new RecoveryState(routing, localNode, null));
+            targetShard = newShard(targetRouting);
+            targetShard.markAsRecovering("store", new RecoveryState(targetShard.routingEntry(), localNode, null));
 
             BiConsumer<String, MappingMetaData> mappingConsumer = (type, mapping) -> {
-                try {
-                    client().admin().indices().preparePutMapping().setConcreteIndex(newShard.indexSettings().getIndex())
-                        .setType(type)
-                        .setSource(mapping.source().string())
-                        .get();
-                } catch (IOException ex) {
-                    throw new ElasticsearchException("failed to stringify mapping source", ex);
-                }
+                assertNull(requestedMappingUpdates.put(type, mapping));
             };
+
+            final IndexShard differentIndex = newShard(new ShardId("index_2", "index_2", 0), true);
+            recoveryShardFromStore(differentIndex);
             expectThrows(IllegalArgumentException.class, () -> {
-                IndexService index = indicesService.indexService(resolveIndex("index"));
-                IndexService index_2 = indicesService.indexService(resolveIndex("index_2"));
-                newShard.recoverFromLocalShards(mappingConsumer, Arrays.asList(index.getShard(0), index_2.getShard(0)));
+                targetShard.recoverFromLocalShards(mappingConsumer, Arrays.asList(sourceShard, differentIndex));
             });
+            closeShards(differentIndex);
 
-            IndexService indexService = indicesService.indexService(resolveIndex("index"));
-            assertTrue(newShard.recoverFromLocalShards(mappingConsumer, Arrays.asList(indexService.getShard(0))));
-            RecoveryState recoveryState = newShard.recoveryState();
+            assertTrue(targetShard.recoverFromLocalShards(mappingConsumer, Arrays.asList(sourceShard)));
+            RecoveryState recoveryState = targetShard.recoveryState();
             assertEquals(RecoveryState.Stage.DONE, recoveryState.getStage());
             assertTrue(recoveryState.getIndex().fileDetails().size() > 0);
             for (RecoveryState.File file : recoveryState.getIndex().fileDetails()) {
@@ -1729,99 +1403,246 @@ public void testRecoverFromLocalShard() throws IOException {
                     assertEquals(file.recovered(), file.length());
                 }
             }
-            routing = ShardRoutingHelper.moveToStarted(routing);
-            newShard.updateRoutingEntry(routing);
-            assertHitCount(client().prepareSearch("index_1").get(), 2);
+            targetShard.updateRoutingEntry(ShardRoutingHelper.moveToStarted(targetShard.routingEntry()));
+            assertDocCount(targetShard, 2);
         }
         // now check that it's persistent ie. that the added shards are committed
         {
-            routing = shard.routingEntry();
-            test.removeShard(0, "b/c simon says so");
-            routing = ShardRoutingHelper.reinitPrimary(routing);
-            final IndexShard newShard = test.createShard(routing);
-            newShard.markAsRecovering("store", new RecoveryState(routing, localNode, null));
-            assertTrue(newShard.recoverFromStore());
-            routing = ShardRoutingHelper.moveToStarted(routing);
-            newShard.updateRoutingEntry(routing);
-            assertHitCount(client().prepareSearch("index_1").get(), 2);
+            final IndexShard newShard = reinitShard(targetShard);
+            recoveryShardFromStore(newShard);
+            assertDocCount(newShard, 2);
+            closeShards(newShard);
         }
 
-        GetMappingsResponse mappingsResponse = client().admin().indices().prepareGetMappings("index_1").get();
-        ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings = mappingsResponse.getMappings();
-        assertNotNull(mappings.get("index_1"));
-        assertNotNull(mappings.get("index_1").get("test"));
-        assertEquals(mappings.get("index_1").get("test").get().source().string(), "{\"test\":{\"properties\":{\"foo\":{\"type\":\"text\"}}}}");
+        assertThat(requestedMappingUpdates, hasKey("test"));
+        assertThat(requestedMappingUpdates.get("test").get().source().string(), equalTo("{\"properties\":{\"foo\":{\"type\":\"text\"}}}"));
+
+        closeShards(sourceShard, targetShard);
+    }
+
+    public void testDocStats() throws IOException {
+        IndexShard indexShard = null;
+        try {
+            indexShard = newStartedShard();
+            final long numDocs = randomIntBetween(2, 32); // at least two documents so we have docs to delete
+            // Delete at least numDocs/10 documents otherwise the number of deleted docs will be below 10%
+            // and forceMerge will refuse to expunge deletes
+            final long numDocsToDelete = randomIntBetween((int) Math.ceil(Math.nextUp(numDocs / 10.0)), Math.toIntExact(numDocs));
+            for (int i = 0; i < numDocs; i++) {
+                final String id = Integer.toString(i);
+                final ParsedDocument doc =
+                    testParsedDocument(id, "test", null, -1, -1, new ParseContext.Document(), new BytesArray(new byte[]{}), null);
+                final Engine.Index index =
+                    new Engine.Index(
+                        new Term("_uid", Uid.createUid(doc.type(), doc.id())),
+                        doc,
+                        Versions.MATCH_ANY,
+                        VersionType.INTERNAL,
+                        PRIMARY,
+                        System.nanoTime(),
+                        -1,
+                        false);
+                Engine.IndexResult indexResult = indexShard.index(index);
+                assertThat(indexResult.getVersion(), equalTo(1L));
+            }
+
+            indexShard.refresh("test");
+            {
+                final DocsStats docsStats = indexShard.docStats();
+                assertThat(docsStats.getCount(), equalTo(numDocs));
+                assertThat(docsStats.getDeleted(), equalTo(0L));
+            }
+
+            final List<Integer> ids = randomSubsetOf(
+                Math.toIntExact(numDocsToDelete),
+                IntStream.range(0, Math.toIntExact(numDocs)).boxed().collect(Collectors.toList()));
+            for (final Integer i : ids) {
+                final String id = Integer.toString(i);
+                final ParsedDocument doc =
+                    testParsedDocument(id, "test", null, -1, -1, new ParseContext.Document(), new BytesArray(new byte[]{}), null);
+                final Engine.Index index =
+                    new Engine.Index(
+                        new Term("_uid", Uid.createUid(doc.type(), doc.id())),
+                        doc,
+                        Versions.MATCH_ANY,
+                        VersionType.INTERNAL,
+                        PRIMARY,
+                        System.nanoTime(),
+                        -1,
+                        false);
+                Engine.IndexResult indexResult = indexShard.index(index);
+                assertThat(indexResult.getVersion(), equalTo(2L));
+            }
+
+            // flush the buffered deletes
+            final FlushRequest flushRequest = new FlushRequest();
+            flushRequest.force(false);
+            flushRequest.waitIfOngoing(false);
+            indexShard.flush(flushRequest);
+
+            indexShard.refresh("test");
+            {
+                final DocsStats docStats = indexShard.docStats();
+                assertThat(docStats.getCount(), equalTo(numDocs));
+                // Lucene will delete a segment if all docs are deleted from it; this means that we lose the deletes when deleting all docs
+                assertThat(docStats.getDeleted(), equalTo(numDocsToDelete == numDocs ? 0 : numDocsToDelete));
+            }
+
+            // merge them away
+            final ForceMergeRequest forceMergeRequest = new ForceMergeRequest();
+            forceMergeRequest.onlyExpungeDeletes(randomBoolean());
+            forceMergeRequest.maxNumSegments(1);
+            indexShard.forceMerge(forceMergeRequest);
+
+            indexShard.refresh("test");
+            {
+                final DocsStats docStats = indexShard.docStats();
+                assertThat(docStats.getCount(), equalTo(numDocs));
+                assertThat(docStats.getDeleted(), equalTo(0L));
+            }
+        } finally {
+            closeShards(indexShard);
+        }
+    }
+
+    /**
+     * here we are simulating the scenario that happens when we do async shard fetching from GatewaySerivce while we are finishing
+     * a recovery and concurrently clean files. This should always be possible without any exception. Yet there was a bug where IndexShard
+     * acquired the index writer lock before it called into the store that has it's own locking for metadata reads
+     */
+    public void testReadSnapshotConcurrently() throws IOException, InterruptedException {
+        IndexShard indexShard = newStartedShard();
+        indexDoc(indexShard, "doc", "0", "{\"foo\" : \"bar\"}");
+        if (randomBoolean()) {
+            indexShard.refresh("test");
+        }
+        indexDoc(indexShard, "doc", "1", "{\"foo\" : \"bar\"}");
+        indexShard.flush(new FlushRequest());
+        closeShards(indexShard);
+
+        final IndexShard newShard = reinitShard(indexShard);
+        Store.MetadataSnapshot storeFileMetaDatas = newShard.snapshotStoreMetadata();
+        assertTrue("at least 2 files, commit and data: " +storeFileMetaDatas.toString(), storeFileMetaDatas.size() > 1);
+        AtomicBoolean stop = new AtomicBoolean(false);
+        CountDownLatch latch = new CountDownLatch(1);
+        expectThrows(AlreadyClosedException.class, () -> newShard.getEngine()); // no engine
+        Thread thread = new Thread(() -> {
+            latch.countDown();
+            while(stop.get() == false){
+                try {
+                    Store.MetadataSnapshot readMeta = newShard.snapshotStoreMetadata();
+                    assertEquals(0, storeFileMetaDatas.recoveryDiff(readMeta).different.size());
+                    assertEquals(0, storeFileMetaDatas.recoveryDiff(readMeta).missing.size());
+                    assertEquals(storeFileMetaDatas.size(), storeFileMetaDatas.recoveryDiff(readMeta).identical.size());
+                } catch (IOException e) {
+                    throw new AssertionError(e);
+                }
+            }
+        });
+        thread.start();
+        latch.await();
 
+        int iters = iterations(10, 100);
+        for (int i = 0; i < iters; i++) {
+            newShard.store().cleanupAndVerify("test", storeFileMetaDatas);
+        }
+        assertTrue(stop.compareAndSet(false, true));
+        thread.join();
+        closeShards(newShard);
     }
 
     /** A dummy repository for testing which just needs restore overridden */
     private abstract static class RestoreOnlyRepository extends AbstractLifecycleComponent implements Repository {
         private final String indexName;
-        public RestoreOnlyRepository(String indexName) {
+
+        RestoreOnlyRepository(String indexName) {
             super(Settings.EMPTY);
             this.indexName = indexName;
         }
+
         @Override
-        protected void doStart() {}
+        protected void doStart() {
+        }
+
         @Override
-        protected void doStop() {}
+        protected void doStop() {
+        }
+
         @Override
-        protected void doClose() {}
+        protected void doClose() {
+        }
+
         @Override
         public RepositoryMetaData getMetadata() {
             return null;
         }
+
         @Override
         public SnapshotInfo getSnapshotInfo(SnapshotId snapshotId) {
             return null;
         }
+
         @Override
         public MetaData getSnapshotMetaData(SnapshotInfo snapshot, List<IndexId> indices) throws IOException {
             return null;
         }
+
         @Override
         public RepositoryData getRepositoryData() {
             Map<IndexId, Set<SnapshotId>> map = new HashMap<>();
-            map.put(new IndexId(indexName, "blah"), Collections.emptySet());
-            return new RepositoryData(Collections.emptyList(), map);
+            map.put(new IndexId(indexName, "blah"), emptySet());
+            return new RepositoryData(EMPTY_REPO_GEN, Collections.emptyMap(), Collections.emptyMap(), map, Collections.emptyList());
         }
+
         @Override
-        public void initializeSnapshot(SnapshotId snapshotId, List<IndexId> indices, MetaData metaData) {}
+        public void initializeSnapshot(SnapshotId snapshotId, List<IndexId> indices, MetaData metaData) {
+        }
+
         @Override
-        public SnapshotInfo finalizeSnapshot(SnapshotId snapshotId, List<IndexId> indices, long startTime, String failure, int totalShards, List<SnapshotShardFailure> shardFailures) {
+        public SnapshotInfo finalizeSnapshot(SnapshotId snapshotId, List<IndexId> indices, long startTime, String failure, int totalShards,
+                                             List<SnapshotShardFailure> shardFailures, long repositoryStateId) {
             return null;
         }
+
         @Override
-        public void deleteSnapshot(SnapshotId snapshotId) {}
+        public void deleteSnapshot(SnapshotId snapshotId, long repositoryStateId) {
+        }
+
         @Override
         public long getSnapshotThrottleTimeInNanos() {
             return 0;
         }
+
         @Override
         public long getRestoreThrottleTimeInNanos() {
             return 0;
         }
+
         @Override
         public String startVerification() {
             return null;
         }
+
         @Override
-        public void endVerification(String verificationToken) {}
+        public void endVerification(String verificationToken) {
+        }
+
         @Override
         public boolean isReadOnly() {
             return false;
         }
+
         @Override
-        public void snapshotShard(IndexShard shard, SnapshotId snapshotId, IndexId indexId, IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus) {}
+        public void snapshotShard(IndexShard shard, SnapshotId snapshotId, IndexId indexId, IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus) {
+        }
+
         @Override
         public IndexShardSnapshotStatus getShardSnapshotStatus(SnapshotId snapshotId, Version version, IndexId indexId, ShardId shardId) {
             return null;
         }
-        @Override
-        public void verify(String verificationToken, DiscoveryNode localNode) {}
-    }
 
-    public static Engine getEngineFromShard(IndexShard shard) {
-        return shard.getEngineOrNull();
+        @Override
+        public void verify(String verificationToken, DiscoveryNode localNode) {
+        }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexingOperationListenerTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexingOperationListenerTests.java
index d1cf8b32f580f..fbb4cbd8c997a 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/IndexingOperationListenerTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/IndexingOperationListenerTests.java
@@ -19,7 +19,11 @@
 package org.elasticsearch.index.shard;
 
 import org.apache.lucene.index.Term;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.engine.InternalEngineTests;
+import org.elasticsearch.index.mapper.ParsedDocument;
+import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.ArrayList;
@@ -28,6 +32,8 @@
 import java.util.List;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import static org.hamcrest.Matchers.is;
+
 public class IndexingOperationListenerTests extends ESTestCase{
 
     // this test also tests if calls are correct if one or more listeners throw exceptions
@@ -38,65 +44,81 @@ public void testListenersAreExecuted() {
         AtomicInteger preDelete = new AtomicInteger();
         AtomicInteger postDelete = new AtomicInteger();
         AtomicInteger postDeleteException = new AtomicInteger();
+        ShardId randomShardId = new ShardId(new Index(randomAlphaOfLength(10), randomAlphaOfLength(10)), randomIntBetween(1, 10));
         IndexingOperationListener listener = new IndexingOperationListener() {
             @Override
-            public Engine.Index preIndex(Engine.Index operation) {
+            public Engine.Index preIndex(ShardId shardId, Engine.Index operation) {
+                assertThat(shardId, is(randomShardId));
                 preIndex.incrementAndGet();
                 return operation;
             }
 
             @Override
-            public void postIndex(Engine.Index index, boolean created) {
-                postIndex.incrementAndGet();
+            public void postIndex(ShardId shardId, Engine.Index index, Engine.IndexResult result) {
+                if (result.hasFailure() == false) {
+                    postIndex.incrementAndGet();
+                } else {
+                    postIndex(shardId, index, result.getFailure());
+                }
             }
 
             @Override
-            public void postIndex(Engine.Index index, Exception ex) {
+            public void postIndex(ShardId shardId, Engine.Index index, Exception ex) {
+                assertThat(shardId, is(randomShardId));
                 postIndexException.incrementAndGet();
             }
 
             @Override
-            public Engine.Delete preDelete(Engine.Delete delete) {
+            public Engine.Delete preDelete(ShardId shardId, Engine.Delete delete) {
+                assertThat(shardId, is(randomShardId));
                 preDelete.incrementAndGet();
                 return delete;
             }
 
             @Override
-            public void postDelete(Engine.Delete delete) {
-                postDelete.incrementAndGet();
+            public void postDelete(ShardId shardId, Engine.Delete delete, Engine.DeleteResult result) {
+                if (result.hasFailure() == false) {
+                    postDelete.incrementAndGet();
+                } else {
+                    postDelete(shardId, delete, result.getFailure());
+                }
             }
 
             @Override
-            public void postDelete(Engine.Delete delete, Exception ex) {
+            public void postDelete(ShardId shardId, Engine.Delete delete, Exception ex) {
+                assertThat(shardId, is(randomShardId));
                 postDeleteException.incrementAndGet();
             }
         };
 
         IndexingOperationListener throwingListener = new IndexingOperationListener() {
             @Override
-            public Engine.Index preIndex(Engine.Index operation) {
+            public Engine.Index preIndex(ShardId shardId, Engine.Index index) {
                 throw new RuntimeException();
             }
 
             @Override
-            public void postIndex(Engine.Index index, boolean created) {
-                throw new RuntimeException();            }
+            public void postIndex(ShardId shardId, Engine.Index index, Engine.IndexResult result) {
+                throw new RuntimeException();
+            }
 
             @Override
-            public void postIndex(Engine.Index index, Exception ex) {
-                throw new RuntimeException();            }
+            public void postIndex(ShardId shardId, Engine.Index index, Exception ex) {
+                throw new RuntimeException();
+            }
 
             @Override
-            public Engine.Delete preDelete(Engine.Delete delete) {
+            public void postDelete(ShardId shardId, Engine.Delete delete, Engine.DeleteResult result) {
                 throw new RuntimeException();
             }
 
             @Override
-            public void postDelete(Engine.Delete delete) {
-                throw new RuntimeException();            }
+            public Engine.Delete preDelete(ShardId shardId, Engine.Delete delete) {
+                throw new RuntimeException();
+            }
 
             @Override
-            public void postDelete(Engine.Delete delete, Exception ex) {
+            public void postDelete(ShardId shardId, Engine.Delete delete, Exception ex) {
                 throw new RuntimeException();
             }
         };
@@ -108,10 +130,12 @@ public void postDelete(Engine.Delete delete, Exception ex) {
             }
         }
         Collections.shuffle(indexingOperationListeners, random());
-        IndexingOperationListener.CompositeListener compositeListener = new IndexingOperationListener.CompositeListener(indexingOperationListeners, logger);
-        Engine.Delete delete = new Engine.Delete("test", "1", new Term("_uid", "1"));
-        Engine.Index index = new Engine.Index(new Term("_uid", "1"), null);
-        compositeListener.postDelete(delete);
+        ParsedDocument doc = InternalEngineTests.createParsedDoc("1", "test", null);
+        IndexingOperationListener.CompositeListener compositeListener =
+                new IndexingOperationListener.CompositeListener(indexingOperationListeners, logger);
+        Engine.Delete delete = new Engine.Delete("test", "1", new Term("_uid", Uid.createUid("test", "1")));
+        Engine.Index index = new Engine.Index(new Term("_uid", Uid.createUid(doc.type(), doc.id())), doc);
+        compositeListener.postDelete(randomShardId, delete, new Engine.DeleteResult(0, false));
         assertEquals(0, preIndex.get());
         assertEquals(0, postIndex.get());
         assertEquals(0, postIndexException.get());
@@ -119,7 +143,7 @@ public void postDelete(Engine.Delete delete, Exception ex) {
         assertEquals(2, postDelete.get());
         assertEquals(0, postDeleteException.get());
 
-        compositeListener.postDelete(delete, new RuntimeException());
+        compositeListener.postDelete(randomShardId, delete, new RuntimeException());
         assertEquals(0, preIndex.get());
         assertEquals(0, postIndex.get());
         assertEquals(0, postIndexException.get());
@@ -127,7 +151,7 @@ public void postDelete(Engine.Delete delete, Exception ex) {
         assertEquals(2, postDelete.get());
         assertEquals(2, postDeleteException.get());
 
-        compositeListener.preDelete(delete);
+        compositeListener.preDelete(randomShardId, delete);
         assertEquals(0, preIndex.get());
         assertEquals(0, postIndex.get());
         assertEquals(0, postIndexException.get());
@@ -135,7 +159,7 @@ public void postDelete(Engine.Delete delete, Exception ex) {
         assertEquals(2, postDelete.get());
         assertEquals(2, postDeleteException.get());
 
-        compositeListener.postIndex(index, false);
+        compositeListener.postIndex(randomShardId, index, new Engine.IndexResult(0, false));
         assertEquals(0, preIndex.get());
         assertEquals(2, postIndex.get());
         assertEquals(0, postIndexException.get());
@@ -143,7 +167,7 @@ public void postDelete(Engine.Delete delete, Exception ex) {
         assertEquals(2, postDelete.get());
         assertEquals(2, postDeleteException.get());
 
-        compositeListener.postIndex(index, new RuntimeException());
+        compositeListener.postIndex(randomShardId, index, new RuntimeException());
         assertEquals(0, preIndex.get());
         assertEquals(2, postIndex.get());
         assertEquals(2, postIndexException.get());
@@ -151,7 +175,7 @@ public void postDelete(Engine.Delete delete, Exception ex) {
         assertEquals(2, postDelete.get());
         assertEquals(2, postDeleteException.get());
 
-        compositeListener.preIndex(index);
+        compositeListener.preIndex(randomShardId, index);
         assertEquals(2, preIndex.get());
         assertEquals(2, postIndex.get());
         assertEquals(2, postIndexException.get());
diff --git a/core/src/test/java/org/elasticsearch/index/shard/NewPathForShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/NewPathForShardTests.java
index f1515cd559b33..fc8fc12e75d6a 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/NewPathForShardTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/NewPathForShardTests.java
@@ -76,7 +76,7 @@ public static void removeMockUsableSpaceFS() throws Exception {
     /** Mock file system that fakes usable space for each FileStore */
     static class MockUsableSpaceFileSystemProvider extends FilterFileSystemProvider {
 
-        public MockUsableSpaceFileSystemProvider(FileSystem inner) {
+        MockUsableSpaceFileSystemProvider(FileSystem inner) {
             super("mockusablespace://", inner);
             final List<FileStore> fileStores = new ArrayList<>();
             fileStores.add(aFileStore);
@@ -99,7 +99,7 @@ static class MockFileStore extends FileStore {
 
         private final String desc;
 
-        public MockFileStore(String desc) {
+        MockFileStore(String desc) {
             this.desc = desc;
         }
 
diff --git a/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java b/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java
index f3f15b2639cd5..546858397ae7d 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java
@@ -37,6 +37,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.codec.CodecService;
@@ -47,6 +48,7 @@
 import org.elasticsearch.index.fieldvisitor.SingleFieldsVisitor;
 import org.elasticsearch.index.mapper.ParseContext.Document;
 import org.elasticsearch.index.mapper.ParsedDocument;
+import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.index.mapper.UidFieldMapper;
 import org.elasticsearch.index.store.DirectoryService;
 import org.elasticsearch.index.store.Store;
@@ -54,6 +56,7 @@
 import org.elasticsearch.test.DummyShardLock;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
+import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.threadpool.ThreadPool.Cancellable;
@@ -64,6 +67,7 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -120,12 +124,14 @@ public void onFailedEngine(String reason, @Nullable Exception e) {
                 // we don't need to notify anybody in this test
             }
         };
+        TranslogHandler translogHandler = new TranslogHandler(xContentRegistry(), shardId.getIndexName(), logger);
         EngineConfig config = new EngineConfig(EngineConfig.OpenMode.CREATE_INDEX_AND_TRANSLOG, shardId, threadPool, indexSettings, null,
                 store, new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()), newMergePolicy(), iwc.getAnalyzer(),
-                iwc.getSimilarity(), new CodecService(null, logger), eventListener, new TranslogHandler(shardId.getIndexName(), logger),
+                iwc.getSimilarity(), new CodecService(null, logger), eventListener, translogHandler,
                 IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), translogConfig,
-                TimeValue.timeValueMinutes(5), listeners, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP);
+                TimeValue.timeValueMinutes(5), Collections.singletonList(listeners), IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP);
         engine = new InternalEngine(config);
+        listeners.setTranslog(engine.getTranslog());
     }
 
     @After
@@ -134,9 +140,40 @@ public void tearDownListeners() throws Exception {
         terminate(threadPool);
     }
 
+    public void testBeforeRefresh() throws Exception {
+        assertEquals(0, listeners.pendingCount());
+        Engine.IndexResult index = index("1");
+        DummyRefreshListener listener = new DummyRefreshListener();
+        assertFalse(listeners.addOrNotify(index.getTranslogLocation(), listener));
+        assertNull(listener.forcedRefresh.get());
+        assertEquals(1, listeners.pendingCount());
+        engine.refresh("I said so");
+        assertFalse(listener.forcedRefresh.get());
+        listener.assertNoError();
+        assertEquals(0, listeners.pendingCount());
+    }
+
+    public void testAfterRefresh() throws Exception {
+        assertEquals(0, listeners.pendingCount());
+        Engine.IndexResult index = index("1");
+        engine.refresh("I said so");
+        if (randomBoolean()) {
+            index(randomFrom("1" /* same document */, "2" /* different document */));
+            if (randomBoolean()) {
+                engine.refresh("I said so");
+            }
+        }
+        DummyRefreshListener listener = new DummyRefreshListener();
+        assertTrue(listeners.addOrNotify(index.getTranslogLocation(), listener));
+        assertFalse(listener.forcedRefresh.get());
+        listener.assertNoError();
+        assertEquals(0, listeners.pendingCount());
+    }
+
     public void testTooMany() throws Exception {
+        assertEquals(0, listeners.pendingCount());
         assertFalse(listeners.refreshNeeded());
-        Engine.Index index = index("1");
+        Engine.IndexResult index = index("1");
 
         // Fill the listener slots
         List<DummyRefreshListener> nonForcedListeners = new ArrayList<>(maxListeners);
@@ -151,6 +188,7 @@ public void testTooMany() throws Exception {
         for (DummyRefreshListener listener : nonForcedListeners) {
             assertNull("Called listener too early!", listener.forcedRefresh.get());
         }
+        assertEquals(maxListeners, listeners.pendingCount());
 
         // Add one more listener which should cause a refresh.
         DummyRefreshListener forcingListener = new DummyRefreshListener();
@@ -164,22 +202,45 @@ public void testTooMany() throws Exception {
             listener.assertNoError();
         }
         assertFalse(listeners.refreshNeeded());
+        assertEquals(0, listeners.pendingCount());
     }
 
-    public void testAfterRefresh() throws Exception {
-        Engine.Index index = index("1");
+    public void testClose() throws Exception {
+        assertEquals(0, listeners.pendingCount());
+        Engine.IndexResult refreshedOperation = index("1");
         engine.refresh("I said so");
-        if (randomBoolean()) {
-            index(randomFrom("1" /* same document */, "2" /* different document */));
-            if (randomBoolean()) {
-                engine.refresh("I said so");
-            }
+        Engine.IndexResult unrefreshedOperation = index("1");
+        {
+            /* Closing flushed pending listeners as though they were refreshed. Since this can only happen when the index is closed and no
+             * longer useful there doesn't seem much point in sending the listener some kind of "I'm closed now, go away" enum value. */
+            DummyRefreshListener listener = new DummyRefreshListener();
+            assertFalse(listeners.addOrNotify(unrefreshedOperation.getTranslogLocation(), listener));
+            assertNull(listener.forcedRefresh.get());
+            listeners.close();
+            assertFalse(listener.forcedRefresh.get());
+            listener.assertNoError();
+            assertFalse(listeners.refreshNeeded());
+            assertEquals(0, listeners.pendingCount());
+        }
+        {
+            // If you add a listener for an already refreshed location then it'll just fire even if closed
+            DummyRefreshListener listener = new DummyRefreshListener();
+            assertTrue(listeners.addOrNotify(refreshedOperation.getTranslogLocation(), listener));
+            assertFalse(listener.forcedRefresh.get());
+            listener.assertNoError();
+            assertFalse(listeners.refreshNeeded());
+            assertEquals(0, listeners.pendingCount());
+        }
+        {
+            // But adding a listener to a non-refreshed location will fail
+            DummyRefreshListener listener = new DummyRefreshListener();
+            Exception e = expectThrows(IllegalStateException.class, () ->
+                listeners.addOrNotify(unrefreshedOperation.getTranslogLocation(), listener));
+            assertEquals("can't wait for refresh on a closed index", e.getMessage());
+            assertNull(listener.forcedRefresh.get());
+            assertFalse(listeners.refreshNeeded());
+            assertEquals(0, listeners.pendingCount());
         }
-
-        DummyRefreshListener listener = new DummyRefreshListener();
-        assertTrue(listeners.addOrNotify(index.getTranslogLocation(), listener));
-        assertFalse(listener.forcedRefresh.get());
-        listener.assertNoError();
     }
 
     /**
@@ -197,7 +258,7 @@ public void testConcurrentRefresh() throws Exception {
         refresher.start();
         try {
             for (int i = 0; i < 1000; i++) {
-                Engine.Index index = index("1");
+                Engine.IndexResult index = index("1");
                 DummyRefreshListener listener = new DummyRefreshListener();
                 boolean immediate = listeners.addOrNotify(index.getTranslogLocation(), listener);
                 if (immediate) {
@@ -218,6 +279,7 @@ public void testConcurrentRefresh() throws Exception {
      * Uses a bunch of threads to index, wait for refresh, and non-realtime get documents to validate that they are visible after waiting
      * regardless of what crazy sequence of events causes the refresh listener to fire.
      */
+    @TestLogging("_root:debug,org.elasticsearch.index.engine.Engine.DW:trace")
     public void testLotsOfThreads() throws Exception {
         int threadCount = between(3, 10);
         maxListeners = between(1, threadCount * 2);
@@ -233,8 +295,8 @@ public void testLotsOfThreads() throws Exception {
                 for (int iteration = 1; iteration <= 50; iteration++) {
                     try {
                         String testFieldValue = String.format(Locale.ROOT, "%s%04d", threadId, iteration);
-                        Engine.Index index = index(threadId, testFieldValue);
-                        assertEquals(iteration, index.version());
+                        Engine.IndexResult index = index(threadId, testFieldValue);
+                        assertEquals(iteration, index.getVersion());
 
                         DummyRefreshListener listener = new DummyRefreshListener();
                         listeners.addOrNotify(index.getTranslogLocation(), listener);
@@ -244,8 +306,8 @@ public void testLotsOfThreads() throws Exception {
                         }
                         listener.assertNoError();
 
-                        Engine.Get get = new Engine.Get(false, index.uid());
-                        try (Engine.GetResult getResult = engine.get(get)) {
+                        Engine.Get get = new Engine.Get(false, "test", threadId, new Term("_uid",  Uid.createUid("test", threadId)));
+                        try (Engine.GetResult getResult = engine.get(get, engine::acquireSearcher)) {
                             assertTrue("document not found", getResult.exists());
                             assertEquals(iteration, getResult.version());
                             SingleFieldsVisitor visitor = new SingleFieldsVisitor("test");
@@ -266,37 +328,35 @@ public void testLotsOfThreads() throws Exception {
         refresher.cancel();
     }
 
-    private Engine.Index index(String id) {
+    private Engine.IndexResult index(String id) throws IOException {
         return index(id, "test");
     }
 
-    private Engine.Index index(String id, String testFieldValue) {
+    private Engine.IndexResult index(String id, String testFieldValue) throws IOException {
         String type = "test";
-        String uid = type + ":" + id;
         Document document = new Document();
         document.add(new TextField("test", testFieldValue, Field.Store.YES));
-        Field uidField = new Field("_uid", type + ":" + id, UidFieldMapper.Defaults.FIELD_TYPE);
+        Field uidField = new Field("_uid", Uid.createUid(type, id), UidFieldMapper.Defaults.FIELD_TYPE);
         Field versionField = new NumericDocValuesField("_version", Versions.MATCH_ANY);
         document.add(uidField);
         document.add(versionField);
         BytesReference source = new BytesArray(new byte[] { 1 });
-        ParsedDocument doc = new ParsedDocument(versionField, id, type, null, -1, -1, Arrays.asList(document), source, null);
-        Engine.Index index = new Engine.Index(new Term("_uid", uid), doc);
-        engine.index(index);
-        return index;
+        ParsedDocument doc = new ParsedDocument(versionField, id, type, null, -1, -1, Arrays.asList(document), source, XContentType.JSON,
+            null);
+        Engine.Index index = new Engine.Index(new Term("_uid", Uid.createUid(doc.type(), doc.id())), doc);
+        return engine.index(index);
     }
 
     private static class DummyRefreshListener implements Consumer<Boolean> {
         /**
          * When the listener is called this captures it's only argument.
          */
-        AtomicReference<Boolean> forcedRefresh = new AtomicReference<>();
+        final AtomicReference<Boolean> forcedRefresh = new AtomicReference<>();
         private volatile Exception error;
 
         @Override
         public void accept(Boolean forcedRefresh) {
             try {
-                assertNotNull(forcedRefresh);
                 Boolean oldValue = this.forcedRefresh.getAndSet(forcedRefresh);
                 assertNull("Listener called twice", oldValue);
             } catch (Exception e) {
diff --git a/core/src/test/java/org/elasticsearch/index/shard/SearchOperationListenerTests.java b/core/src/test/java/org/elasticsearch/index/shard/SearchOperationListenerTests.java
index 1721e5f5e5d83..28bab8da0fdfb 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/SearchOperationListenerTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/SearchOperationListenerTests.java
@@ -18,12 +18,11 @@
  */
 package org.elasticsearch.index.shard;
 
-import org.apache.lucene.index.Term;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.TestSearchContext;
+import org.elasticsearch.transport.TransportRequest;
+import org.elasticsearch.transport.TransportRequest.Empty;
 
 import java.lang.reflect.Proxy;
 import java.util.ArrayList;
@@ -32,6 +31,9 @@
 import java.util.List;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.sameInstance;
+
 public class SearchOperationListenerTests extends ESTestCase {
 
     // this test also tests if calls are correct if one or more listeners throw exceptions
@@ -46,6 +48,7 @@ public void testListenersAreExecuted() {
         AtomicInteger freeContext = new AtomicInteger();
         AtomicInteger newScrollContext =  new AtomicInteger();
         AtomicInteger freeScrollContext =  new AtomicInteger();
+        AtomicInteger validateSearchContext = new AtomicInteger();
         AtomicInteger timeInNanos = new AtomicInteger(randomIntBetween(0, 10));
         SearchOperationListener listener = new SearchOperationListener() {
             @Override
@@ -109,17 +112,26 @@ public void onFreeScrollContext(SearchContext context) {
                 assertNotNull(context);
                 freeScrollContext.incrementAndGet();
             }
+
+            @Override
+            public void validateSearchContext(SearchContext context, TransportRequest request) {
+                assertNotNull(context);
+                validateSearchContext.incrementAndGet();
+            }
         };
 
         SearchOperationListener throwingListener = (SearchOperationListener) Proxy.newProxyInstance(
             SearchOperationListener.class.getClassLoader(),
             new Class[]{SearchOperationListener.class},
             (a,b,c) -> { throw new RuntimeException();});
+        int throwingListeners = 0;
         final List<SearchOperationListener> indexingOperationListeners = new ArrayList<>(Arrays.asList(listener, listener));
         if (randomBoolean()) {
             indexingOperationListeners.add(throwingListener);
+            throwingListeners++;
             if (randomBoolean()) {
                 indexingOperationListeners.add(throwingListener);
+                throwingListeners++;
             }
         }
         Collections.shuffle(indexingOperationListeners, random());
@@ -137,6 +149,7 @@ public void onFreeScrollContext(SearchContext context) {
         assertEquals(0, newScrollContext.get());
         assertEquals(0, freeContext.get());
         assertEquals(0, freeScrollContext.get());
+        assertEquals(0, validateSearchContext.get());
 
         compositeListener.onFetchPhase(ctx, timeInNanos.get());
         assertEquals(0, preFetch.get());
@@ -149,6 +162,7 @@ public void onFreeScrollContext(SearchContext context) {
         assertEquals(0, newScrollContext.get());
         assertEquals(0, freeContext.get());
         assertEquals(0, freeScrollContext.get());
+        assertEquals(0, validateSearchContext.get());
 
         compositeListener.onPreQueryPhase(ctx);
         assertEquals(0, preFetch.get());
@@ -161,6 +175,7 @@ public void onFreeScrollContext(SearchContext context) {
         assertEquals(0, newScrollContext.get());
         assertEquals(0, freeContext.get());
         assertEquals(0, freeScrollContext.get());
+        assertEquals(0, validateSearchContext.get());
 
         compositeListener.onPreFetchPhase(ctx);
         assertEquals(2, preFetch.get());
@@ -173,6 +188,7 @@ public void onFreeScrollContext(SearchContext context) {
         assertEquals(0, newScrollContext.get());
         assertEquals(0, freeContext.get());
         assertEquals(0, freeScrollContext.get());
+        assertEquals(0, validateSearchContext.get());
 
         compositeListener.onFailedFetchPhase(ctx);
         assertEquals(2, preFetch.get());
@@ -185,6 +201,7 @@ public void onFreeScrollContext(SearchContext context) {
         assertEquals(0, newScrollContext.get());
         assertEquals(0, freeContext.get());
         assertEquals(0, freeScrollContext.get());
+        assertEquals(0, validateSearchContext.get());
 
         compositeListener.onFailedQueryPhase(ctx);
         assertEquals(2, preFetch.get());
@@ -197,6 +214,7 @@ public void onFreeScrollContext(SearchContext context) {
         assertEquals(0, newScrollContext.get());
         assertEquals(0, freeContext.get());
         assertEquals(0, freeScrollContext.get());
+        assertEquals(0, validateSearchContext.get());
 
         compositeListener.onNewContext(ctx);
         assertEquals(2, preFetch.get());
@@ -209,6 +227,7 @@ public void onFreeScrollContext(SearchContext context) {
         assertEquals(0, newScrollContext.get());
         assertEquals(0, freeContext.get());
         assertEquals(0, freeScrollContext.get());
+        assertEquals(0, validateSearchContext.get());
 
         compositeListener.onNewScrollContext(ctx);
         assertEquals(2, preFetch.get());
@@ -221,6 +240,7 @@ public void onFreeScrollContext(SearchContext context) {
         assertEquals(2, newScrollContext.get());
         assertEquals(0, freeContext.get());
         assertEquals(0, freeScrollContext.get());
+        assertEquals(0, validateSearchContext.get());
 
         compositeListener.onFreeContext(ctx);
         assertEquals(2, preFetch.get());
@@ -233,6 +253,7 @@ public void onFreeScrollContext(SearchContext context) {
         assertEquals(2, newScrollContext.get());
         assertEquals(2, freeContext.get());
         assertEquals(0, freeScrollContext.get());
+        assertEquals(0, validateSearchContext.get());
 
         compositeListener.onFreeScrollContext(ctx);
         assertEquals(2, preFetch.get());
@@ -245,5 +266,29 @@ public void onFreeScrollContext(SearchContext context) {
         assertEquals(2, newScrollContext.get());
         assertEquals(2, freeContext.get());
         assertEquals(2, freeScrollContext.get());
+        assertEquals(0, validateSearchContext.get());
+
+        if (throwingListeners == 0) {
+            compositeListener.validateSearchContext(ctx, Empty.INSTANCE);
+        } else {
+            RuntimeException expected =
+                expectThrows(RuntimeException.class, () -> compositeListener.validateSearchContext(ctx, Empty.INSTANCE));
+            assertNull(expected.getMessage());
+            assertEquals(throwingListeners - 1, expected.getSuppressed().length);
+            if (throwingListeners > 1) {
+                assertThat(expected.getSuppressed()[0], not(sameInstance(expected)));
+            }
+        }
+        assertEquals(2, preFetch.get());
+        assertEquals(2, preQuery.get());
+        assertEquals(2, failedFetch.get());
+        assertEquals(2, failedQuery.get());
+        assertEquals(2, onQuery.get());
+        assertEquals(2, onFetch.get());
+        assertEquals(2, newContext.get());
+        assertEquals(2, newScrollContext.get());
+        assertEquals(2, freeContext.get());
+        assertEquals(2, freeScrollContext.get());
+        assertEquals(2, validateSearchContext.get());
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/shard/ShardIdTests.java b/core/src/test/java/org/elasticsearch/index/shard/ShardIdTests.java
new file mode 100644
index 0000000000000..d7f6d14760471
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/shard/ShardIdTests.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.shard;
+
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.test.ESTestCase;
+
+public class ShardIdTests extends ESTestCase {
+
+    public void testShardIdFromString() {
+        String indexName = randomAlphaOfLengthBetween(3,50);
+        int shardId = randomInt();
+        ShardId id = ShardId.fromString("["+indexName+"]["+shardId+"]");
+        assertEquals(indexName, id.getIndexName());
+        assertEquals(shardId, id.getId());
+        assertEquals(indexName, id.getIndex().getName());
+        assertEquals(IndexMetaData.INDEX_UUID_NA_VALUE, id.getIndex().getUUID());
+
+        id = ShardId.fromString("[some]weird[0]Name][-125]");
+        assertEquals("some]weird[0]Name", id.getIndexName());
+        assertEquals(-125, id.getId());
+        assertEquals("some]weird[0]Name", id.getIndex().getName());
+        assertEquals(IndexMetaData.INDEX_UUID_NA_VALUE, id.getIndex().getUUID());
+
+        String badId = indexName + "," + shardId; // missing separator
+        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> ShardId.fromString(badId));
+        assertEquals("Unexpected shardId string format, expected [indexName][shardId] but got " + badId, ex.getMessage());
+
+        String badId2 = indexName + "][" + shardId + "]"; // missing opening bracket
+        ex = expectThrows(IllegalArgumentException.class,
+                () -> ShardId.fromString(badId2));
+
+        String badId3 = "[" + indexName + "][" + shardId; // missing closing bracket
+        ex = expectThrows(IllegalArgumentException.class,
+                () -> ShardId.fromString(badId3));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java b/core/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java
index f3433a9669a4d..4284d3573e7e4 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java
@@ -38,7 +38,7 @@
  */
 public class ShardPathTests extends ESTestCase {
     public void testLoadShardPath() throws IOException {
-        try (final NodeEnvironment env = newNodeEnvironment(Settings.builder().build())) {
+        try (NodeEnvironment env = newNodeEnvironment(Settings.builder().build())) {
             Settings.Builder builder = Settings.builder().put(IndexMetaData.SETTING_INDEX_UUID, "0xDEADBEEF")
                     .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT);
             Settings settings = builder.build();
@@ -56,7 +56,7 @@ public void testLoadShardPath() throws IOException {
     }
 
     public void testFailLoadShardPathOnMultiState() throws IOException {
-        try (final NodeEnvironment env = newNodeEnvironment(Settings.builder().build())) {
+        try (NodeEnvironment env = newNodeEnvironment(Settings.builder().build())) {
             final String indexUUID = "0xDEADBEEF";
             Settings.Builder builder = Settings.builder().put(IndexMetaData.SETTING_INDEX_UUID, indexUUID)
                     .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT);
@@ -66,15 +66,14 @@ public void testFailLoadShardPathOnMultiState() throws IOException {
             assumeTrue("This test tests multi data.path but we only got one", paths.length > 1);
             int id = randomIntBetween(1, 10);
             ShardStateMetaData.FORMAT.write(new ShardStateMetaData(id, true, indexUUID, AllocationId.newInitializing()), paths);
-            ShardPath.loadShardPath(logger, env, shardId, IndexSettingsModule.newIndexSettings(shardId.getIndex(), settings));
-            fail("Expected IllegalStateException");
-        } catch (IllegalStateException e) {
+            Exception e = expectThrows(IllegalStateException.class, () ->
+                ShardPath.loadShardPath(logger, env, shardId, IndexSettingsModule.newIndexSettings(shardId.getIndex(), settings)));
             assertThat(e.getMessage(), containsString("more than one shard state found"));
         }
     }
 
     public void testFailLoadShardPathIndexUUIDMissmatch() throws IOException {
-        try (final NodeEnvironment env = newNodeEnvironment(Settings.builder().build())) {
+        try (NodeEnvironment env = newNodeEnvironment(Settings.builder().build())) {
             Settings.Builder builder = Settings.builder().put(IndexMetaData.SETTING_INDEX_UUID, "foobar")
                     .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT);
             Settings settings = builder.build();
@@ -83,9 +82,8 @@ public void testFailLoadShardPathIndexUUIDMissmatch() throws IOException {
             Path path = randomFrom(paths);
             int id = randomIntBetween(1, 10);
             ShardStateMetaData.FORMAT.write(new ShardStateMetaData(id, true, "0xDEADBEEF", AllocationId.newInitializing()), path);
-            ShardPath.loadShardPath(logger, env, shardId, IndexSettingsModule.newIndexSettings(shardId.getIndex(), settings));
-            fail("Expected IllegalStateException");
-        } catch (IllegalStateException e) {
+            Exception e = expectThrows(IllegalStateException.class, () ->
+                ShardPath.loadShardPath(logger, env, shardId, IndexSettingsModule.newIndexSettings(shardId.getIndex(), settings)));
             assertThat(e.getMessage(), containsString("expected: foobar on shard path"));
         }
     }
@@ -93,12 +91,8 @@ public void testFailLoadShardPathIndexUUIDMissmatch() throws IOException {
     public void testIllegalCustomDataPath() {
         Index index = new Index("foo", "foo");
         final Path path = createTempDir().resolve(index.getUUID()).resolve("0");
-        try {
-            new ShardPath(true, path, path, new ShardId(index, 0));
-            fail("Expected IllegalArgumentException");
-        } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), is("shard state path must be different to the data path when using custom data paths"));
-        }
+        Exception e = expectThrows(IllegalArgumentException.class, () -> new ShardPath(true, path, path, new ShardId(index, 0)));
+        assertThat(e.getMessage(), is("shard state path must be different to the data path when using custom data paths"));
     }
 
     public void testValidCtor() {
@@ -135,7 +129,7 @@ public void testGetRootPaths() throws IOException {
             indexSettings = indexSettingsBuilder.build();
             nodeSettings = Settings.EMPTY;
         }
-        try (final NodeEnvironment env = newNodeEnvironment(nodeSettings)) {
+        try (NodeEnvironment env = newNodeEnvironment(nodeSettings)) {
             ShardId shardId = new ShardId("foo", indexUUID, 0);
             Path[] paths = env.availableShardPaths(shardId);
             Path path = randomFrom(paths);
diff --git a/core/src/test/java/org/elasticsearch/index/shard/ShardUtilsTests.java b/core/src/test/java/org/elasticsearch/index/shard/ShardUtilsTests.java
index e960622d1c188..34c1789824ea6 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/ShardUtilsTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/ShardUtilsTests.java
@@ -28,6 +28,7 @@
 import org.apache.lucene.store.BaseDirectoryWrapper;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
+import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
@@ -64,4 +65,7 @@ public void testExtractShardId() throws IOException {
         IOUtils.close(writer, dir);
     }
 
+    public static Engine getShardEngine(IndexShard shard) {
+        return shard.getEngine();
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/similarity/SimilarityTests.java b/core/src/test/java/org/elasticsearch/index/similarity/SimilarityTests.java
index 653eea4196c61..95a537c453e09 100644
--- a/core/src/test/java/org/elasticsearch/index/similarity/SimilarityTests.java
+++ b/core/src/test/java/org/elasticsearch/index/similarity/SimilarityTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.index.similarity;
 
+import org.apache.lucene.search.similarities.BooleanSimilarity;
 import org.apache.lucene.search.similarities.ClassicSimilarity;
 import org.apache.lucene.search.similarities.DFISimilarity;
 import org.apache.lucene.search.similarities.AfterEffectL;
@@ -64,6 +65,7 @@ public void testResolveDefaultSimilarities() {
         SimilarityService similarityService = createIndex("foo").similarityService();
         assertThat(similarityService.getSimilarity("classic").get(), instanceOf(ClassicSimilarity.class));
         assertThat(similarityService.getSimilarity("BM25").get(), instanceOf(BM25Similarity.class));
+        assertThat(similarityService.getSimilarity("boolean").get(), instanceOf(BooleanSimilarity.class));
         assertThat(similarityService.getSimilarity("default"), equalTo(null));
     }
 
@@ -109,6 +111,21 @@ public void testResolveSimilaritiesFromMapping_bm25() throws IOException {
         assertThat(similarity.getDiscountOverlaps(), equalTo(false));
     }
 
+    public void testResolveSimilaritiesFromMapping_boolean() throws IOException {
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startObject("properties")
+            .startObject("field1").field("type", "text").field("similarity", "boolean").endObject()
+            .endObject()
+            .endObject().endObject().string();
+
+        IndexService indexService = createIndex("foo", Settings.EMPTY);
+        DocumentMapper documentMapper = indexService.mapperService()
+            .documentMapperParser()
+            .parse("type", new CompressedXContent(mapping));
+        assertThat(documentMapper.mappers().getMapper("field1").fieldType().similarity(),
+            instanceOf(BooleanSimilarityProvider.class));
+    }
+
     public void testResolveSimilaritiesFromMapping_DFR() throws IOException {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
             .startObject("properties")
@@ -259,4 +276,30 @@ public void testSimilarityDefaultBackCompat() throws IOException {
             assertThat(e.getMessage(), equalTo("Unknown Similarity type [default] for field [field1]"));
         }
     }
+
+    public void testSimilarityDefaultOveriddenBackCompat() throws IOException {
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startObject("properties")
+            .startObject("field1")
+            .field("similarity", "default")
+            .field("type", "text")
+            .endObject()
+            .endObject()
+            .endObject()
+            .endObject().string();
+        Settings settings = Settings.builder()
+            .put(IndexMetaData.SETTING_VERSION_CREATED, VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.V_2_2_0))
+            .put("index.similarity.default.type", "LMJelinekMercer")
+            .put("index.similarity.default.lambda", 0.7f)
+            .build();
+
+        DocumentMapperParser parser = createIndex("test_v2.x", settings).mapperService().documentMapperParser();
+        DocumentMapper documentMapper = parser.parse("type", new CompressedXContent(mapping));
+        assertThat(documentMapper.mappers().getMapper("field1").fieldType().similarity(),
+            instanceOf(LMJelinekMercerSimilarityProvider.class));
+        LMJelinekMercerSimilarity sim =
+            (LMJelinekMercerSimilarity) documentMapper.mappers().getMapper("field1").fieldType().similarity().get();
+        assertThat(sim.getLambda(), equalTo(0.7f));
+        assertThat(documentMapper.mappers().getMapper("field1").fieldType().similarity().name(), equalTo("default"));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/snapshots/blobstore/FileInfoTests.java b/core/src/test/java/org/elasticsearch/index/snapshots/blobstore/FileInfoTests.java
index abaebb88c5ef9..d80448440aec1 100644
--- a/core/src/test/java/org/elasticsearch/index/snapshots/blobstore/FileInfoTests.java
+++ b/core/src/test/java/org/elasticsearch/index/snapshots/blobstore/FileInfoTests.java
@@ -28,11 +28,13 @@
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo;
 import org.elasticsearch.index.store.StoreFileMetaData;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
+
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
@@ -48,7 +50,7 @@ public void testToFromXContent() throws IOException {
             for (int i = 0; i < hash.length; i++) {
                 hash.bytes[i] = randomByte();
             }
-            StoreFileMetaData meta = new StoreFileMetaData("foobar", Math.abs(randomLong()), randomAsciiOfLengthBetween(1, 10), Version.LATEST, hash);
+            StoreFileMetaData meta = new StoreFileMetaData("foobar", Math.abs(randomLong()), randomAlphaOfLengthBetween(1, 10), Version.LATEST, hash);
             ByteSizeValue size = new ByteSizeValue(Math.abs(randomLong()));
             BlobStoreIndexShardSnapshot.FileInfo info = new BlobStoreIndexShardSnapshot.FileInfo("_foobar", meta, size);
             XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON).prettyPrint();
@@ -56,7 +58,7 @@ public void testToFromXContent() throws IOException {
             byte[] xcontent = BytesReference.toBytes(shuffleXContent(builder).bytes());
 
             final BlobStoreIndexShardSnapshot.FileInfo parsedInfo;
-            try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(xcontent)) {
+            try (XContentParser parser = createParser(JsonXContent.jsonXContent, xcontent)) {
                 parser.nextToken();
                 parsedInfo = BlobStoreIndexShardSnapshot.FileInfo.fromXContent(parser);
             }
@@ -117,7 +119,7 @@ public void testInvalidFieldsInFromXContent() throws IOException {
             if (failure == null) {
                 // No failures should read as usual
                 final BlobStoreIndexShardSnapshot.FileInfo parsedInfo;
-                try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(xContent)) {
+                try (XContentParser parser = createParser(JsonXContent.jsonXContent, xContent)) {
                     parser.nextToken();
                     parsedInfo = BlobStoreIndexShardSnapshot.FileInfo.fromXContent(parser);
                 }
@@ -128,14 +130,13 @@ public void testInvalidFieldsInFromXContent() throws IOException {
                 assertEquals("666", parsedInfo.metadata().checksum());
                 assertEquals(Version.LATEST, parsedInfo.metadata().writtenBy());
             } else {
-                try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(xContent)) {
+                try (XContentParser parser = createParser(JsonXContent.jsonXContent, xContent)) {
                     parser.nextToken();
                     BlobStoreIndexShardSnapshot.FileInfo.fromXContent(parser);
                     fail("Should have failed with [" + failure + "]");
                 } catch (ElasticsearchParseException ex) {
                     assertThat(ex.getMessage(), containsString(failure));
                 }
-
             }
         }
     }
diff --git a/core/src/test/java/org/elasticsearch/index/snapshots/blobstore/SlicedInputStreamTests.java b/core/src/test/java/org/elasticsearch/index/snapshots/blobstore/SlicedInputStreamTests.java
index 4bd8ba9cb3e62..1fefbfceac352 100644
--- a/core/src/test/java/org/elasticsearch/index/snapshots/blobstore/SlicedInputStreamTests.java
+++ b/core/src/test/java/org/elasticsearch/index/snapshots/blobstore/SlicedInputStreamTests.java
@@ -18,7 +18,7 @@
  */
 package org.elasticsearch.index.snapshots.blobstore;
 
-import com.carrotsearch.randomizedtesting.generators.RandomInts;
+import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.ByteArrayInputStream;
@@ -111,7 +111,7 @@ private int readFully(InputStream stream, byte[] buffer) throws IOException {
     }
 
     private byte[] randomBytes(Random random) {
-        int length = RandomInts.randomIntBetween(random, 1, 10);
+        int length = RandomNumbers.randomIntBetween(random, 1, 10);
         byte[] data = new byte[length];
         random.nextBytes(data);
         return data;
@@ -121,7 +121,7 @@ private static final class CheckClosedInputStream extends FilterInputStream {
 
         public boolean closed = false;
 
-        public CheckClosedInputStream(InputStream in) {
+        CheckClosedInputStream(InputStream in) {
             super(in);
         }
 
diff --git a/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java b/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java
index e1005c03b71e7..243e06b38eaef 100644
--- a/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java
+++ b/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java
@@ -35,7 +35,6 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.ShardIterator;
@@ -60,8 +59,8 @@
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.IndexShardState;
 import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.indices.recovery.RecoveryFileChunkRequest;
 import org.elasticsearch.indices.recovery.PeerRecoveryTargetService;
+import org.elasticsearch.indices.recovery.RecoveryFileChunkRequest;
 import org.elasticsearch.monitor.fs.FsInfo;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.plugins.Plugin;
@@ -73,7 +72,6 @@
 import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.test.store.MockFSIndexStore;
 import org.elasticsearch.test.transport.MockTransportService;
-import org.elasticsearch.transport.TransportException;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportService;
@@ -159,7 +157,7 @@ public void testCorruptFileAndRecover() throws ExecutionException, InterruptedEx
         }
         indexRandom(true, builders);
         ensureGreen();
-        assertAllSuccessful(client().admin().indices().prepareFlush().setForce(true).setWaitIfOngoing(true).execute().actionGet());
+        assertAllSuccessful(client().admin().indices().prepareFlush().setForce(true).execute().actionGet());
         // we have to flush at least once here since we don't corrupt the translog
         SearchResponse countResponse = client().prepareSearch().setSize(0).get();
         assertHitCount(countResponse, numDocs);
@@ -178,7 +176,8 @@ public void testCorruptFileAndRecover() throws ExecutionException, InterruptedEx
                 .timeout("5m") // sometimes due to cluster rebalacing and random settings default timeout is just not enough.
                 .waitForNoRelocatingShards(true)).actionGet();
         if (health.isTimedOut()) {
-            logger.info("cluster state:\n{}\n{}", client().admin().cluster().prepareState().get().getState().prettyPrint(), client().admin().cluster().preparePendingClusterTasks().get().prettyPrint());
+            logger.info("cluster state:\n{}\n{}",
+                client().admin().cluster().prepareState().get().getState(), client().admin().cluster().preparePendingClusterTasks().get());
             assertThat("timed out waiting for green state", health.isTimedOut(), equalTo(false));
         }
         assertThat(health.getStatus(), equalTo(ClusterHealthStatus.GREEN));
@@ -262,7 +261,7 @@ public void testCorruptPrimaryNoReplica() throws ExecutionException, Interrupted
         }
         indexRandom(true, builders);
         ensureGreen();
-        assertAllSuccessful(client().admin().indices().prepareFlush().setForce(true).setWaitIfOngoing(true).execute().actionGet());
+        assertAllSuccessful(client().admin().indices().prepareFlush().setForce(true).execute().actionGet());
         // we have to flush at least once here since we don't corrupt the translog
         SearchResponse countResponse = client().prepareSearch().setSize(0).get();
         assertHitCount(countResponse, numDocs);
@@ -284,11 +283,12 @@ public void testCorruptPrimaryNoReplica() throws ExecutionException, Interrupted
             .health(Requests.clusterHealthRequest("test")).get();
         if (response.getStatus() != ClusterHealthStatus.RED) {
             logger.info("Cluster turned red in busy loop: {}", didClusterTurnRed);
-            logger.info("cluster state:\n{}\n{}", client().admin().cluster().prepareState().get().getState().prettyPrint(), client().admin().cluster().preparePendingClusterTasks().get().prettyPrint());
+            logger.info("cluster state:\n{}\n{}",
+                client().admin().cluster().prepareState().get().getState(), client().admin().cluster().preparePendingClusterTasks().get());
         }
         assertThat(response.getStatus(), is(ClusterHealthStatus.RED));
         ClusterState state = client().admin().cluster().prepareState().get().getState();
-        GroupShardsIterator shardIterators = state.getRoutingTable().activePrimaryShardsGrouped(new String[]{"test"}, false);
+        GroupShardsIterator<ShardIterator> shardIterators = state.getRoutingTable().activePrimaryShardsGrouped(new String[]{"test"}, false);
         for (ShardIterator iterator : shardIterators) {
             ShardRouting routing;
             while ((routing = iterator.nextOrNull()) != null) {
@@ -346,7 +346,7 @@ public void testCorruptionOnNetworkLayerFinalizingRecovery() throws ExecutionExc
             mockTransportService.addDelegate(internalCluster().getInstance(TransportService.class, unluckyNode.getNode().getName()), new MockTransportService.DelegateTransport(mockTransportService.original()) {
 
                 @Override
-                public void sendRequest(DiscoveryNode node, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException, TransportException {
+                protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException {
                     if (corrupt.get() && action.equals(PeerRecoveryTargetService.Actions.FILE_CHUNK)) {
                         RecoveryFileChunkRequest req = (RecoveryFileChunkRequest) request;
                         byte[] array = BytesRef.deepCopyOf(req.content().toBytesRef()).bytes;
@@ -354,7 +354,7 @@ public void sendRequest(DiscoveryNode node, long requestId, String action, Trans
                         array[i] = (byte) ~array[i]; // flip one byte in the content
                         hasCorrupted.countDown();
                     }
-                    super.sendRequest(node, requestId, action, request, options);
+                    super.sendRequest(connection, requestId, action, request, options);
                 }
             });
         }
@@ -408,7 +408,7 @@ public void testCorruptionOnNetworkLayer() throws ExecutionException, Interrupte
         }
         indexRandom(true, builders);
         ensureGreen();
-        assertAllSuccessful(client().admin().indices().prepareFlush().setForce(true).setWaitIfOngoing(true).execute().actionGet());
+        assertAllSuccessful(client().admin().indices().prepareFlush().setForce(true).execute().actionGet());
         // we have to flush at least once here since we don't corrupt the translog
         SearchResponse countResponse = client().prepareSearch().setSize(0).get();
         assertHitCount(countResponse, numDocs);
@@ -418,7 +418,7 @@ public void testCorruptionOnNetworkLayer() throws ExecutionException, Interrupte
             mockTransportService.addDelegate(internalCluster().getInstance(TransportService.class, unluckyNode.getNode().getName()), new MockTransportService.DelegateTransport(mockTransportService.original()) {
 
                 @Override
-                public void sendRequest(DiscoveryNode node, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException, TransportException {
+                protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException {
                     if (action.equals(PeerRecoveryTargetService.Actions.FILE_CHUNK)) {
                         RecoveryFileChunkRequest req = (RecoveryFileChunkRequest) request;
                         if (truncate && req.length() > 1) {
@@ -432,7 +432,7 @@ public void sendRequest(DiscoveryNode node, long requestId, String action, Trans
                             array[i] = (byte) ~array[i]; // flip one byte in the content
                         }
                     }
-                    super.sendRequest(node, requestId, action, request, options);
+                    super.sendRequest(connection, requestId, action, request, options);
                 }
             });
         }
@@ -445,7 +445,8 @@ public void sendRequest(DiscoveryNode node, long requestId, String action, Trans
         ClusterHealthResponse actionGet = client().admin().cluster()
             .health(Requests.clusterHealthRequest("test").waitForGreenStatus()).actionGet();
         if (actionGet.isTimedOut()) {
-            logger.info("ensureGreen timed out, cluster state:\n{}\n{}", client().admin().cluster().prepareState().get().getState().prettyPrint(), client().admin().cluster().preparePendingClusterTasks().get().prettyPrint());
+            logger.info("ensureGreen timed out, cluster state:\n{}\n{}",
+                client().admin().cluster().prepareState().get().getState(), client().admin().cluster().preparePendingClusterTasks().get());
             assertThat("timed out waiting for green state", actionGet.isTimedOut(), equalTo(false));
         }
         // we are green so primaries got not corrupted.
@@ -473,7 +474,7 @@ public void sendRequest(DiscoveryNode node, long requestId, String action, Trans
      * TODO once checksum verification on snapshotting is implemented this test needs to be fixed or split into several
      * parts... We should also corrupt files on the actual snapshot and check that we don't restore the corrupted shard.
      */
-    @TestLogging("monitor.fs:DEBUG")
+    @TestLogging("org.elasticsearch.monitor.fs:DEBUG")
     public void testCorruptFileThenSnapshotAndRestore() throws ExecutionException, InterruptedException, IOException {
         int numDocs = scaledRandomIntBetween(100, 1000);
         internalCluster().ensureAtLeastNumDataNodes(2);
@@ -491,7 +492,7 @@ public void testCorruptFileThenSnapshotAndRestore() throws ExecutionException, I
         }
         indexRandom(true, builders);
         ensureGreen();
-        assertAllSuccessful(client().admin().indices().prepareFlush().setForce(true).setWaitIfOngoing(true).execute().actionGet());
+        assertAllSuccessful(client().admin().indices().prepareFlush().setForce(true).execute().actionGet());
         // we have to flush at least once here since we don't corrupt the translog
         SearchResponse countResponse = client().prepareSearch().setSize(0).get();
         assertHitCount(countResponse, numDocs);
@@ -546,7 +547,7 @@ public void testReplicaCorruption() throws Exception {
         }
         indexRandom(true, builders);
         ensureGreen();
-        assertAllSuccessful(client().admin().indices().prepareFlush().setForce(true).setWaitIfOngoing(true).execute().actionGet());
+        assertAllSuccessful(client().admin().indices().prepareFlush().setForce(true).execute().actionGet());
         // we have to flush at least once here since we don't corrupt the translog
         SearchResponse countResponse = client().prepareSearch().setSize(0).get();
         assertHitCount(countResponse, numDocs);
diff --git a/core/src/test/java/org/elasticsearch/index/store/CorruptedTranslogIT.java b/core/src/test/java/org/elasticsearch/index/store/CorruptedTranslogIT.java
index 7d658a2a59160..1fe6be53466ee 100644
--- a/core/src/test/java/org/elasticsearch/index/store/CorruptedTranslogIT.java
+++ b/core/src/test/java/org/elasticsearch/index/store/CorruptedTranslogIT.java
@@ -72,7 +72,7 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
     }
 
     public void testCorruptTranslogFiles() throws Exception {
-        internalCluster().startNodesAsync(1, Settings.EMPTY).get();
+        internalCluster().startNodes(1, Settings.EMPTY);
 
         assertAcked(prepareCreate("test").setSettings(Settings.builder()
                 .put("index.number_of_shards", 1)
diff --git a/core/src/test/java/org/elasticsearch/index/store/ExceptionRetryIT.java b/core/src/test/java/org/elasticsearch/index/store/ExceptionRetryIT.java
index aa9de8de87150..373937784b39d 100644
--- a/core/src/test/java/org/elasticsearch/index/store/ExceptionRetryIT.java
+++ b/core/src/test/java/org/elasticsearch/index/store/ExceptionRetryIT.java
@@ -39,6 +39,7 @@
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.transport.ConnectTransportException;
+import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportException;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.transport.TransportRequestOptions;
@@ -99,14 +100,13 @@ public void testRetryDueToExceptionOnNetworkLayer() throws ExecutionException, I
                 dataNode.getNode().getName()));
             mockTransportService.addDelegate(internalCluster().getInstance(TransportService.class, unluckyNode.getNode().getName()),
                 new MockTransportService.DelegateTransport(mockTransportService.original()) {
-
                 @Override
-                public void sendRequest(DiscoveryNode node, long requestId, String action, TransportRequest request,
-                                        TransportRequestOptions options) throws IOException, TransportException {
-                    super.sendRequest(node, requestId, action, request, options);
+                protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request,
+                                           TransportRequestOptions options) throws IOException {
+                    super.sendRequest(connection, requestId, action, request, options);
                     if (action.equals(TransportShardBulkAction.ACTION_NAME) && exceptionThrown.compareAndSet(false, true)) {
                         logger.debug("Throw ConnectTransportException");
-                        throw new ConnectTransportException(node, action);
+                        throw new ConnectTransportException(connection.getNode(), action);
                     }
                 }
             });
diff --git a/core/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java b/core/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java
index fe5410ccaf85b..d9b901493dc1e 100644
--- a/core/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java
+++ b/core/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java
@@ -19,7 +19,6 @@
 package org.elasticsearch.index.store;
 
 import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.FileSwitchDirectory;
 import org.apache.lucene.store.MMapDirectory;
 import org.apache.lucene.store.NIOFSDirectory;
 import org.apache.lucene.store.NoLockFactory;
@@ -54,6 +53,7 @@ public void testStoreDirectory() throws IOException {
         for (IndexModule.Type type : IndexModule.Type.values()) {
             doTestStoreDirectory(index, tempDir, type.name().toLowerCase(Locale.ROOT), type);
         }
+        assertWarnings("[default] store type is deprecated, use [fs] instead");
     }
 
     private void doTestStoreDirectory(Index index, Path tempDir, String typeSettingValue, IndexModule.Type type) throws IOException {
@@ -65,7 +65,7 @@ private void doTestStoreDirectory(Index index, Path tempDir, String typeSettingV
         Settings settings = settingsBuilder.build();
         IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("foo", settings);
         FsDirectoryService service = new FsDirectoryService(indexSettings, null, new ShardPath(false, tempDir, tempDir, new ShardId(index, 0)));
-        try (final Directory directory = service.newFSDirectory(tempDir, NoLockFactory.INSTANCE)) {
+        try (Directory directory = service.newFSDirectory(tempDir, NoLockFactory.INSTANCE)) {
             switch (type) {
                 case NIOFS:
                     assertTrue(type + " " + directory.toString(), directory instanceof NIOFSDirectory);
@@ -110,5 +110,6 @@ public void testUpdateThrottleType() throws IOException {
         assertEquals(StoreRateLimiting.Type.MERGE, store.rateLimiting().getType());
         assertNotSame(indexStoreConfig.getNodeRateLimiter(), store.rateLimiting());
         assertEquals(StoreRateLimiting.Type.ALL, indexStoreConfig.getNodeRateLimiter().getType());
+        assertWarnings("Store rate limiting is deprecated and will be removed in 6.0");
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/store/StoreTests.java b/core/src/test/java/org/elasticsearch/index/store/StoreTests.java
index cea4610685a3a..d5e7a6da8ce5a 100644
--- a/core/src/test/java/org/elasticsearch/index/store/StoreTests.java
+++ b/core/src/test/java/org/elasticsearch/index/store/StoreTests.java
@@ -46,7 +46,6 @@
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.store.RAMDirectory;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.IOUtils;
@@ -469,11 +468,11 @@ private static final class LuceneManagedDirectoryService extends DirectoryServic
         private final Directory dir;
         private final Random random;
 
-        public LuceneManagedDirectoryService(Random random) {
+        LuceneManagedDirectoryService(Random random) {
             this(random, true);
         }
 
-        public LuceneManagedDirectoryService(Random random, boolean preventDoubleWrite) {
+        LuceneManagedDirectoryService(Random random, boolean preventDoubleWrite) {
             super(new ShardId(INDEX_SETTINGS.getIndex(), 1), INDEX_SETTINGS);
             dir = StoreTests.newDirectory(random);
             this.random = random;
@@ -766,7 +765,7 @@ public void testStoreStats() throws IOException {
             initialStoreSize += store.directory().fileLength(extraFiles);
         }
         StoreStats stats = store.stats();
-        assertEquals(stats.getSize().bytes(), initialStoreSize);
+        assertEquals(stats.getSize().getBytes(), initialStoreSize);
 
         Directory dir = store.directory();
         final long length;
diff --git a/core/src/test/java/org/elasticsearch/index/suggest/stats/CompletionsStatsTests.java b/core/src/test/java/org/elasticsearch/index/suggest/stats/CompletionsStatsTests.java
new file mode 100644
index 0000000000000..2d699b2c4571c
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/suggest/stats/CompletionsStatsTests.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.index.suggest.stats;
+
+import org.elasticsearch.common.FieldMemoryStats;
+import org.elasticsearch.common.FieldMemoryStatsTests;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.search.suggest.completion.CompletionStats;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+public class CompletionsStatsTests extends ESTestCase {
+
+    public void testSerialize() throws IOException {
+        FieldMemoryStats map = randomBoolean() ? null : FieldMemoryStatsTests.randomFieldMemoryStats();
+        CompletionStats stats = new CompletionStats(randomNonNegativeLong(), map == null ? null :
+            map);
+        BytesStreamOutput out = new BytesStreamOutput();
+        stats.writeTo(out);
+        CompletionStats read = new CompletionStats();
+        StreamInput input = out.bytes().streamInput();
+        read.readFrom(input);
+        assertEquals(-1, input.read());
+        assertEquals(stats.getSizeInBytes(), read.getSizeInBytes());
+        assertEquals(stats.getFields(), read.getFields());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/suggest/stats/SuggestStatsIT.java b/core/src/test/java/org/elasticsearch/index/suggest/stats/SuggestStatsIT.java
index cb0397990ab73..b92a9f2babea8 100644
--- a/core/src/test/java/org/elasticsearch/index/suggest/stats/SuggestStatsIT.java
+++ b/core/src/test/java/org/elasticsearch/index/suggest/stats/SuggestStatsIT.java
@@ -152,10 +152,10 @@ private SearchRequestBuilder addSuggestions(SearchRequestBuilder request, int i)
 
     private Set<String> nodeIdsWithIndex(String... indices) {
         ClusterState state = client().admin().cluster().prepareState().execute().actionGet().getState();
-        GroupShardsIterator allAssignedShardsGrouped = state.routingTable().allAssignedShardsGrouped(indices, true);
+        GroupShardsIterator<ShardIterator> allAssignedShardsGrouped = state.routingTable().allAssignedShardsGrouped(indices, true);
         Set<String> nodes = new HashSet<>();
         for (ShardIterator shardIterator : allAssignedShardsGrouped) {
-            for (ShardRouting routing : shardIterator.asUnordered()) {
+            for (ShardRouting routing : shardIterator) {
                 if (routing.active()) {
                     nodes.add(routing.currentNodeId());
                 }
diff --git a/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java b/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
index 9961637c32362..3f25b55dd03fc 100644
--- a/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
+++ b/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
@@ -48,6 +48,9 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.VersionType;
+import org.elasticsearch.index.mapper.ParsedDocument;
+import org.elasticsearch.index.mapper.Uid;
+import org.elasticsearch.index.mapper.UidFieldMapper;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.translog.Translog.Location;
 import org.elasticsearch.test.ESTestCase;
@@ -80,11 +83,10 @@
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.CyclicBarrier;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
@@ -206,53 +208,6 @@ private String randomNonTranslogPatternString(int min, int max) {
         return string;
     }
 
-    public void testRead() throws IOException {
-        Location loc0 = translog.getLastWriteLocation();
-        assertNotNull(loc0);
-
-        Translog.Location loc1 = translog.add(new Translog.Index("test", "1", new byte[]{1}));
-        assertThat(loc1, greaterThan(loc0));
-        assertThat(translog.getLastWriteLocation(), greaterThan(loc1));
-        Translog.Location loc2 = translog.add(new Translog.Index("test", "2", new byte[]{2}));
-        assertThat(loc2, greaterThan(loc1));
-        assertThat(translog.getLastWriteLocation(), greaterThan(loc2));
-        assertThat(translog.read(loc1).getSource().source, equalTo(new BytesArray(new byte[]{1})));
-        assertThat(translog.read(loc2).getSource().source, equalTo(new BytesArray(new byte[]{2})));
-
-        Translog.Location lastLocBeforeSync = translog.getLastWriteLocation();
-        translog.sync();
-        assertEquals(lastLocBeforeSync, translog.getLastWriteLocation());
-        assertThat(translog.read(loc1).getSource().source, equalTo(new BytesArray(new byte[]{1})));
-        assertThat(translog.read(loc2).getSource().source, equalTo(new BytesArray(new byte[]{2})));
-
-        Translog.Location loc3 = translog.add(new Translog.Index("test", "2", new byte[]{3}));
-        assertThat(loc3, greaterThan(loc2));
-        assertThat(translog.getLastWriteLocation(), greaterThan(loc3));
-        assertThat(translog.read(loc3).getSource().source, equalTo(new BytesArray(new byte[]{3})));
-
-        lastLocBeforeSync = translog.getLastWriteLocation();
-        translog.sync();
-        assertEquals(lastLocBeforeSync, translog.getLastWriteLocation());
-        assertThat(translog.read(loc3).getSource().source, equalTo(new BytesArray(new byte[]{3})));
-        translog.prepareCommit();
-        /*
-         * The commit adds to the lastWriteLocation even though is isn't really a write. This is just an implementation artifact but it can
-         * safely be ignored because the lastWriteLocation continues to be greater than the Location returned from the last write operation
-         * and less than the location of the next write operation.
-         */
-        assertThat(translog.getLastWriteLocation(), greaterThan(lastLocBeforeSync));
-        assertThat(translog.read(loc3).getSource().source, equalTo(new BytesArray(new byte[]{3})));
-        translog.commit();
-        assertNull(translog.read(loc1));
-        assertNull(translog.read(loc2));
-        assertNull(translog.read(loc3));
-        try {
-            translog.read(new Translog.Location(translog.currentFileGeneration() + 1, 17, 35));
-            fail("generation is greater than the current");
-        } catch (IllegalStateException ex) {
-            // expected
-        }
-    }
 
     public void testSimpleOperations() throws IOException {
         ArrayList<Translog.Operation> ops = new ArrayList<>();
@@ -264,7 +219,7 @@ public void testSimpleOperations() throws IOException {
         assertThat(snapshot, SnapshotMatchers.equalsTo(ops));
         assertThat(snapshot.totalOperations(), equalTo(ops.size()));
 
-        addToTranslogAndList(translog, ops, new Translog.Delete(newUid("2")));
+        addToTranslogAndList(translog, ops, new Translog.Delete("test", "2", newUid("2")));
         snapshot = translog.newSnapshot();
         assertThat(snapshot, SnapshotMatchers.equalsTo(ops));
         assertThat(snapshot.totalOperations(), equalTo(ops.size()));
@@ -324,14 +279,14 @@ public void testStats() throws IOException {
         assertThat(stats.getTranslogSizeInBytes(), greaterThan(lastSize));
         lastSize = stats.getTranslogSizeInBytes();
 
-        translog.add(new Translog.Delete(newUid("2")));
+        translog.add(new Translog.Delete("test", "2", newUid("2")));
         stats = stats();
         total.add(stats);
         assertThat(stats.estimatedNumberOfOperations(), equalTo(2L));
         assertThat(stats.getTranslogSizeInBytes(), greaterThan(lastSize));
         lastSize = stats.getTranslogSizeInBytes();
 
-        translog.add(new Translog.Delete(newUid("3")));
+        translog.add(new Translog.Delete("test", "3", newUid("3")));
         translog.prepareCommit();
         stats = stats();
         total.add(stats);
@@ -344,7 +299,7 @@ public void testStats() throws IOException {
         assertThat(stats.estimatedNumberOfOperations(), equalTo(0L));
         assertThat(stats.getTranslogSizeInBytes(), equalTo(firstOperationPosition));
         assertEquals(6, total.estimatedNumberOfOperations());
-        assertEquals(455, total.getTranslogSizeInBytes());
+        assertEquals(476, total.getTranslogSizeInBytes());
 
         BytesStreamOutput out = new BytesStreamOutput();
         total.writeTo(out);
@@ -352,13 +307,13 @@ public void testStats() throws IOException {
         copy.readFrom(out.bytes().streamInput());
 
         assertEquals(6, copy.estimatedNumberOfOperations());
-        assertEquals(455, copy.getTranslogSizeInBytes());
+        assertEquals(476, copy.getTranslogSizeInBytes());
 
         try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
             builder.startObject();
             copy.toXContent(builder, ToXContent.EMPTY_PARAMS);
             builder.endObject();
-            assertEquals("{\"translog\":{\"operations\":6,\"size_in_bytes\":455}}", builder.string());
+            assertEquals("{\"translog\":{\"operations\":6,\"size_in_bytes\":476}}", builder.string());
         }
 
         try {
@@ -441,15 +396,19 @@ public void assertFileDeleted(Translog translog, long id) {
         assertFalse("translog [" + id + "] still exists", Files.exists(translog.location().resolve(Translog.getFilename(id))));
     }
 
-    static class LocationOperation {
+    static class LocationOperation implements Comparable<LocationOperation> {
         final Translog.Operation operation;
         final Translog.Location location;
 
-        public LocationOperation(Translog.Operation operation, Translog.Location location) {
+        LocationOperation(Translog.Operation operation, Translog.Location location) {
             this.operation = operation;
             this.location = location;
         }
 
+        @Override
+        public int compareTo(LocationOperation o) {
+            return location.compareTo(o.location);
+        }
     }
 
     public void testConcurrentWritesWithVaryingSize() throws Throwable {
@@ -478,8 +437,12 @@ public void testConcurrentWritesWithVaryingSize() throws Throwable {
             threads[i].join(60 * 1000);
         }
 
-        for (LocationOperation locationOperation : writtenOperations) {
-            Translog.Operation op = translog.read(locationOperation.location);
+        List<LocationOperation> collect = writtenOperations.stream().collect(Collectors.toList());
+        Collections.sort(collect);
+        Translog.Snapshot snapshot = translog.newSnapshot();
+        for (LocationOperation locationOperation : collect) {
+            Translog.Operation op = snapshot.next();
+            assertNotNull(op);
             Translog.Operation expectedOp = locationOperation.operation;
             assertEquals(expectedOp.opType(), op.opType());
             switch (op.opType()) {
@@ -505,6 +468,7 @@ public void testConcurrentWritesWithVaryingSize() throws Throwable {
             }
 
         }
+        assertNull(snapshot.next());
 
     }
 
@@ -513,7 +477,7 @@ public void testTranslogChecksums() throws Exception {
 
         int translogOperations = randomIntBetween(10, 100);
         for (int op = 0; op < translogOperations; op++) {
-            String ascii = randomAsciiOfLengthBetween(1, 50);
+            String ascii = randomAlphaOfLengthBetween(1, 50);
             locations.add(translog.add(new Translog.Index("test", "" + op, ascii.getBytes("UTF-8"))));
         }
         translog.sync();
@@ -521,13 +485,16 @@ public void testTranslogChecksums() throws Exception {
         corruptTranslogs(translogDir);
 
         AtomicInteger corruptionsCaught = new AtomicInteger(0);
+        Translog.Snapshot snapshot = translog.newSnapshot();
         for (Translog.Location location : locations) {
             try {
-                translog.read(location);
+                Translog.Operation next = snapshot.next();
+                assertNotNull(next);
             } catch (TranslogCorruptedException e) {
                 corruptionsCaught.incrementAndGet();
             }
         }
+        expectThrows(TranslogCorruptedException.class, () -> snapshot.next());
         assertThat("at least one corruption was caused and caught", corruptionsCaught.get(), greaterThanOrEqualTo(1));
     }
 
@@ -536,7 +503,7 @@ public void testTruncatedTranslogs() throws Exception {
 
         int translogOperations = randomIntBetween(10, 100);
         for (int op = 0; op < translogOperations; op++) {
-            String ascii = randomAsciiOfLengthBetween(1, 50);
+            String ascii = randomAlphaOfLengthBetween(1, 50);
             locations.add(translog.add(new Translog.Index("test", "" + op, ascii.getBytes("UTF-8"))));
         }
         translog.sync();
@@ -544,15 +511,12 @@ public void testTruncatedTranslogs() throws Exception {
         truncateTranslogs(translogDir);
 
         AtomicInteger truncations = new AtomicInteger(0);
+        Translog.Snapshot snap = translog.newSnapshot();
         for (Translog.Location location : locations) {
             try {
-                translog.read(location);
-            } catch (ElasticsearchException e) {
-                if (e.getCause() instanceof EOFException) {
-                    truncations.incrementAndGet();
-                } else {
-                    throw e;
-                }
+                assertNotNull(snap.next());
+            } catch (EOFException e) {
+                truncations.incrementAndGet();
             }
         }
         assertThat("at least one truncation was caused and caught", truncations.get(), greaterThanOrEqualTo(1));
@@ -593,8 +557,12 @@ private void corruptTranslogs(Path directory) throws Exception {
         }
     }
 
-    private Term newUid(String id) {
-        return new Term("_uid", id);
+    private Term newUid(ParsedDocument doc) {
+        return new Term("_uid", Uid.createUid(doc.type(), doc.id()));
+    }
+
+    private Term newUid(String uid) {
+        return new Term("_uid", uid);
     }
 
     public void testVerifyTranslogIsNotDeleted() throws IOException {
@@ -613,11 +581,11 @@ public void testVerifyTranslogIsNotDeleted() throws IOException {
      * Tests that concurrent readers and writes maintain view and snapshot semantics
      */
     public void testConcurrentWriteViewsAndSnapshot() throws Throwable {
-        final Thread[] writers = new Thread[randomIntBetween(1, 10)];
-        final Thread[] readers = new Thread[randomIntBetween(1, 10)];
+        final Thread[] writers = new Thread[randomIntBetween(1, 3)];
+        final Thread[] readers = new Thread[randomIntBetween(1, 3)];
         final int flushEveryOps = randomIntBetween(5, 100);
-        // used to notify main thread that so many operations have been written so it can simulate a flush
-        final AtomicReference<CountDownLatch> writtenOpsLatch = new AtomicReference<>(new CountDownLatch(0));
+        final int maxOps = randomIntBetween(200, 1000);
+        final Object signalReaderSomeDataWasIndexed = new Object();
         final AtomicLong idGenerator = new AtomicLong();
         final CyclicBarrier barrier = new CyclicBarrier(writers.length + readers.length + 1);
 
@@ -638,7 +606,7 @@ public void testConcurrentWriteViewsAndSnapshot() throws Throwable {
                 public void doRun() throws BrokenBarrierException, InterruptedException, IOException {
                     barrier.await();
                     int counter = 0;
-                    while (run.get()) {
+                    while (run.get() && idGenerator.get() < maxOps) {
                         long id = idGenerator.incrementAndGet();
                         final Translog.Operation op;
                         switch (Translog.Operation.Type.values()[((int) (id % Translog.Operation.Type.values().length))]) {
@@ -647,7 +615,7 @@ public void doRun() throws BrokenBarrierException, InterruptedException, IOExcep
                                 op = new Translog.Index("type", "" + id, new byte[]{(byte) id});
                                 break;
                             case DELETE:
-                                op = new Translog.Delete(newUid("" + id));
+                                op = new Translog.Delete("type", "" + id, newUid("" + id));
                                 break;
                             default:
                                 throw new ElasticsearchException("unknown type");
@@ -660,7 +628,14 @@ public void doRun() throws BrokenBarrierException, InterruptedException, IOExcep
                         if (id % writers.length == threadId) {
                             translog.ensureSynced(location);
                         }
-                        writtenOpsLatch.get().countDown();
+                        if (id % flushEveryOps == 0) {
+                            translog.commit();
+                        }
+                        if (id % 7 == 0) {
+                            synchronized (signalReaderSomeDataWasIndexed) {
+                                signalReaderSomeDataWasIndexed.notifyAll();
+                            }
+                        }
                         counter++;
                     }
                     logger.debug("--> [{}] done. wrote [{}] ops.", threadName, counter);
@@ -711,7 +686,7 @@ void newView() throws IOException {
                 protected void doRun() throws Exception {
                     barrier.await();
                     int iter = 0;
-                    while (run.get()) {
+                    while (idGenerator.get() < maxOps) {
                         if (iter++ % 10 == 0) {
                             newView();
                         }
@@ -744,7 +719,11 @@ protected void doRun() throws Exception {
                             }
                         }
                         // slow down things a bit and spread out testing..
-                        writtenOpsLatch.get().await(200, TimeUnit.MILLISECONDS);
+                        synchronized (signalReaderSomeDataWasIndexed) {
+                            if (idGenerator.get() < maxOps){
+                                signalReaderSomeDataWasIndexed.wait();
+                            }
+                        }
                     }
                     closeView();
                     logger.debug("--> [{}] done. tested [{}] snapshots", threadId, iter);
@@ -754,34 +733,27 @@ protected void doRun() throws Exception {
         }
 
         barrier.await();
-        try {
-            for (int iterations = scaledRandomIntBetween(10, 200); iterations > 0 && errors.isEmpty(); iterations--) {
-                writtenOpsLatch.set(new CountDownLatch(flushEveryOps));
-                while (writtenOpsLatch.get().await(200, TimeUnit.MILLISECONDS) == false) {
-                    if (errors.size() > 0) {
-                        break;
-                    }
-                }
-                translog.commit();
-            }
-        } finally {
-            run.set(false);
-            logger.debug("--> waiting for threads to stop");
-            for (Thread thread : writers) {
-                thread.join();
-            }
-            for (Thread thread : readers) {
-                thread.join();
-            }
-            if (errors.size() > 0) {
-                Throwable e = errors.get(0);
-                for (Throwable suppress : errors.subList(1, errors.size())) {
-                    e.addSuppressed(suppress);
-                }
-                throw e;
+        logger.debug("--> waiting for threads to stop");
+        for (Thread thread : writers) {
+            thread.join();
+        }
+        logger.debug("--> waiting for readers to stop");
+        // force stopping, if all writers crashed
+        synchronized (signalReaderSomeDataWasIndexed) {
+            idGenerator.set(Long.MAX_VALUE);
+            signalReaderSomeDataWasIndexed.notifyAll();
+        }
+        for (Thread thread : readers) {
+            thread.join();
+        }
+        if (errors.size() > 0) {
+            Throwable e = errors.get(0);
+            for (Throwable suppress : errors.subList(1, errors.size())) {
+                e.addSuppressed(suppress);
             }
-            logger.info("--> test done. total ops written [{}]", writtenOps.size());
+            throw e;
         }
+        logger.info("--> test done. total ops written [{}]", writtenOps.size());
     }
 
     public void testSyncUpTo() throws IOException {
@@ -860,8 +832,14 @@ public void testLocationComparison() throws IOException {
         }
 
         assertEquals(max.generation, translog.currentFileGeneration());
-        final Translog.Operation read = translog.read(max);
-        assertEquals(read.getSource().source.utf8ToString(), Integer.toString(count));
+        Translog.Snapshot snap = translog.newSnapshot();
+        Translog.Operation next;
+        Translog.Operation maxOp = null;
+        while ((next = snap.next()) != null) {
+            maxOp = next;
+        }
+        assertNotNull(maxOp);
+        assertEquals(maxOp.getSource().source.utf8ToString(), Integer.toString(count));
     }
 
     public static Translog.Location max(Translog.Location a, Translog.Location b) {
@@ -884,30 +862,24 @@ public void testBasicCheckpoint() throws IOException {
             }
         }
         assertEquals(translogOperations, translog.totalOperations());
-        final Translog.Location lastLocation = translog.add(new Translog.Index("test", "" + translogOperations, Integer.toString(translogOperations).getBytes(Charset.forName("UTF-8"))));
+        translog.add(new Translog.Index("test", "" + translogOperations, Integer.toString(translogOperations).getBytes(Charset.forName("UTF-8"))));
 
         final Checkpoint checkpoint = Checkpoint.read(translog.location().resolve(Translog.CHECKPOINT_FILE_NAME));
-        try (final TranslogReader reader = translog.openReader(translog.location().resolve(Translog.getFilename(translog.currentFileGeneration())), checkpoint)) {
+        try (TranslogReader reader = translog.openReader(translog.location().resolve(Translog.getFilename(translog.currentFileGeneration())), checkpoint)) {
             assertEquals(lastSynced + 1, reader.totalOperations());
+            Translog.Snapshot snapshot = reader.newSnapshot();
+
             for (int op = 0; op < translogOperations; op++) {
-                Translog.Location location = locations.get(op);
                 if (op <= lastSynced) {
-                    final Translog.Operation read = reader.read(location);
+                    final Translog.Operation read = snapshot.next();
                     assertEquals(Integer.toString(op), read.getSource().source.utf8ToString());
                 } else {
-                    try {
-                        reader.read(location);
-                        fail("read past checkpoint");
-                    } catch (EOFException ex) {
-
-                    }
+                    Translog.Operation next = snapshot.next();
+                    assertNull(next);
                 }
             }
-            try {
-                reader.read(lastLocation);
-                fail("read past checkpoint");
-            } catch (EOFException ex) {
-            }
+            Translog.Operation next = snapshot.next();
+            assertNull(next);
         }
         assertEquals(translogOperations + 1, translog.totalOperations());
         translog.close();
@@ -1306,7 +1278,7 @@ private static class TranslogThread extends Thread {
         private final Exception[] threadExceptions;
         private final Translog translog;
 
-        public TranslogThread(Translog translog, CountDownLatch downLatch, int opsPerThread, int threadId, Collection<LocationOperation> writtenOperations, Exception[] threadExceptions) {
+        TranslogThread(Translog translog, CountDownLatch downLatch, int opsPerThread, int threadId, Collection<LocationOperation> writtenOperations, Exception[] threadExceptions) {
             this.translog = translog;
             this.downLatch = downLatch;
             this.opsPerThread = opsPerThread;
@@ -1328,7 +1300,7 @@ public void run() {
                                 randomUnicodeOfLengthBetween(1, 20 * 1024).getBytes("UTF-8"));
                             break;
                         case DELETE:
-                            op = new Translog.Delete(new Term("_uid", threadId + "_" + opCount),
+                            op = new Translog.Delete("test", threadId + "_" + opCount, new Term("_uid", threadId + "_" + opCount),
                                 1 + randomInt(100000),
                                 randomFrom(VersionType.values()));
                             break;
@@ -1618,11 +1590,6 @@ ChannelFactory getChannelFactory() {
                     }
                 };
             }
-
-            @Override
-            protected boolean assertBytesAtLocation(Location location, BytesReference expectedBytes) throws IOException {
-                return true; // we don't wanna fail in the assert
-            }
         };
     }
 
@@ -1959,4 +1926,20 @@ public void testPendingDelete() throws IOException {
     public static Translog.Location randomTranslogLocation() {
         return new Translog.Location(randomLong(), randomLong(), randomInt());
     }
+
+    public void testSerialization() throws Exception {
+        // simulate legacy delete serialization
+        BytesStreamOutput out = new BytesStreamOutput();
+        out.writeVInt(Translog.Delete.FORMAT_5_0);
+        out.writeString(UidFieldMapper.NAME);
+        out.writeString("my_type#my_id");
+        out.writeLong(3); // version
+        out.writeByte(VersionType.INTERNAL.getValue());
+        out.writeLong(2); // seq no
+        out.writeLong(0); // primary term
+        StreamInput in = out.bytes().streamInput();
+        Translog.Delete serializedDelete = new Translog.Delete(in);
+        assertEquals("my_type", serializedDelete.type());
+        assertEquals("my_id", serializedDelete.id());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/translog/TruncateTranslogIT.java b/core/src/test/java/org/elasticsearch/index/translog/TruncateTranslogIT.java
index ff4de23a069ee..4ef26b9357c52 100644
--- a/core/src/test/java/org/elasticsearch/index/translog/TruncateTranslogIT.java
+++ b/core/src/test/java/org/elasticsearch/index/translog/TruncateTranslogIT.java
@@ -28,7 +28,6 @@
 import org.apache.lucene.store.Lock;
 import org.apache.lucene.store.LockObtainFailedException;
 import org.apache.lucene.store.NativeFSLockFactory;
-import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
@@ -44,10 +43,10 @@
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.MockEngineFactoryPlugin;
-import org.elasticsearch.index.translog.TruncateTranslogCommand;
 import org.elasticsearch.monitor.fs.FsInfo;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -85,7 +84,7 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
     }
 
     public void testCorruptTranslogTruncation() throws Exception {
-        internalCluster().startNodesAsync(1, Settings.EMPTY).get();
+        internalCluster().startNodes(1, Settings.EMPTY);
 
         assertAcked(prepareCreate("test").setSettings(Settings.builder()
                         .put("index.number_of_shards", 1)
@@ -114,7 +113,7 @@ public void testCorruptTranslogTruncation() throws Exception {
             // Try running it before the shard is closed, it should flip out because it can't acquire the lock
             try {
                 logger.info("--> running truncate while index is open on [{}]", translogDir.toAbsolutePath());
-                ttc.execute(t, options, new HashMap<String, String>());
+                ttc.execute(t, options, null /* TODO: env should be real here, and ttc should actually use it... */);
                 fail("expected the truncate command to fail not being able to acquire the lock");
             } catch (Exception e) {
                 assertThat(e.getMessage(), containsString("Failed to lock shard's directory"));
@@ -162,7 +161,7 @@ public void testCorruptTranslogTruncation() throws Exception {
 
             OptionSet options = parser.parse("-d", translogDir.toAbsolutePath().toString(), "-b");
             logger.info("--> running truncate translog command for [{}]", translogDir.toAbsolutePath());
-            ttc.execute(t, options, new HashMap<String, String>());
+            ttc.execute(t, options, null /* TODO: env should be real here, and ttc should actually use it... */);
             logger.info("--> output:\n{}", t.getOutput());
         }
 
diff --git a/core/src/test/java/org/elasticsearch/indexing/IndexActionIT.java b/core/src/test/java/org/elasticsearch/indexing/IndexActionIT.java
index 1fab2a3808634..a69ed5f3d0b77 100644
--- a/core/src/test/java/org/elasticsearch/indexing/IndexActionIT.java
+++ b/core/src/test/java/org/elasticsearch/indexing/IndexActionIT.java
@@ -18,26 +18,20 @@
  */
 package org.elasticsearch.indexing;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.index.IndexResponse;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.cluster.metadata.MetaDataCreateIndexService;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.indices.InvalidIndexNameException;
-import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.InternalSettingsPlugin;
-import org.elasticsearch.test.VersionUtils;
+import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
+import org.elasticsearch.test.junit.annotations.TestLogging;
 
 import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
 import java.util.Random;
@@ -47,7 +41,6 @@
 import java.util.concurrent.atomic.AtomicIntegerArray;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
@@ -60,6 +53,8 @@ public class IndexActionIT extends ESIntegTestCase {
      * This test tries to simulate load while creating an index and indexing documents
      * while the index is being created.
      */
+
+    @TestLogging("_root:DEBUG,org.elasticsearch.index.shard.IndexShard:TRACE,org.elasticsearch.action.search:TRACE")
     public void testAutoGenerateIdNoDuplicates() throws Exception {
         int numberOfIterations = scaledRandomIntBetween(10, 50);
         for (int i = 0; i < numberOfIterations; i++) {
@@ -69,7 +64,7 @@ public void testAutoGenerateIdNoDuplicates() throws Exception {
             logger.info("indexing [{}] docs", numOfDocs);
             List<IndexRequestBuilder> builders = new ArrayList<>(numOfDocs);
             for (int j = 0; j < numOfDocs; j++) {
-                builders.add(client().prepareIndex("test", "type").setSource("field", "value"));
+                builders.add(client().prepareIndex("test", "type").setSource("field", "value_" + j));
             }
             indexRandom(true, builders);
             logger.info("verifying indexed content");
@@ -77,7 +72,13 @@ public void testAutoGenerateIdNoDuplicates() throws Exception {
             for (int j = 0; j < numOfChecks; j++) {
                 try {
                     logger.debug("running search with all types");
-                    assertHitCount(client().prepareSearch("test").get(), numOfDocs);
+                    SearchResponse response = client().prepareSearch("test").get();
+                    if (response.getHits().totalHits() != numOfDocs) {
+                        final String message = "Count is " + response.getHits().totalHits() + " but " + numOfDocs + " was expected. "
+                            + ElasticsearchAssertions.formatShardStatus(response);
+                        logger.error("{}. search response: \n{}", message, response);
+                        fail(message);
+                    }
                 } catch (Exception e) {
                     logger.error("search for all docs types failed", e);
                     if (firstError == null) {
@@ -86,7 +87,13 @@ public void testAutoGenerateIdNoDuplicates() throws Exception {
                 }
                 try {
                     logger.debug("running search with a specific type");
-                    assertHitCount(client().prepareSearch("test").setTypes("type").get(), numOfDocs);
+                    SearchResponse response = client().prepareSearch("test").setTypes("type").get();
+                    if (response.getHits().totalHits() != numOfDocs) {
+                        final String message = "Count is " + response.getHits().totalHits() + " but " + numOfDocs + " was expected. "
+                            + ElasticsearchAssertions.formatShardStatus(response);
+                        logger.error("{}. search response: \n{}", message, response);
+                        fail(message);
+                    }
                 } catch (Exception e) {
                     logger.error("search for all docs of a specific type failed", e);
                     if (firstError == null) {
@@ -191,7 +198,7 @@ public void testCreateIndexWithLongName() {
         int min = MetaDataCreateIndexService.MAX_INDEX_NAME_BYTES + 1;
         int max = MetaDataCreateIndexService.MAX_INDEX_NAME_BYTES * 2;
         try {
-            createIndex(randomAsciiOfLengthBetween(min, max).toLowerCase(Locale.ROOT));
+            createIndex(randomAlphaOfLengthBetween(min, max).toLowerCase(Locale.ROOT));
             fail("exception should have been thrown on too-long index name");
         } catch (InvalidIndexNameException e) {
             assertThat("exception contains message about index name too long: " + e.getMessage(),
@@ -199,7 +206,7 @@ public void testCreateIndexWithLongName() {
         }
 
         try {
-            client().prepareIndex(randomAsciiOfLengthBetween(min, max).toLowerCase(Locale.ROOT), "mytype").setSource("foo", "bar").get();
+            client().prepareIndex(randomAlphaOfLengthBetween(min, max).toLowerCase(Locale.ROOT), "mytype").setSource("foo", "bar").get();
             fail("exception should have been thrown on too-long index name");
         } catch (InvalidIndexNameException e) {
             assertThat("exception contains message about index name too long: " + e.getMessage(),
@@ -208,7 +215,7 @@ public void testCreateIndexWithLongName() {
 
         try {
             // Catch chars that are more than a single byte
-            client().prepareIndex(randomAsciiOfLength(MetaDataCreateIndexService.MAX_INDEX_NAME_BYTES - 1).toLowerCase(Locale.ROOT) +
+            client().prepareIndex(randomAlphaOfLength(MetaDataCreateIndexService.MAX_INDEX_NAME_BYTES - 1).toLowerCase(Locale.ROOT) +
                             "Ϟ".toLowerCase(Locale.ROOT),
                     "mytype").setSource("foo", "bar").get();
             fail("exception should have been thrown on too-long index name");
@@ -218,7 +225,7 @@ public void testCreateIndexWithLongName() {
         }
 
         // we can create an index of max length
-        createIndex(randomAsciiOfLength(MetaDataCreateIndexService.MAX_INDEX_NAME_BYTES).toLowerCase(Locale.ROOT));
+        createIndex(randomAlphaOfLength(MetaDataCreateIndexService.MAX_INDEX_NAME_BYTES).toLowerCase(Locale.ROOT));
     }
 
     public void testInvalidIndexName() {
@@ -245,21 +252,7 @@ public void testDocumentWithBlankFieldName() {
             }
         );
         assertThat(e.getMessage(), containsString("failed to parse"));
-        assertThat(e.getRootCause().getMessage(), containsString("name cannot be empty string"));
-    }
-
-    @Override
-    protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Collections.singleton(InternalSettingsPlugin.class); // uses index.version.created
-    }
-
-    public void testDocumentWithBlankFieldName2x() {
-        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.V_2_3_4);
-        Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
-        assertAcked(prepareCreate("test1").setSettings(settings));
-        ensureGreen();
-
-        IndexResponse indexResponse = client().prepareIndex("test1", "type", "1").setSource("", "value1_2").execute().actionGet();
-        assertEquals(DocWriteResponse.Result.CREATED, indexResponse.getResult());
+        assertThat(e.getRootCause().getMessage(),
+                containsString("object field starting or ending with a [.] makes object resolution ambiguous: []"));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/indices/DateMathIndexExpressionsIntegrationIT.java b/core/src/test/java/org/elasticsearch/indices/DateMathIndexExpressionsIntegrationIT.java
index e91ed066cc6c4..fe4ab2e363a88 100644
--- a/core/src/test/java/org/elasticsearch/indices/DateMathIndexExpressionsIntegrationIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/DateMathIndexExpressionsIntegrationIT.java
@@ -20,11 +20,15 @@
 package org.elasticsearch.indices;
 
 import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.elasticsearch.action.delete.DeleteResponse;
 import org.elasticsearch.action.get.GetResponse;
+import org.elasticsearch.action.get.MultiGetResponse;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
@@ -45,12 +49,18 @@ public void testIndexNameDateMathExpressions() {
         String index3 = ".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now.minusDays(2));
         createIndex(index1, index2, index3);
 
+        GetSettingsResponse getSettingsResponse = client().admin().indices().prepareGetSettings(index1, index2, index3).get();
+        assertEquals(index1, getSettingsResponse.getSetting(index1, IndexMetaData.SETTING_INDEX_PROVIDED_NAME));
+        assertEquals(index2, getSettingsResponse.getSetting(index2, IndexMetaData.SETTING_INDEX_PROVIDED_NAME));
+        assertEquals(index3, getSettingsResponse.getSetting(index3, IndexMetaData.SETTING_INDEX_PROVIDED_NAME));
+
+
         String dateMathExp1 = "<.marvel-{now/d}>";
         String dateMathExp2 = "<.marvel-{now/d-1d}>";
         String dateMathExp3 = "<.marvel-{now/d-2d}>";
-        client().prepareIndex(dateMathExp1, "type", "1").setSource("{}").get();
-        client().prepareIndex(dateMathExp2, "type", "2").setSource("{}").get();
-        client().prepareIndex(dateMathExp3, "type", "3").setSource("{}").get();
+        client().prepareIndex(dateMathExp1, "type", "1").setSource("{}", XContentType.JSON).get();
+        client().prepareIndex(dateMathExp2, "type", "2").setSource("{}", XContentType.JSON).get();
+        client().prepareIndex(dateMathExp3, "type", "3").setSource("{}", XContentType.JSON).get();
         refresh();
 
         SearchResponse searchResponse = client().prepareSearch(dateMathExp1, dateMathExp2, dateMathExp3).get();
@@ -69,6 +79,17 @@ public void testIndexNameDateMathExpressions() {
         assertThat(getResponse.isExists(), is(true));
         assertThat(getResponse.getId(), equalTo("3"));
 
+        MultiGetResponse mgetResponse = client().prepareMultiGet()
+            .add(dateMathExp1, "type", "1")
+            .add(dateMathExp2, "type", "2")
+            .add(dateMathExp3, "type", "3").get();
+        assertThat(mgetResponse.getResponses()[0].getResponse().isExists(), is(true));
+        assertThat(mgetResponse.getResponses()[0].getResponse().getId(), equalTo("1"));
+        assertThat(mgetResponse.getResponses()[1].getResponse().isExists(), is(true));
+        assertThat(mgetResponse.getResponses()[1].getResponse().getId(), equalTo("2"));
+        assertThat(mgetResponse.getResponses()[2].getResponse().isExists(), is(true));
+        assertThat(mgetResponse.getResponses()[2].getResponse().getId(), equalTo("3"));
+
         IndicesStatsResponse indicesStatsResponse = client().admin().indices().prepareStats(dateMathExp1, dateMathExp2, dateMathExp3).get();
         assertThat(indicesStatsResponse.getIndex(index1), notNullValue());
         assertThat(indicesStatsResponse.getIndex(index2), notNullValue());
@@ -96,9 +117,9 @@ public void testAutoCreateIndexWithDateMathExpression() throws Exception {
         String dateMathExp1 = "<.marvel-{now/d}>";
         String dateMathExp2 = "<.marvel-{now/d-1d}>";
         String dateMathExp3 = "<.marvel-{now/d-2d}>";
-        client().prepareIndex(dateMathExp1, "type", "1").setSource("{}").get();
-        client().prepareIndex(dateMathExp2, "type", "2").setSource("{}").get();
-        client().prepareIndex(dateMathExp3, "type", "3").setSource("{}").get();
+        client().prepareIndex(dateMathExp1, "type", "1").setSource("{}", XContentType.JSON).get();
+        client().prepareIndex(dateMathExp2, "type", "2").setSource("{}", XContentType.JSON).get();
+        client().prepareIndex(dateMathExp3, "type", "3").setSource("{}", XContentType.JSON).get();
         refresh();
 
         SearchResponse searchResponse = client().prepareSearch(dateMathExp1, dateMathExp2, dateMathExp3).get();
@@ -122,6 +143,12 @@ public void testCreateIndexWithDateMathExpression() throws Exception {
         String dateMathExp3 = "<.marvel-{now/d-2d}>";
         createIndex(dateMathExp1, dateMathExp2, dateMathExp3);
 
+
+        GetSettingsResponse getSettingsResponse = client().admin().indices().prepareGetSettings(index1, index2, index3).get();
+        assertEquals(dateMathExp1, getSettingsResponse.getSetting(index1, IndexMetaData.SETTING_INDEX_PROVIDED_NAME));
+        assertEquals(dateMathExp2, getSettingsResponse.getSetting(index2, IndexMetaData.SETTING_INDEX_PROVIDED_NAME));
+        assertEquals(dateMathExp3, getSettingsResponse.getSetting(index3, IndexMetaData.SETTING_INDEX_PROVIDED_NAME));
+
         ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
         assertThat(clusterState.metaData().index(index1), notNullValue());
         assertThat(clusterState.metaData().index(index2), notNullValue());
diff --git a/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java b/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java
index 858e046d67c08..44f8e822c100f 100644
--- a/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java
@@ -21,17 +21,17 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.shard.IndexSearcherWrapper;
 import org.elasticsearch.index.shard.IndexShard;
-import org.elasticsearch.index.shard.IndexShardTests;
+import org.elasticsearch.index.shard.IndexShardIT;
 import org.elasticsearch.indices.recovery.RecoveryState;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -66,7 +66,7 @@ static class MockController extends IndexingMemoryController {
         // Shards that are currently throttled
         final Set<IndexShard> throttled = new HashSet<>();
 
-        public MockController(Settings settings) {
+        MockController(Settings settings) {
             super(Settings.builder()
                             .put("indices.memory.interval", "200h") // disable it
                             .put(settings)
@@ -406,14 +406,11 @@ protected Cancellable scheduleTask(ThreadPool threadPool) {
         imc.forceCheck();
 
         // We must assertBusy because the writeIndexingBufferAsync is done in background (REFRESH) thread pool:
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                try (Engine.Searcher s2 = shard.acquireSearcher("index")) {
-                    // 100 buffered deletes will easily exceed our 1 KB indexing buffer so it should trigger a write:
-                    final long indexingBufferBytes2 = shard.getIndexBufferRAMBytesUsed();
-                    assertTrue(indexingBufferBytes2 < indexingBufferBytes1);
-                }
+        assertBusy(() -> {
+            try (Engine.Searcher s2 = shard.acquireSearcher("index")) {
+                // 100 buffered deletes will easily exceed our 1 KB indexing buffer so it should trigger a write:
+                final long indexingBufferBytes2 = shard.getIndexBufferRAMBytesUsed();
+                assertTrue(indexingBufferBytes2 < indexingBufferBytes1);
             }
         });
     }
@@ -425,7 +422,7 @@ public void testTranslogRecoveryWorksWithIMC() throws IOException {
         IndexService indexService = indicesService.indexService(resolveIndex("test"));
         IndexShard shard = indexService.getShardOrNull(0);
         for (int i = 0; i < 100; i++) {
-            client().prepareIndex("test", "test", Integer.toString(i)).setSource("{\"foo\" : \"bar\"}").get();
+            client().prepareIndex("test", "test", Integer.toString(i)).setSource("{\"foo\" : \"bar\"}", XContentType.JSON).get();
         }
 
         IndexSearcherWrapper wrapper = new IndexSearcherWrapper() {};
@@ -443,7 +440,7 @@ protected void writeIndexingBufferAsync(IndexShard shard) {
                 shard.writeIndexingBuffer();
             }
         };
-        final IndexShard newShard = IndexShardTests.newIndexShard(indexService, shard, wrapper, imc);
+        final IndexShard newShard = IndexShardIT.newIndexShard(indexService, shard, wrapper, imc);
         shardRef.set(newShard);
         try {
             assertEquals(0, imc.availableShards().size());
diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java b/core/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java
index 66d6c16b4e0d5..72ff0b01932a9 100644
--- a/core/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java
@@ -29,10 +29,11 @@
 import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexService;
-import org.elasticsearch.index.NodeServicesProvider;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.shard.IndexEventListener;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason;
 import org.elasticsearch.indices.recovery.RecoveryState;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
@@ -43,11 +44,12 @@
 import static java.util.Collections.emptySet;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
+import static org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.DELETED;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 
 public class IndicesLifecycleListenerSingleNodeTests extends ESSingleNodeTestCase {
 
-    public void testCloseDeleteCallback() throws Throwable {
+    public void testStartDeleteIndexEventCallback() throws Throwable {
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
         assertAcked(client().admin().indices().prepareCreate("test")
                 .setSettings(SETTING_NUMBER_OF_SHARDS, 1, SETTING_NUMBER_OF_REPLICAS, 0));
@@ -57,46 +59,70 @@ public void testCloseDeleteCallback() throws Throwable {
         ShardRouting shardRouting = indicesService.indexService(idx).getShard(0).routingEntry();
         final AtomicInteger counter = new AtomicInteger(1);
         IndexEventListener countingListener = new IndexEventListener() {
+
+            @Override
+            public void beforeIndexCreated(Index index, Settings indexSettings) {
+                assertEquals("test", index.getName());
+                assertEquals(1, counter.get());
+                counter.incrementAndGet();
+            }
+
+            @Override
+            public void afterIndexCreated(IndexService indexService) {
+                assertEquals("test", indexService.index().getName());
+                assertEquals(2, counter.get());
+                counter.incrementAndGet();
+            }
+
             @Override
-            public void afterIndexClosed(Index index, Settings indexSettings) {
-                assertEquals(counter.get(), 5);
+            public void beforeIndexShardCreated(ShardId shardId, Settings indexSettings) {
+                assertEquals(3, counter.get());
                 counter.incrementAndGet();
             }
 
             @Override
-            public void beforeIndexClosed(IndexService indexService) {
-                assertEquals(counter.get(), 1);
+            public void afterIndexShardCreated(IndexShard indexShard) {
+                assertEquals(4, counter.get());
                 counter.incrementAndGet();
             }
 
             @Override
-            public void afterIndexDeleted(Index index, Settings indexSettings) {
-                assertEquals(counter.get(), 6);
+            public void afterIndexShardStarted(IndexShard indexShard) {
+                assertEquals(5, counter.get());
                 counter.incrementAndGet();
             }
 
             @Override
-            public void beforeIndexDeleted(IndexService indexService) {
-                assertEquals(counter.get(), 2);
+            public void beforeIndexRemoved(IndexService indexService, IndexRemovalReason reason) {
+                assertEquals(DELETED, reason);
+                assertEquals(6, counter.get());
                 counter.incrementAndGet();
             }
 
             @Override
             public void beforeIndexShardDeleted(ShardId shardId, Settings indexSettings) {
-                assertEquals(counter.get(), 3);
+                assertEquals(7, counter.get());
                 counter.incrementAndGet();
             }
 
             @Override
             public void afterIndexShardDeleted(ShardId shardId, Settings indexSettings) {
-                assertEquals(counter.get(), 4);
+                assertEquals(8, counter.get());
                 counter.incrementAndGet();
             }
+
+            @Override
+            public void afterIndexRemoved(Index index, IndexSettings indexSettings, IndexRemovalReason reason) {
+                assertEquals(DELETED, reason);
+                assertEquals(9, counter.get());
+                counter.incrementAndGet();
+            }
+
         };
-        indicesService.deleteIndex(idx, "simon says");
+        indicesService.removeIndex(idx, DELETED, "simon says");
         try {
-            NodeServicesProvider nodeServicesProvider = getInstanceFromNode(NodeServicesProvider.class);
-            IndexService index = indicesService.createIndex(nodeServicesProvider, metaData, Arrays.asList(countingListener));
+            IndexService index = indicesService.createIndex(metaData, Arrays.asList(countingListener));
+            assertEquals(3, counter.get());
             idx = index.index();
             ShardRouting newRouting = shardRouting;
             String nodeId = newRouting.currentNodeId();
@@ -106,16 +132,18 @@ public void afterIndexShardDeleted(ShardId shardId, Settings indexSettings) {
             newRouting = ShardRoutingHelper.initialize(newRouting, nodeId);
             IndexShard shard = index.createShard(newRouting);
             shard.updateRoutingEntry(newRouting);
+            assertEquals(5, counter.get());
             final DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(),
                     emptyMap(), emptySet(), Version.CURRENT);
             shard.markAsRecovering("store", new RecoveryState(newRouting, localNode, null));
             shard.recoverFromStore();
             newRouting = ShardRoutingHelper.moveToStarted(newRouting);
             shard.updateRoutingEntry(newRouting);
+            assertEquals(6, counter.get());
         } finally {
-            indicesService.deleteIndex(idx, "simon says");
+            indicesService.removeIndex(idx, DELETED, "simon says");
         }
-        assertEquals(7, counter.get());
+        assertEquals(10, counter.get());
     }
 
 }
diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java b/core/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java
index 04b60b80c0c13..298bb57c49916 100644
--- a/core/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java
@@ -25,8 +25,6 @@
 import java.util.Map;
 import java.util.stream.Collectors;
 
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
 import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
@@ -56,7 +54,7 @@ private static class FakeMetadataMapperParser implements MetadataFieldMapper.Typ
             return null;
         }
         @Override
-        public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName) {
+        public MetadataFieldMapper getDefault(MappedFieldType fieldType, ParserContext context) {
             return null;
         }
     }
diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java b/core/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java
index 13dad84eb9b55..2734fd7828421 100644
--- a/core/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java
@@ -45,7 +45,6 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -444,21 +443,15 @@ public void testAllMissingLenient() throws Exception {
 
     public void testAllMissingStrict() throws Exception {
         createIndex("test1");
-        try {
+        expectThrows(IndexNotFoundException.class, () ->
             client().prepareSearch("test2")
                     .setQuery(matchAllQuery())
-                    .execute().actionGet();
-            fail("Exception should have been thrown.");
-        } catch (IndexNotFoundException e) {
-        }
+                    .execute().actionGet());
 
-        try {
+        expectThrows(IndexNotFoundException.class, () ->
             client().prepareSearch("test2","test3")
                     .setQuery(matchAllQuery())
-                    .execute().actionGet();
-            fail("Exception should have been thrown.");
-        } catch (IndexNotFoundException e) {
-        }
+                    .execute().actionGet());
 
         //you should still be able to run empty searches without things blowing up
         client().prepareSearch().setQuery(matchAllQuery()).execute().actionGet();
diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java b/core/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java
index cd94ee0f8e954..1e9f6cc2bfd6b 100644
--- a/core/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java
@@ -23,25 +23,28 @@
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.Term;
 import org.apache.lucene.search.ConstantScoreScorer;
 import org.apache.lucene.search.ConstantScoreWeight;
 import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.Explanation;
 import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.QueryCachingPolicy;
 import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.ScorerSupplier;
 import org.apache.lucene.search.Weight;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.cache.query.QueryCacheStats;
 import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.indices.IndicesQueryCache;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
+import java.util.Set;
 
 public class IndicesQueryCacheTests extends ESTestCase {
 
@@ -328,4 +331,86 @@ public void testStatsOnEviction() throws IOException {
         cache.close(); // this triggers some assertions
     }
 
+    private static class DummyWeight extends Weight {
+
+        private final Weight weight;
+        private boolean scorerCalled;
+        private boolean scorerSupplierCalled;
+
+        DummyWeight(Weight weight) {
+            super(weight.getQuery());
+            this.weight = weight;
+        }
+
+        @Override
+        public void extractTerms(Set<Term> terms) {
+            weight.extractTerms(terms);
+        }
+
+        @Override
+        public Explanation explain(LeafReaderContext context, int doc) throws IOException {
+            return weight.explain(context, doc);
+        }
+
+        @Override
+        public Scorer scorer(LeafReaderContext context) throws IOException {
+            scorerCalled = true;
+            return weight.scorer(context);
+        }
+
+        @Override
+        public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
+            scorerSupplierCalled = true;
+            return weight.scorerSupplier(context);
+        }
+
+        @Override
+        public float getValueForNormalization() throws IOException {
+            return weight.getValueForNormalization();
+        }
+
+        @Override
+        public void normalize(float norm, float boost) {
+            weight.normalize(norm, boost);
+        }
+
+    }
+
+    public void testDelegatesScorerSupplier() throws Exception {
+        Directory dir = newDirectory();
+        IndexWriter w = new IndexWriter(dir, newIndexWriterConfig());
+        w.addDocument(new Document());
+        DirectoryReader r = DirectoryReader.open(w);
+        w.close();
+        ShardId shard = new ShardId("index", "_na_", 0);
+        r = ElasticsearchDirectoryReader.wrap(r, shard);
+        IndexSearcher s = new IndexSearcher(r);
+        s.setQueryCachingPolicy(new QueryCachingPolicy() {
+            @Override
+            public boolean shouldCache(Query query) throws IOException {
+                return false; // never cache
+            }
+            @Override
+            public void onUse(Query query) {}
+        });
+
+        Settings settings = Settings.builder()
+                .put(IndicesQueryCache.INDICES_CACHE_QUERY_COUNT_SETTING.getKey(), 10)
+                .put(IndicesQueryCache.INDICES_QUERIES_CACHE_ALL_SEGMENTS_SETTING.getKey(), true)
+                .build();
+        IndicesQueryCache cache = new IndicesQueryCache(settings);
+        s.setQueryCache(cache);
+        Query query = new MatchAllDocsQuery();
+        final DummyWeight weight = new DummyWeight(s.createNormalizedWeight(query, false));
+        final Weight cached = cache.doCache(weight, s.getQueryCachingPolicy());
+        assertNotSame(weight, cached);
+        assertFalse(weight.scorerCalled);
+        assertFalse(weight.scorerSupplierCalled);
+        cached.scorerSupplier(s.getIndexReader().leaves().get(0));
+        assertFalse(weight.scorerCalled);
+        assertTrue(weight.scorerSupplierCalled);
+        IOUtils.close(r, dir);
+        cache.onClose(shard);
+        cache.close();
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheIT.java b/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheIT.java
index 7b9f2dbb7cf5b..b76bdf1ce392b 100644
--- a/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheIT.java
@@ -19,9 +19,12 @@
 
 package org.elasticsearch.indices;
 
+import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchType;
+import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
@@ -30,10 +33,13 @@
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.joda.time.chrono.ISOChronology;
+import org.joda.time.format.DateTimeFormat;
 
 import java.util.List;
 
 import static org.elasticsearch.search.aggregations.AggregationBuilders.dateHistogram;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.filter;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.dateRange;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
 import static org.hamcrest.Matchers.equalTo;
@@ -43,29 +49,30 @@ public class IndicesRequestCacheIT extends ESIntegTestCase {
 
     // One of the primary purposes of the query cache is to cache aggs results
     public void testCacheAggs() throws Exception {
-        assertAcked(client().admin().indices().prepareCreate("index")
+        Client client = client();
+        assertAcked(client.admin().indices().prepareCreate("index")
                 .addMapping("type", "f", "type=date")
                 .setSettings(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true).get());
         indexRandom(true,
-                client().prepareIndex("index", "type").setSource("f", "2014-03-10T00:00:00.000Z"),
-                client().prepareIndex("index", "type").setSource("f", "2014-05-13T00:00:00.000Z"));
+                client.prepareIndex("index", "type").setSource("f", "2014-03-10T00:00:00.000Z"),
+                client.prepareIndex("index", "type").setSource("f", "2014-05-13T00:00:00.000Z"));
         ensureSearchable("index");
 
         // This is not a random example: serialization with time zones writes shared strings
         // which used to not work well with the query cache because of the handles stream output
         // see #9500
-        final SearchResponse r1 = client().prepareSearch("index").setSize(0).setSearchType(SearchType.QUERY_THEN_FETCH)
+        final SearchResponse r1 = client.prepareSearch("index").setSize(0).setSearchType(SearchType.QUERY_THEN_FETCH)
                 .addAggregation(dateHistogram("histo").field("f").timeZone(DateTimeZone.forID("+01:00")).minDocCount(0)
                         .dateHistogramInterval(DateHistogramInterval.MONTH))
                 .get();
         assertSearchResponse(r1);
 
         // The cached is actually used
-        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache()
+        assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache()
             .getMemorySizeInBytes(), greaterThan(0L));
 
         for (int i = 0; i < 10; ++i) {
-            final SearchResponse r2 = client().prepareSearch("index").setSize(0)
+            final SearchResponse r2 = client.prepareSearch("index").setSize(0)
                     .setSearchType(SearchType.QUERY_THEN_FETCH).addAggregation(dateHistogram("histo").field("f")
                             .timeZone(DateTimeZone.forID("+01:00")).minDocCount(0).dateHistogramInterval(DateHistogramInterval.MONTH))
                     .get();
@@ -85,336 +92,418 @@ public void testCacheAggs() throws Exception {
     }
 
     public void testQueryRewrite() throws Exception {
-        assertAcked(client().admin().indices().prepareCreate("index").addMapping("type", "s", "type=date")
+        Client client = client();
+        assertAcked(client.admin().indices().prepareCreate("index").addMapping("type", "s", "type=date")
                 .setSettings(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true,
                         IndexMetaData.SETTING_NUMBER_OF_SHARDS, 5,
                         IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
                 .get());
-        indexRandom(true, client().prepareIndex("index", "type", "1").setRouting("1").setSource("s", "2016-03-19"),
-                client().prepareIndex("index", "type", "2").setRouting("1").setSource("s", "2016-03-20"),
-                client().prepareIndex("index", "type", "3").setRouting("1").setSource("s", "2016-03-21"),
-                client().prepareIndex("index", "type", "4").setRouting("2").setSource("s", "2016-03-22"),
-                client().prepareIndex("index", "type", "5").setRouting("2").setSource("s", "2016-03-23"),
-                client().prepareIndex("index", "type", "6").setRouting("2").setSource("s", "2016-03-24"),
-                client().prepareIndex("index", "type", "7").setRouting("3").setSource("s", "2016-03-25"),
-                client().prepareIndex("index", "type", "8").setRouting("3").setSource("s", "2016-03-26"),
-                client().prepareIndex("index", "type", "9").setRouting("3").setSource("s", "2016-03-27"));
+        indexRandom(true, client.prepareIndex("index", "type", "1").setRouting("1").setSource("s", "2016-03-19"),
+                client.prepareIndex("index", "type", "2").setRouting("1").setSource("s", "2016-03-20"),
+                client.prepareIndex("index", "type", "3").setRouting("1").setSource("s", "2016-03-21"),
+                client.prepareIndex("index", "type", "4").setRouting("2").setSource("s", "2016-03-22"),
+                client.prepareIndex("index", "type", "5").setRouting("2").setSource("s", "2016-03-23"),
+                client.prepareIndex("index", "type", "6").setRouting("2").setSource("s", "2016-03-24"),
+                client.prepareIndex("index", "type", "7").setRouting("3").setSource("s", "2016-03-25"),
+                client.prepareIndex("index", "type", "8").setRouting("3").setSource("s", "2016-03-26"),
+                client.prepareIndex("index", "type", "9").setRouting("3").setSource("s", "2016-03-27"));
         ensureSearchable("index");
 
-        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+        assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
                 equalTo(0L));
-        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+        assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
                 equalTo(0L));
 
-        final SearchResponse r1 = client().prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
-                .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-19").lte("2016-03-25")).get();
+        final SearchResponse r1 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
+                .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-19").lte("2016-03-25")).setPreFilterShardSize(Integer.MAX_VALUE)
+            .get();
         assertSearchResponse(r1);
         assertThat(r1.getHits().getTotalHits(), equalTo(7L));
-        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+        assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
                 equalTo(0L));
-        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+        assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
                 equalTo(5L));
 
-        final SearchResponse r2 = client().prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
-                .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-20").lte("2016-03-26")).get();
+        final SearchResponse r2 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
+                .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-20").lte("2016-03-26"))
+            .setPreFilterShardSize(Integer.MAX_VALUE).get();
         assertSearchResponse(r2);
         assertThat(r2.getHits().getTotalHits(), equalTo(7L));
-        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+        assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
                 equalTo(3L));
-        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+        assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
                 equalTo(7L));
 
-        final SearchResponse r3 = client().prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
-                .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-21").lte("2016-03-27")).get();
+        final SearchResponse r3 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
+                .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-21").lte("2016-03-27")).setPreFilterShardSize(Integer.MAX_VALUE)
+            .get();
         assertSearchResponse(r3);
         assertThat(r3.getHits().getTotalHits(), equalTo(7L));
-        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+        assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
                 equalTo(6L));
-        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+        assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
                 equalTo(9L));
     }
 
     public void testQueryRewriteMissingValues() throws Exception {
-        assertAcked(client().admin().indices().prepareCreate("index").addMapping("type", "s", "type=date")
+        Client client = client();
+        assertAcked(client.admin().indices().prepareCreate("index").addMapping("type", "s", "type=date")
                 .setSettings(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true, IndexMetaData.SETTING_NUMBER_OF_SHARDS,
                         1, IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
                 .get());
-        indexRandom(true, client().prepareIndex("index", "type", "1").setSource("s", "2016-03-19"),
-                client().prepareIndex("index", "type", "2").setSource("s", "2016-03-20"),
-                client().prepareIndex("index", "type", "3").setSource("s", "2016-03-21"),
-                client().prepareIndex("index", "type", "4").setSource("s", "2016-03-22"),
-                client().prepareIndex("index", "type", "5").setSource("s", "2016-03-23"),
-                client().prepareIndex("index", "type", "6").setSource("s", "2016-03-24"),
-                client().prepareIndex("index", "type", "7").setSource("other", "value"),
-                client().prepareIndex("index", "type", "8").setSource("s", "2016-03-26"),
-                client().prepareIndex("index", "type", "9").setSource("s", "2016-03-27"));
+        indexRandom(true, client.prepareIndex("index", "type", "1").setSource("s", "2016-03-19"),
+                client.prepareIndex("index", "type", "2").setSource("s", "2016-03-20"),
+                client.prepareIndex("index", "type", "3").setSource("s", "2016-03-21"),
+                client.prepareIndex("index", "type", "4").setSource("s", "2016-03-22"),
+                client.prepareIndex("index", "type", "5").setSource("s", "2016-03-23"),
+                client.prepareIndex("index", "type", "6").setSource("s", "2016-03-24"),
+                client.prepareIndex("index", "type", "7").setSource("other", "value"),
+                client.prepareIndex("index", "type", "8").setSource("s", "2016-03-26"),
+                client.prepareIndex("index", "type", "9").setSource("s", "2016-03-27"));
         ensureSearchable("index");
 
-        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+        assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
                 equalTo(0L));
-        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+        assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
                 equalTo(0L));
 
-        final SearchResponse r1 = client().prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
+        final SearchResponse r1 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
                 .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-19").lte("2016-03-28")).get();
         assertSearchResponse(r1);
         assertThat(r1.getHits().getTotalHits(), equalTo(8L));
-        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+        assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
                 equalTo(0L));
-        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+        assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
                 equalTo(1L));
 
-        final SearchResponse r2 = client().prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
+        final SearchResponse r2 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
                 .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-19").lte("2016-03-28")).get();
         assertSearchResponse(r2);
         assertThat(r2.getHits().getTotalHits(), equalTo(8L));
-        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+        assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
                 equalTo(1L));
-        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+        assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
                 equalTo(1L));
 
-        final SearchResponse r3 = client().prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
+        final SearchResponse r3 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
                 .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-19").lte("2016-03-28")).get();
         assertSearchResponse(r3);
         assertThat(r3.getHits().getTotalHits(), equalTo(8L));
-        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+        assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
                 equalTo(2L));
-        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+        assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
                 equalTo(1L));
     }
 
     public void testQueryRewriteDates() throws Exception {
-        assertAcked(client().admin().indices().prepareCreate("index").addMapping("type", "d", "type=date")
+        Client client = client();
+        assertAcked(client.admin().indices().prepareCreate("index").addMapping("type", "d", "type=date")
                 .setSettings(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true,
                         IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1,
                         IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
                 .get());
-        indexRandom(true, client().prepareIndex("index", "type", "1").setSource("d", "2014-01-01T00:00:00"),
-                client().prepareIndex("index", "type", "2").setSource("d", "2014-02-01T00:00:00"),
-                client().prepareIndex("index", "type", "3").setSource("d", "2014-03-01T00:00:00"),
-                client().prepareIndex("index", "type", "4").setSource("d", "2014-04-01T00:00:00"),
-                client().prepareIndex("index", "type", "5").setSource("d", "2014-05-01T00:00:00"),
-                client().prepareIndex("index", "type", "6").setSource("d", "2014-06-01T00:00:00"),
-                client().prepareIndex("index", "type", "7").setSource("d", "2014-07-01T00:00:00"),
-                client().prepareIndex("index", "type", "8").setSource("d", "2014-08-01T00:00:00"),
-                client().prepareIndex("index", "type", "9").setSource("d", "2014-09-01T00:00:00"));
+        indexRandom(true, client.prepareIndex("index", "type", "1").setSource("d", "2014-01-01T00:00:00"),
+                client.prepareIndex("index", "type", "2").setSource("d", "2014-02-01T00:00:00"),
+                client.prepareIndex("index", "type", "3").setSource("d", "2014-03-01T00:00:00"),
+                client.prepareIndex("index", "type", "4").setSource("d", "2014-04-01T00:00:00"),
+                client.prepareIndex("index", "type", "5").setSource("d", "2014-05-01T00:00:00"),
+                client.prepareIndex("index", "type", "6").setSource("d", "2014-06-01T00:00:00"),
+                client.prepareIndex("index", "type", "7").setSource("d", "2014-07-01T00:00:00"),
+                client.prepareIndex("index", "type", "8").setSource("d", "2014-08-01T00:00:00"),
+                client.prepareIndex("index", "type", "9").setSource("d", "2014-09-01T00:00:00"));
         ensureSearchable("index");
 
-        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+        assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
                 equalTo(0L));
-        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+        assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
                 equalTo(0L));
 
-        final SearchResponse r1 = client().prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
+        final SearchResponse r1 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
                 .setQuery(QueryBuilders.rangeQuery("d").gte("2013-01-01T00:00:00").lte("now"))
                 .get();
         assertSearchResponse(r1);
         assertThat(r1.getHits().getTotalHits(), equalTo(9L));
-        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+        assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
                 equalTo(0L));
-        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+        assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
                 equalTo(1L));
 
-        final SearchResponse r2 = client().prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
+        final SearchResponse r2 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
                 .setQuery(QueryBuilders.rangeQuery("d").gte("2013-01-01T00:00:00").lte("now"))
                 .get();
         assertSearchResponse(r2);
         assertThat(r2.getHits().getTotalHits(), equalTo(9L));
-        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+        assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
                 equalTo(1L));
-        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+        assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
                 equalTo(1L));
 
-        final SearchResponse r3 = client().prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
+        final SearchResponse r3 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
                 .setQuery(QueryBuilders.rangeQuery("d").gte("2013-01-01T00:00:00").lte("now"))
                 .get();
         assertSearchResponse(r3);
         assertThat(r3.getHits().getTotalHits(), equalTo(9L));
-        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+        assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
                 equalTo(2L));
-        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+        assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
                 equalTo(1L));
     }
 
     public void testQueryRewriteDatesWithNow() throws Exception {
-        assertAcked(client().admin().indices().prepareCreate("index-1").addMapping("type", "d", "type=date")
+        Client client = client();
+        assertAcked(client.admin().indices().prepareCreate("index-1").addMapping("type", "d", "type=date")
                 .setSettings(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true, IndexMetaData.SETTING_NUMBER_OF_SHARDS,
                         1, IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
                 .get());
-        assertAcked(client().admin().indices().prepareCreate("index-2").addMapping("type", "d", "type=date")
+        assertAcked(client.admin().indices().prepareCreate("index-2").addMapping("type", "d", "type=date")
                 .setSettings(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true, IndexMetaData.SETTING_NUMBER_OF_SHARDS,
                         1, IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
                 .get());
-        assertAcked(client().admin().indices().prepareCreate("index-3").addMapping("type", "d", "type=date")
+        assertAcked(client.admin().indices().prepareCreate("index-3").addMapping("type", "d", "type=date")
                 .setSettings(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true, IndexMetaData.SETTING_NUMBER_OF_SHARDS,
                         1, IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
                 .get());
         DateTime now = new DateTime(ISOChronology.getInstanceUTC());
-        indexRandom(true, client().prepareIndex("index-1", "type", "1").setSource("d", now),
-                client().prepareIndex("index-1", "type", "2").setSource("d", now.minusDays(1)),
-                client().prepareIndex("index-1", "type", "3").setSource("d", now.minusDays(2)),
-                client().prepareIndex("index-2", "type", "4").setSource("d", now.minusDays(3)),
-                client().prepareIndex("index-2", "type", "5").setSource("d", now.minusDays(4)),
-                client().prepareIndex("index-2", "type", "6").setSource("d", now.minusDays(5)),
-                client().prepareIndex("index-3", "type", "7").setSource("d", now.minusDays(6)),
-                client().prepareIndex("index-3", "type", "8").setSource("d", now.minusDays(7)),
-                client().prepareIndex("index-3", "type", "9").setSource("d", now.minusDays(8)));
+        indexRandom(true, client.prepareIndex("index-1", "type", "1").setSource("d", now),
+                client.prepareIndex("index-1", "type", "2").setSource("d", now.minusDays(1)),
+                client.prepareIndex("index-1", "type", "3").setSource("d", now.minusDays(2)),
+                client.prepareIndex("index-2", "type", "4").setSource("d", now.minusDays(3)),
+                client.prepareIndex("index-2", "type", "5").setSource("d", now.minusDays(4)),
+                client.prepareIndex("index-2", "type", "6").setSource("d", now.minusDays(5)),
+                client.prepareIndex("index-3", "type", "7").setSource("d", now.minusDays(6)),
+                client.prepareIndex("index-3", "type", "8").setSource("d", now.minusDays(7)),
+                client.prepareIndex("index-3", "type", "9").setSource("d", now.minusDays(8)));
         ensureSearchable("index-1", "index-2", "index-3");
 
         assertThat(
-                client().admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+                client.admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
                 equalTo(0L));
         assertThat(
-                client().admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+                client.admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
                 equalTo(0L));
 
         assertThat(
-                client().admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+                client.admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
                 equalTo(0L));
         assertThat(
-                client().admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+                client.admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
                 equalTo(0L));
         assertThat(
-                client().admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+                client.admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
                 equalTo(0L));
         assertThat(
-                client().admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+                client.admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
                 equalTo(0L));
 
-        final SearchResponse r1 = client().prepareSearch("index-*").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
+        final SearchResponse r1 = client.prepareSearch("index-*").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
                 .setQuery(QueryBuilders.rangeQuery("d").gte("now-7d/d").lte("now")).get();
         assertSearchResponse(r1);
         assertThat(r1.getHits().getTotalHits(), equalTo(8L));
         assertThat(
-                client().admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+                client.admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
                 equalTo(0L));
         assertThat(
-                client().admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+                client.admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
                 equalTo(1L));
         assertThat(
-                client().admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+                client.admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
                 equalTo(0L));
         assertThat(
-                client().admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+                client.admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
                 equalTo(1L));
         // Because the query will INTERSECT with the 3rd index it will not be
         // rewritten and will still contain `now` so won't be recorded as a
         // cache miss or cache hit since queries containing now can't be cached
         assertThat(
-                client().admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+                client.admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
                 equalTo(0L));
         assertThat(
-                client().admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+                client.admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
                 equalTo(0L));
 
-        final SearchResponse r2 = client().prepareSearch("index-*").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
+        final SearchResponse r2 = client.prepareSearch("index-*").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
                 .setQuery(QueryBuilders.rangeQuery("d").gte("now-7d/d").lte("now")).get();
         assertSearchResponse(r2);
         assertThat(r2.getHits().getTotalHits(), equalTo(8L));
         assertThat(
-                client().admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+                client.admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
                 equalTo(1L));
         assertThat(
-                client().admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+                client.admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
                 equalTo(1L));
         assertThat(
-                client().admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+                client.admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
                 equalTo(1L));
         assertThat(
-                client().admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+                client.admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
                 equalTo(1L));
         assertThat(
-                client().admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+                client.admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
                 equalTo(0L));
         assertThat(
-                client().admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+                client.admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
                 equalTo(0L));
 
-        final SearchResponse r3 = client().prepareSearch("index-*").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
+        final SearchResponse r3 = client.prepareSearch("index-*").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
                 .setQuery(QueryBuilders.rangeQuery("d").gte("now-7d/d").lte("now")).get();
         assertSearchResponse(r3);
         assertThat(r3.getHits().getTotalHits(), equalTo(8L));
         assertThat(
-                client().admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+                client.admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
                 equalTo(2L));
         assertThat(
-                client().admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+                client.admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
                 equalTo(1L));
         assertThat(
-                client().admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+                client.admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
                 equalTo(2L));
         assertThat(
-                client().admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+                client.admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
                 equalTo(1L));
         assertThat(
-                client().admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+                client.admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
                 equalTo(0L));
         assertThat(
-                client().admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+                client.admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
                 equalTo(0L));
     }
 
     public void testCanCache() throws Exception {
-        assertAcked(client().admin().indices().prepareCreate("index").addMapping("type", "s", "type=date")
+        Client client = client();
+        assertAcked(client.admin().indices().prepareCreate("index").addMapping("type", "s", "type=date")
                 .setSettings(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true, IndexMetaData.SETTING_NUMBER_OF_SHARDS,
-                        5, IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+                        2, IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
                 .get());
-        indexRandom(true, client().prepareIndex("index", "type", "1").setRouting("1").setSource("s", "2016-03-19"),
-                client().prepareIndex("index", "type", "2").setRouting("1").setSource("s", "2016-03-20"),
-                client().prepareIndex("index", "type", "3").setRouting("1").setSource("s", "2016-03-21"),
-                client().prepareIndex("index", "type", "4").setRouting("2").setSource("s", "2016-03-22"),
-                client().prepareIndex("index", "type", "5").setRouting("2").setSource("s", "2016-03-23"),
-                client().prepareIndex("index", "type", "6").setRouting("2").setSource("s", "2016-03-24"),
-                client().prepareIndex("index", "type", "7").setRouting("3").setSource("s", "2016-03-25"),
-                client().prepareIndex("index", "type", "8").setRouting("3").setSource("s", "2016-03-26"),
-                client().prepareIndex("index", "type", "9").setRouting("3").setSource("s", "2016-03-27"));
+        indexRandom(true, client.prepareIndex("index", "type", "1").setRouting("1").setSource("s", "2016-03-19"),
+                client.prepareIndex("index", "type", "2").setRouting("1").setSource("s", "2016-03-20"),
+                client.prepareIndex("index", "type", "3").setRouting("1").setSource("s", "2016-03-21"),
+                client.prepareIndex("index", "type", "4").setRouting("2").setSource("s", "2016-03-22"),
+                client.prepareIndex("index", "type", "5").setRouting("2").setSource("s", "2016-03-23"),
+                client.prepareIndex("index", "type", "6").setRouting("2").setSource("s", "2016-03-24"),
+                client.prepareIndex("index", "type", "7").setRouting("3").setSource("s", "2016-03-25"),
+                client.prepareIndex("index", "type", "8").setRouting("3").setSource("s", "2016-03-26"),
+                client.prepareIndex("index", "type", "9").setRouting("3").setSource("s", "2016-03-27"));
         ensureSearchable("index");
 
-        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+        assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
                 equalTo(0L));
-        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+        assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
                 equalTo(0L));
 
         // If size > 0 we should no cache by default
-        final SearchResponse r1 = client().prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(1)
+        final SearchResponse r1 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(1)
                 .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-19").lte("2016-03-25")).get();
         assertSearchResponse(r1);
         assertThat(r1.getHits().getTotalHits(), equalTo(7L));
-        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+        assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
                 equalTo(0L));
-        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+        assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
                 equalTo(0L));
 
         // If search type is DFS_QUERY_THEN_FETCH we should not cache
-        final SearchResponse r2 = client().prepareSearch("index").setSearchType(SearchType.DFS_QUERY_THEN_FETCH).setSize(0)
+        final SearchResponse r2 = client.prepareSearch("index").setSearchType(SearchType.DFS_QUERY_THEN_FETCH).setSize(0)
                 .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-20").lte("2016-03-26")).get();
         assertSearchResponse(r2);
         assertThat(r2.getHits().getTotalHits(), equalTo(7L));
-        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+        assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
                 equalTo(0L));
-        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+        assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
                 equalTo(0L));
 
         // If search type is DFS_QUERY_THEN_FETCH we should not cache even if
         // the cache flag is explicitly set on the request
-        final SearchResponse r3 = client().prepareSearch("index").setSearchType(SearchType.DFS_QUERY_THEN_FETCH).setSize(0)
+        final SearchResponse r3 = client.prepareSearch("index").setSearchType(SearchType.DFS_QUERY_THEN_FETCH).setSize(0)
                 .setRequestCache(true).setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-20").lte("2016-03-26")).get();
         assertSearchResponse(r3);
         assertThat(r3.getHits().getTotalHits(), equalTo(7L));
-        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+        assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
                 equalTo(0L));
-        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+        assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
                 equalTo(0L));
 
-        // If size > 1 and cache flag is set on the request we should cache
-        final SearchResponse r4 = client().prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(1)
-                .setRequestCache(true).setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-21").lte("2016-03-27")).get();
+        // If the request has an aggregation containing now we should not cache
+        final SearchResponse r4 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
+                .setRequestCache(true).setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-20").lte("2016-03-26"))
+                .addAggregation(filter("foo", QueryBuilders.rangeQuery("s").from("now-10y").to("now"))).get();
         assertSearchResponse(r4);
         assertThat(r4.getHits().getTotalHits(), equalTo(7L));
         assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
                 equalTo(0L));
         assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
-                equalTo(5L));
+                equalTo(0L));
+
+        // If the request has an aggregation containng now we should not cache
+        final SearchResponse r5 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
+                .setRequestCache(true).setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-20").lte("2016-03-26"))
+                .addAggregation(dateRange("foo").field("s").addRange("now-10y", "now")).get();
+        assertSearchResponse(r5);
+        assertThat(r5.getHits().getTotalHits(), equalTo(7L));
+        assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+                equalTo(0L));
+        assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+                equalTo(0L));
+
+        // If size > 1 and cache flag is set on the request we should cache
+        final SearchResponse r6 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(1)
+                .setRequestCache(true).setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-21").lte("2016-03-27")).get();
+        assertSearchResponse(r6);
+        assertThat(r6.getHits().getTotalHits(), equalTo(7L));
+        assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+                equalTo(0L));
+        assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+                equalTo(2L));
+    }
+
+    public void testCacheWithFilteredAlias() {
+        Client client = client();
+        assertAcked(client.admin().indices().prepareCreate("index").addMapping("type", "created_at", "type=date")
+            .setSettings(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true, IndexMetaData.SETTING_NUMBER_OF_SHARDS,
+                1, IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+            .addAlias(new Alias("last_week").filter(QueryBuilders.rangeQuery("created_at").gte("now-7d/d")))
+            .get());
+        DateTime now = new DateTime(DateTimeZone.UTC);
+        client.prepareIndex("index", "type", "1").setRouting("1").setSource("created_at",
+            DateTimeFormat.forPattern("YYYY-MM-dd").print(now)).get();
+        refresh();
+
+        assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+            equalTo(0L));
+        assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+            equalTo(0L));
+
+        SearchResponse r1 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
+            .setQuery(QueryBuilders.rangeQuery("created_at").gte("now-7d/d")).get();
+        assertSearchResponse(r1);
+        assertThat(r1.getHits().getTotalHits(), equalTo(1L));
+        assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+            equalTo(0L));
+        assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+            equalTo(1L));
+
+        r1 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
+            .setQuery(QueryBuilders.rangeQuery("created_at").gte("now-7d/d")).get();
+        assertSearchResponse(r1);
+        assertThat(r1.getHits().getTotalHits(), equalTo(1L));
+        assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+            equalTo(1L));
+        assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+            equalTo(1L));
+
+        r1 = client.prepareSearch("last_week").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0).get();
+        assertSearchResponse(r1);
+        assertThat(r1.getHits().getTotalHits(), equalTo(1L));
+        assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+            equalTo(1L));
+        assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+            equalTo(2L));
+
+        r1 = client.prepareSearch("last_week").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0).get();
+        assertSearchResponse(r1);
+        assertThat(r1.getHits().getTotalHits(), equalTo(1L));
+        assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+            equalTo(2L));
+        assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+            equalTo(2L));
     }
 
 }
diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheTests.java b/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheTests.java
index d43217d978504..3c0674609c57c 100644
--- a/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheTests.java
@@ -31,8 +31,7 @@
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
@@ -44,6 +43,7 @@
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Supplier;
 
 public class IndicesRequestCacheTests extends ESTestCase {
 
@@ -60,23 +60,25 @@ public void testBasicOperationsCache() throws Exception {
         AtomicBoolean indexShard = new AtomicBoolean(true);
 
         // initial cache
-        TestEntity entity = new TestEntity(requestCacheStats, reader, indexShard, 0);
-        BytesReference value = cache.getOrCompute(entity, reader, termQuery.buildAsBytes());
+        TestEntity entity = new TestEntity(requestCacheStats, indexShard);
+        Loader loader = new Loader(reader, 0);
+        BytesReference value = cache.getOrCompute(entity, loader, reader, termQuery.buildAsBytes());
         assertEquals("foo", value.streamInput().readString());
         assertEquals(0, requestCacheStats.stats().getHitCount());
         assertEquals(1, requestCacheStats.stats().getMissCount());
         assertEquals(0, requestCacheStats.stats().getEvictions());
-        assertFalse(entity.loadedFromCache());
+        assertFalse(loader.loadedFromCache);
         assertEquals(1, cache.count());
 
         // cache hit
-        entity = new TestEntity(requestCacheStats, reader, indexShard, 0);
-        value = cache.getOrCompute(entity, reader, termQuery.buildAsBytes());
+        entity = new TestEntity(requestCacheStats, indexShard);
+        loader = new Loader(reader, 0);
+        value = cache.getOrCompute(entity, loader, reader, termQuery.buildAsBytes());
         assertEquals("foo", value.streamInput().readString());
         assertEquals(1, requestCacheStats.stats().getHitCount());
         assertEquals(1, requestCacheStats.stats().getMissCount());
         assertEquals(0, requestCacheStats.stats().getEvictions());
-        assertTrue(entity.loadedFromCache());
+        assertTrue(loader.loadedFromCache);
         assertEquals(1, cache.count());
         assertTrue(requestCacheStats.stats().getMemorySize().bytesAsInt() > value.length());
         assertEquals(1, cache.numRegisteredCloseListeners());
@@ -92,7 +94,7 @@ public void testBasicOperationsCache() throws Exception {
         assertEquals(1, requestCacheStats.stats().getHitCount());
         assertEquals(1, requestCacheStats.stats().getMissCount());
         assertEquals(0, requestCacheStats.stats().getEvictions());
-        assertTrue(entity.loadedFromCache());
+        assertTrue(loader.loadedFromCache);
         assertEquals(0, cache.count());
         assertEquals(0, requestCacheStats.stats().getMemorySize().bytesAsInt());
 
@@ -115,46 +117,50 @@ public void testCacheDifferentReaders() throws Exception {
         DirectoryReader secondReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
 
         // initial cache
-        TestEntity entity = new TestEntity(requestCacheStats, reader, indexShard, 0);
-        BytesReference value = cache.getOrCompute(entity, reader, termQuery.buildAsBytes());
+        TestEntity entity = new TestEntity(requestCacheStats, indexShard);
+        Loader loader = new Loader(reader, 0);
+        BytesReference value = cache.getOrCompute(entity, loader, reader, termQuery.buildAsBytes());
         assertEquals("foo", value.streamInput().readString());
         assertEquals(0, requestCacheStats.stats().getHitCount());
         assertEquals(1, requestCacheStats.stats().getMissCount());
         assertEquals(0, requestCacheStats.stats().getEvictions());
-        assertFalse(entity.loadedFromCache());
+        assertFalse(loader.loadedFromCache);
         assertEquals(1, cache.count());
         assertTrue(requestCacheStats.stats().getMemorySize().bytesAsInt() > value.length());
         final int cacheSize = requestCacheStats.stats().getMemorySize().bytesAsInt();
         assertEquals(1, cache.numRegisteredCloseListeners());
 
         // cache the second
-        TestEntity secondEntity = new TestEntity(requestCacheStats, secondReader, indexShard, 0);
-        value = cache.getOrCompute(secondEntity, secondReader, termQuery.buildAsBytes());
+        TestEntity secondEntity = new TestEntity(requestCacheStats, indexShard);
+        loader = new Loader(secondReader, 0);
+        value = cache.getOrCompute(entity, loader, secondReader, termQuery.buildAsBytes());
         assertEquals("bar", value.streamInput().readString());
         assertEquals(0, requestCacheStats.stats().getHitCount());
         assertEquals(2, requestCacheStats.stats().getMissCount());
         assertEquals(0, requestCacheStats.stats().getEvictions());
-        assertFalse(secondEntity.loadedFromCache());
+        assertFalse(loader.loadedFromCache);
         assertEquals(2, cache.count());
         assertTrue(requestCacheStats.stats().getMemorySize().bytesAsInt() > cacheSize + value.length());
         assertEquals(2, cache.numRegisteredCloseListeners());
 
-        secondEntity = new TestEntity(requestCacheStats, secondReader, indexShard, 0);
-        value = cache.getOrCompute(secondEntity, secondReader, termQuery.buildAsBytes());
+        secondEntity = new TestEntity(requestCacheStats, indexShard);
+        loader = new Loader(secondReader, 0);
+        value = cache.getOrCompute(secondEntity, loader, secondReader, termQuery.buildAsBytes());
         assertEquals("bar", value.streamInput().readString());
         assertEquals(1, requestCacheStats.stats().getHitCount());
         assertEquals(2, requestCacheStats.stats().getMissCount());
         assertEquals(0, requestCacheStats.stats().getEvictions());
-        assertTrue(secondEntity.loadedFromCache());
+        assertTrue(loader.loadedFromCache);
         assertEquals(2, cache.count());
 
-        entity = new TestEntity(requestCacheStats, reader, indexShard, 0);
-        value = cache.getOrCompute(entity, reader, termQuery.buildAsBytes());
+        entity = new TestEntity(requestCacheStats, indexShard);
+        loader = new Loader(reader, 0);
+        value = cache.getOrCompute(entity, loader, reader, termQuery.buildAsBytes());
         assertEquals("foo", value.streamInput().readString());
         assertEquals(2, requestCacheStats.stats().getHitCount());
         assertEquals(2, requestCacheStats.stats().getMissCount());
         assertEquals(0, requestCacheStats.stats().getEvictions());
-        assertTrue(entity.loadedFromCache());
+        assertTrue(loader.loadedFromCache);
         assertEquals(2, cache.count());
 
         // Closing the cache doesn't change returned entities
@@ -162,8 +168,7 @@ public void testCacheDifferentReaders() throws Exception {
         cache.cleanCache();
         assertEquals(2, requestCacheStats.stats().getMissCount());
         assertEquals(0, requestCacheStats.stats().getEvictions());
-        assertTrue(entity.loadedFromCache());
-        assertTrue(secondEntity.loadedFromCache());
+        assertTrue(loader.loadedFromCache);
         assertEquals(1, cache.count());
         assertEquals(cacheSize, requestCacheStats.stats().getMemorySize().bytesAsInt());
         assertEquals(1, cache.numRegisteredCloseListeners());
@@ -179,8 +184,7 @@ public void testCacheDifferentReaders() throws Exception {
         cache.cleanCache();
         assertEquals(2, requestCacheStats.stats().getMissCount());
         assertEquals(0, requestCacheStats.stats().getEvictions());
-        assertTrue(entity.loadedFromCache());
-        assertTrue(secondEntity.loadedFromCache());
+        assertTrue(loader.loadedFromCache);
         assertEquals(0, cache.count());
         assertEquals(0, requestCacheStats.stats().getMemorySize().bytesAsInt());
 
@@ -201,22 +205,24 @@ public void testEviction() throws Exception {
             DirectoryReader reader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer),
                 new ShardId("foo", "bar", 1));
             TermQueryBuilder termQuery = new TermQueryBuilder("id", "0");
-            TestEntity entity = new TestEntity(requestCacheStats, reader, indexShard, 0);
+            TestEntity entity = new TestEntity(requestCacheStats, indexShard);
+            Loader loader = new Loader(reader, 0);
 
             writer.updateDocument(new Term("id", "0"), newDoc(0, "bar"));
             DirectoryReader secondReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer),
                 new ShardId("foo", "bar", 1));
-            TestEntity secondEntity = new TestEntity(requestCacheStats, secondReader, indexShard, 0);
+            TestEntity secondEntity = new TestEntity(requestCacheStats, indexShard);
+            Loader secondLoader = new Loader(secondReader, 0);
 
-            BytesReference value1 = cache.getOrCompute(entity, reader, termQuery.buildAsBytes());
+            BytesReference value1 = cache.getOrCompute(entity, loader, reader, termQuery.buildAsBytes());
             assertEquals("foo", value1.streamInput().readString());
-            BytesReference value2 = cache.getOrCompute(secondEntity, secondReader, termQuery.buildAsBytes());
+            BytesReference value2 = cache.getOrCompute(secondEntity, secondLoader, secondReader, termQuery.buildAsBytes());
             assertEquals("bar", value2.streamInput().readString());
             size = requestCacheStats.stats().getMemorySize();
             IOUtils.close(reader, secondReader, writer, dir, cache);
         }
         IndicesRequestCache cache = new IndicesRequestCache(Settings.builder()
-            .put(IndicesRequestCache.INDICES_CACHE_QUERY_SIZE.getKey(), size.bytes()+1 +"b")
+            .put(IndicesRequestCache.INDICES_CACHE_QUERY_SIZE.getKey(), size.getBytes()+1 +"b")
             .build());
         AtomicBoolean indexShard = new AtomicBoolean(true);
         ShardRequestCache requestCacheStats = new ShardRequestCache();
@@ -227,24 +233,27 @@ public void testEviction() throws Exception {
         DirectoryReader reader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer),
             new ShardId("foo", "bar", 1));
         TermQueryBuilder termQuery = new TermQueryBuilder("id", "0");
-        TestEntity entity = new TestEntity(requestCacheStats, reader, indexShard, 0);
+        TestEntity entity = new TestEntity(requestCacheStats, indexShard);
+        Loader loader = new Loader(reader, 0);
 
         writer.updateDocument(new Term("id", "0"), newDoc(0, "bar"));
         DirectoryReader secondReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer),
             new ShardId("foo", "bar", 1));
-        TestEntity secondEntity = new TestEntity(requestCacheStats, secondReader, indexShard, 0);
+        TestEntity secondEntity = new TestEntity(requestCacheStats, indexShard);
+        Loader secondLoader = new Loader(secondReader, 0);
 
         writer.updateDocument(new Term("id", "0"), newDoc(0, "baz"));
         DirectoryReader thirdReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer),
             new ShardId("foo", "bar", 1));
-        TestEntity thirddEntity = new TestEntity(requestCacheStats, thirdReader, indexShard, 0);
+        TestEntity thirddEntity = new TestEntity(requestCacheStats, indexShard);
+        Loader thirdLoader = new Loader(thirdReader, 0);
 
-        BytesReference value1 = cache.getOrCompute(entity, reader, termQuery.buildAsBytes());
+        BytesReference value1 = cache.getOrCompute(entity, loader, reader, termQuery.buildAsBytes());
         assertEquals("foo", value1.streamInput().readString());
-        BytesReference value2 = cache.getOrCompute(secondEntity, secondReader, termQuery.buildAsBytes());
+        BytesReference value2 = cache.getOrCompute(secondEntity, secondLoader, secondReader, termQuery.buildAsBytes());
         assertEquals("bar", value2.streamInput().readString());
         logger.info("Memory size: {}", requestCacheStats.stats().getMemorySize());
-        BytesReference value3 = cache.getOrCompute(thirddEntity, thirdReader, termQuery.buildAsBytes());
+        BytesReference value3 = cache.getOrCompute(thirddEntity, thirdLoader, thirdReader, termQuery.buildAsBytes());
         assertEquals("baz", value3.streamInput().readString());
         assertEquals(2, cache.count());
         assertEquals(1, requestCacheStats.stats().getEvictions());
@@ -263,25 +272,28 @@ public void testClearAllEntityIdentity() throws Exception {
         DirectoryReader reader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer),
             new ShardId("foo", "bar", 1));
         TermQueryBuilder termQuery = new TermQueryBuilder("id", "0");
-        TestEntity entity = new TestEntity(requestCacheStats, reader, indexShard, 0);
+        TestEntity entity = new TestEntity(requestCacheStats, indexShard);
+        Loader loader = new Loader(reader, 0);
 
         writer.updateDocument(new Term("id", "0"), newDoc(0, "bar"));
         DirectoryReader secondReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer),
             new ShardId("foo", "bar", 1));
-        TestEntity secondEntity = new TestEntity(requestCacheStats, secondReader, indexShard, 0);
+        TestEntity secondEntity = new TestEntity(requestCacheStats, indexShard);
+        Loader secondLoader = new Loader(secondReader, 0);
 
         writer.updateDocument(new Term("id", "0"), newDoc(0, "baz"));
         DirectoryReader thirdReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer),
             new ShardId("foo", "bar", 1));
         AtomicBoolean differentIdentity =  new AtomicBoolean(true);
-        TestEntity thirddEntity = new TestEntity(requestCacheStats, thirdReader, differentIdentity, 0);
+        TestEntity thirddEntity = new TestEntity(requestCacheStats, differentIdentity);
+        Loader thirdLoader = new Loader(thirdReader, 0);
 
-        BytesReference value1 = cache.getOrCompute(entity, reader, termQuery.buildAsBytes());
+        BytesReference value1 = cache.getOrCompute(entity, loader, reader, termQuery.buildAsBytes());
         assertEquals("foo", value1.streamInput().readString());
-        BytesReference value2 = cache.getOrCompute(secondEntity, secondReader, termQuery.buildAsBytes());
+        BytesReference value2 = cache.getOrCompute(secondEntity, secondLoader, secondReader, termQuery.buildAsBytes());
         assertEquals("bar", value2.streamInput().readString());
         logger.info("Memory size: {}", requestCacheStats.stats().getMemorySize());
-        BytesReference value3 = cache.getOrCompute(thirddEntity, thirdReader, termQuery.buildAsBytes());
+        BytesReference value3 = cache.getOrCompute(thirddEntity, thirdLoader, thirdReader, termQuery.buildAsBytes());
         assertEquals("baz", value3.streamInput().readString());
         assertEquals(3, cache.count());
         final long hitCount = requestCacheStats.stats().getHitCount();
@@ -290,7 +302,7 @@ public void testClearAllEntityIdentity() throws Exception {
         cache.cleanCache();
         assertEquals(1, cache.count());
         // third has not been validated since it's a different identity
-        value3 = cache.getOrCompute(thirddEntity, thirdReader, termQuery.buildAsBytes());
+        value3 = cache.getOrCompute(thirddEntity, thirdLoader, thirdReader, termQuery.buildAsBytes());
         assertEquals(hitCount + 1, requestCacheStats.stats().getHitCount());
         assertEquals("baz", value3.streamInput().readString());
 
@@ -304,20 +316,107 @@ public Iterable<Field> newDoc(int id, String value) {
             StringField.TYPE_STORED));
     }
 
+    private static class Loader implements Supplier<BytesReference> {
+
+        private final DirectoryReader reader;
+        private final int id;
+        public boolean loadedFromCache = true;
+
+        Loader(DirectoryReader reader, int id) {
+            super();
+            this.reader = reader;
+            this.id = id;
+        }
+
+        @Override
+        public BytesReference get() {
+            try (BytesStreamOutput out = new BytesStreamOutput()) {
+                IndexSearcher searcher = new IndexSearcher(reader);
+                TopDocs topDocs = searcher.search(new TermQuery(new Term("id", Integer.toString(id))), 1);
+                assertEquals(1, topDocs.totalHits);
+                Document document = reader.document(topDocs.scoreDocs[0].doc);
+                out.writeString(document.get("value"));
+                loadedFromCache = false;
+                return out.bytes();
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        }
+
+    }
+
+    public void testInvalidate() throws Exception {
+        ShardRequestCache requestCacheStats = new ShardRequestCache();
+        IndicesRequestCache cache = new IndicesRequestCache(Settings.EMPTY);
+        Directory dir = newDirectory();
+        IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
+
+        writer.addDocument(newDoc(0, "foo"));
+        DirectoryReader reader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer),
+            new ShardId("foo", "bar", 1));
+        TermQueryBuilder termQuery = new TermQueryBuilder("id", "0");
+        AtomicBoolean indexShard = new AtomicBoolean(true);
+
+        // initial cache
+        TestEntity entity = new TestEntity(requestCacheStats, indexShard);
+        Loader loader = new Loader(reader, 0);
+        BytesReference value = cache.getOrCompute(entity, loader, reader, termQuery.buildAsBytes());
+        assertEquals("foo", value.streamInput().readString());
+        assertEquals(0, requestCacheStats.stats().getHitCount());
+        assertEquals(1, requestCacheStats.stats().getMissCount());
+        assertEquals(0, requestCacheStats.stats().getEvictions());
+        assertFalse(loader.loadedFromCache);
+        assertEquals(1, cache.count());
+
+        // cache hit
+        entity = new TestEntity(requestCacheStats, indexShard);
+        loader = new Loader(reader, 0);
+        value = cache.getOrCompute(entity, loader, reader, termQuery.buildAsBytes());
+        assertEquals("foo", value.streamInput().readString());
+        assertEquals(1, requestCacheStats.stats().getHitCount());
+        assertEquals(1, requestCacheStats.stats().getMissCount());
+        assertEquals(0, requestCacheStats.stats().getEvictions());
+        assertTrue(loader.loadedFromCache);
+        assertEquals(1, cache.count());
+        assertTrue(requestCacheStats.stats().getMemorySize().bytesAsInt() > value.length());
+        assertEquals(1, cache.numRegisteredCloseListeners());
+
+        // load again after invalidate
+        entity = new TestEntity(requestCacheStats, indexShard);
+        loader = new Loader(reader, 0);
+        cache.invalidate(entity, reader,  termQuery.buildAsBytes());
+        value = cache.getOrCompute(entity, loader, reader, termQuery.buildAsBytes());
+        assertEquals("foo", value.streamInput().readString());
+        assertEquals(1, requestCacheStats.stats().getHitCount());
+        assertEquals(2, requestCacheStats.stats().getMissCount());
+        assertEquals(0, requestCacheStats.stats().getEvictions());
+        assertFalse(loader.loadedFromCache);
+        assertEquals(1, cache.count());
+        assertTrue(requestCacheStats.stats().getMemorySize().bytesAsInt() > value.length());
+        assertEquals(1, cache.numRegisteredCloseListeners());
+
+        // release
+        if (randomBoolean()) {
+            reader.close();
+        } else {
+            indexShard.set(false); // closed shard but reader is still open
+            cache.clear(entity);
+        }
+        cache.cleanCache();
+        assertEquals(1, requestCacheStats.stats().getHitCount());
+        assertEquals(2, requestCacheStats.stats().getMissCount());
+        assertEquals(0, requestCacheStats.stats().getEvictions());
+        assertEquals(0, cache.count());
+        assertEquals(0, requestCacheStats.stats().getMemorySize().bytesAsInt());
+
+        IOUtils.close(reader, writer, dir, cache);
+        assertEquals(0, cache.numRegisteredCloseListeners());
+    }
+
     private class TestEntity extends AbstractIndexShardCacheEntity {
         private final AtomicBoolean standInForIndexShard;
         private final ShardRequestCache shardRequestCache;
-        private TestEntity(ShardRequestCache shardRequestCache, DirectoryReader reader, AtomicBoolean standInForIndexShard, int id) {
-            super(new Loader() {
-                @Override
-                public void load(StreamOutput out) throws IOException {
-                    IndexSearcher searcher = new IndexSearcher(reader);
-                    TopDocs topDocs = searcher.search(new TermQuery(new Term("id", Integer.toString(id))), 1);
-                    assertEquals(1, topDocs.totalHits);
-                    Document document = reader.document(topDocs.scoreDocs[0].doc);
-                    out.writeString(document.get("value"));
-                }
-            });
+        private TestEntity(ShardRequestCache shardRequestCache, AtomicBoolean standInForIndexShard) {
             this.standInForIndexShard = standInForIndexShard;
             this.shardRequestCache = shardRequestCache;
         }
@@ -336,5 +435,10 @@ public boolean isOpen() {
         public Object getCacheIdentity() {
             return standInForIndexShard;
         }
+
+        @Override
+        public long ramBytesUsed() {
+            return 42;
+        }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java b/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java
index 577b20c1faecd..03eab6ffca9d6 100644
--- a/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java
@@ -18,7 +18,10 @@
  */
 package org.elasticsearch.indices;
 
+import org.apache.lucene.store.AlreadyClosedException;
 import org.elasticsearch.Version;
+import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
+import org.elasticsearch.action.admin.indices.stats.IndexShardStats;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexGraveyard;
@@ -35,15 +38,31 @@
 import org.elasticsearch.gateway.LocalAllocateDangledIndices;
 import org.elasticsearch.gateway.MetaStateService;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.mapper.Mapper;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.StringFieldMapper;
+import org.elasticsearch.index.shard.IllegalIndexShardStateException;
+import org.elasticsearch.index.shard.IndexShard;
+import org.elasticsearch.index.shard.IndexShardState;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.ShardPath;
+import org.elasticsearch.index.similarity.BM25SimilarityProvider;
+import org.elasticsearch.indices.IndicesService.ShardDeletionCheckResult;
+import org.elasticsearch.plugins.MapperPlugin;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
@@ -52,7 +71,10 @@
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.not;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 public class IndicesServiceTests extends ESSingleNodeTestCase {
 
@@ -64,6 +86,30 @@ public NodeEnvironment getNodeEnvironment() {
         return getInstanceFromNode(NodeEnvironment.class);
     }
 
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        ArrayList<Class<? extends Plugin>> plugins = new ArrayList<>(super.getPlugins());
+        plugins.add(TestPlugin.class);
+        return plugins;
+    }
+
+    public static class TestPlugin extends Plugin implements MapperPlugin {
+
+        public TestPlugin() {}
+
+        @Override
+        public Map<String, Mapper.TypeParser> getMappers() {
+            return Collections.singletonMap("fake-mapper", new StringFieldMapper.TypeParser());
+        }
+
+        @Override
+        public void onIndexModule(IndexModule indexModule) {
+            super.onIndexModule(indexModule);
+            indexModule.addSimilarity("fake-similarity", BM25SimilarityProvider::new);
+        }
+    }
+
+
     @Override
     protected boolean resetNodeAfterTest() {
         return true;
@@ -92,16 +138,19 @@ public void testCanDeleteShardContent() {
                 1).build();
         IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("test", meta.getSettings());
         ShardId shardId = new ShardId(meta.getIndex(), 0);
-        assertFalse("no shard location", indicesService.canDeleteShardContent(shardId, indexSettings));
+        assertEquals("no shard location", indicesService.canDeleteShardContent(shardId, indexSettings),
+            ShardDeletionCheckResult.NO_FOLDER_FOUND);
         IndexService test = createIndex("test");
         shardId = new ShardId(test.index(), 0);
         assertTrue(test.hasShard(0));
-        assertFalse("shard is allocated", indicesService.canDeleteShardContent(shardId, test.getIndexSettings()));
+        assertEquals("shard is allocated", indicesService.canDeleteShardContent(shardId, test.getIndexSettings()),
+            ShardDeletionCheckResult.STILL_ALLOCATED);
         test.removeShard(0, "boom");
-        assertTrue("shard is removed", indicesService.canDeleteShardContent(shardId, test.getIndexSettings()));
+        assertEquals("shard is removed", indicesService.canDeleteShardContent(shardId, test.getIndexSettings()),
+            ShardDeletionCheckResult.FOLDER_FOUND_CAN_DELETE);
         ShardId notAllocated = new ShardId(test.index(), 100);
-        assertFalse("shard that was never on this node should NOT be deletable",
-            indicesService.canDeleteShardContent(notAllocated, test.getIndexSettings()));
+        assertEquals("shard that was never on this node should NOT be deletable",
+            indicesService.canDeleteShardContent(notAllocated, test.getIndexSettings()), ShardDeletionCheckResult.NO_FOLDER_FOUND);
     }
 
     public void testDeleteIndexStore() throws Exception {
@@ -324,4 +373,79 @@ public void onFailure(Throwable e) {
         }
     }
 
+    /**
+     * Tests that teh {@link MapperService} created by {@link IndicesService#createIndexMapperService(IndexMetaData)} contains
+     * custom types and similarities registered by plugins
+     */
+    public void testStandAloneMapperServiceWithPlugins() throws IOException {
+        final String indexName = "test";
+        final Index index = new Index(indexName, UUIDs.randomBase64UUID());
+        final IndicesService indicesService = getIndicesService();
+        final Settings idxSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+            .put(IndexMetaData.SETTING_INDEX_UUID, index.getUUID())
+            .put(IndexModule.SIMILARITY_SETTINGS_PREFIX + ".test.type", "fake-similarity")
+            .build();
+        final IndexMetaData indexMetaData = new IndexMetaData.Builder(index.getName())
+            .settings(idxSettings)
+            .numberOfShards(1)
+            .numberOfReplicas(0)
+            .build();
+        MapperService mapperService = indicesService.createIndexMapperService(indexMetaData);
+        assertNotNull(mapperService.documentMapperParser().parserContext("type").typeParser("fake-mapper"));
+        assertThat(mapperService.documentMapperParser().parserContext("type").getSimilarity("test"),
+            instanceOf(BM25SimilarityProvider.class));
+    }
+
+    public void testStatsByShardDoesNotDieFromExpectedExceptions() {
+        final int shardCount = randomIntBetween(2, 5);
+        final int failedShardId = randomIntBetween(0, shardCount - 1);
+
+        final Index index = new Index("test-index", "abc123");
+        // the shard that is going to fail
+        final ShardId shardId = new ShardId(index, failedShardId);
+
+        final List<IndexShard> shards = new ArrayList<>(shardCount);
+        final List<IndexShardStats> shardStats = new ArrayList<>(shardCount - 1);
+
+        final IndexShardState state = randomFrom(IndexShardState.values());
+        final String message = "TEST - expected";
+
+        final RuntimeException expectedException =
+                randomFrom(new IllegalIndexShardStateException(shardId, state, message), new AlreadyClosedException(message));
+
+        // this allows us to control the indices that exist
+        final IndicesService mockIndicesService = mock(IndicesService.class);
+        final IndexService indexService = mock(IndexService.class);
+
+        // generate fake shards and their responses
+        for (int i = 0; i < shardCount; ++i) {
+            final IndexShard shard = mock(IndexShard.class);
+
+            shards.add(shard);
+
+            if (failedShardId != i) {
+                final IndexShardStats successfulShardStats = mock(IndexShardStats.class);
+
+                shardStats.add(successfulShardStats);
+
+                when(mockIndicesService.indexShardStats(mockIndicesService, shard, CommonStatsFlags.ALL)).thenReturn(successfulShardStats);
+            } else {
+                when(mockIndicesService.indexShardStats(mockIndicesService, shard, CommonStatsFlags.ALL)).thenThrow(expectedException);
+            }
+        }
+
+        when(mockIndicesService.iterator()).thenReturn(Collections.singleton(indexService).iterator());
+        when(indexService.iterator()).thenReturn(shards.iterator());
+        when(indexService.index()).thenReturn(index);
+
+        // real one, which has a logger defined
+        final IndicesService indicesService = getIndicesService();
+
+        final Map<Index, List<IndexShardStats>> indexStats = indicesService.statsByShard(mockIndicesService, CommonStatsFlags.ALL);
+
+        assertThat(indexStats.isEmpty(), equalTo(false));
+        assertThat("index not defined", indexStats.containsKey(index), equalTo(true));
+        assertThat("unexpected shard stats", indexStats.get(index), equalTo(shardStats));
+    }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/indices/NodeIndicesStatsTests.java b/core/src/test/java/org/elasticsearch/indices/NodeIndicesStatsTests.java
new file mode 100644
index 0000000000000..b8ae55a44d28e
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/indices/NodeIndicesStatsTests.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.indices;
+
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Collections;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.object.HasToString.hasToString;
+
+public class NodeIndicesStatsTests extends ESTestCase {
+
+    public void testInvalidLevel() {
+        final NodeIndicesStats stats = new NodeIndicesStats();
+        final String level = randomAlphaOfLength(16);
+        final ToXContent.Params params = new ToXContent.MapParams(Collections.singletonMap("level", level));
+        final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> stats.toXContent(null, params));
+        assertThat(
+            e,
+            hasToString(containsString("level parameter must be one of [indices] or [node] or [shards] but was [" + level + "]")));
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/indices/TermsLookupTests.java b/core/src/test/java/org/elasticsearch/indices/TermsLookupTests.java
index fea69133377f0..07fe5e00a2482 100644
--- a/core/src/test/java/org/elasticsearch/indices/TermsLookupTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/TermsLookupTests.java
@@ -21,7 +21,6 @@
 
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.indices.TermsLookup;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
@@ -30,11 +29,11 @@
 
 public class TermsLookupTests extends ESTestCase {
     public void testTermsLookup() {
-        String index = randomAsciiOfLengthBetween(1, 10);
-        String type = randomAsciiOfLengthBetween(1, 10);
-        String id = randomAsciiOfLengthBetween(1, 10);
-        String path = randomAsciiOfLengthBetween(1, 10);
-        String routing = randomAsciiOfLengthBetween(1, 10);
+        String index = randomAlphaOfLengthBetween(1, 10);
+        String type = randomAlphaOfLengthBetween(1, 10);
+        String id = randomAlphaOfLengthBetween(1, 10);
+        String path = randomAlphaOfLengthBetween(1, 10);
+        String routing = randomAlphaOfLengthBetween(1, 10);
         TermsLookup termsLookup = new TermsLookup(index, type, id, path);
         termsLookup.routing(routing);
         assertEquals(index, termsLookup.index());
@@ -45,9 +44,9 @@ public void testTermsLookup() {
     }
 
     public void testIllegalArguments() {
-        String type = randomAsciiOfLength(5);
-        String id = randomAsciiOfLength(5);
-        String path = randomAsciiOfLength(5);
+        String type = randomAlphaOfLength(5);
+        String id = randomAlphaOfLength(5);
+        String path = randomAlphaOfLength(5);
         switch (randomIntBetween(0, 2)) {
         case 0:
             type = null;
@@ -81,10 +80,10 @@ public void testSerialization() throws IOException {
 
     public static TermsLookup randomTermsLookup() {
         return new TermsLookup(
-                randomBoolean() ? randomAsciiOfLength(10) : null,
-                randomAsciiOfLength(10),
-                randomAsciiOfLength(10),
-                randomAsciiOfLength(10).replace('.', '_')
-        ).routing(randomBoolean() ? randomAsciiOfLength(10) : null);
+                randomBoolean() ? randomAlphaOfLength(10) : null,
+                randomAlphaOfLength(10),
+                randomAlphaOfLength(10),
+                randomAlphaOfLength(10).replace('.', '_')
+        ).routing(randomBoolean() ? randomAlphaOfLength(10) : null);
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java b/core/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java
index 33763571daf38..acd1cc42bb233 100644
--- a/core/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java
@@ -37,13 +37,14 @@
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.inject.ModuleTestCase;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.Analysis;
 import org.elasticsearch.index.analysis.AnalysisRegistry;
-import org.elasticsearch.index.analysis.AnalysisService;
 import org.elasticsearch.index.analysis.AnalysisTestsHelper;
 import org.elasticsearch.index.analysis.CustomAnalyzer;
+import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.analysis.MappingCharFilterFactory;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.analysis.PatternReplaceCharFilterFactory;
@@ -79,11 +80,11 @@
  */
 public class AnalysisModuleTests extends ModuleTestCase {
 
-    public AnalysisService getAnalysisService(Settings settings) throws IOException {
-        return getAnalysisService(getNewRegistry(settings), settings);
+    public IndexAnalyzers getIndexAnalyzers(Settings settings) throws IOException {
+        return getIndexAnalyzers(getNewRegistry(settings), settings);
     }
 
-    public AnalysisService getAnalysisService(AnalysisRegistry registry, Settings settings) throws IOException {
+    public IndexAnalyzers getIndexAnalyzers(AnalysisRegistry registry, Settings settings) throws IOException {
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("test", settings);
         return registry.build(idxSettings);
     }
@@ -136,9 +137,13 @@ public void testAnalyzerAlias() throws IOException {
             .put(IndexMetaData.SETTING_VERSION_CREATED, VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.V_2_3_5))
             .build();
         AnalysisRegistry newRegistry = getNewRegistry(settings);
-        AnalysisService as = getAnalysisService(newRegistry, settings);
-        assertThat(as.analyzer("default").analyzer(), is(instanceOf(KeywordAnalyzer.class)));
-        assertThat(as.analyzer("default_search").analyzer(), is(instanceOf(EnglishAnalyzer.class)));
+        IndexAnalyzers indexAnalyzers = getIndexAnalyzers(newRegistry, settings);
+        assertThat(indexAnalyzers.get("default").analyzer(), is(instanceOf(KeywordAnalyzer.class)));
+        assertThat(indexAnalyzers.get("default_search").analyzer(), is(instanceOf(EnglishAnalyzer.class)));
+        assertWarnings("setting [index.analysis.analyzer.foobar.alias] is only allowed on index [test] because it was created before " +
+                        "5.x; analyzer aliases can no longer be created on new indices.",
+                "setting [index.analysis.analyzer.foobar_search.alias] is only allowed on index [test] because it was created before " +
+                        "5.x; analyzer aliases can no longer be created on new indices.");
     }
 
     public void testAnalyzerAliasReferencesAlias() throws IOException {
@@ -152,10 +157,15 @@ public void testAnalyzerAliasReferencesAlias() throws IOException {
             .put(IndexMetaData.SETTING_VERSION_CREATED, VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.V_2_3_5))
             .build();
         AnalysisRegistry newRegistry = getNewRegistry(settings);
-        AnalysisService as = getAnalysisService(newRegistry, settings);
-        assertThat(as.analyzer("default").analyzer(), is(instanceOf(GermanAnalyzer.class)));
+        IndexAnalyzers indexAnalyzers = getIndexAnalyzers(newRegistry, settings);
+
+        assertThat(indexAnalyzers.get("default").analyzer(), is(instanceOf(GermanAnalyzer.class)));
         // analyzer types are bound early before we resolve aliases
-        assertThat(as.analyzer("default_search").analyzer(), is(instanceOf(StandardAnalyzer.class)));
+        assertThat(indexAnalyzers.get("default_search").analyzer(), is(instanceOf(StandardAnalyzer.class)));
+        assertWarnings("setting [index.analysis.analyzer.foobar.alias] is only allowed on index [test] because it was created before " +
+                "5.x; analyzer aliases can no longer be created on new indices.",
+                "setting [index.analysis.analyzer.foobar_search.alias] is only allowed on index [test] because it was created before " +
+                        "5.x; analyzer aliases can no longer be created on new indices.");
     }
 
     public void testAnalyzerAliasDefault() throws IOException {
@@ -167,9 +177,11 @@ public void testAnalyzerAliasDefault() throws IOException {
             .put(IndexMetaData.SETTING_VERSION_CREATED, VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.V_2_3_5))
             .build();
         AnalysisRegistry newRegistry = getNewRegistry(settings);
-        AnalysisService as = getAnalysisService(newRegistry, settings);
-        assertThat(as.analyzer("default").analyzer(), is(instanceOf(KeywordAnalyzer.class)));
-        assertThat(as.analyzer("default_search").analyzer(), is(instanceOf(KeywordAnalyzer.class)));
+        IndexAnalyzers indexAnalyzers = getIndexAnalyzers(newRegistry, settings);
+        assertThat(indexAnalyzers.get("default").analyzer(), is(instanceOf(KeywordAnalyzer.class)));
+        assertThat(indexAnalyzers.get("default_search").analyzer(), is(instanceOf(KeywordAnalyzer.class)));
+        assertWarnings("setting [index.analysis.analyzer.foobar.alias] is only allowed on index [test] because it was created before " +
+                "5.x; analyzer aliases can no longer be created on new indices.");
     }
 
     public void testAnalyzerAliasMoreThanOnce() throws IOException {
@@ -183,8 +195,12 @@ public void testAnalyzerAliasMoreThanOnce() throws IOException {
             .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
             .build();
         AnalysisRegistry newRegistry = getNewRegistry(settings);
-        IllegalStateException ise = expectThrows(IllegalStateException.class, () -> getAnalysisService(newRegistry, settings));
+        IllegalStateException ise = expectThrows(IllegalStateException.class, () -> getIndexAnalyzers(newRegistry, settings));
         assertEquals("alias [default] is already used by [foobar]", ise.getMessage());
+        assertWarnings("setting [index.analysis.analyzer.foobar.alias] is only allowed on index [test] because it was created before " +
+                "5.x; analyzer aliases can no longer be created on new indices.",
+                "setting [index.analysis.analyzer.foobar1.alias] is only allowed on index [test] because it was created before " +
+                        "5.x; analyzer aliases can no longer be created on new indices.");
     }
 
     public void testAnalyzerAliasNotAllowedPost5x() throws IOException {
@@ -192,11 +208,11 @@ public void testAnalyzerAliasNotAllowedPost5x() throws IOException {
             .put("index.analysis.analyzer.foobar.type", "standard")
             .put("index.analysis.analyzer.foobar.alias","foobaz")
             // analyzer aliases were removed in v5.0.0 alpha6
-            .put(IndexMetaData.SETTING_VERSION_CREATED, VersionUtils.randomVersionBetween(random(), Version.V_5_0_0_alpha6, null))
+            .put(IndexMetaData.SETTING_VERSION_CREATED, VersionUtils.randomVersionBetween(random(), Version.V_5_0_0_beta1, null))
             .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
             .build();
         AnalysisRegistry registry = getNewRegistry(settings);
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> getAnalysisService(registry, settings));
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> getIndexAnalyzers(registry, settings));
         assertEquals("setting [index.analysis.analyzer.foobar.alias] is not supported", e.getMessage());
     }
 
@@ -208,7 +224,7 @@ public void testVersionedAnalyzers() throws Exception {
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_0_0)
                 .build();
         AnalysisRegistry newRegistry = getNewRegistry(settings2);
-        AnalysisService analysisService2 = getAnalysisService(newRegistry, settings2);
+        IndexAnalyzers indexAnalyzers = getIndexAnalyzers(newRegistry, settings2);
 
         // registry always has the current version
         assertThat(newRegistry.getAnalyzer("default"), is(instanceOf(NamedAnalyzer.class)));
@@ -217,20 +233,20 @@ public void testVersionedAnalyzers() throws Exception {
         assertEquals(Version.CURRENT.luceneVersion, defaultNamedAnalyzer.analyzer().getVersion());
 
         // analysis service has the expected version
-        assertThat(analysisService2.analyzer("standard").analyzer(), is(instanceOf(StandardAnalyzer.class)));
-        assertEquals(Version.V_2_0_0.luceneVersion, analysisService2.analyzer("standard").analyzer().getVersion());
-        assertEquals(Version.V_2_0_0.luceneVersion, analysisService2.analyzer("thai").analyzer().getVersion());
+        assertThat(indexAnalyzers.get("standard").analyzer(), is(instanceOf(StandardAnalyzer.class)));
+        assertEquals(Version.V_2_0_0.luceneVersion, indexAnalyzers.get("standard").analyzer().getVersion());
+        assertEquals(Version.V_2_0_0.luceneVersion, indexAnalyzers.get("thai").analyzer().getVersion());
 
-        assertThat(analysisService2.analyzer("custom7").analyzer(), is(instanceOf(StandardAnalyzer.class)));
-        assertEquals(org.apache.lucene.util.Version.fromBits(3,6,0), analysisService2.analyzer("custom7").analyzer().getVersion());
+        assertThat(indexAnalyzers.get("custom7").analyzer(), is(instanceOf(StandardAnalyzer.class)));
+        assertEquals(org.apache.lucene.util.Version.fromBits(3,6,0), indexAnalyzers.get("custom7").analyzer().getVersion());
     }
 
     private void assertTokenFilter(String name, Class<?> clazz) throws IOException {
         Settings settings = Settings.builder()
                                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
                                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build();
-        AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(settings);
-        TokenFilterFactory tokenFilter = analysisService.tokenFilter(name);
+        TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(settings);
+        TokenFilterFactory tokenFilter = analysis.tokenFilter.get(name);
         Tokenizer tokenizer = new WhitespaceTokenizer();
         tokenizer.setReader(new StringReader("foo bar"));
         TokenStream stream = tokenFilter.create(tokenizer);
@@ -238,8 +254,8 @@ private void assertTokenFilter(String name, Class<?> clazz) throws IOException {
     }
 
     private void testSimpleConfiguration(Settings settings) throws IOException {
-        AnalysisService analysisService = getAnalysisService(settings);
-        Analyzer analyzer = analysisService.analyzer("custom1").analyzer();
+        IndexAnalyzers indexAnalyzers = getIndexAnalyzers(settings);
+        Analyzer analyzer = indexAnalyzers.get("custom1").analyzer();
 
         assertThat(analyzer, instanceOf(CustomAnalyzer.class));
         CustomAnalyzer custom1 = (CustomAnalyzer) analyzer;
@@ -249,23 +265,23 @@ private void testSimpleConfiguration(Settings settings) throws IOException {
         StopTokenFilterFactory stop1 = (StopTokenFilterFactory) custom1.tokenFilters()[0];
         assertThat(stop1.stopWords().size(), equalTo(1));
 
-        analyzer = analysisService.analyzer("custom2").analyzer();
+        analyzer = indexAnalyzers.get("custom2").analyzer();
         assertThat(analyzer, instanceOf(CustomAnalyzer.class));
 
         // verify position increment gap
-        analyzer = analysisService.analyzer("custom6").analyzer();
+        analyzer = indexAnalyzers.get("custom6").analyzer();
         assertThat(analyzer, instanceOf(CustomAnalyzer.class));
         CustomAnalyzer custom6 = (CustomAnalyzer) analyzer;
         assertThat(custom6.getPositionIncrementGap("any_string"), equalTo(256));
 
         // verify characters  mapping
-        analyzer = analysisService.analyzer("custom5").analyzer();
+        analyzer = indexAnalyzers.get("custom5").analyzer();
         assertThat(analyzer, instanceOf(CustomAnalyzer.class));
         CustomAnalyzer custom5 = (CustomAnalyzer) analyzer;
         assertThat(custom5.charFilters()[0], instanceOf(MappingCharFilterFactory.class));
 
         // check custom pattern replace filter
-        analyzer = analysisService.analyzer("custom3").analyzer();
+        analyzer = indexAnalyzers.get("custom3").analyzer();
         assertThat(analyzer, instanceOf(CustomAnalyzer.class));
         CustomAnalyzer custom3 = (CustomAnalyzer) analyzer;
         PatternReplaceCharFilterFactory patternReplaceCharFilterFactory = (PatternReplaceCharFilterFactory) custom3.charFilters()[0];
@@ -273,7 +289,7 @@ private void testSimpleConfiguration(Settings settings) throws IOException {
         assertThat(patternReplaceCharFilterFactory.getReplacement(), equalTo("replacedSample $1"));
 
         // check custom class name (my)
-        analyzer = analysisService.analyzer("custom4").analyzer();
+        analyzer = indexAnalyzers.get("custom4").analyzer();
         assertThat(analyzer, instanceOf(CustomAnalyzer.class));
         CustomAnalyzer custom4 = (CustomAnalyzer) analyzer;
         assertThat(custom4.tokenFilters()[0], instanceOf(MyFilterTokenFilterFactory.class));
@@ -307,7 +323,8 @@ public void testWordListPath() throws Exception {
         String[] words = new String[]{"donau", "dampf", "schiff", "spargel", "creme", "suppe"};
 
         Path wordListFile = generateWordList(words);
-        settings = Settings.builder().loadFromSource("index: \n  word_list_path: " + wordListFile.toAbsolutePath()).build();
+        settings = Settings.builder().loadFromSource("index: \n  word_list_path: " + wordListFile.toAbsolutePath(), XContentType.YAML)
+            .build();
 
         Set<?> wordList = Analysis.getWordSet(env, settings, "index.word_list");
         MatcherAssert.assertThat(wordList.size(), equalTo(6));
@@ -333,7 +350,7 @@ public void testUnderscoreInAnalyzerName() throws IOException {
                 .put(IndexMetaData.SETTING_VERSION_CREATED, "1")
                 .build();
         try {
-            getAnalysisService(settings);
+            getIndexAnalyzers(settings);
             fail("This should fail with IllegalArgumentException because the analyzers name starts with _");
         } catch (IllegalArgumentException e) {
             assertThat(e.getMessage(), either(equalTo("analyzer name must not start with '_'. got \"_invalid_name\""))
@@ -350,11 +367,13 @@ public void testUnderscoreInAnalyzerNameAlias() throws IOException {
                 .put(IndexMetaData.SETTING_VERSION_CREATED, VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.V_2_3_5))
                 .build();
         try {
-            getAnalysisService(settings);
+            getIndexAnalyzers(settings);
             fail("This should fail with IllegalArgumentException because the analyzers alias starts with _");
         } catch (IllegalArgumentException e) {
             assertThat(e.getMessage(), equalTo("analyzer name must not start with '_'. got \"_invalid_name\""));
         }
+        assertWarnings("setting [index.analysis.analyzer.valid_name.alias] is only allowed on index [test] because it was " +
+                "created before 5.x; analyzer aliases can no longer be created on new indices.");
     }
 
     public void testDeprecatedPositionOffsetGap() throws IOException {
@@ -365,7 +384,7 @@ public void testDeprecatedPositionOffsetGap() throws IOException {
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
                 .build();
         try {
-            getAnalysisService(settings);
+            getIndexAnalyzers(settings);
             fail("Analyzer should fail if it has position_offset_gap");
         } catch (IllegalArgumentException e) {
             assertThat(e.getMessage(), equalTo("Option [position_offset_gap] in Custom Analyzer [custom] " +
diff --git a/core/src/test/java/org/elasticsearch/indices/analysis/PreBuiltAnalyzerIntegrationIT.java b/core/src/test/java/org/elasticsearch/indices/analysis/PreBuiltAnalyzerIntegrationIT.java
index 6967a88a21ab7..d6e93ce559e11 100644
--- a/core/src/test/java/org/elasticsearch/indices/analysis/PreBuiltAnalyzerIntegrationIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/analysis/PreBuiltAnalyzerIntegrationIT.java
@@ -25,7 +25,6 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.test.ESBackcompatTestCase;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
 
@@ -55,7 +54,7 @@ public void testThatPreBuiltAnalyzersAreNotClosedOnIndexClose() throws Exception
         List<String> indexNames = new ArrayList<>();
         final int numIndices = scaledRandomIntBetween(2, 4);
         for (int i = 0; i < numIndices; i++) {
-            String indexName = randomAsciiOfLength(10).toLowerCase(Locale.ROOT);
+            String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
             indexNames.add(indexName);
 
             int randomInt = randomInt(PreBuiltAnalyzers.values().length-1);
@@ -92,7 +91,7 @@ public void testThatPreBuiltAnalyzersAreNotClosedOnIndexClose() throws Exception
             String randomId = randomInt() + "";
 
             Map<String, Object> data = new HashMap<>();
-            data.put("foo", randomAsciiOfLength(scaledRandomIntBetween(5, 50)));
+            data.put("foo", randomAlphaOfLength(scaledRandomIntBetween(5, 50)));
 
             index(randomIndex, "type", randomId, data);
         }
diff --git a/core/src/test/java/org/elasticsearch/indices/analyze/AnalyzeActionIT.java b/core/src/test/java/org/elasticsearch/indices/analyze/AnalyzeActionIT.java
index cd2c34e5102ae..d414ca917d80b 100644
--- a/core/src/test/java/org/elasticsearch/indices/analyze/AnalyzeActionIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/analyze/AnalyzeActionIT.java
@@ -53,21 +53,25 @@ public void testSimpleAnalyzerTests() throws Exception {
             assertThat(token.getStartOffset(), equalTo(0));
             assertThat(token.getEndOffset(), equalTo(4));
             assertThat(token.getPosition(), equalTo(0));
+            assertThat(token.getPositionLength(), equalTo(1));
             token = analyzeResponse.getTokens().get(1);
             assertThat(token.getTerm(), equalTo("is"));
             assertThat(token.getStartOffset(), equalTo(5));
             assertThat(token.getEndOffset(), equalTo(7));
             assertThat(token.getPosition(), equalTo(1));
+            assertThat(token.getPositionLength(), equalTo(1));
             token = analyzeResponse.getTokens().get(2);
             assertThat(token.getTerm(), equalTo("a"));
             assertThat(token.getStartOffset(), equalTo(8));
             assertThat(token.getEndOffset(), equalTo(9));
             assertThat(token.getPosition(), equalTo(2));
+            assertThat(token.getPositionLength(), equalTo(1));
             token = analyzeResponse.getTokens().get(3);
             assertThat(token.getTerm(), equalTo("test"));
             assertThat(token.getStartOffset(), equalTo(10));
             assertThat(token.getEndOffset(), equalTo(14));
             assertThat(token.getPosition(), equalTo(3));
+            assertThat(token.getPositionLength(), equalTo(1));
         }
     }
 
@@ -107,7 +111,7 @@ public void testAnalyzeWithNoIndex() throws Exception {
         assertThat(analyzeResponse.getTokens().get(0).getPosition(), equalTo(1));
         assertThat(analyzeResponse.getTokens().get(0).getStartOffset(), equalTo(3));
         assertThat(analyzeResponse.getTokens().get(0).getEndOffset(), equalTo(9));
-
+        assertThat(analyzeResponse.getTokens().get(0).getPositionLength(), equalTo(1));
     }
 
     public void testAnalyzeWithCharFilters() throws Exception {
@@ -140,6 +144,54 @@ public void testAnalyzeWithCharFilters() throws Exception {
         assertThat(token.getTerm(), equalTo("fish"));
     }
 
+    public void testAnalyzeWithNonDefaultPostionLength() throws Exception {
+        assertAcked(prepareCreate("test").addAlias(new Alias("alias"))
+            .setSettings(Settings.builder().put(indexSettings())
+                .put("index.analysis.filter.syns.type", "synonym")
+                .putArray("index.analysis.filter.syns.synonyms", "wtf, what the fudge")
+                .put("index.analysis.analyzer.custom_syns.tokenizer", "standard")
+                .putArray("index.analysis.analyzer.custom_syns.filter", "lowercase", "syns")));
+        ensureGreen();
+
+        AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze("say what the fudge").setIndex("test").setAnalyzer("custom_syns").get();
+        assertThat(analyzeResponse.getTokens().size(), equalTo(5));
+
+        AnalyzeResponse.AnalyzeToken token = analyzeResponse.getTokens().get(0);
+        assertThat(token.getTerm(), equalTo("say"));
+        assertThat(token.getPosition(), equalTo(0));
+        assertThat(token.getStartOffset(), equalTo(0));
+        assertThat(token.getEndOffset(), equalTo(3));
+        assertThat(token.getPositionLength(), equalTo(1));
+
+        token = analyzeResponse.getTokens().get(1);
+        assertThat(token.getTerm(), equalTo("what"));
+        assertThat(token.getPosition(), equalTo(1));
+        assertThat(token.getStartOffset(), equalTo(4));
+        assertThat(token.getEndOffset(), equalTo(8));
+        assertThat(token.getPositionLength(), equalTo(1));
+
+        token = analyzeResponse.getTokens().get(2);
+        assertThat(token.getTerm(), equalTo("wtf"));
+        assertThat(token.getPosition(), equalTo(1));
+        assertThat(token.getStartOffset(), equalTo(4));
+        assertThat(token.getEndOffset(), equalTo(18));
+        assertThat(token.getPositionLength(), equalTo(3));
+
+        token = analyzeResponse.getTokens().get(3);
+        assertThat(token.getTerm(), equalTo("the"));
+        assertThat(token.getPosition(), equalTo(2));
+        assertThat(token.getStartOffset(), equalTo(9));
+        assertThat(token.getEndOffset(), equalTo(12));
+        assertThat(token.getPositionLength(), equalTo(1));
+
+        token = analyzeResponse.getTokens().get(4);
+        assertThat(token.getTerm(), equalTo("fudge"));
+        assertThat(token.getPosition(), equalTo(3));
+        assertThat(token.getStartOffset(), equalTo(13));
+        assertThat(token.getEndOffset(), equalTo(18));
+        assertThat(token.getPositionLength(), equalTo(1));
+    }
+
     public void testAnalyzerWithFieldOrTypeTests() throws Exception {
         assertAcked(prepareCreate("test").addAlias(new Alias("alias")));
         ensureGreen();
@@ -157,6 +209,7 @@ public void testAnalyzerWithFieldOrTypeTests() throws Exception {
             assertThat(token.getTerm(), equalTo("test"));
             assertThat(token.getStartOffset(), equalTo(10));
             assertThat(token.getEndOffset(), equalTo(14));
+            assertThat(token.getPositionLength(), equalTo(1));
         }
     }
 
@@ -203,13 +256,14 @@ public void testAnalyzerWithMultiValues() throws Exception {
         assertThat(token.getPosition(), equalTo(3));
         assertThat(token.getStartOffset(), equalTo(10));
         assertThat(token.getEndOffset(), equalTo(14));
+        assertThat(token.getPositionLength(), equalTo(1));
 
         token = analyzeResponse.getTokens().get(5);
         assertThat(token.getTerm(), equalTo("second"));
         assertThat(token.getPosition(), equalTo(105));
         assertThat(token.getStartOffset(), equalTo(19));
         assertThat(token.getEndOffset(), equalTo(25));
-
+        assertThat(token.getPositionLength(), equalTo(1));
     }
 
     public void testDetailAnalyze() throws Exception {
@@ -353,12 +407,14 @@ public void testDetailAnalyzeWithMultiValues() throws Exception {
         assertThat(token.getPosition(), equalTo(3));
         assertThat(token.getStartOffset(), equalTo(10));
         assertThat(token.getEndOffset(), equalTo(14));
+        assertThat(token.getPositionLength(), equalTo(1));
 
         token = analyzeResponse.detail().analyzer().getTokens()[5];
         assertThat(token.getTerm(), equalTo("second"));
         assertThat(token.getPosition(), equalTo(105));
         assertThat(token.getStartOffset(), equalTo(19));
         assertThat(token.getEndOffset(), equalTo(25));
+        assertThat(token.getPositionLength(), equalTo(1));
     }
 
     public void testDetailAnalyzeWithMultiValuesWithCustomAnalyzer() throws Exception {
@@ -398,12 +454,14 @@ public void testDetailAnalyzeWithMultiValuesWithCustomAnalyzer() throws Exceptio
         assertThat(token.getPosition(), equalTo(3));
         assertThat(token.getStartOffset(), equalTo(10));
         assertThat(token.getEndOffset(), equalTo(15));
+        assertThat(token.getPositionLength(), equalTo(1));
 
         token = analyzeResponse.detail().tokenizer().getTokens()[5];
         assertThat(token.getTerm(), equalTo("troubled"));
         assertThat(token.getPosition(), equalTo(105));
         assertThat(token.getStartOffset(), equalTo(20));
         assertThat(token.getEndOffset(), equalTo(28));
+        assertThat(token.getPositionLength(), equalTo(1));
 
         // tokenfilter(snowball)
         assertThat(analyzeResponse.detail().tokenfilters().length, equalTo(2));
@@ -415,12 +473,14 @@ public void testDetailAnalyzeWithMultiValuesWithCustomAnalyzer() throws Exceptio
         assertThat(token.getPosition(), equalTo(3));
         assertThat(token.getStartOffset(), equalTo(10));
         assertThat(token.getEndOffset(), equalTo(15));
+        assertThat(token.getPositionLength(), equalTo(1));
 
         token = analyzeResponse.detail().tokenfilters()[0].getTokens()[5];
         assertThat(token.getTerm(), equalTo("troubl"));
         assertThat(token.getPosition(), equalTo(105));
         assertThat(token.getStartOffset(), equalTo(20));
         assertThat(token.getEndOffset(), equalTo(28));
+        assertThat(token.getPositionLength(), equalTo(1));
 
         // tokenfilter(lowercase)
         assertThat(analyzeResponse.detail().tokenfilters()[1].getName(), equalTo("lowercase"));
@@ -431,14 +491,14 @@ public void testDetailAnalyzeWithMultiValuesWithCustomAnalyzer() throws Exceptio
         assertThat(token.getPosition(), equalTo(3));
         assertThat(token.getStartOffset(), equalTo(10));
         assertThat(token.getEndOffset(), equalTo(15));
+        assertThat(token.getPositionLength(), equalTo(1));
 
         token = analyzeResponse.detail().tokenfilters()[0].getTokens()[5];
         assertThat(token.getTerm(), equalTo("troubl"));
         assertThat(token.getPosition(), equalTo(105));
         assertThat(token.getStartOffset(), equalTo(20));
         assertThat(token.getEndOffset(), equalTo(28));
-
-
+        assertThat(token.getPositionLength(), equalTo(1));
     }
 
     public void testNonExistTokenizer() {
@@ -471,16 +531,19 @@ public void testCustomTokenFilterInRequest() throws Exception {
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getStartOffset(), equalTo(0));
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getEndOffset(), equalTo(3));
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getPosition(), equalTo(0));
+        assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getPositionLength(), equalTo(1));
 
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getTerm(), equalTo("buzz"));
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getStartOffset(), equalTo(4));
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getEndOffset(), equalTo(8));
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getPosition(), equalTo(1));
+        assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getPositionLength(), equalTo(1));
 
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getTerm(), equalTo("test"));
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getStartOffset(), equalTo(9));
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getEndOffset(), equalTo(13));
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getPosition(), equalTo(2));
+        assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getPositionLength(), equalTo(1));
 
         // tokenfilter(lowercase)
         assertThat(analyzeResponse.detail().tokenfilters().length, equalTo(2));
@@ -490,16 +553,19 @@ public void testCustomTokenFilterInRequest() throws Exception {
         assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getStartOffset(), equalTo(0));
         assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getEndOffset(), equalTo(3));
         assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getPosition(), equalTo(0));
+        assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getPositionLength(), equalTo(1));
 
         assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[1].getTerm(), equalTo("buzz"));
         assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[1].getStartOffset(), equalTo(4));
         assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[1].getEndOffset(), equalTo(8));
         assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[1].getPosition(), equalTo(1));
+        assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[1].getPositionLength(), equalTo(1));
 
         assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getTerm(), equalTo("test"));
         assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getStartOffset(), equalTo(9));
         assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getEndOffset(), equalTo(13));
         assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getPosition(), equalTo(2));
+        assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getPositionLength(), equalTo(1));
 
         // tokenfilter({"type": "stop", "stopwords": ["foo", "buzz"]})
         assertThat(analyzeResponse.detail().tokenfilters()[1].getName(), equalTo("_anonymous_tokenfilter_[1]"));
@@ -509,6 +575,7 @@ public void testCustomTokenFilterInRequest() throws Exception {
         assertThat(analyzeResponse.detail().tokenfilters()[1].getTokens()[0].getStartOffset(), equalTo(9));
         assertThat(analyzeResponse.detail().tokenfilters()[1].getTokens()[0].getEndOffset(), equalTo(13));
         assertThat(analyzeResponse.detail().tokenfilters()[1].getTokens()[0].getPosition(), equalTo(2));
+        assertThat(analyzeResponse.detail().tokenfilters()[1].getTokens()[0].getPositionLength(), equalTo(1));
     }
 
 
@@ -536,6 +603,7 @@ public void testCustomCharFilterInRequest() throws Exception {
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getTerm(), equalTo("jeff qit fish"));
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getStartOffset(), equalTo(0));
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getEndOffset(), equalTo(15));
+        assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getPositionLength(), equalTo(1));
     }
 
 
@@ -559,16 +627,51 @@ public void testCustomTokenizerInRequest() throws Exception {
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getStartOffset(), equalTo(0));
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getEndOffset(), equalTo(2));
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getPosition(), equalTo(0));
+        assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getPositionLength(), equalTo(1));
 
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getTerm(), equalTo("oo"));
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getStartOffset(), equalTo(1));
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getEndOffset(), equalTo(3));
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getPosition(), equalTo(1));
+        assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getPositionLength(), equalTo(1));
 
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getTerm(), equalTo("od"));
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getStartOffset(), equalTo(2));
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getEndOffset(), equalTo(4));
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getPosition(), equalTo(2));
+        assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getPositionLength(), equalTo(1));
+    }
+
+    public void testAnalyzeKeywordField() throws IOException {
+        assertAcked(prepareCreate("test").addAlias(new Alias("alias")).addMapping("test", "keyword", "type=keyword"));
+        ensureGreen("test");
+
+        AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze(indexOrAlias(), "ABC").setField("keyword").get();
+        assertThat(analyzeResponse.getTokens().size(), equalTo(1));
+        AnalyzeResponse.AnalyzeToken token = analyzeResponse.getTokens().get(0);
+        assertThat(token.getTerm(), equalTo("ABC"));
+        assertThat(token.getStartOffset(), equalTo(0));
+        assertThat(token.getEndOffset(), equalTo(3));
+        assertThat(token.getPosition(), equalTo(0));
+        assertThat(token.getPositionLength(), equalTo(1));
     }
 
+    public void testAnalyzeNormalizedKeywordField() throws IOException {
+        assertAcked(prepareCreate("test").addAlias(new Alias("alias"))
+            .setSettings(Settings.builder().put(indexSettings())
+                .put("index.analysis.normalizer.my_normalizer.type", "custom")
+                .putArray("index.analysis.normalizer.my_normalizer.filter", "lowercase"))
+            .addMapping("test", "keyword", "type=keyword,normalizer=my_normalizer"));
+        ensureGreen("test");
+
+        AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze(indexOrAlias(), "ABC").setField("keyword").get();
+        assertThat(analyzeResponse.getTokens().size(), equalTo(1));
+        AnalyzeResponse.AnalyzeToken token = analyzeResponse.getTokens().get(0);
+        assertThat(token.getTerm(), equalTo("abc"));
+        assertThat(token.getStartOffset(), equalTo(0));
+        assertThat(token.getEndOffset(), equalTo(3));
+        assertThat(token.getPosition(), equalTo(0));
+        assertThat(token.getPositionLength(), equalTo(1));
+
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java b/core/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java
index 5cb57152a3cd0..8a7ae99e7bdc0 100644
--- a/core/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java
+++ b/core/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java
@@ -26,21 +26,19 @@
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.util.Callback;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.NodeServicesProvider;
 import org.elasticsearch.index.shard.IndexEventListener;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.IndexShardState;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndex;
-import org.elasticsearch.indices.cluster.IndicesClusterStateService.Shard;
 import org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices;
-import org.elasticsearch.indices.recovery.RecoveryState;
+import org.elasticsearch.indices.cluster.IndicesClusterStateService.Shard;
 import org.elasticsearch.indices.recovery.PeerRecoveryTargetService;
+import org.elasticsearch.indices.recovery.RecoveryState;
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
@@ -51,11 +49,13 @@
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentMap;
+import java.util.function.Consumer;
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.unmodifiableMap;
 import static org.elasticsearch.common.collect.MapBuilder.newMapBuilder;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 
 /**
  * Abstract base class for tests against {@link IndicesClusterStateService}
@@ -87,7 +87,11 @@ public void assertClusterStateMatchesNodeState(ClusterState state, IndicesCluste
         RoutingNode localRoutingNode = state.getRoutingNodes().node(state.getNodes().getLocalNodeId());
         if (localRoutingNode != null) {
             if (enableRandomFailures == false) {
-                assertTrue("failed shard cache should be empty", failedShardsCache.isEmpty());
+                // initializing a shard should succeed when enableRandomFailures is disabled
+                // active shards can be failed if state persistence was disabled in an earlier CS update
+                if (failedShardsCache.values().stream().anyMatch(ShardRouting::initializing)) {
+                    fail("failed shard cache should not contain initializing shard routing: " + failedShardsCache.values());
+                }
             }
             // check that all shards in local routing nodes have been allocated
             for (ShardRouting shardRouting : localRoutingNode) {
@@ -96,30 +100,38 @@ public void assertClusterStateMatchesNodeState(ClusterState state, IndicesCluste
 
                 Shard shard = indicesService.getShardOrNull(shardRouting.shardId());
                 ShardRouting failedShard = failedShardsCache.get(shardRouting.shardId());
-                if (enableRandomFailures) {
-                    if (shard == null && failedShard == null) {
-                        fail("Shard with id " + shardRouting + " expected but missing in indicesService and failedShardsCache");
+
+                if (state.blocks().disableStatePersistence()) {
+                    if (shard != null) {
+                        fail("Shard with id " + shardRouting + " should be removed from indicesService due to disabled state persistence");
                     }
+                } else {
                     if (failedShard != null && failedShard.isSameAllocation(shardRouting) == false) {
                         fail("Shard cache has not been properly cleaned for " + failedShard);
                     }
-                } else {
-                    if (shard == null) {
-                        fail("Shard with id " + shardRouting + " expected but missing in indicesService");
+                    if (shard == null && failedShard == null) {
+                        // shard must either be there or there must be a failure
+                        fail("Shard with id " + shardRouting + " expected but missing in indicesService and failedShardsCache");
+                    }
+                    if (enableRandomFailures == false) {
+                        if (shard == null && shardRouting.initializing() && failedShard == shardRouting) {
+                            // initializing a shard should succeed when enableRandomFailures is disabled
+                            fail("Shard with id " + shardRouting + " expected but missing in indicesService " + failedShard);
+                        }
                     }
-                }
 
-                if (shard != null) {
-                    AllocatedIndex<? extends Shard> indexService = indicesService.indexService(index);
-                    assertTrue("Index " + index + " expected but missing in indicesService", indexService != null);
-
-                    // index metadata has been updated
-                    assertThat(indexService.getIndexSettings().getIndexMetaData(), equalTo(indexMetaData));
-                    // shard has been created
-                    if (enableRandomFailures == false || failedShard == null) {
-                        assertTrue("Shard with id " + shardRouting + " expected but missing in indexService", shard != null);
-                        // shard has latest shard routing
-                        assertThat(shard.routingEntry(), equalTo(shardRouting));
+                    if (shard != null) {
+                        AllocatedIndex<? extends Shard> indexService = indicesService.indexService(index);
+                        assertTrue("Index " + index + " expected but missing in indicesService", indexService != null);
+
+                        // index metadata has been updated
+                        assertThat(indexService.getIndexSettings().getIndexMetaData(), equalTo(indexMetaData));
+                        // shard has been created
+                        if (enableRandomFailures == false || failedShard == null) {
+                            assertTrue("Shard with id " + shardRouting + " expected but missing in indexService", shard != null);
+                            // shard has latest shard routing
+                            assertThat(shard.routingEntry(), equalTo(shardRouting));
+                        }
                     }
                 }
             }
@@ -127,6 +139,10 @@ public void assertClusterStateMatchesNodeState(ClusterState state, IndicesCluste
 
         // all other shards / indices have been cleaned up
         for (AllocatedIndex<? extends Shard> indexService : indicesService) {
+            if (state.blocks().disableStatePersistence()) {
+                fail("Index service " + indexService.index() + " should be removed from indicesService due to disabled state persistence");
+            }
+
             assertTrue(state.metaData().getIndexSafe(indexService.index()) != null);
 
             boolean shardsFound = false;
@@ -143,13 +159,9 @@ public void assertClusterStateMatchesNodeState(ClusterState state, IndicesCluste
             }
 
             if (shardsFound == false) {
-                if (enableRandomFailures) {
-                    // check if we have shards of that index in failedShardsCache
-                    // if yes, we might not have cleaned the index as failedShardsCache can be populated by another thread
-                    assertFalse(failedShardsCache.keySet().stream().noneMatch(shardId -> shardId.getIndex().equals(indexService.index())));
-                } else {
-                    fail("index service for index " + indexService.index() + " has no shards");
-                }
+                // check if we have shards of that index in failedShardsCache
+                // if yes, we might not have cleaned the index as failedShardsCache can be populated by another thread
+                assertFalse(failedShardsCache.keySet().stream().noneMatch(shardId -> shardId.getIndex().equals(indexService.index())));
             }
 
         }
@@ -162,7 +174,7 @@ protected class MockIndicesService implements AllocatedIndices<MockIndexShard, M
         private volatile Map<String, MockIndexService> indices = emptyMap();
 
         @Override
-        public synchronized MockIndexService createIndex(NodeServicesProvider nodeServicesProvider, IndexMetaData indexMetaData,
+        public synchronized MockIndexService createIndex(IndexMetaData indexMetaData,
                                                          List<IndexEventListener> buildInIndexListener) throws IOException {
             MockIndexService indexService = new MockIndexService(new IndexSettings(indexMetaData, Settings.EMPTY));
             indices = newMapBuilder(indices).put(indexMetaData.getIndexUUID(), indexService).immutableMap();
@@ -180,23 +192,12 @@ public void deleteUnassignedIndex(String reason, IndexMetaData metaData, Cluster
         }
 
         @Override
-        public synchronized void deleteIndex(Index index, String reason) {
-            if (hasIndex(index) == false) {
-                return;
-            }
-            Map<String, MockIndexService> newIndices = new HashMap<>(indices);
-            newIndices.remove(index.getUUID());
-            indices = unmodifiableMap(newIndices);
-        }
-
-        @Override
-        public synchronized void removeIndex(Index index, String reason) {
-            if (hasIndex(index) == false) {
-                return;
+        public synchronized void removeIndex(Index index, IndexRemovalReason reason, String extraInfo) {
+            if (hasIndex(index)) {
+                Map<String, MockIndexService> newIndices = new HashMap<>(indices);
+                newIndices.remove(index.getUUID());
+                indices = unmodifiableMap(newIndices);
             }
-            Map<String, MockIndexService> newIndices = new HashMap<>(indices);
-            newIndices.remove(index.getUUID());
-            indices = unmodifiableMap(newIndices);
         }
 
         @Override
@@ -210,8 +211,7 @@ public MockIndexShard createShard(ShardRouting shardRouting, RecoveryState recov
                                           PeerRecoveryTargetService recoveryTargetService,
                                           PeerRecoveryTargetService.RecoveryListener recoveryListener,
                                           RepositoriesService repositoriesService,
-                                          NodeServicesProvider nodeServicesProvider, Callback<IndexShard.ShardFailure> onShardFailure)
-            throws IOException {
+                                          Consumer<IndexShard.ShardFailure> onShardFailure) throws IOException {
             failRandomly();
             MockIndexService indexService = indexService(recoveryState.getShardId().getIndex());
             MockIndexShard indexShard = indexService.createShard(shardRouting);
@@ -261,6 +261,9 @@ public boolean updateMapping(IndexMetaData indexMetaData) throws IOException {
         @Override
         public void updateMetaData(IndexMetaData indexMetaData) {
             indexSettings.updateIndexMetaData(indexMetaData);
+            for (MockIndexShard shard: shards.values()) {
+                shard.updateTerm(indexMetaData.primaryTerm(shard.shardId().id()));
+            }
         }
 
         @Override
@@ -270,7 +273,7 @@ public MockIndexShard getShardOrNull(int shardId) {
 
         public synchronized MockIndexShard createShard(ShardRouting routing) throws IOException {
             failRandomly();
-            MockIndexShard shard = new MockIndexShard(routing);
+            MockIndexShard shard = new MockIndexShard(routing, indexSettings.getIndexMetaData().primaryTerm(routing.shardId().id()));
             shards = newMapBuilder(shards).put(routing.id(), shard).immutableMap();
             return shard;
         }
@@ -303,9 +306,11 @@ public Index index() {
     protected class MockIndexShard implements IndicesClusterStateService.Shard {
         private volatile ShardRouting shardRouting;
         private volatile RecoveryState recoveryState;
+        private volatile long term;
 
-        public MockIndexShard(ShardRouting shardRouting) {
+        public MockIndexShard(ShardRouting shardRouting, long term) {
             this.shardRouting = shardRouting;
+            this.term = term;
         }
 
         @Override
@@ -331,9 +336,21 @@ public IndexShardState state() {
         @Override
         public void updateRoutingEntry(ShardRouting shardRouting) throws IOException {
             failRandomly();
-            assert this.shardId().equals(shardRouting.shardId());
-            assert this.shardRouting.isSameAllocation(shardRouting);
+            assertThat(this.shardId(), equalTo(shardRouting.shardId()));
+            assertTrue("current: " + this.shardRouting + ", got: " + shardRouting, this.shardRouting.isSameAllocation(shardRouting));
+            if (this.shardRouting.active()) {
+                assertTrue("and active shard must stay active, current: " + this.shardRouting + ", got: " + shardRouting,
+                    shardRouting.active());
+            }
             this.shardRouting = shardRouting;
         }
+
+        public void updateTerm(long newTerm) {
+            assertThat("term can only be incremented: " + shardRouting, newTerm, greaterThanOrEqualTo(term));
+            if (shardRouting.primary() && shardRouting.active()) {
+                assertThat("term can not be changed on an active primary shard: " + shardRouting, newTerm, equalTo(term));
+            }
+            this.term = newTerm;
+        }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java b/core/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java
index a5e4b2ab7e1fe..5aefbae63cfcf 100644
--- a/core/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java
+++ b/core/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.indices.cluster;
 
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteRequest;
 import org.elasticsearch.action.admin.cluster.reroute.TransportClusterRerouteAction;
@@ -40,9 +41,12 @@
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
 import org.elasticsearch.action.support.master.TransportMasterNodeActionUtils;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateTaskExecutor;
+import org.elasticsearch.cluster.ClusterStateTaskExecutor.ClusterTasksResult;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.EmptyClusterInfoService;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
+import org.elasticsearch.cluster.action.shard.ShardStateAction.ShardEntry;
 import org.elasticsearch.cluster.metadata.AliasValidator;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
@@ -51,27 +55,28 @@
 import org.elasticsearch.cluster.metadata.MetaDataIndexStateService;
 import org.elasticsearch.cluster.metadata.MetaDataIndexUpgradeService;
 import org.elasticsearch.cluster.metadata.MetaDataUpdateSettingsService;
+import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
-import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation;
+import org.elasticsearch.cluster.routing.allocation.FailedShard;
 import org.elasticsearch.cluster.routing.allocation.RandomAllocationDeciderTests;
-import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.elasticsearch.cluster.routing.allocation.decider.ReplicaAfterPrimaryActiveAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.SameShardAllocationDecider;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexService;
-import org.elasticsearch.index.NodeServicesProvider;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.shard.IndexEventListener;
 import org.elasticsearch.indices.IndicesService;
-import org.elasticsearch.test.gateway.NoopGatewayAllocator;
+import org.elasticsearch.test.gateway.TestGatewayAllocator;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportService;
@@ -110,23 +115,22 @@ public class ClusterStateChanges extends AbstractComponent {
     private final TransportClusterRerouteAction transportClusterRerouteAction;
     private final TransportCreateIndexAction transportCreateIndexAction;
 
-    public ClusterStateChanges() {
+    public ClusterStateChanges(NamedXContentRegistry xContentRegistry, ThreadPool threadPool) {
         super(Settings.builder().put(PATH_HOME_SETTING.getKey(), "dummy").build());
 
+        ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
         allocationService = new AllocationService(settings, new AllocationDeciders(settings,
-            new HashSet<>(Arrays.asList(new SameShardAllocationDecider(settings),
+            new HashSet<>(Arrays.asList(new SameShardAllocationDecider(settings, clusterSettings),
                 new ReplicaAfterPrimaryActiveAllocationDecider(settings),
                 new RandomAllocationDeciderTests.RandomAllocationDecider(getRandom())))),
-            NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(settings),
+            new TestGatewayAllocator(), new BalancedShardsAllocator(settings),
             EmptyClusterInfoService.INSTANCE);
         shardFailedClusterStateTaskExecutor = new ShardStateAction.ShardFailedClusterStateTaskExecutor(allocationService, null, logger);
         shardStartedClusterStateTaskExecutor = new ShardStateAction.ShardStartedClusterStateTaskExecutor(allocationService, logger);
-        ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
         ActionFilters actionFilters = new ActionFilters(Collections.emptySet());
         IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(settings);
         DestructiveOperations destructiveOperations = new DestructiveOperations(settings, clusterSettings);
         Environment environment = new Environment(settings);
-        ThreadPool threadPool = null; // it's not used
         Transport transport = null; // it's not used
 
         // mocks
@@ -134,10 +138,10 @@ NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(settings),
         IndicesService indicesService = mock(IndicesService.class);
         // MetaDataCreateIndexService creates indices using its IndicesService instance to check mappings -> fake it here
         try {
-            when(indicesService.createIndex(any(NodeServicesProvider.class), any(IndexMetaData.class), anyList()))
+            when(indicesService.createIndex(any(IndexMetaData.class), anyList()))
                 .then(invocationOnMock -> {
                     IndexService indexService = mock(IndexService.class);
-                    IndexMetaData indexMetaData = (IndexMetaData)invocationOnMock.getArguments()[1];
+                    IndexMetaData indexMetaData = (IndexMetaData)invocationOnMock.getArguments()[0];
                     when(indexService.index()).thenReturn(indexMetaData.getIndex());
                     MapperService mapperService = mock(MapperService.class);
                     when(indexService.mapperService()).thenReturn(mapperService);
@@ -150,23 +154,25 @@ NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(settings),
         }
 
         // services
-        TransportService transportService = new TransportService(settings, transport, threadPool);
-        MetaDataIndexUpgradeService metaDataIndexUpgradeService = new MetaDataIndexUpgradeService(settings, null, null) {
+        TransportService transportService = new TransportService(settings, transport, threadPool,
+            TransportService.NOOP_TRANSPORT_INTERCEPTOR,
+            boundAddress -> DiscoveryNode.createLocal(settings, boundAddress.publishAddress(), UUIDs.randomBase64UUID()), clusterSettings);
+        MetaDataIndexUpgradeService metaDataIndexUpgradeService = new MetaDataIndexUpgradeService(settings, xContentRegistry, null, null,
+            null) {
             // metaData upgrader should do nothing
             @Override
-            public IndexMetaData upgradeIndexMetaData(IndexMetaData indexMetaData) {
+            public IndexMetaData upgradeIndexMetaData(IndexMetaData indexMetaData, Version minimumIndexCompatibilityVersion) {
                 return indexMetaData;
             }
         };
-        NodeServicesProvider nodeServicesProvider = new NodeServicesProvider(threadPool, null, null, null, null, null, clusterService);
         MetaDataIndexStateService indexStateService = new MetaDataIndexStateService(settings, clusterService, allocationService,
-            metaDataIndexUpgradeService, nodeServicesProvider, indicesService);
+            metaDataIndexUpgradeService, indicesService);
         MetaDataDeleteIndexService deleteIndexService = new MetaDataDeleteIndexService(settings, clusterService, allocationService);
         MetaDataUpdateSettingsService metaDataUpdateSettingsService = new MetaDataUpdateSettingsService(settings, clusterService,
-            allocationService, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, indicesService, nodeServicesProvider);
+            allocationService, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, indicesService, threadPool);
         MetaDataCreateIndexService createIndexService = new MetaDataCreateIndexService(settings, clusterService, indicesService,
             allocationService, new AliasValidator(settings), environment,
-            nodeServicesProvider, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, threadPool);
+            IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, threadPool, xContentRegistry);
 
         transportCloseIndexAction = new TransportCloseIndexAction(settings, transportService, clusterService, threadPool,
             indexStateService, clusterSettings, actionFilters, indexNameExpressionResolver, destructiveOperations);
@@ -207,28 +213,33 @@ public ClusterState reroute(ClusterState state, ClusterRerouteRequest request) {
     }
 
     public ClusterState deassociateDeadNodes(ClusterState clusterState, boolean reroute, String reason) {
-        RoutingAllocation.Result rerouteResult = allocationService.deassociateDeadNodes(clusterState, reroute, reason);
-        return ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+        return allocationService.deassociateDeadNodes(clusterState, reroute, reason);
     }
 
-    public ClusterState applyFailedShards(ClusterState clusterState, List<FailedRerouteAllocation.FailedShard> failedShards) {
-        List<ShardStateAction.ShardEntry> entries = failedShards.stream().map(failedShard ->
-            new ShardStateAction.ShardEntry(failedShard.routingEntry.shardId(), failedShard.routingEntry.allocationId().getId(),
-                0L, failedShard.message, failedShard.failure))
+    public ClusterState applyFailedShards(ClusterState clusterState, List<FailedShard> failedShards) {
+        List<ShardEntry> entries = failedShards.stream().map(failedShard ->
+            new ShardEntry(failedShard.getRoutingEntry().shardId(), failedShard.getRoutingEntry().allocationId().getId(),
+                0L, failedShard.getMessage(), failedShard.getFailure()))
             .collect(Collectors.toList());
-        try {
-            return shardFailedClusterStateTaskExecutor.execute(clusterState, entries).resultingState;
-        } catch (Exception e) {
-            throw ExceptionsHelper.convertToRuntime(e);
-        }
+        return runTasks(shardFailedClusterStateTaskExecutor, clusterState, entries);
     }
 
     public ClusterState applyStartedShards(ClusterState clusterState, List<ShardRouting> startedShards) {
-        List<ShardStateAction.ShardEntry> entries = startedShards.stream().map(startedShard ->
-            new ShardStateAction.ShardEntry(startedShard.shardId(), startedShard.allocationId().getId(), 0L, "shard started", null))
+        List<ShardEntry> entries = startedShards.stream().map(startedShard ->
+            new ShardEntry(startedShard.shardId(), startedShard.allocationId().getId(), 0L, "shard started", null))
             .collect(Collectors.toList());
+        return runTasks(shardStartedClusterStateTaskExecutor, clusterState, entries);
+    }
+
+    private <T> ClusterState runTasks(ClusterStateTaskExecutor<T> executor, ClusterState clusterState, List<T> entries) {
         try {
-            return shardStartedClusterStateTaskExecutor.execute(clusterState, entries).resultingState;
+            ClusterTasksResult<T> result = executor.execute(clusterState, entries);
+            for (ClusterStateTaskExecutor.TaskResult taskResult : result.executionResults.values()) {
+                if (taskResult.isSuccess() == false) {
+                    throw taskResult.getFailure();
+                }
+            }
+            return result.resultingState;
         } catch (Exception e) {
             throw ExceptionsHelper.convertToRuntime(e);
         }
diff --git a/core/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java b/core/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java
index e9a1476449f4e..71780f255d1a3 100644
--- a/core/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.indices.cluster;
 
+import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteRequest;
 import org.elasticsearch.action.admin.indices.close.CloseIndexRequest;
@@ -31,6 +32,8 @@
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
+import org.elasticsearch.cluster.block.ClusterBlock;
+import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -38,15 +41,17 @@
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
-import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation.FailedShard;
+import org.elasticsearch.cluster.routing.allocation.FailedShard;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.discovery.DiscoverySettings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.indices.recovery.PeerRecoveryTargetService;
 import org.elasticsearch.repositories.RepositoriesService;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -61,7 +66,8 @@
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
-import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Supplier;
 
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
@@ -72,13 +78,35 @@
 
 public class IndicesClusterStateServiceRandomUpdatesTests extends AbstractIndicesClusterStateServiceTestCase {
 
-    private final ClusterStateChanges cluster = new ClusterStateChanges();
+    private ThreadPool threadPool;
+    private ClusterStateChanges cluster;
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        threadPool = new TestThreadPool(getClass().getName());
+        cluster = new ClusterStateChanges(xContentRegistry(), threadPool);
+    }
+
+    @Override
+    public void tearDown() throws Exception {
+        super.tearDown();
+        terminate(threadPool);
+    }
+
+    /**
+     * needed due to random usage of {@link IndexMetaData#INDEX_SHADOW_REPLICAS_SETTING}. removed once
+     * shadow replicas are removed.
+     */
+    @Override
+    protected boolean enableWarningsCheck() {
+        return false;
+    }
 
     public void testRandomClusterStateUpdates() {
         // we have an IndicesClusterStateService per node in the cluster
         final Map<DiscoveryNode, IndicesClusterStateService> clusterStateServiceMap = new HashMap<>();
         ClusterState state = randomInitialClusterState(clusterStateServiceMap, MockIndicesService::new);
-
         // each of the following iterations represents a new cluster state update processed on all nodes
         for (int i = 0; i < 30; i++) {
             logger.info("Iteration {}", i);
@@ -86,7 +114,14 @@ public void testRandomClusterStateUpdates() {
 
             // calculate new cluster state
             for (int j = 0; j < randomInt(3); j++) { // multiple iterations to simulate batching of cluster states
-                state = randomlyUpdateClusterState(state, clusterStateServiceMap, MockIndicesService::new);
+                try {
+                    state = randomlyUpdateClusterState(state, clusterStateServiceMap, MockIndicesService::new);
+                } catch (AssertionError error) {
+                    ClusterState finalState = state;
+                    logger.error((org.apache.logging.log4j.util.Supplier<?>) () ->
+                        new ParameterizedMessage("failed to random change state. last good state: \n{}", finalState), error);
+                    throw error;
+                }
             }
 
             // apply cluster state to nodes (incl. master)
@@ -94,7 +129,15 @@ public void testRandomClusterStateUpdates() {
                 IndicesClusterStateService indicesClusterStateService = clusterStateServiceMap.get(node);
                 ClusterState localState = adaptClusterStateToLocalNode(state, node);
                 ClusterState previousLocalState = adaptClusterStateToLocalNode(previousState, node);
-                indicesClusterStateService.clusterChanged(new ClusterChangedEvent("simulated change " + i, localState, previousLocalState));
+                final ClusterChangedEvent event = new ClusterChangedEvent("simulated change " + i, localState, previousLocalState);
+                try {
+                    indicesClusterStateService.applyClusterState(event);
+                } catch (AssertionError error) {
+                    logger.error((org.apache.logging.log4j.util.Supplier<?>) () -> new ParameterizedMessage(
+                            "failed to apply change on [{}].\n ***  Previous state ***\n{}\n ***  New state ***\n{}",
+                            node, event.previousState(), event.state()), error);
+                    throw error;
+                }
 
                 // check that cluster state has been properly applied to node
                 assertClusterStateMatchesNodeState(localState, indicesClusterStateService);
@@ -102,7 +145,7 @@ public void testRandomClusterStateUpdates() {
         }
 
         // TODO: check if we can go to green by starting all shards and finishing all iterations
-        logger.info("Final cluster state: {}", state.prettyPrint());
+        logger.info("Final cluster state: {}", state);
     }
 
     /**
@@ -111,7 +154,7 @@ public void testRandomClusterStateUpdates() {
      * are all removed but the on disk contents of those indices remain so that they can later be
      * imported as dangling indices.  Normally, the first cluster state update that the node
      * receives from the new cluster would contain a cluster block that would cause all in-memory
-     * structures to be removed (see {@link IndicesClusterStateService#clusterChanged(ClusterChangedEvent)}),
+     * structures to be removed (see {@link IndicesClusterStateService#applyClusterState(ClusterChangedEvent)}),
      * but in the case where the node joined and was a few cluster state updates behind, it would
      * not have received the cluster block, in which case we still need to remove the in-memory
      * structures while ensuring the data remains on disk.  This test executes this particular
@@ -119,7 +162,7 @@ public void testRandomClusterStateUpdates() {
      */
     public void testJoiningNewClusterOnlyRemovesInMemoryIndexStructures() {
         // a cluster state derived from the initial state that includes a created index
-        String name = "index_" + randomAsciiOfLength(8).toLowerCase(Locale.ROOT);
+        String name = "index_" + randomAlphaOfLength(8).toLowerCase(Locale.ROOT);
         ShardRoutingState[] replicaStates = new ShardRoutingState[randomIntBetween(0, 3)];
         Arrays.fill(replicaStates, ShardRoutingState.INITIALIZING);
         ClusterState stateWithIndex = ClusterStateCreationUtils.state(name, randomBoolean(), ShardRoutingState.INITIALIZING, replicaStates);
@@ -137,9 +180,9 @@ public void testJoiningNewClusterOnlyRemovesInMemoryIndexStructures() {
         // simulate the cluster state change on the node
         ClusterState localState = adaptClusterStateToLocalNode(stateWithIndex, node);
         ClusterState previousLocalState = adaptClusterStateToLocalNode(initialState, node);
-        IndicesClusterStateService indicesCSSvc = createIndicesClusterStateService(RecordingIndicesService::new);
+        IndicesClusterStateService indicesCSSvc = createIndicesClusterStateService(node, RecordingIndicesService::new);
         indicesCSSvc.start();
-        indicesCSSvc.clusterChanged(new ClusterChangedEvent("cluster state change that adds the index", localState, previousLocalState));
+        indicesCSSvc.applyClusterState(new ClusterChangedEvent("cluster state change that adds the index", localState, previousLocalState));
 
         // create a new empty cluster state with a brand new cluster UUID
         ClusterState newClusterState = ClusterState.builder(initialState)
@@ -149,8 +192,8 @@ public void testJoiningNewClusterOnlyRemovesInMemoryIndexStructures() {
         // simulate the cluster state change on the node
         localState = adaptClusterStateToLocalNode(newClusterState, node);
         previousLocalState = adaptClusterStateToLocalNode(stateWithIndex, node);
-        indicesCSSvc.clusterChanged(new ClusterChangedEvent("cluster state change with a new cluster UUID (and doesn't contain the index)",
-                                                            localState, previousLocalState));
+        indicesCSSvc.applyClusterState(new ClusterChangedEvent(
+            "cluster state change with a new cluster UUID (and doesn't contain the index)", localState, previousLocalState));
 
         // check that in memory data structures have been removed once the new cluster state is applied,
         // but the persistent data is still there
@@ -183,7 +226,7 @@ private void updateNodes(ClusterState state, Map<DiscoveryNode, IndicesClusterSt
                              Supplier<MockIndicesService> indicesServiceSupplier) {
         for (DiscoveryNode node : state.nodes()) {
             clusterStateServiceMap.computeIfAbsent(node, discoveryNode -> {
-                IndicesClusterStateService ics = createIndicesClusterStateService(indicesServiceSupplier);
+                IndicesClusterStateService ics = createIndicesClusterStateService(discoveryNode, indicesServiceSupplier);
                 ics.start();
                 return ics;
             });
@@ -200,16 +243,37 @@ private void updateNodes(ClusterState state, Map<DiscoveryNode, IndicesClusterSt
     public ClusterState randomlyUpdateClusterState(ClusterState state,
                                                    Map<DiscoveryNode, IndicesClusterStateService> clusterStateServiceMap,
                                                    Supplier<MockIndicesService> indicesServiceSupplier) {
+        // randomly remove no_master blocks
+        if (randomBoolean() && state.blocks().hasGlobalBlock(DiscoverySettings.NO_MASTER_BLOCK_ID)) {
+            state = ClusterState.builder(state).blocks(
+                ClusterBlocks.builder().blocks(state.blocks()).removeGlobalBlock(DiscoverySettings.NO_MASTER_BLOCK_ID)).build();
+        }
+
+        // randomly add no_master blocks
+        if (rarely() && state.blocks().hasGlobalBlock(DiscoverySettings.NO_MASTER_BLOCK_ID) == false) {
+            ClusterBlock block = randomBoolean() ? DiscoverySettings.NO_MASTER_BLOCK_ALL : DiscoverySettings.NO_MASTER_BLOCK_WRITES;
+            state = ClusterState.builder(state).blocks(ClusterBlocks.builder().blocks(state.blocks()).addGlobalBlock(block)).build();
+        }
+
+        // if no_master block is in place, make no other cluster state changes
+        if (state.blocks().hasGlobalBlock(DiscoverySettings.NO_MASTER_BLOCK_ID)) {
+            return state;
+        }
+
         // randomly create new indices (until we have 200 max)
         for (int i = 0; i < randomInt(5); i++) {
             if (state.metaData().indices().size() > 200) {
                 break;
             }
-            String name = "index_" + randomAsciiOfLength(15).toLowerCase(Locale.ROOT);
-            CreateIndexRequest request = new CreateIndexRequest(name, Settings.builder()
+            String name = "index_" + randomAlphaOfLength(15).toLowerCase(Locale.ROOT);
+            Settings.Builder settingsBuilder = Settings.builder()
                 .put(SETTING_NUMBER_OF_SHARDS, randomIntBetween(1, 3))
-                .put(SETTING_NUMBER_OF_REPLICAS, randomInt(2))
-                .build()).waitForActiveShards(ActiveShardCount.NONE);
+                .put(SETTING_NUMBER_OF_REPLICAS, randomInt(2));
+            if (randomBoolean()) {
+                settingsBuilder.put(IndexMetaData.SETTING_SHADOW_REPLICAS, true)
+                    .put(IndexMetaData.SETTING_SHARED_FILESYSTEM, true);
+            }
+            CreateIndexRequest request = new CreateIndexRequest(name, settingsBuilder.build()).waitForActiveShards(ActiveShardCount.NONE);
             state = cluster.createIndex(state, request);
             assertTrue(state.metaData().hasIndex(name));
         }
@@ -309,6 +373,13 @@ public ClusterState randomlyUpdateClusterState(ClusterState state,
                         state = cluster.deassociateDeadNodes(state, true, "removed and added a node");
                         updateNodes(state, clusterStateServiceMap, indicesServiceSupplier);
                     }
+                    if (randomBoolean()) {
+                        // and add it back
+                        DiscoveryNodes newNodes = DiscoveryNodes.builder(state.nodes()).add(discoveryNode).build();
+                        state = ClusterState.builder(state).nodes(newNodes).build();
+                        state = cluster.reroute(state, new ClusterRerouteRequest());
+                        updateNodes(state, clusterStateServiceMap, indicesServiceSupplier);
+                    }
                 }
             }
         }
@@ -318,12 +389,15 @@ public ClusterState randomlyUpdateClusterState(ClusterState state,
         return state;
     }
 
+    private static final AtomicInteger nodeIdGenerator = new AtomicInteger();
+
     protected DiscoveryNode createNode(DiscoveryNode.Role... mustHaveRoles) {
         Set<DiscoveryNode.Role> roles = new HashSet<>(randomSubsetOf(Sets.newHashSet(DiscoveryNode.Role.values())));
         for (DiscoveryNode.Role mustHaveRole : mustHaveRoles) {
             roles.add(mustHaveRole);
         }
-        return new DiscoveryNode("node_" + randomAsciiOfLength(8), LocalTransportAddress.buildUnique(), Collections.emptyMap(), roles,
+        final String id = String.format(Locale.ROOT, "node_%03d", nodeIdGenerator.incrementAndGet());
+        return new DiscoveryNode(id, id, LocalTransportAddress.buildUnique(), Collections.emptyMap(), roles,
             Version.CURRENT);
     }
 
@@ -331,19 +405,22 @@ private static ClusterState adaptClusterStateToLocalNode(ClusterState state, Dis
         return ClusterState.builder(state).nodes(DiscoveryNodes.builder(state.nodes()).localNodeId(node.getId())).build();
     }
 
-    private IndicesClusterStateService createIndicesClusterStateService(final Supplier<MockIndicesService> indicesServiceSupplier) {
+    private IndicesClusterStateService createIndicesClusterStateService(DiscoveryNode discoveryNode,
+                                                                        final Supplier<MockIndicesService> indicesServiceSupplier) {
         final ThreadPool threadPool = mock(ThreadPool.class);
-        final Executor executor = mock(Executor.class);
-        when(threadPool.generic()).thenReturn(executor);
+        when(threadPool.generic()).thenReturn(mock(ExecutorService.class));
         final MockIndicesService indicesService = indicesServiceSupplier.get();
-        final TransportService transportService = new TransportService(Settings.EMPTY, null, threadPool);
+        final Settings settings = Settings.builder().put("node.name", discoveryNode.getName()).build();
+        final TransportService transportService = new TransportService(settings, null, threadPool,
+            TransportService.NOOP_TRANSPORT_INTERCEPTOR,
+            boundAddress -> DiscoveryNode.createLocal(settings, boundAddress.publishAddress(), UUIDs.randomBase64UUID()), null);
         final ClusterService clusterService = mock(ClusterService.class);
-        final RepositoriesService repositoriesService = new RepositoriesService(Settings.EMPTY, clusterService,
+        final RepositoriesService repositoriesService = new RepositoriesService(settings, clusterService,
             transportService, null);
-        final PeerRecoveryTargetService recoveryTargetService = new PeerRecoveryTargetService(Settings.EMPTY, threadPool,
+        final PeerRecoveryTargetService recoveryTargetService = new PeerRecoveryTargetService(settings, threadPool,
             transportService, null, clusterService);
         final ShardStateAction shardStateAction = mock(ShardStateAction.class);
-        return new IndicesClusterStateService(Settings.EMPTY, indicesService, clusterService,
+        return new IndicesClusterStateService(settings, indicesService, clusterService,
             threadPool, recoveryTargetService, shardStateAction, null, repositoriesService, null, null, null, null, null);
     }
 
@@ -351,11 +428,13 @@ private class RecordingIndicesService extends MockIndicesService {
         private Set<Index> deletedIndices = Collections.emptySet();
 
         @Override
-        public synchronized void deleteIndex(Index index, String reason) {
-            super.deleteIndex(index, reason);
-            Set<Index> newSet = Sets.newHashSet(deletedIndices);
-            newSet.add(index);
-            deletedIndices = Collections.unmodifiableSet(newSet);
+        public synchronized void removeIndex(Index index, IndexRemovalReason reason, String extraInfo) {
+            super.removeIndex(index, reason, extraInfo);
+            if (reason == IndexRemovalReason.DELETED) {
+                Set<Index> newSet = Sets.newHashSet(deletedIndices);
+                newSet.add(index);
+                deletedIndices = Collections.unmodifiableSet(newSet);
+            }
         }
 
         public synchronized boolean isDeleted(Index index) {
diff --git a/core/src/test/java/org/elasticsearch/indices/exists/indices/IndicesExistsIT.java b/core/src/test/java/org/elasticsearch/indices/exists/indices/IndicesExistsIT.java
index 6052add7e3673..a80c61adaffb2 100644
--- a/core/src/test/java/org/elasticsearch/indices/exists/indices/IndicesExistsIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/exists/indices/IndicesExistsIT.java
@@ -29,6 +29,7 @@
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_READ;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_WRITE;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY_ALLOW_DELETE;
 import static org.hamcrest.Matchers.equalTo;
 
 public class IndicesExistsIT extends ESIntegTestCase {
@@ -51,9 +52,9 @@ public void testIndicesExists() throws Exception {
 
     public void testIndicesExistsWithBlocks() {
         createIndex("ro");
-
         // Request is not blocked
-        for (String blockSetting : Arrays.asList(SETTING_BLOCKS_READ, SETTING_BLOCKS_WRITE, SETTING_READ_ONLY)) {
+        for (String blockSetting : Arrays.asList(SETTING_BLOCKS_READ, SETTING_BLOCKS_WRITE, SETTING_READ_ONLY,
+            SETTING_READ_ONLY_ALLOW_DELETE)) {
             try {
                 enableIndexBlock("ro", blockSetting);
                 assertThat(client().admin().indices().prepareExists("ro").execute().actionGet().isExists(), equalTo(true));
diff --git a/core/src/test/java/org/elasticsearch/indices/flush/FlushIT.java b/core/src/test/java/org/elasticsearch/indices/flush/FlushIT.java
index 83369392caba3..50979cf611d07 100644
--- a/core/src/test/java/org/elasticsearch/indices/flush/FlushIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/flush/FlushIT.java
@@ -31,6 +31,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.engine.Engine;
@@ -57,12 +58,12 @@ public void testWaitIfOngoing() throws InterruptedException {
         final int numIters = scaledRandomIntBetween(10, 30);
         for (int i = 0; i < numIters; i++) {
             for (int j = 0; j < 10; j++) {
-                client().prepareIndex("test", "test").setSource("{}").get();
+                client().prepareIndex("test", "test").setSource("{}", XContentType.JSON).get();
             }
             final CountDownLatch latch = new CountDownLatch(10);
             final CopyOnWriteArrayList<Throwable> errors = new CopyOnWriteArrayList<>();
             for (int j = 0; j < 10; j++) {
-                client().admin().indices().prepareFlush("test").setWaitIfOngoing(true).execute(new ActionListener<FlushResponse>() {
+                client().admin().indices().prepareFlush("test").execute(new ActionListener<FlushResponse>() {
                     @Override
                     public void onResponse(FlushResponse flushResponse) {
                         try {
@@ -164,7 +165,7 @@ public void testSyncedFlushWithConcurrentIndexing() throws Exception {
             @Override
             public void run() {
                 while (stop.get() == false) {
-                    client().prepareIndex().setIndex("test").setType("doc").setSource("{}").get();
+                    client().prepareIndex().setIndex("test").setType("doc").setSource("{}", XContentType.JSON).get();
                     numDocs.incrementAndGet();
                 }
             }
diff --git a/core/src/test/java/org/elasticsearch/indices/flush/SyncedFlushSingleNodeTests.java b/core/src/test/java/org/elasticsearch/indices/flush/SyncedFlushSingleNodeTests.java
index ea2a80bada5e3..7023df427bcd3 100644
--- a/core/src/test/java/org/elasticsearch/indices/flush/SyncedFlushSingleNodeTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/flush/SyncedFlushSingleNodeTests.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.lease.Releasable;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.shard.IndexShard;
@@ -44,7 +45,7 @@ public class SyncedFlushSingleNodeTests extends ESSingleNodeTestCase {
 
     public void testModificationPreventsFlushing() throws InterruptedException {
         createIndex("test");
-        client().prepareIndex("test", "test", "1").setSource("{}").get();
+        client().prepareIndex("test", "test", "1").setSource("{}", XContentType.JSON).get();
         IndexService test = getInstanceFromNode(IndicesService.class).indexService(resolveIndex("test"));
         IndexShard shard = test.getShardOrNull(0);
 
@@ -56,7 +57,7 @@ public void testModificationPreventsFlushing() throws InterruptedException {
         assertEquals("exactly one active shard", 1, activeShards.size());
         Map<String, Engine.CommitId> commitIds = SyncedFlushUtil.sendPreSyncRequests(flushService, activeShards, state, shardId);
         assertEquals("exactly one commit id", 1, commitIds.size());
-        client().prepareIndex("test", "test", "2").setSource("{}").get();
+        client().prepareIndex("test", "test", "2").setSource("{}", XContentType.JSON).get();
         String syncId = UUIDs.base64UUID();
         SyncedFlushUtil.LatchedListener<ShardsSyncedFlushResult> listener = new SyncedFlushUtil.LatchedListener<>();
         flushService.sendSyncRequests(syncId, activeShards, state, commitIds, shardId, shardRoutingTable.size(), listener);
@@ -88,7 +89,7 @@ public void testModificationPreventsFlushing() throws InterruptedException {
 
     public void testSingleShardSuccess() throws InterruptedException {
         createIndex("test");
-        client().prepareIndex("test", "test", "1").setSource("{}").get();
+        client().prepareIndex("test", "test", "1").setSource("{}", XContentType.JSON).get();
         IndexService test = getInstanceFromNode(IndicesService.class).indexService(resolveIndex("test"));
         IndexShard shard = test.getShardOrNull(0);
 
@@ -108,7 +109,7 @@ public void testSingleShardSuccess() throws InterruptedException {
 
     public void testSyncFailsIfOperationIsInFlight() throws InterruptedException, ExecutionException {
         createIndex("test");
-        client().prepareIndex("test", "test", "1").setSource("{}").get();
+        client().prepareIndex("test", "test", "1").setSource("{}", XContentType.JSON).get();
         IndexService test = getInstanceFromNode(IndicesService.class).indexService(resolveIndex("test"));
         IndexShard shard = test.getShardOrNull(0);
 
@@ -163,7 +164,7 @@ public void testSyncFailsOnIndexClosedOrMissing() throws InterruptedException {
 
     public void testFailAfterIntermediateCommit() throws InterruptedException {
         createIndex("test");
-        client().prepareIndex("test", "test", "1").setSource("{}").get();
+        client().prepareIndex("test", "test", "1").setSource("{}", XContentType.JSON).get();
         IndexService test = getInstanceFromNode(IndicesService.class).indexService(resolveIndex("test"));
         IndexShard shard = test.getShardOrNull(0);
 
@@ -176,7 +177,7 @@ public void testFailAfterIntermediateCommit() throws InterruptedException {
         Map<String, Engine.CommitId> commitIds = SyncedFlushUtil.sendPreSyncRequests(flushService, activeShards, state, shardId);
         assertEquals("exactly one commit id", 1, commitIds.size());
         if (randomBoolean()) {
-            client().prepareIndex("test", "test", "2").setSource("{}").get();
+            client().prepareIndex("test", "test", "2").setSource("{}", XContentType.JSON).get();
         }
         client().admin().indices().prepareFlush("test").setForce(true).get();
         String syncId = UUIDs.base64UUID();
@@ -196,7 +197,7 @@ public void testFailAfterIntermediateCommit() throws InterruptedException {
 
     public void testFailWhenCommitIsMissing() throws InterruptedException {
         createIndex("test");
-        client().prepareIndex("test", "test", "1").setSource("{}").get();
+        client().prepareIndex("test", "test", "1").setSource("{}", XContentType.JSON).get();
         IndexService test = getInstanceFromNode(IndicesService.class).indexService(resolveIndex("test"));
         IndexShard shard = test.getShardOrNull(0);
 
diff --git a/core/src/test/java/org/elasticsearch/indices/mapping/DedicatedMasterGetFieldMappingIT.java b/core/src/test/java/org/elasticsearch/indices/mapping/DedicatedMasterGetFieldMappingIT.java
index 81c9d12fbb433..f8fe05bc97b8a 100644
--- a/core/src/test/java/org/elasticsearch/indices/mapping/DedicatedMasterGetFieldMappingIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/mapping/DedicatedMasterGetFieldMappingIT.java
@@ -34,7 +34,7 @@ public void before1() throws Exception {
         Settings settings = Settings.builder()
                 .put(Node.NODE_DATA_SETTING.getKey(), false)
                 .build();
-        internalCluster().startNodesAsync(settings, Settings.EMPTY).get();
+        internalCluster().startNodes(settings, Settings.EMPTY);
     }
 
 }
diff --git a/core/src/test/java/org/elasticsearch/indices/mapping/SimpleGetFieldMappingsIT.java b/core/src/test/java/org/elasticsearch/indices/mapping/SimpleGetFieldMappingsIT.java
index ecb4436918acb..f654dfe19f64f 100644
--- a/core/src/test/java/org/elasticsearch/indices/mapping/SimpleGetFieldMappingsIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/mapping/SimpleGetFieldMappingsIT.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.hamcrest.Matchers;
 
@@ -163,7 +164,7 @@ public void testSimpleGetFieldMappingsWithPretty() throws Exception {
 
 
         XContentBuilder prettyJsonBuilder = XContentFactory.jsonBuilder().prettyPrint();
-        prettyJsonBuilder.copyCurrentStructure(XContentFactory.xContent(responseStrings).createParser(responseStrings));
+        prettyJsonBuilder.copyCurrentStructure(createParser(JsonXContent.jsonXContent, responseStrings));
         assertThat(responseStrings, equalTo(prettyJsonBuilder.string()));
 
         params.put("pretty", "false");
@@ -176,7 +177,7 @@ public void testSimpleGetFieldMappingsWithPretty() throws Exception {
         responseStrings = responseBuilder.string();
 
         prettyJsonBuilder = XContentFactory.jsonBuilder().prettyPrint();
-        prettyJsonBuilder.copyCurrentStructure(XContentFactory.xContent(responseStrings).createParser(responseStrings));
+        prettyJsonBuilder.copyCurrentStructure(createParser(JsonXContent.jsonXContent, responseStrings));
         assertThat(responseStrings, not(equalTo(prettyJsonBuilder.string())));
 
     }
diff --git a/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java b/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java
index 361bca6076af4..80001ed16ae21 100644
--- a/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java
@@ -31,6 +31,7 @@
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.client.Requests;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
 import org.elasticsearch.common.breaker.CircuitBreaker;
@@ -40,6 +41,7 @@
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.indices.breaker.BreakerSettings;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.breaker.CircuitBreakerStats;
@@ -56,9 +58,11 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Stream;
 
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
+import static org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_LIMIT_SETTING;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.cardinality;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.terms;
 import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST;
@@ -68,6 +72,7 @@
 import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.Matchers.startsWith;
 
 /**
@@ -81,20 +86,15 @@ private void reset() {
         // clear all caches, we could be very close (or even above) the limit and then we will not be able to reset the breaker settings
         client().admin().indices().prepareClearCache().setFieldDataCache(true).setQueryCache(true).setRequestCache(true).get();
 
-        Settings resetSettings = Settings.builder()
-                .put(HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(),
-                        HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_LIMIT_SETTING.getDefaultRaw(null))
-                .put(HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_OVERHEAD_SETTING.getKey(),
-                        HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_OVERHEAD_SETTING.getDefaultRaw(null))
-                .put(HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(),
-                        HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING.getDefaultRaw(null))
-                .put(HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_OVERHEAD_SETTING.getKey(), 1.0)
-                .put(HierarchyCircuitBreakerService.IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(),
-                        HierarchyCircuitBreakerService.IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_LIMIT_SETTING.getDefaultRaw(null))
-                .put(HierarchyCircuitBreakerService.IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_OVERHEAD_SETTING.getKey(), 1.0)
-                .put(HierarchyCircuitBreakerService.TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(),
-                    HierarchyCircuitBreakerService.TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING.getDefaultRaw(null))
-                .build();
+        Settings.Builder resetSettings = Settings.builder();
+        Stream.of(
+            HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_LIMIT_SETTING,
+            HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_OVERHEAD_SETTING,
+            HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING,
+            HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_OVERHEAD_SETTING,
+            IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_LIMIT_SETTING,
+            HierarchyCircuitBreakerService.IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_OVERHEAD_SETTING,
+            HierarchyCircuitBreakerService.TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING).forEach(s -> resetSettings.putNull(s.getKey()));
         assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(resetSettings));
     }
 
@@ -155,8 +155,11 @@ public void testMemoryBreaker() throws Exception {
         // execute a search that loads field data (sorting on the "test" field)
         // again, this time it should trip the breaker
         SearchRequestBuilder searchRequest = client.prepareSearch("cb-test").setQuery(matchAllQuery()).addSort("test", SortOrder.DESC);
-        assertFailures(searchRequest, RestStatus.INTERNAL_SERVER_ERROR,
-                containsString("Data too large, data for [test] would be larger than limit of [100/100b]"));
+
+        String errMsg = "Data too large, data for [test] would be";
+        assertFailures(searchRequest, RestStatus.INTERNAL_SERVER_ERROR, containsString(errMsg));
+        errMsg = "which is larger than the limit of [100/100b]";
+        assertFailures(searchRequest, RestStatus.INTERNAL_SERVER_ERROR, containsString(errMsg));
 
         NodesStatsResponse stats = client.admin().cluster().prepareNodesStats().setBreaker(true).get();
         int breaks = 0;
@@ -176,7 +179,7 @@ public void testRamAccountingTermsEnum() throws Exception {
 
         // Create an index where the mappings have a field data filter
         assertAcked(prepareCreate("ramtest").setSource("{\"mappings\": {\"type\": {\"properties\": {\"test\": " +
-                "{\"type\": \"text\",\"fielddata\": true,\"fielddata_frequency_filter\": {\"max\": 10000}}}}}}"));
+                "{\"type\": \"text\",\"fielddata\": true,\"fielddata_frequency_filter\": {\"max\": 10000}}}}}}", XContentType.JSON));
 
         ensureGreen("ramtest");
 
@@ -203,9 +206,12 @@ public void testRamAccountingTermsEnum() throws Exception {
 
         // execute a search that loads field data (sorting on the "test" field)
         // again, this time it should trip the breaker
-        assertFailures(client.prepareSearch("ramtest").setQuery(matchAllQuery()).addSort("test", SortOrder.DESC),
-                RestStatus.INTERNAL_SERVER_ERROR,
-                containsString("Data too large, data for [test] would be larger than limit of [100/100b]"));
+        SearchRequestBuilder searchRequest = client.prepareSearch("ramtest").setQuery(matchAllQuery()).addSort("test", SortOrder.DESC);
+
+        String errMsg = "Data too large, data for [test] would be";
+        assertFailures(searchRequest, RestStatus.INTERNAL_SERVER_ERROR, containsString(errMsg));
+        errMsg = "which is larger than the limit of [100/100b]";
+        assertFailures(searchRequest, RestStatus.INTERNAL_SERVER_ERROR, containsString(errMsg));
 
         NodesStatsResponse stats = client.admin().cluster().prepareNodesStats().setBreaker(true).get();
         int breaks = 0;
@@ -220,6 +226,7 @@ public void testRamAccountingTermsEnum() throws Exception {
      * Test that a breaker correctly redistributes to a different breaker, in
      * this case, the fielddata breaker borrows space from the request breaker
      */
+    @TestLogging("_root:DEBUG,org.elasticsearch.action.search:TRACE")
     public void testParentChecking() throws Exception {
         if (noopBreakerUsed()) {
             logger.info("--> noop breakers used, skipping test");
@@ -248,14 +255,20 @@ public void testParentChecking() throws Exception {
             client.prepareSearch("cb-test").setQuery(matchAllQuery()).addSort("test", SortOrder.DESC).get();
             fail("should have thrown an exception");
         } catch (Exception e) {
-            String errMsg = "[fielddata] Data too large, data for [test] would be larger than limit of [10/10b]";
-            assertThat("Exception: [" + e.toString() + "] should contain a CircuitBreakingException",
-                e.toString(), containsString(errMsg));
+            String errMsg = "CircuitBreakingException[[fielddata] Data too large, data for [test] would be";
+            assertThat("Exception: [" + e.toString() + "] should contain a CircuitBreakingException", e.toString(), containsString(errMsg));
+            errMsg = "which is larger than the limit of [10/10b]]";
+            assertThat("Exception: [" + e.toString() + "] should contain a CircuitBreakingException", e.toString(), containsString(errMsg));
         }
 
-        assertFailures(client.prepareSearch("cb-test").setQuery(matchAllQuery()).addSort("test", SortOrder.DESC),
-                RestStatus.INTERNAL_SERVER_ERROR,
-                containsString("Data too large, data for [test] would be larger than limit of [10/10b]"));
+        // execute a search that loads field data (sorting on the "test" field)
+        // again, this time it should trip the breaker
+        SearchRequestBuilder searchRequest = client.prepareSearch("cb-test").setQuery(matchAllQuery()).addSort("test", SortOrder.DESC);
+
+        String errMsg = "Data too large, data for [test] would be";
+        assertFailures(searchRequest, RestStatus.INTERNAL_SERVER_ERROR, containsString(errMsg));
+        errMsg = "which is larger than the limit of [10/10b]";
+        assertFailures(searchRequest, RestStatus.INTERNAL_SERVER_ERROR, containsString(errMsg));
 
         reset();
 
@@ -319,9 +332,8 @@ public void testRequestBreaker() throws Exception {
             fail("aggregation should have tripped the breaker");
         } catch (Exception e) {
             String errMsg = "CircuitBreakingException[[request] Data too large";
-            assertThat("Exception: [" + e.toString() + "] should contain a CircuitBreakingException",
-                e.toString(), containsString(errMsg));
-            errMsg = "would be larger than limit of [10/10b]]";
+            assertThat("Exception: [" + e.toString() + "] should contain a CircuitBreakingException", e.toString(), containsString(errMsg));
+            errMsg = "which is larger than the limit of [10/10b]]";
             assertThat("Exception: [" + e.toString() + "] should contain a CircuitBreakingException", e.toString(), containsString(errMsg));
         }
     }
@@ -357,9 +369,11 @@ public void testBucketBreaker() throws Exception {
             assertTrue("there should be shard failures", resp.getFailedShards() > 0);
             fail("aggregation should have tripped the breaker");
         } catch (Exception e) {
-            String errMsg = "CircuitBreakingException[[request] " +
-                    "Data too large, data for [<agg [my_terms]>] would be larger than limit of [100/100b]]";
-            assertThat("Exception: " + e.toString() + " should contain a CircuitBreakingException",
+            String errMsg = "CircuitBreakingException[[request] Data too large, data for [<agg [my_terms]>] would be";
+            assertThat("Exception: [" + e.toString() + "] should contain a CircuitBreakingException",
+                    e.toString(), containsString(errMsg));
+            errMsg = "which is larger than the limit of [100/100b]]";
+            assertThat("Exception: [" + e.toString() + "] should contain a CircuitBreakingException",
                     e.toString(), containsString(errMsg));
         }
     }
@@ -367,16 +381,13 @@ public void testBucketBreaker() throws Exception {
     /** Issues a cache clear and waits 30 seconds for the field data breaker to be cleared */
     public void clearFieldData() throws Exception {
         client().admin().indices().prepareClearCache().setFieldDataCache(true).execute().actionGet();
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                NodesStatsResponse resp = client().admin().cluster().prepareNodesStats()
-                        .clear().setBreaker(true).get(new TimeValue(15, TimeUnit.SECONDS));
-                for (NodeStats nStats : resp.getNodes()) {
-                    assertThat("fielddata breaker never reset back to 0",
-                            nStats.getBreaker().getStats(CircuitBreaker.FIELDDATA).getEstimated(),
-                            equalTo(0L));
-                }
+        assertBusy(() -> {
+            NodesStatsResponse resp = client().admin().cluster().prepareNodesStats()
+                    .clear().setBreaker(true).get(new TimeValue(15, TimeUnit.SECONDS));
+            for (NodeStats nStats : resp.getNodes()) {
+                assertThat("fielddata breaker never reset back to 0",
+                        nStats.getBreaker().getStats(CircuitBreaker.FIELDDATA).getEstimated(),
+                        equalTo(0L));
             }
         }, 30, TimeUnit.SECONDS);
     }
@@ -410,6 +421,26 @@ public void testCustomCircuitBreakerRegistration() throws Exception {
         assertThat(breaks, greaterThanOrEqualTo(1));
     }
 
+    public void testCanResetUnreasonableSettings() {
+        if (noopBreakerUsed()) {
+            logger.info("--> noop breakers used, skipping test");
+            return;
+        }
+        Settings insane = Settings.builder()
+            .put(IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), "5b")
+            .build();
+        client().admin().cluster().prepareUpdateSettings().setTransientSettings(insane).get();
+
+        // calls updates settings to reset everything to default, checking that the request
+        // is not blocked by the above inflight circuit breaker
+        reset();
+
+        assertThat(client().admin().cluster().prepareState().get()
+            .getState().metaData().transientSettings().get(HierarchyCircuitBreakerService.TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING.getKey()),
+            nullValue());
+
+    }
+
     public void testLimitsRequestSize() throws Exception {
         ByteSizeValue inFlightRequestsLimit = new ByteSizeValue(8, ByteSizeUnit.KB);
         if (noopBreakerUsed()) {
@@ -448,12 +479,12 @@ public void testLimitsRequestSize() throws Exception {
         BulkRequest bulkRequest = new BulkRequest();
         for (int i = 0; i < numRequests; i++) {
             IndexRequest indexRequest = new IndexRequest("index", "type", Integer.toString(i));
-            indexRequest.source("field", "value", "num", i);
+            indexRequest.source(Requests.INDEX_CONTENT_TYPE, "field", "value", "num", i);
             bulkRequest.add(indexRequest);
         }
 
         Settings limitSettings = Settings.builder()
-            .put(HierarchyCircuitBreakerService.IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), inFlightRequestsLimit)
+            .put(IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), inFlightRequestsLimit)
             .build();
 
         assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(limitSettings));
@@ -468,11 +499,11 @@ public void testLimitsRequestSize() throws Exception {
                 for (BulkItemResponse bulkItemResponse : response) {
                     Throwable cause = ExceptionsHelper.unwrapCause(bulkItemResponse.getFailure().getCause());
                     assertThat(cause, instanceOf(CircuitBreakingException.class));
-                    assertEquals(((CircuitBreakingException) cause).getByteLimit(), inFlightRequestsLimit.bytes());
+                    assertEquals(((CircuitBreakingException) cause).getByteLimit(), inFlightRequestsLimit.getBytes());
                 }
             }
         } catch (CircuitBreakingException ex) {
-            assertEquals(ex.getByteLimit(), inFlightRequestsLimit.bytes());
+            assertEquals(ex.getByteLimit(), inFlightRequestsLimit.getBytes());
         }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerUnitTests.java b/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerUnitTests.java
index 033b3bb75caf4..1c8eaf71f8a35 100644
--- a/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerUnitTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerUnitTests.java
@@ -37,7 +37,7 @@
  */
 public class CircuitBreakerUnitTests extends ESTestCase {
     public static long pctBytes(String percentString) {
-        return Settings.EMPTY.getAsMemory("", percentString).bytes();
+        return Settings.EMPTY.getAsMemory("", percentString).getBytes();
     }
 
     public void testBreakerSettingsValidationWithValidSettings() {
diff --git a/core/src/test/java/org/elasticsearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java b/core/src/test/java/org/elasticsearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java
index 7054b0085d0f4..7b87a62288f30 100644
--- a/core/src/test/java/org/elasticsearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.indices.memory.breaker;
 
 import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.FilterDirectoryReader;
 import org.apache.lucene.index.LeafReader;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
@@ -39,16 +40,20 @@
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.search.basic.SearchWithRandomExceptionsIT;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.engine.MockEngineSupport;
 import org.elasticsearch.test.engine.ThrowingLeafReaderWrapper;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Random;
+import java.util.Set;
 import java.util.concurrent.ExecutionException;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllSuccessful;
@@ -60,7 +65,14 @@
 public class RandomExceptionCircuitBreakerIT extends ESIntegTestCase {
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Arrays.asList(RandomExceptionDirectoryReaderWrapper.TestPlugin.class, MockEngineFactoryPlugin.class);
+        return Arrays.asList(RandomExceptionDirectoryReaderWrapper.TestPlugin.class);
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> getMockPlugins() {
+        Set<Class<? extends Plugin>> mocks = new HashSet<>(super.getMockPlugins());
+        mocks.remove(MockEngineFactoryPlugin.class);
+        return mocks;
     }
 
     public void testBreakerWithRandomExceptions() throws IOException, InterruptedException, ExecutionException {
@@ -200,14 +212,19 @@ public static class RandomExceptionDirectoryReaderWrapper extends MockEngineSupp
             Setting.doubleSetting(EXCEPTION_TOP_LEVEL_RATIO_KEY, 0.1d, 0.0d, Property.IndexScope);
         public static final Setting<Double> EXCEPTION_LOW_LEVEL_RATIO_SETTING =
             Setting.doubleSetting(EXCEPTION_LOW_LEVEL_RATIO_KEY, 0.1d, 0.0d, Property.IndexScope);
-        public static class TestPlugin extends Plugin {
+        public static class TestPlugin extends MockEngineFactoryPlugin {
             @Override
             public List<Setting<?>> getSettings() {
-                return Arrays.asList(EXCEPTION_TOP_LEVEL_RATIO_SETTING, EXCEPTION_LOW_LEVEL_RATIO_SETTING);
+                List<Setting<?>> settings = new ArrayList<>();
+                settings.addAll(super.getSettings());
+                settings.add(EXCEPTION_TOP_LEVEL_RATIO_SETTING);
+                settings.add(EXCEPTION_LOW_LEVEL_RATIO_SETTING);
+                return settings;
             }
 
-            public void onModule(MockEngineFactoryPlugin.MockEngineReaderModule module) {
-                module.setReaderClass(RandomExceptionDirectoryReaderWrapper.class);
+            @Override
+            protected Class<? extends FilterDirectoryReader> getReaderWrapperClass() {
+                return RandomExceptionDirectoryReaderWrapper.class;
             }
         }
 
diff --git a/core/src/test/java/org/elasticsearch/indices/recovery/IndexPrimaryRelocationIT.java b/core/src/test/java/org/elasticsearch/indices/recovery/IndexPrimaryRelocationIT.java
index 16a926b9e7a2e..bb6b7df1ce9dc 100644
--- a/core/src/test/java/org/elasticsearch/indices/recovery/IndexPrimaryRelocationIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/recovery/IndexPrimaryRelocationIT.java
@@ -42,9 +42,10 @@
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST)
 public class IndexPrimaryRelocationIT extends ESIntegTestCase {
 
-    private static final int RELOCATION_COUNT = 25;
+    private static final int RELOCATION_COUNT = 15;
 
-    @TestLogging("_root:DEBUG,action.delete:TRACE,action.index:TRACE,index.shard:TRACE,cluster.service:TRACE")
+    @TestLogging("_root:DEBUG,org.elasticsearch.action.delete:TRACE,org.elasticsearch.action.index:TRACE," +
+        "org.elasticsearch.index.shard:TRACE,org.elasticsearch.cluster.service:TRACE,org.elasticsearch.action.bulk:TRACE")
     public void testPrimaryRelocationWhileIndexing() throws Exception {
         internalCluster().ensureAtLeastNumDataNodes(randomIntBetween(2, 3));
         client().admin().indices().prepareCreate("test")
@@ -88,7 +89,10 @@ public void run() {
             if (indexingThread.isAlive() == false) { // indexing process aborted early, no need for more relocations as test has already failed
                 break;
             }
-
+            if (i > 0  && i % 5 == 0) {
+                logger.info("--> [iteration {}] flushing index", i);
+                client().admin().indices().prepareFlush("test").get();
+            }
         }
         finished.set(true);
         indexingThread.join();
diff --git a/core/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java b/core/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java
index a28af0561be21..f5dea3ed84f3e 100644
--- a/core/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java
@@ -31,8 +31,8 @@
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.cluster.action.shard.ShardStateAction;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.RecoverySource.PeerRecoverySource;
 import org.elasticsearch.cluster.routing.RecoverySource.SnapshotRecoverySource;
@@ -42,11 +42,14 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.recovery.RecoveryStats;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.recovery.RecoveryState.Stage;
+import org.elasticsearch.node.RecoverySettingsChunkSizePlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.snapshots.SnapshotState;
@@ -54,12 +57,12 @@
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
 import org.elasticsearch.test.InternalTestCluster;
+import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.test.store.MockFSDirectoryService;
 import org.elasticsearch.test.store.MockFSIndexStore;
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.transport.ConnectTransportException;
 import org.elasticsearch.transport.Transport;
-import org.elasticsearch.transport.TransportException;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportService;
@@ -72,7 +75,10 @@
 import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
 
+import static org.elasticsearch.node.RecoverySettingsChunkSizePlugin.CHUNK_SIZE_SETTING;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.hamcrest.Matchers.equalTo;
@@ -101,7 +107,8 @@ public class IndexRecoveryIT extends ESIntegTestCase {
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Arrays.asList(MockTransportService.TestPlugin.class, MockFSIndexStore.TestPlugin.class);
+        return Arrays.asList(MockTransportService.TestPlugin.class, MockFSIndexStore.TestPlugin.class,
+                RecoverySettingsChunkSizePlugin.class);
     }
 
     private void assertRecoveryStateWithoutStage(RecoveryState state, int shardId, RecoverySource recoverySource, boolean primary,
@@ -136,27 +143,22 @@ private void assertOnGoingRecoveryState(RecoveryState state, int shardId, Recove
     }
 
     private void slowDownRecovery(ByteSizeValue shardSize) {
-        long chunkSize = Math.max(1, shardSize.bytes() / 10);
-        for(RecoverySettings settings : internalCluster().getInstances(RecoverySettings.class)) {
-            setChunkSize(settings, new ByteSizeValue(chunkSize, ByteSizeUnit.BYTES));
-        }
+        long chunkSize = Math.max(1, shardSize.getBytes() / 10);
         assertTrue(client().admin().cluster().prepareUpdateSettings()
                 .setTransientSettings(Settings.builder()
                                 // one chunk per sec..
                                 .put(RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey(), chunkSize, ByteSizeUnit.BYTES)
-                )
-                .get().isAcknowledged());
+                                // small chunks
+                                .put(CHUNK_SIZE_SETTING.getKey(), new ByteSizeValue(chunkSize, ByteSizeUnit.BYTES))
+                ).get().isAcknowledged());
     }
 
     private void restoreRecoverySpeed() {
-        for(RecoverySettings settings : internalCluster().getInstances(RecoverySettings.class)) {
-            setChunkSize(settings, RecoverySettings.DEFAULT_CHUNK_SIZE);
-        }
         assertTrue(client().admin().cluster().prepareUpdateSettings()
                 .setTransientSettings(Settings.builder()
                                 .put(RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey(), "20mb")
-                )
-                .get().isAcknowledged());
+                                .put(CHUNK_SIZE_SETTING.getKey(), RecoverySettings.DEFAULT_CHUNK_SIZE)
+                ).get().isAcknowledged());
     }
 
     public void testGatewayRecovery() throws Exception {
@@ -263,17 +265,13 @@ public void testRerouteRecovery() throws Exception {
 
         logger.info("--> waiting for recovery to start both on source and target");
         final Index index = resolveIndex(INDEX_NAME);
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-
-                IndicesService indicesService = internalCluster().getInstance(IndicesService.class, nodeA);
-                assertThat(indicesService.indexServiceSafe(index).getShard(0).recoveryStats().currentAsSource(),
-                        equalTo(1));
-                indicesService = internalCluster().getInstance(IndicesService.class, nodeB);
-                assertThat(indicesService.indexServiceSafe(index).getShard(0).recoveryStats().currentAsTarget(),
-                        equalTo(1));
-            }
+        assertBusy(() -> {
+            IndicesService indicesService = internalCluster().getInstance(IndicesService.class, nodeA);
+            assertThat(indicesService.indexServiceSafe(index).getShard(0).recoveryStats().currentAsSource(),
+                    equalTo(1));
+            indicesService = internalCluster().getInstance(IndicesService.class, nodeB);
+            assertThat(indicesService.indexServiceSafe(index).getShard(0).recoveryStats().currentAsTarget(),
+                    equalTo(1));
         });
 
         logger.info("--> request recoveries");
@@ -312,19 +310,16 @@ public void run() {
         logger.info("--> checking throttling increases");
         final long finalNodeAThrottling = nodeAThrottling;
         final long finalNodeBThrottling = nodeBThrottling;
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                NodesStatsResponse statsResponse = client().admin().cluster().prepareNodesStats().clear().setIndices(new CommonStatsFlags(CommonStatsFlags.Flag.Recovery)).get();
-                assertThat(statsResponse.getNodes(), hasSize(2));
-                for (NodeStats nodeStats : statsResponse.getNodes()) {
-                    final RecoveryStats recoveryStats = nodeStats.getIndices().getRecoveryStats();
-                    if (nodeStats.getNode().getName().equals(nodeA)) {
-                        assertThat("node A throttling should increase", recoveryStats.throttleTime().millis(), greaterThan(finalNodeAThrottling));
-                    }
-                    if (nodeStats.getNode().getName().equals(nodeB)) {
-                        assertThat("node B throttling should increase", recoveryStats.throttleTime().millis(), greaterThan(finalNodeBThrottling));
-                    }
+        assertBusy(() -> {
+            NodesStatsResponse statsResponse1 = client().admin().cluster().prepareNodesStats().clear().setIndices(new CommonStatsFlags(CommonStatsFlags.Flag.Recovery)).get();
+            assertThat(statsResponse1.getNodes(), hasSize(2));
+            for (NodeStats nodeStats : statsResponse1.getNodes()) {
+                final RecoveryStats recoveryStats = nodeStats.getIndices().getRecoveryStats();
+                if (nodeStats.getNode().getName().equals(nodeA)) {
+                    assertThat("node A throttling should increase", recoveryStats.throttleTime().millis(), greaterThan(finalNodeAThrottling));
+                }
+                if (nodeStats.getNode().getName().equals(nodeB)) {
+                    assertThat("node B throttling should increase", recoveryStats.throttleTime().millis(), greaterThan(finalNodeBThrottling));
                 }
             }
         });
@@ -532,7 +527,7 @@ private IndicesStatsResponse createAndPopulateIndex(String name, int nodeCount,
         for (int i = 0; i < numDocs; i++) {
             docs[i] = client().prepareIndex(name, INDEX_TYPE).
                     setSource("foo-int", randomInt(),
-                            "foo-string", randomAsciiOfLength(32),
+                            "foo-string", randomAlphaOfLength(32),
                             "foo-float", randomFloat());
         }
 
@@ -560,10 +555,8 @@ public void testDisconnectsWhileRecovering() throws Exception {
         // start a master node
         internalCluster().startNode(nodeSettings);
 
-        InternalTestCluster.Async<String> blueFuture = internalCluster().startNodeAsync(Settings.builder().put("node.attr.color", "blue").put(nodeSettings).build());
-        InternalTestCluster.Async<String> redFuture = internalCluster().startNodeAsync(Settings.builder().put("node.attr.color", "red").put(nodeSettings).build());
-        final String blueNodeName = blueFuture.get();
-        final String redNodeName = redFuture.get();
+        final String blueNodeName = internalCluster().startNode(Settings.builder().put("node.attr.color", "blue").put(nodeSettings).build());
+        final String redNodeName = internalCluster().startNode(Settings.builder().put("node.attr.color", "red").put(nodeSettings).build());
 
         ClusterHealthResponse response = client().admin().cluster().prepareHealth().setWaitForNodes(">=3").get();
         assertThat(response.isTimedOut(), is(false));
@@ -580,7 +573,7 @@ public void testDisconnectsWhileRecovering() throws Exception {
         List<IndexRequestBuilder> requests = new ArrayList<>();
         int numDocs = scaledRandomIntBetween(25, 250);
         for (int i = 0; i < numDocs; i++) {
-            requests.add(client().prepareIndex(indexName, "type").setSource("{}"));
+            requests.add(client().prepareIndex(indexName, "type").setSource("{}", XContentType.JSON));
         }
         indexRandom(true, requests);
         ensureSearchable(indexName);
@@ -639,7 +632,7 @@ private class RecoveryActionBlocker extends MockTransportService.DelegateTranspo
         private final String recoveryActionToBlock;
         private final CountDownLatch requestBlocked;
 
-        public RecoveryActionBlocker(boolean dropRequests, String recoveryActionToBlock, Transport delegate, CountDownLatch requestBlocked) {
+        RecoveryActionBlocker(boolean dropRequests, String recoveryActionToBlock, Transport delegate, CountDownLatch requestBlocked) {
             super(delegate);
             this.dropRequests = dropRequests;
             this.recoveryActionToBlock = recoveryActionToBlock;
@@ -647,20 +640,141 @@ public RecoveryActionBlocker(boolean dropRequests, String recoveryActionToBlock,
         }
 
         @Override
-        public void sendRequest(DiscoveryNode node, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException, TransportException {
+        protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request,
+                                   TransportRequestOptions options) throws IOException {
             if (recoveryActionToBlock.equals(action) || requestBlocked.getCount() == 0) {
                 logger.info("--> preventing {} request", action);
                 requestBlocked.countDown();
                 if (dropRequests) {
                     return;
                 }
-                throw new ConnectTransportException(node, "DISCONNECT: prevented " + action + " request");
+                throw new ConnectTransportException(connection.getNode(), "DISCONNECT: prevented " + action + " request");
             }
-            transport.sendRequest(node, requestId, action, request, options);
+            super.sendRequest(connection, requestId, action, request, options);
         }
     }
 
-    public static void setChunkSize(RecoverySettings recoverySettings, ByteSizeValue chunksSize) {
-        recoverySettings.setChunkSize(chunksSize);
+    /**
+     * Tests scenario where recovery target successfully sends recovery request to source but then the channel gets closed while
+     * the source is working on the recovery process.
+     */
+    @TestLogging("_root:DEBUG,org.elasticsearch.indices.recovery:TRACE")
+    public void testDisconnectsDuringRecovery() throws Exception {
+        boolean primaryRelocation = randomBoolean();
+        final String indexName = "test";
+        final Settings nodeSettings = Settings.builder()
+            .put(RecoverySettings.INDICES_RECOVERY_RETRY_DELAY_NETWORK_SETTING.getKey(), TimeValue.timeValueMillis(randomIntBetween(0, 100)))
+            .build();
+        TimeValue disconnectAfterDelay = TimeValue.timeValueMillis(randomIntBetween(0, 100));
+        // start a master node
+        String masterNodeName = internalCluster().startMasterOnlyNode(nodeSettings);
+
+        final String blueNodeName = internalCluster().startNode(Settings.builder().put("node.attr.color", "blue").put(nodeSettings).build());
+        final String redNodeName = internalCluster().startNode(Settings.builder().put("node.attr.color", "red").put(nodeSettings).build());
+
+        client().admin().indices().prepareCreate(indexName)
+            .setSettings(
+                Settings.builder()
+                    .put(IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + "color", "blue")
+                    .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+                    .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+            ).get();
+
+        List<IndexRequestBuilder> requests = new ArrayList<>();
+        int numDocs = scaledRandomIntBetween(25, 250);
+        for (int i = 0; i < numDocs; i++) {
+            requests.add(client().prepareIndex(indexName, "type").setSource("{}", XContentType.JSON));
+        }
+        indexRandom(true, requests);
+        ensureSearchable(indexName);
+        assertHitCount(client().prepareSearch(indexName).get(), numDocs);
+
+        MockTransportService masterTransportService = (MockTransportService) internalCluster().getInstance(TransportService.class, masterNodeName);
+        MockTransportService blueMockTransportService = (MockTransportService) internalCluster().getInstance(TransportService.class, blueNodeName);
+        MockTransportService redMockTransportService = (MockTransportService) internalCluster().getInstance(TransportService.class, redNodeName);
+
+        redMockTransportService.addDelegate(blueMockTransportService, new MockTransportService.DelegateTransport(redMockTransportService.original()) {
+            private final AtomicInteger count = new AtomicInteger();
+
+            @Override
+            protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request,
+                                       TransportRequestOptions options) throws IOException {
+                logger.info("--> sending request {} on {}", action, connection.getNode());
+                if (PeerRecoverySourceService.Actions.START_RECOVERY.equals(action) && count.incrementAndGet() == 1) {
+                    // ensures that it's considered as valid recovery attempt by source
+                    try {
+                        awaitBusy(() -> client(blueNodeName).admin().cluster().prepareState().setLocal(true).get()
+                            .getState().getRoutingTable().index("test").shard(0).getAllInitializingShards().isEmpty() == false);
+                    } catch (InterruptedException e) {
+                        throw new RuntimeException(e);
+                    }
+                    super.sendRequest(connection, requestId, action, request, options);
+                    try {
+                        Thread.sleep(disconnectAfterDelay.millis());
+                    } catch (InterruptedException e) {
+                        throw new RuntimeException(e);
+                    }
+                    throw new ConnectTransportException(connection.getNode(), "DISCONNECT: simulation disconnect after successfully sending " + action + " request");
+                } else {
+                    super.sendRequest(connection, requestId, action, request, options);
+                }
+            }
+        });
+
+        final AtomicBoolean finalized = new AtomicBoolean();
+        blueMockTransportService.addDelegate(redMockTransportService, new MockTransportService.DelegateTransport(blueMockTransportService.original()) {
+            @Override
+            protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request,
+                                       TransportRequestOptions options) throws IOException {
+                logger.info("--> sending request {} on {}", action, connection.getNode());
+                if (action.equals(PeerRecoveryTargetService.Actions.FINALIZE)) {
+                    finalized.set(true);
+                }
+                super.sendRequest(connection, requestId, action, request, options);
+            }
+        });
+
+        for (MockTransportService mockTransportService : Arrays.asList(redMockTransportService, blueMockTransportService)) {
+            mockTransportService.addDelegate(masterTransportService, new MockTransportService.DelegateTransport(mockTransportService.original()) {
+                @Override
+                protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request,
+                                           TransportRequestOptions options) throws IOException {
+                    logger.info("--> sending request {} on {}", action, connection.getNode());
+                    if ((primaryRelocation && finalized.get()) == false) {
+                        assertNotEquals(action, ShardStateAction.SHARD_FAILED_ACTION_NAME);
+                    }
+                    super.sendRequest(connection, requestId, action, request, options);
+                }
+            });
+        }
+
+        if (primaryRelocation) {
+            logger.info("--> starting primary relocation recovery from blue to red");
+            client().admin().indices().prepareUpdateSettings(indexName).setSettings(
+                Settings.builder()
+                    .put(IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + "color", "red")
+            ).get();
+
+            ensureGreen(); // also waits for relocation / recovery to complete
+            // if a primary relocation fails after the source shard has been marked as relocated, both source and target are failed. If the
+            // source shard is moved back to started because the target fails first, it's possible that there is a cluster state where the
+            // shard is marked as started again (and ensureGreen returns), but while applying the cluster state the primary is failed and
+            // will be reallocated. The cluster will thus become green, then red, then green again. Triggering a refresh here before
+            // searching helps, as in contrast to search actions, refresh waits for the closed shard to be reallocated.
+            client().admin().indices().prepareRefresh(indexName).get();
+        } else {
+            logger.info("--> starting replica recovery from blue to red");
+            client().admin().indices().prepareUpdateSettings(indexName).setSettings(
+                Settings.builder()
+                    .put(IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + "color", "red,blue")
+                    .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
+            ).get();
+
+            ensureGreen();
+        }
+
+        for (int i = 0; i < 10; i++) {
+            assertHitCount(client().prepareSearch(indexName).get(), numDocs);
+        }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/indices/recovery/RecoverySettingsDynamicUpdateTests.java b/core/src/test/java/org/elasticsearch/indices/recovery/RecoverySettingsDynamicUpdateTests.java
new file mode 100644
index 0000000000000..54e082098763e
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/indices/recovery/RecoverySettingsDynamicUpdateTests.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.indices.recovery;
+
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.indices.recovery.RecoverySettings;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.concurrent.TimeUnit;
+
+public class RecoverySettingsDynamicUpdateTests extends ESTestCase {
+    private final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+    private final RecoverySettings recoverySettings = new RecoverySettings(Settings.EMPTY, clusterSettings);
+
+    public void testZeroBytesPerSecondIsNoRateLimit() {
+        clusterSettings.applySettings(Settings.builder().put(
+                RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey(), 0).build());
+        assertEquals(null, recoverySettings.rateLimiter());
+    }
+
+    public void testRetryDelayStateSync() {
+        long duration = between(1, 1000);
+        TimeUnit timeUnit = randomFrom(TimeUnit.MILLISECONDS, TimeUnit.SECONDS, TimeUnit.MINUTES, TimeUnit.HOURS);
+        clusterSettings.applySettings(Settings.builder().put(
+                RecoverySettings.INDICES_RECOVERY_RETRY_DELAY_STATE_SYNC_SETTING.getKey(), duration, timeUnit
+        ).build());
+        assertEquals(new TimeValue(duration, timeUnit), recoverySettings.retryDelayStateSync());
+    }
+
+    public void testRetryDelayNetwork() {
+        long duration = between(1, 1000);
+        TimeUnit timeUnit = randomFrom(TimeUnit.MILLISECONDS, TimeUnit.SECONDS, TimeUnit.MINUTES, TimeUnit.HOURS);
+        clusterSettings.applySettings(Settings.builder().put(
+                RecoverySettings.INDICES_RECOVERY_RETRY_DELAY_NETWORK_SETTING.getKey(), duration, timeUnit
+        ).build());
+        assertEquals(new TimeValue(duration, timeUnit), recoverySettings.retryDelayNetwork());
+    }
+
+    public void testActivityTimeout() {
+        long duration = between(1, 1000);
+        TimeUnit timeUnit = randomFrom(TimeUnit.MILLISECONDS, TimeUnit.SECONDS, TimeUnit.MINUTES, TimeUnit.HOURS);
+        clusterSettings.applySettings(Settings.builder().put(
+                RecoverySettings.INDICES_RECOVERY_ACTIVITY_TIMEOUT_SETTING.getKey(), duration, timeUnit
+        ).build());
+        assertEquals(new TimeValue(duration, timeUnit), recoverySettings.activityTimeout());
+    }
+
+    public void testInternalActionTimeout() {
+        long duration = between(1, 1000);
+        TimeUnit timeUnit = randomFrom(TimeUnit.MILLISECONDS, TimeUnit.SECONDS, TimeUnit.MINUTES, TimeUnit.HOURS);
+        clusterSettings.applySettings(Settings.builder().put(
+                RecoverySettings.INDICES_RECOVERY_INTERNAL_ACTION_TIMEOUT_SETTING.getKey(), duration, timeUnit
+        ).build());
+        assertEquals(new TimeValue(duration, timeUnit), recoverySettings.internalActionTimeout());
+    }
+
+    public void testInternalLongActionTimeout() {
+        long duration = between(1, 1000);
+        TimeUnit timeUnit = randomFrom(TimeUnit.MILLISECONDS, TimeUnit.SECONDS, TimeUnit.MINUTES, TimeUnit.HOURS);
+        clusterSettings.applySettings(Settings.builder().put(
+                RecoverySettings.INDICES_RECOVERY_INTERNAL_LONG_ACTION_TIMEOUT_SETTING.getKey(), duration, timeUnit
+        ).build());
+        assertEquals(new TimeValue(duration, timeUnit), recoverySettings.internalActionLongTimeout());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java b/core/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java
index 0678ebef99fb2..d2b756bc0950b 100644
--- a/core/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java
@@ -35,7 +35,6 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.lucene.store.IndexOutputOutputStream;
@@ -81,7 +80,7 @@ public void testSendFiles() throws Throwable {
         Settings settings = Settings.builder().put("indices.recovery.concurrent_streams", 1).
                 put("indices.recovery.concurrent_small_file_streams", 1).build();
         final RecoverySettings recoverySettings = new RecoverySettings(settings, service);
-        StartRecoveryRequest request = new StartRecoveryRequest(shardId,
+        StartRecoveryRequest request = new StartRecoveryRequest(shardId, null,
                 new DiscoveryNode("b", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT),
                 new DiscoveryNode("b", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT),
             null, randomBoolean(), randomLong());
@@ -133,7 +132,7 @@ public void testHandleCorruptedIndexOnSendSendFiles() throws Throwable {
         Settings settings = Settings.builder().put("indices.recovery.concurrent_streams", 1).
                 put("indices.recovery.concurrent_small_file_streams", 1).build();
         final RecoverySettings recoverySettings = new RecoverySettings(settings, service);
-        StartRecoveryRequest request = new StartRecoveryRequest(shardId,
+        StartRecoveryRequest request = new StartRecoveryRequest(shardId, null,
                 new DiscoveryNode("b", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT),
                 new DiscoveryNode("b", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT),
             null, randomBoolean(), randomLong());
@@ -197,7 +196,7 @@ public void testHandleExceptinoOnSendSendFiles() throws Throwable {
         Settings settings = Settings.builder().put("indices.recovery.concurrent_streams", 1).
                 put("indices.recovery.concurrent_small_file_streams", 1).build();
         final RecoverySettings recoverySettings = new RecoverySettings(settings, service);
-        StartRecoveryRequest request = new StartRecoveryRequest(shardId,
+        StartRecoveryRequest request = new StartRecoveryRequest(shardId, null,
                 new DiscoveryNode("b", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT),
                 new DiscoveryNode("b", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT),
             null, randomBoolean(), randomLong());
@@ -256,7 +255,7 @@ protected void failEngine(IOException cause) {
 
     public void testThrowExceptionOnPrimaryRelocatedBeforePhase1Completed() throws IOException {
         final RecoverySettings recoverySettings = new RecoverySettings(Settings.EMPTY, service);
-        StartRecoveryRequest request = new StartRecoveryRequest(shardId,
+        StartRecoveryRequest request = new StartRecoveryRequest(shardId, null,
             new DiscoveryNode("b", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT),
             new DiscoveryNode("b", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT),
             null, false, randomLong());
@@ -286,7 +285,7 @@ public void phase2(Translog.Snapshot snapshot) {
 
     public void testWaitForClusterStateOnPrimaryRelocation() throws IOException, InterruptedException {
         final RecoverySettings recoverySettings = new RecoverySettings(Settings.EMPTY, service);
-        StartRecoveryRequest request = new StartRecoveryRequest(shardId,
+        StartRecoveryRequest request = new StartRecoveryRequest(shardId, null,
             new DiscoveryNode("b", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT),
             new DiscoveryNode("b", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT),
             null, true, randomLong());
diff --git a/core/src/test/java/org/elasticsearch/indices/recovery/RecoveryTargetTests.java b/core/src/test/java/org/elasticsearch/indices/recovery/RecoveryTargetTests.java
index 6d37ae7d0d721..79ee9553167b5 100644
--- a/core/src/test/java/org/elasticsearch/indices/recovery/RecoveryTargetTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/recovery/RecoveryTargetTests.java
@@ -158,12 +158,7 @@ Timer createObj() {
         Timer lastRead = streamer.serializeDeserialize();
         final long time = lastRead.time();
         assertThat(time, lessThanOrEqualTo(timer.time()));
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                assertThat("timer timer should progress compared to captured one ", time, lessThan(timer.time()));
-            }
-        });
+        assertBusy(() -> assertThat("timer timer should progress compared to captured one ", time, lessThan(timer.time())));
         assertThat("captured time shouldn't change", lastRead.time(), equalTo(time));
 
         if (randomBoolean()) {
@@ -522,7 +517,7 @@ Index createObj() {
             @Override
             public void run() {
                 for (int i = 0; i < 1000; i++) {
-                    index.addFileDetail(randomAsciiOfLength(10), 100, true);
+                    index.addFileDetail(randomAlphaOfLength(10), 100, true);
                 }
                 stop.set(true);
             }
diff --git a/core/src/test/java/org/elasticsearch/indices/recovery/StartRecoveryRequestTests.java b/core/src/test/java/org/elasticsearch/indices/recovery/StartRecoveryRequestTests.java
index 7065ffa5dfc6e..af1be7faf8b78 100644
--- a/core/src/test/java/org/elasticsearch/indices/recovery/StartRecoveryRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/recovery/StartRecoveryRequestTests.java
@@ -21,6 +21,7 @@
 
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.io.stream.InputStreamStreamInput;
 import org.elasticsearch.common.io.stream.OutputStreamStreamOutput;
 import org.elasticsearch.common.transport.LocalTransportAddress;
@@ -43,6 +44,7 @@ public void testSerialization() throws Exception {
         Version targetNodeVersion = randomVersion(random());
         StartRecoveryRequest outRequest = new StartRecoveryRequest(
                 new ShardId("test", "_na_", 0),
+                UUIDs.base64UUID(),
                 new DiscoveryNode("a", new LocalTransportAddress("1"), emptyMap(), emptySet(), targetNodeVersion),
                 new DiscoveryNode("b", new LocalTransportAddress("1"), emptyMap(), emptySet(), targetNodeVersion),
                 Store.MetadataSnapshot.EMPTY,
@@ -61,6 +63,11 @@ public void testSerialization() throws Exception {
         inRequest.readFrom(in);
 
         assertThat(outRequest.shardId(), equalTo(inRequest.shardId()));
+        if (targetNodeVersion.onOrAfter(Version.V_5_4_0)) {
+            assertThat(outRequest.targetAllocationId(), equalTo(inRequest.targetAllocationId()));
+        } else {
+            assertNull(inRequest.targetAllocationId());
+        }
         assertThat(outRequest.sourceNode(), equalTo(inRequest.sourceNode()));
         assertThat(outRequest.targetNode(), equalTo(inRequest.targetNode()));
         assertThat(outRequest.metadataSnapshot().asMap(), equalTo(inRequest.metadataSnapshot().asMap()));
diff --git a/core/src/test/java/org/elasticsearch/indices/settings/GetSettingsBlocksIT.java b/core/src/test/java/org/elasticsearch/indices/settings/GetSettingsBlocksIT.java
index d53230d3a9ee8..cb45a639c07eb 100644
--- a/core/src/test/java/org/elasticsearch/indices/settings/GetSettingsBlocksIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/settings/GetSettingsBlocksIT.java
@@ -30,6 +30,7 @@
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_READ;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_WRITE;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY_ALLOW_DELETE;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBlocked;
 import static org.hamcrest.Matchers.equalTo;
@@ -43,7 +44,7 @@ public void testGetSettingsWithBlocks() throws Exception {
                         .put("index.merge.policy.expunge_deletes_allowed", "30")
                         .put(MapperService.INDEX_MAPPER_DYNAMIC_SETTING.getKey(), false)));
 
-        for (String block : Arrays.asList(SETTING_BLOCKS_READ, SETTING_BLOCKS_WRITE, SETTING_READ_ONLY)) {
+        for (String block : Arrays.asList(SETTING_BLOCKS_READ, SETTING_BLOCKS_WRITE, SETTING_READ_ONLY, SETTING_READ_ONLY_ALLOW_DELETE)) {
             try {
                 enableIndexBlock("test", block);
                 GetSettingsResponse response = client().admin().indices().prepareGetSettings("test").get();
diff --git a/core/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java b/core/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java
index 81d1e05e9e045..6f8a775c56c54 100644
--- a/core/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java
@@ -19,27 +19,19 @@
 
 package org.elasticsearch.indices.settings;
 
-import org.apache.logging.log4j.Level;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.core.LogEvent;
-import org.apache.logging.log4j.core.appender.AbstractAppender;
-import org.apache.logging.log4j.core.filter.RegexFilter;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.Priority;
-import org.elasticsearch.common.logging.Loggers;
-import org.elasticsearch.common.logging.TestLoggers;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.IndexService;
-import org.elasticsearch.index.engine.VersionConflictEngineException;
 import org.elasticsearch.index.MergePolicyConfig;
 import org.elasticsearch.index.MergeSchedulerConfig;
+import org.elasticsearch.index.engine.VersionConflictEngineException;
 import org.elasticsearch.index.store.IndexStore;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.indices.IndicesService;
@@ -58,20 +50,34 @@
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBlocked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.nullValue;
 
 public class UpdateSettingsIT extends ESIntegTestCase {
-
+    public void testInvalidUpdateOnClosedIndex() {
+        createIndex("test");
+        assertAcked(client().admin().indices().prepareClose("test").get());
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () ->
+            client()
+                .admin()
+                .indices()
+                .prepareUpdateSettings("test")
+                .setSettings(Settings.builder().put("index.analysis.char_filter.invalid_char.type", "invalid"))
+                .get());
+        assertEquals(exception.getMessage(), "Unknown char_filter type [invalid] for [invalid_char]");
+    }
 
     public void testInvalidDynamicUpdate() {
         createIndex("test");
         IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () ->
-            client().admin().indices().prepareUpdateSettings("test")
-                .setSettings(Settings.builder()
-                    .put("index.dummy", "boom")
-                )
-                .execute().actionGet());
+            client()
+                .admin()
+                .indices()
+                .prepareUpdateSettings("test")
+                .setSettings(Settings.builder().put("index.dummy", "boom"))
+                .execute()
+                .actionGet());
         assertEquals(exception.getCause().getMessage(), "this setting goes boom");
         IndexMetaData indexMetaData = client().admin().cluster().prepareState().execute().actionGet().getState().metaData().index("test");
         assertNotEquals(indexMetaData.getSettings().get("index.dummy"), "invalid dynamic value");
@@ -79,7 +85,7 @@ public void testInvalidDynamicUpdate() {
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Arrays.asList(DummySettingPlugin.class);
+        return Arrays.asList(DummySettingPlugin.class, FinalSettingPlugin.class);
     }
 
     public static class DummySettingPlugin extends Plugin {
@@ -99,67 +105,97 @@ public List<Setting<?>> getSettings() {
         }
     }
 
+    public static class FinalSettingPlugin extends Plugin {
+        public static final Setting<String> FINAL_SETTING = Setting.simpleString("index.final",
+            Setting.Property.IndexScope, Setting.Property.Final);
+        @Override
+        public void onIndexModule(IndexModule indexModule) {
+        }
+
+        @Override
+        public List<Setting<?>> getSettings() {
+            return Collections.singletonList(FINAL_SETTING);
+        }
+    }
+
     public void testResetDefault() {
         createIndex("test");
 
-        client().admin().indices().prepareUpdateSettings("test")
-            .setSettings(Settings.builder()
-                .put("index.refresh_interval", -1)
-                .put("index.translog.flush_threshold_size", "1024b")
-            )
-            .execute().actionGet();
+        client()
+            .admin()
+            .indices()
+            .prepareUpdateSettings("test")
+            .setSettings(Settings.builder().put("index.refresh_interval", -1).put("index.translog.flush_threshold_size", "1024b"))
+            .execute()
+            .actionGet();
         IndexMetaData indexMetaData = client().admin().cluster().prepareState().execute().actionGet().getState().metaData().index("test");
         assertEquals(indexMetaData.getSettings().get("index.refresh_interval"), "-1");
         for (IndicesService service : internalCluster().getInstances(IndicesService.class)) {
             IndexService indexService = service.indexService(resolveIndex("test"));
             if (indexService != null) {
                 assertEquals(indexService.getIndexSettings().getRefreshInterval().millis(), -1);
-                assertEquals(indexService.getIndexSettings().getFlushThresholdSize().bytes(), 1024);
+                assertEquals(indexService.getIndexSettings().getFlushThresholdSize().getBytes(), 1024);
             }
         }
-        client().admin().indices().prepareUpdateSettings("test")
-            .setSettings(Settings.builder()
-                .putNull("index.refresh_interval")
-            )
-            .execute().actionGet();
+        client()
+            .admin()
+            .indices()
+            .prepareUpdateSettings("test")
+            .setSettings(Settings.builder().putNull("index.refresh_interval"))
+            .execute()
+            .actionGet();
         indexMetaData = client().admin().cluster().prepareState().execute().actionGet().getState().metaData().index("test");
         assertNull(indexMetaData.getSettings().get("index.refresh_interval"));
         for (IndicesService service : internalCluster().getInstances(IndicesService.class)) {
             IndexService indexService = service.indexService(resolveIndex("test"));
             if (indexService != null) {
                 assertEquals(indexService.getIndexSettings().getRefreshInterval().millis(), 1000);
-                assertEquals(indexService.getIndexSettings().getFlushThresholdSize().bytes(), 1024);
+                assertEquals(indexService.getIndexSettings().getFlushThresholdSize().getBytes(), 1024);
             }
         }
     }
     public void testOpenCloseUpdateSettings() throws Exception {
         createIndex("test");
-        try {
-            client().admin().indices().prepareUpdateSettings("test")
-                    .setSettings(Settings.builder()
-                            .put("index.refresh_interval", -1) // this one can change
-                            .put("index.fielddata.cache", "none") // this one can't
-                    )
-                    .execute().actionGet();
-            fail();
-        } catch (IllegalArgumentException e) {
-            // all is well
-        }
-
+        expectThrows(IllegalArgumentException.class, () ->
+            client()
+                .admin()
+                .indices()
+                .prepareUpdateSettings("test")
+                .setSettings(Settings.builder()
+                    .put("index.refresh_interval", -1) // this one can change
+                    .put("index.fielddata.cache", "none")) // this one can't
+                .execute()
+                .actionGet()
+        );
+        expectThrows(IllegalArgumentException.class, () ->
+            client()
+                .admin()
+                .indices()
+                .prepareUpdateSettings("test")
+                .setSettings(Settings.builder()
+                    .put("index.refresh_interval", -1) // this one can change
+                    .put("index.final", "no")) // this one can't
+                .execute()
+                .actionGet()
+        );
         IndexMetaData indexMetaData = client().admin().cluster().prepareState().execute().actionGet().getState().metaData().index("test");
         assertThat(indexMetaData.getSettings().get("index.refresh_interval"), nullValue());
         assertThat(indexMetaData.getSettings().get("index.fielddata.cache"), nullValue());
+        assertThat(indexMetaData.getSettings().get("index.final"), nullValue());
 
         // Now verify via dedicated get settings api:
         GetSettingsResponse getSettingsResponse = client().admin().indices().prepareGetSettings("test").get();
         assertThat(getSettingsResponse.getSetting("test", "index.refresh_interval"), nullValue());
         assertThat(getSettingsResponse.getSetting("test", "index.fielddata.cache"), nullValue());
+        assertThat(getSettingsResponse.getSetting("test", "index.final"), nullValue());
 
-        client().admin().indices().prepareUpdateSettings("test")
-                .setSettings(Settings.builder()
-                        .put("index.refresh_interval", -1) // this one can change
-                )
-                .execute().actionGet();
+        client()
+            .admin()
+            .indices()
+            .prepareUpdateSettings("test")
+            .setSettings(Settings.builder().put("index.refresh_interval", -1)) // this one can change
+            .execute()
+            .actionGet();
 
         indexMetaData = client().admin().cluster().prepareState().execute().actionGet().getState().metaData().index("test");
         assertThat(indexMetaData.getSettings().get("index.refresh_interval"), equalTo("-1"));
@@ -170,53 +206,83 @@ public void testOpenCloseUpdateSettings() throws Exception {
         // now close the index, change the non dynamic setting, and see that it applies
 
         // Wait for the index to turn green before attempting to close it
-        ClusterHealthResponse health = client().admin().cluster().prepareHealth().setTimeout("30s").setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().execute().actionGet();
+        ClusterHealthResponse health =
+            client()
+                .admin()
+                .cluster()
+                .prepareHealth()
+                .setTimeout("30s")
+                .setWaitForEvents(Priority.LANGUID)
+                .setWaitForGreenStatus()
+                .execute()
+                .actionGet();
         assertThat(health.isTimedOut(), equalTo(false));
 
         client().admin().indices().prepareClose("test").execute().actionGet();
 
         try {
-            client().admin().indices().prepareUpdateSettings("test")
-                    .setSettings(Settings.builder()
-                                    .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
-                    )
-                    .execute().actionGet();
+            client()
+                .admin()
+                .indices()
+                .prepareUpdateSettings("test")
+                .setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1))
+                .execute()
+                .actionGet();
             fail("can't change number of replicas on a closed index");
         } catch (IllegalArgumentException ex) {
             assertTrue(ex.getMessage(), ex.getMessage().startsWith("Can't update [index.number_of_replicas] on closed indices [[test/"));
             assertTrue(ex.getMessage(), ex.getMessage().endsWith("]] - can leave index in an unopenable state"));
             // expected
         }
-        client().admin().indices().prepareUpdateSettings("test")
-                .setSettings(Settings.builder()
-                        .put("index.refresh_interval", "1s") // this one can change
-                        .put("index.fielddata.cache", "none") // this one can't
-                )
-                .execute().actionGet();
+        client()
+            .admin()
+            .indices()
+            .prepareUpdateSettings("test")
+            .setSettings(Settings.builder()
+                .put("index.refresh_interval", "1s") // this one can change
+                .put("index.fielddata.cache", "none")) // this one can't
+            .execute()
+            .actionGet();
 
         indexMetaData = client().admin().cluster().prepareState().execute().actionGet().getState().metaData().index("test");
         assertThat(indexMetaData.getSettings().get("index.refresh_interval"), equalTo("1s"));
         assertThat(indexMetaData.getSettings().get("index.fielddata.cache"), equalTo("none"));
 
+        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () ->
+            client()
+                .admin()
+                .indices()
+                .prepareUpdateSettings("test")
+                .setSettings(Settings.builder()
+                    .put("index.refresh_interval", -1) // this one can change
+                    .put("index.final", "no")) // this one really can't
+                .execute()
+                .actionGet()
+        );
+        assertThat(ex.getMessage(), containsString("final test setting [index.final], not updateable"));
+        indexMetaData = client().admin().cluster().prepareState().execute().actionGet().getState().metaData().index("test");
+        assertThat(indexMetaData.getSettings().get("index.refresh_interval"), equalTo("1s"));
+        assertThat(indexMetaData.getSettings().get("index.final"), nullValue());
+
+
         // Now verify via dedicated get settings api:
         getSettingsResponse = client().admin().indices().prepareGetSettings("test").get();
         assertThat(getSettingsResponse.getSetting("test", "index.refresh_interval"), equalTo("1s"));
-        assertThat(getSettingsResponse.getSetting("test", "index.fielddata.cache"), equalTo("none"));
+        assertThat(getSettingsResponse.getSetting("test", "index.final"), nullValue());
     }
 
     public void testEngineGCDeletesSetting() throws InterruptedException {
         createIndex("test");
         client().prepareIndex("test", "type", "1").setSource("f", 1).get(); // set version to 1
         client().prepareDelete("test", "type", "1").get(); // sets version to 2
-        client().prepareIndex("test", "type", "1").setSource("f", 2).setVersion(2).get(); // delete is still in cache this should work & set version to 3
-        client().admin().indices().prepareUpdateSettings("test")
-                .setSettings(Settings.builder()
-                        .put("index.gc_deletes", 0)
-                ).get();
+        // delete is still in cache this should work & set version to 3
+        client().prepareIndex("test", "type", "1").setSource("f", 2).setVersion(2).get();
+        client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder().put("index.gc_deletes", 0)).get();
 
         client().prepareDelete("test", "type", "1").get(); // sets version to 4
         Thread.sleep(300); // wait for cache time to change TODO: this needs to be solved better. To be discussed.
-        assertThrows(client().prepareIndex("test", "type", "1").setSource("f", 3).setVersion(4), VersionConflictEngineException.class); // delete is should not be in cache
+        // delete is should not be in cache
+        assertThrows(client().prepareIndex("test", "type", "1").setSource("f", 3).setVersion(4), VersionConflictEngineException.class);
 
     }
 
@@ -262,9 +328,10 @@ public void testUpdateThrottleSettings() {
             .admin()
             .indices()
             .prepareUpdateSettings("test")
-            .setSettings(Settings.builder()
-                         .put(IndexStore.INDEX_STORE_THROTTLE_TYPE_SETTING.getKey(), "merge")
-                             .put(IndexStore.INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC_SETTING.getKey(), "1mb"))
+            .setSettings(
+                Settings.builder()
+                    .put(IndexStore.INDEX_STORE_THROTTLE_TYPE_SETTING.getKey(), "merge")
+                    .put(IndexStore.INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC_SETTING.getKey(), "1mb"))
             .get();
 
         // Make sure setting says it is in fact changed:
@@ -302,8 +369,7 @@ public void testUpdateThrottleSettings() {
             .admin()
             .indices()
             .prepareUpdateSettings("test")
-            .setSettings(Settings.builder()
-                         .put(IndexStore.INDEX_STORE_THROTTLE_TYPE_SETTING.getKey(), "none"))
+            .setSettings(Settings.builder().put(IndexStore.INDEX_STORE_THROTTLE_TYPE_SETTING.getKey(), "none"))
             .get();
 
         // Optimize does a waitForMerges, which we must do to make sure all in-flight (throttled) merges finish:
@@ -352,115 +418,6 @@ public void testUpdateThrottleSettings() {
         logger.info("test: test done");
     }
 
-    private static class MockAppender extends AbstractAppender {
-        public boolean sawUpdateMaxThreadCount;
-        public boolean sawUpdateAutoThrottle;
-
-        public MockAppender(final String name) throws IllegalAccessException {
-            super(name, RegexFilter.createFilter(".*(\n.*)*", new String[0], true, null, null), null);
-        }
-
-        @Override
-        public void append(LogEvent event) {
-            String message = event.getMessage().getFormattedMessage();
-            if (event.getLevel() == Level.TRACE &&
-                event.getLoggerName().endsWith("lucene.iw")) {
-            }
-            if (event.getLevel() == Level.INFO && message.contains("updating [index.merge.scheduler.max_thread_count] from [10000] to [1]")) {
-                sawUpdateMaxThreadCount = true;
-            }
-            if (event.getLevel() == Level.INFO && message.contains("updating [index.merge.scheduler.auto_throttle] from [true] to [false]")) {
-                sawUpdateAutoThrottle = true;
-            }
-        }
-
-    }
-
-    public void testUpdateAutoThrottleSettings() throws IllegalAccessException {
-        MockAppender mockAppender = new MockAppender("testUpdateAutoThrottleSettings");
-        Logger rootLogger = LogManager.getRootLogger();
-        Level savedLevel = rootLogger.getLevel();
-        TestLoggers.addAppender(rootLogger, mockAppender);
-        Loggers.setLevel(rootLogger, Level.TRACE);
-
-        try {
-            // No throttling at first, only 1 non-replicated shard, force lots of merging:
-            assertAcked(prepareCreate("test")
-                        .setSettings(Settings.builder()
-                                     .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1")
-                                     .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0")
-                                     .put(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING.getKey(), "2")
-                                     .put(MergePolicyConfig.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING.getKey(), "2")
-                                     .put(MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING.getKey(), "1")
-                                     .put(MergeSchedulerConfig.MAX_MERGE_COUNT_SETTING.getKey(), "2")
-                                     .put(MergeSchedulerConfig.AUTO_THROTTLE_SETTING.getKey(), "true")
-                                     ));
-
-            // Disable auto throttle:
-            client()
-                .admin()
-                .indices()
-                .prepareUpdateSettings("test")
-                .setSettings(Settings.builder()
-                             .put(MergeSchedulerConfig.AUTO_THROTTLE_SETTING.getKey(), "false"))
-                .get();
-
-            // Make sure we log the change:
-            assertTrue(mockAppender.sawUpdateAutoThrottle);
-
-            // Make sure setting says it is in fact changed:
-            GetSettingsResponse getSettingsResponse = client().admin().indices().prepareGetSettings("test").get();
-            assertThat(getSettingsResponse.getSetting("test", MergeSchedulerConfig.AUTO_THROTTLE_SETTING.getKey()), equalTo("false"));
-        } finally {
-            TestLoggers.removeAppender(rootLogger, mockAppender);
-            Loggers.setLevel(rootLogger, savedLevel);
-        }
-    }
-
-    // #6882: make sure we can change index.merge.scheduler.max_thread_count live
-    public void testUpdateMergeMaxThreadCount() throws IllegalAccessException {
-        MockAppender mockAppender = new MockAppender("testUpdateMergeMaxThreadCount");
-        Logger rootLogger = LogManager.getRootLogger();
-        Level savedLevel = rootLogger.getLevel();
-        TestLoggers.addAppender(rootLogger, mockAppender);
-        Loggers.setLevel(rootLogger, Level.TRACE);
-
-        try {
-
-            assertAcked(prepareCreate("test")
-                        .setSettings(Settings.builder()
-                                     .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1")
-                                     .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0")
-                                     .put(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING.getKey(), "2")
-                                     .put(MergePolicyConfig.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING.getKey(), "2")
-                                     .put(MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING.getKey(), "10000")
-                                     .put(MergeSchedulerConfig.MAX_MERGE_COUNT_SETTING.getKey(), "10000")
-                                     ));
-
-            assertFalse(mockAppender.sawUpdateMaxThreadCount);
-            // Now make a live change to reduce allowed merge threads:
-            client()
-                .admin()
-                .indices()
-                .prepareUpdateSettings("test")
-                .setSettings(Settings.builder()
-                             .put(MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING.getKey(), "1")
-                             )
-                .get();
-
-            // Make sure we log the change:
-            assertTrue(mockAppender.sawUpdateMaxThreadCount);
-
-            // Make sure setting says it is in fact changed:
-            GetSettingsResponse getSettingsResponse = client().admin().indices().prepareGetSettings("test").get();
-            assertThat(getSettingsResponse.getSetting("test", MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING.getKey()), equalTo("1"));
-
-        } finally {
-            TestLoggers.removeAppender(rootLogger, mockAppender);
-            Loggers.setLevel(rootLogger, savedLevel);
-        }
-    }
-
     public void testUpdateSettingsWithBlocks() {
         createIndex("test");
         ensureGreen("test");
@@ -486,4 +443,5 @@ public void testUpdateSettingsWithBlocks() {
             }
         }
     }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/indices/state/CloseIndexDisableCloseAllIT.java b/core/src/test/java/org/elasticsearch/indices/state/CloseIndexDisableCloseAllIT.java
index 8ec629dbbdc5b..54bdfd05008e7 100644
--- a/core/src/test/java/org/elasticsearch/indices/state/CloseIndexDisableCloseAllIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/state/CloseIndexDisableCloseAllIT.java
@@ -18,99 +18,48 @@
  */
 package org.elasticsearch.indices.state;
 
-import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
-import org.elasticsearch.action.admin.indices.close.CloseIndexResponse;
 import org.elasticsearch.action.admin.indices.close.TransportCloseIndexAction;
-import org.elasticsearch.action.support.DestructiveOperations;
-import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
-import org.elasticsearch.test.ESIntegTestCase.Scope;
+import org.junit.After;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.notNullValue;
 
-@ClusterScope(scope=Scope.TEST, numDataNodes=2)
 public class CloseIndexDisableCloseAllIT extends ESIntegTestCase {
-    // Combined multiple tests into one, because cluster scope is test.
-    // The cluster scope is test b/c we can't clear cluster settings.
-    public void testCloseAllRequiresName() {
-        Settings clusterSettings = Settings.builder()
-                .put(DestructiveOperations.REQUIRES_NAME_SETTING.getKey(), true)
-                .build();
-        assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(clusterSettings));
-        createIndex("test1", "test2", "test3");
-        ClusterHealthResponse healthResponse = client().admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet();
-        assertThat(healthResponse.isTimedOut(), equalTo(false));
-
-        // Close all explicitly
-        try {
-            client().admin().indices().prepareClose("_all").execute().actionGet();
-            fail();
-        } catch (IllegalArgumentException e) {
-        }
-
-        // Close all wildcard
-        try {
-            client().admin().indices().prepareClose("*").execute().actionGet();
-            fail();
-        } catch (IllegalArgumentException e) {
-        }
-
-        // Close all wildcard
-        try {
-            client().admin().indices().prepareClose("test*").execute().actionGet();
-            fail();
-        } catch (IllegalArgumentException e) {
-        }
 
-        // Close all wildcard
-        try {
-            client().admin().indices().prepareClose("*", "-test1").execute().actionGet();
-            fail();
-        } catch (IllegalArgumentException e) {
-        }
+    @After
+    public void afterTest() {
+        Settings settings = Settings.builder().put(TransportCloseIndexAction.CLUSTER_INDICES_CLOSE_ENABLE_SETTING.getKey(), (String)null)
+                .build();
+        assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(settings));
+    }
 
-        // Close all wildcard
-        try {
-            client().admin().indices().prepareClose("*", "-test1", "+test1").execute().actionGet();
-            fail();
-        } catch (IllegalArgumentException e) {
-        }
+    public void testCloseAllRequiresName() {
+        createIndex("test1", "test2", "test3");
 
-        CloseIndexResponse closeIndexResponse = client().admin().indices().prepareClose("test3", "test2").execute().actionGet();
-        assertThat(closeIndexResponse.isAcknowledged(), equalTo(true));
+        assertAcked(client().admin().indices().prepareClose("test3", "test2"));
         assertIndexIsClosed("test2", "test3");
 
         // disable closing
-        Client client = client();
         createIndex("test_no_close");
-        healthResponse = client.admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet();
-        assertThat(healthResponse.isTimedOut(), equalTo(false));
-        client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put(TransportCloseIndexAction.CLUSTER_INDICES_CLOSE_ENABLE_SETTING.getKey(), false)).get();
+        Settings settings = Settings.builder().put(TransportCloseIndexAction.CLUSTER_INDICES_CLOSE_ENABLE_SETTING.getKey(), false).build();
+        assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(settings));
 
-        try {
-            client.admin().indices().prepareClose("test_no_close").execute().actionGet();
-            fail("exception expected");
-        } catch (IllegalStateException ex) {
-            assertEquals(ex.getMessage(), "closing indices is disabled - set [cluster.indices.close.enable: true] to enable it. NOTE: closed indices still consume a significant amount of diskspace");
-        }
+        IllegalStateException illegalStateException = expectThrows(IllegalStateException.class,
+                () -> client().admin().indices().prepareClose("test_no_close").get());
+        assertEquals(illegalStateException.getMessage(),
+                "closing indices is disabled - set [cluster.indices.close.enable: true] to enable it. NOTE: closed indices still " +
+                        "consume a significant amount of diskspace");
     }
 
     private void assertIndexIsClosed(String... indices) {
-        checkIndexState(IndexMetaData.State.CLOSE, indices);
-    }
-
-    private void checkIndexState(IndexMetaData.State state, String... indices) {
         ClusterStateResponse clusterStateResponse = client().admin().cluster().prepareState().execute().actionGet();
         for (String index : indices) {
             IndexMetaData indexMetaData = clusterStateResponse.getState().metaData().indices().get(index);
-            assertThat(indexMetaData, notNullValue());
-            assertThat(indexMetaData.getState(), equalTo(state));
+            assertNotNull(indexMetaData);
+            assertEquals(IndexMetaData.State.CLOSE, indexMetaData.getState());
         }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/indices/state/OpenCloseIndexIT.java b/core/src/test/java/org/elasticsearch/indices/state/OpenCloseIndexIT.java
index 8eef10d693b48..19c027b930a28 100644
--- a/core/src/test/java/org/elasticsearch/indices/state/OpenCloseIndexIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/state/OpenCloseIndexIT.java
@@ -43,6 +43,7 @@
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_READ;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_WRITE;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY_ALLOW_DELETE;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBlocked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
@@ -70,22 +71,16 @@ public void testSimpleCloseOpen() {
 
     public void testSimpleCloseMissingIndex() {
         Client client = client();
-        try {
-            client.admin().indices().prepareClose("test1").execute().actionGet();
-            fail("Expected IndexNotFoundException");
-        } catch (IndexNotFoundException e) {
-            assertThat(e.getMessage(), is("no such index"));
-        }
+        Exception e = expectThrows(IndexNotFoundException.class, () ->
+            client.admin().indices().prepareClose("test1").execute().actionGet());
+        assertThat(e.getMessage(), is("no such index"));
     }
 
     public void testSimpleOpenMissingIndex() {
         Client client = client();
-        try {
-            client.admin().indices().prepareOpen("test1").execute().actionGet();
-            fail("Expected IndexNotFoundException");
-        } catch (IndexNotFoundException e) {
-            assertThat(e.getMessage(), is("no such index"));
-        }
+        Exception e = expectThrows(IndexNotFoundException.class, () ->
+            client.admin().indices().prepareOpen("test1").execute().actionGet());
+        assertThat(e.getMessage(), is("no such index"));
     }
 
     public void testCloseOneMissingIndex() {
@@ -93,12 +88,9 @@ public void testCloseOneMissingIndex() {
         createIndex("test1");
         ClusterHealthResponse healthResponse = client.admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet();
         assertThat(healthResponse.isTimedOut(), equalTo(false));
-        try {
-            client.admin().indices().prepareClose("test1", "test2").execute().actionGet();
-            fail("Expected IndexNotFoundException");
-        } catch (IndexNotFoundException e) {
-            assertThat(e.getMessage(), is("no such index"));
-        }
+        Exception e = expectThrows(IndexNotFoundException.class, () ->
+            client.admin().indices().prepareClose("test1", "test2").execute().actionGet());
+        assertThat(e.getMessage(), is("no such index"));
     }
 
     public void testCloseOneMissingIndexIgnoreMissing() {
@@ -117,12 +109,9 @@ public void testOpenOneMissingIndex() {
         createIndex("test1");
         ClusterHealthResponse healthResponse = client.admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet();
         assertThat(healthResponse.isTimedOut(), equalTo(false));
-        try {
-            client.admin().indices().prepareOpen("test1", "test2").execute().actionGet();
-            fail("Expected IndexNotFoundException");
-        } catch (IndexNotFoundException e) {
-            assertThat(e.getMessage(), is("no such index"));
-        }
+        Exception e = expectThrows(IndexNotFoundException.class, () ->
+            client.admin().indices().prepareOpen("test1", "test2").execute().actionGet());
+        assertThat(e.getMessage(), is("no such index"));
     }
 
     public void testOpenOneMissingIndexIgnoreMissing() {
@@ -202,44 +191,67 @@ public void testCloseOpenAllWildcard() {
         assertIndexIsOpened("test1", "test2", "test3");
     }
 
+    // if there are no indices to open/close throw an exception
+    public void testOpenCloseWildCardsNoIndicesDefault() {
+        expectThrows(IndexNotFoundException.class, () -> client().admin().indices().prepareOpen("test").execute().actionGet());
+        expectThrows(IndexNotFoundException.class, () -> client().admin().indices().prepareClose("test").execute().actionGet());
+
+        expectThrows(IndexNotFoundException.class, () -> client().admin().indices().prepareOpen("test*").execute().actionGet());
+        expectThrows(IndexNotFoundException.class, () -> client().admin().indices().prepareClose("test*").execute().actionGet());
+
+        expectThrows(IndexNotFoundException.class, () -> client().admin().indices().prepareOpen("*").execute().actionGet());
+        expectThrows(IndexNotFoundException.class, () -> client().admin().indices().prepareClose("*").execute().actionGet());
+
+        expectThrows(IndexNotFoundException.class, () -> client().admin().indices().prepareOpen("_all").execute().actionGet());
+        expectThrows(IndexNotFoundException.class, () -> client().admin().indices().prepareClose("_all").execute().actionGet());
+    }
+
+    // if there are no indices to open/close and allow_no_indices=true, the open/close is a no-op
+    public void testOpenCloseWildCardsNoIndicesAllowNoIndices() throws InterruptedException, ExecutionException {
+        IndicesOptions openIndicesOptions = IndicesOptions.fromOptions(false, true, false, true);
+        IndicesOptions closeIndicesOptions = IndicesOptions.fromOptions(false, true, true, false);
+
+        expectThrows(IndexNotFoundException.class,
+                () -> client().admin().indices().prepareOpen("test").setIndicesOptions(openIndicesOptions).execute().actionGet());
+        expectThrows(IndexNotFoundException.class,
+                () -> client().admin().indices().prepareClose("test").setIndicesOptions(closeIndicesOptions).execute().actionGet());
+
+        assertAcked(client().admin().indices().prepareOpen("test*").setIndicesOptions(openIndicesOptions).execute().get());
+        assertAcked(client().admin().indices().prepareClose("test*").setIndicesOptions(closeIndicesOptions).execute().get());
+
+        assertAcked(client().admin().indices().prepareOpen("*").setIndicesOptions(openIndicesOptions).execute().get());
+        assertAcked(client().admin().indices().prepareClose("*").setIndicesOptions(closeIndicesOptions).execute().get());
+
+        assertAcked(client().admin().indices().prepareOpen("_all").setIndicesOptions(openIndicesOptions).execute().get());
+        assertAcked(client().admin().indices().prepareClose("_all").setIndicesOptions(closeIndicesOptions).execute().get());
+    }
+
     public void testCloseNoIndex() {
         Client client = client();
-        try {
-            client.admin().indices().prepareClose().execute().actionGet();
-            fail("Expected ActionRequestValidationException");
-        } catch (ActionRequestValidationException e) {
-            assertThat(e.getMessage(), containsString("index is missing"));
-        }
+        Exception e = expectThrows(ActionRequestValidationException.class, () ->
+            client.admin().indices().prepareClose().execute().actionGet());
+        assertThat(e.getMessage(), containsString("index is missing"));
     }
 
     public void testCloseNullIndex() {
         Client client = client();
-        try {
-            client.admin().indices().prepareClose((String[])null).execute().actionGet();
-            fail("Expected ActionRequestValidationException");
-        } catch (ActionRequestValidationException e) {
-            assertThat(e.getMessage(), containsString("index is missing"));
-        }
+        Exception e = expectThrows(ActionRequestValidationException.class, () ->
+            client.admin().indices().prepareClose((String[])null).execute().actionGet());
+        assertThat(e.getMessage(), containsString("index is missing"));
     }
 
     public void testOpenNoIndex() {
         Client client = client();
-        try {
-            client.admin().indices().prepareOpen().execute().actionGet();
-            fail("Expected ActionRequestValidationException");
-        } catch (ActionRequestValidationException e) {
-            assertThat(e.getMessage(), containsString("index is missing"));
-        }
+        Exception e = expectThrows(ActionRequestValidationException.class, () ->
+            client.admin().indices().prepareOpen().execute().actionGet());
+        assertThat(e.getMessage(), containsString("index is missing"));
     }
 
     public void testOpenNullIndex() {
         Client client = client();
-        try {
-            client.admin().indices().prepareOpen((String[])null).execute().actionGet();
-            fail("Expected ActionRequestValidationException");
-        } catch (ActionRequestValidationException e) {
-            assertThat(e.getMessage(), containsString("index is missing"));
-        }
+        Exception e = expectThrows(ActionRequestValidationException.class, () ->
+            client.admin().indices().prepareOpen((String[])null).execute().actionGet());
+        assertThat(e.getMessage(), containsString("index is missing"));
     }
 
     public void testOpenAlreadyOpenedIndex() {
@@ -348,7 +360,7 @@ public void testOpenCloseWithDocs() throws IOException, ExecutionException, Inte
         }
         indexRandom(true, builder);
         if (randomBoolean()) {
-            client().admin().indices().prepareFlush("test").setWaitIfOngoing(true).setForce(true).execute().get();
+            client().admin().indices().prepareFlush("test").setForce(true).execute().get();
         }
         client().admin().indices().prepareClose("test").execute().get();
 
@@ -403,7 +415,7 @@ public void testOpenCloseIndexWithBlocks() {
         assertIndexIsClosed("test");
 
         // Opening an index is blocked
-        for (String blockSetting : Arrays.asList(SETTING_READ_ONLY, SETTING_BLOCKS_METADATA)) {
+        for (String blockSetting : Arrays.asList(SETTING_READ_ONLY, SETTING_READ_ONLY_ALLOW_DELETE, SETTING_BLOCKS_METADATA)) {
             try {
                 enableIndexBlock("test", blockSetting);
                 assertBlocked(client().admin().indices().prepareOpen("test"));
@@ -413,4 +425,4 @@ public void testOpenCloseIndexWithBlocks() {
             }
         }
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/test/java/org/elasticsearch/indices/state/RareClusterStateIT.java b/core/src/test/java/org/elasticsearch/indices/state/RareClusterStateIT.java
index 4964defc62adc..d079ad94892d9 100644
--- a/core/src/test/java/org/elasticsearch/indices/state/RareClusterStateIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/state/RareClusterStateIT.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.indices.state;
 
+import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
@@ -37,14 +38,12 @@
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.discovery.DiscoveryModule;
 import org.elasticsearch.discovery.DiscoverySettings;
 import org.elasticsearch.gateway.GatewayAllocator;
 import org.elasticsearch.index.Index;
@@ -53,6 +52,7 @@
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.discovery.TestZenDiscovery;
 import org.elasticsearch.test.disruption.BlockClusterStateProcessing;
 import org.elasticsearch.test.junit.annotations.TestLogging;
 
@@ -80,6 +80,13 @@
 @ESIntegTestCase.SuppressLocalMode
 @TestLogging("_root:DEBUG")
 public class RareClusterStateIT extends ESIntegTestCase {
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        return Settings.builder().put(super.nodeSettings(nodeOrdinal))
+            .put(TestZenDiscovery.USE_MOCK_PINGS.getKey(), false).build();
+    }
+
     @Override
     protected int numberOfShards() {
         return 1;
@@ -140,8 +147,7 @@ public ClusterState execute(ClusterState currentState) throws Exception {
                 routingTable.addAsRecovery(updatedState.metaData().index(index));
                 updatedState = ClusterState.builder(updatedState).routingTable(routingTable.build()).build();
 
-                RoutingAllocation.Result result = allocationService.reroute(updatedState, "reroute");
-                return ClusterState.builder(updatedState).routingResult(result).build();
+                return allocationService.reroute(updatedState, "reroute");
 
             }
 
@@ -159,8 +165,7 @@ public ClusterState execute(ClusterState currentState) throws Exception {
                 builder.nodes(DiscoveryNodes.builder(currentState.nodes()).remove("_non_existent"));
 
                 currentState = builder.build();
-                RoutingAllocation.Result result = allocationService.deassociateDeadNodes(currentState, true, "reroute");
-                return ClusterState.builder(currentState).routingResult(result).build();
+                return allocationService.deassociateDeadNodes(currentState, true, "reroute");
 
             }
 
@@ -173,9 +178,7 @@ public void onFailure(String source, Exception e) {
 
     @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/14932")
     public void testDeleteCreateInOneBulk() throws Exception {
-        internalCluster().startNodesAsync(2, Settings.builder()
-                .put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), "zen")
-                .build()).get();
+        internalCluster().startNodes(2);
         assertFalse(client().admin().cluster().prepareHealth().setWaitForNodes("2").get().isTimedOut());
         prepareCreate("test").setSettings(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, true).addMapping("type").get();
         ensureGreen("test");
@@ -210,8 +213,11 @@ public void testDelayedMappingPropagationOnPrimary() throws Exception {
         // but the change might not be on the node that performed the indexing
         // operation yet
 
-        Settings settings = Settings.builder().put(DiscoverySettings.PUBLISH_TIMEOUT_SETTING.getKey(), "0ms").build();
-        final List<String> nodeNames = internalCluster().startNodesAsync(2, settings).get();
+        Settings settings = Settings.builder()
+            .put(DiscoverySettings.COMMIT_TIMEOUT_SETTING.getKey(), "30s") // explicitly set so it won't default to publish timeout
+            .put(DiscoverySettings.PUBLISH_TIMEOUT_SETTING.getKey(), "0s") // don't wait post commit as we are blocking things by design
+            .build();
+        final List<String> nodeNames = internalCluster().startNodes(2, settings);
         assertFalse(client().admin().cluster().prepareHealth().setWaitForNodes("2").get().isTimedOut());
 
         final String master = internalCluster().getMasterName();
@@ -265,23 +271,20 @@ public void onFailure(Exception e) {
             }
         });
         // ...and wait for mappings to be available on master
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                ImmutableOpenMap<String, MappingMetaData> indexMappings = client().admin().indices().prepareGetMappings("index").get().getMappings().get("index");
-                assertNotNull(indexMappings);
-                MappingMetaData typeMappings = indexMappings.get("type");
-                assertNotNull(typeMappings);
-                Object properties;
-                try {
-                    properties = typeMappings.getSourceAsMap().get("properties");
-                } catch (IOException e) {
-                    throw new AssertionError(e);
-                }
-                assertNotNull(properties);
-                Object fieldMapping = ((Map<String, Object>) properties).get("field");
-                assertNotNull(fieldMapping);
+        assertBusy(() -> {
+            ImmutableOpenMap<String, MappingMetaData> indexMappings = client().admin().indices().prepareGetMappings("index").get().getMappings().get("index");
+            assertNotNull(indexMappings);
+            MappingMetaData typeMappings = indexMappings.get("type");
+            assertNotNull(typeMappings);
+            Object properties;
+            try {
+                properties = typeMappings.getSourceAsMap().get("properties");
+            } catch (ElasticsearchParseException e) {
+                throw new AssertionError(e);
             }
+            assertNotNull(properties);
+            Object fieldMapping = ((Map<String, Object>) properties).get("field");
+            assertNotNull(fieldMapping);
         });
 
         final AtomicReference<Object> docIndexResponse = new AtomicReference<>();
@@ -306,17 +309,14 @@ public void onFailure(Exception e) {
 
         // Now make sure the indexing request finishes successfully
         disruption.stopDisrupting();
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                assertThat(putMappingResponse.get(), instanceOf(PutMappingResponse.class));
-                PutMappingResponse resp = (PutMappingResponse) putMappingResponse.get();
-                assertTrue(resp.isAcknowledged());
-                assertThat(docIndexResponse.get(), instanceOf(IndexResponse.class));
-                IndexResponse docResp = (IndexResponse) docIndexResponse.get();
-                assertEquals(Arrays.toString(docResp.getShardInfo().getFailures()),
-                        1, docResp.getShardInfo().getTotal());
-            }
+        assertBusy(() -> {
+            assertThat(putMappingResponse.get(), instanceOf(PutMappingResponse.class));
+            PutMappingResponse resp = (PutMappingResponse) putMappingResponse.get();
+            assertTrue(resp.isAcknowledged());
+            assertThat(docIndexResponse.get(), instanceOf(IndexResponse.class));
+            IndexResponse docResp = (IndexResponse) docIndexResponse.get();
+            assertEquals(Arrays.toString(docResp.getShardInfo().getFailures()),
+                    1, docResp.getShardInfo().getTotal());
         });
     }
 
@@ -326,7 +326,11 @@ public void testDelayedMappingPropagationOnReplica() throws Exception {
         // Here we want to test that everything goes well if the mappings that
         // are needed for a document are not available on the replica at the
         // time of indexing it
-        final List<String> nodeNames = internalCluster().startNodesAsync(2).get();
+        final List<String> nodeNames = internalCluster().startNodes(2,
+            Settings.builder()
+                .put(DiscoverySettings.COMMIT_TIMEOUT_SETTING.getKey(), "30s") // explicitly set so it won't default to publish timeout
+                .put(DiscoverySettings.PUBLISH_TIMEOUT_SETTING.getKey(), "0s") // don't wait post commit as we are blocking things by design
+                .build());
         assertFalse(client().admin().cluster().prepareHealth().setWaitForNodes("2").get().isTimedOut());
 
         final String master = internalCluster().getMasterName();
@@ -382,17 +386,14 @@ public void onFailure(Exception e) {
         });
         final Index index = resolveIndex("index");
         // Wait for mappings to be available on master
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                final IndicesService indicesService = internalCluster().getInstance(IndicesService.class, master);
-                final IndexService indexService = indicesService.indexServiceSafe(index);
-                assertNotNull(indexService);
-                final MapperService mapperService = indexService.mapperService();
-                DocumentMapper mapper = mapperService.documentMapper("type");
-                assertNotNull(mapper);
-                assertNotNull(mapper.mappers().getMapper("field"));
-            }
+        assertBusy(() -> {
+            final IndicesService indicesService = internalCluster().getInstance(IndicesService.class, master);
+            final IndexService indexService = indicesService.indexServiceSafe(index);
+            assertNotNull(indexService);
+            final MapperService mapperService = indexService.mapperService();
+            DocumentMapper mapper = mapperService.documentMapper("type");
+            assertNotNull(mapper);
+            assertNotNull(mapper.mappers().getMapper("field"));
         });
 
         final AtomicReference<Object> docIndexResponse = new AtomicReference<>();
@@ -409,12 +410,7 @@ public void onFailure(Exception e) {
         });
 
         // Wait for document to be indexed on primary
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                assertTrue(client().prepareGet("index", "type", "1").setPreference("_primary").get().isExists());
-            }
-        });
+        assertBusy(() -> assertTrue(client().prepareGet("index", "type", "1").setPreference("_primary").get().isExists()));
 
         // The mappings have not been propagated to the replica yet as a consequence the document count not be indexed
         // We wait on purpose to make sure that the document is not indexed because the shard operation is stalled
@@ -425,17 +421,14 @@ public void run() {
 
         // Now make sure the indexing request finishes successfully
         disruption.stopDisrupting();
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                assertThat(putMappingResponse.get(), instanceOf(PutMappingResponse.class));
-                PutMappingResponse resp = (PutMappingResponse) putMappingResponse.get();
-                assertTrue(resp.isAcknowledged());
-                assertThat(docIndexResponse.get(), instanceOf(IndexResponse.class));
-                IndexResponse docResp = (IndexResponse) docIndexResponse.get();
-                assertEquals(Arrays.toString(docResp.getShardInfo().getFailures()),
-                        2, docResp.getShardInfo().getTotal()); // both shards should have succeeded
-            }
+        assertBusy(() -> {
+            assertThat(putMappingResponse.get(), instanceOf(PutMappingResponse.class));
+            PutMappingResponse resp = (PutMappingResponse) putMappingResponse.get();
+            assertTrue(resp.isAcknowledged());
+            assertThat(docIndexResponse.get(), instanceOf(IndexResponse.class));
+            IndexResponse docResp = (IndexResponse) docIndexResponse.get();
+            assertEquals(Arrays.toString(docResp.getShardInfo().getFailures()),
+                    2, docResp.getShardInfo().getTotal()); // both shards should have succeeded
         });
     }
 
diff --git a/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java b/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java
index b95d872a61e99..83a4161ac36f1 100644
--- a/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java
@@ -21,21 +21,26 @@
 
 import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
 import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.action.ShardOperationFailedException;
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.admin.indices.stats.CommonStats;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags.Flag;
 import org.elasticsearch.action.admin.indices.stats.IndexStats;
+import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.elasticsearch.action.admin.indices.stats.ShardStats;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.MergePolicyConfig;
@@ -55,14 +60,27 @@
 import org.elasticsearch.test.ESIntegTestCase.Scope;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.EnumSet;
+import java.util.List;
 import java.util.Random;
+import java.util.concurrent.BrokenBarrierException;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
 
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllSuccessful;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
+import static org.hamcrest.Matchers.emptyCollectionOf;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.is;
@@ -252,12 +270,7 @@ public void testQueryCache() throws Exception {
         }
         indexRandom(true, builders);
         refresh();
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                assertThat(client().admin().indices().prepareStats("idx").setRequestCache(true).get().getTotal().getRequestCache().getMemorySizeInBytes(), equalTo(0L));
-            }
-        });
+        assertBusy(() -> assertThat(client().admin().indices().prepareStats("idx").setRequestCache(true).get().getTotal().getRequestCache().getMemorySizeInBytes(), equalTo(0L)));
 
         for (int i = 0; i < 10; i++) {
             assertThat(client().prepareSearch("idx").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0).get().getHits().getTotalHits(), equalTo((long) numDocs));
@@ -489,11 +502,11 @@ public void testSimpleStats() throws Exception {
         } catch (VersionConflictEngineException e) {}
 
         stats = client().admin().indices().prepareStats().setTypes("type1", "type2").execute().actionGet();
-        assertThat(stats.getIndex("test1").getTotal().getIndexing().getTotal().getIndexFailedCount(), equalTo(2L));
-        assertThat(stats.getIndex("test2").getTotal().getIndexing().getTotal().getIndexFailedCount(), equalTo(1L));
+        assertThat(stats.getIndex("test1").getPrimaries().getIndexing().getTotal().getIndexFailedCount(), equalTo(2L));
+        assertThat(stats.getIndex("test2").getPrimaries().getIndexing().getTotal().getIndexFailedCount(), equalTo(1L));
         assertThat(stats.getPrimaries().getIndexing().getTypeStats().get("type1").getIndexFailedCount(), equalTo(1L));
         assertThat(stats.getPrimaries().getIndexing().getTypeStats().get("type2").getIndexFailedCount(), equalTo(1L));
-        assertThat(stats.getTotal().getIndexing().getTotal().getIndexFailedCount(), equalTo(3L));
+        assertThat(stats.getPrimaries().getIndexing().getTotal().getIndexFailedCount(), equalTo(3L));
     }
 
     public void testMergeStats() {
@@ -725,8 +738,8 @@ public void testFieldDataFieldsParam() throws Exception {
 
         ensureGreen();
 
-        client().prepareIndex("test1", "bar", Integer.toString(1)).setSource("{\"bar\":\"bar\",\"baz\":\"baz\"}").execute().actionGet();
-        client().prepareIndex("test1", "baz", Integer.toString(1)).setSource("{\"bar\":\"bar\",\"baz\":\"baz\"}").execute().actionGet();
+        client().prepareIndex("test1", "bar", Integer.toString(1)).setSource("{\"bar\":\"bar\",\"baz\":\"baz\"}", XContentType.JSON).get();
+        client().prepareIndex("test1", "baz", Integer.toString(1)).setSource("{\"bar\":\"bar\",\"baz\":\"baz\"}", XContentType.JSON).get();
         refresh();
 
         client().prepareSearch("_all").addSort("bar", SortOrder.ASC).addSort("baz", SortOrder.ASC).execute().actionGet();
@@ -739,29 +752,29 @@ public void testFieldDataFieldsParam() throws Exception {
 
         stats = builder.setFieldDataFields("bar").execute().actionGet();
         assertThat(stats.getTotal().fieldData.getMemorySizeInBytes(), greaterThan(0L));
-        assertThat(stats.getTotal().fieldData.getFields().containsKey("bar"), is(true));
+        assertThat(stats.getTotal().fieldData.getFields().containsField("bar"), is(true));
         assertThat(stats.getTotal().fieldData.getFields().get("bar"), greaterThan(0L));
-        assertThat(stats.getTotal().fieldData.getFields().containsKey("baz"), is(false));
+        assertThat(stats.getTotal().fieldData.getFields().containsField("baz"), is(false));
 
         stats = builder.setFieldDataFields("bar", "baz").execute().actionGet();
         assertThat(stats.getTotal().fieldData.getMemorySizeInBytes(), greaterThan(0L));
-        assertThat(stats.getTotal().fieldData.getFields().containsKey("bar"), is(true));
+        assertThat(stats.getTotal().fieldData.getFields().containsField("bar"), is(true));
         assertThat(stats.getTotal().fieldData.getFields().get("bar"), greaterThan(0L));
-        assertThat(stats.getTotal().fieldData.getFields().containsKey("baz"), is(true));
+        assertThat(stats.getTotal().fieldData.getFields().containsField("baz"), is(true));
         assertThat(stats.getTotal().fieldData.getFields().get("baz"), greaterThan(0L));
 
         stats = builder.setFieldDataFields("*").execute().actionGet();
         assertThat(stats.getTotal().fieldData.getMemorySizeInBytes(), greaterThan(0L));
-        assertThat(stats.getTotal().fieldData.getFields().containsKey("bar"), is(true));
+        assertThat(stats.getTotal().fieldData.getFields().containsField("bar"), is(true));
         assertThat(stats.getTotal().fieldData.getFields().get("bar"), greaterThan(0L));
-        assertThat(stats.getTotal().fieldData.getFields().containsKey("baz"), is(true));
+        assertThat(stats.getTotal().fieldData.getFields().containsField("baz"), is(true));
         assertThat(stats.getTotal().fieldData.getFields().get("baz"), greaterThan(0L));
 
         stats = builder.setFieldDataFields("*r").execute().actionGet();
         assertThat(stats.getTotal().fieldData.getMemorySizeInBytes(), greaterThan(0L));
-        assertThat(stats.getTotal().fieldData.getFields().containsKey("bar"), is(true));
+        assertThat(stats.getTotal().fieldData.getFields().containsField("bar"), is(true));
         assertThat(stats.getTotal().fieldData.getFields().get("bar"), greaterThan(0L));
-        assertThat(stats.getTotal().fieldData.getFields().containsKey("baz"), is(false));
+        assertThat(stats.getTotal().fieldData.getFields().containsField("baz"), is(false));
 
     }
 
@@ -772,8 +785,8 @@ public void testCompletionFieldsParam() throws Exception {
                         "{ \"properties\": { \"bar\": { \"type\": \"text\", \"fields\": { \"completion\": { \"type\": \"completion\" }}},\"baz\": { \"type\": \"text\", \"fields\": { \"completion\": { \"type\": \"completion\" }}}}}"));
         ensureGreen();
 
-        client().prepareIndex("test1", "bar", Integer.toString(1)).setSource("{\"bar\":\"bar\",\"baz\":\"baz\"}").execute().actionGet();
-        client().prepareIndex("test1", "baz", Integer.toString(1)).setSource("{\"bar\":\"bar\",\"baz\":\"baz\"}").execute().actionGet();
+        client().prepareIndex("test1", "bar", Integer.toString(1)).setSource("{\"bar\":\"bar\",\"baz\":\"baz\"}", XContentType.JSON).get();
+        client().prepareIndex("test1", "baz", Integer.toString(1)).setSource("{\"bar\":\"bar\",\"baz\":\"baz\"}", XContentType.JSON).get();
         refresh();
 
         IndicesStatsRequestBuilder builder = client().admin().indices().prepareStats();
@@ -784,29 +797,29 @@ public void testCompletionFieldsParam() throws Exception {
 
         stats = builder.setCompletionFields("bar.completion").execute().actionGet();
         assertThat(stats.getTotal().completion.getSizeInBytes(), greaterThan(0L));
-        assertThat(stats.getTotal().completion.getFields().containsKey("bar.completion"), is(true));
+        assertThat(stats.getTotal().completion.getFields().containsField("bar.completion"), is(true));
         assertThat(stats.getTotal().completion.getFields().get("bar.completion"), greaterThan(0L));
-        assertThat(stats.getTotal().completion.getFields().containsKey("baz.completion"), is(false));
+        assertThat(stats.getTotal().completion.getFields().containsField("baz.completion"), is(false));
 
         stats = builder.setCompletionFields("bar.completion", "baz.completion").execute().actionGet();
         assertThat(stats.getTotal().completion.getSizeInBytes(), greaterThan(0L));
-        assertThat(stats.getTotal().completion.getFields().containsKey("bar.completion"), is(true));
+        assertThat(stats.getTotal().completion.getFields().containsField("bar.completion"), is(true));
         assertThat(stats.getTotal().completion.getFields().get("bar.completion"), greaterThan(0L));
-        assertThat(stats.getTotal().completion.getFields().containsKey("baz.completion"), is(true));
+        assertThat(stats.getTotal().completion.getFields().containsField("baz.completion"), is(true));
         assertThat(stats.getTotal().completion.getFields().get("baz.completion"), greaterThan(0L));
 
         stats = builder.setCompletionFields("*").execute().actionGet();
         assertThat(stats.getTotal().completion.getSizeInBytes(), greaterThan(0L));
-        assertThat(stats.getTotal().completion.getFields().containsKey("bar.completion"), is(true));
+        assertThat(stats.getTotal().completion.getFields().containsField("bar.completion"), is(true));
         assertThat(stats.getTotal().completion.getFields().get("bar.completion"), greaterThan(0L));
-        assertThat(stats.getTotal().completion.getFields().containsKey("baz.completion"), is(true));
+        assertThat(stats.getTotal().completion.getFields().containsField("baz.completion"), is(true));
         assertThat(stats.getTotal().completion.getFields().get("baz.completion"), greaterThan(0L));
 
         stats = builder.setCompletionFields("*r*").execute().actionGet();
         assertThat(stats.getTotal().completion.getSizeInBytes(), greaterThan(0L));
-        assertThat(stats.getTotal().completion.getFields().containsKey("bar.completion"), is(true));
+        assertThat(stats.getTotal().completion.getFields().containsField("bar.completion"), is(true));
         assertThat(stats.getTotal().completion.getFields().get("bar.completion"), greaterThan(0L));
-        assertThat(stats.getTotal().completion.getFields().containsKey("baz.completion"), is(false));
+        assertThat(stats.getTotal().completion.getFields().containsField("baz.completion"), is(false));
 
     }
 
@@ -1071,4 +1084,103 @@ public void testFilterCacheStats() throws Exception {
         assertThat(response.getTotal().queryCache.getMemorySizeInBytes(), equalTo(0L));
     }
 
+    /**
+     * Test that we can safely concurrently index and get stats. This test was inspired by a serialization issue that arose due to a race
+     * getting doc stats during heavy indexing. The race could lead to deleted docs being negative which would then be serialized as a
+     * variable-length long. Since serialization of negative longs using a variable-length format was unsupported
+     * ({@link org.elasticsearch.common.io.stream.StreamOutput#writeVLong(long)}), the stream would become corrupted. Here, we want to test
+     * that we can continue to get stats while indexing.
+     */
+    public void testConcurrentIndexingAndStatsRequests() throws BrokenBarrierException, InterruptedException, ExecutionException {
+        final AtomicInteger idGenerator = new AtomicInteger();
+        final int numberOfIndexingThreads = Runtime.getRuntime().availableProcessors();
+        final int numberOfStatsThreads = 4 * numberOfIndexingThreads;
+        final CyclicBarrier barrier = new CyclicBarrier(1 + numberOfIndexingThreads + numberOfStatsThreads);
+        final AtomicBoolean stop = new AtomicBoolean();
+        final List<Thread> threads = new ArrayList<>(numberOfIndexingThreads + numberOfIndexingThreads);
+
+        final CountDownLatch latch = new CountDownLatch(1);
+        final AtomicBoolean failed = new AtomicBoolean();
+        final AtomicReference<List<ShardOperationFailedException>> shardFailures = new AtomicReference<>(new CopyOnWriteArrayList<>());
+        final AtomicReference<List<Exception>> executionFailures = new AtomicReference<>(new CopyOnWriteArrayList<>());
+
+        // increasing the number of shards increases the number of chances any one stats request will hit a race
+        final CreateIndexRequest createIndexRequest =
+            new CreateIndexRequest("test", Settings.builder().put("index.number_of_shards", 10).build());
+        client().admin().indices().create(createIndexRequest).get();
+
+        // start threads that will index concurrently with stats requests
+        for (int i = 0; i < numberOfIndexingThreads; i++) {
+            final Thread thread = new Thread(() -> {
+                try {
+                    barrier.await();
+                } catch (final BrokenBarrierException | InterruptedException e) {
+                    failed.set(true);
+                    executionFailures.get().add(e);
+                    latch.countDown();
+                }
+                while (!stop.get()) {
+                    final String id = Integer.toString(idGenerator.incrementAndGet());
+                    final IndexResponse response =
+                        client()
+                            .prepareIndex("test", "type", id)
+                            .setSource("{}", XContentType.JSON)
+                            .get();
+                    assertThat(response.getResult(), equalTo(DocWriteResponse.Result.CREATED));
+                }
+            });
+            thread.setName("indexing-" + i);
+            threads.add(thread);
+            thread.start();
+        }
+
+        // start threads that will get stats concurrently with indexing
+        for (int i = 0; i < numberOfStatsThreads; i++) {
+            final Thread thread = new Thread(() -> {
+                try {
+                    barrier.await();
+                } catch (final BrokenBarrierException | InterruptedException e) {
+                    failed.set(true);
+                    executionFailures.get().add(e);
+                    latch.countDown();
+                }
+                final IndicesStatsRequest request = new IndicesStatsRequest();
+                request.all();
+                request.indices(new String[0]);
+                while (!stop.get()) {
+                    try {
+                        final IndicesStatsResponse response = client().admin().indices().stats(request).get();
+                        if (response.getFailedShards() > 0) {
+                            failed.set(true);
+                            shardFailures.get().addAll(Arrays.asList(response.getShardFailures()));
+                            latch.countDown();
+                        }
+                    } catch (final ExecutionException | InterruptedException e) {
+                        failed.set(true);
+                        executionFailures.get().add(e);
+                        latch.countDown();
+                    }
+                }
+            });
+            thread.setName("stats-" + i);
+            threads.add(thread);
+            thread.start();
+        }
+
+        // release the hounds
+        barrier.await();
+
+        // wait for a failure, or for fifteen seconds to elapse
+        latch.await(15, TimeUnit.SECONDS);
+
+        // stop all threads and wait for them to complete
+        stop.set(true);
+        for (final Thread thread : threads) {
+            thread.join();
+        }
+
+        assertThat(shardFailures.get(), emptyCollectionOf(ShardOperationFailedException.class));
+        assertThat(executionFailures.get(), emptyCollectionOf(Exception.class));
+    }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java b/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java
index b248fc811f674..f05910cb2f5e3 100644
--- a/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java
@@ -23,10 +23,9 @@
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.LocalClusterUpdateTask;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.RoutingNode;
@@ -45,7 +44,7 @@
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
-import org.elasticsearch.indices.recovery.PeerRecoverySourceService;
+import org.elasticsearch.indices.recovery.PeerRecoveryTargetService;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -53,10 +52,8 @@
 import org.elasticsearch.test.ESIntegTestCase.Scope;
 import org.elasticsearch.test.InternalTestCluster;
 import org.elasticsearch.test.disruption.BlockClusterStateProcessing;
-import org.elasticsearch.test.disruption.SingleNodeDisruption;
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.transport.ConnectTransportException;
-import org.elasticsearch.transport.TransportException;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportService;
@@ -139,18 +136,7 @@ public void testIndexCleanup() throws Exception {
         logger.info("--> move shard from node_1 to node_3, and wait for relocation to finish");
 
         if (randomBoolean()) { // sometimes add cluster-state delay to trigger observers in IndicesStore.ShardActiveRequestHandler
-            SingleNodeDisruption disruption = new BlockClusterStateProcessing(node_3, random());
-            internalCluster().setDisruptionScheme(disruption);
-            MockTransportService transportServiceNode3 = (MockTransportService) internalCluster().getInstance(TransportService.class, node_3);
-            CountDownLatch beginRelocationLatch = new CountDownLatch(1);
-            CountDownLatch endRelocationLatch = new CountDownLatch(1);
-            transportServiceNode3.addTracer(new ReclocationStartEndTracer(logger, beginRelocationLatch, endRelocationLatch));
-            internalCluster().client().admin().cluster().prepareReroute().add(new MoveAllocationCommand("test", 0, node_1, node_3)).get();
-            // wait for relocation to start
-            beginRelocationLatch.await();
-            disruption.startDisrupting();
-            // wait for relocation to finish
-            endRelocationLatch.await();
+            BlockClusterStateProcessing disruption = relocateAndBlockCompletion(logger, "test", 0, node_1, node_3);
             // wait a little so that cluster state observer is registered
             sleep(50);
             disruption.stopDisrupting();
@@ -171,6 +157,56 @@ public void testIndexCleanup() throws Exception {
 
     }
 
+    /**
+     * relocate a shard and block cluster state processing on the relocation target node to activate the shard
+     */
+    public static BlockClusterStateProcessing relocateAndBlockCompletion(Logger logger, String index, int shard, String nodeFrom,
+                                                                         String nodeTo) throws InterruptedException {
+        BlockClusterStateProcessing disruption = new BlockClusterStateProcessing(nodeTo, random());
+        internalCluster().setDisruptionScheme(disruption);
+        MockTransportService transportService = (MockTransportService) internalCluster().getInstance(TransportService.class, nodeTo);
+        ClusterService clusterService = internalCluster().getInstance(ClusterService.class, nodeTo);
+        CountDownLatch beginRelocationLatch = new CountDownLatch(1);
+        CountDownLatch receivedShardExistsRequestLatch = new CountDownLatch(1);
+        // use a tracer on the target node to track relocation start and end
+        transportService.addTracer(new MockTransportService.Tracer() {
+            @Override
+            public void receivedRequest(long requestId, String action) {
+                if (action.equals(PeerRecoveryTargetService.Actions.FILES_INFO)) {
+                    logger.info("received: {}, relocation starts", action);
+                    beginRelocationLatch.countDown();
+                } else if (action.equals(IndicesStore.ACTION_SHARD_EXISTS)) {
+                    // Whenever a node deletes a shard because it was relocated somewhere else, it first
+                    // checks if enough other copies are started somewhere else. The node sends a ShardActiveRequest
+                    // to the other nodes that should have a copy according to cluster state.
+                    receivedShardExistsRequestLatch.countDown();
+                    logger.info("received: {}, relocation done", action);
+                } else if (action.equals(PeerRecoveryTargetService.Actions.WAIT_CLUSTERSTATE)) {
+                    logger.info("received: {}, waiting on cluster state", action);
+                    // ensure that relocation target node is on the same cluster state as relocation source before proceeding with
+                    // this request. If the target does not have the relocating cluster state exposed through ClusterService.state(),
+                    // then waitForClusterState will have to register a ClusterObserver with the ClusterService, which can cause
+                    // a race with the BlockClusterStateProcessing block that is added below.
+                    try {
+                        assertBusy(() -> assertTrue(
+                            clusterService.state().routingTable().index(index).shard(shard).primaryShard().relocating()));
+                    } catch (Exception e) {
+                        throw new RuntimeException(e);
+                    }
+                }
+            }
+        });
+        internalCluster().client().admin().cluster().prepareReroute().add(new MoveAllocationCommand(index, shard, nodeFrom, nodeTo)).get();
+        logger.info("--> waiting for relocation to start");
+        beginRelocationLatch.await();
+        logger.info("--> starting disruption");
+        disruption.startDisrupting();
+        logger.info("--> waiting for relocation to finish");
+        receivedShardExistsRequestLatch.await();
+        logger.info("--> relocation completed (but cluster state processing block still in place)");
+        return disruption;
+    }
+
     /* Test that shard is deleted in case ShardActiveRequest after relocation and next incoming cluster state is an index delete. */
     public void testShardCleanupIfShardDeletionAfterRelocationFailedAndIndexDeleted() throws Exception {
         final String node_1 = internalCluster().startNode();
@@ -201,13 +237,13 @@ public void testShardCleanupIfShardDeletionAfterRelocationFailedAndIndexDeleted(
         final CountDownLatch shardActiveRequestSent = new CountDownLatch(1);
         transportServiceNode_1.addDelegate(transportServiceNode_2, new MockTransportService.DelegateTransport(transportServiceNode_1.original()) {
             @Override
-            public void sendRequest(DiscoveryNode node, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException, TransportException {
+            protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException {
                 if (action.equals("internal:index/shard/exists") && shardActiveRequestSent.getCount() > 0) {
                     shardActiveRequestSent.countDown();
                     logger.info("prevent shard active request from being sent");
-                    throw new ConnectTransportException(node, "DISCONNECT: simulated");
+                    throw new ConnectTransportException(connection.getNode(), "DISCONNECT: simulated");
                 }
-                super.sendRequest(node, requestId, action, request, options);
+                super.sendRequest(connection, requestId, action, request, options);
             }
         });
 
@@ -295,17 +331,14 @@ public void testShardsCleanup() throws Exception {
     }
 
     public void testShardActiveElsewhereDoesNotDeleteAnother() throws Exception {
-        InternalTestCluster.Async<String> masterFuture = internalCluster().startNodeAsync(
-                Settings.builder().put(Node.NODE_MASTER_SETTING.getKey(), true, Node.NODE_DATA_SETTING.getKey(), false).build());
-        InternalTestCluster.Async<List<String>> nodesFutures = internalCluster().startNodesAsync(4,
-                Settings.builder().put(Node.NODE_MASTER_SETTING.getKey(), false, Node.NODE_DATA_SETTING.getKey(), true).build());
-
-        final String masterNode = masterFuture.get();
-        final String node1 = nodesFutures.get().get(0);
-        final String node2 = nodesFutures.get().get(1);
-        final String node3 = nodesFutures.get().get(2);
+        final String masterNode = internalCluster().startMasterOnlyNode();
+        final List<String> nodes = internalCluster().startDataOnlyNodes(4);
+
+        final String node1 = nodes.get(0);
+        final String node2 = nodes.get(1);
+        final String node3 = nodes.get(2);
         // we will use this later on, handy to start now to make sure it has a different data folder that node 1,2 &3
-        final String node4 = nodesFutures.get().get(3);
+        final String node4 = nodes.get(3);
 
         assertAcked(prepareCreate("test").setSettings(Settings.builder()
                         .put(indexSettings())
@@ -343,12 +376,7 @@ public void testShardActiveElsewhereDoesNotDeleteAnother() throws Exception {
         // allocation filtering may not have immediate effect
         // TODO: we should add an easier to do this. It's too much of a song and dance..
         Index index = resolveIndex("test");
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                assertTrue(internalCluster().getInstance(IndicesService.class, node4).hasIndex(index));
-            }
-        });
+        assertBusy(() -> assertTrue(internalCluster().getInstance(IndicesService.class, node4).hasIndex(index)));
 
         // wait for 4 active shards - we should have lost one shard
         assertFalse(client().admin().cluster().prepareHealth().setWaitForActiveShards(4).get().isTimedOut());
@@ -359,8 +387,7 @@ public void run() {
 
         logger.debug("--> starting the two old nodes back");
 
-        internalCluster().startNodesAsync(2,
-                Settings.builder().put(Node.NODE_MASTER_SETTING.getKey(), false, Node.NODE_DATA_SETTING.getKey(), true).build());
+        internalCluster().startDataOnlyNodes(2);
 
         assertFalse(client().admin().cluster().prepareHealth().setWaitForNodes("5").get().isTimedOut());
 
@@ -375,7 +402,7 @@ public void run() {
     }
 
     public void testShardActiveElseWhere() throws Exception {
-        List<String> nodes = internalCluster().startNodesAsync(2).get();
+        List<String> nodes = internalCluster().startNodes(2);
 
         final String masterNode = internalCluster().getMasterName();
         final String nonMasterNode = nodes.get(0).equals(masterNode) ? nodes.get(1) : nodes.get(0);
@@ -404,9 +431,9 @@ public void testShardActiveElseWhere() throws Exception {
         // disable relocations when we do this, to make sure the shards are not relocated from node2
         // due to rebalancing, and delete its content
         client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), EnableAllocationDecider.Rebalance.NONE)).get();
-        internalCluster().getInstance(ClusterService.class, nonMasterNode).submitStateUpdateTask("test", new ClusterStateUpdateTask(Priority.IMMEDIATE) {
+        internalCluster().getInstance(ClusterService.class, nonMasterNode).submitStateUpdateTask("test", new LocalClusterUpdateTask(Priority.IMMEDIATE) {
             @Override
-            public ClusterState execute(ClusterState currentState) throws Exception {
+            public ClusterTasksResult<LocalClusterUpdateTask> execute(ClusterState currentState) throws Exception {
                 IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(index);
                 for (int i = 0; i < numShards; i++) {
                     indexRoutingTableBuilder.addIndexShard(
@@ -415,14 +442,9 @@ public ClusterState execute(ClusterState currentState) throws Exception {
                                     .build()
                     );
                 }
-                return ClusterState.builder(currentState)
+                return newState(ClusterState.builder(currentState)
                         .routingTable(RoutingTable.builder().add(indexRoutingTableBuilder).build())
-                        .build();
-            }
-
-            @Override
-            public boolean runOnlyOnMaster() {
-                return false;
+                        .build());
             }
 
             @Override
@@ -459,45 +481,4 @@ private boolean waitForIndexDeletion(final String server, final Index index) thr
         awaitBusy(() -> !Files.exists(indexDirectory(server, index)));
         return Files.exists(indexDirectory(server, index));
     }
-
-    /**
-     * This Tracer can be used to signal start and end of a recovery.
-     * This is used to test the following:
-     * Whenever a node deletes a shard because it was relocated somewhere else, it first
-     * checks if enough other copies are started somewhere else. The node sends a ShardActiveRequest
-     * to the other nodes that should have a copy according to cluster state.
-     * The nodes that receive this request check if the shard is in state STARTED in which case they
-     * respond with "true". If they have the shard in POST_RECOVERY they register a cluster state
-     * observer that checks at each update if the shard has moved to STARTED.
-     * To test that this mechanism actually works, this can be triggered by blocking the cluster
-     * state processing when a recover starts and only unblocking it shortly after the node receives
-     * the ShardActiveRequest.
-     */
-    public static class ReclocationStartEndTracer extends MockTransportService.Tracer {
-        private final Logger logger;
-        private final CountDownLatch beginRelocationLatch;
-        private final CountDownLatch receivedShardExistsRequestLatch;
-
-        public ReclocationStartEndTracer(Logger logger, CountDownLatch beginRelocationLatch, CountDownLatch receivedShardExistsRequestLatch) {
-            this.logger = logger;
-            this.beginRelocationLatch = beginRelocationLatch;
-            this.receivedShardExistsRequestLatch = receivedShardExistsRequestLatch;
-        }
-
-        @Override
-        public void receivedRequest(long requestId, String action) {
-            if (action.equals(IndicesStore.ACTION_SHARD_EXISTS)) {
-                receivedShardExistsRequestLatch.countDown();
-                logger.info("received: {}, relocation done", action);
-            }
-        }
-
-        @Override
-        public void requestSent(DiscoveryNode node, long requestId, String action, TransportRequestOptions options) {
-            if (action.equals(PeerRecoverySourceService.Actions.START_RECOVERY)) {
-                logger.info("sent: {}, relocation starts", action);
-                beginRelocationLatch.countDown();
-            }
-        }
-    }
 }
diff --git a/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java b/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java
index 7dea17f383022..7f61659bcd5c0 100644
--- a/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java
@@ -20,39 +20,22 @@
 package org.elasticsearch.indices.store;
 
 import org.elasticsearch.Version;
-import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.threadpool.TestThreadPool;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.TransportService;
-import org.junit.After;
-import org.junit.AfterClass;
 import org.junit.Before;
-import org.junit.BeforeClass;
 
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.Set;
-import java.util.concurrent.TimeUnit;
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
-import static org.elasticsearch.Version.CURRENT;
-import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
-import static org.elasticsearch.test.VersionUtils.randomVersion;
 
 /**
  */
@@ -66,54 +49,22 @@ public class IndicesStoreTests extends ESTestCase {
         NOT_STARTED_STATES = set.toArray(new ShardRoutingState[set.size()]);
     }
 
-    private static ThreadPool threadPool;
-
-    private IndicesStore indicesStore;
     private DiscoveryNode localNode;
 
-    private ClusterService clusterService;
-
-    @BeforeClass
-    public static void beforeClass() {
-        threadPool = new TestThreadPool("ShardReplicationTests");
-    }
-
-    @AfterClass
-    public static void afterClass() {
-        ThreadPool.terminate(threadPool, 30, TimeUnit.SECONDS);
-        threadPool = null;
-    }
-
     @Before
-    public void before() {
+    public void createLocalNode() {
         localNode = new DiscoveryNode("abc", new LocalTransportAddress("abc"), emptyMap(), emptySet(), Version.CURRENT);
-        clusterService = createClusterService(threadPool);
-        indicesStore = new IndicesStore(Settings.EMPTY, null, clusterService, new TransportService(clusterService.getSettings(), null, null), null);
-    }
-
-    @After
-    public void tearDown() throws Exception {
-        super.tearDown();
-        clusterService.close();
     }
 
     public void testShardCanBeDeletedNoShardRouting() throws Exception {
-        int numShards = randomIntBetween(1, 7);
-        int numReplicas = randomInt(2);
-
-        ClusterState.Builder clusterState = ClusterState.builder(new ClusterName("test"));
-        clusterState.metaData(MetaData.builder().put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(numShards).numberOfReplicas(numReplicas)));
         IndexShardRoutingTable.Builder routingTable = new IndexShardRoutingTable.Builder(new ShardId("test", "_na_", 1));
-
-        assertFalse(indicesStore.shardCanBeDeleted(clusterState.build(), routingTable.build()));
+        assertFalse(IndicesStore.shardCanBeDeleted(localNode.getId(), routingTable.build()));
     }
 
     public void testShardCanBeDeletedNoShardStarted() throws Exception {
         int numShards = randomIntBetween(1, 7);
         int numReplicas = randomInt(2);
 
-        ClusterState.Builder clusterState = ClusterState.builder(new ClusterName("test"));
-        clusterState.metaData(MetaData.builder().put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(numShards).numberOfReplicas(numReplicas)));
         IndexShardRoutingTable.Builder routingTable = new IndexShardRoutingTable.Builder(new ShardId("test", "_na_", 1));
 
         for (int i = 0; i < numShards; i++) {
@@ -129,99 +80,31 @@ public void testShardCanBeDeletedNoShardStarted() throws Exception {
                 if (state == ShardRoutingState.UNASSIGNED) {
                     unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null);
                 }
-                routingTable.addShard(TestShardRouting.newShardRouting("test", i, "xyz", null, j == 0, state, unassignedInfo));
+                String relocatingNodeId = state == ShardRoutingState.RELOCATING ? randomAlphaOfLength(10) : null;
+                routingTable.addShard(TestShardRouting.newShardRouting("test", i, randomAlphaOfLength(10), relocatingNodeId, j == 0, state, unassignedInfo));
             }
         }
-        assertFalse(indicesStore.shardCanBeDeleted(clusterState.build(), routingTable.build()));
+
+        assertFalse(IndicesStore.shardCanBeDeleted(localNode.getId(), routingTable.build()));
     }
 
     public void testShardCanBeDeletedShardExistsLocally() throws Exception {
         int numShards = randomIntBetween(1, 7);
         int numReplicas = randomInt(2);
 
-        ClusterState.Builder clusterState = ClusterState.builder(new ClusterName("test"));
-        clusterState.metaData(MetaData.builder().put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(numShards).numberOfReplicas(numReplicas)));
-        clusterState.nodes(DiscoveryNodes.builder().localNodeId(localNode.getId()).add(localNode).add(new DiscoveryNode("xyz",
-                new LocalTransportAddress("xyz"), emptyMap(), emptySet(), Version.CURRENT)));
         IndexShardRoutingTable.Builder routingTable = new IndexShardRoutingTable.Builder(new ShardId("test", "_na_", 1));
         int localShardId = randomInt(numShards - 1);
         for (int i = 0; i < numShards; i++) {
-            String nodeId = i == localShardId ? localNode.getId() : randomBoolean() ? "abc" : "xyz";
-            String relocationNodeId = randomBoolean() ? null : randomBoolean() ? localNode.getId() : "xyz";
-            routingTable.addShard(TestShardRouting.newShardRouting("test", i, nodeId, relocationNodeId, true, ShardRoutingState.STARTED));
+            int localNodeIndex = randomInt(numReplicas);
+            boolean primaryOnLocalNode = i == localShardId && localNodeIndex == numReplicas;
+            routingTable.addShard(TestShardRouting.newShardRouting("test", i, primaryOnLocalNode ? localNode.getId() : randomAlphaOfLength(10), true, ShardRoutingState.STARTED));
             for (int j = 0; j < numReplicas; j++) {
-                routingTable.addShard(TestShardRouting.newShardRouting("test", i, nodeId, relocationNodeId, false, ShardRoutingState.STARTED));
+                boolean replicaOnLocalNode = i == localShardId && localNodeIndex == j;
+                routingTable.addShard(TestShardRouting.newShardRouting("test", i, replicaOnLocalNode ? localNode.getId() : randomAlphaOfLength(10), false, ShardRoutingState.STARTED));
             }
         }
 
         // Shard exists locally, can't delete shard
-        assertFalse(indicesStore.shardCanBeDeleted(clusterState.build(), routingTable.build()));
-    }
-
-    public void testShardCanBeDeletedNodeNotInList() throws Exception {
-        int numShards = randomIntBetween(1, 7);
-        int numReplicas = randomInt(2);
-
-        ClusterState.Builder clusterState = ClusterState.builder(new ClusterName("test"));
-        clusterState.metaData(MetaData.builder().put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(numShards).numberOfReplicas(numReplicas)));
-        clusterState.nodes(DiscoveryNodes.builder().localNodeId(localNode.getId()).add(localNode));
-        IndexShardRoutingTable.Builder routingTable = new IndexShardRoutingTable.Builder(new ShardId("test", "_na_", 1));
-        for (int i = 0; i < numShards; i++) {
-            String relocatingNodeId = randomBoolean() ? null : "def";
-            routingTable.addShard(TestShardRouting.newShardRouting("test", i, "xyz", relocatingNodeId, true, ShardRoutingState.STARTED));
-            for (int j = 0; j < numReplicas; j++) {
-                routingTable.addShard(TestShardRouting.newShardRouting("test", i, "xyz", relocatingNodeId, false, ShardRoutingState.STARTED));
-            }
-        }
-
-        // null node -> false
-        assertFalse(indicesStore.shardCanBeDeleted(clusterState.build(), routingTable.build()));
-    }
-
-    public void testShardCanBeDeletedNodeVersion() throws Exception {
-        int numShards = randomIntBetween(1, 7);
-        int numReplicas = randomInt(2);
-
-        // Most of the times don't test bwc and use current version
-        final Version nodeVersion = randomBoolean() ? CURRENT : randomVersion(random());
-        ClusterState.Builder clusterState = ClusterState.builder(new ClusterName("test"));
-        clusterState.metaData(MetaData.builder().put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(numShards).numberOfReplicas(numReplicas)));
-        clusterState.nodes(DiscoveryNodes.builder().localNodeId(localNode.getId()).add(localNode).add(new DiscoveryNode("xyz",
-                new LocalTransportAddress("xyz"), emptyMap(), emptySet(), nodeVersion)));
-        IndexShardRoutingTable.Builder routingTable = new IndexShardRoutingTable.Builder(new ShardId("test", "_na_", 1));
-        for (int i = 0; i < numShards; i++) {
-            routingTable.addShard(TestShardRouting.newShardRouting("test", i, "xyz", null, true, ShardRoutingState.STARTED));
-            for (int j = 0; j < numReplicas; j++) {
-                routingTable.addShard(TestShardRouting.newShardRouting("test", i, "xyz", null, false, ShardRoutingState.STARTED));
-            }
-        }
-
-        // shard exist on other node (abc)
-        assertTrue(indicesStore.shardCanBeDeleted(clusterState.build(), routingTable.build()));
-    }
-
-    public void testShardCanBeDeletedRelocatingNode() throws Exception {
-        int numShards = randomIntBetween(1, 7);
-        int numReplicas = randomInt(2);
-
-        ClusterState.Builder clusterState = ClusterState.builder(new ClusterName("test"));
-        clusterState.metaData(MetaData.builder().put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(numShards).numberOfReplicas(numReplicas)));
-        final Version nodeVersion = randomBoolean() ? CURRENT : randomVersion(random());
-
-        clusterState.nodes(DiscoveryNodes.builder().localNodeId(localNode.getId())
-                .add(localNode)
-                .add(new DiscoveryNode("xyz", new LocalTransportAddress("xyz"), emptyMap(), emptySet(), Version.CURRENT))
-                .add(new DiscoveryNode("def", new LocalTransportAddress("def"), emptyMap(), emptySet(), nodeVersion) // <-- only set relocating, since we're testing that in this test
-                ));
-        IndexShardRoutingTable.Builder routingTable = new IndexShardRoutingTable.Builder(new ShardId("test", "_na_", 1));
-        for (int i = 0; i < numShards; i++) {
-            routingTable.addShard(TestShardRouting.newShardRouting("test", i, "xyz", "def", true, ShardRoutingState.STARTED));
-            for (int j = 0; j < numReplicas; j++) {
-                routingTable.addShard(TestShardRouting.newShardRouting("test", i, "xyz", "def", false, ShardRoutingState.STARTED));
-            }
-        }
-
-        // shard exist on other node (abc and def)
-        assertTrue(indicesStore.shardCanBeDeleted(clusterState.build(), routingTable.build()));
+        assertFalse(IndicesStore.shardCanBeDeleted(localNode.getId(), routingTable.build()));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java b/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java
index 676f26e7d74b9..c71bc950ab25d 100644
--- a/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java
@@ -27,20 +27,20 @@
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.AliasMetaData;
 import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.mapper.MapperParsingException;
-import org.elasticsearch.index.query.MatchQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.index.query.TermQueryBuilder;
-import org.elasticsearch.indices.IndexTemplateAlreadyExistsException;
 import org.elasticsearch.indices.InvalidAliasNameException;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.junit.After;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashSet;
@@ -68,6 +68,11 @@
  */
 public class SimpleIndexTemplateIT extends ESIntegTestCase {
 
+    @After
+    public void cleanupTemplates() {
+        client().admin().indices().prepareDeleteTemplate("*").get();
+    }
+
     public void testSimpleIndexTemplateTests() throws Exception {
         // clean all templates setup by the framework.
         client().admin().indices().prepareDeleteTemplate("*").get();
@@ -105,7 +110,7 @@ public void testSimpleIndexTemplateTests() throws Exception {
                 .addMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties")
                         .startObject("field2").field("type", "text").field("store", false).endObject()
                         .endObject().endObject().endObject())
-                , IndexTemplateAlreadyExistsException.class
+                , IllegalArgumentException.class
         );
 
         response = client().admin().indices().prepareGetTemplates().get();
@@ -113,7 +118,9 @@ public void testSimpleIndexTemplateTests() throws Exception {
 
 
         // index something into test_index, will match on both templates
-        client().prepareIndex("test_index", "type1", "1").setSource("field1", "value1", "field2", "value 2").setRefreshPolicy(IMMEDIATE).get();
+        client().prepareIndex("test_index", "type1", "1")
+                .setSource("field1", "value1", "field2", "value 2")
+                .setRefreshPolicy(IMMEDIATE).get();
 
         ensureGreen();
         SearchResponse searchResponse = client().prepareSearch("test_index")
@@ -126,7 +133,9 @@ public void testSimpleIndexTemplateTests() throws Exception {
         // field2 is not stored.
         assertThat(searchResponse.getHits().getAt(0).field("field2"), nullValue());
 
-        client().prepareIndex("text_index", "type1", "1").setSource("field1", "value1", "field2", "value 2").setRefreshPolicy(IMMEDIATE).get();
+        client().prepareIndex("text_index", "type1", "1")
+                .setSource("field1", "value1", "field2", "value 2")
+                .setRefreshPolicy(IMMEDIATE).get();
 
         ensureGreen();
         // now only match on one template (template_1)
@@ -148,25 +157,47 @@ public void testDeleteIndexTemplate() throws Exception {
         client().admin().indices().preparePutTemplate("template_1")
                 .setTemplate("te*")
                 .setOrder(0)
-                .addMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties")
-                        .startObject("field1").field("type", "text").field("store", true).endObject()
-                        .startObject("field2").field("type", "text").field("store", true).endObject()
-                        .endObject().endObject().endObject())
+                .addMapping("type1", XContentFactory.jsonBuilder()
+                    .startObject()
+                        .startObject("type1")
+                            .startObject("properties")
+                                .startObject("field1")
+                                    .field("type", "text")
+                                    .field("store", true)
+                                .endObject()
+                                .startObject("field2")
+                                    .field("type", "text")
+                                    .field("store", true)
+                                .endObject()
+                            .endObject()
+                        .endObject()
+                    .endObject())
                 .execute().actionGet();
 
         client().admin().indices().preparePutTemplate("template_2")
                 .setTemplate("test*")
                 .setOrder(1)
-                .addMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties")
-                        .startObject("field2").field("type", "text").field("store", "no").endObject()
-                        .endObject().endObject().endObject())
+                .addMapping("type1", XContentFactory.jsonBuilder()
+                    .startObject()
+                        .startObject("type1")
+                            .startObject("properties")
+                                .startObject("field2")
+                                    .field("type", "text")
+                                    .field("store", false)
+                                .endObject()
+                            .endObject()
+                        .endObject()
+                    .endObject())
                 .execute().actionGet();
 
         logger.info("--> explicitly delete template_1");
         admin().indices().prepareDeleteTemplate("template_1").execute().actionGet();
-        assertThat(admin().cluster().prepareState().execute().actionGet().getState().metaData().templates().size(), equalTo(1 + existingTemplates));
-        assertThat(admin().cluster().prepareState().execute().actionGet().getState().metaData().templates().containsKey("template_2"), equalTo(true));
-        assertThat(admin().cluster().prepareState().execute().actionGet().getState().metaData().templates().containsKey("template_1"), equalTo(false));
+
+        ClusterState state = admin().cluster().prepareState().execute().actionGet().getState();
+
+        assertThat(state.metaData().templates().size(), equalTo(1 + existingTemplates));
+        assertThat(state.metaData().templates().containsKey("template_2"), equalTo(true));
+        assertThat(state.metaData().templates().containsKey("template_1"), equalTo(false));
 
 
         logger.info("--> put template_1 back");
@@ -181,11 +212,13 @@ public void testDeleteIndexTemplate() throws Exception {
 
         logger.info("--> delete template*");
         admin().indices().prepareDeleteTemplate("template*").execute().actionGet();
-        assertThat(admin().cluster().prepareState().execute().actionGet().getState().metaData().templates().size(), equalTo(existingTemplates));
+        assertThat(admin().cluster().prepareState().execute().actionGet().getState().metaData().templates().size(),
+                   equalTo(existingTemplates));
 
         logger.info("--> delete * with no templates, make sure we don't get a failure");
         admin().indices().prepareDeleteTemplate("*").execute().actionGet();
-        assertThat(admin().cluster().prepareState().execute().actionGet().getState().metaData().templates().size(), equalTo(0));
+        assertThat(admin().cluster().prepareState().execute().actionGet().getState().metaData().templates().size(),
+                   equalTo(0));
     }
 
     public void testThatGetIndexTemplatesWorks() throws Exception {
@@ -193,6 +226,7 @@ public void testThatGetIndexTemplatesWorks() throws Exception {
         client().admin().indices().preparePutTemplate("template_1")
                 .setTemplate("te*")
                 .setOrder(0)
+                .setVersion(123)
                 .addMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties")
                         .startObject("field1").field("type", "text").field("store", true).endObject()
                         .startObject("field2").field("type", "keyword").field("store", true).endObject()
@@ -205,9 +239,11 @@ public void testThatGetIndexTemplatesWorks() throws Exception {
         assertThat(getTemplate1Response.getIndexTemplates().get(0), is(notNullValue()));
         assertThat(getTemplate1Response.getIndexTemplates().get(0).getTemplate(), is("te*"));
         assertThat(getTemplate1Response.getIndexTemplates().get(0).getOrder(), is(0));
+        assertThat(getTemplate1Response.getIndexTemplates().get(0).getVersion(), is(123));
 
         logger.info("--> get non-existing-template");
-        GetIndexTemplatesResponse getTemplate2Response = client().admin().indices().prepareGetTemplates("non-existing-template").execute().actionGet();
+        GetIndexTemplatesResponse getTemplate2Response =
+            client().admin().indices().prepareGetTemplates("non-existing-template").execute().actionGet();
         assertThat(getTemplate2Response.getIndexTemplates(), hasSize(0));
     }
 
@@ -291,7 +327,6 @@ private void testExpectActionRequestValidationException(String... names) {
                 "get template with " + Arrays.toString(names));
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/pull/8802")
     public void testBrokenMapping() throws Exception {
         // clean all templates setup by the framework.
         client().admin().indices().prepareDeleteTemplate("*").get();
@@ -303,7 +338,7 @@ public void testBrokenMapping() throws Exception {
         MapperParsingException e = expectThrows( MapperParsingException.class,
             () -> client().admin().indices().preparePutTemplate("template_1")
                 .setTemplate("te*")
-                .addMapping("type1", "abcde")
+                .addMapping("type1", "{\"foo\": \"abcde\"}", XContentType.JSON)
                 .get());
         assertThat(e.getMessage(), containsString("Failed to parse mapping "));
 
@@ -340,7 +375,7 @@ public void testIndexTemplateWithAliases() throws Exception {
 
         client().admin().indices().preparePutTemplate("template_with_aliases")
                 .setTemplate("te*")
-                .addMapping("type1", "{\"type1\" : {\"properties\" : {\"value\" : {\"type\" : \"text\"}}}}")
+                .addMapping("type1", "{\"type1\" : {\"properties\" : {\"value\" : {\"type\" : \"text\"}}}}", XContentType.JSON)
                 .addAlias(new Alias("simple_alias"))
                 .addAlias(new Alias("templated_alias-{index}"))
                 .addAlias(new Alias("filtered_alias").filter("{\"type\":{\"value\":\"type2\"}}"))
@@ -348,7 +383,8 @@ public void testIndexTemplateWithAliases() throws Exception {
                         .filter(QueryBuilders.termsQuery("_type",  "typeX", "typeY", "typeZ")))
                 .get();
 
-        assertAcked(prepareCreate("test_index").addMapping("type1").addMapping("type2").addMapping("typeX").addMapping("typeY").addMapping("typeZ"));
+        assertAcked(prepareCreate("test_index")
+                        .addMapping("type1").addMapping("type2").addMapping("typeX").addMapping("typeY").addMapping("typeZ"));
         ensureGreen();
 
         client().prepareIndex("test_index", "type1", "1").setSource("field", "A value").get();
@@ -390,7 +426,7 @@ public void testIndexTemplateWithAliases() throws Exception {
 
     public void testIndexTemplateWithAliasesInSource() {
         client().admin().indices().preparePutTemplate("template_1")
-                .setSource("{\n" +
+                .setSource(new BytesArray("{\n" +
                         "    \"template\" : \"*\",\n" +
                         "    \"aliases\" : {\n" +
                         "        \"my_alias\" : {\n" +
@@ -401,7 +437,7 @@ public void testIndexTemplateWithAliasesInSource() {
                         "            }\n" +
                         "        }\n" +
                         "    }\n" +
-                        "}").get();
+                        "}"), XContentType.JSON).get();
 
 
         assertAcked(prepareCreate("test_index").addMapping("type1").addMapping("type2"));
@@ -582,7 +618,8 @@ public void testStrictAliasParsingInIndicesCreatedViaTemplates() throws Exceptio
                 .setOrder(0)
                 .addMapping("test", "field", "type=text")
                 .addAlias(new Alias("alias1").filter(termQuery("field", "value"))).get();
-        // Indexing into b should succeed, because the field mapping for field 'field' is defined in the _default_ mapping and the test type exists.
+        // Indexing into b should succeed, because the field mapping for field 'field' is defined in the _default_ mapping and
+        //  the test type exists.
         client().admin().indices().preparePutTemplate("template2")
                 .setTemplate("b*")
                 .setOrder(0)
@@ -601,8 +638,8 @@ public void testStrictAliasParsingInIndicesCreatedViaTemplates() throws Exceptio
                 .setOrder(0)
                 .addAlias(new Alias("alias4").filter(termQuery("field", "value"))).get();
 
-        client().prepareIndex("a1", "test", "test").setSource("{}").get();
-        BulkResponse response = client().prepareBulk().add(new IndexRequest("a2", "test", "test").source("{}")).get();
+        client().prepareIndex("a1", "test", "test").setSource("{}", XContentType.JSON).get();
+        BulkResponse response = client().prepareBulk().add(new IndexRequest("a2", "test", "test").source("{}", XContentType.JSON)).get();
         assertThat(response.hasFailures(), is(false));
         assertThat(response.getItems()[0].isFailed(), equalTo(false));
         assertThat(response.getItems()[0].getIndex(), equalTo("a2"));
@@ -610,8 +647,8 @@ public void testStrictAliasParsingInIndicesCreatedViaTemplates() throws Exceptio
         assertThat(response.getItems()[0].getId(), equalTo("test"));
         assertThat(response.getItems()[0].getVersion(), equalTo(1L));
 
-        client().prepareIndex("b1", "test", "test").setSource("{}").get();
-        response = client().prepareBulk().add(new IndexRequest("b2", "test", "test").source("{}")).get();
+        client().prepareIndex("b1", "test", "test").setSource("{}", XContentType.JSON).get();
+        response = client().prepareBulk().add(new IndexRequest("b2", "test", "test").source("{}", XContentType.JSON)).get();
         assertThat(response.hasFailures(), is(false));
         assertThat(response.getItems()[0].isFailed(), equalTo(false));
         assertThat(response.getItems()[0].getIndex(), equalTo("b2"));
@@ -619,8 +656,8 @@ public void testStrictAliasParsingInIndicesCreatedViaTemplates() throws Exceptio
         assertThat(response.getItems()[0].getId(), equalTo("test"));
         assertThat(response.getItems()[0].getVersion(), equalTo(1L));
 
-        client().prepareIndex("c1", "test", "test").setSource("{}").get();
-        response = client().prepareBulk().add(new IndexRequest("c2", "test", "test").source("{}")).get();
+        client().prepareIndex("c1", "test", "test").setSource("{}", XContentType.JSON).get();
+        response = client().prepareBulk().add(new IndexRequest("c2", "test", "test").source("{}", XContentType.JSON)).get();
         assertThat(response.hasFailures(), is(false));
         assertThat(response.getItems()[0].isFailed(), equalTo(false));
         assertThat(response.getItems()[0].getIndex(), equalTo("c2"));
@@ -636,9 +673,9 @@ public void testStrictAliasParsingInIndicesCreatedViaTemplates() throws Exceptio
         // So the aliases defined in the index template for this index will not fail
         // even though the fields in the alias fields don't exist yet and indexing into
         // an index that doesn't exist yet will succeed
-        client().prepareIndex("d1", "test", "test").setSource("{}").get();
+        client().prepareIndex("d1", "test", "test").setSource("{}", XContentType.JSON).get();
 
-        response = client().prepareBulk().add(new IndexRequest("d2", "test", "test").source("{}")).get();
+        response = client().prepareBulk().add(new IndexRequest("d2", "test", "test").source("{}", XContentType.JSON)).get();
         assertThat(response.hasFailures(), is(false));
         assertThat(response.getItems()[0].isFailed(), equalTo(false));
         assertThat(response.getItems()[0].getId(), equalTo("test"));
@@ -668,7 +705,7 @@ public void testCombineTemplates() throws Exception{
                     "                }\n" +
                     "            }\n" +
                     "         }\n" +
-                    "    }\n")
+                    "    }\n", XContentType.JSON)
             .get();
 
         // put template using custom_1 analyzer
@@ -688,5 +725,79 @@ public void testCombineTemplates() throws Exception{
 
     }
 
+    public void testOrderAndVersion() {
+        int order = randomInt();
+        Integer version = randomBoolean() ? randomInt() : null;
+
+        assertAcked(client().admin().indices().preparePutTemplate("versioned_template")
+                                              .setTemplate("te*")
+                                              .setVersion(version)
+                                              .setOrder(order)
+                                              .addMapping("test", "field", "type=text")
+                                              .get());
+
+        GetIndexTemplatesResponse response = client().admin().indices().prepareGetTemplates("versioned_template").get();
+        assertThat(response.getIndexTemplates().size(), equalTo(1));
+        assertThat(response.getIndexTemplates().get(0).getVersion(), equalTo(version));
+        assertThat(response.getIndexTemplates().get(0).getOrder(), equalTo(order));
+    }
 
+    public void testPartitionedTemplate() throws Exception {
+        // clean all templates setup by the framework.
+        client().admin().indices().prepareDeleteTemplate("*").get();
+
+        // check get all templates on an empty index.
+        GetIndexTemplatesResponse response = client().admin().indices().prepareGetTemplates().get();
+        assertThat(response.getIndexTemplates(), empty());
+
+        // provide more partitions than shards
+        IllegalArgumentException eBadSettings = expectThrows(IllegalArgumentException.class,
+            () -> client().admin().indices().preparePutTemplate("template_1")
+                .setTemplate("te*")
+                .setSettings(Settings.builder()
+                    .put("index.number_of_shards", "5")
+                    .put("index.routing_partition_size", "6"))
+                .get());
+        assertThat(eBadSettings.getMessage(), containsString("partition size [6] should be a positive number "
+                + "less than the number of shards [5]"));
+
+        // provide an invalid mapping for a partitioned index
+        IllegalArgumentException eBadMapping = expectThrows(IllegalArgumentException.class,
+            () -> client().admin().indices().preparePutTemplate("template_2")
+                .setTemplate("te*")
+                .addMapping("type", "{\"type\":{\"_routing\":{\"required\":false}}}")
+                .setSettings(Settings.builder()
+                    .put("index.number_of_shards", "6")
+                    .put("index.routing_partition_size", "3"))
+                .get());
+        assertThat(eBadMapping.getMessage(), containsString("must have routing required for partitioned index"));
+
+        // no templates yet
+        response = client().admin().indices().prepareGetTemplates().get();
+        assertEquals(0, response.getIndexTemplates().size());
+
+        // a valid configuration that only provides the partition size
+        assertAcked(client().admin().indices().preparePutTemplate("just_partitions")
+            .setTemplate("te*")
+            .setSettings(Settings.builder()
+                .put("index.routing_partition_size", "6"))
+            .get());
+
+        // create an index with too few shards
+        IllegalArgumentException eBadIndex = expectThrows(IllegalArgumentException.class,
+                () -> prepareCreate("test_bad").setSettings(Settings.builder()
+            .put("index.number_of_shards", 5))
+            .get());
+
+        assertThat(eBadIndex.getMessage(), containsString("partition size [6] should be a positive number "
+                + "less than the number of shards [5]"));
+
+        // finally, create a valid index
+        prepareCreate("test_good").setSettings(Settings.builder()
+            .put("index.number_of_shards", 7))
+            .get();
+
+        GetSettingsResponse getSettingsResponse = client().admin().indices().prepareGetSettings("test_good").get();
+        assertEquals("6", getSettingsResponse.getIndexToSettings().get("test_good").getAsMap().get("index.routing_partition_size"));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java b/core/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java
index f2d9aaa517060..bd3323f0c558a 100644
--- a/core/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java
@@ -19,6 +19,10 @@
 
 package org.elasticsearch.ingest;
 
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.Before;
+
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -26,11 +30,10 @@
 import java.util.List;
 import java.util.Map;
 
-import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.test.ESTestCase;
-import org.junit.Before;
-
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.Matchers.sameInstance;
 import static org.mockito.Mockito.mock;
 
@@ -105,13 +108,36 @@ public void testReadProcessors() throws Exception {
         assertThat(result.get(0), sameInstance(processor));
         assertThat(result.get(1), sameInstance(processor));
 
-        config.add(Collections.singletonMap("unknown_processor", emptyConfig));
-        try {
-            ConfigurationUtils.readProcessorConfigs(config, registry);
-            fail("exception expected");
-        } catch (ElasticsearchParseException e) {
-            assertThat(e.getMessage(), equalTo("No processor type exists with name [unknown_processor]"));
-        }
+        Map<String, Object> unknownTaggedConfig = new HashMap<>();
+        unknownTaggedConfig.put("tag", "my_unknown");
+        config.add(Collections.singletonMap("unknown_processor", unknownTaggedConfig));
+        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class,
+            () -> ConfigurationUtils.readProcessorConfigs(config, registry));
+        assertThat(e.getMessage(), equalTo("No processor type exists with name [unknown_processor]"));
+        assertThat(e.getHeader("processor_tag"), equalTo(Collections.singletonList("my_unknown")));
+        assertThat(e.getHeader("processor_type"), equalTo(Collections.singletonList("unknown_processor")));
+        assertThat(e.getHeader("property_name"), is(nullValue()));
+
+        List<Map<String, Map<String, Object>>> config2 = new ArrayList<>();
+        unknownTaggedConfig = new HashMap<>();
+        unknownTaggedConfig.put("tag", "my_unknown");
+        config2.add(Collections.singletonMap("unknown_processor", unknownTaggedConfig));
+        Map<String, Object> secondUnknonwTaggedConfig = new HashMap<>();
+        secondUnknonwTaggedConfig.put("tag", "my_second_unknown");
+        config2.add(Collections.singletonMap("second_unknown_processor", secondUnknonwTaggedConfig));
+        e = expectThrows(ElasticsearchParseException.class, () -> ConfigurationUtils.readProcessorConfigs(config2, registry));
+        assertThat(e.getMessage(), equalTo("No processor type exists with name [unknown_processor]"));
+        assertThat(e.getHeader("processor_tag"), equalTo(Collections.singletonList("my_unknown")));
+        assertThat(e.getHeader("processor_type"), equalTo(Collections.singletonList("unknown_processor")));
+        assertThat(e.getHeader("property_name"), is(nullValue()));
+
+        assertThat(e.getSuppressed().length, equalTo(1));
+        assertThat(e.getSuppressed()[0], instanceOf(ElasticsearchParseException.class));
+        ElasticsearchParseException e2 = (ElasticsearchParseException) e.getSuppressed()[0];
+        assertThat(e2.getMessage(), equalTo("No processor type exists with name [second_unknown_processor]"));
+        assertThat(e2.getHeader("processor_tag"), equalTo(Collections.singletonList("my_second_unknown")));
+        assertThat(e2.getHeader("processor_type"), equalTo(Collections.singletonList("second_unknown_processor")));
+        assertThat(e2.getHeader("property_name"), is(nullValue()));
     }
 
 }
diff --git a/core/src/test/java/org/elasticsearch/ingest/IngestClientIT.java b/core/src/test/java/org/elasticsearch/ingest/IngestClientIT.java
index 2b78cd83fb632..26a9130bd30cd 100644
--- a/core/src/test/java/org/elasticsearch/ingest/IngestClientIT.java
+++ b/core/src/test/java/org/elasticsearch/ingest/IngestClientIT.java
@@ -36,14 +36,15 @@
 import org.elasticsearch.action.ingest.SimulatePipelineRequest;
 import org.elasticsearch.action.ingest.SimulatePipelineResponse;
 import org.elasticsearch.action.ingest.WritePipelineResponse;
+import org.elasticsearch.client.Requests;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.ExecutionException;
@@ -82,7 +83,7 @@ public void testSimulate() throws Exception {
             .endObject()
             .endArray()
             .endObject().bytes();
-        client().admin().cluster().preparePutPipeline("_id", pipelineSource)
+        client().admin().cluster().preparePutPipeline("_id", pipelineSource, XContentType.JSON)
                 .get();
         GetPipelineResponse getResponse = client().admin().cluster().prepareGetPipeline("_id")
                 .get();
@@ -105,10 +106,10 @@ public void testSimulate() throws Exception {
             .endObject().bytes();
         SimulatePipelineResponse response;
         if (randomBoolean()) {
-            response = client().admin().cluster().prepareSimulatePipeline(bytes)
+            response = client().admin().cluster().prepareSimulatePipeline(bytes, XContentType.JSON)
                 .setId("_id").get();
         } else {
-            SimulatePipelineRequest request = new SimulatePipelineRequest(bytes);
+            SimulatePipelineRequest request = new SimulatePipelineRequest(bytes, XContentType.JSON);
             request.setId("_id");
             response = client().admin().cluster().simulatePipeline(request).get();
         }
@@ -138,14 +139,14 @@ public void testBulkWithIngestFailures() throws Exception {
             .endObject()
             .endArray()
             .endObject().bytes();
-        PutPipelineRequest putPipelineRequest = new PutPipelineRequest("_id", source);
+        PutPipelineRequest putPipelineRequest = new PutPipelineRequest("_id", source, XContentType.JSON);
         client().admin().cluster().putPipeline(putPipelineRequest).get();
 
         int numRequests = scaledRandomIntBetween(32, 128);
         BulkRequest bulkRequest = new BulkRequest();
         for (int i = 0; i < numRequests; i++) {
             IndexRequest indexRequest = new IndexRequest("index", "type", Integer.toString(i)).setPipeline("_id");
-            indexRequest.source("field", "value", "fail", i % 2 == 0);
+            indexRequest.source(Requests.INDEX_CONTENT_TYPE, "field", "value", "fail", i % 2 == 0);
             bulkRequest.add(indexRequest);
         }
 
@@ -178,7 +179,7 @@ public void test() throws Exception {
             .endObject()
             .endArray()
             .endObject().bytes();
-        PutPipelineRequest putPipelineRequest = new PutPipelineRequest("_id", source);
+        PutPipelineRequest putPipelineRequest = new PutPipelineRequest("_id", source, XContentType.JSON);
         client().admin().cluster().putPipeline(putPipelineRequest).get();
 
         GetPipelineRequest getPipelineRequest = new GetPipelineRequest("_id");
@@ -220,7 +221,7 @@ public void testPutWithPipelineFactoryError() throws Exception {
                 .endObject()
                 .endArray()
                 .endObject().bytes();
-        PutPipelineRequest putPipelineRequest = new PutPipelineRequest("_id", source);
+        PutPipelineRequest putPipelineRequest = new PutPipelineRequest("_id", source, XContentType.JSON);
         try {
             client().admin().cluster().putPipeline(putPipelineRequest).get();
         } catch (ExecutionException e) {
@@ -229,9 +230,4 @@ public void testPutWithPipelineFactoryError() throws Exception {
             assertThat(ex.getMessage(), equalTo("processor [test] doesn't support one or more provided configuration parameters [unused]"));
         }
     }
-
-    @Override
-    protected Collection<Class<? extends Plugin>> getMockPlugins() {
-        return Collections.singletonList(TestSeedPlugin.class);
-    }
 }
diff --git a/core/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java b/core/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java
index 1719b58178d0d..25d62deac81b2 100644
--- a/core/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java
@@ -22,20 +22,19 @@
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 
+import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
 import static org.hamcrest.Matchers.both;
 import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.endsWith;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.instanceOf;
@@ -47,13 +46,19 @@
 
 public class IngestDocumentTests extends ESTestCase {
 
+    private static final Date BOGUS_TIMESTAMP = new Date(0L);
+    private static final ZonedDateTime BOGUS_TIMESTAMP_NEW_DATE_FORMAT = ZonedDateTime.of(2016, 10, 23, 0, 0, 0, 0, ZoneOffset.UTC);
     private IngestDocument ingestDocument;
+    private IngestDocument ingestDocumentWithNewDateFormat;
 
-    @Before
-    public void setIngestDocument() {
+    public IngestDocument getTestIngestDocument(boolean newDateFormat) {
         Map<String, Object> document = new HashMap<>();
         Map<String, Object> ingestMap = new HashMap<>();
-        ingestMap.put("timestamp", "bogus_timestamp");
+        if (newDateFormat) {
+            ingestMap.put("timestamp", BOGUS_TIMESTAMP_NEW_DATE_FORMAT);
+        } else {
+            ingestMap.put("timestamp", BOGUS_TIMESTAMP);
+        }
         document.put("_ingest", ingestMap);
         document.put("foo", "bar");
         document.put("int", 123);
@@ -74,7 +79,20 @@ public void setIngestDocument() {
         list.add(null);
 
         document.put("list", list);
-        ingestDocument = new IngestDocument("index", "type", "id", null, null, null, null, document);
+        return new IngestDocument("index", "type", "id", null, null, null,
+            null, document, newDateFormat);
+    }
+
+    @Before
+    public void setIngestDocuments() {
+        ingestDocument = getTestIngestDocument(false);
+        ingestDocumentWithNewDateFormat = getTestIngestDocument(true);
+    }
+
+    public void testDefaultConstructorUsesDateClass() {
+        IngestDocument ingestDocument = new IngestDocument("foo", "bar", "baz", "fuzz", "buzz",
+            "timestamp", "ttl", Collections.emptyMap());
+        assertThat(ingestDocument.getFieldValue("_ingest.timestamp", Object.class).getClass(), equalTo(Date.class));
     }
 
     public void testSimpleGetFieldValue() {
@@ -85,9 +103,16 @@ public void testSimpleGetFieldValue() {
         assertThat(ingestDocument.getFieldValue("_index", String.class), equalTo("index"));
         assertThat(ingestDocument.getFieldValue("_type", String.class), equalTo("type"));
         assertThat(ingestDocument.getFieldValue("_id", String.class), equalTo("id"));
-        assertThat(ingestDocument.getFieldValue("_ingest.timestamp", String.class),
-                both(notNullValue()).and(not(equalTo("bogus_timestamp"))));
-        assertThat(ingestDocument.getFieldValue("_source._ingest.timestamp", String.class), equalTo("bogus_timestamp"));
+        assertThat(ingestDocument.getFieldValue("_ingest.timestamp", Date.class),
+                both(notNullValue()).and(not(equalTo(BOGUS_TIMESTAMP))));
+        assertThat(ingestDocument.getFieldValue("_source._ingest.timestamp", Date.class), equalTo(BOGUS_TIMESTAMP));
+    }
+
+    public void testNewDateFormat() {
+        assertThat(ingestDocumentWithNewDateFormat.getFieldValue("_ingest.timestamp", ZonedDateTime.class),
+            both(notNullValue()).and(not(equalTo(BOGUS_TIMESTAMP_NEW_DATE_FORMAT))));
+        assertThat(ingestDocumentWithNewDateFormat.getFieldValue("_source._ingest.timestamp", ZonedDateTime.class),
+            equalTo(BOGUS_TIMESTAMP_NEW_DATE_FORMAT));
     }
 
     public void testGetSourceObject() {
@@ -914,12 +939,12 @@ public void testEqualsAndHashcode() throws Exception {
         Map<String, Object> sourceAndMetadata = RandomDocumentPicks.randomSource(random());
         int numFields = randomIntBetween(1, IngestDocument.MetaData.values().length);
         for (int i = 0; i < numFields; i++) {
-            sourceAndMetadata.put(randomFrom(IngestDocument.MetaData.values()).getFieldName(), randomAsciiOfLengthBetween(5, 10));
+            sourceAndMetadata.put(randomFrom(IngestDocument.MetaData.values()).getFieldName(), randomAlphaOfLengthBetween(5, 10));
         }
         Map<String, Object> ingestMetadata = new HashMap<>();
         numFields = randomIntBetween(1, 5);
         for (int i = 0; i < numFields; i++) {
-            ingestMetadata.put(randomAsciiOfLengthBetween(5, 10), randomAsciiOfLengthBetween(5, 10));
+            ingestMetadata.put(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLengthBetween(5, 10));
         }
         IngestDocument ingestDocument = new IngestDocument(sourceAndMetadata, ingestMetadata);
 
@@ -934,7 +959,7 @@ public void testEqualsAndHashcode() throws Exception {
         if (randomBoolean()) {
             numFields = randomIntBetween(1, IngestDocument.MetaData.values().length);
             for (int i = 0; i < numFields; i++) {
-                otherSourceAndMetadata.put(randomFrom(IngestDocument.MetaData.values()).getFieldName(), randomAsciiOfLengthBetween(5, 10));
+                otherSourceAndMetadata.put(randomFrom(IngestDocument.MetaData.values()).getFieldName(), randomAlphaOfLengthBetween(5, 10));
             }
             changed = true;
         }
@@ -944,7 +969,7 @@ public void testEqualsAndHashcode() throws Exception {
             otherIngestMetadata = new HashMap<>();
             numFields = randomIntBetween(1, 5);
             for (int i = 0; i < numFields; i++) {
-                otherIngestMetadata.put(randomAsciiOfLengthBetween(5, 10), randomAsciiOfLengthBetween(5, 10));
+                otherIngestMetadata.put(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLengthBetween(5, 10));
             }
             changed = true;
         } else {
@@ -971,11 +996,8 @@ public void testIngestMetadataTimestamp() throws Exception {
         long before = System.currentTimeMillis();
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
         long after = System.currentTimeMillis();
-        String timestampString = (String) ingestDocument.getIngestMetadata().get("timestamp");
-        assertThat(timestampString, notNullValue());
-        assertThat(timestampString, endsWith("+0000"));
-        DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZZ", Locale.ROOT);
-        Date timestamp = df.parse(timestampString);
+        Date timestamp = (Date) ingestDocument.getIngestMetadata().get(IngestDocument.TIMESTAMP);
+        assertThat(timestamp, notNullValue());
         assertThat(timestamp.getTime(), greaterThanOrEqualTo(before));
         assertThat(timestamp.getTime(), lessThanOrEqualTo(after));
     }
@@ -1004,34 +1026,4 @@ public void testSetInvalidSourceField() throws Exception {
         }
     }
 
-    public static void assertIngestDocument(Object a, Object b) {
-        if (a instanceof Map) {
-            Map<?, ?> mapA = (Map<?, ?>) a;
-            Map<?, ?> mapB = (Map<?, ?>) b;
-            for (Map.Entry<?, ?> entry : mapA.entrySet()) {
-                if (entry.getValue() instanceof List || entry.getValue() instanceof Map) {
-                    assertIngestDocument(entry.getValue(), mapB.get(entry.getKey()));
-                }
-            }
-        } else if (a instanceof List) {
-            List<?> listA = (List<?>) a;
-            List<?> listB = (List<?>) b;
-            for (int i = 0; i < listA.size(); i++) {
-                Object value = listA.get(i);
-                if (value instanceof List || value instanceof Map) {
-                    assertIngestDocument(value, listB.get(i));
-                }
-            }
-        } else if (a instanceof byte[]) {
-            assertArrayEquals((byte[]) a, (byte[])b);
-        } else if (a instanceof IngestDocument) {
-            IngestDocument docA = (IngestDocument) a;
-            IngestDocument docB = (IngestDocument) b;
-            assertIngestDocument(docA.getSourceAndMetadata(), docB.getSourceAndMetadata());
-            assertIngestDocument(docA.getIngestMetadata(), docB.getIngestMetadata());
-        } else {
-            String msg = String.format(Locale.ROOT, "Expected %s class to be equal to %s", a.getClass().getName(), b.getClass().getName());
-            assertThat(msg, a, equalTo(b));
-        }
-    }
 }
diff --git a/core/src/test/java/org/elasticsearch/ingest/IngestMetadataTests.java b/core/src/test/java/org/elasticsearch/ingest/IngestMetadataTests.java
index 7924d069f2e6c..518b775d7f802 100644
--- a/core/src/test/java/org/elasticsearch/ingest/IngestMetadataTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/IngestMetadataTests.java
@@ -42,10 +42,10 @@ public class IngestMetadataTests extends ESTestCase {
 
     public void testFromXContent() throws IOException {
         PipelineConfiguration pipeline = new PipelineConfiguration(
-            "1",new BytesArray("{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}")
+            "1", new BytesArray("{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}"), XContentType.JSON
         );
         PipelineConfiguration pipeline2 = new PipelineConfiguration(
-            "2",new BytesArray("{\"processors\": [{\"set\" : {\"field\": \"_field1\", \"value\": \"_value1\"}}]}")
+            "2", new BytesArray("{\"processors\": [{\"set\" : {\"field\": \"_field1\", \"value\": \"_value1\"}}]}"), XContentType.JSON
         );
         Map<String, PipelineConfiguration> map = new HashMap<>();
         map.put(pipeline.getId(), pipeline);
@@ -57,8 +57,8 @@ public void testFromXContent() throws IOException {
         ingestMetadata.toXContent(builder, ToXContent.EMPTY_PARAMS);
         builder.endObject();
         XContentBuilder shuffled = shuffleXContent(builder);
-        final XContentParser parser = XContentFactory.xContent(shuffled.bytes()).createParser(shuffled.bytes());
-        MetaData.Custom custom = ingestMetadata.fromXContent(parser);
+        final XContentParser parser = createParser(shuffled);
+        MetaData.Custom custom = IngestMetadata.fromXContent(parser);
         assertTrue(custom instanceof IngestMetadata);
         IngestMetadata m = (IngestMetadata) custom;
         assertEquals(2, m.getPipelines().size());
@@ -72,14 +72,14 @@ public void testDiff() throws Exception {
         BytesReference pipelineConfig = new BytesArray("{}");
 
         Map<String, PipelineConfiguration> pipelines = new HashMap<>();
-        pipelines.put("1", new PipelineConfiguration("1", pipelineConfig));
-        pipelines.put("2", new PipelineConfiguration("2", pipelineConfig));
+        pipelines.put("1", new PipelineConfiguration("1", pipelineConfig, XContentType.JSON));
+        pipelines.put("2", new PipelineConfiguration("2", pipelineConfig, XContentType.JSON));
         IngestMetadata ingestMetadata1 = new IngestMetadata(pipelines);
 
         pipelines = new HashMap<>();
-        pipelines.put("1", new PipelineConfiguration("1", pipelineConfig));
-        pipelines.put("3", new PipelineConfiguration("3", pipelineConfig));
-        pipelines.put("4", new PipelineConfiguration("4", pipelineConfig));
+        pipelines.put("1", new PipelineConfiguration("1", pipelineConfig, XContentType.JSON));
+        pipelines.put("3", new PipelineConfiguration("3", pipelineConfig, XContentType.JSON));
+        pipelines.put("4", new PipelineConfiguration("4", pipelineConfig, XContentType.JSON));
         IngestMetadata ingestMetadata2 = new IngestMetadata(pipelines);
 
         IngestMetadata.IngestMetadataDiff diff = (IngestMetadata.IngestMetadataDiff) ingestMetadata2.diff(ingestMetadata1);
@@ -92,13 +92,13 @@ public void testDiff() throws Exception {
         IngestMetadata endResult = (IngestMetadata) diff.apply(ingestMetadata2);
         assertThat(endResult, not(equalTo(ingestMetadata1)));
         assertThat(endResult.getPipelines().size(), equalTo(3));
-        assertThat(endResult.getPipelines().get("1"), equalTo(new PipelineConfiguration("1", pipelineConfig)));
-        assertThat(endResult.getPipelines().get("3"), equalTo(new PipelineConfiguration("3", pipelineConfig)));
-        assertThat(endResult.getPipelines().get("4"), equalTo(new PipelineConfiguration("4", pipelineConfig)));
+        assertThat(endResult.getPipelines().get("1"), equalTo(new PipelineConfiguration("1", pipelineConfig, XContentType.JSON)));
+        assertThat(endResult.getPipelines().get("3"), equalTo(new PipelineConfiguration("3", pipelineConfig, XContentType.JSON)));
+        assertThat(endResult.getPipelines().get("4"), equalTo(new PipelineConfiguration("4", pipelineConfig, XContentType.JSON)));
 
         pipelines = new HashMap<>();
-        pipelines.put("1", new PipelineConfiguration("1", new BytesArray("{}")));
-        pipelines.put("2", new PipelineConfiguration("2", new BytesArray("{}")));
+        pipelines.put("1", new PipelineConfiguration("1", new BytesArray("{}"), XContentType.JSON));
+        pipelines.put("2", new PipelineConfiguration("2", new BytesArray("{}"), XContentType.JSON));
         IngestMetadata ingestMetadata3 = new IngestMetadata(pipelines);
 
         diff = (IngestMetadata.IngestMetadataDiff) ingestMetadata3.diff(ingestMetadata1);
@@ -108,12 +108,12 @@ public void testDiff() throws Exception {
         endResult = (IngestMetadata) diff.apply(ingestMetadata3);
         assertThat(endResult, equalTo(ingestMetadata1));
         assertThat(endResult.getPipelines().size(), equalTo(2));
-        assertThat(endResult.getPipelines().get("1"), equalTo(new PipelineConfiguration("1", pipelineConfig)));
-        assertThat(endResult.getPipelines().get("2"), equalTo(new PipelineConfiguration("2", pipelineConfig)));
+        assertThat(endResult.getPipelines().get("1"), equalTo(new PipelineConfiguration("1", pipelineConfig, XContentType.JSON)));
+        assertThat(endResult.getPipelines().get("2"), equalTo(new PipelineConfiguration("2", pipelineConfig, XContentType.JSON)));
 
         pipelines = new HashMap<>();
-        pipelines.put("1", new PipelineConfiguration("1", new BytesArray("{}")));
-        pipelines.put("2", new PipelineConfiguration("2", new BytesArray("{\"key\" : \"value\"}")));
+        pipelines.put("1", new PipelineConfiguration("1", new BytesArray("{}"), XContentType.JSON));
+        pipelines.put("2", new PipelineConfiguration("2", new BytesArray("{\"key\" : \"value\"}"), XContentType.JSON));
         IngestMetadata ingestMetadata4 = new IngestMetadata(pipelines);
 
         diff = (IngestMetadata.IngestMetadataDiff) ingestMetadata4.diff(ingestMetadata1);
@@ -123,7 +123,8 @@ public void testDiff() throws Exception {
         endResult = (IngestMetadata) diff.apply(ingestMetadata4);
         assertThat(endResult, not(equalTo(ingestMetadata1)));
         assertThat(endResult.getPipelines().size(), equalTo(2));
-        assertThat(endResult.getPipelines().get("1"), equalTo(new PipelineConfiguration("1", pipelineConfig)));
-        assertThat(endResult.getPipelines().get("2"), equalTo(new PipelineConfiguration("2", new BytesArray("{\"key\" : \"value\"}"))));
+        assertThat(endResult.getPipelines().get("1"), equalTo(new PipelineConfiguration("1", pipelineConfig, XContentType.JSON)));
+        assertThat(endResult.getPipelines().get("2"),
+            equalTo(new PipelineConfiguration("2", new BytesArray("{\"key\" : \"value\"}"), XContentType.JSON)));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java b/core/src/test/java/org/elasticsearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java
index c91e1e5f6ec0a..645933348879c 100644
--- a/core/src/test/java/org/elasticsearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java
+++ b/core/src/test/java/org/elasticsearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java
@@ -22,7 +22,8 @@
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.action.ingest.WritePipelineResponse;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.node.service.NodeService;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.node.NodeService;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 
@@ -60,11 +61,6 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
         return installPlugin ? Arrays.asList(IngestTestPlugin.class) : Collections.emptyList();
     }
 
-    @Override
-    protected Collection<Class<? extends Plugin>> getMockPlugins() {
-        return Collections.singletonList(TestSeedPlugin.class);
-    }
-
     public void testFailPipelineCreation() throws Exception {
         installPlugin = true;
         String node1 = internalCluster().startNode();
@@ -74,9 +70,9 @@ public void testFailPipelineCreation() throws Exception {
         ensureStableCluster(2, node2);
 
         try {
-            client().admin().cluster().preparePutPipeline("_id", pipelineSource).get();
+            client().admin().cluster().preparePutPipeline("_id", pipelineSource, XContentType.JSON).get();
             fail("exception expected");
-        } catch (IllegalArgumentException e) {
+        } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), containsString("Processor type [test] is not installed on node"));
         }
     }
@@ -87,7 +83,7 @@ public void testFailPipelineCreationProcessorNotInstalledOnMasterNode() throws E
         internalCluster().startNode();
 
         try {
-            client().admin().cluster().preparePutPipeline("_id", pipelineSource).get();
+            client().admin().cluster().preparePutPipeline("_id", pipelineSource, XContentType.JSON).get();
             fail("exception expected");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("No processor type exists with name [test]"));
@@ -100,7 +96,7 @@ public void testFailStartNode() throws Exception {
         installPlugin = true;
         String node1 = internalCluster().startNode();
 
-        WritePipelineResponse response = client().admin().cluster().preparePutPipeline("_id", pipelineSource).get();
+        WritePipelineResponse response = client().admin().cluster().preparePutPipeline("_id", pipelineSource, XContentType.JSON).get();
         assertThat(response.isAcknowledged(), is(true));
         Pipeline pipeline = internalCluster().getInstance(NodeService.class, node1).getIngestService().getPipelineStore().get("_id");
         assertThat(pipeline, notNullValue());
diff --git a/core/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java b/core/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java
index 3a842a4690afa..d36b3390e4164 100644
--- a/core/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java
@@ -23,6 +23,7 @@
 import java.util.Collections;
 import java.util.Map;
 
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.IngestPlugin;
 import org.elasticsearch.test.ESTestCase;
@@ -39,7 +40,8 @@ public Map<String, Processor.Factory> getProcessors(Processor.Parameters paramet
 
     public void testIngestPlugin() {
         ThreadPool tp = Mockito.mock(ThreadPool.class);
-        IngestService ingestService = new IngestService(Settings.EMPTY, tp, null, null, null, Collections.singletonList(DUMMY_PLUGIN));
+        IngestService ingestService = new IngestService(new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
+            Settings.EMPTY, tp, null, null, null, Collections.singletonList(DUMMY_PLUGIN));
         Map<String, Processor.Factory> factories = ingestService.getPipelineStore().getProcessorFactories();
         assertTrue(factories.containsKey("foo"));
         assertEquals(1, factories.size());
@@ -48,7 +50,8 @@ public void testIngestPlugin() {
     public void testIngestPluginDuplicate() {
         ThreadPool tp = Mockito.mock(ThreadPool.class);
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () ->
-            new IngestService(Settings.EMPTY, tp, null, null, null, Arrays.asList(DUMMY_PLUGIN, DUMMY_PLUGIN))
+            new IngestService(new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
+                Settings.EMPTY, tp, null, null, null, Arrays.asList(DUMMY_PLUGIN, DUMMY_PLUGIN))
         );
         assertTrue(e.getMessage(), e.getMessage().contains("already registered"));
     }
diff --git a/core/src/test/java/org/elasticsearch/ingest/PipelineConfigurationTests.java b/core/src/test/java/org/elasticsearch/ingest/PipelineConfigurationTests.java
new file mode 100644
index 0000000000000..f4bea8696dbff
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/ingest/PipelineConfigurationTests.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.xcontent.ContextParser;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+
+public class PipelineConfigurationTests extends ESTestCase {
+
+    public void testSerialization() throws IOException {
+        PipelineConfiguration configuration = new PipelineConfiguration("1",
+            new BytesArray("{}".getBytes(StandardCharsets.UTF_8)), XContentType.JSON);
+        assertEquals(XContentType.JSON, configuration.getXContentType());
+
+        BytesStreamOutput out = new BytesStreamOutput();
+        configuration.writeTo(out);
+        StreamInput in = StreamInput.wrap(out.bytes().toBytesRef().bytes);
+        PipelineConfiguration serialized = PipelineConfiguration.readFrom(in);
+        assertEquals(XContentType.JSON, serialized.getXContentType());
+        assertEquals("{}", serialized.getConfig().utf8ToString());
+    }
+
+    public void testSerializationBwc() throws IOException {
+        final byte[] data = Base64.getDecoder().decode("ATECe30AAAA=");
+        final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2,
+            Version.V_5_1_1, Version.V_5_1_2, Version.V_5_2_0);
+        try (StreamInput in = StreamInput.wrap(data)) {
+            in.setVersion(version);
+            PipelineConfiguration configuration = PipelineConfiguration.readFrom(in);
+            assertEquals(XContentType.JSON, configuration.getXContentType());
+            assertEquals("{}", configuration.getConfig().utf8ToString());
+
+            try (BytesStreamOutput out = new BytesStreamOutput()) {
+                out.setVersion(version);
+                configuration.writeTo(out);
+                assertArrayEquals(data, out.bytes().toBytesRef().bytes);
+            }
+        }
+    }
+
+    public void testParser() throws IOException {
+        ContextParser<Void, PipelineConfiguration> parser = PipelineConfiguration.getParser();
+        XContentType xContentType = randomFrom(XContentType.values());
+        final BytesReference bytes;
+        try (XContentBuilder builder = XContentBuilder.builder(xContentType.xContent())) {
+            new PipelineConfiguration("1", new BytesArray("{}".getBytes(StandardCharsets.UTF_8)), XContentType.JSON)
+                .toXContent(builder, ToXContent.EMPTY_PARAMS);
+            bytes = builder.bytes();
+        }
+
+        XContentParser xContentParser = xContentType.xContent().createParser(NamedXContentRegistry.EMPTY, bytes);
+        PipelineConfiguration parsed = parser.parse(xContentParser, null);
+        assertEquals(xContentType, parsed.getXContentType());
+        assertEquals("{}", XContentHelper.convertToJson(parsed.getConfig(), false, parsed.getXContentType()));
+        assertEquals("1", parsed.getId());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java b/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java
index acf5c26e565d9..7570d4e8ab0ce 100644
--- a/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java
@@ -21,13 +21,16 @@
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.update.UpdateRequest;
+import org.elasticsearch.client.Requests;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.hamcrest.CustomTypeSafeMatcher;
@@ -39,6 +42,7 @@
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
+import java.util.concurrent.ExecutorService;
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 
@@ -68,7 +72,8 @@ public class PipelineExecutionServiceTests extends ESTestCase {
     public void setup() {
         store = mock(PipelineStore.class);
         ThreadPool threadPool = mock(ThreadPool.class);
-        when(threadPool.executor(anyString())).thenReturn(Runnable::run);
+        final ExecutorService executorService = EsExecutors.newDirectExecutorService();
+        when(threadPool.executor(anyString())).thenReturn(executorService);
         executionService = new PipelineExecutionService(store, threadPool);
     }
 
@@ -317,7 +322,7 @@ public void testBulkRequestExecutionWithFailures() throws Exception {
         int numRequest = scaledRandomIntBetween(8, 64);
         int numIndexRequests = 0;
         for (int i = 0; i < numRequest; i++) {
-            ActionRequest request;
+            DocWriteRequest request;
             if (randomBoolean()) {
                 if (randomBoolean()) {
                     request = new DeleteRequest("_index", "_type", "_id");
@@ -326,7 +331,7 @@ public void testBulkRequestExecutionWithFailures() throws Exception {
                 }
             } else {
                 IndexRequest indexRequest = new IndexRequest("_index", "_type", "_id").setPipeline(pipelineId);
-                indexRequest.source("field1", "value1");
+                indexRequest.source(Requests.INDEX_CONTENT_TYPE, "field1", "value1");
                 request = indexRequest;
                 numIndexRequests++;
             }
@@ -356,7 +361,7 @@ public void testBulkRequestExecution() throws Exception {
         int numRequest = scaledRandomIntBetween(8, 64);
         for (int i = 0; i < numRequest; i++) {
             IndexRequest indexRequest = new IndexRequest("_index", "_type", "_id").setPipeline(pipelineId);
-            indexRequest.source("field1", "value1");
+            indexRequest.source(Requests.INDEX_CONTENT_TYPE, "field1", "value1");
             bulkRequest.add(indexRequest);
         }
 
@@ -384,8 +389,8 @@ public void testStats() throws Exception {
         when(store.get("_id2")).thenReturn(new Pipeline("_id2", null, null, new CompoundProcessor(mock(Processor.class))));
 
         Map<String, PipelineConfiguration> configurationMap = new HashMap<>();
-        configurationMap.put("_id1", new PipelineConfiguration("_id1", new BytesArray("{}")));
-        configurationMap.put("_id2", new PipelineConfiguration("_id2", new BytesArray("{}")));
+        configurationMap.put("_id1", new PipelineConfiguration("_id1", new BytesArray("{}"), XContentType.JSON));
+        configurationMap.put("_id2", new PipelineConfiguration("_id2", new BytesArray("{}"), XContentType.JSON));
         executionService.updatePipelineStats(new IngestMetadata(configurationMap));
 
         @SuppressWarnings("unchecked")
@@ -414,14 +419,14 @@ public void testStats() throws Exception {
     // issue: https://github.com/elastic/elasticsearch/issues/18126
     public void testUpdatingStatsWhenRemovingPipelineWorks() throws Exception {
         Map<String, PipelineConfiguration> configurationMap = new HashMap<>();
-        configurationMap.put("_id1", new PipelineConfiguration("_id1", new BytesArray("{}")));
-        configurationMap.put("_id2", new PipelineConfiguration("_id2", new BytesArray("{}")));
+        configurationMap.put("_id1", new PipelineConfiguration("_id1", new BytesArray("{}"), XContentType.JSON));
+        configurationMap.put("_id2", new PipelineConfiguration("_id2", new BytesArray("{}"), XContentType.JSON));
         executionService.updatePipelineStats(new IngestMetadata(configurationMap));
         assertThat(executionService.stats().getStatsPerPipeline(), hasKey("_id1"));
         assertThat(executionService.stats().getStatsPerPipeline(), hasKey("_id2"));
 
         configurationMap = new HashMap<>();
-        configurationMap.put("_id3", new PipelineConfiguration("_id3", new BytesArray("{}")));
+        configurationMap.put("_id3", new PipelineConfiguration("_id3", new BytesArray("{}"), XContentType.JSON));
         executionService.updatePipelineStats(new IngestMetadata(configurationMap));
         assertThat(executionService.stats().getStatsPerPipeline(), not(hasKey("_id1")));
         assertThat(executionService.stats().getStatsPerPipeline(), not(hasKey("_id2")));
@@ -435,7 +440,7 @@ private class IngestDocumentMatcher extends ArgumentMatcher<IngestDocument> {
 
         private final IngestDocument ingestDocument;
 
-        public IngestDocumentMatcher(String index, String type, String id, Map<String, Object> source) {
+        IngestDocumentMatcher(String index, String type, String id, Map<String, Object> source) {
             this.ingestDocument = new IngestDocument(index, type, id, null, null, null, null, source);
         }
 
diff --git a/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java b/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java
index cdbe1e1157084..61a89ffcecc10 100644
--- a/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java
@@ -29,8 +29,10 @@
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.LocalTransportAddress;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
@@ -49,6 +51,7 @@
 
 public class PipelineStoreTests extends ESTestCase {
 
+    private ClusterSettings clusterSettings;
     private PipelineStore store;
 
     @Before
@@ -70,7 +73,7 @@ public String getType() {
 
                 @Override
                 public String getTag() {
-                    return null;
+                    return tag;
                 }
             };
         });
@@ -89,11 +92,12 @@ public String getType() {
 
                 @Override
                 public String getTag() {
-                    return null;
+                    return tag;
                 }
             };
         });
-        store = new PipelineStore(Settings.EMPTY, processorFactories);
+        clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        store = new PipelineStore(clusterSettings, Settings.EMPTY, processorFactories);
     }
 
     public void testUpdatePipelines() {
@@ -103,7 +107,7 @@ public void testUpdatePipelines() {
         assertThat(store.pipelines.size(), is(0));
 
         PipelineConfiguration pipeline = new PipelineConfiguration(
-            "_id",new BytesArray("{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}")
+            "_id",new BytesArray("{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}"), XContentType.JSON
         );
         IngestMetadata ingestMetadata = new IngestMetadata(Collections.singletonMap("_id", pipeline));
         clusterState = ClusterState.builder(clusterState)
@@ -124,7 +128,7 @@ public void testPut() {
         ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build();
 
         // add a new pipeline:
-        PutPipelineRequest putRequest = new PutPipelineRequest(id, new BytesArray("{\"processors\": []}"));
+        PutPipelineRequest putRequest = new PutPipelineRequest(id, new BytesArray("{\"processors\": []}"), XContentType.JSON);
         ClusterState previousClusterState = clusterState;
         clusterState = store.innerPut(putRequest, clusterState);
         store.innerUpdatePipelines(previousClusterState, clusterState);
@@ -135,7 +139,8 @@ public void testPut() {
         assertThat(pipeline.getProcessors().size(), equalTo(0));
 
         // overwrite existing pipeline:
-        putRequest = new PutPipelineRequest(id, new BytesArray("{\"processors\": [], \"description\": \"_description\"}"));
+        putRequest =
+            new PutPipelineRequest(id, new BytesArray("{\"processors\": [], \"description\": \"_description\"}"), XContentType.JSON);
         previousClusterState = clusterState;
         clusterState = store.innerPut(putRequest, clusterState);
         store.innerUpdatePipelines(previousClusterState, clusterState);
@@ -152,7 +157,8 @@ public void testPutWithErrorResponse() {
         assertThat(pipeline, nullValue());
         ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build();
 
-        PutPipelineRequest putRequest = new PutPipelineRequest(id, new BytesArray("{\"description\": \"empty processors\"}"));
+        PutPipelineRequest putRequest =
+            new PutPipelineRequest(id, new BytesArray("{\"description\": \"empty processors\"}"), XContentType.JSON);
         ClusterState previousClusterState = clusterState;
         clusterState = store.innerPut(putRequest, clusterState);
         try {
@@ -167,7 +173,7 @@ public void testPutWithErrorResponse() {
 
     public void testDelete() {
         PipelineConfiguration config = new PipelineConfiguration(
-            "_id",new BytesArray("{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}")
+            "_id",new BytesArray("{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}"), XContentType.JSON
         );
         IngestMetadata ingestMetadata = new IngestMetadata(Collections.singletonMap("_id", config));
         ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build();
@@ -193,13 +199,84 @@ public void testDelete() {
         }
     }
 
+    public void testDeleteUsingWildcard() {
+        HashMap<String, PipelineConfiguration> pipelines = new HashMap<>();
+        BytesArray definition = new BytesArray(
+            "{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}"
+        );
+        pipelines.put("p1", new PipelineConfiguration("p1", definition, XContentType.JSON));
+        pipelines.put("p2", new PipelineConfiguration("p2", definition, XContentType.JSON));
+        pipelines.put("q1", new PipelineConfiguration("q1", definition, XContentType.JSON));
+        IngestMetadata ingestMetadata = new IngestMetadata(pipelines);
+        ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build();
+        ClusterState previousClusterState = clusterState;
+        clusterState = ClusterState.builder(clusterState).metaData(MetaData.builder()
+            .putCustom(IngestMetadata.TYPE, ingestMetadata)).build();
+        store.innerUpdatePipelines(previousClusterState, clusterState);
+        assertThat(store.get("p1"), notNullValue());
+        assertThat(store.get("p2"), notNullValue());
+        assertThat(store.get("q1"), notNullValue());
+
+        // Delete pipeline matching wildcard
+        DeletePipelineRequest deleteRequest = new DeletePipelineRequest("p*");
+        previousClusterState = clusterState;
+        clusterState = store.innerDelete(deleteRequest, clusterState);
+        store.innerUpdatePipelines(previousClusterState, clusterState);
+        assertThat(store.get("p1"), nullValue());
+        assertThat(store.get("p2"), nullValue());
+        assertThat(store.get("q1"), notNullValue());
+
+        // Exception if we used name which does not exist
+        try {
+            store.innerDelete(new DeletePipelineRequest("unknown"), clusterState);
+            fail("exception expected");
+        } catch (ResourceNotFoundException e) {
+            assertThat(e.getMessage(), equalTo("pipeline [unknown] is missing"));
+        }
+
+        // match all wildcard works on last remaining pipeline
+        DeletePipelineRequest matchAllDeleteRequest = new DeletePipelineRequest("*");
+        previousClusterState = clusterState;
+        clusterState = store.innerDelete(matchAllDeleteRequest, clusterState);
+        store.innerUpdatePipelines(previousClusterState, clusterState);
+        assertThat(store.get("p1"), nullValue());
+        assertThat(store.get("p2"), nullValue());
+        assertThat(store.get("q1"), nullValue());
+
+        // match all wildcard does not throw exception if none match
+        store.innerDelete(matchAllDeleteRequest, clusterState);
+    }
+
+    public void testDeleteWithExistingUnmatchedPipelines() {
+        HashMap<String, PipelineConfiguration> pipelines = new HashMap<>();
+        BytesArray definition = new BytesArray(
+            "{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}"
+        );
+        pipelines.put("p1", new PipelineConfiguration("p1", definition, XContentType.JSON));
+        IngestMetadata ingestMetadata = new IngestMetadata(pipelines);
+        ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build();
+        ClusterState previousClusterState = clusterState;
+        clusterState = ClusterState.builder(clusterState).metaData(MetaData.builder()
+            .putCustom(IngestMetadata.TYPE, ingestMetadata)).build();
+        store.innerUpdatePipelines(previousClusterState, clusterState);
+        assertThat(store.get("p1"), notNullValue());
+
+        DeletePipelineRequest deleteRequest = new DeletePipelineRequest("z*");
+        try {
+            store.innerDelete(deleteRequest, clusterState);
+            fail("exception expected");
+        } catch (ResourceNotFoundException e) {
+            assertThat(e.getMessage(), equalTo("pipeline [z*] is missing"));
+        }
+    }
+
     public void testGetPipelines() {
         Map<String, PipelineConfiguration> configs = new HashMap<>();
         configs.put("_id1", new PipelineConfiguration(
-            "_id1", new BytesArray("{\"processors\": []}")
+            "_id1", new BytesArray("{\"processors\": []}"), XContentType.JSON
         ));
         configs.put("_id2", new PipelineConfiguration(
-            "_id2", new BytesArray("{\"processors\": []}")
+            "_id2", new BytesArray("{\"processors\": []}"), XContentType.JSON
         ));
 
         assertThat(store.innerGetPipelines(null, "_id1").isEmpty(), is(true));
@@ -241,7 +318,7 @@ public void testCrud() throws Exception {
         ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build(); // Start empty
 
         PutPipelineRequest putRequest = new PutPipelineRequest(id,
-                new BytesArray("{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}"));
+                new BytesArray("{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}"), XContentType.JSON);
         ClusterState previousClusterState = clusterState;
         clusterState = store.innerPut(putRequest, clusterState);
         store.innerUpdatePipelines(previousClusterState, clusterState);
@@ -262,7 +339,9 @@ public void testCrud() throws Exception {
 
     public void testValidate() throws Exception {
         PutPipelineRequest putRequest = new PutPipelineRequest("_id", new BytesArray(
-                "{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}},{\"remove\" : {\"field\": \"_field\"}}]}"));
+                "{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\", \"tag\": \"tag1\"}}," +
+                    "{\"remove\" : {\"field\": \"_field\", \"tag\": \"tag2\"}}]}"),
+            XContentType.JSON);
 
         DiscoveryNode node1 = new DiscoveryNode("_node_id1", new LocalTransportAddress("_id"),
                 emptyMap(), emptySet(), Version.CURRENT);
@@ -272,12 +351,11 @@ public void testValidate() throws Exception {
         ingestInfos.put(node1, new IngestInfo(Arrays.asList(new ProcessorInfo("set"), new ProcessorInfo("remove"))));
         ingestInfos.put(node2, new IngestInfo(Arrays.asList(new ProcessorInfo("set"))));
 
-        try {
-            store.validatePipeline(ingestInfos, putRequest);
-            fail("exception expected");
-        } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), equalTo("Processor type [remove] is not installed on node [" + node2 + "]"));
-        }
+        ElasticsearchParseException e =
+            expectThrows(ElasticsearchParseException.class, () -> store.validatePipeline(ingestInfos, putRequest));
+        assertEquals("Processor type [remove] is not installed on node [" + node2 + "]", e.getMessage());
+        assertEquals("remove", e.getHeader("processor_type").get(0));
+        assertEquals("tag2", e.getHeader("processor_tag").get(0));
 
         ingestInfos.put(node2, new IngestInfo(Arrays.asList(new ProcessorInfo("set"), new ProcessorInfo("remove"))));
         store.validatePipeline(ingestInfos, putRequest);
@@ -285,13 +363,9 @@ public void testValidate() throws Exception {
 
     public void testValidateNoIngestInfo() throws Exception {
         PutPipelineRequest putRequest = new PutPipelineRequest("_id", new BytesArray(
-                "{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}"));
-        try {
-            store.validatePipeline(Collections.emptyMap(), putRequest);
-            fail("exception expected");
-        } catch (IllegalStateException e) {
-            assertThat(e.getMessage(), equalTo("Ingest info is empty"));
-        }
+                "{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}"), XContentType.JSON);
+        Exception e = expectThrows(IllegalStateException.class, () -> store.validatePipeline(Collections.emptyMap(), putRequest));
+        assertEquals("Ingest info is empty", e.getMessage());
 
         DiscoveryNode discoveryNode = new DiscoveryNode("_node_id", new LocalTransportAddress("_id"),
                 emptyMap(), emptySet(), Version.CURRENT);
@@ -299,4 +373,11 @@ public void testValidateNoIngestInfo() throws Exception {
         store.validatePipeline(Collections.singletonMap(discoveryNode, ingestInfo), putRequest);
     }
 
+    public void testUpdateIngestNewDateFormatSetting() throws Exception {
+        assertFalse(store.isNewIngestDateFormat());
+        clusterSettings.applySettings(Settings.builder().put(IngestService.NEW_INGEST_DATE_FORMAT.getKey(), true).build());
+        assertTrue(store.isNewIngestDateFormat());
+        assertWarnings("[ingest.new_date_format] setting was deprecated in Elasticsearch and will be " +
+            "removed in a future release! See the breaking changes documentation for the next major version.");
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/mget/SimpleMgetIT.java b/core/src/test/java/org/elasticsearch/mget/SimpleMgetIT.java
index 15fac5ab4c2b3..3c30e7610dd26 100644
--- a/core/src/test/java/org/elasticsearch/mget/SimpleMgetIT.java
+++ b/core/src/test/java/org/elasticsearch/mget/SimpleMgetIT.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 import org.elasticsearch.test.ESIntegTestCase;
 
@@ -36,12 +37,14 @@
 import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
 
 public class SimpleMgetIT extends ESIntegTestCase {
+
     public void testThatMgetShouldWorkWithOneIndexMissing() throws IOException {
         createIndex("test");
 
@@ -51,7 +54,7 @@ public void testThatMgetShouldWorkWithOneIndexMissing() throws IOException {
         MultiGetResponse mgetResponse = client().prepareMultiGet()
                 .add(new MultiGetRequest.Item("test", "test", "1"))
                 .add(new MultiGetRequest.Item("nonExistingIndex", "test", "1"))
-                .execute().actionGet();
+                .get();
         assertThat(mgetResponse.getResponses().length, is(2));
 
         assertThat(mgetResponse.getResponses()[0].getIndex(), is("test"));
@@ -60,19 +63,64 @@ public void testThatMgetShouldWorkWithOneIndexMissing() throws IOException {
         assertThat(mgetResponse.getResponses()[1].getIndex(), is("nonExistingIndex"));
         assertThat(mgetResponse.getResponses()[1].isFailed(), is(true));
         assertThat(mgetResponse.getResponses()[1].getFailure().getMessage(), is("no such index"));
-        assertThat(((ElasticsearchException) mgetResponse.getResponses()[1].getFailure().getFailure()).getIndex().getName(), is("nonExistingIndex"));
-
+        assertThat(((ElasticsearchException) mgetResponse.getResponses()[1].getFailure().getFailure()).getIndex().getName(),
+            is("nonExistingIndex"));
 
         mgetResponse = client().prepareMultiGet()
                 .add(new MultiGetRequest.Item("nonExistingIndex", "test", "1"))
-                .execute().actionGet();
+                .get();
         assertThat(mgetResponse.getResponses().length, is(1));
         assertThat(mgetResponse.getResponses()[0].getIndex(), is("nonExistingIndex"));
         assertThat(mgetResponse.getResponses()[0].isFailed(), is(true));
         assertThat(mgetResponse.getResponses()[0].getFailure().getMessage(), is("no such index"));
-        assertThat(((ElasticsearchException) mgetResponse.getResponses()[0].getFailure().getFailure()).getIndex().getName(), is("nonExistingIndex"));
+        assertThat(((ElasticsearchException) mgetResponse.getResponses()[0].getFailure().getFailure()).getIndex().getName(),
+            is("nonExistingIndex"));
+    }
+
+    public void testThatMgetShouldWorkWithMultiIndexAlias() throws IOException {
+        assertAcked(prepareCreate("test").addAlias(new Alias("multiIndexAlias")));
+        assertAcked(prepareCreate("test2").addAlias(new Alias("multiIndexAlias")));
+
+        client().prepareIndex("test", "test", "1").setSource(jsonBuilder().startObject().field("foo", "bar").endObject())
+            .setRefreshPolicy(IMMEDIATE).get();
+
+        MultiGetResponse mgetResponse = client().prepareMultiGet()
+            .add(new MultiGetRequest.Item("test", "test", "1"))
+            .add(new MultiGetRequest.Item("multiIndexAlias", "test", "1"))
+            .get();
+        assertThat(mgetResponse.getResponses().length, is(2));
+
+        assertThat(mgetResponse.getResponses()[0].getIndex(), is("test"));
+        assertThat(mgetResponse.getResponses()[0].isFailed(), is(false));
 
+        assertThat(mgetResponse.getResponses()[1].getIndex(), is("multiIndexAlias"));
+        assertThat(mgetResponse.getResponses()[1].isFailed(), is(true));
+        assertThat(mgetResponse.getResponses()[1].getFailure().getMessage(), containsString("more than one indices"));
 
+        mgetResponse = client().prepareMultiGet()
+            .add(new MultiGetRequest.Item("multiIndexAlias", "test", "1"))
+            .get();
+        assertThat(mgetResponse.getResponses().length, is(1));
+        assertThat(mgetResponse.getResponses()[0].getIndex(), is("multiIndexAlias"));
+        assertThat(mgetResponse.getResponses()[0].isFailed(), is(true));
+        assertThat(mgetResponse.getResponses()[0].getFailure().getMessage(), containsString("more than one indices"));
+    }
+
+    public void testThatMgetShouldWorkWithAliasRouting() throws IOException {
+        assertAcked(prepareCreate("test").addAlias(new Alias("alias1").routing("abc"))
+            .addMapping("test", jsonBuilder()
+                .startObject().startObject("test").startObject("_routing").field("required", true).endObject().endObject().endObject()));
+
+        client().prepareIndex("alias1", "test", "1").setSource(jsonBuilder().startObject().field("foo", "bar").endObject())
+            .setRefreshPolicy(IMMEDIATE).get();
+
+        MultiGetResponse mgetResponse = client().prepareMultiGet()
+            .add(new MultiGetRequest.Item("alias1", "test", "1"))
+            .get();
+        assertEquals(1, mgetResponse.getResponses().length);
+
+        assertEquals("test", mgetResponse.getResponses()[0].getIndex());
+        assertFalse(mgetResponse.getResponses()[0].isFailed());
     }
 
     public void testThatParentPerDocumentIsSupported() throws Exception {
@@ -93,7 +141,7 @@ public void testThatParentPerDocumentIsSupported() throws Exception {
         MultiGetResponse mgetResponse = client().prepareMultiGet()
                 .add(new MultiGetRequest.Item(indexOrAlias(), "test", "1").parent("4"))
                 .add(new MultiGetRequest.Item(indexOrAlias(), "test", "1"))
-                .execute().actionGet();
+                .get();
 
         assertThat(mgetResponse.getResponses().length, is(2));
         assertThat(mgetResponse.getResponses()[0].isFailed(), is(false));
@@ -108,20 +156,22 @@ public void testThatParentPerDocumentIsSupported() throws Exception {
     public void testThatSourceFilteringIsSupported() throws Exception {
         assertAcked(prepareCreate("test").addAlias(new Alias("alias")));
         BytesReference sourceBytesRef = jsonBuilder().startObject()
-                .field("field", "1", "2")
+                .array("field", "1", "2")
                 .startObject("included").field("field", "should be seen").field("hidden_field", "should not be seen").endObject()
                 .field("excluded", "should not be seen")
                 .endObject().bytes();
         for (int i = 0; i < 100; i++) {
-            client().prepareIndex("test", "type", Integer.toString(i)).setSource(sourceBytesRef).get();
+            client().prepareIndex("test", "type", Integer.toString(i)).setSource(sourceBytesRef, XContentType.JSON).get();
         }
 
         MultiGetRequestBuilder request = client().prepareMultiGet();
         for (int i = 0; i < 100; i++) {
             if (i % 2 == 0) {
-                request.add(new MultiGetRequest.Item(indexOrAlias(), "type", Integer.toString(i)).fetchSourceContext(new FetchSourceContext("included", "*.hidden_field")));
+                request.add(new MultiGetRequest.Item(indexOrAlias(), "type", Integer.toString(i))
+                    .fetchSourceContext(new FetchSourceContext(true, new String[] {"included"}, new String[] {"*.hidden_field"})));
             } else {
-                request.add(new MultiGetRequest.Item(indexOrAlias(), "type", Integer.toString(i)).fetchSourceContext(new FetchSourceContext(false)));
+                request.add(new MultiGetRequest.Item(indexOrAlias(), "type", Integer.toString(i))
+                    .fetchSourceContext(new FetchSourceContext(false)));
             }
         }
 
@@ -159,7 +209,7 @@ public void testThatRoutingPerDocumentIsSupported() throws Exception {
         MultiGetResponse mgetResponse = client().prepareMultiGet()
                 .add(new MultiGetRequest.Item(indexOrAlias(), "test", id).routing(routingOtherShard))
                 .add(new MultiGetRequest.Item(indexOrAlias(), "test", id))
-                .execute().actionGet();
+                .get();
 
         assertThat(mgetResponse.getResponses().length, is(2));
         assertThat(mgetResponse.getResponses()[0].isFailed(), is(false));
diff --git a/core/src/test/java/org/elasticsearch/monitor/fs/DeviceStatsTests.java b/core/src/test/java/org/elasticsearch/monitor/fs/DeviceStatsTests.java
index 758f67ff563f3..85aeed9485f9c 100644
--- a/core/src/test/java/org/elasticsearch/monitor/fs/DeviceStatsTests.java
+++ b/core/src/test/java/org/elasticsearch/monitor/fs/DeviceStatsTests.java
@@ -28,7 +28,7 @@ public class DeviceStatsTests extends ESTestCase {
     public void testDeviceStats() {
         final int majorDeviceNumber = randomIntBetween(1, 1 << 8);
         final int minorDeviceNumber = randomIntBetween(0, 1 << 5);
-        final String deviceName = randomAsciiOfLength(3);
+        final String deviceName = randomAlphaOfLength(3);
         final int readsCompleted = randomIntBetween(1, 1 << 16);
         final int sectorsRead = randomIntBetween(8 * readsCompleted, 16 * readsCompleted);
         final int writesCompleted = randomIntBetween(1, 1 << 16);
diff --git a/core/src/test/java/org/elasticsearch/monitor/fs/FsProbeTests.java b/core/src/test/java/org/elasticsearch/monitor/fs/FsProbeTests.java
index 14f7151e40ddb..c1ff620841619 100644
--- a/core/src/test/java/org/elasticsearch/monitor/fs/FsProbeTests.java
+++ b/core/src/test/java/org/elasticsearch/monitor/fs/FsProbeTests.java
@@ -23,18 +23,26 @@
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.env.NodeEnvironment.NodePath;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
+import java.nio.file.FileStore;
+import java.nio.file.Path;
+import java.nio.file.attribute.FileAttributeView;
+import java.nio.file.attribute.FileStoreAttributeView;
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Function;
+import java.util.function.Supplier;
 
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.lessThan;
 import static org.hamcrest.Matchers.isEmptyOrNullString;
 import static org.hamcrest.Matchers.not;
 
@@ -86,6 +94,67 @@ public void testFsInfo() throws IOException {
         }
     }
 
+    public void testFsInfoOverflow() throws Exception {
+        final FsInfo.Path pathStats =
+                new FsInfo.Path(
+                        "/foo/bar",
+                        null,
+                        randomNonNegativeLong(),
+                        randomNonNegativeLong(),
+                        randomNonNegativeLong());
+
+        addUntilOverflow(
+                pathStats,
+                p -> p.total,
+                "total",
+                () -> new FsInfo.Path("/foo/baz", null, randomNonNegativeLong(), 0, 0));
+
+        addUntilOverflow(
+                pathStats,
+                p -> p.free,
+                "free",
+                () -> new FsInfo.Path("/foo/baz", null, 0, randomNonNegativeLong(), 0));
+
+        addUntilOverflow(
+                pathStats,
+                p -> p.available,
+                "available",
+                () -> new FsInfo.Path("/foo/baz", null, 0, 0, randomNonNegativeLong()));
+
+        // even after overflowing these should not be negative
+        assertThat(pathStats.total, greaterThan(0L));
+        assertThat(pathStats.free, greaterThan(0L));
+        assertThat(pathStats.available, greaterThan(0L));
+    }
+
+    private void addUntilOverflow(
+            final FsInfo.Path pathStats,
+            final Function<FsInfo.Path, Long> getter,
+            final String field,
+            final Supplier<FsInfo.Path> supplier) {
+        FsInfo.Path pathToAdd = supplier.get();
+        while ((getter.apply(pathStats) + getter.apply(pathToAdd)) > 0) {
+            // add a path to increase the total bytes until it overflows
+            logger.info(
+                    "--> adding {} bytes to {}, {} will be: {}",
+                    getter.apply(pathToAdd),
+                    getter.apply(pathStats),
+                    field,
+                    getter.apply(pathStats) + getter.apply(pathToAdd));
+            pathStats.add(pathToAdd);
+            pathToAdd = supplier.get();
+        }
+        // this overflows
+        logger.info(
+                "--> adding {} bytes to {}, {} will be: {}",
+                getter.apply(pathToAdd),
+                getter.apply(pathStats),
+                field,
+                getter.apply(pathStats) + getter.apply(pathToAdd));
+        assertThat(getter.apply(pathStats) + getter.apply(pathToAdd), lessThan(0L));
+        pathStats.add(pathToAdd);
+    }
+
     public void testIoStats() {
         final AtomicReference<List<String>> diskStats = new AtomicReference<>();
         diskStats.set(Arrays.asList(
@@ -174,4 +243,74 @@ List<String> readProcDiskStats() throws IOException {
         assertThat(second.totalWriteKilobytes, equalTo(1236L));
     }
 
+    public void testAdjustForHugeFilesystems() throws Exception {
+        NodePath np = new FakeNodePath(createTempDir());
+        assertThat(FsProbe.getFSInfo(np).total, greaterThanOrEqualTo(0L));
+    }
+
+    static class FakeNodePath extends NodeEnvironment.NodePath {
+        public final FileStore fileStore;
+
+        FakeNodePath(Path path) throws IOException {
+            super(path);
+            this.fileStore = new HugeFileStore();
+        }
+    }
+
+    /**
+     * Randomly returns negative values for disk space to simulate https://bugs.openjdk.java.net/browse/JDK-8162520
+     */
+    static class HugeFileStore extends FileStore {
+
+        @Override
+        public String name() {
+            return "myHugeFS";
+        }
+
+        @Override
+        public String type() {
+            return "bigFS";
+        }
+
+        @Override
+        public boolean isReadOnly() {
+            return false;
+        }
+
+        @Override
+        public long getTotalSpace() throws IOException {
+            return randomIntBetween(-1000, 1000);
+        }
+
+        @Override
+        public long getUsableSpace() throws IOException {
+            return 10;
+        }
+
+        @Override
+        public long getUnallocatedSpace() throws IOException {
+            return 10;
+        }
+
+        @Override
+        public boolean supportsFileAttributeView(Class<? extends FileAttributeView> type) {
+            return false;
+        }
+
+        @Override
+        public boolean supportsFileAttributeView(String name) {
+            return false;
+        }
+
+        @Override
+        public <V extends FileStoreAttributeView> V getFileStoreAttributeView(Class<V> type) {
+            throw new UnsupportedOperationException("don't call me");
+        }
+
+        @Override
+        public Object getAttribute(String attribute) throws IOException {
+            throw new UnsupportedOperationException("don't call me");
+        }
+
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/monitor/jvm/JvmGcMonitorServiceSettingsTests.java b/core/src/test/java/org/elasticsearch/monitor/jvm/JvmGcMonitorServiceSettingsTests.java
index dc374f133391c..48817e52d562e 100644
--- a/core/src/test/java/org/elasticsearch/monitor/jvm/JvmGcMonitorServiceSettingsTests.java
+++ b/core/src/test/java/org/elasticsearch/monitor/jvm/JvmGcMonitorServiceSettingsTests.java
@@ -55,7 +55,7 @@ public void testDisabledSetting() throws InterruptedException {
     }
 
     public void testNegativeSetting() throws InterruptedException {
-        String collector = randomAsciiOfLength(5);
+        String collector = randomAlphaOfLength(5);
         Settings settings = Settings.builder().put("monitor.jvm.gc.collector." + collector + ".warn", "-" + randomTimeValue()).build();
         execute(settings, (command, interval, name) -> null, e -> {
             assertThat(e, instanceOf(IllegalArgumentException.class));
@@ -64,7 +64,7 @@ public void testNegativeSetting() throws InterruptedException {
     }
 
     public void testMissingSetting() throws InterruptedException {
-        String collector = randomAsciiOfLength(5);
+        String collector = randomAlphaOfLength(5);
         Set<AbstractMap.SimpleEntry<String, String>> entries = new HashSet<>();
         entries.add(new AbstractMap.SimpleEntry<>("monitor.jvm.gc.collector." + collector + ".warn", randomPositiveTimeValue()));
         entries.add(new AbstractMap.SimpleEntry<>("monitor.jvm.gc.collector." + collector + ".info", randomPositiveTimeValue()));
diff --git a/core/src/test/java/org/elasticsearch/monitor/jvm/JvmGcMonitorServiceTests.java b/core/src/test/java/org/elasticsearch/monitor/jvm/JvmGcMonitorServiceTests.java
index 972072c0ebd25..711988d5a826b 100644
--- a/core/src/test/java/org/elasticsearch/monitor/jvm/JvmGcMonitorServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/monitor/jvm/JvmGcMonitorServiceTests.java
@@ -37,7 +37,7 @@ public void testSlowGcLogging() {
         when(logger.isInfoEnabled()).thenReturn(true);
         when(logger.isDebugEnabled()).thenReturn(true);
         final JvmGcMonitorService.JvmMonitor.Threshold threshold = randomFrom(JvmGcMonitorService.JvmMonitor.Threshold.values());
-        final String name = randomAsciiOfLength(16);
+        final String name = randomAlphaOfLength(16);
         final long seq = randomIntBetween(1, 1 << 30);
         final int elapsedValue = randomIntBetween(1, 1 << 10);
         final long totalCollectionCount = randomIntBetween(1, 16);
@@ -64,7 +64,7 @@ public void testSlowGcLogging() {
         when(gc.getCollectionCount()).thenReturn(totalCollectionCount);
         when(gc.getCollectionTime()).thenReturn(totalCollectionTime);
 
-        final ByteSizeValue maxHeapUsed = new ByteSizeValue(Math.max(lastHeapUsed.bytes(), currentHeapUsed.bytes()) + 1 << 10);
+        final ByteSizeValue maxHeapUsed = new ByteSizeValue(Math.max(lastHeapUsed.getBytes(), currentHeapUsed.getBytes()) + 1 << 10);
 
         JvmGcMonitorService.JvmMonitor.SlowGcEvent slowGcEvent = new JvmGcMonitorService.JvmMonitor.SlowGcEvent(
             gc,
diff --git a/core/src/test/java/org/elasticsearch/monitor/jvm/JvmStatsTests.java b/core/src/test/java/org/elasticsearch/monitor/jvm/JvmStatsTests.java
index d0b1d54171f5b..aa59a914dc3ec 100644
--- a/core/src/test/java/org/elasticsearch/monitor/jvm/JvmStatsTests.java
+++ b/core/src/test/java/org/elasticsearch/monitor/jvm/JvmStatsTests.java
@@ -45,7 +45,7 @@ public void testJvmStats() throws IOException {
         assertNotNull(mem);
         for (ByteSizeValue heap : Arrays.asList(mem.getHeapCommitted(), mem.getHeapMax(), mem.getHeapUsed(), mem.getNonHeapCommitted())) {
             assertNotNull(heap);
-            assertThat(heap.bytes(), greaterThanOrEqualTo(0L));
+            assertThat(heap.getBytes(), greaterThanOrEqualTo(0L));
         }
         assertNotNull(mem.getHeapUsedPercent());
         assertThat(mem.getHeapUsedPercent(), anyOf(equalTo((short) -1), greaterThanOrEqualTo((short) 0)));
@@ -78,9 +78,9 @@ public void testJvmStats() throws IOException {
                 assertTrue(Strings.hasText(bufferPool.getName()));
                 assertThat(bufferPool.getCount(), greaterThanOrEqualTo(0L));
                 assertNotNull(bufferPool.getTotalCapacity());
-                assertThat(bufferPool.getTotalCapacity().bytes(), greaterThanOrEqualTo(0L));
+                assertThat(bufferPool.getTotalCapacity().getBytes(), greaterThanOrEqualTo(0L));
                 assertNotNull(bufferPool.getUsed());
-                assertThat(bufferPool.getUsed().bytes(), anyOf(equalTo(-1L), greaterThanOrEqualTo(0L)));
+                assertThat(bufferPool.getUsed().getBytes(), anyOf(equalTo(-1L), greaterThanOrEqualTo(0L)));
             }
         }
 
diff --git a/core/src/test/java/org/elasticsearch/monitor/os/OsProbeTests.java b/core/src/test/java/org/elasticsearch/monitor/os/OsProbeTests.java
index 2e085a807001c..71305c41f56d7 100644
--- a/core/src/test/java/org/elasticsearch/monitor/os/OsProbeTests.java
+++ b/core/src/test/java/org/elasticsearch/monitor/os/OsProbeTests.java
@@ -22,6 +22,9 @@
 import org.apache.lucene.util.Constants;
 import org.elasticsearch.test.ESTestCase;
 
+import java.util.Arrays;
+import java.util.List;
+
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.both;
@@ -30,13 +33,15 @@
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
+import static org.hamcrest.Matchers.notNullValue;
 
 public class OsProbeTests extends ESTestCase {
+
     private final OsProbe probe = OsProbe.getInstance();
 
     public void testOsInfo() {
         int allocatedProcessors = randomIntBetween(1, Runtime.getRuntime().availableProcessors());
-        long refreshInterval = randomBoolean() ? -1 : randomPositiveLong();
+        long refreshInterval = randomBoolean() ? -1 : randomNonNegativeLong();
         OsInfo info = probe.osInfo(refreshInterval, allocatedProcessors);
         assertNotNull(info);
         assertEquals(refreshInterval, info.getRefreshInterval());
@@ -66,12 +71,6 @@ public void testOsStats() {
             assertThat(loadAverage[0], greaterThanOrEqualTo((double) 0));
             assertThat(loadAverage[1], greaterThanOrEqualTo((double) 0));
             assertThat(loadAverage[2], greaterThanOrEqualTo((double) 0));
-        } else if (Constants.FREE_BSD) {
-            // five- and fifteen-minute load averages not available if linprocfs is not mounted at /compat/linux/proc
-            assertNotNull(loadAverage);
-            assertThat(loadAverage[0], greaterThanOrEqualTo((double) 0));
-            assertThat(loadAverage[1], anyOf(equalTo((double) -1), greaterThanOrEqualTo((double) 0)));
-            assertThat(loadAverage[2], anyOf(equalTo((double) -1), greaterThanOrEqualTo((double) 0)));
         } else if (Constants.MAC_OS_X) {
             // one minute load average is available, but 10-minute and 15-minute load averages are not
             assertNotNull(loadAverage);
@@ -88,25 +87,135 @@ public void testOsStats() {
         }
 
         assertNotNull(stats.getMem());
-        assertThat(stats.getMem().getTotal().bytes(), greaterThan(0L));
-        assertThat(stats.getMem().getFree().bytes(), greaterThan(0L));
+        assertThat(stats.getMem().getTotal().getBytes(), greaterThan(0L));
+        assertThat(stats.getMem().getFree().getBytes(), greaterThan(0L));
         assertThat(stats.getMem().getFreePercent(), allOf(greaterThanOrEqualTo((short) 0), lessThanOrEqualTo((short) 100)));
-        assertThat(stats.getMem().getUsed().bytes(), greaterThan(0L));
+        assertThat(stats.getMem().getUsed().getBytes(), greaterThan(0L));
         assertThat(stats.getMem().getUsedPercent(), allOf(greaterThanOrEqualTo((short) 0), lessThanOrEqualTo((short) 100)));
 
         assertNotNull(stats.getSwap());
         assertNotNull(stats.getSwap().getTotal());
 
-        long total = stats.getSwap().getTotal().bytes();
+        long total = stats.getSwap().getTotal().getBytes();
         if (total > 0) {
-            assertThat(stats.getSwap().getTotal().bytes(), greaterThan(0L));
-            assertThat(stats.getSwap().getFree().bytes(), greaterThan(0L));
-            assertThat(stats.getSwap().getUsed().bytes(), greaterThanOrEqualTo(0L));
+            assertThat(stats.getSwap().getTotal().getBytes(), greaterThan(0L));
+            assertThat(stats.getSwap().getFree().getBytes(), greaterThan(0L));
+            assertThat(stats.getSwap().getUsed().getBytes(), greaterThanOrEqualTo(0L));
         } else {
             // On platforms with no swap
-            assertThat(stats.getSwap().getTotal().bytes(), equalTo(0L));
-            assertThat(stats.getSwap().getFree().bytes(), equalTo(0L));
-            assertThat(stats.getSwap().getUsed().bytes(), equalTo(0L));
+            assertThat(stats.getSwap().getTotal().getBytes(), equalTo(0L));
+            assertThat(stats.getSwap().getFree().getBytes(), equalTo(0L));
+            assertThat(stats.getSwap().getUsed().getBytes(), equalTo(0L));
+        }
+
+        if (Constants.LINUX) {
+            if (stats.getCgroup() != null) {
+                assertThat(stats.getCgroup().getCpuAcctControlGroup(), notNullValue());
+                assertThat(stats.getCgroup().getCpuAcctUsageNanos(), greaterThan(0L));
+                assertThat(stats.getCgroup().getCpuCfsQuotaMicros(), anyOf(equalTo(-1L), greaterThanOrEqualTo(0L)));
+                assertThat(stats.getCgroup().getCpuCfsPeriodMicros(), greaterThanOrEqualTo(0L));
+                assertThat(stats.getCgroup().getCpuStat().getNumberOfElapsedPeriods(), greaterThanOrEqualTo(0L));
+                assertThat(stats.getCgroup().getCpuStat().getNumberOfTimesThrottled(), greaterThanOrEqualTo(0L));
+                assertThat(stats.getCgroup().getCpuStat().getTimeThrottledNanos(), greaterThanOrEqualTo(0L));
+            }
+        } else {
+            assertNull(stats.getCgroup());
         }
     }
+
+    public void testGetSystemLoadAverage() {
+        assumeTrue("test runs on Linux only", Constants.LINUX);
+
+        final OsProbe probe = new OsProbe() {
+            @Override
+            String readProcLoadavg() {
+                return "1.51 1.69 1.99 3/417 23251";
+            }
+        };
+
+        final double[] systemLoadAverage = probe.getSystemLoadAverage();
+
+        assertNotNull(systemLoadAverage);
+        assertThat(systemLoadAverage.length, equalTo(3));
+
+        // avoid silliness with representing doubles
+        assertThat(systemLoadAverage[0], equalTo(Double.parseDouble("1.51")));
+        assertThat(systemLoadAverage[1], equalTo(Double.parseDouble("1.69")));
+        assertThat(systemLoadAverage[2], equalTo(Double.parseDouble("1.99")));
+    }
+
+    public void testCgroupProbe() {
+        assumeTrue("test runs on Linux only", Constants.LINUX);
+
+        final boolean areCgroupStatsAvailable = randomBoolean();
+        final String hierarchy = randomAlphaOfLength(16);
+
+        final OsProbe probe = new OsProbe() {
+
+            @Override
+            List<String> readProcSelfCgroup() {
+                return Arrays.asList(
+                        "10:freezer:/",
+                        "9:net_cls,net_prio:/",
+                        "8:pids:/",
+                        "7:blkio:/",
+                        "6:memory:/",
+                        "5:devices:/user.slice",
+                        "4:hugetlb:/",
+                        "3:perf_event:/",
+                        "2:cpu,cpuacct,cpuset:/" + hierarchy,
+                        "1:name=systemd:/user.slice/user-1000.slice/session-2359.scope",
+                        "0::/cgroup2");
+            }
+
+            @Override
+            String readSysFsCgroupCpuAcctCpuAcctUsage(String controlGroup) {
+                assertThat(controlGroup, equalTo("/" + hierarchy));
+                return "364869866063112";
+            }
+
+            @Override
+            String readSysFsCgroupCpuAcctCpuCfsPeriod(String controlGroup) {
+                assertThat(controlGroup, equalTo("/" + hierarchy));
+                return "100000";
+            }
+
+            @Override
+            String readSysFsCgroupCpuAcctCpuAcctCfsQuota(String controlGroup) {
+                assertThat(controlGroup, equalTo("/" + hierarchy));
+                return "50000";
+            }
+
+            @Override
+            List<String> readSysFsCgroupCpuAcctCpuStat(String controlGroup) {
+                return Arrays.asList(
+                    "nr_periods 17992",
+                    "nr_throttled 1311",
+                    "throttled_time 139298645489");
+            }
+
+            @Override
+            boolean areCgroupStatsAvailable() {
+                return areCgroupStatsAvailable;
+            }
+
+        };
+
+        final OsStats.Cgroup cgroup = probe.osStats().getCgroup();
+
+        if (areCgroupStatsAvailable) {
+            assertNotNull(cgroup);
+            assertThat(cgroup.getCpuAcctControlGroup(), equalTo("/" + hierarchy));
+            assertThat(cgroup.getCpuAcctUsageNanos(), equalTo(364869866063112L));
+            assertThat(cgroup.getCpuControlGroup(), equalTo("/" + hierarchy));
+            assertThat(cgroup.getCpuCfsPeriodMicros(), equalTo(100000L));
+            assertThat(cgroup.getCpuCfsQuotaMicros(), equalTo(50000L));
+            assertThat(cgroup.getCpuStat().getNumberOfElapsedPeriods(), equalTo(17992L));
+            assertThat(cgroup.getCpuStat().getNumberOfTimesThrottled(), equalTo(1311L));
+            assertThat(cgroup.getCpuStat().getTimeThrottledNanos(), equalTo(139298645489L));
+        } else {
+            assertNull(cgroup);
+        }
+    }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/monitor/os/OsStatsTests.java b/core/src/test/java/org/elasticsearch/monitor/os/OsStatsTests.java
index 30d527311b31f..f1e2371db5cb7 100644
--- a/core/src/test/java/org/elasticsearch/monitor/os/OsStatsTests.java
+++ b/core/src/test/java/org/elasticsearch/monitor/os/OsStatsTests.java
@@ -36,7 +36,14 @@ public void testSerialization() throws IOException {
         OsStats.Cpu cpu = new OsStats.Cpu(randomShort(), loadAverages);
         OsStats.Mem mem = new OsStats.Mem(randomLong(), randomLong());
         OsStats.Swap swap = new OsStats.Swap(randomLong(), randomLong());
-        OsStats osStats = new OsStats(System.currentTimeMillis(), cpu, mem, swap);
+        OsStats.Cgroup cgroup = new OsStats.Cgroup(
+            randomAlphaOfLength(8),
+            randomNonNegativeLong(),
+            randomAlphaOfLength(8),
+            randomNonNegativeLong(),
+            randomNonNegativeLong(),
+            new OsStats.Cgroup.CpuStat(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()));
+        OsStats osStats = new OsStats(System.currentTimeMillis(), cpu, mem, swap, cgroup);
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             osStats.writeTo(out);
@@ -49,7 +56,22 @@ public void testSerialization() throws IOException {
                 assertEquals(osStats.getMem().getTotal(), deserializedOsStats.getMem().getTotal());
                 assertEquals(osStats.getSwap().getFree(), deserializedOsStats.getSwap().getFree());
                 assertEquals(osStats.getSwap().getTotal(), deserializedOsStats.getSwap().getTotal());
+                assertEquals(osStats.getCgroup().getCpuAcctControlGroup(), deserializedOsStats.getCgroup().getCpuAcctControlGroup());
+                assertEquals(osStats.getCgroup().getCpuAcctUsageNanos(), deserializedOsStats.getCgroup().getCpuAcctUsageNanos());
+                assertEquals(osStats.getCgroup().getCpuControlGroup(), deserializedOsStats.getCgroup().getCpuControlGroup());
+                assertEquals(osStats.getCgroup().getCpuCfsPeriodMicros(), deserializedOsStats.getCgroup().getCpuCfsPeriodMicros());
+                assertEquals(osStats.getCgroup().getCpuCfsQuotaMicros(), deserializedOsStats.getCgroup().getCpuCfsQuotaMicros());
+                assertEquals(
+                    osStats.getCgroup().getCpuStat().getNumberOfElapsedPeriods(),
+                    deserializedOsStats.getCgroup().getCpuStat().getNumberOfElapsedPeriods());
+                assertEquals(
+                    osStats.getCgroup().getCpuStat().getNumberOfTimesThrottled(),
+                    deserializedOsStats.getCgroup().getCpuStat().getNumberOfTimesThrottled());
+                assertEquals(
+                    osStats.getCgroup().getCpuStat().getTimeThrottledNanos(),
+                    deserializedOsStats.getCgroup().getCpuStat().getTimeThrottledNanos());
             }
         }
     }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/monitor/process/ProcessProbeTests.java b/core/src/test/java/org/elasticsearch/monitor/process/ProcessProbeTests.java
index 61c91368fa08d..4a44c518051a2 100644
--- a/core/src/test/java/org/elasticsearch/monitor/process/ProcessProbeTests.java
+++ b/core/src/test/java/org/elasticsearch/monitor/process/ProcessProbeTests.java
@@ -36,7 +36,7 @@ public class ProcessProbeTests extends ESTestCase {
     private final ProcessProbe probe = ProcessProbe.getInstance();
 
     public void testProcessInfo() {
-        long refreshInterval = randomPositiveLong();
+        long refreshInterval = randomNonNegativeLong();
         ProcessInfo info = probe.processInfo(refreshInterval);
         assertNotNull(info);
         assertEquals(refreshInterval, info.getRefreshInterval());
@@ -70,6 +70,6 @@ public void testProcessStats() {
         ProcessStats.Mem mem = stats.getMem();
         assertNotNull(mem);
         // Commited total virtual memory can return -1 if not supported, let's see which platforms fail
-        assertThat(mem.getTotalVirtual().bytes(), greaterThan(0L));
+        assertThat(mem.getTotalVirtual().getBytes(), greaterThan(0L));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/node/InternalSettingsPreparerTests.java b/core/src/test/java/org/elasticsearch/node/InternalSettingsPreparerTests.java
new file mode 100644
index 0000000000000..5db397ab16d93
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/node/InternalSettingsPreparerTests.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.node;
+
+import org.elasticsearch.cli.MockTerminal;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.common.settings.MockSecureSettings;
+import org.elasticsearch.common.settings.SecureSetting;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsException;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+
+public class InternalSettingsPreparerTests extends ESTestCase {
+
+    Path homeDir;
+    Settings baseEnvSettings;
+
+    @Before
+    public void createBaseEnvSettings() {
+        homeDir = createTempDir();
+        baseEnvSettings = Settings.builder()
+            .put(Environment.PATH_HOME_SETTING.getKey(), homeDir)
+            .build();
+    }
+
+    @After
+    public void clearBaseEnvSettings() {
+        homeDir = null;
+        baseEnvSettings = null;
+    }
+
+    public void testEmptySettings() {
+        Settings settings = InternalSettingsPreparer.prepareSettings(Settings.EMPTY);
+        assertNull(settings.get("node.name")); // a name was not set
+        assertNotNull(settings.get(ClusterName.CLUSTER_NAME_SETTING.getKey())); // a cluster name was set
+        int size = settings.names().size();
+
+        Environment env = InternalSettingsPreparer.prepareEnvironment(baseEnvSettings, null);
+        settings = env.settings();
+        assertNull(settings.get("node.name")); // a name was not set
+        assertNotNull(settings.get(ClusterName.CLUSTER_NAME_SETTING.getKey())); // a cluster name was set
+        assertEquals(settings.toString(), size + 1 /* path.home is in the base settings */, settings.names().size());
+        String home = Environment.PATH_HOME_SETTING.get(baseEnvSettings);
+        String configDir = env.configFile().toString();
+        assertTrue(configDir, configDir.startsWith(home));
+    }
+
+    public void testDefaultClusterName() {
+        Settings settings = InternalSettingsPreparer.prepareSettings(Settings.EMPTY);
+        assertEquals("elasticsearch", settings.get("cluster.name"));
+        settings = InternalSettingsPreparer.prepareSettings(Settings.builder().put("cluster.name", "foobar").build());
+        assertEquals("foobar", settings.get("cluster.name"));
+    }
+
+    public void testReplacePromptPlaceholders() {
+        MockTerminal terminal = new MockTerminal();
+        terminal.addTextInput("text");
+        terminal.addSecretInput("replaced");
+
+        Settings.Builder builder = Settings.builder()
+                .put(baseEnvSettings)
+                .put("password.replace", InternalSettingsPreparer.SECRET_PROMPT_VALUE)
+                .put("dont.replace", "prompt:secret")
+                .put("dont.replace2", "_prompt:secret_")
+                .put("dont.replace3", "_prompt:text__")
+                .put("dont.replace4", "__prompt:text_")
+                .put("dont.replace5", "prompt:secret__")
+                .put("replace_me", InternalSettingsPreparer.TEXT_PROMPT_VALUE);
+        Settings settings = InternalSettingsPreparer.prepareEnvironment(builder.build(), terminal).settings();
+
+        assertThat(settings.get("password.replace"), equalTo("replaced"));
+        assertThat(settings.get("replace_me"), equalTo("text"));
+
+        // verify other values unchanged
+        assertThat(settings.get("dont.replace"), equalTo("prompt:secret"));
+        assertThat(settings.get("dont.replace2"), equalTo("_prompt:secret_"));
+        assertThat(settings.get("dont.replace3"), equalTo("_prompt:text__"));
+        assertThat(settings.get("dont.replace4"), equalTo("__prompt:text_"));
+        assertThat(settings.get("dont.replace5"), equalTo("prompt:secret__"));
+    }
+
+    public void testReplaceSecretPromptPlaceholderWithNullTerminal() {
+        Settings.Builder builder = Settings.builder()
+                .put(baseEnvSettings)
+                .put("replace_me1", InternalSettingsPreparer.SECRET_PROMPT_VALUE);
+        try {
+            InternalSettingsPreparer.prepareEnvironment(builder.build(), null);
+            fail("an exception should have been thrown since no terminal was provided!");
+        } catch (UnsupportedOperationException e) {
+            assertThat(e.getMessage(), containsString("with value [" + InternalSettingsPreparer.SECRET_PROMPT_VALUE + "]"));
+        }
+    }
+
+    public void testReplaceTextPromptPlaceholderWithNullTerminal() {
+        Settings.Builder builder = Settings.builder()
+                .put(baseEnvSettings)
+                .put("replace_me1", InternalSettingsPreparer.TEXT_PROMPT_VALUE);
+        try {
+            InternalSettingsPreparer.prepareEnvironment(builder.build(), null);
+            fail("an exception should have been thrown since no terminal was provided!");
+        } catch (UnsupportedOperationException e) {
+            assertThat(e.getMessage(), containsString("with value [" + InternalSettingsPreparer.TEXT_PROMPT_VALUE + "]"));
+        }
+    }
+
+    public void testGarbageIsNotSwallowed() throws IOException {
+        try {
+            InputStream garbage = getClass().getResourceAsStream("/config/garbage/garbage.yml");
+            Path home = createTempDir();
+            Path config = home.resolve("config");
+            Files.createDirectory(config);
+            Files.copy(garbage, config.resolve("elasticsearch.yml"));
+            InternalSettingsPreparer.prepareEnvironment(Settings.builder()
+                .put(baseEnvSettings)
+                .build(), null);
+        } catch (SettingsException e) {
+            assertEquals("Failed to load settings from [elasticsearch.yml]", e.getMessage());
+        }
+    }
+
+    public void testMultipleSettingsFileNotAllowed() throws IOException {
+        InputStream yaml = getClass().getResourceAsStream("/config/elasticsearch.yaml");
+        InputStream json = getClass().getResourceAsStream("/config/elasticsearch.json");
+        Path config = homeDir.resolve("config");
+        Files.createDirectory(config);
+        Files.copy(yaml, config.resolve("elasticsearch.yaml"));
+        Files.copy(json, config.resolve("elasticsearch.json"));
+
+        SettingsException e = expectThrows(SettingsException.class, () ->
+            InternalSettingsPreparer.prepareEnvironment(Settings.builder().put(baseEnvSettings).build(), null)
+        );
+        assertTrue(e.getMessage(), e.getMessage().contains("multiple settings files found with suffixes"));
+        assertTrue(e.getMessage(), e.getMessage().contains(".yaml"));
+        assertTrue(e.getMessage(), e.getMessage().contains(".json"));
+    }
+
+    public void testYmlExtension() throws IOException {
+        InputStream yaml = getClass().getResourceAsStream("/config/elasticsearch.yaml");
+        Path config = homeDir.resolve("config");
+        Files.createDirectory(config);
+        Files.copy(yaml, config.resolve("elasticsearch.yml"));
+        Environment env = InternalSettingsPreparer.prepareEnvironment(Settings.builder().put(baseEnvSettings).build(), null);
+        assertEquals(".yml", env.configExtension());
+    }
+
+    public void testJsonExtension() throws IOException {
+        InputStream yaml = getClass().getResourceAsStream("/config/elasticsearch.json");
+        Path config = homeDir.resolve("config");
+        Files.createDirectory(config);
+        Files.copy(yaml, config.resolve("elasticsearch.json"));
+        Environment env = InternalSettingsPreparer.prepareEnvironment(Settings.builder().put(baseEnvSettings).build(), null);
+        assertEquals(".json", env.configExtension());
+    }
+
+    public void testSecureSettings() {
+        MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("foo", "secret");
+        Settings input = Settings.builder().put(baseEnvSettings).setSecureSettings(secureSettings).build();
+        Environment env = InternalSettingsPreparer.prepareEnvironment(input, null);
+        Setting<SecureString> fakeSetting = SecureSetting.secureString("foo", null);
+        assertEquals("secret", fakeSetting.get(env.settings()).toString());
+    }
+
+    public void testDefaultPropertiesDoNothing() throws Exception {
+        Map<String, String> props = Collections.singletonMap("default.setting", "foo");
+        Environment env = InternalSettingsPreparer.prepareEnvironment(baseEnvSettings, null, props);
+        assertEquals("foo", env.settings().get("default.setting"));
+        assertNull(env.settings().get("setting"));
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/node/NodeTests.java b/core/src/test/java/org/elasticsearch/node/NodeTests.java
new file mode 100644
index 0000000000000..e99c7b9063161
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/node/NodeTests.java
@@ -0,0 +1,255 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.node;
+
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.util.LuceneTestCase;
+import org.elasticsearch.Version;
+import org.elasticsearch.bootstrap.BootstrapCheck;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.network.NetworkModule;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.BoundTransportAddress;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.InternalTestCluster;
+import org.elasticsearch.transport.MockTcpTransportPlugin;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Locale;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasToString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+
+@LuceneTestCase.SuppressFileSystems(value = "ExtrasFS")
+public class NodeTests extends ESTestCase {
+
+    public void testNodeName() throws IOException {
+        final Path tempDir = createTempDir();
+        final String name = randomBoolean() ? randomAlphaOfLength(10) : null;
+        Settings.Builder settings = Settings.builder()
+            .put(ClusterName.CLUSTER_NAME_SETTING.getKey(), InternalTestCluster.clusterName("single-node-cluster", randomLong()))
+            .put(Environment.PATH_HOME_SETTING.getKey(), tempDir)
+            .put(NetworkModule.HTTP_ENABLED.getKey(), false)
+            .put("transport.type", "mock-socket-network")
+            .put(Node.NODE_DATA_SETTING.getKey(), true);
+        if (name != null) {
+            settings.put(Node.NODE_NAME_SETTING.getKey(), name);
+        }
+        try (Node node = new MockNode(settings.build(), Collections.singleton(MockTcpTransportPlugin.class))) {
+            final Settings nodeSettings = randomBoolean() ? node.settings() : node.getEnvironment().settings();
+            if (name == null) {
+                assertThat(Node.NODE_NAME_SETTING.get(nodeSettings), equalTo(node.getNodeEnvironment().nodeId().substring(0, 7)));
+            } else {
+                assertThat(Node.NODE_NAME_SETTING.get(nodeSettings), equalTo(name));
+            }
+        }
+    }
+
+    public static class CheckPlugin extends Plugin {
+        public static final BootstrapCheck CHECK = new BootstrapCheck() {
+            @Override
+            public boolean check() {
+                return false;
+            }
+
+            @Override
+            public String errorMessage() {
+                return "boom";
+            }
+        };
+        @Override
+        public List<BootstrapCheck> getBootstrapChecks() {
+            return Collections.singletonList(CHECK);
+        }
+    }
+
+    public void testLoadPluginBootstrapChecks() throws IOException {
+        final Path tempDir = createTempDir();
+        final String name = randomBoolean() ? randomAlphaOfLength(10) : null;
+        Settings.Builder settings = Settings.builder()
+            .put(ClusterName.CLUSTER_NAME_SETTING.getKey(), InternalTestCluster.clusterName("single-node-cluster", randomLong()))
+            .put(Environment.PATH_HOME_SETTING.getKey(), tempDir)
+            .put(NetworkModule.HTTP_ENABLED.getKey(), false)
+            .put("transport.type", "mock-socket-network")
+            .put(Node.NODE_DATA_SETTING.getKey(), true);
+        if (name != null) {
+            settings.put(Node.NODE_NAME_SETTING.getKey(), name);
+        }
+        AtomicBoolean executed = new AtomicBoolean(false);
+        try (Node node = new MockNode(settings.build(), Arrays.asList(MockTcpTransportPlugin.class, CheckPlugin.class)) {
+            @Override
+            protected void validateNodeBeforeAcceptingRequests(Settings settings, BoundTransportAddress boundTransportAddress,
+                                                               List<BootstrapCheck> bootstrapChecks) throws NodeValidationException {
+                assertEquals(1, bootstrapChecks.size());
+                assertSame(CheckPlugin.CHECK, bootstrapChecks.get(0));
+                executed.set(true);
+                throw new NodeValidationException("boom");
+            }
+        }) {
+            expectThrows(NodeValidationException.class, () -> node.start());
+            assertTrue(executed.get());
+        }
+    }
+
+    public void testWarnIfPreRelease() {
+        final Logger logger = mock(Logger.class);
+
+        final int id = randomIntBetween(1, 9) * 1000000;
+        final Version releaseVersion = Version.fromId(id + 99);
+        final Version preReleaseVersion = Version.fromId(id + randomIntBetween(0, 98));
+
+        Node.warnIfPreRelease(releaseVersion, false, logger);
+        verifyNoMoreInteractions(logger);
+
+        reset(logger);
+        Node.warnIfPreRelease(releaseVersion, true, logger);
+        verify(logger).warn(
+            "version [{}] is a pre-release version of Elasticsearch and is not suitable for production", releaseVersion + "-SNAPSHOT");
+
+        reset(logger);
+        final boolean isSnapshot = randomBoolean();
+        Node.warnIfPreRelease(preReleaseVersion, isSnapshot, logger);
+        verify(logger).warn(
+            "version [{}] is a pre-release version of Elasticsearch and is not suitable for production",
+            preReleaseVersion + (isSnapshot ? "-SNAPSHOT" : ""));
+
+    }
+
+    public void testNodeAttributes() throws IOException {
+        String attr = randomAlphaOfLength(5);
+        Settings.Builder settings = baseSettings().put(Node.NODE_ATTRIBUTES.getKey() + "test_attr", attr);
+        try (Node node = new MockNode(settings.build(), Collections.singleton(MockTcpTransportPlugin.class))) {
+            final Settings nodeSettings = randomBoolean() ? node.settings() : node.getEnvironment().settings();
+            assertEquals(attr, Node.NODE_ATTRIBUTES.get(nodeSettings).getAsMap().get("test_attr"));
+        }
+
+        // leading whitespace not allowed
+        attr = " leading";
+        settings = baseSettings().put(Node.NODE_ATTRIBUTES.getKey() + "test_attr", attr);
+        try (Node node = new MockNode(settings.build(), Collections.singleton(MockTcpTransportPlugin.class))) {
+            fail("should not allow a node attribute with leading whitespace");
+        } catch (IllegalArgumentException e) {
+            assertEquals("node.attr.test_attr cannot have leading or trailing whitespace [ leading]", e.getMessage());
+        }
+
+        // trailing whitespace not allowed
+        attr = "trailing ";
+        settings = baseSettings().put(Node.NODE_ATTRIBUTES.getKey() + "test_attr", attr);
+        try (Node node = new MockNode(settings.build(), Collections.singleton(MockTcpTransportPlugin.class))) {
+            fail("should not allow a node attribute with trailing whitespace");
+        } catch (IllegalArgumentException e) {
+            assertEquals("node.attr.test_attr cannot have leading or trailing whitespace [trailing ]", e.getMessage());
+        }
+    }
+
+    public void testDefaultPathDataSet() throws IOException {
+        final Path zero = createTempDir().toAbsolutePath();
+        final Path one = createTempDir().toAbsolutePath();
+        final Path defaultPathData = createTempDir().toAbsolutePath();
+        final Settings settings = Settings.builder()
+                .put("path.home", "/home")
+                .put("path.data.0", zero)
+                .put("path.data.1", one)
+                .put("default.path.data", defaultPathData)
+                .build();
+        try (NodeEnvironment nodeEnv = new NodeEnvironment(settings, new Environment(settings))) {
+            final Path defaultPathDataWithNodesAndId = defaultPathData.resolve("nodes/0");
+            Files.createDirectories(defaultPathDataWithNodesAndId);
+            final NodeEnvironment.NodePath defaultNodePath = new NodeEnvironment.NodePath(defaultPathDataWithNodesAndId);
+            final boolean indexExists = randomBoolean();
+            final List<String> indices;
+            if (indexExists) {
+                indices = IntStream.range(0, randomIntBetween(1, 3)).mapToObj(i -> UUIDs.randomBase64UUID()).collect(Collectors.toList());
+                for (final String index : indices) {
+                    Files.createDirectories(defaultNodePath.indicesPath.resolve(index));
+                }
+            } else {
+                indices = Collections.emptyList();
+            }
+            final Logger mock = mock(Logger.class);
+            if (indexExists) {
+                final IllegalStateException e = expectThrows(
+                        IllegalStateException.class,
+                        () -> Node.checkForIndexDataInDefaultPathData(settings, nodeEnv, mock));
+                final String message = String.format(
+                        Locale.ROOT,
+                        "detected index data in default.path.data [%s] where there should not be any; check the logs for details",
+                        defaultPathData);
+                assertThat(e, hasToString(containsString(message)));
+                verify(mock)
+                        .error("detected index data in default.path.data [{}] where there should not be any", defaultNodePath.indicesPath);
+                for (final String index : indices) {
+                    verify(mock).info(
+                            "index folder [{}] in default.path.data [{}] must be moved to any of {}",
+                            index,
+                            defaultNodePath.indicesPath,
+                            Arrays.stream(nodeEnv.nodePaths()).map(np -> np.indicesPath).collect(Collectors.toList()));
+                }
+                verifyNoMoreInteractions(mock);
+            } else {
+                Node.checkForIndexDataInDefaultPathData(settings, nodeEnv, mock);
+                verifyNoMoreInteractions(mock);
+            }
+        }
+    }
+
+    public void testDefaultPathDataNotSet() throws IOException {
+        final Path zero = createTempDir().toAbsolutePath();
+        final Path one = createTempDir().toAbsolutePath();
+        final Settings settings = Settings.builder()
+                .put("path.home", "/home")
+                .put("path.data.0", zero)
+                .put("path.data.1", one)
+                .build();
+        try (NodeEnvironment nodeEnv = new NodeEnvironment(settings, new Environment(settings))) {
+            final Logger mock = mock(Logger.class);
+            Node.checkForIndexDataInDefaultPathData(settings, nodeEnv, mock);
+            verifyNoMoreInteractions(mock);
+        }
+    }
+
+    private static Settings.Builder baseSettings() {
+        final Path tempDir = createTempDir();
+        return Settings.builder()
+                .put(ClusterName.CLUSTER_NAME_SETTING.getKey(), InternalTestCluster.clusterName("single-node-cluster", randomLong()))
+                .put(Environment.PATH_HOME_SETTING.getKey(), tempDir)
+                .put(NetworkModule.HTTP_ENABLED.getKey(), false)
+                .put("transport.type", "mock-socket-network")
+                .put(Node.NODE_DATA_SETTING.getKey(), true);
+    }
+
+
+}
diff --git a/core/src/test/java/org/elasticsearch/node/internal/InternalSettingsPreparerTests.java b/core/src/test/java/org/elasticsearch/node/internal/InternalSettingsPreparerTests.java
deleted file mode 100644
index 2dc95f8e9f15b..0000000000000
--- a/core/src/test/java/org/elasticsearch/node/internal/InternalSettingsPreparerTests.java
+++ /dev/null
@@ -1,163 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.node.internal;
-
-import org.elasticsearch.cli.MockTerminal;
-import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsException;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.test.ESTestCase;
-import org.junit.After;
-import org.junit.Before;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
-
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.equalTo;
-
-public class InternalSettingsPreparerTests extends ESTestCase {
-
-    Settings baseEnvSettings;
-
-    @Before
-    public void createBaseEnvSettings() {
-        baseEnvSettings = Settings.builder()
-            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
-            .build();
-    }
-
-    @After
-    public void clearBaseEnvSettings() {
-        baseEnvSettings = null;
-    }
-
-    public void testEmptySettings() {
-        Settings settings = InternalSettingsPreparer.prepareSettings(Settings.EMPTY);
-        assertNull(settings.get("node.name")); // a name was not set
-        assertNotNull(settings.get(ClusterName.CLUSTER_NAME_SETTING.getKey())); // a cluster name was set
-        int size = settings.names().size();
-
-        Environment env = InternalSettingsPreparer.prepareEnvironment(baseEnvSettings, null);
-        settings = env.settings();
-        assertNull(settings.get("node.name")); // a name was not set
-        assertNotNull(settings.get(ClusterName.CLUSTER_NAME_SETTING.getKey())); // a cluster name was set
-        assertEquals(settings.toString(), size + 1 /* path.home is in the base settings */, settings.names().size());
-        String home = Environment.PATH_HOME_SETTING.get(baseEnvSettings);
-        String configDir = env.configFile().toString();
-        assertTrue(configDir, configDir.startsWith(home));
-    }
-
-    public void testDefaultClusterName() {
-        Settings settings = InternalSettingsPreparer.prepareSettings(Settings.EMPTY);
-        assertEquals("elasticsearch", settings.get("cluster.name"));
-        settings = InternalSettingsPreparer.prepareSettings(Settings.builder().put("cluster.name", "foobar").build());
-        assertEquals("foobar", settings.get("cluster.name"));
-    }
-
-    public void testReplacePromptPlaceholders() {
-        MockTerminal terminal = new MockTerminal();
-        terminal.addTextInput("text");
-        terminal.addSecretInput("replaced");
-
-        Settings.Builder builder = Settings.builder()
-                .put(baseEnvSettings)
-                .put("password.replace", InternalSettingsPreparer.SECRET_PROMPT_VALUE)
-                .put("dont.replace", "prompt:secret")
-                .put("dont.replace2", "_prompt:secret_")
-                .put("dont.replace3", "_prompt:text__")
-                .put("dont.replace4", "__prompt:text_")
-                .put("dont.replace5", "prompt:secret__")
-                .put("replace_me", InternalSettingsPreparer.TEXT_PROMPT_VALUE);
-        Settings settings = InternalSettingsPreparer.prepareEnvironment(builder.build(), terminal).settings();
-
-        assertThat(settings.get("password.replace"), equalTo("replaced"));
-        assertThat(settings.get("replace_me"), equalTo("text"));
-
-        // verify other values unchanged
-        assertThat(settings.get("dont.replace"), equalTo("prompt:secret"));
-        assertThat(settings.get("dont.replace2"), equalTo("_prompt:secret_"));
-        assertThat(settings.get("dont.replace3"), equalTo("_prompt:text__"));
-        assertThat(settings.get("dont.replace4"), equalTo("__prompt:text_"));
-        assertThat(settings.get("dont.replace5"), equalTo("prompt:secret__"));
-    }
-
-    public void testReplaceSecretPromptPlaceholderWithNullTerminal() {
-        Settings.Builder builder = Settings.builder()
-                .put(baseEnvSettings)
-                .put("replace_me1", InternalSettingsPreparer.SECRET_PROMPT_VALUE);
-        try {
-            InternalSettingsPreparer.prepareEnvironment(builder.build(), null);
-            fail("an exception should have been thrown since no terminal was provided!");
-        } catch (UnsupportedOperationException e) {
-            assertThat(e.getMessage(), containsString("with value [" + InternalSettingsPreparer.SECRET_PROMPT_VALUE + "]"));
-        }
-    }
-
-    public void testReplaceTextPromptPlaceholderWithNullTerminal() {
-        Settings.Builder builder = Settings.builder()
-                .put(baseEnvSettings)
-                .put("replace_me1", InternalSettingsPreparer.TEXT_PROMPT_VALUE);
-        try {
-            InternalSettingsPreparer.prepareEnvironment(builder.build(), null);
-            fail("an exception should have been thrown since no terminal was provided!");
-        } catch (UnsupportedOperationException e) {
-            assertThat(e.getMessage(), containsString("with value [" + InternalSettingsPreparer.TEXT_PROMPT_VALUE + "]"));
-        }
-    }
-
-    public void testGarbageIsNotSwallowed() throws IOException {
-        try {
-            InputStream garbage = getClass().getResourceAsStream("/config/garbage/garbage.yml");
-            Path home = createTempDir();
-            Path config = home.resolve("config");
-            Files.createDirectory(config);
-            Files.copy(garbage, config.resolve("elasticsearch.yml"));
-            InternalSettingsPreparer.prepareEnvironment(Settings.builder()
-                .put(baseEnvSettings)
-                .build(), null);
-        } catch (SettingsException e) {
-            assertEquals("Failed to load settings from [elasticsearch.yml]", e.getMessage());
-        }
-    }
-
-    public void testMultipleSettingsFileNotAllowed() throws IOException {
-        InputStream yaml = getClass().getResourceAsStream("/config/elasticsearch.yaml");
-        InputStream properties = getClass().getResourceAsStream("/config/elasticsearch.properties");
-        Path home = createTempDir();
-        Path config = home.resolve("config");
-        Files.createDirectory(config);
-        Files.copy(yaml, config.resolve("elasticsearch.yaml"));
-        Files.copy(properties, config.resolve("elasticsearch.properties"));
-
-        try {
-            InternalSettingsPreparer.prepareEnvironment(Settings.builder()
-                .put(baseEnvSettings)
-                .build(), null);
-        } catch (SettingsException e) {
-            assertTrue(e.getMessage(), e.getMessage().contains("multiple settings files found with suffixes"));
-            assertTrue(e.getMessage(), e.getMessage().contains(".yaml"));
-            assertTrue(e.getMessage(), e.getMessage().contains(".properties"));
-        }
-    }
-}
diff --git a/core/src/test/java/org/elasticsearch/node/service/NodeServiceTests.java b/core/src/test/java/org/elasticsearch/node/service/NodeServiceTests.java
new file mode 100644
index 0000000000000..e9097b81426d4
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/node/service/NodeServiceTests.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.node.service;
+
+import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequest;
+import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
+import org.elasticsearch.common.network.NetworkModule;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+
+import static org.hamcrest.Matchers.hasSize;
+
+public class NodeServiceTests extends ESSingleNodeTestCase {
+
+    @Override
+    protected Settings nodeSettings() {
+        return Settings.builder().put(NetworkModule.HTTP_ENABLED.getKey(), false).build();
+    }
+
+    public void testHttpServerDisabled() {
+        // test for a bug where if HTTP stats were requested but HTTP was disabled, NodeService would hit a NullPointerException
+        NodesStatsResponse response = client().admin().cluster().nodesStats(new NodesStatsRequest().http(true)).actionGet();
+        assertThat(response.getNodes(), hasSize(1));
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java b/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java
index d9134ba5cf348..395afed1dc593 100644
--- a/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java
+++ b/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java
@@ -113,20 +113,20 @@ private static NodeInfo createNodeInfo() {
         if (randomBoolean()) {
             int availableProcessors = randomIntBetween(1, 64);
             int allocatedProcessors = randomIntBetween(1, availableProcessors);
-            long refreshInterval = randomBoolean() ? -1 : randomPositiveLong();
-            String name = randomAsciiOfLengthBetween(3, 10);
-            String arch = randomAsciiOfLengthBetween(3, 10);
-            String version = randomAsciiOfLengthBetween(3, 10);
+            long refreshInterval = randomBoolean() ? -1 : randomNonNegativeLong();
+            String name = randomAlphaOfLengthBetween(3, 10);
+            String arch = randomAlphaOfLengthBetween(3, 10);
+            String version = randomAlphaOfLengthBetween(3, 10);
             osInfo = new OsInfo(refreshInterval, availableProcessors, allocatedProcessors, name, arch, version);
         }
-        ProcessInfo process = randomBoolean() ? null : new ProcessInfo(randomInt(), randomBoolean(), randomPositiveLong());
+        ProcessInfo process = randomBoolean() ? null : new ProcessInfo(randomInt(), randomBoolean(), randomNonNegativeLong());
         JvmInfo jvm = randomBoolean() ? null : JvmInfo.jvmInfo();
         ThreadPoolInfo threadPoolInfo = null;
         if (randomBoolean()) {
             int numThreadPools = randomIntBetween(1, 10);
             List<ThreadPool.Info> threadPoolInfos = new ArrayList<>(numThreadPools);
             for (int i = 0; i < numThreadPools; i++) {
-                threadPoolInfos.add(new ThreadPool.Info(randomAsciiOfLengthBetween(3, 10),
+                threadPoolInfos.add(new ThreadPool.Info(randomAlphaOfLengthBetween(3, 10),
                         randomFrom(ThreadPool.ThreadPoolType.values()), randomInt()));
             }
             threadPoolInfo = new ThreadPoolInfo(threadPoolInfos);
@@ -143,14 +143,14 @@ private static NodeInfo createNodeInfo() {
             int numPlugins = randomIntBetween(0, 5);
             List<PluginInfo> plugins = new ArrayList<>();
             for (int i = 0; i < numPlugins; i++) {
-                plugins.add(new PluginInfo(randomAsciiOfLengthBetween(3, 10), randomAsciiOfLengthBetween(3, 10),
-                        randomAsciiOfLengthBetween(3, 10), randomAsciiOfLengthBetween(3, 10)));
+                plugins.add(new PluginInfo(randomAlphaOfLengthBetween(3, 10), randomAlphaOfLengthBetween(3, 10),
+                        randomAlphaOfLengthBetween(3, 10), randomAlphaOfLengthBetween(3, 10), randomBoolean()));
             }
             int numModules = randomIntBetween(0, 5);
             List<PluginInfo> modules = new ArrayList<>();
             for (int i = 0; i < numModules; i++) {
-                modules.add(new PluginInfo(randomAsciiOfLengthBetween(3, 10), randomAsciiOfLengthBetween(3, 10),
-                        randomAsciiOfLengthBetween(3, 10), randomAsciiOfLengthBetween(3, 10)));
+                modules.add(new PluginInfo(randomAlphaOfLengthBetween(3, 10), randomAlphaOfLengthBetween(3, 10),
+                        randomAlphaOfLengthBetween(3, 10), randomAlphaOfLengthBetween(3, 10), randomBoolean()));
             }
             pluginsAndModules = new PluginsAndModules(plugins, modules);
         }
@@ -160,7 +160,7 @@ private static NodeInfo createNodeInfo() {
             int numProcessors = randomIntBetween(0, 5);
             List<ProcessorInfo> processors = new ArrayList<>(numProcessors);
             for (int i = 0; i < numProcessors; i++) {
-                processors.add(new ProcessorInfo(randomAsciiOfLengthBetween(3, 10)));
+                processors.add(new ProcessorInfo(randomAlphaOfLengthBetween(3, 10)));
             }
             ingestInfo = new IngestInfo(processors);
         }
diff --git a/core/src/test/java/org/elasticsearch/nodesinfo/SimpleNodesInfoIT.java b/core/src/test/java/org/elasticsearch/nodesinfo/SimpleNodesInfoIT.java
index 0916cad60d5ca..8d722346ff25c 100644
--- a/core/src/test/java/org/elasticsearch/nodesinfo/SimpleNodesInfoIT.java
+++ b/core/src/test/java/org/elasticsearch/nodesinfo/SimpleNodesInfoIT.java
@@ -43,7 +43,7 @@
 public class SimpleNodesInfoIT extends ESIntegTestCase {
 
     public void testNodesInfos() throws Exception {
-        List<String> nodesIds = internalCluster().startNodesAsync(2).get();
+        List<String> nodesIds = internalCluster().startNodes(2);
         final String node_1 = nodesIds.get(0);
         final String node_2 = nodesIds.get(1);
 
@@ -82,7 +82,7 @@ public void testNodesInfos() throws Exception {
     }
 
     public void testNodesInfosTotalIndexingBuffer() throws Exception {
-        List<String> nodesIds = internalCluster().startNodesAsync(2).get();
+        List<String> nodesIds = internalCluster().startNodes(2);
         final String node_1 = nodesIds.get(0);
         final String node_2 = nodesIds.get(1);
 
@@ -97,30 +97,29 @@ public void testNodesInfosTotalIndexingBuffer() throws Exception {
         assertThat(response.getNodes().size(), is(2));
         assertThat(response.getNodesMap().get(server1NodeId), notNullValue());
         assertNotNull(response.getNodesMap().get(server1NodeId).getTotalIndexingBuffer());
-        assertThat(response.getNodesMap().get(server1NodeId).getTotalIndexingBuffer().bytes(), greaterThan(0L));
+        assertThat(response.getNodesMap().get(server1NodeId).getTotalIndexingBuffer().getBytes(), greaterThan(0L));
 
         assertThat(response.getNodesMap().get(server2NodeId), notNullValue());
         assertNotNull(response.getNodesMap().get(server2NodeId).getTotalIndexingBuffer());
-        assertThat(response.getNodesMap().get(server2NodeId).getTotalIndexingBuffer().bytes(), greaterThan(0L));
+        assertThat(response.getNodesMap().get(server2NodeId).getTotalIndexingBuffer().getBytes(), greaterThan(0L));
 
         // again, using only the indices flag
         response = client().admin().cluster().prepareNodesInfo().clear().setIndices(true).execute().actionGet();
         assertThat(response.getNodes().size(), is(2));
         assertThat(response.getNodesMap().get(server1NodeId), notNullValue());
         assertNotNull(response.getNodesMap().get(server1NodeId).getTotalIndexingBuffer());
-        assertThat(response.getNodesMap().get(server1NodeId).getTotalIndexingBuffer().bytes(), greaterThan(0L));
+        assertThat(response.getNodesMap().get(server1NodeId).getTotalIndexingBuffer().getBytes(), greaterThan(0L));
 
         assertThat(response.getNodesMap().get(server2NodeId), notNullValue());
         assertNotNull(response.getNodesMap().get(server2NodeId).getTotalIndexingBuffer());
-        assertThat(response.getNodesMap().get(server2NodeId).getTotalIndexingBuffer().bytes(), greaterThan(0L));
+        assertThat(response.getNodesMap().get(server2NodeId).getTotalIndexingBuffer().getBytes(), greaterThan(0L));
     }
 
     public void testAllocatedProcessors() throws Exception {
-        List<String> nodesIds = internalCluster().
-                startNodesAsync(
+        List<String> nodesIds = internalCluster().startNodes(
                         Settings.builder().put(EsExecutors.PROCESSORS_SETTING.getKey(), 3).build(),
                         Settings.builder().put(EsExecutors.PROCESSORS_SETTING.getKey(), 6).build()
-                ).get();
+                );
 
         final String node_1 = nodesIds.get(0);
         final String node_2 = nodesIds.get(1);
diff --git a/core/src/test/java/org/elasticsearch/operateAllIndices/DestructiveOperationsIT.java b/core/src/test/java/org/elasticsearch/operateAllIndices/DestructiveOperationsIT.java
new file mode 100644
index 0000000000000..fea2e4699d5e5
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/operateAllIndices/DestructiveOperationsIT.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.operateAllIndices;
+
+import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
+import org.elasticsearch.action.support.DestructiveOperations;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.junit.After;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.equalTo;
+
+public class DestructiveOperationsIT extends ESIntegTestCase {
+
+    @After
+    public void afterTest() {
+        Settings settings = Settings.builder().put(DestructiveOperations.REQUIRES_NAME_SETTING.getKey(), (String)null).build();
+        assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(settings));
+    }
+
+    public void testDeleteIndexIsRejected() throws Exception {
+        Settings settings = Settings.builder()
+                .put(DestructiveOperations.REQUIRES_NAME_SETTING.getKey(), true)
+                .build();
+        assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(settings));
+
+        createIndex("index1", "1index");
+
+        // Should succeed, since no wildcards
+        assertAcked(client().admin().indices().prepareDelete("1index").get());
+
+        expectThrows(IllegalArgumentException.class, () -> client().admin().indices().prepareDelete("i*").get());
+        expectThrows(IllegalArgumentException.class, () -> client().admin().indices().prepareDelete("_all").get());
+    }
+
+    public void testDeleteIndexDefaultBehaviour() throws Exception {
+        if (randomBoolean()) {
+            Settings settings = Settings.builder()
+                    .put(DestructiveOperations.REQUIRES_NAME_SETTING.getKey(), false)
+                    .build();
+            assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(settings));
+        }
+
+        createIndex("index1", "1index");
+
+        if (randomBoolean()) {
+            assertAcked(client().admin().indices().prepareDelete("_all").get());
+        } else {
+            assertAcked(client().admin().indices().prepareDelete("*").get());
+        }
+
+        assertThat(client().admin().indices().prepareExists("_all").get().isExists(), equalTo(false));
+    }
+
+    public void testCloseIndexIsRejected() throws Exception {
+        Settings settings = Settings.builder()
+                .put(DestructiveOperations.REQUIRES_NAME_SETTING.getKey(), true)
+                .build();
+        assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(settings));
+
+        createIndex("index1", "1index");
+
+        // Should succeed, since no wildcards
+        assertAcked(client().admin().indices().prepareClose("1index").get());
+
+        expectThrows(IllegalArgumentException.class, () -> client().admin().indices().prepareClose("i*").get());
+        expectThrows(IllegalArgumentException.class, () -> client().admin().indices().prepareClose("_all").get());
+    }
+
+    public void testCloseIndexDefaultBehaviour() throws Exception {
+        if (randomBoolean()) {
+            Settings settings = Settings.builder()
+                    .put(DestructiveOperations.REQUIRES_NAME_SETTING.getKey(), false)
+                    .build();
+            assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(settings));
+        }
+
+        createIndex("index1", "1index");
+
+        if (randomBoolean()) {
+            assertAcked(client().admin().indices().prepareClose("_all").get());
+        } else {
+            assertAcked(client().admin().indices().prepareClose("*").get());
+        }
+
+        ClusterState state = client().admin().cluster().prepareState().get().getState();
+        for (ObjectObjectCursor<String, IndexMetaData> indexMetaDataObjectObjectCursor : state.getMetaData().indices()) {
+            assertEquals(IndexMetaData.State.CLOSE, indexMetaDataObjectObjectCursor.value.getState());
+        }
+    }
+
+    public void testOpenIndexIsRejected() throws Exception {
+        Settings settings = Settings.builder()
+                .put(DestructiveOperations.REQUIRES_NAME_SETTING.getKey(), true)
+                .build();
+        assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(settings));
+
+        createIndex("index1", "1index");
+        assertAcked(client().admin().indices().prepareClose("1index", "index1").get());
+
+        expectThrows(IllegalArgumentException.class, () -> client().admin().indices().prepareOpen("i*").get());
+        expectThrows(IllegalArgumentException.class, () -> client().admin().indices().prepareOpen("_all").get());
+    }
+
+    public void testOpenIndexDefaultBehaviour() throws Exception {
+        if (randomBoolean()) {
+            Settings settings = Settings.builder()
+                    .put(DestructiveOperations.REQUIRES_NAME_SETTING.getKey(), false)
+                    .build();
+            assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(settings));
+        }
+
+        createIndex("index1", "1index");
+        assertAcked(client().admin().indices().prepareClose("1index", "index1").get());
+
+        if (randomBoolean()) {
+            assertAcked(client().admin().indices().prepareOpen("_all").get());
+        } else {
+            assertAcked(client().admin().indices().prepareOpen("*").get());
+        }
+
+        ClusterState state = client().admin().cluster().prepareState().get().getState();
+        for (ObjectObjectCursor<String, IndexMetaData> indexMetaDataObjectObjectCursor : state.getMetaData().indices()) {
+            assertEquals(IndexMetaData.State.OPEN, indexMetaDataObjectObjectCursor.value.getState());
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/operateAllIndices/DestructiveOperationsIntegrationIT.java b/core/src/test/java/org/elasticsearch/operateAllIndices/DestructiveOperationsIntegrationIT.java
deleted file mode 100644
index 83f600a8c2f9f..0000000000000
--- a/core/src/test/java/org/elasticsearch/operateAllIndices/DestructiveOperationsIntegrationIT.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.operateAllIndices;
-
-import org.elasticsearch.action.support.DestructiveOperations;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.test.ESIntegTestCase;
-
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.hamcrest.Matchers.equalTo;
-
-/**
- */
-@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST)
-public class DestructiveOperationsIntegrationIT extends ESIntegTestCase {
-    // One test for test performance, since cluster scope is test
-    // The cluster scope is test b/c we can't clear cluster settings.
-    public void testDestructiveOperations() throws Exception {
-        Settings settings = Settings.builder()
-                .put(DestructiveOperations.REQUIRES_NAME_SETTING.getKey(), true)
-                .build();
-        assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(settings));
-
-        assertAcked(client().admin().indices().prepareCreate("index1").get());
-        assertAcked(client().admin().indices().prepareCreate("1index").get());
-
-        // Should succeed, since no wildcards
-        assertAcked(client().admin().indices().prepareDelete("1index").get());
-
-        try {
-            // should fail since index1 is the only index.
-            client().admin().indices().prepareDelete("i*").get();
-            fail();
-        } catch (IllegalArgumentException e) {
-        }
-
-        try {
-            client().admin().indices().prepareDelete("_all").get();
-            fail();
-        } catch (IllegalArgumentException e) {
-        }
-
-        settings = Settings.builder()
-                .put(DestructiveOperations.REQUIRES_NAME_SETTING.getKey(), false)
-                .build();
-        assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(settings));
-
-        assertAcked(client().admin().indices().prepareDelete("_all").get());
-        assertThat(client().admin().indices().prepareExists("_all").get().isExists(), equalTo(false));
-
-        // end delete index:
-        // close index:
-        settings = Settings.builder()
-                .put(DestructiveOperations.REQUIRES_NAME_SETTING.getKey(), true)
-                .build();
-        assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(settings));
-
-        assertAcked(client().admin().indices().prepareCreate("index1").get());
-        assertAcked(client().admin().indices().prepareCreate("1index").get());
-        // Should succeed, since no wildcards
-        assertAcked(client().admin().indices().prepareClose("1index").get());
-
-        try {
-            client().admin().indices().prepareClose("_all").get();
-            fail();
-        } catch (IllegalArgumentException e) {
-        }
-        try {
-            assertAcked(client().admin().indices().prepareOpen("_all").get());
-            fail();
-        } catch (IllegalArgumentException e) {
-        }
-        try {
-            client().admin().indices().prepareClose("*").get();
-            fail();
-        } catch (IllegalArgumentException e) {
-        }
-        try {
-            assertAcked(client().admin().indices().prepareOpen("*").get());
-            fail();
-        } catch (IllegalArgumentException e) {
-        }
-
-        settings = Settings.builder()
-                .put(DestructiveOperations.REQUIRES_NAME_SETTING.getKey(), false)
-                .build();
-        assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(settings));
-        assertAcked(client().admin().indices().prepareClose("_all").get());
-        assertAcked(client().admin().indices().prepareOpen("_all").get());
-
-        // end close index:
-        client().admin().indices().prepareDelete("_all").get();
-    }
-}
diff --git a/core/src/test/java/org/elasticsearch/plugins/PluginInfoTests.java b/core/src/test/java/org/elasticsearch/plugins/PluginInfoTests.java
index 4ad52be8866a8..04afdd5839181 100644
--- a/core/src/test/java/org/elasticsearch/plugins/PluginInfoTests.java
+++ b/core/src/test/java/org/elasticsearch/plugins/PluginInfoTests.java
@@ -56,7 +56,7 @@ public void testReadFromPropertiesNameMissing() throws Exception {
             PluginInfo.readFromProperties(pluginDir);
             fail("expected missing name exception");
         } catch (IllegalArgumentException e) {
-            assertTrue(e.getMessage().contains("Property [name] is missing in"));
+            assertTrue(e.getMessage().contains("property [name] is missing in"));
         }
 
         PluginTestUtil.writeProperties(pluginDir, "name", "");
@@ -64,7 +64,7 @@ public void testReadFromPropertiesNameMissing() throws Exception {
             PluginInfo.readFromProperties(pluginDir);
             fail("expected missing name exception");
         } catch (IllegalArgumentException e) {
-            assertTrue(e.getMessage().contains("Property [name] is missing in"));
+            assertTrue(e.getMessage().contains("property [name] is missing in"));
         }
     }
 
@@ -81,7 +81,8 @@ public void testReadFromPropertiesDescriptionMissing() throws Exception {
 
     public void testReadFromPropertiesVersionMissing() throws Exception {
         Path pluginDir = createTempDir().resolve("fake-plugin");
-        PluginTestUtil.writeProperties(pluginDir, "description", "fake desc", "name", "fake-plugin");
+        PluginTestUtil.writeProperties(
+                pluginDir, "description", "fake desc", "name", "fake-plugin");
         try {
             PluginInfo.readFromProperties(pluginDir);
             fail("expected missing version exception");
@@ -151,7 +152,11 @@ public void testReadFromPropertiesBadJavaVersionFormat() throws Exception {
             PluginInfo.readFromProperties(pluginDir);
             fail("expected bad java version format exception");
         } catch (IllegalStateException e) {
-            assertTrue(e.getMessage(), e.getMessage().equals("version string must be a sequence of nonnegative decimal integers separated by \".\"'s and may have leading zeros but was 1.7.0_80"));
+            assertTrue(
+                    e.getMessage(),
+                    e.getMessage().equals("version string must be a sequence of nonnegative "
+                            + "decimal integers separated by \".\"'s and may have leading zeros "
+                            + "but was 1.7.0_80"));
         }
     }
 
@@ -166,7 +171,8 @@ public void testReadFromPropertiesBogusElasticsearchVersion() throws Exception {
             PluginInfo.readFromProperties(pluginDir);
             fail("expected bogus elasticsearch version exception");
         } catch (IllegalArgumentException e) {
-            assertTrue(e.getMessage().contains("version needs to contain major, minor, and revision"));
+            assertTrue(e.getMessage().contains(
+                    "version needs to contain major, minor, and revision"));
         }
     }
 
@@ -181,7 +187,7 @@ public void testReadFromPropertiesOldElasticsearchVersion() throws Exception {
             PluginInfo.readFromProperties(pluginDir);
             fail("expected old elasticsearch version exception");
         } catch (IllegalArgumentException e) {
-            assertTrue(e.getMessage().contains("Was designed for version [2.0.0]"));
+            assertTrue(e.getMessage().contains("was designed for version [2.0.0]"));
         }
     }
 
@@ -197,17 +203,17 @@ public void testReadFromPropertiesJvmMissingClassname() throws Exception {
             PluginInfo.readFromProperties(pluginDir);
             fail("expected old elasticsearch version exception");
         } catch (IllegalArgumentException e) {
-            assertTrue(e.getMessage().contains("Property [classname] is missing"));
+            assertTrue(e.getMessage().contains("property [classname] is missing"));
         }
     }
 
     public void testPluginListSorted() {
         List<PluginInfo> plugins = new ArrayList<>();
-        plugins.add(new PluginInfo("c", "foo", "dummy", "dummyclass"));
-        plugins.add(new PluginInfo("b", "foo", "dummy", "dummyclass"));
-        plugins.add(new PluginInfo("e", "foo", "dummy", "dummyclass"));
-        plugins.add(new PluginInfo("a", "foo", "dummy", "dummyclass"));
-        plugins.add(new PluginInfo("d", "foo", "dummy", "dummyclass"));
+        plugins.add(new PluginInfo("c", "foo", "dummy", "dummyclass", randomBoolean()));
+        plugins.add(new PluginInfo("b", "foo", "dummy", "dummyclass", randomBoolean()));
+        plugins.add(new PluginInfo("e", "foo", "dummy", "dummyclass", randomBoolean()));
+        plugins.add(new PluginInfo("a", "foo", "dummy", "dummyclass", randomBoolean()));
+        plugins.add(new PluginInfo("d", "foo", "dummy", "dummyclass", randomBoolean()));
         PluginsAndModules pluginsInfo = new PluginsAndModules(plugins, Collections.emptyList());
 
 
@@ -215,4 +221,5 @@ public void testPluginListSorted() {
         List<String> names = infos.stream().map(PluginInfo::getName).collect(Collectors.toList());
         assertThat(names, contains("a", "b", "c", "d", "e"));
     }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java b/core/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java
index 38b072b6dd731..e980081479b0e 100644
--- a/core/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java
@@ -19,18 +19,24 @@
 
 package org.elasticsearch.plugins;
 
+import org.apache.lucene.util.LuceneTestCase;
+import org.elasticsearch.Version;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.IndexModule;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Locale;
 
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.common.inject.AbstractModule;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.index.IndexModule;
-import org.elasticsearch.test.ESTestCase;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.hasToString;
 
+@LuceneTestCase.SuppressFileSystems(value = "ExtrasFS")
 public class PluginsServiceTests extends ESTestCase {
     public static class AdditionalSettingsPlugin1 extends Plugin {
         @Override
@@ -45,21 +51,8 @@ public Settings additionalSettings() {
         }
     }
 
-    public static class FailOnModule extends Plugin {
-        public void onModule(BrokenModule brokenModule) {
-            throw new IllegalStateException("boom");
-        }
-    }
-
     public static class FilterablePlugin extends Plugin implements ScriptPlugin {}
 
-    public static class BrokenModule extends AbstractModule {
-
-        @Override
-        protected void configure() {
-        }
-    }
-
     static PluginsService newPluginsService(Settings settings, Class<? extends Plugin>... classpathPlugins) {
         return new PluginsService(settings, null, new Environment(settings).pluginsFile(), Arrays.asList(classpathPlugins));
     }
@@ -91,19 +84,6 @@ public void testAdditionalSettingsClash() {
         }
     }
 
-    public void testOnModuleExceptionsArePropagated() {
-        Settings settings = Settings.builder()
-                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build();
-        PluginsService service = newPluginsService(settings, FailOnModule.class);
-        try {
-            service.processModule(new BrokenModule());
-            fail("boom");
-        } catch (ElasticsearchException ex) {
-            assertEquals("failed to invoke onModule", ex.getMessage());
-            assertEquals("boom", ex.getCause().getCause().getMessage());
-        }
-    }
-
     public void testExistingPluginMissingDescriptor() throws Exception {
         Path pluginsDir = createTempDir();
         Files.createDirectory(pluginsDir.resolve("plugin-missing-descriptor"));
@@ -111,7 +91,7 @@ public void testExistingPluginMissingDescriptor() throws Exception {
             PluginsService.getPluginBundles(pluginsDir);
             fail();
         } catch (IllegalStateException e) {
-            assertTrue(e.getMessage(), e.getMessage().contains("Could not load plugin descriptor for existing plugin"));
+            assertTrue(e.getMessage(), e.getMessage().contains("Could not load plugin descriptor for existing plugin [plugin-missing-descriptor]"));
         }
     }
 
@@ -125,4 +105,50 @@ public void testFilterPlugins() {
         assertEquals(1, scriptPlugins.size());
         assertEquals(FilterablePlugin.class, scriptPlugins.get(0).getClass());
     }
+
+    public void testHiddenFiles() throws IOException {
+        final Path home = createTempDir();
+        final Settings settings =
+                Settings.builder()
+                        .put(Environment.PATH_HOME_SETTING.getKey(), home)
+                        .build();
+        final Path hidden = home.resolve("plugins").resolve(".hidden");
+        Files.createDirectories(hidden);
+        @SuppressWarnings("unchecked")
+        final IllegalStateException e = expectThrows(
+                IllegalStateException.class,
+                () -> newPluginsService(settings));
+
+        final String expected = "Could not load plugin descriptor for existing plugin [.hidden]";
+        assertThat(e, hasToString(containsString(expected)));
+    }
+
+    public void testStartupWithRemovingMarker() throws IOException {
+        final Path home = createTempDir();
+        final Settings settings =
+                Settings.builder()
+                        .put(Environment.PATH_HOME_SETTING.getKey(), home)
+                        .build();
+        final Path fake = home.resolve("plugins").resolve("fake");
+        Files.createDirectories(fake);
+        Files.createFile(fake.resolve("plugin.jar"));
+        final Path removing = home.resolve("plugins").resolve(".removing-fake");
+        Files.createFile(removing);
+        PluginTestUtil.writeProperties(
+                fake,
+                "description", "fake",
+                "name", "fake",
+                "version", "1.0.0",
+                "elasticsearch.version", Version.CURRENT.toString(),
+                "java.version", System.getProperty("java.specification.version"),
+                "classname", "Fake",
+                "has.native.controller", "false");
+        final IllegalStateException e = expectThrows(IllegalStateException.class, () -> newPluginsService(settings));
+        final String expected = String.format(
+                Locale.ROOT,
+                "found file [%s] from a failed attempt to remove the plugin [fake]; execute [elasticsearch-plugin remove fake]",
+                removing);
+        assertThat(e, hasToString(containsString(expected)));
+    }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/plugins/PluginsTests.java b/core/src/test/java/org/elasticsearch/plugins/PluginsTests.java
new file mode 100644
index 0000000000000..a26bcb1991e4f
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/plugins/PluginsTests.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.plugins;
+
+import org.apache.lucene.util.Constants;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Locale;
+
+public class PluginsTests extends ESTestCase {
+
+    public void testMakePlatformName() {
+        final String platformName = Platforms.platformName(Constants.OS_NAME, Constants.OS_ARCH);
+
+        assertFalse(platformName, platformName.isEmpty());
+        assertTrue(platformName, platformName.equals(platformName.toLowerCase(Locale.ROOT)));
+        assertTrue(platformName, platformName.indexOf("-") > 0);
+        assertTrue(platformName, platformName.indexOf("-") < platformName.length() - 1);
+        assertFalse(platformName, platformName.contains(" "));
+    }
+
+    public void testMakeSpecificPlatformNames() {
+        assertEquals("darwin-x86_64", Platforms.platformName("Mac OS X", "x86_64"));
+        assertEquals("linux-x86_64", Platforms.platformName("Linux", "amd64"));
+        assertEquals("linux-x86", Platforms.platformName("Linux", "i386"));
+        assertEquals("windows-x86_64", Platforms.platformName("Windows Server 2008 R2", "amd64"));
+        assertEquals("windows-x86", Platforms.platformName("Windows Server 2008", "x86"));
+        assertEquals("windows-x86_64", Platforms.platformName("Windows 8.1", "amd64"));
+        assertEquals("sunos-x86_64", Platforms.platformName("SunOS", "amd64"));
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/plugins/spi/NamedXContentProviderTests.java b/core/src/test/java/org/elasticsearch/plugins/spi/NamedXContentProviderTests.java
new file mode 100644
index 0000000000000..3b63d88f39298
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/plugins/spi/NamedXContentProviderTests.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.plugins.spi;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.Streams;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.search.aggregations.Aggregation;
+import org.elasticsearch.search.aggregations.pipeline.ParsedSimpleValue;
+import org.elasticsearch.search.suggest.Suggest;
+import org.elasticsearch.search.suggest.term.TermSuggestion;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.ServiceLoader;
+import java.util.function.Predicate;
+
+public class NamedXContentProviderTests extends ESTestCase {
+
+    public void testSpiFileExists() throws IOException {
+        String serviceFile = "/META-INF/services/" + NamedXContentProvider.class.getName();
+        List<String> implementations = new ArrayList<>();
+        try (InputStream input = NamedXContentProviderTests.class.getResourceAsStream(serviceFile)) {
+            Streams.readAllLines(input, implementations::add);
+        }
+
+        assertEquals(1, implementations.size());
+        assertEquals(TestNamedXContentProvider.class.getName(), implementations.get(0));
+    }
+
+    public void testNamedXContents() {
+        final List<NamedXContentRegistry.Entry> namedXContents = new ArrayList<>();
+        for (NamedXContentProvider service : ServiceLoader.load(NamedXContentProvider.class)) {
+            namedXContents.addAll(service.getNamedXContentParsers());
+        }
+
+        assertEquals(2, namedXContents.size());
+
+        List<Predicate<NamedXContentRegistry.Entry>> predicates = new ArrayList<>(2);
+        predicates.add(e -> Aggregation.class.equals(e.categoryClass) && "test_aggregation".equals(e.name.getPreferredName()));
+        predicates.add(e -> Suggest.Suggestion.class.equals(e.categoryClass) && "test_suggestion".equals(e.name.getPreferredName()));
+        predicates.forEach(predicate -> assertEquals(1, namedXContents.stream().filter(predicate).count()));
+    }
+
+    public static class TestNamedXContentProvider implements NamedXContentProvider {
+
+        public TestNamedXContentProvider() {
+        }
+
+        @Override
+        public List<NamedXContentRegistry.Entry> getNamedXContentParsers() {
+            return Arrays.asList(
+                    new NamedXContentRegistry.Entry(Aggregation.class, new ParseField("test_aggregation"),
+                            (parser, context) -> ParsedSimpleValue.fromXContent(parser, (String) context)),
+                    new NamedXContentRegistry.Entry(Suggest.Suggestion.class, new ParseField("test_suggestion"),
+                            (parser, context) -> TermSuggestion.fromXContent(parser, (String) context))
+            );
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/recovery/FullRollingRestartIT.java b/core/src/test/java/org/elasticsearch/recovery/FullRollingRestartIT.java
index 75904e69c259d..b2707d70fbeb3 100644
--- a/core/src/test/java/org/elasticsearch/recovery/FullRollingRestartIT.java
+++ b/core/src/test/java/org/elasticsearch/recovery/FullRollingRestartIT.java
@@ -75,16 +75,15 @@ public void testFullRollingRestart() throws Exception {
         }
 
         logger.info("--> now start adding nodes");
-        internalCluster().startNodesAsync(2, settings).get();
+        internalCluster().startNode(settings);
+        internalCluster().startNode(settings);
 
         // make sure the cluster state is green, and all has been recovered
         assertTimeout(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout(healthTimeout).setWaitForGreenStatus().setWaitForNoRelocatingShards(true).setWaitForNodes("3"));
 
         logger.info("--> add two more nodes");
-        internalCluster().startNodesAsync(2, settings).get();
-
-        // We now have 5 nodes
-        setMinimumMasterNodes(3);
+        internalCluster().startNode(settings);
+        internalCluster().startNode(settings);
 
         // make sure the cluster state is green, and all has been recovered
         assertTimeout(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout(healthTimeout).setWaitForGreenStatus().setWaitForNoRelocatingShards(true).setWaitForNodes("5"));
@@ -100,9 +99,6 @@ public void testFullRollingRestart() throws Exception {
         // make sure the cluster state is green, and all has been recovered
         assertTimeout(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout(healthTimeout).setWaitForGreenStatus().setWaitForNoRelocatingShards(true).setWaitForNodes("4"));
 
-        // going down to 3 nodes. note that the min_master_node may not be in effect when we shutdown the 4th
-        // node, but that's OK as it is set to 3 before.
-        setMinimumMasterNodes(2);
         internalCluster().stopRandomDataNode();
         // make sure the cluster state is green, and all has been recovered
         assertTimeout(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout(healthTimeout).setWaitForGreenStatus().setWaitForNoRelocatingShards(true).setWaitForNodes("3"));
@@ -118,8 +114,6 @@ public void testFullRollingRestart() throws Exception {
         // make sure the cluster state is green, and all has been recovered
         assertTimeout(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout(healthTimeout).setWaitForGreenStatus().setWaitForNoRelocatingShards(true).setWaitForNodes("2"));
 
-        // closing the 2nd node
-        setMinimumMasterNodes(1);
         internalCluster().stopRandomDataNode();
 
         // make sure the cluster state is yellow, and all has been recovered
@@ -135,7 +129,7 @@ public void testFullRollingRestart() throws Exception {
     public void testNoRebalanceOnRollingRestart() throws Exception {
         // see https://github.com/elastic/elasticsearch/issues/14387
         internalCluster().startMasterOnlyNode(Settings.EMPTY);
-        internalCluster().startDataOnlyNodesAsync(3).get();
+        internalCluster().startDataOnlyNodes(3);
         /**
          * We start 3 nodes and a dedicated master. Restart on of the data-nodes and ensure that we got no relocations.
          * Yet we have 6 shards 0 replica so that means if the restarting node comes back both other nodes are subject
@@ -152,7 +146,8 @@ public void testNoRebalanceOnRollingRestart() throws Exception {
         ClusterState state = client().admin().cluster().prepareState().get().getState();
         RecoveryResponse recoveryResponse = client().admin().indices().prepareRecoveries("test").get();
         for (RecoveryState recoveryState : recoveryResponse.shardRecoveryStates().get("test")) {
-            assertTrue("relocated from: " + recoveryState.getSourceNode() + " to: " + recoveryState.getTargetNode() + "\n" + state.prettyPrint(), recoveryState.getRecoverySource().getType() != RecoverySource.Type.PEER || recoveryState.getPrimary() == false);
+            assertTrue("relocated from: " + recoveryState.getSourceNode() + " to: " + recoveryState.getTargetNode() + "\n" + state,
+                recoveryState.getRecoverySource().getType() != RecoverySource.Type.PEER || recoveryState.getPrimary() == false);
         }
         internalCluster().restartRandomDataNode();
         ensureGreen();
@@ -160,7 +155,8 @@ public void testNoRebalanceOnRollingRestart() throws Exception {
 
         recoveryResponse = client().admin().indices().prepareRecoveries("test").get();
         for (RecoveryState recoveryState : recoveryResponse.shardRecoveryStates().get("test")) {
-           assertTrue("relocated from: " + recoveryState.getSourceNode() + " to: " + recoveryState.getTargetNode()+ "-- \nbefore: \n" + state.prettyPrint() + "\nafter: \n" + afterState.prettyPrint(), recoveryState.getRecoverySource().getType() != RecoverySource.Type.PEER || recoveryState.getPrimary() == false);
+           assertTrue("relocated from: " + recoveryState.getSourceNode() + " to: " + recoveryState.getTargetNode()+ "-- \nbefore: \n" + state,
+               recoveryState.getRecoverySource().getType() != RecoverySource.Type.PEER || recoveryState.getPrimary() == false);
         }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/recovery/RecoveriesCollectionTests.java b/core/src/test/java/org/elasticsearch/recovery/RecoveriesCollectionTests.java
index 5b1073beffaa5..bbc434120d864 100644
--- a/core/src/test/java/org/elasticsearch/recovery/RecoveriesCollectionTests.java
+++ b/core/src/test/java/org/elasticsearch/recovery/RecoveriesCollectionTests.java
@@ -24,10 +24,12 @@
 import org.elasticsearch.index.replication.ESIndexLevelReplicationTestCase;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.index.store.Store;
 import org.elasticsearch.indices.recovery.RecoveriesCollection;
 import org.elasticsearch.indices.recovery.RecoveryFailedException;
 import org.elasticsearch.indices.recovery.RecoveryState;
 import org.elasticsearch.indices.recovery.PeerRecoveryTargetService;
+import org.elasticsearch.indices.recovery.RecoveryTarget;
 
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
@@ -54,10 +56,10 @@ public void testLastAccessTimeUpdate() throws Exception {
             final RecoveriesCollection collection = new RecoveriesCollection(logger, threadPool, v -> {});
             final long recoveryId = startRecovery(collection, shards.getPrimaryNode(), shards.addReplica());
             try (RecoveriesCollection.RecoveryRef status = collection.getRecovery(recoveryId)) {
-                final long lastSeenTime = status.status().lastAccessTime();
+                final long lastSeenTime = status.target().lastAccessTime();
                 assertBusy(() -> {
                     try (RecoveriesCollection.RecoveryRef currentStatus = collection.getRecovery(recoveryId)) {
-                        assertThat("access time failed to update", lastSeenTime, lessThan(currentStatus.status().lastAccessTime()));
+                        assertThat("access time failed to update", lastSeenTime, lessThan(currentStatus.target().lastAccessTime()));
                     }
                 });
             } finally {
@@ -100,7 +102,7 @@ public void testRecoveryCancellation() throws Exception {
             final long recoveryId = startRecovery(collection, shards.getPrimaryNode(), shards.addReplica());
             final long recoveryId2 = startRecovery(collection, shards.getPrimaryNode(), shards.addReplica());
             try (RecoveriesCollection.RecoveryRef recoveryRef = collection.getRecovery(recoveryId)) {
-                ShardId shardId = recoveryRef.status().shardId();
+                ShardId shardId = recoveryRef.target().shardId();
                 assertTrue("failed to cancel recoveries", collection.cancelRecoveriesForShard(shardId, "test"));
                 assertThat("all recoveries should be cancelled", collection.size(), equalTo(0));
             } finally {
@@ -118,30 +120,30 @@ public void testResetRecovery() throws Exception {
             final RecoveriesCollection collection = new RecoveriesCollection(logger, threadPool, v -> {});
             IndexShard shard = shards.addReplica();
             final long recoveryId = startRecovery(collection, shards.getPrimaryNode(), shard);
-            try (RecoveriesCollection.RecoveryRef recovery = collection.getRecovery(recoveryId)) {
-                final int currentAsTarget = shard.recoveryStats().currentAsTarget();
-                final int referencesToStore = recovery.status().store().refCount();
-                String tempFileName = recovery.status().getTempNameForFile("foobar");
-                collection.resetRecovery(recoveryId, recovery.status().shardId());
-                try (RecoveriesCollection.RecoveryRef resetRecovery = collection.getRecovery(recoveryId)) {
-                    assertNotSame(recovery.status(), resetRecovery);
-                    assertSame(recovery.status().CancellableThreads(), resetRecovery.status().CancellableThreads());
-                    assertSame(recovery.status().indexShard(), resetRecovery.status().indexShard());
-                    assertSame(recovery.status().store(), resetRecovery.status().store());
-                    assertEquals(referencesToStore + 1, resetRecovery.status().store().refCount());
-                    assertEquals(currentAsTarget+1, shard.recoveryStats().currentAsTarget()); // we blink for a short moment...
-                    recovery.close();
-                    expectThrows(ElasticsearchException.class, () -> recovery.status().store());
-                    assertEquals(referencesToStore, resetRecovery.status().store().refCount());
-                    String resetTempFileName = resetRecovery.status().getTempNameForFile("foobar");
-                    assertNotEquals(tempFileName, resetTempFileName);
-                }
-                assertEquals(currentAsTarget, shard.recoveryStats().currentAsTarget());
-            }
-            try (RecoveriesCollection.RecoveryRef resetRecovery = collection.getRecovery(recoveryId)) {
+            RecoveryTarget recoveryTarget = collection.getRecoveryTarget(recoveryId);
+            final int currentAsTarget = shard.recoveryStats().currentAsTarget();
+            final int referencesToStore = recoveryTarget.store().refCount();
+            IndexShard indexShard = recoveryTarget.indexShard();
+            Store store = recoveryTarget.store();
+            String tempFileName = recoveryTarget.getTempNameForFile("foobar");
+            RecoveryTarget resetRecovery = collection.resetRecovery(recoveryId, TimeValue.timeValueMinutes(60));
+            final long resetRecoveryId = resetRecovery.recoveryId();
+            assertNotSame(recoveryTarget, resetRecovery);
+            assertNotSame(recoveryTarget.CancellableThreads(), resetRecovery.CancellableThreads());
+            assertSame(indexShard, resetRecovery.indexShard());
+            assertSame(store, resetRecovery.store());
+            assertEquals(referencesToStore, resetRecovery.store().refCount());
+            assertEquals(currentAsTarget, shard.recoveryStats().currentAsTarget());
+            assertEquals(recoveryTarget.refCount(), 0);
+            expectThrows(ElasticsearchException.class, () -> recoveryTarget.store());
+            expectThrows(ElasticsearchException.class, () -> recoveryTarget.indexShard());
+            String resetTempFileName = resetRecovery.getTempNameForFile("foobar");
+            assertNotEquals(tempFileName, resetTempFileName);
+            assertEquals(currentAsTarget, shard.recoveryStats().currentAsTarget());
+            try (RecoveriesCollection.RecoveryRef newRecoveryRef = collection.getRecovery(resetRecoveryId)) {
                 shards.recoverReplica(shard, (s, n) -> {
-                    assertSame(s, resetRecovery.status().indexShard());
-                    return resetRecovery.status();
+                    assertSame(s, newRecoveryRef.target().indexShard());
+                    return newRecoveryRef.target();
                 }, false);
             }
             shards.assertAllEqual(numDocs);
diff --git a/core/src/test/java/org/elasticsearch/recovery/RecoverySettingsTests.java b/core/src/test/java/org/elasticsearch/recovery/RecoverySettingsTests.java
deleted file mode 100644
index 26c22fc3bb03f..0000000000000
--- a/core/src/test/java/org/elasticsearch/recovery/RecoverySettingsTests.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.recovery;
-
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.ByteSizeUnit;
-import org.elasticsearch.indices.recovery.RecoverySettings;
-import org.elasticsearch.test.ESSingleNodeTestCase;
-
-import java.util.concurrent.TimeUnit;
-
-public class RecoverySettingsTests extends ESSingleNodeTestCase {
-    @Override
-    protected boolean resetNodeAfterTest() {
-        return true;
-    }
-
-    public void testAllSettingsAreDynamicallyUpdatable() {
-        innerTestSettings(RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey(), 0, new Validator() {
-            @Override
-            public void validate(RecoverySettings recoverySettings, int expectedValue) {
-                assertEquals(null, recoverySettings.rateLimiter());
-            }
-        });
-        innerTestSettings(RecoverySettings.INDICES_RECOVERY_RETRY_DELAY_STATE_SYNC_SETTING.getKey(), randomIntBetween(1, 200), TimeUnit.MILLISECONDS, new Validator() {
-            @Override
-            public void validate(RecoverySettings recoverySettings, int expectedValue) {
-                assertEquals(expectedValue, recoverySettings.retryDelayStateSync().millis());
-            }
-        });
-        innerTestSettings(RecoverySettings.INDICES_RECOVERY_RETRY_DELAY_NETWORK_SETTING.getKey(), randomIntBetween(1, 200), TimeUnit.MILLISECONDS, new Validator() {
-            @Override
-            public void validate(RecoverySettings recoverySettings, int expectedValue) {
-                assertEquals(expectedValue, recoverySettings.retryDelayNetwork().millis());
-            }
-        });
-        innerTestSettings(RecoverySettings.INDICES_RECOVERY_ACTIVITY_TIMEOUT_SETTING.getKey(), randomIntBetween(1, 200), TimeUnit.MILLISECONDS, new Validator() {
-            @Override
-            public void validate(RecoverySettings recoverySettings, int expectedValue) {
-                assertEquals(expectedValue, recoverySettings.activityTimeout().millis());
-            }
-        });
-        innerTestSettings(RecoverySettings.INDICES_RECOVERY_INTERNAL_ACTION_TIMEOUT_SETTING.getKey(), randomIntBetween(1, 200), TimeUnit.MILLISECONDS, new Validator() {
-            @Override
-            public void validate(RecoverySettings recoverySettings, int expectedValue) {
-                assertEquals(expectedValue, recoverySettings.internalActionTimeout().millis());
-            }
-        });
-        innerTestSettings(RecoverySettings.INDICES_RECOVERY_INTERNAL_LONG_ACTION_TIMEOUT_SETTING.getKey(), randomIntBetween(1, 200), TimeUnit.MILLISECONDS, new Validator() {
-            @Override
-            public void validate(RecoverySettings recoverySettings, int expectedValue) {
-                assertEquals(expectedValue, recoverySettings.internalActionLongTimeout().millis());
-            }
-        });
-    }
-
-    private static class Validator {
-        public void validate(RecoverySettings recoverySettings, int expectedValue) {
-        }
-
-        public void validate(RecoverySettings recoverySettings, boolean expectedValue) {
-        }
-    }
-
-    private void innerTestSettings(String key, int newValue, Validator validator) {
-        client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put(key, newValue)).get();
-        validator.validate(getInstanceFromNode(RecoverySettings.class), newValue);
-    }
-
-    private void innerTestSettings(String key, int newValue, TimeUnit timeUnit, Validator validator) {
-        client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put(key, newValue, timeUnit)).get();
-        validator.validate(getInstanceFromNode(RecoverySettings.class), newValue);
-    }
-
-    private void innerTestSettings(String key, int newValue, ByteSizeUnit byteSizeUnit, Validator validator) {
-        client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put(key, newValue, byteSizeUnit)).get();
-        validator.validate(getInstanceFromNode(RecoverySettings.class), newValue);
-    }
-
-    private void innerTestSettings(String key, boolean newValue, Validator validator) {
-        client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put(key, newValue)).get();
-        validator.validate(getInstanceFromNode(RecoverySettings.class), newValue);
-    }
-
-}
diff --git a/core/src/test/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java b/core/src/test/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java
index 71c96b85fd568..36d9373b0835c 100644
--- a/core/src/test/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java
+++ b/core/src/test/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java
@@ -53,7 +53,7 @@
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoTimeout;
 
-@TestLogging("_root:DEBUG,index.shard:TRACE")
+@TestLogging("_root:DEBUG,org.elasticsearch.index.shard:TRACE")
 public class RecoveryWhileUnderLoadIT extends ESIntegTestCase {
     private final Logger logger = Loggers.getLogger(RecoveryWhileUnderLoadIT.class);
 
@@ -341,12 +341,9 @@ private void logSearchResponse(int numberOfShards, long numberOfDocs, int iterat
     }
 
     private void refreshAndAssert() throws Exception {
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                RefreshResponse actionGet = client().admin().indices().prepareRefresh().get();
-                assertAllSuccessful(actionGet);
-            }
+        assertBusy(() -> {
+            RefreshResponse actionGet = client().admin().indices().prepareRefresh().get();
+            assertAllSuccessful(actionGet);
         }, 5, TimeUnit.MINUTES);
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java b/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java
index 8493a08d70446..6dc2330ebee55 100644
--- a/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java
+++ b/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java
@@ -39,13 +39,14 @@
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.shard.IndexEventListener;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.IndexShardState;
 import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.indices.recovery.RecoveryFileChunkRequest;
 import org.elasticsearch.indices.recovery.PeerRecoveryTargetService;
+import org.elasticsearch.indices.recovery.RecoveryFileChunkRequest;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHits;
@@ -53,12 +54,10 @@
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
-import org.elasticsearch.test.InternalTestCluster;
 import org.elasticsearch.test.MockIndexEventListener;
 import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.transport.Transport;
-import org.elasticsearch.transport.TransportException;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportService;
@@ -77,6 +76,7 @@
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Stream;
 
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@@ -90,7 +90,7 @@
 /**
  */
 @ClusterScope(scope = Scope.TEST, numDataNodes = 0)
-@TestLogging("_root:DEBUG,indices.recovery:TRACE,index.shard.service:TRACE")
+@TestLogging("_root:DEBUG,org.elasticsearch.indices.recovery:TRACE,org.elasticsearch.index.shard.service:TRACE")
 public class RelocationIT extends ESIntegTestCase {
     private final TimeValue ACCEPTABLE_RELOCATION_TIME = new TimeValue(5, TimeUnit.MINUTES);
 
@@ -302,12 +302,12 @@ public void indexShardStateChanged(IndexShard indexShard, @Nullable IndexShardSt
 
             List<IndexRequestBuilder> builders1 = new ArrayList<>();
             for (int numDocs = randomIntBetween(10, 30); numDocs > 0; numDocs--) {
-                builders1.add(client().prepareIndex("test", "type").setSource("{}"));
+                builders1.add(client().prepareIndex("test", "type").setSource("{}", XContentType.JSON));
             }
 
             List<IndexRequestBuilder> builders2 = new ArrayList<>();
             for (int numDocs = randomIntBetween(10, 30); numDocs > 0; numDocs--) {
-                builders2.add(client().prepareIndex("test", "type").setSource("{}"));
+                builders2.add(client().prepareIndex("test", "type").setSource("{}", XContentType.JSON));
             }
 
             logger.info("--> START relocate the shard from {} to {}", nodes[fromNode], nodes[toNode]);
@@ -353,12 +353,13 @@ public void testCancellationCleansTempFiles() throws Exception {
         client().admin().indices().prepareCreate(indexName)
                 .setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1, IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)).get();
 
-        internalCluster().startNodesAsync(2).get();
+        internalCluster().startNode();
+        internalCluster().startNode();
 
         List<IndexRequestBuilder> requests = new ArrayList<>();
         int numDocs = scaledRandomIntBetween(25, 250);
         for (int i = 0; i < numDocs; i++) {
-            requests.add(client().prepareIndex(indexName, "type").setSource("{}"));
+            requests.add(client().prepareIndex(indexName, "type").setSource("{}", XContentType.JSON));
         }
         indexRandom(true, requests);
         assertFalse(client().admin().cluster().prepareHealth().setWaitForNodes("3").setWaitForGreenStatus().get().isTimedOut());
@@ -384,17 +385,14 @@ public void testCancellationCleansTempFiles() throws Exception {
                 .setTransientSettings(Settings.builder().put(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), "none")));
 
         logger.info("--> wait for all replica shards to be removed, on all nodes");
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                for (String node : internalCluster().getNodeNames()) {
-                    if (node.equals(p_node)) {
-                        continue;
-                    }
-                    ClusterState state = client(node).admin().cluster().prepareState().setLocal(true).get().getState();
-                    assertThat(node + " indicates assigned replicas",
-                            state.getRoutingTable().index(indexName).shardsWithState(ShardRoutingState.UNASSIGNED).size(), equalTo(1));
+        assertBusy(() -> {
+            for (String node : internalCluster().getNodeNames()) {
+                if (node.equals(p_node)) {
+                    continue;
                 }
+                ClusterState state = client(node).admin().cluster().prepareState().setLocal(true).get().getState();
+                assertThat(node + " indicates assigned replicas",
+                        state.getRoutingTable().index(indexName).shardsWithState(ShardRoutingState.UNASSIGNED).size(), equalTo(1));
             }
         });
 
@@ -403,20 +401,17 @@ public void run() {
             NodeEnvironment nodeEnvironment = internalCluster().getInstance(NodeEnvironment.class, node);
             for (final Path shardLoc : nodeEnvironment.availableShardPaths(new ShardId(indexName, "_na_", 0))) {
                 if (Files.exists(shardLoc)) {
-                    assertBusy(new Runnable() {
-                        @Override
-                        public void run() {
-                            try {
-                                Files.walkFileTree(shardLoc, new SimpleFileVisitor<Path>() {
-                                    @Override
-                                    public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
-                                        assertThat("found a temporary recovery file: " + file, file.getFileName().toString(), not(startsWith("recovery.")));
-                                        return FileVisitResult.CONTINUE;
-                                    }
-                                });
-                            } catch (IOException e) {
-                                throw new AssertionError("failed to walk file tree starting at [" + shardLoc + "]", e);
-                            }
+                    assertBusy(() -> {
+                        try {
+                            Files.walkFileTree(shardLoc, new SimpleFileVisitor<Path>() {
+                                @Override
+                                public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+                                    assertThat("found a temporary recovery file: " + file, file.getFileName().toString(), not(startsWith("recovery.")));
+                                    return FileVisitResult.CONTINUE;
+                                }
+                            });
+                        } catch (IOException e) {
+                            throw new AssertionError("failed to walk file tree starting at [" + shardLoc + "]", e);
                         }
                     });
                 }
@@ -426,14 +421,15 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO
 
     public void testIndexAndRelocateConcurrently() throws ExecutionException, InterruptedException {
         int halfNodes = randomIntBetween(1, 3);
-        Settings blueSetting = Settings.builder().put("node.attr.color", "blue").build();
-        InternalTestCluster.Async<List<String>> blueFuture = internalCluster().startNodesAsync(halfNodes, blueSetting);
-        Settings redSetting = Settings.builder().put("node.attr.color", "red").build();
-        InternalTestCluster.Async<java.util.List<String>> redFuture = internalCluster().startNodesAsync(halfNodes, redSetting);
-        blueFuture.get();
-        redFuture.get();
-        logger.info("blue nodes: {}", blueFuture.get());
-        logger.info("red nodes: {}", redFuture.get());
+        Settings[] nodeSettings = Stream.concat(
+            Stream.generate(() -> Settings.builder().put("node.attr.color", "blue").build()).limit(halfNodes),
+            Stream.generate(() -> Settings.builder().put("node.attr.color", "red").build()).limit(halfNodes)
+            ).toArray(Settings[]::new);
+        List<String> nodes = internalCluster().startNodes(nodeSettings);
+        String[] blueNodes = nodes.subList(0, halfNodes).stream().toArray(String[]::new);
+        String[] redNodes = nodes.subList(halfNodes, nodes.size()).stream().toArray(String[]::new);
+        logger.info("blue nodes: {}", (Object)blueNodes);
+        logger.info("red nodes: {}", (Object)redNodes);
         ensureStableCluster(halfNodes * 2);
 
         assertAcked(prepareCreate("test").setSettings(Settings.builder()
@@ -441,7 +437,7 @@ public void testIndexAndRelocateConcurrently() throws ExecutionException, Interr
             .put(indexSettings())
             .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, randomInt(halfNodes - 1))
         ));
-        assertAllShardsOnNodes("test", redFuture.get().toArray(new String[2]));
+        assertAllShardsOnNodes("test", redNodes);
         int numDocs = randomIntBetween(100, 150);
         ArrayList<String> ids = new ArrayList<>();
         logger.info(" --> indexing [{}] docs", numDocs);
@@ -486,26 +482,26 @@ class RecoveryCorruption extends MockTransportService.DelegateTransport {
 
         private final CountDownLatch corruptionCount;
 
-        public RecoveryCorruption(Transport transport, CountDownLatch corruptionCount) {
+        RecoveryCorruption(Transport transport, CountDownLatch corruptionCount) {
             super(transport);
             this.corruptionCount = corruptionCount;
         }
 
         @Override
-        public void sendRequest(DiscoveryNode node, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException, TransportException {
+        protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException {
             if (action.equals(PeerRecoveryTargetService.Actions.FILE_CHUNK)) {
                 RecoveryFileChunkRequest chunkRequest = (RecoveryFileChunkRequest) request;
                 if (chunkRequest.name().startsWith(IndexFileNames.SEGMENTS)) {
                     // corrupting the segments_N files in order to make sure future recovery re-send files
-                    logger.debug("corrupting [{}] to {}. file name: [{}]", action, node, chunkRequest.name());
+                    logger.debug("corrupting [{}] to {}. file name: [{}]", action, connection.getNode(), chunkRequest.name());
                     assert chunkRequest.content().toBytesRef().bytes == chunkRequest.content().toBytesRef().bytes : "no internal reference!!";
                     byte[] array = chunkRequest.content().toBytesRef().bytes;
                     array[0] = (byte) ~array[0]; // flip one byte in the content
                     corruptionCount.countDown();
                 }
-                transport.sendRequest(node, requestId, action, request, options);
+                super.sendRequest(connection, requestId, action, request, options);
             } else {
-                transport.sendRequest(node, requestId, action, request, options);
+                super.sendRequest(connection, requestId, action, request, options);
             }
         }
     }
diff --git a/core/src/test/java/org/elasticsearch/recovery/SimpleRecoveryIT.java b/core/src/test/java/org/elasticsearch/recovery/SimpleRecoveryIT.java
index e860cfc9ebd54..bf8bce8ae6334 100644
--- a/core/src/test/java/org/elasticsearch/recovery/SimpleRecoveryIT.java
+++ b/core/src/test/java/org/elasticsearch/recovery/SimpleRecoveryIT.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESIntegTestCase;
 
 import static org.elasticsearch.client.Requests.flushRequest;
@@ -52,12 +53,12 @@ public void testSimpleRecovery() throws Exception {
 
         NumShards numShards = getNumShards("test");
 
-        client().index(indexRequest("test").type("type1").id("1").source(source("1", "test"))).actionGet();
+        client().index(indexRequest("test").type("type1").id("1").source(source("1", "test"), XContentType.JSON)).actionGet();
         FlushResponse flushResponse = client().admin().indices().flush(flushRequest("test")).actionGet();
         assertThat(flushResponse.getTotalShards(), equalTo(numShards.totalNumShards));
         assertThat(flushResponse.getSuccessfulShards(), equalTo(numShards.numPrimaries));
         assertThat(flushResponse.getFailedShards(), equalTo(0));
-        client().index(indexRequest("test").type("type1").id("2").source(source("2", "test"))).actionGet();
+        client().index(indexRequest("test").type("type1").id("2").source(source("2", "test"), XContentType.JSON)).actionGet();
         RefreshResponse refreshResponse = client().admin().indices().refresh(refreshRequest("test")).actionGet();
         assertThat(refreshResponse.getTotalShards(), equalTo(numShards.totalNumShards));
         assertThat(refreshResponse.getSuccessfulShards(), equalTo(numShards.numPrimaries));
diff --git a/core/src/test/java/org/elasticsearch/recovery/TruncatedRecoveryIT.java b/core/src/test/java/org/elasticsearch/recovery/TruncatedRecoveryIT.java
index b261928d041be..564eca54d8e6c 100644
--- a/core/src/test/java/org/elasticsearch/recovery/TruncatedRecoveryIT.java
+++ b/core/src/test/java/org/elasticsearch/recovery/TruncatedRecoveryIT.java
@@ -30,13 +30,14 @@
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.indices.recovery.IndexRecoveryIT;
-import org.elasticsearch.indices.recovery.RecoveryFileChunkRequest;
-import org.elasticsearch.indices.recovery.RecoverySettings;
 import org.elasticsearch.indices.recovery.PeerRecoveryTargetService;
+import org.elasticsearch.indices.recovery.RecoveryFileChunkRequest;
+import org.elasticsearch.node.RecoverySettingsChunkSizePlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.transport.MockTransportService;
+import org.elasticsearch.transport.ConnectionProfile;
+import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportException;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.transport.TransportRequestOptions;
@@ -51,6 +52,7 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import static org.elasticsearch.node.RecoverySettingsChunkSizePlugin.CHUNK_SIZE_SETTING;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
@@ -61,7 +63,7 @@ public class TruncatedRecoveryIT extends ESIntegTestCase {
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Arrays.asList(MockTransportService.TestPlugin.class);
+        return Arrays.asList(MockTransportService.TestPlugin.class, RecoverySettingsChunkSizePlugin.class);
     }
 
     /**
@@ -71,9 +73,9 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
      * Later we allow full recovery to ensure we can still recover and don't run into corruptions.
      */
     public void testCancelRecoveryAndResume() throws Exception {
-        for(RecoverySettings settings : internalCluster().getInstances(RecoverySettings.class)) {
-            IndexRecoveryIT.setChunkSize(settings, new ByteSizeValue(randomIntBetween(50, 300), ByteSizeUnit.BYTES));
-        }
+        assertTrue(client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder()
+                .put(CHUNK_SIZE_SETTING.getKey(), new ByteSizeValue(randomIntBetween(50, 300), ByteSizeUnit.BYTES)))
+                .get().isAcknowledged());
 
         NodesStatsResponse nodeStats = client().admin().cluster().prepareNodesStats().get();
         List<NodeStats> dataNodeStats = new ArrayList<>();
@@ -111,7 +113,7 @@ public void testCancelRecoveryAndResume() throws Exception {
         }
         ensureGreen();
         // ensure we have flushed segments and make them a big one via optimize
-        client().admin().indices().prepareFlush().setForce(true).setWaitIfOngoing(true).get();
+        client().admin().indices().prepareFlush().setForce(true).get();
         client().admin().indices().prepareForceMerge().setMaxNumSegments(1).setFlush(true).get();
 
         final CountDownLatch latch = new CountDownLatch(1);
@@ -121,7 +123,8 @@ public void testCancelRecoveryAndResume() throws Exception {
             mockTransportService.addDelegate(internalCluster().getInstance(TransportService.class, unluckyNode.getNode().getName()), new MockTransportService.DelegateTransport(mockTransportService.original()) {
 
                 @Override
-                public void sendRequest(DiscoveryNode node, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException, TransportException {
+                protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request,
+                                           TransportRequestOptions options) throws IOException {
                     if (action.equals(PeerRecoveryTargetService.Actions.FILE_CHUNK)) {
                         RecoveryFileChunkRequest req = (RecoveryFileChunkRequest) request;
                         logger.debug("file chunk [{}] lastChunk: {}", req, req.lastChunk());
@@ -130,7 +133,7 @@ public void sendRequest(DiscoveryNode node, long requestId, String action, Trans
                             throw new RuntimeException("Caused some truncated files for fun and profit");
                         }
                     }
-                    super.sendRequest(node, requestId, action, request, options);
+                    super.sendRequest(connection, requestId, action, request, options);
                 }
             });
         }
diff --git a/core/src/test/java/org/elasticsearch/repositories/IndexIdTests.java b/core/src/test/java/org/elasticsearch/repositories/IndexIdTests.java
index 30002d54a6bf7..51167d862fdfd 100644
--- a/core/src/test/java/org/elasticsearch/repositories/IndexIdTests.java
+++ b/core/src/test/java/org/elasticsearch/repositories/IndexIdTests.java
@@ -24,7 +24,6 @@
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.test.ESTestCase;
 
@@ -37,7 +36,7 @@ public class IndexIdTests extends ESTestCase {
 
     public void testEqualsAndHashCode() {
         // assert equals and hashcode
-        String name = randomAsciiOfLength(8);
+        String name = randomAlphaOfLength(8);
         String id = UUIDs.randomBase64UUID();
         IndexId indexId1 = new IndexId(name, id);
         IndexId indexId2 = new IndexId(name, id);
@@ -50,7 +49,7 @@ public void testEqualsAndHashCode() {
         assertEquals(indexId1, indexId2);
         assertEquals(indexId1.hashCode(), indexId2.hashCode());
         //assert not equals when name or id differ
-        indexId2 = new IndexId(randomAsciiOfLength(8), id);
+        indexId2 = new IndexId(randomAlphaOfLength(8), id);
         assertNotEquals(indexId1, indexId2);
         assertNotEquals(indexId1.hashCode(), indexId2.hashCode());
         indexId2 = new IndexId(name, UUIDs.randomBase64UUID());
@@ -59,17 +58,17 @@ public void testEqualsAndHashCode() {
     }
 
     public void testSerialization() throws IOException {
-        IndexId indexId = new IndexId(randomAsciiOfLength(8), UUIDs.randomBase64UUID());
+        IndexId indexId = new IndexId(randomAlphaOfLength(8), UUIDs.randomBase64UUID());
         BytesStreamOutput out = new BytesStreamOutput();
         indexId.writeTo(out);
         assertEquals(indexId, new IndexId(out.bytes().streamInput()));
     }
 
     public void testXContent() throws IOException {
-        IndexId indexId = new IndexId(randomAsciiOfLength(8), UUIDs.randomBase64UUID());
+        IndexId indexId = new IndexId(randomAlphaOfLength(8), UUIDs.randomBase64UUID());
         XContentBuilder builder = JsonXContent.contentBuilder();
         indexId.toXContent(builder, ToXContent.EMPTY_PARAMS);
-        XContentParser parser = XContentType.JSON.xContent().createParser(builder.bytes());
+        XContentParser parser = createParser(JsonXContent.jsonXContent, builder.bytes());
         assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
         String name = null;
         String id = null;
diff --git a/core/src/test/java/org/elasticsearch/repositories/RepositoryDataTests.java b/core/src/test/java/org/elasticsearch/repositories/RepositoryDataTests.java
index 1fb34249fd268..40ff1bad9767f 100644
--- a/core/src/test/java/org/elasticsearch/repositories/RepositoryDataTests.java
+++ b/core/src/test/java/org/elasticsearch/repositories/RepositoryDataTests.java
@@ -23,9 +23,9 @@
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.snapshots.SnapshotId;
+import org.elasticsearch.snapshots.SnapshotState;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
@@ -38,6 +38,7 @@
 import java.util.Map;
 import java.util.Set;
 
+import static org.elasticsearch.repositories.RepositoryData.EMPTY_REPO_GEN;
 import static org.hamcrest.Matchers.greaterThan;
 
 /**
@@ -55,25 +56,25 @@ public void testEqualsAndHashCode() {
     public void testXContent() throws IOException {
         RepositoryData repositoryData = generateRandomRepoData();
         XContentBuilder builder = JsonXContent.contentBuilder();
-        repositoryData.toXContent(builder, ToXContent.EMPTY_PARAMS);
-        XContentParser parser = XContentType.JSON.xContent().createParser(builder.bytes());
-        assertEquals(repositoryData, RepositoryData.fromXContent(parser));
+        repositoryData.snapshotsToXContent(builder, ToXContent.EMPTY_PARAMS);
+        XContentParser parser = createParser(JsonXContent.jsonXContent, builder.bytes());
+        long gen = (long) randomIntBetween(0, 500);
+        RepositoryData fromXContent = RepositoryData.snapshotsFromXContent(parser, gen);
+        assertEquals(repositoryData, fromXContent);
+        assertEquals(gen, fromXContent.getGenId());
     }
 
     public void testAddSnapshots() {
         RepositoryData repositoryData = generateRandomRepoData();
         // test that adding the same snapshot id to the repository data throws an exception
-        final SnapshotId snapshotId = repositoryData.getSnapshotIds().get(0);
         Map<String, IndexId> indexIdMap = repositoryData.getIndices();
-        expectThrows(IllegalArgumentException.class,
-            () -> repositoryData.addSnapshot(new SnapshotId(snapshotId.getName(), snapshotId.getUUID()), Collections.emptyList()));
         // test that adding a snapshot and its indices works
-        SnapshotId newSnapshot = new SnapshotId(randomAsciiOfLength(7), UUIDs.randomBase64UUID());
+        SnapshotId newSnapshot = new SnapshotId(randomAlphaOfLength(7), UUIDs.randomBase64UUID());
         List<IndexId> indices = new ArrayList<>();
         Set<IndexId> newIndices = new HashSet<>();
         int numNew = randomIntBetween(1, 10);
         for (int i = 0; i < numNew; i++) {
-            IndexId indexId = new IndexId(randomAsciiOfLength(7), UUIDs.randomBase64UUID());
+            IndexId indexId = new IndexId(randomAlphaOfLength(7), UUIDs.randomBase64UUID());
             newIndices.add(indexId);
             indices.add(indexId);
         }
@@ -82,7 +83,8 @@ public void testAddSnapshots() {
         for (int i = 0; i < numOld; i++) {
             indices.add(indexIdMap.get(indexNames.get(i)));
         }
-        RepositoryData newRepoData = repositoryData.addSnapshot(newSnapshot, indices);
+        RepositoryData newRepoData = repositoryData.addSnapshot(newSnapshot,
+            randomFrom(SnapshotState.SUCCESS, SnapshotState.PARTIAL, SnapshotState.FAILED), indices);
         // verify that the new repository data has the new snapshot and its indices
         assertTrue(newRepoData.getSnapshotIds().contains(newSnapshot));
         for (IndexId indexId : indices) {
@@ -92,19 +94,26 @@ public void testAddSnapshots() {
                 assertEquals(snapshotIds.size(), 1); // if it was a new index, only the new snapshot should be in its set
             }
         }
+        assertEquals(repositoryData.getGenId(), newRepoData.getGenId());
     }
 
     public void testInitIndices() {
         final int numSnapshots = randomIntBetween(1, 30);
-        final List<SnapshotId> snapshotIds = new ArrayList<>(numSnapshots);
+        final Map<String, SnapshotId> snapshotIds = new HashMap<>(numSnapshots);
         for (int i = 0; i < numSnapshots; i++) {
-            snapshotIds.add(new SnapshotId(randomAsciiOfLength(8), UUIDs.randomBase64UUID()));
+            final SnapshotId snapshotId = new SnapshotId(randomAlphaOfLength(8), UUIDs.randomBase64UUID());
+            snapshotIds.put(snapshotId.getUUID(), snapshotId);
         }
-        RepositoryData repositoryData = new RepositoryData(snapshotIds, Collections.emptyMap());
+        RepositoryData repositoryData = new RepositoryData(EMPTY_REPO_GEN, snapshotIds,
+            Collections.emptyMap(), Collections.emptyMap(), Collections.emptyList());
         // test that initializing indices works
         Map<IndexId, Set<SnapshotId>> indices = randomIndices(snapshotIds);
         RepositoryData newRepoData = repositoryData.initIndices(indices);
-        assertEquals(repositoryData.getSnapshotIds(), newRepoData.getSnapshotIds());
+        List<SnapshotId> expected = new ArrayList<>(repositoryData.getSnapshotIds());
+        Collections.sort(expected);
+        List<SnapshotId> actual = new ArrayList<>(newRepoData.getSnapshotIds());
+        Collections.sort(actual);
+        assertEquals(expected, actual);
         for (IndexId indexId : indices.keySet()) {
             assertEquals(indices.get(indexId), newRepoData.getSnapshots(indexId));
         }
@@ -130,35 +139,42 @@ public void testResolveIndexId() {
         String indexName = indexNames.iterator().next();
         IndexId indexId = indices.get(indexName);
         assertEquals(indexId, repositoryData.resolveIndexId(indexName));
-        String notInRepoData = randomAsciiOfLength(5);
+        String notInRepoData = randomAlphaOfLength(5);
         assertFalse(indexName.contains(notInRepoData));
         assertEquals(new IndexId(notInRepoData, notInRepoData), repositoryData.resolveIndexId(notInRepoData));
     }
 
-    public static RepositoryData generateRandomRepoData() {
-        return generateRandomRepoData(new ArrayList<>());
-    }
-
-    public static RepositoryData generateRandomRepoData(final List<SnapshotId> origSnapshotIds) {
-        List<SnapshotId> snapshotIds = randomSnapshots(origSnapshotIds);
-        return new RepositoryData(snapshotIds, randomIndices(snapshotIds));
+    public void testGetSnapshotState() {
+        final SnapshotId snapshotId = new SnapshotId(randomAlphaOfLength(8), UUIDs.randomBase64UUID());
+        final SnapshotState state = randomFrom(SnapshotState.values());
+        final RepositoryData repositoryData = RepositoryData.EMPTY.addSnapshot(snapshotId, state, Collections.emptyList());
+        assertEquals(state, repositoryData.getSnapshotState(snapshotId));
+        assertNull(repositoryData.getSnapshotState(new SnapshotId(randomAlphaOfLength(8), UUIDs.randomBase64UUID())));
     }
 
-    private static List<SnapshotId> randomSnapshots(final List<SnapshotId> origSnapshotIds) {
+    public static RepositoryData generateRandomRepoData() {
+        final int numIndices = randomIntBetween(1, 30);
+        final List<IndexId> indices = new ArrayList<>(numIndices);
+        for (int i = 0; i < numIndices; i++) {
+            indices.add(new IndexId(randomAlphaOfLength(8), UUIDs.randomBase64UUID()));
+        }
         final int numSnapshots = randomIntBetween(1, 30);
-        final List<SnapshotId> snapshotIds = new ArrayList<>(origSnapshotIds);
+        RepositoryData repositoryData = RepositoryData.EMPTY;
         for (int i = 0; i < numSnapshots; i++) {
-            snapshotIds.add(new SnapshotId(randomAsciiOfLength(8), UUIDs.randomBase64UUID()));
+            final SnapshotId snapshotId = new SnapshotId(randomAlphaOfLength(8), UUIDs.randomBase64UUID());
+            final List<IndexId> someIndices = indices.subList(0, randomIntBetween(1, numIndices));
+            repositoryData = repositoryData.addSnapshot(snapshotId, randomFrom(SnapshotState.values()), someIndices);
         }
-        return snapshotIds;
+        return repositoryData;
     }
 
-    private static Map<IndexId, Set<SnapshotId>> randomIndices(final List<SnapshotId> snapshotIds) {
+    private static Map<IndexId, Set<SnapshotId>> randomIndices(final Map<String, SnapshotId> snapshotIdsMap) {
+        final List<SnapshotId> snapshotIds = new ArrayList<>(snapshotIdsMap.values());
         final int totalSnapshots = snapshotIds.size();
         final int numIndices = randomIntBetween(1, 30);
         final Map<IndexId, Set<SnapshotId>> indices = new HashMap<>(numIndices);
         for (int i = 0; i < numIndices; i++) {
-            final IndexId indexId = new IndexId(randomAsciiOfLength(8), UUIDs.randomBase64UUID());
+            final IndexId indexId = new IndexId(randomAlphaOfLength(8), UUIDs.randomBase64UUID());
             final Set<SnapshotId> indexSnapshots = new LinkedHashSet<>();
             final int numIndicesForSnapshot = randomIntBetween(1, numIndices);
             for (int j = 0; j < numIndicesForSnapshot; j++) {
diff --git a/core/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java b/core/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java
index e8408f37b3749..7e4d5cc54a900 100644
--- a/core/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java
+++ b/core/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java
@@ -24,12 +24,16 @@
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.repositories.IndexId;
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.repositories.RepositoryData;
+import org.elasticsearch.repositories.RepositoryException;
 import org.elasticsearch.snapshots.SnapshotId;
+import org.elasticsearch.snapshots.SnapshotState;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
+import java.io.IOException;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -67,7 +71,7 @@ public void testRetrieveSnapshots() throws Exception {
             String id = Integer.toString(i);
             client().prepareIndex(indexName, "type1", id).setSource("text", "sometext").get();
         }
-        client().admin().indices().prepareFlush(indexName).setWaitIfOngoing(true).get();
+        client().admin().indices().prepareFlush(indexName).get();
 
         logger.info("--> create first snapshot");
         CreateSnapshotResponse createSnapshotResponse = client.admin()
@@ -93,9 +97,8 @@ public void testRetrieveSnapshots() throws Exception {
             (BlobStoreRepository) repositoriesService.repository(repositoryName);
         final List<SnapshotId> originalSnapshots = Arrays.asList(snapshotId1, snapshotId2);
 
-        List<SnapshotId> snapshotIds = repository.getSnapshots().stream()
-                                                             .sorted((s1, s2) -> s1.getName().compareTo(s2.getName()))
-                                                             .collect(Collectors.toList());
+        List<SnapshotId> snapshotIds = repository.getRepositoryData().getSnapshotIds().stream()
+            .sorted((s1, s2) -> s1.getName().compareTo(s2.getName())).collect(Collectors.toList());
         assertThat(snapshotIds, equalTo(originalSnapshots));
     }
 
@@ -103,28 +106,24 @@ public void testReadAndWriteSnapshotsThroughIndexFile() throws Exception {
         final BlobStoreRepository repository = setupRepo();
 
         // write to and read from a index file with no entries
-        assertThat(repository.getSnapshots().size(), equalTo(0));
+        assertThat(repository.getRepositoryData().getSnapshotIds().size(), equalTo(0));
         final RepositoryData emptyData = RepositoryData.EMPTY;
-        repository.writeIndexGen(emptyData);
-        final RepositoryData readData = repository.getRepositoryData();
-        assertEquals(readData, emptyData);
-        assertEquals(readData.getIndices().size(), 0);
-        assertEquals(readData.getSnapshotIds().size(), 0);
+        repository.writeIndexGen(emptyData, emptyData.getGenId());
+        RepositoryData repoData = repository.getRepositoryData();
+        assertEquals(repoData, emptyData);
+        assertEquals(repoData.getIndices().size(), 0);
+        assertEquals(repoData.getSnapshotIds().size(), 0);
+        assertEquals(0L, repoData.getGenId());
 
         // write to and read from an index file with snapshots but no indices
-        final int numSnapshots = randomIntBetween(1, 20);
-        final List<SnapshotId> snapshotIds = new ArrayList<>(numSnapshots);
-        for (int i = 0; i < numSnapshots; i++) {
-            snapshotIds.add(new SnapshotId(randomAsciiOfLength(8), UUIDs.randomBase64UUID()));
-        }
-        RepositoryData repositoryData = new RepositoryData(snapshotIds, Collections.emptyMap());
-        repository.writeIndexGen(repositoryData);
-        assertEquals(repository.getRepositoryData(), repositoryData);
+        repoData = addRandomSnapshotsToRepoData(repoData, false);
+        repository.writeIndexGen(repoData, repoData.getGenId());
+        assertEquals(repoData, repository.getRepositoryData());
 
         // write to and read from a index file with random repository data
-        repositoryData = generateRandomRepoData();
-        repository.writeIndexGen(repositoryData);
-        assertThat(repository.getRepositoryData(), equalTo(repositoryData));
+        repoData = addRandomSnapshotsToRepoData(repository.getRepositoryData(), true);
+        repository.writeIndexGen(repoData, repoData.getGenId());
+        assertEquals(repoData, repository.getRepositoryData());
     }
 
     public void testIndexGenerationalFiles() throws Exception {
@@ -132,26 +131,77 @@ public void testIndexGenerationalFiles() throws Exception {
 
         // write to index generational file
         RepositoryData repositoryData = generateRandomRepoData();
-        repository.writeIndexGen(repositoryData);
+        repository.writeIndexGen(repositoryData, repositoryData.getGenId());
         assertThat(repository.getRepositoryData(), equalTo(repositoryData));
         assertThat(repository.latestIndexBlobId(), equalTo(0L));
         assertThat(repository.readSnapshotIndexLatestBlob(), equalTo(0L));
 
         // adding more and writing to a new index generational file
-        repositoryData = generateRandomRepoData();
-        repository.writeIndexGen(repositoryData);
+        repositoryData = addRandomSnapshotsToRepoData(repository.getRepositoryData(), true);
+        repository.writeIndexGen(repositoryData, repositoryData.getGenId());
         assertEquals(repository.getRepositoryData(), repositoryData);
         assertThat(repository.latestIndexBlobId(), equalTo(1L));
         assertThat(repository.readSnapshotIndexLatestBlob(), equalTo(1L));
 
         // removing a snapshot and writing to a new index generational file
-        repositoryData = repositoryData.removeSnapshot(repositoryData.getSnapshotIds().get(0));
-        repository.writeIndexGen(repositoryData);
+        repositoryData = repository.getRepositoryData().removeSnapshot(repositoryData.getSnapshotIds().iterator().next());
+        repository.writeIndexGen(repositoryData, repositoryData.getGenId());
         assertEquals(repository.getRepositoryData(), repositoryData);
         assertThat(repository.latestIndexBlobId(), equalTo(2L));
         assertThat(repository.readSnapshotIndexLatestBlob(), equalTo(2L));
     }
 
+    public void testRepositoryDataConcurrentModificationNotAllowed() throws IOException {
+        final BlobStoreRepository repository = setupRepo();
+
+        // write to index generational file
+        RepositoryData repositoryData = generateRandomRepoData();
+        repository.writeIndexGen(repositoryData, repositoryData.getGenId());
+
+        // write repo data again to index generational file, errors because we already wrote to the
+        // N+1 generation from which this repository data instance was created
+        expectThrows(RepositoryException.class, () -> repository.writeIndexGen(repositoryData, repositoryData.getGenId()));
+    }
+
+    public void testReadAndWriteIncompatibleSnapshots() throws Exception {
+        final BlobStoreRepository repository = setupRepo();
+
+        // write to and read from incompatible snapshots file with no entries
+        assertEquals(0, repository.getRepositoryData().getIncompatibleSnapshotIds().size());
+        RepositoryData emptyData = RepositoryData.EMPTY;
+        repository.writeIndexGen(emptyData, emptyData.getGenId());
+        repository.writeIncompatibleSnapshots(emptyData);
+        RepositoryData readData = repository.getRepositoryData();
+        assertEquals(emptyData, readData);
+        assertEquals(0, readData.getIndices().size());
+        assertEquals(0, readData.getSnapshotIds().size());
+
+        // write to and read from incompatible snapshots with some number of entries
+        final int numSnapshots = randomIntBetween(1, 20);
+        final List<SnapshotId> snapshotIds = new ArrayList<>(numSnapshots);
+        for (int i = 0; i < numSnapshots; i++) {
+            snapshotIds.add(new SnapshotId(randomAlphaOfLength(8), UUIDs.randomBase64UUID()));
+        }
+        RepositoryData repositoryData = new RepositoryData(readData.getGenId(),
+            Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), snapshotIds);
+        repository.blobContainer().deleteBlob("incompatible-snapshots");
+        repository.writeIncompatibleSnapshots(repositoryData);
+        readData = repository.getRepositoryData();
+        assertEquals(repositoryData.getIncompatibleSnapshotIds(), readData.getIncompatibleSnapshotIds());
+    }
+
+    public void testIncompatibleSnapshotsBlobExists() throws Exception {
+        final BlobStoreRepository repository = setupRepo();
+        RepositoryData emptyData = RepositoryData.EMPTY;
+        repository.writeIndexGen(emptyData, emptyData.getGenId());
+        RepositoryData repoData = repository.getRepositoryData();
+        assertEquals(emptyData, repoData);
+        assertTrue(repository.blobContainer().blobExists("incompatible-snapshots"));
+        repoData = addRandomSnapshotsToRepoData(repository.getRepositoryData(), true);
+        repository.writeIndexGen(repoData, repoData.getGenId());
+        assertEquals(0, repository.getRepositoryData().getIncompatibleSnapshotIds().size());
+    }
+
     private BlobStoreRepository setupRepo() {
         final Client client = client();
         final Path location = ESIntegTestCase.randomRepoPath(node().settings());
@@ -170,4 +220,19 @@ private BlobStoreRepository setupRepo() {
         return repository;
     }
 
+    private RepositoryData addRandomSnapshotsToRepoData(RepositoryData repoData, boolean inclIndices) {
+        int numSnapshots = randomIntBetween(1, 20);
+        for (int i = 0; i < numSnapshots; i++) {
+            SnapshotId snapshotId = new SnapshotId(randomAlphaOfLength(8), UUIDs.randomBase64UUID());
+            int numIndices = inclIndices ? randomIntBetween(0, 20) : 0;
+            List<IndexId> indexIds = new ArrayList<>(numIndices);
+            for (int j = 0; j < numIndices; j++) {
+                indexIds.add(new IndexId(randomAlphaOfLength(8), UUIDs.randomBase64UUID()));
+            }
+            repoData = repoData.addSnapshot(snapshotId,
+                randomFrom(SnapshotState.SUCCESS, SnapshotState.PARTIAL, SnapshotState.FAILED), indexIds);
+        }
+        return repoData;
+    }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/rest/BaseRestHandlerTests.java b/core/src/test/java/org/elasticsearch/rest/BaseRestHandlerTests.java
new file mode 100644
index 0000000000000..732614112bd99
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/rest/BaseRestHandlerTests.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.rest;
+
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.Table;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.rest.action.cat.AbstractCatAction;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.rest.FakeRestChannel;
+import org.elasticsearch.test.rest.FakeRestRequest;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.hamcrest.core.StringContains.containsString;
+import static org.hamcrest.object.HasToString.hasToString;
+import static org.mockito.Mockito.mock;
+
+public class BaseRestHandlerTests extends ESTestCase {
+
+    public void testOneUnconsumedParameters() throws Exception {
+        final AtomicBoolean executed = new AtomicBoolean();
+        BaseRestHandler handler = new BaseRestHandler(Settings.EMPTY) {
+            @Override
+            protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+                request.param("consumed");
+                return channel -> executed.set(true);
+            }
+        };
+
+        final HashMap<String, String> params = new HashMap<>();
+        params.put("consumed", randomAlphaOfLength(8));
+        params.put("unconsumed", randomAlphaOfLength(8));
+        RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withParams(params).build();
+        RestChannel channel = new FakeRestChannel(request, randomBoolean(), 1);
+        final IllegalArgumentException e =
+            expectThrows(IllegalArgumentException.class, () -> handler.handleRequest(request, channel, mock(NodeClient.class)));
+        assertThat(e, hasToString(containsString("request [/] contains unrecognized parameter: [unconsumed]")));
+        assertFalse(executed.get());
+    }
+
+    public void testMultipleUnconsumedParameters() throws Exception {
+        final AtomicBoolean executed = new AtomicBoolean();
+        BaseRestHandler handler = new BaseRestHandler(Settings.EMPTY) {
+            @Override
+            protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+                request.param("consumed");
+                return channel -> executed.set(true);
+            }
+        };
+
+        final HashMap<String, String> params = new HashMap<>();
+        params.put("consumed", randomAlphaOfLength(8));
+        params.put("unconsumed-first", randomAlphaOfLength(8));
+        params.put("unconsumed-second", randomAlphaOfLength(8));
+        RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withParams(params).build();
+        RestChannel channel = new FakeRestChannel(request, randomBoolean(), 1);
+        final IllegalArgumentException e =
+            expectThrows(IllegalArgumentException.class, () -> handler.handleRequest(request, channel, mock(NodeClient.class)));
+        assertThat(e, hasToString(containsString("request [/] contains unrecognized parameters: [unconsumed-first], [unconsumed-second]")));
+        assertFalse(executed.get());
+    }
+
+    public void testUnconsumedParametersDidYouMean() throws Exception {
+        final AtomicBoolean executed = new AtomicBoolean();
+        BaseRestHandler handler = new BaseRestHandler(Settings.EMPTY) {
+            @Override
+            protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+                request.param("consumed");
+                request.param("field");
+                request.param("tokenizer");
+                request.param("very_close_to_parameter_1");
+                request.param("very_close_to_parameter_2");
+                return channel -> executed.set(true);
+            }
+
+            @Override
+            protected Set<String> responseParams() {
+                return Collections.singleton("response_param");
+            }
+        };
+
+        final HashMap<String, String> params = new HashMap<>();
+        params.put("consumed", randomAlphaOfLength(8));
+        params.put("flied", randomAlphaOfLength(8));
+        params.put("respones_param", randomAlphaOfLength(8));
+        params.put("tokenzier", randomAlphaOfLength(8));
+        params.put("very_close_to_parametre", randomAlphaOfLength(8));
+        params.put("very_far_from_every_consumed_parameter", randomAlphaOfLength(8));
+        RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withParams(params).build();
+        RestChannel channel = new FakeRestChannel(request, randomBoolean(), 1);
+        final IllegalArgumentException e =
+            expectThrows(IllegalArgumentException.class, () -> handler.handleRequest(request, channel, mock(NodeClient.class)));
+        assertThat(
+            e,
+            hasToString(containsString(
+                "request [/] contains unrecognized parameters: " +
+                    "[flied] -> did you mean [field]?, " +
+                    "[respones_param] -> did you mean [response_param]?, " +
+                    "[tokenzier] -> did you mean [tokenizer]?, " +
+                    "[very_close_to_parametre] -> did you mean any of [very_close_to_parameter_1, very_close_to_parameter_2]?, " +
+                    "[very_far_from_every_consumed_parameter]")));
+        assertFalse(executed.get());
+    }
+
+    public void testUnconsumedResponseParameters() throws Exception {
+        final AtomicBoolean executed = new AtomicBoolean();
+        BaseRestHandler handler = new BaseRestHandler(Settings.EMPTY) {
+            @Override
+            protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+                request.param("consumed");
+                return channel -> executed.set(true);
+            }
+
+            @Override
+            protected Set<String> responseParams() {
+                return Collections.singleton("response_param");
+            }
+        };
+
+        final HashMap<String, String> params = new HashMap<>();
+        params.put("consumed", randomAlphaOfLength(8));
+        params.put("response_param", randomAlphaOfLength(8));
+        RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withParams(params).build();
+        RestChannel channel = new FakeRestChannel(request, randomBoolean(), 1);
+        handler.handleRequest(request, channel, mock(NodeClient.class));
+        assertTrue(executed.get());
+    }
+
+    public void testDefaultResponseParameters() throws Exception {
+        final AtomicBoolean executed = new AtomicBoolean();
+        BaseRestHandler handler = new BaseRestHandler(Settings.EMPTY) {
+            @Override
+            protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+                return channel -> executed.set(true);
+            }
+        };
+
+        final HashMap<String, String> params = new HashMap<>();
+        params.put("format", randomAlphaOfLength(8));
+        params.put("filter_path", randomAlphaOfLength(8));
+        params.put("pretty", randomFrom("true", "false", "", null));
+        params.put("human", null);
+        RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withParams(params).build();
+        RestChannel channel = new FakeRestChannel(request, randomBoolean(), 1);
+        handler.handleRequest(request, channel, mock(NodeClient.class));
+        assertTrue(executed.get());
+    }
+
+    public void testCatResponseParameters() throws Exception {
+        final AtomicBoolean executed = new AtomicBoolean();
+        AbstractCatAction handler = new AbstractCatAction(Settings.EMPTY) {
+            @Override
+            protected RestChannelConsumer doCatRequest(RestRequest request, NodeClient client) {
+                return channel -> executed.set(true);
+            }
+
+            @Override
+            protected void documentation(StringBuilder sb) {
+
+            }
+
+            @Override
+            protected Table getTableWithHeader(RestRequest request) {
+                return null;
+            }
+        };
+
+        final HashMap<String, String> params = new HashMap<>();
+        params.put("format", randomAlphaOfLength(8));
+        params.put("h", randomAlphaOfLength(8));
+        params.put("v", randomAlphaOfLength(8));
+        params.put("ts", randomAlphaOfLength(8));
+        params.put("pri", randomAlphaOfLength(8));
+        params.put("bytes", randomAlphaOfLength(8));
+        params.put("size", randomAlphaOfLength(8));
+        params.put("time", randomAlphaOfLength(8));
+        RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withParams(params).build();
+        RestChannel channel = new FakeRestChannel(request, randomBoolean(), 1);
+        handler.handleRequest(request, channel, mock(NodeClient.class));
+        assertTrue(executed.get());
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java b/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java
index b9f97156df373..04f112f842af3 100644
--- a/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java
+++ b/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java
@@ -20,11 +20,20 @@
 package org.elasticsearch.rest;
 
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.ResourceAlreadyExistsException;
+import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.transport.LocalTransportAddress;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.test.ESTestCase;
@@ -36,23 +45,18 @@
 import java.util.Collections;
 import java.util.Map;
 
+import static org.elasticsearch.ElasticsearchExceptionTests.assertDeepEquals;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
 
 public class BytesRestResponseTests extends ESTestCase {
 
     class UnknownException extends Exception {
-
-        public UnknownException(final String message, final Throwable cause) {
+        UnknownException(final String message, final Throwable cause) {
             super(message, cause);
         }
-
     }
 
     public void testWithHeaders() throws Exception {
@@ -60,6 +64,7 @@ public void testWithHeaders() throws Exception {
         RestChannel channel = randomBoolean() ? new DetailedExceptionRestChannel(request) : new SimpleExceptionRestChannel(request);
 
         BytesRestResponse response = new BytesRestResponse(channel, new WithHeadersException());
+        assertEquals(2, response.getHeaders().size());
         assertThat(response.getHeaders().get("n1"), notNullValue());
         assertThat(response.getHeaders().get("n1"), contains("v11", "v12"));
         assertThat(response.getHeaders().get("n2"), notNullValue());
@@ -147,9 +152,9 @@ public void testConvert() throws IOException {
         RestRequest request = new FakeRestRequest();
         RestChannel channel = new DetailedExceptionRestChannel(request);
         ShardSearchFailure failure = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null),
-                new SearchShardTarget("node_1", new Index("foo", "_na_"), 1));
+                new SearchShardTarget("node_1", new Index("foo", "_na_"), 1, null));
         ShardSearchFailure failure1 = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null),
-                new SearchShardTarget("node_1", new Index("foo", "_na_"), 2));
+                new SearchShardTarget("node_1", new Index("foo", "_na_"), 2, null));
         SearchPhaseExecutionException ex = new SearchPhaseExecutionException("search", "all shards failed",  new ShardSearchFailure[] {failure, failure1});
         BytesRestResponse response = new BytesRestResponse(channel, new RemoteTransportException("foo", ex));
         String text = response.content().utf8ToString();
@@ -161,7 +166,7 @@ public void testConvert() throws IOException {
 
     public void testResponseWhenPathContainsEncodingError() throws IOException {
         final String path = "%a";
-        final RestRequest request = new RestRequest(Collections.emptyMap(), path) {
+        final RestRequest request = new RestRequest(NamedXContentRegistry.EMPTY, Collections.emptyMap(), path, Collections.emptyMap()) {
             @Override
             public Method method() {
                 return null;
@@ -181,16 +186,6 @@ public boolean hasContent() {
             public BytesReference content() {
                 return null;
             }
-
-            @Override
-            public String header(String name) {
-                return null;
-            }
-
-            @Override
-            public Iterable<Map.Entry<String, String>> headers() {
-                return null;
-            }
         };
         final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> RestUtils.decodeComponent(request.rawPath()));
         final RestChannel channel = new DetailedExceptionRestChannel(request);
@@ -214,12 +209,151 @@ public void testResponseWhenInternalServerError() throws IOException {
         assertThat(content, containsString("\"status\":" + 500));
     }
 
+    public void testErrorToAndFromXContent() throws IOException {
+        final boolean detailed = randomBoolean();
+
+        Exception original;
+        ElasticsearchException cause = null;
+        String reason;
+        String type = "exception";
+        RestStatus status = RestStatus.INTERNAL_SERVER_ERROR;
+        boolean addHeadersOrMetadata = false;
+
+        switch (randomIntBetween(0, 5)) {
+            case 0:
+                original = new ElasticsearchException("ElasticsearchException without cause");
+                if (detailed) {
+                    addHeadersOrMetadata = randomBoolean();
+                    reason = "ElasticsearchException without cause";
+                } else {
+                    reason = "ElasticsearchException[ElasticsearchException without cause]";
+                }
+                break;
+            case 1:
+                original = new ElasticsearchException("ElasticsearchException with a cause", new FileNotFoundException("missing"));
+                if (detailed) {
+                    addHeadersOrMetadata = randomBoolean();
+                    type = "exception";
+                    reason = "ElasticsearchException with a cause";
+                    cause = new ElasticsearchException("Elasticsearch exception [type=file_not_found_exception, reason=missing]");
+                } else {
+                    reason = "ElasticsearchException[ElasticsearchException with a cause]";
+                }
+                break;
+            case 2:
+                original = new ResourceNotFoundException("ElasticsearchException with custom status");
+                status = RestStatus.NOT_FOUND;
+                if (detailed) {
+                    addHeadersOrMetadata = randomBoolean();
+                    type = "resource_not_found_exception";
+                    reason = "ElasticsearchException with custom status";
+                } else {
+                    reason = "ResourceNotFoundException[ElasticsearchException with custom status]";
+                }
+                break;
+            case 3:
+                TransportAddress address = LocalTransportAddress.buildUnique();
+                original = new RemoteTransportException("remote", address, "action",
+                        new ResourceAlreadyExistsException("ElasticsearchWrapperException with a cause that has a custom status"));
+                status = RestStatus.BAD_REQUEST;
+                if (detailed) {
+                    type = "resource_already_exists_exception";
+                    reason = "ElasticsearchWrapperException with a cause that has a custom status";
+                } else {
+                    reason = "RemoteTransportException[[remote][" + address.toString() + "][action]]";
+                }
+                break;
+            case 4:
+                original = new RemoteTransportException("ElasticsearchWrapperException with a cause that has a special treatment",
+                        new IllegalArgumentException("wrong"));
+                status = RestStatus.BAD_REQUEST;
+                if (detailed) {
+                    type = "illegal_argument_exception";
+                    reason = "wrong";
+                } else {
+                    reason = "RemoteTransportException[[ElasticsearchWrapperException with a cause that has a special treatment]]";
+                }
+                break;
+            case 5:
+                status = randomFrom(RestStatus.values());
+                original = new ElasticsearchStatusException("ElasticsearchStatusException with random status", status);
+                if (detailed) {
+                    addHeadersOrMetadata = randomBoolean();
+                    type = "status_exception";
+                    reason = "ElasticsearchStatusException with random status";
+                } else {
+                    reason = "ElasticsearchStatusException[ElasticsearchStatusException with random status]";
+                }
+                break;
+            default:
+                throw new UnsupportedOperationException("Failed to generate random exception");
+        }
+
+        String message = "Elasticsearch exception [type=" + type + ", reason=" + reason + "]";
+        ElasticsearchStatusException expected = new ElasticsearchStatusException(message, status, cause);
+
+        if (addHeadersOrMetadata) {
+            ElasticsearchException originalException = ((ElasticsearchException) original);
+            if (randomBoolean()) {
+                originalException.addHeader("foo", "bar", "baz");
+                expected.addHeader("foo", "bar", "baz");
+            }
+            if (randomBoolean()) {
+                originalException.addMetadata("es.metadata_0", "0");
+                expected.addMetadata("es.metadata_0", "0");
+            }
+            if (randomBoolean()) {
+                String resourceType = randomAlphaOfLength(5);
+                String resourceId = randomAlphaOfLength(5);
+                originalException.setResources(resourceType, resourceId);
+                expected.setResources(resourceType, resourceId);
+            }
+            if (randomBoolean()) {
+                originalException.setIndex("_index");
+                expected.setIndex("_index");
+            }
+        }
+
+        final XContentType xContentType = randomFrom(XContentType.values());
+
+        Map<String, String> params = Collections.singletonMap("format", xContentType.mediaType());
+        RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withParams(params).build();
+        RestChannel channel = detailed ? new DetailedExceptionRestChannel(request) : new SimpleExceptionRestChannel(request);
+
+        BytesRestResponse response = new BytesRestResponse(channel, original);
+
+        ElasticsearchException parsedError;
+        try (XContentParser parser = createParser(xContentType.xContent(), response.content())) {
+            parsedError = BytesRestResponse.errorFromXContent(parser);
+            assertNull(parser.nextToken());
+        }
+
+        assertEquals(expected.status(), parsedError.status());
+        assertDeepEquals(expected, parsedError);
+    }
+
+    public void testNoErrorFromXContent() throws IOException {
+        IllegalStateException e = expectThrows(IllegalStateException.class, () -> {
+            try (XContentBuilder builder = XContentBuilder.builder(randomFrom(XContentType.values()).xContent())) {
+                builder.startObject();
+                builder.field("status", randomFrom(RestStatus.values()).getStatus());
+                builder.endObject();
+
+                try (XContentParser parser = createParser(builder.contentType().xContent(), builder.bytes())) {
+                    BytesRestResponse.errorFromXContent(parser);
+                }
+            }
+        });
+        assertEquals("Failed to parse elasticsearch status exception: no exception was found", e.getMessage());
+    }
+
     public static class WithHeadersException extends ElasticsearchException {
 
         WithHeadersException() {
             super("");
             this.addHeader("n1", "v11", "v12");
             this.addHeader("n2", "v21", "v22");
+            this.addMetadata("es.test", "value1", "value2");
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/rest/DeprecationRestHandlerTests.java b/core/src/test/java/org/elasticsearch/rest/DeprecationRestHandlerTests.java
index be0f3b15115b9..0a8ace71d8860 100644
--- a/core/src/test/java/org/elasticsearch/rest/DeprecationRestHandlerTests.java
+++ b/core/src/test/java/org/elasticsearch/rest/DeprecationRestHandlerTests.java
@@ -38,7 +38,7 @@ public class DeprecationRestHandlerTests extends ESTestCase {
     /**
      * Note: Headers should only use US ASCII (and this inevitably becomes one!).
      */
-    private final String deprecationMessage = randomAsciiOfLengthBetween(1, 30);
+    private final String deprecationMessage = randomAlphaOfLengthBetween(1, 30);
     private final DeprecationLogger deprecationLogger = mock(DeprecationLogger.class);
 
     public void testNullHandler() {
@@ -78,8 +78,15 @@ public void testValidHeaderValue() {
         ASCIIHeaderGenerator generator = new ASCIIHeaderGenerator();
         String value = generator.ofCodeUnitsLength(random(), 1, 50);
 
-        assertTrue(DeprecationRestHandler.validHeaderValue(value));
-        assertSame(value, DeprecationRestHandler.requireValidHeader(value));
+        if (value.trim().length() == 0) {
+            // empty text, not a valid header
+            assertFalse(DeprecationRestHandler.validHeaderValue(value));
+            Exception e = expectThrows(IllegalArgumentException.class, () -> DeprecationRestHandler.requireValidHeader(value));
+            assertEquals("header value must contain only US ASCII text", e.getMessage());
+        } else {
+            assertTrue(DeprecationRestHandler.validHeaderValue(value));
+            assertSame(value, DeprecationRestHandler.requireValidHeader(value));
+        }
     }
 
     public void testInvalidHeaderValue() {
@@ -131,7 +138,7 @@ static char[] asciiFromTo(int from, int to) {
         /**
          * Create a generator for characters [32, 126].
          */
-        public ASCIIHeaderGenerator() {
+        ASCIIHeaderGenerator() {
             super(asciiFromTo(32, 126));
         }
     }
diff --git a/core/src/test/java/org/elasticsearch/rest/RestControllerTests.java b/core/src/test/java/org/elasticsearch/rest/RestControllerTests.java
index dca0a16f0f085..24483dbe18438 100644
--- a/core/src/test/java/org/elasticsearch/rest/RestControllerTests.java
+++ b/core/src/test/java/org/elasticsearch/rest/RestControllerTests.java
@@ -19,20 +19,45 @@
 
 package org.elasticsearch.rest;
 
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
 import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.breaker.CircuitBreaker;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.BoundTransportAddress;
+import org.elasticsearch.common.transport.LocalTransportAddress;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.yaml.YamlXContent;
+import org.elasticsearch.http.HttpInfo;
+import org.elasticsearch.http.HttpServerTransport;
+import org.elasticsearch.http.HttpStats;
+import org.elasticsearch.http.HttpTransportSettings;
+import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.rest.FakeRestRequest;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.UnaryOperator;
 
+import static org.hamcrest.Matchers.containsString;
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.doCallRealMethod;
@@ -41,52 +66,75 @@
 
 public class RestControllerTests extends ESTestCase {
 
+    private static final ByteSizeValue BREAKER_LIMIT = new ByteSizeValue(20);
+    private CircuitBreaker inFlightRequestsBreaker;
+    private RestController restController;
+    private HierarchyCircuitBreakerService circuitBreakerService;
+
+    @Before
+    public void setup() {
+        Settings settings = Settings.EMPTY;
+        circuitBreakerService = new HierarchyCircuitBreakerService(
+            Settings.builder()
+                .put(HierarchyCircuitBreakerService.IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), BREAKER_LIMIT)
+                .build(),
+            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
+        // we can do this here only because we know that we don't adjust breaker settings dynamically in the test
+        inFlightRequestsBreaker = circuitBreakerService.getBreaker(CircuitBreaker.IN_FLIGHT_REQUESTS);
+
+        HttpServerTransport httpServerTransport = new TestHttpServerTransport();
+        restController = new RestController(settings, Collections.emptySet(), null, null, circuitBreakerService);
+        restController.registerHandler(RestRequest.Method.GET, "/",
+            (request, channel, client) -> channel.sendResponse(
+                new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY)));
+        restController.registerHandler(RestRequest.Method.GET, "/error", (request, channel, client) -> {
+            throw new IllegalArgumentException("test error");
+        });
+
+        httpServerTransport.start();
+    }
+
+
     public void testApplyRelevantHeaders() throws Exception {
         final ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
         Set<String> headers = new HashSet<>(Arrays.asList("header.1", "header.2"));
-        final RestController restController = new RestController(Settings.EMPTY, headers) {
-            @Override
-            boolean checkRequestParameters(RestRequest request, RestChannel channel) {
-                return true;
-            }
-
-            @Override
-            void executeHandler(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
+        final RestController restController = new RestController(Settings.EMPTY, headers, null, null, circuitBreakerService);
+        Map<String, List<String>> restHeaders = new HashMap<>();
+        restHeaders.put("header.1", Collections.singletonList("true"));
+        restHeaders.put("header.2", Collections.singletonList("true"));
+        restHeaders.put("header.3", Collections.singletonList("false"));
+        restController.dispatchRequest(new FakeRestRequest.Builder(xContentRegistry()).withHeaders(restHeaders).build(), null, null,
+            threadContext, (RestRequest request, RestChannel channel, NodeClient client) -> {
                 assertEquals("true", threadContext.getHeader("header.1"));
                 assertEquals("true", threadContext.getHeader("header.2"));
                 assertNull(threadContext.getHeader("header.3"));
-            }
-        };
-        threadContext.putHeader("header.3", "true");
-        Map<String, String> restHeaders = new HashMap<>();
-        restHeaders.put("header.1", "true");
-        restHeaders.put("header.2", "true");
-        restHeaders.put("header.3", "false");
-        restController.dispatchRequest(new FakeRestRequest.Builder().withHeaders(restHeaders).build(), null, null, threadContext);
-        assertNull(threadContext.getHeader("header.1"));
-        assertNull(threadContext.getHeader("header.2"));
-        assertEquals("true", threadContext.getHeader("header.3"));
+            });
+        // the rest controller relies on the caller to stash the context, so we should expect these values here as we didn't stash the
+        // context in this test
+        assertEquals("true", threadContext.getHeader("header.1"));
+        assertEquals("true", threadContext.getHeader("header.2"));
+        assertNull(threadContext.getHeader("header.3"));
     }
 
     public void testCanTripCircuitBreaker() throws Exception {
-        RestController controller = new RestController(Settings.EMPTY, Collections.emptySet());
+        RestController controller = new RestController(Settings.EMPTY, Collections.emptySet(), null, null, circuitBreakerService);
         // trip circuit breaker by default
         controller.registerHandler(RestRequest.Method.GET, "/trip", new FakeRestHandler(true));
         controller.registerHandler(RestRequest.Method.GET, "/do-not-trip", new FakeRestHandler(false));
 
-        assertTrue(controller.canTripCircuitBreaker(new FakeRestRequest.Builder().withPath("/trip").build()));
+        assertTrue(controller.canTripCircuitBreaker(new FakeRestRequest.Builder(xContentRegistry()).withPath("/trip").build()));
         // assume trip even on unknown paths
-        assertTrue(controller.canTripCircuitBreaker(new FakeRestRequest.Builder().withPath("/unknown-path").build()));
-        assertFalse(controller.canTripCircuitBreaker(new FakeRestRequest.Builder().withPath("/do-not-trip").build()));
+        assertTrue(controller.canTripCircuitBreaker(new FakeRestRequest.Builder(xContentRegistry()).withPath("/unknown-path").build()));
+        assertFalse(controller.canTripCircuitBreaker(new FakeRestRequest.Builder(xContentRegistry()).withPath("/do-not-trip").build()));
     }
 
     public void testRegisterAsDeprecatedHandler() {
         RestController controller = mock(RestController.class);
 
         RestRequest.Method method = randomFrom(RestRequest.Method.values());
-        String path = "/_" + randomAsciiOfLengthBetween(1, 6);
+        String path = "/_" + randomAlphaOfLengthBetween(1, 6);
         RestHandler handler = mock(RestHandler.class);
-        String deprecationMessage = randomAsciiOfLengthBetween(1, 10);
+        String deprecationMessage = randomAlphaOfLengthBetween(1, 10);
         DeprecationLogger logger = mock(DeprecationLogger.class);
 
         // don't want to test everything -- just that it actually wraps the handler
@@ -101,10 +149,10 @@ public void testRegisterWithDeprecatedHandler() {
         final RestController controller = mock(RestController.class);
 
         final RestRequest.Method method = randomFrom(RestRequest.Method.values());
-        final String path = "/_" + randomAsciiOfLengthBetween(1, 6);
+        final String path = "/_" + randomAlphaOfLengthBetween(1, 6);
         final RestHandler handler = mock(RestHandler.class);
         final RestRequest.Method deprecatedMethod = randomFrom(RestRequest.Method.values());
-        final String deprecatedPath = "/_" + randomAsciiOfLengthBetween(1, 6);
+        final String deprecatedPath = "/_" + randomAlphaOfLengthBetween(1, 6);
         final DeprecationLogger logger = mock(DeprecationLogger.class);
 
         final String deprecationMessage = "[" + deprecatedMethod.name() + " " + deprecatedPath + "] is deprecated! Use [" +
@@ -119,6 +167,24 @@ public void testRegisterWithDeprecatedHandler() {
         verify(controller).registerAsDeprecatedHandler(deprecatedMethod, deprecatedPath, handler, deprecationMessage, logger);
     }
 
+    public void testRestHandlerWrapper() throws Exception {
+        AtomicBoolean handlerCalled = new AtomicBoolean(false);
+        AtomicBoolean wrapperCalled = new AtomicBoolean(false);
+        RestHandler handler = (RestRequest request, RestChannel channel, NodeClient client) -> {
+            handlerCalled.set(true);
+        };
+        UnaryOperator<RestHandler> wrapper = h -> {
+            assertSame(handler, h);
+            return (RestRequest request, RestChannel channel, NodeClient client) -> wrapperCalled.set(true);
+        };
+        final RestController restController = new RestController(Settings.EMPTY, Collections.emptySet(), wrapper, null,
+            circuitBreakerService);
+        final ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
+        restController.dispatchRequest(new FakeRestRequest.Builder(xContentRegistry()).build(), null, null, threadContext, handler);
+        assertTrue(wrapperCalled.get());
+        assertFalse(handlerCalled.get());
+    }
+
     /**
      * Useful for testing with deprecation handler.
      */
@@ -139,4 +205,443 @@ public boolean canTripCircuitBreaker() {
             return canTripCircuitBreaker;
         }
     }
+
+    public void testDispatchRequestAddsAndFreesBytesOnSuccess() {
+        int contentLength = BREAKER_LIMIT.bytesAsInt();
+        String content = randomAlphaOfLength(contentLength);
+        TestRestRequest request = new TestRestRequest("/", content, XContentType.JSON);
+        AssertingChannel channel = new AssertingChannel(request, true, RestStatus.OK);
+
+        restController.dispatchRequest(request, channel, new ThreadContext(Settings.EMPTY));
+
+        assertEquals(0, inFlightRequestsBreaker.getTrippedCount());
+        assertEquals(0, inFlightRequestsBreaker.getUsed());
+    }
+
+    public void testDispatchRequestAddsAndFreesBytesOnError() {
+        int contentLength = BREAKER_LIMIT.bytesAsInt();
+        String content = randomAlphaOfLength(contentLength);
+        TestRestRequest request = new TestRestRequest("/error", content, XContentType.JSON);
+        AssertingChannel channel = new AssertingChannel(request, true, RestStatus.BAD_REQUEST);
+
+        restController.dispatchRequest(request, channel, new ThreadContext(Settings.EMPTY));
+
+        assertEquals(0, inFlightRequestsBreaker.getTrippedCount());
+        assertEquals(0, inFlightRequestsBreaker.getUsed());
+    }
+
+    public void testDispatchRequestAddsAndFreesBytesOnlyOnceOnError() {
+        int contentLength = BREAKER_LIMIT.bytesAsInt();
+        String content = randomAlphaOfLength(contentLength);
+        // we will produce an error in the rest handler and one more when sending the error response
+        TestRestRequest request = new TestRestRequest("/error", content, XContentType.JSON);
+        ExceptionThrowingChannel channel = new ExceptionThrowingChannel(request, true);
+
+        restController.dispatchRequest(request, channel, new ThreadContext(Settings.EMPTY));
+
+        assertEquals(0, inFlightRequestsBreaker.getTrippedCount());
+        assertEquals(0, inFlightRequestsBreaker.getUsed());
+    }
+
+    public void testDispatchRequestLimitsBytes() {
+        int contentLength = BREAKER_LIMIT.bytesAsInt() + 1;
+        String content = randomAlphaOfLength(contentLength);
+        TestRestRequest request = new TestRestRequest("/", content, XContentType.JSON);
+        AssertingChannel channel = new AssertingChannel(request, true, RestStatus.SERVICE_UNAVAILABLE);
+
+        restController.dispatchRequest(request, channel, new ThreadContext(Settings.EMPTY));
+
+        assertEquals(1, inFlightRequestsBreaker.getTrippedCount());
+        assertEquals(0, inFlightRequestsBreaker.getUsed());
+    }
+
+    public void testDispatchRequiresContentTypeForRequestsWithContent() {
+        String content = randomAlphaOfLengthBetween(1, BREAKER_LIMIT.bytesAsInt());
+        TestRestRequest request = new TestRestRequest("/", content, null);
+        AssertingChannel channel = new AssertingChannel(request, true, RestStatus.NOT_ACCEPTABLE);
+        restController = new RestController(
+            Settings.builder().put(HttpTransportSettings.SETTING_HTTP_CONTENT_TYPE_REQUIRED.getKey(), true).build(),
+            Collections.emptySet(), null, null, circuitBreakerService);
+        restController.registerHandler(RestRequest.Method.GET, "/",
+            (r, c, client) -> c.sendResponse(
+                new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY)));
+
+        assertFalse(channel.getSendResponseCalled());
+        restController.dispatchRequest(request, channel, new ThreadContext(Settings.EMPTY));
+        assertTrue(channel.getSendResponseCalled());
+    }
+
+    public void testDispatchDoesNotRequireContentTypeForRequestsWithoutContent() {
+        FakeRestRequest fakeRestRequest = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY).build();
+        AssertingChannel channel = new AssertingChannel(fakeRestRequest, true, RestStatus.OK);
+
+        assertFalse(channel.getSendResponseCalled());
+        restController.dispatchRequest(fakeRestRequest, channel, new ThreadContext(Settings.EMPTY));
+        assertTrue(channel.getSendResponseCalled());
+    }
+
+    public void testDispatchWorksWithPlainText() {
+        String content = randomAlphaOfLengthBetween(1, BREAKER_LIMIT.bytesAsInt());
+        FakeRestRequest fakeRestRequest = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY)
+            .withContent(new BytesArray(content), null).withPath("/foo")
+            .withHeaders(Collections.singletonMap("Content-Type", Collections.singletonList("text/plain"))).build();
+        AssertingChannel channel = new AssertingChannel(fakeRestRequest, true, RestStatus.OK);
+        restController.registerHandler(RestRequest.Method.GET, "/foo", new RestHandler() {
+            @Override
+            public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
+                channel.sendResponse(new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY));
+            }
+
+            @Override
+            public boolean supportsPlainText() {
+                return true;
+            }
+        });
+
+        assertFalse(channel.getSendResponseCalled());
+        restController.dispatchRequest(fakeRestRequest, channel, new ThreadContext(Settings.EMPTY));
+        assertTrue(channel.getSendResponseCalled());
+        assertWarnings("Plain text request bodies are deprecated. Use request parameters or body in a supported format.",
+            "Content type detection for rest requests is deprecated. Specify the content type using the [Content-Type] header.");
+    }
+
+    public void testDispatchWorksWithAutodetectOnPlainTextHandler() {
+        String content = "{}";
+        FakeRestRequest fakeRestRequest = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY)
+            .withContent(new BytesArray(content), null).withPath("/foo").build();
+        AssertingChannel channel = new AssertingChannel(fakeRestRequest, true, RestStatus.OK);
+        restController.registerHandler(RestRequest.Method.GET, "/foo", new RestHandler() {
+            @Override
+            public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
+                channel.sendResponse(new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY));
+            }
+
+            @Override
+            public boolean supportsPlainText() {
+                return true;
+            }
+        });
+
+        assertFalse(channel.getSendResponseCalled());
+        restController.dispatchRequest(fakeRestRequest, channel, new ThreadContext(Settings.EMPTY));
+        assertTrue(channel.getSendResponseCalled());
+        assertWarnings("Content type detection for rest requests is deprecated. Specify the content type using the [Content-Type] header.");
+    }
+
+    public void testDispatchWorksWithAutoDetection() {
+        FakeRestRequest fakeRestRequest = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY)
+            .withContent(new BytesArray("{}"), null).withPath("/")
+            .withHeaders(Collections.singletonMap("Content-Type", Collections.singletonList("application/x-www-form-urlencoded"))).build();
+        AssertingChannel channel = new AssertingChannel(fakeRestRequest, true, RestStatus.OK);
+
+        assertFalse(channel.getSendResponseCalled());
+        restController.dispatchRequest(fakeRestRequest, channel, new ThreadContext(Settings.EMPTY));
+        assertTrue(channel.getSendResponseCalled());
+        assertWarnings("Content type detection for rest requests is deprecated. Specify the content type using the [Content-Type] header.");
+    }
+
+    public void testDispatchWorksWithNewlineDelimitedJson() {
+        final String mimeType = "application/x-ndjson";
+        String content = randomAlphaOfLengthBetween(1, BREAKER_LIMIT.bytesAsInt());
+        FakeRestRequest fakeRestRequest = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY)
+            .withContent(new BytesArray(content), null).withPath("/foo")
+            .withHeaders(Collections.singletonMap("Content-Type", Collections.singletonList(mimeType))).build();
+        AssertingChannel channel = new AssertingChannel(fakeRestRequest, true, RestStatus.OK);
+        restController.registerHandler(RestRequest.Method.GET, "/foo", new RestHandler() {
+            @Override
+            public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
+                channel.sendResponse(new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY));
+            }
+
+            @Override
+            public boolean supportsContentStream() {
+                return true;
+            }
+        });
+
+        assertFalse(channel.getSendResponseCalled());
+        restController.dispatchRequest(fakeRestRequest, channel, new ThreadContext(Settings.EMPTY));
+        assertTrue(channel.getSendResponseCalled());
+    }
+
+    public void testDispatchWorksWithLineDelimitedJsonDeprecated() {
+        final String mimeType = "application/x-ldjson";
+        String content = randomAlphaOfLengthBetween(1, BREAKER_LIMIT.bytesAsInt());
+        FakeRestRequest fakeRestRequest = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY)
+            .withContent(new BytesArray(content), null).withPath("/foo")
+            .withHeaders(Collections.singletonMap("Content-Type", Collections.singletonList(mimeType))).build();
+        AssertingChannel channel = new AssertingChannel(fakeRestRequest, true, RestStatus.OK);
+        restController.registerHandler(RestRequest.Method.GET, "/foo", new RestHandler() {
+            @Override
+            public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
+                channel.sendResponse(new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY));
+            }
+
+            @Override
+            public boolean supportsContentStream() {
+                return true;
+            }
+        });
+
+        assertFalse(channel.getSendResponseCalled());
+        restController.dispatchRequest(fakeRestRequest, channel, new ThreadContext(Settings.EMPTY));
+        assertTrue(channel.getSendResponseCalled());
+        assertWarnings(
+                String.format(
+                        Locale.ROOT,
+                        "The Content-Type [%s] has been superseded by [%s] in the specification and should be used instead.",
+                        "application/x-ldjson",
+                        "application/x-ndjson"));
+    }
+
+    public void testDispatchWithContentStream() {
+        final String mimeType = randomFrom("application/json", "application/smile");
+        String content = randomAlphaOfLengthBetween(1, BREAKER_LIMIT.bytesAsInt());
+        FakeRestRequest fakeRestRequest = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY)
+            .withContent(new BytesArray(content), null).withPath("/foo")
+            .withHeaders(Collections.singletonMap("Content-Type", Collections.singletonList(mimeType))).build();
+        AssertingChannel channel = new AssertingChannel(fakeRestRequest, true, RestStatus.OK);
+        restController.registerHandler(RestRequest.Method.GET, "/foo", new RestHandler() {
+            @Override
+            public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
+                channel.sendResponse(new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY));
+            }
+
+            @Override
+            public boolean supportsContentStream() {
+                return true;
+            }
+        });
+
+        assertFalse(channel.getSendResponseCalled());
+        restController.dispatchRequest(fakeRestRequest, channel, new ThreadContext(Settings.EMPTY));
+        assertTrue(channel.getSendResponseCalled());
+    }
+
+    public void testDispatchWithContentStreamAutoDetect() {
+        FakeRestRequest fakeRestRequest = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY)
+            .withContent(new BytesArray("{}"), null).withPath("/foo").build();
+        AssertingChannel channel = new AssertingChannel(fakeRestRequest, true, RestStatus.OK);
+        restController.registerHandler(RestRequest.Method.GET, "/foo", new RestHandler() {
+            @Override
+            public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
+                channel.sendResponse(new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY));
+            }
+
+            @Override
+            public boolean supportsContentStream() {
+                return true;
+            }
+        });
+
+        assertFalse(channel.getSendResponseCalled());
+        restController.dispatchRequest(fakeRestRequest, channel, new ThreadContext(Settings.EMPTY));
+        assertTrue(channel.getSendResponseCalled());
+        assertWarnings("Content type detection for rest requests is deprecated. Specify the content type using the [Content-Type] header.");
+    }
+
+    public void testNonStreamingXContentCausesErrorResponse() throws IOException {
+        // auto detect
+        FakeRestRequest fakeRestRequest = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY)
+            .withContent(YamlXContent.contentBuilder().startObject().endObject().bytes(), null).withPath("/foo").build();
+        AssertingChannel channel = new AssertingChannel(fakeRestRequest, true, RestStatus.NOT_ACCEPTABLE);
+        restController.registerHandler(RestRequest.Method.GET, "/foo", new RestHandler() {
+            @Override
+            public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
+                channel.sendResponse(new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY));
+            }
+
+            @Override
+            public boolean supportsContentStream() {
+                return true;
+            }
+        });
+
+        assertFalse(channel.getSendResponseCalled());
+        restController.dispatchRequest(fakeRestRequest, channel, new ThreadContext(Settings.EMPTY));
+        assertTrue(channel.getSendResponseCalled());
+
+        assertWarnings("Content type detection for rest requests is deprecated. Specify the content type using the [Content-Type] header.");
+
+        // specified
+        fakeRestRequest = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY)
+            .withContent(YamlXContent.contentBuilder().startObject().endObject().bytes(), XContentType.YAML).withPath("/foo").build();
+        channel = new AssertingChannel(fakeRestRequest, true, RestStatus.NOT_ACCEPTABLE);
+        assertFalse(channel.getSendResponseCalled());
+        restController.dispatchRequest(fakeRestRequest, channel, new ThreadContext(Settings.EMPTY));
+        assertTrue(channel.getSendResponseCalled());
+    }
+
+    public void testStrictModeContentStream() {
+        restController = new RestController(
+            Settings.builder().put(HttpTransportSettings.SETTING_HTTP_CONTENT_TYPE_REQUIRED.getKey(), true).build(),
+            Collections.emptySet(), null, null, circuitBreakerService);
+        FakeRestRequest fakeRestRequest = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY)
+            .withContent(new BytesArray("{}"), null).withPath("/foo")
+            .build();
+        AssertingChannel channel = new AssertingChannel(fakeRestRequest, true, RestStatus.NOT_ACCEPTABLE);
+        restController.registerHandler(RestRequest.Method.GET, "/foo", new RestHandler() {
+            @Override
+            public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
+                channel.sendResponse(new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY));
+            }
+
+            @Override
+            public boolean supportsContentStream() {
+                return true;
+            }
+        });
+        assertFalse(channel.getSendResponseCalled());
+        restController.dispatchRequest(fakeRestRequest, channel, new ThreadContext(Settings.EMPTY));
+        assertTrue(channel.getSendResponseCalled());
+    }
+
+    public void testUnknownContentWithContentStream() {
+        FakeRestRequest fakeRestRequest = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY)
+            .withContent(new BytesArray("aaaabbbbb"), null).withPath("/foo")
+            .withHeaders(Collections.singletonMap("Content-Type", Collections.singletonList("foo/bar")))
+            .build();
+        AssertingChannel channel = new AssertingChannel(fakeRestRequest, true, RestStatus.NOT_ACCEPTABLE);
+        restController.registerHandler(RestRequest.Method.GET, "/foo", new RestHandler() {
+            @Override
+            public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
+                channel.sendResponse(new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY));
+            }
+
+            @Override
+            public boolean supportsContentStream() {
+                return true;
+            }
+        });
+        assertFalse(channel.getSendResponseCalled());
+        restController.dispatchRequest(fakeRestRequest, channel, new ThreadContext(Settings.EMPTY));
+        assertTrue(channel.getSendResponseCalled());
+        assertWarnings("Content type detection for rest requests is deprecated. Specify the content type using the [Content-Type] header.");
+    }
+
+    public void testDispatchBadRequest() {
+        final FakeRestRequest fakeRestRequest = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY).build();
+        final AssertingChannel channel = new AssertingChannel(fakeRestRequest, true, RestStatus.BAD_REQUEST);
+        restController.dispatchBadRequest(
+                fakeRestRequest,
+                channel,
+                new ThreadContext(Settings.EMPTY),
+                randomBoolean() ? new IllegalStateException("bad request") : new Throwable("bad request"));
+        assertTrue(channel.getSendResponseCalled());
+        assertThat(channel.getRestResponse().content().utf8ToString(), containsString("bad request"));
+    }
+
+    public void testDispatchBadRequestUnknownCause() {
+        final FakeRestRequest fakeRestRequest = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY).build();
+        final AssertingChannel channel = new AssertingChannel(fakeRestRequest, true, RestStatus.BAD_REQUEST);
+        restController.dispatchBadRequest(fakeRestRequest, channel, new ThreadContext(Settings.EMPTY), null);
+        assertTrue(channel.getSendResponseCalled());
+        assertThat(channel.getRestResponse().content().utf8ToString(), containsString("unknown cause"));
+    }
+
+    private static final class TestHttpServerTransport extends AbstractLifecycleComponent implements
+        HttpServerTransport {
+
+        TestHttpServerTransport() {
+            super(Settings.EMPTY);
+        }
+
+        @Override
+        protected void doStart() {
+        }
+
+        @Override
+        protected void doStop() {
+        }
+
+        @Override
+        protected void doClose() {
+        }
+
+        @Override
+        public BoundTransportAddress boundAddress() {
+            TransportAddress transportAddress = new LocalTransportAddress(randomAlphaOfLengthBetween(1, 5));
+            return new BoundTransportAddress(new TransportAddress[] {transportAddress} ,transportAddress);
+        }
+
+        @Override
+        public HttpInfo info() {
+            return null;
+        }
+
+        @Override
+        public HttpStats stats() {
+            return null;
+        }
+    }
+
+    private static final class AssertingChannel extends AbstractRestChannel {
+
+        private final RestStatus expectedStatus;
+        private final AtomicReference<RestResponse> responseReference = new AtomicReference<>();
+
+        protected AssertingChannel(RestRequest request, boolean detailedErrorsEnabled, RestStatus expectedStatus) {
+            super(request, detailedErrorsEnabled);
+            this.expectedStatus = expectedStatus;
+        }
+
+        @Override
+        public void sendResponse(RestResponse response) {
+            assertEquals(expectedStatus, response.status());
+            responseReference.set(response);
+        }
+
+        RestResponse getRestResponse() {
+            return responseReference.get();
+        }
+
+        boolean getSendResponseCalled() {
+            return getRestResponse() != null;
+        }
+
+    }
+
+    private static final class ExceptionThrowingChannel extends AbstractRestChannel {
+
+        protected ExceptionThrowingChannel(RestRequest request, boolean detailedErrorsEnabled) {
+            super(request, detailedErrorsEnabled);
+        }
+
+        @Override
+        public void sendResponse(RestResponse response) {
+            throw new IllegalStateException("always throwing an exception for testing");
+        }
+    }
+
+    private static final class TestRestRequest extends RestRequest {
+
+        private final BytesReference content;
+
+        private TestRestRequest(String path, String content, XContentType xContentType) {
+            super(NamedXContentRegistry.EMPTY, Collections.emptyMap(), path, xContentType == null ?
+                Collections.emptyMap() : Collections.singletonMap("Content-Type", Collections.singletonList(xContentType.mediaType())));
+            this.content = new BytesArray(content);
+        }
+
+        @Override
+        public Method method() {
+            return Method.GET;
+        }
+
+        @Override
+        public String uri() {
+            return null;
+        }
+
+        @Override
+        public boolean hasContent() {
+            return true;
+        }
+
+        @Override
+        public BytesReference content() {
+            return content;
+        }
+
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/rest/RestFilterChainTests.java b/core/src/test/java/org/elasticsearch/rest/RestFilterChainTests.java
deleted file mode 100644
index 5013d637436d2..0000000000000
--- a/core/src/test/java/org/elasticsearch/rest/RestFilterChainTests.java
+++ /dev/null
@@ -1,211 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.rest;
-
-import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.rest.FakeRestChannel;
-import org.elasticsearch.test.rest.FakeRestRequest;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import static org.hamcrest.CoreMatchers.equalTo;
-
-public class RestFilterChainTests extends ESTestCase {
-    public void testRestFilters() throws Exception {
-
-        RestController restController = new RestController(Settings.EMPTY, Collections.emptySet());
-
-        int numFilters = randomInt(10);
-        Set<Integer> orders = new HashSet<>(numFilters);
-        while (orders.size() < numFilters) {
-            orders.add(randomInt(10));
-        }
-
-        List<RestFilter> filters = new ArrayList<>();
-        for (Integer order : orders) {
-            TestFilter testFilter = new TestFilter(order, randomFrom(Operation.values()));
-            filters.add(testFilter);
-            restController.registerFilter(testFilter);
-        }
-
-        ArrayList<RestFilter> restFiltersByOrder = new ArrayList<>(filters);
-        Collections.sort(restFiltersByOrder, new Comparator<RestFilter>() {
-            @Override
-            public int compare(RestFilter o1, RestFilter o2) {
-                return Integer.compare(o1.order(), o2.order());
-            }
-        });
-
-        List<RestFilter> expectedRestFilters = new ArrayList<>();
-        for (RestFilter filter : restFiltersByOrder) {
-            TestFilter testFilter = (TestFilter) filter;
-            expectedRestFilters.add(testFilter);
-            if (!(testFilter.callback == Operation.CONTINUE_PROCESSING) ) {
-                break;
-            }
-        }
-
-        restController.registerHandler(RestRequest.Method.GET, "/", (request, channel, client) -> {
-            channel.sendResponse(new TestResponse());
-        });
-
-        FakeRestRequest fakeRestRequest = new FakeRestRequest();
-        FakeRestChannel fakeRestChannel = new FakeRestChannel(fakeRestRequest, randomBoolean(), 1);
-        restController.dispatchRequest(fakeRestRequest, fakeRestChannel, null, new ThreadContext(Settings.EMPTY));
-        assertThat(fakeRestChannel.await(), equalTo(true));
-
-
-        List<TestFilter> testFiltersByLastExecution = new ArrayList<>();
-        for (RestFilter restFilter : filters) {
-            testFiltersByLastExecution.add((TestFilter)restFilter);
-        }
-        Collections.sort(testFiltersByLastExecution, new Comparator<TestFilter>() {
-            @Override
-            public int compare(TestFilter o1, TestFilter o2) {
-                return Long.compare(o1.executionToken, o2.executionToken);
-            }
-        });
-
-        ArrayList<TestFilter> finalTestFilters = new ArrayList<>();
-        for (RestFilter filter : testFiltersByLastExecution) {
-            TestFilter testFilter = (TestFilter) filter;
-            finalTestFilters.add(testFilter);
-            if (!(testFilter.callback == Operation.CONTINUE_PROCESSING) ) {
-                break;
-            }
-        }
-
-        assertThat(finalTestFilters.size(), equalTo(expectedRestFilters.size()));
-
-        for (int i = 0; i < finalTestFilters.size(); i++) {
-            TestFilter testFilter = finalTestFilters.get(i);
-            assertThat(testFilter, equalTo(expectedRestFilters.get(i)));
-            assertThat(testFilter.runs.get(), equalTo(1));
-        }
-    }
-
-    public void testTooManyContinueProcessing() throws Exception {
-
-        final int additionalContinueCount = randomInt(10);
-
-        TestFilter testFilter = new TestFilter(randomInt(), (request, channel, client, filterChain) -> {
-            for (int i = 0; i <= additionalContinueCount; i++) {
-                filterChain.continueProcessing(request, channel, null);
-            }
-        });
-
-        RestController restController = new RestController(Settings.EMPTY, Collections.emptySet());
-        restController.registerFilter(testFilter);
-
-        restController.registerHandler(RestRequest.Method.GET, "/", new RestHandler() {
-            @Override
-            public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
-                channel.sendResponse(new TestResponse());
-            }
-        });
-
-        FakeRestRequest fakeRestRequest = new FakeRestRequest();
-        FakeRestChannel fakeRestChannel = new FakeRestChannel(fakeRestRequest, randomBoolean(), additionalContinueCount + 1);
-        restController.dispatchRequest(fakeRestRequest, fakeRestChannel, null, new ThreadContext(Settings.EMPTY));
-        fakeRestChannel.await();
-
-        assertThat(testFilter.runs.get(), equalTo(1));
-
-        assertThat(fakeRestChannel.responses().get(), equalTo(1));
-        assertThat(fakeRestChannel.errors().get(), equalTo(additionalContinueCount));
-    }
-
-    private enum Operation implements Callback {
-        CONTINUE_PROCESSING {
-            @Override
-            public void execute(RestRequest request, RestChannel channel, NodeClient client, RestFilterChain filterChain) throws Exception {
-                filterChain.continueProcessing(request, channel, client);
-            }
-        },
-        CHANNEL_RESPONSE {
-            @Override
-            public void execute(RestRequest request, RestChannel channel, NodeClient client, RestFilterChain filterChain) throws Exception {
-                channel.sendResponse(new TestResponse());
-            }
-        }
-    }
-
-    private interface Callback {
-        void execute(RestRequest request, RestChannel channel, NodeClient client, RestFilterChain filterChain) throws Exception;
-    }
-
-    private final AtomicInteger counter = new AtomicInteger();
-
-    private class TestFilter extends RestFilter {
-        private final int order;
-        private final Callback callback;
-        AtomicInteger runs = new AtomicInteger();
-        volatile int executionToken = Integer.MAX_VALUE; //the filters that don't run will go last in the sorted list
-
-        TestFilter(int order, Callback callback) {
-            this.order = order;
-            this.callback = callback;
-        }
-
-        @Override
-        public void process(RestRequest request, RestChannel channel, NodeClient client, RestFilterChain filterChain) throws Exception {
-            this.runs.incrementAndGet();
-            this.executionToken = counter.incrementAndGet();
-            this.callback.execute(request, channel, client, filterChain);
-        }
-
-        @Override
-        public int order() {
-            return order;
-        }
-
-        @Override
-        public String toString() {
-            return "[order:" + order + ", executionToken:" + executionToken + "]";
-        }
-    }
-
-    private static class TestResponse extends RestResponse {
-        @Override
-        public String contentType() {
-            return null;
-        }
-
-        @Override
-        public BytesReference content() {
-            return null;
-        }
-
-        @Override
-        public RestStatus status() {
-            return RestStatus.OK;
-        }
-    }
-}
diff --git a/core/src/test/java/org/elasticsearch/rest/RestRequestTests.java b/core/src/test/java/org/elasticsearch/rest/RestRequestTests.java
new file mode 100644
index 0000000000000..190537eca62e0
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/rest/RestRequestTests.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.rest;
+
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.collect.MapBuilder;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.singletonMap;
+
+public class RestRequestTests extends ESTestCase {
+    public void testContentParser() throws IOException {
+        Exception e = expectThrows(ElasticsearchParseException.class, () ->
+            new ContentRestRequest("", emptyMap()).contentParser());
+        assertEquals("request body is required", e.getMessage());
+        e = expectThrows(ElasticsearchParseException.class, () ->
+            new ContentRestRequest("", singletonMap("source", "{}")).contentParser());
+        assertEquals("request body is required", e.getMessage());
+        assertEquals(emptyMap(), new ContentRestRequest("{}", emptyMap()).contentParser().map());
+        e = expectThrows(ElasticsearchParseException.class, () ->
+            new ContentRestRequest("", emptyMap(), emptyMap()).contentParser());
+        assertEquals("request body is required", e.getMessage());
+    }
+
+    public void testApplyContentParser() throws IOException {
+        new ContentRestRequest("", emptyMap()).applyContentParser(p -> fail("Shouldn't have been called"));
+        new ContentRestRequest("", singletonMap("source", "{}")).applyContentParser(p -> fail("Shouldn't have been called"));
+        AtomicReference<Object> source = new AtomicReference<>();
+        new ContentRestRequest("{}", emptyMap()).applyContentParser(p -> source.set(p.map()));
+        assertEquals(emptyMap(), source.get());
+    }
+
+    public void testContentOrSourceParam() throws IOException {
+        Exception e = expectThrows(ElasticsearchParseException.class, () ->
+            new ContentRestRequest("", emptyMap()).contentOrSourceParam());
+        assertEquals("request body or source parameter is required", e.getMessage());
+        assertEquals(new BytesArray("stuff"), new ContentRestRequest("stuff", emptyMap()).contentOrSourceParam().v2());
+        assertEquals(new BytesArray("stuff"),
+            new ContentRestRequest("stuff", singletonMap("source", "stuff2")).contentOrSourceParam().v2());
+        assertEquals(new BytesArray("{\"foo\": \"stuff\"}"),
+            new ContentRestRequest("", singletonMap("source", "{\"foo\": \"stuff\"}")).contentOrSourceParam().v2());
+        assertWarnings("Deprecated use of the [source] parameter without the [source_content_type] parameter. " +
+                "Use the [source_content_type] parameter to specify the content type of the source such as [application/json]");
+    }
+
+    public void testHasContentOrSourceParam() throws IOException {
+        assertEquals(false, new ContentRestRequest("", emptyMap()).hasContentOrSourceParam());
+        assertEquals(true, new ContentRestRequest("stuff", emptyMap()).hasContentOrSourceParam());
+        assertEquals(true, new ContentRestRequest("stuff", singletonMap("source", "stuff2")).hasContentOrSourceParam());
+        assertEquals(true, new ContentRestRequest("", singletonMap("source", "stuff")).hasContentOrSourceParam());
+    }
+
+    public void testContentOrSourceParamParser() throws IOException {
+        Exception e = expectThrows(ElasticsearchParseException.class, () ->
+            new ContentRestRequest("", emptyMap()).contentOrSourceParamParser());
+        assertEquals("request body or source parameter is required", e.getMessage());
+        assertEquals(emptyMap(), new ContentRestRequest("{}", emptyMap()).contentOrSourceParamParser().map());
+        assertEquals(emptyMap(), new ContentRestRequest("{}", singletonMap("source", "stuff2")).contentOrSourceParamParser().map());
+        assertEquals(emptyMap(), new ContentRestRequest("", singletonMap("source", "{}")).contentOrSourceParamParser().map());
+        assertWarnings("Deprecated use of the [source] parameter without the [source_content_type] parameter. " +
+                "Use the [source_content_type] parameter to specify the content type of the source such as [application/json]");
+    }
+
+    public void testWithContentOrSourceParamParserOrNull() throws IOException {
+        new ContentRestRequest("", emptyMap()).withContentOrSourceParamParserOrNull(parser -> assertNull(parser));
+        new ContentRestRequest("{}", emptyMap()).withContentOrSourceParamParserOrNull(parser -> assertEquals(emptyMap(), parser.map()));
+        new ContentRestRequest("{}", singletonMap("source", "stuff2")).withContentOrSourceParamParserOrNull(parser ->
+                assertEquals(emptyMap(), parser.map()));
+        new ContentRestRequest("", singletonMap("source", "{}")).withContentOrSourceParamParserOrNull(parser ->
+                assertEquals(emptyMap(), parser.map()));
+        assertWarnings("Deprecated use of the [source] parameter without the [source_content_type] parameter. " +
+                "Use the [source_content_type] parameter to specify the content type of the source such as [application/json]");
+    }
+
+    public void testContentTypeParsing() {
+        for (XContentType xContentType : XContentType.values()) {
+            Map<String, List<String>> map = new HashMap<>();
+            map.put("Content-Type", Collections.singletonList(xContentType.mediaType()));
+            ContentRestRequest restRequest = new ContentRestRequest("", Collections.emptyMap(), map);
+            assertEquals(xContentType, restRequest.getXContentType());
+
+            map = new HashMap<>();
+            map.put("Content-Type", Collections.singletonList(xContentType.mediaTypeWithoutParameters()));
+            restRequest = new ContentRestRequest("", Collections.emptyMap(), map);
+            assertEquals(xContentType, restRequest.getXContentType());
+        }
+    }
+
+    public void testPlainTextSupport() {
+        ContentRestRequest restRequest = new ContentRestRequest(randomAlphaOfLengthBetween(1, 30), Collections.emptyMap(),
+            Collections.singletonMap("Content-Type",
+                Collections.singletonList(randomFrom("text/plain", "text/plain; charset=utf-8", "text/plain;charset=utf-8"))));
+        assertNull(restRequest.getXContentType());
+    }
+
+    public void testMalformedContentTypeHeader() {
+        final String type = randomFrom("text", "text/:ain; charset=utf-8", "text/plain\";charset=utf-8", ":", "/", "t:/plain");
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new ContentRestRequest("", Collections.emptyMap(),
+            Collections.singletonMap("Content-Type", Collections.singletonList(type))));
+        assertEquals("invalid Content-Type header [" + type + "]", e.getMessage());
+    }
+
+    public void testNoContentTypeHeader() {
+        ContentRestRequest contentRestRequest = new ContentRestRequest("", Collections.emptyMap(), Collections.emptyMap());
+        assertNull(contentRestRequest.getXContentType());
+    }
+
+    public void testMultipleContentTypeHeaders() {
+        List<String> headers = new ArrayList<>(randomUnique(() -> randomAlphaOfLengthBetween(1, 16), randomIntBetween(2, 10)));
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new ContentRestRequest("", Collections.emptyMap(),
+            Collections.singletonMap("Content-Type", headers)));
+        assertEquals("only one Content-Type header should be provided", e.getMessage());
+    }
+
+    public void testRequiredContent() {
+        Exception e = expectThrows(ElasticsearchParseException.class, () ->
+            new ContentRestRequest("", emptyMap()).requiredContent());
+        assertEquals("request body is required", e.getMessage());
+        assertEquals(new BytesArray("stuff"), new ContentRestRequest("stuff", emptyMap()).requiredContent());
+        assertEquals(new BytesArray("stuff"),
+            new ContentRestRequest("stuff", MapBuilder.<String, String>newMapBuilder()
+                .put("source", "stuff2").put("source_content_type", "application/json").immutableMap()).requiredContent());
+        e = expectThrows(ElasticsearchParseException.class, () ->
+            new ContentRestRequest("", MapBuilder.<String, String>newMapBuilder()
+                .put("source", "{\"foo\": \"stuff\"}").put("source_content_type", "application/json").immutableMap())
+                .requiredContent());
+        assertEquals("request body is required", e.getMessage());
+        e = expectThrows(IllegalStateException.class, () ->
+            new ContentRestRequest("test", null, Collections.emptyMap()).requiredContent());
+        assertEquals("unknown content type", e.getMessage());
+    }
+
+    private static final class ContentRestRequest extends RestRequest {
+        private final BytesArray content;
+
+        ContentRestRequest(String content, Map<String, String> params) {
+            this(content, params, Collections.singletonMap("Content-Type", Collections.singletonList("application/json")));
+        }
+
+        ContentRestRequest(String content, Map<String, String> params, Map<String, List<String>> headers) {
+            super(NamedXContentRegistry.EMPTY, params, "not used by this test", headers);
+            this.content = new BytesArray(content);
+        }
+
+        @Override
+        public boolean hasContent() {
+            return Strings.hasLength(content);
+        }
+
+        @Override
+        public BytesReference content() {
+            return content;
+        }
+
+        @Override
+        public String uri() {
+            throw new UnsupportedOperationException("Not used by this test");
+        }
+
+        @Override
+        public Method method() {
+            throw new UnsupportedOperationException("Not used by this test");
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/rest/action/RestActionsTests.java b/core/src/test/java/org/elasticsearch/rest/action/RestActionsTests.java
new file mode 100644
index 0000000000000..401cc79b02092
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/rest/action/RestActionsTests.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.rest.action;
+
+import com.fasterxml.jackson.core.io.JsonEOFException;
+import java.util.Arrays;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.index.query.MatchQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+
+import static java.util.Collections.emptyList;
+
+public class RestActionsTests extends ESTestCase {
+
+    private static NamedXContentRegistry xContentRegistry;
+
+    @BeforeClass
+    public static void init() {
+        xContentRegistry = new NamedXContentRegistry(new SearchModule(Settings.EMPTY, false, emptyList()).getNamedXContents());
+    }
+
+    @AfterClass
+    public static void cleanup() {
+        xContentRegistry = null;
+    }
+
+    public void testParseTopLevelBuilder() throws IOException {
+        QueryBuilder query = new MatchQueryBuilder("foo", "bar");
+        String requestBody = "{ \"query\" : " + query.toString() + "}";
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, requestBody)) {
+            QueryBuilder actual = RestActions.getQueryContent(parser);
+            assertEquals(query, actual);
+        }
+    }
+
+    public void testParseTopLevelBuilderEmptyObject() throws IOException {
+        for (String requestBody : Arrays.asList("{}", "")) {
+            try (XContentParser parser = createParser(JsonXContent.jsonXContent, requestBody)) {
+                QueryBuilder query = RestActions.getQueryContent(parser);
+                assertNull(query);
+            }
+        }
+    }
+
+    public void testParseTopLevelBuilderMalformedJson() throws IOException {
+        for (String requestBody : Arrays.asList("\"\"", "\"someString\"", "\"{\"")) {
+            try (XContentParser parser = createParser(JsonXContent.jsonXContent, requestBody)) {
+                ParsingException exception =
+                    expectThrows(ParsingException.class, () -> RestActions.getQueryContent(parser));
+                assertEquals("Expected [START_OBJECT] but found [VALUE_STRING]", exception.getMessage());
+            }
+        }
+    }
+
+    public void testParseTopLevelBuilderIncompleteJson() throws IOException {
+        for (String requestBody : Arrays.asList("{", "{ \"query\" :")) {
+            try (XContentParser parser = createParser(JsonXContent.jsonXContent, requestBody)) {
+                ParsingException exception =
+                    expectThrows(ParsingException.class, () -> RestActions.getQueryContent(parser));
+                assertEquals("Failed to parse", exception.getMessage());
+                assertEquals(JsonEOFException.class, exception.getRootCause().getClass());
+            }
+        }
+    }
+
+    public void testParseTopLevelBuilderUnknownParameter() throws IOException {
+        String requestBody = "{ \"foo\" : \"bar\"}";
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, requestBody)) {
+            ParsingException exception = expectThrows(ParsingException.class, () -> RestActions.getQueryContent(parser));
+            assertEquals("request does not support [foo]", exception.getMessage());
+        }
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return xContentRegistry;
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/rest/action/RestBuilderListenerTests.java b/core/src/test/java/org/elasticsearch/rest/action/RestBuilderListenerTests.java
new file mode 100644
index 0000000000000..2bc0d0bdc8157
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/rest/action/RestBuilderListenerTests.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.rest.action;
+
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.rest.BytesRestResponse;
+import org.elasticsearch.rest.RestResponse;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.rest.FakeRestChannel;
+import org.elasticsearch.test.rest.FakeRestRequest;
+import org.elasticsearch.transport.TransportResponse;
+import org.elasticsearch.transport.TransportResponse.Empty;
+
+import java.util.concurrent.atomic.AtomicReference;
+
+public class RestBuilderListenerTests extends ESTestCase {
+
+    public void testXContentBuilderClosedInBuildResponse() throws Exception {
+        AtomicReference<XContentBuilder> builderAtomicReference = new AtomicReference<>();
+        RestBuilderListener<TransportResponse.Empty> builderListener =
+            new RestBuilderListener<Empty>(new FakeRestChannel(new FakeRestRequest(), randomBoolean(), 1)) {
+                @Override
+                public RestResponse buildResponse(Empty empty, XContentBuilder builder) throws Exception {
+                    builderAtomicReference.set(builder);
+                    builder.close();
+                    return new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY);
+                }
+        };
+
+        builderListener.buildResponse(Empty.INSTANCE);
+        assertNotNull(builderAtomicReference.get());
+        assertTrue(builderAtomicReference.get().generator().isClosed());
+    }
+
+    public void testXContentBuilderNotClosedInBuildResponseAssertionsDisabled() throws Exception {
+        AtomicReference<XContentBuilder> builderAtomicReference = new AtomicReference<>();
+        RestBuilderListener<TransportResponse.Empty> builderListener =
+            new RestBuilderListener<Empty>(new FakeRestChannel(new FakeRestRequest(), randomBoolean(), 1)) {
+                @Override
+                public RestResponse buildResponse(Empty empty, XContentBuilder builder) throws Exception {
+                    builderAtomicReference.set(builder);
+                    return new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY);
+                }
+
+                @Override
+                boolean assertBuilderClosed(XContentBuilder xContentBuilder) {
+                    // don't check the actual builder being closed so we can test auto close
+                    return true;
+                }
+        };
+
+        builderListener.buildResponse(Empty.INSTANCE);
+        assertNotNull(builderAtomicReference.get());
+        assertTrue(builderAtomicReference.get().generator().isClosed());
+    }
+
+    public void testXContentBuilderNotClosedInBuildResponseAssertionsEnabled() throws Exception {
+        assumeTrue("tests are not being run with assertions", RestBuilderListener.class.desiredAssertionStatus());
+
+        RestBuilderListener<TransportResponse.Empty> builderListener =
+            new RestBuilderListener<Empty>(new FakeRestChannel(new FakeRestRequest(), randomBoolean(), 1)) {
+                @Override
+                public RestResponse buildResponse(Empty empty, XContentBuilder builder) throws Exception {
+                    return new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY);
+                }
+        };
+
+        AssertionError error = expectThrows(AssertionError.class, () -> builderListener.buildResponse(Empty.INSTANCE));
+        assertEquals("callers should ensure the XContentBuilder is closed themselves", error.getMessage());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/rest/action/RestMainActionTests.java b/core/src/test/java/org/elasticsearch/rest/action/RestMainActionTests.java
index 0c8a1c9a3cc51..13e6de063e81e 100644
--- a/core/src/test/java/org/elasticsearch/rest/action/RestMainActionTests.java
+++ b/core/src/test/java/org/elasticsearch/rest/action/RestMainActionTests.java
@@ -30,8 +30,6 @@
 import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.rest.RestRequest.Method;
-import org.elasticsearch.rest.action.RestMainAction;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.rest.FakeRestRequest;
 
@@ -45,12 +43,13 @@ public class RestMainActionTests extends ESTestCase {
     public void testHeadResponse() throws Exception {
         final String nodeName = "node1";
         final ClusterName clusterName = new ClusterName("cluster1");
+        final String clusterUUID = randomAlphaOfLengthBetween(10, 20);
         final boolean available = randomBoolean();
         final RestStatus expectedStatus = available ? RestStatus.OK : RestStatus.SERVICE_UNAVAILABLE;
         final Version version = Version.CURRENT;
         final Build build = Build.CURRENT;
 
-        final MainResponse mainResponse = new MainResponse(nodeName, version, clusterName, build, available);
+        final MainResponse mainResponse = new MainResponse(nodeName, version, clusterName, clusterUUID, build, available);
         XContentBuilder builder = JsonXContent.contentBuilder();
         RestRequest restRequest = new FakeRestRequest() {
             @Override
@@ -62,28 +61,29 @@ public Method method() {
         BytesRestResponse response = RestMainAction.convertMainResponse(mainResponse, restRequest, builder);
         assertNotNull(response);
         assertEquals(expectedStatus, response.status());
-        assertEquals(0, response.content().length());
 
-        assertEquals(0, builder.bytes().length());
+        // the empty responses are handled in the HTTP layer so we do
+        // not assert on them here
     }
 
     public void testGetResponse() throws Exception {
         final String nodeName = "node1";
         final ClusterName clusterName = new ClusterName("cluster1");
+        final String clusterUUID = randomAlphaOfLengthBetween(10, 20);
         final boolean available = randomBoolean();
         final RestStatus expectedStatus = available ? RestStatus.OK : RestStatus.SERVICE_UNAVAILABLE;
         final Version version = Version.CURRENT;
         final Build build = Build.CURRENT;
         final boolean prettyPrint = randomBoolean();
 
-        final MainResponse mainResponse = new MainResponse(nodeName, version, clusterName, build, available);
+        final MainResponse mainResponse = new MainResponse(nodeName, version, clusterName, clusterUUID, build, available);
         XContentBuilder builder = JsonXContent.contentBuilder();
 
         Map<String, String> params = new HashMap<>();
         if (prettyPrint == false) {
             params.put("pretty", String.valueOf(prettyPrint));
         }
-        RestRequest restRequest = new FakeRestRequest.Builder().withParams(params).build();
+        RestRequest restRequest = new FakeRestRequest.Builder(xContentRegistry()).withParams(params).build();
 
         BytesRestResponse response = RestMainAction.convertMainResponse(mainResponse, restRequest, builder);
         assertNotNull(response);
diff --git a/core/src/test/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsActionTests.java b/core/src/test/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsActionTests.java
new file mode 100644
index 0000000000000..fb4b9e3c74a7a
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsActionTests.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.rest.action.admin.cluster;
+
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.rest.FakeRestRequest;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.object.HasToString.hasToString;
+import static org.mockito.Mockito.mock;
+
+public class RestNodesStatsActionTests extends ESTestCase {
+
+    private RestNodesStatsAction action;
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        action = new RestNodesStatsAction(Settings.EMPTY, new RestController(Settings.EMPTY, Collections.emptySet(), null, null, null));
+    }
+
+    public void testUnrecognizedMetric() throws IOException {
+        final HashMap<String, String> params = new HashMap<>();
+        final String metric = randomAlphaOfLength(64);
+        params.put("metric", metric);
+        final RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withPath("/_nodes/stats").withParams(params).build();
+        final IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> action.prepareRequest(request, mock(NodeClient.class)));
+        assertThat(e, hasToString(containsString("request [/_nodes/stats] contains unrecognized metric: [" + metric + "]")));
+    }
+
+    public void testUnrecognizedMetricDidYouMean() throws IOException {
+        final HashMap<String, String> params = new HashMap<>();
+        params.put("metric", "os,transprot,unrecognized");
+        final RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withPath("/_nodes/stats").withParams(params).build();
+        final IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> action.prepareRequest(request, mock(NodeClient.class)));
+        assertThat(
+            e,
+            hasToString(
+                containsString(
+                    "request [/_nodes/stats] contains unrecognized metrics: [transprot] -> did you mean [transport]?, [unrecognized]")));
+    }
+
+    public void testAllRequestWithOtherMetrics() throws IOException {
+        final HashMap<String, String> params = new HashMap<>();
+        final String metric = randomSubsetOf(1, RestNodesStatsAction.METRICS.keySet()).get(0);
+        params.put("metric", "_all," + metric);
+        final RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withPath("/_nodes/stats").withParams(params).build();
+        final IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> action.prepareRequest(request, mock(NodeClient.class)));
+        assertThat(e, hasToString(containsString("request [/_nodes/stats] contains _all and individual metrics [_all," + metric + "]")));
+    }
+
+    public void testUnrecognizedIndexMetric() {
+        final HashMap<String, String> params = new HashMap<>();
+        params.put("metric", "indices");
+        final String indexMetric = randomAlphaOfLength(64);
+        params.put("index_metric", indexMetric);
+        final RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withPath("/_nodes/stats").withParams(params).build();
+        final IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> action.prepareRequest(request, mock(NodeClient.class)));
+        assertThat(e, hasToString(containsString("request [/_nodes/stats] contains unrecognized index metric: [" + indexMetric + "]")));
+    }
+
+    public void testUnrecognizedIndexMetricDidYouMean() {
+        final HashMap<String, String> params = new HashMap<>();
+        params.put("metric", "indices");
+        params.put("index_metric", "indexing,stroe,unrecognized");
+        final RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withPath("/_nodes/stats").withParams(params).build();
+        final IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> action.prepareRequest(request, mock(NodeClient.class)));
+        assertThat(
+            e,
+            hasToString(
+                containsString(
+                    "request [/_nodes/stats] contains unrecognized index metrics: [stroe] -> did you mean [store]?, [unrecognized]")));
+    }
+
+    public void testIndexMetricsWithPercolate() throws IOException {
+        final HashMap<String, String> params = new HashMap<>();
+        params.put("metric", "indices");
+        params.put("index_metric", "percolate");
+        final RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withPath("/_nodes/stats").withParams(params).build();
+        action.prepareRequest(request, mock(NodeClient.class));
+        assertWarnings("percolate stats are no longer available and requests for percolate stats will fail starting in 6.0.0");
+    }
+
+    public void testIndexMetricsRequestWithoutIndicesMetric() throws IOException {
+        final HashMap<String, String> params = new HashMap<>();
+        final Set<String> metrics = new HashSet<>(RestNodesStatsAction.METRICS.keySet());
+        metrics.remove("indices");
+        params.put("metric", randomSubsetOf(1, metrics).get(0));
+        final String indexMetric = randomSubsetOf(1, RestNodesStatsAction.FLAGS.keySet()).get(0);
+        params.put("index_metric", indexMetric);
+        final RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withPath("/_nodes/stats").withParams(params).build();
+        final IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> action.prepareRequest(request, mock(NodeClient.class)));
+        assertThat(
+            e,
+            hasToString(
+                containsString("request [/_nodes/stats] contains index metrics [" + indexMetric + "] but indices stats not requested")));
+    }
+
+    public void testIndexMetricsRequestOnAllRequest() throws IOException {
+        final HashMap<String, String> params = new HashMap<>();
+        params.put("metric", "_all");
+        final String indexMetric = randomSubsetOf(1, RestNodesStatsAction.FLAGS.keySet()).get(0);
+        params.put("index_metric", indexMetric);
+        final RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withPath("/_nodes/stats").withParams(params).build();
+        final IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> action.prepareRequest(request, mock(NodeClient.class)));
+        assertThat(
+            e,
+            hasToString(
+                containsString("request [/_nodes/stats] contains index metrics [" + indexMetric + "] but all stats requested")));
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/rest/action/admin/indices/RestAnalyzeActionTests.java b/core/src/test/java/org/elasticsearch/rest/action/admin/indices/RestAnalyzeActionTests.java
index 9b7d4073d0d9b..78ac4b5510fbe 100644
--- a/core/src/test/java/org/elasticsearch/rest/action/admin/indices/RestAnalyzeActionTests.java
+++ b/core/src/test/java/org/elasticsearch/rest/action/admin/indices/RestAnalyzeActionTests.java
@@ -19,30 +19,37 @@
 package org.elasticsearch.rest.action.admin.indices;
 
 import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.rest.FakeRestRequest;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.startsWith;
+import static org.mockito.Mockito.doCallRealMethod;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
 
 public class RestAnalyzeActionTests extends ESTestCase {
 
     public void testParseXContentForAnalyzeRequest() throws Exception {
-        BytesReference content =  XContentFactory.jsonBuilder()
+        XContentParser content = createParser(XContentFactory.jsonBuilder()
             .startObject()
-            .field("text", "THIS IS A TEST")
-            .field("tokenizer", "keyword")
-            .array("filter", "lowercase")
-            .endObject().bytes();
+                .field("text", "THIS IS A TEST")
+                .field("tokenizer", "keyword")
+                .array("filter", "lowercase")
+            .endObject());
 
         AnalyzeRequest analyzeRequest = new AnalyzeRequest("for test");
 
-        RestAnalyzeAction.buildFromContent(content, analyzeRequest, new ParseFieldMatcher(Settings.EMPTY));
+        RestAnalyzeAction.buildFromContent(content, analyzeRequest);
 
         assertThat(analyzeRequest.text().length, equalTo(1));
         assertThat(analyzeRequest.text(), equalTo(new String[]{"THIS IS A TEST"}));
@@ -54,28 +61,28 @@ public void testParseXContentForAnalyzeRequest() throws Exception {
     }
 
     public void testParseXContentForAnalyzeRequestWithCustomFilters() throws Exception {
-        BytesReference content =  XContentFactory.jsonBuilder()
+        XContentParser content = createParser(XContentFactory.jsonBuilder()
             .startObject()
-            .field("text", "THIS IS A TEST")
-            .field("tokenizer", "keyword")
-            .startArray("filter")
-            .value("lowercase")
-            .startObject()
-            .field("type", "stop")
-            .array("stopwords", "foo", "buzz")
-            .endObject()
-            .endArray()
-            .startArray("char_filter")
-            .startObject()
-            .field("type", "mapping")
-            .array("mappings", "ph => f", "qu => q")
-            .endObject()
-            .endArray()
-            .endObject().bytes();
+                .field("text", "THIS IS A TEST")
+                .field("tokenizer", "keyword")
+                .startArray("filter")
+                    .value("lowercase")
+                    .startObject()
+                        .field("type", "stop")
+                        .array("stopwords", "foo", "buzz")
+                    .endObject()
+                .endArray()
+                .startArray("char_filter")
+                    .startObject()
+                        .field("type", "mapping")
+                        .array("mappings", "ph => f", "qu => q")
+                    .endObject()
+                .endArray()
+            .endObject());
 
         AnalyzeRequest analyzeRequest = new AnalyzeRequest("for test");
 
-        RestAnalyzeAction.buildFromContent(content, analyzeRequest, new ParseFieldMatcher(Settings.EMPTY));
+        RestAnalyzeAction.buildFromContent(content, analyzeRequest);
 
         assertThat(analyzeRequest.text().length, equalTo(1));
         assertThat(analyzeRequest.text(), equalTo(new String[]{"THIS IS A TEST"}));
@@ -88,82 +95,97 @@ public void testParseXContentForAnalyzeRequestWithCustomFilters() throws Excepti
     }
 
     public void testParseXContentForAnalyzeRequestWithInvalidJsonThrowsException() throws Exception {
-        AnalyzeRequest analyzeRequest = new AnalyzeRequest("for test");
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-            () -> RestAnalyzeAction.buildFromContent(
-                new BytesArray("{invalid_json}"), analyzeRequest, new ParseFieldMatcher(Settings.EMPTY)));
+        RestAnalyzeAction action = new RestAnalyzeAction(Settings.EMPTY, mock(RestController.class));
+        RestRequest request = new FakeRestRequest.Builder(xContentRegistry())
+            .withContent(new BytesArray("{invalid_json}"), XContentType.JSON).build();
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> action.handleRequest(request, null, null));
         assertThat(e.getMessage(), equalTo("Failed to parse request body"));
     }
 
     public void testParseXContentForAnalyzeRequestWithUnknownParamThrowsException() throws Exception {
         AnalyzeRequest analyzeRequest = new AnalyzeRequest("for test");
-        BytesReference invalidContent = XContentFactory.jsonBuilder()
+        XContentParser invalidContent = createParser(XContentFactory.jsonBuilder()
             .startObject()
-            .field("text", "THIS IS A TEST")
-            .field("unknown", "keyword")
-            .endObject().bytes();
+                .field("text", "THIS IS A TEST")
+                .field("unknown", "keyword")
+            .endObject());
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-            () -> RestAnalyzeAction.buildFromContent(invalidContent, analyzeRequest, new ParseFieldMatcher(Settings.EMPTY)));
+                () -> RestAnalyzeAction.buildFromContent(invalidContent, analyzeRequest));
         assertThat(e.getMessage(), startsWith("Unknown parameter [unknown]"));
     }
 
     public void testParseXContentForAnalyzeRequestWithInvalidStringExplainParamThrowsException() throws Exception {
         AnalyzeRequest analyzeRequest = new AnalyzeRequest("for test");
-        BytesReference invalidExplain = XContentFactory.jsonBuilder()
+        XContentParser invalidExplain = createParser(XContentFactory.jsonBuilder()
             .startObject()
-            .field("explain", "fals")
-            .endObject().bytes();
+                .field("explain", "fals")
+            .endObject());
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-            () -> RestAnalyzeAction.buildFromContent(invalidExplain, analyzeRequest, new ParseFieldMatcher(Settings.EMPTY)));
+            () -> RestAnalyzeAction.buildFromContent(invalidExplain, analyzeRequest));
         assertThat(e.getMessage(), startsWith("explain must be either 'true' or 'false'"));
     }
 
-    public void testDeprecatedParamException() throws Exception {
+    public void testDeprecatedParamIn2xException() throws Exception {
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
             () -> RestAnalyzeAction.buildFromContent(
-                XContentFactory.jsonBuilder()
+                createParser(XContentFactory.jsonBuilder()
                     .startObject()
-                    .field("text", "THIS IS A TEST")
-                    .field("tokenizer", "keyword")
-                    .array("filters", "lowercase")
-                    .endObject().bytes(),
-                new AnalyzeRequest("for test"), new ParseFieldMatcher(Settings.EMPTY)));
+                        .field("text", "THIS IS A TEST")
+                        .field("tokenizer", "keyword")
+                        .array("filters", "lowercase")
+                    .endObject()),
+                new AnalyzeRequest("for test")));
         assertThat(e.getMessage(), startsWith("Unknown parameter [filters]"));
 
-
         e = expectThrows(IllegalArgumentException.class,
             () -> RestAnalyzeAction.buildFromContent(
-                XContentFactory.jsonBuilder()
+                createParser(XContentFactory.jsonBuilder()
                     .startObject()
-                    .field("text", "THIS IS A TEST")
-                    .field("tokenizer", "keyword")
-                    .array("token_filters", "lowercase")
-                    .endObject().bytes(),
-                new AnalyzeRequest("for test"), new ParseFieldMatcher(Settings.EMPTY)));
+                        .field("text", "THIS IS A TEST")
+                        .field("tokenizer", "keyword")
+                        .array("token_filters", "lowercase")
+                    .endObject()),
+                new AnalyzeRequest("for test")));
         assertThat(e.getMessage(), startsWith("Unknown parameter [token_filters]"));
 
-
-        e = expectThrows(IllegalArgumentException.class,
-            () -> RestAnalyzeAction.buildFromContent(
-                XContentFactory.jsonBuilder()
-                    .startObject()
+        XContentParser parser = createParser(XContentFactory.jsonBuilder()
+                .startObject()
                     .field("text", "THIS IS A TEST")
                     .field("tokenizer", "keyword")
                     .array("char_filters", "lowercase")
-                    .endObject().bytes(),
-                new AnalyzeRequest("for test"), new ParseFieldMatcher(Settings.EMPTY)));
+                .endObject());
+        e = expectThrows(IllegalArgumentException.class, () -> RestAnalyzeAction.buildFromContent(parser,
+                new AnalyzeRequest("for test")));
         assertThat(e.getMessage(), startsWith("Unknown parameter [char_filters]"));
 
         e = expectThrows(IllegalArgumentException.class,
             () -> RestAnalyzeAction.buildFromContent(
-                XContentFactory.jsonBuilder()
+                createParser(XContentFactory.jsonBuilder()
                     .startObject()
                     .field("text", "THIS IS A TEST")
                     .field("tokenizer", "keyword")
                     .array("token_filter", "lowercase")
-                    .endObject().bytes()
-                , new AnalyzeRequest("for test"), new ParseFieldMatcher(Settings.EMPTY)));
+                    .endObject())
+                , new AnalyzeRequest("for test")));
         assertThat(e.getMessage(), startsWith("Unknown parameter [token_filter]"));
     }
 
+    public void testDeprecatedReqParamsIn5x() throws Exception {
+        RestAnalyzeAction action = mock(RestAnalyzeAction.class);
+        AnalyzeRequest analyzeRequest = new AnalyzeRequest();
+
+        BytesReference content = new BytesArray("this is test");
+        FakeRestRequest request = new FakeRestRequest.Builder(xContentRegistry())
+            .withContent(content, null)
+            .withMethod(randomFrom(RestRequest.Method.values()))
+            .build();
+
+        doCallRealMethod().when(action).handleBodyContent(request, null, analyzeRequest);
+
+        action.handleBodyContent(request, null, analyzeRequest);
+        verify(action).deprecationLogForText(" plain text bodies is deprecated and " +
+            "this feature will be removed in the next major release. Please use the text param in JSON");
+        assertThat(analyzeRequest.text().length, equalTo(1));
+        assertThat(analyzeRequest.text()[0], equalTo("this is test"));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/rest/action/admin/indices/RestClearIndicesCacheActionTests.java b/core/src/test/java/org/elasticsearch/rest/action/admin/indices/RestClearIndicesCacheActionTests.java
new file mode 100644
index 0000000000000..25a8f350d9a32
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/rest/action/admin/indices/RestClearIndicesCacheActionTests.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.rest.action.admin.indices;
+
+import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.rest.FakeRestRequest;
+
+import java.util.HashMap;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class RestClearIndicesCacheActionTests extends ESTestCase {
+
+    public void testRequestCacheSet() throws Exception {
+        final HashMap<String, String> params = new HashMap<>();
+        params.put("request", "true");
+        final RestRequest restRequest = new FakeRestRequest.Builder(xContentRegistry())
+                                                           .withParams(params).build();
+        ClearIndicesCacheRequest cacheRequest = new ClearIndicesCacheRequest();
+        cacheRequest = RestClearIndicesCacheAction.fromRequest(restRequest, cacheRequest);
+        assertThat(cacheRequest.requestCache(), equalTo(true));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsActionTests.java b/core/src/test/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsActionTests.java
new file mode 100644
index 0000000000000..9321a2488ea90
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsActionTests.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.rest.action.admin.indices;
+
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.rest.FakeRestRequest;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.object.HasToString.hasToString;
+import static org.mockito.Mockito.mock;
+
+public class RestIndicesStatsActionTests extends ESTestCase {
+
+    private RestIndicesStatsAction action;
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        action = new RestIndicesStatsAction(Settings.EMPTY, new RestController(Settings.EMPTY, Collections.emptySet(), null, null, null));
+    }
+
+    public void testUnrecognizedMetric() throws IOException {
+        final HashMap<String, String> params = new HashMap<>();
+        final String metric = randomAlphaOfLength(64);
+        params.put("metric", metric);
+        final RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withPath("/_stats").withParams(params).build();
+        final IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> action.prepareRequest(request, mock(NodeClient.class)));
+        assertThat(e, hasToString(containsString("request [/_stats] contains unrecognized metric: [" + metric + "]")));
+    }
+
+    public void testUnrecognizedMetricDidYouMean() throws IOException {
+        final HashMap<String, String> params = new HashMap<>();
+        params.put("metric", "request_cache,fieldata,unrecognized");
+        final RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withPath("/_stats").withParams(params).build();
+        final IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> action.prepareRequest(request, mock(NodeClient.class)));
+        assertThat(
+            e,
+            hasToString(
+                containsString(
+                    "request [/_stats] contains unrecognized metrics: [fieldata] -> did you mean [fielddata]?, [unrecognized]")));
+    }
+
+    public void testAllRequestWithOtherMetrics() throws IOException {
+        final HashMap<String, String> params = new HashMap<>();
+        final String metric = randomSubsetOf(1, RestIndicesStatsAction.METRICS.keySet()).get(0);
+        params.put("metric", "_all," + metric);
+        final RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withPath("/_stats").withParams(params).build();
+        final IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> action.prepareRequest(request, mock(NodeClient.class)));
+        assertThat(e, hasToString(containsString("request [/_stats] contains _all and individual metrics [_all," + metric + "]")));
+    }
+
+    public void testIndexMetricsWithPercolate() throws IOException {
+        final HashMap<String, String> params = new HashMap<>();
+        params.put("metric", "percolate");
+        final RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withPath("/_stats").withParams(params).build();
+        action.prepareRequest(request, mock(NodeClient.class));
+        assertWarnings("percolate stats are no longer available and requests for percolate stats will fail starting in 6.0.0");
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java b/core/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java
index 6fc0333fecf2c..2a3fc746c9ca6 100644
--- a/core/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java
+++ b/core/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java
@@ -57,6 +57,7 @@
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.search.suggest.completion.CompletionStats;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.rest.FakeRestRequest;
 
 import java.nio.file.Path;
 import java.util.ArrayList;
@@ -73,14 +74,14 @@ public class RestIndicesActionTests extends ESTestCase {
 
     public void testBuildTable() {
         final Settings settings = Settings.EMPTY;
-        final RestController restController = new RestController(settings, Collections.emptySet());
+        final RestController restController = new RestController(settings, Collections.emptySet(), null, null, null);
         final RestIndicesAction action = new RestIndicesAction(settings, restController, new IndexNameExpressionResolver(settings));
 
         // build a (semi-)random table
         final int numIndices = randomIntBetween(0, 5);
         Index[] indices = new Index[numIndices];
         for (int i = 0; i < numIndices; i++) {
-            indices[i] = new Index(randomAsciiOfLength(5), UUIDs.randomBase64UUID());
+            indices[i] = new Index(randomAlphaOfLength(5), UUIDs.randomBase64UUID());
         }
 
         final MetaData.Builder metaDataBuilder = MetaData.builder();
@@ -107,7 +108,7 @@ public void testBuildTable() {
             clusterState.getClusterName().value(), indicesStr, clusterState, 0, 0, 0, TimeValue.timeValueMillis(1000L)
         );
 
-        final Table table = action.buildTable(null, indices, clusterHealth, randomIndicesStatsResponse(indices), metaData);
+        final Table table = action.buildTable(new FakeRestRequest(), indices, clusterHealth, randomIndicesStatsResponse(indices), metaData);
 
         // now, verify the table is correct
         int count = 0;
@@ -133,11 +134,14 @@ public void testBuildTable() {
     private IndicesStatsResponse randomIndicesStatsResponse(final Index[] indices) {
         List<ShardStats> shardStats = new ArrayList<>();
         for (final Index index : indices) {
-            for (int i = 0; i < 2; i++) {
+            int numShards = randomInt(5);
+            int primaryIdx = randomIntBetween(-1, numShards - 1); // -1 means there is no primary shard.
+            for (int i = 0; i < numShards; i++) {
                 ShardId shardId = new ShardId(index, i);
+                boolean primary = (i == primaryIdx);
                 Path path = createTempDir().resolve("indices").resolve(index.getUUID()).resolve(String.valueOf(i));
-                ShardRouting shardRouting = ShardRouting.newUnassigned(shardId, i == 0,
-                    i == 0 ? StoreRecoverySource.EMPTY_STORE_INSTANCE : PeerRecoverySource.INSTANCE,
+                ShardRouting shardRouting = ShardRouting.newUnassigned(shardId, primary,
+                    primary ? StoreRecoverySource.EMPTY_STORE_INSTANCE : PeerRecoverySource.INSTANCE,
                     new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)
                     );
                 shardRouting = shardRouting.initialize("node-0", null, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE);
diff --git a/core/src/test/java/org/elasticsearch/rest/action/cat/RestRecoveryActionTests.java b/core/src/test/java/org/elasticsearch/rest/action/cat/RestRecoveryActionTests.java
index 9495ba7e995a0..6fce7219d67c2 100644
--- a/core/src/test/java/org/elasticsearch/rest/action/cat/RestRecoveryActionTests.java
+++ b/core/src/test/java/org/elasticsearch/rest/action/cat/RestRecoveryActionTests.java
@@ -50,8 +50,8 @@ public class RestRecoveryActionTests extends ESTestCase {
 
     public void testRestRecoveryAction() {
         final Settings settings = Settings.EMPTY;
-        final RestController restController = new RestController(settings, Collections.emptySet());
-        final RestRecoveryAction action = new RestRecoveryAction(settings, restController, restController);
+        final RestController restController = new RestController(settings, Collections.emptySet(), null, null, null);
+        final RestRecoveryAction action = new RestRecoveryAction(settings, restController);
         final int totalShards = randomIntBetween(1, 32);
         final int successfulShards = Math.max(0, totalShards - randomIntBetween(1, 2));
         final int failedShards = totalShards - successfulShards;
@@ -69,11 +69,11 @@ public void testRestRecoveryAction() {
             when(state.getStage()).thenReturn(randomFrom(RecoveryState.Stage.values()));
             final DiscoveryNode sourceNode = randomBoolean() ? mock(DiscoveryNode.class) : null;
             if (sourceNode != null) {
-                when(sourceNode.getHostName()).thenReturn(randomAsciiOfLength(8));
+                when(sourceNode.getHostName()).thenReturn(randomAlphaOfLength(8));
             }
             when(state.getSourceNode()).thenReturn(sourceNode);
             final DiscoveryNode targetNode = mock(DiscoveryNode.class);
-            when(targetNode.getHostName()).thenReturn(randomAsciiOfLength(8));
+            when(targetNode.getHostName()).thenReturn(randomAlphaOfLength(8));
             when(state.getTargetNode()).thenReturn(targetNode);
 
             RecoveryState.Index index = mock(RecoveryState.Index.class);
diff --git a/core/src/test/java/org/elasticsearch/rest/action/cat/RestTableTests.java b/core/src/test/java/org/elasticsearch/rest/action/cat/RestTableTests.java
index f7dc6149052ad..4f7bc1bedd9d6 100644
--- a/core/src/test/java/org/elasticsearch/rest/action/cat/RestTableTests.java
+++ b/core/src/test/java/org/elasticsearch/rest/action/cat/RestTableTests.java
@@ -23,12 +23,12 @@
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.rest.AbstractRestChannel;
 import org.elasticsearch.rest.RestResponse;
-import org.elasticsearch.rest.action.cat.RestTable;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.rest.FakeRestRequest;
 import org.junit.Before;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
@@ -62,11 +62,13 @@ public class RestTableTests extends ESTestCase {
             "  invalidAliasesBulk: \"foo\"\n" +
             "  timestamp: \"foo\"\n" +
             "  epoch: \"foo\"\n";
-    private Table table = new Table();
-    private FakeRestRequest restRequest = new FakeRestRequest();
+    private Table table;
+    private FakeRestRequest restRequest;
 
     @Before
     public void setup() {
+        restRequest = new FakeRestRequest();
+        table = new Table();
         table.startHeaders();
         table.addCell("bulk.foo", "alias:f;desc:foo");
         table.addCell("bulk.bar", "alias:b;desc:bar");
@@ -102,42 +104,42 @@ public void testThatDisplayHeadersAreNotAddedTwice() throws Exception {
     }
 
     public void testThatWeUseTheAcceptHeaderJson() throws Exception {
-        assertResponse(Collections.singletonMap(ACCEPT, APPLICATION_JSON),
+        assertResponse(Collections.singletonMap(ACCEPT, Collections.singletonList(APPLICATION_JSON)),
                 APPLICATION_JSON,
                 JSON_TABLE_BODY);
     }
 
     public void testThatWeUseTheAcceptHeaderYaml() throws Exception {
-        assertResponse(Collections.singletonMap(ACCEPT, APPLICATION_YAML),
+        assertResponse(Collections.singletonMap(ACCEPT, Collections.singletonList(APPLICATION_YAML)),
                 APPLICATION_YAML,
                 YAML_TABLE_BODY);
     }
 
     public void testThatWeUseTheAcceptHeaderSmile() throws Exception {
-        assertResponseContentType(Collections.singletonMap(ACCEPT, APPLICATION_SMILE),
+        assertResponseContentType(Collections.singletonMap(ACCEPT, Collections.singletonList(APPLICATION_SMILE)),
                 APPLICATION_SMILE);
     }
 
     public void testThatWeUseTheAcceptHeaderCbor() throws Exception {
-        assertResponseContentType(Collections.singletonMap(ACCEPT, APPLICATION_CBOR),
+        assertResponseContentType(Collections.singletonMap(ACCEPT, Collections.singletonList(APPLICATION_CBOR)),
                 APPLICATION_CBOR);
     }
 
     public void testThatWeUseTheAcceptHeaderText() throws Exception {
-        assertResponse(Collections.singletonMap(ACCEPT, TEXT_PLAIN),
+        assertResponse(Collections.singletonMap(ACCEPT, Collections.singletonList(TEXT_PLAIN)),
                 TEXT_PLAIN,
                 TEXT_TABLE_BODY);
     }
 
     public void testIgnoreContentType() throws Exception {
-        assertResponse(Collections.singletonMap(CONTENT_TYPE, APPLICATION_JSON),
+        assertResponse(Collections.singletonMap(CONTENT_TYPE, Collections.singletonList(APPLICATION_JSON)),
                 TEXT_PLAIN,
                 TEXT_TABLE_BODY);
     }
 
     public void testThatDisplayHeadersWithoutTimestamp() throws Exception {
         restRequest.params().put("h", "timestamp,epoch,bulk*");
-        restRequest.params().put("ts", "0");
+        restRequest.params().put("ts", "false");
         List<RestTable.DisplayHeader> headers = buildDisplayHeaders(table, restRequest);
 
         List<String> headerNames = getHeaderNames(headers);
@@ -146,8 +148,112 @@ public void testThatDisplayHeadersWithoutTimestamp() throws Exception {
         assertThat(headerNames, not(hasItem("epoch")));
     }
 
-    private RestResponse assertResponseContentType(Map<String, String> headers, String mediaType) throws Exception {
-        FakeRestRequest requestWithAcceptHeader = new FakeRestRequest.Builder().withHeaders(headers).build();
+    public void testCompareRow() {
+        Table table = new Table();
+        table.startHeaders();
+        table.addCell("compare");
+        table.endHeaders();
+
+        for (Integer i : Arrays.asList(1,2,1)) {
+            table.startRow();
+            table.addCell(i);
+            table.endRow();
+        }
+
+        RestTable.TableIndexComparator comparator = new RestTable.TableIndexComparator(table,
+            Collections.singletonList(new RestTable.ColumnOrderElement("compare", false)));
+        assertTrue(comparator.compare(0,1) < 0);
+        assertTrue(comparator.compare(0,2) == 0);
+        assertTrue(comparator.compare(1,2) > 0);
+
+        RestTable.TableIndexComparator reverseComparator = new RestTable.TableIndexComparator(table,
+            Collections.singletonList(new RestTable.ColumnOrderElement("compare", true)));
+
+        assertTrue(reverseComparator.compare(0,1) > 0);
+        assertTrue(reverseComparator.compare(0,2) == 0);
+        assertTrue(reverseComparator.compare(1,2) < 0);
+    }
+
+    public void testRowOutOfBounds() {
+        Table table = new Table();
+        table.startHeaders();
+        table.addCell("compare");
+        table.endHeaders();
+        RestTable.TableIndexComparator comparator = new RestTable.TableIndexComparator(table,
+            Collections.singletonList(new RestTable.ColumnOrderElement("compare", false)));
+        Error e = expectThrows(AssertionError.class, () -> {
+            comparator.compare(0,1);
+        });
+        assertEquals("Invalid comparison of indices (0, 1): Table has 0 rows.", e.getMessage());
+    }
+
+    public void testUnknownHeader() {
+        Table table = new Table();
+        table.startHeaders();
+        table.addCell("compare");
+        table.endHeaders();
+        restRequest.params().put("s", "notaheader");
+        Exception e = expectThrows(UnsupportedOperationException.class, () -> RestTable.getRowOrder(table, restRequest));
+        assertEquals("Unable to sort by unknown sort key `notaheader`", e.getMessage());
+    }
+
+    public void testAliasSort() {
+        Table table = new Table();
+        table.startHeaders();
+        table.addCell("compare", "alias:c;");
+        table.endHeaders();
+        List<Integer> comparisonList = Arrays.asList(3,1,2);
+        for (int i = 0; i < comparisonList.size(); i++) {
+            table.startRow();
+            table.addCell(comparisonList.get(i));
+            table.endRow();
+        }
+        restRequest.params().put("s", "c");
+        List<Integer> rowOrder = RestTable.getRowOrder(table, restRequest);
+        assertEquals(Arrays.asList(1,2,0), rowOrder);
+    }
+
+    public void testReversedSort() {
+        Table table = new Table();
+        table.startHeaders();
+        table.addCell("reversed");
+        table.endHeaders();
+        List<Integer> comparisonList = Arrays.asList(0, 1, 2);
+        for (int i = 0; i < comparisonList.size(); i++) {
+            table.startRow();
+            table.addCell(comparisonList.get(i));
+            table.endRow();
+        }
+        restRequest.params().put("s", "reversed:desc");
+        List<Integer> rowOrder = RestTable.getRowOrder(table, restRequest);
+        assertEquals(Arrays.asList(2,1,0), rowOrder);
+    }
+
+    public void testMultiSort() {
+        Table table = new Table();
+        table.startHeaders();
+        table.addCell("compare");
+        table.addCell("second.compare");
+        table.endHeaders();
+        List<Integer> comparisonList = Arrays.asList(3, 3, 2);
+        List<Integer> secondComparisonList = Arrays.asList(2, 1, 3);
+        for (int i = 0; i < comparisonList.size(); i++) {
+            table.startRow();
+            table.addCell(comparisonList.get(i));
+            table.addCell(secondComparisonList.get(i));
+            table.endRow();
+        }
+        restRequest.params().put("s", "compare,second.compare");
+        List<Integer> rowOrder = RestTable.getRowOrder(table, restRequest);
+        assertEquals(Arrays.asList(2,1,0), rowOrder);
+
+        restRequest.params().put("s", "compare:desc,second.compare");
+        rowOrder = RestTable.getRowOrder(table, restRequest);
+        assertEquals(Arrays.asList(1,0,2), rowOrder);
+    }
+
+    private RestResponse assertResponseContentType(Map<String, List<String>> headers, String mediaType) throws Exception {
+        FakeRestRequest requestWithAcceptHeader = new FakeRestRequest.Builder(xContentRegistry()).withHeaders(headers).build();
         table.startRow();
         table.addCell("foo");
         table.addCell("foo");
@@ -168,7 +274,7 @@ public void sendResponse(RestResponse response) {
         return response;
     }
 
-    private void assertResponse(Map<String, String> headers, String mediaType, String body) throws Exception {
+    private void assertResponse(Map<String, List<String>> headers, String mediaType, String body) throws Exception {
         RestResponse response = assertResponseContentType(headers, mediaType);
         assertThat(response.content().utf8ToString(), equalTo(body));
     }
diff --git a/core/src/test/java/org/elasticsearch/routing/AliasRoutingIT.java b/core/src/test/java/org/elasticsearch/routing/AliasRoutingIT.java
index 621ade9e31de9..dc8b6b54464e4 100644
--- a/core/src/test/java/org/elasticsearch/routing/AliasRoutingIT.java
+++ b/core/src/test/java/org/elasticsearch/routing/AliasRoutingIT.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
+import org.elasticsearch.client.Requests;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -64,7 +65,7 @@ public void testAliasCrudRouting() throws Exception {
         logger.info("--> updating with id [1] and routing through alias");
         client().prepareUpdate("alias0", "type1", "1")
                 .setUpsert(XContentFactory.jsonBuilder().startObject().field("field", 1).endObject())
-                .setDoc("field", "value2")
+                .setDoc(Requests.INDEX_CONTENT_TYPE, "field", "value2")
                 .execute().actionGet();
         for (int i = 0; i < 5; i++) {
             assertThat(client().prepareGet("alias0", "type1", "1").execute().actionGet().isExists(), equalTo(true));
diff --git a/core/src/test/java/org/elasticsearch/routing/PartitionedRoutingIT.java b/core/src/test/java/org/elasticsearch/routing/PartitionedRoutingIT.java
new file mode 100644
index 0000000000000..2f32e3830c1cb
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/routing/PartitionedRoutingIT.java
@@ -0,0 +1,199 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.routing;
+
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.mockito.internal.util.collections.Sets;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+public class PartitionedRoutingIT extends ESIntegTestCase {
+
+    public void testVariousPartitionSizes() throws Exception {
+        for (int shards = 1; shards <= 4; shards++) {
+            for (int partitionSize = 1; partitionSize < shards; partitionSize++) {
+                String index = "index_" + shards + "_" + partitionSize;
+
+                client().admin().indices().prepareCreate(index)
+                    .setSettings(Settings.builder()
+                        .put("index.number_of_shards", shards)
+                        .put("index.routing_partition_size", partitionSize))
+                    .addMapping("type", "{\"type\":{\"_routing\":{\"required\":true}}}")
+                    .execute().actionGet();
+                ensureGreen();
+
+                Map<String, Set<String>> routingToDocumentIds = generateRoutedDocumentIds(index);
+
+                verifyGets(index, routingToDocumentIds);
+                verifyBroadSearches(index, routingToDocumentIds, shards);
+                verifyRoutedSearches(index, routingToDocumentIds, Sets.newSet(partitionSize));
+            }
+        }
+    }
+
+    public void testShrinking() throws Exception {
+        // creates random routing groups and repeatedly halves the index until it is down to 1 shard
+        // verifying that the count is correct for each shrunken index
+        final int partitionSize = 3;
+        final int originalShards = 8;
+        int currentShards = originalShards;
+        String index = "index_" + currentShards;
+
+        client().admin().indices().prepareCreate(index)
+            .setSettings(Settings.builder()
+                .put("index.number_of_shards", currentShards)
+                .put("index.number_of_replicas", numberOfReplicas())
+                .put("index.routing_partition_size", partitionSize))
+            .addMapping("type", "{\"type\":{\"_routing\":{\"required\":true}}}")
+            .execute().actionGet();
+        ensureGreen();
+
+        Map<String, Set<String>> routingToDocumentIds = generateRoutedDocumentIds(index);
+
+        while (true) {
+            int factor = originalShards / currentShards;
+
+            verifyGets(index, routingToDocumentIds);
+            verifyBroadSearches(index, routingToDocumentIds, currentShards);
+
+            // we need the floor and ceiling of the routing_partition_size / factor since the partition size of the shrunken
+            // index will be one of those, depending on the routing value
+            verifyRoutedSearches(index, routingToDocumentIds,
+                Math.floorDiv(partitionSize, factor) == 0 ?
+                    Sets.newSet(1, 2) :
+                    Sets.newSet(Math.floorDiv(partitionSize, factor), -Math.floorDiv(-partitionSize, factor)));
+
+            client().admin().indices().prepareUpdateSettings(index)
+                .setSettings(Settings.builder()
+                    .put("index.routing.allocation.require._name", client().admin().cluster().prepareState().get().getState().nodes()
+                        .getDataNodes().values().toArray(DiscoveryNode.class)[0].getName())
+                    .put("index.blocks.write", true)).get();
+            ensureGreen();
+
+            currentShards = Math.floorDiv(currentShards, 2);
+
+            if (currentShards == 0) {
+                break;
+            }
+
+            String previousIndex = index;
+            index = "index_" + currentShards;
+
+            logger.info("--> shrinking index [" + previousIndex + "] to [" + index + "]");
+            client().admin().indices().prepareShrinkIndex(previousIndex, index)
+                .setSettings(Settings.builder()
+                    .put("index.number_of_shards", currentShards)
+                    .put("index.number_of_replicas", numberOfReplicas())
+                    .build()).get();
+            ensureGreen();
+        }
+    }
+
+    private void verifyRoutedSearches(String index, Map<String, Set<String>> routingToDocumentIds, Set<Integer> expectedShards) {
+        for (Map.Entry<String, Set<String>> routingEntry : routingToDocumentIds.entrySet()) {
+            String routing = routingEntry.getKey();
+            int expectedDocuments = routingEntry.getValue().size();
+
+            SearchResponse response  = client().prepareSearch()
+                .setQuery(QueryBuilders.termQuery("_routing", routing))
+                .setRouting(routing)
+                .setIndices(index)
+                .setSize(100)
+                .execute().actionGet();
+
+            logger.info("--> routed search on index [" + index + "] visited [" + response.getTotalShards()
+                + "] shards for routing [" + routing + "] and got hits [" + response.getHits().totalHits() + "]");
+
+            assertTrue(response.getTotalShards() + " was not in " + expectedShards + " for " + index,
+                    expectedShards.contains(response.getTotalShards()));
+            assertEquals(expectedDocuments, response.getHits().totalHits());
+
+            Set<String> found = new HashSet<>();
+            response.getHits().forEach(h -> found.add(h.getId()));
+
+            assertEquals(routingEntry.getValue(), found);
+        }
+    }
+
+    private void verifyBroadSearches(String index, Map<String, Set<String>> routingToDocumentIds, int expectedShards) {
+        for (Map.Entry<String, Set<String>> routingEntry : routingToDocumentIds.entrySet()) {
+            String routing = routingEntry.getKey();
+            int expectedDocuments = routingEntry.getValue().size();
+
+            SearchResponse response  = client().prepareSearch()
+                .setQuery(QueryBuilders.termQuery("_routing", routing))
+                .setIndices(index)
+                .setSize(100)
+                .execute().actionGet();
+
+            assertEquals(expectedShards, response.getTotalShards());
+            assertEquals(expectedDocuments, response.getHits().totalHits());
+
+            Set<String> found = new HashSet<>();
+            response.getHits().forEach(h -> found.add(h.getId()));
+
+            assertEquals(routingEntry.getValue(), found);
+        }
+    }
+
+    private void verifyGets(String index, Map<String, Set<String>> routingToDocumentIds) {
+        for (Map.Entry<String, Set<String>> routingEntry : routingToDocumentIds.entrySet()) {
+            String routing = routingEntry.getKey();
+
+            for (String id : routingEntry.getValue()) {
+                assertTrue(client().prepareGet(index, "type", id).setRouting(routing).execute().actionGet().isExists());
+            }
+        }
+    }
+
+    private Map<String, Set<String>> generateRoutedDocumentIds(String index) {
+        Map<String, Set<String>> routingToDocumentIds = new HashMap<>();
+        int numRoutingValues = randomIntBetween(5, 15);
+
+        for (int i = 0; i < numRoutingValues; i++) {
+            String routingValue = String.valueOf(i);
+            int numDocuments = randomIntBetween(10, 100);
+            routingToDocumentIds.put(String.valueOf(routingValue), new HashSet<>());
+
+            for (int k = 0; k < numDocuments; k++) {
+                String id = routingValue + "_" + String.valueOf(k);
+                routingToDocumentIds.get(routingValue).add(id);
+
+                client().prepareIndex(index, "type", id)
+                    .setRouting(routingValue)
+                    .setSource("foo", "bar")
+                    .get();
+            }
+        }
+
+        client().admin().indices().prepareRefresh(index).get();
+
+        return routingToDocumentIds;
+    }
+
+
+}
diff --git a/core/src/test/java/org/elasticsearch/routing/SimpleRoutingIT.java b/core/src/test/java/org/elasticsearch/routing/SimpleRoutingIT.java
index d8cf1e7b5ec61..b027a13e867e1 100644
--- a/core/src/test/java/org/elasticsearch/routing/SimpleRoutingIT.java
+++ b/core/src/test/java/org/elasticsearch/routing/SimpleRoutingIT.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.routing;
 
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.RoutingMissingException;
 import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.action.bulk.BulkItemResponse;
@@ -206,13 +207,13 @@ public void testRequiredRoutingCrudApis() throws Exception {
         }
 
         try {
-            client().prepareUpdate(indexOrAlias(), "type1", "1").setDoc("field", "value2").execute().actionGet();
+            client().prepareUpdate(indexOrAlias(), "type1", "1").setDoc(Requests.INDEX_CONTENT_TYPE, "field", "value2").execute().actionGet();
             fail("update with missing routing when routing is required should fail");
         } catch(ElasticsearchException e) {
             assertThat(e.unwrapCause(), instanceOf(RoutingMissingException.class));
         }
 
-        client().prepareUpdate(indexOrAlias(), "type1", "1").setRouting("0").setDoc("field", "value2").execute().actionGet();
+        client().prepareUpdate(indexOrAlias(), "type1", "1").setRouting("0").setDoc(Requests.INDEX_CONTENT_TYPE, "field", "value2").get();
         client().admin().indices().prepareRefresh().execute().actionGet();
 
         for (int i = 0; i < 5; i++) {
@@ -253,13 +254,13 @@ public void testRequiredRoutingBulk() throws Exception {
 
         {
             BulkResponse bulkResponse = client().prepareBulk().add(Requests.indexRequest(indexOrAlias()).type("type1").id("1")
-                .source("field", "value")).execute().actionGet();
+                .source(Requests.INDEX_CONTENT_TYPE, "field", "value")).execute().actionGet();
             assertThat(bulkResponse.getItems().length, equalTo(1));
             assertThat(bulkResponse.hasFailures(), equalTo(true));
 
             for (BulkItemResponse bulkItemResponse : bulkResponse) {
                 assertThat(bulkItemResponse.isFailed(), equalTo(true));
-                assertThat(bulkItemResponse.getOpType(), equalTo("index"));
+                assertThat(bulkItemResponse.getOpType(), equalTo(DocWriteRequest.OpType.INDEX));
                 assertThat(bulkItemResponse.getFailure().getStatus(), equalTo(RestStatus.BAD_REQUEST));
                 assertThat(bulkItemResponse.getFailure().getCause(), instanceOf(RoutingMissingException.class));
                 assertThat(bulkItemResponse.getFailureMessage(), containsString("routing is required for [test]/[type1]/[1]"));
@@ -268,19 +269,20 @@ public void testRequiredRoutingBulk() throws Exception {
 
         {
             BulkResponse bulkResponse = client().prepareBulk().add(Requests.indexRequest(indexOrAlias()).type("type1").id("1").routing("0")
-                .source("field", "value")).execute().actionGet();
+                .source(Requests.INDEX_CONTENT_TYPE, "field", "value")).execute().actionGet();
             assertThat(bulkResponse.hasFailures(), equalTo(false));
         }
 
         {
-            BulkResponse bulkResponse = client().prepareBulk().add(new UpdateRequest(indexOrAlias(), "type1", "1").doc("field", "value2"))
+            BulkResponse bulkResponse = client().prepareBulk().add(new UpdateRequest(indexOrAlias(), "type1", "1")
+                .doc(Requests.INDEX_CONTENT_TYPE, "field", "value2"))
                 .execute().actionGet();
             assertThat(bulkResponse.getItems().length, equalTo(1));
             assertThat(bulkResponse.hasFailures(), equalTo(true));
 
             for (BulkItemResponse bulkItemResponse : bulkResponse) {
                 assertThat(bulkItemResponse.isFailed(), equalTo(true));
-                assertThat(bulkItemResponse.getOpType(), equalTo("update"));
+                assertThat(bulkItemResponse.getOpType(), equalTo(DocWriteRequest.OpType.UPDATE));
                 assertThat(bulkItemResponse.getFailure().getStatus(), equalTo(RestStatus.BAD_REQUEST));
                 assertThat(bulkItemResponse.getFailure().getCause(), instanceOf(RoutingMissingException.class));
                 assertThat(bulkItemResponse.getFailureMessage(), containsString("routing is required for [test]/[type1]/[1]"));
@@ -288,7 +290,8 @@ public void testRequiredRoutingBulk() throws Exception {
         }
 
         {
-            BulkResponse bulkResponse = client().prepareBulk().add(new UpdateRequest(indexOrAlias(), "type1", "1").doc("field", "value2")
+            BulkResponse bulkResponse = client().prepareBulk().add(new UpdateRequest(indexOrAlias(), "type1", "1")
+                .doc(Requests.INDEX_CONTENT_TYPE, "field", "value2")
                 .routing("0")).execute().actionGet();
             assertThat(bulkResponse.hasFailures(), equalTo(false));
         }
@@ -301,7 +304,7 @@ public void testRequiredRoutingBulk() throws Exception {
 
             for (BulkItemResponse bulkItemResponse : bulkResponse) {
                 assertThat(bulkItemResponse.isFailed(), equalTo(true));
-                assertThat(bulkItemResponse.getOpType(), equalTo("delete"));
+                assertThat(bulkItemResponse.getOpType(), equalTo(DocWriteRequest.OpType.DELETE));
                 assertThat(bulkItemResponse.getFailure().getStatus(), equalTo(RestStatus.BAD_REQUEST));
                 assertThat(bulkItemResponse.getFailure().getCause(), instanceOf(RoutingMissingException.class));
                 assertThat(bulkItemResponse.getFailureMessage(), containsString("routing is required for [test]/[type1]/[1]"));
@@ -368,12 +371,12 @@ public void testRequiredRoutingMappingVariousAPIs() throws Exception {
         }
 
         UpdateResponse updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1").setRouting("0")
-                .setDoc("field1", "value1").get();
+                .setDoc(Requests.INDEX_CONTENT_TYPE, "field1", "value1").get();
         assertThat(updateResponse.getId(), equalTo("1"));
         assertThat(updateResponse.getVersion(), equalTo(2L));
 
         try {
-            client().prepareUpdate(indexOrAlias(), "type1", "1").setDoc("field1", "value1").get();
+            client().prepareUpdate(indexOrAlias(), "type1", "1").setDoc(Requests.INDEX_CONTENT_TYPE, "field1", "value1").get();
             fail();
         } catch (RoutingMissingException e) {
             assertThat(e.getMessage(), equalTo("routing is required for [test]/[type1]/[1]"));
diff --git a/core/src/test/java/org/elasticsearch/script/FileScriptTests.java b/core/src/test/java/org/elasticsearch/script/FileScriptTests.java
index 954de105a4f67..0ad3bb08ab21a 100644
--- a/core/src/test/java/org/elasticsearch/script/FileScriptTests.java
+++ b/core/src/test/java/org/elasticsearch/script/FileScriptTests.java
@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.script;
 
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.script.MockScriptEngine.MockCompiledScript;
@@ -26,12 +27,12 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Collections;
-import java.util.HashSet;
-import java.util.Set;
 
 // TODO: these really should just be part of ScriptService tests, there is nothing special about them
 public class FileScriptTests extends ESTestCase {
 
+    private ScriptSettings scriptSettings;
+
     ScriptService makeScriptService(Settings settings) throws Exception {
         Path homeDir = createTempDir();
         Path scriptsDir = homeDir.resolve("config").resolve("scripts");
@@ -48,7 +49,7 @@ ScriptService makeScriptService(Settings settings) throws Exception {
         MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, Collections.singletonMap(scriptSource, script -> "1"));
         ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singleton(scriptEngine));
         ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList());
-        ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
+        scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
         return new ScriptService(settings, new Environment(settings), null, scriptEngineRegistry, scriptContextRegistry, scriptSettings);
     }
 
@@ -56,11 +57,15 @@ public void testFileScriptFound() throws Exception {
         Settings settings = Settings.builder()
             .put("script.engine." + MockScriptEngine.NAME + ".file.aggs", "false").build();
         ScriptService scriptService = makeScriptService(settings);
-        Script script = new Script("script1", ScriptService.ScriptType.FILE, MockScriptEngine.NAME, null);
-        CompiledScript compiledScript = scriptService.compile(script, ScriptContext.Standard.SEARCH, Collections.emptyMap());
+        Script script = new Script(ScriptType.FILE, MockScriptEngine.NAME, "script1", Collections.emptyMap());
+        CompiledScript compiledScript = scriptService.compile(script, ScriptContext.Standard.SEARCH);
         assertNotNull(compiledScript);
         MockCompiledScript executable = (MockCompiledScript) compiledScript.compiled();
         assertEquals("script1.mockscript", executable.getName());
+        assertSettingDeprecationsAndWarnings(ScriptSettingsTests.buildDeprecatedSettingsArray(
+            new Setting[] {ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING},
+            scriptSettings, "script.engine." + MockScriptEngine.NAME + ".file.aggs"),
+            "File scripts are deprecated. Use stored or inline scripts instead.");
     }
 
     public void testAllOpsDisabled() throws Exception {
@@ -71,14 +76,21 @@ public void testAllOpsDisabled() throws Exception {
             .put("script.engine." + MockScriptEngine.NAME + ".file.update", "false")
             .put("script.engine." + MockScriptEngine.NAME + ".file.ingest", "false").build();
         ScriptService scriptService = makeScriptService(settings);
-        Script script = new Script("script1", ScriptService.ScriptType.FILE, MockScriptEngine.NAME, null);
+        Script script = new Script(ScriptType.FILE, MockScriptEngine.NAME, "script1", Collections.emptyMap());
         for (ScriptContext context : ScriptContext.Standard.values()) {
             try {
-                scriptService.compile(script, context, Collections.emptyMap());
+                scriptService.compile(script, context);
                 fail(context.getKey() + " script should have been rejected");
             } catch(Exception e) {
                 assertTrue(e.getMessage(), e.getMessage().contains("scripts of type [file], operation [" + context.getKey() + "] and lang [" + MockScriptEngine.NAME + "] are disabled"));
             }
         }
+        assertSettingDeprecationsAndWarnings(ScriptSettingsTests.buildDeprecatedSettingsArray(
+            new Setting[] {ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING}, scriptSettings,
+            "script.engine." + MockScriptEngine.NAME + ".file.aggs",
+            "script.engine." + MockScriptEngine.NAME + ".file.search",
+            "script.engine." + MockScriptEngine.NAME + ".file.update",
+            "script.engine." + MockScriptEngine.NAME + ".file.ingest"),
+            "File scripts are deprecated. Use stored or inline scripts instead.");
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/script/IndexLookupIT.java b/core/src/test/java/org/elasticsearch/script/IndexLookupIT.java
index 7e57d41aceaa4..601610091a297 100644
--- a/core/src/test/java/org/elasticsearch/script/IndexLookupIT.java
+++ b/core/src/test/java/org/elasticsearch/script/IndexLookupIT.java
@@ -48,7 +48,7 @@
 import java.util.function.Function;
 
 import static java.util.Collections.emptyList;
-import static org.elasticsearch.script.ScriptService.ScriptType;
+
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.hamcrest.Matchers.containsString;
@@ -580,15 +580,19 @@ public void testCallWithDifferentFlagsFails() throws Exception {
                     .addScriptField("tvtest", script)
                     .get();
 
-            assertThat(numPrimaries, greaterThan(1));
+            // (partial) success when at least one shard succeeds
+            assertThat(numPrimaries, greaterThan(response.getShardFailures().length));
             assertThat(response.getFailedShards(), greaterThanOrEqualTo(1));
 
             for (ShardSearchFailure failure : response.getShardFailures()) {
                 assertThat(failure.reason(), containsString(expectedError));
             }
         } catch (SearchPhaseExecutionException e) {
-            assertThat(numPrimaries, equalTo(1));
-            assertThat(e.toString(), containsString(expectedError));
+            // Exception thrown when *all* shards fail
+            assertThat(numPrimaries, equalTo(e.shardFailures().length));
+            for (ShardSearchFailure failure : e.shardFailures()) {
+                assertThat(failure.reason(), containsString(expectedError));
+            }
         }
 
         // Should not throw an exception this way round
@@ -683,7 +687,7 @@ private Script createPositionsArrayScriptDefaultGet(String field, String term, S
     }
 
     private Script createScript(String script) {
-        return new Script(script, ScriptType.INLINE, CustomScriptPlugin.NAME, null);
+        return new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, script, Collections.emptyMap());
     }
 
     public void testFlags() throws Exception {
diff --git a/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java b/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java
index 1ee4c58455b5f..999a243642333 100644
--- a/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java
+++ b/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java
@@ -30,7 +30,6 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
-import org.elasticsearch.script.ScriptService.ScriptType;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
 import org.elasticsearch.watcher.ResourceWatcherService;
@@ -46,24 +45,24 @@ public void testNativeScript() throws InterruptedException {
         Settings settings = Settings.builder()
                 .put("node.name", "testNativeScript")
                 .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
-                .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), false)
                 .build();
         ScriptModule scriptModule = new ScriptModule(settings, new Environment(settings), null,
                 singletonList(new NativeScriptEngineService(settings, singletonMap("my", new MyNativeScriptFactory()))), emptyList());
         List<Setting<?>> scriptSettings = scriptModule.getSettings();
         scriptSettings.add(InternalSettingsPlugin.VERSION_CREATED);
 
-        ExecutableScript executable = scriptModule.getScriptService().executable(
-                new Script("my", ScriptType.INLINE, NativeScriptEngineService.NAME, null), ScriptContext.Standard.SEARCH,
-                Collections.emptyMap());
+        Script script = new Script(ScriptType.INLINE, NativeScriptEngineService.NAME, "my", Collections.emptyMap());
+        CompiledScript compiledScript = scriptModule.getScriptService().compile(script, ScriptContext.Standard.SEARCH);
+        ExecutableScript executable = scriptModule.getScriptService().executable(compiledScript, script.getParams());
         assertThat(executable.run().toString(), equalTo("test"));
+        assertWarnings("Native scripts are deprecated. Use a custom ScriptEngine to write scripts in java.");
     }
 
     public void testFineGrainedSettingsDontAffectNativeScripts() throws IOException {
         Settings.Builder builder = Settings.builder();
         if (randomBoolean()) {
             ScriptType scriptType = randomFrom(ScriptType.values());
-            builder.put("script" + "." + scriptType.getScriptType(), randomBoolean());
+            builder.put("script" + "." + scriptType.getName(), randomBoolean());
         } else {
             ScriptContext scriptContext = randomFrom(ScriptContext.Standard.values());
             builder.put("script" + "." + scriptContext.getKey(), randomBoolean());
@@ -81,9 +80,10 @@ public void testFineGrainedSettingsDontAffectNativeScripts() throws IOException
             scriptContextRegistry, scriptSettings);
 
         for (ScriptContext scriptContext : scriptContextRegistry.scriptContexts()) {
-            assertThat(scriptService.compile(new Script("my", ScriptType.INLINE, NativeScriptEngineService.NAME, null), scriptContext,
-                    Collections.emptyMap()), notNullValue());
+            assertThat(scriptService.compile(new Script(ScriptType.INLINE, NativeScriptEngineService.NAME, "my", Collections.emptyMap()),
+                scriptContext), notNullValue());
         }
+        assertWarnings("Native scripts are deprecated. Use a custom ScriptEngine to write scripts in java.");
     }
 
     public static class MyNativeScriptFactory implements NativeScriptFactory {
diff --git a/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java b/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java
index 16a1c20792f4f..1d627c0d6a34b 100644
--- a/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java
+++ b/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java
@@ -19,29 +19,36 @@
 
 package org.elasticsearch.script;
 
+import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Set;
 
 import static org.hamcrest.Matchers.containsString;
 
 public class ScriptContextTests extends ESTestCase {
 
     private static final String PLUGIN_NAME = "testplugin";
+    private static final String SCRIPT_PLUGIN_CUSTOM_SETTING = "script." + PLUGIN_NAME + "_custom_globally_disabled_op";
+    private static final String SCRIPT_ENGINE_CUSTOM_SETTING = "script.engine." + MockScriptEngine.NAME + ".inline." + PLUGIN_NAME + "_custom_exp_disabled_op";
+
+    private ScriptSettings scriptSettings;
 
     ScriptService makeScriptService() throws Exception {
         Settings settings = Settings.builder()
             .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
             // no file watching, so we don't need a ResourceWatcherService
-            .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), "off")
-            .put("script." + PLUGIN_NAME + "_custom_globally_disabled_op", "false")
-            .put("script.engine." + MockScriptEngine.NAME + ".inline." + PLUGIN_NAME + "_custom_exp_disabled_op", "false")
+            .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), "false")
+            .put(SCRIPT_PLUGIN_CUSTOM_SETTING, "false")
+            .put(SCRIPT_ENGINE_CUSTOM_SETTING, "false")
             .build();
 
         MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, Collections.singletonMap("1", script -> "1"));
@@ -51,51 +58,70 @@ ScriptService makeScriptService() throws Exception {
             new ScriptContext.Plugin(PLUGIN_NAME, "custom_exp_disabled_op"),
             new ScriptContext.Plugin(PLUGIN_NAME, "custom_globally_disabled_op"));
         ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(customContexts);
-        ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
+        scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
+        ScriptService scriptService = new ScriptService(settings, new Environment(settings), null, scriptEngineRegistry, scriptContextRegistry, scriptSettings);
+
+        ClusterState empty = ClusterState.builder(new ClusterName("_name")).build();
+        ScriptMetaData smd = empty.metaData().custom(ScriptMetaData.TYPE);
+        smd = ScriptMetaData.putStoredScript(smd, "1", new StoredScriptSource(MockScriptEngine.NAME, "1", Collections.emptyMap()));
+        MetaData.Builder mdb = MetaData.builder(empty.getMetaData()).putCustom(ScriptMetaData.TYPE, smd);
+        ClusterState stored = ClusterState.builder(empty).metaData(mdb).build();
+        scriptService.clusterChanged(new ClusterChangedEvent("test", stored, empty));
 
-        return new ScriptService(settings, new Environment(settings), null, scriptEngineRegistry, scriptContextRegistry, scriptSettings);
+        return scriptService;
     }
 
+
+
     public void testCustomGlobalScriptContextSettings() throws Exception {
         ScriptService scriptService = makeScriptService();
-        for (ScriptService.ScriptType scriptType : ScriptService.ScriptType.values()) {
+        for (ScriptType scriptType : ScriptType.values()) {
             try {
-                Script script = new Script("1", scriptType, MockScriptEngine.NAME, null);
-                scriptService.compile(script, new ScriptContext.Plugin(PLUGIN_NAME, "custom_globally_disabled_op"), Collections.emptyMap());
+                Script script = new Script(scriptType, MockScriptEngine.NAME, "1", Collections.emptyMap());
+                scriptService.compile(script, new ScriptContext.Plugin(PLUGIN_NAME, "custom_globally_disabled_op"));
                 fail("script compilation should have been rejected");
             } catch (IllegalStateException e) {
                 assertThat(e.getMessage(), containsString("scripts of type [" + scriptType + "], operation [" + PLUGIN_NAME + "_custom_globally_disabled_op] and lang [" + MockScriptEngine.NAME + "] are disabled"));
             }
         }
+        assertSettingDeprecationsAndWarnings(
+            ScriptSettingsTests.buildDeprecatedSettingsArray(new Setting[] {ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING},
+                scriptSettings, SCRIPT_PLUGIN_CUSTOM_SETTING, SCRIPT_ENGINE_CUSTOM_SETTING));
     }
 
     public void testCustomScriptContextSettings() throws Exception {
         ScriptService scriptService = makeScriptService();
-        Script script = new Script("1", ScriptService.ScriptType.INLINE, MockScriptEngine.NAME, null);
+        Script script = new Script(ScriptType.INLINE, MockScriptEngine.NAME, "1", Collections.emptyMap());
         try {
-            scriptService.compile(script, new ScriptContext.Plugin(PLUGIN_NAME, "custom_exp_disabled_op"), Collections.emptyMap());
+            scriptService.compile(script, new ScriptContext.Plugin(PLUGIN_NAME, "custom_exp_disabled_op"));
             fail("script compilation should have been rejected");
         } catch (IllegalStateException e) {
             assertTrue(e.getMessage(), e.getMessage().contains("scripts of type [inline], operation [" + PLUGIN_NAME + "_custom_exp_disabled_op] and lang [" + MockScriptEngine.NAME + "] are disabled"));
         }
 
         // still works for other script contexts
-        assertNotNull(scriptService.compile(script, ScriptContext.Standard.AGGS, Collections.emptyMap()));
-        assertNotNull(scriptService.compile(script, ScriptContext.Standard.SEARCH, Collections.emptyMap()));
-        assertNotNull(scriptService.compile(script, new ScriptContext.Plugin(PLUGIN_NAME, "custom_op"), Collections.emptyMap()));
+        assertNotNull(scriptService.compile(script, ScriptContext.Standard.AGGS));
+        assertNotNull(scriptService.compile(script, ScriptContext.Standard.SEARCH));
+        assertNotNull(scriptService.compile(script, new ScriptContext.Plugin(PLUGIN_NAME, "custom_op")));
+        assertSettingDeprecationsAndWarnings(
+            ScriptSettingsTests.buildDeprecatedSettingsArray(new Setting[] {ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING},
+                scriptSettings, SCRIPT_PLUGIN_CUSTOM_SETTING, SCRIPT_ENGINE_CUSTOM_SETTING));
     }
 
     public void testUnknownPluginScriptContext() throws Exception {
         ScriptService scriptService = makeScriptService();
-        for (ScriptService.ScriptType scriptType : ScriptService.ScriptType.values()) {
+        for (ScriptType scriptType : ScriptType.values()) {
             try {
-                Script script = new Script("1", scriptType, MockScriptEngine.NAME, null);
-                scriptService.compile(script, new ScriptContext.Plugin(PLUGIN_NAME, "unknown"), Collections.emptyMap());
+                Script script = new Script(scriptType, MockScriptEngine.NAME, "1", Collections.emptyMap());
+                scriptService.compile(script, new ScriptContext.Plugin(PLUGIN_NAME, "unknown"));
                 fail("script compilation should have been rejected");
             } catch (IllegalArgumentException e) {
                 assertTrue(e.getMessage(), e.getMessage().contains("script context [" + PLUGIN_NAME + "_unknown] not supported"));
             }
         }
+        assertSettingDeprecationsAndWarnings(
+            ScriptSettingsTests.buildDeprecatedSettingsArray(new Setting[] {ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING},
+                scriptSettings, SCRIPT_PLUGIN_CUSTOM_SETTING, SCRIPT_ENGINE_CUSTOM_SETTING));
     }
 
     public void testUnknownCustomScriptContext() throws Exception {
@@ -106,15 +132,17 @@ public String getKey() {
             }
         };
         ScriptService scriptService = makeScriptService();
-        for (ScriptService.ScriptType scriptType : ScriptService.ScriptType.values()) {
+        for (ScriptType scriptType : ScriptType.values()) {
             try {
-                Script script = new Script("1", scriptType, MockScriptEngine.NAME, null);
-                scriptService.compile(script, context, Collections.emptyMap());
+                Script script = new Script(scriptType, MockScriptEngine.NAME, "1", Collections.emptyMap());
+                scriptService.compile(script, context);
                 fail("script compilation should have been rejected");
             } catch (IllegalArgumentException e) {
                 assertTrue(e.getMessage(), e.getMessage().contains("script context [test] not supported"));
             }
         }
+        assertSettingDeprecationsAndWarnings(
+            ScriptSettingsTests.buildDeprecatedSettingsArray(new Setting[] {ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING},
+                scriptSettings, SCRIPT_PLUGIN_CUSTOM_SETTING, SCRIPT_ENGINE_CUSTOM_SETTING));
     }
-
 }
diff --git a/core/src/test/java/org/elasticsearch/script/ScriptFieldIT.java b/core/src/test/java/org/elasticsearch/script/ScriptFieldIT.java
index b15a2ab7be618..1a4a58e4b1e3d 100644
--- a/core/src/test/java/org/elasticsearch/script/ScriptFieldIT.java
+++ b/core/src/test/java/org/elasticsearch/script/ScriptFieldIT.java
@@ -24,7 +24,6 @@
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.ScriptPlugin;
-import org.elasticsearch.script.ScriptService.ScriptType;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
@@ -32,6 +31,7 @@
 
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ExecutionException;
@@ -61,10 +61,10 @@ public void testNativeScript() throws InterruptedException, ExecutionException {
 
         client().admin().indices().prepareFlush("test").execute().actionGet();
         SearchResponse sr = client().prepareSearch("test").setQuery(QueryBuilders.matchAllQuery())
-                .addScriptField("int", new Script("int", ScriptType.INLINE, "native", null))
-                .addScriptField("float", new Script("float", ScriptType.INLINE, "native", null))
-                .addScriptField("double", new Script("double", ScriptType.INLINE, "native", null))
-                .addScriptField("long", new Script("long", ScriptType.INLINE, "native", null)).execute().actionGet();
+                .addScriptField("int", new Script(ScriptType.INLINE, "native", "int", Collections.emptyMap()))
+                .addScriptField("float", new Script(ScriptType.INLINE, "native", "float", Collections.emptyMap()))
+                .addScriptField("double", new Script(ScriptType.INLINE, "native", "double", Collections.emptyMap()))
+                .addScriptField("long", new Script(ScriptType.INLINE, "native", "long", Collections.emptyMap())).execute().actionGet();
         assertThat(sr.getHits().hits().length, equalTo(6));
         for (SearchHit hit : sr.getHits().getHits()) {
             Object result = hit.getFields().get("int").getValues().get(0);
diff --git a/core/src/test/java/org/elasticsearch/script/ScriptMetaDataTests.java b/core/src/test/java/org/elasticsearch/script/ScriptMetaDataTests.java
index bc3757386f158..38a3922676485 100644
--- a/core/src/test/java/org/elasticsearch/script/ScriptMetaDataTests.java
+++ b/core/src/test/java/org/elasticsearch/script/ScriptMetaDataTests.java
@@ -20,21 +20,12 @@
 
 import org.elasticsearch.cluster.DiffableUtils;
 import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.io.stream.InputStreamStreamInput;
-import org.elasticsearch.common.io.stream.OutputStreamStreamOutput;
-import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESTestCase;
 
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
 import java.io.IOException;
-import java.util.Collections;
-
 
 public class ScriptMetaDataTests extends ESTestCase {
 
@@ -43,114 +34,77 @@ public void testGetScript() throws Exception {
 
         XContentBuilder sourceBuilder = XContentFactory.jsonBuilder();
         sourceBuilder.startObject().startObject("template").field("field", "value").endObject().endObject();
-        builder.storeScript("lang", "template", sourceBuilder.bytes());
+        builder.storeScript("template", StoredScriptSource.parse("lang", sourceBuilder.bytes(), sourceBuilder.contentType()));
 
         sourceBuilder = XContentFactory.jsonBuilder();
         sourceBuilder.startObject().field("template", "value").endObject();
-        builder.storeScript("lang", "template_field", sourceBuilder.bytes());
+        builder.storeScript("template_field", StoredScriptSource.parse("lang", sourceBuilder.bytes(), sourceBuilder.contentType()));
 
         sourceBuilder = XContentFactory.jsonBuilder();
         sourceBuilder.startObject().startObject("script").field("field", "value").endObject().endObject();
-        builder.storeScript("lang", "script", sourceBuilder.bytes());
+        builder.storeScript("script", StoredScriptSource.parse("lang", sourceBuilder.bytes(), sourceBuilder.contentType()));
 
         sourceBuilder = XContentFactory.jsonBuilder();
         sourceBuilder.startObject().field("script", "value").endObject();
-        builder.storeScript("lang", "script_field", sourceBuilder.bytes());
+        builder.storeScript("script_field", StoredScriptSource.parse("lang", sourceBuilder.bytes(), sourceBuilder.contentType()));
 
         sourceBuilder = XContentFactory.jsonBuilder();
         sourceBuilder.startObject().field("field", "value").endObject();
-        builder.storeScript("lang", "any", sourceBuilder.bytes());
+        builder.storeScript("any", StoredScriptSource.parse("lang", sourceBuilder.bytes(), sourceBuilder.contentType()));
 
         ScriptMetaData scriptMetaData = builder.build();
-        assertEquals("{\"field\":\"value\"}", scriptMetaData.getScript("lang", "template"));
-        assertEquals("value", scriptMetaData.getScript("lang", "template_field"));
-        assertEquals("{\"field\":\"value\"}", scriptMetaData.getScript("lang", "script"));
-        assertEquals("value", scriptMetaData.getScript("lang", "script_field"));
-        assertEquals("{\"field\":\"value\"}", scriptMetaData.getScript("lang", "any"));
-    }
-
-    public void testToAndFromXContent() throws IOException {
-        XContentType contentType = randomFrom(XContentType.values());
-        XContentBuilder xContentBuilder = XContentBuilder.builder(contentType.xContent());
-        ScriptMetaData expected = randomScriptMetaData(contentType);
-
-        xContentBuilder.startObject();
-        expected.toXContent(xContentBuilder, new ToXContent.MapParams(Collections.emptyMap()));
-        xContentBuilder.endObject();
-        xContentBuilder = shuffleXContent(xContentBuilder);
-
-        XContentParser parser = XContentHelper.createParser(xContentBuilder.bytes());
-        parser.nextToken();
-        ScriptMetaData result = ScriptMetaData.PROTO.fromXContent(parser);
-        assertEquals(expected, result);
-        assertEquals(expected.hashCode(), result.hashCode());
-    }
-
-    public void testReadFromWriteTo() throws IOException {
-        ScriptMetaData expected = randomScriptMetaData(randomFrom(XContentType.values()));
-        ByteArrayOutputStream out = new ByteArrayOutputStream();
-        expected.writeTo(new OutputStreamStreamOutput(out));
-
-        ScriptMetaData result = ScriptMetaData.PROTO.readFrom(new InputStreamStreamInput(new ByteArrayInputStream(out.toByteArray())));
-        assertEquals(expected, result);
-        assertEquals(expected.hashCode(), result.hashCode());
+        assertEquals("{\"field\":\"value\"}", scriptMetaData.getStoredScript("template", "lang").getSource());
+        assertEquals("value", scriptMetaData.getStoredScript("template_field", "lang").getSource());
+        assertEquals("{\"field\":\"value\"}", scriptMetaData.getStoredScript("script", "lang").getSource());
+        assertEquals("value", scriptMetaData.getStoredScript("script_field", "lang").getSource());
+        assertEquals("{\"field\":\"value\"}", scriptMetaData.getStoredScript("any", "lang").getSource());
     }
 
     public void testDiff() throws Exception {
         ScriptMetaData.Builder builder = new ScriptMetaData.Builder(null);
-        builder.storeScript("lang", "1", new BytesArray("{\"foo\":\"abc\"}"));
-        builder.storeScript("lang", "2", new BytesArray("{\"foo\":\"def\"}"));
-        builder.storeScript("lang", "3", new BytesArray("{\"foo\":\"ghi\"}"));
+        builder.storeScript("1", StoredScriptSource.parse("lang", new BytesArray("{\"foo\":\"abc\"}"), XContentType.JSON));
+        builder.storeScript("2", StoredScriptSource.parse("lang", new BytesArray("{\"foo\":\"def\"}"), XContentType.JSON));
+        builder.storeScript("3", StoredScriptSource.parse("lang", new BytesArray("{\"foo\":\"ghi\"}"), XContentType.JSON));
         ScriptMetaData scriptMetaData1 = builder.build();
 
         builder = new ScriptMetaData.Builder(scriptMetaData1);
-        builder.storeScript("lang", "2", new BytesArray("{\"foo\":\"changed\"}"));
-        builder.deleteScript("lang", "3");
-        builder.storeScript("lang", "4", new BytesArray("{\"foo\":\"jkl\"}"));
+        builder.storeScript("2", StoredScriptSource.parse("lang", new BytesArray("{\"foo\":\"changed\"}"), XContentType.JSON));
+        builder.deleteScript("3", "lang");
+        builder.storeScript("4", StoredScriptSource.parse("lang", new BytesArray("{\"foo\":\"jkl\"}"), XContentType.JSON));
         ScriptMetaData scriptMetaData2 = builder.build();
 
         ScriptMetaData.ScriptMetadataDiff diff = (ScriptMetaData.ScriptMetadataDiff) scriptMetaData2.diff(scriptMetaData1);
-        assertEquals(1, ((DiffableUtils.MapDiff) diff.pipelines).getDeletes().size());
-        assertEquals("lang#3", ((DiffableUtils.MapDiff) diff.pipelines).getDeletes().get(0));
-        assertEquals(1, ((DiffableUtils.MapDiff) diff.pipelines).getDiffs().size());
-        assertNotNull(((DiffableUtils.MapDiff) diff.pipelines).getDiffs().get("lang#2"));
-        assertEquals(1, ((DiffableUtils.MapDiff) diff.pipelines).getUpserts().size());
-        assertNotNull(((DiffableUtils.MapDiff) diff.pipelines).getUpserts().get("lang#4"));
+        assertEquals(2, ((DiffableUtils.MapDiff) diff.pipelines).getDeletes().size());
+        assertEquals("3", ((DiffableUtils.MapDiff) diff.pipelines).getDeletes().get(0));
+        assertEquals(2, ((DiffableUtils.MapDiff) diff.pipelines).getDiffs().size());
+        assertNotNull(((DiffableUtils.MapDiff) diff.pipelines).getDiffs().get("2"));
+        assertEquals(2, ((DiffableUtils.MapDiff) diff.pipelines).getUpserts().size());
+        assertNotNull(((DiffableUtils.MapDiff) diff.pipelines).getUpserts().get("4"));
 
         ScriptMetaData result = (ScriptMetaData) diff.apply(scriptMetaData1);
-        assertEquals(new BytesArray("{\"foo\":\"abc\"}"), result.getScriptAsBytes("lang", "1"));
-        assertEquals(new BytesArray("{\"foo\":\"changed\"}"), result.getScriptAsBytes("lang", "2"));
-        assertEquals(new BytesArray("{\"foo\":\"jkl\"}"), result.getScriptAsBytes("lang", "4"));
+        assertEquals("{\"foo\":\"abc\"}", result.getStoredScript("1", "lang").getSource());
+        assertEquals("{\"foo\":\"changed\"}", result.getStoredScript("2", "lang").getSource());
+        assertEquals("{\"foo\":\"jkl\"}", result.getStoredScript("4", "lang").getSource());
     }
 
     public void testBuilder() {
         ScriptMetaData.Builder builder = new ScriptMetaData.Builder(null);
-        builder.storeScript("_lang", "_id", new BytesArray("{\"script\":\"1 + 1\"}"));
-
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-            () -> builder.storeScript("_lang#", "_id", new BytesArray("{\"foo\": \"bar\"}")));
-        assertEquals("stored script language can't contain: '#'", e.getMessage());
-        e = expectThrows(IllegalArgumentException.class, () -> builder.storeScript("_lang", "_id#", new BytesArray("{\"foo\": \"bar\"}")));
-        assertEquals("stored script id can't contain: '#'", e.getMessage());
-        e = expectThrows(IllegalArgumentException.class, () -> builder.deleteScript("_lang#", "_id"));
-        assertEquals("stored script language can't contain: '#'", e.getMessage());
-        e = expectThrows(IllegalArgumentException.class, () -> builder.deleteScript("_lang", "_id#"));
-        assertEquals("stored script id can't contain: '#'", e.getMessage());
+        builder.storeScript("_id", StoredScriptSource.parse("_lang", new BytesArray("{\"script\":\"1 + 1\"}"), XContentType.JSON));
 
         ScriptMetaData result = builder.build();
-        assertEquals("1 + 1", result.getScript("_lang", "_id"));
+        assertEquals("1 + 1", result.getStoredScript("_id", "_lang").getSource());
     }
 
     private ScriptMetaData randomScriptMetaData(XContentType sourceContentType) throws IOException {
         ScriptMetaData.Builder builder = new ScriptMetaData.Builder(null);
         int numScripts = scaledRandomIntBetween(0, 32);
         for (int i = 0; i < numScripts; i++) {
-            String lang = randomAsciiOfLength(4);
+            String lang = randomAlphaOfLength(4);
             XContentBuilder sourceBuilder = XContentBuilder.builder(sourceContentType.xContent());
-            sourceBuilder.startObject().field(randomAsciiOfLength(4), randomAsciiOfLength(4)).endObject();
-            builder.storeScript(lang, randomAsciiOfLength(i + 1), sourceBuilder.bytes());
+            sourceBuilder.startObject().field("script", randomAlphaOfLength(4)).endObject();
+            builder.storeScript(randomAlphaOfLength(i + 1),
+                StoredScriptSource.parse(lang, sourceBuilder.bytes(), sourceBuilder.contentType()));
         }
         return builder.build();
     }
-
 }
diff --git a/core/src/test/java/org/elasticsearch/script/ScriptModesTests.java b/core/src/test/java/org/elasticsearch/script/ScriptModesTests.java
index e4a1835ab63e6..89c5e6e95d424 100644
--- a/core/src/test/java/org/elasticsearch/script/ScriptModesTests.java
+++ b/core/src/test/java/org/elasticsearch/script/ScriptModesTests.java
@@ -21,15 +21,16 @@
 
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.script.ScriptService.ScriptType;
 import org.elasticsearch.search.lookup.SearchLookup;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.After;
 import org.junit.Before;
 
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
@@ -57,8 +58,8 @@ public void setupScriptEngines() {
         //prevent duplicates using map
         Map<String, ScriptContext.Plugin> contexts = new HashMap<>();
         for (int i = 0; i < randomInt; i++) {
-            String plugin = randomAsciiOfLength(randomIntBetween(1, 10));
-            String operation = randomAsciiOfLength(randomIntBetween(1, 30));
+            String plugin = randomAlphaOfLength(randomIntBetween(1, 10));
+            String operation = randomAlphaOfLength(randomIntBetween(1, 30));
             String context = plugin + "-" + operation;
             contexts.put(context, new ScriptContext.Plugin(plugin, operation));
         }
@@ -96,14 +97,14 @@ public void assertAllSettingsWereChecked() {
     }
 
     public void testDefaultSettings() {
-        this.scriptModes = new ScriptModes(scriptSettings, Settings.EMPTY);
+        this.scriptModes = new ScriptModes(scriptContextRegistry, scriptSettings, Settings.EMPTY);
         assertScriptModesAllOps(true, ScriptType.FILE);
         assertScriptModesAllOps(false, ScriptType.STORED, ScriptType.INLINE);
     }
 
     public void testMissingSetting() {
         assertAllSettingsWereChecked = false;
-        this.scriptModes = new ScriptModes(scriptSettings, Settings.EMPTY);
+        this.scriptModes = new ScriptModes(scriptContextRegistry, scriptSettings, Settings.EMPTY);
         try {
             scriptModes.getScriptEnabled("non_existing", randomFrom(ScriptType.values()), randomFrom(scriptContexts));
             fail("Expected IllegalArgumentException");
@@ -124,11 +125,13 @@ public void testScriptTypeGenericSettings() {
             randomScriptModes[i] = randomBoolean();
         }
         ScriptType[] randomScriptTypes = randomScriptTypesSet.toArray(new ScriptType[randomScriptTypesSet.size()]);
+        List<String> deprecated = new ArrayList<>();
         Settings.Builder builder = Settings.builder();
         for (int i = 0; i < randomInt; i++) {
-            builder.put("script" + "." + randomScriptTypes[i].getScriptType(), randomScriptModes[i]);
+            builder.put("script" + "." + randomScriptTypes[i].getName(), randomScriptModes[i]);
+            deprecated.add("script" + "." + randomScriptTypes[i].getName());
         }
-        this.scriptModes = new ScriptModes(scriptSettings, builder.build());
+        this.scriptModes = new ScriptModes(scriptContextRegistry, scriptSettings, builder.build());
 
         for (int i = 0; i < randomInt; i++) {
             assertScriptModesAllOps(randomScriptModes[i], randomScriptTypes[i]);
@@ -142,6 +145,8 @@ public void testScriptTypeGenericSettings() {
         if (randomScriptTypesSet.contains(ScriptType.INLINE) == false) {
             assertScriptModesAllOps(false, ScriptType.INLINE);
         }
+        assertSettingDeprecationsAndWarnings(
+            ScriptSettingsTests.buildDeprecatedSettingsArray(scriptSettings, deprecated.toArray(new String[] {})));
     }
 
     public void testScriptContextGenericSettings() {
@@ -156,11 +161,13 @@ public void testScriptContextGenericSettings() {
             randomScriptModes[i] = randomBoolean();
         }
         ScriptContext[] randomScriptContexts = randomScriptContextsSet.toArray(new ScriptContext[randomScriptContextsSet.size()]);
+        List<String> deprecated = new ArrayList<>();
         Settings.Builder builder = Settings.builder();
         for (int i = 0; i < randomInt; i++) {
             builder.put("script" + "." + randomScriptContexts[i].getKey(), randomScriptModes[i]);
+            deprecated.add("script" + "." + randomScriptContexts[i].getKey());
         }
-        this.scriptModes = new ScriptModes(scriptSettings, builder.build());
+        this.scriptModes = new ScriptModes(scriptContextRegistry, scriptSettings, builder.build());
 
         for (int i = 0; i < randomInt; i++) {
             assertScriptModesAllTypes(randomScriptModes[i], randomScriptContexts[i]);
@@ -169,6 +176,8 @@ public void testScriptContextGenericSettings() {
         ScriptContext[] complementOf = complementOf(randomScriptContexts);
         assertScriptModes(true, new ScriptType[]{ScriptType.FILE}, complementOf);
         assertScriptModes(false, new ScriptType[]{ScriptType.STORED, ScriptType.INLINE}, complementOf);
+        assertSettingDeprecationsAndWarnings(
+            ScriptSettingsTests.buildDeprecatedSettingsArray(scriptSettings, deprecated.toArray(new String[] {})));
     }
 
     public void testConflictingScriptTypeAndOpGenericSettings() {
@@ -178,11 +187,14 @@ public void testConflictingScriptTypeAndOpGenericSettings() {
                 .put("script.stored", "true")
                 .put("script.inline", "true");
         //operations generic settings have precedence over script type generic settings
-        this.scriptModes = new ScriptModes(scriptSettings, builder.build());
+        this.scriptModes = new ScriptModes(scriptContextRegistry, scriptSettings, builder.build());
         assertScriptModesAllTypes(false, scriptContext);
         ScriptContext[] complementOf = complementOf(scriptContext);
         assertScriptModes(true, new ScriptType[]{ScriptType.FILE, ScriptType.STORED}, complementOf);
         assertScriptModes(true, new ScriptType[]{ScriptType.INLINE}, complementOf);
+        assertSettingDeprecationsAndWarnings(
+            ScriptSettingsTests.buildDeprecatedSettingsArray(
+                scriptSettings, "script." + scriptContext.getKey(), "script.stored", "script.inline"));
     }
 
     private void assertScriptModesAllOps(boolean expectedScriptEnabled, ScriptType... scriptTypes) {
diff --git a/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java b/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
index bc7cb9ffb603d..0d0a5958abd6b 100644
--- a/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
@@ -19,9 +19,7 @@
 package org.elasticsearch.script;
 
 import org.elasticsearch.ResourceNotFoundException;
-import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest;
 import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest;
-import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequest;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.MetaData;
@@ -32,8 +30,8 @@
 import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.env.Environment;
-import org.elasticsearch.script.ScriptService.ScriptType;
 import org.elasticsearch.search.lookup.SearchLookup;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.watcher.ResourceWatcherService;
@@ -42,10 +40,12 @@
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 
 import static org.hamcrest.CoreMatchers.containsString;
@@ -97,11 +97,11 @@ public void setup() throws IOException {
         for (int i = 0; i < randomInt; i++) {
             String plugin;
             do {
-                plugin = randomAsciiOfLength(randomIntBetween(1, 10));
+                plugin = randomAlphaOfLength(randomIntBetween(1, 10));
             } while (ScriptContextRegistry.RESERVED_SCRIPT_CONTEXTS.contains(plugin));
             String operation;
             do {
-                operation = randomAsciiOfLength(randomIntBetween(1, 30));
+                operation = randomAlphaOfLength(randomIntBetween(1, 30));
             } while (ScriptContextRegistry.RESERVED_SCRIPT_CONTEXTS.contains(operation));
             String context = plugin + "_" + operation;
             contexts.put(context, new ScriptContext.Plugin(plugin, operation));
@@ -122,9 +122,9 @@ private void buildScriptService(Settings additionalSettings) throws IOException
         // TODO:
         scriptService = new ScriptService(finalSettings, environment, resourceWatcherService, scriptEngineRegistry, scriptContextRegistry, scriptSettings) {
             @Override
-            String getScriptFromClusterState(String scriptLang, String id) {
+            StoredScriptSource getScriptFromClusterState(String id, String lang) {
                 //mock the script that gets retrieved from an index
-                return "100";
+                return new StoredScriptSource(lang, "100", Collections.emptyMap());
             }
         };
     }
@@ -151,6 +151,7 @@ public void testCompilationCircuitBreaking() throws Exception {
         for (int i = 0; i < largeLimit; i++) {
             scriptService.checkCompilationLimit();
         }
+        assertScriptCompilationWarning();
     }
 
     public void testNotSupportedDisableDynamicSetting() throws IOException {
@@ -170,8 +171,8 @@ public void testScriptsWithoutExtensions() throws IOException {
         Streams.copy("test_file".getBytes("UTF-8"), Files.newOutputStream(testFileWithExt));
         resourceWatcherService.notifyNow();
 
-        CompiledScript compiledScript = scriptService.compile(new Script("test_script", ScriptType.FILE, "test", null),
-                ScriptContext.Standard.SEARCH, Collections.emptyMap());
+        CompiledScript compiledScript = scriptService.compile(new Script(ScriptType.FILE, "test", "test_script", Collections.emptyMap()),
+                ScriptContext.Standard.SEARCH);
         assertThat(compiledScript.compiled(), equalTo((Object) "compiled_test_file"));
 
         Files.delete(testFileNoExt);
@@ -179,12 +180,14 @@ public void testScriptsWithoutExtensions() throws IOException {
         resourceWatcherService.notifyNow();
 
         try {
-            scriptService.compile(new Script("test_script", ScriptType.FILE, "test", null), ScriptContext.Standard.SEARCH,
-                    Collections.emptyMap());
+            scriptService.compile(new Script(ScriptType.FILE, "test", "test_script", Collections.emptyMap()), ScriptContext.Standard.SEARCH);
             fail("the script test_script should no longer exist");
         } catch (IllegalArgumentException ex) {
-            assertThat(ex.getMessage(), containsString("Unable to find on disk file script [test_script] using lang [test]"));
+            assertThat(ex.getMessage(), containsString("unable to find file script [test_script] using lang [test]"));
         }
+        assertWarnings("File scripts are deprecated. Use stored or inline scripts instead.", "[script.max_compilations_per_minute] " +
+                "setting was deprecated in Elasticsearch and will be removed in a future " +
+                "release! See the breaking changes documentation for the next major version.");
     }
 
     public void testScriptCompiledOnceHiddenFileDetected() throws IOException {
@@ -197,32 +200,109 @@ public void testScriptCompiledOnceHiddenFileDetected() throws IOException {
         Streams.copy("test_file_script".getBytes("UTF-8"), Files.newOutputStream(testFileScript));
         resourceWatcherService.notifyNow();
 
-        CompiledScript compiledScript = scriptService.compile(new Script("file_script", ScriptType.FILE, "test", null),
-                ScriptContext.Standard.SEARCH, Collections.emptyMap());
+        CompiledScript compiledScript = scriptService.compile(new Script(ScriptType.FILE, "test", "file_script", Collections.emptyMap()),
+                ScriptContext.Standard.SEARCH);
         assertThat(compiledScript.compiled(), equalTo((Object) "compiled_test_file_script"));
 
         Files.delete(testHiddenFile);
         Files.delete(testFileScript);
         resourceWatcherService.notifyNow();
+        assertWarnings("File scripts are deprecated. Use stored or inline scripts instead.", "[script.max_compilations_per_minute] " +
+                "setting was deprecated in Elasticsearch and will be removed in a future " +
+                "release! See the breaking changes documentation for the next major version.");
     }
 
     public void testInlineScriptCompiledOnceCache() throws IOException {
         buildScriptService(Settings.EMPTY);
-        CompiledScript compiledScript1 = scriptService.compile(new Script("1+1", ScriptType.INLINE, "test", null),
-                randomFrom(scriptContexts), Collections.emptyMap());
-        CompiledScript compiledScript2 = scriptService.compile(new Script("1+1", ScriptType.INLINE, "test", null),
-                randomFrom(scriptContexts), Collections.emptyMap());
+        CompiledScript compiledScript1 = scriptService.compile(new Script(ScriptType.INLINE, "test", "1+1", Collections.emptyMap()),
+                randomFrom(scriptContexts));
+        CompiledScript compiledScript2 = scriptService.compile(new Script(ScriptType.INLINE, "test", "1+1", Collections.emptyMap()),
+                randomFrom(scriptContexts));
         assertThat(compiledScript1.compiled(), sameInstance(compiledScript2.compiled()));
+        assertScriptCompilationWarning();
+    }
+
+    public void testAllowAllScriptTypeSettings() throws IOException {
+        buildScriptService(Settings.EMPTY);
+
+        assertCompileAccepted("painless", "script", ScriptType.INLINE, ScriptContext.Standard.SEARCH);
+        assertCompileAccepted("painless", "script", ScriptType.STORED, ScriptContext.Standard.SEARCH);
+        assertScriptCompilationWarning();
+    }
+
+    public void testAllowAllScriptContextSettings() throws IOException {
+        buildScriptService(Settings.EMPTY);
+
+        assertCompileAccepted("painless", "script", ScriptType.INLINE, ScriptContext.Standard.SEARCH);
+        assertCompileAccepted("painless", "script", ScriptType.INLINE, ScriptContext.Standard.AGGS);
+        assertCompileAccepted("painless", "script", ScriptType.INLINE, ScriptContext.Standard.UPDATE);
+        assertCompileAccepted("painless", "script", ScriptType.INLINE, ScriptContext.Standard.INGEST);
+        assertScriptCompilationWarning();
+    }
+
+    public void testAllowSomeScriptTypeSettings() throws IOException {
+        Settings.Builder builder = Settings.builder();
+        builder.put("script.allowed_types", "inline");
+        builder.put("script.engine.painless.stored", false);
+        buildScriptService(builder.build());
+
+        assertCompileAccepted("painless", "script", ScriptType.INLINE, ScriptContext.Standard.SEARCH);
+        assertCompileRejected("painless", "script", ScriptType.STORED, ScriptContext.Standard.SEARCH);
+
+        assertSettingDeprecationsAndWarnings(
+            ScriptSettingsTests.buildDeprecatedSettingsArray(scriptSettings, "script.engine.painless.stored"),
+            "[script.max_compilations_per_minute] setting was deprecated in Elasticsearch and will be removed in a " +
+                    "future release! See the breaking changes documentation for the next major version.");
+    }
+
+    public void testAllowSomeScriptContextSettings() throws IOException {
+        Settings.Builder builder = Settings.builder();
+        builder.put("script.allowed_contexts", "search, aggs");
+        builder.put("script.update", false);
+        buildScriptService(builder.build());
+
+        assertCompileAccepted("painless", "script", ScriptType.INLINE, ScriptContext.Standard.SEARCH);
+        assertCompileAccepted("painless", "script", ScriptType.INLINE, ScriptContext.Standard.AGGS);
+        assertCompileRejected("painless", "script", ScriptType.INLINE, ScriptContext.Standard.UPDATE);
+
+        assertSettingDeprecationsAndWarnings(
+            ScriptSettingsTests.buildDeprecatedSettingsArray(scriptSettings, "script.update"),
+                "[script.max_compilations_per_minute] setting was deprecated in Elasticsearch and will be removed in a " +
+                        "future release! See the breaking changes documentation for the next major version.");
+    }
+
+    public void testAllowNoScriptTypeSettings() throws IOException {
+        Settings.Builder builder = Settings.builder();
+        builder.put("script.allowed_types", "none");
+        buildScriptService(builder.build());
+
+        assertCompileRejected("painless", "script", ScriptType.INLINE, ScriptContext.Standard.SEARCH);
+        assertCompileRejected("painless", "script", ScriptType.STORED, ScriptContext.Standard.SEARCH);
+        assertScriptCompilationWarning();
+    }
+
+    public void testAllowNoScriptContextSettings() throws IOException {
+        Settings.Builder builder = Settings.builder();
+        builder.put("script.allowed_contexts", "none");
+        buildScriptService(builder.build());
+
+        assertCompileRejected("painless", "script", ScriptType.INLINE, ScriptContext.Standard.SEARCH);
+        assertCompileRejected("painless", "script", ScriptType.INLINE, ScriptContext.Standard.AGGS);
+        assertCompileRejected("painless", "script", ScriptType.INLINE, ScriptContext.Standard.UPDATE);
+        assertCompileRejected("painless", "script", ScriptType.INLINE, ScriptContext.Standard.INGEST);
+        assertScriptCompilationWarning();
     }
 
     public void testDefaultBehaviourFineGrainedSettings() throws IOException {
         Settings.Builder builder = Settings.builder();
         //rarely inject the default settings, which have no effect
+        boolean deprecate = false;
         if (rarely()) {
             builder.put("script.file", "true");
+            deprecate = true;
         }
         buildScriptService(builder.build());
-        createFileScripts("groovy", "mustache", "dtest");
+        createFileScripts("dtest");
 
         for (ScriptContext scriptContext : scriptContexts) {
             // only file scripts are accepted by default
@@ -230,6 +310,16 @@ public void testDefaultBehaviourFineGrainedSettings() throws IOException {
             assertCompileRejected("dtest", "script", ScriptType.STORED, scriptContext);
             assertCompileAccepted("dtest", "file_script", ScriptType.FILE, scriptContext);
         }
+        if (deprecate) {
+            assertSettingDeprecationsAndWarnings(ScriptSettingsTests.buildDeprecatedSettingsArray(scriptSettings, "script.file"),
+                "File scripts are deprecated. Use stored or inline scripts instead.", "[script.max_compilations_per_minute] " +
+                            "setting was deprecated in Elasticsearch and will be removed in a future " +
+                            "release! See the breaking changes documentation for the next major version.");
+        } else {
+            assertWarnings("File scripts are deprecated. Use stored or inline scripts instead.", "[script.max_compilations_per_minute] " +
+                    "setting was deprecated in Elasticsearch and will be removed in a future " +
+                    "release! See the breaking changes documentation for the next major version.");
+        }
     }
 
     public void testFineGrainedSettings() throws IOException {
@@ -263,14 +353,16 @@ public void testFineGrainedSettings() throws IOException {
             } while (engineSettings.containsKey(settingKey));
             engineSettings.put(settingKey, randomBoolean());
         }
+        List<String> deprecated = new ArrayList<>();
         //set the selected fine-grained settings
         Settings.Builder builder = Settings.builder();
         for (Map.Entry<ScriptType, Boolean> entry : scriptSourceSettings.entrySet()) {
             if (entry.getValue()) {
-                builder.put("script" + "." + entry.getKey().getScriptType(), "true");
+                builder.put("script" + "." + entry.getKey().getName(), "true");
             } else {
-                builder.put("script" + "." + entry.getKey().getScriptType(), "false");
+                builder.put("script" + "." + entry.getKey().getName(), "false");
             }
+            deprecated.add("script" + "." + entry.getKey().getName());
         }
         for (Map.Entry<ScriptContext, Boolean> entry : scriptContextSettings.entrySet()) {
             if (entry.getValue()) {
@@ -278,6 +370,7 @@ public void testFineGrainedSettings() throws IOException {
             } else {
                 builder.put("script" + "." + entry.getKey().getKey(), "false");
             }
+            deprecated.add("script" + "." + entry.getKey().getKey());
         }
         for (Map.Entry<String, Boolean> entry : engineSettings.entrySet()) {
             int delimiter = entry.getKey().indexOf('.');
@@ -290,6 +383,7 @@ public void testFineGrainedSettings() throws IOException {
             } else {
                 builder.put("script.engine" + "." + lang + "." + part2, "false");
             }
+            deprecated.add("script.engine" + "." + lang + "." + part2);
         }
 
         buildScriptService(builder.build());
@@ -320,6 +414,11 @@ public void testFineGrainedSettings() throws IOException {
                 }
             }
         }
+        assertSettingDeprecationsAndWarnings(
+            ScriptSettingsTests.buildDeprecatedSettingsArray(scriptSettings, deprecated.toArray(new String[] {})),
+            "File scripts are deprecated. Use stored or inline scripts instead.", "[script.max_compilations_per_minute] setting was " +
+                        "deprecated in Elasticsearch and will be removed in a future " +
+                        "release! See the breaking changes documentation for the next major version.");
     }
 
     public void testCompileNonRegisteredContext() throws IOException {
@@ -327,38 +426,42 @@ public void testCompileNonRegisteredContext() throws IOException {
         String pluginName;
         String unknownContext;
         do {
-            pluginName = randomAsciiOfLength(randomIntBetween(1, 10));
-            unknownContext = randomAsciiOfLength(randomIntBetween(1, 30));
-        } while(scriptContextRegistry.isSupportedContext(new ScriptContext.Plugin(pluginName, unknownContext)));
+            pluginName = randomAlphaOfLength(randomIntBetween(1, 10));
+            unknownContext = randomAlphaOfLength(randomIntBetween(1, 30));
+        } while(scriptContextRegistry.isSupportedContext(new ScriptContext.Plugin(pluginName, unknownContext).getKey()));
 
         String type = scriptEngineService.getType();
         try {
-            scriptService.compile(new Script("test", randomFrom(ScriptType.values()), type, null), new ScriptContext.Plugin(
-                            pluginName, unknownContext), Collections.emptyMap());
+            scriptService.compile(new Script(randomFrom(ScriptType.values()), type, "test", Collections.emptyMap()),
+                new ScriptContext.Plugin(pluginName, unknownContext));
             fail("script compilation should have been rejected");
         } catch(IllegalArgumentException e) {
             assertThat(e.getMessage(), containsString("script context [" + pluginName + "_" + unknownContext + "] not supported"));
         }
+        assertScriptCompilationWarning();
     }
 
     public void testCompileCountedInCompilationStats() throws IOException {
         buildScriptService(Settings.EMPTY);
-        scriptService.compile(new Script("1+1", ScriptType.INLINE, "test", null), randomFrom(scriptContexts),
-                Collections.emptyMap());
+        scriptService.compile(new Script(ScriptType.INLINE, "test", "1+1", Collections.emptyMap()), randomFrom(scriptContexts));
         assertEquals(1L, scriptService.stats().getCompilations());
+        assertScriptCompilationWarning();
     }
 
     public void testExecutableCountedInCompilationStats() throws IOException {
         buildScriptService(Settings.EMPTY);
-        scriptService.executable(new Script("1+1", ScriptType.INLINE, "test", null), randomFrom(scriptContexts), Collections.emptyMap());
+        Script script = new Script(ScriptType.INLINE, "test", "1+1", Collections.emptyMap());
+        CompiledScript compiledScript = scriptService.compile(script, randomFrom(scriptContexts));
+        scriptService.executable(compiledScript, script.getParams());
         assertEquals(1L, scriptService.stats().getCompilations());
+        assertScriptCompilationWarning();
     }
 
     public void testSearchCountedInCompilationStats() throws IOException {
         buildScriptService(Settings.EMPTY);
-        scriptService.search(null, new Script("1+1", ScriptType.INLINE, "test", null), randomFrom(scriptContexts),
-                Collections.emptyMap());
+        scriptService.search(null, new Script(ScriptType.INLINE, "test", "1+1", Collections.emptyMap()), randomFrom(scriptContexts));
         assertEquals(1L, scriptService.stats().getCompilations());
+        assertScriptCompilationWarning();
     }
 
     public void testMultipleCompilationsCountedInCompilationStats() throws IOException {
@@ -366,10 +469,10 @@ public void testMultipleCompilationsCountedInCompilationStats() throws IOExcepti
         int numberOfCompilations = randomIntBetween(1, 1024);
         for (int i = 0; i < numberOfCompilations; i++) {
             scriptService
-                    .compile(new Script(i + " + " + i, ScriptType.INLINE, "test", null), randomFrom(scriptContexts),
-                            Collections.emptyMap());
+                    .compile(new Script(ScriptType.INLINE, "test", i + " + " + i, Collections.emptyMap()), randomFrom(scriptContexts));
         }
         assertEquals(numberOfCompilations, scriptService.stats().getCompilations());
+        assertScriptCompilationWarning();
     }
 
     public void testCompilationStatsOnCacheHit() throws IOException {
@@ -377,24 +480,31 @@ public void testCompilationStatsOnCacheHit() throws IOException {
         builder.put(ScriptService.SCRIPT_CACHE_SIZE_SETTING.getKey(), 1);
         builder.put("script.inline", "true");
         buildScriptService(builder.build());
-        scriptService.executable(new Script("1+1", ScriptType.INLINE, "test", null), randomFrom(scriptContexts), Collections.emptyMap());
-        scriptService.executable(new Script("1+1", ScriptType.INLINE, "test", null), randomFrom(scriptContexts), Collections.emptyMap());
+        Script script = new Script(ScriptType.INLINE, "test", "1+1", Collections.emptyMap());
+        scriptService.compile(script, randomFrom(scriptContexts));
+        scriptService.compile(script, randomFrom(scriptContexts));
         assertEquals(1L, scriptService.stats().getCompilations());
+        assertSettingDeprecationsAndWarnings(
+            ScriptSettingsTests.buildDeprecatedSettingsArray(scriptSettings, "script.inline"), "[script.max_compilations_per_minute] " +
+                        "setting was deprecated in Elasticsearch and will be removed in a future " +
+                        "release! See the breaking changes documentation for the next major version.");
     }
 
     public void testFileScriptCountedInCompilationStats() throws IOException {
         buildScriptService(Settings.EMPTY);
         createFileScripts("test");
-        scriptService.compile(new Script("file_script", ScriptType.FILE, "test", null), randomFrom(scriptContexts),
-                Collections.emptyMap());
+        scriptService.compile(new Script(ScriptType.FILE, "test", "file_script", Collections.emptyMap()), randomFrom(scriptContexts));
         assertEquals(1L, scriptService.stats().getCompilations());
+        assertWarnings("File scripts are deprecated. Use stored or inline scripts instead.", "[script.max_compilations_per_minute] " +
+                "setting was deprecated in Elasticsearch and will be removed in a future " +
+                "release! See the breaking changes documentation for the next major version.");
     }
 
     public void testIndexedScriptCountedInCompilationStats() throws IOException {
         buildScriptService(Settings.EMPTY);
-        scriptService.compile(new Script("script", ScriptType.STORED, "test", null), randomFrom(scriptContexts),
-                Collections.emptyMap());
+        scriptService.compile(new Script(ScriptType.STORED, "test", "script", Collections.emptyMap()), randomFrom(scriptContexts));
         assertEquals(1L, scriptService.stats().getCompilations());
+        assertScriptCompilationWarning();
     }
 
     public void testCacheEvictionCountedInCacheEvictionsStats() throws IOException {
@@ -402,19 +512,29 @@ public void testCacheEvictionCountedInCacheEvictionsStats() throws IOException {
         builder.put(ScriptService.SCRIPT_CACHE_SIZE_SETTING.getKey(), 1);
         builder.put("script.inline", "true");
         buildScriptService(builder.build());
-        scriptService.executable(new Script("1+1", ScriptType.INLINE, "test", null), randomFrom(scriptContexts), Collections.emptyMap());
-        scriptService.executable(new Script("2+2", ScriptType.INLINE, "test", null), randomFrom(scriptContexts), Collections.emptyMap());
+        scriptService.compile(new Script(ScriptType.INLINE, "test", "1+1", Collections.emptyMap()), randomFrom(scriptContexts));
+        scriptService.compile(new Script(ScriptType.INLINE, "test", "2+2", Collections.emptyMap()), randomFrom(scriptContexts));
         assertEquals(2L, scriptService.stats().getCompilations());
         assertEquals(1L, scriptService.stats().getCacheEvictions());
+        assertSettingDeprecationsAndWarnings(
+            ScriptSettingsTests.buildDeprecatedSettingsArray(scriptSettings, "script.inline"), "[script.max_compilations_per_minute] " +
+                        "setting was deprecated in Elasticsearch and will be removed in a future " +
+                        "release! See the breaking changes documentation for the next major version."
+                );
     }
 
     public void testDefaultLanguage() throws IOException {
         Settings.Builder builder = Settings.builder();
         builder.put("script.inline", "true");
         buildScriptService(builder.build());
-        CompiledScript script = scriptService.compile(new Script("1 + 1", ScriptType.INLINE, null, null),
-                randomFrom(scriptContexts), Collections.emptyMap());
+        CompiledScript script = scriptService.compile(
+            new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, "1 + 1", Collections.emptyMap()), randomFrom(scriptContexts));
         assertEquals(script.lang(), Script.DEFAULT_SCRIPT_LANG);
+        assertSettingDeprecationsAndWarnings(
+            ScriptSettingsTests.buildDeprecatedSettingsArray(scriptSettings, "script.inline"), "[script.max_compilations_per_minute] " +
+                        "setting was deprecated in Elasticsearch and will be removed in a future " +
+                        "release! See the breaking changes documentation for the next major version.");
+
     }
 
     public void testStoreScript() throws Exception {
@@ -422,68 +542,43 @@ public void testStoreScript() throws Exception {
                     .field("script", "abc")
                 .endObject().bytes();
 
-        ClusterState empty = ClusterState.builder(new ClusterName("_name")).build();
-        PutStoredScriptRequest request = new PutStoredScriptRequest("_lang", "_id")
-                .script(script);
-        ClusterState result = ScriptService.innerStoreScript(empty, "_lang", request);
-        ScriptMetaData scriptMetaData = result.getMetaData().custom(ScriptMetaData.TYPE);
+        ScriptMetaData scriptMetaData = ScriptMetaData.putStoredScript(null, "_id",
+            StoredScriptSource.parse("_lang", script, XContentType.JSON));
         assertNotNull(scriptMetaData);
-        assertEquals("abc", scriptMetaData.getScript("_lang", "_id"));
+        assertEquals("abc", scriptMetaData.getStoredScript("_id", "_lang").getSource());
     }
 
     public void testDeleteScript() throws Exception {
-        ClusterState cs = ClusterState.builder(new ClusterName("_name"))
-                .metaData(MetaData.builder()
-                        .putCustom(ScriptMetaData.TYPE,
-                                new ScriptMetaData.Builder(null).storeScript("_lang", "_id",
-                                    new BytesArray("{\"script\":\"abc\"}")).build()))
-                .build();
-
-        DeleteStoredScriptRequest request = new DeleteStoredScriptRequest("_lang", "_id");
-        ClusterState result = ScriptService.innerDeleteScript(cs, "_lang", request);
-        ScriptMetaData scriptMetaData = result.getMetaData().custom(ScriptMetaData.TYPE);
+        ScriptMetaData scriptMetaData = ScriptMetaData.putStoredScript(null, "_id",
+            StoredScriptSource.parse("_lang", new BytesArray("{\"script\":\"abc\"}"), XContentType.JSON));
+        scriptMetaData = ScriptMetaData.deleteStoredScript(scriptMetaData, "_id", "_lang");
         assertNotNull(scriptMetaData);
-        assertNull(scriptMetaData.getScript("_lang", "_id"));
-        assertNull(scriptMetaData.getScriptAsBytes("_lang", "_id"));
+        assertNull(scriptMetaData.getStoredScript("_id", "_lang"));
 
+        ScriptMetaData errorMetaData = scriptMetaData;
         ResourceNotFoundException e = expectThrows(ResourceNotFoundException.class, () -> {
-            ScriptService.innerDeleteScript(cs, "_lang", new DeleteStoredScriptRequest("_lang", "_non_existing_id"));
+            ScriptMetaData.deleteStoredScript(errorMetaData, "_id", "_lang");
         });
-        assertEquals("Stored script with id [_non_existing_id] for language [_lang] does not exist", e.getMessage());
+        assertEquals("stored script [_id] using lang [_lang] does not exist and cannot be deleted", e.getMessage());
     }
 
     public void testGetStoredScript() throws Exception {
         buildScriptService(Settings.EMPTY);
         ClusterState cs = ClusterState.builder(new ClusterName("_name"))
-                .metaData(MetaData.builder()
-                        .putCustom(ScriptMetaData.TYPE,
-                                new ScriptMetaData.Builder(null).storeScript("_lang", "_id",
-                                        new BytesArray("{\"script\":\"abc\"}")).build()))
-                .build();
+            .metaData(MetaData.builder()
+                .putCustom(ScriptMetaData.TYPE,
+                    new ScriptMetaData.Builder(null).storeScript("_id",
+                        StoredScriptSource.parse("_lang", new BytesArray("{\"script\":\"abc\"}"), XContentType.JSON)).build()))
+            .build();
 
-        assertEquals("abc", scriptService.getStoredScript(cs, new GetStoredScriptRequest("_lang", "_id")));
-        assertNull(scriptService.getStoredScript(cs, new GetStoredScriptRequest("_lang", "_id2")));
+        assertEquals("abc", scriptService.getStoredScript(cs, new GetStoredScriptRequest("_id", "_lang")).getSource());
+        assertNull(scriptService.getStoredScript(cs, new GetStoredScriptRequest("_id2", "_lang")));
 
         cs = ClusterState.builder(new ClusterName("_name")).build();
-        assertNull(scriptService.getStoredScript(cs, new GetStoredScriptRequest("_lang", "_id")));
-    }
-
-    public void testValidateScriptSize() throws Exception {
-        int maxSize = 0xFFFF;
-        buildScriptService(Settings.EMPTY);
-        // allowed
-        scriptService.validateStoredScript("_id", "test", new BytesArray("{\"script\":\"" + randomAsciiOfLength(maxSize - 13) + "\"}"));
-
-        // disallowed
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-                () -> {
-                    scriptService.validateStoredScript("_id", "test", new BytesArray("{\"script\":\"" + randomAsciiOfLength(maxSize - 12) + "\"}"));
-                });
-        assertThat(e.getMessage(), equalTo(
-                "Limit of script size in bytes [" + maxSize+ "] has been exceeded for script [_id] with size [" + (maxSize + 1) + "]"));
+        assertNull(scriptService.getStoredScript(cs, new GetStoredScriptRequest("_id", "_lang")));
+        assertScriptCompilationWarning();
     }
 
-
     private void createFileScripts(String... langs) throws IOException {
         for (String lang : langs) {
             Path scriptPath = scriptsFilePath.resolve("file_script." + lang);
@@ -494,20 +589,25 @@ private void createFileScripts(String... langs) throws IOException {
 
     private void assertCompileRejected(String lang, String script, ScriptType scriptType, ScriptContext scriptContext) {
         try {
-            scriptService.compile(new Script(script, scriptType, lang, null), scriptContext, Collections.emptyMap());
+            scriptService.compile(new Script(scriptType, lang, script, Collections.emptyMap()), scriptContext);
             fail("compile should have been rejected for lang [" + lang + "], script_type [" + scriptType + "], scripted_op [" + scriptContext + "]");
-        } catch(IllegalStateException e) {
-            //all good
+        } catch (IllegalArgumentException | IllegalStateException e) {
+            // pass
         }
     }
 
     private void assertCompileAccepted(String lang, String script, ScriptType scriptType, ScriptContext scriptContext) {
         assertThat(
-                scriptService.compile(new Script(script, scriptType, lang, null), scriptContext, Collections.emptyMap()),
+                scriptService.compile(new Script(scriptType, lang, script, Collections.emptyMap()), scriptContext),
                 notNullValue()
         );
     }
 
+    private void assertScriptCompilationWarning() {
+        assertWarnings("[script.max_compilations_per_minute] setting was deprecated in Elasticsearch and will be removed in a " +
+                "future release! See the breaking changes documentation for the next major version.");
+    }
+
     public static class TestEngineService implements ScriptEngineService {
 
         public static final String NAME = "test";
diff --git a/core/src/test/java/org/elasticsearch/script/ScriptSettingsTests.java b/core/src/test/java/org/elasticsearch/script/ScriptSettingsTests.java
index 917650d36b81c..67a94f588075b 100644
--- a/core/src/test/java/org/elasticsearch/script/ScriptSettingsTests.java
+++ b/core/src/test/java/org/elasticsearch/script/ScriptSettingsTests.java
@@ -34,6 +34,29 @@
 
 public class ScriptSettingsTests extends ESTestCase {
 
+    public static Setting<?>[] buildDeprecatedSettingsArray(ScriptSettings scriptSettings, String... keys) {
+        return buildDeprecatedSettingsArray(null, scriptSettings, keys);
+    }
+
+    public static Setting<?>[] buildDeprecatedSettingsArray(Setting<?>[] deprecated, ScriptSettings scriptSettings, String... keys) {
+        Setting<?>[] settings = new Setting[keys.length + (deprecated == null ? 0 : deprecated.length)];
+        int count = 0;
+
+        for (Setting<?> setting : scriptSettings.getSettings()) {
+            for (String key : keys) {
+                if (setting.getKey().equals(key)) {
+                    settings[count++] = setting;
+                }
+            }
+        }
+
+        if (deprecated != null) {
+            System.arraycopy(deprecated, 0, settings, keys.length, deprecated.length);
+        }
+
+        return settings;
+    }
+
     public void testDefaultLegacyLanguageIsPainless() {
         ScriptEngineRegistry scriptEngineRegistry =
                 new ScriptEngineRegistry(Collections.singletonList(new CustomScriptEngineService()));
@@ -81,6 +104,7 @@ public void testSettingsAreProperlyPropogated() {
                 assertThat(setting.getDefaultRaw(s), equalTo(Boolean.toString(enabled)));
             }
         }
+        assertSettingDeprecationsAndWarnings(buildDeprecatedSettingsArray(scriptSettings, "script.inline"));
     }
 
     private static class CustomScriptEngineService implements ScriptEngineService {
diff --git a/core/src/test/java/org/elasticsearch/script/ScriptTests.java b/core/src/test/java/org/elasticsearch/script/ScriptTests.java
index 316a1c8451b58..9584bf01a5c83 100644
--- a/core/src/test/java/org/elasticsearch/script/ScriptTests.java
+++ b/core/src/test/java/org/elasticsearch/script/ScriptTests.java
@@ -19,13 +19,11 @@
 
 package org.elasticsearch.script;
 
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.InputStreamStreamInput;
 import org.elasticsearch.common.io.stream.OutputStreamStreamOutput;
 import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESTestCase;
@@ -41,20 +39,18 @@
 public class ScriptTests extends ESTestCase {
 
     public void testScriptParsing() throws IOException {
-        XContent xContent = randomFrom(XContentType.JSON, XContentType.YAML).xContent();
-        Script expectedScript = createScript(xContent);
-        try (XContentBuilder builder = XContentBuilder.builder(xContent)) {
+        Script expectedScript = createScript();
+        try (XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()))) {
             expectedScript.toXContent(builder, ToXContent.EMPTY_PARAMS);
-            try (XContentParser parser = XContentHelper.createParser(builder.bytes())) {
-                Script actualScript = Script.parse(parser, ParseFieldMatcher.STRICT);
+            try (XContentParser parser = createParser(builder)) {
+                Script actualScript = Script.parse(parser);
                 assertThat(actualScript, equalTo(expectedScript));
             }
         }
     }
 
     public void testScriptSerialization() throws IOException {
-        XContent xContent = randomFrom(XContentType.JSON, XContentType.YAML).xContent();
-        Script expectedScript = createScript(xContent);
+        Script expectedScript = createScript();
         try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
             expectedScript.writeTo(new OutputStreamStreamOutput(out));
             try (ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray())) {
@@ -64,26 +60,26 @@ public void testScriptSerialization() throws IOException {
         }
     }
 
-    private Script createScript(XContent xContent) throws IOException {
-        final Map<String, Object> params = randomBoolean() ? null : Collections.singletonMap("key", "value");
-        ScriptService.ScriptType scriptType = randomFrom(ScriptService.ScriptType.values());
+    private Script createScript() throws IOException {
+        final Map<String, Object> params = randomBoolean() ? Collections.emptyMap() : Collections.singletonMap("key", "value");
+        ScriptType scriptType = randomFrom(ScriptType.values());
         String script;
-        if (scriptType == ScriptService.ScriptType.INLINE) {
-            try (XContentBuilder builder = XContentBuilder.builder(xContent)) {
+        if (scriptType == ScriptType.INLINE) {
+            try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
                 builder.startObject();
-                builder.field("field", randomAsciiOfLengthBetween(1, 5));
+                builder.field("field", randomAlphaOfLengthBetween(1, 5));
                 builder.endObject();
                 script = builder.string();
             }
         } else {
-            script = randomAsciiOfLengthBetween(1, 5);
+            script = randomAlphaOfLengthBetween(1, 5);
         }
         return new Script(
-                script,
-                scriptType,
-                randomFrom("_lang1", "_lang2", null),
-                params,
-                scriptType == ScriptService.ScriptType.INLINE ? xContent.type() : null
+            scriptType,
+            scriptType == ScriptType.STORED ? null : randomFrom("_lang1", "_lang2", "_lang3"),
+            script,
+            scriptType == ScriptType.INLINE ?
+                    Collections.singletonMap(Script.CONTENT_TYPE_OPTION, XContentType.JSON.mediaType()) : null, params
         );
     }
 
diff --git a/core/src/test/java/org/elasticsearch/script/StoredScriptSourceTests.java b/core/src/test/java/org/elasticsearch/script/StoredScriptSourceTests.java
new file mode 100644
index 0000000000000..a99c897ec3445
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/script/StoredScriptSourceTests.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.script;
+
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.AbstractSerializingTestCase;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+public class StoredScriptSourceTests extends AbstractSerializingTestCase<StoredScriptSource> {
+
+    @Override
+    protected StoredScriptSource createTestInstance() {
+        String lang = randomAlphaOfLengthBetween(1, 20);
+        XContentType xContentType = randomFrom(XContentType.JSON, XContentType.YAML);
+        try {
+            XContentBuilder template = XContentBuilder.builder(xContentType.xContent());
+            template.startObject();
+            template.startObject("query");
+            template.startObject("match");
+            template.field("title", "{{query_string}}");
+            template.endObject();
+            template.endObject();
+            template.endObject();
+            Map<String, String> options = new HashMap<>();
+            if (randomBoolean()) {
+                options.put(Script.CONTENT_TYPE_OPTION, xContentType.mediaType());
+            }
+            return StoredScriptSource.parse(lang, template.bytes(), xContentType);
+        } catch (IOException e) {
+            throw new AssertionError("Failed to create test instance", e);
+        }
+    }
+
+    @Override
+    protected StoredScriptSource doParseInstance(XContentParser parser) throws IOException {
+        return StoredScriptSource.fromXContent(parser);
+    }
+
+    @Override
+    protected Reader<StoredScriptSource> instanceReader() {
+        return StoredScriptSource::new;
+    }
+
+
+}
diff --git a/core/src/test/java/org/elasticsearch/script/StoredScriptTests.java b/core/src/test/java/org/elasticsearch/script/StoredScriptTests.java
new file mode 100644
index 0000000000000..5fd1002b38ed9
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/script/StoredScriptTests.java
@@ -0,0 +1,338 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.script;
+
+import org.elasticsearch.ResourceNotFoundException;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collections;
+
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.singletonMap;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+
+public class StoredScriptTests extends ESTestCase {
+
+    public void testBasicAddDelete() {
+        StoredScriptSource source = new StoredScriptSource("lang", "code", emptyMap());
+        ScriptMetaData smd = ScriptMetaData.putStoredScript(null, "test", source);
+
+        assertThat(smd.getStoredScript("test", null), equalTo(source));
+        assertThat(smd.getStoredScript("test", "lang"), equalTo(source));
+
+        smd = ScriptMetaData.deleteStoredScript(smd, "test", null);
+
+        assertThat(smd.getStoredScript("test", null), nullValue());
+        assertThat(smd.getStoredScript("test", "lang"), nullValue());
+    }
+
+    public void testDifferentMultiAddDelete() {
+        StoredScriptSource source0 = new StoredScriptSource("lang0", "code0", emptyMap());
+        StoredScriptSource source1 = new StoredScriptSource("lang0", "code1", emptyMap());
+        StoredScriptSource source2 = new StoredScriptSource("lang1", "code2",
+            singletonMap(Script.CONTENT_TYPE_OPTION, XContentType.JSON.mediaType()));
+
+        ScriptMetaData smd = ScriptMetaData.putStoredScript(null, "test0", source0);
+        smd = ScriptMetaData.putStoredScript(smd, "test1", source1);
+        smd = ScriptMetaData.putStoredScript(smd, "test2", source2);
+
+        assertThat(smd.getStoredScript("test0", null), equalTo(source0));
+        assertThat(smd.getStoredScript("test0", "lang0"), equalTo(source0));
+        assertThat(smd.getStoredScript("test1", null), equalTo(source1));
+        assertThat(smd.getStoredScript("test1", "lang0"), equalTo(source1));
+        assertThat(smd.getStoredScript("test2", null), equalTo(source2));
+        assertThat(smd.getStoredScript("test2", "lang1"), equalTo(source2));
+
+        assertThat(smd.getStoredScript("test3", null), nullValue());
+        assertThat(smd.getStoredScript("test0", "lang1"), nullValue());
+        assertThat(smd.getStoredScript("test2", "lang0"), nullValue());
+
+        smd = ScriptMetaData.deleteStoredScript(smd, "test0", null);
+
+        assertThat(smd.getStoredScript("test0", null), nullValue());
+        assertThat(smd.getStoredScript("test0", "lang0"), nullValue());
+        assertThat(smd.getStoredScript("test1", null), equalTo(source1));
+        assertThat(smd.getStoredScript("test1", "lang0"), equalTo(source1));
+        assertThat(smd.getStoredScript("test2", null), equalTo(source2));
+        assertThat(smd.getStoredScript("test2", "lang1"), equalTo(source2));
+
+        assertThat(smd.getStoredScript("test3", null), nullValue());
+        assertThat(smd.getStoredScript("test0", "lang1"), nullValue());
+        assertThat(smd.getStoredScript("test2", "lang0"), nullValue());
+
+        smd = ScriptMetaData.deleteStoredScript(smd, "test2", "lang1");
+
+        assertThat(smd.getStoredScript("test0", null), nullValue());
+        assertThat(smd.getStoredScript("test0", "lang0"), nullValue());
+        assertThat(smd.getStoredScript("test1", null), equalTo(source1));
+        assertThat(smd.getStoredScript("test1", "lang0"), equalTo(source1));
+        assertThat(smd.getStoredScript("test2", null), nullValue());
+        assertThat(smd.getStoredScript("test2", "lang1"), nullValue());
+
+        assertThat(smd.getStoredScript("test3", null), nullValue());
+        assertThat(smd.getStoredScript("test0", "lang1"), nullValue());
+        assertThat(smd.getStoredScript("test2", "lang0"), nullValue());
+
+        smd = ScriptMetaData.deleteStoredScript(smd, "test1", "lang0");
+
+        assertThat(smd.getStoredScript("test0", null), nullValue());
+        assertThat(smd.getStoredScript("test0", "lang0"), nullValue());
+        assertThat(smd.getStoredScript("test1", null), nullValue());
+        assertThat(smd.getStoredScript("test1", "lang0"), nullValue());
+        assertThat(smd.getStoredScript("test2", null), nullValue());
+        assertThat(smd.getStoredScript("test2", "lang1"), nullValue());
+
+        assertThat(smd.getStoredScript("test3", null), nullValue());
+        assertThat(smd.getStoredScript("test0", "lang1"), nullValue());
+        assertThat(smd.getStoredScript("test2", "lang0"), nullValue());
+    }
+
+    public void testSameMultiAddDelete() {
+        StoredScriptSource source0 = new StoredScriptSource("lang0", "code0", emptyMap());
+        StoredScriptSource source1 = new StoredScriptSource("lang1", "code1", emptyMap());
+        StoredScriptSource source2 = new StoredScriptSource("lang2", "code1", emptyMap());
+        StoredScriptSource source3 = new StoredScriptSource("lang1", "code2",
+            singletonMap(Script.CONTENT_TYPE_OPTION, XContentType.JSON.mediaType()));
+
+        ScriptMetaData smd = ScriptMetaData.putStoredScript(null, "test0", source0);
+        smd = ScriptMetaData.putStoredScript(smd, "test0", source1);
+        assertWarnings("stored script [test0] already exists using a different lang [lang0]," +
+            " the new namespace for stored scripts will only use (id) instead of (lang, id)");
+        smd = ScriptMetaData.putStoredScript(smd, "test3", source3);
+        smd = ScriptMetaData.putStoredScript(smd, "test0", source2);
+        assertWarnings("stored script [test0] already exists using a different lang [lang1]," +
+            " the new namespace for stored scripts will only use (id) instead of (lang, id)");
+
+        assertThat(smd.getStoredScript("test0", null), equalTo(source2));
+        assertThat(smd.getStoredScript("test0", "lang0"), equalTo(source0));
+        assertThat(smd.getStoredScript("test0", "lang1"), equalTo(source1));
+        assertThat(smd.getStoredScript("test0", "lang2"), equalTo(source2));
+        assertThat(smd.getStoredScript("test3", null), equalTo(source3));
+        assertThat(smd.getStoredScript("test3", "lang1"), equalTo(source3));
+
+        smd = ScriptMetaData.deleteStoredScript(smd, "test0", "lang1");
+
+        assertThat(smd.getStoredScript("test0", null), equalTo(source2));
+        assertThat(smd.getStoredScript("test0", "lang0"), equalTo(source0));
+        assertThat(smd.getStoredScript("test0", "lang1"), nullValue());
+        assertThat(smd.getStoredScript("test0", "lang2"), equalTo(source2));
+        assertThat(smd.getStoredScript("test3", null), equalTo(source3));
+        assertThat(smd.getStoredScript("test3", "lang1"), equalTo(source3));
+
+        smd = ScriptMetaData.deleteStoredScript(smd, "test0", null);
+
+        assertThat(smd.getStoredScript("test0", null), nullValue());
+        assertThat(smd.getStoredScript("test0", "lang0"), equalTo(source0));
+        assertThat(smd.getStoredScript("test0", "lang1"), nullValue());
+        assertThat(smd.getStoredScript("test0", "lang2"), nullValue());
+        assertThat(smd.getStoredScript("test3", null), equalTo(source3));
+        assertThat(smd.getStoredScript("test3", "lang1"), equalTo(source3));
+
+        smd = ScriptMetaData.deleteStoredScript(smd, "test3", "lang1");
+
+        assertThat(smd.getStoredScript("test0", null), nullValue());
+        assertThat(smd.getStoredScript("test0", "lang0"), equalTo(source0));
+        assertThat(smd.getStoredScript("test0", "lang1"), nullValue());
+        assertThat(smd.getStoredScript("test0", "lang2"), nullValue());
+        assertThat(smd.getStoredScript("test3", null), nullValue());
+        assertThat(smd.getStoredScript("test3", "lang1"), nullValue());
+
+        smd = ScriptMetaData.deleteStoredScript(smd, "test0", "lang0");
+
+        assertThat(smd.getStoredScript("test0", null), nullValue());
+        assertThat(smd.getStoredScript("test0", "lang0"), nullValue());
+        assertThat(smd.getStoredScript("test0", "lang1"), nullValue());
+        assertThat(smd.getStoredScript("test0", "lang2"), nullValue());
+        assertThat(smd.getStoredScript("test3", null), nullValue());
+        assertThat(smd.getStoredScript("test3", "lang1"), nullValue());
+    }
+
+    public void testInvalidDelete() {
+        ResourceNotFoundException rnfe =
+            expectThrows(ResourceNotFoundException.class, () -> ScriptMetaData.deleteStoredScript(null, "test", "lang"));
+        assertThat(rnfe.getMessage(), equalTo("stored script [test] using lang [lang] does not exist and cannot be deleted"));
+
+        rnfe = expectThrows(ResourceNotFoundException.class, () -> ScriptMetaData.deleteStoredScript(null, "test", null));
+        assertThat(rnfe.getMessage(), equalTo("stored script [test] does not exist and cannot be deleted"));
+    }
+
+    public void testSourceParsing() throws Exception {
+        // simple script value string
+        try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) {
+            builder.startObject().field("script", "code").endObject();
+
+            StoredScriptSource parsed = StoredScriptSource.parse("lang", builder.bytes(), XContentType.JSON);
+            StoredScriptSource source = new StoredScriptSource("lang", "code", Collections.emptyMap());
+
+            assertThat(parsed, equalTo(source));
+        }
+
+        // simple template value string
+        try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) {
+            builder.startObject().field("template", "code").endObject();
+
+            StoredScriptSource parsed = StoredScriptSource.parse("lang", builder.bytes(), XContentType.JSON);
+            StoredScriptSource source = new StoredScriptSource("lang", "code", Collections.emptyMap());
+
+            assertThat(parsed, equalTo(source));
+        }
+
+        // complex template with wrapper template object
+        try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) {
+            builder.startObject().field("template").startObject().field("query", "code").endObject().endObject();
+            String code;
+
+            try (XContentBuilder cb = XContentFactory.contentBuilder(builder.contentType())) {
+                code = cb.startObject().field("query", "code").endObject().string();
+            }
+
+            StoredScriptSource parsed = StoredScriptSource.parse("lang", builder.bytes(), XContentType.JSON);
+            StoredScriptSource source = new StoredScriptSource("lang", code, Collections.emptyMap());
+
+            assertThat(parsed, equalTo(source));
+        }
+
+        // complex template with no wrapper object
+        try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) {
+            builder.startObject().field("query", "code").endObject();
+            String code;
+
+            try (XContentBuilder cb = XContentFactory.contentBuilder(builder.contentType())) {
+                code = cb.startObject().field("query", "code").endObject().string();
+            }
+
+            StoredScriptSource parsed = StoredScriptSource.parse("lang", builder.bytes(), XContentType.JSON);
+            StoredScriptSource source = new StoredScriptSource("lang", code, Collections.emptyMap());
+
+            assertThat(parsed, equalTo(source));
+        }
+
+        // complex template using script as the field name
+        try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) {
+            builder.startObject().field("script").startObject().field("query", "code").endObject().endObject();
+            String code;
+
+            try (XContentBuilder cb = XContentFactory.contentBuilder(builder.contentType())) {
+                code = cb.startObject().field("query", "code").endObject().string();
+            }
+
+            StoredScriptSource parsed = StoredScriptSource.parse("lang", builder.bytes(), XContentType.JSON);
+            StoredScriptSource source = new StoredScriptSource("lang", code, Collections.emptyMap());
+
+            assertThat(parsed, equalTo(source));
+        }
+
+        // complex script with script object
+        try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) {
+            builder.startObject().field("script").startObject().field("lang", "lang").field("source", "code").endObject().endObject();
+
+            StoredScriptSource parsed = StoredScriptSource.parse(null, builder.bytes(), XContentType.JSON);
+            StoredScriptSource source = new StoredScriptSource("lang", "code", Collections.emptyMap());
+
+            assertThat(parsed, equalTo(source));
+        }
+
+        // complex script using "code" backcompat
+        try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) {
+            builder.startObject().field("script").startObject().field("lang", "lang").field("code", "code").endObject().endObject();
+
+            StoredScriptSource parsed = StoredScriptSource.parse(null, builder.bytes(), XContentType.JSON);
+            StoredScriptSource source = new StoredScriptSource("lang", "code", Collections.emptyMap());
+
+            assertThat(parsed, equalTo(source));
+        }
+        assertWarnings("Deprecated field [code] used, expected [source] instead");
+
+        // complex script with script object and empty options
+        try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) {
+            builder.startObject().field("script").startObject().field("lang", "lang").field("source", "code")
+                .field("options").startObject().endObject().endObject().endObject();
+
+            StoredScriptSource parsed = StoredScriptSource.parse(null, builder.bytes(), XContentType.JSON);
+            StoredScriptSource source = new StoredScriptSource("lang", "code", Collections.emptyMap());
+
+            assertThat(parsed, equalTo(source));
+        }
+
+        // complex script with embedded template
+        try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) {
+            builder.startObject().field("script").startObject().field("lang", "lang").startObject("source").field("query", "code")
+                .endObject().startObject("options").endObject().endObject().endObject().string();
+            String code;
+
+            try (XContentBuilder cb = XContentFactory.contentBuilder(builder.contentType())) {
+                code = cb.startObject().field("query", "code").endObject().string();
+            }
+
+            StoredScriptSource parsed = StoredScriptSource.parse(null, builder.bytes(), XContentType.JSON);
+            StoredScriptSource source = new StoredScriptSource("lang", code,
+                Collections.singletonMap(Script.CONTENT_TYPE_OPTION, builder.contentType().mediaType()));
+
+            assertThat(parsed, equalTo(source));
+        }
+    }
+
+    public void testSourceParsingErrors() throws Exception {
+        // check for missing lang parameter when parsing a template
+        try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) {
+            builder.startObject().field("template", "code").endObject();
+
+            IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () ->
+                StoredScriptSource.parse(null, builder.bytes(), XContentType.JSON));
+            assertThat(iae.getMessage(), equalTo("unexpected stored script format"));
+        }
+
+        // check for missing lang parameter when parsing a script
+        try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) {
+            builder.startObject().field("script").startObject().field("source", "code").endObject().endObject();
+
+            IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () ->
+                StoredScriptSource.parse(null, builder.bytes(), XContentType.JSON));
+            assertThat(iae.getMessage(), equalTo("must specify lang for stored script"));
+        }
+
+        // check for missing source parameter when parsing a script
+        try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) {
+            builder.startObject().field("script").startObject().field("lang", "lang").endObject().endObject();
+
+            IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () ->
+                StoredScriptSource.parse(null, builder.bytes(), XContentType.JSON));
+            assertThat(iae.getMessage(), equalTo("must specify source for stored script"));
+        }
+
+        // check for illegal options parameter when parsing a script
+        try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) {
+            builder.startObject().field("script").startObject().field("lang", "lang").field("source", "code")
+                .startObject("options").field("option", "option").endObject().endObject().endObject();
+
+            IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () ->
+                StoredScriptSource.parse(null, builder.bytes(), XContentType.JSON));
+            assertThat(iae.getMessage(), equalTo("illegal compiler options [{option=option}] specified"));
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/script/StoredScriptsIT.java b/core/src/test/java/org/elasticsearch/script/StoredScriptsIT.java
index 2bc58bb0d22dd..80a8f4deaa741 100644
--- a/core/src/test/java/org/elasticsearch/script/StoredScriptsIT.java
+++ b/core/src/test/java/org/elasticsearch/script/StoredScriptsIT.java
@@ -20,6 +20,7 @@
 
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 
@@ -50,37 +51,38 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
 
     public void testBasics() {
         assertAcked(client().admin().cluster().preparePutStoredScript()
-                .setScriptLang(LANG)
+                .setLang(LANG)
                 .setId("foobar")
-                .setSource(new BytesArray("{\"script\":\"1\"}")));
+                .setContent(new BytesArray("{\"script\":\"1\"}"), XContentType.JSON));
         String script = client().admin().cluster().prepareGetStoredScript(LANG, "foobar")
-                .get().getStoredScript();
+                .get().getSource().getSource();
         assertNotNull(script);
         assertEquals("1", script);
 
         assertAcked(client().admin().cluster().prepareDeleteStoredScript()
                 .setId("foobar")
-                .setScriptLang(LANG));
-        script = client().admin().cluster().prepareGetStoredScript(LANG, "foobar")
-                .get().getStoredScript();
-        assertNull(script);
+                .setLang(LANG));
+        StoredScriptSource source = client().admin().cluster().prepareGetStoredScript(LANG, "foobar")
+                .get().getSource();
+        assertNull(source);
 
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> client().admin().cluster().preparePutStoredScript()
-                .setScriptLang("lang#")
+                .setLang("lang#")
                 .setId("id#")
-                .setSource(new BytesArray("{}"))
+                .setContent(new BytesArray("{}"), XContentType.JSON)
                 .get());
-        assertEquals("Validation Failed: 1: id can't contain: '#';2: lang can't contain: '#';", e.getMessage());
+        assertEquals("Validation Failed: 1: id cannot contain '#' for stored script;" +
+            "2: lang cannot contain '#' for stored script;", e.getMessage());
     }
 
     public void testMaxScriptSize() {
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> client().admin().cluster().preparePutStoredScript()
-                .setScriptLang(LANG)
+                .setLang(LANG)
                 .setId("foobar")
-                .setSource(new BytesArray(randomAsciiOfLength(SCRIPT_MAX_SIZE_IN_BYTES + 1)))
+                .setContent(new BytesArray(randomAlphaOfLength(SCRIPT_MAX_SIZE_IN_BYTES + 1)), XContentType.JSON)
                 .get()
         );
-        assertEquals("Limit of script size in bytes [64] has been exceeded for script [foobar] with size [65]", e.getMessage());
+        assertEquals("exceeded max allowed stored script size in bytes [64] with size [65] for script [foobar]", e.getMessage());
     }
 
     public static class CustomScriptPlugin extends MockScriptPlugin {
diff --git a/core/src/test/java/org/elasticsearch/search/AbstractSearchTestCase.java b/core/src/test/java/org/elasticsearch/search/AbstractSearchTestCase.java
new file mode 100644
index 0000000000000..5fe9d9d75fbab
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/AbstractSearchTestCase.java
@@ -0,0 +1,269 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search;
+
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.common.CheckedFunction;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.indices.IndicesModule;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.SearchPlugin;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.collapse.CollapseBuilderTests;
+import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilderTests;
+import org.elasticsearch.search.rescore.QueryRescoreBuilderTests;
+import org.elasticsearch.search.suggest.SuggestBuilderTests;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.function.Supplier;
+
+public abstract class AbstractSearchTestCase extends ESTestCase {
+
+    protected NamedWriteableRegistry namedWriteableRegistry;
+    private TestSearchExtPlugin searchExtPlugin;
+    private NamedXContentRegistry xContentRegistry;
+
+    public void setUp() throws Exception {
+        super.setUp();
+        IndicesModule indicesModule = new IndicesModule(Collections.emptyList());
+        searchExtPlugin = new TestSearchExtPlugin();
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.singletonList(searchExtPlugin));
+        List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
+        entries.addAll(indicesModule.getNamedWriteables());
+        entries.addAll(searchModule.getNamedWriteables());
+        namedWriteableRegistry = new NamedWriteableRegistry(entries);
+        xContentRegistry = new NamedXContentRegistry(searchModule.getNamedXContents());
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return xContentRegistry;
+    }
+
+    protected SearchSourceBuilder createSearchSourceBuilder() {
+        Supplier<List<SearchExtBuilder>> randomExtBuilders = () -> {
+            Set<String> elementNames = new HashSet<>(searchExtPlugin.getSupportedElements().keySet());
+            int numSearchExts = randomIntBetween(1, elementNames.size());
+            while(elementNames.size() > numSearchExts) {
+                elementNames.remove(randomFrom(elementNames));
+            }
+            List<SearchExtBuilder> searchExtBuilders = new ArrayList<>();
+            for (String elementName : elementNames) {
+                searchExtBuilders.add(searchExtPlugin.getSupportedElements().get(elementName).apply(randomAlphaOfLengthBetween(3, 10)));
+            }
+            return searchExtBuilders;
+        };
+        return RandomSearchRequestGenerator.randomSearchSourceBuilder(
+                HighlightBuilderTests::randomHighlighterBuilder,
+                SuggestBuilderTests::randomSuggestBuilder,
+                QueryRescoreBuilderTests::randomRescoreBuilder,
+                randomExtBuilders,
+                CollapseBuilderTests::randomCollapseBuilder);
+    }
+
+    protected SearchRequest createSearchRequest() throws IOException {
+        return RandomSearchRequestGenerator.randomSearchRequest(this::createSearchSourceBuilder);
+    }
+
+    private static class TestSearchExtPlugin extends Plugin implements SearchPlugin {
+        private final List<SearchExtSpec<? extends SearchExtBuilder>> searchExtSpecs;
+        private final Map<String, Function<String, ? extends SearchExtBuilder>> supportedElements;
+
+        private TestSearchExtPlugin() {
+            int numSearchExts = randomIntBetween(1, 3);
+            this.searchExtSpecs = new ArrayList<>(numSearchExts);
+            this.supportedElements = new HashMap<>();
+            for (int i = 0; i < numSearchExts; i++) {
+                switch (randomIntBetween(0, 2)) {
+                    case 0:
+                        if (this.supportedElements.put(TestSearchExtBuilder1.NAME, TestSearchExtBuilder1::new) == null) {
+                            this.searchExtSpecs.add(new SearchExtSpec<>(TestSearchExtBuilder1.NAME, TestSearchExtBuilder1::new,
+                                    new TestSearchExtParser<>(TestSearchExtBuilder1::new)));
+                        }
+                        break;
+                    case 1:
+                        if (this.supportedElements.put(TestSearchExtBuilder2.NAME, TestSearchExtBuilder2::new) == null) {
+                            this.searchExtSpecs.add(new SearchExtSpec<>(TestSearchExtBuilder2.NAME, TestSearchExtBuilder2::new,
+                                    new TestSearchExtParser<>(TestSearchExtBuilder2::new)));
+                        }
+                        break;
+                    case 2:
+                        if (this.supportedElements.put(TestSearchExtBuilder3.NAME, TestSearchExtBuilder3::new) == null) {
+                            this.searchExtSpecs.add(new SearchExtSpec<>(TestSearchExtBuilder3.NAME, TestSearchExtBuilder3::new,
+                                    new TestSearchExtParser<>(TestSearchExtBuilder3::new)));
+                        }
+                        break;
+                    default:
+                        throw new UnsupportedOperationException();
+                }
+            }
+        }
+
+        Map<String, Function<String, ? extends SearchExtBuilder>> getSupportedElements() {
+            return supportedElements;
+        }
+
+        @Override
+        public List<SearchExtSpec<?>> getSearchExts() {
+            return searchExtSpecs;
+        }
+    }
+
+    private static class TestSearchExtParser<T extends SearchExtBuilder> implements CheckedFunction<XContentParser, T, IOException> {
+        private final Function<String, T> searchExtBuilderFunction;
+
+        TestSearchExtParser(Function<String, T> searchExtBuilderFunction) {
+            this.searchExtBuilderFunction = searchExtBuilderFunction;
+        }
+
+        @Override
+        public T apply(XContentParser parser) throws IOException {
+            return searchExtBuilderFunction.apply(parseField(parser));
+        }
+
+        String parseField(XContentParser parser) throws IOException {
+            if (parser.currentToken() != XContentParser.Token.START_OBJECT) {
+                throw new ParsingException(parser.getTokenLocation(), "start_object expected, found " + parser.currentToken());
+            }
+            if (parser.nextToken() != XContentParser.Token.FIELD_NAME) {
+                throw new ParsingException(parser.getTokenLocation(), "field_name expected, found " + parser.currentToken());
+            }
+            String field = parser.currentName();
+            if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
+                throw new ParsingException(parser.getTokenLocation(), "start_object expected, found " + parser.currentToken());
+            }
+            if (parser.nextToken() != XContentParser.Token.END_OBJECT) {
+                throw new ParsingException(parser.getTokenLocation(), "end_object expected, found " + parser.currentToken());
+            }
+            if (parser.nextToken() != XContentParser.Token.END_OBJECT) {
+                throw new ParsingException(parser.getTokenLocation(), "end_object expected, found " + parser.currentToken());
+            }
+            return field;
+        }
+    }
+
+    //Would be nice to have a single builder that gets its name as a parameter, but the name wouldn't get a value when the object
+    //is created reading from the stream (constructor that takes a StreamInput) which is a problem as we check that after reading
+    //a named writeable its name is the expected one. That's why we go for the following less dynamic approach.
+    private static class TestSearchExtBuilder1 extends TestSearchExtBuilder {
+        private static final String NAME = "name1";
+
+        TestSearchExtBuilder1(String field) {
+            super(NAME, field);
+        }
+
+        TestSearchExtBuilder1(StreamInput in) throws IOException {
+            super(NAME, in);
+        }
+    }
+
+    private static class TestSearchExtBuilder2 extends TestSearchExtBuilder {
+        private static final String NAME = "name2";
+
+        TestSearchExtBuilder2(String field) {
+            super(NAME, field);
+        }
+
+        TestSearchExtBuilder2(StreamInput in) throws IOException {
+            super(NAME, in);
+        }
+    }
+
+    private static class TestSearchExtBuilder3 extends TestSearchExtBuilder {
+        private static final String NAME = "name3";
+
+        TestSearchExtBuilder3(String field) {
+            super(NAME, field);
+        }
+
+        TestSearchExtBuilder3(StreamInput in) throws IOException {
+            super(NAME, in);
+        }
+    }
+
+    private abstract static class TestSearchExtBuilder extends SearchExtBuilder {
+        final String objectName;
+        protected final String name;
+
+        TestSearchExtBuilder(String name, String objectName) {
+            this.name = name;
+            this.objectName = objectName;
+        }
+
+        TestSearchExtBuilder(String name, StreamInput in) throws IOException {
+            this.name = name;
+            this.objectName = in.readString();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeString(objectName);
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) {
+                return true;
+            }
+            if (o == null || getClass() != o.getClass()) {
+                return false;
+            }
+            TestSearchExtBuilder that = (TestSearchExtBuilder) o;
+            return Objects.equals(objectName, that.objectName) &&
+                    Objects.equals(name, that.name);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(objectName, name);
+        }
+
+        @Override
+        public String getWriteableName() {
+            return name;
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject(name);
+            builder.startObject(objectName);
+            builder.endObject();
+            builder.endObject();
+            return builder;
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/ClearScrollResponseTests.java b/core/src/test/java/org/elasticsearch/search/ClearScrollResponseTests.java
new file mode 100644
index 0000000000000..c4571e02ecc00
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/ClearScrollResponseTests.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search;
+
+import org.elasticsearch.action.search.ClearScrollResponse;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
+
+public class ClearScrollResponseTests extends ESTestCase {
+
+    public void testToXContent() throws IOException {
+        ClearScrollResponse clearScrollResponse = new ClearScrollResponse(true, 10);
+        try (XContentBuilder builder = JsonXContent.contentBuilder()) {
+            clearScrollResponse.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        }
+        assertEquals(true, clearScrollResponse.isSucceeded());
+        assertEquals(10, clearScrollResponse.getNumFreed());
+    }
+
+    public void testToAndFromXContent() throws IOException {
+        XContentType xContentType = randomFrom(XContentType.values());
+        ClearScrollResponse originalResponse = createTestItem();
+        BytesReference originalBytes = toShuffledXContent(originalResponse, xContentType, ToXContent.EMPTY_PARAMS, randomBoolean());
+        ClearScrollResponse parsedResponse;
+        try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) {
+            parsedResponse = ClearScrollResponse.fromXContent(parser);
+        }
+        assertEquals(originalResponse.isSucceeded(), parsedResponse.isSucceeded());
+        assertEquals(originalResponse.getNumFreed(), parsedResponse.getNumFreed());
+        BytesReference parsedBytes = XContentHelper.toXContent(parsedResponse, xContentType, randomBoolean());
+        assertToXContentEquivalent(originalBytes, parsedBytes, xContentType);
+    }
+
+    private static ClearScrollResponse createTestItem() {
+        return new ClearScrollResponse(randomBoolean(), randomIntBetween(0, Integer.MAX_VALUE));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/DocValueFormatTests.java b/core/src/test/java/org/elasticsearch/search/DocValueFormatTests.java
index 049e133b35740..7bf5308eb635e 100644
--- a/core/src/test/java/org/elasticsearch/search/DocValueFormatTests.java
+++ b/core/src/test/java/org/elasticsearch/search/DocValueFormatTests.java
@@ -19,9 +19,6 @@
 
 package org.elasticsearch.search;
 
-import java.util.ArrayList;
-import java.util.List;
-
 import org.apache.lucene.document.InetAddressPoint;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
@@ -34,6 +31,9 @@
 import org.elasticsearch.test.ESTestCase;
 import org.joda.time.DateTimeZone;
 
+import java.util.ArrayList;
+import java.util.List;
+
 public class DocValueFormatTests extends ESTestCase {
 
     public void testSerialization() throws Exception {
@@ -108,6 +108,18 @@ public void testIpFormat() {
                 DocValueFormat.IP.format(new BytesRef(InetAddressPoint.encode(InetAddresses.forString("::1")))));
     }
 
+    public void testDecimalFormat() {
+        DocValueFormat formatter = new DocValueFormat.Decimal("###.##");
+        assertEquals("0", formatter.format(0.0d));
+        assertEquals("1", formatter.format(1d));
+        formatter = new DocValueFormat.Decimal("000.000");
+        assertEquals("-000.500", formatter.format(-0.5));
+        formatter = new DocValueFormat.Decimal("###,###.###");
+        assertEquals("0.86", formatter.format(0.8598023539251286d));
+        formatter = new DocValueFormat.Decimal("###,###.###");
+        assertEquals("859,802.354", formatter.format(0.8598023539251286d * 1_000_000));
+    }
+
     public void testRawParse() {
         assertEquals(-1L, DocValueFormat.RAW.parseLong("-1", randomBoolean(), null));
         assertEquals(1L, DocValueFormat.RAW.parseLong("1", randomBoolean(), null));
@@ -145,4 +157,16 @@ public void testIPParse() {
         assertEquals(new BytesRef(InetAddressPoint.encode(InetAddresses.forString("::1"))),
                 DocValueFormat.IP.parseBytesRef("::1"));
     }
+
+    public void testDecimalParse() {
+        DocValueFormat parser = new DocValueFormat.Decimal("###.##");
+        assertEquals(0.0d, parser.parseDouble(randomFrom("0.0", "0", ".0", ".0000"), true, null), 0.0d);
+        assertEquals(-1.0d, parser.parseDouble(randomFrom("-1.0", "-1", "-1.0", "-1.0000"), true, null), 0.0d);
+        assertEquals(0.0d, parser.parseLong("0", true, null), 0.0d);
+        assertEquals(1.0d, parser.parseLong("1", true, null), 0.0d);
+        parser = new DocValueFormat.Decimal("###,###.###");
+        assertEquals(859802.354d, parser.parseDouble("859,802.354", true, null), 0.0d);
+        assertEquals(0.859d, parser.parseDouble("0.859", true, null), 0.0d);
+        assertEquals(0.8598023539251286d, parser.parseDouble("0.8598023539251286", true, null), 0.0d);
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/NestedIdentityTests.java b/core/src/test/java/org/elasticsearch/search/NestedIdentityTests.java
new file mode 100644
index 0000000000000..0ec29cc2cc0a2
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/NestedIdentityTests.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search;
+
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.search.SearchHit.NestedIdentity;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Supplier;
+
+import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
+
+public class NestedIdentityTests extends ESTestCase {
+
+    public static NestedIdentity createTestItem(int depth) {
+        String field = frequently() ? randomAlphaOfLengthBetween(1, 20) : randomRealisticUnicodeOfCodepointLengthBetween(1, 20);
+        int offset = randomInt(10);
+        NestedIdentity child = null;
+        if (depth > 0) {
+            child = createTestItem(depth - 1);
+        }
+        return new NestedIdentity(field, offset, child);
+    }
+
+    public void testFromXContent() throws IOException {
+        NestedIdentity nestedIdentity = createTestItem(randomInt(3));
+        XContentType xcontentType = randomFrom(XContentType.values());
+        XContentBuilder builder = XContentFactory.contentBuilder(xcontentType);
+        if (randomBoolean()) {
+            builder.prettyPrint();
+        }
+        builder = nestedIdentity.innerToXContent(builder, ToXContent.EMPTY_PARAMS);
+        XContentParser parser = createParser(builder);
+        NestedIdentity parsedNestedIdentity = NestedIdentity.fromXContent(parser);
+        assertEquals(nestedIdentity, parsedNestedIdentity);
+        assertNull(parser.nextToken());
+    }
+
+    public void testToXContent() throws IOException {
+        NestedIdentity nestedIdentity = new NestedIdentity("foo", 5, null);
+        XContentBuilder builder = JsonXContent.contentBuilder();
+        builder.prettyPrint();
+        builder.startObject();
+        nestedIdentity.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        builder.endObject();
+        assertEquals(
+              "{\n" +
+              "  \"_nested\" : {\n" +
+              "    \"field\" : \"foo\",\n" +
+              "    \"offset\" : 5\n" +
+              "  }\n" +
+              "}", builder.string());
+
+        nestedIdentity = new NestedIdentity("foo", 5, new NestedIdentity("bar", 3, null));
+        builder = JsonXContent.contentBuilder();
+        builder.prettyPrint();
+        builder.startObject();
+        nestedIdentity.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        builder.endObject();
+        assertEquals(
+              "{\n" +
+              "  \"_nested\" : {\n" +
+              "    \"field\" : \"foo\",\n" +
+              "    \"offset\" : 5,\n" +
+              "    \"_nested\" : {\n" +
+              "      \"field\" : \"bar\",\n" +
+              "      \"offset\" : 3\n" +
+              "    }\n" +
+              "  }\n" +
+              "}", builder.string());
+    }
+
+    /**
+     * Test equality and hashCode properties
+     */
+    public void testEqualsAndHashcode() {
+        checkEqualsAndHashCode(createTestItem(randomInt(3)), NestedIdentityTests::copy, NestedIdentityTests::mutate);
+    }
+
+    public void testSerialization() throws IOException {
+        NestedIdentity nestedIdentity = createTestItem(randomInt(3));
+        try (BytesStreamOutput output = new BytesStreamOutput()) {
+            nestedIdentity.writeTo(output);
+            try (StreamInput in = output.bytes().streamInput()) {
+                NestedIdentity deserializedCopy = new NestedIdentity(in);
+                assertEquals(nestedIdentity, deserializedCopy);
+                assertEquals(nestedIdentity.hashCode(), deserializedCopy.hashCode());
+                assertNotSame(nestedIdentity, deserializedCopy);
+            }
+        }
+    }
+
+    private static NestedIdentity mutate(NestedIdentity original) {
+        if (original == null) {
+            return createTestItem(0);
+        }
+        List<Supplier<NestedIdentity>> mutations = new ArrayList<>();
+        int offset = original.getOffset();
+        NestedIdentity child = original.getChild();
+        String fieldName = original.getField().string();
+        mutations.add(() ->
+            new NestedIdentity(original.getField().string() + "_prefix", offset, child));
+        mutations.add(() -> new NestedIdentity(fieldName, offset + 1, child));
+        mutations.add(() -> new NestedIdentity(fieldName, offset, mutate(child)));
+        return randomFrom(mutations).get();
+    }
+
+    private static NestedIdentity copy(NestedIdentity original) {
+        NestedIdentity child = original.getChild();
+        return new NestedIdentity(original.getField().string(), original.getOffset(), child != null ? copy(child) : null);
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/search/SearchCancellationIT.java b/core/src/test/java/org/elasticsearch/search/SearchCancellationIT.java
new file mode 100644
index 0000000000000..18e102a396926
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/SearchCancellationIT.java
@@ -0,0 +1,316 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search;
+
+import org.elasticsearch.action.ListenableActionFuture;
+import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse;
+import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
+import org.elasticsearch.action.bulk.BulkRequestBuilder;
+import org.elasticsearch.action.search.SearchAction;
+import org.elasticsearch.action.search.SearchPhaseExecutionException;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.SearchScrollAction;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.PluginsService;
+import org.elasticsearch.plugins.ScriptPlugin;
+import org.elasticsearch.script.AbstractSearchScript;
+import org.elasticsearch.script.ExecutableScript;
+import org.elasticsearch.script.NativeScriptFactory;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptType;
+import org.elasticsearch.tasks.TaskInfo;
+import org.elasticsearch.test.ESIntegTestCase;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.elasticsearch.index.query.QueryBuilders.scriptQuery;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.hasSize;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE)
+public class SearchCancellationIT extends ESIntegTestCase {
+
+    private static final ToXContent.Params FORMAT_PARAMS = new ToXContent.MapParams(Collections.singletonMap("pretty", "false"));
+
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Collections.singleton(ScriptedBlockPlugin.class);
+    }
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        boolean lowLevelCancellation = randomBoolean();
+        logger.info("Using lowLevelCancellation: {}", lowLevelCancellation);
+        return Settings.builder().put(SearchService.LOW_LEVEL_CANCELLATION_SETTING.getKey(), lowLevelCancellation).build();
+    }
+
+    private void indexTestData() {
+        for (int i = 0; i < 5; i++) {
+            // Make sure we have a few segments
+            BulkRequestBuilder bulkRequestBuilder = client().prepareBulk().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
+            for (int j = 0; j < 20; j++) {
+                bulkRequestBuilder.add(client().prepareIndex("test", "type", Integer.toString(i * 5 + j)).setSource("field", "value"));
+            }
+            assertNoFailures(bulkRequestBuilder.get());
+        }
+    }
+
+    private List<ScriptedBlockPlugin> initBlockFactory() {
+        List<ScriptedBlockPlugin> plugins = new ArrayList<>();
+        for (PluginsService pluginsService : internalCluster().getDataNodeInstances(PluginsService.class)) {
+            plugins.addAll(pluginsService.filterPlugins(ScriptedBlockPlugin.class));
+        }
+        for (ScriptedBlockPlugin plugin : plugins) {
+            plugin.scriptedBlockFactory.reset();
+            plugin.scriptedBlockFactory.enableBlock();
+        }
+        return plugins;
+    }
+
+    private void awaitForBlock(List<ScriptedBlockPlugin> plugins) throws Exception {
+        int numberOfShards = getNumShards("test").numPrimaries;
+        assertBusy(() -> {
+            int numberOfBlockedPlugins = 0;
+            for (ScriptedBlockPlugin plugin : plugins) {
+                numberOfBlockedPlugins += plugin.scriptedBlockFactory.hits.get();
+            }
+            logger.info("The plugin blocked on {} out of {} shards", numberOfBlockedPlugins, numberOfShards);
+            assertThat(numberOfBlockedPlugins, greaterThan(0));
+        });
+    }
+
+    private void disableBlocks(List<ScriptedBlockPlugin> plugins) throws Exception {
+        for (ScriptedBlockPlugin plugin : plugins) {
+            plugin.scriptedBlockFactory.disableBlock();
+        }
+    }
+
+    private void cancelSearch(String action) {
+        ListTasksResponse listTasksResponse = client().admin().cluster().prepareListTasks().setActions(action).get();
+        assertThat(listTasksResponse.getTasks(), hasSize(1));
+        TaskInfo searchTask = listTasksResponse.getTasks().get(0);
+
+        logger.info("Cancelling search");
+        CancelTasksResponse cancelTasksResponse = client().admin().cluster().prepareCancelTasks().setTaskId(searchTask.getTaskId()).get();
+        assertThat(cancelTasksResponse.getTasks(), hasSize(1));
+        assertThat(cancelTasksResponse.getTasks().get(0).getTaskId(), equalTo(searchTask.getTaskId()));
+    }
+
+    private SearchResponse ensureSearchWasCancelled(ListenableActionFuture<SearchResponse> searchResponse) {
+        try {
+            SearchResponse response = searchResponse.actionGet();
+            logger.info("Search response {}", response);
+            assertNotEquals("At least one shard should have failed", 0, response.getFailedShards());
+            return response;
+        } catch (SearchPhaseExecutionException ex) {
+            logger.info("All shards failed with", ex);
+            return null;
+        }
+    }
+
+    public void testCancellationDuringQueryPhase() throws Exception {
+
+        List<ScriptedBlockPlugin> plugins = initBlockFactory();
+        indexTestData();
+
+        logger.info("Executing search");
+        ListenableActionFuture<SearchResponse> searchResponse = client().prepareSearch("test").setQuery(
+            scriptQuery(new Script(
+                ScriptType.INLINE, "native", NativeTestScriptedBlockFactory.TEST_NATIVE_BLOCK_SCRIPT, Collections.emptyMap())))
+            .execute();
+
+        awaitForBlock(plugins);
+        cancelSearch(SearchAction.NAME);
+        disableBlocks(plugins);
+        logger.info("Segments {}", XContentHelper.toString(client().admin().indices().prepareSegments("test").get(), FORMAT_PARAMS));
+        ensureSearchWasCancelled(searchResponse);
+    }
+
+    public void testCancellationDuringFetchPhase() throws Exception {
+
+        List<ScriptedBlockPlugin> plugins = initBlockFactory();
+        indexTestData();
+
+        logger.info("Executing search");
+        ListenableActionFuture<SearchResponse> searchResponse = client().prepareSearch("test")
+            .addScriptField("test_field",
+                new Script(ScriptType.INLINE, "native", NativeTestScriptedBlockFactory.TEST_NATIVE_BLOCK_SCRIPT, Collections.emptyMap())
+            ).execute();
+
+        awaitForBlock(plugins);
+        cancelSearch(SearchAction.NAME);
+        disableBlocks(plugins);
+        logger.info("Segments {}", XContentHelper.toString(client().admin().indices().prepareSegments("test").get(), FORMAT_PARAMS));
+        ensureSearchWasCancelled(searchResponse);
+    }
+
+    public void testCancellationOfScrollSearches() throws Exception {
+
+        List<ScriptedBlockPlugin> plugins = initBlockFactory();
+        indexTestData();
+
+        logger.info("Executing search");
+        ListenableActionFuture<SearchResponse> searchResponse = client().prepareSearch("test")
+            .setScroll(TimeValue.timeValueSeconds(10))
+            .setSize(5)
+            .setQuery(
+                scriptQuery(new Script(
+                    ScriptType.INLINE, "native", NativeTestScriptedBlockFactory.TEST_NATIVE_BLOCK_SCRIPT, Collections.emptyMap())))
+            .execute();
+
+        awaitForBlock(plugins);
+        cancelSearch(SearchAction.NAME);
+        disableBlocks(plugins);
+        SearchResponse response = ensureSearchWasCancelled(searchResponse);
+        if (response != null) {
+            // The response might not have failed on all shards - we need to clean scroll
+            logger.info("Cleaning scroll with id {}", response.getScrollId());
+            client().prepareClearScroll().addScrollId(response.getScrollId()).get();
+        }
+    }
+
+
+    public void testCancellationOfScrollSearchesOnFollowupRequests() throws Exception {
+
+        List<ScriptedBlockPlugin> plugins = initBlockFactory();
+        indexTestData();
+
+        // Disable block so the first request would pass
+        disableBlocks(plugins);
+
+        logger.info("Executing search");
+        TimeValue keepAlive = TimeValue.timeValueSeconds(5);
+        SearchResponse searchResponse = client().prepareSearch("test")
+            .setScroll(keepAlive)
+            .setSize(2)
+            .setQuery(
+                scriptQuery(new Script(
+                    ScriptType.INLINE, "native", NativeTestScriptedBlockFactory.TEST_NATIVE_BLOCK_SCRIPT, Collections.emptyMap())))
+            .get();
+
+        assertNotNull(searchResponse.getScrollId());
+
+        // Enable block so the second request would block
+        for (ScriptedBlockPlugin plugin : plugins) {
+            plugin.scriptedBlockFactory.reset();
+            plugin.scriptedBlockFactory.enableBlock();
+        }
+
+        String scrollId = searchResponse.getScrollId();
+        logger.info("Executing scroll with id {}", scrollId);
+        ListenableActionFuture<SearchResponse> scrollResponse = client().prepareSearchScroll(searchResponse.getScrollId())
+            .setScroll(keepAlive).execute();
+
+        awaitForBlock(plugins);
+        cancelSearch(SearchScrollAction.NAME);
+        disableBlocks(plugins);
+
+        SearchResponse response = ensureSearchWasCancelled(scrollResponse);
+        if (response != null) {
+            // The response didn't fail completely - update scroll id
+            scrollId = response.getScrollId();
+        }
+        logger.info("Cleaning scroll with id {}", scrollId);
+        client().prepareClearScroll().addScrollId(scrollId).get();
+    }
+
+
+    public static class ScriptedBlockPlugin extends Plugin implements ScriptPlugin {
+        private NativeTestScriptedBlockFactory scriptedBlockFactory;
+
+        public ScriptedBlockPlugin() {
+            scriptedBlockFactory = new NativeTestScriptedBlockFactory();
+        }
+
+        @Override
+        public List<NativeScriptFactory> getNativeScripts() {
+            return Collections.singletonList(scriptedBlockFactory);
+        }
+    }
+
+    private static class NativeTestScriptedBlockFactory implements NativeScriptFactory {
+
+        public static final String TEST_NATIVE_BLOCK_SCRIPT = "native_test_search_block_script";
+
+        private final AtomicInteger hits = new AtomicInteger();
+
+        private final AtomicBoolean shouldBlock = new AtomicBoolean(true);
+
+        NativeTestScriptedBlockFactory() {
+        }
+
+        public void reset() {
+            hits.set(0);
+        }
+
+        public void disableBlock() {
+            shouldBlock.set(false);
+        }
+
+        public void enableBlock() {
+            shouldBlock.set(true);
+        }
+
+        @Override
+        public ExecutableScript newScript(Map<String, Object> params) {
+            return new NativeTestScriptedBlock();
+        }
+
+        @Override
+        public boolean needsScores() {
+            return false;
+        }
+
+        @Override
+        public String getName() {
+            return TEST_NATIVE_BLOCK_SCRIPT;
+        }
+
+        public class NativeTestScriptedBlock extends AbstractSearchScript {
+            @Override
+            public Object run() {
+                Loggers.getLogger(SearchCancellationIT.class).info("Blocking on the document {}", fields().get("_uid"));
+                hits.incrementAndGet();
+                try {
+                    awaitBusy(() -> shouldBlock.get() == false);
+                } catch (Exception e) {
+                    throw new RuntimeException(e);
+                }
+                return true;
+            }
+        }
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/search/SearchCancellationTests.java b/core/src/test/java/org/elasticsearch/search/SearchCancellationTests.java
new file mode 100644
index 0000000000000..0b658d95a0f2e
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/SearchCancellationTests.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.NoMergePolicy;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.search.LeafCollector;
+import org.apache.lucene.search.TotalHitCountCollector;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.TestUtil;
+import org.elasticsearch.search.query.CancellableCollector;
+import org.elasticsearch.tasks.TaskCancelledException;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+public class SearchCancellationTests extends ESTestCase {
+
+    static Directory dir;
+    static IndexReader reader;
+
+    @BeforeClass
+    public static void setup() throws IOException {
+        dir = newDirectory();
+        RandomIndexWriter w = new RandomIndexWriter(random(), dir);
+        // we need at least 2 segments - so no merges should be allowed
+        w.w.getConfig().setMergePolicy(NoMergePolicy.INSTANCE);
+        w.setDoRandomForceMerge(false);
+        indexRandomDocuments(w, TestUtil.nextInt(random(), 2, 20));
+        w.flush();
+        indexRandomDocuments(w, TestUtil.nextInt(random(), 1, 20));
+        reader = w.getReader();
+        w.close();
+    }
+
+    private static void indexRandomDocuments(RandomIndexWriter w, int numDocs) throws IOException {
+        for (int i = 0; i < numDocs; ++i) {
+            Document doc = new Document();
+            doc.add(new StringField("foo", "bar", Field.Store.NO));
+            w.addDocument(doc);
+        }
+    }
+
+    @AfterClass
+    public static void cleanup() throws IOException {
+        IOUtils.close(reader, dir);
+        dir = null;
+        reader = null;
+    }
+
+    public void testCancellableCollector() throws IOException {
+        TotalHitCountCollector collector = new TotalHitCountCollector();
+        AtomicBoolean cancelled = new AtomicBoolean();
+        CancellableCollector cancellableCollector = new CancellableCollector(cancelled::get, collector);
+        final LeafCollector leafCollector = cancellableCollector.getLeafCollector(reader.leaves().get(0));
+        leafCollector.collect(0);
+        cancelled.set(true);
+        leafCollector.collect(1);
+        expectThrows(TaskCancelledException.class, () -> cancellableCollector.getLeafCollector(reader.leaves().get(1)));
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/search/SearchHitTests.java b/core/src/test/java/org/elasticsearch/search/SearchHitTests.java
new file mode 100644
index 0000000000000..d92058471e1ab
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/SearchHitTests.java
@@ -0,0 +1,291 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search;
+
+import org.apache.lucene.search.Explanation;
+import org.elasticsearch.action.OriginalIndices;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.InputStreamStreamInput;
+import org.elasticsearch.common.text.Text;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.search.SearchHit.NestedIdentity;
+import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
+import org.elasticsearch.search.fetch.subphase.highlight.HighlightFieldTests;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.RandomObjects;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Predicate;
+
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+import static org.elasticsearch.test.XContentTestUtils.insertRandomFields;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+
+public class SearchHitTests extends ESTestCase {
+
+    private static Set<String> META_FIELDS = Sets.newHashSet("_uid", "_all", "_parent", "_routing", "_size", "_timestamp", "_ttl");
+
+    public static SearchHit createTestItem(boolean withOptionalInnerHits) {
+        int internalId = randomInt();
+        String uid = randomAlphaOfLength(10);
+        Text type = new Text(randomAlphaOfLengthBetween(5, 10));
+        NestedIdentity nestedIdentity = null;
+        if (randomBoolean()) {
+            nestedIdentity = NestedIdentityTests.createTestItem(randomIntBetween(0, 2));
+        }
+        Map<String, SearchHitField> fields = new HashMap<>();
+        if (randomBoolean()) {
+            int size = randomIntBetween(0, 10);
+            for (int i = 0; i < size; i++) {
+                Tuple<List<Object>, List<Object>> values = RandomObjects.randomStoredFieldValues(random(),
+                        XContentType.JSON);
+                if (randomBoolean()) {
+                    String metaField = randomFrom(META_FIELDS);
+                    fields.put(metaField, new SearchHitField(metaField, values.v1()));
+                } else {
+                    String fieldName = randomAlphaOfLengthBetween(5, 10);
+                    fields.put(fieldName, new SearchHitField(fieldName, values.v1()));
+                }
+            }
+        }
+        SearchHit hit = new SearchHit(internalId, uid, type, nestedIdentity, fields);
+        if (frequently()) {
+            if (rarely()) {
+                hit.score(Float.NaN);
+            } else {
+                hit.score(randomFloat());
+            }
+        }
+        if (frequently()) {
+            hit.sourceRef(RandomObjects.randomSource(random()));
+        }
+        if (randomBoolean()) {
+            hit.version(randomLong());
+        }
+        if (randomBoolean()) {
+            hit.sortValues(SearchSortValuesTests.createTestItem());
+        }
+        if (randomBoolean()) {
+            int size = randomIntBetween(0, 5);
+            Map<String, HighlightField> highlightFields = new HashMap<>(size);
+            for (int i = 0; i < size; i++) {
+                highlightFields.put(randomAlphaOfLength(5), HighlightFieldTests.createTestItem());
+            }
+            hit.highlightFields(highlightFields);
+        }
+        if (randomBoolean()) {
+            int size = randomIntBetween(0, 5);
+            String[] matchedQueries = new String[size];
+            for (int i = 0; i < size; i++) {
+                matchedQueries[i] = randomAlphaOfLength(5);
+            }
+            hit.matchedQueries(matchedQueries);
+        }
+        if (randomBoolean()) {
+            hit.explanation(createExplanation(randomIntBetween(0, 5)));
+        }
+        if (withOptionalInnerHits) {
+            int innerHitsSize = randomIntBetween(0, 3);
+            Map<String, SearchHits> innerHits = new HashMap<>(innerHitsSize);
+            for (int i = 0; i < innerHitsSize; i++) {
+                innerHits.put(randomAlphaOfLength(5), SearchHitsTests.createTestItem());
+            }
+            hit.setInnerHits(innerHits);
+        }
+        if (randomBoolean()) {
+            hit.shard(new SearchShardTarget(randomAlphaOfLengthBetween(5, 10),
+                    new ShardId(new Index(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLengthBetween(5, 10)), randomInt()), null,
+                    OriginalIndices.NONE));
+        }
+        return hit;
+    }
+
+    public void testFromXContent() throws IOException {
+        SearchHit searchHit = createTestItem(true);
+        boolean humanReadable = randomBoolean();
+        XContentType xContentType = randomFrom(XContentType.values());
+        BytesReference originalBytes = toShuffledXContent(searchHit, xContentType, ToXContent.EMPTY_PARAMS, humanReadable);
+        SearchHit parsed;
+        try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) {
+            parser.nextToken(); // jump to first START_OBJECT
+            parsed = SearchHit.fromXContent(parser);
+            assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());
+            assertNull(parser.nextToken());
+        }
+        assertToXContentEquivalent(originalBytes, toXContent(parsed, xContentType, humanReadable), xContentType);
+    }
+
+    /**
+     * This test adds randomized fields on all json objects and checks that we can parse it to
+     * ensure the parsing is lenient for forward compatibility.
+     * We need to exclude json objects with the "highlight" and "fields" field name since these
+     * objects allow arbitrary keys (the field names that are queries). Also we want to exclude
+     * to add anything under "_source" since it is not parsed, and avoid complexity by excluding
+     * everything under "inner_hits". They are also keyed by arbitrary names and contain SearchHits,
+     * which are already tested elsewhere.
+     */
+    public void testFromXContentLenientParsing() throws IOException {
+        SearchHit searchHit = createTestItem(true);
+        XContentType xContentType = randomFrom(XContentType.values());
+        BytesReference originalBytes = toXContent(searchHit, xContentType, true);
+        Predicate<String> pathsToExclude = path -> (path.endsWith("highlight") || path.endsWith("fields") || path.contains("_source")
+                || path.contains("inner_hits"));
+        BytesReference withRandomFields = insertRandomFields(xContentType, originalBytes, pathsToExclude, random());
+
+        SearchHit parsed;
+        try (XContentParser parser = createParser(xContentType.xContent(), withRandomFields)) {
+            parser.nextToken(); // jump to first START_OBJECT
+            parsed = SearchHit.fromXContent(parser);
+            assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());
+            assertNull(parser.nextToken());
+        }
+        assertToXContentEquivalent(originalBytes, toXContent(parsed, xContentType, true), xContentType);
+    }
+
+    /**
+     * When e.g. with "stored_fields": "_none_", only "_index" and "_score" are returned.
+     */
+    public void testFromXContentWithoutTypeAndId() throws IOException {
+        String hit = "{\"_index\": \"my_index\", \"_score\": 1}";
+        SearchHit parsed;
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, hit)) {
+            parser.nextToken(); // jump to first START_OBJECT
+            parsed = SearchHit.fromXContent(parser);
+            assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());
+            assertNull(parser.nextToken());
+        }
+        assertEquals("my_index", parsed.getIndex());
+        assertEquals(1, parsed.getScore(), Float.MIN_VALUE);
+        assertNull(parsed.getType());
+        assertNull(parsed.getId());
+    }
+
+    public void testToXContent() throws IOException {
+        SearchHit searchHit = new SearchHit(1, "id1", new Text("type"), Collections.emptyMap());
+        searchHit.score(1.5f);
+        XContentBuilder builder = JsonXContent.contentBuilder();
+        searchHit.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        assertEquals("{\"_type\":\"type\",\"_id\":\"id1\",\"_score\":1.5}", builder.string());
+    }
+
+    public void testSerializeShardTarget() throws Exception {
+        String clusterAlias = randomBoolean() ? null : "cluster_alias";
+        SearchShardTarget target = new SearchShardTarget("_node_id", new Index("_index", "_na_"), 0, clusterAlias);
+
+        Map<String, SearchHits> innerHits = new HashMap<>();
+        SearchHit innerHit1 = new SearchHit(0, "_id", new Text("_type"), null);
+        innerHit1.shard(target);
+        SearchHit innerInnerHit2 = new SearchHit(0, "_id", new Text("_type"), null);
+        innerInnerHit2.shard(target);
+        innerHits.put("1", new SearchHits(new SearchHit[]{innerInnerHit2}, 1, 1f));
+        innerHit1.setInnerHits(innerHits);
+        SearchHit innerHit2 = new SearchHit(0, "_id", new Text("_type"), null);
+        innerHit2.shard(target);
+        SearchHit innerHit3 = new SearchHit(0, "_id", new Text("_type"), null);
+        innerHit3.shard(target);
+
+        innerHits = new HashMap<>();
+        SearchHit hit1 = new SearchHit(0, "_id", new Text("_type"), null);
+        innerHits.put("1", new SearchHits(new SearchHit[]{innerHit1, innerHit2}, 1, 1f));
+        innerHits.put("2", new SearchHits(new SearchHit[]{innerHit3}, 1, 1f));
+        hit1.shard(target);
+        hit1.setInnerHits(innerHits);
+
+        SearchHit hit2 = new SearchHit(0, "_id", new Text("_type"), null);
+        hit2.shard(target);
+
+        SearchHits hits = new SearchHits(new SearchHit[]{hit1, hit2}, 2, 1f);
+
+
+        BytesStreamOutput output = new BytesStreamOutput();
+        hits.writeTo(output);
+        InputStream input = output.bytes().streamInput();
+        SearchHits results = SearchHits.readSearchHits(new InputStreamStreamInput(input));
+        assertThat(results.getAt(0).getShard(), equalTo(target));
+        assertThat(results.getAt(0).getInnerHits().get("1").getAt(0).getShard(), notNullValue());
+        assertThat(results.getAt(0).getInnerHits().get("1").getAt(0).getInnerHits().get("1").getAt(0).getShard(), notNullValue());
+        assertThat(results.getAt(0).getInnerHits().get("1").getAt(1).getShard(), notNullValue());
+        assertThat(results.getAt(0).getInnerHits().get("2").getAt(0).getShard(), notNullValue());
+        for (SearchHit hit : results) {
+            assertEquals(clusterAlias, hit.getClusterAlias());
+            if (hit.getInnerHits() != null) {
+                for (SearchHits innerhits : hit.getInnerHits().values()) {
+                    for (SearchHit innerHit : innerhits) {
+                        assertEquals(clusterAlias, innerHit.getClusterAlias());
+                    }
+                }
+            }
+        }
+
+        assertThat(results.getAt(1).getShard(), equalTo(target));
+    }
+
+    public void testNullSource() throws Exception {
+        SearchHit searchHit = new SearchHit(0, "_id", new Text("_type"), null);
+
+        assertThat(searchHit.source(), nullValue());
+        assertThat(searchHit.sourceRef(), nullValue());
+        assertThat(searchHit.sourceAsMap(), nullValue());
+        assertThat(searchHit.sourceAsString(), nullValue());
+        assertThat(searchHit.getSource(), nullValue());
+        assertThat(searchHit.getSourceRef(), nullValue());
+        assertThat(searchHit.getSourceAsString(), nullValue());
+    }
+
+    public void testHasSource() {
+        SearchHit searchHit = new SearchHit(randomInt());
+        assertFalse(searchHit.hasSource());
+        searchHit.sourceRef(new BytesArray("{}"));
+        assertTrue(searchHit.hasSource());
+    }
+
+    private static Explanation createExplanation(int depth) {
+        String description = randomAlphaOfLengthBetween(5, 20);
+        float value = randomFloat();
+        List<Explanation> details = new ArrayList<>();
+        if (depth > 0) {
+            int numberOfDetails = randomIntBetween(1, 3);
+            for (int i = 0; i < numberOfDetails; i++) {
+                details.add(createExplanation(depth - 1));
+            }
+        }
+        return Explanation.match(value, description, details);
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/SearchHitsTests.java b/core/src/test/java/org/elasticsearch/search/SearchHitsTests.java
new file mode 100644
index 0000000000000..c25eb7da81454
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/SearchHitsTests.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search;
+
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.text.Text;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.function.Predicate;
+
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+import static org.elasticsearch.test.XContentTestUtils.insertRandomFields;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
+
+public class SearchHitsTests extends ESTestCase {
+
+    public static SearchHits createTestItem() {
+        int searchHits = randomIntBetween(0, 5);
+        SearchHit[] hits = new SearchHit[searchHits];
+        for (int i = 0; i < searchHits; i++) {
+            hits[i] = SearchHitTests.createTestItem(false); // creating random innerHits could create loops
+        }
+        long totalHits = randomLong();
+        float maxScore = frequently() ? randomFloat() : Float.NaN;
+        return new SearchHits(hits, totalHits, maxScore);
+    }
+
+    public void testFromXContent() throws IOException {
+        SearchHits searchHits = createTestItem();
+        XContentType xcontentType = randomFrom(XContentType.values());
+        boolean humanReadable = randomBoolean();
+        BytesReference originalBytes = toShuffledXContent(searchHits, xcontentType, ToXContent.EMPTY_PARAMS, humanReadable);
+        SearchHits parsed;
+        try (XContentParser parser = createParser(xcontentType.xContent(), originalBytes)) {
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
+            assertEquals(SearchHits.Fields.HITS, parser.currentName());
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            parsed = SearchHits.fromXContent(parser);
+            assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());
+            assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
+            assertNull(parser.nextToken());
+        }
+        assertToXContentEquivalent(originalBytes, toXContent(parsed, xcontentType, humanReadable), xcontentType);
+    }
+
+    /**
+     * This test adds randomized fields on all json objects and checks that we
+     * can parse it to ensure the parsing is lenient for forward compatibility.
+     * We need to exclude json objects with the "highlight" and "fields" field
+     * name since these objects allow arbitrary keys (the field names that are
+     * queries). Also we want to exclude to add anything under "_source" since
+     * it is not parsed.
+     */
+    public void testFromXContentLenientParsing() throws IOException {
+        SearchHits searchHits = createTestItem();
+        XContentType xcontentType = randomFrom(XContentType.values());
+        BytesReference originalBytes = toXContent(searchHits, xcontentType, ToXContent.EMPTY_PARAMS, true);
+        Predicate<String> pathsToExclude = path -> (path.isEmpty() || path.endsWith("highlight") || path.endsWith("fields")
+                || path.contains("_source"));
+        BytesReference withRandomFields = insertRandomFields(xcontentType, originalBytes, pathsToExclude, random());
+        SearchHits parsed = null;
+        try (XContentParser parser = createParser(xcontentType.xContent(), withRandomFields)) {
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
+            assertEquals(SearchHits.Fields.HITS, parser.currentName());
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            parsed = SearchHits.fromXContent(parser);
+            assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());
+            assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
+            assertNull(parser.nextToken());
+        }
+        assertToXContentEquivalent(originalBytes, toXContent(parsed, xcontentType, true), xcontentType);
+    }
+
+    public void testToXContent() throws IOException {
+        SearchHit[] hits = new SearchHit[] {
+                new SearchHit(1, "id1", new Text("type"), Collections.emptyMap()),
+                new SearchHit(2, "id2", new Text("type"), Collections.emptyMap()) };
+
+        long totalHits = 1000;
+        float maxScore = 1.5f;
+        SearchHits searchHits = new SearchHits(hits, totalHits, maxScore);
+        XContentBuilder builder = JsonXContent.contentBuilder();
+        builder.startObject();
+        searchHits.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        builder.endObject();
+        assertEquals("{\"hits\":{\"total\":1000,\"max_score\":1.5," +
+                "\"hits\":[{\"_type\":\"type\",\"_id\":\"id1\",\"_score\":\"-Infinity\"},"+
+                          "{\"_type\":\"type\",\"_id\":\"id2\",\"_score\":\"-Infinity\"}]}}", builder.string());
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/search/SearchModuleTests.java b/core/src/test/java/org/elasticsearch/search/SearchModuleTests.java
index c3324955fe69f..6065cbbdc45ad 100644
--- a/core/src/test/java/org/elasticsearch/search/SearchModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/search/SearchModuleTests.java
@@ -18,31 +18,28 @@
  */
 package org.elasticsearch.search;
 
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.inject.ModuleTestCase;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.index.query.QueryParser;
 import org.elasticsearch.index.query.TermQueryBuilder;
 import org.elasticsearch.index.query.functionscore.GaussDecayFunctionBuilder;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.plugins.SearchPlugin;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.BaseAggregationBuilder;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext;
 import org.elasticsearch.search.aggregations.bucket.significant.heuristics.ChiSquare;
 import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristic;
 import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristicParser;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.terms.TermsParser;
 import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.pipeline.derivative.DerivativePipelineAggregationBuilder;
@@ -50,7 +47,6 @@
 import org.elasticsearch.search.aggregations.pipeline.derivative.InternalDerivative;
 import org.elasticsearch.search.aggregations.pipeline.movavg.models.MovAvgModel;
 import org.elasticsearch.search.aggregations.pipeline.movavg.models.SimpleModel;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
@@ -62,24 +58,28 @@
 import org.elasticsearch.search.fetch.subphase.highlight.Highlighter;
 import org.elasticsearch.search.fetch.subphase.highlight.PlainHighlighter;
 import org.elasticsearch.search.fetch.subphase.highlight.PostingsHighlighter;
-import org.elasticsearch.search.suggest.CustomSuggester;
-import org.elasticsearch.search.suggest.Suggester;
-import org.elasticsearch.search.suggest.completion.CompletionSuggester;
-import org.elasticsearch.search.suggest.phrase.PhraseSuggester;
-import org.elasticsearch.search.suggest.term.TermSuggester;
+import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.search.suggest.CustomSuggesterSearchIT.CustomSuggestionBuilder;
+import org.elasticsearch.search.suggest.SuggestionBuilder;
+import org.elasticsearch.search.suggest.term.TermSuggestionBuilder;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.BiConsumer;
 
 import static java.util.Collections.emptyList;
 import static java.util.Collections.singletonList;
 import static java.util.Collections.singletonMap;
+import static java.util.stream.Collectors.toList;
+import static java.util.stream.Collectors.toSet;
 import static org.hamcrest.Matchers.containsInAnyOrder;
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.hasSize;
 
 public class SearchModuleTests extends ModuleTestCase {
 
@@ -94,13 +94,12 @@ public Map<String, Highlighter> getHighlighters() {
                 () -> new SearchModule(Settings.EMPTY, false, singletonList(registersDupeHighlighter)));
 
         SearchPlugin registersDupeSuggester = new SearchPlugin() {
-            @Override
-            public Map<String,org.elasticsearch.search.suggest.Suggester<?>> getSuggesters() {
-                return singletonMap("term", TermSuggester.INSTANCE);
+            public List<SearchPlugin.SuggesterSpec<?>> getSuggesters() {
+                return singletonList(new SuggesterSpec<>("term", TermSuggestionBuilder::new, TermSuggestionBuilder::fromXContent));
             }
         };
-        expectThrows(IllegalArgumentException.class,
-                () -> new SearchModule(Settings.EMPTY, false, singletonList(registersDupeSuggester)));
+        expectThrows(IllegalArgumentException.class, () -> new NamedXContentRegistry(
+                new SearchModule(Settings.EMPTY, false, singletonList(registersDupeSuggester)).getNamedXContents()));
 
         SearchPlugin registersDupeScoreFunction = new SearchPlugin() {
             @Override
@@ -109,8 +108,8 @@ public List<ScoreFunctionSpec<?>> getScoreFunctions() {
                         GaussDecayFunctionBuilder.PARSER));
             }
         };
-        expectThrows(IllegalArgumentException.class,
-                () -> new SearchModule(Settings.EMPTY, false, singletonList(registersDupeScoreFunction)));
+        expectThrows(IllegalArgumentException.class, () -> new NamedXContentRegistry(
+                new SearchModule(Settings.EMPTY, false, singletonList(registersDupeScoreFunction)).getNamedXContents()));
 
         SearchPlugin registersDupeSignificanceHeuristic = new SearchPlugin() {
             @Override
@@ -144,16 +143,17 @@ public List<SearchPlugin.QuerySpec<?>> getQueries() {
                 return singletonList(new QuerySpec<>(TermQueryBuilder.NAME, TermQueryBuilder::new, TermQueryBuilder::fromXContent));
             }
         };
-        expectThrows(IllegalArgumentException.class, () -> new SearchModule(Settings.EMPTY, false,
-                singletonList(registersDupeQuery)));
+        expectThrows(IllegalArgumentException.class, () -> new NamedXContentRegistry(
+                new SearchModule(Settings.EMPTY, false, singletonList(registersDupeQuery)).getNamedXContents()));
 
         SearchPlugin registersDupeAggregation = new SearchPlugin() {
             public List<AggregationSpec> getAggregations() {
-                return singletonList(new AggregationSpec(TermsAggregationBuilder.NAME, TermsAggregationBuilder::new, new TermsParser()));
+                return singletonList(new AggregationSpec(TermsAggregationBuilder.NAME, TermsAggregationBuilder::new,
+                        TermsAggregationBuilder::parse));
             }
         };
-        expectThrows(IllegalArgumentException.class, () -> new SearchModule(Settings.EMPTY, false,
-                singletonList(registersDupeAggregation)));
+        expectThrows(IllegalArgumentException.class, () -> new NamedXContentRegistry(new SearchModule(Settings.EMPTY, false,
+                singletonList(registersDupeAggregation)).getNamedXContents()));
 
         SearchPlugin registersDupePipelineAggregation = new SearchPlugin() {
             public List<PipelineAggregationSpec> getPipelineAggregations() {
@@ -165,21 +165,34 @@ public List<PipelineAggregationSpec> getPipelineAggregations() {
                             .addResultReader(InternalDerivative::new));
             }
         };
-        expectThrows(IllegalArgumentException.class, () -> new SearchModule(Settings.EMPTY, false,
-                singletonList(registersDupePipelineAggregation)));
+        expectThrows(IllegalArgumentException.class, () -> new NamedXContentRegistry(new SearchModule(Settings.EMPTY, false,
+                singletonList(registersDupePipelineAggregation)).getNamedXContents()));
     }
 
     public void testRegisterSuggester() {
         SearchModule module = new SearchModule(Settings.EMPTY, false, singletonList(new SearchPlugin() {
             @Override
-            public Map<String, Suggester<?>> getSuggesters() {
-                return singletonMap("custom", CustomSuggester.INSTANCE);
+            public List<SuggesterSpec<?>> getSuggesters() {
+                return singletonList(new SuggesterSpec<>("custom", CustomSuggestionBuilder::new, CustomSuggestionBuilder::fromXContent));
             }
         }));
-        assertSame(TermSuggester.INSTANCE, module.getSuggesters().getSuggester("term"));
-        assertSame(PhraseSuggester.INSTANCE, module.getSuggesters().getSuggester("phrase"));
-        assertSame(CompletionSuggester.INSTANCE, module.getSuggesters().getSuggester("completion"));
-        assertSame(CustomSuggester.INSTANCE, module.getSuggesters().getSuggester("custom"));
+        assertEquals(1, module.getNamedXContents().stream()
+                .filter(e -> e.categoryClass.equals(SuggestionBuilder.class) && e.name.match("term")).count());
+        assertEquals(1, module.getNamedXContents().stream()
+                .filter(e -> e.categoryClass.equals(SuggestionBuilder.class) && e.name.match("phrase")).count());
+        assertEquals(1, module.getNamedXContents().stream()
+                .filter(e -> e.categoryClass.equals(SuggestionBuilder.class) && e.name.match("completion")).count());
+        assertEquals(1, module.getNamedXContents().stream()
+                .filter(e -> e.categoryClass.equals(SuggestionBuilder.class) && e.name.match("custom")).count());
+
+        assertEquals(1, module.getNamedWriteables().stream()
+                .filter(e -> e.categoryClass.equals(SuggestionBuilder.class) && e.name.equals("term")).count());
+        assertEquals(1, module.getNamedWriteables().stream()
+                .filter(e -> e.categoryClass.equals(SuggestionBuilder.class) && e.name.equals("phrase")).count());
+        assertEquals(1, module.getNamedWriteables().stream()
+                .filter(e -> e.categoryClass.equals(SuggestionBuilder.class) && e.name.equals("completion")).count());
+        assertEquals(1, module.getNamedWriteables().stream()
+                .filter(e -> e.categoryClass.equals(SuggestionBuilder.class) && e.name.equals("custom")).count());
     }
 
     public void testRegisterHighlighter() {
@@ -199,31 +212,22 @@ public Map<String, Highlighter> getHighlighters() {
     }
 
     public void testRegisteredQueries() throws IOException {
-        SearchModule module = new SearchModule(Settings.EMPTY, false, emptyList());
         List<String> allSupportedQueries = new ArrayList<>();
         Collections.addAll(allSupportedQueries, NON_DEPRECATED_QUERIES);
         Collections.addAll(allSupportedQueries, DEPRECATED_QUERIES);
-        String[] supportedQueries = allSupportedQueries.toArray(new String[allSupportedQueries.size()]);
-        assertThat(module.getQueryParserRegistry().getNames(), containsInAnyOrder(supportedQueries));
-
-        IndicesQueriesRegistry indicesQueriesRegistry = module.getQueryParserRegistry();
-        XContentParser dummyParser = XContentHelper.createParser(new BytesArray("{}"));
-        for (String queryName : supportedQueries) {
-            indicesQueriesRegistry.lookup(queryName, ParseFieldMatcher.EMPTY, dummyParser.getTokenLocation());
-        }
+        SearchModule module = new SearchModule(Settings.EMPTY, false, emptyList());
 
-        for (String queryName : NON_DEPRECATED_QUERIES) {
-            QueryParser<?> queryParser = indicesQueriesRegistry.lookup(queryName, ParseFieldMatcher.STRICT, dummyParser.getTokenLocation());
-            assertThat(queryParser, notNullValue());
-        }
-        for (String queryName : DEPRECATED_QUERIES) {
-            try {
-                indicesQueriesRegistry.lookup(queryName, ParseFieldMatcher.STRICT, dummyParser.getTokenLocation());
-                fail("query is deprecated, getQueryParser should have failed in strict mode");
-            } catch(IllegalArgumentException e) {
-                assertThat(e.getMessage(), containsString("Deprecated field [" + queryName + "] used"));
-            }
-        }
+        Set<String> registeredNonDeprecated = module.getNamedXContents().stream()
+                .filter(e -> e.categoryClass.equals(Optional.class))
+                .map(e -> e.name.getPreferredName())
+                .collect(toSet());
+        Set<String> registeredAll = module.getNamedXContents().stream()
+                .filter(e -> e.categoryClass.equals(Optional.class))
+                .flatMap(e -> Arrays.stream(e.name.getAllNamesIncludedDeprecated()))
+                .collect(toSet());
+
+        assertThat(registeredNonDeprecated, containsInAnyOrder(NON_DEPRECATED_QUERIES));
+        assertThat(registeredAll, containsInAnyOrder(allSupportedQueries.toArray(new String[0])));
     }
 
     public void testRegisterAggregation() {
@@ -233,7 +237,11 @@ public List<AggregationSpec> getAggregations() {
             }
         }));
 
-        assertNotNull(module.getAggregatorParsers().parser("test", ParseFieldMatcher.STRICT));
+        assertThat(
+                module.getNamedXContents().stream()
+                    .filter(entry -> entry.categoryClass.equals(BaseAggregationBuilder.class) && entry.name.match("test"))
+                    .collect(toList()),
+                hasSize(1));
     }
 
     public void testRegisterPipelineAggregation() {
@@ -244,7 +252,11 @@ public List<PipelineAggregationSpec> getPipelineAggregations() {
             }
         }));
 
-        assertNotNull(module.getAggregatorParsers().pipelineParser("test", ParseFieldMatcher.STRICT));
+        assertThat(
+                module.getNamedXContents().stream()
+                    .filter(entry -> entry.categoryClass.equals(BaseAggregationBuilder.class) && entry.name.match("test"))
+                    .collect(toList()),
+                hasSize(1));
     }
 
     private static final String[] NON_DEPRECATED_QUERIES = new String[] {
@@ -263,8 +275,6 @@ public List<PipelineAggregationSpec> getPipelineAggregations() {
             "geo_polygon",
             "geo_shape",
             "geohash_cell",
-            "has_child",
-            "has_parent",
             "ids",
             "indices",
             "match",
@@ -275,7 +285,6 @@ public List<PipelineAggregationSpec> getPipelineAggregations() {
             "more_like_this",
             "multi_match",
             "nested",
-            "parent_id",
             "prefix",
             "query_string",
             "range",
@@ -317,7 +326,7 @@ protected TestAggregationBuilder(StreamInput in) throws IOException {
         }
 
         @Override
-        public String getWriteableName() {
+        public String getType() {
             return "test";
         }
 
@@ -326,7 +335,7 @@ protected void innerWriteTo(StreamOutput out) throws IOException {
         }
 
         @Override
-        protected ValuesSourceAggregatorFactory<ValuesSource, ?> innerBuild(AggregationContext context,
+        protected ValuesSourceAggregatorFactory<ValuesSource, ?> innerBuild(SearchContext context,
                 ValuesSourceConfig<ValuesSource> config, AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
             return null;
         }
@@ -358,7 +367,7 @@ private static class TestPipelineAggregationBuilder extends AbstractPipelineAggr
         /**
          * Read from a stream.
          */
-        public TestPipelineAggregationBuilder(StreamInput in) throws IOException {
+        TestPipelineAggregationBuilder(StreamInput in) throws IOException {
             super(in, "test");
         }
 
@@ -403,7 +412,7 @@ private static class TestPipelineAggregator extends PipelineAggregator {
         /**
          * Read from a stream.
          */
-        public TestPipelineAggregator(StreamInput in) throws IOException {
+        TestPipelineAggregator(StreamInput in) throws IOException {
             super(in);
         }
         @Override
diff --git a/core/src/test/java/org/elasticsearch/search/SearchRequestTests.java b/core/src/test/java/org/elasticsearch/search/SearchRequestTests.java
index 3db3492f4150e..fb83bcfaa4543 100644
--- a/core/src/test/java/org/elasticsearch/search/SearchRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/search/SearchRequestTests.java
@@ -24,51 +24,17 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.indices.IndicesModule;
-import org.elasticsearch.test.ESTestCase;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
+import org.elasticsearch.common.util.ArrayUtils;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.List;
 
-import static java.util.Collections.emptyList;
-import static org.elasticsearch.search.builder.SearchSourceBuilderTests.createSearchSourceBuilder;
+import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
 
-public class SearchRequestTests extends ESTestCase {
-
-    private static NamedWriteableRegistry namedWriteableRegistry;
-
-    @BeforeClass
-    public static void beforeClass() {
-        IndicesModule indicesModule = new IndicesModule(emptyList()) {
-            @Override
-            protected void configure() {
-                bindMapperExtension();
-            }
-        };
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList()) {
-            @Override
-            protected void configureSearch() {
-                // Skip me
-            }
-        };
-        List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
-        entries.addAll(indicesModule.getNamedWriteables());
-        entries.addAll(searchModule.getNamedWriteables());
-        namedWriteableRegistry = new NamedWriteableRegistry(entries);
-    }
-
-    @AfterClass
-    public static void afterClass() {
-        namedWriteableRegistry = null;
-    }
+public class SearchRequestTests extends AbstractSearchTestCase {
 
     public void testSerialization() throws Exception {
         SearchRequest searchRequest = createSearchRequest();
@@ -115,127 +81,26 @@ public void testIllegalArguments() {
     }
 
     public void testEqualsAndHashcode() throws IOException {
-        SearchRequest firstSearchRequest = createSearchRequest();
-        assertNotNull("search request is equal to null", firstSearchRequest);
-        assertNotEquals("search request  is equal to incompatible type", firstSearchRequest, "");
-        assertEquals("search request is not equal to self", firstSearchRequest, firstSearchRequest);
-        assertEquals("same source builder's hashcode returns different values if called multiple times",
-                firstSearchRequest.hashCode(), firstSearchRequest.hashCode());
-
-        SearchRequest secondSearchRequest = copyRequest(firstSearchRequest);
-        assertEquals("search request  is not equal to self", secondSearchRequest, secondSearchRequest);
-        assertEquals("search request is not equal to its copy", firstSearchRequest, secondSearchRequest);
-        assertEquals("search request is not symmetric", secondSearchRequest, firstSearchRequest);
-        assertEquals("search request copy's hashcode is different from original hashcode",
-                firstSearchRequest.hashCode(), secondSearchRequest.hashCode());
-
-        SearchRequest thirdSearchRequest = copyRequest(secondSearchRequest);
-        assertEquals("search request is not equal to self", thirdSearchRequest, thirdSearchRequest);
-        assertEquals("search request is not equal to its copy", secondSearchRequest, thirdSearchRequest);
-        assertEquals("search request copy's hashcode is different from original hashcode",
-                secondSearchRequest.hashCode(), thirdSearchRequest.hashCode());
-        assertEquals("equals is not transitive", firstSearchRequest, thirdSearchRequest);
-        assertEquals("search request copy's hashcode is different from original hashcode",
-                firstSearchRequest.hashCode(), thirdSearchRequest.hashCode());
-        assertEquals("equals is not symmetric", thirdSearchRequest, secondSearchRequest);
-        assertEquals("equals is not symmetric", thirdSearchRequest, firstSearchRequest);
-
-        boolean changed = false;
-        if (randomBoolean()) {
-            secondSearchRequest.indices(generateRandomStringArray(10, 10, false, false));
-            if (Arrays.equals(secondSearchRequest.indices(), firstSearchRequest.indices()) == false) {
-                changed = true;
-            }
-        }
-        if (randomBoolean()) {
-            secondSearchRequest.indicesOptions(
-                    IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()));
-            if (secondSearchRequest.indicesOptions().equals(firstSearchRequest.indicesOptions()) == false) {
-                changed = true;
-            }
-        }
-        if (randomBoolean()) {
-            secondSearchRequest.types(generateRandomStringArray(10, 10, false, false));
-            if (Arrays.equals(secondSearchRequest.types(), firstSearchRequest.types()) == false) {
-                changed = true;
-            }
-        }
-        if (randomBoolean()) {
-            secondSearchRequest.preference(randomAsciiOfLengthBetween(3, 10));
-            if (secondSearchRequest.preference().equals(firstSearchRequest.preference()) == false) {
-                changed = true;
-            }
-        }
-        if (randomBoolean()) {
-            secondSearchRequest.routing(randomAsciiOfLengthBetween(3, 10));
-            if (secondSearchRequest.routing().equals(firstSearchRequest.routing()) == false) {
-                changed = true;
-            }
-        }
-        if (randomBoolean()) {
-            secondSearchRequest.requestCache(randomBoolean());
-            if (secondSearchRequest.requestCache().equals(firstSearchRequest.requestCache()) == false) {
-                changed = true;
-            }
-        }
-        if (randomBoolean()) {
-            secondSearchRequest.scroll(randomPositiveTimeValue());
-            if (secondSearchRequest.scroll().equals(firstSearchRequest.scroll()) == false) {
-                changed = true;
-            }
-        }
-        if (randomBoolean()) {
-            secondSearchRequest.searchType(randomFrom(SearchType.values()));
-            if (secondSearchRequest.searchType() != firstSearchRequest.searchType()) {
-                changed = true;
-            }
-        }
-        if (randomBoolean()) {
-            secondSearchRequest.source(createSearchSourceBuilder());
-            if (secondSearchRequest.source().equals(firstSearchRequest.source()) == false) {
-                changed = true;
-            }
-        }
-
-        if (changed) {
-            assertNotEquals(firstSearchRequest, secondSearchRequest);
-            assertNotEquals(firstSearchRequest.hashCode(), secondSearchRequest.hashCode());
-        } else {
-            assertEquals(firstSearchRequest, secondSearchRequest);
-            assertEquals(firstSearchRequest.hashCode(), secondSearchRequest.hashCode());
-        }
+        checkEqualsAndHashCode(createSearchRequest(), SearchRequestTests::copyRequest, this::mutate);
     }
 
-    public static SearchRequest createSearchRequest() throws IOException {
-        SearchRequest searchRequest = new SearchRequest();
-        if (randomBoolean()) {
-            searchRequest.indices(generateRandomStringArray(10, 10, false, false));
-        }
-        if (randomBoolean()) {
-            searchRequest.indicesOptions(IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()));
-        }
-        if (randomBoolean()) {
-            searchRequest.types(generateRandomStringArray(10, 10, false, false));
-        }
-        if (randomBoolean()) {
-            searchRequest.preference(randomAsciiOfLengthBetween(3, 10));
-        }
-        if (randomBoolean()) {
-            searchRequest.requestCache(randomBoolean());
-        }
-        if (randomBoolean()) {
-            searchRequest.routing(randomAsciiOfLengthBetween(3, 10));
-        }
-        if (randomBoolean()) {
-            searchRequest.scroll(randomPositiveTimeValue());
-        }
-        if (randomBoolean()) {
-            searchRequest.searchType(randomFrom(SearchType.values()));
-        }
-        if (randomBoolean()) {
-            searchRequest.source(createSearchSourceBuilder());
-        }
-        return searchRequest;
+    private SearchRequest mutate(SearchRequest searchRequest) throws IOException {
+        SearchRequest mutation = copyRequest(searchRequest);
+        List<Runnable> mutators = new ArrayList<>();
+        mutators.add(() -> mutation.indices(ArrayUtils.concat(searchRequest.indices(), new String[] { randomAlphaOfLength(10) })));
+        mutators.add(() -> mutation.indicesOptions(randomValueOtherThan(searchRequest.indicesOptions(),
+                () -> IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()))));
+        mutators.add(() -> mutation.types(ArrayUtils.concat(searchRequest.types(), new String[] { randomAlphaOfLength(10) })));
+        mutators.add(() -> mutation.preference(randomValueOtherThan(searchRequest.preference(), () -> randomAlphaOfLengthBetween(3, 10))));
+        mutators.add(() -> mutation.routing(randomValueOtherThan(searchRequest.routing(), () -> randomAlphaOfLengthBetween(3, 10))));
+        mutators.add(() -> mutation.requestCache((randomValueOtherThan(searchRequest.requestCache(), () -> randomBoolean()))));
+        mutators.add(() -> mutation
+                .scroll(randomValueOtherThan(searchRequest.scroll(), () -> new Scroll(new TimeValue(randomNonNegativeLong() % 100000)))));
+        mutators.add(() -> mutation.searchType(randomValueOtherThan(searchRequest.searchType(),
+            () -> randomFrom(SearchType.DFS_QUERY_THEN_FETCH, SearchType.QUERY_THEN_FETCH))));
+        mutators.add(() -> mutation.source(randomValueOtherThan(searchRequest.source(), this::createSearchSourceBuilder)));
+        randomFrom(mutators).run();
+        return mutation;
     }
 
     private static SearchRequest copyRequest(SearchRequest searchRequest) throws IOException {
diff --git a/core/src/test/java/org/elasticsearch/search/SearchServiceTests.java b/core/src/test/java/org/elasticsearch/search/SearchServiceTests.java
index 7de8f6a4988a4..a55e0bef0740b 100644
--- a/core/src/test/java/org/elasticsearch/search/SearchServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/search/SearchServiceTests.java
@@ -19,30 +19,44 @@
 package org.elasticsearch.search;
 
 import com.carrotsearch.hppc.IntArrayList;
+
 import org.apache.lucene.search.Query;
 import org.apache.lucene.store.AlreadyClosedException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
+import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.SearchTask;
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.query.AbstractQueryBuilder;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
+import org.elasticsearch.index.query.MatchNoneQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryRewriteContext;
 import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.query.TermQueryBuilder;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.SearchPlugin;
+import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
+import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.fetch.ShardFetchRequest;
+import org.elasticsearch.search.internal.AliasFilter;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.internal.ShardSearchLocalRequest;
-import org.elasticsearch.search.query.QuerySearchResultProvider;
+import org.elasticsearch.search.suggest.SuggestBuilder;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
 import java.io.IOException;
@@ -55,7 +69,9 @@
 
 import static java.util.Collections.singletonList;
 import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
+import static org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.DELETED;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
 
@@ -71,6 +87,11 @@ protected Collection<Class<? extends Plugin>> getPlugins() {
         return pluginList(FailOnRewriteQueryPlugin.class);
     }
 
+    @Override
+    protected Settings nodeSettings() {
+        return Settings.builder().put("search.default_search_timeout", "5s").build();
+    }
+
     public void testClearOnClose() throws ExecutionException, InterruptedException {
         createIndex("index");
         client().prepareIndex("index", "type", "1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
@@ -140,7 +161,7 @@ public void testSearchWhileIndexDeleted() throws IOException, InterruptedExcepti
             public void run() {
                 startGun.countDown();
                 while(running.get()) {
-                    service.afterIndexDeleted(indexService.index(), indexService.getIndexSettings().getSettings());
+                    service.afterIndexRemoved(indexService.index(), indexService.getIndexSettings(), DELETED);
                     if (randomBoolean()) {
                         // here we trigger some refreshes to ensure the IR go out of scope such that we hit ACE if we access a search
                         // context in a non-sane way.
@@ -171,13 +192,14 @@ public void onFailure(Exception e) {
             final int rounds = scaledRandomIntBetween(100, 10000);
             for (int i = 0; i < rounds; i++) {
                 try {
-                    QuerySearchResultProvider querySearchResultProvider = service.executeQueryPhase(
-                        new ShardSearchLocalRequest(indexShard.shardId(), 1, SearchType.DEFAULT,
-                            new SearchSourceBuilder(), new String[0], false));
+                    SearchPhaseResult searchPhaseResult = service.executeQueryPhase(
+                            new ShardSearchLocalRequest(indexShard.shardId(), 1, SearchType.DEFAULT,
+                                    new SearchSourceBuilder(), new String[0], false, new AliasFilter(null, Strings.EMPTY_ARRAY), 1.0f),
+                        new SearchTask(123L, "", "", "", null));
                     IntArrayList intCursors = new IntArrayList(1);
                     intCursors.add(0);
-                    ShardFetchRequest req = new ShardFetchRequest(querySearchResultProvider.id(), intCursors, null /* not a scroll */);
-                    service.executeFetchPhase(req);
+                    ShardFetchRequest req = new ShardFetchRequest(searchPhaseResult.getRequestId(), intCursors, null /* not a scroll */);
+                    service.executeFetchPhase(req, new SearchTask(123L, "", "", "", null));
                 } catch (AlreadyClosedException ex) {
                     throw ex;
                 } catch (IllegalStateException ex) {
@@ -193,6 +215,53 @@ public void onFailure(Exception e) {
         }
     }
 
+    public void testTimeout() throws IOException {
+        createIndex("index");
+        final SearchService service = getInstanceFromNode(SearchService.class);
+        final IndicesService indicesService = getInstanceFromNode(IndicesService.class);
+        final IndexService indexService = indicesService.indexServiceSafe(resolveIndex("index"));
+        final IndexShard indexShard = indexService.getShard(0);
+        final SearchContext contextWithDefaultTimeout = service.createContext(
+                new ShardSearchLocalRequest(
+                        indexShard.shardId(),
+                        1,
+                        SearchType.DEFAULT,
+                        new SearchSourceBuilder(),
+                        new String[0],
+                        false,
+                        new AliasFilter(null, Strings.EMPTY_ARRAY),
+                        1.0f),
+                null);
+        try {
+            // the search context should inherit the default timeout
+            assertThat(contextWithDefaultTimeout.timeout(), equalTo(TimeValue.timeValueSeconds(5)));
+        } finally {
+            contextWithDefaultTimeout.decRef();
+            service.freeContext(contextWithDefaultTimeout.id());
+        }
+
+        final long seconds = randomIntBetween(6, 10);
+        final SearchContext context = service.createContext(
+                new ShardSearchLocalRequest(
+                        indexShard.shardId(),
+                        1,
+                        SearchType.DEFAULT,
+                        new SearchSourceBuilder().timeout(TimeValue.timeValueSeconds(seconds)),
+                        new String[0],
+                        false,
+                        new AliasFilter(null, Strings.EMPTY_ARRAY),
+                        1.0f),
+            null);
+        try {
+            // the search context should inherit the query timeout
+            assertThat(context.timeout(), equalTo(TimeValue.timeValueSeconds(seconds)));
+        } finally {
+            context.decRef();
+            service.freeContext(context.id());
+        }
+
+    }
+
     public static class FailOnRewriteQueryPlugin extends Plugin implements SearchPlugin {
         @Override
         public List<QuerySpec<?>> getQueries() {
@@ -244,4 +313,53 @@ public String getWriteableName() {
             return null;
         }
     }
+
+    public void testCanMatch() throws IOException {
+        createIndex("index");
+        final SearchService service = getInstanceFromNode(SearchService.class);
+        final IndicesService indicesService = getInstanceFromNode(IndicesService.class);
+        final IndexService indexService = indicesService.indexServiceSafe(resolveIndex("index"));
+        final IndexShard indexShard = indexService.getShard(0);
+        assertTrue(service.canMatch(new ShardSearchLocalRequest(indexShard.shardId(), 1, SearchType.QUERY_THEN_FETCH, null,
+            Strings.EMPTY_ARRAY, false, new AliasFilter(null, Strings.EMPTY_ARRAY), 1f)));
+
+        assertTrue(service.canMatch(new ShardSearchLocalRequest(indexShard.shardId(), 1, SearchType.QUERY_THEN_FETCH,
+            new SearchSourceBuilder(), Strings.EMPTY_ARRAY, false, new AliasFilter(null, Strings.EMPTY_ARRAY), 1f)));
+
+        assertTrue(service.canMatch(new ShardSearchLocalRequest(indexShard.shardId(), 1, SearchType.QUERY_THEN_FETCH,
+            new SearchSourceBuilder().query(new MatchAllQueryBuilder()), Strings.EMPTY_ARRAY, false,
+            new AliasFilter(null, Strings.EMPTY_ARRAY), 1f)));
+
+        assertTrue(service.canMatch(new ShardSearchLocalRequest(indexShard.shardId(), 1, SearchType.QUERY_THEN_FETCH,
+            new SearchSourceBuilder().query(new MatchNoneQueryBuilder())
+            .aggregation(new TermsAggregationBuilder("test", ValueType.STRING).minDocCount(0)), Strings.EMPTY_ARRAY, false,
+            new AliasFilter(null, Strings.EMPTY_ARRAY), 1f)));
+        assertTrue(service.canMatch(new ShardSearchLocalRequest(indexShard.shardId(), 1, SearchType.QUERY_THEN_FETCH,
+            new SearchSourceBuilder().query(new MatchNoneQueryBuilder())
+                .aggregation(new GlobalAggregationBuilder("test")), Strings.EMPTY_ARRAY, false,
+            new AliasFilter(null, Strings.EMPTY_ARRAY), 1f)));
+
+        assertFalse(service.canMatch(new ShardSearchLocalRequest(indexShard.shardId(), 1, SearchType.QUERY_THEN_FETCH,
+            new SearchSourceBuilder().query(new MatchNoneQueryBuilder()), Strings.EMPTY_ARRAY, false,
+            new AliasFilter(null, Strings.EMPTY_ARRAY), 1f)));
+
+    }
+
+    public void testCanRewriteToMatchNone() {
+        assertFalse(SearchService.canRewriteToMatchNone(new SearchSourceBuilder().query(new MatchNoneQueryBuilder())
+            .aggregation(new GlobalAggregationBuilder("test"))));
+        assertFalse(SearchService.canRewriteToMatchNone(new SearchSourceBuilder()));
+        assertFalse(SearchService.canRewriteToMatchNone(null));
+        assertFalse(SearchService.canRewriteToMatchNone(new SearchSourceBuilder().query(new MatchNoneQueryBuilder())
+            .aggregation(new TermsAggregationBuilder("test", ValueType.STRING).minDocCount(0))));
+        assertTrue(SearchService.canRewriteToMatchNone(new SearchSourceBuilder().query(new TermQueryBuilder("foo", "bar"))));
+        assertTrue(SearchService.canRewriteToMatchNone(new SearchSourceBuilder().query(new MatchNoneQueryBuilder())
+            .aggregation(new TermsAggregationBuilder("test", ValueType.STRING).minDocCount(1))));
+        assertFalse(SearchService.canRewriteToMatchNone(new SearchSourceBuilder().query(new MatchNoneQueryBuilder())
+            .aggregation(new TermsAggregationBuilder("test", ValueType.STRING).minDocCount(1))
+            .suggest(new SuggestBuilder())));
+        assertFalse(SearchService.canRewriteToMatchNone(new SearchSourceBuilder().query(new TermQueryBuilder("foo", "bar"))
+            .suggest(new SuggestBuilder())));
+
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/SearchSortValuesTests.java b/core/src/test/java/org/elasticsearch/search/SearchSortValuesTests.java
new file mode 100644
index 0000000000000..316ef3d455984
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/SearchSortValuesTests.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search;
+
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.function.Supplier;
+
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
+
+public class SearchSortValuesTests extends ESTestCase {
+
+    public static SearchSortValues createTestItem() {
+        List<Supplier<Object>> valueSuppliers = new ArrayList<>();
+        // this should reflect all values that are allowed to go through the transport layer
+        valueSuppliers.add(() -> null);
+        valueSuppliers.add(() -> randomInt());
+        valueSuppliers.add(() -> randomLong());
+        valueSuppliers.add(() -> randomDouble());
+        valueSuppliers.add(() -> randomFloat());
+        valueSuppliers.add(() -> randomByte());
+        valueSuppliers.add(() -> randomShort());
+        valueSuppliers.add(() -> randomBoolean());
+        valueSuppliers.add(() -> frequently() ? randomAlphaOfLengthBetween(1, 30) : randomRealisticUnicodeOfCodepointLength(30));
+
+        int size = randomIntBetween(1, 20);
+        Object[] values = new Object[size];
+        for (int i = 0; i < size; i++) {
+            Supplier<Object> supplier = randomFrom(valueSuppliers);
+            values[i] = supplier.get();
+        }
+        return new SearchSortValues(values);
+    }
+
+    public void testFromXContent() throws IOException {
+        SearchSortValues sortValues = createTestItem();
+        XContentType xcontentType = randomFrom(XContentType.values());
+        boolean humanReadable = randomBoolean();
+        BytesReference originalBytes = toShuffledXContent(sortValues, xcontentType, ToXContent.EMPTY_PARAMS, humanReadable);
+
+        SearchSortValues parsed;
+        try (XContentParser parser = createParser(xcontentType.xContent(), originalBytes)) {
+            parser.nextToken(); // skip to the elements start array token, fromXContent advances from there if called
+            parser.nextToken();
+            parser.nextToken();
+            parsed = SearchSortValues.fromXContent(parser);
+            parser.nextToken();
+            assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());
+            assertNull(parser.nextToken());
+        }
+        assertToXContentEquivalent(originalBytes, toXContent(parsed, xcontentType, humanReadable), xcontentType);
+    }
+
+    public void testToXContent() throws IOException {
+        SearchSortValues sortValues = new SearchSortValues(new Object[]{ 1, "foo", 3.0});
+        XContentBuilder builder = JsonXContent.contentBuilder();
+        builder.startObject();
+        sortValues.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        builder.endObject();
+        assertEquals("{\"sort\":[1,\"foo\",3.0]}", builder.string());
+    }
+
+    /**
+     * Test equality and hashCode properties
+     */
+    public void testEqualsAndHashcode() {
+        checkEqualsAndHashCode(createTestItem(), SearchSortValuesTests::copy, SearchSortValuesTests::mutate);
+    }
+
+    public void testSerialization() throws IOException {
+        SearchSortValues sortValues = createTestItem();
+        try (BytesStreamOutput output = new BytesStreamOutput()) {
+            sortValues.writeTo(output);
+            try (StreamInput in = output.bytes().streamInput()) {
+                SearchSortValues deserializedCopy = new SearchSortValues(in);
+                assertEquals(sortValues, deserializedCopy);
+                assertEquals(sortValues.hashCode(), deserializedCopy.hashCode());
+                assertNotSame(sortValues, deserializedCopy);
+            }
+        }
+    }
+
+    private static SearchSortValues mutate(SearchSortValues original) {
+        Object[] sortValues = original.sortValues();
+        if (sortValues.length == 0) {
+            return new SearchSortValues(new Object[] { 1 });
+        }
+        return new SearchSortValues(Arrays.copyOf(sortValues, sortValues.length + 1));
+    }
+
+    private static SearchSortValues copy(SearchSortValues original) {
+        return new SearchSortValues(Arrays.copyOf(original.sortValues(), original.sortValues().length));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/SearchTimeoutIT.java b/core/src/test/java/org/elasticsearch/search/SearchTimeoutIT.java
index 168574059f67b..08fde0da3c9ab 100644
--- a/core/src/test/java/org/elasticsearch/search/SearchTimeoutIT.java
+++ b/core/src/test/java/org/elasticsearch/search/SearchTimeoutIT.java
@@ -28,8 +28,7 @@
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.NativeScriptFactory;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptModule;
-import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.test.ESIntegTestCase;
 
 import java.util.Collection;
@@ -61,7 +60,8 @@ public void testSimpleTimeout() throws Exception {
         client().prepareIndex("test", "type", "1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
 
         SearchResponse searchResponse = client().prepareSearch("test").setTimeout(new TimeValue(10, TimeUnit.MILLISECONDS))
-                .setQuery(scriptQuery(new Script(NativeTestScriptedTimeout.TEST_NATIVE_SCRIPT_TIMEOUT, ScriptType.INLINE, "native", null)))
+                .setQuery(scriptQuery(
+                    new Script(ScriptType.INLINE, "native", NativeTestScriptedTimeout.TEST_NATIVE_SCRIPT_TIMEOUT, Collections.emptyMap())))
                 .execute().actionGet();
         assertThat(searchResponse.isTimedOut(), equalTo(true));
     }
diff --git a/core/src/test/java/org/elasticsearch/search/SearchWithRejectionsIT.java b/core/src/test/java/org/elasticsearch/search/SearchWithRejectionsIT.java
index 6542bad5b8abb..9f0c9ca97eaef 100644
--- a/core/src/test/java/org/elasticsearch/search/SearchWithRejectionsIT.java
+++ b/core/src/test/java/org/elasticsearch/search/SearchWithRejectionsIT.java
@@ -54,7 +54,7 @@ public void testOpenContextsAfterRejections() throws InterruptedException {
 
         int numSearches = 10;
         Future<SearchResponse>[] responses = new Future[numSearches];
-        SearchType searchType = randomFrom(SearchType.DEFAULT, SearchType.QUERY_AND_FETCH, SearchType.QUERY_THEN_FETCH, SearchType.DFS_QUERY_AND_FETCH, SearchType.DFS_QUERY_THEN_FETCH);
+        SearchType searchType = randomFrom(SearchType.DEFAULT, SearchType.QUERY_THEN_FETCH, SearchType.DFS_QUERY_THEN_FETCH);
         logger.info("search type is {}", searchType);
         for (int i = 0; i < numSearches; i++) {
             responses[i] = client().prepareSearch()
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/AggregationCollectorTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/AggregationCollectorTests.java
index 44f1adccc8be7..4ddab52110bcd 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/AggregationCollectorTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/AggregationCollectorTests.java
@@ -19,14 +19,10 @@
 
 package org.elasticsearch.search.aggregations;
 
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
-import org.elasticsearch.search.SearchRequestParsers;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
@@ -61,14 +57,11 @@ public void testNeedsScores() throws Exception {
     }
 
     private boolean needsScores(IndexService index, String agg) throws IOException {
-        AggregatorParsers parser = getInstanceFromNode(SearchRequestParsers.class).aggParsers;
-        IndicesQueriesRegistry queriesRegistry = getInstanceFromNode(IndicesQueriesRegistry.class);
-        XContentParser aggParser = JsonXContent.jsonXContent.createParser(agg);
-        QueryParseContext parseContext = new QueryParseContext(queriesRegistry, aggParser, ParseFieldMatcher.STRICT);
+        XContentParser aggParser = createParser(JsonXContent.jsonXContent, agg);
+        QueryParseContext parseContext = new QueryParseContext(aggParser);
         aggParser.nextToken();
-        SearchContext searchContext = createSearchContext(index);
-        AggregationContext aggContext = new AggregationContext(searchContext);
-        final AggregatorFactories factories = parser.parseAggregators(parseContext).build(aggContext, null);
+        SearchContext context = createSearchContext(index);
+        final AggregatorFactories factories = AggregatorFactories.parseAggregators(parseContext).build(context, null);
         final Aggregator[] aggregators = factories.createTopLevelAggregators();
         assertEquals(1, aggregators.length);
         return aggregators[0].needsScores();
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/AggregationTestScriptsPlugin.java b/core/src/test/java/org/elasticsearch/search/aggregations/AggregationTestScriptsPlugin.java
index 2c320288edf6f..bc98dda41d661 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/AggregationTestScriptsPlugin.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/AggregationTestScriptsPlugin.java
@@ -28,7 +28,8 @@
 import java.util.function.Function;
 
 import static java.util.Collections.singletonMap;
-import static org.elasticsearch.script.ScriptService.ScriptType;
+
+import org.elasticsearch.script.ScriptType;
 
 /**
  * This class contains various mocked scripts that are used in aggregations integration tests.
@@ -43,7 +44,7 @@ public class AggregationTestScriptsPlugin extends MockScriptPlugin {
     //      res[i] = values.get(i) - dec;
     // };
     // return res;
-    public static final Script DECREMENT_ALL_VALUES = new Script("decrement all values", ScriptType.INLINE, NAME, singletonMap("dec", 1));
+    public static final Script DECREMENT_ALL_VALUES = new Script(ScriptType.INLINE, NAME, "decrement all values", singletonMap("dec", 1));
 
     @Override
     protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
@@ -90,7 +91,7 @@ protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
             return doc.get("values");
         });
 
-        scripts.put(DECREMENT_ALL_VALUES.getScript(), vars -> {
+        scripts.put(DECREMENT_ALL_VALUES.getIdOrCode(), vars -> {
             int dec = (int) vars.get("dec");
             Map<?, ?> doc = (Map) vars.get("doc");
             ScriptDocValues.Longs values = (ScriptDocValues.Longs) doc.get("values");
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java
new file mode 100644
index 0000000000000..fb524d1430afc
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java
@@ -0,0 +1,285 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations;
+
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.rest.action.search.RestSearchAction;
+import org.elasticsearch.search.aggregations.Aggregation.CommonFields;
+import org.elasticsearch.search.aggregations.bucket.adjacency.InternalAdjacencyMatrixTests;
+import org.elasticsearch.search.aggregations.bucket.filter.InternalFilterTests;
+import org.elasticsearch.search.aggregations.bucket.filters.InternalFiltersTests;
+import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoHashGridTests;
+import org.elasticsearch.search.aggregations.bucket.global.InternalGlobalTests;
+import org.elasticsearch.search.aggregations.bucket.histogram.InternalDateHistogramTests;
+import org.elasticsearch.search.aggregations.bucket.histogram.InternalHistogramTests;
+import org.elasticsearch.search.aggregations.bucket.missing.InternalMissingTests;
+import org.elasticsearch.search.aggregations.bucket.nested.InternalNestedTests;
+import org.elasticsearch.search.aggregations.bucket.nested.InternalReverseNestedTests;
+import org.elasticsearch.search.aggregations.bucket.sampler.InternalSamplerTests;
+import org.elasticsearch.search.aggregations.bucket.range.InternalBinaryRangeTests;
+import org.elasticsearch.search.aggregations.bucket.range.InternalRangeTests;
+import org.elasticsearch.search.aggregations.bucket.range.date.InternalDateRangeTests;
+import org.elasticsearch.search.aggregations.bucket.range.geodistance.InternalGeoDistanceTests;
+import org.elasticsearch.search.aggregations.bucket.significant.SignificantLongTermsTests;
+import org.elasticsearch.search.aggregations.bucket.significant.SignificantStringTermsTests;
+import org.elasticsearch.search.aggregations.bucket.terms.DoubleTermsTests;
+import org.elasticsearch.search.aggregations.bucket.terms.LongTermsTests;
+import org.elasticsearch.search.aggregations.bucket.terms.StringTermsTests;
+import org.elasticsearch.search.aggregations.metrics.InternalExtendedStatsTests;
+import org.elasticsearch.search.aggregations.metrics.InternalMaxTests;
+import org.elasticsearch.search.aggregations.metrics.InternalStatsBucketTests;
+import org.elasticsearch.search.aggregations.metrics.InternalStatsTests;
+import org.elasticsearch.search.aggregations.metrics.avg.InternalAvgTests;
+import org.elasticsearch.search.aggregations.metrics.cardinality.InternalCardinalityTests;
+import org.elasticsearch.search.aggregations.metrics.geobounds.InternalGeoBoundsTests;
+import org.elasticsearch.search.aggregations.metrics.geocentroid.InternalGeoCentroidTests;
+import org.elasticsearch.search.aggregations.metrics.min.InternalMinTests;
+import org.elasticsearch.search.aggregations.metrics.percentiles.hdr.InternalHDRPercentilesRanksTests;
+import org.elasticsearch.search.aggregations.metrics.percentiles.hdr.InternalHDRPercentilesTests;
+import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.InternalTDigestPercentilesRanksTests;
+import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.InternalTDigestPercentilesTests;
+import org.elasticsearch.search.aggregations.metrics.scripted.InternalScriptedMetricTests;
+import org.elasticsearch.search.aggregations.metrics.sum.InternalSumTests;
+import org.elasticsearch.search.aggregations.metrics.tophits.InternalTopHitsTests;
+import org.elasticsearch.search.aggregations.metrics.valuecount.InternalValueCountTests;
+import org.elasticsearch.search.aggregations.pipeline.InternalSimpleValueTests;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.InternalBucketMetricValueTests;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.percentile.InternalPercentilesBucketTests;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.extended.InternalExtendedStatsBucketTests;
+import org.elasticsearch.search.aggregations.pipeline.derivative.InternalDerivativeTests;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.InternalAggregationTestCase;
+import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+
+import static java.util.Collections.singletonMap;
+import static org.elasticsearch.test.XContentTestUtils.insertRandomFields;
+
+/**
+ * This class tests that aggregations parsing works properly. It checks that we can parse
+ * different aggregations and adds sub-aggregations where applicable.
+ *
+ */
+public class AggregationsTests extends ESTestCase {
+
+    private static final List<InternalAggregationTestCase> aggsTests = getAggsTests();
+
+    private static List<InternalAggregationTestCase> getAggsTests() {
+        List<InternalAggregationTestCase> aggsTests = new ArrayList<>();
+        aggsTests.add(new InternalCardinalityTests());
+        aggsTests.add(new InternalTDigestPercentilesTests());
+        aggsTests.add(new InternalTDigestPercentilesRanksTests());
+        aggsTests.add(new InternalHDRPercentilesTests());
+        aggsTests.add(new InternalHDRPercentilesRanksTests());
+        aggsTests.add(new InternalPercentilesBucketTests());
+        aggsTests.add(new InternalMinTests());
+        aggsTests.add(new InternalMaxTests());
+        aggsTests.add(new InternalAvgTests());
+        aggsTests.add(new InternalSumTests());
+        aggsTests.add(new InternalValueCountTests());
+        aggsTests.add(new InternalSimpleValueTests());
+        aggsTests.add(new InternalDerivativeTests());
+        aggsTests.add(new InternalBucketMetricValueTests());
+        aggsTests.add(new InternalStatsTests());
+        aggsTests.add(new InternalStatsBucketTests());
+        aggsTests.add(new InternalExtendedStatsTests());
+        aggsTests.add(new InternalExtendedStatsBucketTests());
+        aggsTests.add(new InternalGeoBoundsTests());
+        aggsTests.add(new InternalGeoCentroidTests());
+        aggsTests.add(new InternalHistogramTests());
+        aggsTests.add(new InternalDateHistogramTests());
+        aggsTests.add(new LongTermsTests());
+        aggsTests.add(new DoubleTermsTests());
+        aggsTests.add(new StringTermsTests());
+        aggsTests.add(new InternalMissingTests());
+        aggsTests.add(new InternalNestedTests());
+        aggsTests.add(new InternalReverseNestedTests());
+        aggsTests.add(new InternalGlobalTests());
+        aggsTests.add(new InternalFilterTests());
+        aggsTests.add(new InternalSamplerTests());
+        aggsTests.add(new InternalGeoHashGridTests());
+        aggsTests.add(new InternalRangeTests());
+        aggsTests.add(new InternalDateRangeTests());
+        aggsTests.add(new InternalGeoDistanceTests());
+        aggsTests.add(new InternalFiltersTests());
+        aggsTests.add(new InternalAdjacencyMatrixTests());
+        aggsTests.add(new SignificantLongTermsTests());
+        aggsTests.add(new SignificantStringTermsTests());
+        aggsTests.add(new InternalScriptedMetricTests());
+        aggsTests.add(new InternalBinaryRangeTests());
+        aggsTests.add(new InternalTopHitsTests());
+        return Collections.unmodifiableList(aggsTests);
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return new NamedXContentRegistry(InternalAggregationTestCase.getDefaultNamedXContents());
+    }
+
+    @Before
+    public void init() throws Exception {
+        for (InternalAggregationTestCase aggsTest : aggsTests) {
+            if (aggsTest instanceof InternalMultiBucketAggregationTestCase) {
+                // Lower down the number of buckets generated by multi bucket aggregation tests in
+                // order to avoid too many aggregations to be created.
+                ((InternalMultiBucketAggregationTestCase) aggsTest).maxNumberOfBuckets = 3;
+            }
+            aggsTest.setUp();
+        }
+    }
+
+    @After
+    public void cleanUp() throws Exception {
+        for (InternalAggregationTestCase aggsTest : aggsTests) {
+            aggsTest.tearDown();
+        }
+    }
+
+    public void testAllAggsAreBeingTested() {
+        assertEquals(InternalAggregationTestCase.getDefaultNamedXContents().size(), aggsTests.size());
+        Set<String> aggs = aggsTests.stream().map((testCase) -> testCase.createTestInstance().getType()).collect(Collectors.toSet());
+        for (NamedXContentRegistry.Entry entry : InternalAggregationTestCase.getDefaultNamedXContents()) {
+            assertTrue(aggs.contains(entry.name.getPreferredName()));
+        }
+    }
+
+    public void testFromXContent() throws IOException {
+        parseAndAssert(false);
+    }
+
+    public void testFromXContentWithRandomFields() throws IOException {
+        parseAndAssert(true);
+    }
+
+    /**
+     * Test that parsing works for a randomly created Aggregations object with a
+     * randomized aggregation tree. The test randomly chooses an
+     * {@link XContentType}, randomizes the order of the {@link XContent} fields
+     * and randomly sets the `humanReadable` flag when rendering the
+     * {@link XContent}.
+     *
+     * @param addRandomFields
+     *            if set, this will also add random {@link XContent} fields to
+     *            tests that the parsers are lenient to future additions to rest
+     *            responses
+     */
+    private void parseAndAssert(boolean addRandomFields) throws IOException {
+        XContentType xContentType = randomFrom(XContentType.values());
+        final ToXContent.Params params = new ToXContent.MapParams(singletonMap(RestSearchAction.TYPED_KEYS_PARAM, "true"));
+        Aggregations aggregations = createTestInstance();
+        BytesReference originalBytes = toShuffledXContent(aggregations, xContentType, params, randomBoolean());
+        BytesReference mutated;
+        if (addRandomFields) {
+            /*
+             * - don't insert into the root object because it should only contain the named aggregations to test
+             *
+             * - don't insert into the "meta" object, because we pass on everything we find there
+             *
+             * - we don't want to directly insert anything random into "buckets"  objects, they are used with
+             * "keyed" aggregations and contain named bucket objects. Any new named object on this level should
+             * also be a bucket and be parsed as such.
+             *
+             * - we cannot insert randomly into VALUE or VALUES objects e.g. in Percentiles, the keys need to be numeric there
+             *
+             * - we cannot insert into ExtendedMatrixStats "covariance" or "correlation" fields, their syntax is strict
+             */
+            Predicate<String> excludes = path -> (path.isEmpty() || path.endsWith("aggregations")
+                    || path.endsWith(Aggregation.CommonFields.META.getPreferredName())
+                    || path.endsWith(Aggregation.CommonFields.BUCKETS.getPreferredName())
+                    || path.endsWith(CommonFields.VALUES.getPreferredName()) || path.endsWith("covariance") || path.endsWith("correlation")
+                    || path.contains(CommonFields.VALUE.getPreferredName()));
+            mutated = insertRandomFields(xContentType, originalBytes, excludes, random());
+        } else {
+            mutated = originalBytes;
+        }
+        try (XContentParser parser = createParser(xContentType.xContent(), mutated)) {
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
+            assertEquals(Aggregations.AGGREGATIONS_FIELD, parser.currentName());
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            Aggregations parsedAggregations = Aggregations.fromXContent(parser);
+            BytesReference parsedBytes = XContentHelper.toXContent(parsedAggregations, xContentType, randomBoolean());
+            ElasticsearchAssertions.assertToXContentEquivalent(originalBytes, parsedBytes, xContentType);
+        }
+    }
+
+    public void testParsingExceptionOnUnknownAggregation() throws IOException {
+        XContentBuilder builder = XContentFactory.jsonBuilder();
+        builder.startObject();
+        {
+            builder.startObject("unknownAggregation");
+            builder.endObject();
+        }
+        builder.endObject();
+        BytesReference originalBytes = builder.bytes();
+        try (XContentParser parser = createParser(builder.contentType().xContent(), originalBytes)) {
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            ParsingException ex = expectThrows(ParsingException.class, () -> Aggregations.fromXContent(parser));
+            assertEquals("Could not parse aggregation keyed as [unknownAggregation]", ex.getMessage());
+        }
+    }
+
+    public final InternalAggregations createTestInstance() {
+        return createTestInstance(1, 0, 5);
+    }
+
+    private static InternalAggregations createTestInstance(final int minNumAggs, final int currentDepth, final int maxDepth) {
+        int numAggs = randomIntBetween(minNumAggs, 4);
+        List<InternalAggregation> aggs = new ArrayList<>(numAggs);
+        for (int i = 0; i < numAggs; i++) {
+            InternalAggregationTestCase testCase = randomFrom(aggsTests);
+            if (testCase instanceof InternalMultiBucketAggregationTestCase) {
+                InternalMultiBucketAggregationTestCase multiBucketAggTestCase = (InternalMultiBucketAggregationTestCase) testCase;
+                if (currentDepth < maxDepth) {
+                    multiBucketAggTestCase.subAggregationsSupplier = () -> createTestInstance(0, currentDepth + 1, maxDepth);
+                } else {
+                    multiBucketAggTestCase.subAggregationsSupplier = () -> InternalAggregations.EMPTY;
+                }
+            } else if (testCase instanceof InternalSingleBucketAggregationTestCase) {
+                InternalSingleBucketAggregationTestCase singleBucketAggTestCase = (InternalSingleBucketAggregationTestCase) testCase;
+                if (currentDepth < maxDepth) {
+                    singleBucketAggTestCase.subAggregationsSupplier = () -> createTestInstance(0, currentDepth + 1, maxDepth);
+                } else {
+                    singleBucketAggTestCase.subAggregationsSupplier = () -> InternalAggregations.EMPTY;
+                }
+            }
+            aggs.add(testCase.createTestInstance());
+        }
+        return new InternalAggregations(aggs);
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorFactoriesTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorFactoriesTests.java
new file mode 100644
index 0000000000000..0caa8e8d777c2
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorFactoriesTests.java
@@ -0,0 +1,246 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations;
+
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilders;
+import org.elasticsearch.test.AbstractQueryTestCase;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.List;
+import java.util.Random;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static java.util.Collections.emptyList;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+
+public class AggregatorFactoriesTests extends ESTestCase {
+    private String[] currentTypes;
+
+    private NamedXContentRegistry xContentRegistry;
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        // we have to prefer CURRENT since with the range of versions we support
+        // it's rather unlikely to get the current actually.
+        Settings settings = Settings.builder().put("node.name", AbstractQueryTestCase.class.toString())
+                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build();
+        // create some random type with some default field, those types will
+        // stick around for all of the subclasses
+        currentTypes = new String[randomIntBetween(0, 5)];
+        for (int i = 0; i < currentTypes.length; i++) {
+            String type = randomAlphaOfLengthBetween(1, 10);
+            currentTypes[i] = type;
+        }
+        xContentRegistry = new NamedXContentRegistry(new SearchModule(settings, false, emptyList()).getNamedXContents());
+    }
+
+    public void testGetAggregatorFactories_returnsUnmodifiableList() {
+        AggregatorFactories.Builder builder = new AggregatorFactories.Builder().addAggregator(AggregationBuilders.avg("foo"));
+        List<AggregationBuilder> aggregatorFactories = builder.getAggregatorFactories();
+        assertThat(aggregatorFactories.size(), equalTo(1));
+        expectThrows(UnsupportedOperationException.class, () -> aggregatorFactories.add(AggregationBuilders.avg("bar")));
+    }
+
+    public void testGetPipelineAggregatorFactories_returnsUnmodifiableList() {
+        AggregatorFactories.Builder builder = new AggregatorFactories.Builder().addPipelineAggregator(
+            PipelineAggregatorBuilders.avgBucket("foo", "path1"));
+        List<PipelineAggregationBuilder> pipelineAggregatorFactories = builder.getPipelineAggregatorFactories();
+        assertThat(pipelineAggregatorFactories.size(), equalTo(1));
+        expectThrows(UnsupportedOperationException.class,
+            () -> pipelineAggregatorFactories.add(PipelineAggregatorBuilders.avgBucket("bar", "path2")));
+    }
+
+    public void testTwoTypes() throws Exception {
+        XContentBuilder source = JsonXContent.contentBuilder()
+                .startObject()
+                    .startObject("in_stock")
+                        .startObject("filter")
+                            .startObject("range")
+                                .startObject("stock")
+                                    .field("gt", 0)
+                                .endObject()
+                            .endObject()
+                        .endObject()
+                        .startObject("terms")
+                            .field("field", "stock")
+                        .endObject()
+                    .endObject()
+                .endObject();
+        XContentParser parser = createParser(source);
+        QueryParseContext parseContext = new QueryParseContext(parser);
+        assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
+        Exception e = expectThrows(ParsingException.class, () -> AggregatorFactories.parseAggregators(parseContext));
+        assertThat(e.toString(), containsString("Found two aggregation type definitions in [in_stock]: [filter] and [terms]"));
+    }
+
+    public void testTwoAggs() throws Exception {
+        XContentBuilder source = JsonXContent.contentBuilder()
+                .startObject()
+                    .startObject("by_date")
+                        .startObject("date_histogram")
+                            .field("field", "timestamp")
+                            .field("interval", "month")
+                        .endObject()
+                        .startObject("aggs")
+                            .startObject("tag_count")
+                                .startObject("cardinality")
+                                    .field("field", "tag")
+                                .endObject()
+                            .endObject()
+                        .endObject()
+                        .startObject("aggs") // 2nd "aggs": illegal
+                            .startObject("tag_count2")
+                                .startObject("cardinality")
+                                    .field("field", "tag")
+                                .endObject()
+                            .endObject()
+                        .endObject()
+                    .endObject()
+                .endObject();
+        XContentParser parser = createParser(source);
+        QueryParseContext parseContext = new QueryParseContext(parser);
+        assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
+        Exception e = expectThrows(ParsingException.class, () -> AggregatorFactories.parseAggregators(parseContext));
+        assertThat(e.toString(), containsString("Found two sub aggregation definitions under [by_date]"));
+    }
+
+    public void testInvalidAggregationName() throws Exception {
+        Matcher matcher = Pattern.compile("[^\\[\\]>]+").matcher("");
+        String name;
+        Random rand = random();
+        int len = randomIntBetween(1, 5);
+        char[] word = new char[len];
+        while (true) {
+            for (int i = 0; i < word.length; i++) {
+                word[i] = (char) rand.nextInt(127);
+            }
+            name = String.valueOf(word);
+            if (!matcher.reset(name).matches()) {
+                break;
+            }
+        }
+
+        XContentBuilder source = JsonXContent.contentBuilder()
+                .startObject()
+                    .startObject(name)
+                        .startObject("filter")
+                            .startObject("range")
+                                .startObject("stock")
+                                    .field("gt", 0)
+                                .endObject()
+                            .endObject()
+                        .endObject()
+                    .endObject()
+                .endObject();
+        XContentParser parser = createParser(source);
+        QueryParseContext parseContext = new QueryParseContext(parser);
+        assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
+        Exception e = expectThrows(ParsingException.class, () -> AggregatorFactories.parseAggregators(parseContext));
+        assertThat(e.toString(), containsString("Invalid aggregation name [" + name + "]"));
+    }
+
+    public void testSameAggregationName() throws Exception {
+        final String name = randomAlphaOfLengthBetween(1, 10);
+        XContentBuilder source = JsonXContent.contentBuilder()
+                .startObject()
+                    .startObject(name)
+                        .startObject("terms")
+                            .field("field", "a")
+                        .endObject()
+                    .endObject()
+                    .startObject(name)
+                        .startObject("terms")
+                            .field("field", "b")
+                        .endObject()
+                    .endObject()
+                .endObject();
+        XContentParser parser = createParser(source);
+        QueryParseContext parseContext = new QueryParseContext(parser);
+        assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
+        Exception e = expectThrows(IllegalArgumentException.class, () -> AggregatorFactories.parseAggregators(parseContext));
+        assertThat(e.toString(), containsString("Two sibling aggregations cannot have the same name: [" + name + "]"));
+    }
+
+    public void testMissingName() throws Exception {
+        XContentBuilder source = JsonXContent.contentBuilder()
+                .startObject()
+                    .startObject("by_date")
+                        .startObject("date_histogram")
+                            .field("field", "timestamp")
+                            .field("interval", "month")
+                        .endObject()
+                        .startObject("aggs")
+                            // the aggregation name is missing
+                            //.startObject("tag_count")
+                            .startObject("cardinality")
+                                .field("field", "tag")
+                            .endObject()
+                            //.endObject()
+                        .endObject()
+                    .endObject()
+                .endObject();
+        XContentParser parser = createParser(source);
+        QueryParseContext parseContext = new QueryParseContext(parser);
+        assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
+        Exception e = expectThrows(ParsingException.class, () -> AggregatorFactories.parseAggregators(parseContext));
+        assertThat(e.toString(), containsString("Expected [START_OBJECT] under [field], but got a [VALUE_STRING] in [cardinality]"));
+    }
+
+    public void testMissingType() throws Exception {
+        XContentBuilder source = JsonXContent.contentBuilder()
+                .startObject()
+                    .startObject("by_date")
+                        .startObject("date_histogram")
+                            .field("field", "timestamp")
+                            .field("interval", "month")
+                        .endObject()
+                        .startObject("aggs")
+                            .startObject("tag_count")
+                                // the aggregation type is missing
+                                //.startObject("cardinality")
+                                .field("field", "tag")
+                                //.endObject()
+                            .endObject()
+                        .endObject()
+                    .endObject()
+                .endObject();
+        XContentParser parser = createParser(source);
+        QueryParseContext parseContext = new QueryParseContext(parser);
+        assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
+        Exception e = expectThrows(ParsingException.class, () -> AggregatorFactories.parseAggregators(parseContext));
+        assertThat(e.toString(), containsString("Expected [START_OBJECT] under [field], but got a [VALUE_STRING] in [tag_count]"));
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return xContentRegistry;
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java
deleted file mode 100644
index a7476381b8eac..0000000000000
--- a/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java
+++ /dev/null
@@ -1,355 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.aggregations;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.inject.AbstractModule;
-import org.elasticsearch.common.inject.Injector;
-import org.elasticsearch.common.inject.ModulesBuilder;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsModule;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.json.JsonXContent;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.search.SearchRequestParsers;
-import org.elasticsearch.test.AbstractQueryTestCase;
-import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.indices.IndicesModule;
-import org.elasticsearch.indices.breaker.CircuitBreakerService;
-import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
-import org.elasticsearch.script.ScriptModule;
-import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.search.SearchModule;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.IndexSettingsModule;
-import org.elasticsearch.test.InternalSettingsPlugin;
-import org.elasticsearch.test.VersionUtils;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Random;
-import java.util.function.Function;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import static java.util.Collections.emptyList;
-import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
-import static org.elasticsearch.test.ClusterServiceUtils.setState;
-import static org.hamcrest.Matchers.containsString;
-
-public class AggregatorParsingTests extends ESTestCase {
-
-    private static Injector injector;
-    private static Index index;
-
-    private static String[] currentTypes;
-
-    protected static String[] getCurrentTypes() {
-        return currentTypes;
-    }
-
-    private static NamedWriteableRegistry namedWriteableRegistry;
-
-    protected static AggregatorParsers aggParsers;
-    protected static IndicesQueriesRegistry queriesRegistry;
-    protected static ParseFieldMatcher parseFieldMatcher;
-
-    /**
-     * Setup for the whole base test class.
-     */
-    @BeforeClass
-    public static void init() throws IOException {
-        // we have to prefer CURRENT since with the range of versions we support
-        // it's rather unlikely to get the current actually.
-        Version version = randomBoolean() ? Version.CURRENT
-                : VersionUtils.randomVersionBetween(random(), Version.V_2_0_0_beta1, Version.CURRENT);
-        Settings settings = Settings.builder().put("node.name", AbstractQueryTestCase.class.toString())
-                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
-                .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), false).build();
-
-        index = new Index(randomAsciiOfLengthBetween(1, 10), "_na_");
-        Settings indexSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
-        final ThreadPool threadPool = new ThreadPool(settings);
-        final ClusterService clusterService = createClusterService(threadPool);
-        setState(clusterService, new ClusterState.Builder(clusterService.state()).metaData(new MetaData.Builder()
-                .put(new IndexMetaData.Builder(index.getName()).settings(indexSettings).numberOfShards(1).numberOfReplicas(0))));
-        ScriptModule scriptModule = newTestScriptModule();
-        List<Setting<?>> scriptSettings = scriptModule.getSettings();
-        scriptSettings.add(InternalSettingsPlugin.VERSION_CREATED);
-        SettingsModule settingsModule = new SettingsModule(settings, scriptSettings, Collections.emptyList());
-
-        IndicesModule indicesModule = new IndicesModule(Collections.emptyList()) {
-            @Override
-            protected void configure() {
-                bindMapperExtension();
-            }
-        };
-        SearchModule searchModule = new SearchModule(settings, false, emptyList()) {
-            @Override
-            protected void configureSearch() {
-                // Skip me
-            }
-        };
-        List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
-        entries.addAll(indicesModule.getNamedWriteables());
-        entries.addAll(searchModule.getNamedWriteables());
-        namedWriteableRegistry = new NamedWriteableRegistry(entries);
-        injector = new ModulesBuilder().add(
-            (b) -> {
-                b.bind(Environment.class).toInstance(new Environment(settings));
-                b.bind(ThreadPool.class).toInstance(threadPool);
-                b.bind(ScriptService.class).toInstance(scriptModule.getScriptService());
-            },
-            settingsModule, indicesModule, searchModule,
-            new IndexSettingsModule(index, settings),
-            new AbstractModule() {
-                @Override
-                protected void configure() {
-                    bind(ClusterService.class).toInstance(clusterService);
-                    bind(CircuitBreakerService.class).toInstance(new NoneCircuitBreakerService());
-                    bind(NamedWriteableRegistry.class).toInstance(namedWriteableRegistry);
-                }
-            }).createInjector();
-        aggParsers = injector.getInstance(SearchRequestParsers.class).aggParsers;
-        // create some random type with some default field, those types will
-        // stick around for all of the subclasses
-        currentTypes = new String[randomIntBetween(0, 5)];
-        for (int i = 0; i < currentTypes.length; i++) {
-            String type = randomAsciiOfLengthBetween(1, 10);
-            currentTypes[i] = type;
-        }
-        queriesRegistry = injector.getInstance(IndicesQueriesRegistry.class);
-        parseFieldMatcher = ParseFieldMatcher.STRICT;
-    }
-
-    @AfterClass
-    public static void afterClass() throws Exception {
-        injector.getInstance(ClusterService.class).close();
-        terminate(injector.getInstance(ThreadPool.class));
-        injector = null;
-        index = null;
-        aggParsers = null;
-        currentTypes = null;
-        namedWriteableRegistry = null;
-    }
-
-    public void testTwoTypes() throws Exception {
-        String source = JsonXContent.contentBuilder()
-                .startObject()
-                .startObject("in_stock")
-                .startObject("filter")
-                .startObject("range")
-                .startObject("stock")
-                .field("gt", 0)
-                .endObject()
-                .endObject()
-                .endObject()
-                .startObject("terms")
-                .field("field", "stock")
-                .endObject()
-                .endObject()
-                .endObject().string();
-        try {
-            XContentParser parser = XContentFactory.xContent(source).createParser(source);
-            QueryParseContext parseContext = new QueryParseContext(queriesRegistry, parser, parseFieldMatcher);
-            assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
-            aggParsers.parseAggregators(parseContext);
-            fail();
-        } catch (ParsingException e) {
-            assertThat(e.toString(), containsString("Found two aggregation type definitions in [in_stock]: [filter] and [terms]"));
-        }
-    }
-
-    public void testTwoAggs() throws Exception {
-        String source = JsonXContent.contentBuilder()
-                .startObject()
-                    .startObject("by_date")
-                        .startObject("date_histogram")
-                            .field("field", "timestamp")
-                            .field("interval", "month")
-                        .endObject()
-                        .startObject("aggs")
-                            .startObject("tag_count")
-                                .startObject("cardinality")
-                                    .field("field", "tag")
-                                .endObject()
-                            .endObject()
-                        .endObject()
-                        .startObject("aggs") // 2nd "aggs": illegal
-                            .startObject("tag_count2")
-                                .startObject("cardinality")
-                                    .field("field", "tag")
-                                .endObject()
-                            .endObject()
-                        .endObject()
-                    .endObject()
-                .endObject().string();
-        try {
-            XContentParser parser = XContentFactory.xContent(source).createParser(source);
-            QueryParseContext parseContext = new QueryParseContext(queriesRegistry, parser, parseFieldMatcher);
-            assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
-            aggParsers.parseAggregators(parseContext);
-            fail();
-        } catch (ParsingException e) {
-            assertThat(e.toString(), containsString("Found two sub aggregation definitions under [by_date]"));
-        }
-    }
-
-    public void testInvalidAggregationName() throws Exception {
-        Matcher matcher = Pattern.compile("[^\\[\\]>]+").matcher("");
-        String name;
-        Random rand = random();
-        int len = randomIntBetween(1, 5);
-        char[] word = new char[len];
-        while (true) {
-            for (int i = 0; i < word.length; i++) {
-                word[i] = (char) rand.nextInt(127);
-            }
-            name = String.valueOf(word);
-            if (!matcher.reset(name).matches()) {
-                break;
-            }
-        }
-
-        String source = JsonXContent.contentBuilder()
-                .startObject()
-                    .startObject(name)
-                        .startObject("filter")
-                            .startObject("range")
-                                .startObject("stock")
-                                    .field("gt", 0)
-                                .endObject()
-                            .endObject()
-                        .endObject()
-                    .endObject()
-                .endObject().string();
-        try {
-            XContentParser parser = XContentFactory.xContent(source).createParser(source);
-            QueryParseContext parseContext = new QueryParseContext(queriesRegistry, parser, parseFieldMatcher);
-            assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
-            aggParsers.parseAggregators(parseContext);
-            fail();
-        } catch (ParsingException e) {
-            assertThat(e.toString(), containsString("Invalid aggregation name [" + name + "]"));
-        }
-    }
-
-    public void testSameAggregationName() throws Exception {
-        final String name = randomAsciiOfLengthBetween(1, 10);
-        String source = JsonXContent.contentBuilder()
-                .startObject()
-                .startObject(name)
-                .startObject("terms")
-                .field("field", "a")
-                .endObject()
-                .endObject()
-                .startObject(name)
-                .startObject("terms")
-                .field("field", "b")
-                .endObject()
-                .endObject()
-                .endObject().string();
-        try {
-            XContentParser parser = XContentFactory.xContent(source).createParser(source);
-            QueryParseContext parseContext = new QueryParseContext(queriesRegistry, parser, parseFieldMatcher);
-            assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
-            aggParsers.parseAggregators(parseContext);
-            fail();
-        } catch (IllegalArgumentException e) {
-            assertThat(e.toString(), containsString("Two sibling aggregations cannot have the same name: [" + name + "]"));
-        }
-    }
-
-    public void testMissingName() throws Exception {
-        String source = JsonXContent.contentBuilder()
-                .startObject()
-                    .startObject("by_date")
-                        .startObject("date_histogram")
-                            .field("field", "timestamp")
-                            .field("interval", "month")
-                        .endObject()
-                        .startObject("aggs")
-                            // the aggregation name is missing
-                            //.startObject("tag_count")
-                            .startObject("cardinality")
-                                .field("field", "tag")
-                            .endObject()
-                            //.endObject()
-                        .endObject()
-                    .endObject()
-                .endObject().string();
-        try {
-            XContentParser parser = XContentFactory.xContent(source).createParser(source);
-            QueryParseContext parseContext = new QueryParseContext(queriesRegistry, parser, parseFieldMatcher);
-            assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
-            aggParsers.parseAggregators(parseContext);
-            fail();
-        } catch (ParsingException e) {
-            // All Good
-        }
-    }
-
-    public void testMissingType() throws Exception {
-        String source = JsonXContent.contentBuilder()
-                .startObject()
-                    .startObject("by_date")
-                        .startObject("date_histogram")
-                            .field("field", "timestamp")
-                            .field("interval", "month")
-                        .endObject()
-                        .startObject("aggs")
-                            .startObject("tag_count")
-                                // the aggregation type is missing
-                                //.startObject("cardinality")
-                                .field("field", "tag")
-                                //.endObject()
-                            .endObject()
-                        .endObject()
-                    .endObject()
-                .endObject().string();
-        try {
-            XContentParser parser = XContentFactory.xContent(source).createParser(source);
-            QueryParseContext parseContext = new QueryParseContext(queriesRegistry, parser, parseFieldMatcher);
-            assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
-            aggParsers.parseAggregators(parseContext);
-            fail();
-        } catch (ParsingException e) {
-            // All Good
-        }
-    }
-}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
deleted file mode 100644
index b9c0e5f09c1a5..0000000000000
--- a/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
+++ /dev/null
@@ -1,301 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.aggregations;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.inject.Injector;
-import org.elasticsearch.common.inject.ModulesBuilder;
-import org.elasticsearch.common.inject.util.Providers;
-import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsModule;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.search.SearchRequestParsers;
-import org.elasticsearch.test.AbstractQueryTestCase;
-import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.indices.IndicesModule;
-import org.elasticsearch.indices.breaker.CircuitBreakerService;
-import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
-import org.elasticsearch.script.ScriptModule;
-import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.search.SearchModule;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.IndexSettingsModule;
-import org.elasticsearch.test.InternalSettingsPlugin;
-import org.elasticsearch.test.VersionUtils;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-import static java.util.Collections.emptyList;
-import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
-import static org.elasticsearch.test.ClusterServiceUtils.setState;
-import static org.hamcrest.Matchers.equalTo;
-
-public abstract class BaseAggregationTestCase<AB extends AbstractAggregationBuilder<AB>> extends ESTestCase {
-
-    protected static final String STRING_FIELD_NAME = "mapped_string";
-    protected static final String INT_FIELD_NAME = "mapped_int";
-    protected static final String DOUBLE_FIELD_NAME = "mapped_double";
-    protected static final String BOOLEAN_FIELD_NAME = "mapped_boolean";
-    protected static final String DATE_FIELD_NAME = "mapped_date";
-    protected static final String IP_FIELD_NAME = "mapped_ip";
-    protected static final String OBJECT_FIELD_NAME = "mapped_object";
-    protected static final String[] mappedFieldNames = new String[]{STRING_FIELD_NAME, INT_FIELD_NAME,
-            DOUBLE_FIELD_NAME, BOOLEAN_FIELD_NAME, DATE_FIELD_NAME, IP_FIELD_NAME, OBJECT_FIELD_NAME};
-
-    private static Injector injector;
-    private static Index index;
-
-    private static String[] currentTypes;
-
-    protected static String[] getCurrentTypes() {
-        return currentTypes;
-    }
-
-    private static NamedWriteableRegistry namedWriteableRegistry;
-
-    protected static AggregatorParsers aggParsers;
-    protected static IndicesQueriesRegistry queriesRegistry;
-    protected static ParseFieldMatcher parseFieldMatcher;
-
-    protected abstract AB createTestAggregatorBuilder();
-
-    /**
-     * Setup for the whole base test class.
-     */
-    @BeforeClass
-    public static void init() throws IOException {
-        index = new Index(randomAsciiOfLengthBetween(1, 10), "_na_");
-        injector = buildInjector(index);
-        namedWriteableRegistry = injector.getInstance(NamedWriteableRegistry.class);
-        aggParsers = injector.getInstance(SearchRequestParsers.class).aggParsers;
-        //create some random type with some default field, those types will stick around for all of the subclasses
-        currentTypes = new String[randomIntBetween(0, 5)];
-        for (int i = 0; i < currentTypes.length; i++) {
-            String type = randomAsciiOfLengthBetween(1, 10);
-            currentTypes[i] = type;
-        }
-        queriesRegistry = injector.getInstance(IndicesQueriesRegistry.class);
-        parseFieldMatcher = ParseFieldMatcher.STRICT;
-    }
-
-    public static final Injector buildInjector(Index index) {
-        // we have to prefer CURRENT since with the range of versions we support it's rather unlikely to get the current actually.
-        Version version = randomBoolean() ? Version.CURRENT
-            : VersionUtils.randomVersionBetween(random(), Version.V_2_0_0_beta1, Version.CURRENT);
-        Settings settings = Settings.builder()
-            .put("node.name", AbstractQueryTestCase.class.toString())
-            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
-            .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), false)
-            .build();
-
-        Settings indexSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
-        final ThreadPool threadPool = new ThreadPool(settings);
-        final ClusterService clusterService = createClusterService(threadPool);
-        setState(clusterService, new ClusterState.Builder(clusterService.state()).metaData(new MetaData.Builder()
-            .put(new IndexMetaData.Builder(index.getName()).settings(indexSettings).numberOfShards(1).numberOfReplicas(0))));
-        ScriptModule scriptModule = newTestScriptModule();
-        List<Setting<?>> scriptSettings = scriptModule.getSettings();
-        scriptSettings.add(InternalSettingsPlugin.VERSION_CREATED);
-        SettingsModule settingsModule = new SettingsModule(settings, scriptSettings, Collections.emptyList());
-        IndicesModule indicesModule = new IndicesModule(Collections.emptyList()) {
-            @Override
-            protected void configure() {
-                bindMapperExtension();
-            }
-        };
-        SearchModule searchModule = new SearchModule(settings, false, emptyList()) {
-            @Override
-            protected void configureSearch() {
-                // Skip me
-            }
-        };
-        List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
-        entries.addAll(indicesModule.getNamedWriteables());
-        entries.addAll(searchModule.getNamedWriteables());
-        NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(entries);
-        return new ModulesBuilder().add(
-            (b) -> {
-                b.bind(Environment.class).toInstance(new Environment(settings));
-                b.bind(ThreadPool.class).toInstance(threadPool);
-                b.bind(ScriptService.class).toInstance(scriptModule.getScriptService());
-                b.bind(ClusterService.class).toProvider(Providers.of(clusterService));
-                b.bind(CircuitBreakerService.class).to(NoneCircuitBreakerService.class);
-                b.bind(NamedWriteableRegistry.class).toInstance(namedWriteableRegistry);
-            },
-            settingsModule, indicesModule, searchModule, new IndexSettingsModule(index, settings)
-        ).createInjector();
-    }
-
-
-    @AfterClass
-    public static void afterClass() throws Exception {
-        injector.getInstance(ClusterService.class).close();
-        terminate(injector.getInstance(ThreadPool.class));
-        injector = null;
-        index = null;
-        aggParsers = null;
-        currentTypes = null;
-        namedWriteableRegistry = null;
-    }
-
-    /**
-     * Generic test that creates new AggregatorFactory from the test
-     * AggregatorFactory and checks both for equality and asserts equality on
-     * the two queries.
-     */
-    public void testFromXContent() throws IOException {
-        AB testAgg = createTestAggregatorBuilder();
-        AggregatorFactories.Builder factoriesBuilder = AggregatorFactories.builder().addAggregator(testAgg);
-        XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
-        if (randomBoolean()) {
-            builder.prettyPrint();
-        }
-        factoriesBuilder.toXContent(builder, ToXContent.EMPTY_PARAMS);
-        XContentBuilder shuffled = shuffleXContent(builder);
-        XContentParser parser = XContentFactory.xContent(shuffled.bytes()).createParser(shuffled.bytes());
-        QueryParseContext parseContext = new QueryParseContext(queriesRegistry, parser, parseFieldMatcher);
-        assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
-        assertSame(XContentParser.Token.FIELD_NAME, parser.nextToken());
-        assertEquals(testAgg.name, parser.currentName());
-        assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
-        assertSame(XContentParser.Token.FIELD_NAME, parser.nextToken());
-        assertEquals(testAgg.type.name(), parser.currentName());
-        assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
-        AggregationBuilder newAgg = aggParsers.parser(testAgg.getType(), ParseFieldMatcher.STRICT).parse(testAgg.name, parseContext);
-        assertSame(XContentParser.Token.END_OBJECT, parser.currentToken());
-        assertSame(XContentParser.Token.END_OBJECT, parser.nextToken());
-        assertSame(XContentParser.Token.END_OBJECT, parser.nextToken());
-        assertNull(parser.nextToken());
-        assertNotNull(newAgg);
-        assertNotSame(newAgg, testAgg);
-        assertEquals(testAgg, newAgg);
-        assertEquals(testAgg.hashCode(), newAgg.hashCode());
-    }
-
-    /**
-     * Test serialization and deserialization of the test AggregatorFactory.
-     */
-
-    public void testSerialization() throws IOException {
-        AB testAgg = createTestAggregatorBuilder();
-        try (BytesStreamOutput output = new BytesStreamOutput()) {
-            output.writeNamedWriteable(testAgg);
-            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
-                AggregationBuilder deserialized = in.readNamedWriteable(AggregationBuilder.class);
-                assertEquals(testAgg, deserialized);
-                assertEquals(testAgg.hashCode(), deserialized.hashCode());
-                assertNotSame(testAgg, deserialized);
-            }
-        }
-    }
-
-
-    public void testEqualsAndHashcode() throws IOException {
-        AB firstAgg = createTestAggregatorBuilder();
-        assertFalse("aggregation is equal to null", firstAgg.equals(null));
-        assertFalse("aggregation is equal to incompatible type", firstAgg.equals(""));
-        assertTrue("aggregation is not equal to self", firstAgg.equals(firstAgg));
-        assertThat("same aggregation's hashcode returns different values if called multiple times", firstAgg.hashCode(),
-                equalTo(firstAgg.hashCode()));
-
-        AB secondQuery = copyAggregation(firstAgg);
-        assertTrue("aggregation is not equal to self", secondQuery.equals(secondQuery));
-        assertTrue("aggregation is not equal to its copy", firstAgg.equals(secondQuery));
-        assertTrue("equals is not symmetric", secondQuery.equals(firstAgg));
-        assertThat("aggregation copy's hashcode is different from original hashcode", secondQuery.hashCode(), equalTo(firstAgg.hashCode()));
-
-        AB thirdQuery = copyAggregation(secondQuery);
-        assertTrue("aggregation is not equal to self", thirdQuery.equals(thirdQuery));
-        assertTrue("aggregation is not equal to its copy", secondQuery.equals(thirdQuery));
-        assertThat("aggregation copy's hashcode is different from original hashcode", secondQuery.hashCode(),
-                equalTo(thirdQuery.hashCode()));
-        assertTrue("equals is not transitive", firstAgg.equals(thirdQuery));
-        assertThat("aggregation copy's hashcode is different from original hashcode", firstAgg.hashCode(), equalTo(thirdQuery.hashCode()));
-        assertTrue("equals is not symmetric", thirdQuery.equals(secondQuery));
-        assertTrue("equals is not symmetric", thirdQuery.equals(firstAgg));
-    }
-
-    // we use the streaming infra to create a copy of the query provided as
-    // argument
-    private AB copyAggregation(AB agg) throws IOException {
-        try (BytesStreamOutput output = new BytesStreamOutput()) {
-            agg.writeTo(output);
-            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
-                @SuppressWarnings("unchecked")
-                AB secondAgg = (AB) namedWriteableRegistry.getReader(AggregationBuilder.class, agg.getWriteableName()).read(in);
-                return secondAgg;
-            }
-        }
-    }
-
-    protected String[] getRandomTypes() {
-        String[] types;
-        if (currentTypes.length > 0 && randomBoolean()) {
-            int numberOfQueryTypes = randomIntBetween(1, currentTypes.length);
-            types = new String[numberOfQueryTypes];
-            for (int i = 0; i < numberOfQueryTypes; i++) {
-                types[i] = randomFrom(currentTypes);
-            }
-        } else {
-            if (randomBoolean()) {
-                types = new String[]{MetaData.ALL};
-            } else {
-                types = new String[0];
-            }
-        }
-        return types;
-    }
-
-    public String randomNumericField() {
-        int randomInt = randomInt(3);
-        switch (randomInt) {
-            case 0:
-                return DATE_FIELD_NAME;
-            case 1:
-                return DOUBLE_FIELD_NAME;
-            case 2:
-            default:
-                return INT_FIELD_NAME;
-        }
-    }
-}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
index b0952b2de6033..0a738ec022beb 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
@@ -20,30 +20,33 @@
 package org.elasticsearch.search.aggregations;
 
 import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.inject.Injector;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.index.Index;
+import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
-import org.elasticsearch.search.SearchRequestParsers;
+import org.elasticsearch.indices.IndicesModule;
+import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregationBuilder;
+import org.elasticsearch.test.AbstractQueryTestCase;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
 
 import java.io.IOException;
-import static org.hamcrest.Matchers.equalTo;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import static java.util.Collections.emptyList;
+import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
+import static org.hamcrest.Matchers.hasSize;
 
 public abstract class BasePipelineAggregationTestCase<AF extends AbstractPipelineAggregationBuilder<AF>> extends ESTestCase {
 
@@ -53,51 +56,40 @@ public abstract class BasePipelineAggregationTestCase<AF extends AbstractPipelin
     protected static final String BOOLEAN_FIELD_NAME = "mapped_boolean";
     protected static final String DATE_FIELD_NAME = "mapped_date";
 
-    private static Injector injector;
-    private static Index index;
-
-    private static String[] currentTypes;
+    private String[] currentTypes;
 
-    protected static String[] getCurrentTypes() {
+    protected String[] getCurrentTypes() {
         return currentTypes;
     }
 
-    private static NamedWriteableRegistry namedWriteableRegistry;
-
-    protected static AggregatorParsers aggParsers;
-    protected static ParseFieldMatcher parseFieldMatcher;
-    protected static IndicesQueriesRegistry queriesRegistry;
+    private NamedWriteableRegistry namedWriteableRegistry;
+    private NamedXContentRegistry xContentRegistry;
 
     protected abstract AF createTestAggregatorFactory();
 
     /**
      * Setup for the whole base test class.
      */
-    @BeforeClass
-    public static void init() throws IOException {
-        index = new Index(randomAsciiOfLengthBetween(1, 10), "_na_");
-        injector = BaseAggregationTestCase.buildInjector(index);
-        namedWriteableRegistry = injector.getInstance(NamedWriteableRegistry.class);
-        aggParsers = injector.getInstance(SearchRequestParsers.class).aggParsers;
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        Settings settings = Settings.builder()
+            .put("node.name", AbstractQueryTestCase.class.toString())
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
+            .build();
+        IndicesModule indicesModule = new IndicesModule(Collections.emptyList());
+        SearchModule searchModule = new SearchModule(settings, false, emptyList());
+        List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
+        entries.addAll(indicesModule.getNamedWriteables());
+        entries.addAll(searchModule.getNamedWriteables());
+        namedWriteableRegistry = new NamedWriteableRegistry(entries);
+        xContentRegistry = new NamedXContentRegistry(searchModule.getNamedXContents());
         //create some random type with some default field, those types will stick around for all of the subclasses
         currentTypes = new String[randomIntBetween(0, 5)];
         for (int i = 0; i < currentTypes.length; i++) {
-            String type = randomAsciiOfLengthBetween(1, 10);
+            String type = randomAlphaOfLengthBetween(1, 10);
             currentTypes[i] = type;
         }
-        queriesRegistry = injector.getInstance(IndicesQueriesRegistry.class);
-        parseFieldMatcher = ParseFieldMatcher.STRICT;
-    }
-
-    @AfterClass
-    public static void afterClass() throws Exception {
-        injector.getInstance(ClusterService.class).close();
-        terminate(injector.getInstance(ThreadPool.class));
-        injector = null;
-        index = null;
-        aggParsers = null;
-        currentTypes = null;
-        namedWriteableRegistry = null;
     }
 
     /**
@@ -105,7 +97,6 @@ public static void afterClass() throws Exception {
      * AggregatorFactory and checks both for equality and asserts equality on
      * the two queries.
      */
-
     public void testFromXContent() throws IOException {
         AF testAgg = createTestAggregatorFactory();
         AggregatorFactories.Builder factoriesBuilder = AggregatorFactories.builder().skipResolveOrder().addPipelineAggregator(testAgg);
@@ -116,33 +107,30 @@ public void testFromXContent() throws IOException {
         }
         factoriesBuilder.toXContent(builder, ToXContent.EMPTY_PARAMS);
         XContentBuilder shuffled = shuffleXContent(builder);
-        XContentParser parser = XContentFactory.xContent(shuffled.bytes()).createParser(shuffled.bytes());
-        QueryParseContext parseContext = new QueryParseContext(queriesRegistry, parser, parseFieldMatcher);
+        XContentParser parser = createParser(shuffled);
         String contentString = factoriesBuilder.toString();
         logger.info("Content string: {}", contentString);
-        assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
-        assertSame(XContentParser.Token.FIELD_NAME, parser.nextToken());
-        assertEquals(testAgg.getName(), parser.currentName());
-        assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
-        assertSame(XContentParser.Token.FIELD_NAME, parser.nextToken());
-        assertEquals(testAgg.type(), parser.currentName());
-        assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
-        PipelineAggregationBuilder newAgg = aggParsers.pipelineParser(testAgg.getWriteableName(), ParseFieldMatcher.STRICT)
-                .parse(testAgg.getName(), parseContext);
-        assertSame(XContentParser.Token.END_OBJECT, parser.currentToken());
-        assertSame(XContentParser.Token.END_OBJECT, parser.nextToken());
-        assertSame(XContentParser.Token.END_OBJECT, parser.nextToken());
-        assertNull(parser.nextToken());
-        assertNotNull(newAgg);
+        PipelineAggregationBuilder newAgg = parse(parser);
         assertNotSame(newAgg, testAgg);
         assertEquals(testAgg, newAgg);
         assertEquals(testAgg.hashCode(), newAgg.hashCode());
     }
 
+    protected PipelineAggregationBuilder parse(XContentParser parser) throws IOException {
+        QueryParseContext parseContext = new QueryParseContext(parser);
+        assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
+        AggregatorFactories.Builder parsed = AggregatorFactories.parseAggregators(parseContext);
+        assertThat(parsed.getAggregatorFactories(), hasSize(0));
+        assertThat(parsed.getPipelineAggregatorFactories(), hasSize(1));
+        PipelineAggregationBuilder newAgg = parsed.getPipelineAggregatorFactories().get(0);
+        assertNull(parser.nextToken());
+        assertNotNull(newAgg);
+        return newAgg;
+    }
+
     /**
      * Test serialization and deserialization of the test AggregatorFactory.
      */
-
     public void testSerialization() throws IOException {
         AF testAgg = createTestAggregatorFactory();
         try (BytesStreamOutput output = new BytesStreamOutput()) {
@@ -158,28 +146,9 @@ public void testSerialization() throws IOException {
 
 
     public void testEqualsAndHashcode() throws IOException {
-        AF firstAgg = createTestAggregatorFactory();
-        assertFalse("aggregation is equal to null", firstAgg.equals(null));
-        assertFalse("aggregation is equal to incompatible type", firstAgg.equals(""));
-        assertTrue("aggregation is not equal to self", firstAgg.equals(firstAgg));
-        assertThat("same aggregation's hashcode returns different values if called multiple times", firstAgg.hashCode(),
-                equalTo(firstAgg.hashCode()));
-
-        AF secondQuery = copyAggregation(firstAgg);
-        assertTrue("aggregation is not equal to self", secondQuery.equals(secondQuery));
-        assertTrue("aggregation is not equal to its copy", firstAgg.equals(secondQuery));
-        assertTrue("equals is not symmetric", secondQuery.equals(firstAgg));
-        assertThat("aggregation copy's hashcode is different from original hashcode", secondQuery.hashCode(), equalTo(firstAgg.hashCode()));
-
-        AF thirdQuery = copyAggregation(secondQuery);
-        assertTrue("aggregation is not equal to self", thirdQuery.equals(thirdQuery));
-        assertTrue("aggregation is not equal to its copy", secondQuery.equals(thirdQuery));
-        assertThat("aggregation copy's hashcode is different from original hashcode", secondQuery.hashCode(),
-                equalTo(thirdQuery.hashCode()));
-        assertTrue("equals is not transitive", firstAgg.equals(thirdQuery));
-        assertThat("aggregation copy's hashcode is different from original hashcode", firstAgg.hashCode(), equalTo(thirdQuery.hashCode()));
-        assertTrue("equals is not symmetric", thirdQuery.equals(secondQuery));
-        assertTrue("equals is not symmetric", thirdQuery.equals(firstAgg));
+        // TODO we only change name and boost, we should extend by any sub-test supplying a "mutate" method that randomly changes one
+        // aspect of the object under test
+        checkEqualsAndHashCode(createTestAggregatorFactory(), this::copyAggregation);
     }
 
     // we use the streaming infra to create a copy of the query provided as
@@ -225,4 +194,9 @@ public String randomNumericField() {
                 return INT_FIELD_NAME;
         }
     }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return xContentRegistry;
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/CombiIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/CombiIT.java
index f72ecdb8b1c55..82666d9faa205 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/CombiIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/CombiIT.java
@@ -30,7 +30,7 @@
 import org.elasticsearch.test.ESIntegTestCase;
 import org.hamcrest.Matchers;
 
-import java.util.Collection;
+import java.util.List;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram;
@@ -98,7 +98,7 @@ public void testMultipleAggsOnSameField_WithDifferentRequiredValueSourceType() t
 
         Terms terms = aggs.get("values");
         assertNotNull(terms);
-        Collection<Terms.Bucket> buckets = terms.getBuckets();
+        List<? extends Terms.Bucket> buckets = terms.getBuckets();
         assertThat(buckets.size(), equalTo(values.size()));
         for (Terms.Bucket bucket : buckets) {
             values.remove(((Number) bucket.getKey()).intValue());
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/EquivalenceIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/EquivalenceIT.java
index 3f03d39f11606..6c08c1697244f 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/EquivalenceIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/EquivalenceIT.java
@@ -30,7 +30,7 @@
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.MockScriptPlugin;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode;
 import org.elasticsearch.search.aggregations.bucket.filter.Filter;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
@@ -383,7 +383,7 @@ public void testDuelTermsHistogram() throws Exception {
                         terms("terms")
                                 .field("values")
                                 .collectMode(randomFrom(SubAggCollectionMode.values()))
-                                .script(new Script("floor(_value / interval)", ScriptType.INLINE, CustomScriptPlugin.NAME, params))
+                                .script(new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "floor(_value / interval)", params))
                                 .size(maxNumTerms))
                 .addAggregation(
                         histogram("histo")
@@ -485,10 +485,10 @@ public void testReduce() throws Exception {
     }
 
     private void assertEquals(Terms t1, Terms t2) {
-        List<Terms.Bucket> t1Buckets = t1.getBuckets();
-        List<Terms.Bucket> t2Buckets = t1.getBuckets();
+        List<? extends Terms.Bucket> t1Buckets = t1.getBuckets();
+        List<? extends Terms.Bucket> t2Buckets = t1.getBuckets();
         assertEquals(t1Buckets.size(), t2Buckets.size());
-        for (Iterator<Terms.Bucket> it1 = t1Buckets.iterator(), it2 = t2Buckets.iterator(); it1.hasNext(); ) {
+        for (Iterator<? extends Terms.Bucket> it1 = t1Buckets.iterator(), it2 = t2Buckets.iterator(); it1.hasNext(); ) {
             final Terms.Bucket b1 = it1.next();
             final Terms.Bucket b2 = it2.next();
             assertEquals(b1.getDocCount(), b2.getDocCount());
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/FiltersAggsRewriteIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/FiltersAggsRewriteIT.java
new file mode 100644
index 0000000000000..a3836b2fce046
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/FiltersAggsRewriteIT.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations;
+
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.query.WrapperQueryBuilder;
+import org.elasticsearch.search.aggregations.bucket.filters.FiltersAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.filters.FiltersAggregator;
+import org.elasticsearch.search.aggregations.bucket.filters.InternalFilters;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+
+import java.io.IOException;
+
+public class FiltersAggsRewriteIT extends ESSingleNodeTestCase {
+
+    public void testWrapperQueryIsRewritten() throws IOException {
+        createIndex("test", Settings.EMPTY, "test", "title", "type=text");
+        client().prepareIndex("test", "test", "1").setSource("title", "foo bar baz").get();
+        client().prepareIndex("test", "test", "2").setSource("title", "foo foo foo").get();
+        client().prepareIndex("test", "test", "3").setSource("title", "bar baz bax").get();
+        client().admin().indices().prepareRefresh("test").get();
+
+        XContentType xContentType = randomFrom(XContentType.values());
+        BytesReference bytesReference;
+        try (XContentBuilder builder = XContentFactory.contentBuilder(xContentType)) {
+            builder.startObject();
+            {
+                builder.startObject("terms");
+                {
+                    builder.array("title", "foo");
+                }
+                builder.endObject();
+            }
+            builder.endObject();
+            bytesReference = builder.bytes();
+        }
+        FiltersAggregationBuilder builder = new FiltersAggregationBuilder("titles", new FiltersAggregator.KeyedFilter("titleterms",
+                new WrapperQueryBuilder(bytesReference)));
+        SearchResponse searchResponse = client().prepareSearch("test").setSize(0).addAggregation(builder).get();
+        assertEquals(3, searchResponse.getHits().getTotalHits());
+        InternalFilters filters = searchResponse.getAggregations().get("titles");
+        assertEquals(1, filters.getBuckets().size());
+        assertEquals(2, filters.getBuckets().get(0).getDocCount());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregationTestCase.java
new file mode 100644
index 0000000000000..9ae55a66b250c
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregationTestCase.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations;
+
+import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.test.InternalAggregationTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Supplier;
+
+import static java.util.Collections.emptyList;
+import static java.util.Collections.emptyMap;
+
+public abstract class InternalMultiBucketAggregationTestCase<T extends InternalAggregation & MultiBucketsAggregation>
+        extends InternalAggregationTestCase<T> {
+
+    private static final int DEFAULT_MAX_NUMBER_OF_BUCKETS = 10;
+
+    Supplier<InternalAggregations> subAggregationsSupplier;
+    int maxNumberOfBuckets = DEFAULT_MAX_NUMBER_OF_BUCKETS;
+
+    protected int randomNumberOfBuckets() {
+        return randomIntBetween(minNumberOfBuckets(), maxNumberOfBuckets());
+    }
+
+    protected int minNumberOfBuckets() {
+        return 0;
+    }
+
+    protected int maxNumberOfBuckets() {
+        return maxNumberOfBuckets;
+    }
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        if (randomBoolean()) {
+            subAggregationsSupplier = () -> InternalAggregations.EMPTY;
+        } else {
+            subAggregationsSupplier = () -> {
+                final int numAggregations = randomIntBetween(1, 3);
+                List<InternalAggregation> aggs = new ArrayList<>();
+                for (int i = 0; i < numAggregations; i++) {
+                    aggs.add(createTestInstance(randomAlphaOfLength(5), emptyList(), emptyMap(), InternalAggregations.EMPTY));
+                }
+                return new InternalAggregations(aggs);
+            };
+        }
+    }
+
+    @Override
+    protected final T createTestInstance(String name, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
+        T instance = createTestInstance(name, pipelineAggregators, metaData, subAggregationsSupplier.get());
+        assert instance.getBuckets().size() <= maxNumberOfBuckets() :
+                "Maximum number of buckets exceeded for " + instance.getClass().getSimpleName() + " aggregation";
+        return instance;
+    }
+
+    protected abstract T createTestInstance(String name, List<PipelineAggregator> pipelineAggregators,
+                                            Map<String, Object> metaData, InternalAggregations aggregations);
+
+    protected abstract Class<? extends ParsedMultiBucketAggregation> implementationClass();
+
+    @Override
+    protected final void assertFromXContent(T aggregation, ParsedAggregation parsedAggregation) {
+        assertMultiBucketsAggregations(aggregation, parsedAggregation, false);
+    }
+
+    public void testIterators() throws IOException {
+        final T aggregation = createTestInstance();
+        assertMultiBucketsAggregations(aggregation, parseAndAssert(aggregation, false, false), true);
+    }
+
+    private void assertMultiBucketsAggregations(Aggregation expected, Aggregation actual, boolean checkOrder) {
+        assertTrue(expected instanceof MultiBucketsAggregation);
+        MultiBucketsAggregation expectedMultiBucketsAggregation = (MultiBucketsAggregation) expected;
+
+        assertTrue(actual instanceof MultiBucketsAggregation);
+        MultiBucketsAggregation actualMultiBucketsAggregation = (MultiBucketsAggregation) actual;
+
+        assertMultiBucketsAggregation(expectedMultiBucketsAggregation, actualMultiBucketsAggregation, checkOrder);
+
+        List<? extends MultiBucketsAggregation.Bucket> expectedBuckets = expectedMultiBucketsAggregation.getBuckets();
+        List<? extends MultiBucketsAggregation.Bucket> actualBuckets = actualMultiBucketsAggregation.getBuckets();
+        assertEquals(expectedBuckets.size(), actualBuckets.size());
+
+        if (checkOrder) {
+            Iterator<? extends MultiBucketsAggregation.Bucket> expectedIt = expectedBuckets.iterator();
+            Iterator<? extends MultiBucketsAggregation.Bucket> actualIt = actualBuckets.iterator();
+            while (expectedIt.hasNext()) {
+                MultiBucketsAggregation.Bucket expectedBucket = expectedIt.next();
+                MultiBucketsAggregation.Bucket actualBucket = actualIt.next();
+                assertBucket(expectedBucket, actualBucket, true);
+            }
+        } else {
+            for (MultiBucketsAggregation.Bucket expectedBucket : expectedBuckets) {
+                final Object expectedKey = expectedBucket.getKey();
+                boolean found = false;
+
+                for (MultiBucketsAggregation.Bucket actualBucket : actualBuckets) {
+                    final Object actualKey = actualBucket.getKey();
+                    if ((actualKey != null && actualKey.equals(expectedKey)) || (actualKey == null && expectedKey == null)) {
+                        found = true;
+                        assertBucket(expectedBucket, actualBucket, false);
+                        break;
+                    }
+                }
+                assertTrue("Failed to find bucket with key [" + expectedBucket.getKey() + "]", found);
+            }
+        }
+    }
+
+    protected void assertMultiBucketsAggregation(MultiBucketsAggregation expected, MultiBucketsAggregation actual, boolean checkOrder) {
+        Class<? extends ParsedMultiBucketAggregation> parsedClass = implementationClass();
+        assertNotNull("Parsed aggregation class must not be null", parsedClass);
+        assertTrue(parsedClass.isInstance(actual));
+
+        assertTrue(expected instanceof InternalAggregation);
+        assertEquals(expected.getName(), actual.getName());
+        assertEquals(expected.getMetaData(), actual.getMetaData());
+        assertEquals(expected.getType(), actual.getType());
+    }
+
+    protected void assertBucket(MultiBucketsAggregation.Bucket expected, MultiBucketsAggregation.Bucket actual, boolean checkOrder) {
+        assertTrue(expected instanceof InternalMultiBucketAggregation.InternalBucket);
+        assertTrue(actual instanceof ParsedMultiBucketAggregation.ParsedBucket);
+
+        assertEquals(expected.getKey(), actual.getKey());
+        assertEquals(expected.getKeyAsString(), actual.getKeyAsString());
+        assertEquals(expected.getDocCount(), actual.getDocCount());
+
+        Aggregations expectedAggregations = expected.getAggregations();
+        Aggregations actualAggregations = actual.getAggregations();
+        assertEquals(expectedAggregations.asList().size(), actualAggregations.asList().size());
+
+        if (checkOrder) {
+            Iterator<Aggregation> expectedIt = expectedAggregations.iterator();
+            Iterator<Aggregation> actualIt = actualAggregations.iterator();
+
+            while (expectedIt.hasNext()) {
+                Aggregation expectedAggregation = expectedIt.next();
+                Aggregation actualAggregation = actualIt.next();
+                assertMultiBucketsAggregations(expectedAggregation, actualAggregation, true);
+            }
+        } else {
+            for (Aggregation expectedAggregation : expectedAggregations) {
+                Aggregation actualAggregation = actualAggregations.get(expectedAggregation.getName());
+                assertNotNull(actualAggregation);
+                assertMultiBucketsAggregations(expectedAggregation, actualAggregation, false);
+            }
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/MultiBucketAggregatorWrapperTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/MultiBucketAggregatorWrapperTests.java
new file mode 100644
index 0000000000000..0a83b2ec5c794
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/MultiBucketAggregatorWrapperTests.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations;
+
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.memory.MemoryIndex;
+import org.apache.lucene.search.Scorer;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.MockBigArrays;
+import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
+import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.same;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+
+public class MultiBucketAggregatorWrapperTests extends ESTestCase {
+
+    public void testNoNullScorerIsDelegated() throws Exception {
+        LeafReaderContext leafReaderContext = MemoryIndex.fromDocument(Collections.emptyList(), new MockAnalyzer(random()))
+                .createSearcher().getIndexReader().leaves().get(0);
+        BigArrays bigArrays = new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService());
+        SearchContext searchContext = mock(SearchContext.class);
+        when(searchContext.bigArrays()).thenReturn(bigArrays);
+
+        Aggregator aggregator = mock(Aggregator.class);
+        AggregatorFactory aggregatorFactory = new TestAggregatorFactory(searchContext, aggregator);
+        LeafBucketCollector wrappedCollector = mock(LeafBucketCollector.class);
+        when(aggregator.getLeafCollector(leafReaderContext)).thenReturn(wrappedCollector);
+        Aggregator wrapper = AggregatorFactory.asMultiBucketAggregator(aggregatorFactory, searchContext, null);
+
+        LeafBucketCollector collector = wrapper.getLeafCollector(leafReaderContext);
+
+        collector.collect(0, 0);
+        // setScorer should not be invoked as it has not been set
+        // Only collect should be invoked:
+        verify(wrappedCollector).collect(0, 0);
+        verifyNoMoreInteractions(wrappedCollector);
+
+        reset(wrappedCollector);
+        Scorer scorer = mock(Scorer.class);
+        collector.setScorer(scorer);
+        collector.collect(0, 1);
+        verify(wrappedCollector).setScorer(same(scorer));
+        verify(wrappedCollector).collect(0, 0);
+        verifyNoMoreInteractions(wrappedCollector);
+        wrapper.close();
+    }
+
+    static class TestAggregatorFactory extends AggregatorFactory {
+
+        private final Aggregator aggregator;
+
+        TestAggregatorFactory(SearchContext context, Aggregator aggregator) throws IOException {
+            super("_name", context, null, new AggregatorFactories.Builder(), Collections.emptyMap());
+            this.aggregator = aggregator;
+        }
+
+        @Override
+        protected Aggregator createInternal(Aggregator parent, boolean collectsFromSingleBucket, List list,
+                                            Map metaData) throws IOException {
+            return aggregator;
+        }
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/AdjacencyMatrixIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/AdjacencyMatrixIT.java
new file mode 100644
index 0000000000000..81dce8002e84d
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/AdjacencyMatrixIT.java
@@ -0,0 +1,385 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.action.search.SearchPhaseExecutionException;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.bucket.adjacency.AdjacencyMatrix;
+import org.elasticsearch.search.aggregations.bucket.adjacency.AdjacencyMatrix.Bucket;
+import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
+import org.elasticsearch.search.aggregations.metrics.avg.Avg;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.hamcrest.Matchers;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
+import static org.elasticsearch.index.query.QueryBuilders.termQuery;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.adjacencyMatrix;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.avg;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.core.IsNull.notNullValue;
+
+@ESIntegTestCase.SuiteScopeTestCase
+public class AdjacencyMatrixIT extends ESIntegTestCase {
+
+    static int numDocs, numSingleTag1Docs, numSingleTag2Docs, numTag1Docs, numTag2Docs, numMultiTagDocs;
+    static final int MAX_NUM_FILTERS = 3;
+
+    @Override
+    public void setupSuiteScopeCluster() throws Exception {
+        createIndex("idx");
+        createIndex("idx2");
+        assertAcked(client().admin().indices().prepareUpdateSettings("idx")
+                .setSettings(
+                        Settings.builder().put(IndexSettings.MAX_ADJACENCY_MATRIX_FILTERS_SETTING.getKey(), MAX_NUM_FILTERS))
+                .get());
+
+        numDocs = randomIntBetween(5, 20);
+        numTag1Docs = randomIntBetween(1, numDocs - 1);
+        numTag2Docs = randomIntBetween(1, numDocs - numTag1Docs);
+        List<IndexRequestBuilder> builders = new ArrayList<>();
+        for (int i = 0; i < numTag1Docs; i++) {
+            numSingleTag1Docs++;
+            XContentBuilder source = jsonBuilder().startObject().field("value", i + 1).field("tag", "tag1").endObject();
+            builders.add(client().prepareIndex("idx", "type", "" + i).setSource(source));
+            if (randomBoolean()) {
+                // randomly index the document twice so that we have deleted
+                // docs that match the filter
+                builders.add(client().prepareIndex("idx", "type", "" + i).setSource(source));
+            }
+        }
+        for (int i = numTag1Docs; i < (numTag1Docs + numTag2Docs); i++) {
+            numSingleTag2Docs++;
+            XContentBuilder source = jsonBuilder().startObject().field("value", i + 1).field("tag", "tag2").endObject();
+            builders.add(client().prepareIndex("idx", "type", "" + i).setSource(source));
+            if (randomBoolean()) {
+                builders.add(client().prepareIndex("idx", "type", "" + i).setSource(source));
+            }
+        }
+        for (int i = numTag1Docs + numTag2Docs; i < numDocs; i++) {
+            numMultiTagDocs++;
+            numTag1Docs++;
+            numTag2Docs++;
+            XContentBuilder source = jsonBuilder().startObject().field("value", i + 1).array("tag", "tag1", "tag2").endObject();
+            builders.add(client().prepareIndex("idx", "type", "" + i).setSource(source));
+            if (randomBoolean()) {
+                builders.add(client().prepareIndex("idx", "type", "" + i).setSource(source));
+            }
+        }
+        prepareCreate("empty_bucket_idx").addMapping("type", "value", "type=integer").execute().actionGet();
+        for (int i = 0; i < 2; i++) {
+            builders.add(client().prepareIndex("empty_bucket_idx", "type", "" + i)
+                    .setSource(jsonBuilder().startObject().field("value", i * 2).endObject()));
+        }
+        indexRandom(true, builders);
+        ensureSearchable();
+    }
+
+    public void testSimple() throws Exception {
+        SearchResponse response = client().prepareSearch("idx")
+                .addAggregation(adjacencyMatrix("tags",
+                        newMap("tag1", termQuery("tag", "tag1")).add("tag2", termQuery("tag", "tag2"))))
+                .execute().actionGet();
+
+        assertSearchResponse(response);
+
+        AdjacencyMatrix matrix = response.getAggregations().get("tags");
+        assertThat(matrix, notNullValue());
+        assertThat(matrix.getName(), equalTo("tags"));
+
+        int expected = numMultiTagDocs > 0 ? 3 : 2;
+        assertThat(matrix.getBuckets().size(), equalTo(expected));
+
+        AdjacencyMatrix.Bucket bucket = matrix.getBucketByKey("tag1");
+        assertThat(bucket, Matchers.notNullValue());
+        assertThat(bucket.getDocCount(), equalTo((long) numTag1Docs));
+
+        bucket = matrix.getBucketByKey("tag2");
+        assertThat(bucket, Matchers.notNullValue());
+        assertThat(bucket.getDocCount(), equalTo((long) numTag2Docs));
+
+        bucket = matrix.getBucketByKey("tag1&tag2");
+        if (numMultiTagDocs == 0) {
+            assertThat(bucket, Matchers.nullValue());
+        } else {
+            assertThat(bucket, Matchers.notNullValue());
+            assertThat(bucket.getDocCount(), equalTo((long) numMultiTagDocs));
+        }
+
+    }
+
+    public void testCustomSeparator() throws Exception {
+        SearchResponse response = client().prepareSearch("idx")
+                .addAggregation(adjacencyMatrix("tags", "\t",
+                        newMap("tag1", termQuery("tag", "tag1")).add("tag2", termQuery("tag", "tag2"))))
+                .execute().actionGet();
+
+        assertSearchResponse(response);
+
+        AdjacencyMatrix matrix = response.getAggregations().get("tags");
+        assertThat(matrix, notNullValue());
+
+        AdjacencyMatrix.Bucket bucket = matrix.getBucketByKey("tag1\ttag2");
+        if (numMultiTagDocs == 0) {
+            assertThat(bucket, Matchers.nullValue());
+        } else {
+            assertThat(bucket, Matchers.notNullValue());
+            assertThat(bucket.getDocCount(), equalTo((long) numMultiTagDocs));
+        }
+
+    }
+
+
+    // See NullPointer issue when filters are empty:
+    // https://github.com/elastic/elasticsearch/issues/8438
+    public void testEmptyFilterDeclarations() throws Exception {
+        QueryBuilder emptyFilter = new BoolQueryBuilder();
+        SearchResponse response = client().prepareSearch("idx")
+                .addAggregation(adjacencyMatrix("tags",
+                        newMap("all", emptyFilter).add("tag1", termQuery("tag", "tag1"))))
+                .execute().actionGet();
+
+        assertSearchResponse(response);
+
+        AdjacencyMatrix filters = response.getAggregations().get("tags");
+        assertThat(filters, notNullValue());
+        AdjacencyMatrix.Bucket allBucket = filters.getBucketByKey("all");
+        assertThat(allBucket.getDocCount(), equalTo((long) numDocs));
+
+        AdjacencyMatrix.Bucket bucket = filters.getBucketByKey("tag1");
+        assertThat(bucket, Matchers.notNullValue());
+        assertThat(bucket.getDocCount(), equalTo((long) numTag1Docs));
+    }
+
+    public void testWithSubAggregation() throws Exception {
+        BoolQueryBuilder boolQ = new BoolQueryBuilder();
+        boolQ.must(termQuery("tag", "tag1"));
+        boolQ.must(termQuery("tag", "tag2"));
+        SearchResponse response = client().prepareSearch("idx")
+                .addAggregation(
+                        adjacencyMatrix("tags",
+                                newMap("tag1", termQuery("tag", "tag1"))
+                                    .add("tag2", termQuery("tag", "tag2"))
+                                    .add("both", boolQ))
+                               .subAggregation(avg("avg_value").field("value")))
+                .execute().actionGet();
+
+        assertSearchResponse(response);
+
+        AdjacencyMatrix matrix = response.getAggregations().get("tags");
+        assertThat(matrix, notNullValue());
+        assertThat(matrix.getName(), equalTo("tags"));
+
+        int expectedBuckets = 0;
+        if (numTag1Docs > 0) {
+            expectedBuckets ++;
+        }
+        if (numTag2Docs > 0) {
+            expectedBuckets ++;
+        }
+        if (numMultiTagDocs > 0) {
+            // both, both&tag1, both&tag2, tag1&tag2
+            expectedBuckets += 4;
+        }
+
+        assertThat(matrix.getBuckets().size(), equalTo(expectedBuckets));
+        assertThat(((InternalAggregation)matrix).getProperty("_bucket_count"), equalTo(expectedBuckets));
+
+        Object[] propertiesKeys = (Object[]) ((InternalAggregation)matrix).getProperty("_key");
+        Object[] propertiesDocCounts = (Object[]) ((InternalAggregation)matrix).getProperty("_count");
+        Object[] propertiesCounts = (Object[]) ((InternalAggregation)matrix).getProperty("avg_value.value");
+
+        assertEquals(expectedBuckets, propertiesKeys.length);
+        assertEquals(propertiesKeys.length, propertiesDocCounts.length);
+        assertEquals(propertiesKeys.length, propertiesCounts.length);
+
+        for (int i = 0; i < propertiesCounts.length; i++) {
+            AdjacencyMatrix.Bucket bucket = matrix.getBucketByKey(propertiesKeys[i].toString());
+            assertThat(bucket, Matchers.notNullValue());
+            Avg avgValue = bucket.getAggregations().get("avg_value");
+            assertThat(avgValue, notNullValue());
+            assertThat((long) propertiesDocCounts[i], equalTo(bucket.getDocCount()));
+            assertThat((double) propertiesCounts[i], equalTo(avgValue.getValue()));
+        }
+
+        AdjacencyMatrix.Bucket tag1Bucket = matrix.getBucketByKey("tag1");
+        assertThat(tag1Bucket, Matchers.notNullValue());
+        assertThat(tag1Bucket.getDocCount(), equalTo((long) numTag1Docs));
+        long sum = 0;
+        for (int i = 0; i < numSingleTag1Docs; i++) {
+            sum += i + 1;
+        }
+        for (int i = numSingleTag1Docs + numSingleTag2Docs; i < numDocs; i++) {
+            sum += i + 1;
+        }
+        assertThat(tag1Bucket.getAggregations().asList().isEmpty(), is(false));
+        Avg avgBucket1Value = tag1Bucket.getAggregations().get("avg_value");
+        assertThat(avgBucket1Value, notNullValue());
+        assertThat(avgBucket1Value.getName(), equalTo("avg_value"));
+        assertThat(avgBucket1Value.getValue(), equalTo((double) sum / numTag1Docs));
+
+        Bucket tag2Bucket = matrix.getBucketByKey("tag2");
+        assertThat(tag2Bucket, Matchers.notNullValue());
+        assertThat(tag2Bucket.getDocCount(), equalTo((long) numTag2Docs));
+        sum = 0;
+        for (int i = numSingleTag1Docs; i < numDocs; i++) {
+            sum += i + 1;
+        }
+        assertThat(tag2Bucket.getAggregations().asList().isEmpty(), is(false));
+        Avg avgBucket2Value = tag2Bucket.getAggregations().get("avg_value");
+        assertThat(avgBucket2Value, notNullValue());
+        assertThat(avgBucket2Value.getName(), equalTo("avg_value"));
+        assertThat(avgBucket2Value.getValue(), equalTo((double) sum / numTag2Docs));
+
+        // Check intersection buckets are computed correctly by comparing with
+        // ANDed query bucket results
+        Bucket bucketBothQ = matrix.getBucketByKey("both");
+        if (numMultiTagDocs == 0) {
+            // Empty intersections are not returned.
+            assertThat(bucketBothQ, Matchers.nullValue());
+            Bucket bucketIntersectQ = matrix.getBucketByKey("tag1&tag2");
+            assertThat(bucketIntersectQ, Matchers.nullValue());
+            Bucket tag1Both = matrix.getBucketByKey("both&tag1");
+            assertThat(tag1Both, Matchers.nullValue());
+        } else
+        {
+            assertThat(bucketBothQ, Matchers.notNullValue());
+            assertThat(bucketBothQ.getDocCount(), equalTo((long) numMultiTagDocs));
+            Avg avgValueBothQ = bucketBothQ.getAggregations().get("avg_value");
+
+            Bucket bucketIntersectQ = matrix.getBucketByKey("tag1&tag2");
+            assertThat(bucketIntersectQ, Matchers.notNullValue());
+            assertThat(bucketIntersectQ.getDocCount(), equalTo((long) numMultiTagDocs));
+            Avg avgValueIntersectQ = bucketBothQ.getAggregations().get("avg_value");
+            assertThat(avgValueIntersectQ.getValue(), equalTo(avgValueBothQ.getValue()));
+
+            Bucket tag1Both = matrix.getBucketByKey("both&tag1");
+            assertThat(tag1Both, Matchers.notNullValue());
+            assertThat(tag1Both.getDocCount(), equalTo((long) numMultiTagDocs));
+            Avg avgValueTag1BothIntersectQ = tag1Both.getAggregations().get("avg_value");
+            assertThat(avgValueTag1BothIntersectQ.getValue(), equalTo(avgValueBothQ.getValue()));
+        }
+
+
+    }
+
+    public void testTooLargeMatrix() throws Exception{
+
+        // Create more filters than is permitted by index settings.
+        MapBuilder filtersMap = new MapBuilder();
+        for (int i = 0; i <= MAX_NUM_FILTERS; i++) {
+            filtersMap.add("tag" + i, termQuery("tag", "tag" + i));
+        }
+
+        try {
+            client().prepareSearch("idx")
+                .addAggregation(adjacencyMatrix("tags", "\t", filtersMap))
+                .execute().actionGet();
+            fail("SearchPhaseExecutionException should have been thrown");
+        } catch (SearchPhaseExecutionException ex) {
+            assertThat(ex.getCause().getMessage(), containsString("Number of filters is too large"));
+        }
+    }
+
+    public void testAsSubAggregation() {
+        SearchResponse response = client().prepareSearch("idx").addAggregation(histogram("histo").field("value").interval(2L)
+                .subAggregation(adjacencyMatrix("matrix", newMap("all", matchAllQuery())))).get();
+
+        assertSearchResponse(response);
+
+        Histogram histo = response.getAggregations().get("histo");
+        assertThat(histo, notNullValue());
+        assertThat(histo.getBuckets().size(), greaterThanOrEqualTo(1));
+
+        for (Histogram.Bucket bucket : histo.getBuckets()) {
+            AdjacencyMatrix matrix = bucket.getAggregations().get("matrix");
+            assertThat(matrix, notNullValue());
+            assertThat(matrix.getBuckets().size(), equalTo(1));
+            AdjacencyMatrix.Bucket filterBucket = matrix.getBuckets().get(0);
+            assertEquals(bucket.getDocCount(), filterBucket.getDocCount());
+        }
+    }
+
+    public void testWithContextBasedSubAggregation() throws Exception {
+
+        try {
+            client().prepareSearch("idx")
+                    .addAggregation(adjacencyMatrix("tags",
+                            newMap("tag1", termQuery("tag", "tag1")).add("tag2", termQuery("tag", "tag2")))
+                            .subAggregation(avg("avg_value")))
+                    .execute().actionGet();
+
+            fail("expected execution to fail - an attempt to have a context based numeric sub-aggregation, but there is not value source"
+                    + "context which the sub-aggregation can inherit");
+
+        } catch (ElasticsearchException e) {
+            assertThat(e.getMessage(), is("all shards failed"));
+        }
+    }
+
+    public void testEmptyAggregation() throws Exception {
+        SearchResponse searchResponse = client()
+                .prepareSearch("empty_bucket_idx").setQuery(matchAllQuery()).addAggregation(histogram("histo").field("value").interval(1L)
+                        .minDocCount(0).subAggregation(adjacencyMatrix("matrix", newMap("all", matchAllQuery()))))
+                .execute().actionGet();
+
+        assertThat(searchResponse.getHits().getTotalHits(), equalTo(2L));
+        Histogram histo = searchResponse.getAggregations().get("histo");
+        assertThat(histo, Matchers.notNullValue());
+        Histogram.Bucket bucket = histo.getBuckets().get(1);
+        assertThat(bucket, Matchers.notNullValue());
+
+        AdjacencyMatrix matrix = bucket.getAggregations().get("matrix");
+        assertThat(matrix, notNullValue());
+        AdjacencyMatrix.Bucket all = matrix.getBucketByKey("all");
+        assertThat(all, Matchers.nullValue());
+    }
+
+    // Helper methods for building maps of QueryBuilders
+    static MapBuilder newMap(String name, QueryBuilder builder) {
+        return new MapBuilder().add(name, builder);
+    }
+
+    static class MapBuilder extends HashMap<String, QueryBuilder> {
+        public MapBuilder add(String name, QueryBuilder builder) {
+            put(name, builder);
+            return this;
+        }
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/BooleanTermsIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/BooleanTermsIT.java
index e38f33cfa2b3c..cfb290284aeaa 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/BooleanTermsIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/BooleanTermsIT.java
@@ -78,7 +78,7 @@ public void setupSuiteScopeCluster() throws Exception {
             builders[i] = client().prepareIndex("idx", "type").setSource(jsonBuilder()
                     .startObject()
                     .field(SINGLE_VALUED_FIELD_NAME, singleValue)
-                    .field(MULTI_VALUED_FIELD_NAME, multiValue)
+                    .array(MULTI_VALUED_FIELD_NAME, multiValue)
                     .endObject());
         }
         indexRandom(true, builders);
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenIT.java
deleted file mode 100644
index f4234423531c9..0000000000000
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenIT.java
+++ /dev/null
@@ -1,466 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.search.aggregations.bucket;
-
-import org.apache.lucene.search.join.ScoreMode;
-import org.elasticsearch.action.index.IndexRequestBuilder;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.update.UpdateResponse;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.search.SearchHit;
-import org.elasticsearch.search.aggregations.AggregationBuilders;
-import org.elasticsearch.search.aggregations.bucket.children.Children;
-import org.elasticsearch.search.aggregations.bucket.terms.Terms;
-import org.elasticsearch.search.aggregations.metrics.sum.Sum;
-import org.elasticsearch.search.aggregations.metrics.tophits.TopHits;
-import org.elasticsearch.search.sort.SortOrder;
-import org.elasticsearch.test.ESIntegTestCase;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import static org.elasticsearch.index.query.QueryBuilders.hasChildQuery;
-import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
-import static org.elasticsearch.index.query.QueryBuilders.termQuery;
-import static org.elasticsearch.search.aggregations.AggregationBuilders.children;
-import static org.elasticsearch.search.aggregations.AggregationBuilders.sum;
-import static org.elasticsearch.search.aggregations.AggregationBuilders.terms;
-import static org.elasticsearch.search.aggregations.AggregationBuilders.topHits;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.greaterThan;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.sameInstance;
-
-/**
- */
-@ESIntegTestCase.SuiteScopeTestCase
-public class ChildrenIT extends ESIntegTestCase {
-
-    private static final Map<String, Control> categoryToControl = new HashMap<>();
-
-    @Override
-    public void setupSuiteScopeCluster() throws Exception {
-        assertAcked(
-                prepareCreate("test")
-                    .addMapping("article", "category", "type=keyword")
-                    .addMapping("comment", "_parent", "type=article", "commenter", "type=keyword")
-        );
-
-        List<IndexRequestBuilder> requests = new ArrayList<>();
-        String[] uniqueCategories = new String[randomIntBetween(1, 25)];
-        for (int i = 0; i < uniqueCategories.length; i++) {
-            uniqueCategories[i] = Integer.toString(i);
-        }
-        int catIndex = 0;
-
-        int numParentDocs = randomIntBetween(uniqueCategories.length, uniqueCategories.length * 5);
-        for (int i = 0; i < numParentDocs; i++) {
-            String id = Integer.toString(i);
-
-            // TODO: this array is always of length 1, and testChildrenAggs fails if this is changed
-            String[] categories = new String[randomIntBetween(1,1)];
-            for (int j = 0; j < categories.length; j++) {
-                String category = categories[j] = uniqueCategories[catIndex++ % uniqueCategories.length];
-                Control control = categoryToControl.get(category);
-                if (control == null) {
-                    categoryToControl.put(category, control = new Control(category));
-                }
-                control.articleIds.add(id);
-            }
-
-            requests.add(client().prepareIndex("test", "article", id).setCreate(true).setSource("category", categories, "randomized", true));
-        }
-
-        String[] commenters = new String[randomIntBetween(5, 50)];
-        for (int i = 0; i < commenters.length; i++) {
-            commenters[i] = Integer.toString(i);
-        }
-
-        int id = 0;
-        for (Control control : categoryToControl.values()) {
-            for (String articleId : control.articleIds) {
-                int numChildDocsPerParent = randomIntBetween(0, 5);
-                for (int i = 0; i < numChildDocsPerParent; i++) {
-                    String commenter = commenters[id % commenters.length];
-                    String idValue = Integer.toString(id++);
-                    control.commentIds.add(idValue);
-                    Set<String> ids = control.commenterToCommentId.get(commenter);
-                    if (ids == null) {
-                        control.commenterToCommentId.put(commenter, ids = new HashSet<>());
-                    }
-                    ids.add(idValue);
-                    requests.add(client().prepareIndex("test", "comment", idValue).setCreate(true).setParent(articleId).setSource("commenter", commenter));
-                }
-            }
-        }
-
-        requests.add(client().prepareIndex("test", "article", "a").setSource("category", new String[]{"a"}, "randomized", false));
-        requests.add(client().prepareIndex("test", "article", "b").setSource("category", new String[]{"a", "b"}, "randomized", false));
-        requests.add(client().prepareIndex("test", "article", "c").setSource("category", new String[]{"a", "b", "c"}, "randomized", false));
-        requests.add(client().prepareIndex("test", "article", "d").setSource("category", new String[]{"c"}, "randomized", false));
-        requests.add(client().prepareIndex("test", "comment", "a").setParent("a").setSource("{}"));
-        requests.add(client().prepareIndex("test", "comment", "c").setParent("c").setSource("{}"));
-
-        indexRandom(true, requests);
-        ensureSearchable("test");
-    }
-
-    public void testChildrenAggs() throws Exception {
-        SearchResponse searchResponse = client().prepareSearch("test")
-                .setQuery(matchQuery("randomized", true))
-                .addAggregation(
-                        terms("category").field("category").size(10000).subAggregation(children("to_comment", "comment")
-                                .subAggregation(
-                                        terms("commenters").field("commenter").size(10000).subAggregation(
-                                                topHits("top_comments")
-                                        ))
-                        )
-                ).get();
-        assertSearchResponse(searchResponse);
-
-        Terms categoryTerms = searchResponse.getAggregations().get("category");
-        assertThat(categoryTerms.getBuckets().size(), equalTo(categoryToControl.size()));
-        for (Map.Entry<String, Control> entry1 : categoryToControl.entrySet()) {
-            Terms.Bucket categoryBucket = categoryTerms.getBucketByKey(entry1.getKey());
-            assertThat(categoryBucket.getKeyAsString(), equalTo(entry1.getKey()));
-            assertThat(categoryBucket.getDocCount(), equalTo((long) entry1.getValue().articleIds.size()));
-
-            Children childrenBucket = categoryBucket.getAggregations().get("to_comment");
-            assertThat(childrenBucket.getName(), equalTo("to_comment"));
-            assertThat(childrenBucket.getDocCount(), equalTo((long) entry1.getValue().commentIds.size()));
-            assertThat((long) childrenBucket.getProperty("_count"), equalTo((long) entry1.getValue().commentIds.size()));
-
-            Terms commentersTerms = childrenBucket.getAggregations().get("commenters");
-            assertThat((Terms) childrenBucket.getProperty("commenters"), sameInstance(commentersTerms));
-            assertThat(commentersTerms.getBuckets().size(), equalTo(entry1.getValue().commenterToCommentId.size()));
-            for (Map.Entry<String, Set<String>> entry2 : entry1.getValue().commenterToCommentId.entrySet()) {
-                Terms.Bucket commentBucket = commentersTerms.getBucketByKey(entry2.getKey());
-                assertThat(commentBucket.getKeyAsString(), equalTo(entry2.getKey()));
-                assertThat(commentBucket.getDocCount(), equalTo((long) entry2.getValue().size()));
-
-                TopHits topHits = commentBucket.getAggregations().get("top_comments");
-                for (SearchHit searchHit : topHits.getHits().getHits()) {
-                    assertThat(entry2.getValue().contains(searchHit.getId()), is(true));
-                }
-            }
-        }
-    }
-
-    public void testParentWithMultipleBuckets() throws Exception {
-        SearchResponse searchResponse = client().prepareSearch("test")
-                .setQuery(matchQuery("randomized", false))
-                .addAggregation(
-                        terms("category").field("category").size(10000).subAggregation(
-                        children("to_comment", "comment").subAggregation(topHits("top_comments").sort("_uid", SortOrder.ASC))
-                        )
-                ).get();
-        assertSearchResponse(searchResponse);
-
-        Terms categoryTerms = searchResponse.getAggregations().get("category");
-        assertThat(categoryTerms.getBuckets().size(), equalTo(3));
-
-        for (Terms.Bucket bucket : categoryTerms.getBuckets()) {
-            logger.info("bucket={}", bucket.getKey());
-            Children childrenBucket = bucket.getAggregations().get("to_comment");
-            TopHits topHits = childrenBucket.getAggregations().get("top_comments");
-            logger.info("total_hits={}", topHits.getHits().getTotalHits());
-            for (SearchHit searchHit : topHits.getHits()) {
-                logger.info("hit= {} {} {}", searchHit.sortValues()[0], searchHit.getType(), searchHit.getId());
-            }
-        }
-
-        Terms.Bucket categoryBucket = categoryTerms.getBucketByKey("a");
-        assertThat(categoryBucket.getKeyAsString(), equalTo("a"));
-        assertThat(categoryBucket.getDocCount(), equalTo(3L));
-
-        Children childrenBucket = categoryBucket.getAggregations().get("to_comment");
-        assertThat(childrenBucket.getName(), equalTo("to_comment"));
-        assertThat(childrenBucket.getDocCount(), equalTo(2L));
-        TopHits topHits = childrenBucket.getAggregations().get("top_comments");
-        assertThat(topHits.getHits().totalHits(), equalTo(2L));
-        assertThat(topHits.getHits().getAt(0).getId(), equalTo("a"));
-        assertThat(topHits.getHits().getAt(0).getType(), equalTo("comment"));
-        assertThat(topHits.getHits().getAt(1).getId(), equalTo("c"));
-        assertThat(topHits.getHits().getAt(1).getType(), equalTo("comment"));
-
-        categoryBucket = categoryTerms.getBucketByKey("b");
-        assertThat(categoryBucket.getKeyAsString(), equalTo("b"));
-        assertThat(categoryBucket.getDocCount(), equalTo(2L));
-
-        childrenBucket = categoryBucket.getAggregations().get("to_comment");
-        assertThat(childrenBucket.getName(), equalTo("to_comment"));
-        assertThat(childrenBucket.getDocCount(), equalTo(1L));
-        topHits = childrenBucket.getAggregations().get("top_comments");
-        assertThat(topHits.getHits().totalHits(), equalTo(1L));
-        assertThat(topHits.getHits().getAt(0).getId(), equalTo("c"));
-        assertThat(topHits.getHits().getAt(0).getType(), equalTo("comment"));
-
-        categoryBucket = categoryTerms.getBucketByKey("c");
-        assertThat(categoryBucket.getKeyAsString(), equalTo("c"));
-        assertThat(categoryBucket.getDocCount(), equalTo(2L));
-
-        childrenBucket = categoryBucket.getAggregations().get("to_comment");
-        assertThat(childrenBucket.getName(), equalTo("to_comment"));
-        assertThat(childrenBucket.getDocCount(), equalTo(1L));
-        topHits = childrenBucket.getAggregations().get("top_comments");
-        assertThat(topHits.getHits().totalHits(), equalTo(1L));
-        assertThat(topHits.getHits().getAt(0).getId(), equalTo("c"));
-        assertThat(topHits.getHits().getAt(0).getType(), equalTo("comment"));
-    }
-
-    public void testWithDeletes() throws Exception {
-        String indexName = "xyz";
-        assertAcked(
-                prepareCreate(indexName)
-                        .addMapping("parent")
-                        .addMapping("child", "_parent", "type=parent", "count", "type=long")
-        );
-
-        List<IndexRequestBuilder> requests = new ArrayList<>();
-        requests.add(client().prepareIndex(indexName, "parent", "1").setSource("{}"));
-        requests.add(client().prepareIndex(indexName, "child", "0").setParent("1").setSource("count", 1));
-        requests.add(client().prepareIndex(indexName, "child", "1").setParent("1").setSource("count", 1));
-        requests.add(client().prepareIndex(indexName, "child", "2").setParent("1").setSource("count", 1));
-        requests.add(client().prepareIndex(indexName, "child", "3").setParent("1").setSource("count", 1));
-        indexRandom(true, requests);
-
-        for (int i = 0; i < 10; i++) {
-            SearchResponse searchResponse = client().prepareSearch(indexName)
-                    .addAggregation(children("children", "child").subAggregation(sum("counts").field("count")))
-                    .get();
-
-            assertNoFailures(searchResponse);
-            Children children = searchResponse.getAggregations().get("children");
-            assertThat(children.getDocCount(), equalTo(4L));
-
-            Sum count = children.getAggregations().get("counts");
-            assertThat(count.getValue(), equalTo(4.));
-
-            String idToUpdate = Integer.toString(randomInt(3));
-            /*
-             * The whole point of this test is to test these things with deleted
-             * docs in the index so we turn off detect_noop to make sure that
-             * the updates cause that.
-             */
-            UpdateResponse updateResponse = client().prepareUpdate(indexName, "child", idToUpdate)
-                    .setParent("1")
-                    .setDoc("count", 1)
-                    .setDetectNoop(false)
-                    .get();
-            assertThat(updateResponse.getVersion(), greaterThan(1L));
-            refresh();
-        }
-    }
-
-    public void testNonExistingChildType() throws Exception {
-        SearchResponse searchResponse = client().prepareSearch("test")
-                .addAggregation(
-children("non-existing", "xyz")
-                ).get();
-        assertSearchResponse(searchResponse);
-
-        Children children = searchResponse.getAggregations().get("non-existing");
-        assertThat(children.getName(), equalTo("non-existing"));
-        assertThat(children.getDocCount(), equalTo(0L));
-    }
-
-    public void testPostCollection() throws Exception {
-        String indexName = "prodcatalog";
-        String masterType = "masterprod";
-        String childType = "variantsku";
-        assertAcked(
-                prepareCreate(indexName)
-                        .addMapping(masterType, "brand", "type=text", "name", "type=keyword", "material", "type=text")
-                        .addMapping(childType, "_parent", "type=masterprod", "color", "type=keyword", "size", "type=keyword")
-        );
-
-        List<IndexRequestBuilder> requests = new ArrayList<>();
-        requests.add(client().prepareIndex(indexName, masterType, "1").setSource("brand", "Levis", "name", "Style 501", "material", "Denim"));
-        requests.add(client().prepareIndex(indexName, childType, "0").setParent("1").setSource("color", "blue", "size", "32"));
-        requests.add(client().prepareIndex(indexName, childType, "1").setParent("1").setSource("color", "blue", "size", "34"));
-        requests.add(client().prepareIndex(indexName, childType, "2").setParent("1").setSource("color", "blue", "size", "36"));
-        requests.add(client().prepareIndex(indexName, childType, "3").setParent("1").setSource("color", "black", "size", "38"));
-        requests.add(client().prepareIndex(indexName, childType, "4").setParent("1").setSource("color", "black", "size", "40"));
-        requests.add(client().prepareIndex(indexName, childType, "5").setParent("1").setSource("color", "gray", "size", "36"));
-
-        requests.add(client().prepareIndex(indexName, masterType, "2").setSource("brand", "Wrangler", "name", "Regular Cut", "material", "Leather"));
-        requests.add(client().prepareIndex(indexName, childType, "6").setParent("2").setSource("color", "blue", "size", "32"));
-        requests.add(client().prepareIndex(indexName, childType, "7").setParent("2").setSource("color", "blue", "size", "34"));
-        requests.add(client().prepareIndex(indexName, childType, "8").setParent("2").setSource("color", "black", "size", "36"));
-        requests.add(client().prepareIndex(indexName, childType, "9").setParent("2").setSource("color", "black", "size", "38"));
-        requests.add(client().prepareIndex(indexName, childType, "10").setParent("2").setSource("color", "black", "size", "40"));
-        requests.add(client().prepareIndex(indexName, childType, "11").setParent("2").setSource("color", "orange", "size", "36"));
-        requests.add(client().prepareIndex(indexName, childType, "12").setParent("2").setSource("color", "green", "size", "44"));
-        indexRandom(true, requests);
-
-        SearchResponse response = client().prepareSearch(indexName).setTypes(masterType)
-                .setQuery(hasChildQuery(childType, termQuery("color", "orange"), ScoreMode.None))
-.addAggregation(children("my-refinements", childType)
-                                .subAggregation(terms("my-colors").field("color"))
-                                .subAggregation(terms("my-sizes").field("size"))
-                ).get();
-        assertNoFailures(response);
-        assertHitCount(response, 1);
-
-        Children childrenAgg = response.getAggregations().get("my-refinements");
-        assertThat(childrenAgg.getDocCount(), equalTo(7L));
-
-        Terms termsAgg = childrenAgg.getAggregations().get("my-colors");
-        assertThat(termsAgg.getBuckets().size(), equalTo(4));
-        assertThat(termsAgg.getBucketByKey("black").getDocCount(), equalTo(3L));
-        assertThat(termsAgg.getBucketByKey("blue").getDocCount(), equalTo(2L));
-        assertThat(termsAgg.getBucketByKey("green").getDocCount(), equalTo(1L));
-        assertThat(termsAgg.getBucketByKey("orange").getDocCount(), equalTo(1L));
-
-        termsAgg = childrenAgg.getAggregations().get("my-sizes");
-        assertThat(termsAgg.getBuckets().size(), equalTo(6));
-        assertThat(termsAgg.getBucketByKey("36").getDocCount(), equalTo(2L));
-        assertThat(termsAgg.getBucketByKey("32").getDocCount(), equalTo(1L));
-        assertThat(termsAgg.getBucketByKey("34").getDocCount(), equalTo(1L));
-        assertThat(termsAgg.getBucketByKey("38").getDocCount(), equalTo(1L));
-        assertThat(termsAgg.getBucketByKey("40").getDocCount(), equalTo(1L));
-        assertThat(termsAgg.getBucketByKey("44").getDocCount(), equalTo(1L));
-    }
-
-    public void testHierarchicalChildrenAggs() {
-        String indexName = "geo";
-        String grandParentType = "continent";
-        String parentType = "country";
-        String childType = "city";
-        assertAcked(
-                prepareCreate(indexName)
-                        .setSettings(Settings.builder()
-                                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
-                                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
-                        )
-                        .addMapping(grandParentType, "name", "type=keyword")
-                        .addMapping(parentType, "_parent", "type=" + grandParentType)
-                        .addMapping(childType, "_parent", "type=" + parentType)
-        );
-
-        client().prepareIndex(indexName, grandParentType, "1").setSource("name", "europe").get();
-        client().prepareIndex(indexName, parentType, "2").setParent("1").setSource("name", "belgium").get();
-        client().prepareIndex(indexName, childType, "3").setParent("2").setRouting("1").setSource("name", "brussels").get();
-        refresh();
-
-        SearchResponse response = client().prepareSearch(indexName)
-                .setQuery(matchQuery("name", "europe"))
-                .addAggregation(
-                children(parentType, parentType).subAggregation(children(childType, childType).subAggregation(
-                                        terms("name").field("name")
-                                )
-                        )
-                )
-                .get();
-        assertNoFailures(response);
-        assertHitCount(response, 1);
-
-        Children children = response.getAggregations().get(parentType);
-        assertThat(children.getName(), equalTo(parentType));
-        assertThat(children.getDocCount(), equalTo(1L));
-        children = children.getAggregations().get(childType);
-        assertThat(children.getName(), equalTo(childType));
-        assertThat(children.getDocCount(), equalTo(1L));
-        Terms terms = children.getAggregations().get("name");
-        assertThat(terms.getBuckets().size(), equalTo(1));
-        assertThat(terms.getBuckets().get(0).getKey().toString(), equalTo("brussels"));
-        assertThat(terms.getBuckets().get(0).getDocCount(), equalTo(1L));
-    }
-
-    public void testPostCollectAllLeafReaders() throws Exception {
-        // The 'towns' and 'parent_names' aggs operate on parent docs and if child docs are in different segments we need
-        // to ensure those segments which child docs are also evaluated to in the post collect phase.
-
-        // Before we only evaluated segments that yielded matches in 'towns' and 'parent_names' aggs, which caused
-        // us to miss to evaluate child docs in segments we didn't have parent matches for.
-
-        assertAcked(
-            prepareCreate("index")
-                .addMapping("parentType", "name", "type=keyword", "town", "type=keyword")
-                .addMapping("childType", "_parent", "type=parentType", "name", "type=keyword", "age", "type=integer")
-        );
-        List<IndexRequestBuilder> requests = new ArrayList<>();
-        requests.add(client().prepareIndex("index", "parentType", "1").setSource("name", "Bob", "town", "Memphis"));
-        requests.add(client().prepareIndex("index", "parentType", "2").setSource("name", "Alice", "town", "Chicago"));
-        requests.add(client().prepareIndex("index", "parentType", "3").setSource("name", "Bill", "town", "Chicago"));
-        requests.add(client().prepareIndex("index", "childType", "1").setSource("name", "Jill", "age", 5).setParent("1"));
-        requests.add(client().prepareIndex("index", "childType", "2").setSource("name", "Joey", "age", 3).setParent("1"));
-        requests.add(client().prepareIndex("index", "childType", "3").setSource("name", "John", "age", 2).setParent("2"));
-        requests.add(client().prepareIndex("index", "childType", "4").setSource("name", "Betty", "age", 6).setParent("3"));
-        requests.add(client().prepareIndex("index", "childType", "5").setSource("name", "Dan", "age", 1).setParent("3"));
-        indexRandom(true, requests);
-
-        SearchResponse response = client().prepareSearch("index")
-            .setSize(0)
-            .addAggregation(AggregationBuilders.terms("towns").field("town")
-                .subAggregation(AggregationBuilders.terms("parent_names").field("name")
-.subAggregation(AggregationBuilders.children("child_docs", "childType"))
-                )
-            )
-            .get();
-
-        Terms towns = response.getAggregations().get("towns");
-        assertThat(towns.getBuckets().size(), equalTo(2));
-        assertThat(towns.getBuckets().get(0).getKeyAsString(), equalTo("Chicago"));
-        assertThat(towns.getBuckets().get(0).getDocCount(), equalTo(2L));
-
-        Terms parents = towns.getBuckets().get(0).getAggregations().get("parent_names");
-        assertThat(parents.getBuckets().size(), equalTo(2));
-        assertThat(parents.getBuckets().get(0).getKeyAsString(), equalTo("Alice"));
-        assertThat(parents.getBuckets().get(0).getDocCount(), equalTo(1L));
-        Children children = parents.getBuckets().get(0).getAggregations().get("child_docs");
-        assertThat(children.getDocCount(), equalTo(1L));
-
-        assertThat(parents.getBuckets().get(1).getKeyAsString(), equalTo("Bill"));
-        assertThat(parents.getBuckets().get(1).getDocCount(), equalTo(1L));
-        children = parents.getBuckets().get(1).getAggregations().get("child_docs");
-        assertThat(children.getDocCount(), equalTo(2L));
-
-        assertThat(towns.getBuckets().get(1).getKeyAsString(), equalTo("Memphis"));
-        assertThat(towns.getBuckets().get(1).getDocCount(), equalTo(1L));
-        parents = towns.getBuckets().get(1).getAggregations().get("parent_names");
-        assertThat(parents.getBuckets().size(), equalTo(1));
-        assertThat(parents.getBuckets().get(0).getKeyAsString(), equalTo("Bob"));
-        assertThat(parents.getBuckets().get(0).getDocCount(), equalTo(1L));
-        children = parents.getBuckets().get(0).getAggregations().get("child_docs");
-        assertThat(children.getDocCount(), equalTo(2L));
-    }
-
-    private static final class Control {
-
-        final String category;
-        final Set<String> articleIds = new HashSet<>();
-        final Set<String> commentIds = new HashSet<>();
-        final Map<String, Set<String>> commenterToCommentId = new HashMap<>();
-
-        private Control(String category) {
-            this.category = category;
-        }
-    }
-
-}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenTests.java
deleted file mode 100644
index c7844f29d05d4..0000000000000
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenTests.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.aggregations.bucket;
-
-import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
-import org.elasticsearch.search.aggregations.bucket.children.ChildrenAggregationBuilder;
-
-public class ChildrenTests extends BaseAggregationTestCase<ChildrenAggregationBuilder> {
-
-    @Override
-    protected ChildrenAggregationBuilder createTestAggregatorBuilder() {
-        String name = randomAsciiOfLengthBetween(3, 20);
-        String childType = randomAsciiOfLengthBetween(5, 40);
-        ChildrenAggregationBuilder factory = new ChildrenAggregationBuilder(name, childType);
-        return factory;
-    }
-
-}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java
index 66dd387623aeb..56e48e4fa0deb 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java
@@ -28,7 +28,8 @@
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.script.ScriptType;
+import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.bucket.DateScriptMocks.DateScriptsMockPlugin;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
 import org.elasticsearch.search.aggregations.bucket.histogram.ExtendedBounds;
@@ -50,7 +51,6 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 
@@ -380,10 +380,10 @@ public void testSingleValuedFieldWithSubAggregation() throws Exception {
         assertThat(histo.getName(), equalTo("histo"));
         List<? extends Bucket> buckets = histo.getBuckets();
         assertThat(buckets.size(), equalTo(3));
-        assertThat(histo.getProperty("_bucket_count"), equalTo(3));
-        Object[] propertiesKeys = (Object[]) histo.getProperty("_key");
-        Object[] propertiesDocCounts = (Object[]) histo.getProperty("_count");
-        Object[] propertiesCounts = (Object[]) histo.getProperty("sum.value");
+        assertThat(((InternalAggregation)histo).getProperty("_bucket_count"), equalTo(3));
+        Object[] propertiesKeys = (Object[]) ((InternalAggregation)histo).getProperty("_key");
+        Object[] propertiesDocCounts = (Object[]) ((InternalAggregation)histo).getProperty("_count");
+        Object[] propertiesCounts = (Object[]) ((InternalAggregation)histo).getProperty("sum.value");
 
         DateTime key = new DateTime(2012, 1, 1, 0, 0, DateTimeZone.UTC);
         Histogram.Bucket bucket = buckets.get(0);
@@ -500,7 +500,7 @@ public void testSingleValuedFieldWithValueScript() throws Exception {
         SearchResponse response = client().prepareSearch("idx")
                 .addAggregation(dateHistogram("histo")
                         .field("date")
-                        .script(new Script(DateScriptMocks.PlusOneMonthScript.NAME, ScriptType.INLINE, "native", params))
+                        .script(new Script(ScriptType.INLINE, "native", DateScriptMocks.PlusOneMonthScript.NAME, params))
                         .dateHistogramInterval(DateHistogramInterval.MONTH)).execute().actionGet();
 
         assertSearchResponse(response);
@@ -636,7 +636,7 @@ public void testMultiValuedFieldWithValueScript() throws Exception {
         SearchResponse response = client().prepareSearch("idx")
                 .addAggregation(dateHistogram("histo")
                         .field("dates")
-                        .script(new Script(DateScriptMocks.PlusOneMonthScript.NAME, ScriptType.INLINE, "native", params))
+                        .script(new Script(ScriptType.INLINE, "native", DateScriptMocks.PlusOneMonthScript.NAME, params))
                         .dateHistogramInterval(DateHistogramInterval.MONTH)).execute().actionGet();
 
         assertSearchResponse(response);
@@ -688,8 +688,8 @@ public void testScriptSingleValue() throws Exception {
         Map<String, Object> params = new HashMap<>();
         params.put("fieldname", "date");
         SearchResponse response = client().prepareSearch("idx")
-                .addAggregation(dateHistogram("histo").script(new Script(DateScriptMocks.ExtractFieldScript.NAME,
-                        ScriptType.INLINE, "native", params)).dateHistogramInterval(DateHistogramInterval.MONTH))
+                .addAggregation(dateHistogram("histo").script(new Script(ScriptType.INLINE, "native", DateScriptMocks.ExtractFieldScript.NAME,
+                    params)).dateHistogramInterval(DateHistogramInterval.MONTH))
                 .execute().actionGet();
 
         assertSearchResponse(response);
@@ -726,8 +726,8 @@ public void testScriptMultiValued() throws Exception {
         Map<String, Object> params = new HashMap<>();
         params.put("fieldname", "dates");
         SearchResponse response = client().prepareSearch("idx")
-                .addAggregation(dateHistogram("histo").script(new Script(DateScriptMocks.ExtractFieldScript.NAME,
-                        ScriptType.INLINE, "native", params)).dateHistogramInterval(DateHistogramInterval.MONTH))
+                .addAggregation(dateHistogram("histo").script(new Script(ScriptType.INLINE, "native", DateScriptMocks.ExtractFieldScript.NAME,
+                    params)).dateHistogramInterval(DateHistogramInterval.MONTH))
                 .execute().actionGet();
 
         assertSearchResponse(response);
@@ -1009,20 +1009,13 @@ public void testSingleValueFieldWithExtendedBoundsTimezone() throws Exception {
 
         DateMathParser parser = new DateMathParser(Joda.getStrictStandardDateFormatter());
 
-        final Callable<Long> callable = new Callable<Long>() {
-            @Override
-            public Long call() throws Exception {
-                return System.currentTimeMillis();
-            }
-        };
-
         // we pick a random timezone offset of +12/-12 hours and insert two documents
         // one at 00:00 in that time zone and one at 12:00
         List<IndexRequestBuilder> builders = new ArrayList<>();
         int timeZoneHourOffset = randomIntBetween(-12, 12);
         DateTimeZone timezone = DateTimeZone.forOffsetHours(timeZoneHourOffset);
-        DateTime timeZoneStartToday = new DateTime(parser.parse("now/d", callable, false, timezone), DateTimeZone.UTC);
-        DateTime timeZoneNoonToday = new DateTime(parser.parse("now/d+12h", callable, false, timezone), DateTimeZone.UTC);
+        DateTime timeZoneStartToday = new DateTime(parser.parse("now/d", System::currentTimeMillis, false, timezone), DateTimeZone.UTC);
+        DateTime timeZoneNoonToday = new DateTime(parser.parse("now/d+12h", System::currentTimeMillis, false, timezone), DateTimeZone.UTC);
         builders.add(indexDoc(index, timeZoneStartToday, 1));
         builders.add(indexDoc(index, timeZoneNoonToday, 2));
         indexRandom(true, builders);
@@ -1057,7 +1050,61 @@ public Long call() throws Exception {
                 assertThat(bucket.getDocCount(), equalTo(0L));
             }
         }
-        internalCluster().wipeIndices("test12278");
+        internalCluster().wipeIndices(index);
+    }
+
+    /**
+     * Test date histogram aggregation with day interval, offset and
+     * extended bounds (see https://github.com/elastic/elasticsearch/issues/23776)
+     */
+    public void testSingleValueFieldWithExtendedBoundsOffset() throws Exception {
+        String index = "test23776";
+        prepareCreate(index)
+                .setSettings(Settings.builder().put(indexSettings()).put("index.number_of_shards", 1).put("index.number_of_replicas", 0))
+                .execute().actionGet();
+
+        List<IndexRequestBuilder> builders = new ArrayList<>();
+        builders.add(indexDoc(index, DateTime.parse("2016-01-03T08:00:00.000Z"), 1));
+        builders.add(indexDoc(index, DateTime.parse("2016-01-03T08:00:00.000Z"), 2));
+        builders.add(indexDoc(index, DateTime.parse("2016-01-06T08:00:00.000Z"), 3));
+        builders.add(indexDoc(index, DateTime.parse("2016-01-06T08:00:00.000Z"), 4));
+        indexRandom(true, builders);
+        ensureSearchable(index);
+
+        SearchResponse response = null;
+        // retrieve those docs with the same time zone and extended bounds
+        response = client()
+                .prepareSearch(index)
+                .addAggregation(
+                        dateHistogram("histo").field("date").dateHistogramInterval(DateHistogramInterval.days(1)).offset("+6h").minDocCount(0)
+                                .extendedBounds(new ExtendedBounds("2016-01-01T06:00:00Z", "2016-01-08T08:00:00Z"))
+                ).execute().actionGet();
+        assertSearchResponse(response);
+
+        Histogram histo = response.getAggregations().get("histo");
+        assertThat(histo, notNullValue());
+        assertThat(histo.getName(), equalTo("histo"));
+        List<? extends Bucket> buckets = histo.getBuckets();
+        assertThat(buckets.size(), equalTo(8));
+
+        assertEquals("2016-01-01T06:00:00.000Z", buckets.get(0).getKeyAsString());
+        assertEquals(0, buckets.get(0).getDocCount());
+        assertEquals("2016-01-02T06:00:00.000Z", buckets.get(1).getKeyAsString());
+        assertEquals(0, buckets.get(1).getDocCount());
+        assertEquals("2016-01-03T06:00:00.000Z", buckets.get(2).getKeyAsString());
+        assertEquals(2, buckets.get(2).getDocCount());
+        assertEquals("2016-01-04T06:00:00.000Z", buckets.get(3).getKeyAsString());
+        assertEquals(0, buckets.get(3).getDocCount());
+        assertEquals("2016-01-05T06:00:00.000Z", buckets.get(4).getKeyAsString());
+        assertEquals(0, buckets.get(4).getDocCount());
+        assertEquals("2016-01-06T06:00:00.000Z", buckets.get(5).getKeyAsString());
+        assertEquals(2, buckets.get(5).getDocCount());
+        assertEquals("2016-01-07T06:00:00.000Z", buckets.get(6).getKeyAsString());
+        assertEquals(0, buckets.get(6).getDocCount());
+        assertEquals("2016-01-08T06:00:00.000Z", buckets.get(7).getKeyAsString());
+        assertEquals(0, buckets.get(7).getDocCount());
+
+        internalCluster().wipeIndices(index);
     }
 
     public void testSingleValueWithMultipleDateFormatsFromMapping() throws Exception {
@@ -1203,4 +1250,46 @@ public void testDSTEndTransition() throws Exception {
         assertThat(((DateTime) buckets.get(2).getKey()).getMillis() - ((DateTime) buckets.get(1).getKey()).getMillis(), equalTo(3600000L));
         assertThat(((DateTime) buckets.get(3).getKey()).getMillis() - ((DateTime) buckets.get(2).getKey()).getMillis(), equalTo(3600000L));
     }
+
+    /**
+     * Make sure that a request using a script does not get cached and a request
+     * not using a script does get cached.
+     */
+    public void testDontCacheScripts() throws Exception {
+        assertAcked(prepareCreate("cache_test_idx").addMapping("type", "d", "type=date")
+                .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1))
+                .get());
+        indexRandom(true, client().prepareIndex("cache_test_idx", "type", "1").setSource("d", date(1, 1)),
+                client().prepareIndex("cache_test_idx", "type", "2").setSource("d", date(2, 1)));
+
+        // Make sure we are starting with a clear cache
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(0L));
+
+        // Test that a request using a script does not get cached
+        Map<String, Object> params = new HashMap<>();
+        params.put("fieldname", "d");
+        SearchResponse r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(dateHistogram("histo").field("d")
+                .script(new Script(ScriptType.INLINE, "native", DateScriptMocks.PlusOneMonthScript.NAME, params))
+                .dateHistogramInterval(DateHistogramInterval.MONTH)).get();
+        assertSearchResponse(r);
+
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(0L));
+
+        // To make sure that the cache is working test that a request not using
+        // a script is cached
+        r = client().prepareSearch("cache_test_idx").setSize(0)
+                .addAggregation(dateHistogram("histo").field("d").dateHistogramInterval(DateHistogramInterval.MONTH)).get();
+        assertSearchResponse(r);
+
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(1L));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramOffsetIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramOffsetIT.java
index 9be3a83bbc910..52b5de85a9f4f 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramOffsetIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramOffsetIT.java
@@ -97,7 +97,7 @@ public void testSingleValueWithPositiveOffset() throws Exception {
         assertThat(response.getHits().getTotalHits(), equalTo(5L));
 
         Histogram histo = response.getAggregations().get("date_histo");
-        List<Histogram.Bucket> buckets = histo.getBuckets();
+        List<? extends Histogram.Bucket> buckets = histo.getBuckets();
         assertThat(buckets.size(), equalTo(2));
 
         checkBucketFor(buckets.get(0), new DateTime(2014, 3, 10, 2, 0, DateTimeZone.UTC), 2L);
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java
index 1e25068100487..238c252ec0c60 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java
@@ -19,10 +19,13 @@
 package org.elasticsearch.search.aggregations.bucket;
 
 import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.script.ScriptType;
+import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.bucket.DateScriptMocks.DateScriptsMockPlugin;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
 import org.elasticsearch.search.aggregations.bucket.range.Range;
@@ -49,6 +52,7 @@
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.core.IsNull.notNullValue;
 import static org.hamcrest.core.IsNull.nullValue;
@@ -121,7 +125,7 @@ public void testDateMath() throws Exception {
         if (randomBoolean()) {
             rangeBuilder.field("date");
         } else {
-            rangeBuilder.script(new Script(DateScriptMocks.ExtractFieldScript.NAME, ScriptType.INLINE, "native", params));
+            rangeBuilder.script(new Script(ScriptType.INLINE, "native", DateScriptMocks.ExtractFieldScript.NAME, params));
         }
         SearchResponse response = client()
                 .prepareSearch("idx")
@@ -421,10 +425,10 @@ public void testSingleValuedFieldWithSubAggregation() throws Exception {
         assertThat(range.getName(), equalTo("range"));
         List<? extends Bucket> buckets = range.getBuckets();
         assertThat(buckets.size(), equalTo(3));
-        assertThat(range.getProperty("_bucket_count"), equalTo(3));
-        Object[] propertiesKeys = (Object[]) range.getProperty("_key");
-        Object[] propertiesDocCounts = (Object[]) range.getProperty("_count");
-        Object[] propertiesCounts = (Object[]) range.getProperty("sum.value");
+        assertThat(((InternalAggregation)range).getProperty("_bucket_count"), equalTo(3));
+        Object[] propertiesKeys = (Object[]) ((InternalAggregation)range).getProperty("_key");
+        Object[] propertiesDocCounts = (Object[]) ((InternalAggregation)range).getProperty("_count");
+        Object[] propertiesCounts = (Object[]) ((InternalAggregation)range).getProperty("sum.value");
 
         Range.Bucket bucket = buckets.get(0);
         assertThat(bucket, notNullValue());
@@ -543,7 +547,7 @@ public void testMultiValuedFieldWithValueScript() throws Exception {
         SearchResponse response = client().prepareSearch("idx")
                 .addAggregation(dateRange("range")
                         .field("dates")
-                                .script(new Script(DateScriptMocks.PlusOneMonthScript.NAME, ScriptType.INLINE, "native", params))
+                                .script(new Script(ScriptType.INLINE, "native", DateScriptMocks.PlusOneMonthScript.NAME, params))
                                 .addUnboundedTo(date(2, 15)).addRange(date(2, 15), date(3, 15)).addUnboundedFrom(date(3, 15))).execute()
                 .actionGet();
 
@@ -599,7 +603,7 @@ public void testScriptSingleValue() throws Exception {
         params.put("fieldname", "date");
         SearchResponse response = client().prepareSearch("idx")
                 .addAggregation(dateRange("range")
-                        .script(new Script(DateScriptMocks.ExtractFieldScript.NAME, ScriptType.INLINE, "native", params))
+                        .script(new Script(ScriptType.INLINE, "native", DateScriptMocks.ExtractFieldScript.NAME, params))
                         .addUnboundedTo(date(2, 15))
                         .addRange(date(2, 15), date(3, 15))
                         .addUnboundedFrom(date(3, 15)))
@@ -661,7 +665,7 @@ public void testScriptMultiValued() throws Exception {
         SearchResponse response = client()
                 .prepareSearch("idx")
                 .addAggregation(
-                        dateRange("range").script(new Script(DateScriptMocks.ExtractFieldScript.NAME, ScriptType.INLINE, "native", params))
+                        dateRange("range").script(new Script(ScriptType.INLINE, "native", DateScriptMocks.ExtractFieldScript.NAME, params))
                         .addUnboundedTo(date(2, 15)).addRange(date(2, 15), date(3, 15))
                         .addUnboundedFrom(date(3, 15))).execute().actionGet();
 
@@ -866,4 +870,64 @@ public void testEmptyAggregation() throws Exception {
         assertThat(buckets.get(0).getDocCount(), equalTo(0L));
         assertThat(buckets.get(0).getAggregations().asList().isEmpty(), is(true));
     }
+
+    public void testNoRangesInQuery()  {
+        try {
+            client().prepareSearch("idx")
+                .addAggregation(dateRange("my_date_range_agg").field("value"))
+                .execute().actionGet();
+            fail();
+        } catch (SearchPhaseExecutionException spee){
+            Throwable rootCause = spee.getCause().getCause();
+            assertThat(rootCause, instanceOf(IllegalArgumentException.class));
+            assertEquals(rootCause.getMessage(), "No [ranges] specified for the [my_date_range_agg] aggregation");
+        }
+    }
+
+    /**
+     * Make sure that a request using a script does not get cached and a request
+     * not using a script does get cached.
+     */
+    public void testDontCacheScripts() throws Exception {
+        assertAcked(prepareCreate("cache_test_idx").addMapping("type", "date", "type=date")
+                .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1))
+                .get());
+        indexRandom(true,
+                client().prepareIndex("cache_test_idx", "type", "1")
+                        .setSource(jsonBuilder().startObject().field("date", date(1, 1)).endObject()),
+                client().prepareIndex("cache_test_idx", "type", "2")
+                        .setSource(jsonBuilder().startObject().field("date", date(2, 1)).endObject()));
+
+        // Make sure we are starting with a clear cache
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(0L));
+
+        // Test that a request using a script does not get cached
+        Map<String, Object> params = new HashMap<>();
+        params.put("fieldname", "date");
+        SearchResponse r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(dateRange("foo").field("date")
+                .script(new Script(ScriptType.INLINE, "native", DateScriptMocks.PlusOneMonthScript.NAME, params))
+                .addRange(new DateTime(2012, 1, 1, 0, 0, 0, 0, DateTimeZone.UTC), new DateTime(2013, 1, 1, 0, 0, 0, 0, DateTimeZone.UTC)))
+                .get();
+        assertSearchResponse(r);
+
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(0L));
+
+        // To make sure that the cache is working test that a request not using
+        // a script is cached
+        r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(dateRange("foo").field("date")
+                .addRange(new DateTime(2012, 1, 1, 0, 0, 0, 0, DateTimeZone.UTC), new DateTime(2013, 1, 1, 0, 0, 0, 0, DateTimeZone.UTC)))
+                .get();
+        assertSearchResponse(r);
+
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(1L));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeTests.java
index ee32915fdd7df..8987616b77141 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeTests.java
@@ -32,7 +32,7 @@ protected DateRangeAggregationBuilder createTestAggregatorBuilder() {
         for (int i = 0; i < numRanges; i++) {
             String key = null;
             if (randomBoolean()) {
-                key = randomAsciiOfLengthBetween(1, 20);
+                key = randomAlphaOfLengthBetween(1, 20);
             }
             double from = randomBoolean() ? Double.NEGATIVE_INFINITY : randomIntBetween(Integer.MIN_VALUE, Integer.MAX_VALUE - 1000);
             double to = randomBoolean() ? Double.POSITIVE_INFINITY
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DiversifiedSamplerIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DiversifiedSamplerIT.java
index 96fbf17480c27..c8803b7e790e1 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DiversifiedSamplerIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DiversifiedSamplerIT.java
@@ -32,6 +32,7 @@
 import org.elasticsearch.test.ESIntegTestCase;
 
 import java.util.Collection;
+import java.util.List;
 
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
@@ -99,7 +100,7 @@ public void testIssue10719() throws Exception {
         // Tests that we can refer to nested elements under a sample in a path
         // statement
         boolean asc = randomBoolean();
-        SearchResponse response = client().prepareSearch("test").setTypes("book").setSearchType(SearchType.QUERY_AND_FETCH)
+        SearchResponse response = client().prepareSearch("test").setTypes("book").setSearchType(SearchType.QUERY_THEN_FETCH)
                 .addAggregation(terms("genres")
                         .field("genre")
                         .order(Terms.Order.aggregation("sample>max_price.value", asc))
@@ -108,7 +109,7 @@ public void testIssue10719() throws Exception {
                 ).execute().actionGet();
         assertSearchResponse(response);
         Terms genres = response.getAggregations().get("genres");
-        Collection<Bucket> genreBuckets = genres.getBuckets();
+        Collection<? extends Bucket> genreBuckets = genres.getBuckets();
         // For this test to be useful we need >1 genre bucket to compare
         assertThat(genreBuckets.size(), greaterThan(1));
         double lastMaxPrice = asc ? Double.MIN_VALUE : Double.MAX_VALUE;
@@ -132,7 +133,7 @@ public void testSimpleDiversity() throws Exception {
         sampleAgg.field("author").maxDocsPerValue(MAX_DOCS_PER_AUTHOR).executionHint(randomExecutionHint());
         sampleAgg.subAggregation(terms("authors").field("author"));
         SearchResponse response = client().prepareSearch("test")
-                .setSearchType(SearchType.QUERY_AND_FETCH)
+                .setSearchType(SearchType.QUERY_THEN_FETCH)
                 .setQuery(new TermQueryBuilder("genre", "fantasy"))
                 .setFrom(0).setSize(60)
                 .addAggregation(sampleAgg)
@@ -141,7 +142,7 @@ public void testSimpleDiversity() throws Exception {
         assertSearchResponse(response);
         Sampler sample = response.getAggregations().get("sample");
         Terms authors = sample.getAggregations().get("authors");
-        Collection<Bucket> testBuckets = authors.getBuckets();
+        List<? extends Bucket> testBuckets = authors.getBuckets();
 
         for (Terms.Bucket testBucket : testBuckets) {
             assertThat(testBucket.getDocCount(), lessThanOrEqualTo((long) NUM_SHARDS * MAX_DOCS_PER_AUTHOR));
@@ -158,15 +159,15 @@ public void testNestedDiversity() throws Exception {
         sampleAgg.subAggregation(terms("authors").field("author"));
 
         rootTerms.subAggregation(sampleAgg);
-        SearchResponse response = client().prepareSearch("test").setSearchType(SearchType.QUERY_AND_FETCH)
+        SearchResponse response = client().prepareSearch("test").setSearchType(SearchType.QUERY_THEN_FETCH)
                 .addAggregation(rootTerms).execute().actionGet();
         assertSearchResponse(response);
         Terms genres = response.getAggregations().get("genres");
-        Collection<Bucket> genreBuckets = genres.getBuckets();
+        List<? extends Bucket> genreBuckets = genres.getBuckets();
         for (Terms.Bucket genreBucket : genreBuckets) {
             Sampler sample = genreBucket.getAggregations().get("sample");
             Terms authors = sample.getAggregations().get("authors");
-            Collection<Bucket> testBuckets = authors.getBuckets();
+            List<? extends Bucket> testBuckets = authors.getBuckets();
 
             for (Terms.Bucket testBucket : testBuckets) {
                 assertThat(testBucket.getDocCount(), lessThanOrEqualTo((long) NUM_SHARDS * MAX_DOCS_PER_AUTHOR));
@@ -188,14 +189,14 @@ public void testNestedSamples() throws Exception {
         sampleAgg.subAggregation(terms("genres").field("genre"));
 
         rootSample.subAggregation(sampleAgg);
-        SearchResponse response = client().prepareSearch("test").setSearchType(SearchType.QUERY_AND_FETCH).addAggregation(rootSample)
+        SearchResponse response = client().prepareSearch("test").setSearchType(SearchType.QUERY_THEN_FETCH).addAggregation(rootSample)
                 .execute().actionGet();
         assertSearchResponse(response);
         Sampler genreSample = response.getAggregations().get("genreSample");
         Sampler sample = genreSample.getAggregations().get("sample");
 
         Terms genres = sample.getAggregations().get("genres");
-        Collection<Bucket> testBuckets = genres.getBuckets();
+        List<? extends Bucket> testBuckets = genres.getBuckets();
         for (Terms.Bucket testBucket : testBuckets) {
             assertThat(testBucket.getDocCount(), lessThanOrEqualTo((long) NUM_SHARDS * MAX_DOCS_PER_GENRE));
         }
@@ -213,7 +214,7 @@ public void testPartiallyUnmappedDiversifyField() throws Exception {
         DiversifiedAggregationBuilder sampleAgg = new DiversifiedAggregationBuilder("sample").shardSize(100).field("author")
                 .maxDocsPerValue(1);
         sampleAgg.subAggregation(terms("authors").field("author"));
-        SearchResponse response = client().prepareSearch("idx_unmapped_author", "test").setSearchType(SearchType.QUERY_AND_FETCH)
+        SearchResponse response = client().prepareSearch("idx_unmapped_author", "test").setSearchType(SearchType.QUERY_THEN_FETCH)
                 .setQuery(new TermQueryBuilder("genre", "fantasy")).setFrom(0).setSize(60).addAggregation(sampleAgg)
                 .execute().actionGet();
         assertSearchResponse(response);
@@ -229,7 +230,7 @@ public void testWhollyUnmappedDiversifyField() throws Exception {
         DiversifiedAggregationBuilder sampleAgg = new DiversifiedAggregationBuilder("sample").shardSize(100);
         sampleAgg.field("author").maxDocsPerValue(MAX_DOCS_PER_AUTHOR).executionHint(randomExecutionHint());
         sampleAgg.subAggregation(terms("authors").field("author"));
-        SearchResponse response = client().prepareSearch("idx_unmapped", "idx_unmapped_author").setSearchType(SearchType.QUERY_AND_FETCH)
+        SearchResponse response = client().prepareSearch("idx_unmapped", "idx_unmapped_author").setSearchType(SearchType.QUERY_THEN_FETCH)
                 .setQuery(new TermQueryBuilder("genre", "fantasy")).setFrom(0).setSize(60).addAggregation(sampleAgg).execute().actionGet();
         assertSearchResponse(response);
         Sampler sample = response.getAggregations().get("sample");
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DiversifiedSamplerTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DiversifiedSamplerTests.java
index b68caad0ea7f2..d23cf556b5af1 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DiversifiedSamplerTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DiversifiedSamplerTests.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.search.aggregations.bucket;
 
-import org.elasticsearch.script.Script;
 import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
 import org.elasticsearch.search.aggregations.bucket.sampler.DiversifiedAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.sampler.SamplerAggregator.ExecutionMode;
@@ -30,19 +29,7 @@ public class DiversifiedSamplerTests extends BaseAggregationTestCase<Diversified
     protected final DiversifiedAggregationBuilder createTestAggregatorBuilder() {
         DiversifiedAggregationBuilder factory = new DiversifiedAggregationBuilder("foo");
         String field = randomNumericField();
-        int randomFieldBranch = randomInt(3);
-        switch (randomFieldBranch) {
-        case 0:
-            factory.field(field);
-            break;
-        case 1:
-            factory.field(field);
-            factory.script(new Script("_value + 1"));
-            break;
-        case 2:
-            factory.script(new Script("doc[" + field + "] + 1"));
-            break;
-        }
+        randomFieldOrScript(factory, field);
         if (randomBoolean()) {
             factory.missing("MISSING");
         }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DoubleTermsIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DoubleTermsIT.java
index ed43ba8bd63d6..2eae86a4c7941 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DoubleTermsIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DoubleTermsIT.java
@@ -21,16 +21,20 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.ScoreAccessor;
 import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.aggregations.AggregationTestScriptsPlugin;
 import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode;
+import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.bucket.filter.Filter;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
+import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket;
 import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude;
 import org.elasticsearch.search.aggregations.metrics.avg.Avg;
 import org.elasticsearch.search.aggregations.metrics.max.Max;
@@ -46,17 +50,18 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Set;
 import java.util.function.Function;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.functionScoreQuery;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.scriptFunction;
-import static org.elasticsearch.script.ScriptService.ScriptType;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.avg;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.extendedStats;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.filter;
@@ -358,6 +363,43 @@ private void testIncludeExcludeResults(double[] includes, double[] excludes, dou
             assertThat(bucket.getDocCount(), equalTo(1L));
         }
     }
+    
+    public void testSingleValueFieldWithPartitionedFiltering() throws Exception {
+        runTestFieldWithPartitionedFiltering(SINGLE_VALUED_FIELD_NAME);
+    }
+    
+    public void testMultiValueFieldWithPartitionedFiltering() throws Exception {
+        runTestFieldWithPartitionedFiltering(MULTI_VALUED_FIELD_NAME);
+    }
+    
+    private void runTestFieldWithPartitionedFiltering(String field) throws Exception {
+        // Find total number of unique terms
+        SearchResponse allResponse = client().prepareSearch("idx").setTypes("type")
+                .addAggregation(terms("terms").field(field).size(10000).collectMode(randomFrom(SubAggCollectionMode.values())))
+                .execute().actionGet();
+        assertSearchResponse(allResponse);
+        Terms terms = allResponse.getAggregations().get("terms");
+        assertThat(terms, notNullValue());
+        assertThat(terms.getName(), equalTo("terms"));
+        int expectedCardinality = terms.getBuckets().size();
+
+        // Gather terms using partitioned aggregations
+        final int numPartitions = randomIntBetween(2, 4);
+        Set<Number> foundTerms = new HashSet<>();
+        for (int partition = 0; partition < numPartitions; partition++) {
+            SearchResponse response = client().prepareSearch("idx").setTypes("type").addAggregation(terms("terms").field(field)
+                    .includeExclude(new IncludeExclude(partition, numPartitions)).collectMode(randomFrom(SubAggCollectionMode.values())))
+                    .execute().actionGet();
+            assertSearchResponse(response);
+            terms = response.getAggregations().get("terms");
+            assertThat(terms, notNullValue());
+            assertThat(terms.getName(), equalTo("terms"));
+            for (Bucket bucket : terms.getBuckets()) {
+                assertTrue(foundTerms.add(bucket.getKeyAsNumber()));
+            }
+        }
+        assertEquals(expectedCardinality, foundTerms.size());
+    }   
 
     public void testSingleValueFieldOrderedByTermAsc() throws Exception {
         SearchResponse response = client().prepareSearch("idx").setTypes("type")
@@ -426,10 +468,10 @@ public void testSingleValuedFieldWithSubAggregation() throws Exception {
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
         assertThat(terms.getBuckets().size(), equalTo(5));
-        assertThat(terms.getProperty("_bucket_count"), equalTo(5));
-        Object[] propertiesKeys = (Object[]) terms.getProperty("_key");
-        Object[] propertiesDocCounts = (Object[]) terms.getProperty("_count");
-        Object[] propertiesCounts = (Object[]) terms.getProperty("sum.value");
+        assertThat(((InternalAggregation)terms).getProperty("_bucket_count"), equalTo(5));
+        Object[] propertiesKeys = (Object[]) ((InternalAggregation)terms).getProperty("_key");
+        Object[] propertiesDocCounts = (Object[]) ((InternalAggregation)terms).getProperty("_count");
+        Object[] propertiesCounts = (Object[]) ((InternalAggregation)terms).getProperty("sum.value");
 
         for (int i = 0; i < 5; i++) {
             Terms.Bucket bucket = terms.getBucketByKey("" + (double) i);
@@ -451,7 +493,7 @@ public void testSingleValuedFieldWithValueScript() throws Exception {
                 .addAggregation(terms("terms")
                         .field(SINGLE_VALUED_FIELD_NAME)
                         .collectMode(randomFrom(SubAggCollectionMode.values()))
-                                .script(new Script("_value + 1", ScriptType.INLINE, CustomScriptPlugin.NAME, null)))
+                                .script(new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_value + 1", Collections.emptyMap())))
                 .get();
 
         assertSearchResponse(response);
@@ -504,7 +546,7 @@ public void testMultiValuedFieldWithValueScript() throws Exception {
                 .addAggregation(terms("terms")
                         .field(MULTI_VALUED_FIELD_NAME)
                         .collectMode(randomFrom(SubAggCollectionMode.values()))
-                                .script(new Script("_value + 1", ScriptType.INLINE, CustomScriptPlugin.NAME, null)))
+                                .script(new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_value + 1", Collections.emptyMap())))
                 .get();
 
         assertSearchResponse(response);
@@ -533,7 +575,8 @@ public void testMultiValuedFieldWithValueScriptNotUnique() throws Exception {
                 .addAggregation(terms("terms")
                         .field(MULTI_VALUED_FIELD_NAME)
                         .collectMode(randomFrom(SubAggCollectionMode.values()))
-                        .script(new Script("(long) (_value / 1000 + 1)", ScriptType.INLINE, CustomScriptPlugin.NAME, null)))
+                        .script(
+                            new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "(long) (_value / 1000 + 1)", Collections.emptyMap())))
                 .get();
 
         assertSearchResponse(response);
@@ -575,8 +618,8 @@ public void testScriptSingleValue() throws Exception {
                 .addAggregation(
                         terms("terms")
                                 .collectMode(randomFrom(SubAggCollectionMode.values()))
-                                .script(new Script("doc['" + MULTI_VALUED_FIELD_NAME + "'].value", ScriptType.INLINE,
-                                        CustomScriptPlugin.NAME, null)))
+                                .script(new Script(ScriptType.INLINE,
+                                    CustomScriptPlugin.NAME, "doc['" + MULTI_VALUED_FIELD_NAME + "'].value", Collections.emptyMap())))
                 .get();
 
         assertSearchResponse(response);
@@ -602,8 +645,8 @@ public void testScriptMultiValued() throws Exception {
                 .addAggregation(
                         terms("terms")
                                 .collectMode(randomFrom(SubAggCollectionMode.values()))
-                                .script(new Script("doc['" + MULTI_VALUED_FIELD_NAME + "']", ScriptType.INLINE,
-                                        CustomScriptPlugin.NAME, null)))
+                                .script(new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['" + MULTI_VALUED_FIELD_NAME + "']",
+                                    Collections.emptyMap())))
                 .get();
 
         assertSearchResponse(response);
@@ -804,7 +847,7 @@ public void testSingleValuedFieldOrderedBySingleBucketSubAggregationAsc() throws
         assertThat(tags.getName(), equalTo("num_tags"));
         assertThat(tags.getBuckets().size(), equalTo(2));
 
-        Iterator<Terms.Bucket> iters = tags.getBuckets().iterator();
+        Iterator<? extends Terms.Bucket> iters = tags.getBuckets().iterator();
 
         Terms.Bucket tag = iters.next();
         assertThat(tag, notNullValue());
@@ -846,7 +889,7 @@ public void testSingleValuedFieldOrderedBySubAggregationAscMultiHierarchyLevels(
         assertThat(tags.getName(), equalTo("tags"));
         assertThat(tags.getBuckets().size(), equalTo(2));
 
-        Iterator<Terms.Bucket> iters = tags.getBuckets().iterator();
+        Iterator<? extends Terms.Bucket> iters = tags.getBuckets().iterator();
 
         // the max for "1" is 2
         // the max for "0" is 4
@@ -1077,10 +1120,11 @@ public void testSingleValuedFieldOrderedByMultiValueExtendedStatsAsc() throws Ex
     }
 
     public void testScriptScore() {
-        Script scoringScript =
-                new Script("doc['" + SINGLE_VALUED_FIELD_NAME + "'].value", ScriptType.INLINE, CustomScriptPlugin .NAME, null);
+        Script scoringScript = new Script(
+            ScriptType.INLINE, CustomScriptPlugin .NAME, "doc['" + SINGLE_VALUED_FIELD_NAME + "'].value", Collections.emptyMap());
 
-        Script aggregationScript = new Script("ceil(_score.doubleValue()/3)", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
+        Script aggregationScript = new Script(
+            ScriptType.INLINE, CustomScriptPlugin.NAME, "ceil(_score.doubleValue()/3)", Collections.emptyMap());
 
         SearchResponse response = client()
                 .prepareSearch("idx")
@@ -1179,4 +1223,43 @@ private void assertMultiSortResponse(double[] expectedKeys, Terms.Order... order
     public void testOtherDocCount() {
         testOtherDocCount(SINGLE_VALUED_FIELD_NAME, MULTI_VALUED_FIELD_NAME);
     }
+
+    /**
+     * Make sure that a request using a script does not get cached and a request
+     * not using a script does get cached.
+     */
+    public void testDontCacheScripts() throws Exception {
+        assertAcked(prepareCreate("cache_test_idx").addMapping("type", "d", "type=float")
+                .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1))
+                .get());
+        indexRandom(true, client().prepareIndex("cache_test_idx", "type", "1").setSource("s", 1.5),
+                client().prepareIndex("cache_test_idx", "type", "2").setSource("s", 2.5));
+
+        // Make sure we are starting with a clear cache
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(0L));
+
+        // Test that a request using a script does not get cached
+        SearchResponse r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(
+                terms("terms").field("d").script(
+                    new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_value + 1", Collections.emptyMap()))).get();
+        assertSearchResponse(r);
+
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(0L));
+
+        // To make sure that the cache is working test that a request not using
+        // a script is cached
+        r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(terms("terms").field("d")).get();
+        assertSearchResponse(r);
+
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(1L));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FilterIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FilterIT.java
index 0c8b9a22c37b1..1578da3aa3a72 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FilterIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FilterIT.java
@@ -21,15 +21,14 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.bucket.filter.Filter;
 import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
@@ -127,8 +126,8 @@ public void testEmptyFilterDeclarations() throws Exception {
      */
     public void testEmptyFilter() throws Exception {
         String emtpyFilterBody = "{ }";
-        XContentParser parser = XContentFactory.xContent(emtpyFilterBody).createParser(emtpyFilterBody);
-        QueryParseContext parseContext = new QueryParseContext(new IndicesQueriesRegistry(), parser, ParseFieldMatcher.EMPTY);
+        XContentParser parser = createParser(JsonXContent.jsonXContent, emtpyFilterBody);
+        QueryParseContext parseContext = new QueryParseContext(parser);
         AggregationBuilder filterAgg = FilterAggregationBuilder.parse("tag1", parseContext);
 
         SearchResponse response = client().prepareSearch("idx").addAggregation(filterAgg).execute().actionGet();
@@ -153,7 +152,7 @@ public void testWithSubAggregation() throws Exception {
         assertThat(filter, notNullValue());
         assertThat(filter.getName(), equalTo("tag1"));
         assertThat(filter.getDocCount(), equalTo((long) numTag1Docs));
-        assertThat((long) filter.getProperty("_count"), equalTo((long) numTag1Docs));
+        assertThat((long) ((InternalAggregation)filter).getProperty("_count"), equalTo((long) numTag1Docs));
 
         long sum = 0;
         for (int i = 0; i < numTag1Docs; ++i) {
@@ -164,7 +163,7 @@ public void testWithSubAggregation() throws Exception {
         assertThat(avgValue, notNullValue());
         assertThat(avgValue.getName(), equalTo("avg_value"));
         assertThat(avgValue.getValue(), equalTo((double) sum / numTag1Docs));
-        assertThat((double) filter.getProperty("avg_value.value"), equalTo((double) sum / numTag1Docs));
+        assertThat((double) ((InternalAggregation)filter).getProperty("avg_value.value"), equalTo((double) sum / numTag1Docs));
     }
 
     public void testAsSubAggregation() {
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java
index 619bba7da1289..3ee868b90e846 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java
@@ -22,15 +22,14 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.bucket.filters.Filters;
 import org.elasticsearch.search.aggregations.bucket.filters.FiltersAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.filters.FiltersAggregator.KeyedFilter;
@@ -172,10 +171,10 @@ public void testWithSubAggregation() throws Exception {
         assertThat(filters.getName(), equalTo("tags"));
 
         assertThat(filters.getBuckets().size(), equalTo(2));
-        assertThat(filters.getProperty("_bucket_count"), equalTo(2));
-        Object[] propertiesKeys = (Object[]) filters.getProperty("_key");
-        Object[] propertiesDocCounts = (Object[]) filters.getProperty("_count");
-        Object[] propertiesCounts = (Object[]) filters.getProperty("avg_value.value");
+        assertThat(((InternalAggregation)filters).getProperty("_bucket_count"), equalTo(2));
+        Object[] propertiesKeys = (Object[]) ((InternalAggregation)filters).getProperty("_key");
+        Object[] propertiesDocCounts = (Object[]) ((InternalAggregation)filters).getProperty("_count");
+        Object[] propertiesCounts = (Object[]) ((InternalAggregation)filters).getProperty("avg_value.value");
 
         Filters.Bucket bucket = filters.getBucketByKey("tag1");
         assertThat(bucket, Matchers.notNullValue());
@@ -212,9 +211,9 @@ public void testWithSubAggregation() throws Exception {
 
     public void testEmptyFilter() throws Exception {
         String emtpyFilterBody = "{ \"filters\" : [ {} ] }";
-        XContentParser parser = XContentFactory.xContent(emtpyFilterBody).createParser(emtpyFilterBody);
+        XContentParser parser = createParser(JsonXContent.jsonXContent, emtpyFilterBody);
         parser.nextToken();
-        QueryParseContext parseContext = new QueryParseContext(new IndicesQueriesRegistry(), parser, ParseFieldMatcher.EMPTY);
+        QueryParseContext parseContext = new QueryParseContext(parser);
         AggregationBuilder filtersAgg = FiltersAggregationBuilder.parse("tag1", parseContext);
 
         SearchResponse response = client().prepareSearch("idx").addAggregation(filtersAgg).execute().actionGet();
@@ -229,9 +228,9 @@ public void testEmptyFilter() throws Exception {
 
     public void testEmptyKeyedFilter() throws Exception {
         String emtpyFilterBody = "{ \"filters\" : {\"foo\" : {} } }";
-        XContentParser parser = XContentFactory.xContent(emtpyFilterBody).createParser(emtpyFilterBody);
+        XContentParser parser = createParser(JsonXContent.jsonXContent, emtpyFilterBody);
         parser.nextToken();
-        QueryParseContext parseContext = new QueryParseContext(new IndicesQueriesRegistry(), parser, ParseFieldMatcher.EMPTY);
+        QueryParseContext parseContext = new QueryParseContext(parser);
         AggregationBuilder filtersAgg = FiltersAggregationBuilder.parse("tag1", parseContext);
 
         SearchResponse response = client().prepareSearch("idx").addAggregation(filtersAgg)
@@ -427,10 +426,10 @@ public void testOtherWithSubAggregation() throws Exception {
         assertThat(filters.getName(), equalTo("tags"));
 
         assertThat(filters.getBuckets().size(), equalTo(3));
-        assertThat(filters.getProperty("_bucket_count"), equalTo(3));
-        Object[] propertiesKeys = (Object[]) filters.getProperty("_key");
-        Object[] propertiesDocCounts = (Object[]) filters.getProperty("_count");
-        Object[] propertiesCounts = (Object[]) filters.getProperty("avg_value.value");
+        assertThat(((InternalAggregation)filters).getProperty("_bucket_count"), equalTo(3));
+        Object[] propertiesKeys = (Object[]) ((InternalAggregation)filters).getProperty("_key");
+        Object[] propertiesDocCounts = (Object[]) ((InternalAggregation)filters).getProperty("_count");
+        Object[] propertiesCounts = (Object[]) ((InternalAggregation)filters).getProperty("avg_value.value");
 
         Filters.Bucket bucket = filters.getBucketByKey("tag1");
         assertThat(bucket, Matchers.notNullValue());
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceIT.java
index 288e9d4dcc5a1..2d13c0312a8e4 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceIT.java
@@ -20,6 +20,7 @@
 
 import org.elasticsearch.Version;
 import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.geo.GeoPoint;
@@ -28,6 +29,7 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode;
+import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
 import org.elasticsearch.search.aggregations.bucket.range.Range;
 import org.elasticsearch.search.aggregations.bucket.range.Range.Bucket;
@@ -51,6 +53,7 @@
 import static org.elasticsearch.search.aggregations.AggregationBuilders.terms;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.sameInstance;
 import static org.hamcrest.core.IsNull.notNullValue;
@@ -349,10 +352,10 @@ public void testWithSubAggregation() throws Exception {
         assertThat(geoDist.getName(), equalTo("amsterdam_rings"));
         List<? extends Bucket> buckets = geoDist.getBuckets();
         assertThat(geoDist.getBuckets().size(), equalTo(3));
-        assertThat(geoDist.getProperty("_bucket_count"), equalTo(3));
-        Object[] propertiesKeys = (Object[]) geoDist.getProperty("_key");
-        Object[] propertiesDocCounts = (Object[]) geoDist.getProperty("_count");
-        Object[] propertiesCities = (Object[]) geoDist.getProperty("cities");
+        assertThat(((InternalAggregation)geoDist).getProperty("_bucket_count"), equalTo(3));
+        Object[] propertiesKeys = (Object[]) ((InternalAggregation)geoDist).getProperty("_key");
+        Object[] propertiesDocCounts = (Object[]) ((InternalAggregation)geoDist).getProperty("_count");
+        Object[] propertiesCities = (Object[]) ((InternalAggregation)geoDist).getProperty("cities");
 
         Range.Bucket bucket = buckets.get(0);
         assertThat(bucket, notNullValue());
@@ -442,6 +445,19 @@ public void testEmptyAggregation() throws Exception {
         assertThat(buckets.get(0).getDocCount(), equalTo(0L));
     }
 
+    public void testNoRangesInQuery()  {
+        try {
+            client().prepareSearch("idx")
+                .addAggregation(geoDistance("geo_dist", new GeoPoint(52.3760, 4.894)))
+                .execute().actionGet();
+            fail();
+        } catch (SearchPhaseExecutionException spee){
+            Throwable rootCause = spee.getCause().getCause();
+            assertThat(rootCause, instanceOf(IllegalArgumentException.class));
+            assertEquals(rootCause.getMessage(), "No [ranges] specified for the [geo_dist] aggregation");
+        }
+    }
+
     public void testMultiValues() throws Exception {
         SearchResponse response = client().prepareSearch("idx-multi")
                 .addAggregation(geoDistance("amsterdam_rings", new GeoPoint(52.3760, 4.894))
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceRangeTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceRangeTests.java
index 65b80537c7ac4..7f8af68b381a7 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceRangeTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceRangeTests.java
@@ -24,7 +24,7 @@
 import org.elasticsearch.common.unit.DistanceUnit;
 import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
 import org.elasticsearch.search.aggregations.bucket.range.geodistance.GeoDistanceAggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.range.geodistance.GeoDistanceParser.Range;
+import org.elasticsearch.search.aggregations.bucket.range.geodistance.GeoDistanceAggregationBuilder.Range;
 import org.elasticsearch.test.geo.RandomShapeGenerator;
 
 public class GeoDistanceRangeTests extends BaseAggregationTestCase<GeoDistanceAggregationBuilder> {
@@ -37,7 +37,7 @@ protected GeoDistanceAggregationBuilder createTestAggregatorBuilder() {
         for (int i = 0; i < numRanges; i++) {
             String key = null;
             if (randomBoolean()) {
-                key = randomAsciiOfLengthBetween(1, 20);
+                key = randomAlphaOfLengthBetween(1, 20);
             }
             double from = randomBoolean() ? 0 : randomIntBetween(0, Integer.MAX_VALUE - 1000);
             double to = randomBoolean() ? Double.POSITIVE_INFINITY
@@ -45,7 +45,7 @@ protected GeoDistanceAggregationBuilder createTestAggregatorBuilder() {
                             : randomIntBetween((int) from, Integer.MAX_VALUE));
             factory.addRange(new Range(key, from, to));
         }
-        factory.field(randomAsciiOfLengthBetween(1, 20));
+        factory.field(randomAlphaOfLengthBetween(1, 20));
         if (randomBoolean()) {
             factory.keyed(randomBoolean());
         }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridIT.java
index 733d2feb86470..7f158e0732f97 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridIT.java
@@ -31,6 +31,7 @@
 import org.elasticsearch.index.query.GeoBoundingBoxQueryBuilder;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
+import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.bucket.filter.Filter;
 import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGrid;
 import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGrid.Bucket;
@@ -156,9 +157,9 @@ public void testSimple() throws Exception {
             assertSearchResponse(response);
 
             GeoHashGrid geoGrid = response.getAggregations().get("geohashgrid");
-            List<Bucket> buckets = geoGrid.getBuckets();
-            Object[] propertiesKeys = (Object[]) geoGrid.getProperty("_key");
-            Object[] propertiesDocCounts = (Object[]) geoGrid.getProperty("_count");
+            List<? extends Bucket> buckets = geoGrid.getBuckets();
+            Object[] propertiesKeys = (Object[]) ((InternalAggregation)geoGrid).getProperty("_key");
+            Object[] propertiesDocCounts = (Object[]) ((InternalAggregation)geoGrid).getProperty("_count");
             for (int i = 0; i < buckets.size(); i++) {
                 GeoHashGrid.Bucket cell = buckets.get(i);
                 String geohash = cell.getKeyAsString();
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridTests.java
index 688289f02dd13..fb7b3984cdf2b 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridTests.java
@@ -26,7 +26,7 @@ public class GeoHashGridTests extends BaseAggregationTestCase<GeoGridAggregation
 
     @Override
     protected GeoGridAggregationBuilder createTestAggregatorBuilder() {
-        String name = randomAsciiOfLengthBetween(3, 20);
+        String name = randomAlphaOfLengthBetween(3, 20);
         GeoGridAggregationBuilder factory = new GeoGridAggregationBuilder(name);
         if (randomBoolean()) {
             int precision = randomIntBetween(1, 12);
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GlobalIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GlobalIT.java
index 9cebfeb9824e4..a10810e82e303 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GlobalIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GlobalIT.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.bucket.global.Global;
 import org.elasticsearch.search.aggregations.metrics.stats.Stats;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -85,11 +86,11 @@ public void testWithStatsSubAggregator() throws Exception {
         assertThat(global, notNullValue());
         assertThat(global.getName(), equalTo("global"));
         assertThat(global.getDocCount(), equalTo((long) numDocs));
-        assertThat((long) global.getProperty("_count"), equalTo((long) numDocs));
+        assertThat((long) ((InternalAggregation)global).getProperty("_count"), equalTo((long) numDocs));
         assertThat(global.getAggregations().asList().isEmpty(), is(false));
 
         Stats stats = global.getAggregations().get("value_stats");
-        assertThat((Stats) global.getProperty("value_stats"), sameInstance(stats));
+        assertThat((Stats) ((InternalAggregation)global).getProperty("value_stats"), sameInstance(stats));
         assertThat(stats, notNullValue());
         assertThat(stats.getName(), equalTo("value_stats"));
         long sum = 0;
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GlobalTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GlobalTests.java
index a874eff839b9e..957f0f366339c 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GlobalTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GlobalTests.java
@@ -26,7 +26,7 @@ public class GlobalTests extends BaseAggregationTestCase<GlobalAggregationBuilde
 
     @Override
     protected GlobalAggregationBuilder createTestAggregatorBuilder() {
-        return new GlobalAggregationBuilder(randomAsciiOfLengthBetween(3, 20));
+        return new GlobalAggregationBuilder(randomAlphaOfLengthBetween(3, 20));
     }
 
 }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/HistogramIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/HistogramIT.java
index e24f0d39d4b33..683e7924419a3 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/HistogramIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/HistogramIT.java
@@ -21,11 +21,13 @@
 import com.carrotsearch.hppc.LongHashSet;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.MockScriptPlugin;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.script.ScriptType;
+import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.bucket.filter.Filter;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket;
@@ -354,10 +356,10 @@ public void testSingleValuedFieldWithSubAggregation() throws Exception {
         assertThat(histo, notNullValue());
         assertThat(histo.getName(), equalTo("histo"));
         assertThat(histo.getBuckets().size(), equalTo(numValueBuckets));
-        assertThat(histo.getProperty("_bucket_count"), equalTo(numValueBuckets));
-        Object[] propertiesKeys = (Object[]) histo.getProperty("_key");
-        Object[] propertiesDocCounts = (Object[]) histo.getProperty("_count");
-        Object[] propertiesCounts = (Object[]) histo.getProperty("sum.value");
+        assertThat(((InternalAggregation)histo).getProperty("_bucket_count"), equalTo(numValueBuckets));
+        Object[] propertiesKeys = (Object[]) ((InternalAggregation)histo).getProperty("_key");
+        Object[] propertiesDocCounts = (Object[]) ((InternalAggregation)histo).getProperty("_count");
+        Object[] propertiesCounts = (Object[]) ((InternalAggregation)histo).getProperty("sum.value");
 
         // TODO: use diamond once JI-9019884 is fixed
         List<Histogram.Bucket> buckets = new ArrayList<>(histo.getBuckets());
@@ -561,7 +563,7 @@ public void testSingleValuedFieldWithValueScript() throws Exception {
                 .addAggregation(
                         histogram("histo")
                                 .field(SINGLE_VALUED_FIELD_NAME)
-                                .script(new Script("_value + 1", ScriptType.INLINE, CustomScriptPlugin.NAME, emptyMap()))
+                                .script(new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_value + 1", emptyMap()))
                                 .interval(interval))
                 .execute().actionGet();
 
@@ -638,7 +640,7 @@ public void testMultiValuedFieldWithValueScript() throws Exception {
                 .addAggregation(
                         histogram("histo")
                                 .field(MULTI_VALUED_FIELD_NAME)
-                                .script(new Script("_value + 1", ScriptType.INLINE, CustomScriptPlugin.NAME, emptyMap()))
+                                .script(new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_value + 1", emptyMap()))
                                 .interval(interval))
                 .execute().actionGet();
 
@@ -674,7 +676,7 @@ public void testScriptSingleValue() throws Exception {
         SearchResponse response = client().prepareSearch("idx")
                 .addAggregation(
                         histogram("histo")
-                            .script(new Script("doc['l_value'].value", ScriptType.INLINE, CustomScriptPlugin.NAME, emptyMap()))
+                            .script(new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['l_value'].value", emptyMap()))
                             .interval(interval))
                 .execute().actionGet();
 
@@ -698,7 +700,7 @@ public void testScriptMultiValued() throws Exception {
         SearchResponse response = client().prepareSearch("idx")
                 .addAggregation(
                         histogram("histo")
-                                .script(new Script("doc['l_values']", ScriptType.INLINE, CustomScriptPlugin.NAME, emptyMap()))
+                                .script(new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['l_values']", emptyMap()))
                                 .interval(interval))
                 .execute().actionGet();
 
@@ -988,11 +990,50 @@ public void testDecimalIntervalAndOffset() throws Exception {
         assertSearchResponse(r);
 
         Histogram histogram = r.getAggregations().get("histo");
-        List<Bucket> buckets = histogram.getBuckets();
+        List<? extends Bucket> buckets = histogram.getBuckets();
         assertEquals(2, buckets.size());
         assertEquals(-0.65, (double) buckets.get(0).getKey(), 0.01d);
         assertEquals(1, buckets.get(0).getDocCount());
         assertEquals(0.05, (double) buckets.get(1).getKey(), 0.01d);
         assertEquals(1, buckets.get(1).getDocCount());
     }
+
+    /**
+     * Make sure that a request using a script does not get cached and a request
+     * not using a script does get cached.
+     */
+    public void testDontCacheScripts() throws Exception {
+        assertAcked(prepareCreate("cache_test_idx").addMapping("type", "d", "type=float")
+                .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1))
+                .get());
+        indexRandom(true, client().prepareIndex("cache_test_idx", "type", "1").setSource("d", -0.6),
+                client().prepareIndex("cache_test_idx", "type", "2").setSource("d", 0.1));
+
+        // Make sure we are starting with a clear cache
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(0L));
+
+        // Test that a request using a script does not get cached
+        SearchResponse r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(histogram("histo").field("d")
+                .script(new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_value + 1", emptyMap())).interval(0.7).offset(0.05)).get();
+        assertSearchResponse(r);
+
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(0L));
+
+        // To make sure that the cache is working test that a request not using
+        // a script is cached
+        r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(histogram("histo").field("d").interval(0.7).offset(0.05))
+                .get();
+        assertSearchResponse(r);
+
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(1L));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeIT.java
index d9f2ed7c365c6..b9bb46501db73 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeIT.java
@@ -18,18 +18,9 @@
  */
 package org.elasticsearch.search.aggregations.bucket;
 
-import org.elasticsearch.cluster.health.ClusterHealthStatus;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
-import static org.hamcrest.Matchers.containsString;
-
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
+import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.common.inject.internal.Nullable;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.ScriptPlugin;
@@ -37,12 +28,22 @@
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.NativeScriptFactory;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptModule;
-import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.search.aggregations.bucket.range.Range;
 import org.elasticsearch.test.ESIntegTestCase;
 
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.instanceOf;
+
 @ESIntegTestCase.SuiteScopeTestCase
 public class IpRangeIT extends ESIntegTestCase {
 
@@ -209,7 +210,7 @@ public void testRejectsScript() {
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
                 () -> client().prepareSearch("idx").addAggregation(
                         AggregationBuilders.ipRange("my_range")
-                        .script(new Script(DummyScript.NAME, ScriptType.INLINE, "native", Collections.emptyMap())) ).get());
+                        .script(new Script(ScriptType.INLINE, "native", DummyScript.NAME, Collections.emptyMap())) ).get());
         assertThat(e.getMessage(), containsString("[ip_range] does not support scripts"));
     }
 
@@ -218,10 +219,24 @@ public void testRejectsValueScript() {
                 () -> client().prepareSearch("idx").addAggregation(
                         AggregationBuilders.ipRange("my_range")
                         .field("ip")
-                        .script(new Script(DummyScript.NAME, ScriptType.INLINE, "native", Collections.emptyMap())) ).get());
+                        .script(new Script(ScriptType.INLINE, "native", DummyScript.NAME, Collections.emptyMap())) ).get());
         assertThat(e.getMessage(), containsString("[ip_range] does not support scripts"));
     }
 
+    public void testNoRangesInQuery()  {
+        try {
+            client().prepareSearch("idx").addAggregation(
+                AggregationBuilders.ipRange("my_range")
+                    .field("ip"))
+                .execute().actionGet();
+            fail();
+        } catch (SearchPhaseExecutionException spee){
+            Throwable rootCause = spee.getCause().getCause();
+            assertThat(rootCause, instanceOf(IllegalArgumentException.class));
+            assertEquals(rootCause.getMessage(), "No [ranges] specified for the [my_range] aggregation");
+        }
+    }
+
     public static class DummyScriptPlugin extends Plugin implements ScriptPlugin {
         @Override
         public List<NativeScriptFactory> getNativeScripts() {
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeTests.java
index 5d86571f08f51..eb07d059e5a53 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeTests.java
@@ -51,7 +51,7 @@ protected IpRangeAggregationBuilder createTestAggregatorBuilder() {
         for (int i = 0; i < numRanges; i++) {
             String key = null;
             if (randomBoolean()) {
-                key = randomAsciiOfLengthBetween(1, 20);
+                key = randomAlphaOfLengthBetween(1, 20);
             }
             switch (randomInt(3)) {
             case 0:
@@ -87,4 +87,41 @@ protected IpRangeAggregationBuilder createTestAggregatorBuilder() {
         return factory;
     }
 
+    public void testMask() {
+        IpRangeAggregationBuilder b1 = new IpRangeAggregationBuilder("foo");
+        IpRangeAggregationBuilder b2 = new IpRangeAggregationBuilder("foo");
+        b1.addMaskRange("bar", "192.168.10.12/16");
+        b2.addRange("bar", "192.168.0.0", "192.169.0.0");
+        assertEquals(b1, b2);
+
+        b1 = new IpRangeAggregationBuilder("foo");
+        b2 = new IpRangeAggregationBuilder("foo");
+        b1.addMaskRange("bar", "192.168.0.0/31");
+        b2.addRange("bar", "192.168.0.0", "192.168.0.2");
+        assertEquals(b1, b2);
+
+        b1 = new IpRangeAggregationBuilder("foo");
+        b2 = new IpRangeAggregationBuilder("foo");
+        b1.addMaskRange("bar", "0.0.0.0/0");
+        b2.addRange("bar", "0.0.0.0", "::1:0:0:0");
+        assertEquals(b1, b2);
+
+        b1 = new IpRangeAggregationBuilder("foo");
+        b2 = new IpRangeAggregationBuilder("foo");
+        b1.addMaskRange("bar", "fe80::821f:2ff:fe4a:c5bd/64");
+        b2.addRange("bar", "fe80::", "fe80:0:0:1::");
+        assertEquals(b1, b2);
+
+        b1 = new IpRangeAggregationBuilder("foo");
+        b2 = new IpRangeAggregationBuilder("foo");
+        b1.addMaskRange("bar", "::/16");
+        b2.addRange("bar", null, "1::");
+        assertEquals(b1, b2);
+
+        b1 = new IpRangeAggregationBuilder("foo");
+        b2 = new IpRangeAggregationBuilder("foo");
+        b1.addMaskRange("bar", "::/0");
+        b2.addRange("bar", null, null);
+        assertEquals(b1, b2);
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpTermsIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpTermsIT.java
index 4a03303c66e12..3a69812df1fe9 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpTermsIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpTermsIT.java
@@ -22,11 +22,46 @@
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
 
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.index.fielddata.ScriptDocValues;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
+import org.elasticsearch.search.aggregations.AggregationTestScriptsPlugin;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+import java.util.function.Function;
+
 public class IpTermsIT extends AbstractTermsTestCase {
 
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Collections.singleton(CustomScriptPlugin.class);
+    }
+
+    public static class CustomScriptPlugin extends AggregationTestScriptsPlugin {
+
+        @Override
+        protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
+            Map<String, Function<Map<String, Object>, Object>> scripts = super.pluginScripts();
+
+            scripts.put("doc['ip'].value", vars -> {
+                Map<?, ?> doc = (Map<?,?>) vars.get("doc");
+                return doc.get("ip");
+            });
+
+            scripts.put("doc['ip'].values", vars -> {
+                Map<?, ?> doc = (Map<?,?>) vars.get("doc");
+                return ((ScriptDocValues<?>) doc.get("ip")).get(0);
+            });
+
+            return scripts;
+        }
+    }
+
     public void testBasics() throws Exception {
         assertAcked(prepareCreate("index").addMapping("type", "ip", "type=ip"));
         indexRandom(true,
@@ -51,4 +86,55 @@ public void testBasics() throws Exception {
         assertEquals("2001:db8::2:1", bucket2.getKeyAsString());
     }
 
+    public void testScriptValue() throws Exception {
+        assertAcked(prepareCreate("index").addMapping("type", "ip", "type=ip"));
+        indexRandom(true,
+                client().prepareIndex("index", "type", "1").setSource("ip", "192.168.1.7"),
+                client().prepareIndex("index", "type", "2").setSource("ip", "192.168.1.7"),
+                client().prepareIndex("index", "type", "3").setSource("ip", "2001:db8::2:1"));
+
+        Script script = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME,
+                "doc['ip'].value", Collections.emptyMap());
+        SearchResponse response = client().prepareSearch("index").addAggregation(
+                AggregationBuilders.terms("my_terms").script(script).executionHint(randomExecutionHint())).get();
+        assertSearchResponse(response);
+        Terms terms = response.getAggregations().get("my_terms");
+        assertEquals(2, terms.getBuckets().size());
+
+        Terms.Bucket bucket1 = terms.getBuckets().get(0);
+        assertEquals(2, bucket1.getDocCount());
+        assertEquals("192.168.1.7", bucket1.getKey());
+        assertEquals("192.168.1.7", bucket1.getKeyAsString());
+
+        Terms.Bucket bucket2 = terms.getBuckets().get(1);
+        assertEquals(1, bucket2.getDocCount());
+        assertEquals("2001:db8::2:1", bucket2.getKey());
+        assertEquals("2001:db8::2:1", bucket2.getKeyAsString());
+    }
+
+    public void testScriptValues() throws Exception {
+        assertAcked(prepareCreate("index").addMapping("type", "ip", "type=ip"));
+        indexRandom(true,
+                client().prepareIndex("index", "type", "1").setSource("ip", "192.168.1.7"),
+                client().prepareIndex("index", "type", "2").setSource("ip", "192.168.1.7"),
+                client().prepareIndex("index", "type", "3").setSource("ip", "2001:db8::2:1"));
+
+        Script script = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME,
+                "doc['ip'].values", Collections.emptyMap());
+        SearchResponse response = client().prepareSearch("index").addAggregation(
+                AggregationBuilders.terms("my_terms").script(script).executionHint(randomExecutionHint())).get();
+        assertSearchResponse(response);
+        Terms terms = response.getAggregations().get("my_terms");
+        assertEquals(2, terms.getBuckets().size());
+
+        Terms.Bucket bucket1 = terms.getBuckets().get(0);
+        assertEquals(2, bucket1.getDocCount());
+        assertEquals("192.168.1.7", bucket1.getKey());
+        assertEquals("192.168.1.7", bucket1.getKeyAsString());
+
+        Terms.Bucket bucket2 = terms.getBuckets().get(1);
+        assertEquals(1, bucket2.getDocCount());
+        assertEquals("2001:db8::2:1", bucket2.getKey());
+        assertEquals("2001:db8::2:1", bucket2.getKeyAsString());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/LongTermsIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/LongTermsIT.java
index 6d5e190b542b2..cc2f916587468 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/LongTermsIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/LongTermsIT.java
@@ -21,15 +21,19 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.aggregations.AggregationTestScriptsPlugin;
 import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode;
+import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.bucket.filter.Filter;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
+import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket;
 import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude;
 import org.elasticsearch.search.aggregations.metrics.avg.Avg;
 import org.elasticsearch.search.aggregations.metrics.max.Max;
@@ -45,15 +49,16 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Set;
 import java.util.function.Function;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
-import static org.elasticsearch.script.ScriptService.ScriptType;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.avg;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.extendedStats;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.filter;
@@ -301,31 +306,85 @@ public void testSingleValueFieldWithFiltering() throws Exception {
         long includes[] = { 1, 2, 3, 98 };
         long excludes[] = { -1, 2, 4 };
         long empty[] = {};
-        testIncludeExcludeResults(includes, empty, new long[] { 1, 2, 3 });
-        testIncludeExcludeResults(includes, excludes, new long[] { 1, 3 });
-        testIncludeExcludeResults(empty, excludes, new long[] { 0, 1, 3 });
+        testIncludeExcludeResults(1, includes, empty, new long[] { 1, 2, 3 }, new long[0]);
+        testIncludeExcludeResults(1, includes, excludes, new long[] { 1, 3 }, new long[0]);
+        testIncludeExcludeResults(1, empty, excludes, new long[] { 0, 1, 3 }, new long[0]);
+
+        testIncludeExcludeResults(0, includes, empty, new long[] { 1, 2, 3}, new long[] { 98 });
+        testIncludeExcludeResults(0, includes, excludes, new long[] { 1, 3 }, new long[] { 98 });
+        testIncludeExcludeResults(0, empty, excludes, new long[] { 0, 1, 3 }, new long[] {5, 6, 7, 8, 9, 10, 11});
     }
 
-    private void testIncludeExcludeResults(long[] includes, long[] excludes, long[] expecteds) {
+    private void testIncludeExcludeResults(int minDocCount, long[] includes, long[] excludes,
+                                           long[] expectedWithCounts, long[] expectedZeroCounts) {
         SearchResponse response = client().prepareSearch("idx").setTypes("type")
                 .addAggregation(terms("terms")
                         .field(SINGLE_VALUED_FIELD_NAME)
                         .includeExclude(new IncludeExclude(includes, excludes))
-                        .collectMode(randomFrom(SubAggCollectionMode.values())))
+                        .collectMode(randomFrom(SubAggCollectionMode.values()))
+                        .minDocCount(minDocCount))
                 .execute().actionGet();
         assertSearchResponse(response);
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
-        assertThat(terms.getBuckets().size(), equalTo(expecteds.length));
+        assertThat(terms.getBuckets().size(), equalTo(expectedWithCounts.length + expectedZeroCounts.length));
 
-        for (int i = 0; i < expecteds.length; i++) {
-            Terms.Bucket bucket = terms.getBucketByKey("" + expecteds[i]);
+        for (int i = 0; i < expectedWithCounts.length; i++) {
+            Terms.Bucket bucket = terms.getBucketByKey("" + expectedWithCounts[i]);
             assertThat(bucket, notNullValue());
             assertThat(bucket.getDocCount(), equalTo(1L));
         }
+
+        for (int i = 0; i < expectedZeroCounts.length; i++) {
+            Terms.Bucket bucket = terms.getBucketByKey("" + expectedZeroCounts[i]);
+            assertThat(bucket, notNullValue());
+            assertThat(bucket.getDocCount(), equalTo(0L));
+        }
     }
 
+
+
+    public void testSingleValueFieldWithPartitionedFiltering() throws Exception {
+        runTestFieldWithPartitionedFiltering(SINGLE_VALUED_FIELD_NAME);
+    }
+
+    public void testMultiValueFieldWithPartitionedFiltering() throws Exception {
+        runTestFieldWithPartitionedFiltering(MULTI_VALUED_FIELD_NAME);
+    }
+
+    private void runTestFieldWithPartitionedFiltering(String field) throws Exception {
+        // Find total number of unique terms
+        SearchResponse allResponse = client().prepareSearch("idx").setTypes("type")
+                .addAggregation(terms("terms").field(field).collectMode(randomFrom(SubAggCollectionMode.values()))).execute().actionGet();
+        assertSearchResponse(allResponse);
+        Terms terms = allResponse.getAggregations().get("terms");
+        assertThat(terms, notNullValue());
+        assertThat(terms.getName(), equalTo("terms"));
+        int expectedCardinality = terms.getBuckets().size();
+
+        // Gather terms using partitioned aggregations
+        final int numPartitions = randomIntBetween(2, 4);
+        Set<Number> foundTerms = new HashSet<>();
+        for (int partition = 0; partition < numPartitions; partition++) {
+            SearchResponse response = client().prepareSearch("idx").setTypes("type")
+                    .addAggregation(
+                            terms("terms").field(field).includeExclude(new IncludeExclude(partition, numPartitions))
+                                    .collectMode(randomFrom(SubAggCollectionMode.values())))
+                    .execute().actionGet();
+            assertSearchResponse(response);
+            terms = response.getAggregations().get("terms");
+            assertThat(terms, notNullValue());
+            assertThat(terms.getName(), equalTo("terms"));
+            for (Bucket bucket : terms.getBuckets()) {
+                assertFalse(foundTerms.contains(bucket.getKeyAsNumber()));
+                foundTerms.add(bucket.getKeyAsNumber());
+            }
+        }
+        assertEquals(expectedCardinality, foundTerms.size());
+    }
+
+
     public void testSingleValueFieldWithMaxSize() throws Exception {
         SearchResponse response = client().prepareSearch("idx").setTypes("high_card_type")
                 .addAggregation(terms("terms")
@@ -417,9 +476,9 @@ public void testSingleValuedFieldWithSubAggregation() throws Exception {
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
         assertThat(terms.getBuckets().size(), equalTo(5));
-        Object[] propertiesKeys = (Object[]) terms.getProperty("_key");
-        Object[] propertiesDocCounts = (Object[]) terms.getProperty("_count");
-        Object[] propertiesCounts = (Object[]) terms.getProperty("sum.value");
+        Object[] propertiesKeys = (Object[]) ((InternalAggregation)terms).getProperty("_key");
+        Object[] propertiesDocCounts = (Object[]) ((InternalAggregation)terms).getProperty("_count");
+        Object[] propertiesCounts = (Object[]) ((InternalAggregation)terms).getProperty("sum.value");
 
         for (int i = 0; i < 5; i++) {
             Terms.Bucket bucket = terms.getBucketByKey("" + i);
@@ -441,7 +500,7 @@ public void testSingleValuedFieldWithValueScript() throws Exception {
                 .addAggregation(terms("terms")
                         .field(SINGLE_VALUED_FIELD_NAME)
                         .collectMode(randomFrom(SubAggCollectionMode.values()))
-                        .script(new Script("_value + 1", ScriptType.INLINE, CustomScriptPlugin.NAME, null)))
+                        .script(new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_value + 1", Collections.emptyMap())))
                 .get();
 
         assertSearchResponse(response);
@@ -494,7 +553,7 @@ public void testMultiValuedFieldWithValueScript() throws Exception {
                 .addAggregation(terms("terms")
                         .field(MULTI_VALUED_FIELD_NAME)
                         .collectMode(randomFrom(SubAggCollectionMode.values()))
-                                .script(new Script("_value - 1", ScriptType.INLINE, CustomScriptPlugin.NAME, null)))
+                                .script(new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_value - 1", Collections.emptyMap())))
                 .get();
 
         assertSearchResponse(response);
@@ -523,7 +582,8 @@ public void testMultiValuedFieldWithValueScriptNotUnique() throws Exception {
                 .addAggregation(terms("terms")
                         .field(MULTI_VALUED_FIELD_NAME)
                         .collectMode(randomFrom(SubAggCollectionMode.values()))
-                                .script(new Script("floor(_value / 1000 + 1)", ScriptType.INLINE, CustomScriptPlugin.NAME, null)))
+                                .script(new Script(
+                                    ScriptType.INLINE, CustomScriptPlugin.NAME, "floor(_value / 1000 + 1)", Collections.emptyMap())))
                 .get();
 
         assertSearchResponse(response);
@@ -559,7 +619,8 @@ public void testMultiValuedFieldWithValueScriptNotUnique() throws Exception {
     */
 
     public void testScriptSingleValue() throws Exception {
-        Script script = new Script("doc['" + SINGLE_VALUED_FIELD_NAME + "'].value", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
+        Script script =
+            new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['" + SINGLE_VALUED_FIELD_NAME + "'].value", Collections.emptyMap());
 
         SearchResponse response = client().prepareSearch("idx").setTypes("type")
                 .addAggregation(terms("terms")
@@ -585,7 +646,8 @@ public void testScriptSingleValue() throws Exception {
     }
 
     public void testScriptMultiValued() throws Exception {
-        Script script = new Script("doc['" + MULTI_VALUED_FIELD_NAME + "']", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
+        Script script =
+            new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['" + MULTI_VALUED_FIELD_NAME + "']", Collections.emptyMap());
 
         SearchResponse response = client().prepareSearch("idx").setTypes("type")
                 .addAggregation(terms("terms")
@@ -791,7 +853,7 @@ public void testSingleValuedFieldOrderedBySingleBucketSubAggregationAsc() throws
         assertThat(tags.getName(), equalTo("num_tags"));
         assertThat(tags.getBuckets().size(), equalTo(2));
 
-        Iterator<Terms.Bucket> iters = tags.getBuckets().iterator();
+        Iterator<? extends Terms.Bucket> iters = tags.getBuckets().iterator();
 
         Terms.Bucket tag = iters.next();
         assertThat(tag, notNullValue());
@@ -830,7 +892,7 @@ public void testSingleValuedFieldOrderedBySubAggregationAscMultiHierarchyLevels(
         assertThat(tags.getName(), equalTo("tags"));
         assertThat(tags.getBuckets().size(), equalTo(2));
 
-        Iterator<Terms.Bucket> iters = tags.getBuckets().iterator();
+        Iterator<? extends Terms.Bucket> iters = tags.getBuckets().iterator();
 
         // the max for "1" is 2
         // the max for "0" is 4
@@ -1136,4 +1198,43 @@ private void assertMultiSortResponse(long[] expectedKeys, Terms.Order... order)
     public void testOtherDocCount() {
         testOtherDocCount(SINGLE_VALUED_FIELD_NAME, MULTI_VALUED_FIELD_NAME);
     }
+
+    /**
+     * Make sure that a request using a script does not get cached and a request
+     * not using a script does get cached.
+     */
+    public void testDontCacheScripts() throws Exception {
+        assertAcked(prepareCreate("cache_test_idx").addMapping("type", "d", "type=long")
+                .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1))
+                .get());
+        indexRandom(true, client().prepareIndex("cache_test_idx", "type", "1").setSource("s", 1),
+                client().prepareIndex("cache_test_idx", "type", "2").setSource("s", 2));
+
+        // Make sure we are starting with a clear cache
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(0L));
+
+        // Test that a request using a script does not get cached
+        SearchResponse r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(
+                terms("terms").field("d").script(
+                    new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_value + 1", Collections.emptyMap()))).get();
+        assertSearchResponse(r);
+
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(0L));
+
+        // To make sure that the cache is working test that a request not using
+        // a script is cached
+        r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(terms("terms").field("d")).get();
+        assertSearchResponse(r);
+
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(1L));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/MinDocCountIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/MinDocCountIT.java
index 012df7bfbffcb..e1e8f1ba660cd 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/MinDocCountIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/MinDocCountIT.java
@@ -29,7 +29,7 @@
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.aggregations.AggregationTestScriptsPlugin;
 import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
@@ -151,8 +151,8 @@ TermsAggregationBuilder apply(TermsAggregationBuilder builder, String field) {
         YES {
             @Override
             TermsAggregationBuilder apply(TermsAggregationBuilder builder, String field) {
-                return builder.script(new org.elasticsearch.script.Script("doc['" + field + "'].values", ScriptService.ScriptType.INLINE,
-                 CustomScriptPlugin.NAME, null));
+                return builder.script(new org.elasticsearch.script.Script(ScriptType.INLINE,
+                    CustomScriptPlugin.NAME, "doc['" + field + "'].values", Collections.emptyMap()));
             }
         };
         abstract TermsAggregationBuilder apply(TermsAggregationBuilder builder, String field);
@@ -161,8 +161,8 @@ TermsAggregationBuilder apply(TermsAggregationBuilder builder, String field) {
     // check that terms2 is a subset of terms1
     private void assertSubset(Terms terms1, Terms terms2, long minDocCount, int size, String include) {
         final Matcher matcher = include == null ? null : Pattern.compile(include).matcher("");;
-        final Iterator<Terms.Bucket> it1 = terms1.getBuckets().iterator();
-        final Iterator<Terms.Bucket> it2 = terms2.getBuckets().iterator();
+        final Iterator<? extends Terms.Bucket> it1 = terms1.getBuckets().iterator();
+        final Iterator<? extends Terms.Bucket> it2 = terms2.getBuckets().iterator();
         int size2 = 0;
         while (it1.hasNext()) {
             final Terms.Bucket bucket1 = it1.next();
@@ -170,9 +170,9 @@ private void assertSubset(Terms terms1, Terms terms2, long minDocCount, int size
                 if (size2++ == size) {
                     break;
                 }
-                assertTrue(it2.hasNext());
+                assertTrue("minDocCount: " + minDocCount, it2.hasNext());
                 final Terms.Bucket bucket2 = it2.next();
-                assertEquals(bucket1.getDocCount(), bucket2.getDocCount());
+                assertEquals("minDocCount: " + minDocCount, bucket1.getDocCount(), bucket2.getDocCount());
             }
         }
         assertFalse(it2.hasNext());
@@ -336,24 +336,8 @@ private void testMinDocCountOnTerms(String field, Script script, Terms.Order ord
                             .shardSize(cardinality + randomInt(10))
                             .minDocCount(minDocCount)).request();
             final SearchResponse response = client().search(request).get();
-            try {
-                assertAllSuccessful(response);
-                assertSubset(allTerms, (Terms) response.getAggregations().get("terms"), minDocCount, size, include);
-            } catch (AssertionError ae) {
-                if (!retry) {
-                    throw ae;
-                }
-                logger.info("test failed. trying to see if it recovers after 1m.", ae);
-                try {
-                    Thread.sleep(60000);
-                    logger.debug("1m passed. retrying.");
-                    testMinDocCountOnTerms(field, script, order, include, false);
-                } catch (Exception secondFailure) {
-                    secondFailure.addSuppressed(ae);
-                    logger.error("exception on retry (will re-throw the original in a sec)", secondFailure);
-                }
-                throw ae;
-            }
+            assertAllSuccessful(response);
+            assertSubset(allTerms, (Terms) response.getAggregations().get("terms"), minDocCount, size, include);
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/MissingIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/MissingIT.java
index ade53a2164a68..ac4d8ac315f7c 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/MissingIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/MissingIT.java
@@ -20,6 +20,7 @@
 
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
 import org.elasticsearch.search.aggregations.bucket.missing.Missing;
 import org.elasticsearch.search.aggregations.metrics.avg.Avg;
@@ -142,7 +143,7 @@ public void testWithSubAggregation() throws Exception {
         assertThat(missing, notNullValue());
         assertThat(missing.getName(), equalTo("missing_tag"));
         assertThat(missing.getDocCount(), equalTo((long) numDocsMissing + numDocsUnmapped));
-        assertThat((long) missing.getProperty("_count"), equalTo((long) numDocsMissing + numDocsUnmapped));
+        assertThat((long) ((InternalAggregation)missing).getProperty("_count"), equalTo((long) numDocsMissing + numDocsUnmapped));
         assertThat(missing.getAggregations().asList().isEmpty(), is(false));
 
         long sum = 0;
@@ -156,7 +157,7 @@ public void testWithSubAggregation() throws Exception {
         assertThat(avgValue, notNullValue());
         assertThat(avgValue.getName(), equalTo("avg_value"));
         assertThat(avgValue.getValue(), equalTo((double) sum / (numDocsMissing + numDocsUnmapped)));
-        assertThat((double) missing.getProperty("avg_value.value"), equalTo((double) sum / (numDocsMissing + numDocsUnmapped)));
+        assertThat((double) ((InternalAggregation)missing).getProperty("avg_value.value"), equalTo((double) sum / (numDocsMissing + numDocsUnmapped)));
     }
 
     public void testEmptyAggregation() throws Exception {
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java
index c8b7aa6ad5580..d5f93f0daa704 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java
@@ -24,7 +24,9 @@
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode;
+import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.bucket.filter.Filter;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
 import org.elasticsearch.search.aggregations.bucket.nested.Nested;
@@ -244,7 +246,7 @@ public void testNestedWithSubTermsAgg() throws Exception {
         assertThat(nested, notNullValue());
         assertThat(nested.getName(), equalTo("nested"));
         assertThat(nested.getDocCount(), equalTo(docCount));
-        assertThat(nested.getProperty("_count"), equalTo(docCount));
+        assertThat(((InternalAggregation)nested).getProperty("_count"), equalTo(docCount));
         assertThat(nested.getAggregations().asList().isEmpty(), is(false));
 
         LongTerms values = nested.getAggregations().get("values");
@@ -262,7 +264,7 @@ public void testNestedWithSubTermsAgg() throws Exception {
                 assertEquals(counts[i], bucket.getDocCount());
             }
         }
-        assertThat(nested.getProperty("values"), sameInstance(values));
+        assertThat(((InternalAggregation)nested).getProperty("values"), sameInstance(values));
     }
 
     public void testNestedAsSubAggregation() throws Exception {
@@ -397,8 +399,8 @@ public void testParentFilterResolvedCorrectly() throws Exception {
         ensureGreen("idx2");
 
         List<IndexRequestBuilder> indexRequests = new ArrayList<>(2);
-        indexRequests.add(client().prepareIndex("idx2", "provider", "1").setSource("{\"dates\": {\"month\": {\"label\": \"2014-11\", \"end\": \"2014-11-30\", \"start\": \"2014-11-01\"}, \"day\": \"2014-11-30\"}, \"comments\": [{\"cid\": 3,\"identifier\": \"29111\"}, {\"cid\": 4,\"tags\": [{\"tid\" :44,\"name\": \"Roles\"}], \"identifier\": \"29101\"}]}"));
-        indexRequests.add(client().prepareIndex("idx2", "provider", "2").setSource("{\"dates\": {\"month\": {\"label\": \"2014-12\", \"end\": \"2014-12-31\", \"start\": \"2014-12-01\"}, \"day\": \"2014-12-03\"}, \"comments\": [{\"cid\": 1, \"identifier\": \"29111\"}, {\"cid\": 2,\"tags\": [{\"tid\" : 22, \"name\": \"DataChannels\"}], \"identifier\": \"29101\"}]}"));
+        indexRequests.add(client().prepareIndex("idx2", "provider", "1").setSource("{\"dates\": {\"month\": {\"label\": \"2014-11\", \"end\": \"2014-11-30\", \"start\": \"2014-11-01\"}, \"day\": \"2014-11-30\"}, \"comments\": [{\"cid\": 3,\"identifier\": \"29111\"}, {\"cid\": 4,\"tags\": [{\"tid\" :44,\"name\": \"Roles\"}], \"identifier\": \"29101\"}]}", XContentType.JSON));
+        indexRequests.add(client().prepareIndex("idx2", "provider", "2").setSource("{\"dates\": {\"month\": {\"label\": \"2014-12\", \"end\": \"2014-12-31\", \"start\": \"2014-12-01\"}, \"day\": \"2014-12-03\"}, \"comments\": [{\"cid\": 1, \"identifier\": \"29111\"}, {\"cid\": 2,\"tags\": [{\"tid\" : 22, \"name\": \"DataChannels\"}], \"identifier\": \"29101\"}]}", XContentType.JSON));
         indexRandom(true, indexRequests);
 
         SearchResponse response = client().prepareSearch("idx2").setTypes("provider")
@@ -468,7 +470,7 @@ public void testNestedSameDocIdProcessedMultipleTime() throws Exception {
 
         client().prepareIndex("idx4", "product", "1").setSource(jsonBuilder().startObject()
                     .field("name", "product1")
-                    .field("categories", "1", "2", "3", "4")
+                    .array("categories", "1", "2", "3", "4")
                     .startArray("property")
                         .startObject().field("id", 1).endObject()
                         .startObject().field("id", 2).endObject()
@@ -477,7 +479,7 @@ public void testNestedSameDocIdProcessedMultipleTime() throws Exception {
                 .endObject()).get();
         client().prepareIndex("idx4", "product", "2").setSource(jsonBuilder().startObject()
                 .field("name", "product2")
-                .field("categories", "1", "2")
+                .array("categories", "1", "2")
                 .startArray("property")
                 .startObject().field("id", 1).endObject()
                 .startObject().field("id", 5).endObject()
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/RangeIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/RangeIT.java
index d3e934d875fbe..c2a2405098dab 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/RangeIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/RangeIT.java
@@ -19,12 +19,16 @@
 package org.elasticsearch.search.aggregations.bucket;
 
 import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.aggregations.AggregationTestScriptsPlugin;
 import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode;
+import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
 import org.elasticsearch.search.aggregations.bucket.range.Range;
 import org.elasticsearch.search.aggregations.bucket.range.Range.Bucket;
@@ -36,19 +40,21 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.function.Function;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
-import static org.elasticsearch.script.ScriptService.ScriptType;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.range;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.sum;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.terms;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.core.IsNull.notNullValue;
 import static org.hamcrest.core.IsNull.nullValue;
@@ -331,9 +337,9 @@ public void testSingleValuedFieldWithSubAggregation() throws Exception {
         assertThat(range.getName(), equalTo("range"));
         List<? extends Bucket> buckets = range.getBuckets();
         assertThat(range.getBuckets().size(), equalTo(3));
-        Object[] propertiesKeys = (Object[]) range.getProperty("_key");
-        Object[] propertiesDocCounts = (Object[]) range.getProperty("_count");
-        Object[] propertiesCounts = (Object[]) range.getProperty("sum.value");
+        Object[] propertiesKeys = (Object[]) ((InternalAggregation)range).getProperty("_key");
+        Object[] propertiesDocCounts = (Object[]) ((InternalAggregation)range).getProperty("_count");
+        Object[] propertiesCounts = (Object[]) ((InternalAggregation)range).getProperty("sum.value");
 
         Range.Bucket bucket = buckets.get(0);
         assertThat(bucket, notNullValue());
@@ -391,7 +397,7 @@ public void testSingleValuedFieldWithValueScript() throws Exception {
                 .addAggregation(
                         range("range")
                                 .field(SINGLE_VALUED_FIELD_NAME)
-                                .script(new Script("_value + 1", ScriptType.INLINE, CustomScriptPlugin.NAME, null))
+                                .script(new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_value + 1", Collections.emptyMap()))
                                 .addUnboundedTo(3)
                                 .addRange(3, 6)
                                 .addUnboundedFrom(6))
@@ -511,7 +517,7 @@ public void testMultiValuedFieldWithValueScript() throws Exception {
                 .addAggregation(
                         range("range")
                                 .field(MULTI_VALUED_FIELD_NAME)
-                                .script(new Script("_value + 1", ScriptType.INLINE, CustomScriptPlugin.NAME, null))
+                                .script(new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_value + 1", Collections.emptyMap()))
                                 .addUnboundedTo(3)
                                 .addRange(3, 6)
                                 .addUnboundedFrom(6))
@@ -572,7 +578,8 @@ public void testMultiValuedFieldWithValueScript() throws Exception {
      */
 
     public void testScriptSingleValue() throws Exception {
-        Script script = new Script("doc['" + SINGLE_VALUED_FIELD_NAME + "'].value", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
+        Script script =
+            new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['" + SINGLE_VALUED_FIELD_NAME + "'].value", Collections.emptyMap());
         SearchResponse response = client()
                 .prepareSearch("idx")
                 .addAggregation(
@@ -656,8 +663,23 @@ public void testEmptyRange() throws Exception {
         assertThat(bucket.getDocCount(), equalTo(0L));
     }
 
+    public void testNoRangesInQuery()  {
+        try {
+            client().prepareSearch("idx")
+                .addAggregation(range("foobar")
+                    .field(SINGLE_VALUED_FIELD_NAME))
+                .execute().actionGet();
+            fail();
+        } catch (SearchPhaseExecutionException spee){
+            Throwable rootCause = spee.getCause().getCause();
+            assertThat(rootCause, instanceOf(IllegalArgumentException.class));
+            assertEquals(rootCause.getMessage(), "No [ranges] specified for the [foobar] aggregation");
+        }
+    }
+
     public void testScriptMultiValued() throws Exception {
-        Script script = new Script("doc['" + MULTI_VALUED_FIELD_NAME + "'].values", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
+        Script script =
+            new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['" + MULTI_VALUED_FIELD_NAME + "'].values", Collections.emptyMap());
 
         SearchResponse response = client()
                 .prepareSearch("idx")
@@ -907,4 +929,47 @@ public void testEmptyAggregation() throws Exception {
         assertThat(buckets.get(0).getDocCount(), equalTo(0L));
 
     }
+
+    /**
+     * Make sure that a request using a script does not get cached and a request
+     * not using a script does get cached.
+     */
+    public void testDontCacheScripts() throws Exception {
+        assertAcked(prepareCreate("cache_test_idx").addMapping("type", "i", "type=integer")
+                .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1))
+                .get());
+        indexRandom(true,
+                client().prepareIndex("cache_test_idx", "type", "1").setSource(jsonBuilder().startObject().field("i", 1).endObject()),
+                client().prepareIndex("cache_test_idx", "type", "2").setSource(jsonBuilder().startObject().field("i", 2).endObject()));
+
+        // Make sure we are starting with a clear cache
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(0L));
+
+        // Test that a request using a script does not get cached
+        Map<String, Object> params = new HashMap<>();
+        params.put("fieldname", "date");
+        SearchResponse r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(
+                range("foo").field("i").script(
+                    new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_value + 1", Collections.emptyMap())).addRange(0, 10))
+                .get();
+        assertSearchResponse(r);
+
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(0L));
+
+        // To make sure that the cache is working test that a request not using
+        // a script is cached
+        r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(range("foo").field("i").addRange(0, 10)).get();
+        assertSearchResponse(r);
+
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(1L));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/RangeTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/RangeTests.java
index 22b4eae842182..89bcf4ebc048c 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/RangeTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/RangeTests.java
@@ -32,7 +32,7 @@ protected RangeAggregationBuilder createTestAggregatorBuilder() {
         for (int i = 0; i < numRanges; i++) {
             String key = null;
             if (randomBoolean()) {
-                key = randomAsciiOfLengthBetween(1, 20);
+                key = randomAlphaOfLengthBetween(1, 20);
             }
             double from = randomBoolean() ? Double.NEGATIVE_INFINITY : randomIntBetween(Integer.MIN_VALUE, Integer.MAX_VALUE - 1000);
             double to = randomBoolean() ? Double.POSITIVE_INFINITY
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ReverseNestedIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ReverseNestedIT.java
index db7c301883a48..3fdd32d22bb3b 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ReverseNestedIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ReverseNestedIT.java
@@ -23,13 +23,13 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode;
+import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.bucket.filter.Filter;
 import org.elasticsearch.search.aggregations.bucket.nested.Nested;
 import org.elasticsearch.search.aggregations.bucket.nested.ReverseNested;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCount;
 import org.elasticsearch.test.ESIntegTestCase;
-import org.hamcrest.Matchers;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -167,9 +167,9 @@ public void testSimpleReverseNestedToRoot() throws Exception {
         assertThat(bucket.getKeyAsString(), equalTo("1"));
         assertThat(bucket.getDocCount(), equalTo(6L));
         ReverseNested reverseNested = bucket.getAggregations().get("nested1_to_field1");
-        assertThat(reverseNested.getProperty("_count"), equalTo(5L));
+        assertThat(((InternalAggregation)reverseNested).getProperty("_count"), equalTo(5L));
         Terms tags = reverseNested.getAggregations().get("field1");
-        assertThat(reverseNested.getProperty("field1"), sameInstance(tags));
+        assertThat(((InternalAggregation)reverseNested).getProperty("field1"), sameInstance(tags));
         List<Terms.Bucket> tagsBuckets = new ArrayList<>(tags.getBuckets());
         assertThat(tagsBuckets.size(), equalTo(6));
         assertThat(tagsBuckets.get(0).getKeyAsString(), equalTo("c"));
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SamplerIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SamplerIT.java
index 980d792013a6d..ebd078de67402 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SamplerIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SamplerIT.java
@@ -31,6 +31,7 @@
 import org.elasticsearch.test.ESIntegTestCase;
 
 import java.util.Collection;
+import java.util.List;
 
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
@@ -95,7 +96,7 @@ public void testIssue10719() throws Exception {
         // Tests that we can refer to nested elements under a sample in a path
         // statement
         boolean asc = randomBoolean();
-        SearchResponse response = client().prepareSearch("test").setTypes("book").setSearchType(SearchType.QUERY_AND_FETCH)
+        SearchResponse response = client().prepareSearch("test").setTypes("book").setSearchType(SearchType.QUERY_THEN_FETCH)
                 .addAggregation(terms("genres")
                         .field("genre")
                         .order(Terms.Order.aggregation("sample>max_price.value", asc))
@@ -104,7 +105,7 @@ public void testIssue10719() throws Exception {
                 ).execute().actionGet();
         assertSearchResponse(response);
         Terms genres = response.getAggregations().get("genres");
-        Collection<Bucket> genreBuckets = genres.getBuckets();
+        List<? extends Bucket> genreBuckets = genres.getBuckets();
         // For this test to be useful we need >1 genre bucket to compare
         assertThat(genreBuckets.size(), greaterThan(1));
         double lastMaxPrice = asc ? Double.MIN_VALUE : Double.MAX_VALUE;
@@ -125,12 +126,12 @@ public void testIssue10719() throws Exception {
     public void testSimpleSampler() throws Exception {
         SamplerAggregationBuilder sampleAgg = sampler("sample").shardSize(100);
         sampleAgg.subAggregation(terms("authors").field("author"));
-        SearchResponse response = client().prepareSearch("test").setSearchType(SearchType.QUERY_AND_FETCH)
+        SearchResponse response = client().prepareSearch("test").setSearchType(SearchType.QUERY_THEN_FETCH)
                 .setQuery(new TermQueryBuilder("genre", "fantasy")).setFrom(0).setSize(60).addAggregation(sampleAgg).execute().actionGet();
         assertSearchResponse(response);
         Sampler sample = response.getAggregations().get("sample");
         Terms authors = sample.getAggregations().get("authors");
-        Collection<Bucket> testBuckets = authors.getBuckets();
+        List<? extends Bucket> testBuckets = authors.getBuckets();
 
         long maxBooksPerAuthor = 0;
         for (Terms.Bucket testBucket : testBuckets) {
@@ -143,7 +144,7 @@ public void testUnmappedChildAggNoDiversity() throws Exception {
         SamplerAggregationBuilder sampleAgg = sampler("sample").shardSize(100);
         sampleAgg.subAggregation(terms("authors").field("author"));
         SearchResponse response = client().prepareSearch("idx_unmapped")
-                .setSearchType(SearchType.QUERY_AND_FETCH)
+                .setSearchType(SearchType.QUERY_THEN_FETCH)
                 .setQuery(new TermQueryBuilder("genre", "fantasy"))
                 .setFrom(0).setSize(60)
                 .addAggregation(sampleAgg)
@@ -160,7 +161,7 @@ public void testPartiallyUnmappedChildAggNoDiversity() throws Exception {
         SamplerAggregationBuilder sampleAgg = sampler("sample").shardSize(100);
         sampleAgg.subAggregation(terms("authors").field("author"));
         SearchResponse response = client().prepareSearch("idx_unmapped", "test")
-                .setSearchType(SearchType.QUERY_AND_FETCH)
+                .setSearchType(SearchType.QUERY_THEN_FETCH)
                 .setQuery(new TermQueryBuilder("genre", "fantasy"))
                 .setFrom(0).setSize(60).setExplain(true)
                 .addAggregation(sampleAgg)
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardSizeTermsIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardSizeTermsIT.java
index ecc16b85f1312..748c5f886f669 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardSizeTermsIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardSizeTermsIT.java
@@ -22,8 +22,8 @@
 import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 
-import java.util.Collection;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
@@ -43,7 +43,7 @@ public void testNoShardSizeString() throws Exception {
                 .execute().actionGet();
 
         Terms  terms = response.getAggregations().get("keys");
-        Collection<Terms.Bucket> buckets = terms.getBuckets();
+        List<? extends Terms.Bucket> buckets = terms.getBuckets();
         assertThat(buckets.size(), equalTo(3));
         Map<String, Long> expected = new HashMap<>();
         expected.put("1", 8L);
@@ -66,7 +66,7 @@ public void testShardSizeEqualsSizeString() throws Exception {
                 .execute().actionGet();
 
         Terms  terms = response.getAggregations().get("keys");
-        Collection<Terms.Bucket> buckets = terms.getBuckets();
+        List<? extends Terms.Bucket> buckets = terms.getBuckets();
         assertThat(buckets.size(), equalTo(3));
         Map<String, Long> expected = new HashMap<>();
         expected.put("1", 8L);
@@ -90,7 +90,7 @@ public void testWithShardSizeString() throws Exception {
                 .execute().actionGet();
 
         Terms terms = response.getAggregations().get("keys");
-        Collection<Terms.Bucket> buckets = terms.getBuckets();
+        List<? extends Terms.Bucket> buckets = terms.getBuckets();
         assertThat(buckets.size(), equalTo(3)); // we still only return 3 entries (based on the 'size' param)
         Map<String, Long> expected = new HashMap<>();
         expected.put("1", 8L);
@@ -114,7 +114,7 @@ public void testWithShardSizeStringSingleShard() throws Exception {
                 .execute().actionGet();
 
         Terms terms = response.getAggregations().get("keys");
-        Collection<Terms.Bucket> buckets = terms.getBuckets();
+        List<? extends Terms.Bucket> buckets = terms.getBuckets();
         assertThat(buckets.size(), equalTo(3)); // we still only return 3 entries (based on the 'size' param)
         Map<String, Long> expected = new HashMap<>();
         expected.put("1", 5L);
@@ -137,7 +137,7 @@ public void testNoShardSizeTermOrderString() throws Exception {
                 .execute().actionGet();
 
         Terms  terms = response.getAggregations().get("keys");
-        Collection<Terms.Bucket> buckets = terms.getBuckets();
+        List<? extends Terms.Bucket> buckets = terms.getBuckets();
         assertThat(buckets.size(), equalTo(3));
         Map<String, Long> expected = new HashMap<>();
         expected.put("1", 8L);
@@ -160,7 +160,7 @@ public void testNoShardSizeLong() throws Exception {
                 .execute().actionGet();
 
         Terms terms = response.getAggregations().get("keys");
-        Collection<Terms.Bucket> buckets = terms.getBuckets();
+        List<? extends Terms.Bucket> buckets = terms.getBuckets();
         assertThat(buckets.size(), equalTo(3));
         Map<Integer, Long> expected = new HashMap<>();
         expected.put(1, 8L);
@@ -183,7 +183,7 @@ public void testShardSizeEqualsSizeLong() throws Exception {
                 .execute().actionGet();
 
         Terms terms = response.getAggregations().get("keys");
-        Collection<Terms.Bucket> buckets = terms.getBuckets();
+        List<? extends Terms.Bucket> buckets = terms.getBuckets();
         assertThat(buckets.size(), equalTo(3));
         Map<Integer, Long> expected = new HashMap<>();
         expected.put(1, 8L);
@@ -206,7 +206,7 @@ public void testWithShardSizeLong() throws Exception {
                 .execute().actionGet();
 
         Terms terms = response.getAggregations().get("keys");
-        Collection<Terms.Bucket> buckets = terms.getBuckets();
+        List<? extends Terms.Bucket> buckets = terms.getBuckets();
         assertThat(buckets.size(), equalTo(3)); // we still only return 3 entries (based on the 'size' param)
         Map<Integer, Long> expected = new HashMap<>();
         expected.put(1, 8L);
@@ -230,7 +230,7 @@ public void testWithShardSizeLongSingleShard() throws Exception {
                 .execute().actionGet();
 
         Terms terms = response.getAggregations().get("keys");
-        Collection<Terms.Bucket> buckets = terms.getBuckets();
+        List<? extends Terms.Bucket> buckets = terms.getBuckets();
         assertThat(buckets.size(), equalTo(3)); // we still only return 3 entries (based on the 'size' param)
         Map<Integer, Long> expected = new HashMap<>();
         expected.put(1, 5L);
@@ -253,7 +253,7 @@ public void testNoShardSizeTermOrderLong() throws Exception {
                 .execute().actionGet();
 
         Terms terms = response.getAggregations().get("keys");
-        Collection<Terms.Bucket> buckets = terms.getBuckets();
+        List<? extends Terms.Bucket> buckets = terms.getBuckets();
         assertThat(buckets.size(), equalTo(3));
         Map<Integer, Long> expected = new HashMap<>();
         expected.put(1, 8L);
@@ -276,7 +276,7 @@ public void testNoShardSizeDouble() throws Exception {
                 .execute().actionGet();
 
         Terms terms = response.getAggregations().get("keys");
-        Collection<Terms.Bucket> buckets = terms.getBuckets();
+        List<? extends Terms.Bucket> buckets = terms.getBuckets();
         assertThat(buckets.size(), equalTo(3));
         Map<Integer, Long> expected = new HashMap<>();
         expected.put(1, 8L);
@@ -299,7 +299,7 @@ public void testShardSizeEqualsSizeDouble() throws Exception {
                 .execute().actionGet();
 
         Terms terms = response.getAggregations().get("keys");
-        Collection<Terms.Bucket> buckets = terms.getBuckets();
+        List<? extends Terms.Bucket> buckets = terms.getBuckets();
         assertThat(buckets.size(), equalTo(3));
         Map<Integer, Long> expected = new HashMap<>();
         expected.put(1, 8L);
@@ -322,7 +322,7 @@ public void testWithShardSizeDouble() throws Exception {
                 .execute().actionGet();
 
         Terms terms = response.getAggregations().get("keys");
-        Collection<Terms.Bucket> buckets = terms.getBuckets();
+        List<? extends Terms.Bucket> buckets = terms.getBuckets();
         assertThat(buckets.size(), equalTo(3));
         Map<Integer, Long> expected = new HashMap<>();
         expected.put(1, 8L);
@@ -345,7 +345,7 @@ public void testWithShardSizeDoubleSingleShard() throws Exception {
                 .execute().actionGet();
 
         Terms terms = response.getAggregations().get("keys");
-        Collection<Terms.Bucket> buckets = terms.getBuckets();
+        List<? extends Terms.Bucket> buckets = terms.getBuckets();
         assertThat(buckets.size(), equalTo(3));
         Map<Integer, Long> expected = new HashMap<>();
         expected.put(1, 5L);
@@ -368,7 +368,7 @@ public void testNoShardSizeTermOrderDouble() throws Exception {
                 .execute().actionGet();
 
         Terms terms = response.getAggregations().get("keys");
-        Collection<Terms.Bucket> buckets = terms.getBuckets();
+        List<? extends Terms.Bucket> buckets = terms.getBuckets();
         assertThat(buckets.size(), equalTo(3));
         Map<Integer, Long> expected = new HashMap<>();
         expected.put(1, 8L);
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsIT.java
index 7284901e94c22..d94a2cfd44921 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsIT.java
@@ -130,7 +130,7 @@ public void setupSuiteScopeCluster() throws Exception {
 
     public void testStructuredAnalysis() throws Exception {
         SearchResponse response = client().prepareSearch("test")
-                .setSearchType(SearchType.QUERY_AND_FETCH)
+                .setSearchType(SearchType.QUERY_THEN_FETCH)
                 .setQuery(new TermQueryBuilder("_all", "terje"))
                 .setFrom(0).setSize(60).setExplain(true)
                 .addAggregation(significantTerms("mySignificantTerms").field("fact_category").executionHint(randomExecutionHint())
@@ -146,7 +146,7 @@ public void testStructuredAnalysis() throws Exception {
     public void testStructuredAnalysisWithIncludeExclude() throws Exception {
         long[] excludeTerms = { MUSIC_CATEGORY };
         SearchResponse response = client().prepareSearch("test")
-                .setSearchType(SearchType.QUERY_AND_FETCH)
+                .setSearchType(SearchType.QUERY_THEN_FETCH)
                 .setQuery(new TermQueryBuilder("_all", "paul"))
                 .setFrom(0).setSize(60).setExplain(true)
                 .addAggregation(significantTerms("mySignificantTerms").field("fact_category").executionHint(randomExecutionHint())
@@ -226,7 +226,7 @@ public void testIncludeExcludeExactValues() throws Exception {
 
     public void testUnmapped() throws Exception {
         SearchResponse response = client().prepareSearch("idx_unmapped")
-                .setSearchType(SearchType.QUERY_AND_FETCH)
+                .setSearchType(SearchType.QUERY_THEN_FETCH)
                 .setQuery(new TermQueryBuilder("_all", "terje"))
                 .setFrom(0).setSize(60).setExplain(true)
                 .addAggregation(significantTerms("mySignificantTerms").field("fact_category").executionHint(randomExecutionHint())
@@ -240,7 +240,7 @@ public void testUnmapped() throws Exception {
 
     public void testTextAnalysis() throws Exception {
         SearchResponse response = client().prepareSearch("test")
-                .setSearchType(SearchType.QUERY_AND_FETCH)
+                .setSearchType(SearchType.QUERY_THEN_FETCH)
                 .setQuery(new TermQueryBuilder("_all", "terje"))
                 .setFrom(0).setSize(60).setExplain(true)
                 .addAggregation(significantTerms("mySignificantTerms").field("description").executionHint(randomExecutionHint())
@@ -254,7 +254,7 @@ public void testTextAnalysis() throws Exception {
 
     public void testTextAnalysisGND() throws Exception {
         SearchResponse response = client().prepareSearch("test")
-                .setSearchType(SearchType.QUERY_AND_FETCH)
+                .setSearchType(SearchType.QUERY_THEN_FETCH)
                 .setQuery(new TermQueryBuilder("_all", "terje"))
                 .setFrom(0).setSize(60).setExplain(true)
                 .addAggregation(significantTerms("mySignificantTerms").field("description").executionHint(randomExecutionHint()).significanceHeuristic(new GND(true))
@@ -268,7 +268,7 @@ public void testTextAnalysisGND() throws Exception {
 
     public void testTextAnalysisChiSquare() throws Exception {
         SearchResponse response = client().prepareSearch("test")
-                .setSearchType(SearchType.QUERY_AND_FETCH)
+                .setSearchType(SearchType.QUERY_THEN_FETCH)
                 .setQuery(new TermQueryBuilder("_all", "terje"))
                 .setFrom(0).setSize(60).setExplain(true)
                 .addAggregation(significantTerms("mySignificantTerms").field("description").executionHint(randomExecutionHint()).significanceHeuristic(new ChiSquare(false,true))
@@ -283,7 +283,7 @@ public void testTextAnalysisChiSquare() throws Exception {
     public void testTextAnalysisPercentageScore() throws Exception {
         SearchResponse response = client()
                 .prepareSearch("test")
-                .setSearchType(SearchType.QUERY_AND_FETCH)
+                .setSearchType(SearchType.QUERY_THEN_FETCH)
                 .setQuery(new TermQueryBuilder("_all", "terje"))
                 .setFrom(0)
                 .setSize(60)
@@ -302,7 +302,7 @@ public void testBadFilteredAnalysis() throws Exception {
         // We search for the name of a snowboarder but use music-related content (fact_category:1)
         // as the background source of term statistics.
         SearchResponse response = client().prepareSearch("test")
-                .setSearchType(SearchType.QUERY_AND_FETCH)
+                .setSearchType(SearchType.QUERY_THEN_FETCH)
                 .setQuery(new TermQueryBuilder("_all", "terje"))
                 .setFrom(0).setSize(60).setExplain(true)
                 .addAggregation(significantTerms("mySignificantTerms").field("description")
@@ -326,7 +326,7 @@ public void testBadFilteredAnalysis() throws Exception {
 
     public void testFilteredAnalysis() throws Exception {
         SearchResponse response = client().prepareSearch("test")
-                .setSearchType(SearchType.QUERY_AND_FETCH)
+                .setSearchType(SearchType.QUERY_THEN_FETCH)
                 .setQuery(new TermQueryBuilder("_all", "weller"))
                 .setFrom(0).setSize(60).setExplain(true)
                 .addAggregation(significantTerms("mySignificantTerms").field("description")
@@ -351,7 +351,7 @@ public void testNestedAggs() throws Exception {
                 { "paul", "smith" },
                 { "craig", "kelly", "terje", "haakonsen", "burton" }};
         SearchResponse response = client().prepareSearch("test")
-                .setSearchType(SearchType.QUERY_AND_FETCH)
+                .setSearchType(SearchType.QUERY_THEN_FETCH)
                 .addAggregation(terms("myCategories").field("fact_category").minDocCount(2)
                         .subAggregation(
                                    significantTerms("mySignificantTerms").field("description")
@@ -376,7 +376,7 @@ public void testNestedAggs() throws Exception {
 
     public void testPartiallyUnmapped() throws Exception {
         SearchResponse response = client().prepareSearch("idx_unmapped", "test")
-                .setSearchType(SearchType.QUERY_AND_FETCH)
+                .setSearchType(SearchType.QUERY_THEN_FETCH)
                 .setQuery(new TermQueryBuilder("_all", "terje"))
                 .setFrom(0).setSize(60).setExplain(true)
                 .addAggregation(significantTerms("mySignificantTerms").field("description")
@@ -391,7 +391,7 @@ public void testPartiallyUnmapped() throws Exception {
 
     public void testPartiallyUnmappedWithFormat() throws Exception {
         SearchResponse response = client().prepareSearch("idx_unmapped", "test")
-                .setSearchType(SearchType.QUERY_AND_FETCH)
+                .setSearchType(SearchType.QUERY_THEN_FETCH)
                 .setQuery(boolQuery().should(termQuery("_all", "the")).should(termQuery("_all", "terje")))
                 .setFrom(0).setSize(60).setExplain(true)
                 .addAggregation(significantTerms("mySignificantTerms")
@@ -428,7 +428,7 @@ private void checkExpectedStringTermsFound(SignificantTerms topTerms) {
 
     public void testDefaultSignificanceHeuristic() throws Exception {
         SearchResponse response = client().prepareSearch("test")
-                .setSearchType(SearchType.QUERY_AND_FETCH)
+                .setSearchType(SearchType.QUERY_THEN_FETCH)
                 .setQuery(new TermQueryBuilder("_all", "terje"))
                 .setFrom(0).setSize(60).setExplain(true)
                 .addAggregation(significantTerms("mySignificantTerms")
@@ -445,7 +445,7 @@ public void testDefaultSignificanceHeuristic() throws Exception {
 
     public void testMutualInformation() throws Exception {
         SearchResponse response = client().prepareSearch("test")
-                .setSearchType(SearchType.QUERY_AND_FETCH)
+                .setSearchType(SearchType.QUERY_THEN_FETCH)
                 .setQuery(new TermQueryBuilder("_all", "terje"))
                 .setFrom(0).setSize(60).setExplain(true)
                 .addAggregation(significantTerms("mySignificantTerms")
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java
index fab1f8b7d3ed5..1b34a4b659856 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java
@@ -22,16 +22,20 @@
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardException;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.ScriptPlugin;
 import org.elasticsearch.plugins.SearchPlugin;
 import org.elasticsearch.script.NativeScriptFactory;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.aggregations.Aggregation;
 import org.elasticsearch.search.aggregations.Aggregations;
 import org.elasticsearch.search.aggregations.bucket.filter.InternalFilter;
@@ -47,7 +51,6 @@
 import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristicParser;
 import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
-import org.elasticsearch.search.aggregations.support.XContentParseContext;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.search.aggregations.bucket.SharedSignificantTermsTestMethods;
 
@@ -55,6 +58,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
@@ -238,9 +242,9 @@ public double getScore(long subsetFreq, long subsetSize, long supersetFreq, long
             return subsetFreq / subsetSize > supersetFreq / supersetSize ? 2.0 : 1.0;
         }
 
-        public static SignificanceHeuristic parse(XContentParseContext context)
+        public static SignificanceHeuristic parse(QueryParseContext context)
                 throws IOException, QueryShardException {
-            context.getParser().nextToken();
+            context.parser().nextToken();
             return new SimpleHeuristic();
         }
     }
@@ -268,7 +272,7 @@ public void testXContentResponse() throws Exception {
 
         XContentBuilder responseBuilder = XContentFactory.jsonBuilder();
         responseBuilder.startObject();
-        classes.toXContent(responseBuilder, null);
+        classes.toXContent(responseBuilder, ToXContent.EMPTY_PARAMS);
         responseBuilder.endObject();
 
         String result = "{\"class\":{\"doc_count_error_upper_bound\":0,\"sum_other_doc_count\":0,"
@@ -278,6 +282,7 @@ public void testXContentResponse() throws Exception {
                 + "\"doc_count\":4,"
                 + "\"sig_terms\":{"
                 + "\"doc_count\":4,"
+                + "\"bg_count\":7,"
                 + "\"buckets\":["
                 + "{"
                 + "\"key\":" + (type.equals("long") ? "0," : "\"0\",")
@@ -293,6 +298,7 @@ public void testXContentResponse() throws Exception {
                 + "\"doc_count\":3,"
                 + "\"sig_terms\":{"
                 + "\"doc_count\":3,"
+                + "\"bg_count\":7,"
                 + "\"buckets\":["
                 + "{"
                 + "\"key\":" + (type.equals("long") ? "1," : "\"1\",")
@@ -306,7 +312,7 @@ public void testXContentResponse() throws Exception {
 
     public void testDeletesIssue7951() throws Exception {
         String settings = "{\"index.number_of_shards\": 1, \"index.number_of_replicas\": 0}";
-        assertAcked(prepareCreate(INDEX_NAME).setSettings(settings)
+        assertAcked(prepareCreate(INDEX_NAME).setSettings(settings, XContentType.JSON)
                 .addMapping("doc", "text", "type=keyword", CLASS_FIELD, "type=keyword"));
         String[] cat1v1 = {"constant", "one"};
         String[] cat1v2 = {"constant", "uno"};
@@ -437,13 +443,13 @@ public void testScoresEqualForPositiveAndNegative(SignificanceHeuristic heuristi
         assertSearchResponse(response);
         StringTerms classes = response.getAggregations().get("class");
         assertThat(classes.getBuckets().size(), equalTo(2));
-        Iterator<Terms.Bucket> classBuckets = classes.getBuckets().iterator();
+        Iterator<? extends Terms.Bucket> classBuckets = classes.getBuckets().iterator();
 
         Aggregations aggregations = classBuckets.next().getAggregations();
         SignificantTerms sigTerms = aggregations.get("mySignificantTerms");
 
-        Collection<SignificantTerms.Bucket> classA = sigTerms.getBuckets();
-        Iterator<SignificantTerms.Bucket> classBBucketIterator = sigTerms.getBuckets().iterator();
+        List<? extends SignificantTerms.Bucket> classA = sigTerms.getBuckets();
+        Iterator<SignificantTerms.Bucket> classBBucketIterator = sigTerms.iterator();
         assertThat(classA.size(), greaterThan(0));
         for (SignificantTerms.Bucket classABucket : classA) {
             SignificantTerms.Bucket classBBucket = classBBucketIterator.next();
@@ -509,14 +515,13 @@ public void testScriptScore() throws ExecutionException, InterruptedException, I
     }
 
     private ScriptHeuristic getScriptSignificanceHeuristic() throws IOException {
-        Script script = null;
+        Script script;
         if (randomBoolean()) {
-            Map<String, Object> params = null;
-            params = new HashMap<>();
+            Map<String, Object> params = new HashMap<>();
             params.put("param", randomIntBetween(1, 100));
-            script = new Script("native_significance_score_script_with_params", ScriptType.INLINE, "native", params);
+            script = new Script(ScriptType.INLINE, "native", "native_significance_score_script_with_params", params);
         } else {
-            script = new Script("native_significance_score_script_no_params", ScriptType.INLINE, "native", null);
+            script = new Script(ScriptType.INLINE, "native", "native_significance_score_script_no_params", Collections.emptyMap());
         }
         return new ScriptHeuristic(script);
     }
@@ -547,4 +552,43 @@ public void testReduceFromSeveralShards() throws IOException, ExecutionException
         SharedSignificantTermsTestMethods.aggregateAndCheckFromSeveralShards(this);
     }
 
+    /**
+     * Make sure that a request using a script does not get cached and a request
+     * not using a script does get cached.
+     */
+    public void testDontCacheScripts() throws Exception {
+        assertAcked(prepareCreate("cache_test_idx").addMapping("type", "d", "type=long")
+                .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1))
+                .get());
+        indexRandom(true, client().prepareIndex("cache_test_idx", "type", "1").setSource("s", 1),
+                client().prepareIndex("cache_test_idx", "type", "2").setSource("s", 2));
+
+        // Make sure we are starting with a clear cache
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(0L));
+
+        // Test that a request using a script does not get cached
+        ScriptHeuristic scriptHeuristic = getScriptSignificanceHeuristic();
+        SearchResponse r = client().prepareSearch("cache_test_idx").setSize(0)
+                .addAggregation(significantTerms("foo").field("s").significanceHeuristic(scriptHeuristic)).get();
+        assertSearchResponse(r);
+
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(0L));
+
+        // To make sure that the cache is working test that a request not using
+        // a script is cached
+        r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(significantTerms("foo").field("s")).get();
+        assertSearchResponse(r);
+
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(1L));
+    }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsTests.java
index c8434924a3898..9fe1c0ea479e0 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsTests.java
@@ -22,7 +22,6 @@
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.automaton.RegExp;
 import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.script.Script;
 import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
 import org.elasticsearch.search.aggregations.bucket.significant.SignificantTermsAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.significant.heuristics.ChiSquare;
@@ -34,6 +33,7 @@
 import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristic;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorFactory.ExecutionMode;
 import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude;
+
 import java.util.SortedSet;
 import java.util.TreeSet;
 
@@ -51,24 +51,11 @@ public class SignificantTermsTests extends BaseAggregationTestCase<SignificantTe
 
     @Override
     protected SignificantTermsAggregationBuilder createTestAggregatorBuilder() {
-        String name = randomAsciiOfLengthBetween(3, 20);
+        String name = randomAlphaOfLengthBetween(3, 20);
         SignificantTermsAggregationBuilder factory = new SignificantTermsAggregationBuilder(name, null);
-        String field = randomAsciiOfLengthBetween(3, 20);
-        int randomFieldBranch = randomInt(2);
-        switch (randomFieldBranch) {
-        case 0:
-            factory.field(field);
-            break;
-        case 1:
-            factory.field(field);
-            factory.script(new Script("_value + 1"));
-            break;
-        case 2:
-            factory.script(new Script("doc[" + field + "] + 1"));
-            break;
-        default:
-            fail();
-        }
+        String field = randomAlphaOfLengthBetween(3, 20);
+        randomFieldOrScript(factory, field);
+
         if (randomBoolean()) {
             factory.missing("MISSING");
         }
@@ -88,7 +75,7 @@ protected SignificantTermsAggregationBuilder createTestAggregatorBuilder() {
             case 2:
             case 3:
             case 4:
-                minDocCount = randomInt();
+                minDocCount = randomIntBetween(0, Integer.MAX_VALUE);
                 break;
             }
             factory.bucketCountThresholds().setMinDocCount(minDocCount);
@@ -102,7 +89,7 @@ protected SignificantTermsAggregationBuilder createTestAggregatorBuilder() {
             case 2:
             case 3:
             case 4:
-                shardMinDocCount = randomInt();
+                shardMinDocCount = randomIntBetween(0, Integer.MAX_VALUE);
                 break;
             default:
                 fail();
@@ -131,7 +118,7 @@ protected SignificantTermsAggregationBuilder createTestAggregatorBuilder() {
                 SortedSet<BytesRef> includeValues = new TreeSet<>();
                 int numIncs = randomIntBetween(1, 20);
                 for (int i = 0; i < numIncs; i++) {
-                    includeValues.add(new BytesRef(randomAsciiOfLengthBetween(1, 30)));
+                    includeValues.add(new BytesRef(randomAlphaOfLengthBetween(1, 30)));
                 }
                 SortedSet<BytesRef> excludeValues = null;
                 incExc = new IncludeExclude(includeValues, excludeValues);
@@ -141,7 +128,7 @@ protected SignificantTermsAggregationBuilder createTestAggregatorBuilder() {
                 SortedSet<BytesRef> excludeValues2 = new TreeSet<>();
                 int numExcs2 = randomIntBetween(1, 20);
                 for (int i = 0; i < numExcs2; i++) {
-                    excludeValues2.add(new BytesRef(randomAsciiOfLengthBetween(1, 30)));
+                    excludeValues2.add(new BytesRef(randomAlphaOfLengthBetween(1, 30)));
                 }
                 incExc = new IncludeExclude(includeValues2, excludeValues2);
                 break;
@@ -149,12 +136,12 @@ protected SignificantTermsAggregationBuilder createTestAggregatorBuilder() {
                 SortedSet<BytesRef> includeValues3 = new TreeSet<>();
                 int numIncs3 = randomIntBetween(1, 20);
                 for (int i = 0; i < numIncs3; i++) {
-                    includeValues3.add(new BytesRef(randomAsciiOfLengthBetween(1, 30)));
+                    includeValues3.add(new BytesRef(randomAlphaOfLengthBetween(1, 30)));
                 }
                 SortedSet<BytesRef> excludeValues3 = new TreeSet<>();
                 int numExcs3 = randomIntBetween(1, 20);
                 for (int i = 0; i < numExcs3; i++) {
-                    excludeValues3.add(new BytesRef(randomAsciiOfLengthBetween(1, 30)));
+                    excludeValues3.add(new BytesRef(randomAlphaOfLengthBetween(1, 30)));
                 }
                 incExc = new IncludeExclude(includeValues3, excludeValues3);
                 break;
@@ -179,7 +166,7 @@ protected SignificantTermsAggregationBuilder createTestAggregatorBuilder() {
                 significanceHeuristic = new MutualInformation(randomBoolean(), randomBoolean());
                 break;
             case 4:
-                significanceHeuristic = new ScriptHeuristic(new Script("foo"));
+                significanceHeuristic = new ScriptHeuristic(mockScript("foo"));
                 break;
             case 5:
                 significanceHeuristic = new JLHScore();
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/StringTermsIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/StringTermsIT.java
index 8ccccf3f33abb..06c635d942ef9 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/StringTermsIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/StringTermsIT.java
@@ -24,17 +24,21 @@
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.index.mapper.IndexFieldMapper;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.aggregations.AggregationExecutionException;
 import org.elasticsearch.search.aggregations.AggregationTestScriptsPlugin;
 import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode;
+import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.bucket.filter.Filter;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
+import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorFactory.ExecutionMode;
 import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude;
 import org.elasticsearch.search.aggregations.metrics.avg.Avg;
@@ -52,16 +56,17 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Set;
 import java.util.function.Function;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
-import static org.elasticsearch.script.ScriptService.ScriptType;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.avg;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.count;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.extendedStats;
@@ -252,8 +257,8 @@ public void testSingleValueField() throws Exception {
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
         assertThat(terms.getBuckets().size(), equalTo(5));
-        Object[] propertiesKeys = (Object[]) terms.getProperty("_key");
-        Object[] propertiesDocCounts = (Object[]) terms.getProperty("_count");
+        Object[] propertiesKeys = (Object[]) ((InternalAggregation)terms).getProperty("_key");
+        Object[] propertiesDocCounts = (Object[]) ((InternalAggregation)terms).getProperty("_count");
 
         for (int i = 0; i < 5; i++) {
             Terms.Bucket bucket = terms.getBucketByKey("val" + i);
@@ -454,6 +459,44 @@ public void testSingleValueFieldWithExactTermFiltering() throws Exception {
         }
 
     }
+        
+    public void testSingleValueFieldWithPartitionedFiltering() throws Exception {
+        runTestFieldWithPartitionedFiltering(SINGLE_VALUED_FIELD_NAME);
+    }
+    
+    public void testMultiValueFieldWithPartitionedFiltering() throws Exception {
+        runTestFieldWithPartitionedFiltering(MULTI_VALUED_FIELD_NAME);
+    }
+    
+    private void runTestFieldWithPartitionedFiltering(String field) throws Exception {
+        // Find total number of unique terms
+        SearchResponse allResponse = client().prepareSearch("idx").setTypes("type")
+                .addAggregation(terms("terms").field(field).size(10000).collectMode(randomFrom(SubAggCollectionMode.values())))
+                .execute().actionGet();
+        assertSearchResponse(allResponse);
+        Terms terms = allResponse.getAggregations().get("terms");
+        assertThat(terms, notNullValue());
+        assertThat(terms.getName(), equalTo("terms"));
+        int expectedCardinality = terms.getBuckets().size();
+
+        // Gather terms using partitioned aggregations
+        final int numPartitions = randomIntBetween(2, 4);
+        Set<String> foundTerms = new HashSet<>();
+        for (int partition = 0; partition < numPartitions; partition++) {
+            SearchResponse response = client().prepareSearch("idx").setTypes("type").addAggregation(terms("terms").field(field)
+                    .includeExclude(new IncludeExclude(partition, numPartitions)).collectMode(randomFrom(SubAggCollectionMode.values())))
+                    .execute().actionGet();
+            assertSearchResponse(response);
+            terms = response.getAggregations().get("terms");
+            assertThat(terms, notNullValue());
+            assertThat(terms.getName(), equalTo("terms"));
+            for (Bucket bucket : terms.getBuckets()) {
+                assertTrue(foundTerms.add(bucket.getKeyAsString()));
+            }
+        }
+        assertEquals(expectedCardinality, foundTerms.size());
+    }    
+    
 
     public void testSingleValueFieldWithMaxSize() throws Exception {
         SearchResponse response = client()
@@ -547,9 +590,9 @@ public void testSingleValuedFieldWithSubAggregation() throws Exception {
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
         assertThat(terms.getBuckets().size(), equalTo(5));
-        Object[] propertiesKeys = (Object[]) terms.getProperty("_key");
-        Object[] propertiesDocCounts = (Object[]) terms.getProperty("_count");
-        Object[] propertiesCounts = (Object[]) terms.getProperty("count.value");
+        Object[] propertiesKeys = (Object[]) ((InternalAggregation)terms).getProperty("_key");
+        Object[] propertiesDocCounts = (Object[]) ((InternalAggregation)terms).getProperty("_count");
+        Object[] propertiesCounts = (Object[]) ((InternalAggregation)terms).getProperty("count.value");
 
         for (int i = 0; i < 5; i++) {
             Terms.Bucket bucket = terms.getBucketByKey("val" + i);
@@ -574,7 +617,7 @@ public void testSingleValuedFieldWithValueScript() throws Exception {
                                 .executionHint(randomExecutionHint())
                                 .field(SINGLE_VALUED_FIELD_NAME)
                                 .collectMode(randomFrom(SubAggCollectionMode.values()))
-                                .script(new Script("'foo_' + _value", ScriptType.INLINE, CustomScriptPlugin.NAME, null)))
+                                .script(new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "'foo_' + _value", Collections.emptyMap())))
                 .get();
 
         assertSearchResponse(response);
@@ -601,7 +644,8 @@ public void testMultiValuedFieldWithValueScriptNotUnique() throws Exception {
                                 .executionHint(randomExecutionHint())
                                 .field(MULTI_VALUED_FIELD_NAME)
                                 .collectMode(randomFrom(SubAggCollectionMode.values()))
-                                .script(new Script("_value.substring(0,3)", ScriptType.INLINE, CustomScriptPlugin.NAME, null)))
+                                .script(new Script(
+                                    ScriptType.INLINE, CustomScriptPlugin.NAME, "_value.substring(0,3)", Collections.emptyMap())))
                 .get();
 
         assertSearchResponse(response);
@@ -651,7 +695,8 @@ public void testMultiValuedScript() throws Exception {
                 .addAggregation(
                         terms("terms")
                             .executionHint(randomExecutionHint())
-                            .script(new Script("doc['" + MULTI_VALUED_FIELD_NAME + "']", ScriptType.INLINE, CustomScriptPlugin.NAME, null))
+                            .script(new Script(ScriptType.INLINE,
+                                CustomScriptPlugin.NAME, "doc['" + MULTI_VALUED_FIELD_NAME + "']", Collections.emptyMap()))
                             .collectMode(randomFrom(SubAggCollectionMode.values())))
                 .get();
 
@@ -683,7 +728,7 @@ public void testMultiValuedFieldWithValueScript() throws Exception {
                                 .executionHint(randomExecutionHint())
                                 .field(MULTI_VALUED_FIELD_NAME)
                                 .collectMode(randomFrom(SubAggCollectionMode.values()))
-                                .script(new Script("'foo_' + _value", ScriptType.INLINE, CustomScriptPlugin.NAME, null)))
+                                .script(new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "'foo_' + _value", Collections.emptyMap())))
                 .get();
 
         assertSearchResponse(response);
@@ -718,7 +763,8 @@ public void testMultiValuedFieldWithValueScript() throws Exception {
      */
 
     public void testScriptSingleValue() throws Exception {
-        Script script = new Script("doc['" + SINGLE_VALUED_FIELD_NAME + "'].value", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
+        Script script =
+            new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['" + SINGLE_VALUED_FIELD_NAME + "'].value", Collections.emptyMap());
 
         SearchResponse response = client()
                 .prepareSearch("idx")
@@ -746,7 +792,8 @@ public void testScriptSingleValue() throws Exception {
     }
 
     public void testScriptSingleValueExplicitSingleValue() throws Exception {
-        Script script = new Script("doc['" + SINGLE_VALUED_FIELD_NAME + "'].value", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
+        Script script =
+            new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['" + SINGLE_VALUED_FIELD_NAME + "'].value", Collections.emptyMap());
 
         SearchResponse response = client()
                 .prepareSearch("idx")
@@ -781,7 +828,8 @@ public void testScriptMultiValued() throws Exception {
                         terms("terms")
                             .collectMode(randomFrom(SubAggCollectionMode.values()))
                             .executionHint(randomExecutionHint())
-                            .script(new Script("doc['" + MULTI_VALUED_FIELD_NAME + "']", ScriptType.INLINE, CustomScriptPlugin.NAME, null)))
+                            .script(new Script(ScriptType.INLINE,
+                                CustomScriptPlugin.NAME, "doc['" + MULTI_VALUED_FIELD_NAME + "']", Collections.emptyMap())))
                 .get();
 
         assertSearchResponse(response);
@@ -968,7 +1016,7 @@ public void testSingleValuedFieldOrderedBySingleBucketSubAggregationAsc() throws
         assertThat(tags.getName(), equalTo("tags"));
         assertThat(tags.getBuckets().size(), equalTo(2));
 
-        Iterator<Terms.Bucket> iters = tags.getBuckets().iterator();
+        Iterator<? extends Terms.Bucket> iters = tags.getBuckets().iterator();
 
         Terms.Bucket tag = iters.next();
         assertThat(tag, notNullValue());
@@ -1010,7 +1058,7 @@ public void testSingleValuedFieldOrderedBySubAggregationAscMultiHierarchyLevels(
         assertThat(tags.getName(), equalTo("tags"));
         assertThat(tags.getBuckets().size(), equalTo(2));
 
-        Iterator<Terms.Bucket> iters = tags.getBuckets().iterator();
+        Iterator<? extends Terms.Bucket> iters = tags.getBuckets().iterator();
 
         // the max for "more" is 2
         // the max for "less" is 4
@@ -1046,10 +1094,10 @@ public void testSingleValuedFieldOrderedBySubAggregationAscMultiHierarchyLevels(
 
     public void testSingleValuedFieldOrderedBySubAggregationAscMultiHierarchyLevelsSpecialChars() throws Exception {
         StringBuilder filter2NameBuilder = new StringBuilder("filt.er2");
-        filter2NameBuilder.append(randomAsciiOfLengthBetween(3, 10).replace("[", "").replace("]", "").replace(">", ""));
+        filter2NameBuilder.append(randomAlphaOfLengthBetween(3, 10).replace("[", "").replace("]", "").replace(">", ""));
         String filter2Name = filter2NameBuilder.toString();
         StringBuilder statsNameBuilder = new StringBuilder("st.ats");
-        statsNameBuilder.append(randomAsciiOfLengthBetween(3, 10).replace("[", "").replace("]", "").replace(">", ""));
+        statsNameBuilder.append(randomAlphaOfLengthBetween(3, 10).replace("[", "").replace("]", "").replace(">", ""));
         String statsName = statsNameBuilder.toString();
         boolean asc = randomBoolean();
         SearchResponse response = client()
@@ -1073,7 +1121,7 @@ public void testSingleValuedFieldOrderedBySubAggregationAscMultiHierarchyLevelsS
         assertThat(tags.getName(), equalTo("tags"));
         assertThat(tags.getBuckets().size(), equalTo(2));
 
-        Iterator<Terms.Bucket> iters = tags.getBuckets().iterator();
+        Iterator<? extends Terms.Bucket> iters = tags.getBuckets().iterator();
 
         // the max for "more" is 2
         // the max for "less" is 4
@@ -1109,10 +1157,10 @@ public void testSingleValuedFieldOrderedBySubAggregationAscMultiHierarchyLevelsS
 
     public void testSingleValuedFieldOrderedBySubAggregationAscMultiHierarchyLevelsSpecialCharsNoDotNotation() throws Exception {
         StringBuilder filter2NameBuilder = new StringBuilder("filt.er2");
-        filter2NameBuilder.append(randomAsciiOfLengthBetween(3, 10).replace("[", "").replace("]", "").replace(">", ""));
+        filter2NameBuilder.append(randomAlphaOfLengthBetween(3, 10).replace("[", "").replace("]", "").replace(">", ""));
         String filter2Name = filter2NameBuilder.toString();
         StringBuilder statsNameBuilder = new StringBuilder("st.ats");
-        statsNameBuilder.append(randomAsciiOfLengthBetween(3, 10).replace("[", "").replace("]", "").replace(">", ""));
+        statsNameBuilder.append(randomAlphaOfLengthBetween(3, 10).replace("[", "").replace("]", "").replace(">", ""));
         String statsName = statsNameBuilder.toString();
         boolean asc = randomBoolean();
         SearchResponse response = client()
@@ -1136,7 +1184,7 @@ public void testSingleValuedFieldOrderedBySubAggregationAscMultiHierarchyLevelsS
         assertThat(tags.getName(), equalTo("tags"));
         assertThat(tags.getBuckets().size(), equalTo(2));
 
-        Iterator<Terms.Bucket> iters = tags.getBuckets().iterator();
+        Iterator<? extends Terms.Bucket> iters = tags.getBuckets().iterator();
 
         // the max for "more" is 2
         // the max for "less" is 4
@@ -1511,4 +1559,45 @@ public void testIndexMetaField() throws Exception {
     public void testOtherDocCount() {
         testOtherDocCount(SINGLE_VALUED_FIELD_NAME, MULTI_VALUED_FIELD_NAME);
     }
+
+    /**
+     * Make sure that a request using a script does not get cached and a request
+     * not using a script does get cached.
+     */
+    public void testDontCacheScripts() throws Exception {
+        assertAcked(prepareCreate("cache_test_idx").addMapping("type", "d", "type=keyword")
+                .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1))
+                .get());
+        indexRandom(true, client().prepareIndex("cache_test_idx", "type", "1").setSource("s", "foo"),
+                client().prepareIndex("cache_test_idx", "type", "2").setSource("s", "bar"));
+
+        // Make sure we are starting with a clear cache
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(0L));
+
+        // Test that a request using a script does not get cached
+        SearchResponse r = client().prepareSearch("cache_test_idx").setSize(0)
+                .addAggregation(
+                        terms("terms").field("d").script(
+                            new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "'foo_' + _value", Collections.emptyMap())))
+                .get();
+        assertSearchResponse(r);
+
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(0L));
+
+        // To make sure that the cache is working test that a request not using
+        // a script is cached
+        r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(terms("terms").field("d")).get();
+        assertSearchResponse(r);
+
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(1L));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsDocCountErrorIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsDocCountErrorIT.java
index 6bbe1dc22b2d7..01e05f403d745 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsDocCountErrorIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsDocCountErrorIT.java
@@ -32,7 +32,9 @@
 
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.sum;
@@ -52,6 +54,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase {
     private static final String LONG_FIELD_NAME = "l_value";
     private static final String DOUBLE_FIELD_NAME = "d_value";
 
+
     public static String randomExecutionHint() {
         return randomBoolean() ? null : randomFrom(ExecutionMode.values()).toString();
     }
@@ -96,6 +99,68 @@ public void setupSuiteScopeCluster() throws Exception {
                     .field(DOUBLE_FIELD_NAME, 1.0 * randomInt(numUniqueTerms))
                     .endObject()));
         }
+        assertAcked(prepareCreate("idx_fixed_docs_0").addMapping("type", STRING_FIELD_NAME, "type=keyword")
+                .setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)));
+        Map<String, Integer> shard0DocsPerTerm = new HashMap<>();
+        shard0DocsPerTerm.put("A", 25);
+        shard0DocsPerTerm.put("B", 18);
+        shard0DocsPerTerm.put("C", 6);
+        shard0DocsPerTerm.put("D", 3);
+        shard0DocsPerTerm.put("E", 2);
+        shard0DocsPerTerm.put("F", 2);
+        shard0DocsPerTerm.put("G", 2);
+        shard0DocsPerTerm.put("H", 2);
+        shard0DocsPerTerm.put("I", 1);
+        shard0DocsPerTerm.put("J", 1);
+        for (Map.Entry<String, Integer> entry : shard0DocsPerTerm.entrySet()) {
+            for (int i = 0; i < entry.getValue(); i++) {
+                String term = entry.getKey();
+                builders.add(client().prepareIndex("idx_fixed_docs_0", "type", term + "-" + i)
+                        .setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, term).endObject()));
+            }
+        }
+
+        assertAcked(prepareCreate("idx_fixed_docs_1").addMapping("type", STRING_FIELD_NAME, "type=keyword")
+                .setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)));
+        Map<String, Integer> shard1DocsPerTerm = new HashMap<>();
+        shard1DocsPerTerm.put("A", 30);
+        shard1DocsPerTerm.put("B", 25);
+        shard1DocsPerTerm.put("F", 17);
+        shard1DocsPerTerm.put("Z", 16);
+        shard1DocsPerTerm.put("G", 15);
+        shard1DocsPerTerm.put("H", 14);
+        shard1DocsPerTerm.put("I", 10);
+        shard1DocsPerTerm.put("Q", 6);
+        shard1DocsPerTerm.put("J", 8);
+        shard1DocsPerTerm.put("C", 4);
+        for (Map.Entry<String, Integer> entry : shard1DocsPerTerm.entrySet()) {
+            for (int i = 0; i < entry.getValue(); i++) {
+                String term = entry.getKey();
+                builders.add(client().prepareIndex("idx_fixed_docs_1", "type", term + "-" + i)
+                        .setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, term).field("shard", 1).endObject()));
+            }
+        }
+
+        assertAcked(prepareCreate("idx_fixed_docs_2")
+                .addMapping("type", STRING_FIELD_NAME, "type=keyword")
+                .setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)));
+        Map<String, Integer> shard2DocsPerTerm = new HashMap<>();
+        shard2DocsPerTerm.put("A", 45);
+        shard2DocsPerTerm.put("C", 44);
+        shard2DocsPerTerm.put("Z", 36);
+        shard2DocsPerTerm.put("G", 30);
+        shard2DocsPerTerm.put("E", 29);
+        shard2DocsPerTerm.put("H", 28);
+        shard2DocsPerTerm.put("Q", 2);
+        shard2DocsPerTerm.put("D", 1);
+        for (Map.Entry<String, Integer> entry : shard2DocsPerTerm.entrySet()) {
+            for (int i = 0; i < entry.getValue(); i++) {
+                String term = entry.getKey();
+                builders.add(client().prepareIndex("idx_fixed_docs_2", "type", term + "-" + i)
+                        .setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, term).field("shard", 2).endObject()));
+            }
+        }
+
         indexRandom(true, builders);
         ensureSearchable();
     }
@@ -110,7 +175,7 @@ private void assertDocCountErrorWithinBounds(int size, SearchResponse accurateRe
         assertThat(testTerms, notNullValue());
         assertThat(testTerms.getName(), equalTo("terms"));
         assertThat(testTerms.getDocCountError(), greaterThanOrEqualTo(0L));
-        Collection<Bucket> testBuckets = testTerms.getBuckets();
+        List<? extends Bucket> testBuckets = testTerms.getBuckets();
         assertThat(testBuckets.size(), lessThanOrEqualTo(size));
         assertThat(accurateTerms.getBuckets().size(), greaterThanOrEqualTo(testBuckets.size()));
 
@@ -144,7 +209,7 @@ private void assertNoDocCountError(int size, SearchResponse accurateResponse, Se
         assertThat(testTerms, notNullValue());
         assertThat(testTerms.getName(), equalTo("terms"));
         assertThat(testTerms.getDocCountError(), equalTo(0L));
-        Collection<Bucket> testBuckets = testTerms.getBuckets();
+        List<? extends Bucket> testBuckets = testTerms.getBuckets();
         assertThat(testBuckets.size(), lessThanOrEqualTo(size));
         assertThat(accurateTerms.getBuckets().size(), greaterThanOrEqualTo(testBuckets.size()));
 
@@ -162,7 +227,7 @@ private void assertNoDocCountErrorSingleResponse(int size, SearchResponse testRe
         assertThat(testTerms, notNullValue());
         assertThat(testTerms.getName(), equalTo("terms"));
         assertThat(testTerms.getDocCountError(), equalTo(0L));
-        Collection<Bucket> testBuckets = testTerms.getBuckets();
+        List<? extends Bucket> testBuckets = testTerms.getBuckets();
         assertThat(testBuckets.size(), lessThanOrEqualTo(size));
 
         for (Terms.Bucket testBucket : testBuckets) {
@@ -181,7 +246,7 @@ private void assertUnboundedDocCountError(int size, SearchResponse accurateRespo
         assertThat(testTerms, notNullValue());
         assertThat(testTerms.getName(), equalTo("terms"));
         assertThat(testTerms.getDocCountError(),anyOf(equalTo(-1L), equalTo(0L)));
-        Collection<Bucket> testBuckets = testTerms.getBuckets();
+        List<? extends Bucket> testBuckets = testTerms.getBuckets();
         assertThat(testBuckets.size(), lessThanOrEqualTo(size));
         assertThat(accurateTerms.getBuckets().size(), greaterThanOrEqualTo(testBuckets.size()));
 
@@ -901,5 +966,60 @@ public void testDoubleValueFieldSubAggDesc() throws Exception {
 
         assertUnboundedDocCountError(size, accurateResponse, testResponse);
     }
+    
+    /**
+     * Test a case where we know exactly how many of each term is on each shard
+     * so we know the exact error value for each term. To do this we search over
+     * 3 one-shard indices.
+     */
+    public void testFixedDocs() throws Exception {
+        SearchResponse response = client().prepareSearch("idx_fixed_docs_0", "idx_fixed_docs_1", "idx_fixed_docs_2").setTypes("type")
+                .addAggregation(terms("terms")
+                        .executionHint(randomExecutionHint())
+                        .field(STRING_FIELD_NAME)
+                        .showTermDocCountError(true)
+                        .size(5).shardSize(5)
+                        .collectMode(randomFrom(SubAggCollectionMode.values())))
+                .execute().actionGet();
+        assertSearchResponse(response);
+        
+        Terms terms = response.getAggregations().get("terms");
+        assertThat(terms, notNullValue());
+        assertThat(terms.getDocCountError(), equalTo(46L));
+        List<? extends Bucket> buckets = terms.getBuckets();
+        assertThat(buckets, notNullValue());
+        assertThat(buckets.size(), equalTo(5));
+        
+        Bucket bucket = buckets.get(0);
+        assertThat(bucket, notNullValue());
+        assertThat(bucket.getKey(), equalTo("A"));
+        assertThat(bucket.getDocCount(), equalTo(100L));
+        assertThat(bucket.getDocCountError(), equalTo(0L));
+        
+        bucket = buckets.get(1);
+        assertThat(bucket, notNullValue());
+        assertThat(bucket.getKey(), equalTo("Z"));
+        assertThat(bucket.getDocCount(), equalTo(52L));
+        assertThat(bucket.getDocCountError(), equalTo(2L));
+        
+        bucket = buckets.get(2);
+        assertThat(bucket, notNullValue());
+        assertThat(bucket.getKey(), equalTo("C"));
+        assertThat(bucket.getDocCount(), equalTo(50L));
+        assertThat(bucket.getDocCountError(), equalTo(15L));
+        
+        
+        bucket = buckets.get(3);
+        assertThat(bucket, notNullValue());
+        assertThat(bucket.getKey(), equalTo("G"));
+        assertThat(bucket.getDocCount(), equalTo(45L));
+        assertThat(bucket.getDocCountError(), equalTo(2L));
+        
+        bucket = buckets.get(4);
+        assertThat(bucket, notNullValue());
+        assertThat(bucket.getKey(), equalTo("B"));
+        assertThat(bucket.getDocCount(), equalTo(43L));
+        assertThat(bucket.getDocCountError(), equalTo(29L));
+    }
 
 }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsShardMinDocCountIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsShardMinDocCountIT.java
index 2ea319daa9c3c..6f618fd5afe0f 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsShardMinDocCountIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsShardMinDocCountIT.java
@@ -20,6 +20,7 @@
 
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.aggregations.bucket.filter.InternalFilter;
 import org.elasticsearch.search.aggregations.bucket.significant.SignificantTerms;
@@ -102,10 +103,10 @@ private void addTermsDocs(String term, int numInClass, int numNotInClass, List<I
         String sourceClass = "{\"text\": \"" + term + "\", \"class\":" + "true" + "}";
         String sourceNotClass = "{\"text\": \"" + term + "\", \"class\":" + "false" + "}";
         for (int i = 0; i < numInClass; i++) {
-            builders.add(client().prepareIndex(index, type).setSource(sourceClass));
+            builders.add(client().prepareIndex(index, type).setSource(sourceClass, XContentType.JSON));
         }
         for (int i = 0; i < numNotInClass; i++) {
-            builders.add(client().prepareIndex(index, type).setSource(sourceNotClass));
+            builders.add(client().prepareIndex(index, type).setSource(sourceNotClass, XContentType.JSON));
         }
     }
 
@@ -155,7 +156,7 @@ public void testShardMinDocCountTermsTest() throws Exception {
     private void addTermsDocs(String term, int numDocs, List<IndexRequestBuilder> builders) {
         String sourceClass = "{\"text\": \"" + term + "\"}";
         for (int i = 0; i < numDocs; i++) {
-            builders.add(client().prepareIndex(index, type).setSource(sourceClass));
+            builders.add(client().prepareIndex(index, type).setSource(sourceClass, XContentType.JSON));
         }
 
     }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsTests.java
index c8a14e82bf405..42f6ef78f4b17 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsTests.java
@@ -21,13 +21,13 @@
 
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.automaton.RegExp;
-import org.elasticsearch.script.Script;
 import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode;
 import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorFactory.ExecutionMode;
 import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude;
+
 import java.util.ArrayList;
 import java.util.List;
 import java.util.SortedSet;
@@ -47,32 +47,18 @@ public class TermsTests extends BaseAggregationTestCase<TermsAggregationBuilder>
 
     @Override
     protected TermsAggregationBuilder createTestAggregatorBuilder() {
-        String name = randomAsciiOfLengthBetween(3, 20);
+        String name = randomAlphaOfLengthBetween(3, 20);
         TermsAggregationBuilder factory = new TermsAggregationBuilder(name, null);
-        String field = randomAsciiOfLengthBetween(3, 20);
-        int randomFieldBranch = randomInt(2);
-        switch (randomFieldBranch) {
-        case 0:
-            factory.field(field);
-            break;
-        case 1:
-            factory.field(field);
-            factory.script(new Script("_value + 1"));
-            break;
-        case 2:
-            factory.script(new Script("doc[" + field + "] + 1"));
-            break;
-        default:
-            fail();
-        }
+        String field = randomAlphaOfLengthBetween(3, 20);
+        randomFieldOrScript(factory, field);
         if (randomBoolean()) {
             factory.missing("MISSING");
         }
         if (randomBoolean()) {
-            factory.bucketCountThresholds().setRequiredSize(randomIntBetween(1, Integer.MAX_VALUE));
+            factory.size(randomIntBetween(1, Integer.MAX_VALUE));
         }
         if (randomBoolean()) {
-            factory.bucketCountThresholds().setShardSize(randomIntBetween(1, Integer.MAX_VALUE));
+            factory.shardSize(randomIntBetween(1, Integer.MAX_VALUE));
         }
         if (randomBoolean()) {
             int minDocCount = randomInt(4);
@@ -83,12 +69,12 @@ protected TermsAggregationBuilder createTestAggregatorBuilder() {
             case 2:
             case 3:
             case 4:
-                minDocCount = randomInt();
+                minDocCount = randomIntBetween(0, Integer.MAX_VALUE);
                 break;
             default:
                 fail();
             }
-            factory.bucketCountThresholds().setMinDocCount(minDocCount);
+            factory.minDocCount(minDocCount);
         }
         if (randomBoolean()) {
             int shardMinDocCount = randomInt(4);
@@ -99,12 +85,12 @@ protected TermsAggregationBuilder createTestAggregatorBuilder() {
             case 2:
             case 3:
             case 4:
-                shardMinDocCount = randomInt();
+                shardMinDocCount = randomIntBetween(0, Integer.MAX_VALUE);
                 break;
             default:
                 fail();
             }
-            factory.bucketCountThresholds().setShardMinDocCount(shardMinDocCount);
+            factory.shardMinDocCount(shardMinDocCount);
         }
         if (randomBoolean()) {
             factory.collectMode(randomFrom(SubAggCollectionMode.values()));
@@ -117,7 +103,7 @@ protected TermsAggregationBuilder createTestAggregatorBuilder() {
         }
         if (randomBoolean()) {
             IncludeExclude incExc = null;
-            switch (randomInt(5)) {
+            switch (randomInt(6)) {
             case 0:
                 incExc = new IncludeExclude(new RegExp("foobar"), null);
                 break;
@@ -131,7 +117,7 @@ protected TermsAggregationBuilder createTestAggregatorBuilder() {
                 SortedSet<BytesRef> includeValues = new TreeSet<>();
                 int numIncs = randomIntBetween(1, 20);
                 for (int i = 0; i < numIncs; i++) {
-                    includeValues.add(new BytesRef(randomAsciiOfLengthBetween(1, 30)));
+                    includeValues.add(new BytesRef(randomAlphaOfLengthBetween(1, 30)));
                 }
                 SortedSet<BytesRef> excludeValues = null;
                 incExc = new IncludeExclude(includeValues, excludeValues);
@@ -141,7 +127,7 @@ protected TermsAggregationBuilder createTestAggregatorBuilder() {
                 SortedSet<BytesRef> excludeValues2 = new TreeSet<>();
                 int numExcs2 = randomIntBetween(1, 20);
                 for (int i = 0; i < numExcs2; i++) {
-                    excludeValues2.add(new BytesRef(randomAsciiOfLengthBetween(1, 30)));
+                    excludeValues2.add(new BytesRef(randomAlphaOfLengthBetween(1, 30)));
                 }
                 incExc = new IncludeExclude(includeValues2, excludeValues2);
                 break;
@@ -149,15 +135,20 @@ protected TermsAggregationBuilder createTestAggregatorBuilder() {
                 SortedSet<BytesRef> includeValues3 = new TreeSet<>();
                 int numIncs3 = randomIntBetween(1, 20);
                 for (int i = 0; i < numIncs3; i++) {
-                    includeValues3.add(new BytesRef(randomAsciiOfLengthBetween(1, 30)));
+                    includeValues3.add(new BytesRef(randomAlphaOfLengthBetween(1, 30)));
                 }
                 SortedSet<BytesRef> excludeValues3 = new TreeSet<>();
                 int numExcs3 = randomIntBetween(1, 20);
                 for (int i = 0; i < numExcs3; i++) {
-                    excludeValues3.add(new BytesRef(randomAsciiOfLengthBetween(1, 30)));
+                    excludeValues3.add(new BytesRef(randomAlphaOfLengthBetween(1, 30)));
                 }
                 incExc = new IncludeExclude(includeValues3, excludeValues3);
                 break;
+            case 6:
+                final int numPartitions = randomIntBetween(1, 100);
+                final int partition = randomIntBetween(0, numPartitions - 1);
+                incExc = new IncludeExclude(partition, numPartitions);
+                break;
             default:
                 fail();
             }
@@ -183,10 +174,10 @@ private List<Terms.Order> randomOrder() {
             orders.add(Terms.Order.count(randomBoolean()));
             break;
         case 2:
-            orders.add(Terms.Order.aggregation(randomAsciiOfLengthBetween(3, 20), randomBoolean()));
+            orders.add(Terms.Order.aggregation(randomAlphaOfLengthBetween(3, 20), randomBoolean()));
             break;
         case 3:
-            orders.add(Terms.Order.aggregation(randomAsciiOfLengthBetween(3, 20), randomAsciiOfLengthBetween(3, 20), randomBoolean()));
+            orders.add(Terms.Order.aggregation(randomAlphaOfLengthBetween(3, 20), randomAlphaOfLengthBetween(3, 20), randomBoolean()));
             break;
         case 4:
             int numOrders = randomIntBetween(1, 3);
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/adjacency/InternalAdjacencyMatrixTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/adjacency/InternalAdjacencyMatrixTests.java
new file mode 100644
index 0000000000000..bd349200b3d81
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/adjacency/InternalAdjacencyMatrixTests.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.adjacency;
+
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.aggregations.InternalMultiBucketAggregationTestCase;
+import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class InternalAdjacencyMatrixTests extends InternalMultiBucketAggregationTestCase<InternalAdjacencyMatrix> {
+
+    private List<String> keys;
+
+    @Override
+    protected int maxNumberOfBuckets() {
+        return 10;
+    }
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        keys = new ArrayList<>();
+        // InternalAdjacencyMatrix represents the upper triangular matrix:
+        // 2 filters (matrix of 2x2) generates 3 buckets
+        // 3 filters generates 6 buckets
+        // 4 filters generates 10 buckets
+        int numFilters = randomIntBetween(2, 4);
+        String[] filters = new String[numFilters];
+        for (int i = 0; i < numFilters; i++) {
+            filters[i] = randomAlphaOfLength(5);
+        }
+        for (int i = 0; i < filters.length; i++) {
+            keys.add(filters[i]);
+            for (int j = i + 1; j < filters.length; j++) {
+                if (filters[i].compareTo(filters[j]) <= 0) {
+                    keys.add(filters[i] + "&" + filters[j]);
+                } else {
+                    keys.add(filters[j] + "&" + filters[i]);
+                }
+            }
+        }
+    }
+
+    @Override
+    protected InternalAdjacencyMatrix createTestInstance(String name, List<PipelineAggregator> pipelineAggregators,
+            Map<String, Object> metaData, InternalAggregations aggregations) {
+        final List<InternalAdjacencyMatrix.InternalBucket> buckets = new ArrayList<>();
+        for (int i = 0; i < keys.size(); ++i) {
+            String key = keys.get(i);
+            int docCount = randomIntBetween(0, 1000);
+            buckets.add(new InternalAdjacencyMatrix.InternalBucket(key, docCount, aggregations));
+        }
+        return new InternalAdjacencyMatrix(name, buckets, pipelineAggregators, metaData);
+    }
+
+    @Override
+    protected Class<? extends ParsedMultiBucketAggregation> implementationClass() {
+        return ParsedAdjacencyMatrix.class;
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregatorTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregatorTests.java
new file mode 100644
index 0000000000000..fb615e66dfb57
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregatorTests.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.bucket.filter;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.MultiReader;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.store.Directory;
+import org.elasticsearch.index.mapper.KeywordFieldMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.AggregatorTestCase;
+import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregatorFactory;
+import org.elasticsearch.search.aggregations.bucket.filter.InternalFilter;
+import org.hamcrest.Matchers;
+import org.junit.Before;
+
+import java.io.IOException;
+
+public class FilterAggregatorTests extends AggregatorTestCase {
+    private MappedFieldType fieldType;
+
+    @Before
+    public void setUpTest() throws Exception {
+        super.setUp();
+        fieldType = new KeywordFieldMapper.KeywordFieldType();
+        fieldType.setHasDocValues(true);
+        fieldType.setIndexOptions(IndexOptions.DOCS);
+        fieldType.setName("field");
+    }
+
+    public void testEmpty() throws Exception {
+        Directory directory = newDirectory();
+        RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
+        indexWriter.close();
+        IndexReader indexReader = DirectoryReader.open(directory);
+        IndexSearcher indexSearcher = newSearcher(indexReader, true, true);
+        QueryBuilder filter = QueryBuilders.termQuery("field", randomAlphaOfLength(5));
+        FilterAggregationBuilder builder = new FilterAggregationBuilder("test", filter);
+        InternalFilter response = search(indexSearcher, new MatchAllDocsQuery(), builder,
+                fieldType);
+        assertEquals(response.getDocCount(), 0);
+        indexReader.close();
+        directory.close();
+    }
+
+    public void testRandom() throws Exception {
+        Directory directory = newDirectory();
+        RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
+        int numDocs = randomIntBetween(100, 200);
+        int maxTerm = randomIntBetween(10, 50);
+        int[] expectedBucketCount = new int[maxTerm];
+        Document document = new Document();
+        for (int i = 0; i < numDocs; i++) {
+            if (frequently()) {
+                // make sure we have more than one segment to test the merge
+                indexWriter.getReader().close();
+            }
+            int value = randomInt(maxTerm-1);
+            expectedBucketCount[value] += 1;
+            document.add(new Field("field", Integer.toString(value), fieldType));
+            indexWriter.addDocument(document);
+            document.clear();
+        }
+        indexWriter.close();
+
+        IndexReader indexReader = DirectoryReader.open(directory);
+        IndexSearcher indexSearcher = newSearcher(indexReader, true, true);
+        int value = randomInt(maxTerm - 1);
+        QueryBuilder filter = QueryBuilders.termQuery("field", Integer.toString(value));
+        FilterAggregationBuilder builder = new FilterAggregationBuilder("test", filter);
+
+        for (boolean doReduce : new boolean[] {true, false}) {
+            final InternalFilter response;
+            if (doReduce) {
+                response = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), builder,
+                        fieldType);
+            } else {
+                response = search(indexSearcher, new MatchAllDocsQuery(), builder, fieldType);
+            }
+            assertEquals(response.getDocCount(), (long) expectedBucketCount[value]);
+        }
+        indexReader.close();
+        directory.close();
+    }
+
+    public void testParsedAsFilter() throws IOException {
+        IndexReader indexReader = new MultiReader();
+        IndexSearcher indexSearcher = newSearcher(indexReader);
+        QueryBuilder filter = QueryBuilders.boolQuery()
+                .must(QueryBuilders.termQuery("field", "foo"))
+                .should(QueryBuilders.termQuery("field", "bar"));
+        FilterAggregationBuilder builder = new FilterAggregationBuilder("test", filter);
+        AggregatorFactory<?> factory = createAggregatorFactory(builder, indexSearcher, fieldType);
+        assertThat(factory, Matchers.instanceOf(FilterAggregatorFactory.class));
+        FilterAggregatorFactory filterFactory = (FilterAggregatorFactory) factory;
+        Query parsedQuery = filterFactory.weight.getQuery();
+        assertThat(parsedQuery, Matchers.instanceOf(BooleanQuery.class));
+        assertEquals(2, ((BooleanQuery) parsedQuery).clauses().size());
+        // means the bool query has been parsed as a filter, if it was a query minShouldMatch would
+        // be 0
+        assertEquals(1, ((BooleanQuery) parsedQuery).getMinimumNumberShouldMatch());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/InternalFilterTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/InternalFilterTests.java
new file mode 100644
index 0000000000000..4e2531de15d44
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/InternalFilterTests.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.filter;
+
+import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.aggregations.InternalSingleBucketAggregationTestCase;
+import org.elasticsearch.search.aggregations.bucket.ParsedSingleBucketAggregation;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+
+import java.util.List;
+import java.util.Map;
+
+public class InternalFilterTests extends InternalSingleBucketAggregationTestCase<InternalFilter> {
+    @Override
+    protected InternalFilter createTestInstance(String name, long docCount, InternalAggregations aggregations,
+            List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
+        return new InternalFilter(name, docCount, aggregations, pipelineAggregators, metaData);
+    }
+
+    @Override
+    protected Class<? extends ParsedSingleBucketAggregation> implementationClass() {
+        return ParsedFilter.class;
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregatorTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregatorTests.java
new file mode 100644
index 0000000000000..4597eabb777cf
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregatorTests.java
@@ -0,0 +1,228 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.bucket.filters;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.MultiReader;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.store.Directory;
+import org.elasticsearch.index.mapper.KeywordFieldMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.AggregatorTestCase;
+import org.elasticsearch.search.aggregations.bucket.filters.FiltersAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.filters.FiltersAggregator;
+import org.elasticsearch.search.aggregations.bucket.filters.FiltersAggregatorFactory;
+import org.elasticsearch.search.aggregations.bucket.filters.InternalFilters;
+import org.hamcrest.Matchers;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+public class FiltersAggregatorTests extends AggregatorTestCase {
+    private MappedFieldType fieldType;
+
+    @Before
+    public void setUpTest() throws Exception {
+        super.setUp();
+        fieldType = new KeywordFieldMapper.KeywordFieldType();
+        fieldType.setHasDocValues(true);
+        fieldType.setIndexOptions(IndexOptions.DOCS);
+        fieldType.setName("field");
+    }
+
+    public void testEmpty() throws Exception {
+        Directory directory = newDirectory();
+        RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
+        indexWriter.close();
+        IndexReader indexReader = DirectoryReader.open(directory);
+        IndexSearcher indexSearcher = newSearcher(indexReader, true, true);
+        int numFilters = randomIntBetween(1, 10);
+        QueryBuilder[] filters = new QueryBuilder[numFilters];
+        for (int i = 0; i < filters.length; i++) {
+            filters[i] = QueryBuilders.termQuery("field", randomAlphaOfLength(5));
+        }
+        FiltersAggregationBuilder builder = new FiltersAggregationBuilder("test", filters);
+        builder.otherBucketKey("other");
+        InternalFilters response = search(indexSearcher, new MatchAllDocsQuery(), builder, fieldType);
+        assertEquals(response.getBuckets().size(), numFilters);
+        for (InternalFilters.InternalBucket filter : response.getBuckets()) {
+            assertEquals(filter.getDocCount(), 0);
+        }
+        indexReader.close();
+        directory.close();
+    }
+
+    public void testKeyedFilter() throws Exception {
+        Directory directory = newDirectory();
+        RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
+        Document document = new Document();
+        document.add(new Field("field", "foo", fieldType));
+        indexWriter.addDocument(document);
+        document.clear();
+        document.add(new Field("field", "else", fieldType));
+        indexWriter.addDocument(document);
+        // make sure we have more than one segment to test the merge
+        indexWriter.commit();
+        document.add(new Field("field", "foo", fieldType));
+        indexWriter.addDocument(document);
+        document.clear();
+        document.add(new Field("field", "bar", fieldType));
+        indexWriter.addDocument(document);
+        document.clear();
+        document.add(new Field("field", "foobar", fieldType));
+        indexWriter.addDocument(document);
+        indexWriter.commit();
+        document.clear();
+        document.add(new Field("field", "something", fieldType));
+        indexWriter.addDocument(document);
+        indexWriter.commit();
+        document.clear();
+        document.add(new Field("field", "foobar", fieldType));
+        indexWriter.addDocument(document);
+        indexWriter.close();
+
+        IndexReader indexReader = DirectoryReader.open(directory);
+        IndexSearcher indexSearcher = newSearcher(indexReader, true, true);
+
+        FiltersAggregator.KeyedFilter[] keys = new FiltersAggregator.KeyedFilter[6];
+        keys[0] = new FiltersAggregator.KeyedFilter("foobar", QueryBuilders.termQuery("field", "foobar"));
+        keys[1] = new FiltersAggregator.KeyedFilter("bar", QueryBuilders.termQuery("field", "bar"));
+        keys[2] = new FiltersAggregator.KeyedFilter("foo", QueryBuilders.termQuery("field", "foo"));
+        keys[3] = new FiltersAggregator.KeyedFilter("foo2", QueryBuilders.termQuery("field", "foo"));
+        keys[4] = new FiltersAggregator.KeyedFilter("same", QueryBuilders.termQuery("field", "foo"));
+        // filter name already present so it should be merge with the previous one ?
+        keys[5] = new FiltersAggregator.KeyedFilter("same", QueryBuilders.termQuery("field", "bar"));
+        FiltersAggregationBuilder builder = new FiltersAggregationBuilder("test", keys);
+        builder.otherBucket(true);
+        builder.otherBucketKey("other");
+        for (boolean doReduce : new boolean[] {true, false}) {
+            final InternalFilters filters;
+            if (doReduce) {
+                filters = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), builder, fieldType);
+            } else {
+                filters = search(indexSearcher, new MatchAllDocsQuery(), builder, fieldType);
+            }
+            assertEquals(filters.getBuckets().size(), 7);
+            assertEquals(filters.getBucketByKey("foobar").getDocCount(), 2);
+            assertEquals(filters.getBucketByKey("foo").getDocCount(), 2);
+            assertEquals(filters.getBucketByKey("foo2").getDocCount(), 2);
+            assertEquals(filters.getBucketByKey("bar").getDocCount(), 1);
+            assertEquals(filters.getBucketByKey("same").getDocCount(), 1);
+            assertEquals(filters.getBucketByKey("other").getDocCount(), 2);
+        }
+
+        indexReader.close();
+        directory.close();
+    }
+
+    public void testRandom() throws Exception {
+        Directory directory = newDirectory();
+        RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
+        int numDocs = randomIntBetween(100, 200);
+        int maxTerm = randomIntBetween(10, 50);
+        int[] expectedBucketCount = new int[maxTerm];
+        Document document = new Document();
+        for (int i = 0; i < numDocs; i++) {
+            if (frequently()) {
+                // make sure we have more than one segment to test the merge
+                indexWriter.commit();
+            }
+            int value = randomInt(maxTerm-1);
+            expectedBucketCount[value] += 1;
+            document.add(new Field("field", Integer.toString(value), fieldType));
+            indexWriter.addDocument(document);
+            document.clear();
+        }
+        indexWriter.close();
+
+        IndexReader indexReader = DirectoryReader.open(directory);
+        IndexSearcher indexSearcher = newSearcher(indexReader, true, true);
+        int numFilters = randomIntBetween(1, 10);
+        QueryBuilder[] filters = new QueryBuilder[numFilters];
+        int[] filterTerms = new int[numFilters];
+        int expectedOtherCount = numDocs;
+        Set<Integer> filterSet = new HashSet<>();
+        for (int i = 0; i < filters.length; i++) {
+            int value = randomInt(maxTerm-1);
+            filters[i] = QueryBuilders.termQuery("field", Integer.toString(value));
+            filterTerms[i] = value;
+            if (filterSet.contains(value) == false) {
+                expectedOtherCount -= expectedBucketCount[value];
+                filterSet.add(value);
+            }
+        }
+        FiltersAggregationBuilder builder = new FiltersAggregationBuilder("test", filters);
+        builder.otherBucket(true);
+        builder.otherBucketKey("other");
+
+        for (boolean doReduce : new boolean[] {true, false}) {
+            final InternalFilters response;
+            if (doReduce) {
+                response = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), builder, fieldType);
+            } else {
+                response = search(indexSearcher, new MatchAllDocsQuery(), builder, fieldType);
+            }
+            List<InternalFilters.InternalBucket> buckets = response.getBuckets();
+            assertEquals(buckets.size(), filters.length+1);
+
+            for (InternalFilters.InternalBucket bucket : buckets) {
+                if ("other".equals(bucket.getKey())) {
+                    assertEquals(bucket.getDocCount(), expectedOtherCount);
+                } else {
+                    int index = Integer.parseInt(bucket.getKey());
+                    assertEquals(bucket.getDocCount(), (long) expectedBucketCount[filterTerms[index]]);
+                }
+            }
+        }
+        indexReader.close();
+        directory.close();
+    }
+
+    public void testParsedAsFilter() throws IOException {
+        IndexReader indexReader = new MultiReader();
+        IndexSearcher indexSearcher = newSearcher(indexReader);
+        QueryBuilder filter = QueryBuilders.boolQuery()
+                .must(QueryBuilders.termQuery("field", "foo"))
+                .should(QueryBuilders.termQuery("field", "bar"));
+        FiltersAggregationBuilder builder = new FiltersAggregationBuilder("test", filter);
+        AggregatorFactory<?> factory = createAggregatorFactory(builder, indexSearcher, fieldType);
+        assertThat(factory, Matchers.instanceOf(FiltersAggregatorFactory.class));
+        FiltersAggregatorFactory filtersFactory = (FiltersAggregatorFactory) factory;
+        Query parsedQuery = filtersFactory.weights[0].getQuery();
+        assertThat(parsedQuery, Matchers.instanceOf(BooleanQuery.class));
+        assertEquals(2, ((BooleanQuery) parsedQuery).clauses().size());
+        // means the bool query has been parsed as a filter, if it was a query minShouldMatch would
+        // be 0
+        assertEquals(1, ((BooleanQuery) parsedQuery).getMinimumNumberShouldMatch());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/filters/InternalFiltersTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/filters/InternalFiltersTests.java
new file mode 100644
index 0000000000000..a1f991ac92d13
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/filters/InternalFiltersTests.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.filters;
+
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.aggregations.InternalMultiBucketAggregationTestCase;
+import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+public class InternalFiltersTests extends InternalMultiBucketAggregationTestCase<InternalFilters> {
+
+    private boolean keyed;
+    private List<String> keys;
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        keyed = randomBoolean();
+        keys = new ArrayList<>();
+        int numBuckets = randomNumberOfBuckets();
+        for (int i = 0; i < numBuckets; i++) {
+            if (keyed) {
+                keys.add(randomAlphaOfLength(5));
+            } else {
+                // this is what the FiltersAggregationBuilder ctor does when not providing KeyedFilter
+                keys.add(String.valueOf(i));
+            }
+        }
+    }
+
+    @Override
+    protected InternalFilters createTestInstance(String name, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData,
+            InternalAggregations aggregations) {
+        final List<InternalFilters.InternalBucket> buckets = new ArrayList<>();
+        for (int i = 0; i < keys.size(); ++i) {
+            String key = keys.get(i);
+            int docCount = randomIntBetween(0, 1000);
+            buckets.add(new InternalFilters.InternalBucket(key, docCount, aggregations, keyed));
+        }
+        return new InternalFilters(name, buckets, keyed, pipelineAggregators, metaData);
+    }
+
+    @Override
+    protected Class<? extends ParsedMultiBucketAggregation> implementationClass() {
+        return ParsedFilters.class;
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridParserTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridParserTests.java
index 9f2ecfec3e9d9..49c44b222454f 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridParserTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridParserTests.java
@@ -18,81 +18,61 @@
  */
 package org.elasticsearch.search.aggregations.bucket.geogrid;
 
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.test.ESTestCase;
 
-public class GeoHashGridParserTests extends ESTestCase {
-    private static final IndicesQueriesRegistry mockRegistry = new IndicesQueriesRegistry();
+import static org.hamcrest.Matchers.instanceOf;
 
+public class GeoHashGridParserTests extends ESTestCase {
     public void testParseValidFromInts() throws Exception {
         int precision = randomIntBetween(1, 12);
-        XContentParser stParser = JsonXContent.jsonXContent.createParser(
+        XContentParser stParser = createParser(JsonXContent.jsonXContent, 
                 "{\"field\":\"my_loc\", \"precision\":" + precision + ", \"size\": 500, \"shard_size\": 550}");
-        QueryParseContext parseContext = new QueryParseContext(mockRegistry,
-                stParser, ParseFieldMatcher.STRICT);
+        QueryParseContext parseContext = new QueryParseContext(stParser);
         XContentParser.Token token = stParser.nextToken();
         assertSame(XContentParser.Token.START_OBJECT, token);
-        GeoHashGridParser parser = new GeoHashGridParser();
         // can create a factory
-        assertNotNull(parser.parse("geohash_grid", parseContext));
+        assertNotNull(GeoGridAggregationBuilder.parse("geohash_grid", parseContext));
     }
 
     public void testParseValidFromStrings() throws Exception {
         int precision = randomIntBetween(1, 12);
-        XContentParser stParser = JsonXContent.jsonXContent.createParser(
+        XContentParser stParser = createParser(JsonXContent.jsonXContent, 
                 "{\"field\":\"my_loc\", \"precision\":\"" + precision + "\", \"size\": \"500\", \"shard_size\": \"550\"}");
-        QueryParseContext parseContext = new QueryParseContext(mockRegistry, stParser, ParseFieldMatcher.STRICT);
+        QueryParseContext parseContext = new QueryParseContext(stParser);
         XContentParser.Token token = stParser.nextToken();
         assertSame(XContentParser.Token.START_OBJECT, token);
-        GeoHashGridParser parser = new GeoHashGridParser();
         // can create a factory
-        assertNotNull(parser.parse("geohash_grid", parseContext));
-    }
-
-    public void testParseErrorOnNonIntPrecision() throws Exception {
-        XContentParser stParser = JsonXContent.jsonXContent.createParser("{\"field\":\"my_loc\", \"precision\":\"2.0\"}");
-        QueryParseContext parseContext = new QueryParseContext(mockRegistry, stParser, ParseFieldMatcher.STRICT);
-        XContentParser.Token token = stParser.nextToken();
-        assertSame(XContentParser.Token.START_OBJECT, token);
-        GeoHashGridParser parser = new GeoHashGridParser();
-        try {
-            parser.parse("geohash_grid", parseContext);
-            fail();
-        } catch (NumberFormatException ex) {
-            assertEquals("For input string: \"2.0\"", ex.getMessage());
-        }
+        assertNotNull(GeoGridAggregationBuilder.parse("geohash_grid", parseContext));
     }
 
     public void testParseErrorOnBooleanPrecision() throws Exception {
-        XContentParser stParser = JsonXContent.jsonXContent.createParser("{\"field\":\"my_loc\", \"precision\":false}");
-        QueryParseContext parseContext = new QueryParseContext(mockRegistry, stParser, ParseFieldMatcher.STRICT);
+        XContentParser stParser = createParser(JsonXContent.jsonXContent, "{\"field\":\"my_loc\", \"precision\":false}");
+        QueryParseContext parseContext = new QueryParseContext(stParser);
         XContentParser.Token token = stParser.nextToken();
         assertSame(XContentParser.Token.START_OBJECT, token);
-        GeoHashGridParser parser = new GeoHashGridParser();
         try {
-            parser.parse("geohash_grid", parseContext);
+            GeoGridAggregationBuilder.parse("geohash_grid", parseContext);
             fail();
-        } catch (ParsingException ex) {
-            assertEquals("Unexpected token VALUE_BOOLEAN [precision] in [geohash_grid].", ex.getMessage());
+        } catch (IllegalArgumentException ex) {
+            assertEquals("[geohash_grid] precision doesn't support values of type: VALUE_BOOLEAN", ex.getMessage());
         }
     }
 
     public void testParseErrorOnPrecisionOutOfRange() throws Exception {
-        XContentParser stParser = JsonXContent.jsonXContent.createParser("{\"field\":\"my_loc\", \"precision\":\"13\"}");
-        QueryParseContext parseContext = new QueryParseContext(mockRegistry, stParser, ParseFieldMatcher.STRICT);
+        XContentParser stParser = createParser(JsonXContent.jsonXContent, "{\"field\":\"my_loc\", \"precision\":\"13\"}");
+        QueryParseContext parseContext = new QueryParseContext(stParser);
         XContentParser.Token token = stParser.nextToken();
         assertSame(XContentParser.Token.START_OBJECT, token);
-        GeoHashGridParser parser = new GeoHashGridParser();
         try {
-            parser.parse("geohash_grid", parseContext);
+            GeoGridAggregationBuilder.parse("geohash_grid", parseContext);
             fail();
-        } catch (IllegalArgumentException ex) {
-            assertEquals("Invalid geohash aggregation precision of 13. Must be between 1 and 12.", ex.getMessage());
+        } catch (ParsingException ex) {
+            assertThat(ex.getCause(), instanceOf(IllegalArgumentException.class));
+            assertEquals("Invalid geohash aggregation precision of 13. Must be between 1 and 12.", ex.getCause().getMessage());
         }
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoHashGridTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoHashGridTests.java
new file mode 100644
index 0000000000000..f5df9d179d1d1
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoHashGridTests.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.bucket.geogrid;
+
+import org.apache.lucene.index.IndexWriter;
+import org.elasticsearch.common.geo.GeoHashUtils;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.aggregations.InternalMultiBucketAggregationTestCase;
+import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class InternalGeoHashGridTests extends InternalMultiBucketAggregationTestCase<InternalGeoHashGrid> {
+
+    @Override
+    protected int minNumberOfBuckets() {
+        return 1;
+    }
+
+    @Override
+    protected int maxNumberOfBuckets() {
+        return 3;
+    }
+
+    @Override
+    protected InternalGeoHashGrid createTestInstance(String name,
+                                                     List<PipelineAggregator> pipelineAggregators,
+                                                     Map<String, Object> metaData,
+                                                     InternalAggregations aggregations) {
+        int size = randomNumberOfBuckets();
+        List<InternalGeoHashGrid.Bucket> buckets = new ArrayList<>(size);
+        for (int i = 0; i < size; i++) {
+            double latitude = randomDoubleBetween(-90.0, 90.0, false);
+            double longitude = randomDoubleBetween(-180.0, 180.0, false);
+
+            long geoHashAsLong = GeoHashUtils.longEncode(longitude, latitude, 4);
+            buckets.add(new InternalGeoHashGrid.Bucket(geoHashAsLong, randomInt(IndexWriter.MAX_DOCS), aggregations));
+        }
+        return new InternalGeoHashGrid(name, size, buckets, pipelineAggregators, metaData);
+    }
+
+    @Override
+    protected Class<? extends ParsedMultiBucketAggregation> implementationClass() {
+        return ParsedGeoHashGrid.class;
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/global/InternalGlobalTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/global/InternalGlobalTests.java
new file mode 100644
index 0000000000000..a949fc6de7913
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/global/InternalGlobalTests.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.global;
+
+import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.aggregations.InternalSingleBucketAggregationTestCase;
+import org.elasticsearch.search.aggregations.bucket.ParsedSingleBucketAggregation;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+
+import java.util.List;
+import java.util.Map;
+
+public class InternalGlobalTests extends InternalSingleBucketAggregationTestCase<InternalGlobal> {
+    @Override
+    protected InternalGlobal createTestInstance(String name, long docCount, InternalAggregations aggregations,
+            List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
+        return new InternalGlobal(name, docCount, aggregations, pipelineAggregators, metaData);
+    }
+
+    @Override
+    protected Class<? extends ParsedSingleBucketAggregation> implementationClass() {
+        return ParsedGlobal.class;
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java
new file mode 100644
index 0000000000000..9d3374630a9c7
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java
@@ -0,0 +1,404 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.histogram;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.LongPoint;
+import org.apache.lucene.document.SortedNumericDocValuesField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.MatchNoDocsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.store.Directory;
+import org.elasticsearch.index.mapper.DateFieldMapper;
+import org.elasticsearch.search.aggregations.AggregatorTestCase;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.function.Consumer;
+
+public class DateHistogramAggregatorTests extends AggregatorTestCase {
+
+    private static final String DATE_FIELD = "date";
+    private static final String INSTANT_FIELD = "instant";
+
+    private static final List<String> dataset = Arrays.asList(
+            "2010-03-12T01:07:45",
+            "2010-04-27T03:43:34",
+            "2012-05-18T04:11:00",
+            "2013-05-29T05:11:31",
+            "2013-10-31T08:24:05",
+            "2015-02-13T13:09:32",
+            "2015-06-24T13:47:43",
+            "2015-11-13T16:14:34",
+            "2016-03-04T17:09:50",
+            "2017-12-12T22:55:46");
+
+    public void testMatchNoDocs() throws IOException {
+        testBothCases(new MatchNoDocsQuery(), dataset,
+                aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.YEAR).field(DATE_FIELD),
+                histogram -> assertEquals(0, histogram.getBuckets().size())
+        );
+    }
+
+    public void testMatchAllDocs() throws IOException {
+        Query query = new MatchAllDocsQuery();
+
+        testSearchCase(query, dataset,
+                aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.YEAR).field(DATE_FIELD),
+                histogram -> assertEquals(6, histogram.getBuckets().size())
+        );
+        testSearchAndReduceCase(query, dataset,
+                aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.YEAR).field(DATE_FIELD),
+                histogram -> assertEquals(8, histogram.getBuckets().size())
+        );
+        testBothCases(query, dataset,
+                aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.YEAR).field(DATE_FIELD).minDocCount(1L),
+                histogram -> assertEquals(6, histogram.getBuckets().size())
+        );
+    }
+
+    public void testNoDocs() throws IOException {
+        Query query = new MatchNoDocsQuery();
+        List<String> dates = Collections.emptyList();
+        Consumer<DateHistogramAggregationBuilder> aggregation = agg ->
+                agg.dateHistogramInterval(DateHistogramInterval.YEAR).field(DATE_FIELD);
+
+        testSearchCase(query, dates, aggregation,
+                histogram -> assertEquals(0, histogram.getBuckets().size())
+        );
+        testSearchAndReduceCase(query, dates, aggregation,
+                histogram -> assertNull(histogram)
+        );
+    }
+
+    public void testAggregateWrongField() throws IOException {
+        testBothCases(new MatchAllDocsQuery(), dataset,
+                aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.YEAR).field("wrong_field"),
+                histogram -> assertEquals(0, histogram.getBuckets().size())
+        );
+    }
+
+    public void testIntervalYear() throws IOException {
+        testBothCases(LongPoint.newRangeQuery(INSTANT_FIELD, asLong("2015-01-01"), asLong("2017-12-31")), dataset,
+                aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.YEAR).field(DATE_FIELD),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(3, buckets.size());
+
+                    Histogram.Bucket bucket = buckets.get(0);
+                    assertEquals("2015-01-01T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(3, bucket.getDocCount());
+
+                    bucket = buckets.get(1);
+                    assertEquals("2016-01-01T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(2);
+                    assertEquals("2017-01-01T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+                }
+        );
+    }
+
+    public void testIntervalMonth() throws IOException {
+        testBothCases(new MatchAllDocsQuery(),
+                Arrays.asList("2017-01-01", "2017-02-02", "2017-02-03", "2017-03-04", "2017-03-05", "2017-03-06"),
+                aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.MONTH).field(DATE_FIELD),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(3, buckets.size());
+
+                    Histogram.Bucket bucket = buckets.get(0);
+                    assertEquals("2017-01-01T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(1);
+                    assertEquals("2017-02-01T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(2, bucket.getDocCount());
+
+                    bucket = buckets.get(2);
+                    assertEquals("2017-03-01T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(3, bucket.getDocCount());
+                }
+        );
+    }
+
+    public void testIntervalDay() throws IOException {
+        testBothCases(new MatchAllDocsQuery(),
+                Arrays.asList(
+                        "2017-02-01",
+                        "2017-02-02",
+                        "2017-02-02",
+                        "2017-02-03",
+                        "2017-02-03",
+                        "2017-02-03",
+                        "2017-02-05"
+                ),
+                aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.DAY).field(DATE_FIELD).minDocCount(1L),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(4, buckets.size());
+
+                    Histogram.Bucket bucket = buckets.get(0);
+                    assertEquals("2017-02-01T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(1);
+                    assertEquals("2017-02-02T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(2, bucket.getDocCount());
+
+                    bucket = buckets.get(2);
+                    assertEquals("2017-02-03T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(3, bucket.getDocCount());
+
+                    bucket = buckets.get(3);
+                    assertEquals("2017-02-05T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+                }
+        );
+    }
+
+    public void testIntervalHour() throws IOException {
+        testBothCases(new MatchAllDocsQuery(),
+                Arrays.asList(
+                        "2017-02-01T09:02:00.000Z",
+                        "2017-02-01T09:35:00.000Z",
+                        "2017-02-01T10:15:00.000Z",
+                        "2017-02-01T13:06:00.000Z",
+                        "2017-02-01T14:04:00.000Z",
+                        "2017-02-01T14:05:00.000Z",
+                        "2017-02-01T15:59:00.000Z",
+                        "2017-02-01T16:06:00.000Z",
+                        "2017-02-01T16:48:00.000Z",
+                        "2017-02-01T16:59:00.000Z"
+                ),
+                aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.HOUR).field(DATE_FIELD).minDocCount(1L),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(6, buckets.size());
+
+                    Histogram.Bucket bucket = buckets.get(0);
+                    assertEquals("2017-02-01T09:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(2, bucket.getDocCount());
+
+                    bucket = buckets.get(1);
+                    assertEquals("2017-02-01T10:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(2);
+                    assertEquals("2017-02-01T13:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(3);
+                    assertEquals("2017-02-01T14:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(2, bucket.getDocCount());
+
+                    bucket = buckets.get(4);
+                    assertEquals("2017-02-01T15:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(5);
+                    assertEquals("2017-02-01T16:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(3, bucket.getDocCount());
+                }
+        );
+    }
+
+    public void testIntervalMinute() throws IOException {
+        testBothCases(new MatchAllDocsQuery(),
+                Arrays.asList(
+                        "2017-02-01T09:02:35.000Z",
+                        "2017-02-01T09:02:59.000Z",
+                        "2017-02-01T09:15:37.000Z",
+                        "2017-02-01T09:16:04.000Z",
+                        "2017-02-01T09:16:42.000Z"
+                ),
+                aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.MINUTE).field(DATE_FIELD).minDocCount(1L),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(3, buckets.size());
+
+                    Histogram.Bucket bucket = buckets.get(0);
+                    assertEquals("2017-02-01T09:02:00.000Z", bucket.getKeyAsString());
+                    assertEquals(2, bucket.getDocCount());
+
+                    bucket = buckets.get(1);
+                    assertEquals("2017-02-01T09:15:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(2);
+                    assertEquals("2017-02-01T09:16:00.000Z", bucket.getKeyAsString());
+                    assertEquals(2, bucket.getDocCount());
+                }
+        );
+    }
+
+    public void testIntervalSecond() throws IOException {
+        testBothCases(new MatchAllDocsQuery(),
+                Arrays.asList(
+                        "2017-02-01T00:00:05.015Z",
+                        "2017-02-01T00:00:11.299Z",
+                        "2017-02-01T00:00:11.074Z",
+                        "2017-02-01T00:00:37.688Z",
+                        "2017-02-01T00:00:37.210Z",
+                        "2017-02-01T00:00:37.380Z"
+                ),
+                aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.SECOND).field(DATE_FIELD).minDocCount(1L),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(3, buckets.size());
+
+                    Histogram.Bucket bucket = buckets.get(0);
+                    assertEquals("2017-02-01T00:00:05.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(1);
+                    assertEquals("2017-02-01T00:00:11.000Z", bucket.getKeyAsString());
+                    assertEquals(2, bucket.getDocCount());
+
+                    bucket = buckets.get(2);
+                    assertEquals("2017-02-01T00:00:37.000Z", bucket.getKeyAsString());
+                    assertEquals(3, bucket.getDocCount());
+                }
+        );
+    }
+
+    public void testMinDocCount() throws IOException {
+        Query query = LongPoint.newRangeQuery(INSTANT_FIELD, asLong("2017-02-01T00:00:00.000Z"), asLong("2017-02-01T00:00:30.000Z"));
+        List<String> timestamps = Arrays.asList(
+                "2017-02-01T00:00:05.015Z",
+                "2017-02-01T00:00:11.299Z",
+                "2017-02-01T00:00:11.074Z",
+                "2017-02-01T00:00:13.688Z",
+                "2017-02-01T00:00:21.380Z"
+        );
+
+        // 5 sec interval with minDocCount = 0
+        testSearchAndReduceCase(query, timestamps,
+                aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.seconds(5)).field(DATE_FIELD).minDocCount(0L),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(4, buckets.size());
+
+                    Histogram.Bucket bucket = buckets.get(0);
+                    assertEquals("2017-02-01T00:00:05.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(1);
+                    assertEquals("2017-02-01T00:00:10.000Z", bucket.getKeyAsString());
+                    assertEquals(3, bucket.getDocCount());
+
+                    bucket = buckets.get(2);
+                    assertEquals("2017-02-01T00:00:15.000Z", bucket.getKeyAsString());
+                    assertEquals(0, bucket.getDocCount());
+
+                    bucket = buckets.get(3);
+                    assertEquals("2017-02-01T00:00:20.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+                }
+        );
+
+        // 5 sec interval with minDocCount = 3
+        testSearchAndReduceCase(query, timestamps,
+                aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.seconds(5)).field(DATE_FIELD).minDocCount(3L),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(1, buckets.size());
+
+                    Histogram.Bucket bucket = buckets.get(0);
+                    assertEquals("2017-02-01T00:00:10.000Z", bucket.getKeyAsString());
+                    assertEquals(3, bucket.getDocCount());
+                }
+        );
+    }
+
+    private void testSearchCase(Query query, List<String> dataset,
+                                Consumer<DateHistogramAggregationBuilder> configure,
+                                Consumer<Histogram> verify) throws IOException {
+        executeTestCase(false, query, dataset, configure, verify);
+    }
+
+    private void testSearchAndReduceCase(Query query, List<String> dataset,
+                                         Consumer<DateHistogramAggregationBuilder> configure,
+                                         Consumer<Histogram> verify) throws IOException {
+        executeTestCase(true, query, dataset, configure, verify);
+    }
+
+    private void testBothCases(Query query, List<String> dataset,
+                               Consumer<DateHistogramAggregationBuilder> configure,
+                               Consumer<Histogram> verify) throws IOException {
+        testSearchCase(query, dataset, configure, verify);
+        testSearchAndReduceCase(query, dataset, configure, verify);
+    }
+
+    private void executeTestCase(boolean reduced, Query query, List<String> dataset,
+                                 Consumer<DateHistogramAggregationBuilder> configure,
+                                 Consumer<Histogram> verify) throws IOException {
+
+        try (Directory directory = newDirectory()) {
+            try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
+                Document document = new Document();
+                for (String date : dataset) {
+                    if (frequently()) {
+                        indexWriter.commit();
+                    }
+
+                    long instant = asLong(date);
+                    document.add(new SortedNumericDocValuesField(DATE_FIELD, instant));
+                    document.add(new LongPoint(INSTANT_FIELD, instant));
+                    indexWriter.addDocument(document);
+                    document.clear();
+                }
+            }
+
+            try (IndexReader indexReader = DirectoryReader.open(directory)) {
+                IndexSearcher indexSearcher = newSearcher(indexReader, true, true);
+
+                DateHistogramAggregationBuilder aggregationBuilder = new DateHistogramAggregationBuilder("_name");
+                if (configure != null) {
+                    configure.accept(aggregationBuilder);
+                }
+
+                DateFieldMapper.Builder builder = new DateFieldMapper.Builder("_name");
+                DateFieldMapper.DateFieldType fieldType = builder.fieldType();
+                fieldType.setHasDocValues(true);
+                fieldType.setName(aggregationBuilder.field());
+
+                InternalDateHistogram histogram;
+                if (reduced) {
+                    histogram = searchAndReduce(indexSearcher, query, aggregationBuilder, fieldType);
+                } else {
+                    histogram = search(indexSearcher, query, aggregationBuilder, fieldType);
+                }
+                verify.accept(histogram);
+            }
+        }
+    }
+
+    private static long asLong(String dateTime) {
+        return DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parser().parseDateTime(dateTime).getMillis();
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/ExtendedBoundsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/ExtendedBoundsTests.java
index 6b7e51cc90f64..16ba3ed6958b7 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/ExtendedBoundsTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/ExtendedBoundsTests.java
@@ -19,22 +19,24 @@
 
 package org.elasticsearch.search.aggregations.bucket.histogram;
 
-import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.joda.FormatDateTimeFormatter;
 import org.elasticsearch.common.joda.Joda;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.SearchParseException;
-import org.elasticsearch.search.aggregations.bucket.histogram.ExtendedBounds;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.threadpool.ThreadPoolStats;
 import org.joda.time.DateTimeZone;
 import org.joda.time.Instant;
 
@@ -92,8 +94,13 @@ public static ExtendedBounds unparsed(ExtendedBounds template) {
 
     public void testParseAndValidate() {
         long now = randomLong();
+        Settings indexSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1).build();
         SearchContext context = mock(SearchContext.class);
-        when(context.nowInMillis()).thenReturn(now);
+        QueryShardContext qsc = new QueryShardContext(0,
+                new IndexSettings(IndexMetaData.builder("foo").settings(indexSettings).build(), indexSettings), null, null, null, null,
+                null, xContentRegistry(), null, null, () -> now);
+        when(context.getQueryShardContext()).thenReturn(qsc);
         FormatDateTimeFormatter formatter = Joda.forPattern("dateOptionalTime");
         DocValueFormat format = new DocValueFormat.DateTime(formatter, DateTimeZone.UTC);
 
@@ -155,7 +162,7 @@ public void testXContentRoundTrip() throws Exception {
             orig.toXContent(out, ToXContent.EMPTY_PARAMS);
             out.endObject();
 
-            try (XContentParser in = JsonXContent.jsonXContent.createParser(out.bytes())) {
+            try (XContentParser in = createParser(JsonXContent.jsonXContent, out.bytes())) {
                 XContentParser.Token token = in.currentToken();
                 assertNull(token);
 
@@ -166,7 +173,7 @@ public void testXContentRoundTrip() throws Exception {
                 assertThat(token, equalTo(XContentParser.Token.FIELD_NAME));
                 assertThat(in.currentName(), equalTo(ExtendedBounds.EXTENDED_BOUNDS_FIELD.getPreferredName()));
 
-                ExtendedBounds read = ExtendedBounds.PARSER.apply(in, () -> ParseFieldMatcher.STRICT);
+                ExtendedBounds read = ExtendedBounds.PARSER.apply(in, null);
                 assertEquals(orig, read);
             } catch (Exception e) {
                 throw new Exception("Error parsing [" + out.bytes().utf8ToString() + "]", e);
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogramTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogramTests.java
new file mode 100644
index 0000000000000..dd92f84e111ab
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogramTests.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.histogram;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.aggregations.InternalMultiBucketAggregationTestCase;
+import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.joda.time.DateTime;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+import static org.elasticsearch.common.unit.TimeValue.timeValueHours;
+import static org.elasticsearch.common.unit.TimeValue.timeValueMinutes;
+import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
+
+public class InternalDateHistogramTests extends InternalMultiBucketAggregationTestCase<InternalDateHistogram> {
+
+    private boolean keyed;
+    private DocValueFormat format;
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        keyed = randomBoolean();
+        format = randomNumericDocValueFormat();
+    }
+
+    @Override
+    protected InternalDateHistogram createTestInstance(String name,
+                                                       List<PipelineAggregator> pipelineAggregators,
+                                                       Map<String, Object> metaData,
+                                                       InternalAggregations aggregations) {
+        int nbBuckets = randomNumberOfBuckets();
+        List<InternalDateHistogram.Bucket> buckets = new ArrayList<>(nbBuckets);
+        long startingDate = System.currentTimeMillis();
+
+        long interval = randomIntBetween(1, 3);
+        long intervalMillis = randomFrom(timeValueSeconds(interval), timeValueMinutes(interval), timeValueHours(interval)).getMillis();
+
+        for (int i = 0; i < nbBuckets; i++) {
+            long key = startingDate + (intervalMillis * i);
+            buckets.add(i, new InternalDateHistogram.Bucket(key, randomIntBetween(1, 100), keyed, format, aggregations));
+        }
+
+        InternalOrder order = (InternalOrder) randomFrom(InternalHistogram.Order.KEY_ASC, InternalHistogram.Order.KEY_DESC);
+        return new InternalDateHistogram(name, buckets, order, 1, 0L, null, format, keyed, pipelineAggregators, metaData);
+    }
+
+    @Override
+    protected Class<? extends ParsedMultiBucketAggregation> implementationClass() {
+        return ParsedDateHistogram.class;
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogramTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogramTests.java
new file mode 100644
index 0000000000000..9d1c0d12a8e04
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogramTests.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.histogram;
+
+import org.apache.lucene.util.TestUtil;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.aggregations.InternalMultiBucketAggregationTestCase;
+import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+public class InternalHistogramTests extends InternalMultiBucketAggregationTestCase<InternalHistogram> {
+
+    private boolean keyed;
+    private DocValueFormat format;
+
+    @Override
+    public void setUp() throws Exception{
+        super.setUp();
+        keyed = randomBoolean();
+        format = randomNumericDocValueFormat();
+    }
+
+    @Override
+    protected InternalHistogram createTestInstance(String name,
+                                                   List<PipelineAggregator> pipelineAggregators,
+                                                   Map<String, Object> metaData,
+                                                   InternalAggregations aggregations) {
+        final int base = randomInt(50) - 30;
+        final int numBuckets = randomNumberOfBuckets();
+        final int interval = randomIntBetween(1, 3);
+        List<InternalHistogram.Bucket> buckets = new ArrayList<>();
+        for (int i = 0; i < numBuckets; ++i) {
+            final int docCount = TestUtil.nextInt(random(), 1, 50);
+            buckets.add(new InternalHistogram.Bucket(base + i * interval, docCount, keyed, format, aggregations));
+        }
+        InternalOrder order = (InternalOrder) randomFrom(InternalHistogram.Order.KEY_ASC, InternalHistogram.Order.KEY_DESC);
+        return new InternalHistogram(name, buckets, order, 1, null, format, keyed, pipelineAggregators, metaData);
+    }
+
+    // issue 26787
+    public void testHandlesNaN() {
+        InternalHistogram histogram = createTestInstance();
+        InternalHistogram histogram2 = createTestInstance();
+        List<InternalHistogram.Bucket> buckets = histogram.getBuckets();
+        if (buckets == null || buckets.isEmpty()) {
+            return;
+        }
+
+        // Set the key of one bucket to NaN. Must be the last bucket because NaN is greater than everything else.
+        List<InternalHistogram.Bucket> newBuckets = new ArrayList<>(buckets.size());
+        if (buckets.size() > 1) {
+            newBuckets.addAll(buckets.subList(0, buckets.size() - 1));
+        }
+        InternalHistogram.Bucket b = buckets.get(buckets.size() - 1);
+        newBuckets.add(new InternalHistogram.Bucket(Double.NaN, b.docCount, keyed, b.format, b.aggregations));
+        
+        InternalHistogram newHistogram = histogram.create(newBuckets);
+        newHistogram.doReduce(Arrays.asList(newHistogram, histogram2), new InternalAggregation.ReduceContext(null, null, false));
+    }
+
+    @Override
+    protected Class<? extends ParsedMultiBucketAggregation> implementationClass() {
+        return ParsedHistogram.class;
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/InternalMissingTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/InternalMissingTests.java
new file mode 100644
index 0000000000000..6e6af95d937f9
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/InternalMissingTests.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.missing;
+
+import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.aggregations.InternalSingleBucketAggregationTestCase;
+import org.elasticsearch.search.aggregations.bucket.ParsedSingleBucketAggregation;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+
+import java.util.List;
+import java.util.Map;
+
+public class InternalMissingTests extends InternalSingleBucketAggregationTestCase<InternalMissing> {
+    @Override
+    protected InternalMissing createTestInstance(String name, long docCount, InternalAggregations aggregations,
+            List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
+        return new InternalMissing(name, docCount, aggregations, pipelineAggregators, metaData);
+    }
+
+    @Override
+    protected Class<? extends ParsedSingleBucketAggregation> implementationClass() {
+        return ParsedMissing.class;
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/InternalNestedTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/InternalNestedTests.java
new file mode 100644
index 0000000000000..bf45bbe718ab9
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/InternalNestedTests.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.nested;
+
+import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.aggregations.InternalSingleBucketAggregationTestCase;
+import org.elasticsearch.search.aggregations.bucket.ParsedSingleBucketAggregation;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+
+import java.util.List;
+import java.util.Map;
+
+public class InternalNestedTests extends InternalSingleBucketAggregationTestCase<InternalNested> {
+    @Override
+    protected InternalNested createTestInstance(String name, long docCount, InternalAggregations aggregations,
+            List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
+        return new InternalNested(name, docCount, aggregations, pipelineAggregators, metaData);
+    }
+
+    @Override
+    protected Class<? extends ParsedSingleBucketAggregation> implementationClass() {
+        return ParsedNested.class;
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/InternalReverseNestedTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/InternalReverseNestedTests.java
new file mode 100644
index 0000000000000..49dd04aff354e
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/InternalReverseNestedTests.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.nested;
+
+import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.aggregations.InternalSingleBucketAggregationTestCase;
+import org.elasticsearch.search.aggregations.bucket.ParsedSingleBucketAggregation;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+
+import java.util.List;
+import java.util.Map;
+
+public class InternalReverseNestedTests extends InternalSingleBucketAggregationTestCase<InternalReverseNested> {
+    @Override
+    protected InternalReverseNested createTestInstance(String name, long docCount, InternalAggregations aggregations,
+            List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
+        return new InternalReverseNested(name, docCount, aggregations, pipelineAggregators, metaData);
+    }
+
+    @Override
+    protected Class<? extends ParsedSingleBucketAggregation> implementationClass() {
+        return ParsedReverseNested.class;
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java
index a3bb846c3e14a..245ae3a294f80 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java
@@ -45,7 +45,6 @@
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.BucketCollector;
 import org.elasticsearch.search.aggregations.SearchContextAggregations;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
@@ -119,14 +118,13 @@ public void testResetRootDocId() throws Exception {
         IndexSearcher searcher = new IndexSearcher(directoryReader);
 
         indexService.mapperService().merge("test", new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef("test", "nested_field", "type=nested").string()), MapperService.MergeReason.MAPPING_UPDATE, false);
-        SearchContext searchContext = createSearchContext(indexService);
-        AggregationContext context = new AggregationContext(searchContext);
+        SearchContext context = createSearchContext(indexService);
 
         AggregatorFactories.Builder builder = AggregatorFactories.builder();
         NestedAggregationBuilder factory = new NestedAggregationBuilder("test", "nested_field");
         builder.addAggregator(factory);
         AggregatorFactories factories = builder.build(context, null);
-        searchContext.aggregations(new SearchContextAggregations(factories));
+        context.aggregations(new SearchContextAggregations(factories));
         Aggregator[] aggs = factories.createTopLevelAggregators();
         BucketCollector collector = BucketCollector.wrap(Arrays.asList(aggs));
         collector.preCollection();
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedTests.java
index 29dde100a0891..ff2563fc4cc83 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedTests.java
@@ -25,7 +25,7 @@ public class NestedTests extends BaseAggregationTestCase<NestedAggregationBuilde
 
     @Override
     protected NestedAggregationBuilder createTestAggregatorBuilder() {
-        return new NestedAggregationBuilder(randomAsciiOfLengthBetween(1, 20), randomAsciiOfLengthBetween(3, 40));
+        return new NestedAggregationBuilder(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(3, 40));
     }
 
 }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorTests.java
new file mode 100644
index 0000000000000..70e6c3559409a
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorTests.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.nested;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.SortedNumericDocValuesField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.store.Directory;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.NumberFieldMapper;
+import org.elasticsearch.index.mapper.TypeFieldMapper;
+import org.elasticsearch.index.mapper.UidFieldMapper;
+import org.elasticsearch.search.aggregations.AggregatorTestCase;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.metrics.max.InternalMax;
+import org.elasticsearch.search.aggregations.metrics.max.MaxAggregationBuilder;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class ReverseNestedAggregatorTests extends AggregatorTestCase {
+
+    private static final String VALUE_FIELD_NAME = "number";
+    private static final String NESTED_OBJECT = "nested_object";
+    private static final String NESTED_AGG = "nestedAgg";
+    private static final String REVERSE_AGG_NAME = "reverseNestedAgg";
+    private static final String MAX_AGG_NAME = "maxAgg";
+
+
+    public void testNoDocs() throws IOException {
+        try (Directory directory = newDirectory()) {
+            try (RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) {
+                // intentionally not writing any docs
+            }
+            try (IndexReader indexReader = wrap(DirectoryReader.open(directory))) {
+                NestedAggregationBuilder nestedBuilder = new NestedAggregationBuilder(NESTED_AGG,
+                        NESTED_OBJECT);
+                ReverseNestedAggregationBuilder reverseNestedBuilder
+                    = new ReverseNestedAggregationBuilder(REVERSE_AGG_NAME);
+                nestedBuilder.subAggregation(reverseNestedBuilder);
+                MaxAggregationBuilder maxAgg = new MaxAggregationBuilder(MAX_AGG_NAME)
+                        .field(VALUE_FIELD_NAME);
+                reverseNestedBuilder.subAggregation(maxAgg);
+                MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(
+                        NumberFieldMapper.NumberType.LONG);
+                fieldType.setName(VALUE_FIELD_NAME);
+
+                Nested nested = search(newSearcher(indexReader, false, true),
+                        new MatchAllDocsQuery(), nestedBuilder, fieldType);
+                ReverseNested reverseNested = (ReverseNested)
+                        ((InternalAggregation)nested).getProperty(REVERSE_AGG_NAME);
+                assertEquals(REVERSE_AGG_NAME, reverseNested.getName());
+                assertEquals(0, reverseNested.getDocCount());
+
+                InternalMax max = (InternalMax)
+                        ((InternalAggregation)reverseNested).getProperty(MAX_AGG_NAME);
+                assertEquals(MAX_AGG_NAME, max.getName());
+                assertEquals(Double.NEGATIVE_INFINITY, max.getValue(), Double.MIN_VALUE);
+            }
+        }
+    }
+
+    public void testMaxFromParentDocs() throws IOException {
+        int numParentDocs = randomIntBetween(1, 20);
+        int expectedParentDocs = 0;
+        int expectedNestedDocs = 0;
+        double expectedMaxValue = Double.NEGATIVE_INFINITY;
+        try (Directory directory = newDirectory()) {
+            try (RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) {
+                for (int i = 0; i < numParentDocs; i++) {
+                    List<Document> documents = new ArrayList<>();
+                    int numNestedDocs = randomIntBetween(0, 20);
+                    for (int nested = 0; nested < numNestedDocs; nested++) {
+                        Document document = new Document();
+                        document.add(new Field(UidFieldMapper.NAME, "type#" + i,
+                                UidFieldMapper.Defaults.NESTED_FIELD_TYPE));
+                        document.add(new Field(TypeFieldMapper.NAME, "__" + NESTED_OBJECT,
+                                TypeFieldMapper.Defaults.FIELD_TYPE));
+                        documents.add(document);
+                        expectedNestedDocs++;
+                    }
+                    Document document = new Document();
+                    document.add(new Field(UidFieldMapper.NAME, "type#" + i,
+                            UidFieldMapper.Defaults.FIELD_TYPE));
+                    document.add(new Field(TypeFieldMapper.NAME, "test",
+                            TypeFieldMapper.Defaults.FIELD_TYPE));
+                    long value = randomNonNegativeLong() % 10000;
+                    document.add(new SortedNumericDocValuesField(VALUE_FIELD_NAME, value));
+                    if (numNestedDocs > 0) {
+                        expectedMaxValue = Math.max(expectedMaxValue, value);
+                        expectedParentDocs++;
+                    }
+                    documents.add(document);
+                    iw.addDocuments(documents);
+                }
+                iw.commit();
+            }
+            try (IndexReader indexReader = wrap(DirectoryReader.open(directory))) {
+                NestedAggregationBuilder nestedBuilder = new NestedAggregationBuilder(NESTED_AGG,
+                        NESTED_OBJECT);
+                ReverseNestedAggregationBuilder reverseNestedBuilder
+                    = new ReverseNestedAggregationBuilder(REVERSE_AGG_NAME);
+                nestedBuilder.subAggregation(reverseNestedBuilder);
+                MaxAggregationBuilder maxAgg = new MaxAggregationBuilder(MAX_AGG_NAME)
+                        .field(VALUE_FIELD_NAME);
+                reverseNestedBuilder.subAggregation(maxAgg);
+                MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(
+                        NumberFieldMapper.NumberType.LONG);
+                fieldType.setName(VALUE_FIELD_NAME);
+
+                Nested nested = search(newSearcher(indexReader, false, true),
+                        new MatchAllDocsQuery(), nestedBuilder, fieldType);
+                assertEquals(expectedNestedDocs, nested.getDocCount());
+
+                ReverseNested reverseNested = (ReverseNested)
+                        ((InternalAggregation)nested).getProperty(REVERSE_AGG_NAME);
+                assertEquals(REVERSE_AGG_NAME, reverseNested.getName());
+                assertEquals(expectedParentDocs, reverseNested.getDocCount());
+
+                InternalMax max = (InternalMax)
+                        ((InternalAggregation)reverseNested).getProperty(MAX_AGG_NAME);
+                assertEquals(MAX_AGG_NAME, max.getName());
+                assertEquals(expectedMaxValue, max.getValue(), Double.MIN_VALUE);
+            }
+        }
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedTests.java
index 97dbf3718af95..76423a700595a 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedTests.java
@@ -25,9 +25,9 @@ public class ReverseNestedTests extends BaseAggregationTestCase<ReverseNestedAgg
 
     @Override
     protected ReverseNestedAggregationBuilder createTestAggregatorBuilder() {
-        ReverseNestedAggregationBuilder factory = new ReverseNestedAggregationBuilder(randomAsciiOfLengthBetween(1, 20));
+        ReverseNestedAggregationBuilder factory = new ReverseNestedAggregationBuilder(randomAlphaOfLengthBetween(1, 20));
         if (randomBoolean()) {
-            factory.path(randomAsciiOfLengthBetween(3, 40));
+            factory.path(randomAlphaOfLengthBetween(3, 40));
         }
         return factory;
     }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRangeTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRangeTests.java
new file mode 100644
index 0000000000000..8bcf6899752f0
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRangeTests.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.bucket.range;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation;
+import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+public class InternalBinaryRangeTests extends InternalRangeTestCase<InternalBinaryRange> {
+
+    private List<Tuple<BytesRef, BytesRef>> ranges;
+
+    @Override
+    protected int minNumberOfBuckets() {
+        return 1;
+    }
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+
+        List<Tuple<BytesRef, BytesRef>> listOfRanges = new ArrayList<>();
+        if (randomBoolean()) {
+            listOfRanges.add(Tuple.tuple(null, new BytesRef(randomAlphaOfLength(15))));
+        }
+        if (randomBoolean()) {
+            listOfRanges.add(Tuple.tuple(new BytesRef(randomAlphaOfLength(15)), null));
+        }
+        if (randomBoolean()) {
+            listOfRanges.add(Tuple.tuple(null, null));
+        }
+
+        final int numRanges = Math.max(0, randomNumberOfBuckets() - listOfRanges.size());
+        for (int i = 0; i < numRanges; i++) {
+            BytesRef[] values = new BytesRef[2];
+            values[0] = new BytesRef(randomAlphaOfLength(15));
+            values[1] = new BytesRef(randomAlphaOfLength(15));
+            Arrays.sort(values);
+            listOfRanges.add(Tuple.tuple(values[0], values[1]));
+        }
+        Collections.shuffle(listOfRanges, random());
+        ranges = Collections.unmodifiableList(listOfRanges);
+    }
+
+    @Override
+    protected InternalBinaryRange createTestInstance(String name,
+                                                     List<PipelineAggregator> pipelineAggregators,
+                                                     Map<String, Object> metaData,
+                                                     InternalAggregations aggregations,
+                                                     boolean keyed) {
+        DocValueFormat format = DocValueFormat.RAW;
+        List<InternalBinaryRange.Bucket> buckets = new ArrayList<>();
+
+        int nullKey = randomBoolean() ? randomIntBetween(0, ranges.size() -1) : -1;
+        for (int i = 0; i < ranges.size(); ++i) {
+            final int docCount = randomIntBetween(1, 100);
+            final String key = (i == nullKey) ? null: randomAlphaOfLength(10);
+            buckets.add(new InternalBinaryRange.Bucket(format, keyed, key, ranges.get(i).v1(), ranges.get(i).v2(), docCount, aggregations));
+        }
+        return new InternalBinaryRange(name, format, keyed, buckets, pipelineAggregators, metaData);
+    }
+
+    @Override
+    protected Class<? extends ParsedMultiBucketAggregation> implementationClass() {
+        return ParsedBinaryRange.class;
+    }
+
+    @Override
+    protected Class<? extends InternalMultiBucketAggregation.InternalBucket> internalRangeBucketClass() {
+        return InternalBinaryRange.Bucket.class;
+    }
+
+    @Override
+    protected Class<? extends ParsedMultiBucketAggregation.ParsedBucket> parsedRangeBucketClass() {
+        return ParsedBinaryRange.ParsedBucket.class;
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/range/InternalRangeTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/range/InternalRangeTestCase.java
new file mode 100644
index 0000000000000..2a70ced343887
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/range/InternalRangeTestCase.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.range;
+
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation;
+import org.elasticsearch.search.aggregations.InternalMultiBucketAggregationTestCase;
+import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation;
+import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+public abstract class InternalRangeTestCase<T extends InternalAggregation & Range> extends InternalMultiBucketAggregationTestCase<T> {
+
+    private boolean keyed;
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        keyed = randomBoolean();
+    }
+
+    @Override
+    protected T createTestInstance(String name,
+                                   List<PipelineAggregator> pipelineAggregators,
+                                   Map<String, Object> metaData,
+                                   InternalAggregations aggregations) {
+        return createTestInstance(name, pipelineAggregators, metaData, aggregations, keyed);
+    }
+
+    protected abstract T createTestInstance(String name,
+                                            List<PipelineAggregator> pipelineAggregators,
+                                            Map<String, Object> metaData,
+                                            InternalAggregations aggregations,
+                                            boolean keyed);
+    @Override
+    protected void assertBucket(MultiBucketsAggregation.Bucket expected, MultiBucketsAggregation.Bucket actual, boolean checkOrder) {
+        super.assertBucket(expected, actual, checkOrder);
+
+        Class<?> internalBucketClass = internalRangeBucketClass();
+        assertNotNull("Internal bucket class must not be null", internalBucketClass);
+        assertTrue(internalBucketClass.isInstance(expected));
+
+        Class<?> parsedBucketClass = parsedRangeBucketClass();
+        assertNotNull("Parsed bucket class must not be null", parsedBucketClass);
+        assertTrue(parsedBucketClass.isInstance(actual));
+
+        Range.Bucket expectedRange = (Range.Bucket) expected;
+        Range.Bucket actualRange = (Range.Bucket) actual;
+
+        assertEquals(expectedRange.getFrom(), actualRange.getFrom());
+        assertEquals(expectedRange.getFromAsString(), actualRange.getFromAsString());
+        assertEquals(expectedRange.getTo(), actualRange.getTo());
+        assertEquals(expectedRange.getToAsString(), actualRange.getToAsString());
+    }
+
+    protected abstract Class<? extends InternalMultiBucketAggregation.InternalBucket> internalRangeBucketClass();
+
+    protected abstract Class<? extends ParsedMultiBucketAggregation.ParsedBucket> parsedRangeBucketClass();
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/range/InternalRangeTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/range/InternalRangeTests.java
new file mode 100644
index 0000000000000..b0edb47534e35
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/range/InternalRangeTests.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.range;
+
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation;
+import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+public class InternalRangeTests extends InternalRangeTestCase<InternalRange> {
+
+    private DocValueFormat format;
+    private List<Tuple<Double, Double>> ranges;
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        format = randomNumericDocValueFormat();
+
+        List<Tuple<Double, Double>> listOfRanges = new ArrayList<>();
+        if (rarely()) {
+            listOfRanges.add(Tuple.tuple(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY));
+        }
+        if (rarely()) {
+            listOfRanges.add(Tuple.tuple(Double.NEGATIVE_INFINITY, randomDouble()));
+        }
+        if (rarely()) {
+            listOfRanges.add(Tuple.tuple(randomDouble(), Double.POSITIVE_INFINITY));
+        }
+
+        final int interval = randomFrom(1, 5, 10, 25, 50, 100);
+        final int numRanges = Math.max(0, randomNumberOfBuckets() - listOfRanges.size());
+        final double max = (double) numRanges * interval;
+
+        for (int i = 0; numRanges - listOfRanges.size() > 0; i++) {
+            double from = i * interval;
+            double to = from + interval;
+
+            Tuple<Double, Double> range;
+            if (randomBoolean()) {
+                range = Tuple.tuple(from, to);
+            } else {
+                // Add some overlapping range
+                range = Tuple.tuple(randomFrom(0.0, max / 3), randomFrom(max, max / 2, max / 3 * 2));
+            }
+            listOfRanges.add(range);
+        }
+        Collections.shuffle(listOfRanges, random());
+        ranges = Collections.unmodifiableList(listOfRanges);
+    }
+
+    @Override
+    protected InternalRange createTestInstance(String name,
+                                               List<PipelineAggregator> pipelineAggregators,
+                                               Map<String, Object> metaData,
+                                               InternalAggregations aggregations,
+                                               boolean keyed) {
+        final List<InternalRange.Bucket> buckets = new ArrayList<>();
+        for (int i = 0; i < ranges.size(); ++i) {
+            Tuple<Double, Double> range = ranges.get(i);
+            int docCount = randomIntBetween(0, 1000);
+            double from = range.v1();
+            double to = range.v2();
+            buckets.add(new InternalRange.Bucket("range_" + i, from, to, docCount, aggregations, keyed, format));
+        }
+        return new InternalRange<>(name, buckets, format, keyed, pipelineAggregators, metaData);
+    }
+
+    @Override
+    protected Class<? extends ParsedMultiBucketAggregation> implementationClass() {
+        return ParsedRange.class;
+    }
+
+    @Override
+    protected Class<? extends InternalMultiBucketAggregation.InternalBucket> internalRangeBucketClass() {
+        return InternalRange.Bucket.class;
+    }
+
+    @Override
+    protected Class<? extends ParsedMultiBucketAggregation.ParsedBucket> parsedRangeBucketClass() {
+        return ParsedRange.ParsedBucket.class;
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/range/date/InternalDateRangeTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/range/date/InternalDateRangeTests.java
new file mode 100644
index 0000000000000..dfda0749118c9
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/range/date/InternalDateRangeTests.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.range.date;
+
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation;
+import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation;
+import org.elasticsearch.search.aggregations.bucket.range.InternalRangeTestCase;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+
+public class InternalDateRangeTests extends InternalRangeTestCase<InternalDateRange> {
+
+    private DocValueFormat format;
+    private List<Tuple<Double, Double>> dateRanges;
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        format = randomNumericDocValueFormat();
+
+        Function<DateTime, DateTime> interval = randomFrom(dateTime -> dateTime.plusSeconds(1), dateTime -> dateTime.plusMinutes(1),
+                dateTime -> dateTime.plusHours(1), dateTime -> dateTime.plusDays(1), dateTime -> dateTime.plusMonths(1), dateTime ->
+                        dateTime.plusYears(1));
+
+        final int numRanges = randomNumberOfBuckets();
+        final List<Tuple<Double, Double>> listOfRanges = new ArrayList<>(numRanges);
+
+        DateTime date = new DateTime(DateTimeZone.UTC);
+        double start = date.getMillis();
+        double end = 0;
+        for (int i = 0; i < numRanges; i++) {
+            double from = date.getMillis();
+            date = interval.apply(date);
+            double to = date.getMillis();
+            if (to > end) {
+                end = to;
+            }
+
+            if (randomBoolean()) {
+                listOfRanges.add(Tuple.tuple(from, to));
+            } else {
+                // Add some overlapping range
+                listOfRanges.add(Tuple.tuple(start, randomDoubleBetween(start, end, false)));
+            }
+        }
+        Collections.shuffle(listOfRanges, random());
+        dateRanges = Collections.unmodifiableList(listOfRanges);
+    }
+
+    @Override
+    protected InternalDateRange createTestInstance(String name,
+                                                   List<PipelineAggregator> pipelineAggregators,
+                                                   Map<String, Object> metaData,
+                                                   InternalAggregations aggregations,
+                                                   boolean keyed) {
+        final List<InternalDateRange.Bucket> buckets = new ArrayList<>();
+        for (int i = 0; i < dateRanges.size(); ++i) {
+            Tuple<Double, Double> range = dateRanges.get(i);
+            int docCount = randomIntBetween(0, 1000);
+            double from = range.v1();
+            double to = range.v2();
+            buckets.add(new InternalDateRange.Bucket("range_" + i, from, to, docCount, aggregations, keyed, format));
+        }
+        return new InternalDateRange(name, buckets, format, keyed, pipelineAggregators, metaData);
+    }
+
+    @Override
+    protected Class<? extends ParsedMultiBucketAggregation> implementationClass() {
+        return ParsedDateRange.class;
+    }
+
+    @Override
+    protected Class<? extends InternalMultiBucketAggregation.InternalBucket> internalRangeBucketClass() {
+        return InternalDateRange.Bucket.class;
+    }
+
+    @Override
+    protected Class<? extends ParsedMultiBucketAggregation.ParsedBucket> parsedRangeBucketClass() {
+        return ParsedDateRange.ParsedBucket.class;
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/InternalGeoDistanceTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/InternalGeoDistanceTests.java
new file mode 100644
index 0000000000000..ceb182ee08821
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/InternalGeoDistanceTests.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.range.geodistance;
+
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation;
+import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation;
+import org.elasticsearch.search.aggregations.bucket.range.InternalRangeTestCase;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+public class InternalGeoDistanceTests extends InternalRangeTestCase<InternalGeoDistance> {
+
+    private List<Tuple<Double, Double>> geoDistanceRanges;
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+
+        final int interval = randomFrom(1, 5, 10, 25, 50, 100);
+        final int numRanges = randomNumberOfBuckets();
+        final double max = (double) numRanges * interval;
+
+        List<Tuple<Double, Double>> listOfRanges = new ArrayList<>(numRanges);
+        for (int i = 0; i < numRanges; i++) {
+            double from = i * interval;
+            double to = from + interval;
+
+            Tuple<Double, Double> range;
+            if (randomBoolean()) {
+                range = Tuple.tuple(from, to);
+            } else {
+                // Add some overlapping range
+                range = Tuple.tuple(randomFrom(0.0, max / 3), randomFrom(max, max / 2, max / 3 * 2));
+            }
+            listOfRanges.add(range);
+        }
+        Collections.shuffle(listOfRanges, random());
+        geoDistanceRanges = Collections.unmodifiableList(listOfRanges);
+    }
+
+    @Override
+    protected InternalGeoDistance createTestInstance(String name,
+                                                     List<PipelineAggregator> pipelineAggregators,
+                                                     Map<String, Object> metaData,
+                                                     InternalAggregations aggregations,
+                                                     boolean keyed) {
+        final List<InternalGeoDistance.Bucket> buckets = new ArrayList<>();
+        for (int i = 0; i < geoDistanceRanges.size(); ++i) {
+            Tuple<Double, Double> range = geoDistanceRanges.get(i);
+            int docCount = randomIntBetween(0, 1000);
+            double from = range.v1();
+            double to = range.v2();
+            buckets.add(new InternalGeoDistance.Bucket("range_" + i, from, to, docCount, aggregations, keyed));
+        }
+        return new InternalGeoDistance(name, buckets, keyed, pipelineAggregators, metaData);
+    }
+
+    @Override
+    protected Class<? extends ParsedMultiBucketAggregation> implementationClass() {
+        return ParsedGeoDistance.class;
+    }
+
+    @Override
+    protected Class<? extends InternalMultiBucketAggregation.InternalBucket> internalRangeBucketClass() {
+        return InternalGeoDistance.Bucket.class;
+    }
+
+    @Override
+    protected Class<? extends ParsedMultiBucketAggregation.ParsedBucket> parsedRangeBucketClass() {
+        return ParsedGeoDistance.ParsedBucket.class;
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/sampler/InternalSamplerTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/sampler/InternalSamplerTests.java
new file mode 100644
index 0000000000000..44116ae980a60
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/sampler/InternalSamplerTests.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.bucket.sampler;
+
+import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.aggregations.InternalSingleBucketAggregationTestCase;
+import org.elasticsearch.search.aggregations.bucket.ParsedSingleBucketAggregation;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+
+import java.util.List;
+import java.util.Map;
+
+public class InternalSamplerTests extends InternalSingleBucketAggregationTestCase<InternalSampler> {
+    @Override
+    protected InternalSampler createTestInstance(String name, long docCount, InternalAggregations aggregations,
+                                                 List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
+        return new InternalSampler(name, docCount, aggregations, pipelineAggregators, metaData);
+    }
+
+    @Override
+    protected Class<? extends ParsedSingleBucketAggregation> implementationClass() {
+        return ParsedSampler.class;
+    }
+}
\ No newline at end of file
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregatorTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregatorTests.java
new file mode 100644
index 0000000000000..900db1399955d
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregatorTests.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.sampler;
+
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.SortedNumericDocValuesField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.store.Directory;
+import org.elasticsearch.index.analysis.AnalyzerScope;
+import org.elasticsearch.index.analysis.NamedAnalyzer;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.NumberFieldMapper;
+import org.elasticsearch.index.mapper.TextFieldMapper.TextFieldType;
+import org.elasticsearch.search.aggregations.AggregatorTestCase;
+import org.elasticsearch.search.aggregations.metrics.min.Min;
+import org.elasticsearch.search.aggregations.metrics.min.MinAggregationBuilder;
+
+import java.io.IOException;
+
+public class SamplerAggregatorTests extends AggregatorTestCase {
+    /**
+     * Uses the sampler aggregation to find the minimum value of a field out of the top 3 scoring documents in a search.
+     */
+    public void testSampler() throws IOException {
+        TextFieldType textFieldType = new TextFieldType();
+        textFieldType.setIndexAnalyzer(new NamedAnalyzer("foo", AnalyzerScope.GLOBAL, new StandardAnalyzer()));
+        MappedFieldType numericFieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
+        numericFieldType.setName("int");
+
+        IndexWriterConfig indexWriterConfig = newIndexWriterConfig();
+        indexWriterConfig.setMaxBufferedDocs(100);
+        indexWriterConfig.setRAMBufferSizeMB(100); // flush on open to have a single segment with predictable docIds
+        try (Directory dir = newDirectory();
+                IndexWriter w = new IndexWriter(dir, indexWriterConfig)) {
+            for (long value : new long[] {7, 3, -10, -6, 5, 50}) {
+                Document doc = new Document();
+                StringBuilder text = new StringBuilder();
+                for (int i = 0; i < value; i++) {
+                    text.append("good ");
+                }
+                doc.add(new Field("text", text.toString(), textFieldType));
+                doc.add(new SortedNumericDocValuesField("int", value));
+                w.addDocument(doc);
+            }
+
+            SamplerAggregationBuilder aggBuilder = new SamplerAggregationBuilder("sampler")
+                    .shardSize(3)
+                    .subAggregation(new MinAggregationBuilder("min")
+                            .field("int"));
+            try (IndexReader reader = DirectoryReader.open(w)) {
+                assertEquals("test expects a single segment", 1, reader.leaves().size());
+                IndexSearcher searcher = new IndexSearcher(reader);
+                Sampler sampler = searchAndReduce(searcher, new TermQuery(new Term("text", "good")), aggBuilder, textFieldType,
+                        numericFieldType);
+                Min min = sampler.getAggregations().get("min");
+                assertEquals(5.0, min.getValue(), 0);
+            }
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/InternalSignificantTermsTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/InternalSignificantTermsTestCase.java
new file mode 100644
index 0000000000000..b78e4147245db
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/InternalSignificantTermsTestCase.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.significant;
+
+import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.aggregations.InternalMultiBucketAggregationTestCase;
+import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
+import org.elasticsearch.search.aggregations.bucket.significant.heuristics.ChiSquare;
+import org.elasticsearch.search.aggregations.bucket.significant.heuristics.GND;
+import org.elasticsearch.search.aggregations.bucket.significant.heuristics.JLHScore;
+import org.elasticsearch.search.aggregations.bucket.significant.heuristics.MutualInformation;
+import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristic;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+public abstract class InternalSignificantTermsTestCase extends InternalMultiBucketAggregationTestCase<InternalSignificantTerms<?, ?>> {
+
+    private SignificanceHeuristic significanceHeuristic;
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        significanceHeuristic = randomSignificanceHeuristic();
+    }
+
+    @Override
+    protected final InternalSignificantTerms createTestInstance(String name,
+                                                                List<PipelineAggregator> pipelineAggregators,
+                                                                Map<String, Object> metaData,
+                                                                InternalAggregations aggregations) {
+        final int requiredSize = randomIntBetween(1, 5);
+        final int numBuckets = randomNumberOfBuckets();
+
+        long subsetSize = 0;
+        long supersetSize = 0;
+
+        int[] subsetDfs = new int[numBuckets];
+        int[] supersetDfs = new int[numBuckets];
+
+        for (int i = 0; i < numBuckets; ++i) {
+            int subsetDf = randomIntBetween(1, 10);
+            subsetDfs[i] = subsetDf;
+
+            int supersetDf = randomIntBetween(subsetDf, 20);
+            supersetDfs[i] = supersetDf;
+
+            subsetSize += subsetDf;
+            supersetSize += supersetDf;
+        }
+        return createTestInstance(name, pipelineAggregators, metaData, aggregations, requiredSize, numBuckets, subsetSize, subsetDfs,
+                supersetSize, supersetDfs, significanceHeuristic);
+    }
+
+    protected abstract InternalSignificantTerms createTestInstance(String name,
+                                                                   List<PipelineAggregator> pipelineAggregators,
+                                                                   Map<String, Object> metaData,
+                                                                   InternalAggregations aggregations,
+                                                                   int requiredSize, int numBuckets,
+                                                                   long subsetSize, int[] subsetDfs,
+                                                                   long supersetSize, int[] supersetDfs,
+                                                                   SignificanceHeuristic significanceHeuristic);
+
+    @Override
+    protected void assertMultiBucketsAggregation(MultiBucketsAggregation expected, MultiBucketsAggregation actual, boolean checkOrder) {
+        super.assertMultiBucketsAggregation(expected, actual, checkOrder);
+
+        assertTrue(expected instanceof InternalSignificantTerms);
+        assertTrue(actual instanceof ParsedSignificantTerms);
+
+        InternalSignificantTerms expectedSigTerms = (InternalSignificantTerms) expected;
+        ParsedSignificantTerms actualSigTerms = (ParsedSignificantTerms) actual;
+        assertEquals(expectedSigTerms.getSubsetSize(), actualSigTerms.getSubsetSize());
+        assertEquals(expectedSigTerms.getSupersetSize(), actualSigTerms.getSupersetSize());
+
+        for (SignificantTerms.Bucket bucket : (SignificantTerms) expected) {
+            String key = bucket.getKeyAsString();
+            assertBucket(expectedSigTerms.getBucketByKey(key), actualSigTerms.getBucketByKey(key), checkOrder);
+        }
+    }
+
+    @Override
+    protected void assertBucket(MultiBucketsAggregation.Bucket expected, MultiBucketsAggregation.Bucket actual, boolean checkOrder) {
+        super.assertBucket(expected, actual, checkOrder);
+
+        assertTrue(expected instanceof InternalSignificantTerms.Bucket);
+        assertTrue(actual instanceof ParsedSignificantTerms.ParsedBucket);
+
+        SignificantTerms.Bucket expectedSigTerm = (SignificantTerms.Bucket) expected;
+        SignificantTerms.Bucket actualSigTerm = (SignificantTerms.Bucket) actual;
+
+        assertEquals(expectedSigTerm.getSignificanceScore(), actualSigTerm.getSignificanceScore(), 0.0);
+        assertEquals(expectedSigTerm.getSubsetDf(), actualSigTerm.getSubsetDf());
+        assertEquals(expectedSigTerm.getDocCount(), actualSigTerm.getSubsetDf());
+        assertEquals(expectedSigTerm.getSupersetDf(), actualSigTerm.getSupersetDf());
+        assertEquals(expectedSigTerm.getSubsetSize(), actualSigTerm.getSubsetSize());
+        assertEquals(expectedSigTerm.getSupersetSize(), actualSigTerm.getSupersetSize());
+    }
+
+    private static SignificanceHeuristic randomSignificanceHeuristic() {
+        return randomFrom(
+                new JLHScore(),
+                new MutualInformation(randomBoolean(), randomBoolean()),
+                new GND(randomBoolean()),
+                new ChiSquare(randomBoolean(), randomBoolean()));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificanceHeuristicTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificanceHeuristicTests.java
index 93c2c1edd00c8..dcefbffb10196 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificanceHeuristicTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificanceHeuristicTests.java
@@ -19,9 +19,8 @@
 package org.elasticsearch.search.aggregations.bucket.significant;
 
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.Version;
-import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.InputStreamStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -35,7 +34,6 @@
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.search.SearchShardTarget;
@@ -49,7 +47,6 @@
 import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristic;
 import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristicParser;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.TestSearchContext;
 
@@ -68,6 +65,7 @@
 import static java.util.Collections.singletonList;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.significantTerms;
 import static org.elasticsearch.test.VersionUtils.randomVersion;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
@@ -78,7 +76,7 @@
 public class SignificanceHeuristicTests extends ESTestCase {
     static class SignificantTermsTestSearchContext extends TestSearchContext {
 
-        public SignificantTermsTestSearchContext() {
+        SignificantTermsTestSearchContext() {
             super(null);
         }
 
@@ -89,7 +87,7 @@ public int numberOfShards() {
 
         @Override
         public SearchShardTarget shardTarget() {
-            return new SearchShardTarget("no node, this is a unit test", new Index("no index, this is a unit test", "_na_"), 0);
+            return new SearchShardTarget("no node, this is a unit test", new Index("no index, this is a unit test", "_na_"), 0, null);
         }
     }
 
@@ -148,7 +146,8 @@ SignificanceHeuristic getRandomSignificanceheuristic() {
 
     public void testReduce() {
         List<InternalAggregation> aggs = createInternalAggregations();
-        SignificantTerms reducedAgg = (SignificantTerms) aggs.get(0).doReduce(aggs, null);
+        InternalAggregation.ReduceContext context = new InternalAggregation.ReduceContext(null, null, true);
+        SignificantTerms reducedAgg = (SignificantTerms) aggs.get(0).doReduce(aggs, context);
         assertThat(reducedAgg.getBuckets().size(), equalTo(2));
         assertThat(reducedAgg.getBuckets().get(0).getSubsetDf(), equalTo(8L));
         assertThat(reducedAgg.getBuckets().get(0).getSubsetSize(), equalTo(16L));
@@ -219,84 +218,82 @@ SignificantLongTerms.Bucket createBucket(long subsetDF, long subsetSize, long su
     public void testBuilderAndParser() throws Exception {
         SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList());
         ParseFieldRegistry<SignificanceHeuristicParser> heuristicParserMapper = searchModule.getSignificanceHeuristicParserRegistry();
-        SearchContext searchContext = new SignificantTermsTestSearchContext();
 
         // test jlh with string
-        assertTrue(parseFromString(heuristicParserMapper, searchContext, "\"jlh\":{}") instanceof JLHScore);
+        assertTrue(parseFromString(heuristicParserMapper, "\"jlh\":{}") instanceof JLHScore);
         // test gnd with string
-        assertTrue(parseFromString(heuristicParserMapper, searchContext, "\"gnd\":{}") instanceof GND);
+        assertTrue(parseFromString(heuristicParserMapper, "\"gnd\":{}") instanceof GND);
         // test mutual information with string
         boolean includeNegatives = randomBoolean();
         boolean backgroundIsSuperset = randomBoolean();
         String mutual = "\"mutual_information\":{\"include_negatives\": " + includeNegatives + ", \"background_is_superset\":"
                 + backgroundIsSuperset + "}";
         assertEquals(new MutualInformation(includeNegatives, backgroundIsSuperset),
-                parseFromString(heuristicParserMapper, searchContext, mutual));
+                parseFromString(heuristicParserMapper, mutual));
         String chiSquare = "\"chi_square\":{\"include_negatives\": " + includeNegatives + ", \"background_is_superset\":"
                 + backgroundIsSuperset + "}";
         assertEquals(new ChiSquare(includeNegatives, backgroundIsSuperset),
-                parseFromString(heuristicParserMapper, searchContext, chiSquare));
+                parseFromString(heuristicParserMapper, chiSquare));
 
         // test with builders
-        assertThat(parseFromBuilder(heuristicParserMapper, searchContext, new JLHScore()), instanceOf(JLHScore.class));
-        assertThat(parseFromBuilder(heuristicParserMapper, searchContext, new GND(backgroundIsSuperset)), instanceOf(GND.class));
+        assertThat(parseFromBuilder(heuristicParserMapper, new JLHScore()), instanceOf(JLHScore.class));
+        assertThat(parseFromBuilder(heuristicParserMapper, new GND(backgroundIsSuperset)), instanceOf(GND.class));
         assertEquals(new MutualInformation(includeNegatives, backgroundIsSuperset),
-                parseFromBuilder(heuristicParserMapper, searchContext, new MutualInformation(includeNegatives, backgroundIsSuperset)));
+                parseFromBuilder(heuristicParserMapper, new MutualInformation(includeNegatives, backgroundIsSuperset)));
         assertEquals(new ChiSquare(includeNegatives, backgroundIsSuperset),
-                parseFromBuilder(heuristicParserMapper, searchContext, new ChiSquare(includeNegatives, backgroundIsSuperset)));
+                parseFromBuilder(heuristicParserMapper, new ChiSquare(includeNegatives, backgroundIsSuperset)));
 
         // test exceptions
         String faultyHeuristicdefinition = "\"mutual_information\":{\"include_negatives\": false, \"some_unknown_field\": false}";
         String expectedError = "unknown field [some_unknown_field]";
-        checkParseException(heuristicParserMapper, searchContext, faultyHeuristicdefinition, expectedError);
+        checkParseException(heuristicParserMapper, faultyHeuristicdefinition, expectedError);
 
         faultyHeuristicdefinition = "\"chi_square\":{\"unknown_field\": true}";
         expectedError = "unknown field [unknown_field]";
-        checkParseException(heuristicParserMapper, searchContext, faultyHeuristicdefinition, expectedError);
+        checkParseException(heuristicParserMapper, faultyHeuristicdefinition, expectedError);
 
         faultyHeuristicdefinition = "\"jlh\":{\"unknown_field\": true}";
         expectedError = "expected an empty object, but found ";
-        checkParseException(heuristicParserMapper, searchContext, faultyHeuristicdefinition, expectedError);
+        checkParseException(heuristicParserMapper, faultyHeuristicdefinition, expectedError);
 
         faultyHeuristicdefinition = "\"gnd\":{\"unknown_field\": true}";
         expectedError = "unknown field [unknown_field]";
-        checkParseException(heuristicParserMapper, searchContext, faultyHeuristicdefinition, expectedError);
+        checkParseException(heuristicParserMapper, faultyHeuristicdefinition, expectedError);
     }
 
     protected void checkParseException(ParseFieldRegistry<SignificanceHeuristicParser> significanceHeuristicParserRegistry,
-            SearchContext searchContext, String faultyHeuristicDefinition, String expectedError) throws IOException {
+            String faultyHeuristicDefinition, String expectedError) throws IOException {
 
-        IndicesQueriesRegistry registry = new IndicesQueriesRegistry();
         try {
-            XContentParser stParser = JsonXContent.jsonXContent.createParser(
+            XContentParser stParser = createParser(JsonXContent.jsonXContent,
                     "{\"field\":\"text\", " + faultyHeuristicDefinition + ",\"min_doc_count\":200}");
-            QueryParseContext parseContext = new QueryParseContext(registry, stParser, ParseFieldMatcher.STRICT);
+            QueryParseContext parseContext = new QueryParseContext(stParser);
             stParser.nextToken();
-            new SignificantTermsParser(significanceHeuristicParserRegistry, registry).parse("testagg", parseContext);
+            SignificantTermsAggregationBuilder.getParser(significanceHeuristicParserRegistry).parse("testagg", parseContext);
             fail();
-        } catch (ElasticsearchParseException e) {
-            assertTrue(e.getMessage().contains(expectedError));
+        } catch (ParsingException e) {
+            assertThat(e.getCause().getMessage(), containsString(expectedError));
         }
     }
 
     protected SignificanceHeuristic parseFromBuilder(ParseFieldRegistry<SignificanceHeuristicParser> significanceHeuristicParserRegistry,
-            SearchContext searchContext, SignificanceHeuristic significanceHeuristic) throws IOException {
+            SignificanceHeuristic significanceHeuristic) throws IOException {
         SignificantTermsAggregationBuilder stBuilder = significantTerms("testagg");
         stBuilder.significanceHeuristic(significanceHeuristic).field("text").minDocCount(200);
         XContentBuilder stXContentBuilder = XContentFactory.jsonBuilder();
         stBuilder.internalXContent(stXContentBuilder, null);
-        XContentParser stParser = JsonXContent.jsonXContent.createParser(stXContentBuilder.string());
-        return parseSignificanceHeuristic(significanceHeuristicParserRegistry, searchContext, stParser);
+        XContentParser stParser = createParser(JsonXContent.jsonXContent, stXContentBuilder.string());
+        return parseSignificanceHeuristic(significanceHeuristicParserRegistry, stParser);
     }
 
     private SignificanceHeuristic parseSignificanceHeuristic(
-            ParseFieldRegistry<SignificanceHeuristicParser> significanceHeuristicParserRegistry, SearchContext searchContext,
+            ParseFieldRegistry<SignificanceHeuristicParser> significanceHeuristicParserRegistry,
             XContentParser stParser) throws IOException {
-        IndicesQueriesRegistry registry = new IndicesQueriesRegistry();
-        QueryParseContext parseContext = new QueryParseContext(registry, stParser, ParseFieldMatcher.STRICT);
+        QueryParseContext parseContext = new QueryParseContext(stParser);
         stParser.nextToken();
-        SignificantTermsAggregationBuilder aggregatorFactory = (SignificantTermsAggregationBuilder) new SignificantTermsParser(
-                significanceHeuristicParserRegistry, registry).parse("testagg", parseContext);
+        SignificantTermsAggregationBuilder aggregatorFactory =
+                (SignificantTermsAggregationBuilder) SignificantTermsAggregationBuilder.getParser(
+                significanceHeuristicParserRegistry).parse("testagg", parseContext);
         stParser.nextToken();
         assertThat(aggregatorFactory.getBucketCountThresholds().getMinDocCount(), equalTo(200L));
         assertThat(stParser.currentToken(), equalTo(null));
@@ -305,10 +302,10 @@ private SignificanceHeuristic parseSignificanceHeuristic(
     }
 
     protected SignificanceHeuristic parseFromString(ParseFieldRegistry<SignificanceHeuristicParser> significanceHeuristicParserRegistry,
-            SearchContext searchContext, String heuristicString) throws IOException {
-        XContentParser stParser = JsonXContent.jsonXContent.createParser(
+            String heuristicString) throws IOException {
+        XContentParser stParser = createParser(JsonXContent.jsonXContent,
                 "{\"field\":\"text\", " + heuristicString + ", \"min_doc_count\":200}");
-        return parseSignificanceHeuristic(significanceHeuristicParserRegistry, searchContext, stParser);
+        return parseSignificanceHeuristic(significanceHeuristicParserRegistry, stParser);
     }
 
     void testBackgroundAssertions(SignificanceHeuristic heuristicIsSuperset, SignificanceHeuristic heuristicNotSuperset) {
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTermsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTermsTests.java
new file mode 100644
index 0000000000000..ceb9431b8bfa2
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTermsTests.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.significant;
+
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation;
+import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristic;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class SignificantLongTermsTests extends InternalSignificantTermsTestCase {
+
+    private DocValueFormat format;
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        format = randomNumericDocValueFormat();
+    }
+
+    @Override
+    protected InternalSignificantTerms createTestInstance(String name,
+                                                          List<PipelineAggregator> pipelineAggregators,
+                                                          Map<String, Object> metaData,
+                                                          InternalAggregations aggs,
+                                                          int requiredSize, int numBuckets,
+                                                          long subsetSize, int[] subsetDfs,
+                                                          long supersetSize, int[] supersetDfs,
+                                                          SignificanceHeuristic significanceHeuristic) {
+
+        List<SignificantLongTerms.Bucket> buckets = new ArrayList<>(numBuckets);
+        Set<Long> terms = new HashSet<>();
+        for (int i = 0; i < numBuckets; ++i) {
+            long term = randomValueOtherThanMany(l -> terms.add(l) == false, random()::nextLong);
+            buckets.add(new SignificantLongTerms.Bucket(subsetDfs[i], subsetSize, supersetDfs[i], supersetSize, term, aggs, format));
+        }
+        return new SignificantLongTerms(name, requiredSize, 1L, pipelineAggregators, metaData, format, subsetSize,
+                supersetSize, significanceHeuristic, buckets);
+    }
+
+    @Override
+    protected Class<? extends ParsedMultiBucketAggregation> implementationClass() {
+        return ParsedSignificantLongTerms.class;
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTermsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTermsTests.java
new file mode 100644
index 0000000000000..605f7b5e6230c
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTermsTests.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.significant;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation;
+import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristic;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class SignificantStringTermsTests extends InternalSignificantTermsTestCase {
+
+    @Override
+    protected InternalSignificantTerms createTestInstance(String name,
+                                                          List<PipelineAggregator> pipelineAggregators,
+                                                          Map<String, Object> metaData,
+                                                          InternalAggregations aggs,
+                                                          int requiredSize, int numBuckets,
+                                                          long subsetSize, int[] subsetDfs,
+                                                          long supersetSize, int[] supersetDfs,
+                                                          SignificanceHeuristic significanceHeuristic) {
+        DocValueFormat format = DocValueFormat.RAW;
+        List<SignificantStringTerms.Bucket> buckets = new ArrayList<>(numBuckets);
+        Set<BytesRef> terms = new HashSet<>();
+        for (int i = 0; i < numBuckets; ++i) {
+            BytesRef term = randomValueOtherThanMany(b -> terms.add(b) == false, () -> new BytesRef(randomAlphaOfLength(10)));
+            buckets.add(new SignificantStringTerms.Bucket(term, subsetDfs[i], subsetSize, supersetDfs[i], supersetSize, aggs, format));
+        }
+        return new SignificantStringTerms(name, requiredSize, 1L, pipelineAggregators, metaData, format, subsetSize,
+                supersetSize, significanceHeuristic, buckets);
+    }
+
+    @Override
+    protected Class<? extends ParsedMultiBucketAggregation> implementationClass() {
+        return ParsedSignificantStringTerms.class;
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorTests.java
new file mode 100644
index 0000000000000..e2625039df5f9
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorTests.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.significant;
+
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.MultiReader;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.elasticsearch.index.mapper.KeywordFieldMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.AggregatorTestCase;
+import org.elasticsearch.search.aggregations.support.ValueType;
+import org.hamcrest.Matchers;
+import org.junit.Before;
+
+import java.io.IOException;
+
+public class SignificantTermsAggregatorTests extends AggregatorTestCase {
+
+    private MappedFieldType fieldType;
+
+    @Before
+    public void setUpTest() throws Exception {
+        super.setUp();
+        fieldType = new KeywordFieldMapper.KeywordFieldType();
+        fieldType.setHasDocValues(true);
+        fieldType.setIndexOptions(IndexOptions.DOCS);
+        fieldType.setName("field");
+    }
+
+    public void testParsedAsFilter() throws IOException {
+        IndexReader indexReader = new MultiReader();
+        IndexSearcher indexSearcher = newSearcher(indexReader);
+        QueryBuilder filter = QueryBuilders.boolQuery()
+                .must(QueryBuilders.termQuery("field", "foo"))
+                .should(QueryBuilders.termQuery("field", "bar"));
+        SignificantTermsAggregationBuilder builder = new SignificantTermsAggregationBuilder(
+                "test", ValueType.STRING)
+                .field("field")
+                .backgroundFilter(filter);
+        AggregatorFactory<?> factory = createAggregatorFactory(builder, indexSearcher, fieldType);
+        assertThat(factory, Matchers.instanceOf(SignificantTermsAggregatorFactory.class));
+        SignificantTermsAggregatorFactory sigTermsFactory =
+                (SignificantTermsAggregatorFactory) factory;
+        Query parsedQuery = sigTermsFactory.filter;
+        assertThat(parsedQuery, Matchers.instanceOf(BooleanQuery.class));
+        assertEquals(2, ((BooleanQuery) parsedQuery).clauses().size());
+        // means the bool query has been parsed as a filter, if it was a query minShouldMatch would
+        // be 0
+        assertEquals(1, ((BooleanQuery) parsedQuery).getMinimumNumberShouldMatch());
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTermsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTermsTests.java
new file mode 100644
index 0000000000000..f558f89b49331
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTermsTests.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.terms;
+
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class DoubleTermsTests extends InternalTermsTestCase {
+
+    @Override
+    protected InternalTerms<?, ?> createTestInstance(String name,
+                                                     List<PipelineAggregator> pipelineAggregators,
+                                                     Map<String, Object> metaData,
+                                                     InternalAggregations aggregations,
+                                                     boolean showTermDocCountError,
+                                                     long docCountError) {
+        Terms.Order order = Terms.Order.count(false);
+        long minDocCount = 1;
+        int requiredSize = 3;
+        int shardSize = requiredSize + 2;
+        DocValueFormat format = randomNumericDocValueFormat();
+        long otherDocCount = 0;
+        List<DoubleTerms.Bucket> buckets = new ArrayList<>();
+        final int numBuckets = randomNumberOfBuckets();
+        Set<Double> terms = new HashSet<>();
+        for (int i = 0; i < numBuckets; ++i) {
+            double term = randomValueOtherThanMany(d -> terms.add(d) == false, random()::nextDouble);
+            int docCount = randomIntBetween(1, 100);
+            buckets.add(new DoubleTerms.Bucket(term, docCount, aggregations, showTermDocCountError, docCountError, format));
+        }
+        return new DoubleTerms(name, order, requiredSize, minDocCount, pipelineAggregators,
+                metaData, format, shardSize, showTermDocCountError, otherDocCount, buckets, docCountError);
+    }
+
+    @Override
+    protected Class<? extends ParsedMultiBucketAggregation> implementationClass() {
+        return ParsedDoubleTerms.class;
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTermsTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTermsTestCase.java
new file mode 100644
index 0000000000000..37f401fcb5de0
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTermsTestCase.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.terms;
+
+import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.aggregations.InternalMultiBucketAggregationTestCase;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.junit.Before;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+public abstract class InternalTermsTestCase extends InternalMultiBucketAggregationTestCase<InternalTerms<?, ?>> {
+
+    private boolean showDocCount;
+    private long docCountError;
+
+    @Before
+    public void init() {
+        showDocCount = randomBoolean();
+        docCountError = showDocCount ? randomInt(1000) : -1;
+    }
+
+    @Override
+    protected InternalTerms<?, ?> createTestInstance(String name,
+                                                     List<PipelineAggregator> pipelineAggregators,
+                                                     Map<String, Object> metaData,
+                                                     InternalAggregations aggregations) {
+        return createTestInstance(name, pipelineAggregators, metaData, aggregations, showDocCount, docCountError);
+    }
+
+    protected abstract InternalTerms<?, ?> createTestInstance(String name,
+                                                              List<PipelineAggregator> pipelineAggregators,
+                                                              Map<String, Object> metaData,
+                                                              InternalAggregations aggregations,
+                                                              boolean showTermDocCountError,
+                                                              long docCountError);
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/LongTermsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/LongTermsTests.java
new file mode 100644
index 0000000000000..e8429691953a8
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/LongTermsTests.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.terms;
+
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class LongTermsTests extends InternalTermsTestCase {
+
+    @Override
+    protected InternalTerms<?, ?> createTestInstance(String name,
+                                                     List<PipelineAggregator> pipelineAggregators,
+                                                     Map<String, Object> metaData,
+                                                     InternalAggregations aggregations,
+                                                     boolean showTermDocCountError,
+                                                     long docCountError) {
+        Terms.Order order = Terms.Order.count(false);
+        long minDocCount = 1;
+        int requiredSize = 3;
+        int shardSize = requiredSize + 2;
+        DocValueFormat format = randomNumericDocValueFormat();
+        long otherDocCount = 0;
+        List<LongTerms.Bucket> buckets = new ArrayList<>();
+        final int numBuckets = randomNumberOfBuckets();
+        Set<Long> terms = new HashSet<>();
+        for (int i = 0; i < numBuckets; ++i) {
+            long term = randomValueOtherThanMany(l -> terms.add(l) == false, random()::nextLong);
+            int docCount = randomIntBetween(1, 100);
+            buckets.add(new LongTerms.Bucket(term, docCount, aggregations, showTermDocCountError, docCountError, format));
+        }
+        return new LongTerms(name, order, requiredSize, minDocCount, pipelineAggregators,
+                metaData, format, shardSize, showTermDocCountError, otherDocCount, buckets, docCountError);
+    }
+
+    @Override
+    protected Class<? extends ParsedMultiBucketAggregation> implementationClass() {
+        return ParsedLongTerms.class;
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsTests.java
new file mode 100644
index 0000000000000..7e7bb6f0692ba
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsTests.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.terms;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class StringTermsTests extends InternalTermsTestCase {
+
+    @Override
+    protected InternalTerms<?, ?> createTestInstance(String name,
+                                                     List<PipelineAggregator> pipelineAggregators,
+                                                     Map<String, Object> metaData,
+                                                     InternalAggregations aggregations,
+                                                     boolean showTermDocCountError,
+                                                     long docCountError) {
+        Terms.Order order = Terms.Order.count(false);
+        long minDocCount = 1;
+        int requiredSize = 3;
+        int shardSize = requiredSize + 2;
+        DocValueFormat format = DocValueFormat.RAW;
+        long otherDocCount = 0;
+        List<StringTerms.Bucket> buckets = new ArrayList<>();
+        final int numBuckets = randomNumberOfBuckets();
+        Set<BytesRef> terms = new HashSet<>();
+        for (int i = 0; i < numBuckets; ++i) {
+            BytesRef term = randomValueOtherThanMany(b -> terms.add(b) == false, () -> new BytesRef(randomAlphaOfLength(10)));
+            int docCount = randomIntBetween(1, 100);
+            buckets.add(new StringTerms.Bucket(term, docCount, aggregations, showTermDocCountError, docCountError, format));
+        }
+        return new StringTerms(name, order, requiredSize, minDocCount, pipelineAggregators,
+                metaData, format, shardSize, showTermDocCountError, otherDocCount, buckets, docCountError);
+    }
+
+    @Override
+    protected Class<? extends ParsedMultiBucketAggregation> implementationClass() {
+        return ParsedStringTerms.class;
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactoryTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactoryTests.java
index 7d7ffac4fb324..fe32bff86b877 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactoryTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactoryTests.java
@@ -21,10 +21,11 @@
 
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.elasticsearch.test.ESTestCase;
 
 import static org.hamcrest.Matchers.equalTo;
 
-public class TermsAggregatorFactoryTests extends ESSingleNodeTestCase {
+public class TermsAggregatorFactoryTests extends ESTestCase {
     public void testSubAggCollectMode() throws Exception {
         assertThat(TermsAggregatorFactory.subAggCollectionMode(Integer.MAX_VALUE, -1),
             equalTo(Aggregator.SubAggCollectionMode.DEPTH_FIRST));
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java
new file mode 100644
index 0000000000000..a6999118e139c
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.bucket.terms;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.SortedSetDocValuesField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.index.mapper.KeywordFieldMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.NumberFieldMapper;
+import org.elasticsearch.search.aggregations.AggregationBuilders;
+import org.elasticsearch.search.aggregations.Aggregator;
+import org.elasticsearch.search.aggregations.AggregatorTestCase;
+import org.elasticsearch.search.aggregations.support.ValueType;
+
+
+import static org.hamcrest.Matchers.instanceOf;
+
+public class TermsAggregatorTests extends AggregatorTestCase {
+    public void testGlobalOrdinalsExecutionHint() throws Exception {
+        Directory directory = newDirectory();
+        RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
+        indexWriter.close();
+        IndexReader indexReader = DirectoryReader.open(directory);
+        // We do not use LuceneTestCase.newSearcher because we need a DirectoryReader
+        IndexSearcher indexSearcher = new IndexSearcher(indexReader);
+
+        TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.STRING)
+            .field("string")
+            .collectMode(Aggregator.SubAggCollectionMode.BREADTH_FIRST);
+        MappedFieldType fieldType = new KeywordFieldMapper.KeywordFieldType();
+        fieldType.setName("string");
+        fieldType.setHasDocValues(true);
+
+        MappedFieldType fieldTypeNumber = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.INTEGER);
+        fieldTypeNumber.setName("number");
+        fieldTypeNumber.setHasDocValues(true);
+
+        TermsAggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType, fieldTypeNumber);
+        assertThat(aggregator, instanceOf(GlobalOrdinalsStringTermsAggregator.class));
+        GlobalOrdinalsStringTermsAggregator globalAgg = (GlobalOrdinalsStringTermsAggregator) aggregator;
+        assertFalse(globalAgg.remapGlobalOrds());
+        aggregator.close();
+
+        aggregationBuilder
+            .subAggregation(AggregationBuilders.max("max").field("number"));
+        aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType);
+        assertThat(aggregator, instanceOf(GlobalOrdinalsStringTermsAggregator.class));
+        globalAgg = (GlobalOrdinalsStringTermsAggregator) aggregator;
+        assertFalse(globalAgg.remapGlobalOrds());
+        aggregator.close();
+
+        aggregationBuilder
+            .order(Terms.Order.aggregation("max", true));
+        aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType);
+        assertThat(aggregator, instanceOf(GlobalOrdinalsStringTermsAggregator.class));
+        globalAgg = (GlobalOrdinalsStringTermsAggregator) aggregator;
+        assertTrue(globalAgg.remapGlobalOrds());
+        aggregator.close();
+
+        aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.STRING)
+            .field("string")
+            .executionHint("global_ordinals_hash");
+        aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType);
+        assertThat(aggregator, instanceOf(GlobalOrdinalsStringTermsAggregator.class));
+        globalAgg = (GlobalOrdinalsStringTermsAggregator) aggregator;
+        assertTrue(globalAgg.remapGlobalOrds());
+        aggregator.close();
+
+        indexReader.close();
+        directory.close();
+    }
+
+    public void testTermsAggregator() throws Exception {
+        Directory directory = newDirectory();
+        RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
+        Document document = new Document();
+        document.add(new SortedSetDocValuesField("string", new BytesRef("a")));
+        document.add(new SortedSetDocValuesField("string", new BytesRef("b")));
+        indexWriter.addDocument(document);
+        document = new Document();
+        document.add(new SortedSetDocValuesField("string", new BytesRef("c")));
+        document.add(new SortedSetDocValuesField("string", new BytesRef("a")));
+        indexWriter.addDocument(document);
+        document = new Document();
+        document.add(new SortedSetDocValuesField("string", new BytesRef("b")));
+        document.add(new SortedSetDocValuesField("string", new BytesRef("d")));
+        indexWriter.addDocument(document);
+        indexWriter.close();
+
+        IndexReader indexReader = DirectoryReader.open(directory);
+        // We do not use LuceneTestCase.newSearcher because we need a DirectoryReader
+        IndexSearcher indexSearcher = new IndexSearcher(indexReader);
+
+        for (TermsAggregatorFactory.ExecutionMode executionMode : TermsAggregatorFactory.ExecutionMode.values()) {
+            TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.STRING)
+                .executionHint(executionMode.toString())
+                .field("string")
+                .order(Terms.Order.term(true));
+            MappedFieldType fieldType = new KeywordFieldMapper.KeywordFieldType();
+            fieldType.setName("string");
+            fieldType.setHasDocValues(true );
+            try (TermsAggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType)) {
+                aggregator.preCollection();
+                indexSearcher.search(new MatchAllDocsQuery(), aggregator);
+                aggregator.postCollection();
+                Terms result = (Terms) aggregator.buildAggregation(0L);
+                assertEquals(4, result.getBuckets().size());
+                assertEquals("a", result.getBuckets().get(0).getKeyAsString());
+                assertEquals(2L, result.getBuckets().get(0).getDocCount());
+                assertEquals("b", result.getBuckets().get(1).getKeyAsString());
+                assertEquals(2L, result.getBuckets().get(1).getDocCount());
+                assertEquals("c", result.getBuckets().get(2).getKeyAsString());
+                assertEquals(1L, result.getBuckets().get(2).getDocCount());
+                assertEquals("d", result.getBuckets().get(3).getKeyAsString());
+                assertEquals(1L, result.getBuckets().get(3).getDocCount());
+            }
+        }
+        indexReader.close();
+        directory.close();
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java
index 5cc6ec586307d..91d27946175a0 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java
@@ -74,7 +74,7 @@ public abstract class AbstractGeoTestCase extends ESIntegTestCase {
     public void setupSuiteScopeCluster() throws Exception {
         createIndex(UNMAPPED_IDX_NAME);
         assertAcked(prepareCreate(IDX_NAME)
-                .addMapping("type", SINGLE_VALUED_FIELD_NAME, "type=geo_point,geohash_prefix=true,geohash_precision=12",
+                .addMapping("type", SINGLE_VALUED_FIELD_NAME, "type=geo_point",
                         MULTI_VALUED_FIELD_NAME, "type=geo_point", NUMBER_FIELD_NAME, "type=long", "tag", "type=keyword"));
 
         singleTopLeft = new GeoPoint(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY);
@@ -185,9 +185,8 @@ public void setupSuiteScopeCluster() throws Exception {
                 .order(SortOrder.ASC)).setSize(5000).get();
         assertSearchResponse(response);
         long totalHits = response.getHits().totalHits();
-        XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
+        XContentBuilder builder = XContentFactory.jsonBuilder();
         response.toXContent(builder, ToXContent.EMPTY_PARAMS);
-        builder.endObject();
         logger.info("Full high_card_idx Response Content:\n{ {} }", builder.string());
         for (int i = 0; i < totalHits; i++) {
             SearchHit searchHit = response.getHits().getAt(i);
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractNumericMetricTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractNumericMetricTestCase.java
index f1ccf344a7c44..975a1610a61d9 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractNumericMetricTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractNumericMetricTestCase.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.search.aggregations.metrics;
 
-import org.elasticsearch.script.Script;
 import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
@@ -31,19 +30,7 @@ public abstract class AbstractNumericMetricTestCase<AF extends ValuesSourceAggre
     protected final AF createTestAggregatorBuilder() {
         AF factory = doCreateTestAggregatorFactory();
         String field = randomNumericField();
-        int randomFieldBranch = randomInt(3);
-        switch (randomFieldBranch) {
-        case 0:
-            factory.field(field);
-            break;
-        case 1:
-            factory.field(field);
-            factory.script(new Script("_value + 1"));
-            break;
-        case 2:
-            factory.script(new Script("doc[" + field + "] + 1"));
-            break;
-        }
+        randomFieldOrScript(factory, field);
         if (randomBoolean()) {
             factory.missing("MISSING");
         }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AdjacencyMatrixTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AdjacencyMatrixTests.java
new file mode 100644
index 0000000000000..a12dbad59fbfb
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AdjacencyMatrixTests.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics;
+
+import org.elasticsearch.index.query.MatchNoneQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
+import org.elasticsearch.search.aggregations.bucket.adjacency.AdjacencyMatrixAggregationBuilder;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class AdjacencyMatrixTests extends BaseAggregationTestCase<AdjacencyMatrixAggregationBuilder> {
+
+    @Override
+    protected AdjacencyMatrixAggregationBuilder createTestAggregatorBuilder() {
+
+        int size = randomIntBetween(1, 20);
+        AdjacencyMatrixAggregationBuilder factory;
+        Map<String, QueryBuilder> filters = new HashMap<>(size);
+        for (String key : randomUnique(() -> randomAlphaOfLengthBetween(1, 20), size)) {
+            filters.put(key, QueryBuilders.termQuery(randomAlphaOfLengthBetween(5, 20), randomAlphaOfLengthBetween(5, 20)));
+        }
+        factory = new AdjacencyMatrixAggregationBuilder(randomAlphaOfLengthBetween(1, 20), filters)
+                .separator(randomFrom("&","+","\t"));       
+        return factory;
+    }
+
+    /**
+     * Test that when passing in keyed filters as a map they are equivalent
+     */
+    public void testFiltersSameMap() {
+        Map<String, QueryBuilder> original = new HashMap<>();
+        original.put("bbb", new MatchNoneQueryBuilder());
+        original.put("aaa", new MatchNoneQueryBuilder());
+        AdjacencyMatrixAggregationBuilder builder;
+        builder = new AdjacencyMatrixAggregationBuilder("my-agg", original);
+        assertEquals(original, builder.filters());
+        assert original != builder.filters();
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java
index ccf03df37fcd4..1bf61c828058f 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java
@@ -29,8 +29,9 @@
 import org.elasticsearch.script.LeafSearchScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptEngineService;
-import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.script.SearchScript;
+import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.bucket.filter.Filter;
 import org.elasticsearch.search.aggregations.bucket.global.Global;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
@@ -56,7 +57,9 @@
 import static org.elasticsearch.search.aggregations.AggregationBuilders.global;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.terms;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
@@ -143,9 +146,9 @@ public void testSingleValuedFieldGetProperty() throws Exception {
         assertThat(avg.getName(), equalTo("avg"));
         double expectedAvgValue = (double) (1+2+3+4+5+6+7+8+9+10) / 10;
         assertThat(avg.getValue(), equalTo(expectedAvgValue));
-        assertThat((Avg) global.getProperty("avg"), equalTo(avg));
-        assertThat((double) global.getProperty("avg.value"), equalTo(expectedAvgValue));
-        assertThat((double) avg.getProperty("value"), equalTo(expectedAvgValue));
+        assertThat((Avg) ((InternalAggregation)global).getProperty("avg"), equalTo(avg));
+        assertThat((double) ((InternalAggregation)global).getProperty("avg.value"), equalTo(expectedAvgValue));
+        assertThat((double) ((InternalAggregation)avg).getProperty("value"), equalTo(expectedAvgValue));
     }
 
     @Override
@@ -168,7 +171,7 @@ public void testSingleValuedFieldWithValueScript() throws Exception {
         SearchResponse searchResponse = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
                 .addAggregation(avg("avg").field("value")
-                        .script(new Script("", ScriptType.INLINE, FieldValueScriptEngine.NAME, null)))
+                        .script(new Script(ScriptType.INLINE, FieldValueScriptEngine.NAME, "", Collections.emptyMap())))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
@@ -185,7 +188,7 @@ public void testSingleValuedFieldWithValueScriptWithParams() throws Exception {
         SearchResponse searchResponse = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
                 .addAggregation(avg("avg").field("value")
-                        .script(new Script("", ScriptType.INLINE, FieldValueScriptEngine.NAME, params)))
+                        .script(new Script(ScriptType.INLINE, FieldValueScriptEngine.NAME, "", params)))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
@@ -229,7 +232,7 @@ public void testMultiValuedFieldWithValueScript() throws Exception {
         SearchResponse searchResponse = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
                 .addAggregation(avg("avg").field("values")
-                        .script(new Script("", ScriptType.INLINE, FieldValueScriptEngine.NAME, null)))
+                        .script(new Script(ScriptType.INLINE, FieldValueScriptEngine.NAME, "", Collections.emptyMap())))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
@@ -246,7 +249,7 @@ public void testMultiValuedFieldWithValueScriptWithParams() throws Exception {
         SearchResponse searchResponse = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
                 .addAggregation(avg("avg").field("values")
-                        .script(new Script("", ScriptType.INLINE, FieldValueScriptEngine.NAME, params)))
+                        .script(new Script(ScriptType.INLINE, FieldValueScriptEngine.NAME, "", params)))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
@@ -262,7 +265,7 @@ public void testScriptSingleValued() throws Exception {
         SearchResponse searchResponse = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
                 .addAggregation(avg("avg")
-                        .script(new Script("value", ScriptType.INLINE, ExtractFieldScriptEngine.NAME, null)))
+                        .script(new Script(ScriptType.INLINE, ExtractFieldScriptEngine.NAME, "value", Collections.emptyMap())))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
@@ -279,7 +282,7 @@ public void testScriptSingleValuedWithParams() throws Exception {
         SearchResponse searchResponse = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
                 .addAggregation(avg("avg")
-                        .script(new Script("value", ScriptType.INLINE, ExtractFieldScriptEngine.NAME, params)))
+                        .script(new Script(ScriptType.INLINE, ExtractFieldScriptEngine.NAME, "value", params)))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
@@ -295,7 +298,7 @@ public void testScriptMultiValued() throws Exception {
         SearchResponse searchResponse = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
                 .addAggregation(avg("avg")
-                        .script(new Script("values", ScriptType.INLINE, ExtractFieldScriptEngine.NAME, null)))
+                        .script(new Script(ScriptType.INLINE, ExtractFieldScriptEngine.NAME, "values", Collections.emptyMap())))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
@@ -312,7 +315,7 @@ public void testScriptMultiValuedWithParams() throws Exception {
         SearchResponse searchResponse = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
                 .addAggregation(avg("avg")
-                        .script(new Script("values", ScriptType.INLINE, ExtractFieldScriptEngine.NAME, params)))
+                        .script(new Script(ScriptType.INLINE, ExtractFieldScriptEngine.NAME, "values", params)))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
@@ -334,7 +337,7 @@ public void testOrderByEmptyAggregation() throws Exception {
 
         Terms terms = searchResponse.getAggregations().get("terms");
         assertThat(terms, notNullValue());
-        List<Terms.Bucket> buckets = terms.getBuckets();
+        List<? extends Terms.Bucket> buckets = terms.getBuckets();
         assertThat(buckets, notNullValue());
         assertThat(buckets.size(), equalTo(10));
 
@@ -353,6 +356,45 @@ public void testOrderByEmptyAggregation() throws Exception {
         }
     }
 
+    /**
+     * Make sure that a request using a script does not get cached and a request
+     * not using a script does get cached.
+     */
+    public void testDontCacheScripts() throws Exception {
+        assertAcked(prepareCreate("cache_test_idx").addMapping("type", "d", "type=long")
+                .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1))
+                .get());
+        indexRandom(true, client().prepareIndex("cache_test_idx", "type", "1").setSource("s", 1),
+                client().prepareIndex("cache_test_idx", "type", "2").setSource("s", 2));
+
+        // Make sure we are starting with a clear cache
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(0L));
+
+        // Test that a request using a script does not get cached
+        SearchResponse r = client().prepareSearch("cache_test_idx").setSize(0)
+                .addAggregation(avg("foo").field("d").script(
+                    new Script(ScriptType.INLINE, FieldValueScriptEngine.NAME, "", Collections.emptyMap()))).get();
+        assertSearchResponse(r);
+
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(0L));
+
+        // To make sure that the cache is working test that a request not using
+        // a script is cached
+        r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(avg("foo").field("d")).get();
+        assertSearchResponse(r);
+
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(1L));
+    }
+
     /**
      * Mock plugin for the {@link ExtractFieldScriptEngine}
      */
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/CardinalityIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/CardinalityIT.java
index 594eba7ddb5f9..b8b33b97e4d00 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/CardinalityIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/CardinalityIT.java
@@ -26,7 +26,9 @@
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.MockScriptPlugin;
 import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode;
+import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.bucket.global.Global;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 import org.elasticsearch.search.aggregations.metrics.cardinality.Cardinality;
@@ -41,10 +43,10 @@
 import static java.util.Collections.emptyMap;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
-import static org.elasticsearch.script.ScriptService.ScriptType;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.cardinality;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.global;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.terms;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
@@ -134,11 +136,11 @@ public void setupSuiteScopeCluster() throws Exception {
             builders[i] = client().prepareIndex("idx", "type").setSource(jsonBuilder()
                     .startObject()
                         .field("str_value", "s" + i)
-                        .field("str_values", new String[]{"s" + (i * 2), "s" + (i * 2 + 1)})
+                        .array("str_values", new String[]{"s" + (i * 2), "s" + (i * 2 + 1)})
                         .field("l_value", i)
-                        .field("l_values", new int[] {i * 2, i * 2 + 1})
+                        .array("l_values", new int[] {i * 2, i * 2 + 1})
                         .field("d_value", i)
-                        .field("d_values", new double[]{i * 2, i * 2 + 1})
+                        .array("d_values", new double[]{i * 2, i * 2 + 1})
                     .endObject());
         }
         indexRandom(true, builders);
@@ -243,9 +245,9 @@ public void testSingleValuedNumericGetProperty() throws Exception {
         assertThat(cardinality.getName(), equalTo("cardinality"));
         long expectedValue = numDocs;
         assertCount(cardinality, expectedValue);
-        assertThat((Cardinality) global.getProperty("cardinality"), equalTo(cardinality));
-        assertThat((double) global.getProperty("cardinality.value"), equalTo((double) cardinality.getValue()));
-        assertThat((double) cardinality.getProperty("value"), equalTo((double) cardinality.getValue()));
+        assertThat(((InternalAggregation)global).getProperty("cardinality"), equalTo(cardinality));
+        assertThat(((InternalAggregation)global).getProperty("cardinality.value"), equalTo((double) cardinality.getValue()));
+        assertThat((double) ((InternalAggregation)cardinality).getProperty("value"), equalTo((double) cardinality.getValue()));
     }
 
     public void testSingleValuedNumericHashed() throws Exception {
@@ -305,7 +307,7 @@ public void testSingleValuedStringScript() throws Exception {
                 .addAggregation(
                         cardinality("cardinality")
                                 .precisionThreshold(precisionThreshold)
-                                .script(new Script("doc['str_value'].value", ScriptType.INLINE, CustomScriptPlugin.NAME, emptyMap())))
+                                .script(new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['str_value'].value", emptyMap())))
                 .execute().actionGet();
 
         assertSearchResponse(response);
@@ -321,7 +323,7 @@ public void testMultiValuedStringScript() throws Exception {
                 .addAggregation(
                         cardinality("cardinality")
                                 .precisionThreshold(precisionThreshold)
-                                .script(new Script("doc['str_values'].values", ScriptType.INLINE, CustomScriptPlugin.NAME, emptyMap())))
+                                .script(new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['str_values'].values", emptyMap())))
                 .execute().actionGet();
 
         assertSearchResponse(response);
@@ -333,7 +335,7 @@ public void testMultiValuedStringScript() throws Exception {
     }
 
     public void testSingleValuedNumericScript() throws Exception {
-        Script script = new Script("doc[' + singleNumericField() + '].value", ScriptType.INLINE, CustomScriptPlugin.NAME, emptyMap());
+        Script script = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc[' + singleNumericField() + '].value", emptyMap());
         SearchResponse response = client().prepareSearch("idx").setTypes("type")
                 .addAggregation(cardinality("cardinality").precisionThreshold(precisionThreshold).script(script))
                 .execute().actionGet();
@@ -347,7 +349,8 @@ public void testSingleValuedNumericScript() throws Exception {
     }
 
     public void testMultiValuedNumericScript() throws Exception {
-        Script script = new Script("doc[' + multiNumericField(false) + '].values", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
+        Script script =
+            new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc[' + multiNumericField(false) + '].values", Collections.emptyMap());
         SearchResponse response = client().prepareSearch("idx").setTypes("type")
                 .addAggregation(cardinality("cardinality").precisionThreshold(precisionThreshold).script(script))
                 .execute().actionGet();
@@ -366,7 +369,7 @@ public void testSingleValuedStringValueScript() throws Exception {
                         cardinality("cardinality")
                                 .precisionThreshold(precisionThreshold)
                                 .field("str_value")
-                                .script(new Script("_value", ScriptType.INLINE, CustomScriptPlugin.NAME, emptyMap())))
+                                .script(new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_value", emptyMap())))
                 .execute().actionGet();
 
         assertSearchResponse(response);
@@ -383,7 +386,7 @@ public void testMultiValuedStringValueScript() throws Exception {
                         cardinality("cardinality")
                                 .precisionThreshold(precisionThreshold)
                                 .field("str_values")
-                                .script(new Script("_value", ScriptType.INLINE, CustomScriptPlugin.NAME, emptyMap())))
+                                .script(new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_value", emptyMap())))
                 .execute().actionGet();
 
         assertSearchResponse(response);
@@ -400,7 +403,7 @@ public void testSingleValuedNumericValueScript() throws Exception {
                         cardinality("cardinality")
                                 .precisionThreshold(precisionThreshold)
                                 .field(singleNumericField())
-                                .script(new Script("_value", ScriptType.INLINE, CustomScriptPlugin.NAME, emptyMap())))
+                                .script(new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_value", emptyMap())))
                 .execute().actionGet();
 
         assertSearchResponse(response);
@@ -417,7 +420,7 @@ public void testMultiValuedNumericValueScript() throws Exception {
                         cardinality("cardinality")
                                 .precisionThreshold(precisionThreshold)
                                 .field(multiNumericField(false))
-                                .script(new Script("_value", ScriptType.INLINE, CustomScriptPlugin.NAME, emptyMap())))
+                                .script(new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_value", emptyMap())))
                 .execute().actionGet();
 
         assertSearchResponse(response);
@@ -445,4 +448,44 @@ public void testAsSubAgg() throws Exception {
             assertCount(count, 2);
         }
     }
+
+    /**
+     * Make sure that a request using a script does not get cached and a request
+     * not using a script does get cached.
+     */
+    public void testDontCacheScripts() throws Exception {
+        assertAcked(prepareCreate("cache_test_idx").addMapping("type", "d", "type=long")
+                .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1))
+                .get());
+        indexRandom(true, client().prepareIndex("cache_test_idx", "type", "1").setSource("s", 1),
+                client().prepareIndex("cache_test_idx", "type", "2").setSource("s", 2));
+
+        // Make sure we are starting with a clear cache
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(0L));
+
+        // Test that a request using a script does not get cached
+        SearchResponse r = client().prepareSearch("cache_test_idx").setSize(0)
+                .addAggregation(
+                        cardinality("foo").field("d").script(new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_value", emptyMap())))
+                .get();
+        assertSearchResponse(r);
+
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(0L));
+
+        // To make sure that the cache is working test that a request not using
+        // a script is cached
+        r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(cardinality("foo").field("d")).get();
+        assertSearchResponse(r);
+
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(1L));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsIT.java
index 6f4f891ea65a0..3903dd8b0bc76 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsIT.java
@@ -19,10 +19,12 @@
 package org.elasticsearch.search.aggregations.metrics;
 
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.aggregations.AggregationTestScriptsPlugin;
+import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.bucket.filter.Filter;
 import org.elasticsearch.search.aggregations.bucket.global.Global;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
@@ -46,7 +48,9 @@
 import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.missing;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.terms;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
@@ -210,7 +214,6 @@ public void testSingleValuedField_WithFormatter() throws Exception {
         assertThat(stats.getSum(), equalTo((double) 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10));
         assertThat(stats.getSumAsString(), equalTo("0055.0"));
         assertThat(stats.getCount(), equalTo(10L));
-        assertThat(stats.getCountAsString(), equalTo("0010.0"));
         assertThat(stats.getSumOfSquares(), equalTo((double) 1 + 4 + 9 + 16 + 25 + 36 + 49 + 64 + 81 + 100));
         assertThat(stats.getSumOfSquaresAsString(), equalTo("0385.0"));
         assertThat(stats.getVariance(), equalTo(variance(1, 2, 3, 4, 5, 6, 7, 8, 9, 10)));
@@ -237,33 +240,33 @@ public void testSingleValuedFieldGetProperty() throws Exception {
         ExtendedStats stats = global.getAggregations().get("stats");
         assertThat(stats, notNullValue());
         assertThat(stats.getName(), equalTo("stats"));
-        ExtendedStats statsFromProperty = (ExtendedStats) global.getProperty("stats");
+        ExtendedStats statsFromProperty = (ExtendedStats) ((InternalAggregation)global).getProperty("stats");
         assertThat(statsFromProperty, notNullValue());
         assertThat(statsFromProperty, sameInstance(stats));
         double expectedAvgValue = (double) (1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10) / 10;
         assertThat(stats.getAvg(), equalTo(expectedAvgValue));
-        assertThat((double) global.getProperty("stats.avg"), equalTo(expectedAvgValue));
+        assertThat((double) ((InternalAggregation)global).getProperty("stats.avg"), equalTo(expectedAvgValue));
         double expectedMinValue = 1.0;
         assertThat(stats.getMin(), equalTo(expectedMinValue));
-        assertThat((double) global.getProperty("stats.min"), equalTo(expectedMinValue));
+        assertThat((double) ((InternalAggregation)global).getProperty("stats.min"), equalTo(expectedMinValue));
         double expectedMaxValue = 10.0;
         assertThat(stats.getMax(), equalTo(expectedMaxValue));
-        assertThat((double) global.getProperty("stats.max"), equalTo(expectedMaxValue));
+        assertThat((double) ((InternalAggregation)global).getProperty("stats.max"), equalTo(expectedMaxValue));
         double expectedSumValue = 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10;
         assertThat(stats.getSum(), equalTo(expectedSumValue));
-        assertThat((double) global.getProperty("stats.sum"), equalTo(expectedSumValue));
+        assertThat((double) ((InternalAggregation)global).getProperty("stats.sum"), equalTo(expectedSumValue));
         long expectedCountValue = 10;
         assertThat(stats.getCount(), equalTo(expectedCountValue));
-        assertThat((double) global.getProperty("stats.count"), equalTo((double) expectedCountValue));
+        assertThat((double) ((InternalAggregation)global).getProperty("stats.count"), equalTo((double) expectedCountValue));
         double expectedSumOfSquaresValue = (double) 1 + 4 + 9 + 16 + 25 + 36 + 49 + 64 + 81 + 100;
         assertThat(stats.getSumOfSquares(), equalTo(expectedSumOfSquaresValue));
-        assertThat((double) global.getProperty("stats.sum_of_squares"), equalTo(expectedSumOfSquaresValue));
+        assertThat((double) ((InternalAggregation)global).getProperty("stats.sum_of_squares"), equalTo(expectedSumOfSquaresValue));
         double expectedVarianceValue = variance(1, 2, 3, 4, 5, 6, 7, 8, 9, 10);
         assertThat(stats.getVariance(), equalTo(expectedVarianceValue));
-        assertThat((double) global.getProperty("stats.variance"), equalTo(expectedVarianceValue));
+        assertThat((double) ((InternalAggregation)global).getProperty("stats.variance"), equalTo(expectedVarianceValue));
         double expectedStdDevValue = stdDev(1, 2, 3, 4, 5, 6, 7, 8, 9, 10);
         assertThat(stats.getStdDeviation(), equalTo(expectedStdDevValue));
-        assertThat((double) global.getProperty("stats.std_deviation"), equalTo(expectedStdDevValue));
+        assertThat((double) ((InternalAggregation)global).getProperty("stats.std_deviation"), equalTo(expectedStdDevValue));
     }
 
     @Override
@@ -298,7 +301,8 @@ public void testSingleValuedFieldWithValueScript() throws Exception {
                 .addAggregation(
                         extendedStats("stats")
                                 .field("value")
-                                .script(new Script("_value + 1", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, null))
+                                .script(new Script(ScriptType.INLINE,
+                                    AggregationTestScriptsPlugin.NAME, "_value + 1", Collections.emptyMap()))
                                 .sigma(sigma))
                 .execute().actionGet();
 
@@ -328,7 +332,7 @@ public void testSingleValuedFieldWithValueScriptWithParams() throws Exception {
                 .addAggregation(
                         extendedStats("stats")
                                 .field("value")
-                                .script(new Script("_value + inc", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params))
+                                .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value + inc", params))
                                 .sigma(sigma))
                 .execute().actionGet();
 
@@ -380,7 +384,8 @@ public void testMultiValuedFieldWithValueScript() throws Exception {
                 .addAggregation(
                         extendedStats("stats")
                                 .field("values")
-                                .script(new Script("_value - 1", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, null))
+                                .script(new Script(ScriptType.INLINE,
+                                    AggregationTestScriptsPlugin.NAME, "_value - 1", Collections.emptyMap()))
                                 .sigma(sigma))
                 .execute().actionGet();
 
@@ -410,7 +415,7 @@ public void testMultiValuedFieldWithValueScriptWithParams() throws Exception {
                 .addAggregation(
                         extendedStats("stats")
                                 .field("values")
-                                .script(new Script("_value - dec", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params))
+                                .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value - dec", params))
                                 .sigma(sigma))
                 .get();
 
@@ -437,7 +442,8 @@ public void testScriptSingleValued() throws Exception {
                 .setQuery(matchAllQuery())
                 .addAggregation(
                         extendedStats("stats")
-                                .script(new Script("doc['value'].value", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, null))
+                                .script(new Script(ScriptType.INLINE,
+                                    AggregationTestScriptsPlugin.NAME, "doc['value'].value", Collections.emptyMap()))
                                 .sigma(sigma))
                 .execute().actionGet();
 
@@ -462,7 +468,7 @@ public void testScriptSingleValuedWithParams() throws Exception {
         Map<String, Object> params = new HashMap<>();
         params.put("inc", 1);
 
-        Script script = new Script("doc['value'].value + inc", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params);
+        Script script = new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "doc['value'].value + inc", params);
 
         double sigma = randomDouble() * randomIntBetween(1, 10);
         SearchResponse searchResponse = client().prepareSearch("idx")
@@ -496,7 +502,8 @@ public void testScriptMultiValued() throws Exception {
                 .setQuery(matchAllQuery())
                 .addAggregation(
                         extendedStats("stats")
-                                .script(new Script("doc['values'].values", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, null))
+                                .script(new Script(ScriptType.INLINE,
+                                    AggregationTestScriptsPlugin.NAME, "doc['values'].values", Collections.emptyMap()))
                                 .sigma(sigma))
                 .execute().actionGet();
 
@@ -521,8 +528,8 @@ public void testScriptMultiValuedWithParams() throws Exception {
         Map<String, Object> params = new HashMap<>();
         params.put("dec", 1);
 
-        Script script = new Script("[ doc['value'].value, doc['value'].value - dec ]", ScriptType.INLINE,
-                AggregationTestScriptsPlugin.NAME, params);
+        Script script = new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "[ doc['value'].value, doc['value'].value - dec ]",
+            params);
 
         double sigma = randomDouble() * randomIntBetween(1, 10);
         SearchResponse searchResponse = client().prepareSearch("idx")
@@ -597,7 +604,7 @@ public void testOrderByEmptyAggregation() throws Exception {
 
         Terms terms = searchResponse.getAggregations().get("terms");
         assertThat(terms, notNullValue());
-        List<Terms.Bucket> buckets = terms.getBuckets();
+        List<? extends Terms.Bucket> buckets = terms.getBuckets();
         assertThat(buckets, notNullValue());
         assertThat(buckets.size(), equalTo(10));
 
@@ -630,4 +637,44 @@ private void checkUpperLowerBounds(ExtendedStats stats, double sigma) {
         assertThat(stats.getStdDeviationBound(ExtendedStats.Bounds.LOWER), equalTo(stats.getAvg() - (stats.getStdDeviation() * sigma)));
     }
 
+    /**
+     * Make sure that a request using a script does not get cached and a request
+     * not using a script does get cached.
+     */
+    public void testDontCacheScripts() throws Exception {
+        assertAcked(prepareCreate("cache_test_idx").addMapping("type", "d", "type=long")
+                .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1))
+                .get());
+        indexRandom(true, client().prepareIndex("cache_test_idx", "type", "1").setSource("s", 1),
+                client().prepareIndex("cache_test_idx", "type", "2").setSource("s", 2));
+
+        // Make sure we are starting with a clear cache
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(0L));
+
+        // Test that a request using a script does not get cached
+        SearchResponse r = client().prepareSearch("cache_test_idx").setSize(0)
+                .addAggregation(extendedStats("foo").field("d")
+                        .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value + 1", Collections.emptyMap())))
+                .get();
+        assertSearchResponse(r);
+
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(0L));
+
+        // To make sure that the cache is working test that a request not using
+        // a script is cached
+        r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(extendedStats("foo").field("d")).get();
+        assertSearchResponse(r);
+
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(1L));
+    }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/FilterTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/FilterTests.java
index 1b563d531a84d..1fa2906bd8c71 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/FilterTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/FilterTests.java
@@ -27,11 +27,8 @@ public class FilterTests extends BaseAggregationTestCase<FilterAggregationBuilde
 
     @Override
     protected FilterAggregationBuilder createTestAggregatorBuilder() {
-        FilterAggregationBuilder factory = new FilterAggregationBuilder(randomAsciiOfLengthBetween(1, 20),
-                QueryBuilders.termQuery(randomAsciiOfLengthBetween(5, 20), randomAsciiOfLengthBetween(5, 20)));
-        // NORELEASE make RandomQueryBuilder work outside of the
-        // AbstractQueryTestCase
-        // builder.query(RandomQueryBuilder.createQuery(getRandom()));
+        FilterAggregationBuilder factory = new FilterAggregationBuilder(randomAlphaOfLengthBetween(1, 20),
+                QueryBuilders.termQuery(randomAlphaOfLengthBetween(5, 20), randomAlphaOfLengthBetween(5, 20)));
         return factory;
     }
 
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/FiltersTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/FiltersTests.java
index 89fc38b7cd850..f6691d5b27fd9 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/FiltersTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/FiltersTests.java
@@ -19,12 +19,19 @@
 
 package org.elasticsearch.search.aggregations.metrics;
 
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.MatchNoneQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
-import org.elasticsearch.search.aggregations.bucket.filters.FiltersAggregator.KeyedFilter;
 import org.elasticsearch.search.aggregations.bucket.filters.FiltersAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.filters.FiltersAggregator.KeyedFilter;
+
+import java.io.IOException;
 
 public class FiltersTests extends BaseAggregationTestCase<FiltersAggregationBuilder> {
 
@@ -36,23 +43,23 @@ protected FiltersAggregationBuilder createTestAggregatorBuilder() {
         if (randomBoolean()) {
             KeyedFilter[] filters = new KeyedFilter[size];
             int i = 0;
-            for (String key : randomUnique(() -> randomAsciiOfLengthBetween(1, 20), size)) {
+            for (String key : randomUnique(() -> randomAlphaOfLengthBetween(1, 20), size)) {
                 filters[i++] = new KeyedFilter(key,
-                        QueryBuilders.termQuery(randomAsciiOfLengthBetween(5, 20), randomAsciiOfLengthBetween(5, 20)));
+                        QueryBuilders.termQuery(randomAlphaOfLengthBetween(5, 20), randomAlphaOfLengthBetween(5, 20)));
             }
-            factory = new FiltersAggregationBuilder(randomAsciiOfLengthBetween(1, 20), filters);
+            factory = new FiltersAggregationBuilder(randomAlphaOfLengthBetween(1, 20), filters);
         } else {
             QueryBuilder[] filters = new QueryBuilder[size];
             for (int i = 0; i < size; i++) {
-                filters[i] = QueryBuilders.termQuery(randomAsciiOfLengthBetween(5, 20), randomAsciiOfLengthBetween(5, 20));
+                filters[i] = QueryBuilders.termQuery(randomAlphaOfLengthBetween(5, 20), randomAlphaOfLengthBetween(5, 20));
             }
-            factory = new FiltersAggregationBuilder(randomAsciiOfLengthBetween(1, 20), filters);
+            factory = new FiltersAggregationBuilder(randomAlphaOfLengthBetween(1, 20), filters);
         }
         if (randomBoolean()) {
             factory.otherBucket(randomBoolean());
         }
         if (randomBoolean()) {
-            factory.otherBucketKey(randomAsciiOfLengthBetween(1, 20));
+            factory.otherBucketKey(randomAlphaOfLengthBetween(1, 20));
         }
         return factory;
     }
@@ -73,4 +80,41 @@ public void testFiltersSortedByKey() {
         assertEquals("aaa", original[1].key());
     }
 
+    public void testOtherBucket() throws IOException {
+        XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
+        builder.startObject();
+        builder.startArray("filters").endArray();
+        builder.endObject();
+        XContentParser parser = createParser(shuffleXContent(builder));
+        parser.nextToken();
+        QueryParseContext context = new QueryParseContext(parser);
+        FiltersAggregationBuilder filters = FiltersAggregationBuilder.parse("agg_name", context);
+        // The other bucket is disabled by default
+        assertFalse(filters.otherBucket());
+
+        builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
+        builder.startObject();
+        builder.startArray("filters").endArray();
+        builder.field("other_bucket_key", "some_key");
+        builder.endObject();
+        parser = createParser(shuffleXContent(builder));
+        parser.nextToken();
+        context = new QueryParseContext(parser);
+        filters = FiltersAggregationBuilder.parse("agg_name", context);
+        // but setting a key enables it automatically
+        assertTrue(filters.otherBucket());
+
+        builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
+        builder.startObject();
+        builder.startArray("filters").endArray();
+        builder.field("other_bucket", false);
+        builder.field("other_bucket_key", "some_key");
+        builder.endObject();
+        parser = createParser(shuffleXContent(builder));
+        parser.nextToken();
+        context = new QueryParseContext(parser);
+        filters = FiltersAggregationBuilder.parse("agg_name", context);
+        // unless the other bucket is explicitly disabled
+        assertFalse(filters.otherBucket());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsIT.java
index 3afba951b4a53..1a97cb49164a4 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsIT.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.util.BigArray;
+import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.bucket.global.Global;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket;
@@ -87,17 +88,19 @@ public void testSingleValuedField_getProperty() throws Exception {
         GeoBounds geobounds = global.getAggregations().get(aggName);
         assertThat(geobounds, notNullValue());
         assertThat(geobounds.getName(), equalTo(aggName));
-        assertThat((GeoBounds) global.getProperty(aggName), sameInstance(geobounds));
+        assertThat((GeoBounds) ((InternalAggregation)global).getProperty(aggName), sameInstance(geobounds));
         GeoPoint topLeft = geobounds.topLeft();
         GeoPoint bottomRight = geobounds.bottomRight();
         assertThat(topLeft.lat(), closeTo(singleTopLeft.lat(), GEOHASH_TOLERANCE));
         assertThat(topLeft.lon(), closeTo(singleTopLeft.lon(), GEOHASH_TOLERANCE));
         assertThat(bottomRight.lat(), closeTo(singleBottomRight.lat(), GEOHASH_TOLERANCE));
         assertThat(bottomRight.lon(), closeTo(singleBottomRight.lon(), GEOHASH_TOLERANCE));
-        assertThat((double) global.getProperty(aggName + ".top"), closeTo(singleTopLeft.lat(), GEOHASH_TOLERANCE));
-        assertThat((double) global.getProperty(aggName + ".left"), closeTo(singleTopLeft.lon(), GEOHASH_TOLERANCE));
-        assertThat((double) global.getProperty(aggName + ".bottom"), closeTo(singleBottomRight.lat(), GEOHASH_TOLERANCE));
-        assertThat((double) global.getProperty(aggName + ".right"), closeTo(singleBottomRight.lon(), GEOHASH_TOLERANCE));
+        assertThat((double) ((InternalAggregation)global).getProperty(aggName + ".top"), closeTo(singleTopLeft.lat(), GEOHASH_TOLERANCE));
+        assertThat((double) ((InternalAggregation)global).getProperty(aggName + ".left"), closeTo(singleTopLeft.lon(), GEOHASH_TOLERANCE));
+        assertThat((double) ((InternalAggregation)global).getProperty(aggName + ".bottom"),
+                closeTo(singleBottomRight.lat(), GEOHASH_TOLERANCE));
+        assertThat((double) ((InternalAggregation)global).getProperty(aggName + ".right"),
+                closeTo(singleBottomRight.lon(), GEOHASH_TOLERANCE));
     }
 
     public void testMultiValuedField() throws Exception {
@@ -230,7 +233,7 @@ public void testSingleValuedFieldAsSubAggToHighCardTermsAgg() {
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
-        List<Bucket> buckets = terms.getBuckets();
+        List<? extends Bucket> buckets = terms.getBuckets();
         assertThat(buckets.size(), equalTo(10));
         for (int i = 0; i < 10; i++) {
             Bucket bucket = buckets.get(i);
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsTests.java
index e086598dbfe16..9f5bd13b5f664 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsTests.java
@@ -26,8 +26,8 @@ public class GeoBoundsTests extends BaseAggregationTestCase<GeoBoundsAggregation
 
     @Override
     protected GeoBoundsAggregationBuilder createTestAggregatorBuilder() {
-        GeoBoundsAggregationBuilder factory = new GeoBoundsAggregationBuilder(randomAsciiOfLengthBetween(1, 20));
-        String field = randomAsciiOfLengthBetween(3, 20);
+        GeoBoundsAggregationBuilder factory = new GeoBoundsAggregationBuilder(randomAlphaOfLengthBetween(1, 20));
+        String field = randomAlphaOfLengthBetween(3, 20);
         factory.field(field);
         if (randomBoolean()) {
             factory.wrapLongitude(randomBoolean());
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidIT.java
index da869919040d1..32b036606d399 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidIT.java
@@ -21,6 +21,7 @@
 
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.geo.GeoPoint;
+import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGrid;
 import org.elasticsearch.search.aggregations.bucket.global.Global;
 import org.elasticsearch.search.aggregations.metrics.geocentroid.GeoCentroid;
@@ -58,6 +59,7 @@ public void testEmptyAggregation() throws Exception {
         assertThat(geoCentroid.getName(), equalTo(aggName));
         GeoPoint centroid = geoCentroid.centroid();
         assertThat(centroid, equalTo(null));
+        assertEquals(0, geoCentroid.count());
     }
 
     public void testUnmapped() throws Exception {
@@ -71,6 +73,7 @@ public void testUnmapped() throws Exception {
         assertThat(geoCentroid.getName(), equalTo(aggName));
         GeoPoint centroid = geoCentroid.centroid();
         assertThat(centroid, equalTo(null));
+        assertEquals(0, geoCentroid.count());
     }
 
     public void testPartiallyUnmapped() throws Exception {
@@ -85,6 +88,7 @@ public void testPartiallyUnmapped() throws Exception {
         GeoPoint centroid = geoCentroid.centroid();
         assertThat(centroid.lat(), closeTo(singleCentroid.lat(), GEOHASH_TOLERANCE));
         assertThat(centroid.lon(), closeTo(singleCentroid.lon(), GEOHASH_TOLERANCE));
+        assertEquals(numDocs, geoCentroid.count());
     }
 
     public void testSingleValuedField() throws Exception {
@@ -100,6 +104,7 @@ public void testSingleValuedField() throws Exception {
         GeoPoint centroid = geoCentroid.centroid();
         assertThat(centroid.lat(), closeTo(singleCentroid.lat(), GEOHASH_TOLERANCE));
         assertThat(centroid.lon(), closeTo(singleCentroid.lon(), GEOHASH_TOLERANCE));
+        assertEquals(numDocs, geoCentroid.count());
     }
 
     public void testSingleValueFieldGetProperty() throws Exception {
@@ -119,14 +124,17 @@ public void testSingleValueFieldGetProperty() throws Exception {
         GeoCentroid geoCentroid = global.getAggregations().get(aggName);
         assertThat(geoCentroid, notNullValue());
         assertThat(geoCentroid.getName(), equalTo(aggName));
-        assertThat((GeoCentroid) global.getProperty(aggName), sameInstance(geoCentroid));
+        assertThat((GeoCentroid) ((InternalAggregation)global).getProperty(aggName), sameInstance(geoCentroid));
         GeoPoint centroid = geoCentroid.centroid();
         assertThat(centroid.lat(), closeTo(singleCentroid.lat(), GEOHASH_TOLERANCE));
         assertThat(centroid.lon(), closeTo(singleCentroid.lon(), GEOHASH_TOLERANCE));
-        assertThat(((GeoPoint) global.getProperty(aggName + ".value")).lat(), closeTo(singleCentroid.lat(), GEOHASH_TOLERANCE));
-        assertThat(((GeoPoint) global.getProperty(aggName + ".value")).lon(), closeTo(singleCentroid.lon(), GEOHASH_TOLERANCE));
-        assertThat((double) global.getProperty(aggName + ".lat"), closeTo(singleCentroid.lat(), GEOHASH_TOLERANCE));
-        assertThat((double) global.getProperty(aggName + ".lon"), closeTo(singleCentroid.lon(), GEOHASH_TOLERANCE));
+        assertThat(((GeoPoint) ((InternalAggregation)global).getProperty(aggName + ".value")).lat(),
+                closeTo(singleCentroid.lat(), GEOHASH_TOLERANCE));
+        assertThat(((GeoPoint) ((InternalAggregation)global).getProperty(aggName + ".value")).lon(),
+                closeTo(singleCentroid.lon(), GEOHASH_TOLERANCE));
+        assertThat((double) ((InternalAggregation)global).getProperty(aggName + ".lat"), closeTo(singleCentroid.lat(), GEOHASH_TOLERANCE));
+        assertThat((double) ((InternalAggregation)global).getProperty(aggName + ".lon"), closeTo(singleCentroid.lon(), GEOHASH_TOLERANCE));
+        assertEquals(numDocs, (long) ((InternalAggregation) global).getProperty(aggName + ".count"));
     }
 
     public void testMultiValuedField() throws Exception {
@@ -142,6 +150,7 @@ public void testMultiValuedField() throws Exception {
         GeoPoint centroid = geoCentroid.centroid();
         assertThat(centroid.lat(), closeTo(multiCentroid.lat(), GEOHASH_TOLERANCE));
         assertThat(centroid.lon(), closeTo(multiCentroid.lon(), GEOHASH_TOLERANCE));
+        assertEquals(2 * numDocs, geoCentroid.count());
     }
 
     public void testSingleValueFieldAsSubAggToGeohashGrid() throws Exception {
@@ -154,9 +163,8 @@ public void testSingleValueFieldAsSubAggToGeohashGrid() throws Exception {
         GeoHashGrid grid = response.getAggregations().get("geoGrid");
         assertThat(grid, notNullValue());
         assertThat(grid.getName(), equalTo("geoGrid"));
-        List<GeoHashGrid.Bucket> buckets = grid.getBuckets();
-        for (int i=0; i < buckets.size(); ++i) {
-            GeoHashGrid.Bucket cell = buckets.get(i);
+        List<? extends GeoHashGrid.Bucket> buckets = grid.getBuckets();
+        for (GeoHashGrid.Bucket cell : buckets) {
             String geohash = cell.getKeyAsString();
             GeoPoint expectedCentroid = expectedCentroidsForGeoHash.get(geohash);
             GeoCentroid centroidAgg = cell.getAggregations().get(aggName);
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidTests.java
index 1ea21a1ff1dba..90067df601387 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidTests.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.search.aggregations.metrics;
 
-import org.elasticsearch.script.Script;
 import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
 import org.elasticsearch.search.aggregations.metrics.geocentroid.GeoCentroidAggregationBuilder;
 
@@ -27,21 +26,9 @@ public class GeoCentroidTests extends BaseAggregationTestCase<GeoCentroidAggrega
 
     @Override
     protected GeoCentroidAggregationBuilder createTestAggregatorBuilder() {
-        GeoCentroidAggregationBuilder factory = new GeoCentroidAggregationBuilder(randomAsciiOfLengthBetween(1, 20));
+        GeoCentroidAggregationBuilder factory = new GeoCentroidAggregationBuilder(randomAlphaOfLengthBetween(1, 20));
         String field = randomNumericField();
-        int randomFieldBranch = randomInt(3);
-        switch (randomFieldBranch) {
-        case 0:
-            factory.field(field);
-            break;
-        case 1:
-            factory.field(field);
-            factory.script(new Script("_value + 1"));
-            break;
-        case 2:
-            factory.script(new Script("doc[" + field + "] + 1"));
-            break;
-        }
+        randomFieldOrScript(factory, field);
         if (randomBoolean()) {
             factory.missing("0,0");
         }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksIT.java
index 57bdc7d5dfcd0..5b56e6b7efbf2 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksIT.java
@@ -20,10 +20,12 @@
 
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.aggregations.AggregationTestScriptsPlugin;
+import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.bucket.filter.Filter;
 import org.elasticsearch.search.aggregations.bucket.global.Global;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
@@ -49,7 +51,9 @@
 import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.percentileRanks;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.terms;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
@@ -206,7 +210,7 @@ public void testSingleValuedFieldGetProperty() throws Exception {
         PercentileRanks values = global.getAggregations().get("percentile_ranks");
         assertThat(values, notNullValue());
         assertThat(values.getName(), equalTo("percentile_ranks"));
-        assertThat(global.getProperty("percentile_ranks"), sameInstance(values));
+        assertThat(((InternalAggregation)global).getProperty("percentile_ranks"), sameInstance(values));
 
     }
 
@@ -257,7 +261,7 @@ public void testSingleValuedFieldWithValueScript() throws Exception {
                                 .method(PercentilesMethod.HDR)
                                 .numberOfSignificantValueDigits(sigDigits)
                                 .field("value")
-                                .script(new Script("_value - 1", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()))
+                                .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value - 1", emptyMap()))
                                 .values(pcts))
                 .execute().actionGet();
 
@@ -281,7 +285,7 @@ public void testSingleValuedFieldWithValueScriptWithParams() throws Exception {
                                 .method(PercentilesMethod.HDR)
                                 .numberOfSignificantValueDigits(sigDigits)
                                 .field("value")
-                                .script(new Script("_value - dec", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params))
+                                .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value - dec", params))
                                 .values(pcts))
                 .execute().actionGet();
 
@@ -321,7 +325,7 @@ public void testMultiValuedFieldWithValueScript() throws Exception {
                                 .method(PercentilesMethod.HDR)
                                 .numberOfSignificantValueDigits(sigDigits)
                                 .field("values")
-                                .script(new Script("_value - 1", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()))
+                                .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value - 1", emptyMap()))
                                 .values(pcts))
                 .execute().actionGet();
 
@@ -342,7 +346,7 @@ public void testMultiValuedFieldWithValueScriptReverse() throws Exception {
                                 .method(PercentilesMethod.HDR)
                                 .numberOfSignificantValueDigits(sigDigits)
                                 .field("values")
-                                .script(new Script("20 - _value", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()))
+                                .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "20 - _value", emptyMap()))
                                 .values(pcts))
                 .execute().actionGet();
 
@@ -366,7 +370,7 @@ public void testMultiValuedFieldWithValueScriptWithParams() throws Exception {
                                 .method(PercentilesMethod.HDR)
                                 .numberOfSignificantValueDigits(sigDigits)
                                 .field("values")
-                                .script(new Script("_value - dec", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params))
+                                .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value - dec", params))
                                 .values(pcts))
                 .execute().actionGet();
 
@@ -387,7 +391,7 @@ public void testScriptSingleValued() throws Exception {
                         percentileRanks("percentile_ranks")
                                 .method(PercentilesMethod.HDR)
                                 .numberOfSignificantValueDigits(sigDigits)
-                                .script(new Script("doc['value'].value", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()))
+                                .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "doc['value'].value", emptyMap()))
                                 .values(pcts))
                 .execute().actionGet();
 
@@ -403,7 +407,7 @@ public void testScriptSingleValuedWithParams() throws Exception {
         Map<String, Object> params = new HashMap<>();
         params.put("dec", 1);
 
-        Script script = new Script("doc['value'].value - dec", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params);
+        Script script = new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "doc['value'].value - dec", params);
 
         final double[] pcts = randomPercents(minValue - 1, maxValue - 1);
         SearchResponse searchResponse = client()
@@ -428,7 +432,7 @@ public void testScriptMultiValued() throws Exception {
         int sigDigits = randomSignificantDigits();
         final double[] pcts = randomPercents(minValues, maxValues);
 
-        Script script = new Script("doc['values'].values", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap());
+        Script script = new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "doc['values'].values", emptyMap());
 
         SearchResponse searchResponse = client()
                 .prepareSearch("idx")
@@ -511,7 +515,7 @@ public void testOrderByEmptyAggregation() throws Exception {
 
         Terms terms = searchResponse.getAggregations().get("terms");
         assertThat(terms, notNullValue());
-        List<Terms.Bucket> buckets = terms.getBuckets();
+        List<? extends Terms.Bucket> buckets = terms.getBuckets();
         assertThat(buckets, notNullValue());
         assertThat(buckets.size(), equalTo(10));
 
@@ -530,4 +534,45 @@ public void testOrderByEmptyAggregation() throws Exception {
         }
     }
 
+    /**
+     * Make sure that a request using a script does not get cached and a request
+     * not using a script does get cached.
+     */
+    public void testDontCacheScripts() throws Exception {
+        assertAcked(prepareCreate("cache_test_idx").addMapping("type", "d", "type=long")
+                .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1))
+                .get());
+        indexRandom(true, client().prepareIndex("cache_test_idx", "type", "1").setSource("s", 1),
+                client().prepareIndex("cache_test_idx", "type", "2").setSource("s", 2));
+
+        // Make sure we are starting with a clear cache
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(0L));
+
+        // Test that a request using a script does not get cached
+        SearchResponse r = client()
+                .prepareSearch("cache_test_idx").setSize(0).addAggregation(percentileRanks("foo").method(PercentilesMethod.HDR).field("d")
+                        .values(50.0).script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value - 1", emptyMap())))
+                .get();
+        assertSearchResponse(r);
+
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(0L));
+
+        // To make sure that the cache is working test that a request not using
+        // a script is cached
+        r = client().prepareSearch("cache_test_idx").setSize(0)
+                .addAggregation(percentileRanks("foo").method(PercentilesMethod.HDR).field("d").values(50.0)).get();
+        assertSearchResponse(r);
+
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(1L));
+    }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesIT.java
index 8112551f53cba..56fb14402ad5d 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesIT.java
@@ -20,11 +20,13 @@
 
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.aggregations.AggregationTestScriptsPlugin;
+import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.bucket.filter.Filter;
 import org.elasticsearch.search.aggregations.bucket.global.Global;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
@@ -49,7 +51,9 @@
 import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.percentiles;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.terms;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
 import static org.hamcrest.Matchers.closeTo;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
@@ -208,7 +212,7 @@ public void testSingleValuedFieldGetProperty() throws Exception {
         Percentiles percentiles = global.getAggregations().get("percentiles");
         assertThat(percentiles, notNullValue());
         assertThat(percentiles.getName(), equalTo("percentiles"));
-        assertThat(global.getProperty("percentiles"), sameInstance(percentiles));
+        assertThat(((InternalAggregation)global).getProperty("percentiles"), sameInstance(percentiles));
 
     }
 
@@ -242,7 +246,7 @@ public void testSingleValuedFieldWithValueScript() throws Exception {
                                 .numberOfSignificantValueDigits(sigDigits)
                                 .method(PercentilesMethod.HDR)
                                 .field("value")
-                                .script(new Script("_value - 1", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()))
+                                .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value - 1", emptyMap()))
                                 .percentiles(pcts))
                 .execute().actionGet();
 
@@ -267,7 +271,7 @@ public void testSingleValuedFieldWithValueScriptWithParams() throws Exception {
                                 .numberOfSignificantValueDigits(sigDigits)
                                 .method(PercentilesMethod.HDR)
                                 .field("value")
-                                .script(new Script("_value - dec", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params))
+                                .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value - dec", params))
                                 .percentiles(pcts))
                 .execute().actionGet();
 
@@ -307,7 +311,7 @@ public void testMultiValuedFieldWithValueScript() throws Exception {
                                 .numberOfSignificantValueDigits(sigDigits)
                                 .method(PercentilesMethod.HDR)
                                 .field("values")
-                                .script(new Script("_value - 1", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()))
+                                .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value - 1", emptyMap()))
                                 .percentiles(pcts))
                 .execute().actionGet();
 
@@ -328,7 +332,7 @@ public void testMultiValuedFieldWithValueScriptReverse() throws Exception {
                                 .numberOfSignificantValueDigits(sigDigits)
                                 .method(PercentilesMethod.HDR)
                                 .field("values")
-                                .script(new Script("20 - _value", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()))
+                                .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "20 - _value", emptyMap()))
                                 .percentiles(pcts))
                 .execute().actionGet();
 
@@ -353,7 +357,7 @@ public void testMultiValuedFieldWithValueScriptWithParams() throws Exception {
                                 .numberOfSignificantValueDigits(sigDigits)
                                 .method(PercentilesMethod.HDR)
                                 .field("values")
-                                .script(new Script("_value - dec", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params))
+                                .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value - dec", params))
                                 .percentiles(pcts))
                 .execute().actionGet();
 
@@ -374,7 +378,7 @@ public void testScriptSingleValued() throws Exception {
                         percentiles("percentiles")
                                 .numberOfSignificantValueDigits(sigDigits)
                                 .method(PercentilesMethod.HDR)
-                                .script(new Script("doc['value'].value", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()))
+                                .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "doc['value'].value", emptyMap()))
                                 .percentiles(pcts))
                 .execute().actionGet();
 
@@ -389,7 +393,7 @@ public void testScriptSingleValuedWithParams() throws Exception {
         Map<String, Object> params = new HashMap<>();
         params.put("dec", 1);
 
-        Script script = new Script("doc['value'].value - dec", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params);
+        Script script = new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "doc['value'].value - dec", params);
 
         final double[] pcts = randomPercentiles();
         int sigDigits = randomSignificantDigits();
@@ -415,7 +419,7 @@ public void testScriptMultiValued() throws Exception {
         final double[] pcts = randomPercentiles();
         int sigDigits = randomSignificantDigits();
 
-        Script script = new Script("doc['values'].values", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap());
+        Script script = new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "doc['values'].values", emptyMap());
 
         SearchResponse searchResponse = client()
                 .prepareSearch("idx")
@@ -502,7 +506,7 @@ public void testOrderByEmptyAggregation() throws Exception {
 
         Terms terms = searchResponse.getAggregations().get("terms");
         assertThat(terms, notNullValue());
-        List<Terms.Bucket> buckets = terms.getBuckets();
+        List<? extends Terms.Bucket> buckets = terms.getBuckets();
         assertThat(buckets, notNullValue());
         assertThat(buckets.size(), equalTo(10));
 
@@ -521,4 +525,45 @@ public void testOrderByEmptyAggregation() throws Exception {
         }
     }
 
-}
\ No newline at end of file
+    /**
+     * Make sure that a request using a script does not get cached and a request
+     * not using a script does get cached.
+     */
+    public void testDontCacheScripts() throws Exception {
+        assertAcked(prepareCreate("cache_test_idx").addMapping("type", "d", "type=long")
+                .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1))
+                .get());
+        indexRandom(true, client().prepareIndex("cache_test_idx", "type", "1").setSource("s", 1),
+                client().prepareIndex("cache_test_idx", "type", "2").setSource("s", 2));
+
+        // Make sure we are starting with a clear cache
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(0L));
+
+        // Test that a request using a script does not get cached
+        SearchResponse r = client().prepareSearch("cache_test_idx").setSize(0)
+                .addAggregation(percentiles("foo").method(PercentilesMethod.HDR).field("d").percentiles(50.0)
+                        .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value - 1", emptyMap())))
+                .get();
+        assertSearchResponse(r);
+
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(0L));
+
+        // To make sure that the cache is working test that a request not using
+        // a script is cached
+        r = client().prepareSearch("cache_test_idx").setSize(0)
+                .addAggregation(percentiles("foo").method(PercentilesMethod.HDR).field("d").percentiles(50.0)).get();
+        assertSearchResponse(r);
+
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(1L));
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalExtendedStatsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalExtendedStatsTests.java
new file mode 100644
index 0000000000000..22c22892d6cf8
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalExtendedStatsTests.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics;
+
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.ParsedAggregation;
+import org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStats.Bounds;
+import org.elasticsearch.search.aggregations.metrics.stats.extended.InternalExtendedStats;
+import org.elasticsearch.search.aggregations.metrics.stats.extended.ParsedExtendedStats;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.test.InternalAggregationTestCase;
+
+import java.util.List;
+import java.util.Map;
+
+public class InternalExtendedStatsTests extends InternalAggregationTestCase<InternalExtendedStats> {
+
+    private double sigma;
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        this.sigma = randomDoubleBetween(0, 10, true);
+    }
+
+    @Override
+    protected InternalExtendedStats createTestInstance(String name, List<PipelineAggregator> pipelineAggregators,
+            Map<String, Object> metaData) {
+        long count = frequently() ? randomIntBetween(1, Integer.MAX_VALUE) : 0;
+        double min = randomDoubleBetween(-1000000, 1000000, true);
+        double max = randomDoubleBetween(-1000000, 1000000, true);
+        double sum = randomDoubleBetween(-1000000, 1000000, true);
+        DocValueFormat format = randomNumericDocValueFormat();
+        return createInstance(name, count, sum, min, max, randomDoubleBetween(0, 1000000, true), sigma, format, pipelineAggregators,
+                metaData);
+    }
+
+    protected InternalExtendedStats createInstance(String name, long count, double sum, double min, double max, double sumOfSqrs,
+            double sigma, DocValueFormat formatter, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
+        return new InternalExtendedStats(name, count, sum, min, max, sumOfSqrs, sigma, formatter, pipelineAggregators, metaData);
+    }
+
+    @Override
+    protected void assertFromXContent(InternalExtendedStats aggregation, ParsedAggregation parsedAggregation) {
+        assertTrue(parsedAggregation instanceof ParsedExtendedStats);
+        ParsedExtendedStats parsed = (ParsedExtendedStats) parsedAggregation;
+        InternalStatsTests.assertStats(aggregation, parsed);
+
+        long count = aggregation.getCount();
+        // for count == 0, fields are rendered as `null`, so  we test that we parse to default values used also in the reduce phase
+        assertEquals(count > 0 ? aggregation.getSumOfSquares() : 0 , parsed.getSumOfSquares(), 0);
+        assertEquals(count > 0 ? aggregation.getVariance() : 0 , parsed.getVariance(), 0);
+        assertEquals(count > 0 ? aggregation.getStdDeviation() : 0 , parsed.getStdDeviation(), 0);
+        assertEquals(count > 0 ? aggregation.getStdDeviationBound(Bounds.LOWER) : 0 , parsed.getStdDeviationBound(Bounds.LOWER), 0);
+        assertEquals(count > 0 ? aggregation.getStdDeviationBound(Bounds.UPPER) : 0 , parsed.getStdDeviationBound(Bounds.UPPER), 0);
+        // also as_string values are only rendered for count != 0
+        if (count > 0) {
+            assertEquals(aggregation.getSumOfSquaresAsString(), parsed.getSumOfSquaresAsString());
+            assertEquals(aggregation.getVarianceAsString(), parsed.getVarianceAsString());
+            assertEquals(aggregation.getStdDeviationAsString(), parsed.getStdDeviationAsString());
+            assertEquals(aggregation.getStdDeviationBoundAsString(Bounds.LOWER), parsed.getStdDeviationBoundAsString(Bounds.LOWER));
+            assertEquals(aggregation.getStdDeviationBoundAsString(Bounds.UPPER), parsed.getStdDeviationBoundAsString(Bounds.UPPER));
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalMaxTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalMaxTests.java
new file mode 100644
index 0000000000000..95d97c3c908e6
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalMaxTests.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics;
+
+
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.ParsedAggregation;
+import org.elasticsearch.search.aggregations.metrics.max.InternalMax;
+import org.elasticsearch.search.aggregations.metrics.max.ParsedMax;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.test.InternalAggregationTestCase;
+
+import java.util.List;
+import java.util.Map;
+
+public class InternalMaxTests extends InternalAggregationTestCase<InternalMax> {
+
+    @Override
+    protected InternalMax createTestInstance(String name, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
+        double value = frequently() ? randomDouble() : randomFrom(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY);
+        DocValueFormat formatter = randomNumericDocValueFormat();
+        return new InternalMax(name, value, formatter, pipelineAggregators, metaData);
+    }
+
+    @Override
+    protected void assertFromXContent(InternalMax max, ParsedAggregation parsedAggregation) {
+        ParsedMax parsed = ((ParsedMax) parsedAggregation);
+        if (Double.isInfinite(max.getValue()) == false) {
+            assertEquals(max.getValue(), parsed.getValue(), Double.MIN_VALUE);
+            assertEquals(max.getValueAsString(), parsed.getValueAsString());
+        } else {
+            // we write Double.NEGATIVE_INFINITY and Double.POSITIVE_INFINITY to xContent as 'null', so we
+            // cannot differentiate between them. Also we cannot recreate the exact String representation
+            assertEquals(parsed.getValue(), Double.NEGATIVE_INFINITY, 0);
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalStatsBucketTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalStatsBucketTests.java
new file mode 100644
index 0000000000000..028a2ea8a12c4
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalStatsBucketTests.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics;
+
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.ParsedAggregation;
+import org.elasticsearch.search.aggregations.metrics.stats.InternalStats;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.InternalStatsBucket;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.ParsedStatsBucket;
+
+import java.util.List;
+import java.util.Map;
+
+public class InternalStatsBucketTests extends InternalStatsTests {
+
+    @Override
+    protected InternalStatsBucket createInstance(String name, long count, double sum, double min, double max,
+            DocValueFormat formatter, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
+        return new InternalStatsBucket(name, count, sum, min, max, formatter, pipelineAggregators, metaData);
+    }
+
+    @Override
+    protected void assertFromXContent(InternalStats aggregation, ParsedAggregation parsedAggregation) {
+        super.assertFromXContent(aggregation, parsedAggregation);
+        assertTrue(parsedAggregation instanceof ParsedStatsBucket);
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalStatsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalStatsTests.java
new file mode 100644
index 0000000000000..9164886c225d9
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalStatsTests.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.metrics;
+
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.ParsedAggregation;
+import org.elasticsearch.search.aggregations.metrics.stats.InternalStats;
+import org.elasticsearch.search.aggregations.metrics.stats.ParsedStats;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.test.InternalAggregationTestCase;
+
+import java.util.List;
+import java.util.Map;
+
+public class InternalStatsTests extends InternalAggregationTestCase<InternalStats> {
+
+    @Override
+    protected InternalStats createTestInstance(String name, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
+        long count = frequently() ? randomIntBetween(1, Integer.MAX_VALUE) : 0;
+        double min = randomDoubleBetween(-1000000, 1000000, true);
+        double max = randomDoubleBetween(-1000000, 1000000, true);
+        double sum = randomDoubleBetween(-1000000, 1000000, true);
+        DocValueFormat format = randomNumericDocValueFormat();
+        return createInstance(name, count, sum, min, max, format, pipelineAggregators, metaData);
+    }
+
+    protected InternalStats createInstance(String name, long count, double sum, double min, double max, DocValueFormat formatter,
+            List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
+        return new InternalStats(name, count, sum, min, max, formatter, pipelineAggregators, metaData);
+    }
+
+    @Override
+    protected void assertFromXContent(InternalStats aggregation, ParsedAggregation parsedAggregation) {
+        assertTrue(parsedAggregation instanceof ParsedStats);
+        ParsedStats parsed = (ParsedStats) parsedAggregation;
+        assertStats(aggregation, parsed);
+    }
+
+    static void assertStats(InternalStats aggregation, ParsedStats parsed) {
+        long count = aggregation.getCount();
+        assertEquals(count, parsed.getCount());
+        // for count == 0, fields are rendered as `null`, so  we test that we parse to default values used also in the reduce phase
+        assertEquals(count > 0 ? aggregation.getMin() : Double.POSITIVE_INFINITY , parsed.getMin(), 0);
+        assertEquals(count > 0 ? aggregation.getMax() : Double.NEGATIVE_INFINITY, parsed.getMax(), 0);
+        assertEquals(count > 0 ? aggregation.getSum() : 0, parsed.getSum(), 0);
+        assertEquals(count > 0 ? aggregation.getAvg() : 0, parsed.getAvg(), 0);
+        // also as_string values are only rendered for count != 0
+        if (count > 0) {
+            assertEquals(aggregation.getMinAsString(), parsed.getMinAsString());
+            assertEquals(aggregation.getMaxAsString(), parsed.getMaxAsString());
+            assertEquals(aggregation.getSumAsString(), parsed.getSumAsString());
+            assertEquals(aggregation.getAvgAsString(), parsed.getAvgAsString());
+        }
+    }
+}
+
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxIT.java
index d4ac8835ee0b4..7db38b9989886 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxIT.java
@@ -19,10 +19,12 @@
 package org.elasticsearch.search.aggregations.metrics;
 
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.aggregations.AggregationTestScriptsPlugin;
+import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.bucket.filter.Filter;
 import org.elasticsearch.search.aggregations.bucket.global.Global;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
@@ -44,7 +46,9 @@
 import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.max;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.terms;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
 
@@ -138,9 +142,9 @@ public void testSingleValuedFieldGetProperty() throws Exception {
         assertThat(max.getName(), equalTo("max"));
         double expectedMaxValue = 10.0;
         assertThat(max.getValue(), equalTo(expectedMaxValue));
-        assertThat((Max) global.getProperty("max"), equalTo(max));
-        assertThat((double) global.getProperty("max.value"), equalTo(expectedMaxValue));
-        assertThat((double) max.getProperty("value"), equalTo(expectedMaxValue));
+        assertThat((Max) ((InternalAggregation)global).getProperty("max"), equalTo(max));
+        assertThat((double) ((InternalAggregation)global).getProperty("max.value"), equalTo(expectedMaxValue));
+        assertThat((double) ((InternalAggregation)max).getProperty("value"), equalTo(expectedMaxValue));
     }
 
     @Override
@@ -165,7 +169,7 @@ public void testSingleValuedFieldWithValueScript() throws Exception {
                 .addAggregation(
                         max("max")
                                 .field("value")
-                                .script(new Script("_value + 1", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap())))
+                                .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value + 1", emptyMap())))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
@@ -185,7 +189,7 @@ public void testSingleValuedFieldWithValueScriptWithParams() throws Exception {
                 .addAggregation(
                         max("max")
                                 .field("value")
-                                .script(new Script("_value + inc", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params)))
+                                .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value + inc", params)))
                 .get();
 
         assertHitCount(searchResponse, 10);
@@ -218,7 +222,7 @@ public void testMultiValuedFieldWithValueScript() throws Exception {
                 .addAggregation(
                         max("max")
                                 .field("values")
-                                .script(new Script("_value + 1", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap())))
+                                .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value + 1", emptyMap())))
                 .get();
 
         assertHitCount(searchResponse, 10);
@@ -238,7 +242,7 @@ public void testMultiValuedFieldWithValueScriptWithParams() throws Exception {
                 .addAggregation(
                         max("max")
                                 .field("values")
-                                .script(new Script("_value + inc", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params)))
+                                .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value + inc", params)))
                 .get();
 
         assertHitCount(searchResponse, 10);
@@ -255,7 +259,7 @@ public void testScriptSingleValued() throws Exception {
                 .setQuery(matchAllQuery())
                 .addAggregation(
                         max("max")
-                                .script(new Script("doc['value'].value", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap())))
+                                .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "doc['value'].value", emptyMap())))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
@@ -271,7 +275,7 @@ public void testScriptSingleValuedWithParams() throws Exception {
         Map<String, Object> params = new HashMap<>();
         params.put("inc", 1);
 
-        Script script = new Script("doc['value'].value + inc", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params);
+        Script script = new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "doc['value'].value + inc", params);
 
         SearchResponse searchResponse = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
@@ -292,7 +296,8 @@ public void testScriptMultiValued() throws Exception {
                 .setQuery(matchAllQuery())
                 .addAggregation(
                         max("max")
-                                .script(new Script("doc['values'].values", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, null)))
+                                .script(new Script(ScriptType.INLINE,
+                                    AggregationTestScriptsPlugin.NAME, "doc['values'].values", Collections.emptyMap())))
                 .get();
 
         assertHitCount(searchResponse, 10);
@@ -308,8 +313,8 @@ public void testScriptMultiValuedWithParams() throws Exception {
         Map<String, Object> params = new HashMap<>();
         params.put("inc", 1);
 
-        Script script = new Script("[ doc['value'].value, doc['value'].value + inc ]", ScriptType.INLINE,
-                AggregationTestScriptsPlugin.NAME, params);
+        Script script = new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "[ doc['value'].value, doc['value'].value + inc ]",
+            params);
 
         SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery())
                 .addAggregation(max("max").script(script))
@@ -334,7 +339,7 @@ public void testOrderByEmptyAggregation() throws Exception {
 
         Terms terms = searchResponse.getAggregations().get("terms");
         assertThat(terms, notNullValue());
-        List<Terms.Bucket> buckets = terms.getBuckets();
+        List<? extends Terms.Bucket> buckets = terms.getBuckets();
         assertThat(buckets, notNullValue());
         assertThat(buckets.size(), equalTo(10));
 
@@ -352,4 +357,43 @@ public void testOrderByEmptyAggregation() throws Exception {
 
         }
     }
-}
\ No newline at end of file
+
+    /**
+     * Make sure that a request using a script does not get cached and a request
+     * not using a script does get cached.
+     */
+    public void testDontCacheScripts() throws Exception {
+        assertAcked(prepareCreate("cache_test_idx").addMapping("type", "d", "type=long")
+                .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1))
+                .get());
+        indexRandom(true, client().prepareIndex("cache_test_idx", "type", "1").setSource("s", 1),
+                client().prepareIndex("cache_test_idx", "type", "2").setSource("s", 2));
+
+        // Make sure we are starting with a clear cache
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(0L));
+
+        // Test that a request using a script does not get cached
+        SearchResponse r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(
+                max("foo").field("d").script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value + 1", emptyMap())))
+                .get();
+        assertSearchResponse(r);
+
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(0L));
+
+        // To make sure that the cache is working test that a request not using
+        // a script is cached
+        r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(max("foo").field("d")).get();
+        assertSearchResponse(r);
+
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(1L));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/MinIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/MinIT.java
index 56c12fbc77f61..d3039ad155b4f 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/MinIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/MinIT.java
@@ -19,10 +19,12 @@
 package org.elasticsearch.search.aggregations.metrics;
 
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.aggregations.AggregationTestScriptsPlugin;
+import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.bucket.filter.Filter;
 import org.elasticsearch.search.aggregations.bucket.global.Global;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
@@ -44,7 +46,9 @@
 import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.min;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.terms;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
 
@@ -139,9 +143,9 @@ public void testSingleValuedFieldGetProperty() throws Exception {
         assertThat(min.getName(), equalTo("min"));
         double expectedMinValue = 1.0;
         assertThat(min.getValue(), equalTo(expectedMinValue));
-        assertThat((Min) global.getProperty("min"), equalTo(min));
-        assertThat((double) global.getProperty("min.value"), equalTo(expectedMinValue));
-        assertThat((double) min.getProperty("value"), equalTo(expectedMinValue));
+        assertThat((Min) ((InternalAggregation)global).getProperty("min"), equalTo(min));
+        assertThat((double) ((InternalAggregation)global).getProperty("min.value"), equalTo(expectedMinValue));
+        assertThat((double) ((InternalAggregation)min).getProperty("value"), equalTo(expectedMinValue));
     }
 
     @Override
@@ -166,7 +170,7 @@ public void testSingleValuedFieldWithValueScript() throws Exception {
                 .addAggregation(
                         min("min")
                                 .field("value")
-                                .script(new Script("_value - 1", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap())))
+                                .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value - 1", emptyMap())))
                 .get();
 
         assertHitCount(searchResponse, 10);
@@ -182,7 +186,7 @@ public void testSingleValuedFieldWithValueScriptWithParams() throws Exception {
         Map<String, Object> params = new HashMap<>();
         params.put("dec", 1);
 
-        Script script = new Script("_value - dec", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params);
+        Script script = new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value - dec", params);
 
         SearchResponse searchResponse = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
@@ -219,7 +223,7 @@ public void testMultiValuedFieldWithValueScript() throws Exception {
                 .addAggregation(
                         min("min")
                                 .field("values")
-                                .script(new Script("_value - 1", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap())))
+                                .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value - 1", emptyMap())))
                 .get();
 
         assertHitCount(searchResponse, 10);
@@ -237,7 +241,7 @@ public void testMultiValuedFieldWithValueScriptReverse() throws Exception {
                 .addAggregation(
                         min("min")
                                 .field("values")
-                                .script(new Script("_value * -1", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap())))
+                                .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value * -1", emptyMap())))
                 .get();
 
         assertHitCount(searchResponse, 10);
@@ -253,7 +257,7 @@ public void testMultiValuedFieldWithValueScriptWithParams() throws Exception {
         Map<String, Object> params = new HashMap<>();
         params.put("dec", 1);
 
-        Script script = new Script("_value - dec", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params);
+        Script script = new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value - dec", params);
 
         SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery())
                 .addAggregation(min("min").field("values").script(script))
@@ -269,7 +273,7 @@ public void testMultiValuedFieldWithValueScriptWithParams() throws Exception {
 
     @Override
     public void testScriptSingleValued() throws Exception {
-        Script script = new Script("doc['value'].value", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap());
+        Script script = new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "doc['value'].value", emptyMap());
 
         SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery())
                 .addAggregation(min("min").script(script))
@@ -288,7 +292,7 @@ public void testScriptSingleValuedWithParams() throws Exception {
         Map<String, Object> params = new HashMap<>();
         params.put("dec", 1);
 
-        Script script = new Script("doc['value'].value - dec", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params);
+        Script script = new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "doc['value'].value - dec", params);
 
         SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery())
                 .addAggregation(min("min").script(script))
@@ -304,7 +308,7 @@ public void testScriptSingleValuedWithParams() throws Exception {
 
     @Override
     public void testScriptMultiValued() throws Exception {
-        Script script = new Script("doc['values'].values", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap());
+        Script script = new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "doc['values'].values", emptyMap());
         SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery())
                 .addAggregation(min("min").script(script))
                 .get();
@@ -347,7 +351,7 @@ public void testOrderByEmptyAggregation() throws Exception {
 
         Terms terms = searchResponse.getAggregations().get("terms");
         assertThat(terms, notNullValue());
-        List<Terms.Bucket> buckets = terms.getBuckets();
+        List<? extends Terms.Bucket> buckets = terms.getBuckets();
         assertThat(buckets, notNullValue());
         assertThat(buckets.size(), equalTo(10));
 
@@ -365,4 +369,43 @@ public void testOrderByEmptyAggregation() throws Exception {
 
         }
     }
-}
\ No newline at end of file
+
+    /**
+     * Make sure that a request using a script does not get cached and a request
+     * not using a script does get cached.
+     */
+    public void testDontCacheScripts() throws Exception {
+        assertAcked(prepareCreate("cache_test_idx").addMapping("type", "d", "type=long")
+                .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1))
+                .get());
+        indexRandom(true, client().prepareIndex("cache_test_idx", "type", "1").setSource("s", 1),
+                client().prepareIndex("cache_test_idx", "type", "2").setSource("s", 2));
+
+        // Make sure we are starting with a clear cache
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(0L));
+
+        // Test that a request using a script does not get cached
+        SearchResponse r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(
+                min("foo").field("d").script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value - 1", emptyMap())))
+                .get();
+        assertSearchResponse(r);
+
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(0L));
+
+        // To make sure that the cache is working test that a request not using
+        // a script is cached
+        r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(min("foo").field("d")).get();
+        assertSearchResponse(r);
+
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(1L));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/MissingTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/MissingTests.java
index 979747ade2e6c..168e01a269f32 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/MissingTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/MissingTests.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.search.aggregations.metrics;
 
-import org.elasticsearch.script.Script;
 import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
 import org.elasticsearch.search.aggregations.bucket.missing.MissingAggregationBuilder;
 
@@ -29,19 +28,7 @@ public class MissingTests extends BaseAggregationTestCase<MissingAggregationBuil
     protected final MissingAggregationBuilder createTestAggregatorBuilder() {
         MissingAggregationBuilder factory = new MissingAggregationBuilder("foo", null);
         String field = randomNumericField();
-        int randomFieldBranch = randomInt(3);
-        switch (randomFieldBranch) {
-        case 0:
-            factory.field(field);
-            break;
-        case 1:
-            factory.field(field);
-            factory.script(new Script("_value + 1"));
-            break;
-        case 2:
-            factory.script(new Script("doc[" + field + "] + 1"));
-            break;
-        }
+        randomFieldOrScript(factory, field);
         return factory;
     }
 
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/PercentileRanksTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/PercentileRanksTests.java
index 1907733fbd827..b78a67e9511ce 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/PercentileRanksTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/PercentileRanksTests.java
@@ -27,7 +27,7 @@ public class PercentileRanksTests extends BaseAggregationTestCase<PercentileRank
 
     @Override
     protected PercentileRanksAggregationBuilder createTestAggregatorBuilder() {
-        PercentileRanksAggregationBuilder factory = new PercentileRanksAggregationBuilder(randomAsciiOfLengthBetween(1, 20));
+        PercentileRanksAggregationBuilder factory = new PercentileRanksAggregationBuilder(randomAlphaOfLengthBetween(1, 20));
         if (randomBoolean()) {
             factory.keyed(randomBoolean());
         }
@@ -44,19 +44,7 @@ protected PercentileRanksAggregationBuilder createTestAggregatorBuilder() {
             factory.compression(randomIntBetween(1, 50000));
         }
         String field = randomNumericField();
-        int randomFieldBranch = randomInt(3);
-        switch (randomFieldBranch) {
-        case 0:
-            factory.field(field);
-            break;
-        case 1:
-            factory.field(field);
-            factory.script(new Script("_value + 1"));
-            break;
-        case 2:
-            factory.script(new Script("doc[" + field + "] + 1"));
-            break;
-        }
+        randomFieldOrScript(factory, field);
         if (randomBoolean()) {
             factory.missing("MISSING");
         }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/PercentilesTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/PercentilesTests.java
index b5539f8c1be8e..679bf324172b9 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/PercentilesTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/PercentilesTests.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.search.aggregations.metrics;
 
-import org.elasticsearch.script.Script;
 import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
 import org.elasticsearch.search.aggregations.metrics.percentiles.PercentilesAggregationBuilder;
 
@@ -27,7 +26,7 @@ public class PercentilesTests extends BaseAggregationTestCase<PercentilesAggrega
 
     @Override
     protected PercentilesAggregationBuilder createTestAggregatorBuilder() {
-        PercentilesAggregationBuilder factory = new PercentilesAggregationBuilder(randomAsciiOfLengthBetween(1, 20));
+        PercentilesAggregationBuilder factory = new PercentilesAggregationBuilder(randomAlphaOfLengthBetween(1, 20));
         if (randomBoolean()) {
             factory.keyed(randomBoolean());
         }
@@ -46,19 +45,7 @@ protected PercentilesAggregationBuilder createTestAggregatorBuilder() {
             factory.compression(randomIntBetween(1, 50000));
         }
         String field = randomNumericField();
-        int randomFieldBranch = randomInt(3);
-        switch (randomFieldBranch) {
-        case 0:
-            factory.field(field);
-            break;
-        case 1:
-            factory.field(field);
-            factory.script(new Script("_value + 1"));
-            break;
-        case 2:
-            factory.script(new Script("doc[" + field + "] + 1"));
-            break;
-        }
+        randomFieldOrScript(factory, field);
         if (randomBoolean()) {
             factory.missing("MISSING");
         }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java
index e1800b2f9f185..63921114a918b 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java
@@ -23,14 +23,16 @@
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.MockScriptPlugin;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.aggregations.Aggregation;
 import org.elasticsearch.search.aggregations.Aggregations;
+import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.bucket.global.Global;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket;
@@ -204,7 +206,7 @@ public void setupSuiteScopeCluster() throws Exception {
         numDocs = randomIntBetween(10, 100);
         for (int i = 0; i < numDocs; i++) {
             builders.add(client().prepareIndex("idx", "type", "" + i).setSource(
-                    jsonBuilder().startObject().field("value", randomAsciiOfLengthBetween(5, 15))
+                    jsonBuilder().startObject().field("value", randomAlphaOfLengthBetween(5, 15))
                             .field("l_value", i).endObject()));
         }
         indexRandom(true, builders);
@@ -228,24 +230,24 @@ public void setupSuiteScopeCluster() throws Exception {
         // the id of the stored script is used in test method while the source of the stored script
         // must match a predefined script from CustomScriptPlugin.pluginScripts() method
         assertAcked(client().admin().cluster().preparePutStoredScript()
-                .setScriptLang(CustomScriptPlugin.NAME)
+                .setLang(CustomScriptPlugin.NAME)
                 .setId("initScript_stored")
-                .setSource(new BytesArray("{\"script\":\"vars.multiplier = 3\"}")));
+                .setContent(new BytesArray("{\"script\":\"vars.multiplier = 3\"}"), XContentType.JSON));
 
         assertAcked(client().admin().cluster().preparePutStoredScript()
-                .setScriptLang(CustomScriptPlugin.NAME)
+                .setLang(CustomScriptPlugin.NAME)
                 .setId("mapScript_stored")
-                .setSource(new BytesArray("{\"script\":\"_agg.add(vars.multiplier)\"}")));
+                .setContent(new BytesArray("{\"script\":\"_agg.add(vars.multiplier)\"}"), XContentType.JSON));
 
         assertAcked(client().admin().cluster().preparePutStoredScript()
-                .setScriptLang(CustomScriptPlugin.NAME)
+                .setLang(CustomScriptPlugin.NAME)
                 .setId("combineScript_stored")
-                .setSource(new BytesArray("{\"script\":\"sum agg values as a new aggregation\"}")));
+                .setContent(new BytesArray("{\"script\":\"sum agg values as a new aggregation\"}"), XContentType.JSON));
 
         assertAcked(client().admin().cluster().preparePutStoredScript()
-                .setScriptLang(CustomScriptPlugin.NAME)
+                .setLang(CustomScriptPlugin.NAME)
                 .setId("reduceScript_stored")
-                .setSource(new BytesArray("{\"script\":\"sum aggs of agg values as a new aggregation\"}")));
+                .setContent(new BytesArray("{\"script\":\"sum aggs of agg values as a new aggregation\"}"), XContentType.JSON));
 
         indexRandom(true, builders);
         ensureSearchable();
@@ -277,7 +279,7 @@ protected Settings nodeSettings(int nodeOrdinal) {
     }
 
     public void testMap() {
-        Script mapScript = new Script("_agg['count'] = 1", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
+        Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg['count'] = 1", Collections.emptyMap());
 
         SearchResponse response = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
@@ -317,7 +319,7 @@ public void testMapWithParams() {
         Map<String, Object> params = new HashMap<>();
         params.put("_agg", new ArrayList<>());
 
-        Script mapScript = new Script("_agg.add(1)", ScriptType.INLINE, CustomScriptPlugin.NAME, params);
+        Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg.add(1)", params);
 
         SearchResponse response = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
@@ -365,8 +367,10 @@ public void testInitMapWithParams() {
                 .addAggregation(
                         scriptedMetric("scripted")
                                 .params(params)
-                                .initScript(new Script("vars.multiplier = 3", ScriptType.INLINE, CustomScriptPlugin.NAME, null))
-                                .mapScript(new Script("_agg.add(vars.multiplier)", ScriptType.INLINE, CustomScriptPlugin.NAME, null)))
+                                .initScript(
+                                    new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "vars.multiplier = 3", Collections.emptyMap()))
+                                .mapScript(new Script(ScriptType.INLINE, CustomScriptPlugin.NAME,
+                                    "_agg.add(vars.multiplier)", Collections.emptyMap())))
                 .get();
         assertSearchResponse(response);
         assertThat(response.getHits().getTotalHits(), equalTo(numDocs));
@@ -404,8 +408,9 @@ public void testMapCombineWithParams() {
         params.put("_agg", new ArrayList<>());
         params.put("vars", varsMap);
 
-        Script mapScript = new Script("_agg.add(1)", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
-        Script combineScript = new Script("sum agg values as a new aggregation", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
+        Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg.add(1)", Collections.emptyMap());
+        Script combineScript =
+            new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum agg values as a new aggregation", Collections.emptyMap());
 
         SearchResponse response = client()
                 .prepareSearch("idx")
@@ -455,9 +460,10 @@ public void testInitMapCombineWithParams() {
         params.put("_agg", new ArrayList<>());
         params.put("vars", varsMap);
 
-        Script initScript = new Script("vars.multiplier = 3", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
-        Script mapScript = new Script("_agg.add(vars.multiplier)", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
-        Script combineScript = new Script("sum agg values as a new aggregation", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
+        Script initScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "vars.multiplier = 3", Collections.emptyMap());
+        Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg.add(vars.multiplier)", Collections.emptyMap());
+        Script combineScript =
+            new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum agg values as a new aggregation", Collections.emptyMap());
 
         SearchResponse response = client()
                 .prepareSearch("idx")
@@ -508,10 +514,12 @@ public void testInitMapCombineReduceWithParams() {
         params.put("_agg", new ArrayList<>());
         params.put("vars", varsMap);
 
-        Script initScript = new Script("vars.multiplier = 3", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
-        Script mapScript = new Script("_agg.add(vars.multiplier)", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
-        Script combineScript = new Script("sum agg values as a new aggregation", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
-        Script reduceScript = new Script("sum aggs of agg values as a new aggregation", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
+        Script initScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "vars.multiplier = 3", Collections.emptyMap());
+        Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg.add(vars.multiplier)", Collections.emptyMap());
+        Script combineScript =
+            new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum agg values as a new aggregation", Collections.emptyMap());
+        Script reduceScript =
+            new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum aggs of agg values as a new aggregation", Collections.emptyMap());
 
         SearchResponse response = client()
                 .prepareSearch("idx")
@@ -551,10 +559,12 @@ public void testInitMapCombineReduceGetProperty() throws Exception {
         params.put("_agg", new ArrayList<>());
         params.put("vars", varsMap);
 
-        Script initScript = new Script("vars.multiplier = 3", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
-        Script mapScript = new Script("_agg.add(vars.multiplier)", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
-        Script combineScript = new Script("sum agg values as a new aggregation", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
-        Script reduceScript = new Script("sum aggs of agg values as a new aggregation", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
+        Script initScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "vars.multiplier = 3", Collections.emptyMap());
+        Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg.add(vars.multiplier)", Collections.emptyMap());
+        Script combineScript =
+            new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum agg values as a new aggregation", Collections.emptyMap());
+        Script reduceScript =
+            new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum aggs of agg values as a new aggregation", Collections.emptyMap());
 
         SearchResponse searchResponse = client()
                 .prepareSearch("idx")
@@ -591,10 +601,9 @@ public void testInitMapCombineReduceGetProperty() throws Exception {
         assertThat(object, notNullValue());
         assertThat(object, instanceOf(Number.class));
         assertThat(((Number) object).longValue(), equalTo(numDocs * 3));
-        assertThat((ScriptedMetric) global.getProperty("scripted"), sameInstance(scriptedMetricAggregation));
-        assertThat((List) global.getProperty("scripted.value"), sameInstance((List) aggregationList));
-        assertThat((List) scriptedMetricAggregation.getProperty("value"), sameInstance((List) aggregationList));
-
+        assertThat(((InternalAggregation)global).getProperty("scripted"), sameInstance(scriptedMetricAggregation));
+        assertThat((List) ((InternalAggregation)global).getProperty("scripted.value"), sameInstance((List) aggregationList));
+        assertThat((List) ((InternalAggregation)scriptedMetricAggregation).getProperty("value"), sameInstance((List) aggregationList));
     }
 
     public void testMapCombineReduceWithParams() {
@@ -605,9 +614,11 @@ public void testMapCombineReduceWithParams() {
         params.put("_agg", new ArrayList<>());
         params.put("vars", varsMap);
 
-        Script mapScript = new Script("_agg.add(vars.multiplier)", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
-        Script combineScript = new Script("sum agg values as a new aggregation", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
-        Script reduceScript = new Script("sum aggs of agg values as a new aggregation", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
+        Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg.add(vars.multiplier)", Collections.emptyMap());
+        Script combineScript =
+            new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum agg values as a new aggregation", Collections.emptyMap());
+        Script reduceScript =
+            new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum aggs of agg values as a new aggregation", Collections.emptyMap());
 
         SearchResponse response = client()
                 .prepareSearch("idx")
@@ -645,9 +656,10 @@ public void testInitMapReduceWithParams() {
         params.put("_agg", new ArrayList<>());
         params.put("vars", varsMap);
 
-        Script initScript = new Script("vars.multiplier = 3", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
-        Script mapScript = new Script("_agg.add(vars.multiplier)", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
-        Script reduceScript = new Script("sum aggs of agg values as a new aggregation", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
+        Script initScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "vars.multiplier = 3", Collections.emptyMap());
+        Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg.add(vars.multiplier)", Collections.emptyMap());
+        Script reduceScript =
+            new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum aggs of agg values as a new aggregation", Collections.emptyMap());
 
         SearchResponse response = client()
                 .prepareSearch("idx")
@@ -684,8 +696,9 @@ public void testMapReduceWithParams() {
         params.put("_agg", new ArrayList<>());
         params.put("vars", varsMap);
 
-        Script mapScript = new Script("_agg.add(vars.multiplier)", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
-        Script reduceScript = new Script("sum aggs of agg values as a new aggregation", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
+        Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg.add(vars.multiplier)", Collections.emptyMap());
+        Script reduceScript =
+            new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum aggs of agg values as a new aggregation", Collections.emptyMap());
 
         SearchResponse response = client()
                 .prepareSearch("idx")
@@ -725,11 +738,12 @@ public void testInitMapCombineReduceWithParamsAndReduceParams() {
         Map<String, Object> reduceParams = new HashMap<>();
         reduceParams.put("multiplier", 4);
 
-        Script initScript = new Script("vars.multiplier = 3", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
-        Script mapScript = new Script("_agg.add(vars.multiplier)", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
-        Script combineScript = new Script("sum agg values as a new aggregation", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
-        Script reduceScript = new Script("multiplied sum aggs of agg values as a new aggregation", ScriptType.INLINE,
-                CustomScriptPlugin.NAME, reduceParams);
+        Script initScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "vars.multiplier = 3", Collections.emptyMap());
+        Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg.add(vars.multiplier)", Collections.emptyMap());
+        Script combineScript =
+            new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum agg values as a new aggregation", Collections.emptyMap());
+        Script reduceScript =
+            new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "multiplied sum aggs of agg values as a new aggregation", reduceParams);
 
         SearchResponse response = client()
                 .prepareSearch("idx")
@@ -774,10 +788,14 @@ public void testInitMapCombineReduceWithParamsStored() {
                 .addAggregation(
                         scriptedMetric("scripted")
                                 .params(params)
-                                .initScript(new Script("initScript_stored", ScriptType.STORED, CustomScriptPlugin.NAME, null))
-                                .mapScript(new Script("mapScript_stored", ScriptType.STORED, CustomScriptPlugin.NAME, null))
-                                .combineScript(new Script("combineScript_stored", ScriptType.STORED, CustomScriptPlugin.NAME, null))
-                                .reduceScript(new Script("reduceScript_stored", ScriptType.STORED, CustomScriptPlugin.NAME, null)))
+                                .initScript(
+                                    new Script(ScriptType.STORED, CustomScriptPlugin.NAME, "initScript_stored", Collections.emptyMap()))
+                                .mapScript(
+                                    new Script(ScriptType.STORED, CustomScriptPlugin.NAME, "mapScript_stored", Collections.emptyMap()))
+                                .combineScript(
+                                    new Script(ScriptType.STORED, CustomScriptPlugin.NAME, "combineScript_stored", Collections.emptyMap()))
+                                .reduceScript(
+                                    new Script(ScriptType.STORED, CustomScriptPlugin.NAME, "reduceScript_stored", Collections.emptyMap())))
                 .get();
         assertSearchResponse(response);
         assertThat(response.getHits().getTotalHits(), equalTo(numDocs));
@@ -810,10 +828,12 @@ public void testInitMapCombineReduceWithParamsFile() {
                 .addAggregation(
                         scriptedMetric("scripted")
                                 .params(params)
-                                .initScript(new Script("init_script", ScriptType.FILE, CustomScriptPlugin.NAME, null))
-                                .mapScript(new Script("map_script", ScriptType.FILE, CustomScriptPlugin.NAME, null))
-                                .combineScript(new Script("combine_script", ScriptType.FILE, CustomScriptPlugin.NAME, null))
-                                .reduceScript(new Script("reduce_script", ScriptType.FILE, CustomScriptPlugin.NAME, null)))
+                                .initScript(new Script(ScriptType.FILE, CustomScriptPlugin.NAME, "init_script", Collections.emptyMap()))
+                                .mapScript(new Script(ScriptType.FILE, CustomScriptPlugin.NAME, "map_script", Collections.emptyMap()))
+                                .combineScript(
+                                    new Script(ScriptType.FILE, CustomScriptPlugin.NAME, "combine_script", Collections.emptyMap()))
+                                .reduceScript(
+                                    new Script(ScriptType.FILE, CustomScriptPlugin.NAME, "reduce_script", Collections.emptyMap())))
                 .get();
         assertSearchResponse(response);
         assertThat(response.getHits().getTotalHits(), equalTo(numDocs));
@@ -841,10 +861,12 @@ public void testInitMapCombineReduceWithParamsAsSubAgg() {
         params.put("_agg", new ArrayList<>());
         params.put("vars", varsMap);
 
-        Script initScript = new Script("vars.multiplier = 3", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
-        Script mapScript = new Script("_agg.add(vars.multiplier)", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
-        Script combineScript = new Script("sum agg values as a new aggregation", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
-        Script reduceScript = new Script("sum aggs of agg values as a new aggregation", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
+        Script initScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "vars.multiplier = 3", Collections.emptyMap());
+        Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg.add(vars.multiplier)", Collections.emptyMap());
+        Script combineScript =
+            new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum agg values as a new aggregation", Collections.emptyMap());
+        Script reduceScript =
+            new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum aggs of agg values as a new aggregation", Collections.emptyMap());
 
         SearchResponse response = client()
                 .prepareSearch("idx")
@@ -900,10 +922,12 @@ public void testEmptyAggregation() throws Exception {
         params.put("_agg", new ArrayList<>());
         params.put("vars", varsMap);
 
-        Script initScript = new Script("vars.multiplier = 3", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
-        Script mapScript = new Script("_agg.add(vars.multiplier)", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
-        Script combineScript = new Script("sum agg values as a new aggregation", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
-        Script reduceScript = new Script("sum aggs of agg values as a new aggregation", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
+        Script initScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "vars.multiplier = 3", Collections.emptyMap());
+        Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg.add(vars.multiplier)", Collections.emptyMap());
+        Script combineScript =
+            new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum agg values as a new aggregation", Collections.emptyMap());
+        Script reduceScript =
+            new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum aggs of agg values as a new aggregation", Collections.emptyMap());
 
         SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx")
                 .setQuery(matchAllQuery())
@@ -933,4 +957,33 @@ public void testEmptyAggregation() throws Exception {
         assertThat(aggregationResult.size(), equalTo(1));
         assertThat(aggregationResult.get(0), equalTo(0));
     }
+
+    /**
+     * Make sure that a request using a script does not get cached and a request
+     * not using a script does get cached.
+     */
+    public void testDontCacheScripts() throws Exception {
+        Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg['count'] = 1", Collections.emptyMap());
+        assertAcked(prepareCreate("cache_test_idx").addMapping("type", "d", "type=long")
+                .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1))
+                .get());
+        indexRandom(true, client().prepareIndex("cache_test_idx", "type", "1").setSource("s", 1),
+                client().prepareIndex("cache_test_idx", "type", "2").setSource("s", 2));
+
+        // Make sure we are starting with a clear cache
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(0L));
+
+        // Test that a request using a script does not get cached
+        SearchResponse r = client().prepareSearch("cache_test_idx").setSize(0)
+                .addAggregation(scriptedMetric("foo").mapScript(mapScript)).get();
+        assertSearchResponse(r);
+
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(0L));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricTests.java
index e4f96fae76200..3ca4dfd253d9b 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricTests.java
@@ -20,10 +20,11 @@
 package org.elasticsearch.search.aggregations.metrics;
 
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
 import org.elasticsearch.search.aggregations.metrics.scripted.ScriptedMetricAggregationBuilder;
 
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -31,7 +32,7 @@ public class ScriptedMetricTests extends BaseAggregationTestCase<ScriptedMetricA
 
     @Override
     protected ScriptedMetricAggregationBuilder createTestAggregatorBuilder() {
-        ScriptedMetricAggregationBuilder factory = new ScriptedMetricAggregationBuilder(randomAsciiOfLengthBetween(1, 20));
+        ScriptedMetricAggregationBuilder factory = new ScriptedMetricAggregationBuilder(randomAlphaOfLengthBetween(1, 20));
         if (randomBoolean()) {
             factory.initScript(randomScript("initScript"));
         }
@@ -52,9 +53,11 @@ protected ScriptedMetricAggregationBuilder createTestAggregatorBuilder() {
 
     private Script randomScript(String script) {
         if (randomBoolean()) {
-            return new Script(script);
+            return mockScript(script);
         } else {
-            return new Script(script, randomFrom(ScriptType.values()), randomFrom("my_lang", null), null);
+            ScriptType type = randomFrom(ScriptType.values());
+            return new Script(
+                type, type == ScriptType.STORED ? null : randomFrom("my_lang", Script.DEFAULT_SCRIPT_LANG), script, Collections.emptyMap());
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsIT.java
index 510df4c572bf0..9231f09396307 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsIT.java
@@ -22,10 +22,12 @@
 import org.apache.logging.log4j.util.Supplier;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.ShardSearchFailure;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.aggregations.AggregationTestScriptsPlugin;
+import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.bucket.filter.Filter;
 import org.elasticsearch.search.aggregations.bucket.global.Global;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
@@ -47,7 +49,9 @@
 import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.stats;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.terms;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
@@ -158,7 +162,6 @@ public void testSingleValuedField_WithFormatter() throws Exception {
         assertThat(stats.getSum(), equalTo((double) 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10));
         assertThat(stats.getSumAsString(), equalTo("0055.0"));
         assertThat(stats.getCount(), equalTo(10L));
-        assertThat(stats.getCountAsString(), equalTo("0010.0"));
     }
 
     @Override
@@ -178,24 +181,24 @@ public void testSingleValuedFieldGetProperty() throws Exception {
         Stats stats = global.getAggregations().get("stats");
         assertThat(stats, notNullValue());
         assertThat(stats.getName(), equalTo("stats"));
-        Stats statsFromProperty = (Stats) global.getProperty("stats");
+        Stats statsFromProperty = (Stats) ((InternalAggregation)global).getProperty("stats");
         assertThat(statsFromProperty, notNullValue());
         assertThat(statsFromProperty, sameInstance(stats));
         double expectedAvgValue = (double) (1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10) / 10;
         assertThat(stats.getAvg(), equalTo(expectedAvgValue));
-        assertThat((double) global.getProperty("stats.avg"), equalTo(expectedAvgValue));
+        assertThat((double) ((InternalAggregation)global).getProperty("stats.avg"), equalTo(expectedAvgValue));
         double expectedMinValue = 1.0;
         assertThat(stats.getMin(), equalTo(expectedMinValue));
-        assertThat((double) global.getProperty("stats.min"), equalTo(expectedMinValue));
+        assertThat((double) ((InternalAggregation)global).getProperty("stats.min"), equalTo(expectedMinValue));
         double expectedMaxValue = 10.0;
         assertThat(stats.getMax(), equalTo(expectedMaxValue));
-        assertThat((double) global.getProperty("stats.max"), equalTo(expectedMaxValue));
+        assertThat((double) ((InternalAggregation)global).getProperty("stats.max"), equalTo(expectedMaxValue));
         double expectedSumValue = 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10;
         assertThat(stats.getSum(), equalTo(expectedSumValue));
-        assertThat((double) global.getProperty("stats.sum"), equalTo(expectedSumValue));
+        assertThat((double) ((InternalAggregation)global).getProperty("stats.sum"), equalTo(expectedSumValue));
         long expectedCountValue = 10;
         assertThat(stats.getCount(), equalTo(expectedCountValue));
-        assertThat((double) global.getProperty("stats.count"), equalTo((double) expectedCountValue));
+        assertThat((double) ((InternalAggregation)global).getProperty("stats.count"), equalTo((double) expectedCountValue));
     }
 
     @Override
@@ -226,7 +229,7 @@ public void testSingleValuedFieldWithValueScript() throws Exception {
                 .addAggregation(
                         stats("stats")
                                 .field("value")
-                                .script(new Script("_value + 1", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap())))
+                                .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value + 1", emptyMap())))
                 .get();
 
         assertShardExecutionState(searchResponse, 0);
@@ -252,7 +255,7 @@ public void testSingleValuedFieldWithValueScriptWithParams() throws Exception {
                 .addAggregation(
                         stats("stats")
                                 .field("value")
-                                .script(new Script("_value + inc", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params)))
+                                .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value + inc", params)))
                 .get();
 
         assertShardExecutionState(searchResponse, 0);
@@ -297,7 +300,7 @@ public void testMultiValuedFieldWithValueScript() throws Exception {
                 .addAggregation(
                         stats("stats")
                                 .field("values")
-                                .script(new Script("_value - 1", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap())))
+                                .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value - 1", emptyMap())))
                 .get();
 
         assertShardExecutionState(searchResponse, 0);
@@ -323,7 +326,7 @@ public void testMultiValuedFieldWithValueScriptWithParams() throws Exception {
                 .addAggregation(
                         stats("stats")
                                 .field("values")
-                                .script(new Script("_value - dec", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params)))
+                                .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value - dec", params)))
                 .get();
 
         assertShardExecutionState(searchResponse, 0);
@@ -346,7 +349,7 @@ public void testScriptSingleValued() throws Exception {
                 .setQuery(matchAllQuery())
                 .addAggregation(
                         stats("stats")
-                                .script(new Script("doc['value'].value", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap())))
+                                .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "doc['value'].value", emptyMap())))
                 .get();
 
         assertShardExecutionState(searchResponse, 0);
@@ -368,7 +371,7 @@ public void testScriptSingleValuedWithParams() throws Exception {
         Map<String, Object> params = new HashMap<>();
         params.put("inc", 1);
 
-        Script script = new Script("doc['value'].value + inc", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params);
+        Script script = new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "doc['value'].value + inc", params);
 
         SearchResponse searchResponse = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
@@ -391,7 +394,7 @@ public void testScriptSingleValuedWithParams() throws Exception {
 
     @Override
     public void testScriptMultiValued() throws Exception {
-        Script script = new Script("doc['values'].values", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap());
+        Script script = new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "doc['values'].values", emptyMap());
 
         SearchResponse searchResponse = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
@@ -417,8 +420,8 @@ public void testScriptMultiValuedWithParams() throws Exception {
         Map<String, Object> params = new HashMap<>();
         params.put("dec", 1);
 
-        Script script = new Script("[ doc['value'].value, doc['value'].value - dec ]", ScriptType.INLINE,
-                AggregationTestScriptsPlugin.NAME, params);
+        Script script = new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "[ doc['value'].value, doc['value'].value - dec ]",
+            params);
 
         SearchResponse searchResponse = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
@@ -452,7 +455,7 @@ public void testOrderByEmptyAggregation() throws Exception {
 
         Terms terms = searchResponse.getAggregations().get("terms");
         assertThat(terms, notNullValue());
-        List<Terms.Bucket> buckets = terms.getBuckets();
+        List<? extends Terms.Bucket> buckets = terms.getBuckets();
         assertThat(buckets, notNullValue());
         assertThat(buckets.size(), equalTo(10));
 
@@ -485,4 +488,43 @@ private void assertShardExecutionState(SearchResponse response, int expectedFail
         }
         assertThat("Not all shards are initialized", response.getSuccessfulShards(), equalTo(response.getTotalShards()));
     }
+
+    /**
+     * Make sure that a request using a script does not get cached and a request
+     * not using a script does get cached.
+     */
+    public void testDontCacheScripts() throws Exception {
+        assertAcked(prepareCreate("cache_test_idx").addMapping("type", "d", "type=long")
+                .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1))
+                .get());
+        indexRandom(true, client().prepareIndex("cache_test_idx", "type", "1").setSource("s", 1),
+                client().prepareIndex("cache_test_idx", "type", "2").setSource("s", 2));
+
+        // Make sure we are starting with a clear cache
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(0L));
+
+        // Test that a request using a script does not get cached
+        SearchResponse r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(
+                stats("foo").field("d").script(
+                    new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value + 1", Collections.emptyMap()))).get();
+        assertSearchResponse(r);
+
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(0L));
+
+        // To make sure that the cache is working test that a request not using
+        // a script is cached
+        r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(stats("foo").field("d")).get();
+        assertSearchResponse(r);
+
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(1L));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
index 186bc8fb27b76..9ab13084fd76d 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
@@ -29,8 +29,9 @@
 import org.elasticsearch.script.LeafSearchScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptEngineService;
-import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.script.SearchScript;
+import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.bucket.filter.Filter;
 import org.elasticsearch.search.aggregations.bucket.global.Global;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
@@ -44,6 +45,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -55,7 +57,9 @@
 import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.sum;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.terms;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
 
@@ -152,9 +156,9 @@ public void testSingleValuedFieldGetProperty() throws Exception {
         assertThat(sum.getName(), equalTo("sum"));
         double expectedSumValue = (double) 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10;
         assertThat(sum.getValue(), equalTo(expectedSumValue));
-        assertThat((Sum) global.getProperty("sum"), equalTo(sum));
-        assertThat((double) global.getProperty("sum.value"), equalTo(expectedSumValue));
-        assertThat((double) sum.getProperty("value"), equalTo(expectedSumValue));
+        assertThat((Sum) ((InternalAggregation)global).getProperty("sum"), equalTo(sum));
+        assertThat((double) ((InternalAggregation)global).getProperty("sum.value"), equalTo(expectedSumValue));
+        assertThat((double) ((InternalAggregation)sum).getProperty("value"), equalTo(expectedSumValue));
     }
 
     @Override
@@ -176,7 +180,8 @@ public void testSingleValuedFieldPartiallyUnmapped() throws Exception {
     public void testSingleValuedFieldWithValueScript() throws Exception {
         SearchResponse searchResponse = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
-                .addAggregation(sum("sum").field("value").script(new Script("", ScriptType.INLINE, FieldValueScriptEngine.NAME, null)))
+                .addAggregation(sum("sum").field("value").script(
+                    new Script(ScriptType.INLINE, FieldValueScriptEngine.NAME, "", Collections.emptyMap())))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
@@ -193,7 +198,7 @@ public void testSingleValuedFieldWithValueScriptWithParams() throws Exception {
         params.put("increment", 1);
         SearchResponse searchResponse = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
-                .addAggregation(sum("sum").field("value").script(new Script("", ScriptType.INLINE, FieldValueScriptEngine.NAME, params)))
+                .addAggregation(sum("sum").field("value").script(new Script(ScriptType.INLINE, FieldValueScriptEngine.NAME, "", params)))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
@@ -208,7 +213,8 @@ public void testSingleValuedFieldWithValueScriptWithParams() throws Exception {
     public void testScriptSingleValued() throws Exception {
         SearchResponse searchResponse = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
-                .addAggregation(sum("sum").script(new Script("value", ScriptType.INLINE, ExtractFieldScriptEngine.NAME, null)))
+                .addAggregation(sum("sum").script(
+                    new Script(ScriptType.INLINE, ExtractFieldScriptEngine.NAME, "value", Collections.emptyMap())))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
@@ -225,7 +231,7 @@ public void testScriptSingleValuedWithParams() throws Exception {
         params.put("inc", 1);
         SearchResponse searchResponse = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
-                .addAggregation(sum("sum").script(new Script("value", ScriptType.INLINE, ExtractFieldScriptEngine.NAME, params)))
+                .addAggregation(sum("sum").script(new Script(ScriptType.INLINE, ExtractFieldScriptEngine.NAME, "value", params)))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
@@ -240,7 +246,8 @@ public void testScriptSingleValuedWithParams() throws Exception {
     public void testScriptMultiValued() throws Exception {
         SearchResponse searchResponse = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
-                .addAggregation(sum("sum").script(new Script("values", ScriptType.INLINE, ExtractFieldScriptEngine.NAME, null)))
+                .addAggregation(sum("sum").script(
+                    new Script(ScriptType.INLINE, ExtractFieldScriptEngine.NAME, "values", Collections.emptyMap())))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
@@ -258,7 +265,7 @@ public void testScriptMultiValuedWithParams() throws Exception {
         SearchResponse searchResponse = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
                 .addAggregation(
-                        sum("sum").script(new Script("values", ScriptType.INLINE, ExtractFieldScriptEngine.NAME, params)))
+                        sum("sum").script(new Script(ScriptType.INLINE, ExtractFieldScriptEngine.NAME, "values", params)))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
@@ -290,7 +297,8 @@ public void testMultiValuedFieldWithValueScript() throws Exception {
 
         SearchResponse searchResponse = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
-                .addAggregation(sum("sum").field("values").script(new Script("", ScriptType.INLINE, FieldValueScriptEngine.NAME, null)))
+                .addAggregation(sum("sum").field("values").script(
+                    new Script(ScriptType.INLINE, FieldValueScriptEngine.NAME, "", Collections.emptyMap())))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
@@ -306,7 +314,7 @@ public void testMultiValuedFieldWithValueScriptWithParams() throws Exception {
         Map<String, Object> params = new HashMap<>();
         params.put("increment", 1);
         SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery())
-                .addAggregation(sum("sum").field("values").script(new Script("", ScriptType.INLINE, FieldValueScriptEngine.NAME, params)))
+                .addAggregation(sum("sum").field("values").script(new Script(ScriptType.INLINE, FieldValueScriptEngine.NAME, "", params)))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
@@ -328,7 +336,7 @@ public void testOrderByEmptyAggregation() throws Exception {
 
         Terms terms = searchResponse.getAggregations().get("terms");
         assertThat(terms, notNullValue());
-        List<Terms.Bucket> buckets = terms.getBuckets();
+        List<? extends Terms.Bucket> buckets = terms.getBuckets();
         assertThat(buckets, notNullValue());
         assertThat(buckets.size(), equalTo(10));
 
@@ -347,6 +355,45 @@ public void testOrderByEmptyAggregation() throws Exception {
         }
     }
 
+    /**
+     * Make sure that a request using a script does not get cached and a request
+     * not using a script does get cached.
+     */
+    public void testDontCacheScripts() throws Exception {
+        assertAcked(prepareCreate("cache_test_idx").addMapping("type", "d", "type=long")
+                .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1))
+                .get());
+        indexRandom(true, client().prepareIndex("cache_test_idx", "type", "1").setSource("s", 1),
+                client().prepareIndex("cache_test_idx", "type", "2").setSource("s", 2));
+
+        // Make sure we are starting with a clear cache
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(0L));
+
+        // Test that a request using a script does not get cached
+        SearchResponse r = client().prepareSearch("cache_test_idx").setSize(0)
+                .addAggregation(sum("foo").field("d").script(
+                    new Script(ScriptType.INLINE, FieldValueScriptEngine.NAME, "", Collections.emptyMap()))).get();
+        assertSearchResponse(r);
+
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(0L));
+
+        // To make sure that the cache is working test that a request not using
+        // a script is cached
+        r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(sum("foo").field("d")).get();
+        assertSearchResponse(r);
+
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(1L));
+    }
+
     /**
      * Mock plugin for the {@link ExtractFieldScriptEngine}
      */
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksIT.java
index e50b89d8b96cc..f1943747ceb43 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksIT.java
@@ -20,13 +20,13 @@
 
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.CollectionUtils;
-import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.script.MockScriptPlugin;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.aggregations.AggregationTestScriptsPlugin;
+import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.bucket.filter.Filter;
 import org.elasticsearch.search.aggregations.bucket.global.Global;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
@@ -43,7 +43,6 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.function.Function;
 
 import static java.util.Collections.emptyMap;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
@@ -53,7 +52,9 @@
 import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.percentileRanks;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.terms;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
@@ -199,7 +200,7 @@ public void testSingleValuedFieldGetProperty() throws Exception {
         PercentileRanks values = global.getAggregations().get("percentile_ranks");
         assertThat(values, notNullValue());
         assertThat(values.getName(), equalTo("percentile_ranks"));
-        assertThat(global.getProperty("percentile_ranks"), sameInstance(values));
+        assertThat(((InternalAggregation)global).getProperty("percentile_ranks"), sameInstance(values));
     }
 
     public void testSingleValuedFieldOutsideRange() throws Exception {
@@ -242,7 +243,7 @@ public void testSingleValuedFieldWithValueScript() throws Exception {
                         randomCompression(
                                 percentileRanks("percentile_ranks"))
                                 .field("value")
-                                .script(new Script("_value - 1", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()))
+                                .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value - 1", emptyMap()))
                                 .values(pcts))
                 .execute().actionGet();
 
@@ -263,7 +264,7 @@ public void testSingleValuedFieldWithValueScriptWithParams() throws Exception {
                         randomCompression(
                                 percentileRanks("percentile_ranks"))
                                 .field("value")
-                                .script(new Script("_value - dec", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params))
+                                .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value - dec", params))
                                 .values(pcts))
                 .execute().actionGet();
 
@@ -298,7 +299,7 @@ public void testMultiValuedFieldWithValueScript() throws Exception {
                         randomCompression(
                                 percentileRanks("percentile_ranks"))
                                 .field("values")
-                                .script(new Script("_value - 1", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()))
+                                .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value - 1", emptyMap()))
                                 .values(pcts))
                 .execute().actionGet();
 
@@ -316,7 +317,7 @@ public void testMultiValuedFieldWithValueScriptReverse() throws Exception {
                         randomCompression(
                                 percentileRanks("percentile_ranks"))
                                 .field("values")
-                                .script(new Script("_value * -1", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()))
+                                .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value * -1", emptyMap()))
                                 .values(pcts))
                 .execute().actionGet();
 
@@ -337,7 +338,7 @@ public void testMultiValuedFieldWithValueScriptWithParams() throws Exception {
                         randomCompression(
                                 percentileRanks("percentile_ranks"))
                                 .field("values")
-                                .script(new Script("_value - dec", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params))
+                                .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value - dec", params))
                                 .values(pcts))
                 .execute().actionGet();
 
@@ -355,7 +356,7 @@ public void testScriptSingleValued() throws Exception {
                 .addAggregation(
                         randomCompression(
                                 percentileRanks("percentile_ranks"))
-                                .script(new Script("doc['value'].value", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()))
+                                .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "doc['value'].value", emptyMap()))
                                 .values(pcts))
                 .execute().actionGet();
 
@@ -370,7 +371,7 @@ public void testScriptSingleValuedWithParams() throws Exception {
         Map<String, Object> params = new HashMap<>();
         params.put("dec", 1);
 
-        Script script = new Script("doc['value'].value - dec", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params);
+        Script script = new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "doc['value'].value - dec", params);
 
         final double[] pcts = randomPercents(minValue - 1, maxValue - 1);
         SearchResponse searchResponse = client().prepareSearch("idx")
@@ -391,7 +392,7 @@ public void testScriptSingleValuedWithParams() throws Exception {
     @Override
     public void testScriptMultiValued() throws Exception {
         final double[] pcts = randomPercents(minValues, maxValues);
-        Script script = new Script("doc['values'].values", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap());
+        Script script = new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "doc['values'].values", emptyMap());
         SearchResponse searchResponse = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
                 .addAggregation(
@@ -465,7 +466,7 @@ public void testOrderByEmptyAggregation() throws Exception {
 
         Terms terms = searchResponse.getAggregations().get("terms");
         assertThat(terms, notNullValue());
-        List<Terms.Bucket> buckets = terms.getBuckets();
+        List<? extends Terms.Bucket> buckets = terms.getBuckets();
         assertThat(buckets, notNullValue());
         assertThat(buckets.size(), equalTo(10));
 
@@ -484,4 +485,42 @@ public void testOrderByEmptyAggregation() throws Exception {
         }
     }
 
+    /**
+     * Make sure that a request using a script does not get cached and a request
+     * not using a script does get cached.
+     */
+    public void testDontCacheScripts() throws Exception {
+        assertAcked(prepareCreate("cache_test_idx").addMapping("type", "d", "type=long")
+                .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1))
+                .get());
+        indexRandom(true, client().prepareIndex("cache_test_idx", "type", "1").setSource("s", 1),
+                client().prepareIndex("cache_test_idx", "type", "2").setSource("s", 2));
+
+        // Make sure we are starting with a clear cache
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(0L));
+
+        // Test that a request using a script does not get cached
+        SearchResponse r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(percentileRanks("foo").field("d").values(50.0)
+                .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value - 1", emptyMap()))).get();
+        assertSearchResponse(r);
+
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(0L));
+
+        // To make sure that the cache is working test that a request not using
+        // a script is cached
+        r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(percentileRanks("foo").field("d").values(50.0)).get();
+        assertSearchResponse(r);
+
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(1L));
+    }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentilesIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentilesIT.java
index b0268351954e0..2589e9977a6c9 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentilesIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentilesIT.java
@@ -20,11 +20,13 @@
 
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.aggregations.AggregationTestScriptsPlugin;
+import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.bucket.filter.Filter;
 import org.elasticsearch.search.aggregations.bucket.global.Global;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
@@ -50,7 +52,9 @@
 import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.percentiles;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.terms;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
@@ -195,7 +199,7 @@ public void testSingleValuedFieldGetProperty() throws Exception {
         Percentiles percentiles = global.getAggregations().get("percentiles");
         assertThat(percentiles, notNullValue());
         assertThat(percentiles.getName(), equalTo("percentiles"));
-        assertThat(global.getProperty("percentiles"), sameInstance(percentiles));
+        assertThat(((InternalAggregation)global).getProperty("percentiles"), sameInstance(percentiles));
     }
 
     @Override
@@ -223,7 +227,7 @@ public void testSingleValuedFieldWithValueScript() throws Exception {
                         randomCompression(
                                 percentiles("percentiles"))
                                     .field("value")
-                                    .script(new Script("_value - 1", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()))
+                                    .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value - 1", emptyMap()))
                                     .percentiles(pcts))
                 .execute().actionGet();
 
@@ -244,7 +248,7 @@ public void testSingleValuedFieldWithValueScriptWithParams() throws Exception {
                         randomCompression(
                                 percentiles("percentiles"))
                                     .field("value")
-                                    .script(new Script("_value - dec", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params))
+                                    .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value - dec", params))
                                 .percentiles(pcts))
                 .execute().actionGet();
 
@@ -277,7 +281,7 @@ public void testMultiValuedFieldWithValueScript() throws Exception {
                         randomCompression(
                                 percentiles("percentiles"))
                                     .field("values")
-                                .script(new Script("_value - 1", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()))
+                                .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value - 1", emptyMap()))
                                 .percentiles(pcts))
                 .execute().actionGet();
 
@@ -295,7 +299,7 @@ public void testMultiValuedFieldWithValueScriptReverse() throws Exception {
                         randomCompression(
                                 percentiles("percentiles"))
                                     .field("values")
-                                    .script(new Script("_value * -1", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()))
+                                    .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value * -1", emptyMap()))
                                     .percentiles(pcts))
                 .execute().actionGet();
 
@@ -316,7 +320,7 @@ public void testMultiValuedFieldWithValueScriptWithParams() throws Exception {
                         randomCompression(
                                 percentiles("percentiles"))
                                     .field("values")
-                                    .script(new Script("_value - dec", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params))
+                                    .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value - dec", params))
                                     .percentiles(pcts))
                 .execute().actionGet();
 
@@ -328,7 +332,7 @@ public void testMultiValuedFieldWithValueScriptWithParams() throws Exception {
 
     @Override
     public void testScriptSingleValued() throws Exception {
-        Script script = new Script("doc['value'].value", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap());
+        Script script = new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "doc['value'].value", emptyMap());
         final double[] pcts = randomPercentiles();
         SearchResponse searchResponse = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
@@ -350,7 +354,7 @@ public void testScriptSingleValuedWithParams() throws Exception {
         Map<String, Object> params = new HashMap<>();
         params.put("dec", 1);
 
-        Script script = new Script("doc['value'].value - dec", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params);
+        Script script = new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "doc['value'].value - dec", params);
 
         final double[] pcts = randomPercentiles();
         SearchResponse searchResponse = client().prepareSearch("idx")
@@ -371,7 +375,7 @@ public void testScriptSingleValuedWithParams() throws Exception {
     @Override
     public void testScriptMultiValued() throws Exception {
         final double[] pcts = randomPercentiles();
-        Script script = new Script("doc['values'].values", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap());
+        Script script = new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "doc['values'].values", emptyMap());
 
         SearchResponse searchResponse = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
@@ -447,7 +451,7 @@ public void testOrderByEmptyAggregation() throws Exception {
 
         Terms terms = searchResponse.getAggregations().get("terms");
         assertThat(terms, notNullValue());
-        List<Terms.Bucket> buckets = terms.getBuckets();
+        List<? extends Terms.Bucket> buckets = terms.getBuckets();
         assertThat(buckets, notNullValue());
         assertThat(buckets.size(), equalTo(10));
 
@@ -465,4 +469,43 @@ public void testOrderByEmptyAggregation() throws Exception {
 
         }
     }
+
+    /**
+     * Make sure that a request using a script does not get cached and a request
+     * not using a script does get cached.
+     */
+    public void testDontCacheScripts() throws Exception {
+        assertAcked(prepareCreate("cache_test_idx").addMapping("type", "d", "type=long")
+                .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1))
+                .get());
+        indexRandom(true, client().prepareIndex("cache_test_idx", "type", "1").setSource("s", 1),
+                client().prepareIndex("cache_test_idx", "type", "2").setSource("s", 2));
+
+        // Make sure we are starting with a clear cache
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(0L));
+
+        // Test that a request using a script does not get cached
+        SearchResponse r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(percentiles("foo").field("d")
+                .percentiles(50.0).script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value - 1", emptyMap())))
+                .get();
+        assertSearchResponse(r);
+
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(0L));
+
+        // To make sure that the cache is working test that a request not using
+        // a script is cached
+        r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(percentiles("foo").field("d").percentiles(50.0)).get();
+        assertSearchResponse(r);
+
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(1L));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java
index 45d44c863a44b..6819fddf3e3e5 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java
@@ -25,18 +25,21 @@
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchType;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.MockScriptEngine;
 import org.elasticsearch.script.MockScriptPlugin;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHitField;
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode;
+import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.bucket.global.Global;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
 import org.elasticsearch.search.aggregations.bucket.nested.Nested;
@@ -46,6 +49,8 @@
 import org.elasticsearch.search.aggregations.metrics.tophits.TopHits;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
+import org.elasticsearch.search.rescore.RescoreBuilder;
+import org.elasticsearch.search.sort.ScriptSortBuilder.ScriptSortType;
 import org.elasticsearch.search.sort.SortBuilders;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -84,9 +89,6 @@
 import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.Matchers.sameInstance;
 
-/**
- *
- */
 @ESIntegTestCase.SuiteScopeTestCase()
 public class TopHitsIT extends ESIntegTestCase {
 
@@ -113,7 +115,8 @@ public static String randomExecutionHint() {
 
     @Override
     public void setupSuiteScopeCluster() throws Exception {
-        assertAcked(prepareCreate("idx").addMapping("type", TERMS_AGGS_FIELD, "type=keyword", "group", "type=keyword"));
+        assertAcked(prepareCreate("idx").addMapping("type", TERMS_AGGS_FIELD, "type=keyword"));
+        assertAcked(prepareCreate("field-collapsing").addMapping("type", "group", "type=keyword"));
         createIndex("empty");
         assertAcked(prepareCreate("articles").addMapping("article", jsonBuilder().startObject().startObject("article").startObject("properties")
                 .startObject(TERMS_AGGS_FIELD)
@@ -158,49 +161,49 @@ public void setupSuiteScopeCluster() throws Exception {
                     .endObject()));
         }
 
-        builders.add(client().prepareIndex("idx", "field-collapsing", "1").setSource(jsonBuilder()
+        builders.add(client().prepareIndex("field-collapsing", "type", "1").setSource(jsonBuilder()
                 .startObject()
                 .field("group", "a")
                 .field("text", "term x y z b")
                 .endObject()));
-        builders.add(client().prepareIndex("idx", "field-collapsing", "2").setSource(jsonBuilder()
+        builders.add(client().prepareIndex("field-collapsing", "type", "2").setSource(jsonBuilder()
                 .startObject()
                 .field("group", "a")
                 .field("text", "term x y z n rare")
                 .field("value", 1)
                 .endObject()));
-        builders.add(client().prepareIndex("idx", "field-collapsing", "3").setSource(jsonBuilder()
+        builders.add(client().prepareIndex("field-collapsing", "type", "3").setSource(jsonBuilder()
                 .startObject()
                 .field("group", "b")
                 .field("text", "x y z term")
                 .endObject()));
-        builders.add(client().prepareIndex("idx", "field-collapsing", "4").setSource(jsonBuilder()
+        builders.add(client().prepareIndex("field-collapsing", "type", "4").setSource(jsonBuilder()
                 .startObject()
                 .field("group", "b")
                 .field("text", "x y term")
                 .endObject()));
-        builders.add(client().prepareIndex("idx", "field-collapsing", "5").setSource(jsonBuilder()
+        builders.add(client().prepareIndex("field-collapsing", "type", "5").setSource(jsonBuilder()
                 .startObject()
                 .field("group", "b")
                 .field("text", "x term")
                 .endObject()));
-        builders.add(client().prepareIndex("idx", "field-collapsing", "6").setSource(jsonBuilder()
+        builders.add(client().prepareIndex("field-collapsing", "type", "6").setSource(jsonBuilder()
                 .startObject()
                 .field("group", "b")
                 .field("text", "term rare")
                 .field("value", 3)
                 .endObject()));
-        builders.add(client().prepareIndex("idx", "field-collapsing", "7").setSource(jsonBuilder()
+        builders.add(client().prepareIndex("field-collapsing", "type", "7").setSource(jsonBuilder()
                 .startObject()
                 .field("group", "c")
                 .field("text", "x y z term")
                 .endObject()));
-        builders.add(client().prepareIndex("idx", "field-collapsing", "8").setSource(jsonBuilder()
+        builders.add(client().prepareIndex("field-collapsing", "type", "8").setSource(jsonBuilder()
                 .startObject()
                 .field("group", "c")
                 .field("text", "x y term b")
                 .endObject()));
-        builders.add(client().prepareIndex("idx", "field-collapsing", "9").setSource(jsonBuilder()
+        builders.add(client().prepareIndex("field-collapsing", "type", "9").setSource(jsonBuilder()
                 .startObject()
                 .field("group", "c")
                 .field("text", "rare x term")
@@ -268,7 +271,6 @@ private String key(Terms.Bucket bucket) {
     public void testBasics() throws Exception {
         SearchResponse response = client()
                 .prepareSearch("idx")
-                .setTypes("type")
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
                         .field(TERMS_AGGS_FIELD)
@@ -293,22 +295,21 @@ public void testBasics() throws Exception {
             assertThat(bucket.getDocCount(), equalTo(10L));
             TopHits topHits = bucket.getAggregations().get("hits");
             SearchHits hits = topHits.getHits();
-            assertThat(hits.totalHits(), equalTo(10L));
+            assertThat(hits.getTotalHits(), equalTo(10L));
             assertThat(hits.getHits().length, equalTo(3));
             higestSortValue += 10;
-            assertThat((Long) hits.getAt(0).sortValues()[0], equalTo(higestSortValue));
-            assertThat((Long) hits.getAt(1).sortValues()[0], equalTo(higestSortValue - 1));
-            assertThat((Long) hits.getAt(2).sortValues()[0], equalTo(higestSortValue - 2));
+            assertThat((Long) hits.getAt(0).getSortValues()[0], equalTo(higestSortValue));
+            assertThat((Long) hits.getAt(1).getSortValues()[0], equalTo(higestSortValue - 1));
+            assertThat((Long) hits.getAt(2).getSortValues()[0], equalTo(higestSortValue - 2));
 
-            assertThat(hits.getAt(0).sourceAsMap().size(), equalTo(4));
+            assertThat(hits.getAt(0).getSourceAsMap().size(), equalTo(4));
         }
     }
 
     public void testIssue11119() throws Exception {
         // Test that top_hits aggregation is fed scores if query results size=0
         SearchResponse response = client()
-                .prepareSearch("idx")
-                .setTypes("field-collapsing")
+                .prepareSearch("field-collapsing")
                 .setSize(0)
                 .setQuery(matchQuery("text", "x y z"))
                 .addAggregation(terms("terms").executionHint(randomExecutionHint()).field("group").subAggregation(topHits("hits")))
@@ -317,8 +318,8 @@ public void testIssue11119() throws Exception {
         assertSearchResponse(response);
 
         assertThat(response.getHits().getTotalHits(), equalTo(8L));
-        assertThat(response.getHits().hits().length, equalTo(0));
-        assertThat(response.getHits().maxScore(), equalTo(0f));
+        assertThat(response.getHits().getHits().length, equalTo(0));
+        assertThat(response.getHits().getMaxScore(), equalTo(0f));
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
@@ -342,8 +343,7 @@ public void testIssue11119() throws Exception {
         // tied up with the need to feed scores into the agg tree even when
         // users don't want ranked set of query results.)
         response = client()
-                .prepareSearch("idx")
-                .setTypes("field-collapsing")
+                .prepareSearch("field-collapsing")
                 .setSize(0)
                 .setMinScore(0.0001f)
                 .setQuery(matchQuery("text", "x y z"))
@@ -353,8 +353,8 @@ public void testIssue11119() throws Exception {
         assertSearchResponse(response);
 
         assertThat(response.getHits().getTotalHits(), equalTo(8L));
-        assertThat(response.getHits().hits().length, equalTo(0));
-        assertThat(response.getHits().maxScore(), equalTo(0f));
+        assertThat(response.getHits().getHits().length, equalTo(0));
+        assertThat(response.getHits().getMaxScore(), equalTo(0f));
         terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
@@ -363,7 +363,7 @@ public void testIssue11119() throws Exception {
 
 
     public void testBreadthFirstWithScoreNeeded() throws Exception {
-        SearchResponse response = client().prepareSearch("idx").setTypes("type")
+        SearchResponse response = client().prepareSearch("idx")
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
                         .collectMode(SubAggCollectionMode.BREADTH_FIRST)
@@ -385,15 +385,15 @@ public void testBreadthFirstWithScoreNeeded() throws Exception {
             assertThat(bucket.getDocCount(), equalTo(10L));
             TopHits topHits = bucket.getAggregations().get("hits");
             SearchHits hits = topHits.getHits();
-            assertThat(hits.totalHits(), equalTo(10L));
+            assertThat(hits.getTotalHits(), equalTo(10L));
             assertThat(hits.getHits().length, equalTo(3));
 
-            assertThat(hits.getAt(0).sourceAsMap().size(), equalTo(4));
+            assertThat(hits.getAt(0).getSourceAsMap().size(), equalTo(4));
         }
     }
 
     public void testBreadthFirstWithAggOrderAndScoreNeeded() throws Exception {
-        SearchResponse response = client().prepareSearch("idx").setTypes("type")
+        SearchResponse response = client().prepareSearch("idx")
             .addAggregation(terms("terms")
                 .executionHint(randomExecutionHint())
                 .collectMode(SubAggCollectionMode.BREADTH_FIRST)
@@ -416,11 +416,11 @@ public void testBreadthFirstWithAggOrderAndScoreNeeded() throws Exception {
             assertThat(bucket.getDocCount(), equalTo(10L));
             TopHits topHits = bucket.getAggregations().get("hits");
             SearchHits hits = topHits.getHits();
-            assertThat(hits.totalHits(), equalTo(10L));
+            assertThat(hits.getTotalHits(), equalTo(10L));
             assertThat(hits.getHits().length, equalTo(3));
 
-            assertThat(hits.getAt(0).sourceAsMap().size(), equalTo(4));
-            id --;
+            assertThat(hits.getAt(0).getSourceAsMap().size(), equalTo(4));
+            id--;
         }
     }
 
@@ -439,19 +439,20 @@ public void testBasicsGetProperty() throws Exception {
         TopHits topHits = global.getAggregations().get("hits");
         assertThat(topHits, notNullValue());
         assertThat(topHits.getName(), equalTo("hits"));
-        assertThat((TopHits) global.getProperty("hits"), sameInstance(topHits));
+        assertThat((TopHits) ((InternalAggregation)global).getProperty("hits"), sameInstance(topHits));
 
     }
 
     public void testPagination() throws Exception {
         int size = randomIntBetween(1, 10);
         int from = randomIntBetween(0, 10);
-        SearchResponse response = client().prepareSearch("idx").setTypes("type")
+        SearchResponse response = client().prepareSearch("idx")
                 .addAggregation(terms("terms")
                                 .executionHint(randomExecutionHint())
                                 .field(TERMS_AGGS_FIELD)
                                 .subAggregation(
-                                        topHits("hits").sort(SortBuilders.fieldSort(SORT_FIELD).order(SortOrder.DESC))
+                                        topHits("hits")
+                                            .sort(SortBuilders.fieldSort(SORT_FIELD).order(SortOrder.DESC))
                                                 .from(from)
                                                 .size(size)
                                 )
@@ -460,7 +461,6 @@ public void testPagination() throws Exception {
         assertSearchResponse(response);
 
         SearchResponse control = client().prepareSearch("idx")
-                .setTypes("type")
                 .setFrom(from)
                 .setSize(size)
                 .setPostFilter(QueryBuilders.termQuery(TERMS_AGGS_FIELD, "val0"))
@@ -479,17 +479,18 @@ public void testPagination() throws Exception {
         assertThat(bucket.getDocCount(), equalTo(10L));
         TopHits topHits = bucket.getAggregations().get("hits");
         SearchHits hits = topHits.getHits();
-        assertThat(hits.totalHits(), equalTo(controlHits.totalHits()));
+        assertThat(hits.getTotalHits(), equalTo(controlHits.getTotalHits()));
         assertThat(hits.getHits().length, equalTo(controlHits.getHits().length));
         for (int i = 0; i < hits.getHits().length; i++) {
-            logger.info("{}: top_hits: [{}][{}] control: [{}][{}]", i, hits.getAt(i).id(), hits.getAt(i).sortValues()[0], controlHits.getAt(i).id(), controlHits.getAt(i).sortValues()[0]);
-            assertThat(hits.getAt(i).id(), equalTo(controlHits.getAt(i).id()));
-            assertThat(hits.getAt(i).sortValues()[0], equalTo(controlHits.getAt(i).sortValues()[0]));
+            logger.info("{}: top_hits: [{}][{}] control: [{}][{}]", i, hits.getAt(i).getId(), hits.getAt(i).getSortValues()[0],
+                controlHits.getAt(i).getId(), controlHits.getAt(i).getSortValues()[0]);
+            assertThat(hits.getAt(i).getId(), equalTo(controlHits.getAt(i).getId()));
+            assertThat(hits.getAt(i).getSortValues()[0], equalTo(controlHits.getAt(i).getSortValues()[0]));
         }
     }
 
     public void testSortByBucket() throws Exception {
-        SearchResponse response = client().prepareSearch("idx").setTypes("type")
+        SearchResponse response = client().prepareSearch("idx")
                 .addAggregation(terms("terms")
                                 .executionHint(randomExecutionHint())
                                 .field(TERMS_AGGS_FIELD)
@@ -516,11 +517,11 @@ public void testSortByBucket() throws Exception {
             assertThat(bucket.getDocCount(), equalTo(10L));
             TopHits topHits = bucket.getAggregations().get("hits");
             SearchHits hits = topHits.getHits();
-            assertThat(hits.totalHits(), equalTo(10L));
+            assertThat(hits.getTotalHits(), equalTo(10L));
             assertThat(hits.getHits().length, equalTo(3));
-            assertThat((Long) hits.getAt(0).sortValues()[0], equalTo(higestSortValue));
-            assertThat((Long) hits.getAt(1).sortValues()[0], equalTo(higestSortValue - 1));
-            assertThat((Long) hits.getAt(2).sortValues()[0], equalTo(higestSortValue - 2));
+            assertThat(hits.getAt(0).getSortValues()[0], equalTo(higestSortValue));
+            assertThat(hits.getAt(1).getSortValues()[0], equalTo(higestSortValue - 1));
+            assertThat(hits.getAt(2).getSortValues()[0], equalTo(higestSortValue - 2));
             Max max = bucket.getAggregations().get("max_sort");
             assertThat(max.getValue(), equalTo(((Long) higestSortValue).doubleValue()));
             higestSortValue -= 10;
@@ -529,8 +530,7 @@ public void testSortByBucket() throws Exception {
 
     public void testFieldCollapsing() throws Exception {
         SearchResponse response = client()
-                .prepareSearch("idx")
-                .setTypes("field-collapsing")
+                .prepareSearch("field-collapsing")
                 .setSearchType(SearchType.DFS_QUERY_THEN_FETCH)
                 .setQuery(matchQuery("text", "term rare"))
                 .addAggregation(
@@ -544,34 +544,34 @@ public void testFieldCollapsing() throws Exception {
         assertThat(terms.getName(), equalTo("terms"));
         assertThat(terms.getBuckets().size(), equalTo(3));
 
-        Iterator<Terms.Bucket> bucketIterator = terms.getBuckets().iterator();
+        Iterator<? extends Terms.Bucket> bucketIterator = terms.getBuckets().iterator();
         Terms.Bucket bucket = bucketIterator.next();
         assertThat(key(bucket), equalTo("b"));
         TopHits topHits = bucket.getAggregations().get("hits");
         SearchHits hits = topHits.getHits();
-        assertThat(hits.totalHits(), equalTo(4L));
+        assertThat(hits.getTotalHits(), equalTo(4L));
         assertThat(hits.getHits().length, equalTo(1));
-        assertThat(hits.getAt(0).id(), equalTo("6"));
+        assertThat(hits.getAt(0).getId(), equalTo("6"));
 
         bucket = bucketIterator.next();
         assertThat(key(bucket), equalTo("c"));
         topHits = bucket.getAggregations().get("hits");
         hits = topHits.getHits();
-        assertThat(hits.totalHits(), equalTo(3L));
+        assertThat(hits.getTotalHits(), equalTo(3L));
         assertThat(hits.getHits().length, equalTo(1));
-        assertThat(hits.getAt(0).id(), equalTo("9"));
+        assertThat(hits.getAt(0).getId(), equalTo("9"));
 
         bucket = bucketIterator.next();
         assertThat(key(bucket), equalTo("a"));
         topHits = bucket.getAggregations().get("hits");
         hits = topHits.getHits();
-        assertThat(hits.totalHits(), equalTo(2L));
+        assertThat(hits.getTotalHits(), equalTo(2L));
         assertThat(hits.getHits().length, equalTo(1));
-        assertThat(hits.getAt(0).id(), equalTo("2"));
+        assertThat(hits.getAt(0).getId(), equalTo("2"));
     }
 
     public void testFetchFeatures() {
-        SearchResponse response = client().prepareSearch("idx").setTypes("type")
+        SearchResponse response = client().prepareSearch("idx")
                 .setQuery(matchQuery("text", "text").queryName("test"))
                 .addAggregation(terms("terms")
                                 .executionHint(randomExecutionHint())
@@ -582,7 +582,7 @@ public void testFetchFeatures() {
                                             .explain(true)
                                             .storedField("text")
                                             .fieldDataField("field1")
-                                            .scriptField("script", new Script("5", ScriptService.ScriptType.INLINE, MockScriptEngine.NAME, Collections.emptyMap()))
+                                            .scriptField("script", new Script(ScriptType.INLINE, MockScriptEngine.NAME, "5", Collections.emptyMap()))
                                             .fetchSource("text", null)
                                             .version(true)
                                 )
@@ -598,7 +598,7 @@ public void testFetchFeatures() {
         for (Terms.Bucket bucket : terms.getBuckets()) {
             TopHits topHits = bucket.getAggregations().get("hits");
             SearchHits hits = topHits.getHits();
-            assertThat(hits.totalHits(), equalTo(10L));
+            assertThat(hits.getTotalHits(), equalTo(10L));
             assertThat(hits.getHits().length, equalTo(1));
 
             SearchHit hit = hits.getAt(0);
@@ -606,30 +606,30 @@ public void testFetchFeatures() {
             assertThat(highlightField.getFragments().length, equalTo(1));
             assertThat(highlightField.getFragments()[0].string(), equalTo("some <em>text</em> to entertain"));
 
-            Explanation explanation = hit.explanation();
+            Explanation explanation = hit.getExplanation();
             assertThat(explanation.toString(), containsString("text:text"));
 
-            long version = hit.version();
+            long version = hit.getVersion();
             assertThat(version, equalTo(1L));
 
-            assertThat(hit.matchedQueries()[0], equalTo("test"));
+            assertThat(hit.getMatchedQueries()[0], equalTo("test"));
 
             SearchHitField field = hit.field("field1");
             assertThat(field.getValue().toString(), equalTo("5"));
 
-            assertThat(hit.getSource().get("text").toString(), equalTo("some text to entertain"));
+            assertThat(hit.getSourceAsMap().get("text").toString(), equalTo("some text to entertain"));
 
             field = hit.field("script");
             assertThat(field.getValue().toString(), equalTo("5"));
 
-            assertThat(hit.sourceAsMap().size(), equalTo(1));
-            assertThat(hit.sourceAsMap().get("text").toString(), equalTo("some text to entertain"));
+            assertThat(hit.getSourceAsMap().size(), equalTo(1));
+            assertThat(hit.getSourceAsMap().get("text").toString(), equalTo("some text to entertain"));
         }
     }
 
     public void testInvalidSortField() throws Exception {
         try {
-            client().prepareSearch("idx").setTypes("type")
+            client().prepareSearch("idx")
                     .addAggregation(terms("terms")
                                     .executionHint(randomExecutionHint())
                                     .field(TERMS_AGGS_FIELD)
@@ -644,7 +644,7 @@ public void testInvalidSortField() throws Exception {
     }
 
     public void testEmptyIndex() throws Exception {
-        SearchResponse response = client().prepareSearch("empty").setTypes("type")
+        SearchResponse response = client().prepareSearch("empty")
                 .addAggregation(topHits("hits"))
                 .get();
         assertSearchResponse(response);
@@ -652,14 +652,14 @@ public void testEmptyIndex() throws Exception {
         TopHits hits = response.getAggregations().get("hits");
         assertThat(hits, notNullValue());
         assertThat(hits.getName(), equalTo("hits"));
-        assertThat(hits.getHits().totalHits(), equalTo(0L));
+        assertThat(hits.getHits().getTotalHits(), equalTo(0L));
     }
 
     public void testTrackScores() throws Exception {
         boolean[] trackScores = new boolean[]{true, false};
         for (boolean trackScore : trackScores) {
             logger.info("Track score={}", trackScore);
-            SearchResponse response = client().prepareSearch("idx").setTypes("field-collapsing")
+            SearchResponse response = client().prepareSearch("field-collapsing")
                     .setQuery(matchQuery("text", "term rare"))
                     .addAggregation(terms("terms")
                                     .field("group")
@@ -667,7 +667,7 @@ public void testTrackScores() throws Exception {
                                             topHits("hits")
                                                     .trackScores(trackScore)
                                                     .size(1)
-                                                    .sort("_uid", SortOrder.DESC)
+                                                    .sort("_index", SortOrder.DESC)
                                     )
                     )
                     .get();
@@ -683,21 +683,21 @@ public void testTrackScores() throws Exception {
             TopHits topHits = bucket.getAggregations().get("hits");
             SearchHits hits = topHits.getHits();
             assertThat(hits.getMaxScore(), trackScore ? not(equalTo(Float.NaN)) : equalTo(Float.NaN));
-            assertThat(hits.getAt(0).score(), trackScore ? not(equalTo(Float.NaN)) : equalTo(Float.NaN));
+            assertThat(hits.getAt(0).getScore(), trackScore ? not(equalTo(Float.NaN)) : equalTo(Float.NaN));
 
             bucket = terms.getBucketByKey("b");
             assertThat(key(bucket), equalTo("b"));
             topHits = bucket.getAggregations().get("hits");
             hits = topHits.getHits();
             assertThat(hits.getMaxScore(), trackScore ? not(equalTo(Float.NaN)) : equalTo(Float.NaN));
-            assertThat(hits.getAt(0).score(), trackScore ? not(equalTo(Float.NaN)) : equalTo(Float.NaN));
+            assertThat(hits.getAt(0).getScore(), trackScore ? not(equalTo(Float.NaN)) : equalTo(Float.NaN));
 
             bucket = terms.getBucketByKey("c");
             assertThat(key(bucket), equalTo("c"));
             topHits = bucket.getAggregations().get("hits");
             hits = topHits.getHits();
             assertThat(hits.getMaxScore(), trackScore ? not(equalTo(Float.NaN)) : equalTo(Float.NaN));
-            assertThat(hits.getAt(0).score(), trackScore ? not(equalTo(Float.NaN)) : equalTo(Float.NaN));
+            assertThat(hits.getAt(0).getScore(), trackScore ? not(equalTo(Float.NaN)) : equalTo(Float.NaN));
         }
     }
 
@@ -724,31 +724,31 @@ public void testTopHitsInNestedSimple() throws Exception {
         assertThat(bucket.getDocCount(), equalTo(1L));
         TopHits topHits = bucket.getAggregations().get("top-comments");
         SearchHits searchHits = topHits.getHits();
-        assertThat(searchHits.totalHits(), equalTo(1L));
+        assertThat(searchHits.getTotalHits(), equalTo(1L));
         assertThat(searchHits.getAt(0).getNestedIdentity().getField().string(), equalTo("comments"));
         assertThat(searchHits.getAt(0).getNestedIdentity().getOffset(), equalTo(0));
-        assertThat((Integer) searchHits.getAt(0).getSource().get("date"), equalTo(1));
+        assertThat((Integer) searchHits.getAt(0).getSourceAsMap().get("date"), equalTo(1));
 
         bucket = terms.getBucketByKey("b");
         assertThat(bucket.getDocCount(), equalTo(2L));
         topHits = bucket.getAggregations().get("top-comments");
         searchHits = topHits.getHits();
-        assertThat(searchHits.totalHits(), equalTo(2L));
+        assertThat(searchHits.getTotalHits(), equalTo(2L));
         assertThat(searchHits.getAt(0).getNestedIdentity().getField().string(), equalTo("comments"));
         assertThat(searchHits.getAt(0).getNestedIdentity().getOffset(), equalTo(1));
-        assertThat((Integer) searchHits.getAt(0).getSource().get("date"), equalTo(2));
+        assertThat((Integer) searchHits.getAt(0).getSourceAsMap().get("date"), equalTo(2));
         assertThat(searchHits.getAt(1).getNestedIdentity().getField().string(), equalTo("comments"));
         assertThat(searchHits.getAt(1).getNestedIdentity().getOffset(), equalTo(0));
-        assertThat((Integer) searchHits.getAt(1).getSource().get("date"), equalTo(3));
+        assertThat((Integer) searchHits.getAt(1).getSourceAsMap().get("date"), equalTo(3));
 
         bucket = terms.getBucketByKey("c");
         assertThat(bucket.getDocCount(), equalTo(1L));
         topHits = bucket.getAggregations().get("top-comments");
         searchHits = topHits.getHits();
-        assertThat(searchHits.totalHits(), equalTo(1L));
+        assertThat(searchHits.getTotalHits(), equalTo(1L));
         assertThat(searchHits.getAt(0).getNestedIdentity().getField().string(), equalTo("comments"));
         assertThat(searchHits.getAt(0).getNestedIdentity().getOffset(), equalTo(1));
-        assertThat((Integer) searchHits.getAt(0).getSource().get("date"), equalTo(4));
+        assertThat((Integer) searchHits.getAt(0).getSourceAsMap().get("date"), equalTo(4));
     }
 
     public void testTopHitsInSecondLayerNested() throws Exception {
@@ -770,7 +770,7 @@ public void testTopHitsInSecondLayerNested() throws Exception {
         assertThat(toComments.getDocCount(), equalTo(4L));
 
         TopHits topComments = toComments.getAggregations().get("top-comments");
-        assertThat(topComments.getHits().totalHits(), equalTo(4L));
+        assertThat(topComments.getHits().getTotalHits(), equalTo(4L));
         assertThat(topComments.getHits().getHits().length, equalTo(4));
 
         assertThat(topComments.getHits().getAt(0).getId(), equalTo("2"));
@@ -797,53 +797,53 @@ public void testTopHitsInSecondLayerNested() throws Exception {
         assertThat(toReviewers.getDocCount(), equalTo(7L));
 
         TopHits topReviewers = toReviewers.getAggregations().get("top-reviewers");
-        assertThat(topReviewers.getHits().totalHits(), equalTo(7L));
+        assertThat(topReviewers.getHits().getTotalHits(), equalTo(7L));
         assertThat(topReviewers.getHits().getHits().length, equalTo(7));
 
         assertThat(topReviewers.getHits().getAt(0).getId(), equalTo("1"));
-        assertThat((String) topReviewers.getHits().getAt(0).sourceAsMap().get("name"), equalTo("user a"));
+        assertThat((String) topReviewers.getHits().getAt(0).getSourceAsMap().get("name"), equalTo("user a"));
         assertThat(topReviewers.getHits().getAt(0).getNestedIdentity().getField().string(), equalTo("comments"));
         assertThat(topReviewers.getHits().getAt(0).getNestedIdentity().getOffset(), equalTo(0));
         assertThat(topReviewers.getHits().getAt(0).getNestedIdentity().getChild().getField().string(), equalTo("reviewers"));
         assertThat(topReviewers.getHits().getAt(0).getNestedIdentity().getChild().getOffset(), equalTo(0));
 
         assertThat(topReviewers.getHits().getAt(1).getId(), equalTo("1"));
-        assertThat((String) topReviewers.getHits().getAt(1).sourceAsMap().get("name"), equalTo("user b"));
+        assertThat((String) topReviewers.getHits().getAt(1).getSourceAsMap().get("name"), equalTo("user b"));
         assertThat(topReviewers.getHits().getAt(1).getNestedIdentity().getField().string(), equalTo("comments"));
         assertThat(topReviewers.getHits().getAt(1).getNestedIdentity().getOffset(), equalTo(0));
         assertThat(topReviewers.getHits().getAt(1).getNestedIdentity().getChild().getField().string(), equalTo("reviewers"));
         assertThat(topReviewers.getHits().getAt(1).getNestedIdentity().getChild().getOffset(), equalTo(1));
 
         assertThat(topReviewers.getHits().getAt(2).getId(), equalTo("1"));
-        assertThat((String) topReviewers.getHits().getAt(2).sourceAsMap().get("name"), equalTo("user c"));
+        assertThat((String) topReviewers.getHits().getAt(2).getSourceAsMap().get("name"), equalTo("user c"));
         assertThat(topReviewers.getHits().getAt(2).getNestedIdentity().getField().string(), equalTo("comments"));
         assertThat(topReviewers.getHits().getAt(2).getNestedIdentity().getOffset(), equalTo(0));
         assertThat(topReviewers.getHits().getAt(2).getNestedIdentity().getChild().getField().string(), equalTo("reviewers"));
         assertThat(topReviewers.getHits().getAt(2).getNestedIdentity().getChild().getOffset(), equalTo(2));
 
         assertThat(topReviewers.getHits().getAt(3).getId(), equalTo("1"));
-        assertThat((String) topReviewers.getHits().getAt(3).sourceAsMap().get("name"), equalTo("user c"));
+        assertThat((String) topReviewers.getHits().getAt(3).getSourceAsMap().get("name"), equalTo("user c"));
         assertThat(topReviewers.getHits().getAt(3).getNestedIdentity().getField().string(), equalTo("comments"));
         assertThat(topReviewers.getHits().getAt(3).getNestedIdentity().getOffset(), equalTo(1));
         assertThat(topReviewers.getHits().getAt(3).getNestedIdentity().getChild().getField().string(), equalTo("reviewers"));
         assertThat(topReviewers.getHits().getAt(3).getNestedIdentity().getChild().getOffset(), equalTo(0));
 
         assertThat(topReviewers.getHits().getAt(4).getId(), equalTo("1"));
-        assertThat((String) topReviewers.getHits().getAt(4).sourceAsMap().get("name"), equalTo("user d"));
+        assertThat((String) topReviewers.getHits().getAt(4).getSourceAsMap().get("name"), equalTo("user d"));
         assertThat(topReviewers.getHits().getAt(4).getNestedIdentity().getField().string(), equalTo("comments"));
         assertThat(topReviewers.getHits().getAt(4).getNestedIdentity().getOffset(), equalTo(1));
         assertThat(topReviewers.getHits().getAt(4).getNestedIdentity().getChild().getField().string(), equalTo("reviewers"));
         assertThat(topReviewers.getHits().getAt(4).getNestedIdentity().getChild().getOffset(), equalTo(1));
 
         assertThat(topReviewers.getHits().getAt(5).getId(), equalTo("1"));
-        assertThat((String) topReviewers.getHits().getAt(5).sourceAsMap().get("name"), equalTo("user e"));
+        assertThat((String) topReviewers.getHits().getAt(5).getSourceAsMap().get("name"), equalTo("user e"));
         assertThat(topReviewers.getHits().getAt(5).getNestedIdentity().getField().string(), equalTo("comments"));
         assertThat(topReviewers.getHits().getAt(5).getNestedIdentity().getOffset(), equalTo(1));
         assertThat(topReviewers.getHits().getAt(5).getNestedIdentity().getChild().getField().string(), equalTo("reviewers"));
         assertThat(topReviewers.getHits().getAt(5).getNestedIdentity().getChild().getOffset(), equalTo(2));
 
         assertThat(topReviewers.getHits().getAt(6).getId(), equalTo("2"));
-        assertThat((String) topReviewers.getHits().getAt(6).sourceAsMap().get("name"), equalTo("user f"));
+        assertThat((String) topReviewers.getHits().getAt(6).getSourceAsMap().get("name"), equalTo("user f"));
         assertThat(topReviewers.getHits().getAt(0).getNestedIdentity().getField().string(), equalTo("comments"));
         assertThat(topReviewers.getHits().getAt(0).getNestedIdentity().getOffset(), equalTo(0));
         assertThat(topReviewers.getHits().getAt(0).getNestedIdentity().getChild().getField().string(), equalTo("reviewers"));
@@ -864,14 +864,14 @@ public void testNestedFetchFeatures() {
                         nested("to-comments", "comments").subAggregation(
                                 topHits("top-comments").size(1).highlighter(new HighlightBuilder().field(hlField)).explain(true)
                                                 .fieldDataField("comments.user")
-                                        .scriptField("script", new Script("5", ScriptService.ScriptType.INLINE, MockScriptEngine.NAME, Collections.emptyMap())).fetchSource("comments.message", null)
+                                        .scriptField("script", new Script(ScriptType.INLINE, MockScriptEngine.NAME, "5", Collections.emptyMap())).fetchSource("comments.message", null)
                                         .version(true).sort("comments.date", SortOrder.ASC))).get();
         assertHitCount(searchResponse, 2);
         Nested nested = searchResponse.getAggregations().get("to-comments");
         assertThat(nested.getDocCount(), equalTo(4L));
 
         SearchHits hits = ((TopHits) nested.getAggregations().get("top-comments")).getHits();
-        assertThat(hits.totalHits(), equalTo(4L));
+        assertThat(hits.getTotalHits(), equalTo(4L));
         SearchHit searchHit = hits.getAt(0);
         assertThat(searchHit.getId(), equalTo("1"));
         assertThat(searchHit.getNestedIdentity().getField().string(), equalTo("comments"));
@@ -883,14 +883,14 @@ public void testNestedFetchFeatures() {
 
         // Can't explain nested hit with the main query, since both are in a different scopes, also the nested doc may not even have matched with the main query
         // If top_hits would have a query option then we can explain that query
-        Explanation explanation = searchHit.explanation();
+        Explanation explanation = searchHit.getExplanation();
         assertFalse(explanation.isMatch());
 
         // Returns the version of the root document. Nested docs don't have a separate version
-        long version = searchHit.version();
+        long version = searchHit.getVersion();
         assertThat(version, equalTo(1L));
 
-        assertThat(searchHit.matchedQueries(), arrayContaining("test"));
+        assertThat(searchHit.getMatchedQueries(), arrayContaining("test"));
 
         SearchHitField field = searchHit.field("comments.user");
         assertThat(field.getValue().toString(), equalTo("a"));
@@ -898,8 +898,8 @@ public void testNestedFetchFeatures() {
         field = searchHit.field("script");
         assertThat(field.getValue().toString(), equalTo("5"));
 
-        assertThat(searchHit.sourceAsMap().size(), equalTo(1));
-        assertThat(XContentMapValues.extractValue("comments.message", searchHit.sourceAsMap()), equalTo("some comment"));
+        assertThat(searchHit.getSourceAsMap().size(), equalTo(1));
+        assertThat(XContentMapValues.extractValue("comments.message", searchHit.getSourceAsMap()), equalTo("some comment"));
     }
 
     public void testTopHitsInNested() throws Exception {
@@ -929,11 +929,11 @@ public void testTopHitsInNested() throws Exception {
 
             TopHits hits = nested.getAggregations().get("comments");
             SearchHits searchHits = hits.getHits();
-            assertThat(searchHits.totalHits(), equalTo(numNestedDocs));
+            assertThat(searchHits.getTotalHits(), equalTo(numNestedDocs));
             for (int j = 0; j < 3; j++) {
                 assertThat(searchHits.getAt(j).getNestedIdentity().getField().string(), equalTo("comments"));
                 assertThat(searchHits.getAt(j).getNestedIdentity().getOffset(), equalTo(0));
-                assertThat((Integer) searchHits.getAt(j).sourceAsMap().get("id"), equalTo(0));
+                assertThat((Integer) searchHits.getAt(j).getSourceAsMap().get("id"), equalTo(0));
 
                 HighlightField highlightField = searchHits.getAt(j).getHighlightFields().get("comments.message");
                 assertThat(highlightField.getFragments().length, equalTo(1));
@@ -945,7 +945,6 @@ public void testTopHitsInNested() throws Exception {
     public void testDontExplode() throws Exception {
         SearchResponse response = client()
                 .prepareSearch("idx")
-                .setTypes("type")
                 .addAggregation(terms("terms")
                                 .executionHint(randomExecutionHint())
                                 .field(TERMS_AGGS_FIELD)
@@ -960,7 +959,6 @@ public void testDontExplode() throws Exception {
     public void testNoStoredFields() throws Exception {
         SearchResponse response = client()
             .prepareSearch("idx")
-            .setTypes("type")
             .addAggregation(terms("terms")
                 .executionHint(randomExecutionHint())
                 .field(TERMS_AGGS_FIELD)
@@ -984,12 +982,160 @@ public void testNoStoredFields() throws Exception {
             assertThat(bucket.getDocCount(), equalTo(10L));
             TopHits topHits = bucket.getAggregations().get("hits");
             SearchHits hits = topHits.getHits();
-            assertThat(hits.totalHits(), equalTo(10L));
+            assertThat(hits.getTotalHits(), equalTo(10L));
             assertThat(hits.getHits().length, equalTo(3));
             for (SearchHit hit : hits) {
-                assertThat(hit.source(), nullValue());
-                assertThat(hit.id(), nullValue());
-                assertThat(hit.type(), nullValue());
+                assertThat(hit.getSourceAsMap(), nullValue());
+                assertThat(hit.getId(), nullValue());
+                assertThat(hit.getType(), nullValue());
+            }
+        }
+    }
+
+    /**
+     * Make sure that a request using a script does not get cached and a request
+     * not using a script does get cached.
+     */
+    public void testDontCacheScripts() throws Exception {
+        try {
+            assertAcked(prepareCreate("cache_test_idx").addMapping("type", "d", "type=long")
+                .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1))
+                .get());
+            indexRandom(true, client().prepareIndex("cache_test_idx", "type", "1").setSource("s", 1),
+                client().prepareIndex("cache_test_idx", "type", "2").setSource("s", 2));
+
+            // Make sure we are starting with a clear cache
+            assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+            assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(0L));
+
+            // Test that a request using a script field does not get cached
+            SearchResponse r = client().prepareSearch("cache_test_idx").setSize(0)
+                .addAggregation(topHits("foo").scriptField("bar",
+                    new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "5", Collections.emptyMap()))).get();
+            assertSearchResponse(r);
+
+            assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+            assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(0L));
+
+            // Test that a request using a script sort does not get cached
+            r = client().prepareSearch("cache_test_idx").setSize(0)
+                .addAggregation(topHits("foo").sort(
+                    SortBuilders.scriptSort(
+                        new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "5", Collections.emptyMap()), ScriptSortType.STRING)))
+                .get();
+            assertSearchResponse(r);
+
+            assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+            assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(0L));
+
+            // To make sure that the cache is working test that a request not using
+            // a script is cached
+            r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(topHits("foo")).get();
+            assertSearchResponse(r);
+
+            assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+            assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(1L));
+        } finally {
+            assertAcked(client().admin().indices().prepareDelete("cache_test_idx")); // delete this - if we use tests.iters it would fail
+        }
+    }
+
+    public void testWithRescore() {
+        // Rescore with default sort on relevancy (score)
+        {
+            SearchResponse response = client()
+                .prepareSearch("idx")
+                .addRescorer(
+                    RescoreBuilder.queryRescorer(new MatchAllQueryBuilder().boost(3.0f))
+                )
+                .addAggregation(terms("terms")
+                    .field(TERMS_AGGS_FIELD)
+                    .subAggregation(
+                        topHits("hits")
+                    )
+                )
+                .get();
+            Terms terms = response.getAggregations().get("terms");
+            for (Terms.Bucket bucket : terms.getBuckets()) {
+                TopHits topHits = bucket.getAggregations().get("hits");
+                for (SearchHit hit : topHits.getHits().getHits()) {
+                    assertThat(hit.getScore(), equalTo(4.0f));
+                }
+            }
+        }
+
+        {
+            SearchResponse response = client()
+                .prepareSearch("idx")
+                .addRescorer(
+                    RescoreBuilder.queryRescorer(new MatchAllQueryBuilder().boost(3.0f))
+                )
+                .addAggregation(terms("terms")
+                    .field(TERMS_AGGS_FIELD)
+                    .subAggregation(
+                        topHits("hits").sort(SortBuilders.scoreSort())
+                    )
+                )
+                .get();
+            Terms terms = response.getAggregations().get("terms");
+            for (Terms.Bucket bucket : terms.getBuckets()) {
+                TopHits topHits = bucket.getAggregations().get("hits");
+                for (SearchHit hit : topHits.getHits().getHits()) {
+                    assertThat(hit.getScore(), equalTo(4.0f));
+                }
+            }
+        }
+
+        // Rescore should not be applied if the sort order is not relevancy
+        {
+            SearchResponse response = client()
+                .prepareSearch("idx")
+                .addRescorer(
+                    RescoreBuilder.queryRescorer(new MatchAllQueryBuilder().boost(3.0f))
+                )
+                .addAggregation(terms("terms")
+                    .field(TERMS_AGGS_FIELD)
+                    .subAggregation(
+                        topHits("hits").sort(SortBuilders.fieldSort("_type"))
+                    )
+                )
+                .get();
+            Terms terms = response.getAggregations().get("terms");
+            for (Terms.Bucket bucket : terms.getBuckets()) {
+                TopHits topHits = bucket.getAggregations().get("hits");
+                for (SearchHit hit : topHits.getHits().getHits()) {
+                    assertThat(hit.getScore(), equalTo(Float.NaN));
+                }
+            }
+        }
+
+        {
+            SearchResponse response = client()
+                .prepareSearch("idx")
+                .addRescorer(
+                    RescoreBuilder.queryRescorer(new MatchAllQueryBuilder().boost(3.0f))
+                )
+                .addAggregation(terms("terms")
+                    .field(TERMS_AGGS_FIELD)
+                    .subAggregation(
+                        topHits("hits").sort(SortBuilders.scoreSort()).sort(SortBuilders.fieldSort("_type"))
+                    )
+                )
+                .get();
+            Terms terms = response.getAggregations().get("terms");
+            for (Terms.Bucket bucket : terms.getBuckets()) {
+                TopHits topHits = bucket.getAggregations().get("hits");
+                for (SearchHit hit : topHits.getHits().getHits()) {
+                    assertThat(hit.getScore(), equalTo(Float.NaN));
+                }
             }
         }
     }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsTests.java
index 3f2b4c44620ad..9e7ae8361768c 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsTests.java
@@ -19,12 +19,12 @@
 
 package org.elasticsearch.search.aggregations.metrics;
 
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.test.AbstractQueryTestCase;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.aggregations.AggregationInitializationException;
+import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
 import org.elasticsearch.search.aggregations.metrics.tophits.TopHitsAggregationBuilder;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
@@ -32,6 +32,7 @@
 import org.elasticsearch.search.sort.ScriptSortBuilder.ScriptSortType;
 import org.elasticsearch.search.sort.SortBuilders;
 import org.elasticsearch.search.sort.SortOrder;
+import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -72,7 +73,7 @@ protected final TopHitsAggregationBuilder createTestAggregatorBuilder() {
                 int fieldsSize = randomInt(25);
                 List<String> fields = new ArrayList<>(fieldsSize);
                 for (int i = 0; i < fieldsSize; i++) {
-                    fields.add(randomAsciiOfLengthBetween(5, 50));
+                    fields.add(randomAlphaOfLengthBetween(5, 50));
                 }
                 factory.storedFields(fields);
                 break;
@@ -82,16 +83,16 @@ protected final TopHitsAggregationBuilder createTestAggregatorBuilder() {
         if (randomBoolean()) {
             int fieldDataFieldsSize = randomInt(25);
             for (int i = 0; i < fieldDataFieldsSize; i++) {
-                factory.fieldDataField(randomAsciiOfLengthBetween(5, 50));
+                factory.fieldDataField(randomAlphaOfLengthBetween(5, 50));
             }
         }
         if (randomBoolean()) {
             int scriptFieldsSize = randomInt(25);
             for (int i = 0; i < scriptFieldsSize; i++) {
                 if (randomBoolean()) {
-                    factory.scriptField(randomAsciiOfLengthBetween(5, 50), new Script("foo"), randomBoolean());
+                    factory.scriptField(randomAlphaOfLengthBetween(5, 50), mockScript("foo"), randomBoolean());
                 } else {
-                    factory.scriptField(randomAsciiOfLengthBetween(5, 50), new Script("foo"));
+                    factory.scriptField(randomAlphaOfLengthBetween(5, 50), mockScript("foo"));
                 }
             }
         }
@@ -100,33 +101,34 @@ protected final TopHitsAggregationBuilder createTestAggregatorBuilder() {
             int branch = randomInt(5);
             String[] includes = new String[randomIntBetween(0, 20)];
             for (int i = 0; i < includes.length; i++) {
-                includes[i] = randomAsciiOfLengthBetween(5, 20);
+                includes[i] = randomAlphaOfLengthBetween(5, 20);
             }
             String[] excludes = new String[randomIntBetween(0, 20)];
             for (int i = 0; i < excludes.length; i++) {
-                excludes[i] = randomAsciiOfLengthBetween(5, 20);
+                excludes[i] = randomAlphaOfLengthBetween(5, 20);
             }
             switch (branch) {
-            case 0:
-                fetchSourceContext = new FetchSourceContext(randomBoolean());
-                break;
-            case 1:
-                fetchSourceContext = new FetchSourceContext(includes, excludes);
-                break;
-            case 2:
-                fetchSourceContext = new FetchSourceContext(randomAsciiOfLengthBetween(5, 20), randomAsciiOfLengthBetween(5, 20));
-                break;
-            case 3:
-                fetchSourceContext = new FetchSourceContext(true, includes, excludes);
-                break;
-            case 4:
-                fetchSourceContext = new FetchSourceContext(includes);
-                break;
-            case 5:
-                fetchSourceContext = new FetchSourceContext(randomAsciiOfLengthBetween(5, 20));
-                break;
-            default:
-                throw new IllegalStateException();
+                case 0:
+                    fetchSourceContext = new FetchSourceContext(randomBoolean());
+                    break;
+                case 1:
+                    fetchSourceContext = new FetchSourceContext(true, includes, excludes);
+                    break;
+                case 2:
+                    fetchSourceContext = new FetchSourceContext(true, new String[]{randomAlphaOfLengthBetween(5, 20)},
+                        new String[]{randomAlphaOfLengthBetween(5, 20)});
+                    break;
+                case 3:
+                    fetchSourceContext = new FetchSourceContext(true, includes, excludes);
+                    break;
+                case 4:
+                    fetchSourceContext = new FetchSourceContext(true, includes, null);
+                    break;
+                case 5:
+                    fetchSourceContext = new FetchSourceContext(true, new String[] {randomAlphaOfLengthBetween(5, 20)}, null);
+                    break;
+                default:
+                    throw new IllegalStateException();
             }
             factory.fetchSource(fetchSourceContext);
         }
@@ -136,29 +138,31 @@ protected final TopHitsAggregationBuilder createTestAggregatorBuilder() {
                 int branch = randomInt(5);
                 switch (branch) {
                 case 0:
-                    factory.sort(SortBuilders.fieldSort(randomAsciiOfLengthBetween(5, 20)).order(randomFrom(SortOrder.values())));
+                    factory.sort(SortBuilders.fieldSort(randomAlphaOfLengthBetween(5, 20)).order(randomFrom(SortOrder.values())));
                     break;
                 case 1:
-                    factory.sort(SortBuilders.geoDistanceSort(randomAsciiOfLengthBetween(5, 20), AbstractQueryTestCase.randomGeohash(1, 12))
+                    factory.sort(SortBuilders.geoDistanceSort(randomAlphaOfLengthBetween(5, 20), AbstractQueryTestCase.randomGeohash(1, 12))
                             .order(randomFrom(SortOrder.values())));
                     break;
                 case 2:
                     factory.sort(SortBuilders.scoreSort().order(randomFrom(SortOrder.values())));
                     break;
                 case 3:
-                    factory.sort(SortBuilders.scriptSort(new Script("foo"), ScriptSortType.NUMBER).order(randomFrom(SortOrder.values())));
+                    factory.sort(SortBuilders.scriptSort(mockScript("foo"), ScriptSortType.NUMBER).order(randomFrom(SortOrder.values())));
                     break;
                 case 4:
-                    factory.sort(randomAsciiOfLengthBetween(5, 20));
+                    factory.sort(randomAlphaOfLengthBetween(5, 20));
                     break;
                 case 5:
-                    factory.sort(randomAsciiOfLengthBetween(5, 20), randomFrom(SortOrder.values()));
+                    factory.sort(randomAlphaOfLengthBetween(5, 20), randomFrom(SortOrder.values()));
                     break;
                 }
             }
         }
         if (randomBoolean()) {
-            factory.highlighter(HighlightBuilderTests.randomHighlighterBuilder());
+            // parent test shuffles xContent, we need to make sure highlight fields are ordered
+            factory.highlighter(
+                    HighlightBuilderTests.randomHighlighterBuilder().useExplicitFieldOrder(true));
         }
         return factory;
     }
@@ -184,15 +188,11 @@ public void testFailWithSubAgg() throws Exception {
             "      }\n" +
             "    }\n" +
             "}";
-        try {
-            XContentParser parser = XContentFactory.xContent(source).createParser(source);
-            QueryParseContext parseContext = new QueryParseContext(queriesRegistry, parser, parseFieldMatcher);
-            assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
-            aggParsers.parseAggregators(parseContext);
-            fail();
-        } catch (AggregationInitializationException e) {
-            assertThat(e.toString(), containsString("Aggregator [top_tags_hits] of type [top_hits] cannot accept sub-aggregations"));
-        }
+        XContentParser parser = createParser(JsonXContent.jsonXContent, source);
+        QueryParseContext parseContext = new QueryParseContext(parser);
+        assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
+        Exception e = expectThrows(AggregationInitializationException.class, () -> AggregatorFactories.parseAggregators(parseContext));
+        assertThat(e.toString(), containsString("Aggregator [top_tags_hits] of type [top_hits] cannot accept sub-aggregations"));
     }
 
 }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java
index 7de37fdb6a806..16eda8f72d831 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java
@@ -28,12 +28,13 @@
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.LeafSearchScript;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptEngineRegistry;
 import org.elasticsearch.script.ScriptEngineService;
-import org.elasticsearch.script.ScriptModule;
-import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.script.SearchScript;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.bucket.filter.Filter;
 import org.elasticsearch.search.aggregations.bucket.global.Global;
+import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCount;
 import org.elasticsearch.search.lookup.LeafSearchLookup;
 import org.elasticsearch.search.lookup.SearchLookup;
@@ -48,9 +49,14 @@
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
+import static org.elasticsearch.index.query.QueryBuilders.termQuery;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.count;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.global;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.terms;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.filter;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
 
@@ -126,9 +132,9 @@ public void testSingleValuedFieldGetProperty() throws Exception {
         assertThat(valueCount, notNullValue());
         assertThat(valueCount.getName(), equalTo("count"));
         assertThat(valueCount.getValue(), equalTo(10L));
-        assertThat((ValueCount) global.getProperty("count"), equalTo(valueCount));
-        assertThat((double) global.getProperty("count.value"), equalTo(10d));
-        assertThat((double) valueCount.getProperty("value"), equalTo(10d));
+        assertThat((ValueCount) ((InternalAggregation)global).getProperty("count"), equalTo(valueCount));
+        assertThat((double) ((InternalAggregation)global).getProperty("count.value"), equalTo(10d));
+        assertThat((double) ((InternalAggregation)valueCount).getProperty("value"), equalTo(10d));
     }
 
     public void testSingleValuedFieldPartiallyUnmapped() throws Exception {
@@ -161,7 +167,7 @@ public void testMultiValuedField() throws Exception {
 
     public void testSingleValuedScript() throws Exception {
         SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery())
-                .addAggregation(count("count").script(new Script("value", ScriptType.INLINE, FieldValueScriptEngine.NAME, null))).execute().actionGet();
+                .addAggregation(count("count").script(new Script(ScriptType.INLINE, FieldValueScriptEngine.NAME, "value", Collections.emptyMap()))).execute().actionGet();
 
         assertHitCount(searchResponse, 10);
 
@@ -173,7 +179,7 @@ public void testSingleValuedScript() throws Exception {
 
     public void testMultiValuedScript() throws Exception {
         SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery())
-                .addAggregation(count("count").script(new Script("values", ScriptType.INLINE, FieldValueScriptEngine.NAME, null))).execute().actionGet();
+                .addAggregation(count("count").script(new Script(ScriptType.INLINE, FieldValueScriptEngine.NAME, "values", Collections.emptyMap()))).execute().actionGet();
 
         assertHitCount(searchResponse, 10);
 
@@ -186,7 +192,7 @@ public void testMultiValuedScript() throws Exception {
     public void testSingleValuedScriptWithParams() throws Exception {
         Map<String, Object> params = Collections.singletonMap("s", "value");
         SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery())
-                .addAggregation(count("count").script(new Script("", ScriptType.INLINE, FieldValueScriptEngine.NAME, params))).execute().actionGet();
+                .addAggregation(count("count").script(new Script(ScriptType.INLINE, FieldValueScriptEngine.NAME, "", params))).execute().actionGet();
 
         assertHitCount(searchResponse, 10);
 
@@ -199,7 +205,7 @@ public void testSingleValuedScriptWithParams() throws Exception {
     public void testMultiValuedScriptWithParams() throws Exception {
         Map<String, Object> params = Collections.singletonMap("s", "values");
         SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery())
-                .addAggregation(count("count").script(new Script("", ScriptType.INLINE, FieldValueScriptEngine.NAME, params))).execute().actionGet();
+                .addAggregation(count("count").script(new Script(ScriptType.INLINE, FieldValueScriptEngine.NAME, "", params))).execute().actionGet();
 
         assertHitCount(searchResponse, 10);
 
@@ -209,6 +215,75 @@ public void testMultiValuedScriptWithParams() throws Exception {
         assertThat(valueCount.getValue(), equalTo(20L));
     }
 
+    /**
+     * Make sure that a request using a script does not get cached and a request
+     * not using a script does get cached.
+     */
+    public void testDontCacheScripts() throws Exception {
+        assertAcked(prepareCreate("cache_test_idx").addMapping("type", "d", "type=long")
+                .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1))
+                .get());
+        indexRandom(true, client().prepareIndex("cache_test_idx", "type", "1").setSource("s", 1),
+                client().prepareIndex("cache_test_idx", "type", "2").setSource("s", 2));
+
+        // Make sure we are starting with a clear cache
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(0L));
+
+        // Test that a request using a script does not get cached
+        SearchResponse r = client().prepareSearch("cache_test_idx").setSize(0)
+                .addAggregation(count("foo").field("d").script(
+                    new Script(ScriptType.INLINE, FieldValueScriptEngine.NAME, "value", Collections.emptyMap())))
+                .get();
+        assertSearchResponse(r);
+
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(0L));
+
+        // To make sure that the cache is working test that a request not using
+        // a script is cached
+        r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(count("foo").field("d")).get();
+        assertSearchResponse(r);
+
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+                .getMissCount(), equalTo(1L));
+    }
+
+    public void testOrderByEmptyAggregation() throws Exception {
+        SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery())
+                .addAggregation(terms("terms").field("value").order(Terms.Order.compound(Terms.Order.aggregation("filter>count", true)))
+                        .subAggregation(filter("filter", termQuery("value", 100)).subAggregation(count("count").field("value"))))
+                .get();
+
+        assertHitCount(searchResponse, 10);
+
+        Terms terms = searchResponse.getAggregations().get("terms");
+        assertThat(terms, notNullValue());
+        List<? extends Terms.Bucket> buckets = terms.getBuckets();
+        assertThat(buckets, notNullValue());
+        assertThat(buckets.size(), equalTo(10));
+
+        for (int i = 0; i < 10; i++) {
+            Terms.Bucket bucket = buckets.get(i);
+            assertThat(bucket, notNullValue());
+            assertThat(bucket.getKeyAsNumber(), equalTo((long) i + 1));
+            assertThat(bucket.getDocCount(), equalTo(1L));
+            Filter filter = bucket.getAggregations().get("filter");
+            assertThat(filter, notNullValue());
+            assertThat(filter.getDocCount(), equalTo(0L));
+            ValueCount count = filter.getAggregations().get("count");
+            assertThat(count, notNullValue());
+            assertThat(count.value(), equalTo(0.0));
+
+        }
+    }
+
     /**
      * Mock plugin for the {@link FieldValueScriptEngine}
      */
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountTests.java
index 99d4d41839cef..9a3ed32604488 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountTests.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.search.aggregations.metrics;
 
-import org.elasticsearch.script.Script;
 import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
 import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCountAggregationBuilder;
 
@@ -29,19 +28,7 @@ public class ValueCountTests extends BaseAggregationTestCase<ValueCountAggregati
     protected final ValueCountAggregationBuilder createTestAggregatorBuilder() {
         ValueCountAggregationBuilder factory = new ValueCountAggregationBuilder("foo", null);
         String field = randomNumericField();
-        int randomFieldBranch = randomInt(3);
-        switch (randomFieldBranch) {
-        case 0:
-            factory.field(field);
-            break;
-        case 1:
-            factory.field(field);
-            factory.script(new Script("_value + 1"));
-            break;
-        case 2:
-            factory.script(new Script("doc[" + field + "] + 1"));
-            break;
-        }
+        randomFieldOrScript(factory, field);
         if (randomBoolean()) {
             factory.missing("MISSING");
         }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/avg/AvgAggregatorTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/avg/AvgAggregatorTests.java
new file mode 100644
index 0000000000000..9ba7ecb71b70c
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/avg/AvgAggregatorTests.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.avg;
+
+import org.apache.lucene.document.IntPoint;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.SortedNumericDocValuesField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.search.FieldValueQuery;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.store.Directory;
+import org.elasticsearch.common.CheckedConsumer;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.NumberFieldMapper;
+import org.elasticsearch.search.aggregations.AggregatorTestCase;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.function.Consumer;
+
+import static java.util.Collections.singleton;
+
+public class AvgAggregatorTests extends AggregatorTestCase {
+
+    public void testNoDocs() throws IOException {
+        testCase(new MatchAllDocsQuery(), iw -> {
+            // Intentionally not writing any docs
+        }, avg -> {
+            assertEquals(Double.NaN, avg.getValue(), 0);
+        });
+    }
+
+    public void testNoMatchingField() throws IOException {
+        testCase(new MatchAllDocsQuery(), iw -> {
+            iw.addDocument(singleton(new SortedNumericDocValuesField("wrong_number", 7)));
+            iw.addDocument(singleton(new SortedNumericDocValuesField("wrong_number", 3)));
+        }, avg -> {
+            assertEquals(Double.NaN, avg.getValue(), 0);
+        });
+    }
+
+    public void testSomeMatchesSortedNumericDocValues() throws IOException {
+        testCase(new FieldValueQuery("number"), iw -> {
+            iw.addDocument(singleton(new SortedNumericDocValuesField("number", 7)));
+            iw.addDocument(singleton(new SortedNumericDocValuesField("number", 2)));
+            iw.addDocument(singleton(new SortedNumericDocValuesField("number", 3)));
+        }, avg -> {
+            assertEquals(4, avg.getValue(), 0);
+        });
+    }
+
+    public void testSomeMatchesNumericDocValues() throws IOException {
+        testCase(new FieldValueQuery("number"), iw -> {
+            iw.addDocument(singleton(new NumericDocValuesField("number", 7)));
+            iw.addDocument(singleton(new NumericDocValuesField("number", 2)));
+            iw.addDocument(singleton(new NumericDocValuesField("number", 3)));
+        }, avg -> {
+            assertEquals(4, avg.getValue(), 0);
+        });
+    }
+
+    public void testQueryFiltering() throws IOException {
+        testCase(IntPoint.newRangeQuery("number", 0, 3), iw -> {
+            iw.addDocument(Arrays.asList(new IntPoint("number", 7), new SortedNumericDocValuesField("number", 7)));
+            iw.addDocument(Arrays.asList(new IntPoint("number", 1), new SortedNumericDocValuesField("number", 2)));
+            iw.addDocument(Arrays.asList(new IntPoint("number", 3), new SortedNumericDocValuesField("number", 3)));
+        }, avg -> {
+            assertEquals(2.5, avg.getValue(), 0);
+        });
+    }
+
+    public void testQueryFiltersAll() throws IOException {
+        testCase(IntPoint.newRangeQuery("number", -1, 0), iw -> {
+            iw.addDocument(Arrays.asList(new IntPoint("number", 7), new SortedNumericDocValuesField("number", 7)));
+            iw.addDocument(Arrays.asList(new IntPoint("number", 1), new SortedNumericDocValuesField("number", 2)));
+            iw.addDocument(Arrays.asList(new IntPoint("number", 3), new SortedNumericDocValuesField("number", 7)));
+        }, avg -> {
+            assertEquals(Double.NaN, avg.getValue(), 0);
+        });
+    }
+
+    private void testCase(Query query, CheckedConsumer<RandomIndexWriter, IOException> buildIndex, Consumer<InternalAvg> verify)
+            throws IOException {
+        Directory directory = newDirectory();
+        RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
+        buildIndex.accept(indexWriter);
+        indexWriter.close();
+
+        IndexReader indexReader = DirectoryReader.open(directory);
+        IndexSearcher indexSearcher = newSearcher(indexReader, true, true);
+
+        AvgAggregationBuilder aggregationBuilder = new AvgAggregationBuilder("_name").field("number");
+        MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
+        fieldType.setName("number");
+        try (AvgAggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType)) {
+            aggregator.preCollection();
+            indexSearcher.search(query, aggregator);
+            aggregator.postCollection();
+            verify.accept((InternalAvg) aggregator.buildAggregation(0L));
+        }
+        indexReader.close();
+        directory.close();
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/avg/InternalAvgTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/avg/InternalAvgTests.java
new file mode 100644
index 0000000000000..8fa0c050a8aba
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/avg/InternalAvgTests.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.avg;
+
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.ParsedAggregation;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.test.InternalAggregationTestCase;
+
+import java.util.List;
+import java.util.Map;
+
+public class InternalAvgTests extends InternalAggregationTestCase<InternalAvg> {
+
+    @Override
+    protected InternalAvg createTestInstance(String name, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
+        DocValueFormat formatter = randomNumericDocValueFormat();
+        long count = frequently() ? randomNonNegativeLong() % 100000 : 0;
+        return new InternalAvg(name, randomDoubleBetween(0, 100000, true), count, formatter, pipelineAggregators, metaData);
+    }
+
+    @Override
+    protected void assertFromXContent(InternalAvg avg, ParsedAggregation parsedAggregation) {
+        ParsedAvg parsed = ((ParsedAvg) parsedAggregation);
+        assertEquals(avg.getValue(), parsed.getValue(), Double.MIN_VALUE);
+        // we don't print out VALUE_AS_STRING for avg.getCount() == 0, so we cannot get the exact same value back
+        if (avg.getCount() != 0) {
+            assertEquals(avg.getValueAsString(), parsed.getValueAsString());
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityTests.java
index ab0377c633168..1b3a18581768e 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityTests.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.search.aggregations.metrics.cardinality;
 
-import org.elasticsearch.script.Script;
 import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
 
 public class CardinalityTests extends BaseAggregationTestCase<CardinalityAggregationBuilder> {
@@ -28,19 +27,7 @@ public class CardinalityTests extends BaseAggregationTestCase<CardinalityAggrega
     protected final CardinalityAggregationBuilder createTestAggregatorBuilder() {
         CardinalityAggregationBuilder factory = new CardinalityAggregationBuilder("foo", null);
         String field = randomNumericField();
-        int randomFieldBranch = randomInt(3);
-        switch (randomFieldBranch) {
-        case 0:
-            factory.field(field);
-            break;
-        case 1:
-            factory.field(field);
-            factory.script(new Script("_value + 1"));
-            break;
-        case 2:
-            factory.script(new Script("doc[" + field + "] + 1"));
-            break;
-        }
+        randomFieldOrScript(factory, field);
         if (randomBoolean()) {
             factory.missing("MISSING");
         }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/cardinality/InternalCardinalityTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/cardinality/InternalCardinalityTests.java
new file mode 100644
index 0000000000000..3b65a91fa89ba
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/cardinality/InternalCardinalityTests.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.cardinality;
+
+import org.elasticsearch.common.lease.Releasables;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.MockBigArrays;
+import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
+import org.elasticsearch.search.aggregations.ParsedAggregation;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.test.InternalAggregationTestCase;
+import org.junit.After;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class InternalCardinalityTests extends InternalAggregationTestCase<InternalCardinality> {
+    private static List<HyperLogLogPlusPlus> algos;
+    private static int p;
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        algos = new ArrayList<>();
+        p = randomIntBetween(HyperLogLogPlusPlus.MIN_PRECISION, HyperLogLogPlusPlus.MAX_PRECISION);
+    }
+
+    @After //we force @After to have it run before ESTestCase#after otherwise it fails
+    @Override
+    public void tearDown() throws Exception {
+        super.tearDown();
+        Releasables.close(algos);
+        algos.clear();
+        algos = null;
+    }
+
+    @Override
+    protected InternalCardinality createTestInstance(String name,
+            List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
+        HyperLogLogPlusPlus hllpp = new HyperLogLogPlusPlus(p,
+                new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService()), 1);
+        algos.add(hllpp);
+        for (int i = 0; i < 100; i++) {
+            hllpp.collect(0, randomIntBetween(1, 100));
+        }
+        return new InternalCardinality(name, hllpp, pipelineAggregators, metaData);
+    }
+
+    @Override
+    protected void assertFromXContent(InternalCardinality aggregation, ParsedAggregation parsedAggregation) {
+        assertTrue(parsedAggregation instanceof ParsedCardinality);
+        ParsedCardinality parsed = (ParsedCardinality) parsedAggregation;
+
+        assertEquals(aggregation.getValue(), parsed.getValue(), Double.MIN_VALUE);
+        assertEquals(aggregation.getValueAsString(), parsed.getValueAsString());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/geobounds/InternalGeoBoundsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/geobounds/InternalGeoBoundsTests.java
new file mode 100644
index 0000000000000..e734ee5eb53e6
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/geobounds/InternalGeoBoundsTests.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.geobounds;
+
+import org.elasticsearch.search.aggregations.ParsedAggregation;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.test.InternalAggregationTestCase;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+public class InternalGeoBoundsTests extends InternalAggregationTestCase<InternalGeoBounds> {
+
+    @Override
+    protected InternalGeoBounds createTestInstance(String name, List<PipelineAggregator> pipelineAggregators,
+                                                   Map<String, Object> metaData) {
+        // we occasionally want to test top = Double.NEGATIVE_INFINITY since this triggers empty xContent object
+        double top = frequently() ? randomDouble() : Double.NEGATIVE_INFINITY;
+        InternalGeoBounds geo = new InternalGeoBounds(name,
+            top, randomDouble(), randomDouble(), randomDouble(),
+            randomDouble(), randomDouble(), randomBoolean(),
+            pipelineAggregators, Collections.emptyMap());
+        return geo;
+    }
+
+    @Override
+    protected void assertFromXContent(InternalGeoBounds aggregation, ParsedAggregation parsedAggregation) {
+        assertTrue(parsedAggregation instanceof ParsedGeoBounds);
+        ParsedGeoBounds parsed = (ParsedGeoBounds) parsedAggregation;
+
+        assertEquals(aggregation.topLeft(), parsed.topLeft());
+        assertEquals(aggregation.bottomRight(), parsed.bottomRight());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroidTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroidTests.java
new file mode 100644
index 0000000000000..3d1e370967767
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroidTests.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.metrics.geocentroid;
+
+import org.apache.lucene.geo.GeoEncodingUtils;
+import org.elasticsearch.common.geo.GeoPoint;
+import org.elasticsearch.search.aggregations.ParsedAggregation;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.test.InternalAggregationTestCase;
+import org.elasticsearch.test.geo.RandomGeoGenerator;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+public class InternalGeoCentroidTests extends InternalAggregationTestCase<InternalGeoCentroid> {
+
+    @Override
+    protected InternalGeoCentroid createTestInstance(String name, List<PipelineAggregator> pipelineAggregators,
+                                                     Map<String, Object> metaData) {
+        GeoPoint centroid = RandomGeoGenerator.randomPoint(random());
+
+        // Re-encode lat/longs to avoid rounding issue when testing InternalGeoCentroid#hashCode() and
+        // InternalGeoCentroid#equals()
+        int encodedLon = GeoEncodingUtils.encodeLongitude(centroid.lon());
+        centroid.resetLon(GeoEncodingUtils.decodeLongitude(encodedLon));
+        int encodedLat = GeoEncodingUtils.encodeLatitude(centroid.lat());
+        centroid.resetLat(GeoEncodingUtils.decodeLatitude(encodedLat));
+        long count = randomIntBetween(0, 1000);
+        if (count == 0) {
+            centroid = null;
+        }
+        return new InternalGeoCentroid(name, centroid, count, Collections.emptyList(), Collections.emptyMap());
+    }
+
+    @Override
+    protected void assertFromXContent(InternalGeoCentroid aggregation, ParsedAggregation parsedAggregation) {
+        assertTrue(parsedAggregation instanceof ParsedGeoCentroid);
+        ParsedGeoCentroid parsed = (ParsedGeoCentroid) parsedAggregation;
+
+        assertEquals(aggregation.centroid(), parsed.centroid());
+        assertEquals(aggregation.count(), parsed.count());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/min/InternalMinTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/min/InternalMinTests.java
new file mode 100644
index 0000000000000..58f587ac0c32a
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/min/InternalMinTests.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.min;
+
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.ParsedAggregation;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.test.InternalAggregationTestCase;
+
+import java.util.List;
+import java.util.Map;
+
+public class InternalMinTests extends InternalAggregationTestCase<InternalMin> {
+    @Override
+    protected InternalMin createTestInstance(String name, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
+        double value = frequently() ? randomDouble() : randomFrom(new Double[] { Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY });
+        DocValueFormat formatter = randomNumericDocValueFormat();
+        return new InternalMin(name, value, formatter, pipelineAggregators, metaData);
+    }
+
+    @Override
+    protected void assertFromXContent(InternalMin min, ParsedAggregation parsedAggregation) {
+        ParsedMin parsed = ((ParsedMin) parsedAggregation);
+        if (Double.isInfinite(min.getValue()) == false) {
+            assertEquals(min.getValue(), parsed.getValue(), Double.MIN_VALUE);
+            assertEquals(min.getValueAsString(), parsed.getValueAsString());
+        } else {
+            // we write Double.NEGATIVE_INFINITY and Double.POSITIVE_INFINITY to xContent as 'null', so we
+            // cannot differentiate between them. Also we cannot recreate the exact String representation
+            assertEquals(parsed.getValue(), Double.POSITIVE_INFINITY, 0);
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregatorTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregatorTests.java
new file mode 100644
index 0000000000000..48b0b115e8a3c
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregatorTests.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.metrics.min;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.SortedNumericDocValuesField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.store.Directory;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.NumberFieldMapper;
+import org.elasticsearch.search.aggregations.AggregatorTestCase;
+
+public class MinAggregatorTests extends AggregatorTestCase {
+
+    public void testMinAggregator_numericDv() throws Exception {
+        Directory directory = newDirectory();
+        RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
+        Document document = new Document();
+        document.add(new NumericDocValuesField("number", 9));
+        indexWriter.addDocument(document);
+        document = new Document();
+        document.add(new NumericDocValuesField("number", 7));
+        indexWriter.addDocument(document);
+        document = new Document();
+        document.add(new NumericDocValuesField("number", 5));
+        indexWriter.addDocument(document);
+        document = new Document();
+        document.add(new NumericDocValuesField("number", 3));
+        indexWriter.addDocument(document);
+        document = new Document();
+        document.add(new NumericDocValuesField("number", 1));
+        indexWriter.addDocument(document);
+        document = new Document();
+        document.add(new NumericDocValuesField("number", -1));
+        indexWriter.addDocument(document);
+        indexWriter.close();
+
+        IndexReader indexReader = DirectoryReader.open(directory);
+        IndexSearcher indexSearcher = newSearcher(indexReader, true, true);
+
+        MinAggregationBuilder aggregationBuilder = new MinAggregationBuilder("_name").field("number");
+        MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
+        fieldType.setName("number");
+        try (MinAggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType)) {
+            aggregator.preCollection();
+            indexSearcher.search(new MatchAllDocsQuery(), aggregator);
+            aggregator.postCollection();
+            InternalMin result = (InternalMin) aggregator.buildAggregation(0L);
+            assertEquals(-1.0, result.getValue(), 0);
+        }
+        indexReader.close();
+        directory.close();
+    }
+
+    public void testMinAggregator_sortedNumericDv() throws Exception {
+        Directory directory = newDirectory();
+        RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
+        Document document = new Document();
+        document.add(new SortedNumericDocValuesField("number", 9));
+        document.add(new SortedNumericDocValuesField("number", 7));
+        indexWriter.addDocument(document);
+        document = new Document();
+        document.add(new SortedNumericDocValuesField("number", 5));
+        document.add(new SortedNumericDocValuesField("number", 3));
+        indexWriter.addDocument(document);
+        document = new Document();
+        document.add(new SortedNumericDocValuesField("number", 1));
+        document.add(new SortedNumericDocValuesField("number", -1));
+        indexWriter.addDocument(document);
+        indexWriter.close();
+
+        IndexReader indexReader = DirectoryReader.open(directory);
+        IndexSearcher indexSearcher = newSearcher(indexReader, true, true);
+
+        MinAggregationBuilder aggregationBuilder = new MinAggregationBuilder("_name").field("number");
+        MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
+        fieldType.setName("number");
+        try (MinAggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType)) {
+            aggregator.preCollection();
+            indexSearcher.search(new MatchAllDocsQuery(), aggregator);
+            aggregator.postCollection();
+            InternalMin result = (InternalMin) aggregator.buildAggregation(0L);
+            assertEquals(-1.0, result.getValue(), 0);
+        }
+        indexReader.close();
+        directory.close();
+    }
+
+    public void testMinAggregator_noValue() throws Exception {
+        Directory directory = newDirectory();
+        RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
+        Document document = new Document();
+        document.add(new SortedNumericDocValuesField("number1", 7));
+        indexWriter.addDocument(document);
+        document = new Document();
+        document.add(new SortedNumericDocValuesField("number1", 3));
+        indexWriter.addDocument(document);
+        document = new Document();
+        document.add(new SortedNumericDocValuesField("number1", 1));
+        indexWriter.addDocument(document);
+        indexWriter.close();
+
+        IndexReader indexReader = DirectoryReader.open(directory);
+        IndexSearcher indexSearcher = newSearcher(indexReader, true, true);
+
+        MinAggregationBuilder aggregationBuilder = new MinAggregationBuilder("_name").field("number2");
+        MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
+        fieldType.setName("number2");
+        try (MinAggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType)) {
+            aggregator.preCollection();
+            indexSearcher.search(new MatchAllDocsQuery(), aggregator);
+            aggregator.postCollection();
+            InternalMin result = (InternalMin) aggregator.buildAggregation(0L);
+            assertEquals(Double.POSITIVE_INFINITY, result.getValue(), 0);
+        }
+        indexReader.close();
+        directory.close();
+    }
+
+    public void testMinAggregator_noDocs() throws Exception {
+        Directory directory = newDirectory();
+        RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
+        indexWriter.close();
+
+        IndexReader indexReader = DirectoryReader.open(directory);
+        IndexSearcher indexSearcher = newSearcher(indexReader, true, true);
+
+        MinAggregationBuilder aggregationBuilder = new MinAggregationBuilder("_name").field("number");
+        MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
+        fieldType.setName("number");
+        try (MinAggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType)) {
+            aggregator.preCollection();
+            indexSearcher.search(new MatchAllDocsQuery(), aggregator);
+            aggregator.postCollection();
+            InternalMin result = (InternalMin) aggregator.buildAggregation(0L);
+            assertEquals(Double.POSITIVE_INFINITY, result.getValue(), 0);
+        }
+        indexReader.close();
+        directory.close();
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractPercentilesTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractPercentilesTestCase.java
new file mode 100644
index 0000000000000..e54a2a8b9a14f
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractPercentilesTestCase.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.percentiles;
+
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.Aggregation.CommonFields;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.test.InternalAggregationTestCase;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Predicate;
+
+public abstract class AbstractPercentilesTestCase<T extends InternalAggregation & Iterable<Percentile>>
+        extends InternalAggregationTestCase<T> {
+
+    private double[] percents;
+    private boolean keyed;
+    private DocValueFormat docValueFormat;
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        percents = randomPercents(false);
+        keyed = randomBoolean();
+        docValueFormat = randomNumericDocValueFormat();
+    }
+
+    @Override
+    protected T createTestInstance(String name, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
+        int numValues = randomInt(100);
+        double[] values = new double[numValues];
+        for (int i = 0; i < numValues; ++i) {
+            values[i] = randomDouble();
+        }
+        return createTestInstance(name, pipelineAggregators, metaData, keyed, docValueFormat, percents, values);
+    }
+
+    protected abstract T createTestInstance(String name, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData,
+                                            boolean keyed, DocValueFormat format, double[] percents, double[] values);
+
+    protected abstract Class<? extends ParsedPercentiles> implementationClass();
+
+    public void testPercentilesIterators() throws IOException {
+        final T aggregation = createTestInstance();
+        final Iterable<Percentile> parsedAggregation = parseAndAssert(aggregation, false, false);
+
+        Iterator<Percentile> it = aggregation.iterator();
+        Iterator<Percentile> parsedIt = parsedAggregation.iterator();
+        while (it.hasNext()) {
+            assertEquals(it.next(), parsedIt.next());
+        }
+    }
+
+    public static double[] randomPercents(boolean sorted) {
+        List<Double> randomCdfValues = randomSubsetOf(randomIntBetween(1, 7), 0.01d, 0.05d, 0.25d, 0.50d, 0.75d, 0.95d, 0.99d);
+        double[] percents = new double[randomCdfValues.size()];
+        for (int i = 0; i < randomCdfValues.size(); i++) {
+            percents[i] = randomCdfValues.get(i);
+        }
+        if (sorted) {
+            Arrays.sort(percents);
+        }
+        return percents;
+    }
+
+    @Override
+    protected Predicate<String> excludePathsFromXContentInsertion() {
+        return path -> path.endsWith(CommonFields.VALUES.getPreferredName());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/InternalPercentilesRanksTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/InternalPercentilesRanksTestCase.java
new file mode 100644
index 0000000000000..f45b7cce51e37
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/InternalPercentilesRanksTestCase.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.percentiles;
+
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.ParsedAggregation;
+
+public abstract class InternalPercentilesRanksTestCase<T extends InternalAggregation & PercentileRanks>
+        extends AbstractPercentilesTestCase<T> {
+
+    @Override
+    protected final void assertFromXContent(T aggregation, ParsedAggregation parsedAggregation) {
+        assertTrue(parsedAggregation instanceof PercentileRanks);
+        PercentileRanks parsedPercentileRanks = (PercentileRanks) parsedAggregation;
+
+        for (Percentile percentile : aggregation) {
+            Double value = percentile.getValue();
+            assertEquals(aggregation.percent(value), parsedPercentileRanks.percent(value), 0);
+            assertEquals(aggregation.percentAsString(value), parsedPercentileRanks.percentAsString(value));
+        }
+
+        Class<? extends ParsedPercentiles> parsedClass = implementationClass();
+        assertTrue(parsedClass != null && parsedClass.isInstance(parsedAggregation));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/InternalPercentilesTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/InternalPercentilesTestCase.java
new file mode 100644
index 0000000000000..be105f2af80b6
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/InternalPercentilesTestCase.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.percentiles;
+
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.ParsedAggregation;
+
+import java.util.List;
+
+public abstract class InternalPercentilesTestCase<T extends InternalAggregation & Percentiles> extends AbstractPercentilesTestCase<T> {
+
+    @Override
+    protected final void assertFromXContent(T aggregation, ParsedAggregation parsedAggregation) {
+        assertTrue(parsedAggregation instanceof Percentiles);
+        Percentiles parsedPercentiles = (Percentiles) parsedAggregation;
+
+        for (Percentile percentile : aggregation) {
+            Double percent = percentile.getPercent();
+            assertEquals(aggregation.percentile(percent), parsedPercentiles.percentile(percent), 0);
+            assertEquals(aggregation.percentileAsString(percent), parsedPercentiles.percentileAsString(percent));
+        }
+
+        Class<? extends ParsedPercentiles> parsedClass = implementationClass();
+        assertTrue(parsedClass != null && parsedClass.isInstance(parsedAggregation));
+    }
+
+    public static double[] randomPercents() {
+        List<Double> randomCdfValues = randomSubsetOf(randomIntBetween(1, 7), 0.01d, 0.05d, 0.25d, 0.50d, 0.75d, 0.95d, 0.99d);
+        double[] percents = new double[randomCdfValues.size()];
+        for (int i = 0; i < randomCdfValues.size(); i++) {
+            percents[i] = randomCdfValues.get(i);
+        }
+        return percents;
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentilesRanksTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentilesRanksTests.java
new file mode 100644
index 0000000000000..c837a71346a59
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentilesRanksTests.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.percentiles.hdr;
+
+import org.HdrHistogram.DoubleHistogram;
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.metrics.percentiles.InternalPercentilesRanksTestCase;
+import org.elasticsearch.search.aggregations.metrics.percentiles.ParsedPercentiles;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+public class InternalHDRPercentilesRanksTests extends InternalPercentilesRanksTestCase<InternalHDRPercentileRanks> {
+
+    @Override
+    protected InternalHDRPercentileRanks createTestInstance(String name, List<PipelineAggregator> aggregators, Map<String, Object> metadata,
+                                                            boolean keyed, DocValueFormat format, double[] percents, double[] values) {
+
+        final DoubleHistogram state = new DoubleHistogram(3);
+        Arrays.stream(values).forEach(state::recordValue);
+
+        return new InternalHDRPercentileRanks(name, percents, state, keyed, format, aggregators, metadata);
+    }
+
+    @Override
+    protected Class<? extends ParsedPercentiles> implementationClass() {
+        return ParsedHDRPercentileRanks.class;
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentilesTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentilesTests.java
new file mode 100644
index 0000000000000..fea13b0fe91f1
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentilesTests.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.percentiles.hdr;
+
+import org.HdrHistogram.DoubleHistogram;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.metrics.percentiles.InternalPercentilesTestCase;
+import org.elasticsearch.search.aggregations.metrics.percentiles.ParsedPercentiles;
+import org.elasticsearch.search.aggregations.metrics.percentiles.Percentile;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import static java.util.Collections.emptyList;
+import static java.util.Collections.emptyMap;
+
+public class InternalHDRPercentilesTests extends InternalPercentilesTestCase<InternalHDRPercentiles> {
+
+    @Override
+    protected InternalHDRPercentiles createTestInstance(String name,
+                                                        List<PipelineAggregator> pipelineAggregators,
+                                                        Map<String, Object>  metaData,
+                                                        boolean keyed, DocValueFormat format, double[] percents, double[] values) {
+        final DoubleHistogram state = new DoubleHistogram(3);
+        Arrays.stream(values).forEach(state::recordValue);
+
+        return new InternalHDRPercentiles(name, percents, state, keyed, format, pipelineAggregators, metaData);
+    }
+
+    @Override
+    protected Class<? extends ParsedPercentiles> implementationClass() {
+        return ParsedHDRPercentiles.class;
+    }
+
+    public void testIterator() {
+        final double[] percents =  randomPercents(false);
+        final double[] values = new double[frequently() ? randomIntBetween(1, 10) : 0];
+        for (int i = 0; i < values.length; ++i) {
+            values[i] = randomDouble();
+        }
+
+        InternalHDRPercentiles aggregation =
+                createTestInstance("test", emptyList(), emptyMap(), false, randomNumericDocValueFormat(), percents, values);
+
+        Iterator<Percentile> iterator = aggregation.iterator();
+        for (double percent : percents) {
+            assertTrue(iterator.hasNext());
+
+            Percentile percentile = iterator.next();
+            assertEquals(percent, percentile.getPercent(), 0.0d);
+            assertEquals(aggregation.percentile(percent), percentile.getValue(), 0.0d);
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentilesRanksTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentilesRanksTests.java
new file mode 100644
index 0000000000000..d392b692d7168
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentilesRanksTests.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.percentiles.tdigest;
+
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.metrics.percentiles.InternalPercentilesRanksTestCase;
+import org.elasticsearch.search.aggregations.metrics.percentiles.ParsedPercentiles;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+public class InternalTDigestPercentilesRanksTests extends InternalPercentilesRanksTestCase<InternalTDigestPercentileRanks> {
+
+    @Override
+    protected InternalTDigestPercentileRanks createTestInstance(String name, List<PipelineAggregator> aggregators,
+                                                                Map<String, Object> metadata,
+                                                                boolean keyed, DocValueFormat format, double[] percents, double[] values) {
+        final TDigestState state = new TDigestState(100);
+        Arrays.stream(values).forEach(state::add);
+
+        assertEquals(state.centroidCount(), values.length);
+        return new InternalTDigestPercentileRanks(name, percents, state, keyed, format, aggregators, metadata);
+    }
+
+    @Override
+    protected Class<? extends ParsedPercentiles> implementationClass() {
+        return ParsedTDigestPercentileRanks.class;
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentilesTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentilesTests.java
new file mode 100644
index 0000000000000..ab82c7c4ecde2
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentilesTests.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.percentiles.tdigest;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.metrics.percentiles.InternalPercentilesTestCase;
+import org.elasticsearch.search.aggregations.metrics.percentiles.ParsedPercentiles;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+public class InternalTDigestPercentilesTests extends InternalPercentilesTestCase<InternalTDigestPercentiles> {
+
+    @Override
+    protected InternalTDigestPercentiles createTestInstance(String name,
+                                                            List<PipelineAggregator> pipelineAggregators,
+                                                            Map<String, Object> metaData,
+                                                            boolean keyed, DocValueFormat format, double[] percents, double[] values) {
+        final TDigestState state = new TDigestState(100);
+        Arrays.stream(values).forEach(state::add);
+
+        assertEquals(state.centroidCount(), values.length);
+        return new InternalTDigestPercentiles(name, percents, state, keyed, format, pipelineAggregators, metaData);
+    }
+
+    @Override
+    protected Class<? extends ParsedPercentiles> implementationClass() {
+        return ParsedTDigestPercentiles.class;
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetricTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetricTests.java
new file mode 100644
index 0000000000000..4ed0f16c73559
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetricTests.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.scripted;
+
+import org.elasticsearch.common.geo.GeoPoint;
+import org.elasticsearch.script.MockScriptEngine;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptType;
+import org.elasticsearch.search.aggregations.Aggregation.CommonFields;
+import org.elasticsearch.search.aggregations.ParsedAggregation;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.test.InternalAggregationTestCase;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Predicate;
+import java.util.function.Supplier;
+
+public class InternalScriptedMetricTests extends InternalAggregationTestCase<InternalScriptedMetric> {
+
+    private static final String REDUCE_SCRIPT_NAME = "reduceScript";
+    private boolean hasReduceScript;
+    private Supplier<Object>[] valueTypes;
+    private final Supplier<Object>[] leafValueSuppliers = new Supplier[] { () -> randomInt(), () -> randomLong(), () -> randomDouble(),
+            () -> randomFloat(), () -> randomBoolean(), () -> randomAlphaOfLength(5), () -> new GeoPoint(randomDouble(), randomDouble()),
+            () -> null };
+    private final Supplier<Object>[] nestedValueSuppliers = new Supplier[] { () -> new HashMap<String, Object>(),
+            () -> new ArrayList<>() };
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        hasReduceScript = randomBoolean();
+        // we want the same value types (also for nested lists, maps) for all random aggregations
+        int levels = randomIntBetween(1, 3);
+        valueTypes = new Supplier[levels];
+        for (int i = 0; i < levels; i++) {
+            if (i < levels - 1) {
+                valueTypes[i] = randomFrom(nestedValueSuppliers);
+            } else {
+                // the last one needs to be a leaf value, not map or list
+                valueTypes[i] = randomFrom(leafValueSuppliers);
+            }
+        }
+    }
+
+    @Override
+    protected InternalScriptedMetric createTestInstance(String name, List<PipelineAggregator> pipelineAggregators,
+            Map<String, Object> metaData) {
+        Map<String, Object> params = new HashMap<>();
+        if (randomBoolean()) {
+            params.put(randomAlphaOfLength(5), randomAlphaOfLength(5));
+        }
+        Script reduceScript = null;
+        if (hasReduceScript) {
+            reduceScript = new Script(ScriptType.INLINE, MockScriptEngine.NAME, REDUCE_SCRIPT_NAME, params);
+        }
+        Object randomValue = randomValue(valueTypes, 0);
+        return new InternalScriptedMetric(name, randomValue, reduceScript, pipelineAggregators, metaData);
+    }
+
+    @SuppressWarnings("unchecked")
+    private static Object randomValue(Supplier<Object>[] valueTypes, int level) {
+        Object value = valueTypes[level].get();
+        if (value instanceof Map) {
+            int elements = randomIntBetween(1, 5);
+            Map<String, Object> map = (Map<String, Object>) value;
+            for (int i = 0; i < elements; i++) {
+                map.put(randomAlphaOfLength(5), randomValue(valueTypes, level + 1));
+            }
+        } else if (value instanceof List) {
+            int elements = randomIntBetween(1,5);
+            List<Object> list = (List<Object>) value;
+            for (int i = 0; i < elements; i++) {
+                list.add(randomValue(valueTypes, level + 1));
+            }
+        }
+        return value;
+    }
+
+    @Override
+    protected void assertFromXContent(InternalScriptedMetric aggregation, ParsedAggregation parsedAggregation) {
+        assertTrue(parsedAggregation instanceof ParsedScriptedMetric);
+        ParsedScriptedMetric parsed = (ParsedScriptedMetric) parsedAggregation;
+
+        assertValues(aggregation.aggregation(), parsed.aggregation());
+    }
+
+    private static void assertValues(Object expected, Object actual) {
+        if (expected instanceof Long) {
+            // longs that fit into the integer range are parsed back as integer
+            if (actual instanceof Integer) {
+                assertEquals(((Long) expected).intValue(), actual);
+            } else {
+                assertEquals(expected, actual);
+            }
+        } else if (expected instanceof Float) {
+            // based on the xContent type, floats are sometimes parsed back as doubles
+            if (actual instanceof Double) {
+                assertEquals(expected, ((Double) actual).floatValue());
+            } else {
+                assertEquals(expected, actual);
+            }
+        } else if (expected instanceof GeoPoint) {
+            assertTrue(actual instanceof Map);
+            GeoPoint point = (GeoPoint) expected;
+            Map<String, Object> pointMap = (Map<String, Object>) actual;
+            assertEquals(point.getLat(), pointMap.get("lat"));
+            assertEquals(point.getLon(), pointMap.get("lon"));
+        } else if (expected instanceof Map) {
+            Map<String, Object> expectedMap = (Map<String, Object>) expected;
+            Map<String, Object> actualMap = (Map<String, Object>) actual;
+            assertEquals(expectedMap.size(), actualMap.size());
+            for (String key : expectedMap.keySet()) {
+                assertValues(expectedMap.get(key), actualMap.get(key));
+            }
+        } else if (expected instanceof List) {
+                List<Object> expectedList = (List<Object>) expected;
+                List<Object> actualList = (List<Object>) actual;
+                assertEquals(expectedList.size(), actualList.size());
+                Iterator<Object> actualIterator = actualList.iterator();
+                for (Object element : expectedList) {
+                    assertValues(element, actualIterator.next());
+                }
+        } else {
+            assertEquals(expected, actual);
+        }
+    }
+
+    @Override
+    protected Predicate<String> excludePathsFromXContentInsertion() {
+        return path -> path.contains(CommonFields.VALUE.getPreferredName());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorTests.java
new file mode 100644
index 0000000000000..6e5081bc0bd9a
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorTests.java
@@ -0,0 +1,216 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.scripted;
+
+import org.apache.lucene.document.SortedNumericDocValuesField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.store.Directory;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.indices.breaker.CircuitBreakerService;
+import org.elasticsearch.script.MockScriptEngine;
+import org.elasticsearch.script.ScoreAccessor;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptContextRegistry;
+import org.elasticsearch.script.ScriptEngineRegistry;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptSettings;
+import org.elasticsearch.script.ScriptType;
+import org.elasticsearch.search.aggregations.AggregatorTestCase;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+
+import static java.util.Collections.singleton;
+
+public class ScriptedMetricAggregatorTests extends AggregatorTestCase {
+
+    private static final String AGG_NAME = "scriptedMetric";
+    private static final Script INIT_SCRIPT = new Script(ScriptType.INLINE, MockScriptEngine.NAME, "initScript", Collections.emptyMap());
+    private static final Script MAP_SCRIPT = new Script(ScriptType.INLINE, MockScriptEngine.NAME, "mapScript", Collections.emptyMap());
+    private static final Script COMBINE_SCRIPT = new Script(ScriptType.INLINE, MockScriptEngine.NAME, "combineScript",
+            Collections.emptyMap());
+
+    private static final Script INIT_SCRIPT_SCORE = new Script(ScriptType.INLINE, MockScriptEngine.NAME, "initScriptScore",
+            Collections.emptyMap());
+    private static final Script MAP_SCRIPT_SCORE = new Script(ScriptType.INLINE, MockScriptEngine.NAME, "mapScriptScore",
+            Collections.emptyMap());
+    private static final Script COMBINE_SCRIPT_SCORE = new Script(ScriptType.INLINE, MockScriptEngine.NAME, "combineScriptScore",
+            Collections.emptyMap());
+    private static final Map<String, Function<Map<String, Object>, Object>> SCRIPTS = new HashMap<>();
+
+
+    @BeforeClass
+    @SuppressWarnings("unchecked")
+    public static void initMockScripts() {
+        SCRIPTS.put("initScript", params -> {
+            Map<String, Object> agg = (Map<String, Object>) params.get("_agg");
+            agg.put("collector", new ArrayList<Integer>());
+            return agg;
+            });
+        SCRIPTS.put("mapScript", params -> {
+            Map<String, Object> agg = (Map<String, Object>) params.get("_agg");
+            ((List<Integer>) agg.get("collector")).add(1); // just add 1 for each doc the script is run on
+            return agg;
+        });
+        SCRIPTS.put("combineScript", params -> {
+            Map<String, Object> agg = (Map<String, Object>) params.get("_agg");
+            return ((List<Integer>) agg.get("collector")).stream().mapToInt(Integer::intValue).sum();
+        });
+
+        SCRIPTS.put("initScriptScore", params -> {
+            Map<String, Object> agg = (Map<String, Object>) params.get("_agg");
+            agg.put("collector", new ArrayList<Double>());
+            return agg;
+            });
+        SCRIPTS.put("mapScriptScore", params -> {
+            Map<String, Object> agg = (Map<String, Object>) params.get("_agg");
+            ((List<Double>) agg.get("collector")).add(((ScoreAccessor) params.get("_score")).doubleValue());
+            return agg;
+        });
+        SCRIPTS.put("combineScriptScore", params -> {
+            Map<String, Object> agg = (Map<String, Object>) params.get("_agg");
+            return ((List<Double>) agg.get("collector")).stream().mapToDouble(Double::doubleValue).sum();
+        });
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testNoDocs() throws IOException {
+        try (Directory directory = newDirectory()) {
+            try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
+                // intentionally not writing any docs
+            }
+            try (IndexReader indexReader = DirectoryReader.open(directory)) {
+                ScriptedMetricAggregationBuilder aggregationBuilder = new ScriptedMetricAggregationBuilder(AGG_NAME);
+                aggregationBuilder.mapScript(MAP_SCRIPT); // map script is mandatory, even if its not used in this case
+                ScriptedMetric scriptedMetric = search(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder);
+                assertEquals(AGG_NAME, scriptedMetric.getName());
+                assertNotNull(scriptedMetric.aggregation());
+                assertEquals(0, ((HashMap<Object, String>) scriptedMetric.aggregation()).size());
+            }
+        }
+    }
+
+    /**
+     * without combine script, the "_aggs" map should contain a list of the size of the number of documents matched
+     */
+    @SuppressWarnings("unchecked")
+    public void testScriptedMetricWithoutCombine() throws IOException {
+        try (Directory directory = newDirectory()) {
+            int numDocs = randomInt(100);
+            try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
+                for (int i = 0; i < numDocs; i++) {
+                    indexWriter.addDocument(singleton(new SortedNumericDocValuesField("number", i)));
+                }
+            }
+            try (IndexReader indexReader = DirectoryReader.open(directory)) {
+                ScriptedMetricAggregationBuilder aggregationBuilder = new ScriptedMetricAggregationBuilder(AGG_NAME);
+                aggregationBuilder.initScript(INIT_SCRIPT).mapScript(MAP_SCRIPT);
+                ScriptedMetric scriptedMetric = search(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder);
+                assertEquals(AGG_NAME, scriptedMetric.getName());
+                assertNotNull(scriptedMetric.aggregation());
+                Map<String, Object> agg = (Map<String, Object>) scriptedMetric.aggregation();
+                assertEquals(numDocs, ((List<Integer>) agg.get("collector")).size());
+            }
+        }
+    }
+
+    /**
+     * test that combine script sums the list produced by the "mapScript"
+     */
+    public void testScriptedMetricWithCombine() throws IOException {
+        try (Directory directory = newDirectory()) {
+            Integer numDocs = randomInt(100);
+            try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
+                for (int i = 0; i < numDocs; i++) {
+                    indexWriter.addDocument(singleton(new SortedNumericDocValuesField("number", i)));
+                }
+            }
+            try (IndexReader indexReader = DirectoryReader.open(directory)) {
+                ScriptedMetricAggregationBuilder aggregationBuilder = new ScriptedMetricAggregationBuilder(AGG_NAME);
+                aggregationBuilder.initScript(INIT_SCRIPT).mapScript(MAP_SCRIPT).combineScript(COMBINE_SCRIPT);
+                ScriptedMetric scriptedMetric = search(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder);
+                assertEquals(AGG_NAME, scriptedMetric.getName());
+                assertNotNull(scriptedMetric.aggregation());
+                assertEquals(numDocs, scriptedMetric.aggregation());
+            }
+        }
+    }
+
+    /**
+     * test that uses the score of the documents
+     */
+    public void testScriptedMetricWithCombineAccessesScores() throws IOException {
+        try (Directory directory = newDirectory()) {
+            Integer numDocs = randomInt(100);
+            try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
+                for (int i = 0; i < numDocs; i++) {
+                    indexWriter.addDocument(singleton(new SortedNumericDocValuesField("number", i)));
+                }
+            }
+            try (IndexReader indexReader = DirectoryReader.open(directory)) {
+                ScriptedMetricAggregationBuilder aggregationBuilder = new ScriptedMetricAggregationBuilder(AGG_NAME);
+                aggregationBuilder.initScript(INIT_SCRIPT_SCORE).mapScript(MAP_SCRIPT_SCORE).combineScript(COMBINE_SCRIPT_SCORE);
+                ScriptedMetric scriptedMetric = search(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder);
+                assertEquals(AGG_NAME, scriptedMetric.getName());
+                assertNotNull(scriptedMetric.aggregation());
+                // all documents have score of 1.0
+                assertEquals((double) numDocs, scriptedMetric.aggregation());
+            }
+        }
+    }
+
+    /**
+     * We cannot use Mockito for mocking QueryShardContext in this case because
+     * script-related methods (e.g. QueryShardContext#getLazyExecutableScript)
+     * is final and cannot be mocked
+     */
+    @Override
+    protected QueryShardContext queryShardContextMock(MapperService mapperService, final MappedFieldType[] fieldTypes,
+            CircuitBreakerService circuitBreakerService) {
+        Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build();
+        MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, SCRIPTS);
+        ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singletonList(scriptEngine));
+        ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList());
+        ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
+        ScriptService scriptService;
+        try {
+            scriptService =  new ScriptService(settings, new Environment(settings), null, scriptEngineRegistry, scriptContextRegistry,
+                    scriptSettings);
+        } catch (IOException e) {
+            throw new ElasticsearchException(e);
+        }
+        return new QueryShardContext(0, mapperService.getIndexSettings(), null, null, mapperService, null, scriptService,
+                xContentRegistry(), null, null, System::currentTimeMillis);
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/sum/InternalSumTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/sum/InternalSumTests.java
new file mode 100644
index 0000000000000..1df83b989bbea
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/sum/InternalSumTests.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.metrics.sum;
+
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.ParsedAggregation;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.test.InternalAggregationTestCase;
+
+import java.util.List;
+import java.util.Map;
+
+public class InternalSumTests extends InternalAggregationTestCase<InternalSum> {
+
+    @Override
+    protected InternalSum createTestInstance(String name, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
+        double value = frequently() ? randomDouble() : randomFrom(new Double[] { Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY });
+        DocValueFormat formatter = randomFrom(new DocValueFormat.Decimal("###.##"), DocValueFormat.BOOLEAN, DocValueFormat.RAW);
+        return new InternalSum(name, value, formatter, pipelineAggregators, metaData);
+    }
+
+    @Override
+    protected void assertFromXContent(InternalSum sum, ParsedAggregation parsedAggregation) {
+        ParsedSum parsed = ((ParsedSum) parsedAggregation);
+        assertEquals(sum.getValue(), parsed.getValue(), Double.MIN_VALUE);
+        assertEquals(sum.getValueAsString(), parsed.getValueAsString());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/tophits/InternalTopHitsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/tophits/InternalTopHitsTests.java
new file mode 100644
index 0000000000000..44f444728320d
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/tophits/InternalTopHitsTests.java
@@ -0,0 +1,214 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.tophits;
+
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.search.FieldComparator;
+import org.apache.lucene.search.FieldDoc;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.TopFieldDocs;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.common.text.Text;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchHitField;
+import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.search.aggregations.ParsedAggregation;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.test.InternalAggregationTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static java.lang.Math.max;
+import static java.lang.Math.min;
+import static java.util.Comparator.comparing;
+
+public class InternalTopHitsTests extends InternalAggregationTestCase<InternalTopHits> {
+    /**
+     * Should the test instances look like they are sorted by some fields (true) or sorted by score (false).
+     */
+    private boolean testInstancesLookSortedByField;
+    /**
+     * Fields shared by all instances created by {@link #createTestInstance(String, List, Map)}.
+     */
+    private SortField[] testInstancesSortFields;
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        testInstancesLookSortedByField = randomBoolean();
+        testInstancesSortFields = testInstancesLookSortedByField ? randomSortFields() : new SortField[0];
+    }
+
+    @Override
+    protected InternalTopHits createTestInstance(String name, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
+        int from = 0;
+        int requestedSize = between(1, 40);
+        int actualSize = between(0, requestedSize);
+
+        float maxScore = Float.MIN_VALUE;
+        ScoreDoc[] scoreDocs = new ScoreDoc[actualSize];
+        SearchHit[] hits = new SearchHit[actualSize];
+        Set<Integer> usedDocIds = new HashSet<>();
+        for (int i = 0; i < actualSize; i++) {
+            float score = randomFloat();
+            maxScore = max(maxScore, score);
+            int docId = randomValueOtherThanMany(usedDocIds::contains, () -> between(0, IndexWriter.MAX_DOCS));
+            usedDocIds.add(docId);
+
+            Map<String, SearchHitField> searchHitFields = new HashMap<>();
+            if (testInstancesLookSortedByField) {
+                Object[] fields = new Object[testInstancesSortFields.length];
+                for (int f = 0; f < testInstancesSortFields.length; f++) {
+                    fields[f] = randomOfType(testInstancesSortFields[f].getType());
+                }
+                scoreDocs[i] = new FieldDoc(docId, score, fields);
+            } else {
+                scoreDocs[i] = new ScoreDoc(docId, score);
+            }
+            hits[i] = new SearchHit(docId, Integer.toString(i), new Text("test"), searchHitFields);
+            hits[i].score(score);
+        }
+        int totalHits = between(actualSize, 500000);
+        SearchHits searchHits = new SearchHits(hits, totalHits, maxScore);
+
+        TopDocs topDocs;
+        Arrays.sort(scoreDocs, scoreDocComparator());
+        if (testInstancesLookSortedByField) {
+            topDocs = new TopFieldDocs(totalHits, scoreDocs, testInstancesSortFields, maxScore);
+        } else {
+            topDocs = new TopDocs(totalHits, scoreDocs, maxScore);
+        }
+
+        return new InternalTopHits(name, from, requestedSize, topDocs, searchHits, pipelineAggregators, metaData);
+    }
+
+    @Override
+    protected void assertFromXContent(InternalTopHits aggregation, ParsedAggregation parsedAggregation) throws IOException {
+        final SearchHits expectedSearchHits = aggregation.getHits();
+
+        assertTrue(parsedAggregation instanceof ParsedTopHits);
+        ParsedTopHits parsed = (ParsedTopHits) parsedAggregation;
+        final SearchHits actualSearchHits = parsed.getHits();
+
+        assertEquals(expectedSearchHits.getTotalHits(), actualSearchHits.getTotalHits());
+        assertEquals(expectedSearchHits.getMaxScore(), actualSearchHits.getMaxScore(), 0.0f);
+
+        List<SearchHit> expectedHits = Arrays.asList(expectedSearchHits.getHits());
+        List<SearchHit> actualHits = Arrays.asList(actualSearchHits.getHits());
+
+        assertEquals(expectedHits.size(), actualHits.size());
+        for (int i = 0; i < expectedHits.size(); i++) {
+            SearchHit expected = expectedHits.get(i);
+            SearchHit actual = actualHits.get(i);
+
+            assertEquals(expected.getIndex(), actual.getIndex());
+            assertEquals(expected.getType(), actual.getType());
+            assertEquals(expected.getId(), actual.getId());
+            assertEquals(expected.getVersion(), actual.getVersion());
+            assertEquals(expected.getScore(), actual.getScore(), 0.0f);
+            assertEquals(expected.getFields(), actual.getFields());
+            assertEquals(expected.getSourceAsMap(), actual.getSourceAsMap());
+        }
+    }
+
+    private Object randomOfType(SortField.Type type) {
+        switch (type) {
+        case CUSTOM:
+            throw new UnsupportedOperationException();
+        case DOC:
+            return between(0, IndexWriter.MAX_DOCS);
+        case DOUBLE:
+            return randomDouble();
+        case FLOAT:
+            return randomFloat();
+        case INT:
+            return randomInt();
+        case LONG:
+            return randomLong();
+        case REWRITEABLE:
+            throw new UnsupportedOperationException();
+        case SCORE:
+            return randomFloat();
+        case STRING:
+            return new BytesRef(randomAlphaOfLength(5));
+        case STRING_VAL:
+            return new BytesRef(randomAlphaOfLength(5));
+        default:
+            throw new UnsupportedOperationException("Unkown SortField.Type: " + type);
+        }
+    }
+
+    private SortField[] randomSortFields() {
+        SortField[] sortFields = new SortField[between(1, 5)];
+        Set<String> usedSortFields = new HashSet<>();
+        for (int i = 0; i < sortFields.length; i++) {
+            String sortField = randomValueOtherThanMany(usedSortFields::contains, () -> randomAlphaOfLength(5));
+            usedSortFields.add(sortField);
+            SortField.Type type = randomValueOtherThanMany(t -> t == SortField.Type.CUSTOM || t == SortField.Type.REWRITEABLE,
+                    () -> randomFrom(SortField.Type.values()));
+            sortFields[i] = new SortField(sortField, type);
+        }
+        return sortFields;
+    }
+
+    private Comparator<ScoreDoc> scoreDocComparator() {
+        return innerScoreDocComparator().thenComparing(s -> s.shardIndex);
+    }
+
+    private Comparator<ScoreDoc> innerScoreDocComparator() {
+        if (testInstancesLookSortedByField) {
+            // Values passed to getComparator shouldn't matter
+            @SuppressWarnings("rawtypes")
+            FieldComparator[] comparators = new FieldComparator[testInstancesSortFields.length];
+            for (int i = 0; i < testInstancesSortFields.length; i++) {
+                comparators[i] = testInstancesSortFields[i].getComparator(0, 0);
+            }
+            return (lhs, rhs) -> {
+                FieldDoc l = (FieldDoc) lhs;
+                FieldDoc r = (FieldDoc) rhs;
+                int i = 0;
+                while (i < l.fields.length) {
+                    @SuppressWarnings("unchecked")
+                    int c = comparators[i].compareValues(l.fields[i], r.fields[i]);
+                    if (c != 0) {
+                        return c;
+                    }
+                    i++;
+                }
+                return 0;
+            };
+        } else {
+            Comparator<ScoreDoc> comparator = comparing(d -> d.score);
+            return comparator.reversed();
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorTests.java
new file mode 100644
index 0000000000000..43686a1465e60
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorTests.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.metrics.tophits;
+
+import org.apache.lucene.analysis.core.KeywordAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.SortedSetDocValuesField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.queryparser.classic.QueryParser;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.MatchNoDocsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.index.mapper.KeywordFieldMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.Uid;
+import org.elasticsearch.index.mapper.UidFieldMapper;
+import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.search.aggregations.Aggregation;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.AggregatorTestCase;
+import org.elasticsearch.search.aggregations.bucket.terms.Terms;
+import org.elasticsearch.search.sort.SortOrder;
+
+import java.io.IOException;
+
+import static org.elasticsearch.search.aggregations.AggregationBuilders.terms;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.topHits;
+
+public class TopHitsAggregatorTests extends AggregatorTestCase {
+    public void testTopLevel() throws Exception {
+        Aggregation result;
+        if (randomBoolean()) {
+            result = testCase(new MatchAllDocsQuery(), topHits("_name").sort("string", SortOrder.DESC));
+        } else {
+            Query query = new QueryParser("string", new KeywordAnalyzer()).parse("d^1000 c^100 b^10 a^1");
+            result = testCase(query, topHits("_name"));
+        }
+        SearchHits searchHits = ((TopHits) result).getHits();
+        assertEquals(3L, searchHits.getTotalHits());
+        assertEquals("3", searchHits.getAt(0).getId());
+        assertEquals("type", searchHits.getAt(0).getType());
+        assertEquals("2", searchHits.getAt(1).getId());
+        assertEquals("type", searchHits.getAt(1).getType());
+        assertEquals("1", searchHits.getAt(2).getId());
+        assertEquals("type", searchHits.getAt(2).getType());
+    }
+
+    public void testNoResults() throws Exception {
+        TopHits result = (TopHits) testCase(new MatchNoDocsQuery(), topHits("_name").sort("string", SortOrder.DESC));
+        SearchHits searchHits = ((TopHits) result).getHits();
+        assertEquals(0L, searchHits.getTotalHits());
+    }
+
+    /**
+     * Tests {@code top_hits} inside of {@code terms}. While not strictly a unit test this is a fairly common way to run {@code top_hits}
+     * and serves as a good example of running {@code top_hits} inside of another aggregation.
+     */
+    public void testInsideTerms() throws Exception {
+        Aggregation result;
+        if (randomBoolean()) {
+            result = testCase(new MatchAllDocsQuery(),
+                    terms("term").field("string")
+                        .subAggregation(topHits("top").sort("string", SortOrder.DESC)));
+        } else {
+            Query query = new QueryParser("string", new KeywordAnalyzer()).parse("d^1000 c^100 b^10 a^1");
+            result = testCase(query,
+                    terms("term").field("string")
+                        .subAggregation(topHits("top")));
+        }
+        Terms terms = (Terms) result;
+
+        // The "a" bucket
+        TopHits hits = (TopHits) terms.getBucketByKey("a").getAggregations().get("top");
+        SearchHits searchHits = (hits).getHits();
+        assertEquals(2L, searchHits.getTotalHits());
+        assertEquals("2", searchHits.getAt(0).getId());
+        assertEquals("1", searchHits.getAt(1).getId());
+
+        // The "b" bucket
+        searchHits = ((TopHits) terms.getBucketByKey("b").getAggregations().get("top")).getHits();
+        assertEquals(2L, searchHits.getTotalHits());
+        assertEquals("3", searchHits.getAt(0).getId());
+        assertEquals("1", searchHits.getAt(1).getId());
+
+        // The "c" bucket
+        searchHits = ((TopHits) terms.getBucketByKey("c").getAggregations().get("top")).getHits();
+        assertEquals(1L, searchHits.getTotalHits());
+        assertEquals("2", searchHits.getAt(0).getId());
+
+        // The "d" bucket
+        searchHits = ((TopHits) terms.getBucketByKey("d").getAggregations().get("top")).getHits();
+        assertEquals(1L, searchHits.getTotalHits());
+        assertEquals("3", searchHits.getAt(0).getId());
+    }
+
+    private static final MappedFieldType STRING_FIELD_TYPE = new KeywordFieldMapper.KeywordFieldType();
+    static {
+        STRING_FIELD_TYPE.setName("string");
+        STRING_FIELD_TYPE.setHasDocValues(true);
+    }
+
+    private Aggregation testCase(Query query, AggregationBuilder builder) throws IOException {
+        Directory directory = newDirectory();
+        RandomIndexWriter iw = new RandomIndexWriter(random(), directory);
+        iw.addDocument(document("1", "a", "b"));
+        iw.addDocument(document("2", "c", "a"));
+        iw.addDocument(document("3", "b", "d"));
+        iw.close();
+
+        IndexReader indexReader = DirectoryReader.open(directory);
+        // We do not use LuceneTestCase.newSearcher because we need a DirectoryReader for "testInsideTerms"
+        IndexSearcher indexSearcher = new IndexSearcher(indexReader);
+
+        Aggregation result = searchAndReduce(indexSearcher, query, builder, STRING_FIELD_TYPE);
+        indexReader.close();
+        directory.close();
+        return result;
+    }
+
+    private Document document(String id, String... stringValues) {
+        Document document = new Document();
+        document.add(new Field(UidFieldMapper.NAME, Uid.createUid("type", id), UidFieldMapper.Defaults.FIELD_TYPE));
+        for (String stringValue : stringValues) {
+            document.add(new Field("string", stringValue, STRING_FIELD_TYPE));
+            document.add(new SortedSetDocValuesField("string", new BytesRef(stringValue)));
+        }
+        return document;
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/valuecount/InternalValueCountTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/valuecount/InternalValueCountTests.java
new file mode 100644
index 0000000000000..9e3f5c2db1f17
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/valuecount/InternalValueCountTests.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.valuecount;
+
+import org.elasticsearch.search.aggregations.ParsedAggregation;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.test.InternalAggregationTestCase;
+
+import java.util.List;
+import java.util.Map;
+
+public class InternalValueCountTests extends InternalAggregationTestCase<InternalValueCount> {
+
+    @Override
+    protected InternalValueCount createTestInstance(String name, List<PipelineAggregator> pipelineAggregators,
+            Map<String, Object> metaData) {
+        return new InternalValueCount(name, randomIntBetween(0, 100), pipelineAggregators, metaData);
+    }
+
+    @Override
+    protected void assertFromXContent(InternalValueCount valueCount, ParsedAggregation parsedAggregation) {
+        assertEquals(valueCount.getValue(), ((ParsedValueCount) parsedAggregation).getValue(), 0);
+        assertEquals(valueCount.getValueAsString(), ((ParsedValueCount) parsedAggregation).getValueAsString());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountAggregatorTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountAggregatorTests.java
new file mode 100644
index 0000000000000..bc60524b28d8b
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountAggregatorTests.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.valuecount;
+
+import org.apache.lucene.document.IntPoint;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.SortedNumericDocValuesField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.search.FieldValueQuery;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.CheckedConsumer;
+import org.elasticsearch.index.mapper.BooleanFieldMapper;
+import org.elasticsearch.index.mapper.DateFieldMapper;
+import org.elasticsearch.index.mapper.IpFieldMapper;
+import org.elasticsearch.index.mapper.KeywordFieldMapper;
+import org.elasticsearch.index.mapper.LatLonPointFieldMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.NumberFieldMapper;
+import org.elasticsearch.search.aggregations.AggregatorTestCase;
+import org.elasticsearch.search.aggregations.support.ValueType;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.function.Consumer;
+
+import static java.util.Collections.singleton;
+
+public class ValueCountAggregatorTests extends AggregatorTestCase {
+
+    private static final String FIELD_NAME = "field";
+
+    public void testNoDocs() throws IOException {
+        for (ValueType valueType : ValueType.values()) {
+            testCase(new MatchAllDocsQuery(), valueType, iw -> {
+                // Intentionally not writing any docs
+            }, count -> assertEquals(0L, count.getValue()));
+        }
+    }
+
+    public void testNoMatchingField() throws IOException {
+        testCase(new MatchAllDocsQuery(), ValueType.LONG, iw -> {
+            iw.addDocument(singleton(new SortedNumericDocValuesField("wrong_number", 7)));
+            iw.addDocument(singleton(new SortedNumericDocValuesField("wrong_number", 1)));
+        }, count -> assertEquals(0L, count.getValue()));
+    }
+
+    public void testSomeMatchesSortedNumericDocValues() throws IOException {
+        testCase(new FieldValueQuery(FIELD_NAME), ValueType.NUMERIC, iw -> {
+            iw.addDocument(singleton(new SortedNumericDocValuesField("wrong_number", 7)));
+            iw.addDocument(singleton(new SortedNumericDocValuesField(FIELD_NAME, 7)));
+            iw.addDocument(singleton(new SortedNumericDocValuesField(FIELD_NAME, 1)));
+        }, count -> assertEquals(2L, count.getValue()));
+    }
+
+    public void testSomeMatchesNumericDocValues() throws IOException {
+        testCase(new FieldValueQuery(FIELD_NAME), ValueType.NUMBER, iw -> {
+            iw.addDocument(singleton(new NumericDocValuesField(FIELD_NAME, 7)));
+            iw.addDocument(singleton(new NumericDocValuesField(FIELD_NAME, 1)));
+        }, count -> assertEquals(2L, count.getValue()));
+    }
+
+    public void testQueryFiltering() throws IOException {
+        testCase(IntPoint.newRangeQuery("level", 0, 5), ValueType.STRING, iw -> {
+            iw.addDocument(Arrays.asList(new IntPoint("level", 0), new SortedDocValuesField(FIELD_NAME, new BytesRef("foo"))));
+            iw.addDocument(Arrays.asList(new IntPoint("level", 1), new SortedDocValuesField(FIELD_NAME, new BytesRef("bar"))));
+            iw.addDocument(Arrays.asList(new IntPoint("level", 3), new SortedDocValuesField(FIELD_NAME, new BytesRef("foo"))));
+            iw.addDocument(Arrays.asList(new IntPoint("level", 5), new SortedDocValuesField(FIELD_NAME, new BytesRef("baz"))));
+            iw.addDocument(Arrays.asList(new IntPoint("level", 7), new SortedDocValuesField(FIELD_NAME, new BytesRef("baz"))));
+        }, count -> assertEquals(4L, count.getValue()));
+    }
+
+    public void testQueryFiltersAll() throws IOException {
+        testCase(IntPoint.newRangeQuery("level", -1, 0), ValueType.STRING, iw -> {
+            iw.addDocument(Arrays.asList(new IntPoint("level", 3), new SortedDocValuesField(FIELD_NAME, new BytesRef("foo"))));
+            iw.addDocument(Arrays.asList(new IntPoint("level", 5), new SortedDocValuesField(FIELD_NAME, new BytesRef("baz"))));
+        }, count -> assertEquals(0L, count.getValue()));
+    }
+
+    private void testCase(Query query,
+                          ValueType valueType,
+                          CheckedConsumer<RandomIndexWriter, IOException> indexer,
+                          Consumer<ValueCount> verify) throws IOException {
+
+        try (Directory directory = newDirectory()) {
+            try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
+                indexer.accept(indexWriter);
+            }
+
+            try (IndexReader indexReader = DirectoryReader.open(directory)) {
+                IndexSearcher indexSearcher = newSearcher(indexReader, true, true);
+
+                MappedFieldType fieldType = createMappedFieldType(valueType);
+                fieldType.setName(FIELD_NAME);
+                fieldType.setHasDocValues(true);
+
+                ValueCountAggregationBuilder aggregationBuilder = new ValueCountAggregationBuilder("_name", valueType);
+                aggregationBuilder.field(FIELD_NAME);
+
+                try (ValueCountAggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType)) {
+                    aggregator.preCollection();
+                    indexSearcher.search(query, aggregator);
+                    aggregator.postCollection();
+
+                    verify.accept((ValueCount) aggregator.buildAggregation(0L));
+                }
+            }
+        }
+    }
+
+    private static MappedFieldType createMappedFieldType(ValueType valueType) {
+        switch (valueType) {
+            case BOOLEAN:
+                return new BooleanFieldMapper.BooleanFieldType();
+            case STRING:
+                return new KeywordFieldMapper.KeywordFieldType();
+            case DOUBLE:
+                return new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.DOUBLE);
+            case NUMBER:
+            case NUMERIC:
+            case LONG:
+                return new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
+            case DATE:
+                return new DateFieldMapper.Builder("_name").fieldType();
+            case IP:
+                return new IpFieldMapper.Builder("_name").fieldType();
+            case GEOPOINT:
+                return new LatLonPointFieldMapper.Builder("_name").fieldType();
+            default:
+                throw new IllegalArgumentException("Test does not support value type [" + valueType + "]");
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/AvgBucketIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/AvgBucketIT.java
index d62e93fa5e9ac..4f6ff0e32eda8 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/AvgBucketIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/AvgBucketIT.java
@@ -140,7 +140,7 @@ public void testDocCountAsSubAgg() throws Exception {
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
-        List<Terms.Bucket> termsBuckets = terms.getBuckets();
+        List<? extends Terms.Bucket> termsBuckets = terms.getBuckets();
         assertThat(termsBuckets.size(), equalTo(interval));
 
         for (int i = 0; i < interval; ++i) {
@@ -182,7 +182,7 @@ public void testMetricTopLevel() throws Exception {
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
-        List<Terms.Bucket> buckets = terms.getBuckets();
+        List<? extends Terms.Bucket> buckets = terms.getBuckets();
         assertThat(buckets.size(), equalTo(interval));
 
         double bucketSum = 0;
@@ -223,7 +223,7 @@ public void testMetricAsSubAgg() throws Exception {
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
-        List<Terms.Bucket> termsBuckets = terms.getBuckets();
+        List<? extends Terms.Bucket> termsBuckets = terms.getBuckets();
         assertThat(termsBuckets.size(), equalTo(interval));
 
         for (int i = 0; i < interval; ++i) {
@@ -277,7 +277,7 @@ public void testMetricAsSubAggWithInsertZeros() throws Exception {
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
-        List<Terms.Bucket> termsBuckets = terms.getBuckets();
+        List<? extends Terms.Bucket> termsBuckets = terms.getBuckets();
         assertThat(termsBuckets.size(), equalTo(interval));
 
         for (int i = 0; i < interval; ++i) {
@@ -322,7 +322,7 @@ public void testNoBuckets() throws Exception {
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
-        List<Terms.Bucket> buckets = terms.getBuckets();
+        List<? extends Terms.Bucket> buckets = terms.getBuckets();
         assertThat(buckets.size(), equalTo(0));
 
         InternalSimpleValue avgBucketValue = response.getAggregations().get("avg_bucket");
@@ -349,7 +349,7 @@ public void testNested() throws Exception {
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
-        List<Terms.Bucket> termsBuckets = terms.getBuckets();
+        List<? extends Terms.Bucket> termsBuckets = terms.getBuckets();
         assertThat(termsBuckets.size(), equalTo(interval));
 
         double aggTermsSum = 0;
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptIT.java
index 101e52fcadb72..5b8ea2d5c67d9 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptIT.java
@@ -23,16 +23,16 @@
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.MockScriptPlugin;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
 import org.elasticsearch.search.aggregations.bucket.range.Range;
 import org.elasticsearch.search.aggregations.metrics.sum.Sum;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
 import org.elasticsearch.test.ESIntegTestCase;
-import org.joda.time.DateTime;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -165,8 +165,9 @@ public void testInlineScript() {
                                 .subAggregation(sum("field4Sum").field(FIELD_4_NAME))
                                 .subAggregation(
                                         bucketScript("seriesArithmetic",
-                                                new Script("_value0 + _value1 + _value2", ScriptType.INLINE, CustomScriptPlugin.NAME, null)
-                                                , "field2Sum", "field3Sum", "field4Sum")))
+                                                new Script(ScriptType.INLINE,
+                                                    CustomScriptPlugin.NAME, "_value0 + _value1 + _value2", Collections.emptyMap()),
+                                            "field2Sum", "field3Sum", "field4Sum")))
                 .execute().actionGet();
 
         assertSearchResponse(response);
@@ -211,7 +212,8 @@ public void testInlineScript2() {
                                 .subAggregation(sum("field4Sum").field(FIELD_4_NAME))
                                 .subAggregation(
                                         bucketScript("seriesArithmetic",
-                                                new Script("_value0 + _value1 / _value2", ScriptType.INLINE, CustomScriptPlugin.NAME, null),
+                                                new Script(ScriptType.INLINE,
+                                                    CustomScriptPlugin.NAME, "_value0 + _value1 / _value2", Collections.emptyMap()),
                                                 "field2Sum", "field3Sum", "field4Sum")))
                 .execute().actionGet();
 
@@ -257,7 +259,7 @@ public void testInlineScriptWithDateRange() {
                     .subAggregation(sum("field4Sum").field(FIELD_4_NAME))
                     .subAggregation(
                         bucketScript("seriesArithmetic",
-                            new Script("_value0 + _value1 + _value2", ScriptType.INLINE, CustomScriptPlugin.NAME, null)
+                            new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_value0 + _value1 + _value2", Collections.emptyMap())
                             , "field2Sum", "field3Sum", "field4Sum")))
             .execute().actionGet();
 
@@ -301,7 +303,7 @@ public void testInlineScriptSingleVariable() {
                                 .subAggregation(sum("field2Sum").field(FIELD_2_NAME))
                                 .subAggregation(
                                         bucketScript("seriesArithmetic",
-                                                new Script("_value0", ScriptType.INLINE, CustomScriptPlugin.NAME, null),
+                                                new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_value0", Collections.emptyMap()),
                                                 "field2Sum")))
                 .execute().actionGet();
 
@@ -345,7 +347,8 @@ public void testInlineScriptNamedVars() {
                                 .subAggregation(sum("field4Sum").field(FIELD_4_NAME))
                                 .subAggregation(
                                         bucketScript("seriesArithmetic", bucketsPathsMap,
-                                                new Script("foo + bar + baz", ScriptType.INLINE, CustomScriptPlugin.NAME, null))))
+                                                new Script(ScriptType.INLINE,
+                                                    CustomScriptPlugin.NAME, "foo + bar + baz", Collections.emptyMap()))))
                 .execute().actionGet();
 
         assertSearchResponse(response);
@@ -382,7 +385,7 @@ public void testInlineScriptWithParams() {
         Map<String, Object> params = new HashMap<>();
         params.put("factor", 3);
 
-        Script script = new Script("(_value0 + _value1 + _value2) * factor", ScriptType.INLINE, CustomScriptPlugin.NAME, params);
+        Script script = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "(_value0 + _value1 + _value2) * factor", params);
 
         SearchResponse response = client()
                 .prepareSearch("idx")
@@ -438,7 +441,8 @@ public void testInlineScriptInsertZeros() {
                                 .subAggregation(sum("field4Sum").field(FIELD_4_NAME))
                                 .subAggregation(
                                         bucketScript("seriesArithmetic",
-                                                new Script("_value0 + _value1 + _value2", ScriptType.INLINE, CustomScriptPlugin.NAME, null),
+                                                new Script(ScriptType.INLINE,
+                                                    CustomScriptPlugin.NAME, "_value0 + _value1 + _value2", Collections.emptyMap()),
                                                 "field2Sum", "field3Sum", "field4Sum").gapPolicy(GapPolicy.INSERT_ZEROS)))
                 .execute().actionGet();
 
@@ -477,9 +481,9 @@ public void testInlineScriptInsertZeros() {
     public void testStoredScript() {
         assertAcked(client().admin().cluster().preparePutStoredScript()
                 .setId("my_script")
-                .setScriptLang(CustomScriptPlugin.NAME)
+                .setLang(CustomScriptPlugin.NAME)
                 // Script source is not interpreted but it references a pre-defined script from CustomScriptPlugin
-                .setSource(new BytesArray("{ \"script\": \"my_script\" }")));
+                .setContent(new BytesArray("{ \"script\": \"my_script\" }"), XContentType.JSON));
 
         SearchResponse response = client()
                 .prepareSearch("idx")
@@ -492,7 +496,7 @@ public void testStoredScript() {
                                 .subAggregation(sum("field4Sum").field(FIELD_4_NAME))
                                 .subAggregation(
                                         bucketScript("seriesArithmetic",
-                                                new Script("my_script", ScriptType.STORED, CustomScriptPlugin.NAME, null),
+                                                new Script(ScriptType.STORED, CustomScriptPlugin.NAME, "my_script", Collections.emptyMap()),
                                                 "field2Sum", "field3Sum", "field4Sum"))).execute().actionGet();
 
         assertSearchResponse(response);
@@ -537,7 +541,8 @@ public void testUnmapped() throws Exception {
                                 .subAggregation(sum("field4Sum").field(FIELD_4_NAME))
                                 .subAggregation(
                                         bucketScript("seriesArithmetic",
-                                                new Script("_value0 + _value1 + _value2", ScriptType.INLINE, CustomScriptPlugin.NAME, null),
+                                                new Script(ScriptType.INLINE,
+                                                    CustomScriptPlugin.NAME, "_value0 + _value1 + _value2", Collections.emptyMap()),
                                                 "field2Sum", "field3Sum", "field4Sum")))
                                 .execute().actionGet();
 
@@ -561,7 +566,8 @@ public void testPartiallyUnmapped() throws Exception {
                                 .subAggregation(sum("field4Sum").field(FIELD_4_NAME))
                                 .subAggregation(
                                         bucketScript("seriesArithmetic",
-                                                new Script("_value0 + _value1 + _value2", ScriptType.INLINE, CustomScriptPlugin.NAME, null),
+                                                new Script(ScriptType.INLINE,
+                                                    CustomScriptPlugin.NAME, "_value0 + _value1 + _value2", Collections.emptyMap()),
                                                 "field2Sum", "field3Sum", "field4Sum"))).execute().actionGet();
 
         assertSearchResponse(response);
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptTests.java
index 252c1247a490c..c213619183be1 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptTests.java
@@ -20,7 +20,7 @@
 package org.elasticsearch.search.aggregations.pipeline;
 
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.aggregations.BasePipelineAggregationTestCase;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
 import org.elasticsearch.search.aggregations.pipeline.bucketscript.BucketScriptPipelineAggregationBuilder;
@@ -32,26 +32,27 @@ public class BucketScriptTests extends BasePipelineAggregationTestCase<BucketScr
 
     @Override
     protected BucketScriptPipelineAggregationBuilder createTestAggregatorFactory() {
-        String name = randomAsciiOfLengthBetween(3, 20);
+        String name = randomAlphaOfLengthBetween(3, 20);
         Map<String, String> bucketsPaths = new HashMap<>();
         int numBucketPaths = randomIntBetween(1, 10);
         for (int i = 0; i < numBucketPaths; i++) {
-            bucketsPaths.put(randomAsciiOfLengthBetween(1, 20), randomAsciiOfLengthBetween(1, 40));
+            bucketsPaths.put(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 40));
         }
         Script script;
         if (randomBoolean()) {
-            script = new Script("script");
+            script = mockScript("script");
         } else {
-            Map<String, Object> params = null;
+            Map<String, Object> params = new HashMap<>();
             if (randomBoolean()) {
-                params = new HashMap<String, Object>();
                 params.put("foo", "bar");
             }
-            script = new Script("script", randomFrom(ScriptType.values()), randomFrom("my_lang", null), params);
+            ScriptType type = randomFrom(ScriptType.values());
+            script = new Script(type, type == ScriptType.STORED ? null : randomFrom("my_lang", Script.DEFAULT_SCRIPT_LANG),
+                "script", params);
         }
         BucketScriptPipelineAggregationBuilder factory = new BucketScriptPipelineAggregationBuilder(name, bucketsPaths, script);
         if (randomBoolean()) {
-            factory.format(randomAsciiOfLengthBetween(1, 10));
+            factory.format(randomAlphaOfLengthBetween(1, 10));
         }
         if (randomBoolean()) {
             factory.gapPolicy(randomFrom(GapPolicy.values()));
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorIT.java
index 26c0d3a4bbef2..68904c94ca396 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorIT.java
@@ -23,11 +23,11 @@
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.MockScriptPlugin;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService.ScriptType;
-import org.elasticsearch.search.aggregations.bucket.histogram.ExtendedBounds;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket;
 import org.elasticsearch.search.aggregations.metrics.sum.Sum;
@@ -177,8 +177,8 @@ private XContentBuilder newDocBuilder(int field1Value, int field2Value, int fiel
     }
 
     public void testInlineScript() {
-        Script script =
-                new Script("Double.isNaN(_value0) ? false : (_value0 + _value1 > 100)", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
+        Script script = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME,
+            "Double.isNaN(_value0) ? false : (_value0 + _value1 > 100)", Collections.emptyMap());
 
         SearchResponse response = client().prepareSearch("idx")
                 .addAggregation(histogram("histo").field(FIELD_1_NAME).interval(interval)
@@ -206,7 +206,8 @@ public void testInlineScript() {
     }
 
     public void testInlineScriptNoBucketsPruned() {
-        Script script = new Script("Double.isNaN(_value0) ? true : (_value0 < 10000)", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
+        Script script = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME,
+            "Double.isNaN(_value0) ? true : (_value0 < 10000)", Collections.emptyMap());
 
         SearchResponse response = client()
                 .prepareSearch("idx")
@@ -239,7 +240,8 @@ public void testInlineScriptNoBucketsPruned() {
     }
 
     public void testInlineScriptNoBucketsLeft() {
-        Script script = new Script("Double.isNaN(_value0) ? false : (_value0 > 10000)", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
+        Script script = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME,
+            "Double.isNaN(_value0) ? false : (_value0 > 10000)", Collections.emptyMap());
 
         SearchResponse response = client()
                 .prepareSearch("idx")
@@ -262,7 +264,8 @@ public void testInlineScriptNoBucketsLeft() {
     }
 
     public void testInlineScript2() {
-        Script script = new Script("Double.isNaN(_value0) ? false : (_value0 < _value1)", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
+        Script script = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME,
+            "Double.isNaN(_value0) ? false : (_value0 < _value1)", Collections.emptyMap());
 
         SearchResponse response = client()
                 .prepareSearch("idx")
@@ -295,7 +298,8 @@ public void testInlineScript2() {
     }
 
     public void testInlineScriptSingleVariable() {
-        Script script = new Script("Double.isNaN(_value0) ? false : (_value0 > 100)", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
+        Script script = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME,
+            "Double.isNaN(_value0) ? false : (_value0 > 100)", Collections.emptyMap());
 
         SearchResponse response = client()
                 .prepareSearch("idx")
@@ -324,8 +328,8 @@ public void testInlineScriptSingleVariable() {
     }
 
     public void testInlineScriptNamedVars() {
-        Script script = new Script("Double.isNaN(my_value1) ? false : (my_value1 + my_value2 > 100)", ScriptType.INLINE,
-                CustomScriptPlugin.NAME, null);
+        Script script = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME,
+            "Double.isNaN(my_value1) ? false : (my_value1 + my_value2 > 100)", Collections.emptyMap());
 
         Map<String, String> bucketPathsMap = new HashMap<>();
         bucketPathsMap.put("my_value1", "field2Sum");
@@ -361,8 +365,8 @@ public void testInlineScriptNamedVars() {
     }
 
     public void testInlineScriptWithParams() {
-        Script script = new Script("Double.isNaN(_value0) ? false : (_value0 + _value1 > threshold)", ScriptType.INLINE,
-                CustomScriptPlugin.NAME, Collections.singletonMap("threshold", 100));
+        Script script = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME,
+            "Double.isNaN(_value0) ? false : (_value0 + _value1 > threshold)", Collections.singletonMap("threshold", 100));
 
         SearchResponse response = client().prepareSearch("idx")
                 .addAggregation(
@@ -394,7 +398,7 @@ public void testInlineScriptWithParams() {
     }
 
     public void testInlineScriptInsertZeros() {
-        Script script = new Script("_value0 + _value1 > 100", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
+        Script script = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_value0 + _value1 > 100", Collections.emptyMap());
 
         SearchResponse response = client().prepareSearch("idx")
                 .addAggregation(
@@ -429,11 +433,12 @@ public void testInlineScriptInsertZeros() {
     public void testStoredScript() {
         assertAcked(client().admin().cluster().preparePutStoredScript()
                 .setId("my_script")
-                .setScriptLang(CustomScriptPlugin.NAME)
+                .setLang(CustomScriptPlugin.NAME)
                 // Source is not interpreted but my_script is defined in CustomScriptPlugin
-                .setSource(new BytesArray("{ \"script\": \"Double.isNaN(_value0) ? false : (_value0 + _value1 > 100)\" }")));
+                .setContent(new BytesArray("{ \"script\": \"Double.isNaN(_value0) ? false : (_value0 + _value1 > 100)\" }"),
+                    XContentType.JSON));
 
-        Script script = new Script("my_script", ScriptType.STORED, CustomScriptPlugin.NAME, null);
+        Script script = new Script(ScriptType.STORED, CustomScriptPlugin.NAME, "my_script", Collections.emptyMap());
 
         SearchResponse response = client()
                 .prepareSearch("idx")
@@ -466,8 +471,8 @@ public void testStoredScript() {
     }
 
     public void testUnmapped() throws Exception {
-        Script script = new Script("Double.isNaN(_value0) ? false : (_value0 + _value1 > 100)", ScriptType.INLINE,
-                CustomScriptPlugin.NAME, null);
+        Script script = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME,
+            "Double.isNaN(_value0) ? false : (_value0 + _value1 > 100)", Collections.emptyMap());
 
         SearchResponse response = client().prepareSearch("idx_unmapped")
                 .addAggregation(
@@ -488,8 +493,8 @@ public void testUnmapped() throws Exception {
     }
 
     public void testPartiallyUnmapped() throws Exception {
-        Script script = new Script("Double.isNaN(_value0) ? false : (_value0 + _value1 > 100)", ScriptType.INLINE,
-                CustomScriptPlugin.NAME, null);
+        Script script = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME,
+            "Double.isNaN(_value0) ? false : (_value0 + _value1 > 100)", Collections.emptyMap());
 
         SearchResponse response = client().prepareSearch("idx", "idx_unmapped")
                 .addAggregation(
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorTests.java
index 5ffbda13f9c94..8dd63942d866f 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorTests.java
@@ -20,7 +20,7 @@
 package org.elasticsearch.search.aggregations.pipeline;
 
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.aggregations.BasePipelineAggregationTestCase;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
 import org.elasticsearch.search.aggregations.pipeline.bucketselector.BucketSelectorPipelineAggregationBuilder;
@@ -32,22 +32,23 @@ public class BucketSelectorTests extends BasePipelineAggregationTestCase<BucketS
 
     @Override
     protected BucketSelectorPipelineAggregationBuilder createTestAggregatorFactory() {
-        String name = randomAsciiOfLengthBetween(3, 20);
+        String name = randomAlphaOfLengthBetween(3, 20);
         Map<String, String> bucketsPaths = new HashMap<>();
         int numBucketPaths = randomIntBetween(1, 10);
         for (int i = 0; i < numBucketPaths; i++) {
-            bucketsPaths.put(randomAsciiOfLengthBetween(1, 20), randomAsciiOfLengthBetween(1, 40));
+            bucketsPaths.put(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 40));
         }
         Script script;
         if (randomBoolean()) {
-            script = new Script("script");
+            script = mockScript("script");
         } else {
-            Map<String, Object> params = null;
+            Map<String, Object> params = new HashMap<>();
             if (randomBoolean()) {
-                params = new HashMap<String, Object>();
                 params.put("foo", "bar");
             }
-            script = new Script("script", randomFrom(ScriptType.values()), randomFrom("my_lang", null), params);
+            ScriptType type = randomFrom(ScriptType.values());
+            script =
+                new Script(type, type == ScriptType.STORED ? null : randomFrom("my_lang", Script.DEFAULT_SCRIPT_LANG), "script", params);
         }
         BucketSelectorPipelineAggregationBuilder factory = new BucketSelectorPipelineAggregationBuilder(name, bucketsPaths, script);
         if (randomBoolean()) {
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumTests.java
index 07cd9010ed0e8..3b1514a8c38fd 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumTests.java
@@ -26,11 +26,11 @@ public class CumulativeSumTests extends BasePipelineAggregationTestCase<Cumulati
 
     @Override
     protected CumulativeSumPipelineAggregationBuilder createTestAggregatorFactory() {
-        String name = randomAsciiOfLengthBetween(3, 20);
-        String bucketsPath = randomAsciiOfLengthBetween(3, 20);
+        String name = randomAlphaOfLengthBetween(3, 20);
+        String bucketsPath = randomAlphaOfLengthBetween(3, 20);
         CumulativeSumPipelineAggregationBuilder factory = new CumulativeSumPipelineAggregationBuilder(name, bucketsPath);
         if (randomBoolean()) {
-            factory.format(randomAsciiOfLengthBetween(1, 10));
+            factory.format(randomAlphaOfLengthBetween(1, 10));
         }
         return factory;
     }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/DateDerivativeIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/DateDerivativeIT.java
index 3d4394a0f3b96..34bf83b122b1c 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/DateDerivativeIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/DateDerivativeIT.java
@@ -22,6 +22,8 @@
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.index.mapper.DateFieldMapper;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket;
@@ -349,9 +351,9 @@ public void testSingleValuedFieldWithSubAggregation() throws Exception {
         assertThat(histo.getName(), equalTo("histo"));
         List<? extends Bucket> buckets = histo.getBuckets();
         assertThat(buckets.size(), equalTo(3));
-        Object[] propertiesKeys = (Object[]) histo.getProperty("_key");
-        Object[] propertiesDocCounts = (Object[]) histo.getProperty("_count");
-        Object[] propertiesCounts = (Object[]) histo.getProperty("sum.value");
+        Object[] propertiesKeys = (Object[]) ((InternalAggregation)histo).getProperty("_key");
+        Object[] propertiesDocCounts = (Object[]) ((InternalAggregation)histo).getProperty("_count");
+        Object[] propertiesCounts = (Object[]) ((InternalAggregation)histo).getProperty("sum.value");
 
         DateTime key = new DateTime(2012, 1, 1, 0, 0, DateTimeZone.UTC);
         Histogram.Bucket bucket = buckets.get(0);
@@ -380,7 +382,8 @@ public void testSingleValuedFieldWithSubAggregation() throws Exception {
         deriv = bucket.getAggregations().get("deriv");
         assertThat(deriv, notNullValue());
         assertThat(deriv.value(), equalTo(4.0));
-        assertThat((double) bucket.getProperty("histo", AggregationPath.parse("deriv.value").getPathElementsAsStringList()), equalTo(4.0));
+        assertThat(((InternalMultiBucketAggregation.InternalBucket)bucket).getProperty(
+                "histo", AggregationPath.parse("deriv.value").getPathElementsAsStringList()), equalTo(4.0));
         assertThat((DateTime) propertiesKeys[1], equalTo(key));
         assertThat((long) propertiesDocCounts[1], equalTo(2L));
         assertThat((double) propertiesCounts[1], equalTo(5.0));
@@ -397,7 +400,8 @@ public void testSingleValuedFieldWithSubAggregation() throws Exception {
         deriv = bucket.getAggregations().get("deriv");
         assertThat(deriv, notNullValue());
         assertThat(deriv.value(), equalTo(10.0));
-        assertThat((double) bucket.getProperty("histo", AggregationPath.parse("deriv.value").getPathElementsAsStringList()), equalTo(10.0));
+        assertThat(((InternalMultiBucketAggregation.InternalBucket)bucket).getProperty(
+                "histo", AggregationPath.parse("deriv.value").getPathElementsAsStringList()), equalTo(10.0));
         assertThat((DateTime) propertiesKeys[2], equalTo(key));
         assertThat((long) propertiesDocCounts[2], equalTo(3L));
         assertThat((double) propertiesCounts[2], equalTo(15.0));
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/DerivativeIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/DerivativeIT.java
index 5a780712f0246..5561b61f49288 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/DerivativeIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/DerivativeIT.java
@@ -25,6 +25,8 @@
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket;
 import org.elasticsearch.search.aggregations.metrics.stats.Stats;
@@ -259,9 +261,9 @@ public void testSingleValueAggDerivative() throws Exception {
         assertThat(deriv, notNullValue());
         assertThat(deriv.getName(), equalTo("histo"));
         assertThat(deriv.getBuckets().size(), equalTo(numValueBuckets));
-        Object[] propertiesKeys = (Object[]) deriv.getProperty("_key");
-        Object[] propertiesDocCounts = (Object[]) deriv.getProperty("_count");
-        Object[] propertiesSumCounts = (Object[]) deriv.getProperty("sum.value");
+        Object[] propertiesKeys = (Object[]) ((InternalAggregation)deriv).getProperty("_key");
+        Object[] propertiesDocCounts = (Object[]) ((InternalAggregation)deriv).getProperty("_count");
+        Object[] propertiesSumCounts = (Object[]) ((InternalAggregation)deriv).getProperty("sum.value");
 
         List<Bucket> buckets = new ArrayList<Bucket>(deriv.getBuckets());
         Long expectedSumPreviousBucket = Long.MIN_VALUE; // start value, gets
@@ -278,7 +280,8 @@ public void testSingleValueAggDerivative() throws Exception {
                 assertThat(sumDeriv, notNullValue());
                 long sumDerivValue = expectedSum - expectedSumPreviousBucket;
                 assertThat(sumDeriv.value(), equalTo((double) sumDerivValue));
-                assertThat((double) bucket.getProperty("histo", AggregationPath.parse("deriv.value").getPathElementsAsStringList()),
+                assertThat(((InternalMultiBucketAggregation.InternalBucket)bucket).getProperty("histo",
+                        AggregationPath.parse("deriv.value").getPathElementsAsStringList()),
                         equalTo((double) sumDerivValue));
             } else {
                 assertThat(sumDeriv, nullValue());
@@ -304,9 +307,9 @@ public void testMultiValueAggDerivative() throws Exception {
         assertThat(deriv, notNullValue());
         assertThat(deriv.getName(), equalTo("histo"));
         assertThat(deriv.getBuckets().size(), equalTo(numValueBuckets));
-        Object[] propertiesKeys = (Object[]) deriv.getProperty("_key");
-        Object[] propertiesDocCounts = (Object[]) deriv.getProperty("_count");
-        Object[] propertiesSumCounts = (Object[]) deriv.getProperty("stats.sum");
+        Object[] propertiesKeys = (Object[]) ((InternalAggregation)deriv).getProperty("_key");
+        Object[] propertiesDocCounts = (Object[]) ((InternalAggregation)deriv).getProperty("_count");
+        Object[] propertiesSumCounts = (Object[]) ((InternalAggregation)deriv).getProperty("stats.sum");
 
         List<Bucket> buckets = new ArrayList<Bucket>(deriv.getBuckets());
         Long expectedSumPreviousBucket = Long.MIN_VALUE; // start value, gets
@@ -323,7 +326,8 @@ public void testMultiValueAggDerivative() throws Exception {
                 assertThat(sumDeriv, notNullValue());
                 long sumDerivValue = expectedSum - expectedSumPreviousBucket;
                 assertThat(sumDeriv.value(), equalTo((double) sumDerivValue));
-                assertThat((double) bucket.getProperty("histo", AggregationPath.parse("deriv.value").getPathElementsAsStringList()),
+                assertThat(((InternalMultiBucketAggregation.InternalBucket)bucket).getProperty("histo",
+                        AggregationPath.parse("deriv.value").getPathElementsAsStringList()),
                         equalTo((double) sumDerivValue));
             } else {
                 assertThat(sumDeriv, nullValue());
@@ -450,7 +454,7 @@ public void testDocCountDerivativeWithGaps_insertZeros() throws Exception {
         Histogram deriv = searchResponse.getAggregations().get("histo");
         assertThat(deriv, Matchers.notNullValue());
         assertThat(deriv.getName(), equalTo("histo"));
-        List<Bucket> buckets = deriv.getBuckets();
+        List<? extends Bucket> buckets = deriv.getBuckets();
         assertThat(buckets.size(), equalTo(valueCounts_empty.length));
 
         for (int i = 0; i < valueCounts_empty.length; i++) {
@@ -479,7 +483,7 @@ public void testSingleValueAggDerivativeWithGaps() throws Exception {
         Histogram deriv = searchResponse.getAggregations().get("histo");
         assertThat(deriv, Matchers.notNullValue());
         assertThat(deriv.getName(), equalTo("histo"));
-        List<Bucket> buckets = deriv.getBuckets();
+        List<? extends Bucket> buckets = deriv.getBuckets();
         assertThat(buckets.size(), equalTo(valueCounts_empty.length));
 
         double lastSumValue = Double.NaN;
@@ -521,7 +525,7 @@ public void testSingleValueAggDerivativeWithGaps_insertZeros() throws Exception
         Histogram deriv = searchResponse.getAggregations().get("histo");
         assertThat(deriv, Matchers.notNullValue());
         assertThat(deriv.getName(), equalTo("histo"));
-        List<Bucket> buckets = deriv.getBuckets();
+        List<? extends Bucket> buckets = deriv.getBuckets();
         assertThat(buckets.size(), equalTo(valueCounts_empty.length));
 
         double lastSumValue = Double.NaN;
@@ -560,7 +564,7 @@ public void testSingleValueAggDerivativeWithGaps_random() throws Exception {
         Histogram deriv = searchResponse.getAggregations().get("histo");
         assertThat(deriv, Matchers.notNullValue());
         assertThat(deriv.getName(), equalTo("histo"));
-        List<Bucket> buckets = deriv.getBuckets();
+        List<? extends Bucket> buckets = deriv.getBuckets();
         assertThat(buckets.size(), equalTo(numBuckets_empty_rnd));
 
         double lastSumValue = Double.NaN;
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/DerivativeTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/DerivativeTests.java
index 678bb0b5c4d94..4b98e2ee6b263 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/DerivativeTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/DerivativeTests.java
@@ -27,11 +27,11 @@ public class DerivativeTests extends BasePipelineAggregationTestCase<DerivativeP
 
     @Override
     protected DerivativePipelineAggregationBuilder createTestAggregatorFactory() {
-        String name = randomAsciiOfLengthBetween(3, 20);
-        String bucketsPath = randomAsciiOfLengthBetween(3, 20);
+        String name = randomAlphaOfLengthBetween(3, 20);
+        String bucketsPath = randomAlphaOfLengthBetween(3, 20);
         DerivativePipelineAggregationBuilder factory = new DerivativePipelineAggregationBuilder(name, bucketsPath);
         if (randomBoolean()) {
-            factory.format(randomAsciiOfLengthBetween(1, 10));
+            factory.format(randomAlphaOfLengthBetween(1, 10));
         }
         if (randomBoolean()) {
             factory.gapPolicy(randomFrom(GapPolicy.values()));
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketIT.java
index d4310e581c026..607124ecb159d 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketIT.java
@@ -211,7 +211,7 @@ public void testDocCountAsSubAgg() throws Exception {
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
-        List<Terms.Bucket> termsBuckets = terms.getBuckets();
+        List<? extends Terms.Bucket> termsBuckets = terms.getBuckets();
         assertThat(termsBuckets.size(), equalTo(interval));
 
         for (int i = 0; i < interval; ++i) {
@@ -262,7 +262,7 @@ public void testMetricTopLevel() throws Exception {
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
-        List<Terms.Bucket> buckets = terms.getBuckets();
+        List<? extends Terms.Bucket> buckets = terms.getBuckets();
         assertThat(buckets.size(), equalTo(interval));
 
         double bucketSum = 0;
@@ -312,7 +312,7 @@ public void testMetricAsSubAgg() throws Exception {
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
-        List<Terms.Bucket> termsBuckets = terms.getBuckets();
+        List<? extends Terms.Bucket> termsBuckets = terms.getBuckets();
         assertThat(termsBuckets.size(), equalTo(interval));
 
         for (int i = 0; i < interval; ++i) {
@@ -375,7 +375,7 @@ public void testMetricAsSubAggWithInsertZeros() throws Exception {
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
-        List<Terms.Bucket> termsBuckets = terms.getBuckets();
+        List<? extends Terms.Bucket> termsBuckets = terms.getBuckets();
         assertThat(termsBuckets.size(), equalTo(interval));
 
         for (int i = 0; i < interval; ++i) {
@@ -429,7 +429,7 @@ public void testNoBuckets() throws Exception {
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
-        List<Terms.Bucket> buckets = terms.getBuckets();
+        List<? extends Terms.Bucket> buckets = terms.getBuckets();
         assertThat(buckets.size(), equalTo(0));
 
         ExtendedStatsBucket extendedStatsBucketValue = response.getAggregations().get("extended_stats_bucket");
@@ -487,7 +487,7 @@ public void testNested() throws Exception {
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
-        List<Terms.Bucket> termsBuckets = terms.getBuckets();
+        List<? extends Terms.Bucket> termsBuckets = terms.getBuckets();
         assertThat(termsBuckets.size(), equalTo(interval));
 
         double aggTermsSum = 0;
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/InternalSimpleValueTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/InternalSimpleValueTests.java
new file mode 100644
index 0000000000000..42678364efdec
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/InternalSimpleValueTests.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.pipeline;
+
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.ParsedAggregation;
+import org.elasticsearch.test.InternalAggregationTestCase;
+
+import java.util.List;
+import java.util.Map;
+
+public class InternalSimpleValueTests extends InternalAggregationTestCase<InternalSimpleValue> {
+
+    @Override
+    protected InternalSimpleValue createTestInstance(String name, List<PipelineAggregator> pipelineAggregators,
+            Map<String, Object> metaData) {
+        DocValueFormat formatter = randomNumericDocValueFormat();
+        double value = frequently() ? randomDoubleBetween(0, 100000, true)
+                : randomFrom(new Double[] { Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, Double.NaN });
+        return new InternalSimpleValue(name, value, formatter, pipelineAggregators, metaData);
+    }
+
+    @Override
+    protected void assertFromXContent(InternalSimpleValue simpleValue, ParsedAggregation parsedAggregation) {
+        ParsedSimpleValue parsed = ((ParsedSimpleValue) parsedAggregation);
+        if (Double.isInfinite(simpleValue.getValue()) == false && Double.isNaN(simpleValue.getValue()) == false) {
+            assertEquals(simpleValue.getValue(), parsed.value(), 0);
+            assertEquals(simpleValue.getValueAsString(), parsed.getValueAsString());
+        } else {
+            // we write Double.NEGATIVE_INFINITY, Double.POSITIVE amd Double.NAN to xContent as 'null', so we
+            // cannot differentiate between them. Also we cannot recreate the exact String representation
+            assertEquals(parsed.value(), Double.NaN, 0);
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/MaxBucketIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/MaxBucketIT.java
index aba941ebb4b6a..632f11f7ec73d 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/MaxBucketIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/MaxBucketIT.java
@@ -149,7 +149,7 @@ public void testDocCountAsSubAgg() throws Exception {
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
-        List<Terms.Bucket> termsBuckets = terms.getBuckets();
+        List<? extends Terms.Bucket> termsBuckets = terms.getBuckets();
         assertThat(termsBuckets.size(), equalTo(interval));
 
         for (int i = 0; i < interval; ++i) {
@@ -196,7 +196,7 @@ public void testMetricTopLevel() throws Exception {
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
-        List<Terms.Bucket> buckets = terms.getBuckets();
+        List<? extends Terms.Bucket> buckets = terms.getBuckets();
         assertThat(buckets.size(), equalTo(interval));
 
         List<String> maxKeys = new ArrayList<>();
@@ -242,7 +242,7 @@ public void testMetricAsSubAgg() throws Exception {
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
-        List<Terms.Bucket> termsBuckets = terms.getBuckets();
+        List<? extends Terms.Bucket> termsBuckets = terms.getBuckets();
         assertThat(termsBuckets.size(), equalTo(interval));
 
         for (int i = 0; i < interval; ++i) {
@@ -348,7 +348,7 @@ public void testMetricAsSubAggWithInsertZeros() throws Exception {
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
-        List<Terms.Bucket> termsBuckets = terms.getBuckets();
+        List<? extends Terms.Bucket> termsBuckets = terms.getBuckets();
         assertThat(termsBuckets.size(), equalTo(interval));
 
         for (int i = 0; i < interval; ++i) {
@@ -397,7 +397,7 @@ public void testNoBuckets() throws Exception {
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
-        List<Terms.Bucket> buckets = terms.getBuckets();
+        List<? extends Terms.Bucket> buckets = terms.getBuckets();
         assertThat(buckets.size(), equalTo(0));
 
         InternalBucketMetricValue maxBucketValue = response.getAggregations().get("max_bucket");
@@ -425,7 +425,7 @@ public void testNested() throws Exception {
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
-        List<Terms.Bucket> termsBuckets = terms.getBuckets();
+        List<? extends Terms.Bucket> termsBuckets = terms.getBuckets();
         assertThat(termsBuckets.size(), equalTo(interval));
 
         List<String> maxTermsKeys = new ArrayList<>();
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/MinBucketIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/MinBucketIT.java
index cbd6824b3a4a3..04fdd0c313354 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/MinBucketIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/MinBucketIT.java
@@ -146,7 +146,7 @@ public void testDocCountAsSubAgg() throws Exception {
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
-        List<Terms.Bucket> termsBuckets = terms.getBuckets();
+        List<? extends Terms.Bucket> termsBuckets = terms.getBuckets();
         assertThat(termsBuckets.size(), equalTo(interval));
 
         for (int i = 0; i < interval; ++i) {
@@ -193,7 +193,7 @@ public void testMetricTopLevel() throws Exception {
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
-        List<Terms.Bucket> buckets = terms.getBuckets();
+        List<? extends Terms.Bucket> buckets = terms.getBuckets();
         assertThat(buckets.size(), equalTo(interval));
 
         List<String> minKeys = new ArrayList<>();
@@ -239,7 +239,7 @@ public void testMetricAsSubAgg() throws Exception {
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
-        List<Terms.Bucket> termsBuckets = terms.getBuckets();
+        List<? extends Terms.Bucket> termsBuckets = terms.getBuckets();
         assertThat(termsBuckets.size(), equalTo(interval));
 
         for (int i = 0; i < interval; ++i) {
@@ -298,7 +298,7 @@ public void testMetricAsSubAggWithInsertZeros() throws Exception {
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
-        List<Terms.Bucket> termsBuckets = terms.getBuckets();
+        List<? extends Terms.Bucket> termsBuckets = terms.getBuckets();
         assertThat(termsBuckets.size(), equalTo(interval));
 
         for (int i = 0; i < interval; ++i) {
@@ -347,7 +347,7 @@ public void testNoBuckets() throws Exception {
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
-        List<Terms.Bucket> buckets = terms.getBuckets();
+        List<? extends Terms.Bucket> buckets = terms.getBuckets();
         assertThat(buckets.size(), equalTo(0));
 
         InternalBucketMetricValue minBucketValue = response.getAggregations().get("min_bucket");
@@ -375,7 +375,7 @@ public void testNested() throws Exception {
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
-        List<Terms.Bucket> termsBuckets = terms.getBuckets();
+        List<? extends Terms.Bucket> termsBuckets = terms.getBuckets();
         assertThat(termsBuckets.size(), equalTo(interval));
 
         List<String> minTermsKeys = new ArrayList<>();
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketIT.java
index 3ad2367c5c54c..e23e5441431eb 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketIT.java
@@ -34,6 +34,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.Iterator;
 import java.util.List;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
@@ -51,7 +52,7 @@
 public class PercentilesBucketIT extends ESIntegTestCase {
 
     private static final String SINGLE_VALUED_FIELD_NAME = "l_value";
-    private static final double[] PERCENTS = {1.0, 25.0, 50.0, 75.0, 99.0};
+    private static final double[] PERCENTS = {0.0, 1.0, 25.0, 50.0, 75.0, 99.0, 100.0};
     static int numDocs;
     static int interval;
     static int minRandomValue;
@@ -123,11 +124,7 @@ public void testDocCountopLevel() throws Exception {
         PercentilesBucket percentilesBucketValue = response.getAggregations().get("percentiles_bucket");
         assertThat(percentilesBucketValue, notNullValue());
         assertThat(percentilesBucketValue.getName(), equalTo("percentiles_bucket"));
-        for (Double p : PERCENTS) {
-            double expected = values[(int)((p / 100) * values.length)];
-            assertThat(percentilesBucketValue.percentile(p), equalTo(expected));
-        }
-
+        assertPercentileBucket(PERCENTS, values, percentilesBucketValue);
     }
 
     public void testDocCountAsSubAgg() throws Exception {
@@ -148,7 +145,7 @@ public void testDocCountAsSubAgg() throws Exception {
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
-        List<Terms.Bucket> termsBuckets = terms.getBuckets();
+        List<? extends Terms.Bucket> termsBuckets = terms.getBuckets();
         assertThat(termsBuckets.size(), equalTo(interval));
 
         for (int i = 0; i < interval; ++i) {
@@ -174,10 +171,7 @@ public void testDocCountAsSubAgg() throws Exception {
             PercentilesBucket percentilesBucketValue = termsBucket.getAggregations().get("percentiles_bucket");
             assertThat(percentilesBucketValue, notNullValue());
             assertThat(percentilesBucketValue.getName(), equalTo("percentiles_bucket"));
-            for (Double p : PERCENTS) {
-                double expected = values[(int)((p / 100) * values.length)];
-                assertThat(percentilesBucketValue.percentile(p), equalTo(expected));
-            }
+            assertPercentileBucket(PERCENTS, values, percentilesBucketValue);
         }
     }
 
@@ -193,7 +187,7 @@ public void testMetricTopLevel() throws Exception {
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
-        List<Terms.Bucket> buckets = terms.getBuckets();
+        List<? extends Terms.Bucket> buckets = terms.getBuckets();
         assertThat(buckets.size(), equalTo(interval));
 
         double[] values = new double[interval];
@@ -212,10 +206,7 @@ public void testMetricTopLevel() throws Exception {
         PercentilesBucket percentilesBucketValue = response.getAggregations().get("percentiles_bucket");
         assertThat(percentilesBucketValue, notNullValue());
         assertThat(percentilesBucketValue.getName(), equalTo("percentiles_bucket"));
-        for (Double p : PERCENTS) {
-            double expected = values[(int)((p / 100) * values.length)];
-            assertThat(percentilesBucketValue.percentile(p), equalTo(expected));
-        }
+        assertPercentileBucket(PERCENTS, values, percentilesBucketValue);
     }
 
     public void testMetricTopLevelDefaultPercents() throws Exception {
@@ -229,7 +220,7 @@ public void testMetricTopLevelDefaultPercents() throws Exception {
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
-        List<Terms.Bucket> buckets = terms.getBuckets();
+        List<? extends Terms.Bucket> buckets = terms.getBuckets();
         assertThat(buckets.size(), equalTo(interval));
 
         double[] values = new double[interval];
@@ -248,11 +239,7 @@ public void testMetricTopLevelDefaultPercents() throws Exception {
         PercentilesBucket percentilesBucketValue = response.getAggregations().get("percentiles_bucket");
         assertThat(percentilesBucketValue, notNullValue());
         assertThat(percentilesBucketValue.getName(), equalTo("percentiles_bucket"));
-        for (Percentile p : percentilesBucketValue) {
-            double expected = values[(int)((p.getPercent() / 100) * values.length)];
-            assertThat(percentilesBucketValue.percentile(p.getPercent()), equalTo(expected));
-            assertThat(p.getValue(), equalTo(expected));
-        }
+        assertPercentileBucket(values, percentilesBucketValue);
     }
 
     public void testMetricAsSubAgg() throws Exception {
@@ -274,7 +261,7 @@ public void testMetricAsSubAgg() throws Exception {
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
-        List<Terms.Bucket> termsBuckets = terms.getBuckets();
+        List<? extends Terms.Bucket> termsBuckets = terms.getBuckets();
         assertThat(termsBuckets.size(), equalTo(interval));
 
         for (int i = 0; i < interval; ++i) {
@@ -304,10 +291,7 @@ public void testMetricAsSubAgg() throws Exception {
             PercentilesBucket percentilesBucketValue = termsBucket.getAggregations().get("percentiles_bucket");
             assertThat(percentilesBucketValue, notNullValue());
             assertThat(percentilesBucketValue.getName(), equalTo("percentiles_bucket"));
-            for (Double p : PERCENTS) {
-                double expected = values.get((int) ((p / 100) * values.size()));
-                assertThat(percentilesBucketValue.percentile(p), equalTo(expected));
-            }
+            assertPercentileBucket(PERCENTS, values.stream().mapToDouble(Double::doubleValue).toArray(), percentilesBucketValue);
         }
     }
 
@@ -332,7 +316,7 @@ public void testMetricAsSubAggWithInsertZeros() throws Exception {
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
-        List<Terms.Bucket> termsBuckets = terms.getBuckets();
+        List<? extends Terms.Bucket> termsBuckets = terms.getBuckets();
         assertThat(termsBuckets.size(), equalTo(interval));
 
         for (int i = 0; i < interval; ++i) {
@@ -361,10 +345,7 @@ public void testMetricAsSubAggWithInsertZeros() throws Exception {
             PercentilesBucket percentilesBucketValue = termsBucket.getAggregations().get("percentiles_bucket");
             assertThat(percentilesBucketValue, notNullValue());
             assertThat(percentilesBucketValue.getName(), equalTo("percentiles_bucket"));
-            for (Double p : PERCENTS) {
-                double expected = values[(int)((p / 100) * values.length)];
-                assertThat(percentilesBucketValue.percentile(p), equalTo(expected));
-            }
+            assertPercentileBucket(PERCENTS, values, percentilesBucketValue);
         }
     }
 
@@ -380,7 +361,7 @@ public void testNoBuckets() throws Exception {
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
-        List<Terms.Bucket> buckets = terms.getBuckets();
+        List<? extends Terms.Bucket> buckets = terms.getBuckets();
         assertThat(buckets.size(), equalTo(0));
 
         PercentilesBucket percentilesBucketValue = response.getAggregations().get("percentiles_bucket");
@@ -403,7 +384,7 @@ public void testWrongPercents() throws Exception {
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
-        List<Terms.Bucket> buckets = terms.getBuckets();
+        List<? extends Terms.Bucket> buckets = terms.getBuckets();
         assertThat(buckets.size(), equalTo(0));
 
         PercentilesBucket percentilesBucketValue = response.getAggregations().get("percentiles_bucket");
@@ -489,7 +470,7 @@ public void testNested() throws Exception {
                                 .subAggregation(
                                         histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(interval)
                                                 .extendedBounds(minRandomValue, maxRandomValue))
-                                .subAggregation(percentilesBucket("percentile_histo_bucket", "histo>_count")))
+                                .subAggregation(percentilesBucket("percentile_histo_bucket", "histo>_count").percents(PERCENTS)))
                 .addAggregation(percentilesBucket("percentile_terms_bucket", "terms>percentile_histo_bucket.50")
                         .percents(PERCENTS)).execute().actionGet();
 
@@ -498,7 +479,7 @@ public void testNested() throws Exception {
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
-        List<Terms.Bucket> termsBuckets = terms.getBuckets();
+        List<? extends Terms.Bucket> termsBuckets = terms.getBuckets();
         assertThat(termsBuckets.size(), equalTo(interval));
 
         double[] values = new double[termsBuckets.size()];
@@ -525,10 +506,7 @@ public void testNested() throws Exception {
             PercentilesBucket percentilesBucketValue = termsBucket.getAggregations().get("percentile_histo_bucket");
             assertThat(percentilesBucketValue, notNullValue());
             assertThat(percentilesBucketValue.getName(), equalTo("percentile_histo_bucket"));
-            for (Double p : PERCENTS) {
-                double expected = innerValues[(int)((p / 100) * innerValues.length)];
-                assertThat(percentilesBucketValue.percentile(p), equalTo(expected));
-            }
+            assertPercentileBucket(PERCENTS, innerValues, percentilesBucketValue);
             values[i] = percentilesBucketValue.percentile(50.0);
         }
 
@@ -537,10 +515,7 @@ public void testNested() throws Exception {
         PercentilesBucket percentilesBucketValue = response.getAggregations().get("percentile_terms_bucket");
         assertThat(percentilesBucketValue, notNullValue());
         assertThat(percentilesBucketValue.getName(), equalTo("percentile_terms_bucket"));
-        for (Double p : PERCENTS) {
-            double expected = values[(int)((p / 100) * values.length)];
-            assertThat(percentilesBucketValue.percentile(p), equalTo(expected));
-        }
+        assertPercentileBucket(PERCENTS, values, percentilesBucketValue);
     }
 
     public void testNestedWithDecimal() throws Exception {
@@ -564,7 +539,7 @@ public void testNestedWithDecimal() throws Exception {
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
-        List<Terms.Bucket> termsBuckets = terms.getBuckets();
+        List<? extends Terms.Bucket> termsBuckets = terms.getBuckets();
         assertThat(termsBuckets.size(), equalTo(interval));
 
         double[] values = new double[termsBuckets.size()];
@@ -591,10 +566,7 @@ public void testNestedWithDecimal() throws Exception {
             PercentilesBucket percentilesBucketValue = termsBucket.getAggregations().get("percentile_histo_bucket");
             assertThat(percentilesBucketValue, notNullValue());
             assertThat(percentilesBucketValue.getName(), equalTo("percentile_histo_bucket"));
-            for (Double p : percent) {
-                double expected = innerValues[(int)((p / 100) * innerValues.length)];
-                assertThat(percentilesBucketValue.percentile(p), equalTo(expected));
-            }
+            assertPercentileBucket(innerValues, percentilesBucketValue);
             values[i] = percentilesBucketValue.percentile(99.9);
         }
 
@@ -608,4 +580,22 @@ public void testNestedWithDecimal() throws Exception {
             assertThat(percentilesBucketValue.percentile(p), equalTo(expected));
         }
     }
+
+    private void assertPercentileBucket(double[] values, PercentilesBucket percentiles) {
+        for (Percentile percentile : percentiles) {
+            assertEquals(percentiles.percentile(percentile.getPercent()), percentile.getValue(), 0d);
+            int index = (int) Math.round((percentile.getPercent() / 100.0) * (values.length - 1));
+            assertThat(percentile.getValue(), equalTo(values[index]));
+        }
+    }
+
+    private void assertPercentileBucket(double[] percents, double[] values, PercentilesBucket percentiles) {
+        Iterator<Percentile> it = percentiles.iterator();
+        for (int i = 0; i < percents.length; ++i) {
+            assertTrue(it.hasNext());
+            assertEquals(percents[i], it.next().getPercent(), 0d);
+        }
+        assertFalse(it.hasNext());
+        assertPercentileBucket(values, percentiles);
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/SerialDifferenceTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/SerialDifferenceTests.java
index 01b3174382842..7e71be69b64b7 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/SerialDifferenceTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/SerialDifferenceTests.java
@@ -27,11 +27,11 @@ public class SerialDifferenceTests extends BasePipelineAggregationTestCase<Seria
 
     @Override
     protected SerialDiffPipelineAggregationBuilder createTestAggregatorFactory() {
-        String name = randomAsciiOfLengthBetween(3, 20);
-        String bucketsPath = randomAsciiOfLengthBetween(3, 20);
+        String name = randomAlphaOfLengthBetween(3, 20);
+        String bucketsPath = randomAlphaOfLengthBetween(3, 20);
         SerialDiffPipelineAggregationBuilder factory = new SerialDiffPipelineAggregationBuilder(name, bucketsPath);
         if (randomBoolean()) {
-            factory.format(randomAsciiOfLengthBetween(1, 10));
+            factory.format(randomAlphaOfLengthBetween(1, 10));
         }
         if (randomBoolean()) {
             factory.gapPolicy(randomFrom(GapPolicy.values()));
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/StatsBucketIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/StatsBucketIT.java
index c38dc99bdf939..231005f1b5b62 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/StatsBucketIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/StatsBucketIT.java
@@ -147,7 +147,7 @@ public void testDocCountAsSubAgg() throws Exception {
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
-        List<Terms.Bucket> termsBuckets = terms.getBuckets();
+        List<? extends Terms.Bucket> termsBuckets = terms.getBuckets();
         assertThat(termsBuckets.size(), equalTo(interval));
 
         for (int i = 0; i < interval; ++i) {
@@ -195,7 +195,7 @@ public void testMetricTopLevel() throws Exception {
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
-        List<Terms.Bucket> buckets = terms.getBuckets();
+        List<? extends Terms.Bucket> buckets = terms.getBuckets();
         assertThat(buckets.size(), equalTo(interval));
 
         double bucketSum = 0;
@@ -242,7 +242,7 @@ public void testMetricAsSubAgg() throws Exception {
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
-        List<Terms.Bucket> termsBuckets = terms.getBuckets();
+        List<? extends Terms.Bucket> termsBuckets = terms.getBuckets();
         assertThat(termsBuckets.size(), equalTo(interval));
 
         for (int i = 0; i < interval; ++i) {
@@ -302,7 +302,7 @@ public void testMetricAsSubAggWithInsertZeros() throws Exception {
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
-        List<Terms.Bucket> termsBuckets = terms.getBuckets();
+        List<? extends Terms.Bucket> termsBuckets = terms.getBuckets();
         assertThat(termsBuckets.size(), equalTo(interval));
 
         for (int i = 0; i < interval; ++i) {
@@ -353,7 +353,7 @@ public void testNoBuckets() throws Exception {
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
-        List<Terms.Bucket> buckets = terms.getBuckets();
+        List<? extends Terms.Bucket> buckets = terms.getBuckets();
         assertThat(buckets.size(), equalTo(0));
 
         StatsBucket statsBucketValue = response.getAggregations().get("stats_bucket");
@@ -380,7 +380,7 @@ public void testNested() throws Exception {
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
-        List<Terms.Bucket> termsBuckets = terms.getBuckets();
+        List<? extends Terms.Bucket> termsBuckets = terms.getBuckets();
         assertThat(termsBuckets.size(), equalTo(interval));
 
         double aggTermsSum = 0;
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/SumBucketIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/SumBucketIT.java
index 09582430046f1..048dfac8648d4 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/SumBucketIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/SumBucketIT.java
@@ -137,7 +137,7 @@ public void testDocCountAsSubAgg() throws Exception {
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
-        List<Terms.Bucket> termsBuckets = terms.getBuckets();
+        List<? extends Terms.Bucket> termsBuckets = terms.getBuckets();
         assertThat(termsBuckets.size(), equalTo(interval));
 
         for (int i = 0; i < interval; ++i) {
@@ -176,7 +176,7 @@ public void testMetricTopLevel() throws Exception {
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
-        List<Terms.Bucket> buckets = terms.getBuckets();
+        List<? extends Terms.Bucket> buckets = terms.getBuckets();
         assertThat(buckets.size(), equalTo(interval));
 
         double bucketSum = 0;
@@ -214,7 +214,7 @@ public void testMetricAsSubAgg() throws Exception {
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
-        List<Terms.Bucket> termsBuckets = terms.getBuckets();
+        List<? extends Terms.Bucket> termsBuckets = terms.getBuckets();
         assertThat(termsBuckets.size(), equalTo(interval));
 
         for (int i = 0; i < interval; ++i) {
@@ -265,7 +265,7 @@ public void testMetricAsSubAggWithInsertZeros() throws Exception {
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
-        List<Terms.Bucket> termsBuckets = terms.getBuckets();
+        List<? extends Terms.Bucket> termsBuckets = terms.getBuckets();
         assertThat(termsBuckets.size(), equalTo(interval));
 
         for (int i = 0; i < interval; ++i) {
@@ -307,7 +307,7 @@ public void testNoBuckets() throws Exception {
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
-        List<Terms.Bucket> buckets = terms.getBuckets();
+        List<? extends Terms.Bucket> buckets = terms.getBuckets();
         assertThat(buckets.size(), equalTo(0));
 
         InternalSimpleValue sumBucketValue = response.getAggregations().get("sum_bucket");
@@ -334,7 +334,7 @@ public void testNested() throws Exception {
         Terms terms = response.getAggregations().get("terms");
         assertThat(terms, notNullValue());
         assertThat(terms.getName(), equalTo("terms"));
-        List<Terms.Bucket> termsBuckets = terms.getBuckets();
+        List<? extends Terms.Bucket> termsBuckets = terms.getBuckets();
         assertThat(termsBuckets.size(), equalTo(interval));
 
         double aggTermsSum = 0;
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/AbstractBucketMetricsTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/AbstractBucketMetricsTestCase.java
index 13ed4027b3a37..2b53a236c3bc8 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/AbstractBucketMetricsTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/AbstractBucketMetricsTestCase.java
@@ -27,11 +27,11 @@ public abstract class AbstractBucketMetricsTestCase<PAF extends BucketMetricsPip
 
     @Override
     protected final PAF createTestAggregatorFactory() {
-        String name = randomAsciiOfLengthBetween(3, 20);
-        String bucketsPath = randomAsciiOfLengthBetween(3, 20);
+        String name = randomAlphaOfLengthBetween(3, 20);
+        String bucketsPath = randomAlphaOfLengthBetween(3, 20);
         PAF factory = doCreateTestAggregatorFactory(name, bucketsPath);
         if (randomBoolean()) {
-            factory.format(randomAsciiOfLengthBetween(1, 10));
+            factory.format(randomAlphaOfLengthBetween(1, 10));
         }
         if (randomBoolean()) {
             factory.gapPolicy(randomFrom(GapPolicy.values()));
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/ExtendedStatsBucketTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/ExtendedStatsBucketTests.java
index 390501d200231..b77c0bdd972cb 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/ExtendedStatsBucketTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/ExtendedStatsBucketTests.java
@@ -19,9 +19,8 @@
 
 package org.elasticsearch.search.aggregations.pipeline.bucketmetrics;
 
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.extended.ExtendedStatsBucketPipelineAggregationBuilder;
 
 import static org.hamcrest.Matchers.equalTo;
@@ -38,20 +37,18 @@ protected ExtendedStatsBucketPipelineAggregationBuilder doCreateTestAggregatorFa
     }
 
     public void testSigmaFromInt() throws Exception {
-        String content = XContentFactory.jsonBuilder()
+        XContentBuilder content = XContentFactory.jsonBuilder()
             .startObject()
-                .field("sigma", 5)
-                .field("buckets_path", "test")
-            .endObject()
-            .string();
-
-        XContentParser parser = XContentFactory.xContent(content).createParser(content);
-        QueryParseContext parseContext = new QueryParseContext(queriesRegistry, parser, parseFieldMatcher);
-        parser.nextToken(); // skip object start
-
-        ExtendedStatsBucketPipelineAggregationBuilder builder = (ExtendedStatsBucketPipelineAggregationBuilder) aggParsers
-            .pipelineParser(ExtendedStatsBucketPipelineAggregationBuilder.NAME, parseFieldMatcher)
-            .parse("test", parseContext);
+                .startObject("name")
+                    .startObject("extended_stats_bucket")
+                        .field("sigma", 5)
+                        .field("buckets_path", "test")
+                    .endObject()
+                .endObject()
+            .endObject();
+
+        ExtendedStatsBucketPipelineAggregationBuilder builder = (ExtendedStatsBucketPipelineAggregationBuilder) parse(
+                createParser(content));
 
         assertThat(builder.sigma(), equalTo(5.0));
     }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/InternalBucketMetricValueTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/InternalBucketMetricValueTests.java
new file mode 100644
index 0000000000000..1d1dee735581b
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/InternalBucketMetricValueTests.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.pipeline.bucketmetrics;
+
+import org.elasticsearch.search.aggregations.ParsedAggregation;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.test.InternalAggregationTestCase;
+
+import java.util.List;
+import java.util.Map;
+
+public class InternalBucketMetricValueTests extends InternalAggregationTestCase<InternalBucketMetricValue> {
+
+    @Override
+    protected InternalBucketMetricValue createTestInstance(String name, List<PipelineAggregator> pipelineAggregators,
+            Map<String, Object> metaData) {
+        double value = frequently() ? randomDoubleBetween(-10000, 100000, true)
+                : randomFrom(new Double[] { Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, Double.NaN });
+        String[] keys = new String[randomIntBetween(0, 5)];
+        for (int i = 0; i < keys.length; i++) {
+            keys[i] = randomAlphaOfLength(10);
+        }
+        return new InternalBucketMetricValue(name, keys, value, randomNumericDocValueFormat(), pipelineAggregators, metaData);
+    }
+
+    @Override
+    protected void assertFromXContent(InternalBucketMetricValue bucketMetricValue, ParsedAggregation parsedAggregation) {
+        BucketMetricValue parsed = ((BucketMetricValue) parsedAggregation);
+        assertArrayEquals(bucketMetricValue.keys(), parsed.keys());
+        if (Double.isInfinite(bucketMetricValue.value()) == false) {
+            assertEquals(bucketMetricValue.value(), parsed.value(), 0);
+            assertEquals(bucketMetricValue.getValueAsString(), parsed.getValueAsString());
+        } else {
+            // we write Double.NEGATIVE_INFINITY and Double.POSITIVE_INFINITY to xContent as 'null', so we
+            // cannot differentiate between them. Also we cannot recreate the exact String representation
+            assertEquals(parsed.value(), Double.NEGATIVE_INFINITY, 0);
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/PercentilesBucketTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/PercentilesBucketTests.java
index 71bf937fbb22e..b04b5743c68cb 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/PercentilesBucketTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/PercentilesBucketTests.java
@@ -19,9 +19,8 @@
 
 package org.elasticsearch.search.aggregations.pipeline.bucketmetrics;
 
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.percentile.PercentilesBucketPipelineAggregationBuilder;
 
 import static org.hamcrest.Matchers.equalTo;
@@ -43,20 +42,17 @@ protected PercentilesBucketPipelineAggregationBuilder doCreateTestAggregatorFact
     }
 
     public void testPercentsFromMixedArray() throws Exception {
-        String content = XContentFactory.jsonBuilder()
+        XContentBuilder content = XContentFactory.jsonBuilder()
             .startObject()
-                .field("buckets_path", "test")
-                .array("percents", 0, 20.0, 50, 75.99)
-            .endObject()
-            .string();
-
-        XContentParser parser = XContentFactory.xContent(content).createParser(content);
-        QueryParseContext parseContext = new QueryParseContext(queriesRegistry, parser, parseFieldMatcher);
-        parser.nextToken(); // skip object start
-
-        PercentilesBucketPipelineAggregationBuilder builder = (PercentilesBucketPipelineAggregationBuilder) aggParsers
-            .pipelineParser(PercentilesBucketPipelineAggregationBuilder.NAME, parseFieldMatcher)
-            .parse("test", parseContext);
+                .startObject("name")
+                    .startObject("percentiles_bucket")
+                        .field("buckets_path", "test")
+                        .array("percents", 0, 20.0, 50, 75.99)
+                    .endObject()
+                .endObject()
+            .endObject();
+
+        PercentilesBucketPipelineAggregationBuilder builder = (PercentilesBucketPipelineAggregationBuilder) parse(createParser(content));
 
         assertThat(builder.percents(), equalTo(new double[]{0.0, 20.0, 50.0, 75.99}));
     }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/InternalPercentilesBucketTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/InternalPercentilesBucketTests.java
new file mode 100644
index 0000000000000..a77ec9747040d
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/InternalPercentilesBucketTests.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.pipeline.bucketmetrics.percentile;
+
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.Aggregation.CommonFields;
+import org.elasticsearch.search.aggregations.ParsedAggregation;
+import org.elasticsearch.search.aggregations.metrics.percentiles.Percentile;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.test.InternalAggregationTestCase;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Predicate;
+
+import static org.elasticsearch.search.aggregations.metrics.percentiles.InternalPercentilesTestCase.randomPercents;
+
+public class InternalPercentilesBucketTests extends InternalAggregationTestCase<InternalPercentilesBucket> {
+
+    @Override
+    protected InternalPercentilesBucket createTestInstance(String name, List<PipelineAggregator> pipelineAggregators,
+                                                           Map<String, Object> metaData) {
+        return createTestInstance(name, pipelineAggregators, metaData, randomPercents());
+    }
+
+    private static InternalPercentilesBucket createTestInstance(String name, List<PipelineAggregator> pipelineAggregators,
+                                                                Map<String, Object> metaData, double[] percents) {
+        final double[] percentiles = new double[percents.length];
+        for (int i = 0; i < percents.length; ++i) {
+            percentiles[i] = frequently() ? randomDouble() : Double.NaN;
+        }
+        return new InternalPercentilesBucket(name, percents, percentiles, DocValueFormat.RAW, pipelineAggregators, metaData);
+    }
+
+    @Override
+    protected final void assertFromXContent(InternalPercentilesBucket aggregation, ParsedAggregation parsedAggregation) {
+        assertTrue(parsedAggregation instanceof ParsedPercentilesBucket);
+        ParsedPercentilesBucket parsedPercentiles = (ParsedPercentilesBucket) parsedAggregation;
+
+        for (Percentile percentile : aggregation) {
+            Double percent = percentile.getPercent();
+            assertEquals(aggregation.percentile(percent), parsedPercentiles.percentile(percent), 0);
+            // we cannot ensure we get the same as_string output for Double.NaN values since they are rendered as
+            // null and we don't have a formatted string representation in the rest output
+            if (Double.isNaN(aggregation.percentile(percent)) == false) {
+                assertEquals(aggregation.percentileAsString(percent), parsedPercentiles.percentileAsString(percent));
+            }
+        }
+    }
+
+    /**
+     * check that we don't rely on the percent array order and that the iterator returns the values in the original order
+     */
+    public void testPercentOrder() {
+        final double[] percents =  new double[]{ 0.50, 0.25, 0.01, 0.99, 0.60 };
+        InternalPercentilesBucket aggregation = createTestInstance("test", Collections.emptyList(), Collections.emptyMap(), percents);
+        Iterator<Percentile> iterator = aggregation.iterator();
+        for (double percent : percents) {
+            assertTrue(iterator.hasNext());
+            Percentile percentile = iterator.next();
+            assertEquals(percent, percentile.getPercent(), 0.0d);
+            assertEquals(aggregation.percentile(percent), percentile.getValue(), 0.0d);
+        }
+    }
+
+    public void testErrorOnDifferentArgumentSize() {
+        final double[] percents =  new double[]{ 0.1, 0.2, 0.3};
+        final double[] percentiles =  new double[]{ 0.10, 0.2};
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new InternalPercentilesBucket("test", percents,
+                percentiles, DocValueFormat.RAW, Collections.emptyList(), Collections.emptyMap()));
+        assertEquals("The number of provided percents and percentiles didn't match. percents: [0.1, 0.2, 0.3], percentiles: [0.1, 0.2]",
+                e.getMessage());
+    }
+
+    public void testParsedAggregationIteratorOrder() throws IOException {
+        final InternalPercentilesBucket aggregation = createTestInstance();
+        final Iterable<Percentile> parsedAggregation = parseAndAssert(aggregation, false, false);
+        Iterator<Percentile> it = aggregation.iterator();
+        Iterator<Percentile> parsedIt = parsedAggregation.iterator();
+        while (it.hasNext()) {
+            assertEquals(it.next(), parsedIt.next());
+        }
+    }
+
+    @Override
+    protected Predicate<String> excludePathsFromXContentInsertion() {
+        return path -> path.endsWith(CommonFields.VALUES.getPreferredName());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/InternalExtendedStatsBucketTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/InternalExtendedStatsBucketTests.java
new file mode 100644
index 0000000000000..5bb86dd421a6d
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/InternalExtendedStatsBucketTests.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.extended;
+
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.ParsedAggregation;
+import org.elasticsearch.search.aggregations.metrics.InternalExtendedStatsTests;
+import org.elasticsearch.search.aggregations.metrics.stats.extended.InternalExtendedStats;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+public class InternalExtendedStatsBucketTests extends InternalExtendedStatsTests {
+
+    @Override
+    protected InternalExtendedStatsBucket createInstance(String name, long count, double sum, double min, double max, double sumOfSqrs,
+            double sigma, DocValueFormat formatter, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
+        return new InternalExtendedStatsBucket(name, count, sum, min, max, sumOfSqrs, sigma, formatter, pipelineAggregators,
+                Collections.emptyMap());
+    }
+
+    @Override
+    protected void assertFromXContent(InternalExtendedStats aggregation, ParsedAggregation parsedAggregation) {
+        super.assertFromXContent(aggregation, parsedAggregation);
+        assertTrue(parsedAggregation instanceof ParsedExtendedStatsBucket);
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/derivative/InternalDerivativeTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/derivative/InternalDerivativeTests.java
new file mode 100644
index 0000000000000..d12d58bdae23e
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/derivative/InternalDerivativeTests.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.pipeline.derivative;
+
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.ParsedAggregation;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.test.InternalAggregationTestCase;
+
+import java.util.List;
+import java.util.Map;
+
+public class InternalDerivativeTests extends InternalAggregationTestCase<InternalDerivative> {
+
+    @Override
+    protected InternalDerivative createTestInstance(String name,
+            List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
+        DocValueFormat formatter = randomNumericDocValueFormat();
+        double value = frequently() ? randomDoubleBetween(-100000, 100000, true)
+                : randomFrom(new Double[] { Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, Double.NaN });
+        double normalizationFactor = frequently() ? randomDoubleBetween(0, 100000, true) : 0;
+        return new InternalDerivative(name, value, normalizationFactor, formatter, pipelineAggregators, metaData);
+    }
+
+    @Override
+    protected void assertFromXContent(InternalDerivative derivative, ParsedAggregation parsedAggregation) {
+        ParsedDerivative parsed = ((ParsedDerivative) parsedAggregation);
+        if (Double.isInfinite(derivative.getValue()) == false && Double.isNaN(derivative.getValue()) == false) {
+            assertEquals(derivative.getValue(), parsed.value(), Double.MIN_VALUE);
+            assertEquals(derivative.getValueAsString(), parsed.getValueAsString());
+        } else {
+            // we write Double.NEGATIVE_INFINITY, Double.POSITIVE amd Double.NAN to xContent as 'null', so we
+            // cannot differentiate between them. Also we cannot recreate the exact String representation
+            assertEquals(parsed.value(), Double.NaN, Double.MIN_VALUE);
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/moving/avg/MovAvgIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/moving/avg/MovAvgIT.java
index f24dfe4227097..bbe6ecc3a4e68 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/moving/avg/MovAvgIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/moving/avg/MovAvgIT.java
@@ -19,9 +19,12 @@
 
 package org.elasticsearch.search.aggregations.pipeline.moving.avg;
 
+import org.elasticsearch.action.bulk.BulkRequestBuilder;
+import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.common.collect.EvictingQueue;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket;
@@ -41,6 +44,7 @@
 import org.elasticsearch.test.ESIntegTestCase;
 import org.hamcrest.Matchers;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -67,6 +71,7 @@
 public class MovAvgIT extends ESIntegTestCase {
     private static final String INTERVAL_FIELD = "l_value";
     private static final String VALUE_FIELD = "v_value";
+    private static final String VALUE_FIELD2 = "v_value2";
 
     static int interval;
     static int numBuckets;
@@ -1204,6 +1209,68 @@ public void testCheckIfTunableCanBeMinimized() {
         }
     }
 
+    public void testPredictWithNonEmptyBuckets() throws Exception {
+
+        createIndex("predict_non_empty");
+        BulkRequestBuilder bulkBuilder = client().prepareBulk().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
+
+        for (int i = 0; i < 10; i++) {
+            bulkBuilder.add(client().prepareIndex("predict_non_empty", "type").setSource(
+                jsonBuilder().startObject().field(INTERVAL_FIELD, i)
+                    .field(VALUE_FIELD, 10)
+                    .field(VALUE_FIELD2, 10)
+                    .endObject()));
+        }
+        for (int i = 10; i < 20; i++) {
+            // Extra so there is a bucket that only has second field
+            bulkBuilder.add(client().prepareIndex("predict_non_empty", "type").setSource(
+                jsonBuilder().startObject().field(INTERVAL_FIELD, i).field(VALUE_FIELD2, 10).endObject()));
+        }
+
+        bulkBuilder.execute().actionGet();
+        ensureSearchable();
+
+        SearchResponse response = client()
+            .prepareSearch("predict_non_empty")
+            .setTypes("type")
+            .addAggregation(
+                histogram("histo")
+                    .field(INTERVAL_FIELD)
+                    .interval(1)
+                    .subAggregation(max("max").field(VALUE_FIELD))
+                    .subAggregation(max("max2").field(VALUE_FIELD2))
+                    .subAggregation(
+                        movingAvg("movavg_values", "max")
+                            .window(windowSize)
+                            .modelBuilder(new SimpleModel.SimpleModelBuilder())
+                            .gapPolicy(BucketHelpers.GapPolicy.SKIP).predict(5))).execute().actionGet();
+
+        assertSearchResponse(response);
+
+        Histogram histo = response.getAggregations().get("histo");
+        assertThat(histo, notNullValue());
+        assertThat(histo.getName(), equalTo("histo"));
+        List<? extends Bucket> buckets = histo.getBuckets();
+        assertThat("Size of buckets array is not correct.", buckets.size(), equalTo(20));
+
+        SimpleValue current = buckets.get(0).getAggregations().get("movavg_values");
+        assertThat(current, nullValue());
+
+        for (int i = 1; i < 20; i++) {
+            Bucket bucket = buckets.get(i);
+            assertThat(bucket, notNullValue());
+            assertThat(bucket.getKey(), equalTo((double)i));
+            assertThat(bucket.getDocCount(), equalTo(1L));
+            SimpleValue movAvgAgg = bucket.getAggregations().get("movavg_values");
+            if (i < 15) {
+                assertThat(movAvgAgg, notNullValue());
+                assertThat(movAvgAgg.value(), equalTo(10d));
+            } else {
+                assertThat(movAvgAgg, nullValue());
+            }
+        }
+    }
+
     private void assertValidIterators(Iterator expectedBucketIter, Iterator expectedCountsIter, Iterator expectedValuesIter) {
         if (!expectedBucketIter.hasNext()) {
             fail("`expectedBucketIter` iterator ended before `actual` iterator, size mismatch");
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/moving/avg/MovAvgTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/moving/avg/MovAvgTests.java
index e34a56c496a9d..869a7cd58ed8e 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/moving/avg/MovAvgTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/moving/avg/MovAvgTests.java
@@ -19,10 +19,7 @@
 
 package org.elasticsearch.search.aggregations.pipeline.moving.avg;
 
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.search.aggregations.BasePipelineAggregationTestCase;
 import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
@@ -32,17 +29,17 @@
 import org.elasticsearch.search.aggregations.pipeline.movavg.models.HoltWintersModel;
 import org.elasticsearch.search.aggregations.pipeline.movavg.models.HoltWintersModel.SeasonalityType;
 import org.elasticsearch.search.aggregations.pipeline.movavg.models.LinearModel;
-import org.elasticsearch.search.aggregations.pipeline.movavg.models.SimpleModel;;
+import org.elasticsearch.search.aggregations.pipeline.movavg.models.SimpleModel;
 
 public class MovAvgTests extends BasePipelineAggregationTestCase<MovAvgPipelineAggregationBuilder> {
 
     @Override
     protected MovAvgPipelineAggregationBuilder createTestAggregatorFactory() {
-        String name = randomAsciiOfLengthBetween(3, 20);
-        String bucketsPath = randomAsciiOfLengthBetween(3, 20);
+        String name = randomAlphaOfLengthBetween(3, 20);
+        String bucketsPath = randomAlphaOfLengthBetween(3, 20);
         MovAvgPipelineAggregationBuilder factory = new MovAvgPipelineAggregationBuilder(name, bucketsPath);
         if (randomBoolean()) {
-            factory.format(randomAsciiOfLengthBetween(1, 10));
+            factory.format(randomAlphaOfLengthBetween(1, 10));
         }
         if (randomBoolean()) {
             factory.gapPolicy(randomFrom(GapPolicy.values()));
@@ -102,26 +99,11 @@ public void testDefaultParsing() throws Exception {
         String json = "{" +
             "    \"commits_moving_avg\": {" +
             "        \"moving_avg\": {" +
-                "            \"buckets_path\": \"commits\"" +
+            "            \"buckets_path\": \"commits\"" +
             "        }" +
             "    }" +
             "}";
-        XContentParser parser = XContentFactory.xContent(json).createParser(json);
-        QueryParseContext parseContext = new QueryParseContext(queriesRegistry, parser, parseFieldMatcher);
-        assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
-        assertSame(XContentParser.Token.FIELD_NAME, parser.nextToken());
-        assertEquals(expected.getName(), parser.currentName());
-        assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
-        assertSame(XContentParser.Token.FIELD_NAME, parser.nextToken());
-        assertEquals(expected.type(), parser.currentName());
-        assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
-        PipelineAggregationBuilder newAgg = aggParsers.pipelineParser(expected.getWriteableName(), ParseFieldMatcher.STRICT)
-                .parse(expected.getName(), parseContext);
-        assertSame(XContentParser.Token.END_OBJECT, parser.currentToken());
-        assertSame(XContentParser.Token.END_OBJECT, parser.nextToken());
-        assertSame(XContentParser.Token.END_OBJECT, parser.nextToken());
-        assertNull(parser.nextToken());
-        assertNotNull(newAgg);
+        PipelineAggregationBuilder newAgg = parse(createParser(JsonXContent.jsonXContent, json));
         assertNotSame(newAgg, expected);
         assertEquals(expected, newAgg);
         assertEquals(expected.hashCode(), newAgg.hashCode());
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/moving/avg/MovAvgUnitTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/moving/avg/MovAvgUnitTests.java
index 0f94a4cbc0601..34d203443604a 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/moving/avg/MovAvgUnitTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/moving/avg/MovAvgUnitTests.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.search.aggregations.pipeline.moving.avg;
 
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.collect.EvictingQueue;
 import org.elasticsearch.search.aggregations.pipeline.movavg.models.EwmaModel;
 import org.elasticsearch.search.aggregations.pipeline.movavg.models.HoltLinearModel;
@@ -605,7 +604,7 @@ public void testNumericValidation() {
                 settings.put("alpha", v);
 
                 try {
-                    parser.parse(settings, "pipeline", 10, ParseFieldMatcher.STRICT);
+                    parser.parse(settings, "pipeline", 10);
                 } catch (ParseException e) {
                     fail(parser + " parser should not have thrown SearchParseException while parsing [" +
                             v.getClass().getSimpleName() +"]");
@@ -619,7 +618,7 @@ public void testNumericValidation() {
             settings.put("gamma", "abc");
 
             try {
-                parser.parse(settings, "pipeline", 10, ParseFieldMatcher.STRICT);
+                parser.parse(settings, "pipeline", 10);
             } catch (ParseException e) {
                 //all good
                 continue;
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/support/IncludeExcludeTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/support/IncludeExcludeTests.java
index 90b8d14ef54b5..144cb4a838c50 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/support/IncludeExcludeTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/support/IncludeExcludeTests.java
@@ -24,6 +24,13 @@
 import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.LongBitSet;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude;
 import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude.OrdinalsFilter;
@@ -34,7 +41,6 @@
 import java.util.TreeSet;
 
 public class IncludeExcludeTests extends ESTestCase {
-
     public void testEmptyTermsWithOrds() throws IOException {
         IncludeExclude inexcl = new IncludeExclude(
                 new TreeSet<>(Collections.singleton(new BytesRef("foo"))),
@@ -125,4 +131,180 @@ public int cardinality() {
         assertFalse(acceptedOrds.get(0));
     }
 
+    public void testPartitionedEquals() throws IOException {
+        IncludeExclude serialized = serialize(new IncludeExclude(3, 20), IncludeExclude.INCLUDE_FIELD);
+        assertFalse(serialized.isRegexBased());
+        assertTrue(serialized.isPartitionBased());
+
+        IncludeExclude same = new IncludeExclude(3, 20);
+        assertEquals(serialized, same);
+        assertEquals(serialized.hashCode(), same.hashCode());
+
+        IncludeExclude differentParam1 = new IncludeExclude(4, 20);
+        assertFalse(serialized.equals(differentParam1));
+        assertTrue(serialized.hashCode() != differentParam1.hashCode());
+
+        IncludeExclude differentParam2 = new IncludeExclude(3, 21);
+        assertFalse(serialized.equals(differentParam2));
+        assertTrue(serialized.hashCode() != differentParam2.hashCode());
+    }
+
+    public void testExactIncludeValuesEquals() throws IOException {
+        String[] incValues = { "a", "b" };
+        String[] differentIncValues = { "a", "c" };
+        IncludeExclude serialized = serialize(new IncludeExclude(incValues, null), IncludeExclude.INCLUDE_FIELD);
+        assertFalse(serialized.isPartitionBased());
+        assertFalse(serialized.isRegexBased());
+
+        IncludeExclude same = new IncludeExclude(incValues, null);
+        assertEquals(serialized, same);
+        assertEquals(serialized.hashCode(), same.hashCode());
+
+        IncludeExclude different = new IncludeExclude(differentIncValues, null);
+        assertFalse(serialized.equals(different));
+        assertTrue(serialized.hashCode() != different.hashCode());
+    }
+
+    public void testExactExcludeValuesEquals() throws IOException {
+        String[] excValues = { "a", "b" };
+        String[] differentExcValues = { "a", "c" };
+        IncludeExclude serialized = serialize(new IncludeExclude(null, excValues), IncludeExclude.EXCLUDE_FIELD);
+        assertFalse(serialized.isPartitionBased());
+        assertFalse(serialized.isRegexBased());
+
+        IncludeExclude same = new IncludeExclude(null, excValues);
+        assertEquals(serialized, same);
+        assertEquals(serialized.hashCode(), same.hashCode());
+
+        IncludeExclude different = new IncludeExclude(null, differentExcValues);
+        assertFalse(serialized.equals(different));
+        assertTrue(serialized.hashCode() != different.hashCode());
+    }
+
+    public void testRegexInclude() throws IOException {
+        String incRegex = "foo.*";
+        String differentRegex = "bar.*";
+        IncludeExclude serialized = serialize(new IncludeExclude(incRegex, null), IncludeExclude.INCLUDE_FIELD);
+        assertFalse(serialized.isPartitionBased());
+        assertTrue(serialized.isRegexBased());
+
+        IncludeExclude same = new IncludeExclude(incRegex, null);
+        assertEquals(serialized, same);
+        assertEquals(serialized.hashCode(), same.hashCode());
+
+        IncludeExclude different = new IncludeExclude(differentRegex, null);
+        assertFalse(serialized.equals(different));
+        assertTrue(serialized.hashCode() != different.hashCode());
+    }
+
+    public void testRegexExclude() throws IOException {
+        String excRegex = "foo.*";
+        String differentRegex = "bar.*";
+        IncludeExclude serialized = serialize(new IncludeExclude(null, excRegex), IncludeExclude.EXCLUDE_FIELD);
+        assertFalse(serialized.isPartitionBased());
+        assertTrue(serialized.isRegexBased());
+
+        IncludeExclude same = new IncludeExclude(null, excRegex);
+        assertEquals(serialized, same);
+        assertEquals(serialized.hashCode(), same.hashCode());
+
+        IncludeExclude different = new IncludeExclude(null, differentRegex);
+        assertFalse(serialized.equals(different));
+        assertTrue(serialized.hashCode() != different.hashCode());
+    }
+
+    // Serializes/deserializes an IncludeExclude statement with a single clause
+    private IncludeExclude serialize(IncludeExclude incExc, ParseField field) throws IOException {
+        XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
+        if (randomBoolean()) {
+            builder.prettyPrint();
+        }
+        builder.startObject();
+        incExc.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        builder.endObject();
+
+        XContentParser parser = createParser(builder);
+        XContentParser.Token token = parser.nextToken();
+        assertEquals(token, XContentParser.Token.START_OBJECT);
+        token = parser.nextToken();
+        assertEquals(token, XContentParser.Token.FIELD_NAME);
+        assertEquals(field.getPreferredName(), parser.currentName());
+        token = parser.nextToken();
+
+        if (field.getPreferredName().equalsIgnoreCase("include")) {
+            return IncludeExclude.parseInclude(parser);
+        } else if (field.getPreferredName().equalsIgnoreCase("exclude")) {
+            return IncludeExclude.parseExclude(parser);
+        } else {
+            throw new IllegalArgumentException(
+                    "Unexpected field name serialized in test: " + field.getPreferredName());
+        }
+    }
+
+    public void testRegexIncludeAndExclude() throws IOException {
+        String incRegex = "foo.*";
+        String excRegex = "football";
+        String differentExcRegex = "foosball";
+        IncludeExclude serialized = serializeMixedRegex(new IncludeExclude(incRegex, excRegex));
+        assertFalse(serialized.isPartitionBased());
+        assertTrue(serialized.isRegexBased());
+
+        IncludeExclude same = new IncludeExclude(incRegex, excRegex);
+        assertEquals(serialized, same);
+        assertEquals(serialized.hashCode(), same.hashCode());
+
+        IncludeExclude different = new IncludeExclude(incRegex, differentExcRegex);
+        assertFalse(serialized.equals(different));
+        assertTrue(serialized.hashCode() != different.hashCode());
+    }
+
+    // Serializes/deserializes the IncludeExclude statement with include AND
+    // exclude clauses
+    private IncludeExclude serializeMixedRegex(IncludeExclude incExc) throws IOException {
+        XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
+        if (randomBoolean()) {
+            builder.prettyPrint();
+        }
+        builder.startObject();
+        incExc.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        builder.endObject();
+
+        XContentParser parser = createParser(builder);
+        XContentParser.Token token = parser.nextToken();
+        assertEquals(token, XContentParser.Token.START_OBJECT);
+
+        IncludeExclude inc = null;
+        IncludeExclude exc = null;
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            assertEquals(XContentParser.Token.FIELD_NAME, token);
+            if (IncludeExclude.INCLUDE_FIELD.match(parser.currentName())) {
+                token = parser.nextToken();
+                inc = IncludeExclude.parseInclude(parser);
+            } else if (IncludeExclude.EXCLUDE_FIELD.match(parser.currentName())) {
+                token = parser.nextToken();
+                exc = IncludeExclude.parseExclude(parser);
+            } else {
+                throw new IllegalArgumentException("Unexpected field name serialized in test: " + parser.currentName());
+            }
+        }
+        assertNotNull(inc);
+        assertNotNull(exc);
+        // Include and Exclude clauses are parsed independently and then merged
+        return IncludeExclude.merge(inc, exc);
+    }
+
+    public void testMixRegexAndPartition() throws Exception {
+        XContentBuilder builder = JsonXContent.contentBuilder()
+                .startObject()
+                    .field("pattern", "a.*")
+                    .field("partition", 1)
+                    .field("num_partitions", 3)
+                .endObject();
+        try (XContentParser parser = createParser(builder)) {
+            parser.nextToken();
+            IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> IncludeExclude.parseInclude(parser));
+            assertEquals("Cannot mix pattern-based and partition-based includes", e.getMessage());
+            assertWarnings("Deprecated field [pattern] used, replaced by [Put patterns directly under the [include] or [exclude]]");
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java
index 3b4e84f8e7cca..2c819a671e55d 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java
@@ -293,4 +293,9 @@ public int count() {
             }
         }
     }
+
+    public void testFloatingPointDetection() {
+        assertFalse(MissingValues.replaceMissing(ValuesSource.Numeric.EMPTY, 3).isFloatingPoint());
+        assertTrue(MissingValues.replaceMissing(ValuesSource.Numeric.EMPTY, 3.5).isFloatingPoint());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java
index dcf6cfdb0ba0b..f338e961312bd 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java
@@ -108,6 +108,27 @@ public void testLongs() {
         }
     }
 
+    public void testBooleans() {
+        final Object[][] values = new Boolean[randomInt(10)][];
+        for (int i = 0; i < values.length; ++i) {
+            Boolean[] booleans = new Boolean[randomInt(8)];
+            for (int j = 0; j < booleans.length; ++j) {
+                booleans[j] = randomBoolean();
+            }
+            Arrays.sort(booleans);
+            values[i] = booleans;
+        }
+        FakeSearchScript script = new FakeSearchScript(values);
+        ScriptLongValues scriptValues = new ScriptLongValues(script);
+        for (int i = 0; i < values.length; ++i) {
+            scriptValues.setDocument(i);
+            assertEquals(values[i].length, scriptValues.count());
+            for (int j = 0; j < values[i].length; ++j) {
+                assertEquals(values[i][j], scriptValues.valueAt(j) == 1L);
+            }
+        }
+    }
+
     public void testDoubles() {
         final Object[][] values = new Double[randomInt(10)][];
         for (int i = 0; i < values.length; ++i) {
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/support/ValueTypeTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/support/ValueTypeTests.java
new file mode 100644
index 0000000000000..84995277fb7da
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/support/ValueTypeTests.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.support;
+
+import org.elasticsearch.test.ESTestCase;
+
+public class ValueTypeTests extends ESTestCase {
+
+    public void testResolve() {
+        assertEquals(ValueType.STRING, ValueType.resolveForScript("string"));
+        assertEquals(ValueType.DOUBLE, ValueType.resolveForScript("float"));
+        assertEquals(ValueType.DOUBLE, ValueType.resolveForScript("double"));
+        assertEquals(ValueType.LONG, ValueType.resolveForScript("byte"));
+        assertEquals(ValueType.LONG, ValueType.resolveForScript("short"));
+        assertEquals(ValueType.LONG, ValueType.resolveForScript("integer"));
+        assertEquals(ValueType.LONG, ValueType.resolveForScript("long"));
+        assertEquals(ValueType.DATE, ValueType.resolveForScript("date"));
+        assertEquals(ValueType.IP, ValueType.resolveForScript("ip"));
+        assertEquals(ValueType.BOOLEAN, ValueType.resolveForScript("boolean"));
+    }
+
+    public void testCompatibility() {
+        assertTrue(ValueType.DOUBLE.isA(ValueType.NUMERIC));
+        assertTrue(ValueType.LONG.isA(ValueType.NUMERIC));
+        assertTrue(ValueType.DATE.isA(ValueType.LONG));
+        assertTrue(ValueType.NUMERIC.isA(ValueType.NUMBER));
+        assertTrue(ValueType.BOOLEAN.isA(ValueType.NUMBER));
+        assertFalse(ValueType.STRING.isA(ValueType.NUMBER));
+        assertFalse(ValueType.DATE.isA(ValueType.IP));
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfigTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfigTests.java
new file mode 100644
index 0000000000000..5f8c658f234a9
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfigTests.java
@@ -0,0 +1,264 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.support;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.engine.Engine.Searcher;
+import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+
+public class ValuesSourceConfigTests extends ESSingleNodeTestCase {
+
+    public void testKeyword() throws Exception {
+        IndexService indexService = createIndex("index", Settings.EMPTY, "type",
+                "bytes", "type=keyword");
+        client().prepareIndex("index", "type", "1")
+                .setSource("bytes", "abc")
+                .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+                .get();
+
+        try (Searcher searcher = indexService.getShard(0).acquireSearcher("test")) {
+            QueryShardContext context = indexService.newQueryShardContext(0, searcher.reader(), () -> 42L);
+
+            ValuesSourceConfig<ValuesSource.Bytes> config = ValuesSourceConfig.resolve(
+                    context, null, "bytes", null, null, null, null);
+            ValuesSource.Bytes valuesSource = config.toValuesSource(context);
+            LeafReaderContext ctx = searcher.reader().leaves().get(0);
+            SortedBinaryDocValues values = valuesSource.bytesValues(ctx);
+            values.setDocument(0);
+            assertEquals(1, values.count());
+            assertEquals(new BytesRef("abc"), values.valueAt(0));
+        }
+    }
+
+    public void testEmptyKeyword() throws Exception {
+        IndexService indexService = createIndex("index", Settings.EMPTY, "type",
+                "bytes", "type=keyword");
+        client().prepareIndex("index", "type", "1")
+                .setSource()
+                .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+                .get();
+
+        try (Searcher searcher = indexService.getShard(0).acquireSearcher("test")) {
+            QueryShardContext context = indexService.newQueryShardContext(0, searcher.reader(), () -> 42L);
+
+            ValuesSourceConfig<ValuesSource.Bytes> config = ValuesSourceConfig.resolve(
+                    context, null, "bytes", null, null, null, null);
+            ValuesSource.Bytes valuesSource = config.toValuesSource(context);
+            LeafReaderContext ctx = searcher.reader().leaves().get(0);
+            SortedBinaryDocValues values = valuesSource.bytesValues(ctx);
+            values.setDocument(0);
+            assertEquals(0, values.count());
+
+            config = ValuesSourceConfig.resolve(
+                    context, null, "bytes", null, "abc", null, null);
+            valuesSource = config.toValuesSource(context);
+            values = valuesSource.bytesValues(ctx);
+            values.setDocument(0);
+            assertEquals(1, values.count());
+            assertEquals(new BytesRef("abc"), values.valueAt(0));
+        }
+    }
+
+    public void testUnmappedKeyword() throws Exception {
+        IndexService indexService = createIndex("index", Settings.EMPTY, "type");
+        client().prepareIndex("index", "type", "1")
+                .setSource()
+                .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+                .get();
+
+        try (Searcher searcher = indexService.getShard(0).acquireSearcher("test")) {
+            QueryShardContext context = indexService.newQueryShardContext(0, searcher.reader(), () -> 42L);
+            ValuesSourceConfig<ValuesSource.Bytes> config = ValuesSourceConfig.resolve(
+                    context, ValueType.STRING, "bytes", null, null, null, null);
+            ValuesSource.Bytes valuesSource = config.toValuesSource(context);
+            assertNull(valuesSource);
+
+            config = ValuesSourceConfig.resolve(
+                    context, ValueType.STRING, "bytes", null, "abc", null, null);
+            valuesSource = config.toValuesSource(context);
+            LeafReaderContext ctx = searcher.reader().leaves().get(0);
+            SortedBinaryDocValues values = valuesSource.bytesValues(ctx);
+            values.setDocument(0);
+            assertEquals(1, values.count());
+            assertEquals(new BytesRef("abc"), values.valueAt(0));
+        }
+    }
+
+    public void testLong() throws Exception {
+        IndexService indexService = createIndex("index", Settings.EMPTY, "type",
+                "long", "type=long");
+        client().prepareIndex("index", "type", "1")
+                .setSource("long", 42)
+                .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+                .get();
+
+        try (Searcher searcher = indexService.getShard(0).acquireSearcher("test")) {
+            QueryShardContext context = indexService.newQueryShardContext(0, searcher.reader(), () -> 42L);
+
+            ValuesSourceConfig<ValuesSource.Numeric> config = ValuesSourceConfig.resolve(
+                    context, null, "long", null, null, null, null);
+            ValuesSource.Numeric valuesSource = config.toValuesSource(context);
+            LeafReaderContext ctx = searcher.reader().leaves().get(0);
+            SortedNumericDocValues values = valuesSource.longValues(ctx);
+            values.setDocument(0);
+            assertEquals(1, values.count());
+            assertEquals(42, values.valueAt(0));
+        }
+    }
+
+    public void testEmptyLong() throws Exception {
+        IndexService indexService = createIndex("index", Settings.EMPTY, "type",
+                "long", "type=long");
+        client().prepareIndex("index", "type", "1")
+                .setSource()
+                .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+                .get();
+
+        try (Searcher searcher = indexService.getShard(0).acquireSearcher("test")) {
+            QueryShardContext context = indexService.newQueryShardContext(0, searcher.reader(), () -> 42L);
+
+            ValuesSourceConfig<ValuesSource.Numeric> config = ValuesSourceConfig.resolve(
+                    context, null, "long", null, null, null, null);
+            ValuesSource.Numeric valuesSource = config.toValuesSource(context);
+            LeafReaderContext ctx = searcher.reader().leaves().get(0);
+            SortedNumericDocValues values = valuesSource.longValues(ctx);
+            values.setDocument(0);
+            assertEquals(0, values.count());
+
+            config = ValuesSourceConfig.resolve(
+                    context, null, "long", null, 42, null, null);
+            valuesSource = config.toValuesSource(context);
+            values = valuesSource.longValues(ctx);
+            values.setDocument(0);
+            assertEquals(1, values.count());
+            assertEquals(42, values.valueAt(0));
+        }
+    }
+
+    public void testUnmappedLong() throws Exception {
+        IndexService indexService = createIndex("index", Settings.EMPTY, "type");
+        client().prepareIndex("index", "type", "1")
+                .setSource()
+                .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+                .get();
+
+        try (Searcher searcher = indexService.getShard(0).acquireSearcher("test")) {
+            QueryShardContext context = indexService.newQueryShardContext(0, searcher.reader(), () -> 42L);
+
+            ValuesSourceConfig<ValuesSource.Numeric> config = ValuesSourceConfig.resolve(
+                    context, ValueType.NUMBER, "long", null, null, null, null);
+            ValuesSource.Numeric valuesSource = config.toValuesSource(context);
+            assertNull(valuesSource);
+
+            config = ValuesSourceConfig.resolve(
+                    context, ValueType.NUMBER, "long", null, 42, null, null);
+            valuesSource = config.toValuesSource(context);
+            LeafReaderContext ctx = searcher.reader().leaves().get(0);
+            SortedNumericDocValues values = valuesSource.longValues(ctx);
+            values.setDocument(0);
+            assertEquals(1, values.count());
+            assertEquals(42, values.valueAt(0));
+        }
+    }
+
+    public void testBoolean() throws Exception {
+        IndexService indexService = createIndex("index", Settings.EMPTY, "type",
+                "bool", "type=boolean");
+        client().prepareIndex("index", "type", "1")
+                .setSource("bool", true)
+                .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+                .get();
+
+        try (Searcher searcher = indexService.getShard(0).acquireSearcher("test")) {
+            QueryShardContext context = indexService.newQueryShardContext(0, searcher.reader(), () -> 42L);
+
+            ValuesSourceConfig<ValuesSource.Numeric> config = ValuesSourceConfig.resolve(
+                    context, null, "bool", null, null, null, null);
+            ValuesSource.Numeric valuesSource = config.toValuesSource(context);
+            LeafReaderContext ctx = searcher.reader().leaves().get(0);
+            SortedNumericDocValues values = valuesSource.longValues(ctx);
+            values.setDocument(0);
+            assertEquals(1, values.count());
+            assertEquals(1, values.valueAt(0));
+        }
+    }
+
+    public void testEmptyBoolean() throws Exception {
+        IndexService indexService = createIndex("index", Settings.EMPTY, "type",
+                "bool", "type=boolean");
+        client().prepareIndex("index", "type", "1")
+                .setSource()
+                .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+                .get();
+
+        try (Searcher searcher = indexService.getShard(0).acquireSearcher("test")) {
+            QueryShardContext context = indexService.newQueryShardContext(0, searcher.reader(), () -> 42L);
+
+            ValuesSourceConfig<ValuesSource.Numeric> config = ValuesSourceConfig.resolve(
+                    context, null, "bool", null, null, null, null);
+            ValuesSource.Numeric valuesSource = config.toValuesSource(context);
+            LeafReaderContext ctx = searcher.reader().leaves().get(0);
+            SortedNumericDocValues values = valuesSource.longValues(ctx);
+            values.setDocument(0);
+            assertEquals(0, values.count());
+
+            config = ValuesSourceConfig.resolve(
+                    context, null, "bool", null, true, null, null);
+            valuesSource = config.toValuesSource(context);
+            values = valuesSource.longValues(ctx);
+            values.setDocument(0);
+            assertEquals(1, values.count());
+            assertEquals(1, values.valueAt(0));
+        }
+    }
+
+    public void testUnmappedBoolean() throws Exception {
+        IndexService indexService = createIndex("index", Settings.EMPTY, "type");
+        client().prepareIndex("index", "type", "1")
+                .setSource()
+                .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+                .get();
+
+        try (Searcher searcher = indexService.getShard(0).acquireSearcher("test")) {
+            QueryShardContext context = indexService.newQueryShardContext(0, searcher.reader(), () -> 42L);
+
+            ValuesSourceConfig<ValuesSource.Numeric> config = ValuesSourceConfig.resolve(
+                    context, ValueType.BOOLEAN, "bool", null, null, null, null);
+            ValuesSource.Numeric valuesSource = config.toValuesSource(context);
+            assertNull(valuesSource);
+
+            config = ValuesSourceConfig.resolve(
+                    context, ValueType.BOOLEAN, "bool", null, true, null, null);
+            valuesSource = config.toValuesSource(context);
+            LeafReaderContext ctx = searcher.reader().leaves().get(0);
+            SortedNumericDocValues values = valuesSource.longValues(ctx);
+            values.setDocument(0);
+            assertEquals(1, values.count());
+            assertEquals(1, values.valueAt(0));
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/basic/SearchWhileCreatingIndexIT.java b/core/src/test/java/org/elasticsearch/search/basic/SearchWhileCreatingIndexIT.java
index 96f6f055a3289..c83b3ccc9e505 100644
--- a/core/src/test/java/org/elasticsearch/search/basic/SearchWhileCreatingIndexIT.java
+++ b/core/src/test/java/org/elasticsearch/search/basic/SearchWhileCreatingIndexIT.java
@@ -56,11 +56,11 @@ private void searchWhileCreatingIndex(boolean createIndex, int numberOfReplicas)
 
         // TODO: randomize the wait for active shards value on index creation and ensure the appropriate
         // number of data nodes are started for the randomized active shard count value
-        String id = randomAsciiOfLength(5);
+        String id = randomAlphaOfLength(5);
         // we will go the primary or the replica, but in a
         // randomized re-creatable manner
         int counter = 0;
-        String preference = randomAsciiOfLength(5);
+        String preference = randomAlphaOfLength(5);
 
         logger.info("running iteration for id {}, preference {}", id, preference);
 
diff --git a/core/src/test/java/org/elasticsearch/search/basic/SearchWithRandomExceptionsIT.java b/core/src/test/java/org/elasticsearch/search/basic/SearchWithRandomExceptionsIT.java
index dc2898424fd5d..e406cb72aeab8 100644
--- a/core/src/test/java/org/elasticsearch/search/basic/SearchWithRandomExceptionsIT.java
+++ b/core/src/test/java/org/elasticsearch/search/basic/SearchWithRandomExceptionsIT.java
@@ -44,10 +44,13 @@
 import org.elasticsearch.test.engine.ThrowingLeafReaderWrapper;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Random;
+import java.util.Set;
 import java.util.concurrent.ExecutionException;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@@ -56,7 +59,14 @@ public class SearchWithRandomExceptionsIT extends ESIntegTestCase {
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Arrays.asList(RandomExceptionDirectoryReaderWrapper.TestPlugin.class, MockEngineFactoryPlugin.class);
+        return Arrays.asList(RandomExceptionDirectoryReaderWrapper.TestPlugin.class);
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> getMockPlugins() {
+        Set<Class<? extends Plugin>> mocks = new HashSet<>(super.getMockPlugins());
+        mocks.remove(MockEngineFactoryPlugin.class);
+        return mocks;
     }
 
     public void testRandomExceptions() throws IOException, InterruptedException, ExecutionException {
@@ -153,17 +163,22 @@ public void testRandomExceptions() throws IOException, InterruptedException, Exe
 
     public static class RandomExceptionDirectoryReaderWrapper extends MockEngineSupport.DirectoryReaderWrapper {
 
-        public static class TestPlugin extends Plugin {
+        public static class TestPlugin extends MockEngineFactoryPlugin {
             public static final Setting<Double> EXCEPTION_TOP_LEVEL_RATIO_SETTING =
                 Setting.doubleSetting(EXCEPTION_TOP_LEVEL_RATIO_KEY, 0.1d, 0.0d, Property.IndexScope);
             public static final Setting<Double> EXCEPTION_LOW_LEVEL_RATIO_SETTING =
                 Setting.doubleSetting(EXCEPTION_LOW_LEVEL_RATIO_KEY, 0.1d, 0.0d, Property.IndexScope);
             @Override
             public List<Setting<?>> getSettings() {
-                return Arrays.asList(EXCEPTION_TOP_LEVEL_RATIO_SETTING, EXCEPTION_LOW_LEVEL_RATIO_SETTING);
+                List<Setting<?>> settings = new ArrayList<>();
+                settings.addAll(super.getSettings());
+                settings.add(EXCEPTION_TOP_LEVEL_RATIO_SETTING);
+                settings.add(EXCEPTION_LOW_LEVEL_RATIO_SETTING);
+                return settings;
             }
-            public void onModule(MockEngineFactoryPlugin.MockEngineReaderModule module) {
-                module.setReaderClass(RandomExceptionDirectoryReaderWrapper.class);
+            @Override
+            protected Class<? extends FilterDirectoryReader> getReaderWrapperClass() {
+                return RandomExceptionDirectoryReaderWrapper.class;
             }
         }
 
diff --git a/core/src/test/java/org/elasticsearch/search/basic/SearchWithRandomIOExceptionsIT.java b/core/src/test/java/org/elasticsearch/search/basic/SearchWithRandomIOExceptionsIT.java
index 07f696e491ac3..61dd798f5e111 100644
--- a/core/src/test/java/org/elasticsearch/search/basic/SearchWithRandomIOExceptionsIT.java
+++ b/core/src/test/java/org/elasticsearch/search/basic/SearchWithRandomIOExceptionsIT.java
@@ -99,7 +99,7 @@ public void testRandomDirectoryIOExceptions() throws IOException, InterruptedExc
                 client().prepareIndex("test", "type", "init" + i).setSource("test", "init").get();
             }
             client().admin().indices().prepareRefresh("test").execute().get();
-            client().admin().indices().prepareFlush("test").setWaitIfOngoing(true).execute().get();
+            client().admin().indices().prepareFlush("test").execute().get();
             client().admin().indices().prepareClose("test").execute().get();
             client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder()
                 .put(MockFSDirectoryService.RANDOM_IO_EXCEPTION_RATE_SETTING.getKey(), exceptionRate)
diff --git a/core/src/test/java/org/elasticsearch/search/basic/TransportTwoNodesSearchIT.java b/core/src/test/java/org/elasticsearch/search/basic/TransportTwoNodesSearchIT.java
index 49c24ba995279..fd84d5b62c66f 100644
--- a/core/src/test/java/org/elasticsearch/search/basic/TransportTwoNodesSearchIT.java
+++ b/core/src/test/java/org/elasticsearch/search/basic/TransportTwoNodesSearchIT.java
@@ -32,7 +32,7 @@
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.functionscore.ScriptScoreFunctionBuilder;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
@@ -43,13 +43,12 @@
 import org.elasticsearch.test.ESIntegTestCase;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
 import java.util.TreeSet;
 
-import static org.elasticsearch.action.search.SearchType.DFS_QUERY_AND_FETCH;
 import static org.elasticsearch.action.search.SearchType.DFS_QUERY_THEN_FETCH;
-import static org.elasticsearch.action.search.SearchType.QUERY_AND_FETCH;
 import static org.elasticsearch.action.search.SearchType.QUERY_THEN_FETCH;
 import static org.elasticsearch.client.Requests.createIndexRequest;
 import static org.elasticsearch.client.Requests.searchRequest;
@@ -60,7 +59,6 @@
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
 import static org.elasticsearch.search.builder.SearchSourceBuilder.searchSource;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
-import static org.hamcrest.Matchers.endsWith;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
@@ -150,8 +148,15 @@ public void testDfsQueryThenFetch() throws Exception {
                 assertThat(hit.explanation(), notNullValue());
                 assertThat(hit.explanation().getDetails().length, equalTo(1));
                 assertThat(hit.explanation().getDetails()[0].getDetails().length, equalTo(2));
-                assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDescription(),
-                    endsWith("idf(docFreq=100, docCount=100)"));
+                assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDetails().length, equalTo(2));
+                assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDetails()[0].getDescription(),
+                    equalTo("docFreq"));
+                assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDetails()[0].getValue(),
+                    equalTo(100.0f));
+                assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDetails()[1].getDescription(),
+                    equalTo("docCount"));
+                assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDetails()[1].getValue(),
+                    equalTo(100.0f));
                 assertThat("id[" + hit.id() + "] -> " + hit.explanation().toString(), hit.id(), equalTo(Integer.toString(100 - total - i - 1)));
             }
             total += hits.length;
@@ -178,8 +183,15 @@ public void testDfsQueryThenFetchWithSort() throws Exception {
                 assertThat(hit.explanation(), notNullValue());
                 assertThat(hit.explanation().getDetails().length, equalTo(1));
                 assertThat(hit.explanation().getDetails()[0].getDetails().length, equalTo(2));
-                assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDescription(),
-                    endsWith("idf(docFreq=100, docCount=100)"));
+                assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDetails().length, equalTo(2));
+                assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDetails()[0].getDescription(),
+                    equalTo("docFreq"));
+                assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDetails()[0].getValue(),
+                    equalTo(100.0f));
+                assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDetails()[1].getDescription(),
+                    equalTo("docCount"));
+                assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDetails()[1].getValue(),
+                    equalTo(100.0f));
                 assertThat("id[" + hit.id() + "]", hit.id(), equalTo(Integer.toString(total + i)));
             }
             total += hits.length;
@@ -265,90 +277,6 @@ public void testQueryThenFetchWithSort() throws Exception {
         assertEquals(100, total);
     }
 
-    public void testQueryAndFetch() throws Exception {
-        prepareData(3);
-
-        SearchSourceBuilder source = searchSource()
-                .query(termQuery("multi", "test"))
-                .from(0).size(20).explain(true);
-
-        Set<String> expectedIds = new HashSet<>();
-        for (int i = 0; i < 100; i++) {
-            expectedIds.add(Integer.toString(i));
-        }
-
-        SearchResponse searchResponse = client().search(searchRequest("test").source(source).searchType(QUERY_AND_FETCH).scroll(new Scroll(timeValueMinutes(10)))).actionGet();
-        assertNoFailures(searchResponse);
-        assertThat(searchResponse.getHits().totalHits(), equalTo(100L));
-        assertThat(searchResponse.getHits().hits().length, equalTo(60)); // 20 per shard
-        for (int i = 0; i < 60; i++) {
-            SearchHit hit = searchResponse.getHits().hits()[i];
-//            System.out.println(hit.shard() + ": " +  hit.explanation());
-            assertThat(hit.explanation(), notNullValue());
-            // we can't really check here, since its query and fetch, and not controlling distribution
-//            assertThat("id[" + hit.id() + "]", hit.id(), equalTo(Integer.toString(100 - i - 1)));
-            assertThat("make sure we don't have duplicates", expectedIds.remove(hit.id()), notNullValue());
-        }
-
-        do {
-            searchResponse = client().prepareSearchScroll(searchResponse.getScrollId()).setScroll("10m").get();
-            assertThat(searchResponse.getHits().totalHits(), equalTo(100L));
-            assertThat(searchResponse.getHits().hits().length, lessThanOrEqualTo(40));
-            for (int i = 0; i < searchResponse.getHits().hits().length; i++) {
-                SearchHit hit = searchResponse.getHits().hits()[i];
-                // we don't do perfect sorting when it comes to scroll with Query+Fetch
-                assertThat("make sure we don't have duplicates", expectedIds.remove(hit.id()), notNullValue());
-            }
-        } while (searchResponse.getHits().getHits().length > 0);
-        clearScroll(searchResponse.getScrollId());
-        assertThat("make sure we got all [" + expectedIds + "]", expectedIds.size(), equalTo(0));
-    }
-
-    public void testDfsQueryAndFetch() throws Exception {
-        prepareData(3);
-
-        SearchSourceBuilder source = searchSource()
-                .query(termQuery("multi", "test"))
-                .from(0).size(20).explain(true);
-
-        Set<String> expectedIds = new HashSet<>();
-        for (int i = 0; i < 100; i++) {
-            expectedIds.add(Integer.toString(i));
-        }
-
-
-        //SearchResponse searchResponse = client().search(searchRequest("test").source(source).searchType(DFS_QUERY_AND_FETCH).scroll(new Scroll(timeValueMinutes(10)))).actionGet();
-        SearchResponse searchResponse = client().prepareSearch("test").setSearchType(DFS_QUERY_AND_FETCH).setScroll("10m").setSource(source).get();
-        assertNoFailures(searchResponse);
-        assertThat(searchResponse.getHits().totalHits(), equalTo(100L));
-        assertThat(searchResponse.getHits().hits().length, equalTo(60)); // 20 per shard
-        for (int i = 0; i < 60; i++) {
-            SearchHit hit = searchResponse.getHits().hits()[i];
-//            System.out.println(hit.shard() + ": " +  hit.explanation());
-            assertThat(hit.explanation(), notNullValue());
-            assertThat(hit.explanation().getDetails().length, equalTo(1));
-            assertThat(hit.explanation().getDetails()[0].getDetails().length, equalTo(2));
-            assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDescription(),
-                endsWith("idf(docFreq=100, docCount=100)"));
-//            assertThat("id[" + hit.id() + "]", hit.id(), equalTo(Integer.toString(100 - i - 1)));
-            assertThat("make sure we don't have duplicates", expectedIds.remove(hit.id()), notNullValue());
-        }
-
-        do {
-            searchResponse = client().prepareSearchScroll(searchResponse.getScrollId()).setScroll("10m").get();
-
-            assertThat(searchResponse.getHits().totalHits(), equalTo(100L));
-            assertThat(searchResponse.getHits().hits().length, lessThanOrEqualTo(40));
-            for (int i = 0; i < searchResponse.getHits().hits().length; i++) {
-                SearchHit hit = searchResponse.getHits().hits()[i];
-                // we don't do perfect sorting when it comes to scroll with Query+Fetch
-                assertThat("make sure we don't have duplicates", expectedIds.remove(hit.id()), notNullValue());
-            }
-        } while (searchResponse.getHits().hits().length > 0);
-        clearScroll(searchResponse.getScrollId());
-        assertThat("make sure we got all [" + expectedIds + "]", expectedIds.size(), equalTo(0));
-    }
-
     public void testSimpleFacets() throws Exception {
         prepareData();
 
@@ -399,7 +327,7 @@ public void testFailedSearchWithWrongFrom() throws Exception {
         SearchSourceBuilder source = searchSource()
                 .query(termQuery("multi", "test"))
                 .from(1000).size(20).explain(true);
-        SearchResponse response = client().search(searchRequest("test").searchType(DFS_QUERY_AND_FETCH).source(source)).actionGet();
+        SearchResponse response = client().search(searchRequest("test").searchType(DFS_QUERY_THEN_FETCH).source(source)).actionGet();
         assertThat(response.getHits().hits().length, equalTo(0));
         assertThat(response.getTotalShards(), equalTo(test.numPrimaries));
         assertThat(response.getSuccessfulShards(), equalTo(test.numPrimaries));
@@ -409,7 +337,7 @@ public void testFailedSearchWithWrongFrom() throws Exception {
         assertNoFailures(response);
         assertThat(response.getHits().hits().length, equalTo(0));
 
-        response = client().search(searchRequest("test").searchType(DFS_QUERY_AND_FETCH).source(source)).actionGet();
+        response = client().search(searchRequest("test").searchType(DFS_QUERY_THEN_FETCH).source(source)).actionGet();
         assertNoFailures(response);
         assertThat(response.getHits().hits().length, equalTo(0));
 
@@ -450,7 +378,7 @@ public void testFailedMultiSearchWithWrongQueryWithFunctionScore() throws Except
 
         MultiSearchResponse response = client().prepareMultiSearch()
                 // Add custom score query with bogus script
-                .add(client().prepareSearch("test").setQuery(QueryBuilders.functionScoreQuery(QueryBuilders.termQuery("nid", 1), new ScriptScoreFunctionBuilder(new Script("foo", ScriptService.ScriptType.INLINE, "bar", null)))))
+                .add(client().prepareSearch("test").setQuery(QueryBuilders.functionScoreQuery(QueryBuilders.termQuery("nid", 1), new ScriptScoreFunctionBuilder(new Script(ScriptType.INLINE, "bar", "foo", Collections.emptyMap())))))
                 .add(client().prepareSearch("test").setQuery(QueryBuilders.termQuery("nid", 2)))
                 .add(client().prepareSearch("test").setQuery(QueryBuilders.matchAllQuery()))
                 .execute().actionGet();
diff --git a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
index 0742fca357dfc..3e4cdd7e88052 100644
--- a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
@@ -20,393 +20,38 @@
 package org.elasticsearch.search.builder;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.Version;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.inject.AbstractModule;
-import org.elasticsearch.common.inject.Injector;
-import org.elasticsearch.common.inject.ModulesBuilder;
-import org.elasticsearch.common.inject.util.Providers;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsModule;
-import org.elasticsearch.common.text.Text;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.index.Index;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.indices.IndicesModule;
-import org.elasticsearch.indices.breaker.CircuitBreakerService;
-import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
-import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptModule;
-import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.search.SearchModule;
-import org.elasticsearch.search.SearchRequestParsers;
-import org.elasticsearch.search.aggregations.AggregationBuilders;
-import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
-import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilderTests;
-import org.elasticsearch.search.rescore.QueryRescoreBuilderTests;
+import org.elasticsearch.index.query.RandomQueryBuilder;
+import org.elasticsearch.search.AbstractSearchTestCase;
 import org.elasticsearch.search.rescore.QueryRescorerBuilder;
-import org.elasticsearch.search.searchafter.SearchAfterBuilder;
-import org.elasticsearch.search.slice.SliceBuilder;
 import org.elasticsearch.search.sort.FieldSortBuilder;
 import org.elasticsearch.search.sort.ScoreSortBuilder;
-import org.elasticsearch.search.sort.ScriptSortBuilder.ScriptSortType;
-import org.elasticsearch.search.sort.SortBuilders;
 import org.elasticsearch.search.sort.SortOrder;
-import org.elasticsearch.search.suggest.SuggestBuilderTests;
-import org.elasticsearch.test.AbstractQueryTestCase;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.IndexSettingsModule;
-import org.elasticsearch.test.InternalSettingsPlugin;
-import org.elasticsearch.test.VersionUtils;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
+import org.elasticsearch.test.EqualsHashCodeTestUtils;
 
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
+import java.util.Map;
 
-import static java.util.Collections.emptyList;
-import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
-import static org.elasticsearch.test.ClusterServiceUtils.setState;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasToString;
 
-public class SearchSourceBuilderTests extends ESTestCase {
-    private static Injector injector;
-
-    private static NamedWriteableRegistry namedWriteableRegistry;
-
-    private static SearchRequestParsers searchRequestParsers;
-
-    private static Index index;
-
-    private static String[] currentTypes;
-
-    private static ParseFieldMatcher parseFieldMatcher;
-
-    @BeforeClass
-    public static void init() throws IOException {
-        // we have to prefer CURRENT since with the range of versions we support
-        // it's rather unlikely to get the current actually.
-        Version version = randomBoolean() ? Version.CURRENT
-                : VersionUtils.randomVersionBetween(random(), Version.V_2_0_0_beta1, Version.CURRENT);
-        Settings settings = Settings.builder()
-                .put("node.name", AbstractQueryTestCase.class.toString())
-                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
-                .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), false).build();
-
-        index = new Index(randomAsciiOfLengthBetween(1, 10), "_na_");
-        Settings indexSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
-        final ThreadPool threadPool = new ThreadPool(settings);
-        final ClusterService clusterService = createClusterService(threadPool);
-        setState(clusterService, new ClusterState.Builder(clusterService.state()).metaData(new MetaData.Builder()
-                .put(new IndexMetaData.Builder(index.getName()).settings(indexSettings).numberOfShards(1).numberOfReplicas(0))));
-        ScriptModule scriptModule = newTestScriptModule();
-        List<Setting<?>> scriptSettings = scriptModule.getSettings();
-        scriptSettings.add(InternalSettingsPlugin.VERSION_CREATED);
-        SettingsModule settingsModule = new SettingsModule(settings, scriptSettings, Collections.emptyList());
-        IndicesModule indicesModule = new IndicesModule(Collections.emptyList()) {
-            @Override
-            protected void configure() {
-                bindMapperExtension();
-            }
-        };
-        SearchModule searchModule = new SearchModule(settings, false, emptyList()) {
-            @Override
-            protected void configureSearch() {
-                // Skip me
-            }
-        };
-        List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
-        entries.addAll(indicesModule.getNamedWriteables());
-        entries.addAll(searchModule.getNamedWriteables());
-        namedWriteableRegistry = new NamedWriteableRegistry(entries);
-        injector = new ModulesBuilder().add(
-                (b) -> {
-                    b.bind(Environment.class).toInstance(new Environment(settings));
-                    b.bind(ThreadPool.class).toInstance(threadPool);
-                    b.bind(ScriptService.class).toInstance(scriptModule.getScriptService());
-                },
-                settingsModule, indicesModule, searchModule,
-                new IndexSettingsModule(index, settings),
-                new AbstractModule() {
-                    @Override
-                    protected void configure() {
-                        bind(ClusterService.class).toProvider(Providers.of(clusterService));
-                        bind(CircuitBreakerService.class).to(NoneCircuitBreakerService.class);
-                        bind(NamedWriteableRegistry.class).toInstance(namedWriteableRegistry);
-                    }
-                }
-        ).createInjector();
-        searchRequestParsers = injector.getInstance(SearchRequestParsers.class);
-        // create some random type with some default field, those types will
-        // stick around for all of the subclasses
-        currentTypes = new String[randomIntBetween(0, 5)];
-        for (int i = 0; i < currentTypes.length; i++) {
-            String type = randomAsciiOfLengthBetween(1, 10);
-            currentTypes[i] = type;
-        }
-        parseFieldMatcher = ParseFieldMatcher.STRICT;
-    }
-
-    @AfterClass
-    public static void afterClass() throws Exception {
-        injector.getInstance(ClusterService.class).close();
-        terminate(injector.getInstance(ThreadPool.class));
-        injector = null;
-        index = null;
-        searchRequestParsers = null;
-        currentTypes = null;
-        namedWriteableRegistry = null;
-    }
-
-    public static SearchSourceBuilder createSearchSourceBuilder() throws IOException {
-        SearchSourceBuilder builder = new SearchSourceBuilder();
-        if (randomBoolean()) {
-            builder.from(randomIntBetween(0, 10000));
-        }
-        if (randomBoolean()) {
-            builder.size(randomIntBetween(0, 10000));
-        }
-        if (randomBoolean()) {
-            builder.explain(randomBoolean());
-        }
-        if (randomBoolean()) {
-            builder.version(randomBoolean());
-        }
-        if (randomBoolean()) {
-            builder.trackScores(randomBoolean());
-        }
-        if (randomBoolean()) {
-            builder.minScore(randomFloat() * 1000);
-        }
-        if (randomBoolean()) {
-            builder.timeout(TimeValue.parseTimeValue(randomTimeValue(), null, "timeout"));
-        }
-        if (randomBoolean()) {
-            builder.terminateAfter(randomIntBetween(1, 100000));
-        }
-        // if (randomBoolean()) {
-        // builder.defaultRescoreWindowSize(randomIntBetween(1, 100));
-        // }
-
-        switch(randomInt(2)) {
-            case 0:
-                builder.storedFields();
-                break;
-            case 1:
-                builder.storedField("_none_");
-                break;
-            case 2:
-                int fieldsSize = randomInt(25);
-                List<String> fields = new ArrayList<>(fieldsSize);
-                for (int i = 0; i < fieldsSize; i++) {
-                    fields.add(randomAsciiOfLengthBetween(5, 50));
-                }
-                builder.storedFields(fields);
-                break;
-            default:
-                throw new IllegalStateException();
-        }
-
-        if (randomBoolean()) {
-            int scriptFieldsSize = randomInt(25);
-            for (int i = 0; i < scriptFieldsSize; i++) {
-                if (randomBoolean()) {
-                    builder.scriptField(randomAsciiOfLengthBetween(5, 50), new Script("foo"), randomBoolean());
-                } else {
-                    builder.scriptField(randomAsciiOfLengthBetween(5, 50), new Script("foo"));
-                }
-            }
-        }
-        if (randomBoolean()) {
-            FetchSourceContext fetchSourceContext;
-            int branch = randomInt(5);
-            String[] includes = new String[randomIntBetween(0, 20)];
-            for (int i = 0; i < includes.length; i++) {
-                includes[i] = randomAsciiOfLengthBetween(5, 20);
-            }
-            String[] excludes = new String[randomIntBetween(0, 20)];
-            for (int i = 0; i < excludes.length; i++) {
-                excludes[i] = randomAsciiOfLengthBetween(5, 20);
-            }
-            switch (branch) {
-                case 0:
-                    fetchSourceContext = new FetchSourceContext(randomBoolean());
-                    break;
-                case 1:
-                    fetchSourceContext = new FetchSourceContext(includes, excludes);
-                    break;
-                case 2:
-                    fetchSourceContext = new FetchSourceContext(randomAsciiOfLengthBetween(5, 20), randomAsciiOfLengthBetween(5, 20));
-                    break;
-                case 3:
-                    fetchSourceContext = new FetchSourceContext(true, includes, excludes);
-                    break;
-                case 4:
-                    fetchSourceContext = new FetchSourceContext(includes);
-                    break;
-                case 5:
-                    fetchSourceContext = new FetchSourceContext(randomAsciiOfLengthBetween(5, 20));
-                    break;
-                default:
-                    throw new IllegalStateException();
-            }
-            builder.fetchSource(fetchSourceContext);
-        }
-        if (randomBoolean()) {
-            int size = randomIntBetween(0, 20);
-            List<String> statsGroups = new ArrayList<>(size);
-            for (int i = 0; i < size; i++) {
-                statsGroups.add(randomAsciiOfLengthBetween(5, 20));
-            }
-            builder.stats(statsGroups);
-        }
-        if (randomBoolean()) {
-            int indexBoostSize = randomIntBetween(1, 10);
-            for (int i = 0; i < indexBoostSize; i++) {
-                builder.indexBoost(randomAsciiOfLengthBetween(5, 20), randomFloat() * 10);
-            }
-        }
-        if (randomBoolean()) {
-            builder.query(QueryBuilders.termQuery(randomAsciiOfLengthBetween(5, 20), randomAsciiOfLengthBetween(5, 20)));
-        }
-        if (randomBoolean()) {
-            builder.postFilter(QueryBuilders.termQuery(randomAsciiOfLengthBetween(5, 20), randomAsciiOfLengthBetween(5, 20)));
-        }
-        if (randomBoolean()) {
-            int numSorts = randomIntBetween(1, 5);
-            for (int i = 0; i < numSorts; i++) {
-                int branch = randomInt(5);
-                switch (branch) {
-                    case 0:
-                        builder.sort(SortBuilders.fieldSort(randomAsciiOfLengthBetween(5, 20)).order(randomFrom(SortOrder.values())));
-                        break;
-                    case 1:
-                        builder.sort(SortBuilders.geoDistanceSort(randomAsciiOfLengthBetween(5, 20),
-                                AbstractQueryTestCase.randomGeohash(1, 12)).order(randomFrom(SortOrder.values())));
-                        break;
-                    case 2:
-                        builder.sort(SortBuilders.scoreSort().order(randomFrom(SortOrder.values())));
-                        break;
-                    case 3:
-                        builder.sort(SortBuilders.scriptSort(new Script("foo"),
-                                ScriptSortType.NUMBER).order(randomFrom(SortOrder.values())));
-                        break;
-                    case 4:
-                        builder.sort(randomAsciiOfLengthBetween(5, 20));
-                        break;
-                    case 5:
-                        builder.sort(randomAsciiOfLengthBetween(5, 20), randomFrom(SortOrder.values()));
-                        break;
-                }
-            }
-        }
-
-        if (randomBoolean()) {
-            int numSearchFrom = randomIntBetween(1, 5);
-            // We build a json version of the search_from first in order to
-            // ensure that every number type remain the same before/after xcontent (de)serialization.
-            // This is not a problem because the final type of each field value is extracted from associated sort field.
-            // This little trick ensure that equals and hashcode are the same when using the xcontent serialization.
-            XContentBuilder jsonBuilder = XContentFactory.jsonBuilder();
-            jsonBuilder.startObject();
-            jsonBuilder.startArray("search_from");
-            for (int i = 0; i < numSearchFrom; i++) {
-                int branch = randomInt(8);
-                switch (branch) {
-                    case 0:
-                        jsonBuilder.value(randomInt());
-                        break;
-                    case 1:
-                        jsonBuilder.value(randomFloat());
-                        break;
-                    case 2:
-                        jsonBuilder.value(randomLong());
-                        break;
-                    case 3:
-                        jsonBuilder.value(randomDouble());
-                        break;
-                    case 4:
-                        jsonBuilder.value(randomAsciiOfLengthBetween(5, 20));
-                        break;
-                    case 5:
-                        jsonBuilder.value(randomBoolean());
-                        break;
-                    case 6:
-                        jsonBuilder.value(randomByte());
-                        break;
-                    case 7:
-                        jsonBuilder.value(randomShort());
-                        break;
-                    case 8:
-                        jsonBuilder.value(new Text(randomAsciiOfLengthBetween(5, 20)));
-                        break;
-                }
-            }
-            jsonBuilder.endArray();
-            jsonBuilder.endObject();
-            XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(jsonBuilder.bytes());
-            parser.nextToken();
-            parser.nextToken();
-            parser.nextToken();
-            builder.searchAfter(SearchAfterBuilder.fromXContent(parser, null).getSortValues());
-        }
-        if (randomBoolean()) {
-            builder.highlighter(HighlightBuilderTests.randomHighlighterBuilder());
-        }
-        if (randomBoolean()) {
-            builder.suggest(SuggestBuilderTests.randomSuggestBuilder());
-        }
-        if (randomBoolean()) {
-            int numRescores = randomIntBetween(1, 5);
-            for (int i = 0; i < numRescores; i++) {
-                builder.addRescorer(QueryRescoreBuilderTests.randomRescoreBuilder());
-            }
-        }
-        if (randomBoolean()) {
-            builder.aggregation(AggregationBuilders.avg(randomAsciiOfLengthBetween(5, 20)));
-        }
-        if (randomBoolean()) {
-            XContentBuilder xContentBuilder = XContentFactory.jsonBuilder();
-            xContentBuilder.startObject();
-            xContentBuilder.field("term_vectors_fetch", randomAsciiOfLengthBetween(5, 20));
-            xContentBuilder.endObject();
-            builder.ext(xContentBuilder);
-        }
-        if (randomBoolean()) {
-            String field = randomBoolean() ? null : randomAsciiOfLengthBetween(5, 20);
-            int max = between(2, 1000);
-            int id = randomInt(max-1);
-            if (field == null) {
-                builder.slice(new SliceBuilder(id, max));
-            } else {
-                builder.slice(new SliceBuilder(field, id, max));
-            }
-        }
-        return builder;
-    }
+public class SearchSourceBuilderTests extends AbstractSearchTestCase {
 
     public void testFromXContent() throws IOException {
         SearchSourceBuilder testSearchSourceBuilder = createSearchSourceBuilder();
@@ -415,31 +60,23 @@ public void testFromXContent() throws IOException {
             builder.prettyPrint();
         }
         testSearchSourceBuilder.toXContent(builder, ToXContent.EMPTY_PARAMS);
-        assertParseSearchSource(testSearchSourceBuilder, builder.bytes());
-    }
-
-    private static void assertParseSearchSource(SearchSourceBuilder testBuilder, BytesReference searchSourceAsBytes) throws IOException {
-        assertParseSearchSource(testBuilder, searchSourceAsBytes, ParseFieldMatcher.STRICT);
+        assertParseSearchSource(testSearchSourceBuilder, createParser(builder));
     }
 
-    private static void assertParseSearchSource(SearchSourceBuilder testBuilder, BytesReference searchSourceAsBytes, ParseFieldMatcher pfm)
-            throws IOException {
-        XContentParser parser = XContentFactory.xContent(searchSourceAsBytes).createParser(searchSourceAsBytes);
-        QueryParseContext parseContext = new QueryParseContext(searchRequestParsers.queryParsers, parser, pfm);
+    private static void assertParseSearchSource(SearchSourceBuilder testBuilder, XContentParser parser) throws IOException {
+        QueryParseContext parseContext = new QueryParseContext(parser);
         if (randomBoolean()) {
             parser.nextToken(); // sometimes we move it on the START_OBJECT to
                                 // test the embedded case
         }
-        SearchSourceBuilder newBuilder = SearchSourceBuilder.fromXContent(parseContext, searchRequestParsers.aggParsers,
-            searchRequestParsers.suggesters);
+        SearchSourceBuilder newBuilder = SearchSourceBuilder.fromXContent(parseContext);
         assertNull(parser.nextToken());
         assertEquals(testBuilder, newBuilder);
         assertEquals(testBuilder.hashCode(), newBuilder.hashCode());
     }
 
-    private static QueryParseContext createParseContext(XContentParser parser) {
-        QueryParseContext context = new QueryParseContext(searchRequestParsers.queryParsers, parser, parseFieldMatcher);
-        return context;
+    private QueryParseContext createParseContext(XContentParser parser) {
+        return new QueryParseContext(parser);
     }
 
     public void testSerialization() throws IOException {
@@ -456,57 +93,28 @@ public void testSerialization() throws IOException {
     }
 
     public void testEqualsAndHashcode() throws IOException {
-        SearchSourceBuilder firstBuilder = createSearchSourceBuilder();
-        assertNotNull("source builder is equal to null", firstBuilder);
-        assertFalse("source builder is equal to incompatible type", firstBuilder.equals(""));
-        assertTrue("source builder is not equal to self", firstBuilder.equals(firstBuilder));
-        assertThat("same source builder's hashcode returns different values if called multiple times", firstBuilder.hashCode(),
-                equalTo(firstBuilder.hashCode()));
-
-        SearchSourceBuilder secondBuilder = copyBuilder(firstBuilder);
-        assertTrue("source builder is not equal to self", secondBuilder.equals(secondBuilder));
-        assertTrue("source builder is not equal to its copy", firstBuilder.equals(secondBuilder));
-        assertTrue("source builder is not symmetric", secondBuilder.equals(firstBuilder));
-        assertThat("source builder copy's hashcode is different from original hashcode",
-                secondBuilder.hashCode(), equalTo(firstBuilder.hashCode()));
-
-        SearchSourceBuilder thirdBuilder = copyBuilder(secondBuilder);
-        assertTrue("source builder is not equal to self", thirdBuilder.equals(thirdBuilder));
-        assertTrue("source builder is not equal to its copy", secondBuilder.equals(thirdBuilder));
-        assertThat("source builder copy's hashcode is different from original hashcode",
-                secondBuilder.hashCode(), equalTo(thirdBuilder.hashCode()));
-        assertTrue("equals is not transitive", firstBuilder.equals(thirdBuilder));
-        assertThat("source builder copy's hashcode is different from original hashcode",
-                firstBuilder.hashCode(), equalTo(thirdBuilder.hashCode()));
-        assertTrue("equals is not symmetric", thirdBuilder.equals(secondBuilder));
-        assertTrue("equals is not symmetric", thirdBuilder.equals(firstBuilder));
+        // TODO add test checking that changing any member of this class produces an object that is not equal to the original
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(createSearchSourceBuilder(), this::copyBuilder);
     }
 
     //we use the streaming infra to create a copy of the builder provided as argument
-    protected static SearchSourceBuilder copyBuilder(SearchSourceBuilder builder) throws IOException {
-        try (BytesStreamOutput output = new BytesStreamOutput()) {
-            builder.writeTo(output);
-            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
-                return new SearchSourceBuilder(in);
-            }
-        }
+    private SearchSourceBuilder copyBuilder(SearchSourceBuilder original) throws IOException {
+        return ESTestCase.copyWriteable(original, namedWriteableRegistry, SearchSourceBuilder::new);
     }
 
     public void testParseIncludeExclude() throws IOException {
         {
             String restContent = " { \"_source\": { \"includes\": \"include\", \"excludes\": \"*.field2\"}}";
-            try (XContentParser parser = XContentFactory.xContent(restContent).createParser(restContent)) {
-                SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser),
-                        searchRequestParsers.aggParsers, searchRequestParsers.suggesters);
+            try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) {
+                SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser));
                 assertArrayEquals(new String[]{"*.field2"}, searchSourceBuilder.fetchSource().excludes());
                 assertArrayEquals(new String[]{"include"}, searchSourceBuilder.fetchSource().includes());
             }
         }
         {
             String restContent = " { \"_source\": false}";
-            try (XContentParser parser = XContentFactory.xContent(restContent).createParser(restContent)) {
-                SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser),
-                    searchRequestParsers.aggParsers, searchRequestParsers.suggesters);
+            try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) {
+                SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser));
                 assertArrayEquals(new String[]{}, searchSourceBuilder.fetchSource().excludes());
                 assertArrayEquals(new String[]{}, searchSourceBuilder.fetchSource().includes());
                 assertFalse(searchSourceBuilder.fetchSource().fetchSource());
@@ -514,12 +122,30 @@ public void testParseIncludeExclude() throws IOException {
         }
     }
 
+    public void testMultipleQueryObjectsAreRejected() throws Exception {
+        String restContent =
+                " { \"query\": {\n" +
+                "    \"multi_match\": {\n" +
+                "      \"query\": \"workd\",\n" +
+                "      \"fields\": [\"title^5\", \"plain_body\"]\n" +
+                "    },\n" +
+                "    \"filters\": {\n" +
+                "      \"terms\": {\n" +
+                "        \"status\": [ 3 ]\n" +
+                "      }\n" +
+                "    }\n" +
+                "  } }";
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) {
+            ParsingException e = expectThrows(ParsingException.class, () -> SearchSourceBuilder.fromXContent(createParseContext(parser)));
+            assertEquals("[multi_match] malformed query, expected [END_OBJECT] but found [FIELD_NAME]", e.getMessage());
+        }
+    }
+
     public void testParseSort() throws IOException {
         {
             String restContent = " { \"sort\": \"foo\"}";
-            try (XContentParser parser = XContentFactory.xContent(restContent).createParser(restContent)) {
-                SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser),
-                    searchRequestParsers.aggParsers, searchRequestParsers.suggesters);
+            try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) {
+                SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser));
                 assertEquals(1, searchSourceBuilder.sorts().size());
                 assertEquals(new FieldSortBuilder("foo"), searchSourceBuilder.sorts().get(0));
             }
@@ -533,9 +159,8 @@ public void testParseSort() throws IOException {
                     "        { \"age\" : \"desc\" },\n" +
                     "        \"_score\"\n" +
                     "    ]}";
-            try (XContentParser parser = XContentFactory.xContent(restContent).createParser(restContent)) {
-                SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser),
-                    searchRequestParsers.aggParsers, searchRequestParsers.suggesters);
+            try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) {
+                SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser));
                 assertEquals(5, searchSourceBuilder.sorts().size());
                 assertEquals(new FieldSortBuilder("post_date"), searchSourceBuilder.sorts().get(0));
                 assertEquals(new FieldSortBuilder("user"), searchSourceBuilder.sorts().get(1));
@@ -557,9 +182,8 @@ public void testAggsParsing() throws IOException {
                     "        }\n" +
                     "    }\n" +
                     "}\n";
-            try (XContentParser parser = XContentFactory.xContent(restContent).createParser(restContent)) {
-                SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser),
-                    searchRequestParsers.aggParsers, searchRequestParsers.suggesters);
+            try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) {
+                SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser));
                 assertEquals(1, searchSourceBuilder.aggregations().count());
             }
         }
@@ -573,9 +197,8 @@ public void testAggsParsing() throws IOException {
                     "        }\n" +
                     "    }\n" +
                     "}\n";
-            try (XContentParser parser = XContentFactory.xContent(restContent).createParser(restContent)) {
-                SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser),
-                    searchRequestParsers.aggParsers, searchRequestParsers.suggesters);
+            try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) {
+                SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser));
                 assertEquals(1, searchSourceBuilder.aggregations().count());
             }
         }
@@ -599,9 +222,8 @@ public void testParseRescore() throws IOException {
                 "        }\n" +
                 "    }\n" +
                 "}\n";
-            try (XContentParser parser = XContentFactory.xContent(restContent).createParser(restContent)) {
-                SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser),
-                    searchRequestParsers.aggParsers, searchRequestParsers.suggesters);
+            try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) {
+                SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser));
                 assertEquals(1, searchSourceBuilder.rescores().size());
                 assertEquals(new QueryRescorerBuilder(QueryBuilders.matchQuery("content", "baz")).windowSize(50),
                         searchSourceBuilder.rescores().get(0));
@@ -622,9 +244,8 @@ public void testParseRescore() throws IOException {
                 "        }\n" +
                 "    } ]\n" +
                 "}\n";
-            try (XContentParser parser = XContentFactory.xContent(restContent).createParser(restContent)) {
-                SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser),
-                    searchRequestParsers.aggParsers, searchRequestParsers.suggesters);
+            try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) {
+                SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser));
                 assertEquals(1, searchSourceBuilder.rescores().size());
                 assertEquals(new QueryRescorerBuilder(QueryBuilders.matchQuery("content", "baz")).windowSize(50),
                         searchSourceBuilder.rescores().get(0));
@@ -635,9 +256,8 @@ public void testParseRescore() throws IOException {
     public void testTimeoutWithUnits() throws IOException {
         final String timeout = randomTimeValue();
         final String query = "{ \"query\": { \"match_all\": {}}, \"timeout\": \"" + timeout + "\"}";
-        try (XContentParser parser = XContentFactory.xContent(query).createParser(query)) {
-            final SearchSourceBuilder builder = SearchSourceBuilder.fromXContent(createParseContext(parser),
-                searchRequestParsers.aggParsers, searchRequestParsers.suggesters);
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, query)) {
+            final SearchSourceBuilder builder = SearchSourceBuilder.fromXContent(createParseContext(parser));
             assertThat(builder.timeout(), equalTo(TimeValue.parseTimeValue(timeout, null, "timeout")));
         }
     }
@@ -645,46 +265,124 @@ public void testTimeoutWithUnits() throws IOException {
     public void testTimeoutWithoutUnits() throws IOException {
         final int timeout = randomIntBetween(1, 1024);
         final String query = "{ \"query\": { \"match_all\": {}}, \"timeout\": \"" + timeout + "\"}";
-        try (XContentParser parser = XContentFactory.xContent(query).createParser(query)) {
-            final ElasticsearchParseException e =
-                    expectThrows(
-                            ElasticsearchParseException.class,
-                            () -> SearchSourceBuilder.fromXContent(createParseContext(parser),
-                                searchRequestParsers.aggParsers, searchRequestParsers.suggesters));
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, query)) {
+            final ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> SearchSourceBuilder.fromXContent(
+                    createParseContext(parser)));
             assertThat(e, hasToString(containsString("unit is missing or unrecognized")));
         }
     }
 
+    public void testToXContent() throws  IOException {
+        //verify that only what is set gets printed out through toXContent
+        XContentType xContentType = randomFrom(XContentType.values());
+        {
+            SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+            XContentBuilder builder = XContentFactory.contentBuilder(xContentType);
+            searchSourceBuilder.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            BytesReference source = builder.bytes();
+            Map<String, Object> sourceAsMap = XContentHelper.convertToMap(source, false, xContentType).v2();
+            assertEquals(0, sourceAsMap.size());
+        }
+        {
+            SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+            searchSourceBuilder.query(RandomQueryBuilder.createQuery(random()));
+            XContentBuilder builder = XContentFactory.contentBuilder(xContentType);
+            searchSourceBuilder.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            BytesReference source = builder.bytes();
+            Map<String, Object> sourceAsMap = XContentHelper.convertToMap(source, false, xContentType).v2();
+            assertEquals(1, sourceAsMap.size());
+            assertEquals("query", sourceAsMap.keySet().iterator().next());
+        }
+    }
+
     public void testEmptyPostFilter() throws IOException {
         SearchSourceBuilder builder = new SearchSourceBuilder();
         String query = "{ \"post_filter\": {} }";
-        assertParseSearchSource(builder, new BytesArray(query), ParseFieldMatcher.EMPTY);
+        assertParseSearchSource(builder, createParser(JsonXContent.jsonXContent, query));
+        assertWarnings("query malformed, empty clause found at [1:19]");
     }
 
     public void testEmptyQuery() throws IOException {
         SearchSourceBuilder builder = new SearchSourceBuilder();
         String query = "{ \"query\": {} }";
-        assertParseSearchSource(builder, new BytesArray(query), ParseFieldMatcher.EMPTY);
+        assertParseSearchSource(builder, createParser(JsonXContent.jsonXContent, query));
+        assertWarnings("query malformed, empty clause found at [1:13]");
     }
 
-    public void testSearchRequestBuilderSerializationWithIndexBoost() throws Exception {
-        SearchSourceBuilder searchSourceBuilder = createSearchSourceBuilder();
-        createIndexBoost(searchSourceBuilder);
-        try (BytesStreamOutput output = new BytesStreamOutput()) {
-            searchSourceBuilder.writeTo(output);
-            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
-                SearchSourceBuilder deserializedSearchSourceBuilder = new SearchSourceBuilder(in);
-                BytesStreamOutput deserializedOutput = new BytesStreamOutput();
-                deserializedSearchSourceBuilder.writeTo(deserializedOutput);
-                assertEquals(output.bytes(), deserializedOutput.bytes());
+    public void testParseIndicesBoost() throws IOException {
+        {
+            String restContent = " { \"indices_boost\": {\"foo\": 1.0, \"bar\": 2.0}}";
+            try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) {
+                SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser));
+                assertEquals(2, searchSourceBuilder.indexBoosts().size());
+                assertEquals(new SearchSourceBuilder.IndexBoost("foo", 1.0f), searchSourceBuilder.indexBoosts().get(0));
+                assertEquals(new SearchSourceBuilder.IndexBoost("bar", 2.0f), searchSourceBuilder.indexBoosts().get(1));
+                assertWarnings("Object format in indices_boost is deprecated, please use array format instead");
             }
         }
+
+        {
+            String restContent = "{" +
+                "    \"indices_boost\" : [\n" +
+                "        { \"foo\" : 1.0 },\n" +
+                "        { \"bar\" : 2.0 },\n" +
+                "        { \"baz\" : 3.0 }\n" +
+                "    ]}";
+            try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) {
+                SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser));
+                assertEquals(3, searchSourceBuilder.indexBoosts().size());
+                assertEquals(new SearchSourceBuilder.IndexBoost("foo", 1.0f), searchSourceBuilder.indexBoosts().get(0));
+                assertEquals(new SearchSourceBuilder.IndexBoost("bar", 2.0f), searchSourceBuilder.indexBoosts().get(1));
+                assertEquals(new SearchSourceBuilder.IndexBoost("baz", 3.0f), searchSourceBuilder.indexBoosts().get(2));
+            }
+        }
+
+        {
+            String restContent = "{" +
+                "    \"indices_boost\" : [\n" +
+                "        { \"foo\" : 1.0, \"bar\": 2.0}\n" + // invalid format
+                "    ]}";
+
+            assertIndicesBoostParseErrorMessage(restContent, "Expected [END_OBJECT] in [indices_boost] but found [FIELD_NAME]");
+        }
+
+        {
+            String restContent = "{" +
+                "    \"indices_boost\" : [\n" +
+                "        {}\n" + // invalid format
+                "    ]}";
+
+            assertIndicesBoostParseErrorMessage(restContent, "Expected [FIELD_NAME] in [indices_boost] but found [END_OBJECT]");
+        }
+
+        {
+            String restContent = "{" +
+                "    \"indices_boost\" : [\n" +
+                "        { \"foo\" : \"bar\"}\n" + // invalid format
+                "    ]}";
+
+            assertIndicesBoostParseErrorMessage(restContent, "Expected [VALUE_NUMBER] in [indices_boost] but found [VALUE_STRING]");
+        }
+
+        {
+            String restContent = "{" +
+                "    \"indices_boost\" : [\n" +
+                "        { \"foo\" : {\"bar\": 1}}\n" + // invalid format
+                "    ]}";
+
+            assertIndicesBoostParseErrorMessage(restContent, "Expected [VALUE_NUMBER] in [indices_boost] but found [START_OBJECT]");
+        }
+    }
+
+    public void testNegativeFromErrors() {
+        IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> new SearchSourceBuilder().from(-2));
+        assertEquals("[from] parameter cannot be negative", expected.getMessage());
     }
 
-    private void createIndexBoost(SearchSourceBuilder searchSourceBuilder) {
-        int indexBoostSize = randomIntBetween(1, 10);
-        for (int i = 0; i < indexBoostSize; i++) {
-            searchSourceBuilder.indexBoost(randomAsciiOfLengthBetween(5, 20), randomFloat() * 10);
+    private void assertIndicesBoostParseErrorMessage(String restContent, String expectedErrorMessage) throws IOException {
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) {
+            ParsingException e = expectThrows(ParsingException.class, () -> SearchSourceBuilder.fromXContent(createParseContext(parser)));
+            assertEquals(expectedErrorMessage, e.getMessage());
         }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java b/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java
deleted file mode 100644
index 3aa98942833b3..0000000000000
--- a/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java
+++ /dev/null
@@ -1,1937 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.search.child;
-
-import org.apache.lucene.search.join.ScoreMode;
-import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
-import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
-import org.elasticsearch.action.explain.ExplainResponse;
-import org.elasticsearch.action.index.IndexRequestBuilder;
-import org.elasticsearch.action.search.SearchPhaseExecutionException;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.search.SearchType;
-import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
-import org.elasticsearch.common.lucene.search.function.CombineFunction;
-import org.elasticsearch.common.lucene.search.function.FiltersFunctionScoreQuery;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.index.IndexModule;
-import org.elasticsearch.index.query.BoolQueryBuilder;
-import org.elasticsearch.index.query.HasChildQueryBuilder;
-import org.elasticsearch.index.query.HasParentQueryBuilder;
-import org.elasticsearch.index.query.IdsQueryBuilder;
-import org.elasticsearch.index.query.MatchAllQueryBuilder;
-import org.elasticsearch.index.query.MatchQueryBuilder;
-import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
-import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.search.aggregations.AggregationBuilders;
-import org.elasticsearch.search.aggregations.bucket.filter.Filter;
-import org.elasticsearch.search.aggregations.bucket.global.Global;
-import org.elasticsearch.search.aggregations.bucket.terms.Terms;
-import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
-import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
-import org.elasticsearch.search.sort.SortBuilders;
-import org.elasticsearch.search.sort.SortOrder;
-import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
-import org.elasticsearch.test.ESIntegTestCase.Scope;
-import org.hamcrest.Matchers;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
-
-import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
-import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery;
-import static org.elasticsearch.index.query.QueryBuilders.hasParentQuery;
-import static org.elasticsearch.index.query.QueryBuilders.hasChildQuery;
-import static org.elasticsearch.index.query.QueryBuilders.idsQuery;
-import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
-import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
-import static org.elasticsearch.index.query.QueryBuilders.multiMatchQuery;
-import static org.elasticsearch.index.query.QueryBuilders.parentId;
-import static org.elasticsearch.index.query.QueryBuilders.prefixQuery;
-import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery;
-import static org.elasticsearch.index.query.QueryBuilders.termQuery;
-import static org.elasticsearch.index.query.QueryBuilders.termsQuery;
-import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.fieldValueFactorFunction;
-import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.weightFactorFunction;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHit;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasId;
-import static org.hamcrest.Matchers.anyOf;
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.greaterThanOrEqualTo;
-import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.notNullValue;
-
-@ClusterScope(scope = Scope.SUITE)
-public class ChildQuerySearchIT extends ESIntegTestCase {
-
-    @Override
-    public Settings indexSettings() {
-        return Settings.builder().put(super.indexSettings())
-            // aggressive filter caching so that we can assert on the filter cache size
-            .put(IndexModule.INDEX_QUERY_CACHE_ENABLED_SETTING.getKey(), true)
-            .put(IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING.getKey(), true)
-            .build();
-    }
-
-    public void testSelfReferentialIsForbidden() {
-        try {
-            prepareCreate("test").addMapping("type", "_parent", "type=type").get();
-            fail("self referential should be forbidden");
-        } catch (Exception e) {
-            Throwable cause = e.getCause();
-            assertThat(cause, instanceOf(IllegalArgumentException.class));
-            assertThat(cause.getMessage(), equalTo("The [_parent.type] option can't point to the same type"));
-        }
-    }
-
-    public void testMultiLevelChild() throws Exception {
-        assertAcked(prepareCreate("test")
-                .addMapping("parent")
-                .addMapping("child", "_parent", "type=parent")
-                .addMapping("grandchild", "_parent", "type=child"));
-        ensureGreen();
-
-        client().prepareIndex("test", "parent", "p1").setSource("p_field", "p_value1").get();
-        client().prepareIndex("test", "child", "c1").setSource("c_field", "c_value1").setParent("p1").get();
-        client().prepareIndex("test", "grandchild", "gc1").setSource("gc_field", "gc_value1")
-                .setParent("c1").setRouting("p1").get();
-        refresh();
-
-        SearchResponse searchResponse = client()
-                .prepareSearch("test")
-                .setQuery(
-                        boolQuery()
-                                .must(matchAllQuery())
-                                .filter(hasChildQuery(
-                                        "child",
-                                        boolQuery().must(termQuery("c_field", "c_value1"))
-                                                .filter(hasChildQuery("grandchild", termQuery("gc_field", "gc_value1"), ScoreMode.None))
-                                        , ScoreMode.None))).get();
-        assertNoFailures(searchResponse);
-        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
-        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("p1"));
-
-        searchResponse = client().prepareSearch("test")
-                .setQuery(boolQuery().must(matchAllQuery()).filter(hasParentQuery("parent", termQuery("p_field", "p_value1"), false))).execute()
-                .actionGet();
-        assertNoFailures(searchResponse);
-        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
-        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("c1"));
-
-        searchResponse = client().prepareSearch("test")
-                .setQuery(boolQuery().must(matchAllQuery()).filter(hasParentQuery("child", termQuery("c_field", "c_value1"), false))).execute()
-                .actionGet();
-        assertNoFailures(searchResponse);
-        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
-        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("gc1"));
-
-        searchResponse = client().prepareSearch("test").setQuery(hasParentQuery("parent", termQuery("p_field", "p_value1"), false)).execute()
-                .actionGet();
-        assertNoFailures(searchResponse);
-        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
-        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("c1"));
-
-        searchResponse = client().prepareSearch("test").setQuery(hasParentQuery("child", termQuery("c_field", "c_value1"), false)).execute()
-                .actionGet();
-        assertNoFailures(searchResponse);
-        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
-        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("gc1"));
-    }
-
-    // see #2744
-    public void test2744() throws IOException {
-        assertAcked(prepareCreate("test")
-                .addMapping("foo")
-                .addMapping("test", "_parent", "type=foo"));
-        ensureGreen();
-
-        // index simple data
-        client().prepareIndex("test", "foo", "1").setSource("foo", 1).get();
-        client().prepareIndex("test", "test").setSource("foo", 1).setParent("1").get();
-        refresh();
-        SearchResponse searchResponse = client().prepareSearch("test").
-                setQuery(hasChildQuery("test", matchQuery("foo", 1), ScoreMode.None))
-                .get();
-        assertNoFailures(searchResponse);
-        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
-        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("1"));
-
-    }
-
-    public void testSimpleChildQuery() throws Exception {
-        assertAcked(prepareCreate("test")
-                .addMapping("parent")
-                .addMapping("child", "_parent", "type=parent"));
-        ensureGreen();
-
-        // index simple data
-        client().prepareIndex("test", "parent", "p1").setSource("p_field", "p_value1").get();
-        client().prepareIndex("test", "child", "c1").setSource("c_field", "red").setParent("p1").get();
-        client().prepareIndex("test", "child", "c2").setSource("c_field", "yellow").setParent("p1").get();
-        client().prepareIndex("test", "parent", "p2").setSource("p_field", "p_value2").get();
-        client().prepareIndex("test", "child", "c3").setSource("c_field", "blue").setParent("p2").get();
-        client().prepareIndex("test", "child", "c4").setSource("c_field", "red").setParent("p2").get();
-        refresh();
-
-        // TEST FETCHING _parent from child
-        SearchResponse searchResponse = client().prepareSearch("test").setQuery(idsQuery("child").addIds("c1")).storedFields("_parent").execute()
-                .actionGet();
-        assertNoFailures(searchResponse);
-        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
-        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("c1"));
-        assertThat(searchResponse.getHits().getAt(0).field("_parent").value().toString(), equalTo("p1"));
-
-        // TEST matching on parent
-        searchResponse = client().prepareSearch("test").setQuery(termQuery("_parent#parent", "p1")).storedFields("_parent").get();
-        assertNoFailures(searchResponse);
-        assertThat(searchResponse.getHits().totalHits(), equalTo(2L));
-        assertThat(searchResponse.getHits().getAt(0).id(), anyOf(equalTo("c1"), equalTo("c2")));
-        assertThat(searchResponse.getHits().getAt(0).field("_parent").value().toString(), equalTo("p1"));
-        assertThat(searchResponse.getHits().getAt(1).id(), anyOf(equalTo("c1"), equalTo("c2")));
-        assertThat(searchResponse.getHits().getAt(1).field("_parent").value().toString(), equalTo("p1"));
-
-        searchResponse = client().prepareSearch("test").setQuery(queryStringQuery("_parent#parent:p1")).storedFields("_parent").get();
-        assertNoFailures(searchResponse);
-        assertThat(searchResponse.getHits().totalHits(), equalTo(2L));
-        assertThat(searchResponse.getHits().getAt(0).id(), anyOf(equalTo("c1"), equalTo("c2")));
-        assertThat(searchResponse.getHits().getAt(0).field("_parent").value().toString(), equalTo("p1"));
-        assertThat(searchResponse.getHits().getAt(1).id(), anyOf(equalTo("c1"), equalTo("c2")));
-        assertThat(searchResponse.getHits().getAt(1).field("_parent").value().toString(), equalTo("p1"));
-
-        // HAS CHILD
-        searchResponse = client().prepareSearch("test").setQuery(randomHasChild("child", "c_field", "yellow"))
-                .get();
-        assertHitCount(searchResponse, 1L);
-        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
-        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("p1"));
-
-        searchResponse = client().prepareSearch("test").setQuery(randomHasChild("child", "c_field", "blue")).execute()
-                .actionGet();
-        assertHitCount(searchResponse, 1L);
-        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("p2"));
-
-        searchResponse = client().prepareSearch("test").setQuery(randomHasChild("child", "c_field", "red")).get();
-        assertHitCount(searchResponse, 2L);
-        assertThat(searchResponse.getHits().getAt(0).id(), anyOf(equalTo("p2"), equalTo("p1")));
-        assertThat(searchResponse.getHits().getAt(1).id(), anyOf(equalTo("p2"), equalTo("p1")));
-
-        // HAS PARENT
-        searchResponse = client().prepareSearch("test")
-                .setQuery(randomHasParent("parent", "p_field", "p_value2")).get();
-        assertNoFailures(searchResponse);
-        assertHitCount(searchResponse, 2L);
-        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("c3"));
-        assertThat(searchResponse.getHits().getAt(1).id(), equalTo("c4"));
-
-        searchResponse = client().prepareSearch("test")
-                .setQuery(randomHasParent("parent", "p_field", "p_value1")).get();
-        assertHitCount(searchResponse, 2L);
-        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("c1"));
-        assertThat(searchResponse.getHits().getAt(1).id(), equalTo("c2"));
-    }
-
-    // Issue #3290
-    public void testCachingBugWithFqueryFilter() throws Exception {
-        assertAcked(prepareCreate("test")
-                .addMapping("parent")
-                .addMapping("child", "_parent", "type=parent"));
-        ensureGreen();
-        List<IndexRequestBuilder> builders = new ArrayList<>();
-        // index simple data
-        for (int i = 0; i < 10; i++) {
-            builders.add(client().prepareIndex("test", "parent", Integer.toString(i)).setSource("p_field", i));
-        }
-        indexRandom(randomBoolean(), builders);
-        builders.clear();
-        for (int j = 0; j < 2; j++) {
-            for (int i = 0; i < 10; i++) {
-                builders.add(client().prepareIndex("test", "child", Integer.toString(i)).setSource("c_field", i).setParent("" + 0));
-            }
-            for (int i = 0; i < 10; i++) {
-                builders.add(client().prepareIndex("test", "child", Integer.toString(i + 10)).setSource("c_field", i + 10).setParent(Integer.toString(i)));
-            }
-
-            if (randomBoolean()) {
-                break; // randomly break out and dont' have deletes / updates
-            }
-        }
-        indexRandom(true, builders);
-
-        for (int i = 1; i <= 10; i++) {
-            logger.info("Round {}", i);
-            SearchResponse searchResponse = client().prepareSearch("test")
-                    .setQuery(constantScoreQuery(hasChildQuery("child", matchAllQuery(), ScoreMode.Max)))
-                    .get();
-            assertNoFailures(searchResponse);
-            searchResponse = client().prepareSearch("test")
-                    .setQuery(constantScoreQuery(hasParentQuery("parent", matchAllQuery(), true)))
-                    .get();
-            assertNoFailures(searchResponse);
-        }
-    }
-
-    public void testHasParentFilter() throws Exception {
-        assertAcked(prepareCreate("test")
-                .addMapping("parent")
-                .addMapping("child", "_parent", "type=parent"));
-        ensureGreen();
-        Map<String, Set<String>> parentToChildren = new HashMap<>();
-        // Childless parent
-        client().prepareIndex("test", "parent", "p0").setSource("p_field", "p0").get();
-        parentToChildren.put("p0", new HashSet<>());
-
-        String previousParentId = null;
-        int numChildDocs = 32;
-        int numChildDocsPerParent = 0;
-        List<IndexRequestBuilder> builders = new ArrayList<>();
-        for (int i = 1; i <= numChildDocs; i++) {
-
-            if (previousParentId == null || i % numChildDocsPerParent == 0) {
-                previousParentId = "p" + i;
-                builders.add(client().prepareIndex("test", "parent", previousParentId).setSource("p_field", previousParentId));
-                numChildDocsPerParent++;
-            }
-
-            String childId = "c" + i;
-            builders.add(client().prepareIndex("test", "child", childId).setSource("c_field", childId).setParent(previousParentId));
-
-            if (!parentToChildren.containsKey(previousParentId)) {
-                parentToChildren.put(previousParentId, new HashSet<>());
-            }
-            assertThat(parentToChildren.get(previousParentId).add(childId), is(true));
-        }
-        indexRandom(true, builders.toArray(new IndexRequestBuilder[builders.size()]));
-
-        assertThat(parentToChildren.isEmpty(), equalTo(false));
-        for (Map.Entry<String, Set<String>> parentToChildrenEntry : parentToChildren.entrySet()) {
-            SearchResponse searchResponse = client().prepareSearch("test")
-                    .setQuery(constantScoreQuery(hasParentQuery("parent", termQuery("p_field", parentToChildrenEntry.getKey()), false)))
-                    .setSize(numChildDocsPerParent).get();
-
-            assertNoFailures(searchResponse);
-            Set<String> childIds = parentToChildrenEntry.getValue();
-            assertThat(searchResponse.getHits().totalHits(), equalTo((long) childIds.size()));
-            for (int i = 0; i < searchResponse.getHits().totalHits(); i++) {
-                assertThat(childIds.remove(searchResponse.getHits().getAt(i).id()), is(true));
-                assertThat(searchResponse.getHits().getAt(i).score(), is(1.0f));
-            }
-            assertThat(childIds.size(), is(0));
-        }
-    }
-
-    public void testSimpleChildQueryWithFlush() throws Exception {
-        assertAcked(prepareCreate("test")
-                .addMapping("parent")
-                .addMapping("child", "_parent", "type=parent"));
-        ensureGreen();
-
-        // index simple data with flushes, so we have many segments
-        client().prepareIndex("test", "parent", "p1").setSource("p_field", "p_value1").get();
-        client().admin().indices().prepareFlush().get();
-        client().prepareIndex("test", "child", "c1").setSource("c_field", "red").setParent("p1").get();
-        client().admin().indices().prepareFlush().get();
-        client().prepareIndex("test", "child", "c2").setSource("c_field", "yellow").setParent("p1").get();
-        client().admin().indices().prepareFlush().get();
-        client().prepareIndex("test", "parent", "p2").setSource("p_field", "p_value2").get();
-        client().admin().indices().prepareFlush().get();
-        client().prepareIndex("test", "child", "c3").setSource("c_field", "blue").setParent("p2").get();
-        client().admin().indices().prepareFlush().get();
-        client().prepareIndex("test", "child", "c4").setSource("c_field", "red").setParent("p2").get();
-        client().admin().indices().prepareFlush().get();
-        refresh();
-
-        // HAS CHILD QUERY
-
-        SearchResponse searchResponse = client().prepareSearch("test")
-                .setQuery(hasChildQuery("child", termQuery("c_field", "yellow"), ScoreMode.None))
-                .get();
-        assertNoFailures(searchResponse);
-        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
-        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("p1"));
-
-        searchResponse = client().prepareSearch("test")
-                .setQuery(hasChildQuery("child", termQuery("c_field", "blue"), ScoreMode.None))
-                .get();
-        assertNoFailures(searchResponse);
-        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
-        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("p2"));
-
-        searchResponse = client().prepareSearch("test")
-                .setQuery(hasChildQuery("child", termQuery("c_field", "red"), ScoreMode.None))
-                .get();
-        assertNoFailures(searchResponse);
-        assertThat(searchResponse.getHits().totalHits(), equalTo(2L));
-        assertThat(searchResponse.getHits().getAt(0).id(), anyOf(equalTo("p2"), equalTo("p1")));
-        assertThat(searchResponse.getHits().getAt(1).id(), anyOf(equalTo("p2"), equalTo("p1")));
-
-        // HAS CHILD FILTER
-        searchResponse = client().prepareSearch("test")
-                .setQuery(constantScoreQuery(hasChildQuery("child", termQuery("c_field", "yellow"), ScoreMode.None)))
-                .get();
-        assertNoFailures(searchResponse);
-        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
-        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("p1"));
-
-        searchResponse = client().prepareSearch("test")
-                .setQuery(constantScoreQuery(hasChildQuery("child", termQuery("c_field", "blue"), ScoreMode.None)))
-                .get();
-        assertNoFailures(searchResponse);
-        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
-        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("p2"));
-
-        searchResponse = client().prepareSearch("test")
-                .setQuery(constantScoreQuery(hasChildQuery("child", termQuery("c_field", "red"), ScoreMode.None)))
-                .get();
-        assertNoFailures(searchResponse);
-        assertThat(searchResponse.getHits().totalHits(), equalTo(2L));
-        assertThat(searchResponse.getHits().getAt(0).id(), anyOf(equalTo("p2"), equalTo("p1")));
-        assertThat(searchResponse.getHits().getAt(1).id(), anyOf(equalTo("p2"), equalTo("p1")));
-    }
-
-    public void testScopedFacet() throws Exception {
-        assertAcked(prepareCreate("test")
-                .addMapping("parent")
-                .addMapping("child", "_parent", "type=parent", "c_field", "type=keyword"));
-        ensureGreen();
-
-        // index simple data
-        client().prepareIndex("test", "parent", "p1").setSource("p_field", "p_value1").get();
-        client().prepareIndex("test", "child", "c1").setSource("c_field", "red").setParent("p1").get();
-        client().prepareIndex("test", "child", "c2").setSource("c_field", "yellow").setParent("p1").get();
-        client().prepareIndex("test", "parent", "p2").setSource("p_field", "p_value2").get();
-        client().prepareIndex("test", "child", "c3").setSource("c_field", "blue").setParent("p2").get();
-        client().prepareIndex("test", "child", "c4").setSource("c_field", "red").setParent("p2").get();
-
-        refresh();
-
-        SearchResponse searchResponse = client()
-                .prepareSearch("test")
-                .setQuery(hasChildQuery("child", boolQuery().should(termQuery("c_field", "red")).should(termQuery("c_field", "yellow")), ScoreMode.None))
-                .addAggregation(AggregationBuilders.global("global").subAggregation(
-                        AggregationBuilders.filter("filter", boolQuery().should(termQuery("c_field", "red")).should(termQuery("c_field", "yellow"))).subAggregation(
-                                AggregationBuilders.terms("facet1").field("c_field")))).get();
-        assertNoFailures(searchResponse);
-        assertThat(searchResponse.getHits().totalHits(), equalTo(2L));
-        assertThat(searchResponse.getHits().getAt(0).id(), anyOf(equalTo("p2"), equalTo("p1")));
-        assertThat(searchResponse.getHits().getAt(1).id(), anyOf(equalTo("p2"), equalTo("p1")));
-
-        Global global = searchResponse.getAggregations().get("global");
-        Filter filter = global.getAggregations().get("filter");
-        Terms termsFacet = filter.getAggregations().get("facet1");
-        assertThat(termsFacet.getBuckets().size(), equalTo(2));
-        assertThat(termsFacet.getBuckets().get(0).getKeyAsString(), equalTo("red"));
-        assertThat(termsFacet.getBuckets().get(0).getDocCount(), equalTo(2L));
-        assertThat(termsFacet.getBuckets().get(1).getKeyAsString(), equalTo("yellow"));
-        assertThat(termsFacet.getBuckets().get(1).getDocCount(), equalTo(1L));
-    }
-
-    public void testDeletedParent() throws Exception {
-        assertAcked(prepareCreate("test")
-                .addMapping("parent")
-                .addMapping("child", "_parent", "type=parent"));
-        ensureGreen();
-        // index simple data
-        client().prepareIndex("test", "parent", "p1").setSource("p_field", "p_value1").get();
-        client().prepareIndex("test", "child", "c1").setSource("c_field", "red").setParent("p1").get();
-        client().prepareIndex("test", "child", "c2").setSource("c_field", "yellow").setParent("p1").get();
-        client().prepareIndex("test", "parent", "p2").setSource("p_field", "p_value2").get();
-        client().prepareIndex("test", "child", "c3").setSource("c_field", "blue").setParent("p2").get();
-        client().prepareIndex("test", "child", "c4").setSource("c_field", "red").setParent("p2").get();
-
-        refresh();
-
-        SearchResponse searchResponse = client().prepareSearch("test")
-                .setQuery(constantScoreQuery(hasChildQuery("child", termQuery("c_field", "yellow"), ScoreMode.None))).get();
-        assertNoFailures(searchResponse);
-        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
-        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("p1"));
-        assertThat(searchResponse.getHits().getAt(0).sourceAsString(), containsString("\"p_value1\""));
-
-        // update p1 and see what that we get updated values...
-
-        client().prepareIndex("test", "parent", "p1").setSource("p_field", "p_value1_updated").get();
-        client().admin().indices().prepareRefresh().get();
-
-        searchResponse = client().prepareSearch("test")
-                .setQuery(constantScoreQuery(hasChildQuery("child", termQuery("c_field", "yellow"), ScoreMode.None))).get();
-        assertNoFailures(searchResponse);
-        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
-        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("p1"));
-        assertThat(searchResponse.getHits().getAt(0).sourceAsString(), containsString("\"p_value1_updated\""));
-    }
-
-    public void testDfsSearchType() throws Exception {
-        assertAcked(prepareCreate("test")
-                .addMapping("parent")
-                .addMapping("child", "_parent", "type=parent"));
-        ensureGreen();
-
-        // index simple data
-        client().prepareIndex("test", "parent", "p1").setSource("p_field", "p_value1").get();
-        client().prepareIndex("test", "child", "c1").setSource("c_field", "red").setParent("p1").get();
-        client().prepareIndex("test", "child", "c2").setSource("c_field", "yellow").setParent("p1").get();
-        client().prepareIndex("test", "parent", "p2").setSource("p_field", "p_value2").get();
-        client().prepareIndex("test", "child", "c3").setSource("c_field", "blue").setParent("p2").get();
-        client().prepareIndex("test", "child", "c4").setSource("c_field", "red").setParent("p2").get();
-
-        refresh();
-
-        SearchResponse searchResponse = client().prepareSearch("test").setSearchType(SearchType.DFS_QUERY_THEN_FETCH)
-                .setQuery(boolQuery().mustNot(hasChildQuery("child", boolQuery().should(queryStringQuery("c_field:*")), ScoreMode.None)))
-                .get();
-        assertNoFailures(searchResponse);
-
-        searchResponse = client().prepareSearch("test").setSearchType(SearchType.DFS_QUERY_THEN_FETCH)
-                .setQuery(boolQuery().mustNot(hasParentQuery("parent", boolQuery().should(queryStringQuery("p_field:*")), false))).execute()
-                .actionGet();
-        assertNoFailures(searchResponse);
-    }
-
-    public void testHasChildAndHasParentFailWhenSomeSegmentsDontContainAnyParentOrChildDocs() throws Exception {
-        assertAcked(prepareCreate("test")
-                .addMapping("parent")
-                .addMapping("child", "_parent", "type=parent"));
-        ensureGreen();
-
-        client().prepareIndex("test", "parent", "1").setSource("p_field", 1).get();
-        client().prepareIndex("test", "child", "1").setParent("1").setSource("c_field", 1).get();
-        client().admin().indices().prepareFlush("test").get();
-
-        client().prepareIndex("test", "type1", "1").setSource("p_field", 1).get();
-        client().admin().indices().prepareFlush("test").get();
-
-        SearchResponse searchResponse = client().prepareSearch("test")
-                .setQuery(boolQuery().must(matchAllQuery()).filter(hasChildQuery("child", matchAllQuery(), ScoreMode.None))).get();
-        assertNoFailures(searchResponse);
-        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
-
-        searchResponse = client().prepareSearch("test")
-                .setQuery(boolQuery().must(matchAllQuery()).filter(hasParentQuery("parent", matchAllQuery(), false))).get();
-        assertNoFailures(searchResponse);
-        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
-    }
-
-    public void testCountApiUsage() throws Exception {
-        assertAcked(prepareCreate("test")
-                .addMapping("parent")
-                .addMapping("child", "_parent", "type=parent"));
-        ensureGreen();
-
-        String parentId = "p1";
-        client().prepareIndex("test", "parent", parentId).setSource("p_field", "1").get();
-        client().prepareIndex("test", "child", "c1").setSource("c_field", "1").setParent(parentId).get();
-        refresh();
-
-        SearchResponse countResponse = client().prepareSearch("test").setSize(0)
-                .setQuery(hasChildQuery("child", termQuery("c_field", "1"), ScoreMode.Max))
-                .get();
-        assertHitCount(countResponse, 1L);
-
-        countResponse = client().prepareSearch("test").setSize(0).setQuery(hasParentQuery("parent", termQuery("p_field", "1"), true))
-                .get();
-        assertHitCount(countResponse, 1L);
-
-        countResponse = client().prepareSearch("test").setSize(0)
-                .setQuery(constantScoreQuery(hasChildQuery("child", termQuery("c_field", "1"), ScoreMode.None)))
-                .get();
-        assertHitCount(countResponse, 1L);
-
-        countResponse = client().prepareSearch("test").setSize(0).setQuery(constantScoreQuery(hasParentQuery("parent", termQuery("p_field", "1"), false)))
-                .get();
-        assertHitCount(countResponse, 1L);
-    }
-
-    public void testExplainUsage() throws Exception {
-        assertAcked(prepareCreate("test")
-                .addMapping("parent")
-                .addMapping("child", "_parent", "type=parent"));
-        ensureGreen();
-
-        String parentId = "p1";
-        client().prepareIndex("test", "parent", parentId).setSource("p_field", "1").get();
-        client().prepareIndex("test", "child", "c1").setSource("c_field", "1").setParent(parentId).get();
-        refresh();
-
-        SearchResponse searchResponse = client().prepareSearch("test")
-                .setExplain(true)
-                .setQuery(hasChildQuery("child", termQuery("c_field", "1"), ScoreMode.Max))
-                .get();
-        assertHitCount(searchResponse, 1L);
-        assertThat(searchResponse.getHits().getAt(0).explanation().getDescription(), containsString("join value p1"));
-
-        searchResponse = client().prepareSearch("test")
-                .setExplain(true)
-                .setQuery(hasParentQuery("parent", termQuery("p_field", "1"), true))
-                .get();
-        assertHitCount(searchResponse, 1L);
-        assertThat(searchResponse.getHits().getAt(0).explanation().getDescription(), containsString("join value p1"));
-
-        ExplainResponse explainResponse = client().prepareExplain("test", "parent", parentId)
-                .setQuery(hasChildQuery("child", termQuery("c_field", "1"), ScoreMode.Max))
-                .get();
-        assertThat(explainResponse.isExists(), equalTo(true));
-        assertThat(explainResponse.getExplanation().getDetails()[0].getDescription(), containsString("join value p1"));
-    }
-
-    List<IndexRequestBuilder> createDocBuilders() {
-        List<IndexRequestBuilder> indexBuilders = new ArrayList<>();
-        // Parent 1 and its children
-        indexBuilders.add(client().prepareIndex().setType("parent").setId("1").setIndex("test").setSource("p_field", "p_value1"));
-        indexBuilders.add(client().prepareIndex().setType("child").setId("1").setIndex("test")
-                .setSource("c_field1", 1, "c_field2", 0).setParent("1"));
-        indexBuilders.add(client().prepareIndex().setType("child").setId("2").setIndex("test")
-                .setSource("c_field1", 1, "c_field2", 0).setParent("1"));
-        indexBuilders.add(client().prepareIndex().setType("child").setId("3").setIndex("test")
-                .setSource("c_field1", 2, "c_field2", 0).setParent("1"));
-        indexBuilders.add(client().prepareIndex().setType("child").setId("4").setIndex("test")
-                .setSource("c_field1", 2, "c_field2", 0).setParent("1"));
-        indexBuilders.add(client().prepareIndex().setType("child").setId("5").setIndex("test")
-                .setSource("c_field1", 1, "c_field2", 1).setParent("1"));
-        indexBuilders.add(client().prepareIndex().setType("child").setId("6").setIndex("test")
-                .setSource("c_field1", 1, "c_field2", 2).setParent("1"));
-
-        // Parent 2 and its children
-        indexBuilders.add(client().prepareIndex().setType("parent").setId("2").setIndex("test").setSource("p_field", "p_value2"));
-        indexBuilders.add(client().prepareIndex().setType("child").setId("7").setIndex("test")
-                .setSource("c_field1", 3, "c_field2", 0).setParent("2"));
-        indexBuilders.add(client().prepareIndex().setType("child").setId("8").setIndex("test")
-                .setSource("c_field1", 1, "c_field2", 1).setParent("2"));
-        indexBuilders.add(client().prepareIndex().setType("child").setId("9").setIndex("test")
-                .setSource("c_field1", 1, "c_field2", 1).setParent("p")); // why
-        // "p"????
-        indexBuilders.add(client().prepareIndex().setType("child").setId("10").setIndex("test")
-                .setSource("c_field1", 1, "c_field2", 1).setParent("2"));
-        indexBuilders.add(client().prepareIndex().setType("child").setId("11").setIndex("test")
-                .setSource("c_field1", 1, "c_field2", 1).setParent("2"));
-        indexBuilders.add(client().prepareIndex().setType("child").setId("12").setIndex("test")
-                .setSource("c_field1", 1, "c_field2", 2).setParent("2"));
-
-        // Parent 3 and its children
-
-        indexBuilders.add(client().prepareIndex().setType("parent").setId("3").setIndex("test")
-                .setSource("p_field1", "p_value3", "p_field2", 5));
-        indexBuilders.add(client().prepareIndex().setType("child").setId("13").setIndex("test")
-                .setSource("c_field1", 4, "c_field2", 0, "c_field3", 0).setParent("3"));
-        indexBuilders.add(client().prepareIndex().setType("child").setId("14").setIndex("test")
-                .setSource("c_field1", 1, "c_field2", 1, "c_field3", 1).setParent("3"));
-        indexBuilders.add(client().prepareIndex().setType("child").setId("15").setIndex("test")
-                .setSource("c_field1", 1, "c_field2", 2, "c_field3", 2).setParent("3")); // why
-        // "p"????
-        indexBuilders.add(client().prepareIndex().setType("child").setId("16").setIndex("test")
-                .setSource("c_field1", 1, "c_field2", 2, "c_field3", 3).setParent("3"));
-        indexBuilders.add(client().prepareIndex().setType("child").setId("17").setIndex("test")
-                .setSource("c_field1", 1, "c_field2", 2, "c_field3", 4).setParent("3"));
-        indexBuilders.add(client().prepareIndex().setType("child").setId("18").setIndex("test")
-                .setSource("c_field1", 1, "c_field2", 2, "c_field3", 5).setParent("3"));
-        indexBuilders.add(client().prepareIndex().setType("child1").setId("1").setIndex("test")
-                .setSource("c_field1", 1, "c_field2", 2, "c_field3", 6).setParent("3"));
-
-        return indexBuilders;
-    }
-
-    public void testScoreForParentChildQueriesWithFunctionScore() throws Exception {
-        assertAcked(prepareCreate("test")
-                .addMapping("parent")
-                .addMapping("child", "_parent", "type=parent")
-                .addMapping("child1", "_parent", "type=parent"));
-        ensureGreen();
-
-        indexRandom(true, createDocBuilders().toArray(new IndexRequestBuilder[0]));
-        SearchResponse response = client()
-                .prepareSearch("test")
-                .setQuery(
-                        QueryBuilders.hasChildQuery(
-                                "child",
-                                QueryBuilders.functionScoreQuery(matchQuery("c_field2", 0),
-                                        fieldValueFactorFunction("c_field1"))
-                                        .boostMode(CombineFunction.REPLACE), ScoreMode.Total)).get();
-
-        assertThat(response.getHits().totalHits(), equalTo(3L));
-        assertThat(response.getHits().hits()[0].id(), equalTo("1"));
-        assertThat(response.getHits().hits()[0].score(), equalTo(6f));
-        assertThat(response.getHits().hits()[1].id(), equalTo("3"));
-        assertThat(response.getHits().hits()[1].score(), equalTo(4f));
-        assertThat(response.getHits().hits()[2].id(), equalTo("2"));
-        assertThat(response.getHits().hits()[2].score(), equalTo(3f));
-
-        response = client()
-                .prepareSearch("test")
-                .setQuery(
-                        QueryBuilders.hasChildQuery(
-                                "child",
-                                QueryBuilders.functionScoreQuery(matchQuery("c_field2", 0),
-                                        fieldValueFactorFunction("c_field1"))
-                                        .boostMode(CombineFunction.REPLACE), ScoreMode.Max)).get();
-
-        assertThat(response.getHits().totalHits(), equalTo(3L));
-        assertThat(response.getHits().hits()[0].id(), equalTo("3"));
-        assertThat(response.getHits().hits()[0].score(), equalTo(4f));
-        assertThat(response.getHits().hits()[1].id(), equalTo("2"));
-        assertThat(response.getHits().hits()[1].score(), equalTo(3f));
-        assertThat(response.getHits().hits()[2].id(), equalTo("1"));
-        assertThat(response.getHits().hits()[2].score(), equalTo(2f));
-
-        response = client()
-                .prepareSearch("test")
-                .setQuery(
-                        QueryBuilders.hasChildQuery(
-                                "child",
-                                QueryBuilders.functionScoreQuery(matchQuery("c_field2", 0),
-                                        fieldValueFactorFunction("c_field1"))
-                                        .boostMode(CombineFunction.REPLACE), ScoreMode.Avg)).get();
-
-        assertThat(response.getHits().totalHits(), equalTo(3L));
-        assertThat(response.getHits().hits()[0].id(), equalTo("3"));
-        assertThat(response.getHits().hits()[0].score(), equalTo(4f));
-        assertThat(response.getHits().hits()[1].id(), equalTo("2"));
-        assertThat(response.getHits().hits()[1].score(), equalTo(3f));
-        assertThat(response.getHits().hits()[2].id(), equalTo("1"));
-        assertThat(response.getHits().hits()[2].score(), equalTo(1.5f));
-
-        response = client()
-                .prepareSearch("test")
-                .setQuery(
-                        QueryBuilders.hasParentQuery(
-                                "parent",
-                                QueryBuilders.functionScoreQuery(matchQuery("p_field1", "p_value3"),
-                                        fieldValueFactorFunction("p_field2"))
-                                        .boostMode(CombineFunction.REPLACE), true))
-                .addSort(SortBuilders.fieldSort("c_field3")).addSort(SortBuilders.scoreSort()).get();
-
-        assertThat(response.getHits().totalHits(), equalTo(7L));
-        assertThat(response.getHits().hits()[0].id(), equalTo("13"));
-        assertThat(response.getHits().hits()[0].score(), equalTo(5f));
-        assertThat(response.getHits().hits()[1].id(), equalTo("14"));
-        assertThat(response.getHits().hits()[1].score(), equalTo(5f));
-        assertThat(response.getHits().hits()[2].id(), equalTo("15"));
-        assertThat(response.getHits().hits()[2].score(), equalTo(5f));
-        assertThat(response.getHits().hits()[3].id(), equalTo("16"));
-        assertThat(response.getHits().hits()[3].score(), equalTo(5f));
-        assertThat(response.getHits().hits()[4].id(), equalTo("17"));
-        assertThat(response.getHits().hits()[4].score(), equalTo(5f));
-        assertThat(response.getHits().hits()[5].id(), equalTo("18"));
-        assertThat(response.getHits().hits()[5].score(), equalTo(5f));
-        assertThat(response.getHits().hits()[6].id(), equalTo("1"));
-        assertThat(response.getHits().hits()[6].score(), equalTo(5f));
-    }
-
-    // Issue #2536
-    public void testParentChildQueriesCanHandleNoRelevantTypesInIndex() throws Exception {
-        assertAcked(prepareCreate("test")
-                .addMapping("parent")
-                .addMapping("child", "_parent", "type=parent"));
-        ensureGreen();
-
-        SearchResponse response = client().prepareSearch("test")
-                .setQuery(QueryBuilders.hasChildQuery("child", matchQuery("text", "value"), ScoreMode.None)).get();
-        assertNoFailures(response);
-        assertThat(response.getHits().totalHits(), equalTo(0L));
-
-        client().prepareIndex("test", "child1").setSource(jsonBuilder().startObject().field("text", "value").endObject())
-                .setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
-
-        response = client().prepareSearch("test")
-                .setQuery(QueryBuilders.hasChildQuery("child", matchQuery("text", "value"), ScoreMode.None)).get();
-        assertNoFailures(response);
-        assertThat(response.getHits().totalHits(), equalTo(0L));
-
-        response = client().prepareSearch("test").setQuery(QueryBuilders.hasChildQuery("child", matchQuery("text", "value"), ScoreMode.Max))
-                .get();
-        assertNoFailures(response);
-        assertThat(response.getHits().totalHits(), equalTo(0L));
-
-        response = client().prepareSearch("test").setQuery(QueryBuilders.hasParentQuery("parent", matchQuery("text", "value"), false)).get();
-        assertNoFailures(response);
-        assertThat(response.getHits().totalHits(), equalTo(0L));
-
-        response = client().prepareSearch("test").setQuery(QueryBuilders.hasParentQuery("parent", matchQuery("text", "value"), true))
-                .get();
-        assertNoFailures(response);
-        assertThat(response.getHits().totalHits(), equalTo(0L));
-    }
-
-    public void testHasChildAndHasParentFilter_withFilter() throws Exception {
-        assertAcked(prepareCreate("test")
-                .addMapping("parent")
-                .addMapping("child", "_parent", "type=parent"));
-        ensureGreen();
-
-        client().prepareIndex("test", "parent", "1").setSource("p_field", 1).get();
-        client().prepareIndex("test", "child", "2").setParent("1").setSource("c_field", 1).get();
-        client().admin().indices().prepareFlush("test").get();
-
-        client().prepareIndex("test", "type1", "3").setSource("p_field", 2).get();
-        client().admin().indices().prepareFlush("test").get();
-
-        SearchResponse searchResponse = client().prepareSearch("test")
-                .setQuery(boolQuery().must(matchAllQuery()).filter(hasChildQuery("child", termQuery("c_field", 1), ScoreMode.None)))
-                .get();
-        assertNoFailures(searchResponse);
-        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
-        assertThat(searchResponse.getHits().hits()[0].id(), equalTo("1"));
-
-        searchResponse = client().prepareSearch("test")
-                .setQuery(boolQuery().must(matchAllQuery()).filter(hasParentQuery("parent", termQuery("p_field", 1), false))).get();
-        assertNoFailures(searchResponse);
-        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
-        assertThat(searchResponse.getHits().hits()[0].id(), equalTo("2"));
-    }
-
-    public void testHasChildAndHasParentWrappedInAQueryFilter() throws Exception {
-        assertAcked(prepareCreate("test")
-                .addMapping("parent")
-                .addMapping("child", "_parent", "type=parent"));
-        ensureGreen();
-
-        // query filter in case for p/c shouldn't execute per segment, but rather
-        client().prepareIndex("test", "parent", "1").setSource("p_field", 1).get();
-        client().admin().indices().prepareFlush("test").setForce(true).get();
-        client().prepareIndex("test", "child", "2").setParent("1").setSource("c_field", 1).get();
-        refresh();
-
-        SearchResponse searchResponse = client().prepareSearch("test")
-                .setQuery(boolQuery().must(matchAllQuery()).filter(hasChildQuery("child", matchQuery("c_field", 1), ScoreMode.None)))
-                .get();
-        assertSearchHit(searchResponse, 1, hasId("1"));
-
-        searchResponse = client().prepareSearch("test")
-                .setQuery(boolQuery().must(matchAllQuery()).filter(hasParentQuery("parent", matchQuery("p_field", 1), false))).get();
-        assertSearchHit(searchResponse, 1, hasId("2"));
-
-        searchResponse = client().prepareSearch("test")
-                .setQuery(boolQuery().must(matchAllQuery()).filter(boolQuery().must(hasChildQuery("child", matchQuery("c_field", 1), ScoreMode.None))))
-                .get();
-        assertSearchHit(searchResponse, 1, hasId("1"));
-
-        searchResponse = client().prepareSearch("test")
-                .setQuery(boolQuery().must(matchAllQuery()).filter(boolQuery().must(hasParentQuery("parent", matchQuery("p_field", 1), false)))).get();
-        assertSearchHit(searchResponse, 1, hasId("2"));
-    }
-
-    public void testSimpleQueryRewrite() throws Exception {
-        assertAcked(prepareCreate("test")
-                .addMapping("parent", "p_field", "type=keyword")
-                .addMapping("child", "_parent", "type=parent", "c_field", "type=keyword"));
-        ensureGreen();
-
-        // index simple data
-        int childId = 0;
-        for (int i = 0; i < 10; i++) {
-            String parentId = String.format(Locale.ROOT, "p%03d", i);
-            client().prepareIndex("test", "parent", parentId).setSource("p_field", parentId).get();
-            int j = childId;
-            for (; j < childId + 50; j++) {
-                String childUid = String.format(Locale.ROOT, "c%03d", j);
-                client().prepareIndex("test", "child", childUid).setSource("c_field", childUid).setParent(parentId).get();
-            }
-            childId = j;
-        }
-        refresh();
-
-        SearchType[] searchTypes = new SearchType[]{SearchType.QUERY_THEN_FETCH, SearchType.DFS_QUERY_THEN_FETCH};
-        for (SearchType searchType : searchTypes) {
-            SearchResponse searchResponse = client().prepareSearch("test").setSearchType(searchType)
-                    .setQuery(hasChildQuery("child", prefixQuery("c_field", "c"), ScoreMode.Max))
-                    .addSort("p_field", SortOrder.ASC)
-                    .setSize(5).get();
-            assertNoFailures(searchResponse);
-            assertThat(searchResponse.getHits().totalHits(), equalTo(10L));
-            assertThat(searchResponse.getHits().hits()[0].id(), equalTo("p000"));
-            assertThat(searchResponse.getHits().hits()[1].id(), equalTo("p001"));
-            assertThat(searchResponse.getHits().hits()[2].id(), equalTo("p002"));
-            assertThat(searchResponse.getHits().hits()[3].id(), equalTo("p003"));
-            assertThat(searchResponse.getHits().hits()[4].id(), equalTo("p004"));
-
-            searchResponse = client().prepareSearch("test").setSearchType(searchType)
-                    .setQuery(hasParentQuery("parent", prefixQuery("p_field", "p"), true)).addSort("c_field", SortOrder.ASC)
-                    .setSize(5).get();
-            assertNoFailures(searchResponse);
-            assertThat(searchResponse.getHits().totalHits(), equalTo(500L));
-            assertThat(searchResponse.getHits().hits()[0].id(), equalTo("c000"));
-            assertThat(searchResponse.getHits().hits()[1].id(), equalTo("c001"));
-            assertThat(searchResponse.getHits().hits()[2].id(), equalTo("c002"));
-            assertThat(searchResponse.getHits().hits()[3].id(), equalTo("c003"));
-            assertThat(searchResponse.getHits().hits()[4].id(), equalTo("c004"));
-        }
-    }
-
-    // Issue #3144
-    public void testReIndexingParentAndChildDocuments() throws Exception {
-        assertAcked(prepareCreate("test")
-                .addMapping("parent")
-                .addMapping("child", "_parent", "type=parent"));
-        ensureGreen();
-
-        // index simple data
-        client().prepareIndex("test", "parent", "p1").setSource("p_field", "p_value1").get();
-        client().prepareIndex("test", "child", "c1").setSource("c_field", "red").setParent("p1").get();
-        client().prepareIndex("test", "child", "c2").setSource("c_field", "yellow").setParent("p1").get();
-        client().prepareIndex("test", "parent", "p2").setSource("p_field", "p_value2").get();
-        client().prepareIndex("test", "child", "c3").setSource("c_field", "x").setParent("p2").get();
-        client().prepareIndex("test", "child", "c4").setSource("c_field", "x").setParent("p2").get();
-
-        refresh();
-
-        SearchResponse searchResponse = client().prepareSearch("test")
-                .setQuery(hasChildQuery("child", termQuery("c_field", "yellow"), ScoreMode.Total)).get();
-        assertNoFailures(searchResponse);
-        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
-        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("p1"));
-        assertThat(searchResponse.getHits().getAt(0).sourceAsString(), containsString("\"p_value1\""));
-
-        searchResponse = client()
-                .prepareSearch("test")
-                .setQuery(
-                        boolQuery().must(matchQuery("c_field", "x")).must(
-                                hasParentQuery("parent", termQuery("p_field", "p_value2"), true))).get();
-        assertNoFailures(searchResponse);
-        assertThat(searchResponse.getHits().totalHits(), equalTo(2L));
-        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("c3"));
-        assertThat(searchResponse.getHits().getAt(1).id(), equalTo("c4"));
-
-        // re-index
-        for (int i = 0; i < 10; i++) {
-            client().prepareIndex("test", "parent", "p1").setSource("p_field", "p_value1").get();
-            client().prepareIndex("test", "child", "d" + i).setSource("c_field", "red").setParent("p1").get();
-            client().prepareIndex("test", "parent", "p2").setSource("p_field", "p_value2").get();
-            client().prepareIndex("test", "child", "c3").setSource("c_field", "x").setParent("p2").get();
-            client().admin().indices().prepareRefresh("test").get();
-        }
-
-        searchResponse = client().prepareSearch("test").setQuery(hasChildQuery("child", termQuery("c_field", "yellow"), ScoreMode.Total))
-                .get();
-        assertNoFailures(searchResponse);
-        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
-        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("p1"));
-        assertThat(searchResponse.getHits().getAt(0).sourceAsString(), containsString("\"p_value1\""));
-
-        searchResponse = client()
-                .prepareSearch("test")
-                .setQuery(
-                        boolQuery().must(matchQuery("c_field", "x")).must(
-                                hasParentQuery("parent", termQuery("p_field", "p_value2"), true))).get();
-        assertNoFailures(searchResponse);
-        assertThat(searchResponse.getHits().totalHits(), equalTo(2L));
-        assertThat(searchResponse.getHits().getAt(0).id(), Matchers.anyOf(equalTo("c3"), equalTo("c4")));
-        assertThat(searchResponse.getHits().getAt(1).id(), Matchers.anyOf(equalTo("c3"), equalTo("c4")));
-    }
-
-    // Issue #3203
-    public void testHasChildQueryWithMinimumScore() throws Exception {
-        assertAcked(prepareCreate("test")
-                .addMapping("parent")
-                .addMapping("child", "_parent", "type=parent"));
-        ensureGreen();
-
-        // index simple data
-        client().prepareIndex("test", "parent", "p1").setSource("p_field", "p_value1").get();
-        client().prepareIndex("test", "child", "c1").setSource("c_field", "x").setParent("p1").get();
-        client().prepareIndex("test", "parent", "p2").setSource("p_field", "p_value2").get();
-        client().prepareIndex("test", "child", "c3").setSource("c_field", "x").setParent("p2").get();
-        client().prepareIndex("test", "child", "c4").setSource("c_field", "x").setParent("p2").get();
-        client().prepareIndex("test", "child", "c5").setSource("c_field", "x").setParent("p2").get();
-        refresh();
-
-        SearchResponse searchResponse = client().prepareSearch("test").setQuery(hasChildQuery("child", matchAllQuery(), ScoreMode.Total))
-                .setMinScore(3) // Score needs to be 3 or above!
-                .get();
-        assertNoFailures(searchResponse);
-        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
-        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("p2"));
-        assertThat(searchResponse.getHits().getAt(0).score(), equalTo(3.0f));
-    }
-
-    public void testParentFieldQuery() throws Exception {
-        assertAcked(prepareCreate("test")
-                .setSettings(Settings.builder().put(indexSettings())
-                        .put("index.refresh_interval", -1))
-                .addMapping("parent")
-                .addMapping("child", "_parent", "type=parent"));
-        ensureGreen();
-
-        SearchResponse response = client().prepareSearch("test").setQuery(termQuery("_parent", "p1"))
-                .get();
-        assertHitCount(response, 0L);
-
-        client().prepareIndex("test", "child", "c1").setSource("{}").setParent("p1").get();
-        refresh();
-
-        response = client().prepareSearch("test").setQuery(termQuery("_parent#parent", "p1")).get();
-        assertHitCount(response, 1L);
-
-        response = client().prepareSearch("test").setQuery(queryStringQuery("_parent#parent:p1")).get();
-        assertHitCount(response, 1L);
-
-        client().prepareIndex("test", "child", "c2").setSource("{}").setParent("p2").get();
-        refresh();
-        response = client().prepareSearch("test").setQuery(termsQuery("_parent#parent", "p1", "p2")).get();
-        assertHitCount(response, 2L);
-
-        response = client().prepareSearch("test")
-                .setQuery(boolQuery()
-                    .should(termQuery("_parent#parent", "p1"))
-                    .should(termQuery("_parent#parent", "p2"))
-                ).get();
-        assertHitCount(response, 2L);
-    }
-
-    public void testParentIdQuery() throws Exception {
-        assertAcked(prepareCreate("test")
-            .setSettings(Settings.builder().put(indexSettings())
-                .put("index.refresh_interval", -1))
-            .addMapping("parent")
-            .addMapping("child", "_parent", "type=parent"));
-        ensureGreen();
-
-        client().prepareIndex("test", "child", "c1").setSource("{}").setParent("p1").get();
-        refresh();
-
-        SearchResponse response = client().prepareSearch("test").setQuery(parentId("child", "p1")).get();
-        assertHitCount(response, 1L);
-
-        client().prepareIndex("test", "child", "c2").setSource("{}").setParent("p2").get();
-        refresh();
-
-        response = client().prepareSearch("test")
-            .setQuery(boolQuery()
-                .should(parentId("child", "p1"))
-                .should(parentId("child", "p2"))
-            ).get();
-        assertHitCount(response, 2L);
-    }
-
-    public void testHasChildNotBeingCached() throws IOException {
-        assertAcked(prepareCreate("test")
-                .addMapping("parent")
-                .addMapping("child", "_parent", "type=parent"));
-        ensureGreen();
-
-        // index simple data
-        client().prepareIndex("test", "parent", "p1").setSource("p_field", "p_value1").get();
-        client().prepareIndex("test", "parent", "p2").setSource("p_field", "p_value2").get();
-        client().prepareIndex("test", "parent", "p3").setSource("p_field", "p_value3").get();
-        client().prepareIndex("test", "parent", "p4").setSource("p_field", "p_value4").get();
-        client().prepareIndex("test", "parent", "p5").setSource("p_field", "p_value5").get();
-        client().prepareIndex("test", "parent", "p6").setSource("p_field", "p_value6").get();
-        client().prepareIndex("test", "parent", "p7").setSource("p_field", "p_value7").get();
-        client().prepareIndex("test", "parent", "p8").setSource("p_field", "p_value8").get();
-        client().prepareIndex("test", "parent", "p9").setSource("p_field", "p_value9").get();
-        client().prepareIndex("test", "parent", "p10").setSource("p_field", "p_value10").get();
-        client().prepareIndex("test", "child", "c1").setParent("p1").setSource("c_field", "blue").get();
-        client().admin().indices().prepareFlush("test").get();
-        client().admin().indices().prepareRefresh("test").get();
-
-        SearchResponse searchResponse = client().prepareSearch("test")
-                .setQuery(constantScoreQuery(hasChildQuery("child", termQuery("c_field", "blue"), ScoreMode.None)))
-                .get();
-        assertNoFailures(searchResponse);
-        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
-
-        client().prepareIndex("test", "child", "c2").setParent("p2").setSource("c_field", "blue").get();
-        client().admin().indices().prepareRefresh("test").get();
-
-        searchResponse = client().prepareSearch("test")
-                .setQuery(constantScoreQuery(hasChildQuery("child", termQuery("c_field", "blue"), ScoreMode.None)))
-                .get();
-        assertNoFailures(searchResponse);
-        assertThat(searchResponse.getHits().totalHits(), equalTo(2L));
-    }
-
-    private QueryBuilder randomHasChild(String type, String field, String value) {
-        if (randomBoolean()) {
-            if (randomBoolean()) {
-                return constantScoreQuery(hasChildQuery(type, termQuery(field, value), ScoreMode.None));
-            } else {
-                return boolQuery().must(matchAllQuery()).filter(hasChildQuery(type, termQuery(field, value), ScoreMode.None));
-            }
-        } else {
-            return hasChildQuery(type, termQuery(field, value), ScoreMode.None);
-        }
-    }
-
-    private QueryBuilder randomHasParent(String type, String field, String value) {
-        if (randomBoolean()) {
-            if (randomBoolean()) {
-                return constantScoreQuery(hasParentQuery(type, termQuery(field, value), false));
-            } else {
-                return boolQuery().must(matchAllQuery()).filter(hasParentQuery(type, termQuery(field, value), false));
-            }
-        } else {
-            return hasParentQuery(type, termQuery(field, value), false);
-        }
-    }
-
-    // Issue #3818
-    public void testHasChildQueryOnlyReturnsSingleChildType() {
-        assertAcked(prepareCreate("grandissue")
-                .addMapping("grandparent", "name", "type=text")
-                .addMapping("parent", "_parent", "type=grandparent")
-                .addMapping("child_type_one", "_parent", "type=parent")
-                .addMapping("child_type_two", "_parent", "type=parent"));
-
-        client().prepareIndex("grandissue", "grandparent", "1").setSource("name", "Grandpa").get();
-        client().prepareIndex("grandissue", "parent", "2").setParent("1").setSource("name", "Dana").get();
-        client().prepareIndex("grandissue", "child_type_one", "3").setParent("2").setRouting("1")
-                .setSource("name", "William")
-                .get();
-        client().prepareIndex("grandissue", "child_type_two", "4").setParent("2").setRouting("1")
-                .setSource("name", "Kate")
-                .get();
-        refresh();
-
-        SearchResponse searchResponse = client().prepareSearch("grandissue").setQuery(
-                boolQuery().must(
-                        hasChildQuery(
-                                "parent",
-                                boolQuery().must(
-                                        hasChildQuery(
-                                                "child_type_one",
-                                                boolQuery().must(
-                                                        queryStringQuery("name:William*").analyzeWildcard(true)
-                                                ),
-                                                ScoreMode.None)
-                                ),
-                                ScoreMode.None)
-                )
-        ).get();
-        assertHitCount(searchResponse, 1L);
-
-        searchResponse = client().prepareSearch("grandissue").setQuery(
-                boolQuery().must(
-                        hasChildQuery(
-                                "parent",
-                                boolQuery().must(
-                                        hasChildQuery(
-                                                "child_type_two",
-                                                boolQuery().must(
-                                                        queryStringQuery("name:William*").analyzeWildcard(true)
-                                                ),
-                                                ScoreMode.None)
-                                ),
-                                ScoreMode.None)
-                )
-        ).get();
-        assertHitCount(searchResponse, 0L);
-    }
-
-    public void testIndexChildDocWithNoParentMapping() throws IOException {
-        assertAcked(prepareCreate("test")
-                .addMapping("parent")
-                .addMapping("child1"));
-        ensureGreen();
-
-        client().prepareIndex("test", "parent", "p1").setSource("p_field", "p_value1").get();
-        try {
-            client().prepareIndex("test", "child1", "c1").setParent("p1").setSource("c_field", "blue").get();
-            fail();
-        } catch (IllegalArgumentException e) {
-            assertThat(e.toString(), containsString("Can't specify parent if no parent field has been configured"));
-        }
-        try {
-            client().prepareIndex("test", "child2", "c2").setParent("p1").setSource("c_field", "blue").get();
-            fail();
-        } catch (IllegalArgumentException e) {
-            assertThat(e.toString(), containsString("Can't specify parent if no parent field has been configured"));
-        }
-
-        refresh();
-    }
-
-    public void testAddingParentToExistingMapping() throws IOException {
-        createIndex("test");
-        ensureGreen();
-
-        PutMappingResponse putMappingResponse = client().admin().indices().preparePutMapping("test").setType("child").setSource("number", "type=integer")
-                .get();
-        assertThat(putMappingResponse.isAcknowledged(), equalTo(true));
-
-        GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings("test").get();
-        Map<String, Object> mapping = getMappingsResponse.getMappings().get("test").get("child").getSourceAsMap();
-        assertThat(mapping.size(), greaterThanOrEqualTo(1)); // there are potentially some meta fields configured randomly
-        assertThat(mapping.get("properties"), notNullValue());
-
-        try {
-            // Adding _parent metadata field to existing mapping is prohibited:
-            client().admin().indices().preparePutMapping("test").setType("child").setSource(jsonBuilder().startObject().startObject("child")
-                    .startObject("_parent").field("type", "parent").endObject()
-                    .endObject().endObject()).get();
-            fail();
-        } catch (IllegalArgumentException e) {
-            assertThat(e.toString(), containsString("The _parent field's type option can't be changed: [null]->[parent]"));
-        }
-    }
-
-    public void testHasChildQueryWithNestedInnerObjects() throws Exception {
-        assertAcked(prepareCreate("test")
-                .addMapping("parent", "objects", "type=nested")
-                .addMapping("child", "_parent", "type=parent"));
-        ensureGreen();
-
-        client().prepareIndex("test", "parent", "p1")
-                .setSource(jsonBuilder().startObject().field("p_field", "1").startArray("objects")
-                        .startObject().field("i_field", "1").endObject()
-                        .startObject().field("i_field", "2").endObject()
-                        .startObject().field("i_field", "3").endObject()
-                        .startObject().field("i_field", "4").endObject()
-                        .startObject().field("i_field", "5").endObject()
-                        .startObject().field("i_field", "6").endObject()
-                        .endArray().endObject())
-                .get();
-        client().prepareIndex("test", "parent", "p2")
-                .setSource(jsonBuilder().startObject().field("p_field", "2").startArray("objects")
-                        .startObject().field("i_field", "1").endObject()
-                        .startObject().field("i_field", "2").endObject()
-                        .endArray().endObject())
-                .get();
-        client().prepareIndex("test", "child", "c1").setParent("p1").setSource("c_field", "blue").get();
-        client().prepareIndex("test", "child", "c2").setParent("p1").setSource("c_field", "red").get();
-        client().prepareIndex("test", "child", "c3").setParent("p2").setSource("c_field", "red").get();
-        refresh();
-
-        ScoreMode scoreMode = randomFrom(ScoreMode.values());
-        SearchResponse searchResponse = client().prepareSearch("test")
-                .setQuery(boolQuery().must(QueryBuilders.hasChildQuery("child", termQuery("c_field", "blue"), scoreMode)).filter(boolQuery().mustNot(termQuery("p_field", "3"))))
-                .get();
-        assertNoFailures(searchResponse);
-        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
-
-        searchResponse = client().prepareSearch("test")
-                .setQuery(boolQuery().must(QueryBuilders.hasChildQuery("child", termQuery("c_field", "red"), scoreMode)).filter(boolQuery().mustNot(termQuery("p_field", "3"))))
-                .get();
-        assertNoFailures(searchResponse);
-        assertThat(searchResponse.getHits().totalHits(), equalTo(2L));
-    }
-
-    public void testNamedFilters() throws Exception {
-        assertAcked(prepareCreate("test")
-                .addMapping("parent")
-                .addMapping("child", "_parent", "type=parent"));
-        ensureGreen();
-
-        String parentId = "p1";
-        client().prepareIndex("test", "parent", parentId).setSource("p_field", "1").get();
-        client().prepareIndex("test", "child", "c1").setSource("c_field", "1").setParent(parentId).get();
-        refresh();
-
-        SearchResponse searchResponse = client().prepareSearch("test").setQuery(hasChildQuery("child", termQuery("c_field", "1"), ScoreMode.Max).queryName("test"))
-                .get();
-        assertHitCount(searchResponse, 1L);
-        assertThat(searchResponse.getHits().getAt(0).getMatchedQueries().length, equalTo(1));
-        assertThat(searchResponse.getHits().getAt(0).getMatchedQueries()[0], equalTo("test"));
-
-        searchResponse = client().prepareSearch("test").setQuery(hasParentQuery("parent", termQuery("p_field", "1"), true).queryName("test"))
-                .get();
-        assertHitCount(searchResponse, 1L);
-        assertThat(searchResponse.getHits().getAt(0).getMatchedQueries().length, equalTo(1));
-        assertThat(searchResponse.getHits().getAt(0).getMatchedQueries()[0], equalTo("test"));
-
-        searchResponse = client().prepareSearch("test").setQuery(constantScoreQuery(hasChildQuery("child", termQuery("c_field", "1"), ScoreMode.None).queryName("test")))
-                .get();
-        assertHitCount(searchResponse, 1L);
-        assertThat(searchResponse.getHits().getAt(0).getMatchedQueries().length, equalTo(1));
-        assertThat(searchResponse.getHits().getAt(0).getMatchedQueries()[0], equalTo("test"));
-
-        searchResponse = client().prepareSearch("test").setQuery(constantScoreQuery(hasParentQuery("parent", termQuery("p_field", "1"), false).queryName("test")))
-                .get();
-        assertHitCount(searchResponse, 1L);
-        assertThat(searchResponse.getHits().getAt(0).getMatchedQueries().length, equalTo(1));
-        assertThat(searchResponse.getHits().getAt(0).getMatchedQueries()[0], equalTo("test"));
-    }
-
-    public void testParentChildQueriesNoParentType() throws Exception {
-        assertAcked(prepareCreate("test")
-                .setSettings(Settings.builder()
-                        .put(indexSettings())
-                        .put("index.refresh_interval", -1)));
-        ensureGreen();
-
-        String parentId = "p1";
-        client().prepareIndex("test", "parent", parentId).setSource("p_field", "1").get();
-        refresh();
-
-        try {
-            client().prepareSearch("test")
-                    .setQuery(hasChildQuery("child", termQuery("c_field", "1"), ScoreMode.None))
-                    .get();
-            fail();
-        } catch (SearchPhaseExecutionException e) {
-            assertThat(e.status(), equalTo(RestStatus.BAD_REQUEST));
-        }
-
-        try {
-            client().prepareSearch("test")
-                    .setQuery(hasChildQuery("child", termQuery("c_field", "1"), ScoreMode.Max))
-                    .get();
-            fail();
-        } catch (SearchPhaseExecutionException e) {
-            assertThat(e.status(), equalTo(RestStatus.BAD_REQUEST));
-        }
-
-        try {
-            client().prepareSearch("test")
-                    .setPostFilter(hasChildQuery("child", termQuery("c_field", "1"), ScoreMode.None))
-                    .get();
-            fail();
-        } catch (SearchPhaseExecutionException e) {
-            assertThat(e.status(), equalTo(RestStatus.BAD_REQUEST));
-        }
-
-        try {
-            client().prepareSearch("test")
-                    .setQuery(hasParentQuery("parent", termQuery("p_field", "1"), true))
-                    .get();
-            fail();
-        } catch (SearchPhaseExecutionException e) {
-            assertThat(e.status(), equalTo(RestStatus.BAD_REQUEST));
-        }
-
-        try {
-            client().prepareSearch("test")
-                    .setPostFilter(hasParentQuery("parent", termQuery("p_field", "1"), false))
-                    .get();
-            fail();
-        } catch (SearchPhaseExecutionException e) {
-            assertThat(e.status(), equalTo(RestStatus.BAD_REQUEST));
-        }
-    }
-
-    public void testParentChildCaching() throws Exception {
-        assertAcked(prepareCreate("test")
-                .setSettings(
-                        Settings.builder()
-                                .put(indexSettings())
-                                .put("index.refresh_interval", -1)
-                )
-                .addMapping("parent")
-                .addMapping("child", "_parent", "type=parent"));
-        ensureGreen();
-
-        // index simple data
-        client().prepareIndex("test", "parent", "p1").setSource("p_field", "p_value1").get();
-        client().prepareIndex("test", "parent", "p2").setSource("p_field", "p_value2").get();
-        client().prepareIndex("test", "child", "c1").setParent("p1").setSource("c_field", "blue").get();
-        client().prepareIndex("test", "child", "c2").setParent("p1").setSource("c_field", "red").get();
-        client().prepareIndex("test", "child", "c3").setParent("p2").setSource("c_field", "red").get();
-        client().admin().indices().prepareForceMerge("test").setMaxNumSegments(1).setFlush(true).get();
-        client().prepareIndex("test", "parent", "p3").setSource("p_field", "p_value3").get();
-        client().prepareIndex("test", "parent", "p4").setSource("p_field", "p_value4").get();
-        client().prepareIndex("test", "child", "c4").setParent("p3").setSource("c_field", "green").get();
-        client().prepareIndex("test", "child", "c5").setParent("p3").setSource("c_field", "blue").get();
-        client().prepareIndex("test", "child", "c6").setParent("p4").setSource("c_field", "blue").get();
-        client().admin().indices().prepareFlush("test").get();
-        client().admin().indices().prepareRefresh("test").get();
-
-        for (int i = 0; i < 2; i++) {
-            SearchResponse searchResponse = client().prepareSearch()
-                    .setQuery(boolQuery().must(matchAllQuery()).filter(boolQuery()
-                            .must(QueryBuilders.hasChildQuery("child", matchQuery("c_field", "red"), ScoreMode.None))
-                            .must(matchAllQuery())))
-                    .get();
-            assertThat(searchResponse.getHits().totalHits(), equalTo(2L));
-        }
-
-
-        client().prepareIndex("test", "child", "c3").setParent("p2").setSource("c_field", "blue").get();
-        client().admin().indices().prepareRefresh("test").get();
-
-        SearchResponse searchResponse = client().prepareSearch()
-                .setQuery(boolQuery().must(matchAllQuery()).filter(boolQuery()
-                        .must(QueryBuilders.hasChildQuery("child", matchQuery("c_field", "red"), ScoreMode.None))
-                        .must(matchAllQuery())))
-                .get();
-
-        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
-    }
-
-    public void testParentChildQueriesViaScrollApi() throws Exception {
-        assertAcked(prepareCreate("test")
-                .addMapping("parent")
-                .addMapping("child", "_parent", "type=parent"));
-        ensureGreen();
-        for (int i = 0; i < 10; i++) {
-            client().prepareIndex("test", "parent", "p" + i).setSource("{}").get();
-            client().prepareIndex("test", "child", "c" + i).setSource("{}").setParent("p" + i).get();
-        }
-
-        refresh();
-
-        QueryBuilder[] queries = new QueryBuilder[]{
-                hasChildQuery("child", matchAllQuery(), ScoreMode.None),
-                boolQuery().must(matchAllQuery()).filter(hasChildQuery("child", matchAllQuery(), ScoreMode.None)),
-                hasParentQuery("parent", matchAllQuery(), false),
-                boolQuery().must(matchAllQuery()).filter(hasParentQuery("parent", matchAllQuery(), false))
-        };
-
-        for (QueryBuilder query : queries) {
-            SearchResponse scrollResponse = client().prepareSearch("test")
-                    .setScroll(TimeValue.timeValueSeconds(30))
-                    .setSize(1)
-                    .addStoredField("_id")
-                    .setQuery(query)
-                    .execute()
-                    .actionGet();
-
-            assertNoFailures(scrollResponse);
-            assertThat(scrollResponse.getHits().totalHits(), equalTo(10L));
-            int scannedDocs = 0;
-            do {
-                assertThat(scrollResponse.getHits().totalHits(), equalTo(10L));
-                scannedDocs += scrollResponse.getHits().getHits().length;
-                scrollResponse = client()
-                        .prepareSearchScroll(scrollResponse.getScrollId())
-                        .setScroll(TimeValue.timeValueSeconds(30)).get();
-            } while (scrollResponse.getHits().getHits().length > 0);
-            clearScroll(scrollResponse.getScrollId());
-            assertThat(scannedDocs, equalTo(10));
-        }
-    }
-
-    // Issue #5783
-    public void testQueryBeforeChildType() throws Exception {
-        assertAcked(prepareCreate("test")
-                .addMapping("features")
-                .addMapping("posts", "_parent", "type=features")
-                .addMapping("specials"));
-        ensureGreen();
-
-        client().prepareIndex("test", "features", "1").setSource("field", "foo").get();
-        client().prepareIndex("test", "posts", "1").setParent("1").setSource("field", "bar").get();
-        refresh();
-
-        SearchResponse resp;
-        resp = client().prepareSearch("test")
-                .setSource(new SearchSourceBuilder().query(QueryBuilders.hasChildQuery("posts", QueryBuilders.matchQuery("field", "bar"), ScoreMode.None)))
-                .get();
-        assertHitCount(resp, 1L);
-    }
-
-    // Issue #6256
-    public void testParentFieldInMultiMatchField() throws Exception {
-        assertAcked(prepareCreate("test")
-                .addMapping("type1")
-                .addMapping("type2", "_parent", "type=type1")
-        );
-        ensureGreen();
-
-        client().prepareIndex("test", "type2", "1").setParent("1").setSource("field", "value").get();
-        refresh();
-
-        SearchResponse response = client().prepareSearch("test")
-                .setQuery(multiMatchQuery("1", "_parent#type1"))
-                .get();
-
-        assertThat(response.getHits().totalHits(), equalTo(1L));
-        assertThat(response.getHits().getAt(0).id(), equalTo("1"));
-    }
-
-    public void testTypeIsAppliedInHasParentInnerQuery() throws Exception {
-        assertAcked(prepareCreate("test")
-                .addMapping("parent")
-                .addMapping("child", "_parent", "type=parent"));
-        ensureGreen();
-
-        List<IndexRequestBuilder> indexRequests = new ArrayList<>();
-        indexRequests.add(client().prepareIndex("test", "parent", "1").setSource("field1", "a"));
-        indexRequests.add(client().prepareIndex("test", "child", "1").setParent("1").setSource("{}"));
-        indexRequests.add(client().prepareIndex("test", "child", "2").setParent("1").setSource("{}"));
-        indexRandom(true, indexRequests);
-
-        SearchResponse searchResponse = client().prepareSearch("test")
-                .setQuery(constantScoreQuery(hasParentQuery("parent", boolQuery().mustNot(termQuery("field1", "a")), false)))
-                .get();
-        assertHitCount(searchResponse, 0L);
-
-        searchResponse = client().prepareSearch("test")
-                .setQuery(hasParentQuery("parent", constantScoreQuery(boolQuery().mustNot(termQuery("field1", "a"))), false))
-                .get();
-        assertHitCount(searchResponse, 0L);
-
-        searchResponse = client().prepareSearch("test")
-                .setQuery(constantScoreQuery(hasParentQuery("parent", termQuery("field1", "a"), false)))
-                .get();
-        assertHitCount(searchResponse, 2L);
-
-        searchResponse = client().prepareSearch("test")
-                .setQuery(hasParentQuery("parent", constantScoreQuery(termQuery("field1", "a")), false))
-                .get();
-        assertHitCount(searchResponse, 2L);
-    }
-
-    private List<IndexRequestBuilder> createMinMaxDocBuilders() {
-        List<IndexRequestBuilder> indexBuilders = new ArrayList<>();
-        // Parent 1 and its children
-        indexBuilders.add(client().prepareIndex().setType("parent").setId("1").setIndex("test").setSource("id",1));
-        indexBuilders.add(client().prepareIndex().setType("child").setId("10").setIndex("test")
-                .setSource("foo", "one").setParent("1"));
-
-        // Parent 2 and its children
-        indexBuilders.add(client().prepareIndex().setType("parent").setId("2").setIndex("test").setSource("id",2));
-        indexBuilders.add(client().prepareIndex().setType("child").setId("11").setIndex("test")
-                .setSource("foo", "one").setParent("2"));
-        indexBuilders.add(client().prepareIndex().setType("child").setId("12").setIndex("test")
-                .setSource("foo", "one two").setParent("2"));
-
-        // Parent 3 and its children
-        indexBuilders.add(client().prepareIndex().setType("parent").setId("3").setIndex("test").setSource("id",3));
-        indexBuilders.add(client().prepareIndex().setType("child").setId("13").setIndex("test")
-                .setSource("foo", "one").setParent("3"));
-        indexBuilders.add(client().prepareIndex().setType("child").setId("14").setIndex("test")
-                .setSource("foo", "one two").setParent("3"));
-        indexBuilders.add(client().prepareIndex().setType("child").setId("15").setIndex("test")
-                .setSource("foo", "one two three").setParent("3"));
-
-        // Parent 4 and its children
-        indexBuilders.add(client().prepareIndex().setType("parent").setId("4").setIndex("test").setSource("id",4));
-        indexBuilders.add(client().prepareIndex().setType("child").setId("16").setIndex("test")
-                .setSource("foo", "one").setParent("4"));
-        indexBuilders.add(client().prepareIndex().setType("child").setId("17").setIndex("test")
-                .setSource("foo", "one two").setParent("4"));
-        indexBuilders.add(client().prepareIndex().setType("child").setId("18").setIndex("test")
-                .setSource("foo", "one two three").setParent("4"));
-        indexBuilders.add(client().prepareIndex().setType("child").setId("19").setIndex("test")
-                .setSource("foo", "one two three four").setParent("4"));
-
-        return indexBuilders;
-    }
-
-    private SearchResponse minMaxQuery(ScoreMode scoreMode, int minChildren, Integer maxChildren) throws SearchPhaseExecutionException {
-        HasChildQueryBuilder hasChildQuery = hasChildQuery(
-                "child",
-                QueryBuilders.functionScoreQuery(constantScoreQuery(QueryBuilders.termQuery("foo", "two")),
-                        new FunctionScoreQueryBuilder.FilterFunctionBuilder[]{
-                                new FunctionScoreQueryBuilder.FilterFunctionBuilder(weightFactorFunction(1)),
-                                new FunctionScoreQueryBuilder.FilterFunctionBuilder(QueryBuilders.termQuery("foo", "three"), weightFactorFunction(1)),
-                                new FunctionScoreQueryBuilder.FilterFunctionBuilder(QueryBuilders.termQuery("foo", "four"), weightFactorFunction(1))
-                        }).boostMode(CombineFunction.REPLACE).scoreMode(FiltersFunctionScoreQuery.ScoreMode.SUM), scoreMode)
-                .minMaxChildren(minChildren, maxChildren != null ? maxChildren : HasChildQueryBuilder.DEFAULT_MAX_CHILDREN);
-
-        return client()
-                .prepareSearch("test")
-                .setQuery(hasChildQuery)
-                .addSort("_score", SortOrder.DESC).addSort("id", SortOrder.ASC).get();
-    }
-
-    public void testMinMaxChildren() throws Exception {
-        assertAcked(prepareCreate("test")
-                .addMapping("parent", "id", "type=long")
-                .addMapping("child", "_parent", "type=parent"));
-        ensureGreen();
-
-        indexRandom(true, createMinMaxDocBuilders().toArray(new IndexRequestBuilder[0]));
-        SearchResponse response;
-
-        // Score mode = NONE
-        response = minMaxQuery(ScoreMode.None, 0, null);
-
-        assertThat(response.getHits().totalHits(), equalTo(3L));
-        assertThat(response.getHits().hits()[0].id(), equalTo("2"));
-        assertThat(response.getHits().hits()[0].score(), equalTo(1f));
-        assertThat(response.getHits().hits()[1].id(), equalTo("3"));
-        assertThat(response.getHits().hits()[1].score(), equalTo(1f));
-        assertThat(response.getHits().hits()[2].id(), equalTo("4"));
-        assertThat(response.getHits().hits()[2].score(), equalTo(1f));
-
-        response = minMaxQuery(ScoreMode.None, 1, null);
-
-        assertThat(response.getHits().totalHits(), equalTo(3L));
-        assertThat(response.getHits().hits()[0].id(), equalTo("2"));
-        assertThat(response.getHits().hits()[0].score(), equalTo(1f));
-        assertThat(response.getHits().hits()[1].id(), equalTo("3"));
-        assertThat(response.getHits().hits()[1].score(), equalTo(1f));
-        assertThat(response.getHits().hits()[2].id(), equalTo("4"));
-        assertThat(response.getHits().hits()[2].score(), equalTo(1f));
-
-        response = minMaxQuery(ScoreMode.None, 2, null);
-
-        assertThat(response.getHits().totalHits(), equalTo(2L));
-        assertThat(response.getHits().hits()[0].id(), equalTo("3"));
-        assertThat(response.getHits().hits()[0].score(), equalTo(1f));
-        assertThat(response.getHits().hits()[1].id(), equalTo("4"));
-        assertThat(response.getHits().hits()[1].score(), equalTo(1f));
-
-        response = minMaxQuery(ScoreMode.None, 3, null);
-
-        assertThat(response.getHits().totalHits(), equalTo(1L));
-        assertThat(response.getHits().hits()[0].id(), equalTo("4"));
-        assertThat(response.getHits().hits()[0].score(), equalTo(1f));
-
-        response = minMaxQuery(ScoreMode.None, 4, null);
-
-        assertThat(response.getHits().totalHits(), equalTo(0L));
-
-        response = minMaxQuery(ScoreMode.None, 0, 4);
-
-        assertThat(response.getHits().totalHits(), equalTo(3L));
-        assertThat(response.getHits().hits()[0].id(), equalTo("2"));
-        assertThat(response.getHits().hits()[0].score(), equalTo(1f));
-        assertThat(response.getHits().hits()[1].id(), equalTo("3"));
-        assertThat(response.getHits().hits()[1].score(), equalTo(1f));
-        assertThat(response.getHits().hits()[2].id(), equalTo("4"));
-        assertThat(response.getHits().hits()[2].score(), equalTo(1f));
-
-        response = minMaxQuery(ScoreMode.None, 0, 3);
-
-        assertThat(response.getHits().totalHits(), equalTo(3L));
-        assertThat(response.getHits().hits()[0].id(), equalTo("2"));
-        assertThat(response.getHits().hits()[0].score(), equalTo(1f));
-        assertThat(response.getHits().hits()[1].id(), equalTo("3"));
-        assertThat(response.getHits().hits()[1].score(), equalTo(1f));
-        assertThat(response.getHits().hits()[2].id(), equalTo("4"));
-        assertThat(response.getHits().hits()[2].score(), equalTo(1f));
-
-        response = minMaxQuery(ScoreMode.None, 0, 2);
-
-        assertThat(response.getHits().totalHits(), equalTo(2L));
-        assertThat(response.getHits().hits()[0].id(), equalTo("2"));
-        assertThat(response.getHits().hits()[0].score(), equalTo(1f));
-        assertThat(response.getHits().hits()[1].id(), equalTo("3"));
-        assertThat(response.getHits().hits()[1].score(), equalTo(1f));
-
-        response = minMaxQuery(ScoreMode.None, 2, 2);
-
-        assertThat(response.getHits().totalHits(), equalTo(1L));
-        assertThat(response.getHits().hits()[0].id(), equalTo("3"));
-        assertThat(response.getHits().hits()[0].score(), equalTo(1f));
-
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> minMaxQuery(ScoreMode.None, 3, 2));
-        assertThat(e.getMessage(), equalTo("[has_child] 'max_children' is less than 'min_children'"));
-
-        // Score mode = SUM
-        response = minMaxQuery(ScoreMode.Total, 0, null);
-
-        assertThat(response.getHits().totalHits(), equalTo(3L));
-        assertThat(response.getHits().hits()[0].id(), equalTo("4"));
-        assertThat(response.getHits().hits()[0].score(), equalTo(6f));
-        assertThat(response.getHits().hits()[1].id(), equalTo("3"));
-        assertThat(response.getHits().hits()[1].score(), equalTo(3f));
-        assertThat(response.getHits().hits()[2].id(), equalTo("2"));
-        assertThat(response.getHits().hits()[2].score(), equalTo(1f));
-
-        response = minMaxQuery(ScoreMode.Total, 1, null);
-
-        assertThat(response.getHits().totalHits(), equalTo(3L));
-        assertThat(response.getHits().hits()[0].id(), equalTo("4"));
-        assertThat(response.getHits().hits()[0].score(), equalTo(6f));
-        assertThat(response.getHits().hits()[1].id(), equalTo("3"));
-        assertThat(response.getHits().hits()[1].score(), equalTo(3f));
-        assertThat(response.getHits().hits()[2].id(), equalTo("2"));
-        assertThat(response.getHits().hits()[2].score(), equalTo(1f));
-
-        response = minMaxQuery(ScoreMode.Total, 2, null);
-
-        assertThat(response.getHits().totalHits(), equalTo(2L));
-        assertThat(response.getHits().hits()[0].id(), equalTo("4"));
-        assertThat(response.getHits().hits()[0].score(), equalTo(6f));
-        assertThat(response.getHits().hits()[1].id(), equalTo("3"));
-        assertThat(response.getHits().hits()[1].score(), equalTo(3f));
-
-        response = minMaxQuery(ScoreMode.Total, 3, null);
-
-        assertThat(response.getHits().totalHits(), equalTo(1L));
-        assertThat(response.getHits().hits()[0].id(), equalTo("4"));
-        assertThat(response.getHits().hits()[0].score(), equalTo(6f));
-
-        response = minMaxQuery(ScoreMode.Total, 4, null);
-
-        assertThat(response.getHits().totalHits(), equalTo(0L));
-
-        response = minMaxQuery(ScoreMode.Total, 0, 4);
-
-        assertThat(response.getHits().totalHits(), equalTo(3L));
-        assertThat(response.getHits().hits()[0].id(), equalTo("4"));
-        assertThat(response.getHits().hits()[0].score(), equalTo(6f));
-        assertThat(response.getHits().hits()[1].id(), equalTo("3"));
-        assertThat(response.getHits().hits()[1].score(), equalTo(3f));
-        assertThat(response.getHits().hits()[2].id(), equalTo("2"));
-        assertThat(response.getHits().hits()[2].score(), equalTo(1f));
-
-        response = minMaxQuery(ScoreMode.Total, 0, 3);
-
-        assertThat(response.getHits().totalHits(), equalTo(3L));
-        assertThat(response.getHits().hits()[0].id(), equalTo("4"));
-        assertThat(response.getHits().hits()[0].score(), equalTo(6f));
-        assertThat(response.getHits().hits()[1].id(), equalTo("3"));
-        assertThat(response.getHits().hits()[1].score(), equalTo(3f));
-        assertThat(response.getHits().hits()[2].id(), equalTo("2"));
-        assertThat(response.getHits().hits()[2].score(), equalTo(1f));
-
-        response = minMaxQuery(ScoreMode.Total, 0, 2);
-
-        assertThat(response.getHits().totalHits(), equalTo(2L));
-        assertThat(response.getHits().hits()[0].id(), equalTo("3"));
-        assertThat(response.getHits().hits()[0].score(), equalTo(3f));
-        assertThat(response.getHits().hits()[1].id(), equalTo("2"));
-        assertThat(response.getHits().hits()[1].score(), equalTo(1f));
-
-        response = minMaxQuery(ScoreMode.Total, 2, 2);
-
-        assertThat(response.getHits().totalHits(), equalTo(1L));
-        assertThat(response.getHits().hits()[0].id(), equalTo("3"));
-        assertThat(response.getHits().hits()[0].score(), equalTo(3f));
-
-        e = expectThrows(IllegalArgumentException.class, () -> minMaxQuery(ScoreMode.Total, 3, 2));
-        assertThat(e.getMessage(), equalTo("[has_child] 'max_children' is less than 'min_children'"));
-
-        // Score mode = MAX
-        response = minMaxQuery(ScoreMode.Max, 0, null);
-
-        assertThat(response.getHits().totalHits(), equalTo(3L));
-        assertThat(response.getHits().hits()[0].id(), equalTo("4"));
-        assertThat(response.getHits().hits()[0].score(), equalTo(3f));
-        assertThat(response.getHits().hits()[1].id(), equalTo("3"));
-        assertThat(response.getHits().hits()[1].score(), equalTo(2f));
-        assertThat(response.getHits().hits()[2].id(), equalTo("2"));
-        assertThat(response.getHits().hits()[2].score(), equalTo(1f));
-
-        response = minMaxQuery(ScoreMode.Max, 1, null);
-
-        assertThat(response.getHits().totalHits(), equalTo(3L));
-        assertThat(response.getHits().hits()[0].id(), equalTo("4"));
-        assertThat(response.getHits().hits()[0].score(), equalTo(3f));
-        assertThat(response.getHits().hits()[1].id(), equalTo("3"));
-        assertThat(response.getHits().hits()[1].score(), equalTo(2f));
-        assertThat(response.getHits().hits()[2].id(), equalTo("2"));
-        assertThat(response.getHits().hits()[2].score(), equalTo(1f));
-
-        response = minMaxQuery(ScoreMode.Max, 2, null);
-
-        assertThat(response.getHits().totalHits(), equalTo(2L));
-        assertThat(response.getHits().hits()[0].id(), equalTo("4"));
-        assertThat(response.getHits().hits()[0].score(), equalTo(3f));
-        assertThat(response.getHits().hits()[1].id(), equalTo("3"));
-        assertThat(response.getHits().hits()[1].score(), equalTo(2f));
-
-        response = minMaxQuery(ScoreMode.Max, 3, null);
-
-        assertThat(response.getHits().totalHits(), equalTo(1L));
-        assertThat(response.getHits().hits()[0].id(), equalTo("4"));
-        assertThat(response.getHits().hits()[0].score(), equalTo(3f));
-
-        response = minMaxQuery(ScoreMode.Max, 4, null);
-
-        assertThat(response.getHits().totalHits(), equalTo(0L));
-
-        response = minMaxQuery(ScoreMode.Max, 0, 4);
-
-        assertThat(response.getHits().totalHits(), equalTo(3L));
-        assertThat(response.getHits().hits()[0].id(), equalTo("4"));
-        assertThat(response.getHits().hits()[0].score(), equalTo(3f));
-        assertThat(response.getHits().hits()[1].id(), equalTo("3"));
-        assertThat(response.getHits().hits()[1].score(), equalTo(2f));
-        assertThat(response.getHits().hits()[2].id(), equalTo("2"));
-        assertThat(response.getHits().hits()[2].score(), equalTo(1f));
-
-        response = minMaxQuery(ScoreMode.Max, 0, 3);
-
-        assertThat(response.getHits().totalHits(), equalTo(3L));
-        assertThat(response.getHits().hits()[0].id(), equalTo("4"));
-        assertThat(response.getHits().hits()[0].score(), equalTo(3f));
-        assertThat(response.getHits().hits()[1].id(), equalTo("3"));
-        assertThat(response.getHits().hits()[1].score(), equalTo(2f));
-        assertThat(response.getHits().hits()[2].id(), equalTo("2"));
-        assertThat(response.getHits().hits()[2].score(), equalTo(1f));
-
-        response = minMaxQuery(ScoreMode.Max, 0, 2);
-
-        assertThat(response.getHits().totalHits(), equalTo(2L));
-        assertThat(response.getHits().hits()[0].id(), equalTo("3"));
-        assertThat(response.getHits().hits()[0].score(), equalTo(2f));
-        assertThat(response.getHits().hits()[1].id(), equalTo("2"));
-        assertThat(response.getHits().hits()[1].score(), equalTo(1f));
-
-        response = minMaxQuery(ScoreMode.Max, 2, 2);
-
-        assertThat(response.getHits().totalHits(), equalTo(1L));
-        assertThat(response.getHits().hits()[0].id(), equalTo("3"));
-        assertThat(response.getHits().hits()[0].score(), equalTo(2f));
-
-        e = expectThrows(IllegalArgumentException.class, () -> minMaxQuery(ScoreMode.Max, 3, 2));
-        assertThat(e.getMessage(), equalTo("[has_child] 'max_children' is less than 'min_children'"));
-
-        // Score mode = AVG
-        response = minMaxQuery(ScoreMode.Avg, 0, null);
-
-        assertThat(response.getHits().totalHits(), equalTo(3L));
-        assertThat(response.getHits().hits()[0].id(), equalTo("4"));
-        assertThat(response.getHits().hits()[0].score(), equalTo(2f));
-        assertThat(response.getHits().hits()[1].id(), equalTo("3"));
-        assertThat(response.getHits().hits()[1].score(), equalTo(1.5f));
-        assertThat(response.getHits().hits()[2].id(), equalTo("2"));
-        assertThat(response.getHits().hits()[2].score(), equalTo(1f));
-
-        response = minMaxQuery(ScoreMode.Avg, 1, null);
-
-        assertThat(response.getHits().totalHits(), equalTo(3L));
-        assertThat(response.getHits().hits()[0].id(), equalTo("4"));
-        assertThat(response.getHits().hits()[0].score(), equalTo(2f));
-        assertThat(response.getHits().hits()[1].id(), equalTo("3"));
-        assertThat(response.getHits().hits()[1].score(), equalTo(1.5f));
-        assertThat(response.getHits().hits()[2].id(), equalTo("2"));
-        assertThat(response.getHits().hits()[2].score(), equalTo(1f));
-
-        response = minMaxQuery(ScoreMode.Avg, 2, null);
-
-        assertThat(response.getHits().totalHits(), equalTo(2L));
-        assertThat(response.getHits().hits()[0].id(), equalTo("4"));
-        assertThat(response.getHits().hits()[0].score(), equalTo(2f));
-        assertThat(response.getHits().hits()[1].id(), equalTo("3"));
-        assertThat(response.getHits().hits()[1].score(), equalTo(1.5f));
-
-        response = minMaxQuery(ScoreMode.Avg, 3, null);
-
-        assertThat(response.getHits().totalHits(), equalTo(1L));
-        assertThat(response.getHits().hits()[0].id(), equalTo("4"));
-        assertThat(response.getHits().hits()[0].score(), equalTo(2f));
-
-        response = minMaxQuery(ScoreMode.Avg, 4, null);
-
-        assertThat(response.getHits().totalHits(), equalTo(0L));
-
-        response = minMaxQuery(ScoreMode.Avg, 0, 4);
-
-        assertThat(response.getHits().totalHits(), equalTo(3L));
-        assertThat(response.getHits().hits()[0].id(), equalTo("4"));
-        assertThat(response.getHits().hits()[0].score(), equalTo(2f));
-        assertThat(response.getHits().hits()[1].id(), equalTo("3"));
-        assertThat(response.getHits().hits()[1].score(), equalTo(1.5f));
-        assertThat(response.getHits().hits()[2].id(), equalTo("2"));
-        assertThat(response.getHits().hits()[2].score(), equalTo(1f));
-
-        response = minMaxQuery(ScoreMode.Avg, 0, 3);
-
-        assertThat(response.getHits().totalHits(), equalTo(3L));
-        assertThat(response.getHits().hits()[0].id(), equalTo("4"));
-        assertThat(response.getHits().hits()[0].score(), equalTo(2f));
-        assertThat(response.getHits().hits()[1].id(), equalTo("3"));
-        assertThat(response.getHits().hits()[1].score(), equalTo(1.5f));
-        assertThat(response.getHits().hits()[2].id(), equalTo("2"));
-        assertThat(response.getHits().hits()[2].score(), equalTo(1f));
-
-        response = minMaxQuery(ScoreMode.Avg, 0, 2);
-
-        assertThat(response.getHits().totalHits(), equalTo(2L));
-        assertThat(response.getHits().hits()[0].id(), equalTo("3"));
-        assertThat(response.getHits().hits()[0].score(), equalTo(1.5f));
-        assertThat(response.getHits().hits()[1].id(), equalTo("2"));
-        assertThat(response.getHits().hits()[1].score(), equalTo(1f));
-
-        response = minMaxQuery(ScoreMode.Avg, 2, 2);
-
-        assertThat(response.getHits().totalHits(), equalTo(1L));
-        assertThat(response.getHits().hits()[0].id(), equalTo("3"));
-        assertThat(response.getHits().hits()[0].score(), equalTo(1.5f));
-
-        e = expectThrows(IllegalArgumentException.class, () -> minMaxQuery(ScoreMode.Avg, 3, 2));
-        assertThat(e.getMessage(), equalTo("[has_child] 'max_children' is less than 'min_children'"));
-    }
-
-    public void testParentFieldToNonExistingType() {
-        assertAcked(prepareCreate("test").addMapping("parent").addMapping("child", "_parent", "type=parent2"));
-        client().prepareIndex("test", "parent", "1").setSource("{}").get();
-        client().prepareIndex("test", "child", "1").setParent("1").setSource("{}").get();
-        refresh();
-
-        try {
-            client().prepareSearch("test")
-                    .setQuery(QueryBuilders.hasChildQuery("child", matchAllQuery(), ScoreMode.None))
-                    .get();
-            fail();
-        } catch (SearchPhaseExecutionException e) {
-        }
-    }
-
-    public void testHasParentInnerQueryType() {
-        assertAcked(prepareCreate("test").addMapping("parent-type").addMapping("child-type", "_parent", "type=parent-type"));
-        client().prepareIndex("test", "child-type", "child-id").setParent("parent-id").setSource("{}").get();
-        client().prepareIndex("test", "parent-type", "parent-id").setSource("{}").get();
-        refresh();
-        //make sure that when we explicitly set a type, the inner query is executed in the context of the parent type instead
-        SearchResponse searchResponse = client().prepareSearch("test").setTypes("child-type").setQuery(
-                QueryBuilders.hasParentQuery("parent-type", new IdsQueryBuilder().addIds("parent-id"), false)).get();
-        assertSearchHits(searchResponse, "child-id");
-    }
-
-    public void testHasChildInnerQueryType() {
-        assertAcked(prepareCreate("test").addMapping("parent-type").addMapping("child-type", "_parent", "type=parent-type"));
-        client().prepareIndex("test", "child-type", "child-id").setParent("parent-id").setSource("{}").get();
-        client().prepareIndex("test", "parent-type", "parent-id").setSource("{}").get();
-        refresh();
-        //make sure that when we explicitly set a type, the inner query is executed in the context of the child type instead
-        SearchResponse searchResponse = client().prepareSearch("test").setTypes("parent-type").setQuery(
-                QueryBuilders.hasChildQuery("child-type", new IdsQueryBuilder().addIds("child-id"), ScoreMode.None)).get();
-        assertSearchHits(searchResponse, "parent-id");
-    }
-
-    public void testHighlightersIgnoreParentChild() {
-        assertAcked(prepareCreate("test")
-                .addMapping("parent-type", "searchText", "type=text,term_vector=with_positions_offsets,index_options=offsets")
-                .addMapping("child-type", "_parent", "type=parent-type", "searchText",
-                        "type=text,term_vector=with_positions_offsets,index_options=offsets"));
-        client().prepareIndex("test", "parent-type", "parent-id").setSource("searchText", "quick brown fox").get();
-        client().prepareIndex("test", "child-type", "child-id").setParent("parent-id").setSource("searchText", "quick brown fox").get();
-        refresh();
-
-        String[] highlightTypes = new String[] {"plain", "fvh", "postings"};
-        for (String highlightType : highlightTypes) {
-            logger.info("Testing with highlight type [{}]", highlightType);
-            SearchResponse searchResponse = client().prepareSearch("test")
-                    .setQuery(new BoolQueryBuilder()
-                            .must(new MatchQueryBuilder("searchText", "fox"))
-                            .must(new HasChildQueryBuilder("child-type", new MatchAllQueryBuilder(), ScoreMode.None))
-                    )
-                    .highlighter(new HighlightBuilder().field(new HighlightBuilder.Field("searchText").highlighterType(highlightType)))
-                    .get();
-            assertHitCount(searchResponse, 1);
-            assertThat(searchResponse.getHits().getAt(0).id(), equalTo("parent-id"));
-            HighlightField highlightField = searchResponse.getHits().getAt(0).getHighlightFields().get("searchText");
-            assertThat(highlightField.getFragments()[0].string(), equalTo("quick brown <em>fox</em>"));
-
-            searchResponse = client().prepareSearch("test")
-                    .setQuery(new BoolQueryBuilder()
-                            .must(new MatchQueryBuilder("searchText", "fox"))
-                            .must(new HasParentQueryBuilder("parent-type", new MatchAllQueryBuilder(), false))
-                    )
-                    .highlighter(new HighlightBuilder().field(new HighlightBuilder.Field("searchText").highlighterType(highlightType)))
-                    .get();
-            assertHitCount(searchResponse, 1);
-            assertThat(searchResponse.getHits().getAt(0).id(), equalTo("child-id"));
-            highlightField = searchResponse.getHits().getAt(0).getHighlightFields().get("searchText");
-            assertThat(highlightField.getFragments()[0].string(), equalTo("quick brown <em>fox</em>"));
-        }
-    }
-
-}
diff --git a/core/src/test/java/org/elasticsearch/search/child/ParentFieldLoadingIT.java b/core/src/test/java/org/elasticsearch/search/child/ParentFieldLoadingIT.java
index 5aead911caa57..21e7e51c38b22 100644
--- a/core/src/test/java/org/elasticsearch/search/child/ParentFieldLoadingIT.java
+++ b/core/src/test/java/org/elasticsearch/search/child/ParentFieldLoadingIT.java
@@ -26,12 +26,13 @@
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.MergePolicyConfig;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.MergePolicyConfig;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -69,8 +70,8 @@ public void testEagerParentFieldLoading() throws Exception {
                 .addMapping("child", childMapping(false)));
         ensureGreen();
 
-        client().prepareIndex("test", "parent", "1").setSource("{}").get();
-        client().prepareIndex("test", "child", "1").setParent("1").setSource("{}").get();
+        client().prepareIndex("test", "parent", "1").setSource("{}", XContentType.JSON).get();
+        client().prepareIndex("test", "child", "1").setParent("1").setSource("{}", XContentType.JSON).get();
         refresh();
 
         ClusterStatsResponse response = client().admin().cluster().prepareClusterStats().get();
@@ -84,8 +85,8 @@ public void testEagerParentFieldLoading() throws Exception {
                 .addMapping("child", "_parent", "type=parent"));
         ensureGreen();
 
-        client().prepareIndex("test", "parent", "1").setSource("{}").get();
-        client().prepareIndex("test", "child", "1").setParent("1").setSource("{}").get();
+        client().prepareIndex("test", "parent", "1").setSource("{}", XContentType.JSON).get();
+        client().prepareIndex("test", "child", "1").setParent("1").setSource("{}", XContentType.JSON).get();
         refresh();
 
         response = client().admin().cluster().prepareClusterStats().get();
@@ -101,9 +102,9 @@ public void testEagerParentFieldLoading() throws Exception {
 
         // Need to do 2 separate refreshes, otherwise we have 1 segment and then we can't measure if global ordinals
         // is loaded by the size of the field data cache, because global ordinals on 1 segment shards takes no extra memory.
-        client().prepareIndex("test", "parent", "1").setSource("{}").get();
+        client().prepareIndex("test", "parent", "1").setSource("{}", XContentType.JSON).get();
         refresh();
-        client().prepareIndex("test", "child", "1").setParent("1").setSource("{}").get();
+        client().prepareIndex("test", "child", "1").setParent("1").setSource("{}", XContentType.JSON).get();
         refresh();
 
         response = client().admin().cluster().prepareClusterStats().get();
@@ -117,8 +118,8 @@ public void testChangingEagerParentFieldLoadingAtRuntime() throws Exception {
                 .addMapping("child", "_parent", "type=parent"));
         ensureGreen();
 
-        client().prepareIndex("test", "parent", "1").setSource("{}").get();
-        client().prepareIndex("test", "child", "1").setParent("1").setSource("{}").get();
+        client().prepareIndex("test", "parent", "1").setSource("{}", XContentType.JSON).get();
+        client().prepareIndex("test", "child", "1").setParent("1").setSource("{}", XContentType.JSON).get();
         refresh();
 
         ClusterStatsResponse response = client().admin().cluster().prepareClusterStats().get();
@@ -130,30 +131,27 @@ public void testChangingEagerParentFieldLoadingAtRuntime() throws Exception {
                 .get();
         assertAcked(putMappingResponse);
         Index test = resolveIndex("test");
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                ClusterState clusterState = internalCluster().clusterService().state();
-                ShardRouting shardRouting = clusterState.routingTable().index("test").shard(0).getShards().get(0);
-                String nodeName = clusterState.getNodes().get(shardRouting.currentNodeId()).getName();
-
-                boolean verified = false;
-                IndicesService indicesService = internalCluster().getInstance(IndicesService.class, nodeName);
-                IndexService indexService = indicesService.indexService(test);
-                if (indexService != null) {
-                    MapperService mapperService = indexService.mapperService();
-                    DocumentMapper documentMapper = mapperService.documentMapper("child");
-                    if (documentMapper != null) {
-                        verified = documentMapper.parentFieldMapper().fieldType().eagerGlobalOrdinals();
-                    }
+        assertBusy(() -> {
+            ClusterState clusterState = internalCluster().clusterService().state();
+            ShardRouting shardRouting = clusterState.routingTable().index("test").shard(0).getShards().get(0);
+            String nodeName = clusterState.getNodes().get(shardRouting.currentNodeId()).getName();
+
+            boolean verified = false;
+            IndicesService indicesService = internalCluster().getInstance(IndicesService.class, nodeName);
+            IndexService indexService = indicesService.indexService(test);
+            if (indexService != null) {
+                MapperService mapperService = indexService.mapperService();
+                DocumentMapper documentMapper = mapperService.documentMapper("child");
+                if (documentMapper != null) {
+                    verified = documentMapper.parentFieldMapper().fieldType().eagerGlobalOrdinals();
                 }
-                assertTrue(verified);
             }
+            assertTrue(verified);
         });
 
         // Need to add a new doc otherwise the refresh doesn't trigger a new searcher
         // Because it ends up in its own segment, but isn't of type parent or child, this doc doesn't contribute to the size of the fielddata cache
-        client().prepareIndex("test", "dummy", "dummy").setSource("{}").get();
+        client().prepareIndex("test", "dummy", "dummy").setSource("{}", XContentType.JSON).get();
         refresh();
         response = client().admin().cluster().prepareClusterStats().get();
         assertThat(response.getIndicesStats().getFieldData().getMemorySizeInBytes(), greaterThan(0L));
diff --git a/core/src/test/java/org/elasticsearch/search/collapse/CollapseBuilderTests.java b/core/src/test/java/org/elasticsearch/search/collapse/CollapseBuilderTests.java
new file mode 100644
index 0000000000000..e310d2c7b1a8d
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/collapse/CollapseBuilderTests.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.collapse;
+
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.RAMDirectory;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.mapper.KeywordFieldMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.NumberFieldMapper;
+import org.elasticsearch.index.query.InnerHitBuilder;
+import org.elasticsearch.index.query.InnerHitBuilderTests;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.search.AbstractSearchTestCase;
+import org.elasticsearch.search.SearchContextException;
+import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.EqualsHashCodeTestUtils;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static java.util.Collections.emptyList;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class CollapseBuilderTests extends AbstractSearchTestCase {
+    private static NamedWriteableRegistry namedWriteableRegistry;
+
+    @BeforeClass
+    public static void init() {
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList());
+        namedWriteableRegistry = new NamedWriteableRegistry(searchModule.getNamedWriteables());
+    }
+
+    @AfterClass
+    public static void afterClass() throws Exception {
+        namedWriteableRegistry = null;
+    }
+
+    public static CollapseBuilder randomCollapseBuilder() {
+        return randomCollapseBuilder(true);
+    }
+
+    public static CollapseBuilder randomCollapseBuilder(boolean multiInnerHits) {
+        CollapseBuilder builder = new CollapseBuilder(randomAlphaOfLength(10));
+        builder.setMaxConcurrentGroupRequests(randomIntBetween(1, 48));
+        int numInnerHits = randomIntBetween(0, multiInnerHits ? 5 : 1);
+        if (numInnerHits == 1) {
+            InnerHitBuilder innerHit = InnerHitBuilderTests.randomInnerHits();
+            builder.setInnerHits(innerHit);
+        } else if (numInnerHits > 1) {
+            List<InnerHitBuilder> innerHits = new ArrayList<>(numInnerHits);
+            for (int i = 0; i < numInnerHits; i++) {
+                innerHits.add(InnerHitBuilderTests.randomInnerHits());
+            }
+
+            builder.setInnerHits(innerHits);
+        }
+
+        return builder;
+    }
+
+    public void testSerialization() throws IOException {
+        CollapseBuilder testBuilder = randomCollapseBuilder();
+        try (BytesStreamOutput output = new BytesStreamOutput()) {
+            testBuilder.writeTo(output);
+            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
+                CollapseBuilder deserializedBuilder = new CollapseBuilder(in);
+                assertEquals(deserializedBuilder, testBuilder);
+                assertEquals(deserializedBuilder.hashCode(), testBuilder.hashCode());
+                assertNotSame(deserializedBuilder, testBuilder);
+            }
+        }
+    }
+
+    public void testEqualsAndHashcode() throws IOException {
+        CollapseBuilder instance1 = randomCollapseBuilder();
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(instance1, this::copyBuilder);
+    }
+
+    //we use the streaming infra to create a copy of the builder provided as argument
+    private CollapseBuilder copyBuilder(CollapseBuilder original) throws IOException {
+        return ESTestCase.copyWriteable(original, namedWriteableRegistry, CollapseBuilder::new);
+    }
+
+    private SearchContext mockSearchContext() {
+        SearchContext context = mock(SearchContext.class);
+        QueryShardContext shardContext = mock(QueryShardContext.class);
+        when(context.getQueryShardContext()).thenReturn(shardContext);
+        when(context.scrollContext()).thenReturn(null);
+        when(context.rescore()).thenReturn(null);
+        when(context.searchAfter()).thenReturn(null);
+        return context;
+    }
+
+    public void testBuild() throws IOException {
+        Directory dir = new RAMDirectory();
+        try (IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())))) {
+            writer.commit();
+        }
+        SearchContext searchContext = mockSearchContext();
+        try (IndexReader reader = DirectoryReader.open(dir)) {
+            when(searchContext.getQueryShardContext().getIndexReader()).thenReturn(reader);
+            MappedFieldType numberFieldType =
+                new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
+            MappedFieldType keywordFieldType =
+                new KeywordFieldMapper.KeywordFieldType();
+            for (MappedFieldType fieldType : new MappedFieldType[] {numberFieldType, keywordFieldType}) {
+                fieldType.setName("field");
+                fieldType.setHasDocValues(true);
+                when(searchContext.getQueryShardContext().fieldMapper("field")).thenReturn(fieldType);
+                CollapseBuilder builder = new CollapseBuilder("field");
+                CollapseContext collapseContext = builder.build(searchContext);
+                assertEquals(collapseContext.getFieldType(), fieldType);
+
+                fieldType.setIndexOptions(IndexOptions.NONE);
+                collapseContext = builder.build(searchContext);
+                assertEquals(collapseContext.getFieldType(), fieldType);
+
+                fieldType.setHasDocValues(false);
+                SearchContextException exc = expectThrows(SearchContextException.class, () -> builder.build(searchContext));
+                assertEquals(exc.getMessage(), "cannot collapse on field `field` without `doc_values`");
+
+                fieldType.setHasDocValues(true);
+                builder.setInnerHits(new InnerHitBuilder());
+                exc = expectThrows(SearchContextException.class, () -> builder.build(searchContext));
+                assertEquals(exc.getMessage(),
+                    "cannot expand `inner_hits` for collapse field `field`, " +
+                        "only indexed field can retrieve `inner_hits`");
+            }
+        }
+    }
+
+    public void testBuildWithSearchContextExceptions() throws IOException {
+        SearchContext context = mockSearchContext();
+        {
+            CollapseBuilder builder = new CollapseBuilder("unknown_field");
+            SearchContextException exc = expectThrows(SearchContextException.class, () -> builder.build(context));
+            assertEquals(exc.getMessage(), "no mapping found for `unknown_field` in order to collapse on");
+        }
+
+        {
+            MappedFieldType fieldType = new MappedFieldType() {
+                @Override
+                public MappedFieldType clone() {
+                    return null;
+                }
+
+                @Override
+                public String typeName() {
+                    return null;
+                }
+
+                @Override
+                public Query termQuery(Object value, QueryShardContext context) {
+                    return null;
+                }
+            };
+            fieldType.setName("field");
+            fieldType.setHasDocValues(true);
+            when(context.getQueryShardContext().fieldMapper("field")).thenReturn(fieldType);
+            CollapseBuilder builder = new CollapseBuilder("field");
+            SearchContextException exc = expectThrows(SearchContextException.class, () -> builder.build(context));
+            assertEquals(exc.getMessage(), "unknown type for collapse field `field`, only keywords and numbers are accepted");
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/controller/SearchPhaseControllerTests.java b/core/src/test/java/org/elasticsearch/search/controller/SearchPhaseControllerTests.java
deleted file mode 100644
index 33ef23db6edb6..0000000000000
--- a/core/src/test/java/org/elasticsearch/search/controller/SearchPhaseControllerTests.java
+++ /dev/null
@@ -1,234 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.controller;
-
-import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.TopDocs;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.text.Text;
-import org.elasticsearch.common.util.BigArrays;
-import org.elasticsearch.common.util.concurrent.AtomicArray;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.search.SearchShardTarget;
-import org.elasticsearch.search.fetch.FetchSearchResult;
-import org.elasticsearch.search.fetch.FetchSearchResultProvider;
-import org.elasticsearch.search.internal.InternalSearchHit;
-import org.elasticsearch.search.internal.InternalSearchHits;
-import org.elasticsearch.search.internal.InternalSearchResponse;
-import org.elasticsearch.search.query.QuerySearchResult;
-import org.elasticsearch.search.query.QuerySearchResultProvider;
-import org.elasticsearch.search.suggest.Suggest;
-import org.elasticsearch.search.suggest.completion.CompletionSuggestion;
-import org.elasticsearch.test.ESTestCase;
-import org.junit.Before;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.instanceOf;
-
-public class SearchPhaseControllerTests extends ESTestCase {
-    private SearchPhaseController searchPhaseController;
-
-    @Before
-    public void setup() {
-        searchPhaseController = new SearchPhaseController(Settings.EMPTY, BigArrays.NON_RECYCLING_INSTANCE, null, null);
-    }
-
-    public void testSort() throws Exception {
-        List<CompletionSuggestion> suggestions = new ArrayList<>();
-        for (int i = 0; i < randomIntBetween(1, 5); i++) {
-            suggestions.add(new CompletionSuggestion(randomAsciiOfLength(randomIntBetween(1, 5)), randomIntBetween(1, 20)));
-        }
-        int nShards = randomIntBetween(1, 20);
-        int queryResultSize = randomBoolean() ? 0 : randomIntBetween(1, nShards * 2);
-        AtomicArray<QuerySearchResultProvider> results = generateQueryResults(nShards, suggestions, queryResultSize);
-        ScoreDoc[] sortedDocs = searchPhaseController.sortDocs(true, results);
-        int accumulatedLength = Math.min(queryResultSize, getTotalQueryHits(results));
-        for (Suggest.Suggestion<?> suggestion : reducedSuggest(results)) {
-            int suggestionSize = suggestion.getEntries().get(0).getOptions().size();
-            accumulatedLength += suggestionSize;
-        }
-        assertThat(sortedDocs.length, equalTo(accumulatedLength));
-    }
-
-    public void testMerge() throws IOException {
-        List<CompletionSuggestion> suggestions = new ArrayList<>();
-        for (int i = 0; i < randomIntBetween(1, 5); i++) {
-            suggestions.add(new CompletionSuggestion(randomAsciiOfLength(randomIntBetween(1, 5)), randomIntBetween(1, 20)));
-        }
-        int nShards = randomIntBetween(1, 20);
-        int queryResultSize = randomBoolean() ? 0 : randomIntBetween(1, nShards * 2);
-        AtomicArray<QuerySearchResultProvider> queryResults = generateQueryResults(nShards, suggestions, queryResultSize);
-
-        // calculate offsets and score doc array
-        List<ScoreDoc> mergedScoreDocs = new ArrayList<>();
-        ScoreDoc[] mergedSearchDocs = getTopShardDocs(queryResults);
-        mergedScoreDocs.addAll(Arrays.asList(mergedSearchDocs));
-        Suggest mergedSuggest = reducedSuggest(queryResults);
-        for (Suggest.Suggestion<?> suggestion : mergedSuggest) {
-            if (suggestion instanceof CompletionSuggestion) {
-                CompletionSuggestion completionSuggestion = ((CompletionSuggestion) suggestion);
-                mergedScoreDocs.addAll(completionSuggestion.getOptions().stream()
-                    .map(CompletionSuggestion.Entry.Option::getDoc)
-                    .collect(Collectors.toList()));
-            }
-        }
-        ScoreDoc[] sortedDocs = mergedScoreDocs.toArray(new ScoreDoc[mergedScoreDocs.size()]);
-
-        InternalSearchResponse mergedResponse = searchPhaseController.merge(true, sortedDocs, queryResults,
-            generateFetchResults(nShards, mergedSearchDocs, mergedSuggest));
-        assertThat(mergedResponse.hits().getHits().length, equalTo(mergedSearchDocs.length));
-        Suggest suggestResult = mergedResponse.suggest();
-        for (Suggest.Suggestion<?> suggestion : mergedSuggest) {
-            assertThat(suggestion, instanceOf(CompletionSuggestion.class));
-            if (suggestion.getEntries().get(0).getOptions().size() > 0) {
-                CompletionSuggestion suggestionResult = suggestResult.getSuggestion(suggestion.getName());
-                assertNotNull(suggestionResult);
-                List<CompletionSuggestion.Entry.Option> options = suggestionResult.getEntries().get(0).getOptions();
-                assertThat(options.size(), equalTo(suggestion.getEntries().get(0).getOptions().size()));
-                for (CompletionSuggestion.Entry.Option option : options) {
-                    assertNotNull(option.getHit());
-                }
-            }
-        }
-    }
-
-    private AtomicArray<QuerySearchResultProvider> generateQueryResults(int nShards,
-                                                                        List<CompletionSuggestion> suggestions,
-                                                                        int searchHitsSize) {
-        AtomicArray<QuerySearchResultProvider> queryResults = new AtomicArray<>(nShards);
-        for (int shardIndex = 0; shardIndex < nShards; shardIndex++) {
-            QuerySearchResult querySearchResult = new QuerySearchResult(shardIndex,
-                new SearchShardTarget("", new Index("", ""), shardIndex));
-            TopDocs topDocs = new TopDocs(0, new ScoreDoc[0], 0);
-            if (searchHitsSize > 0) {
-                int nDocs = randomIntBetween(0, searchHitsSize);
-                ScoreDoc[] scoreDocs = new ScoreDoc[nDocs];
-                float maxScore = 0F;
-                for (int i = 0; i < nDocs; i++) {
-                    float score = Math.abs(randomFloat());
-                    scoreDocs[i] = new ScoreDoc(i, score);
-                    if (score > maxScore) {
-                        maxScore = score;
-                    }
-                }
-                topDocs = new TopDocs(scoreDocs.length, scoreDocs, maxScore);
-            }
-            List<CompletionSuggestion> shardSuggestion = new ArrayList<>();
-            for (CompletionSuggestion completionSuggestion : suggestions) {
-                CompletionSuggestion suggestion = new CompletionSuggestion(
-                    completionSuggestion.getName(), completionSuggestion.getSize());
-                final CompletionSuggestion.Entry completionEntry = new CompletionSuggestion.Entry(new Text(""), 0, 5);
-                suggestion.addTerm(completionEntry);
-                int optionSize = randomIntBetween(1, suggestion.getSize());
-                float maxScore = randomIntBetween(suggestion.getSize(), (int) Float.MAX_VALUE);
-                for (int i = 0; i < optionSize; i++) {
-                    completionEntry.addOption(new CompletionSuggestion.Entry.Option(i, new Text(""), maxScore,
-                        Collections.emptyMap()));
-                    float dec = randomIntBetween(0, optionSize);
-                    if (dec <= maxScore) {
-                        maxScore -= dec;
-                    }
-                }
-                suggestion.setShardIndex(shardIndex);
-                shardSuggestion.add(suggestion);
-            }
-            querySearchResult.topDocs(topDocs, null);
-            querySearchResult.size(searchHitsSize);
-            querySearchResult.suggest(new Suggest(new ArrayList<>(shardSuggestion)));
-            queryResults.set(shardIndex, querySearchResult);
-        }
-        return queryResults;
-    }
-
-    private int getTotalQueryHits(AtomicArray<QuerySearchResultProvider> results) {
-        int resultCount = 0;
-        for (AtomicArray.Entry<QuerySearchResultProvider> shardResult : results.asList()) {
-            resultCount += shardResult.value.queryResult().topDocs().totalHits;
-        }
-        return resultCount;
-    }
-
-    private Suggest reducedSuggest(AtomicArray<QuerySearchResultProvider> results) {
-        Map<String, List<Suggest.Suggestion<CompletionSuggestion.Entry>>> groupedSuggestion = new HashMap<>();
-        for (AtomicArray.Entry<QuerySearchResultProvider> entry : results.asList()) {
-            for (Suggest.Suggestion<?> suggestion : entry.value.queryResult().suggest()) {
-                List<Suggest.Suggestion<CompletionSuggestion.Entry>> suggests =
-                    groupedSuggestion.computeIfAbsent(suggestion.getName(), s -> new ArrayList<>());
-                suggests.add((Suggest.Suggestion<CompletionSuggestion.Entry>) suggestion);
-            }
-        }
-        return new Suggest(groupedSuggestion.values().stream().map(CompletionSuggestion::reduceTo)
-            .collect(Collectors.toList()));
-    }
-
-    private ScoreDoc[] getTopShardDocs(AtomicArray<QuerySearchResultProvider> results) throws IOException {
-        List<AtomicArray.Entry<QuerySearchResultProvider>> resultList = results.asList();
-        TopDocs[] shardTopDocs = new TopDocs[resultList.size()];
-        for (int i = 0; i < resultList.size(); i++) {
-            shardTopDocs[i] = resultList.get(i).value.queryResult().topDocs();
-        }
-        int topN = Math.min(results.get(0).queryResult().size(), getTotalQueryHits(results));
-        return TopDocs.merge(topN, shardTopDocs).scoreDocs;
-    }
-
-    private AtomicArray<FetchSearchResultProvider> generateFetchResults(int nShards, ScoreDoc[] mergedSearchDocs, Suggest mergedSuggest) {
-        AtomicArray<FetchSearchResultProvider> fetchResults = new AtomicArray<>(nShards);
-        for (int shardIndex = 0; shardIndex < nShards; shardIndex++) {
-            float maxScore = -1F;
-            SearchShardTarget shardTarget = new SearchShardTarget("", new Index("", ""), shardIndex);
-            FetchSearchResult fetchSearchResult = new FetchSearchResult(shardIndex, shardTarget);
-            List<InternalSearchHit> internalSearchHits = new ArrayList<>();
-            for (ScoreDoc scoreDoc : mergedSearchDocs) {
-                if (scoreDoc.shardIndex == shardIndex) {
-                    internalSearchHits.add(new InternalSearchHit(scoreDoc.doc, "", new Text(""), Collections.emptyMap()));
-                    if (scoreDoc.score > maxScore) {
-                        maxScore = scoreDoc.score;
-                    }
-                }
-            }
-            for (Suggest.Suggestion<?> suggestion : mergedSuggest) {
-                if (suggestion instanceof CompletionSuggestion) {
-                    for (CompletionSuggestion.Entry.Option option : ((CompletionSuggestion) suggestion).getOptions()) {
-                        ScoreDoc doc = option.getDoc();
-                        if (doc.shardIndex == shardIndex) {
-                            internalSearchHits.add(new InternalSearchHit(doc.doc, "", new Text(""), Collections.emptyMap()));
-                            if (doc.score > maxScore) {
-                                maxScore = doc.score;
-                            }
-                        }
-                    }
-                }
-            }
-            InternalSearchHit[] hits = internalSearchHits.toArray(new InternalSearchHit[internalSearchHits.size()]);
-            fetchSearchResult.hits(new InternalSearchHits(hits, hits.length, maxScore));
-            fetchResults.set(shardIndex, fetchSearchResult);
-        }
-        return fetchResults;
-    }
-}
diff --git a/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java b/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java
index f8da0312f8837..82e767ada8baf 100644
--- a/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java
+++ b/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java
@@ -26,16 +26,18 @@
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.termvectors.TermVectorsRequest;
 import org.elasticsearch.action.termvectors.TermVectorsResponse;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.termvectors.TermVectorsService;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.SearchPlugin;
+import org.elasticsearch.search.SearchExtBuilder;
 import org.elasticsearch.search.SearchHitField;
-import org.elasticsearch.search.SearchParseElement;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.elasticsearch.search.internal.InternalSearchHitField;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
@@ -48,21 +50,27 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 
 import static java.util.Collections.singletonList;
-import static java.util.Collections.singletonMap;
 import static org.elasticsearch.client.Requests.indexRequest;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
 import static org.hamcrest.CoreMatchers.equalTo;
 
-@ClusterScope(scope = Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 1)
+@ClusterScope(scope = Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 2)
 public class FetchSubPhasePluginIT extends ESIntegTestCase {
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
         return Collections.singletonList(FetchTermVectorsPlugin.class);
     }
 
+    @Override
+    protected Collection<Class<? extends Plugin>> transportClientPlugins() {
+        return nodePlugins();
+    }
+
+    @SuppressWarnings("unchecked")
     public void testPlugin() throws Exception {
         client().admin()
                 .indices()
@@ -84,12 +92,11 @@ public void testPlugin() throws Exception {
 
         client().admin().indices().prepareRefresh().execute().actionGet();
 
-        XContentBuilder extSource = jsonBuilder().startObject()
-                .field("term_vectors_fetch", "test")
-                .endObject();
-         SearchResponse response = client().prepareSearch().setSource(new SearchSourceBuilder().ext(extSource)).get();
+         SearchResponse response = client().prepareSearch().setSource(new SearchSourceBuilder()
+                 .ext(Collections.singletonList(new TermVectorsFetchBuilder("test")))).get();
         assertSearchResponse(response);
-        assertThat(((Map<String, Integer>) response.getHits().getAt(0).field("term_vectors_fetch").getValues().get(0)).get("i"), equalTo(2));
+        assertThat(((Map<String, Integer>) response.getHits().getAt(0).field("term_vectors_fetch").getValues().get(0)).get("i"),
+                equalTo(2));
         assertThat(((Map<String, Integer>) response.getHits().getAt(0).field("term_vectors_fetch").getValues().get(0)).get("am"),
                 equalTo(2));
         assertThat(((Map<String, Integer>) response.getHits().getAt(0).field("term_vectors_fetch").getValues().get(0)).get("sam"),
@@ -101,46 +108,35 @@ public static class FetchTermVectorsPlugin extends Plugin implements SearchPlugi
         public List<FetchSubPhase> getFetchSubPhases(FetchPhaseConstructionContext context) {
             return singletonList(new TermVectorsFetchSubPhase());
         }
-    }
-
-    public static final class TermVectorsFetchSubPhase implements FetchSubPhase {
-
-        public static final ContextFactory<TermVectorsFetchContext> CONTEXT_FACTORY = new ContextFactory<TermVectorsFetchContext>() {
-
-            @Override
-            public String getName() {
-                return NAMES[0];
-            }
-
-            @Override
-            public TermVectorsFetchContext newContextInstance() {
-                return new TermVectorsFetchContext();
-            }
-        };
-
-        public static final String[] NAMES = {"term_vectors_fetch"};
 
         @Override
-        public Map<String, ? extends SearchParseElement> parseElements() {
-            return singletonMap("term_vectors_fetch", new TermVectorsFetchParseElement());
+        public List<SearchExtSpec<?>> getSearchExts() {
+            return Collections.singletonList(new SearchExtSpec<>(TermVectorsFetchSubPhase.NAME,
+                    TermVectorsFetchBuilder::new, TermVectorsFetchBuilder::fromXContent));
         }
+    }
+
+    private static final class TermVectorsFetchSubPhase implements FetchSubPhase {
+        private static final String NAME = "term_vectors_fetch";
 
         @Override
         public void hitExecute(SearchContext context, HitContext hitContext) {
-            if (context.getFetchSubPhaseContext(CONTEXT_FACTORY).hitExecutionNeeded() == false) {
+            TermVectorsFetchBuilder fetchSubPhaseBuilder = (TermVectorsFetchBuilder)context.getSearchExt(NAME);
+            if (fetchSubPhaseBuilder == null) {
                 return;
             }
-            String field = context.getFetchSubPhaseContext(CONTEXT_FACTORY).getField();
-
+            String field = fetchSubPhaseBuilder.getField();
             if (hitContext.hit().fieldsOrNull() == null) {
                 hitContext.hit().fields(new HashMap<>());
             }
-            SearchHitField hitField = hitContext.hit().fields().get(NAMES[0]);
+            SearchHitField hitField = hitContext.hit().fields().get(NAME);
             if (hitField == null) {
-                hitField = new InternalSearchHitField(NAMES[0], new ArrayList<>(1));
-                hitContext.hit().fields().put(NAMES[0], hitField);
+                hitField = new SearchHitField(NAME, new ArrayList<>(1));
+                hitContext.hit().fields().put(NAME, hitField);
             }
-            TermVectorsResponse termVector = TermVectorsService.getTermVectors(context.indexShard(), new TermVectorsRequest(context.indexShard().shardId().getIndex().getName(), hitContext.hit().type(), hitContext.hit().id()));
+            TermVectorsRequest termVectorsRequest = new TermVectorsRequest(context.indexShard().shardId().getIndex().getName(),
+                    hitContext.hit().type(), hitContext.hit().id());
+            TermVectorsResponse termVector = TermVectorsService.getTermVectors(context.indexShard(), termVectorsRequest);
             try {
                 Map<String, Integer> tv = new HashMap<>();
                 TermsEnum terms = termVector.getFields().terms(field).iterator();
@@ -155,36 +151,65 @@ public void hitExecute(SearchContext context, HitContext hitContext) {
         }
     }
 
-    public static class TermVectorsFetchParseElement implements SearchParseElement {
-
-        @Override
-        public void parse(XContentParser parser, SearchContext context) throws Exception {
-            TermVectorsFetchContext fetchSubPhaseContext = context.getFetchSubPhaseContext(TermVectorsFetchSubPhase.CONTEXT_FACTORY);
-            // this is to make sure that the SubFetchPhase knows it should execute
-            fetchSubPhaseContext.setHitExecutionNeeded(true);
+    private static final class TermVectorsFetchBuilder extends SearchExtBuilder {
+        public static TermVectorsFetchBuilder fromXContent(XContentParser parser) throws IOException {
+            String field;
             XContentParser.Token token = parser.currentToken();
             if (token == XContentParser.Token.VALUE_STRING) {
-                String fieldName = parser.text();
-                fetchSubPhaseContext.setField(fieldName);
+                field = parser.text();
             } else {
-                throw new IllegalStateException("Expected a VALUE_STRING but got " + token);
+                throw new ParsingException(parser.getTokenLocation(), "Expected a VALUE_STRING but got " + token);
+            }
+            if (field == null) {
+                throw new ParsingException(parser.getTokenLocation(), "no fields specified for " + TermVectorsFetchSubPhase.NAME);
             }
+            return new TermVectorsFetchBuilder(field);
         }
-    }
-
-    public static class TermVectorsFetchContext extends FetchSubPhaseContext {
 
-        private String field = null;
+        private final String field;
 
-        public TermVectorsFetchContext() {
+        private TermVectorsFetchBuilder(String field) {
+            this.field = field;
         }
 
-        public void setField(String field) {
-            this.field = field;
+        private TermVectorsFetchBuilder(StreamInput in) throws IOException {
+            this.field = in.readString();
         }
 
-        public String getField() {
+        private String getField() {
             return field;
         }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) {
+                return true;
+            }
+            if (o == null || getClass() != o.getClass()) {
+                return false;
+            }
+            TermVectorsFetchBuilder that = (TermVectorsFetchBuilder) o;
+            return Objects.equals(field, that.field);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(field);
+        }
+
+        @Override
+        public String getWriteableName() {
+            return TermVectorsFetchSubPhase.NAME;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeString(field);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            return builder.field(TermVectorsFetchSubPhase.NAME, field);
+        }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/fetch/subphase/FetchSourceSubPhaseTests.java b/core/src/test/java/org/elasticsearch/search/fetch/subphase/FetchSourceSubPhaseTests.java
index 43461929bd9e4..4e27c2a6d7cc6 100644
--- a/core/src/test/java/org/elasticsearch/search/fetch/subphase/FetchSourceSubPhaseTests.java
+++ b/core/src/test/java/org/elasticsearch/search/fetch/subphase/FetchSourceSubPhaseTests.java
@@ -23,8 +23,10 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.shard.IndexShard;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.search.fetch.FetchSubPhase;
-import org.elasticsearch.search.internal.InternalSearchHit;
+import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.lookup.SearchLookup;
 import org.elasticsearch.test.ESTestCase;
@@ -33,36 +35,10 @@
 import java.io.IOException;
 import java.util.Collections;
 
-public class FetchSourceSubPhaseTests extends ESTestCase {
-
-    static class FetchSourceSubPhaseTestSearchContext extends TestSearchContext {
-
-        FetchSourceContext context;
-        BytesReference source;
-
-        FetchSourceSubPhaseTestSearchContext(FetchSourceContext context, BytesReference source) {
-            super(null);
-            this.context = context;
-            this.source = source;
-        }
-
-        @Override
-        public boolean sourceRequested() {
-            return context != null && context.fetchSource();
-        }
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
-        @Override
-        public FetchSourceContext fetchSourceContext() {
-            return context;
-        }
-
-        @Override
-        public SearchLookup lookup() {
-            SearchLookup lookup = super.lookup();
-            lookup.source().setSource(source);
-            return lookup;
-        }
-    }
+public class FetchSourceSubPhaseTests extends ESTestCase {
 
     public void testFetchSource() throws IOException {
         XContentBuilder source = XContentFactory.jsonBuilder().startObject()
@@ -109,11 +85,14 @@ public void testSourceDisabled() throws IOException {
         hitContext = hitExecute(null, false, null, null);
         assertNull(hitContext.hit().sourceAsMap());
 
-        hitContext = hitExecute(null, true, "field1", null);
-        assertNull(hitContext.hit().sourceAsMap());
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> hitExecute(null, true, "field1", null));
+        assertEquals("unable to fetch fields from _source field: _source is disabled in the mappings " +
+                "for index [index]", exception.getMessage());
 
-        hitContext = hitExecuteMultiple(null, true, new String[]{"*"}, new String[]{"field2"});
-        assertNull(hitContext.hit().sourceAsMap());
+        exception = expectThrows(IllegalArgumentException.class,
+                () -> hitExecuteMultiple(null, true, new String[]{"*"}, new String[]{"field2"}));
+        assertEquals("unable to fetch fields from _source field: _source is disabled in the mappings " +
+                "for index [index]", exception.getMessage());
     }
 
     private FetchSubPhase.HitContext hitExecute(XContentBuilder source, boolean fetchSource, String include, String exclude) {
@@ -126,9 +105,45 @@ private FetchSubPhase.HitContext hitExecuteMultiple(XContentBuilder source, bool
         FetchSourceContext fetchSourceContext = new FetchSourceContext(fetchSource, includes, excludes);
         SearchContext searchContext = new FetchSourceSubPhaseTestSearchContext(fetchSourceContext, source == null ? null : source.bytes());
         FetchSubPhase.HitContext hitContext = new FetchSubPhase.HitContext();
-        hitContext.reset(new InternalSearchHit(1, null, null, null), null, 1, null);
+        hitContext.reset(new SearchHit(1, null, null, null), null, 1, null);
         FetchSourceSubPhase phase = new FetchSourceSubPhase();
         phase.hitExecute(searchContext, hitContext);
         return hitContext;
     }
+
+    private static class FetchSourceSubPhaseTestSearchContext extends TestSearchContext {
+        final FetchSourceContext context;
+        final BytesReference source;
+        final IndexShard indexShard;
+
+        FetchSourceSubPhaseTestSearchContext(FetchSourceContext context, BytesReference source) {
+            super(null);
+            this.context = context;
+            this.source = source;
+            this.indexShard = mock(IndexShard.class);
+            when(indexShard.shardId()).thenReturn(new ShardId("index", "index", 1));
+        }
+
+        @Override
+        public boolean sourceRequested() {
+            return context != null && context.fetchSource();
+        }
+
+        @Override
+        public FetchSourceContext fetchSourceContext() {
+            return context;
+        }
+
+        @Override
+        public SearchLookup lookup() {
+            SearchLookup lookup = new SearchLookup(this.mapperService(), this.fieldData(), null);
+            lookup.source().setSource(source);
+            return lookup;
+        }
+
+        @Override
+        public IndexShard indexShard() {
+            return indexShard;
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java b/core/src/test/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java
index 1e43ffe532e3d..e5bff8888ee5e 100644
--- a/core/src/test/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java
+++ b/core/src/test/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java
@@ -21,10 +21,13 @@
 
 import org.apache.lucene.search.join.ScoreMode;
 import org.apache.lucene.util.ArrayUtil;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.InnerHitBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
@@ -32,12 +35,11 @@
 import org.elasticsearch.script.MockScriptEngine;
 import org.elasticsearch.script.MockScriptPlugin;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
 import org.elasticsearch.search.sort.FieldSortBuilder;
-import org.elasticsearch.search.sort.SortBuilders;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.test.ESIntegTestCase;
 
@@ -45,7 +47,6 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.function.Function;
 
@@ -53,9 +54,7 @@
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.common.xcontent.support.XContentMapValues.extractValue;
 import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
-import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery;
-import static org.elasticsearch.index.query.QueryBuilders.hasChildQuery;
-import static org.elasticsearch.index.query.QueryBuilders.hasParentQuery;
+import static org.elasticsearch.index.query.QueryBuilders.indicesQuery;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
 import static org.elasticsearch.index.query.QueryBuilders.nestedQuery;
@@ -65,6 +64,7 @@
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHit;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasId;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -85,6 +85,13 @@ protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
         }
     }
 
+    @Override
+    public Settings indexSettings() {
+        return Settings.builder()
+            .put(super.indexSettings())
+            .build();
+    }
+
     public void testSimpleNested() throws Exception {
         assertAcked(prepareCreate("articles").addMapping("article", jsonBuilder().startObject().startObject("article")
                 .startObject("properties")
@@ -123,7 +130,7 @@ public void testSimpleNested() throws Exception {
 
         SearchResponse response = client().prepareSearch("articles")
                 .setQuery(nestedQuery("comments", matchQuery("comments.message", "fox"), ScoreMode.Avg)
-                        .innerHit(new InnerHitBuilder().setName("comment"))
+                        .innerHit(new InnerHitBuilder("comment"))
                 ).get();
         assertNoFailures(response);
         assertHitCount(response, 1);
@@ -141,7 +148,7 @@ public void testSimpleNested() throws Exception {
 
         response = client().prepareSearch("articles")
                 .setQuery(nestedQuery("comments", matchQuery("comments.message", "elephant"), ScoreMode.Avg)
-                        .innerHit(new InnerHitBuilder().setName("comment"))
+                        .innerHit(new InnerHitBuilder("comment"))
                 ).get();
         assertNoFailures(response);
         assertHitCount(response, 1);
@@ -167,9 +174,8 @@ public void testSimpleNested() throws Exception {
                                 .setExplain(true)
                                 .addDocValueField("comments.message")
                                 .addScriptField("script",
-                                        new Script("5", ScriptService.ScriptType.INLINE, MockScriptEngine.NAME, Collections.emptyMap()))
-                                .setSize(1)
-                )).get();
+                                        new Script(ScriptType.INLINE, MockScriptEngine.NAME, "5", Collections.emptyMap()))
+                                .setSize(1))).get();
         assertNoFailures(response);
         innerHits = response.getHits().getAt(0).getInnerHits().get("comments");
         assertThat(innerHits.getTotalHits(), equalTo(2L));
@@ -190,7 +196,9 @@ public void testRandomNested() throws Exception {
         int[] field2InnerObjects = new int[numDocs];
         for (int i = 0; i < numDocs; i++) {
             int numInnerObjects = field1InnerObjects[i] = scaledRandomIntBetween(1, numDocs);
-            XContentBuilder source = jsonBuilder().startObject().startArray("field1");
+            XContentBuilder source = jsonBuilder().startObject()
+                    .field("foo", i)
+                    .startArray("field1");
             for (int j = 0; j < numInnerObjects; j++) {
                 source.startObject().field("x", "y").endObject();
             }
@@ -200,20 +208,20 @@ public void testRandomNested() throws Exception {
                 source.startObject().field("x", "y").endObject();
             }
             source.endArray().endObject();
-            requestBuilders.add(client().prepareIndex("idx", "type", String.format(Locale.ENGLISH, "%03d", i)).setSource(source));
+            requestBuilders.add(client().prepareIndex("idx", "type", Integer.toString(i)).setSource(source));
         }
         indexRandom(true, requestBuilders);
 
         int size = randomIntBetween(0, numDocs);
         BoolQueryBuilder boolQuery = new BoolQueryBuilder();
-        boolQuery.should(nestedQuery("field1", matchAllQuery(), ScoreMode.Avg).innerHit(new InnerHitBuilder().setName("a").setSize(size)
+        boolQuery.should(nestedQuery("field1", matchAllQuery(), ScoreMode.Avg).innerHit(new InnerHitBuilder("a").setSize(size)
                 .addSort(new FieldSortBuilder("_doc").order(SortOrder.DESC))));
-        boolQuery.should(nestedQuery("field2", matchAllQuery(), ScoreMode.Avg).innerHit(new InnerHitBuilder().setName("b")
+        boolQuery.should(nestedQuery("field2", matchAllQuery(), ScoreMode.Avg).innerHit(new InnerHitBuilder("b")
                 .addSort(new FieldSortBuilder("_doc").order(SortOrder.DESC)).setSize(size)));
         SearchResponse searchResponse = client().prepareSearch("idx")
                 .setQuery(boolQuery)
                 .setSize(numDocs)
-                .addSort("_uid", SortOrder.ASC)
+                .addSort("foo", SortOrder.ASC)
                 .get();
 
         assertNoFailures(searchResponse);
@@ -242,260 +250,6 @@ public void testRandomNested() throws Exception {
         }
     }
 
-    public void testSimpleParentChild() throws Exception {
-        assertAcked(prepareCreate("articles")
-                .addMapping("article", "title", "type=text")
-                .addMapping("comment", "_parent", "type=article", "message", "type=text,fielddata=true")
-        );
-
-        List<IndexRequestBuilder> requests = new ArrayList<>();
-        requests.add(client().prepareIndex("articles", "article", "1").setSource("title", "quick brown fox"));
-        requests.add(client().prepareIndex("articles", "comment", "1").setParent("1").setSource("message", "fox eat quick"));
-        requests.add(client().prepareIndex("articles", "comment", "2").setParent("1").setSource("message", "fox ate rabbit x y z"));
-        requests.add(client().prepareIndex("articles", "comment", "3").setParent("1").setSource("message", "rabbit got away"));
-        requests.add(client().prepareIndex("articles", "article", "2").setSource("title", "big gray elephant"));
-        requests.add(client().prepareIndex("articles", "comment", "4").setParent("2").setSource("message", "elephant captured"));
-        requests.add(client().prepareIndex("articles", "comment", "5").setParent("2").setSource("message", "mice squashed by elephant x"));
-        requests.add(client().prepareIndex("articles", "comment", "6").setParent("2").setSource("message", "elephant scared by mice x y"));
-        indexRandom(true, requests);
-
-        SearchResponse response = client().prepareSearch("articles")
-                .setQuery(hasChildQuery("comment", matchQuery("message", "fox"), ScoreMode.None).innerHit(new InnerHitBuilder()))
-                .get();
-        assertNoFailures(response);
-        assertHitCount(response, 1);
-        assertSearchHit(response, 1, hasId("1"));
-        assertThat(response.getHits().getAt(0).getShard(), notNullValue());
-
-        assertThat(response.getHits().getAt(0).getInnerHits().size(), equalTo(1));
-        SearchHits innerHits = response.getHits().getAt(0).getInnerHits().get("comment");
-        assertThat(innerHits.totalHits(), equalTo(2L));
-
-        assertThat(innerHits.getAt(0).getId(), equalTo("1"));
-        assertThat(innerHits.getAt(0).type(), equalTo("comment"));
-        assertThat(innerHits.getAt(1).getId(), equalTo("2"));
-        assertThat(innerHits.getAt(1).type(), equalTo("comment"));
-
-        response = client().prepareSearch("articles")
-                .setQuery(hasChildQuery("comment", matchQuery("message", "elephant"), ScoreMode.None).innerHit(new InnerHitBuilder()))
-                .get();
-        assertNoFailures(response);
-        assertHitCount(response, 1);
-        assertSearchHit(response, 1, hasId("2"));
-
-        assertThat(response.getHits().getAt(0).getInnerHits().size(), equalTo(1));
-        innerHits = response.getHits().getAt(0).getInnerHits().get("comment");
-        assertThat(innerHits.totalHits(), equalTo(3L));
-
-        assertThat(innerHits.getAt(0).getId(), equalTo("4"));
-        assertThat(innerHits.getAt(0).type(), equalTo("comment"));
-        assertThat(innerHits.getAt(1).getId(), equalTo("5"));
-        assertThat(innerHits.getAt(1).type(), equalTo("comment"));
-        assertThat(innerHits.getAt(2).getId(), equalTo("6"));
-        assertThat(innerHits.getAt(2).type(), equalTo("comment"));
-
-        response = client().prepareSearch("articles")
-                .setQuery(
-                        hasChildQuery("comment", matchQuery("message", "fox"), ScoreMode.None).innerHit(
-                                new InnerHitBuilder()
-                                        .addDocValueField("message")
-                                        .setHighlightBuilder(new HighlightBuilder().field("message"))
-                                        .setExplain(true).setSize(1)
-                                        .addScriptField("script", new Script("5", ScriptService.ScriptType.INLINE,
-                                                MockScriptEngine.NAME, Collections.emptyMap()))
-                        )
-                ).get();
-        assertNoFailures(response);
-        innerHits = response.getHits().getAt(0).getInnerHits().get("comment");
-        assertThat(innerHits.getHits().length, equalTo(1));
-        assertThat(innerHits.getAt(0).getHighlightFields().get("message").getFragments()[0].string(), equalTo("<em>fox</em> eat quick"));
-        assertThat(innerHits.getAt(0).explanation().toString(), containsString("weight(message:fox"));
-        assertThat(innerHits.getAt(0).getFields().get("message").getValue().toString(), equalTo("eat"));
-        assertThat(innerHits.getAt(0).getFields().get("script").getValue().toString(), equalTo("5"));
-    }
-
-    public void testRandomParentChild() throws Exception {
-        assertAcked(prepareCreate("idx")
-                        .addMapping("parent")
-                        .addMapping("child1", "_parent", "type=parent")
-                        .addMapping("child2", "_parent", "type=parent")
-        );
-        int numDocs = scaledRandomIntBetween(5, 50);
-        List<IndexRequestBuilder> requestBuilders = new ArrayList<>();
-
-        int child1 = 0;
-        int child2 = 0;
-        int[] child1InnerObjects = new int[numDocs];
-        int[] child2InnerObjects = new int[numDocs];
-        for (int parent = 0; parent < numDocs; parent++) {
-            String parentId = String.format(Locale.ENGLISH, "%03d", parent);
-            requestBuilders.add(client().prepareIndex("idx", "parent", parentId).setSource("{}"));
-
-            int numChildDocs = child1InnerObjects[parent] = scaledRandomIntBetween(1, numDocs);
-            int limit = child1 + numChildDocs;
-            for (; child1 < limit; child1++) {
-                requestBuilders.add(client().prepareIndex("idx", "child1",
-                        String.format(Locale.ENGLISH, "%04d", child1)).setParent(parentId).setSource("{}"));
-            }
-            numChildDocs = child2InnerObjects[parent] = scaledRandomIntBetween(1, numDocs);
-            limit = child2 + numChildDocs;
-            for (; child2 < limit; child2++) {
-                requestBuilders.add(client().prepareIndex("idx", "child2",
-                        String.format(Locale.ENGLISH, "%04d", child2)).setParent(parentId).setSource("{}"));
-            }
-        }
-        indexRandom(true, requestBuilders);
-
-        int size = randomIntBetween(0, numDocs);
-        BoolQueryBuilder boolQuery = new BoolQueryBuilder();
-        boolQuery.should(constantScoreQuery(hasChildQuery("child1", matchAllQuery(), ScoreMode.None)
-                .innerHit(new InnerHitBuilder().setName("a").addSort(new FieldSortBuilder("_uid").order(SortOrder.ASC)).setSize(size))));
-        boolQuery.should(constantScoreQuery(hasChildQuery("child2", matchAllQuery(), ScoreMode.None)
-                .innerHit(new InnerHitBuilder().setName("b").addSort(new FieldSortBuilder("_uid").order(SortOrder.ASC)).setSize(size))));
-        SearchResponse searchResponse = client().prepareSearch("idx")
-                .setSize(numDocs)
-                .setTypes("parent")
-                .addSort("_uid", SortOrder.ASC)
-                .setQuery(boolQuery)
-                .get();
-
-        assertNoFailures(searchResponse);
-        assertHitCount(searchResponse, numDocs);
-        assertThat(searchResponse.getHits().getHits().length, equalTo(numDocs));
-
-        int offset1 = 0;
-        int offset2 = 0;
-        for (int parent = 0; parent < numDocs; parent++) {
-            SearchHit searchHit = searchResponse.getHits().getAt(parent);
-            assertThat(searchHit.getType(), equalTo("parent"));
-            assertThat(searchHit.getId(), equalTo(String.format(Locale.ENGLISH, "%03d", parent)));
-            assertThat(searchHit.getShard(), notNullValue());
-
-            SearchHits inner = searchHit.getInnerHits().get("a");
-            assertThat(inner.totalHits(), equalTo((long) child1InnerObjects[parent]));
-            for (int child = 0; child < child1InnerObjects[parent] && child < size; child++) {
-                SearchHit innerHit =  inner.getAt(child);
-                assertThat(innerHit.getType(), equalTo("child1"));
-                String childId = String.format(Locale.ENGLISH, "%04d", offset1 + child);
-                assertThat(innerHit.getId(), equalTo(childId));
-                assertThat(innerHit.getNestedIdentity(), nullValue());
-            }
-            offset1 += child1InnerObjects[parent];
-
-            inner = searchHit.getInnerHits().get("b");
-            assertThat(inner.totalHits(), equalTo((long) child2InnerObjects[parent]));
-            for (int child = 0; child < child2InnerObjects[parent] && child < size; child++) {
-                SearchHit innerHit = inner.getAt(child);
-                assertThat(innerHit.getType(), equalTo("child2"));
-                String childId = String.format(Locale.ENGLISH, "%04d", offset2 + child);
-                assertThat(innerHit.getId(), equalTo(childId));
-                assertThat(innerHit.getNestedIdentity(), nullValue());
-            }
-            offset2 += child2InnerObjects[parent];
-        }
-    }
-
-    public void testInnerHitsOnHasParent() throws Exception {
-        assertAcked(prepareCreate("stack")
-                        .addMapping("question", "body", "type=text")
-                        .addMapping("answer", "_parent", "type=question", "body", "type=text")
-        );
-        List<IndexRequestBuilder> requests = new ArrayList<>();
-        requests.add(client().prepareIndex("stack", "question", "1").setSource("body", "I'm using HTTPS + Basic authentication "
-                + "to protect a resource. How can I throttle authentication attempts to protect against brute force attacks?"));
-        requests.add(client().prepareIndex("stack", "answer", "1").setParent("1").setSource("body",
-                "install fail2ban and enable rules for apache"));
-        requests.add(client().prepareIndex("stack", "question", "2").setSource("body",
-                "I have firewall rules set up and also denyhosts installed.\\ndo I also need to install fail2ban?"));
-        requests.add(client().prepareIndex("stack", "answer", "2").setParent("2").setSource("body",
-                "Denyhosts protects only ssh; Fail2Ban protects all daemons."));
-        indexRandom(true, requests);
-
-        SearchResponse response = client().prepareSearch("stack")
-                .setTypes("answer")
-                .addSort("_uid", SortOrder.ASC)
-                .setQuery(
-                        boolQuery()
-                                .must(matchQuery("body", "fail2ban"))
-                                .must(hasParentQuery("question", matchAllQuery(), false).innerHit(new InnerHitBuilder()))
-                ).get();
-        assertNoFailures(response);
-        assertHitCount(response, 2);
-
-        SearchHit searchHit = response.getHits().getAt(0);
-        assertThat(searchHit.getId(), equalTo("1"));
-        assertThat(searchHit.getType(), equalTo("answer"));
-        assertThat(searchHit.getInnerHits().get("question").getTotalHits(), equalTo(1L));
-        assertThat(searchHit.getInnerHits().get("question").getAt(0).getType(), equalTo("question"));
-        assertThat(searchHit.getInnerHits().get("question").getAt(0).id(), equalTo("1"));
-
-        searchHit = response.getHits().getAt(1);
-        assertThat(searchHit.getId(), equalTo("2"));
-        assertThat(searchHit.getType(), equalTo("answer"));
-        assertThat(searchHit.getInnerHits().get("question").getTotalHits(), equalTo(1L));
-        assertThat(searchHit.getInnerHits().get("question").getAt(0).getType(), equalTo("question"));
-        assertThat(searchHit.getInnerHits().get("question").getAt(0).id(), equalTo("2"));
-    }
-
-    public void testParentChildMultipleLayers() throws Exception {
-        assertAcked(prepareCreate("articles")
-                        .addMapping("article", "title", "type=text")
-                        .addMapping("comment", "_parent", "type=article", "message", "type=text")
-                        .addMapping("remark", "_parent", "type=comment", "message", "type=text")
-        );
-
-        List<IndexRequestBuilder> requests = new ArrayList<>();
-        requests.add(client().prepareIndex("articles", "article", "1").setSource("title", "quick brown fox"));
-        requests.add(client().prepareIndex("articles", "comment", "1").setParent("1").setSource("message", "fox eat quick"));
-        requests.add(client().prepareIndex("articles", "remark", "1").setParent("1").setRouting("1").setSource("message", "good"));
-        requests.add(client().prepareIndex("articles", "article", "2").setSource("title", "big gray elephant"));
-        requests.add(client().prepareIndex("articles", "comment", "2").setParent("2").setSource("message", "elephant captured"));
-        requests.add(client().prepareIndex("articles", "remark", "2").setParent("2").setRouting("2").setSource("message", "bad"));
-        indexRandom(true, requests);
-
-        SearchResponse response = client().prepareSearch("articles")
-                .setQuery(hasChildQuery("comment",
-                            hasChildQuery("remark", matchQuery("message", "good"), ScoreMode.None).innerHit(new InnerHitBuilder()),
-                        ScoreMode.None).innerHit(new InnerHitBuilder()))
-                .get();
-
-        assertNoFailures(response);
-        assertHitCount(response, 1);
-        assertSearchHit(response, 1, hasId("1"));
-
-        assertThat(response.getHits().getAt(0).getInnerHits().size(), equalTo(1));
-        SearchHits innerHits = response.getHits().getAt(0).getInnerHits().get("comment");
-        assertThat(innerHits.totalHits(), equalTo(1L));
-        assertThat(innerHits.getAt(0).getId(), equalTo("1"));
-        assertThat(innerHits.getAt(0).type(), equalTo("comment"));
-
-        innerHits = innerHits.getAt(0).getInnerHits().get("remark");
-        assertThat(innerHits.totalHits(), equalTo(1L));
-        assertThat(innerHits.getAt(0).getId(), equalTo("1"));
-        assertThat(innerHits.getAt(0).type(), equalTo("remark"));
-
-        response = client().prepareSearch("articles")
-                .setQuery(hasChildQuery("comment",
-                        hasChildQuery("remark", matchQuery("message", "bad"), ScoreMode.None).innerHit(new InnerHitBuilder()),
-                        ScoreMode.None).innerHit(new InnerHitBuilder()))
-                .get();
-
-        assertNoFailures(response);
-        assertHitCount(response, 1);
-        assertSearchHit(response, 1, hasId("2"));
-
-        assertThat(response.getHits().getAt(0).getInnerHits().size(), equalTo(1));
-        innerHits = response.getHits().getAt(0).getInnerHits().get("comment");
-        assertThat(innerHits.totalHits(), equalTo(1L));
-        assertThat(innerHits.getAt(0).getId(), equalTo("2"));
-        assertThat(innerHits.getAt(0).type(), equalTo("comment"));
-
-        innerHits = innerHits.getAt(0).getInnerHits().get("remark");
-        assertThat(innerHits.totalHits(), equalTo(1L));
-        assertThat(innerHits.getAt(0).getId(), equalTo("2"));
-        assertThat(innerHits.getAt(0).type(), equalTo("remark"));
-    }
-
     public void testNestedMultipleLayers() throws Exception {
         assertAcked(prepareCreate("articles").addMapping("article", jsonBuilder().startObject()
                 .startObject("article").startObject("properties")
@@ -543,7 +297,7 @@ public void testNestedMultipleLayers() throws Exception {
                 .setQuery(
                         nestedQuery("comments",
                                 nestedQuery("comments.remarks", matchQuery("comments.remarks.message", "good"), ScoreMode.Avg)
-                                        .innerHit(new InnerHitBuilder().setName("remark")),
+                                        .innerHit(new InnerHitBuilder("remark")),
                                 ScoreMode.Avg).innerHit(new InnerHitBuilder())
                 ).get();
         assertNoFailures(response);
@@ -586,7 +340,7 @@ public void testNestedMultipleLayers() throws Exception {
                 .setQuery(
                         nestedQuery("comments",
                                 nestedQuery("comments.remarks", matchQuery("comments.remarks.message", "bad"), ScoreMode.Avg)
-                                        .innerHit(new InnerHitBuilder().setName("remark")),
+                                        .innerHit(new InnerHitBuilder("remark")),
                                 ScoreMode.Avg).innerHit(new InnerHitBuilder())
                 ).get();
         assertNoFailures(response);
@@ -621,7 +375,8 @@ public void testNestedDefinedAsObject() throws Exception {
         indexRandom(true, requests);
 
         SearchResponse response = client().prepareSearch("articles")
-                .setQuery(nestedQuery("comments", matchQuery("comments.message", "fox"), ScoreMode.Avg).innerHit(new InnerHitBuilder()))
+                .setQuery(nestedQuery("comments", matchQuery("comments.message", "fox"), ScoreMode.Avg)
+                        .innerHit(new InnerHitBuilder()))
                 .get();
         assertNoFailures(response);
         assertHitCount(response, 1);
@@ -636,6 +391,9 @@ public void testNestedDefinedAsObject() throws Exception {
 
     public void testInnerHitsWithObjectFieldThatHasANestedField() throws Exception {
         assertAcked(prepareCreate("articles")
+                        // number_of_shards = 1, because then we catch the expected exception in the same way.
+                        // (See expectThrows(...) below)
+                        .setSettings(Settings.builder().put("index.number_of_shards", 1))
                         .addMapping("article", jsonBuilder().startObject()
                                         .startObject("properties")
                                             .startObject("comments")
@@ -652,32 +410,54 @@ public void testInnerHitsWithObjectFieldThatHasANestedField() throws Exception {
         List<IndexRequestBuilder> requests = new ArrayList<>();
         requests.add(client().prepareIndex("articles", "article", "1").setSource(jsonBuilder().startObject()
                 .field("title", "quick brown fox")
-                .startObject("comments")
-                .startArray("messages")
-                    .startObject().field("message", "fox eat quick").endObject()
-                    .startObject().field("message", "bear eat quick").endObject()
+                .startArray("comments")
+                    .startObject()
+                        .startArray("messages")
+                            .startObject().field("message", "fox eat quick").endObject()
+                            .startObject().field("message", "bear eat quick").endObject()
+                        .endArray()
+                    .endObject()
+                    .startObject()
+                        .startArray("messages")
+                            .startObject().field("message", "no fox").endObject()
+                        .endArray()
+                    .endObject()
                 .endArray()
-                .endObject()
                 .endObject()));
         indexRandom(true, requests);
 
+        Exception e = expectThrows(Exception.class, () -> client().prepareSearch("articles").setQuery(nestedQuery("comments.messages",
+            matchQuery("comments.messages.message", "fox"), ScoreMode.Avg).innerHit(new InnerHitBuilder())).get());
+        assertEquals("Cannot execute inner hits. One or more parent object fields of nested field [comments.messages] are " +
+            "not nested. All parent fields need to be nested fields too", e.getCause().getCause().getMessage());
+
+        e = expectThrows(Exception.class, () -> client().prepareSearch("articles").setQuery(nestedQuery("comments.messages",
+            matchQuery("comments.messages.message", "fox"), ScoreMode.Avg).innerHit(new InnerHitBuilder()
+            .setFetchSourceContext(new FetchSourceContext(true)))).get());
+        assertEquals("Cannot execute inner hits. One or more parent object fields of nested field [comments.messages] are " +
+            "not nested. All parent fields need to be nested fields too", e.getCause().getCause().getMessage());
+
         SearchResponse response = client().prepareSearch("articles")
                 .setQuery(nestedQuery("comments.messages", matchQuery("comments.messages.message", "fox"), ScoreMode.Avg)
-                        .innerHit(new InnerHitBuilder())).get();
+                        .innerHit(new InnerHitBuilder().setFetchSourceContext(new FetchSourceContext(false)))).get();
         assertNoFailures(response);
         assertHitCount(response, 1);
-        SearchHit hit = response.getHits().getAt(0); 
+        SearchHit hit = response.getHits().getAt(0);
         assertThat(hit.id(), equalTo("1"));
         SearchHits messages = hit.getInnerHits().get("comments.messages");
-        assertThat(messages.getTotalHits(), equalTo(1L));
+        assertThat(messages.getTotalHits(), equalTo(2L));
         assertThat(messages.getAt(0).id(), equalTo("1"));
         assertThat(messages.getAt(0).getNestedIdentity().getField().string(), equalTo("comments.messages"));
-        assertThat(messages.getAt(0).getNestedIdentity().getOffset(), equalTo(0));
+        assertThat(messages.getAt(0).getNestedIdentity().getOffset(), equalTo(2));
         assertThat(messages.getAt(0).getNestedIdentity().getChild(), nullValue());
+        assertThat(messages.getAt(1).getId(), equalTo("1"));
+        assertThat(messages.getAt(1).getNestedIdentity().getField().string(), equalTo("comments.messages"));
+        assertThat(messages.getAt(1).getNestedIdentity().getOffset(), equalTo(0));
+        assertThat(messages.getAt(1).getNestedIdentity().getChild(), nullValue());
 
         response = client().prepareSearch("articles")
                 .setQuery(nestedQuery("comments.messages", matchQuery("comments.messages.message", "bear"), ScoreMode.Avg)
-                        .innerHit(new InnerHitBuilder())).get();
+                        .innerHit(new InnerHitBuilder().setFetchSourceContext(new FetchSourceContext(false)))).get();
         assertNoFailures(response);
         assertHitCount(response, 1);
         hit = response.getHits().getAt(0);
@@ -698,7 +478,7 @@ public void testInnerHitsWithObjectFieldThatHasANestedField() throws Exception {
         indexRandom(true, requests);
         response = client().prepareSearch("articles")
                 .setQuery(nestedQuery("comments.messages", matchQuery("comments.messages.message", "fox"), ScoreMode.Avg)
-                        .innerHit(new InnerHitBuilder())).get();
+                        .innerHit(new InnerHitBuilder().setFetchSourceContext(new FetchSourceContext(false)))).get();
         assertNoFailures(response);
         assertHitCount(response, 1);
         hit = response.getHits().getAt(0);;
@@ -711,81 +491,6 @@ public void testInnerHitsWithObjectFieldThatHasANestedField() throws Exception {
         assertThat(messages.getAt(0).getNestedIdentity().getChild(), nullValue());
     }
 
-    public void testRoyals() throws Exception {
-        assertAcked(
-                prepareCreate("royals")
-                        .addMapping("king")
-                        .addMapping("prince", "_parent", "type=king")
-                        .addMapping("duke", "_parent", "type=prince")
-                        .addMapping("earl", "_parent", "type=duke")
-                        .addMapping("baron", "_parent", "type=earl")
-        );
-
-        List<IndexRequestBuilder> requests = new ArrayList<>();
-        requests.add(client().prepareIndex("royals", "king", "king").setSource("{}"));
-        requests.add(client().prepareIndex("royals", "prince", "prince").setParent("king").setSource("{}"));
-        requests.add(client().prepareIndex("royals", "duke", "duke").setParent("prince").setRouting("king").setSource("{}"));
-        requests.add(client().prepareIndex("royals", "earl", "earl1").setParent("duke").setRouting("king").setSource("{}"));
-        requests.add(client().prepareIndex("royals", "earl", "earl2").setParent("duke").setRouting("king").setSource("{}"));
-        requests.add(client().prepareIndex("royals", "earl", "earl3").setParent("duke").setRouting("king").setSource("{}"));
-        requests.add(client().prepareIndex("royals", "earl", "earl4").setParent("duke").setRouting("king").setSource("{}"));
-        requests.add(client().prepareIndex("royals", "baron", "baron1").setParent("earl1").setRouting("king").setSource("{}"));
-        requests.add(client().prepareIndex("royals", "baron", "baron2").setParent("earl2").setRouting("king").setSource("{}"));
-        requests.add(client().prepareIndex("royals", "baron", "baron3").setParent("earl3").setRouting("king").setSource("{}"));
-        requests.add(client().prepareIndex("royals", "baron", "baron4").setParent("earl4").setRouting("king").setSource("{}"));
-        indexRandom(true, requests);
-
-        SearchResponse response = client().prepareSearch("royals")
-                .setTypes("duke")
-                .setQuery(boolQuery()
-                        .filter(hasParentQuery("prince",
-                                hasParentQuery("king", matchAllQuery(), false).innerHit(new InnerHitBuilder().setName("kings")),
-                                false).innerHit(new InnerHitBuilder().setName("princes"))
-                        )
-                        .filter(hasChildQuery("earl",
-                                hasChildQuery("baron", matchAllQuery(), ScoreMode.None).innerHit(new InnerHitBuilder().setName("barons")),
-                                ScoreMode.None).innerHit(new InnerHitBuilder()
-                                    .addSort(SortBuilders.fieldSort("_uid").order(SortOrder.ASC))
-                                    .setName("earls")
-                                    .setSize(4))
-                        )
-                )
-                .get();
-        assertHitCount(response, 1);
-        assertThat(response.getHits().getAt(0).getId(), equalTo("duke"));
-
-        SearchHits innerHits = response.getHits().getAt(0).getInnerHits().get("earls");
-        assertThat(innerHits.getTotalHits(), equalTo(4L));
-        assertThat(innerHits.getAt(0).getId(), equalTo("earl1"));
-        assertThat(innerHits.getAt(1).getId(), equalTo("earl2"));
-        assertThat(innerHits.getAt(2).getId(), equalTo("earl3"));
-        assertThat(innerHits.getAt(3).getId(), equalTo("earl4"));
-
-        SearchHits innerInnerHits = innerHits.getAt(0).getInnerHits().get("barons");
-        assertThat(innerInnerHits.totalHits(), equalTo(1L));
-        assertThat(innerInnerHits.getAt(0).getId(), equalTo("baron1"));
-
-        innerInnerHits = innerHits.getAt(1).getInnerHits().get("barons");
-        assertThat(innerInnerHits.totalHits(), equalTo(1L));
-        assertThat(innerInnerHits.getAt(0).getId(), equalTo("baron2"));
-
-        innerInnerHits = innerHits.getAt(2).getInnerHits().get("barons");
-        assertThat(innerInnerHits.totalHits(), equalTo(1L));
-        assertThat(innerInnerHits.getAt(0).getId(), equalTo("baron3"));
-
-        innerInnerHits = innerHits.getAt(3).getInnerHits().get("barons");
-        assertThat(innerInnerHits.totalHits(), equalTo(1L));
-        assertThat(innerInnerHits.getAt(0).getId(), equalTo("baron4"));
-
-        innerHits = response.getHits().getAt(0).getInnerHits().get("princes");
-        assertThat(innerHits.getTotalHits(), equalTo(1L));
-        assertThat(innerHits.getAt(0).getId(), equalTo("prince"));
-
-        innerInnerHits = innerHits.getAt(0).getInnerHits().get("kings");
-        assertThat(innerInnerHits.totalHits(), equalTo(1L));
-        assertThat(innerInnerHits.getAt(0).getId(), equalTo("king"));
-    }
-
     public void testMatchesQueriesNestedInnerHits() throws Exception {
         XContentBuilder builder = jsonBuilder().startObject()
                 .startObject("type1")
@@ -890,78 +595,23 @@ public void testMatchesQueriesNestedInnerHits() throws Exception {
         }
     }
 
-    public void testMatchesQueriesParentChildInnerHits() throws Exception {
-        assertAcked(prepareCreate("index").addMapping("child", "_parent", "type=parent"));
-        List<IndexRequestBuilder> requests = new ArrayList<>();
-        requests.add(client().prepareIndex("index", "parent", "1").setSource("{}"));
-        requests.add(client().prepareIndex("index", "child", "1").setParent("1").setSource("field", "value1"));
-        requests.add(client().prepareIndex("index", "child", "2").setParent("1").setSource("field", "value2"));
-        requests.add(client().prepareIndex("index", "parent", "2").setSource("{}"));
-        requests.add(client().prepareIndex("index", "child", "3").setParent("2").setSource("field", "value1"));
-        indexRandom(true, requests);
-
-        SearchResponse response = client().prepareSearch("index")
-                .setQuery(hasChildQuery("child", matchQuery("field", "value1").queryName("_name1"), ScoreMode.None)
-                        .innerHit(new InnerHitBuilder()))
-                .addSort("_uid", SortOrder.ASC)
-                .get();
-        assertHitCount(response, 2);
-        assertThat(response.getHits().getAt(0).id(), equalTo("1"));
-        assertThat(response.getHits().getAt(0).getInnerHits().get("child").getTotalHits(), equalTo(1L));
-        assertThat(response.getHits().getAt(0).getInnerHits().get("child").getAt(0).getMatchedQueries().length, equalTo(1));
-        assertThat(response.getHits().getAt(0).getInnerHits().get("child").getAt(0).getMatchedQueries()[0], equalTo("_name1"));
-
-        assertThat(response.getHits().getAt(1).id(), equalTo("2"));
-        assertThat(response.getHits().getAt(1).getInnerHits().get("child").getTotalHits(), equalTo(1L));
-        assertThat(response.getHits().getAt(1).getInnerHits().get("child").getAt(0).getMatchedQueries().length, equalTo(1));
-        assertThat(response.getHits().getAt(1).getInnerHits().get("child").getAt(0).getMatchedQueries()[0], equalTo("_name1"));
-
-        QueryBuilder query = hasChildQuery("child", matchQuery("field", "value2").queryName("_name2"), ScoreMode.None)
-                .innerHit(new InnerHitBuilder());
-        response = client().prepareSearch("index")
-                .setQuery(query)
-                .addSort("_uid", SortOrder.ASC)
-                .get();
-        assertHitCount(response, 1);
-        assertThat(response.getHits().getAt(0).id(), equalTo("1"));
-        assertThat(response.getHits().getAt(0).getInnerHits().get("child").getTotalHits(), equalTo(1L));
-        assertThat(response.getHits().getAt(0).getInnerHits().get("child").getAt(0).getMatchedQueries().length, equalTo(1));
-        assertThat(response.getHits().getAt(0).getInnerHits().get("child").getAt(0).getMatchedQueries()[0], equalTo("_name2"));
-    }
-
     public void testDontExplode() throws Exception {
-        assertAcked(prepareCreate("index1").addMapping("child", "_parent", "type=parent"));
-        List<IndexRequestBuilder> requests = new ArrayList<>();
-        requests.add(client().prepareIndex("index1", "parent", "1").setSource("{}"));
-        requests.add(client().prepareIndex("index1", "child", "1").setParent("1").setSource("field", "value1"));
-        indexRandom(true, requests);
-
-        QueryBuilder query = hasChildQuery("child", matchQuery("field", "value1"), ScoreMode.None)
-                .innerHit(new InnerHitBuilder().setSize(ArrayUtil.MAX_ARRAY_LENGTH - 1));
-        SearchResponse response = client().prepareSearch("index1")
-                .setQuery(query)
-                .addSort("_uid", SortOrder.ASC)
-                .get();
-        assertNoFailures(response);
-        assertHitCount(response, 1);
-
-        assertAcked(prepareCreate("index2").addMapping("type", "nested", "type=nested"));
-        client().prepareIndex("index2", "type", "1").setSource(jsonBuilder().startObject()
-                .startArray("nested")
-                .startObject()
-                .field("field", "value1")
-                .endObject()
-                .endArray()
-                .endObject())
-        .setRefreshPolicy(IMMEDIATE)
-        .get();
-
-        query = nestedQuery("nested", matchQuery("nested.field", "value1"), ScoreMode.Avg)
-                .innerHit(new InnerHitBuilder().setSize(ArrayUtil.MAX_ARRAY_LENGTH - 1));
-        response = client().prepareSearch("index2")
-                .setQuery(query)
-                .addSort("_uid", SortOrder.ASC)
-                .get();
+        assertAcked(prepareCreate("index").addMapping("type", "nested", "type=nested"));
+        client().prepareIndex("index", "type", "1").setSource(jsonBuilder().startObject()
+            .startArray("nested")
+            .startObject()
+            .field("field", "value1")
+            .endObject()
+            .endArray()
+            .endObject())
+            .setRefreshPolicy(IMMEDIATE)
+            .get();
+
+        QueryBuilder query = nestedQuery("nested", matchQuery("nested.field", "value1"), ScoreMode.Avg)
+            .innerHit(new InnerHitBuilder().setSize(ArrayUtil.MAX_ARRAY_LENGTH - 1));
+        SearchResponse response = client().prepareSearch("index")
+            .setQuery(query)
+            .get();
         assertNoFailures(response);
         assertHitCount(response, 1);
     }
@@ -982,7 +632,8 @@ public void testNestedSourceFiltering() throws Exception {
         // other features (like in the query dsl or aggs) in order for consistency:
         SearchResponse response = client().prepareSearch()
                 .setQuery(nestedQuery("comments", matchQuery("comments.message", "fox"), ScoreMode.None)
-                .innerHit(new InnerHitBuilder().setFetchSourceContext(new FetchSourceContext("comments.message"))))
+                .innerHit(new InnerHitBuilder().setFetchSourceContext(new FetchSourceContext(true,
+                    new String[]{"comments.message"}, null))))
                 .get();
         assertNoFailures(response);
         assertHitCount(response, 1);
@@ -994,4 +645,50 @@ public void testNestedSourceFiltering() throws Exception {
                 equalTo("fox ate rabbit x y z"));
     }
 
+    public void testInnerHitsWithIgnoreUnmapped() throws Exception {
+        assertAcked(prepareCreate("index1")
+            .addMapping("parent_type", "nested_type", "type=nested")
+            .addMapping("child_type", "_parent", "type=parent_type")
+        );
+        createIndex("index2");
+        client().prepareIndex("index1", "parent_type", "1").setSource("nested_type", Collections.singletonMap("key", "value")).get();
+        client().prepareIndex("index1", "child_type", "2").setParent("1").setSource("{}", XContentType.JSON).get();
+        client().prepareIndex("index2", "type", "3").setSource("key", "value").get();
+        refresh();
+
+        SearchResponse response = client().prepareSearch("index1", "index2")
+            .setQuery(boolQuery()
+                .should(nestedQuery("nested_type", matchAllQuery(), ScoreMode.None).ignoreUnmapped(true)
+                        .innerHit(new InnerHitBuilder().setIgnoreUnmapped(true)))
+                .should(termQuery("key", "value"))
+            )
+            .get();
+        assertNoFailures(response);
+        assertHitCount(response, 2);
+        assertSearchHits(response, "1", "3");
+    }
+
+    public void testInnerHitsInsideIndicesQuery() throws Exception {
+        assertAcked(prepareCreate("index1").addMapping("message", "comments", "type=nested"));
+        client().prepareIndex("index1", "message", "1").setSource(jsonBuilder().startObject()
+            .field("message", "quick brown fox")
+            .startArray("comments")
+            .startObject().field("message", "fox eat quick").endObject()
+            .startObject().field("message", "fox ate rabbit x y z").endObject()
+            .startObject().field("message", "rabbit got away").endObject()
+            .endArray()
+            .endObject()).get();
+        refresh();
+
+        SearchResponse response = client().prepareSearch()
+            .setQuery(indicesQuery(nestedQuery("comments", matchQuery("comments.message", "fox"), ScoreMode.None)
+                .innerHit(new InnerHitBuilder()), "index1"))
+            .get();
+        assertNoFailures(response);
+        assertHitCount(response, 1);
+
+        assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getTotalHits(), equalTo(2L));
+        assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).getNestedIdentity().getOffset(), equalTo(0));
+        assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getAt(1).getNestedIdentity().getOffset(), equalTo(1));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesIT.java b/core/src/test/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesIT.java
index 170638b295f1f..4eab9d2969943 100644
--- a/core/src/test/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesIT.java
+++ b/core/src/test/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesIT.java
@@ -334,7 +334,7 @@ public void testMatchedWithShould() throws Exception {
             SearchResponse searchResponse = client().prepareSearch()
                     .setQuery(
                             boolQuery()
-                                    .minimumNumberShouldMatch(1)
+                                    .minimumShouldMatch(1)
                                     .should(queryStringQuery("dolor").queryName("dolor"))
                                     .should(queryStringQuery("elit").queryName("elit"))
                     )
diff --git a/core/src/test/java/org/elasticsearch/search/fetch/subphase/NestedChildrenFilterTests.java b/core/src/test/java/org/elasticsearch/search/fetch/subphase/NestedChildrenFilterTests.java
deleted file mode 100644
index 440d90bdba4b9..0000000000000
--- a/core/src/test/java/org/elasticsearch/search/fetch/subphase/NestedChildrenFilterTests.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.fetch.subphase;
-
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.LegacyIntField;
-import org.apache.lucene.document.StringField;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.RandomIndexWriter;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.ConstantScoreQuery;
-import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.TotalHitCountCollector;
-import org.apache.lucene.search.Weight;
-import org.apache.lucene.search.join.BitSetProducer;
-import org.apache.lucene.search.join.QueryBitSetProducer;
-import org.apache.lucene.store.Directory;
-import org.elasticsearch.search.fetch.FetchSubPhase;
-import org.elasticsearch.search.fetch.subphase.InnerHitsContext.NestedInnerHits.NestedChildrenQuery;
-import org.elasticsearch.test.ESTestCase;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.hamcrest.Matchers.equalTo;
-
-/**
- */
-public class NestedChildrenFilterTests extends ESTestCase {
-    public void testNestedChildrenFilter() throws Exception {
-        int numParentDocs = scaledRandomIntBetween(0, 32);
-        int maxChildDocsPerParent = scaledRandomIntBetween(8, 16);
-
-        Directory dir = newDirectory();
-        RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
-        for (int i = 0; i < numParentDocs; i++) {
-            int numChildDocs = scaledRandomIntBetween(0, maxChildDocsPerParent);
-            List<Document> docs = new ArrayList<>(numChildDocs + 1);
-            for (int j = 0; j < numChildDocs; j++) {
-                Document childDoc = new Document();
-                childDoc.add(new StringField("type", "child", Field.Store.NO));
-                docs.add(childDoc);
-            }
-
-            Document parenDoc = new Document();
-            parenDoc.add(new StringField("type", "parent", Field.Store.NO));
-            parenDoc.add(new LegacyIntField("num_child_docs", numChildDocs, Field.Store.YES));
-            docs.add(parenDoc);
-            writer.addDocuments(docs);
-        }
-
-        IndexReader reader = writer.getReader();
-        writer.close();
-
-        IndexSearcher searcher = new IndexSearcher(reader);
-        FetchSubPhase.HitContext hitContext = new FetchSubPhase.HitContext();
-        BitSetProducer parentFilter = new QueryBitSetProducer(new TermQuery(new Term("type", "parent")));
-        Query childFilter = new TermQuery(new Term("type", "child"));
-        int checkedParents = 0;
-        final Weight parentsWeight = searcher.createNormalizedWeight(new TermQuery(new Term("type", "parent")), false);
-        for (LeafReaderContext leaf : reader.leaves()) {
-            DocIdSetIterator parents = parentsWeight.scorer(leaf).iterator();
-            for (int parentDoc = parents.nextDoc(); parentDoc != DocIdSetIterator.NO_MORE_DOCS ; parentDoc = parents.nextDoc()) {
-                int expectedChildDocs = leaf.reader().document(parentDoc).getField("num_child_docs").numericValue().intValue();
-                hitContext.reset(null, leaf, parentDoc, searcher);
-                NestedChildrenQuery nestedChildrenFilter = new NestedChildrenQuery(parentFilter, childFilter, hitContext);
-                TotalHitCountCollector totalHitCountCollector = new TotalHitCountCollector();
-                searcher.search(new ConstantScoreQuery(nestedChildrenFilter), totalHitCountCollector);
-                assertThat(totalHitCountCollector.getTotalHits(), equalTo(expectedChildDocs));
-                checkedParents++;
-            }
-        }
-        assertThat(checkedParents, equalTo(numParentDocs));
-        reader.close();
-        dir.close();
-    }
-
-}
diff --git a/core/src/test/java/org/elasticsearch/search/fetch/subphase/ParentFieldSubFetchPhaseTests.java b/core/src/test/java/org/elasticsearch/search/fetch/subphase/ParentFieldSubFetchPhaseTests.java
new file mode 100644
index 0000000000000..968424eee735c
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/fetch/subphase/ParentFieldSubFetchPhaseTests.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.fetch.subphase;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.mapper.ContentPath;
+import org.elasticsearch.index.mapper.Mapper;
+import org.elasticsearch.index.mapper.ParentFieldMapper;
+import org.elasticsearch.test.ESTestCase;
+
+public class ParentFieldSubFetchPhaseTests extends ESTestCase {
+
+    public void testGetParentId() throws Exception {
+        ParentFieldMapper fieldMapper = createParentFieldMapper();
+        Directory directory = newDirectory();
+        IndexWriter indexWriter = new IndexWriter(directory, newIndexWriterConfig());
+        Document document = new Document();
+        document.add(new SortedDocValuesField(fieldMapper.fieldType().name(), new BytesRef("1")));
+        indexWriter.addDocument(document);
+        indexWriter.close();
+
+        IndexReader indexReader = DirectoryReader.open(directory);
+        String id = ParentFieldSubFetchPhase.getParentId(fieldMapper, indexReader.leaves().get(0).reader(), 0);
+        assertEquals("1", id);
+
+        indexReader.close();
+        directory.close();
+    }
+
+    public void testGetParentIdNoParentField() throws Exception {
+        ParentFieldMapper fieldMapper = createParentFieldMapper();
+        Directory directory = newDirectory();
+        IndexWriter indexWriter = new IndexWriter(directory, newIndexWriterConfig());
+        Document document = new Document();
+        document.add(new SortedDocValuesField("different_field", new BytesRef("1")));
+        indexWriter.addDocument(document);
+        indexWriter.close();
+
+        IndexReader indexReader = DirectoryReader.open(directory);
+        String id = ParentFieldSubFetchPhase.getParentId(fieldMapper, indexReader.leaves().get(0).reader(), 0);
+        assertNull(id);
+
+        indexReader.close();
+        directory.close();
+    }
+
+    private ParentFieldMapper createParentFieldMapper() {
+        Settings settings = Settings.builder()
+                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                .build();
+        return new ParentFieldMapper.Builder("type")
+                .type("parent_type")
+                .build(new Mapper.BuilderContext(settings, new ContentPath(0)));
+    }
+
+
+}
diff --git a/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java b/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java
index ed272040f516e..2d1fb3d986c30 100644
--- a/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java
@@ -22,19 +22,18 @@
 import org.apache.lucene.search.Query;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.ContentPath;
@@ -47,11 +46,8 @@
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.query.TermQueryBuilder;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.search.SearchModule;
-import org.elasticsearch.search.fetch.subphase.highlight.AbstractHighlighterBuilder;
-import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
-import org.elasticsearch.search.fetch.subphase.highlight.SearchContextHighlight;
+import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder.BoundaryScannerType;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder.Field;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder.Order;
 import org.elasticsearch.search.fetch.subphase.highlight.SearchContextHighlight.FieldOptions;
@@ -72,14 +68,14 @@
 import java.util.function.Function;
 
 import static java.util.Collections.emptyList;
+import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.not;
 
 public class HighlightBuilderTests extends ESTestCase {
 
     private static final int NUMBER_OF_TESTBUILDERS = 20;
     private static NamedWriteableRegistry namedWriteableRegistry;
-    private static IndicesQueriesRegistry indicesQueriesRegistry;
+    private static NamedXContentRegistry xContentRegistry;
 
     /**
      * setup for the whole base test class
@@ -88,13 +84,13 @@ public class HighlightBuilderTests extends ESTestCase {
     public static void init() {
         SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList());
         namedWriteableRegistry = new NamedWriteableRegistry(searchModule.getNamedWriteables());
-        indicesQueriesRegistry = searchModule.getQueryParserRegistry();
+        xContentRegistry = new NamedXContentRegistry(searchModule.getNamedXContents());
     }
 
     @AfterClass
     public static void afterClass() throws Exception {
         namedWriteableRegistry = null;
-        indicesQueriesRegistry = null;
+        xContentRegistry = null;
     }
 
     /**
@@ -115,31 +111,7 @@ public void testSerialization() throws IOException {
      */
     public void testEqualsAndHashcode() throws IOException {
         for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) {
-            HighlightBuilder firstBuilder = randomHighlighterBuilder();
-            assertFalse("highlighter is equal to null", firstBuilder.equals(null));
-            assertFalse("highlighter is equal to incompatible type", firstBuilder.equals(""));
-            assertTrue("highlighter is not equal to self", firstBuilder.equals(firstBuilder));
-            assertThat("same highlighter's hashcode returns different values if called multiple times", firstBuilder.hashCode(),
-                    equalTo(firstBuilder.hashCode()));
-            assertThat("different highlighters should not be equal", mutate(firstBuilder), not(equalTo(firstBuilder)));
-
-            HighlightBuilder secondBuilder = serializedCopy(firstBuilder);
-            assertTrue("highlighter is not equal to self", secondBuilder.equals(secondBuilder));
-            assertTrue("highlighter is not equal to its copy", firstBuilder.equals(secondBuilder));
-            assertTrue("equals is not symmetric", secondBuilder.equals(firstBuilder));
-            assertThat("highlighter copy's hashcode is different from original hashcode", secondBuilder.hashCode(),
-                    equalTo(firstBuilder.hashCode()));
-
-            HighlightBuilder thirdBuilder = serializedCopy(secondBuilder);
-            assertTrue("highlighter is not equal to self", thirdBuilder.equals(thirdBuilder));
-            assertTrue("highlighter is not equal to its copy", secondBuilder.equals(thirdBuilder));
-            assertThat("highlighter copy's hashcode is different from original hashcode", secondBuilder.hashCode(),
-                    equalTo(thirdBuilder.hashCode()));
-            assertTrue("equals is not transitive", firstBuilder.equals(thirdBuilder));
-            assertThat("highlighter copy's hashcode is different from original hashcode", firstBuilder.hashCode(),
-                    equalTo(thirdBuilder.hashCode()));
-            assertTrue("equals is not symmetric", thirdBuilder.equals(secondBuilder));
-            assertTrue("equals is not symmetric", thirdBuilder.equals(firstBuilder));
+            checkEqualsAndHashCode(randomHighlighterBuilder(), HighlightBuilderTests::serializedCopy, HighlightBuilderTests::mutate);
         }
     }
 
@@ -153,11 +125,20 @@ public void testFromXContent() throws IOException {
             if (randomBoolean()) {
                 builder.prettyPrint();
             }
-            highlightBuilder.toXContent(builder, ToXContent.EMPTY_PARAMS);
-            XContentBuilder shuffled = shuffleXContent(builder);
 
-            XContentParser parser = XContentHelper.createParser(shuffled.bytes());
-            QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.EMPTY);
+            XContentBuilder shuffled;
+            if (randomBoolean()) {
+                //this way `fields` is printed out as a json array
+                highlightBuilder.useExplicitFieldOrder(true);
+                highlightBuilder.toXContent(builder, ToXContent.EMPTY_PARAMS);
+                shuffled = shuffleXContent(builder);
+            } else {
+                highlightBuilder.toXContent(builder, ToXContent.EMPTY_PARAMS);
+                shuffled = shuffleXContent(builder, "fields");
+            }
+
+            XContentParser parser = createParser(shuffled);
+            QueryParseContext context = new QueryParseContext(parser);
             parser.nextToken();
             HighlightBuilder secondHighlightBuilder;
             try {
@@ -196,9 +177,9 @@ public void testUnknownArrayNameExpection() throws IOException {
         }
     }
 
-    private static <T extends Throwable> T expectParseThrows(Class<T> exceptionClass, String highlightElement) throws IOException {
-        XContentParser parser = XContentFactory.xContent(highlightElement).createParser(highlightElement);
-        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.STRICT);
+    private <T extends Throwable> T expectParseThrows(Class<T> exceptionClass, String highlightElement) throws IOException {
+        XContentParser parser = createParser(JsonXContent.jsonXContent, highlightElement);
+        QueryParseContext context = new QueryParseContext(parser);
         return expectThrows(exceptionClass, () -> HighlightBuilder.fromXContent(context));
     }
 
@@ -290,11 +271,11 @@ public void testTwoFieldsInObjectInFieldsArray() throws IOException {
     public void testBuildSearchContextHighlight() throws IOException {
         Settings indexSettings = Settings.builder()
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
-        Index index = new Index(randomAsciiOfLengthBetween(1, 10), "_na_");
+        Index index = new Index(randomAlphaOfLengthBetween(1, 10), "_na_");
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(index, indexSettings);
         // shard context will only need indicesQueriesRegistry for building Query objects nested in highlighter
-        QueryShardContext mockShardContext = new QueryShardContext(idxSettings, null, null, null, null, null, indicesQueriesRegistry,
-                null, null, null) {
+        QueryShardContext mockShardContext = new QueryShardContext(0, idxSettings, null, null, null, null, null, xContentRegistry(),
+                null, null, System::currentTimeMillis) {
             @Override
             public MappedFieldType fieldMapper(String name) {
                 TextFieldMapper.Builder builder = new TextFieldMapper.Builder(name);
@@ -317,6 +298,7 @@ public MappedFieldType fieldMapper(String name) {
                         mergeBeforeChek(highlightBuilder, fieldBuilder, fieldOptions);
 
                 checkSame.accept(AbstractHighlighterBuilder::boundaryChars, FieldOptions::boundaryChars);
+                checkSame.accept(AbstractHighlighterBuilder::boundaryScannerType, FieldOptions::boundaryScannerType);
                 checkSame.accept(AbstractHighlighterBuilder::boundaryMaxScan, FieldOptions::boundaryMaxScan);
                 checkSame.accept(AbstractHighlighterBuilder::fragmentSize, FieldOptions::fragmentCharSize);
                 checkSame.accept(AbstractHighlighterBuilder::fragmenter, FieldOptions::fragmenter);
@@ -335,7 +317,7 @@ public MappedFieldType fieldMapper(String name) {
                     String[] copy = Arrays.copyOf(fieldBuilder.matchedFields, fieldBuilder.matchedFields.length);
                     Arrays.sort(copy);
                     assertArrayEquals(copy,
-                            new TreeSet<String>(fieldOptions.matchedFields()).toArray(new String[fieldOptions.matchedFields().size()]));
+                            new TreeSet<>(fieldOptions.matchedFields()).toArray(new String[fieldOptions.matchedFields().size()]));
                 } else {
                     assertNull(fieldOptions.matchedFields());
                 }
@@ -406,9 +388,9 @@ public void testParsingTagsSchema() throws IOException {
         String highlightElement = "{\n" +
                 "    \"tags_schema\" : \"styled\"\n" +
                 "}\n";
-        XContentParser parser = XContentFactory.xContent(highlightElement).createParser(highlightElement);
+        XContentParser parser = createParser(JsonXContent.jsonXContent, highlightElement);
 
-        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.EMPTY);
+        QueryParseContext context = new QueryParseContext(parser);
         HighlightBuilder highlightBuilder = HighlightBuilder.fromXContent(context);
         assertArrayEquals("setting tags_schema 'styled' should alter pre_tags", HighlightBuilder.DEFAULT_STYLED_PRE_TAG,
                 highlightBuilder.preTags());
@@ -418,9 +400,9 @@ public void testParsingTagsSchema() throws IOException {
         highlightElement = "{\n" +
                 "    \"tags_schema\" : \"default\"\n" +
                 "}\n";
-        parser = XContentFactory.xContent(highlightElement).createParser(highlightElement);
+        parser = createParser(JsonXContent.jsonXContent, highlightElement);
 
-        context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.EMPTY);
+        context = new QueryParseContext(parser);
         highlightBuilder = HighlightBuilder.fromXContent(context);
         assertArrayEquals("setting tags_schema 'default' should alter pre_tags", HighlightBuilder.DEFAULT_PRE_TAGS,
                 highlightBuilder.preTags());
@@ -439,23 +421,23 @@ public void testParsingTagsSchema() throws IOException {
      */
     public void testParsingEmptyStructure() throws IOException {
         String highlightElement = "{ }";
-        XContentParser parser = XContentFactory.xContent(highlightElement).createParser(highlightElement);
+        XContentParser parser = createParser(JsonXContent.jsonXContent, highlightElement);
 
-        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.EMPTY);
+        QueryParseContext context = new QueryParseContext(parser);
         HighlightBuilder highlightBuilder = HighlightBuilder.fromXContent(context);
         assertEquals("expected plain HighlightBuilder", new HighlightBuilder(), highlightBuilder);
 
         highlightElement = "{ \"fields\" : { } }";
-        parser = XContentFactory.xContent(highlightElement).createParser(highlightElement);
+        parser = createParser(JsonXContent.jsonXContent, highlightElement);
 
-        context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.EMPTY);
+        context = new QueryParseContext(parser);
         highlightBuilder = HighlightBuilder.fromXContent(context);
         assertEquals("defining no field should return plain HighlightBuilder", new HighlightBuilder(), highlightBuilder);
 
         highlightElement = "{ \"fields\" : { \"foo\" : { } } }";
-        parser = XContentFactory.xContent(highlightElement).createParser(highlightElement);
+        parser = createParser(JsonXContent.jsonXContent, highlightElement);
 
-        context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.EMPTY);
+        context = new QueryParseContext(parser);
         highlightBuilder = HighlightBuilder.fromXContent(context);
         assertEquals("expected HighlightBuilder with field", new HighlightBuilder().field(new Field("foo")), highlightBuilder);
     }
@@ -523,7 +505,7 @@ public static HighlightBuilder randomHighlighterBuilder() {
         }
         int numberOfFields = randomIntBetween(1,5);
         for (int i = 0; i < numberOfFields; i++) {
-            Field field = new Field(i + "_" + randomAsciiOfLengthBetween(1, 10));
+            Field field = new Field(i + "_" + randomAlphaOfLengthBetween(1, 10));
             setRandomCommonOptions(field);
             if (randomBoolean()) {
                 field.fragmentOffset(randomIntBetween(1, 100));
@@ -536,7 +518,7 @@ public static HighlightBuilder randomHighlighterBuilder() {
         return testHighlighter;
     }
 
-    @SuppressWarnings({ "rawtypes", "unchecked" })
+    @SuppressWarnings({ "rawtypes"})
     private static void setRandomCommonOptions(AbstractHighlighterBuilder highlightBuilder) {
         if (randomBoolean()) {
             // need to set this together, otherwise parsing will complain
@@ -550,10 +532,10 @@ private static void setRandomCommonOptions(AbstractHighlighterBuilder highlightB
             highlightBuilder.numOfFragments(randomIntBetween(0, 10));
         }
         if (randomBoolean()) {
-            highlightBuilder.highlighterType(randomAsciiOfLengthBetween(1, 10));
+            highlightBuilder.highlighterType(randomAlphaOfLengthBetween(1, 10));
         }
         if (randomBoolean()) {
-            highlightBuilder.fragmenter(randomAsciiOfLengthBetween(1, 10));
+            highlightBuilder.fragmenter(randomAlphaOfLengthBetween(1, 10));
         }
         if (randomBoolean()) {
             QueryBuilder highlightQuery;
@@ -566,7 +548,7 @@ private static void setRandomCommonOptions(AbstractHighlighterBuilder highlightB
                 break;
             default:
             case 2:
-                highlightQuery = new TermQueryBuilder(randomAsciiOfLengthBetween(1, 10), randomAsciiOfLengthBetween(1, 10));
+                highlightQuery = new TermQueryBuilder(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10));
                 break;
             }
             highlightQuery.boost((float) randomDoubleBetween(0, 10, false));
@@ -586,11 +568,22 @@ private static void setRandomCommonOptions(AbstractHighlighterBuilder highlightB
         if (randomBoolean()) {
             highlightBuilder.forceSource(randomBoolean());
         }
+        if (randomBoolean()) {
+            if (randomBoolean()) {
+                highlightBuilder.boundaryScannerType(randomFrom(BoundaryScannerType.values()));
+            } else {
+                // also test the string setter
+                highlightBuilder.boundaryScannerType(randomFrom(BoundaryScannerType.values()).toString());
+            }
+        }
         if (randomBoolean()) {
             highlightBuilder.boundaryMaxScan(randomIntBetween(0, 10));
         }
         if (randomBoolean()) {
-            highlightBuilder.boundaryChars(randomAsciiOfLengthBetween(1, 10).toCharArray());
+            highlightBuilder.boundaryChars(randomAlphaOfLengthBetween(1, 10).toCharArray());
+        }
+        if (randomBoolean()) {
+            highlightBuilder.boundaryScannerLocale(randomLocale(random()).toLanguageTag());
         }
         if (randomBoolean()) {
             highlightBuilder.noMatchSize(randomIntBetween(0, 10));
@@ -600,12 +593,12 @@ private static void setRandomCommonOptions(AbstractHighlighterBuilder highlightB
         }
         if (randomBoolean()) {
             int items = randomIntBetween(0, 5);
-            Map<String, Object> options = new HashMap<String, Object>(items);
+            Map<String, Object> options = new HashMap<>(items);
             for (int i = 0; i < items; i++) {
                 Object value = null;
                 switch (randomInt(2)) {
                 case 0:
-                    value = randomAsciiOfLengthBetween(1, 10);
+                    value = randomAlphaOfLengthBetween(1, 10);
                     break;
                 case 1:
                     value = new Integer(randomInt(1000));
@@ -614,7 +607,7 @@ private static void setRandomCommonOptions(AbstractHighlighterBuilder highlightB
                     value = new Boolean(randomBoolean());
                     break;
                 }
-                options.put(randomAsciiOfLengthBetween(1, 10), value);
+                options.put(randomAlphaOfLengthBetween(1, 10), value);
             }
         }
         if (randomBoolean()) {
@@ -638,13 +631,13 @@ private static void mutateCommonOptions(AbstractHighlighterBuilder highlightBuil
             highlightBuilder.numOfFragments(randomIntBetween(11, 20));
             break;
         case 5:
-            highlightBuilder.highlighterType(randomAsciiOfLengthBetween(11, 20));
+            highlightBuilder.highlighterType(randomAlphaOfLengthBetween(11, 20));
             break;
         case 6:
-            highlightBuilder.fragmenter(randomAsciiOfLengthBetween(11, 20));
+            highlightBuilder.fragmenter(randomAlphaOfLengthBetween(11, 20));
             break;
         case 7:
-            highlightBuilder.highlightQuery(new TermQueryBuilder(randomAsciiOfLengthBetween(11, 20), randomAsciiOfLengthBetween(11, 20)));
+            highlightBuilder.highlightQuery(new TermQueryBuilder(randomAlphaOfLengthBetween(11, 20), randomAlphaOfLengthBetween(11, 20)));
             break;
         case 8:
             if (highlightBuilder.order() == Order.NONE) {
@@ -663,7 +656,7 @@ private static void mutateCommonOptions(AbstractHighlighterBuilder highlightBuil
             highlightBuilder.boundaryMaxScan(randomIntBetween(11, 20));
             break;
         case 12:
-            highlightBuilder.boundaryChars(randomAsciiOfLengthBetween(11, 20).toCharArray());
+            highlightBuilder.boundaryChars(randomAlphaOfLengthBetween(11, 20).toCharArray());
             break;
         case 13:
             highlightBuilder.noMatchSize(randomIntBetween(11, 20));
@@ -673,9 +666,9 @@ private static void mutateCommonOptions(AbstractHighlighterBuilder highlightBuil
             break;
         case 15:
             int items = 6;
-            Map<String, Object> options = new HashMap<String, Object>(items);
+            Map<String, Object> options = new HashMap<>(items);
             for (int i = 0; i < items; i++) {
-                options.put(randomAsciiOfLengthBetween(1, 10), randomAsciiOfLengthBetween(1, 10));
+                options.put(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10));
             }
             highlightBuilder.options(options);
             break;
@@ -699,9 +692,9 @@ private static Boolean toggleOrSet(Boolean flag) {
      */
     private static String[] randomStringArray(int minSize, int maxSize) {
         int size = randomIntBetween(minSize, maxSize);
-        Set<String> randomStrings = new HashSet<String>(size);
+        Set<String> randomStrings = new HashSet<>(size);
         for (int f = 0; f < size; f++) {
-            randomStrings.add(randomAsciiOfLengthBetween(3, 10));
+            randomStrings.add(randomAlphaOfLengthBetween(3, 10));
         }
         return randomStrings.toArray(new String[randomStrings.size()]);
     }
@@ -719,11 +712,11 @@ private static HighlightBuilder mutate(HighlightBuilder original) throws IOExcep
                 case 0:
                     mutation.useExplicitFieldOrder(!original.useExplicitFieldOrder()); break;
                 case 1:
-                    mutation.encoder(original.encoder() + randomAsciiOfLength(2)); break;
+                    mutation.encoder(original.encoder() + randomAlphaOfLength(2)); break;
                 case 2:
                     if (randomBoolean()) {
                         // add another field
-                        mutation.field(new Field(randomAsciiOfLength(10)));
+                        mutation.field(new Field(randomAlphaOfLength(10)));
                     } else {
                         // change existing fields
                         List<Field> originalFields = original.fields();
@@ -741,11 +734,11 @@ private static HighlightBuilder mutate(HighlightBuilder original) throws IOExcep
     }
 
     private static HighlightBuilder serializedCopy(HighlightBuilder original) throws IOException {
-        try (BytesStreamOutput output = new BytesStreamOutput()) {
-            original.writeTo(output);
-            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
-                return new HighlightBuilder(in);
-            }
-        }
+        return ESTestCase.copyWriteable(original, namedWriteableRegistry, HighlightBuilder::new);
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return xContentRegistry;
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightFieldTests.java b/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightFieldTests.java
new file mode 100644
index 0000000000000..b4bf2950b7d07
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightFieldTests.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.fetch.subphase.highlight;
+
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.text.Text;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
+
+public class HighlightFieldTests extends ESTestCase {
+
+    public static HighlightField createTestItem() {
+        String name = frequently() ? randomAlphaOfLengthBetween(5, 20) : randomRealisticUnicodeOfCodepointLengthBetween(5, 20);
+        Text[] fragments = null;
+        if (frequently()) {
+            int size = randomIntBetween(0, 5);
+            fragments = new Text[size];
+            for (int i = 0; i < size; i++) {
+                fragments[i] = new Text(
+                        frequently() ? randomAlphaOfLengthBetween(10, 30) : randomRealisticUnicodeOfCodepointLengthBetween(10, 30));
+            }
+        }
+        return new HighlightField(name, fragments);
+    }
+
+    public void testFromXContent() throws IOException {
+        HighlightField highlightField = createTestItem();
+        XContentType xcontentType = randomFrom(XContentType.values());
+        XContentBuilder builder = XContentFactory.contentBuilder(xcontentType);
+        if (randomBoolean()) {
+            builder.prettyPrint();
+        }
+        builder.startObject(); // we need to wrap xContent output in proper object to create a parser for it
+        builder = highlightField.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        builder.endObject();
+        XContentParser parser = createParser(builder);
+        parser.nextToken(); // skip to the opening object token, fromXContent advances from here and starts with the field name
+        parser.nextToken();
+        HighlightField parsedField = HighlightField.fromXContent(parser);
+        assertEquals(highlightField, parsedField);
+        if (highlightField.fragments() != null) {
+            assertEquals(XContentParser.Token.END_ARRAY, parser.currentToken());
+        }
+        assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
+        assertNull(parser.nextToken());
+    }
+
+    public void testToXContent() throws IOException {
+        HighlightField field = new HighlightField("foo", new Text[] { new Text("bar"), new Text("baz") });
+        XContentBuilder builder = JsonXContent.contentBuilder();
+        builder.prettyPrint();
+        builder.startObject();
+        field.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        builder.endObject();
+        assertEquals(
+          "{\n" +
+          "  \"foo\" : [\n" +
+          "    \"bar\",\n" +
+          "    \"baz\"\n" +
+          "  ]\n" +
+          "}", builder.string());
+
+        field = new HighlightField("foo", null);
+        builder = JsonXContent.contentBuilder();
+        builder.prettyPrint();
+        builder.startObject();
+        field.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        builder.endObject();
+        assertEquals(
+          "{\n" +
+          "  \"foo\" : null\n" +
+          "}", builder.string());
+    }
+
+    /**
+     * Test equality and hashCode properties
+     */
+    public void testEqualsAndHashcode() {
+        checkEqualsAndHashCode(createTestItem(), HighlightFieldTests::copy, HighlightFieldTests::mutate);
+    }
+
+    public void testSerialization() throws IOException {
+        HighlightField testField = createTestItem();
+        try (BytesStreamOutput output = new BytesStreamOutput()) {
+            testField.writeTo(output);
+            try (StreamInput in = output.bytes().streamInput()) {
+                HighlightField deserializedCopy = HighlightField.readHighlightField(in);
+                assertEquals(testField, deserializedCopy);
+                assertEquals(testField.hashCode(), deserializedCopy.hashCode());
+                assertNotSame(testField, deserializedCopy);
+            }
+        }
+    }
+
+    private static HighlightField mutate(HighlightField original) {
+        Text[] fragments = original.getFragments();
+        if (randomBoolean()) {
+            return new HighlightField(original.getName()+"_suffix", fragments);
+        } else {
+            if (fragments == null) {
+                fragments = new Text[]{new Text("field")};
+            } else {
+                fragments = Arrays.copyOf(fragments, fragments.length + 1);
+                fragments[fragments.length - 1] = new Text("something new");
+            }
+            return new HighlightField(original.getName(), fragments);
+        }
+    }
+
+    private static HighlightField copy(HighlightField original) {
+        return new HighlightField(original.getName(), original.getFragments());
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java b/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java
index 33e8cb3784f3b..a920d8b6afd5d 100644
--- a/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java
+++ b/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java
@@ -19,7 +19,6 @@
 package org.elasticsearch.search.fetch.subphase.highlight;
 
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
-
 import org.apache.lucene.search.join.ScoreMode;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.index.IndexRequestBuilder;
@@ -27,10 +26,12 @@
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.Settings.Builder;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.AbstractQueryBuilder;
 import org.elasticsearch.index.query.IdsQueryBuilder;
 import org.elasticsearch.index.query.MatchQueryBuilder;
@@ -38,21 +39,28 @@
 import org.elasticsearch.index.query.Operator;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
+import org.elasticsearch.index.query.functionscore.RandomScoreFunctionBuilder;
 import org.elasticsearch.index.search.MatchQuery;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder.BoundaryScannerType;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder.Field;
+import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
 import org.hamcrest.Matcher;
 import org.hamcrest.Matchers;
+import org.joda.time.DateTime;
+import org.joda.time.chrono.ISOChronology;
 
 import java.io.IOException;
-import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.Locale;
 import java.util.Map;
 
 import static org.elasticsearch.client.Requests.searchRequest;
@@ -83,6 +91,7 @@
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNotHighlighted;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
 import static org.elasticsearch.test.hamcrest.RegexMatcher.matches;
 import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.containsString;
@@ -93,10 +102,37 @@
 import static org.hamcrest.Matchers.startsWith;
 
 public class HighlighterSearchIT extends ESIntegTestCase {
+    private static final String[] ALL_TYPES = new String[] {"plain", "postings", "fvh", "unified"};
+    private static final String[] UNIFIED_AND_NULL = new String[] {null, "unified"};
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Arrays.asList(InternalSettingsPlugin.class);
+        return Collections.singletonList(InternalSettingsPlugin.class);
+    }
+
+    public void testHighlightingWithStoredKeyword() throws IOException {
+        XContentBuilder mappings = jsonBuilder();
+        mappings.startObject();
+        mappings.startObject("type")
+            .startObject("properties")
+            .startObject("text")
+            .field("type", "keyword")
+            .field("store", true)
+            .endObject()
+            .endObject()
+            .endObject();
+        mappings.endObject();
+        assertAcked(prepareCreate("test")
+            .addMapping("type", mappings));
+        client().prepareIndex("test", "type", "1")
+            .setSource(jsonBuilder().startObject().field("text", "foo").endObject())
+            .get();
+        refresh();
+        for (String type : UNIFIED_AND_NULL) {
+            SearchResponse search = client().prepareSearch().setQuery(matchQuery("text", "foo"))
+                .highlighter(new HighlightBuilder().field(new Field("text")).highlighterType(type)).get();
+            assertHighlight(search, 0, "text", 0, equalTo("<em>foo</em>"));
+        }
     }
 
     public void testHighlightingWithWildcardName() throws IOException {
@@ -120,10 +156,11 @@ public void testHighlightingWithWildcardName() throws IOException {
                 .setSource(jsonBuilder().startObject().field("text", "text").endObject())
                 .get();
         refresh();
-        String highlighter = randomFrom("plain", "postings", "fvh");
-        SearchResponse search = client().prepareSearch().setQuery(constantScoreQuery(matchQuery("text", "text")))
-                .highlighter(new HighlightBuilder().field(new Field("*").highlighterType(highlighter))).get();
-        assertHighlight(search, 0, "text", 0, equalTo("<em>text</em>"));
+        for (String type : ALL_TYPES) {
+            SearchResponse search = client().prepareSearch().setQuery(constantScoreQuery(matchQuery("text", "text")))
+                .highlighter(new HighlightBuilder().field(new Field("*").highlighterType(type))).get();
+            assertHighlight(search, 0, "text", 0, equalTo("<em>text</em>"));
+        }
     }
 
     public void testHighlightingWhenFieldsAreNotStoredThereIsNoSource() throws IOException {
@@ -155,14 +192,15 @@ public void testHighlightingWhenFieldsAreNotStoredThereIsNoSource() throws IOExc
                 .setSource(jsonBuilder().startObject().field("unstored_text", "text").field("text", "text").endObject())
                 .get();
         refresh();
-        String highlighter = randomFrom("plain", "postings", "fvh");
-        SearchResponse search = client().prepareSearch().setQuery(constantScoreQuery(matchQuery("text", "text")))
-                .highlighter(new HighlightBuilder().field(new Field("*").highlighterType(highlighter))).get();
-        assertHighlight(search, 0, "text", 0, equalTo("<em>text</em>"));
-        search = client().prepareSearch().setQuery(constantScoreQuery(matchQuery("text", "text")))
+        for (String type : ALL_TYPES) {
+            SearchResponse search = client().prepareSearch().setQuery(constantScoreQuery(matchQuery("text", "text")))
+                .highlighter(new HighlightBuilder().field(new Field("*").highlighterType(type))).get();
+            assertHighlight(search, 0, "text", 0, equalTo("<em>text</em>"));
+            search = client().prepareSearch().setQuery(constantScoreQuery(matchQuery("text", "text")))
                 .highlighter(new HighlightBuilder().field(new Field("unstored_text"))).get();
-        assertNoFailures(search);
-        assertThat(search.getHits().getAt(0).getHighlightFields().size(), equalTo(0));
+            assertNoFailures(search);
+            assertThat(search.getHits().getAt(0).getHighlightFields().size(), equalTo(0));
+        }
     }
 
     // see #3486
@@ -339,7 +377,6 @@ public void testEnsureNoNegativeOffsets() throws Exception {
                             + "and should be highlighted")
                 .get();
         refresh();
-
         SearchResponse search = client().prepareSearch()
                 .setQuery(matchQuery("long_term", "thisisaverylongwordandmakessurethisfails foo highlighed"))
                 .highlighter(new HighlightBuilder().field("long_term", 18, 1))
@@ -389,23 +426,25 @@ public void testSourceLookupHighlightingUsingPlainHighlighter() throws Exception
         }
         indexRandom(true, indexRequestBuilders);
 
-        SearchResponse search = client().prepareSearch()
+        for (String type : UNIFIED_AND_NULL) {
+            SearchResponse search = client().prepareSearch()
                 .setQuery(matchQuery("title", "bug"))
-                .highlighter(new HighlightBuilder().field("title", -1, 0))
+                .highlighter(new HighlightBuilder().field("title", -1, 0).highlighterType(type))
                 .get();
 
-        for (int i = 0; i < indexRequestBuilders.length; i++) {
-            assertHighlight(search, i, "title", 0, equalTo("This is a test on the highlighting <em>bug</em> present in elasticsearch"));
-        }
+            for (int i = 0; i < indexRequestBuilders.length; i++) {
+                assertHighlight(search, i, "title", 0, equalTo("This is a test on the highlighting <em>bug</em> present in elasticsearch"));
+            }
 
-        search = client().prepareSearch()
+            search = client().prepareSearch()
                 .setQuery(matchQuery("attachments.body", "attachment"))
-                .highlighter(new HighlightBuilder().field("attachments.body", -1, 0))
+                .highlighter(new HighlightBuilder().field("attachments.body", -1, 0).highlighterType(type))
                 .get();
 
-        for (int i = 0; i < indexRequestBuilders.length; i++) {
-            assertHighlight(search, i, "attachments.body", 0, equalTo("<em>attachment</em> 1"));
-            assertHighlight(search, i, "attachments.body", 1, equalTo("<em>attachment</em> 2"));
+            for (int i = 0; i < indexRequestBuilders.length; i++) {
+                assertHighlight(search, i, "attachments.body", 0, equalTo("<em>attachment</em> 1"));
+                assertHighlight(search, i, "attachments.body", 1, equalTo("<em>attachment</em> 2"));
+            }
         }
     }
 
@@ -441,23 +480,25 @@ public void testSourceLookupHighlightingUsingFastVectorHighlighter() throws Exce
         }
         indexRandom(true, indexRequestBuilders);
 
-        SearchResponse search = client().prepareSearch()
+        for (String type : UNIFIED_AND_NULL) {
+            SearchResponse search = client().prepareSearch()
                 .setQuery(matchQuery("title", "bug"))
-                .highlighter(new HighlightBuilder().field("title", -1, 0))
+                .highlighter(new HighlightBuilder().field("title", -1, 0).highlighterType(type))
                 .get();
 
-        for (int i = 0; i < indexRequestBuilders.length; i++) {
-            assertHighlight(search, i, "title", 0, equalTo("This is a test on the highlighting <em>bug</em> present in elasticsearch"));
-        }
+            for (int i = 0; i < indexRequestBuilders.length; i++) {
+                assertHighlight(search, i, "title", 0, equalTo("This is a test on the highlighting <em>bug</em> present in elasticsearch"));
+            }
 
-        search = client().prepareSearch()
+            search = client().prepareSearch()
                 .setQuery(matchQuery("attachments.body", "attachment"))
-                .highlighter(new HighlightBuilder().field("attachments.body", -1, 2))
+                .highlighter(new HighlightBuilder().field("attachments.body", -1, 2).highlighterType(type))
                 .execute().get();
 
-        for (int i = 0; i < 5; i++) {
-            assertHighlight(search, i, "attachments.body", 0, equalTo("<em>attachment</em> 1"));
-            assertHighlight(search, i, "attachments.body", 1, equalTo("<em>attachment</em> 2"));
+            for (int i = 0; i < 5; i++) {
+                assertHighlight(search, i, "attachments.body", 0, equalTo("<em>attachment</em> 1"));
+                assertHighlight(search, i, "attachments.body", 1, equalTo("<em>attachment</em> 2"));
+            }
         }
     }
 
@@ -505,24 +546,28 @@ public void testSourceLookupHighlightingUsingPostingsHighlighter() throws Except
             assertHighlight(search, i, "title", 1, 2, equalTo("This is the second <em>bug</em> to perform highlighting on."));
         }
 
-        search = client().prepareSearch()
+        for (String type : UNIFIED_AND_NULL) {
+            search = client().prepareSearch()
                 .setQuery(matchQuery("title", "bug"))
                 //sentences will be generated out of each value
-                .highlighter(new HighlightBuilder().field("title")).get();
+                .highlighter(new HighlightBuilder().field("title").highlighterType(type)).get();
 
-        for (int i = 0; i < indexRequestBuilders.length; i++) {
-            assertHighlight(search, i, "title", 0, equalTo("This is a test on the highlighting <em>bug</em> present in elasticsearch."));
-            assertHighlight(search, i, "title", 1, 2, equalTo("This is the second <em>bug</em> to perform highlighting on."));
-        }
+            for (int i = 0; i < indexRequestBuilders.length; i++) {
+                assertHighlight(search, i, "title", 0,
+                    equalTo("This is a test on the highlighting <em>bug</em> present in elasticsearch."));
+                assertHighlight(search, i, "title", 1, 2,
+                    equalTo("This is the second <em>bug</em> to perform highlighting on."));
+            }
 
-        search = client().prepareSearch()
+            search = client().prepareSearch()
                 .setQuery(matchQuery("attachments.body", "attachment"))
-                .highlighter(new HighlightBuilder().field("attachments.body", -1, 2))
+                .highlighter(new HighlightBuilder().field("attachments.body", -1, 2).highlighterType(type))
                 .get();
 
-        for (int i = 0; i < indexRequestBuilders.length; i++) {
-            assertHighlight(search, i, "attachments.body", 0, equalTo("<em>attachment</em> for this test"));
-            assertHighlight(search, i, "attachments.body", 1, 2, equalTo("<em>attachment</em> 2"));
+            for (int i = 0; i < indexRequestBuilders.length; i++) {
+                assertHighlight(search, i, "attachments.body", 0, equalTo("<em>attachment</em> for this test"));
+                assertHighlight(search, i, "attachments.body", 1, 2, equalTo("<em>attachment</em> 2"));
+            }
         }
     }
 
@@ -625,48 +670,34 @@ public void testForceSourceWithSourceDisabled() throws Exception {
                 .setSource("field1", "The quick brown fox jumps over the lazy dog", "field2", "second field content").get();
         refresh();
 
-        //works using stored field
-        SearchResponse searchResponse = client().prepareSearch("test")
+        for (String type : UNIFIED_AND_NULL) {
+            //works using stored field
+            SearchResponse searchResponse = client().prepareSearch("test")
                 .setQuery(termQuery("field1", "quick"))
-                .highlighter(new HighlightBuilder().field(new Field("field1").preTags("<xxx>").postTags("</xxx>")))
+                .highlighter(new HighlightBuilder().field(new Field("field1").preTags("<xxx>").postTags("</xxx>").highlighterType(type)))
                 .get();
-        assertHighlight(searchResponse, 0, "field1", 0, 1, equalTo("The <xxx>quick</xxx> brown fox jumps over the lazy dog"));
+            assertHighlight(searchResponse, 0, "field1", 0, 1, equalTo("The <xxx>quick</xxx> brown fox jumps over the lazy dog"));
 
-        assertFailures(client().prepareSearch("test")
-                        .setQuery(termQuery("field1", "quick"))
-                        .highlighter(
-                                new HighlightBuilder().field(new Field("field1").preTags("<xxx>").postTags("</xxx>")
-                                        .highlighterType("plain").forceSource(true))),
+            assertFailures(client().prepareSearch("test")
+                    .setQuery(termQuery("field1", "quick"))
+                    .highlighter(
+                        new HighlightBuilder().field(new Field("field1").preTags("<xxx>").postTags("</xxx>")
+                            .highlighterType(type).forceSource(true))),
                 RestStatus.BAD_REQUEST,
                 containsString("source is forced for fields [field1] but type [type1] has disabled _source"));
 
-        assertFailures(client().prepareSearch("test")
-                        .setQuery(termQuery("field1", "quick"))
-                        .highlighter(
-                                new HighlightBuilder().field(new Field("field1").preTags("<xxx>").postTags("</xxx>").highlighterType("fvh")
-                                        .forceSource(true))),
+            SearchSourceBuilder searchSource = SearchSourceBuilder.searchSource().query(termQuery("field1", "quick"))
+                .highlighter(highlight().forceSource(true).field("field1").highlighterType(type));
+            assertFailures(client().prepareSearch("test").setSource(searchSource),
                 RestStatus.BAD_REQUEST,
                 containsString("source is forced for fields [field1] but type [type1] has disabled _source"));
 
-        assertFailures(client().prepareSearch("test")
-                .setQuery(termQuery("field1", "quick"))
-                        .highlighter(
-                                new HighlightBuilder().field(new Field("field1").preTags("<xxx>").postTags("</xxx>")
-                                        .highlighterType("postings").forceSource(true))),
-                RestStatus.BAD_REQUEST,
-                containsString("source is forced for fields [field1] but type [type1] has disabled _source"));
-
-        SearchSourceBuilder searchSource = SearchSourceBuilder.searchSource().query(termQuery("field1", "quick"))
-                .highlighter(highlight().forceSource(true).field("field1"));
-        assertFailures(client().prepareSearch("test").setSource(searchSource),
-                RestStatus.BAD_REQUEST,
-                containsString("source is forced for fields [field1] but type [type1] has disabled _source"));
-
-        searchSource = SearchSourceBuilder.searchSource().query(termQuery("field1", "quick"))
-                .highlighter(highlight().forceSource(true).field("field*"));
-        assertFailures(client().prepareSearch("test").setSource(searchSource),
+            searchSource = SearchSourceBuilder.searchSource().query(termQuery("field1", "quick"))
+                .highlighter(highlight().forceSource(true).field("field*").highlighterType(type));
+            assertFailures(client().prepareSearch("test").setSource(searchSource),
                 RestStatus.BAD_REQUEST,
                 matches("source is forced for fields \\[field\\d, field\\d\\] but type \\[type1\\] has disabled _source"));
+        }
     }
 
     public void testPlainHighlighter() throws Exception {
@@ -794,7 +825,120 @@ public void testFastVectorHighlighter() throws Exception {
         searchResponse = client().prepareSearch("test").setSource(source).get();
 
         assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The <em>quick</em> brown fox jumps over"));
+    }
+
+    public void testHighlighterWithSentenceBoundaryScanner() throws Exception {
+        assertAcked(prepareCreate("test").addMapping("type1", type1TermVectorMapping()));
+        ensureGreen();
+
+        indexRandom(true, client().prepareIndex("test", "type1")
+                .setSource("field1", "A sentence with few words. Another sentence with even more words."));
+
+        for (String type : new String[] {"unified", "fvh"}) {
+            logger.info("--> highlighting and searching on 'field' with sentence boundary_scanner");
+            SearchSourceBuilder source = searchSource()
+                .query(termQuery("field1", "sentence"))
+                .highlighter(highlight()
+                    .field("field1", 21, 2)
+                    .highlighterType(type)
+                    .preTags("<xxx>").postTags("</xxx>")
+                    .boundaryScannerType(BoundaryScannerType.SENTENCE));
+            SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
+
+            assertHighlight(searchResponse, 0, "field1", 0, 2, anyOf(
+                equalTo("A <xxx>sentence</xxx> with few words"),
+                equalTo("A <xxx>sentence</xxx> with few words. ")
+            ));
+
+            assertHighlight(searchResponse, 0, "field1", 1, 2, anyOf(
+                equalTo("Another <xxx>sentence</xxx> with"),
+                equalTo("Another <xxx>sentence</xxx> with even more words. ")
+            ));
+        }
+    }
+
+    public void testHighlighterWithSentenceBoundaryScannerAndLocale() throws Exception {
+        assertAcked(prepareCreate("test").addMapping("type1", type1TermVectorMapping()));
+        ensureGreen();
+
+        indexRandom(true, client().prepareIndex("test", "type1")
+                .setSource("field1", "A sentence with few words. Another sentence with even more words."));
 
+        for (String type : new String[] {"fvh", "unified"}) {
+            logger.info("--> highlighting and searching on 'field' with sentence boundary_scanner");
+            SearchSourceBuilder source = searchSource()
+                .query(termQuery("field1", "sentence"))
+                .highlighter(highlight()
+                    .field("field1", 21, 2)
+                    .highlighterType(type)
+                    .preTags("<xxx>").postTags("</xxx>")
+                    .boundaryScannerType(BoundaryScannerType.SENTENCE)
+                    .boundaryScannerLocale(Locale.ENGLISH.toLanguageTag()));
+
+            SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
+
+            assertHighlight(searchResponse, 0, "field1", 0, 2, anyOf(
+                equalTo("A <xxx>sentence</xxx> with few words"),
+                equalTo("A <xxx>sentence</xxx> with few words. ")
+            ));
+
+            assertHighlight(searchResponse, 0, "field1", 1, 2, anyOf(
+                equalTo("Another <xxx>sentence</xxx> with"),
+                equalTo("Another <xxx>sentence</xxx> with even more words. ")
+            ));
+        }
+    }
+
+    public void testHighlighterWithWordBoundaryScanner() throws Exception {
+        assertAcked(prepareCreate("test").addMapping("type1", type1TermVectorMapping()));
+        ensureGreen();
+
+        indexRandom(true, client().prepareIndex("test", "type1")
+                .setSource("field1", "some quick and hairy brown:fox jumped over the lazy dog"));
+
+        logger.info("--> highlighting and searching on 'field' with word boundary_scanner");
+        for (String type : new String[] {"unified", "fvh"}) {
+            SearchSourceBuilder source = searchSource()
+                    .query(termQuery("field1", "some"))
+                    .highlighter(highlight()
+                            .field("field1", 23, 1)
+                            .highlighterType(type)
+                            .preTags("<xxx>").postTags("</xxx>")
+                            .boundaryScannerType(BoundaryScannerType.WORD));
+
+            SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
+
+            assertHighlight(searchResponse, 0, "field1", 0, 1, anyOf(
+                equalTo("<xxx>some</xxx> quick and hairy brown"),
+                equalTo("<xxx>some</xxx>")
+            ));
+        }
+    }
+
+    public void testHighlighterWithWordBoundaryScannerAndLocale() throws Exception {
+        assertAcked(prepareCreate("test").addMapping("type1", type1TermVectorMapping()));
+        ensureGreen();
+
+        indexRandom(true, client().prepareIndex("test", "type1")
+                .setSource("field1", "some quick and hairy brown:fox jumped over the lazy dog"));
+
+        for (String type : new String[] {"unified", "fvh"}) {
+            SearchSourceBuilder source = searchSource()
+                .query(termQuery("field1", "some"))
+                .highlighter(highlight()
+                    .field("field1", 23, 1)
+                    .highlighterType(type)
+                    .preTags("<xxx>").postTags("</xxx>")
+                    .boundaryScannerType(BoundaryScannerType.WORD)
+                    .boundaryScannerLocale(Locale.ENGLISH.toLanguageTag()));
+
+            SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
+
+            assertHighlight(searchResponse, 0, "field1", 0, 1, anyOf(
+                equalTo("<xxx>some</xxx> quick and hairy brown"),
+                equalTo("<xxx>some</xxx>")
+            ));
+        }
     }
 
     /**
@@ -1056,13 +1200,16 @@ public void testSameContent() throws Exception {
         }
         indexRandom(true, indexRequestBuilders);
 
-        SearchResponse search = client().prepareSearch()
+        for (String type : UNIFIED_AND_NULL) {
+            SearchResponse search = client().prepareSearch()
                 .setQuery(matchQuery("title", "bug"))
-                .highlighter(new HighlightBuilder().field("title", -1, 0))
+                .highlighter(new HighlightBuilder().field("title", -1, 0).highlighterType(type))
                 .get();
 
-        for (int i = 0; i < 5; i++) {
-            assertHighlight(search, i, "title", 0, 1, equalTo("This is a test on the highlighting <em>bug</em> present in elasticsearch"));
+            for (int i = 0; i < 5; i++) {
+                assertHighlight(search, i, "title", 0, 1, equalTo("This is a test on the highlighting <em>bug</em> " +
+                    "present in elasticsearch"));
+            }
         }
     }
 
@@ -1099,14 +1246,16 @@ public void testEscapeHtml() throws Exception {
         }
         indexRandom(true, indexRequestBuilders);
 
-        SearchResponse search = client().prepareSearch()
+        for (String type : UNIFIED_AND_NULL) {
+            SearchResponse search = client().prepareSearch()
                 .setQuery(matchQuery("title", "test"))
-                .highlighter(new HighlightBuilder().encoder("html").field("title", 50, 1, 10))
+                .highlighter(new HighlightBuilder().encoder("html").field("title", 50, 1, 10).highlighterType(type))
                 .get();
 
-        for (int i = 0; i < indexRequestBuilders.length; i++) {
-            assertHighlight(search, i, "title", 0, 1,
-                    equalTo("This is a html escaping highlighting <em>test</em> for *&amp;? elasticsearch"));
+            for (int i = 0; i < indexRequestBuilders.length; i++) {
+                assertHighlight(search, i, "title", 0, 1,
+                    startsWith("This is a html escaping highlighting <em>test</em> for *&amp;?"));
+            }
         }
     }
 
@@ -1151,21 +1300,24 @@ public void testMultiMapperVectorWithStore() throws Exception {
         ensureGreen();
         client().prepareIndex("test", "type1", "1").setSource("title", "this is a test").get();
         refresh();
-        // simple search on body with standard analyzer with a simple field query
-        SearchResponse search = client().prepareSearch()
+
+        for (String type : UNIFIED_AND_NULL) {
+            // simple search on body with standard analyzer with a simple field query
+            SearchResponse search = client().prepareSearch()
                 .setQuery(matchQuery("title", "this is a test"))
-                .highlighter(new HighlightBuilder().encoder("html").field("title", 50, 1))
+                .highlighter(new HighlightBuilder().encoder("html").field("title", 50, 1).highlighterType(type))
                 .get();
 
-        assertHighlight(search, 0, "title", 0, 1, equalTo("this is a <em>test</em>"));
+            assertHighlight(search, 0, "title", 0, 1, equalTo("this is a <em>test</em>"));
 
-        // search on title.key and highlight on title
-        search = client().prepareSearch()
+            // search on title.key and highlight on title
+            search = client().prepareSearch()
                 .setQuery(matchQuery("title.key", "this is a test"))
-                .highlighter(new HighlightBuilder().encoder("html").field("title.key", 50, 1))
+                .highlighter(new HighlightBuilder().encoder("html").field("title.key", 50, 1).highlighterType(type))
                 .get();
 
-        assertHighlight(search, 0, "title.key", 0, 1, equalTo("<em>this</em> <em>is</em> <em>a</em> <em>test</em>"));
+            assertHighlight(search, 0, "title.key", 0, 1, equalTo("<em>this</em> <em>is</em> <em>a</em> <em>test</em>"));
+        }
     }
 
     public void testMultiMapperVectorFromSource() throws Exception {
@@ -1190,21 +1342,23 @@ public void testMultiMapperVectorFromSource() throws Exception {
         client().prepareIndex("test", "type1", "1").setSource("title", "this is a test").get();
         refresh();
 
-        // simple search on body with standard analyzer with a simple field query
-        SearchResponse search = client().prepareSearch()
+        for (String type : UNIFIED_AND_NULL) {
+            // simple search on body with standard analyzer with a simple field query
+            SearchResponse search = client().prepareSearch()
                 .setQuery(matchQuery("title", "this is a test"))
-                .highlighter(new HighlightBuilder().encoder("html").field("title", 50, 1))
+                .highlighter(new HighlightBuilder().encoder("html").field("title", 50, 1).highlighterType(type))
                 .get();
 
-        assertHighlight(search, 0, "title", 0, 1, equalTo("this is a <em>test</em>"));
+            assertHighlight(search, 0, "title", 0, 1, equalTo("this is a <em>test</em>"));
 
-        // search on title.key and highlight on title.key
-        search = client().prepareSearch()
+            // search on title.key and highlight on title.key
+            search = client().prepareSearch()
                 .setQuery(matchQuery("title.key", "this is a test"))
-                .highlighter(new HighlightBuilder().encoder("html").field("title.key", 50, 1))
+                .highlighter(new HighlightBuilder().encoder("html").field("title.key", 50, 1).highlighterType(type))
                 .get();
 
-        assertHighlight(search, 0, "title.key", 0, 1, equalTo("<em>this</em> <em>is</em> <em>a</em> <em>test</em>"));
+            assertHighlight(search, 0, "title.key", 0, 1, equalTo("<em>this</em> <em>is</em> <em>a</em> <em>test</em>"));
+        }
     }
 
     public void testMultiMapperNoVectorWithStore() throws Exception {
@@ -1230,21 +1384,24 @@ public void testMultiMapperNoVectorWithStore() throws Exception {
         client().prepareIndex("test", "type1", "1").setSource("title", "this is a test").get();
         refresh();
 
-        // simple search on body with standard analyzer with a simple field query
-        SearchResponse search = client().prepareSearch()
+
+        for (String type : UNIFIED_AND_NULL) {
+            // simple search on body with standard analyzer with a simple field query
+            SearchResponse search = client().prepareSearch()
                 .setQuery(matchQuery("title", "this is a test"))
-                .highlighter(new HighlightBuilder().encoder("html").field("title", 50, 1))
+                .highlighter(new HighlightBuilder().encoder("html").field("title", 50, 1).highlighterType(type))
                 .get();
 
-        assertHighlight(search, 0, "title", 0, 1, equalTo("this is a <em>test</em>"));
+            assertHighlight(search, 0, "title", 0, 1, equalTo("this is a <em>test</em>"));
 
-        // search on title.key and highlight on title
-        search = client().prepareSearch()
+            // search on title.key and highlight on title
+            search = client().prepareSearch()
                 .setQuery(matchQuery("title.key", "this is a test"))
-                .highlighter(new HighlightBuilder().encoder("html").field("title.key", 50, 1))
+                .highlighter(new HighlightBuilder().encoder("html").field("title.key", 50, 1).highlighterType(type))
                 .get();
 
-        assertHighlight(search, 0, "title.key", 0, 1, equalTo("<em>this</em> <em>is</em> <em>a</em> <em>test</em>"));
+            assertHighlight(search, 0, "title.key", 0, 1, equalTo("<em>this</em> <em>is</em> <em>a</em> <em>test</em>"));
+        }
     }
 
     public void testMultiMapperNoVectorFromSource() throws Exception {
@@ -1269,21 +1426,23 @@ public void testMultiMapperNoVectorFromSource() throws Exception {
         client().prepareIndex("test", "type1", "1").setSource("title", "this is a test").get();
         refresh();
 
-        // simple search on body with standard analyzer with a simple field query
-        SearchResponse search = client().prepareSearch()
+        for (String type : UNIFIED_AND_NULL) {
+            // simple search on body with standard analyzer with a simple field query
+            SearchResponse search = client().prepareSearch()
                 .setQuery(matchQuery("title", "this is a test"))
-                .highlighter(new HighlightBuilder().encoder("html").field("title", 50, 1))
+                .highlighter(new HighlightBuilder().encoder("html").field("title", 50, 1).highlighterType(type))
                 .get();
 
-        assertHighlight(search, 0, "title", 0, 1, equalTo("this is a <em>test</em>"));
+            assertHighlight(search, 0, "title", 0, 1, equalTo("this is a <em>test</em>"));
 
-        // search on title.key and highlight on title.key
-        search = client().prepareSearch()
+            // search on title.key and highlight on title.key
+            search = client().prepareSearch()
                 .setQuery(matchQuery("title.key", "this is a test"))
-                .highlighter(new HighlightBuilder().encoder("html").field("title.key", 50, 1))
+                .highlighter(new HighlightBuilder().encoder("html").field("title.key", 50, 1).highlighterType(type))
                 .get();
 
-        assertHighlight(search, 0, "title.key", 0, 1, equalTo("<em>this</em> <em>is</em> <em>a</em> <em>test</em>"));
+            assertHighlight(search, 0, "title.key", 0, 1, equalTo("<em>this</em> <em>is</em> <em>a</em> <em>test</em>"));
+        }
     }
 
     public void testFastVectorHighlighterShouldFailIfNoTermVectors() throws Exception {
@@ -1390,14 +1549,16 @@ public void testBoostingQuery() {
                 .setSource("field1", "this is a test", "field2", "The quick brown fox jumps over the lazy dog").get();
         refresh();
 
-        logger.info("--> highlighting and searching on field1");
-        SearchSourceBuilder source = searchSource()
+        for (String type : UNIFIED_AND_NULL) {
+            logger.info("--> highlighting and searching on field1");
+            SearchSourceBuilder source = searchSource()
                 .query(boostingQuery(termQuery("field2", "brown"), termQuery("field2", "foobar")).negativeBoost(0.5f))
-                .highlighter(highlight().field("field2").order("score").preTags("<x>").postTags("</x>"));
+                .highlighter(highlight().field("field2").order("score").preTags("<x>").postTags("</x>").highlighterType(type));
 
-        SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
+            SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
 
-        assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The quick <x>brown</x> fox jumps over the lazy dog"));
+            assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The quick <x>brown</x> fox jumps over the lazy dog"));
+        }
     }
 
     public void testBoostingQueryTermVector() throws IOException {
@@ -1428,12 +1589,14 @@ public void testCommonTermsQuery() {
         refresh();
 
         logger.info("--> highlighting and searching on field1");
-        SearchSourceBuilder source = searchSource()
+        for (String type : UNIFIED_AND_NULL) {
+            SearchSourceBuilder source = searchSource()
                 .query(commonTermsQuery("field2", "quick brown").cutoffFrequency(100))
-                .highlighter(highlight().field("field2").order("score").preTags("<x>").postTags("</x>"));
+                .highlighter(highlight().field("field2").order("score").preTags("<x>").postTags("</x>").highlighterType(type));
 
-        SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
-        assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The <x>quick</x> <x>brown</x> fox jumps over the lazy dog"));
+            SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
+            assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The <x>quick</x> <x>brown</x> fox jumps over the lazy dog"));
+        }
     }
 
     public void testCommonTermsTermVector() throws IOException {
@@ -1473,63 +1636,99 @@ public void testPhrasePrefix() throws IOException {
         client().prepareIndex("test", "type1", "1").setSource("field1", "The quick browse button is a fancy thing, right bro?").get();
         refresh();
         logger.info("--> highlighting and searching on field0");
-        SearchSourceBuilder source = searchSource()
-                .query(matchPhrasePrefixQuery("field0", "quick bro"))
-                .highlighter(highlight().field("field0").order("score").preTags("<x>").postTags("</x>"));
 
-        SearchResponse searchResponse = client().search(searchRequest("test").source(source)).actionGet();
+        for (String type : UNIFIED_AND_NULL) {
+            SearchSourceBuilder source = searchSource()
+                .query(matchPhrasePrefixQuery("field0", "bro"))
+                .highlighter(highlight().field("field0").order("score").preTags("<x>").postTags("</x>").highlighterType(type));
 
-        assertHighlight(searchResponse, 0, "field0", 0, 1, equalTo("The <x>quick</x> <x>brown</x> fox jumps over the lazy dog"));
+            SearchResponse searchResponse = client().search(searchRequest("test").source(source)).actionGet();
 
-        logger.info("--> highlighting and searching on field1");
-        source = searchSource()
+            assertHighlight(searchResponse, 0, "field0", 0, 1, equalTo("The quick <x>brown</x> fox jumps over the lazy dog"));
+
+            source = searchSource()
+                .query(matchPhrasePrefixQuery("field0", "quick bro"))
+                .highlighter(highlight().field("field0").order("score").preTags("<x>").postTags("</x>").highlighterType(type));
+
+            searchResponse = client().search(searchRequest("test").source(source)).actionGet();
+
+            assertHighlight(searchResponse, 0, "field0", 0, 1, equalTo("The <x>quick</x> <x>brown</x> fox jumps over the lazy dog"));
+
+            logger.info("--> highlighting and searching on field1");
+            source = searchSource()
+                .query(boolQuery()
+                    .should(matchPhrasePrefixQuery("field1", "test"))
+                    .should(matchPhrasePrefixQuery("field1", "bro"))
+                )
+                .highlighter(highlight().field("field1").order("score").preTags("<x>").postTags("</x>").highlighterType(type));
+
+            searchResponse = client().search(searchRequest("test").source(source)).actionGet();
+            assertThat(searchResponse.getHits().totalHits, equalTo(2L));
+            for (int i = 0; i < 2; i++) {
+                assertHighlight(searchResponse, i, "field1", 0, 1, anyOf(
+                    equalTo("The quick <x>browse</x> button is a fancy thing, right <x>bro</x>?"),
+                    equalTo("The quick <x>brown</x> fox jumps over the lazy dog")));
+            }
+
+            source = searchSource()
                 .query(matchPhrasePrefixQuery("field1", "quick bro"))
-                .highlighter(highlight().field("field1").order("score").preTags("<x>").postTags("</x>"));
+                .highlighter(highlight().field("field1").order("score").preTags("<x>").postTags("</x>").highlighterType(type));
 
-        searchResponse = client().search(searchRequest("test").source(source)).actionGet();
+            searchResponse = client().search(searchRequest("test").source(source)).actionGet();
 
-        assertHighlight(searchResponse, 0, "field1", 0, 1, anyOf(
+            assertHighlight(searchResponse, 0, "field1", 0, 1, anyOf(
                 equalTo("The <x>quick browse</x> button is a fancy thing, right bro?"),
-                equalTo("The <x>quick brown</x> fox jumps over the lazy dog")));
-        assertHighlight(searchResponse, 1, "field1", 0, 1, anyOf(
+                equalTo("The <x>quick brown</x> fox jumps over the lazy dog"),
+                equalTo("The <x>quick</x> <x>browse</x> button is a fancy thing, right bro?"),
+                equalTo("The <x>quick</x> <x>brown</x> fox jumps over the lazy dog")));
+            assertHighlight(searchResponse, 1, "field1", 0, 1, anyOf(
                 equalTo("The <x>quick browse</x> button is a fancy thing, right bro?"),
-                equalTo("The <x>quick brown</x> fox jumps over the lazy dog")));
+                equalTo("The <x>quick brown</x> fox jumps over the lazy dog"),
+                equalTo("The <x>quick</x> <x>browse</x> button is a fancy thing, right bro?"),
+                equalTo("The <x>quick</x> <x>brown</x> fox jumps over the lazy dog")));
 
-        // with synonyms
-        client().prepareIndex("test", "type2", "0").setSource(
+            // with synonyms
+            client().prepareIndex("test", "type2", "0").setSource(
                 "field4", "The quick brown fox jumps over the lazy dog",
                 "field3", "The quick brown fox jumps over the lazy dog").get();
-        client().prepareIndex("test", "type2", "1").setSource(
+            client().prepareIndex("test", "type2", "1").setSource(
                 "field4", "The quick browse button is a fancy thing, right bro?").get();
-        client().prepareIndex("test", "type2", "2").setSource(
+            client().prepareIndex("test", "type2", "2").setSource(
                 "field4", "a quick fast blue car").get();
-        refresh();
+            refresh();
 
-        source = searchSource().postFilter(typeQuery("type2")).query(matchPhrasePrefixQuery("field3", "fast bro"))
-                .highlighter(highlight().field("field3").order("score").preTags("<x>").postTags("</x>"));
+            source = searchSource().postFilter(typeQuery("type2")).query(matchPhrasePrefixQuery("field3", "fast bro"))
+                .highlighter(highlight().field("field3").order("score").preTags("<x>").postTags("</x>").highlighterType(type));
 
-        searchResponse = client().search(searchRequest("test").source(source)).actionGet();
+            searchResponse = client().search(searchRequest("test").source(source)).actionGet();
 
-        assertHighlight(searchResponse, 0, "field3", 0, 1, equalTo("The <x>quick</x> <x>brown</x> fox jumps over the lazy dog"));
+            assertHighlight(searchResponse, 0, "field3", 0, 1, equalTo("The <x>quick</x> <x>brown</x> fox jumps over the lazy dog"));
 
-        logger.info("--> highlighting and searching on field4");
-        source = searchSource().postFilter(typeQuery("type2")).query(matchPhrasePrefixQuery("field4", "the fast bro"))
-                .highlighter(highlight().field("field4").order("score").preTags("<x>").postTags("</x>"));
-        searchResponse = client().search(searchRequest("test").source(source)).actionGet();
+            logger.info("--> highlighting and searching on field4");
+            source = searchSource().postFilter(typeQuery("type2")).query(matchPhrasePrefixQuery("field4", "the fast bro"))
+                .highlighter(highlight().field("field4").order("score").preTags("<x>").postTags("</x>").highlighterType(type));
+            searchResponse = client().search(searchRequest("test").source(source)).actionGet();
 
-        assertHighlight(searchResponse, 0, "field4", 0, 1, anyOf(
+            assertHighlight(searchResponse, 0, "field4", 0, 1, anyOf(
                 equalTo("<x>The quick browse</x> button is a fancy thing, right bro?"),
-                equalTo("<x>The quick brown</x> fox jumps over the lazy dog")));
-        assertHighlight(searchResponse, 1, "field4", 0, 1, anyOf(
+                equalTo("<x>The quick brown</x> fox jumps over the lazy dog"),
+                equalTo("<x>The</x> <x>quick</x> <x>browse</x> button is a fancy thing, right bro?"),
+                equalTo("<x>The</x> <x>quick</x> <x>brown</x> fox jumps over the lazy dog")));
+            assertHighlight(searchResponse, 1, "field4", 0, 1, anyOf(
                 equalTo("<x>The quick browse</x> button is a fancy thing, right bro?"),
-                equalTo("<x>The quick brown</x> fox jumps over the lazy dog")));
-
-        logger.info("--> highlighting and searching on field4");
-        source = searchSource().postFilter(typeQuery("type2")).query(matchPhrasePrefixQuery("field4", "a fast quick blue ca"))
-                .highlighter(highlight().field("field4").order("score").preTags("<x>").postTags("</x>"));
-        searchResponse = client().search(searchRequest("test").source(source)).actionGet();
-
-        assertHighlight(searchResponse, 0, "field4", 0, 1, equalTo("<x>a quick fast blue car</x>"));
+                equalTo("<x>The quick brown</x> fox jumps over the lazy dog"),
+                equalTo("<x>The</x> <x>quick</x> <x>browse</x> button is a fancy thing, right bro?"),
+                equalTo("<x>The</x> <x>quick</x> <x>brown</x> fox jumps over the lazy dog")));
+
+            logger.info("--> highlighting and searching on field4");
+            source = searchSource().postFilter(typeQuery("type2")).query(matchPhrasePrefixQuery("field4", "a fast quick blue ca"))
+                .highlighter(highlight().field("field4").order("score").preTags("<x>").postTags("</x>").highlighterType(type));
+            searchResponse = client().search(searchRequest("test").source(source)).actionGet();
+
+            assertHighlight(searchResponse, 0, "field4", 0, 1,
+                anyOf(equalTo("<x>a quick fast blue car</x>"),
+                      equalTo("<x>a</x> <x>quick</x> <x>fast</x> <x>blue</x> <x>car</x>")));
+        }
     }
 
     public void testPlainHighlightDifferentFragmenter() throws Exception {
@@ -1537,7 +1736,7 @@ public void testPlainHighlightDifferentFragmenter() throws Exception {
                 .addMapping("type1", "tags", "type=text"));
         ensureGreen();
         client().prepareIndex("test", "type1", "1")
-                .setSource(jsonBuilder().startObject().field("tags",
+                .setSource(jsonBuilder().startObject().array("tags",
                         "this is a really long tag i would like to highlight",
                         "here is another one that is very long tag and has the tag token near the end").endObject()).get();
         refresh();
@@ -1620,13 +1819,15 @@ public void testMissingStoredField() throws Exception {
                         .endObject()).get();
         refresh();
 
-        // This query used to fail when the field to highlight was absent
-        SearchResponse response = client().prepareSearch("test")
+        for (String type : UNIFIED_AND_NULL) {
+            // This query used to fail when the field to highlight was absent
+            SearchResponse response = client().prepareSearch("test")
                 .setQuery(QueryBuilders.matchQuery("field", "highlight").type(MatchQuery.Type.BOOLEAN))
                 .highlighter(
-                        new HighlightBuilder().field(new HighlightBuilder.Field("highlight_field").fragmentSize(-1).numOfFragments(1)
-                                .fragmenter("simple"))).get();
-        assertThat(response.getHits().hits()[0].highlightFields().isEmpty(), equalTo(true));
+                    new HighlightBuilder().field(new HighlightBuilder.Field("highlight_field").fragmentSize(-1).numOfFragments(1)
+                        .fragmenter("simple")).highlighterType(type)).get();
+            assertThat(response.getHits().hits()[0].highlightFields().isEmpty(), equalTo(true));
+        }
     }
 
     // Issue #3211
@@ -1671,11 +1872,13 @@ public void testResetTwice() throws Exception {
                 .setSource("text", "elasticsearch test").get();
         refresh();
 
-        SearchResponse response = client().prepareSearch("test")
+        for (String type : UNIFIED_AND_NULL) {
+            SearchResponse response = client().prepareSearch("test")
                 .setQuery(QueryBuilders.matchQuery("text", "test").type(MatchQuery.Type.BOOLEAN))
-                .highlighter(new HighlightBuilder().field("text")).execute().actionGet();
-        // PatternAnalyzer will throw an exception if it is resetted twice
-        assertHitCount(response, 1L);
+                .highlighter(new HighlightBuilder().field("text").highlighterType(type)).execute().actionGet();
+            // PatternAnalyzer will throw an exception if it is resetted twice
+            assertHitCount(response, 1L);
+        }
     }
 
     public void testHighlightUsesHighlightQuery() throws IOException {
@@ -1686,55 +1889,30 @@ public void testHighlightUsesHighlightQuery() throws IOException {
         index("test", "type1", "1", "text", "Testing the highlight query feature");
         refresh();
 
-        HighlightBuilder.Field field = new HighlightBuilder.Field("text");
-
-        HighlightBuilder highlightBuilder = new HighlightBuilder().field(field);
-        SearchRequestBuilder search = client().prepareSearch("test").setQuery(QueryBuilders.matchQuery("text", "testing"))
+        for (String type : ALL_TYPES) {
+            HighlightBuilder.Field field = new HighlightBuilder.Field("text");
+            HighlightBuilder highlightBuilder = new HighlightBuilder().field(field).highlighterType(type);
+            SearchRequestBuilder search = client().prepareSearch("test").setQuery(QueryBuilders.matchQuery("text", "testing"))
                 .highlighter(highlightBuilder);
-        Matcher<String> searchQueryMatcher = equalTo("<em>Testing</em> the highlight query feature");
-
-        field.highlighterType("plain");
-        SearchResponse response = search.get();
-        assertHighlight(response, 0, "text", 0, searchQueryMatcher);
-        field.highlighterType("fvh");
-        response = search.get();
-        assertHighlight(response, 0, "text", 0, searchQueryMatcher);
-        field.highlighterType("postings");
-        response = search.get();
-        assertHighlight(response, 0, "text", 0, searchQueryMatcher);
-
-        field = new HighlightBuilder.Field("text");
-
-        Matcher<String> hlQueryMatcher = equalTo("Testing the highlight <em>query</em> feature");
-        field.highlightQuery(matchQuery("text", "query"));
-        highlightBuilder = new HighlightBuilder().field(field);
-        search = client().prepareSearch("test").setQuery(QueryBuilders.matchQuery("text", "testing")).highlighter(highlightBuilder);
-
-        field.highlighterType("fvh");
-        response = search.get();
-        assertHighlight(response, 0, "text", 0, hlQueryMatcher);
-
-        field.highlighterType("plain");
-        response = search.get();
-        assertHighlight(response, 0, "text", 0, hlQueryMatcher);
-
-        field.highlighterType("postings");
-        response = search.get();
-        assertHighlight(response, 0, "text", 0, hlQueryMatcher);
-
-        // Make sure the highlightQuery is taken into account when it is set on the highlight context instead of the field
-        highlightBuilder.highlightQuery(matchQuery("text", "query"));
-        field.highlighterType("fvh").highlightQuery(null);
-        response = search.get();
-        assertHighlight(response, 0, "text", 0, hlQueryMatcher);
-
-        field.highlighterType("plain");
-        response = search.get();
-        assertHighlight(response, 0, "text", 0, hlQueryMatcher);
-
-        field.highlighterType("postings");
-        response = search.get();
-        assertHighlight(response, 0, "text", 0, hlQueryMatcher);
+            Matcher<String> searchQueryMatcher = equalTo("<em>Testing</em> the highlight query feature");
+
+            SearchResponse response = search.get();
+            assertHighlight(response, 0, "text", 0, searchQueryMatcher);
+            field = new HighlightBuilder.Field("text");
+
+            Matcher<String> hlQueryMatcher = equalTo("Testing the highlight <em>query</em> feature");
+            field.highlightQuery(matchQuery("text", "query"));
+            highlightBuilder = new HighlightBuilder().field(field);
+            search = client().prepareSearch("test").setQuery(QueryBuilders.matchQuery("text", "testing")).highlighter(highlightBuilder);
+            response = search.get();
+            assertHighlight(response, 0, "text", 0, hlQueryMatcher);
+
+            // Make sure the highlightQuery is taken into account when it is set on the highlight context instead of the field
+            highlightBuilder.highlightQuery(matchQuery("text", "query"));
+            field.highlighterType(type).highlightQuery(null);
+            response = search.get();
+            assertHighlight(response, 0, "text", 0, hlQueryMatcher);
+        }
     }
 
     private static String randomStoreField() {
@@ -1769,6 +1947,10 @@ public void testHighlightNoMatchSize() throws IOException {
         response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
         assertNotHighlighted(response, 0, "text");
 
+        field.highlighterType("unified");
+        response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
+        assertNotHighlighted(response, 0, "text");
+
         // When noMatchSize is set to 0 you also shouldn't get any
         field.highlighterType("plain").noMatchSize(0);
         response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
@@ -1782,6 +1964,10 @@ public void testHighlightNoMatchSize() throws IOException {
         response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
         assertNotHighlighted(response, 0, "text");
 
+        field.highlighterType("unified");
+        response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
+        assertNotHighlighted(response, 0, "text");
+
         // When noMatchSize is between 0 and the size of the string
         field.highlighterType("plain").noMatchSize(21);
         response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
@@ -1792,6 +1978,11 @@ public void testHighlightNoMatchSize() throws IOException {
         response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
         assertHighlight(response, 0, "text", 0, 1, equalTo("I am pretty long so some"));
 
+        // Unified hl also works but the fragment is longer than the plain highlighter because of the boundary is the word
+        field.highlighterType("unified");
+        response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
+        assertHighlight(response, 0, "text", 0, 1, equalTo("I am pretty long so some"));
+
         // Postings hl also works but the fragment is the whole first sentence (size ignored)
         field.highlighterType("postings");
         response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
@@ -1806,6 +1997,10 @@ public void testHighlightNoMatchSize() throws IOException {
         response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
         assertHighlight(response, 0, "text", 0, 1, equalTo(text));
 
+        field.highlighterType("unified");
+        response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
+        assertHighlight(response, 0, "text", 0, 1, equalTo(text));
+
         //no difference using postings hl as the noMatchSize is ignored (just needs to be greater than 0)
         field.highlighterType("postings");
         response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
@@ -1820,6 +2015,11 @@ public void testHighlightNoMatchSize() throws IOException {
         response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
         assertHighlight(response, 0, "text", 0, 1, equalTo(text));
 
+        // unified hl returns the first sentence as the noMatchSize does not cross sentence boundary.
+        field.highlighterType("unified");
+        response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
+        assertHighlight(response, 0, "text", 0, 1, equalTo(text));
+
         //no difference using postings hl as the noMatchSize is ignored (just needs to be greater than 0)
         field.highlighterType("postings");
         response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
@@ -1834,6 +2034,10 @@ public void testHighlightNoMatchSize() throws IOException {
         response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field).noMatchSize(21)).get();
         assertHighlight(response, 0, "text", 0, 1, equalTo("I am pretty long so some"));
 
+        field.highlighterType("unified");
+        response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field).noMatchSize(21)).get();
+        assertHighlight(response, 0, "text", 0, 1, equalTo("I am pretty long so some"));
+
         field.highlighterType("postings");
         response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field).noMatchSize(21)).get();
         assertHighlight(response, 0, "text", 0, 1, equalTo("I am pretty long so some of me should get cut off."));
@@ -1850,6 +2054,10 @@ public void testHighlightNoMatchSize() throws IOException {
         field.highlighterType("postings");
         response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
         assertNotHighlighted(response, 0, "text");
+
+        field.highlighterType("unified");
+        response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
+        assertNotHighlighted(response, 0, "text");
     }
 
     public void testHighlightNoMatchSizeWithMultivaluedFields() throws IOException {
@@ -1875,6 +2083,10 @@ public void testHighlightNoMatchSizeWithMultivaluedFields() throws IOException {
         response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
         assertHighlight(response, 0, "text", 0, 1, equalTo("I am pretty long so some"));
 
+        field.highlighterType("unified");
+        response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
+        assertHighlight(response, 0, "text", 0, 1, equalTo("I am pretty long so some"));
+
         // Postings hl also works but the fragment is the whole first sentence (size ignored)
         field.highlighterType("postings");
         response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
@@ -1903,6 +2115,13 @@ public void testHighlightNoMatchSizeWithMultivaluedFields() throws IOException {
                 .highlighter(new HighlightBuilder().field(field)).get();
         assertNotHighlighted(response, 0, "text");
 
+        // except for the unified highlighter which starts from the first string with actual content
+        field.highlighterType("unified");
+        response = client().prepareSearch("test")
+            .setQuery(idsQueryBuilder)
+            .highlighter(new HighlightBuilder().field(field)).get();
+        assertHighlight(response, 0, "text", 0, 1, equalTo("I am short"));
+
         // But if the field was actually empty then you should get no highlighting field
         index("test", "type1", "3", "text", new String[] {});
         refresh();
@@ -1925,6 +2144,12 @@ public void testHighlightNoMatchSizeWithMultivaluedFields() throws IOException {
                 .highlighter(new HighlightBuilder().field(field)).get();
         assertNotHighlighted(response, 0, "text");
 
+        field.highlighterType("unified");
+        response = client().prepareSearch("test")
+            .setQuery(idsQueryBuilder)
+            .highlighter(new HighlightBuilder().field(field)).get();
+        assertNotHighlighted(response, 0, "text");
+
         // Same for if the field doesn't even exist on the document
         index("test", "type1", "4");
         refresh();
@@ -1942,7 +2167,7 @@ public void testHighlightNoMatchSizeWithMultivaluedFields() throws IOException {
                 .highlighter(new HighlightBuilder().field(field)).get();
         assertNotHighlighted(response, 0, "text");
 
-        field.highlighterType("fvh");
+        field.highlighterType("unified");
         response = client().prepareSearch("test")
                 .setQuery(idsQueryBuilder)
                 .highlighter(new HighlightBuilder().field(field)).get();
@@ -1962,6 +2187,10 @@ public void testHighlightNoMatchSizeWithMultivaluedFields() throws IOException {
         field.highlighterType("postings");
         response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
         assertNotHighlighted(response, 0, "text");
+
+        field.highlighterType("unified");
+        response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
+        assertNotHighlighted(response, 0, "text");
     }
 
     public void testHighlightNoMatchSizeNumberOfFragments() throws IOException {
@@ -1988,6 +2217,11 @@ public void testHighlightNoMatchSizeNumberOfFragments() throws IOException {
         response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
         assertHighlight(response, 0, "text", 0, 1, equalTo("This is the first sentence"));
 
+        field.highlighterType("unified");
+        response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
+        assertHighlight(response, 0, "text", 0, 1, equalTo("This is the first sentence"));
+
+
         // Postings hl also works but the fragment is the whole first sentence (size ignored)
         field.highlighterType("postings");
         response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
@@ -2009,6 +2243,11 @@ public void testHighlightNoMatchSizeNumberOfFragments() throws IOException {
         response = client().prepareSearch("test").setQuery(queryBuilder).highlighter(new HighlightBuilder().field(field)).get();
         assertHighlight(response, 0, "text", 0, 2, equalTo("This is the <em>third</em> sentence. This is the fourth sentence."));
         assertHighlight(response, 0, "text", 1, 2, equalTo("This is the <em>fifth</em> sentence"));
+
+        field.highlighterType("unified");
+        response = client().prepareSearch("test").setQuery(queryBuilder).highlighter(new HighlightBuilder().field(field)).get();
+        assertHighlight(response, 0, "text", 0, 2, equalTo("This is the <em>third</em> sentence. This is the fourth sentence."));
+        assertHighlight(response, 0, "text", 1, 2, equalTo("This is the <em>fifth</em> sentence"));
     }
 
     public void testPostingsHighlighter() throws Exception {
@@ -2019,55 +2258,62 @@ public void testPostingsHighlighter() throws Exception {
                 .setSource("field1", "this is a test", "field2", "The quick brown fox jumps over the lazy quick dog").get();
         refresh();
 
-        logger.info("--> highlighting and searching on field1");
-        SearchSourceBuilder source = searchSource()
+        for (String type : UNIFIED_AND_NULL) {
+            logger.info("--> highlighting and searching on field1");
+            SearchSourceBuilder source = searchSource()
                 .query(termQuery("field1", "test"))
-                .highlighter(highlight().field("field1").preTags("<xxx>").postTags("</xxx>"));
-        SearchResponse searchResponse = client().search(searchRequest("test").source(source)).actionGet();
+                .highlighter(highlight().field("field1").preTags("<xxx>").postTags("</xxx>").highlighterType(type));
+            SearchResponse searchResponse = client().search(searchRequest("test").source(source)).actionGet();
 
-        assertHighlight(searchResponse, 0, "field1", 0, 1, equalTo("this is a <xxx>test</xxx>"));
+            assertHighlight(searchResponse, 0, "field1", 0, 1, equalTo("this is a <xxx>test</xxx>"));
 
-        logger.info("--> searching on field1, highlighting on field1");
-        source = searchSource()
+            logger.info("--> searching on field1, highlighting on field1");
+            source = searchSource()
                 .query(termQuery("field1", "test"))
-                .highlighter(highlight().field("field1").preTags("<xxx>").postTags("</xxx>"));
+                .highlighter(highlight().field("field1").preTags("<xxx>").postTags("</xxx>").highlighterType(type));
 
-        searchResponse = client().search(searchRequest("test").source(source)).actionGet();
+            searchResponse = client().search(searchRequest("test").source(source)).actionGet();
 
-        assertHighlight(searchResponse, 0, "field1", 0, 1, equalTo("this is a <xxx>test</xxx>"));
+            assertHighlight(searchResponse, 0, "field1", 0, 1, equalTo("this is a <xxx>test</xxx>"));
 
-        logger.info("--> searching on field2, highlighting on field2");
-        source = searchSource()
+            logger.info("--> searching on field2, highlighting on field2");
+            source = searchSource()
                 .query(termQuery("field2", "quick"))
-                .highlighter(highlight().field("field2").order("score").preTags("<xxx>").postTags("</xxx>"));
+                .highlighter(highlight().field("field2").order("score").preTags("<xxx>").postTags("</xxx>").highlighterType(type));
 
-        searchResponse = client().search(searchRequest("test").source(source)).actionGet();
+            searchResponse = client().search(searchRequest("test").source(source)).actionGet();
 
-        assertHighlight(searchResponse, 0, "field2", 0, 1,
+            assertHighlight(searchResponse, 0, "field2", 0, 1,
                 equalTo("The <xxx>quick</xxx> brown fox jumps over the lazy <xxx>quick</xxx> dog"));
 
-        logger.info("--> searching on field2, highlighting on field2");
-        source = searchSource()
+            logger.info("--> searching on field2, highlighting on field2");
+            source = searchSource()
                 .query(matchPhraseQuery("field2", "quick brown"))
-                .highlighter(highlight().field("field2").preTags("<xxx>").postTags("</xxx>"));
+                .highlighter(highlight().field("field2").preTags("<xxx>").postTags("</xxx>").highlighterType(type));
 
-        searchResponse = client().search(searchRequest("test").source(source)).actionGet();
+            searchResponse = client().search(searchRequest("test").source(source)).actionGet();
 
-        //phrase query results in highlighting all different terms regardless of their positions
-        assertHighlight(searchResponse, 0, "field2", 0, 1,
-                equalTo("The <xxx>quick</xxx> <xxx>brown</xxx> fox jumps over the lazy <xxx>quick</xxx> dog"));
+            if (type == null) {
+                //phrase query results in highlighting all different terms regardless of their positions
+                assertHighlight(searchResponse, 0, "field2", 0, 1,
+                    equalTo("The <xxx>quick</xxx> <xxx>brown</xxx> fox jumps over the lazy <xxx>quick</xxx> dog"));
+            } else {
+                assertHighlight(searchResponse, 0, "field2", 0, 1,
+                    equalTo("The <xxx>quick</xxx> <xxx>brown</xxx> fox jumps over the lazy quick dog"));
+            }
 
-        //lets fall back to the standard highlighter then, what people would do to highlight query matches
-        logger.info("--> searching on field2, highlighting on field2, falling back to the plain highlighter");
-        source = searchSource()
+            //lets fall back to the standard highlighter then, what people would do to highlight query matches
+            logger.info("--> searching on field2, highlighting on field2, falling back to the plain highlighter");
+            source = searchSource()
                 .query(matchPhraseQuery("_all", "quick brown"))
                 .highlighter(highlight()
                         .field("field2").preTags("<xxx>").postTags("</xxx>").highlighterType("plain").requireFieldMatch(false));
 
-        searchResponse = client().search(searchRequest("test").source(source)).actionGet();
+            searchResponse = client().search(searchRequest("test").source(source)).actionGet();
 
-        assertHighlight(searchResponse, 0, "field2", 0, 1,
+            assertHighlight(searchResponse, 0, "field2", 0, 1,
                 equalTo("The <xxx>quick</xxx> <xxx>brown</xxx> fox jumps over the lazy quick dog"));
+        }
     }
 
     public void testPostingsHighlighterMultipleFields() throws Exception {
@@ -2079,13 +2325,15 @@ public void testPostingsHighlighterMultipleFields() throws Exception {
                 "field2", "The <b>slow<b> brown fox. Second sentence.");
         refresh();
 
-        SearchResponse response = client().prepareSearch("test")
+        for (String type : UNIFIED_AND_NULL) {
+            SearchResponse response = client().prepareSearch("test")
                 .setQuery(QueryBuilders.matchQuery("field1", "fox"))
                 .highlighter(
-                        new HighlightBuilder().field(new HighlightBuilder.Field("field1").preTags("<1>").postTags("</1>")
-                                .requireFieldMatch(true)))
+                    new HighlightBuilder().field(new Field("field1").preTags("<1>").postTags("</1>")
+                        .requireFieldMatch(true).highlighterType(type)))
                 .get();
-        assertHighlight(response, 0, "field1", 0, 1, equalTo("The <b>quick<b> brown <1>fox</1>."));
+            assertHighlight(response, 0, "field1", 0, 1, equalTo("The <b>quick<b> brown <1>fox</1>."));
+        }
     }
 
     public void testPostingsHighlighterNumberOfFragments() throws Exception {
@@ -2099,51 +2347,54 @@ public void testPostingsHighlighterNumberOfFragments() throws Exception {
                     + "The quick brown dog jumps over the lazy fox.").get();
         refresh();
 
-        logger.info("--> highlighting and searching on field1");
-        SearchSourceBuilder source = searchSource()
+
+        for (String type : UNIFIED_AND_NULL) {
+            logger.info("--> highlighting and searching on field1");
+            SearchSourceBuilder source = searchSource()
                 .query(termQuery("field1", "fox"))
                 .highlighter(highlight()
-                        .field(new HighlightBuilder.Field("field1").numOfFragments(5).preTags("<field1>").postTags("</field1>")));
+                    .field(new Field("field1").numOfFragments(5).preTags("<field1>").postTags("</field1>").highlighterType(type)));
 
-        SearchResponse searchResponse = client().search(searchRequest("test").source(source)).actionGet();
+            SearchResponse searchResponse = client().search(searchRequest("test").source(source)).actionGet();
 
-        assertHighlight(searchResponse, 0, "field1", 0, equalTo("The quick brown <field1>fox</field1> jumps over the lazy dog."));
-        assertHighlight(searchResponse, 0, "field1", 1, equalTo("The lazy red <field1>fox</field1> jumps over the quick dog."));
-        assertHighlight(searchResponse, 0, "field1", 2, 3, equalTo("The quick brown dog jumps over the lazy <field1>fox</field1>."));
+            assertHighlight(searchResponse, 0, "field1", 0, equalTo("The quick brown <field1>fox</field1> jumps over the lazy dog."));
+            assertHighlight(searchResponse, 0, "field1", 1, equalTo("The lazy red <field1>fox</field1> jumps over the quick dog."));
+            assertHighlight(searchResponse, 0, "field1", 2, 3, equalTo("The quick brown dog jumps over the lazy <field1>fox</field1>."));
 
-        client().prepareIndex("test", "type1", "2")
-                .setSource("field1", new String[] {
-                        "The quick brown fox jumps over the lazy dog. Second sentence not finished",
-                        "The lazy red fox jumps over the quick dog.",
-                        "The quick brown dog jumps over the lazy fox."}).get();
-        refresh();
+            client().prepareIndex("test", "type1", "2")
+                .setSource("field1", new String[]{
+                    "The quick brown fox jumps over the lazy dog. Second sentence not finished",
+                    "The lazy red fox jumps over the quick dog.",
+                    "The quick brown dog jumps over the lazy fox."}).get();
+            refresh();
 
-        source = searchSource()
+            source = searchSource()
                 .query(termQuery("field1", "fox"))
                 .highlighter(highlight()
-                        .field(new HighlightBuilder.Field("field1").numOfFragments(0).preTags("<field1>").postTags("</field1>")));
+                    .field(new Field("field1").numOfFragments(0).preTags("<field1>").postTags("</field1>").highlighterType(type)));
 
-        searchResponse = client().search(searchRequest("test").source(source)).actionGet();
-        assertHitCount(searchResponse, 2L);
+            searchResponse = client().search(searchRequest("test").source(source)).actionGet();
+            assertHitCount(searchResponse, 2L);
 
-        for (SearchHit searchHit : searchResponse.getHits()) {
-            if ("1".equals(searchHit.id())) {
-                assertHighlight(searchHit, "field1", 0, 1, equalTo("The quick brown <field1>fox</field1> jumps over the lazy dog. "
+            for (SearchHit searchHit : searchResponse.getHits()) {
+                if ("1".equals(searchHit.id())) {
+                    assertHighlight(searchHit, "field1", 0, 1, equalTo("The quick brown <field1>fox</field1> jumps over the lazy dog. "
                         + "The lazy red <field1>fox</field1> jumps over the quick dog. "
                         + "The quick brown dog jumps over the lazy <field1>fox</field1>."));
-            } else if ("2".equals(searchHit.id())) {
-                assertHighlight(searchHit, "field1", 0, 3,
+                } else if ("2".equals(searchHit.id())) {
+                    assertHighlight(searchHit, "field1", 0, 3,
                         equalTo("The quick brown <field1>fox</field1> jumps over the lazy dog. Second sentence not finished"));
-                assertHighlight(searchHit, "field1", 1, 3, equalTo("The lazy red <field1>fox</field1> jumps over the quick dog."));
-                assertHighlight(searchHit, "field1", 2, 3, equalTo("The quick brown dog jumps over the lazy <field1>fox</field1>."));
-            } else {
-                fail("Only hits with id 1 and 2 are returned");
+                    assertHighlight(searchHit, "field1", 1, 3, equalTo("The lazy red <field1>fox</field1> jumps over the quick dog."));
+                    assertHighlight(searchHit, "field1", 2, 3, equalTo("The quick brown dog jumps over the lazy <field1>fox</field1>."));
+                } else {
+                    fail("Only hits with id 1 and 2 are returned");
+                }
             }
         }
     }
 
     public void testMultiMatchQueryHighlight() throws IOException {
-        String[] highlighterTypes = new String[] {"fvh", "plain", "postings"};
+        String[] highlighterTypes = new String[] {"fvh", "plain", "postings", "unified"};
         XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type1")
                 .startObject("_all").field("store", true).field("index_options", "offsets").endObject()
                 .startObject("properties")
@@ -2212,24 +2463,26 @@ public void testPostingsHighlighterOrderByScore() throws Exception {
                             + "This one contains no matches."}).get();
         refresh();
 
-        logger.info("--> highlighting and searching on field1");
-        SearchSourceBuilder source = searchSource()
+        for (String type : UNIFIED_AND_NULL) {
+            logger.info("--> highlighting and searching on field1");
+            SearchSourceBuilder source = searchSource()
                 .query(termQuery("field1", "sentence"))
-                .highlighter(highlight().field("field1").order("score"));
+                .highlighter(highlight().field("field1").order("score").highlighterType(type));
 
-        SearchResponse searchResponse = client().search(searchRequest("test").source(source)).actionGet();
+            SearchResponse searchResponse = client().search(searchRequest("test").source(source)).actionGet();
 
-        Map<String,HighlightField> highlightFieldMap = searchResponse.getHits().getAt(0).highlightFields();
-        assertThat(highlightFieldMap.size(), equalTo(1));
-        HighlightField field1 = highlightFieldMap.get("field1");
-        assertThat(field1.fragments().length, equalTo(5));
-        assertThat(field1.fragments()[0].string(),
+            Map<String, HighlightField> highlightFieldMap = searchResponse.getHits().getAt(0).highlightFields();
+            assertThat(highlightFieldMap.size(), equalTo(1));
+            HighlightField field1 = highlightFieldMap.get("field1");
+            assertThat(field1.fragments().length, equalTo(5));
+            assertThat(field1.fragments()[0].string(),
                 equalTo("This <em>sentence</em> contains three <em>sentence</em> occurrences (<em>sentence</em>)."));
-        assertThat(field1.fragments()[1].string(), equalTo("This <em>sentence</em> contains two <em>sentence</em> matches."));
-        assertThat(field1.fragments()[2].string(), equalTo("This is the second value's first <em>sentence</em>."));
-        assertThat(field1.fragments()[3].string(), equalTo("This <em>sentence</em> contains one match, not that short."));
-        assertThat(field1.fragments()[4].string(),
+            assertThat(field1.fragments()[1].string(), equalTo("This <em>sentence</em> contains two <em>sentence</em> matches."));
+            assertThat(field1.fragments()[2].string(), equalTo("This is the second value's first <em>sentence</em>."));
+            assertThat(field1.fragments()[3].string(), equalTo("This <em>sentence</em> contains one match, not that short."));
+            assertThat(field1.fragments()[4].string(),
                 equalTo("One <em>sentence</em> match here and scored lower since the text is quite long, not that appealing."));
+        }
     }
 
     public void testPostingsHighlighterEscapeHtml() throws Exception {
@@ -2243,12 +2496,15 @@ public void testPostingsHighlighterEscapeHtml() throws Exception {
         }
         indexRandom(true, indexRequestBuilders);
 
-        SearchResponse searchResponse = client().prepareSearch()
+        for (String type : UNIFIED_AND_NULL) {
+            SearchResponse searchResponse = client().prepareSearch()
                 .setQuery(matchQuery("title", "test"))
-                .highlighter(new HighlightBuilder().field("title").encoder("html")).get();
+                .highlighter(new HighlightBuilder().field("title").encoder("html").highlighterType(type)).get();
 
-        for (int i = 0; i < indexRequestBuilders.length; i++) {
-            assertHighlight(searchResponse, i, "title", 0, 1, equalTo("This is a html escaping highlighting <em>test</em> for *&amp;?"));
+            for (int i = 0; i < indexRequestBuilders.length; i++) {
+                assertHighlight(searchResponse, i, "title", 0, 1,
+                    equalTo("This is a html escaping highlighting <em>test</em> for *&amp;?"));
+            }
         }
     }
 
@@ -2272,25 +2528,29 @@ public void testPostingsHighlighterMultiMapperWithStore() throws Exception {
         ensureGreen();
         client().prepareIndex("test", "type1", "1").setSource("title", "this is a test . Second sentence.").get();
         refresh();
-        // simple search on body with standard analyzer with a simple field query
-        SearchResponse searchResponse = client().prepareSearch()
+
+        for (String type : UNIFIED_AND_NULL) {
+            // simple search on body with standard analyzer with a simple field query
+            SearchResponse searchResponse = client().prepareSearch()
                 //lets make sure we analyze the query and we highlight the resulting terms
                 .setQuery(matchQuery("title", "This is a Test"))
-.highlighter(new HighlightBuilder().field("title")).get();
+                .highlighter(new HighlightBuilder().field("title").highlighterType(type)).get();
 
-        assertHitCount(searchResponse, 1L);
-        SearchHit hit = searchResponse.getHits().getAt(0);
-        //stopwords are not highlighted since not indexed
-        assertHighlight(hit, "title", 0, 1, equalTo("this is a <em>test</em> ."));
+            assertHitCount(searchResponse, 1L);
+            SearchHit hit = searchResponse.getHits().getAt(0);
+            //stopwords are not highlighted since not indexed
+            assertHighlight(hit, "title", 0, 1, equalTo("this is a <em>test</em> ."));
 
-        // search on title.key and highlight on title
-        searchResponse = client().prepareSearch()
+            // search on title.key and highlight on title
+            searchResponse = client().prepareSearch()
                 .setQuery(matchQuery("title.key", "this is a test"))
                 .highlighter(new HighlightBuilder().field("title.key")).get();
-        assertHitCount(searchResponse, 1L);
+            assertHitCount(searchResponse, 1L);
 
-        //stopwords are now highlighted since we used only whitespace analyzer here
-        assertHighlight(searchResponse, 0, "title.key", 0, 1, equalTo("<em>this</em> <em>is</em> <em>a</em> <em>test</em> ."));
+            //stopwords are now highlighted since we used only whitespace analyzer here
+            assertHighlight(searchResponse, 0, "title.key", 0, 1,
+                equalTo("<em>this</em> <em>is</em> <em>a</em> <em>test</em> ."));
+        }
     }
 
     public void testPostingsHighlighterMultiMapperFromSource() throws Exception {
@@ -2315,20 +2575,22 @@ public void testPostingsHighlighterMultiMapperFromSource() throws Exception {
         client().prepareIndex("test", "type1", "1").setSource("title", "this is a test").get();
         refresh();
 
-        // simple search on body with standard analyzer with a simple field query
-        SearchResponse searchResponse = client().prepareSearch()
+        for (String type : UNIFIED_AND_NULL) {
+            // simple search on body with standard analyzer with a simple field query
+            SearchResponse searchResponse = client().prepareSearch()
                 .setQuery(matchQuery("title", "this is a test"))
                 .highlighter(new HighlightBuilder().field("title"))
                 .get();
 
-        assertHighlight(searchResponse, 0, "title", 0, 1, equalTo("this is a <em>test</em>"));
+            assertHighlight(searchResponse, 0, "title", 0, 1, equalTo("this is a <em>test</em>"));
 
-        // search on title.key and highlight on title.key
-        searchResponse = client().prepareSearch()
+            // search on title.key and highlight on title.key
+            searchResponse = client().prepareSearch()
                 .setQuery(matchQuery("title.key", "this is a test"))
-                .highlighter(new HighlightBuilder().field("title.key")).get();
+                .highlighter(new HighlightBuilder().field("title.key").highlighterType(type)).get();
 
-        assertHighlight(searchResponse, 0, "title.key", 0, 1, equalTo("<em>this</em> <em>is</em> <em>a</em> <em>test</em>"));
+            assertHighlight(searchResponse, 0, "title.key", 0, 1, equalTo("<em>this</em> <em>is</em> <em>a</em> <em>test</em>"));
+        }
     }
 
     public void testPostingsHighlighterShouldFailIfNoOffsets() throws Exception {
@@ -2379,13 +2641,15 @@ public void testPostingsHighlighterBoostingQuery() throws IOException {
             .setSource("field1", "this is a test", "field2", "The quick brown fox jumps over the lazy dog! Second sentence.").get();
         refresh();
 
-        logger.info("--> highlighting and searching on field1");
-        SearchSourceBuilder source = searchSource()
+        for (String type : UNIFIED_AND_NULL) {
+            logger.info("--> highlighting and searching on field1");
+            SearchSourceBuilder source = searchSource()
                 .query(boostingQuery(termQuery("field2", "brown"), termQuery("field2", "foobar")).negativeBoost(0.5f))
-                .highlighter(highlight().field("field2").preTags("<x>").postTags("</x>"));
-        SearchResponse searchResponse = client().search(searchRequest("test").source(source)).actionGet();
+                .highlighter(highlight().field("field2").preTags("<x>").postTags("</x>").highlighterType(type));
+            SearchResponse searchResponse = client().search(searchRequest("test").source(source)).actionGet();
 
-        assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The quick <x>brown</x> fox jumps over the lazy dog!"));
+            assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The quick <x>brown</x> fox jumps over the lazy dog!"));
+        }
     }
 
     public void testPostingsHighlighterCommonTermsQuery() throws IOException {
@@ -2395,13 +2659,16 @@ public void testPostingsHighlighterCommonTermsQuery() throws IOException {
         client().prepareIndex("test", "type1")
             .setSource("field1", "this is a test", "field2", "The quick brown fox jumps over the lazy dog! Second sentence.").get();
         refresh();
-        logger.info("--> highlighting and searching on field1");
-        SearchSourceBuilder source = searchSource().query(commonTermsQuery("field2", "quick brown").cutoffFrequency(100))
-                .highlighter(highlight().field("field2").preTags("<x>").postTags("</x>"));
-        SearchResponse searchResponse = client().search(searchRequest("test").source(source)).actionGet();
-        assertHitCount(searchResponse, 1L);
 
-        assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The <x>quick</x> <x>brown</x> fox jumps over the lazy dog!"));
+        for (String type : UNIFIED_AND_NULL) {
+            logger.info("--> highlighting and searching on field1");
+            SearchSourceBuilder source = searchSource().query(commonTermsQuery("field2", "quick brown").cutoffFrequency(100))
+                .highlighter(highlight().field("field2").preTags("<x>").postTags("</x>").highlighterType(type));
+            SearchResponse searchResponse = client().search(searchRequest("test").source(source)).actionGet();
+            assertHitCount(searchResponse, 1L);
+
+            assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The <x>quick</x> <x>brown</x> fox jumps over the lazy dog!"));
+        }
     }
 
     private static XContentBuilder type1PostingsffsetsMapping() throws IOException {
@@ -2422,11 +2689,12 @@ public void testPostingsHighlighterPrefixQuery() throws Exception {
         refresh();
         logger.info("--> highlighting and searching on field2");
 
-        SearchSourceBuilder source = searchSource().query(prefixQuery("field2", "qui"))
-                .highlighter(highlight().field("field2"));
-        SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
-        assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The <em>quick</em> brown fox jumps over the lazy dog!"));
-
+        for (String type : UNIFIED_AND_NULL) {
+            SearchSourceBuilder source = searchSource().query(prefixQuery("field2", "qui"))
+                .highlighter(highlight().field("field2").highlighterType(type));
+            SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
+            assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The <em>quick</em> brown fox jumps over the lazy dog!"));
+        }
     }
 
     public void testPostingsHighlighterFuzzyQuery() throws Exception {
@@ -2436,12 +2704,15 @@ public void testPostingsHighlighterFuzzyQuery() throws Exception {
         client().prepareIndex("test", "type1")
             .setSource("field1", "this is a test", "field2", "The quick brown fox jumps over the lazy dog! Second sentence.").get();
         refresh();
-        logger.info("--> highlighting and searching on field2");
-        SearchSourceBuilder source = searchSource().query(fuzzyQuery("field2", "quck"))
-                .highlighter(highlight().field("field2"));
-        SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
 
-        assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The <em>quick</em> brown fox jumps over the lazy dog!"));
+        for (String type : UNIFIED_AND_NULL) {
+            logger.info("--> highlighting and searching on field2");
+            SearchSourceBuilder source = searchSource().query(fuzzyQuery("field2", "quck"))
+                .highlighter(highlight().field("field2").highlighterType(type));
+            SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
+
+            assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The <em>quick</em> brown fox jumps over the lazy dog!"));
+        }
     }
 
     public void testPostingsHighlighterRegexpQuery() throws Exception {
@@ -2451,12 +2722,15 @@ public void testPostingsHighlighterRegexpQuery() throws Exception {
         client().prepareIndex("test", "type1")
             .setSource("field1", "this is a test", "field2", "The quick brown fox jumps over the lazy dog! Second sentence.").get();
         refresh();
-        logger.info("--> highlighting and searching on field2");
-        SearchSourceBuilder source = searchSource().query(regexpQuery("field2", "qu[a-l]+k"))
-                .highlighter(highlight().field("field2"));
-        SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
 
-        assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The <em>quick</em> brown fox jumps over the lazy dog!"));
+        for (String type : UNIFIED_AND_NULL) {
+            logger.info("--> highlighting and searching on field2");
+            SearchSourceBuilder source = searchSource().query(regexpQuery("field2", "qu[a-l]+k"))
+                .highlighter(highlight().field("field2").highlighterType(type));
+            SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
+
+            assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The <em>quick</em> brown fox jumps over the lazy dog!"));
+        }
     }
 
     public void testPostingsHighlighterWildcardQuery() throws Exception {
@@ -2466,19 +2740,22 @@ public void testPostingsHighlighterWildcardQuery() throws Exception {
         client().prepareIndex("test", "type1")
             .setSource("field1", "this is a test", "field2", "The quick brown fox jumps over the lazy dog! Second sentence.").get();
         refresh();
-        logger.info("--> highlighting and searching on field2");
-        SearchSourceBuilder source = searchSource().query(wildcardQuery("field2", "qui*"))
-                .highlighter(highlight().field("field2"));
-        SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
 
-        assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The <em>quick</em> brown fox jumps over the lazy dog!"));
+        for (String type : UNIFIED_AND_NULL) {
+            logger.info("--> highlighting and searching on field2");
+            SearchSourceBuilder source = searchSource().query(wildcardQuery("field2", "qui*"))
+                .highlighter(highlight().field("field2").highlighterType(type));
+            SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
 
-        source = searchSource().query(wildcardQuery("field2", "qu*k"))
-                .highlighter(highlight().field("field2"));
-        searchResponse = client().prepareSearch("test").setSource(source).get();
-        assertHitCount(searchResponse, 1L);
+            assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The <em>quick</em> brown fox jumps over the lazy dog!"));
+
+            source = searchSource().query(wildcardQuery("field2", "qu*k"))
+                .highlighter(highlight().field("field2").highlighterType(type));
+            searchResponse = client().prepareSearch("test").setSource(source).get();
+            assertHitCount(searchResponse, 1L);
 
-        assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The <em>quick</em> brown fox jumps over the lazy dog!"));
+            assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The <em>quick</em> brown fox jumps over the lazy dog!"));
+        }
     }
 
     public void testPostingsHighlighterTermRangeQuery() throws Exception {
@@ -2487,12 +2764,15 @@ public void testPostingsHighlighterTermRangeQuery() throws Exception {
 
         client().prepareIndex("test", "type1").setSource("field1", "this is a test", "field2", "aaab").get();
         refresh();
-        logger.info("--> highlighting and searching on field2");
-        SearchSourceBuilder source = searchSource().query(rangeQuery("field2").gte("aaaa").lt("zzzz"))
-                .highlighter(highlight().field("field2"));
-        SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
 
-        assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("<em>aaab</em>"));
+        for (String type : UNIFIED_AND_NULL) {
+            logger.info("--> highlighting and searching on field2");
+            SearchSourceBuilder source = searchSource().query(rangeQuery("field2").gte("aaaa").lt("zzzz"))
+                .highlighter(highlight().field("field2").highlighterType(type));
+            SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
+
+            assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("<em>aaab</em>"));
+        }
     }
 
     public void testPostingsHighlighterQueryString() throws Exception {
@@ -2502,11 +2782,14 @@ public void testPostingsHighlighterQueryString() throws Exception {
         client().prepareIndex("test", "type1")
             .setSource("field1", "this is a test", "field2", "The quick brown fox jumps over the lazy dog! Second sentence.").get();
         refresh();
-        logger.info("--> highlighting and searching on field2");
-        SearchSourceBuilder source = searchSource().query(queryStringQuery("qui*").defaultField("field2"))
-                .highlighter(highlight().field("field2"));
-        SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
-        assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The <em>quick</em> brown fox jumps over the lazy dog!"));
+
+        for (String type : UNIFIED_AND_NULL) {
+            logger.info("--> highlighting and searching on field2");
+            SearchSourceBuilder source = searchSource().query(queryStringQuery("qui*").defaultField("field2"))
+                .highlighter(highlight().field("field2").highlighterType(type));
+            SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
+            assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The <em>quick</em> brown fox jumps over the lazy dog!"));
+        }
     }
 
     public void testPostingsHighlighterRegexpQueryWithinConstantScoreQuery() throws Exception {
@@ -2516,11 +2799,13 @@ public void testPostingsHighlighterRegexpQueryWithinConstantScoreQuery() throws
         client().prepareIndex("test", "type1").setSource("field1", "The photography word will get highlighted").get();
         refresh();
 
-        logger.info("--> highlighting and searching on field1");
-        SearchSourceBuilder source = searchSource().query(constantScoreQuery(regexpQuery("field1", "pho[a-z]+")))
-                .highlighter(highlight().field("field1"));
-        SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
-        assertHighlight(searchResponse, 0, "field1", 0, 1, equalTo("The <em>photography</em> word will get highlighted"));
+        for (String type : UNIFIED_AND_NULL) {
+            logger.info("--> highlighting and searching on field1");
+            SearchSourceBuilder source = searchSource().query(constantScoreQuery(regexpQuery("field1", "pho[a-z]+")))
+                .highlighter(highlight().field("field1").highlighterType(type));
+            SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
+            assertHighlight(searchResponse, 0, "field1", 0, 1, equalTo("The <em>photography</em> word will get highlighted"));
+        }
     }
 
     public void testPostingsHighlighterMultiTermQueryMultipleLevels() throws Exception {
@@ -2530,14 +2815,16 @@ public void testPostingsHighlighterMultiTermQueryMultipleLevels() throws Excepti
         client().prepareIndex("test", "type1").setSource("field1", "The photography word will get highlighted").get();
         refresh();
 
-        logger.info("--> highlighting and searching on field1");
-        SearchSourceBuilder source = searchSource().query(boolQuery()
+        for (String type : UNIFIED_AND_NULL) {
+            logger.info("--> highlighting and searching on field1");
+            SearchSourceBuilder source = searchSource().query(boolQuery()
                 .should(boolQuery().mustNot(QueryBuilders.existsQuery("field1")))
                 .should(matchQuery("field1", "test"))
                 .should(constantScoreQuery(queryStringQuery("field1:photo*"))))
-                .highlighter(highlight().field("field1"));
-        SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
-        assertHighlight(searchResponse, 0, "field1", 0, 1, equalTo("The <em>photography</em> word will get highlighted"));
+                .highlighter(highlight().field("field1").highlighterType(type));
+            SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
+            assertHighlight(searchResponse, 0, "field1", 0, 1, equalTo("The <em>photography</em> word will get highlighted"));
+        }
     }
 
     public void testPostingsHighlighterPrefixQueryWithinBooleanQuery() throws Exception {
@@ -2547,12 +2834,14 @@ public void testPostingsHighlighterPrefixQueryWithinBooleanQuery() throws Except
         client().prepareIndex("test", "type1").setSource("field1", "The photography word will get highlighted").get();
         refresh();
 
-        logger.info("--> highlighting and searching on field1");
-        SearchSourceBuilder source = searchSource()
+        for (String type : UNIFIED_AND_NULL) {
+            logger.info("--> highlighting and searching on field1");
+            SearchSourceBuilder source = searchSource()
                 .query(boolQuery().must(prefixQuery("field1", "photo")).should(matchQuery("field1", "test").minimumShouldMatch("0")))
-                .highlighter(highlight().field("field1"));
-        SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
-        assertHighlight(searchResponse, 0, "field1", 0, 1, equalTo("The <em>photography</em> word will get highlighted"));
+                .highlighter(highlight().field("field1").highlighterType(type));
+            SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
+            assertHighlight(searchResponse, 0, "field1", 0, 1, equalTo("The <em>photography</em> word will get highlighted"));
+        }
     }
 
     public void testPostingsHighlighterQueryStringWithinFilteredQuery() throws Exception {
@@ -2562,13 +2851,15 @@ public void testPostingsHighlighterQueryStringWithinFilteredQuery() throws Excep
         client().prepareIndex("test", "type1").setSource("field1", "The photography word will get highlighted").get();
         refresh();
 
-        logger.info("--> highlighting and searching on field1");
-        SearchSourceBuilder source = searchSource().query(boolQuery()
+        for (String type : UNIFIED_AND_NULL) {
+            logger.info("--> highlighting and searching on field1");
+            SearchSourceBuilder source = searchSource().query(boolQuery()
                 .must(queryStringQuery("field1:photo*"))
                 .mustNot(existsQuery("field_null")))
-                .highlighter(highlight().field("field1"));
-        SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
-        assertHighlight(searchResponse, 0, "field1", 0, 1, equalTo("The <em>photography</em> word will get highlighted"));
+                .highlighter(highlight().field("field1").highlighterType(type));
+            SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
+            assertHighlight(searchResponse, 0, "field1", 0, 1, equalTo("The <em>photography</em> word will get highlighted"));
+        }
     }
 
     public void testPostingsHighlighterManyDocs() throws Exception {
@@ -2582,7 +2873,7 @@ public void testPostingsHighlighterManyDocs() throws Exception {
         for (int i = 0; i < COUNT; i++) {
             //generating text with word to highlight in a different position
             //(https://github.com/elastic/elasticsearch/issues/4103)
-            String prefix = randomAsciiOfLengthBetween(5, 30);
+            String prefix = randomAlphaOfLengthBetween(5, 30);
             prefixes.put(String.valueOf(i), prefix);
             indexRequestBuilders[i] = client().prepareIndex("test", "type1", Integer.toString(i)).setSource("field1", "Sentence " + prefix
                     + " test. Sentence two.");
@@ -2590,18 +2881,20 @@ public void testPostingsHighlighterManyDocs() throws Exception {
         logger.info("--> indexing docs");
         indexRandom(true, indexRequestBuilders);
 
-        logger.info("--> searching explicitly on field1 and highlighting on it");
-        SearchRequestBuilder searchRequestBuilder = client().prepareSearch()
+        for (String type : UNIFIED_AND_NULL) {
+            logger.info("--> searching explicitly on field1 and highlighting on it");
+            SearchRequestBuilder searchRequestBuilder = client().prepareSearch()
                 .setSize(COUNT)
                 .setQuery(termQuery("field1", "test"))
-                .highlighter(new HighlightBuilder().field("field1"));
-        SearchResponse searchResponse =
+                .highlighter(new HighlightBuilder().field("field1").highlighterType(type));
+            SearchResponse searchResponse =
                 searchRequestBuilder.get();
-        assertHitCount(searchResponse, COUNT);
-        assertThat(searchResponse.getHits().hits().length, equalTo(COUNT));
-        for (SearchHit hit : searchResponse.getHits()) {
-            String prefix = prefixes.get(hit.id());
-            assertHighlight(hit, "field1", 0, 1, equalTo("Sentence " + prefix + " <em>test</em>."));
+            assertHitCount(searchResponse, COUNT);
+            assertThat(searchResponse.getHits().hits().length, equalTo(COUNT));
+            for (SearchHit hit : searchResponse.getHits()) {
+                String prefix = prefixes.get(hit.id());
+                assertHighlight(hit, "field1", 0, 1, equalTo("Sentence " + prefix + " <em>test</em>."));
+            }
         }
     }
 
@@ -2616,7 +2909,7 @@ public void testDoesNotHighlightTypeName() throws Exception {
 
         indexRandom(true, client().prepareIndex("test", "typename").setSource("foo", "test typename"));
 
-        for (String highlighter: new String[] {"plain", "fvh", "postings"}) {
+        for (String highlighter : ALL_TYPES) {
             SearchResponse response = client().prepareSearch("test").setTypes("typename").setQuery(matchQuery("foo", "test"))
                     .highlighter(new HighlightBuilder().field("foo").highlighterType(highlighter).requireFieldMatch(false)).get();
             assertHighlight(response, 0, "foo", 0, 1, equalTo("<em>test</em> typename"));
@@ -2635,7 +2928,7 @@ public void testDoesNotHighlightAliasFilters() throws Exception {
 
         indexRandom(true, client().prepareIndex("test", "typename").setSource("foo", "test japanese"));
 
-        for (String highlighter: new String[] {"plain", "fvh", "postings"}) {
+        for (String highlighter : ALL_TYPES) {
             SearchResponse response = client().prepareSearch("filtered_alias").setTypes("typename").setQuery(matchQuery("foo", "test"))
                     .highlighter(new HighlightBuilder().field("foo").highlighterType(highlighter).requireFieldMatch(false)).get();
             assertHighlight(response, 0, "foo", 0, 1, equalTo("<em>test</em> japanese"));
@@ -2727,7 +3020,6 @@ public void testGeoFieldHighlightingWithDifferentHighlighters() throws IOExcepti
             .startObject("properties")
             .startObject("geo_point")
             .field("type", "geo_point")
-            .field("geohash", true)
             .endObject()
             .startObject("text")
             .field("type", "text")
@@ -2744,7 +3036,7 @@ public void testGeoFieldHighlightingWithDifferentHighlighters() throws IOExcepti
             .setSource(jsonBuilder().startObject().field("text", "Arbitrary text field which will should not cause a failure").endObject())
             .get();
         refresh();
-        String highlighterType = randomFrom("plain", "fvh", "postings");
+        String highlighterType = randomFrom("plain", "fvh", "postings", "unified");
         QueryBuilder query = QueryBuilders.boolQuery().should(QueryBuilders.geoBoundingBoxQuery("geo_point")
             .setCorners(61.10078883158897, -170.15625, -64.92354174306496, 118.47656249999999))
             .should(QueryBuilders.termQuery("text", "failure"));
@@ -2756,6 +3048,45 @@ public void testGeoFieldHighlightingWithDifferentHighlighters() throws IOExcepti
         assertThat(search.getHits().getAt(0).highlightFields().get("text").fragments().length, equalTo(1));
     }
 
+    public void testGeoFieldHighlightingWhenQueryGetsRewritten() throws IOException {
+        // same as above but in this example the query gets rewritten during highlighting
+        // see https://github.com/elastic/elasticsearch/issues/17537#issuecomment-244939633
+        XContentBuilder mappings = jsonBuilder();
+        mappings.startObject();
+        mappings.startObject("jobs")
+            .startObject("_all")
+            .field("enabled", false)
+            .endObject()
+            .startObject("properties")
+            .startObject("loc")
+            .field("type", "geo_point")
+            .endObject()
+            .startObject("jd")
+            .field("type", "string")
+            .endObject()
+            .endObject()
+            .endObject();
+        mappings.endObject();
+        assertAcked(prepareCreate("test")
+            .addMapping("jobs", mappings));
+        ensureYellow();
+
+        client().prepareIndex("test", "jobs", "1")
+            .setSource(jsonBuilder().startObject().field("jd", "some आवश्यकता है- आर्य समाज अनाथालय, 68 सिविल लाइन्स, बरेली को एक पुरूष" +
+                " रस text")
+                .field("loc", "12.934059,77.610741").endObject())
+            .get();
+        refresh();
+
+        QueryBuilder query = QueryBuilders.functionScoreQuery(QueryBuilders.boolQuery().filter(QueryBuilders.geoBoundingBoxQuery("loc")
+            .setCorners(new GeoPoint(48.934059, 41.610741), new GeoPoint(-23.065941, 113.610741))));
+        SearchResponse search = client().prepareSearch().setSource(
+            new SearchSourceBuilder().query(query).highlighter(new HighlightBuilder().highlighterType("plain").field("jd"))).get();
+        assertNoFailures(search);
+        assertThat(search.getHits().totalHits(), equalTo(1L));
+    }
+
+
     public void testKeywordFieldHighlighting() throws IOException {
         // check that keyword highlighting works
         XContentBuilder mappings = jsonBuilder();
@@ -2851,4 +3182,193 @@ public void testACopyFieldWithNestedQuery() throws Exception {
         assertThat(field.getFragments()[0].string(), equalTo("<em>brown</em>"));
         assertThat(field.getFragments()[1].string(), equalTo("<em>cow</em>"));
     }
+
+    public void testFunctionScoreQueryHighlight() throws Exception {
+        client().prepareIndex("test", "type", "1")
+            .setSource(jsonBuilder().startObject().field("text", "brown").endObject())
+            .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+            .get();
+
+        for (String type : UNIFIED_AND_NULL) {
+            SearchResponse searchResponse = client().prepareSearch()
+                .setQuery(new FunctionScoreQueryBuilder(QueryBuilders.prefixQuery("text", "bro")))
+                .highlighter(new HighlightBuilder()
+                    .field(new Field("text")).highlighterType(type))
+                .get();
+            assertHitCount(searchResponse, 1);
+            HighlightField field = searchResponse.getHits().getAt(0).highlightFields().get("text");
+            assertThat(field.getFragments().length, equalTo(1));
+            assertThat(field.getFragments()[0].string(), equalTo("<em>brown</em>"));
+        }
+    }
+
+    public void testFiltersFunctionScoreQueryHighlight() throws Exception {
+        client().prepareIndex("test", "type", "1")
+            .setSource(jsonBuilder().startObject().field("text", "brown").field("enable", "yes").endObject())
+            .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+            .get();
+        FunctionScoreQueryBuilder.FilterFunctionBuilder filterBuilder =
+            new FunctionScoreQueryBuilder.FilterFunctionBuilder(QueryBuilders.termQuery("enable", "yes"),
+                new RandomScoreFunctionBuilder());
+
+        for (String type : UNIFIED_AND_NULL) {
+            SearchResponse searchResponse = client().prepareSearch()
+                .setQuery(new FunctionScoreQueryBuilder(QueryBuilders.prefixQuery("text", "bro"),
+                    new FunctionScoreQueryBuilder.FilterFunctionBuilder[]{filterBuilder}))
+                .highlighter(new HighlightBuilder()
+                    .field(new Field("text")).highlighterType(type))
+                .get();
+            assertHitCount(searchResponse, 1);
+            HighlightField field = searchResponse.getHits().getAt(0).highlightFields().get("text");
+            assertThat(field.getFragments().length, equalTo(1));
+            assertThat(field.getFragments()[0].string(), equalTo("<em>brown</em>"));
+        }
+    }
+
+    public void testSynonyms() throws IOException {
+        Builder builder = Settings.builder()
+            .put(indexSettings())
+            .put("index.analysis.analyzer.synonym.tokenizer", "whitespace")
+            .putArray("index.analysis.analyzer.synonym.filter", "synonym", "lowercase")
+            .put("index.analysis.filter.synonym.type", "synonym")
+            .putArray("index.analysis.filter.synonym.synonyms", "fast,quick");
+
+        assertAcked(prepareCreate("test").setSettings(builder.build())
+            .addMapping("type1", "field1",
+                "type=text,term_vector=with_positions_offsets,search_analyzer=synonym," +
+                    "analyzer=english,index_options=offsets"));
+        ensureGreen();
+
+        client().prepareIndex("test", "type1", "0").setSource(
+            "field1", "The quick brown fox jumps over the lazy dog").get();
+        refresh();
+        for (String highlighterType : ALL_TYPES) {
+            logger.info("--> highlighting (type=" + highlighterType + ") and searching on field1");
+            SearchSourceBuilder source = searchSource()
+                .query(matchQuery("field1", "quick brown fox").operator(Operator.AND))
+                .highlighter(
+                    highlight()
+                        .field("field1")
+                        .order("score")
+                        .preTags("<x>")
+                        .postTags("</x>")
+                        .highlighterType(highlighterType));
+            SearchResponse searchResponse = client().search(searchRequest("test").source(source)).actionGet();
+            assertHighlight(searchResponse, 0, "field1", 0, 1,
+                equalTo("The <x>quick</x> <x>brown</x> <x>fox</x> jumps over the lazy dog"));
+
+            source = searchSource()
+                .query(matchQuery("field1", "fast brown fox").operator(Operator.AND))
+                .highlighter(highlight().field("field1").order("score").preTags("<x>").postTags("</x>"));
+            searchResponse = client().search(searchRequest("test").source(source)).actionGet();
+            assertHighlight(searchResponse, 0, "field1", 0, 1,
+                equalTo("The <x>quick</x> <x>brown</x> <x>fox</x> jumps over the lazy dog"));
+        }
+    }
+
+    public void testHighlightQueryRewriteDatesWithNow() throws Exception {
+        assertAcked(client().admin().indices().prepareCreate("index-1").addMapping("type", "d", "type=date",
+            "field", "type=text,store=true,term_vector=with_positions_offsets")
+            .setSettings("index.number_of_replicas", 0, "index.number_of_shards", 2)
+            .get());
+        DateTime now = new DateTime(ISOChronology.getInstanceUTC());
+        indexRandom(true, client().prepareIndex("index-1", "type", "1").setSource("d", now, "field", "hello world"),
+            client().prepareIndex("index-1", "type", "2").setSource("d", now.minusDays(1), "field", "hello"),
+            client().prepareIndex("index-1", "type", "3").setSource("d", now.minusDays(2), "field", "world"));
+        ensureSearchable("index-1");
+        for (int i = 0; i < 5; i++) {
+            final SearchResponse r1 = client().prepareSearch("index-1")
+                .addSort("d", SortOrder.DESC)
+                .setTrackScores(true)
+                .highlighter(highlight()
+                    .field("field")
+                    .preTags("<x>")
+                    .postTags("</x>")
+                ).setQuery(QueryBuilders.boolQuery().must(
+                    QueryBuilders.rangeQuery("d").gte("now-12h").lte("now").includeLower(true).includeUpper(true).boost(1.0f))
+                    .should(QueryBuilders.termQuery("field", "hello")))
+                .get();
+
+            assertSearchResponse(r1);
+            assertThat(r1.getHits().getTotalHits(), equalTo(1L));
+            assertHighlight(r1, 0, "field", 0, 1,
+                equalTo("<x>hello</x> world"));
+        }
+    }
+
+    public void testWithNestedQuery() throws Exception {
+        String mapping = jsonBuilder().startObject().startObject("type").startObject("properties")
+            .startObject("text")
+                .field("type", "text")
+                .field("index_options", "offsets")
+                .field("term_vector", "with_positions_offsets")
+            .endObject()
+            .startObject("foo")
+                .field("type", "nested")
+                .startObject("properties")
+                    .startObject("text")
+                        .field("type", "text")
+                    .endObject()
+                .endObject()
+            .endObject()
+            .endObject().endObject().endObject().string();
+        prepareCreate("test").addMapping("type", mapping, XContentType.JSON).get();
+
+        client().prepareIndex("test", "type", "1").setSource(jsonBuilder().startObject()
+            .startArray("foo")
+                .startObject().field("text", "brown").endObject()
+                .startObject().field("text", "cow").endObject()
+            .endArray()
+            .field("text", "brown")
+            .endObject()).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+            .get();
+
+        for (String type : new String[] {"unified", "plain"}) {
+            SearchResponse searchResponse = client().prepareSearch()
+                .setQuery(nestedQuery("foo", matchQuery("foo.text", "brown cow"), ScoreMode.None))
+                .highlighter(new HighlightBuilder()
+                    .field(new Field("foo.text").highlighterType(type)))
+                .get();
+            assertHitCount(searchResponse, 1);
+            HighlightField field = searchResponse.getHits().getAt(0).getHighlightFields().get("foo.text");
+            assertThat(field.getFragments().length, equalTo(2));
+            assertThat(field.getFragments()[0].string(), equalTo("<em>brown</em>"));
+            assertThat(field.getFragments()[1].string(), equalTo("<em>cow</em>"));
+
+            searchResponse = client().prepareSearch()
+                .setQuery(nestedQuery("foo", prefixQuery("foo.text", "bro"), ScoreMode.None))
+                .highlighter(new HighlightBuilder()
+                    .field(new Field("foo.text").highlighterType(type)))
+                .get();
+            assertHitCount(searchResponse, 1);
+            field = searchResponse.getHits().getAt(0).getHighlightFields().get("foo.text");
+            assertThat(field.getFragments().length, equalTo(1));
+            assertThat(field.getFragments()[0].string(), equalTo("<em>brown</em>"));
+
+            searchResponse = client().prepareSearch()
+                .setQuery(nestedQuery("foo", prefixQuery("foo.text", "bro"), ScoreMode.None))
+                .highlighter(new HighlightBuilder()
+                    .field(new Field("foo.text").highlighterType("plain")))
+                .get();
+            assertHitCount(searchResponse, 1);
+            field = searchResponse.getHits().getAt(0).getHighlightFields().get("foo.text");
+            assertThat(field.getFragments().length, equalTo(1));
+            assertThat(field.getFragments()[0].string(), equalTo("<em>brown</em>"));
+        }
+
+        // For unified and fvh highlighters we just check that the nested query is correctly extracted
+        // but we highlight the root text field since nested documents cannot be highlighted with postings nor term vectors
+        // directly.
+        for (String type : new String[] { "unified", "fvh" }) {
+            SearchResponse searchResponse = client().prepareSearch()
+                .setQuery(nestedQuery("foo", prefixQuery("foo.text", "bro"), ScoreMode.None))
+                .highlighter(new HighlightBuilder()
+                    .field(new Field("text").highlighterType(type).requireFieldMatch(false)))
+                .get();
+            assertHitCount(searchResponse, 1);
+            HighlightField field = searchResponse.getHits().getAt(0).getHighlightFields().get("text");
+            assertThat(field.getFragments().length, equalTo(1));
+            assertThat(field.getFragments()[0].string(), equalTo("<em>brown</em>"));
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighterTests.java b/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighterTests.java
index 428751e8859a8..b923c2464d831 100644
--- a/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighterTests.java
+++ b/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighterTests.java
@@ -68,8 +68,11 @@ public void checkGeoQueryHighlighting(Query geoQuery) throws IOException, Invali
         String fragment = highlighter.getBestFragment(fieldNameAnalyzer.tokenStream("text", "Arbitrary text field which should not cause " +
             "a failure"), "Arbitrary text field which should not cause a failure");
         assertThat(fragment, equalTo("Arbitrary text field which should not cause a <B>failure</B>"));
-        // TODO: This test will fail if we pass in an instance of GeoPointInBBoxQueryImpl too. Should we also find a way to work around that
-        // or can the query not be rewritten before it is passed into the highlighter?
+        Query rewritten = boolQuery.rewrite(null);
+        highlighter = new org.apache.lucene.search.highlight.Highlighter(new CustomQueryScorer(rewritten));
+        fragment = highlighter.getBestFragment(fieldNameAnalyzer.tokenStream("text", "Arbitrary text field which should not cause " +
+            "a failure"), "Arbitrary text field which should not cause a failure");
+        assertThat(fragment, equalTo("Arbitrary text field which should not cause a <B>failure</B>"));
     }
 
     public void testGeoPointInBBoxQueryHighlighting() throws IOException, InvalidTokenOffsetsException {
diff --git a/core/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java b/core/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java
index b091149fa011e..7df21f59481c1 100644
--- a/core/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java
+++ b/core/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.common.joda.Joda;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.index.mapper.TimestampFieldMapper;
@@ -36,7 +37,7 @@
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.script.MockScriptPlugin;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHitField;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
@@ -80,7 +81,7 @@ public class SearchFieldsIT extends ESIntegTestCase {
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Arrays.asList(CustomScriptPlugin.class);
+        return Collections.singletonList(CustomScriptPlugin.class);
     }
 
     public static class CustomScriptPlugin extends MockScriptPlugin {
@@ -285,15 +286,18 @@ public void testScriptDocAndFields() throws Exception {
         SearchResponse response = client().prepareSearch()
                 .setQuery(matchAllQuery())
                 .addSort("num1", SortOrder.ASC)
-                .addScriptField("sNum1", new Script("doc['num1'].value", ScriptType.INLINE, CustomScriptPlugin.NAME, null))
-                .addScriptField("sNum1_field", new Script("_fields['num1'].value", ScriptType.INLINE, CustomScriptPlugin.NAME, null))
-                .addScriptField("date1", new Script("doc['date'].date.millis", ScriptType.INLINE, CustomScriptPlugin.NAME, null))
+                .addScriptField("sNum1",
+                    new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['num1'].value", Collections.emptyMap()))
+                .addScriptField("sNum1_field",
+                    new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_fields['num1'].value", Collections.emptyMap()))
+                .addScriptField("date1",
+                    new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['date'].date.millis", Collections.emptyMap()))
                 .execute().actionGet();
 
         assertNoFailures(response);
 
         assertThat(response.getHits().totalHits(), equalTo(3L));
-        assertThat(response.getHits().getAt(0).hasSource(), equalTo(true));
+        assertFalse(response.getHits().getAt(0).hasSource());
         assertThat(response.getHits().getAt(0).id(), equalTo("1"));
         Set<String> fields = new HashSet<>(response.getHits().getAt(0).fields().keySet());
         fields.remove(TimestampFieldMapper.NAME); // randomly enabled via templates
@@ -321,7 +325,7 @@ public void testScriptDocAndFields() throws Exception {
         response = client().prepareSearch()
                 .setQuery(matchAllQuery())
                 .addSort("num1", SortOrder.ASC)
-                .addScriptField("sNum1", new Script("doc['num1'].value * factor", ScriptType.INLINE, CustomScriptPlugin.NAME, params))
+                .addScriptField("sNum1", new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['num1'].value * factor", params))
                 .get();
 
         assertThat(response.getHits().totalHits(), equalTo(3L));
@@ -357,7 +361,7 @@ public void testUidBasedScriptFields() throws Exception {
                 .setQuery(matchAllQuery())
                 .addSort("num1", SortOrder.ASC)
                 .setSize(numDocs)
-                .addScriptField("uid", new Script("_fields._uid.value", ScriptType.INLINE, CustomScriptPlugin.NAME, null))
+                .addScriptField("uid", new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_fields._uid.value", Collections.emptyMap()))
                 .get();
 
         assertNoFailures(response);
@@ -375,7 +379,7 @@ public void testUidBasedScriptFields() throws Exception {
                 .setQuery(matchAllQuery())
                 .addSort("num1", SortOrder.ASC)
                 .setSize(numDocs)
-                .addScriptField("id", new Script("_fields._id.value", ScriptType.INLINE, CustomScriptPlugin.NAME, null))
+                .addScriptField("id", new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_fields._id.value", Collections.emptyMap()))
                 .get();
 
         assertNoFailures(response);
@@ -393,7 +397,8 @@ public void testUidBasedScriptFields() throws Exception {
                 .setQuery(matchAllQuery())
                 .addSort("num1", SortOrder.ASC)
                 .setSize(numDocs)
-                .addScriptField("type", new Script("_fields._type.value", ScriptType.INLINE, CustomScriptPlugin.NAME, null))
+                .addScriptField("type",
+                    new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_fields._type.value", Collections.emptyMap()))
                 .get();
 
         assertNoFailures(response);
@@ -411,9 +416,10 @@ public void testUidBasedScriptFields() throws Exception {
                 .setQuery(matchAllQuery())
                 .addSort("num1", SortOrder.ASC)
                 .setSize(numDocs)
-                .addScriptField("id", new Script("_fields._id.value", ScriptType.INLINE, CustomScriptPlugin.NAME, null))
-                .addScriptField("uid", new Script("_fields._uid.value", ScriptType.INLINE, CustomScriptPlugin.NAME, null))
-                .addScriptField("type", new Script("_fields._type.value", ScriptType.INLINE, CustomScriptPlugin.NAME, null))
+                .addScriptField("id", new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_fields._id.value", Collections.emptyMap()))
+                .addScriptField("uid", new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_fields._uid.value", Collections.emptyMap()))
+                .addScriptField("type",
+                    new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_fields._type.value", Collections.emptyMap()))
                 .get();
 
         assertNoFailures(response);
@@ -444,11 +450,13 @@ public void testScriptFieldUsingSource() throws Exception {
 
         SearchResponse response = client().prepareSearch()
                 .setQuery(matchAllQuery())
-                .addScriptField("s_obj1", new Script("_source.obj1", ScriptType.INLINE, CustomScriptPlugin.NAME, null))
-                .addScriptField("s_obj1_test", new Script("_source.obj1.test", ScriptType.INLINE, CustomScriptPlugin.NAME, null))
-                .addScriptField("s_obj2", new Script("_source.obj2", ScriptType.INLINE, CustomScriptPlugin.NAME, null))
-                .addScriptField("s_obj2_arr2", new Script("_source.obj2.arr2", ScriptType.INLINE, CustomScriptPlugin.NAME, null))
-                .addScriptField("s_arr3", new Script("_source.arr3", ScriptType.INLINE, CustomScriptPlugin.NAME, null))
+                .addScriptField("s_obj1", new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_source.obj1", Collections.emptyMap()))
+                .addScriptField("s_obj1_test",
+                    new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_source.obj1.test", Collections.emptyMap()))
+                .addScriptField("s_obj2", new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_source.obj2", Collections.emptyMap()))
+                .addScriptField("s_obj2_arr2",
+                    new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_source.obj2.arr2", Collections.emptyMap()))
+                .addScriptField("s_arr3", new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_source.arr3", Collections.emptyMap()))
                 .get();
 
         assertThat("Failures " + Arrays.toString(response.getShardFailures()), response.getShardFailures().length, equalTo(0));
@@ -481,7 +489,8 @@ public void testScriptFieldsForNullReturn() throws Exception {
 
         SearchResponse response = client().prepareSearch()
                 .setQuery(matchAllQuery())
-                .addScriptField("test_script_1", new Script("return null", ScriptType.INLINE, CustomScriptPlugin.NAME, null))
+                .addScriptField("test_script_1",
+                    new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "return null", Collections.emptyMap()))
                 .get();
 
         assertNoFailures(response);
@@ -693,8 +702,8 @@ public void testGetFieldsComplexField() throws Exception {
                 .endArray()
                 .endObject().bytes();
 
-        client().prepareIndex("my-index", "my-type1", "1").setSource(source).get();
-        client().prepareIndex("my-index", "my-type2", "1").setRefreshPolicy(IMMEDIATE).setSource(source).get();
+        client().prepareIndex("my-index", "my-type1", "1").setSource(source, XContentType.JSON).get();
+        client().prepareIndex("my-index", "my-type2", "1").setRefreshPolicy(IMMEDIATE).setSource(source, XContentType.JSON).get();
 
 
         String field = "field1.field2.field3.field4";
@@ -770,6 +779,9 @@ public void testFieldsPulledFromFieldData() throws Exception {
                             .startObject("binary_field")
                                 .field("type", "binary")
                             .endObject()
+                            .startObject("ip_field")
+                                .field("type", "ip")
+                            .endObject()
                         .endObject()
                     .endObject()
                 .endObject()
@@ -788,6 +800,7 @@ public void testFieldsPulledFromFieldData() throws Exception {
                 .field("double_field", 6.0d)
                 .field("date_field", Joda.forPattern("dateOptionalTime").printer().print(new DateTime(2012, 3, 22, 0, 0, DateTimeZone.UTC)))
                 .field("boolean_field", true)
+                .field("ip_field", "::1")
                 .endObject()).execute().actionGet();
 
         client().admin().indices().prepareRefresh().execute().actionGet();
@@ -802,7 +815,8 @@ public void testFieldsPulledFromFieldData() throws Exception {
                 .addDocValueField("float_field")
                 .addDocValueField("double_field")
                 .addDocValueField("date_field")
-                .addDocValueField("boolean_field");
+                .addDocValueField("boolean_field")
+                .addDocValueField("ip_field");
         SearchResponse searchResponse = builder.execute().actionGet();
 
         assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
@@ -810,7 +824,8 @@ public void testFieldsPulledFromFieldData() throws Exception {
         Set<String> fields = new HashSet<>(searchResponse.getHits().getAt(0).fields().keySet());
         fields.remove(TimestampFieldMapper.NAME); // randomly enabled via templates
         assertThat(fields, equalTo(newHashSet("byte_field", "short_field", "integer_field", "long_field",
-                "float_field", "double_field", "date_field", "boolean_field", "text_field", "keyword_field")));
+                "float_field", "double_field", "date_field", "boolean_field", "text_field", "keyword_field",
+                "ip_field")));
 
         assertThat(searchResponse.getHits().getAt(0).fields().get("byte_field").value().toString(), equalTo("1"));
         assertThat(searchResponse.getHits().getAt(0).fields().get("short_field").value().toString(), equalTo("2"));
@@ -819,9 +834,10 @@ public void testFieldsPulledFromFieldData() throws Exception {
         assertThat(searchResponse.getHits().getAt(0).fields().get("float_field").value(), equalTo((Object) 5.0));
         assertThat(searchResponse.getHits().getAt(0).fields().get("double_field").value(), equalTo((Object) 6.0d));
         assertThat(searchResponse.getHits().getAt(0).fields().get("date_field").value(), equalTo((Object) 1332374400000L));
-        assertThat(searchResponse.getHits().getAt(0).fields().get("boolean_field").value(), equalTo((Object) 1L));
+        assertThat(searchResponse.getHits().getAt(0).fields().get("boolean_field").value(), equalTo((Object) true));
         assertThat(searchResponse.getHits().getAt(0).fields().get("text_field").value(), equalTo("foo"));
         assertThat(searchResponse.getHits().getAt(0).fields().get("keyword_field").value(), equalTo("foo"));
+        assertThat(searchResponse.getHits().getAt(0).fields().get("ip_field").value(), equalTo("::1"));
     }
 
     public void testScriptFields() throws Exception {
@@ -847,7 +863,8 @@ public void testScriptFields() throws Exception {
         ensureSearchable();
         SearchRequestBuilder req = client().prepareSearch("index");
         for (String field : Arrays.asList("s", "ms", "l", "ml", "d", "md")) {
-            req.addScriptField(field, new Script("doc['" + field + "'].values", ScriptType.INLINE, CustomScriptPlugin.NAME, null));
+            req.addScriptField(field,
+                new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['" + field + "'].values", Collections.emptyMap()));
         }
         SearchResponse resp = req.get();
         assertSearchResponse(resp);
diff --git a/core/src/test/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java b/core/src/test/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java
index 1c87ccfe4cb07..9b732bdc00dda 100644
--- a/core/src/test/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java
+++ b/core/src/test/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java
@@ -32,7 +32,7 @@
 import org.elasticsearch.script.ExplainableSearchScript;
 import org.elasticsearch.script.NativeScriptFactory;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -44,6 +44,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ExecutionException;
@@ -76,7 +77,8 @@ public void testNativeExplainScript() throws InterruptedException, IOException,
         SearchResponse response = client().search(searchRequest().searchType(SearchType.QUERY_THEN_FETCH).source(
                         searchSource().explain(true).query(
                                 functionScoreQuery(termQuery("text", "text"),
-                                        scriptFunction(new Script("native_explainable_script", ScriptType.INLINE, "native", null)))
+                                        scriptFunction(
+                                            new Script(ScriptType.INLINE, "native", "native_explainable_script", Collections.emptyMap())))
                                         .boostMode(CombineFunction.REPLACE)))).actionGet();
 
         ElasticsearchAssertions.assertNoFailures(response);
diff --git a/core/src/test/java/org/elasticsearch/search/functionscore/FunctionScoreIT.java b/core/src/test/java/org/elasticsearch/search/functionscore/FunctionScoreIT.java
index 3854d200116bd..e9cb9d72a8b5e 100644
--- a/core/src/test/java/org/elasticsearch/search/functionscore/FunctionScoreIT.java
+++ b/core/src/test/java/org/elasticsearch/search/functionscore/FunctionScoreIT.java
@@ -48,7 +48,8 @@
 import static org.elasticsearch.index.query.QueryBuilders.functionScoreQuery;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
 import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.scriptFunction;
-import static org.elasticsearch.script.ScriptService.ScriptType;
+
+import org.elasticsearch.script.ScriptType;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.terms;
 import static org.elasticsearch.search.builder.SearchSourceBuilder.searchSource;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@@ -93,8 +94,8 @@ public void testScriptScoresNested() throws IOException {
         index(INDEX, TYPE, "1", jsonBuilder().startObject().field("dummy_field", 1).endObject());
         refresh();
 
-        Script scriptOne = new Script("1", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
-        Script scriptTwo = new Script("get score value", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
+        Script scriptOne = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "1", Collections.emptyMap());
+        Script scriptTwo = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "get score value", Collections.emptyMap());
 
         SearchResponse response = client().search(
                 searchRequest().source(
@@ -117,7 +118,7 @@ public void testScriptScoresWithAgg() throws IOException {
         index(INDEX, TYPE, "1", jsonBuilder().startObject().field("dummy_field", 1).endObject());
         refresh();
 
-        Script script = new Script("get score value", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
+        Script script = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "get score value", Collections.emptyMap());
 
         SearchResponse response = client().search(
                 searchRequest().source(
@@ -145,7 +146,7 @@ public void testMinScoreFunctionScoreBasic() throws IOException {
         refresh();
         ensureYellow();
 
-        Script script = new Script("doc['random_score']", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
+        Script script = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['random_score']", Collections.emptyMap());
         SearchResponse searchResponse = client().search(
                 searchRequest().source(searchSource().query(functionScoreQuery(scriptFunction(script)).setMinScore(minScore)))
         ).actionGet();
@@ -177,7 +178,7 @@ public void testMinScoreFunctionScoreManyDocsAndRandomMinScore() throws IOExcept
             docs.add(client().prepareIndex(INDEX, TYPE, Integer.toString(i)).setSource("num", i + scoreOffset));
         }
         indexRandom(true, docs);
-        Script script = new Script("return (doc['num'].value)", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
+        Script script = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "return (doc['num'].value)", Collections.emptyMap());
         int numMatchingDocs = numDocs + scoreOffset - minScore;
         if (numMatchingDocs < 0) {
             numMatchingDocs = 0;
diff --git a/core/src/test/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java b/core/src/test/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java
index 364186572d95a..30c9b9f1fba40 100644
--- a/core/src/test/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java
+++ b/core/src/test/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java
@@ -66,6 +66,7 @@
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasId;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasScore;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.notNullValue;
 
@@ -100,6 +101,7 @@ public void testEnforceWindowSize() {
                     numDocsWith100AsAScore += 1;
                 }
             }
+            assertThat(searchResponse.getHits().maxScore(), equalTo(searchResponse.getHits().getHits()[0].score()));
             // we cannot assert that they are equal since some shards might not have docs at all
             assertThat(numDocsWith100AsAScore, lessThanOrEqualTo(numShards));
         }
@@ -125,6 +127,7 @@ public void testRescorePhrase() throws Exception {
                                 .setRescoreQueryWeight(2), 5).execute().actionGet();
 
         assertThat(searchResponse.getHits().totalHits(), equalTo(3L));
+        assertThat(searchResponse.getHits().maxScore(), equalTo(searchResponse.getHits().getHits()[0].score()));
         assertThat(searchResponse.getHits().getHits()[0].getId(), equalTo("1"));
         assertThat(searchResponse.getHits().getHits()[1].getId(), equalTo("3"));
         assertThat(searchResponse.getHits().getHits()[2].getId(), equalTo("2"));
@@ -145,6 +148,7 @@ public void testRescorePhrase() throws Exception {
                 .actionGet();
 
         assertHitCount(searchResponse, 3);
+        assertThat(searchResponse.getHits().maxScore(), equalTo(searchResponse.getHits().getHits()[0].score()));
         assertFirstHit(searchResponse, hasId("1"));
         assertSecondHit(searchResponse, hasId("2"));
         assertThirdHit(searchResponse, hasId("3"));
@@ -206,6 +210,7 @@ public void testMoreDocs() throws Exception {
 
         assertThat(searchResponse.getHits().hits().length, equalTo(5));
         assertHitCount(searchResponse, 9);
+        assertThat(searchResponse.getHits().maxScore(), equalTo(searchResponse.getHits().getHits()[0].score()));
         assertFirstHit(searchResponse, hasId("2"));
         assertSecondHit(searchResponse, hasId("6"));
         assertThirdHit(searchResponse, hasId("3"));
@@ -222,6 +227,7 @@ public void testMoreDocs() throws Exception {
 
         assertThat(searchResponse.getHits().hits().length, equalTo(5));
         assertHitCount(searchResponse, 9);
+        assertThat(searchResponse.getHits().maxScore(), greaterThan(searchResponse.getHits().getHits()[0].score()));
         assertFirstHit(searchResponse, hasId("3"));
     }
 
@@ -255,6 +261,7 @@ public void testSmallRescoreWindow() throws Exception {
             .setSize(5).execute().actionGet();
         assertThat(searchResponse.getHits().hits().length, equalTo(4));
         assertHitCount(searchResponse, 4);
+        assertThat(searchResponse.getHits().maxScore(), equalTo(searchResponse.getHits().getHits()[0].score()));
         assertFirstHit(searchResponse, hasId("3"));
         assertSecondHit(searchResponse, hasId("6"));
         assertThirdHit(searchResponse, hasId("1"));
@@ -271,6 +278,7 @@ public void testSmallRescoreWindow() throws Exception {
         // Only top 2 hits were re-ordered:
         assertThat(searchResponse.getHits().hits().length, equalTo(4));
         assertHitCount(searchResponse, 4);
+        assertThat(searchResponse.getHits().maxScore(), equalTo(searchResponse.getHits().getHits()[0].score()));
         assertFirstHit(searchResponse, hasId("6"));
         assertSecondHit(searchResponse, hasId("3"));
         assertThirdHit(searchResponse, hasId("1"));
@@ -288,6 +296,7 @@ public void testSmallRescoreWindow() throws Exception {
         // Only top 3 hits were re-ordered:
         assertThat(searchResponse.getHits().hits().length, equalTo(4));
         assertHitCount(searchResponse, 4);
+        assertThat(searchResponse.getHits().maxScore(), equalTo(searchResponse.getHits().getHits()[0].score()));
         assertFirstHit(searchResponse, hasId("6"));
         assertSecondHit(searchResponse, hasId("1"));
         assertThirdHit(searchResponse, hasId("3"));
@@ -324,6 +333,7 @@ public void testRescorerMadeScoresWorse() throws Exception {
             .setSize(5).execute().actionGet();
         assertThat(searchResponse.getHits().hits().length, equalTo(4));
         assertHitCount(searchResponse, 4);
+        assertThat(searchResponse.getHits().maxScore(), equalTo(searchResponse.getHits().getHits()[0].score()));
         assertFirstHit(searchResponse, hasId("3"));
         assertSecondHit(searchResponse, hasId("6"));
         assertThirdHit(searchResponse, hasId("1"));
@@ -339,6 +349,7 @@ public void testRescorerMadeScoresWorse() throws Exception {
                                 .setQueryWeight(1.0f).setRescoreQueryWeight(-1f), 3).execute().actionGet();
 
         // 6 and 1 got worse, and then the hit (2) outside the rescore window were sorted ahead:
+        assertThat(searchResponse.getHits().maxScore(), equalTo(searchResponse.getHits().getHits()[0].score()));
         assertFirstHit(searchResponse, hasId("3"));
         assertSecondHit(searchResponse, hasId("2"));
         assertThirdHit(searchResponse, hasId("6"));
@@ -598,6 +609,7 @@ public void testScoring() throws Exception {
 
                 assertHitCount(rescored, 4);
 
+                assertThat(rescored.getHits().maxScore(), equalTo(rescored.getHits().getHits()[0].score()));
                 if ("total".equals(scoreMode) || "".equals(scoreMode)) {
                     assertFirstHit(rescored, hasId(String.valueOf(i + 1)));
                     assertSecondHit(rescored, hasId(String.valueOf(i)));
@@ -675,6 +687,7 @@ public void testMultipleRescores() throws Exception {
                 .boostMode(CombineFunction.REPLACE)).setScoreMode(QueryRescoreMode.Total);
         request.clearRescorers().addRescorer(ninetyIsGood, numDocs).addRescorer(oneToo, 10);
         response = request.setSize(2).get();
+        assertThat(response.getHits().maxScore(), equalTo(response.getHits().getHits()[0].score()));
         assertFirstHit(response, hasId("91"));
         assertFirstHit(response, hasScore(2001.0f));
         assertSecondHit(response, hasScore(1001.0f)); // Not sure which one it is but it is ninety something
diff --git a/core/src/test/java/org/elasticsearch/search/functionscore/RandomScoreFunctionIT.java b/core/src/test/java/org/elasticsearch/search/functionscore/RandomScoreFunctionIT.java
index 9c2b41eabe6b2..a6f7eb760d272 100644
--- a/core/src/test/java/org/elasticsearch/search/functionscore/RandomScoreFunctionIT.java
+++ b/core/src/test/java/org/elasticsearch/search/functionscore/RandomScoreFunctionIT.java
@@ -27,7 +27,7 @@
 import org.elasticsearch.script.MockScriptPlugin;
 import org.elasticsearch.script.ScoreAccessor;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.hamcrest.CoreMatchers;
@@ -169,7 +169,7 @@ public void testScoreAccessWithinScript() throws Exception {
         params.put("factor", randomIntBetween(2, 4));
 
         // Test for accessing _score
-        Script script = new Script("log(doc['index'].value + (factor * _score))", ScriptType.INLINE, NAME, params);
+        Script script = new Script(ScriptType.INLINE, NAME, "log(doc['index'].value + (factor * _score))", params);
         SearchResponse resp = client()
                 .prepareSearch("test")
                 .setQuery(
@@ -185,7 +185,7 @@ public void testScoreAccessWithinScript() throws Exception {
         assertThat(firstHit.getScore(), greaterThan(1f));
 
         // Test for accessing _score.intValue()
-        script = new Script("log(doc['index'].value + (factor * _score.intValue()))", ScriptType.INLINE, NAME, params);
+        script = new Script(ScriptType.INLINE, NAME, "log(doc['index'].value + (factor * _score.intValue()))", params);
         resp = client()
                 .prepareSearch("test")
                 .setQuery(
@@ -201,7 +201,7 @@ public void testScoreAccessWithinScript() throws Exception {
         assertThat(firstHit.getScore(), greaterThan(1f));
 
         // Test for accessing _score.longValue()
-        script = new Script("log(doc['index'].value + (factor * _score.longValue()))", ScriptType.INLINE, NAME, params);
+        script = new Script(ScriptType.INLINE, NAME, "log(doc['index'].value + (factor * _score.longValue()))", params);
         resp = client()
                 .prepareSearch("test")
                 .setQuery(
@@ -217,7 +217,7 @@ public void testScoreAccessWithinScript() throws Exception {
         assertThat(firstHit.getScore(), greaterThan(1f));
 
         // Test for accessing _score.floatValue()
-        script = new Script("log(doc['index'].value + (factor * _score.floatValue()))", ScriptType.INLINE, NAME, params);
+        script = new Script(ScriptType.INLINE, NAME, "log(doc['index'].value + (factor * _score.floatValue()))", params);
         resp = client()
                 .prepareSearch("test")
                 .setQuery(
@@ -233,7 +233,7 @@ public void testScoreAccessWithinScript() throws Exception {
         assertThat(firstHit.getScore(), greaterThan(1f));
 
         // Test for accessing _score.doubleValue()
-        script = new Script("log(doc['index'].value + (factor * _score.doubleValue()))", ScriptType.INLINE, NAME, params);
+        script = new Script(ScriptType.INLINE, NAME, "log(doc['index'].value + (factor * _score.doubleValue()))", params);
         resp = client()
                 .prepareSearch("test")
                 .setQuery(
diff --git a/core/src/test/java/org/elasticsearch/search/geo/GeoDistanceIT.java b/core/src/test/java/org/elasticsearch/search/geo/GeoDistanceIT.java
index 4ea143ed7f9de..6c9acd7e8a771 100644
--- a/core/src/test/java/org/elasticsearch/search/geo/GeoDistanceIT.java
+++ b/core/src/test/java/org/elasticsearch/search/geo/GeoDistanceIT.java
@@ -20,30 +20,41 @@
 package org.elasticsearch.search.geo;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.geo.GeoHashUtils;
+import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.geo.GeoUtils;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.DistanceUnit;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
+import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.MockScriptPlugin;
 import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptType;
+import org.elasticsearch.search.aggregations.AggregationBuilders;
+import org.elasticsearch.search.aggregations.Aggregations;
+import org.elasticsearch.search.aggregations.bucket.range.Range;
+import org.elasticsearch.search.aggregations.bucket.range.geodistance.InternalGeoDistance;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
 import org.elasticsearch.test.VersionUtils;
+import org.junit.Before;
 
+import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.function.Function;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.elasticsearch.script.ScriptService.ScriptType;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.closeTo;
 
@@ -63,7 +74,6 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
     public static class CustomScriptPlugin extends MockScriptPlugin {
 
         @Override
-        @SuppressWarnings("unchecked")
         protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
             Map<String, Function<Map<String, Object>, Object>> scripts = new HashMap<>();
 
@@ -83,15 +93,14 @@ protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
             return scripts;
         }
 
-        @SuppressWarnings("unchecked")
         static Double distanceScript(Map<String, Object> vars, Function<ScriptDocValues.GeoPoints, Double> distance) {
             Map<?, ?> doc = (Map) vars.get("doc");
             return distance.apply((ScriptDocValues.GeoPoints) doc.get("location"));
         }
     }
 
-    public void testDistanceScript() throws Exception {
-
+    @Before
+    public void setupTestIndex() throws IOException {
         Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT);
         Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
         XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type1")
@@ -102,7 +111,9 @@ public void testDistanceScript() throws Exception {
         xContentBuilder.endObject().endObject().endObject().endObject();
         assertAcked(prepareCreate("test").setSettings(settings).addMapping("type1", xContentBuilder));
         ensureGreen();
+    }
 
+    public void testDistanceScript() throws Exception {
         client().prepareIndex("test", "type1", "1")
                 .setSource(jsonBuilder().startObject()
                         .field("name", "TestPosition")
@@ -117,7 +128,7 @@ public void testDistanceScript() throws Exception {
 
         // Test doc['location'].arcDistance(lat, lon)
         SearchResponse searchResponse1 = client().prepareSearch().addStoredField("_source")
-                .addScriptField("distance", new Script("arcDistance", ScriptType.INLINE, CustomScriptPlugin.NAME, null))
+                .addScriptField("distance", new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "arcDistance", Collections.emptyMap()))
                 .get();
         Double resultDistance1 = searchResponse1.getHits().getHits()[0].getFields().get("distance").getValue();
         assertThat(resultDistance1,
@@ -125,16 +136,16 @@ public void testDistanceScript() throws Exception {
 
         // Test doc['location'].planeDistance(lat, lon)
         SearchResponse searchResponse2 = client().prepareSearch().addStoredField("_source")
-                .addScriptField("distance", new Script("planeDistance", ScriptType.INLINE,
-                    CustomScriptPlugin.NAME, null)).get();
+                .addScriptField("distance", new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "planeDistance",
+                    Collections.emptyMap())).get();
         Double resultDistance2 = searchResponse2.getHits().getHits()[0].getFields().get("distance").getValue();
         assertThat(resultDistance2,
                 closeTo(GeoUtils.planeDistance(src_lat, src_lon, tgt_lat, tgt_lon), 0.01d));
 
         // Test doc['location'].geohashDistance(lat, lon)
         SearchResponse searchResponse4 = client().prepareSearch().addStoredField("_source")
-                .addScriptField("distance", new Script("geohashDistance", ScriptType.INLINE,
-                    CustomScriptPlugin.NAME, null)).get();
+                .addScriptField("distance", new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "geohashDistance",
+                    Collections.emptyMap())).get();
         Double resultDistance4 = searchResponse4.getHits().getHits()[0].getFields().get("distance").getValue();
         assertThat(resultDistance4,
                 closeTo(GeoUtils.arcDistance(src_lat, src_lon, GeoHashUtils.decodeLatitude(tgt_geohash),
@@ -142,18 +153,55 @@ public void testDistanceScript() throws Exception {
 
         // Test doc['location'].arcDistance(lat, lon + 360)/1000d
         SearchResponse searchResponse5 = client().prepareSearch().addStoredField("_source")
-                .addScriptField("distance", new Script("arcDistance(lat, lon + 360)/1000d", ScriptType.INLINE,
-                    CustomScriptPlugin.NAME, null)).get();
+                .addScriptField("distance", new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "arcDistance(lat, lon + 360)/1000d",
+                    Collections.emptyMap())).get();
         Double resultArcDistance5 = searchResponse5.getHits().getHits()[0].getFields().get("distance").getValue();
         assertThat(resultArcDistance5,
                 closeTo(GeoUtils.arcDistance(src_lat, src_lon, tgt_lat, tgt_lon)/1000d, 0.01d));
 
         // Test doc['location'].arcDistance(lat + 360, lon)/1000d
         SearchResponse searchResponse6 = client().prepareSearch().addStoredField("_source")
-                .addScriptField("distance", new Script("arcDistance(lat + 360, lon)/1000d", ScriptType.INLINE,
-                    CustomScriptPlugin.NAME, null)).get();
+                .addScriptField("distance", new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "arcDistance(lat + 360, lon)/1000d",
+                    Collections.emptyMap())).get();
         Double resultArcDistance6 = searchResponse6.getHits().getHits()[0].getFields().get("distance").getValue();
         assertThat(resultArcDistance6,
                 closeTo(GeoUtils.arcDistance(src_lat, src_lon, tgt_lat, tgt_lon)/1000d, 0.01d));
     }
+
+    public void testGeoDistanceAggregation() throws IOException {
+        client().prepareIndex("test", "type1", "1")
+            .setSource(jsonBuilder().startObject()
+                .field("name", "TestPosition")
+                .startObject("location")
+                .field("lat", src_lat)
+                .field("lon", src_lon)
+                .endObject()
+                .endObject())
+            .get();
+
+        refresh();
+
+        SearchRequestBuilder search = client().prepareSearch("test");
+        String name = "TestPosition";
+
+        search.setQuery(QueryBuilders.matchAllQuery())
+            .setTypes("type1")
+            .addAggregation(AggregationBuilders.geoDistance(name, new GeoPoint(tgt_lat, tgt_lon))
+            .field("location")
+            .unit(DistanceUnit.MILES)
+            .addRange(0, 25000));
+
+        search.setSize(0); // no hits please
+
+        SearchResponse response = search.get();
+        Aggregations aggregations = response.getAggregations();
+        assertNotNull(aggregations);
+        InternalGeoDistance geoDistance = aggregations.get(name);
+        assertNotNull(geoDistance);
+
+        List<? extends Range.Bucket> buckets = ((Range) geoDistance).getBuckets();
+        assertNotNull("Buckets should not be null", buckets);
+        assertEquals("Unexpected number of buckets",  1, buckets.size());
+        assertEquals("Unexpected doc count for geo distance", 1, buckets.get(0).getDocCount());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java b/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java
index e6d7700c845d7..3fdeaa51ffb6d 100644
--- a/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java
+++ b/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java
@@ -48,7 +48,9 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.mapper.LatLonPointFieldMapper;
 import org.elasticsearch.index.query.GeohashCellQuery;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.SearchHit;
@@ -241,7 +243,7 @@ public void testShapeRelations() throws Exception {
                                 new CoordinatesBuilder().coordinate(-4, -4).coordinate(-4, 4).coordinate(4, 4).coordinate(4, -4).close()));
         BytesReference data = jsonBuilder().startObject().field("area", polygon).endObject().bytes();
 
-        client().prepareIndex("shapes", "polygon", "1").setSource(data).execute().actionGet();
+        client().prepareIndex("shapes", "polygon", "1").setSource(data, XContentType.JSON).execute().actionGet();
         client().admin().indices().prepareRefresh().execute().actionGet();
 
         // Point in polygon
@@ -303,7 +305,7 @@ public void testShapeRelations() throws Exception {
                             new CoordinatesBuilder().coordinate(-4, -4).coordinate(-4, 4).coordinate(4, 4).coordinate(4, -4).close()));
 
         data = jsonBuilder().startObject().field("area", inverse).endObject().bytes();
-        client().prepareIndex("shapes", "polygon", "2").setSource(data).execute().actionGet();
+        client().prepareIndex("shapes", "polygon", "2").setSource(data, XContentType.JSON).execute().actionGet();
         client().admin().indices().prepareRefresh().execute().actionGet();
 
         // re-check point on polygon hole
@@ -337,7 +339,7 @@ public void testShapeRelations() throws Exception {
                 .coordinate(170, -10).coordinate(190, -10).coordinate(190, 10).coordinate(170, 10).close());
 
         data = jsonBuilder().startObject().field("area", builder).endObject().bytes();
-        client().prepareIndex("shapes", "polygon", "1").setSource(data).execute().actionGet();
+        client().prepareIndex("shapes", "polygon", "1").setSource(data, XContentType.JSON).execute().actionGet();
         client().admin().indices().prepareRefresh().execute().actionGet();
 
         // Create a polygon crossing longitude 180 with hole.
@@ -346,7 +348,7 @@ public void testShapeRelations() throws Exception {
                     .hole(new LineStringBuilder(new CoordinatesBuilder().coordinate(175, -5).coordinate(185, -5).coordinate(185, 5).coordinate(175, 5).close()));
 
         data = jsonBuilder().startObject().field("area", builder).endObject().bytes();
-        client().prepareIndex("shapes", "polygon", "1").setSource(data).execute().actionGet();
+        client().prepareIndex("shapes", "polygon", "1").setSource(data, XContentType.JSON).execute().actionGet();
         client().admin().indices().prepareRefresh().execute().actionGet();
 
         result = client().prepareSearch()
@@ -398,7 +400,7 @@ public void testBulk() throws Exception {
 
         client().admin().indices().prepareCreate("countries").setSettings(settings)
                 .addMapping("country", xContentBuilder.string()).execute().actionGet();
-        BulkResponse bulk = client().prepareBulk().add(bulkAction, 0, bulkAction.length, null, null).execute().actionGet();
+        BulkResponse bulk = client().prepareBulk().add(bulkAction, 0, bulkAction.length, null, null, xContentBuilder.contentType()).get();
 
         for (BulkItemResponse item : bulk.getItems()) {
             assertFalse("unable to index data", item.isFailed());
@@ -433,8 +435,10 @@ public void testBulk() throws Exception {
             String name = hit.getId();
             if (version.before(Version.V_2_2_0)) {
                 point.resetFromString(hit.fields().get("pin").getValue().toString());
-            } else {
+            } else if (version.before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
                 point.resetFromIndexHash(hit.fields().get("pin").getValue());
+            } else {
+                point.resetFromString(hit.getFields().get("pin").getValue());
             }
             double dist = distance(point.getLat(), point.getLon(), 51.11, 9.851);
 
@@ -446,7 +450,7 @@ public void testBulk() throws Exception {
         }
     }
 
-    public void testGeohashCellFilter() throws IOException {
+    public void testLegacyGeohashCellFilter() throws IOException {
         String geohash = randomhash(10);
         logger.info("Testing geohash_cell filter for [{}]", geohash);
 
@@ -457,8 +461,11 @@ public void testGeohashCellFilter() throws IOException {
         logger.info("Parent Neighbors {}", parentNeighbors);
 
         ensureYellow();
+        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.V_5_0_0_alpha5);
+        Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
 
-        client().admin().indices().prepareCreate("locations").addMapping("location", "pin", "type=geo_point,geohash_prefix=true,lat_lon=false").execute().actionGet();
+        client().admin().indices().prepareCreate("locations").setSettings(settings).addMapping("location", "pin",
+            "type=geo_point,geohash_prefix=true,lat_lon=false").execute().actionGet();
 
         // Index a pin
         client().prepareIndex("locations", "location", "1").setCreate(true).setSource("pin", geohash).execute().actionGet();
diff --git a/core/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java b/core/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java
index 067c2dcac0556..7e8aa1ad9d0fe 100644
--- a/core/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java
+++ b/core/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java
@@ -21,6 +21,7 @@
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.locationtech.spatial4j.shape.Rectangle;
 import com.vividsolutions.jts.geom.Coordinate;
 
@@ -69,7 +70,8 @@ public void testNullShape() throws Exception {
         client().admin().indices().prepareCreate("test").addMapping("type1", mapping).execute().actionGet();
         ensureGreen();
 
-        client().prepareIndex("test", "type1", "aNullshape").setSource("{\"location\": null}").setRefreshPolicy(IMMEDIATE).get();
+        client().prepareIndex("test", "type1", "aNullshape").setSource("{\"location\": null}", XContentType.JSON)
+            .setRefreshPolicy(IMMEDIATE).get();
         GetResponse result = client().prepareGet("test", "type1", "aNullshape").execute().actionGet();
         assertThat(result.getField("location"), nullValue());
     }
@@ -252,8 +254,8 @@ public void testShapeFetchingPath() throws Exception {
                 .setSource(
                         String.format(
                                 Locale.ROOT, "{ %s, \"1\" : { %s, \"2\" : { %s, \"3\" : { %s } }} }", location, location, location, location
-                        )
-                ).setRefreshPolicy(IMMEDIATE).get();
+                        ), XContentType.JSON)
+            .setRefreshPolicy(IMMEDIATE).get();
         client().prepareIndex("test", "type", "1")
                 .setSource(jsonBuilder().startObject().startObject("location")
                         .field("type", "polygon")
diff --git a/core/src/test/java/org/elasticsearch/search/internal/AliasFilterTests.java b/core/src/test/java/org/elasticsearch/search/internal/AliasFilterTests.java
new file mode 100644
index 0000000000000..ba21a63f683f5
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/internal/AliasFilterTests.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.internal;
+
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.query.TermQueryBuilder;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.EqualsHashCodeTestUtils;
+
+import java.util.Arrays;
+
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.instanceOf;
+
+public class AliasFilterTests extends ESTestCase {
+
+    public void testEqualsAndHashCode() {
+        final QueryBuilder filter = QueryBuilders.termQuery("field", "value");
+        final String[] aliases = new String[] { "alias_0", "alias_1" };
+        final AliasFilter aliasFilter = new AliasFilter(filter, aliases);
+        final EqualsHashCodeTestUtils.CopyFunction<AliasFilter> aliasFilterCopyFunction = x -> {
+            assertThat(x.getQueryBuilder(), instanceOf(TermQueryBuilder.class));
+            final BytesStreamOutput out = new BytesStreamOutput();
+            x.getQueryBuilder().writeTo(out);
+            final QueryBuilder otherFilter = new TermQueryBuilder(out.bytes().streamInput());
+            final String[] otherAliases = Arrays.copyOf(x.getAliases(), x.getAliases().length);
+            return new AliasFilter(otherFilter, otherAliases);
+        };
+
+        final EqualsHashCodeTestUtils.MutateFunction<AliasFilter> aliasFilterMutationFunction = x -> {
+            assertThat(x.getQueryBuilder(), instanceOf(TermQueryBuilder.class));
+            final BytesStreamOutput out = new BytesStreamOutput();
+            x.getQueryBuilder().writeTo(out);
+            final QueryBuilder otherFilter = new TermQueryBuilder(out.bytes().streamInput());
+            assertThat(x.getAliases().length, greaterThan(0));
+            final String[] otherAliases = Arrays.copyOf(x.getAliases(), x.getAliases().length - 1);
+            return new AliasFilter(otherFilter, otherAliases);
+        };
+
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(aliasFilter, aliasFilterCopyFunction, aliasFilterMutationFunction);
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/search/internal/DefaultSearchContextTests.java b/core/src/test/java/org/elasticsearch/search/internal/DefaultSearchContextTests.java
deleted file mode 100644
index 9ab5af9f393b0..0000000000000
--- a/core/src/test/java/org/elasticsearch/search/internal/DefaultSearchContextTests.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.internal;
-
-import org.apache.lucene.queries.TermsQuery;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.MatchAllDocsQuery;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.lucene.search.Queries;
-import org.elasticsearch.index.mapper.TypeFieldMapper;
-import org.elasticsearch.test.ESTestCase;
-
-import static org.apache.lucene.search.BooleanClause.Occur.FILTER;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.nullValue;
-
-public class DefaultSearchContextTests extends ESTestCase {
-
-    public void testCreateSearchFilter() {
-        Query searchFilter = DefaultSearchContext.createSearchFilter(new String[]{"type1", "type2"}, null, randomBoolean());
-        Query expectedQuery = new BooleanQuery.Builder()
-            .add(new TermsQuery(TypeFieldMapper.NAME, new BytesRef("type1"), new BytesRef("type2")), FILTER)
-            .build();
-        assertThat(searchFilter, equalTo(expectedQuery));
-
-        searchFilter = DefaultSearchContext.createSearchFilter(new String[]{"type1", "type2"}, new MatchAllDocsQuery(), randomBoolean());
-        expectedQuery = new BooleanQuery.Builder()
-            .add(new TermsQuery(TypeFieldMapper.NAME, new BytesRef("type1"), new BytesRef("type2")), FILTER)
-            .add(new MatchAllDocsQuery(), FILTER)
-            .build();
-        assertThat(searchFilter, equalTo(expectedQuery));
-
-        searchFilter = DefaultSearchContext.createSearchFilter(null, null, false);
-        assertThat(searchFilter, nullValue());
-
-        searchFilter = DefaultSearchContext.createSearchFilter(null, null, true);
-        expectedQuery = new BooleanQuery.Builder().add(Queries.newNonNestedFilter(), FILTER).build();
-        assertThat(searchFilter, equalTo(expectedQuery));
-
-        searchFilter = DefaultSearchContext.createSearchFilter(null, new MatchAllDocsQuery(), true);
-        expectedQuery = new BooleanQuery.Builder()
-            .add(new MatchAllDocsQuery(), FILTER)
-            .add(Queries.newNonNestedFilter(), FILTER)
-            .build();
-        assertThat(searchFilter, equalTo(expectedQuery));
-
-        searchFilter = DefaultSearchContext.createSearchFilter(null, new MatchAllDocsQuery(), false);
-        expectedQuery = new BooleanQuery.Builder()
-            .add(new MatchAllDocsQuery(), FILTER)
-            .build();
-        assertThat(searchFilter, equalTo(expectedQuery));
-    }
-
-}
diff --git a/core/src/test/java/org/elasticsearch/search/internal/InternalSearchHitTests.java b/core/src/test/java/org/elasticsearch/search/internal/InternalSearchHitTests.java
deleted file mode 100644
index dedd47d3e432e..0000000000000
--- a/core/src/test/java/org/elasticsearch/search/internal/InternalSearchHitTests.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.internal;
-
-import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.io.stream.InputStreamStreamInput;
-import org.elasticsearch.common.text.Text;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.search.SearchShardTarget;
-import org.elasticsearch.test.ESTestCase;
-
-import java.io.ByteArrayInputStream;
-import java.io.InputStream;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.nullValue;
-
-public class InternalSearchHitTests extends ESTestCase {
-
-    public void testSerializeShardTarget() throws Exception {
-        SearchShardTarget target = new SearchShardTarget("_node_id", new Index("_index", "_na_"), 0);
-
-        Map<String, InternalSearchHits> innerHits = new HashMap<>();
-        InternalSearchHit innerHit1 = new InternalSearchHit(0, "_id", new Text("_type"), null);
-        innerHit1.shardTarget(target);
-        InternalSearchHit innerInnerHit2 = new InternalSearchHit(0, "_id", new Text("_type"), null);
-        innerInnerHit2.shardTarget(target);
-        innerHits.put("1", new InternalSearchHits(new InternalSearchHit[]{innerInnerHit2}, 1, 1f));
-        innerHit1.setInnerHits(innerHits);
-        InternalSearchHit innerHit2 = new InternalSearchHit(0, "_id", new Text("_type"), null);
-        innerHit2.shardTarget(target);
-        InternalSearchHit innerHit3 = new InternalSearchHit(0, "_id", new Text("_type"), null);
-        innerHit3.shardTarget(target);
-
-        innerHits = new HashMap<>();
-        InternalSearchHit hit1 = new InternalSearchHit(0, "_id", new Text("_type"), null);
-        innerHits.put("1", new InternalSearchHits(new InternalSearchHit[]{innerHit1, innerHit2}, 1, 1f));
-        innerHits.put("2", new InternalSearchHits(new InternalSearchHit[]{innerHit3}, 1, 1f));
-        hit1.shardTarget(target);
-        hit1.setInnerHits(innerHits);
-
-        InternalSearchHit hit2 = new InternalSearchHit(0, "_id", new Text("_type"), null);
-        hit2.shardTarget(target);
-
-        InternalSearchHits hits = new InternalSearchHits(new InternalSearchHit[]{hit1, hit2}, 2, 1f);
-
-        InternalSearchHits.StreamContext context = new InternalSearchHits.StreamContext();
-        context.streamShardTarget(InternalSearchHits.StreamContext.ShardTargetType.STREAM);
-        BytesStreamOutput output = new BytesStreamOutput();
-        hits.writeTo(output, context);
-        InputStream input = output.bytes().streamInput();
-        context = new InternalSearchHits.StreamContext();
-        context.streamShardTarget(InternalSearchHits.StreamContext.ShardTargetType.STREAM);
-        InternalSearchHits results = InternalSearchHits.readSearchHits(new InputStreamStreamInput(input), context);
-        assertThat(results.getAt(0).shard(), equalTo(target));
-        assertThat(results.getAt(0).getInnerHits().get("1").getAt(0).shard(), nullValue());
-        assertThat(results.getAt(0).getInnerHits().get("1").getAt(0).getInnerHits().get("1").getAt(0).shard(), nullValue());
-        assertThat(results.getAt(0).getInnerHits().get("1").getAt(1).shard(), nullValue());
-        assertThat(results.getAt(0).getInnerHits().get("2").getAt(0).shard(), nullValue());
-        assertThat(results.getAt(1).shard(), equalTo(target));
-    }
-
-}
diff --git a/core/src/test/java/org/elasticsearch/search/internal/ScrollContextTests.java b/core/src/test/java/org/elasticsearch/search/internal/ScrollContextTests.java
new file mode 100644
index 0000000000000..de4863dd92a08
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/internal/ScrollContextTests.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.internal;
+
+import org.elasticsearch.test.ESTestCase;
+
+public class ScrollContextTests extends ESTestCase {
+
+    public void testStoringObjectsInScrollContext() {
+        final ScrollContext scrollContext = new ScrollContext();
+        final String key = randomAlphaOfLengthBetween(1, 16);
+        assertNull(scrollContext.getFromContext(key));
+
+        final String value = randomAlphaOfLength(6);
+        scrollContext.putInContext(key, value);
+
+        assertEquals(value, scrollContext.getFromContext(key));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/internal/ShardSearchTransportRequestTests.java b/core/src/test/java/org/elasticsearch/search/internal/ShardSearchTransportRequestTests.java
index c2e99b09dc330..0dabd6349e095 100644
--- a/core/src/test/java/org/elasticsearch/search/internal/ShardSearchTransportRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/search/internal/ShardSearchTransportRequestTests.java
@@ -19,59 +19,47 @@
 
 package org.elasticsearch.search.internal;
 
+import org.elasticsearch.Version;
+import org.elasticsearch.action.OriginalIndices;
 import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.cluster.routing.ShardRouting;
-import org.elasticsearch.cluster.routing.ShardRoutingState;
-import org.elasticsearch.cluster.routing.TestShardRouting;
-import org.elasticsearch.cluster.routing.UnassignedInfo;
+import org.elasticsearch.cluster.metadata.AliasMetaData;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.CheckedFunction;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.query.RandomQueryBuilder;
 import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.indices.IndicesModule;
-import org.elasticsearch.search.SearchModule;
-import org.elasticsearch.search.SearchRequestTests;
-import org.elasticsearch.test.ESTestCase;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
+import org.elasticsearch.indices.InvalidAliasNameException;
+import org.elasticsearch.search.AbstractSearchTestCase;
 
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
+import java.util.Base64;
+import java.util.Optional;
 
-import static java.util.Collections.emptyList;
+import static org.elasticsearch.index.query.QueryBuilders.termQuery;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
 
-public class ShardSearchTransportRequestTests extends ESTestCase {
-
-    private static NamedWriteableRegistry namedWriteableRegistry;
-
-    @BeforeClass
-    public static void beforeClass() {
-        IndicesModule indicesModule = new IndicesModule(emptyList()) {
-            @Override
-            protected void configure() {
-                bindMapperExtension();
-            }
-        };
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList()) {
-            @Override
-            protected void configureSearch() {
-                // Skip me
-            }
-        };
-        List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
-        entries.addAll(indicesModule.getNamedWriteables());
-        entries.addAll(searchModule.getNamedWriteables());
-        namedWriteableRegistry = new NamedWriteableRegistry(entries);
-    }
-
-    @AfterClass
-    public static void afterClass() {
-        namedWriteableRegistry = null;
-    }
+public class ShardSearchTransportRequestTests extends AbstractSearchTestCase {
+    private IndexMetaData baseMetaData = IndexMetaData.builder("test").settings(Settings.builder()
+        .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build())
+        .numberOfShards(1).numberOfReplicas(1).build();
 
     public void testSerialization() throws Exception {
         ShardSearchTransportRequest shardSearchTransportRequest = createShardSearchTransportRequest();
@@ -81,7 +69,7 @@ public void testSerialization() throws Exception {
                 ShardSearchTransportRequest deserializedRequest = new ShardSearchTransportRequest();
                 deserializedRequest.readFrom(in);
                 assertEquals(deserializedRequest.scroll(), shardSearchTransportRequest.scroll());
-                assertArrayEquals(deserializedRequest.filteringAliases(), shardSearchTransportRequest.filteringAliases());
+                assertEquals(deserializedRequest.filteringAliases(), shardSearchTransportRequest.filteringAliases());
                 assertArrayEquals(deserializedRequest.indices(), shardSearchTransportRequest.indices());
                 assertArrayEquals(deserializedRequest.types(), shardSearchTransportRequest.types());
                 assertEquals(deserializedRequest.indicesOptions(), shardSearchTransportRequest.indicesOptions());
@@ -93,22 +81,160 @@ public void testSerialization() throws Exception {
                 assertEquals(deserializedRequest.numberOfShards(), shardSearchTransportRequest.numberOfShards());
                 assertEquals(deserializedRequest.cacheKey(), shardSearchTransportRequest.cacheKey());
                 assertNotSame(deserializedRequest, shardSearchTransportRequest);
+                assertEquals(deserializedRequest.filteringAliases(), shardSearchTransportRequest.filteringAliases());
+                assertEquals(deserializedRequest.indexBoost(), shardSearchTransportRequest.indexBoost(), 0.0f);
             }
         }
     }
 
-    private static ShardSearchTransportRequest createShardSearchTransportRequest() throws IOException {
-        SearchRequest searchRequest = SearchRequestTests.createSearchRequest();
-        ShardId shardId = new ShardId(randomAsciiOfLengthBetween(2, 10), randomAsciiOfLengthBetween(2, 10), randomInt());
-        ShardRouting shardRouting = TestShardRouting.newShardRouting(shardId, null, null, randomBoolean(), ShardRoutingState.UNASSIGNED,
-            new UnassignedInfo(randomFrom(UnassignedInfo.Reason.values()), "reason"));
-        String[] filteringAliases;
+    private ShardSearchTransportRequest createShardSearchTransportRequest() throws IOException {
+        SearchRequest searchRequest = createSearchRequest();
+        ShardId shardId = new ShardId(randomAlphaOfLengthBetween(2, 10), randomAlphaOfLengthBetween(2, 10), randomInt());
+        final AliasFilter filteringAliases;
         if (randomBoolean()) {
-            filteringAliases = generateRandomStringArray(10, 10, false, false);
+            String[] strings = generateRandomStringArray(10, 10, false, false);
+            filteringAliases = new AliasFilter(RandomQueryBuilder.createQuery(random()), strings);
         } else {
-            filteringAliases = Strings.EMPTY_ARRAY;
+            filteringAliases = new AliasFilter(null, Strings.EMPTY_ARRAY);
+        }
+        return new ShardSearchTransportRequest(new OriginalIndices(searchRequest), searchRequest, shardId,
+                randomIntBetween(1, 100), filteringAliases, randomBoolean() ? 1.0f : randomFloat(), Math.abs(randomLong()), null);
+    }
+
+    public void testFilteringAliases() throws Exception {
+        IndexMetaData indexMetaData = baseMetaData;
+        indexMetaData = add(indexMetaData, "cats", filter(termQuery("animal", "cat")));
+        indexMetaData = add(indexMetaData, "dogs", filter(termQuery("animal", "dog")));
+        indexMetaData = add(indexMetaData, "all", null);
+
+        assertThat(indexMetaData.getAliases().containsKey("cats"), equalTo(true));
+        assertThat(indexMetaData.getAliases().containsKey("dogs"), equalTo(true));
+        assertThat(indexMetaData.getAliases().containsKey("turtles"), equalTo(false));
+
+        assertEquals(aliasFilter(indexMetaData, "cats"), QueryBuilders.termQuery("animal", "cat"));
+        assertEquals(aliasFilter(indexMetaData, "cats", "dogs"), QueryBuilders.boolQuery().should(QueryBuilders.termQuery("animal", "cat"))
+            .should(QueryBuilders.termQuery("animal", "dog")));
+
+        // Non-filtering alias should turn off all filters because filters are ORed
+        assertThat(aliasFilter(indexMetaData,"all"), nullValue());
+        assertThat(aliasFilter(indexMetaData, "cats", "all"), nullValue());
+        assertThat(aliasFilter(indexMetaData, "all", "cats"), nullValue());
+
+        indexMetaData = add(indexMetaData, "cats", filter(termQuery("animal", "feline")));
+        indexMetaData = add(indexMetaData, "dogs", filter(termQuery("animal", "canine")));
+        assertEquals(aliasFilter(indexMetaData, "dogs", "cats"),QueryBuilders.boolQuery()
+            .should(QueryBuilders.termQuery("animal", "canine"))
+            .should(QueryBuilders.termQuery("animal", "feline")));
+    }
+
+    public void testRemovedAliasFilter() throws Exception {
+        IndexMetaData indexMetaData = baseMetaData;
+        indexMetaData = add(indexMetaData, "cats", filter(termQuery("animal", "cat")));
+        indexMetaData = remove(indexMetaData, "cats");
+        try {
+            aliasFilter(indexMetaData, "cats");
+            fail("Expected InvalidAliasNameException");
+        } catch (InvalidAliasNameException e) {
+            assertThat(e.getMessage(), containsString("Invalid alias name [cats]"));
+        }
+    }
+
+    public void testUnknownAliasFilter() throws Exception {
+        IndexMetaData indexMetaData = baseMetaData;
+        indexMetaData = add(indexMetaData, "cats", filter(termQuery("animal", "cat")));
+        indexMetaData = add(indexMetaData, "dogs", filter(termQuery("animal", "dog")));
+        IndexMetaData finalIndexMetadata = indexMetaData;
+        expectThrows(InvalidAliasNameException.class, () -> aliasFilter(finalIndexMetadata, "unknown"));
+    }
+
+    public static CompressedXContent filter(QueryBuilder filterBuilder) throws IOException {
+        XContentBuilder builder = XContentFactory.jsonBuilder();
+        filterBuilder.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        builder.close();
+        return new CompressedXContent(builder.string());
+    }
+
+    private IndexMetaData remove(IndexMetaData indexMetaData, String alias) {
+        IndexMetaData build = IndexMetaData.builder(indexMetaData).removeAlias(alias).build();
+        return build;
+    }
+
+    private IndexMetaData add(IndexMetaData indexMetaData, String alias, @Nullable CompressedXContent filter) {
+        return IndexMetaData.builder(indexMetaData).putAlias(AliasMetaData.builder(alias).filter(filter).build()).build();
+    }
+
+    public QueryBuilder aliasFilter(IndexMetaData indexMetaData, String... aliasNames) {
+        CheckedFunction<byte[], Optional<QueryBuilder>, IOException> filterParser = bytes -> {
+            try (XContentParser parser = XContentFactory.xContent(bytes).createParser(xContentRegistry(), bytes)) {
+                return new QueryParseContext(parser).parseInnerQueryBuilder();
+            }
+        };
+        return ShardSearchRequest.parseAliasFilter(filterParser, indexMetaData, aliasNames);
+    }
+
+    // BWC test for changes from #20916
+    public void testSerialize50Request() throws IOException {
+        BytesArray requestBytes = new BytesArray(Base64.getDecoder()
+            // this is a base64 encoded request generated with the same input
+            .decode("AAh4cXptdEhJcgdnT0d1ZldWyfL/sgQBJAHkDAMBAAIBAQ4TWlljWlZ5TkVmRU5xQnFQVHBjVBRZbUpod2pRV2dDSXVxRXpRaEdGVBRFZWFJY0plT2hn" +
+                "UEpISFhmSXR6Qw5XZ1hQcmFidWhWalFSQghuUWNwZ2JjQxBtZldRREJPaGF3UnlQSE56EVhQSUtRa25Iekh3bU5kbGVECWlFT2NIeEh3RgZIYXpMTWgUeGJq" +
+                "VU9Tdkdua3RORU5QZkNrb1EOalRyWGh5WXhvZ3plV2UUcWlXZFl2eUFUSXdPVGdMUUtYTHAJU3RKR3JxQkVJEkdEQ01xUHpnWWNaT3N3U3prSRIUeURlVFpM" +
+                "Q1lBZERZcWpDb3NOVWIST1NyQlZtdUNrd0F1UXRvdVRjEGp6RlVMd1dqc3VtUVNaTk0JT3N2cnpLQ3ZLBmRpS1J6cgdYbmVhZnBxBUlTUU9pEEJMcm1ERXVs" +
+                "eXhESlBoVkgTaWdUUmtVZGh4d0FFc2ZKRm9ZahNrb01XTnFFd2NWSVVDU3pWS2xBC3JVTWV3V2tUUWJUE3VGQU1Hd21CYUFMTmNQZkxobXUIZ3dxWHBxWXcF" +
+                "bmNDZUEOTFBSTEpYZVF6Z3d2eE0PV1BucUFacll6WWRxa1hCDGxkbXNMaVRzcUZXbAtSY0NsY3FNdlJQcv8BAP////8PAQAAARQAAQp5THlIcHdQeGtMAAAB" +
+                "AQAAAAEDbkVLAQMBCgACAAADAQABAAAAAQhIc25wRGxQbwEBQgABAAACAQMAAAEIAAAJMF9OSG9kSmh2HwABAwljRW5MVWxFbVQFemlxWG8KcXZQTkRUUGJk" +
+                "bgECCkpMbXVMT1dtVnkISEdUUHhsd0cBAAEJAAABA2lkcz+rKsUAAAAAAAAAAAECAQYAAgwxX0ZlRWxSQkhzQ07/////DwABAAEDCnRyYXFHR1hjVHkKTERY" +
+                "aE1HRWVySghuSWtzbEtXUwABCgEHSlRwQnhwdwAAAQECAgAAAAAAAQcyX3FlYmNDGQEEBklxZU9iUQdTc01Gek5YCWlMd2xuamNRQwNiVncAAUHt61kAAQR0" +
+                "ZXJtP4AAAAANbUtDSnpHU3lidm5KUBUMaVpqeG9vcm5QSFlvAAEBLGdtcWxuRWpWTXdvTlhMSHh0RWlFdHBnbEF1cUNmVmhoUVlwRFZxVllnWWV1A2ZvbwEA" +
+                "AQhwYWlubGVzc/8AALk4AAAAAAABAAAAAAAAAwpKU09PU0ZmWnhFClVqTGxMa2p3V2gKdUJwZ3R3dXFER5Hg97uT7MOmPgEADw"));
+        try (StreamInput in = new NamedWriteableAwareStreamInput(requestBytes.streamInput(), namedWriteableRegistry)) {
+            in.setVersion(Version.V_5_0_0);
+            ShardSearchTransportRequest readRequest = new ShardSearchTransportRequest();
+            readRequest.readFrom(in);
+            assertEquals(0, in.available());
+            IllegalStateException illegalStateException = expectThrows(IllegalStateException.class, () -> readRequest.filteringAliases());
+            assertEquals("alias filter for aliases: [JSOOSFfZxE, UjLlLkjwWh, uBpgtwuqDG] must be rewritten first",
+                illegalStateException.getMessage());
+            IndexMetaData.Builder indexMetadata = new IndexMetaData.Builder(baseMetaData)
+                .putAlias(AliasMetaData.newAliasMetaDataBuilder("JSOOSFfZxE").filter("{\"term\" : {\"foo\" : \"bar\"}}"))
+                .putAlias(AliasMetaData.newAliasMetaDataBuilder("UjLlLkjwWh").filter("{\"term\" : {\"foo\" : \"bar1\"}}"))
+                .putAlias(AliasMetaData.newAliasMetaDataBuilder("uBpgtwuqDG").filter("{\"term\" : {\"foo\" : \"bar2\"}}"));
+            IndexSettings indexSettings = new IndexSettings(indexMetadata.build(), Settings.EMPTY);
+            final long nowInMillis = randomNonNegativeLong();
+            QueryShardContext context = new QueryShardContext(
+                0, indexSettings, null, null, null, null, null, xContentRegistry(), null, null, () -> nowInMillis);
+            readRequest.rewrite(context);
+            QueryBuilder queryBuilder = readRequest.filteringAliases();
+            assertEquals(queryBuilder, QueryBuilders.boolQuery()
+                .should(QueryBuilders.termQuery("foo", "bar"))
+                .should(QueryBuilders.termQuery("foo", "bar1"))
+                .should(QueryBuilders.termQuery("foo", "bar2"))
+            );
+            BytesStreamOutput output = new BytesStreamOutput();
+            output.setVersion(Version.V_5_0_0);
+            readRequest.writeTo(output);
+            assertEquals(output.bytes().toBytesRef(), requestBytes.toBytesRef());
+        }
+    }
+
+    // BWC test for changes from #21393
+    public void testSerialize50RequestForIndexBoost() throws IOException {
+        BytesArray requestBytes = new BytesArray(Base64.getDecoder()
+            // this is a base64 encoded request generated with the same input
+            .decode("AAZpbmRleDEWTjEyM2trbHFUT21XZDY1Z2VDYlo5ZwABBAABAAIA/wD/////DwABBmluZGV4MUAAAAAAAAAAAP////8PAAAAAAAAAgAAAA" +
+                "AAAPa/q8mOKwIAJg=="));
+
+        try (StreamInput in = new NamedWriteableAwareStreamInput(requestBytes.streamInput(), namedWriteableRegistry)) {
+            in.setVersion(Version.V_5_0_0);
+            ShardSearchTransportRequest readRequest = new ShardSearchTransportRequest();
+            readRequest.readFrom(in);
+            assertEquals(0, in.available());
+            assertEquals(2.0f, readRequest.indexBoost(), 0);
+
+            BytesStreamOutput output = new BytesStreamOutput();
+            output.setVersion(Version.V_5_0_0);
+            readRequest.writeTo(output);
+            assertEquals(output.bytes().toBytesRef(), requestBytes.toBytesRef());
         }
-        return new ShardSearchTransportRequest(searchRequest, shardRouting,
-                randomIntBetween(1, 100), filteringAliases, Math.abs(randomLong()));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/morelikethis/MoreLikeThisIT.java b/core/src/test/java/org/elasticsearch/search/morelikethis/MoreLikeThisIT.java
index 21f15860759e5..890defdf993cd 100644
--- a/core/src/test/java/org/elasticsearch/search/morelikethis/MoreLikeThisIT.java
+++ b/core/src/test/java/org/elasticsearch/search/morelikethis/MoreLikeThisIT.java
@@ -622,4 +622,18 @@ public void testSelectFields() throws IOException, ExecutionException, Interrupt
         assertSearchResponse(response);
         assertHitCount(response, 1);
     }
+
+    public void testWithRouting() throws IOException {
+        client().prepareIndex("index", "type", "1").setRouting("3").setSource("text", "this is a document").get();
+        client().prepareIndex("index", "type", "2").setRouting("1").setSource("text", "this is another document").get();
+        client().prepareIndex("index", "type", "3").setRouting("4").setSource("text", "this is yet another document").get();
+        refresh("index");
+
+        Item item = new Item("index", "type", "2").routing("1");
+        MoreLikeThisQueryBuilder moreLikeThisQueryBuilder = new MoreLikeThisQueryBuilder(new String[]{"text"}, null, new Item[]{item});
+        moreLikeThisQueryBuilder.minTermFreq(1);
+        moreLikeThisQueryBuilder.minDocFreq(1);
+        SearchResponse searchResponse = client().prepareSearch("index").setQuery(moreLikeThisQueryBuilder).get();
+        assertEquals(2, searchResponse.getHits().totalHits);
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/msearch/MultiSearchIT.java b/core/src/test/java/org/elasticsearch/search/msearch/MultiSearchIT.java
index ccc3102f3a80b..c7e78f2cc28fd 100644
--- a/core/src/test/java/org/elasticsearch/search/msearch/MultiSearchIT.java
+++ b/core/src/test/java/org/elasticsearch/search/msearch/MultiSearchIT.java
@@ -21,6 +21,7 @@
 
 import org.elasticsearch.action.search.MultiSearchRequest;
 import org.elasticsearch.action.search.MultiSearchResponse;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.test.ESIntegTestCase;
 
@@ -59,7 +60,7 @@ public void testSimpleMultiSearchMoreRequests() {
         createIndex("test");
         int numDocs = randomIntBetween(0, 16);
         for (int i = 0; i < numDocs; i++) {
-            client().prepareIndex("test", "type", Integer.toString(i)).setSource("{}").get();
+            client().prepareIndex("test", "type", Integer.toString(i)).setSource("{}", XContentType.JSON).get();
         }
         refresh();
 
diff --git a/core/src/test/java/org/elasticsearch/search/nested/SimpleNestedIT.java b/core/src/test/java/org/elasticsearch/search/nested/SimpleNestedIT.java
index ec18e528a4007..1ee5bec6a7638 100644
--- a/core/src/test/java/org/elasticsearch/search/nested/SimpleNestedIT.java
+++ b/core/src/test/java/org/elasticsearch/search/nested/SimpleNestedIT.java
@@ -164,6 +164,113 @@ public void testSimpleNested() throws Exception {
         assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
     }
 
+    public void testSimpleNested_singleType() throws Exception {
+        assertAcked(prepareCreate("test")
+            .setSettings(Settings.builder().put("index.mapping.single_type", true))
+            .addMapping("type1", "nested1", "type=nested"));
+        ensureGreen();
+
+        // check on no data, see it works
+        SearchResponse searchResponse = client().prepareSearch("test").setQuery(termQuery("_all", "n_value1_1")).execute().actionGet();
+        assertThat(searchResponse.getHits().totalHits(), equalTo(0L));
+        searchResponse = client().prepareSearch("test").setQuery(termQuery("n_field1", "n_value1_1")).execute().actionGet();
+        assertThat(searchResponse.getHits().totalHits(), equalTo(0L));
+
+        client().prepareIndex("test", "type1", "1").setSource(jsonBuilder().startObject()
+            .field("field1", "value1")
+            .startArray("nested1")
+            .startObject()
+            .field("n_field1", "n_value1_1")
+            .field("n_field2", "n_value2_1")
+            .endObject()
+            .startObject()
+            .field("n_field1", "n_value1_2")
+            .field("n_field2", "n_value2_2")
+            .endObject()
+            .endArray()
+            .endObject()).execute().actionGet();
+
+        waitForRelocation(ClusterHealthStatus.GREEN);
+        // flush, so we fetch it from the index (as see that we filter nested docs)
+        flush();
+        GetResponse getResponse = client().prepareGet("test", "type1", "1").get();
+        assertThat(getResponse.isExists(), equalTo(true));
+        assertThat(getResponse.getSourceAsBytes(), notNullValue());
+
+        // check the numDocs
+        assertDocumentCount("test", 3);
+
+        // check that _all is working on nested docs
+        searchResponse = client().prepareSearch("test").setQuery(termQuery("_all", "n_value1_1")).execute().actionGet();
+        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
+        searchResponse = client().prepareSearch("test").setQuery(termQuery("n_field1", "n_value1_1")).execute().actionGet();
+        assertThat(searchResponse.getHits().totalHits(), equalTo(0L));
+
+        // search for something that matches the nested doc, and see that we don't find the nested doc
+        searchResponse = client().prepareSearch("test").setQuery(matchAllQuery()).get();
+        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
+        searchResponse = client().prepareSearch("test").setQuery(termQuery("n_field1", "n_value1_1")).get();
+        assertThat(searchResponse.getHits().totalHits(), equalTo(0L));
+
+        // now, do a nested query
+        searchResponse = client().prepareSearch("test").setQuery(nestedQuery("nested1", termQuery("nested1.n_field1", "n_value1_1"), ScoreMode.Avg)).get();
+        assertNoFailures(searchResponse);
+        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
+
+        searchResponse = client().prepareSearch("test").setQuery(nestedQuery("nested1", termQuery("nested1.n_field1", "n_value1_1"), ScoreMode.Avg)).setSearchType(SearchType.DFS_QUERY_THEN_FETCH).get();
+        assertNoFailures(searchResponse);
+        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
+
+        // add another doc, one that would match if it was not nested...
+
+        client().prepareIndex("test", "type1", "2").setSource(jsonBuilder().startObject()
+            .field("field1", "value1")
+            .startArray("nested1")
+            .startObject()
+            .field("n_field1", "n_value1_1")
+            .field("n_field2", "n_value2_2")
+            .endObject()
+            .startObject()
+            .field("n_field1", "n_value1_2")
+            .field("n_field2", "n_value2_1")
+            .endObject()
+            .endArray()
+            .endObject()).execute().actionGet();
+        waitForRelocation(ClusterHealthStatus.GREEN);
+        // flush, so we fetch it from the index (as see that we filter nested docs)
+        flush();
+        assertDocumentCount("test", 6);
+
+        searchResponse = client().prepareSearch("test").setQuery(nestedQuery("nested1",
+            boolQuery().must(termQuery("nested1.n_field1", "n_value1_1")).must(termQuery("nested1.n_field2", "n_value2_1")), ScoreMode.Avg)).execute().actionGet();
+        assertNoFailures(searchResponse);
+        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
+
+        // filter
+        searchResponse = client().prepareSearch("test").setQuery(boolQuery().must(matchAllQuery()).mustNot(nestedQuery("nested1",
+            boolQuery().must(termQuery("nested1.n_field1", "n_value1_1")).must(termQuery("nested1.n_field2", "n_value2_1")), ScoreMode.Avg))).execute().actionGet();
+        assertNoFailures(searchResponse);
+        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
+
+        // check with type prefix
+        searchResponse = client().prepareSearch("test").setQuery(nestedQuery("nested1",
+            boolQuery().must(termQuery("nested1.n_field1", "n_value1_1")).must(termQuery("nested1.n_field2", "n_value2_1")), ScoreMode.Avg)).execute().actionGet();
+        assertNoFailures(searchResponse);
+        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
+
+        // check delete, so all is gone...
+        DeleteResponse deleteResponse = client().prepareDelete("test", "type1", "2").execute().actionGet();
+        assertEquals(DocWriteResponse.Result.DELETED, deleteResponse.getResult());
+
+        // flush, so we fetch it from the index (as see that we filter nested docs)
+        flush();
+        assertDocumentCount("test", 3);
+
+        searchResponse = client().prepareSearch("test").setQuery(nestedQuery("nested1", termQuery("nested1.n_field1", "n_value1_1"), ScoreMode.Avg)).execute().actionGet();
+        assertNoFailures(searchResponse);
+        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
+    }
+
     public void testMultiNested() throws Exception {
         assertAcked(prepareCreate("test")
                 .addMapping("type1", jsonBuilder().startObject().startObject("type1").startObject("properties")
@@ -317,9 +424,9 @@ public void testExplain() throws Exception {
                 .execute().actionGet();
         assertNoFailures(searchResponse);
         assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
-        Explanation explanation = searchResponse.getHits().hits()[0].explanation();
+        Explanation explanation = searchResponse.getHits().hits()[0].getExplanation();
         assertThat(explanation.getValue(), equalTo(searchResponse.getHits().getHits()[0].score()));
-        assertThat(explanation.toString(), startsWith("0.36464313 = sum of:\n  0.36464313 = Score based on 2 child docs in range from 0 to 1"));
+        assertThat(explanation.toString(), startsWith("0.36464313 = Score based on 2 child docs in range from 0 to 1"));
     }
 
     public void testSimpleNestedSorting() throws Exception {
diff --git a/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java b/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java
index 3758715cbacb1..4b3b890a20d70 100644
--- a/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java
+++ b/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESIntegTestCase;
 
@@ -90,15 +91,15 @@ public void testNoPreferenceRandom() throws Exception {
 
         final Client client = internalCluster().smartClient();
         SearchResponse searchResponse = client.prepareSearch("test").setQuery(matchAllQuery()).execute().actionGet();
-        String firstNodeId = searchResponse.getHits().getAt(0).shard().nodeId();
+        String firstNodeId = searchResponse.getHits().getAt(0).shard().getNodeId();
         searchResponse = client.prepareSearch("test").setQuery(matchAllQuery()).execute().actionGet();
-        String secondNodeId = searchResponse.getHits().getAt(0).shard().nodeId();
+        String secondNodeId = searchResponse.getHits().getAt(0).shard().getNodeId();
 
         assertThat(firstNodeId, not(equalTo(secondNodeId)));
     }
 
     public void testSimplePreference() throws Exception {
-        client().admin().indices().prepareCreate("test").setSettings("{\"number_of_replicas\": 1}").get();
+        client().admin().indices().prepareCreate("test").setSettings("{\"number_of_replicas\": 1}", XContentType.JSON).get();
         ensureGreen();
 
         client().prepareIndex("test", "type1").setSource("field1", "value1").execute().actionGet();
@@ -131,7 +132,7 @@ public void testSimplePreference() throws Exception {
     }
 
     public void testReplicaPreference() throws Exception {
-        client().admin().indices().prepareCreate("test").setSettings("{\"number_of_replicas\": 0}").get();
+        client().admin().indices().prepareCreate("test").setSettings("{\"number_of_replicas\": 0}", XContentType.JSON).get();
         ensureGreen();
 
         client().prepareIndex("test", "type1").setSource("field1", "value1").execute().actionGet();
@@ -147,7 +148,7 @@ public void testReplicaPreference() throws Exception {
         SearchResponse resp = client().prepareSearch().setQuery(matchAllQuery()).setPreference("_replica_first").execute().actionGet();
         assertThat(resp.getHits().totalHits(), equalTo(1L));
 
-        client().admin().indices().prepareUpdateSettings("test").setSettings("{\"number_of_replicas\": 1}").get();
+        client().admin().indices().prepareUpdateSettings("test").setSettings("{\"number_of_replicas\": 1}", XContentType.JSON).get();
         ensureGreen("test");
 
         resp = client().prepareSearch().setQuery(matchAllQuery()).setPreference("_replica").execute().actionGet();
@@ -220,7 +221,7 @@ private void assertSearchOnRandomNodes(SearchRequestBuilder request) {
         for (int i = 0; i < 2; i++) {
             SearchResponse searchResponse = request.execute().actionGet();
             assertThat(searchResponse.getHits().getHits().length, greaterThan(0));
-            hitNodes.add(searchResponse.getHits().getAt(0).shard().nodeId());
+            hitNodes.add(searchResponse.getHits().getAt(0).shard().getNodeId());
         }
         assertThat(hitNodes.size(), greaterThan(1));
     }
diff --git a/core/src/test/java/org/elasticsearch/search/profile/ProfileResultTests.java b/core/src/test/java/org/elasticsearch/search/profile/ProfileResultTests.java
new file mode 100644
index 0000000000000..4e9a87e7488a2
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/profile/ProfileResultTests.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.profile;
+
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Predicate;
+
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.elasticsearch.test.XContentTestUtils.insertRandomFields;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
+
+public class ProfileResultTests extends ESTestCase {
+
+    public static ProfileResult createTestItem(int depth) {
+        String type = randomAlphaOfLengthBetween(5, 10);
+        String description = randomAlphaOfLengthBetween(5, 10);
+        int timingsSize = randomIntBetween(0, 5);
+        Map<String, Long> timings = new HashMap<>(timingsSize);
+        for (int i = 0; i < timingsSize; i++) {
+            long time = randomNonNegativeLong() / timingsSize;
+            if (randomBoolean()) {
+                // also often use "small" values in tests
+                time = randomNonNegativeLong() % 10000;
+            }
+            timings.put(randomAlphaOfLengthBetween(5, 10), time); // don't overflow Long.MAX_VALUE;
+        }
+        int childrenSize = depth > 0 ? randomIntBetween(0, 1) : 0;
+        List<ProfileResult> children = new ArrayList<>(childrenSize);
+        for (int i = 0; i < childrenSize; i++) {
+            children.add(createTestItem(depth - 1));
+        }
+        return new ProfileResult(type, description, timings, children);
+    }
+
+    public void testFromXContent() throws IOException {
+        doFromXContentTestWithRandomFields(false);
+    }
+
+    /**
+     * This test adds random fields and objects to the xContent rendered out to ensure we can parse it
+     * back to be forward compatible with additions to the xContent
+     */
+    public void testFromXContentWithRandomFields() throws IOException {
+        doFromXContentTestWithRandomFields(true);
+    }
+
+    private void doFromXContentTestWithRandomFields(boolean addRandomFields) throws IOException {
+        ProfileResult profileResult = createTestItem(2);
+        XContentType xContentType = randomFrom(XContentType.values());
+        boolean humanReadable = randomBoolean();
+        BytesReference originalBytes = toShuffledXContent(profileResult, xContentType, ToXContent.EMPTY_PARAMS, humanReadable);
+        BytesReference mutated;
+        if (addRandomFields) {
+            // "breakdown" just consists of key/value pairs, we shouldn't add anything random there
+            Predicate<String> excludeFilter = (s) -> s.endsWith(ProfileResult.BREAKDOWN.getPreferredName());
+            mutated = insertRandomFields(xContentType, originalBytes, excludeFilter, random());
+        } else {
+            mutated = originalBytes;
+        }
+        ProfileResult parsed;
+        try (XContentParser parser = createParser(xContentType.xContent(), mutated)) {
+            ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
+            parsed = ProfileResult.fromXContent(parser);
+            assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());
+            assertNull(parser.nextToken());
+        }
+        assertEquals(profileResult.getTime(), parsed.getTime());
+        assertToXContentEquivalent(originalBytes, toXContent(parsed, xContentType, humanReadable), xContentType);
+    }
+
+    public void testToXContent() throws IOException {
+        List<ProfileResult> children = new ArrayList<>();
+        children.add(new ProfileResult("child1", "desc1", Collections.singletonMap("key1", 100L), Collections.emptyList()));
+        children.add(new ProfileResult("child2", "desc2", Collections.singletonMap("key1", 123356L), Collections.emptyList()));
+        Map<String, Long> timings = new HashMap<>();
+        timings.put("key1", 123456L);
+        timings.put("key2", 100000L);
+        ProfileResult result = new ProfileResult("someType", "some description", timings, children);
+        XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint();
+        result.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        assertEquals("{\n" +
+                "  \"type\" : \"someType\",\n" +
+                "  \"description\" : \"some description\",\n" +
+                "  \"time\" : \"0.2234560000ms\",\n" +
+                "  \"time_in_nanos\" : 223456,\n" +
+                "  \"breakdown\" : {\n" +
+                "    \"key1\" : 123456,\n" +
+                "    \"key2\" : 100000\n" +
+                "  },\n" +
+                "  \"children\" : [\n" +
+                "    {\n" +
+                "      \"type\" : \"child1\",\n" +
+                "      \"description\" : \"desc1\",\n" +
+                "      \"time\" : \"0.0001000000000ms\",\n" +
+                "      \"time_in_nanos\" : 100,\n" +
+                "      \"breakdown\" : {\n" +
+                "        \"key1\" : 100\n" +
+                "      }\n" +
+                "    },\n" +
+                "    {\n" +
+                "      \"type\" : \"child2\",\n" +
+                "      \"description\" : \"desc2\",\n" +
+                "      \"time\" : \"0.1233560000ms\",\n" +
+                "      \"time_in_nanos\" : 123356,\n" +
+                "      \"breakdown\" : {\n" +
+                "        \"key1\" : 123356\n" +
+                "      }\n" +
+                "    }\n" +
+                "  ]\n" +
+          "}", builder.string());
+
+        builder = XContentFactory.jsonBuilder().prettyPrint();
+        result.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        assertEquals("{\n" +
+                "  \"type\" : \"someType\",\n" +
+                "  \"description\" : \"some description\",\n" +
+                "  \"time\" : \"0.2234560000ms\",\n" +
+                "  \"time_in_nanos\" : 223456,\n" +
+                "  \"breakdown\" : {\n" +
+                "    \"key1\" : 123456,\n" +
+                "    \"key2\" : 100000\n" +
+                "  },\n" +
+                "  \"children\" : [\n" +
+                "    {\n" +
+                "      \"type\" : \"child1\",\n" +
+                "      \"description\" : \"desc1\",\n" +
+                "      \"time\" : \"0.0001000000000ms\",\n" +
+                "      \"time_in_nanos\" : 100,\n" +
+                "      \"breakdown\" : {\n" +
+                "        \"key1\" : 100\n" +
+                "      }\n" +
+                "    },\n" +
+                "    {\n" +
+                "      \"type\" : \"child2\",\n" +
+                "      \"description\" : \"desc2\",\n" +
+                "      \"time\" : \"0.1233560000ms\",\n" +
+                "      \"time_in_nanos\" : 123356,\n" +
+                "      \"breakdown\" : {\n" +
+                "        \"key1\" : 123356\n" +
+                "      }\n" +
+                "    }\n" +
+                "  ]\n" +
+          "}", builder.string());
+
+        result = new ProfileResult("profileName", "some description", Collections.singletonMap("key1", 12345678L), Collections.emptyList());
+        builder = XContentFactory.jsonBuilder().prettyPrint();
+        result.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        assertEquals("{\n" +
+                "  \"type\" : \"profileName\",\n" +
+                "  \"description\" : \"some description\",\n" +
+                "  \"time\" : \"12.34567800ms\",\n" +
+                "  \"time_in_nanos\" : 12345678,\n" +
+                "  \"breakdown\" : {\n" +
+                "    \"key1\" : 12345678\n" +
+                "  }\n" +
+              "}", builder.string());
+
+        result = new ProfileResult("profileName", "some description", Collections.singletonMap("key1", 1234567890L),
+                Collections.emptyList());
+        builder = XContentFactory.jsonBuilder().prettyPrint();
+        result.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        assertEquals("{\n" +
+                "  \"type\" : \"profileName\",\n" +
+                "  \"description\" : \"some description\",\n" +
+                "  \"time\" : \"1234.567890ms\",\n" +
+                "  \"time_in_nanos\" : 1234567890,\n" +
+                "  \"breakdown\" : {\n" +
+                "    \"key1\" : 1234567890\n" +
+                "  }\n" +
+              "}", builder.string());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/profile/SearchProfileShardResultsTests.java b/core/src/test/java/org/elasticsearch/search/profile/SearchProfileShardResultsTests.java
new file mode 100644
index 0000000000000..7bc9b18860641
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/profile/SearchProfileShardResultsTests.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.profile;
+
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.search.profile.aggregation.AggregationProfileShardResult;
+import org.elasticsearch.search.profile.aggregation.AggregationProfileShardResultTests;
+import org.elasticsearch.search.profile.query.QueryProfileShardResult;
+import org.elasticsearch.search.profile.query.QueryProfileShardResultTests;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Predicate;
+
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureFieldName;
+import static org.elasticsearch.test.XContentTestUtils.insertRandomFields;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;;
+
+public class SearchProfileShardResultsTests  extends ESTestCase {
+
+    public static SearchProfileShardResults createTestItem() {
+        int size = rarely() ? 0 : randomIntBetween(1, 2);
+        Map<String, ProfileShardResult> searchProfileResults = new HashMap<>(size);
+        for (int i = 0; i < size; i++) {
+            List<QueryProfileShardResult> queryProfileResults = new ArrayList<>();
+            int queryItems = rarely() ? 0 : randomIntBetween(1, 2);
+            for (int q = 0; q < queryItems; q++) {
+                queryProfileResults.add(QueryProfileShardResultTests.createTestItem());
+            }
+            AggregationProfileShardResult aggProfileShardResult = AggregationProfileShardResultTests.createTestItem(1);
+            searchProfileResults.put(randomAlphaOfLengthBetween(5, 10), new ProfileShardResult(queryProfileResults, aggProfileShardResult));
+        }
+        return new SearchProfileShardResults(searchProfileResults);
+    }
+
+    public void testFromXContent() throws IOException {
+        doFromXContentTestWithRandomFields(false);
+    }
+
+    /**
+     * This test adds random fields and objects to the xContent rendered out to ensure we can parse it
+     * back to be forward compatible with additions to the xContent
+     */
+    public void testFromXContentWithRandomFields() throws IOException {
+        doFromXContentTestWithRandomFields(true);
+    }
+
+    private void doFromXContentTestWithRandomFields(boolean addRandomFields) throws IOException {
+        SearchProfileShardResults shardResult = createTestItem();
+        XContentType xContentType = randomFrom(XContentType.values());
+        boolean humanReadable = randomBoolean();
+        BytesReference originalBytes = toShuffledXContent(shardResult, xContentType, ToXContent.EMPTY_PARAMS, humanReadable);
+        BytesReference mutated;
+        if (addRandomFields) {
+            // The ProfileResults "breakdown" section just consists of key/value pairs, we shouldn't add anything random there
+            // also we don't want to insert into the root object here, its just the PROFILE_FIELD itself
+            Predicate<String> excludeFilter = (s) -> (s.isEmpty() || s.endsWith(ProfileResult.BREAKDOWN.getPreferredName()));
+            mutated = insertRandomFields(xContentType, originalBytes, excludeFilter, random());
+        } else {
+            mutated = originalBytes;
+        }
+        SearchProfileShardResults parsed;
+        try (XContentParser parser = createParser(xContentType.xContent(), mutated)) {
+            ensureExpectedToken(parser.nextToken(), XContentParser.Token.START_OBJECT, parser::getTokenLocation);
+            ensureFieldName(parser, parser.nextToken(), SearchProfileShardResults.PROFILE_FIELD);
+            ensureExpectedToken(parser.nextToken(), XContentParser.Token.START_OBJECT, parser::getTokenLocation);
+            parsed = SearchProfileShardResults.fromXContent(parser);
+            assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());
+            assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
+            assertNull(parser.nextToken());
+        }
+        assertToXContentEquivalent(originalBytes, toXContent(parsed, xContentType, humanReadable), xContentType);
+
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/search/profile/TimerTests.java b/core/src/test/java/org/elasticsearch/search/profile/TimerTests.java
new file mode 100644
index 0000000000000..fc1bd76f39ecd
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/profile/TimerTests.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.profile;
+
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+public class TimerTests extends ESTestCase {
+
+    public void testTimingInterval() {
+        final AtomicLong nanoTimeCallCounter = new AtomicLong();
+        Timer t = new Timer() {
+            long time = 50;
+            @Override
+            long nanoTime() {
+                nanoTimeCallCounter.incrementAndGet();
+                return time += 1;
+            }
+        };
+        for (int i = 0; i < 100000; ++i) {
+            t.start();
+            t.stop();
+            if (i < 256) {
+                // for the first 256 calls, nanoTime() is called
+                // once for `start` and once for `stop`
+                assertEquals((i + 1) * 2, nanoTimeCallCounter.get());
+            }
+        }
+        // only called nanoTime() 3356 times, which is significantly less than 100000
+        assertEquals(3356L, nanoTimeCallCounter.get());
+    }
+
+    public void testExtrapolate() {
+        Timer t = new Timer() {
+            long time = 50;
+            @Override
+            long nanoTime() {
+                return time += 42;
+            }
+        };
+        for (int i = 1; i < 100000; ++i) {
+            t.start();
+            t.stop();
+            assertEquals(i, t.getCount());
+            // Make sure the cumulated timing is 42 times the number of calls as expected
+            assertEquals(i * 42L, t.getApproximateTiming());
+        }
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/search/profile/aggregation/AggregationProfileShardResultTests.java b/core/src/test/java/org/elasticsearch/search/profile/aggregation/AggregationProfileShardResultTests.java
new file mode 100644
index 0000000000000..534767eacd1f4
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/profile/aggregation/AggregationProfileShardResultTests.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the \"License\"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.profile.aggregation;
+
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParserUtils;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.search.profile.ProfileResult;
+import org.elasticsearch.search.profile.ProfileResultTests;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
+
+public class AggregationProfileShardResultTests extends ESTestCase {
+
+    public static AggregationProfileShardResult createTestItem(int depth) {
+        int size = randomIntBetween(0, 5);
+        List<ProfileResult> aggProfileResults = new ArrayList<>(size);
+        for (int i = 0; i < size; i++) {
+            aggProfileResults.add(ProfileResultTests.createTestItem(1));
+        }
+        return new AggregationProfileShardResult(aggProfileResults);
+    }
+
+    public void testFromXContent() throws IOException {
+        AggregationProfileShardResult profileResult = createTestItem(2);
+        XContentType xContentType = randomFrom(XContentType.values());
+        boolean humanReadable = randomBoolean();
+        BytesReference originalBytes = toShuffledXContent(profileResult, xContentType, ToXContent.EMPTY_PARAMS, humanReadable);
+
+        AggregationProfileShardResult parsed;
+        try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) {
+            XContentParserUtils.ensureExpectedToken(parser.nextToken(), XContentParser.Token.START_OBJECT, parser::getTokenLocation);
+            XContentParserUtils.ensureFieldName(parser, parser.nextToken(), AggregationProfileShardResult.AGGREGATIONS);
+            XContentParserUtils.ensureExpectedToken(parser.nextToken(), XContentParser.Token.START_ARRAY, parser::getTokenLocation);
+            parsed = AggregationProfileShardResult.fromXContent(parser);
+            assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
+            assertNull(parser.nextToken());
+        }
+        assertToXContentEquivalent(originalBytes, toXContent(parsed, xContentType, humanReadable), xContentType);
+    }
+
+    public void testToXContent() throws IOException {
+        List<ProfileResult> profileResults = new ArrayList<>();
+        Map<String, Long> timings = new HashMap<>();
+        timings.put("timing1", 2000L);
+        timings.put("timing2", 4000L);
+        ProfileResult profileResult = new ProfileResult("someType", "someDescription", timings, Collections.emptyList());
+        profileResults.add(profileResult);
+        AggregationProfileShardResult aggProfileResults = new AggregationProfileShardResult(profileResults);
+        BytesReference xContent = toXContent(aggProfileResults, XContentType.JSON, randomBoolean());
+        assertEquals("{\"aggregations\":["
+                        + "{\"type\":\"someType\","
+                            + "\"description\":\"someDescription\","
+                            + "\"time\":\"0.006000000000ms\","
+                            + "\"time_in_nanos\":6000,"
+                            + "\"breakdown\":{\"timing1\":2000,\"timing2\":4000}"
+                        + "}"
+                   + "]}", xContent.utf8ToString());
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/search/profile/aggregation/AggregationProfilerIT.java b/core/src/test/java/org/elasticsearch/search/profile/aggregation/AggregationProfilerIT.java
index 342da16f5087e..9914938854d03 100644
--- a/core/src/test/java/org/elasticsearch/search/profile/aggregation/AggregationProfilerIT.java
+++ b/core/src/test/java/org/elasticsearch/search/profile/aggregation/AggregationProfilerIT.java
@@ -28,8 +28,6 @@
 import org.elasticsearch.search.aggregations.metrics.max.MaxAggregator;
 import org.elasticsearch.search.profile.ProfileResult;
 import org.elasticsearch.search.profile.ProfileShardResult;
-import org.elasticsearch.search.profile.aggregation.AggregationProfileShardResult;
-import org.elasticsearch.search.profile.aggregation.AggregationTimingType;
 import org.elasticsearch.test.ESIntegTestCase;
 import java.util.ArrayList;
 import java.util.List;
@@ -68,7 +66,7 @@ protected void setupSuiteScopeCluster() throws Exception {
 
         String[] randomStrings = new String[randomIntBetween(2, 10)];
         for (int i = 0; i < randomStrings.length; i++) {
-            randomStrings[i] = randomAsciiOfLength(10);
+            randomStrings[i] = randomAlphaOfLength(10);
         }
 
         for (int i = 0; i < 5; i++) {
@@ -156,7 +154,7 @@ public void testMultiLevelProfile() {
 
             ProfileResult termsAggResult = histoAggResult.getProfiledChildren().get(0);
             assertThat(termsAggResult, notNullValue());
-            assertThat(termsAggResult.getQueryName(), equalTo(GlobalOrdinalsStringTermsAggregator.WithHash.class.getName()));
+            assertThat(termsAggResult.getQueryName(), equalTo(GlobalOrdinalsStringTermsAggregator.class.getName()));
             assertThat(termsAggResult.getLuceneDescription(), equalTo("terms"));
             assertThat(termsAggResult.getTime(), greaterThan(0L));
             Map<String, Long> termsBreakdown = termsAggResult.getTimeBreakdown();
@@ -226,7 +224,7 @@ public void testMultiLevelProfileBreadthFirst() {
 
             ProfileResult termsAggResult = histoAggResult.getProfiledChildren().get(0);
             assertThat(termsAggResult, notNullValue());
-            assertThat(termsAggResult.getQueryName(), equalTo(GlobalOrdinalsStringTermsAggregator.WithHash.class.getName()));
+            assertThat(termsAggResult.getQueryName(), equalTo(GlobalOrdinalsStringTermsAggregator.class.getName()));
             assertThat(termsAggResult.getLuceneDescription(), equalTo("terms"));
             assertThat(termsAggResult.getTime(), greaterThan(0L));
             Map<String, Long> termsBreakdown = termsAggResult.getTimeBreakdown();
@@ -357,7 +355,7 @@ public void testComplexProfile() {
 
             ProfileResult tagsAggResult = histoAggResult.getProfiledChildren().get(0);
             assertThat(tagsAggResult, notNullValue());
-            assertThat(tagsAggResult.getQueryName(), equalTo(GlobalOrdinalsStringTermsAggregator.WithHash.class.getName()));
+            assertThat(tagsAggResult.getQueryName(), equalTo(GlobalOrdinalsStringTermsAggregator.class.getName()));
             assertThat(tagsAggResult.getLuceneDescription(), equalTo("tags"));
             assertThat(tagsAggResult.getTime(), greaterThan(0L));
             Map<String, Long> tagsBreakdown = tagsAggResult.getTimeBreakdown();
@@ -408,7 +406,7 @@ public void testComplexProfile() {
 
             ProfileResult stringsAggResult = histoAggResult.getProfiledChildren().get(1);
             assertThat(stringsAggResult, notNullValue());
-            assertThat(stringsAggResult.getQueryName(), equalTo(GlobalOrdinalsStringTermsAggregator.WithHash.class.getName()));
+            assertThat(stringsAggResult.getQueryName(), equalTo(GlobalOrdinalsStringTermsAggregator.class.getName()));
             assertThat(stringsAggResult.getLuceneDescription(), equalTo("strings"));
             assertThat(stringsAggResult.getTime(), greaterThan(0L));
             Map<String, Long> stringsBreakdown = stringsAggResult.getTimeBreakdown();
@@ -459,7 +457,7 @@ public void testComplexProfile() {
 
             tagsAggResult = stringsAggResult.getProfiledChildren().get(2);
             assertThat(tagsAggResult, notNullValue());
-            assertThat(tagsAggResult.getQueryName(), equalTo(GlobalOrdinalsStringTermsAggregator.WithHash.class.getName()));
+            assertThat(tagsAggResult.getQueryName(), equalTo(GlobalOrdinalsStringTermsAggregator.class.getName()));
             assertThat(tagsAggResult.getLuceneDescription(), equalTo("tags"));
             assertThat(tagsAggResult.getTime(), greaterThan(0L));
             tagsBreakdown = tagsAggResult.getTimeBreakdown();
diff --git a/core/src/test/java/org/elasticsearch/search/profile/query/CollectorResultTests.java b/core/src/test/java/org/elasticsearch/search/profile/query/CollectorResultTests.java
new file mode 100644
index 0000000000000..2323c195f2cc5
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/profile/query/CollectorResultTests.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.profile.query;
+
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.elasticsearch.test.XContentTestUtils.insertRandomFields;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
+
+public class CollectorResultTests extends ESTestCase {
+
+    public static CollectorResult createTestItem(int depth) {
+        String name = randomAlphaOfLengthBetween(5, 10);
+        String reason = randomAlphaOfLengthBetween(5, 10);
+        long time = randomNonNegativeLong();
+        if (randomBoolean()) {
+            // also often use relatively "small" values, otherwise we will mostly test huge longs
+            time = time % 100000;
+        }
+        int size = randomIntBetween(0, 5);
+        List<CollectorResult> children = new ArrayList<>(size);
+        if (depth > 0) {
+            for (int i = 0; i < size; i++) {
+                children.add(createTestItem(depth - 1));
+            }
+        }
+        return new CollectorResult(name, reason, time, children);
+    }
+
+    public void testFromXContent() throws IOException {
+        doFromXContentTestWithRandomFields(false);
+    }
+
+    public void testFromXContentWithRandomFields() throws IOException {
+        doFromXContentTestWithRandomFields(true);
+    }
+
+    private void doFromXContentTestWithRandomFields(boolean addRandomFields) throws IOException {
+        CollectorResult collectorResult = createTestItem(1);
+        XContentType xContentType = randomFrom(XContentType.values());
+        boolean humanReadable = randomBoolean();
+        BytesReference originalBytes = toShuffledXContent(collectorResult, xContentType, ToXContent.EMPTY_PARAMS, humanReadable);
+        BytesReference mutated;
+        if (addRandomFields) {
+            mutated = insertRandomFields(xContentType, originalBytes, null, random());
+        } else {
+            mutated = originalBytes;
+        }
+        try (XContentParser parser = createParser(xContentType.xContent(), mutated)) {
+            ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
+            CollectorResult parsed = CollectorResult.fromXContent(parser);
+            assertNull(parser.nextToken());
+            assertToXContentEquivalent(originalBytes, toXContent(parsed, xContentType, humanReadable), xContentType);
+        }
+    }
+
+    public void testToXContent() throws IOException {
+        List<CollectorResult> children = new ArrayList<>();
+        children.add(new CollectorResult("child1", "reason1", 100L, Collections.emptyList()));
+        children.add(new CollectorResult("child2", "reason1", 123356L, Collections.emptyList()));
+        CollectorResult result = new CollectorResult("collectorName", "some reason", 123456L, children);
+        XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint();
+        result.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        assertEquals("{\n" +
+            "  \"name\" : \"collectorName\",\n" +
+            "  \"reason\" : \"some reason\",\n" +
+            "  \"time\" : \"0.1234560000ms\",\n" +
+            "  \"time_in_nanos\" : 123456,\n" +
+            "  \"children\" : [\n" +
+            "    {\n" +
+            "      \"name\" : \"child1\",\n" +
+            "      \"reason\" : \"reason1\",\n" +
+            "      \"time\" : \"0.0001000000000ms\",\n" +
+            "      \"time_in_nanos\" : 100\n" +
+            "    },\n" +
+            "    {\n" +
+            "      \"name\" : \"child2\",\n" +
+            "      \"reason\" : \"reason1\",\n" +
+            "      \"time\" : \"0.1233560000ms\",\n" +
+            "      \"time_in_nanos\" : 123356\n" +
+            "    }\n" +
+            "  ]\n" +
+          "}", builder.string());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/profile/query/QueryProfileShardResultTests.java b/core/src/test/java/org/elasticsearch/search/profile/query/QueryProfileShardResultTests.java
new file mode 100644
index 0000000000000..6ffece354fdf0
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/profile/query/QueryProfileShardResultTests.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.profile.query;
+
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParserUtils;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.search.profile.ProfileResult;
+import org.elasticsearch.search.profile.ProfileResultTests;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
+
+public class QueryProfileShardResultTests extends ESTestCase {
+
+    public static QueryProfileShardResult createTestItem() {
+        int size = randomIntBetween(0, 5);
+        List<ProfileResult> queryProfileResults = new ArrayList<>(size);
+        for (int i = 0; i < size; i++) {
+            queryProfileResults.add(ProfileResultTests.createTestItem(1));
+        }
+        CollectorResult profileCollector = CollectorResultTests.createTestItem(2);
+        long rewriteTime = randomNonNegativeLong();
+        if (randomBoolean()) {
+            rewriteTime = rewriteTime % 1000; // make sure to often test this with small values too
+        }
+        return new QueryProfileShardResult(queryProfileResults, rewriteTime, profileCollector);
+    }
+
+    public void testFromXContent() throws IOException {
+        QueryProfileShardResult profileResult = createTestItem();
+        XContentType xContentType = randomFrom(XContentType.values());
+        boolean humanReadable = randomBoolean();
+        BytesReference originalBytes = toShuffledXContent(profileResult, xContentType, ToXContent.EMPTY_PARAMS, humanReadable);
+
+        QueryProfileShardResult parsed;
+        try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) {
+            XContentParserUtils.ensureExpectedToken(parser.nextToken(), XContentParser.Token.START_OBJECT, parser::getTokenLocation);
+            parsed = QueryProfileShardResult.fromXContent(parser);
+            assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());
+            assertNull(parser.nextToken());
+        }
+        assertToXContentEquivalent(originalBytes, toXContent(parsed, xContentType, humanReadable), xContentType);
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java b/core/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java
index 152709d628d54..6c40a27ae6d70 100644
--- a/core/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java
+++ b/core/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java
@@ -22,16 +22,25 @@
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field.Store;
 import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.search.ConstantScoreWeight;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.Explanation;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.LeafCollector;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.RandomApproximationQuery;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.ScorerSupplier;
 import org.apache.lucene.search.Sort;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TotalHitCountCollector;
+import org.apache.lucene.search.Weight;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.TestUtil;
@@ -45,6 +54,7 @@
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
@@ -56,7 +66,7 @@ public class QueryProfilerTests extends ESTestCase {
     static ContextIndexSearcher searcher;
 
     @BeforeClass
-    public static void before() throws IOException {
+    public static void setup() throws IOException {
         dir = newDirectory();
         RandomIndexWriter w = new RandomIndexWriter(random(), dir);
         final int numDocs = TestUtil.nextInt(random(), 1, 20);
@@ -76,7 +86,7 @@ public static void before() throws IOException {
     }
 
     @AfterClass
-    public static void after() throws IOException {
+    public static void cleanup() throws IOException {
         IOUtils.close(reader, dir);
         dir = null;
         reader = null;
@@ -191,4 +201,76 @@ public void testCollector() throws IOException {
         leafCollector.collect(0);
         assertThat(profileCollector.getTime(), greaterThan(time));
     }
+
+    private static class DummyQuery extends Query {
+
+        @Override
+        public String toString(String field) {
+            return getClass().getSimpleName();
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            return this == obj;
+        }
+
+        @Override
+        public int hashCode() {
+            return 0;
+        }
+
+        @Override
+        public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException {
+            return new ConstantScoreWeight(this) {
+                @Override
+                public void extractTerms(Set<Term> terms) {
+                    throw new UnsupportedOperationException();
+                }
+
+                @Override
+                public Explanation explain(LeafReaderContext context, int doc) throws IOException {
+                    throw new UnsupportedOperationException();
+                }
+
+                @Override
+                public Scorer scorer(LeafReaderContext context) throws IOException {
+                    throw new UnsupportedOperationException();
+                }
+
+                @Override
+                public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
+                    final Weight weight = this;
+                    return new ScorerSupplier() {
+
+                        @Override
+                        public Scorer get(boolean randomAccess) throws IOException {
+                            throw new UnsupportedOperationException();
+                        }
+
+                        @Override
+                        public long cost() {
+                            return 42;
+                        }
+                    };
+                }
+            };
+        }
+    }
+
+    public void testScorerSupplier() throws IOException {
+        Directory dir = newDirectory();
+        IndexWriter w = new IndexWriter(dir, newIndexWriterConfig());
+        w.addDocument(new Document());
+        DirectoryReader reader = DirectoryReader.open(w);
+        w.close();
+        IndexSearcher s = newSearcher(reader);
+        s.setQueryCache(null);
+        Weight weight = s.createNormalizedWeight(new DummyQuery(), randomBoolean());
+        // exception when getting the scorer
+        expectThrows(UnsupportedOperationException.class, () ->  weight.scorer(s.getIndexReader().leaves().get(0)));
+        // no exception, means scorerSupplier is delegated
+        weight.scorerSupplier(s.getIndexReader().leaves().get(0));
+        reader.close();
+        dir.close();
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/profile/query/RandomQueryGenerator.java b/core/src/test/java/org/elasticsearch/search/profile/query/RandomQueryGenerator.java
index d40f85aa04e17..14fe8d58132f9 100644
--- a/core/src/test/java/org/elasticsearch/search/profile/query/RandomQueryGenerator.java
+++ b/core/src/test/java/org/elasticsearch/search/profile/query/RandomQueryGenerator.java
@@ -195,7 +195,6 @@ private static QueryBuilder randomCommonTermsQuery(List<String> fields, int numD
         return q;
     }
 
-    @Deprecated
     private static QueryBuilder randomFuzzyQuery(List<String> fields) {
 
         QueryBuilder q = QueryBuilders.fuzzyQuery(randomField(fields), randomQueryString(1));
diff --git a/core/src/test/java/org/elasticsearch/search/query/MultiMatchQueryIT.java b/core/src/test/java/org/elasticsearch/search/query/MultiMatchQueryIT.java
index cb55f88ff80ec..d0d47a085a15d 100644
--- a/core/src/test/java/org/elasticsearch/search/query/MultiMatchQueryIT.java
+++ b/core/src/test/java/org/elasticsearch/search/query/MultiMatchQueryIT.java
@@ -122,6 +122,13 @@ public void init() throws Exception {
                 "last_name", "",
                 "category", "marvel hero",
                 "skill", 1));
+
+        builders.add(client().prepareIndex("test", "test", "nowHero").setSource(
+                "full_name", "now sort of",
+                "first_name", "now",
+                "last_name", "",
+                "category", "marvel hero",
+                "skill", 1));
         List<String> firstNames = new ArrayList<>();
         fill(firstNames, "Captain", between(15, 25));
         fill(firstNames, "Ultimate", between(5, 10));
@@ -164,6 +171,9 @@ private XContentBuilder createMapping() throws IOException {
                 .field("norms", false)
                 .field("copy_to", "last_name_phrase")
                 .endObject()
+                .startObject("date")
+                .field("type", "date")
+                .endObject()
                 .endObject()
                 .endObject().endObject();
     }
@@ -260,8 +270,8 @@ public void testSingleField() throws NoSuchFieldException, IllegalAccessExceptio
                     .addSort("_uid", SortOrder.ASC)
                     .setQuery(multiMatchQueryBuilder).get();
             MatchQueryBuilder matchQueryBuilder = QueryBuilders.matchQuery(field, builder.toString());
-            if (getType(multiMatchQueryBuilder) != null) {
-                matchQueryBuilder.type(MatchQuery.Type.valueOf(getType(multiMatchQueryBuilder).matchQueryType().toString()));
+            if (multiMatchQueryBuilder.getType() != null) {
+                matchQueryBuilder.type(MatchQuery.Type.valueOf(multiMatchQueryBuilder.getType().matchQueryType().toString()));
             }
             SearchResponse matchResp = client().prepareSearch("test")
                     // _uid tie sort
@@ -569,9 +579,10 @@ public void testCrossFieldMode() throws ExecutionException, InterruptedException
 
         // test if boosts work
         searchResponse = client().prepareSearch("test")
-                .setQuery(randomizeType(multiMatchQuery("the ultimate", "full_name", "first_name", "last_name", "category").field("last_name", 10)
+                .setQuery(randomizeType(multiMatchQuery("the ultimate", "full_name", "first_name", "category").field("last_name", 10)
                         .type(MultiMatchQueryBuilder.Type.CROSS_FIELDS)
                         .operator(Operator.AND))).get();
+        assertHitCount(searchResponse, 2L);
         assertFirstHit(searchResponse, hasId("ultimate1"));   // has ultimate in the last_name and that is boosted
         assertSecondHit(searchResponse, hasId("ultimate2"));
         assertThat(searchResponse.getHits().hits()[0].getScore(), greaterThan(searchResponse.getHits().hits()[1].getScore()));
@@ -582,6 +593,7 @@ public void testCrossFieldMode() throws ExecutionException, InterruptedException
                 .setQuery(randomizeType(multiMatchQuery("the ultimate", "full_name", "first_name", "last_name", "category")
                         .type(MultiMatchQueryBuilder.Type.CROSS_FIELDS)
                         .operator(Operator.AND))).get();
+        assertHitCount(searchResponse, 2L);
         assertFirstHit(searchResponse, hasId("ultimate2"));
         assertSecondHit(searchResponse, hasId("ultimate1"));
         assertThat(searchResponse.getHits().hits()[0].getScore(), greaterThan(searchResponse.getHits().hits()[1].getScore()));
@@ -590,28 +602,33 @@ public void testCrossFieldMode() throws ExecutionException, InterruptedException
         searchResponse = client().prepareSearch("test")
                 .setQuery(randomizeType(multiMatchQuery("15", "skill")
                         .type(MultiMatchQueryBuilder.Type.CROSS_FIELDS))).get();
+        assertHitCount(searchResponse, 1L);
         assertFirstHit(searchResponse, hasId("theone"));
 
         searchResponse = client().prepareSearch("test")
                 .setQuery(randomizeType(multiMatchQuery("15", "skill", "first_name")
                         .type(MultiMatchQueryBuilder.Type.CROSS_FIELDS))).get();
+        assertHitCount(searchResponse, 1L);
         assertFirstHit(searchResponse, hasId("theone"));
 
         // Two numeric fields together caused trouble at one point!
         searchResponse = client().prepareSearch("test")
                 .setQuery(randomizeType(multiMatchQuery("15", "int-field", "skill")
                         .type(MultiMatchQueryBuilder.Type.CROSS_FIELDS))).get();
+        assertHitCount(searchResponse, 1L);
         assertFirstHit(searchResponse, hasId("theone"));
 
         searchResponse = client().prepareSearch("test")
                 .setQuery(randomizeType(multiMatchQuery("15", "int-field", "first_name", "skill")
                         .type(MultiMatchQueryBuilder.Type.CROSS_FIELDS))).get();
+        assertHitCount(searchResponse, 1L);
         assertFirstHit(searchResponse, hasId("theone"));
 
         searchResponse = client().prepareSearch("test")
                 .setQuery(randomizeType(multiMatchQuery("alpha 15", "first_name", "skill")
                         .type(MultiMatchQueryBuilder.Type.CROSS_FIELDS)
                         .lenient(true))).get();
+        assertHitCount(searchResponse, 1L);
         assertFirstHit(searchResponse, hasId("ultimate1"));
         /*
          * Doesn't find theone because "alpha 15" isn't a number and we don't
@@ -624,7 +641,17 @@ public void testCrossFieldMode() throws ExecutionException, InterruptedException
                 .setQuery(randomizeType(multiMatchQuery("alpha 15", "int-field", "first_name", "skill")
                         .type(MultiMatchQueryBuilder.Type.CROSS_FIELDS)
                         .lenient(true))).get();
+        assertHitCount(searchResponse, 1L);
         assertFirstHit(searchResponse, hasId("ultimate1"));
+
+
+        // Check that cross fields works with date fields
+        searchResponse = client().prepareSearch("test")
+                .setQuery(randomizeType(multiMatchQuery("now", "f*", "date")
+                        .type(MultiMatchQueryBuilder.Type.CROSS_FIELDS)).lenient(true))
+                .get();
+        assertHitCount(searchResponse, 1L);
+        assertFirstHit(searchResponse, hasId("nowHero"));
     }
 
     private static void assertEquivalent(String query, SearchResponse left, SearchResponse right) {
@@ -647,21 +674,21 @@ private static void assertEquivalent(String query, SearchResponse left, SearchRe
         }
     }
 
-    public static List<String> fill(List<String> list, String value, int times) {
+    private static List<String> fill(List<String> list, String value, int times) {
         for (int i = 0; i < times; i++) {
             list.add(value);
         }
         return list;
     }
 
-    public List<String> fillRandom(List<String> list, int times) {
+    private static List<String> fillRandom(List<String> list, int times) {
         for (int i = 0; i < times; i++) {
-            list.add(randomAsciiOfLengthBetween(1, 5));
+            list.add(randomAlphaOfLength(5));
         }
         return list;
     }
 
-    public <T> T randomPickExcept(List<T> fromList, T butNot) {
+    private static <T> T randomPickExcept(List<T> fromList, T butNot) {
         while (true) {
             T t = RandomPicks.randomFrom(random(), fromList);
             if (t.equals(butNot)) {
@@ -671,9 +698,9 @@ public <T> T randomPickExcept(List<T> fromList, T butNot) {
         }
     }
 
-    public MultiMatchQueryBuilder randomizeType(MultiMatchQueryBuilder builder) {
+    private static MultiMatchQueryBuilder randomizeType(MultiMatchQueryBuilder builder) {
         try {
-            MultiMatchQueryBuilder.Type type = getType(builder);
+            MultiMatchQueryBuilder.Type type = builder.getType();
             if (type == null && randomBoolean()) {
                 return builder;
             }
@@ -715,8 +742,4 @@ public MultiMatchQueryBuilder randomizeType(MultiMatchQueryBuilder builder) {
             throw new RuntimeException(ex);
         }
     }
-
-    private MultiMatchQueryBuilder.Type getType(MultiMatchQueryBuilder builder) throws NoSuchFieldException, IllegalAccessException {
-        return builder.getType();
-    }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java b/core/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java
index 5bbae82b7d7ae..143d3c21e5fa1 100644
--- a/core/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java
+++ b/core/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java
@@ -40,6 +40,7 @@
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.Weight;
 import org.apache.lucene.store.Directory;
+import org.elasticsearch.action.search.SearchTask;
 import org.elasticsearch.index.query.ParsedQuery;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.TestSearchContext;
@@ -54,6 +55,7 @@ private void countTestCase(Query query, IndexReader reader, boolean shouldCollec
         TestSearchContext context = new TestSearchContext(null);
         context.parsedQuery(new ParsedQuery(query));
         context.setSize(0);
+        context.setTask(new SearchTask(123L, "", "", "", null));
 
         IndexSearcher searcher = new IndexSearcher(reader);
         final AtomicBoolean collected = new AtomicBoolean();
@@ -64,7 +66,7 @@ protected void search(List<LeafReaderContext> leaves, Weight weight, Collector c
             }
         };
 
-        final boolean rescore = QueryPhase.execute(context, contextSearcher);
+        final boolean rescore = QueryPhase.execute(context, contextSearcher, checkCancelled -> {});
         assertFalse(rescore);
         assertEquals(searcher.count(query), context.queryResult().topDocs().totalHits);
         assertEquals(shouldCollect, collected.get());
@@ -123,6 +125,7 @@ public void testPostFilterDisablesCountOptimization() throws Exception {
         TestSearchContext context = new TestSearchContext(null);
         context.parsedQuery(new ParsedQuery(new MatchAllDocsQuery()));
         context.setSize(0);
+        context.setTask(new SearchTask(123L, "", "", "", null));
 
         final AtomicBoolean collected = new AtomicBoolean();
         IndexSearcher contextSearcher = new IndexSearcher(new MultiReader()) {
@@ -132,12 +135,12 @@ protected void search(List<LeafReaderContext> leaves, Weight weight, Collector c
             }
         };
 
-        QueryPhase.execute(context, contextSearcher);
+        QueryPhase.execute(context, contextSearcher, checkCancelled -> {});
         assertEquals(0, context.queryResult().topDocs().totalHits);
         assertFalse(collected.get());
 
         context.parsedPostFilter(new ParsedQuery(new MatchNoDocsQuery()));
-        QueryPhase.execute(context, contextSearcher);
+        QueryPhase.execute(context, contextSearcher, checkCancelled -> {});
         assertEquals(0, context.queryResult().topDocs().totalHits);
         assertTrue(collected.get());
     }
@@ -146,6 +149,7 @@ public void testMinScoreDisablesCountOptimization() throws Exception {
         TestSearchContext context = new TestSearchContext(null);
         context.parsedQuery(new ParsedQuery(new MatchAllDocsQuery()));
         context.setSize(0);
+        context.setTask(new SearchTask(123L, "", "", "", null));
 
         final AtomicBoolean collected = new AtomicBoolean();
         IndexSearcher contextSearcher = new IndexSearcher(new MultiReader()) {
@@ -155,12 +159,12 @@ protected void search(List<LeafReaderContext> leaves, Weight weight, Collector c
             }
         };
 
-        QueryPhase.execute(context, contextSearcher);
+        QueryPhase.execute(context, contextSearcher, checkCancelled -> {});
         assertEquals(0, context.queryResult().topDocs().totalHits);
         assertFalse(collected.get());
 
         context.minimumScore(1);
-        QueryPhase.execute(context, contextSearcher);
+        QueryPhase.execute(context, contextSearcher, checkCancelled -> {});
         assertEquals(0, context.queryResult().topDocs().totalHits);
         assertTrue(collected.get());
     }
diff --git a/core/src/test/java/org/elasticsearch/search/query/QueryStringIT.java b/core/src/test/java/org/elasticsearch/search/query/QueryStringIT.java
new file mode 100644
index 0000000000000..9f651af094a37
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/query/QueryStringIT.java
@@ -0,0 +1,340 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.query;
+
+import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery;
+import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoSearchHits;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+
+import org.apache.lucene.util.LuceneTestCase;
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
+import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.query.Operator;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.query.QueryStringQueryBuilder;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+public class QueryStringIT extends ESIntegTestCase {
+
+    @Before
+    public void setup() throws Exception {
+        String indexBody = copyToStringFromClasspath("/org/elasticsearch/search/query/all-query-index.json");
+        prepareCreate("test").setSource(indexBody, XContentType.JSON).get();
+        ensureGreen("test");
+    }
+
+    private QueryStringQueryBuilder lenientQuery(String queryText) {
+        return queryStringQuery(queryText).lenient(true);
+    }
+
+    public void testBasicAllQuery() throws Exception {
+        List<IndexRequestBuilder> reqs = new ArrayList<>();
+        reqs.add(client().prepareIndex("test", "doc", "1").setSource("f1", "foo bar baz"));
+        reqs.add(client().prepareIndex("test", "doc", "2").setSource("f2", "Bar"));
+        reqs.add(client().prepareIndex("test", "doc", "3").setSource("f3", "foo bar baz"));
+        indexRandom(true, false, reqs);
+
+        SearchResponse resp = client().prepareSearch("test").setQuery(queryStringQuery("foo")).get();
+        assertHitCount(resp, 2L);
+        assertHits(resp.getHits(), "1", "3");
+
+        resp = client().prepareSearch("test").setQuery(queryStringQuery("bar")).get();
+        assertHitCount(resp, 2L);
+        assertHits(resp.getHits(), "1", "3");
+
+        resp = client().prepareSearch("test").setQuery(queryStringQuery("Bar")).get();
+        assertHitCount(resp, 3L);
+        assertHits(resp.getHits(), "1", "2", "3");
+
+        resp = client().prepareSearch("test").setQuery(queryStringQuery("foa")).get();
+        assertHitCount(resp, 1L);
+        assertHits(resp.getHits(), "3");
+    }
+
+    public void testWithDate() throws Exception {
+        List<IndexRequestBuilder> reqs = new ArrayList<>();
+        reqs.add(client().prepareIndex("test", "doc", "1").setSource("f1", "foo", "f_date", "2015/09/02"));
+        reqs.add(client().prepareIndex("test", "doc", "2").setSource("f1", "bar", "f_date", "2015/09/01"));
+        indexRandom(true, false, reqs);
+
+        SearchResponse resp = client().prepareSearch("test").setQuery(queryStringQuery("foo bar")).get();
+        assertHits(resp.getHits(), "1", "2");
+        assertHitCount(resp, 2L);
+
+        resp = client().prepareSearch("test").setQuery(queryStringQuery("\"2015/09/02\"")).get();
+        assertHits(resp.getHits(), "1");
+        assertHitCount(resp, 1L);
+
+        resp = client().prepareSearch("test").setQuery(queryStringQuery("bar \"2015/09/02\"")).get();
+        assertHits(resp.getHits(), "1", "2");
+        assertHitCount(resp, 2L);
+
+        resp = client().prepareSearch("test").setQuery(queryStringQuery("\"2015/09/02\" \"2015/09/01\"")).get();
+        assertHits(resp.getHits(), "1", "2");
+        assertHitCount(resp, 2L);
+    }
+
+    public void testWithLotsOfTypes() throws Exception {
+        List<IndexRequestBuilder> reqs = new ArrayList<>();
+        reqs.add(client().prepareIndex("test", "doc", "1").setSource("f1", "foo",
+                        "f_date", "2015/09/02",
+                        "f_float", "1.7",
+                        "f_ip", "127.0.0.1"));
+        reqs.add(client().prepareIndex("test", "doc", "2").setSource("f1", "bar",
+                        "f_date", "2015/09/01",
+                        "f_float", "1.8",
+                        "f_ip", "127.0.0.2"));
+        indexRandom(true, false, reqs);
+
+        SearchResponse resp = client().prepareSearch("test").setQuery(queryStringQuery("foo bar")).get();
+        assertHits(resp.getHits(), "1", "2");
+        assertHitCount(resp, 2L);
+
+        resp = client().prepareSearch("test").setQuery(queryStringQuery("\"2015/09/02\"")).get();
+        assertHits(resp.getHits(), "1");
+        assertHitCount(resp, 1L);
+
+        resp = client().prepareSearch("test").setQuery(queryStringQuery("127.0.0.2 \"2015/09/02\"")).get();
+        assertHits(resp.getHits(), "1", "2");
+        assertHitCount(resp, 2L);
+
+        resp = client().prepareSearch("test").setQuery(queryStringQuery("127.0.0.1 1.8")).get();
+        assertHits(resp.getHits(), "1", "2");
+        assertHitCount(resp, 2L);
+    }
+
+    public void testDocWithAllTypes() throws Exception {
+        List<IndexRequestBuilder> reqs = new ArrayList<>();
+        String docBody = copyToStringFromClasspath("/org/elasticsearch/search/query/all-example-document.json");
+        reqs.add(client().prepareIndex("test", "doc", "1").setSource(docBody, XContentType.JSON));
+        indexRandom(true, false, reqs);
+
+        SearchResponse resp = client().prepareSearch("test").setQuery(queryStringQuery("foo")).get();
+        assertHits(resp.getHits(), "1");
+        resp = client().prepareSearch("test").setQuery(queryStringQuery("Bar")).get();
+        assertHits(resp.getHits(), "1");
+        resp = client().prepareSearch("test").setQuery(queryStringQuery("Baz")).get();
+        assertHits(resp.getHits(), "1");
+        resp = client().prepareSearch("test").setQuery(queryStringQuery("sbaz")).get();
+        assertHits(resp.getHits(), "1");
+        resp = client().prepareSearch("test").setQuery(queryStringQuery("19")).get();
+        assertHits(resp.getHits(), "1");
+        // nested doesn't match because it's hidden
+        resp = client().prepareSearch("test").setQuery(queryStringQuery("1476383971")).get();
+        assertHits(resp.getHits(), "1");
+        // bool doesn't match
+        resp = client().prepareSearch("test").setQuery(queryStringQuery("7")).get();
+        assertHits(resp.getHits(), "1");
+        resp = client().prepareSearch("test").setQuery(queryStringQuery("23")).get();
+        assertHits(resp.getHits(), "1");
+        resp = client().prepareSearch("test").setQuery(queryStringQuery("1293")).get();
+        assertHits(resp.getHits(), "1");
+        resp = client().prepareSearch("test").setQuery(queryStringQuery("42")).get();
+        assertHits(resp.getHits(), "1");
+        resp = client().prepareSearch("test").setQuery(queryStringQuery("1.7")).get();
+        assertHits(resp.getHits(), "1");
+        resp = client().prepareSearch("test").setQuery(queryStringQuery("1.5")).get();
+        assertHits(resp.getHits(), "1");
+        resp = client().prepareSearch("test").setQuery(queryStringQuery("12.23")).get();
+        assertHits(resp.getHits(), "1");
+        resp = client().prepareSearch("test").setQuery(queryStringQuery("127.0.0.1")).get();
+        assertHits(resp.getHits(), "1");
+        // binary doesn't match
+        // suggest doesn't match
+        // geo_point doesn't match
+        // geo_shape doesn't match
+    }
+
+    public void testKeywordWithWhitespace() throws Exception {
+        List<IndexRequestBuilder> reqs = new ArrayList<>();
+        reqs.add(client().prepareIndex("test", "doc", "1").setSource("f2", "Foo Bar"));
+        reqs.add(client().prepareIndex("test", "doc", "2").setSource("f1", "bar"));
+        reqs.add(client().prepareIndex("test", "doc", "3").setSource("f1", "foo bar"));
+        indexRandom(true, false, reqs);
+
+        SearchResponse resp = client().prepareSearch("test").setQuery(queryStringQuery("foo")).get();
+        assertHits(resp.getHits(), "3");
+        assertHitCount(resp, 1L);
+
+        resp = client().prepareSearch("test").setQuery(queryStringQuery("bar")).get();
+        assertHits(resp.getHits(), "2", "3");
+        assertHitCount(resp, 2L);
+
+        resp = client().prepareSearch("test")
+                .setQuery(queryStringQuery("Foo Bar").splitOnWhitespace(false))
+                .get();
+        assertHits(resp.getHits(), "1", "2", "3");
+        assertHitCount(resp, 3L);
+    }
+
+    public void testExplicitAllFieldsRequested() throws Exception {
+        String indexBody = copyToStringFromClasspath("/org/elasticsearch/search/query/all-query-index-with-all.json");
+        prepareCreate("test2").setSource(indexBody, XContentType.JSON).get();
+        ensureGreen("test2");
+
+        List<IndexRequestBuilder> reqs = new ArrayList<>();
+        reqs.add(client().prepareIndex("test2", "doc", "1").setSource("f1", "foo", "f2", "eggplant"));
+        indexRandom(true, false, reqs);
+
+        SearchResponse resp = client().prepareSearch("test2").setQuery(
+                queryStringQuery("foo eggplent").defaultOperator(Operator.AND)).get();
+        assertHitCount(resp, 0L);
+
+        resp = client().prepareSearch("test2").setQuery(
+                queryStringQuery("foo eggplent").defaultOperator(Operator.AND).useAllFields(true)).get();
+        assertHits(resp.getHits(), "1");
+        assertHitCount(resp, 1L);
+
+        Exception e = expectThrows(Exception.class, () ->
+                client().prepareSearch("test2").setQuery(
+                        queryStringQuery("blah").field("f1").useAllFields(true)).get());
+        assertThat(ExceptionsHelper.detailedMessage(e),
+                containsString("cannot use [all_fields] parameter in conjunction with [default_field] or [fields]"));
+    }
+
+    @LuceneTestCase.AwaitsFix(bugUrl="currently can't perform phrase queries on fields that don't support positions")
+    public void testPhraseQueryOnFieldWithNoPositions() throws Exception {
+        List<IndexRequestBuilder> reqs = new ArrayList<>();
+        reqs.add(client().prepareIndex("test", "doc", "1").setSource("f1", "foo bar", "f4", "eggplant parmesan"));
+        reqs.add(client().prepareIndex("test", "doc", "2").setSource("f1", "foo bar", "f4", "chicken parmesan"));
+        indexRandom(true, false, reqs);
+
+        SearchResponse resp = client().prepareSearch("test").setQuery(queryStringQuery("\"eggplant parmesan\"")).get();
+        assertHits(resp.getHits(), "1");
+        assertHitCount(resp, 1L);
+    }
+
+    private void setupIndexWithGraph(String index) throws Exception {
+        CreateIndexRequestBuilder builder = prepareCreate(index).setSettings(
+            Settings.builder()
+                .put(indexSettings())
+                .put("index.analysis.filter.graphsyns.type", "synonym_graph")
+                .putArray("index.analysis.filter.graphsyns.synonyms", "wtf, what the fudge", "foo, bar baz")
+                .put("index.analysis.analyzer.lower_graphsyns.type", "custom")
+                .put("index.analysis.analyzer.lower_graphsyns.tokenizer", "standard")
+                .putArray("index.analysis.analyzer.lower_graphsyns.filter", "lowercase", "graphsyns")
+        );
+
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject(index).startObject("properties")
+            .startObject("field").field("type", "text").endObject().endObject().endObject().endObject();
+
+        assertAcked(builder.addMapping(index, mapping));
+        ensureGreen();
+
+        List<IndexRequestBuilder> builders = new ArrayList<>();
+        builders.add(client().prepareIndex(index, index, "1").setSource("field", "say wtf happened foo"));
+        builders.add(client().prepareIndex(index, index, "2").setSource("field", "bar baz what the fudge man"));
+        builders.add(client().prepareIndex(index, index, "3").setSource("field", "wtf"));
+        builders.add(client().prepareIndex(index, index, "4").setSource("field", "what is the name for fudge"));
+        builders.add(client().prepareIndex(index, index, "5").setSource("field", "bar two three"));
+        builders.add(client().prepareIndex(index, index, "6").setSource("field", "bar baz two three"));
+
+        indexRandom(true, false, builders);
+    }
+
+    public void testGraphQueries() throws Exception {
+        String index = "graph_test_index";
+        setupIndexWithGraph(index);
+
+        // phrase
+        SearchResponse searchResponse = client().prepareSearch(index).setQuery(
+            QueryBuilders.queryStringQuery("\"foo two three\"")
+                .defaultField("field")
+                .analyzer("lower_graphsyns")).get();
+
+        assertHitCount(searchResponse, 1L);
+        assertSearchHits(searchResponse, "6");
+
+        // and
+        searchResponse = client().prepareSearch(index).setQuery(
+            QueryBuilders.queryStringQuery("say what the fudge")
+                .defaultField("field")
+                .splitOnWhitespace(false)
+                .defaultOperator(Operator.AND)
+                .analyzer("lower_graphsyns")).get();
+
+        assertHitCount(searchResponse, 1L);
+        assertSearchHits(searchResponse, "1");
+
+        // and, split on whitespace means we should not recognize the multi-word synonym
+        searchResponse = client().prepareSearch(index).setQuery(
+            QueryBuilders.queryStringQuery("say what the fudge")
+                .defaultField("field")
+                .splitOnWhitespace(true)
+                .defaultOperator(Operator.AND)
+                .analyzer("lower_graphsyns")).get();
+
+        assertNoSearchHits(searchResponse);
+
+        // or
+        searchResponse = client().prepareSearch(index).setQuery(
+            QueryBuilders.queryStringQuery("three what the fudge foo")
+                .defaultField("field")
+                .splitOnWhitespace(false)
+                .defaultOperator(Operator.OR)
+                .analyzer("lower_graphsyns")).get();
+
+        assertHitCount(searchResponse, 6L);
+        assertSearchHits(searchResponse, "1", "2", "3", "4", "5", "6");
+
+        // min should match
+        searchResponse = client().prepareSearch(index).setQuery(
+            QueryBuilders.queryStringQuery("three what the fudge foo")
+                .defaultField("field")
+                .splitOnWhitespace(false)
+                .defaultOperator(Operator.OR)
+                .analyzer("lower_graphsyns")
+                .minimumShouldMatch("80%")).get();
+
+        assertHitCount(searchResponse, 3L);
+        assertSearchHits(searchResponse, "1", "2", "6");
+    }
+
+    private void assertHits(SearchHits hits, String... ids) {
+        assertThat(hits.totalHits(), equalTo((long) ids.length));
+        Set<String> hitIds = new HashSet<>();
+        for (SearchHit hit : hits.getHits()) {
+            hitIds.add(hit.id());
+        }
+        assertThat(hitIds, containsInAnyOrder(ids));
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java b/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
index 1d54b412d6c74..5ca65c205aba7 100644
--- a/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
+++ b/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
@@ -31,11 +31,13 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.MatchQueryBuilder;
 import org.elasticsearch.index.query.MultiMatchQueryBuilder;
 import org.elasticsearch.index.query.Operator;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.query.RangeQueryBuilder;
 import org.elasticsearch.index.query.TermQueryBuilder;
 import org.elasticsearch.index.query.WrapperQueryBuilder;
 import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders;
@@ -64,7 +66,6 @@
 import static org.elasticsearch.index.query.QueryBuilders.existsQuery;
 import static org.elasticsearch.index.query.QueryBuilders.functionScoreQuery;
 import static org.elasticsearch.index.query.QueryBuilders.fuzzyQuery;
-import static org.elasticsearch.index.query.QueryBuilders.hasChildQuery;
 import static org.elasticsearch.index.query.QueryBuilders.idsQuery;
 import static org.elasticsearch.index.query.QueryBuilders.indicesQuery;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
@@ -480,19 +481,19 @@ public void testQueryStringAnalyzedWildcard() throws Exception {
         client().prepareIndex("test", "type1", "1").setSource("field1", "value_1", "field2", "value_2").get();
         refresh();
 
-        SearchResponse searchResponse = client().prepareSearch().setQuery(queryStringQuery("value*").analyzeWildcard(true)).get();
+        SearchResponse searchResponse = client().prepareSearch().setQuery(queryStringQuery("value*")).get();
         assertHitCount(searchResponse, 1L);
 
-        searchResponse = client().prepareSearch().setQuery(queryStringQuery("*ue*").analyzeWildcard(true)).get();
+        searchResponse = client().prepareSearch().setQuery(queryStringQuery("*ue*")).get();
         assertHitCount(searchResponse, 1L);
 
-        searchResponse = client().prepareSearch().setQuery(queryStringQuery("*ue_1").analyzeWildcard(true)).get();
+        searchResponse = client().prepareSearch().setQuery(queryStringQuery("*ue_1")).get();
         assertHitCount(searchResponse, 1L);
 
-        searchResponse = client().prepareSearch().setQuery(queryStringQuery("val*e_1").analyzeWildcard(true)).get();
+        searchResponse = client().prepareSearch().setQuery(queryStringQuery("val*e_1")).get();
         assertHitCount(searchResponse, 1L);
 
-        searchResponse = client().prepareSearch().setQuery(queryStringQuery("v?l*e?1").analyzeWildcard(true)).get();
+        searchResponse = client().prepareSearch().setQuery(queryStringQuery("v?l*e?1")).get();
         assertHitCount(searchResponse, 1L);
     }
 
@@ -502,18 +503,14 @@ public void testLowercaseExpandedTerms() {
         client().prepareIndex("test", "type1", "1").setSource("field1", "value_1", "field2", "value_2").get();
         refresh();
 
-        SearchResponse searchResponse = client().prepareSearch().setQuery(queryStringQuery("VALUE_3~1").lowercaseExpandedTerms(true)).get();
+        SearchResponse searchResponse = client().prepareSearch().setQuery(queryStringQuery("VALUE_3~1")).get();
         assertHitCount(searchResponse, 1L);
-        searchResponse = client().prepareSearch().setQuery(queryStringQuery("VALUE_3~1").lowercaseExpandedTerms(false)).get();
-        assertHitCount(searchResponse, 0L);
-        searchResponse = client().prepareSearch().setQuery(queryStringQuery("ValUE_*").lowercaseExpandedTerms(true)).get();
+        searchResponse = client().prepareSearch().setQuery(queryStringQuery("ValUE_*")).get();
         assertHitCount(searchResponse, 1L);
         searchResponse = client().prepareSearch().setQuery(queryStringQuery("vAl*E_1")).get();
         assertHitCount(searchResponse, 1L);
         searchResponse = client().prepareSearch().setQuery(queryStringQuery("[VALUE_1 TO VALUE_3]")).get();
         assertHitCount(searchResponse, 1L);
-        searchResponse = client().prepareSearch().setQuery(queryStringQuery("[VALUE_1 TO VALUE_3]").lowercaseExpandedTerms(false)).get();
-        assertHitCount(searchResponse, 0L);
     }
 
     // Issue #3540
@@ -532,16 +529,13 @@ public void testDateRangeInQueryString() {
         SearchResponse searchResponse = client().prepareSearch().setQuery(queryStringQuery("past:[now-2M/d TO now/d]")).get();
         assertHitCount(searchResponse, 1L);
 
-        searchResponse = client().prepareSearch().setQuery(queryStringQuery("future:[now/d TO now+2M/d]").lowercaseExpandedTerms(false)).get();
+        searchResponse = client().prepareSearch().setQuery(queryStringQuery("future:[now/d TO now+2M/d]")).get();
         assertHitCount(searchResponse, 1L);
 
-        try {
-            client().prepareSearch().setQuery(queryStringQuery("future:[now/D TO now+2M/d]").lowercaseExpandedTerms(false)).get();
-            fail("expected SearchPhaseExecutionException (total failure)");
-        } catch (SearchPhaseExecutionException e) {
-            assertThat(e.status(), equalTo(RestStatus.BAD_REQUEST));
-            assertThat(e.toString(), containsString("unit [D] not supported for date math"));
-        }
+        SearchPhaseExecutionException e = expectThrows(SearchPhaseExecutionException.class, () -> client().prepareSearch()
+                .setQuery(queryStringQuery("future:[now/D TO now+2M/d]")).get());
+        assertThat(e.status(), equalTo(RestStatus.BAD_REQUEST));
+        assertThat(e.toString(), containsString("unit [D] not supported for date math"));
     }
 
     // Issue #7880
@@ -776,12 +770,7 @@ public void testMatchQueryNumeric() throws Exception {
         searchResponse = client().prepareSearch().setQuery(matchQuery("double", "2")).get();
         assertHitCount(searchResponse, 1L);
         assertFirstHit(searchResponse, hasId("2"));
-        try {
-            client().prepareSearch().setQuery(matchQuery("double", "2 3 4")).get();
-            fail("SearchPhaseExecutionException should have been thrown");
-        } catch (SearchPhaseExecutionException ex) {
-            // number format exception
-        }
+        expectThrows(SearchPhaseExecutionException.class, () -> client().prepareSearch().setQuery(matchQuery("double", "2 3 4")).get());
     }
 
     public void testMultiMatchQuery() throws Exception {
@@ -957,7 +946,7 @@ public void testBoolQueryMinShouldMatchBiggerThanNumberOfShouldClauses() throws
             .should(boolQuery()
                 .should(termQuery("field1", "value1"))
                 .should(termQuery("field1", "value2"))
-                .minimumNumberShouldMatch(3));
+                .minimumShouldMatch(3));
         SearchResponse searchResponse = client().prepareSearch().setQuery(boolQuery).get();
         assertHitCount(searchResponse, 1L);
         assertFirstHit(searchResponse, hasId("1"));
@@ -967,9 +956,9 @@ public void testBoolQueryMinShouldMatchBiggerThanNumberOfShouldClauses() throws
             .should(boolQuery()
                 .should(termQuery("field1", "value1"))
                 .should(termQuery("field1", "value2"))
-                .minimumNumberShouldMatch(1))
+                .minimumShouldMatch(1))
             // Only one should clause is defined, returns no docs.
-            .minimumNumberShouldMatch(2);
+            .minimumShouldMatch(2);
         searchResponse = client().prepareSearch().setQuery(boolQuery).get();
         assertHitCount(searchResponse, 0L);
 
@@ -978,8 +967,8 @@ public void testBoolQueryMinShouldMatchBiggerThanNumberOfShouldClauses() throws
             .should(boolQuery()
                 .should(termQuery("field1", "value1"))
                 .should(termQuery("field1", "value2"))
-                .minimumNumberShouldMatch(3))
-            .minimumNumberShouldMatch(1);
+                .minimumShouldMatch(3))
+            .minimumShouldMatch(1);
         searchResponse = client().prepareSearch().setQuery(boolQuery).get();
         assertHitCount(searchResponse, 1L);
         assertFirstHit(searchResponse, hasId("1"));
@@ -989,7 +978,7 @@ public void testBoolQueryMinShouldMatchBiggerThanNumberOfShouldClauses() throws
             .must(boolQuery()
                 .should(termQuery("field1", "value1"))
                 .should(termQuery("field1", "value2"))
-                .minimumNumberShouldMatch(3));
+                .minimumShouldMatch(3));
         searchResponse = client().prepareSearch().setQuery(boolQuery).get();
         assertHitCount(searchResponse, 0L);
     }
@@ -1764,38 +1753,6 @@ public void testIndicesQuery() throws Exception {
         assertFirstHit(searchResponse, hasId("1"));
     }
 
-    // See #2416
-    public void testIndicesQuerySkipParsing() throws Exception {
-        createIndex("simple");
-        assertAcked(prepareCreate("related")
-                .addMapping("child", jsonBuilder().startObject().startObject("child").startObject("_parent").field("type", "parent")
-                        .endObject().endObject().endObject()));
-
-        client().prepareIndex("simple", "lone").setId("1").setSource("text", "value1").get();
-        client().prepareIndex("related", "parent").setId("2").setSource("text", "parent").get();
-        client().prepareIndex("related", "child").setId("3").setParent("2").setSource("text", "value2").get();
-        refresh();
-
-        //has_child fails if executed on "simple" index
-        try {
-            client().prepareSearch("simple")
-                    .setQuery(hasChildQuery("child", matchQuery("text", "value"), ScoreMode.None)).get();
-            fail("Should have failed as has_child query can only be executed against parent-child types");
-        } catch (SearchPhaseExecutionException e) {
-            assertThat(e.shardFailures().length, greaterThan(0));
-            for (ShardSearchFailure shardSearchFailure : e.shardFailures()) {
-                assertThat(shardSearchFailure.reason(), containsString("no mapping found for type [child]"));
-            }
-        }
-
-        //has_child doesn't get parsed for "simple" index
-        SearchResponse searchResponse = client().prepareSearch("related", "simple")
-                .setQuery(indicesQuery(hasChildQuery("child", matchQuery("text", "value2"), ScoreMode.None), "related")
-                        .noMatchQuery(matchQuery("text", "value1"))).get();
-        assertHitCount(searchResponse, 2L);
-        assertSearchHits(searchResponse, "1", "2");
-    }
-
     public void testIndicesQueryMissingIndices() throws IOException, ExecutionException, InterruptedException {
         createIndex("index1");
         createIndex("index2");
@@ -1920,7 +1877,6 @@ public void testDateProvidedAsNumber() throws ExecutionException, InterruptedExc
         assertHitCount(client().prepareSearch("test").setSize(0).setQuery(rangeQuery("field").lte(-999999999999L)).get(), 3);
     }
 
-    @AwaitsFix(bugUrl = "NOCOMMIT")
     public void testRangeQueryWithTimeZone() throws Exception {
         assertAcked(prepareCreate("test")
                 .addMapping("type1", "date", "type=date", "num", "type=integer"));
@@ -1983,14 +1939,10 @@ public void testRangeQueryWithTimeZone() throws Exception {
         assertThat(searchResponse.getHits().getAt(0).getId(), is("3"));
 
         // When we use long values, it means we have ms since epoch UTC based so we don't apply any transformation
-        try {
+        Exception e = expectThrows(SearchPhaseExecutionException.class, () ->
             client().prepareSearch("test")
                     .setQuery(QueryBuilders.rangeQuery("date").from(1388534400000L).to(1388537940999L).timeZone("+01:00"))
-                    .get();
-            fail("A Range Filter using ms since epoch with a TimeZone should raise a ParsingException");
-        } catch (SearchPhaseExecutionException e) {
-            // We expect it
-        }
+                    .get());
 
         searchResponse = client().prepareSearch("test")
                 .setQuery(QueryBuilders.rangeQuery("date").from("2014-01-01").to("2014-01-01T00:59:00").timeZone("-01:00"))
@@ -2005,19 +1957,15 @@ public void testRangeQueryWithTimeZone() throws Exception {
         assertThat(searchResponse.getHits().getAt(0).getId(), is("4"));
 
         // A Range Filter on a numeric field with a TimeZone should raise an exception
-        try {
+        e = expectThrows(SearchPhaseExecutionException.class, () ->
             client().prepareSearch("test")
                     .setQuery(QueryBuilders.rangeQuery("num").from("0").to("4").timeZone("-01:00"))
-                    .get();
-            fail("A Range Filter on a numeric field with a TimeZone should raise a ParsingException");
-        } catch (SearchPhaseExecutionException e) {
-            // We expect it
-        }
+                    .get());
     }
 
     public void testSearchEmptyDoc() {
-        assertAcked(prepareCreate("test").setSettings("{\"index.analysis.analyzer.default.type\":\"keyword\"}"));
-        client().prepareIndex("test", "type1", "1").setSource("{}").get();
+        assertAcked(prepareCreate("test").setSettings("{\"index.analysis.analyzer.default.type\":\"keyword\"}", XContentType.JSON));
+        client().prepareIndex("test", "type1", "1").setSource("{}", XContentType.JSON).get();
 
         refresh();
         assertHitCount(client().prepareSearch().setQuery(matchAllQuery()).get(), 1L);
@@ -2114,4 +2062,20 @@ public void testQueryStringParserCache() throws Exception {
             assertThat(i + " expected: " + first + " actual: " + actual, Float.compare(first, actual), equalTo(0));
         }
     }
+
+    public void testRangeQueryRangeFields_24744() throws Exception {
+        assertAcked(prepareCreate("test")
+            .addMapping("type1", "int_range", "type=integer_range"));
+
+        client().prepareIndex("test", "type1", "1")
+            .setSource(jsonBuilder()
+                .startObject()
+                .startObject("int_range").field("gte", 10).field("lte", 20).endObject()
+                .endObject()).get();
+        refresh();
+
+        RangeQueryBuilder range = new RangeQueryBuilder("int_range").relation("intersects").from(Integer.MIN_VALUE).to(Integer.MAX_VALUE);
+        SearchResponse searchResponse = client().prepareSearch("test").setQuery(range).get();
+        assertHitCount(searchResponse, 1);
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java b/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java
index 9502a8183152e..7f19ad908b072 100644
--- a/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java
+++ b/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java
@@ -19,25 +19,34 @@
 
 package org.elasticsearch.search.query;
 
+import org.apache.lucene.util.LuceneTestCase;
+import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
+import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.Operator;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.SimpleQueryStringFlag;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.test.ESIntegTestCase;
 
 import java.io.IOException;
-import java.util.Locale;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
 import java.util.concurrent.ExecutionException;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
-import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery;
 import static org.elasticsearch.index.query.QueryBuilders.simpleQueryStringQuery;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
+import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFailures;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFirstHit;
@@ -45,6 +54,8 @@
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasId;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 
 /**
@@ -158,49 +169,6 @@ public void testSimpleQueryStringMinimumShouldMatch() throws Exception {
         assertSearchHits(searchResponse, "6", "7", "8");
     }
 
-    public void testSimpleQueryStringLowercasing() {
-        createIndex("test");
-        client().prepareIndex("test", "type1", "1").setSource("body", "Professional").get();
-        refresh();
-
-        SearchResponse searchResponse = client().prepareSearch().setQuery(simpleQueryStringQuery("Professio*")).get();
-        assertHitCount(searchResponse, 1L);
-        assertSearchHits(searchResponse, "1");
-
-        searchResponse = client().prepareSearch().setQuery(
-                simpleQueryStringQuery("Professio*").lowercaseExpandedTerms(false)).get();
-        assertHitCount(searchResponse, 0L);
-
-        searchResponse = client().prepareSearch().setQuery(
-                simpleQueryStringQuery("Professionan~1")).get();
-        assertHitCount(searchResponse, 1L);
-        assertSearchHits(searchResponse, "1");
-
-        searchResponse = client().prepareSearch().setQuery(
-                simpleQueryStringQuery("Professionan~1").lowercaseExpandedTerms(false)).get();
-        assertHitCount(searchResponse, 0L);
-    }
-
-    public void testQueryStringLocale() {
-        createIndex("test");
-        client().prepareIndex("test", "type1", "1").setSource("body", "bılly").get();
-        refresh();
-
-        SearchResponse searchResponse = client().prepareSearch().setQuery(simpleQueryStringQuery("BILL*")).get();
-        assertHitCount(searchResponse, 0L);
-        searchResponse = client().prepareSearch().setQuery(queryStringQuery("body:BILL*")).get();
-        assertHitCount(searchResponse, 0L);
-
-        searchResponse = client().prepareSearch().setQuery(
-                simpleQueryStringQuery("BILL*").locale(new Locale("tr", "TR"))).get();
-        assertHitCount(searchResponse, 1L);
-        assertSearchHits(searchResponse, "1");
-        searchResponse = client().prepareSearch().setQuery(
-                queryStringQuery("body:BILL*").locale(new Locale("tr", "TR"))).get();
-        assertHitCount(searchResponse, 1L);
-        assertSearchHits(searchResponse, "1");
-    }
-
     public void testNestedFieldSimpleQueryString() throws IOException {
         assertAcked(prepareCreate("test")
                 .addMapping("type1", jsonBuilder()
@@ -342,7 +310,7 @@ public void testSimpleQueryStringAnalyzeWildcard() throws ExecutionException, In
         refresh();
 
         SearchResponse searchResponse = client().prepareSearch()
-                .setQuery(simpleQueryStringQuery("Köln*").analyzeWildcard(true).field("location")).get();
+                .setQuery(simpleQueryStringQuery("Köln*").field("location")).get();
         assertNoFailures(searchResponse);
         assertHitCount(searchResponse, 1L);
         assertSearchHits(searchResponse, "1");
@@ -393,8 +361,216 @@ public void testEmptySimpleQueryStringWithAnalysis() throws Exception {
         refresh();
 
         SearchResponse searchResponse = client().prepareSearch()
-                .setQuery(simpleQueryStringQuery("the*").analyzeWildcard(true).field("body")).get();
+                .setQuery(simpleQueryStringQuery("the*").field("body")).get();
         assertNoFailures(searchResponse);
         assertHitCount(searchResponse, 0L);
     }
+
+    public void testBasicAllQuery() throws Exception {
+        String indexBody = copyToStringFromClasspath("/org/elasticsearch/search/query/all-query-index.json");
+        prepareCreate("test").setSource(indexBody).get();
+        ensureGreen("test");
+
+        List<IndexRequestBuilder> reqs = new ArrayList<>();
+        reqs.add(client().prepareIndex("test", "doc", "1").setSource("f1", "foo bar baz"));
+        reqs.add(client().prepareIndex("test", "doc", "2").setSource("f2", "Bar"));
+        reqs.add(client().prepareIndex("test", "doc", "3").setSource("f3", "foo bar baz"));
+        indexRandom(true, false, reqs);
+
+        SearchResponse resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("foo")).get();
+        assertHitCount(resp, 2L);
+        assertHits(resp.getHits(), "1", "3");
+
+        resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("bar")).get();
+        assertHitCount(resp, 2L);
+        assertHits(resp.getHits(), "1", "3");
+
+        resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("Bar")).get();
+        assertHitCount(resp, 3L);
+        assertHits(resp.getHits(), "1", "2", "3");
+
+        resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("foa")).get();
+        assertHitCount(resp, 1L);
+        assertHits(resp.getHits(), "3");
+    }
+
+    public void testWithDate() throws Exception {
+        String indexBody = copyToStringFromClasspath("/org/elasticsearch/search/query/all-query-index.json");
+        prepareCreate("test").setSource(indexBody).get();
+        ensureGreen("test");
+
+        List<IndexRequestBuilder> reqs = new ArrayList<>();
+        reqs.add(client().prepareIndex("test", "doc", "1").setSource("f1", "foo", "f_date", "2015/09/02"));
+        reqs.add(client().prepareIndex("test", "doc", "2").setSource("f1", "bar", "f_date", "2015/09/01"));
+        indexRandom(true, false, reqs);
+
+        SearchResponse resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("foo bar")).get();
+        assertHits(resp.getHits(), "1", "2");
+        assertHitCount(resp, 2L);
+
+        resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("\"2015/09/02\"")).get();
+        assertHits(resp.getHits(), "1");
+        assertHitCount(resp, 1L);
+
+        resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("bar \"2015/09/02\"")).get();
+        assertHits(resp.getHits(), "1", "2");
+        assertHitCount(resp, 2L);
+
+        resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("\"2015/09/02\" \"2015/09/01\"")).get();
+        assertHits(resp.getHits(), "1", "2");
+        assertHitCount(resp, 2L);
+    }
+
+    public void testWithLotsOfTypes() throws Exception {
+        String indexBody = copyToStringFromClasspath("/org/elasticsearch/search/query/all-query-index.json");
+        prepareCreate("test").setSource(indexBody).get();
+        ensureGreen("test");
+
+        List<IndexRequestBuilder> reqs = new ArrayList<>();
+        reqs.add(client().prepareIndex("test", "doc", "1").setSource("f1", "foo",
+                        "f_date", "2015/09/02",
+                        "f_float", "1.7",
+                        "f_ip", "127.0.0.1"));
+        reqs.add(client().prepareIndex("test", "doc", "2").setSource("f1", "bar",
+                        "f_date", "2015/09/01",
+                        "f_float", "1.8",
+                        "f_ip", "127.0.0.2"));
+        indexRandom(true, false, reqs);
+
+        SearchResponse resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("foo bar")).get();
+        assertHits(resp.getHits(), "1", "2");
+        assertHitCount(resp, 2L);
+
+        resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("\"2015/09/02\"")).get();
+        assertHits(resp.getHits(), "1");
+        assertHitCount(resp, 1L);
+
+        resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("127.0.0.2 \"2015/09/02\"")).get();
+        assertHits(resp.getHits(), "1", "2");
+        assertHitCount(resp, 2L);
+
+        resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("127.0.0.1 1.8")).get();
+        assertHits(resp.getHits(), "1", "2");
+        assertHitCount(resp, 2L);
+    }
+
+    public void testDocWithAllTypes() throws Exception {
+        String indexBody = copyToStringFromClasspath("/org/elasticsearch/search/query/all-query-index.json");
+        prepareCreate("test").setSource(indexBody).get();
+        ensureGreen("test");
+
+        List<IndexRequestBuilder> reqs = new ArrayList<>();
+        String docBody = copyToStringFromClasspath("/org/elasticsearch/search/query/all-example-document.json");
+        reqs.add(client().prepareIndex("test", "doc", "1").setSource(docBody, XContentType.JSON));
+        indexRandom(true, false, reqs);
+
+        SearchResponse resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("foo")).get();
+        assertHits(resp.getHits(), "1");
+        resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("Bar")).get();
+        assertHits(resp.getHits(), "1");
+        resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("Baz")).get();
+        assertHits(resp.getHits(), "1");
+        resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("sbaz")).get();
+        assertHits(resp.getHits(), "1");
+        resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("19")).get();
+        assertHits(resp.getHits(), "1");
+        // nested doesn't match because it's hidden
+        resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("1476383971")).get();
+        assertHits(resp.getHits(), "1");
+        // bool doesn't match
+        resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("7")).get();
+        assertHits(resp.getHits(), "1");
+        resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("23")).get();
+        assertHits(resp.getHits(), "1");
+        resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("1293")).get();
+        assertHits(resp.getHits(), "1");
+        resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("42")).get();
+        assertHits(resp.getHits(), "1");
+        resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("1.7")).get();
+        assertHits(resp.getHits(), "1");
+        resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("1.5")).get();
+        assertHits(resp.getHits(), "1");
+        resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("12.23")).get();
+        assertHits(resp.getHits(), "1");
+        resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("127.0.0.1")).get();
+        assertHits(resp.getHits(), "1");
+        // binary doesn't match
+        // suggest doesn't match
+        // geo_point doesn't match
+        // geo_shape doesn't match
+
+        resp = client().prepareSearch("test").setQuery(
+                simpleQueryStringQuery("foo Bar 19 127.0.0.1").defaultOperator(Operator.AND)).get();
+        assertHits(resp.getHits(), "1");
+    }
+
+    public void testKeywordWithWhitespace() throws Exception {
+        String indexBody = copyToStringFromClasspath("/org/elasticsearch/search/query/all-query-index.json");
+        prepareCreate("test").setSource(indexBody).get();
+        ensureGreen("test");
+
+        List<IndexRequestBuilder> reqs = new ArrayList<>();
+        reqs.add(client().prepareIndex("test", "doc", "1").setSource("f2", "Foo Bar"));
+        reqs.add(client().prepareIndex("test", "doc", "2").setSource("f1", "bar"));
+        reqs.add(client().prepareIndex("test", "doc", "3").setSource("f1", "foo bar"));
+        indexRandom(true, false, reqs);
+
+        SearchResponse resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("foo")).get();
+        assertHits(resp.getHits(), "3");
+        assertHitCount(resp, 1L);
+
+        resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("bar")).get();
+        assertHits(resp.getHits(), "2", "3");
+        assertHitCount(resp, 2L);
+    }
+
+    public void testExplicitAllFieldsRequested() throws Exception {
+        String indexBody = copyToStringFromClasspath("/org/elasticsearch/search/query/all-query-index-with-all.json");
+        prepareCreate("test").setSource(indexBody).get();
+        ensureGreen("test");
+
+        List<IndexRequestBuilder> reqs = new ArrayList<>();
+        reqs.add(client().prepareIndex("test", "doc", "1").setSource("f1", "foo", "f2", "eggplant"));
+        indexRandom(true, false, reqs);
+
+        SearchResponse resp = client().prepareSearch("test").setQuery(
+                simpleQueryStringQuery("foo eggplent").defaultOperator(Operator.AND)).get();
+        assertHitCount(resp, 0L);
+
+        resp = client().prepareSearch("test").setQuery(
+                simpleQueryStringQuery("foo eggplent").defaultOperator(Operator.AND).useAllFields(true)).get();
+        assertHits(resp.getHits(), "1");
+        assertHitCount(resp, 1L);
+
+        Exception e = expectThrows(Exception.class, () ->
+                client().prepareSearch("test").setQuery(
+                        simpleQueryStringQuery("blah").field("f1").useAllFields(true)).get());
+        assertThat(ExceptionsHelper.detailedMessage(e),
+                containsString("cannot use [all_fields] parameter in conjunction with [fields]"));
+    }
+
+    @LuceneTestCase.AwaitsFix(bugUrl="currently can't perform phrase queries on fields that don't support positions")
+    public void testPhraseQueryOnFieldWithNoPositions() throws Exception {
+        String indexBody = copyToStringFromClasspath("/org/elasticsearch/search/query/all-query-index.json");
+        prepareCreate("test").setSource(indexBody).get();
+        ensureGreen("test");
+
+        List<IndexRequestBuilder> reqs = new ArrayList<>();
+        reqs.add(client().prepareIndex("test", "doc", "1").setSource("f1", "foo bar", "f4", "eggplant parmesan"));
+        reqs.add(client().prepareIndex("test", "doc", "2").setSource("f1", "foo bar", "f4", "chicken parmesan"));
+        indexRandom(true, false, reqs);
+
+        SearchResponse resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("\"eggplant parmesan\"")).get();
+        assertHits(resp.getHits(), "1");
+        assertHitCount(resp, 1L);
+    }
+
+    private void assertHits(SearchHits hits, String... ids) {
+        assertThat(hits.totalHits(), equalTo((long) ids.length));
+        Set<String> hitIds = new HashSet<>();
+        for (SearchHit hit : hits.getHits()) {
+            hitIds.add(hit.id());
+        }
+        assertThat(hitIds, containsInAnyOrder(ids));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java b/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java
index 1af4a2b1788b6..ee68ea60036d8 100644
--- a/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java
@@ -23,19 +23,16 @@
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.ContentPath;
 import org.elasticsearch.index.mapper.MappedFieldType;
@@ -45,7 +42,6 @@
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.search.rescore.QueryRescorer.QueryRescoreContext;
 import org.elasticsearch.test.ESTestCase;
@@ -56,14 +52,13 @@
 import java.io.IOException;
 
 import static java.util.Collections.emptyList;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.not;
+import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
 
 public class QueryRescoreBuilderTests extends ESTestCase {
 
     private static final int NUMBER_OF_TESTBUILDERS = 20;
     private static NamedWriteableRegistry namedWriteableRegistry;
-    private static IndicesQueriesRegistry indicesQueriesRegistry;
+    private static NamedXContentRegistry xContentRegistry;
 
     /**
      * setup for the whole base test class
@@ -72,13 +67,13 @@ public class QueryRescoreBuilderTests extends ESTestCase {
     public static void init() {
         SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList());
         namedWriteableRegistry = new NamedWriteableRegistry(searchModule.getNamedWriteables());
-        indicesQueriesRegistry = searchModule.getQueryParserRegistry();
+        xContentRegistry = new NamedXContentRegistry(searchModule.getNamedXContents());
     }
 
     @AfterClass
     public static void afterClass() throws Exception {
         namedWriteableRegistry = null;
-        indicesQueriesRegistry = null;
+        xContentRegistry = null;
     }
 
     /**
@@ -87,7 +82,7 @@ public static void afterClass() throws Exception {
     public void testSerialization() throws IOException {
         for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) {
             RescoreBuilder<?> original = randomRescoreBuilder();
-            RescoreBuilder<?> deserialized = serializedCopy(original);
+            RescoreBuilder<?> deserialized = copy(original);
             assertEquals(deserialized, original);
             assertEquals(deserialized.hashCode(), original.hashCode());
             assertNotSame(deserialized, original);
@@ -99,34 +94,15 @@ public void testSerialization() throws IOException {
      */
     public void testEqualsAndHashcode() throws IOException {
         for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) {
-            RescoreBuilder<?> firstBuilder = randomRescoreBuilder();
-            assertFalse("rescore builder is equal to null", firstBuilder.equals(null));
-            assertFalse("rescore builder is equal to incompatible type", firstBuilder.equals(""));
-            assertTrue("rescore builder is not equal to self", firstBuilder.equals(firstBuilder));
-            assertThat("same rescore builder's hashcode returns different values if called multiple times", firstBuilder.hashCode(),
-                    equalTo(firstBuilder.hashCode()));
-            assertThat("different rescore builder should not be equal", mutate(firstBuilder), not(equalTo(firstBuilder)));
-
-            RescoreBuilder<?> secondBuilder = serializedCopy(firstBuilder);
-            assertTrue("rescore builder is not equal to self", secondBuilder.equals(secondBuilder));
-            assertTrue("rescore builder is not equal to its copy", firstBuilder.equals(secondBuilder));
-            assertTrue("equals is not symmetric", secondBuilder.equals(firstBuilder));
-            assertThat("rescore builder copy's hashcode is different from original hashcode", secondBuilder.hashCode(),
-                    equalTo(firstBuilder.hashCode()));
-
-            RescoreBuilder<?> thirdBuilder = serializedCopy(secondBuilder);
-            assertTrue("rescore builder is not equal to self", thirdBuilder.equals(thirdBuilder));
-            assertTrue("rescore builder is not equal to its copy", secondBuilder.equals(thirdBuilder));
-            assertThat("rescore builder copy's hashcode is different from original hashcode", secondBuilder.hashCode(),
-                    equalTo(thirdBuilder.hashCode()));
-            assertTrue("equals is not transitive", firstBuilder.equals(thirdBuilder));
-            assertThat("rescore builder copy's hashcode is different from original hashcode", firstBuilder.hashCode(),
-                    equalTo(thirdBuilder.hashCode()));
-            assertTrue("equals is not symmetric", thirdBuilder.equals(secondBuilder));
-            assertTrue("equals is not symmetric", thirdBuilder.equals(firstBuilder));
+            checkEqualsAndHashCode(randomRescoreBuilder(), this::copy, QueryRescoreBuilderTests::mutate);
         }
     }
 
+    private RescoreBuilder<?> copy(RescoreBuilder<?> original) throws IOException {
+        return copyWriteable(original, namedWriteableRegistry,
+                namedWriteableRegistry.getReader(RescoreBuilder.class, original.getWriteableName()));
+    }
+
     /**
      *  creates random rescorer, renders it to xContent and back to new instance that should be equal to original
      */
@@ -141,8 +117,8 @@ public void testFromXContent() throws IOException {
             XContentBuilder shuffled = shuffleXContent(builder);
 
 
-            XContentParser parser = XContentHelper.createParser(shuffled.bytes());
-            QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.STRICT);
+            XContentParser parser = createParser(shuffled);
+            QueryParseContext context = new QueryParseContext(parser);
             parser.nextToken();
             RescoreBuilder<?> secondRescoreBuilder = RescoreBuilder.parseFromXContent(context);
             assertNotSame(rescoreBuilder, secondRescoreBuilder);
@@ -156,12 +132,13 @@ public void testFromXContent() throws IOException {
      * than the test builder
      */
     public void testBuildRescoreSearchContext() throws ElasticsearchParseException, IOException {
+        final long nowInMillis = randomNonNegativeLong();
         Settings indexSettings = Settings.builder()
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
-        IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAsciiOfLengthBetween(1, 10), indexSettings);
+        IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAlphaOfLengthBetween(1, 10), indexSettings);
         // shard context will only need indicesQueriesRegistry for building Query objects nested in query rescorer
-        QueryShardContext mockShardContext = new QueryShardContext(idxSettings, null, null, null, null, null, indicesQueriesRegistry,
-                null, null, null) {
+        QueryShardContext mockShardContext = new QueryShardContext(0, idxSettings, null, null, null, null, null, xContentRegistry(),
+                null, null, () -> nowInMillis) {
             @Override
             public MappedFieldType fieldMapper(String name) {
                 TextFieldMapper.Builder builder = new TextFieldMapper.Builder(name);
@@ -272,16 +249,21 @@ public void testUnknownFieldsExpection() throws IOException {
     /**
      * create a new parser from the rescorer string representation and reset context with it
      */
-    private static QueryParseContext createContext(String rescoreElement) throws IOException {
-        XContentParser parser = XContentFactory.xContent(rescoreElement).createParser(rescoreElement);
-        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.STRICT);
+    private QueryParseContext createContext(String rescoreElement) throws IOException {
+        XContentParser parser = createParser(JsonXContent.jsonXContent, rescoreElement);
+        QueryParseContext context = new QueryParseContext(parser);
         // move to first token, this is where the internal fromXContent
         assertTrue(parser.nextToken() == XContentParser.Token.START_OBJECT);
         return context;
     }
 
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return xContentRegistry;
+    }
+
     private static RescoreBuilder<?> mutate(RescoreBuilder<?> original) throws IOException {
-        RescoreBuilder<?> mutation = serializedCopy(original);
+        RescoreBuilder<?> mutation = ESTestCase.copyWriteable(original, namedWriteableRegistry, QueryRescorerBuilder::new);
         if (randomBoolean()) {
             Integer windowSize = original.windowSize();
             if (windowSize != null) {
@@ -321,7 +303,7 @@ private static RescoreBuilder<?> mutate(RescoreBuilder<?> original) throws IOExc
      */
     public static QueryRescorerBuilder randomRescoreBuilder() {
         QueryBuilder queryBuilder = new MatchAllQueryBuilder().boost(randomFloat())
-                .queryName(randomAsciiOfLength(20));
+                .queryName(randomAlphaOfLength(20));
         org.elasticsearch.search.rescore.QueryRescorerBuilder rescorer = new
                 org.elasticsearch.search.rescore.QueryRescorerBuilder(queryBuilder);
         if (randomBoolean()) {
@@ -338,14 +320,4 @@ public static QueryRescorerBuilder randomRescoreBuilder() {
         }
         return rescorer;
     }
-
-    private static RescoreBuilder<?> serializedCopy(RescoreBuilder<?> original) throws IOException {
-        try (BytesStreamOutput output = new BytesStreamOutput()) {
-            output.writeNamedWriteable(original);
-            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
-                return in.readNamedWriteable(RescoreBuilder.class);
-            }
-        }
-    }
-
 }
diff --git a/core/src/test/java/org/elasticsearch/search/scriptfilter/ScriptQuerySearchIT.java b/core/src/test/java/org/elasticsearch/search/scriptfilter/ScriptQuerySearchIT.java
index 6422bf7a134a2..8d235f0a78f88 100644
--- a/core/src/test/java/org/elasticsearch/search/scriptfilter/ScriptQuerySearchIT.java
+++ b/core/src/test/java/org/elasticsearch/search/scriptfilter/ScriptQuerySearchIT.java
@@ -21,15 +21,19 @@
 
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.MockScriptPlugin;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.test.ESIntegTestCase;
 
+import java.io.IOException;
+import java.util.Base64;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
@@ -37,11 +41,13 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Function;
 
+import static java.util.Collections.emptyMap;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.scriptQuery;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.equalTo;
 
-@ESIntegTestCase.ClusterScope(scope= ESIntegTestCase.Scope.SUITE)
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE)
 public class ScriptQuerySearchIT extends ESIntegTestCase {
 
     @Override
@@ -74,6 +80,16 @@ protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
                 return num1.getValue() > param1;
             });
 
+            scripts.put("doc['binaryData'].get(0).length", vars -> {
+                Map<?, ?> doc = (Map) vars.get("doc");
+                return ((ScriptDocValues.BytesRefs) doc.get("binaryData")).get(0).length;
+            });
+
+            scripts.put("doc['binaryData'].get(0).length > 15", vars -> {
+                Map<?, ?> doc = (Map) vars.get("doc");
+                return ((ScriptDocValues.BytesRefs) doc.get("binaryData")).get(0).length > 15;
+            });
+
             return scripts;
         }
     }
@@ -87,6 +103,57 @@ public Settings indexSettings() {
                 .build();
     }
 
+    public void testCustomScriptBinaryField() throws Exception {
+        final byte[] randomBytesDoc1 = getRandomBytes(15);
+        final byte[] randomBytesDoc2 = getRandomBytes(16);
+
+        assertAcked(
+                client().admin().indices().prepareCreate("my-index")
+                        .addMapping("my-type", createMappingSource("binary"))
+                        .setSettings(indexSettings())
+        );
+        client().prepareIndex("my-index", "my-type", "1")
+                .setSource(jsonBuilder().startObject().field("binaryData",
+                        Base64.getEncoder().encodeToString(randomBytesDoc1)).endObject())
+                .get();
+        flush();
+        client().prepareIndex("my-index", "my-type", "2")
+                .setSource(jsonBuilder().startObject().field("binaryData",
+                        Base64.getEncoder().encodeToString(randomBytesDoc2)).endObject())
+                .get();
+        flush();
+        refresh();
+
+        SearchResponse response = client().prepareSearch()
+                .setQuery(scriptQuery(
+                        new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['binaryData'].get(0).length > 15", emptyMap())))
+                .addScriptField("sbinaryData",
+                        new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['binaryData'].get(0).length", emptyMap()))
+                .get();
+
+        assertThat(response.getHits().totalHits(), equalTo(1L));
+        assertThat(response.getHits().getAt(0).id(), equalTo("2"));
+        assertThat(response.getHits().getAt(0).fields().get("sbinaryData").values().get(0), equalTo(16));
+
+    }
+
+    private byte[] getRandomBytes(int len) {
+        final byte[] randomBytes = new byte[len];
+        random().nextBytes(randomBytes);
+        return randomBytes;
+    }
+
+    private XContentBuilder createMappingSource(String fieldType) throws IOException {
+        return XContentFactory.jsonBuilder().startObject().startObject("my-type")
+                .startObject("properties")
+                .startObject("binaryData")
+                .field("type", fieldType)
+                .field("doc_values", "true")
+                .endObject()
+                .endObject()
+                .endObject().endObject();
+    }
+
     public void testCustomScriptBoost() throws Exception {
         createIndex("test");
         client().prepareIndex("test", "type1", "1")
@@ -104,9 +171,11 @@ public void testCustomScriptBoost() throws Exception {
 
         logger.info("running doc['num1'].value > 1");
         SearchResponse response = client().prepareSearch()
-                .setQuery(scriptQuery(new Script("doc['num1'].value > 1", ScriptType.INLINE, CustomScriptPlugin.NAME, null)))
+                .setQuery(scriptQuery(
+                        new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['num1'].value > 1", Collections.emptyMap())))
                 .addSort("num1", SortOrder.ASC)
-                .addScriptField("sNum1", new Script("doc['num1'].value", ScriptType.INLINE, CustomScriptPlugin.NAME, null))
+                .addScriptField("sNum1",
+                        new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['num1'].value", Collections.emptyMap()))
                 .get();
 
         assertThat(response.getHits().totalHits(), equalTo(2L));
@@ -121,9 +190,10 @@ public void testCustomScriptBoost() throws Exception {
         logger.info("running doc['num1'].value > param1");
         response = client()
                 .prepareSearch()
-                .setQuery(scriptQuery(new Script("doc['num1'].value > param1", ScriptType.INLINE, CustomScriptPlugin.NAME, params)))
+                .setQuery(scriptQuery(new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['num1'].value > param1", params)))
                 .addSort("num1", SortOrder.ASC)
-                .addScriptField("sNum1", new Script("doc['num1'].value", ScriptType.INLINE, CustomScriptPlugin.NAME, null))
+                .addScriptField("sNum1",
+                        new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['num1'].value", Collections.emptyMap()))
                 .get();
 
         assertThat(response.getHits().totalHits(), equalTo(1L));
@@ -135,9 +205,10 @@ public void testCustomScriptBoost() throws Exception {
         logger.info("running doc['num1'].value > param1");
         response = client()
                 .prepareSearch()
-                .setQuery(scriptQuery(new Script("doc['num1'].value > param1", ScriptType.INLINE, CustomScriptPlugin.NAME, params)))
+                .setQuery(scriptQuery(new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['num1'].value > param1", params)))
                 .addSort("num1", SortOrder.ASC)
-                .addScriptField("sNum1", new Script("doc['num1'].value", ScriptType.INLINE, CustomScriptPlugin.NAME, null))
+                .addScriptField("sNum1",
+                        new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['num1'].value", Collections.emptyMap()))
                 .get();
 
         assertThat(response.getHits().totalHits(), equalTo(3L));
diff --git a/core/src/test/java/org/elasticsearch/search/scroll/DuelScrollIT.java b/core/src/test/java/org/elasticsearch/search/scroll/DuelScrollIT.java
index 0ece1e5d83e54..15e806c228a72 100644
--- a/core/src/test/java/org/elasticsearch/search/scroll/DuelScrollIT.java
+++ b/core/src/test/java/org/elasticsearch/search/scroll/DuelScrollIT.java
@@ -99,38 +99,6 @@ public void testDuelQueryThenFetch() throws Exception {
         clearScroll(scrollId);
     }
 
-    public void testDuelQueryAndFetch() throws Exception {
-        // *_QUERY_AND_FETCH search types are tricky: the ordering can be incorrect, since it returns num_shards * (from + size)
-        // a subsequent scroll call can return hits that should have been in the hits of the first scroll call.
-
-        TestContext context = create(SearchType.DFS_QUERY_AND_FETCH, SearchType.QUERY_AND_FETCH);
-        SearchResponse searchScrollResponse = client().prepareSearch("index")
-                .setSearchType(context.searchType)
-                .addSort(context.sort)
-                .setSize(context.scrollRequestSize)
-                .setScroll("10m").get();
-
-        assertNoFailures(searchScrollResponse);
-        assertThat(searchScrollResponse.getHits().getTotalHits(), equalTo((long) context.numDocs));
-
-        int counter = searchScrollResponse.getHits().hits().length;
-        String scrollId = searchScrollResponse.getScrollId();
-        while (true) {
-            searchScrollResponse = client().prepareSearchScroll(scrollId).setScroll("10m").get();
-            assertNoFailures(searchScrollResponse);
-            assertThat(searchScrollResponse.getHits().getTotalHits(), equalTo((long) context.numDocs));
-            if (searchScrollResponse.getHits().hits().length == 0) {
-                break;
-            }
-
-            counter += searchScrollResponse.getHits().hits().length;
-            scrollId = searchScrollResponse.getScrollId();
-        }
-
-        assertThat(counter, equalTo(context.numDocs));
-        clearScroll(scrollId);
-    }
-
 
     private TestContext create(SearchType... searchTypes) throws Exception {
         assertAcked(prepareCreate("index").addMapping("type", jsonBuilder().startObject().startObject("type").startObject("properties")
@@ -290,16 +258,9 @@ private void testDuelIndexOrder(SearchType searchType, boolean trackScores, int
         }
     }
 
-    public void testDuelIndexOrderQueryAndFetch() throws Exception {
-        final SearchType searchType = RandomPicks.randomFrom(random(), Arrays.asList(SearchType.QUERY_AND_FETCH, SearchType.DFS_QUERY_AND_FETCH));
-        // QUERY_AND_FETCH only works with a single shard
-        final int numDocs = createIndex(true);
-        testDuelIndexOrder(searchType, false, numDocs);
-        testDuelIndexOrder(searchType, true, numDocs);
-    }
-
     public void testDuelIndexOrderQueryThenFetch() throws Exception {
-        final SearchType searchType = RandomPicks.randomFrom(random(), Arrays.asList(SearchType.QUERY_THEN_FETCH, SearchType.DFS_QUERY_THEN_FETCH));
+        final SearchType searchType = RandomPicks.randomFrom(random(), Arrays.asList(SearchType.QUERY_THEN_FETCH,
+            SearchType.DFS_QUERY_THEN_FETCH));
         final int numDocs = createIndex(false);
         testDuelIndexOrder(searchType, false, numDocs);
         testDuelIndexOrder(searchType, true, numDocs);
diff --git a/core/src/test/java/org/elasticsearch/search/scroll/RestClearScrollActionTests.java b/core/src/test/java/org/elasticsearch/search/scroll/RestClearScrollActionTests.java
new file mode 100644
index 0000000000000..5341464c6ccf2
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/scroll/RestClearScrollActionTests.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.scroll;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.search.ClearScrollRequest;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.rest.RestChannel;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.search.RestClearScrollAction;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.rest.FakeRestChannel;
+import org.elasticsearch.test.rest.FakeRestRequest;
+import org.mockito.ArgumentCaptor;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyObject;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+
+public class RestClearScrollActionTests extends ESTestCase {
+
+    public void testParseClearScrollRequestWithInvalidJsonThrowsException() throws Exception {
+        RestClearScrollAction action = new RestClearScrollAction(Settings.EMPTY, mock(RestController.class));
+        RestRequest request = new FakeRestRequest.Builder(xContentRegistry())
+            .withContent(new BytesArray("{invalid_json}"), XContentType.JSON).build();
+        Exception e = expectThrows(IllegalArgumentException.class, () -> action.prepareRequest(request, null));
+        assertThat(e.getMessage(), equalTo("Failed to parse request body"));
+    }
+
+    public void testBodyParamsOverrideQueryStringParams() throws Exception {
+        NodeClient nodeClient = mock(NodeClient.class);
+        doNothing().when(nodeClient).searchScroll(any(), any());
+
+        RestClearScrollAction action = new RestClearScrollAction(Settings.EMPTY, mock(RestController.class));
+        RestRequest request = new FakeRestRequest.Builder(xContentRegistry())
+                .withParams(Collections.singletonMap("scroll_id", "QUERY_STRING"))
+                .withContent(new BytesArray("{\"scroll_id\": [\"BODY\"]}"), XContentType.JSON).build();
+        FakeRestChannel channel = new FakeRestChannel(request, false, 0);
+        action.handleRequest(request, channel, nodeClient);
+
+        ArgumentCaptor<ClearScrollRequest> argument = ArgumentCaptor.forClass(ClearScrollRequest.class);
+        verify(nodeClient).clearScroll(argument.capture(), anyObject());
+        ClearScrollRequest clearScrollRequest = argument.getValue();
+        List<String> scrollIds = clearScrollRequest.getScrollIds();
+        assertEquals(1, scrollIds.size());
+        assertEquals("BODY", scrollIds.get(0));
+    }
+
+    public void testParseClearScrollPlaintext() throws Exception {
+        RestClearScrollAction action = new RestClearScrollAction(Settings.EMPTY, mock(RestController.class));
+        NodeClient mockNodeClient = mock(NodeClient.class);
+        final List<String> scrollIds = Arrays.asList(generateRandomStringArray(4, 30, false, false));
+        final String content = scrollIds.stream().collect(Collectors.joining(","));
+        FakeRestRequest fakeRestRequest = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY)
+                .withContent(new BytesArray(content), null)
+                .withHeaders(Collections.singletonMap("Content-Type", Collections.singletonList("text/plain")))
+                .build();
+        action.handleRequest(fakeRestRequest, mock(RestChannel.class), mockNodeClient);
+        ArgumentCaptor<ClearScrollRequest> captor = ArgumentCaptor.forClass(ClearScrollRequest.class);
+        verify(mockNodeClient).clearScroll(captor.capture(), any(ActionListener.class));
+
+        assertEquals(scrollIds, captor.getValue().getScrollIds());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/scroll/RestSearchScrollActionTests.java b/core/src/test/java/org/elasticsearch/search/scroll/RestSearchScrollActionTests.java
new file mode 100644
index 0000000000000..49c2788e1e19f
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/scroll/RestSearchScrollActionTests.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.scroll;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.search.SearchScrollRequest;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.rest.RestChannel;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.search.RestSearchScrollAction;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.rest.FakeRestChannel;
+import org.elasticsearch.test.rest.FakeRestRequest;
+import org.mockito.ArgumentCaptor;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import java.util.Collections;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyObject;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+
+public class RestSearchScrollActionTests extends ESTestCase {
+
+    public void testParseSearchScrollRequestWithInvalidJsonThrowsException() throws Exception {
+        RestSearchScrollAction action = new RestSearchScrollAction(Settings.EMPTY, mock(RestController.class));
+        RestRequest request = new FakeRestRequest.Builder(xContentRegistry())
+            .withContent(new BytesArray("{invalid_json}"), XContentType.JSON).build();
+        Exception e = expectThrows(IllegalArgumentException.class, () -> action.prepareRequest(request, null));
+        assertThat(e.getMessage(), equalTo("Failed to parse request body"));
+    }
+
+    public void testBodyParamsOverrideQueryStringParams() throws Exception {
+        NodeClient nodeClient = mock(NodeClient.class);
+        doNothing().when(nodeClient).searchScroll(any(), any());
+
+        RestSearchScrollAction action = new RestSearchScrollAction(Settings.EMPTY, mock(RestController.class));
+        Map<String, String> params = new HashMap<>();
+        params.put("scroll_id", "QUERY_STRING");
+        params.put("scroll", "1000m");
+        RestRequest request = new FakeRestRequest.Builder(xContentRegistry())
+                .withParams(params)
+                .withContent(new BytesArray("{\"scroll_id\":\"BODY\", \"scroll\":\"1m\"}"), XContentType.JSON).build();
+        FakeRestChannel channel = new FakeRestChannel(request, false, 0);
+        action.handleRequest(request, channel, nodeClient);
+
+        ArgumentCaptor<SearchScrollRequest> argument = ArgumentCaptor.forClass(SearchScrollRequest.class);
+        verify(nodeClient).searchScroll(argument.capture(), anyObject());
+        SearchScrollRequest searchScrollRequest = argument.getValue();
+        assertEquals("BODY", searchScrollRequest.scrollId());
+        assertEquals("1m", searchScrollRequest.scroll().keepAlive().getStringRep());
+    }
+
+    public void testParseSearchScrollPlaintext() throws Exception {
+        RestSearchScrollAction action = new RestSearchScrollAction(Settings.EMPTY, mock(RestController.class));
+        NodeClient mockNodeClient = mock(NodeClient.class);
+        final String scrollId = randomAlphaOfLengthBetween(1, 30);
+        FakeRestRequest fakeRestRequest = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY)
+            .withContent(new BytesArray(scrollId), null)
+            .withHeaders(Collections.singletonMap("Content-Type", Collections.singletonList("text/plain")))
+            .build();
+        action.handleRequest(fakeRestRequest, mock(RestChannel.class), mockNodeClient);
+        ArgumentCaptor<SearchScrollRequest> captor = ArgumentCaptor.forClass(SearchScrollRequest.class);
+        verify(mockNodeClient).searchScroll(captor.capture(), any(ActionListener.class));
+
+        assertEquals(scrollId, captor.getValue().scrollId());
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java b/core/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java
index 275bc1c0c81b6..1f368cb35daf4 100644
--- a/core/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java
+++ b/core/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java
@@ -19,27 +19,21 @@
 
 package org.elasticsearch.search.scroll;
 
-import org.elasticsearch.action.search.ClearScrollRequest;
 import org.elasticsearch.action.search.ClearScrollResponse;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.search.SearchScrollRequest;
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.Priority;
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.rest.action.search.RestClearScrollAction;
-import org.elasticsearch.rest.action.search.RestSearchScrollAction;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.sort.FieldSortBuilder;
 import org.elasticsearch.search.sort.SortOrder;
@@ -60,13 +54,10 @@
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows;
-import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
-import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
-import static org.hamcrest.Matchers.startsWith;
 
 /**
  * Tests for scrolling.
@@ -500,84 +491,6 @@ public void testStringSortMissingAscTerminates() throws Exception {
         assertThat(response.getHits().getHits().length, equalTo(0));
     }
 
-    public void testParseSearchScrollRequest() throws Exception {
-        BytesReference content = XContentFactory.jsonBuilder()
-            .startObject()
-            .field("scroll_id", "SCROLL_ID")
-            .field("scroll", "1m")
-            .endObject().bytes();
-
-        SearchScrollRequest searchScrollRequest = new SearchScrollRequest();
-        RestSearchScrollAction.buildFromContent(content, searchScrollRequest);
-
-        assertThat(searchScrollRequest.scrollId(), equalTo("SCROLL_ID"));
-        assertThat(searchScrollRequest.scroll().keepAlive(), equalTo(TimeValue.parseTimeValue("1m", null, "scroll")));
-    }
-
-    public void testParseSearchScrollRequestWithInvalidJsonThrowsException() throws Exception {
-        SearchScrollRequest searchScrollRequest = new SearchScrollRequest();
-        try {
-            RestSearchScrollAction.buildFromContent(new BytesArray("{invalid_json}"), searchScrollRequest);
-            fail("expected parseContent failure");
-        } catch (Exception e) {
-            assertThat(e, instanceOf(IllegalArgumentException.class));
-            assertThat(e.getMessage(), equalTo("Failed to parse request body"));
-        }
-    }
-
-    public void testParseSearchScrollRequestWithUnknownParamThrowsException() throws Exception {
-        SearchScrollRequest searchScrollRequest = new SearchScrollRequest();
-        BytesReference invalidContent = XContentFactory.jsonBuilder().startObject()
-            .field("scroll_id", "value_2")
-            .field("unknown", "keyword")
-            .endObject().bytes();
-
-        try {
-            RestSearchScrollAction.buildFromContent(invalidContent, searchScrollRequest);
-            fail("expected parseContent failure");
-        } catch (Exception e) {
-            assertThat(e, instanceOf(IllegalArgumentException.class));
-            assertThat(e.getMessage(), startsWith("Unknown parameter [unknown]"));
-        }
-    }
-
-    public void testParseClearScrollRequest() throws Exception {
-        BytesReference content = XContentFactory.jsonBuilder().startObject()
-            .array("scroll_id", "value_1", "value_2")
-            .endObject().bytes();
-        ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
-        RestClearScrollAction.buildFromContent(content, clearScrollRequest);
-        assertThat(clearScrollRequest.scrollIds(), contains("value_1", "value_2"));
-    }
-
-    public void testParseClearScrollRequestWithInvalidJsonThrowsException() throws Exception {
-        ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
-
-        try {
-            RestClearScrollAction.buildFromContent(new BytesArray("{invalid_json}"), clearScrollRequest);
-            fail("expected parseContent failure");
-        } catch (Exception e) {
-            assertThat(e, instanceOf(IllegalArgumentException.class));
-            assertThat(e.getMessage(), equalTo("Failed to parse request body"));
-        }
-    }
-
-    public void testParseClearScrollRequestWithUnknownParamThrowsException() throws Exception {
-        BytesReference invalidContent = XContentFactory.jsonBuilder().startObject()
-            .array("scroll_id", "value_1", "value_2")
-            .field("unknown", "keyword")
-            .endObject().bytes();
-        ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
-
-        try {
-            RestClearScrollAction.buildFromContent(invalidContent, clearScrollRequest);
-            fail("expected parseContent failure");
-        } catch (Exception e) {
-            assertThat(e, instanceOf(IllegalArgumentException.class));
-            assertThat(e.getMessage(), startsWith("Unknown parameter [unknown]"));
-        }
-    }
-
     public void testCloseAndReopenOrDeleteWithActiveScroll() throws IOException {
         createIndex("test");
         for (int i = 0; i < 100; i++) {
@@ -607,16 +520,8 @@ public void testCloseAndReopenOrDeleteWithActiveScroll() throws IOException {
 
     private void assertToXContentResponse(ClearScrollResponse response, boolean succeed, int numFreed) throws IOException {
         XContentBuilder builder = XContentFactory.jsonBuilder();
-        builder.startObject();
         response.toXContent(builder, ToXContent.EMPTY_PARAMS);
-        builder.endObject();
-
-        BytesReference bytesReference = builder.bytes();
-        Map<String, Object> map;
-        try (XContentParser parser = XContentFactory.xContent(bytesReference).createParser(bytesReference)) {
-            map = parser.map();
-        }
-
+        Map<String, Object> map = XContentHelper.convertToMap(builder.bytes(), false, builder.contentType()).v2();
         assertThat(map.get("succeeded"), is(succeed));
         assertThat(map.get("num_freed"), equalTo(numFreed));
     }
diff --git a/core/src/test/java/org/elasticsearch/search/searchafter/SearchAfterBuilderTests.java b/core/src/test/java/org/elasticsearch/search/searchafter/SearchAfterBuilderTests.java
index 967c1c68134ee..d023133135173 100644
--- a/core/src/test/java/org/elasticsearch/search/searchafter/SearchAfterBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/searchafter/SearchAfterBuilderTests.java
@@ -19,50 +19,34 @@
 
 package org.elasticsearch.search.searchafter;
 
-import org.elasticsearch.common.ParseFieldMatcher;
+import org.apache.lucene.document.LatLonDocValuesField;
+import org.apache.lucene.search.FieldComparator;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.SortedNumericSortField;
+import org.apache.lucene.search.SortedSetSortField;
 import org.elasticsearch.common.geo.GeoPoint;
-import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.text.Text;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.index.query.MatchAllQueryBuilder;
-import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.index.query.QueryParser;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.search.MultiValueMode;
 import org.elasticsearch.test.ESTestCase;
-import org.hamcrest.Matchers;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
 
 import java.io.IOException;
+import java.util.Collections;
+
+import static org.elasticsearch.search.searchafter.SearchAfterBuilder.extractSortType;
+import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
 import static org.hamcrest.Matchers.equalTo;
 
 public class SearchAfterBuilderTests extends ESTestCase {
     private static final int NUMBER_OF_TESTBUILDERS = 20;
-    private static IndicesQueriesRegistry indicesQueriesRegistry;
-
-    /**
-     * setup for the whole base test class
-     */
-    @BeforeClass
-    public static void init() {
-        indicesQueriesRegistry = new IndicesQueriesRegistry();
-        QueryParser<MatchAllQueryBuilder> parser = MatchAllQueryBuilder::fromXContent;
-        indicesQueriesRegistry.register(parser, MatchAllQueryBuilder.NAME);
-    }
 
-    @AfterClass
-    public static void afterClass() throws Exception {
-        indicesQueriesRegistry = null;
-    }
-
-    private SearchAfterBuilder randomSearchFromBuilder() throws IOException {
+    private static SearchAfterBuilder randomSearchAfterBuilder() throws IOException {
         int numSearchFrom = randomIntBetween(1, 10);
         SearchAfterBuilder searchAfterBuilder = new SearchAfterBuilder();
         Object[] values = new Object[numSearchFrom];
@@ -82,7 +66,7 @@ private SearchAfterBuilder randomSearchFromBuilder() throws IOException {
                     values[i] = randomDouble();
                     break;
                 case 4:
-                    values[i] = randomAsciiOfLengthBetween(5, 20);
+                    values[i] = randomAlphaOfLengthBetween(5, 20);
                     break;
                 case 5:
                     values[i] = randomBoolean();
@@ -94,7 +78,7 @@ private SearchAfterBuilder randomSearchFromBuilder() throws IOException {
                     values[i] = randomShort();
                     break;
                 case 8:
-                    values[i] = new Text(randomAsciiOfLengthBetween(5, 20));
+                    values[i] = new Text(randomAlphaOfLengthBetween(5, 20));
                     break;
                 case 9:
                     values[i] = null;
@@ -130,7 +114,7 @@ private SearchAfterBuilder randomJsonSearchFromBuilder() throws IOException {
                     jsonBuilder.value(randomDouble());
                     break;
                 case 4:
-                    jsonBuilder.value(randomAsciiOfLengthBetween(5, 20));
+                    jsonBuilder.value(randomAlphaOfLengthBetween(5, 20));
                     break;
                 case 5:
                     jsonBuilder.value(randomBoolean());
@@ -142,7 +126,7 @@ private SearchAfterBuilder randomJsonSearchFromBuilder() throws IOException {
                     jsonBuilder.value(randomShort());
                     break;
                 case 8:
-                    jsonBuilder.value(new Text(randomAsciiOfLengthBetween(5, 20)));
+                    jsonBuilder.value(new Text(randomAlphaOfLengthBetween(5, 20)));
                     break;
                 case 9:
                     jsonBuilder.nullValue();
@@ -151,25 +135,20 @@ private SearchAfterBuilder randomJsonSearchFromBuilder() throws IOException {
         }
         jsonBuilder.endArray();
         jsonBuilder.endObject();
-        XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(jsonBuilder.bytes());
+        XContentParser parser = createParser(JsonXContent.jsonXContent, jsonBuilder.bytes());
         parser.nextToken();
         parser.nextToken();
         parser.nextToken();
-        return SearchAfterBuilder.fromXContent(parser, null);
+        return SearchAfterBuilder.fromXContent(parser);
     }
 
     private static SearchAfterBuilder serializedCopy(SearchAfterBuilder original) throws IOException {
-        try (BytesStreamOutput output = new BytesStreamOutput()) {
-            original.writeTo(output);
-            try (StreamInput in = output.bytes().streamInput()) {
-                return new SearchAfterBuilder(in);
-            }
-        }
+        return copyWriteable(original, new NamedWriteableRegistry(Collections.emptyList()), SearchAfterBuilder::new);
     }
 
     public void testSerialization() throws Exception {
         for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) {
-            SearchAfterBuilder original = randomSearchFromBuilder();
+            SearchAfterBuilder original = randomSearchAfterBuilder();
             SearchAfterBuilder deserialized = serializedCopy(original);
             assertEquals(deserialized, original);
             assertEquals(deserialized.hashCode(), original.hashCode());
@@ -179,30 +158,8 @@ public void testSerialization() throws Exception {
 
     public void testEqualsAndHashcode() throws Exception {
         for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) {
-            SearchAfterBuilder firstBuilder = randomSearchFromBuilder();
-            assertFalse("searchFrom is equal to null", firstBuilder.equals(null));
-            assertFalse("searchFrom is equal to incompatible type", firstBuilder.equals(""));
-            assertTrue("searchFrom is not equal to self", firstBuilder.equals(firstBuilder));
-            assertThat("same searchFrom's hashcode returns different values if called multiple times", firstBuilder.hashCode(),
-                    equalTo(firstBuilder.hashCode()));
-
-            SearchAfterBuilder secondBuilder = serializedCopy(firstBuilder);
-            assertTrue("searchFrom is not equal to self", secondBuilder.equals(secondBuilder));
-            assertTrue("searchFrom is not equal to its copy", firstBuilder.equals(secondBuilder));
-            assertTrue("equals is not symmetric", secondBuilder.equals(firstBuilder));
-            assertThat("searchFrom copy's hashcode is different from original hashcode", secondBuilder.hashCode(),
-                    equalTo(firstBuilder.hashCode()));
-
-            SearchAfterBuilder thirdBuilder = serializedCopy(secondBuilder);
-            assertTrue("searchFrom is not equal to self", thirdBuilder.equals(thirdBuilder));
-            assertTrue("searchFrom is not equal to its copy", secondBuilder.equals(thirdBuilder));
-            assertThat("searchFrom copy's hashcode is different from original hashcode", secondBuilder.hashCode(),
-                    equalTo(thirdBuilder.hashCode()));
-            assertTrue("equals is not transitive", firstBuilder.equals(thirdBuilder));
-            assertThat("searchFrom copy's hashcode is different from original hashcode", firstBuilder.hashCode(),
-                    equalTo(thirdBuilder.hashCode()));
-            assertTrue("searchFrom is not symmetric", thirdBuilder.equals(secondBuilder));
-            assertTrue("searchFrom is not symmetric", thirdBuilder.equals(firstBuilder));
+            // TODO add equals tests with mutating the original object
+            checkEqualsAndHashCode(randomSearchAfterBuilder(), SearchAfterBuilderTests::serializedCopy);
         }
     }
 
@@ -216,12 +173,11 @@ public void testFromXContent() throws Exception {
             builder.startObject();
             searchAfterBuilder.innerToXContent(builder);
             builder.endObject();
-            XContentParser parser = XContentHelper.createParser(shuffleXContent(builder).bytes());
-            new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.STRICT);
+            XContentParser parser = createParser(shuffleXContent(builder));
             parser.nextToken();
             parser.nextToken();
             parser.nextToken();
-            SearchAfterBuilder secondSearchAfterBuilder = SearchAfterBuilder.fromXContent(parser, null);
+            SearchAfterBuilder secondSearchAfterBuilder = SearchAfterBuilder.fromXContent(parser);
             assertNotSame(searchAfterBuilder, secondSearchAfterBuilder);
             assertEquals(searchAfterBuilder, secondSearchAfterBuilder);
             assertEquals(searchAfterBuilder.hashCode(), secondSearchAfterBuilder.hashCode());
@@ -234,7 +190,7 @@ public void testWithNullArray() throws Exception {
             builder.setSortValues(null);
             fail("Should fail on null array.");
         } catch (NullPointerException e) {
-            assertThat(e.getMessage(), Matchers.equalTo("Values cannot be null."));
+            assertThat(e.getMessage(), equalTo("Values cannot be null."));
         }
     }
 
@@ -244,27 +200,52 @@ public void testWithEmptyArray() throws Exception {
             builder.setSortValues(new Object[0]);
             fail("Should fail on empty array.");
         } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), Matchers.equalTo("Values must contains at least one value."));
+            assertThat(e.getMessage(), equalTo("Values must contains at least one value."));
         }
     }
 
     /**
-     * Explicitly tests what you can't list as a sortValue. What you can list is tested by {@link #randomSearchFromBuilder()}.
+     * Explicitly tests what you can't list as a sortValue. What you can list is tested by {@link #randomSearchAfterBuilder()}.
      */
     public void testBadTypes() throws IOException {
         randomSearchFromBuilderWithSortValueThrows(new Object());
         randomSearchFromBuilderWithSortValueThrows(new GeoPoint(0, 0));
-        randomSearchFromBuilderWithSortValueThrows(randomSearchFromBuilder());
+        randomSearchFromBuilderWithSortValueThrows(randomSearchAfterBuilder());
         randomSearchFromBuilderWithSortValueThrows(this);
     }
 
-    private void randomSearchFromBuilderWithSortValueThrows(Object containing) throws IOException {
+    private static void randomSearchFromBuilderWithSortValueThrows(Object containing) throws IOException {
         // Get a valid one
-        SearchAfterBuilder builder = randomSearchFromBuilder();
+        SearchAfterBuilder builder = randomSearchAfterBuilder();
         // Now replace its values with one containing the passed in object
         Object[] values = builder.getSortValues();
         values[between(0, values.length - 1)] = containing;
         Exception e = expectThrows(IllegalArgumentException.class, () -> builder.setSortValues(values));
         assertEquals(e.getMessage(), "Can't handle search_after field value of type [" + containing.getClass() + "]");
     }
+
+    public void testExtractSortType() throws Exception {
+        SortField.Type type = extractSortType(LatLonDocValuesField.newDistanceSort("field", 0.0, 180.0));
+        assertThat(type, equalTo(SortField.Type.DOUBLE));
+        IndexFieldData.XFieldComparatorSource source = new IndexFieldData.XFieldComparatorSource() {
+            @Override
+            public SortField.Type reducedType() {
+                return SortField.Type.STRING;
+            }
+
+            @Override
+            public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) {
+                return null;
+            }
+        };
+
+        type = extractSortType(new SortField("field", source));
+        assertThat(type, equalTo(SortField.Type.STRING));
+
+        type = extractSortType(new SortedNumericSortField("field", SortField.Type.DOUBLE));
+        assertThat(type, equalTo(SortField.Type.DOUBLE));
+
+        type = extractSortType(new SortedSetSortField("field", false));
+        assertThat(type, equalTo(SortField.Type.STRING));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/searchafter/SearchAfterIT.java b/core/src/test/java/org/elasticsearch/search/searchafter/SearchAfterIT.java
index 08c661a130b28..b036d4348d57b 100644
--- a/core/src/test/java/org/elasticsearch/search/searchafter/SearchAfterIT.java
+++ b/core/src/test/java/org/elasticsearch/search/searchafter/SearchAfterIT.java
@@ -24,13 +24,13 @@
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.SearchContextException;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.transport.RemoteTransportException;
 import org.hamcrest.Matchers;
 
 import java.util.List;
@@ -65,9 +65,11 @@ public void testsShouldFail() throws Exception {
 
             fail("Should fail on search_after cannot be used with scroll.");
         } catch (SearchPhaseExecutionException e) {
-            assertThat(e.getCause().getClass(), Matchers.equalTo(RemoteTransportException.class));
-            assertThat(e.getCause().getCause().getClass(), Matchers.equalTo(SearchContextException.class));
-            assertThat(e.getCause().getCause().getMessage(), Matchers.equalTo("`search_after` cannot be used in a scroll context."));
+            assertTrue(e.shardFailures().length > 0);
+            for (ShardSearchFailure failure : e.shardFailures()) {
+                assertThat(failure.getCause().getClass(), Matchers.equalTo(SearchContextException.class));
+                assertThat(failure.getCause().getMessage(), Matchers.equalTo("`search_after` cannot be used in a scroll context."));
+            }
         }
         try {
             client().prepareSearch("test")
@@ -79,9 +81,11 @@ public void testsShouldFail() throws Exception {
 
             fail("Should fail on search_after cannot be used with from > 0.");
         } catch (SearchPhaseExecutionException e) {
-            assertThat(e.getCause().getClass(), Matchers.equalTo(RemoteTransportException.class));
-            assertThat(e.getCause().getCause().getClass(), Matchers.equalTo(SearchContextException.class));
-            assertThat(e.getCause().getCause().getMessage(), Matchers.equalTo("`from` parameter must be set to 0 when `search_after` is used."));
+            assertTrue(e.shardFailures().length > 0);
+            for (ShardSearchFailure failure : e.shardFailures()) {
+                assertThat(failure.getCause().getClass(), Matchers.equalTo(SearchContextException.class));
+                assertThat(failure.getCause().getMessage(), Matchers.equalTo("`from` parameter must be set to 0 when `search_after` is used."));
+            }
         }
 
         try {
@@ -92,9 +96,11 @@ public void testsShouldFail() throws Exception {
 
             fail("Should fail on search_after on score only is disabled");
         } catch (SearchPhaseExecutionException e) {
-            assertThat(e.getCause().getClass(), Matchers.equalTo(RemoteTransportException.class));
-            assertThat(e.getCause().getCause().getClass(), Matchers.equalTo(IllegalArgumentException.class));
-            assertThat(e.getCause().getCause().getMessage(), Matchers.equalTo("Sort must contain at least one field."));
+            assertTrue(e.shardFailures().length > 0);
+            for (ShardSearchFailure failure : e.shardFailures()) {
+                assertThat(failure.getCause().getClass(), Matchers.equalTo(IllegalArgumentException.class));
+                assertThat(failure.getCause().getMessage(), Matchers.equalTo("Sort must contain at least one field."));
+            }
         }
 
         try {
@@ -106,9 +112,11 @@ public void testsShouldFail() throws Exception {
                     .get();
             fail("Should fail on search_after size differs from sort field size");
         } catch (SearchPhaseExecutionException e) {
-            assertThat(e.getCause().getClass(), Matchers.equalTo(RemoteTransportException.class));
-            assertThat(e.getCause().getCause().getClass(), Matchers.equalTo(IllegalArgumentException.class));
-            assertThat(e.getCause().getCause().getMessage(), Matchers.equalTo("search_after has 1 value(s) but sort has 2."));
+            assertTrue(e.shardFailures().length > 0);
+            for (ShardSearchFailure failure : e.shardFailures()) {
+                assertThat(failure.getCause().getClass(), Matchers.equalTo(IllegalArgumentException.class));
+                assertThat(failure.getCause().getMessage(), Matchers.equalTo("search_after has 1 value(s) but sort has 2."));
+            }
         }
 
         try {
@@ -119,9 +127,11 @@ public void testsShouldFail() throws Exception {
                     .get();
             fail("Should fail on search_after size differs from sort field size");
         } catch (SearchPhaseExecutionException e) {
-            assertThat(e.getCause().getClass(), Matchers.equalTo(RemoteTransportException.class));
-            assertThat(e.getCause().getCause().getClass(), Matchers.equalTo(IllegalArgumentException.class));
-            assertThat(e.getCause().getCause().getMessage(), Matchers.equalTo("search_after has 2 value(s) but sort has 1."));
+            for (ShardSearchFailure failure : e.shardFailures()) {
+                assertTrue(e.shardFailures().length > 0);
+                assertThat(failure.getCause().getClass(), Matchers.equalTo(IllegalArgumentException.class));
+                assertThat(failure.getCause().getMessage(), Matchers.equalTo("search_after has 2 value(s) but sort has 1."));
+            }
         }
 
         try {
@@ -133,9 +143,11 @@ public void testsShouldFail() throws Exception {
 
             fail("Should fail on search_after on score only is disabled");
         } catch (SearchPhaseExecutionException e) {
-            assertThat(e.getCause().getClass(), Matchers.equalTo(RemoteTransportException.class));
-            assertThat(e.getCause().getCause().getClass(), Matchers.equalTo(IllegalArgumentException.class));
-            assertThat(e.getCause().getCause().getMessage(), Matchers.equalTo("Failed to parse search_after value for field [field1]."));
+            assertTrue(e.shardFailures().length > 0);
+            for (ShardSearchFailure failure : e.shardFailures()) {
+                assertThat(failure.getCause().getClass(), Matchers.equalTo(IllegalArgumentException.class));
+                assertThat(failure.getCause().getMessage(), Matchers.equalTo("Failed to parse search_after value for field [field1]."));
+            }
         }
     }
 
@@ -188,7 +200,7 @@ public void testWithSimpleTypes() throws Exception {
                         values.add(randomDouble());
                         break;
                     case 6:
-                        values.add(randomAsciiOfLengthBetween(5, 20));
+                        values.add(randomAlphaOfLengthBetween(5, 20));
                         break;
                 }
             }
diff --git a/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java b/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java
index 9597899104652..f63f13b6dd24b 100644
--- a/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java
+++ b/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.rest.RestStatus;
@@ -52,6 +53,7 @@
 import static org.hamcrest.Matchers.containsString;
 
 public class SimpleSearchIT extends ESIntegTestCase {
+
     public void testSearchNullIndex() {
         expectThrows(NullPointerException.class,
                 () -> client().prepareSearch((String) null).setQuery(QueryBuilders.termQuery("_id", "XXX1")).get());
@@ -288,7 +290,7 @@ public void testSimpleTerminateAfterCount() throws Exception {
 
     public void testInsaneFromAndSize() throws Exception {
         createIndex("idx");
-        indexRandom(true, client().prepareIndex("idx", "type").setSource("{}"));
+        indexRandom(true, client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON));
 
         assertWindowFails(client().prepareSearch("idx").setFrom(Integer.MAX_VALUE));
         assertWindowFails(client().prepareSearch("idx").setSize(Integer.MAX_VALUE));
@@ -296,7 +298,7 @@ public void testInsaneFromAndSize() throws Exception {
 
     public void testTooLargeFromAndSize() throws Exception {
         createIndex("idx");
-        indexRandom(true, client().prepareIndex("idx", "type").setSource("{}"));
+        indexRandom(true, client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON));
 
         assertWindowFails(client().prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)));
         assertWindowFails(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) + 1));
@@ -306,7 +308,7 @@ public void testTooLargeFromAndSize() throws Exception {
 
     public void testLargeFromAndSizeSucceeds() throws Exception {
         createIndex("idx");
-        indexRandom(true, client().prepareIndex("idx", "type").setSource("{}"));
+        indexRandom(true, client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON));
 
         assertHitCount(client().prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) - 10).get(), 1);
         assertHitCount(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)).get(), 1);
@@ -317,7 +319,7 @@ public void testLargeFromAndSizeSucceeds() throws Exception {
     public void testTooLargeFromAndSizeOkBySetting() throws Exception {
         prepareCreate("idx").setSettings(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(),
                 IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 2).get();
-        indexRandom(true, client().prepareIndex("idx", "type").setSource("{}"));
+        indexRandom(true, client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON));
 
         assertHitCount(client().prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)).get(), 1);
         assertHitCount(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) + 1).get(), 1);
@@ -332,7 +334,7 @@ public void testTooLargeFromAndSizeOkByDynamicSetting() throws Exception {
                         Settings.builder().put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(),
                                 IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 2))
                 .get());
-        indexRandom(true, client().prepareIndex("idx", "type").setSource("{}"));
+        indexRandom(true, client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON));
 
         assertHitCount(client().prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)).get(), 1);
         assertHitCount(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) + 1).get(), 1);
@@ -342,7 +344,7 @@ public void testTooLargeFromAndSizeOkByDynamicSetting() throws Exception {
 
     public void testTooLargeFromAndSizeBackwardsCompatibilityRecommendation() throws Exception {
         prepareCreate("idx").setSettings(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), Integer.MAX_VALUE).get();
-        indexRandom(true, client().prepareIndex("idx", "type").setSource("{}"));
+        indexRandom(true, client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON));
 
         assertHitCount(client().prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 10).get(), 1);
         assertHitCount(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 10).get(), 1);
@@ -352,7 +354,7 @@ public void testTooLargeFromAndSizeBackwardsCompatibilityRecommendation() throws
 
     public void testTooLargeRescoreWindow() throws Exception {
         createIndex("idx");
-        indexRandom(true, client().prepareIndex("idx", "type").setSource("{}"));
+        indexRandom(true, client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON));
 
         assertRescoreWindowFails(Integer.MAX_VALUE);
         assertRescoreWindowFails(IndexSettings.MAX_RESCORE_WINDOW_SETTING.get(Settings.EMPTY) + 1);
@@ -362,7 +364,7 @@ public void testTooLargeRescoreOkBySetting() throws Exception {
         int defaultMaxWindow = IndexSettings.MAX_RESCORE_WINDOW_SETTING.get(Settings.EMPTY);
         prepareCreate("idx").setSettings(IndexSettings.MAX_RESCORE_WINDOW_SETTING.getKey(),
                 defaultMaxWindow * 2).get();
-        indexRandom(true, client().prepareIndex("idx", "type").setSource("{}"));
+        indexRandom(true, client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON));
 
         assertHitCount(
                 client().prepareSearch("idx").addRescorer(new QueryRescorerBuilder(matchAllQuery()).windowSize(defaultMaxWindow + 1)).get(),
@@ -373,7 +375,7 @@ public void testTooLargeRescoreOkByResultWindowSetting() throws Exception {
         int defaultMaxWindow = IndexSettings.MAX_RESCORE_WINDOW_SETTING.get(Settings.EMPTY);
         prepareCreate("idx").setSettings(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), // Note that this is the RESULT window.
                 defaultMaxWindow * 2).get();
-        indexRandom(true, client().prepareIndex("idx", "type").setSource("{}"));
+        indexRandom(true, client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON));
 
         assertHitCount(
                 client().prepareSearch("idx").addRescorer(new QueryRescorerBuilder(matchAllQuery()).windowSize(defaultMaxWindow + 1)).get(),
@@ -387,7 +389,7 @@ public void testTooLargeRescoreOkByDynamicSetting() throws Exception {
                 .setSettings(
                         Settings.builder().put(IndexSettings.MAX_RESCORE_WINDOW_SETTING.getKey(), defaultMaxWindow * 2))
                 .get());
-        indexRandom(true, client().prepareIndex("idx", "type").setSource("{}"));
+        indexRandom(true, client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON));
 
         assertHitCount(
                 client().prepareSearch("idx").addRescorer(new QueryRescorerBuilder(matchAllQuery()).windowSize(defaultMaxWindow + 1)).get(),
@@ -402,7 +404,7 @@ public void testTooLargeRescoreOkByDynamicResultWindowSetting() throws Exception
                         // Note that this is the RESULT window
                         Settings.builder().put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), defaultMaxWindow * 2))
                 .get());
-        indexRandom(true, client().prepareIndex("idx", "type").setSource("{}"));
+        indexRandom(true, client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON));
 
         assertHitCount(
                 client().prepareSearch("idx").addRescorer(new QueryRescorerBuilder(matchAllQuery()).windowSize(defaultMaxWindow + 1)).get(),
@@ -417,7 +419,8 @@ public void testQueryNumericFieldWithRegex() throws Exception {
             client().prepareSearch("idx").setQuery(QueryBuilders.regexpQuery("num", "34")).get();
             fail("SearchPhaseExecutionException should have been thrown");
         } catch (SearchPhaseExecutionException ex) {
-            assertThat(ex.getCause().getCause().getMessage(), containsString("Can only use regexp queries on keyword and text fields"));
+            assertThat(ex.getRootCause().getMessage(),
+                containsString("Can only use regexp queries on keyword and text fields"));
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/search/slice/SearchSliceIT.java b/core/src/test/java/org/elasticsearch/search/slice/SearchSliceIT.java
index 4a27dcb596470..9da652045fb24 100644
--- a/core/src/test/java/org/elasticsearch/search/slice/SearchSliceIT.java
+++ b/core/src/test/java/org/elasticsearch/search/slice/SearchSliceIT.java
@@ -82,7 +82,7 @@ private int setupIndex(boolean withDocs) throws IOException, ExecutionException,
         for (int i = 0; i < NUM_DOCS; i++) {
             XContentBuilder builder = jsonBuilder();
             builder.startObject();
-            builder.field("invalid_random_kw", randomAsciiOfLengthBetween(5, 20));
+            builder.field("invalid_random_kw", randomAlphaOfLengthBetween(5, 20));
             builder.field("random_int", randomInt());
             builder.field("static_int", 0);
             builder.field("invalid_random_int", randomInt());
diff --git a/core/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java b/core/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java
index 695c926976b05..33c4f502b7033 100644
--- a/core/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java
@@ -25,79 +25,63 @@
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.RAMDirectory;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.UidFieldMapper;
-import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.index.query.QueryParser;
 import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.test.ESTestCase;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
 
 import java.io.IOException;
-import java.util.List;
 import java.util.ArrayList;
-import java.util.Map;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.containsString;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
 public class SliceBuilderTests extends ESTestCase {
     private static final int MAX_SLICE = 20;
-    private static IndicesQueriesRegistry indicesQueriesRegistry;
-
-    /**
-     * setup for the whole base test class
-     */
-    @BeforeClass
-    public static void init() {
-        indicesQueriesRegistry = new IndicesQueriesRegistry();
-        QueryParser<MatchAllQueryBuilder> parser = MatchAllQueryBuilder::fromXContent;
-        indicesQueriesRegistry.register(parser, MatchAllQueryBuilder.NAME);
-    }
-
-    @AfterClass
-    public static void afterClass() throws Exception {
-        indicesQueriesRegistry = null;
-    }
 
-    private SliceBuilder randomSliceBuilder() throws IOException {
+    private static SliceBuilder randomSliceBuilder() throws IOException {
         int max = randomIntBetween(2, MAX_SLICE);
-        int id = randomInt(max - 1);
-        String field = randomAsciiOfLengthBetween(5, 20);
+        int id = randomIntBetween(1, max - 1);
+        String field = randomAlphaOfLengthBetween(5, 20);
         return new SliceBuilder(field, id, max);
     }
 
     private static SliceBuilder serializedCopy(SliceBuilder original) throws IOException {
-        try (BytesStreamOutput output = new BytesStreamOutput()) {
-            original.writeTo(output);
-            try (StreamInput in = output.bytes().streamInput()) {
-                return new SliceBuilder(in);
-            }
+        return copyWriteable(original, new NamedWriteableRegistry(Collections.emptyList()), SliceBuilder::new);
+    }
+
+    private static SliceBuilder mutate(SliceBuilder original) throws IOException {
+        switch (randomIntBetween(0, 2)) {
+            case 0: return new SliceBuilder(original.getField() + "_xyz", original.getId(), original.getMax());
+            case 1: return new SliceBuilder(original.getField(), original.getId() - 1, original.getMax());
+            case 2:
+            default: return new SliceBuilder(original.getField(), original.getId(), original.getMax() + 1);
         }
     }
 
@@ -110,29 +94,7 @@ public void testSerialization() throws Exception {
     }
 
     public void testEqualsAndHashcode() throws Exception {
-        SliceBuilder firstBuilder = randomSliceBuilder();
-        assertFalse("sliceBuilder is equal to null", firstBuilder.equals(null));
-        assertFalse("sliceBuilder is equal to incompatible type", firstBuilder.equals(""));
-        assertTrue("sliceBuilder is not equal to self", firstBuilder.equals(firstBuilder));
-        assertThat("same searchFrom's hashcode returns different values if called multiple times",
-            firstBuilder.hashCode(), equalTo(firstBuilder.hashCode()));
-
-        SliceBuilder secondBuilder = serializedCopy(firstBuilder);
-        assertTrue("sliceBuilder is not equal to self", secondBuilder.equals(secondBuilder));
-        assertTrue("sliceBuilder is not equal to its copy", firstBuilder.equals(secondBuilder));
-        assertTrue("equals is not symmetric", secondBuilder.equals(firstBuilder));
-        assertThat("sliceBuilder copy's hashcode is different from original hashcode", secondBuilder.hashCode(),
-            equalTo(firstBuilder.hashCode()));
-        SliceBuilder thirdBuilder = serializedCopy(secondBuilder);
-        assertTrue("sliceBuilder is not equal to self", thirdBuilder.equals(thirdBuilder));
-        assertTrue("sliceBuilder is not equal to its copy", secondBuilder.equals(thirdBuilder));
-        assertThat("sliceBuilder copy's hashcode is different from original hashcode", secondBuilder.hashCode(),
-            equalTo(thirdBuilder.hashCode()));
-        assertTrue("equals is not transitive", firstBuilder.equals(thirdBuilder));
-        assertThat("sliceBuilder copy's hashcode is different from original hashcode", firstBuilder.hashCode(),
-            equalTo(thirdBuilder.hashCode()));
-        assertTrue("sliceBuilder is not symmetric", thirdBuilder.equals(secondBuilder));
-        assertTrue("sliceBuilder is not symmetric", thirdBuilder.equals(firstBuilder));
+        checkEqualsAndHashCode(randomSliceBuilder(), SliceBuilderTests::serializedCopy, SliceBuilderTests::mutate);
     }
 
     public void testFromXContent() throws Exception {
@@ -144,9 +106,8 @@ public void testFromXContent() throws Exception {
         builder.startObject();
         sliceBuilder.innerToXContent(builder);
         builder.endObject();
-        XContentParser parser = XContentHelper.createParser(shuffleXContent(builder).bytes());
-        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser,
-            ParseFieldMatcher.STRICT);
+        XContentParser parser = createParser(shuffleXContent(builder));
+        QueryParseContext context = new QueryParseContext(parser);
         SliceBuilder secondSliceBuilder = SliceBuilder.fromXContent(context);
         assertNotSame(sliceBuilder, secondSliceBuilder);
         assertEquals(sliceBuilder, secondSliceBuilder);
@@ -199,6 +160,14 @@ public Query termQuery(Object value, @Nullable QueryShardContext context) {
             fieldType.setHasDocValues(false);
             when(context.fieldMapper(UidFieldMapper.NAME)).thenReturn(fieldType);
             when(context.getIndexReader()).thenReturn(reader);
+            Settings settings = Settings.builder()
+                    .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                    .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 2)
+                    .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+                    .build();
+            IndexMetaData indexState = IndexMetaData.builder("index").settings(settings).build();
+            IndexSettings indexSettings = new IndexSettings(indexState, Settings.EMPTY);
+            when(context.getIndexSettings()).thenReturn(indexSettings);
             SliceBuilder builder = new SliceBuilder(5, 10);
             Query query = builder.toFilter(context, 0, 1);
             assertThat(query, instanceOf(TermsSliceQuery.class));
diff --git a/core/src/test/java/org/elasticsearch/search/slice/TermsSliceQueryTests.java b/core/src/test/java/org/elasticsearch/search/slice/TermsSliceQueryTests.java
index e00dabc63638f..3fa4ce410529a 100644
--- a/core/src/test/java/org/elasticsearch/search/slice/TermsSliceQueryTests.java
+++ b/core/src/test/java/org/elasticsearch/search/slice/TermsSliceQueryTests.java
@@ -34,6 +34,7 @@
 
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.StringHelper;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.test.ESTestCase;
 
@@ -72,7 +73,8 @@ public void testSearch() throws Exception {
             Document doc = new Document();
             String uuid = UUIDs.base64UUID();
             BytesRef br = new BytesRef(uuid);
-            int id = Math.floorMod(br.hashCode(), max);
+            int hashCode = StringHelper.murmurhash3_x86_32(br, TermsSliceQuery.SEED);
+            int id = Math.floorMod(hashCode, max);
             sliceCounters[id] ++;
             doc.add(new StringField("uuid", uuid, Field.Store.YES));
             w.addDocument(doc);
diff --git a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
index 84f83c46e1a22..960b16b6c26b7 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
@@ -23,17 +23,12 @@
 import org.apache.lucene.util.Accountable;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.env.Environment;
@@ -42,10 +37,10 @@
 import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
 import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.mapper.ContentPath;
-import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.mapper.ObjectMapper;
 import org.elasticsearch.index.mapper.LegacyDoubleFieldMapper.DoubleFieldType;
+import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper.BuilderContext;
+import org.elasticsearch.index.mapper.ObjectMapper;
 import org.elasticsearch.index.mapper.ObjectMapper.Nested;
 import org.elasticsearch.index.query.IdsQueryBuilder;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
@@ -55,7 +50,6 @@
 import org.elasticsearch.index.query.TermQueryBuilder;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptContext;
@@ -64,6 +58,7 @@
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.ScriptServiceTests.TestEngineService;
 import org.elasticsearch.script.ScriptSettings;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.test.ESTestCase;
@@ -75,18 +70,16 @@
 import java.io.IOException;
 import java.nio.file.Path;
 import java.util.Collections;
-import java.util.Map;
 
 import static java.util.Collections.emptyList;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.not;
+import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
 
 public abstract class AbstractSortTestCase<T extends SortBuilder<T>> extends ESTestCase {
+    private static final int NUMBER_OF_TESTBUILDERS = 20;
 
     protected static NamedWriteableRegistry namedWriteableRegistry;
 
-    private static final int NUMBER_OF_TESTBUILDERS = 20;
-    static IndicesQueriesRegistry indicesQueriesRegistry;
+    private static NamedXContentRegistry xContentRegistry;
     private static ScriptService scriptService;
 
     @BeforeClass
@@ -103,20 +96,20 @@ public static void init() throws IOException {
         scriptService = new ScriptService(baseSettings, environment,
                 new ResourceWatcherService(baseSettings, null), scriptEngineRegistry, scriptContextRegistry, scriptSettings) {
             @Override
-            public CompiledScript compile(Script script, ScriptContext scriptContext, Map<String, String> params) {
+            public CompiledScript compile(Script script, ScriptContext scriptContext) {
                 return new CompiledScript(ScriptType.INLINE, "mockName", "test", script);
             }
         };
 
         SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList());
         namedWriteableRegistry = new NamedWriteableRegistry(searchModule.getNamedWriteables());
-        indicesQueriesRegistry = searchModule.getQueryParserRegistry();
+        xContentRegistry = new NamedXContentRegistry(searchModule.getNamedXContents());
     }
 
     @AfterClass
     public static void afterClass() throws Exception {
         namedWriteableRegistry = null;
-        indicesQueriesRegistry = null;
+        xContentRegistry = null;
     }
 
     /** Returns random sort that is put under test */
@@ -141,7 +134,7 @@ public void testFromXContent() throws IOException {
             }
             testItem.toXContent(builder, ToXContent.EMPTY_PARAMS);
             XContentBuilder shuffled = shuffleXContent(builder);
-            XContentParser itemParser = XContentHelper.createParser(shuffled.bytes());
+            XContentParser itemParser = createParser(shuffled);
             itemParser.nextToken();
 
             /*
@@ -151,7 +144,7 @@ public void testFromXContent() throws IOException {
             String elementName = itemParser.currentName();
             itemParser.nextToken();
 
-            QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, itemParser, ParseFieldMatcher.STRICT);
+            QueryParseContext context = new QueryParseContext(itemParser);
             T parsedItem = fromXContent(context, elementName);
             assertNotSame(testItem, parsedItem);
             assertEquals(testItem, parsedItem);
@@ -180,7 +173,7 @@ public void testBuildSortField() throws IOException {
     public void testSerialization() throws IOException {
         for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) {
             T testsort = createTestItem();
-            T deserializedsort = copyItem(testsort);
+            T deserializedsort = copy(testsort);
             assertEquals(testsort, deserializedsort);
             assertEquals(testsort.hashCode(), deserializedsort.hashCode());
             assertNotSame(testsort, deserializedsort);
@@ -192,34 +185,12 @@ public void testSerialization() throws IOException {
      */
     public void testEqualsAndHashcode() throws IOException {
         for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) {
-            T firstsort = createTestItem();
-            assertFalse("sort is equal to null", firstsort.equals(null));
-            assertFalse("sort is equal to incompatible type", firstsort.equals(""));
-            assertTrue("sort is not equal to self", firstsort.equals(firstsort));
-            assertThat("same sort's hashcode returns different values if called multiple times", firstsort.hashCode(),
-                    equalTo(firstsort.hashCode()));
-            assertThat("different sorts should not be equal", mutate(firstsort), not(equalTo(firstsort)));
-            assertThat("different sorts should have different hashcode", mutate(firstsort).hashCode(), not(equalTo(firstsort.hashCode())));
-
-            T secondsort = copyItem(firstsort);
-            assertTrue("sort is not equal to self", secondsort.equals(secondsort));
-            assertTrue("sort is not equal to its copy", firstsort.equals(secondsort));
-            assertTrue("equals is not symmetric", secondsort.equals(firstsort));
-            assertThat("sort copy's hashcode is different from original hashcode", secondsort.hashCode(), equalTo(firstsort.hashCode()));
-
-            T thirdsort = copyItem(secondsort);
-            assertTrue("sort is not equal to self", thirdsort.equals(thirdsort));
-            assertTrue("sort is not equal to its copy", secondsort.equals(thirdsort));
-            assertThat("sort copy's hashcode is different from original hashcode", secondsort.hashCode(), equalTo(thirdsort.hashCode()));
-            assertTrue("equals is not transitive", firstsort.equals(thirdsort));
-            assertThat("sort copy's hashcode is different from original hashcode", firstsort.hashCode(), equalTo(thirdsort.hashCode()));
-            assertTrue("equals is not symmetric", thirdsort.equals(secondsort));
-            assertTrue("equals is not symmetric", thirdsort.equals(firstsort));
+            checkEqualsAndHashCode(createTestItem(), this::copy, this::mutate);
         }
     }
 
     protected QueryShardContext createMockShardContext() {
-        Index index = new Index(randomAsciiOfLengthBetween(1, 10), "_na_");
+        Index index = new Index(randomAlphaOfLengthBetween(1, 10), "_na_");
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(index,
             Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build());
         IndicesFieldDataCache cache = new IndicesFieldDataCache(Settings.EMPTY, null);
@@ -235,8 +206,9 @@ public void onRemoval(ShardId shardId, Accountable accountable) {
             public void onCache(ShardId shardId, Accountable accountable) {
             }
         });
-        return new QueryShardContext(idxSettings, bitsetFilterCache, ifds, null, null, scriptService,
-                indicesQueriesRegistry, null, null, null) {
+        long nowInMillis = randomNonNegativeLong();
+        return new QueryShardContext(0, idxSettings, bitsetFilterCache, ifds, null, null, scriptService,
+                xContentRegistry(), null, null, () -> nowInMillis) {
             @Override
             public MappedFieldType fieldMapper(String name) {
                 return provideMappedFieldType(name);
@@ -245,7 +217,7 @@ public MappedFieldType fieldMapper(String name) {
             @Override
             public ObjectMapper getObjectMapper(String name) {
                 BuilderContext context = new BuilderContext(this.getIndexSettings().getSettings(), new ContentPath());
-                return (ObjectMapper) new ObjectMapper.Builder<>(name).nested(Nested.newNested(false, false)).build(context);
+                return new ObjectMapper.Builder<>(name).nested(Nested.newNested(false, false)).build(context);
             }
         };
     }
@@ -261,25 +233,28 @@ protected MappedFieldType provideMappedFieldType(String name) {
         return doubleFieldType;
     }
 
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return xContentRegistry;
+    }
+
     protected static QueryBuilder randomNestedFilter() {
         int id = randomIntBetween(0, 2);
         switch(id) {
             case 0: return (new MatchAllQueryBuilder()).boost(randomFloat());
             case 1: return (new IdsQueryBuilder()).boost(randomFloat());
             case 2: return (new TermQueryBuilder(
-                    randomAsciiOfLengthBetween(1, 10),
+                    randomAlphaOfLengthBetween(1, 10),
                     randomDouble()).boost(randomFloat()));
             default: throw new IllegalStateException("Only three query builders supported for testing sort");
         }
     }
 
     @SuppressWarnings("unchecked")
-    private T copyItem(T original) throws IOException {
-        try (BytesStreamOutput output = new BytesStreamOutput()) {
-            original.writeTo(output);
-            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
-                return (T) namedWriteableRegistry.getReader(SortBuilder.class, original.getWriteableName()).read(in);
-            }
-        }
+    private T copy(T original) throws IOException {
+        /* The cast below is required to make Java 9 happy. Java 8 infers the T in copyWriterable to be the same as AbstractSortTestCase's
+         * T but Java 9 infers it to be SortBuilder. */
+        return (T) copyWriteable(original, namedWriteableRegistry,
+                namedWriteableRegistry.getReader(SortBuilder.class, original.getWriteableName()));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java
index baaf3ac5d3c8f..57e79ce98c379 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java
@@ -20,10 +20,8 @@
 package org.elasticsearch.search.sort;
 
 import org.apache.lucene.search.SortField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.DocValueFormat;
 
@@ -41,12 +39,12 @@ protected FieldSortBuilder createTestItem() {
     private List<Object> missingContent = Arrays.asList(
             "_last",
             "_first",
-            randomAsciiOfLength(10), randomUnicodeOfCodepointLengthBetween(5, 15),
+            Integer.toString(randomInt()),
             randomInt());
 
 
     public FieldSortBuilder randomFieldSortBuilder() {
-        String fieldName = rarely() ? FieldSortBuilder.DOC_FIELD_NAME : randomAsciiOfLengthBetween(1, 10);
+        String fieldName = rarely() ? FieldSortBuilder.DOC_FIELD_NAME : randomAlphaOfLengthBetween(1, 10);
         FieldSortBuilder builder = new FieldSortBuilder(fieldName);
         if (randomBoolean()) {
             builder.order(randomFrom(SortOrder.values()));
@@ -57,7 +55,7 @@ public FieldSortBuilder randomFieldSortBuilder() {
         }
 
         if (randomBoolean()) {
-            builder.unmappedType(randomAsciiOfLengthBetween(1, 10));
+            builder.unmappedType(randomAlphaOfLengthBetween(1, 10));
         }
 
         if (randomBoolean()) {
@@ -69,7 +67,7 @@ public FieldSortBuilder randomFieldSortBuilder() {
         }
 
         if (randomBoolean()) {
-            builder.setNestedPath(randomAsciiOfLengthBetween(1, 10));
+            builder.setNestedPath(randomAlphaOfLengthBetween(1, 10));
         }
 
         return builder;
@@ -83,7 +81,7 @@ protected FieldSortBuilder mutate(FieldSortBuilder original) throws IOException
         case 0:
             mutated.setNestedPath(randomValueOtherThan(
                     original.getNestedPath(),
-                    () -> randomAsciiOfLengthBetween(1, 10)));
+                    () -> randomAlphaOfLengthBetween(1, 10)));
             break;
         case 1:
             mutated.setNestedFilter(randomValueOtherThan(
@@ -96,7 +94,7 @@ protected FieldSortBuilder mutate(FieldSortBuilder original) throws IOException
         case 3:
             mutated.unmappedType(randomValueOtherThan(
                     original.unmappedType(),
-                    () -> randomAsciiOfLengthBetween(1, 10)));
+                    () -> randomAlphaOfLengthBetween(1, 10)));
             break;
         case 4:
             mutated.missing(randomValueOtherThan(original.missing(), () -> randomFrom(missingContent)));
@@ -129,18 +127,18 @@ protected void sortFieldAssertions(FieldSortBuilder builder, SortField sortField
     public void testReverseOptionFails() throws IOException {
         String json = "{ \"post_date\" : {\"reverse\" : true} },\n";
 
-        XContentParser parser = XContentFactory.xContent(json).createParser(json);
+        XContentParser parser = createParser(JsonXContent.jsonXContent, json);
         // need to skip until parser is located on second START_OBJECT
         parser.nextToken();
         parser.nextToken();
         parser.nextToken();
 
-        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.STRICT);
+        QueryParseContext context = new QueryParseContext(parser);
 
         try {
           FieldSortBuilder.fromXContent(context, "");
           fail("adding reverse sorting option should fail with an exception");
-        } catch (ParsingException e) {
+        } catch (IllegalArgumentException e) {
             // all good
         }
     }
diff --git a/core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java b/core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java
index 78c77e15f3868..fd25b8d662a60 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java
@@ -23,14 +23,17 @@
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.TestUtil;
 import org.apache.lucene.util.UnicodeUtil;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders;
@@ -38,6 +41,7 @@
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
+import org.elasticsearch.test.VersionUtils;
 import org.hamcrest.Matchers;
 
 import java.io.IOException;
@@ -91,7 +95,7 @@ public void testIssue8226() {
                 assertAcked(prepareCreate("test_" + i).addAlias(new Alias("test")));
             }
             if (i > 0) {
-                client().prepareIndex("test_" + i, "foo", "" + i).setSource("{\"entry\": " + i + "}").get();
+                client().prepareIndex("test_" + i, "foo", "" + i).setSource("{\"entry\": " + i + "}", XContentType.JSON).get();
             }
         }
         refresh();
@@ -122,11 +126,11 @@ public void testIssue8226() {
         }
     }
 
-    @LuceneTestCase.BadApple(bugUrl = "simon is working on this")
     public void testIssue6614() throws ExecutionException, InterruptedException {
         List<IndexRequestBuilder> builders = new ArrayList<>();
         boolean strictTimeBasedIndices = randomBoolean();
         final int numIndices = randomIntBetween(2, 25); // at most 25 days in the month
+        int docs = 0;
         for (int i = 0; i < numIndices; i++) {
           final String indexId = strictTimeBasedIndices ? "idx_" + i : "idx";
           if (strictTimeBasedIndices || i == 0) {
@@ -142,9 +146,10 @@ public void testIssue6614() throws ExecutionException, InterruptedException {
                                     String.format(Locale.ROOT, "%02d", j+1) +
                                     ":00:00"));
           }
+            indexRandom(true, builders);
+            docs += builders.size();
+            builders.clear();
         }
-        int docs = builders.size();
-        indexRandom(true, builders);
         SearchResponse allDocsResponse = client().prepareSearch().setQuery(
                 QueryBuilders.boolQuery().must(QueryBuilders.termQuery("foo", "bar")).must(
                         QueryBuilders.rangeQuery("timeUpdated").gte("2014/0" + randomIntBetween(1, 7) + "/01")))
@@ -421,9 +426,9 @@ public void testIssue2986() {
         assertAcked(client().admin().indices().prepareCreate("test")
                 .addMapping("type", "field1", "type=keyword").get());
 
-        client().prepareIndex("test", "post", "1").setSource("{\"field1\":\"value1\"}").execute().actionGet();
-        client().prepareIndex("test", "post", "2").setSource("{\"field1\":\"value2\"}").execute().actionGet();
-        client().prepareIndex("test", "post", "3").setSource("{\"field1\":\"value3\"}").execute().actionGet();
+        client().prepareIndex("test", "post", "1").setSource("{\"field1\":\"value1\"}", XContentType.JSON).execute().actionGet();
+        client().prepareIndex("test", "post", "2").setSource("{\"field1\":\"value2\"}", XContentType.JSON).execute().actionGet();
+        client().prepareIndex("test", "post", "3").setSource("{\"field1\":\"value3\"}", XContentType.JSON).execute().actionGet();
         refresh();
         SearchResponse result = client().prepareSearch("test").setQuery(matchAllQuery()).setTrackScores(true)
                 .addSort("field1", SortOrder.ASC).execute().actionGet();
@@ -855,7 +860,18 @@ public void testSortMissingStrings() throws IOException {
     }
 
     public void testIgnoreUnmapped() throws Exception {
-        createIndex("test");
+        assertIgnoreUnmapped(Version.CURRENT);
+    }
+
+    public void testIgnoreUnmappedBWC() throws Exception{
+        // test BWC with indices created in 2.x
+        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.V_2_4_6);
+        assertIgnoreUnmapped(version);
+    }
+
+    private void assertIgnoreUnmapped(Version version) throws IOException {
+        prepareCreate("test")
+            .setSettings(Settings.builder().put(indexSettings()).put("index.version.created", version).build()).get();
 
         client().prepareIndex("test", "type1", "1").setSource(jsonBuilder().startObject()
                 .field("id", "1")
diff --git a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceIT.java b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceIT.java
index 95c20381cfded..558a934abfd85 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceIT.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceIT.java
@@ -67,8 +67,8 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
         return Arrays.asList(InternalSettingsPlugin.class);
     }
 
-    public void testSimpleDistance() throws Exception {
-        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT);
+    public void testLegacyGeoDistanceRangeQuery() throws Exception {
+        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.V_2_4_0);
         Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
         XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties")
                 .startObject("location").field("type", "geo_point");
@@ -230,7 +230,7 @@ public void testDistanceSortingMVFields() throws Exception {
                 .actionGet();
 
         client().prepareIndex("test", "type1", "3")
-                .setSource(jsonBuilder().startObject().field("names", "Times Square", "Tribeca").startArray("locations")
+                .setSource(jsonBuilder().startObject().array("names", "Times Square", "Tribeca").startArray("locations")
                         // to NY: 5.286 km
                         .startObject().field("lat", 40.759011).field("lon", -73.9844722).endObject()
                         // to NY: 0.4621 km
@@ -238,7 +238,7 @@ public void testDistanceSortingMVFields() throws Exception {
                 .execute().actionGet();
 
         client().prepareIndex("test", "type1", "4")
-                .setSource(jsonBuilder().startObject().field("names", "Wall Street", "Soho").startArray("locations")
+                .setSource(jsonBuilder().startObject().array("names", "Wall Street", "Soho").startArray("locations")
                         // to NY: 1.055 km
                         .startObject().field("lat", 40.7051157).field("lon", -74.0088305).endObject()
                         // to NY: 1.258 km
@@ -246,7 +246,7 @@ public void testDistanceSortingMVFields() throws Exception {
                 .execute().actionGet();
 
         client().prepareIndex("test", "type1", "5")
-                .setSource(jsonBuilder().startObject().field("names", "Greenwich Village", "Brooklyn").startArray("locations")
+                .setSource(jsonBuilder().startObject().array("names", "Greenwich Village", "Brooklyn").startArray("locations")
                         // to NY: 2.029 km
                         .startObject().field("lat", 40.731033).field("lon", -73.9962255).endObject()
                         // to NY: 8.572 km
@@ -355,14 +355,14 @@ public void testDistanceSortingWithMissingGeoPoint() throws Exception {
         ensureGreen();
 
         client().prepareIndex("test", "type1", "1")
-                .setSource(jsonBuilder().startObject().field("names", "Times Square", "Tribeca").startArray("locations")
+                .setSource(jsonBuilder().startObject().array("names", "Times Square", "Tribeca").startArray("locations")
                         // to NY: 5.286 km
                         .startObject().field("lat", 40.759011).field("lon", -73.9844722).endObject()
                         // to NY: 0.4621 km
                         .startObject().field("lat", 40.718266).field("lon", -74.007819).endObject().endArray().endObject())
                 .execute().actionGet();
 
-        client().prepareIndex("test", "type1", "2").setSource(jsonBuilder().startObject().field("names", "Wall Street", "Soho").endObject())
+        client().prepareIndex("test", "type1", "2").setSource(jsonBuilder().startObject().array("names", "Wall Street", "Soho").endObject())
                 .execute().actionGet();
 
         refresh();
@@ -375,7 +375,7 @@ public void testDistanceSortingWithMissingGeoPoint() throws Exception {
         assertHitCount(searchResponse, 2);
         assertOrderedSearchHits(searchResponse, "1", "2");
         assertThat(((Number) searchResponse.getHits().getAt(0).sortValues()[0]).doubleValue(), closeTo(462.1d, 10d));
-        assertThat(((Number) searchResponse.getHits().getAt(1).sortValues()[0]).doubleValue(), equalTo(Double.MAX_VALUE));
+        assertThat(((Number) searchResponse.getHits().getAt(1).sortValues()[0]).doubleValue(), equalTo(Double.POSITIVE_INFINITY));
 
         // Order: Desc
         searchResponse = client().prepareSearch("test").setQuery(matchAllQuery())
@@ -385,7 +385,7 @@ public void testDistanceSortingWithMissingGeoPoint() throws Exception {
         // Doc with missing geo point is first, is consistent with 0.20.x
         assertHitCount(searchResponse, 2);
         assertOrderedSearchHits(searchResponse, "2", "1");
-        assertThat(((Number) searchResponse.getHits().getAt(0).sortValues()[0]).doubleValue(), equalTo(Double.MAX_VALUE));
+        assertThat(((Number) searchResponse.getHits().getAt(0).sortValues()[0]).doubleValue(), equalTo(Double.POSITIVE_INFINITY));
         assertThat(((Number) searchResponse.getHits().getAt(1).sortValues()[0]).doubleValue(), closeTo(5286d, 10d));
     }
 
@@ -523,9 +523,9 @@ public void testDistanceSortingNestedFields() throws Exception {
         assertFirstHit(searchResponse, hasId("4"));
         assertSearchHits(searchResponse, "1", "2", "3", "4");
         assertThat(((Number) searchResponse.getHits().getAt(0).sortValues()[0]).doubleValue(), closeTo(8572.0d, 10d));
-        assertThat(((Number) searchResponse.getHits().getAt(1).sortValues()[0]).doubleValue(), equalTo(Double.MAX_VALUE));
-        assertThat(((Number) searchResponse.getHits().getAt(2).sortValues()[0]).doubleValue(), equalTo(Double.MAX_VALUE));
-        assertThat(((Number) searchResponse.getHits().getAt(3).sortValues()[0]).doubleValue(), equalTo(Double.MAX_VALUE));
+        assertThat(((Number) searchResponse.getHits().getAt(1).sortValues()[0]).doubleValue(), equalTo(Double.POSITIVE_INFINITY));
+        assertThat(((Number) searchResponse.getHits().getAt(2).sortValues()[0]).doubleValue(), equalTo(Double.POSITIVE_INFINITY));
+        assertThat(((Number) searchResponse.getHits().getAt(3).sortValues()[0]).doubleValue(), equalTo(Double.POSITIVE_INFINITY));
 
         try {
                 client().prepareSearch("companies").setQuery(matchAllQuery())
@@ -592,18 +592,16 @@ public void testDuelOptimizations() throws Exception {
             final double originLat = randomLat();
             final double originLon = randomLon();
             final String distance = DistanceUnit.KILOMETERS.toString(randomIntBetween(1, 10000));
-            for (GeoDistance geoDistance : Arrays.asList(GeoDistance.ARC, GeoDistance.SLOPPY_ARC)) {
-                logger.info("Now testing GeoDistance={}, distance={}, origin=({}, {})", geoDistance, distance, originLat, originLon);
-                GeoDistanceQueryBuilder qb = QueryBuilders.geoDistanceQuery("location").point(originLat, originLon).distance(distance)
-                        .geoDistance(geoDistance);
-                long matches;
-                for (String optimizeBbox : Arrays.asList("none", "memory", "indexed")) {
-                    qb.optimizeBbox(optimizeBbox);
-                    SearchResponse resp = client().prepareSearch("index").setSize(0).setQuery(QueryBuilders.constantScoreQuery(qb))
-                        .execute().actionGet();
-                    matches = assertDuelOptimization(resp);
-                    logger.info("{} -> {} hits", optimizeBbox, matches);
-                }
+            logger.info("Now testing distance={}, origin=({}, {})", distance, originLat, originLon);
+            GeoDistanceQueryBuilder qb = QueryBuilders.geoDistanceQuery("location").point(originLat, originLon).distance(distance)
+                .geoDistance(GeoDistance.ARC);
+            long matches;
+            for (String optimizeBbox : Arrays.asList("none", "memory", "indexed")) {
+                qb.optimizeBbox(optimizeBbox);
+                SearchResponse resp = client().prepareSearch("index").setSize(0).setQuery(QueryBuilders.constantScoreQuery(qb))
+                    .execute().actionGet();
+                matches = assertDuelOptimization(resp);
+                logger.info("{} -> {} hits", optimizeBbox, matches);
             }
         }
     }
diff --git a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderIT.java b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderIT.java
index 2eef021be3967..6529b990255c3 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderIT.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderIT.java
@@ -25,7 +25,6 @@
 import org.elasticsearch.common.geo.GeoDistance;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.text.Text;
 import org.elasticsearch.common.unit.DistanceUnit;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.query.GeoValidationMethod;
@@ -70,7 +69,7 @@ public void testManyToManyGeoPoints() throws ExecutionException, InterruptedExce
          * |___________________________
          * 1   2   3   4   5   6   7
          */
-        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT);
+        Version version = randomBoolean() ? Version.CURRENT : VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT);
         Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
         assertAcked(prepareCreate("index").setSettings(settings).addMapping("type", LOCATION_FIELD, "type=geo_point"));
         XContentBuilder d1Builder = jsonBuilder();
@@ -97,35 +96,35 @@ public void testManyToManyGeoPoints() throws ExecutionException, InterruptedExce
 
         SearchResponse searchResponse = client().prepareSearch()
                 .setQuery(matchAllQuery())
-                .addSort(new GeoDistanceSortBuilder(LOCATION_FIELD, q).sortMode(SortMode.MIN).order(SortOrder.ASC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS))
+                .addSort(new GeoDistanceSortBuilder(LOCATION_FIELD, q).sortMode(SortMode.MIN).order(SortOrder.ASC))
                 .execute().actionGet();
         assertOrderedSearchHits(searchResponse, "d1", "d2");
-        assertThat((Double)searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(2, 2, 3, 2, DistanceUnit.KILOMETERS), 0.01d));
-        assertThat((Double)searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(2, 1, 5, 1, DistanceUnit.KILOMETERS), 0.01d));
+        assertThat((Double)searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.ARC.calculate(2, 2, 3, 2, DistanceUnit.METERS), 10d));
+        assertThat((Double)searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.ARC.calculate(2, 1, 5, 1, DistanceUnit.METERS), 10d));
 
         searchResponse = client().prepareSearch()
                 .setQuery(matchAllQuery())
-                .addSort(new GeoDistanceSortBuilder(LOCATION_FIELD, q).sortMode(SortMode.MIN).order(SortOrder.DESC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS))
+                .addSort(new GeoDistanceSortBuilder(LOCATION_FIELD, q).sortMode(SortMode.MIN).order(SortOrder.DESC))
                 .execute().actionGet();
         assertOrderedSearchHits(searchResponse, "d2", "d1");
-        assertThat((Double)searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(2, 1, 5, 1, DistanceUnit.KILOMETERS), 0.01d));
-        assertThat((Double)searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(2, 2, 3, 2, DistanceUnit.KILOMETERS), 0.01d));
+        assertThat((Double)searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.ARC.calculate(2, 1, 5, 1, DistanceUnit.METERS), 10d));
+        assertThat((Double)searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.ARC.calculate(2, 2, 3, 2, DistanceUnit.METERS), 10d));
 
         searchResponse = client().prepareSearch()
                 .setQuery(matchAllQuery())
-                .addSort(new GeoDistanceSortBuilder(LOCATION_FIELD, q).sortMode(SortMode.MAX).order(SortOrder.ASC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS))
+                .addSort(new GeoDistanceSortBuilder(LOCATION_FIELD, q).sortMode(SortMode.MAX).order(SortOrder.ASC))
                 .execute().actionGet();
         assertOrderedSearchHits(searchResponse, "d1", "d2");
-        assertThat((Double)searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(2, 2, 4, 1, DistanceUnit.KILOMETERS), 0.01d));
-        assertThat((Double)searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(2, 1, 6, 2, DistanceUnit.KILOMETERS), 0.01d));
+        assertThat((Double)searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.ARC.calculate(2, 2, 4, 1, DistanceUnit.METERS), 10d));
+        assertThat((Double)searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.ARC.calculate(2, 1, 6, 2, DistanceUnit.METERS), 10d));
 
         searchResponse = client().prepareSearch()
                 .setQuery(matchAllQuery())
-                .addSort(new GeoDistanceSortBuilder(LOCATION_FIELD, q).sortMode(SortMode.MAX).order(SortOrder.DESC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS))
+                .addSort(new GeoDistanceSortBuilder(LOCATION_FIELD, q).sortMode(SortMode.MAX).order(SortOrder.DESC))
                 .execute().actionGet();
         assertOrderedSearchHits(searchResponse, "d2", "d1");
-        assertThat((Double)searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(2, 1, 6, 2, DistanceUnit.KILOMETERS), 0.01d));
-        assertThat((Double)searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(2, 2, 4, 1, DistanceUnit.KILOMETERS), 0.01d));
+        assertThat((Double)searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.ARC.calculate(2, 1, 6, 2, DistanceUnit.METERS), 10d));
+        assertThat((Double)searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.ARC.calculate(2, 2, 4, 1, DistanceUnit.METERS), 10d));
     }
 
     public void testSingeToManyAvgMedian() throws ExecutionException, InterruptedException, IOException {
@@ -135,7 +134,7 @@ public void testSingeToManyAvgMedian() throws ExecutionException, InterruptedExc
          * d1 = (0, 1), (0, 4), (0, 10); so avg. distance is 5, median distance is 4
          * d2 = (0, 1), (0, 5), (0, 6); so avg. distance is 4, median distance is 5
          */
-        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT);
+        Version version = randomBoolean() ? Version.CURRENT : VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT);
         Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
         assertAcked(prepareCreate("index").setSettings(settings).addMapping("type", LOCATION_FIELD, "type=geo_point"));
         XContentBuilder d1Builder = jsonBuilder();
@@ -155,19 +154,19 @@ public void testSingeToManyAvgMedian() throws ExecutionException, InterruptedExc
 
         SearchResponse searchResponse = client().prepareSearch()
                 .setQuery(matchAllQuery())
-                .addSort(new GeoDistanceSortBuilder(LOCATION_FIELD, q).sortMode(SortMode.AVG).order(SortOrder.ASC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS))
+                .addSort(new GeoDistanceSortBuilder(LOCATION_FIELD, q).sortMode(SortMode.AVG).order(SortOrder.ASC))
                 .execute().actionGet();
         assertOrderedSearchHits(searchResponse, "d2", "d1");
-        assertThat((Double)searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(0, 0, 0, 4, DistanceUnit.KILOMETERS), 0.01d));
-        assertThat((Double)searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(0, 0, 0, 5, DistanceUnit.KILOMETERS), 0.01d));
+        assertThat((Double)searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.ARC.calculate(0, 0, 0, 4, DistanceUnit.METERS), 10d));
+        assertThat((Double)searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.ARC.calculate(0, 0, 0, 5, DistanceUnit.METERS), 10d));
 
         searchResponse = client().prepareSearch()
                 .setQuery(matchAllQuery())
-                .addSort(new GeoDistanceSortBuilder(LOCATION_FIELD, q).sortMode(SortMode.MEDIAN).order(SortOrder.ASC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS))
+                .addSort(new GeoDistanceSortBuilder(LOCATION_FIELD, q).sortMode(SortMode.MEDIAN).order(SortOrder.ASC))
                 .execute().actionGet();
         assertOrderedSearchHits(searchResponse, "d1", "d2");
-        assertThat((Double)searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(0, 0, 0, 4, DistanceUnit.KILOMETERS), 0.01d));
-        assertThat((Double)searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(0, 0, 0, 5, DistanceUnit.KILOMETERS), 0.01d));
+        assertThat((Double)searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.ARC.calculate(0, 0, 0, 4, DistanceUnit.METERS), 10d));
+        assertThat((Double)searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.ARC.calculate(0, 0, 0, 5, DistanceUnit.METERS), 10d));
     }
 
     protected void createShuffeldJSONArray(XContentBuilder builder, GeoPoint[] pointsArray) throws IOException {
@@ -195,7 +194,7 @@ public void testManyToManyGeoPointsWithDifferentFormats() throws ExecutionExcept
          * |______________________
          * 1   2   3   4   5   6
          */
-        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT);
+        Version version = randomBoolean() ? Version.CURRENT : VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT);
         Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
         assertAcked(prepareCreate("index").setSettings(settings).addMapping("type", LOCATION_FIELD, "type=geo_point"));
         XContentBuilder d1Builder = jsonBuilder();
@@ -236,19 +235,19 @@ public void testManyToManyGeoPointsWithDifferentFormats() throws ExecutionExcept
 
         SearchResponse searchResponse = client().prepareSearch()
                 .setQuery(matchAllQuery())
-                .addSort(geoDistanceSortBuilder.sortMode(SortMode.MIN).order(SortOrder.ASC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS))
+                .addSort(geoDistanceSortBuilder.sortMode(SortMode.MIN).order(SortOrder.ASC))
                 .execute().actionGet();
         assertOrderedSearchHits(searchResponse, "d1", "d2");
-        assertThat((Double) searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(2.5, 1, 2, 1, DistanceUnit.KILOMETERS), 1.e-4));
-        assertThat((Double) searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(4.5, 1, 2, 1, DistanceUnit.KILOMETERS), 1.e-4));
+        assertThat((Double) searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.ARC.calculate(2.5, 1, 2, 1, DistanceUnit.METERS), 1.e-1));
+        assertThat((Double) searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.ARC.calculate(4.5, 1, 2, 1, DistanceUnit.METERS), 1.e-1));
 
         searchResponse = client().prepareSearch()
                 .setQuery(matchAllQuery())
-                .addSort(geoDistanceSortBuilder.sortMode(SortMode.MAX).order(SortOrder.ASC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS))
+                .addSort(geoDistanceSortBuilder.sortMode(SortMode.MAX).order(SortOrder.ASC))
                 .execute().actionGet();
         assertOrderedSearchHits(searchResponse, "d1", "d2");
-        assertThat((Double) searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(3.25, 4, 2, 1, DistanceUnit.KILOMETERS), 1.e-4));
-        assertThat((Double) searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(5.25, 4, 2, 1, DistanceUnit.KILOMETERS), 1.e-4));
+        assertThat((Double) searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.ARC.calculate(3.25, 4, 2, 1, DistanceUnit.METERS), 1.e-1));
+        assertThat((Double) searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.ARC.calculate(5.25, 4, 2, 1, DistanceUnit.METERS), 1.e-1));
 
     }
 
@@ -264,7 +263,7 @@ public void testSinglePointGeoDistanceSort() throws ExecutionException, Interrup
 
         SearchResponse searchResponse = client().prepareSearch()
                 .setQuery(matchAllQuery())
-                .addSort(geoDistanceSortBuilder.sortMode(SortMode.MIN).order(SortOrder.ASC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS))
+                .addSort(geoDistanceSortBuilder.sortMode(SortMode.MIN).order(SortOrder.ASC))
                 .execute().actionGet();
         checkCorrectSortOrderForGeoSort(searchResponse);
 
@@ -272,7 +271,7 @@ public void testSinglePointGeoDistanceSort() throws ExecutionException, Interrup
 
         searchResponse = client().prepareSearch()
                 .setQuery(matchAllQuery())
-                .addSort(geoDistanceSortBuilder.sortMode(SortMode.MIN).order(SortOrder.ASC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS))
+                .addSort(geoDistanceSortBuilder.sortMode(SortMode.MIN).order(SortOrder.ASC))
                 .execute().actionGet();
         checkCorrectSortOrderForGeoSort(searchResponse);
 
@@ -280,7 +279,7 @@ public void testSinglePointGeoDistanceSort() throws ExecutionException, Interrup
 
         searchResponse = client().prepareSearch()
                 .setQuery(matchAllQuery())
-                .addSort(geoDistanceSortBuilder.sortMode(SortMode.MIN).order(SortOrder.ASC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS))
+                .addSort(geoDistanceSortBuilder.sortMode(SortMode.MIN).order(SortOrder.ASC))
                 .execute().actionGet();
         checkCorrectSortOrderForGeoSort(searchResponse);
 
@@ -288,36 +287,35 @@ public void testSinglePointGeoDistanceSort() throws ExecutionException, Interrup
                 .prepareSearch()
                 .setSource(
                         new SearchSourceBuilder().sort(SortBuilders.geoDistanceSort(LOCATION_FIELD, 2.0, 2.0)
-                                .unit(DistanceUnit.KILOMETERS).geoDistance(GeoDistance.PLANE))).execute().actionGet();
+                                )).execute().actionGet();
         checkCorrectSortOrderForGeoSort(searchResponse);
 
         searchResponse = client()
                 .prepareSearch()
                 .setSource(
                         new SearchSourceBuilder().sort(SortBuilders.geoDistanceSort(LOCATION_FIELD, "s037ms06g7h0")
-                                .unit(DistanceUnit.KILOMETERS).geoDistance(GeoDistance.PLANE))).execute().actionGet();
+                                )).execute().actionGet();
         checkCorrectSortOrderForGeoSort(searchResponse);
 
         searchResponse = client()
                 .prepareSearch()
                 .setSource(
                         new SearchSourceBuilder().sort(SortBuilders.geoDistanceSort(LOCATION_FIELD, 2.0, 2.0)
-                                .unit(DistanceUnit.KILOMETERS).geoDistance(GeoDistance.PLANE))).execute().actionGet();
+                                )).execute().actionGet();
         checkCorrectSortOrderForGeoSort(searchResponse);
 
         searchResponse = client()
                 .prepareSearch()
                 .setSource(
                         new SearchSourceBuilder().sort(SortBuilders.geoDistanceSort(LOCATION_FIELD, 2.0, 2.0)
-                                .unit(DistanceUnit.KILOMETERS).geoDistance(GeoDistance.PLANE)
                                 .validation(GeoValidationMethod.COERCE))).execute().actionGet();
         checkCorrectSortOrderForGeoSort(searchResponse);
     }
 
     private static void checkCorrectSortOrderForGeoSort(SearchResponse searchResponse) {
         assertOrderedSearchHits(searchResponse, "d2", "d1");
-        assertThat((Double) searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(2, 2, 1, 2, DistanceUnit.KILOMETERS), 1.e-4));
-        assertThat((Double) searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(2, 2, 1, 1, DistanceUnit.KILOMETERS), 1.e-4));
+        assertThat((Double) searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.ARC.calculate(2, 2, 1, 2, DistanceUnit.METERS), 1.e-1));
+        assertThat((Double) searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.ARC.calculate(2, 2, 1, 1, DistanceUnit.METERS), 1.e-1));
     }
 
     protected void createQPoints(List<String> qHashes, List<GeoPoint> qPoints) {
diff --git a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java
index 4dd14cc523fd0..dd201f1dd46cc 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java
@@ -20,25 +20,22 @@
 package org.elasticsearch.search.sort;
 
 
-import org.apache.lucene.queryparser.xml.builders.MatchAllDocsQueryBuilder;
-import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.document.LatLonDocValuesField;
 import org.apache.lucene.search.SortField;
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.geo.GeoDistance;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.unit.DistanceUnit;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.mapper.GeoPointFieldMapper;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.index.mapper.LatLonPointFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.query.GeoValidationMethod;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
+import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.test.geo.RandomGeoGenerator;
 
@@ -55,7 +52,7 @@ protected GeoDistanceSortBuilder createTestItem() {
     }
 
     public static GeoDistanceSortBuilder randomGeoDistanceSortBuilder() {
-        String fieldName = randomAsciiOfLengthBetween(1, 10);
+        String fieldName = randomAlphaOfLengthBetween(1, 10);
         GeoDistanceSortBuilder result = null;
 
         int id = randomIntBetween(0, 2);
@@ -99,7 +96,7 @@ public static GeoDistanceSortBuilder randomGeoDistanceSortBuilder() {
             result.setNestedPath(
                     randomValueOtherThan(
                             result.getNestedPath(),
-                            () -> randomAsciiOfLengthBetween(1, 10)));
+                            () -> randomAlphaOfLengthBetween(1, 10)));
         }
         if (randomBoolean()) {
             result.validation(randomValueOtherThan(result.validation(), () -> randomFrom(GeoValidationMethod.values())));
@@ -110,7 +107,7 @@ public static GeoDistanceSortBuilder randomGeoDistanceSortBuilder() {
 
     @Override
     protected MappedFieldType provideMappedFieldType(String name) {
-        MappedFieldType clone = GeoPointFieldMapper.Defaults.FIELD_TYPE.clone();
+        MappedFieldType clone = LatLonPointFieldMapper.Defaults.FIELD_TYPE.clone();
         clone.setName(name);
         return clone;
     }
@@ -171,7 +168,7 @@ protected GeoDistanceSortBuilder mutate(GeoDistanceSortBuilder original) throws
         case 7:
             result.setNestedPath(randomValueOtherThan(
                     result.getNestedPath(),
-                    () -> randomAsciiOfLengthBetween(1, 10)));
+                    () -> randomAlphaOfLengthBetween(1, 10)));
             break;
         case 8:
             result.validation(randomValueOtherThan(result.validation(), () -> randomFrom(GeoValidationMethod.values())));
@@ -182,7 +179,6 @@ protected GeoDistanceSortBuilder mutate(GeoDistanceSortBuilder original) throws
 
     @Override
     protected void sortFieldAssertions(GeoDistanceSortBuilder builder, SortField sortField, DocValueFormat format) throws IOException {
-        assertEquals(SortField.Type.CUSTOM, sortField.getType());
         assertEquals(builder.order() == SortOrder.ASC ? false : true, sortField.getReverse());
         assertEquals(builder.fieldName(), sortField.getField());
     }
@@ -207,10 +203,10 @@ public void testReverseOptionFailsWhenNonStringField() throws IOException {
                 "  } ],\n" +
                 "  \"reverse\" : true\n" +
                 "}";
-        XContentParser itemParser = XContentHelper.createParser(new BytesArray(json));
+        XContentParser itemParser = createParser(JsonXContent.jsonXContent, json);
         itemParser.nextToken();
 
-        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, itemParser, ParseFieldMatcher.STRICT);
+        QueryParseContext context = new QueryParseContext(itemParser);
 
         try {
           GeoDistanceSortBuilder.fromXContent(context, "");
@@ -228,10 +224,10 @@ public void testReverseOptionFailsWhenStringFieldButResetting() throws IOExcepti
                 "  } ],\n" +
                 "  \"reverse\" : \"true\"\n" +
                 "}";
-        XContentParser itemParser = XContentHelper.createParser(new BytesArray(json));
+        XContentParser itemParser = createParser(JsonXContent.jsonXContent, json);
         itemParser.nextToken();
 
-        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, itemParser, ParseFieldMatcher.STRICT);
+        QueryParseContext context = new QueryParseContext(itemParser);
 
         try {
           GeoDistanceSortBuilder.fromXContent(context, "");
@@ -245,10 +241,10 @@ public void testReverseOptionFailsBuildWhenInvalidGeoHashString() throws IOExcep
         String json = "{\n" +
                 "  \"reverse\" : \"false\"\n" +
                 "}";
-        XContentParser itemParser = XContentHelper.createParser(new BytesArray(json));
+        XContentParser itemParser = createParser(JsonXContent.jsonXContent, json);
         itemParser.nextToken();
 
-        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, itemParser, ParseFieldMatcher.STRICT);
+        QueryParseContext context = new QueryParseContext(itemParser);
 
         try {
           GeoDistanceSortBuilder item = GeoDistanceSortBuilder.fromXContent(context, "");
@@ -260,7 +256,7 @@ public void testReverseOptionFailsBuildWhenInvalidGeoHashString() throws IOExcep
             assertEquals("illegal latitude value [269.384765625] for [GeoDistanceSort] for field [reverse].", e.getMessage());
         }
     }
-    
+
     public void testCoerceIsDeprecated() throws IOException {
         String json = "{\n" +
                 "  \"testname\" : [ {\n" +
@@ -268,18 +264,16 @@ public void testCoerceIsDeprecated() throws IOException {
                 "    \"lon\" : -51.94128329747579\n" +
                 "  } ],\n" +
                 "  \"unit\" : \"m\",\n" +
-                "  \"distance_type\" : \"sloppy_arc\",\n" +
-                "  \"mode\" : \"SUM\",\n" +
+                "  \"distance_type\" : \"arc\",\n" +
+                "  \"mode\" : \"MAX\",\n" +
                 "  \"coerce\" : true\n" +
                 "}";
-        XContentParser itemParser = XContentHelper.createParser(new BytesArray(json));
+        XContentParser itemParser = createParser(JsonXContent.jsonXContent, json);
         itemParser.nextToken();
 
-        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, itemParser, ParseFieldMatcher.STRICT);
-
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> GeoDistanceSortBuilder.fromXContent(context, ""));
-        assertTrue(e.getMessage().startsWith("Deprecated field "));
-        
+        QueryParseContext context = new QueryParseContext(itemParser);
+        GeoDistanceSortBuilder.fromXContent(context, "");
+        assertWarnings("Deprecated field [coerce] used, replaced by [validation_method]");
     }
 
     public void testIgnoreMalformedIsDeprecated() throws IOException {
@@ -289,18 +283,16 @@ public void testIgnoreMalformedIsDeprecated() throws IOException {
                 "    \"lon\" : -51.94128329747579\n" +
                 "  } ],\n" +
                 "  \"unit\" : \"m\",\n" +
-                "  \"distance_type\" : \"sloppy_arc\",\n" +
-                "  \"mode\" : \"SUM\",\n" +
+                "  \"distance_type\" : \"arc\",\n" +
+                "  \"mode\" : \"MAX\",\n" +
                 "  \"ignore_malformed\" : true\n" +
                 "}";
-        XContentParser itemParser = XContentHelper.createParser(new BytesArray(json));
+        XContentParser itemParser = createParser(JsonXContent.jsonXContent, json);
         itemParser.nextToken();
 
-        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, itemParser, ParseFieldMatcher.STRICT);
-
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> GeoDistanceSortBuilder.fromXContent(context, ""));
-        assertTrue(e.getMessage().startsWith("Deprecated field "));
-        
+        QueryParseContext context = new QueryParseContext(itemParser);
+        GeoDistanceSortBuilder.fromXContent(context, "");
+        assertWarnings("Deprecated field [ignore_malformed] used, replaced by [validation_method]");
     }
 
     public void testSortModeSumIsRejectedInJSON() throws IOException {
@@ -310,13 +302,13 @@ public void testSortModeSumIsRejectedInJSON() throws IOException {
                 "    \"lon\" : -51.94128329747579\n" +
                 "  } ],\n" +
                 "  \"unit\" : \"m\",\n" +
-                "  \"distance_type\" : \"sloppy_arc\",\n" +
+                "  \"distance_type\" : \"arc\",\n" +
                 "  \"mode\" : \"SUM\"\n" +
                 "}";
-        XContentParser itemParser = XContentHelper.createParser(new BytesArray(json));
+        XContentParser itemParser = createParser(JsonXContent.jsonXContent, json);
         itemParser.nextToken();
 
-        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, itemParser, ParseFieldMatcher.STRICT);
+        QueryParseContext context = new QueryParseContext(itemParser);
 
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> GeoDistanceSortBuilder.fromXContent(context, ""));
         assertEquals("sort_mode [sum] isn't supported for sorting by geo distance", e.getMessage());
@@ -327,7 +319,7 @@ public void testGeoDistanceSortCanBeParsedFromGeoHash() throws IOException {
                 "    \"VDcvDuFjE\" : [ \"7umzzv8eychg\", \"dmdgmt5z13uw\", " +
                 "    \"ezu09wxw6v4c\", \"kc7s3515p6k6\", \"jgeuvjwrmfzn\", \"kcpcfj7ruyf8\" ],\n" +
                 "    \"unit\" : \"m\",\n" +
-                "    \"distance_type\" : \"sloppy_arc\",\n" +
+                "    \"distance_type\" : \"arc\",\n" +
                 "    \"mode\" : \"MAX\",\n" +
                 "    \"nested_filter\" : {\n" +
                 "      \"ids\" : {\n" +
@@ -338,10 +330,10 @@ public void testGeoDistanceSortCanBeParsedFromGeoHash() throws IOException {
                 "    },\n" +
                 "    \"validation_method\" : \"STRICT\"\n" +
                 "  }";
-        XContentParser itemParser = XContentHelper.createParser(new BytesArray(json));
+        XContentParser itemParser = createParser(JsonXContent.jsonXContent, json);
         itemParser.nextToken();
 
-        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, itemParser, ParseFieldMatcher.STRICT);
+        QueryParseContext context = new QueryParseContext(itemParser);
 
         GeoDistanceSortBuilder result = GeoDistanceSortBuilder.fromXContent(context, json);
         assertEquals("[-19.700583312660456, -2.8225036337971687, "
@@ -457,13 +449,13 @@ public void testGeoDistanceSortDeprecatedSortModeException() throws Exception {
         sortBuilder.field("unit", "km");
         sortBuilder.field("sort_mode", "max");
         sortBuilder.endObject();
-        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> parse(sortBuilder));
-        assertEquals("Deprecated field [sort_mode] used, expected [mode] instead", ex.getMessage());
+        parse(sortBuilder);
+        assertWarnings("Deprecated field [sort_mode] used, expected [mode] instead");
     }
 
-    private static GeoDistanceSortBuilder parse(XContentBuilder sortBuilder) throws Exception {
-        XContentParser parser = XContentHelper.createParser(sortBuilder.bytes());
-        QueryParseContext parseContext = new QueryParseContext(new IndicesQueriesRegistry(), parser, ParseFieldMatcher.STRICT);
+    private GeoDistanceSortBuilder parse(XContentBuilder sortBuilder) throws Exception {
+        XContentParser parser = createParser(sortBuilder);
+        QueryParseContext parseContext = new QueryParseContext(parser);
         parser.nextToken();
         return GeoDistanceSortBuilder.fromXContent(parseContext, null);
     }
@@ -472,4 +464,40 @@ private static GeoDistanceSortBuilder parse(XContentBuilder sortBuilder) throws
     protected GeoDistanceSortBuilder fromXContent(QueryParseContext context, String fieldName) throws IOException {
         return GeoDistanceSortBuilder.fromXContent(context, fieldName);
     }
+
+    public void testCommonCaseIsOptimized() throws IOException {
+        // make sure the below tests test something...
+        assertFalse(SortField.class.equals(LatLonDocValuesField.newDistanceSort("random_field_name", 3.5, 2.1).getClass()));
+
+        QueryShardContext context = createMockShardContext();
+        // The common case should use LatLonDocValuesField.newDistanceSort
+        GeoDistanceSortBuilder builder = new GeoDistanceSortBuilder("", new GeoPoint(3.5, 2.1));
+        SortFieldAndFormat sort = builder.build(context);
+        assertEquals(LatLonDocValuesField.newDistanceSort("random_field_name", 3.5, 2.1).getClass(), sort.field.getClass());
+
+        // however this might be disabled by fancy options
+        builder = new GeoDistanceSortBuilder("random_field_name", new GeoPoint(3.5, 2.1), new GeoPoint(3.0, 4));
+        sort = builder.build(context);
+        assertEquals(SortField.class, sort.field.getClass()); // 2 points -> plain SortField with a custom comparator
+
+        builder = new GeoDistanceSortBuilder("random_field_name", new GeoPoint(3.5, 2.1));
+        builder.unit(DistanceUnit.KILOMETERS);
+        sort = builder.build(context);
+        assertEquals(SortField.class, sort.field.getClass()); // km rather than m -> plain SortField with a custom comparator
+
+        builder = new GeoDistanceSortBuilder("random_field_name", new GeoPoint(3.5, 2.1));
+        builder.order(SortOrder.DESC);
+        sort = builder.build(context);
+        assertEquals(SortField.class, sort.field.getClass()); // descending means the max value should be considered rather than min
+
+        builder = new GeoDistanceSortBuilder("random_field_name", new GeoPoint(3.5, 2.1));
+        builder.setNestedPath("some_nested_path");
+        sort = builder.build(context);
+        assertEquals(SortField.class, sort.field.getClass()); // can't use LatLon optimized sorting with nested fields
+
+        builder = new GeoDistanceSortBuilder("random_field_name", new GeoPoint(3.5, 2.1));
+        builder.order(SortOrder.DESC);
+        sort = builder.build(context);
+        assertEquals(SortField.class, sort.field.getClass()); // can't use LatLon optimized sorting with DESC sorting
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/sort/ScoreSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/ScoreSortBuilderTests.java
index 94e0054b39711..1d4389c4e3341 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/ScoreSortBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/ScoreSortBuilderTests.java
@@ -21,14 +21,10 @@
 
 
 import org.apache.lucene.search.SortField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.DocValueFormat;
-import org.junit.Rule;
-import org.junit.rules.ExpectedException;
 
 import java.io.IOException;
 
@@ -50,16 +46,12 @@ protected ScoreSortBuilder mutate(ScoreSortBuilder original) throws IOException
         return result;
     }
 
-    @Rule
-    public ExpectedException exceptionRule = ExpectedException.none();
-
     /**
      * test passing null to {@link ScoreSortBuilder#order(SortOrder)} is illegal
      */
     public void testIllegalOrder() {
-            exceptionRule.expect(NullPointerException.class);
-            exceptionRule.expectMessage("sort order cannot be null.");
-            new ScoreSortBuilder().order(null);
+        Exception e = expectThrows(NullPointerException.class, () -> new ScoreSortBuilder().order(null));
+        assertEquals("sort order cannot be null.", e.getMessage());
     }
 
     /**
@@ -69,31 +61,31 @@ public void testIllegalOrder() {
     public void testParseOrder() throws IOException {
         SortOrder order = randomBoolean() ? SortOrder.ASC : SortOrder.DESC;
         String scoreSortString = "{ \"_score\": { \"order\": \""+ order.toString() +"\" }}";
-        XContentParser parser = XContentFactory.xContent(scoreSortString).createParser(scoreSortString);
+        XContentParser parser = createParser(JsonXContent.jsonXContent, scoreSortString);
         // need to skip until parser is located on second START_OBJECT
         parser.nextToken();
         parser.nextToken();
         parser.nextToken();
 
-        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.STRICT);
+        QueryParseContext context = new QueryParseContext(parser);
         ScoreSortBuilder scoreSort = ScoreSortBuilder.fromXContent(context, "_score");
         assertEquals(order, scoreSort.order());
     }
 
     public void testReverseOptionFails() throws IOException {
         String json = "{ \"_score\": { \"reverse\": true }}";
-        XContentParser parser = XContentFactory.xContent(json).createParser(json);
+        XContentParser parser = createParser(JsonXContent.jsonXContent, json);
         // need to skip until parser is located on second START_OBJECT
         parser.nextToken();
         parser.nextToken();
         parser.nextToken();
 
-        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.EMPTY);
+        QueryParseContext context = new QueryParseContext(parser);
 
         try {
           ScoreSortBuilder.fromXContent(context, "_score");
           fail("adding reverse sorting option should fail with an exception");
-        } catch (ParsingException e) {
+        } catch (IllegalArgumentException e) {
             // all good
         }
     }
diff --git a/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java
index e96b02c0e2c87..49ae3734b4c83 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java
@@ -21,19 +21,16 @@
 
 
 import org.apache.lucene.search.SortField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.sort.ScriptSortBuilder.ScriptSortType;
-import org.junit.Rule;
-import org.junit.rules.ExpectedException;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
 
@@ -46,7 +43,7 @@ protected ScriptSortBuilder createTestItem() {
 
     public static ScriptSortBuilder randomScriptSortBuilder() {
         ScriptSortType type = randomBoolean() ? ScriptSortType.NUMBER : ScriptSortType.STRING;
-        ScriptSortBuilder builder = new ScriptSortBuilder(new Script(randomAsciiOfLengthBetween(5, 10)),
+        ScriptSortBuilder builder = new ScriptSortBuilder(mockScript(randomAlphaOfLengthBetween(5, 10)),
                 type);
         if (randomBoolean()) {
                 builder.order(randomFrom(SortOrder.values()));
@@ -66,7 +63,7 @@ public static ScriptSortBuilder randomScriptSortBuilder() {
             builder.setNestedFilter(randomNestedFilter());
         }
         if (randomBoolean()) {
-            builder.setNestedPath(randomAsciiOfLengthBetween(1, 10));
+            builder.setNestedPath(randomAlphaOfLengthBetween(1, 10));
         }
         return builder;
     }
@@ -79,7 +76,7 @@ protected ScriptSortBuilder mutate(ScriptSortBuilder original) throws IOExceptio
             Script script = original.script();
             ScriptSortType type = original.type();
             if (randomBoolean()) {
-                result = new ScriptSortBuilder(new Script(script.getScript() + "_suffix"), type);
+                result = new ScriptSortBuilder(mockScript(script.getIdOrCode() + "_suffix"), type);
             } else {
                 result = new ScriptSortBuilder(script, type.equals(ScriptSortType.NUMBER) ? ScriptSortType.STRING : ScriptSortType.NUMBER);
             }
@@ -146,19 +143,14 @@ public void testScriptSortType() {
         assertEquals(ScriptSortType.NUMBER, ScriptSortType.fromString("NUMBER"));
     }
 
-    @Rule
-    public ExpectedException exceptionRule = ExpectedException.none();
-
     public void testScriptSortTypeNull() {
-        exceptionRule.expect(NullPointerException.class);
-        exceptionRule.expectMessage("input string is null");
-        ScriptSortType.fromString(null);
+        Exception e = expectThrows(NullPointerException.class, () -> ScriptSortType.fromString(null));
+        assertEquals("input string is null", e.getMessage());
     }
 
     public void testScriptSortTypeIllegalArgument() {
-        exceptionRule.expect(IllegalArgumentException.class);
-        exceptionRule.expectMessage("Unknown ScriptSortType [xyz]");
-        ScriptSortType.fromString("xyz");
+        Exception e = expectThrows(IllegalArgumentException.class, () -> ScriptSortType.fromString("xyz"));
+        assertEquals("Unknown ScriptSortType [xyz]", e.getMessage());
     }
 
     public void testParseJson() throws IOException {
@@ -166,7 +158,7 @@ public void testParseJson() throws IOException {
                 "\"_script\" : {\n" +
                     "\"type\" : \"number\",\n" +
                     "\"script\" : {\n" +
-                        "\"inline\": \"doc['field_name'].value * factor\",\n" +
+                        "\"source\": \"doc['field_name'].value * factor\",\n" +
                         "\"params\" : {\n" +
                             "\"factor\" : 1.1\n" +
                             "}\n" +
@@ -174,14 +166,14 @@ public void testParseJson() throws IOException {
                     "\"mode\" : \"max\",\n" +
                     "\"order\" : \"asc\"\n" +
                 "} }\n";
-        XContentParser parser = XContentFactory.xContent(scriptSort).createParser(scriptSort);
+        XContentParser parser = createParser(JsonXContent.jsonXContent, scriptSort);
         parser.nextToken();
         parser.nextToken();
         parser.nextToken();
 
-        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.STRICT);
+        QueryParseContext context = new QueryParseContext(parser);
         ScriptSortBuilder builder = ScriptSortBuilder.fromXContent(context, null);
-        assertEquals("doc['field_name'].value * factor", builder.script().getScript());
+        assertEquals("doc['field_name'].value * factor", builder.script().getIdOrCode());
         assertEquals(Script.DEFAULT_SCRIPT_LANG, builder.script().getLang());
         assertEquals(1.1, builder.script().getParams().get("factor"));
         assertEquals(ScriptType.INLINE, builder.script().getType());
@@ -200,16 +192,16 @@ public void testParseJson_simple() throws IOException {
                 "\"mode\" : \"max\",\n" +
                 "\"order\" : \"asc\"\n" +
                 "} }\n";
-        XContentParser parser = XContentFactory.xContent(scriptSort).createParser(scriptSort);
+        XContentParser parser = createParser(JsonXContent.jsonXContent, scriptSort);
         parser.nextToken();
         parser.nextToken();
         parser.nextToken();
 
-        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.STRICT);
+        QueryParseContext context = new QueryParseContext(parser);
         ScriptSortBuilder builder = ScriptSortBuilder.fromXContent(context, null);
-        assertEquals("doc['field_name'].value", builder.script().getScript());
+        assertEquals("doc['field_name'].value", builder.script().getIdOrCode());
         assertEquals(Script.DEFAULT_SCRIPT_LANG, builder.script().getLang());
-        assertNull(builder.script().getParams());
+        assertEquals(builder.script().getParams(), Collections.emptyMap());
         assertEquals(ScriptType.INLINE, builder.script().getType());
         assertEquals(ScriptSortType.NUMBER, builder.type());
         assertEquals(SortOrder.ASC, builder.order());
@@ -220,52 +212,49 @@ public void testParseJson_simple() throws IOException {
 
     public void testParseBadFieldNameExceptions() throws IOException {
         String scriptSort = "{\"_script\" : {" + "\"bad_field\" : \"number\"" + "} }";
-        XContentParser parser = XContentFactory.xContent(scriptSort).createParser(scriptSort);
+        XContentParser parser = createParser(JsonXContent.jsonXContent, scriptSort);
         parser.nextToken();
         parser.nextToken();
         parser.nextToken();
 
-        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.STRICT);
-        exceptionRule.expect(ParsingException.class);
-        exceptionRule.expectMessage("failed to parse field [bad_field]");
-        ScriptSortBuilder.fromXContent(context, null);
+        QueryParseContext context = new QueryParseContext(parser);
+        Exception e = expectThrows(IllegalArgumentException.class, () -> ScriptSortBuilder.fromXContent(context, null));
+        assertEquals("[_script] unknown field [bad_field], parser not found", e.getMessage());
     }
 
     public void testParseBadFieldNameExceptionsOnStartObject() throws IOException {
 
         String scriptSort = "{\"_script\" : {" + "\"bad_field\" : { \"order\" : \"asc\" } } }";
-        XContentParser parser = XContentFactory.xContent(scriptSort).createParser(scriptSort);
+        XContentParser parser = createParser(JsonXContent.jsonXContent, scriptSort);
         parser.nextToken();
         parser.nextToken();
         parser.nextToken();
 
-        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.STRICT);
-        exceptionRule.expect(ParsingException.class);
-        exceptionRule.expectMessage("failed to parse field [bad_field]");
-        ScriptSortBuilder.fromXContent(context, null);
+        QueryParseContext context = new QueryParseContext(parser);
+        Exception e = expectThrows(IllegalArgumentException.class, () -> ScriptSortBuilder.fromXContent(context, null));
+        assertEquals("[_script] unknown field [bad_field], parser not found", e.getMessage());
     }
 
     public void testParseUnexpectedToken() throws IOException {
         String scriptSort = "{\"_script\" : {" + "\"script\" : [ \"order\" : \"asc\" ] } }";
-        XContentParser parser = XContentFactory.xContent(scriptSort).createParser(scriptSort);
+        XContentParser parser = createParser(JsonXContent.jsonXContent, scriptSort);
         parser.nextToken();
         parser.nextToken();
         parser.nextToken();
 
-        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.STRICT);
-        exceptionRule.expect(ParsingException.class);
-        exceptionRule.expectMessage("unexpected token [START_ARRAY]");
-        ScriptSortBuilder.fromXContent(context, null);
+        QueryParseContext context = new QueryParseContext(parser);
+        Exception e = expectThrows(IllegalArgumentException.class, () -> ScriptSortBuilder.fromXContent(context, null));
+        assertEquals("[_script] script doesn't support values of type: START_ARRAY", e.getMessage());
     }
 
     /**
      * script sort of type {@link ScriptSortType} does not work with {@link SortMode#AVG}, {@link SortMode#MEDIAN} or {@link SortMode#SUM}
      */
     public void testBadSortMode() throws IOException {
-        ScriptSortBuilder builder = new ScriptSortBuilder(new Script("something"), ScriptSortType.STRING);
-        exceptionRule.expect(IllegalArgumentException.class);
-        exceptionRule.expectMessage("script sort of type [string] doesn't support mode");
-        builder.sortMode(SortMode.fromString(randomFrom(new String[]{"avg", "median", "sum"})));
+        ScriptSortBuilder builder = new ScriptSortBuilder(mockScript("something"), ScriptSortType.STRING);
+        String sortMode = randomFrom(new String[] { "avg", "median", "sum" });
+        Exception e = expectThrows(IllegalArgumentException.class, () -> builder.sortMode(SortMode.fromString(sortMode)));
+        assertEquals("script sort of type [string] doesn't support mode [" + sortMode + "]", e.getMessage());
     }
 
     @Override
diff --git a/core/src/test/java/org/elasticsearch/search/sort/SimpleSortIT.java b/core/src/test/java/org/elasticsearch/search/sort/SimpleSortIT.java
index 80cda7d7b56ab..d73da7e507646 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/SimpleSortIT.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/SimpleSortIT.java
@@ -25,7 +25,6 @@
 import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.geo.GeoUtils;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.MockScriptPlugin;
@@ -49,7 +48,8 @@
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
-import static org.elasticsearch.script.ScriptService.ScriptType;
+
+import org.elasticsearch.script.ScriptType;
 import static org.elasticsearch.search.sort.SortBuilders.scriptSort;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
@@ -183,7 +183,7 @@ public void testSimpleSorts() throws Exception {
         // STRING script
         int size = 1 + random.nextInt(10);
 
-        Script script = new Script("doc['str_value'].value", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
+        Script script = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['str_value'].value", Collections.emptyMap());
 
         SearchResponse searchResponse = client().prepareSearch()
                 .setQuery(matchAllQuery())
@@ -251,9 +251,9 @@ public void testSortMinValueScript() throws IOException {
                     .setSource(jsonBuilder()
                             .startObject()
                                 .field("ord", i)
-                                .field("svalue", new String[]{"" + i, "" + (i + 1), "" + (i + 2)})
-                                .field("lvalue", new long[]{i, i + 1, i + 2})
-                                .field("dvalue", new double[]{i, i + 1, i + 2})
+                                .array("svalue", new String[]{"" + i, "" + (i + 1), "" + (i + 2)})
+                                .array("lvalue", new long[]{i, i + 1, i + 2})
+                                .array("dvalue", new double[]{i, i + 1, i + 2})
                                 .startObject("gvalue")
                                     .field("lat", (double) i + 1)
                                     .field("lon", (double) i)
@@ -275,7 +275,7 @@ public void testSortMinValueScript() throws IOException {
         // test the long values
         SearchResponse searchResponse = client().prepareSearch()
                 .setQuery(matchAllQuery())
-                .addScriptField("min", new Script("get min long", ScriptType.INLINE, CustomScriptPlugin.NAME, null))
+                .addScriptField("min", new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "get min long", Collections.emptyMap()))
                 .addSort(SortBuilders.fieldSort("ord").order(SortOrder.ASC).unmappedType("long"))
                 .setSize(10)
                 .get();
@@ -291,7 +291,7 @@ public void testSortMinValueScript() throws IOException {
         // test the double values
         searchResponse = client().prepareSearch()
                 .setQuery(matchAllQuery())
-                .addScriptField("min", new Script("get min double", ScriptType.INLINE, CustomScriptPlugin.NAME, null))
+                .addScriptField("min", new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "get min double", Collections.emptyMap()))
                 .addSort(SortBuilders.fieldSort("ord").order(SortOrder.ASC).unmappedType("long"))
                 .setSize(10)
                 .get();
@@ -307,7 +307,7 @@ public void testSortMinValueScript() throws IOException {
         // test the string values
         searchResponse = client().prepareSearch()
                 .setQuery(matchAllQuery())
-                .addScriptField("min", new Script("get min string", ScriptType.INLINE, CustomScriptPlugin.NAME, null))
+                .addScriptField("min", new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "get min string", Collections.emptyMap()))
                 .addSort(SortBuilders.fieldSort("ord").order(SortOrder.ASC).unmappedType("long"))
                 .setSize(10)
                 .get();
@@ -323,7 +323,8 @@ public void testSortMinValueScript() throws IOException {
         // test the geopoint values
         searchResponse = client().prepareSearch()
                 .setQuery(matchAllQuery())
-                .addScriptField("min", new Script("get min geopoint lon", ScriptType.INLINE, CustomScriptPlugin.NAME, null))
+                .addScriptField("min",
+                    new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "get min geopoint lon", Collections.emptyMap()))
                 .addSort(SortBuilders.fieldSort("ord").order(SortOrder.ASC).unmappedType("long"))
                 .setSize(10)
                 .get();
@@ -381,7 +382,7 @@ public void testDocumentsWithNullValue() throws Exception {
         flush();
         refresh();
 
-        Script scripField = new Script("doc['id'].value", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
+        Script scripField = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['id'].value", Collections.emptyMap());
 
         SearchResponse searchResponse = client().prepareSearch()
                 .setQuery(matchAllQuery())
@@ -398,7 +399,7 @@ public void testDocumentsWithNullValue() throws Exception {
 
         searchResponse = client().prepareSearch()
                 .setQuery(matchAllQuery())
-                .addScriptField("id", new Script("doc['id'].values[0]", ScriptType.INLINE, CustomScriptPlugin.NAME, null))
+                .addScriptField("id", new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['id'].values[0]", Collections.emptyMap()))
                 .addSort("svalue", SortOrder.ASC)
                 .get();
 
@@ -466,7 +467,7 @@ public void test2920() throws IOException {
         }
         refresh();
 
-        Script sortScript = new Script("\u0027\u0027", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
+        Script sortScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "\u0027\u0027", Collections.emptyMap());
         SearchResponse searchResponse = client().prepareSearch()
                 .setQuery(matchAllQuery())
                 .addSort(scriptSort(sortScript, ScriptSortType.STRING))
diff --git a/core/src/test/java/org/elasticsearch/search/sort/SortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/SortBuilderTests.java
index 9fa19146e8219..b1a6af88de133 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/SortBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/SortBuilderTests.java
@@ -19,18 +19,15 @@
 
 package org.elasticsearch.search.sort;
 
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.geo.GeoPoint;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.AfterClass;
@@ -44,24 +41,19 @@
 import static java.util.Collections.emptyList;
 
 public class SortBuilderTests extends ESTestCase {
-
     private static final int NUMBER_OF_RUNS = 20;
 
-    protected static NamedWriteableRegistry namedWriteableRegistry;
-
-    static IndicesQueriesRegistry indicesQueriesRegistry;
+    private static NamedXContentRegistry xContentRegistry;
 
     @BeforeClass
     public static void init() {
         SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList());
-        namedWriteableRegistry = new NamedWriteableRegistry(searchModule.getNamedWriteables());
-        indicesQueriesRegistry = searchModule.getQueryParserRegistry();
+        xContentRegistry = new NamedXContentRegistry(searchModule.getNamedXContents());
     }
 
     @AfterClass
     public static void afterClass() throws Exception {
-        namedWriteableRegistry = null;
-        indicesQueriesRegistry = null;
+        xContentRegistry = null;
     }
 
     /**
@@ -197,7 +189,7 @@ public static List<SortBuilder<?>> randomSortBuilderList() {
                 list.add(new ScoreSortBuilder());
                 break;
             case 1:
-                String fieldName = rarely() ? FieldSortBuilder.DOC_FIELD_NAME : randomAsciiOfLengthBetween(1, 10);
+                String fieldName = rarely() ? FieldSortBuilder.DOC_FIELD_NAME : randomAlphaOfLengthBetween(1, 10);
                 list.add(new FieldSortBuilder(fieldName));
                 break;
             case 2:
@@ -238,9 +230,14 @@ public void testMultiFieldSort() throws IOException {
         assertEquals(new ScoreSortBuilder(), result.get(5));
     }
 
-    private static List<SortBuilder<?>> parseSort(String jsonString) throws IOException {
-        XContentParser itemParser = XContentHelper.createParser(new BytesArray(jsonString));
-        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, itemParser, ParseFieldMatcher.STRICT);
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return xContentRegistry;
+    }
+
+    private List<SortBuilder<?>> parseSort(String jsonString) throws IOException {
+        XContentParser itemParser = createParser(JsonXContent.jsonXContent, jsonString);
+        QueryParseContext context = new QueryParseContext(itemParser);
 
         assertEquals(XContentParser.Token.START_OBJECT, itemParser.nextToken());
         assertEquals(XContentParser.Token.FIELD_NAME, itemParser.nextToken());
diff --git a/core/src/test/java/org/elasticsearch/search/sort/SortModeTests.java b/core/src/test/java/org/elasticsearch/search/sort/SortModeTests.java
index 29deb6dd76dab..e0329347dba6c 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/SortModeTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/SortModeTests.java
@@ -20,16 +20,11 @@
 package org.elasticsearch.search.sort;
 
 import org.elasticsearch.test.ESTestCase;
-import org.junit.Rule;
-import org.junit.rules.ExpectedException;
 
 import java.util.Locale;
 
 public class SortModeTests extends ESTestCase {
 
-    @Rule
-    public ExpectedException exceptionRule = ExpectedException.none();
-
     public void testSortMode() {
         // we rely on these ordinals in serialization, so changing them breaks bwc.
         assertEquals(0, SortMode.MIN.ordinal());
@@ -50,16 +45,11 @@ public void testSortMode() {
         }
     }
 
-    public void testParseNull() {
-        exceptionRule.expect(NullPointerException.class);
-        exceptionRule.expectMessage("input string is null");
-        SortMode.fromString(null);
-    }
+    public void testParsingFromStringExceptions() {
+        Exception e = expectThrows(NullPointerException.class, () -> SortMode.fromString(null));
+        assertEquals("input string is null", e.getMessage());
 
-    public void testIllegalArgument() {
-        exceptionRule.expect(IllegalArgumentException.class);
-        exceptionRule.expectMessage("Unknown SortMode [xyz]");
-        SortMode.fromString("xyz");
+        e = expectThrows(IllegalArgumentException.class, () -> SortMode.fromString("xyz"));
+        assertEquals("Unknown SortMode [xyz]", e.getMessage());
     }
-
 }
diff --git a/core/src/test/java/org/elasticsearch/search/stats/SearchStatsIT.java b/core/src/test/java/org/elasticsearch/search/stats/SearchStatsIT.java
index 1c296a3724a3a..11806a1cea9f6 100644
--- a/core/src/test/java/org/elasticsearch/search/stats/SearchStatsIT.java
+++ b/core/src/test/java/org/elasticsearch/search/stats/SearchStatsIT.java
@@ -34,7 +34,7 @@
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.MockScriptPlugin;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
 import org.elasticsearch.test.ESIntegTestCase;
 
@@ -121,7 +121,8 @@ public void testSimpleStats() throws Exception {
             SearchResponse searchResponse = internalCluster().coordOnlyNodeClient().prepareSearch()
                     .setQuery(QueryBuilders.termQuery("field", "value")).setStats("group1", "group2")
                     .highlighter(new HighlightBuilder().field("field"))
-                    .addScriptField("script1", new Script("_source.field", ScriptType.INLINE, CustomScriptPlugin.NAME, null))
+                    .addScriptField("script1",
+                        new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_source.field", Collections.emptyMap()))
                     .setSize(100)
                     .execute().actionGet();
             assertHitCount(searchResponse, docsTest1 + docsTest2);
@@ -164,10 +165,10 @@ public void testSimpleStats() throws Exception {
 
     private Set<String> nodeIdsWithIndex(String... indices) {
         ClusterState state = client().admin().cluster().prepareState().execute().actionGet().getState();
-        GroupShardsIterator allAssignedShardsGrouped = state.routingTable().allAssignedShardsGrouped(indices, true);
+        GroupShardsIterator<ShardIterator> allAssignedShardsGrouped = state.routingTable().allAssignedShardsGrouped(indices, true);
         Set<String> nodes = new HashSet<>();
         for (ShardIterator shardIterator : allAssignedShardsGrouped) {
-            for (ShardRouting routing : shardIterator.asUnordered()) {
+            for (ShardRouting routing : shardIterator) {
                 if (routing.active()) {
                     nodes.add(routing.currentNodeId());
                 }
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java b/core/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java
index d1fc671d322a8..7fd674486ed71 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java
@@ -19,20 +19,15 @@
 
 package org.elasticsearch.search.suggest;
 
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.AfterClass;
@@ -41,16 +36,13 @@
 import java.io.IOException;
 
 import static java.util.Collections.emptyList;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.not;
+import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
 
 public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBuilder<SB>> extends ESTestCase {
 
     private static final int NUMBER_OF_TESTBUILDERS = 20;
     protected static NamedWriteableRegistry namedWriteableRegistry;
-    protected static IndicesQueriesRegistry queriesRegistry;
-    protected static ParseFieldMatcher parseFieldMatcher;
-    protected static Suggesters suggesters;
+    protected static NamedXContentRegistry xContentRegistry;
 
     /**
      * setup for the whole base test class
@@ -59,16 +51,13 @@ public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBui
     public static void init() throws IOException {
         SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList());
         namedWriteableRegistry = new NamedWriteableRegistry(searchModule.getNamedWriteables());
-        queriesRegistry = searchModule.getQueryParserRegistry();
-        suggesters = searchModule.getSuggesters();
-        parseFieldMatcher = ParseFieldMatcher.STRICT;
+        xContentRegistry = new NamedXContentRegistry(searchModule.getNamedXContents());
     }
 
     @AfterClass
     public static void afterClass() throws Exception {
         namedWriteableRegistry = null;
-        suggesters = null;
-        queriesRegistry = null;
+        xContentRegistry = null;
     }
 
     /**
@@ -77,7 +66,7 @@ public static void afterClass() throws Exception {
     public void testSerialization() throws IOException {
         for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) {
             SB original = randomTestBuilder();
-            SB deserialized = serializedCopy(original);
+            SB deserialized = copy(original);
             assertEquals(deserialized, original);
             assertEquals(deserialized.hashCode(), original.hashCode());
             assertNotSame(deserialized, original);
@@ -93,10 +82,10 @@ protected SB randomTestBuilder() {
     }
 
     public static void setCommonPropertiesOnRandomBuilder(SuggestionBuilder<?> randomSuggestion) {
-        randomSuggestion.text(randomAsciiOfLengthBetween(2, 20)); // have to set the text because we don't know if the global text was set
-        maybeSet(randomSuggestion::prefix, randomAsciiOfLengthBetween(2, 20));
-        maybeSet(randomSuggestion::regex, randomAsciiOfLengthBetween(2, 20));
-        maybeSet(randomSuggestion::analyzer, randomAsciiOfLengthBetween(2, 20));
+        randomSuggestion.text(randomAlphaOfLengthBetween(2, 20)); // have to set the text because we don't know if the global text was set
+        maybeSet(randomSuggestion::prefix, randomAlphaOfLengthBetween(2, 20));
+        maybeSet(randomSuggestion::regex, randomAlphaOfLengthBetween(2, 20));
+        maybeSet(randomSuggestion::analyzer, randomAlphaOfLengthBetween(2, 20));
         maybeSet(randomSuggestion::size, randomIntBetween(1, 20));
         maybeSet(randomSuggestion::shardSize, randomIntBetween(1, 20));
     }
@@ -111,32 +100,7 @@ public static void setCommonPropertiesOnRandomBuilder(SuggestionBuilder<?> rando
      */
     public void testEqualsAndHashcode() throws IOException {
         for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) {
-            SB firstBuilder = randomTestBuilder();
-            assertFalse("suggestion builder is equal to null", firstBuilder.equals(null));
-            assertFalse("suggestion builder is equal to incompatible type", firstBuilder.equals(""));
-            assertTrue("suggestion builder is not equal to self", firstBuilder.equals(firstBuilder));
-            assertThat("same suggestion builder's hashcode returns different values if called multiple times", firstBuilder.hashCode(),
-                    equalTo(firstBuilder.hashCode()));
-        final SB mutate = mutate(firstBuilder);
-        assertThat("different suggestion builders should not be equal", mutate, not(equalTo(firstBuilder)));
-
-            SB secondBuilder = serializedCopy(firstBuilder);
-            assertTrue("suggestion builder is not equal to self", secondBuilder.equals(secondBuilder));
-            assertTrue("suggestion builder is not equal to its copy", firstBuilder.equals(secondBuilder));
-            assertTrue("equals is not symmetric", secondBuilder.equals(firstBuilder));
-            assertThat("suggestion builder copy's hashcode is different from original hashcode", secondBuilder.hashCode(),
-                    equalTo(firstBuilder.hashCode()));
-
-            SB thirdBuilder = serializedCopy(secondBuilder);
-            assertTrue("suggestion builder is not equal to self", thirdBuilder.equals(thirdBuilder));
-            assertTrue("suggestion builder is not equal to its copy", secondBuilder.equals(thirdBuilder));
-            assertThat("suggestion builder copy's hashcode is different from original hashcode", secondBuilder.hashCode(),
-                    equalTo(thirdBuilder.hashCode()));
-            assertTrue("equals is not transitive", firstBuilder.equals(thirdBuilder));
-            assertThat("suggestion builder copy's hashcode is different from original hashcode", firstBuilder.hashCode(),
-                    equalTo(thirdBuilder.hashCode()));
-            assertTrue("equals is not symmetric", thirdBuilder.equals(secondBuilder));
-            assertTrue("equals is not symmetric", thirdBuilder.equals(firstBuilder));
+            checkEqualsAndHashCode(randomTestBuilder(), this::copy, this::mutate);
         }
     }
 
@@ -156,12 +120,11 @@ public void testFromXContent() throws IOException {
             xContentBuilder.endObject();
 
             XContentBuilder shuffled = shuffleXContent(xContentBuilder, shuffleProtectedFields());
-            XContentParser parser = XContentHelper.createParser(shuffled.bytes());
-            QueryParseContext context = new QueryParseContext(queriesRegistry, parser, parseFieldMatcher);
+            XContentParser parser = createParser(shuffled);
             // we need to skip the start object and the name, those will be parsed by outer SuggestBuilder
             parser.nextToken();
 
-            SuggestionBuilder<?> secondSuggestionBuilder = SuggestionBuilder.fromXContent(context, suggesters);
+            SuggestionBuilder<?> secondSuggestionBuilder = SuggestionBuilder.fromXContent(parser);
             assertNotSame(suggestionBuilder, secondSuggestionBuilder);
             assertEquals(suggestionBuilder, secondSuggestionBuilder);
             assertEquals(suggestionBuilder.hashCode(), secondSuggestionBuilder.hashCode());
@@ -177,22 +140,22 @@ protected String[] shuffleProtectedFields() {
     }
 
     private SB mutate(SB firstBuilder) throws IOException {
-        SB mutation = serializedCopy(firstBuilder);
+        SB mutation = copy(firstBuilder);
         assertNotSame(mutation, firstBuilder);
         // change ither one of the shared SuggestionBuilder parameters, or delegate to the specific tests mutate method
         if (randomBoolean()) {
             switch (randomIntBetween(0, 5)) {
             case 0:
-                mutation.text(randomValueOtherThan(mutation.text(), () -> randomAsciiOfLengthBetween(2, 20)));
+                mutation.text(randomValueOtherThan(mutation.text(), () -> randomAlphaOfLengthBetween(2, 20)));
                 break;
             case 1:
-                mutation.prefix(randomValueOtherThan(mutation.prefix(), () -> randomAsciiOfLengthBetween(2, 20)));
+                mutation.prefix(randomValueOtherThan(mutation.prefix(), () -> randomAlphaOfLengthBetween(2, 20)));
                 break;
             case 2:
-                mutation.regex(randomValueOtherThan(mutation.regex(), () -> randomAsciiOfLengthBetween(2, 20)));
+                mutation.regex(randomValueOtherThan(mutation.regex(), () -> randomAlphaOfLengthBetween(2, 20)));
                 break;
             case 3:
-                mutation.analyzer(randomValueOtherThan(mutation.analyzer(), () -> randomAsciiOfLengthBetween(2, 20)));
+                mutation.analyzer(randomValueOtherThan(mutation.analyzer(), () -> randomAlphaOfLengthBetween(2, 20)));
                 break;
             case 4:
                 mutation.size(randomValueOtherThan(mutation.size(), () -> randomIntBetween(1, 20)));
@@ -214,18 +177,13 @@ private SB mutate(SB firstBuilder) throws IOException {
     protected abstract void mutateSpecificParameters(SB firstBuilder) throws IOException;
 
     @SuppressWarnings("unchecked")
-    protected SB serializedCopy(SB original) throws IOException {
-        try (BytesStreamOutput output = new BytesStreamOutput()) {
-            output.writeNamedWriteable(original);
-            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
-                return (SB) in.readNamedWriteable(SuggestionBuilder.class);
-            }
-        }
+    protected SB copy(SB original) throws IOException {
+        return copyWriteable(original, namedWriteableRegistry,
+                (Writeable.Reader<SB>) namedWriteableRegistry.getReader(SuggestionBuilder.class, original.getWriteableName()));
     }
 
-    protected static QueryParseContext newParseContext(final String xcontent) throws IOException {
-        XContentParser parser = XContentFactory.xContent(xcontent).createParser(xcontent);
-        final QueryParseContext parseContext = new QueryParseContext(queriesRegistry, parser, parseFieldMatcher);
-        return parseContext;
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return xContentRegistry;
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearch2xIT.java b/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearch2xIT.java
index 49ad540d956b0..c6602d3046e57 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearch2xIT.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearch2xIT.java
@@ -33,6 +33,7 @@
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.FieldMemoryStats;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -99,7 +100,7 @@ public void testSimple() throws Exception {
             client().prepareIndex(INDEX, TYPE, "" + i)
                 .setSource(jsonBuilder()
                     .startObject().startObject(FIELD)
-                    .field("input", input[i])
+                    .array("input", input[i])
                     .endObject()
                     .endObject()
                 )
@@ -758,7 +759,7 @@ public void testThatStatsAreWorking() throws Exception {
         // regexes
         IndicesStatsResponse regexFieldStats = client().admin().indices().prepareStats(INDEX)
             .setIndices(INDEX).setCompletion(true).setCompletionFields("*").get();
-        ObjectLongHashMap<String> fields = regexFieldStats.getIndex(INDEX).getPrimaries().completion.getFields();
+        FieldMemoryStats fields = regexFieldStats.getIndex(INDEX).getPrimaries().completion.getFields();
         long regexSizeInBytes = fields.get(FIELD) + fields.get(otherField);
         assertThat(regexSizeInBytes, is(totalSizeInBytes));
     }
@@ -941,7 +942,7 @@ private void createData(boolean optimize) throws IOException, InterruptedExcepti
             builders[i] = client().prepareIndex(INDEX, TYPE, "" + i)
                 .setSource(jsonBuilder()
                     .startObject().startObject(FIELD)
-                    .field("input", input[i])
+                    .array("input", input[i])
                     .field("output", surface[i])
                     .startObject("payload").field("id", i).endObject()
                     .field("weight", 1) // WE FORCEFULLY INDEX A BOGUS WEIGHT
@@ -955,7 +956,7 @@ private void createData(boolean optimize) throws IOException, InterruptedExcepti
             builders[i] = client().prepareIndex(INDEX, TYPE, "n" + i)
                 .setSource(jsonBuilder()
                     .startObject().startObject(FIELD)
-                    .field("input", input[i])
+                    .array("input", input[i])
                     .field("output", surface[i])
                     .startObject("payload").field("id", i).endObject()
                     .field("weight", weight[i])
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java b/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java
index 14fab9d72b2e5..e25d9325abb35 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java
@@ -18,7 +18,6 @@
  */
 package org.elasticsearch.search.suggest;
 
-import com.carrotsearch.hppc.ObjectLongHashMap;
 import com.carrotsearch.randomizedtesting.generators.RandomStrings;
 
 import org.apache.lucene.analysis.TokenStreamToAutomaton;
@@ -31,10 +30,10 @@
 import org.elasticsearch.action.admin.indices.segments.ShardSegments;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.elasticsearch.action.index.IndexRequestBuilder;
-import org.elasticsearch.action.search.ReduceSearchPhaseException;
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.FieldMemoryStats;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -58,7 +57,6 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
@@ -72,12 +70,10 @@
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllSuccessful;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHit;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasId;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasScore;
 import static org.hamcrest.Matchers.contains;
-import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
@@ -120,6 +116,36 @@ public void testPrefix() throws Exception {
         assertSuggestions("foo", prefix, "suggestion10", "suggestion9", "suggestion8", "suggestion7", "suggestion6");
     }
 
+    /**
+     * test that suggestion works if prefix is either provided via {@link CompletionSuggestionBuilder#text(String)} or
+     * {@link SuggestBuilder#setGlobalText(String)}
+     */
+    public void testTextAndGlobalText() throws Exception {
+        final CompletionMappingBuilder mapping = new CompletionMappingBuilder();
+        createIndexAndMapping(mapping);
+        int numDocs = 10;
+        List<IndexRequestBuilder> indexRequestBuilders = new ArrayList<>();
+        for (int i = 1; i <= numDocs; i++) {
+            indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "" + i).setSource(jsonBuilder().startObject().startObject(FIELD)
+                    .field("input", "suggestion" + i).field("weight", i).endObject().endObject()));
+        }
+        indexRandom(true, indexRequestBuilders);
+        CompletionSuggestionBuilder noText = SuggestBuilders.completionSuggestion(FIELD);
+        SearchResponse searchResponse = client().prepareSearch(INDEX)
+                .suggest(new SuggestBuilder().addSuggestion("foo", noText).setGlobalText("sugg")).execute().actionGet();
+        assertSuggestions(searchResponse, "foo", "suggestion10", "suggestion9", "suggestion8", "suggestion7", "suggestion6");
+
+        CompletionSuggestionBuilder withText = SuggestBuilders.completionSuggestion(FIELD).text("sugg");
+        searchResponse = client().prepareSearch(INDEX)
+                .suggest(new SuggestBuilder().addSuggestion("foo", withText)).execute().actionGet();
+        assertSuggestions(searchResponse, "foo", "suggestion10", "suggestion9", "suggestion8", "suggestion7", "suggestion6");
+
+        // test that suggestion text takes precedence over global text
+        searchResponse = client().prepareSearch(INDEX)
+                .suggest(new SuggestBuilder().addSuggestion("foo", withText).setGlobalText("bogus")).execute().actionGet();
+        assertSuggestions(searchResponse, "foo", "suggestion10", "suggestion9", "suggestion8", "suggestion7", "suggestion6");
+    }
+
     public void testRegex() throws Exception {
         final CompletionMappingBuilder mapping = new CompletionMappingBuilder();
         createIndexAndMapping(mapping);
@@ -209,7 +235,7 @@ public void testMixedCompletion() throws Exception {
             client().prepareSearch(INDEX, otherIndex).suggest(new SuggestBuilder().addSuggestion("foo", prefix))
                 .execute().actionGet();
             fail("querying on mixed completion suggester should throw an error");
-        } catch (ReduceSearchPhaseException e) {
+        } catch (SearchPhaseExecutionException e) {
             assertThat(e.getCause().getMessage(), containsString("detected mixed suggestion results"));
         }
     }
@@ -273,8 +299,8 @@ public void testSuggestDocument() throws Exception {
         for (CompletionSuggestion.Entry.Option option : options) {
             assertThat(option.getText().toString(), equalTo("suggestion" + id));
             assertSearchHit(option.getHit(), hasId("" + id));
-            assertSearchHit(option.getHit(), hasScore(((float) id)));
-            assertNotNull(option.getHit().source());
+            assertSearchHit(option.getHit(), hasScore(id));
+            assertNotNull(option.getHit().getSourceAsMap());
             id--;
         }
     }
@@ -308,8 +334,8 @@ public void testSuggestDocumentNoSource() throws Exception {
         for (CompletionSuggestion.Entry.Option option : options) {
             assertThat(option.getText().toString(), equalTo("suggestion" + id));
             assertSearchHit(option.getHit(), hasId("" + id));
-            assertSearchHit(option.getHit(), hasScore(((float) id)));
-            assertNull(option.getHit().source());
+            assertSearchHit(option.getHit(), hasScore(id));
+            assertNull(option.getHit().getSourceAsMap());
             id--;
         }
     }
@@ -345,9 +371,9 @@ public void testSuggestDocumentSourceFiltering() throws Exception {
         for (CompletionSuggestion.Entry.Option option : options) {
             assertThat(option.getText().toString(), equalTo("suggestion" + id));
             assertSearchHit(option.getHit(), hasId("" + id));
-            assertSearchHit(option.getHit(), hasScore(((float) id)));
-            assertNotNull(option.getHit().source());
-            Set<String> sourceFields = option.getHit().sourceAsMap().keySet();
+            assertSearchHit(option.getHit(), hasScore(id));
+            assertNotNull(option.getHit().getSourceAsMap());
+            Set<String> sourceFields = option.getHit().getSourceAsMap().keySet();
             assertThat(sourceFields, contains("a"));
             assertThat(sourceFields, not(contains("b")));
             id--;
@@ -807,7 +833,7 @@ public void testThatStatsAreWorking() throws Exception {
 
         // regexes
         IndicesStatsResponse regexFieldStats = client().admin().indices().prepareStats(INDEX).setIndices(INDEX).setCompletion(true).setCompletionFields("*").get();
-        ObjectLongHashMap<String> fields = regexFieldStats.getIndex(INDEX).getPrimaries().completion.getFields();
+        FieldMemoryStats fields = regexFieldStats.getIndex(INDEX).getPrimaries().completion.getFields();
         long regexSizeInBytes = fields.get(FIELD) + fields.get(otherField);
         assertThat(regexSizeInBytes, is(totalSizeInBytes));
     }
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestionOptionTests.java b/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestionOptionTests.java
new file mode 100644
index 0000000000000..df4c6898cc7d6
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestionOptionTests.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.suggest;
+
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.text.Text;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchHitTests;
+import org.elasticsearch.search.suggest.completion.CompletionSuggestion;
+import org.elasticsearch.search.suggest.completion.CompletionSuggestion.Entry.Option;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Predicate;
+
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+import static org.elasticsearch.test.XContentTestUtils.insertRandomFields;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
+
+public class CompletionSuggestionOptionTests extends ESTestCase {
+
+    public static Option createTestItem() {
+        Text text = new Text(randomAlphaOfLengthBetween(5, 15));
+        int docId = randomInt();
+        int numberOfContexts = randomIntBetween(0, 3);
+        Map<String, Set<CharSequence>> contexts = new HashMap<>();
+        for (int i = 0; i < numberOfContexts; i++) {
+            int numberOfValues = randomIntBetween(0, 3);
+            Set<CharSequence> values = new HashSet<>();
+            for (int v = 0; v < numberOfValues; v++) {
+                values.add(randomAlphaOfLengthBetween(5, 15));
+            }
+            contexts.put(randomAlphaOfLengthBetween(5, 15), values);
+        }
+        SearchHit hit = null;
+        float score = randomFloat();
+        if (randomBoolean()) {
+            hit = SearchHitTests.createTestItem(false);
+            score = hit.getScore();
+        }
+        Option option = new CompletionSuggestion.Entry.Option(docId, text, score, contexts);
+        option.setHit(hit);
+        return option;
+    }
+
+    public void testFromXContent() throws IOException {
+        doTestFromXContent(false);
+    }
+
+    public void testFromXContentWithRandomFields() throws IOException {
+        doTestFromXContent(true);
+    }
+
+    private void doTestFromXContent(boolean addRandomFields) throws IOException {
+        Option option = createTestItem();
+        XContentType xContentType = randomFrom(XContentType.values());
+        boolean humanReadable = randomBoolean();
+        BytesReference originalBytes = toShuffledXContent(option, xContentType, ToXContent.EMPTY_PARAMS, humanReadable);
+        BytesReference mutated;
+        if (addRandomFields) {
+            // "contexts" is an object consisting of key/array pairs, we shouldn't add anything random there
+            // also there can be inner search hits fields inside this option, we need to exclude another couple of paths
+            // where we cannot add random stuff
+            Predicate<String> excludeFilter = (path) -> (path.endsWith(CompletionSuggestion.Entry.Option.CONTEXTS.getPreferredName())
+                    || path.endsWith("highlight") || path.endsWith("fields") || path.contains("_source") || path.contains("inner_hits"));
+            mutated = insertRandomFields(xContentType, originalBytes, excludeFilter, random());
+        } else {
+            mutated = originalBytes;
+        }
+        Option parsed;
+        try (XContentParser parser = createParser(xContentType.xContent(), mutated)) {
+            parsed = Option.fromXContent(parser);
+            assertNull(parser.nextToken());
+        }
+        assertEquals(option.getText(), parsed.getText());
+        assertEquals(option.getHighlighted(), parsed.getHighlighted());
+        assertEquals(option.getScore(), parsed.getScore(), Float.MIN_VALUE);
+        assertEquals(option.collateMatch(), parsed.collateMatch());
+        assertEquals(option.getContexts(), parsed.getContexts());
+        assertToXContentEquivalent(originalBytes, toXContent(parsed, xContentType, humanReadable), xContentType);
+    }
+
+    public void testToXContent() throws IOException {
+        Map<String, Set<CharSequence>> contexts = Collections.singletonMap("key", Collections.singleton("value"));
+        CompletionSuggestion.Entry.Option option = new CompletionSuggestion.Entry.Option(1, new Text("someText"), 1.3f, contexts);
+        BytesReference xContent = toXContent(option, XContentType.JSON, randomBoolean());
+        assertEquals("{\"text\":\"someText\",\"score\":1.3,\"contexts\":{\"key\":[\"value\"]}}"
+                   , xContent.utf8ToString());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/ContextCompletionSuggestSearchIT.java b/core/src/test/java/org/elasticsearch/search/suggest/ContextCompletionSuggestSearchIT.java
index 9fae823c43b36..eb30c32cf6232 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/ContextCompletionSuggestSearchIT.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/ContextCompletionSuggestSearchIT.java
@@ -19,9 +19,9 @@
 package org.elasticsearch.search.suggest;
 
 import com.carrotsearch.randomizedtesting.generators.RandomStrings;
-
 import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
 import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.geo.GeoHashUtils;
 import org.elasticsearch.common.geo.GeoPoint;
@@ -29,6 +29,7 @@
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.suggest.CompletionSuggestSearchIT.CompletionMappingBuilder;
 import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder;
 import org.elasticsearch.search.suggest.completion.context.CategoryContextMapping;
@@ -51,6 +52,8 @@
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
+import static org.hamcrest.core.IsEqual.equalTo;
 
 @SuppressCodecs("*") // requires custom completion format
 public class ContextCompletionSuggestSearchIT extends ESIntegTestCase {
@@ -157,6 +160,27 @@ public void testContextFuzzy() throws Exception {
         assertSuggestions("foo", prefix, "sugxgestion9", "sugxgestion8", "sugxgestion7", "sugxgestion6", "sugxgestion5");
     }
 
+    public void testContextFilteringWorksWithUTF8Categories() throws Exception {
+        CategoryContextMapping contextMapping = ContextBuilder.category("cat").field("cat").build();
+        LinkedHashMap<String, ContextMapping> map = new LinkedHashMap<>(Collections.singletonMap("cat", contextMapping));
+        final CompletionMappingBuilder mapping = new CompletionMappingBuilder().context(map);
+        createIndexAndMapping(mapping);
+        IndexResponse indexResponse = client().prepareIndex(INDEX, TYPE, "1")
+                .setSource(jsonBuilder().startObject()
+                        .startObject(FIELD)
+                        .field("input", "suggestion")
+                        .endObject()
+                        .field("cat", "ctx\\u00e4")
+                        .endObject())
+                .get();
+        assertThat(indexResponse.status(), equalTo(RestStatus.CREATED));
+        assertNoFailures(client().admin().indices().prepareRefresh(INDEX).get());
+        CompletionSuggestionBuilder contextSuggestQuery = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg")
+                .contexts(Collections.singletonMap("cat",
+                        Collections.singletonList(CategoryQueryContext.builder().setCategory("ctx\\u00e4").build())));
+        assertSuggestions("foo", contextSuggestQuery, "suggestion");
+    }
+
     public void testSingleContextFiltering() throws Exception {
         CategoryContextMapping contextMapping = ContextBuilder.category("cat").field("cat").build();
         LinkedHashMap<String, ContextMapping> map = new LinkedHashMap<String, ContextMapping>(Collections.singletonMap("cat", contextMapping));
@@ -557,7 +581,8 @@ public void testGeoNeighbours() throws Exception {
     }
 
     public void testGeoField() throws Exception {
-
+//        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.V_5_0_0_alpha5);
+//        Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
         XContentBuilder mapping = jsonBuilder();
         mapping.startObject();
         mapping.startObject(TYPE);
@@ -605,7 +630,7 @@ public void testGeoField() throws Exception {
 
         refresh();
 
-        String suggestionName = randomAsciiOfLength(10);
+        String suggestionName = randomAlphaOfLength(10);
         CompletionSuggestionBuilder context = SuggestBuilders.completionSuggestion(FIELD).text("h").size(10)
                 .contexts(Collections.singletonMap("st", Collections.singletonList(GeoQueryContext.builder().setGeoPoint(new GeoPoint(52.52, 13.4)).build())));
         SearchResponse searchResponse = client().prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion(suggestionName, context)).get();
@@ -634,37 +659,42 @@ private void createIndexAndMappingAndSettings(Settings settings, CompletionMappi
                 .field("preserve_separators", completionMappingBuilder.preserveSeparators)
                 .field("preserve_position_increments", completionMappingBuilder.preservePositionIncrements);
 
+        List<String> categoryContextFields = new ArrayList<>();
         if (completionMappingBuilder.contextMappings != null) {
-            mapping = mapping.startArray("contexts");
+            mapping.startArray("contexts");
             for (Map.Entry<String, ContextMapping> contextMapping : completionMappingBuilder.contextMappings.entrySet()) {
-                mapping = mapping.startObject()
+                mapping.startObject()
                         .field("name", contextMapping.getValue().name())
                         .field("type", contextMapping.getValue().type().name());
                 switch (contextMapping.getValue().type()) {
                     case CATEGORY:
                         final String fieldName = ((CategoryContextMapping) contextMapping.getValue()).getFieldName();
                         if (fieldName != null) {
-                            mapping = mapping.field("path", fieldName);
+                            mapping.field("path", fieldName);
+                            categoryContextFields.add(fieldName);
                         }
                         break;
                     case GEO:
                         final String name = ((GeoContextMapping) contextMapping.getValue()).getFieldName();
-                        mapping = mapping
-                                .field("precision", ((GeoContextMapping) contextMapping.getValue()).getPrecision());
+                        mapping.field("precision", ((GeoContextMapping) contextMapping.getValue()).getPrecision());
                         if (name != null) {
                             mapping.field("path", name);
                         }
                         break;
                 }
 
-                mapping = mapping.endObject();
+                mapping.endObject();
             }
 
-            mapping = mapping.endArray();
+            mapping.endArray();
         }
-        mapping = mapping.endObject()
-                .endObject().endObject()
+        mapping.endObject();
+        for (String fieldName : categoryContextFields) {
+            mapping.startObject(fieldName)
+                .field("type", randomBoolean() ? "keyword" : "text")
                 .endObject();
+        }
+        mapping.endObject().endObject().endObject();
 
         assertAcked(client().admin().indices().prepareCreate(INDEX)
                 .setSettings(Settings.builder().put(indexSettings()).put(settings))
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/ContextSuggestSearch2xIT.java b/core/src/test/java/org/elasticsearch/search/suggest/ContextSuggestSearch2xIT.java
index a3d72f671f7ed..858dddc22e80a 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/ContextSuggestSearch2xIT.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/ContextSuggestSearch2xIT.java
@@ -119,7 +119,7 @@ public void testBasicGeo() throws Exception {
 
         client().admin().indices().prepareRefresh(INDEX).get();
 
-        String suggestionName = randomAsciiOfLength(10);
+        String suggestionName = randomAlphaOfLength(10);
         CompletionSuggestionBuilder context = SuggestBuilders.completionSuggestion(FIELD).text("h").size(10).contexts(
             new CompletionSuggestionBuilder.Contexts2x().addGeoLocation("st", 52.52, 13.4));
 
@@ -161,7 +161,7 @@ public void testMultiLevelGeo() throws Exception {
         client().admin().indices().prepareRefresh(INDEX).get();
 
         for (int precision = 1; precision <= 12; precision++) {
-            String suggestionName = randomAsciiOfLength(10);
+            String suggestionName = randomAlphaOfLength(10);
             CompletionSuggestionBuilder context = new CompletionSuggestionBuilder(FIELD).text("h").size(10)
                 .contexts(new CompletionSuggestionBuilder.Contexts2x().addGeoLocation("st", 52.529172, 13.407333, precision));
 
@@ -190,7 +190,7 @@ public void testMappingIdempotency() throws Exception {
             .startObject("context")
             .startObject("location")
             .field("type", "geo")
-            .field("precision", precision)
+            .array("precision", precision)
             .endObject()
             .endObject()
             .endObject().endObject()
@@ -209,7 +209,7 @@ public void testMappingIdempotency() throws Exception {
             .startObject("context")
             .startObject("location")
             .field("type", "geo")
-            .field("precision", precision)
+            .array("precision", precision)
             .endObject()
             .endObject()
             .endObject().endObject()
@@ -269,7 +269,7 @@ public void testGeoField() throws Exception {
 
         refresh();
 
-        String suggestionName = randomAsciiOfLength(10);
+        String suggestionName = randomAlphaOfLength(10);
         CompletionSuggestionBuilder context = SuggestBuilders.completionSuggestion(FIELD).text("h").size(10)
             .contexts(new CompletionSuggestionBuilder.Contexts2x()
             .addGeoLocation("st", 52.52, 13.4));
@@ -314,7 +314,7 @@ public void testSimpleGeo() throws Exception {
             { "pizza - treptow", "pizza", "food" } };
 
         for (int i = 0; i < locations.length; i++) {
-            XContentBuilder source = jsonBuilder().startObject().startObject(FIELD).field("input", input[i])
+            XContentBuilder source = jsonBuilder().startObject().startObject(FIELD).array("input", input[i])
                 .startObject("context").field("st", locations[i]).endObject().field("payload", locations[i])
                 .endObject().endObject();
             client().prepareIndex(INDEX, TYPE, "" + i).setSource(source).execute().actionGet();
@@ -343,7 +343,7 @@ public void testSimplePrefix() throws Exception {
             .addMapping(TYPE, createMapping(TYPE, ContextBuilder.category("st"))));
 
         for (int i = 0; i < HEROS.length; i++) {
-            XContentBuilder source = jsonBuilder().startObject().startObject(FIELD).field("input", HEROS[i])
+            XContentBuilder source = jsonBuilder().startObject().startObject(FIELD).array("input", HEROS[i])
                 .startObject("context").field("st", i%3).endObject()
                 .startObject("payload").field("group", i % 3).field("id", i).endObject()
                 .endObject().endObject();
@@ -376,12 +376,12 @@ public void testTypeCategoryIsActuallyCalledCategory() throws Exception {
         XContentBuilder doc1 = jsonBuilder();
         doc1.startObject().startObject("suggest_field")
             .field("input", "backpack_red")
-            .startObject("context").field("color", "red", "all_colors").endObject()
+            .startObject("context").array("color", "red", "all_colors").endObject()
             .endObject().endObject();
         XContentBuilder doc2 = jsonBuilder();
         doc2.startObject().startObject("suggest_field")
             .field("input", "backpack_green")
-            .startObject("context").field("color", "green", "all_colors").endObject()
+            .startObject("context").array("color", "green", "all_colors").endObject()
             .endObject().endObject();
 
         client().prepareIndex(INDEX, TYPE, "1")
@@ -451,7 +451,7 @@ public void testSimpleField() throws Exception {
             client().prepareIndex(INDEX, TYPE, "" + i)
                 .setSource(
                     jsonBuilder().startObject().field("category", Integer.toString(i % 3)).startObject(FIELD)
-                        .field("input", HEROS[i])
+                        .array("input", HEROS[i])
                         .startObject("context").endObject().field("payload", Integer.toString(i % 3))
                         .endObject().endObject()).execute().actionGet();
         }
@@ -508,7 +508,7 @@ public void testMultiValueField() throws Exception {
             client().prepareIndex(INDEX, TYPE, "" + i)
                 .setSource(
                     jsonBuilder().startObject().startArray("category").value(Integer.toString(i % 3)).value("other").endArray()
-                        .startObject(FIELD).field("input", HEROS[i]).startObject("context").endObject()
+                        .startObject(FIELD).array("input", HEROS[i]).startObject("context").endObject()
                         .field("payload", Integer.toString(i % 3)).endObject().endObject()).execute().actionGet();
         }
 
@@ -535,7 +535,7 @@ public void testMultiContext() throws Exception {
             client().prepareIndex(INDEX, TYPE, "" + i)
                 .setSource(
                     jsonBuilder().startObject().field("categoryA").value("" + (char) ('0' + (i % 3))).field("categoryB")
-                        .value("" + (char) ('A' + (i % 3))).startObject(FIELD).field("input", HEROS[i])
+                        .value("" + (char) ('A' + (i % 3))).startObject(FIELD).array("input", HEROS[i])
                         .startObject("context").endObject().field("payload", ((char) ('0' + (i % 3))) + "" + (char) ('A' + (i % 3)))
                         .endObject().endObject()).execute().actionGet();
         }
@@ -561,7 +561,7 @@ public void testMultiContextWithFuzzyLogic() throws Exception {
 
         for (int i = 0; i < HEROS.length; i++) {
             String source = jsonBuilder().startObject().field("categoryA", "" + (char) ('0' + (i % 3)))
-                .field("categoryB", "" + (char) ('a' + (i % 3))).startObject(FIELD).field("input", HEROS[i])
+                .field("categoryB", "" + (char) ('a' + (i % 3))).startObject(FIELD).array("input", HEROS[i])
                 .startObject("context").endObject().startObject("payload").field("categoryA", "" + (char) ('0' + (i % 3)))
                 .field("categoryB", "" + (char) ('a' + (i % 3))).endObject().endObject().endObject().string();
             client().prepareIndex(INDEX, TYPE, "" + i).setSource(source).execute().actionGet();
@@ -599,7 +599,7 @@ public void testSimpleType() throws Exception {
             String type = types[i % types.length];
             client().prepareIndex(INDEX, type, "" + i)
                 .setSource(
-                    jsonBuilder().startObject().startObject(FIELD).field("input", HEROS[i])
+                    jsonBuilder().startObject().startObject(FIELD).array("input", HEROS[i])
                         .startObject("context").endObject().field("payload", type).endObject().endObject()).execute().actionGet();
         }
 
@@ -878,7 +878,7 @@ public void testThatPrecisionIsRequired() throws Exception {
     }
 
     public void assertGeoSuggestionsInRange(String location, String suggest, double precision) throws IOException {
-        String suggestionName = randomAsciiOfLength(10);
+        String suggestionName = randomAlphaOfLength(10);
         CompletionSuggestionBuilder context = SuggestBuilders.completionSuggestion(FIELD).text(suggest).size(10)
             .contexts(new CompletionSuggestionBuilder.Contexts2x().addGeoLocation("st", location));
         SearchRequestBuilder suggestionRequest = client().prepareSearch(INDEX)
@@ -902,7 +902,7 @@ public void assertGeoSuggestionsInRange(String location, String suggest, double
     }
 
     public void assertPrefixSuggestions(long prefix, String suggest, String... hits) throws IOException {
-        String suggestionName = randomAsciiOfLength(10);
+        String suggestionName = randomAlphaOfLength(10);
         CompletionSuggestionBuilder context = SuggestBuilders.completionSuggestion(FIELD).text(suggest)
             .size(hits.length + 1).contexts(new CompletionSuggestionBuilder.Contexts2x()
                 .addCategory("st", Long.toString(prefix)));
@@ -929,7 +929,7 @@ public void assertPrefixSuggestions(long prefix, String suggest, String... hits)
 
     public void assertContextWithFuzzySuggestions(String[] prefix1, String[] prefix2, String suggest, String... hits)
         throws IOException {
-        String suggestionName = randomAsciiOfLength(10);
+        String suggestionName = randomAlphaOfLength(10);
         CompletionSuggestionBuilder context = SuggestBuilders.completionSuggestion(FIELD).prefix(suggest, Fuzziness.TWO)
             .size(hits.length + 10).contexts(
                 new CompletionSuggestionBuilder.Contexts2x().addContextField("st", prefix1).addContextField("nd", prefix2));
@@ -959,7 +959,7 @@ public void assertContextWithFuzzySuggestions(String[] prefix1, String[] prefix2
     }
 
     public void assertFieldSuggestions(String value, String suggest, String... hits) throws IOException {
-        String suggestionName = randomAsciiOfLength(10);
+        String suggestionName = randomAlphaOfLength(10);
         CompletionSuggestionBuilder context = SuggestBuilders.completionSuggestion(FIELD).text(suggest).size(10)
             .contexts(new CompletionSuggestionBuilder.Contexts2x().addContextField("st", value));
         SearchRequestBuilder suggestionRequest = client().prepareSearch(INDEX).suggest(
@@ -985,7 +985,7 @@ public void assertFieldSuggestions(String value, String suggest, String... hits)
     }
 
     public void assertDoubleFieldSuggestions(String field1, String field2, String suggest, String... hits) throws IOException {
-        String suggestionName = randomAsciiOfLength(10);
+        String suggestionName = randomAlphaOfLength(10);
         CompletionSuggestionBuilder context = SuggestBuilders.completionSuggestion(FIELD).text(suggest).size(10)
             .contexts(new CompletionSuggestionBuilder.Contexts2x()
             .addContextField("st", field1).addContextField("nd", field2));
@@ -1011,7 +1011,7 @@ public void assertDoubleFieldSuggestions(String field1, String field2, String su
     }
 
     public void assertMultiContextSuggestions(String value1, String value2, String suggest, String... hits) throws IOException {
-        String suggestionName = randomAsciiOfLength(10);
+        String suggestionName = randomAlphaOfLength(10);
         CompletionSuggestionBuilder context = SuggestBuilders.completionSuggestion(FIELD).text(suggest).size(10)
             .contexts(new CompletionSuggestionBuilder.Contexts2x()
             .addContextField("st", value1).addContextField("nd", value2));
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggester.java b/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggester.java
index 48b570e880a97..63c8aa1e66816 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggester.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggester.java
@@ -20,11 +20,8 @@
 
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.util.CharsRefBuilder;
-import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.text.Text;
-import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.search.suggest.CustomSuggesterSearchIT.CustomSuggestionBuilder;
 
 import java.io.IOException;
 import java.util.Locale;
@@ -63,14 +60,4 @@ public CustomSuggestionsContext(QueryShardContext context, Map<String, Object> o
             this.options = options;
         }
     }
-
-    @Override
-    public SuggestionBuilder<?> innerFromXContent(QueryParseContext context) throws IOException {
-        return CustomSuggestionBuilder.innerFromXContent(context);
-    }
-
-    @Override
-    public SuggestionBuilder<?> read(StreamInput in) throws IOException {
-        return new CustomSuggestionBuilder(in);
-    }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggesterPlugin.java b/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggesterPlugin.java
deleted file mode 100644
index 583ac365fad9e..0000000000000
--- a/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggesterPlugin.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.search.suggest;
-
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.plugins.SearchPlugin;
-
-import java.util.Map;
-
-import static java.util.Collections.singletonMap;
-
-public class CustomSuggesterPlugin extends Plugin implements SearchPlugin {
-    @Override
-    public Map<String, Suggester<?>> getSuggesters() {
-        return singletonMap("custom", CustomSuggester.INSTANCE);
-    }
-}
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggesterSearchIT.java b/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggesterSearchIT.java
index 38d2a3976dc39..d5eee4d3dee7c 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggesterSearchIT.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggesterSearchIT.java
@@ -21,7 +21,6 @@
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -29,9 +28,9 @@
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.SearchPlugin;
 import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
@@ -46,6 +45,7 @@
 import java.util.Map;
 import java.util.Objects;
 
+import static java.util.Collections.singletonList;
 import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.hamcrest.Matchers.hasSize;
@@ -66,6 +66,14 @@ protected Collection<Class<? extends Plugin>> transportClientPlugins() {
         return Arrays.asList(CustomSuggesterPlugin.class);
     }
 
+    public static class CustomSuggesterPlugin extends Plugin implements SearchPlugin {
+        @Override
+        public List<SuggesterSpec<?>> getSuggesters() {
+            return singletonList(new SuggesterSpec<CustomSuggestionBuilder>("custom", CustomSuggestionBuilder::new,
+                    CustomSuggestionBuilder::fromXContent));
+        }
+    }
+
     public void testThatCustomSuggestersCanBeRegisteredAndWork() throws Exception {
         createIndex("test");
         client().prepareIndex("test", "test", "1").setSource(jsonBuilder()
@@ -74,9 +82,9 @@ public void testThatCustomSuggestersCanBeRegisteredAndWork() throws Exception {
                 .endObject())
                 .setRefreshPolicy(IMMEDIATE).get();
 
-        String randomText = randomAsciiOfLength(10);
-        String randomField = randomAsciiOfLength(10);
-        String randomSuffix = randomAsciiOfLength(10);
+        String randomText = randomAlphaOfLength(10);
+        String randomField = randomAlphaOfLength(10);
+        String randomSuffix = randomAlphaOfLength(10);
         SuggestBuilder suggestBuilder = new SuggestBuilder();
         suggestBuilder.addSuggestion("someName", new CustomSuggestionBuilder(randomField, randomSuffix).text(randomText));
         SearchRequestBuilder searchRequestBuilder = client().prepareSearch("test").setTypes("test").setFrom(0).setSize(1)
@@ -140,9 +148,7 @@ protected int doHashCode() {
             return Objects.hash(randomSuffix);
         }
 
-        static CustomSuggestionBuilder innerFromXContent(QueryParseContext parseContext) throws IOException {
-            XContentParser parser = parseContext.parser();
-            ParseFieldMatcher parseFieldMatcher = parseContext.getParseFieldMatcher();
+        public static CustomSuggestionBuilder fromXContent(XContentParser parser) throws IOException {
             XContentParser.Token token;
             String currentFieldName = null;
             String fieldname = null;
@@ -154,15 +160,15 @@ static CustomSuggestionBuilder innerFromXContent(QueryParseContext parseContext)
                 if (token == XContentParser.Token.FIELD_NAME) {
                     currentFieldName = parser.currentName();
                 } else if (token.isValue()) {
-                    if (parseFieldMatcher.match(currentFieldName, SuggestionBuilder.ANALYZER_FIELD)) {
+                    if (SuggestionBuilder.ANALYZER_FIELD.match(currentFieldName)) {
                         analyzer = parser.text();
-                    } else if (parseFieldMatcher.match(currentFieldName, SuggestionBuilder.FIELDNAME_FIELD)) {
+                    } else if (SuggestionBuilder.FIELDNAME_FIELD.match(currentFieldName)) {
                         fieldname = parser.text();
-                    } else if (parseFieldMatcher.match(currentFieldName, SuggestionBuilder.SIZE_FIELD)) {
+                    } else if (SuggestionBuilder.SIZE_FIELD.match(currentFieldName)) {
                         sizeField = parser.intValue();
-                    } else if (parseFieldMatcher.match(currentFieldName, SuggestionBuilder.SHARDSIZE_FIELD)) {
+                    } else if (SuggestionBuilder.SHARDSIZE_FIELD.match(currentFieldName)) {
                         shardSize = parser.intValue();
-                    } else if (parseFieldMatcher.match(currentFieldName, RANDOM_SUFFIX_FIELD)) {
+                    } else if (RANDOM_SUFFIX_FIELD.match(currentFieldName)) {
                         suffix = parser.text();
                     }
                 } else {
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/SuggestBuilderTests.java b/core/src/test/java/org/elasticsearch/search/suggest/SuggestBuilderTests.java
index badcce9250ffd..7c63b59ffc61a 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/SuggestBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/SuggestBuilderTests.java
@@ -19,23 +19,20 @@
 
 package org.elasticsearch.search.suggest;
 
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.search.suggest.completion.CompletionSuggesterBuilderTests;
-import org.elasticsearch.search.suggest.completion.WritableTestCase;
 import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilderTests;
 import org.elasticsearch.search.suggest.term.TermSuggestionBuilderTests;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.EqualsHashCodeTestUtils;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
@@ -43,11 +40,13 @@
 import java.util.Map.Entry;
 
 import static java.util.Collections.emptyList;
+import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
 
-public class SuggestBuilderTests extends WritableTestCase<SuggestBuilder> {
+public class SuggestBuilderTests extends ESTestCase {
 
+    private static final int NUMBER_OF_RUNS = 20;
     private static NamedWriteableRegistry namedWriteableRegistry;
-    private static Suggesters suggesters;
+    private static NamedXContentRegistry xContentRegistry;
 
     /**
      * Setup for the whole base test class.
@@ -56,18 +55,13 @@ public class SuggestBuilderTests extends WritableTestCase<SuggestBuilder> {
     public static void init() {
         SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList());
         namedWriteableRegistry = new NamedWriteableRegistry(searchModule.getNamedWriteables());
-        suggesters = searchModule.getSuggesters();
+        xContentRegistry = new NamedXContentRegistry(searchModule.getNamedXContents());
     }
 
     @AfterClass
     public static void afterClass() {
         namedWriteableRegistry = null;
-        suggesters = null;
-    }
-
-    @Override
-    protected NamedWriteableRegistry provideNamedWritableRegistry() {
-        return namedWriteableRegistry;
+        xContentRegistry = null;
     }
 
     /**
@@ -75,21 +69,45 @@ protected NamedWriteableRegistry provideNamedWritableRegistry() {
      */
     public void testFromXContent() throws IOException {
         for (int runs = 0; runs < NUMBER_OF_RUNS; runs++) {
-            SuggestBuilder suggestBuilder = createTestModel();
+            SuggestBuilder suggestBuilder = randomSuggestBuilder();
             XContentBuilder xContentBuilder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
             if (randomBoolean()) {
                 xContentBuilder.prettyPrint();
             }
             suggestBuilder.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS);
-            XContentParser parser = XContentHelper.createParser(xContentBuilder.bytes());
-            QueryParseContext context = new QueryParseContext(new IndicesQueriesRegistry(), parser, ParseFieldMatcher.STRICT);
-            SuggestBuilder secondSuggestBuilder = SuggestBuilder.fromXContent(context, suggesters);
+            XContentParser parser = createParser(xContentBuilder);
+            SuggestBuilder secondSuggestBuilder = SuggestBuilder.fromXContent(parser);
             assertNotSame(suggestBuilder, secondSuggestBuilder);
             assertEquals(suggestBuilder, secondSuggestBuilder);
             assertEquals(suggestBuilder.hashCode(), secondSuggestBuilder.hashCode());
         }
     }
 
+    /**
+     * Test equality and hashCode properties
+     */
+    public void testEqualsAndHashcode() throws IOException {
+        for (int runs = 0; runs < NUMBER_OF_RUNS; runs++) {
+            // explicit about type parameters, see: https://bugs.eclipse.org/bugs/show_bug.cgi?id=481649
+            EqualsHashCodeTestUtils.<SuggestBuilder>checkEqualsAndHashCode(randomSuggestBuilder(), original -> {
+                return copyWriteable(original, namedWriteableRegistry, SuggestBuilder::new);
+            }, this::createMutation);
+        }
+    }
+
+    /**
+     * Test serialization and deserialization
+     */
+    public void testSerialization() throws IOException {
+        for (int i = 0; i < NUMBER_OF_RUNS; i++) {
+            SuggestBuilder suggestBuilder = randomSuggestBuilder();
+            SuggestBuilder deserializedModel = copyWriteable(suggestBuilder, namedWriteableRegistry, SuggestBuilder::new);
+            assertEquals(suggestBuilder, deserializedModel);
+            assertEquals(suggestBuilder.hashCode(), deserializedModel.hashCode());
+            assertNotSame(suggestBuilder, deserializedModel);
+        }
+    }
+
     public void testIllegalSuggestionName() {
         try {
             new SuggestBuilder().addSuggestion(null, PhraseSuggestionBuilderTests.randomPhraseSuggestionBuilder());
@@ -107,38 +125,27 @@ public void testIllegalSuggestionName() {
         }
     }
 
-    @Override
-    protected SuggestBuilder createTestModel() {
-        return randomSuggestBuilder();
-    }
-
-    @Override
     protected SuggestBuilder createMutation(SuggestBuilder original) throws IOException {
         SuggestBuilder mutation = new SuggestBuilder().setGlobalText(original.getGlobalText());
         for (Entry<String, SuggestionBuilder<?>> suggestionBuilder : original.getSuggestions().entrySet()) {
             mutation.addSuggestion(suggestionBuilder.getKey(), suggestionBuilder.getValue());
         }
         if (randomBoolean()) {
-            mutation.setGlobalText(randomAsciiOfLengthBetween(5, 60));
+            mutation.setGlobalText(randomAlphaOfLengthBetween(5, 60));
         } else {
-            mutation.addSuggestion(randomAsciiOfLength(10), PhraseSuggestionBuilderTests.randomPhraseSuggestionBuilder());
+            mutation.addSuggestion(randomAlphaOfLength(10), PhraseSuggestionBuilderTests.randomPhraseSuggestionBuilder());
         }
         return mutation;
     }
 
-    @Override
-    protected SuggestBuilder readFrom(StreamInput in) throws IOException {
-        return new SuggestBuilder(in);
-    }
-
     public static SuggestBuilder randomSuggestBuilder() {
         SuggestBuilder builder = new SuggestBuilder();
         if (randomBoolean()) {
-            builder.setGlobalText(randomAsciiOfLengthBetween(1, 20));
+            builder.setGlobalText(randomAlphaOfLengthBetween(1, 20));
         }
         final int numSuggestions = randomIntBetween(1, 5);
         for (int i = 0; i < numSuggestions; i++) {
-            builder.addSuggestion(randomAsciiOfLengthBetween(5, 10), randomSuggestionBuilder());
+            builder.addSuggestion(randomAlphaOfLengthBetween(5, 10), randomSuggestionBuilder());
         }
         return builder;
     }
@@ -152,4 +159,8 @@ private static SuggestionBuilder<?> randomSuggestionBuilder() {
         }
     }
 
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return xContentRegistry;
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/SuggestSearchIT.java b/core/src/test/java/org/elasticsearch/search/suggest/SuggestSearchIT.java
index b10272329e549..1881ce0e6d2ec 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/SuggestSearchIT.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/SuggestSearchIT.java
@@ -141,7 +141,7 @@ public void testSuggestAcrossMultipleIndices() throws IOException {
         try {
             searchSuggest("test", termSuggest);
             fail(" can not suggest across multiple indices with different analysis chains");
-        } catch (ReduceSearchPhaseException ex) {
+        } catch (SearchPhaseExecutionException ex) {
             assertThat(ex.getCause(), instanceOf(IllegalStateException.class));
             assertThat(ex.getCause().getMessage(),
                     anyOf(endsWith("Suggest entries have different sizes actual [1] expected [2]"),
@@ -160,7 +160,7 @@ public void testSuggestAcrossMultipleIndices() throws IOException {
         try {
             searchSuggest("test", termSuggest);
             fail(" can not suggest across multiple indices with different analysis chains");
-        } catch (ReduceSearchPhaseException ex) {
+        } catch (SearchPhaseExecutionException ex) {
             assertThat(ex.getCause(), instanceOf(IllegalStateException.class));
             assertThat(ex.getCause().getMessage(), anyOf(endsWith("Suggest entries have different text actual [ABCD] expected [abcd]"),
                     endsWith("Suggest entries have different text actual [abcd] expected [ABCD]")));
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/SuggestTests.java b/core/src/test/java/org/elasticsearch/search/suggest/SuggestTests.java
index bed6d5997d73f..f1a630fab3742 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/SuggestTests.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/SuggestTests.java
@@ -19,25 +19,129 @@
 
 package org.elasticsearch.search.suggest;
 
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.text.Text;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.rest.action.search.RestSearchAction;
+import org.elasticsearch.search.suggest.Suggest.Suggestion;
+import org.elasticsearch.search.suggest.Suggest.Suggestion.Entry;
+import org.elasticsearch.search.suggest.Suggest.Suggestion.Entry.Option;
 import org.elasticsearch.search.suggest.completion.CompletionSuggestion;
 import org.elasticsearch.search.suggest.phrase.PhraseSuggestion;
 import org.elasticsearch.search.suggest.term.TermSuggestion;
 import org.elasticsearch.test.ESTestCase;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureFieldName;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
 import static org.hamcrest.Matchers.equalTo;
 
 public class SuggestTests extends ESTestCase {
 
+    private static final NamedXContentRegistry xContentRegistry;
+    private static final List<NamedXContentRegistry.Entry> namedXContents;
+
+    static {
+        namedXContents = new ArrayList<>();
+        namedXContents.add(new NamedXContentRegistry.Entry(Suggest.Suggestion.class, new ParseField("term"),
+                (parser, context) -> TermSuggestion.fromXContent(parser, (String)context)));
+        namedXContents.add(new NamedXContentRegistry.Entry(Suggest.Suggestion.class, new ParseField("phrase"),
+                (parser, context) -> PhraseSuggestion.fromXContent(parser, (String)context)));
+        namedXContents.add(new NamedXContentRegistry.Entry(Suggest.Suggestion.class, new ParseField("completion"),
+                (parser, context) -> CompletionSuggestion.fromXContent(parser, (String)context)));
+        xContentRegistry = new NamedXContentRegistry(namedXContents);
+    }
+
+    public static List<NamedXContentRegistry.Entry> getDefaultNamedXContents() {
+        return namedXContents;
+    }
+
+    static NamedXContentRegistry getSuggestersRegistry() {
+        return xContentRegistry;
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return getSuggestersRegistry();
+    }
+
+    public static Suggest createTestItem() {
+        int numEntries = randomIntBetween(0, 5);
+        List<Suggestion<? extends Entry<? extends Option>>> suggestions = new ArrayList<>();
+        for (int i = 0; i < numEntries; i++) {
+            suggestions.add(SuggestionTests.createTestItem());
+        }
+        return new Suggest(suggestions);
+    }
+
+    public void testFromXContent() throws IOException {
+        ToXContent.Params params = new ToXContent.MapParams(Collections.singletonMap(RestSearchAction.TYPED_KEYS_PARAM, "true"));
+        Suggest suggest = createTestItem();
+        XContentType xContentType = randomFrom(XContentType.values());
+        boolean humanReadable = randomBoolean();
+        BytesReference originalBytes = toShuffledXContent(suggest, xContentType, params, humanReadable);
+        Suggest parsed;
+        try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) {
+            ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
+            ensureFieldName(parser, parser.nextToken(), Suggest.NAME);
+            ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
+            parsed = Suggest.fromXContent(parser);
+            assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());
+            assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
+            assertNull(parser.nextToken());
+        }
+        assertEquals(suggest.size(), parsed.size());
+        for (Suggestion suggestion : suggest) {
+            Suggestion<? extends Entry<? extends Option>> parsedSuggestion = parsed.getSuggestion(suggestion.getName());
+            assertNotNull(parsedSuggestion);
+            assertEquals(suggestion.getClass(), parsedSuggestion.getClass());
+        }
+        assertToXContentEquivalent(originalBytes, toXContent(parsed, xContentType, params, humanReadable), xContentType);
+    }
+
+    public void testToXContent() throws IOException {
+        Option option = new Option(new Text("someText"), new Text("somethingHighlighted"), 1.3f, true);
+        Entry<Option> entry = new Entry<>(new Text("entryText"), 42, 313);
+        entry.addOption(option);
+        Suggestion<Entry<Option>> suggestion = new Suggestion<>("suggestionName", 5);
+        suggestion.addTerm(entry);
+        Suggest suggest = new Suggest(Collections.singletonList(suggestion));
+        BytesReference xContent = toXContent(suggest, XContentType.JSON, randomBoolean());
+        assertEquals(
+                "{\"suggest\":"
+                        + "{\"suggestionName\":"
+                            + "[{\"text\":\"entryText\","
+                            + "\"offset\":42,"
+                            + "\"length\":313,"
+                            + "\"options\":[{\"text\":\"someText\","
+                                        + "\"highlighted\":\"somethingHighlighted\","
+                                        + "\"score\":1.3,"
+                                        + "\"collate_match\":true}]"
+                            + "}]"
+                        + "}"
+                +"}",
+                xContent.utf8ToString());
+    }
+
     public void testFilter() throws Exception {
         List<Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>>> suggestions;
-        CompletionSuggestion completionSuggestion = new CompletionSuggestion(randomAsciiOfLength(10), 2);
-        PhraseSuggestion phraseSuggestion = new PhraseSuggestion(randomAsciiOfLength(10), 2);
-        TermSuggestion termSuggestion = new TermSuggestion(randomAsciiOfLength(10), 2, SortBy.SCORE);
+        CompletionSuggestion completionSuggestion = new CompletionSuggestion(randomAlphaOfLength(10), 2);
+        PhraseSuggestion phraseSuggestion = new PhraseSuggestion(randomAlphaOfLength(10), 2);
+        TermSuggestion termSuggestion = new TermSuggestion(randomAlphaOfLength(10), 2, SortBy.SCORE);
         suggestions = Arrays.asList(completionSuggestion, phraseSuggestion, termSuggestion);
         Suggest suggest = new Suggest(suggestions);
         List<PhraseSuggestion> phraseSuggestions = suggest.filter(PhraseSuggestion.class);
@@ -56,7 +160,7 @@ public void testSuggestionOrdering() throws Exception {
         suggestions = new ArrayList<>();
         int n = randomIntBetween(2, 5);
         for (int i = 0; i < n; i++) {
-            suggestions.add(new CompletionSuggestion(randomAsciiOfLength(10), randomIntBetween(3, 5)));
+            suggestions.add(new CompletionSuggestion(randomAlphaOfLength(10), randomIntBetween(3, 5)));
         }
         Collections.shuffle(suggestions, random());
         Suggest suggest = new Suggest(suggestions);
@@ -70,4 +174,22 @@ public void testSuggestionOrdering() throws Exception {
         }
     }
 
+
+    public void testParsingExceptionOnUnknownSuggestion() throws IOException {
+        XContentBuilder builder = XContentFactory.jsonBuilder();
+        builder.startObject();
+        {
+            builder.startArray("unknownSuggestion");
+            builder.endArray();
+        }
+        builder.endObject();
+        BytesReference originalBytes = builder.bytes();
+        try (XContentParser parser = createParser(builder.contentType().xContent(), originalBytes)) {
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            ParsingException ex = expectThrows(ParsingException.class, () -> Suggest.fromXContent(parser));
+            assertEquals("Could not parse suggestion keyed as [unknownSuggestion]", ex.getMessage());
+        }
+    }
+
+
 }
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/SuggestionEntryTests.java b/core/src/test/java/org/elasticsearch/search/suggest/SuggestionEntryTests.java
new file mode 100644
index 0000000000000..46971a5537b26
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/suggest/SuggestionEntryTests.java
@@ -0,0 +1,180 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.suggest;
+
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.text.Text;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.search.suggest.Suggest.Suggestion.Entry;
+import org.elasticsearch.search.suggest.Suggest.Suggestion.Entry.Option;
+import org.elasticsearch.search.suggest.completion.CompletionSuggestion;
+import org.elasticsearch.search.suggest.phrase.PhraseSuggestion;
+import org.elasticsearch.search.suggest.term.TermSuggestion;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.function.Predicate;
+import java.util.function.Supplier;
+
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.elasticsearch.test.XContentTestUtils.insertRandomFields;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
+
+public class SuggestionEntryTests extends ESTestCase {
+
+    private static final Map<Class<? extends Entry>, Function<XContentParser, ? extends Entry>> ENTRY_PARSERS = new HashMap<>();
+    static {
+        ENTRY_PARSERS.put(TermSuggestion.Entry.class, TermSuggestion.Entry::fromXContent);
+        ENTRY_PARSERS.put(PhraseSuggestion.Entry.class, PhraseSuggestion.Entry::fromXContent);
+        ENTRY_PARSERS.put(CompletionSuggestion.Entry.class, CompletionSuggestion.Entry::fromXContent);
+    }
+
+    /**
+     * Create a randomized Suggestion.Entry
+     */
+    @SuppressWarnings("unchecked")
+    public static <O extends Option> Entry<O> createTestItem(Class<? extends Entry> entryType) {
+        Text entryText = new Text(randomAlphaOfLengthBetween(5, 15));
+        int offset = randomInt();
+        int length = randomInt();
+        Entry entry;
+        Supplier<Option> supplier;
+        if (entryType == TermSuggestion.Entry.class) {
+            entry = new TermSuggestion.Entry(entryText, offset, length);
+            supplier = TermSuggestionOptionTests::createTestItem;
+        } else if (entryType == PhraseSuggestion.Entry.class) {
+            entry = new PhraseSuggestion.Entry(entryText, offset, length, randomDouble());
+            supplier = SuggestionOptionTests::createTestItem;
+        } else if (entryType == CompletionSuggestion.Entry.class) {
+            entry = new CompletionSuggestion.Entry(entryText, offset, length);
+            supplier = CompletionSuggestionOptionTests::createTestItem;
+        } else {
+            throw new UnsupportedOperationException("entryType not supported [" + entryType + "]");
+        }
+        int numOptions = randomIntBetween(0, 5);
+        for (int i = 0; i < numOptions; i++) {
+            entry.addOption(supplier.get());
+        }
+        return entry;
+    }
+
+    public void testFromXContent() throws IOException {
+        doTestFromXContent(false);
+    }
+
+    public void testFromXContentWithRandomFields() throws IOException {
+        doTestFromXContent(true);
+    }
+
+    @SuppressWarnings("unchecked")
+    private void doTestFromXContent(boolean addRandomFields) throws IOException {
+        for (Class<? extends Entry> entryType : ENTRY_PARSERS.keySet()) {
+            Entry<Option> entry = createTestItem(entryType);
+            XContentType xContentType = randomFrom(XContentType.values());
+            boolean humanReadable = randomBoolean();
+            BytesReference originalBytes = toShuffledXContent(entry, xContentType, ToXContent.EMPTY_PARAMS, humanReadable);
+            BytesReference mutated;
+            if (addRandomFields) {
+                // "contexts" is an object consisting of key/array pairs, we shouldn't add anything random there
+                // also there can be inner search hits fields inside this option, we need to exclude another couple of paths
+                // where we cannot add random stuff
+                Predicate<String> excludeFilter = (
+                        path) -> (path.endsWith(CompletionSuggestion.Entry.Option.CONTEXTS.getPreferredName()) || path.endsWith("highlight")
+                                || path.endsWith("fields") || path.contains("_source") || path.contains("inner_hits"));
+                mutated = insertRandomFields(xContentType, originalBytes, excludeFilter, random());
+            } else {
+                mutated = originalBytes;
+            }
+            Entry<Option> parsed;
+            try (XContentParser parser = createParser(xContentType.xContent(), mutated)) {
+                ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
+                parsed = ENTRY_PARSERS.get(entry.getClass()).apply(parser);
+                assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());
+                assertNull(parser.nextToken());
+            }
+            assertEquals(entry.getClass(), parsed.getClass());
+            assertEquals(entry.getText(), parsed.getText());
+            assertEquals(entry.getLength(), parsed.getLength());
+            assertEquals(entry.getOffset(), parsed.getOffset());
+            assertEquals(entry.getOptions().size(), parsed.getOptions().size());
+            for (int i = 0; i < entry.getOptions().size(); i++) {
+                assertEquals(entry.getOptions().get(i).getClass(), parsed.getOptions().get(i).getClass());
+            }
+            assertToXContentEquivalent(originalBytes, toXContent(parsed, xContentType, humanReadable), xContentType);
+        }
+    }
+
+    public void testToXContent() throws IOException {
+        Option option = new Option(new Text("someText"), new Text("somethingHighlighted"), 1.3f, true);
+        Entry<Option> entry = new Entry<>(new Text("entryText"), 42, 313);
+        entry.addOption(option);
+        BytesReference xContent = toXContent(entry, XContentType.JSON, randomBoolean());
+        assertEquals(
+                "{\"text\":\"entryText\","
+                + "\"offset\":42,"
+                + "\"length\":313,"
+                + "\"options\":["
+                    + "{\"text\":\"someText\","
+                    + "\"highlighted\":\"somethingHighlighted\","
+                    + "\"score\":1.3,"
+                    + "\"collate_match\":true}"
+                + "]}", xContent.utf8ToString());
+
+        org.elasticsearch.search.suggest.term.TermSuggestion.Entry.Option termOption =
+                new org.elasticsearch.search.suggest.term.TermSuggestion.Entry.Option(new Text("termSuggestOption"), 42, 3.13f);
+        entry = new Entry<>(new Text("entryText"), 42, 313);
+        entry.addOption(termOption);
+        xContent = toXContent(entry, XContentType.JSON, randomBoolean());
+        assertEquals(
+                "{\"text\":\"entryText\","
+                + "\"offset\":42,"
+                + "\"length\":313,"
+                + "\"options\":["
+                    + "{\"text\":\"termSuggestOption\","
+                    + "\"score\":3.13,"
+                    + "\"freq\":42}"
+                + "]}", xContent.utf8ToString());
+
+        org.elasticsearch.search.suggest.completion.CompletionSuggestion.Entry.Option completionOption =
+                new org.elasticsearch.search.suggest.completion.CompletionSuggestion.Entry.Option(-1, new Text("completionOption"),
+                        3.13f, Collections.singletonMap("key", Collections.singleton("value")));
+        entry = new Entry<>(new Text("entryText"), 42, 313);
+        entry.addOption(completionOption);
+        xContent = toXContent(entry, XContentType.JSON, randomBoolean());
+        assertEquals(
+                "{\"text\":\"entryText\","
+                + "\"offset\":42,"
+                + "\"length\":313,"
+                + "\"options\":["
+                    + "{\"text\":\"completionOption\","
+                    + "\"score\":3.13,"
+                    + "\"contexts\":{\"key\":[\"value\"]}"
+                    + "}"
+                + "]}", xContent.utf8ToString());
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/SuggestionOptionTests.java b/core/src/test/java/org/elasticsearch/search/suggest/SuggestionOptionTests.java
new file mode 100644
index 0000000000000..7385dd38f45b4
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/suggest/SuggestionOptionTests.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.suggest;
+
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.text.Text;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.search.suggest.Suggest.Suggestion.Entry.Option;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.elasticsearch.test.XContentTestUtils.insertRandomFields;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
+
+public class SuggestionOptionTests extends ESTestCase {
+
+    public static Option createTestItem() {
+        Text text = new Text(randomAlphaOfLengthBetween(5, 15));
+        float score = randomFloat();
+        Text highlighted = randomFrom((Text) null, new Text(randomAlphaOfLengthBetween(5, 15)));
+        Boolean collateMatch = randomFrom((Boolean) null, randomBoolean());
+        return new Option(text, highlighted, score, collateMatch);
+    }
+
+    public void testFromXContent() throws IOException {
+        doTestFromXContent(false);
+    }
+
+    public void testFromXContentWithRandomFields() throws IOException {
+        doTestFromXContent(true);
+    }
+
+    private void doTestFromXContent(boolean addRandomFields) throws IOException {
+        Option option = createTestItem();
+        XContentType xContentType = randomFrom(XContentType.values());
+        boolean humanReadable = randomBoolean();
+        BytesReference originalBytes = toShuffledXContent(option, xContentType, ToXContent.EMPTY_PARAMS, humanReadable);
+        BytesReference mutated;
+        if (addRandomFields) {
+            mutated = insertRandomFields(xContentType, originalBytes, null, random());
+        } else {
+            mutated = originalBytes;
+        }
+        Option parsed;
+        try (XContentParser parser = createParser(xContentType.xContent(), mutated)) {
+            ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
+            parsed = Option.fromXContent(parser);
+            assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());
+            assertNull(parser.nextToken());
+        }
+        assertEquals(option.getText(), parsed.getText());
+        assertEquals(option.getHighlighted(), parsed.getHighlighted());
+        assertEquals(option.getScore(), parsed.getScore(), Float.MIN_VALUE);
+        assertEquals(option.collateMatch(), parsed.collateMatch());
+        assertToXContentEquivalent(originalBytes, toXContent(parsed, xContentType, humanReadable), xContentType);
+    }
+
+    public void testToXContent() throws IOException {
+        Option option = new Option(new Text("someText"), new Text("somethingHighlighted"), 1.3f, true);
+        BytesReference xContent = toXContent(option, XContentType.JSON, randomBoolean());
+        assertEquals("{\"text\":\"someText\","
+                      + "\"highlighted\":\"somethingHighlighted\","
+                      + "\"score\":1.3,"
+                      + "\"collate_match\":true"
+                   + "}"
+                   , xContent.utf8ToString());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/SuggestionTests.java b/core/src/test/java/org/elasticsearch/search/suggest/SuggestionTests.java
new file mode 100644
index 0000000000000..3c56597299dd3
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/suggest/SuggestionTests.java
@@ -0,0 +1,268 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.suggest;
+
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.text.Text;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContent;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.rest.action.search.RestSearchAction;
+import org.elasticsearch.search.suggest.Suggest.Suggestion;
+import org.elasticsearch.search.suggest.Suggest.Suggestion.Entry;
+import org.elasticsearch.search.suggest.Suggest.Suggestion.Entry.Option;
+import org.elasticsearch.search.suggest.completion.CompletionSuggestion;
+import org.elasticsearch.search.suggest.phrase.PhraseSuggestion;
+import org.elasticsearch.search.suggest.term.TermSuggestion;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Predicate;
+import java.util.function.Supplier;
+
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.elasticsearch.test.XContentTestUtils.insertRandomFields;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
+
+public class SuggestionTests extends ESTestCase {
+
+    @SuppressWarnings("unchecked")
+    private static final Class<Suggestion<? extends Entry<? extends Option>>>[] SUGGESTION_TYPES = new Class[] {
+        TermSuggestion.class, PhraseSuggestion.class, CompletionSuggestion.class
+    };
+
+    public static Suggestion<? extends Entry<? extends Option>> createTestItem() {
+        return createTestItem(randomFrom(SUGGESTION_TYPES));
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return SuggestTests.getSuggestersRegistry();
+    }
+
+    @SuppressWarnings({ "unchecked", "rawtypes" })
+    public static Suggestion<? extends Entry<? extends Option>> createTestItem(Class<? extends Suggestion> type) {
+        String name = randomAlphaOfLengthBetween(5, 10);
+        // note: size will not be rendered via "toXContent", only passed on internally on transport layer
+        int size = randomInt();
+        Supplier<Entry> entrySupplier;
+        Suggestion suggestion;
+        if (type == TermSuggestion.class) {
+            suggestion = new TermSuggestion(name, size, randomFrom(SortBy.values()));
+            entrySupplier = () -> SuggestionEntryTests.createTestItem(TermSuggestion.Entry.class);
+        } else if (type == PhraseSuggestion.class) {
+            suggestion = new PhraseSuggestion(name, size);
+            entrySupplier = () -> SuggestionEntryTests.createTestItem(PhraseSuggestion.Entry.class);
+        } else if (type == CompletionSuggestion.class) {
+            suggestion = new CompletionSuggestion(name, size);
+            entrySupplier = () -> SuggestionEntryTests.createTestItem(CompletionSuggestion.Entry.class);
+        } else {
+            throw new UnsupportedOperationException("type not supported [" + type + "]");
+        }
+        int numEntries;
+        if (frequently()) {
+            if (type == CompletionSuggestion.class) {
+                numEntries = 1; // CompletionSuggestion can have max. one entry
+            } else {
+                numEntries = randomIntBetween(1, 5);
+            }
+        } else {
+            numEntries = 0; // also occasionally test zero entries
+        }
+        for (int i = 0; i < numEntries; i++) {
+            suggestion.addTerm(entrySupplier.get());
+        }
+        return suggestion;
+    }
+
+    public void testFromXContent() throws IOException {
+        doTestFromXContent(false);
+    }
+
+    public void testFromXContentWithRandomFields() throws IOException {
+        doTestFromXContent(true);
+    }
+
+    @SuppressWarnings({ "rawtypes" })
+    private void doTestFromXContent(boolean addRandomFields) throws IOException {
+        ToXContent.Params params = new ToXContent.MapParams(Collections.singletonMap(RestSearchAction.TYPED_KEYS_PARAM, "true"));
+        for (Class<Suggestion<? extends Entry<? extends Option>>> type : SUGGESTION_TYPES) {
+            Suggestion suggestion = createTestItem(type);
+            XContentType xContentType = randomFrom(XContentType.values());
+            boolean humanReadable = randomBoolean();
+            BytesReference originalBytes = toShuffledXContent(suggestion, xContentType, params, humanReadable);
+            BytesReference mutated;
+            if (addRandomFields) {
+                // - "contexts" is an object consisting of key/array pairs, we shouldn't add anything random there
+                // - there can be inner search hits fields inside this option where we cannot add random stuff
+                // - the root object should be excluded since it contains the named suggestion arrays
+                Predicate<String> excludeFilter = path -> (path.isEmpty()
+                        || path.endsWith(CompletionSuggestion.Entry.Option.CONTEXTS.getPreferredName()) || path.endsWith("highlight")
+                        || path.endsWith("fields") || path.contains("_source") || path.contains("inner_hits"));
+                mutated = insertRandomFields(xContentType, originalBytes, excludeFilter, random());
+            } else {
+                mutated = originalBytes;
+            }
+            Suggestion parsed;
+            try (XContentParser parser = createParser(xContentType.xContent(), mutated)) {
+                ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
+                ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.nextToken(), parser::getTokenLocation);
+                ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.nextToken(), parser::getTokenLocation);
+                parsed = Suggestion.fromXContent(parser);
+                assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
+                assertNull(parser.nextToken());
+            }
+            assertEquals(suggestion.getName(), parsed.getName());
+            assertEquals(suggestion.getEntries().size(), parsed.getEntries().size());
+            // We don't parse size via xContent, instead we set it to -1 on the client side
+            assertEquals(-1, parsed.getSize());
+            assertToXContentEquivalent(originalBytes, toXContent(parsed, xContentType, params, humanReadable), xContentType);
+        }
+    }
+
+    /**
+     * test that we parse nothing if RestSearchAction.TYPED_KEYS_PARAM isn't set while rendering xContent and we cannot find
+     * suggestion type information
+     */
+    public void testFromXContentWithoutTypeParam() throws IOException {
+        XContentType xContentType = randomFrom(XContentType.values());
+        BytesReference originalBytes = toXContent(createTestItem(), xContentType, ToXContent.EMPTY_PARAMS, randomBoolean());
+        try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) {
+            ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
+            ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.nextToken(), parser::getTokenLocation);
+            ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.nextToken(), parser::getTokenLocation);
+            assertNull(Suggestion.fromXContent(parser));
+            ensureExpectedToken(XContentParser.Token.END_OBJECT, parser.nextToken(), parser::getTokenLocation);
+        }
+    }
+
+    public void testUnknownSuggestionTypeThrows() throws IOException {
+        XContent xContent = JsonXContent.jsonXContent;
+        String suggestionString =
+                 "{\"unknownType#suggestionName\":"
+                    + "[{\"text\":\"entryText\","
+                    + "\"offset\":42,"
+                    + "\"length\":313,"
+                    + "\"options\":[{\"text\":\"someText\","
+                                + "\"highlighted\":\"somethingHighlighted\","
+                                + "\"score\":1.3,"
+                                + "\"collate_match\":true}]"
+                            + "}]"
+                + "}";
+        try (XContentParser parser = xContent.createParser(xContentRegistry(), suggestionString)) {
+            ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
+            ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.nextToken(), parser::getTokenLocation);
+            ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.nextToken(), parser::getTokenLocation);
+            ParsingException e = expectThrows(ParsingException.class, () -> Suggestion.fromXContent(parser));
+            assertEquals("Unknown Suggestion [unknownType]", e.getMessage());
+        }
+    }
+
+    public void testToXContent() throws IOException {
+        ToXContent.Params params = new ToXContent.MapParams(Collections.singletonMap(RestSearchAction.TYPED_KEYS_PARAM, "true"));
+        {
+            Option option = new Option(new Text("someText"), new Text("somethingHighlighted"), 1.3f, true);
+            Entry<Option> entry = new Entry<>(new Text("entryText"), 42, 313);
+            entry.addOption(option);
+            Suggestion<Entry<Option>> suggestion = new Suggestion<>("suggestionName", 5);
+            suggestion.addTerm(entry);
+            BytesReference xContent = toXContent(suggestion, XContentType.JSON, params, randomBoolean());
+            assertEquals(
+                    "{\"suggestion#suggestionName\":[{"
+                            + "\"text\":\"entryText\","
+                            + "\"offset\":42,"
+                            + "\"length\":313,"
+                            + "\"options\":[{"
+                                + "\"text\":\"someText\","
+                                + "\"highlighted\":\"somethingHighlighted\","
+                                + "\"score\":1.3,"
+                                + "\"collate_match\":true}]"
+                            + "}]"
+                    + "}", xContent.utf8ToString());
+        }
+        {
+            Option option = new Option(new Text("someText"), new Text("somethingHighlighted"), 1.3f, true);
+            PhraseSuggestion.Entry entry = new PhraseSuggestion.Entry(new Text("entryText"), 42, 313, 1.0);
+            entry.addOption(option);
+            PhraseSuggestion suggestion = new PhraseSuggestion("suggestionName", 5);
+            suggestion.addTerm(entry);
+            BytesReference xContent = toXContent(suggestion, XContentType.JSON, params, randomBoolean());
+            assertEquals(
+                    "{\"phrase#suggestionName\":[{"
+                            + "\"text\":\"entryText\","
+                            + "\"offset\":42,"
+                            + "\"length\":313,"
+                            + "\"options\":[{"
+                                + "\"text\":\"someText\","
+                                + "\"highlighted\":\"somethingHighlighted\","
+                                + "\"score\":1.3,"
+                                + "\"collate_match\":true}]"
+                            + "}]"
+                    + "}", xContent.utf8ToString());
+        }
+        {
+            TermSuggestion.Entry.Option option = new TermSuggestion.Entry.Option(new Text("someText"), 10, 1.3f);
+            TermSuggestion.Entry entry = new TermSuggestion.Entry(new Text("entryText"), 42, 313);
+            entry.addOption(option);
+            TermSuggestion suggestion = new TermSuggestion("suggestionName", 5, SortBy.SCORE);
+            suggestion.addTerm(entry);
+            BytesReference xContent = toXContent(suggestion, XContentType.JSON, params, randomBoolean());
+            assertEquals(
+                    "{\"term#suggestionName\":[{"
+                        + "\"text\":\"entryText\","
+                        + "\"offset\":42,"
+                        + "\"length\":313,"
+                        + "\"options\":[{"
+                            + "\"text\":\"someText\","
+                            + "\"score\":1.3,"
+                            + "\"freq\":10}]"
+                        + "}]"
+                    + "}", xContent.utf8ToString());
+        }
+        {
+            Map<String, Set<CharSequence>> contexts = Collections.singletonMap("key", Collections.singleton("value"));
+            CompletionSuggestion.Entry.Option option = new CompletionSuggestion.Entry.Option(1, new Text("someText"), 1.3f, contexts);
+            CompletionSuggestion.Entry entry = new CompletionSuggestion.Entry(new Text("entryText"), 42, 313);
+            entry.addOption(option);
+            CompletionSuggestion suggestion = new CompletionSuggestion("suggestionName", 5);
+            suggestion.addTerm(entry);
+            BytesReference xContent = toXContent(suggestion, XContentType.JSON, params, randomBoolean());
+            assertEquals(
+                    "{\"completion#suggestionName\":[{"
+                        + "\"text\":\"entryText\","
+                        + "\"offset\":42,"
+                        + "\"length\":313,"
+                        + "\"options\":[{"
+                            + "\"text\":\"someText\","
+                            + "\"score\":1.3,"
+                            + "\"contexts\":{\"key\":[\"value\"]}"
+                        + "}]"
+                    + "}]}", xContent.utf8ToString());
+        }
+    }
+}
\ No newline at end of file
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/TermSuggestionOptionTests.java b/core/src/test/java/org/elasticsearch/search/suggest/TermSuggestionOptionTests.java
new file mode 100644
index 0000000000000..f25fe09341ec1
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/suggest/TermSuggestionOptionTests.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.suggest;
+
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.text.Text;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.search.suggest.term.TermSuggestion.Entry.Option;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.elasticsearch.test.XContentTestUtils.insertRandomFields;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
+
+public class TermSuggestionOptionTests extends ESTestCase {
+
+    public static Option createTestItem() {
+        Text text = new Text(randomAlphaOfLengthBetween(5, 15));
+        float score = randomFloat();
+        int freq = randomInt();
+        return new Option(text, freq, score);
+    }
+
+    public void testFromXContent() throws IOException {
+        doTestFromXContent(false);
+    }
+
+    public void testFromXContentWithRandomFields() throws IOException {
+        doTestFromXContent(true);
+    }
+
+    private void doTestFromXContent(boolean addRandomFields) throws IOException {
+        Option option = createTestItem();
+        XContentType xContentType = randomFrom(XContentType.values());
+        boolean humanReadable = randomBoolean();
+        BytesReference originalBytes = toShuffledXContent(option, xContentType, ToXContent.EMPTY_PARAMS, humanReadable);
+        BytesReference mutated;
+        if (addRandomFields) {
+            mutated = insertRandomFields(xContentType, originalBytes, null, random());
+        } else {
+            mutated = originalBytes;
+        }
+        Option parsed;
+        try (XContentParser parser = createParser(xContentType.xContent(), mutated)) {
+            ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
+            parsed = Option.fromXContent(parser);
+            assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());
+            assertNull(parser.nextToken());
+        }
+        assertEquals(option.getText(), parsed.getText());
+        assertEquals(option.getScore(), parsed.getScore(), Float.MIN_VALUE);
+        assertEquals(option.getFreq(), parsed.getFreq());
+        assertToXContentEquivalent(originalBytes, toXContent(parsed, xContentType, humanReadable), xContentType);
+    }
+
+    public void testToXContent() throws IOException {
+        Option option = new Option(new Text("someText"), 100, 1.3f);
+        BytesReference xContent = toXContent(option, XContentType.JSON, randomBoolean());
+        assertEquals("{\"text\":\"someText\",\"score\":1.3,\"freq\":100}"
+                   , xContent.utf8ToString());
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/completion/CategoryContextMappingTests.java b/core/src/test/java/org/elasticsearch/search/suggest/completion/CategoryContextMappingTests.java
index de912bb9636c4..7b1cf8061eae3 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/completion/CategoryContextMappingTests.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/completion/CategoryContextMappingTests.java
@@ -20,22 +20,32 @@
 package org.elasticsearch.search.suggest.completion;
 
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.IntPoint;
+import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.SortedSetDocValuesField;
+import org.apache.lucene.document.StoredField;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.search.suggest.document.ContextSuggestField;
-import org.elasticsearch.common.ParseFieldMatcher;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.index.mapper.CompletionFieldMapper.CompletionFieldType;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.FieldMapper;
+import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.MapperParsingException;
+import org.elasticsearch.index.mapper.NumberFieldMapper;
 import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
+import org.elasticsearch.index.mapper.SourceToParse;
+import org.elasticsearch.index.mapper.StringFieldType;
+import org.elasticsearch.index.mapper.TextFieldMapper;
 import org.elasticsearch.search.suggest.completion.context.CategoryContextMapping;
 import org.elasticsearch.search.suggest.completion.context.ContextBuilder;
 import org.elasticsearch.search.suggest.completion.context.ContextMapping;
@@ -45,6 +55,7 @@
 import java.util.Set;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 
 public class CategoryContextMappingTests extends ESSingleNodeTestCase {
@@ -77,7 +88,7 @@ public void testIndexingWithNoContexts() throws Exception {
                 .field("weight", 4)
                 .endObject()
                 .startObject()
-                .field("input", "suggestion5", "suggestion6", "suggestion7")
+                .array("input", "suggestion5", "suggestion6", "suggestion7")
                 .field("weight", 5)
                 .endObject()
                 .endArray()
@@ -107,7 +118,7 @@ public void testIndexingWithSimpleContexts() throws Exception {
                 .startObject()
                 .startArray("completion")
                 .startObject()
-                .field("input", "suggestion5", "suggestion6", "suggestion7")
+                .array("input", "suggestion5", "suggestion6", "suggestion7")
                 .startObject("contexts")
                 .field("ctx", "ctx1")
                 .endObject()
@@ -120,6 +131,103 @@ public void testIndexingWithSimpleContexts() throws Exception {
         assertContextSuggestFields(fields, 3);
     }
 
+    public void testIndexingWithSimpleNumberContexts() throws Exception {
+        String mapping = jsonBuilder().startObject().startObject("type1")
+                .startObject("properties").startObject("completion")
+                .field("type", "completion")
+                .startArray("contexts")
+                .startObject()
+                .field("name", "ctx")
+                .field("type", "category")
+                .endObject()
+                .endArray()
+                .endObject().endObject()
+                .endObject().endObject().string();
+
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion");
+        MappedFieldType completionFieldType = fieldMapper.fieldType();
+        ParsedDocument parsedDocument = defaultMapper.parse("test", "type1", "1", jsonBuilder()
+                .startObject()
+                .startArray("completion")
+                .startObject()
+                .array("input", "suggestion5", "suggestion6", "suggestion7")
+                .startObject("contexts")
+                .field("ctx", 100)
+                .endObject()
+                .field("weight", 5)
+                .endObject()
+                .endArray()
+                .endObject()
+                .bytes());
+        IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.name());
+        assertContextSuggestFields(fields, 3);
+    }
+
+    public void testIndexingWithSimpleBooleanContexts() throws Exception {
+        String mapping = jsonBuilder().startObject().startObject("type1")
+                .startObject("properties").startObject("completion")
+                .field("type", "completion")
+                .startArray("contexts")
+                .startObject()
+                .field("name", "ctx")
+                .field("type", "category")
+                .endObject()
+                .endArray()
+                .endObject().endObject()
+                .endObject().endObject().string();
+
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion");
+        MappedFieldType completionFieldType = fieldMapper.fieldType();
+        ParsedDocument parsedDocument = defaultMapper.parse("test", "type1", "1", jsonBuilder()
+                .startObject()
+                .startArray("completion")
+                .startObject()
+                .array("input", "suggestion5", "suggestion6", "suggestion7")
+                .startObject("contexts")
+                .field("ctx", true)
+                .endObject()
+                .field("weight", 5)
+                .endObject()
+                .endArray()
+                .endObject()
+                .bytes());
+        IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.name());
+        assertContextSuggestFields(fields, 3);
+    }
+
+    public void testIndexingWithSimpleNULLContexts() throws Exception {
+        String mapping = jsonBuilder().startObject().startObject("type1")
+                .startObject("properties").startObject("completion")
+                .field("type", "completion")
+                .startArray("contexts")
+                .startObject()
+                .field("name", "ctx")
+                .field("type", "category")
+                .endObject()
+                .endArray()
+                .endObject().endObject()
+                .endObject().endObject().string();
+
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        XContentBuilder builder = jsonBuilder()
+                .startObject()
+                .startArray("completion")
+                .startObject()
+                .array("input", "suggestion5", "suggestion6", "suggestion7")
+                .startObject("contexts")
+                .nullField("ctx")
+                .endObject()
+                .field("weight", 5)
+                .endObject()
+                .endArray()
+                .endObject();
+
+        Exception e = expectThrows(MapperParsingException.class, () -> defaultMapper.parse("test", "type1", "1", builder.bytes()));
+        assertEquals("contexts must be a string, number or boolean or a list of string, number or boolean, but was [VALUE_NULL]", e.getCause().getMessage());
+    }
+
     public void testIndexingWithContextList() throws Exception {
         String mapping = jsonBuilder().startObject().startObject("type1")
                 .startObject("properties").startObject("completion")
@@ -139,7 +247,7 @@ public void testIndexingWithContextList() throws Exception {
         ParsedDocument parsedDocument = defaultMapper.parse("test", "type1", "1", jsonBuilder()
                 .startObject()
                 .startObject("completion")
-                .field("input", "suggestion5", "suggestion6", "suggestion7")
+                .array("input", "suggestion5", "suggestion6", "suggestion7")
                 .startObject("contexts")
                 .array("ctx", "ctx1", "ctx2", "ctx3")
                 .endObject()
@@ -151,6 +259,66 @@ public void testIndexingWithContextList() throws Exception {
         assertContextSuggestFields(fields, 3);
     }
 
+    public void testIndexingWithMixedTypeContextList() throws Exception {
+        String mapping = jsonBuilder().startObject().startObject("type1")
+                .startObject("properties").startObject("completion")
+                .field("type", "completion")
+                .startArray("contexts")
+                .startObject()
+                .field("name", "ctx")
+                .field("type", "category")
+                .endObject()
+                .endArray()
+                .endObject().endObject()
+                .endObject().endObject().string();
+
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion");
+        MappedFieldType completionFieldType = fieldMapper.fieldType();
+        ParsedDocument parsedDocument = defaultMapper.parse("test", "type1", "1", jsonBuilder()
+                .startObject()
+                .startObject("completion")
+                .array("input", "suggestion5", "suggestion6", "suggestion7")
+                .startObject("contexts")
+                .array("ctx", "ctx1", true, 100)
+                .endObject()
+                .field("weight", 5)
+                .endObject()
+                .endObject()
+                .bytes());
+        IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.name());
+        assertContextSuggestFields(fields, 3);
+    }
+
+    public void testIndexingWithMixedTypeContextListHavingNULL() throws Exception {
+        String mapping = jsonBuilder().startObject().startObject("type1")
+                .startObject("properties").startObject("completion")
+                .field("type", "completion")
+                .startArray("contexts")
+                .startObject()
+                .field("name", "ctx")
+                .field("type", "category")
+                .endObject()
+                .endArray()
+                .endObject().endObject()
+                .endObject().endObject().string();
+
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        XContentBuilder builder = jsonBuilder()
+                .startObject()
+                .startObject("completion")
+                .array("input", "suggestion5", "suggestion6", "suggestion7")
+                .startObject("contexts")
+                .array("ctx", "ctx1", true, 100, null)
+                .endObject()
+                .field("weight", 5)
+                .endObject()
+                .endObject();
+
+        Exception e = expectThrows(MapperParsingException.class, () -> defaultMapper.parse("test", "type1", "1", builder.bytes()));
+        assertEquals("context array must have string, number or boolean values, but was [VALUE_NULL]", e.getCause().getMessage());
+    }
+
     public void testIndexingWithMultipleContexts() throws Exception {
         String mapping = jsonBuilder().startObject().startObject("type1")
                 .startObject("properties").startObject("completion")
@@ -175,7 +343,7 @@ public void testIndexingWithMultipleContexts() throws Exception {
                 .startObject()
                 .startArray("completion")
                 .startObject()
-                .field("input", "suggestion5", "suggestion6", "suggestion7")
+                .array("input", "suggestion5", "suggestion6", "suggestion7")
                 .field("weight", 5)
                 .startObject("contexts")
                 .array("ctx", "ctx1", "ctx2", "ctx3")
@@ -191,7 +359,7 @@ public void testIndexingWithMultipleContexts() throws Exception {
 
     public void testQueryContextParsingBasic() throws Exception {
         XContentBuilder builder = jsonBuilder().value("context1");
-        XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.bytes());
+        XContentParser parser = createParser(JsonXContent.jsonXContent, builder.bytes());
         CategoryContextMapping mapping = ContextBuilder.category("cat").build();
         List<ContextMapping.InternalQueryContext> internalQueryContexts = mapping.parseQueryContext(createParseContext(parser));
         assertThat(internalQueryContexts.size(), equalTo(1));
@@ -200,12 +368,43 @@ public void testQueryContextParsingBasic() throws Exception {
         assertThat(internalQueryContexts.get(0).isPrefix, equalTo(false));
     }
 
+    public void testBooleanQueryContextParsingBasic() throws Exception {
+        XContentBuilder builder = jsonBuilder().value(true);
+        XContentParser parser = createParser(JsonXContent.jsonXContent, builder.bytes());
+        CategoryContextMapping mapping = ContextBuilder.category("cat").build();
+        List<ContextMapping.InternalQueryContext> internalQueryContexts = mapping.parseQueryContext(createParseContext(parser));
+        assertThat(internalQueryContexts.size(), equalTo(1));
+        assertThat(internalQueryContexts.get(0).context, equalTo("true"));
+        assertThat(internalQueryContexts.get(0).boost, equalTo(1));
+        assertThat(internalQueryContexts.get(0).isPrefix, equalTo(false));
+    }
+
+    public void testNumberQueryContextParsingBasic() throws Exception {
+        XContentBuilder builder = jsonBuilder().value(10);
+        XContentParser parser = createParser(JsonXContent.jsonXContent, builder.bytes());
+        CategoryContextMapping mapping = ContextBuilder.category("cat").build();
+        List<ContextMapping.InternalQueryContext> internalQueryContexts = mapping.parseQueryContext(createParseContext(parser));
+        assertThat(internalQueryContexts.size(), equalTo(1));
+        assertThat(internalQueryContexts.get(0).context, equalTo("10"));
+        assertThat(internalQueryContexts.get(0).boost, equalTo(1));
+        assertThat(internalQueryContexts.get(0).isPrefix, equalTo(false));
+    }
+
+    public void testNULLQueryContextParsingBasic() throws Exception {
+        XContentBuilder builder = jsonBuilder().nullValue();
+        XContentParser parser = createParser(JsonXContent.jsonXContent, builder.bytes());
+        CategoryContextMapping mapping = ContextBuilder.category("cat").build();
+
+        Exception e = expectThrows(ElasticsearchParseException.class, () -> mapping.parseQueryContext(createParseContext(parser)));
+        assertEquals("category context must be an object, string, number or boolean", e.getMessage());
+    }
+
     public void testQueryContextParsingArray() throws Exception {
         XContentBuilder builder = jsonBuilder().startArray()
-                .value("context1")
-                .value("context2")
+                    .value("context1")
+                    .value("context2")
                 .endArray();
-        XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.bytes());
+        XContentParser parser = createParser(JsonXContent.jsonXContent, builder.bytes());
         CategoryContextMapping mapping = ContextBuilder.category("cat").build();
         List<ContextMapping.InternalQueryContext> internalQueryContexts = mapping.parseQueryContext(createParseContext(parser));
         assertThat(internalQueryContexts.size(), equalTo(2));
@@ -217,13 +416,53 @@ public void testQueryContextParsingArray() throws Exception {
         assertThat(internalQueryContexts.get(1).isPrefix, equalTo(false));
     }
 
+    public void testQueryContextParsingMixedTypeValuesArray() throws Exception {
+        XContentBuilder builder = jsonBuilder().startArray()
+                    .value("context1")
+                    .value("context2")
+                    .value(true)
+                    .value(10)
+                .endArray();
+        XContentParser parser = createParser(JsonXContent.jsonXContent, builder.bytes());
+        CategoryContextMapping mapping = ContextBuilder.category("cat").build();
+        List<ContextMapping.InternalQueryContext> internalQueryContexts = mapping.parseQueryContext(createParseContext(parser));
+        assertThat(internalQueryContexts.size(), equalTo(4));
+        assertThat(internalQueryContexts.get(0).context, equalTo("context1"));
+        assertThat(internalQueryContexts.get(0).boost, equalTo(1));
+        assertThat(internalQueryContexts.get(0).isPrefix, equalTo(false));
+        assertThat(internalQueryContexts.get(1).context, equalTo("context2"));
+        assertThat(internalQueryContexts.get(1).boost, equalTo(1));
+        assertThat(internalQueryContexts.get(1).isPrefix, equalTo(false));
+        assertThat(internalQueryContexts.get(2).context, equalTo("true"));
+        assertThat(internalQueryContexts.get(2).boost, equalTo(1));
+        assertThat(internalQueryContexts.get(2).isPrefix, equalTo(false));
+        assertThat(internalQueryContexts.get(3).context, equalTo("10"));
+        assertThat(internalQueryContexts.get(3).boost, equalTo(1));
+        assertThat(internalQueryContexts.get(3).isPrefix, equalTo(false));
+    }
+
+    public void testQueryContextParsingMixedTypeValuesArrayHavingNULL() throws Exception {
+        XContentBuilder builder = jsonBuilder().startArray()
+                    .value("context1")
+                    .value("context2")
+                    .value(true)
+                    .value(10)
+                    .nullValue()
+                .endArray();
+        XContentParser parser = createParser(JsonXContent.jsonXContent, builder.bytes());
+        CategoryContextMapping mapping = ContextBuilder.category("cat").build();
+
+        Exception e = expectThrows(ElasticsearchParseException.class, () -> mapping.parseQueryContext(createParseContext(parser)));
+        assertEquals("category context must be an object, string, number or boolean", e.getMessage());
+    }
+
     public void testQueryContextParsingObject() throws Exception {
         XContentBuilder builder = jsonBuilder().startObject()
                 .field("context", "context1")
                 .field("boost", 10)
                 .field("prefix", true)
                 .endObject();
-        XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.bytes());
+        XContentParser parser = createParser(JsonXContent.jsonXContent, builder.bytes());
         CategoryContextMapping mapping = ContextBuilder.category("cat").build();
         List<ContextMapping.InternalQueryContext> internalQueryContexts = mapping.parseQueryContext(createParseContext(parser));
         assertThat(internalQueryContexts.size(), equalTo(1));
@@ -232,6 +471,48 @@ public void testQueryContextParsingObject() throws Exception {
         assertThat(internalQueryContexts.get(0).isPrefix, equalTo(true));
     }
 
+    public void testQueryContextParsingObjectHavingBoolean() throws Exception {
+        XContentBuilder builder = jsonBuilder().startObject()
+                .field("context", false)
+                .field("boost", 10)
+                .field("prefix", true)
+                .endObject();
+        XContentParser parser = createParser(JsonXContent.jsonXContent, builder.bytes());
+        CategoryContextMapping mapping = ContextBuilder.category("cat").build();
+        List<ContextMapping.InternalQueryContext> internalQueryContexts = mapping.parseQueryContext(createParseContext(parser));
+        assertThat(internalQueryContexts.size(), equalTo(1));
+        assertThat(internalQueryContexts.get(0).context, equalTo("false"));
+        assertThat(internalQueryContexts.get(0).boost, equalTo(10));
+        assertThat(internalQueryContexts.get(0).isPrefix, equalTo(true));
+    }
+
+    public void testQueryContextParsingObjectHavingNumber() throws Exception {
+        XContentBuilder builder = jsonBuilder().startObject()
+                .field("context", 333)
+                .field("boost", 10)
+                .field("prefix", true)
+                .endObject();
+        XContentParser parser = createParser(JsonXContent.jsonXContent, builder.bytes());
+        CategoryContextMapping mapping = ContextBuilder.category("cat").build();
+        List<ContextMapping.InternalQueryContext> internalQueryContexts = mapping.parseQueryContext(createParseContext(parser));
+        assertThat(internalQueryContexts.size(), equalTo(1));
+        assertThat(internalQueryContexts.get(0).context, equalTo("333"));
+        assertThat(internalQueryContexts.get(0).boost, equalTo(10));
+        assertThat(internalQueryContexts.get(0).isPrefix, equalTo(true));
+    }
+
+    public void testQueryContextParsingObjectHavingNULL() throws Exception {
+        XContentBuilder builder = jsonBuilder().startObject()
+                .nullField("context")
+                .field("boost", 10)
+                .field("prefix", true)
+                .endObject();
+        XContentParser parser = createParser(JsonXContent.jsonXContent, builder.bytes());
+        CategoryContextMapping mapping = ContextBuilder.category("cat").build();
+
+        Exception e = expectThrows(ElasticsearchParseException.class, () -> mapping.parseQueryContext(createParseContext(parser)));
+        assertEquals("category context must be a string, number or boolean", e.getMessage());
+    }
 
     public void testQueryContextParsingObjectArray() throws Exception {
         XContentBuilder builder = jsonBuilder().startArray()
@@ -246,7 +527,7 @@ public void testQueryContextParsingObjectArray() throws Exception {
                 .field("prefix", false)
                 .endObject()
                 .endArray();
-        XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.bytes());
+        XContentParser parser = createParser(JsonXContent.jsonXContent, builder.bytes());
         CategoryContextMapping mapping = ContextBuilder.category("cat").build();
         List<ContextMapping.InternalQueryContext> internalQueryContexts = mapping.parseQueryContext(createParseContext(parser));
         assertThat(internalQueryContexts.size(), equalTo(2));
@@ -258,8 +539,84 @@ public void testQueryContextParsingObjectArray() throws Exception {
         assertThat(internalQueryContexts.get(1).isPrefix, equalTo(false));
     }
 
+    public void testQueryContextParsingMixedTypeObjectArray() throws Exception {
+        XContentBuilder builder = jsonBuilder().startArray()
+                .startObject()
+                .field("context", "context1")
+                .field("boost", 2)
+                .field("prefix", true)
+                .endObject()
+                .startObject()
+                .field("context", "context2")
+                .field("boost", 3)
+                .field("prefix", false)
+                .endObject()
+                .startObject()
+                .field("context", true)
+                .field("boost", 3)
+                .field("prefix", false)
+                .endObject()
+                .startObject()
+                .field("context", 333)
+                .field("boost", 3)
+                .field("prefix", false)
+                .endObject()
+                .endArray();
+        XContentParser parser = createParser(JsonXContent.jsonXContent, builder.bytes());
+        CategoryContextMapping mapping = ContextBuilder.category("cat").build();
+        List<ContextMapping.InternalQueryContext> internalQueryContexts = mapping.parseQueryContext(createParseContext(parser));
+        assertThat(internalQueryContexts.size(), equalTo(4));
+        assertThat(internalQueryContexts.get(0).context, equalTo("context1"));
+        assertThat(internalQueryContexts.get(0).boost, equalTo(2));
+        assertThat(internalQueryContexts.get(0).isPrefix, equalTo(true));
+        assertThat(internalQueryContexts.get(1).context, equalTo("context2"));
+        assertThat(internalQueryContexts.get(1).boost, equalTo(3));
+        assertThat(internalQueryContexts.get(1).isPrefix, equalTo(false));
+        assertThat(internalQueryContexts.get(2).context, equalTo("true"));
+        assertThat(internalQueryContexts.get(2).boost, equalTo(3));
+        assertThat(internalQueryContexts.get(2).isPrefix, equalTo(false));
+        assertThat(internalQueryContexts.get(3).context, equalTo("333"));
+        assertThat(internalQueryContexts.get(3).boost, equalTo(3));
+        assertThat(internalQueryContexts.get(3).isPrefix, equalTo(false));
+    }
+
+    public void testQueryContextParsingMixedTypeObjectArrayHavingNULL() throws Exception {
+        XContentBuilder builder = jsonBuilder().startArray()
+                .startObject()
+                .field("context", "context1")
+                .field("boost", 2)
+                .field("prefix", true)
+                .endObject()
+                .startObject()
+                .field("context", "context2")
+                .field("boost", 3)
+                .field("prefix", false)
+                .endObject()
+                .startObject()
+                .field("context", true)
+                .field("boost", 3)
+                .field("prefix", false)
+                .endObject()
+                .startObject()
+                .field("context", 333)
+                .field("boost", 3)
+                .field("prefix", false)
+                .endObject()
+                .startObject()
+                .nullField("context")
+                .field("boost", 3)
+                .field("prefix", false)
+                .endObject()
+                .endArray();
+        XContentParser parser = createParser(JsonXContent.jsonXContent, builder.bytes());
+        CategoryContextMapping mapping = ContextBuilder.category("cat").build();
+
+        Exception e = expectThrows(ElasticsearchParseException.class, () -> mapping.parseQueryContext(createParseContext(parser)));
+        assertEquals("category context must be a string, number or boolean", e.getMessage());
+    }
+
     private static QueryParseContext createParseContext(XContentParser parser) {
-        return new QueryParseContext(new IndicesQueriesRegistry(), parser, ParseFieldMatcher.STRICT);
+        return new QueryParseContext(parser);
     }
 
     public void testQueryContextParsingMixed() throws Exception {
@@ -270,26 +627,117 @@ public void testQueryContextParsingMixed() throws Exception {
                 .field("prefix", true)
                 .endObject()
                 .value("context2")
+                .value(false)
+                .startObject()
+                .field("context", 333)
+                .field("boost", 2)
+                .field("prefix", true)
+                .endObject()
                 .endArray();
-        XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.bytes());
+        XContentParser parser = createParser(JsonXContent.jsonXContent, builder.bytes());
         CategoryContextMapping mapping = ContextBuilder.category("cat").build();
         List<ContextMapping.InternalQueryContext> internalQueryContexts = mapping.parseQueryContext(createParseContext(parser));
-        assertThat(internalQueryContexts.size(), equalTo(2));
+        assertThat(internalQueryContexts.size(), equalTo(4));
         assertThat(internalQueryContexts.get(0).context, equalTo("context1"));
         assertThat(internalQueryContexts.get(0).boost, equalTo(2));
         assertThat(internalQueryContexts.get(0).isPrefix, equalTo(true));
         assertThat(internalQueryContexts.get(1).context, equalTo("context2"));
         assertThat(internalQueryContexts.get(1).boost, equalTo(1));
         assertThat(internalQueryContexts.get(1).isPrefix, equalTo(false));
+        assertThat(internalQueryContexts.get(2).context, equalTo("false"));
+        assertThat(internalQueryContexts.get(2).boost, equalTo(1));
+        assertThat(internalQueryContexts.get(2).isPrefix, equalTo(false));
+        assertThat(internalQueryContexts.get(3).context, equalTo("333"));
+        assertThat(internalQueryContexts.get(3).boost, equalTo(2));
+        assertThat(internalQueryContexts.get(3).isPrefix, equalTo(true));
+    }
+
+    public void testQueryContextParsingMixedHavingNULL() throws Exception {
+        XContentBuilder builder = jsonBuilder().startArray()
+                .startObject()
+                .field("context", "context1")
+                .field("boost", 2)
+                .field("prefix", true)
+                .endObject()
+                .value("context2")
+                .value(false)
+                .startObject()
+                .field("context", 333)
+                .field("boost", 2)
+                .field("prefix", true)
+                .endObject()
+                .nullValue()
+                .endArray();
+        XContentParser parser = createParser(JsonXContent.jsonXContent, builder.bytes());
+        CategoryContextMapping mapping = ContextBuilder.category("cat").build();
+
+        Exception e = expectThrows(ElasticsearchParseException.class, () -> mapping.parseQueryContext(createParseContext(parser)));
+        assertEquals("category context must be an object, string, number or boolean", e.getMessage());
+    }
+
+    public void testUnknownQueryContextParsing() throws Exception {
+        String mapping = jsonBuilder().startObject().startObject("type1")
+                .startObject("properties").startObject("completion")
+                .field("type", "completion")
+                .startArray("contexts")
+                .startObject()
+                .field("name", "ctx")
+                .field("type", "category")
+                .endObject()
+                .startObject()
+                .field("name", "type")
+                .field("type", "category")
+                .endObject()
+                .endArray()
+                .endObject().endObject()
+                .endObject().endObject().string();
+
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion");
+        CompletionFieldType completionFieldType = (CompletionFieldType) fieldMapper.fieldType();
+
+        Exception e = expectThrows(IllegalArgumentException.class, () -> completionFieldType.getContextMappings().get("brand"));
+        assertEquals("Unknown context name [brand], must be one of [ctx, type]", e.getMessage());
     }
 
     public void testParsingContextFromDocument() throws Exception {
         CategoryContextMapping mapping = ContextBuilder.category("cat").field("category").build();
         ParseContext.Document document = new ParseContext.Document();
-        document.add(new StringField("category", "category1", Field.Store.NO));
+
+        KeywordFieldMapper.KeywordFieldType keyword = new KeywordFieldMapper.KeywordFieldType();
+        keyword.setName("category");
+        document.add(new Field(keyword.name(), new BytesRef("category1"), keyword));
+        // Ignore doc values
+        document.add(new SortedSetDocValuesField(keyword.name(), new BytesRef("category1")));
         Set<CharSequence> context = mapping.parseContext(document);
         assertThat(context.size(), equalTo(1));
         assertTrue(context.contains("category1"));
+
+
+        document = new ParseContext.Document();
+        TextFieldMapper.TextFieldType text = new TextFieldMapper.TextFieldType();
+        text.setName("category");
+        document.add(new Field(text.name(), "category1", text));
+        // Ignore stored field
+        document.add(new StoredField(text.name(), "category1", text));
+        context = mapping.parseContext(document);
+        assertThat(context.size(), equalTo(1));
+        assertTrue(context.contains("category1"));
+
+        document = new ParseContext.Document();
+        document.add(new SortedSetDocValuesField("category", new BytesRef("category")));
+        context = mapping.parseContext(document);
+        assertThat(context.size(), equalTo(0));
+
+        document = new ParseContext.Document();
+        document.add(new SortedDocValuesField("category", new BytesRef("category")));
+        context = mapping.parseContext(document);
+        assertThat(context.size(), equalTo(0));
+
+        final ParseContext.Document doc = new ParseContext.Document();
+        doc.add(new IntPoint("category", 36));
+        IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> mapping.parseContext(doc));
+        assertThat(exc.getMessage(), containsString("Failed to parse context field [category]"));
     }
 
     static void assertContextSuggestFields(IndexableField[] fields, int expected) {
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/completion/CategoryQueryContextTests.java b/core/src/test/java/org/elasticsearch/search/suggest/completion/CategoryQueryContextTests.java
index 59d7bbd61b905..658f145b1c635 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/completion/CategoryQueryContextTests.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/completion/CategoryQueryContextTests.java
@@ -28,7 +28,7 @@ public class CategoryQueryContextTests extends QueryContextTestCase<CategoryQuer
 
     public static CategoryQueryContext randomCategoryQueryContext() {
         final CategoryQueryContext.Builder builder = CategoryQueryContext.builder();
-        builder.setCategory(randomAsciiOfLength(10));
+        builder.setCategory(randomAlphaOfLength(10));
         maybeSet(builder::setBoost, randomIntBetween(1, 10));
         maybeSet(builder::setPrefix, randomBoolean());
         return builder.build();
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/completion/CompletionSuggesterBuilderTests.java b/core/src/test/java/org/elasticsearch/search/suggest/completion/CompletionSuggesterBuilderTests.java
index 8e0ff3953ffda..77c149df7a773 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/completion/CompletionSuggesterBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/completion/CompletionSuggesterBuilderTests.java
@@ -54,20 +54,20 @@ private static class BuilderAndInfo {
 
     private static BuilderAndInfo randomSuggestionBuilderWithContextInfo() {
         final BuilderAndInfo builderAndInfo = new BuilderAndInfo();
-        CompletionSuggestionBuilder testBuilder = new CompletionSuggestionBuilder(randomAsciiOfLengthBetween(2, 20));
+        CompletionSuggestionBuilder testBuilder = new CompletionSuggestionBuilder(randomAlphaOfLengthBetween(2, 20));
         setCommonPropertiesOnRandomBuilder(testBuilder);
         switch (randomIntBetween(0, 3)) {
             case 0:
-                testBuilder.prefix(randomAsciiOfLength(10));
+                testBuilder.prefix(randomAlphaOfLength(10));
                 break;
             case 1:
-                testBuilder.prefix(randomAsciiOfLength(10), FuzzyOptionsTests.randomFuzzyOptions());
+                testBuilder.prefix(randomAlphaOfLength(10), FuzzyOptionsTests.randomFuzzyOptions());
                 break;
             case 2:
-                testBuilder.prefix(randomAsciiOfLength(10), randomFrom(Fuzziness.ZERO, Fuzziness.ONE, Fuzziness.TWO));
+                testBuilder.prefix(randomAlphaOfLength(10), randomFrom(Fuzziness.ZERO, Fuzziness.ONE, Fuzziness.TWO));
                 break;
             case 3:
-                testBuilder.regex(randomAsciiOfLength(10), RegexOptionsTests.randomRegexOptions());
+                testBuilder.regex(randomAlphaOfLength(10), RegexOptionsTests.randomRegexOptions());
                 break;
         }
         Map<String, List<? extends ToXContent>> contextMap = new HashMap<>();
@@ -77,7 +77,7 @@ private static BuilderAndInfo randomSuggestionBuilderWithContextInfo() {
             for (int i = 0; i < numContext; i++) {
                 contexts.add(CategoryQueryContextTests.randomCategoryQueryContext());
             }
-            String name = randomAsciiOfLength(10);
+            String name = randomAlphaOfLength(10);
             contextMap.put(name, contexts);
             builderAndInfo.catContexts.add(name);
         }
@@ -87,7 +87,7 @@ private static BuilderAndInfo randomSuggestionBuilderWithContextInfo() {
             for (int i = 0; i < numContext; i++) {
                 contexts.add(GeoQueryContextTests.randomGeoQueryContext());
             }
-            String name = randomAsciiOfLength(10);
+            String name = randomAlphaOfLength(10);
             contextMap.put(name, contexts);
             builderAndInfo.geoContexts.add(name);
         }
@@ -114,7 +114,7 @@ protected void mutateSpecificParameters(CompletionSuggestionBuilder builder) thr
                 for (int i = 0; i < nCatContext; i++) {
                     contexts.add(CategoryQueryContextTests.randomCategoryQueryContext());
                 }
-                builder.contexts(Collections.singletonMap(randomAsciiOfLength(10), contexts));
+                builder.contexts(Collections.singletonMap(randomAlphaOfLength(10), contexts));
                 break;
             case 1:
                 int nGeoContext = randomIntBetween(1, 5);
@@ -122,16 +122,16 @@ protected void mutateSpecificParameters(CompletionSuggestionBuilder builder) thr
                 for (int i = 0; i < nGeoContext; i++) {
                     geoContexts.add(GeoQueryContextTests.randomGeoQueryContext());
                 }
-                builder.contexts(Collections.singletonMap(randomAsciiOfLength(10), geoContexts));
+                builder.contexts(Collections.singletonMap(randomAlphaOfLength(10), geoContexts));
                 break;
             case 2:
-                builder.prefix(randomAsciiOfLength(10), FuzzyOptionsTests.randomFuzzyOptions());
+                builder.prefix(randomAlphaOfLength(10), FuzzyOptionsTests.randomFuzzyOptions());
                 break;
             case 3:
-                builder.prefix(randomAsciiOfLength(10), randomFrom(Fuzziness.ZERO, Fuzziness.ONE, Fuzziness.TWO));
+                builder.prefix(randomAlphaOfLength(10), randomFrom(Fuzziness.ZERO, Fuzziness.ONE, Fuzziness.TWO));
                 break;
             case 4:
-                builder.regex(randomAsciiOfLength(10), RegexOptionsTests.randomRegexOptions());
+                builder.regex(randomAlphaOfLength(10), RegexOptionsTests.randomRegexOptions());
                 break;
             default:
                 throw new IllegalStateException("should not through");
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionTests.java b/core/src/test/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionTests.java
index 60722fb5f6da7..4b0e60a1d00db 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionTests.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionTests.java
@@ -35,7 +35,7 @@ public class CompletionSuggestionTests extends ESTestCase {
     public void testToReduce() throws Exception {
         List<Suggest.Suggestion<CompletionSuggestion.Entry>> shardSuggestions = new ArrayList<>();
         int nShards = randomIntBetween(1, 10);
-        String name = randomAsciiOfLength(10);
+        String name = randomAlphaOfLength(10);
         int size = randomIntBetween(3, 5);
         for (int i = 0; i < nShards; i++) {
             CompletionSuggestion suggestion = new CompletionSuggestion(name, size);
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/completion/FuzzyOptionsTests.java b/core/src/test/java/org/elasticsearch/search/suggest/completion/FuzzyOptionsTests.java
index 2a378bf1d78f4..4df7b9bf41503 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/completion/FuzzyOptionsTests.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/completion/FuzzyOptionsTests.java
@@ -19,12 +19,20 @@
 
 package org.elasticsearch.search.suggest.completion;
 
-import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.unit.Fuzziness;
+import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
 
-public class FuzzyOptionsTests extends WritableTestCase<FuzzyOptions> {
+import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
+
+public class FuzzyOptionsTests extends ESTestCase {
+
+    private static final int NUMBER_OF_RUNS = 20;
 
     public static FuzzyOptions randomFuzzyOptions() {
         final FuzzyOptions.Builder builder = FuzzyOptions.builder();
@@ -41,49 +49,45 @@ public static FuzzyOptions randomFuzzyOptions() {
         return builder.build();
     }
 
-    @Override
-    protected FuzzyOptions createTestModel() {
-        return randomFuzzyOptions();
-    }
-
-    @Override
     protected FuzzyOptions createMutation(FuzzyOptions original) throws IOException {
         final FuzzyOptions.Builder builder = FuzzyOptions.builder();
-        builder.setFuzziness(original.getEditDistance())
-            .setFuzzyPrefixLength(original.getFuzzyPrefixLength())
-            .setFuzzyMinLength(original.getFuzzyMinLength())
-            .setMaxDeterminizedStates(original.getMaxDeterminizedStates())
-            .setTranspositions(original.isTranspositions())
-            .setUnicodeAware(original.isUnicodeAware());
-        switch (randomIntBetween(0, 5)) {
-            case 0:
-                builder.setFuzziness(randomValueOtherThan(original.getEditDistance(), () -> randomFrom(0, 1, 2)));
-                break;
-            case 1:
-                builder.setFuzzyPrefixLength(randomValueOtherThan(original.getFuzzyPrefixLength(), () ->
-                    randomIntBetween(1, 3)));
-                break;
-            case 2:
-                builder.setFuzzyMinLength(randomValueOtherThan(original.getFuzzyMinLength(), () ->
-                    randomIntBetween(1, 3)));
-                break;
-            case 3:
-                builder.setMaxDeterminizedStates(randomValueOtherThan(original.getMaxDeterminizedStates(), () ->
-                    randomIntBetween(1, 10)));
-                break;
-            case 4:
-                builder.setTranspositions(!original.isTranspositions());
-                break;
-            case 5:
-                builder.setUnicodeAware(!original.isUnicodeAware());
-                break;
-        }
+        builder.setFuzziness(original.getEditDistance()).setFuzzyPrefixLength(original.getFuzzyPrefixLength())
+                .setFuzzyMinLength(original.getFuzzyMinLength()).setMaxDeterminizedStates(original.getMaxDeterminizedStates())
+                .setTranspositions(original.isTranspositions()).setUnicodeAware(original.isUnicodeAware());
+        List<Runnable> mutators = new ArrayList<>();
+        mutators.add(() -> builder.setFuzziness(randomValueOtherThan(original.getEditDistance(), () -> randomFrom(0, 1, 2))));
+
+        mutators.add(
+                () -> builder.setFuzzyPrefixLength(randomValueOtherThan(original.getFuzzyPrefixLength(), () -> randomIntBetween(1, 3))));
+        mutators.add(() -> builder.setFuzzyMinLength(randomValueOtherThan(original.getFuzzyMinLength(), () -> randomIntBetween(1, 3))));
+        mutators.add(() -> builder
+                .setMaxDeterminizedStates(randomValueOtherThan(original.getMaxDeterminizedStates(), () -> randomIntBetween(1, 10))));
+        mutators.add(() -> builder.setTranspositions(!original.isTranspositions()));
+        mutators.add(() -> builder.setUnicodeAware(!original.isUnicodeAware()));
+        randomFrom(mutators).run();
         return builder.build();
     }
 
-    @Override
-    protected FuzzyOptions readFrom(StreamInput in) throws IOException {
-        return new FuzzyOptions(in);
+    /**
+     * Test serialization and deserialization
+     */
+    public void testSerialization() throws IOException {
+        for (int i = 0; i < NUMBER_OF_RUNS; i++) {
+            FuzzyOptions testModel = randomFuzzyOptions();
+            FuzzyOptions deserializedModel = copyWriteable(testModel, new NamedWriteableRegistry(Collections.emptyList()),
+                    FuzzyOptions::new);
+            assertEquals(testModel, deserializedModel);
+            assertEquals(testModel.hashCode(), deserializedModel.hashCode());
+            assertNotSame(testModel, deserializedModel);
+        }
+    }
+
+    public void testEqualsAndHashCode() throws IOException {
+        for (int i = 0; i < NUMBER_OF_RUNS; i++) {
+            checkEqualsAndHashCode(randomFuzzyOptions(),
+                    original -> copyWriteable(original, new NamedWriteableRegistry(Collections.emptyList()), FuzzyOptions::new),
+                    this::createMutation);
+        }
     }
 
     public void testIllegalArguments() {
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/completion/GeoContextMappingTests.java b/core/src/test/java/org/elasticsearch/search/suggest/completion/GeoContextMappingTests.java
index 33624ed80027f..f4c65b247800e 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/completion/GeoContextMappingTests.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/completion/GeoContextMappingTests.java
@@ -20,18 +20,15 @@
 package org.elasticsearch.search.suggest.completion;
 
 import org.apache.lucene.index.IndexableField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.search.suggest.completion.context.ContextBuilder;
 import org.elasticsearch.search.suggest.completion.context.ContextMapping;
 import org.elasticsearch.search.suggest.completion.context.GeoContextMapping;
@@ -77,7 +74,7 @@ public void testIndexingWithNoContexts() throws Exception {
                 .field("weight", 4)
                 .endObject()
                 .startObject()
-                .field("input", "suggestion5", "suggestion6", "suggestion7")
+                .array("input", "suggestion5", "suggestion6", "suggestion7")
                 .field("weight", 5)
                 .endObject()
                 .endArray()
@@ -108,7 +105,7 @@ public void testIndexingWithSimpleContexts() throws Exception {
                 .startObject()
                 .startArray("completion")
                 .startObject()
-                .field("input", "suggestion5", "suggestion6", "suggestion7")
+                .array("input", "suggestion5", "suggestion6", "suggestion7")
                 .startObject("contexts")
                 .startObject("ctx")
                 .field("lat", 43.6624803)
@@ -143,7 +140,7 @@ public void testIndexingWithContextList() throws Exception {
         ParsedDocument parsedDocument = defaultMapper.parse("test", "type1", "1", jsonBuilder()
                 .startObject()
                     .startObject("completion")
-                        .field("input", "suggestion5", "suggestion6", "suggestion7")
+                        .array("input", "suggestion5", "suggestion6", "suggestion7")
                         .startObject("contexts")
                             .startArray("ctx")
                                 .startObject()
@@ -188,7 +185,7 @@ public void testIndexingWithMultipleContexts() throws Exception {
                 .startObject()
                 .startArray("completion")
                 .startObject()
-                .field("input", "suggestion5", "suggestion6", "suggestion7")
+                .array("input", "suggestion5", "suggestion6", "suggestion7")
                 .field("weight", 5)
                 .startObject("contexts")
                 .array("loc1", "ezs42e44yx96")
@@ -204,7 +201,7 @@ public void testIndexingWithMultipleContexts() throws Exception {
 
     public void testParsingQueryContextBasic() throws Exception {
         XContentBuilder builder = jsonBuilder().value("ezs42e44yx96");
-        XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.bytes());
+        XContentParser parser = createParser(JsonXContent.jsonXContent, builder.bytes());
         GeoContextMapping mapping = ContextBuilder.geo("geo").build();
         List<ContextMapping.InternalQueryContext> internalQueryContexts = mapping.parseQueryContext(createParseContext(parser));
         assertThat(internalQueryContexts.size(), equalTo(1 + 8));
@@ -223,7 +220,7 @@ public void testParsingQueryContextGeoPoint() throws Exception {
                 .field("lat", 23.654242)
                 .field("lon", 90.047153)
                 .endObject();
-        XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.bytes());
+        XContentParser parser = createParser(JsonXContent.jsonXContent, builder.bytes());
         GeoContextMapping mapping = ContextBuilder.geo("geo").build();
         List<ContextMapping.InternalQueryContext> internalQueryContexts = mapping.parseQueryContext(createParseContext(parser));
         assertThat(internalQueryContexts.size(), equalTo(1 + 8));
@@ -246,7 +243,7 @@ public void testParsingQueryContextObject() throws Exception {
                 .field("boost", 10)
                 .array("neighbours", 1, 2, 3)
                 .endObject();
-        XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.bytes());
+        XContentParser parser = createParser(JsonXContent.jsonXContent, builder.bytes());
         GeoContextMapping mapping = ContextBuilder.geo("geo").build();
         List<ContextMapping.InternalQueryContext> internalQueryContexts = mapping.parseQueryContext(createParseContext(parser));
         assertThat(internalQueryContexts.size(), equalTo(1 + 1 + 8 + 1 + 8 + 1 + 8));
@@ -284,7 +281,7 @@ public void testParsingQueryContextObjectArray() throws Exception {
                 .array("neighbours", 5)
                 .endObject()
                 .endArray();
-        XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.bytes());
+        XContentParser parser = createParser(JsonXContent.jsonXContent, builder.bytes());
         GeoContextMapping mapping = ContextBuilder.geo("geo").build();
         List<ContextMapping.InternalQueryContext> internalQueryContexts = mapping.parseQueryContext(createParseContext(parser));
         assertThat(internalQueryContexts.size(), equalTo(1 + 1 + 8 + 1 + 8 + 1 + 8 + 1 + 1 + 8));
@@ -327,7 +324,7 @@ public void testParsingQueryContextMixed() throws Exception {
                 .field("lon", 92.112583)
                 .endObject()
                 .endArray();
-        XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.bytes());
+        XContentParser parser = createParser(JsonXContent.jsonXContent, builder.bytes());
         GeoContextMapping mapping = ContextBuilder.geo("geo").build();
         List<ContextMapping.InternalQueryContext> internalQueryContexts = mapping.parseQueryContext(createParseContext(parser));
         assertThat(internalQueryContexts.size(), equalTo(1 + 1 + 8 + 1 + 8 + 1 + 8));
@@ -353,6 +350,6 @@ public void testParsingQueryContextMixed() throws Exception {
     }
 
     private static QueryParseContext createParseContext(XContentParser parser) {
-        return new QueryParseContext(new IndicesQueriesRegistry(), parser, ParseFieldMatcher.STRICT);
+        return new QueryParseContext(parser);
     };
 }
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/completion/QueryContextTestCase.java b/core/src/test/java/org/elasticsearch/search/suggest/completion/QueryContextTestCase.java
index 2960a6e23d879..192aa8ee1d50a 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/completion/QueryContextTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/completion/QueryContextTestCase.java
@@ -19,14 +19,11 @@
 
 package org.elasticsearch.search.suggest.completion;
 
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
@@ -50,10 +47,9 @@ public void testToXContext() throws IOException {
             QC toXContent = createTestModel();
             XContentBuilder builder = XContentFactory.jsonBuilder();
             toXContent.toXContent(builder, ToXContent.EMPTY_PARAMS);
-            BytesReference bytesReference = builder.bytes();
-            XContentParser parser = XContentFactory.xContent(bytesReference).createParser(bytesReference);
+            XContentParser parser = createParser(builder);
             parser.nextToken();
-            QC fromXContext = fromXContent(new QueryParseContext(new IndicesQueriesRegistry(), parser, ParseFieldMatcher.STRICT));
+            QC fromXContext = fromXContent(new QueryParseContext(parser));
             assertEquals(toXContent, fromXContext);
             assertEquals(toXContent.hashCode(), fromXContext.hashCode());
         }
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/completion/RegexOptionsTests.java b/core/src/test/java/org/elasticsearch/search/suggest/completion/RegexOptionsTests.java
index 81df2dfd7da20..a133ddf37029e 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/completion/RegexOptionsTests.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/completion/RegexOptionsTests.java
@@ -19,12 +19,16 @@
 
 package org.elasticsearch.search.suggest.completion;
 
-import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.index.query.RegexpFlag;
+import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
+import java.util.Collections;
 
-public class RegexOptionsTests extends WritableTestCase<RegexOptions> {
+public class RegexOptionsTests extends ESTestCase {
+
+    private static final int NUMBER_OF_RUNS = 20;
 
     public static RegexOptions randomRegexOptions() {
         final RegexOptions.Builder builder = RegexOptions.builder();
@@ -42,21 +46,24 @@ public static RegexOptions randomRegexOptions() {
         return builder.build();
     }
 
-    @Override
-    protected RegexOptions createTestModel() {
-        return randomRegexOptions();
-    }
-
-    @Override
     protected RegexOptions createMutation(RegexOptions original) throws IOException {
         final RegexOptions.Builder builder = RegexOptions.builder();
         builder.setMaxDeterminizedStates(randomValueOtherThan(original.getMaxDeterminizedStates(), () -> randomIntBetween(1, 10)));
         return builder.build();
     }
 
-    @Override
-    protected RegexOptions readFrom(StreamInput in) throws IOException {
-        return new RegexOptions(in);
+    /**
+     * Test serialization and deserialization
+     */
+    public void testSerialization() throws IOException {
+        for (int i = 0; i < NUMBER_OF_RUNS; i++) {
+            RegexOptions testOptions = randomRegexOptions();
+            RegexOptions deserializedModel = copyWriteable(testOptions, new NamedWriteableRegistry(Collections.emptyList()),
+                    RegexOptions::new);
+            assertEquals(testOptions, deserializedModel);
+            assertEquals(testOptions.hashCode(), deserializedModel.hashCode());
+            assertNotSame(testOptions, deserializedModel);
+        }
     }
 
     public void testIllegalArgument() {
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/completion/WritableTestCase.java b/core/src/test/java/org/elasticsearch/search/suggest/completion/WritableTestCase.java
deleted file mode 100644
index 9f6b452ce20e3..0000000000000
--- a/core/src/test/java/org/elasticsearch/search/suggest/completion/WritableTestCase.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.suggest.completion;
-
-import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.test.ESTestCase;
-
-import java.io.IOException;
-import java.util.Collections;
-
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.not;
-
-/**
- * Base class for testing serialization and equality for
- * {@link Writeable} models
- */
-public abstract class WritableTestCase<M extends Writeable> extends ESTestCase {
-
-    protected static final int NUMBER_OF_RUNS = 20;
-
-    /**
-     * create random model that is put under test
-     */
-    protected abstract M createTestModel();
-
-    /**
-     * mutate the given model so the returned model is different
-     */
-    protected abstract M createMutation(M original) throws IOException;
-
-    /**
-     * Read from a stream.
-     */
-    protected abstract M readFrom(StreamInput in) throws IOException;
-
-    /**
-     * Test serialization and deserialization of the tested model.
-     */
-    public void testSerialization() throws IOException {
-        for (int i = 0; i < NUMBER_OF_RUNS; i++) {
-            M testModel = createTestModel();
-            M deserializedModel = copyModel(testModel);
-            assertEquals(testModel, deserializedModel);
-            assertEquals(testModel.hashCode(), deserializedModel.hashCode());
-            assertNotSame(testModel, deserializedModel);
-        }
-    }
-
-    /**
-     * Test equality and hashCode properties
-     */
-    @SuppressWarnings("unchecked")
-    public void testEqualsAndHashcode() throws IOException {
-        M firstModel = createTestModel();
-        String modelName = firstModel.getClass().getSimpleName();
-        assertFalse(modelName + " is equal to null", firstModel.equals(null));
-        assertFalse(modelName + " is equal to incompatible type", firstModel.equals(""));
-        assertTrue(modelName + " is not equal to self", firstModel.equals(firstModel));
-        assertThat("same "+ modelName + "'s hashcode returns different values if called multiple times", firstModel.hashCode(),
-                equalTo(firstModel.hashCode()));
-        assertThat("different " + modelName + " should not be equal", createMutation(firstModel), not(equalTo(firstModel)));
-
-        M secondModel = copyModel(firstModel);
-        assertTrue(modelName + " is not equal to self", secondModel.equals(secondModel));
-        assertTrue(modelName + " is not equal to its copy", firstModel.equals(secondModel));
-        assertTrue("equals is not symmetric", secondModel.equals(firstModel));
-        assertThat(modelName + " copy's hashcode is different from original hashcode", secondModel.hashCode(),
-                equalTo(firstModel.hashCode()));
-
-        M thirdModel = copyModel(secondModel);
-        assertTrue(modelName + " is not equal to self", thirdModel.equals(thirdModel));
-        assertTrue(modelName + " is not equal to its copy", secondModel.equals(thirdModel));
-        assertThat(modelName + " copy's hashcode is different from original hashcode", secondModel.hashCode(),
-                equalTo(thirdModel.hashCode()));
-        assertTrue("equals is not transitive", firstModel.equals(thirdModel));
-        assertThat(modelName + " copy's hashcode is different from original hashcode", firstModel.hashCode(),
-                equalTo(thirdModel.hashCode()));
-        assertTrue(modelName + " equals is not symmetric", thirdModel.equals(secondModel));
-        assertTrue(modelName + " equals is not symmetric", thirdModel.equals(firstModel));
-    }
-
-    private M copyModel(M original) throws IOException {
-        try (BytesStreamOutput output = new BytesStreamOutput()) {
-            original.writeTo(output);
-            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), provideNamedWritableRegistry())) {
-                return readFrom(in);
-            }
-        }
-    }
-
-    protected NamedWriteableRegistry provideNamedWritableRegistry() {
-        return new NamedWriteableRegistry(Collections.emptyList());
-    }
-}
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorTests.java b/core/src/test/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorTests.java
index 846d3193f6d71..621ec2089a19e 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorTests.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorTests.java
@@ -19,38 +19,35 @@
 
 package org.elasticsearch.search.suggest.phrase;
 
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.search.suggest.phrase.PhraseSuggestionContext.DirectCandidateGenerator;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
-import static org.hamcrest.Matchers.equalTo;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.function.Supplier;
 
-public class DirectCandidateGeneratorTests extends ESTestCase{
+import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
 
-    private static final IndicesQueriesRegistry mockRegistry = new IndicesQueriesRegistry();
+public class DirectCandidateGeneratorTests extends ESTestCase {
     private static final int NUMBER_OF_RUNS = 20;
 
-
-
     /**
      * Test serialization and deserialization of the generator
      */
     public void testSerialization() throws IOException {
         for (int runs = 0; runs < NUMBER_OF_RUNS; runs++) {
             DirectCandidateGeneratorBuilder original = randomCandidateGenerator();
-            DirectCandidateGeneratorBuilder deserialized = serializedCopy(original);
+            DirectCandidateGeneratorBuilder deserialized = copy(original);
             assertEquals(deserialized, original);
             assertEquals(deserialized.hashCode(), original.hashCode());
             assertNotSame(deserialized, original);
@@ -62,49 +59,40 @@ public void testSerialization() throws IOException {
      */
     public void testEqualsAndHashcode() throws IOException {
         for (int runs = 0; runs < NUMBER_OF_RUNS; runs++) {
-            DirectCandidateGeneratorBuilder first = randomCandidateGenerator();
-            assertFalse("generator is equal to null", first.equals(null));
-            assertFalse("generator is equal to incompatible type", first.equals(""));
-            assertTrue("generator is not equal to self", first.equals(first));
-            assertThat("same generator's hashcode returns different values if called multiple times", first.hashCode(),
-                    equalTo(first.hashCode()));
-
-            DirectCandidateGeneratorBuilder second = serializedCopy(first);
-            assertTrue("generator is not equal to self", second.equals(second));
-            assertTrue("generator is not equal to its copy", first.equals(second));
-            assertTrue("equals is not symmetric", second.equals(first));
-            assertThat("generator copy's hashcode is different from original hashcode", second.hashCode(), equalTo(first.hashCode()));
-
-            DirectCandidateGeneratorBuilder third = serializedCopy(second);
-            assertTrue("generator is not equal to self", third.equals(third));
-            assertTrue("generator is not equal to its copy", second.equals(third));
-            assertThat("generator copy's hashcode is different from original hashcode", second.hashCode(), equalTo(third.hashCode()));
-            assertTrue("equals is not transitive", first.equals(third));
-            assertThat("generator copy's hashcode is different from original hashcode", first.hashCode(), equalTo(third.hashCode()));
-            assertTrue("equals is not symmetric", third.equals(second));
-            assertTrue("equals is not symmetric", third.equals(first));
-
-            // test for non-equality, check that all fields are covered by changing one by one
-            first = new DirectCandidateGeneratorBuilder("aaa");
-            assertEquals(first, serializedCopy(first));
-            second = new DirectCandidateGeneratorBuilder("bbb");
-            assertNotEquals(first, second);
-            assertNotEquals(first.accuracy(0.1f), serializedCopy(first).accuracy(0.2f));
-            assertNotEquals(first.maxEdits(1), serializedCopy(first).maxEdits(2));
-            assertNotEquals(first.maxInspections(1), serializedCopy(first).maxInspections(2));
-            assertNotEquals(first.maxTermFreq(0.1f), serializedCopy(first).maxTermFreq(0.2f));
-            assertNotEquals(first.minDocFreq(0.1f), serializedCopy(first).minDocFreq(0.2f));
-            assertNotEquals(first.minWordLength(1), serializedCopy(first).minWordLength(2));
-            assertNotEquals(first.postFilter("postFilter"), serializedCopy(first).postFilter("postFilter_other"));
-            assertNotEquals(first.preFilter("preFilter"), serializedCopy(first).preFilter("preFilter_other"));
-            assertNotEquals(first.prefixLength(1), serializedCopy(first).prefixLength(2));
-            assertNotEquals(first.size(1), serializedCopy(first).size(2));
-            assertNotEquals(first.sort("score"), serializedCopy(first).sort("frequency"));
-            assertNotEquals(first.stringDistance("levenstein"), serializedCopy(first).sort("ngram"));
-            assertNotEquals(first.suggestMode("missing"), serializedCopy(first).suggestMode("always"));
+            final DirectCandidateGeneratorBuilder original = randomCandidateGenerator();
+            checkEqualsAndHashCode(original, DirectCandidateGeneratorTests::copy, DirectCandidateGeneratorTests::mutate);
         }
     }
 
+    private static DirectCandidateGeneratorBuilder mutate(DirectCandidateGeneratorBuilder original) throws IOException {
+        DirectCandidateGeneratorBuilder mutation = copy(original);
+        List<Supplier<DirectCandidateGeneratorBuilder>> mutators = new ArrayList<>();
+        mutators.add(() -> new DirectCandidateGeneratorBuilder(original.field() + "_other"));
+        mutators.add(() -> mutation.accuracy(original.accuracy() == null ? 0.1f : original.accuracy() + 0.1f));
+        mutators.add(() -> {
+            Integer maxEdits = original.maxEdits() == null ? 1 : original.maxEdits();
+            if (maxEdits == 1) {
+                maxEdits = 2;
+            } else {
+                maxEdits = 1;
+            }
+            return mutation.maxEdits(maxEdits);
+        });
+        mutators.add(() -> mutation.maxInspections(original.maxInspections() == null ? 1 : original.maxInspections() + 1));
+        mutators.add(() -> mutation.minWordLength(original.minWordLength() == null ? 1 : original.minWordLength() + 1));
+        mutators.add(() -> mutation.prefixLength(original.prefixLength() == null ? 1 : original.prefixLength() + 1));
+        mutators.add(() -> mutation.size(original.size() == null ? 1 : original.size() + 1));
+        mutators.add(() -> mutation.maxTermFreq(original.maxTermFreq() == null ? 0.1f : original.maxTermFreq() + 0.1f));
+        mutators.add(() -> mutation.minDocFreq(original.minDocFreq() == null ? 0.1f : original.minDocFreq() + 0.1f));
+        mutators.add(() -> mutation.postFilter(original.postFilter() == null ? "postFilter" : original.postFilter() + "_other"));
+        mutators.add(() -> mutation.preFilter(original.preFilter() == null ? "preFilter" : original.preFilter() + "_other"));
+        mutators.add(() -> mutation.sort(original.sort() == null ? "score" : original.sort() + "_other"));
+        mutators.add(
+                () -> mutation.stringDistance(original.stringDistance() == null ? "levenstein" : original.stringDistance() + "_other"));
+        mutators.add(() -> mutation.suggestMode(original.suggestMode() == null ? "missing" : original.suggestMode() + "_other"));
+        return randomFrom(mutators).get();
+    }
+
     /**
      *  creates random candidate generator, renders it to xContent and back to new instance that should be equal to original
      */
@@ -116,10 +104,9 @@ public void testFromXContent() throws IOException {
                 builder.prettyPrint();
             }
             generator.toXContent(builder, ToXContent.EMPTY_PARAMS);
-            XContentParser parser = XContentHelper.createParser(shuffleXContent(builder).bytes());
-            QueryParseContext context = new QueryParseContext(mockRegistry, parser, ParseFieldMatcher.STRICT);
+            XContentParser parser = createParser(shuffleXContent(builder));
             parser.nextToken();
-            DirectCandidateGeneratorBuilder secondGenerator = DirectCandidateGeneratorBuilder.fromXContent(context);
+            DirectCandidateGeneratorBuilder secondGenerator = DirectCandidateGeneratorBuilder.fromXContent(parser);
             assertNotSame(generator, secondGenerator);
             assertEquals(generator, secondGenerator);
             assertEquals(generator.hashCode(), secondGenerator.hashCode());
@@ -174,11 +161,10 @@ public void testIllegalXContent() throws IOException {
                 "[direct_generator] size doesn't support values of type: START_ARRAY");
     }
 
-    private static void assertIllegalXContent(String directGenerator, Class<? extends Exception> exceptionClass, String exceptionMsg)
+    private void assertIllegalXContent(String directGenerator, Class<? extends Exception> exceptionClass, String exceptionMsg)
             throws IOException {
-        XContentParser parser = XContentFactory.xContent(directGenerator).createParser(directGenerator);
-        QueryParseContext context = new QueryParseContext(mockRegistry, parser, ParseFieldMatcher.STRICT);
-        Exception e = expectThrows(exceptionClass, () -> DirectCandidateGeneratorBuilder.fromXContent(context));
+        XContentParser parser = createParser(JsonXContent.jsonXContent, directGenerator);
+        Exception e = expectThrows(exceptionClass, () -> DirectCandidateGeneratorBuilder.fromXContent(parser));
         assertEquals(exceptionMsg, e.getMessage());
     }
 
@@ -186,7 +172,7 @@ private static void assertIllegalXContent(String directGenerator, Class<? extend
      * create random {@link DirectCandidateGeneratorBuilder}
      */
     public static DirectCandidateGeneratorBuilder randomCandidateGenerator() {
-        DirectCandidateGeneratorBuilder generator = new DirectCandidateGeneratorBuilder(randomAsciiOfLength(10));
+        DirectCandidateGeneratorBuilder generator = new DirectCandidateGeneratorBuilder(randomAlphaOfLength(10));
         maybeSet(generator::accuracy, randomFloat());
         maybeSet(generator::maxEdits, randomIntBetween(1, 2));
         maybeSet(generator::maxInspections, randomIntBetween(1, 20));
@@ -194,8 +180,8 @@ public static DirectCandidateGeneratorBuilder randomCandidateGenerator() {
         maybeSet(generator::minDocFreq, randomFloat());
         maybeSet(generator::minWordLength, randomIntBetween(1, 20));
         maybeSet(generator::prefixLength, randomIntBetween(1, 20));
-        maybeSet(generator::preFilter, randomAsciiOfLengthBetween(1, 20));
-        maybeSet(generator::postFilter, randomAsciiOfLengthBetween(1, 20));
+        maybeSet(generator::preFilter, randomAlphaOfLengthBetween(1, 20));
+        maybeSet(generator::postFilter, randomAlphaOfLengthBetween(1, 20));
         maybeSet(generator::size, randomIntBetween(1, 20));
         maybeSet(generator::sort, randomFrom("score", "frequency"));
         maybeSet(generator::stringDistance, randomFrom("internal", "damerau_levenshtein", "levenstein", "jarowinkler", "ngram"));
@@ -203,12 +189,7 @@ public static DirectCandidateGeneratorBuilder randomCandidateGenerator() {
         return generator;
     }
 
-    private static DirectCandidateGeneratorBuilder serializedCopy(DirectCandidateGeneratorBuilder original) throws IOException {
-        try (BytesStreamOutput output = new BytesStreamOutput()) {
-            original.writeTo(output);
-            try (StreamInput in = output.bytes().streamInput()) {
-                return new DirectCandidateGeneratorBuilder(in);
-            }
-        }
+    private static DirectCandidateGeneratorBuilder copy(DirectCandidateGeneratorBuilder original) throws IOException {
+        return copyWriteable(original, new NamedWriteableRegistry(Collections.emptyList()), DirectCandidateGeneratorBuilder::new);
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/phrase/LaplaceModelTests.java b/core/src/test/java/org/elasticsearch/search/suggest/phrase/LaplaceModelTests.java
index dae87ef057079..22917c6c31292 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/phrase/LaplaceModelTests.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/phrase/LaplaceModelTests.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.search.suggest.phrase;
 
-import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
 
@@ -54,7 +54,7 @@ void assertWordScorer(WordScorer wordScorer, SmoothingModel input) {
     }
 
     @Override
-    protected SmoothingModel fromXContent(QueryParseContext context) throws IOException {
-        return Laplace.innerFromXContent(context);
+    protected SmoothingModel fromXContent(XContentParser parser) throws IOException {
+        return Laplace.fromXContent(parser);
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/phrase/LinearInterpolationModelTests.java b/core/src/test/java/org/elasticsearch/search/suggest/phrase/LinearInterpolationModelTests.java
index b3689977840ea..7efbab4584fc0 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/phrase/LinearInterpolationModelTests.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/phrase/LinearInterpolationModelTests.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.search.suggest.phrase;
 
-import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
 
@@ -73,7 +73,7 @@ void assertWordScorer(WordScorer wordScorer, SmoothingModel in) {
     }
 
     @Override
-    protected SmoothingModel fromXContent(QueryParseContext context) throws IOException {
-        return LinearInterpolation.innerFromXContent(context);
+    protected SmoothingModel fromXContent(XContentParser parser) throws IOException {
+        return LinearInterpolation.fromXContent(parser);
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilderTests.java b/core/src/test/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilderTests.java
index 4c5b3b8ca6088..8b10a17064e56 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilderTests.java
@@ -33,13 +33,13 @@ protected PhraseSuggestionBuilder randomSuggestionBuilder() {
     }
 
     public static PhraseSuggestionBuilder randomPhraseSuggestionBuilder() {
-        PhraseSuggestionBuilder testBuilder = new PhraseSuggestionBuilder(randomAsciiOfLengthBetween(2, 20));
+        PhraseSuggestionBuilder testBuilder = new PhraseSuggestionBuilder(randomAlphaOfLengthBetween(2, 20));
         setCommonPropertiesOnRandomBuilder(testBuilder);
         maybeSet(testBuilder::maxErrors, randomFloat());
-        maybeSet(testBuilder::separator, randomAsciiOfLengthBetween(1, 10));
+        maybeSet(testBuilder::separator, randomAlphaOfLengthBetween(1, 10));
         maybeSet(testBuilder::realWordErrorLikelihood, randomFloat());
         maybeSet(testBuilder::confidence, randomFloat());
-        maybeSet(testBuilder::collateQuery, randomAsciiOfLengthBetween(3, 20));
+        maybeSet(testBuilder::collateQuery, randomAlphaOfLengthBetween(3, 20));
         // collate query prune and parameters will only be used when query is set
         if (testBuilder.collateQuery() != null) {
             maybeSet(testBuilder::collatePrune, randomBoolean());
@@ -47,14 +47,14 @@ public static PhraseSuggestionBuilder randomPhraseSuggestionBuilder() {
                 Map<String, Object> collateParams = new HashMap<>();
                 int numParams = randomIntBetween(1, 5);
                 for (int i = 0; i < numParams; i++) {
-                    collateParams.put(randomAsciiOfLength(5), randomAsciiOfLength(5));
+                    collateParams.put(randomAlphaOfLength(5), randomAlphaOfLength(5));
                 }
                 testBuilder.collateParams(collateParams );
             }
         }
         if (randomBoolean()) {
             // preTag, postTag
-            testBuilder.highlight(randomAsciiOfLengthBetween(3, 20), randomAsciiOfLengthBetween(3, 20));
+            testBuilder.highlight(randomAlphaOfLengthBetween(3, 20), randomAlphaOfLengthBetween(3, 20));
         }
         maybeSet(testBuilder::gramSize, randomIntBetween(1, 5));
         maybeSet(testBuilder::forceUnigrams, randomBoolean());
@@ -106,14 +106,14 @@ protected void mutateSpecificParameters(PhraseSuggestionBuilder builder) throws
             builder.tokenLimit(randomValueOtherThan(builder.tokenLimit(), () -> randomIntBetween(1, 20)));
             break;
         case 5:
-            builder.separator(randomValueOtherThan(builder.separator(), () -> randomAsciiOfLengthBetween(1, 10)));
+            builder.separator(randomValueOtherThan(builder.separator(), () -> randomAlphaOfLengthBetween(1, 10)));
             break;
         case 6:
             Script collateQuery = builder.collateQuery();
             if (collateQuery != null) {
-                builder.collateQuery(randomValueOtherThan(collateQuery.getScript(), () -> randomAsciiOfLengthBetween(3, 20)));
+                builder.collateQuery(randomValueOtherThan(collateQuery.getIdOrCode(), () -> randomAlphaOfLengthBetween(3, 20)));
             } else {
-                builder.collateQuery(randomAsciiOfLengthBetween(3, 20));
+                builder.collateQuery(randomAlphaOfLengthBetween(3, 20));
             }
             break;
         case 7:
@@ -126,7 +126,7 @@ protected void mutateSpecificParameters(PhraseSuggestionBuilder builder) throws
                 // simply double both values
                 builder.highlight(builder.preTag() + builder.preTag(), builder.postTag() + builder.postTag());
             } else {
-                builder.highlight(randomAsciiOfLengthBetween(3, 20), randomAsciiOfLengthBetween(3, 20));
+                builder.highlight(randomAlphaOfLengthBetween(3, 20), randomAlphaOfLengthBetween(3, 20));
             }
             break;
         case 9:
@@ -134,7 +134,7 @@ protected void mutateSpecificParameters(PhraseSuggestionBuilder builder) throws
             break;
         case 10:
             Map<String, Object> collateParams = builder.collateParams() == null ? new HashMap<>(1) : builder.collateParams();
-            collateParams.put(randomAsciiOfLength(5), randomAsciiOfLength(5));
+            collateParams.put(randomAlphaOfLength(5), randomAlphaOfLength(5));
             builder.collateParams(collateParams);
             break;
         case 11:
@@ -155,7 +155,7 @@ public void testInvalidParameters() throws IOException {
         e = expectThrows(IllegalArgumentException.class, () -> new PhraseSuggestionBuilder(""));
         assertEquals("suggestion field name is empty", e.getMessage());
 
-        PhraseSuggestionBuilder builder = new PhraseSuggestionBuilder(randomAsciiOfLengthBetween(2, 20));
+        PhraseSuggestionBuilder builder = new PhraseSuggestionBuilder(randomAlphaOfLengthBetween(2, 20));
 
         e = expectThrows(IllegalArgumentException.class, () -> builder.gramSize(0));
         assertEquals("gramSize must be >= 1", e.getMessage());
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/phrase/SmoothingModelTestCase.java b/core/src/test/java/org/elasticsearch/search/suggest/phrase/SmoothingModelTestCase.java
index c3d001b1fbcfc..e75d01739ccb8 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/phrase/SmoothingModelTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/phrase/SmoothingModelTestCase.java
@@ -30,20 +30,13 @@
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.MultiFields;
 import org.apache.lucene.store.RAMDirectory;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.AfterClass;
@@ -55,8 +48,7 @@
 import java.util.List;
 import java.util.Map;
 
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.not;
+import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
 
 public abstract class SmoothingModelTestCase extends ESTestCase {
 
@@ -89,7 +81,7 @@ public static void afterClass() throws Exception {
      */
     protected abstract SmoothingModel createMutation(SmoothingModel original) throws IOException;
 
-    protected abstract SmoothingModel fromXContent(QueryParseContext context) throws IOException;
+    protected abstract SmoothingModel fromXContent(XContentParser parser) throws IOException;
 
     /**
      * Test that creates new smoothing model from a random test smoothing model and checks both for equality
@@ -103,10 +95,9 @@ public void testFromXContent() throws IOException {
         contentBuilder.startObject();
         testModel.innerToXContent(contentBuilder, ToXContent.EMPTY_PARAMS);
         contentBuilder.endObject();
-        XContentParser parser = XContentHelper.createParser(shuffleXContent(contentBuilder).bytes());
-        QueryParseContext context = new QueryParseContext(new IndicesQueriesRegistry(), parser, ParseFieldMatcher.STRICT);
+        XContentParser parser = createParser(shuffleXContent(contentBuilder));
         parser.nextToken();  // go to start token, real parsing would do that in the outer element parser
-        SmoothingModel parsedModel = fromXContent(context);
+        SmoothingModel parsedModel = fromXContent(parser);
         assertNotSame(testModel, parsedModel);
         assertEquals(testModel, parsedModel);
         assertEquals(testModel.hashCode(), parsedModel.hashCode());
@@ -117,7 +108,6 @@ public void testFromXContent() throws IOException {
      */
     public void testBuildWordScorer() throws IOException {
         SmoothingModel testModel = createTestModel();
-
         Map<String, Analyzer> mapping = new HashMap<>();
         mapping.put("field", new WhitespaceAnalyzer());
         PerFieldAnalyzerWrapper wrapper = new PerFieldAnalyzerWrapper(new WhitespaceAnalyzer(), mapping);
@@ -142,7 +132,7 @@ public void testBuildWordScorer() throws IOException {
      */
     public void testSerialization() throws IOException {
         SmoothingModel testModel = createTestModel();
-        SmoothingModel deserializedModel = copyModel(testModel);
+        SmoothingModel deserializedModel = copy(testModel);
         assertEquals(testModel, deserializedModel);
         assertEquals(testModel.hashCode(), deserializedModel.hashCode());
         assertNotSame(testModel, deserializedModel);
@@ -151,42 +141,12 @@ public void testSerialization() throws IOException {
     /**
      * Test equality and hashCode properties
      */
-    @SuppressWarnings("unchecked")
     public void testEqualsAndHashcode() throws IOException {
-        SmoothingModel firstModel = createTestModel();
-        assertFalse("smoothing model is equal to null", firstModel.equals(null));
-        assertFalse("smoothing model is equal to incompatible type", firstModel.equals(""));
-        assertTrue("smoothing model is not equal to self", firstModel.equals(firstModel));
-        assertThat("same smoothing model's hashcode returns different values if called multiple times", firstModel.hashCode(),
-                equalTo(firstModel.hashCode()));
-        assertThat("different smoothing models should not be equal", createMutation(firstModel), not(equalTo(firstModel)));
-
-        SmoothingModel secondModel = copyModel(firstModel);
-        assertTrue("smoothing model is not equal to self", secondModel.equals(secondModel));
-        assertTrue("smoothing model is not equal to its copy", firstModel.equals(secondModel));
-        assertTrue("equals is not symmetric", secondModel.equals(firstModel));
-        assertThat("smoothing model copy's hashcode is different from original hashcode", secondModel.hashCode(),
-                equalTo(firstModel.hashCode()));
-
-        SmoothingModel thirdModel = copyModel(secondModel);
-        assertTrue("smoothing model is not equal to self", thirdModel.equals(thirdModel));
-        assertTrue("smoothing model is not equal to its copy", secondModel.equals(thirdModel));
-        assertThat("smoothing model copy's hashcode is different from original hashcode", secondModel.hashCode(),
-                equalTo(thirdModel.hashCode()));
-        assertTrue("equals is not transitive", firstModel.equals(thirdModel));
-        assertThat("smoothing model copy's hashcode is different from original hashcode", firstModel.hashCode(),
-                equalTo(thirdModel.hashCode()));
-        assertTrue("equals is not symmetric", thirdModel.equals(secondModel));
-        assertTrue("equals is not symmetric", thirdModel.equals(firstModel));
+        checkEqualsAndHashCode(createTestModel(), this::copy, this::createMutation);
     }
 
-    static SmoothingModel copyModel(SmoothingModel original) throws IOException {
-        try (BytesStreamOutput output = new BytesStreamOutput()) {
-            original.writeTo(output);
-            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
-                return namedWriteableRegistry.getReader(SmoothingModel.class, original.getWriteableName()).read(in);
-            }
-        }
+    private SmoothingModel copy(SmoothingModel original) throws IOException {
+        return ESTestCase.copyWriteable(original, namedWriteableRegistry,
+                namedWriteableRegistry.getReader(SmoothingModel.class, original.getWriteableName()));
     }
-
 }
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/phrase/StupidBackoffModelTests.java b/core/src/test/java/org/elasticsearch/search/suggest/phrase/StupidBackoffModelTests.java
index 57f84c3754347..c0e0abecb3770 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/phrase/StupidBackoffModelTests.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/phrase/StupidBackoffModelTests.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.search.suggest.phrase;
 
-import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
 
@@ -53,7 +53,7 @@ void assertWordScorer(WordScorer wordScorer, SmoothingModel input) {
     }
 
     @Override
-    protected SmoothingModel fromXContent(QueryParseContext context) throws IOException {
-        return StupidBackoff.innerFromXContent(context);
+    protected SmoothingModel fromXContent(XContentParser parser) throws IOException {
+        return StupidBackoff.fromXContent(parser);
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilderTests.java b/core/src/test/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilderTests.java
index 588957137ff03..c5471f3ee0acf 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilderTests.java
@@ -21,8 +21,9 @@
 
 import com.carrotsearch.randomizedtesting.generators.RandomStrings;
 
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.search.suggest.AbstractSuggestionBuilderTestCase;
-import org.elasticsearch.search.suggest.DirectSpellcheckerSettings;
 import org.elasticsearch.search.suggest.SortBy;
 import org.elasticsearch.search.suggest.SuggestBuilder;
 import org.elasticsearch.search.suggest.term.TermSuggestionBuilder.StringDistanceImpl;
@@ -57,7 +58,7 @@ protected TermSuggestionBuilder randomSuggestionBuilder() {
      * Creates a random TermSuggestionBuilder
      */
     public static TermSuggestionBuilder randomTermSuggestionBuilder() {
-        TermSuggestionBuilder testBuilder = new TermSuggestionBuilder(randomAsciiOfLengthBetween(2, 20));
+        TermSuggestionBuilder testBuilder = new TermSuggestionBuilder(randomAlphaOfLengthBetween(2, 20));
         setCommonPropertiesOnRandomBuilder(testBuilder);
         maybeSet(testBuilder::suggestMode, randomSuggestMode());
         maybeSet(testBuilder::accuracy, randomFloat());
@@ -150,7 +151,7 @@ public void testInvalidParameters() throws IOException {
         e = expectThrows(IllegalArgumentException.class, () -> new TermSuggestionBuilder(""));
         assertEquals("suggestion field name is empty", e.getMessage());
 
-        TermSuggestionBuilder builder = new TermSuggestionBuilder(randomAsciiOfLengthBetween(2, 20));
+        TermSuggestionBuilder builder = new TermSuggestionBuilder(randomAlphaOfLengthBetween(2, 20));
 
         // test invalid accuracy values
         expectThrows(IllegalArgumentException.class, () -> builder.accuracy(-0.5f));
@@ -192,7 +193,7 @@ public void testInvalidParameters() throws IOException {
     }
 
     public void testDefaultValuesSet() {
-        TermSuggestionBuilder builder = new TermSuggestionBuilder(randomAsciiOfLengthBetween(2, 20));
+        TermSuggestionBuilder builder = new TermSuggestionBuilder(randomAlphaOfLengthBetween(2, 20));
         assertEquals(DEFAULT_ACCURACY, builder.accuracy(), Float.MIN_VALUE);
         assertEquals(DEFAULT_MAX_EDITS, builder.maxEdits());
         assertEquals(DEFAULT_MAX_INSPECTIONS, builder.maxInspections());
@@ -215,32 +216,11 @@ public void testMalformedJson() {
                          "    }\n" +
                          "  }\n" +
                          "}";
-        try {
-            final SuggestBuilder suggestBuilder = SuggestBuilder.fromXContent(newParseContext(suggest), suggesters);
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, suggest)) {
+            final SuggestBuilder suggestBuilder = SuggestBuilder.fromXContent(parser);
             fail("Should not have been able to create SuggestBuilder from malformed JSON: " + suggestBuilder);
         } catch (Exception e) {
             assertThat(e.getMessage(), containsString("parsing failed"));
         }
     }
-
-    private void assertSpellcheckerSettings(DirectSpellcheckerSettings oldSettings, DirectSpellcheckerSettings newSettings) {
-        final double delta = 0.0d;
-        // make sure the objects aren't the same
-        assertNotSame(oldSettings, newSettings);
-        // make sure the objects aren't null
-        assertNotNull(oldSettings);
-        assertNotNull(newSettings);
-        // and now, make sure they are equal..
-        assertEquals(oldSettings.accuracy(), newSettings.accuracy(), delta);
-        assertEquals(oldSettings.maxEdits(), newSettings.maxEdits());
-        assertEquals(oldSettings.maxInspections(), newSettings.maxInspections());
-        assertEquals(oldSettings.maxTermFreq(), newSettings.maxTermFreq(), delta);
-        assertEquals(oldSettings.minDocFreq(), newSettings.minDocFreq(), delta);
-        assertEquals(oldSettings.minWordLength(), newSettings.minWordLength());
-        assertEquals(oldSettings.prefixLength(), newSettings.prefixLength());
-        assertEquals(oldSettings.sort(), newSettings.sort());
-        assertEquals(oldSettings.stringDistance().getClass(), newSettings.stringDistance().getClass());
-        assertEquals(oldSettings.suggestMode().getClass(), newSettings.suggestMode().getClass());
-    }
-
 }
diff --git a/core/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java b/core/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java
index 5562e2d4026fa..0483ea8a0284d 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java
@@ -141,6 +141,20 @@ public SnapshotInfo waitForCompletion(String repository, String snapshotName, Ti
         return null;
     }
 
+    public static String blockMasterFromFinalizingSnapshotOnIndexFile(final String repositoryName) {
+        final String masterName = internalCluster().getMasterName();
+        ((MockRepository)internalCluster().getInstance(RepositoriesService.class, masterName)
+            .repository(repositoryName)).setBlockOnWriteIndexFile(true);
+        return masterName;
+    }
+
+    public static String blockMasterFromFinalizingSnapshotOnSnapFile(final String repositoryName) {
+        final String masterName = internalCluster().getMasterName();
+        ((MockRepository)internalCluster().getInstance(RepositoriesService.class, masterName)
+            .repository(repositoryName)).setBlockAndFailOnWriteSnapFiles(true);
+        return masterName;
+    }
+
     public static String blockNodeWithIndex(final String repositoryName, final String indexName) {
         for(String node : internalCluster().nodesInclude(indexName)) {
             ((MockRepository)internalCluster().getInstance(RepositoriesService.class, node).repository(repositoryName))
@@ -182,18 +196,15 @@ public static void unblockNode(final String repository, final String node) {
     }
 
     protected void assertBusyPendingTasks(final String taskPrefix, final int expectedCount) throws Exception {
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                PendingClusterTasksResponse tasks = client().admin().cluster().preparePendingClusterTasks().get();
-                int count = 0;
-                for(PendingClusterTask task : tasks) {
-                    if (task.getSource().toString().startsWith(taskPrefix)) {
-                        count++;
-                    }
+        assertBusy(() -> {
+            PendingClusterTasksResponse tasks = client().admin().cluster().preparePendingClusterTasks().get();
+            int count = 0;
+            for(PendingClusterTask task : tasks) {
+                if (task.getSource().toString().startsWith(taskPrefix)) {
+                    count++;
                 }
-                assertThat(count, greaterThanOrEqualTo(expectedCount));
             }
+            assertThat(count, greaterThanOrEqualTo(expectedCount));
         }, 1, TimeUnit.MINUTES);
     }
 
diff --git a/core/src/test/java/org/elasticsearch/snapshots/BlobStoreFormatIT.java b/core/src/test/java/org/elasticsearch/snapshots/BlobStoreFormatIT.java
index 183fd17b60fb8..d8754c44368ff 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/BlobStoreFormatIT.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/BlobStoreFormatIT.java
@@ -21,7 +21,7 @@
 
 import org.elasticsearch.ElasticsearchCorruptionException;
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.blobstore.BlobContainer;
 import org.elasticsearch.common.blobstore.BlobMetaData;
 import org.elasticsearch.common.blobstore.BlobPath;
@@ -35,7 +35,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.FromXContentBuilder;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -61,16 +61,13 @@
 
 public class BlobStoreFormatIT extends AbstractSnapshotIntegTestCase {
 
-    private static final ParseFieldMatcher parseFieldMatcher = new ParseFieldMatcher(Settings.EMPTY);
-
     public static final String BLOB_CODEC = "blob";
 
-    private static class BlobObj implements ToXContent, FromXContentBuilder<BlobObj> {
-        public static final BlobObj PROTO = new BlobObj("");
+    private static class BlobObj implements ToXContent {
 
         private final String text;
 
-        public BlobObj(String text) {
+        BlobObj(String text) {
             this.text = text;
         }
 
@@ -78,8 +75,7 @@ public String getText() {
             return text;
         }
 
-        @Override
-        public BlobObj fromXContent(XContentParser parser, ParseFieldMatcher parseFieldMatcher) throws IOException {
+        public static BlobObj fromXContent(XContentParser parser) throws IOException {
             String text = null;
             XContentParser.Token token = parser.currentToken();
             if (token == null) {
@@ -125,8 +121,9 @@ private static final class LegacyEmulationBlobStoreFormat<T extends ToXContent>
 
         protected final boolean compress;
 
-        public LegacyEmulationBlobStoreFormat(String blobNameFormat, FromXContentBuilder<T> reader, ParseFieldMatcher parseFieldMatcher, boolean compress, XContentType xContentType) {
-            super(blobNameFormat, reader, parseFieldMatcher);
+        LegacyEmulationBlobStoreFormat(String blobNameFormat, CheckedFunction<XContentParser, T, IOException> reader,
+                                              NamedXContentRegistry namedXContentRegistry, boolean compress, XContentType xContentType) {
+            super(blobNameFormat, reader, namedXContentRegistry);
             this.xContentType = xContentType;
             this.compress = compress;
         }
@@ -163,12 +160,18 @@ private void write(T obj, StreamOutput streamOutput) throws IOException {
     public void testBlobStoreOperations() throws IOException {
         BlobStore blobStore = createTestBlobStore();
         BlobContainer blobContainer = blobStore.blobContainer(BlobPath.cleanPath());
-        ChecksumBlobStoreFormat<BlobObj> checksumJSON = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj.PROTO, parseFieldMatcher, false, XContentType.JSON);
-        ChecksumBlobStoreFormat<BlobObj> checksumSMILE = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj.PROTO, parseFieldMatcher, false, XContentType.SMILE);
-        ChecksumBlobStoreFormat<BlobObj> checksumSMILECompressed = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj.PROTO, parseFieldMatcher, true, XContentType.SMILE);
-        LegacyEmulationBlobStoreFormat<BlobObj> legacyJSON = new LegacyEmulationBlobStoreFormat<>("%s", BlobObj.PROTO, parseFieldMatcher, false, XContentType.JSON);
-        LegacyEmulationBlobStoreFormat<BlobObj> legacySMILE = new LegacyEmulationBlobStoreFormat<>("%s", BlobObj.PROTO, parseFieldMatcher, false, XContentType.SMILE);
-        LegacyEmulationBlobStoreFormat<BlobObj> legacySMILECompressed = new LegacyEmulationBlobStoreFormat<>("%s", BlobObj.PROTO, parseFieldMatcher, true, XContentType.SMILE);
+        ChecksumBlobStoreFormat<BlobObj> checksumJSON = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
+            xContentRegistry(), false, XContentType.JSON);
+        ChecksumBlobStoreFormat<BlobObj> checksumSMILE = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
+            xContentRegistry(), false, XContentType.SMILE);
+        ChecksumBlobStoreFormat<BlobObj> checksumSMILECompressed = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
+            xContentRegistry(), true, XContentType.SMILE);
+        LegacyEmulationBlobStoreFormat<BlobObj> legacyJSON = new LegacyEmulationBlobStoreFormat<>("%s", BlobObj::fromXContent,
+            xContentRegistry(), false, XContentType.JSON);
+        LegacyEmulationBlobStoreFormat<BlobObj> legacySMILE = new LegacyEmulationBlobStoreFormat<>("%s", BlobObj::fromXContent,
+            xContentRegistry(), false, XContentType.SMILE);
+        LegacyEmulationBlobStoreFormat<BlobObj> legacySMILECompressed = new LegacyEmulationBlobStoreFormat<>("%s", BlobObj::fromXContent,
+            xContentRegistry(), true, XContentType.SMILE);
 
         // Write blobs in different formats
         checksumJSON.write(new BlobObj("checksum json"), blobContainer, "check-json");
@@ -202,8 +205,10 @@ public void testCompressionIsApplied() throws IOException {
         for (int i = 0; i < randomIntBetween(100, 300); i++) {
             veryRedundantText.append("Blah ");
         }
-        ChecksumBlobStoreFormat<BlobObj> checksumFormat = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj.PROTO, parseFieldMatcher, false, randomBoolean() ? XContentType.SMILE : XContentType.JSON);
-        ChecksumBlobStoreFormat<BlobObj> checksumFormatComp = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj.PROTO, parseFieldMatcher, true, randomBoolean() ? XContentType.SMILE : XContentType.JSON);
+        ChecksumBlobStoreFormat<BlobObj> checksumFormat = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
+            xContentRegistry(), false, randomBoolean() ? XContentType.SMILE : XContentType.JSON);
+        ChecksumBlobStoreFormat<BlobObj> checksumFormatComp = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
+            xContentRegistry(), true, randomBoolean() ? XContentType.SMILE : XContentType.JSON);
         BlobObj blobObj = new BlobObj(veryRedundantText.toString());
         checksumFormatComp.write(blobObj, blobContainer, "blob-comp");
         checksumFormat.write(blobObj, blobContainer, "blob-not-comp");
@@ -215,9 +220,10 @@ public void testCompressionIsApplied() throws IOException {
     public void testBlobCorruption() throws IOException {
         BlobStore blobStore = createTestBlobStore();
         BlobContainer blobContainer = blobStore.blobContainer(BlobPath.cleanPath());
-        String testString = randomAsciiOfLength(randomInt(10000));
+        String testString = randomAlphaOfLength(randomInt(10000));
         BlobObj blobObj = new BlobObj(testString);
-        ChecksumBlobStoreFormat<BlobObj> checksumFormat = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj.PROTO, parseFieldMatcher, randomBoolean(), randomBoolean() ? XContentType.SMILE : XContentType.JSON);
+        ChecksumBlobStoreFormat<BlobObj> checksumFormat = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
+            xContentRegistry(), randomBoolean(), randomBoolean() ? XContentType.SMILE : XContentType.JSON);
         checksumFormat.write(blobObj, blobContainer, "test-path");
         assertEquals(checksumFormat.read(blobContainer, "test-path").getText(), testString);
         randomCorruption(blobContainer, "test-path");
@@ -234,7 +240,7 @@ public void testBlobCorruption() throws IOException {
     public void testAtomicWrite() throws Exception {
         final BlobStore blobStore = createTestBlobStore();
         final BlobContainer blobContainer = blobStore.blobContainer(BlobPath.cleanPath());
-        String testString = randomAsciiOfLength(randomInt(10000));
+        String testString = randomAlphaOfLength(randomInt(10000));
         final CountDownLatch block = new CountDownLatch(1);
         final CountDownLatch unblock = new CountDownLatch(1);
         final BlobObj blobObj = new BlobObj(testString) {
@@ -251,7 +257,8 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par
                 return builder;
             }
         };
-        final ChecksumBlobStoreFormat<BlobObj> checksumFormat = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj.PROTO, parseFieldMatcher, randomBoolean(), randomBoolean() ? XContentType.SMILE : XContentType.JSON);
+        final ChecksumBlobStoreFormat<BlobObj> checksumFormat = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
+            xContentRegistry(), randomBoolean(), randomBoolean() ? XContentType.SMILE : XContentType.JSON);
         ExecutorService threadPool = Executors.newFixedThreadPool(1);
         try {
             Future<Void> future = threadPool.submit(new Callable<Void>() {
diff --git a/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java b/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java
index f512f1da5387b..c76dd4d1d1116 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java
@@ -35,24 +35,35 @@
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.NamedDiff;
+import org.elasticsearch.cluster.SnapshotsInProgress;
+import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.metadata.MetaDataIndexStateService;
 import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Priority;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.discovery.zen.ZenDiscovery;
-import org.elasticsearch.discovery.zen.elect.ElectMasterService;
-import org.elasticsearch.index.store.IndexStore;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.discovery.zen.ElectMasterService;
+import org.elasticsearch.env.Environment;
 import org.elasticsearch.indices.recovery.RecoveryState;
 import org.elasticsearch.indices.ttl.IndicesTTLService;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.repositories.RepositoryMissingException;
 import org.elasticsearch.rest.AbstractRestChannel;
+import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.action.admin.cluster.RestClusterStateAction;
@@ -65,13 +76,13 @@
 import org.elasticsearch.test.TestCustomMetaData;
 import org.elasticsearch.test.rest.FakeRestRequest;
 
+import java.io.IOException;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.EnumSet;
 import java.util.List;
-import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
@@ -88,27 +99,65 @@
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
+import static org.mockito.Mockito.mock;
 
 /**
  */
 @ClusterScope(scope = Scope.TEST, numDataNodes = 0, transportClientRatio = 0)
 @ESIntegTestCase.SuppressLocalMode // TODO only restorePersistentSettingsTest needs this maybe factor out?
 public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCase {
+
+    public static class TestCustomMetaDataPlugin extends Plugin {
+
+        private final List<NamedWriteableRegistry.Entry> namedWritables = new ArrayList<>();
+        private final List<NamedXContentRegistry.Entry> namedXContents = new ArrayList<>();
+
+        public TestCustomMetaDataPlugin() {
+            registerBuiltinWritables();
+        }
+
+        private <T extends MetaData.Custom> void registerMetaDataCustom(String name, Writeable.Reader<T> reader,
+                                                                        Writeable.Reader<NamedDiff> diffReader,
+                                                                        CheckedFunction<XContentParser, T, IOException> parser) {
+            namedWritables.add(new NamedWriteableRegistry.Entry(MetaData.Custom.class, name, reader));
+            namedWritables.add(new NamedWriteableRegistry.Entry(NamedDiff.class, name, diffReader));
+            namedXContents.add(new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(name), parser));
+        }
+
+        private void registerBuiltinWritables() {
+            registerMetaDataCustom(SnapshottableMetadata.TYPE, SnapshottableMetadata::readFrom,
+                SnapshottableMetadata::readDiffFrom, SnapshottableMetadata::fromXContent);
+            registerMetaDataCustom(NonSnapshottableMetadata.TYPE, NonSnapshottableMetadata::readFrom,
+                NonSnapshottableMetadata::readDiffFrom, NonSnapshottableMetadata::fromXContent);
+            registerMetaDataCustom(SnapshottableGatewayMetadata.TYPE, SnapshottableGatewayMetadata::readFrom,
+                SnapshottableGatewayMetadata::readDiffFrom, SnapshottableGatewayMetadata::fromXContent);
+            registerMetaDataCustom(NonSnapshottableGatewayMetadata.TYPE, NonSnapshottableGatewayMetadata::readFrom,
+                NonSnapshottableGatewayMetadata::readDiffFrom, NonSnapshottableGatewayMetadata::fromXContent);
+            registerMetaDataCustom(SnapshotableGatewayNoApiMetadata.TYPE, SnapshotableGatewayNoApiMetadata::readFrom,
+                NonSnapshottableGatewayMetadata::readDiffFrom, SnapshotableGatewayNoApiMetadata::fromXContent);
+        }
+
+        @Override
+        public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
+            return namedWritables;
+        }
+
+        @Override
+        public List<NamedXContentRegistry.Entry> getNamedXContent() {
+            return namedXContents;
+        }
+    }
+
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Arrays.asList(MockRepository.Plugin.class);
+        return Arrays.asList(MockRepository.Plugin.class, TestCustomMetaDataPlugin.class);
     }
 
     public void testRestorePersistentSettings() throws Exception {
         logger.info("--> start 2 nodes");
-        Settings nodeSettings = Settings.builder()
-                .put("discovery.type", "zen")
-                .put(ZenDiscovery.PING_TIMEOUT_SETTING.getKey(), "200ms")
-                .put("discovery.initial_state_timeout", "500ms")
-                .build();
-        internalCluster().startNode(nodeSettings);
+        internalCluster().startNode();
         Client client = client();
-        String secondNode = internalCluster().startNode(nodeSettings);
+        String secondNode = internalCluster().startNode();
         logger.info("--> wait for the second node to join the cluster");
         assertThat(client.admin().cluster().prepareHealth().setWaitForNodes("2").get().isTimedOut(), equalTo(false));
 
@@ -170,19 +219,16 @@ public void testRestoreCustomMetadata() throws Exception {
         Client client = client();
         createIndex("test-idx");
         logger.info("--> add custom persistent metadata");
-        updateClusterState(new ClusterStateUpdater() {
-            @Override
-            public ClusterState execute(ClusterState currentState) throws Exception {
-                ClusterState.Builder builder = ClusterState.builder(currentState);
-                MetaData.Builder metadataBuilder = MetaData.builder(currentState.metaData());
-                metadataBuilder.putCustom(SnapshottableMetadata.TYPE, new SnapshottableMetadata("before_snapshot_s"));
-                metadataBuilder.putCustom(NonSnapshottableMetadata.TYPE, new NonSnapshottableMetadata("before_snapshot_ns"));
-                metadataBuilder.putCustom(SnapshottableGatewayMetadata.TYPE, new SnapshottableGatewayMetadata("before_snapshot_s_gw"));
-                metadataBuilder.putCustom(NonSnapshottableGatewayMetadata.TYPE, new NonSnapshottableGatewayMetadata("before_snapshot_ns_gw"));
-                metadataBuilder.putCustom(SnapshotableGatewayNoApiMetadata.TYPE, new SnapshotableGatewayNoApiMetadata("before_snapshot_s_gw_noapi"));
-                builder.metaData(metadataBuilder);
-                return builder.build();
-            }
+        updateClusterState(currentState -> {
+            ClusterState.Builder builder = ClusterState.builder(currentState);
+            MetaData.Builder metadataBuilder = MetaData.builder(currentState.metaData());
+            metadataBuilder.putCustom(SnapshottableMetadata.TYPE, new SnapshottableMetadata("before_snapshot_s"));
+            metadataBuilder.putCustom(NonSnapshottableMetadata.TYPE, new NonSnapshottableMetadata("before_snapshot_ns"));
+            metadataBuilder.putCustom(SnapshottableGatewayMetadata.TYPE, new SnapshottableGatewayMetadata("before_snapshot_s_gw"));
+            metadataBuilder.putCustom(NonSnapshottableGatewayMetadata.TYPE, new NonSnapshottableGatewayMetadata("before_snapshot_ns_gw"));
+            metadataBuilder.putCustom(SnapshotableGatewayNoApiMetadata.TYPE, new SnapshotableGatewayNoApiMetadata("before_snapshot_s_gw_noapi"));
+            builder.metaData(metadataBuilder);
+            return builder.build();
         });
 
         logger.info("--> create repository");
@@ -197,27 +243,24 @@ public ClusterState execute(ClusterState currentState) throws Exception {
         assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").execute().actionGet().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));
 
         logger.info("--> change custom persistent metadata");
-        updateClusterState(new ClusterStateUpdater() {
-            @Override
-            public ClusterState execute(ClusterState currentState) throws Exception {
-                ClusterState.Builder builder = ClusterState.builder(currentState);
-                MetaData.Builder metadataBuilder = MetaData.builder(currentState.metaData());
-                if (randomBoolean()) {
-                    metadataBuilder.putCustom(SnapshottableMetadata.TYPE, new SnapshottableMetadata("after_snapshot_s"));
-                } else {
-                    metadataBuilder.removeCustom(SnapshottableMetadata.TYPE);
-                }
-                metadataBuilder.putCustom(NonSnapshottableMetadata.TYPE, new NonSnapshottableMetadata("after_snapshot_ns"));
-                if (randomBoolean()) {
-                    metadataBuilder.putCustom(SnapshottableGatewayMetadata.TYPE, new SnapshottableGatewayMetadata("after_snapshot_s_gw"));
-                } else {
-                    metadataBuilder.removeCustom(SnapshottableGatewayMetadata.TYPE);
-                }
-                metadataBuilder.putCustom(NonSnapshottableGatewayMetadata.TYPE, new NonSnapshottableGatewayMetadata("after_snapshot_ns_gw"));
-                metadataBuilder.removeCustom(SnapshotableGatewayNoApiMetadata.TYPE);
-                builder.metaData(metadataBuilder);
-                return builder.build();
+        updateClusterState(currentState -> {
+            ClusterState.Builder builder = ClusterState.builder(currentState);
+            MetaData.Builder metadataBuilder = MetaData.builder(currentState.metaData());
+            if (randomBoolean()) {
+                metadataBuilder.putCustom(SnapshottableMetadata.TYPE, new SnapshottableMetadata("after_snapshot_s"));
+            } else {
+                metadataBuilder.removeCustom(SnapshottableMetadata.TYPE);
             }
+            metadataBuilder.putCustom(NonSnapshottableMetadata.TYPE, new NonSnapshottableMetadata("after_snapshot_ns"));
+            if (randomBoolean()) {
+                metadataBuilder.putCustom(SnapshottableGatewayMetadata.TYPE, new SnapshottableGatewayMetadata("after_snapshot_s_gw"));
+            } else {
+                metadataBuilder.removeCustom(SnapshottableGatewayMetadata.TYPE);
+            }
+            metadataBuilder.putCustom(NonSnapshottableGatewayMetadata.TYPE, new NonSnapshottableGatewayMetadata("after_snapshot_ns_gw"));
+            metadataBuilder.removeCustom(SnapshotableGatewayNoApiMetadata.TYPE);
+            builder.metaData(metadataBuilder);
+            return builder.build();
         });
 
         logger.info("--> delete repository");
@@ -309,7 +352,7 @@ public void testSnapshotDuringNodeShutdown() throws Exception {
                 .setType("mock").setSettings(
                         Settings.builder()
                                 .put("location", randomRepoPath())
-                                .put("random", randomAsciiOfLength(10))
+                                .put("random", randomAlphaOfLength(10))
                                 .put("wait_after_unblock", 200)
                 ).get();
         assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
@@ -357,7 +400,7 @@ public void testSnapshotWithStuckNode() throws Exception {
                 .setType("mock").setSettings(
                         Settings.builder()
                                 .put("location", repo)
-                                .put("random", randomAsciiOfLength(10))
+                                .put("random", randomAlphaOfLength(10))
                                 .put("wait_after_unblock", 200)
                 ).get();
         assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
@@ -393,11 +436,13 @@ public void testSnapshotWithStuckNode() throws Exception {
 
         logger.info("--> making sure that snapshot no longer exists");
         assertThrows(client().admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").execute(), SnapshotMissingException.class);
-        // Subtract three files that will remain in the repository:
+        // Subtract four files that will remain in the repository:
         //   (1) index-1
         //   (2) index-0 (because we keep the previous version) and
         //   (3) index-latest
-        assertThat("not all files were deleted during snapshot cancellation", numberOfFilesBeforeSnapshot, equalTo(numberOfFiles(repo) - 3));
+        //   (4) incompatible-snapshots
+        assertThat("not all files were deleted during snapshot cancellation",
+            numberOfFilesBeforeSnapshot, equalTo(numberOfFiles(repo) - 4));
         logger.info("--> done");
     }
 
@@ -470,15 +515,12 @@ public void testRestoreIndexWithMissingShards() throws Exception {
             client().admin().cluster().prepareCreateSnapshot("test-repo", "test-snap-2")
                     .setIndices("test-idx-all", "test-idx-none", "test-idx-some")
                     .setWaitForCompletion(false).setPartial(true).execute().actionGet();
-            assertBusy(new Runnable() {
-                @Override
-                public void run() {
-                    SnapshotsStatusResponse snapshotsStatusResponse = client().admin().cluster().prepareSnapshotStatus("test-repo").setSnapshots("test-snap-2").get();
-                    List<SnapshotStatus> snapshotStatuses = snapshotsStatusResponse.getSnapshots();
-                    assertEquals(snapshotStatuses.size(), 1);
-                    logger.trace("current snapshot status [{}]", snapshotStatuses.get(0));
-                    assertTrue(snapshotStatuses.get(0).getState().completed());
-                }
+            assertBusy(() -> {
+                SnapshotsStatusResponse snapshotsStatusResponse = client().admin().cluster().prepareSnapshotStatus("test-repo").setSnapshots("test-snap-2").get();
+                List<SnapshotStatus> snapshotStatuses = snapshotsStatusResponse.getSnapshots();
+                assertEquals(snapshotStatuses.size(), 1);
+                logger.trace("current snapshot status [{}]", snapshotStatuses.get(0));
+                assertTrue(snapshotStatuses.get(0).getState().completed());
             }, 1, TimeUnit.MINUTES);
             SnapshotsStatusResponse snapshotsStatusResponse = client().admin().cluster().prepareSnapshotStatus("test-repo").setSnapshots("test-snap-2").get();
             List<SnapshotStatus> snapshotStatuses = snapshotsStatusResponse.getSnapshots();
@@ -491,15 +533,12 @@ public void run() {
 
             // There is slight delay between snapshot being marked as completed in the cluster state and on the file system
             // After it was marked as completed in the cluster state - we need to check if it's completed on the file system as well
-            assertBusy(new Runnable() {
-                @Override
-                public void run() {
-                    GetSnapshotsResponse response = client().admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap-2").get();
-                    assertThat(response.getSnapshots().size(), equalTo(1));
-                    SnapshotInfo snapshotInfo = response.getSnapshots().get(0);
-                    assertTrue(snapshotInfo.state().completed());
-                    assertEquals(SnapshotState.PARTIAL, snapshotInfo.state());
-                }
+            assertBusy(() -> {
+                GetSnapshotsResponse response = client().admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap-2").get();
+                assertThat(response.getSnapshots().size(), equalTo(1));
+                SnapshotInfo snapshotInfo = response.getSnapshots().get(0);
+                assertTrue(snapshotInfo.state().completed());
+                assertEquals(SnapshotState.PARTIAL, snapshotInfo.state());
             }, 1, TimeUnit.MINUTES);
         } else {
             logger.info("checking snapshot completion using wait_for_completion flag");
@@ -631,7 +670,7 @@ public void testRegistrationFailure() {
     public void testThatSensitiveRepositorySettingsAreNotExposed() throws Exception {
         Settings nodeSettings = Settings.builder().put().build();
         logger.info("--> start two nodes");
-        internalCluster().startNodesAsync(2, nodeSettings).get();
+        internalCluster().startNodes(2, nodeSettings);
         // Register mock repositories
         client().admin().cluster().preparePutRepository("test-repo")
                 .setType("mock").setSettings(Settings.builder()
@@ -641,7 +680,8 @@ public void testThatSensitiveRepositorySettingsAreNotExposed() throws Exception
         ).get();
 
         NodeClient nodeClient = internalCluster().getInstance(NodeClient.class);
-        RestGetRepositoriesAction getRepoAction = internalCluster().getInstance(RestGetRepositoriesAction.class);
+        RestGetRepositoriesAction getRepoAction = new RestGetRepositoriesAction(nodeSettings, mock(RestController.class),
+                internalCluster().getInstance(SettingsFilter.class));
         RestRequest getRepoRequest = new FakeRestRequest();
         getRepoRequest.params().put("repository", "test-repo");
         final CountDownLatch getRepoLatch = new CountDownLatch(1);
@@ -663,7 +703,8 @@ public void sendResponse(RestResponse response) {
             throw getRepoError.get();
         }
 
-        RestClusterStateAction clusterStateAction = internalCluster().getInstance(RestClusterStateAction.class);
+        RestClusterStateAction clusterStateAction = new RestClusterStateAction(nodeSettings, mock(RestController.class),
+                internalCluster().getInstance(SettingsFilter.class));
         RestRequest clusterStateRequest = new FakeRestRequest();
         final CountDownLatch clusterStateLatch = new CountDownLatch(1);
         final AtomicReference<AssertionError> clusterStateError = new AtomicReference<>();
@@ -683,109 +724,6 @@ public void sendResponse(RestResponse response) {
         if (clusterStateError.get() != null) {
             throw clusterStateError.get();
         }
-
-    }
-
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/12621")
-    public void testChaosSnapshot() throws Exception {
-        final List<String> indices = new CopyOnWriteArrayList<>();
-        int initialNodes = between(1, 3);
-        logger.info("--> start {} nodes", initialNodes);
-        for (int i = 0; i < initialNodes; i++) {
-            internalCluster().startNode();
-        }
-
-        logger.info("-->  creating repository");
-        assertAcked(client().admin().cluster().preparePutRepository("test-repo")
-                .setType("fs").setSettings(Settings.builder()
-                        .put("location", randomRepoPath())
-                        .put("compress", randomBoolean())
-                        .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)));
-
-        int initialIndices = between(1, 3);
-        logger.info("--> create {} indices", initialIndices);
-        for (int i = 0; i < initialIndices; i++) {
-            createTestIndex("test-" + i);
-            indices.add("test-" + i);
-        }
-
-        int asyncNodes = between(0, 5);
-        logger.info("--> start {} additional nodes asynchronously", asyncNodes);
-        InternalTestCluster.Async<List<String>> asyncNodesFuture = internalCluster().startNodesAsync(asyncNodes);
-
-        int asyncIndices = between(0, 10);
-        logger.info("--> create {} additional indices asynchronously", asyncIndices);
-        Thread[] asyncIndexThreads = new Thread[asyncIndices];
-        for (int i = 0; i < asyncIndices; i++) {
-            final int cur = i;
-            asyncIndexThreads[i] = new Thread(new Runnable() {
-                @Override
-                public void run() {
-                    createTestIndex("test-async-" + cur);
-                    indices.add("test-async-" + cur);
-
-                }
-            });
-            asyncIndexThreads[i].start();
-        }
-
-        logger.info("--> snapshot");
-
-        ListenableActionFuture<CreateSnapshotResponse> snapshotResponseFuture = client().admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-*").setPartial(true).execute();
-
-        long start = System.currentTimeMillis();
-        // Produce chaos for 30 sec or until snapshot is done whatever comes first
-        int randomIndices = 0;
-        while (System.currentTimeMillis() - start < 30000 && !snapshotIsDone("test-repo", "test-snap")) {
-            Thread.sleep(100);
-            int chaosType = randomInt(10);
-            if (chaosType < 4) {
-                // Randomly delete an index
-                if (indices.size() > 0) {
-                    String index = indices.remove(randomInt(indices.size() - 1));
-                    logger.info("--> deleting random index [{}]", index);
-                    internalCluster().wipeIndices(index);
-                }
-            } else if (chaosType < 6) {
-                // Randomly shutdown a node
-                if (cluster().size() > 1) {
-                    logger.info("--> shutting down random node");
-                    internalCluster().stopRandomDataNode();
-                }
-            } else if (chaosType < 8) {
-                // Randomly create an index
-                String index = "test-rand-" + randomIndices;
-                logger.info("--> creating random index [{}]", index);
-                createTestIndex(index);
-                randomIndices++;
-            } else {
-                // Take a break
-                logger.info("--> noop");
-            }
-        }
-
-        logger.info("--> waiting for async indices creation to finish");
-        for (int i = 0; i < asyncIndices; i++) {
-            asyncIndexThreads[i].join();
-        }
-
-        logger.info("--> update index settings to back to normal");
-        assertAcked(client().admin().indices().prepareUpdateSettings("test-*").setSettings(Settings.builder()
-                        .put(IndexStore.INDEX_STORE_THROTTLE_TYPE_SETTING.getKey(), "node")
-        ));
-
-        // Make sure that snapshot finished - doesn't matter if it failed or succeeded
-        try {
-            CreateSnapshotResponse snapshotResponse = snapshotResponseFuture.get();
-            SnapshotInfo snapshotInfo = snapshotResponse.getSnapshotInfo();
-            assertNotNull(snapshotInfo);
-            logger.info("--> snapshot is done with state [{}], total shards [{}], successful shards [{}]", snapshotInfo.state(), snapshotInfo.totalShards(), snapshotInfo.successfulShards());
-        } catch (Exception ex) {
-            logger.info("--> snapshot didn't start properly", ex);
-        }
-
-        asyncNodesFuture.get();
-        logger.info("--> done");
     }
 
     public void testMasterShutdownDuringSnapshot() throws Exception {
@@ -826,7 +764,7 @@ public void testMasterShutdownDuringSnapshot() throws Exception {
         final ClusterService clusterService = internalCluster().clusterService(internalCluster().getMasterName());
         BlockingClusterStateListener snapshotListener = new BlockingClusterStateListener(clusterService, "update_snapshot [", "update snapshot state", Priority.HIGH);
         try {
-            clusterService.addFirst(snapshotListener);
+            clusterService.addListener(snapshotListener);
             logger.info("--> snapshot");
             dataNodeClient().admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(false).setIndices("test-idx").get();
 
@@ -840,18 +778,15 @@ public void testMasterShutdownDuringSnapshot() throws Exception {
             snapshotListener.unblock();
 
         } finally {
-            clusterService.remove(snapshotListener);
+            clusterService.removeListener(snapshotListener);
         }
 
         logger.info("--> wait until the snapshot is done");
 
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                GetSnapshotsResponse snapshotsStatusResponse = client().admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get();
-                SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots().get(0);
-                assertTrue(snapshotInfo.state().completed());
-            }
+        assertBusy(() -> {
+            GetSnapshotsResponse snapshotsStatusResponse = client().admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get();
+            SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots().get(0);
+            assertTrue(snapshotInfo.state().completed());
         }, 1, TimeUnit.MINUTES);
 
         logger.info("--> verify that snapshot was succesful");
@@ -864,65 +799,223 @@ public void run() {
     }
 
 
-    private boolean snapshotIsDone(String repository, String snapshot) {
-        try {
-            SnapshotsStatusResponse snapshotsStatusResponse = client().admin().cluster().prepareSnapshotStatus(repository).setSnapshots(snapshot).get();
-            if (snapshotsStatusResponse.getSnapshots().isEmpty()) {
-                return false;
-            }
-            for (SnapshotStatus snapshotStatus : snapshotsStatusResponse.getSnapshots()) {
-                if (snapshotStatus.getState().completed()) {
-                    return true;
-                }
-            }
-            return false;
-        } catch (SnapshotMissingException ex) {
-            return false;
+    public void testMasterAndDataShutdownDuringSnapshot() throws Exception {
+        logger.info("-->  starting three master nodes and two data nodes");
+        internalCluster().startMasterOnlyNodes(3);
+        internalCluster().startDataOnlyNodes(2);
+
+        final Client client = client();
+
+        logger.info("-->  creating repository");
+        assertAcked(client.admin().cluster().preparePutRepository("test-repo")
+            .setType("mock").setSettings(Settings.builder()
+                .put("location", randomRepoPath())
+                .put("compress", randomBoolean())
+                .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)));
+
+        assertAcked(prepareCreate("test-idx", 0, Settings.builder().put("number_of_shards", between(1, 20))
+            .put("number_of_replicas", 0)));
+        ensureGreen();
+
+        logger.info("--> indexing some data");
+        final int numdocs = randomIntBetween(10, 100);
+        IndexRequestBuilder[] builders = new IndexRequestBuilder[numdocs];
+        for (int i = 0; i < builders.length; i++) {
+            builders[i] = client().prepareIndex("test-idx", "type1", Integer.toString(i)).setSource("field1", "bar " + i);
+        }
+        indexRandom(true, builders);
+        flushAndRefresh();
+
+        final int numberOfShards = getNumShards("test-idx").numPrimaries;
+        logger.info("number of shards: {}", numberOfShards);
+
+        final String masterNode = blockMasterFromFinalizingSnapshotOnSnapFile("test-repo");
+        final String dataNode = blockNodeWithIndex("test-repo", "test-idx");
+
+        dataNodeClient().admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(false).setIndices("test-idx").get();
+
+        logger.info("--> stopping data node {}", dataNode);
+        stopNode(dataNode);
+        logger.info("--> stopping master node {} ", masterNode);
+        internalCluster().stopCurrentMasterNode();
+
+        logger.info("--> wait until the snapshot is done");
+
+        assertBusy(() -> {
+            GetSnapshotsResponse snapshotsStatusResponse = client().admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get();
+            SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots().get(0);
+            assertTrue(snapshotInfo.state().completed());
+        }, 1, TimeUnit.MINUTES);
+
+        logger.info("--> verify that snapshot was partial");
+
+        GetSnapshotsResponse snapshotsStatusResponse = client().admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get();
+        SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots().get(0);
+        assertEquals(SnapshotState.PARTIAL, snapshotInfo.state());
+        assertNotEquals(snapshotInfo.totalShards(), snapshotInfo.successfulShards());
+        assertThat(snapshotInfo.failedShards(), greaterThan(0));
+        for (SnapshotShardFailure failure : snapshotInfo.shardFailures()) {
+            assertNotNull(failure.reason());
         }
     }
 
-    private void createTestIndex(String name) {
-        assertAcked(prepareCreate(name, 0, Settings.builder().put("number_of_shards", between(1, 6))
-                .put("number_of_replicas", between(1, 6))));
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/25281")
+    public void testMasterShutdownDuringFailedSnapshot() throws Exception {
+        logger.info("-->  starting two master nodes and two data nodes");
+        internalCluster().startMasterOnlyNodes(2);
+        internalCluster().startDataOnlyNodes(2);
+
+        logger.info("-->  creating repository");
+        assertAcked(client().admin().cluster().preparePutRepository("test-repo")
+            .setType("mock").setSettings(Settings.builder()
+                .put("location", randomRepoPath())
+                .put("compress", randomBoolean())
+                .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)));
+
+        assertAcked(prepareCreate("test-idx", 0, Settings.builder()
+            .put("number_of_shards", 6).put("number_of_replicas", 0)));
+        ensureGreen();
 
-        logger.info("--> indexing some data into {}", name);
-        for (int i = 0; i < between(10, 500); i++) {
-            index(name, "doc", Integer.toString(i), "foo", "bar" + i);
+        logger.info("-->  indexing some data");
+        final int numdocs = randomIntBetween(50, 100);
+        IndexRequestBuilder[] builders = new IndexRequestBuilder[numdocs];
+        for (int i = 0; i < builders.length; i++) {
+            builders[i] = client().prepareIndex("test-idx", "type1",
+                Integer.toString(i)).setSource("field1", "bar " + i);
         }
+        indexRandom(true, builders);
+        flushAndRefresh();
 
-        assertAcked(client().admin().indices().prepareUpdateSettings(name).setSettings(Settings.builder()
-                        .put(IndexStore.INDEX_STORE_THROTTLE_TYPE_SETTING.getKey(), "all")
-                        .put(IndexStore.INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC_SETTING.getKey(), between(100, 50000))
-        ));
+        logger.info("-->  stopping random data node, which should cause shards to go missing");
+        internalCluster().stopRandomDataNode();
+        assertBusy(() ->
+            assertEquals(ClusterHealthStatus.RED, client().admin().cluster().prepareHealth().get().getStatus()),
+            30, TimeUnit.SECONDS);
+
+        final String masterNode = blockMasterFromFinalizingSnapshotOnIndexFile("test-repo");
+
+        logger.info("-->  snapshot");
+        client().admin().cluster().prepareCreateSnapshot("test-repo", "test-snap")
+            .setWaitForCompletion(false).setIndices("test-idx").get();
+
+        logger.info("--> waiting for block to kick in on " + masterNode);
+        waitForBlock(masterNode, "test-repo", TimeValue.timeValueSeconds(60));
+
+        logger.info("-->  stopping master node");
+        internalCluster().stopCurrentMasterNode();
+
+        logger.info("-->  wait until the snapshot is done");
+        assertBusy(() -> {
+            GetSnapshotsResponse snapshotsStatusResponse = client().admin().cluster()
+                .prepareGetSnapshots("test-repo").setSnapshots("test-snap").setIgnoreUnavailable(true).get();
+            assertEquals(1, snapshotsStatusResponse.getSnapshots().size());
+            SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots().get(0);
+            assertTrue(snapshotInfo.state().completed());
+            ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
+            SnapshotsInProgress snapshotsInProgress = clusterState.custom(SnapshotsInProgress.TYPE);
+            assertEquals(0, snapshotsInProgress.entries().size());
+        }, 30, TimeUnit.SECONDS);
+
+        logger.info("-->  verify that snapshot failed");
+        GetSnapshotsResponse snapshotsStatusResponse = client().admin().cluster()
+            .prepareGetSnapshots("test-repo").setSnapshots("test-snap").get();
+        SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots().get(0);
+        assertEquals(SnapshotState.FAILED, snapshotInfo.state());
     }
 
-    static {
-        MetaData.registerPrototype(SnapshottableMetadata.TYPE, SnapshottableMetadata.PROTO);
-        MetaData.registerPrototype(NonSnapshottableMetadata.TYPE, NonSnapshottableMetadata.PROTO);
-        MetaData.registerPrototype(SnapshottableGatewayMetadata.TYPE, SnapshottableGatewayMetadata.PROTO);
-        MetaData.registerPrototype(NonSnapshottableGatewayMetadata.TYPE, NonSnapshottableGatewayMetadata.PROTO);
-        MetaData.registerPrototype(SnapshotableGatewayNoApiMetadata.TYPE, SnapshotableGatewayNoApiMetadata.PROTO);
+    /**
+     * Tests that a shrunken index (created via the shrink APIs) and subsequently snapshotted
+     * can be restored when the node the shrunken index was created on is no longer part of
+     * the cluster.
+     */
+    public void testRestoreShrinkIndex() throws Exception {
+        logger.info("-->  starting a master node and a data node");
+        internalCluster().startMasterOnlyNode();
+        internalCluster().startDataOnlyNode();
+
+        final Client client = client();
+        final String repo = "test-repo";
+        final String snapshot = "test-snap";
+        final String sourceIdx = "test-idx";
+        final String shrunkIdx = "test-idx-shrunk";
+
+        logger.info("-->  creating repository");
+        assertAcked(client.admin().cluster().preparePutRepository(repo).setType("fs")
+            .setSettings(Settings.builder().put("location", randomRepoPath())
+                             .put("compress", randomBoolean())));
+
+        assertAcked(prepareCreate(sourceIdx, 0, Settings.builder()
+            .put("number_of_shards", between(2, 10)).put("number_of_replicas", 0)));
+        ensureGreen();
+
+        logger.info("--> indexing some data");
+        IndexRequestBuilder[] builders = new IndexRequestBuilder[randomIntBetween(10, 100)];
+        for (int i = 0; i < builders.length; i++) {
+            builders[i] = client().prepareIndex(sourceIdx, "type1",
+                Integer.toString(i)).setSource("field1", "bar " + i);
+        }
+        indexRandom(true, builders);
+        flushAndRefresh();
+
+        logger.info("--> shrink the index");
+        assertAcked(client.admin().indices().prepareUpdateSettings(sourceIdx)
+            .setSettings(Settings.builder().put("index.blocks.write", true)).get());
+        assertAcked(client.admin().indices().prepareShrinkIndex(sourceIdx, shrunkIdx).get());
+
+        logger.info("--> snapshot the shrunk index");
+        CreateSnapshotResponse createResponse = client.admin().cluster()
+            .prepareCreateSnapshot(repo, snapshot)
+            .setWaitForCompletion(true).setIndices(shrunkIdx).get();
+        assertEquals(SnapshotState.SUCCESS, createResponse.getSnapshotInfo().state());
+
+        logger.info("--> delete index and stop the data node");
+        assertAcked(client.admin().indices().prepareDelete(sourceIdx).get());
+        assertAcked(client.admin().indices().prepareDelete(shrunkIdx).get());
+        internalCluster().stopRandomDataNode();
+        client().admin().cluster().prepareHealth().setTimeout("30s").setWaitForNodes("1");
+
+        logger.info("--> start a new data node");
+        final Settings dataSettings = Settings.builder()
+            .put(Node.NODE_NAME_SETTING.getKey(), randomAlphaOfLength(5))
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()) // to get a new node id
+            .build();
+        internalCluster().startDataOnlyNode(dataSettings);
+        client().admin().cluster().prepareHealth().setTimeout("30s").setWaitForNodes("2");
+
+        logger.info("--> restore the shrunk index and ensure all shards are allocated");
+        RestoreSnapshotResponse restoreResponse = client().admin().cluster()
+            .prepareRestoreSnapshot(repo, snapshot).setWaitForCompletion(true)
+            .setIndices(shrunkIdx).get();
+        assertEquals(restoreResponse.getRestoreInfo().totalShards(),
+            restoreResponse.getRestoreInfo().successfulShards());
+        ensureYellow();
     }
 
     public static class SnapshottableMetadata extends TestCustomMetaData {
         public static final String TYPE = "test_snapshottable";
 
-        public static final SnapshottableMetadata PROTO = new SnapshottableMetadata("");
-
         public SnapshottableMetadata(String data) {
             super(data);
         }
 
         @Override
-        public String type() {
+        public String getWriteableName() {
             return TYPE;
         }
 
-        @Override
-        protected TestCustomMetaData newTestCustomMetaData(String data) {
-            return new SnapshottableMetadata(data);
+        public static SnapshottableMetadata readFrom(StreamInput in) throws IOException {
+            return readFrom(SnapshottableMetadata::new, in);
+        }
+
+        public static NamedDiff<MetaData.Custom> readDiffFrom(StreamInput in) throws IOException {
+            return readDiffFrom(TYPE, in);
         }
 
+        public static SnapshottableMetadata fromXContent(XContentParser parser) throws IOException {
+            return fromXContent(SnapshottableMetadata::new, parser);
+        }
+
+
         @Override
         public EnumSet<MetaData.XContentContext> context() {
             return MetaData.API_AND_SNAPSHOT;
@@ -932,20 +1025,25 @@ public EnumSet<MetaData.XContentContext> context() {
     public static class NonSnapshottableMetadata extends TestCustomMetaData {
         public static final String TYPE = "test_non_snapshottable";
 
-        public static final NonSnapshottableMetadata PROTO = new NonSnapshottableMetadata("");
-
         public NonSnapshottableMetadata(String data) {
             super(data);
         }
 
         @Override
-        public String type() {
+        public String getWriteableName() {
             return TYPE;
         }
 
-        @Override
-        protected NonSnapshottableMetadata newTestCustomMetaData(String data) {
-            return new NonSnapshottableMetadata(data);
+        public static NonSnapshottableMetadata readFrom(StreamInput in) throws IOException {
+            return readFrom(NonSnapshottableMetadata::new, in);
+        }
+
+        public static NamedDiff<MetaData.Custom> readDiffFrom(StreamInput in) throws IOException {
+            return readDiffFrom(TYPE, in);
+        }
+
+        public static NonSnapshottableMetadata fromXContent(XContentParser parser) throws IOException {
+            return fromXContent(NonSnapshottableMetadata::new, parser);
         }
 
         @Override
@@ -957,20 +1055,25 @@ public EnumSet<MetaData.XContentContext> context() {
     public static class SnapshottableGatewayMetadata extends TestCustomMetaData {
         public static final String TYPE = "test_snapshottable_gateway";
 
-        public static final SnapshottableGatewayMetadata PROTO = new SnapshottableGatewayMetadata("");
-
         public SnapshottableGatewayMetadata(String data) {
             super(data);
         }
 
         @Override
-        public String type() {
+        public String getWriteableName() {
             return TYPE;
         }
 
-        @Override
-        protected TestCustomMetaData newTestCustomMetaData(String data) {
-            return new SnapshottableGatewayMetadata(data);
+        public static SnapshottableGatewayMetadata readFrom(StreamInput in) throws IOException {
+            return readFrom(SnapshottableGatewayMetadata::new, in);
+        }
+
+        public static NamedDiff<MetaData.Custom> readDiffFrom(StreamInput in) throws IOException {
+            return readDiffFrom(TYPE, in);
+        }
+
+        public static SnapshottableGatewayMetadata fromXContent(XContentParser parser) throws IOException {
+            return fromXContent(SnapshottableGatewayMetadata::new, parser);
         }
 
         @Override
@@ -982,20 +1085,25 @@ public EnumSet<MetaData.XContentContext> context() {
     public static class NonSnapshottableGatewayMetadata extends TestCustomMetaData {
         public static final String TYPE = "test_non_snapshottable_gateway";
 
-        public static final NonSnapshottableGatewayMetadata PROTO = new NonSnapshottableGatewayMetadata("");
-
         public NonSnapshottableGatewayMetadata(String data) {
             super(data);
         }
 
         @Override
-        public String type() {
+        public String getWriteableName() {
             return TYPE;
         }
 
-        @Override
-        protected NonSnapshottableGatewayMetadata newTestCustomMetaData(String data) {
-            return new NonSnapshottableGatewayMetadata(data);
+        public static NonSnapshottableGatewayMetadata readFrom(StreamInput in) throws IOException {
+            return readFrom(NonSnapshottableGatewayMetadata::new, in);
+        }
+
+        public static NamedDiff<MetaData.Custom> readDiffFrom(StreamInput in) throws IOException {
+            return readDiffFrom(TYPE, in);
+        }
+
+        public static NonSnapshottableGatewayMetadata fromXContent(XContentParser parser) throws IOException {
+            return fromXContent(NonSnapshottableGatewayMetadata::new, parser);
         }
 
         @Override
@@ -1008,20 +1116,21 @@ public EnumSet<MetaData.XContentContext> context() {
     public static class SnapshotableGatewayNoApiMetadata extends TestCustomMetaData {
         public static final String TYPE = "test_snapshottable_gateway_no_api";
 
-        public static final SnapshotableGatewayNoApiMetadata PROTO = new SnapshotableGatewayNoApiMetadata("");
-
         public SnapshotableGatewayNoApiMetadata(String data) {
             super(data);
         }
 
         @Override
-        public String type() {
+        public String getWriteableName() {
             return TYPE;
         }
 
-        @Override
-        protected SnapshotableGatewayNoApiMetadata newTestCustomMetaData(String data) {
-            return new SnapshotableGatewayNoApiMetadata(data);
+        public static SnapshotableGatewayNoApiMetadata readFrom(StreamInput in) throws IOException {
+            return readFrom(SnapshotableGatewayNoApiMetadata::new, in);
+        }
+
+        public static SnapshotableGatewayNoApiMetadata fromXContent(XContentParser parser) throws IOException {
+            return fromXContent(SnapshotableGatewayNoApiMetadata::new, parser);
         }
 
         @Override
diff --git a/core/src/test/java/org/elasticsearch/snapshots/MinThreadsSnapshotRestoreIT.java b/core/src/test/java/org/elasticsearch/snapshots/MinThreadsSnapshotRestoreIT.java
new file mode 100644
index 0000000000000..680bfea6b3b3c
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/snapshots/MinThreadsSnapshotRestoreIT.java
@@ -0,0 +1,211 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.snapshots;
+
+import org.elasticsearch.action.ListenableActionFuture;
+import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotResponse;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.repositories.RepositoriesService;
+import org.elasticsearch.snapshots.mockstore.MockRepository;
+
+import java.util.Collection;
+import java.util.Collections;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.containsString;
+
+/**
+ * Tests for snapshot/restore that require at least 2 threads available
+ * in the thread pool (for example, tests that use the mock repository that
+ * block on master).
+ */
+public class MinThreadsSnapshotRestoreIT extends AbstractSnapshotIntegTestCase {
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        return Settings.builder().put(super.nodeSettings(nodeOrdinal))
+                   .put("thread_pool.snapshot.core", 2)
+                   .put("thread_pool.snapshot.max", 2)
+                   .build();
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Collections.singleton(MockRepository.Plugin.class);
+    }
+
+    public void testConcurrentSnapshotDeletionsNotAllowed() throws Exception {
+        logger.info("--> creating repository");
+        final String repo = "test-repo";
+        assertAcked(client().admin().cluster().preparePutRepository(repo).setType("mock").setSettings(
+            Settings.builder()
+                .put("location", randomRepoPath())
+                .put("random", randomAlphaOfLength(10))
+                .put("wait_after_unblock", 200)).get());
+
+        logger.info("--> snapshot twice");
+        final String index = "test-idx1";
+        assertAcked(prepareCreate(index, 1, Settings.builder().put("number_of_shards", 1).put("number_of_replicas", 0)));
+        for (int i = 0; i < 10; i++) {
+            index(index, "doc", Integer.toString(i), "foo", "bar" + i);
+        }
+        refresh();
+        final String snapshot1 = "test-snap1";
+        client().admin().cluster().prepareCreateSnapshot(repo, snapshot1).setWaitForCompletion(true).get();
+        final String index2 = "test-idx2";
+        assertAcked(prepareCreate(index2, 1, Settings.builder().put("number_of_shards", 1).put("number_of_replicas", 0)));
+        for (int i = 0; i < 10; i++) {
+            index(index2, "doc", Integer.toString(i), "foo", "bar" + i);
+        }
+        refresh();
+        final String snapshot2 = "test-snap2";
+        client().admin().cluster().prepareCreateSnapshot(repo, snapshot2).setWaitForCompletion(true).get();
+
+        String blockedNode = internalCluster().getMasterName();
+        ((MockRepository)internalCluster().getInstance(RepositoriesService.class, blockedNode).repository(repo)).blockOnDataFiles(true);
+        logger.info("--> start deletion of first snapshot");
+        ListenableActionFuture<DeleteSnapshotResponse> future =
+            client().admin().cluster().prepareDeleteSnapshot(repo, snapshot2).execute();
+        logger.info("--> waiting for block to kick in on node [{}]", blockedNode);
+        waitForBlock(blockedNode, repo, TimeValue.timeValueSeconds(10));
+
+        logger.info("--> try deleting the second snapshot, should fail because the first deletion is in progress");
+        try {
+            client().admin().cluster().prepareDeleteSnapshot(repo, snapshot1).get();
+            fail("should not be able to delete snapshots concurrently");
+        } catch (ConcurrentSnapshotExecutionException e) {
+            assertThat(e.getMessage(), containsString("cannot delete - another snapshot is currently being deleted"));
+        }
+
+        logger.info("--> unblocking blocked node [{}]", blockedNode);
+        unblockNode(repo, blockedNode);
+
+        logger.info("--> wait until first snapshot is finished");
+        assertAcked(future.actionGet());
+
+        logger.info("--> delete second snapshot, which should now work");
+        client().admin().cluster().prepareDeleteSnapshot(repo, snapshot1).get();
+        assertTrue(client().admin().cluster().prepareGetSnapshots(repo).setSnapshots("_all").get().getSnapshots().isEmpty());
+    }
+
+    public void testSnapshottingWithInProgressDeletionNotAllowed() throws Exception {
+        logger.info("--> creating repository");
+        final String repo = "test-repo";
+        assertAcked(client().admin().cluster().preparePutRepository(repo).setType("mock").setSettings(
+            Settings.builder()
+                .put("location", randomRepoPath())
+                .put("random", randomAlphaOfLength(10))
+                .put("wait_after_unblock", 200)).get());
+
+        logger.info("--> snapshot");
+        final String index = "test-idx";
+        assertAcked(prepareCreate(index, 1, Settings.builder().put("number_of_shards", 1).put("number_of_replicas", 0)));
+        for (int i = 0; i < 10; i++) {
+            index(index, "doc", Integer.toString(i), "foo", "bar" + i);
+        }
+        refresh();
+        final String snapshot1 = "test-snap1";
+        client().admin().cluster().prepareCreateSnapshot(repo, snapshot1).setWaitForCompletion(true).get();
+
+        String blockedNode = internalCluster().getMasterName();
+        ((MockRepository)internalCluster().getInstance(RepositoriesService.class, blockedNode).repository(repo)).blockOnDataFiles(true);
+        logger.info("--> start deletion of snapshot");
+        ListenableActionFuture<DeleteSnapshotResponse> future =
+            client().admin().cluster().prepareDeleteSnapshot(repo, snapshot1).execute();
+        logger.info("--> waiting for block to kick in on node [{}]", blockedNode);
+        waitForBlock(blockedNode, repo, TimeValue.timeValueSeconds(10));
+
+        logger.info("--> try creating a second snapshot, should fail because the deletion is in progress");
+        final String snapshot2 = "test-snap2";
+        try {
+            client().admin().cluster().prepareCreateSnapshot(repo, snapshot2).setWaitForCompletion(true).get();
+            fail("should not be able to create a snapshot while another is being deleted");
+        } catch (ConcurrentSnapshotExecutionException e) {
+            assertThat(e.getMessage(), containsString("cannot snapshot while a snapshot deletion is in-progress"));
+        }
+
+        logger.info("--> unblocking blocked node [{}]", blockedNode);
+        unblockNode(repo, blockedNode);
+
+        logger.info("--> wait until snapshot deletion is finished");
+        assertAcked(future.actionGet());
+
+        logger.info("--> creating second snapshot, which should now work");
+        client().admin().cluster().prepareCreateSnapshot(repo, snapshot2).setWaitForCompletion(true).get();
+        assertEquals(1, client().admin().cluster().prepareGetSnapshots(repo).setSnapshots("_all").get().getSnapshots().size());
+    }
+
+    public void testRestoreWithInProgressDeletionsNotAllowed() throws Exception {
+        logger.info("--> creating repository");
+        final String repo = "test-repo";
+        assertAcked(client().admin().cluster().preparePutRepository(repo).setType("mock").setSettings(
+            Settings.builder()
+                .put("location", randomRepoPath())
+                .put("random", randomAlphaOfLength(10))
+                .put("wait_after_unblock", 200)).get());
+
+        logger.info("--> snapshot");
+        final String index = "test-idx";
+        assertAcked(prepareCreate(index, 1, Settings.builder().put("number_of_shards", 1).put("number_of_replicas", 0)));
+        for (int i = 0; i < 10; i++) {
+            index(index, "doc", Integer.toString(i), "foo", "bar" + i);
+        }
+        refresh();
+        final String snapshot1 = "test-snap1";
+        client().admin().cluster().prepareCreateSnapshot(repo, snapshot1).setWaitForCompletion(true).get();
+        final String index2 = "test-idx2";
+        assertAcked(prepareCreate(index2, 1, Settings.builder().put("number_of_shards", 1).put("number_of_replicas", 0)));
+        for (int i = 0; i < 10; i++) {
+            index(index2, "doc", Integer.toString(i), "foo", "bar" + i);
+        }
+        refresh();
+        final String snapshot2 = "test-snap2";
+        client().admin().cluster().prepareCreateSnapshot(repo, snapshot2).setWaitForCompletion(true).get();
+        client().admin().indices().prepareClose(index, index2).get();
+
+        String blockedNode = internalCluster().getMasterName();
+        ((MockRepository)internalCluster().getInstance(RepositoriesService.class, blockedNode).repository(repo)).blockOnDataFiles(true);
+        logger.info("--> start deletion of snapshot");
+        ListenableActionFuture<DeleteSnapshotResponse> future =
+            client().admin().cluster().prepareDeleteSnapshot(repo, snapshot2).execute();
+        logger.info("--> waiting for block to kick in on node [{}]", blockedNode);
+        waitForBlock(blockedNode, repo, TimeValue.timeValueSeconds(10));
+
+        logger.info("--> try restoring the other snapshot, should fail because the deletion is in progress");
+        try {
+            client().admin().cluster().prepareRestoreSnapshot(repo, snapshot1).setWaitForCompletion(true).get();
+            fail("should not be able to restore a snapshot while another is being deleted");
+        } catch (ConcurrentSnapshotExecutionException e) {
+            assertThat(e.getMessage(), containsString("cannot restore a snapshot while a snapshot deletion is in-progress"));
+        }
+
+        logger.info("--> unblocking blocked node [{}]", blockedNode);
+        unblockNode(repo, blockedNode);
+
+        logger.info("--> wait until snapshot deletion is finished");
+        assertAcked(future.actionGet());
+
+        logger.info("--> restoring snapshot, which should now work");
+        client().admin().cluster().prepareRestoreSnapshot(repo, snapshot1).setWaitForCompletion(true).get();
+        assertEquals(1, client().admin().cluster().prepareGetSnapshots(repo).setSnapshots("_all").get().getSnapshots().size());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
index 3e43cc8304578..c865bb2e70582 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
@@ -34,10 +34,13 @@
 import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotStatus;
 import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
+import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptResponse;
 import org.elasticsearch.action.admin.indices.flush.FlushResponse;
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
 import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
 import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.action.ingest.DeletePipelineRequest;
+import org.elasticsearch.action.ingest.GetPipelineResponse;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.client.Client;
@@ -51,20 +54,33 @@
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.cluster.metadata.MetaDataIndexStateService;
+import org.elasticsearch.cluster.routing.IndexRoutingTable;
+import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.store.IndexStore;
-import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.InvalidIndexNameException;
+import org.elasticsearch.ingest.IngestTestPlugin;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.repositories.IndexId;
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.repositories.RepositoryData;
 import org.elasticsearch.repositories.RepositoryException;
+import org.elasticsearch.script.MockScriptEngine;
+import org.elasticsearch.script.StoredScriptsIT;
+import org.elasticsearch.snapshots.mockstore.MockRepository;
 import org.elasticsearch.test.junit.annotations.TestLogging;
 
 import java.nio.channels.SeekableByteChannel;
@@ -73,16 +89,21 @@
 import java.nio.file.StandardOpenOption;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.IndexSettings.INDEX_REFRESH_INTERVAL_SETTING;
 import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@@ -106,6 +127,14 @@
 import static org.hamcrest.Matchers.startsWith;
 
 public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Arrays.asList(IngestTestPlugin.class,
+            StoredScriptsIT.CustomScriptPlugin.class,
+            MockRepository.Plugin.class);
+    }
+
     public void testBasicWorkFlow() throws Exception {
         Client client = client();
 
@@ -422,8 +451,8 @@ public void testRestoreTemplates() throws Exception {
 
         logger.info("-->  creating test template");
         assertThat(client.admin().indices().preparePutTemplate("test-template").setTemplate("te*").addMapping("test-mapping", XContentFactory.jsonBuilder().startObject().startObject("test-mapping").startObject("properties")
-            .startObject("field1").field("type", "string").field("store", "yes").endObject()
-            .startObject("field2").field("type", "string").field("store", "yes").field("index", "not_analyzed").endObject()
+            .startObject("field1").field("type", "string").field("store", "true").endObject()
+            .startObject("field2").field("type", "string").field("store", "true").field("index", "not_analyzed").endObject()
             .endObject().endObject().endObject()).get().isAcknowledged(), equalTo(true));
 
         logger.info("--> snapshot");
@@ -456,11 +485,39 @@ public void testIncludeGlobalState() throws Exception {
         assertAcked(client.admin().cluster().preparePutRepository("test-repo")
                 .setType("fs").setSettings(Settings.builder().put("location", location)));
 
-        logger.info("-->  creating test template");
-        assertThat(client.admin().indices().preparePutTemplate("test-template").setTemplate("te*").addMapping("test-mapping", XContentFactory.jsonBuilder().startObject().startObject("test-mapping").startObject("properties")
-            .startObject("field1").field("type", "string").field("store", "yes").endObject()
-            .startObject("field2").field("type", "string").field("store", "yes").field("index", "not_analyzed").endObject()
-            .endObject().endObject().endObject()).get().isAcknowledged(), equalTo(true));
+        boolean testTemplate = randomBoolean();
+        boolean testPipeline = randomBoolean();
+        boolean testScript = (testTemplate == false && testPipeline == false) || randomBoolean(); // At least something should be stored
+
+        if(testTemplate) {
+            logger.info("-->  creating test template");
+            assertThat(client.admin().indices().preparePutTemplate("test-template").setTemplate("te*").addMapping("test-mapping", XContentFactory.jsonBuilder().startObject().startObject("test-mapping").startObject("properties")
+                .startObject("field1").field("type", "string").field("store", "true").endObject()
+                .startObject("field2").field("type", "string").field("store", "true").field("index", "not_analyzed").endObject()
+                .endObject().endObject().endObject()).get().isAcknowledged(), equalTo(true));
+        }
+
+        if(testPipeline) {
+            logger.info("-->  creating test pipeline");
+            BytesReference pipelineSource = jsonBuilder().startObject()
+                .field("description", "my_pipeline")
+                .startArray("processors")
+                .startObject()
+                .startObject("test")
+                .endObject()
+                .endObject()
+                .endArray()
+                .endObject().bytes();
+            assertAcked(client().admin().cluster().preparePutPipeline("barbaz", pipelineSource, XContentType.JSON).get());
+        }
+
+        if(testScript) {
+            logger.info("-->  creating test script");
+            assertAcked(client().admin().cluster().preparePutStoredScript()
+                .setLang(MockScriptEngine.NAME)
+                .setId("foobar")
+                .setContent(new BytesArray("{\"script\":\"1\"}"), XContentType.JSON));
+        }
 
         logger.info("--> snapshot without global state");
         CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap-no-global-state").setIndices().setIncludeGlobalState(false).setWaitForCompletion(true).get();
@@ -474,26 +531,52 @@ public void testIncludeGlobalState() throws Exception {
         assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(0));
         assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap-with-global-state").get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));
 
-        logger.info("-->  delete test template");
-        cluster().wipeTemplates("test-template");
-        GetIndexTemplatesResponse getIndexTemplatesResponse = client().admin().indices().prepareGetTemplates().get();
-        assertIndexTemplateMissing(getIndexTemplatesResponse, "test-template");
+        if (testTemplate) {
+            logger.info("-->  delete test template");
+            cluster().wipeTemplates("test-template");
+            GetIndexTemplatesResponse getIndexTemplatesResponse = client().admin().indices().prepareGetTemplates().get();
+            assertIndexTemplateMissing(getIndexTemplatesResponse, "test-template");
+        }
+
+        if (testPipeline) {
+            logger.info("-->  delete test pipeline");
+            assertAcked(client().admin().cluster().deletePipeline(new DeletePipelineRequest("barbaz")).get());
+        }
+
+        if (testScript) {
+            logger.info("-->  delete test script");
+            assertAcked(client().admin().cluster().prepareDeleteStoredScript(MockScriptEngine.NAME, "foobar").get());
+        }
 
         logger.info("--> try restoring cluster state from snapshot without global state");
         RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap-no-global-state").setWaitForCompletion(true).setRestoreGlobalState(true).execute().actionGet();
         assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), equalTo(0));
 
         logger.info("--> check that template wasn't restored");
-        getIndexTemplatesResponse = client().admin().indices().prepareGetTemplates().get();
+        GetIndexTemplatesResponse getIndexTemplatesResponse = client().admin().indices().prepareGetTemplates().get();
         assertIndexTemplateMissing(getIndexTemplatesResponse, "test-template");
 
         logger.info("--> restore cluster state");
         restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap-with-global-state").setWaitForCompletion(true).setRestoreGlobalState(true).execute().actionGet();
         assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), equalTo(0));
 
-        logger.info("--> check that template is restored");
-        getIndexTemplatesResponse = client().admin().indices().prepareGetTemplates().get();
-        assertIndexTemplateExists(getIndexTemplatesResponse, "test-template");
+        if (testTemplate) {
+            logger.info("--> check that template is restored");
+            getIndexTemplatesResponse = client().admin().indices().prepareGetTemplates().get();
+            assertIndexTemplateExists(getIndexTemplatesResponse, "test-template");
+        }
+
+        if (testPipeline) {
+            logger.info("--> check that pipeline is restored");
+            GetPipelineResponse getPipelineResponse = client().admin().cluster().prepareGetPipeline("barbaz").get();
+            assertTrue(getPipelineResponse.isFound());
+        }
+
+        if (testScript) {
+            logger.info("--> check that script is restored");
+            GetStoredScriptResponse getStoredScriptResponse = client().admin().cluster().prepareGetStoredScript(MockScriptEngine.NAME, "foobar").get();
+            assertNotNull(getStoredScriptResponse.getSource());
+        }
 
         createIndex("test-idx");
         ensureGreen();
@@ -511,9 +594,19 @@ public void testIncludeGlobalState() throws Exception {
         assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
         assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap-no-global-state-with-index").get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));
 
-        logger.info("-->  delete test template and index ");
+        logger.info("-->  delete global state and index ");
         cluster().wipeIndices("test-idx");
-        cluster().wipeTemplates("test-template");
+        if (testTemplate) {
+            cluster().wipeTemplates("test-template");
+        }
+        if (testPipeline) {
+            assertAcked(client().admin().cluster().deletePipeline(new DeletePipelineRequest("barbaz")).get());
+        }
+
+        if (testScript) {
+            assertAcked(client().admin().cluster().prepareDeleteStoredScript(MockScriptEngine.NAME, "foobar").get());
+        }
+
         getIndexTemplatesResponse = client().admin().indices().prepareGetTemplates().get();
         assertIndexTemplateMissing(getIndexTemplatesResponse, "test-template");
 
@@ -522,9 +615,11 @@ public void testIncludeGlobalState() throws Exception {
         assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
         assertThat(restoreSnapshotResponse.getRestoreInfo().failedShards(), equalTo(0));
 
-        logger.info("--> check that template wasn't restored but index was");
+        logger.info("--> check that global state wasn't restored but index was");
         getIndexTemplatesResponse = client().admin().indices().prepareGetTemplates().get();
         assertIndexTemplateMissing(getIndexTemplatesResponse, "test-template");
+        assertFalse(client().admin().cluster().prepareGetPipeline("barbaz").get().isFound());
+        assertNull(client().admin().cluster().prepareGetStoredScript(MockScriptEngine.NAME, "foobar").get().getSource());
         assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L));
 
     }
@@ -537,7 +632,7 @@ public void testSnapshotFileFailureDuringSnapshot() throws Exception {
                 .setType("mock").setSettings(
                         Settings.builder()
                                 .put("location", randomRepoPath())
-                                .put("random", randomAsciiOfLength(10))
+                                .put("random", randomAlphaOfLength(10))
                                 .put("random_control_io_exception_rate", 0.2))
                 .setVerify(false));
 
@@ -587,7 +682,7 @@ public void testDataFileFailureDuringSnapshot() throws Exception {
                 .setType("mock").setSettings(
                         Settings.builder()
                                 .put("location", randomRepoPath())
-                                .put("random", randomAsciiOfLength(10))
+                                .put("random", randomAlphaOfLength(10))
                                 .put("random_data_file_io_exception_rate", 0.3)));
 
         createIndex("test-idx");
@@ -672,7 +767,7 @@ public void testDataFileFailureDuringRestore() throws Exception {
                 .setType("mock").setSettings(
                         Settings.builder()
                                 .put("location", repositoryLocation)
-                                .put("random", randomAsciiOfLength(10))
+                                .put("random", randomAlphaOfLength(10))
                                 .put("random_data_file_io_exception_rate", 0.3)));
 
         // Test restore after index deletion
@@ -686,6 +781,58 @@ public void testDataFileFailureDuringRestore() throws Exception {
         logger.info("--> total number of simulated failures during restore: [{}]", getFailureCount("test-repo"));
     }
 
+    @TestLogging("org.elasticsearch.cluster.routing:TRACE,org.elasticsearch.snapshots:TRACE")
+    public void testDataFileCorruptionDuringRestore() throws Exception {
+        Path repositoryLocation = randomRepoPath();
+        Client client = client();
+        logger.info("-->  creating repository");
+        assertAcked(client.admin().cluster().preparePutRepository("test-repo")
+            .setType("fs").setSettings(Settings.builder().put("location", repositoryLocation)));
+
+        createIndex("test-idx");
+        ensureGreen();
+
+        logger.info("--> indexing some data");
+        for (int i = 0; i < 100; i++) {
+            index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i);
+        }
+        refresh();
+        assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L));
+
+        logger.info("--> snapshot");
+        CreateSnapshotResponse createSnapshotResponse = client.admin().cluster()
+            .prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx").get();
+        assertThat(createSnapshotResponse.getSnapshotInfo().state(), equalTo(SnapshotState.SUCCESS));
+        assertThat(createSnapshotResponse.getSnapshotInfo().totalShards(),
+            equalTo(createSnapshotResponse.getSnapshotInfo().successfulShards()));
+
+        logger.info("-->  update repository with mock version");
+        assertAcked(client.admin().cluster().preparePutRepository("test-repo")
+            .setType("mock").setSettings(
+                Settings.builder()
+                    .put("location", repositoryLocation)
+                    .put("random", randomAlphaOfLength(10))
+                    .put("use_lucene_corruption", true)
+                    .put("max_failure_number", Long.MAX_VALUE)
+                    .put("random_data_file_io_exception_rate", 1.0)));
+
+        // Test restore after index deletion
+        logger.info("--> delete index");
+        cluster().wipeIndices("test-idx");
+        logger.info("--> restore corrupt index");
+        RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster()
+            .prepareRestoreSnapshot("test-repo", "test-snap").setMasterNodeTimeout("30s")
+            .setWaitForCompletion(true).execute().actionGet();
+        assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
+        assertThat(restoreSnapshotResponse.getRestoreInfo().failedShards(),
+            equalTo(restoreSnapshotResponse.getRestoreInfo().totalShards()));
+        // we have to delete the index here manually, otherwise the cluster will keep
+        // trying to allocate the shards for the index, even though the restore operation
+        // is completed and marked as failed, which can lead to nodes having pending
+        // cluster states to process in their queue when the test is finished
+        cluster().wipeIndices("test-idx");
+    }
+
     public void testDeletionOfFailingToRecoverIndexShouldStopRestore() throws Exception {
         Path repositoryLocation = randomRepoPath();
         Client client = client();
@@ -713,7 +860,7 @@ public void testDeletionOfFailingToRecoverIndexShouldStopRestore() throws Except
                 .setType("mock").setSettings(
                         Settings.builder()
                                 .put("location", repositoryLocation)
-                                .put("random", randomAsciiOfLength(10))
+                                .put("random", randomAlphaOfLength(10))
                                 .put("random_data_file_io_exception_rate", 1.0) // Fail completely
                 ));
 
@@ -822,8 +969,8 @@ public void testDeleteSnapshot() throws Exception {
         logger.info("--> delete the last snapshot");
         client.admin().cluster().prepareDeleteSnapshot("test-repo", lastSnapshot).get();
         logger.info("--> make sure that number of files is back to what it was when the first snapshot was made, " +
-                    "plus one because one backup index-N file should remain");
-        assertThat(numberOfFiles(repo), equalTo(numberOfFiles[0] + 1));
+                    "plus two because one backup index-N file should remain and incompatible-snapshots");
+        assertThat(numberOfFiles(repo), equalTo(numberOfFiles[0] + 2));
     }
 
     public void testDeleteSnapshotWithMissingIndexAndShardMetadata() throws Exception {
@@ -935,6 +1082,44 @@ public void testDeleteSnapshotWithCorruptedSnapshotFile() throws Exception {
         assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
     }
 
+    public void testSnapshotWithMissingShardLevelIndexFile() throws Exception {
+        Path repo = randomRepoPath();
+        logger.info("-->  creating repository at {}", repo.toAbsolutePath());
+        assertAcked(client().admin().cluster().preparePutRepository("test-repo").setType("fs").setSettings(
+            Settings.builder().put("location", repo).put("compress", false)));
+
+        createIndex("test-idx-1", "test-idx-2");
+        logger.info("--> indexing some data");
+        indexRandom(true,
+            client().prepareIndex("test-idx-1", "doc").setSource("foo", "bar"),
+            client().prepareIndex("test-idx-2", "doc").setSource("foo", "bar"));
+
+        logger.info("--> creating snapshot");
+        client().admin().cluster().prepareCreateSnapshot("test-repo", "test-snap-1")
+            .setWaitForCompletion(true).setIndices("test-idx-*").get();
+
+        logger.info("--> deleting shard level index file");
+        try (Stream<Path> files = Files.list(repo.resolve("indices"))) {
+            files.forEach(indexPath ->
+                IOUtils.deleteFilesIgnoringExceptions(indexPath.resolve("0").resolve("index-0"))
+            );
+        }
+
+        logger.info("--> creating another snapshot");
+        CreateSnapshotResponse createSnapshotResponse =
+            client().admin().cluster().prepareCreateSnapshot("test-repo", "test-snap-2")
+                .setWaitForCompletion(true).setIndices("test-idx-1").get();
+        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
+        assertEquals(createSnapshotResponse.getSnapshotInfo().successfulShards(), createSnapshotResponse.getSnapshotInfo().totalShards());
+
+        logger.info("--> restoring the first snapshot, the repository should not have lost any shard data despite deleting index-N, " +
+                        "because it should have iterated over the snap-*.data files as backup");
+        client().admin().indices().prepareDelete("test-idx-1", "test-idx-2").get();
+        RestoreSnapshotResponse restoreSnapshotResponse =
+            client().admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap-1").setWaitForCompletion(true).get();
+        assertEquals(0, restoreSnapshotResponse.getRestoreInfo().failedShards());
+    }
+
     public void testSnapshotClosedIndex() throws Exception {
         Client client = client();
 
@@ -1108,7 +1293,7 @@ public void testMoveShardWhileSnapshotting() throws Exception {
                 .setType("mock").setSettings(
                         Settings.builder()
                                 .put("location", repositoryLocation)
-                                .put("random", randomAsciiOfLength(10))
+                                .put("random", randomAlphaOfLength(10))
                                 .put("wait_after_unblock", 200)));
 
         // Create index on 2 nodes and make sure each node has a primary by setting no replicas
@@ -1169,7 +1354,7 @@ public void testDeleteRepositoryWhileSnapshotting() throws Exception {
                 .setType("mock").setSettings(
                         Settings.builder()
                                 .put("location", repositoryLocation)
-                                .put("random", randomAsciiOfLength(10))
+                                .put("random", randomAlphaOfLength(10))
                                 .put("wait_after_unblock", 200)
                 ).get();
         assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
@@ -1368,8 +1553,8 @@ public void testThrottling() throws Exception {
                         .put("location", repositoryLocation)
                         .put("compress", randomBoolean())
                         .put("chunk_size", randomIntBetween(1000, 10000), ByteSizeUnit.BYTES)
-                        .put("max_restore_bytes_per_sec", throttleRestore ? "0.5k" : "0")
-                        .put("max_snapshot_bytes_per_sec", throttleSnapshot ? "0.5k" : "0")));
+                        .put("max_restore_bytes_per_sec", throttleRestore ? "10k" : "0")
+                        .put("max_snapshot_bytes_per_sec", throttleSnapshot ? "10k" : "0")));
 
         createIndex("test-idx");
         ensureGreen();
@@ -1422,7 +1607,7 @@ public void testSnapshotStatus() throws Exception {
                 .setType("mock").setSettings(
                         Settings.builder()
                                 .put("location", repositoryLocation)
-                                .put("random", randomAsciiOfLength(10))
+                                .put("random", randomAlphaOfLength(10))
                                 .put("wait_after_unblock", 200)
                 ).get();
         assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
@@ -1775,7 +1960,7 @@ public void testRecreateBlocksOnRestore() throws Exception {
             Settings initialSettings = initialSettingsBuilder.build();
             logger.info("--> using initial block settings {}", initialSettings.getAsMap());
 
-            if (!initialSettings.getAsMap().isEmpty()) {
+            if (!initialSettings.isEmpty()) {
                 logger.info("--> apply initial blocks to index");
                 client().admin().indices().prepareUpdateSettings("test-idx").setSettings(initialSettingsBuilder).get();
             }
@@ -2108,9 +2293,9 @@ public void testDeleteOrphanSnapshot() throws Exception {
             public ClusterState execute(ClusterState currentState) {
                 // Simulate orphan snapshot
                 ImmutableOpenMap.Builder<ShardId, ShardSnapshotStatus> shards = ImmutableOpenMap.builder();
-                shards.put(new ShardId(idxName, "_na_", 0), new ShardSnapshotStatus("unknown-node", State.ABORTED));
-                shards.put(new ShardId(idxName, "_na_", 1), new ShardSnapshotStatus("unknown-node", State.ABORTED));
-                shards.put(new ShardId(idxName, "_na_", 2), new ShardSnapshotStatus("unknown-node", State.ABORTED));
+                shards.put(new ShardId(idxName, "_na_", 0), new ShardSnapshotStatus("unknown-node", State.ABORTED, "aborted"));
+                shards.put(new ShardId(idxName, "_na_", 1), new ShardSnapshotStatus("unknown-node", State.ABORTED, "aborted"));
+                shards.put(new ShardId(idxName, "_na_", 2), new ShardSnapshotStatus("unknown-node", State.ABORTED, "aborted"));
                 List<Entry> entries = new ArrayList<>();
                 entries.add(new Entry(new Snapshot(repositoryName,
                                                    createSnapshotResponse.getSnapshotInfo().snapshotId()),
@@ -2119,6 +2304,7 @@ public ClusterState execute(ClusterState currentState) {
                                                    State.ABORTED,
                                                    Collections.singletonList(indexId),
                                                    System.currentTimeMillis(),
+                                                   repositoryData.getGenId(),
                                                    shards.build()));
                 return ClusterState.builder(currentState).putCustom(SnapshotsInProgress.TYPE, new SnapshotsInProgress(Collections.unmodifiableList(entries))).build();
             }
@@ -2177,7 +2363,7 @@ public void testBatchingShardUpdateTask() throws Exception {
         final ClusterService clusterService = internalCluster().clusterService(internalCluster().getMasterName());
         BlockingClusterStateListener snapshotListener = new BlockingClusterStateListener(clusterService, "update_snapshot [", "update snapshot state", Priority.HIGH);
         try {
-            clusterService.addFirst(snapshotListener);
+            clusterService.addListener(snapshotListener);
             logger.info("--> snapshot");
             ListenableActionFuture<CreateSnapshotResponse> snapshotFuture = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx").execute();
 
@@ -2192,39 +2378,13 @@ public void testBatchingShardUpdateTask() throws Exception {
             assertEquals(numberOfShards, createSnapshotResponse.getSnapshotInfo().successfulShards());
 
         } finally {
-            clusterService.remove(snapshotListener);
+            clusterService.removeListener(snapshotListener);
         }
 
         // Check that we didn't timeout
         assertFalse(snapshotListener.timedOut());
         // Check that cluster state update task was called only once
         assertEquals(1, snapshotListener.count());
-
-        logger.info("--> close indices");
-        client.admin().indices().prepareClose("test-idx").get();
-
-        BlockingClusterStateListener restoreListener = new BlockingClusterStateListener(clusterService, "restore_snapshot[", "update snapshot state", Priority.HIGH);
-
-        try {
-            clusterService.addFirst(restoreListener);
-            logger.info("--> restore snapshot");
-            ListenableActionFuture<RestoreSnapshotResponse> futureRestore = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).execute();
-
-            // Await until shard updates are in pending state.
-            assertBusyPendingTasks("update snapshot state", numberOfShards);
-            restoreListener.unblock();
-
-            RestoreSnapshotResponse restoreSnapshotResponse = futureRestore.actionGet();
-            assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), equalTo(numberOfShards));
-
-        } finally {
-            clusterService.remove(restoreListener);
-        }
-
-        // Check that we didn't timeout
-        assertFalse(restoreListener.timedOut());
-        // Check that cluster state update task was called only once
-        assertEquals(1, restoreListener.count());
     }
 
     public void testSnapshotName() throws Exception {
@@ -2343,7 +2503,7 @@ public void testCannotCreateSnapshotsWithSameName() throws Exception {
                                            .get();
             fail("should not be allowed to create a snapshot with the same name as an already existing snapshot: " +
                  createSnapshotResponse.getSnapshotInfo().snapshotId());
-        } catch (SnapshotCreationException e) {
+        } catch (InvalidSnapshotNameException e) {
             assertThat(e.getMessage(), containsString("snapshot with the same name already exists"));
         }
 
@@ -2398,11 +2558,31 @@ public void testGetSnapshotsRequest() throws Exception {
         }
         refresh();
 
+        // make sure we return only the in-progress snapshot when taking the first snapshot on a clean repository
+        // take initial snapshot with a block, making sure we only get 1 in-progress snapshot returned
+        // block a node so the create snapshot operation can remain in progress
+        final String initialBlockedNode = blockNodeWithIndex(repositoryName, indexName);
+        ListenableActionFuture<CreateSnapshotResponse> responseListener =
+            client.admin().cluster().prepareCreateSnapshot(repositoryName, "snap-on-empty-repo")
+                .setWaitForCompletion(false)
+                .setIndices(indexName)
+                .execute();
+        waitForBlock(initialBlockedNode, repositoryName, TimeValue.timeValueSeconds(60)); // wait for block to kick in
+        getSnapshotsResponse = client.admin().cluster()
+                                   .prepareGetSnapshots("test-repo")
+                                   .setSnapshots(randomFrom("_all", "_current", "snap-on-*", "*-on-empty-repo", "snap-on-empty-repo"))
+                                   .get();
+        assertEquals(1, getSnapshotsResponse.getSnapshots().size());
+        assertEquals("snap-on-empty-repo", getSnapshotsResponse.getSnapshots().get(0).snapshotId().getName());
+        unblockNode(repositoryName, initialBlockedNode); // unblock node
+        responseListener.actionGet(TimeValue.timeValueMillis(10000L)); // timeout after 10 seconds
+        client.admin().cluster().prepareDeleteSnapshot(repositoryName, "snap-on-empty-repo").get();
+
         final int numSnapshots = randomIntBetween(1, 3) + 1;
-        logger.info("--> take {} snapshot(s)", numSnapshots);
+        logger.info("--> take {} snapshot(s)", numSnapshots - 1);
         final String[] snapshotNames = new String[numSnapshots];
         for (int i = 0; i < numSnapshots - 1; i++) {
-            final String snapshotName = randomAsciiOfLength(8).toLowerCase(Locale.ROOT);
+            final String snapshotName = randomAlphaOfLength(8).toLowerCase(Locale.ROOT);
             CreateSnapshotResponse createSnapshotResponse = client.admin()
                                                                   .cluster()
                                                                   .prepareCreateSnapshot(repositoryName, snapshotName)
@@ -2419,7 +2599,7 @@ public void testGetSnapshotsRequest() throws Exception {
         }
         refresh();
 
-        final String inProgressSnapshot = randomAsciiOfLength(8).toLowerCase(Locale.ROOT);
+        final String inProgressSnapshot = randomAlphaOfLength(8).toLowerCase(Locale.ROOT);
         snapshotNames[numSnapshots - 1] = inProgressSnapshot;
         // block a node so the create snapshot operation can remain in progress
         final String blockedNode = blockNodeWithIndex(repositoryName, indexName);
@@ -2431,9 +2611,19 @@ public void testGetSnapshotsRequest() throws Exception {
 
         logger.info("--> get all snapshots with a current in-progress");
         // with ignore unavailable set to true, should not throw an exception
+        final List<String> snapshotsToGet = new ArrayList<>();
+        if (randomBoolean()) {
+            // use _current plus the individual names of the finished snapshots
+            snapshotsToGet.add("_current");
+            for (int i = 0; i < numSnapshots - 1; i++) {
+                snapshotsToGet.add(snapshotNames[i]);
+            }
+        } else {
+            snapshotsToGet.add("_all");
+        }
         getSnapshotsResponse = client.admin().cluster()
                                              .prepareGetSnapshots(repositoryName)
-                                             .addSnapshots("_all")
+                                             .setSnapshots(snapshotsToGet.toArray(Strings.EMPTY_ARRAY))
                                              .get();
         List<String> sortedNames = Arrays.asList(snapshotNames);
         Collections.sort(sortedNames);
@@ -2475,4 +2665,282 @@ public void testGetSnapshotsRequest() throws Exception {
         waitForCompletion(repositoryName, inProgressSnapshot, TimeValue.timeValueSeconds(60));
     }
 
+    /**
+     * This test ensures that when a shard is removed from a node (perhaps due to the node
+     * leaving the cluster, then returning), all snapshotting of that shard is aborted, so
+     * all Store references held onto by the snapshot are released.
+     *
+     * See https://github.com/elastic/elasticsearch/issues/20876
+     */
+    public void testSnapshotCanceledOnRemovedShard() throws Exception {
+        final int numPrimaries = 1;
+        final int numReplicas = 1;
+        final int numDocs = 100;
+        final String repo = "test-repo";
+        final String index = "test-idx";
+        final String snapshot = "test-snap";
+
+        assertAcked(prepareCreate(index, 1,
+            Settings.builder().put("number_of_shards", numPrimaries).put("number_of_replicas", numReplicas)));
+
+        logger.info("--> indexing some data");
+        for (int i = 0; i < numDocs; i++) {
+            index(index, "doc", Integer.toString(i), "foo", "bar" + i);
+        }
+        refresh();
+
+        logger.info("--> creating repository");
+        PutRepositoryResponse putRepositoryResponse =
+            client().admin().cluster().preparePutRepository(repo).setType("mock").setSettings(Settings.builder()
+                .put("location", randomRepoPath())
+                .put("random", randomAlphaOfLength(10))
+                .put("wait_after_unblock", 200)
+            ).get();
+        assertTrue(putRepositoryResponse.isAcknowledged());
+
+        String blockedNode = blockNodeWithIndex(repo, index);
+
+        logger.info("--> snapshot");
+        client().admin().cluster().prepareCreateSnapshot(repo, snapshot)
+            .setWaitForCompletion(false)
+            .execute();
+
+        logger.info("--> waiting for block to kick in on node [{}]", blockedNode);
+        waitForBlock(blockedNode, repo, TimeValue.timeValueSeconds(10));
+
+        logger.info("--> removing primary shard that is being snapshotted");
+        ClusterState clusterState = internalCluster().clusterService(internalCluster().getMasterName()).state();
+        IndexRoutingTable indexRoutingTable = clusterState.getRoutingTable().index(index);
+        String nodeWithPrimary = clusterState.nodes().get(indexRoutingTable.shard(0).primaryShard().currentNodeId()).getName();
+        assertNotNull("should be at least one node with a primary shard", nodeWithPrimary);
+        IndicesService indicesService = internalCluster().getInstance(IndicesService.class, nodeWithPrimary);
+        IndexService indexService = indicesService.indexService(resolveIndex(index));
+        indexService.removeShard(0, "simulate node removal");
+
+        logger.info("--> unblocking blocked node [{}]", blockedNode);
+        unblockNode(repo, blockedNode);
+
+        logger.info("--> ensuring snapshot is aborted and the aborted shard was marked as failed");
+        SnapshotInfo snapshotInfo = waitForCompletion(repo, snapshot, TimeValue.timeValueSeconds(10));
+        assertEquals(1, snapshotInfo.shardFailures().size());
+        assertEquals(0, snapshotInfo.shardFailures().get(0).shardId());
+        assertEquals("IndexShardSnapshotFailedException[Aborted]", snapshotInfo.shardFailures().get(0).reason());
+    }
+
+    public void testSnapshotSucceedsAfterSnapshotFailure() throws Exception {
+        logger.info("--> creating repository");
+        final Path repoPath = randomRepoPath();
+        final Client client = client();
+        assertAcked(client.admin().cluster().preparePutRepository("test-repo").setType("mock").setVerify(false).setSettings(
+                Settings.builder()
+                    .put("location", repoPath)
+                    .put("random_control_io_exception_rate", randomIntBetween(5, 20) / 100f)
+                    // test that we can take a snapshot after a failed one, even if a partial index-N was written
+                    .put("atomic_move", false)
+                    .put("random", randomAlphaOfLength(10))));
+
+        logger.info("--> indexing some data");
+        assertAcked(prepareCreate("test-idx").setSettings(
+            // the less the number of shards, the less control files we have, so we are giving a higher probability of
+            // triggering an IOException toward the end when writing the pending-index-* files, which are the files
+            // that caused problems with writing subsequent snapshots if they happened to be lingering in the repository
+            Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 0)));
+        ensureGreen();
+        final int numDocs = randomIntBetween(1, 5);
+        for (int i = 0; i < numDocs; i++) {
+            index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i);
+        }
+        refresh();
+        assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo((long) numDocs));
+
+        logger.info("--> snapshot with potential I/O failures");
+        try {
+            CreateSnapshotResponse createSnapshotResponse =
+                client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap")
+                    .setWaitForCompletion(true)
+                    .setIndices("test-idx")
+                    .get();
+            if (createSnapshotResponse.getSnapshotInfo().totalShards() != createSnapshotResponse.getSnapshotInfo().successfulShards()) {
+                assertThat(getFailureCount("test-repo"), greaterThan(0L));
+                assertThat(createSnapshotResponse.getSnapshotInfo().shardFailures().size(), greaterThan(0));
+                for (SnapshotShardFailure shardFailure : createSnapshotResponse.getSnapshotInfo().shardFailures()) {
+                    assertThat(shardFailure.reason(), containsString("Random IOException"));
+                }
+            }
+        } catch (SnapshotCreationException | RepositoryException ex) {
+            // sometimes, the snapshot will fail with a top level I/O exception
+            assertThat(ExceptionsHelper.stackTrace(ex), containsString("Random IOException"));
+        }
+
+        logger.info("--> snapshot with no I/O failures");
+        assertAcked(client.admin().cluster().preparePutRepository("test-repo-2").setType("mock").setSettings(
+            Settings.builder().put("location", repoPath)));
+        CreateSnapshotResponse createSnapshotResponse =
+            client.admin().cluster().prepareCreateSnapshot("test-repo-2", "test-snap-2")
+                .setWaitForCompletion(true)
+                .setIndices("test-idx")
+                .get();
+        assertEquals(0, createSnapshotResponse.getSnapshotInfo().failedShards());
+        GetSnapshotsResponse getSnapshotsResponse = client.admin().cluster().prepareGetSnapshots("test-repo-2")
+                                                        .addSnapshots("test-snap-2").get();
+        assertEquals(SnapshotState.SUCCESS, getSnapshotsResponse.getSnapshots().get(0).state());
+    }
+
+    public void testSnapshotStatusOnFailedIndex() throws Exception {
+        logger.info("--> creating repository");
+        final Path repoPath = randomRepoPath();
+        final Client client = client();
+        assertAcked(client.admin().cluster()
+            .preparePutRepository("test-repo")
+            .setType("fs")
+            .setVerify(false)
+            .setSettings(Settings.builder().put("location", repoPath)));
+
+        logger.info("--> creating good index");
+        assertAcked(prepareCreate("test-idx-good")
+            .setSettings(Settings.builder()
+                .put(SETTING_NUMBER_OF_SHARDS, 1)
+                .put(SETTING_NUMBER_OF_REPLICAS, 0)));
+        ensureGreen();
+        final int numDocs = randomIntBetween(1, 5);
+        for (int i = 0; i < numDocs; i++) {
+            index("test-idx-good", "doc", Integer.toString(i), "foo", "bar" + i);
+        }
+        refresh();
+
+        logger.info("--> creating bad index");
+        assertAcked(prepareCreate("test-idx-bad")
+            .setWaitForActiveShards(ActiveShardCount.NONE)
+            .setSettings(Settings.builder()
+                .put(SETTING_NUMBER_OF_SHARDS, 1)
+                .put(SETTING_NUMBER_OF_REPLICAS, 0)
+                // set shard allocation to none so the primary cannot be
+                // allocated - simulates a "bad" index that fails to snapshot
+                .put(EnableAllocationDecider.INDEX_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(),
+                     "none")));
+
+        logger.info("--> snapshot bad index and get status");
+        client.admin().cluster()
+            .prepareCreateSnapshot("test-repo", "test-snap1")
+            .setWaitForCompletion(true)
+            .setIndices("test-idx-bad")
+            .get();
+        SnapshotsStatusResponse snapshotsStatusResponse = client.admin().cluster()
+            .prepareSnapshotStatus("test-repo")
+            .setSnapshots("test-snap1")
+            .get();
+        assertEquals(1, snapshotsStatusResponse.getSnapshots().size());
+        assertEquals(State.FAILED, snapshotsStatusResponse.getSnapshots().get(0).getState());
+
+        logger.info("--> snapshot both good and bad index and get status");
+        client.admin().cluster()
+            .prepareCreateSnapshot("test-repo", "test-snap2")
+            .setWaitForCompletion(true)
+            .setIndices("test-idx-good", "test-idx-bad")
+            .get();
+        snapshotsStatusResponse = client.admin().cluster()
+            .prepareSnapshotStatus("test-repo")
+            .setSnapshots("test-snap2")
+            .get();
+        assertEquals(1, snapshotsStatusResponse.getSnapshots().size());
+        // verify a FAILED status is returned instead of a 500 status code
+        // see https://github.com/elastic/elasticsearch/issues/23716
+        SnapshotStatus snapshotStatus = snapshotsStatusResponse.getSnapshots().get(0);
+        assertEquals(State.FAILED, snapshotStatus.getState());
+        for (SnapshotIndexShardStatus shardStatus : snapshotStatus.getShards()) {
+            assertEquals(SnapshotIndexShardStage.FAILURE, shardStatus.getStage());
+            if (shardStatus.getIndex().equals("test-idx-good")) {
+                assertEquals("skipped", shardStatus.getFailure());
+            } else {
+                assertEquals("primary shard is not allocated", shardStatus.getFailure());
+            }
+        }
+    }
+
+    public void testGetSnapshotsFromIndexBlobOnly() throws Exception {
+        logger.info("--> creating repository");
+        final Path repoPath = randomRepoPath();
+        final Client client = client();
+        assertAcked(client.admin().cluster()
+            .preparePutRepository("test-repo")
+            .setType("fs")
+            .setVerify(false)
+            .setSettings(Settings.builder().put("location", repoPath)));
+
+        logger.info("--> creating random number of indices");
+        final int numIndices = randomIntBetween(1, 10);
+        for (int i = 0; i < numIndices; i++) {
+            assertAcked(prepareCreate("test-idx-" + i).setSettings(Settings.builder()
+                .put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 0)));
+        }
+
+        logger.info("--> creating random number of snapshots");
+        final int numSnapshots = randomIntBetween(1, 10);
+        final Map<String, List<String>> indicesPerSnapshot = new HashMap<>();
+        for (int i = 0; i < numSnapshots; i++) {
+            // index some additional docs (maybe) for each index
+            for (int j = 0; j < numIndices; j++) {
+                if (randomBoolean()) {
+                    final int numDocs = randomIntBetween(1, 5);
+                    for (int k = 0; k < numDocs; k++) {
+                        index("test-idx-" + j, "doc", Integer.toString(k), "foo", "bar" + k);
+                    }
+                    refresh();
+                }
+            }
+            final boolean all = randomBoolean();
+            boolean atLeastOne = false;
+            List<String> indices = new ArrayList<>();
+            for (int j = 0; j < numIndices; j++) {
+                if (all || randomBoolean() || !atLeastOne) {
+                    indices.add("test-idx-" + j);
+                    atLeastOne = true;
+                }
+            }
+            final String snapshotName = "test-snap-" + i;
+            indicesPerSnapshot.put(snapshotName, indices);
+            client.admin().cluster()
+                .prepareCreateSnapshot("test-repo", snapshotName)
+                .setWaitForCompletion(true)
+                .setIndices(indices.toArray(new String[indices.size()]))
+                .get();
+        }
+
+        logger.info("--> verify _all returns snapshot info");
+        GetSnapshotsResponse response = client().admin().cluster()
+            .prepareGetSnapshots("test-repo")
+            .setSnapshots("_all")
+            .setVerbose(false)
+            .get();
+        assertEquals(indicesPerSnapshot.size(), response.getSnapshots().size());
+        verifySnapshotInfo(response, indicesPerSnapshot);
+
+        logger.info("--> verify wildcard returns snapshot info");
+        response = client().admin().cluster()
+            .prepareGetSnapshots("test-repo")
+            .setSnapshots("test-snap-*")
+            .setVerbose(false)
+            .get();
+        assertEquals(indicesPerSnapshot.size(), response.getSnapshots().size());
+        verifySnapshotInfo(response, indicesPerSnapshot);
+
+        logger.info("--> verify individual requests return snapshot info");
+        for (int i = 0; i < numSnapshots; i++) {
+            response = client().admin().cluster()
+                .prepareGetSnapshots("test-repo")
+                .setSnapshots("test-snap-" + i)
+                .setVerbose(false)
+                .get();
+            assertEquals(1, response.getSnapshots().size());
+            verifySnapshotInfo(response, indicesPerSnapshot);
+        }
+    }
+
+    private void verifySnapshotInfo(final GetSnapshotsResponse response, final Map<String, List<String>> indicesPerSnapshot) {
+        for (SnapshotInfo snapshotInfo : response.getSnapshots()) {
+            final List<String> expected = snapshotInfo.indices();
+            assertEquals(expected, indicesPerSnapshot.get(snapshotInfo.snapshotId().getName()));
+            assertEquals(SnapshotState.SUCCESS, snapshotInfo.state());
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/snapshots/SnapshotRequestsTests.java b/core/src/test/java/org/elasticsearch/snapshots/SnapshotRequestsTests.java
index c178b2a6f83bd..a8e1eeaec106b 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/SnapshotRequestsTests.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/SnapshotRequestsTests.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
@@ -76,10 +77,9 @@ public void testRestoreSnapshotRequestParsing() throws IOException {
             builder.endArray();
         }
 
-        byte[] bytes = BytesReference.toBytes(builder.endObject().bytes());
+        BytesReference bytes = builder.endObject().bytes();
 
-
-        request.source(bytes);
+        request.source(XContentHelper.convertToMap(bytes, true, builder.contentType()).v2());
 
         assertEquals("test-repo", request.repository());
         assertEquals("test-snap", request.snapshot());
@@ -135,10 +135,9 @@ public void testCreateSnapshotRequestParsing() throws IOException {
             builder.endArray();
         }
 
-        byte[] bytes = BytesReference.toBytes(builder.endObject().bytes());
-
+        BytesReference bytes = builder.endObject().bytes();
 
-        request.source(bytes);
+        request.source(XContentHelper.convertToMap(bytes, true, builder.contentType()).v2());
 
         assertEquals("test-repo", request.repository());
         assertEquals("test-snap", request.snapshot());
diff --git a/core/src/test/java/org/elasticsearch/snapshots/SnapshotTests.java b/core/src/test/java/org/elasticsearch/snapshots/SnapshotTests.java
index 41cfa3d414135..3d7277b9c68d3 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/SnapshotTests.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/SnapshotTests.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.snapshots;
 
+import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest;
+import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.test.ESTestCase;
@@ -44,11 +46,25 @@ public void testSnapshotEquals() {
     }
 
     public void testSerialization() throws IOException {
-        final SnapshotId snapshotId = new SnapshotId(randomAsciiOfLength(randomIntBetween(2, 8)), UUIDs.randomBase64UUID());
-        final Snapshot original = new Snapshot(randomAsciiOfLength(randomIntBetween(2, 8)), snapshotId);
+        final SnapshotId snapshotId = new SnapshotId(randomAlphaOfLength(randomIntBetween(2, 8)), UUIDs.randomBase64UUID());
+        final Snapshot original = new Snapshot(randomAlphaOfLength(randomIntBetween(2, 8)), snapshotId);
         final BytesStreamOutput out = new BytesStreamOutput();
         original.writeTo(out);
         assertThat(new Snapshot(out.bytes().streamInput()), equalTo(original));
     }
 
+    public void testCreateSnapshotRequestDescrptions() {
+        CreateSnapshotRequest createSnapshotRequest = new CreateSnapshotRequest();
+        createSnapshotRequest.snapshot("snapshot_name");
+        createSnapshotRequest.repository("repo_name");
+        assertEquals("snapshot [repo_name:snapshot_name]", createSnapshotRequest.getDescription());
+    }
+
+    public void testRestoreSnapshotRequestDescrptions() {
+        RestoreSnapshotRequest restoreSnapshotRequest = new RestoreSnapshotRequest();
+        restoreSnapshotRequest.snapshot("snapshot_name");
+        restoreSnapshotRequest.repository("repo_name");
+        assertEquals("snapshot [repo_name:snapshot_name]", restoreSnapshotRequest.getDescription());
+    }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java b/core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java
index 06c4ec10af056..3a5b068cd8977 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java
@@ -33,6 +33,8 @@
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.atomic.AtomicLong;
 
+import com.carrotsearch.randomizedtesting.RandomizedContext;
+import org.apache.lucene.index.CorruptIndexException;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.metadata.RepositoryMetaData;
@@ -44,6 +46,7 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.plugins.RepositoryPlugin;
 import org.elasticsearch.repositories.Repository;
@@ -61,8 +64,8 @@ public static class Plugin extends org.elasticsearch.plugins.Plugin implements R
 
 
         @Override
-        public Map<String, Repository.Factory> getRepositories(Environment env) {
-            return Collections.singletonMap("mock", (metadata) -> new MockRepository(metadata, env));
+        public Map<String, Repository.Factory> getRepositories(Environment env, NamedXContentRegistry namedXContentRegistry) {
+            return Collections.singletonMap("mock", (metadata) -> new MockRepository(metadata, env, namedXContentRegistry));
         }
 
         @Override
@@ -81,6 +84,8 @@ public long getFailureCount() {
 
     private final double randomDataFileIOExceptionRate;
 
+    private final boolean useLuceneCorruptionException;
+
     private final long maximumNumberOfFailures;
 
     private final long waitAfterUnblock;
@@ -95,18 +100,31 @@ public long getFailureCount() {
 
     private volatile boolean blockOnDataFiles;
 
+    /** Allows blocking on writing the index-N blob; this is a way to enforce blocking the
+     *  finalization of a snapshot, while permitting other IO operations to proceed unblocked. */
+    private volatile boolean blockOnWriteIndexFile;
+
+    /** Allows blocking on writing the snapshot file at the end of snapshot creation to simulate a died master node */
+    private volatile boolean blockAndFailOnWriteSnapFile;
+
+    private volatile boolean atomicMove;
+
     private volatile boolean blocked = false;
 
-    public MockRepository(RepositoryMetaData metadata, Environment environment) throws IOException {
-        super(overrideSettings(metadata, environment), environment);
+    public MockRepository(RepositoryMetaData metadata, Environment environment,
+                          NamedXContentRegistry namedXContentRegistry) throws IOException {
+        super(overrideSettings(metadata, environment), environment, namedXContentRegistry);
         randomControlIOExceptionRate = metadata.settings().getAsDouble("random_control_io_exception_rate", 0.0);
         randomDataFileIOExceptionRate = metadata.settings().getAsDouble("random_data_file_io_exception_rate", 0.0);
+        useLuceneCorruptionException = metadata.settings().getAsBoolean("use_lucene_corruption", false);
         maximumNumberOfFailures = metadata.settings().getAsLong("max_failure_number", 100L);
         blockOnControlFiles = metadata.settings().getAsBoolean("block_on_control", false);
         blockOnDataFiles = metadata.settings().getAsBoolean("block_on_data", false);
         blockOnInitialization = metadata.settings().getAsBoolean("block_on_init", false);
+        blockAndFailOnWriteSnapFile = metadata.settings().getAsBoolean("block_on_snap", false);
         randomPrefix = metadata.settings().get("random", "default");
         waitAfterUnblock = metadata.settings().getAsLong("wait_after_unblock", 0L);
+        atomicMove = metadata.settings().getAsBoolean("atomic_move", true);
         logger.info("starting mock repository with random prefix {}", randomPrefix);
         mockBlobStore = new MockBlobStore(super.blobStore());
     }
@@ -147,29 +165,27 @@ protected BlobStore blobStore() {
         return mockBlobStore;
     }
 
-    public void unblock() {
-        unblockExecution();
+    public synchronized void unblock() {
+        blocked = false;
+        // Clean blocking flags, so we wouldn't try to block again
+        blockOnDataFiles = false;
+        blockOnControlFiles = false;
+        blockOnInitialization = false;
+        blockOnWriteIndexFile = false;
+        blockAndFailOnWriteSnapFile = false;
+        this.notifyAll();
     }
 
     public void blockOnDataFiles(boolean blocked) {
         blockOnDataFiles = blocked;
     }
 
-    public void blockOnControlFiles(boolean blocked) {
-        blockOnControlFiles = blocked;
+    public void setBlockAndFailOnWriteSnapFiles(boolean blocked) {
+        blockAndFailOnWriteSnapFile = blocked;
     }
 
-    public boolean blockOnDataFiles() {
-        return blockOnDataFiles;
-    }
-
-    public synchronized void unblockExecution() {
-        blocked = false;
-        // Clean blocking flags, so we wouldn't try to block again
-        blockOnDataFiles = false;
-        blockOnControlFiles = false;
-        blockOnInitialization = false;
-        this.notifyAll();
+    public void setBlockOnWriteIndexFile(boolean blocked) {
+        blockOnWriteIndexFile = blocked;
     }
 
     public boolean blocked() {
@@ -180,7 +196,8 @@ private synchronized boolean blockExecution() {
         logger.debug("Blocking execution");
         boolean wasBlocked = false;
         try {
-            while (blockOnDataFiles || blockOnControlFiles || blockOnInitialization) {
+            while (blockOnDataFiles || blockOnControlFiles || blockOnInitialization || blockOnWriteIndexFile ||
+                blockAndFailOnWriteSnapFile) {
                 blocked = true;
                 this.wait();
                 wasBlocked = true;
@@ -245,40 +262,49 @@ private void maybeIOExceptionOrBlock(String blobName) throws IOException {
                 if (blobName.startsWith("__")) {
                     if (shouldFail(blobName, randomDataFileIOExceptionRate) && (incrementAndGetFailureCount() < maximumNumberOfFailures)) {
                         logger.info("throwing random IOException for file [{}] at path [{}]", blobName, path());
-                        throw new IOException("Random IOException");
-                    } else if (blockOnDataFiles) {
-                        logger.info("blocking I/O operation for file [{}] at path [{}]", blobName, path());
-                        if (blockExecution() && waitAfterUnblock > 0) {
-                            try {
-                                // Delay operation after unblocking
-                                // So, we can start node shutdown while this operation is still running.
-                                Thread.sleep(waitAfterUnblock);
-                            } catch (InterruptedException ex) {
-                                //
-                            }
+                        if (useLuceneCorruptionException) {
+                            throw new CorruptIndexException("Random corruption", "random file");
+                        } else {
+                            throw new IOException("Random IOException");
                         }
+                    } else if (blockOnDataFiles) {
+                        blockExecutionAndMaybeWait(blobName);
                     }
                 } else {
                     if (shouldFail(blobName, randomControlIOExceptionRate) && (incrementAndGetFailureCount() < maximumNumberOfFailures)) {
                         logger.info("throwing random IOException for file [{}] at path [{}]", blobName, path());
                         throw new IOException("Random IOException");
                     } else if (blockOnControlFiles) {
-                        logger.info("blocking I/O operation for file [{}] at path [{}]", blobName, path());
-                        if (blockExecution() && waitAfterUnblock > 0) {
-                            try {
-                                // Delay operation after unblocking
-                                // So, we can start node shutdown while this operation is still running.
-                                Thread.sleep(waitAfterUnblock);
-                            } catch (InterruptedException ex) {
-                                //
-                            }
-                        }
+                        blockExecutionAndMaybeWait(blobName);
+                    } else if (blobName.startsWith("snap-") && blockAndFailOnWriteSnapFile) {
+                        blockExecutionAndFail(blobName);
                     }
                 }
             }
 
+            private void blockExecutionAndMaybeWait(final String blobName) {
+                logger.info("blocking I/O operation for file [{}] at path [{}]", blobName, path());
+                if (blockExecution() && waitAfterUnblock > 0) {
+                    try {
+                        // Delay operation after unblocking
+                        // So, we can start node shutdown while this operation is still running.
+                        Thread.sleep(waitAfterUnblock);
+                    } catch (InterruptedException ex) {
+                        //
+                    }
+                }
+            }
+
+            /**
+             * Blocks an I/O operation on the blob fails and throws an exception when unblocked
+             */
+            private void blockExecutionAndFail(final String blobName) throws IOException {
+                logger.info("blocking I/O operation for file [{}] at path [{}]", blobName, path());
+                blockExecution();
+                throw new IOException("exception after block");
+            }
 
-            public MockBlobContainer(BlobContainer delegate) {
+            MockBlobContainer(BlobContainer delegate) {
                 super(delegate);
             }
 
@@ -313,14 +339,31 @@ public Map<String, BlobMetaData> listBlobsByPrefix(String blobNamePrefix) throws
 
             @Override
             public void move(String sourceBlob, String targetBlob) throws IOException {
-                maybeIOExceptionOrBlock(targetBlob);
-                super.move(sourceBlob, targetBlob);
+                if (blockOnWriteIndexFile && targetBlob.startsWith("index-")) {
+                    blockExecutionAndMaybeWait(targetBlob);
+                }
+                if (atomicMove) {
+                    // atomic move since this inherits from FsBlobContainer which provides atomic moves
+                    maybeIOExceptionOrBlock(targetBlob);
+                    super.move(sourceBlob, targetBlob);
+                } else {
+                    // simulate a non-atomic move, since many blob container implementations
+                    // will not have an atomic move, and we should be able to handle that
+                    maybeIOExceptionOrBlock(targetBlob);
+                    super.writeBlob(targetBlob, super.readBlob(sourceBlob), 0L);
+                    super.deleteBlob(sourceBlob);
+                }
             }
 
             @Override
             public void writeBlob(String blobName, InputStream inputStream, long blobSize) throws IOException {
                 maybeIOExceptionOrBlock(blobName);
                 super.writeBlob(blobName, inputStream, blobSize);
+                if (RandomizedContext.current().getRandom().nextBoolean()) {
+                    // for network based repositories, the blob may have been written but we may still
+                    // get an error with the client connection, so an IOException here simulates this
+                    maybeIOExceptionOrBlock(blobName);
+                }
             }
         }
     }
diff --git a/core/src/test/java/org/elasticsearch/tasks/TaskResultTests.java b/core/src/test/java/org/elasticsearch/tasks/TaskResultTests.java
index abf7bbf108cc1..e70c2b7119421 100644
--- a/core/src/test/java/org/elasticsearch/tasks/TaskResultTests.java
+++ b/core/src/test/java/org/elasticsearch/tasks/TaskResultTests.java
@@ -20,14 +20,13 @@
 package org.elasticsearch.tasks;
 
 import org.elasticsearch.client.Requests;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESTestCase;
@@ -67,8 +66,8 @@ public void testXContentRoundTrip() throws IOException {
         try (XContentBuilder builder = XContentBuilder.builder(randomFrom(XContentType.values()).xContent())) {
             result.toXContent(builder, ToXContent.EMPTY_PARAMS);
             try (XContentBuilder shuffled = shuffleXContent(builder);
-                    XContentParser parser = XContentHelper.createParser(shuffled.bytes())) {
-                read = TaskResult.PARSER.apply(parser, () -> ParseFieldMatcher.STRICT);
+                 XContentParser parser = createParser(shuffled)) {
+                read = TaskResult.PARSER.apply(parser, null);
             }
         } catch (IOException e) {
             throw new IOException("Error processing [" + result + "]", e);
@@ -76,25 +75,61 @@ public void testXContentRoundTrip() throws IOException {
         assertEquals(result, read);
     }
 
+    public void testTaskInfoIsForwardCompatible() throws IOException {
+        TaskInfo taskInfo = randomTaskInfo();
+        TaskInfo read;
+        try (XContentBuilder builder = XContentBuilder.builder(randomFrom(XContentType.values()).xContent())) {
+            builder.startObject();
+            taskInfo.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            builder.endObject();
+            try (XContentBuilder withExtraFields = addRandomUnknownFields(builder)) {
+                try (XContentBuilder shuffled = shuffleXContent(withExtraFields)) {
+                    try (XContentParser parser = createParser(shuffled)) {
+                        read = TaskInfo.PARSER.apply(parser, null);
+                    }
+                }
+            }
+        } catch (IOException e) {
+            throw new IOException("Error processing [" + taskInfo + "]", e);
+        }
+        assertEquals(taskInfo, read);
+    }
+
+    private XContentBuilder addRandomUnknownFields(XContentBuilder builder) throws IOException {
+        try (XContentParser parser = createParser(builder)) {
+            Map<String, Object> map = parser.mapOrdered();
+            int numberOfNewFields = randomIntBetween(2, 10);
+            for (int i = 0; i < numberOfNewFields; i++) {
+                if (randomBoolean()) {
+                    map.put("unknown_field" + i, randomAlphaOfLength(20));
+                } else {
+                    map.put("unknown_field" + i, Collections.singletonMap("inner", randomAlphaOfLength(20)));
+                }
+            }
+            XContentBuilder xContentBuilder = XContentFactory.contentBuilder(parser.contentType());
+            return xContentBuilder.map(map);
+        }
+    }
+
     private static TaskResult randomTaskResult() throws IOException {
         switch (between(0, 2)) {
-        case 0:
-            return new TaskResult(randomBoolean(), randomTaskInfo());
-        case 1:
-            return new TaskResult(randomTaskInfo(), new RuntimeException("error"));
-        case 2:
-            return new TaskResult(randomTaskInfo(), randomTaskResponse());
-        default:
-            throw new UnsupportedOperationException("Unsupported random TaskResult constructor");
+            case 0:
+                return new TaskResult(randomBoolean(), randomTaskInfo());
+            case 1:
+                return new TaskResult(randomTaskInfo(), new RuntimeException("error"));
+            case 2:
+                return new TaskResult(randomTaskInfo(), randomTaskResponse());
+            default:
+                throw new UnsupportedOperationException("Unsupported random TaskResult constructor");
         }
     }
 
     private static TaskInfo randomTaskInfo() throws IOException {
         TaskId taskId = randomTaskId();
-        String type = randomAsciiOfLength(5);
-        String action = randomAsciiOfLength(5);
+        String type = randomAlphaOfLength(5);
+        String action = randomAlphaOfLength(5);
         Task.Status status = randomBoolean() ? randomRawTaskStatus() : null;
-        String description = randomBoolean() ? randomAsciiOfLength(5) : null;
+        String description = randomBoolean() ? randomAlphaOfLength(5) : null;
         long startTime = randomLong();
         long runningTimeNanos = randomLong();
         boolean cancellable = randomBoolean();
@@ -103,7 +138,7 @@ private static TaskInfo randomTaskInfo() throws IOException {
     }
 
     private static TaskId randomTaskId() {
-        return new TaskId(randomAsciiOfLength(5), randomLong());
+        return new TaskId(randomAlphaOfLength(5), randomLong());
     }
 
     private static RawTaskStatus randomRawTaskStatus() throws IOException {
@@ -111,7 +146,7 @@ private static RawTaskStatus randomRawTaskStatus() throws IOException {
             builder.startObject();
             int fields = between(0, 10);
             for (int f = 0; f < fields; f++) {
-                builder.field(randomAsciiOfLength(5), randomAsciiOfLength(5));
+                builder.field(randomAlphaOfLength(5), randomAlphaOfLength(5));
             }
             builder.endObject();
             return new RawTaskStatus(builder.bytes());
@@ -122,7 +157,7 @@ private static ToXContent randomTaskResponse() {
         Map<String, String> result = new TreeMap<>();
         int fields = between(0, 10);
         for (int f = 0; f < fields; f++) {
-            result.put(randomAsciiOfLength(5), randomAsciiOfLength(5));
+            result.put(randomAlphaOfLength(5), randomAlphaOfLength(5));
         }
         return new ToXContent() {
             @Override
diff --git a/core/src/test/java/org/elasticsearch/test/AbstractSerializingTestCase.java b/core/src/test/java/org/elasticsearch/test/AbstractSerializingTestCase.java
new file mode 100644
index 0000000000000..392dcf1542a1c
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/test/AbstractSerializingTestCase.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.test;
+
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+
+public abstract class AbstractSerializingTestCase<T extends ToXContent & Writeable> extends AbstractWireSerializingTestCase<T> {
+
+    /**
+     * Generic test that creates new instance from the test instance and checks
+     * both for equality and asserts equality on the two instances.
+     */
+    public void testFromXContent() throws IOException {
+        for (int runs = 0; runs < NUMBER_OF_TEST_RUNS; runs++) {
+            T testInstance = createTestInstance();
+            XContentType xContentType = randomFrom(XContentType.values());
+            XContentBuilder builder = toXContent(testInstance, xContentType);
+            XContentBuilder shuffled = shuffleXContent(builder);
+            assertParsedInstance(xContentType, shuffled.bytes(), testInstance);
+            for (Map.Entry<String, T> alternateVersion : getAlternateVersions().entrySet()) {
+                String instanceAsString = alternateVersion.getKey();
+                assertParsedInstance(XContentType.JSON, new BytesArray(instanceAsString), alternateVersion.getValue());
+            }
+        }
+    }
+
+    private void assertParsedInstance(XContentType xContentType, BytesReference instanceAsBytes, T expectedInstance)
+            throws IOException {
+
+        XContentParser parser = createParser(XContentFactory.xContent(xContentType), instanceAsBytes);
+        T newInstance = parseInstance(parser);
+        assertNotSame(newInstance, expectedInstance);
+        assertEquals(expectedInstance, newInstance);
+        assertEquals(expectedInstance.hashCode(), newInstance.hashCode());
+    }
+
+    private T parseInstance(XContentParser parser) throws IOException {
+        T parsedInstance = doParseInstance(parser);
+        assertNull(parser.nextToken());
+        return parsedInstance;
+    }
+
+    /**
+     * Parses to a new instance using the provided {@link XContentParser}
+     */
+    protected abstract T doParseInstance(XContentParser parser) throws IOException;
+
+    /**
+     * Renders the provided instance in XContent
+     *
+     * @param instance
+     *            the instance to render
+     * @param contentType
+     *            the content type to render to
+     */
+    protected XContentBuilder toXContent(T instance, XContentType contentType)
+            throws IOException {
+        XContentBuilder builder = XContentFactory.contentBuilder(contentType);
+        if (randomBoolean()) {
+            builder.prettyPrint();
+        }
+        if (instance.isFragment()) {
+            builder.startObject();
+        }
+        instance.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        if (instance.isFragment()) {
+            builder.endObject();
+        }
+        return builder;
+    }
+
+    /**
+     * Returns alternate string representation of the instance that need to be
+     * tested as they are never used as output of the test instance. By default
+     * there are no alternate versions.
+     *
+     * These alternatives must be JSON strings.
+     */
+    protected Map<String, T> getAlternateVersions() {
+        return Collections.emptyMap();
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/test/AbstractWireSerializingTestCase.java b/core/src/test/java/org/elasticsearch/test/AbstractWireSerializingTestCase.java
new file mode 100644
index 0000000000000..bf65a7f4bdd4a
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/test/AbstractWireSerializingTestCase.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.test;
+
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteable;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+
+import java.io.IOException;
+import java.util.Collections;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public abstract class AbstractWireSerializingTestCase<T extends Writeable> extends ESTestCase {
+    protected static final int NUMBER_OF_TEST_RUNS = 20;
+
+    /**
+     * Creates a random test instance to use in the tests. This method will be
+     * called multiple times during test execution and should return a different
+     * random instance each time it is called.
+     */
+    protected abstract T createTestInstance();
+
+    /**
+     * Returns a {@link Reader} that can be used to de-serialize the instance
+     */
+    protected abstract Reader<T> instanceReader();
+
+    /**
+     * Tests that the equals and hashcode methods are consistent and copied
+     * versions of the instance have are equal.
+     */
+    public void testEqualsAndHashcode() throws IOException {
+        for (int runs = 0; runs < NUMBER_OF_TEST_RUNS; runs++) {
+            T firstInstance = createTestInstance();
+            assertFalse("instance is equal to null", firstInstance.equals(null));
+            assertFalse("instance is equal to incompatible type", firstInstance.equals(""));
+            assertEquals("instance is not equal to self", firstInstance, firstInstance);
+            assertThat("same instance's hashcode returns different values if called multiple times", firstInstance.hashCode(),
+                    equalTo(firstInstance.hashCode()));
+
+            T secondInstance = copyInstance(firstInstance);
+            assertEquals("instance is not equal to self", secondInstance, secondInstance);
+            assertEquals("instance is not equal to its copy", firstInstance, secondInstance);
+            assertEquals("equals is not symmetric", secondInstance, firstInstance);
+            assertThat("instance copy's hashcode is different from original hashcode", secondInstance.hashCode(),
+                    equalTo(firstInstance.hashCode()));
+
+            T thirdInstance = copyInstance(secondInstance);
+            assertEquals("instance is not equal to self", thirdInstance, thirdInstance);
+            assertEquals("instance is not equal to its copy", secondInstance, thirdInstance);
+            assertThat("instance copy's hashcode is different from original hashcode", secondInstance.hashCode(),
+                    equalTo(thirdInstance.hashCode()));
+            assertEquals("equals is not transitive", firstInstance, thirdInstance);
+            assertThat("instance copy's hashcode is different from original hashcode", firstInstance.hashCode(),
+                    equalTo(thirdInstance.hashCode()));
+            assertEquals("equals is not symmetric", thirdInstance, secondInstance);
+            assertEquals("equals is not symmetric", thirdInstance, firstInstance);
+        }
+    }
+
+    /**
+     * Test serialization and deserialization of the test instance.
+     */
+    public void testSerialization() throws IOException {
+        for (int runs = 0; runs < NUMBER_OF_TEST_RUNS; runs++) {
+            T testInstance = createTestInstance();
+            assertSerialization(testInstance);
+        }
+    }
+
+    /**
+     * Serialize the given instance and asserts that both are equal
+     */
+    protected T assertSerialization(T testInstance) throws IOException {
+        T deserializedInstance = copyInstance(testInstance);
+        assertEquals(testInstance, deserializedInstance);
+        assertEquals(testInstance.hashCode(), deserializedInstance.hashCode());
+        assertNotSame(testInstance, deserializedInstance);
+        return deserializedInstance;
+    }
+
+    private T copyInstance(T instance) throws IOException {
+        try (BytesStreamOutput output = new BytesStreamOutput()) {
+            instance.writeTo(output);
+            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(),
+                    getNamedWriteableRegistry())) {
+                return instanceReader().read(in);
+            }
+        }
+    }
+
+    /**
+     * Get the {@link NamedWriteableRegistry} to use when de-serializing the object.
+     * 
+     * Override this method if you need to register {@link NamedWriteable}s for the test object to de-serialize.
+     * 
+     * By default this will return a {@link NamedWriteableRegistry} with no registered {@link NamedWriteable}s
+     */
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return new NamedWriteableRegistry(Collections.emptyList());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/test/geo/RandomShapeGenerator.java b/core/src/test/java/org/elasticsearch/test/geo/RandomShapeGenerator.java
index 20c82e6f518f3..12f84f23c28f5 100644
--- a/core/src/test/java/org/elasticsearch/test/geo/RandomShapeGenerator.java
+++ b/core/src/test/java/org/elasticsearch/test/geo/RandomShapeGenerator.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.test.geo;
 
-import com.carrotsearch.randomizedtesting.generators.RandomInts;
+import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
 import com.vividsolutions.jts.algorithm.ConvexHull;
 import com.vividsolutions.jts.geom.Coordinate;
 import com.vividsolutions.jts.geom.Geometry;
@@ -56,12 +56,12 @@ public class RandomShapeGenerator extends RandomGeoGenerator {
     protected static final double xDIVISIBLE = 2;
     protected static boolean ST_VALIDATE = true;
 
-    public static enum ShapeType {
+    public enum ShapeType {
         POINT, MULTIPOINT, LINESTRING, MULTILINESTRING, POLYGON;
         private static final ShapeType[] types = values();
 
         public static ShapeType randomType(Random r) {
-            return types[RandomInts.randomIntBetween(r, 0, types.length - 1)];
+            return types[RandomNumbers.randomIntBetween(r, 0, types.length - 1)];
         }
     }
 
@@ -115,7 +115,7 @@ protected static GeometryCollectionBuilder createGeometryCollection(Random r, Po
             throws InvalidShapeException {
         if (numGeometries <= 0) {
             // cap geometry collection at 4 shapes (to save test time)
-            numGeometries = RandomInts.randomIntBetween(r, 2, 4);
+            numGeometries = RandomNumbers.randomIntBetween(r, 2, 4);
         }
 
         if (nearPoint == null) {
@@ -187,7 +187,7 @@ private static ShapeBuilder createShape(Random r, Point nearPoint, Rectangle wit
                 // for random testing having a maximum number of 10 points for a line string is more than sufficient
                 // if this number gets out of hand, the number of self intersections for a linestring can become
                 // (n^2-n)/2 and computing the relation intersection matrix will become NP-Hard
-                int numPoints = RandomInts.randomIntBetween(r, 3, 10);
+                int numPoints = RandomNumbers.randomIntBetween(r, 3, 10);
                 CoordinatesBuilder coordinatesBuilder = new CoordinatesBuilder();
                 for (int i=0; i<numPoints; ++i) {
                     p = xRandomPointIn(r, within);
@@ -197,12 +197,12 @@ private static ShapeBuilder createShape(Random r, Point nearPoint, Rectangle wit
                 return pcb;
             case MULTILINESTRING:
                 MultiLineStringBuilder mlsb = new MultiLineStringBuilder();
-                for (int i=0; i<RandomInts.randomIntBetween(r, 1, 10); ++i) {
+                for (int i=0; i<RandomNumbers.randomIntBetween(r, 1, 10); ++i) {
                     mlsb.linestring((LineStringBuilder) createShape(r, nearPoint, within, ShapeType.LINESTRING, false));
                 }
                 return mlsb;
             case POLYGON:
-                numPoints = RandomInts.randomIntBetween(r, 5, 25);
+                numPoints = RandomNumbers.randomIntBetween(r, 5, 25);
                 Coordinate[] coordinates = new Coordinate[numPoints];
                 for (int i=0; i<numPoints; ++i) {
                     p = (Point) createShape(r, nearPoint, within, ShapeType.POINT, false).build();
@@ -257,7 +257,7 @@ private static Rectangle xRandomRectangle(Random r, Point nearP, Rectangle bound
         if (nearP == null)
             nearP = xRandomPointIn(r, bounds);
 
-        if (small == true) {
+        if (small) {
             // between 3 and 6 degrees
             final double latRange = 3 * r.nextDouble() + 3;
             final double lonRange = 3 * r.nextDouble() + 3;
@@ -298,7 +298,7 @@ public static Rectangle xRandomRectangle(Random r, Point nearP, boolean small) {
     }
 
     private static boolean rarely(Random r) {
-        return RandomInts.randomInt(r, 100) >= 90;
+        return r.nextInt(100) >= 90;
     }
 
     private static Range xRandomRange(Random r, double near, Range bounds) {
diff --git a/core/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java b/core/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java
index 7fc1490de73f6..6dd4fa384e99b 100644
--- a/core/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java
+++ b/core/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java
@@ -21,6 +21,7 @@
 
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.search.aggregations.Aggregation;
 import org.elasticsearch.search.aggregations.bucket.significant.SignificantTerms;
 import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
@@ -80,7 +81,7 @@ public static void index01Docs(String type, String settings, ESIntegTestCase tes
         if (type.equals("text")) {
             textMappings += ",fielddata=true";
         }
-        assertAcked(testCase.prepareCreate(INDEX_NAME).setSettings(settings)
+        assertAcked(testCase.prepareCreate(INDEX_NAME).setSettings(settings, XContentType.JSON)
                 .addMapping("doc", "text", textMappings, CLASS_FIELD, "type=keyword"));
         String[] gb = {"0", "1"};
         List<IndexRequestBuilder> indexRequestBuilderList = new ArrayList<>();
diff --git a/core/src/test/java/org/elasticsearch/threadpool/ScalingThreadPoolTests.java b/core/src/test/java/org/elasticsearch/threadpool/ScalingThreadPoolTests.java
index d065abb884c6b..5e7227052d361 100644
--- a/core/src/test/java/org/elasticsearch/threadpool/ScalingThreadPoolTests.java
+++ b/core/src/test/java/org/elasticsearch/threadpool/ScalingThreadPoolTests.java
@@ -29,10 +29,10 @@
 import java.util.concurrent.Executor;
 import java.util.concurrent.TimeUnit;
 import java.util.function.BiConsumer;
+import java.util.function.Function;
 
 import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.hasToString;
 
 public class ScalingThreadPoolTests extends ESThreadPoolTestCase {
 
@@ -95,13 +95,8 @@ public void testScalingThreadPoolConfiguration() throws InterruptedException {
         });
     }
 
-    @FunctionalInterface
-    private interface SizeFunction {
-        int size(int numberOfProcessors);
-    }
-
     private int expectedSize(final String threadPoolName, final int numberOfProcessors) {
-        final Map<String, SizeFunction> sizes = new HashMap<>();
+        final Map<String, Function<Integer, Integer>> sizes = new HashMap<>();
         sizes.put(ThreadPool.Names.GENERIC, n -> ThreadPool.boundedBy(4 * n, 128, 512));
         sizes.put(ThreadPool.Names.MANAGEMENT, n -> 5);
         sizes.put(ThreadPool.Names.FLUSH, ThreadPool::halfNumberOfProcessorsMaxFive);
@@ -110,7 +105,7 @@ private int expectedSize(final String threadPoolName, final int numberOfProcesso
         sizes.put(ThreadPool.Names.SNAPSHOT, ThreadPool::halfNumberOfProcessorsMaxFive);
         sizes.put(ThreadPool.Names.FETCH_SHARD_STARTED, ThreadPool::twiceNumberOfProcessors);
         sizes.put(ThreadPool.Names.FETCH_SHARD_STORE, ThreadPool::twiceNumberOfProcessors);
-        return sizes.get(threadPoolName).size(numberOfProcessors);
+        return sizes.get(threadPoolName).apply(numberOfProcessors);
     }
 
     public void testScalingThreadPoolIsBounded() throws InterruptedException {
@@ -198,13 +193,4 @@ public void runScalingThreadPoolTest(
             terminateThreadPoolIfNeeded(threadPool);
         }
     }
-
-    private static Settings settings(final String setting, final int value) {
-        return settings(setting, Integer.toString(value));
-    }
-
-    private static Settings settings(final String setting, final String value) {
-        return Settings.builder().put(setting, value).build();
-    }
-
 }
diff --git a/core/src/test/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java b/core/src/test/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java
index 974929dddf239..83660e0f4e708 100644
--- a/core/src/test/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java
+++ b/core/src/test/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java
@@ -34,7 +34,6 @@
 import org.elasticsearch.test.ESIntegTestCase.Scope;
 import org.elasticsearch.test.InternalTestCluster;
 import org.elasticsearch.test.hamcrest.RegexMatcher;
-import org.elasticsearch.tribe.TribeIT;
 
 import java.io.IOException;
 import java.lang.management.ManagementFactory;
@@ -76,11 +75,11 @@ public void testThreadNames() throws Exception {
             builders[i] = client().prepareIndex("idx", "type").setSource(jsonBuilder()
                     .startObject()
                     .field("str_value", "s" + i)
-                    .field("str_values", new String[]{"s" + (i * 2), "s" + (i * 2 + 1)})
+                    .array("str_values", new String[]{"s" + (i * 2), "s" + (i * 2 + 1)})
                     .field("l_value", i)
-                    .field("l_values", new int[]{i * 2, i * 2 + 1})
+                    .array("l_values", new int[]{i * 2, i * 2 + 1})
                     .field("d_value", i)
-                    .field("d_values", new double[]{i * 2, i * 2 + 1})
+                    .array("d_values", new double[]{i * 2, i * 2 + 1})
                     .endObject());
         }
         indexRandom(true, builders);
@@ -109,7 +108,7 @@ public void testThreadNames() throws Exception {
             String nodePrefix = "(" + Pattern.quote(InternalTestCluster.TRANSPORT_CLIENT_PREFIX) + ")?(" +
                     Pattern.quote(ESIntegTestCase.SUITE_CLUSTER_NODE_PREFIX) + "|" +
                     Pattern.quote(ESIntegTestCase.TEST_CLUSTER_NODE_PREFIX) + "|" +
-                    Pattern.quote(TribeIT.SECOND_CLUSTER_NODE_PREFIX) + ")";
+                    Pattern.quote("node_tribe2") + ")";
             assertThat(threadName, RegexMatcher.matches("\\[" + nodePrefix + "\\d+\\]"));
         }
     }
@@ -138,7 +137,7 @@ private Map<String, Object> getPoolSettingsThroughJson(ThreadPoolInfo info, Stri
         builder.endObject();
         builder.close();
         Map<String, Object> poolsMap;
-        try (XContentParser parser = JsonXContent.jsonXContent.createParser(builder.string())) {
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, builder.string())) {
             poolsMap = parser.map();
         }
         return (Map<String, Object>) ((Map<String, Object>) poolsMap.get("thread_pool")).get(poolName);
diff --git a/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolSerializationTests.java b/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolSerializationTests.java
index bf0f5f6e60666..33047b2d12a6a 100644
--- a/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolSerializationTests.java
+++ b/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolSerializationTests.java
@@ -19,7 +19,6 @@
 package org.elasticsearch.threadpool;
 
 import org.elasticsearch.Version;
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
@@ -27,8 +26,7 @@
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
@@ -80,11 +78,7 @@ public void testThatToXContentWritesOutUnboundedCorrectly() throws Exception {
         info.toXContent(builder, ToXContent.EMPTY_PARAMS);
         builder.endObject();
 
-        BytesReference bytesReference = builder.bytes();
-        Map<String, Object> map;
-        try (XContentParser parser = XContentFactory.xContent(bytesReference).createParser(bytesReference)) {
-            map = parser.map();
-        }
+        Map<String, Object> map = XContentHelper.convertToMap(builder.bytes(), false, builder.contentType()).v2();
         assertThat(map, hasKey("foo"));
         map = (Map<String, Object>) map.get("foo");
         assertThat(map, hasKey("queue_size"));
@@ -106,11 +100,7 @@ public void testThatToXContentWritesInteger() throws Exception {
         info.toXContent(builder, ToXContent.EMPTY_PARAMS);
         builder.endObject();
 
-        BytesReference bytesReference = builder.bytes();
-        Map<String, Object> map;
-        try (XContentParser parser = XContentFactory.xContent(bytesReference).createParser(bytesReference)) {
-            map = parser.map();
-        }
+        Map<String, Object> map = XContentHelper.convertToMap(builder.bytes(), false, builder.contentType()).v2();
         assertThat(map, hasKey("foo"));
         map = (Map<String, Object>) map.get("foo");
         assertThat(map, hasKey("queue_size"));
diff --git a/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolStatsTests.java b/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolStatsTests.java
index a7312f594069c..bcfca78bb6d57 100644
--- a/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolStatsTests.java
+++ b/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolStatsTests.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
@@ -78,7 +79,7 @@ public void testThreadPoolStatsToXContent() throws IOException {
                 builder.endObject();
             }
 
-            try (XContentParser parser = XContentType.JSON.xContent().createParser(os.bytes())) {
+            try (XContentParser parser = createParser(JsonXContent.jsonXContent, os.bytes())) {
                 XContentParser.Token token = parser.currentToken();
                 assertNull(token);
 
diff --git a/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolTests.java b/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolTests.java
index daad1a51a085b..b277de64a9d9f 100644
--- a/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolTests.java
+++ b/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolTests.java
@@ -46,4 +46,17 @@ public void testBoundedByBetweenMinAndMax() {
         assertThat(ThreadPool.boundedBy(value, min, max), equalTo(value));
     }
 
+    public void testAbsoluteTime() throws Exception {
+        TestThreadPool threadPool = new TestThreadPool("test");
+        try {
+            long currentTime = System.currentTimeMillis();
+            long gotTime = threadPool.absoluteTimeInMillis();
+            long delta = Math.abs(gotTime - currentTime);
+            assertTrue("thread pool cached absolute time " + gotTime + " is too far from real current time " + currentTime,
+                delta < 10000); // the delta can be large, we just care it is the same order of magnitude
+        } finally {
+            threadPool.shutdown();
+            threadPool.close();
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/timestamp/SimpleTimestampIT.java b/core/src/test/java/org/elasticsearch/timestamp/SimpleTimestampIT.java
index f65e3a742f642..2fb6333587379 100644
--- a/core/src/test/java/org/elasticsearch/timestamp/SimpleTimestampIT.java
+++ b/core/src/test/java/org/elasticsearch/timestamp/SimpleTimestampIT.java
@@ -69,33 +69,33 @@ public void testSimpleTimestamp() throws Exception {
         long now2 = System.currentTimeMillis();
 
         // non realtime get (stored)
-        GetResponse getResponse = client().prepareGet("test", "type1", "1").setFields("_timestamp").setRealtime(randomBoolean()).execute().actionGet();
+        GetResponse getResponse = client().prepareGet("test", "type1", "1").setStoredFields("_timestamp").setRealtime(randomBoolean()).execute().actionGet();
         long timestamp = ((Number) getResponse.getField("_timestamp").getValue()).longValue();
         assertThat(timestamp, greaterThanOrEqualTo(now1));
         assertThat(timestamp, lessThanOrEqualTo(now2));
         // verify its the same timestamp when going the replica
-        getResponse = client().prepareGet("test", "type1", "1").setFields("_timestamp").setRealtime(randomBoolean()).execute().actionGet();
+        getResponse = client().prepareGet("test", "type1", "1").setStoredFields("_timestamp").setRealtime(randomBoolean()).execute().actionGet();
         assertThat(((Number) getResponse.getField("_timestamp").getValue()).longValue(), equalTo(timestamp));
 
         logger.info("--> check with custom timestamp (numeric)");
         client().prepareIndex("test", "type1", "1").setSource("field1", "value1").setTimestamp("10").setRefreshPolicy(IMMEDIATE).get();
 
-        getResponse = client().prepareGet("test", "type1", "1").setFields("_timestamp").setRealtime(false).execute().actionGet();
+        getResponse = client().prepareGet("test", "type1", "1").setStoredFields("_timestamp").setRealtime(false).execute().actionGet();
         timestamp = ((Number) getResponse.getField("_timestamp").getValue()).longValue();
         assertThat(timestamp, equalTo(10L));
         // verify its the same timestamp when going the replica
-        getResponse = client().prepareGet("test", "type1", "1").setFields("_timestamp").setRealtime(false).execute().actionGet();
+        getResponse = client().prepareGet("test", "type1", "1").setStoredFields("_timestamp").setRealtime(false).execute().actionGet();
         assertThat(((Number) getResponse.getField("_timestamp").getValue()).longValue(), equalTo(timestamp));
 
         logger.info("--> check with custom timestamp (string)");
         client().prepareIndex("test", "type1", "1").setSource("field1", "value1").setTimestamp("1970-01-01T00:00:00.020")
                 .setRefreshPolicy(IMMEDIATE).get();
 
-        getResponse = client().prepareGet("test", "type1", "1").setFields("_timestamp").setRealtime(false).execute().actionGet();
+        getResponse = client().prepareGet("test", "type1", "1").setStoredFields("_timestamp").setRealtime(false).execute().actionGet();
         timestamp = ((Number) getResponse.getField("_timestamp").getValue()).longValue();
         assertThat(timestamp, equalTo(20L));
         // verify its the same timestamp when going the replica
-        getResponse = client().prepareGet("test", "type1", "1").setFields("_timestamp").setRealtime(false).execute().actionGet();
+        getResponse = client().prepareGet("test", "type1", "1").setStoredFields("_timestamp").setRealtime(false).execute().actionGet();
         assertThat(((Number) getResponse.getField("_timestamp").getValue()).longValue(), equalTo(timestamp));
     }
 
diff --git a/core/src/test/java/org/elasticsearch/transport/ConnectionProfileTests.java b/core/src/test/java/org/elasticsearch/transport/ConnectionProfileTests.java
new file mode 100644
index 0000000000000..b18b57e371782
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/transport/ConnectionProfileTests.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.transport;
+
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.test.ESTestCase;
+import org.hamcrest.Matchers;
+
+import java.util.EnumSet;
+
+public class ConnectionProfileTests extends ESTestCase {
+
+    public void testBuildConnectionProfile() {
+        ConnectionProfile.Builder builder = new ConnectionProfile.Builder();
+        TimeValue connectTimeout = TimeValue.timeValueMillis(randomIntBetween(1, 10));
+        TimeValue handshaketTimeout = TimeValue.timeValueMillis(randomIntBetween(1, 10));
+        final boolean setConnectTimeout = randomBoolean();
+        if (setConnectTimeout) {
+            builder.setConnectTimeout(connectTimeout);
+        }
+        final boolean setHandshakeTimeout = randomBoolean();
+        if (setHandshakeTimeout) {
+            builder.setHandshakeTimeout(handshaketTimeout);
+        }
+        builder.addConnections(1, TransportRequestOptions.Type.BULK);
+        builder.addConnections(2, TransportRequestOptions.Type.STATE, TransportRequestOptions.Type.RECOVERY);
+        builder.addConnections(3, TransportRequestOptions.Type.PING);
+        IllegalStateException illegalStateException = expectThrows(IllegalStateException.class, builder::build);
+        assertEquals("not all types are added for this connection profile - missing types: [REG]", illegalStateException.getMessage());
+
+        IllegalArgumentException illegalArgumentException = expectThrows(IllegalArgumentException.class,
+            () -> builder.addConnections(4, TransportRequestOptions.Type.REG, TransportRequestOptions.Type.PING));
+        assertEquals("type [PING] is already registered", illegalArgumentException.getMessage());
+        builder.addConnections(4, TransportRequestOptions.Type.REG);
+        ConnectionProfile build = builder.build();
+        if (randomBoolean()) {
+            build = new ConnectionProfile.Builder(build).build();
+        }
+        assertEquals(10, build.getNumConnections());
+        if (setConnectTimeout) {
+            assertEquals(connectTimeout, build.getConnectTimeout());
+        } else {
+            assertNull(build.getConnectTimeout());
+        }
+
+        if (setHandshakeTimeout) {
+            assertEquals(handshaketTimeout, build.getHandshakeTimeout());
+        } else {
+            assertNull(build.getHandshakeTimeout());
+        }
+
+        Integer[] array = new Integer[10];
+        for (int i = 0; i < array.length; i++) {
+            array[i] = i;
+        }
+        final int numIters = randomIntBetween(5, 10);
+        assertEquals(4, build.getHandles().size());
+        assertEquals(0, build.getHandles().get(0).offset);
+        assertEquals(1, build.getHandles().get(0).length);
+        assertEquals(EnumSet.of(TransportRequestOptions.Type.BULK), build.getHandles().get(0).getTypes());
+        Integer channel = build.getHandles().get(0).getChannel(array);
+        for (int i = 0; i < numIters; i++) {
+            assertEquals(0, channel.intValue());
+        }
+
+        assertEquals(1, build.getHandles().get(1).offset);
+        assertEquals(2, build.getHandles().get(1).length);
+        assertEquals(EnumSet.of(TransportRequestOptions.Type.STATE, TransportRequestOptions.Type.RECOVERY),
+            build.getHandles().get(1).getTypes());
+        channel = build.getHandles().get(1).getChannel(array);
+        for (int i = 0; i < numIters; i++) {
+            assertThat(channel, Matchers.anyOf(Matchers.is(1), Matchers.is(2)));
+        }
+
+        assertEquals(3, build.getHandles().get(2).offset);
+        assertEquals(3, build.getHandles().get(2).length);
+        assertEquals(EnumSet.of(TransportRequestOptions.Type.PING), build.getHandles().get(2).getTypes());
+        channel = build.getHandles().get(2).getChannel(array);
+        for (int i = 0; i < numIters; i++) {
+            assertThat(channel, Matchers.anyOf(Matchers.is(3), Matchers.is(4), Matchers.is(5)));
+        }
+
+        assertEquals(6, build.getHandles().get(3).offset);
+        assertEquals(4, build.getHandles().get(3).length);
+        assertEquals(EnumSet.of(TransportRequestOptions.Type.REG), build.getHandles().get(3).getTypes());
+        channel = build.getHandles().get(3).getChannel(array);
+        for (int i = 0; i < numIters; i++) {
+            assertThat(channel, Matchers.anyOf(Matchers.is(6), Matchers.is(7), Matchers.is(8), Matchers.is(9)));
+        }
+
+        assertEquals(3, build.getNumConnectionsPerType(TransportRequestOptions.Type.PING));
+        assertEquals(4, build.getNumConnectionsPerType(TransportRequestOptions.Type.REG));
+        assertEquals(2, build.getNumConnectionsPerType(TransportRequestOptions.Type.STATE));
+        assertEquals(2, build.getNumConnectionsPerType(TransportRequestOptions.Type.RECOVERY));
+        assertEquals(1, build.getNumConnectionsPerType(TransportRequestOptions.Type.BULK));
+    }
+
+    public void testNoChannels() {
+        ConnectionProfile.Builder builder = new ConnectionProfile.Builder();
+        builder.addConnections(1, TransportRequestOptions.Type.BULK,
+            TransportRequestOptions.Type.STATE,
+            TransportRequestOptions.Type.RECOVERY,
+            TransportRequestOptions.Type.REG);
+        builder.addConnections(0, TransportRequestOptions.Type.PING);
+        ConnectionProfile build = builder.build();
+        Integer[] array = new Integer[]{Integer.valueOf(0)};
+        assertEquals(Integer.valueOf(0), build.getHandles().get(0).getChannel(array));
+        expectThrows(IllegalStateException.class, () -> build.getHandles().get(1).getChannel(array));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java b/core/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java
new file mode 100644
index 0000000000000..599f1dccda0aa
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java
@@ -0,0 +1,962 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.transport;
+
+import org.apache.lucene.store.AlreadyClosedException;
+import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.Build;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.LatchedActionListener;
+import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
+import org.elasticsearch.action.admin.cluster.node.info.NodesInfoAction;
+import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest;
+import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
+import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsAction;
+import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsGroup;
+import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsRequest;
+import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsResponse;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.BoundTransportAddress;
+import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.elasticsearch.common.transport.LocalTransportAddress;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.CancellableThreads;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.http.HttpInfo;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.transport.MockTransportService;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.hamcrest.Matchers;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.BrokenBarrierException;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Function;
+
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.emptySet;
+
+public class RemoteClusterConnectionTests extends ESTestCase {
+
+    private final ThreadPool threadPool = new TestThreadPool(getClass().getName());
+
+    @Override
+    public void tearDown() throws Exception {
+        super.tearDown();
+        ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS);
+    }
+
+    private MockTransportService startTransport(String id, List<DiscoveryNode> knownNodes, Version version) {
+        return startTransport(id, knownNodes, version, threadPool);
+    }
+
+    public static MockTransportService startTransport(String id, List<DiscoveryNode> knownNodes, Version version, ThreadPool threadPool) {
+        return startTransport(id, knownNodes, version, threadPool, Settings.EMPTY);
+    }
+
+    public static MockTransportService startTransport(
+            final String id,
+            final List<DiscoveryNode> knownNodes,
+            final Version version,
+            final ThreadPool threadPool,
+            final Settings settings) {
+        boolean success = false;
+        final Settings s = Settings.builder().put(settings).put("node.name", id).build();
+        ClusterName clusterName = ClusterName.CLUSTER_NAME_SETTING.get(s);
+        MockTransportService newService = MockTransportService.createNewService(s, version, threadPool, null);
+        try {
+            newService.registerRequestHandler(ClusterSearchShardsAction.NAME, ClusterSearchShardsRequest::new, ThreadPool.Names.SAME,
+                    (request, channel) -> {
+                        channel.sendResponse(new ClusterSearchShardsResponse(new ClusterSearchShardsGroup[0],
+                                knownNodes.toArray(new DiscoveryNode[0]), Collections.emptyMap()));
+                    });
+            newService.registerRequestHandler(ClusterStateAction.NAME, ClusterStateRequest::new, ThreadPool.Names.SAME,
+                    (request, channel) -> {
+                        DiscoveryNodes.Builder builder = DiscoveryNodes.builder();
+                        for (DiscoveryNode node : knownNodes) {
+                            builder.add(node);
+                        }
+                        ClusterState build = ClusterState.builder(clusterName).nodes(builder.build()).build();
+                        channel.sendResponse(new ClusterStateResponse(clusterName, build));
+                    });
+            newService.start();
+            newService.acceptIncomingRequests();
+            success = true;
+            return newService;
+        } finally {
+            if (success == false) {
+                newService.close();
+            }
+        }
+    }
+
+    public void testDiscoverSingleNode() throws Exception {
+        List<DiscoveryNode> knownNodes = new CopyOnWriteArrayList<>();
+        try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT);
+             MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT)) {
+            DiscoveryNode seedNode = seedTransport.getLocalDiscoNode();
+            DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode();
+            knownNodes.add(seedTransport.getLocalDiscoNode());
+            knownNodes.add(discoverableTransport.getLocalDiscoNode());
+            Collections.shuffle(knownNodes, random());
+
+            try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) {
+                service.start();
+                service.acceptIncomingRequests();
+                try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
+                    Arrays.asList(seedNode), service, Integer.MAX_VALUE, n -> true)) {
+                    updateSeedNodes(connection, Arrays.asList(seedNode));
+                    assertTrue(service.nodeConnected(seedNode));
+                    assertTrue(service.nodeConnected(discoverableNode));
+                    assertTrue(connection.assertNoRunningConnections());
+                }
+            }
+        }
+    }
+
+    public void testDiscoverSingleNodeWithIncompatibleSeed() throws Exception {
+        List<DiscoveryNode> knownNodes = new CopyOnWriteArrayList<>();
+        try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT);
+             MockTransportService incomaptibleTransport = startTransport("incompat_seed_node", knownNodes, Version.fromString("2.0.0"));
+             MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT)) {
+            DiscoveryNode seedNode = seedTransport.getLocalDiscoNode();
+            DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode();
+            DiscoveryNode incompatibleSeedNode = incomaptibleTransport.getLocalDiscoNode();
+            knownNodes.add(seedTransport.getLocalDiscoNode());
+            knownNodes.add(discoverableTransport.getLocalDiscoNode());
+            knownNodes.add(incomaptibleTransport.getLocalDiscoNode());
+            Collections.shuffle(knownNodes, random());
+            List<DiscoveryNode> seedNodes = Arrays.asList(incompatibleSeedNode, seedNode);
+            Collections.shuffle(seedNodes, random());
+
+            try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) {
+                service.start();
+                service.acceptIncomingRequests();
+                try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
+                    seedNodes, service, Integer.MAX_VALUE, n -> true)) {
+                    updateSeedNodes(connection, seedNodes);
+                    assertTrue(service.nodeConnected(seedNode));
+                    assertTrue(service.nodeConnected(discoverableNode));
+                    assertFalse(service.nodeConnected(incompatibleSeedNode));
+                    assertTrue(connection.assertNoRunningConnections());
+                }
+            }
+        }
+    }
+
+    public void testNodeDisconnected() throws Exception {
+        List<DiscoveryNode> knownNodes = new CopyOnWriteArrayList<>();
+        try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT);
+             MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT);
+             MockTransportService spareTransport = startTransport("spare_node", knownNodes, Version.CURRENT)) {
+            DiscoveryNode seedNode = seedTransport.getLocalDiscoNode();
+            DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode();
+            DiscoveryNode spareNode = spareTransport.getLocalDiscoNode();
+            knownNodes.add(seedTransport.getLocalDiscoNode());
+            knownNodes.add(discoverableTransport.getLocalDiscoNode());
+            Collections.shuffle(knownNodes, random());
+
+            try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) {
+                service.start();
+                service.acceptIncomingRequests();
+                try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
+                    Arrays.asList(seedNode), service, Integer.MAX_VALUE, n -> true)) {
+                    updateSeedNodes(connection, Arrays.asList(seedNode));
+                    assertTrue(service.nodeConnected(seedNode));
+                    assertTrue(service.nodeConnected(discoverableNode));
+                    assertFalse(service.nodeConnected(spareNode));
+                    knownNodes.add(spareNode);
+                    CountDownLatch latchDisconnect = new CountDownLatch(1);
+                    CountDownLatch latchConnected = new CountDownLatch(1);
+                    service.addConnectionListener(new TransportConnectionListener() {
+                        @Override
+                        public void onNodeDisconnected(DiscoveryNode node) {
+                            if (node.equals(discoverableNode)) {
+                                latchDisconnect.countDown();
+                            }
+                        }
+
+                        @Override
+                        public void onNodeConnected(DiscoveryNode node) {
+                            if (node.equals(spareNode)) {
+                                latchConnected.countDown();
+                            }
+                        }
+                    });
+
+                    discoverableTransport.close();
+                    // now make sure we try to connect again to other nodes once we got disconnected
+                    assertTrue(latchDisconnect.await(10, TimeUnit.SECONDS));
+                    assertTrue(latchConnected.await(10, TimeUnit.SECONDS));
+                    assertTrue(service.nodeConnected(spareNode));
+                }
+            }
+        }
+    }
+
+    public void testFilterDiscoveredNodes() throws Exception {
+        List<DiscoveryNode> knownNodes = new CopyOnWriteArrayList<>();
+        try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT);
+             MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT)) {
+            DiscoveryNode seedNode = seedTransport.getLocalDiscoNode();
+            DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode();
+            knownNodes.add(seedTransport.getLocalDiscoNode());
+            knownNodes.add(discoverableTransport.getLocalDiscoNode());
+            DiscoveryNode rejectedNode = randomBoolean() ? seedNode : discoverableNode;
+            Collections.shuffle(knownNodes, random());
+
+            try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) {
+                service.start();
+                service.acceptIncomingRequests();
+                try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
+                    Arrays.asList(seedNode), service, Integer.MAX_VALUE, n -> n.equals(rejectedNode) == false)) {
+                    updateSeedNodes(connection, Arrays.asList(seedNode));
+                    if (rejectedNode.equals(seedNode)) {
+                        assertFalse(service.nodeConnected(seedNode));
+                        assertTrue(service.nodeConnected(discoverableNode));
+                    } else {
+                        assertTrue(service.nodeConnected(seedNode));
+                        assertFalse(service.nodeConnected(discoverableNode));
+                    }
+                    assertTrue(connection.assertNoRunningConnections());
+                }
+            }
+        }
+    }
+
+    private void updateSeedNodes(RemoteClusterConnection connection, List<DiscoveryNode> seedNodes) throws Exception {
+        CountDownLatch latch = new CountDownLatch(1);
+        AtomicReference<Exception> exceptionAtomicReference = new AtomicReference<>();
+        ActionListener<Void> listener = ActionListener.wrap(x -> latch.countDown(), x -> {
+            exceptionAtomicReference.set(x);
+            latch.countDown();
+        });
+        connection.updateSeedNodes(seedNodes, listener);
+        latch.await();
+        if (exceptionAtomicReference.get() != null) {
+            throw exceptionAtomicReference.get();
+        }
+    }
+
+    public void testConnectWithIncompatibleTransports() throws Exception {
+        List<DiscoveryNode> knownNodes = new CopyOnWriteArrayList<>();
+        try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.fromString("2.0.0"))) {
+            DiscoveryNode seedNode = seedTransport.getLocalDiscoNode();
+            knownNodes.add(seedTransport.getLocalDiscoNode());
+
+            try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) {
+                service.start();
+                service.acceptIncomingRequests();
+                try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
+                    Arrays.asList(seedNode), service, Integer.MAX_VALUE, n -> true)) {
+                    expectThrows(Exception.class, () -> updateSeedNodes(connection, Arrays.asList(seedNode)));
+                    assertFalse(service.nodeConnected(seedNode));
+                    assertTrue(connection.assertNoRunningConnections());
+                }
+            }
+        }
+    }
+
+    @SuppressForbidden(reason = "calls getLocalHost here but it's fine in this case")
+    public void testSlowNodeCanBeCanceled() throws IOException, InterruptedException {
+        try (ServerSocket socket = new ServerSocket()) {
+            socket.bind(new InetSocketAddress(InetAddress.getLocalHost(), 0), 1);
+            socket.setReuseAddress(true);
+            DiscoveryNode seedNode = new DiscoveryNode("TEST", new InetSocketTransportAddress(socket.getInetAddress(),
+                socket.getLocalPort()), emptyMap(),
+                emptySet(), Version.CURRENT);
+            CountDownLatch acceptedLatch = new CountDownLatch(1);
+            CountDownLatch closeRemote = new CountDownLatch(1);
+            Thread t = new Thread() {
+                @Override
+                public void run() {
+                    try (Socket accept = socket.accept()) {
+                        acceptedLatch.countDown();
+                        closeRemote.await();
+                    } catch (IOException e) {
+                        // that's fine we might close
+                    } catch (InterruptedException e) {
+                        Thread.currentThread().interrupt();
+                    }
+                }
+            };
+            t.start();
+
+            try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) {
+                service.start();
+                service.acceptIncomingRequests();
+                CountDownLatch listenerCalled = new CountDownLatch(1);
+                AtomicReference<Exception> exceptionReference = new AtomicReference<>();
+                try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
+                    Arrays.asList(seedNode), service, Integer.MAX_VALUE, n -> true)) {
+                    ActionListener<Void> listener = ActionListener.wrap(x -> {
+                        listenerCalled.countDown();
+                        fail("expected exception");
+                    }, x -> {
+                        exceptionReference.set(x);
+                        listenerCalled.countDown();
+                    });
+                    connection.updateSeedNodes(Arrays.asList(seedNode), listener);
+                    acceptedLatch.await();
+                    connection.close(); // now close it, this should trigger an interrupt on the socket and we can move on
+                    assertTrue(connection.assertNoRunningConnections());
+                }
+                closeRemote.countDown();
+                listenerCalled.await();
+                assertNotNull(exceptionReference.get());
+                expectThrows(CancellableThreads.ExecutionCancelledException.class, () -> {throw exceptionReference.get();});
+
+            }
+        }
+    }
+
+    public void testFetchShards() throws Exception {
+        List<DiscoveryNode> knownNodes = new CopyOnWriteArrayList<>();
+        try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT);
+             MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT)) {
+            DiscoveryNode seedNode = seedTransport.getLocalDiscoNode();
+            knownNodes.add(seedTransport.getLocalDiscoNode());
+            knownNodes.add(discoverableTransport.getLocalDiscoNode());
+            Collections.shuffle(knownNodes, random());
+            try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) {
+                service.start();
+                service.acceptIncomingRequests();
+                try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
+                    Arrays.asList(seedNode), service, Integer.MAX_VALUE, n -> true)) {
+                    if (randomBoolean()) {
+                        updateSeedNodes(connection, Arrays.asList(seedNode));
+                    }
+
+                    SearchRequest request = new SearchRequest("test-index");
+                    CountDownLatch responseLatch = new CountDownLatch(1);
+                    AtomicReference<ClusterSearchShardsResponse> reference = new AtomicReference<>();
+                    AtomicReference<Exception> failReference = new AtomicReference<>();
+                    ActionListener<ClusterSearchShardsResponse> shardsListener = ActionListener.wrap(
+                        x -> {
+                            reference.set(x);
+                            responseLatch.countDown();
+                        },
+                        x -> {
+                            failReference.set(x);
+                            responseLatch.countDown();
+                        });
+                    ClusterSearchShardsRequest searchShardsRequest = new ClusterSearchShardsRequest(new String[]{"test-index"})
+                        .indicesOptions(request.indicesOptions()).local(true).preference(request.preference())
+                        .routing(request.routing());
+                    connection.fetchSearchShards(searchShardsRequest, shardsListener);
+                    responseLatch.await();
+                    assertNull(failReference.get());
+                    assertNotNull(reference.get());
+                    ClusterSearchShardsResponse clusterSearchShardsResponse = reference.get();
+                    assertEquals(knownNodes, Arrays.asList(clusterSearchShardsResponse.getNodes()));
+                    assertTrue(connection.assertNoRunningConnections());
+                }
+            }
+        }
+    }
+
+    public void testTriggerUpdatesConcurrently() throws IOException, InterruptedException {
+        List<DiscoveryNode> knownNodes = new CopyOnWriteArrayList<>();
+        try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT);
+             MockTransportService seedTransport1 = startTransport("seed_node_1", knownNodes, Version.CURRENT);
+             MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT)) {
+            DiscoveryNode seedNode = seedTransport.getLocalDiscoNode();
+            DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode();
+            DiscoveryNode seedNode1 = seedTransport1.getLocalDiscoNode();
+            knownNodes.add(seedTransport.getLocalDiscoNode());
+            knownNodes.add(discoverableTransport.getLocalDiscoNode());
+            knownNodes.add(seedTransport1.getLocalDiscoNode());
+            Collections.shuffle(knownNodes, random());
+            List<DiscoveryNode> seedNodes = Arrays.asList(seedNode1, seedNode);
+            Collections.shuffle(seedNodes, random());
+
+            try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) {
+                service.start();
+                service.acceptIncomingRequests();
+                try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
+                    seedNodes, service, Integer.MAX_VALUE, n -> true)) {
+                    int numThreads = randomIntBetween(4, 10);
+                    Thread[] threads = new Thread[numThreads];
+                    CyclicBarrier barrier = new CyclicBarrier(numThreads);
+                    for (int i = 0; i < threads.length; i++) {
+                        final int numConnectionAttempts = randomIntBetween(10, 200);
+                        threads[i] = new Thread() {
+                            @Override
+                            public void run() {
+                                try {
+                                    barrier.await();
+                                    CountDownLatch latch = new CountDownLatch(numConnectionAttempts);
+                                    for (int i = 0; i < numConnectionAttempts; i++) {
+                                        AtomicBoolean executed = new AtomicBoolean(false);
+                                        ActionListener<Void> listener = ActionListener.wrap(x -> {
+                                            assertTrue(executed.compareAndSet(false, true));
+                                            latch.countDown();}, x -> {
+                                            assertTrue(executed.compareAndSet(false, true));
+                                            latch.countDown();
+                                            if (x instanceof RejectedExecutionException) {
+                                                // that's fine
+                                            } else {
+                                                throw new AssertionError(x);
+                                            }
+                                        });
+                                        connection.updateSeedNodes(seedNodes, listener);
+                                    }
+                                    latch.await();
+                                } catch (Exception ex) {
+                                    throw new AssertionError(ex);
+                                }
+                            }
+                        };
+                        threads[i].start();
+                    }
+
+                    for (int i = 0; i < threads.length; i++) {
+                        threads[i].join();
+                    }
+                    assertTrue(service.nodeConnected(seedNode));
+                    assertTrue(service.nodeConnected(discoverableNode));
+                    assertTrue(service.nodeConnected(seedNode1));
+                    assertTrue(connection.assertNoRunningConnections());
+                }
+            }
+        }
+    }
+
+    public void testCloseWhileConcurrentlyConnecting() throws IOException, InterruptedException, BrokenBarrierException {
+        List<DiscoveryNode> knownNodes = new CopyOnWriteArrayList<>();
+        try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT);
+             MockTransportService seedTransport1 = startTransport("seed_node_1", knownNodes, Version.CURRENT);
+             MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT)) {
+            DiscoveryNode seedNode = seedTransport.getLocalDiscoNode();
+            DiscoveryNode seedNode1 = seedTransport1.getLocalDiscoNode();
+            knownNodes.add(seedTransport.getLocalDiscoNode());
+            knownNodes.add(discoverableTransport.getLocalDiscoNode());
+            knownNodes.add(seedTransport1.getLocalDiscoNode());
+            Collections.shuffle(knownNodes, random());
+            List<DiscoveryNode> seedNodes = Arrays.asList(seedNode1, seedNode);
+            Collections.shuffle(seedNodes, random());
+
+            try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) {
+                service.start();
+                service.acceptIncomingRequests();
+                try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
+                    seedNodes, service, Integer.MAX_VALUE, n -> true)) {
+                    int numThreads = randomIntBetween(4, 10);
+                    Thread[] threads = new Thread[numThreads];
+                    CyclicBarrier barrier = new CyclicBarrier(numThreads + 1);
+                    for (int i = 0; i < threads.length; i++) {
+                        final int numConnectionAttempts = randomIntBetween(10, 100);
+                        threads[i] = new Thread() {
+                            @Override
+                            public void run() {
+                                try {
+                                    barrier.await();
+                                    CountDownLatch latch = new CountDownLatch(numConnectionAttempts);
+                                    for (int i = 0; i < numConnectionAttempts; i++) {
+                                        AtomicReference<Exception> executed = new AtomicReference<>();
+                                        ActionListener<Void> listener = ActionListener.wrap(
+                                            x -> {
+                                                if (executed.compareAndSet(null, new RuntimeException())) {
+                                                    latch.countDown();
+                                                } else {
+                                                    throw new AssertionError("shit's been called twice", executed.get());
+                                                }
+                                            },
+                                            x -> {
+                                                if (executed.compareAndSet(null, x)) {
+                                                    latch.countDown();
+                                                } else {
+                                                    final String message = x.getMessage();
+                                                    if ((executed.get().getClass() == x.getClass()
+                                                        && "operation was cancelled reason [connect handler is closed]".equals(message)
+                                                        && message.equals(executed.get().getMessage())) == false) {
+                                                        // we do cancel the operation and that means that if timing allows it, the caller
+                                                        // of a blocking call as well as the handler will get the exception from the
+                                                        // ExecutionCancelledException concurrently. unless that is the case we fail
+                                                        // if we get called more than once!
+                                                        AssertionError assertionError = new AssertionError("shit's been called twice", x);
+                                                        assertionError.addSuppressed(executed.get());
+                                                        throw assertionError;
+                                                    }
+                                                }
+                                                if (x instanceof RejectedExecutionException || x instanceof AlreadyClosedException
+                                                    || x instanceof CancellableThreads.ExecutionCancelledException) {
+                                                    // that's fine
+                                                } else {
+                                                    throw new AssertionError(x);
+                                                }
+                                        });
+                                        connection.updateSeedNodes(seedNodes, listener);
+                                    }
+                                    latch.await();
+                                } catch (Exception ex) {
+                                    throw new AssertionError(ex);
+                                }
+                            }
+                        };
+                        threads[i].start();
+                    }
+                    barrier.await();
+                    connection.close();
+                }
+            }
+        }
+    }
+
+    private static void installNodeStatsHandler(TransportService service, DiscoveryNode...nodes) {
+        service.registerRequestHandler(NodesInfoAction.NAME, NodesInfoRequest::new, ThreadPool.Names.SAME, false, false,
+            (request, channel) -> {
+                List<NodeInfo> nodeInfos = new ArrayList<>();
+                int port = 80;
+                for (DiscoveryNode node : nodes) {
+                    HttpInfo http = new HttpInfo(new BoundTransportAddress(new TransportAddress[]{node.getAddress()},
+                        new InetSocketTransportAddress(((InetSocketTransportAddress)node.getAddress()).address().getAddress(),
+                            port++)), 100);
+                    nodeInfos.add(new NodeInfo(node.getVersion(), Build.CURRENT, node, null, null, null, null, null, null, http, null,
+                        null, null));
+                }
+                channel.sendResponse(new NodesInfoResponse(ClusterName.DEFAULT, nodeInfos, Collections.emptyList()));
+            });
+
+    }
+
+    public void testGetConnectionInfo() throws Exception {
+        List<DiscoveryNode> knownNodes = new CopyOnWriteArrayList<>();
+        try (MockTransportService transport1 = startTransport("seed_node", knownNodes, Version.CURRENT);
+             MockTransportService transport2 = startTransport("seed_node_1", knownNodes, Version.CURRENT);
+             MockTransportService transport3 = startTransport("discoverable_node", knownNodes, Version.CURRENT)) {
+            DiscoveryNode node1 = transport1.getLocalDiscoNode();
+            DiscoveryNode node2 = transport3.getLocalDiscoNode();
+            DiscoveryNode node3 = transport2.getLocalDiscoNode();
+            knownNodes.add(transport1.getLocalDiscoNode());
+            knownNodes.add(transport3.getLocalDiscoNode());
+            knownNodes.add(transport2.getLocalDiscoNode());
+            Collections.shuffle(knownNodes, random());
+            List<DiscoveryNode> seedNodes = Arrays.asList(node3, node1, node2);
+            Collections.shuffle(seedNodes, random());
+
+            try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) {
+                service.start();
+                service.acceptIncomingRequests();
+                int maxNumConnections = randomIntBetween(1, 5);
+                try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
+                    seedNodes, service, maxNumConnections, n -> true)) {
+                    // test no nodes connected
+                    RemoteConnectionInfo remoteConnectionInfo = assertSerialization(getRemoteConnectionInfo(connection));
+                    assertNotNull(remoteConnectionInfo);
+                    assertEquals(0, remoteConnectionInfo.numNodesConnected);
+                    assertEquals(0, remoteConnectionInfo.seedNodes.size());
+                    assertEquals(0, remoteConnectionInfo.httpAddresses.size());
+                    assertEquals(maxNumConnections, remoteConnectionInfo.connectionsPerCluster);
+                    assertEquals("test-cluster", remoteConnectionInfo.clusterAlias);
+                    updateSeedNodes(connection, seedNodes);
+                    expectThrows(RemoteTransportException.class, () -> getRemoteConnectionInfo(connection));
+
+                    for (MockTransportService s : Arrays.asList(transport1, transport2, transport3)) {
+                        installNodeStatsHandler(s, node1, node2, node3);
+                    }
+
+                    remoteConnectionInfo = getRemoteConnectionInfo(connection);
+                    remoteConnectionInfo = assertSerialization(remoteConnectionInfo);
+                    assertNotNull(remoteConnectionInfo);
+                    assertEquals(connection.getNumNodesConnected(), remoteConnectionInfo.numNodesConnected);
+                    assertEquals(Math.min(3, maxNumConnections), connection.getNumNodesConnected());
+                    assertEquals(3, remoteConnectionInfo.seedNodes.size());
+                    assertEquals(remoteConnectionInfo.httpAddresses.size(), Math.min(3, maxNumConnections));
+                    assertEquals(maxNumConnections, remoteConnectionInfo.connectionsPerCluster);
+                    assertEquals("test-cluster", remoteConnectionInfo.clusterAlias);
+                    for (TransportAddress address : remoteConnectionInfo.httpAddresses) {
+                        assertTrue("port range mismatch: " + address.getPort(), address.getPort() >= 80 && address.getPort() <= 90);
+                    }
+                }
+            }
+        }
+    }
+
+    public void testRemoteConnectionInfo() throws IOException {
+        RemoteConnectionInfo stats = new RemoteConnectionInfo("test_cluster",
+            Arrays.asList(new LocalTransportAddress("1")),
+            Arrays.asList(new LocalTransportAddress("80")),
+            4, 3, TimeValue.timeValueMinutes(30));
+        assertSerialization(stats);
+
+        RemoteConnectionInfo stats1 = new RemoteConnectionInfo("test_cluster",
+            Arrays.asList(new LocalTransportAddress("1")),
+            Arrays.asList(new LocalTransportAddress("80")),
+            4, 4, TimeValue.timeValueMinutes(30));
+        assertSerialization(stats1);
+        assertNotEquals(stats, stats1);
+
+        stats1 = new RemoteConnectionInfo("test_cluster_1",
+            Arrays.asList(new LocalTransportAddress("1")),
+            Arrays.asList(new LocalTransportAddress("80")),
+            4, 3, TimeValue.timeValueMinutes(30));
+        assertSerialization(stats1);
+        assertNotEquals(stats, stats1);
+
+        stats1 = new RemoteConnectionInfo("test_cluster",
+            Arrays.asList(new LocalTransportAddress("15")),
+            Arrays.asList(new LocalTransportAddress("80")),
+            4, 3, TimeValue.timeValueMinutes(30));
+        assertSerialization(stats1);
+        assertNotEquals(stats, stats1);
+
+        stats1 = new RemoteConnectionInfo("test_cluster",
+            Arrays.asList(new LocalTransportAddress("1")),
+            Arrays.asList(new LocalTransportAddress("87")),
+            4, 3, TimeValue.timeValueMinutes(30));
+        assertSerialization(stats1);
+        assertNotEquals(stats, stats1);
+
+        stats1 = new RemoteConnectionInfo("test_cluster",
+            Arrays.asList(new LocalTransportAddress("1")),
+            Arrays.asList(new LocalTransportAddress("80")),
+            4, 3, TimeValue.timeValueMinutes(325));
+        assertSerialization(stats1);
+        assertNotEquals(stats, stats1);
+
+        stats1 = new RemoteConnectionInfo("test_cluster",
+            Arrays.asList(new LocalTransportAddress("1")),
+            Arrays.asList(new LocalTransportAddress("80")),
+            5, 3, TimeValue.timeValueMinutes(30));
+        assertSerialization(stats1);
+        assertNotEquals(stats, stats1);
+    }
+
+    private RemoteConnectionInfo assertSerialization(RemoteConnectionInfo info) throws IOException {
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            out.setVersion(Version.CURRENT);
+            info.writeTo(out);
+            StreamInput in = out.bytes().streamInput();
+            in.setVersion(Version.CURRENT);
+            RemoteConnectionInfo remoteConnectionInfo = new RemoteConnectionInfo(in);
+            assertEquals(info, remoteConnectionInfo);
+            assertEquals(info.hashCode(), remoteConnectionInfo.hashCode());
+            return  randomBoolean() ? info : remoteConnectionInfo;
+        }
+    }
+
+    public void testRenderConnectionInfoXContent() throws IOException {
+        RemoteConnectionInfo stats = new RemoteConnectionInfo("test_cluster",
+            Arrays.asList(new LocalTransportAddress("1")),
+            Arrays.asList(new LocalTransportAddress("2")),
+            4, 3, TimeValue.timeValueMinutes(30));
+        stats = assertSerialization(stats);
+        XContentBuilder builder = XContentFactory.jsonBuilder();
+        builder.startObject();
+        stats.toXContent(builder, null);
+        builder.endObject();
+        assertEquals("{\"test_cluster\":{\"seeds\":[\"local[1]\"],\"http_addresses\":[\"local[2]\"],\"connected\":true," +
+            "\"num_nodes_connected\":3,\"max_connections_per_cluster\":4,\"initial_connect_timeout\":\"30m\"}}", builder.string());
+
+        stats = new RemoteConnectionInfo("some_other_cluster",
+            Arrays.asList(new LocalTransportAddress("1"), new LocalTransportAddress("2")),
+            Arrays.asList(new LocalTransportAddress("80"), new LocalTransportAddress("81")),
+            2, 0, TimeValue.timeValueSeconds(30));
+        stats = assertSerialization(stats);
+        builder = XContentFactory.jsonBuilder();
+        builder.startObject();
+        stats.toXContent(builder, null);
+        builder.endObject();
+        assertEquals("{\"some_other_cluster\":{\"seeds\":[\"local[1]\",\"local[2]\"],\"http_addresses\":[\"local[80]\",\"local[81]\"]," +
+                "\"connected\":false,\"num_nodes_connected\":0,\"max_connections_per_cluster\":2,\"initial_connect_timeout\":\"30s\"}}",
+            builder.string());
+    }
+
+    private RemoteConnectionInfo getRemoteConnectionInfo(RemoteClusterConnection connection) throws Exception {
+        AtomicReference<RemoteConnectionInfo> statsRef = new AtomicReference<>();
+        AtomicReference<Exception> exceptionRef = new AtomicReference<>();
+        CountDownLatch latch = new CountDownLatch(1);
+        connection.getConnectionInfo(new ActionListener<RemoteConnectionInfo>() {
+            @Override
+            public void onResponse(RemoteConnectionInfo remoteConnectionInfo) {
+                statsRef.set(remoteConnectionInfo);
+                latch.countDown();
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                exceptionRef.set(e);
+                latch.countDown();
+            }
+        });
+        latch.await();
+        if (exceptionRef.get() != null) {
+            throw exceptionRef.get();
+        }
+        return statsRef.get();
+    }
+
+    public void testEnsureConnected() throws IOException, InterruptedException {
+        List<DiscoveryNode> knownNodes = new CopyOnWriteArrayList<>();
+        try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT);
+            MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT)) {
+            DiscoveryNode seedNode = seedTransport.getLocalDiscoNode();
+            DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode();
+            knownNodes.add(seedTransport.getLocalDiscoNode());
+            knownNodes.add(discoverableTransport.getLocalDiscoNode());
+            Collections.shuffle(knownNodes, random());
+
+            try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) {
+                service.start();
+                service.acceptIncomingRequests();
+                try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
+                    Arrays.asList(seedNode), service, Integer.MAX_VALUE, n -> true)) {
+                    assertFalse(service.nodeConnected(seedNode));
+                    assertFalse(service.nodeConnected(discoverableNode));
+                    assertTrue(connection.assertNoRunningConnections());
+                    CountDownLatch latch = new CountDownLatch(1);
+                    connection.ensureConnected(new LatchedActionListener<>(new ActionListener<Void>() {
+                        @Override
+                        public void onResponse(Void aVoid) {
+                        }
+
+                        @Override
+                        public void onFailure(Exception e) {
+                            throw new AssertionError(e);
+                        }
+                    }, latch));
+                    latch.await();
+                    assertTrue(service.nodeConnected(seedNode));
+                    assertTrue(service.nodeConnected(discoverableNode));
+                    assertTrue(connection.assertNoRunningConnections());
+
+                    // exec again we are already connected
+                    connection.ensureConnected(new LatchedActionListener<>(new ActionListener<Void>() {
+                        @Override
+                        public void onResponse(Void aVoid) {
+                        }
+
+                        @Override
+                        public void onFailure(Exception e) {
+                            throw new AssertionError(e);
+                        }
+                    }, latch));
+                    latch.await();
+                    assertTrue(service.nodeConnected(seedNode));
+                    assertTrue(service.nodeConnected(discoverableNode));
+                    assertTrue(connection.assertNoRunningConnections());
+                }
+            }
+        }
+    }
+
+    public void testCollectNodes() throws Exception {
+        List<DiscoveryNode> knownNodes = new CopyOnWriteArrayList<>();
+        try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT)) {
+            DiscoveryNode seedNode = seedTransport.getLocalDiscoNode();
+            knownNodes.add(seedTransport.getLocalDiscoNode());
+            try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) {
+                service.start();
+                service.acceptIncomingRequests();
+                try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
+                    Arrays.asList(seedNode), service, Integer.MAX_VALUE, n -> true)) {
+                    if (randomBoolean()) {
+                        updateSeedNodes(connection, Arrays.asList(seedNode));
+                    }
+                    CountDownLatch responseLatch = new CountDownLatch(1);
+                    AtomicReference<Function<String, DiscoveryNode>> reference = new AtomicReference<>();
+                    AtomicReference<Exception> failReference = new AtomicReference<>();
+                    ActionListener<Function<String, DiscoveryNode>> shardsListener = ActionListener.wrap(
+                        x -> {
+                            reference.set(x);
+                            responseLatch.countDown();
+                        },
+                        x -> {
+                            failReference.set(x);
+                            responseLatch.countDown();
+                        });
+                    connection.collectNodes(shardsListener);
+                    responseLatch.await();
+                    assertNull(failReference.get());
+                    assertNotNull(reference.get());
+                    Function<String, DiscoveryNode> function = reference.get();
+                    assertEquals(seedNode, function.apply(seedNode.getId()));
+                    assertNull(function.apply(seedNode.getId() + "foo"));
+                    assertTrue(connection.assertNoRunningConnections());
+                }
+            }
+        }
+    }
+
+    public void testConnectedNodesConcurrentAccess() throws IOException, InterruptedException {
+        List<DiscoveryNode> knownNodes = new CopyOnWriteArrayList<>();
+        List<MockTransportService> discoverableTransports = new CopyOnWriteArrayList<>();
+        try {
+            final int numDiscoverableNodes = randomIntBetween(5, 20);
+            List<DiscoveryNode> discoverableNodes = new ArrayList<>(numDiscoverableNodes);
+            for (int i = 0; i < numDiscoverableNodes; i++ ) {
+                MockTransportService transportService = startTransport("discoverable_node" + i, knownNodes, Version.CURRENT);
+                discoverableNodes.add(transportService.getLocalDiscoNode());
+                discoverableTransports.add(transportService);
+            }
+
+            List<DiscoveryNode> seedNodes = randomSubsetOf(discoverableNodes);
+            Collections.shuffle(seedNodes, random());
+
+            try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) {
+                service.start();
+                service.acceptIncomingRequests();
+                try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
+                    seedNodes, service, Integer.MAX_VALUE, n -> true)) {
+                    final int numGetThreads = randomIntBetween(4, 10);
+                    final Thread[] getThreads = new Thread[numGetThreads];
+                    final int numModifyingThreads = randomIntBetween(4, 10);
+                    final Thread[] modifyingThreads = new Thread[numModifyingThreads];
+                    CyclicBarrier barrier = new CyclicBarrier(numGetThreads + numModifyingThreads);
+                    for (int i = 0; i < getThreads.length; i++) {
+                        final int numGetCalls = randomIntBetween(1000, 10000);
+                        getThreads[i] = new Thread(() -> {
+                            try {
+                                barrier.await();
+                                for (int j = 0; j < numGetCalls; j++) {
+                                    try {
+                                        DiscoveryNode node = connection.getConnectedNode();
+                                        assertNotNull(node);
+                                    } catch (IllegalStateException e) {
+                                        if (e.getMessage().startsWith("No node available for cluster:") == false) {
+                                            throw e;
+                                        }
+                                    }
+                                }
+                            } catch (Exception ex) {
+                                throw new AssertionError(ex);
+                            }
+                        });
+                        getThreads[i].start();
+                    }
+
+                    final AtomicInteger counter = new AtomicInteger();
+                    for (int i = 0; i < modifyingThreads.length; i++) {
+                        final int numDisconnects = randomIntBetween(5, 10);
+                        modifyingThreads[i] = new Thread(() -> {
+                            try {
+                                barrier.await();
+                                for (int j = 0; j < numDisconnects; j++) {
+                                    if (randomBoolean()) {
+                                        MockTransportService transportService =
+                                            startTransport("discoverable_node_added" + counter.incrementAndGet(), knownNodes,
+                                                Version.CURRENT);
+                                        discoverableTransports.add(transportService);
+                                        connection.addConnectedNode(transportService.getLocalDiscoNode());
+                                    } else {
+                                        DiscoveryNode node = randomFrom(discoverableNodes);
+                                        connection.onNodeDisconnected(node);
+                                    }
+                                }
+                            } catch (Exception ex) {
+                                throw new AssertionError(ex);
+                            }
+                        });
+                        modifyingThreads[i].start();
+                    }
+
+                    for (Thread thread : getThreads) {
+                        thread.join();
+                    }
+                    for (Thread thread : modifyingThreads) {
+                        thread.join();
+                    }
+                }
+            }
+        } finally {
+            IOUtils.closeWhileHandlingException(discoverableTransports);
+        }
+    }
+
+    public void testClusterNameIsChecked() throws Exception {
+        List<DiscoveryNode> knownNodes = new CopyOnWriteArrayList<>();
+        List<DiscoveryNode> otherClusterKnownNodes = new CopyOnWriteArrayList<>();
+
+        Settings settings = Settings.builder().put("cluster.name", "testClusterNameIsChecked").build();
+        try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT, threadPool, settings);
+            MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT, threadPool,
+                settings);
+             MockTransportService otherClusterTransport = startTransport("other_cluster_discoverable_node", otherClusterKnownNodes,
+                 Version.CURRENT, threadPool, Settings.builder().put("cluster.name", "otherCluster").build());
+            MockTransportService otherClusterDiscoverable= startTransport("other_cluster_discoverable_node", otherClusterKnownNodes,
+             Version.CURRENT, threadPool, Settings.builder().put("cluster.name", "otherCluster").build())) {
+            DiscoveryNode seedNode = seedTransport.getLocalDiscoNode();
+            DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode();
+            knownNodes.add(seedTransport.getLocalDiscoNode());
+            knownNodes.add(discoverableTransport.getLocalDiscoNode());
+            otherClusterKnownNodes.add(otherClusterDiscoverable.getLocalDiscoNode());
+            otherClusterKnownNodes.add(otherClusterTransport.getLocalDiscoNode());
+            Collections.shuffle(knownNodes, random());
+
+            try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) {
+                service.start();
+                service.acceptIncomingRequests();
+                try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
+                    Arrays.asList(seedNode), service, Integer.MAX_VALUE, n -> true)) {
+                    updateSeedNodes(connection, Arrays.asList(seedNode));
+                    assertTrue(service.nodeConnected(seedNode));
+                    assertTrue(service.nodeConnected(discoverableNode));
+                    assertTrue(connection.assertNoRunningConnections());
+                    List<DiscoveryNode> discoveryNodes = Arrays.asList(otherClusterTransport.getLocalDiscoNode(), seedNode);
+                    Collections.shuffle(discoveryNodes, random());
+                    updateSeedNodes(connection, discoveryNodes);
+                    assertTrue(service.nodeConnected(seedNode));
+                    for (DiscoveryNode otherClusterNode : otherClusterKnownNodes) {
+                        assertFalse(service.nodeConnected(otherClusterNode));
+                    }
+                    assertFalse(service.nodeConnected(otherClusterTransport.getLocalDiscoNode()));
+                    assertTrue(service.nodeConnected(discoverableNode));
+                    assertTrue(connection.assertNoRunningConnections());
+                    IllegalStateException illegalStateException = expectThrows(IllegalStateException.class, () ->
+                        updateSeedNodes(connection, Arrays.asList(otherClusterTransport.getLocalDiscoNode())));
+                    assertThat(illegalStateException.getMessage(),
+                        Matchers.startsWith("handshake failed, mismatched cluster name [Cluster [otherCluster]]" +
+                            " - {other_cluster_discoverable_node}"));
+                }
+            }
+        }
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java b/core/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java
new file mode 100644
index 0000000000000..fcbfbec989ec3
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java
@@ -0,0 +1,470 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.transport;
+
+import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.transport.MockTransportService;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.hamcrest.Matchers;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BiFunction;
+
+public class RemoteClusterServiceTests extends ESTestCase {
+
+    private final ThreadPool threadPool = new TestThreadPool(getClass().getName());
+
+    @Override
+    public void tearDown() throws Exception {
+        super.tearDown();
+        ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS);
+    }
+
+    private MockTransportService startTransport(
+            final String id,
+            final List<DiscoveryNode> knownNodes,
+            final Version version) {
+        return startTransport(id, knownNodes, version, Settings.EMPTY);
+    }
+
+    private MockTransportService startTransport(
+            final String id,
+            final List<DiscoveryNode> knownNodes,
+            final Version version,
+            final Settings settings) {
+        return RemoteClusterConnectionTests.startTransport(
+                id, knownNodes, version, threadPool, settings);
+    }
+
+    public void testSettingsAreRegistered() {
+        assertTrue(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS.contains(RemoteClusterAware.REMOTE_CLUSTERS_SEEDS));
+        assertTrue(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS.contains(RemoteClusterService.REMOTE_CONNECTIONS_PER_CLUSTER));
+        assertTrue(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS.contains(RemoteClusterService.REMOTE_INITIAL_CONNECTION_TIMEOUT_SETTING));
+        assertTrue(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS.contains(RemoteClusterService.REMOTE_NODE_ATTRIBUTE));
+    }
+
+    public void testRemoteClusterSeedSetting() {
+        // simple validation
+        Settings settings = Settings.builder()
+            .put("search.remote.foo.seeds", "192.168.0.1:8080")
+            .put("search.remote.bar.seed", "[::1]:9090").build();
+        RemoteClusterAware.REMOTE_CLUSTERS_SEEDS.getAllConcreteSettings(settings).forEach(setting -> setting.get(settings));
+
+        Settings brokenSettings = Settings.builder()
+            .put("search.remote.foo.seeds", "192.168.0.1").build();
+        expectThrows(IllegalArgumentException.class, () ->
+        RemoteClusterAware.REMOTE_CLUSTERS_SEEDS.getAllConcreteSettings(brokenSettings).forEach(setting -> setting.get(brokenSettings)));
+    }
+
+    public void testBuiltRemoteClustersSeeds() throws Exception {
+        Map<String, List<DiscoveryNode>> map = RemoteClusterService.buildRemoteClustersSeeds(
+            Settings.builder().put("search.remote.foo.seeds", "192.168.0.1:8080").put("search.remote.bar.seeds", "[::1]:9090").build());
+        assertEquals(2, map.size());
+        assertTrue(map.containsKey("foo"));
+        assertTrue(map.containsKey("bar"));
+        assertEquals(1, map.get("foo").size());
+        assertEquals(1, map.get("bar").size());
+
+        DiscoveryNode foo = map.get("foo").get(0);
+
+        assertEquals(foo.getAddress(), new InetSocketTransportAddress(new InetSocketAddress(InetAddress.getByName("192.168.0.1"), 8080)));
+        assertEquals(foo.getId(), "foo#192.168.0.1:8080");
+        assertEquals(foo.getVersion(), Version.CURRENT.minimumCompatibilityVersion());
+
+        DiscoveryNode bar = map.get("bar").get(0);
+        assertEquals(bar.getAddress(), new InetSocketTransportAddress(new InetSocketAddress(InetAddress.getByName("[::1]"), 9090)));
+        assertEquals(bar.getId(), "bar#[::1]:9090");
+        assertEquals(bar.getVersion(), Version.CURRENT.minimumCompatibilityVersion());
+    }
+
+
+    public void testGroupClusterIndices() throws IOException {
+        List<DiscoveryNode> knownNodes = new CopyOnWriteArrayList<>();
+        try (MockTransportService seedTransport = startTransport("cluster_1_node", knownNodes, Version.CURRENT);
+             MockTransportService otherSeedTransport = startTransport("cluster_2_node", knownNodes, Version.CURRENT)) {
+            DiscoveryNode seedNode = seedTransport.getLocalDiscoNode();
+            DiscoveryNode otherSeedNode = otherSeedTransport.getLocalDiscoNode();
+            knownNodes.add(seedTransport.getLocalDiscoNode());
+            knownNodes.add(otherSeedTransport.getLocalDiscoNode());
+            Collections.shuffle(knownNodes, random());
+
+            try (MockTransportService transportService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool,
+                null)) {
+                transportService.start();
+                transportService.acceptIncomingRequests();
+                Settings.Builder builder = Settings.builder();
+                builder.putArray("search.remote.cluster_1.seeds", seedNode.getAddress().toString());
+                builder.putArray("search.remote.cluster_2.seeds", otherSeedNode.getAddress().toString());
+                try (RemoteClusterService service = new RemoteClusterService(builder.build(), transportService)) {
+                    assertFalse(service.isCrossClusterSearchEnabled());
+                    service.initializeRemoteClusters();
+                    assertTrue(service.isCrossClusterSearchEnabled());
+                    assertTrue(service.isRemoteClusterRegistered("cluster_1"));
+                    assertTrue(service.isRemoteClusterRegistered("cluster_2"));
+                    assertFalse(service.isRemoteClusterRegistered("foo"));
+                    Map<String, List<String>> perClusterIndices = service.groupClusterIndices(new String[]{"foo:bar", "cluster_1:bar",
+                        "cluster_2:foo:bar", "cluster_1:test", "cluster_2:foo*", "foo", "cluster*:baz", "*:boo", "no*match:boo"},
+                        i -> false);
+                    String[] localIndices = perClusterIndices.computeIfAbsent(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY,
+                        k -> Collections.emptyList()).toArray(new String[0]);
+                    assertNotNull(perClusterIndices.remove(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY));
+                    assertArrayEquals(new String[]{"foo:bar", "foo", "no*match:boo"}, localIndices);
+                    assertEquals(2, perClusterIndices.size());
+                    assertEquals(Arrays.asList("bar", "test", "baz", "boo"), perClusterIndices.get("cluster_1"));
+                    assertEquals(Arrays.asList("foo:bar", "foo*", "baz", "boo"), perClusterIndices.get("cluster_2"));
+
+                    IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () ->
+                        service.groupClusterIndices(new String[]{"foo:bar", "cluster_1:bar",
+                            "cluster_2:foo:bar", "cluster_1:test", "cluster_2:foo*", "foo"}, i -> "cluster_1:bar".equals(i)));
+
+                    assertEquals("Can not filter indices; index cluster_1:bar exists but there is also a remote cluster named:" +
+                            " cluster_1", iae.getMessage());
+                }
+            }
+        }
+    }
+
+    public void testIncrementallyAddClusters() throws IOException {
+        List<DiscoveryNode> knownNodes = new CopyOnWriteArrayList<>();
+        try (MockTransportService seedTransport = startTransport("cluster_1_node", knownNodes, Version.CURRENT);
+             MockTransportService otherSeedTransport = startTransport("cluster_2_node", knownNodes, Version.CURRENT)) {
+            DiscoveryNode seedNode = seedTransport.getLocalDiscoNode();
+            DiscoveryNode otherSeedNode = otherSeedTransport.getLocalDiscoNode();
+            knownNodes.add(seedTransport.getLocalDiscoNode());
+            knownNodes.add(otherSeedTransport.getLocalDiscoNode());
+            Collections.shuffle(knownNodes, random());
+
+            try (MockTransportService transportService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool,
+                null)) {
+                transportService.start();
+                transportService.acceptIncomingRequests();
+                Settings.Builder builder = Settings.builder();
+                builder.putArray("search.remote.cluster_1.seeds", seedNode.getAddress().toString());
+                builder.putArray("search.remote.cluster_2.seeds", otherSeedNode.getAddress().toString());
+                try (RemoteClusterService service = new RemoteClusterService(Settings.EMPTY, transportService)) {
+                    assertFalse(service.isCrossClusterSearchEnabled());
+                    service.initializeRemoteClusters();
+                    assertFalse(service.isCrossClusterSearchEnabled());
+                    service.updateRemoteCluster("cluster_1", Collections.singletonList(((InetSocketTransportAddress)seedNode.getAddress())
+                        .address()));
+                    assertTrue(service.isCrossClusterSearchEnabled());
+                    assertTrue(service.isRemoteClusterRegistered("cluster_1"));
+                    service.updateRemoteCluster("cluster_2", Collections.singletonList(((InetSocketTransportAddress)otherSeedNode
+                        .getAddress()).address()));
+                    assertTrue(service.isCrossClusterSearchEnabled());
+                    assertTrue(service.isRemoteClusterRegistered("cluster_1"));
+                    assertTrue(service.isRemoteClusterRegistered("cluster_2"));
+                    service.updateRemoteCluster("cluster_2", Collections.emptyList());
+                    assertFalse(service.isRemoteClusterRegistered("cluster_2"));
+                    IllegalArgumentException iae = expectThrows(IllegalArgumentException.class,
+                        () -> service.updateRemoteCluster(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY, Collections.emptyList()));
+                    assertEquals("remote clusters must not have the empty string as its key", iae.getMessage());
+                }
+            }
+        }
+    }
+
+    // we need at least one node that is >= 5.3 for cross cluster search
+    public void testPre53NodesAreRejected() throws IOException {
+        List<DiscoveryNode> knownNodes = new CopyOnWriteArrayList<>();
+        try (MockTransportService seedTransport = startTransport("cluster_1_node", knownNodes, Version.fromString("5.1.99"));
+             MockTransportService otherSeedTransport = startTransport("cluster_2_node", knownNodes, Version.CURRENT)) {
+            DiscoveryNode seedNode = seedTransport.getLocalDiscoNode();
+            DiscoveryNode otherSeedNode = otherSeedTransport.getLocalDiscoNode();
+            knownNodes.add(seedNode);
+            knownNodes.add(otherSeedNode);
+            assertTrue(seedNode.getVersion().before(Version.V_5_3_0));
+            Collections.shuffle(knownNodes, random());
+
+            try (MockTransportService transportService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool,
+                null)) {
+                transportService.start();
+                transportService.acceptIncomingRequests();
+                Settings.Builder builder = Settings.builder();
+                builder.putArray("search.remote.cluster.seeds", knownNodes.get(0).getAddress().toString());
+                try (RemoteClusterService service = new RemoteClusterService(builder.build(), transportService)) {
+                    service.initializeRemoteClusters();
+                    assertTrue(service.isCrossClusterSearchEnabled());
+                    assertTrue(service.isRemoteClusterRegistered("cluster"));
+                    assertFalse(transportService.nodeConnected(seedNode));
+                    assertTrue(transportService.nodeConnected(otherSeedNode));
+                }
+            }
+        }
+    }
+
+    public void testRemoteNodeAttribute() throws IOException, InterruptedException {
+        final Settings settings =
+                Settings.builder().put("search.remote.node.attr", "gateway").build();
+        final List<DiscoveryNode> knownNodes = new CopyOnWriteArrayList<>();
+        final Settings gateway = Settings.builder().put("node.attr.gateway", true).build();
+        try (MockTransportService c1N1 =
+                     startTransport("cluster_1_node_1", knownNodes, Version.CURRENT);
+             MockTransportService c1N2 =
+                     startTransport("cluster_1_node_2", knownNodes, Version.CURRENT, gateway);
+             MockTransportService c2N1 =
+                     startTransport("cluster_2_node_1", knownNodes, Version.CURRENT);
+             MockTransportService c2N2 =
+                     startTransport("cluster_2_node_2", knownNodes, Version.CURRENT, gateway)) {
+            final DiscoveryNode c1N1Node = c1N1.getLocalDiscoNode();
+            final DiscoveryNode c1N2Node = c1N2.getLocalDiscoNode();
+            final DiscoveryNode c2N1Node = c2N1.getLocalDiscoNode();
+            final DiscoveryNode c2N2Node = c2N2.getLocalDiscoNode();
+            knownNodes.add(c1N1Node);
+            knownNodes.add(c1N2Node);
+            knownNodes.add(c2N1Node);
+            knownNodes.add(c2N2Node);
+            Collections.shuffle(knownNodes, random());
+
+            try (MockTransportService transportService = MockTransportService.createNewService(
+                    settings,
+                    Version.CURRENT,
+                    threadPool,
+                    null)) {
+                transportService.start();
+                transportService.acceptIncomingRequests();
+                final Settings.Builder builder = Settings.builder();
+                builder.putArray(
+                        "search.remote.cluster_1.seeds", c1N1Node.getAddress().toString());
+                builder.putArray(
+                        "search.remote.cluster_2.seeds", c2N1Node.getAddress().toString());
+                try (RemoteClusterService service =
+                             new RemoteClusterService(settings, transportService)) {
+                    assertFalse(service.isCrossClusterSearchEnabled());
+                    service.initializeRemoteClusters();
+                    assertFalse(service.isCrossClusterSearchEnabled());
+
+                    final InetSocketAddress c1N1Address =
+                            ((InetSocketTransportAddress)c1N1Node.getAddress()).address();
+                    final InetSocketAddress c1N2Address =
+                            ((InetSocketTransportAddress)c1N2Node.getAddress()).address();
+                    final InetSocketAddress c2N1Address =
+                            ((InetSocketTransportAddress)c2N1Node.getAddress()).address();
+                    final InetSocketAddress c2N2Address =
+                            ((InetSocketTransportAddress)c2N2Node.getAddress()).address();
+
+                    final CountDownLatch firstLatch = new CountDownLatch(1);
+                    service.updateRemoteCluster(
+                            "cluster_1",
+                            Arrays.asList(c1N1Address, c1N2Address),
+                            connectionListener(firstLatch));
+                    firstLatch.await();
+
+                    final CountDownLatch secondLatch = new CountDownLatch(1);
+                    service.updateRemoteCluster(
+                            "cluster_2",
+                            Arrays.asList(c2N1Address, c2N2Address),
+                            connectionListener(secondLatch));
+                    secondLatch.await();
+
+                    assertTrue(service.isCrossClusterSearchEnabled());
+                    assertTrue(service.isRemoteClusterRegistered("cluster_1"));
+                    assertFalse(service.isRemoteNodeConnected("cluster_1", c1N1Node));
+                    assertTrue(service.isRemoteNodeConnected("cluster_1", c1N2Node));
+                    assertTrue(service.isRemoteClusterRegistered("cluster_2"));
+                    assertFalse(service.isRemoteNodeConnected("cluster_2", c2N1Node));
+                    assertTrue(service.isRemoteNodeConnected("cluster_2", c2N2Node));
+                }
+            }
+        }
+    }
+
+    private ActionListener<Void> connectionListener(final CountDownLatch latch) {
+        return ActionListener.wrap(x -> latch.countDown(), x -> fail());
+    }
+
+    public void testCollectNodes() throws InterruptedException, IOException {
+        final Settings settings = Settings.EMPTY;
+        final List<DiscoveryNode> knownNodes_c1 = new CopyOnWriteArrayList<>();
+        final List<DiscoveryNode> knownNodes_c2 = new CopyOnWriteArrayList<>();
+
+        try (MockTransportService c1N1 =
+                 startTransport("cluster_1_node_1", knownNodes_c1, Version.CURRENT, settings);
+             MockTransportService c1N2 =
+                 startTransport("cluster_1_node_2", knownNodes_c1, Version.CURRENT, settings);
+             MockTransportService c2N1 =
+                 startTransport("cluster_2_node_1", knownNodes_c2, Version.CURRENT, settings);
+             MockTransportService c2N2 =
+                 startTransport("cluster_2_node_2", knownNodes_c2, Version.CURRENT, settings)) {
+            final DiscoveryNode c1N1Node = c1N1.getLocalDiscoNode();
+            final DiscoveryNode c1N2Node = c1N2.getLocalDiscoNode();
+            final DiscoveryNode c2N1Node = c2N1.getLocalDiscoNode();
+            final DiscoveryNode c2N2Node = c2N2.getLocalDiscoNode();
+            knownNodes_c1.add(c1N1Node);
+            knownNodes_c1.add(c1N2Node);
+            knownNodes_c2.add(c2N1Node);
+            knownNodes_c2.add(c2N2Node);
+            Collections.shuffle(knownNodes_c1, random());
+            Collections.shuffle(knownNodes_c2, random());
+
+            try (MockTransportService transportService = MockTransportService.createNewService(
+                settings,
+                Version.CURRENT,
+                threadPool,
+                null)) {
+                transportService.start();
+                transportService.acceptIncomingRequests();
+                final Settings.Builder builder = Settings.builder();
+                builder.putArray(
+                    "search.remote.cluster_1.seeds", c1N1Node.getAddress().toString());
+                builder.putArray(
+                    "search.remote.cluster_2.seeds", c2N1Node.getAddress().toString());
+                try (RemoteClusterService service =
+                         new RemoteClusterService(settings, transportService)) {
+                    assertFalse(service.isCrossClusterSearchEnabled());
+                    service.initializeRemoteClusters();
+                    assertFalse(service.isCrossClusterSearchEnabled());
+
+                    final InetSocketAddress c1N1Address = ((InetSocketTransportAddress)c1N1Node.getAddress()).address();
+                    final InetSocketAddress c1N2Address = ((InetSocketTransportAddress)c1N2Node.getAddress()).address();
+                    final InetSocketAddress c2N1Address = ((InetSocketTransportAddress)c2N1Node.getAddress()).address();
+                    final InetSocketAddress c2N2Address = ((InetSocketTransportAddress)c2N2Node.getAddress()).address();
+
+                    final CountDownLatch firstLatch = new CountDownLatch(1);
+                    service.updateRemoteCluster(
+                        "cluster_1",
+                        Arrays.asList(c1N1Address, c1N2Address),
+                        connectionListener(firstLatch));
+                    firstLatch.await();
+
+                    final CountDownLatch secondLatch = new CountDownLatch(1);
+                    service.updateRemoteCluster(
+                        "cluster_2",
+                        Arrays.asList(c2N1Address, c2N2Address),
+                        connectionListener(secondLatch));
+                    secondLatch.await();
+                    CountDownLatch latch = new CountDownLatch(1);
+                    service.collectNodes(new HashSet<>(Arrays.asList("cluster_1", "cluster_2")),
+                        new ActionListener<BiFunction<String, String, DiscoveryNode>>() {
+                        @Override
+                        public void onResponse(BiFunction<String, String, DiscoveryNode> func) {
+                            try {
+                                assertEquals(c1N1Node, func.apply("cluster_1", c1N1Node.getId()));
+                                assertEquals(c1N2Node, func.apply("cluster_1", c1N2Node.getId()));
+                                assertEquals(c2N1Node, func.apply("cluster_2", c2N1Node.getId()));
+                                assertEquals(c2N2Node, func.apply("cluster_2", c2N2Node.getId()));
+                            } finally {
+                                latch.countDown();
+                            }
+                        }
+
+                        @Override
+                        public void onFailure(Exception e) {
+                            try {
+                                throw new AssertionError(e);
+                            } finally {
+                                latch.countDown();
+                            }
+                        }
+                    });
+                    latch.await();
+                    {
+                        CountDownLatch failLatch = new CountDownLatch(1);
+                        AtomicReference<Exception> ex = new AtomicReference<>();
+                        service.collectNodes(new HashSet<>(Arrays.asList("cluster_1", "cluster_2", "no such cluster")),
+                            new ActionListener<BiFunction<String, String, DiscoveryNode>>() {
+                                @Override
+                                public void onResponse(BiFunction<String, String, DiscoveryNode> stringStringDiscoveryNodeBiFunction) {
+                                    try {
+                                        fail("should not be called");
+                                    } finally {
+                                        failLatch.countDown();
+                                    }
+                                }
+
+                                @Override
+                                public void onFailure(Exception e) {
+                                    try {
+                                        ex.set(e);
+                                    } finally {
+                                        failLatch.countDown();
+                                    }
+                                }
+                            });
+                        failLatch.await();
+                        assertNotNull(ex.get());
+                        assertTrue(ex.get() instanceof IllegalArgumentException);
+                        assertEquals("no such remote cluster: [no such cluster]", ex.get().getMessage());
+                    }
+                    {
+
+                        logger.info("closing all source nodes");
+                        // close all targets and check for the transport level failure path
+                        IOUtils.close(c1N1, c1N2, c2N1, c2N2);
+                        logger.info("all source nodes are closed");
+                        CountDownLatch failLatch = new CountDownLatch(1);
+                        AtomicReference<Exception> ex = new AtomicReference<>();
+                        service.collectNodes(new HashSet<>(Arrays.asList("cluster_1", "cluster_2")),
+                            new ActionListener<BiFunction<String, String, DiscoveryNode>>() {
+                                @Override
+                                public void onResponse(BiFunction<String, String, DiscoveryNode> stringStringDiscoveryNodeBiFunction) {
+                                    try {
+                                        fail("should not be called");
+                                    } finally {
+                                        failLatch.countDown();
+                                    }
+                                }
+
+                                @Override
+                                public void onFailure(Exception e) {
+                                    try {
+                                        ex.set(e);
+                                    } finally {
+                                        failLatch.countDown();
+                                    }
+                                }
+                            });
+                        failLatch.await();
+                        assertNotNull(ex.get());
+                        if (ex.get() instanceof  IllegalStateException) {
+                            assertThat(ex.get().getMessage(), Matchers.anyOf(Matchers.equalTo("no seed node left"), Matchers.startsWith
+                                ("No node available for cluster:")));
+                        } else {
+                            if (ex.get() instanceof TransportException == false) {
+                                // we have an issue for this see #25301
+                                logger.error("expected TransportException but got a different one see #25301", ex.get());
+                            }
+                            assertTrue("expected TransportException but got a different one [" + ex.get().getClass().toString() + "]",
+                                ex.get() instanceof TransportException);
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/transport/TCPTransportTests.java b/core/src/test/java/org/elasticsearch/transport/TCPTransportTests.java
index da1dcf43e5d07..39f83c7c48dbb 100644
--- a/core/src/test/java/org/elasticsearch/transport/TCPTransportTests.java
+++ b/core/src/test/java/org/elasticsearch/transport/TCPTransportTests.java
@@ -19,8 +19,31 @@
 
 package org.elasticsearch.transport;
 
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.compress.CompressorFactory;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.threadpool.ThreadPool;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
+
+import static org.hamcrest.Matchers.equalTo;
 
 /** Unit tests for TCPTransport */
 public class TCPTransportTests extends ESTestCase {
@@ -127,4 +150,173 @@ public void testAddressLimit() throws Exception {
         assertEquals(101, addresses[1].getPort());
         assertEquals(102, addresses[2].getPort());
     }
+
+    public void testCompressRequest() throws IOException {
+        final boolean compressed = randomBoolean();
+        final AtomicBoolean called = new AtomicBoolean(false);
+        Req request = new Req(randomRealisticUnicodeOfLengthBetween(10, 100));
+        ThreadPool threadPool = new TestThreadPool(TCPTransportTests.class.getName());
+        AtomicReference<IOException> exceptionReference = new AtomicReference<>();
+        try {
+            TcpTransport transport = new TcpTransport("test", Settings.builder().put("transport.tcp.compress", compressed).build(),
+                threadPool, new BigArrays(Settings.EMPTY, null), null, null, null) {
+                @Override
+                protected InetSocketAddress getLocalAddress(Object o) {
+                    return null;
+                }
+
+                @Override
+                protected Object bind(String name, InetSocketAddress address) throws IOException {
+                    return null;
+                }
+
+                @Override
+                protected void closeChannels(List channel, boolean blocking) throws IOException {
+
+                }
+
+                @Override
+                protected void sendMessage(Object o, BytesReference reference, ActionListener listener) {
+                    try {
+                        StreamInput streamIn = reference.streamInput();
+                        streamIn.skip(TcpHeader.MARKER_BYTES_SIZE);
+                        int len = streamIn.readInt();
+                        long requestId = streamIn.readLong();
+                        assertEquals(42, requestId);
+                        byte status = streamIn.readByte();
+                        Version version = Version.fromId(streamIn.readInt());
+                        assertEquals(Version.CURRENT, version);
+                        assertEquals(compressed, TransportStatus.isCompress(status));
+                        called.compareAndSet(false, true);
+                        if (compressed) {
+                            final int bytesConsumed = TcpHeader.HEADER_SIZE;
+                            streamIn = CompressorFactory.compressor(reference.slice(bytesConsumed, reference.length() - bytesConsumed))
+                                .streamInput(streamIn);
+                        }
+                        threadPool.getThreadContext().readHeaders(streamIn);
+                        assertEquals("foobar", streamIn.readString());
+                        Req readReq = new Req("");
+                        readReq.readFrom(streamIn);
+                        assertEquals(request.value, readReq.value);
+                    } catch (IOException e) {
+                        exceptionReference.set(e);
+                    }
+                }
+
+                @Override
+                protected NodeChannels connectToChannels(
+                        DiscoveryNode node, ConnectionProfile profile, Consumer onChannelClose) throws IOException {
+                    return new NodeChannels(node, new Object[profile.getNumConnections()], profile);
+                }
+
+                @Override
+                protected boolean isOpen(Object o) {
+                    return false;
+                }
+
+                @Override
+                public long serverOpen() {
+                    return 0;
+                }
+
+                @Override
+                public NodeChannels getConnection(DiscoveryNode node) {
+                    return new NodeChannels(node, new Object[MockTcpTransport.LIGHT_PROFILE.getNumConnections()],
+                        MockTcpTransport.LIGHT_PROFILE);
+                }
+            };
+            DiscoveryNode node = new DiscoveryNode("foo", new LocalTransportAddress("1"), Version.CURRENT);
+            Transport.Connection connection = transport.getConnection(node);
+            connection.sendRequest(42, "foobar", request, TransportRequestOptions.EMPTY);
+            assertTrue(called.get());
+            assertNull("IOException while sending message.", exceptionReference.get());
+        } finally {
+            ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS);
+        }
+    }
+
+    private static final class Req extends TransportRequest {
+        public String value;
+
+        private Req(String value) {
+            this.value = value;
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            value = in.readString();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeString(value);
+        }
+    }
+
+    public void testConnectionProfileResolve() {
+        final ConnectionProfile defaultProfile = TcpTransport.buildDefaultConnectionProfile(Settings.EMPTY);
+        assertEquals(defaultProfile, TcpTransport.resolveConnectionProfile(null, defaultProfile));
+
+        final ConnectionProfile.Builder builder = new ConnectionProfile.Builder();
+        builder.addConnections(randomIntBetween(0, 5), TransportRequestOptions.Type.BULK);
+        builder.addConnections(randomIntBetween(0, 5), TransportRequestOptions.Type.RECOVERY);
+        builder.addConnections(randomIntBetween(0, 5), TransportRequestOptions.Type.REG);
+        builder.addConnections(randomIntBetween(0, 5), TransportRequestOptions.Type.STATE);
+        builder.addConnections(randomIntBetween(0, 5), TransportRequestOptions.Type.PING);
+
+        final boolean connectionTimeoutSet = randomBoolean();
+        if (connectionTimeoutSet) {
+            builder.setConnectTimeout(TimeValue.timeValueMillis(randomNonNegativeLong()));
+        }
+        final boolean connectionHandshakeSet = randomBoolean();
+        if (connectionHandshakeSet) {
+            builder.setHandshakeTimeout(TimeValue.timeValueMillis(randomNonNegativeLong()));
+        }
+
+        final ConnectionProfile profile = builder.build();
+        final ConnectionProfile resolved = TcpTransport.resolveConnectionProfile(profile, defaultProfile);
+        assertNotEquals(resolved, defaultProfile);
+        assertThat(resolved.getNumConnections(), equalTo(profile.getNumConnections()));
+        assertThat(resolved.getHandles(), equalTo(profile.getHandles()));
+
+        assertThat(resolved.getConnectTimeout(),
+            equalTo(connectionTimeoutSet ? profile.getConnectTimeout() : defaultProfile.getConnectTimeout()));
+        assertThat(resolved.getHandshakeTimeout(),
+            equalTo(connectionHandshakeSet ? profile.getHandshakeTimeout() : defaultProfile.getHandshakeTimeout()));
+    }
+
+    public void testDefaultConnectionProfile() {
+        ConnectionProfile profile = TcpTransport.buildDefaultConnectionProfile(Settings.EMPTY);
+        assertEquals(13, profile.getNumConnections());
+        assertEquals(1, profile.getNumConnectionsPerType(TransportRequestOptions.Type.PING));
+        assertEquals(6, profile.getNumConnectionsPerType(TransportRequestOptions.Type.REG));
+        assertEquals(1, profile.getNumConnectionsPerType(TransportRequestOptions.Type.STATE));
+        assertEquals(2, profile.getNumConnectionsPerType(TransportRequestOptions.Type.RECOVERY));
+        assertEquals(3, profile.getNumConnectionsPerType(TransportRequestOptions.Type.BULK));
+
+        profile = TcpTransport.buildDefaultConnectionProfile(Settings.builder().put("node.master", false).build());
+        assertEquals(12, profile.getNumConnections());
+        assertEquals(1, profile.getNumConnectionsPerType(TransportRequestOptions.Type.PING));
+        assertEquals(6, profile.getNumConnectionsPerType(TransportRequestOptions.Type.REG));
+        assertEquals(0, profile.getNumConnectionsPerType(TransportRequestOptions.Type.STATE));
+        assertEquals(2, profile.getNumConnectionsPerType(TransportRequestOptions.Type.RECOVERY));
+        assertEquals(3, profile.getNumConnectionsPerType(TransportRequestOptions.Type.BULK));
+
+        profile = TcpTransport.buildDefaultConnectionProfile(Settings.builder().put("node.data", false).build());
+        assertEquals(11, profile.getNumConnections());
+        assertEquals(1, profile.getNumConnectionsPerType(TransportRequestOptions.Type.PING));
+        assertEquals(6, profile.getNumConnectionsPerType(TransportRequestOptions.Type.REG));
+        assertEquals(1, profile.getNumConnectionsPerType(TransportRequestOptions.Type.STATE));
+        assertEquals(0, profile.getNumConnectionsPerType(TransportRequestOptions.Type.RECOVERY));
+        assertEquals(3, profile.getNumConnectionsPerType(TransportRequestOptions.Type.BULK));
+
+        profile = TcpTransport.buildDefaultConnectionProfile(Settings.builder().put("node.data", false).put("node.master", false).build());
+        assertEquals(10, profile.getNumConnections());
+        assertEquals(1, profile.getNumConnectionsPerType(TransportRequestOptions.Type.PING));
+        assertEquals(6, profile.getNumConnectionsPerType(TransportRequestOptions.Type.REG));
+        assertEquals(0, profile.getNumConnectionsPerType(TransportRequestOptions.Type.STATE));
+        assertEquals(0, profile.getNumConnectionsPerType(TransportRequestOptions.Type.RECOVERY));
+        assertEquals(3, profile.getNumConnectionsPerType(TransportRequestOptions.Type.BULK));
+    }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/transport/TransportActionProxyTests.java b/core/src/test/java/org/elasticsearch/transport/TransportActionProxyTests.java
new file mode 100644
index 0000000000000..936b24bc8654f
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/transport/TransportActionProxyTests.java
@@ -0,0 +1,273 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.transport;
+
+import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.transport.MockTransportService;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.concurrent.CountDownLatch;
+
+public class TransportActionProxyTests extends ESTestCase {
+    protected ThreadPool threadPool;
+    // we use always a non-alpha or beta version here otherwise minimumCompatibilityVersion will be different for the two used versions
+    private static final Version CURRENT_VERSION = Version.fromString(String.valueOf(Version.CURRENT.major) + ".0.0");
+    protected static final Version version0 = CURRENT_VERSION.minimumCompatibilityVersion();
+
+    protected DiscoveryNode nodeA;
+    protected MockTransportService serviceA;
+
+    protected static final Version version1 = Version.fromId(CURRENT_VERSION.id + 1);
+    protected DiscoveryNode nodeB;
+    protected MockTransportService serviceB;
+
+    protected DiscoveryNode nodeC;
+    protected MockTransportService serviceC;
+
+
+    @Override
+    @Before
+    public void setUp() throws Exception {
+        super.setUp();
+        threadPool = new TestThreadPool(getClass().getName());
+        serviceA = buildService(version0); // this one supports dynamic tracer updates
+        nodeA = serviceA.getLocalDiscoNode();
+        serviceB = buildService(version1); // this one doesn't support dynamic tracer updates
+        nodeB = serviceB.getLocalDiscoNode();
+        serviceC = buildService(version1); // this one doesn't support dynamic tracer updates
+        nodeC = serviceC.getLocalDiscoNode();
+    }
+
+    @Override
+    public void tearDown() throws Exception {
+        super.tearDown();
+        IOUtils.close(serviceA, serviceB, serviceC, () -> {
+            try {
+                terminate(threadPool);
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+            }
+        });
+    }
+
+    private MockTransportService buildService(final Version version) {
+        MockTransportService service = MockTransportService.local(Settings.EMPTY, version, threadPool, null);
+            service.start();
+            service.acceptIncomingRequests();
+        return service;
+
+    }
+
+
+    public void testSendMessage() throws InterruptedException {
+        serviceA.registerRequestHandler("/test", SimpleTestRequest::new, ThreadPool.Names.SAME,
+            (request, channel) -> {
+                assertEquals(request.sourceNode, "TS_A");
+                SimpleTestResponse response = new SimpleTestResponse();
+                response.targetNode = "TS_A";
+                channel.sendResponse(response);
+            });
+        TransportActionProxy.registerProxyAction(serviceA, "/test", SimpleTestResponse::new);
+        serviceA.connectToNode(nodeB);
+
+        serviceB.registerRequestHandler("/test", SimpleTestRequest::new, ThreadPool.Names.SAME,
+            (request, channel) -> {
+                assertEquals(request.sourceNode, "TS_A");
+                SimpleTestResponse response = new SimpleTestResponse();
+                response.targetNode = "TS_B";
+                channel.sendResponse(response);
+            });
+        TransportActionProxy.registerProxyAction(serviceB, "/test", SimpleTestResponse::new);
+        serviceB.connectToNode(nodeC);
+        serviceC.registerRequestHandler("/test", SimpleTestRequest::new, ThreadPool.Names.SAME,
+            (request, channel) -> {
+                assertEquals(request.sourceNode, "TS_A");
+                SimpleTestResponse response = new SimpleTestResponse();
+                response.targetNode = "TS_C";
+                channel.sendResponse(response);
+            });
+        TransportActionProxy.registerProxyAction(serviceC, "/test", SimpleTestResponse::new);
+
+        CountDownLatch latch = new CountDownLatch(1);
+        serviceA.sendRequest(nodeB, TransportActionProxy.getProxyAction("/test"), TransportActionProxy.wrapRequest(nodeC,
+            new SimpleTestRequest("TS_A")), new TransportResponseHandler<SimpleTestResponse>() {
+                @Override
+                public SimpleTestResponse newInstance() {
+                    return new SimpleTestResponse();
+                }
+
+                @Override
+                public void handleResponse(SimpleTestResponse response) {
+                    try {
+                        assertEquals("TS_C", response.targetNode);
+                    } finally {
+                        latch.countDown();
+                    }
+                }
+
+                @Override
+                public void handleException(TransportException exp) {
+                    try {
+                    throw new AssertionError(exp);
+                    } finally {
+                        latch.countDown();
+                    }
+                }
+
+                @Override
+                public String executor() {
+                    return ThreadPool.Names.SAME;
+                }
+            });
+        latch.await();
+    }
+
+    public void testException() throws InterruptedException {
+        serviceA.registerRequestHandler("/test", SimpleTestRequest::new, ThreadPool.Names.SAME,
+            (request, channel) -> {
+                assertEquals(request.sourceNode, "TS_A");
+                SimpleTestResponse response = new SimpleTestResponse();
+                response.targetNode = "TS_A";
+                channel.sendResponse(response);
+            });
+        TransportActionProxy.registerProxyAction(serviceA, "/test", SimpleTestResponse::new);
+        serviceA.connectToNode(nodeB);
+
+        serviceB.registerRequestHandler("/test", SimpleTestRequest::new, ThreadPool.Names.SAME,
+            (request, channel) -> {
+                assertEquals(request.sourceNode, "TS_A");
+                SimpleTestResponse response = new SimpleTestResponse();
+                response.targetNode = "TS_B";
+                channel.sendResponse(response);
+            });
+        TransportActionProxy.registerProxyAction(serviceB, "/test", SimpleTestResponse::new);
+        serviceB.connectToNode(nodeC);
+        serviceC.registerRequestHandler("/test", SimpleTestRequest::new, ThreadPool.Names.SAME,
+            (request, channel) -> {
+                throw new ElasticsearchException("greetings from TS_C");
+            });
+        TransportActionProxy.registerProxyAction(serviceC, "/test", SimpleTestResponse::new);
+
+        CountDownLatch latch = new CountDownLatch(1);
+        serviceA.sendRequest(nodeB, TransportActionProxy.getProxyAction("/test"), TransportActionProxy.wrapRequest(nodeC,
+            new SimpleTestRequest("TS_A")), new TransportResponseHandler<SimpleTestResponse>() {
+                @Override
+                public SimpleTestResponse newInstance() {
+                    return new SimpleTestResponse();
+                }
+
+                @Override
+                public void handleResponse(SimpleTestResponse response) {
+                    try {
+                        fail("expected exception");
+                    } finally {
+                        latch.countDown();
+                    }
+                }
+
+                @Override
+                public void handleException(TransportException exp) {
+                    try {
+                        Throwable cause = ExceptionsHelper.unwrapCause(exp);
+                        assertEquals("greetings from TS_C", cause.getMessage());
+                    } finally {
+                        latch.countDown();
+                    }
+                }
+
+                @Override
+                public String executor() {
+                    return ThreadPool.Names.SAME;
+                }
+            });
+        latch.await();
+    }
+
+    public static class SimpleTestRequest extends TransportRequest {
+        String sourceNode;
+
+        public SimpleTestRequest(String sourceNode) {
+            this.sourceNode = sourceNode;
+        }
+        public SimpleTestRequest() {}
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            super.readFrom(in);
+            sourceNode = in.readString();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeString(sourceNode);
+        }
+    }
+
+    public static class SimpleTestResponse extends TransportResponse {
+        String targetNode;
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            super.readFrom(in);
+            targetNode = in.readString();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeString(targetNode);
+        }
+    }
+
+    public void testGetAction() {
+        String action = "foo/bar";
+        String proxyAction = TransportActionProxy.getProxyAction(action);
+        assertTrue(proxyAction.endsWith(action));
+        assertEquals("internal:transport/proxy/foo/bar", proxyAction);
+    }
+
+    public void testUnwrap() {
+        TransportRequest transportRequest = TransportActionProxy.wrapRequest(nodeA, TransportService.HandshakeRequest.INSTANCE);
+        assertTrue(transportRequest instanceof TransportActionProxy.ProxyRequest);
+        assertSame(TransportService.HandshakeRequest.INSTANCE, TransportActionProxy.unwrapRequest(transportRequest));
+    }
+
+    public void testIsProxyAction() {
+        String action = "foo/bar";
+        String proxyAction = TransportActionProxy.getProxyAction(action);
+        assertTrue(TransportActionProxy.isProxyAction(proxyAction));
+        assertFalse(TransportActionProxy.isProxyAction(action));
+    }
+
+    public void testIsProxyRequest() {
+        assertTrue(TransportActionProxy.isProxyRequest(new TransportActionProxy.ProxyRequest<>(() -> null)));
+        assertFalse(TransportActionProxy.isProxyRequest(TransportRequest.Empty.INSTANCE));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/transport/TransportModuleTests.java b/core/src/test/java/org/elasticsearch/transport/TransportModuleTests.java
deleted file mode 100644
index e5c734cbfb669..0000000000000
--- a/core/src/test/java/org/elasticsearch/transport/TransportModuleTests.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.transport;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.inject.ModuleTestCase;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.indices.breaker.CircuitBreakerService;
-import org.elasticsearch.test.transport.AssertingLocalTransport;
-import org.elasticsearch.threadpool.ThreadPool;
-
-/** Unit tests for module registering custom transport and transport service */
-public class TransportModuleTests extends ModuleTestCase {
-
-
-
-    static class FakeTransport extends AssertingLocalTransport {
-        @Inject
-        public FakeTransport(Settings settings, CircuitBreakerService circuitBreakerService, ThreadPool threadPool,
-                             NamedWriteableRegistry namedWriteableRegistry) {
-            super(settings, circuitBreakerService, threadPool, namedWriteableRegistry);
-        }
-    }
-
-    static class FakeTransportService extends TransportService {
-        @Inject
-        public FakeTransportService(Settings settings, Transport transport, ThreadPool threadPool) {
-            super(settings, transport, threadPool);
-        }
-    }
-}
diff --git a/core/src/test/java/org/elasticsearch/transport/TransportServiceHandshakeTests.java b/core/src/test/java/org/elasticsearch/transport/TransportServiceHandshakeTests.java
index cfc4de6b3c014..ab882b4031600 100644
--- a/core/src/test/java/org/elasticsearch/transport/TransportServiceHandshakeTests.java
+++ b/core/src/test/java/org/elasticsearch/transport/TransportServiceHandshakeTests.java
@@ -35,6 +35,7 @@
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -65,20 +66,18 @@ private NetworkHandle startServices(String nodeNameAndId, Settings settings, Ver
                         new NoneCircuitBreakerService(),
                         new NamedWriteableRegistry(Collections.emptyList()),
                         new NetworkService(settings, Collections.emptyList()));
-        TransportService transportService = new MockTransportService(settings, transport, threadPool);
+        TransportService transportService = new MockTransportService(settings, transport, threadPool,
+            TransportService.NOOP_TRANSPORT_INTERCEPTOR, (boundAddress) -> new DiscoveryNode(
+            nodeNameAndId,
+            nodeNameAndId,
+            boundAddress.publishAddress(),
+            emptyMap(),
+            emptySet(),
+            version), null);
         transportService.start();
         transportService.acceptIncomingRequests();
-        DiscoveryNode node =
-                new DiscoveryNode(
-                        nodeNameAndId,
-                        nodeNameAndId,
-                        transportService.boundAddress().publishAddress(),
-                        emptyMap(),
-                        emptySet(),
-                        version);
-        transportService.setLocalNode(node);
         transportServices.add(transportService);
-        return new NetworkHandle(transportService, node);
+        return new NetworkHandle(transportService, transportService.getLocalNode());
     }
 
     @After
@@ -96,7 +95,7 @@ public static void terminateThreadPool() {
         threadPool = null;
     }
 
-    public void testConnectToNodeLight() {
+    public void testConnectToNodeLight() throws IOException {
         Settings settings = Settings.builder().put("cluster.name", "test").build();
 
         NetworkHandle handleA = startServices("TS_A", settings, Version.CURRENT);
@@ -111,14 +110,14 @@ public void testConnectToNodeLight() {
             emptyMap(),
             emptySet(),
             Version.CURRENT.minimumCompatibilityVersion());
-        DiscoveryNode connectedNode =
-                handleA.transportService.connectToNodeLightAndHandshake(discoveryNode, timeout);
-        assertNotNull(connectedNode);
-
-        // the name and version should be updated
-        assertEquals(connectedNode.getName(), "TS_B");
-        assertEquals(connectedNode.getVersion(), handleB.discoveryNode.getVersion());
-        assertTrue(handleA.transportService.nodeConnected(discoveryNode));
+        try (Transport.Connection connection = handleA.transportService.openConnection(discoveryNode, MockTcpTransport.LIGHT_PROFILE)){
+            DiscoveryNode connectedNode = handleA.transportService.handshake(connection, timeout);
+            assertNotNull(connectedNode);
+            // the name and version should be updated
+            assertEquals(connectedNode.getName(), "TS_B");
+            assertEquals(connectedNode.getVersion(), handleB.discoveryNode.getVersion());
+            assertFalse(handleA.transportService.nodeConnected(discoveryNode));
+        }
     }
 
     public void testMismatchedClusterName() {
@@ -131,8 +130,12 @@ public void testMismatchedClusterName() {
             emptyMap(),
             emptySet(),
             Version.CURRENT.minimumCompatibilityVersion());
-        IllegalStateException ex = expectThrows(IllegalStateException.class, () -> handleA.transportService.connectToNodeLightAndHandshake(
-                discoveryNode, timeout));
+        IllegalStateException ex = expectThrows(IllegalStateException.class, () -> {
+            try (Transport.Connection connection = handleA.transportService.openConnection(discoveryNode,
+                MockTcpTransport.LIGHT_PROFILE)) {
+                handleA.transportService.handshake(connection, timeout);
+            }
+        });
         assertThat(ex.getMessage(), containsString("handshake failed, mismatched cluster name [Cluster [b]]"));
         assertFalse(handleA.transportService.nodeConnected(discoveryNode));
 }
@@ -148,40 +151,39 @@ public void testIncompatibleVersions() {
             emptyMap(),
             emptySet(),
             Version.CURRENT.minimumCompatibilityVersion());
-        IllegalStateException ex = expectThrows(IllegalStateException.class, () -> handleA.transportService.connectToNodeLightAndHandshake(
-            discoveryNode, timeout));
+        IllegalStateException ex = expectThrows(IllegalStateException.class, () -> {
+            try (Transport.Connection connection = handleA.transportService.openConnection(discoveryNode,
+                MockTcpTransport.LIGHT_PROFILE)) {
+                handleA.transportService.handshake(connection, timeout);
+            }
+        });
         assertThat(ex.getMessage(), containsString("handshake failed, incompatible version"));
         assertFalse(handleA.transportService.nodeConnected(discoveryNode));
     }
 
-    public void testIgnoreMismatchedClusterName() {
-        Settings settings = Settings.builder().put("cluster.name", "a").build();
-
+    public void testNodeConnectWithDifferentNodeId() {
+        Settings settings = Settings.builder().put("cluster.name", "test").build();
         NetworkHandle handleA = startServices("TS_A", settings, Version.CURRENT);
-        NetworkHandle handleB =
-                startServices(
-                        "TS_B",
-                        Settings.builder().put("cluster.name", "b").build(),
-                        VersionUtils.randomVersionBetween(random(), Version.CURRENT.minimumCompatibilityVersion(), Version.CURRENT)
-                );
+        NetworkHandle handleB = startServices("TS_B", settings, Version.CURRENT);
         DiscoveryNode discoveryNode = new DiscoveryNode(
-            "",
+            randomAlphaOfLength(10),
             handleB.discoveryNode.getAddress(),
             emptyMap(),
             emptySet(),
-            Version.CURRENT.minimumCompatibilityVersion());
-        DiscoveryNode connectedNode = handleA.transportService.connectToNodeLightAndHandshake(discoveryNode, timeout, false);
-        assertNotNull(connectedNode);
-        assertEquals(connectedNode.getName(), "TS_B");
-        assertEquals(connectedNode.getVersion(), handleB.discoveryNode.getVersion());
-        assertTrue(handleA.transportService.nodeConnected(discoveryNode));
+            handleB.discoveryNode.getVersion());
+        ConnectTransportException ex = expectThrows(ConnectTransportException.class, () -> {
+            handleA.transportService.connectToNode(discoveryNode, MockTcpTransport.LIGHT_PROFILE);
+        });
+        assertThat(ex.getMessage(), containsString("unexpected remote node"));
+        assertFalse(handleA.transportService.nodeConnected(discoveryNode));
     }
 
+
     private static class NetworkHandle {
         private TransportService transportService;
         private DiscoveryNode discoveryNode;
 
-        public NetworkHandle(TransportService transportService, DiscoveryNode discoveryNode) {
+        NetworkHandle(TransportService transportService, DiscoveryNode discoveryNode) {
             this.transportService = transportService;
             this.discoveryNode = discoveryNode;
         }
diff --git a/core/src/test/java/org/elasticsearch/transport/local/SimpleLocalTransportTests.java b/core/src/test/java/org/elasticsearch/transport/local/SimpleLocalTransportTests.java
index c4458b7883d27..6880e322d6ec1 100644
--- a/core/src/test/java/org/elasticsearch/transport/local/SimpleLocalTransportTests.java
+++ b/core/src/test/java/org/elasticsearch/transport/local/SimpleLocalTransportTests.java
@@ -20,18 +20,32 @@
 package org.elasticsearch.transport.local;
 
 import org.elasticsearch.Version;
-import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.transport.AbstractSimpleTransportTestCase;
+import org.elasticsearch.transport.Transport;
+
+import java.io.IOException;
 
 public class SimpleLocalTransportTests extends AbstractSimpleTransportTestCase {
 
     @Override
-    protected MockTransportService build(Settings settings, Version version) {
-        MockTransportService transportService = MockTransportService.local(settings, version, threadPool);
+    protected MockTransportService build(Settings settings, Version version, ClusterSettings clusterSettings, boolean doHandshake) {
+        // we ignore the doHandshake boolean here since this only applied to TcpTransport subclasses
+        MockTransportService transportService = MockTransportService.local(settings, version, threadPool, clusterSettings);
         transportService.start();
         return transportService;
     }
+
+    @Override
+    public void testChannelCloseWhileConnecting() throws IOException {
+        // this test does not apply to local transport
+    }
+
+    @Override
+    protected void closeConnectionChannel(final Transport transport, final Transport.Connection connection) throws IOException {
+        throw new UnsupportedOperationException();
+    }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/tribe/TribeIT.java b/core/src/test/java/org/elasticsearch/tribe/TribeIT.java
index 96911f0f91c66..9ed438282a092 100644
--- a/core/src/test/java/org/elasticsearch/tribe/TribeIT.java
+++ b/core/src/test/java/org/elasticsearch/tribe/TribeIT.java
@@ -19,432 +19,621 @@
 
 package org.elasticsearch.tribe;
 
-import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
-import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
-import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
+import org.elasticsearch.action.support.DestructiveOperations;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.client.Requests;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.NamedDiff;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.TransportAddress;
-import org.elasticsearch.discovery.DiscoveryModule;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.discovery.DiscoverySettings;
 import org.elasticsearch.discovery.MasterNotDiscoveredException;
-import org.elasticsearch.discovery.zen.ping.unicast.UnicastZenPing;
 import org.elasticsearch.node.Node;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.InternalTestCluster;
 import org.elasticsearch.test.NodeConfigurationSource;
-import org.elasticsearch.test.TestCluster;
+import org.elasticsearch.test.TestCustomMetaData;
+import org.elasticsearch.tribe.TribeServiceTests.MergableCustomMetaData1;
+import org.elasticsearch.tribe.TribeServiceTests.MergableCustomMetaData2;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
-import java.util.Map;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
 import java.util.function.Function;
+import java.util.function.Predicate;
+import java.util.function.UnaryOperator;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
 
+import static java.util.stream.Collectors.toSet;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.core.Is.is;
 
 /**
  * Note, when talking to tribe client, no need to set the local flag on master read operations, it
  * does it by default.
  */
-@LuceneTestCase.SuppressFileSystems("ExtrasFS") // doesn't work with potential multi data path from test cluster yet
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0)
 public class TribeIT extends ESIntegTestCase {
 
-    public static final String SECOND_CLUSTER_NODE_PREFIX = "node_tribe2";
+    private static final String TRIBE_NODE = "tribe_node";
 
+    private static InternalTestCluster cluster1;
     private static InternalTestCluster cluster2;
 
-    private Node tribeNode;
-    private Client tribeClient;
+    /**
+     * A predicate that is used to select none of the remote clusters
+     **/
+    private static final Predicate<InternalTestCluster> NONE = c -> false;
+
+    /**
+     * A predicate that is used to select the remote cluster 1 only
+     **/
+    private static final Predicate<InternalTestCluster> CLUSTER1_ONLY = c -> c.getClusterName().equals(cluster1.getClusterName());
+
+    /**
+     * A predicate that is used to select the remote cluster 2 only
+     **/
+    private static final Predicate<InternalTestCluster> CLUSTER2_ONLY = c -> c.getClusterName().equals(cluster2.getClusterName());
+
+    /**
+     * A predicate that is used to select the the two remote clusters
+     **/
+    private static final Predicate<InternalTestCluster> ALL = c -> true;
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        return Settings.builder()
+                .put(super.nodeSettings(nodeOrdinal))
+                // Required to delete _all indices on remote clusters
+                .put(DestructiveOperations.REQUIRES_NAME_SETTING.getKey(), false)
+                .build();
+    }
+
+    public static class TestCustomMetaDataPlugin extends Plugin {
+
+        private final List<NamedWriteableRegistry.Entry> namedWritables = new ArrayList<>();
+
+        public TestCustomMetaDataPlugin() {
+            registerBuiltinWritables();
+        }
+
+        private <T extends MetaData.Custom> void registerMetaDataCustom(String name, Writeable.Reader<? extends T> reader,
+                                                                        Writeable.Reader<NamedDiff> diffReader) {
+            namedWritables.add(new NamedWriteableRegistry.Entry(MetaData.Custom.class, name, reader));
+            namedWritables.add(new NamedWriteableRegistry.Entry(NamedDiff.class, name, diffReader));
+        }
+
+        private void registerBuiltinWritables() {
+            registerMetaDataCustom(MergableCustomMetaData1.TYPE, MergableCustomMetaData1::readFrom, MergableCustomMetaData1::readDiffFrom);
+            registerMetaDataCustom(MergableCustomMetaData2.TYPE, MergableCustomMetaData2::readFrom, MergableCustomMetaData2::readDiffFrom);
+        }
+
+        @Override
+        public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
+            return namedWritables;
+        }
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        ArrayList<Class<? extends Plugin>> plugins = new ArrayList<>();
+        plugins.addAll(getMockPlugins());
+        plugins.add(TestCustomMetaDataPlugin.class);
+        return plugins;
+    }
 
     @Before
-    public  void setupSecondCluster() throws Exception {
+    public void startRemoteClusters() {
+        final int minNumDataNodes = 2;
+        final int maxNumDataNodes = 4;
+        final NodeConfigurationSource nodeConfigurationSource = getNodeConfigSource();
+        final Collection<Class<? extends Plugin>> plugins = nodePlugins();
+
+        if (cluster1 == null) {
+            cluster1 = new InternalTestCluster(randomLong(), createTempDir(), true, true, minNumDataNodes, maxNumDataNodes,
+                    UUIDs.randomBase64UUID(random()), nodeConfigurationSource, 0, false, "cluster_1",
+                    plugins, Function.identity());
+        }
+
         if (cluster2 == null) {
-            final  NodeConfigurationSource configSource = getNodeConfigSource();
-            cluster2 = new InternalTestCluster(randomLong(), createTempDir(), true, 2, 2,
-                UUIDs.randomBase64UUID(random()), configSource, 0, false, SECOND_CLUSTER_NODE_PREFIX, getMockPlugins(),
-                Function.identity());
-            cluster2.beforeTest(random(), 0.1);
-            cluster2.ensureAtLeastNumDataNodes(2);
+            cluster2 = new InternalTestCluster(randomLong(), createTempDir(), true, true, minNumDataNodes, maxNumDataNodes,
+                    UUIDs.randomBase64UUID(random()), nodeConfigurationSource, 0, false, "cluster_2",
+                    plugins, Function.identity());
         }
-    }
 
-    @AfterClass
-    public static void tearDownSecondCluster() {
-        if (cluster2 != null) {
+        doWithAllClusters(c -> {
             try {
-                cluster2.close();
-            } finally {
-                cluster2 = null;
+                c.beforeTest(random(), 0.1);
+                c.ensureAtLeastNumDataNodes(minNumDataNodes);
+            } catch (Exception e) {
+                throw new RuntimeException("Failed to set up remote cluster [" + c.getClusterName() + "]", e);
             }
-        }
+        });
     }
 
     @After
-    public void tearDownTribeNode() throws IOException {
-        if (cluster2 != null) {
+    public void wipeRemoteClusters() {
+        doWithAllClusters(c -> {
+            final String clusterName = c.getClusterName();
             try {
-                cluster2.wipe(Collections.<String>emptySet());
-            } finally {
-                cluster2.afterTest();
+                c.client().admin().indices().prepareDelete(MetaData.ALL).get();
+                c.afterTest();
+            } catch (IOException e) {
+                throw new RuntimeException("Failed to clean up remote cluster [" + clusterName + "]", e);
             }
+        });
+    }
+
+    @AfterClass
+    public static void stopRemoteClusters() {
+        try {
+            doWithAllClusters(InternalTestCluster::close);
+        } finally {
+            cluster1 = null;
+            cluster2 = null;
         }
-        if (tribeNode != null) {
-            tribeNode.close();
-            tribeNode = null;
-        }
     }
 
-    private void setupTribeNode(Settings settings) {
-        Map<String,String> asMap = internalCluster().getDefaultSettings().getAsMap();
-        Settings.Builder tribe1Defaults = Settings.builder();
-        Settings.Builder tribe2Defaults = Settings.builder();
-        for (Map.Entry<String, String> entry : asMap.entrySet()) {
-            if (entry.getKey().startsWith("path.")) {
-                continue;
+    private Releasable startTribeNode() throws Exception {
+        return startTribeNode(ALL, Settings.EMPTY);
+    }
+
+    private Releasable startTribeNode(Predicate<InternalTestCluster> filter, Settings settings) throws Exception {
+        final String node = internalCluster().startNode(createTribeSettings(filter).put(settings).build());
+
+        // wait for node to be connected to all tribe clusters
+        final Set<String> expectedNodes = Sets.newHashSet(internalCluster().getNodeNames());
+        doWithAllClusters(filter, c -> {
+            // Adds the tribe client node dedicated to this remote cluster
+            for (String tribeNode : internalCluster().getNodeNames()) {
+                expectedNodes.add(tribeNode + "/" + c.getClusterName());
             }
-            tribe1Defaults.put("tribe.t1." + entry.getKey(), entry.getValue());
-            tribe2Defaults.put("tribe.t2." + entry.getKey(), entry.getValue());
-        }
-        // give each tribe it's unicast hosts to connect to
-        tribe1Defaults.putArray("tribe.t1." + UnicastZenPing.DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING.getKey(), getUnicastHosts(internalCluster().client()));
-        tribe1Defaults.putArray("tribe.t2." + UnicastZenPing.DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING.getKey(), getUnicastHosts(cluster2.client()));
-
-        Settings merged = Settings.builder()
-                .put(internalCluster().getDefaultSettings())
-                .put("tribe.t1.cluster.name", internalCluster().getClusterName())
-                .put("tribe.t2.cluster.name", cluster2.getClusterName())
-                .put("tribe.t1.transport.type", "local")
-                .put("tribe.t2.transport.type", "local")
-                .put("tribe.t1.discovery.type", "local")
-                .put("tribe.t2.discovery.type", "local")
-                .put("transport.type", "local")
-                .put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), "local")
-                .put("tribe.blocks.write", false)
-                .put(NetworkModule.HTTP_ENABLED.getKey(), false)
-                .put(settings)
-
-                .put(tribe1Defaults.build())
-                .put(tribe2Defaults.build())
-                .put("node.name", "tribe_node") // make sure we can identify threads from this node
-                .build();
+            // Adds the remote clusters nodes names
+            Collections.addAll(expectedNodes, c.getNodeNames());
+        });
+        assertBusy(() -> {
+            ClusterState state = client().admin().cluster().prepareState().setNodes(true).get().getState();
+            Set<String> nodes = StreamSupport.stream(state.getNodes().spliterator(), false).map(DiscoveryNode::getName).collect(toSet());
+            assertThat(nodes, containsInAnyOrder(expectedNodes.toArray()));
+        });
+        // wait for join to be fully applied on all nodes in the tribe clusters, see https://github.com/elastic/elasticsearch/issues/23695
+        doWithAllClusters(filter, c -> {
+            assertFalse(c.client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).get().isTimedOut());
+        });
 
-        tribeNode = new Node(merged).start();
-        tribeClient = tribeNode.client();
+        return () -> {
+            try {
+                while(internalCluster().getNodeNames().length > 0) {
+                    internalCluster().stopRandomNode(s -> true);
+                }
+            } catch (Exception e) {
+                throw new RuntimeException("Failed to close tribe node [" + node + "]", e);
+            }
+        };
     }
 
-    public void testGlobalReadWriteBlocks() throws Exception {
-        logger.info("create 2 indices, test1 on t1, and test2 on t2");
-        internalCluster().client().admin().indices().prepareCreate("test1").get();
-        assertAcked(cluster2.client().admin().indices().prepareCreate("test2"));
+    private Settings.Builder createTribeSettings(Predicate<InternalTestCluster> filter) {
+        assertNotNull(filter);
+
+        final Settings.Builder settings = Settings.builder();
+        settings.put(Node.NODE_NAME_SETTING.getKey(), TRIBE_NODE);
+        settings.put(Node.NODE_DATA_SETTING.getKey(), false);
+        settings.put(Node.NODE_MASTER_SETTING.getKey(), true);
+        settings.put(NetworkModule.HTTP_ENABLED.getKey(), false);
+        settings.put(NetworkModule.TRANSPORT_TYPE_SETTING.getKey(), NetworkModule.LOCAL_TRANSPORT);
+
+        doWithAllClusters(filter, c -> {
+            String tribeSetting = "tribe." + c.getClusterName() + ".";
+            settings.put(tribeSetting + ClusterName.CLUSTER_NAME_SETTING.getKey(), c.getClusterName());
+            settings.put(tribeSetting + DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING.getKey(), "100ms");
+            settings.put(tribeSetting + NetworkModule.TRANSPORT_TYPE_SETTING.getKey(), NetworkModule.LOCAL_TRANSPORT);
+        });
 
+        return settings;
+    }
 
-        setupTribeNode(Settings.builder()
+    public void testGlobalReadWriteBlocks() throws Exception {
+        Settings additionalSettings = Settings.builder()
                 .put("tribe.blocks.write", true)
                 .put("tribe.blocks.metadata", true)
-                .build());
+                .build();
 
-        logger.info("wait till tribe has the same nodes as the 2 clusters");
-        awaitSameNodeCounts();
-        // wait till the tribe node connected to the cluster, by checking if the index exists in the cluster state
-        logger.info("wait till test1 and test2 exists in the tribe node state");
-        awaitIndicesInClusterState("test1", "test2");
+        try (Releasable tribeNode = startTribeNode(ALL, additionalSettings)) {
+            // Creates 2 indices, test1 on cluster1 and test2 on cluster2
+            assertAcked(cluster1.client().admin().indices().prepareCreate("test1"));
+            ensureGreen(cluster1.client());
 
-        try {
-            tribeClient.prepareIndex("test1", "type1", "1").setSource("field1", "value1").execute().actionGet();
-            fail("cluster block should be thrown");
-        } catch (ClusterBlockException e) {
-            // all is well!
-        }
-        try {
-            tribeClient.admin().indices().prepareForceMerge("test1").execute().actionGet();
-            fail("cluster block should be thrown");
-        } catch (ClusterBlockException e) {
-            // all is well!
-        }
-        try {
-            tribeClient.admin().indices().prepareForceMerge("test2").execute().actionGet();
-            fail("cluster block should be thrown");
-        } catch (ClusterBlockException e) {
-            // all is well!
+            assertAcked(cluster2.client().admin().indices().prepareCreate("test2"));
+            ensureGreen(cluster2.client());
+
+            // Wait for the tribe node to retrieve the indices into its cluster state
+            assertIndicesExist(client(), "test1", "test2");
+
+            // Writes not allowed through the tribe node
+            ClusterBlockException e = expectThrows(ClusterBlockException.class, () -> {
+                client().prepareIndex("test1", "type1").setSource("field", "value").get();
+            });
+            assertThat(e.getMessage(), containsString("blocked by: [BAD_REQUEST/11/tribe node, write not allowed]"));
+
+            e = expectThrows(ClusterBlockException.class, () -> client().prepareIndex("test2", "type2").setSource("field", "value").get());
+            assertThat(e.getMessage(), containsString("blocked by: [BAD_REQUEST/11/tribe node, write not allowed]"));
+
+            e = expectThrows(ClusterBlockException.class, () -> client().admin().indices().prepareForceMerge("test1").get());
+            assertThat(e.getMessage(), containsString("blocked by: [BAD_REQUEST/10/tribe node, metadata not allowed]"));
+
+            e = expectThrows(ClusterBlockException.class, () -> client().admin().indices().prepareForceMerge("test2").get());
+            assertThat(e.getMessage(), containsString("blocked by: [BAD_REQUEST/10/tribe node, metadata not allowed]"));
         }
     }
 
     public void testIndexWriteBlocks() throws Exception {
-        logger.info("create 2 indices, test1 on t1, and test2 on t2");
-        assertAcked(internalCluster().client().admin().indices().prepareCreate("test1"));
-        assertAcked(internalCluster().client().admin().indices().prepareCreate("block_test1"));
-        assertAcked(cluster2.client().admin().indices().prepareCreate("test2"));
-        assertAcked(cluster2.client().admin().indices().prepareCreate("block_test2"));
-
-        setupTribeNode(Settings.builder()
+        Settings additionalSettings = Settings.builder()
                 .put("tribe.blocks.write.indices", "block_*")
-                .build());
-        logger.info("wait till tribe has the same nodes as the 2 clusters");
-        awaitSameNodeCounts();
-        // wait till the tribe node connected to the cluster, by checking if the index exists in the cluster state
-        logger.info("wait till test1 and test2 exists in the tribe node state");
-        awaitIndicesInClusterState("test1", "test2", "block_test1", "block_test2");
-
-        tribeClient.prepareIndex("test1", "type1", "1").setSource("field1", "value1").get();
-        try {
-            tribeClient.prepareIndex("block_test1", "type1", "1").setSource("field1", "value1").get();
-            fail("cluster block should be thrown");
-        } catch (ClusterBlockException e) {
-            // all is well!
-        }
+                .build();
 
-        tribeClient.prepareIndex("test2", "type1", "1").setSource("field1", "value1").get();
-        try {
-            tribeClient.prepareIndex("block_test2", "type1", "1").setSource("field1", "value1").get();
-            fail("cluster block should be thrown");
-        } catch (ClusterBlockException e) {
-            // all is well!
+        try (Releasable tribeNode = startTribeNode(ALL, additionalSettings)) {
+            // Creates 2 indices on each remote cluster, test1 and block_test1 on cluster1 and test2 and block_test2 on cluster2
+            assertAcked(cluster1.client().admin().indices().prepareCreate("test1"));
+            assertAcked(cluster1.client().admin().indices().prepareCreate("block_test1"));
+            ensureGreen(cluster1.client());
+
+            assertAcked(cluster2.client().admin().indices().prepareCreate("test2"));
+            assertAcked(cluster2.client().admin().indices().prepareCreate("block_test2"));
+            ensureGreen(cluster2.client());
+
+            // Wait for the tribe node to retrieve the indices into its cluster state
+            assertIndicesExist(client(), "test1", "test2", "block_test1", "block_test2");
+
+            // Writes allowed through the tribe node for test1/test2 indices
+            client().prepareIndex("test1", "type1").setSource("field", "value").get();
+            client().prepareIndex("test2", "type2").setSource("field", "value").get();
+
+            ClusterBlockException e;
+            e = expectThrows(ClusterBlockException.class, () -> client().prepareIndex("block_test1", "type1").setSource("foo", 0).get());
+            assertThat(e.getMessage(), containsString("blocked by: [FORBIDDEN/8/index write (api)]"));
+
+            e = expectThrows(ClusterBlockException.class, () -> client().prepareIndex("block_test2", "type2").setSource("foo", 0).get());
+            assertThat(e.getMessage(), containsString("blocked by: [FORBIDDEN/8/index write (api)]"));
         }
     }
 
     public void testOnConflictDrop() throws Exception {
-        logger.info("create 2 indices, test1 on t1, and test2 on t2");
-        assertAcked(cluster().client().admin().indices().prepareCreate("conflict"));
-        assertAcked(cluster2.client().admin().indices().prepareCreate("conflict"));
-        assertAcked(cluster().client().admin().indices().prepareCreate("test1"));
-        assertAcked(cluster2.client().admin().indices().prepareCreate("test2"));
-
-        setupTribeNode(Settings.builder()
+        Settings additionalSettings = Settings.builder()
                 .put("tribe.on_conflict", "drop")
-                .build());
-
-        logger.info("wait till tribe has the same nodes as the 2 clusters");
-        awaitSameNodeCounts();
-
-        // wait till the tribe node connected to the cluster, by checking if the index exists in the cluster state
-        logger.info("wait till test1 and test2 exists in the tribe node state");
-        awaitIndicesInClusterState("test1", "test2");
+                .build();
 
-        assertThat(tribeClient.admin().cluster().prepareState().get().getState().getMetaData().index("test1").getSettings().get("tribe.name"), equalTo("t1"));
-        assertThat(tribeClient.admin().cluster().prepareState().get().getState().getMetaData().index("test2").getSettings().get("tribe.name"), equalTo("t2"));
-        assertThat(tribeClient.admin().cluster().prepareState().get().getState().getMetaData().hasIndex("conflict"), equalTo(false));
+        try (Releasable tribeNode = startTribeNode(ALL, additionalSettings)) {
+            // Creates 2 indices on each remote cluster, test1 and conflict on cluster1 and test2 and also conflict on cluster2
+            assertAcked(cluster1.client().admin().indices().prepareCreate("test1"));
+            assertAcked(cluster1.client().admin().indices().prepareCreate("conflict"));
+            ensureGreen(cluster1.client());
+
+            assertAcked(cluster2.client().admin().indices().prepareCreate("test2"));
+            assertAcked(cluster2.client().admin().indices().prepareCreate("conflict"));
+            ensureGreen(cluster2.client());
+
+            // Wait for the tribe node to retrieve the indices into its cluster state
+            assertIndicesExist(client(), "test1", "test2");
+
+            ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
+            assertThat(clusterState.getMetaData().hasIndex("test1"), is(true));
+            assertThat(clusterState.getMetaData().index("test1").getSettings().get("tribe.name"), equalTo(cluster1.getClusterName()));
+            assertThat(clusterState.getMetaData().hasIndex("test2"), is(true));
+            assertThat(clusterState.getMetaData().index("test2").getSettings().get("tribe.name"), equalTo(cluster2.getClusterName()));
+            assertThat(clusterState.getMetaData().hasIndex("conflict"), is(false));
+        }
     }
 
     public void testOnConflictPrefer() throws Exception {
-        testOnConflictPrefer(randomBoolean() ? "t1" : "t2");
-    }
+        final String preference = randomFrom(cluster1, cluster2).getClusterName();
+        Settings additionalSettings = Settings.builder()
+                .put("tribe.on_conflict", "prefer_" + preference)
+                .build();
 
-    private void testOnConflictPrefer(String tribe) throws Exception {
-        logger.info("testing preference for tribe {}", tribe);
-
-        logger.info("create 2 indices, test1 on t1, and test2 on t2");
-        assertAcked(internalCluster().client().admin().indices().prepareCreate("conflict"));
-        assertAcked(cluster2.client().admin().indices().prepareCreate("conflict"));
-        assertAcked(internalCluster().client().admin().indices().prepareCreate("test1"));
-        assertAcked(cluster2.client().admin().indices().prepareCreate("test2"));
-
-        setupTribeNode(Settings.builder()
-                .put("tribe.on_conflict", "prefer_" + tribe)
-                .build());
-        logger.info("wait till tribe has the same nodes as the 2 clusters");
-        awaitSameNodeCounts();
-        // wait till the tribe node connected to the cluster, by checking if the index exists in the cluster state
-        logger.info("wait till test1 and test2 exists in the tribe node state");
-        awaitIndicesInClusterState("test1", "test2", "conflict");
-
-        assertThat(tribeClient.admin().cluster().prepareState().get().getState().getMetaData().index("test1").getSettings().get("tribe.name"), equalTo("t1"));
-        assertThat(tribeClient.admin().cluster().prepareState().get().getState().getMetaData().index("test2").getSettings().get("tribe.name"), equalTo("t2"));
-        assertThat(tribeClient.admin().cluster().prepareState().get().getState().getMetaData().index("conflict").getSettings().get("tribe.name"), equalTo(tribe));
+        try (Releasable tribeNode = startTribeNode(ALL, additionalSettings)) {
+            assertAcked(cluster1.client().admin().indices().prepareCreate("test1"));
+            assertAcked(cluster1.client().admin().indices().prepareCreate("shared"));
+            ensureGreen(cluster1.client());
+
+            assertAcked(cluster2.client().admin().indices().prepareCreate("test2"));
+            assertAcked(cluster2.client().admin().indices().prepareCreate("shared"));
+            ensureGreen(cluster2.client());
+
+            // Wait for the tribe node to retrieve the indices into its cluster state
+            assertIndicesExist(client(), "test1", "test2", "shared");
+
+            ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
+            assertThat(clusterState.getMetaData().hasIndex("test1"), is(true));
+            assertThat(clusterState.getMetaData().index("test1").getSettings().get("tribe.name"), equalTo(cluster1.getClusterName()));
+            assertThat(clusterState.getMetaData().hasIndex("test2"), is(true));
+            assertThat(clusterState.getMetaData().index("test2").getSettings().get("tribe.name"), equalTo(cluster2.getClusterName()));
+            assertThat(clusterState.getMetaData().hasIndex("shared"), is(true));
+            assertThat(clusterState.getMetaData().index("shared").getSettings().get("tribe.name"), equalTo(preference));
+        }
     }
 
     public void testTribeOnOneCluster() throws Exception {
-        setupTribeNode(Settings.EMPTY);
-        logger.info("create 2 indices, test1 on t1, and test2 on t2");
-        assertAcked(internalCluster().client().admin().indices().prepareCreate("test1"));
-        assertAcked(cluster2.client().admin().indices().prepareCreate("test2"));
-
-
-        // wait till the tribe node connected to the cluster, by checking if the index exists in the cluster state
-        logger.info("wait till test1 and test2 exists in the tribe node state");
-        awaitIndicesInClusterState("test1", "test2");
-
-        logger.info("wait till tribe has the same nodes as the 2 clusters");
-        awaitSameNodeCounts();
+        try (Releasable tribeNode = startTribeNode()) {
+            // Creates 2 indices, test1 on cluster1 and test2 on cluster2
+            assertAcked(cluster1.client().admin().indices().prepareCreate("test1"));
+            ensureGreen(cluster1.client());
+
+            assertAcked(cluster2.client().admin().indices().prepareCreate("test2"));
+            ensureGreen(cluster2.client());
+
+            // Wait for the tribe node to retrieve the indices into its cluster state
+            assertIndicesExist(client(), "test1", "test2");
+
+            // Creates two docs using the tribe node
+            indexRandom(true,
+                    client().prepareIndex("test1", "type1", "1").setSource("field1", "value1"),
+                    client().prepareIndex("test2", "type1", "1").setSource("field1", "value1")
+            );
+
+            // Verify that documents are searchable using the tribe node
+            assertHitCount(client().prepareSearch().get(), 2L);
+
+            // Using assertBusy to check that the mappings are in the tribe node cluster state
+            assertBusy(() -> {
+                ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
+                assertThat(clusterState.getMetaData().index("test1").mapping("type1"), notNullValue());
+                assertThat(clusterState.getMetaData().index("test2").mapping("type1"), notNullValue());
+            });
+
+            // More documents with another type
+            indexRandom(true,
+                    client().prepareIndex("test1", "type2", "1").setSource("field1", "value1"),
+                    client().prepareIndex("test2", "type2", "1").setSource("field1", "value1")
+            );
+            assertHitCount(client().prepareSearch().get(), 4L);
+            assertBusy(() -> {
+                ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
+                assertThat(clusterState.getMetaData().index("test1").mapping("type1"), notNullValue());
+                assertThat(clusterState.getMetaData().index("test1").mapping("type2"), notNullValue());
+
+                assertThat(clusterState.getMetaData().index("test2").mapping("type1"), notNullValue());
+                assertThat(clusterState.getMetaData().index("test2").mapping("type2"), notNullValue());
+            });
+
+            // Make sure master level write operations fail... (we don't really have a master)
+            expectThrows(MasterNotDiscoveredException.class, () -> {
+                client().admin().indices().prepareCreate("tribe_index").setMasterNodeTimeout("10ms").get();
+            });
+
+            // Now delete an index and makes sure it's reflected in cluster state
+            cluster2.client().admin().indices().prepareDelete("test2").get();
+            assertBusy(() -> {
+                ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
+                assertFalse(clusterState.getMetaData().hasIndex("test2"));
+                assertFalse(clusterState.getRoutingTable().hasIndex("test2"));
+            });
+        }
+    }
 
-        assertThat(tribeClient.admin().cluster().prepareHealth().setWaitForGreenStatus().get().getStatus(), equalTo(ClusterHealthStatus.GREEN));
+    public void testCloseAndOpenIndex() throws Exception {
+        // Creates an index on remote cluster 1
+        assertTrue(cluster1.client().admin().indices().prepareCreate("first").get().isAcknowledged());
+        ensureGreen(cluster1.client());
+
+        // Closes the index
+        assertTrue(cluster1.client().admin().indices().prepareClose("first").get().isAcknowledged());
+
+        try (Releasable tribeNode = startTribeNode()) {
+            // The closed index is not part of the tribe node cluster state
+            ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
+            assertFalse(clusterState.getMetaData().hasIndex("first"));
+
+            // Open the index, it becomes part of the tribe node cluster state
+            assertTrue(cluster1.client().admin().indices().prepareOpen("first").get().isAcknowledged());
+            assertIndicesExist(client(), "first");
+
+            // Create a second index, wait till it is seen from within the tribe node
+            assertTrue(cluster2.client().admin().indices().prepareCreate("second").get().isAcknowledged());
+            assertIndicesExist(client(), "first", "second");
+            ensureGreen(cluster2.client());
+
+            // Close the second index, wait till it gets removed from the tribe node cluster state
+            assertTrue(cluster2.client().admin().indices().prepareClose("second").get().isAcknowledged());
+            assertIndicesExist(client(), "first");
+
+            // Open the second index, wait till it gets added back to the tribe node cluster state
+            assertTrue(cluster2.client().admin().indices().prepareOpen("second").get().isAcknowledged());
+            assertIndicesExist(client(), "first", "second");
+            ensureGreen(cluster2.client());
+        }
+    }
 
-        logger.info("create 2 docs through the tribe node");
-        tribeClient.prepareIndex("test1", "type1", "1").setSource("field1", "value1").get();
-        tribeClient.prepareIndex("test2", "type1", "1").setSource("field1", "value1").get();
-        tribeClient.admin().indices().prepareRefresh().get();
+    /**
+     * Test that the tribe node's cluster state correctly reflect the number of nodes
+     * of the remote clusters the tribe node is connected to.
+     */
+    public void testClusterStateNodes() throws Exception {
+        List<Predicate<InternalTestCluster>> predicates = Arrays.asList(NONE, CLUSTER1_ONLY, CLUSTER2_ONLY, ALL);
+        Collections.shuffle(predicates, random());
 
-        logger.info("verify they are there");
-        assertHitCount(tribeClient.prepareSearch().setSize(0).get(), 2L);
-        assertHitCount(tribeClient.prepareSearch().get(), 2L);
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                ClusterState tribeState = tribeNode.client().admin().cluster().prepareState().get().getState();
-                assertThat(tribeState.getMetaData().index("test1").mapping("type1"), notNullValue());
-                assertThat(tribeState.getMetaData().index("test2").mapping("type1"), notNullValue());
+        for (Predicate<InternalTestCluster> predicate : predicates) {
+            try (Releasable tribeNode = startTribeNode(predicate, Settings.EMPTY)) {
             }
-        });
-
-
-        logger.info("write to another type");
-        tribeClient.prepareIndex("test1", "type2", "1").setSource("field1", "value1").get();
-        tribeClient.prepareIndex("test2", "type2", "1").setSource("field1", "value1").get();
-        assertNoFailures(tribeClient.admin().indices().prepareRefresh().get());
+        }
+    }
 
+    public void testMergingRemovedCustomMetaData() throws Exception {
+        removeCustomMetaData(cluster1, MergableCustomMetaData1.TYPE);
+        removeCustomMetaData(cluster2, MergableCustomMetaData1.TYPE);
+        MergableCustomMetaData1 customMetaData1 = new MergableCustomMetaData1("a");
+        MergableCustomMetaData1 customMetaData2 = new MergableCustomMetaData1("b");
+        try (Releasable tribeNode = startTribeNode()) {
+            putCustomMetaData(cluster1, customMetaData1);
+            putCustomMetaData(cluster2, customMetaData2);
+            assertCustomMetaDataUpdated(internalCluster(), customMetaData2);
+            removeCustomMetaData(cluster2, customMetaData2.getWriteableName());
+            assertCustomMetaDataUpdated(internalCluster(), customMetaData1);
+        }
+    }
 
-        logger.info("verify they are there");
-        assertHitCount(tribeClient.prepareSearch().setSize(0).get(), 4L);
-        assertHitCount(tribeClient.prepareSearch().get(), 4L);
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                ClusterState tribeState = tribeNode.client().admin().cluster().prepareState().get().getState();
-                assertThat(tribeState.getMetaData().index("test1").mapping("type1"), notNullValue());
-                assertThat(tribeState.getMetaData().index("test1").mapping("type2"), notNullValue());
-                assertThat(tribeState.getMetaData().index("test2").mapping("type1"), notNullValue());
-                assertThat(tribeState.getMetaData().index("test2").mapping("type2"), notNullValue());
-            }
-        });
+    public void testMergingCustomMetaData() throws Exception {
+        removeCustomMetaData(cluster1, MergableCustomMetaData1.TYPE);
+        removeCustomMetaData(cluster2, MergableCustomMetaData1.TYPE);
+        MergableCustomMetaData1 customMetaData1 = new MergableCustomMetaData1(randomAlphaOfLength(10));
+        MergableCustomMetaData1 customMetaData2 = new MergableCustomMetaData1(randomAlphaOfLength(10));
+        List<MergableCustomMetaData1> customMetaDatas = Arrays.asList(customMetaData1, customMetaData2);
+        Collections.sort(customMetaDatas, (cm1, cm2) -> cm2.getData().compareTo(cm1.getData()));
+        final MergableCustomMetaData1 tribeNodeCustomMetaData = customMetaDatas.get(0);
+        try (Releasable tribeNode = startTribeNode()) {
+            putCustomMetaData(cluster1, customMetaData1);
+            assertCustomMetaDataUpdated(internalCluster(), customMetaData1);
+            putCustomMetaData(cluster2, customMetaData2);
+            assertCustomMetaDataUpdated(internalCluster(), tribeNodeCustomMetaData);
+        }
+    }
 
-        logger.info("make sure master level write operations fail... (we don't really have a master)");
-        try {
-            tribeClient.admin().indices().prepareCreate("tribe_index").setMasterNodeTimeout("10ms").get();
-            fail();
-        } catch (MasterNotDiscoveredException e) {
-            // all is well!
+    public void testMergingMultipleCustomMetaData() throws Exception {
+        removeCustomMetaData(cluster1, MergableCustomMetaData1.TYPE);
+        removeCustomMetaData(cluster2, MergableCustomMetaData1.TYPE);
+        MergableCustomMetaData1 firstCustomMetaDataType1 = new MergableCustomMetaData1(randomAlphaOfLength(10));
+        MergableCustomMetaData1 secondCustomMetaDataType1 = new MergableCustomMetaData1(randomAlphaOfLength(10));
+        MergableCustomMetaData2 firstCustomMetaDataType2 = new MergableCustomMetaData2(randomAlphaOfLength(10));
+        MergableCustomMetaData2 secondCustomMetaDataType2 = new MergableCustomMetaData2(randomAlphaOfLength(10));
+        List<MergableCustomMetaData1> mergedCustomMetaDataType1 = Arrays.asList(firstCustomMetaDataType1, secondCustomMetaDataType1);
+        List<MergableCustomMetaData2> mergedCustomMetaDataType2 = Arrays.asList(firstCustomMetaDataType2, secondCustomMetaDataType2);
+        Collections.sort(mergedCustomMetaDataType1, (cm1, cm2) -> cm2.getData().compareTo(cm1.getData()));
+        Collections.sort(mergedCustomMetaDataType2, (cm1, cm2) -> cm2.getData().compareTo(cm1.getData()));
+        try (Releasable tribeNode = startTribeNode()) {
+            // test putting multiple custom md types propagates to tribe
+            putCustomMetaData(cluster1, firstCustomMetaDataType1);
+            putCustomMetaData(cluster1, firstCustomMetaDataType2);
+            assertCustomMetaDataUpdated(internalCluster(), firstCustomMetaDataType1);
+            assertCustomMetaDataUpdated(internalCluster(), firstCustomMetaDataType2);
+
+            // test multiple same type custom md is merged and propagates to tribe
+            putCustomMetaData(cluster2, secondCustomMetaDataType1);
+            assertCustomMetaDataUpdated(internalCluster(), firstCustomMetaDataType2);
+            assertCustomMetaDataUpdated(internalCluster(), mergedCustomMetaDataType1.get(0));
+
+            // test multiple same type custom md is merged and propagates to tribe
+            putCustomMetaData(cluster2, secondCustomMetaDataType2);
+            assertCustomMetaDataUpdated(internalCluster(), mergedCustomMetaDataType1.get(0));
+            assertCustomMetaDataUpdated(internalCluster(), mergedCustomMetaDataType2.get(0));
+
+            // test removing custom md is propagates to tribe
+            removeCustomMetaData(cluster2, secondCustomMetaDataType1.getWriteableName());
+            assertCustomMetaDataUpdated(internalCluster(), firstCustomMetaDataType1);
+            assertCustomMetaDataUpdated(internalCluster(), mergedCustomMetaDataType2.get(0));
+            removeCustomMetaData(cluster2, secondCustomMetaDataType2.getWriteableName());
+            assertCustomMetaDataUpdated(internalCluster(), firstCustomMetaDataType1);
+            assertCustomMetaDataUpdated(internalCluster(), firstCustomMetaDataType2);
         }
+    }
 
-        logger.info("delete an index, and make sure its reflected");
-        cluster2.client().admin().indices().prepareDelete("test2").get();
-        awaitIndicesNotInClusterState("test2");
+    private static void assertCustomMetaDataUpdated(InternalTestCluster cluster,
+                                                    TestCustomMetaData expectedCustomMetaData) throws Exception {
+        assertBusy(() -> {
+            ClusterState tribeState = cluster.getInstance(ClusterService.class, cluster.getNodeNames()[0]).state();
+            MetaData.Custom custom = tribeState.metaData().custom(expectedCustomMetaData.getWriteableName());
+            assertNotNull(custom);
+            assertThat(custom, equalTo(expectedCustomMetaData));
+        });
+    }
 
-        try {
-            logger.info("stop a node, make sure its reflected");
-            cluster2.stopRandomDataNode();
-            awaitSameNodeCounts();
-        } finally {
-            cluster2.startNode();
-            awaitSameNodeCounts();
-        }
+    private void removeCustomMetaData(InternalTestCluster cluster, final String customMetaDataType) {
+        logger.info("removing custom_md type [{}] from [{}]", customMetaDataType, cluster.getClusterName());
+        updateMetaData(cluster, builder -> builder.removeCustom(customMetaDataType));
     }
 
-    public void testCloseAndOpenIndex() throws Exception {
-        //create an index and close it even before starting the tribe node
-        assertAcked(internalCluster().client().admin().indices().prepareCreate("test1"));
-        ensureGreen(internalCluster());
-        assertAcked(internalCluster().client().admin().indices().prepareClose("test1"));
-
-        setupTribeNode(Settings.EMPTY);
-        awaitSameNodeCounts();
-
-        //the closed index is not part of the tribe node cluster state
-        ClusterState tribeState = tribeNode.client().admin().cluster().prepareState().get().getState();
-        assertThat(tribeState.getMetaData().hasIndex("test1"), equalTo(false));
-
-        //open the index, it becomes part of the tribe node cluster state
-        assertAcked(internalCluster().client().admin().indices().prepareOpen("test1"));
-        awaitIndicesInClusterState("test1");
-        ensureGreen(internalCluster());
-
-        //create a second index, wait till it is seen from within the tribe node
-        assertAcked(cluster2.client().admin().indices().prepareCreate("test2"));
-        awaitIndicesInClusterState("test1", "test2");
-        ensureGreen(cluster2);
-
-        //close the second index, wait till it gets removed from the tribe node cluster state
-        assertAcked(cluster2.client().admin().indices().prepareClose("test2"));
-        awaitIndicesNotInClusterState("test2");
-
-        //open the second index, wait till it gets added back to the tribe node cluster state
-        assertAcked(cluster2.client().admin().indices().prepareOpen("test2"));
-        awaitIndicesInClusterState("test1", "test2");
-        ensureGreen(cluster2);
+    private void putCustomMetaData(InternalTestCluster cluster, final TestCustomMetaData customMetaData) {
+        logger.info("putting custom_md type [{}] with data[{}] from [{}]", customMetaData.getWriteableName(),
+                customMetaData.getData(), cluster.getClusterName());
+        updateMetaData(cluster, builder -> builder.putCustom(customMetaData.getWriteableName(), customMetaData));
     }
 
-    private void awaitIndicesInClusterState(final String... indices) throws Exception {
-        assertBusy(new Runnable() {
+    private static void updateMetaData(InternalTestCluster cluster, UnaryOperator<MetaData.Builder> addCustoms) {
+        ClusterService clusterService = cluster.getInstance(ClusterService.class, cluster.getMasterName());
+        final CountDownLatch latch = new CountDownLatch(1);
+        clusterService.submitStateUpdateTask("update customMetaData", new ClusterStateUpdateTask(Priority.IMMEDIATE) {
             @Override
-            public void run() {
-                ClusterState tribeState = tribeNode.client().admin().cluster().prepareState().get().getState();
-                for (String index : indices) {
-                    assertTrue(tribeState.getMetaData().hasIndex(index));
-                    assertTrue(tribeState.getRoutingTable().hasIndex(index));
-                }
+            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                latch.countDown();
             }
-        });
-    }
 
-    private void awaitIndicesNotInClusterState(final String... indices) throws Exception {
-        assertBusy(new Runnable() {
             @Override
-            public void run() {
-                ClusterState tribeState = tribeNode.client().admin().cluster().prepareState().get().getState();
-                for (String index : indices) {
-                    assertFalse(tribeState.getMetaData().hasIndex(index));
-                    assertFalse(tribeState.getRoutingTable().hasIndex(index));
-                }
+            public ClusterState execute(ClusterState currentState) throws Exception {
+                MetaData.Builder builder = MetaData.builder(currentState.metaData());
+                builder = addCustoms.apply(builder);
+                return ClusterState.builder(currentState).metaData(builder).build();
             }
-        });
-    }
 
-    private void ensureGreen(TestCluster testCluster) {
-        ClusterHealthResponse actionGet = testCluster.client().admin().cluster()
-                .health(Requests.clusterHealthRequest().waitForGreenStatus().waitForEvents(Priority.LANGUID).waitForNoRelocatingShards(true)).actionGet();
-        if (actionGet.isTimedOut()) {
-            logger.info("ensureGreen timed out, cluster state:\n{}\n{}", testCluster.client().admin().cluster().prepareState().get().getState().prettyPrint(), testCluster.client().admin().cluster().preparePendingClusterTasks().get().prettyPrint());
-            assertThat("timed out waiting for green state", actionGet.isTimedOut(), equalTo(false));
+            @Override
+            public void onFailure(String source, Exception e) {
+                fail("failed to apply cluster state from [" + source + "] with " + e.getMessage());
+            }
+        });
+        try {
+            latch.await(1, TimeUnit.MINUTES);
+        } catch (InterruptedException e) {
+            fail("latch waiting on publishing custom md interrupted [" + e.getMessage() + "]");
         }
-        assertThat(actionGet.getStatus(), equalTo(ClusterHealthStatus.GREEN));
+        assertThat("timed out trying to add custom metadata to " + cluster.getClusterName(), latch.getCount(), equalTo(0L));
+
     }
 
-    private void awaitSameNodeCounts() throws Exception {
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                DiscoveryNodes tribeNodes = tribeNode.client().admin().cluster().prepareState().get().getState().getNodes();
-                assertThat(countDataNodesForTribe("t1", tribeNodes), equalTo(internalCluster().client().admin().cluster().prepareState().get().getState().getNodes().getDataNodes().size()));
-                assertThat(countDataNodesForTribe("t2", tribeNodes), equalTo(cluster2.client().admin().cluster().prepareState().get().getState().getNodes().getDataNodes().size()));
+    private void assertIndicesExist(Client client, String... indices) throws Exception {
+        assertBusy(() -> {
+            ClusterState state = client.admin().cluster().prepareState().setRoutingTable(true).setMetaData(true).get().getState();
+            assertThat(state.getMetaData().getIndices().size(), equalTo(indices.length));
+            for (String index : indices) {
+                assertTrue(state.getMetaData().hasIndex(index));
+                assertTrue(state.getRoutingTable().hasIndex(index));
             }
         });
     }
 
-    private int countDataNodesForTribe(String tribeName, DiscoveryNodes nodes) {
-        int count = 0;
-        for (DiscoveryNode node : nodes) {
-            if (!node.isDataNode()) {
-                continue;
-            }
-            if (tribeName.equals(node.getAttributes().get("tribe.name"))) {
-                count++;
-            }
-        }
-        return count;
+    private void ensureGreen(Client client) throws Exception {
+        assertBusy(() -> {
+            ClusterHealthResponse clusterHealthResponse = client.admin().cluster() .prepareHealth()
+                    .setWaitForActiveShards(0)
+                    .setWaitForEvents(Priority.LANGUID)
+                    .setWaitForNoRelocatingShards(true)
+                    .get();
+            assertThat(clusterHealthResponse.getStatus(), equalTo(ClusterHealthStatus.GREEN));
+            assertFalse(clusterHealthResponse.isTimedOut());
+        });
     }
 
-    public String[] getUnicastHosts(Client client) {
-        ArrayList<String> unicastHosts = new ArrayList<>();
-        NodesInfoResponse nodeInfos = client.admin().cluster().prepareNodesInfo().clear().setTransport(true).get();
-        for (NodeInfo info : nodeInfos.getNodes()) {
-            TransportAddress address = info.getTransport().getAddress().publishAddress();
-            unicastHosts.add(address.getAddress() + ":" + address.getPort());
-        }
-        return unicastHosts.toArray(new String[unicastHosts.size()]);
+    private static void doWithAllClusters(Consumer<InternalTestCluster> consumer) {
+        doWithAllClusters(cluster -> cluster != null, consumer);
+    }
+
+    private static void doWithAllClusters(Predicate<InternalTestCluster> predicate, Consumer<InternalTestCluster> consumer) {
+        Stream.of(cluster1, cluster2).filter(predicate).forEach(consumer);
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/tribe/TribeServiceTests.java b/core/src/test/java/org/elasticsearch/tribe/TribeServiceTests.java
index 43ee8fee15192..0be58079878f0 100644
--- a/core/src/test/java/org/elasticsearch/tribe/TribeServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/tribe/TribeServiceTests.java
@@ -19,9 +19,32 @@
 
 package org.elasticsearch.tribe;
 
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.NamedDiff;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.network.NetworkModule;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.node.MockNode;
+import org.elasticsearch.node.Node;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.TestCustomMetaData;
+import org.elasticsearch.transport.MockTcpTransportPlugin;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.instanceOf;
 
 public class TribeServiceTests extends ESTestCase {
     public void testMinimalSettings() {
@@ -38,7 +61,7 @@ public void testMinimalSettings() {
         assertEquals("false", clientSettings.get("node.ingest"));
         assertEquals("false", clientSettings.get("node.local_storage"));
         assertEquals("3707202549613653169", clientSettings.get("node.id.seed")); // should be fixed by the parent id and tribe name
-        assertEquals(9, clientSettings.getAsMap().size());
+        assertEquals(9, clientSettings.size());
     }
 
     public void testEnvironmentSettings() {
@@ -60,6 +83,7 @@ public void testEnvironmentSettings() {
             TribeService.buildClientSettings("tribe1", "parent_id", globalSettings, tribeSettings);
         });
         assertTrue(e.getMessage(), e.getMessage().contains("Setting [path.home] not allowed in tribe client"));
+        assertSettingDeprecationsAndWarnings(new Setting[] {Environment.PATH_SCRIPTS_SETTING});
     }
 
     public void testPassthroughSettings() {
@@ -96,4 +120,157 @@ public void testPassthroughSettings() {
         assertEquals("7.7.7.7", clientSettings.get("transport.bind_host"));
         assertEquals("8.8.8.8", clientSettings.get("transport.publish_host"));
     }
+
+    public void testMergeCustomMetaDataSimple() {
+        Map<String, MetaData.Custom> mergedCustoms =
+                TribeService.mergeChangedCustomMetaData(Collections.singleton(MergableCustomMetaData1.TYPE),
+                        s -> Collections.singletonList(new MergableCustomMetaData1("data1")));
+        TestCustomMetaData mergedCustom = (TestCustomMetaData) mergedCustoms.get(MergableCustomMetaData1.TYPE);
+        assertThat(mergedCustom, instanceOf(MergableCustomMetaData1.class));
+        assertNotNull(mergedCustom);
+        assertEquals(mergedCustom.getData(), "data1");
+    }
+
+    public void testMergeCustomMetaData() {
+        Map<String, MetaData.Custom> mergedCustoms =
+                TribeService.mergeChangedCustomMetaData(Collections.singleton(MergableCustomMetaData1.TYPE),
+                        s -> Arrays.asList(new MergableCustomMetaData1("data1"), new MergableCustomMetaData1("data2")));
+        TestCustomMetaData mergedCustom = (TestCustomMetaData) mergedCustoms.get(MergableCustomMetaData1.TYPE);
+        assertThat(mergedCustom, instanceOf(MergableCustomMetaData1.class));
+        assertNotNull(mergedCustom);
+        assertEquals(mergedCustom.getData(), "data2");
+    }
+
+    public void testMergeMultipleCustomMetaData() {
+        Map<String, List<TribeService.MergableCustomMetaData>> inputMap = new HashMap<>();
+        inputMap.put(MergableCustomMetaData1.TYPE,
+                Arrays.asList(new MergableCustomMetaData1("data10"), new MergableCustomMetaData1("data11")));
+        inputMap.put(MergableCustomMetaData2.TYPE,
+                Arrays.asList(new MergableCustomMetaData2("data21"), new MergableCustomMetaData2("data20")));
+        Map<String, MetaData.Custom> mergedCustoms = TribeService.mergeChangedCustomMetaData(
+                        Sets.newHashSet(MergableCustomMetaData1.TYPE, MergableCustomMetaData2.TYPE), inputMap::get);
+        TestCustomMetaData mergedCustom = (TestCustomMetaData) mergedCustoms.get(MergableCustomMetaData1.TYPE);
+        assertNotNull(mergedCustom);
+        assertThat(mergedCustom, instanceOf(MergableCustomMetaData1.class));
+        assertEquals(mergedCustom.getData(), "data11");
+        mergedCustom = (TestCustomMetaData) mergedCustoms.get(MergableCustomMetaData2.TYPE);
+        assertNotNull(mergedCustom);
+        assertThat(mergedCustom, instanceOf(MergableCustomMetaData2.class));
+        assertEquals(mergedCustom.getData(), "data21");
+    }
+
+    public void testMergeCustomMetaDataFromMany() {
+        Map<String, List<TribeService.MergableCustomMetaData>> inputMap = new HashMap<>();
+        int n = randomIntBetween(3, 5);
+        List<TribeService.MergableCustomMetaData> customList1 = new ArrayList<>();
+        for (int i = 0; i <= n; i++) {
+            customList1.add(new MergableCustomMetaData1("data1"+String.valueOf(i)));
+        }
+        Collections.shuffle(customList1, random());
+        inputMap.put(MergableCustomMetaData1.TYPE, customList1);
+        List<TribeService.MergableCustomMetaData> customList2 = new ArrayList<>();
+        for (int i = 0; i <= n; i++) {
+            customList2.add(new MergableCustomMetaData2("data2"+String.valueOf(i)));
+        }
+        Collections.shuffle(customList2, random());
+        inputMap.put(MergableCustomMetaData2.TYPE, customList2);
+
+        Map<String, MetaData.Custom> mergedCustoms = TribeService.mergeChangedCustomMetaData(
+                        Sets.newHashSet(MergableCustomMetaData1.TYPE, MergableCustomMetaData2.TYPE), inputMap::get);
+        TestCustomMetaData mergedCustom = (TestCustomMetaData) mergedCustoms.get(MergableCustomMetaData1.TYPE);
+        assertNotNull(mergedCustom);
+        assertThat(mergedCustom, instanceOf(MergableCustomMetaData1.class));
+        assertEquals(mergedCustom.getData(), "data1"+String.valueOf(n));
+        mergedCustom = (TestCustomMetaData) mergedCustoms.get(MergableCustomMetaData2.TYPE);
+        assertNotNull(mergedCustom);
+        assertThat(mergedCustom, instanceOf(MergableCustomMetaData2.class));
+        assertEquals(mergedCustom.getData(), "data2"+String.valueOf(n));
+    }
+
+    public void testTribeNodeDeprecation() throws IOException {
+        final Path tempDir = createTempDir();
+        Settings.Builder settings = Settings.builder()
+            .put("node.name", "test-node")
+            .put("path.home", tempDir)
+            .put(NetworkModule.HTTP_ENABLED.getKey(), false)
+            .put(NetworkModule.TRANSPORT_TYPE_SETTING.getKey(), "mock-socket-network");
+
+        final boolean tribeServiceEnable = randomBoolean();
+        if (tribeServiceEnable) {
+            String clusterName = "single-node-cluster";
+            String tribeSetting = "tribe." + clusterName + ".";
+            settings.put(tribeSetting + ClusterName.CLUSTER_NAME_SETTING.getKey(), clusterName)
+                .put(tribeSetting + NetworkModule.TRANSPORT_TYPE_SETTING.getKey(), "mock-socket-network");
+        }
+        try (Node node = new MockNode(settings.build(),Collections.singleton(MockTcpTransportPlugin.class) )) {
+            if (tribeServiceEnable) {
+                assertWarnings("tribe nodes are deprecated in favor of cross-cluster search and will be removed in Elasticsearch 7.0.0");
+            }
+        }
+    }
+
+    static class MergableCustomMetaData1 extends TestCustomMetaData
+            implements TribeService.MergableCustomMetaData<MergableCustomMetaData1> {
+        public static final String TYPE = "custom_md_1";
+
+        protected MergableCustomMetaData1(String data) {
+            super(data);
+        }
+
+        @Override
+        public String getWriteableName() {
+            return TYPE;
+        }
+
+        public static MergableCustomMetaData1 readFrom(StreamInput in) throws IOException {
+            return readFrom(MergableCustomMetaData1::new, in);
+        }
+
+        public static NamedDiff<MetaData.Custom> readDiffFrom(StreamInput in) throws IOException {
+            return readDiffFrom(TYPE, in);
+        }
+
+        @Override
+        public EnumSet<MetaData.XContentContext> context() {
+            return EnumSet.of(MetaData.XContentContext.GATEWAY);
+        }
+
+        @Override
+        public MergableCustomMetaData1 merge(MergableCustomMetaData1 other) {
+            return (getData().compareTo(other.getData()) >= 0) ? this : other;
+        }
+    }
+
+    static class MergableCustomMetaData2 extends TestCustomMetaData
+            implements TribeService.MergableCustomMetaData<MergableCustomMetaData2> {
+        public static final String TYPE = "custom_md_2";
+
+        protected MergableCustomMetaData2(String data) {
+            super(data);
+        }
+
+        @Override
+        public String getWriteableName() {
+            return TYPE;
+        }
+
+        public static MergableCustomMetaData2 readFrom(StreamInput in) throws IOException {
+            return readFrom(MergableCustomMetaData2::new, in);
+        }
+
+        public static NamedDiff<MetaData.Custom> readDiffFrom(StreamInput in) throws IOException {
+            return readDiffFrom(TYPE, in);
+        }
+
+
+        @Override
+        public EnumSet<MetaData.XContentContext> context() {
+            return EnumSet.of(MetaData.XContentContext.GATEWAY);
+        }
+
+        @Override
+        public MergableCustomMetaData2 merge(MergableCustomMetaData2 other) {
+            return (getData().compareTo(other.getData()) >= 0) ? this : other;
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java b/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java
index 87bf66d64b1ac..5716e57c96fb7 100644
--- a/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java
+++ b/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java
@@ -117,7 +117,7 @@ public void testSimpleTTL() throws Exception {
 
         // realtime get check
         long currentTime = System.currentTimeMillis();
-        GetResponse getResponse = client().prepareGet("test", "type1", "1").setFields("_ttl").get();
+        GetResponse getResponse = client().prepareGet("test", "type1", "1").setStoredFields("_ttl").get();
         long ttl0;
         if (getResponse.isExists()) {
             ttl0 = ((Number) getResponse.getField("_ttl").getValue()).longValue();
@@ -127,7 +127,7 @@ public void testSimpleTTL() throws Exception {
         }
         // verify the ttl is still decreasing when going to the replica
         currentTime = System.currentTimeMillis();
-        getResponse = client().prepareGet("test", "type1", "1").setFields("_ttl").get();
+        getResponse = client().prepareGet("test", "type1", "1").setStoredFields("_ttl").get();
         if (getResponse.isExists()) {
             ttl0 = ((Number) getResponse.getField("_ttl").getValue()).longValue();
             assertThat(ttl0, lessThanOrEqualTo(providedTTLValue - (currentTime - now)));
@@ -136,7 +136,7 @@ public void testSimpleTTL() throws Exception {
         }
         // non realtime get (stored)
         currentTime = System.currentTimeMillis();
-        getResponse = client().prepareGet("test", "type1", "1").setFields("_ttl").setRealtime(false).get();
+        getResponse = client().prepareGet("test", "type1", "1").setStoredFields("_ttl").setRealtime(false).get();
         if (getResponse.isExists()) {
             ttl0 = ((Number) getResponse.getField("_ttl").getValue()).longValue();
             assertThat(ttl0, lessThanOrEqualTo(providedTTLValue - (currentTime - now)));
@@ -145,7 +145,7 @@ public void testSimpleTTL() throws Exception {
         }
         // non realtime get going the replica
         currentTime = System.currentTimeMillis();
-        getResponse = client().prepareGet("test", "type1", "1").setFields("_ttl").setRealtime(false).get();
+        getResponse = client().prepareGet("test", "type1", "1").setStoredFields("_ttl").setRealtime(false).get();
         if (getResponse.isExists()) {
             ttl0 = ((Number) getResponse.getField("_ttl").getValue()).longValue();
             assertThat(ttl0, lessThanOrEqualTo(providedTTLValue - (currentTime - now)));
@@ -154,10 +154,10 @@ public void testSimpleTTL() throws Exception {
         }
 
         // no TTL provided so no TTL fetched
-        getResponse = client().prepareGet("test", "type1", "no_ttl").setFields("_ttl").setRealtime(true).execute().actionGet();
+        getResponse = client().prepareGet("test", "type1", "no_ttl").setStoredFields("_ttl").setRealtime(true).execute().actionGet();
         assertThat(getResponse.getField("_ttl"), nullValue());
         // no TTL provided make sure it has default TTL
-        getResponse = client().prepareGet("test", "type2", "default_ttl").setFields("_ttl").setRealtime(true).execute().actionGet();
+        getResponse = client().prepareGet("test", "type2", "default_ttl").setStoredFields("_ttl").setRealtime(true).execute().actionGet();
         ttl0 = ((Number) getResponse.getField("_ttl").getValue()).longValue();
         assertThat(ttl0, greaterThan(0L));
 
@@ -190,28 +190,28 @@ public void testSimpleTTL() throws Exception {
         ));
 
         // realtime get check
-        getResponse = client().prepareGet("test", "type1", "1").setFields("_ttl").setRealtime(true).execute().actionGet();
+        getResponse = client().prepareGet("test", "type1", "1").setStoredFields("_ttl").setRealtime(true).execute().actionGet();
         assertThat(getResponse.isExists(), equalTo(false));
-        getResponse = client().prepareGet("test", "type1", "with_routing").setRouting("routing").setFields("_ttl").setRealtime(true).execute().actionGet();
+        getResponse = client().prepareGet("test", "type1", "with_routing").setRouting("routing").setStoredFields("_ttl").setRealtime(true).execute().actionGet();
         assertThat(getResponse.isExists(), equalTo(false));
         // replica realtime get check
-        getResponse = client().prepareGet("test", "type1", "1").setFields("_ttl").setRealtime(true).execute().actionGet();
+        getResponse = client().prepareGet("test", "type1", "1").setStoredFields("_ttl").setRealtime(true).execute().actionGet();
         assertThat(getResponse.isExists(), equalTo(false));
-        getResponse = client().prepareGet("test", "type1", "with_routing").setRouting("routing").setFields("_ttl").setRealtime(true).execute().actionGet();
+        getResponse = client().prepareGet("test", "type1", "with_routing").setRouting("routing").setStoredFields("_ttl").setRealtime(true).execute().actionGet();
         assertThat(getResponse.isExists(), equalTo(false));
 
         // Need to run a refresh, in order for the non realtime get to work.
         client().admin().indices().prepareRefresh("test").execute().actionGet();
 
         // non realtime get (stored) check
-        getResponse = client().prepareGet("test", "type1", "1").setFields("_ttl").setRealtime(false).execute().actionGet();
+        getResponse = client().prepareGet("test", "type1", "1").setStoredFields("_ttl").setRealtime(false).execute().actionGet();
         assertThat(getResponse.isExists(), equalTo(false));
-        getResponse = client().prepareGet("test", "type1", "with_routing").setRouting("routing").setFields("_ttl").setRealtime(false).execute().actionGet();
+        getResponse = client().prepareGet("test", "type1", "with_routing").setRouting("routing").setStoredFields("_ttl").setRealtime(false).execute().actionGet();
         assertThat(getResponse.isExists(), equalTo(false));
         // non realtime get going the replica check
-        getResponse = client().prepareGet("test", "type1", "1").setFields("_ttl").setRealtime(false).execute().actionGet();
+        getResponse = client().prepareGet("test", "type1", "1").setStoredFields("_ttl").setRealtime(false).execute().actionGet();
         assertThat(getResponse.isExists(), equalTo(false));
-        getResponse = client().prepareGet("test", "type1", "with_routing").setRouting("routing").setFields("_ttl").setRealtime(false).execute().actionGet();
+        getResponse = client().prepareGet("test", "type1", "with_routing").setRouting("routing").setStoredFields("_ttl").setRealtime(false).execute().actionGet();
         assertThat(getResponse.isExists(), equalTo(false));
     }
 
@@ -287,7 +287,7 @@ private long updateAndGetTtl(UpdateRequestBuilder update) {
     }
 
     private long getTtl(String type, Object id) {
-        GetResponse getResponse = client().prepareGet("test", type, id.toString()).setFields("_ttl").execute()
+        GetResponse getResponse = client().prepareGet("test", type, id.toString()).setStoredFields("_ttl").execute()
                 .actionGet();
         return ((Number) getResponse.getField("_ttl").getValue()).longValue();
     }
diff --git a/core/src/test/java/org/elasticsearch/update/TimestampTTLBWIT.java b/core/src/test/java/org/elasticsearch/update/TimestampTTLBWIT.java
index e81b4decb2d43..39f1c7746344a 100644
--- a/core/src/test/java/org/elasticsearch/update/TimestampTTLBWIT.java
+++ b/core/src/test/java/org/elasticsearch/update/TimestampTTLBWIT.java
@@ -31,7 +31,7 @@
 import org.elasticsearch.index.engine.DocumentMissingException;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHitField;
 import org.elasticsearch.search.sort.SortOrder;
@@ -115,7 +115,7 @@ public void testUpdate() throws Exception {
 
         try {
             client().prepareUpdate(indexOrAlias(), "type1", "1")
-                    .setScript(new Script("field", ScriptService.ScriptType.INLINE, "field_inc", null)).execute().actionGet();
+                    .setScript(new Script(ScriptType.INLINE, "field_inc", "field", Collections.emptyMap())).execute().actionGet();
             fail();
         } catch (DocumentMissingException e) {
             // all is well
@@ -123,20 +123,20 @@ public void testUpdate() throws Exception {
 
         // check TTL is kept after an update without TTL
         client().prepareIndex("test", "type1", "2").setSource("field", 1).setTTL(86400000L).setRefreshPolicy(IMMEDIATE).get();
-        GetResponse getResponse = client().prepareGet("test", "type1", "2").setFields("_ttl").execute().actionGet();
+        GetResponse getResponse = client().prepareGet("test", "type1", "2").setStoredFields("_ttl").execute().actionGet();
         long ttl = ((Number) getResponse.getField("_ttl").getValue()).longValue();
         assertThat(ttl, greaterThan(0L));
         client().prepareUpdate(indexOrAlias(), "type1", "2")
-                .setScript(new Script("field", ScriptService.ScriptType.INLINE, "field_inc", null)).execute().actionGet();
-        getResponse = client().prepareGet("test", "type1", "2").setFields("_ttl").execute().actionGet();
+                .setScript(new Script(ScriptType.INLINE, "field_inc", "field", Collections.emptyMap())).execute().actionGet();
+        getResponse = client().prepareGet("test", "type1", "2").setStoredFields("_ttl").execute().actionGet();
         ttl = ((Number) getResponse.getField("_ttl").getValue()).longValue();
         assertThat(ttl, greaterThan(0L));
 
         // check TTL update
         client().prepareUpdate(indexOrAlias(), "type1", "2")
-                .setScript(new Script("", ScriptService.ScriptType.INLINE, "put_values",
-                        Collections.singletonMap("_ctx", Collections.singletonMap("_ttl", 3600000)))).execute().actionGet();
-        getResponse = client().prepareGet("test", "type1", "2").setFields("_ttl").execute().actionGet();
+                .setScript(new Script(ScriptType.INLINE, "put_values", "",
+                    Collections.singletonMap("_ctx", Collections.singletonMap("_ttl", 3600000)))).execute().actionGet();
+        getResponse = client().prepareGet("test", "type1", "2").setStoredFields("_ttl").execute().actionGet();
         ttl = ((Number) getResponse.getField("_ttl").getValue()).longValue();
         assertThat(ttl, greaterThan(0L));
         assertThat(ttl, lessThanOrEqualTo(3600000L));
@@ -144,10 +144,10 @@ public void testUpdate() throws Exception {
         // check timestamp update
         client().prepareIndex("test", "type1", "3").setSource("field", 1).setRefreshPolicy(IMMEDIATE).get();
         client().prepareUpdate(indexOrAlias(), "type1", "3")
-                .setScript(new Script("", ScriptService.ScriptType.INLINE, "put_values",
-                        Collections.singletonMap("_ctx", Collections.singletonMap("_timestamp", "2009-11-15T14:12:12")))).execute()
+                .setScript(new Script(ScriptType.INLINE, "put_values", "",
+                    Collections.singletonMap("_ctx", Collections.singletonMap("_timestamp", "2009-11-15T14:12:12")))).execute()
                 .actionGet();
-        getResponse = client().prepareGet("test", "type1", "3").setFields("_timestamp").execute().actionGet();
+        getResponse = client().prepareGet("test", "type1", "3").setStoredFields("_timestamp").execute().actionGet();
         long timestamp = ((Number) getResponse.getField("_timestamp").getValue()).longValue();
         assertThat(timestamp, equalTo(1258294332000L));
     }
@@ -198,7 +198,7 @@ public void testContextVariables() throws Exception {
         // Update the first object and note context variables values
         UpdateResponse updateResponse = client().prepareUpdate("test", "subtype1", "id1")
                 .setRouting("routing1")
-                .setScript(new Script("", ScriptService.ScriptType.INLINE, "extract_ctx", null))
+                .setScript(new Script(ScriptType.INLINE, "extract_ctx", "", Collections.emptyMap()))
                 .execute().actionGet();
 
         assertEquals(2, updateResponse.getVersion());
@@ -215,7 +215,7 @@ public void testContextVariables() throws Exception {
 
         // Idem with the second object
         updateResponse = client().prepareUpdate("test", "type1", "parentId1")
-                .setScript(new Script("", ScriptService.ScriptType.INLINE, "extract_ctx", null))
+                .setScript(new Script(ScriptType.INLINE, "extract_ctx", "", Collections.emptyMap()))
                 .execute().actionGet();
 
         assertEquals(2, updateResponse.getVersion());
diff --git a/core/src/test/java/org/elasticsearch/update/UpdateByNativeScriptIT.java b/core/src/test/java/org/elasticsearch/update/UpdateByNativeScriptIT.java
index 31ce9861e4532..ea218c5858908 100644
--- a/core/src/test/java/org/elasticsearch/update/UpdateByNativeScriptIT.java
+++ b/core/src/test/java/org/elasticsearch/update/UpdateByNativeScriptIT.java
@@ -26,8 +26,7 @@
 import org.elasticsearch.script.NativeScriptEngineService;
 import org.elasticsearch.script.NativeScriptFactory;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptModule;
-import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
@@ -61,7 +60,7 @@ public void testThatUpdateUsingNativeScriptWorks() throws Exception {
         Map<String, Object> params = new HashMap<>();
         params.put("foo", "SETVALUE");
         client().prepareUpdate("test", "type", "1")
-                .setScript(new Script("custom", ScriptService.ScriptType.INLINE, NativeScriptEngineService.NAME, params)).get();
+                .setScript(new Script(ScriptType.INLINE, NativeScriptEngineService.NAME, "custom", params)).get();
 
         Map<String, Object> data = client().prepareGet("test", "type", "1").get().getSource();
         assertThat(data, hasKey("foo"));
@@ -94,7 +93,7 @@ static class CustomScript extends AbstractExecutableScript {
         private Map<String, Object> params;
         private Map<String, Object> vars = new HashMap<>(2);
 
-        public CustomScript(Map<String, Object> params) {
+        CustomScript(Map<String, Object> params) {
             this.params = params;
         }
 
diff --git a/core/src/test/java/org/elasticsearch/update/UpdateIT.java b/core/src/test/java/org/elasticsearch/update/UpdateIT.java
index 7ea68dc10f6b6..5bf3d97638074 100644
--- a/core/src/test/java/org/elasticsearch/update/UpdateIT.java
+++ b/core/src/test/java/org/elasticsearch/update/UpdateIT.java
@@ -34,6 +34,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.MergePolicyConfig;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.engine.DocumentMissingException;
@@ -44,7 +45,7 @@
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptEngineService;
-import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.lookup.SearchLookup;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -369,7 +370,7 @@ public void testUpsert() throws Exception {
 
         UpdateResponse updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1")
                 .setUpsert(XContentFactory.jsonBuilder().startObject().field("field", 1).endObject())
-                .setScript(new Script("field", ScriptService.ScriptType.INLINE, "field_inc", null))
+                .setScript(new Script(ScriptType.INLINE, "field_inc", "field", Collections.emptyMap()))
                 .execute().actionGet();
         assertEquals(DocWriteResponse.Result.CREATED, updateResponse.getResult());
         assertThat(updateResponse.getIndex(), equalTo("test"));
@@ -381,7 +382,7 @@ public void testUpsert() throws Exception {
 
         updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1")
                 .setUpsert(XContentFactory.jsonBuilder().startObject().field("field", 1).endObject())
-                .setScript(new Script("field", ScriptService.ScriptType.INLINE, "field_inc", null))
+                .setScript(new Script(ScriptType.INLINE, "field_inc", "field", Collections.emptyMap()))
                 .execute().actionGet();
         assertEquals(DocWriteResponse.Result.UPDATED, updateResponse.getResult());
         assertThat(updateResponse.getIndex(), equalTo("test"));
@@ -410,7 +411,7 @@ public void testScriptedUpsert() throws Exception {
         UpdateResponse updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1")
                 .setUpsert(XContentFactory.jsonBuilder().startObject().field("balance", openingBalance).endObject())
                 .setScriptedUpsert(true)
-                .setScript(new Script("", ScriptService.ScriptType.INLINE, "scripted_upsert", params))
+                .setScript(new Script(ScriptType.INLINE, "scripted_upsert", "", params))
                 .execute().actionGet();
         assertEquals(DocWriteResponse.Result.CREATED, updateResponse.getResult());
         assertThat(updateResponse.getIndex(), equalTo("test"));
@@ -424,7 +425,7 @@ public void testScriptedUpsert() throws Exception {
         updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1")
                 .setUpsert(XContentFactory.jsonBuilder().startObject().field("balance", openingBalance).endObject())
                 .setScriptedUpsert(true)
-                .setScript(new Script("", ScriptService.ScriptType.INLINE, "scripted_upsert", params))
+                .setScript(new Script(ScriptType.INLINE, "scripted_upsert", "", params))
                 .execute().actionGet();
         assertEquals(DocWriteResponse.Result.UPDATED, updateResponse.getResult());
         assertThat(updateResponse.getIndex(), equalTo("test"));
@@ -468,8 +469,8 @@ public void testUpsertFields() throws Exception {
 
         UpdateResponse updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1")
                 .setUpsert(XContentFactory.jsonBuilder().startObject().field("bar", "baz").endObject())
-                .setScript(new Script("", ScriptService.ScriptType.INLINE, "put_values", Collections.singletonMap("extra", "foo")))
-                .setFields("_source")
+                .setScript(new Script(ScriptType.INLINE, "put_values", "", Collections.singletonMap("extra", "foo")))
+                .setFetchSource(true)
                 .execute().actionGet();
 
         assertThat(updateResponse.getIndex(), equalTo("test"));
@@ -480,7 +481,7 @@ public void testUpsertFields() throws Exception {
 
         updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1")
                 .setUpsert(XContentFactory.jsonBuilder().startObject().field("bar", "baz").endObject())
-                .setScript(new Script("", ScriptService.ScriptType.INLINE, "put_values", Collections.singletonMap("extra", "foo")))
+                .setScript(new Script(ScriptType.INLINE, "put_values", "", Collections.singletonMap("extra", "foo")))
                 .setFields("_source")
                 .execute().actionGet();
 
@@ -498,24 +499,24 @@ public void testVersionedUpdate() throws Exception {
         index("test", "type", "1", "text", "value"); // version is now 1
 
         assertThrows(client().prepareUpdate(indexOrAlias(), "type", "1")
-                        .setScript(new Script("", ScriptService.ScriptType.INLINE, "put_values", Collections.singletonMap("text", "v2"))).setVersion(2)
+                        .setScript(new Script(ScriptType.INLINE, "put_values", "", Collections.singletonMap("text", "v2"))).setVersion(2)
                         .execute(),
                 VersionConflictEngineException.class);
 
         client().prepareUpdate(indexOrAlias(), "type", "1")
-                .setScript(new Script("", ScriptService.ScriptType.INLINE, "put_values", Collections.singletonMap("text", "v2"))).setVersion(1).get();
+                .setScript(new Script(ScriptType.INLINE, "put_values", "", Collections.singletonMap("text", "v2"))).setVersion(1).get();
         assertThat(client().prepareGet("test", "type", "1").get().getVersion(), equalTo(2L));
 
         // and again with a higher version..
         client().prepareUpdate(indexOrAlias(), "type", "1")
-                .setScript(new Script("", ScriptService.ScriptType.INLINE, "put_values", Collections.singletonMap("text", "v3"))).setVersion(2).get();
+                .setScript(new Script(ScriptType.INLINE, "put_values", "", Collections.singletonMap("text", "v3"))).setVersion(2).get();
 
         assertThat(client().prepareGet("test", "type", "1").get().getVersion(), equalTo(3L));
 
         // after delete
         client().prepareDelete("test", "type", "1").get();
         assertThrows(client().prepareUpdate("test", "type", "1")
-                        .setScript(new Script("", ScriptService.ScriptType.INLINE, "put_values", Collections.singletonMap("text", "v2"))).setVersion(3)
+                        .setScript(new Script(ScriptType.INLINE, "put_values", "", Collections.singletonMap("text", "v2"))).setVersion(3)
                         .execute(),
                 DocumentMissingException.class);
 
@@ -523,20 +524,26 @@ public void testVersionedUpdate() throws Exception {
         client().prepareIndex("test", "type", "2").setSource("text", "value").setVersion(10).setVersionType(VersionType.EXTERNAL).get();
 
         assertThrows(client().prepareUpdate(indexOrAlias(), "type", "2")
-                        .setScript(new Script("", ScriptService.ScriptType.INLINE, "put_values", Collections.singletonMap("text", "v2"))).setVersion(2)
+                        .setScript(new Script(ScriptType.INLINE, "put_values", "", Collections.singletonMap("text", "v2"))).setVersion(2)
                         .setVersionType(VersionType.EXTERNAL).execute(),
                 ActionRequestValidationException.class);
 
+
+        // With force version
+        client().prepareUpdate(indexOrAlias(), "type", "2")
+                .setScript(new Script(ScriptType.INLINE, "put_values", "", Collections.singletonMap("text", "v10")))
+                .setVersion(10).setVersionType(VersionType.FORCE).get();
+
         GetResponse get = get("test", "type", "2");
         assertThat(get.getVersion(), equalTo(10L));
-        assertThat((String) get.getSource().get("text"), equalTo("value"));
+        assertThat((String) get.getSource().get("text"), equalTo("v10"));
 
         // upserts - the combination with versions is a bit weird. Test are here to ensure we do not change our behavior unintentionally
 
         // With internal versions, tt means "if object is there with version X, update it or explode. If it is not there, index.
         client().prepareUpdate(indexOrAlias(), "type", "3")
-                .setScript(new Script("", ScriptService.ScriptType.INLINE, "put_values", Collections.singletonMap("text", "v2")))
-                .setVersion(10).setUpsert("{ \"text\": \"v0\" }").get();
+                .setScript(new Script(ScriptType.INLINE, "put_values", "", Collections.singletonMap("text", "v2")))
+                .setVersion(10).setUpsert("{ \"text\": \"v0\" }", XContentType.JSON).get();
         get = get("test", "type", "3");
         assertThat(get.getVersion(), equalTo(1L));
         assertThat((String) get.getSource().get("text"), equalTo("v0"));
@@ -548,8 +555,8 @@ public void testVersionedUpdate() throws Exception {
     public void testIndexAutoCreation() throws Exception {
         UpdateResponse updateResponse = client().prepareUpdate("test", "type1", "1")
                 .setUpsert(XContentFactory.jsonBuilder().startObject().field("bar", "baz").endObject())
-                .setScript(new Script("", ScriptService.ScriptType.INLINE, "put_values", Collections.singletonMap("extra", "foo")))
-                .setFields("_source")
+                .setScript(new Script(ScriptType.INLINE, "put_values", "", Collections.singletonMap("extra", "foo")))
+                .setFetchSource(true)
                 .execute().actionGet();
 
         assertThat(updateResponse.getIndex(), equalTo("test"));
@@ -565,7 +572,7 @@ public void testUpdate() throws Exception {
 
         try {
             client().prepareUpdate(indexOrAlias(), "type1", "1")
-                    .setScript(new Script("field", ScriptService.ScriptType.INLINE, "field_inc", null)).execute().actionGet();
+                    .setScript(new Script(ScriptType.INLINE, "field_inc", "field", Collections.emptyMap())).execute().actionGet();
             fail();
         } catch (DocumentMissingException e) {
             // all is well
@@ -574,7 +581,7 @@ public void testUpdate() throws Exception {
         client().prepareIndex("test", "type1", "1").setSource("field", 1).execute().actionGet();
 
         UpdateResponse updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1")
-                .setScript(new Script("field", ScriptService.ScriptType.INLINE, "field_inc", null)).execute().actionGet();
+                .setScript(new Script(ScriptType.INLINE, "field_inc", "field", Collections.emptyMap())).execute().actionGet();
         assertThat(updateResponse.getVersion(), equalTo(2L));
         assertEquals(DocWriteResponse.Result.UPDATED, updateResponse.getResult());
         assertThat(updateResponse.getIndex(), equalTo("test"));
@@ -587,7 +594,7 @@ public void testUpdate() throws Exception {
         Map<String, Object> params = new HashMap<>();
         params.put("inc", 3);
         updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1")
-                .setScript(new Script("field", ScriptService.ScriptType.INLINE, "field_inc", params)).execute().actionGet();
+                .setScript(new Script(ScriptType.INLINE, "field_inc", "field", params)).execute().actionGet();
         assertThat(updateResponse.getVersion(), equalTo(3L));
         assertEquals(DocWriteResponse.Result.UPDATED, updateResponse.getResult());
         assertThat(updateResponse.getIndex(), equalTo("test"));
@@ -599,7 +606,7 @@ public void testUpdate() throws Exception {
 
         // check noop
         updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1")
-                .setScript(new Script("", ScriptService.ScriptType.INLINE, "put_values", Collections.singletonMap("_ctx", Collections.singletonMap("op", "none")))).execute().actionGet();
+                .setScript(new Script(ScriptType.INLINE, "put_values", "", Collections.singletonMap("_ctx", Collections.singletonMap("op", "none")))).execute().actionGet();
         assertThat(updateResponse.getVersion(), equalTo(3L));
         assertEquals(DocWriteResponse.Result.NOOP, updateResponse.getResult());
         assertThat(updateResponse.getIndex(), equalTo("test"));
@@ -611,7 +618,7 @@ public void testUpdate() throws Exception {
 
         // check delete
         updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1")
-                .setScript(new Script("", ScriptService.ScriptType.INLINE, "put_values", Collections.singletonMap("_ctx", Collections.singletonMap("op", "delete")))).execute().actionGet();
+                .setScript(new Script(ScriptType.INLINE, "put_values", "", Collections.singletonMap("_ctx", Collections.singletonMap("op", "delete")))).execute().actionGet();
         assertThat(updateResponse.getVersion(), equalTo(4L));
         assertEquals(DocWriteResponse.Result.DELETED, updateResponse.getResult());
         assertThat(updateResponse.getIndex(), equalTo("test"));
@@ -624,14 +631,30 @@ public void testUpdate() throws Exception {
         // check fields parameter
         client().prepareIndex("test", "type1", "1").setSource("field", 1).execute().actionGet();
         updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1")
-                .setScript(new Script("field", ScriptService.ScriptType.INLINE, "field_inc", null)).setFields("_source", "field")
-                .execute().actionGet();
+            .setScript(new Script(ScriptType.INLINE, "field_inc", "field", Collections.emptyMap()))
+            .setFields("field")
+            .setFetchSource(true)
+            .execute().actionGet();
         assertThat(updateResponse.getIndex(), equalTo("test"));
         assertThat(updateResponse.getGetResult(), notNullValue());
         assertThat(updateResponse.getGetResult().getIndex(), equalTo("test"));
         assertThat(updateResponse.getGetResult().sourceRef(), notNullValue());
         assertThat(updateResponse.getGetResult().field("field").getValue(), notNullValue());
 
+        // check _source parameter
+        client().prepareIndex("test", "type1", "1").setSource("field1", 1, "field2", 2).execute().actionGet();
+        updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1")
+            .setScript(new Script(ScriptType.INLINE, "field_inc", "field1", Collections.emptyMap()))
+            .setFetchSource("field1", "field2")
+            .get();
+        assertThat(updateResponse.getIndex(), equalTo("test"));
+        assertThat(updateResponse.getGetResult(), notNullValue());
+        assertThat(updateResponse.getGetResult().getIndex(), equalTo("test"));
+        assertThat(updateResponse.getGetResult().sourceRef(), notNullValue());
+        assertThat(updateResponse.getGetResult().field("field1"), nullValue());
+        assertThat(updateResponse.getGetResult().sourceAsMap().size(), equalTo(1));
+        assertThat(updateResponse.getGetResult().sourceAsMap().get("field1"), equalTo(2));
+
         // check updates without script
         // add new field
         client().prepareIndex("test", "type1", "1").setSource("field", 1).execute().actionGet();
@@ -684,7 +707,7 @@ public void testUpdateRequestWithBothScriptAndDoc() throws Exception {
         try {
             client().prepareUpdate(indexOrAlias(), "type1", "1")
                     .setDoc(XContentFactory.jsonBuilder().startObject().field("field", 1).endObject())
-                    .setScript(new Script("field", ScriptService.ScriptType.INLINE, "field_inc", null))
+                    .setScript(new Script(ScriptType.INLINE, "field_inc", "field", Collections.emptyMap()))
                     .execute().actionGet();
             fail("Should have thrown ActionRequestValidationException");
         } catch (ActionRequestValidationException e) {
@@ -699,7 +722,7 @@ public void testUpdateRequestWithScriptAndShouldUpsertDoc() throws Exception {
         ensureGreen();
         try {
             client().prepareUpdate(indexOrAlias(), "type1", "1")
-                    .setScript(new Script("field", ScriptService.ScriptType.INLINE, "field_inc", null))
+                    .setScript(new Script(ScriptType.INLINE, "field_inc", "field", Collections.emptyMap()))
                     .setDocAsUpsert(true)
                     .execute().actionGet();
             fail("Should have thrown ActionRequestValidationException");
@@ -751,7 +774,7 @@ public void testContextVariables() throws Exception {
         // Update the first object and note context variables values
         UpdateResponse updateResponse = client().prepareUpdate("test", "subtype1", "id1")
                 .setRouting("routing1")
-                .setScript(new Script("", ScriptService.ScriptType.INLINE, "extract_ctx", null))
+                .setScript(new Script(ScriptType.INLINE, "extract_ctx", "", Collections.emptyMap()))
                 .execute().actionGet();
 
         assertEquals(2, updateResponse.getVersion());
@@ -767,7 +790,7 @@ public void testContextVariables() throws Exception {
 
         // Idem with the second object
         updateResponse = client().prepareUpdate("test", "type1", "parentId1")
-                .setScript(new Script("", ScriptService.ScriptType.INLINE, "extract_ctx", null))
+                .setScript(new Script(ScriptType.INLINE, "extract_ctx", "", Collections.emptyMap()))
                 .execute().actionGet();
 
         assertEquals(2, updateResponse.getVersion());
@@ -806,13 +829,13 @@ public void run() {
                             }
                             if (useBulkApi) {
                                 UpdateRequestBuilder updateRequestBuilder = client().prepareUpdate(indexOrAlias(), "type1", Integer.toString(i))
-                                        .setScript(new Script("field", ScriptService.ScriptType.INLINE, "field_inc", null))
+                                        .setScript(new Script(ScriptType.INLINE, "field_inc", "field", Collections.emptyMap()))
                                         .setRetryOnConflict(Integer.MAX_VALUE)
                                         .setUpsert(jsonBuilder().startObject().field("field", 1).endObject());
                                 client().prepareBulk().add(updateRequestBuilder).execute().actionGet();
                             } else {
                                 client().prepareUpdate(indexOrAlias(), "type1", Integer.toString(i))
-                                        .setScript(new Script("field", ScriptService.ScriptType.INLINE, "field_inc", null))
+                                        .setScript(new Script(ScriptType.INLINE, "field_inc", "field", Collections.emptyMap()))
                                         .setRetryOnConflict(Integer.MAX_VALUE)
                                         .setUpsert(jsonBuilder().startObject().field("field", 1).endObject())
                                         .execute().actionGet();
@@ -874,7 +897,7 @@ final class UpdateThread extends Thread {
             private final Semaphore updateRequestsOutstanding = new Semaphore(maxUpdateRequests);
             private final Semaphore deleteRequestsOutstanding = new Semaphore(maxDeleteRequests);
 
-            public UpdateThread(int numberOfIds, int updatesPerId) {
+            UpdateThread(int numberOfIds, int updatesPerId) {
                 this.numberOfIds = numberOfIds;
                 this.updatesPerId = updatesPerId;
             }
@@ -882,7 +905,7 @@ public UpdateThread(int numberOfIds, int updatesPerId) {
             final class UpdateListener implements ActionListener<UpdateResponse> {
                 int id;
 
-                public UpdateListener(int id) {
+                UpdateListener(int id) {
                     this.id = id;
                 }
 
@@ -904,7 +927,7 @@ public void onFailure(Exception e) {
             final class DeleteListener implements ActionListener<DeleteResponse> {
                 int id;
 
-                public DeleteListener(int id) {
+                DeleteListener(int id) {
                     this.id = id;
                 }
 
@@ -932,7 +955,7 @@ public void run(){
                             updateRequestsOutstanding.acquire();
                             try {
                                 UpdateRequest ur = client().prepareUpdate("test", "type1", Integer.toString(j))
-                                        .setScript(new Script("field", ScriptService.ScriptType.INLINE, "field_inc", null))
+                                        .setScript(new Script(ScriptType.INLINE, "field_inc", "field", Collections.emptyMap()))
                                         .setRetryOnConflict(retryOnConflict)
                                         .setUpsert(jsonBuilder().startObject().field("field", 1).endObject())
                                         .request();
@@ -1032,7 +1055,7 @@ private void waitForOutstandingRequests(TimeValue timeOut, Semaphore requestsOut
         //All the previous operations should be complete or failed at this point
         for (int i = 0; i < numberOfIdsPerThread; ++i) {
             UpdateResponse ur = client().prepareUpdate("test", "type1", Integer.toString(i))
-                    .setScript(new Script("field", ScriptService.ScriptType.INLINE, "field_inc", null))
+                    .setScript(new Script(ScriptType.INLINE, "field_inc", "field", Collections.emptyMap()))
                 .setRetryOnConflict(Integer.MAX_VALUE)
                 .setUpsert(jsonBuilder().startObject().field("field", 1).endObject())
                 .execute().actionGet();
diff --git a/core/src/test/java/org/elasticsearch/update/UpdateNoopIT.java b/core/src/test/java/org/elasticsearch/update/UpdateNoopIT.java
index f14d91465f6e2..ae232c2c6872b 100644
--- a/core/src/test/java/org/elasticsearch/update/UpdateNoopIT.java
+++ b/core/src/test/java/org/elasticsearch/update/UpdateNoopIT.java
@@ -51,9 +51,9 @@ public void testSingleField() throws Exception {
 
     public void testTwoFields() throws Exception {
         // Use random keys so we get random iteration order.
-        String key1 = 1 + randomAsciiOfLength(3);
-        String key2 = 2 + randomAsciiOfLength(3);
-        String key3 = 3 + randomAsciiOfLength(3);
+        String key1 = 1 + randomAlphaOfLength(3);
+        String key2 = 2 + randomAlphaOfLength(3);
+        String key3 = 3 + randomAlphaOfLength(3);
         updateAndCheckSource(1, fields(key1, "foo", key2, "baz"));
         updateAndCheckSource(1, fields(key1, "foo", key2, "baz"));
         updateAndCheckSource(2, fields(key1, "foo", key2, "bir"));
@@ -90,9 +90,9 @@ public void testArrayField() throws Exception {
 
     public void testMap() throws Exception {
         // Use random keys so we get variable iteration order.
-        String key1 = 1 + randomAsciiOfLength(3);
-        String key2 = 2 + randomAsciiOfLength(3);
-        String key3 = 3 + randomAsciiOfLength(3);
+        String key1 = 1 + randomAlphaOfLength(3);
+        String key2 = 2 + randomAlphaOfLength(3);
+        String key3 = 3 + randomAlphaOfLength(3);
         updateAndCheckSource(1, XContentFactory.jsonBuilder().startObject()
                 .startObject("test")
                     .field(key1, "foo")
@@ -245,7 +245,7 @@ private void updateAndCheckSource(long expectedVersion, Boolean detectNoop, XCon
 
     private UpdateResponse update(Boolean detectNoop, long expectedVersion, XContentBuilder xContentBuilder) {
         UpdateRequestBuilder updateRequest = client().prepareUpdate("test", "type1", "1")
-                .setDoc(xContentBuilder.bytes().utf8ToString())
+                .setDoc(xContentBuilder)
                 .setDocAsUpsert(true)
                 .setFields("_source");
         if (detectNoop != null) {
diff --git a/core/src/test/java/org/elasticsearch/validate/SimpleValidateQueryIT.java b/core/src/test/java/org/elasticsearch/validate/SimpleValidateQueryIT.java
index 79ee4389527e7..5bcac4db906e7 100644
--- a/core/src/test/java/org/elasticsearch/validate/SimpleValidateQueryIT.java
+++ b/core/src/test/java/org/elasticsearch/validate/SimpleValidateQueryIT.java
@@ -39,11 +39,14 @@
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.List;
 
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
 import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
+import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
@@ -255,6 +258,35 @@ public void testExplainWithRewriteValidateQuery() throws Exception {
                 containsString("field:huge field:pidgin"), true);
     }
 
+    public void testExplainWithRewriteValidateQueryAllShards() throws Exception {
+        client().admin().indices().prepareCreate("test")
+            .addMapping("type1", "field", "type=text,analyzer=whitespace")
+            .setSettings(SETTING_NUMBER_OF_SHARDS, 2).get();
+        // We are relying on specific routing behaviors for the result to be right, so
+        // we cannot randomize the number of shards or change ids here.
+        client().prepareIndex("test", "type1", "1")
+            .setSource("field", "quick lazy huge brown pidgin").get();
+        client().prepareIndex("test", "type1", "2")
+            .setSource("field", "the quick brown fox").get();
+        client().prepareIndex("test", "type1", "3")
+            .setSource("field", "the quick lazy huge brown fox jumps over the tree").get();
+        client().prepareIndex("test", "type1", "4")
+            .setSource("field", "the lazy dog quacks like a duck").get();
+        refresh();
+
+        // prefix queries
+        assertExplanations(QueryBuilders.matchPhrasePrefixQuery("field", "qu"),
+            Arrays.asList(
+                equalTo("field:quick"),
+                allOf(containsString("field:quick"), containsString("field:quacks"))
+            ), true, true);
+        assertExplanations(QueryBuilders.matchPhrasePrefixQuery("field", "ju"),
+            Arrays.asList(
+                equalTo("field:jumps"),
+                equalTo("field:\"ju*\"")
+            ), true, true);
+    }
+
     public void testIrrelevantPropertiesBeforeQuery() throws IOException {
         createIndex("test");
         ensureGreen();
@@ -283,4 +315,22 @@ private static void assertExplanation(QueryBuilder queryBuilder, Matcher<String>
         assertThat(response.getQueryExplanation().get(0).getExplanation(), matcher);
         assertThat(response.isValid(), equalTo(true));
     }
+
+    private static void assertExplanations(QueryBuilder queryBuilder,
+                                           List<Matcher<String>> matchers, boolean withRewrite,
+                                           boolean allShards) {
+        ValidateQueryResponse response = client().admin().indices().prepareValidateQuery("test")
+            .setTypes("type1")
+            .setQuery(queryBuilder)
+            .setExplain(true)
+            .setRewrite(withRewrite)
+            .setAllShards(allShards)
+            .execute().actionGet();
+        assertThat(response.getQueryExplanation().size(), equalTo(matchers.size()));
+        for (int i = 0; i < matchers.size(); i++) {
+            assertThat(response.getQueryExplanation().get(i).getError(), nullValue());
+            assertThat(response.getQueryExplanation().get(i).getExplanation(), matchers.get(i));
+            assertThat(response.isValid(), equalTo(true));
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/versioning/SimpleVersioningIT.java b/core/src/test/java/org/elasticsearch/versioning/SimpleVersioningIT.java
index b80c5bd8e2c87..5914c66fa4967 100644
--- a/core/src/test/java/org/elasticsearch/versioning/SimpleVersioningIT.java
+++ b/core/src/test/java/org/elasticsearch/versioning/SimpleVersioningIT.java
@@ -20,16 +20,15 @@
 
 import org.apache.lucene.util.TestUtil;
 import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.delete.DeleteResponse;
-import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.VersionType;
-import org.elasticsearch.index.engine.FlushNotAllowedEngineException;
 import org.elasticsearch.index.engine.VersionConflictEngineException;
 import org.elasticsearch.test.ESIntegTestCase;
 
@@ -72,6 +71,36 @@ public void testExternalVersioningInitialDelete() throws Exception {
         assertThat(indexResponse.getVersion(), equalTo(18L));
     }
 
+    public void testForce() throws Exception {
+        createIndex("test");
+        ensureGreen("test"); // we are testing force here which doesn't work if we are recovering at the same time - zzzzz...
+        IndexResponse indexResponse = client().prepareIndex("test", "type", "1").setSource("field1", "value1_1").setVersion(12).setVersionType(VersionType.FORCE).get();
+        assertThat(indexResponse.getVersion(), equalTo(12L));
+
+        indexResponse = client().prepareIndex("test", "type", "1").setSource("field1", "value1_2").setVersion(12).setVersionType(VersionType.FORCE).get();
+        assertThat(indexResponse.getVersion(), equalTo(12L));
+
+        indexResponse = client().prepareIndex("test", "type", "1").setSource("field1", "value1_2").setVersion(14).setVersionType(VersionType.FORCE).get();
+        assertThat(indexResponse.getVersion(), equalTo(14L));
+
+        indexResponse = client().prepareIndex("test", "type", "1").setSource("field1", "value1_1").setVersion(13).setVersionType(VersionType.FORCE).get();
+        assertThat(indexResponse.getVersion(), equalTo(13L));
+
+        client().admin().indices().prepareRefresh().execute().actionGet();
+        if (randomBoolean()) {
+            refresh();
+        }
+        for (int i = 0; i < 10; i++) {
+            assertThat(client().prepareGet("test", "type", "1").get().getVersion(), equalTo(13L));
+        }
+
+        // deleting with a lower version works.
+        long v = randomIntBetween(12, 14);
+        DeleteResponse deleteResponse = client().prepareDelete("test", "type", "1").setVersion(v).setVersionType(VersionType.FORCE).get();
+        assertEquals(DocWriteResponse.Result.DELETED, deleteResponse.getResult());
+        assertThat(deleteResponse.getVersion(), equalTo(v));
+    }
+
     public void testExternalGTE() throws Exception {
         createIndex("test");
 
@@ -329,7 +358,7 @@ private IDSource getRandomIDs() {
                 ids = new IDSource() {
                     @Override
                     public String next() {
-                        return TestUtil.randomSimpleString(random);
+                        return TestUtil.randomSimpleString(random, 1, 10);
                     }
                 };
                 break;
@@ -339,7 +368,7 @@ public String next() {
                 ids = new IDSource() {
                     @Override
                     public String next() {
-                        return TestUtil.randomRealisticUnicodeString(random);
+                        return TestUtil.randomRealisticUnicodeString(random, 1, 20);
                     }
                 };
                 break;
@@ -617,11 +646,7 @@ public void run() {
                             }
                             if (threadRandom.nextInt(100) == 7) {
                                 logger.trace("--> {}: TEST: now flush at {}", threadID, System.nanoTime() - startTime);
-                                try {
-                                    flush();
-                                } catch (FlushNotAllowedEngineException fnaee) {
-                                    // OK
-                                }
+                                flush();
                                 logger.trace("--> {}: TEST: flush done at {}", threadID, System.nanoTime() - startTime);
                             }
                         }
@@ -694,7 +719,7 @@ public void testDeleteNotLost() throws Exception {
         client()
                 .prepareIndex("test", "type", "id")
                 .setSource("foo", "bar")
-                .setOpType(IndexRequest.OpType.INDEX)
+                .setOpType(DocWriteRequest.OpType.INDEX)
                 .setVersion(10)
                 .setVersionType(VersionType.EXTERNAL)
                 .execute()
@@ -763,7 +788,7 @@ public void testGCDeletesZero() throws Exception {
         client()
                 .prepareIndex("test", "type", "id")
                 .setSource("foo", "bar")
-                .setOpType(IndexRequest.OpType.INDEX)
+                .setOpType(DocWriteRequest.OpType.INDEX)
                 .setVersion(10)
                 .setVersionType(VersionType.EXTERNAL)
                 .execute()
diff --git a/core/src/test/resources/META-INF/services/org.elasticsearch.plugins.spi.NamedXContentProvider b/core/src/test/resources/META-INF/services/org.elasticsearch.plugins.spi.NamedXContentProvider
new file mode 100644
index 0000000000000..8ec7461c6676f
--- /dev/null
+++ b/core/src/test/resources/META-INF/services/org.elasticsearch.plugins.spi.NamedXContentProvider
@@ -0,0 +1 @@
+org.elasticsearch.plugins.spi.NamedXContentProviderTests$TestNamedXContentProvider
\ No newline at end of file
diff --git a/core/src/test/resources/indices/bwc/2.0.0-rc1.zip b/core/src/test/resources/indices/bwc/2.0.0-rc1.zip
new file mode 100644
index 0000000000000..8c440725e9cd9
Binary files /dev/null and b/core/src/test/resources/indices/bwc/2.0.0-rc1.zip differ
diff --git a/core/src/test/resources/indices/bwc/compressed-repo-1.7.4.zip b/core/src/test/resources/indices/bwc/compressed-repo-1.7.4.zip
new file mode 100644
index 0000000000000..9edf7d57527e5
Binary files /dev/null and b/core/src/test/resources/indices/bwc/compressed-repo-1.7.4.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-2.0.0-beta1.zip b/core/src/test/resources/indices/bwc/index-2.0.0-beta1.zip
index ca3d11099ce32..97291291a69a8 100644
Binary files a/core/src/test/resources/indices/bwc/index-2.0.0-beta1.zip and b/core/src/test/resources/indices/bwc/index-2.0.0-beta1.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-2.0.0-beta2.zip b/core/src/test/resources/indices/bwc/index-2.0.0-beta2.zip
index 47496a9f012f2..8ea2aa0cec9f5 100644
Binary files a/core/src/test/resources/indices/bwc/index-2.0.0-beta2.zip and b/core/src/test/resources/indices/bwc/index-2.0.0-beta2.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-2.0.0-rc1.zip b/core/src/test/resources/indices/bwc/index-2.0.0-rc1.zip
deleted file mode 100644
index 3b4599594102e..0000000000000
Binary files a/core/src/test/resources/indices/bwc/index-2.0.0-rc1.zip and /dev/null differ
diff --git a/core/src/test/resources/indices/bwc/index-2.0.0.zip b/core/src/test/resources/indices/bwc/index-2.0.0.zip
index 2dae323f69e20..ad79a34baa7e2 100644
Binary files a/core/src/test/resources/indices/bwc/index-2.0.0.zip and b/core/src/test/resources/indices/bwc/index-2.0.0.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-2.0.1.zip b/core/src/test/resources/indices/bwc/index-2.0.1.zip
index 2d0d5f42d5022..ae67f158825a4 100644
Binary files a/core/src/test/resources/indices/bwc/index-2.0.1.zip and b/core/src/test/resources/indices/bwc/index-2.0.1.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-2.0.2.zip b/core/src/test/resources/indices/bwc/index-2.0.2.zip
index f6a9492b33f22..1a5ff97a99fff 100644
Binary files a/core/src/test/resources/indices/bwc/index-2.0.2.zip and b/core/src/test/resources/indices/bwc/index-2.0.2.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-2.1.0.zip b/core/src/test/resources/indices/bwc/index-2.1.0.zip
index 347d9cb31e920..5b61aa6e6321d 100644
Binary files a/core/src/test/resources/indices/bwc/index-2.1.0.zip and b/core/src/test/resources/indices/bwc/index-2.1.0.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-2.1.1.zip b/core/src/test/resources/indices/bwc/index-2.1.1.zip
index 6981c9af4a9aa..afb54ea6495b2 100644
Binary files a/core/src/test/resources/indices/bwc/index-2.1.1.zip and b/core/src/test/resources/indices/bwc/index-2.1.1.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-2.1.2.zip b/core/src/test/resources/indices/bwc/index-2.1.2.zip
index 57162675b14c0..4ee1feea85391 100644
Binary files a/core/src/test/resources/indices/bwc/index-2.1.2.zip and b/core/src/test/resources/indices/bwc/index-2.1.2.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-2.2.0.zip b/core/src/test/resources/indices/bwc/index-2.2.0.zip
index 81ff74d5abfbd..ee6ceebe65e8b 100644
Binary files a/core/src/test/resources/indices/bwc/index-2.2.0.zip and b/core/src/test/resources/indices/bwc/index-2.2.0.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-2.2.1.zip b/core/src/test/resources/indices/bwc/index-2.2.1.zip
index 7e640e4158f87..0fc5dea97edd3 100644
Binary files a/core/src/test/resources/indices/bwc/index-2.2.1.zip and b/core/src/test/resources/indices/bwc/index-2.2.1.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-2.2.2.zip b/core/src/test/resources/indices/bwc/index-2.2.2.zip
index f6c5c7653d13e..5fbaf71aebc3f 100644
Binary files a/core/src/test/resources/indices/bwc/index-2.2.2.zip and b/core/src/test/resources/indices/bwc/index-2.2.2.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-2.3.0.zip b/core/src/test/resources/indices/bwc/index-2.3.0.zip
index c09e5d8ba1936..d0f01d5cee3f5 100644
Binary files a/core/src/test/resources/indices/bwc/index-2.3.0.zip and b/core/src/test/resources/indices/bwc/index-2.3.0.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-2.3.1.zip b/core/src/test/resources/indices/bwc/index-2.3.1.zip
index de10f7926dff4..75198c2d092fb 100644
Binary files a/core/src/test/resources/indices/bwc/index-2.3.1.zip and b/core/src/test/resources/indices/bwc/index-2.3.1.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-2.3.2.zip b/core/src/test/resources/indices/bwc/index-2.3.2.zip
index eff6c8cd15627..74f6dd91b796d 100644
Binary files a/core/src/test/resources/indices/bwc/index-2.3.2.zip and b/core/src/test/resources/indices/bwc/index-2.3.2.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-2.3.3.zip b/core/src/test/resources/indices/bwc/index-2.3.3.zip
index 751819741b314..0b8b963f9f17b 100644
Binary files a/core/src/test/resources/indices/bwc/index-2.3.3.zip and b/core/src/test/resources/indices/bwc/index-2.3.3.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-2.3.4.zip b/core/src/test/resources/indices/bwc/index-2.3.4.zip
index b69f100398a79..06d78147cfcb3 100644
Binary files a/core/src/test/resources/indices/bwc/index-2.3.4.zip and b/core/src/test/resources/indices/bwc/index-2.3.4.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-2.3.5.zip b/core/src/test/resources/indices/bwc/index-2.3.5.zip
index dd64e69995456..485ab7b9cc3b9 100644
Binary files a/core/src/test/resources/indices/bwc/index-2.3.5.zip and b/core/src/test/resources/indices/bwc/index-2.3.5.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-2.4.0.zip b/core/src/test/resources/indices/bwc/index-2.4.0.zip
index 14bd436b1642c..377faf38ef8de 100644
Binary files a/core/src/test/resources/indices/bwc/index-2.4.0.zip and b/core/src/test/resources/indices/bwc/index-2.4.0.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-2.4.1.zip b/core/src/test/resources/indices/bwc/index-2.4.1.zip
new file mode 100644
index 0000000000000..febb6a3bcdb56
Binary files /dev/null and b/core/src/test/resources/indices/bwc/index-2.4.1.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-2.4.2.zip b/core/src/test/resources/indices/bwc/index-2.4.2.zip
new file mode 100644
index 0000000000000..35e8d0f66254a
Binary files /dev/null and b/core/src/test/resources/indices/bwc/index-2.4.2.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-2.4.3.zip b/core/src/test/resources/indices/bwc/index-2.4.3.zip
new file mode 100644
index 0000000000000..b4af34aed86bb
Binary files /dev/null and b/core/src/test/resources/indices/bwc/index-2.4.3.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-2.4.4.zip b/core/src/test/resources/indices/bwc/index-2.4.4.zip
new file mode 100644
index 0000000000000..696e865284165
Binary files /dev/null and b/core/src/test/resources/indices/bwc/index-2.4.4.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-2.4.5.zip b/core/src/test/resources/indices/bwc/index-2.4.5.zip
new file mode 100644
index 0000000000000..253944a3aee09
Binary files /dev/null and b/core/src/test/resources/indices/bwc/index-2.4.5.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-2.4.6.zip b/core/src/test/resources/indices/bwc/index-2.4.6.zip
new file mode 100644
index 0000000000000..3d24b00d3df56
Binary files /dev/null and b/core/src/test/resources/indices/bwc/index-2.4.6.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-5.0.0.zip b/core/src/test/resources/indices/bwc/index-5.0.0.zip
new file mode 100644
index 0000000000000..a04b829c2cf99
Binary files /dev/null and b/core/src/test/resources/indices/bwc/index-5.0.0.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-5.0.1.zip b/core/src/test/resources/indices/bwc/index-5.0.1.zip
new file mode 100644
index 0000000000000..256e162fa92f2
Binary files /dev/null and b/core/src/test/resources/indices/bwc/index-5.0.1.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-5.0.2.zip b/core/src/test/resources/indices/bwc/index-5.0.2.zip
new file mode 100644
index 0000000000000..d02b44fbeff1a
Binary files /dev/null and b/core/src/test/resources/indices/bwc/index-5.0.2.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-5.1.1.zip b/core/src/test/resources/indices/bwc/index-5.1.1.zip
new file mode 100644
index 0000000000000..9df43e70f4d55
Binary files /dev/null and b/core/src/test/resources/indices/bwc/index-5.1.1.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-5.1.2.zip b/core/src/test/resources/indices/bwc/index-5.1.2.zip
new file mode 100644
index 0000000000000..021735879d104
Binary files /dev/null and b/core/src/test/resources/indices/bwc/index-5.1.2.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-5.2.0.zip b/core/src/test/resources/indices/bwc/index-5.2.0.zip
new file mode 100644
index 0000000000000..cdcb9b01a5c1c
Binary files /dev/null and b/core/src/test/resources/indices/bwc/index-5.2.0.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-5.2.1.zip b/core/src/test/resources/indices/bwc/index-5.2.1.zip
new file mode 100644
index 0000000000000..8957d724b69da
Binary files /dev/null and b/core/src/test/resources/indices/bwc/index-5.2.1.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-5.2.2.zip b/core/src/test/resources/indices/bwc/index-5.2.2.zip
new file mode 100644
index 0000000000000..89c90c0290262
Binary files /dev/null and b/core/src/test/resources/indices/bwc/index-5.2.2.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-5.3.0.zip b/core/src/test/resources/indices/bwc/index-5.3.0.zip
new file mode 100644
index 0000000000000..5d44e7fb529d0
Binary files /dev/null and b/core/src/test/resources/indices/bwc/index-5.3.0.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-5.3.1.zip b/core/src/test/resources/indices/bwc/index-5.3.1.zip
new file mode 100644
index 0000000000000..251dc0e41a149
Binary files /dev/null and b/core/src/test/resources/indices/bwc/index-5.3.1.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-5.3.2.zip b/core/src/test/resources/indices/bwc/index-5.3.2.zip
new file mode 100644
index 0000000000000..2c760d5f871fc
Binary files /dev/null and b/core/src/test/resources/indices/bwc/index-5.3.2.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-5.3.3.zip b/core/src/test/resources/indices/bwc/index-5.3.3.zip
new file mode 100644
index 0000000000000..4d6f4f348fad0
Binary files /dev/null and b/core/src/test/resources/indices/bwc/index-5.3.3.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-5.4.0.zip b/core/src/test/resources/indices/bwc/index-5.4.0.zip
new file mode 100644
index 0000000000000..e8473b3aa30bb
Binary files /dev/null and b/core/src/test/resources/indices/bwc/index-5.4.0.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-5.4.1.zip b/core/src/test/resources/indices/bwc/index-5.4.1.zip
new file mode 100644
index 0000000000000..82ad144a70811
Binary files /dev/null and b/core/src/test/resources/indices/bwc/index-5.4.1.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-5.4.2.zip b/core/src/test/resources/indices/bwc/index-5.4.2.zip
new file mode 100644
index 0000000000000..c6aef9feeaae0
Binary files /dev/null and b/core/src/test/resources/indices/bwc/index-5.4.2.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-5.4.3.zip b/core/src/test/resources/indices/bwc/index-5.4.3.zip
new file mode 100644
index 0000000000000..0ba6d1e0e1bed
Binary files /dev/null and b/core/src/test/resources/indices/bwc/index-5.4.3.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-5.5.0.zip b/core/src/test/resources/indices/bwc/index-5.5.0.zip
new file mode 100644
index 0000000000000..4d48b7bca79c3
Binary files /dev/null and b/core/src/test/resources/indices/bwc/index-5.5.0.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-5.5.1.zip b/core/src/test/resources/indices/bwc/index-5.5.1.zip
new file mode 100644
index 0000000000000..dcf57efb61bc7
Binary files /dev/null and b/core/src/test/resources/indices/bwc/index-5.5.1.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-5.5.2.zip b/core/src/test/resources/indices/bwc/index-5.5.2.zip
new file mode 100644
index 0000000000000..5f1f5e0805f86
Binary files /dev/null and b/core/src/test/resources/indices/bwc/index-5.5.2.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-5.5.3.zip b/core/src/test/resources/indices/bwc/index-5.5.3.zip
new file mode 100644
index 0000000000000..1f780f83bf9d8
Binary files /dev/null and b/core/src/test/resources/indices/bwc/index-5.5.3.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-5.6.0.zip b/core/src/test/resources/indices/bwc/index-5.6.0.zip
new file mode 100644
index 0000000000000..4d18b37a22b05
Binary files /dev/null and b/core/src/test/resources/indices/bwc/index-5.6.0.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-5.6.1.zip b/core/src/test/resources/indices/bwc/index-5.6.1.zip
new file mode 100644
index 0000000000000..1019213846ee6
Binary files /dev/null and b/core/src/test/resources/indices/bwc/index-5.6.1.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-5.6.2.zip b/core/src/test/resources/indices/bwc/index-5.6.2.zip
new file mode 100644
index 0000000000000..e5e20406ebeb1
Binary files /dev/null and b/core/src/test/resources/indices/bwc/index-5.6.2.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-5.6.3.zip b/core/src/test/resources/indices/bwc/index-5.6.3.zip
new file mode 100644
index 0000000000000..90629dee4f5a2
Binary files /dev/null and b/core/src/test/resources/indices/bwc/index-5.6.3.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-2.0.0-beta1.zip b/core/src/test/resources/indices/bwc/repo-2.0.0-beta1.zip
index 4a46dbc8382ad..20c7fa6839ca6 100644
Binary files a/core/src/test/resources/indices/bwc/repo-2.0.0-beta1.zip and b/core/src/test/resources/indices/bwc/repo-2.0.0-beta1.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-2.0.0-beta2.zip b/core/src/test/resources/indices/bwc/repo-2.0.0-beta2.zip
index 6e4080a914654..1db8e893fe657 100644
Binary files a/core/src/test/resources/indices/bwc/repo-2.0.0-beta2.zip and b/core/src/test/resources/indices/bwc/repo-2.0.0-beta2.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-2.0.0-rc1.zip b/core/src/test/resources/indices/bwc/repo-2.0.0-rc1.zip
index deb36fee11968..59b31f5cc3b97 100644
Binary files a/core/src/test/resources/indices/bwc/repo-2.0.0-rc1.zip and b/core/src/test/resources/indices/bwc/repo-2.0.0-rc1.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-2.0.0.zip b/core/src/test/resources/indices/bwc/repo-2.0.0.zip
index 8042696cb90e1..1f2580da2621d 100644
Binary files a/core/src/test/resources/indices/bwc/repo-2.0.0.zip and b/core/src/test/resources/indices/bwc/repo-2.0.0.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-2.0.1.zip b/core/src/test/resources/indices/bwc/repo-2.0.1.zip
index 6e9b3d0aedefe..dc39cab29bf4a 100644
Binary files a/core/src/test/resources/indices/bwc/repo-2.0.1.zip and b/core/src/test/resources/indices/bwc/repo-2.0.1.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-2.0.2.zip b/core/src/test/resources/indices/bwc/repo-2.0.2.zip
index 4dd61b0f26aeb..5dbef58e7dbf8 100644
Binary files a/core/src/test/resources/indices/bwc/repo-2.0.2.zip and b/core/src/test/resources/indices/bwc/repo-2.0.2.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-2.1.0.zip b/core/src/test/resources/indices/bwc/repo-2.1.0.zip
index b641e0b5bbab2..6f47aef2033c8 100644
Binary files a/core/src/test/resources/indices/bwc/repo-2.1.0.zip and b/core/src/test/resources/indices/bwc/repo-2.1.0.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-2.1.1.zip b/core/src/test/resources/indices/bwc/repo-2.1.1.zip
index e08cde10b331e..9e507a226fee4 100644
Binary files a/core/src/test/resources/indices/bwc/repo-2.1.1.zip and b/core/src/test/resources/indices/bwc/repo-2.1.1.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-2.1.2.zip b/core/src/test/resources/indices/bwc/repo-2.1.2.zip
index f9829c219f0de..ebcb3c5e873f8 100644
Binary files a/core/src/test/resources/indices/bwc/repo-2.1.2.zip and b/core/src/test/resources/indices/bwc/repo-2.1.2.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-2.2.0.zip b/core/src/test/resources/indices/bwc/repo-2.2.0.zip
index 703184dac1ea1..5dc1b17ce9174 100644
Binary files a/core/src/test/resources/indices/bwc/repo-2.2.0.zip and b/core/src/test/resources/indices/bwc/repo-2.2.0.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-2.2.1.zip b/core/src/test/resources/indices/bwc/repo-2.2.1.zip
index c665f79c11c2b..c7e6adcc550f7 100644
Binary files a/core/src/test/resources/indices/bwc/repo-2.2.1.zip and b/core/src/test/resources/indices/bwc/repo-2.2.1.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-2.2.2.zip b/core/src/test/resources/indices/bwc/repo-2.2.2.zip
index 9e5e6fdd30d62..8efcab39b06ba 100644
Binary files a/core/src/test/resources/indices/bwc/repo-2.2.2.zip and b/core/src/test/resources/indices/bwc/repo-2.2.2.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-2.3.0.zip b/core/src/test/resources/indices/bwc/repo-2.3.0.zip
index f41df41224d72..6dc684290dc15 100644
Binary files a/core/src/test/resources/indices/bwc/repo-2.3.0.zip and b/core/src/test/resources/indices/bwc/repo-2.3.0.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-2.3.1.zip b/core/src/test/resources/indices/bwc/repo-2.3.1.zip
index 78e736986ab8a..3d32fd1caa140 100644
Binary files a/core/src/test/resources/indices/bwc/repo-2.3.1.zip and b/core/src/test/resources/indices/bwc/repo-2.3.1.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-2.3.2.zip b/core/src/test/resources/indices/bwc/repo-2.3.2.zip
index b160856326af4..b54f951b56a51 100644
Binary files a/core/src/test/resources/indices/bwc/repo-2.3.2.zip and b/core/src/test/resources/indices/bwc/repo-2.3.2.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-2.3.3.zip b/core/src/test/resources/indices/bwc/repo-2.3.3.zip
index 411cbea5a224e..515acc63d8a57 100644
Binary files a/core/src/test/resources/indices/bwc/repo-2.3.3.zip and b/core/src/test/resources/indices/bwc/repo-2.3.3.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-2.3.4.zip b/core/src/test/resources/indices/bwc/repo-2.3.4.zip
index 4afa60f7c78fa..9c24ecd1bb600 100644
Binary files a/core/src/test/resources/indices/bwc/repo-2.3.4.zip and b/core/src/test/resources/indices/bwc/repo-2.3.4.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-2.3.5.zip b/core/src/test/resources/indices/bwc/repo-2.3.5.zip
index 5d2d00de96189..746979f68c26b 100644
Binary files a/core/src/test/resources/indices/bwc/repo-2.3.5.zip and b/core/src/test/resources/indices/bwc/repo-2.3.5.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-2.4.0.zip b/core/src/test/resources/indices/bwc/repo-2.4.0.zip
index c5f3c0d0759f4..a396872da3dcb 100644
Binary files a/core/src/test/resources/indices/bwc/repo-2.4.0.zip and b/core/src/test/resources/indices/bwc/repo-2.4.0.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-2.4.1.zip b/core/src/test/resources/indices/bwc/repo-2.4.1.zip
new file mode 100644
index 0000000000000..3beb57e4ddcf5
Binary files /dev/null and b/core/src/test/resources/indices/bwc/repo-2.4.1.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-2.4.2.zip b/core/src/test/resources/indices/bwc/repo-2.4.2.zip
new file mode 100644
index 0000000000000..45c67737e6a1c
Binary files /dev/null and b/core/src/test/resources/indices/bwc/repo-2.4.2.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-2.4.3.zip b/core/src/test/resources/indices/bwc/repo-2.4.3.zip
new file mode 100644
index 0000000000000..bba8762219794
Binary files /dev/null and b/core/src/test/resources/indices/bwc/repo-2.4.3.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-2.4.4.zip b/core/src/test/resources/indices/bwc/repo-2.4.4.zip
new file mode 100644
index 0000000000000..eaef762816585
Binary files /dev/null and b/core/src/test/resources/indices/bwc/repo-2.4.4.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-2.4.5.zip b/core/src/test/resources/indices/bwc/repo-2.4.5.zip
new file mode 100644
index 0000000000000..ec1effccc35be
Binary files /dev/null and b/core/src/test/resources/indices/bwc/repo-2.4.5.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-2.4.6.zip b/core/src/test/resources/indices/bwc/repo-2.4.6.zip
new file mode 100644
index 0000000000000..dfa61a1fe7dc1
Binary files /dev/null and b/core/src/test/resources/indices/bwc/repo-2.4.6.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-5.0.0.zip b/core/src/test/resources/indices/bwc/repo-5.0.0.zip
new file mode 100644
index 0000000000000..df2878e47a80f
Binary files /dev/null and b/core/src/test/resources/indices/bwc/repo-5.0.0.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-5.0.1.zip b/core/src/test/resources/indices/bwc/repo-5.0.1.zip
new file mode 100644
index 0000000000000..dd2484b8a4054
Binary files /dev/null and b/core/src/test/resources/indices/bwc/repo-5.0.1.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-5.0.2.zip b/core/src/test/resources/indices/bwc/repo-5.0.2.zip
new file mode 100644
index 0000000000000..7b8fca8a0ece2
Binary files /dev/null and b/core/src/test/resources/indices/bwc/repo-5.0.2.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-5.1.1.zip b/core/src/test/resources/indices/bwc/repo-5.1.1.zip
new file mode 100644
index 0000000000000..8300b4d0dea64
Binary files /dev/null and b/core/src/test/resources/indices/bwc/repo-5.1.1.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-5.1.2.zip b/core/src/test/resources/indices/bwc/repo-5.1.2.zip
new file mode 100644
index 0000000000000..e371b79e4c360
Binary files /dev/null and b/core/src/test/resources/indices/bwc/repo-5.1.2.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-5.2.0.zip b/core/src/test/resources/indices/bwc/repo-5.2.0.zip
new file mode 100644
index 0000000000000..ed475b9779af8
Binary files /dev/null and b/core/src/test/resources/indices/bwc/repo-5.2.0.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-5.2.1.zip b/core/src/test/resources/indices/bwc/repo-5.2.1.zip
new file mode 100644
index 0000000000000..d211c79ce0aa4
Binary files /dev/null and b/core/src/test/resources/indices/bwc/repo-5.2.1.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-5.2.2.zip b/core/src/test/resources/indices/bwc/repo-5.2.2.zip
new file mode 100644
index 0000000000000..2086c6b8e0aca
Binary files /dev/null and b/core/src/test/resources/indices/bwc/repo-5.2.2.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-5.3.0.zip b/core/src/test/resources/indices/bwc/repo-5.3.0.zip
new file mode 100644
index 0000000000000..bc0f26be5774d
Binary files /dev/null and b/core/src/test/resources/indices/bwc/repo-5.3.0.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-5.3.1.zip b/core/src/test/resources/indices/bwc/repo-5.3.1.zip
new file mode 100644
index 0000000000000..3f94bbc650875
Binary files /dev/null and b/core/src/test/resources/indices/bwc/repo-5.3.1.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-5.3.2.zip b/core/src/test/resources/indices/bwc/repo-5.3.2.zip
new file mode 100644
index 0000000000000..7ad0edcb5cff9
Binary files /dev/null and b/core/src/test/resources/indices/bwc/repo-5.3.2.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-5.3.3.zip b/core/src/test/resources/indices/bwc/repo-5.3.3.zip
new file mode 100644
index 0000000000000..9743f9c7a37ad
Binary files /dev/null and b/core/src/test/resources/indices/bwc/repo-5.3.3.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-5.4.0.zip b/core/src/test/resources/indices/bwc/repo-5.4.0.zip
new file mode 100644
index 0000000000000..53c565b477627
Binary files /dev/null and b/core/src/test/resources/indices/bwc/repo-5.4.0.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-5.4.1.zip b/core/src/test/resources/indices/bwc/repo-5.4.1.zip
new file mode 100644
index 0000000000000..c9539ee969754
Binary files /dev/null and b/core/src/test/resources/indices/bwc/repo-5.4.1.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-5.4.2.zip b/core/src/test/resources/indices/bwc/repo-5.4.2.zip
new file mode 100644
index 0000000000000..5e71fe33451e5
Binary files /dev/null and b/core/src/test/resources/indices/bwc/repo-5.4.2.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-5.4.3.zip b/core/src/test/resources/indices/bwc/repo-5.4.3.zip
new file mode 100644
index 0000000000000..e1de034b67ee6
Binary files /dev/null and b/core/src/test/resources/indices/bwc/repo-5.4.3.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-5.5.0.zip b/core/src/test/resources/indices/bwc/repo-5.5.0.zip
new file mode 100644
index 0000000000000..18a1f39d9cef9
Binary files /dev/null and b/core/src/test/resources/indices/bwc/repo-5.5.0.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-5.5.1.zip b/core/src/test/resources/indices/bwc/repo-5.5.1.zip
new file mode 100644
index 0000000000000..3745ea9e758c8
Binary files /dev/null and b/core/src/test/resources/indices/bwc/repo-5.5.1.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-5.5.2.zip b/core/src/test/resources/indices/bwc/repo-5.5.2.zip
new file mode 100644
index 0000000000000..2e5516fd20edb
Binary files /dev/null and b/core/src/test/resources/indices/bwc/repo-5.5.2.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-5.5.3.zip b/core/src/test/resources/indices/bwc/repo-5.5.3.zip
new file mode 100644
index 0000000000000..8a6d842973cd6
Binary files /dev/null and b/core/src/test/resources/indices/bwc/repo-5.5.3.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-5.6.0.zip b/core/src/test/resources/indices/bwc/repo-5.6.0.zip
new file mode 100644
index 0000000000000..06bb2e9487d6d
Binary files /dev/null and b/core/src/test/resources/indices/bwc/repo-5.6.0.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-5.6.1.zip b/core/src/test/resources/indices/bwc/repo-5.6.1.zip
new file mode 100644
index 0000000000000..b4af2fa2e8b01
Binary files /dev/null and b/core/src/test/resources/indices/bwc/repo-5.6.1.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-5.6.2.zip b/core/src/test/resources/indices/bwc/repo-5.6.2.zip
new file mode 100644
index 0000000000000..dacf126e7e25c
Binary files /dev/null and b/core/src/test/resources/indices/bwc/repo-5.6.2.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-5.6.3.zip b/core/src/test/resources/indices/bwc/repo-5.6.3.zip
new file mode 100644
index 0000000000000..949fc6199b305
Binary files /dev/null and b/core/src/test/resources/indices/bwc/repo-5.6.3.zip differ
diff --git a/core/src/test/resources/org/elasticsearch/action/admin/indices/template/filebeat-1.3.template.json b/core/src/test/resources/org/elasticsearch/action/admin/indices/template/filebeat-1.3.template.json
new file mode 100644
index 0000000000000..9bb41829e4395
--- /dev/null
+++ b/core/src/test/resources/org/elasticsearch/action/admin/indices/template/filebeat-1.3.template.json
@@ -0,0 +1,42 @@
+{
+  "mappings": {
+    "_default_": {
+      "_all": {
+        "enabled": true,
+        "norms": {
+          "enabled": false
+        }
+      },
+      "dynamic_templates": [
+        {
+          "template1": {
+            "mapping": {
+              "doc_values": true,
+              "ignore_above": 1024,
+              "index": "not_analyzed",
+              "type": "{dynamic_type}"
+            },
+            "match": "*"
+          }
+        }
+      ],
+      "properties": {
+        "@timestamp": {
+          "type": "date"
+        },
+        "message": {
+          "type": "string",
+          "index": "analyzed"
+        },
+        "offset": {
+          "type": "long",
+          "doc_values": "true"
+        }
+      }
+    }
+  },
+  "settings": {
+    "index.refresh_interval": "5s"
+  },
+  "template": "filebeat-*"
+}
diff --git a/core/src/test/resources/org/elasticsearch/action/admin/indices/template/logstash-2.x.template.json b/core/src/test/resources/org/elasticsearch/action/admin/indices/template/logstash-2.x.template.json
new file mode 100644
index 0000000000000..0126c7111a513
--- /dev/null
+++ b/core/src/test/resources/org/elasticsearch/action/admin/indices/template/logstash-2.x.template.json
@@ -0,0 +1,95 @@
+{
+  "template" : "logstash-*",
+  "settings" : {
+    "index.refresh_interval" : "5s"
+  },
+  "mappings" : {
+    "_default_" : {
+      "_all" : {"enabled" : true, "omit_norms" : true},
+      "dynamic_templates" : [ {
+        "message_field" : {
+          "match" : "message",
+          "match_mapping_type" : "string",
+          "mapping" : {
+            "type" : "string", "index" : "analyzed", "omit_norms" : true,
+            "fielddata" : { "format" : "disabled" }
+          }
+        }
+      }, {
+        "string_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "string",
+          "mapping" : {
+            "type" : "string", "index" : "analyzed", "omit_norms" : true,
+            "fielddata" : { "format" : "disabled" },
+            "fields" : {
+              "raw" : {"type": "string", "index" : "not_analyzed", "doc_values" : true, "ignore_above" : 256}
+            }
+          }
+        }
+      }, {
+        "float_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "float",
+          "mapping" : { "type" : "float", "doc_values" : true }
+        }
+      }, {
+        "double_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "double",
+          "mapping" : { "type" : "double", "doc_values" : true }
+        }
+      }, {
+        "byte_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "byte",
+          "mapping" : { "type" : "byte", "doc_values" : true }
+        }
+      }, {
+        "short_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "short",
+          "mapping" : { "type" : "short", "doc_values" : true }
+        }
+      }, {
+        "integer_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "integer",
+          "mapping" : { "type" : "integer", "doc_values" : true }
+        }
+      }, {
+        "long_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "long",
+          "mapping" : { "type" : "long", "doc_values" : true }
+        }
+      }, {
+        "date_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "date",
+          "mapping" : { "type" : "date", "doc_values" : true }
+        }
+      }, {
+        "geo_point_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "geo_point",
+          "mapping" : { "type" : "geo_point", "doc_values" : true }
+        }
+      } ],
+      "properties" : {
+        "@timestamp": { "type": "date", "doc_values" : true },
+        "@version": { "type": "string", "index": "not_analyzed", "doc_values" : true },
+        "geoip"  : {
+          "type" : "object",
+          "dynamic": true,
+          "properties" : {
+            "ip": { "type": "ip", "doc_values" : true },
+            "location" : { "type" : "geo_point", "doc_values" : true },
+            "latitude" : { "type" : "float", "doc_values" : true },
+            "longitude" : { "type" : "float", "doc_values" : true }
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/core/src/test/resources/org/elasticsearch/action/admin/indices/template/logstash-5.x.template.json b/core/src/test/resources/org/elasticsearch/action/admin/indices/template/logstash-5.x.template.json
new file mode 100644
index 0000000000000..fc6406c457ae1
--- /dev/null
+++ b/core/src/test/resources/org/elasticsearch/action/admin/indices/template/logstash-5.x.template.json
@@ -0,0 +1,45 @@
+{
+  "template" : "logstash-*",
+  "settings" : {
+    "index.refresh_interval" : "5s"
+  },
+  "mappings" : {
+    "_default_" : {
+      "_all" : {"enabled" : true, "norms" : false},
+      "dynamic_templates" : [ {
+        "message_field" : {
+          "match" : "message",
+          "match_mapping_type" : "string",
+          "mapping" : {
+            "type" : "string", "index" : "analyzed", "norms" : false,
+            "fielddata" : { "format" : "disabled" }
+          }
+        }
+      }, {
+        "string_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "string",
+          "mapping" : {
+            "type" : "text", "norms" : false,
+            "fields" : {
+              "keyword" : { "type": "keyword" }
+            }
+          }
+        }
+      } ],
+      "properties" : {
+        "@timestamp": { "type": "date", "include_in_all": false },
+        "@version": { "type": "keyword", "include_in_all": false },
+        "geoip"  : {
+          "dynamic": true,
+          "properties" : {
+            "ip": { "type": "ip" },
+            "location" : { "type" : "geo_point" },
+            "latitude" : { "type" : "half_float" },
+            "longitude" : { "type" : "half_float" }
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/core/src/test/resources/org/elasticsearch/action/admin/indices/template/packetbeat-1.3.template.json b/core/src/test/resources/org/elasticsearch/action/admin/indices/template/packetbeat-1.3.template.json
new file mode 100644
index 0000000000000..cacf9c27711b2
--- /dev/null
+++ b/core/src/test/resources/org/elasticsearch/action/admin/indices/template/packetbeat-1.3.template.json
@@ -0,0 +1,63 @@
+{
+  "mappings": {
+    "_default_": {
+      "_all": {
+        "enabled": true,
+        "norms": {
+          "enabled": false
+        }
+      },
+      "dynamic_templates": [
+        {
+          "template1": {
+            "mapping": {
+              "doc_values": true,
+              "ignore_above": 1024,
+              "index": "not_analyzed",
+              "type": "{dynamic_type}"
+            },
+            "match": "*"
+          }
+        }
+      ],
+      "properties": {
+        "@timestamp": {
+          "type": "date"
+        },
+        "client_location": {
+          "type": "geo_point"
+        },
+        "params": {
+          "index": "analyzed",
+          "norms": {
+            "enabled": false
+          },
+          "type": "string"
+        },
+        "query": {
+          "doc_values": true,
+          "index": "not_analyzed",
+          "type": "string"
+        },
+        "request": {
+          "index": "analyzed",
+          "norms": {
+            "enabled": false
+          },
+          "type": "string"
+        },
+        "response": {
+          "index": "analyzed",
+          "norms": {
+            "enabled": false
+          },
+          "type": "string"
+        }
+      }
+    }
+  },
+  "settings": {
+    "index.refresh_interval": "5s"
+  },
+  "template": "packetbeat-*"
+}
\ No newline at end of file
diff --git a/core/src/test/resources/org/elasticsearch/action/admin/indices/template/topbeat-1.3.template.json b/core/src/test/resources/org/elasticsearch/action/admin/indices/template/topbeat-1.3.template.json
new file mode 100644
index 0000000000000..9f35b2155f453
--- /dev/null
+++ b/core/src/test/resources/org/elasticsearch/action/admin/indices/template/topbeat-1.3.template.json
@@ -0,0 +1,110 @@
+{
+  "mappings": {
+    "_default_": {
+      "_all": {
+        "enabled": true,
+        "norms": {
+          "enabled": false
+        }
+      },
+      "dynamic_templates": [
+        {
+          "template1": {
+            "mapping": {
+              "doc_values": true,
+              "ignore_above": 1024,
+              "index": "not_analyzed",
+              "type": "{dynamic_type}"
+            },
+            "match": "*"
+          }
+        }
+      ],
+      "properties": {
+        "@timestamp": {
+          "type": "date"
+        },
+        "cpu": {
+          "properties": {
+            "system_p": {
+              "doc_values": "true",
+              "type": "float"
+            },
+            "user_p": {
+              "doc_values": "true",
+              "type": "float"
+            }
+          }
+        },
+        "fs": {
+          "properties": {
+            "used_p": {
+              "doc_values": "true",
+              "type": "float"
+            }
+          }
+        },
+        "load": {
+          "properties": {
+            "load1": {
+              "doc_values": "true",
+              "type": "float"
+            },
+            "load15": {
+              "doc_values": "true",
+              "type": "float"
+            },
+            "load5": {
+              "doc_values": "true",
+              "type": "float"
+            }
+          }
+        },
+        "mem": {
+          "properties": {
+            "actual_used_p": {
+              "doc_values": "true",
+              "type": "float"
+            },
+            "used_p": {
+              "doc_values": "true",
+              "type": "float"
+            }
+          }
+        },
+        "proc": {
+          "properties": {
+            "cpu": {
+              "properties": {
+                "user_p": {
+                  "doc_values": "true",
+                  "type": "float"
+                }
+              }
+            },
+            "mem": {
+              "properties": {
+                "rss_p": {
+                  "doc_values": "true",
+                  "type": "float"
+                }
+              }
+            }
+          }
+        },
+        "swap": {
+          "properties": {
+            "used_p": {
+              "doc_values": "true",
+              "type": "float"
+            }
+          }
+        }
+      }
+    }
+  },
+  "settings": {
+    "index.refresh_interval": "5s"
+  },
+  "template": "topbeat-*"
+}
diff --git a/core/src/test/resources/org/elasticsearch/action/admin/indices/template/winlogbeat-1.3.template.json b/core/src/test/resources/org/elasticsearch/action/admin/indices/template/winlogbeat-1.3.template.json
new file mode 100644
index 0000000000000..45e19590f8230
--- /dev/null
+++ b/core/src/test/resources/org/elasticsearch/action/admin/indices/template/winlogbeat-1.3.template.json
@@ -0,0 +1,38 @@
+{
+  "mappings": {
+    "_default_": {
+      "_all": {
+        "enabled": true,
+        "norms": {
+          "enabled": false
+        }
+      },
+      "dynamic_templates": [
+        {
+          "template1": {
+            "mapping": {
+              "doc_values": true,
+              "ignore_above": 1024,
+              "index": "not_analyzed",
+              "type": "{dynamic_type}"
+            },
+            "match": "*"
+          }
+        }
+      ],
+      "properties": {
+        "@timestamp": {
+          "type": "date"
+        },
+        "message": {
+          "index": "analyzed",
+          "type": "string"
+        }
+      }
+    }
+  },
+  "settings": {
+    "index.refresh_interval": "5s"
+  },
+  "template": "winlogbeat-*"
+}
diff --git a/core/src/test/resources/org/elasticsearch/action/bulk/simple-bulk4.json b/core/src/test/resources/org/elasticsearch/action/bulk/simple-bulk4.json
index 362e4a473e988..bbcc0dbd6868d 100644
--- a/core/src/test/resources/org/elasticsearch/action/bulk/simple-bulk4.json
+++ b/core/src/test/resources/org/elasticsearch/action/bulk/simple-bulk4.json
@@ -1,7 +1,7 @@
 { "update" : {"_id" : "1", "_retry_on_conflict" : 2} }
 { "doc" : {"field" : "value"} }
 { "update" : { "_id" : "0", "_type" : "type1", "_index" : "index1" } }
-{ "script" : { "inline" : "counter += param1", "lang" : "javascript", "params" : {"param1" : 1}}, "upsert" : {"counter" : 1}}
+{ "script" : { "source" : "counter += param1", "lang" : "javascript", "params" : {"param1" : 1}}, "upsert" : {"counter" : 1}}
 { "delete" : { "_id" : "2" } }
 { "create" : { "_id" : "3" } }
 { "field1" : "value3" }
diff --git a/core/src/test/resources/org/elasticsearch/search/query/all-example-document.json b/core/src/test/resources/org/elasticsearch/search/query/all-example-document.json
new file mode 100644
index 0000000000000..9e4d04930a71a
--- /dev/null
+++ b/core/src/test/resources/org/elasticsearch/search/query/all-example-document.json
@@ -0,0 +1,36 @@
+{
+  "f1": "foo",
+  "f2": "Bar",
+  "f3": "foo bar baz",
+  "f_multi": "Foo Bar Baz",
+  "f_object": {
+    "sub1": "sfoo",
+    "sub2":"sbar",
+    "sub3":19
+  },
+  "f_nested": {
+    "nest1": "nfoo",
+    "nest2":"nbar",
+    "nest3":21
+  },
+  "f_date": "1476383971",
+  "f_bool": "true",
+  "f_byte": "7",
+  "f_short": "23",
+  "f_int": "1293",
+  "f_long": "42",
+  "f_float": "1.7",
+  "f_hfloat": "1.5",
+  "f_sfloat": "12.23",
+  "f_ip": "127.0.0.1",
+  "f_binary": "VGhpcyBpcyBzb21lIGJpbmFyeSBkYXRhCg==",
+  "f_suggest": {
+    "input": ["Nevermind", "Nirvana"],
+    "weight": 34
+  },
+  "f_geop": "41.12,-71.34",
+  "f_geos": {
+    "type": "point",
+    "coordinates": [-77.03653, 38.897676]
+  }
+}
diff --git a/core/src/test/resources/org/elasticsearch/search/query/all-query-index-with-all.json b/core/src/test/resources/org/elasticsearch/search/query/all-query-index-with-all.json
new file mode 100644
index 0000000000000..05de13b4261a2
--- /dev/null
+++ b/core/src/test/resources/org/elasticsearch/search/query/all-query-index-with-all.json
@@ -0,0 +1,35 @@
+{
+  "settings": {
+    "index": {
+      "number_of_shards": 1,
+      "number_of_replicas": 0,
+      "analysis": {
+        "analyzer": {
+          "my_ngrams": {
+            "type": "custom",
+            "tokenizer": "standard",
+            "filter": ["my_ngrams"]
+          }
+        },
+        "filter": {
+          "my_ngrams": {
+            "type": "ngram",
+            "min_gram": 2,
+            "max_gram": 2
+          }
+        }
+      }
+    }
+  },
+  "mappings": {
+    "doc": {
+      "_all": {
+        "enabled": true
+      },
+      "properties": {
+        "f1": {"type": "text"},
+        "f2": {"type": "text", "analyzer": "my_ngrams"}
+      }
+    }
+  }
+}
diff --git a/core/src/test/resources/org/elasticsearch/search/query/all-query-index.json b/core/src/test/resources/org/elasticsearch/search/query/all-query-index.json
new file mode 100644
index 0000000000000..1a67abcfcb38f
--- /dev/null
+++ b/core/src/test/resources/org/elasticsearch/search/query/all-query-index.json
@@ -0,0 +1,80 @@
+{
+  "settings": {
+    "index": {
+      "number_of_shards": 1,
+      "number_of_replicas": 0,
+      "analysis": {
+        "analyzer": {
+          "my_ngrams": {
+            "type": "custom",
+            "tokenizer": "standard",
+            "filter": ["my_ngrams"]
+          }
+        },
+        "filter": {
+          "my_ngrams": {
+            "type": "ngram",
+            "min_gram": 2,
+            "max_gram": 2
+          }
+        }
+      }
+    }
+  },
+  "mappings": {
+    "doc": {
+      "_all": {
+        "enabled": false
+      },
+      "properties": {
+        "f1": {"type": "text"},
+        "f2": {"type": "keyword"},
+        "f3": {"type": "text", "analyzer": "my_ngrams"},
+        "f4": {
+          "type": "text",
+          "index_options": "docs"
+        },
+        "f_multi": {
+          "type": "text",
+          "fields": {
+            "raw": {"type": "keyword"},
+            "f_token_count": {"type": "token_count", "analyzer": "standard"}
+          }
+        },
+        "f_object": {
+          "type": "object",
+          "properties": {
+            "sub1": {"type": "text"},
+            "sub2": {"type": "keyword"},
+            "sub3": {"type": "integer"}
+          }
+        },
+        "f_nested": {
+          "type": "nested",
+          "properties": {
+            "nest1": {"type": "text"},
+            "nest2": {"type": "keyword"},
+            "nest3": {"type": "integer"}
+          }
+        },
+        "f_date": {
+          "type": "date",
+          "format": "yyyy/MM/dd||epoch_millis"
+        },
+        "f_bool": {"type": "boolean"},
+        "f_byte": {"type": "byte"},
+        "f_short": {"type": "short"},
+        "f_int": {"type": "integer"},
+        "f_long": {"type": "long"},
+        "f_float": {"type": "float"},
+        "f_hfloat": {"type": "half_float"},
+        "f_sfloat": {"type": "scaled_float", "scaling_factor": 100},
+        "f_ip": {"type": "ip"},
+        "f_binary": {"type": "binary"},
+        "f_suggest": {"type": "completion"},
+        "f_geop": {"type": "geo_point"},
+        "f_geos": {"type": "geo_shape"}
+      }
+    }
+  }
+}
diff --git a/dev-tools/create_bwc_index.py b/dev-tools/create_bwc_index.py
index 5a262d23cb63e..fe04edde01fee 100644
--- a/dev-tools/create_bwc_index.py
+++ b/dev-tools/create_bwc_index.py
@@ -15,6 +15,7 @@
 # language governing permissions and limitations under the License.
 
 import argparse
+import base64
 import glob
 import logging
 import os
@@ -47,6 +48,28 @@ def rarely():
 def frequently():
   return not rarely()
 
+def capabilities_of(version):
+  current_version = parse_version(version)
+
+  return {
+    'warmers': current_version < parse_version('2.0.0-alpha1'),
+    'dots_in_field_names': current_version >= parse_version('2.4.0'),
+    'lenient_booleans': current_version < parse_version('6.0.0-alpha1')
+  }
+
+
+def falsy(lenient):
+  return random.choice(['off', 'no', '0', 0, 'false', False]) if lenient else False
+
+
+def truthy(lenient):
+  return random.choice(['on', 'yes', '1', 1, 'true', True]) if lenient else True
+
+
+def random_bool(lenient):
+  return random.choice([falsy, truthy])(lenient)
+
+
 # asserts the correctness of the given hits given they are sorted asc
 def assert_sort(hits):
   values = [hit['sort'] for hit in hits['hits']['hits']]
@@ -58,21 +81,27 @@ def assert_sort(hits):
 
 # Indexes the given number of document into the given index
 # and randomly runs refresh, optimize and flush commands
-def index_documents(es, index_name, type, num_docs, supports_dots_in_field_names):
+def index_documents(es, index_name, type, num_docs, capabilities):
   logging.info('Indexing %s docs' % num_docs)
-  index(es, index_name, type, num_docs, supports_dots_in_field_names, True)
+  index(es, index_name, type, num_docs, capabilities, flush=True)
   logging.info('Flushing index')
   es.indices.flush(index=index_name)
 
-def index(es, index_name, type, num_docs, supports_dots_in_field_names, flush=False):
+def index(es, index_name, type, num_docs, capabilities, flush=False):
   for id in range(0, num_docs):
-    body = {'string': str(random.randint(0, 100)),
-            'long_sort': random.randint(0, 100),
-            'double_sort' : float(random.randint(0, 100)),
-            'bool' : random.choice([True, False])}
-    if supports_dots_in_field_names:
+    lenient_bool = capabilities['lenient_booleans']
+    body = {
+          'string': str(random.randint(0, 100)),
+          'long_sort': random.randint(0, 100),
+          'double_sort': float(random.randint(0, 100)),
+          # be sure to create a "proper" boolean (True, False) for the first document so that automapping is correct
+          'bool': random_bool(lenient_bool) if id > 0 else random.choice([True, False])
+        }
+    if capabilities['dots_in_field_names']:
       body['field.with.dots'] = str(random.randint(0, 100))
 
+    body['binary'] = base64.b64encode(bytearray(random.getrandbits(8) for _ in range(16))).decode('ascii')
+
     es.index(index=index_name, doc_type=type, id=id, body=body)
 
     if rarely():
@@ -80,10 +109,11 @@ def index(es, index_name, type, num_docs, supports_dots_in_field_names, flush=Fa
     if rarely() and flush:
       es.indices.flush(index=index_name, force=frequently())
 
-def reindex_docs(es, index_name, type, num_docs, supports_dots_in_field_names):
+def reindex_docs(es, index_name, type, num_docs, capabilities):
   logging.info('Re-indexing %s docs' % num_docs)
+  capabilities['lenient_booleans'] = False
   # reindex some docs after the flush such that we have something in the translog
-  index(es, index_name, type, num_docs, supports_dots_in_field_names)
+  index(es, index_name, type, num_docs, capabilities)
 
 def delete_by_query(es, version, index_name, doc_type):
 
@@ -103,7 +133,7 @@ def delete_by_query(es, version, index_name, doc_type):
     return
 
   deleted_count = es.count(index=index_name, doc_type=doc_type, body=query)['count']
-    
+
   result = es.delete_by_query(index=index_name,
                               doc_type=doc_type,
                               body=query)
@@ -113,9 +143,13 @@ def delete_by_query(es, version, index_name, doc_type):
 
   logging.info('Deleted %d docs' % deleted_count)
 
-def run_basic_asserts(es, index_name, type, num_docs):
+def run_basic_asserts(es, version, index_name, type, num_docs):
   count = es.count(index=index_name)['count']
   assert count == num_docs, 'Expected %r but got %r documents' % (num_docs, count)
+  if parse_version(version) < parse_version('5.1.0'):
+    # This alias isn't allowed to be created after 5.1 so we can verify that we can still use it
+    count = es.count(index='#' + index_name)['count']
+    assert count == num_docs, 'Expected %r but got %r documents' % (num_docs, count)
   for _ in range(0, num_docs):
     random_doc_id = random.randint(0, num_docs-1)
     doc = es.get(index=index_name, doc_type=type, id=random_doc_id)
@@ -193,9 +227,12 @@ def generate_index(client, version, index_name):
   client.indices.delete(index=index_name, ignore=404)
   logging.info('Create single shard test index')
 
+  capabilities = capabilities_of(version)
+  lenient_booleans = capabilities['lenient_booleans']
+
   mappings = {}
   warmers = {}
-  if parse_version(version) < parse_version('2.0.0-alpha1'):
+  if capabilities['warmers']:
     warmers['warmer1'] = {
       'source': {
         'query': {
@@ -242,7 +279,7 @@ def generate_index(client, version, index_name):
     }
     mappings['meta_fields'] = {
       '_routing': {
-        'required': 'false'
+        'required': falsy(lenient_booleans)
       },
     }
     mappings['custom_formats'] = {
@@ -259,18 +296,25 @@ def generate_index(client, version, index_name):
     }
     mappings['auto_boost'] = {
       '_all': {
-        'auto_boost': True
+        'auto_boost': truthy(lenient_booleans)
       }
     }
   mappings['doc'] = {'properties' : {}}
-  supports_dots_in_field_names = parse_version(version) >= parse_version("2.4.0")
-  if supports_dots_in_field_names:
-    mappings["doc"]['properties'].update({
+
+  if capabilities['dots_in_field_names']:
+    if parse_version(version) < parse_version("5.0.0-alpha1"):
+      mappings["doc"]['properties'].update({
         'field.with.dots': {
           'type': 'string',
           'boost': 4
         }
       })
+    else:
+      mappings["doc"]['properties'].update({
+        'field.with.dots': {
+          'type': 'text'
+        }
+      })
 
   if parse_version(version) < parse_version("5.0.0-alpha1"):
     mappings['norms'] = {
@@ -305,7 +349,7 @@ def generate_index(client, version, index_name):
       'properties': {
         'string_with_norms_disabled': {
           'type': 'text',
-          'norms' : False
+          'norms': False
         },
         'string_with_norms_enabled': {
           'type': 'keyword',
@@ -322,6 +366,12 @@ def generate_index(client, version, index_name):
         }
     })
 
+  # test back-compat of stored binary fields
+  mappings['doc']['properties']['binary'] = {
+    'type': 'binary',
+    'store': truthy(lenient_booleans),
+    }
+
   settings = {
     'number_of_shards': 1,
     'number_of_replicas': 0,
@@ -339,7 +389,10 @@ def generate_index(client, version, index_name):
   if warmers:
     body['warmers'] = warmers
   client.indices.create(index=index_name, body=body)
-  health = client.cluster.health(wait_for_status='green', wait_for_relocating_shards=0)
+  if parse_version(version) < parse_version("5.0.0-alpha1"):
+    health = client.cluster.health(wait_for_status='green')
+  else:
+    health = client.cluster.health(wait_for_status='green', params={"wait_for_no_relocating_shards": "true"})
   assert health['timed_out'] == False, 'cluster health timed out %s' % health
 
   num_docs = random.randint(2000, 3000)
@@ -348,10 +401,13 @@ def generate_index(client, version, index_name):
     # lighter index for it to keep bw tests reasonable
     # see https://github.com/elastic/elasticsearch/issues/5817
     num_docs = int(num_docs / 10)
-  index_documents(client, index_name, 'doc', num_docs, supports_dots_in_field_names)
+  index_documents(client, index_name, 'doc', num_docs, capabilities)
+  if parse_version(version) < parse_version('5.1.0'):
+    logging.info("Adding a alias that can't be created in 5.1+ so we can assert that we can still use it")
+    client.indices.put_alias(index=index_name, name='#' + index_name)
   logging.info('Running basic asserts on the data added')
-  run_basic_asserts(client, index_name, 'doc', num_docs)
-  return num_docs, supports_dots_in_field_names
+  run_basic_asserts(client, version, index_name, 'doc', num_docs)
+  return num_docs, capabilities
 
 def snapshot_index(client, version, repo_dir):
   persistent = {
@@ -373,7 +429,9 @@ def snapshot_index(client, version, repo_dir):
     },
     "mappings": {
       "type1": {
-        "_source": { "enabled" : False }
+        "_source": {
+          "enabled": falsy(capabilities_of(version)['lenient_booleans'])
+        }
       }
     },
     "aliases": {
@@ -461,7 +519,7 @@ def create_bwc_index(cfg, version):
     node = start_node(version, release_dir, data_dir, repo_dir, cfg.tcp_port, cfg.http_port)
     client = create_client(cfg.http_port)
     index_name = 'index-%s' % version.lower()
-    num_docs, supports_dots_in_field_names = generate_index(client, version, index_name)
+    num_docs, capabilities = generate_index(client, version, index_name)
     if snapshot_supported:
       snapshot_index(client, version, repo_dir)
 
@@ -470,7 +528,7 @@ def create_bwc_index(cfg, version):
     # will already have the deletions applied on upgrade.
     if version.startswith('0.') or version.startswith('1.'):
       delete_by_query(client, version, index_name, 'doc')
-    reindex_docs(client, index_name, 'doc', min(100, num_docs), supports_dots_in_field_names)
+    reindex_docs(client, index_name, 'doc', min(100, num_docs), capabilities)
 
     shutdown_node(node)
     node = None
@@ -483,7 +541,7 @@ def create_bwc_index(cfg, version):
     if node is not None:
       # This only happens if we've hit an exception:
       shutdown_node(node)
-      
+
     shutil.rmtree(tmp_dir)
 
 def shutdown_node(node):
@@ -522,4 +580,3 @@ def main():
     main()
   except KeyboardInterrupt:
     print('Caught keyboard interrupt, exiting...')
-
diff --git a/dev-tools/get-bwc-version.py b/dev-tools/get-bwc-version.py
index 54c559d1dc8f1..ba7df263d9646 100644
--- a/dev-tools/get-bwc-version.py
+++ b/dev-tools/get-bwc-version.py
@@ -9,7 +9,7 @@
 #     http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on 
+# software distributed under the License is distributed on
 # an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 # either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
@@ -38,7 +38,7 @@ def parse_config():
 
 def main():
   c = parse_config()
-  
+
   if not os.path.exists(c.path):
     print('Creating %s' % c.path)
     os.mkdir(c.path)
@@ -53,7 +53,7 @@ def main():
       shutil.rmtree(version_dir)
     else:
       print('Version %s exists at %s' % (c.version, version_dir))
-      return 
+      return
 
   # before 1.4.0, the zip file contains windows scripts, and tar.gz contained *nix scripts
   if is_windows:
@@ -64,17 +64,17 @@ def main():
   if c.version == '1.2.0':
     # 1.2.0 was pulled from download.elasticsearch.org because of routing bug:
     url = 'http://central.maven.org/maven2/org/elasticsearch/elasticsearch/1.2.0/%s' % filename
-  elif c.version.startswith('0.') or c.version.startswith('1.'):
+  elif c.version.startswith('0.') or c.version.startswith('1.') or c.version.startswith('2.'):
     url = 'https://download.elasticsearch.org/elasticsearch/elasticsearch/%s' % filename
   else:
-    url = 'http://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/%s/%s' % (c.version, filename)
+    url = 'https://artifacts.elastic.co/downloads/elasticsearch/%s' % filename
   print('Downloading %s' % url)
   urllib.request.urlretrieve(url, filename)
 
   print('Extracting to %s' % version_dir)
   if is_windows:
     archive = zipfile.ZipFile(filename)
-    archive.extractall() 
+    archive.extractall()
   else:
     # for some reason python's tarfile module has trouble with ES tgz?
     subprocess.check_call('tar -xzf %s' % filename, shell=True)
diff --git a/dev-tools/prepare_release_candidate.py b/dev-tools/prepare_release_candidate.py
deleted file mode 100644
index 84c10f4b9e4fd..0000000000000
--- a/dev-tools/prepare_release_candidate.py
+++ /dev/null
@@ -1,417 +0,0 @@
-# Licensed to Elasticsearch under one or more contributor
-# license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright
-# ownership. Elasticsearch licenses this file to you under
-# the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance  with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on
-# an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
-# either express or implied. See the License for the specific
-# language governing permissions and limitations under the License.
-
-# Prepare a release
-#
-# 1. Update the Version.java to remove the snapshot bit
-# 2. Remove the -SNAPSHOT suffix in all pom.xml files
-#
-# USAGE:
-#
-# python3 ./dev-tools/prepare-release.py
-#
-# Note: Ensure the script is run from the elasticsearch top level directory
-#
-
-import fnmatch
-import argparse
-from prepare_release_update_documentation import update_reference_docs
-import subprocess
-import tempfile
-import re
-import os
-import shutil
-from functools import partial
-import sys
-
-VERSION_FILE = 'core/src/main/java/org/elasticsearch/Version.java'
-POM_FILE = 'pom.xml'
-MAIL_TEMPLATE = """
-Hi all
-
-The new release candidate for %(version)s is now available, including the x-plugins and RPM/deb repos.  This release is based on:
-
- * Elasticsearch commit: %(hash)s - https://github.com/elastic/elasticsearch/commit/%(hash)s
- * X-Plugins commit:     FILL_IN_X-PLUGINS_HASH - https://github.com/elastic/x-plugins/commit/FILL_IN_X-PLUGINS_HASH
-
-The packages may be downloaded from the following URLs:
-
- * ZIP    - http://%(bucket)s/elasticsearch/staging/%(version)s-%(hash)s/org/elasticsearch/distribution/zip/elasticsearch/%(version)s/elasticsearch-%(version)s.zip
- * tar.gz - http://%(bucket)s/elasticsearch/staging/%(version)s-%(hash)s/org/elasticsearch/distribution/tar/elasticsearch/%(version)s/elasticsearch-%(version)s.tar.gz
- * RPM    - http://%(bucket)s/elasticsearch/staging/%(version)s-%(hash)s/org/elasticsearch/distribution/rpm/elasticsearch/%(version)s/elasticsearch-%(version)s.rpm
- * deb    - http://%(bucket)s/elasticsearch/staging/%(version)s-%(hash)s/org/elasticsearch/distribution/deb/elasticsearch/%(version)s/elasticsearch-%(version)s.deb
-
-Plugins can be installed as follows:
-
-    ES_JAVA_OPTS="-Des.plugins.staging=true" bin/elasticsearch-plugin install cloud-aws
-
-The same goes for the x-plugins:
-
-    ES_JAVA_OPTS="-Des.plugins.staging=true" bin/elasticsearch-plugin install license
-    ES_JAVA_OPTS="-Des.plugins.staging=true" bin/elasticsearch-plugin install marvel-agent
-    ES_JAVA_OPTS="-Des.plugins.staging=true" bin/elasticsearch-plugin install shield
-    ES_JAVA_OPTS="-Des.plugins.staging=true" bin/elasticsearch-plugin install watcher
-
-To install the deb from an APT repo:
-
-APT line sources.list line:
-
-deb http://%(bucket)s/elasticsearch/staging/%(version)s-%(hash)s/repos/%(package_repo_version)s/debian/ stable main
-
-To install the RPM, create a YUM file like:
-
-    /etc/yum.repos.d/elasticsearch.repo
-
-containing:
-
-[elasticsearch-2.0]
-name=Elasticsearch repository for packages
-baseurl=http://%(bucket)s/elasticsearch/staging/%(version)s-%(hash)s/repos/%(package_repo_version)s/centos
-gpgcheck=1
-gpgkey=http://packages.elastic.co/GPG-KEY-elasticsearch
-enabled=1
-
-To smoke-test the release please run:
-
- python3 -B ./dev-tools/smoke_test_rc.py --version %(version)s --hash %(hash)s --plugins license,shield,watcher
-
-NOTE: this script requires JAVA_HOME to point to a Java 7 Runtime
-
-"""
-
-# console colors
-COLOR_OK = '\033[92m'
-COLOR_END = '\033[0m'
-COLOR_FAIL = '\033[91m'
-
-def run(command, env_vars=None):
-  if env_vars:
-    for key, value in env_vars.items():
-      os.putenv(key, value)
-  print('*** Running: %s%s%s' % (COLOR_OK, command, COLOR_END))
-  if os.system(command):
-    raise RuntimeError('    FAILED: %s' % (command))
-
-def ensure_checkout_is_clean():
-  # Make sure no local mods:
-  s = subprocess.check_output('git diff --shortstat', shell=True).decode('utf-8')
-  if len(s) > 0:
-    raise RuntimeError('git diff --shortstat is non-empty got:\n%s' % s)
-
-  # Make sure no untracked files:
-  s = subprocess.check_output('git status', shell=True).decode('utf-8', errors='replace')
-  if 'Untracked files:' in s:
-    if 'dev-tools/__pycache__/' in s:
-      print('*** NOTE: invoke python with -B to prevent __pycache__ directories ***')
-    raise RuntimeError('git status shows untracked files got:\n%s' % s)
-
-  # Make sure we have all changes from origin:
-  if 'is behind' in s:
-    raise RuntimeError('git status shows not all changes pulled from origin; try running "git pull origin" in this branch got:\n%s' % (s))
-
-  # Make sure we no local unpushed changes (this is supposed to be a clean area):
-  if 'is ahead' in s:
-    raise RuntimeError('git status shows local commits; try running "git fetch origin", "git checkout ", "git reset --hard origin/" in this branch got:\n%s' % (s))
-
-# Reads the given file and applies the
-# callback to it. If the callback changed
-# a line the given file is replaced with
-# the modified input.
-def process_file(file_path, line_callback):
-  fh, abs_path = tempfile.mkstemp()
-  modified = False
-  with open(abs_path,'w', encoding='utf-8') as new_file:
-    with open(file_path, encoding='utf-8') as old_file:
-      for line in old_file:
-        new_line = line_callback(line)
-        modified = modified or (new_line != line)
-        new_file.write(new_line)
-  os.close(fh)
-  if modified:
-    #Remove original file
-    os.remove(file_path)
-    #Move new file
-    shutil.move(abs_path, file_path)
-    return True
-  else:
-    # nothing to do - just remove the tmp file
-    os.remove(abs_path)
-    return False
-
-# Moves the Version.java file from a snapshot to a release
-def remove_version_snapshot(version_file, release):
-  # 1.0.0.Beta1 -> 1_0_0_Beta1
-  release = release.replace('.', '_')
-  release = release.replace('-', '_')
-  pattern = 'new Version(V_%s_ID, true' % (release)
-  replacement = 'new Version(V_%s_ID, false' % (release)
-  def callback(line):
-    return line.replace(pattern, replacement)
-  processed = process_file(version_file, callback)
-  if not processed:
-    raise RuntimeError('failed to remove snapshot version for %s' % (release))
-
-def rename_local_meta_files(path):
-  for root, _, file_names in os.walk(path):
-    for file_name in fnmatch.filter(file_names, 'maven-metadata-local.xml*'):
-      full_path = os.path.join(root, file_name)
-      os.rename(full_path, os.path.join(root, file_name.replace('-local', '')))
-
-# Checks the pom.xml for the release version.
-# This method fails if the pom file has no SNAPSHOT version set ie.
-# if the version is already on a release version we fail.
-# Returns the next version string ie. 0.90.7
-def find_release_version():
-  with open('pom.xml', encoding='utf-8') as file:
-    for line in file:
-      match = re.search(r'<version>(.+)-SNAPSHOT</version>', line)
-      if match:
-        return match.group(1)
-    raise RuntimeError('Could not find release version in branch')
-
-# Checks if the produced RPM is signed with the supplied GPG key
-def ensure_rpm_is_signed(rpm, gpg_key):
-  rpm_check_signature_cmd = 'rpm -v -K %s | grep -qi %s' % (rpm, gpg_key)
-  try:
-    subprocess.check_output(rpm_check_signature_cmd, shell=True)
-  except:
-    raise RuntimeError('Aborting. RPM does not seem to be signed, check with: rpm -v -K %s' % rpm)
-
-# Checks if a command exists, needed for external binaries
-def check_command_exists(name, cmd):
-  try:
-    subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
-  except subprocess.CalledProcessError:
-    raise RuntimeError('Could not run command %s - please make sure it is installed and in $PATH' % (name))
-
-def run_and_print(text, run_function):
-  try:
-    print(text, end='')
-    run_function()
-    print(COLOR_OK + 'OK' + COLOR_END)
-    return True
-  except RuntimeError:
-    print(COLOR_FAIL + 'NOT OK' + COLOR_END)
-    return False
-
-def check_env_var(text, env_var):
-  try:
-    print(text, end='')
-    os.environ[env_var]
-    print(COLOR_OK + 'OK' + COLOR_END)
-    return True
-  except KeyError:
-    print(COLOR_FAIL + 'NOT OK' + COLOR_END)
-    return False
-
-def check_environment_and_commandline_tools(check_only):
-  checks = list()
-  checks.append(check_env_var('Checking for AWS env configuration AWS_SECRET_KEY... ', 'AWS_SECRET_KEY'))
-  checks.append(check_env_var('Checking for AWS env configuration AWS_ACCESS_KEY... ', 'AWS_ACCESS_KEY'))
-  checks.append(run_and_print('Checking command: rpm...            ', partial(check_command_exists, 'rpm', 'rpm --version')))
-  checks.append(run_and_print('Checking command: dpkg...           ', partial(check_command_exists, 'dpkg', 'dpkg --version')))
-  checks.append(run_and_print('Checking command: gpg...            ', partial(check_command_exists, 'gpg', 'gpg --version')))
-  checks.append(run_and_print('Checking command: expect...         ', partial(check_command_exists, 'expect', 'expect -v')))
-  checks.append(run_and_print('Checking command: createrepo...     ', partial(check_command_exists, 'createrepo', 'createrepo --version')))
-  checks.append(run_and_print('Checking command: s3cmd...          ', partial(check_command_exists, 's3cmd', 's3cmd --version')))
-  checks.append(run_and_print('Checking command: deb-s3...         ', partial(check_command_exists, 'deb-s3', 'deb-s3 -h')))
-  checks.append(run_and_print('Checking command: rpm-s3...         ', partial(check_command_exists, 'rpm-s3', 'rpm-s3 -h')))
-
-  if check_only:
-    sys.exit(0)
-
-  if False in checks:
-    print("Exiting due to failing checks")
-    sys.exit(0)
-
-if __name__ == "__main__":
-  parser = argparse.ArgumentParser(description='Builds and publishes a Elasticsearch Release')
-  parser.add_argument('--deploy-sonatype', dest='deploy_sonatype', action='store_true',
-                      help='Installs and Deploys the release on a sonatype staging repository.')
-  parser.add_argument('--deploy-s3', dest='deploy_s3', action='store_true',
-                      help='Pushes artifacts to the S3 staging area')
-  parser.add_argument('--deploy-s3-repos', dest='deploy_s3_repos', action='store_true',
-                      help='Creates package repositories in S3 repo')
-  parser.add_argument('--no-install', dest='no_install', action='store_true',
-                      help='Does not run "mvn install", expects this to be run already and reuses artifacts from local repo, only useful with --deploy-s3/--deploy-s3-repos, after sonatype deplomeny to ensure same artifacts')
-  parser.add_argument('--skip-doc-check', dest='skip_doc_check', action='store_false',
-                      help='Skips any checks for pending documentation changes')
-  parser.add_argument('--skip-tests', dest='skip_tests', action='store_true',
-                      help='Skips any test runs')
-  parser.add_argument('--gpg-key', dest='gpg_key', default="D88E42B4",
-                      help='Allows you to specify a different gpg_key to be used instead of the default release key')
-  parser.add_argument('--bucket', '-b', dest='bucket', default="download.elasticsearch.org",
-                      help='Allows you to specify a different s3 bucket to upload the artifacts to')
-  parser.add_argument('--quiet', dest='quiet', action='store_true',
-                      help='Runs the script in quiet mode')
-  parser.add_argument('--check', dest='check', action='store_true',
-                      help='Checks and reports for all requirements and then exits')
-
-  # by default, we only run mvn install and don't push anything repo
-  parser.set_defaults(deploy_sonatype=False)
-  parser.set_defaults(deploy_s3=False)
-  parser.set_defaults(deploy_s3_repos=False)
-  parser.set_defaults(no_install=False)
-  # other defaults
-  parser.set_defaults(skip_doc_check=False)
-  parser.set_defaults(quiet=False)
-  parser.set_defaults(skip_tests=False)
-
-  args = parser.parse_args()
-  skip_doc_check = args.skip_doc_check
-  gpg_key = args.gpg_key
-  bucket = args.bucket
-  deploy_sonatype = args.deploy_sonatype
-  deploy_s3 = args.deploy_s3
-  deploy_s3_repos = args.deploy_s3_repos
-  run_mvn_install = not args.no_install
-  skip_tests = args.skip_tests
-
-  check_environment_and_commandline_tools(args.check)
-
-  if not run_mvn_install and deploy_sonatype:
-    print('Using --no-install and --deploy-sonatype together does not work. Exiting')
-    sys.exit(-1)
-
-  print('*** Preparing a release candidate: ', end='')
-  print('deploy sonatype: %s%s%s' % (COLOR_OK if deploy_sonatype else COLOR_FAIL, 'yes' if deploy_sonatype else 'no', COLOR_END), end='')
-  print(', deploy s3: %s%s%s' % (COLOR_OK if deploy_s3 else COLOR_FAIL, 'yes' if deploy_s3 else 'no', COLOR_END), end='')
-  print(', deploy s3 repos: %s%s%s' % (COLOR_OK if deploy_s3_repos else COLOR_FAIL, 'yes' if deploy_s3_repos else 'no', COLOR_END), end='')
-  print('')
-
-  shortHash = subprocess.check_output('git log --pretty=format:"%h" -n 1', shell=True).decode('utf-8')
-  releaseDirectory = os.getenv('HOME') + '/elastic-releases'
-  release_version = find_release_version()
-  localRepo = '%s/elasticsearch-%s-%s' % (releaseDirectory, release_version, shortHash)
-  localRepoElasticsearch = localRepo + '/org/elasticsearch'
-
-  ensure_checkout_is_clean()
-  if not re.match('(\d+\.\d+)\.*',release_version):
-    raise RuntimeError('illegal release version format: %s' % (release_version))
-  package_repo_version = '%s.x' % re.match('(\d+)\.*', release_version).group(1)
-
-  print('*** Preparing release version: [%s]' % release_version)
-
-  if not skip_doc_check:
-    print('*** Check for pending documentation changes')
-    pending_files = update_reference_docs(release_version)
-    if pending_files:
-      raise RuntimeError('pending coming[%s] documentation changes found in %s' % (release_version, pending_files))
-
-  run('cd dev-tools && mvn versions:set -DnewVersion=%s -DgenerateBackupPoms=false' % (release_version))
-  run('cd rest-api-spec && mvn versions:set -DnewVersion=%s -DgenerateBackupPoms=false' % (release_version))
-  run('mvn versions:set -DnewVersion=%s -DgenerateBackupPoms=false' % (release_version))
-
-  remove_version_snapshot(VERSION_FILE, release_version)
-
-  print('*** Done removing snapshot version. DO NOT COMMIT THIS, WHEN CREATING A RELEASE CANDIDATE.')
-
-  if not os.path.exists(releaseDirectory):
-    os.mkdir(releaseDirectory)
-  if os.path.exists(localRepoElasticsearch) and run_mvn_install:
-    print('clean local repository %s' % localRepoElasticsearch)
-    shutil.rmtree(localRepoElasticsearch)
-
-  mvn_target = 'deploy' if deploy_sonatype else 'install'
-  tests = '-DskipTests' if skip_tests else '-Dskip.integ.tests=true'
-  install_command = 'mvn clean %s -Prelease %s -Dgpg.key="%s" -Dpackaging.rpm.rpmbuild=/usr/bin/rpmbuild -Drpm.sign=true -Dmaven.repo.local=%s -Dno.commit.pattern="\\bno(n|)commit\\b" -Dforbidden.test.signatures=""' % (mvn_target, tests, gpg_key, localRepo)
-  clean_repo_command = 'find %s -name _remote.repositories -exec rm {} \;' % (localRepoElasticsearch)
-
-  if not run_mvn_install:
-    print('')
-    print('*** By choosing --no-install we assume you ran the following commands successfully:')
-    print('  %s' % (install_command))
-    print('  1. Remove all _remote.repositories: %s' % (clean_repo_command))
-    rename_metadata_files_command = 'for i in $(find %s -name "maven-metadata-local.xml*") ; do mv "$i" "${i/-local/}" ; done' % (localRepoElasticsearch)
-    print('  2. Rename all maven metadata files: %s' % (rename_metadata_files_command))
-  else:
-    for cmd in [install_command, clean_repo_command]:
-      run(cmd)
-    rename_local_meta_files(localRepoElasticsearch)
-
-  rpm = '%s/distribution/rpm/elasticsearch/%s/elasticsearch-%s.rpm' % (localRepoElasticsearch, release_version, release_version)
-  print('Ensuring that RPM has been signed')
-  ensure_rpm_is_signed(rpm, gpg_key)
-
-  # repository push commands
-  s3cmd_sync_to_staging_bucket_cmd = 's3cmd sync -P %s s3://%s/elasticsearch/staging/%s-%s/org/' % (localRepoElasticsearch, bucket, release_version, shortHash)
-  s3_bucket_sync_to = '%s/elasticsearch/staging/%s-%s/repos/' % (bucket, release_version, shortHash)
-  s3cmd_sync_official_repo_cmd = 's3cmd sync s3://packages.elasticsearch.org/elasticsearch/%s s3://%s' % (package_repo_version, s3_bucket_sync_to)
-
-  debs3_prefix = 'elasticsearch/staging/%s-%s/repos/%s/debian' % (release_version, shortHash, package_repo_version)
-  debs3_upload_cmd = 'deb-s3 upload --preserve-versions %s/distribution/deb/elasticsearch/%s/elasticsearch-%s.deb -b %s --prefix %s --sign %s --arch amd64' % (localRepoElasticsearch, release_version, release_version, bucket, debs3_prefix, gpg_key)
-  debs3_list_cmd = 'deb-s3 list -b %s --prefix %s' % (bucket, debs3_prefix)
-  debs3_verify_cmd = 'deb-s3 verify -b %s --prefix %s' % (bucket, debs3_prefix)
-  rpms3_prefix = 'elasticsearch/staging/%s-%s/repos/%s/centos' % (release_version, shortHash, package_repo_version)
-  # external-1 is the alias name for the us-east-1 region. This is used by rpm-s3 to construct the hostname
-  rpms3_upload_cmd = 'rpm-s3 -v -b %s -p %s --sign --visibility public-read -k 100 %s -r external-1' % (bucket, rpms3_prefix, rpm)
-
-  if deploy_s3:
-    run(s3cmd_sync_to_staging_bucket_cmd)
-  else:
-    print('')
-    print('*** To push a release candidate to s3 run: ')
-    print('  1. Sync %s into S3 bucket' % (localRepoElasticsearch))
-    print ('    %s' % (s3cmd_sync_to_staging_bucket_cmd))
-
-  if deploy_s3_repos:
-    print('*** Syncing official package repository into staging s3 bucket')
-    run(s3cmd_sync_official_repo_cmd)
-    print('*** Uploading debian package (you will be prompted for the passphrase!)')
-    run(debs3_upload_cmd)
-    run(debs3_list_cmd)
-    run(debs3_verify_cmd)
-    print('*** Uploading rpm package (you will be prompted for the passphrase!)')
-    run(rpms3_upload_cmd)
-  else:
-    print('*** To create repositories on S3 run:')
-    print('    1. Sync existing repo into staging: %s' % s3cmd_sync_official_repo_cmd)
-    print('    2. Upload debian package (and sign it): %s' % debs3_upload_cmd)
-    print('    3. List all debian packages: %s' % debs3_list_cmd)
-    print('    4. Verify debian packages: %s' % debs3_verify_cmd)
-    print('    5. Upload RPM: %s' % rpms3_upload_cmd)
-    print('')
-    print('NOTE: the above mvn command will promt you several times for the GPG passphrase of the key you specified you can alternatively pass it via -Dgpg.passphrase=yourPassPhrase')
-    print(' since RPM signing doesn\'t support gpg-agents the recommended way to set the password is to add a release profile to your settings.xml:')
-    print("""
-  <profiles>
-    <profile>
-      <id>release</id>
-      <properties>
-        <gpg.passphrase>YourPasswordGoesHere</gpg.passphrase>
-      </properties>
-    </profile>
-  </profiles>
-    """)
-    print('NOTE: Running s3cmd might require you to create a config file with your credentials, if the s3cmd does not support suppliying them via the command line!')
-
-  print('*** Once the release is deployed and published send out the following mail to dev@elastic.co:')
-  string_format_dict = {'version' : release_version, 'hash': shortHash, 'package_repo_version' : package_repo_version, 'bucket': bucket}
-  print(MAIL_TEMPLATE % string_format_dict)
-
-  print('')
-  print('You can verify that pushing to the staging repository pushed all the artifacts by running (log into sonatype to find out the correct id):')
-  print(' python -B dev-tools/validate-maven-repository.py %s https://oss.sonatype.org/service/local/repositories/orgelasticsearch-IDTOFINDOUT/content/org/elasticsearch ' %(localRepoElasticsearch))
-
-  print('')
-  print('To publish the release and the repo on S3 execute the following commands:')
-  print('   s3cmd cp --recursive s3://%(bucket)s/elasticsearch/staging/%(version)s-%(hash)s/repos/%(package_repo_version)s/ s3://packages.elasticsearch.org/elasticsearch/%(package_repo_version)s'  % string_format_dict)
-  print('   s3cmd cp --recursive s3://%(bucket)s/elasticsearch/staging/%(version)s-%(hash)s/org/ s3://%(bucket)s/elasticsearch/release/org'  % string_format_dict)
-  print('Now go ahead and tag the release:')
-  print('   git tag -a v%(version)s %(hash)s'  % string_format_dict)
-  print('   git push origin v%(version)s' % string_format_dict )
-
-
diff --git a/dev-tools/smoke_test_rc.py b/dev-tools/smoke_test_rc.py
index 33abbf96345ab..0d34fb366ee38 100644
--- a/dev-tools/smoke_test_rc.py
+++ b/dev-tools/smoke_test_rc.py
@@ -44,6 +44,7 @@
 import argparse
 import tempfile
 import os
+from os.path import basename, dirname, isdir, join
 import signal
 import shutil
 import urllib
@@ -55,29 +56,16 @@
 import base64
 from urllib.parse import urlparse
 
-from prepare_release_candidate import run
 from http.client import HTTPConnection
 
-DEFAULT_PLUGINS = ["analysis-icu",
-                   "analysis-kuromoji",
-                   "analysis-phonetic",
-                   "analysis-smartcn",
-                   "analysis-stempel",
-                   "discovery-azure-classic",
-                   "discovery-ec2",
-                   "discovery-gce",
-                   "ingest-attachment",
-                   "ingest-geoip",
-                   "lang-javascript",
-                   "lang-python",
-                   "mapper-attachments",
-                   "mapper-murmur3",
-                   "mapper-size",
-                   "repository-azure",
-                   "repository-gcs",
-                   "repository-hdfs",
-                   "repository-s3",
-                   "store-smb"]
+def find_official_plugins():
+    plugins_dir = join(dirname(dirname(__file__)), 'plugins')
+    plugins = []
+    for plugin in os.listdir(plugins_dir):
+        if isdir(join(plugins_dir, plugin)):
+            plugins.append(plugin)
+    return plugins
+DEFAULT_PLUGINS = find_official_plugins()
 
 try:
   JAVA_HOME = os.environ['JAVA_HOME']
@@ -86,6 +74,18 @@
   Please set JAVA_HOME in the env before running release tool
   On OSX use: export JAVA_HOME=`/usr/libexec/java_home -v '1.8*'`""")
 
+# console colors
+COLOR_OK = '\033[92m'
+COLOR_END = '\033[0m'
+
+def run(command, env_vars=None):
+  if env_vars:
+    for key, value in env_vars.items():
+      os.putenv(key, value)
+  print('*** Running: %s%s%s' % (COLOR_OK, command, COLOR_END))
+  if os.system(command):
+    raise RuntimeError('    FAILED: %s' % (command))
+
 def java_exe():
   path = JAVA_HOME
   return 'export JAVA_HOME="%s" PATH="%s/bin:$PATH" JAVACMD="%s/bin/java"' % (path, path, path)
@@ -95,7 +95,6 @@ def verify_java_version(version):
   if ' version "%s.' % version not in s:
     raise RuntimeError('got wrong version for java %s:\n%s' % (version, s))
 
-
 def sha1(file):
   with open(file, 'rb') as f:
     return hashlib.sha1(f.read()).hexdigest()
@@ -104,7 +103,6 @@ def read_fully(file):
   with open(file, encoding='utf-8') as f:
      return f.read()
 
-
 def wait_for_node_startup(es_dir, timeout=60, header={}):
   print('     Waiting until node becomes available for at most %s seconds' % timeout)
   for _ in range(timeout):
@@ -225,7 +223,7 @@ def smoke_test_release(release, files, expected_hash, plugins):
           if expected_hash != version['build_hash'].strip():
             raise RuntimeError('HEAD hash does not match expected [%s] but got [%s]' % (expected_hash, version['build_hash']))
           print('  Verify if plugins are listed in _nodes')
-          conn.request('GET', '/_nodes?plugin=true&pretty=true', headers=headers)
+          conn.request('GET', '/_nodes/plugins?pretty=true', headers=headers)
           res = conn.getresponse()
           if res.status == 200:
             nodes = json.loads(res.read().decode("utf-8"))['nodes']
diff --git a/dev-tools/validate-maven-repository.py b/dev-tools/validate-maven-repository.py
deleted file mode 100644
index 6bf84a3a1855c..0000000000000
--- a/dev-tools/validate-maven-repository.py
+++ /dev/null
@@ -1,130 +0,0 @@
-# Licensed to Elasticsearch under one or more contributor
-# license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright
-# ownership. Elasticsearch licenses this file to you under
-# the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance  with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on
-# an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
-# either express or implied. See the License for the specific
-# language governing permissions and limitations under the License.
-
-# Helper python script to check if a sonatype staging repo contains
-# all the required files compared to a local repository
-#
-# The script does the following steps
-#
-# 1. Scans the local maven repo for all files in /org/elasticsearch
-# 2. Opens a HTTP connection to the staging repo
-# 3. Executes a HEAD request for each file found in step one
-# 4. Compares the content-length response header with the real file size
-# 5. Return an error if those two numbers differ
-#
-# A pre requirement to run this, is to find out via the oss.sonatype.org web UI, how that repo is named
-# - After logging in you go to 'Staging repositories' and search for the one you just created
-# - Click into the `Content` tab
-# - Open any artifact (not a directory)
-# - Copy the link of `Repository Path` on the right and reuse that part of the URL
-#
-# Alternatively you can just use the name of the repository and reuse the rest (ie. the repository
-# named for the example below would have been named orgelasticsearch-1012)
-#
-#
-# Example call
-#   python dev-tools/validate-maven-repository.py /path/to/repo/org/elasticsearch/ \
-#          https://oss.sonatype.org/service/local/repositories/orgelasticsearch-1012/content/org/elasticsearch
-
-import sys
-import os
-import httplib
-import urlparse
-import re
-
-# Draw a simple progress bar, a couple of hundred HEAD requests might take a while
-# Note, when drawing this, it uses the carriage return character, so you should not
-# write anything in between
-def drawProgressBar(percent, barLen = 40):
-    sys.stdout.write("\r")
-    progress = ""
-    for i in range(barLen):
-        if i < int(barLen * percent):
-            progress += "="
-        else:
-            progress += " "
-    sys.stdout.write("[ %s ] %.2f%%" % (progress, percent * 100))
-    sys.stdout.flush()
-
-if __name__ == "__main__":
-  if len(sys.argv) != 3:
-    print 'Usage: %s <localRep> <stagingRepo> [user:pass]' % (sys.argv[0])
-    print ''
-    print 'Example: %s /tmp/my-maven-repo/org/elasticsearch https://oss.sonatype.org/service/local/repositories/orgelasticsearch-1012/content/org/elasticsearch' % (sys.argv[0])
-  else:
-    sys.argv[1] = re.sub('/$', '', sys.argv[1])
-    sys.argv[2] = re.sub('/$', '', sys.argv[2])
-
-    localMavenRepo = sys.argv[1]
-    endpoint = sys.argv[2]
-
-    filesToCheck = []
-    foundSignedFiles = False
-
-    for root, dirs, files in os.walk(localMavenRepo):
-      for file in files:
-        # no metadata files (they get renamed from maven-metadata-local.xml to maven-metadata.xml while deploying)
-        # no .properties and .repositories files (they don't get uploaded)
-        if not file.startswith('maven-metadata') and not file.endswith('.properties') and not file.endswith('.repositories'):
-          filesToCheck.append(os.path.join(root, file))
-        if file.endswith('.asc'):
-          foundSignedFiles = True
-
-    print "Need to check %i files" % len(filesToCheck)
-    if not foundSignedFiles:
-      print '### Warning: No signed .asc files found'
-
-    # set up http
-    parsed_uri = urlparse.urlparse(endpoint)
-    domain = parsed_uri.netloc
-    if parsed_uri.scheme == 'https':
-      conn = httplib.HTTPSConnection(domain)
-    else:
-      conn = httplib.HTTPConnection(domain)
-    #conn.set_debuglevel(5)
-
-    drawProgressBar(0)
-    errors = []
-    for idx, file in enumerate(filesToCheck):
-      request_uri = parsed_uri.path + file[len(localMavenRepo):]
-      conn.request("HEAD", request_uri)
-      res = conn.getresponse()
-      res.read() # useless call for head, but prevents httplib.ResponseNotReady raise
-
-      absolute_url = parsed_uri.scheme + '://' + parsed_uri.netloc + request_uri
-      if res.status == 200:
-        content_length = res.getheader('content-length')
-        local_file_size = os.path.getsize(file)
-        if int(content_length) != int(local_file_size):
-          errors.append('LENGTH MISMATCH:   %s differs in size. local %s <=> %s remote' % (absolute_url, content_length, local_file_size))
-      elif res.status == 404:
-        errors.append('MISSING:           %s' % absolute_url)
-      elif res.status == 301 or res.status == 302:
-        errors.append('REDIRECT:          %s to %s' % (absolute_url, res.getheader('location')))
-      else:
-        errors.append('ERROR:             %s http response: %s %s' %(absolute_url, res.status, res.reason))
-
-      # update progressbar at the end
-      drawProgressBar((idx+1)/float(len(filesToCheck)))
-
-    print
-
-    if len(errors) != 0:
-      print 'The following errors occurred (%s out of %s files)' % (len(errors), len(filesToCheck))
-      print
-      for error in errors:
-        print error
-      sys.exit(-1)
diff --git a/distribution/build.gradle b/distribution/build.gradle
index edf9b6a70cd1d..9a2f8f65f9439 100644
--- a/distribution/build.gradle
+++ b/distribution/build.gradle
@@ -23,6 +23,7 @@ import org.apache.tools.ant.taskdefs.condition.Os
 import org.elasticsearch.gradle.BuildPlugin
 import org.elasticsearch.gradle.EmptyDirTask
 import org.elasticsearch.gradle.MavenFilteringHack
+import org.elasticsearch.gradle.NoticeTask
 import org.elasticsearch.gradle.precommit.DependencyLicensesTask
 import org.elasticsearch.gradle.precommit.UpdateShasTask
 import org.elasticsearch.gradle.test.RunTask
@@ -39,9 +40,22 @@ buildscript {
   }
 }
 
-// this is common configuration for distributions, but we also add it here for the license check to use
-ext.dependencyFiles = project(':core').configurations.runtime.copyRecursive()
+Collection distributions = project.subprojects.findAll {
+   it.path.contains(':tools') == false && it.name != 'bwc' }
 
+/*****************************************************************************
+ *                                Notice file                                *
+ *****************************************************************************/
+
+// integ test zip only uses core, so a different notice file is needed there
+task buildCoreNotice(type: NoticeTask) {
+  licensesDir new File(project(':core').projectDir, 'licenses')
+}
+
+// other distributions include notices from modules as well, which are added below later
+task buildFullNotice(type: NoticeTask) {
+  licensesDir new File(project(':core').projectDir, 'licenses')
+}
 
 /*****************************************************************************
  *                                  Modules                                  *
@@ -58,6 +72,12 @@ ext.restTestExpansions = [
 // depend on it, but we don't actually configure it until here so we can do a single
 // loop over modules to also setup cross task dependencies and increment our modules counter
 project.rootProject.subprojects.findAll { it.path.startsWith(':modules:') }.each { Project module ->
+  buildFullNotice {
+    def defaultLicensesDir = new File(module.projectDir, 'licenses')
+    if (defaultLicensesDir.exists()) {
+      licensesDir defaultLicensesDir
+    }
+  }
   buildModules {
     dependsOn({ project(module.path).bundlePlugin })
     into(module.name) {
@@ -65,25 +85,21 @@ project.rootProject.subprojects.findAll { it.path.startsWith(':modules:') }.each
     }
   }
   // We would like to make sure integ tests for the distribution run after
-  // integ tests for the modules included in the distribution. However, gradle
-  // has a bug where depending on a task with a finalizer can sometimes not make
-  // the finalizer task follow the original task immediately. To work around this,
-  // we make the mustRunAfter the finalizer task itself.
-  // See https://discuss.gradle.org/t/cross-project-task-dependencies-ordering-screws-up-finalizers/13190
-  project.configure(project.subprojects.findAll { it.name != 'integ-test-zip' }) { Project distribution ->
+  // integ tests for the modules included in the distribution.
+  project.configure(distributions.findAll { it.name != 'integ-test-zip'}) { Project distribution ->
     distribution.afterEvaluate({
       // some integTest tasks will have multiple finalizers
-      distribution.integTest.mustRunAfter module.tasks.find { t -> t.name.matches(".*integTest\$") }.getFinalizedBy()
+      distribution.integTest.mustRunAfter module.tasks.find { t -> t.name.matches(".*integTest\$") }
     })
   }
   // also want to make sure the module's integration tests run after the integ-test-zip (ie rest tests)
   module.afterEvaluate({
-    module.integTest.mustRunAfter(':distribution:integ-test-zip:integTest#stop')
+    module.integTest.mustRunAfter(':distribution:integ-test-zip:integTest')
   })
   restTestExpansions['expected.modules.count'] += 1
 }
 
-// Integ tests work over the rest http layer, so we need a transport included with the integ test zip. 
+// Integ tests work over the rest http layer, so we need a transport included with the integ test zip.
 // All transport modules are included so that they may be randomized for testing
 task buildTransportModules(type: Sync) {
   into 'build/transport-modules'
@@ -104,23 +120,23 @@ task clean(type: Delete) {
   delete 'build'
 }
 
-subprojects {
+configure(distributions) {
   /*****************************************************************************
    *                            Rest test config                               *
    *****************************************************************************/
+  apply plugin: 'elasticsearch.standalone-rest-test'
   apply plugin: 'elasticsearch.rest-test'
   project.integTest {
-    dependsOn project.assemble
     includePackaged project.name == 'integ-test-zip'
-    cluster {
-      distribution = project.name
-    }
     if (project.name != 'integ-test-zip') {
-      // see note above with module mustRunAfter about why integTest#stop is used here
-      mustRunAfter ':distribution:integ-test-zip:integTest#stop'
+      mustRunAfter ':distribution:integ-test-zip:integTest'
     }
   }
-  
+  project.integTestCluster {
+    dependsOn project.assemble
+    distribution = project.name
+  }
+
   processTestResources {
     inputs.properties(project(':distribution').restTestExpansions)
     MavenFilteringHack.filter(it, project(':distribution').restTestExpansions)
@@ -132,13 +148,6 @@ subprojects {
   // note: the group must be correct before applying the nexus plugin, or it will capture the wrong value...
   project.group = "org.elasticsearch.distribution.${project.name}"
   project.archivesBaseName = 'elasticsearch'
-  apply plugin: 'com.bmuschko.nexus'
-  // we must create our own install task, because it is only added when the java plugin is added
-  task install(type: Upload, description: "Installs the 'archives' artifacts into the local Maven repository.", group: 'Upload') {
-    configuration = configurations.archives
-    MavenRepositoryHandlerConvention repositoriesHandler = (MavenRepositoryHandlerConvention)getRepositories().getConvention().getPlugin(MavenRepositoryHandlerConvention)
-    repositoriesHandler.mavenInstaller()
-  }
 
   // TODO: the map needs to be an input of the tasks, so that when it changes, the task will re-run...
   /*****************************************************************************
@@ -153,7 +162,10 @@ subprojects {
     libFiles = copySpec {
       into 'lib'
       from project(':core').jar
-      from project(':distribution').dependencyFiles
+      from project(':core').configurations.runtime
+      // delay add tools using closures, since they have not yet been configured, so no jar task exists yet
+      from { project(':distribution:tools:java-version-checker').jar }
+      from { project(':distribution:tools:plugin-cli').jar }
     }
 
     modulesFiles = copySpec {
@@ -182,36 +194,47 @@ subprojects {
 
     commonFiles = copySpec {
       from rootProject.projectDir
-      include 'NOTICE.txt'
       include 'LICENSE.txt'
       include 'README.textile'
     }
+
+    noticeFile = copySpec {
+      if (project.name == 'integ-test-zip') {
+        from buildCoreNotice
+      } else {
+        from buildFullNotice
+      }
+    }
   }
 
    /*****************************************************************************
    *                           Publishing setup                                *
    *****************************************************************************/
-  BuildPlugin.configurePomGeneration(project)
-  apply plugin: 'nebula.info-scm'
-  apply plugin: 'nebula.maven-base-publish'
-  apply plugin: 'nebula.maven-scm'
-  publishing {
-    publications {
-      nebula {
-        artifactId 'elasticsearch'
-      }
-    }
+  if (['zip', 'integ-test-zip'].contains(it.name)) {
+    BuildPlugin.configurePomGeneration(project)
+    apply plugin: 'nebula.info-scm'
+    apply plugin: 'nebula.maven-base-publish'
+    apply plugin: 'nebula.maven-scm'
   }
 }
 
 /*****************************************************************************
  *                         Zip and tgz configuration                         *
  *****************************************************************************/
-configure(subprojects.findAll { ['zip', 'tar', 'integ-test-zip'].contains(it.name) }) {
+configure(distributions.findAll { ['zip', 'tar', 'integ-test-zip'].contains(it.name) }) {
+  // CopySpec does not make it easy to create an empty director so we create the directory that we want, and then point CopySpec to its
+  // parent to copy to the root of the distribution
+  File plugins = new File(buildDir, 'plugins-hack/plugins')
+  task createPluginsDir(type: EmptyDirTask) {
+    dir "${plugins}"
+    dirMode 0755
+  }
   project.ext.archivesFiles = copySpec {
     into("elasticsearch-${version}") {
       with libFiles
       into('config') {
+        dirMode 0750
+        fileMode 0660
         with configFiles
       }
       into('bin') {
@@ -224,7 +247,14 @@ configure(subprojects.findAll { ['zip', 'tar', 'integ-test-zip'].contains(it.nam
           MavenFilteringHack.filter(it, expansions)
         }
       }
+      into('') {
+        from {
+          dirMode 0755
+          plugins.getParent()
+        }
+      }
       with commonFiles
+      with noticeFile
       from('../src/main/resources') {
         include 'bin/*.exe'
       }
@@ -254,8 +284,14 @@ configure(subprojects.findAll { ['zip', 'tar', 'integ-test-zip'].contains(it.nam
  * 3. ospackage really wants to suck up some of the debian control scripts
  *    directly from the filesystem. It doesn't want to process them through
  *    MavenFilteringHack or any other copy-style action.
+ *
+ * The following commands are useful when it comes to check the user/group
+ * and files permissions set within the RPM and DEB packages:
+ *
+ *    rpm -qlp --dump path/to/elasticsearch.rpm
+ *    dpkg -c path/to/elasticsearch.deb
  */
-configure(subprojects.findAll { ['deb', 'rpm'].contains(it.name) }) {
+configure(distributions.findAll { ['deb', 'rpm'].contains(it.name) }) {
   integTest.enabled = Os.isFamily(Os.FAMILY_WINDOWS) == false
   File packagingFiles = new File(buildDir, 'packaging')
   project.ext.packagingFiles = packagingFiles
@@ -288,8 +324,6 @@ configure(subprojects.findAll { ['deb', 'rpm'].contains(it.name) }) {
     dependsOn createEtc, createEtcScripts
     with configFiles
     into "${packagingFiles}/etc/elasticsearch"
-    fileMode 0640
-    dirMode 0750
     /* Explicitly declare the output files so this task doesn't consider itself
       up to date when the directory is created, which it would by default. And
       that'll happen when createEtc runs. */
@@ -372,12 +406,14 @@ configure(subprojects.findAll { ['deb', 'rpm'].contains(it.name) }) {
         exclude 'LICENSE.txt'
       }
     }
+    with noticeFile
 
     configurationFile '/etc/elasticsearch/elasticsearch.yml'
     configurationFile '/etc/elasticsearch/jvm.options'
     configurationFile '/etc/elasticsearch/log4j2.properties'
     into('/etc/elasticsearch') {
-      fileMode 0750
+      dirMode 0750
+      fileMode 0660
       permissionGroup 'elasticsearch'
       includeEmptyDirs true
       createDirectoryEntry true
@@ -399,51 +435,38 @@ configure(subprojects.findAll { ['deb', 'rpm'].contains(it.name) }) {
     }
     configurationFile '/etc/init.d/elasticsearch'
     into('/etc/init.d') {
-      fileMode 0755
+      fileMode 0750
       fileType CONFIG | NOREPLACE
       from "${packagingFiles}/init.d/elasticsearch"
     }
     configurationFile project.expansions['path.env']
     into(new File(project.expansions['path.env']).getParent()) {
-      fileMode 0644
-      dirMode 0755
       fileType CONFIG | NOREPLACE
+      fileMode 0660
       from "${project.packagingFiles}/env/elasticsearch"
     }
 
     /**
      * Suck up all the empty directories that we need to install into the path.
      */
-    Closure suckUpEmptyDirectories = { path, u, g ->
+    Closure suckUpEmptyDirectories = { path, u, g, mode ->
       into(path) {
-        fileMode 0755
         from "${packagingFiles}/${path}"
         includeEmptyDirs true
         createDirectoryEntry true
         user u
         permissionGroup g
+        dirMode mode
+        fileMode mode
       }
     }
-    suckUpEmptyDirectories('/var/run', 'elasticsearch', 'elasticsearch')
-    suckUpEmptyDirectories('/var/log', 'elasticsearch', 'elasticsearch')
-    suckUpEmptyDirectories('/var/lib', 'elasticsearch', 'elasticsearch')
-    suckUpEmptyDirectories('/usr/share/elasticsearch', 'root', 'root')
+    suckUpEmptyDirectories('/var/run', 'elasticsearch', 'elasticsearch', 0755)
+    suckUpEmptyDirectories('/var/log', 'elasticsearch', 'elasticsearch', 0750)
+    suckUpEmptyDirectories('/var/lib', 'elasticsearch', 'elasticsearch', 0750)
+    suckUpEmptyDirectories('/usr/share/elasticsearch', 'root', 'root', 0755)
   }
 }
 
-// TODO: dependency checks should really be when building the jar itself, which would remove the need
-// for this hackery and instead we can do this inside the BuildPlugin
-task dependencyLicenses(type: DependencyLicensesTask) {
-  dependsOn = [dependencyFiles]
-  dependencies = dependencyFiles
-  mapping from: /lucene-.*/, to: 'lucene'
-  mapping from: /jackson-.*/, to: 'jackson'
-}
-task check(group: 'Verification', description: 'Runs all checks.', dependsOn: dependencyLicenses) {} // dummy task!
-task updateShas(type: UpdateShasTask) {
-  parentTask = dependencyLicenses
-}
-
 task run(type: RunTask) {
   distribution = 'zip'
 }
diff --git a/distribution/bwc/build.gradle b/distribution/bwc/build.gradle
new file mode 100644
index 0000000000000..7ae153958785f
--- /dev/null
+++ b/distribution/bwc/build.gradle
@@ -0,0 +1,133 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import org.elasticsearch.gradle.LoggedExec
+
+/**
+ * This is a dummy project which does a local checkout of the previous
+ * wire compat version's branch, and builds a snapshot. This allows backcompat
+ * tests to test against the next unreleased version, closest to this version,
+ * without relying on snapshots.
+ */
+
+String bwcVersion = wireCompatVersions[-1]
+if (bwcVersion.endsWith('-SNAPSHOT')) {
+  apply plugin: 'distribution'
+  // Not published so no need to assemble
+  tasks.remove(assemble)
+  build.dependsOn.remove('assemble')
+
+  def (String major, String minor, String bugfix) = bwcVersion.split('\\.')
+  def (String currentMajor, String currentMinor, String currentBugfix) = version.split('\\.')
+  String bwcBranch = currentMajor != major ? "${major}.x" : "${major}.${minor}"
+  File checkoutDir = file("${buildDir}/bwc/checkout-${bwcBranch}")
+  task createClone(type: LoggedExec) {
+    onlyIf { checkoutDir.exists() == false }
+    commandLine = ['git', 'clone', rootDir, checkoutDir]
+  }
+
+  // we use regular Exec here to ensure we always get output, regardless of logging level
+  task findUpstream(type: Exec) {
+    dependsOn createClone
+    workingDir = checkoutDir
+    commandLine = ['git', 'remote', '-v']
+    ignoreExitValue = true
+    ByteArrayOutputStream output = new ByteArrayOutputStream()
+    standardOutput = output
+    doLast {
+      if (execResult.exitValue != 0) {
+        output.toString('UTF-8').eachLine { line -> logger.error(line) }
+        execResult.assertNormalExitValue()
+      }
+      project.ext.upstreamExists = false
+      output.toString('UTF-8').eachLine {
+        if (it.contains("upstream")) {
+          project.ext.upstreamExists = true
+        }
+      }
+    }
+  }
+
+  task addUpstream(type: LoggedExec) {
+    dependsOn findUpstream
+    onlyIf { project.ext.upstreamExists == false }
+    workingDir = checkoutDir
+    commandLine = ['git', 'remote', 'add', 'upstream', 'https://github.com/elastic/elasticsearch.git']
+  }
+
+  task fetchLatest(type: LoggedExec) {
+    onlyIf { project.gradle.startParameter.isOffline() == false }
+    dependsOn addUpstream
+    workingDir = checkoutDir
+    commandLine = ['git', 'fetch', 'upstream']
+  }
+
+  // this is an Exec task so that the SHA that is checked out is logged
+  String buildMetadataKey = "bwc_refspec_${project.path.substring(1)}"
+  task checkoutBwcBranch(type: Exec) {
+    def String refspec = System.getProperty("tests.bwc.refspec", buildMetadata.get(buildMetadataKey, "upstream/${bwcBranch}"))
+    dependsOn fetchLatest
+    workingDir = checkoutDir
+    commandLine = ['git', 'checkout', refspec]
+  }
+
+  File buildMetadataFile = project.file("build/${project.name}/build_metadata")
+  task writeBuildMetadata(type: LoggedExec) {
+    dependsOn checkoutBwcBranch
+    workingDir = checkoutDir
+    commandLine = ['git', 'rev-parse', 'HEAD']
+    ignoreExitValue = true
+    ByteArrayOutputStream output = new ByteArrayOutputStream()
+    standardOutput = output
+    doLast {
+      if (execResult.exitValue != 0) {
+        output.toString('UTF-8').eachLine { line -> logger.error(line) }
+        execResult.assertNormalExitValue()
+      }
+      project.mkdir(buildMetadataFile.parent)
+      buildMetadataFile.setText("${buildMetadataKey}=${output.toString('UTF-8')}", 'UTF-8')
+    }
+  }
+
+  File bwcDeb = file("${checkoutDir}/distribution/deb/build/distributions/elasticsearch-${bwcVersion}.deb")
+  File bwcRpm = file("${checkoutDir}/distribution/rpm/build/distributions/elasticsearch-${bwcVersion}.rpm")
+  File bwcZip = file("${checkoutDir}/distribution/zip/build/distributions/elasticsearch-${bwcVersion}.zip")
+  task buildBwcVersion(type: GradleBuild) {
+    dependsOn checkoutBwcBranch, writeBuildMetadata
+    dir = checkoutDir
+    tasks = [':distribution:deb:assemble', ':distribution:rpm:assemble', ':distribution:zip:assemble']
+    startParameter.systemPropertiesArgs = ['build.snapshot': 'true']
+    doLast {
+      List missing = [bwcDeb, bwcRpm, bwcZip].grep { file ->
+        false == file.exists() }
+      if (false == missing.empty) {
+        throw new InvalidUserDataException(
+          "Building bwc version didn't generate expected files ${missing}")
+      }
+    }
+  }
+
+
+  artifacts {
+    'default' file: bwcDeb, name: 'elasticsearch', type: 'deb', builtBy: buildBwcVersion
+    'default' file: bwcRpm, name: 'elasticsearch', type: 'rpm', builtBy: buildBwcVersion
+    'default' file: bwcZip, name: 'elasticsearch', type: 'zip', builtBy: buildBwcVersion
+  }
+}
diff --git a/distribution/deb/src/main/packaging/init.d/elasticsearch b/distribution/deb/src/main/packaging/init.d/elasticsearch
index f04008ba7965f..f08e2b24cfa63 100755
--- a/distribution/deb/src/main/packaging/init.d/elasticsearch
+++ b/distribution/deb/src/main/packaging/init.d/elasticsearch
@@ -70,6 +70,10 @@ if [ -f "$DEFAULT" ]; then
 	. "$DEFAULT"
 fi
 
+if [ "$ES_USER" != "elasticsearch" ] || [ "$ES_GROUP" != "elasticsearch" ]; then
+    echo "WARNING: ES_USER and ES_GROUP are deprecated and will be removed in the next major version of Elasticsearch, got: [$ES_USER:$ES_GROUP]"
+fi
+
 # CONF_FILE setting was removed
 if [ ! -z "$CONF_FILE" ]; then
     echo "CONF_FILE setting is no longer supported. elasticsearch.yml must be placed in the config directory and cannot be renamed."
@@ -84,6 +88,17 @@ DAEMON_OPTS="-d -p $PID_FILE -Edefault.path.logs=$LOG_DIR -Edefault.path.data=$D
 export ES_JAVA_OPTS
 export JAVA_HOME
 export ES_INCLUDE
+export ES_JVM_OPTIONS
+
+# export unsupported variables so bin/elasticsearch can reject them and inform the user these are unsupported
+if test -n "$ES_MIN_MEM"; then export ES_MIN_MEM; fi
+if test -n "$ES_MAX_MEM"; then export ES_MAX_MEM; fi
+if test -n "$ES_HEAP_SIZE"; then export ES_HEAP_SIZE; fi
+if test -n "$ES_HEAP_NEWSIZE"; then export ES_HEAP_NEWSIZE; fi
+if test -n "$ES_DIRECT_SIZE"; then export ES_DIRECT_SIZE; fi
+if test -n "$ES_USE_IPV4"; then export ES_USE_IPV4; fi
+if test -n "$ES_GC_OPTS"; then export ES_GC_OPTS; fi
+if test -n "$ES_GC_LOG_FILE"; then export ES_GC_LOG_FILE; fi
 
 if [ ! -x "$DAEMON" ]; then
 	echo "The elasticsearch startup script does not exists or it is not executable, tried: $DAEMON"
@@ -137,7 +152,7 @@ case "$1" in
 	fi
 
 	# Start Daemon
-	start-stop-daemon -d $ES_HOME --start -b --user "$ES_USER" -c "$ES_USER" --pidfile "$PID_FILE" --exec $DAEMON -- $DAEMON_OPTS
+	start-stop-daemon -d $ES_HOME --start --user "$ES_USER" -c "$ES_USER" --pidfile "$PID_FILE" --exec $DAEMON -- $DAEMON_OPTS
 	return=$?
 	if [ $return -eq 0 ]; then
 		i=0
diff --git a/distribution/deb/src/test/java/org/elasticsearch/test/rest/DebClientYamlTestSuiteIT.java b/distribution/deb/src/test/java/org/elasticsearch/test/rest/DebClientYamlTestSuiteIT.java
index a94f0faf2d2b9..0fcdecde0684e 100644
--- a/distribution/deb/src/test/java/org/elasticsearch/test/rest/DebClientYamlTestSuiteIT.java
+++ b/distribution/deb/src/test/java/org/elasticsearch/test/rest/DebClientYamlTestSuiteIT.java
@@ -20,12 +20,8 @@
 package org.elasticsearch.test.rest;
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
-
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-
-import java.io.IOException;
 
 /** Rest integration test. Runs against a cluster started by {@code gradle integTest} */
 public class DebClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
@@ -34,7 +30,7 @@ public DebClientYamlTestSuiteIT(ClientYamlTestCandidate testCandidate) {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
-        return createParameters(0, 1);
+    public static Iterable<Object[]> parameters() throws Exception {
+        return createParameters();
     }
 }
diff --git a/distribution/integ-test-zip/build.gradle b/distribution/integ-test-zip/build.gradle
index f1f3a2c26c560..a99768f7cf9a9 100644
--- a/distribution/integ-test-zip/build.gradle
+++ b/distribution/integ-test-zip/build.gradle
@@ -17,7 +17,11 @@
  * under the License.
  */
 
+import org.elasticsearch.gradle.plugin.PluginBuildPlugin
+import org.apache.tools.ant.taskdefs.condition.Os
+
 task buildZip(type: Zip) {
+  dependsOn createPluginsDir
   baseName = 'elasticsearch'
   with archivesFiles
 }
@@ -30,8 +34,38 @@ artifacts {
 publishing {
   publications {
     nebula {
+      artifactId 'elasticsearch'
       artifact buildZip
     }
+    /* HUGE HACK: the underlying maven publication library refuses to deploy any attached artifacts
+     * when the packaging type is set to 'pom'. But Sonatype's OSS repositories require source files
+     * for artifacts that are of type 'zip'. We already publish the source and javadoc for Elasticsearch
+     * under the various other subprojects. So here we create another publication using the same
+     * name that has the "real" pom, and rely on the fact that gradle will execute the publish tasks
+     * in alphabetical order. This lets us publish the zip file and even though the pom says the
+     * type is 'pom' instead of 'zip'. We cannot setup a dependency between the tasks because the
+     * publishing tasks are created *extremely* late in the configuration phase, so that we cannot get
+     * ahold of the actual task. Furthermore, this entire hack only exists so we can make publishing to
+     * maven local work, since we publish to maven central externally. */
+    nebulaRealPom(MavenPublication) {
+      artifactId 'elasticsearch'
+      pom.packaging = 'pom'
+      pom.withXml { XmlProvider xml ->
+        Node root = xml.asNode()
+        root.appendNode('name', 'Elasticsearch')
+        root.appendNode('description', 'A Distributed RESTful Search Engine')
+        root.appendNode('url', PluginBuildPlugin.urlFromOrigin(project.scminfo.origin))
+        Node scmNode = root.appendNode('scm')
+        scmNode.appendNode('url', project.scminfo.origin)
+      }
+    }
+  }
+}
+
+integTestRunner {
+  if (Os.isFamily(Os.FAMILY_WINDOWS) && System.getProperty('tests.timeoutSuite') == null) {
+    // override the suite timeout to 30 mins for windows, because it has the most inefficient filesystem known to man
+    systemProperty 'tests.timeoutSuite', '1800000!'
   }
 }
 
diff --git a/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/CreatedLocationHeaderIT.java b/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/CreatedLocationHeaderIT.java
index 71ec194866266..c61b736bf6db1 100644
--- a/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/CreatedLocationHeaderIT.java
+++ b/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/CreatedLocationHeaderIT.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.test.rest;
 
+import org.apache.http.entity.ContentType;
 import org.apache.http.entity.StringEntity;
 import org.elasticsearch.client.Response;
 
@@ -47,12 +48,14 @@ public void testIndexWithoutId() throws IOException {
     public void testUpsert() throws IOException {
         locationTestCase(client().performRequest("POST", "test/test/1/_update", emptyMap(), new StringEntity("{"
                 + "\"doc\": {\"test\": \"test\"},"
-                + "\"doc_as_upsert\": true}")));
+                + "\"doc_as_upsert\": true}", ContentType.APPLICATION_JSON)));
     }
 
     private void locationTestCase(String method, String url) throws IOException {
-        locationTestCase(client().performRequest(method, url, emptyMap(), new StringEntity("{\"test\": \"test\"}")));
-        locationTestCase(client().performRequest(method, url + "?routing=cat", emptyMap(), new StringEntity("{\"test\": \"test\"}")));
+        locationTestCase(client().performRequest(method, url, emptyMap(),
+            new StringEntity("{\"test\": \"test\"}", ContentType.APPLICATION_JSON)));
+        locationTestCase(client().performRequest(method, url + "?routing=cat", emptyMap(),
+            new StringEntity("{\"test\": \"test\"}", ContentType.APPLICATION_JSON)));
     }
 
     private void locationTestCase(Response response) throws IOException {
diff --git a/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/IntegTestZipClientYamlTestSuiteIT.java b/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/IntegTestZipClientYamlTestSuiteIT.java
index ca54a3becd9bd..cd3a778280153 100644
--- a/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/IntegTestZipClientYamlTestSuiteIT.java
+++ b/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/IntegTestZipClientYamlTestSuiteIT.java
@@ -23,9 +23,6 @@
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-
-import java.io.IOException;
 
 /** Rest integration test. Runs against a cluster started by {@code gradle integTest} */
 public class IntegTestZipClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
@@ -34,7 +31,7 @@ public IntegTestZipClientYamlTestSuiteIT(ClientYamlTestCandidate testCandidate)
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
-        return createParameters(0, 1);
+    public static Iterable<Object[]> parameters() throws Exception {
+        return createParameters();
     }
 }
diff --git a/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/RequestsWithoutContentIT.java b/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/RequestsWithoutContentIT.java
new file mode 100644
index 0000000000000..ce72af26628a1
--- /dev/null
+++ b/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/RequestsWithoutContentIT.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.rest;
+
+import org.elasticsearch.client.ResponseException;
+
+import java.io.IOException;
+
+import static org.hamcrest.CoreMatchers.containsString;
+
+public class RequestsWithoutContentIT extends ESRestTestCase {
+
+    public void testIndexMissingBody() throws IOException {
+        ResponseException responseException = expectThrows(ResponseException.class, () -> client().performRequest(
+                randomBoolean() ? "POST" : "PUT", "/idx/type/123"));
+        assertResponseException(responseException, "request body is required");
+    }
+
+    public void testBulkMissingBody() throws IOException {
+        ResponseException responseException = expectThrows(ResponseException.class, () -> client().performRequest(
+                randomBoolean() ? "POST" : "PUT", "/_bulk"));
+        assertResponseException(responseException, "request body is required");
+    }
+
+    public void testPutSettingsMissingBody() throws IOException {
+        ResponseException responseException = expectThrows(ResponseException.class, () -> client().performRequest(
+                "PUT", "/_settings"));
+        assertResponseException(responseException, "request body is required");
+    }
+
+    public void testPutMappingsMissingBody() throws IOException {
+        ResponseException responseException = expectThrows(ResponseException.class, () -> client().performRequest(
+                randomBoolean() ? "POST" : "PUT", "/test_index/test_type/_mapping"));
+        assertResponseException(responseException, "request body is required");
+    }
+
+    public void testPutIndexTemplateMissingBody() throws IOException {
+        ResponseException responseException = expectThrows(ResponseException.class, () -> client().performRequest(
+                randomBoolean() ? "PUT" : "POST", "/_template/my_template"));
+        assertResponseException(responseException, "request body is required");
+    }
+
+    public void testMultiSearchMissingBody() throws IOException {
+        ResponseException responseException = expectThrows(ResponseException.class, () -> client().performRequest(
+                randomBoolean() ? "POST" : "GET", "/_msearch"));
+        assertResponseException(responseException, "request body or source parameter is required");
+    }
+
+    public void testPutPipelineMissingBody() throws IOException {
+        ResponseException responseException = expectThrows(ResponseException.class, () -> client().performRequest(
+                "PUT", "/_ingest/pipeline/my_pipeline"));
+        assertResponseException(responseException, "request body or source parameter is required");
+    }
+
+    public void testSimulatePipelineMissingBody() throws IOException {
+        ResponseException responseException = expectThrows(ResponseException.class, () -> client().performRequest(
+                randomBoolean() ? "POST" : "GET", "/_ingest/pipeline/my_pipeline/_simulate"));
+        assertResponseException(responseException, "request body or source parameter is required");
+    }
+
+    public void testPutScriptMissingBody() throws IOException {
+        ResponseException responseException = expectThrows(ResponseException.class, () -> client().performRequest(
+                randomBoolean() ? "POST" : "PUT", "/_scripts/lang"));
+        assertResponseException(responseException, "request body is required");
+    }
+
+    private static void assertResponseException(ResponseException responseException, String message) {
+        assertEquals(400, responseException.getResponse().getStatusLine().getStatusCode());
+        assertThat(responseException.getMessage(), containsString(message));
+    }
+}
diff --git a/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/WaitForRefreshAndCloseTests.java b/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/WaitForRefreshAndCloseTests.java
new file mode 100644
index 0000000000000..ca69a95deeda9
--- /dev/null
+++ b/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/WaitForRefreshAndCloseTests.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.rest;
+
+import org.apache.http.HttpEntity;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.util.EntityUtils;
+import org.elasticsearch.action.ActionFuture;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.client.ResponseListener;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+
+import static java.util.Collections.emptyMap;
+
+/**
+ * Tests that wait for refresh is fired if the index is closed.
+ */
+public class WaitForRefreshAndCloseTests extends ESRestTestCase {
+    @Before
+    public void setupIndex() throws IOException {
+        try {
+            client().performRequest("DELETE", indexName());
+        } catch (ResponseException e) {
+            // If we get an error, it should be because the index doesn't exist
+            assertEquals(404, e.getResponse().getStatusLine().getStatusCode());
+        }
+        client().performRequest("PUT", indexName(), emptyMap(), new StringEntity("{\"settings\":{\"refresh_interval\":-1}}"));
+    }
+
+    @After
+    public void cleanupIndex() throws IOException {
+        client().performRequest("DELETE", indexName());
+    }
+
+    private String indexName() {
+        return getTestName().toLowerCase(Locale.ROOT);
+    }
+
+    private String docPath() {
+        return indexName() + "/test/1";
+    }
+
+    public void testIndexAndThenClose() throws Exception {
+        closeWhileListenerEngaged(start("PUT", "", new StringEntity("{\"test\":\"test\"}")));
+    }
+
+    public void testUpdateAndThenClose() throws Exception {
+        client().performRequest("PUT", docPath(), emptyMap(), new StringEntity("{\"test\":\"test\"}"));
+        closeWhileListenerEngaged(start("POST", "/_update", new StringEntity("{\"doc\":{\"name\":\"test\"}}")));
+    }
+
+    public void testDeleteAndThenClose() throws Exception {
+        client().performRequest("PUT", docPath(), emptyMap(), new StringEntity("{\"test\":\"test\"}"));
+        closeWhileListenerEngaged(start("DELETE", "", null));
+    }
+
+    private void closeWhileListenerEngaged(ActionFuture<String> future) throws Exception {
+        // Wait for the refresh listener to start waiting
+        assertBusy(() -> {
+            Map<String, Object> stats;
+            try {
+                stats = entityAsMap(client().performRequest("GET", indexName() + "/_stats/refresh"));
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+            @SuppressWarnings("unchecked")
+            Map<String, Object> indices = (Map<String, Object>) stats.get("indices");
+            @SuppressWarnings("unchecked")
+            Map<String, Object> theIndex = (Map<String, Object>) indices.get(indexName());
+            @SuppressWarnings("unchecked")
+            Map<String, Object> total = (Map<String, Object>) theIndex.get("total");
+            @SuppressWarnings("unchecked")
+            Map<String, Object> refresh = (Map<String, Object>) total.get("refresh");
+            int listeners = (int) refresh.get("listeners");
+            assertEquals(1, listeners);
+        });
+
+        // Close the index. That should flush the listener.
+        client().performRequest("POST", indexName() + "/_close");
+
+        // The request shouldn't fail. It certainly shouldn't hang.
+        future.get();
+    }
+
+    private ActionFuture<String> start(String method, String path, HttpEntity body) {
+        PlainActionFuture<String> future = new PlainActionFuture<>();
+        Map<String, String> params = new HashMap<>();
+        params.put("refresh", "wait_for");
+        params.put("error_trace", "");
+        client().performRequestAsync(method, docPath() + path, params, body, new ResponseListener() {
+            @Override
+            public void onSuccess(Response response) {
+                try {
+                    future.onResponse(EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8));
+                } catch (IOException e) {
+                    future.onFailure(e);
+                }
+            }
+
+            @Override
+            public void onFailure(Exception exception) {
+                future.onFailure(exception);
+            }
+        });
+        return future;
+    }
+}
diff --git a/distribution/licenses/HdrHistogram-2.1.6.jar.sha1 b/distribution/licenses/HdrHistogram-2.1.6.jar.sha1
deleted file mode 100644
index 26fc16f2e87f0..0000000000000
--- a/distribution/licenses/HdrHistogram-2.1.6.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-7495feb7f71ee124bd2a7e7d83590e296d71d80e
\ No newline at end of file
diff --git a/distribution/licenses/hppc-0.7.1.jar.sha1 b/distribution/licenses/hppc-0.7.1.jar.sha1
deleted file mode 100644
index d3e00e1c47698..0000000000000
--- a/distribution/licenses/hppc-0.7.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-8b5057f74ea378c0150a1860874a3ebdcb713767
diff --git a/distribution/licenses/jackson-core-2.8.1.jar.sha1 b/distribution/licenses/jackson-core-2.8.1.jar.sha1
deleted file mode 100644
index b92131d6fab45..0000000000000
--- a/distribution/licenses/jackson-core-2.8.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-fd13b1c033741d48291315c6370f7d475a42dccf
\ No newline at end of file
diff --git a/distribution/licenses/jackson-dataformat-cbor-2.8.1.jar.sha1 b/distribution/licenses/jackson-dataformat-cbor-2.8.1.jar.sha1
deleted file mode 100644
index 7f1609bfd85b2..0000000000000
--- a/distribution/licenses/jackson-dataformat-cbor-2.8.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3a6fb7e75c9972559a78cf5cfc5a48a41a13ea40
\ No newline at end of file
diff --git a/distribution/licenses/jackson-dataformat-smile-2.8.1.jar.sha1 b/distribution/licenses/jackson-dataformat-smile-2.8.1.jar.sha1
deleted file mode 100644
index 114d656a388d3..0000000000000
--- a/distribution/licenses/jackson-dataformat-smile-2.8.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-005b73867bc12224946fc67fc8d49d9f5e698d7f
\ No newline at end of file
diff --git a/distribution/licenses/jackson-dataformat-yaml-2.8.1.jar.sha1 b/distribution/licenses/jackson-dataformat-yaml-2.8.1.jar.sha1
deleted file mode 100644
index 32ce0f7434484..0000000000000
--- a/distribution/licenses/jackson-dataformat-yaml-2.8.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-eb63166c723b0b4b9fb5298fca232a2f6612ec34
\ No newline at end of file
diff --git a/distribution/licenses/jna-4.2.2.jar.sha1 b/distribution/licenses/jna-4.2.2.jar.sha1
deleted file mode 100644
index 8b1acbbe5d7ab..0000000000000
--- a/distribution/licenses/jna-4.2.2.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-5012450aee579c3118ff09461d5ce210e0cdc2a9
\ No newline at end of file
diff --git a/distribution/licenses/joda-convert-1.2.jar.sha1 b/distribution/licenses/joda-convert-1.2.jar.sha1
deleted file mode 100644
index 37c3e87058082..0000000000000
--- a/distribution/licenses/joda-convert-1.2.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-35ec554f0cd00c956cc69051514d9488b1374dec
diff --git a/distribution/licenses/joda-convert-LICENSE.txt b/distribution/licenses/joda-convert-LICENSE.txt
deleted file mode 100644
index 75b52484ea471..0000000000000
--- a/distribution/licenses/joda-convert-LICENSE.txt
+++ /dev/null
@@ -1,202 +0,0 @@
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
diff --git a/distribution/licenses/joda-convert-NOTICE.txt b/distribution/licenses/joda-convert-NOTICE.txt
deleted file mode 100644
index dffbcf31cacf6..0000000000000
--- a/distribution/licenses/joda-convert-NOTICE.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-=============================================================================
-= NOTICE file corresponding to section 4d of the Apache License Version 2.0 =
-=============================================================================
-This product includes software developed by
-Joda.org (http://www.joda.org/).
diff --git a/distribution/licenses/joda-time-2.9.4.jar.sha1 b/distribution/licenses/joda-time-2.9.4.jar.sha1
deleted file mode 100644
index e9ea891bfeee2..0000000000000
--- a/distribution/licenses/joda-time-2.9.4.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-1c295b462f16702ebe720bbb08f62e1ba80da41b
\ No newline at end of file
diff --git a/distribution/licenses/jts-1.13.jar.sha1 b/distribution/licenses/jts-1.13.jar.sha1
deleted file mode 100644
index 2899139aab8c1..0000000000000
--- a/distribution/licenses/jts-1.13.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3ccfb9b60f04d71add996a666ceb8902904fd805
diff --git a/distribution/licenses/log4j-1.2-api-2.6.2.jar.sha1 b/distribution/licenses/log4j-1.2-api-2.6.2.jar.sha1
deleted file mode 100644
index 5bf4bcab46fad..0000000000000
--- a/distribution/licenses/log4j-1.2-api-2.6.2.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3b4c5a8b734b6a29b2f03380535a48da6284b210
\ No newline at end of file
diff --git a/distribution/licenses/log4j-api-2.6.2.jar.sha1 b/distribution/licenses/log4j-api-2.6.2.jar.sha1
deleted file mode 100644
index e4f9af7497c94..0000000000000
--- a/distribution/licenses/log4j-api-2.6.2.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-bd1b74a5d170686362091c7cf596bbc3adf5c09b
\ No newline at end of file
diff --git a/distribution/licenses/log4j-core-2.6.2.jar.sha1 b/distribution/licenses/log4j-core-2.6.2.jar.sha1
deleted file mode 100644
index 0ac4323411c56..0000000000000
--- a/distribution/licenses/log4j-core-2.6.2.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-00a91369f655eb1639c6aece5c5eb5108db18306
\ No newline at end of file
diff --git a/distribution/licenses/lucene-analyzers-common-6.2.0.jar.sha1 b/distribution/licenses/lucene-analyzers-common-6.2.0.jar.sha1
deleted file mode 100644
index 57aec3f4ac21d..0000000000000
--- a/distribution/licenses/lucene-analyzers-common-6.2.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d254d52dd394b5079129f3d5f3bf4f2d44a5936e
\ No newline at end of file
diff --git a/distribution/licenses/lucene-backward-codecs-6.2.0.jar.sha1 b/distribution/licenses/lucene-backward-codecs-6.2.0.jar.sha1
deleted file mode 100644
index 04aefc62f613d..0000000000000
--- a/distribution/licenses/lucene-backward-codecs-6.2.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b625bb21456b3c0d1e5e431bced125cb060c1abd
\ No newline at end of file
diff --git a/distribution/licenses/lucene-core-6.2.0.jar.sha1 b/distribution/licenses/lucene-core-6.2.0.jar.sha1
deleted file mode 100644
index 2d74124e62411..0000000000000
--- a/distribution/licenses/lucene-core-6.2.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-849ee62525a294416802be2cacc66c80352f6f13
\ No newline at end of file
diff --git a/distribution/licenses/lucene-grouping-6.2.0.jar.sha1 b/distribution/licenses/lucene-grouping-6.2.0.jar.sha1
deleted file mode 100644
index 6ba525a038fc5..0000000000000
--- a/distribution/licenses/lucene-grouping-6.2.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-9527fedfd5acc624b2bb3f862bd99fb8f470b053
\ No newline at end of file
diff --git a/distribution/licenses/lucene-highlighter-6.2.0.jar.sha1 b/distribution/licenses/lucene-highlighter-6.2.0.jar.sha1
deleted file mode 100644
index c258e3fb85026..0000000000000
--- a/distribution/licenses/lucene-highlighter-6.2.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-7ca342372a3f45e32bbd21cecaa757e38eccb8a5
\ No newline at end of file
diff --git a/distribution/licenses/lucene-join-6.2.0.jar.sha1 b/distribution/licenses/lucene-join-6.2.0.jar.sha1
deleted file mode 100644
index 01989e96a588e..0000000000000
--- a/distribution/licenses/lucene-join-6.2.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-da0b8de98511abd4fe9c7d48a353d17854c5ed46
\ No newline at end of file
diff --git a/distribution/licenses/lucene-memory-6.2.0.jar.sha1 b/distribution/licenses/lucene-memory-6.2.0.jar.sha1
deleted file mode 100644
index b8a4a87efe26a..0000000000000
--- a/distribution/licenses/lucene-memory-6.2.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-bc9e075b1ee051c8e5246c237c38d8e71dab8a66
\ No newline at end of file
diff --git a/distribution/licenses/lucene-misc-6.2.0.jar.sha1 b/distribution/licenses/lucene-misc-6.2.0.jar.sha1
deleted file mode 100644
index f4e081865ad43..0000000000000
--- a/distribution/licenses/lucene-misc-6.2.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-94ddde6312566a4da4a50a88e453b6c82c759b41
\ No newline at end of file
diff --git a/distribution/licenses/lucene-queries-6.2.0.jar.sha1 b/distribution/licenses/lucene-queries-6.2.0.jar.sha1
deleted file mode 100644
index f7270a23afe0f..0000000000000
--- a/distribution/licenses/lucene-queries-6.2.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-dce47238f78e3e97d91dc6fefa9f46f07866bc2b
\ No newline at end of file
diff --git a/distribution/licenses/lucene-queryparser-6.2.0.jar.sha1 b/distribution/licenses/lucene-queryparser-6.2.0.jar.sha1
deleted file mode 100644
index 8e95aa600ec3c..0000000000000
--- a/distribution/licenses/lucene-queryparser-6.2.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-17ef728ac15e668bfa1105321611548424637645
\ No newline at end of file
diff --git a/distribution/licenses/lucene-sandbox-6.2.0.jar.sha1 b/distribution/licenses/lucene-sandbox-6.2.0.jar.sha1
deleted file mode 100644
index 1f34be3033df0..0000000000000
--- a/distribution/licenses/lucene-sandbox-6.2.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-520183f7b9aba77a26e224760c420a3844b0631a
\ No newline at end of file
diff --git a/distribution/licenses/lucene-spatial-6.2.0.jar.sha1 b/distribution/licenses/lucene-spatial-6.2.0.jar.sha1
deleted file mode 100644
index 22e81792e4057..0000000000000
--- a/distribution/licenses/lucene-spatial-6.2.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-8dba929b66927b936fbc76103b109ad6c824daee
\ No newline at end of file
diff --git a/distribution/licenses/lucene-spatial-extras-6.2.0.jar.sha1 b/distribution/licenses/lucene-spatial-extras-6.2.0.jar.sha1
deleted file mode 100644
index d5e8f379d78c0..0000000000000
--- a/distribution/licenses/lucene-spatial-extras-6.2.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3b5a6ef5cd90c0218a72e9e2f7e60104be2447da
\ No newline at end of file
diff --git a/distribution/licenses/lucene-spatial3d-6.2.0.jar.sha1 b/distribution/licenses/lucene-spatial3d-6.2.0.jar.sha1
deleted file mode 100644
index d0ce5275a269c..0000000000000
--- a/distribution/licenses/lucene-spatial3d-6.2.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-fcdb0567725722c5145149d1502848b6a96ec18d
\ No newline at end of file
diff --git a/distribution/licenses/lucene-suggest-6.2.0.jar.sha1 b/distribution/licenses/lucene-suggest-6.2.0.jar.sha1
deleted file mode 100644
index 39392ad115816..0000000000000
--- a/distribution/licenses/lucene-suggest-6.2.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3d9d526c51f483d27f425c75d7e56bc0849242d6
\ No newline at end of file
diff --git a/distribution/licenses/t-digest-3.0.jar.sha1 b/distribution/licenses/t-digest-3.0.jar.sha1
deleted file mode 100644
index 6deec27800f12..0000000000000
--- a/distribution/licenses/t-digest-3.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-84ccf145ac2215e6bfa63baa3101c0af41017cfc
diff --git a/distribution/rpm/build.gradle b/distribution/rpm/build.gradle
index 185f558ff029f..6f8299522ca35 100644
--- a/distribution/rpm/build.gradle
+++ b/distribution/rpm/build.gradle
@@ -18,7 +18,7 @@
  */
 
 task buildRpm(type: Rpm) {
-  dependsOn dependencyFiles, preparePackagingFiles
+  dependsOn preparePackagingFiles
   baseName 'elasticsearch' // this is what pom generation uses for artifactId
   // Follow elasticsearch's rpm file naming convention
   archiveName "${packageName}-${project.version}.rpm"
@@ -36,6 +36,17 @@ task buildRpm(type: Rpm) {
   fileMode 0644
   addParentDirs false
   // TODO ospackage doesn't support icon but we used to have one
+
+  // Declare the folders so that the RPM package manager removes
+  // them when upgrading or removing the package
+  directory('/usr/share/elasticsearch/bin', 0755)
+  directory('/usr/share/elasticsearch/lib', 0755)
+  directory('/usr/share/elasticsearch/modules', 0755)
+  modulesFiles.eachFile { FileCopyDetails fcp ->
+    if (fcp.name == "plugin-descriptor.properties") {
+      directory('/usr/share/elasticsearch/modules/' + fcp.file.parentFile.name, 0755)
+    }
+  }
 }
 
 artifacts {
diff --git a/distribution/rpm/src/main/packaging/init.d/elasticsearch b/distribution/rpm/src/main/packaging/init.d/elasticsearch
index 8f1d93dcbdc4f..e63e3e6de5046 100644
--- a/distribution/rpm/src/main/packaging/init.d/elasticsearch
+++ b/distribution/rpm/src/main/packaging/init.d/elasticsearch
@@ -49,6 +49,10 @@ if [ -f "$ES_ENV_FILE" ]; then
     . "$ES_ENV_FILE"
 fi
 
+if [ "$ES_USER" != "elasticsearch" ] || [ "$ES_GROUP" != "elasticsearch" ]; then
+    echo "WARNING: ES_USER and ES_GROUP are deprecated and will be removed in the next major version of Elasticsearch, got: [$ES_USER:$ES_GROUP]"
+fi
+
 # CONF_FILE setting was removed
 if [ ! -z "$CONF_FILE" ]; then
     echo "CONF_FILE setting is no longer supported. elasticsearch.yml must be placed in the config directory and cannot be renamed."
@@ -60,9 +64,20 @@ prog="elasticsearch"
 pidfile="$PID_DIR/${prog}.pid"
 
 export ES_JAVA_OPTS
-export ES_STARTUP_SLEEP_TIME
 export JAVA_HOME
 export ES_INCLUDE
+export ES_JVM_OPTIONS
+export ES_STARTUP_SLEEP_TIME
+
+# export unsupported variables so bin/elasticsearch can reject them and inform the user these are unsupported
+if test -n "$ES_MIN_MEM"; then export ES_MIN_MEM; fi
+if test -n "$ES_MAX_MEM"; then export ES_MAX_MEM; fi
+if test -n "$ES_HEAP_SIZE"; then export ES_HEAP_SIZE; fi
+if test -n "$ES_HEAP_NEWSIZE"; then export ES_HEAP_NEWSIZE; fi
+if test -n "$ES_DIRECT_SIZE"; then export ES_DIRECT_SIZE; fi
+if test -n "$ES_USE_IPV4"; then export ES_USE_IPV4; fi
+if test -n "$ES_GC_OPTS"; then export ES_GC_OPTS; fi
+if test -n "$ES_GC_LOG_FILE"; then export ES_GC_LOG_FILE; fi
 
 lockfile=/var/lock/subsys/$prog
 
diff --git a/distribution/rpm/src/test/java/org/elasticsearch/test/rest/RpmClientYamlTestSuiteIT.java b/distribution/rpm/src/test/java/org/elasticsearch/test/rest/RpmClientYamlTestSuiteIT.java
index a5b7f46269ff9..388f5ee6517cc 100644
--- a/distribution/rpm/src/test/java/org/elasticsearch/test/rest/RpmClientYamlTestSuiteIT.java
+++ b/distribution/rpm/src/test/java/org/elasticsearch/test/rest/RpmClientYamlTestSuiteIT.java
@@ -23,9 +23,6 @@
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-
-import java.io.IOException;
 
 /** Rest integration test. Runs against a cluster started by {@code gradle integTest} */
 public class RpmClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
@@ -34,7 +31,7 @@ public RpmClientYamlTestSuiteIT(ClientYamlTestCandidate testCandidate) {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
-        return createParameters(0, 1);
+    public static Iterable<Object[]> parameters() throws Exception {
+        return createParameters();
     }
 }
diff --git a/distribution/src/main/packaging/scripts/postinst b/distribution/src/main/packaging/scripts/postinst
index 451ec3457d36e..6811ad6a11406 100644
--- a/distribution/src/main/packaging/scripts/postinst
+++ b/distribution/src/main/packaging/scripts/postinst
@@ -51,6 +51,11 @@ case "$1" in
     ;;
 esac
 
+# to pick up /usr/lib/sysctl.d/elasticsearch.conf
+if command -v systemctl > /dev/null; then
+    systemctl restart systemd-sysctl.service || true
+fi
+
 if [ "x$IS_UPGRADE" != "xtrue" ]; then
     if command -v systemctl >/dev/null; then
         echo "### NOT starting on installation, please execute the following statements to configure elasticsearch service to start automatically using systemd"
@@ -99,11 +104,12 @@ fi
 chown -R $ES_USER:$ES_GROUP /var/lib/elasticsearch
 chown -R $ES_USER:$ES_GROUP /var/log/elasticsearch
 chown -R root:$ES_GROUP /etc/elasticsearch
+chmod g+s /etc/elasticsearch
 chmod 0750 /etc/elasticsearch
 chmod 0750 /etc/elasticsearch/scripts
 
 if [ -f /etc/sysconfig/elasticsearch ]; then
-   chmod 0644 /etc/sysconfig/elasticsearch
+   chmod 0660 /etc/sysconfig/elasticsearch
 fi
 
 ${scripts.footer}
diff --git a/distribution/src/main/packaging/scripts/preinst b/distribution/src/main/packaging/scripts/preinst
index 7ffbd3129a0ca..14df9333327de 100644
--- a/distribution/src/main/packaging/scripts/preinst
+++ b/distribution/src/main/packaging/scripts/preinst
@@ -21,6 +21,10 @@ if [ -f "$ES_ENV_FILE" ]; then
     . "$ES_ENV_FILE"
 fi
 
+if [ "$ES_USER" != "elasticsearch" ] || [ "$ES_GROUP" != "elasticsearch" ]; then
+    echo "WARNING: ES_USER and ES_GROUP are deprecated and will be removed in the next major version of Elasticsearch, got: [$ES_USER:$ES_GROUP]"
+fi
+
 case "$1" in
 
     # Debian ####################################################
diff --git a/distribution/src/main/packaging/scripts/prerm b/distribution/src/main/packaging/scripts/prerm
index 592f2f9948f84..bca03b2576600 100644
--- a/distribution/src/main/packaging/scripts/prerm
+++ b/distribution/src/main/packaging/scripts/prerm
@@ -79,12 +79,13 @@ if [ "$REMOVE_SERVICE" = "true" ]; then
     if command -v update-rc.d >/dev/null; then
         update-rc.d elasticsearch remove >/dev/null || true
     fi
-fi
 
-SCRIPTS_DIR="/etc/elasticsearch/scripts"
-# delete the scripts directory if and only if empty
-if [ -d "$SCRIPTS_DIR" ]; then
-    rmdir --ignore-fail-on-non-empty "$SCRIPTS_DIR"
+    SCRIPTS_DIR="/etc/elasticsearch/scripts"
+    # delete the scripts directory if and only if empty
+    if [ -d "$SCRIPTS_DIR" ]; then
+        rmdir --ignore-fail-on-non-empty "$SCRIPTS_DIR"
+    fi
 fi
 
+
 ${scripts.footer}
diff --git a/distribution/src/main/packaging/systemd/elasticsearch.service b/distribution/src/main/packaging/systemd/elasticsearch.service
index 8971bc2bc1968..45b741779f79c 100644
--- a/distribution/src/main/packaging/systemd/elasticsearch.service
+++ b/distribution/src/main/packaging/systemd/elasticsearch.service
@@ -21,20 +21,31 @@ ExecStartPre=/usr/share/elasticsearch/bin/elasticsearch-systemd-pre-exec
 
 ExecStart=/usr/share/elasticsearch/bin/elasticsearch \
                                                 -p ${PID_DIR}/elasticsearch.pid \
+                                                --quiet \
                                                 -Edefault.path.logs=${LOG_DIR} \
                                                 -Edefault.path.data=${DATA_DIR} \
                                                 -Edefault.path.conf=${CONF_DIR}
 
+# StandardOutput is configured to redirect to journalctl since
+# some error messages may be logged in standard output before
+# elasticsearch logging system is initialized. Elasticsearch
+# stores its logs in /var/log/elasticsearch and does not use
+# journalctl by default. If you also want to enable journalctl
+# logging, you can simply remove the "quiet" option from ExecStart.
 StandardOutput=journal
 StandardError=inherit
 
 # Specifies the maximum file descriptor number that can be opened by this process
 LimitNOFILE=65536
 
-# Specifies the maximum number of bytes of memory that may be locked into RAM
-# Set to "infinity" if you use the 'bootstrap.memory_lock: true' option
-# in elasticsearch.yml and 'MAX_LOCKED_MEMORY=unlimited' in ${path.env}
-#LimitMEMLOCK=infinity
+# Specifies the maximum number of processes
+LimitNPROC=2048
+
+# Specifies the maximum size of virtual memory
+LimitAS=infinity
+
+# Specifies the maximum file size
+LimitFSIZE=infinity
 
 # Disable timeout logic and wait until process is stopped
 TimeoutStopSec=0
@@ -42,6 +53,9 @@ TimeoutStopSec=0
 # SIGTERM signal is used to stop the Java process
 KillSignal=SIGTERM
 
+# Send the signal only to the JVM rather than its control group
+KillMode=process
+
 # Java process is never killed
 SendSIGKILL=no
 
diff --git a/distribution/src/main/resources/bin/elasticsearch b/distribution/src/main/resources/bin/elasticsearch
index b41304f70d06d..8a0eda60f0ce3 100755
--- a/distribution/src/main/resources/bin/elasticsearch
+++ b/distribution/src/main/resources/bin/elasticsearch
@@ -47,7 +47,7 @@
 
 if echo '${project.name}' | grep project.name > /dev/null ; then
     cat >&2 << EOF
-Error: You must build the project with Maven or download a pre-built package
+Error: You must build the project with Gradle or download a pre-built package
 before you can run Elasticsearch. See 'Building from Source' in README.textile
 or visit https://www.elastic.co/download to get a pre-built package.
 EOF
@@ -172,6 +172,13 @@ if [ "x$JAVA_OPTS" != "x" ]; then
     echo "Please pass JVM parameters via ES_JAVA_OPTS instead"
 fi
 
+"$JAVA" -cp "$ES_CLASSPATH" org.elasticsearch.tools.JavaVersionChecker
+
+if [ $? -ne 0 ]; then
+    echo "Elasticsearch requires at least Java 8 but your Java version from $JAVA does not meet this requirement"
+    exit 1
+fi
+
 # full hostname passed through cut for portability on systems that do not support hostname -s
 # export on separate line for shells that do not support combining definition and export
 HOSTNAME=`hostname | cut -d. -f1`
diff --git a/distribution/src/main/resources/bin/elasticsearch-keystore b/distribution/src/main/resources/bin/elasticsearch-keystore
new file mode 100755
index 0000000000000..d99d11aed49fb
--- /dev/null
+++ b/distribution/src/main/resources/bin/elasticsearch-keystore
@@ -0,0 +1,91 @@
+#!/bin/bash
+
+CDPATH=""
+SCRIPT="$0"
+
+# SCRIPT may be an arbitrarily deep series of symlinks. Loop until we have the concrete path.
+while [ -h "$SCRIPT" ] ; do
+  ls=`ls -ld "$SCRIPT"`
+  # Drop everything prior to ->
+  link=`expr "$ls" : '.*-> \(.*\)$'`
+  if expr "$link" : '/.*' > /dev/null; then
+    SCRIPT="$link"
+  else
+    SCRIPT=`dirname "$SCRIPT"`/"$link"
+  fi
+done
+
+# determine elasticsearch home
+ES_HOME=`dirname "$SCRIPT"`/..
+
+# make ELASTICSEARCH_HOME absolute
+ES_HOME=`cd "$ES_HOME"; pwd`
+
+
+# Sets the default values for elasticsearch variables used in this script
+if [ -z "$CONF_DIR" ]; then
+  CONF_DIR="${path.conf}"
+fi
+
+# The default env file is defined at building/packaging time.
+# For a ${project.name} package, the value is "${path.env}".
+ES_ENV_FILE="${path.env}"
+
+# If an include is specified with the ES_INCLUDE environment variable, use it
+if [ -n "$ES_INCLUDE" ]; then
+    ES_ENV_FILE="$ES_INCLUDE"
+fi
+
+# Source the environment file
+if [ -n "$ES_ENV_FILE" ]; then
+
+  # If the ES_ENV_FILE is not found, try to resolve the path
+  # against the ES_HOME directory
+  if [ ! -f "$ES_ENV_FILE" ]; then
+      ES_ENV_FILE="$ELASTIC_HOME/$ES_ENV_FILE"
+  fi
+
+  . "$ES_ENV_FILE"
+  if [ $? -ne 0 ]; then
+      echo "Unable to source environment file: $ES_ENV_FILE" >&2
+      exit 1
+  fi
+fi
+
+# don't let JAVA_TOOL_OPTIONS slip in (e.g. crazy agents in ubuntu)
+# works around https://bugs.launchpad.net/ubuntu/+source/jayatana/+bug/1441487
+if [ "x$JAVA_TOOL_OPTIONS" != "x" ]; then
+    echo "Warning: Ignoring JAVA_TOOL_OPTIONS=$JAVA_TOOL_OPTIONS"
+    unset JAVA_TOOL_OPTIONS
+fi
+
+# CONF_FILE setting was removed
+if [ ! -z "$CONF_FILE" ]; then
+    echo "CONF_FILE setting is no longer supported. elasticsearch.yml must be placed in the config directory and cannot be renamed."
+    exit 1
+fi
+
+if [ -x "$JAVA_HOME/bin/java" ]; then
+    JAVA=$JAVA_HOME/bin/java
+else
+    JAVA=`which java`
+fi
+
+if [ ! -x "$JAVA" ]; then
+    echo "Could not find any executable java binary. Please install java in your PATH or set JAVA_HOME"
+    exit 1
+fi
+
+# full hostname passed through cut for portability on systems that do not support hostname -s
+# export on separate line for shells that do not support combining definition and export
+HOSTNAME=`hostname | cut -d. -f1`
+export HOSTNAME
+
+declare -a args=("$@")
+path_props=(-Des.path.home="$ES_HOME")
+
+if [ -e "$CONF_DIR" ]; then
+  path_props=("${path_props[@]}" -Des.path.conf="$CONF_DIR")
+fi
+
+exec "$JAVA" $ES_JAVA_OPTS -Delasticsearch "${path_props[@]}" -cp "$ES_HOME/lib/*" org.elasticsearch.common.settings.KeyStoreCli "${args[@]}"
diff --git a/distribution/src/main/resources/bin/elasticsearch-keystore.bat b/distribution/src/main/resources/bin/elasticsearch-keystore.bat
new file mode 100644
index 0000000000000..ceb7e96665d7e
--- /dev/null
+++ b/distribution/src/main/resources/bin/elasticsearch-keystore.bat
@@ -0,0 +1,30 @@
+@echo off
+
+SETLOCAL enabledelayedexpansion
+
+IF DEFINED JAVA_HOME (
+  set JAVA=%JAVA_HOME%\bin\java.exe
+) ELSE (
+  FOR %%I IN (java.exe) DO set JAVA=%%~$PATH:I
+)
+IF NOT EXIST "%JAVA%" (
+  ECHO Could not find any executable java binary. Please install java in your PATH or set JAVA_HOME 1>&2
+  EXIT /B 1
+)
+
+set SCRIPT_DIR=%~dp0
+for %%I in ("%SCRIPT_DIR%..") do set ES_HOME=%%~dpfI
+
+TITLE Elasticsearch Plugin Manager ${project.version}
+
+SET path_props=-Des.path.home="%ES_HOME%"
+IF DEFINED CONF_DIR (
+  SET path_props=!path_props! -Des.path.conf="%CONF_DIR%"
+)
+
+SET args=%*
+SET HOSTNAME=%COMPUTERNAME%
+
+"%JAVA%" %ES_JAVA_OPTS% !path_props! -cp "%ES_HOME%/lib/*;" "org.elasticsearch.common.settings.KeyStoreCli" !args!
+
+ENDLOCAL
diff --git a/distribution/src/main/resources/bin/elasticsearch-plugin b/distribution/src/main/resources/bin/elasticsearch-plugin
index 06f8c5b8c2719..098d9124498e1 100755
--- a/distribution/src/main/resources/bin/elasticsearch-plugin
+++ b/distribution/src/main/resources/bin/elasticsearch-plugin
@@ -82,9 +82,10 @@ HOSTNAME=`hostname | cut -d. -f1`
 export HOSTNAME
 
 declare -a args=("$@")
+path_props=(-Des.path.home="$ES_HOME")
 
 if [ -e "$CONF_DIR" ]; then
-  args=("${args[@]}" -Edefault.path.conf="$CONF_DIR")
+  path_props=("${path_props[@]}" -Des.path.conf="$CONF_DIR")
 fi
 
-exec "$JAVA" $ES_JAVA_OPTS -Delasticsearch -Des.path.home="$ES_HOME" -cp "$ES_HOME/lib/*" org.elasticsearch.plugins.PluginCli "${args[@]}"
+exec "$JAVA" $ES_JAVA_OPTS -Delasticsearch "${path_props[@]}" -cp "$ES_HOME/lib/*" org.elasticsearch.plugins.PluginCli "${args[@]}"
diff --git a/distribution/src/main/resources/bin/elasticsearch-plugin.bat b/distribution/src/main/resources/bin/elasticsearch-plugin.bat
index 54c0b86b88023..88552089346e0 100644
--- a/distribution/src/main/resources/bin/elasticsearch-plugin.bat
+++ b/distribution/src/main/resources/bin/elasticsearch-plugin.bat
@@ -3,11 +3,11 @@
 SETLOCAL enabledelayedexpansion
 
 IF DEFINED JAVA_HOME (
-  set JAVA=%JAVA_HOME%\bin\java.exe
+  set JAVA="%JAVA_HOME%\bin\java.exe"
 ) ELSE (
   FOR %%I IN (java.exe) DO set JAVA=%%~$PATH:I
 )
-IF NOT EXIST "%JAVA%" (
+IF NOT EXIST %JAVA% (
   ECHO Could not find any executable java binary. Please install java in your PATH or set JAVA_HOME 1>&2
   EXIT /B 1
 )
@@ -17,9 +17,14 @@ for %%I in ("%SCRIPT_DIR%..") do set ES_HOME=%%~dpfI
 
 TITLE Elasticsearch Plugin Manager ${project.version}
 
+SET path_props=-Des.path.home="%ES_HOME%"
+IF DEFINED CONF_DIR (
+  SET path_props=!path_props! -Des.path.conf="%CONF_DIR%"
+)
+
 SET args=%*
 SET HOSTNAME=%COMPUTERNAME%
 
-"%JAVA%" %ES_JAVA_OPTS% -Des.path.home="%ES_HOME%" -cp "%ES_HOME%/lib/*;" "org.elasticsearch.plugins.PluginCli" !args!
+%JAVA% %ES_JAVA_OPTS% !path_props! -cp "%ES_HOME%/lib/*;" "org.elasticsearch.plugins.PluginCli" !args!
 
 ENDLOCAL
diff --git a/distribution/src/main/resources/bin/elasticsearch-service.bat b/distribution/src/main/resources/bin/elasticsearch-service.bat
new file mode 100644
index 0000000000000..28cf8d00322aa
--- /dev/null
+++ b/distribution/src/main/resources/bin/elasticsearch-service.bat
@@ -0,0 +1,349 @@
+@echo off
+SETLOCAL enabledelayedexpansion
+
+TITLE Elasticsearch Service ${project.version}
+
+rem TODO: remove for Elasticsearch 6.x
+set bad_env_var=0
+if not "%ES_MIN_MEM%" == "" set bad_env_var=1
+if not "%ES_MAX_MEM%" == "" set bad_env_var=1
+if not "%ES_HEAP_SIZE%" == "" set bad_env_var=1
+if not "%ES_HEAP_NEWSIZE%" == "" set bad_env_var=1
+if not "%ES_DIRECT_SIZE%" == "" set bad_env_var=1
+if not "%ES_USE_IPV4%" == "" set bad_env_var=1
+if not "%ES_GC_OPTS%" == "" set bad_env_var=1
+if not "%ES_GC_LOG_FILE%" == "" set bad_env_var=1
+if %bad_env_var% == 1 (
+    echo Error: encountered environment variables that are no longer supported
+    echo Use jvm.options or ES_JAVA_OPTS to configure the JVM
+    if not "%ES_MIN_MEM%" == "" echo ES_MIN_MEM=%ES_MIN_MEM%: set -Xms%ES_MIN_MEM% in jvm.options or add "-Xms%ES_MIN_MEM%" to ES_JAVA_OPTS
+    if not "%ES_MAX_MEM%" == "" echo ES_MAX_MEM=%ES_MAX_MEM%: set -Xms%ES_MAX_MEM% in jvm.options or add "-Xmx%ES_MAX_MEM%" to ES_JAVA_OPTS
+    if not "%ES_HEAP_SIZE%" == "" echo ES_HEAP_SIZE=%ES_HEAP_SIZE%: set -Xms%ES_HEAP_SIZE% and -Xmx%ES_HEAP_SIZE% in jvm.options or add "-Xms%ES_HEAP_SIZE% -Xmx%ES_HEAP_SIZE%" to ES_JAVA_OPTS
+    if not "%ES_HEAP_NEWSIZE%" == "" echo ES_HEAP_NEWSIZE=%ES_HEAP_NEWSIZE%: set -Xmn%ES_HEAP_NEWSIZE% in jvm.options or add "-Xmn%ES_HEAP_SIZE%" to ES_JAVA_OPTS
+    if not "%ES_DIRECT_SIZE%" == "" echo ES_DIRECT_SIZE=%ES_DIRECT_SIZE%: set -XX:MaxDirectMemorySize=%ES_DIRECT_SIZE% in jvm.options or add "-XX:MaxDirectMemorySize=%ES_DIRECT_SIZE%" to ES_JAVA_OPTS
+    if not "%ES_USE_IPV4%" == "" echo ES_USE_IPV4=%ES_USE_IPV4%: set -Djava.net.preferIPv4Stack=true in jvm.options or add "-Djava.net.preferIPv4Stack=true" to ES_JAVA_OPTS
+    if not "%ES_GC_OPTS%" == "" echo ES_GC_OPTS=%ES_GC_OPTS%: set %ES_GC_OPTS: = and % in jvm.options or add "%ES_GC_OPTS%" to ES_JAVA_OPTS
+    if not "%ES_GC_LOG_FILE%" == "" echo ES_GC_LOG_FILE=%ES_GC_LOG_FILE%: set -Xloggc:%ES_GC_LOG_FILE% in jvm.options or add "-Xloggc:%ES_GC_LOG_FILE%" to ES_JAVA_OPTS"
+    exit /b 1
+)
+rem end TODO: remove for Elasticsearch 6.x
+
+IF DEFINED JAVA_HOME (
+  SET JAVA="%JAVA_HOME%\bin\java.exe"
+) ELSE (
+  FOR %%I IN (java.exe) DO set JAVA=%%~$PATH:I
+)
+IF NOT EXIST %JAVA% (
+  ECHO Could not find any executable java binary. Please install java in your PATH or set JAVA_HOME 1>&2
+  EXIT /B 1
+)
+IF DEFINED JAVA_HOME GOTO :cont
+
+IF NOT %JAVA:~-13% == "\bin\java.exe" (
+  FOR /f "tokens=2 delims=[]" %%I IN ('dir %JAVA%') DO @set JAVA=%%I
+)
+IF %JAVA:~-13% == "\bin\java.exe" (
+  SET JAVA_HOME=%JAVA:~0,-13%
+)
+
+:cont
+if not "%CONF_FILE%" == "" goto conffileset
+
+set SCRIPT_DIR=%~dp0
+for %%I in ("%SCRIPT_DIR%..") do set ES_HOME=%%~dpfI
+
+%JAVA% -Xmx50M -version > nul 2>&1
+
+if errorlevel 1 (
+	echo Warning: Could not start JVM to detect version, defaulting to x86:
+	goto x86
+)
+
+%JAVA% -Xmx50M -version 2>&1 | "%windir%\System32\find" "64-Bit" >nul:
+
+if errorlevel 1 goto x86
+set EXECUTABLE=%ES_HOME%\bin\elasticsearch-service-x64.exe
+set SERVICE_ID=elasticsearch-service-x64
+set ARCH=64-bit
+goto checkExe
+
+:x86
+set EXECUTABLE=%ES_HOME%\bin\elasticsearch-service-x86.exe
+set SERVICE_ID=elasticsearch-service-x86
+set ARCH=32-bit
+
+:checkExe
+if EXIST "%EXECUTABLE%" goto okExe
+echo elasticsearch-service-(x86|x64).exe was not found...
+
+:okExe
+set ES_VERSION=${project.version}
+
+if "%LOG_DIR%" == "" set LOG_DIR=%ES_HOME%\logs
+
+if "x%1x" == "xx" goto displayUsage
+set SERVICE_CMD=%1
+shift
+if "x%1x" == "xx" goto checkServiceCmd
+set SERVICE_ID=%1
+
+:checkServiceCmd
+
+if "%LOG_OPTS%" == "" set LOG_OPTS=--LogPath "%LOG_DIR%" --LogPrefix "%SERVICE_ID%" --StdError auto --StdOutput auto
+
+if /i %SERVICE_CMD% == install goto doInstall
+if /i %SERVICE_CMD% == remove goto doRemove
+if /i %SERVICE_CMD% == start goto doStart
+if /i %SERVICE_CMD% == stop goto doStop
+if /i %SERVICE_CMD% == manager goto doManagment
+echo Unknown option "%SERVICE_CMD%"
+
+:displayUsage
+echo.
+echo Usage: elasticsearch-service.bat install^|remove^|start^|stop^|manager [SERVICE_ID]
+goto:eof
+
+:doStart
+"%EXECUTABLE%" //ES//%SERVICE_ID% %LOG_OPTS%
+if not errorlevel 1 goto started
+echo Failed starting '%SERVICE_ID%' service
+goto:eof
+:started
+echo The service '%SERVICE_ID%' has been started
+goto:eof
+
+:doStop
+"%EXECUTABLE%" //SS//%SERVICE_ID% %LOG_OPTS%
+if not errorlevel 1 goto stopped
+echo Failed stopping '%SERVICE_ID%' service
+goto:eof
+:stopped
+echo The service '%SERVICE_ID%' has been stopped
+goto:eof
+
+:doManagment
+set EXECUTABLE_MGR=%ES_HOME%\bin\elasticsearch-service-mgr.exe
+"%EXECUTABLE_MGR%" //ES//%SERVICE_ID%
+if not errorlevel 1 goto managed
+echo Failed starting service manager for '%SERVICE_ID%'
+goto:eof
+:managed
+echo Successfully started service manager for '%SERVICE_ID%'.
+goto:eof
+
+:doRemove
+rem Remove the service
+"%EXECUTABLE%" //DS//%SERVICE_ID% %LOG_OPTS%
+if not errorlevel 1 goto removed
+echo Failed removing '%SERVICE_ID%' service
+goto:eof
+:removed
+echo The service '%SERVICE_ID%' has been removed
+goto:eof
+
+:doInstall
+echo Installing service      :  "%SERVICE_ID%"
+echo Using JAVA_HOME (%ARCH%):  "%JAVA_HOME%"
+
+rem Check JVM server dll first
+if exist "%JAVA_HOME%\jre\bin\server\jvm.dll" (
+	set JVM_DLL=\jre\bin\server\jvm.dll
+	goto foundJVM
+)
+
+rem Check 'server' JRE (JRE installed on Windows Server)
+if exist "%JAVA_HOME%\bin\server\jvm.dll" (
+	set JVM_DLL=\bin\server\jvm.dll
+	goto foundJVM
+)
+
+rem Fallback to 'client' JRE
+if exist "%JAVA_HOME%\bin\client\jvm.dll" (
+	set JVM_DLL=\bin\client\jvm.dll
+	echo Warning: JAVA_HOME points to a JRE and not JDK installation; a client (not a server^) JVM will be used...
+) else (
+	echo JAVA_HOME points to an invalid Java installation (no jvm.dll found in "%JAVA_HOME%"^). Exiting...
+	goto:eof
+)
+
+:foundJVM
+if "%ES_JVM_OPTIONS%" == "" (
+set ES_JVM_OPTIONS=%ES_HOME%\config\jvm.options
+)
+
+if not "%ES_JAVA_OPTS%" == "" set ES_JAVA_OPTS=%ES_JAVA_OPTS: =;%
+
+@setlocal
+for /F "usebackq delims=" %%a in (`findstr /b \- "%ES_JVM_OPTIONS%" ^| findstr /b /v "\-server \-client"`) do set JVM_OPTIONS=!JVM_OPTIONS!%%a;
+@endlocal & set ES_JAVA_OPTS=%JVM_OPTIONS%%ES_JAVA_OPTS%
+
+if "%ES_JAVA_OPTS:~-1%"==";" set ES_JAVA_OPTS=%ES_JAVA_OPTS:~0,-1%
+
+@setlocal EnableDelayedExpansion
+for %%a in ("%ES_JAVA_OPTS:;=","%") do (
+  set var=%%a
+  if "!var:~1,4!" == "-Xms" (
+    if not "!JVM_MS!" == "" (
+      echo duplicate min heap size settings found
+      goto:eof
+    )
+    set XMS=!var:~5,-1!
+    call:convertxm !XMS! JVM_MS
+  )
+  if "!var:~1,16!" == "-XX:MinHeapSize=" (
+    if not "!JVM_MS!" == "" (
+      echo duplicate min heap size settings found
+      goto:eof
+    )
+    set XMS=!var:~17,-1!
+    call:convertxm !XMS! JVM_MS
+  )
+  if "!var:~1,4!" == "-Xmx" (
+    if not "!JVM_MX!" == "" (
+      echo duplicate max heap size settings found
+      goto:eof
+    )
+    set XMX=!var:~5,-1!
+    call:convertxm !XMX! JVM_MX
+  )
+  if "!var:~1,16!" == "-XX:MaxHeapSize=" (
+    if not "!JVM_MX!" == "" (
+      echo duplicate max heap size settings found
+      goto:eof
+    )
+    set XMX=!var:~17,-1!
+    call:convertxm !XMX! JVM_MX
+  )
+  if "!var:~1,4!" == "-Xss" (
+    if not "!JVM_SS!" == "" (
+      echo duplicate thread stack size settings found
+      exit 1
+    )
+    set XSS=!var:~5,-1!
+    call:convertxk !XSS! JVM_SS
+  )
+  if "!var:~1,20!" == "-XX:ThreadStackSize=" (
+    if not "!JVM_SS!" == "" (
+      echo duplicate thread stack size settings found
+      goto:eof
+    )
+    set XSS=!var:~21,-1!
+    call:convertxk !XSS! JVM_SS
+  )
+)
+@endlocal & set JVM_MS=%JVM_MS% & set JVM_MX=%JVM_MX% & set JVM_SS=%JVM_SS%
+
+if "%JVM_MS%" == "" (
+  echo minimum heap size not set; configure using -Xms via "%ES_JVM_OPTIONS%" or ES_JAVA_OPTS
+  goto:eof
+)
+if "%JVM_MX%" == "" (
+  echo maximum heap size not set; configure using -Xmx via "%ES_JVM_OPTIONS%" or ES_JAVA_OPTS
+  goto:eof
+)
+if "%JVM_SS%" == "" (
+  echo thread stack size not set; configure using -Xss via "%ES_JVM_OPTIONS%" or ES_JAVA_OPTS
+  goto:eof
+)
+
+CALL "%ES_HOME%\bin\elasticsearch.in.bat"
+if "%DATA_DIR%" == "" set DATA_DIR=%ES_HOME%\data
+
+if "%CONF_DIR%" == "" set CONF_DIR=%ES_HOME%\config
+
+set ES_PARAMS=-Delasticsearch;-Des.path.home="%ES_HOME%";-Des.default.path.logs="%LOG_DIR%";-Des.default.path.data="%DATA_DIR%";-Des.default.path.conf="%CONF_DIR%"
+
+if "%ES_START_TYPE%" == "" set ES_START_TYPE=manual
+if "%ES_STOP_TIMEOUT%" == "" set ES_STOP_TIMEOUT=0
+
+if "%SERVICE_DISPLAY_NAME%" == "" set SERVICE_DISPLAY_NAME=Elasticsearch %ES_VERSION% (%SERVICE_ID%)
+if "%SERVICE_DESCRIPTION%" == "" set SERVICE_DESCRIPTION=Elasticsearch %ES_VERSION% Windows Service - https://elastic.co
+
+if not "%SERVICE_USERNAME%" == "" (
+	if not "%SERVICE_PASSWORD%" == "" (
+		set SERVICE_PARAMS=%SERVICE_PARAMS% --ServiceUser "%SERVICE_USERNAME%" --ServicePassword "%SERVICE_PASSWORD%"
+	)
+)
+
+"%EXECUTABLE%" //IS//%SERVICE_ID% --Startup %ES_START_TYPE% --StopTimeout %ES_STOP_TIMEOUT% --StartClass org.elasticsearch.bootstrap.Elasticsearch --StopClass org.elasticsearch.bootstrap.Elasticsearch --StartMethod main --StopMethod close --Classpath "%ES_CLASSPATH%" --JvmMs %JVM_MS% --JvmMx %JVM_MX% --JvmSs %JVM_SS% --JvmOptions %ES_JAVA_OPTS% ++JvmOptions %ES_PARAMS% %LOG_OPTS% --PidFile "%SERVICE_ID%.pid" --DisplayName "%SERVICE_DISPLAY_NAME%" --Description "%SERVICE_DESCRIPTION%" --Jvm "%%JAVA_HOME%%%JVM_DLL%" --StartMode jvm --StopMode jvm --StartPath "%ES_HOME%" %SERVICE_PARAMS%
+
+if not errorlevel 1 goto installed
+echo Failed installing '%SERVICE_ID%' service
+goto:eof
+
+:installed
+echo The service '%SERVICE_ID%' has been installed.
+goto:eof
+
+:err
+echo JAVA_HOME environment variable must be set!
+pause
+goto:eof
+
+rem ---
+rem Function for converting Xm[s|x] values into MB which Commons Daemon accepts
+rem ---
+:convertxm
+set value=%~1
+rem extract last char (unit)
+set unit=%value:~-1%
+rem assume the unit is specified
+set conv=%value:~0,-1%
+
+if "%unit%" == "k" goto kilo
+if "%unit%" == "K" goto kilo
+if "%unit%" == "m" goto mega
+if "%unit%" == "M" goto mega
+if "%unit%" == "g" goto giga
+if "%unit%" == "G" goto giga
+
+rem no unit found, must be bytes; consider the whole value
+set conv=%value%
+rem convert to KB
+set /a conv=%conv% / 1024
+:kilo
+rem convert to MB
+set /a conv=%conv% / 1024
+goto mega
+:giga
+rem convert to MB
+set /a conv=%conv% * 1024
+:mega
+set "%~2=%conv%"
+goto:eof
+
+:convertxk
+set value=%~1
+rem extract last char (unit)
+set unit=%value:~-1%
+rem assume the unit is specified
+set conv=%value:~0,-1%
+
+if "%unit%" == "k" goto kilo
+if "%unit%" == "K" goto kilo
+if "%unit%" == "m" goto mega
+if "%unit%" == "M" goto mega
+if "%unit%" == "g" goto giga
+if "%unit%" == "G" goto giga
+
+rem no unit found, must be bytes; consider the whole value
+set conv=%value%
+rem convert to KB
+set /a conv=%conv% / 1024
+goto kilo
+:mega
+rem convert to KB
+set /a conv=%conv% * 1024
+goto kilo
+:giga
+rem convert to KB
+set /a conv=%conv% * 1024 * 1024
+:kilo
+set "%~2=%conv%"
+goto:eof
+
+:conffileset
+echo CONF_FILE setting is no longer supported. elasticsearch.yml must be placed in the config directory and cannot be renamed.
+goto:eof
+
+ENDLOCAL
diff --git a/distribution/src/main/resources/bin/elasticsearch.bat b/distribution/src/main/resources/bin/elasticsearch.bat
index 416bb37c85eef..d98de740d2923 100644
--- a/distribution/src/main/resources/bin/elasticsearch.bat
+++ b/distribution/src/main/resources/bin/elasticsearch.bat
@@ -62,7 +62,7 @@ SET HOSTNAME=%COMPUTERNAME%
 
 if "%ES_JVM_OPTIONS%" == "" (
 rem '0' is the batch file, '~dp' appends the drive and path
-set ES_JVM_OPTIONS=%~dp0\..\config\jvm.options
+set "ES_JVM_OPTIONS=%~dp0\..\config\jvm.options"
 )
 
 @setlocal
@@ -79,6 +79,6 @@ IF ERRORLEVEL 1 (
 	EXIT /B %ERRORLEVEL%
 )
 
-"%JAVA%" %ES_JAVA_OPTS% %ES_PARAMS% -cp "%ES_CLASSPATH%" "org.elasticsearch.bootstrap.Elasticsearch" !newparams!
+%JAVA% %ES_JAVA_OPTS% %ES_PARAMS% -cp "%ES_CLASSPATH%" "org.elasticsearch.bootstrap.Elasticsearch" !newparams!
 
 ENDLOCAL
diff --git a/distribution/src/main/resources/bin/elasticsearch.in.bat b/distribution/src/main/resources/bin/elasticsearch.in.bat
index 2d73ed2b57b75..d7b985d082459 100644
--- a/distribution/src/main/resources/bin/elasticsearch.in.bat
+++ b/distribution/src/main/resources/bin/elasticsearch.in.bat
@@ -1,11 +1,11 @@
 @echo off
 
 IF DEFINED JAVA_HOME (
-  set JAVA=%JAVA_HOME%\bin\java.exe
+  set JAVA="%JAVA_HOME%\bin\java.exe"
 ) ELSE (
-  FOR %%I IN (java.exe) DO set JAVA=%%~$PATH:I
+  FOR %%I IN (java.exe) DO set JAVA="%%~$PATH:I"
 )
-IF NOT EXIST "%JAVA%" (
+IF NOT EXIST %JAVA% (
   ECHO Could not find any executable java binary. Please install java in your PATH or set JAVA_HOME 1>&2
   EXIT /B 1
 )
@@ -15,11 +15,19 @@ for %%I in ("%SCRIPT_DIR%..") do set ES_HOME=%%~dpfI
 
 REM check in case a user was using this mechanism
 if "%ES_CLASSPATH%" == "" (
-set ES_CLASSPATH=!ES_HOME!/lib/elasticsearch-${project.version}.jar;!ES_HOME!/lib/*
+set ES_CLASSPATH=!ES_HOME!/lib/*
 ) else (
 ECHO Error: Don't modify the classpath with ES_CLASSPATH, Best is to add 1>&2
 ECHO additional elements via the plugin mechanism, or if code must really be 1>&2
 ECHO added to the main classpath, add jars to lib\, unsupported 1>&2
 EXIT /B 1
 )
+
+%JAVA% -cp "%ES_CLASSPATH%" "org.elasticsearch.tools.JavaVersionChecker"
+
+IF ERRORLEVEL 1 (
+    ECHO Elasticsearch requires at least Java 8 but your Java version from %JAVA% does not meet this requirement
+    EXIT /B 1
+)
+
 set ES_PARAMS=-Delasticsearch -Des.path.home="%ES_HOME%"
diff --git a/distribution/src/main/resources/bin/elasticsearch.in.sh b/distribution/src/main/resources/bin/elasticsearch.in.sh
index 58b26a2d6ebc7..2d22439225697 100644
--- a/distribution/src/main/resources/bin/elasticsearch.in.sh
+++ b/distribution/src/main/resources/bin/elasticsearch.in.sh
@@ -10,4 +10,4 @@ EOF
     exit 1
 fi
 
-ES_CLASSPATH="$ES_HOME/lib/elasticsearch-${project.version}.jar:$ES_HOME/lib/*"
+ES_CLASSPATH="$ES_HOME/lib/*"
diff --git a/distribution/src/main/resources/bin/service.bat b/distribution/src/main/resources/bin/service.bat
deleted file mode 100644
index ccb2d2b7efcc3..0000000000000
--- a/distribution/src/main/resources/bin/service.bat
+++ /dev/null
@@ -1,349 +0,0 @@
-@echo off
-SETLOCAL enabledelayedexpansion
-
-TITLE Elasticsearch Service ${project.version}
-
-rem TODO: remove for Elasticsearch 6.x
-set bad_env_var=0
-if not "%ES_MIN_MEM%" == "" set bad_env_var=1
-if not "%ES_MAX_MEM%" == "" set bad_env_var=1
-if not "%ES_HEAP_SIZE%" == "" set bad_env_var=1
-if not "%ES_HEAP_NEWSIZE%" == "" set bad_env_var=1
-if not "%ES_DIRECT_SIZE%" == "" set bad_env_var=1
-if not "%ES_USE_IPV4%" == "" set bad_env_var=1
-if not "%ES_GC_OPTS%" == "" set bad_env_var=1
-if not "%ES_GC_LOG_FILE%" == "" set bad_env_var=1
-if %bad_env_var% == 1 (
-    echo Error: encountered environment variables that are no longer supported
-    echo Use jvm.options or ES_JAVA_OPTS to configure the JVM
-    if not "%ES_MIN_MEM%" == "" echo ES_MIN_MEM=%ES_MIN_MEM%: set -Xms%ES_MIN_MEM% in jvm.options or add "-Xms%ES_MIN_MEM%" to ES_JAVA_OPTS
-    if not "%ES_MAX_MEM%" == "" echo ES_MAX_MEM=%ES_MAX_MEM%: set -Xms%ES_MAX_MEM% in jvm.options or add "-Xmx%ES_MAX_MEM%" to ES_JAVA_OPTS
-    if not "%ES_HEAP_SIZE%" == "" echo ES_HEAP_SIZE=%ES_HEAP_SIZE%: set -Xms%ES_HEAP_SIZE% and -Xmx%ES_HEAP_SIZE% in jvm.options or add "-Xms%ES_HEAP_SIZE% -Xmx%ES_HEAP_SIZE%" to ES_JAVA_OPTS
-    if not "%ES_HEAP_NEWSIZE%" == "" echo ES_HEAP_NEWSIZE=%ES_HEAP_NEWSIZE%: set -Xmn%ES_HEAP_NEWSIZE% in jvm.options or add "-Xmn%ES_HEAP_SIZE%" to ES_JAVA_OPTS
-    if not "%ES_DIRECT_SIZE%" == "" echo ES_DIRECT_SIZE=%ES_DIRECT_SIZE%: set -XX:MaxDirectMemorySize=%ES_DIRECT_SIZE% in jvm.options or add "-XX:MaxDirectMemorySize=%ES_DIRECT_SIZE%" to ES_JAVA_OPTS
-    if not "%ES_USE_IPV4%" == "" echo ES_USE_IPV4=%ES_USE_IPV4%: set -Djava.net.preferIPv4Stack=true in jvm.options or add "-Djava.net.preferIPv4Stack=true" to ES_JAVA_OPTS
-    if not "%ES_GC_OPTS%" == "" echo ES_GC_OPTS=%ES_GC_OPTS%: set %ES_GC_OPTS: = and % in jvm.options or add "%ES_GC_OPTS%" to ES_JAVA_OPTS
-    if not "%ES_GC_LOG_FILE%" == "" echo ES_GC_LOG_FILE=%ES_GC_LOG_FILE%: set -Xloggc:%ES_GC_LOG_FILE% in jvm.options or add "-Xloggc:%ES_GC_LOG_FILE%" to ES_JAVA_OPTS"
-    exit /b 1
-)
-rem end TODO: remove for Elasticsearch 6.x
-
-IF DEFINED JAVA_HOME (
-  SET JAVA=%JAVA_HOME%\bin\java.exe
-) ELSE (
-  FOR %%I IN (java.exe) DO set JAVA=%%~$PATH:I
-)
-IF NOT EXIST "%JAVA%" (
-  ECHO Could not find any executable java binary. Please install java in your PATH or set JAVA_HOME 1>&2
-  EXIT /B 1
-)
-IF DEFINED JAVA_HOME GOTO :cont
-
-IF NOT "%JAVA:~-13%" == "\bin\java.exe" (
-  FOR /f "tokens=2 delims=[]" %%I IN ('dir %JAVA%') DO @set JAVA=%%I
-)
-IF "%JAVA:~-13%" == "\bin\java.exe" (
-  SET JAVA_HOME=%JAVA:~0,-13%
-)
-
-:cont
-if not "%CONF_FILE%" == "" goto conffileset
-
-set SCRIPT_DIR=%~dp0
-for %%I in ("%SCRIPT_DIR%..") do set ES_HOME=%%~dpfI
-
-"%JAVA%" -Xmx50M -version > nul 2>&1
-
-if errorlevel 1 (
-	echo Warning: Could not start JVM to detect version, defaulting to x86:
-	goto x86
-)
-
-"%JAVA%" -Xmx50M -version 2>&1 | "%windir%\System32\find" "64-Bit" >nul:
-
-if errorlevel 1 goto x86
-set EXECUTABLE=%ES_HOME%\bin\elasticsearch-service-x64.exe
-set SERVICE_ID=elasticsearch-service-x64
-set ARCH=64-bit
-goto checkExe
-
-:x86
-set EXECUTABLE=%ES_HOME%\bin\elasticsearch-service-x86.exe
-set SERVICE_ID=elasticsearch-service-x86
-set ARCH=32-bit
-
-:checkExe
-if EXIST "%EXECUTABLE%" goto okExe
-echo elasticsearch-service-(x86|x64).exe was not found...
-
-:okExe
-set ES_VERSION=${project.version}
-
-if "%LOG_DIR%" == "" set LOG_DIR=%ES_HOME%\logs
-
-if "x%1x" == "xx" goto displayUsage
-set SERVICE_CMD=%1
-shift
-if "x%1x" == "xx" goto checkServiceCmd
-set SERVICE_ID=%1
-
-:checkServiceCmd
-
-if "%LOG_OPTS%" == "" set LOG_OPTS=--LogPath "%LOG_DIR%" --LogPrefix "%SERVICE_ID%" --StdError auto --StdOutput auto
-
-if /i %SERVICE_CMD% == install goto doInstall
-if /i %SERVICE_CMD% == remove goto doRemove
-if /i %SERVICE_CMD% == start goto doStart
-if /i %SERVICE_CMD% == stop goto doStop
-if /i %SERVICE_CMD% == manager goto doManagment
-echo Unknown option "%SERVICE_CMD%"
-
-:displayUsage
-echo.
-echo Usage: service.bat install^|remove^|start^|stop^|manager [SERVICE_ID]
-goto:eof
-
-:doStart
-"%EXECUTABLE%" //ES//%SERVICE_ID% %LOG_OPTS%
-if not errorlevel 1 goto started
-echo Failed starting '%SERVICE_ID%' service
-goto:eof
-:started
-echo The service '%SERVICE_ID%' has been started
-goto:eof
-
-:doStop
-"%EXECUTABLE%" //SS//%SERVICE_ID% %LOG_OPTS%
-if not errorlevel 1 goto stopped
-echo Failed stopping '%SERVICE_ID%' service
-goto:eof
-:stopped
-echo The service '%SERVICE_ID%' has been stopped
-goto:eof
-
-:doManagment
-set EXECUTABLE_MGR=%ES_HOME%\bin\elasticsearch-service-mgr.exe
-"%EXECUTABLE_MGR%" //ES//%SERVICE_ID%
-if not errorlevel 1 goto managed
-echo Failed starting service manager for '%SERVICE_ID%'
-goto:eof
-:managed
-echo Successfully started service manager for '%SERVICE_ID%'.
-goto:eof
-
-:doRemove
-rem Remove the service
-"%EXECUTABLE%" //DS//%SERVICE_ID% %LOG_OPTS%
-if not errorlevel 1 goto removed
-echo Failed removing '%SERVICE_ID%' service
-goto:eof
-:removed
-echo The service '%SERVICE_ID%' has been removed
-goto:eof
-
-:doInstall
-echo Installing service      :  "%SERVICE_ID%"
-echo Using JAVA_HOME (%ARCH%):  "%JAVA_HOME%"
-
-rem Check JVM server dll first
-if exist "%JAVA_HOME%"\jre\bin\server\jvm.dll (
-	set JVM_DLL=\jre\bin\server\jvm.dll
-	goto foundJVM
-)
-
-rem Check 'server' JRE (JRE installed on Windows Server)
-if exist "%JAVA_HOME%"\bin\server\jvm.dll (
-	set JVM_DLL=\bin\server\jvm.dll
-	goto foundJVM
-)
-
-rem Fallback to 'client' JRE
-if exist "%JAVA_HOME%"\bin\client\jvm.dll (
-	set JVM_DLL=\bin\client\jvm.dll
-	echo Warning: JAVA_HOME points to a JRE and not JDK installation; a client (not a server^) JVM will be used...
-) else (
-	echo JAVA_HOME points to an invalid Java installation (no jvm.dll found in "%JAVA_HOME%"^). Exiting...
-	goto:eof
-)
-
-:foundJVM
-if "%ES_JVM_OPTIONS%" == "" (
-set ES_JVM_OPTIONS=%ES_HOME%\config\jvm.options
-)
-
-if not "%ES_JAVA_OPTS%" == "" set ES_JAVA_OPTS=%ES_JAVA_OPTS: =;%
-
-@setlocal
-for /F "usebackq delims=" %%a in (`findstr /b \- "%ES_JVM_OPTIONS%" ^| findstr /b /v "\-server \-client"`) do set JVM_OPTIONS=!JVM_OPTIONS!%%a;
-@endlocal & set ES_JAVA_OPTS=%JVM_OPTIONS%%ES_JAVA_OPTS%
-
-if "%ES_JAVA_OPTS:~-1%"==";" set ES_JAVA_OPTS=%ES_JAVA_OPTS:~0,-1%
-
-@setlocal EnableDelayedExpansion
-for %%a in ("%ES_JAVA_OPTS:;=","%") do (
-  set var=%%a
-  if "!var:~1,4!" == "-Xms" (
-    if not "!JVM_MS!" == "" (
-      echo duplicate min heap size settings found
-      goto:eof
-    )
-    set XMS=!var:~5,-1!
-    call:convertxm !XMS! JVM_MS
-  )
-  if "!var:~1,16!" == "-XX:MinHeapSize=" (
-    if not "!JVM_MS!" == "" (
-      echo duplicate min heap size settings found
-      goto:eof
-    )
-    set XMS=!var:~17,-1!
-    call:convertxm !XMS! JVM_MS
-  )
-  if "!var:~1,4!" == "-Xmx" (
-    if not "!JVM_MX!" == "" (
-      echo duplicate max heap size settings found
-      goto:eof
-    )
-    set XMX=!var:~5,-1!
-    call:convertxm !XMX! JVM_MX
-  )
-  if "!var:~1,16!" == "-XX:MaxHeapSize=" (
-    if not "!JVM_MX!" == "" (
-      echo duplicate max heap size settings found
-      goto:eof
-    )
-    set XMX=!var:~17,-1!
-    call:convertxm !XMX! JVM_MX
-  )
-  if "!var:~1,4!" == "-Xss" (
-    if not "!JVM_SS!" == "" (
-      echo duplicate thread stack size settings found
-      exit 1
-    )
-    set XSS=!var:~5,-1!
-    call:convertxk !XSS! JVM_SS
-  )
-  if "!var:~1,20!" == "-XX:ThreadStackSize=" (
-    if not "!JVM_SS!" == "" (
-      echo duplicate thread stack size settings found
-      goto:eof
-    )
-    set XSS=!var:~21,-1!
-    call:convertxk !XSS! JVM_SS
-  )
-)
-@endlocal & set JVM_MS=%JVM_MS% & set JVM_MX=%JVM_MX% & set JVM_SS=%JVM_SS%
-
-if "%JVM_MS%" == "" (
-  echo minimum heap size not set; configure via %ES_JVM_OPTIONS% or ES_JAVA_OPTS
-  goto:eof
-)
-if "%JVM_MX%" == "" (
-  echo maximum heap size not set; configure via %ES_JVM_OPTIONS% or ES_JAVA_OPTS
-  goto:eof
-)
-if "%JVM_SS%" == "" (
-  echo thread stack size not set; configure via %ES_JVM_OPTIONS% or ES_JAVA_OPTS
-  goto:eof
-)
-
-CALL "%ES_HOME%\bin\elasticsearch.in.bat"
-if "%DATA_DIR%" == "" set DATA_DIR=%ES_HOME%\data
-
-if "%CONF_DIR%" == "" set CONF_DIR=%ES_HOME%\config
-
-set ES_PARAMS=-Delasticsearch;-Des.path.home="%ES_HOME%";-Des.default.path.logs="%LOG_DIR%";-Des.default.path.data="%DATA_DIR%";-Des.default.path.conf="%CONF_DIR%"
-
-if "%ES_START_TYPE%" == "" set ES_START_TYPE=manual
-if "%ES_STOP_TIMEOUT%" == "" set ES_STOP_TIMEOUT=0
-
-if "%SERVICE_DISPLAY_NAME%" == "" set SERVICE_DISPLAY_NAME=Elasticsearch %ES_VERSION% (%SERVICE_ID%)
-if "%SERVICE_DESCRIPTION%" == "" set SERVICE_DESCRIPTION=Elasticsearch %ES_VERSION% Windows Service - https://elastic.co
-
-if not "%SERVICE_USERNAME%" == "" (
-	if not "%SERVICE_PASSWORD%" == "" (
-		set SERVICE_PARAMS=%SERVICE_PARAMS% --ServiceUser "%SERVICE_USERNAME%" --ServicePassword "%SERVICE_PASSWORD%"
-	)
-)
-
-"%EXECUTABLE%" //IS//%SERVICE_ID% --Startup %ES_START_TYPE% --StopTimeout %ES_STOP_TIMEOUT% --StartClass org.elasticsearch.bootstrap.Elasticsearch --StopClass org.elasticsearch.bootstrap.Elasticsearch --StartMethod main --StopMethod close --Classpath "%ES_CLASSPATH%" --JvmMs %JVM_MS% --JvmMx %JVM_MX% --JvmSs %JVM_SS% --JvmOptions %ES_JAVA_OPTS% ++JvmOptions %ES_PARAMS% %LOG_OPTS% --PidFile "%SERVICE_ID%.pid" --DisplayName "%SERVICE_DISPLAY_NAME%" --Description "%SERVICE_DESCRIPTION%" --Jvm "%%JAVA_HOME%%%JVM_DLL%" --StartMode jvm --StopMode jvm --StartPath "%ES_HOME%" %SERVICE_PARAMS%
-
-if not errorlevel 1 goto installed
-echo Failed installing '%SERVICE_ID%' service
-goto:eof
-
-:installed
-echo The service '%SERVICE_ID%' has been installed.
-goto:eof
-
-:err
-echo JAVA_HOME environment variable must be set!
-pause
-goto:eof
-
-rem ---
-rem Function for converting Xm[s|x] values into MB which Commons Daemon accepts
-rem ---
-:convertxm
-set value=%~1
-rem extract last char (unit)
-set unit=%value:~-1%
-rem assume the unit is specified
-set conv=%value:~0,-1%
-
-if "%unit%" == "k" goto kilo
-if "%unit%" == "K" goto kilo
-if "%unit%" == "m" goto mega
-if "%unit%" == "M" goto mega
-if "%unit%" == "g" goto giga
-if "%unit%" == "G" goto giga
-
-rem no unit found, must be bytes; consider the whole value
-set conv=%value%
-rem convert to KB
-set /a conv=%conv% / 1024
-:kilo
-rem convert to MB
-set /a conv=%conv% / 1024
-goto mega
-:giga
-rem convert to MB
-set /a conv=%conv% * 1024
-:mega
-set "%~2=%conv%"
-goto:eof
-
-:convertxk
-set value=%~1
-rem extract last char (unit)
-set unit=%value:~-1%
-rem assume the unit is specified
-set conv=%value:~0,-1%
-
-if "%unit%" == "k" goto kilo
-if "%unit%" == "K" goto kilo
-if "%unit%" == "m" goto mega
-if "%unit%" == "M" goto mega
-if "%unit%" == "g" goto giga
-if "%unit%" == "G" goto giga
-
-rem no unit found, must be bytes; consider the whole value
-set conv=%value%
-rem convert to KB
-set /a conv=%conv% / 1024
-goto kilo
-:mega
-rem convert to KB
-set /a conv=%conv% * 1024
-goto kilo
-:giga
-rem convert to KB
-set /a conv=%conv% * 1024 * 1024
-:kilo
-set "%~2=%conv%"
-goto:eof
-
-:conffileset
-echo CONF_FILE setting is no longer supported. elasticsearch.yml must be placed in the config directory and cannot be renamed.
-goto:eof
-
-ENDLOCAL
diff --git a/distribution/src/main/resources/config/elasticsearch.yml b/distribution/src/main/resources/config/elasticsearch.yml
index 0b99be87e0a70..15e841fe39061 100644
--- a/distribution/src/main/resources/config/elasticsearch.yml
+++ b/distribution/src/main/resources/config/elasticsearch.yml
@@ -7,8 +7,8 @@
 # The primary way of configuring a node is via this file. This template lists
 # the most important settings you may want to configure for a production cluster.
 #
-# Please see the documentation for further information on configuration options:
-# <http://www.elastic.co/guide/en/elasticsearch/reference/current/setup-configuration.html>
+# Please consult the documentation for further information on configuration options:
+# https://www.elastic.co/guide/en/elasticsearch/reference/index.html
 #
 # ---------------------------------- Cluster -----------------------------------
 #
@@ -24,7 +24,7 @@
 #
 # Add custom attributes to the node:
 #
-#node.rack: r1
+#node.attr.rack: r1
 #
 # ----------------------------------- Paths ------------------------------------
 #
@@ -58,8 +58,7 @@
 #
 #http.port: 9200
 #
-# For more information, see the documentation at:
-# <http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-network.html>
+# For more information, consult the network module documentation.
 #
 # --------------------------------- Discovery ----------------------------------
 #
@@ -68,12 +67,11 @@
 #
 #discovery.zen.ping.unicast.hosts: ["host1", "host2"]
 #
-# Prevent the "split brain" by configuring the majority of nodes (total number of nodes / 2 + 1):
+# Prevent the "split brain" by configuring the majority of nodes (total number of master-eligible nodes / 2 + 1):
 #
 #discovery.zen.minimum_master_nodes: 3
 #
-# For more information, see the documentation at:
-# <http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-discovery.html>
+# For more information, consult the zen discovery module documentation.
 #
 # ---------------------------------- Gateway -----------------------------------
 #
@@ -81,15 +79,10 @@
 #
 #gateway.recover_after_nodes: 3
 #
-# For more information, see the documentation at:
-# <http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-gateway.html>
+# For more information, consult the gateway module documentation.
 #
 # ---------------------------------- Various -----------------------------------
 #
-# Disable starting multiple nodes on a single system:
-#
-#node.max_local_storage_nodes: 1
-#
 # Require explicit names when deleting indices:
 #
 #action.destructive_requires_name: true
diff --git a/distribution/src/main/resources/config/jvm.options b/distribution/src/main/resources/config/jvm.options
index fc352b14668c9..a14f5ad33a076 100644
--- a/distribution/src/main/resources/config/jvm.options
+++ b/distribution/src/main/resources/config/jvm.options
@@ -39,17 +39,17 @@
 
 ## optimizations
 
-# disable calls to System#gc
--XX:+DisableExplicitGC
-
 # pre-touch memory pages used by the JVM during initialization
 -XX:+AlwaysPreTouch
 
 ## basic
 
-# force the server VM
+# force the server VM (remove on 32-bit client JVMs)
 -server
 
+# explicitly set the stack size (reduce to 320k on 32-bit client JVMs)
+-Xss1m
+
 # set to headless, just in case
 -Djava.awt.headless=true
 
@@ -59,9 +59,13 @@
 # use our provided JNA always versus the system one
 -Djna.nosys=true
 
-# flags to keep Netty from being unsafe
+# use old-style file permissions on JDK9
+-Djdk.io.permissionsUseCanonicalPath=true
+
+# flags to configure Netty
 -Dio.netty.noUnsafe=true
 -Dio.netty.noKeySetOptimization=true
+-Dio.netty.recycler.maxCapacityPerThread=0
 
 # log4j 2
 -Dlog4j.shutdownHookEnabled=false
@@ -91,6 +95,13 @@
 # ensure the directory exists
 #-Xloggc:${loggc}
 
+# By default, the GC log file will not rotate.
+# By uncommenting the lines below, the GC log file
+# will be rotated every 128MB at most 32 times.
+#-XX:+UseGCLogFileRotation
+#-XX:NumberOfGCLogFiles=32
+#-XX:GCLogFileSize=128M
+
 # Elasticsearch 5.0.0 will throw an exception on unquoted field names in JSON.
 # If documents were already indexed with unquoted fields in a previous version
 # of Elasticsearch, some operations may throw errors.
diff --git a/distribution/src/main/resources/config/log4j2.properties b/distribution/src/main/resources/config/log4j2.properties
index 2cfe038cc847d..f344d0aee5541 100644
--- a/distribution/src/main/resources/config/log4j2.properties
+++ b/distribution/src/main/resources/config/log4j2.properties
@@ -1,20 +1,20 @@
 status = error
 
 # log action execution errors for easier debugging
-logger.action.name = action
+logger.action.name = org.elasticsearch.action
 logger.action.level = debug
 
 appender.console.type = Console
 appender.console.name = console
 appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %m%n
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n
 
 appender.rolling.type = RollingFile
 appender.rolling.name = rolling
-appender.rolling.fileName = ${sys:es.logs}.log
+appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log
 appender.rolling.layout.type = PatternLayout
-appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %.10000m%n
-appender.rolling.filePattern = ${sys:es.logs}-%d{yyyy-MM-dd}.log
+appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n
+appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}.log
 appender.rolling.policies.type = Policies
 appender.rolling.policies.time.type = TimeBasedTriggeringPolicy
 appender.rolling.policies.time.interval = 1
@@ -26,27 +26,27 @@ rootLogger.appenderRef.rolling.ref = rolling
 
 appender.deprecation_rolling.type = RollingFile
 appender.deprecation_rolling.name = deprecation_rolling
-appender.deprecation_rolling.fileName = ${sys:es.logs}_deprecation.log
+appender.deprecation_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.log
 appender.deprecation_rolling.layout.type = PatternLayout
-appender.deprecation_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %.10000m%n
-appender.deprecation_rolling.filePattern = ${sys:es.logs}_deprecation-%i.log.gz
+appender.deprecation_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n
+appender.deprecation_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation-%i.log.gz
 appender.deprecation_rolling.policies.type = Policies
 appender.deprecation_rolling.policies.size.type = SizeBasedTriggeringPolicy
 appender.deprecation_rolling.policies.size.size = 1GB
 appender.deprecation_rolling.strategy.type = DefaultRolloverStrategy
 appender.deprecation_rolling.strategy.max = 4
 
-logger.deprecation.name = deprecation
+logger.deprecation.name = org.elasticsearch.deprecation
 logger.deprecation.level = warn
 logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling
 logger.deprecation.additivity = false
 
 appender.index_search_slowlog_rolling.type = RollingFile
 appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling
-appender.index_search_slowlog_rolling.fileName = ${sys:es.logs}_index_search_slowlog.log
+appender.index_search_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog.log
 appender.index_search_slowlog_rolling.layout.type = PatternLayout
-appender.index_search_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %.10000m%n
-appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs}_index_search_slowlog-%d{yyyy-MM-dd}.log
+appender.index_search_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n
+appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog-%d{yyyy-MM-dd}.log
 appender.index_search_slowlog_rolling.policies.type = Policies
 appender.index_search_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy
 appender.index_search_slowlog_rolling.policies.time.interval = 1
@@ -59,10 +59,10 @@ logger.index_search_slowlog_rolling.additivity = false
 
 appender.index_indexing_slowlog_rolling.type = RollingFile
 appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling
-appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs}_index_indexing_slowlog.log
+appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog.log
 appender.index_indexing_slowlog_rolling.layout.type = PatternLayout
-appender.index_indexing_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %.10000m%n
-appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs}_index_indexing_slowlog-%d{yyyy-MM-dd}.log
+appender.index_indexing_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n
+appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog-%d{yyyy-MM-dd}.log
 appender.index_indexing_slowlog_rolling.policies.type = Policies
 appender.index_indexing_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy
 appender.index_indexing_slowlog_rolling.policies.time.interval = 1
diff --git a/distribution/tar/build.gradle b/distribution/tar/build.gradle
index 724256590568a..2140061ee4803 100644
--- a/distribution/tar/build.gradle
+++ b/distribution/tar/build.gradle
@@ -18,6 +18,7 @@
  */
 
 task buildTar(type: Tar) {
+  dependsOn createPluginsDir
   baseName = 'elasticsearch'
   extension = 'tar.gz'
   with archivesFiles
@@ -28,9 +29,5 @@ task buildTar(type: Tar) {
 
 artifacts {
   'default' buildTar
-  project.afterEvaluate {
-    // gradle is broken for extensions that contain a dot, so we must be explicit about the name of the .asc file
-    project.signArchives.singleSignature.type = 'tar.gz.asc'
-  }
 }
 
diff --git a/distribution/tar/src/test/java/org/elasticsearch/test/rest/TarClientYamlTestSuiteIT.java b/distribution/tar/src/test/java/org/elasticsearch/test/rest/TarClientYamlTestSuiteIT.java
index 73d323f7d507c..391d6fe688fd5 100644
--- a/distribution/tar/src/test/java/org/elasticsearch/test/rest/TarClientYamlTestSuiteIT.java
+++ b/distribution/tar/src/test/java/org/elasticsearch/test/rest/TarClientYamlTestSuiteIT.java
@@ -23,9 +23,6 @@
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-
-import java.io.IOException;
 
 /** Rest integration test. Runs against a cluster started by {@code gradle integTest} */
 public class TarClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
@@ -34,7 +31,7 @@ public TarClientYamlTestSuiteIT(ClientYamlTestCandidate testCandidate) {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
-        return createParameters(0, 1);
+    public static Iterable<Object[]> parameters() throws Exception {
+        return createParameters();
     }
 }
diff --git a/distribution/tools/java-version-checker/build.gradle b/distribution/tools/java-version-checker/build.gradle
new file mode 100644
index 0000000000000..7b2a76037cc96
--- /dev/null
+++ b/distribution/tools/java-version-checker/build.gradle
@@ -0,0 +1,42 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import org.elasticsearch.gradle.precommit.PrecommitTasks
+import org.gradle.api.JavaVersion
+
+apply plugin: 'elasticsearch.build'
+apply plugin: 'ru.vyarus.animalsniffer'
+
+sourceCompatibility = JavaVersion.VERSION_1_6
+targetCompatibility = JavaVersion.VERSION_1_6
+
+dependencies {
+    signature "org.codehaus.mojo.signature:java16:1.0@signature"
+}
+
+forbiddenApisMain {
+    // java-version-checker does not depend on core so only JDK signatures should be checked
+    signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
+}
+
+javadoc.enabled = false
+test.enabled = false
+loggerUsageCheck.enabled = false
+jarHell.enabled=false
diff --git a/distribution/tools/java-version-checker/src/main/java/org/elasticsearch/tools/JavaVersionChecker.java b/distribution/tools/java-version-checker/src/main/java/org/elasticsearch/tools/JavaVersionChecker.java
new file mode 100644
index 0000000000000..f1c5828ff5f9c
--- /dev/null
+++ b/distribution/tools/java-version-checker/src/main/java/org/elasticsearch/tools/JavaVersionChecker.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.tools;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Simple program that checks if the runtime Java version is at least 1.8.
+ */
+final class JavaVersionChecker {
+
+    private JavaVersionChecker() {
+    }
+
+    private static final List<Integer> JAVA_8 = Arrays.asList(1, 8);
+
+    /**
+     * The main entry point. The exit code is 0 if the Java version is at least 1.8, otherwise the exit code is 1.
+     *
+     * @param args the args to the program which are rejected if not empty
+     */
+    public static void main(final String[] args) {
+        // no leniency!
+        if (args.length != 0) {
+            throw new IllegalArgumentException("expected zero arguments but was: " + Arrays.toString(args));
+        }
+        final String javaSpecificationVersion = System.getProperty("java.specification.version");
+        final List<Integer> current = parse(javaSpecificationVersion);
+        if (compare(current, JAVA_8) < 0) {
+            exit(1);
+        }
+        exit(0);
+    }
+
+    private static List<Integer> parse(final String value) {
+        if (!value.matches("^0*[0-9]+(\\.[0-9]+)*$")) {
+            throw new IllegalArgumentException(value);
+        }
+
+        final List<Integer> version = new ArrayList<Integer>();
+        final String[] components = value.split("\\.");
+        for (final String component : components) {
+            version.add(Integer.valueOf(component));
+        }
+        return version;
+    }
+
+    private static int compare(final List<Integer> left, final List<Integer> right) {
+        // lexicographically compare two lists, treating missing entries as zeros
+        final int len = Math.max(left.size(), right.size());
+        for (int i = 0; i < len; i++) {
+            final int l = (i < left.size()) ? left.get(i) : 0;
+            final int r = (i < right.size()) ? right.get(i) : 0;
+            if (l < r) {
+                return -1;
+            }
+            if (r < l) {
+                return 1;
+            }
+        }
+        return 0;
+    }
+
+    @SuppressForbidden(reason = "exit")
+    private static void exit(final int status) {
+        System.exit(status);
+    }
+
+}
diff --git a/distribution/tools/java-version-checker/src/main/java/org/elasticsearch/tools/SuppressForbidden.java b/distribution/tools/java-version-checker/src/main/java/org/elasticsearch/tools/SuppressForbidden.java
new file mode 100644
index 0000000000000..ce7b789b2ea7f
--- /dev/null
+++ b/distribution/tools/java-version-checker/src/main/java/org/elasticsearch/tools/SuppressForbidden.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.tools;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+/**
+ * Annotation to suppress forbidden-apis errors inside a whole class, a method, or a field.
+ */
+@Retention(RetentionPolicy.CLASS)
+@Target({ ElementType.CONSTRUCTOR, ElementType.FIELD, ElementType.METHOD, ElementType.TYPE })
+@interface SuppressForbidden {
+    String reason();
+}
diff --git a/distribution/tools/plugin-cli/build.gradle b/distribution/tools/plugin-cli/build.gradle
new file mode 100644
index 0000000000000..ae3dca9ef87af
--- /dev/null
+++ b/distribution/tools/plugin-cli/build.gradle
@@ -0,0 +1,32 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+apply plugin: 'elasticsearch.build'
+
+dependencies {
+  provided "org.elasticsearch:elasticsearch:${version}"
+  testCompile "org.elasticsearch.test:framework:${version}"
+  testCompile 'com.google.jimfs:jimfs:1.1'
+  testCompile 'com.google.guava:guava:18.0'
+}
+
+test {
+  // TODO: find a way to add permissions for the tests in this module
+  systemProperty 'tests.security.manager', 'false'
+}
diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
new file mode 100644
index 0000000000000..4be0bbc7fc41e
--- /dev/null
+++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
@@ -0,0 +1,706 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.plugins;
+
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+import org.apache.lucene.search.spell.LevensteinDistance;
+import org.apache.lucene.util.CollectionUtil;
+import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.Version;
+import org.elasticsearch.bootstrap.JarHell;
+import org.elasticsearch.cli.EnvironmentAwareCommand;
+import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cli.UserException;
+import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.hash.MessageDigests;
+import org.elasticsearch.common.io.FileSystemUtils;
+import org.elasticsearch.env.Environment;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.net.URLConnection;
+import java.net.URLDecoder;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.DirectoryStream;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.StandardCopyOption;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.nio.file.attribute.PosixFileAttributeView;
+import java.nio.file.attribute.PosixFileAttributes;
+import java.nio.file.attribute.PosixFilePermission;
+import java.nio.file.attribute.PosixFilePermissions;
+import java.security.MessageDigest;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Objects;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+
+import static org.elasticsearch.cli.Terminal.Verbosity.VERBOSE;
+
+/**
+ * A command for the plugin cli to install a plugin into elasticsearch.
+ *
+ * The install command takes a plugin id, which may be any of the following:
+ * <ul>
+ * <li>An official elasticsearch plugin name</li>
+ * <li>Maven coordinates to a plugin zip</li>
+ * <li>A URL to a plugin zip</li>
+ * </ul>
+ *
+ * Plugins are packaged as zip files. Each packaged plugin must contain a
+ * plugin properties file. See {@link PluginInfo}.
+ * <p>
+ * The installation process first extracts the plugin files into a temporary
+ * directory in order to verify the plugin satisfies the following requirements:
+ * <ul>
+ * <li>Jar hell does not exist, either between the plugin's own jars, or with elasticsearch</li>
+ * <li>The plugin is not a module already provided with elasticsearch</li>
+ * <li>If the plugin contains extra security permissions, the policy file is validated</li>
+ * </ul>
+ * <p>
+ * A plugin may also contain an optional {@code bin} directory which contains scripts. The
+ * scripts will be installed into a subdirectory of the elasticsearch bin directory, using
+ * the name of the plugin, and the scripts will be marked executable.
+ * <p>
+ * A plugin may also contain an optional {@code config} directory which contains configuration
+ * files specific to the plugin. The config files be installed into a subdirectory of the
+ * elasticsearch config directory, using the name of the plugin. If any files to be installed
+ * already exist, they will be skipped.
+ */
+class InstallPluginCommand extends EnvironmentAwareCommand {
+
+    private static final String PROPERTY_STAGING_ID = "es.plugins.staging";
+
+    // exit codes for install
+    /** A plugin with the same name is already installed. */
+    static final int PLUGIN_EXISTS = 1;
+    /** The plugin zip is not properly structured. */
+    static final int PLUGIN_MALFORMED = 2;
+
+
+    /** The builtin modules, which are plugins, but cannot be installed or removed. */
+    static final Set<String> MODULES;
+    static {
+        try (InputStream stream = InstallPluginCommand.class.getResourceAsStream("/modules.txt");
+            BufferedReader reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) {
+            Set<String> modules = new HashSet<>();
+            String line = reader.readLine();
+            while (line != null) {
+                modules.add(line.trim());
+                line = reader.readLine();
+            }
+            MODULES = Collections.unmodifiableSet(modules);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /** The official plugins that can be installed simply by name. */
+    static final Set<String> OFFICIAL_PLUGINS;
+    static {
+        try (InputStream stream = InstallPluginCommand.class.getResourceAsStream("/plugins.txt");
+            BufferedReader reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) {
+            Set<String> plugins = new TreeSet<>(); // use tree set to get sorting for help command
+            String line = reader.readLine();
+            while (line != null) {
+                plugins.add(line.trim());
+                line = reader.readLine();
+            }
+            plugins.add("x-pack");
+            OFFICIAL_PLUGINS = Collections.unmodifiableSet(plugins);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private final OptionSpec<Void> batchOption;
+    private final OptionSpec<String> arguments;
+
+    static final Set<PosixFilePermission> BIN_DIR_PERMS;
+    static final Set<PosixFilePermission> BIN_FILES_PERMS;
+    static final Set<PosixFilePermission> CONFIG_DIR_PERMS;
+    static final Set<PosixFilePermission> CONFIG_FILES_PERMS;
+    static final Set<PosixFilePermission> PLUGIN_DIR_PERMS;
+    static final Set<PosixFilePermission> PLUGIN_FILES_PERMS;
+
+    static {
+        // Bin directory get chmod 755
+        BIN_DIR_PERMS = Collections.unmodifiableSet(PosixFilePermissions.fromString("rwxr-xr-x"));
+
+        // Bin files also get chmod 755
+        BIN_FILES_PERMS = BIN_DIR_PERMS;
+
+        // Config directory get chmod 750
+        CONFIG_DIR_PERMS = Collections.unmodifiableSet(PosixFilePermissions.fromString("rwxr-x---"));
+
+        // Config files get chmod 660
+        CONFIG_FILES_PERMS = Collections.unmodifiableSet(PosixFilePermissions.fromString("rw-rw----"));
+
+        // Plugin directory get chmod 755
+        PLUGIN_DIR_PERMS = BIN_DIR_PERMS;
+
+        // Plugins files get chmod 644
+        PLUGIN_FILES_PERMS = Collections.unmodifiableSet(PosixFilePermissions.fromString("rw-r--r--"));
+    }
+
+    InstallPluginCommand() {
+        super("Install a plugin");
+        this.batchOption = parser.acceptsAll(Arrays.asList("b", "batch"),
+                "Enable batch mode explicitly, automatic confirmation of security permission");
+        this.arguments = parser.nonOptions("plugin id");
+    }
+
+    @Override
+    protected void printAdditionalHelp(Terminal terminal) {
+        terminal.println("The following official plugins may be installed by name:");
+        for (String plugin : OFFICIAL_PLUGINS) {
+            terminal.println("  " + plugin);
+        }
+        terminal.println("");
+    }
+
+    @Override
+    protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
+        String pluginId = arguments.value(options);
+        boolean isBatch = options.has(batchOption) || System.console() == null;
+        execute(terminal, pluginId, isBatch, env);
+    }
+
+    // pkg private for testing
+    void execute(Terminal terminal, String pluginId, boolean isBatch, Environment env) throws Exception {
+        if (pluginId == null) {
+            throw new UserException(ExitCodes.USAGE, "plugin id is required");
+        }
+
+        if (Files.exists(env.pluginsFile()) == false) {
+            terminal.println("Plugins directory [" + env.pluginsFile() + "] does not exist. Creating...");
+            Files.createDirectory(env.pluginsFile());
+        }
+
+        Path pluginZip = download(terminal, pluginId, env.tmpFile());
+        Path extractedZip = unzip(pluginZip, env.pluginsFile());
+        install(terminal, isBatch, extractedZip, env);
+    }
+
+    /** Downloads the plugin and returns the file it was downloaded to. */
+    private Path download(Terminal terminal, String pluginId, Path tmpDir) throws Exception {
+        if (OFFICIAL_PLUGINS.contains(pluginId)) {
+            final String url = getElasticUrl(terminal, getStagingHash(), Version.CURRENT, pluginId, Platforms.PLATFORM_NAME);
+            terminal.println("-> Downloading " + pluginId + " from elastic");
+            return downloadZipAndChecksum(terminal, url, tmpDir, false);
+        }
+
+        // now try as maven coordinates, a valid URL would only have a colon and slash
+        String[] coordinates = pluginId.split(":");
+        if (coordinates.length == 3 && pluginId.contains("/") == false) {
+            String mavenUrl = getMavenUrl(terminal, coordinates, Platforms.PLATFORM_NAME);
+            terminal.println("-> Downloading " + pluginId + " from maven central");
+            return downloadZipAndChecksum(terminal, mavenUrl, tmpDir, true);
+        }
+
+        // fall back to plain old URL
+        if (pluginId.contains(":/") == false) {
+            // definitely not a valid url, so assume it is a plugin name
+            List<String> plugins = checkMisspelledPlugin(pluginId);
+            String msg = "Unknown plugin " + pluginId;
+            if (plugins.isEmpty() == false) {
+                msg += ", did you mean " + (plugins.size() == 1 ? "[" + plugins.get(0) + "]": "any of " + plugins.toString()) + "?";
+            }
+            throw new UserException(ExitCodes.USAGE, msg);
+        }
+        terminal.println("-> Downloading " + URLDecoder.decode(pluginId, "UTF-8"));
+        return downloadZip(terminal, pluginId, tmpDir);
+    }
+
+    // pkg private so tests can override
+    String getStagingHash() {
+        return System.getProperty(PROPERTY_STAGING_ID);
+    }
+
+    /** Returns the url for an official elasticsearch plugin. */
+    private String getElasticUrl(Terminal terminal, String stagingHash, Version version,
+                                        String pluginId, String platform) throws IOException {
+        final String baseUrl;
+        if (stagingHash != null) {
+            baseUrl = String.format(Locale.ROOT,
+                "https://staging.elastic.co/%s-%s/downloads/elasticsearch-plugins/%s", version, stagingHash, pluginId);
+        } else {
+            baseUrl = String.format(Locale.ROOT,
+                "https://artifacts.elastic.co/downloads/elasticsearch-plugins/%s", pluginId);
+        }
+        final String platformUrl = String.format(Locale.ROOT, "%s/%s-%s-%s.zip", baseUrl, pluginId, platform, version);
+        if (urlExists(terminal, platformUrl)) {
+            return platformUrl;
+        }
+        return String.format(Locale.ROOT, "%s/%s-%s.zip", baseUrl, pluginId, version);
+    }
+
+    /** Returns the url for an elasticsearch plugin in maven. */
+    private String getMavenUrl(Terminal terminal, String[] coordinates, String platform) throws IOException {
+        final String groupId = coordinates[0].replace(".", "/");
+        final String artifactId = coordinates[1];
+        final String version = coordinates[2];
+        final String baseUrl = String.format(Locale.ROOT, "https://repo1.maven.org/maven2/%s/%s/%s", groupId, artifactId, version);
+        final String platformUrl = String.format(Locale.ROOT, "%s/%s-%s-%s.zip", baseUrl, artifactId, platform, version);
+        if (urlExists(terminal, platformUrl)) {
+            return platformUrl;
+        }
+        return String.format(Locale.ROOT, "%s/%s-%s.zip", baseUrl, artifactId, version);
+    }
+
+    /**
+     * Returns {@code true} if the given url exists, and {@code false} otherwise.
+     *
+     * The given url must be {@code https} and existing means a {@code HEAD} request returns 200.
+     */
+    // pkg private for tests to manipulate
+    @SuppressForbidden(reason = "Make HEAD request using URLConnection.connect()")
+    boolean urlExists(Terminal terminal, String urlString) throws IOException {
+        terminal.println(VERBOSE, "Checking if url exists: " + urlString);
+        URL url = new URL(urlString);
+        assert "https".equals(url.getProtocol()) : "Only http urls can be checked";
+        HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
+        urlConnection.addRequestProperty("User-Agent", "elasticsearch-plugin-installer");
+        urlConnection.setRequestMethod("HEAD");
+        urlConnection.connect();
+        return urlConnection.getResponseCode() == 200;
+    }
+
+    /** Returns all the official plugin names that look similar to pluginId. **/
+    private List<String> checkMisspelledPlugin(String pluginId) {
+        LevensteinDistance ld = new LevensteinDistance();
+        List<Tuple<Float, String>> scoredKeys = new ArrayList<>();
+        for (String officialPlugin : OFFICIAL_PLUGINS) {
+            float distance = ld.getDistance(pluginId, officialPlugin);
+            if (distance > 0.7f) {
+                scoredKeys.add(new Tuple<>(distance, officialPlugin));
+            }
+        }
+        CollectionUtil.timSort(scoredKeys, (a, b) -> b.v1().compareTo(a.v1()));
+        return scoredKeys.stream().map((a) -> a.v2()).collect(Collectors.toList());
+    }
+
+    /** Downloads a zip from the url, into a temp file under the given temp dir. */
+    @SuppressForbidden(reason = "We use getInputStream to download plugins")
+    Path downloadZip(Terminal terminal, String urlString, Path tmpDir) throws IOException {
+        terminal.println(VERBOSE, "Retrieving zip from " + urlString);
+        URL url = new URL(urlString);
+        Path zip = Files.createTempFile(tmpDir, null, ".zip");
+        URLConnection urlConnection = url.openConnection();
+        urlConnection.addRequestProperty("User-Agent", "elasticsearch-plugin-installer");
+        int contentLength = urlConnection.getContentLength();
+        try (InputStream in = new TerminalProgressInputStream(urlConnection.getInputStream(), contentLength, terminal)) {
+            // must overwrite since creating the temp file above actually created the file
+            Files.copy(in, zip, StandardCopyOption.REPLACE_EXISTING);
+        }
+        return zip;
+    }
+
+    /**
+     * content length might be -1 for unknown and progress only makes sense if the content length is greater than 0
+     */
+    private class TerminalProgressInputStream extends ProgressInputStream {
+
+        private final Terminal terminal;
+        private int width = 50;
+        private final boolean enabled;
+
+        TerminalProgressInputStream(InputStream is, int expectedTotalSize, Terminal terminal) {
+            super(is, expectedTotalSize);
+            this.terminal = terminal;
+            this.enabled = expectedTotalSize > 0;
+        }
+
+        @Override
+        public void onProgress(int percent) {
+            if (enabled) {
+                int currentPosition = percent * width / 100;
+                StringBuilder sb = new StringBuilder("\r[");
+                sb.append(String.join("=", Collections.nCopies(currentPosition, "")));
+                if (currentPosition > 0 && percent < 100) {
+                    sb.append(">");
+                }
+                sb.append(String.join(" ", Collections.nCopies(width - currentPosition, "")));
+                sb.append("] %s   ");
+                if (percent == 100) {
+                    sb.append("\n");
+                }
+                terminal.print(Terminal.Verbosity.NORMAL, String.format(Locale.ROOT, sb.toString(), percent + "%"));
+            }
+        }
+    }
+
+    /** Downloads a zip from the url, as well as a SHA512 (or SHA1) checksum, and checks the checksum. */
+    // pkg private for tests
+    @SuppressForbidden(reason = "We use openStream to download plugins")
+    private Path downloadZipAndChecksum(Terminal terminal, String urlString, Path tmpDir, boolean allowSha1) throws Exception {
+        Path zip = downloadZip(terminal, urlString, tmpDir);
+        pathsToDeleteOnShutdown.add(zip);
+        String checksumUrlString = urlString + ".sha512";
+        URL checksumUrl = openUrl(checksumUrlString);
+        String digestAlgo = "SHA-512";
+        if (checksumUrl == null && allowSha1) {
+            checksumUrlString = urlString + ".sha1";
+            checksumUrl = openUrl(checksumUrlString);
+            digestAlgo = "SHA-1";
+        }
+        if (checksumUrl == null) {
+            throw new UserException(ExitCodes.IO_ERROR, "Plugin checksum missing: " + checksumUrlString);
+        }
+        final String expectedChecksum;
+        try (InputStream in = checksumUrl.openStream()) {
+            BufferedReader checksumReader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8));
+            expectedChecksum = checksumReader.readLine();
+            if (checksumReader.readLine() != null) {
+                throw new UserException(ExitCodes.IO_ERROR, "Invalid checksum file at " + checksumUrl);
+            }
+        }
+
+        byte[] zipbytes = Files.readAllBytes(zip);
+        String gotChecksum = MessageDigests.toHexString(MessageDigest.getInstance(digestAlgo).digest(zipbytes));
+        if (expectedChecksum.equals(gotChecksum) == false) {
+            throw new UserException(ExitCodes.IO_ERROR,
+                digestAlgo + " mismatch, expected " + expectedChecksum + " but got " + gotChecksum);
+        }
+
+        return zip;
+    }
+
+    /**
+     * Creates a URL and opens a connection.
+     *
+     * If the URL returns a 404, {@code null} is returned, otherwise the open URL opject is returned.
+     */
+    // pkg private for tests
+    URL openUrl(String urlString) throws Exception {
+        URL checksumUrl = new URL(urlString);
+        HttpURLConnection connection = (HttpURLConnection)checksumUrl.openConnection();
+        if (connection.getResponseCode() == 404) {
+            return null;
+        }
+        return checksumUrl;
+    }
+
+    private Path unzip(Path zip, Path pluginsDir) throws IOException, UserException {
+        // unzip plugin to a staging temp dir
+
+        final Path target = stagingDirectory(pluginsDir);
+        pathsToDeleteOnShutdown.add(target);
+
+        boolean hasEsDir = false;
+        try (ZipInputStream zipInput = new ZipInputStream(Files.newInputStream(zip))) {
+            ZipEntry entry;
+            byte[] buffer = new byte[8192];
+            while ((entry = zipInput.getNextEntry()) != null) {
+                if (entry.getName().startsWith("elasticsearch/") == false) {
+                    // only extract the elasticsearch directory
+                    continue;
+                }
+                hasEsDir = true;
+                Path targetFile = target.resolve(entry.getName().substring("elasticsearch/".length()));
+
+                // Using the entry name as a path can result in an entry outside of the plugin dir,
+                // either if the name starts with the root of the filesystem, or it is a relative
+                // entry like ../whatever. This check attempts to identify both cases by first
+                // normalizing the path (which removes foo/..) and ensuring the normalized entry
+                // is still rooted with the target plugin directory.
+                if (targetFile.normalize().startsWith(target) == false) {
+                    throw new UserException(PLUGIN_MALFORMED, "Zip contains entry name '" +
+                        entry.getName() + "' resolving outside of plugin directory");
+                }
+
+                // be on the safe side: do not rely on that directories are always extracted
+                // before their children (although this makes sense, but is it guaranteed?)
+                if (!Files.isSymbolicLink(targetFile.getParent())) {
+                    Files.createDirectories(targetFile.getParent());
+                }
+                if (entry.isDirectory() == false) {
+                    try (OutputStream out = Files.newOutputStream(targetFile)) {
+                        int len;
+                        while ((len = zipInput.read(buffer)) >= 0) {
+                            out.write(buffer, 0, len);
+                        }
+                    }
+                }
+                zipInput.closeEntry();
+            }
+        }
+        Files.delete(zip);
+        if (hasEsDir == false) {
+            IOUtils.rm(target);
+            throw new UserException(PLUGIN_MALFORMED,
+                                    "`elasticsearch` directory is missing in the plugin zip");
+        }
+        return target;
+    }
+
+    private Path stagingDirectory(Path pluginsDir) throws IOException {
+        try {
+            return Files.createTempDirectory(pluginsDir, ".installing-", PosixFilePermissions.asFileAttribute(PLUGIN_DIR_PERMS));
+        } catch (IllegalArgumentException e) {
+            // Jimfs throws an IAE where it should throw an UOE
+            // remove when google/jimfs#30 is integrated into Jimfs
+            // and the Jimfs test dependency is upgraded to include
+            // this pull request
+            final StackTraceElement[] elements = e.getStackTrace();
+            if (elements.length >= 1 &&
+                elements[0].getClassName().equals("com.google.common.jimfs.AttributeService") &&
+                elements[0].getMethodName().equals("setAttributeInternal")) {
+                return stagingDirectoryWithoutPosixPermissions(pluginsDir);
+            } else {
+                throw e;
+            }
+        } catch (UnsupportedOperationException e) {
+            return stagingDirectoryWithoutPosixPermissions(pluginsDir);
+        }
+    }
+
+    private Path stagingDirectoryWithoutPosixPermissions(Path pluginsDir) throws IOException {
+        return Files.createTempDirectory(pluginsDir, ".installing-");
+    }
+
+    /** Load information about the plugin, and verify it can be installed with no errors. */
+    private PluginInfo verify(Terminal terminal, Path pluginRoot, boolean isBatch, Environment env) throws Exception {
+        // read and validate the plugin descriptor
+        PluginInfo info = PluginInfo.readFromProperties(pluginRoot);
+
+        // checking for existing version of the plugin
+        final Path destination = env.pluginsFile().resolve(info.getName());
+        if (Files.exists(destination)) {
+            final String message = String.format(
+                Locale.ROOT,
+                "plugin directory [%s] already exists; if you need to update the plugin, " +
+                    "uninstall it first using command 'remove %s'",
+                destination.toAbsolutePath(),
+                info.getName());
+            throw new UserException(PLUGIN_EXISTS, message);
+        }
+
+        PluginsService.checkForFailedPluginRemovals(env.pluginsFile());
+
+        terminal.println(VERBOSE, info.toString());
+
+        // don't let user install plugin as a module...
+        // they might be unavoidably in maven central and are packaged up the same way)
+        if (MODULES.contains(info.getName())) {
+            throw new UserException(ExitCodes.USAGE, "plugin '" + info.getName() +
+                "' cannot be installed like this, it is a system module");
+        }
+
+        // check for jar hell before any copying
+        jarHellCheck(pluginRoot, env.pluginsFile());
+
+        // read optional security policy (extra permissions)
+        // if it exists, confirm or warn the user
+        Path policy = pluginRoot.resolve(PluginInfo.ES_PLUGIN_POLICY);
+        if (Files.exists(policy)) {
+            PluginSecurity.readPolicy(info, policy, terminal, env::tmpFile, isBatch);
+        }
+
+        return info;
+    }
+
+    /** check a candidate plugin for jar hell before installing it */
+    void jarHellCheck(Path candidate, Path pluginsDir) throws Exception {
+        // create list of current jars in classpath
+        final Set<URL> jars = new HashSet<>(JarHell.parseClassPath());
+
+        // read existing bundles. this does some checks on the installation too.
+        PluginsService.getPluginBundles(pluginsDir);
+
+        // add plugin jars to the list
+        Path pluginJars[] = FileSystemUtils.files(candidate, "*.jar");
+        for (Path jar : pluginJars) {
+            if (jars.add(jar.toUri().toURL()) == false) {
+                throw new IllegalStateException("jar hell! duplicate plugin jar: " + jar);
+            }
+        }
+        // TODO: no jars should be an error
+        // TODO: verify the classname exists in one of the jars!
+
+        // check combined (current classpath + new jars to-be-added)
+        JarHell.checkJarHell(jars);
+    }
+
+    /**
+     * Installs the plugin from {@code tmpRoot} into the plugins dir.
+     * If the plugin has a bin dir and/or a config dir, those are copied.
+     */
+    private void install(Terminal terminal, boolean isBatch, Path tmpRoot, Environment env) throws Exception {
+        List<Path> deleteOnFailure = new ArrayList<>();
+        deleteOnFailure.add(tmpRoot);
+
+        try {
+            PluginInfo info = verify(terminal, tmpRoot, isBatch, env);
+            final Path destination = env.pluginsFile().resolve(info.getName());
+
+            Path tmpBinDir = tmpRoot.resolve("bin");
+            if (Files.exists(tmpBinDir)) {
+                Path destBinDir = env.binFile().resolve(info.getName());
+                deleteOnFailure.add(destBinDir);
+                installBin(info, tmpBinDir, destBinDir);
+            }
+
+            Path tmpConfigDir = tmpRoot.resolve("config");
+            if (Files.exists(tmpConfigDir)) {
+                // some files may already exist, and we don't remove plugin config files on plugin removal,
+                // so any installed config files are left on failure too
+                installConfig(info, tmpConfigDir, env.configFile().resolve(info.getName()));
+            }
+
+            Files.move(tmpRoot, destination, StandardCopyOption.ATOMIC_MOVE);
+            Files.walkFileTree(destination, new SimpleFileVisitor<Path>() {
+                @Override
+                public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException {
+                    if ("bin".equals(file.getParent().getFileName().toString())) {
+                        setFileAttributes(file, BIN_FILES_PERMS);
+                    } else {
+                        setFileAttributes(file, PLUGIN_FILES_PERMS);
+                    }
+                    return FileVisitResult.CONTINUE;
+                }
+
+                @Override
+                public FileVisitResult postVisitDirectory(final Path dir, final IOException exc) throws IOException {
+                    setFileAttributes(dir, PLUGIN_DIR_PERMS);
+                    return FileVisitResult.CONTINUE;
+                }
+            });
+
+            terminal.println("-> Installed " + info.getName());
+
+        } catch (Exception installProblem) {
+            try {
+                IOUtils.rm(deleteOnFailure.toArray(new Path[0]));
+            } catch (IOException exceptionWhileRemovingFiles) {
+                installProblem.addSuppressed(exceptionWhileRemovingFiles);
+            }
+            throw installProblem;
+        }
+    }
+
+    /** Copies the files from {@code tmpBinDir} into {@code destBinDir}, along with permissions from dest dirs parent. */
+    private void installBin(PluginInfo info, Path tmpBinDir, Path destBinDir) throws Exception {
+        if (Files.isDirectory(tmpBinDir) == false) {
+            throw new UserException(PLUGIN_MALFORMED, "bin in plugin " + info.getName() + " is not a directory");
+        }
+        Files.createDirectory(destBinDir);
+        setFileAttributes(destBinDir, BIN_DIR_PERMS);
+
+        try (DirectoryStream<Path> stream = Files.newDirectoryStream(tmpBinDir)) {
+            for (Path srcFile : stream) {
+                if (Files.isDirectory(srcFile)) {
+                    throw new UserException(PLUGIN_MALFORMED, "Directories not allowed in bin dir " +
+                        "for plugin " + info.getName() + ", found " + srcFile.getFileName());
+                }
+
+                Path destFile = destBinDir.resolve(tmpBinDir.relativize(srcFile));
+                Files.copy(srcFile, destFile);
+                setFileAttributes(destFile, BIN_FILES_PERMS);
+            }
+        }
+        IOUtils.rm(tmpBinDir); // clean up what we just copied
+    }
+
+    /**
+     * Copies the files from {@code tmpConfigDir} into {@code destConfigDir}.
+     * Any files existing in both the source and destination will be skipped.
+     */
+    private void installConfig(PluginInfo info, Path tmpConfigDir, Path destConfigDir) throws Exception {
+        if (Files.isDirectory(tmpConfigDir) == false) {
+            throw new UserException(PLUGIN_MALFORMED,
+                "config in plugin " + info.getName() + " is not a directory");
+        }
+
+        Files.createDirectories(destConfigDir);
+        setFileAttributes(destConfigDir, CONFIG_DIR_PERMS);
+        final PosixFileAttributeView destConfigDirAttributesView =
+            Files.getFileAttributeView(destConfigDir.getParent(), PosixFileAttributeView.class);
+        final PosixFileAttributes destConfigDirAttributes =
+            destConfigDirAttributesView != null ? destConfigDirAttributesView.readAttributes() : null;
+        if (destConfigDirAttributes != null) {
+            setOwnerGroup(destConfigDir, destConfigDirAttributes);
+        }
+
+        try (DirectoryStream<Path> stream = Files.newDirectoryStream(tmpConfigDir)) {
+            for (Path srcFile : stream) {
+                if (Files.isDirectory(srcFile)) {
+                    throw new UserException(PLUGIN_MALFORMED,
+                        "Directories not allowed in config dir for plugin " + info.getName());
+                }
+
+                Path destFile = destConfigDir.resolve(tmpConfigDir.relativize(srcFile));
+                if (Files.exists(destFile) == false) {
+                    Files.copy(srcFile, destFile);
+                    setFileAttributes(destFile, CONFIG_FILES_PERMS);
+                    if (destConfigDirAttributes != null) {
+                        setOwnerGroup(destFile, destConfigDirAttributes);
+                    }
+                }
+            }
+        }
+        IOUtils.rm(tmpConfigDir); // clean up what we just copied
+    }
+
+    private static void setOwnerGroup(final Path path, final PosixFileAttributes attributes) throws IOException {
+        Objects.requireNonNull(attributes);
+        PosixFileAttributeView fileAttributeView = Files.getFileAttributeView(path, PosixFileAttributeView.class);
+        assert fileAttributeView != null;
+        fileAttributeView.setOwner(attributes.owner());
+        fileAttributeView.setGroup(attributes.group());
+    }
+
+    /**
+     * Sets the attributes for a path iff posix attributes are supported
+     */
+    private static void setFileAttributes(final Path path, final Set<PosixFilePermission> permissions) throws IOException {
+        PosixFileAttributeView fileAttributeView = Files.getFileAttributeView(path, PosixFileAttributeView.class);
+        if (fileAttributeView != null) {
+            Files.setPosixFilePermissions(path, permissions);
+        }
+    }
+
+    private final List<Path> pathsToDeleteOnShutdown = new ArrayList<>();
+
+    @Override
+    public void close() throws IOException {
+        IOUtils.rm(pathsToDeleteOnShutdown.toArray(new Path[pathsToDeleteOnShutdown.size()]));
+    }
+
+}
diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/ListPluginsCommand.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/ListPluginsCommand.java
new file mode 100644
index 0000000000000..c2b5ce34b5469
--- /dev/null
+++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/ListPluginsCommand.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.plugins;
+
+import joptsimple.OptionSet;
+import org.elasticsearch.cli.EnvironmentAwareCommand;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.env.Environment;
+
+import java.io.IOException;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * A command for the plugin cli to list plugins installed in elasticsearch.
+ */
+class ListPluginsCommand extends EnvironmentAwareCommand {
+
+    ListPluginsCommand() {
+        super("Lists installed elasticsearch plugins");
+    }
+
+    @Override
+    protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
+        if (Files.exists(env.pluginsFile()) == false) {
+            throw new IOException("Plugins directory missing: " + env.pluginsFile());
+        }
+
+        terminal.println(Terminal.Verbosity.VERBOSE, "Plugins directory: " + env.pluginsFile());
+        final List<Path> plugins = new ArrayList<>();
+        try (DirectoryStream<Path> paths = Files.newDirectoryStream(env.pluginsFile())) {
+            for (Path plugin : paths) {
+                plugins.add(plugin);
+            }
+        }
+        Collections.sort(plugins);
+        for (final Path plugin : plugins) {
+            terminal.println(Terminal.Verbosity.SILENT, plugin.getFileName().toString());
+            try {
+                PluginInfo info = PluginInfo.readFromProperties(env.pluginsFile().resolve(plugin.toAbsolutePath()));
+                terminal.println(Terminal.Verbosity.VERBOSE, info.toString());
+            } catch (IllegalArgumentException e) {
+                if (e.getMessage().contains("incompatible with version")) {
+                    terminal.println("WARNING: " + e.getMessage());
+                } else {
+                    throw e;
+                }
+            }
+        }
+    }
+}
diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/PluginCli.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/PluginCli.java
new file mode 100644
index 0000000000000..ccc96c94eb7f8
--- /dev/null
+++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/PluginCli.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.plugins;
+
+import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.cli.Command;
+import org.elasticsearch.cli.MultiCommand;
+import org.elasticsearch.cli.Terminal;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+
+/**
+ * A cli tool for adding, removing and listing plugins for elasticsearch.
+ */
+public class PluginCli extends MultiCommand {
+
+    private final Collection<Command> commands;
+
+    private PluginCli() {
+        super("A tool for managing installed elasticsearch plugins");
+        subcommands.put("list", new ListPluginsCommand());
+        subcommands.put("install", new InstallPluginCommand());
+        subcommands.put("remove", new RemovePluginCommand());
+        commands = Collections.unmodifiableCollection(subcommands.values());
+    }
+
+    public static void main(String[] args) throws Exception {
+        exit(new PluginCli().main(args, Terminal.DEFAULT));
+    }
+
+    @Override
+    public void close() throws IOException {
+        IOUtils.close(commands);
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/plugins/ProgressInputStream.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/ProgressInputStream.java
similarity index 97%
rename from core/src/main/java/org/elasticsearch/plugins/ProgressInputStream.java
rename to distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/ProgressInputStream.java
index 16e1f203bb387..64d9437fa24b0 100644
--- a/core/src/main/java/org/elasticsearch/plugins/ProgressInputStream.java
+++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/ProgressInputStream.java
@@ -37,7 +37,7 @@ abstract class ProgressInputStream extends FilterInputStream {
     private int currentPercent;
     private int count = 0;
 
-    public ProgressInputStream(InputStream is, int expectedTotalSize) {
+    ProgressInputStream(InputStream is, int expectedTotalSize) {
         super(is);
         this.expectedTotalSize = expectedTotalSize;
         this.currentPercent = 0;
diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/RemovePluginCommand.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/RemovePluginCommand.java
new file mode 100644
index 0000000000000..3cd5a3290439e
--- /dev/null
+++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/RemovePluginCommand.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.plugins;
+
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.cli.EnvironmentAwareCommand;
+import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cli.UserException;
+import org.elasticsearch.env.Environment;
+
+import java.io.IOException;
+import java.nio.file.FileAlreadyExistsException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Locale;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.elasticsearch.cli.Terminal.Verbosity.VERBOSE;
+
+/**
+ * A command for the plugin CLI to remove a plugin from Elasticsearch.
+ */
+class RemovePluginCommand extends EnvironmentAwareCommand {
+
+    private final OptionSpec<Void> purgeOption;
+    private final OptionSpec<String> arguments;
+
+    RemovePluginCommand() {
+        super("removes a plugin from Elasticsearch");
+        this.purgeOption = parser.acceptsAll(Arrays.asList("p", "purge"), "Purge plugin configuration files");
+        this.arguments = parser.nonOptions("plugin name");
+    }
+
+    @Override
+    protected void execute(final Terminal terminal, final OptionSet options, final Environment env) throws Exception {
+        final String pluginName = arguments.value(options);
+        final boolean purge = options.has(purgeOption);
+        execute(terminal, env, pluginName, purge);
+    }
+
+    /**
+     * Remove the plugin specified by {@code pluginName}.
+     *
+     * @param terminal   the terminal to use for input/output
+     * @param env        the environment for the local node
+     * @param pluginName the name of the plugin to remove
+     * @param purge      if true, plugin configuration files will be removed but otherwise preserved
+     * @throws IOException   if any I/O exception occurs while performing a file operation
+     * @throws UserException if plugin name is null
+     * @throws UserException if plugin directory does not exist
+     * @throws UserException if the plugin bin directory is not a directory
+     */
+    void execute(
+            final Terminal terminal,
+            final Environment env,
+            final String pluginName,
+            final boolean purge) throws IOException, UserException {
+        if (pluginName == null) {
+            throw new UserException(ExitCodes.USAGE, "plugin name is required");
+        }
+
+        terminal.println("-> removing [" + pluginName + "]...");
+
+        final Path pluginDir = env.pluginsFile().resolve(pluginName);
+        final Path pluginConfigDir = env.configFile().resolve(pluginName);
+        final Path removing = env.pluginsFile().resolve(".removing-" + pluginName);
+        /*
+         * If the plugin does not exist and the plugin config does not exist, fail to the user that the plugin is not found, unless there's
+         * a marker file left from a previously failed attempt in which case we proceed to clean up the marker file. Or, if the plugin does
+         * not exist, the plugin config does, and we are not purging, again fail to the user that the plugin is not found.
+         */
+        if ((!Files.exists(pluginDir) && !Files.exists(pluginConfigDir) && !Files.exists(removing))
+                || (!Files.exists(pluginDir) && Files.exists(pluginConfigDir) && !purge)) {
+            final String message = String.format(
+                    Locale.ROOT, "plugin [%s] not found; run 'elasticsearch-plugin list' to get list of installed plugins", pluginName);
+            throw new UserException(ExitCodes.CONFIG, message);
+        }
+
+        final List<Path> pluginPaths = new ArrayList<>();
+
+        /*
+         * Add the contents of the plugin directory before creating the marker file and adding it to the list of paths to be deleted so
+         * that the marker file is the last file to be deleted.
+         */
+        if (Files.exists(pluginDir)) {
+            try (Stream<Path> paths = Files.list(pluginDir)) {
+                pluginPaths.addAll(paths.collect(Collectors.toList()));
+            }
+            terminal.println(VERBOSE, "removing [" + pluginDir + "]");
+        }
+
+        final Path pluginBinDir = env.binFile().resolve(pluginName);
+        if (Files.exists(pluginBinDir)) {
+            if (!Files.isDirectory(pluginBinDir)) {
+                throw new UserException(ExitCodes.IO_ERROR, "bin dir for " + pluginName + " is not a directory");
+            }
+            try (Stream<Path> paths = Files.list(pluginBinDir)) {
+                pluginPaths.addAll(paths.collect(Collectors.toList()));
+            }
+            pluginPaths.add(pluginBinDir);
+            terminal.println(VERBOSE, "removing [" + pluginBinDir + "]");
+        }
+
+        if (Files.exists(pluginConfigDir)) {
+            if (purge) {
+                try (Stream<Path> paths = Files.list(pluginConfigDir)) {
+                    pluginPaths.addAll(paths.collect(Collectors.toList()));
+                }
+                pluginPaths.add(pluginConfigDir);
+                terminal.println(VERBOSE, "removing [" + pluginConfigDir + "]");
+            } else {
+                /*
+                 * By default we preserve the config files in case the user is upgrading the plugin, but we print a message so the user
+                 * knows in case they want to remove manually.
+                 */
+                final String message = String.format(
+                        Locale.ROOT,
+                        "-> preserving plugin config files [%s] in case of upgrade; use --purge if not needed",
+                        pluginConfigDir);
+                terminal.println(message);
+            }
+        }
+
+        /*
+         * We are going to create a marker file in the plugin directory that indicates that this plugin is a state of removal. If the
+         * removal fails, the existence of this marker file indicates that the plugin is in a garbage state. We check for existence of this
+         * marker file during startup so that we do not startup with plugins in such a garbage state. Up to this point, we have not done
+         * anything destructive, so we create the marker file as the last action before executing destructive operations. We place this
+         * marker file in the root plugin directory (not the specific plugin directory) so that we do not have to create the specific plugin
+         * directory if it does not exist (we are purging configuration files).
+         */
+        try {
+            Files.createFile(removing);
+        } catch (final FileAlreadyExistsException e) {
+            /*
+             * We need to suppress the marker file already existing as we could be in this state if a previous removal attempt failed and
+             * the user is attempting to remove the plugin again.
+             */
+            terminal.println(VERBOSE, "marker file [" + removing + "] already exists");
+        }
+
+        // add the plugin directory
+        pluginPaths.add(pluginDir);
+
+        // finally, add the marker file
+        pluginPaths.add(removing);
+
+        IOUtils.rm(pluginPaths.toArray(new Path[pluginPaths.size()]));
+    }
+
+}
diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java
new file mode 100644
index 0000000000000..54253587556e9
--- /dev/null
+++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java
@@ -0,0 +1,895 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.plugins;
+
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import com.google.common.jimfs.Configuration;
+import com.google.common.jimfs.Jimfs;
+import org.apache.lucene.util.LuceneTestCase;
+import org.elasticsearch.Version;
+import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.MockTerminal;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cli.UserException;
+import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.hash.MessageDigests;
+import org.elasticsearch.common.io.PathUtils;
+import org.elasticsearch.common.io.PathUtilsForTesting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.PosixPermissionsResetter;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.StringReader;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.DirectoryStream;
+import java.nio.file.FileAlreadyExistsException;
+import java.nio.file.FileSystem;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.StandardCopyOption;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.nio.file.attribute.GroupPrincipal;
+import java.nio.file.attribute.PosixFileAttributeView;
+import java.nio.file.attribute.PosixFileAttributes;
+import java.nio.file.attribute.PosixFilePermission;
+import java.nio.file.attribute.UserPrincipal;
+import java.security.MessageDigest;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.hasToString;
+import static org.hamcrest.Matchers.not;
+
+@LuceneTestCase.SuppressFileSystems("*")
+public class InstallPluginCommandTests extends ESTestCase {
+
+    private InstallPluginCommand skipJarHellCommand;
+    private InstallPluginCommand defaultCommand;
+
+    private final Function<String, Path> temp;
+
+    private final FileSystem fs;
+    private final boolean isPosix;
+    private final boolean isReal;
+    private final String javaIoTmpdir;
+
+    @SuppressForbidden(reason = "sets java.io.tmpdir")
+    public InstallPluginCommandTests(FileSystem fs, Function<String, Path> temp) {
+        this.fs = fs;
+        this.temp = temp;
+        this.isPosix = fs.supportedFileAttributeViews().contains("posix");
+        this.isReal = fs == PathUtils.getDefaultFileSystem();
+        PathUtilsForTesting.installMock(fs);
+        javaIoTmpdir = System.getProperty("java.io.tmpdir");
+        System.setProperty("java.io.tmpdir", temp.apply("tmpdir").toString());
+    }
+
+    @Before
+    public void setUp() throws Exception {
+        super.setUp();
+        skipJarHellCommand = new InstallPluginCommand() {
+            @Override
+            void jarHellCheck(Path candidate, Path pluginsDir) throws Exception {
+                // no jarhell check
+            }
+        };
+        defaultCommand = new InstallPluginCommand();
+    }
+
+    @After
+    @SuppressForbidden(reason = "resets java.io.tmpdir")
+    public void tearDown() throws Exception {
+        defaultCommand.close();
+        skipJarHellCommand.close();
+        System.setProperty("java.io.tmpdir", javaIoTmpdir);
+        PathUtilsForTesting.teardown();
+        super.tearDown();
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() {
+        class Parameter {
+            private final FileSystem fileSystem;
+            private final Function<String, Path> temp;
+
+            Parameter(FileSystem fileSystem, String root) {
+                this(fileSystem, s -> {
+                    try {
+                        return Files.createTempDirectory(fileSystem.getPath(root), s);
+                    } catch (IOException e) {
+                        throw new RuntimeException(e);
+                    }
+                });
+            }
+
+            Parameter(FileSystem fileSystem, Function<String, Path> temp) {
+                this.fileSystem = fileSystem;
+                this.temp = temp;
+            }
+        }
+        List<Parameter> parameters = new ArrayList<>();
+        parameters.add(new Parameter(Jimfs.newFileSystem(Configuration.windows()), "c:\\"));
+        parameters.add(new Parameter(Jimfs.newFileSystem(toPosix(Configuration.osX())), "/"));
+        parameters.add(new Parameter(Jimfs.newFileSystem(toPosix(Configuration.unix())), "/"));
+        parameters.add(new Parameter(PathUtils.getDefaultFileSystem(), LuceneTestCase::createTempDir ));
+        return parameters.stream().map(p -> new Object[] { p.fileSystem, p.temp }).collect(Collectors.toList());
+    }
+
+    private static Configuration toPosix(Configuration configuration) {
+        return configuration.toBuilder().setAttributeViews("basic", "owner", "posix", "unix").build();
+    }
+
+    /** Creates a test environment with bin, config and plugins directories. */
+    static Tuple<Path, Environment> createEnv(FileSystem fs, Function<String, Path> temp) throws IOException {
+        Path home = temp.apply("install-plugin-command-tests");
+        Files.createDirectories(home.resolve("bin"));
+        Files.createFile(home.resolve("bin").resolve("elasticsearch"));
+        Files.createDirectories(home.resolve("config"));
+        Files.createFile(home.resolve("config").resolve("elasticsearch.yml"));
+        Path plugins = Files.createDirectories(home.resolve("plugins"));
+        assertTrue(Files.exists(plugins));
+        Settings settings = Settings.builder()
+            .put("path.home", home)
+            .build();
+        return Tuple.tuple(home, new Environment(settings));
+    }
+
+    static Path createPluginDir(Function<String, Path> temp) throws IOException {
+        return temp.apply("pluginDir");
+    }
+
+    /** creates a fake jar file with empty class files */
+    static void writeJar(Path jar, String... classes) throws IOException {
+        try (ZipOutputStream stream = new ZipOutputStream(Files.newOutputStream(jar))) {
+            for (String clazz : classes) {
+                stream.putNextEntry(new ZipEntry(clazz + ".class")); // no package names, just support simple classes
+            }
+        }
+    }
+
+    static Path writeZip(Path structure, String prefix) throws IOException {
+        Path zip = createTempDir().resolve(structure.getFileName() + ".zip");
+        try (ZipOutputStream stream = new ZipOutputStream(Files.newOutputStream(zip))) {
+            Files.walkFileTree(structure, new SimpleFileVisitor<Path>() {
+                @Override
+                public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+                    String target = (prefix == null ? "" : prefix + "/") + structure.relativize(file).toString();
+                    stream.putNextEntry(new ZipEntry(target));
+                    Files.copy(file, stream);
+                    return FileVisitResult.CONTINUE;
+                }
+            });
+        }
+        return zip;
+    }
+
+    /** creates a plugin .zip and returns the url for testing */
+    static String createPluginUrl(String name, Path structure) throws IOException {
+        return createPlugin(name, structure, false).toUri().toURL().toString();
+    }
+
+    static Path createPlugin(String name, Path structure, boolean createSecurityPolicyFile) throws IOException {
+        PluginTestUtil.writeProperties(structure,
+            "description", "fake desc",
+            "name", name,
+            "version", "1.0",
+            "elasticsearch.version", Version.CURRENT.toString(),
+            "java.version", System.getProperty("java.specification.version"),
+            "classname", "FakePlugin");
+        if (createSecurityPolicyFile) {
+            String securityPolicyContent = "grant {\n  permission java.lang.RuntimePermission \"setFactory\";\n};\n";
+            Files.write(structure.resolve("plugin-security.policy"), securityPolicyContent.getBytes(StandardCharsets.UTF_8));
+        }
+        writeJar(structure.resolve("plugin.jar"), "FakePlugin");
+        return writeZip(structure, "elasticsearch");
+    }
+
+    MockTerminal installPlugin(String pluginUrl, Path home) throws Exception {
+        return installPlugin(pluginUrl, home, skipJarHellCommand);
+    }
+
+    MockTerminal installPlugin(String pluginUrl, Path home, InstallPluginCommand command) throws Exception {
+        Environment env = new Environment(Settings.builder().put("path.home", home).build());
+        MockTerminal terminal = new MockTerminal();
+        command.execute(terminal, pluginUrl, true, env);
+        return terminal;
+    }
+
+    void assertPlugin(String name, Path original, Environment env) throws IOException {
+        Path got = env.pluginsFile().resolve(name);
+        assertTrue("dir " + name + " exists", Files.exists(got));
+
+        if (isPosix) {
+            Set<PosixFilePermission> perms = Files.getPosixFilePermissions(got);
+            assertThat(
+                perms,
+                containsInAnyOrder(
+                    PosixFilePermission.OWNER_READ,
+                    PosixFilePermission.OWNER_WRITE,
+                    PosixFilePermission.OWNER_EXECUTE,
+                    PosixFilePermission.GROUP_READ,
+                    PosixFilePermission.GROUP_EXECUTE,
+                    PosixFilePermission.OTHERS_READ,
+                    PosixFilePermission.OTHERS_EXECUTE));
+        }
+
+        assertTrue("jar was copied", Files.exists(got.resolve("plugin.jar")));
+        assertFalse("bin was not copied", Files.exists(got.resolve("bin")));
+        assertFalse("config was not copied", Files.exists(got.resolve("config")));
+        if (Files.exists(original.resolve("bin"))) {
+            Path binDir = env.binFile().resolve(name);
+            assertTrue("bin dir exists", Files.exists(binDir));
+            assertTrue("bin is a dir", Files.isDirectory(binDir));
+            PosixFileAttributes binAttributes = null;
+            if (isPosix) {
+                binAttributes = Files.readAttributes(env.binFile(), PosixFileAttributes.class);
+            }
+            try (DirectoryStream<Path> stream = Files.newDirectoryStream(binDir)) {
+                for (Path file : stream) {
+                    assertFalse("not a dir", Files.isDirectory(file));
+                    if (isPosix) {
+                        PosixFileAttributes attributes = Files.readAttributes(file, PosixFileAttributes.class);
+                        assertEquals(InstallPluginCommand.BIN_FILES_PERMS, attributes.permissions());
+                    }
+                }
+            }
+        }
+        if (Files.exists(original.resolve("config"))) {
+            Path configDir = env.configFile().resolve(name);
+            assertTrue("config dir exists", Files.exists(configDir));
+            assertTrue("config is a dir", Files.isDirectory(configDir));
+
+            UserPrincipal user = null;
+            GroupPrincipal group = null;
+
+            if (isPosix) {
+                PosixFileAttributes configAttributes =
+                        Files.getFileAttributeView(env.configFile(), PosixFileAttributeView.class).readAttributes();
+                user = configAttributes.owner();
+                group = configAttributes.group();
+
+                PosixFileAttributes attributes = Files.getFileAttributeView(configDir, PosixFileAttributeView.class).readAttributes();
+                assertThat(attributes.owner(), equalTo(user));
+                assertThat(attributes.group(), equalTo(group));
+            }
+
+            try (DirectoryStream<Path> stream = Files.newDirectoryStream(configDir)) {
+                for (Path file : stream) {
+                    assertFalse("not a dir", Files.isDirectory(file));
+
+                    if (isPosix) {
+                        PosixFileAttributes attributes = Files.readAttributes(file, PosixFileAttributes.class);
+                        if (user != null) {
+                            assertThat(attributes.owner(), equalTo(user));
+                        }
+                        if (group != null) {
+                            assertThat(attributes.group(), equalTo(group));
+                        }
+                    }
+                }
+            }
+        }
+        assertInstallCleaned(env);
+    }
+
+    void assertInstallCleaned(Environment env) throws IOException {
+        try (DirectoryStream<Path> stream = Files.newDirectoryStream(env.pluginsFile())) {
+            for (Path file : stream) {
+                if (file.getFileName().toString().startsWith(".installing")) {
+                    fail("Installation dir still exists, " + file);
+                }
+            }
+        }
+    }
+
+    public void testMissingPluginId() throws IOException {
+        final Tuple<Path, Environment> env = createEnv(fs, temp);
+        final UserException e = expectThrows(UserException.class, () -> installPlugin(null, env.v1()));
+        assertTrue(e.getMessage(), e.getMessage().contains("plugin id is required"));
+    }
+
+    public void testSomethingWorks() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
+        String pluginZip = createPluginUrl("fake", pluginDir);
+        installPlugin(pluginZip, env.v1());
+        assertPlugin("fake", pluginDir, env.v2());
+    }
+
+    public void testInstallFailsIfPreviouslyRemovedPluginFailed() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
+        String pluginZip = createPluginUrl("fake", pluginDir);
+        final Path removing = env.v2().pluginsFile().resolve(".removing-failed");
+        Files.createDirectory(removing);
+        final IllegalStateException e = expectThrows(IllegalStateException.class, () -> installPlugin(pluginZip, env.v1()));
+        final String expected = String.format(
+                Locale.ROOT,
+                "found file [%s] from a failed attempt to remove the plugin [failed]; execute [elasticsearch-plugin remove failed]",
+                removing);
+        assertThat(e, hasToString(containsString(expected)));
+    }
+
+    public void testSpaceInUrl() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
+        String pluginZip = createPluginUrl("fake", pluginDir);
+        Path pluginZipWithSpaces = createTempFile("foo bar", ".zip");
+        try (InputStream in = new URL(pluginZip).openStream()) {
+            Files.copy(in, pluginZipWithSpaces, StandardCopyOption.REPLACE_EXISTING);
+        }
+        installPlugin(pluginZipWithSpaces.toUri().toURL().toString(), env.v1());
+        assertPlugin("fake", pluginDir, env.v2());
+    }
+
+    public void testMalformedUrlNotMaven() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        // has two colons, so it appears similar to maven coordinates
+        MalformedURLException e = expectThrows(MalformedURLException.class, () -> installPlugin("://host:1234", env.v1()));
+        assertTrue(e.getMessage(), e.getMessage().contains("no protocol"));
+    }
+
+    public void testUnknownPlugin() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        UserException e = expectThrows(UserException.class, () -> installPlugin("foo", env.v1()));
+        assertTrue(e.getMessage(), e.getMessage().contains("Unknown plugin foo"));
+    }
+
+    public void testPluginsDirMissing() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Files.delete(env.v2().pluginsFile());
+        Path pluginDir = createPluginDir(temp);
+        String pluginZip = createPluginUrl("fake", pluginDir);
+        installPlugin(pluginZip, env.v1());
+        assertPlugin("fake", pluginDir, env.v2());
+    }
+
+    public void testPluginsDirReadOnly() throws Exception {
+        assumeTrue("posix and filesystem", isPosix && isReal);
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
+        try (PosixPermissionsResetter pluginsAttrs = new PosixPermissionsResetter(env.v2().pluginsFile())) {
+            pluginsAttrs.setPermissions(new HashSet<>());
+            String pluginZip = createPluginUrl("fake", pluginDir);
+            IOException e = expectThrows(IOException.class, () -> installPlugin(pluginZip, env.v1()));
+            assertTrue(e.getMessage(), e.getMessage().contains(env.v2().pluginsFile().toString()));
+        }
+        assertInstallCleaned(env.v2());
+    }
+
+    public void testBuiltinModule() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
+        String pluginZip = createPluginUrl("lang-painless", pluginDir);
+        UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1()));
+        assertTrue(e.getMessage(), e.getMessage().contains("is a system module"));
+        assertInstallCleaned(env.v2());
+    }
+
+    public void testJarHell() throws Exception {
+        // jar hell test needs a real filesystem
+        assumeTrue("real filesystem", isReal);
+        Tuple<Path, Environment> environment = createEnv(fs, temp);
+        Path pluginDirectory = createPluginDir(temp);
+        writeJar(pluginDirectory.resolve("other.jar"), "FakePlugin");
+        String pluginZip = createPluginUrl("fake", pluginDirectory); // adds plugin.jar with FakePlugin
+        IllegalStateException e = expectThrows(IllegalStateException.class,
+            () -> installPlugin(pluginZip, environment.v1(), defaultCommand));
+        assertTrue(e.getMessage(), e.getMessage().contains("jar hell"));
+        assertInstallCleaned(environment.v2());
+    }
+
+    public void testIsolatedPlugins() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        // these both share the same FakePlugin class
+        Path pluginDir1 = createPluginDir(temp);
+        String pluginZip1 = createPluginUrl("fake1", pluginDir1);
+        installPlugin(pluginZip1, env.v1());
+        Path pluginDir2 = createPluginDir(temp);
+        String pluginZip2 = createPluginUrl("fake2", pluginDir2);
+        installPlugin(pluginZip2, env.v1());
+        assertPlugin("fake1", pluginDir1, env.v2());
+        assertPlugin("fake2", pluginDir2, env.v2());
+    }
+
+    public void testExistingPlugin() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
+        String pluginZip = createPluginUrl("fake", pluginDir);
+        installPlugin(pluginZip, env.v1());
+        UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1()));
+        assertTrue(e.getMessage(), e.getMessage().contains("already exists"));
+        assertInstallCleaned(env.v2());
+    }
+
+    public void testBin() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
+        Path binDir = pluginDir.resolve("bin");
+        Files.createDirectory(binDir);
+        Files.createFile(binDir.resolve("somescript"));
+        String pluginZip = createPluginUrl("fake", pluginDir);
+        installPlugin(pluginZip, env.v1());
+        assertPlugin("fake", pluginDir, env.v2());
+    }
+
+    public void testBinNotDir() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
+        Path binDir = pluginDir.resolve("bin");
+        Files.createFile(binDir);
+        String pluginZip = createPluginUrl("fake", pluginDir);
+        UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1()));
+        assertTrue(e.getMessage(), e.getMessage().contains("not a directory"));
+        assertInstallCleaned(env.v2());
+    }
+
+    public void testBinContainsDir() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
+        Path dirInBinDir = pluginDir.resolve("bin").resolve("foo");
+        Files.createDirectories(dirInBinDir);
+        Files.createFile(dirInBinDir.resolve("somescript"));
+        String pluginZip = createPluginUrl("fake", pluginDir);
+        UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1()));
+        assertTrue(e.getMessage(), e.getMessage().contains("Directories not allowed in bin dir for plugin"));
+        assertInstallCleaned(env.v2());
+    }
+
+    public void testBinConflict() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
+        Path binDir = pluginDir.resolve("bin");
+        Files.createDirectory(binDir);
+        Files.createFile(binDir.resolve("somescript"));
+        String pluginZip = createPluginUrl("elasticsearch", pluginDir);
+        FileAlreadyExistsException e = expectThrows(FileAlreadyExistsException.class, () -> installPlugin(pluginZip, env.v1()));
+        assertTrue(e.getMessage(), e.getMessage().contains(env.v2().binFile().resolve("elasticsearch").toString()));
+        assertInstallCleaned(env.v2());
+    }
+
+    public void testBinPermissions() throws Exception {
+        assumeTrue("posix filesystem", isPosix);
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
+        Path binDir = pluginDir.resolve("bin");
+        Files.createDirectory(binDir);
+        Files.createFile(binDir.resolve("somescript"));
+        String pluginZip = createPluginUrl("fake", pluginDir);
+        try (PosixPermissionsResetter binAttrs = new PosixPermissionsResetter(env.v2().binFile())) {
+            Set<PosixFilePermission> perms = binAttrs.getCopyPermissions();
+            // make sure at least one execute perm is missing, so we know we forced it during installation
+            perms.remove(PosixFilePermission.GROUP_EXECUTE);
+            binAttrs.setPermissions(perms);
+            installPlugin(pluginZip, env.v1());
+            assertPlugin("fake", pluginDir, env.v2());
+        }
+    }
+
+    public void testPluginPermissions() throws Exception {
+        assumeTrue("posix filesystem", isPosix);
+
+        final Tuple<Path, Environment> env = createEnv(fs, temp);
+        final Path pluginDir = createPluginDir(temp);
+        final Path resourcesDir = pluginDir.resolve("resources");
+        final Path platformDir = pluginDir.resolve("platform");
+        final Path platformNameDir = platformDir.resolve("linux-x86_64");
+        final Path platformBinDir = platformNameDir.resolve("bin");
+        Files.createDirectories(platformBinDir);
+
+        Files.createFile(pluginDir.resolve("fake-" + Version.CURRENT.toString() + ".jar"));
+        Files.createFile(platformBinDir.resolve("fake_executable"));
+        Files.createDirectory(resourcesDir);
+        Files.createFile(resourcesDir.resolve("resource"));
+
+        final String pluginZip = createPluginUrl("fake", pluginDir);
+
+        installPlugin(pluginZip, env.v1());
+        assertPlugin("fake", pluginDir, env.v2());
+
+        final Path fake = env.v2().pluginsFile().resolve("fake");
+        final Path resources = fake.resolve("resources");
+        final Path platform = fake.resolve("platform");
+        final Path platformName = platform.resolve("linux-x86_64");
+        final Path bin = platformName.resolve("bin");
+        assert755(fake);
+        assert644(fake.resolve("fake-" + Version.CURRENT + ".jar"));
+        assert755(resources);
+        assert644(resources.resolve("resource"));
+        assert755(platform);
+        assert755(platformName);
+        assert755(bin.resolve("fake_executable"));
+    }
+
+    private void assert644(final Path path) throws IOException {
+        final Set<PosixFilePermission> permissions = Files.getPosixFilePermissions(path);
+        assertTrue(permissions.contains(PosixFilePermission.OWNER_READ));
+        assertTrue(permissions.contains(PosixFilePermission.OWNER_WRITE));
+        assertFalse(permissions.contains(PosixFilePermission.OWNER_EXECUTE));
+        assertTrue(permissions.contains(PosixFilePermission.GROUP_READ));
+        assertFalse(permissions.contains(PosixFilePermission.GROUP_WRITE));
+        assertFalse(permissions.contains(PosixFilePermission.GROUP_EXECUTE));
+        assertTrue(permissions.contains(PosixFilePermission.OTHERS_READ));
+        assertFalse(permissions.contains(PosixFilePermission.OTHERS_WRITE));
+        assertFalse(permissions.contains(PosixFilePermission.OTHERS_EXECUTE));
+    }
+
+    private void assert755(final Path path) throws IOException {
+        final Set<PosixFilePermission> permissions = Files.getPosixFilePermissions(path);
+        assertTrue(permissions.contains(PosixFilePermission.OWNER_READ));
+        assertTrue(permissions.contains(PosixFilePermission.OWNER_WRITE));
+        assertTrue(permissions.contains(PosixFilePermission.OWNER_EXECUTE));
+        assertTrue(permissions.contains(PosixFilePermission.GROUP_READ));
+        assertFalse(permissions.contains(PosixFilePermission.GROUP_WRITE));
+        assertTrue(permissions.contains(PosixFilePermission.GROUP_EXECUTE));
+        assertTrue(permissions.contains(PosixFilePermission.OTHERS_READ));
+        assertFalse(permissions.contains(PosixFilePermission.OTHERS_WRITE));
+        assertTrue(permissions.contains(PosixFilePermission.OTHERS_EXECUTE));
+    }
+
+    public void testConfig() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
+        Path configDir = pluginDir.resolve("config");
+        Files.createDirectory(configDir);
+        Files.createFile(configDir.resolve("custom.yaml"));
+        String pluginZip = createPluginUrl("fake", pluginDir);
+        installPlugin(pluginZip, env.v1());
+        assertPlugin("fake", pluginDir, env.v2());
+    }
+
+    public void testExistingConfig() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Path envConfigDir = env.v2().configFile().resolve("fake");
+        Files.createDirectories(envConfigDir);
+        Files.write(envConfigDir.resolve("custom.yaml"), "existing config".getBytes(StandardCharsets.UTF_8));
+        Path pluginDir = createPluginDir(temp);
+        Path configDir = pluginDir.resolve("config");
+        Files.createDirectory(configDir);
+        Files.write(configDir.resolve("custom.yaml"), "new config".getBytes(StandardCharsets.UTF_8));
+        Files.createFile(configDir.resolve("other.yaml"));
+        String pluginZip = createPluginUrl("fake", pluginDir);
+        installPlugin(pluginZip, env.v1());
+        assertPlugin("fake", pluginDir, env.v2());
+        List<String> configLines = Files.readAllLines(envConfigDir.resolve("custom.yaml"), StandardCharsets.UTF_8);
+        assertEquals(1, configLines.size());
+        assertEquals("existing config", configLines.get(0));
+        assertTrue(Files.exists(envConfigDir.resolve("other.yaml")));
+    }
+
+    public void testConfigNotDir() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
+        Path configDir = pluginDir.resolve("config");
+        Files.createFile(configDir);
+        String pluginZip = createPluginUrl("fake", pluginDir);
+        UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1()));
+        assertTrue(e.getMessage(), e.getMessage().contains("not a directory"));
+        assertInstallCleaned(env.v2());
+    }
+
+    public void testConfigContainsDir() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
+        Path dirInConfigDir = pluginDir.resolve("config").resolve("foo");
+        Files.createDirectories(dirInConfigDir);
+        Files.createFile(dirInConfigDir.resolve("myconfig.yml"));
+        String pluginZip = createPluginUrl("fake", pluginDir);
+        UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1()));
+        assertTrue(e.getMessage(), e.getMessage().contains("Directories not allowed in config dir for plugin"));
+        assertInstallCleaned(env.v2());
+    }
+
+    public void testConfigConflict() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
+        Path configDir = pluginDir.resolve("config");
+        Files.createDirectory(configDir);
+        Files.createFile(configDir.resolve("myconfig.yml"));
+        String pluginZip = createPluginUrl("elasticsearch.yml", pluginDir);
+        FileAlreadyExistsException e = expectThrows(FileAlreadyExistsException.class, () -> installPlugin(pluginZip, env.v1()));
+        assertTrue(e.getMessage(), e.getMessage().contains(env.v2().configFile().resolve("elasticsearch.yml").toString()));
+        assertInstallCleaned(env.v2());
+    }
+
+    public void testMissingDescriptor() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
+        Files.createFile(pluginDir.resolve("fake.yml"));
+        String pluginZip = writeZip(pluginDir, "elasticsearch").toUri().toURL().toString();
+        NoSuchFileException e = expectThrows(NoSuchFileException.class, () -> installPlugin(pluginZip, env.v1()));
+        assertTrue(e.getMessage(), e.getMessage().contains("plugin-descriptor.properties"));
+        assertInstallCleaned(env.v2());
+    }
+
+    public void testMissingDirectory() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
+        Files.createFile(pluginDir.resolve(PluginInfo.ES_PLUGIN_PROPERTIES));
+        String pluginZip = writeZip(pluginDir, null).toUri().toURL().toString();
+        UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1()));
+        assertTrue(e.getMessage(), e.getMessage().contains("`elasticsearch` directory is missing in the plugin zip"));
+        assertInstallCleaned(env.v2());
+    }
+
+    public void testZipRelativeOutsideEntryName() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Path zip = createTempDir().resolve("broken.zip");
+        try (ZipOutputStream stream = new ZipOutputStream(Files.newOutputStream(zip))) {
+            stream.putNextEntry(new ZipEntry("elasticsearch/../blah"));
+        }
+        String pluginZip = zip.toUri().toURL().toString();
+        UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1()));
+        assertTrue(e.getMessage(), e.getMessage().contains("resolving outside of plugin directory"));
+    }
+
+    public void testOfficialPluginsHelpSorted() throws Exception {
+        MockTerminal terminal = new MockTerminal();
+        new InstallPluginCommand() {
+            @Override
+            protected boolean addShutdownHook() {
+                return false;
+            }
+        }.main(new String[] { "--help" }, terminal);
+        try (BufferedReader reader = new BufferedReader(new StringReader(terminal.getOutput()))) {
+            String line = reader.readLine();
+
+            // first find the beginning of our list of official plugins
+            while (line.endsWith("may be installed by name:") == false) {
+                line = reader.readLine();
+            }
+
+            // now check each line compares greater than the last, until we reach an empty line
+            String prev = reader.readLine();
+            line = reader.readLine();
+            while (line != null && line.trim().isEmpty() == false) {
+                assertTrue(prev + " < " + line, prev.compareTo(line) < 0);
+                prev = line;
+                line = reader.readLine();
+            }
+        }
+    }
+
+    public void testOfficialPluginsIncludesXpack() throws Exception {
+        MockTerminal terminal = new MockTerminal();
+        new InstallPluginCommand() {
+            @Override
+            protected boolean addShutdownHook() {
+                return false;
+            }
+        }.main(new String[] { "--help" }, terminal);
+        assertTrue(terminal.getOutput(), terminal.getOutput().contains("x-pack"));
+    }
+
+    public void testInstallMisspelledOfficialPlugins() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        UserException e = expectThrows(UserException.class, () -> installPlugin("xpack", env.v1()));
+        assertThat(e.getMessage(), containsString("Unknown plugin xpack, did you mean [x-pack]?"));
+
+        e = expectThrows(UserException.class, () -> installPlugin("analysis-smartnc", env.v1()));
+        assertThat(e.getMessage(), containsString("Unknown plugin analysis-smartnc, did you mean [analysis-smartcn]?"));
+
+        e = expectThrows(UserException.class, () -> installPlugin("repository", env.v1()));
+        assertThat(e.getMessage(), containsString("Unknown plugin repository, did you mean any of [repository-s3, repository-gcs]?"));
+
+        e = expectThrows(UserException.class, () -> installPlugin("unknown_plugin", env.v1()));
+        assertThat(e.getMessage(), containsString("Unknown plugin unknown_plugin"));
+    }
+
+    public void testBatchFlag() throws Exception {
+        MockTerminal terminal = new MockTerminal();
+        installPlugin(terminal, true);
+        assertThat(terminal.getOutput(), containsString("WARNING: plugin requires additional permissions"));
+    }
+
+    public void testQuietFlagDisabled() throws Exception {
+        MockTerminal terminal = new MockTerminal();
+        terminal.setVerbosity(randomFrom(Terminal.Verbosity.NORMAL, Terminal.Verbosity.VERBOSE));
+        installPlugin(terminal, false);
+        assertThat(terminal.getOutput(), containsString("100%"));
+    }
+
+    public void testQuietFlagEnabled() throws Exception {
+        MockTerminal terminal = new MockTerminal();
+        terminal.setVerbosity(Terminal.Verbosity.SILENT);
+        installPlugin(terminal, false);
+        assertThat(terminal.getOutput(), not(containsString("100%")));
+    }
+
+    public void testPluginAlreadyInstalled() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
+        String pluginZip = createPluginUrl("fake", pluginDir);
+        installPlugin(pluginZip, env.v1());
+        final UserException e = expectThrows(UserException.class,
+            () -> installPlugin(pluginZip, env.v1(), randomFrom(skipJarHellCommand, defaultCommand)));
+        assertThat(
+            e.getMessage(),
+            equalTo("plugin directory [" + env.v2().pluginsFile().resolve("fake") + "] already exists; " +
+                "if you need to update the plugin, uninstall it first using command 'remove fake'"));
+    }
+
+    private void installPlugin(MockTerminal terminal, boolean isBatch) throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
+        // if batch is enabled, we also want to add a security policy
+        String pluginZip = createPlugin("fake", pluginDir, isBatch).toUri().toURL().toString();
+        skipJarHellCommand.execute(terminal, pluginZip, isBatch, env.v2());
+    }
+
+    public MockTerminal assertInstallPluginFromUrl(String pluginId, String name, String url, String stagingHash,
+                                                   String shaExtension, Function<byte[], String> shaCalculator) throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
+        Path pluginZip = createPlugin(name, pluginDir, false);
+        InstallPluginCommand command = new InstallPluginCommand() {
+            @Override
+            Path downloadZip(Terminal terminal, String urlString, Path tmpDir) throws IOException {
+                assertEquals(url, urlString);
+                Path downloadedPath = tmpDir.resolve("downloaded.zip");
+                Files.copy(pluginZip, downloadedPath);
+                return downloadedPath;
+            }
+            @Override
+            URL openUrl(String urlString) throws Exception {
+                String expectedUrl = url + shaExtension;
+                if (expectedUrl.equals(urlString)) {
+                    // calc sha an return file URL to it
+                    Path shaFile = temp.apply("shas").resolve("downloaded.zip" + shaExtension);
+                    byte[] zipbytes = Files.readAllBytes(pluginZip);
+                    String checksum = shaCalculator.apply(zipbytes);
+                    Files.write(shaFile, checksum.getBytes(StandardCharsets.UTF_8));
+                    return shaFile.toUri().toURL();
+                }
+                return null;
+            }
+            @Override
+            boolean urlExists(Terminal terminal, String urlString) throws IOException {
+                return urlString.equals(url);
+            }
+            @Override
+            String getStagingHash() {
+                return stagingHash;
+            }
+            @Override
+            void jarHellCheck(Path candidate, Path pluginsDir) throws Exception {
+                // no jarhell check
+            }
+        };
+        MockTerminal terminal = installPlugin(pluginId, env.v1(), command);
+        assertPlugin(name, pluginDir, env.v2());
+        return terminal;
+    }
+
+    public void assertInstallPluginFromUrl(String pluginId, String name, String url, String stagingHash) throws Exception {
+        MessageDigest digest = MessageDigest.getInstance("SHA-512");
+        assertInstallPluginFromUrl(pluginId, name, url, stagingHash, ".sha512",
+            bytes -> MessageDigests.toHexString(digest.digest(bytes)));
+    }
+
+    public void testOfficalPlugin() throws Exception {
+        String url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-" + Version.CURRENT + ".zip";
+        assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null);
+    }
+
+    public void testOfficalPluginStaging() throws Exception {
+        String url = "https://staging.elastic.co/" + Version.CURRENT + "-abc123/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-"
+            + Version.CURRENT + ".zip";
+        assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, "abc123");
+    }
+
+    public void testOfficalPlatformPlugin() throws Exception {
+        String url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-" + Platforms.PLATFORM_NAME +
+            "-" + Version.CURRENT + ".zip";
+        assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null);
+    }
+
+    public void testOfficalPlatformPluginStaging() throws Exception {
+        String url = "https://staging.elastic.co/" + Version.CURRENT + "-abc123/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-"
+            + Platforms.PLATFORM_NAME + "-"+ Version.CURRENT + ".zip";
+        assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, "abc123");
+    }
+
+    public void testMavenPlugin() throws Exception {
+        String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-1.0.0.zip";
+        assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null);
+    }
+
+    public void testMavenPlatformPlugin() throws Exception {
+        String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-" + Platforms.PLATFORM_NAME + "-1.0.0.zip";
+        assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null);
+    }
+
+    public void testMavenSha1Backcompat() throws Exception {
+        String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-1.0.0.zip";
+        MessageDigest digest = MessageDigest.getInstance("SHA-1");
+        MockTerminal terminal = assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null,
+                                               ".sha1", bytes -> MessageDigests.toHexString(digest.digest(bytes)));
+    }
+
+    public void testOfficialShaMissing() throws Exception {
+        String url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-" + Version.CURRENT + ".zip";
+        MessageDigest digest = MessageDigest.getInstance("SHA-1");
+        UserException e = expectThrows(UserException.class, () ->
+            assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, ".sha1",
+                                       bytes -> MessageDigests.toHexString(digest.digest(bytes))));
+        assertEquals(ExitCodes.IO_ERROR, e.exitCode);
+        assertEquals("Plugin checksum missing: " + url + ".sha512", e.getMessage());
+    }
+
+    public void testMavenShaMissing() throws Exception {
+        String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-1.0.0.zip";
+        UserException e = expectThrows(UserException.class, () ->
+            assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null, ".dne", bytes -> null));
+        assertEquals(ExitCodes.IO_ERROR, e.exitCode);
+        assertEquals("Plugin checksum missing: " + url + ".sha1", e.getMessage());
+    }
+
+    public void testInvalidShaFile() throws Exception {
+        String url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-" + Version.CURRENT + ".zip";
+        MessageDigest digest = MessageDigest.getInstance("SHA-512");
+        UserException e = expectThrows(UserException.class, () ->
+            assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, ".sha512",
+                bytes -> MessageDigests.toHexString(digest.digest(bytes)) + "\nfoobar"));
+        assertEquals(ExitCodes.IO_ERROR, e.exitCode);
+        assertTrue(e.getMessage(), e.getMessage().startsWith("Invalid checksum file"));
+    }
+
+    public void testSha512Mismatch() throws Exception {
+        String url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-" + Version.CURRENT + ".zip";
+        UserException e = expectThrows(UserException.class, () ->
+            assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, ".sha512",
+                bytes -> "foobar"));
+        assertEquals(ExitCodes.IO_ERROR, e.exitCode);
+        assertTrue(e.getMessage(), e.getMessage().contains("SHA-512 mismatch, expected foobar"));
+    }
+
+    public void testSha1Mismatch() throws Exception {
+        String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-1.0.0.zip";
+        UserException e = expectThrows(UserException.class, () ->
+            assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null,
+                ".sha1", bytes -> "foobar"));
+        assertEquals(ExitCodes.IO_ERROR, e.exitCode);
+        assertTrue(e.getMessage(), e.getMessage().contains("SHA-1 mismatch, expected foobar"));
+    }
+
+}
diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/ListPluginsCommandTests.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/ListPluginsCommandTests.java
new file mode 100644
index 0000000000000..7ebc2f0709bce
--- /dev/null
+++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/ListPluginsCommandTests.java
@@ -0,0 +1,240 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.plugins;
+
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Locale;
+import java.util.stream.Collectors;
+
+import org.apache.lucene.util.LuceneTestCase;
+import org.elasticsearch.Version;
+import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.MockTerminal;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.Before;
+
+@LuceneTestCase.SuppressFileSystems("*")
+public class ListPluginsCommandTests extends ESTestCase {
+
+    private Path home;
+    private Environment env;
+
+    @Before
+    public void setUp() throws Exception {
+        super.setUp();
+        home = createTempDir();
+        Files.createDirectories(home.resolve("plugins"));
+        Settings settings = Settings.builder()
+                .put("path.home", home)
+                .build();
+        env = new Environment(settings);
+    }
+
+    static MockTerminal listPlugins(Path home) throws Exception {
+        return listPlugins(home, new String[0]);
+    }
+
+    static MockTerminal listPlugins(Path home, String[] args) throws Exception {
+        String[] argsAndHome = new String[args.length + 1];
+        System.arraycopy(args, 0, argsAndHome, 0, args.length);
+        argsAndHome[args.length] = "-Epath.home=" + home;
+        MockTerminal terminal = new MockTerminal();
+        int status = new ListPluginsCommand() {
+            @Override
+            protected boolean addShutdownHook() {
+                return false;
+            }
+        }.main(argsAndHome, terminal);
+        assertEquals(ExitCodes.OK, status);
+        return terminal;
+    }
+
+    private static String buildMultiline(String... args){
+        return Arrays.stream(args).collect(Collectors.joining("\n", "", "\n"));
+    }
+
+    private static void buildFakePlugin(
+            final Environment env,
+            final String description,
+            final String name,
+            final String classname) throws IOException {
+        buildFakePlugin(env, description, name, classname, false);
+    }
+
+    private static void buildFakePlugin(
+            final Environment env,
+            final String description,
+            final String name,
+            final String classname,
+            final boolean hasNativeController) throws IOException {
+        PluginTestUtil.writeProperties(
+                env.pluginsFile().resolve(name),
+                "description", description,
+                "name", name,
+                "version", "1.0",
+                "elasticsearch.version", Version.CURRENT.toString(),
+                "java.version", System.getProperty("java.specification.version"),
+                "classname", classname,
+                "has.native.controller", Boolean.toString(hasNativeController));
+    }
+
+    public void testPluginsDirMissing() throws Exception {
+        Files.delete(env.pluginsFile());
+        IOException e = expectThrows(IOException.class, () -> listPlugins(home));
+        assertEquals("Plugins directory missing: " + env.pluginsFile(), e.getMessage());
+    }
+
+    public void testNoPlugins() throws Exception {
+        MockTerminal terminal = listPlugins(home);
+        assertTrue(terminal.getOutput(), terminal.getOutput().isEmpty());
+    }
+
+    public void testOnePlugin() throws Exception {
+        buildFakePlugin(env, "fake desc", "fake", "org.fake");
+        MockTerminal terminal = listPlugins(home);
+        assertEquals(buildMultiline("fake"), terminal.getOutput());
+    }
+
+    public void testTwoPlugins() throws Exception {
+        buildFakePlugin(env, "fake desc", "fake1", "org.fake");
+        buildFakePlugin(env, "fake desc 2", "fake2", "org.fake");
+        MockTerminal terminal = listPlugins(home);
+        assertEquals(buildMultiline("fake1", "fake2"), terminal.getOutput());
+    }
+
+    public void testPluginWithVerbose() throws Exception {
+        buildFakePlugin(env, "fake desc", "fake_plugin", "org.fake");
+        String[] params = { "-v" };
+        MockTerminal terminal = listPlugins(home, params);
+        assertEquals(
+                buildMultiline(
+                        "Plugins directory: " + env.pluginsFile(),
+                        "fake_plugin",
+                        "- Plugin information:",
+                        "Name: fake_plugin",
+                        "Description: fake desc",
+                        "Version: 1.0",
+                        "Native Controller: false",
+                        " * Classname: org.fake"),
+                terminal.getOutput());
+    }
+
+    public void testPluginWithNativeController() throws Exception {
+        buildFakePlugin(env, "fake desc 1", "fake_plugin1", "org.fake", true);
+        String[] params = { "-v" };
+        MockTerminal terminal = listPlugins(home, params);
+        assertEquals(
+                buildMultiline(
+                        "Plugins directory: " + env.pluginsFile(),
+                        "fake_plugin1",
+                        "- Plugin information:",
+                        "Name: fake_plugin1",
+                        "Description: fake desc 1",
+                        "Version: 1.0",
+                        "Native Controller: true",
+                        " * Classname: org.fake"),
+                terminal.getOutput());
+    }
+
+    public void testPluginWithVerboseMultiplePlugins() throws Exception {
+        buildFakePlugin(env, "fake desc 1", "fake_plugin1", "org.fake");
+        buildFakePlugin(env, "fake desc 2", "fake_plugin2", "org.fake2");
+        String[] params = { "-v" };
+        MockTerminal terminal = listPlugins(home, params);
+        assertEquals(
+                buildMultiline(
+                        "Plugins directory: " + env.pluginsFile(),
+                        "fake_plugin1",
+                        "- Plugin information:",
+                        "Name: fake_plugin1",
+                        "Description: fake desc 1",
+                        "Version: 1.0",
+                        "Native Controller: false",
+                        " * Classname: org.fake",
+                        "fake_plugin2",
+                        "- Plugin information:",
+                        "Name: fake_plugin2",
+                        "Description: fake desc 2",
+                        "Version: 1.0",
+                        "Native Controller: false",
+                        " * Classname: org.fake2"),
+                terminal.getOutput());
+    }
+
+    public void testPluginWithoutVerboseMultiplePlugins() throws Exception {
+        buildFakePlugin(env, "fake desc 1", "fake_plugin1", "org.fake");
+        buildFakePlugin(env, "fake desc 2", "fake_plugin2", "org.fake2");
+        MockTerminal terminal = listPlugins(home, new String[0]);
+        String output = terminal.getOutput();
+        assertEquals(buildMultiline("fake_plugin1", "fake_plugin2"), output);
+    }
+
+    public void testPluginWithoutDescriptorFile() throws Exception{
+        final Path pluginDir = env.pluginsFile().resolve("fake1");
+        Files.createDirectories(pluginDir);
+        NoSuchFileException e = expectThrows(NoSuchFileException.class, () -> listPlugins(home));
+        assertEquals(pluginDir.resolve(PluginInfo.ES_PLUGIN_PROPERTIES).toString(), e.getFile());
+    }
+
+    public void testPluginWithWrongDescriptorFile() throws Exception{
+        final Path pluginDir = env.pluginsFile().resolve("fake1");
+        PluginTestUtil.writeProperties(pluginDir, "description", "fake desc");
+        IllegalArgumentException e = expectThrows(
+                IllegalArgumentException.class,
+                () -> listPlugins(home));
+        final Path descriptorPath = pluginDir.resolve(PluginInfo.ES_PLUGIN_PROPERTIES);
+        assertEquals(
+                "property [name] is missing in [" + descriptorPath.toString() + "]",
+                e.getMessage());
+    }
+
+    public void testExistingIncompatiblePlugin() throws Exception {
+        PluginTestUtil.writeProperties(env.pluginsFile().resolve("fake_plugin1"),
+            "description", "fake desc 1",
+            "name", "fake_plugin1",
+            "version", "1.0",
+            "elasticsearch.version", Version.fromString("1.0.0").toString(),
+            "java.version", System.getProperty("java.specification.version"),
+            "classname", "org.fake1");
+        buildFakePlugin(env, "fake desc 2", "fake_plugin2", "org.fake2");
+
+        MockTerminal terminal = listPlugins(home);
+        final String message = String.format(Locale.ROOT,
+                "plugin [%s] is incompatible with version [%s]; was designed for version [%s]",
+                "fake_plugin1",
+                Version.CURRENT.toString(),
+                "1.0.0");
+        assertEquals(
+                "fake_plugin1\n" + "WARNING: " + message + "\n" + "fake_plugin2\n",
+                terminal.getOutput());
+
+        String[] params = {"-s"};
+        terminal = listPlugins(home, params);
+        assertEquals("fake_plugin1\nfake_plugin2\n", terminal.getOutput());
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/plugins/ProgressInputStreamTests.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/ProgressInputStreamTests.java
similarity index 100%
rename from core/src/test/java/org/elasticsearch/plugins/ProgressInputStreamTests.java
rename to distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/ProgressInputStreamTests.java
diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java
new file mode 100644
index 0000000000000..5879f2821d979
--- /dev/null
+++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java
@@ -0,0 +1,210 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.plugins;
+
+import org.apache.lucene.util.LuceneTestCase;
+import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.MockTerminal;
+import org.elasticsearch.cli.UserException;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.Before;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.StringReader;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.not;
+import static org.hamcrest.Matchers.hasToString;
+
+@LuceneTestCase.SuppressFileSystems("*")
+public class RemovePluginCommandTests extends ESTestCase {
+
+    private Path home;
+    private Environment env;
+
+    @Override
+    @Before
+    public void setUp() throws Exception {
+        super.setUp();
+        home = createTempDir();
+        Files.createDirectories(home.resolve("bin"));
+        Files.createFile(home.resolve("bin").resolve("elasticsearch"));
+        Files.createDirectories(home.resolve("plugins"));
+        Settings settings = Settings.builder()
+                .put("path.home", home)
+                .build();
+        env = new Environment(settings);
+    }
+
+    static MockTerminal removePlugin(String name, Path home, boolean purge) throws Exception {
+        Environment env = new Environment(Settings.builder().put("path.home", home).build());
+        MockTerminal terminal = new MockTerminal();
+        new RemovePluginCommand().execute(terminal, env, name, purge);
+        return terminal;
+    }
+
+    static void assertRemoveCleaned(Environment env) throws IOException {
+        try (DirectoryStream<Path> stream = Files.newDirectoryStream(env.pluginsFile())) {
+            for (Path file : stream) {
+                if (file.getFileName().toString().startsWith(".removing")) {
+                    fail("Removal dir still exists, " + file);
+                }
+            }
+        }
+    }
+
+    public void testMissing() throws Exception {
+        UserException e = expectThrows(UserException.class, () -> removePlugin("dne", home, randomBoolean()));
+        assertTrue(e.getMessage(), e.getMessage().contains("plugin [dne] not found"));
+        assertRemoveCleaned(env);
+    }
+
+    public void testBasic() throws Exception {
+        Files.createDirectory(env.pluginsFile().resolve("fake"));
+        Files.createFile(env.pluginsFile().resolve("fake").resolve("plugin.jar"));
+        Files.createDirectory(env.pluginsFile().resolve("fake").resolve("subdir"));
+        Files.createDirectory(env.pluginsFile().resolve("other"));
+        removePlugin("fake", home, randomBoolean());
+        assertFalse(Files.exists(env.pluginsFile().resolve("fake")));
+        assertTrue(Files.exists(env.pluginsFile().resolve("other")));
+        assertRemoveCleaned(env);
+    }
+
+    public void testBin() throws Exception {
+        Files.createDirectories(env.pluginsFile().resolve("fake"));
+        Path binDir = env.binFile().resolve("fake");
+        Files.createDirectories(binDir);
+        Files.createFile(binDir.resolve("somescript"));
+        removePlugin("fake", home, randomBoolean());
+        assertFalse(Files.exists(env.pluginsFile().resolve("fake")));
+        assertTrue(Files.exists(env.binFile().resolve("elasticsearch")));
+        assertFalse(Files.exists(binDir));
+        assertRemoveCleaned(env);
+    }
+
+    public void testBinNotDir() throws Exception {
+        Files.createDirectories(env.pluginsFile().resolve("elasticsearch"));
+        UserException e = expectThrows(UserException.class, () -> removePlugin("elasticsearch", home, randomBoolean()));
+        assertTrue(e.getMessage(), e.getMessage().contains("not a directory"));
+        assertTrue(Files.exists(env.pluginsFile().resolve("elasticsearch"))); // did not remove
+        assertTrue(Files.exists(env.binFile().resolve("elasticsearch")));
+        assertRemoveCleaned(env);
+    }
+
+    public void testConfigDirPreserved() throws Exception {
+        Files.createDirectories(env.pluginsFile().resolve("fake"));
+        final Path configDir = env.configFile().resolve("fake");
+        Files.createDirectories(configDir);
+        Files.createFile(configDir.resolve("fake.yml"));
+        final MockTerminal terminal = removePlugin("fake", home, false);
+        assertTrue(Files.exists(env.configFile().resolve("fake")));
+        assertThat(terminal.getOutput(), containsString(expectedConfigDirPreservedMessage(configDir)));
+        assertRemoveCleaned(env);
+    }
+
+    public void testPurgePluginExists() throws Exception {
+        Files.createDirectories(env.pluginsFile().resolve("fake"));
+        final Path configDir = env.configFile().resolve("fake");
+        if (randomBoolean()) {
+            Files.createDirectories(configDir);
+            Files.createFile(configDir.resolve("fake.yml"));
+        }
+        final MockTerminal terminal = removePlugin("fake", home, true);
+        assertFalse(Files.exists(env.configFile().resolve("fake")));
+        assertThat(terminal.getOutput(), not(containsString(expectedConfigDirPreservedMessage(configDir))));
+        assertRemoveCleaned(env);
+    }
+
+    public void testPurgePluginDoesNotExist() throws Exception {
+        final Path configDir = env.configFile().resolve("fake");
+        Files.createDirectories(configDir);
+        Files.createFile(configDir.resolve("fake.yml"));
+        final MockTerminal terminal = removePlugin("fake", home, true);
+        assertFalse(Files.exists(env.configFile().resolve("fake")));
+        assertThat(terminal.getOutput(), not(containsString(expectedConfigDirPreservedMessage(configDir))));
+        assertRemoveCleaned(env);
+    }
+
+    public void testPurgeNothingExists() throws Exception {
+        final UserException e = expectThrows(UserException.class, () -> removePlugin("fake", home, true));
+        assertThat(e, hasToString(containsString("plugin [fake] not found")));
+    }
+
+    public void testPurgeOnlyMarkerFileExists() throws Exception {
+        final Path configDir = env.configFile().resolve("fake");
+        final Path removing = env.pluginsFile().resolve(".removing-fake");
+        Files.createFile(removing);
+        final MockTerminal terminal = removePlugin("fake", home, randomBoolean());
+        assertFalse(Files.exists(removing));
+        assertThat(terminal.getOutput(), not(containsString(expectedConfigDirPreservedMessage(configDir))));
+    }
+
+    public void testNoConfigDirPreserved() throws Exception {
+        Files.createDirectories(env.pluginsFile().resolve("fake"));
+        final Path configDir = env.configFile().resolve("fake");
+        final MockTerminal terminal = removePlugin("fake", home, randomBoolean());
+        assertThat(terminal.getOutput(), not(containsString(expectedConfigDirPreservedMessage(configDir))));
+    }
+
+    public void testRemoveUninstalledPluginErrors() throws Exception {
+        UserException e = expectThrows(UserException.class, () -> removePlugin("fake", home, randomBoolean()));
+        assertEquals(ExitCodes.CONFIG, e.exitCode);
+        assertEquals("plugin [fake] not found; run 'elasticsearch-plugin list' to get list of installed plugins", e.getMessage());
+
+        MockTerminal terminal = new MockTerminal();
+        new RemovePluginCommand() {
+            @Override
+            protected boolean addShutdownHook() {
+                return false;
+            }
+        }.main(new String[] { "-Epath.home=" + home, "fake" }, terminal);
+        try (BufferedReader reader = new BufferedReader(new StringReader(terminal.getOutput()))) {
+            assertEquals("-> removing [fake]...", reader.readLine());
+            assertEquals("ERROR: plugin [fake] not found; run 'elasticsearch-plugin list' to get list of installed plugins",
+                    reader.readLine());
+            assertNull(reader.readLine());
+        }
+    }
+
+    public void testMissingPluginName() throws Exception {
+        UserException e = expectThrows(UserException.class, () -> removePlugin(null, home, randomBoolean()));
+        assertEquals(ExitCodes.USAGE, e.exitCode);
+        assertEquals("plugin name is required", e.getMessage());
+    }
+
+    public void testRemoveWhenRemovingMarker() throws Exception {
+        Files.createDirectory(env.pluginsFile().resolve("fake"));
+        Files.createFile(env.pluginsFile().resolve("fake").resolve("plugin.jar"));
+        Files.createFile(env.pluginsFile().resolve(".removing-fake"));
+        removePlugin("fake", home, randomBoolean());
+    }
+
+    private String expectedConfigDirPreservedMessage(final Path configDir) {
+        return "-> preserving plugin config files [" + configDir + "] in case of upgrade; use --purge if not needed";
+    }
+
+}
+
diff --git a/distribution/zip/build.gradle b/distribution/zip/build.gradle
index 67f99aa884a9d..80da41319958c 100644
--- a/distribution/zip/build.gradle
+++ b/distribution/zip/build.gradle
@@ -17,7 +17,10 @@
  * under the License.
  */
 
+import org.elasticsearch.gradle.plugin.PluginBuildPlugin
+
 task buildZip(type: Zip) {
+  dependsOn createPluginsDir
   baseName = 'elasticsearch'
   with archivesFiles
 }
@@ -30,10 +33,32 @@ artifacts {
 publishing {
   publications {
     nebula {
+      artifactId 'elasticsearch'
       artifact buildZip
     }
+    /* HUGE HACK: the underlying maven publication library refuses to deploy any attached artifacts
+     * when the packaging type is set to 'pom'. But Sonatype's OSS repositories require source files
+     * for artifacts that are of type 'zip'. We already publish the source and javadoc for Elasticsearch
+     * under the various other subprojects. So here we create another publication using the same
+     * name that has the "real" pom, and rely on the fact that gradle will execute the publish tasks
+     * in alphabetical order. This lets us publish the zip file and even though the pom says the
+     * type is 'pom' instead of 'zip'. We cannot setup a dependency between the tasks because the
+     * publishing tasks are created *extremely* late in the configuration phase, so that we cannot get
+     * ahold of the actual task. Furthermore, this entire hack only exists so we can make publishing to
+     * maven local work, since we publish to maven central externally. */
+    nebulaRealPom(MavenPublication) {
+      artifactId 'elasticsearch'
+      pom.packaging = 'pom'
+      pom.withXml { XmlProvider xml ->
+        Node root = xml.asNode()
+        root.appendNode('name', 'Elasticsearch')
+        root.appendNode('description', 'A Distributed RESTful Search Engine')
+        root.appendNode('url', PluginBuildPlugin.urlFromOrigin(project.scminfo.origin))
+        Node scmNode = root.appendNode('scm')
+        scmNode.appendNode('url', project.scminfo.origin)
+      }
+    }
   }
 }
 
 integTest.dependsOn buildZip
-
diff --git a/distribution/zip/src/test/java/org/elasticsearch/test/rest/ZipClientYamlTestSuiteIT.java b/distribution/zip/src/test/java/org/elasticsearch/test/rest/ZipClientYamlTestSuiteIT.java
index 329c8259d0b6e..dc08af2528b52 100644
--- a/distribution/zip/src/test/java/org/elasticsearch/test/rest/ZipClientYamlTestSuiteIT.java
+++ b/distribution/zip/src/test/java/org/elasticsearch/test/rest/ZipClientYamlTestSuiteIT.java
@@ -23,9 +23,6 @@
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-
-import java.io.IOException;
 
 /** Rest integration test. Runs against a cluster started by {@code gradle integTest} */
 public class ZipClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
@@ -34,7 +31,7 @@ public ZipClientYamlTestSuiteIT(ClientYamlTestCandidate testCandidate) {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
-        return createParameters(0, 1);
+    public static Iterable<Object[]> parameters() throws Exception {
+        return createParameters();
     }
 }
diff --git a/docs/README.asciidoc b/docs/README.asciidoc
index 5da211c662215..f76682b4ce346 100644
--- a/docs/README.asciidoc
+++ b/docs/README.asciidoc
@@ -4,21 +4,28 @@ Elasticsearch documentation build process.
 See: https://github.com/elastic/docs
 
 Snippets marked with `// CONSOLE` are automatically annotated with "VIEW IN
-SENSE" in the documentation and are automatically tested by the command
-`gradle :docs:check`. By default `// CONSOLE` snippet runs as its own isolated
-test. You can manipulate the test execution in the following ways:
+CONSOLE" and "COPY AS CURL" in the documentation and are automatically tested
+by the command `gradle :docs:check`. To test just the docs from a single page,
+use e.g. `gradle :docs:check -Dtests.method=*rollover*`.
+
+By default each `// CONSOLE` snippet runs as its own isolated test. You can
+manipulate the test execution in the following ways:
 
 * `// TEST`: Explicitly marks a snippet as a test. Snippets marked this way
-are tests even if they don't have `// CONSOLE`.
-  * `// TEST[s/foo/bar/]`: Replace `foo` with `bar` in the test. This should be
-  used sparingly because it makes the test "lie". Sometimes, though, you can use
-  it to make the tests more clear.
+are tests even if they don't have `// CONSOLE` but usually `// TEST` is used
+for its modifiers:
+  * `// TEST[s/foo/bar/]`: Replace `foo` with `bar` in the generated test. This
+  should be used sparingly because it makes the snippet "lie". Sometimes,
+  though, you can use it to make the snippet more clear more clear. Keep in
+  mind the that if there are multiple substitutions then they are applied in
+  the order that they are defined.
   * `// TEST[catch:foo]`: Used to expect errors in the requests. Replace `foo`
   with `request` to expect a 400 error, for example. If the snippet contains
   multiple requests then only the last request will expect the error.
   * `// TEST[continued]`: Continue the test started in the last snippet. Between
-  tests the nodes are cleaned: indexes are removed, etc. This will prevent that.
-  This is really useful when you have text and snippets that work together to
+  tests the nodes are cleaned: indexes are removed, etc. This prevents that
+  from happening between snippets because the two snippets are a single test.
+  This is most useful when you have text and snippets that work together to
   tell the story of some use case because it merges the snippets (and thus the
   use case) into one big test.
   * `// TEST[skip:reason]`: Skip this test. Replace `reason` with the actual
@@ -33,10 +40,25 @@ are tests even if they don't have `// CONSOLE`.
   text then the test fails. If the response includes `Warning` headers that
   aren't expected then the test fails.
 * `// TESTRESPONSE`: Matches this snippet against the body of the response of
-  the last test. If the response is JSON then order is ignored. With
-  `// TEST[continued]` you can make tests that contain multiple command snippets
-  and multiple response snippets.
-  * `// TESTRESPONSE[s/foo/bar/]`: Substitutions. See `// TEST[s/foo/bar]`.
+  the last test. If the response is JSON then order is ignored. If you add
+  `// TEST[continued]` to the snippet after `// TESTRESPONSE` it will continue
+  in the same test, allowing you to interleave requests with responses to check.
+  * `// TESTRESPONSE[s/foo/bar/]`: Substitutions. See `// TEST[s/foo/bar]` for
+  how it works. These are much more common than `// TEST[s/foo/bar]` because
+  they are useful for eliding portions of the response that are not pertinent
+  to the documentation.
+    * One interesting difference here is that you often want to match against
+    the response from Elasticsearch. To do that you can reference the "body" of
+    the response like this: `// TESTRESPONSE[s/"took": 25/"took": $body.took/]`.
+    Note the `$body` string. This says "I don't expect that 25 number in the
+    response, just match against what is in the response." Instead of writing
+    the path into the response after `$body` you can write `$_path` which
+    "figures out" the path. This is especially useful for making sweeping
+    assertions like "I made up all the numbers in this example, don't compare
+    them" which looks like `// TESTRESPONSE[s/\d+/$body.$_path/]`.
+  * `// TESTRESPONSE[_cat]`: Add substitutions for testing `_cat` responses. Use
+  this after all other substitutions so it doesn't make other substitutions
+  difficult.
 * `// TESTSETUP`: Marks this snippet as the "setup" for all other snippets in
   this file. This is a somewhat natural way of structuring documentation. You
   say "this is the data we use to explain this feature" then you add the
diff --git a/docs/Versions.asciidoc b/docs/Versions.asciidoc
new file mode 100644
index 0000000000000..024225f945e80
--- /dev/null
+++ b/docs/Versions.asciidoc
@@ -0,0 +1,56 @@
+:version:               5.6.3
+:major-version:         5.x
+:lucene_version:        6.6.1
+:lucene_version_path:   6_6_1
+:branch:                5.6
+:jdk:                   1.8.0_131
+
+//////////
+release-state can be: released | prerelease | unreleased
+//////////
+
+:release-state:   released
+
+:issue:           https://github.com/elastic/elasticsearch/issues/
+:pull:            https://github.com/elastic/elasticsearch/pull/
+
+:docker-image:    docker.elastic.co/elasticsearch/elasticsearch:{version}
+:plugin_url:      https://artifacts.elastic.co/downloads/elasticsearch-plugins
+
+///////
+Javadoc roots used to generate links from Painless's API reference
+///////
+:java8-javadoc: https://docs.oracle.com/javase/8/docs/api
+:java9-javadoc: http://download.java.net/java/jigsaw/docs/api
+:joda-time-javadoc: http://www.joda.org/joda-time/apidocs
+:lucene-core-javadoc: http://lucene.apache.org/core/{lucene_version_path}/core
+
+ifeval::["{release-state}"=="unreleased"]
+:elasticsearch-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/elasticsearch/{version}-SNAPSHOT
+:transport-client-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/client/transport/{version}-SNAPSHOT
+:rest-client-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/client/elasticsearch-rest-client/{version}-SNAPSHOT
+:rest-client-sniffer-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/client/elasticsearch-rest-client-sniffer/{version}-SNAPSHOT
+:rest-high-level-client-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/client/elasticsearch-rest-high-level-client/{version}-SNAPSHOT
+:painless-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/painless/lang-painless/{version}-SNAPSHOT
+:parent-join-client-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/plugin/parent-join-client/{version}-SNAPSHOT
+:percolator-client-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/plugin/percolator-client/{version}-SNAPSHOT
+:matrixstats-client-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/plugin/aggs-matrix-stats-client/{version}-SNAPSHOT
+endif::[]
+
+ifeval::["{release-state}"!="unreleased"]
+:elasticsearch-javadoc: https://artifacts.elastic.co/javadoc/org/elasticsearch/elasticsearch/{version}
+:transport-client-javadoc: https://artifacts.elastic.co/javadoc/org/elasticsearch/client/transport/{version}
+:rest-client-javadoc: https://artifacts.elastic.co/javadoc/org/elasticsearch/client/elasticsearch-rest-client/{version}
+:rest-client-sniffer-javadoc: https://artifacts.elastic.co/javadoc/org/elasticsearch/client/elasticsearch-rest-client-sniffer/{version}
+:rest-high-level-client-javadoc: https://artifacts.elastic.co/javadoc/org/elasticsearch/client/elasticsearch-rest-high-level-client/{version}
+:painless-javadoc: https://artifacts.elastic.co/javadoc/org/elasticsearch/painless/lang-painless/{version}
+:parent-join-client-javadoc: https://artifacts.elastic.co/javadoc/org/elasticsearch/plugin/parent-join-client/{version}
+:percolator-client-javadoc: https://artifacts.elastic.co/javadoc/org/elasticsearch/plugin/percolator-client/{version}
+:matrixstats-client-javadoc: https://artifacts.elastic.co/javadoc/org/elasticsearch/plugin/aggs-matrix-stats-client/{version}
+endif::[]
+
+///////
+Shared attribute values are pulled from elastic/docs
+///////
+
+include::{asciidoc-dir}/../../shared/attributes.asciidoc[]
diff --git a/docs/build.gradle b/docs/build.gradle
index d930dfb5b6083..c7a0213e41bae 100644
--- a/docs/build.gradle
+++ b/docs/build.gradle
@@ -19,21 +19,93 @@
 
 apply plugin: 'elasticsearch.docs-test'
 
-integTest {
-  cluster {
-    setting 'script.inline', 'true'
-    setting 'script.stored', 'true'
-    setting 'script.max_compilations_per_minute', '1000'
-    Closure configFile = {
-      extraConfigFile it, "src/test/cluster/config/$it"
-    }
-    configFile 'scripts/my_script.js'
-    configFile 'scripts/my_script.py'
-    configFile 'userdict_ja.txt'
-    configFile 'KeywordTokenizer.rbbi'
-    // Whitelist reindexing from the local node so we can test it.
-    setting 'reindex.remote.whitelist', 'myself'
+/* List of files that have snippets that probably should be converted to
+ * `// CONSOLE` and `// TESTRESPONSE` but have yet to be converted. Try and
+ * only remove entries from this list. When it is empty we'll remove it
+ * entirely and have a party! There will be cake and everything.... */
+buildRestTests.expectedUnconvertedCandidates = [
+  'reference/aggregations/bucket/iprange-aggregation.asciidoc',
+  'reference/aggregations/bucket/nested-aggregation.asciidoc',
+  'reference/aggregations/bucket/range-aggregation.asciidoc',
+  'reference/aggregations/bucket/reverse-nested-aggregation.asciidoc',
+  'reference/aggregations/bucket/significantterms-aggregation.asciidoc',
+  'reference/aggregations/bucket/terms-aggregation.asciidoc',
+  'reference/aggregations/matrix/stats-aggregation.asciidoc',
+  'reference/aggregations/metrics/extendedstats-aggregation.asciidoc',
+  'reference/aggregations/metrics/percentile-aggregation.asciidoc',
+  'reference/aggregations/metrics/percentile-rank-aggregation.asciidoc',
+  'reference/aggregations/metrics/scripted-metric-aggregation.asciidoc',
+  'reference/aggregations/metrics/stats-aggregation.asciidoc',
+  'reference/aggregations/metrics/tophits-aggregation.asciidoc',
+  'reference/aggregations/pipeline.asciidoc',
+  'reference/aggregations/pipeline/avg-bucket-aggregation.asciidoc',
+  'reference/aggregations/pipeline/bucket-script-aggregation.asciidoc',
+  'reference/aggregations/pipeline/bucket-selector-aggregation.asciidoc',
+  'reference/aggregations/pipeline/cumulative-sum-aggregation.asciidoc',
+  'reference/aggregations/pipeline/derivative-aggregation.asciidoc',
+  'reference/aggregations/pipeline/extended-stats-bucket-aggregation.asciidoc',
+  'reference/aggregations/pipeline/max-bucket-aggregation.asciidoc',
+  'reference/aggregations/pipeline/min-bucket-aggregation.asciidoc',
+  'reference/aggregations/pipeline/movavg-aggregation.asciidoc',
+  'reference/aggregations/pipeline/percentiles-bucket-aggregation.asciidoc',
+  'reference/aggregations/pipeline/serial-diff-aggregation.asciidoc',
+  'reference/aggregations/pipeline/stats-bucket-aggregation.asciidoc',
+  'reference/aggregations/pipeline/sum-bucket-aggregation.asciidoc',
+  'reference/cluster/allocation-explain.asciidoc',
+  'reference/cluster/nodes-info.asciidoc',
+  'reference/cluster/pending.asciidoc',
+  'reference/cluster/state.asciidoc',
+  'reference/cluster/stats.asciidoc',
+  'reference/cluster/tasks.asciidoc',
+  'reference/docs/delete-by-query.asciidoc',
+  'reference/docs/reindex.asciidoc',
+  'reference/docs/update-by-query.asciidoc',
+  'reference/index-modules/similarity.asciidoc',
+  'reference/index-modules/store.asciidoc',
+  'reference/index-modules/translog.asciidoc',
+  'reference/indices/flush.asciidoc',
+  'reference/indices/get-settings.asciidoc',
+  'reference/indices/put-mapping.asciidoc',
+  'reference/indices/recovery.asciidoc',
+  'reference/indices/segments.asciidoc',
+  'reference/indices/shard-stores.asciidoc',
+  'reference/migration/migrate_5_0/docs.asciidoc',
+  'reference/migration/migrate_5_0/mapping.asciidoc',
+  'reference/migration/migrate_5_0/scripting.asciidoc',
+  'reference/release-notes/5.0.0-alpha5.asciidoc',
+  'reference/query-dsl/exists-query.asciidoc',
+  'reference/search/field-stats.asciidoc',
+  'reference/search/profile.asciidoc'
+]
+
+integTestCluster {
+  setting 'script.inline', 'true'
+  setting 'script.stored', 'true'
+  setting 'script.max_compilations_per_minute', '1000'
+  /* Enable regexes in painless so our tests don't complain about example
+   * snippets that use them. */
+  setting 'script.painless.regex.enabled', 'true'
+  Closure configFile = {
+    extraConfigFile it, "src/test/cluster/config/$it"
   }
+  configFile 'scripts/calculate_score.painless'
+  configFile 'scripts/my_script.js'
+  configFile 'scripts/my_script.py'
+  configFile 'scripts/my_script.painless'
+  configFile 'scripts/my_init_script.painless'
+  configFile 'scripts/my_map_script.painless'
+  configFile 'scripts/my_combine_script.painless'
+  configFile 'scripts/my_reduce_script.painless'
+  configFile 'analysis/example_word_list.txt'
+  configFile 'analysis/hyphenation_patterns.xml'
+  configFile 'analysis/synonym.txt'
+  configFile 'analysis/stemmer_override.txt'
+  configFile 'userdict_ja.txt'
+  configFile 'KeywordTokenizer.rbbi'
+  extraConfigFile 'hunspell/en_US/en_US.aff', '../core/src/test/resources/indices/analyze/conf_dir/hunspell/en_US/en_US.aff'
+  extraConfigFile 'hunspell/en_US/en_US.dic', '../core/src/test/resources/indices/analyze/conf_dir/hunspell/en_US/en_US.dic'
+  // Whitelist reindexing from the local node so we can test it.
+  setting 'reindex.remote.whitelist', '127.0.0.1:*'
 }
 
 // Build the cluser with all plugins
@@ -45,10 +117,8 @@ project.rootProject.subprojects.findAll { it.parent.path == ':plugins' }.each {
     return
   }
   subproj.afterEvaluate { // need to wait until the project has been configured
-    integTest {
-      cluster {
-        plugin subproj.path
-      }
+    integTestCluster {
+      plugin subproj.path
     }
   }
 }
@@ -65,18 +135,28 @@ buildRestTests.docs = fileTree(projectDir) {
 Closure setupTwitter = { String name, int count ->
   buildRestTests.setups[name] = '''
   - do:
-    indices.create:
-      index: twitter
-      body:
-        settings:
-          number_of_shards: 1
-          number_of_replicas: 1
-  - do:
-    bulk:
-      index: twitter
-      type: tweet
-      refresh: true
-      body: |'''
+        indices.create:
+          index: twitter
+          body:
+            settings:
+              number_of_shards: 1
+              number_of_replicas: 1
+            mappings:
+              tweet:
+                properties:
+                  user:
+                    type: keyword
+                    doc_values: true
+                  date:
+                    type: date
+                  likes:
+                    type: long
+  - do:
+        bulk:
+          index: twitter
+          type: tweet
+          refresh: true
+          body: |'''
     for (int i = 0; i < count; i++) {
       String user, text
       if (i == 0) {
@@ -87,8 +167,8 @@ Closure setupTwitter = { String name, int count ->
         text = "some message with the number $i"
       }
       buildRestTests.setups[name] += """
-        {"index":{"_id": "$i"}}
-        {"user": "$user", "message": "$text", "date": "2009-11-15T14:12:12", "likes": $i}"""
+            {"index":{"_id": "$i"}}
+            {"user": "$user", "message": "$text", "date": "2009-11-15T14:12:12", "likes": $i}"""
     }
   }
 setupTwitter('twitter', 5)
@@ -107,37 +187,214 @@ buildRestTests.setups['host'] = '''
   - set: {nodes.$master.http.publish_address: host}
 '''
 
-// Used by pipeline aggregation docs
+// Used by scripted metric docs
+buildRestTests.setups['ledger'] = '''
+  - do:
+        indices.create:
+          index: ledger
+          body:
+            settings:
+              number_of_shards: 2
+              number_of_replicas: 1
+            mappings:
+              sale:
+                properties:
+                  type:
+                    type: keyword
+                  amount:
+                    type: double
+  - do:
+        bulk:
+          index: ledger
+          type: item
+          refresh: true
+          body: |
+            {"index":{}}
+            {"date": "2015/01/01 00:00:00", "amount": 200, "type": "sale", "description": "something"}
+            {"index":{}}
+            {"date": "2015/01/01 00:00:00", "amount": 10, "type": "expense", "decription": "another thing"}
+            {"index":{}}
+            {"date": "2015/01/01 00:00:00", "amount": 150, "type": "sale", "description": "blah"}
+            {"index":{}}
+            {"date": "2015/01/01 00:00:00", "amount": 50, "type": "expense", "description": "cost of blah"}
+            {"index":{}}
+            {"date": "2015/01/01 00:00:00", "amount": 50, "type": "expense", "description": "advertisement"}'''
+
+// Used by aggregation docs
 buildRestTests.setups['sales'] = '''
   - do:
-    indices.create:
-      index: sales
-      body:
-        settings:
-          number_of_shards: 2
-          number_of_replicas: 1
-        mappings:
-          sale:
-            properties:
-              type:
-                type: keyword
-  - do:
-    bulk:
-      index: sales
-      type: sale
-      refresh: true
-      body: |
-        {"index":{}}
-        {"date": "2015/01/01 00:00:00", "price": 200, "type": "hat"}
-        {"index":{}}
-        {"date": "2015/01/01 00:00:00", "price": 200, "type": "t-shirt"}
-        {"index":{}}
-        {"date": "2015/01/01 00:00:00", "price": 150, "type": "bag"}
-        {"index":{}}
-        {"date": "2015/02/01 00:00:00", "price": 50, "type": "hat"}
-        {"index":{}}
-        {"date": "2015/02/01 00:00:00", "price": 10, "type": "t-shirt"}
-        {"index":{}}
-        {"date": "2015/03/01 00:00:00", "price": 200, "type": "hat"}
-        {"index":{}}
-        {"date": "2015/03/01 00:00:00", "price": 175, "type": "t-shirt"}'''
+        indices.create:
+          index: sales
+          body:
+            settings:
+              number_of_shards: 2
+              number_of_replicas: 1
+            mappings:
+              sale:
+                properties:
+                  type:
+                    type: keyword
+  - do:
+        bulk:
+          index: sales
+          type: sale
+          refresh: true
+          body: |
+            {"index":{}}
+            {"date": "2015/01/01 00:00:00", "price": 200, "promoted": true, "rating": 1, "type": "hat"}
+            {"index":{}}
+            {"date": "2015/01/01 00:00:00", "price": 200, "promoted": true, "rating": 1, "type": "t-shirt"}
+            {"index":{}}
+            {"date": "2015/01/01 00:00:00", "price": 150, "promoted": true, "rating": 5, "type": "bag"}
+            {"index":{}}
+            {"date": "2015/02/01 00:00:00", "price": 50, "promoted": false, "rating": 1, "type": "hat"}
+            {"index":{}}
+            {"date": "2015/02/01 00:00:00", "price": 10, "promoted": true, "rating": 4, "type": "t-shirt"}
+            {"index":{}}
+            {"date": "2015/03/01 00:00:00", "price": 200, "promoted": true, "rating": 1, "type": "hat"}
+            {"index":{}}
+            {"date": "2015/03/01 00:00:00", "price": 175, "promoted": false, "rating": 2, "type": "t-shirt"}'''
+
+// Dummy bank account data used by getting-started.asciidoc
+buildRestTests.setups['bank'] = '''
+  - do:
+        bulk:
+          index: bank
+          type: account
+          refresh: true
+          body: |
+#bank_data#
+'''
+/* Load the actual accounts only if we're going to use them. This complicates
+ * dependency checking but that is a small price to pay for not building a
+ * 400kb string every time we start the build. */
+File accountsFile = new File("$projectDir/src/test/resources/accounts.json")
+buildRestTests.inputs.file(accountsFile)
+buildRestTests.doFirst {
+  String accounts = accountsFile.getText('UTF-8')
+  // Indent like a yaml test needs
+  accounts = accounts.replaceAll('(?m)^', '            ')
+  buildRestTests.setups['bank'] =
+    buildRestTests.setups['bank'].replace('#bank_data#', accounts)
+}
+
+buildRestTests.setups['range_index'] = '''
+  - do :
+        indices.create:
+          index: range_index
+          body:
+            settings:
+              number_of_shards: 2
+              number_of_replicas: 1
+            mappings:
+              my_type:
+                properties:
+                  expected_attendees:
+                    type: integer_range
+                  time_frame:
+                    type: date_range
+                    format: yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis
+  - do:
+        bulk:
+          index: range_index
+          type: my_type
+          refresh: true
+          body: |
+            {"index":{"_id": 1}}
+            {"expected_attendees": {"gte": 10, "lte": 20}, "time_frame": {"gte": "2015-10-31 12:00:00", "lte": "2015-11-01"}}'''
+
+// Used by index boost doc
+buildRestTests.setups['index_boost'] = '''
+  - do:
+      indices.create:
+          index:  index1
+  - do:
+      indices.create:
+          index:  index2
+
+  - do:
+      indices.put_alias:
+        index: index1
+        name: alias1
+'''
+// Used by sampler and diversified-sampler aggregation docs
+buildRestTests.setups['stackoverflow'] = '''
+  - do:
+        indices.create:
+          index: stackoverflow
+          body:
+            settings:
+              number_of_shards: 1
+              number_of_replicas: 1
+            mappings:
+              question:
+                properties:
+                  author:
+                    type: keyword
+                  tags:
+                    type: keyword
+  - do:
+        bulk:
+          index: stackoverflow
+          type: question
+          refresh: true
+          body: |'''
+
+// Make Kibana strongly connected to elasticsearch and logstash
+// Make Kibana rarer (and therefore higher-ranking) than Javascript
+// Make Javascript strongly connected to jquery and angular
+// Make Cabana strongly connected to elasticsearch but only as a result of a single author
+
+for (int i = 0; i < 150; i++) {
+  buildRestTests.setups['stackoverflow'] += """
+            {"index":{}}
+            {"author": "very_relevant_$i", "tags": ["elasticsearch", "kibana"]}"""
+}
+for (int i = 0; i < 50; i++) {
+  buildRestTests.setups['stackoverflow'] += """
+            {"index":{}}
+            {"author": "very_relevant_$i", "tags": ["logstash", "kibana"]}"""
+}
+for (int i = 0; i < 200; i++) {
+  buildRestTests.setups['stackoverflow'] += """
+            {"index":{}}
+            {"author": "partially_relevant_$i", "tags": ["javascript", "jquery"]}"""
+}
+for (int i = 0; i < 200; i++) {
+  buildRestTests.setups['stackoverflow'] += """
+            {"index":{}}
+            {"author": "partially_relevant_$i", "tags": ["javascript", "angular"]}"""
+}
+for (int i = 0; i < 50; i++) {
+  buildRestTests.setups['stackoverflow'] += """
+            {"index":{}}
+            {"author": "noisy author", "tags": ["elasticsearch", "cabana"]}"""
+}
+buildRestTests.setups['stackoverflow'] += """
+"""
+
+// Used by some aggregations
+buildRestTests.setups['exams'] = '''
+  - do:
+        indices.create:
+          index: exams
+          body:
+            settings:
+              number_of_shards: 1
+              number_of_replicas: 1
+            mappings:
+              exam:
+                properties:
+                  grade:
+                    type: byte
+  - do:
+        bulk:
+          index: exams
+          type: exam
+          refresh: true
+          body: |
+            {"index":{}}
+            {"grade": 100}
+            {"index":{}}
+            {"grade": 50}'''
+
diff --git a/docs/community-clients/index.asciidoc b/docs/community-clients/index.asciidoc
index 707d976536b74..db35064170b8b 100644
--- a/docs/community-clients/index.asciidoc
+++ b/docs/community-clients/index.asciidoc
@@ -6,7 +6,7 @@ Besides the link:/guide[officially supported Elasticsearch clients], there are
 a number of clients that have been contributed by the community for various languages:
 
 * <<clojure>>
-* <<cold-fusion>>
+* <<coldfusion>>
 * <<erlang>>
 * <<go>>
 * <<groovy>>
@@ -14,6 +14,7 @@ a number of clients that have been contributed by the community for various lang
 * <<java>>
 * <<javascript>>
 * <<kotlin>>
+* <<lua>>
 * <<dotnet>>
 * <<ocaml>>
 * <<perl>>
@@ -32,13 +33,16 @@ a number of clients that have been contributed by the community for various lang
 * http://github.com/clojurewerkz/elastisch[Elastisch]:
   Clojure client.
 
-[[cold-fusion]]
-== Cold Fusion
+[[coldfusion]]
+== ColdFusion (CFML)
+
+* https://www.forgebox.io/view/cbelasticsearch[cbElasticSearch]
+  Native ColdFusion (CFML) support for the ColdBox MVC Platform which provides you with a fluent search interface for Elasticsearch, in addition to a CacheBox Cache provider and a Logbox Appender for logging.
 
 The following project appears to be abandoned:
 
 * https://github.com/jasonfill/ColdFusion-ElasticSearch-Client[ColdFusion-Elasticsearch-Client]
-  Cold Fusion client for Elasticsearch
+  ColdFusion client for Elasticsearch
 
 [[erlang]]
 == Erlang
@@ -113,6 +117,12 @@ The following project appears to be abandoned:
 * https://github.com/mbuhot/eskotlin[ES Kotlin]:
   Elasticsearch Query DSL for kotlin based on the {client}/java-api/current/index.html[official Elasticsearch Java client].
 
+[[lua]]
+== Lua
+
+* https://github.com/DhavalKapil/elasticsearch-lua[elasticsearch-lua]:
+  Lua client for elasticsearch
+
 [[dotnet]]
 == .NET
 
@@ -174,14 +184,17 @@ The following projects appear to be abandoned:
 
 [[r]]
 == R
-* https://github.com/Tomesch/elasticsearch[elasticsearch]
-  R client for Elasticsearch
 
 * https://github.com/ropensci/elastic[elastic]:
   A low-level R client for Elasticsearch.
 
 * https://github.com/ropensci/elasticdsl[elasticdsl]:
   A high-level R DSL for Elasticsearch, wrapping the elastic R client.
+  
+The following projects appear to be abandoned:
+
+* https://github.com/Tomesch/elasticsearch[elasticsearch]
+  R client for Elasticsearch
 
 [[ruby]]
 == Ruby
@@ -216,16 +229,19 @@ The following projects appear to be abandoned:
 * https://github.com/sksamuel/elastic4s[elastic4s]:
   Scala DSL.
 
-* https://github.com/scalastuff/esclient[esclient]:
-  Thin Scala client.
-
 * https://github.com/gphat/wabisabi[wabisabi]:
   Asynchronous REST API Scala client.
+  
+* https://github.com/workday/escalar[escalar]:
+  Type-safe Scala wrapper for the REST API.
 
 * https://github.com/SumoLogic/elasticsearch-client[elasticsearch-client]:
   Scala DSL that uses the REST API. Akka and AWS helpers included.
 
-The following project appears to be abandoned:
+The following projects appear to be abandoned:
+
+* https://github.com/scalastuff/esclient[esclient]:
+  Thin Scala client.
 
 * https://github.com/bsadeh/scalastic[scalastic]:
   Scala client.
diff --git a/docs/groovy-api/delete.asciidoc b/docs/groovy-api/delete.asciidoc
index 18f11e67775a1..790f3ff7f6110 100644
--- a/docs/groovy-api/delete.asciidoc
+++ b/docs/groovy-api/delete.asciidoc
@@ -2,7 +2,7 @@
 == Delete API
 
 The delete API is very similar to the
-// {java}/java-docs-delete.html[]
+// {javaclient}/java-docs-delete.html[]
 Java delete API, here is an
 example:
 
diff --git a/docs/groovy-api/get.asciidoc b/docs/groovy-api/get.asciidoc
index 824c18f8f11e4..7bb8ffe46b08a 100644
--- a/docs/groovy-api/get.asciidoc
+++ b/docs/groovy-api/get.asciidoc
@@ -2,7 +2,7 @@
 == Get API
 
 The get API is very similar to the
-// {java}/java-docs-get.html[]
+// {javaclient}/java-docs-get.html[]
 Java get API. The main benefit
 of using groovy is handling the source content. It can be automatically
 converted to a `Map` which means using Groovy to navigate it is simple:
diff --git a/docs/groovy-api/index.asciidoc b/docs/groovy-api/index.asciidoc
index 3ed4ff9e2e64b..dd819c07c1e73 100644
--- a/docs/groovy-api/index.asciidoc
+++ b/docs/groovy-api/index.asciidoc
@@ -1,7 +1,6 @@
 = Groovy API
-:ref:  http://www.elastic.co/guide/en/elasticsearch/reference/current
-:java: http://www.elastic.co/guide/en/elasticsearch/client/java-api/current
-:version: 5.0.0-alpha5
+
+include::../Versions.asciidoc[]
 
 [preface]
 == Preface
@@ -13,7 +12,7 @@ asynchronous in nature (they either accept a listener, or return a
 future).
 
 The Groovy API is a wrapper on top of the
-{java}[Java API] exposing it in a groovier
+{javaclient}[Java API] exposing it in a groovier
 manner. The execution options for each API follow a similar manner and
 covered in <<anatomy>>.
 
diff --git a/docs/groovy-api/index_.asciidoc b/docs/groovy-api/index_.asciidoc
index b63a212352a40..b3a96334bcf9e 100644
--- a/docs/groovy-api/index_.asciidoc
+++ b/docs/groovy-api/index_.asciidoc
@@ -2,7 +2,7 @@
 == Index API
 
 The index API is very similar to the
-// {java}/java-docs-index.html[]
+// {javaclient}/java-docs-index.html[]
 Java index API. The Groovy
 extension to it is the ability to provide the indexed source using a
 closure. For example:
diff --git a/docs/groovy-api/search.asciidoc b/docs/groovy-api/search.asciidoc
index d0b74a4d6582e..f0650b8233bf5 100644
--- a/docs/groovy-api/search.asciidoc
+++ b/docs/groovy-api/search.asciidoc
@@ -2,7 +2,7 @@
 == Search API
 
 The search API is very similar to the
-// {java}/java-search.html[]
+// {javaclient}/java-search.html[]
 Java search API. The Groovy
 extension allows to provide the search source to execute as a `Closure`
 including the query itself (similar to GORM criteria builder):
diff --git a/docs/java-api/admin/indices/put-mapping.asciidoc b/docs/java-api/admin/indices/put-mapping.asciidoc
index 9b085975077cc..6c2a5406528bc 100644
--- a/docs/java-api/admin/indices/put-mapping.asciidoc
+++ b/docs/java-api/admin/indices/put-mapping.asciidoc
@@ -60,7 +60,7 @@ You can use the same API to update an existing mapping:
 [source,java]
 --------------------------------------------------
 client.admin().indices().preparePutMapping("twitter")   <1>
-        .setType("tweet")                               <2>
+        .setType("user")                                <2>
         .setSource("{\n" +                              <3>
                 "  \"properties\": {\n" +
                 "    \"user_name\": {\n" +
diff --git a/docs/java-api/aggregations/bucket/children-aggregation.asciidoc b/docs/java-api/aggregations/bucket/children-aggregation.asciidoc
index 1bf8a5b26e69e..f6a23fdafe976 100644
--- a/docs/java-api/aggregations/bucket/children-aggregation.asciidoc
+++ b/docs/java-api/aggregations/bucket/children-aggregation.asciidoc
@@ -24,7 +24,7 @@ Import Aggregation definition classes:
 
 [source,java]
 --------------------------------------------------
-import org.elasticsearch.search.aggregations.bucket.children.Children;
+import org.elasticsearch.join.aggregations.Children;
 --------------------------------------------------
 
 [source,java]
diff --git a/docs/java-api/aggregations/bucket/datehistogram-aggregation.asciidoc b/docs/java-api/aggregations/bucket/datehistogram-aggregation.asciidoc
index 9617fbc50f746..99b871730e6d8 100644
--- a/docs/java-api/aggregations/bucket/datehistogram-aggregation.asciidoc
+++ b/docs/java-api/aggregations/bucket/datehistogram-aggregation.asciidoc
@@ -16,7 +16,7 @@ AggregationBuilder aggregation =
         AggregationBuilders
                 .dateHistogram("agg")
                 .field("dateOfBirth")
-                .interval(DateHistogramInterval.YEAR);
+                .dateHistogramInterval(DateHistogramInterval.YEAR);
 --------------------------------------------------
 
 Or if you want to set an interval of 10 days:
@@ -27,7 +27,7 @@ AggregationBuilder aggregation =
         AggregationBuilders
                 .dateHistogram("agg")
                 .field("dateOfBirth")
-                .interval(DateHistogramInterval.days(10));
+                .dateHistogramInterval(DateHistogramInterval.days(10));
 --------------------------------------------------
 
 
diff --git a/docs/java-api/aggregations/bucket/filters-aggregation.asciidoc b/docs/java-api/aggregations/bucket/filters-aggregation.asciidoc
index 1734bff6a8afa..0b782304dacc0 100644
--- a/docs/java-api/aggregations/bucket/filters-aggregation.asciidoc
+++ b/docs/java-api/aggregations/bucket/filters-aggregation.asciidoc
@@ -14,8 +14,9 @@ Here is an example on how to create the aggregation request:
 --------------------------------------------------
 AggregationBuilder aggregation =
     AggregationBuilders
-        .filters("agg", new KeyedFilter("men", QueryBuilders.termQuery("gender", "male")), 
-            new KeyedFilter("women", QueryBuilders.termQuery("gender", "female")));
+        .filters("agg",
+            new FiltersAggregator.KeyedFilter("men", QueryBuilders.termQuery("gender", "male")),
+            new FiltersAggregator.KeyedFilter("women", QueryBuilders.termQuery("gender", "female")));
 --------------------------------------------------
 
 
diff --git a/docs/java-api/aggregations/bucket/histogram-aggregation.asciidoc b/docs/java-api/aggregations/bucket/histogram-aggregation.asciidoc
index bc1803e861708..28e9cd3ecd0ff 100644
--- a/docs/java-api/aggregations/bucket/histogram-aggregation.asciidoc
+++ b/docs/java-api/aggregations/bucket/histogram-aggregation.asciidoc
@@ -36,7 +36,7 @@ Histogram agg = sr.getAggregations().get("agg");
 
 // For each entry
 for (Histogram.Bucket entry : agg.getBuckets()) {
-    Long key = (Long) entry.getKey();       // Key
+    Number key = (Number) entry.getKey();   // Key
     long docCount = entry.getDocCount();    // Doc count
 
     logger.info("key [{}], doc_count [{}]", key, docCount);
diff --git a/docs/java-api/aggregations/bucket/reverse-nested-aggregation.asciidoc b/docs/java-api/aggregations/bucket/reverse-nested-aggregation.asciidoc
index 688e9d7d075c1..635b0e8cf77ee 100644
--- a/docs/java-api/aggregations/bucket/reverse-nested-aggregation.asciidoc
+++ b/docs/java-api/aggregations/bucket/reverse-nested-aggregation.asciidoc
@@ -14,7 +14,7 @@ Here is an example on how to create the aggregation request:
 --------------------------------------------------
 AggregationBuilder aggregation =
     AggregationBuilders
-        .nested("agg").path("resellers")
+        .nested("agg", "resellers")
         .subAggregation(
                 AggregationBuilders
                         .terms("name").field("resellers.name")
diff --git a/docs/java-api/aggregations/metrics/avg-aggregation.asciidoc b/docs/java-api/aggregations/metrics/avg-aggregation.asciidoc
index 74c35c0898e7c..511cbabf5c848 100644
--- a/docs/java-api/aggregations/metrics/avg-aggregation.asciidoc
+++ b/docs/java-api/aggregations/metrics/avg-aggregation.asciidoc
@@ -12,7 +12,7 @@ Here is an example on how to create the aggregation request:
 
 [source,java]
 --------------------------------------------------
-MetricsAggregationBuilder aggregation =
+AvgAggregationBuilder aggregation =
         AggregationBuilders
                 .avg("agg")
                 .field("height");
diff --git a/docs/java-api/aggregations/metrics/cardinality-aggregation.asciidoc b/docs/java-api/aggregations/metrics/cardinality-aggregation.asciidoc
index edde64c1ed2b2..8a854e553f4a3 100644
--- a/docs/java-api/aggregations/metrics/cardinality-aggregation.asciidoc
+++ b/docs/java-api/aggregations/metrics/cardinality-aggregation.asciidoc
@@ -12,7 +12,7 @@ Here is an example on how to create the aggregation request:
 
 [source,java]
 --------------------------------------------------
-MetricsAggregationBuilder aggregation =
+CardinalityAggregationBuilder aggregation =
         AggregationBuilders
                 .cardinality("agg")
                 .field("tags");
diff --git a/docs/java-api/aggregations/metrics/extendedstats-aggregation.asciidoc b/docs/java-api/aggregations/metrics/extendedstats-aggregation.asciidoc
index 20d8db036d90c..8f2f12ede6849 100644
--- a/docs/java-api/aggregations/metrics/extendedstats-aggregation.asciidoc
+++ b/docs/java-api/aggregations/metrics/extendedstats-aggregation.asciidoc
@@ -12,7 +12,7 @@ Here is an example on how to create the aggregation request:
 
 [source,java]
 --------------------------------------------------
-MetricsAggregationBuilder aggregation =
+ExtendedStatsAggregationBuilder aggregation =
         AggregationBuilders
                 .extendedStats("agg")
                 .field("height");
diff --git a/docs/java-api/aggregations/metrics/geobounds-aggregation.asciidoc b/docs/java-api/aggregations/metrics/geobounds-aggregation.asciidoc
index e49962a58f70e..ecffabbd58388 100644
--- a/docs/java-api/aggregations/metrics/geobounds-aggregation.asciidoc
+++ b/docs/java-api/aggregations/metrics/geobounds-aggregation.asciidoc
@@ -1,5 +1,5 @@
 [[java-aggs-metrics-geobounds]]
-==== Cardinality Aggregation
+==== Geo Bounds Aggregation
 
 Here is how you can use
 {ref}/search-aggregations-metrics-geobounds-aggregation.html[Geo Bounds Aggregation]
@@ -13,7 +13,7 @@ Here is an example on how to create the aggregation request:
 [source,java]
 --------------------------------------------------
 GeoBoundsBuilder aggregation =
-        AggregationBuilders
+        GeoBoundsAggregationBuilder
                 .geoBounds("agg")
                 .field("address.location")
                 .wrapLongitude(true);
diff --git a/docs/java-api/aggregations/metrics/max-aggregation.asciidoc b/docs/java-api/aggregations/metrics/max-aggregation.asciidoc
index 765e8c9eaa5d2..9bd393698429b 100644
--- a/docs/java-api/aggregations/metrics/max-aggregation.asciidoc
+++ b/docs/java-api/aggregations/metrics/max-aggregation.asciidoc
@@ -12,7 +12,7 @@ Here is an example on how to create the aggregation request:
 
 [source,java]
 --------------------------------------------------
-MetricsAggregationBuilder aggregation =
+MaxAggregationBuilder aggregation =
         AggregationBuilders
                 .max("agg")
                 .field("height");
diff --git a/docs/java-api/aggregations/metrics/min-aggregation.asciidoc b/docs/java-api/aggregations/metrics/min-aggregation.asciidoc
index 53df26897aa1a..0205cae44d8f8 100644
--- a/docs/java-api/aggregations/metrics/min-aggregation.asciidoc
+++ b/docs/java-api/aggregations/metrics/min-aggregation.asciidoc
@@ -12,7 +12,7 @@ Here is an example on how to create the aggregation request:
 
 [source,java]
 --------------------------------------------------
-MetricsAggregationBuilder aggregation =
+MinAggregationBuilder aggregation =
         AggregationBuilders
                 .min("agg")
                 .field("height");
diff --git a/docs/java-api/aggregations/metrics/percentile-aggregation.asciidoc b/docs/java-api/aggregations/metrics/percentile-aggregation.asciidoc
index a01c291e3ee07..ad54fbf5a46be 100644
--- a/docs/java-api/aggregations/metrics/percentile-aggregation.asciidoc
+++ b/docs/java-api/aggregations/metrics/percentile-aggregation.asciidoc
@@ -12,7 +12,7 @@ Here is an example on how to create the aggregation request:
 
 [source,java]
 --------------------------------------------------
-MetricsAggregationBuilder aggregation =
+PercentilesAggregationBuilder aggregation =
         AggregationBuilders
                 .percentiles("agg")
                 .field("height");
@@ -22,7 +22,7 @@ You can provide your own percentiles instead of using defaults:
 
 [source,java]
 --------------------------------------------------
-MetricsAggregationBuilder aggregation =
+PercentilesAggregationBuilder aggregation =
         AggregationBuilders
                 .percentiles("agg")
                 .field("height")
diff --git a/docs/java-api/aggregations/metrics/percentile-rank-aggregation.asciidoc b/docs/java-api/aggregations/metrics/percentile-rank-aggregation.asciidoc
index 5cb615b8a7675..a846d59f82029 100644
--- a/docs/java-api/aggregations/metrics/percentile-rank-aggregation.asciidoc
+++ b/docs/java-api/aggregations/metrics/percentile-rank-aggregation.asciidoc
@@ -12,11 +12,11 @@ Here is an example on how to create the aggregation request:
 
 [source,java]
 --------------------------------------------------
-MetricsAggregationBuilder aggregation =
+PercentileRanksAggregationBuilder aggregation =
         AggregationBuilders
                 .percentileRanks("agg")
                 .field("height")
-                .percentiles(1.24, 1.91, 2.22);
+                .values(1.24, 1.91, 2.22);
 --------------------------------------------------
 
 
diff --git a/docs/java-api/aggregations/metrics/scripted-metric-aggregation.asciidoc b/docs/java-api/aggregations/metrics/scripted-metric-aggregation.asciidoc
index e9c79ed59d880..3474333066a0a 100644
--- a/docs/java-api/aggregations/metrics/scripted-metric-aggregation.asciidoc
+++ b/docs/java-api/aggregations/metrics/scripted-metric-aggregation.asciidoc
@@ -26,45 +26,33 @@ Here is an example on how to create the aggregation request:
 
 [source,java]
 --------------------------------------------------
-MetricsAggregationBuilder aggregation =
-        AggregationBuilders
-                .scriptedMetric("agg")
-                .initScript("_agg['heights'] = []")
-                .mapScript(new Script("if (doc['gender'].value == \"male\") " +
-                        "{ _agg.heights.add(doc['height'].value) } " +
-                        "else " +
-                        "{ _agg.heights.add(-1 * doc['height'].value) }"));
+ScriptedMetricAggregationBuilder aggregation = AggregationBuilders
+    .scriptedMetric("agg")
+    .initScript(new Script("params._agg.heights = []"))
+    .mapScript(new Script("params._agg.heights.add(doc.gender.value == 'male' ? doc.height.value : -1.0 * doc.height.value)"));
 --------------------------------------------------
 
 You can also specify a `combine` script which will be executed on each shard:
 
 [source,java]
 --------------------------------------------------
-MetricsAggregationBuilder aggregation =
-        AggregationBuilders
-                .scriptedMetric("agg")
-                .initScript(new Script("_agg['heights'] = []"))
-                .mapScript(new Script("if (doc['gender'].value == \"male\") " +
-                        "{ _agg.heights.add(doc['height'].value) } " +
-                        "else " +
-                        "{ _agg.heights.add(-1 * doc['height'].value) }"))
-                .combineScript(new Script("heights_sum = 0; for (t in _agg.heights) { heights_sum += t }; return heights_sum"));
+ScriptedMetricAggregationBuilder aggregation = AggregationBuilders
+    .scriptedMetric("agg")
+    .initScript(new Script("params._agg.heights = []"))
+    .mapScript(new Script("params._agg.heights.add(doc.gender.value == 'male' ? doc.height.value : -1.0 * doc.height.value)"))
+    .combineScript(new Script("double heights_sum = 0.0; for (t in params._agg.heights) { heights_sum += t } return heights_sum"));
 --------------------------------------------------
 
 You can also specify a `reduce` script which will be executed on the node which gets the request:
 
 [source,java]
 --------------------------------------------------
-MetricsAggregationBuilder aggregation =
-        AggregationBuilders
-                .scriptedMetric("agg")
-                .initScript(new Script("_agg['heights'] = []"))
-                .mapScript(new Script("if (doc['gender'].value == \"male\") " +
-                        "{ _agg.heights.add(doc['height'].value) } " +
-                        "else " +
-                        "{ _agg.heights.add(-1 * doc['height'].value) }"))
-                .combineScript(new Script("heights_sum = 0; for (t in _agg.heights) { heights_sum += t }; return heights_sum"))
-                .reduceScript(new Script("heights_sum = 0; for (a in _aggs) { heights_sum += a }; return heights_sum"));
+ScriptedMetricAggregationBuilder aggregation = AggregationBuilders
+    .scriptedMetric("agg")
+    .initScript(new Script("params._agg.heights = []"))
+    .mapScript(new Script("params._agg.heights.add(doc.gender.value == 'male' ? doc.height.value : -1.0 * doc.height.value)"))
+    .combineScript(new Script("double heights_sum = 0.0; for (t in params._agg.heights) { heights_sum += t } return heights_sum"))
+    .reduceScript(new Script("double heights_sum = 0.0; for (a in params._aggs) { heights_sum += a } return heights_sum"));
 --------------------------------------------------
 
 
diff --git a/docs/java-api/aggregations/metrics/stats-aggregation.asciidoc b/docs/java-api/aggregations/metrics/stats-aggregation.asciidoc
index 094c372966b56..260d9c01cb944 100644
--- a/docs/java-api/aggregations/metrics/stats-aggregation.asciidoc
+++ b/docs/java-api/aggregations/metrics/stats-aggregation.asciidoc
@@ -12,7 +12,7 @@ Here is an example on how to create the aggregation request:
 
 [source,java]
 --------------------------------------------------
-MetricsAggregationBuilder aggregation =
+StatsAggregationBuilder aggregation =
         AggregationBuilders
                 .stats("agg")
                 .field("height");
diff --git a/docs/java-api/aggregations/metrics/sum-aggregation.asciidoc b/docs/java-api/aggregations/metrics/sum-aggregation.asciidoc
index 6aea336ff7206..453616916d755 100644
--- a/docs/java-api/aggregations/metrics/sum-aggregation.asciidoc
+++ b/docs/java-api/aggregations/metrics/sum-aggregation.asciidoc
@@ -12,7 +12,7 @@ Here is an example on how to create the aggregation request:
 
 [source,java]
 --------------------------------------------------
-MetricsAggregationBuilder aggregation =
+SumAggregationBuilder aggregation =
         AggregationBuilders
                 .sum("agg")
                 .field("height");
diff --git a/docs/java-api/aggregations/metrics/valuecount-aggregation.asciidoc b/docs/java-api/aggregations/metrics/valuecount-aggregation.asciidoc
index 1acf7cea481eb..b180d22af33cd 100644
--- a/docs/java-api/aggregations/metrics/valuecount-aggregation.asciidoc
+++ b/docs/java-api/aggregations/metrics/valuecount-aggregation.asciidoc
@@ -12,7 +12,7 @@ Here is an example on how to create the aggregation request:
 
 [source,java]
 --------------------------------------------------
-MetricsAggregationBuilder aggregation =
+ValueCountAggregationBuilder aggregation =
         AggregationBuilders
                 .count("agg")
                 .field("height");
diff --git a/docs/java-api/client.asciidoc b/docs/java-api/client.asciidoc
index dc8558a85f652..d09661a8a87bd 100644
--- a/docs/java-api/client.asciidoc
+++ b/docs/java-api/client.asciidoc
@@ -23,6 +23,10 @@ cluster.
 
 ==============================
 
+WARNING: The `TransportClient` is aimed to be replaced by the Java High Level REST
+Client, which executes HTTP requests instead of serialized Java requests. The
+`TransportClient` will be deprecated in upcoming versions of Elasticsearch and it
+is advised to use the Java High Level REST Client instead.
 
 [[transport-client]]
 === Transport Client
@@ -77,7 +81,7 @@ In order to enable sniffing, set `client.transport.sniff` to `true`:
 
 [source,java]
 --------------------------------------------------
-Settings settings = Settings.settingsBuilder()
+Settings settings = Settings.builder()
         .put("client.transport.sniff", true).build();
 TransportClient client = new PreBuiltTransportClient(settings);
 --------------------------------------------------
diff --git a/docs/java-api/docs.asciidoc b/docs/java-api/docs.asciidoc
index 9ace406ee42af..c355714bdd636 100644
--- a/docs/java-api/docs.asciidoc
+++ b/docs/java-api/docs.asciidoc
@@ -7,6 +7,7 @@ This section describes the following CRUD APIs:
 * <<java-docs-index>>
 * <<java-docs-get>>
 * <<java-docs-delete>>
+* <<java-docs-delete-by-query>>
 * <<java-docs-update>>
 
 .Multi-document APIs
diff --git a/docs/java-api/docs/bulk.asciidoc b/docs/java-api/docs/bulk.asciidoc
index 0b43b89c07cc5..07849164a6806 100644
--- a/docs/java-api/docs/bulk.asciidoc
+++ b/docs/java-api/docs/bulk.asciidoc
@@ -71,7 +71,7 @@ BulkProcessor bulkProcessor = BulkProcessor.builder(
                                   Throwable failure) { ... } <4>
         })
         .setBulkActions(10000) <5>
-        .setBulkSize(new ByteSizeValue(1, ByteSizeUnit.GB)) <6>
+        .setBulkSize(new ByteSizeValue(5, ByteSizeUnit.MB)) <6>
         .setFlushInterval(TimeValue.timeValueSeconds(5)) <7>
         .setConcurrentRequests(1) <8>
         .setBackoffPolicy(
@@ -85,15 +85,26 @@ BulkProcessor bulkProcessor = BulkProcessor.builder(
     with `response.hasFailures()`
 <4> This method is called when the bulk failed and raised a `Throwable`
 <5> We want to execute the bulk every 10 000 requests
-<6> We want to flush the bulk every 1gb
+<6> We want to flush the bulk every 5mb
 <7> We want to flush the bulk every 5 seconds whatever the number of requests
-<8> Set the number of concurrent requests. A value of 0 means that only a single request will  be allowed to be
+<8> Set the number of concurrent requests. A value of 0 means that only a single request will be allowed to be
     executed. A value of 1 means 1 concurrent request is allowed to be executed while accumulating new bulk requests.
 <9> Set a custom backoff policy which will initially wait for 100ms, increase exponentially and retries up to three
     times. A retry is attempted whenever one or more bulk item requests have failed with an `EsRejectedExecutionException`
     which indicates that there were too little compute resources available for processing the request. To disable backoff,
     pass `BackoffPolicy.noBackoff()`.
 
+By default, `BulkProcessor`:
+
+* sets bulkActions to `1000`
+* sets bulkSize to `5mb`
+* does not set flushInterval
+* sets concurrentRequests to 1, which means an asynchronous execution of the flush operation.
+* sets backoffPolicy to an exponential backoff with 8 retries and a start delay of 50ms. The total wait time is roughly 5.1 seconds.
+
+[[java-docs-bulk-processor-requests]]
+==== Add requests
+
 Then you can simply add your requests to the `BulkProcessor`:
 
 [source,java]
@@ -102,13 +113,8 @@ bulkProcessor.add(new IndexRequest("twitter", "tweet", "1").source(/* your doc h
 bulkProcessor.add(new DeleteRequest("twitter", "tweet", "2"));
 --------------------------------------------------
 
-By default, `BulkProcessor`:
-
-* sets bulkActions to `1000`
-* sets bulkSize to `5mb`
-* does not set flushInterval
-* sets concurrentRequests to 1
-* sets backoffPolicy to an exponential backoff with 8 retries and a start delay of 50ms. The total wait time is roughly 5.1 seconds.
+[[java-docs-bulk-processor-close]]
+==== Closing the Bulk Processor
 
 When all documents are loaded to the `BulkProcessor` it can be closed by using `awaitClose` or `close` methods:
 
@@ -129,3 +135,33 @@ Both methods flush any remaining documents and disable all other scheduled flush
 all bulk requests to complete then returns `true`, if the specified waiting time elapses before all bulk requests complete,
 `false` is returned. The `close` method doesn't wait for any remaining bulk requests to complete and exits immediately.
 
+[[java-docs-bulk-processor-tests]]
+==== Using Bulk Processor in tests
+
+If you are running tests with elasticsearch and are using the `BulkProcessor` to populate your dataset
+you should better set the number of concurrent requests to `0` so the flush operation of the bulk will be executed
+in a synchronous manner:
+
+[source,java]
+--------------------------------------------------
+BulkProcessor bulkProcessor = BulkProcessor.builder(client, new BulkProcessor.Listener() { /* Listener methods */ })
+        .setBulkActions(10000)
+        .setConcurrentRequests(0)
+        .build();
+
+// Add your requests
+bulkProcessor.add(/* Your requests */);
+
+// Flush any remaining requests
+bulkProcessor.flush();
+
+// Or close the bulkProcessor if you don't need it anymore
+bulkProcessor.close();
+
+// Refresh your indices
+client.admin().indices().prepareRefresh().get();
+
+// Now you can start searching!
+client.prepareSearch().get();
+--------------------------------------------------
+
diff --git a/docs/java-api/docs/delete.asciidoc b/docs/java-api/docs/delete.asciidoc
index b9efd01242df5..3e3788cda2472 100644
--- a/docs/java-api/docs/delete.asciidoc
+++ b/docs/java-api/docs/delete.asciidoc
@@ -35,3 +35,48 @@ DeleteResponse response = client.prepareDelete("twitter", "tweet", "1")
         .setOperationThreaded(false)
         .get();
 --------------------------------------------------
+
+[[java-docs-delete-by-query]]
+=== Delete By Query API
+
+The delete by query API allows one to delete a given set of documents based on
+the result of a query:
+
+[source,java]
+--------------------------------------------------
+BulkByScrollResponse response =
+    DeleteByQueryAction.INSTANCE.newRequestBuilder(client)
+        .filter(QueryBuilders.matchQuery("gender", "male")) <1>
+        .source("persons")                                  <2>
+        .get();                                             <3>
+
+long deleted = response.getDeleted();                       <4>
+--------------------------------------------------
+<1> query
+<2> index
+<3> execute the operation
+<4> number of deleted documents
+
+As it can be a long running operation, if you wish to do it asynchronously, you can call `execute` instead of `get`
+and provide a listener like:
+
+[source,java]
+--------------------------------------------------
+DeleteByQueryAction.INSTANCE.newRequestBuilder(client)
+    .filter(QueryBuilders.matchQuery("gender", "male"))                  <1>
+    .source("persons")                                                   <2>
+    .execute(new ActionListener<BulkByScrollResponse>() {           <3>
+        @Override
+        public void onResponse(BulkByScrollResponse response) {
+            long deleted = response.getDeleted();                        <4>
+        }
+        @Override
+        public void onFailure(Exception e) {
+            // Handle the exception
+        }
+    });
+--------------------------------------------------
+<1> query
+<2> index
+<3> listener
+<4> number of deleted documents
diff --git a/docs/java-api/docs/index_.asciidoc b/docs/java-api/docs/index_.asciidoc
index 2b29f15fabd5e..b32955d9d4f50 100644
--- a/docs/java-api/docs/index_.asciidoc
+++ b/docs/java-api/docs/index_.asciidoc
@@ -60,8 +60,9 @@ json.put("message","trying out Elasticsearch");
 [[java-docs-index-generate-beans]]
 ===== Serialize your beans
 
-Elasticsearch already uses http://wiki.fasterxml.com/JacksonHome[Jackson].
-So you can use it to serialize your beans to JSON:
+You can use http://wiki.fasterxml.com/JacksonHome[Jackson] to serialize
+your beans to JSON. Please add http://search.maven.org/#search%7Cga%7C1%7Cjackson-databind[Jackson Databind]
+ to your project. Then you can use `ObjectMapper` to serialize your beans:
 
 [source,java]
 --------------------------------------------------
@@ -139,7 +140,7 @@ String json = "{" +
     "}";
 
 IndexResponse response = client.prepareIndex("twitter", "tweet")
-        .setSource(json)
+        .setSource(json, XContentType.JSON)
         .get();
 --------------------------------------------------
 
@@ -155,8 +156,8 @@ String _type = response.getType();
 String _id = response.getId();
 // Version (if it's the first time you index this document, you will get: 1)
 long _version = response.getVersion();
-// isCreated() is true if the document is a new one, false if it has been updated
-boolean created = response.isCreated();
+// status has stored current instance statement.
+RestStatus status = response.status();
 --------------------------------------------------
 
 For more information on the index operation, check out the REST
diff --git a/docs/java-api/docs/update-by-query.asciidoc b/docs/java-api/docs/update-by-query.asciidoc
index a94899668efa2..8b3d2d71c400a 100644
--- a/docs/java-api/docs/update-by-query.asciidoc
+++ b/docs/java-api/docs/update-by-query.asciidoc
@@ -1,8 +1,6 @@
 [[docs-update-by-query]]
 == Update By Query API
 
-experimental[The update-by-query API is new and should still be considered experimental.  The API may change in ways that are not backwards compatible]
-
 The simplest usage of `updateByQuery` updates each
 document in an index without changing the source. This usage enables
 <<picking-up-a-new-property,picking up a new property>> or another online
@@ -14,7 +12,7 @@ UpdateByQueryRequestBuilder updateByQuery = UpdateByQueryAction.INSTANCE.newRequ
 
 updateByQuery.source("source_index").abortOnVersionConflict(false);
 
-BulkIndexByScrollResponse response = updateByQuery.get();
+BulkByScrollResponse response = updateByQuery.get();
 --------------------------------------------------
 
 Calls to the `updateByQuery` API start by getting a snapshot of the index, indexing
@@ -27,7 +25,7 @@ When the versions match, `updateByQuery` updates the document
 and increments the version number.
 
 All update and query failures cause `updateByQuery` to abort. These failures are
-available from the `BulkIndexByScrollResponse#getIndexingFailures` method. Any
+available from the `BulkByScrollResponse#getIndexingFailures` method. Any
 successful updates remain and are not rolled back. While the first failure
 causes the abort, the response contains all of the failures generated by the
 failed bulk request.
@@ -52,7 +50,7 @@ updateByQuery.source("source_index")
         .size(1000)
         .script(new Script("ctx._source.awesome = 'absolutely'", ScriptType.INLINE, "painless", emptyMap()));
 
-BulkIndexByScrollResponse response = updateByQuery.get();
+BulkByScrollResponse response = updateByQuery.get();
 --------------------------------------------------
 
 `UpdateByQueryRequestBuilder` also enables direct access to the query used
@@ -66,7 +64,7 @@ UpdateByQueryRequestBuilder updateByQuery = UpdateByQueryAction.INSTANCE.newRequ
 updateByQuery.source("source_index")
         .source().setSize(500);
 
-BulkIndexByScrollResponse response = updateByQuery.get();
+BulkByScrollResponse response = updateByQuery.get();
 --------------------------------------------------
 
 You can also combine `size` with sorting to limit the documents updated:
@@ -78,7 +76,7 @@ UpdateByQueryRequestBuilder updateByQuery = UpdateByQueryAction.INSTANCE.newRequ
 updateByQuery.source("source_index").size(100)
         .source().addSort("cat", SortOrder.DESC);
 
-BulkIndexByScrollResponse response = updateByQuery.get();
+BulkByScrollResponse response = updateByQuery.get();
 --------------------------------------------------
 
 In addition to changing the `_source` field for the document, you can use a
@@ -97,7 +95,7 @@ updateByQuery.source("source_index")
           + "} else {"
           + "ctx._source.awesome = 'absolutely'}", ScriptType.INLINE, "painless", emptyMap()));
 
-BulkIndexByScrollResponse response = updateByQuery.get();
+BulkByScrollResponse response = updateByQuery.get();
 --------------------------------------------------
 
 As in the <<docs-update,Update API>>, you can set the value of `ctx.op` to change the
@@ -131,7 +129,7 @@ UpdateByQueryRequestBuilder updateByQuery = UpdateByQueryAction.INSTANCE.newRequ
 
 updateByQuery.source("foo", "bar").source().setTypes("a", "b");
 
-BulkIndexByScrollResponse response = updateByQuery.get();
+BulkByScrollResponse response = updateByQuery.get();
 --------------------------------------------------
 
 If you provide a `routing` value then the process copies the routing value to the scroll query,
@@ -143,7 +141,7 @@ UpdateByQueryRequestBuilder updateByQuery = UpdateByQueryAction.INSTANCE.newRequ
 
 updateByQuery.source().setRouting("cat");
 
-BulkIndexByScrollResponse response = updateByQuery.get();
+BulkByScrollResponse response = updateByQuery.get();
 --------------------------------------------------
 
 `updateByQuery` can also use the <<ingest>> feature by
@@ -155,7 +153,7 @@ UpdateByQueryRequestBuilder updateByQuery = UpdateByQueryAction.INSTANCE.newRequ
 
 updateByQuery.setPipeline("hurray");
 
-BulkIndexByScrollResponse response = updateByQuery.get();
+BulkByScrollResponse response = updateByQuery.get();
 --------------------------------------------------
 
 [float]
diff --git a/docs/java-api/index.asciidoc b/docs/java-api/index.asciidoc
index d8e2d8848bbd3..77ef6087211d4 100644
--- a/docs/java-api/index.asciidoc
+++ b/docs/java-api/index.asciidoc
@@ -1,7 +1,7 @@
 [[java-api]]
 = Java API
-:ref: http://www.elastic.co/guide/en/elasticsearch/reference/master
-:version: 5.0.0-alpha5
+
+include::../Versions.asciidoc[]
 
 [preface]
 == Preface
@@ -17,6 +17,14 @@ Additionally, operations on a client may be accumulated and executed in
 Note, all the APIs are exposed through the
 Java API (actually, the Java API is used internally to execute them).
 
+WARNING: Starting from version 5.6.0, a new Java client has been
+released: the {java-rest}/java-rest-high.html[Java High Level REST Client].
+This new client is designed to replace the `TransportClient` in Java
+applications which will be deprecated in future versions of Elasticsearch.
+
+== Javadoc
+
+The javadoc for the transport client can be found at {transport-client-javadoc}/index.html.
 
 == Maven Repository
 
@@ -35,6 +43,65 @@ For example, you can define the latest version in your `pom.xml` file:
 </dependency>
 --------------------------------------------------
 
+=== Log4j 2 Logger
+
+You need to also include Log4j 2 dependencies:
+
+["source","xml",subs="attributes"]
+--------------------------------------------------
+<dependency>
+    <groupId>org.apache.logging.log4j</groupId>
+    <artifactId>log4j-core</artifactId>
+    <version>2.9.1</version>
+</dependency>
+--------------------------------------------------
+
+And also provide a Log4j 2 configuration file in your classpath.
+For example, you can add in your `src/main/resources` project dir a `log4j2.properties` file like:
+
+
+["source","properties",subs="attributes"]
+--------------------------------------------------
+appender.console.type = Console
+appender.console.name = console
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%m%n
+
+rootLogger.level = info
+rootLogger.appenderRef.console.ref = console
+--------------------------------------------------
+
+=== Using another Logger
+
+If you want to use another logger than Log4j 2, you can use http://www.slf4j.org/[SLF4J] bridge to do that:
+
+["source","xml",subs="attributes"]
+--------------------------------------------------
+<dependency>
+    <groupId>org.apache.logging.log4j</groupId>
+    <artifactId>log4j-to-slf4j</artifactId>
+    <version>2.9.1</version>
+</dependency>
+<dependency>
+    <groupId>org.slf4j</groupId>
+    <artifactId>slf4j-api</artifactId>
+    <version>1.7.24</version>
+</dependency>
+--------------------------------------------------
+
+http://www.slf4j.org/manual.html[This page] lists implementations you can use. Pick your favorite logger
+and add it as a dependency. As an example, we will use the `slf4j-simple` logger:
+
+["source","xml",subs="attributes"]
+--------------------------------------------------
+<dependency>
+    <groupId>org.slf4j</groupId>
+    <artifactId>slf4j-simple</artifactId>
+    <version>1.7.21</version>
+</dependency>
+--------------------------------------------------
+
+
 == Dealing with JAR dependency conflicts
 
 If you want to use Elasticsearch in your Java application, you may have to deal with version conflicts with third party
@@ -90,51 +157,6 @@ it to the `transformers`:
 </transformer>
 --------------------------------------------------
 
-
-== Deploying in JBoss EAP6 module
-
-Elasticsearch and Lucene classes need to be in the same JBoss module.
-
-You should define a `module.xml` file like this:
-
-[source,xml]
---------------------------------------------------
-<?xml version="1.0" encoding="UTF-8"?>
-<module xmlns="urn:jboss:module:1.1" name="org.elasticsearch">
-  <resources>
-    <!-- Elasticsearch -->
-    <resource-root path="elasticsearch-2.0.0.jar"/>
-    <!-- Lucene -->
-    <resource-root path="lucene-core-5.1.0.jar"/>
-    <resource-root path="lucene-analyzers-common-5.1.0.jar"/>
-    <resource-root path="lucene-queries-5.1.0.jar"/>
-    <resource-root path="lucene-memory-5.1.0.jar"/>
-    <resource-root path="lucene-highlighter-5.1.0.jar"/>
-    <resource-root path="lucene-queryparser-5.1.0.jar"/>
-    <resource-root path="lucene-sandbox-5.1.0.jar"/>
-    <resource-root path="lucene-suggest-5.1.0.jar"/>
-    <resource-root path="lucene-misc-5.1.0.jar"/>
-    <resource-root path="lucene-join-5.1.0.jar"/>
-    <resource-root path="lucene-grouping-5.1.0.jar"/>
-    <resource-root path="lucene-spatial-5.1.0.jar"/>
-    <resource-root path="lucene-expressions-5.1.0.jar"/>
-    <!-- Insert other resources here -->
-  </resources>
-
-  <dependencies>
-    <module name="sun.jdk" export="true" >
-        <imports>
-            <include path="sun/misc/Unsafe" />
-        </imports>
-    </module>
-    <module name="org.apache.log4j"/>
-    <module name="org.apache.commons.logging"/>
-    <module name="javax.api"/>
-  </dependencies>
-</module>
---------------------------------------------------
-
-
 include::client.asciidoc[]
 
 include::docs.asciidoc[]
diff --git a/docs/java-api/query-dsl/function-score-query.asciidoc b/docs/java-api/query-dsl/function-score-query.asciidoc
index 0915814ae1b56..9e731edb8ba7b 100644
--- a/docs/java-api/query-dsl/function-score-query.asciidoc
+++ b/docs/java-api/query-dsl/function-score-query.asciidoc
@@ -19,6 +19,7 @@ FilterFunctionBuilder[] functions = {
         new FunctionScoreQueryBuilder.FilterFunctionBuilder(
                 exponentialDecayFunction("age", 0L, 1L))      <3>
 };
+QueryBuilder qb = QueryBuilders.functionScoreQuery(functions);
 --------------------------------------------------
 <1> Add a first function based on a query
 <2> And randomize the score based on a given seed
diff --git a/docs/java-api/query-dsl/fuzzy-query.asciidoc b/docs/java-api/query-dsl/fuzzy-query.asciidoc
index 1ea5983072340..e871bc9d0b4d3 100644
--- a/docs/java-api/query-dsl/fuzzy-query.asciidoc
+++ b/docs/java-api/query-dsl/fuzzy-query.asciidoc
@@ -1,8 +1,6 @@
 [[java-query-dsl-fuzzy-query]]
 ==== Fuzzy Query
 
-deprecated[3.0.0, Will be removed without a replacement for `string` fields. Note that the `fuzziness` parameter is still supported for match queries and in suggesters. Use range queries for `date` and `numeric` fields instead.]
-
 See {ref}/query-dsl-fuzzy-query.html[Fuzzy Query]
 
 [source,java]
diff --git a/docs/java-api/query-dsl/geo-distance-query.asciidoc b/docs/java-api/query-dsl/geo-distance-query.asciidoc
index 4209c329f7c82..d03ba2017f594 100644
--- a/docs/java-api/query-dsl/geo-distance-query.asciidoc
+++ b/docs/java-api/query-dsl/geo-distance-query.asciidoc
@@ -7,15 +7,9 @@ See {ref}/query-dsl-geo-distance-query.html[Geo Distance Query]
 --------------------------------------------------
 QueryBuilder qb = geoDistanceQuery("pin.location")  <1>
     .point(40, -70)                                 <2>
-    .distance(200, DistanceUnit.KILOMETERS)         <3>
-    .optimizeBbox("memory")                         <4>
-    .geoDistance(GeoDistance.ARC);                  <5>
-
+    .distance(200, DistanceUnit.KILOMETERS);        <3>
 --------------------------------------------------
 <1> field
 <2> center point
 <3> distance from center point
-<4> optimize bounding box: `memory`, `indexed` or `none`
-<5> distance computation mode: `GeoDistance.SLOPPY_ARC` (default), `GeoDistance.ARC` (slightly more precise but
-    significantly slower) or `GeoDistance.PLANE` (faster, but inaccurate on long distances and close to the poles)
 
diff --git a/docs/java-api/query-dsl/geo-distance-range-query.asciidoc b/docs/java-api/query-dsl/geo-distance-range-query.asciidoc
deleted file mode 100644
index 1abe02a5e810a..0000000000000
--- a/docs/java-api/query-dsl/geo-distance-range-query.asciidoc
+++ /dev/null
@@ -1,26 +0,0 @@
-[[java-query-dsl-geo-distance-range-query]]
-==== Geo Distance Range Query
-
-See {ref}/query-dsl-geo-distance-range-query.html[Geo Distance Range Query]
-
-[source,java]
---------------------------------------------------
-QueryBuilder qb = geoDistanceRangeQuery("pin.location",         <1>
-        new GeoPoint(40, -70))                                  <2>
-    .from("200km")                                              <3>
-    .to("400km")                                                <4>
-    .includeLower(true)                                         <5>
-    .includeUpper(false)                                        <6>
-    .optimizeBbox("memory")                                     <7>
-    .geoDistance(GeoDistance.ARC);                              <8>
---------------------------------------------------
-<1> field
-<2> center point
-<3> starting distance from center point
-<4> ending distance from center point
-<5> include lower value means that `from` is `gt` when `false` or `gte` when `true`
-<6> include upper value means that `to` is `lt` when `false` or `lte` when `true`
-<7> optimize bounding box: `memory`, `indexed` or `none`
-<8> distance computation mode: `GeoDistance.SLOPPY_ARC` (default), `GeoDistance.ARC` (slightly more precise but
-    significantly slower) or `GeoDistance.PLANE` (faster, but inaccurate on long distances and close to the poles)
-
diff --git a/docs/java-api/query-dsl/geo-polygon-query.asciidoc b/docs/java-api/query-dsl/geo-polygon-query.asciidoc
index 1ee344c3098c1..362e1751bb935 100644
--- a/docs/java-api/query-dsl/geo-polygon-query.asciidoc
+++ b/docs/java-api/query-dsl/geo-polygon-query.asciidoc
@@ -5,7 +5,7 @@ See {ref}/query-dsl-geo-polygon-query.html[Geo Polygon Query]
 
 [source,java]
 --------------------------------------------------
-List<GeoPoint> points = new ArrayList<GeoPoint>();     <1>
+List<GeoPoint> points = new ArrayList<>();             <1>
 points.add(new GeoPoint(40, -70));
 points.add(new GeoPoint(30, -80));
 points.add(new GeoPoint(20, -90));
diff --git a/docs/java-api/query-dsl/geo-queries.asciidoc b/docs/java-api/query-dsl/geo-queries.asciidoc
index 18ed0f37c141a..10df4ff5e8716 100644
--- a/docs/java-api/query-dsl/geo-queries.asciidoc
+++ b/docs/java-api/query-dsl/geo-queries.asciidoc
@@ -21,29 +21,14 @@ The queries in this group are:
     Finds document with geo-points within the specified distance of a central
     point.
 
-<<java-query-dsl-geo-distance-range-query,`geo_distance_range`>> query::
-
-    Like the `geo_point` query, but the range starts at a specified distance
-    from the central point.
-
 <<java-query-dsl-geo-polygon-query,`geo_polygon`>> query::
 
     Find documents with geo-points within the specified polygon.
 
-<<java-query-dsl-geohash-cell-query,`geohash_cell`>> query::
-
-    Find geo-points whose geohash intersects with the geohash of the specified
-    point.
-
-
 include::geo-shape-query.asciidoc[]
 
 include::geo-bounding-box-query.asciidoc[]
 
 include::geo-distance-query.asciidoc[]
 
-include::geo-distance-range-query.asciidoc[]
-
 include::geo-polygon-query.asciidoc[]
-
-include::geohash-cell-query.asciidoc[]
diff --git a/docs/java-api/query-dsl/geo-shape-query.asciidoc b/docs/java-api/query-dsl/geo-shape-query.asciidoc
index e08410acbdb0a..62e8ee199c9e1 100644
--- a/docs/java-api/query-dsl/geo-shape-query.asciidoc
+++ b/docs/java-api/query-dsl/geo-shape-query.asciidoc
@@ -39,31 +39,32 @@ import org.elasticsearch.common.geo.builders.ShapeBuilder;
 
 [source,java]
 --------------------------------------------------
-GeoShapeQueryBuilder qb = geoShapeQuery(
-        "pin.location",                 <1>
-        ShapeBuilder.newMultiPoint()    <2>
-             .point(0, 0)
-             .point(0, 10)
-             .point(10, 10)
-             .point(10, 0)
-             .point(0, 0));
-qb.relation(ShapeRelation.WITHIN);      <3>
+List<Coordinate> points = new ArrayList<>();
+points.add(new Coordinate(0, 0));
+points.add(new Coordinate(0, 10));
+points.add(new Coordinate(10, 10));
+points.add(new Coordinate(10, 0));
+points.add(new Coordinate(0, 0));
 
+QueryBuilder qb = geoShapeQuery(
+        "pin.location",                         <1>
+        ShapeBuilders.newMultiPoint(points)     <2>
+        .relation(ShapeRelation.WITHIN);        <3>
 --------------------------------------------------
 <1> field
 <2> shape
-<3> relation can be `ShapeRelation.WITHIN`, `ShapeRelation.INTERSECTS` or `ShapeRelation.DISJOINT`
+<3> relation can be `ShapeRelation.CONTAINS`, `ShapeRelation.WITHIN`, `ShapeRelation.INTERSECTS` or `ShapeRelation.DISJOINT`
 
 [source,java]
 --------------------------------------------------
 // Using pre-indexed shapes
-GeoShapeQueryBuilder qb = geoShapeQuery(
-        "pin.location",             <1>
-        "DEU",                      <2>
-        "countries");               <3>
-qb.relation(ShapeRelation.WITHIN))  <4>
-    .indexedShapeIndex("shapes")    <5>
-    .indexedShapePath("location");  <6>
+QueryBuilder qb = geoShapeQuery(
+        "pin.location",                  <1>
+        "DEU",                           <2>
+        "countries")                     <3>
+        .relation(ShapeRelation.WITHIN)) <4>
+        .indexedShapeIndex("shapes")     <5>
+        .indexedShapePath("location");   <6>
 --------------------------------------------------
 <1> field
 <2> The ID of the document that containing the pre-indexed shape.
diff --git a/docs/java-api/query-dsl/geohash-cell-query.asciidoc b/docs/java-api/query-dsl/geohash-cell-query.asciidoc
deleted file mode 100644
index 7aad54892a575..0000000000000
--- a/docs/java-api/query-dsl/geohash-cell-query.asciidoc
+++ /dev/null
@@ -1,17 +0,0 @@
-[[java-query-dsl-geohash-cell-query]]
-==== Geohash Cell Query
-
-See {ref}/query-dsl-geohash-cell-query.html[Geohash Cell Query]
-
-[source,java]
---------------------------------------------------
-QueryBuilder qb = geoHashCellQuery("pin.location",  <1>
-            new GeoPoint(13.4080, 52.5186))         <2>
-        .neighbors(true)                            <3>
-        .precision(3);                              <4>
---------------------------------------------------
-<1> field
-<2> point. Can also be a hash like `u30`
-<3> The `neighbors` option of the filter offers the possibility to filter cells
-    next to the given cell.
-<4> precision level
diff --git a/docs/java-api/query-dsl/has-child-query.asciidoc b/docs/java-api/query-dsl/has-child-query.asciidoc
index 755bb9a820f1d..4402916c1394e 100644
--- a/docs/java-api/query-dsl/has-child-query.asciidoc
+++ b/docs/java-api/query-dsl/has-child-query.asciidoc
@@ -3,13 +3,26 @@
 
 See {ref}/query-dsl-has-child-query.html[Has Child Query]
 
+When using the `has_child` query it is important to use the `PreBuiltTransportClient` instead of the regular client:
+
+[source,java]
+--------------------------------------------------
+Settings settings = Settings.builder().put("cluster.name", "elasticsearch").build();
+TransportClient client = new PreBuiltTransportClient(settings);
+client.addTransportAddress(new InetSocketTransportAddress(new InetSocketAddress(InetAddresses.forString("127.0.0.1"), 9300)));
+--------------------------------------------------
+
+Otherwise the parent-join module doesn't get loaded and the `has_child` query can't be used from the transport client.
+
 [source,java]
 --------------------------------------------------
-QueryBuilder qb = hasChildQuery(
+QueryBuilder qb = JoinQueryBuilders.hasChildQuery(
     "blog_tag",                     <1>
-    termQuery("tag","something")    <2>
+    termQuery("tag","something"),   <2>
+    ScoreMode.Avg                   <3>
 );
 --------------------------------------------------
 <1> child type to query against
 <2> query
+<3> score mode can be `ScoreMode.Avg`, `ScoreMode.Max`, `ScoreMode.Min`, `ScoreMode.None` or `ScoreMode.Total`
 
diff --git a/docs/java-api/query-dsl/has-parent-query.asciidoc b/docs/java-api/query-dsl/has-parent-query.asciidoc
index 7c984346265cb..77047ba3ea5d2 100644
--- a/docs/java-api/query-dsl/has-parent-query.asciidoc
+++ b/docs/java-api/query-dsl/has-parent-query.asciidoc
@@ -3,12 +3,25 @@
 
 See {ref}/query-dsl-has-parent-query.html[Has Parent]
 
+When using the `has_parent` query it is important to use the `PreBuiltTransportClient` instead of the regular client:
+
+[source,java]
+--------------------------------------------------
+Settings settings = Settings.builder().put("cluster.name", "elasticsearch").build();
+TransportClient client = new PreBuiltTransportClient(settings);
+client.addTransportAddress(new InetSocketTransportAddress(new InetSocketAddress(InetAddresses.forString("127.0.0.1"), 9300)));
+--------------------------------------------------
+
+Otherwise the parent-join module doesn't get loaded and the `has_parent` query can't be used from the transport client.
+
 [source,java]
 --------------------------------------------------
-QueryBuilder qb = hasParentQuery(
+QueryBuilder qb = JoinQueryBuilders.hasParentQuery(
     "blog",                         <1>
-    termQuery("tag","something")    <2>
+    termQuery("tag","something"),   <2>
+    false                           <3>
 );
 --------------------------------------------------
 <1> parent type to query against
 <2> query
+<3> whether the score from the parent hit should propagate to the child hit
diff --git a/docs/java-api/query-dsl/indices-query.asciidoc b/docs/java-api/query-dsl/indices-query.asciidoc
index 4667c5c62ffdd..6f923ab1ff7c2 100644
--- a/docs/java-api/query-dsl/indices-query.asciidoc
+++ b/docs/java-api/query-dsl/indices-query.asciidoc
@@ -1,6 +1,8 @@
 [[java-query-dsl-indices-query]]
 ==== Indices Query
 
+deprecated[5.0.0, Search on the '_index' field instead]
+
 See {ref}/query-dsl-indices-query.html[Indices Query]
 
 [source,java]
diff --git a/docs/java-api/query-dsl/nested-query.asciidoc b/docs/java-api/query-dsl/nested-query.asciidoc
index 69fa1082c1987..d6da597eecdd2 100644
--- a/docs/java-api/query-dsl/nested-query.asciidoc
+++ b/docs/java-api/query-dsl/nested-query.asciidoc
@@ -9,10 +9,10 @@ QueryBuilder qb = nestedQuery(
         "obj1",                       <1>
         boolQuery()                   <2>
                 .must(matchQuery("obj1.name", "blue"))
-                .must(rangeQuery("obj1.count").gt(5))
-    )
-    .scoreMode(ScoreMode.Avg);        <3>
+                .must(rangeQuery("obj1.count").gt(5)),
+        ScoreMode.Avg                 <3>
+    );
 --------------------------------------------------
 <1> path to nested document
 <2> your query. Any fields referenced inside the query must use the complete path (fully qualified).
-<3> score mode could be `max`, `total`, `avg` (default) or `none`
+<3> score mode could be `ScoreMode.Max`, `ScoreMode.Min`, `ScoreMode.Total`, `ScoreMode.Avg` or `ScoreMode.None`
diff --git a/docs/java-api/query-dsl/percolate-query.asciidoc b/docs/java-api/query-dsl/percolate-query.asciidoc
index 186d707379e66..a5651392b628e 100644
--- a/docs/java-api/query-dsl/percolate-query.asciidoc
+++ b/docs/java-api/query-dsl/percolate-query.asciidoc
@@ -1,21 +1,15 @@
 [[java-query-percolate-query]]
-==== Percolate query
+==== Percolate Query
 
-See: {ref}/query-dsl-percolate-query.html[Percolate Query]
+See:
+ * {ref}/query-dsl-percolate-query.html[Percolate Query]
 
-In order to use the `percolate` query from the Java API your
-the percolator module dependency should be on the classpath and
-the transport client should be loaded with the percolator plugin:
 
 [source,java]
 --------------------------------------------------
-TransportClient transportClient = TransportClient.builder()
-        .settings(Settings.builder().put("node.name", "node"))
-        .addPlugin(PercolatorPlugin.class)
-        .build();
-transportClient.addTransportAddress(
-        new InetSocketTransportAddress(new InetSocketAddress(InetAddresses.forString("127.0.0.1"), 9300))
-);
+Settings settings = Settings.builder().put("cluster.name", "elasticsearch").build();
+TransportClient client = new PreBuiltTransportClient(settings);
+client.addTransportAddress(new InetSocketTransportAddress(new InetSocketAddress(InetAddresses.forString("127.0.0.1"), 9300)));
 --------------------------------------------------
 
 Before the `percolate` query can be used an `percolator` mapping should be added and
@@ -26,6 +20,7 @@ a document containing a percolator query should be indexed:
 // create an index with a percolator field with the name 'query':
 client.admin().indices().prepareCreate("myIndexName")
                         .addMapping("query", "query", "type=percolator")
+                        .addMapping("docs", "content", "type=text")
                         .get();
 
 //This is the query we're registering in the percolator
@@ -37,7 +32,7 @@ client.prepareIndex("myIndexName", "query", "myDesignatedQueryName")
         .startObject()
             .field("query", qb) // Register the query
         .endObject())
-    .setRefresh(true) // Needed when the query shall be available immediately
+    .setRefreshPolicy(RefreshPolicy.IMMEDIATE) // Needed when the query shall be available immediately
     .get();
 --------------------------------------------------
 
@@ -51,13 +46,14 @@ code:
 --------------------------------------------------
 //Build a document to check against the percolator
 XContentBuilder docBuilder = XContentFactory.jsonBuilder().startObject();
-docBuilder.field("doc").startObject(); //This is needed to designate the document
 docBuilder.field("content", "This is amazing!");
-docBuilder.endObject(); //End of the doc field
 docBuilder.endObject(); //End of the JSON root object
+
+PercolateQueryBuilder percolateQuery = new PercolateQueryBuilder("query", "docs", docBuilder.bytes());
+
 // Percolate, by executing the percolator query in the query dsl:
 SearchResponse response = client().prepareSearch("myIndexName")
-        .setQuery(QueryBuilders.percolateQuery("query", ""myDocumentType", docBuilder.bytes()))
+        .setQuery(percolateQuery))
         .get();
 //Iterate over the results
 for(SearchHit hit : response.getHits()) {
diff --git a/docs/java-api/query-dsl/script-query.asciidoc b/docs/java-api/query-dsl/script-query.asciidoc
index 5d30cab41800a..a79af0f2cef05 100644
--- a/docs/java-api/query-dsl/script-query.asciidoc
+++ b/docs/java-api/query-dsl/script-query.asciidoc
@@ -25,13 +25,13 @@ You can use it then with:
 --------------------------------------------------
 QueryBuilder qb = scriptQuery(
     new Script(
-        "myscript",                          <1>
-        ScriptType.FILE,       <2>
-        "painless",                          <3>
-        ImmutableMap.of("param1", 5))        <4>
+        ScriptType.FILE,                       <1>
+        "painless",                            <2>
+        "myscript",                            <3>
+        Collections.singletonMap("param1", 5)) <4>
 );
 --------------------------------------------------
-<1> Script name
-<2> Script type: either `ScriptType.FILE`, `ScriptType.INLINE` or `ScriptType.INDEXED`
-<3> Scripting engine
+<1> Script type: either `ScriptType.FILE`, `ScriptType.INLINE` or `ScriptType.INDEXED`
+<2> Scripting engine
+<3> Script name
 <4> Parameters as a `Map` of `<String, Object>`
diff --git a/docs/java-api/query-dsl/span-containing-query.asciidoc b/docs/java-api/query-dsl/span-containing-query.asciidoc
index 81859eb93f3ec..a9f762f929d69 100644
--- a/docs/java-api/query-dsl/span-containing-query.asciidoc
+++ b/docs/java-api/query-dsl/span-containing-query.asciidoc
@@ -7,7 +7,7 @@ See {ref}/query-dsl-span-containing-query.html[Span Containing Query]
 --------------------------------------------------
 QueryBuilder qb = spanContainingQuery(
     spanNearQuery(spanTermQuery("field1","bar"), 5) <1>
-          .clause(spanTermQuery("field1","baz"))
+          .addClause(spanTermQuery("field1","baz"))
           .inOrder(true),
     spanTermQuery("field1","foo"));                 <2>
 --------------------------------------------------
diff --git a/docs/java-api/query-dsl/span-near-query.asciidoc b/docs/java-api/query-dsl/span-near-query.asciidoc
index d18d2d74958c9..6e811d595811d 100644
--- a/docs/java-api/query-dsl/span-near-query.asciidoc
+++ b/docs/java-api/query-dsl/span-near-query.asciidoc
@@ -8,13 +8,11 @@ See {ref}/query-dsl-span-near-query.html[Span Near Query]
 QueryBuilder qb = spanNearQuery(
     spanTermQuery("field","value1"),            <1>
     12)                                         <2>
-    .clause(spanTermQuery("field","value2"))    <1>
-    .clause(spanTermQuery("field","value3"))    <1>
-    .inOrder(false)                             <3>
-    .collectPayloads(false);                    <4>
+    .addClause(spanTermQuery("field","value2")) <1>
+    .addClause(spanTermQuery("field","value3")) <1>
+    .inOrder(false);                            <3>
 --------------------------------------------------
 <1> span term queries
 <2> slop factor: the maximum number of intervening unmatched positions
 <3> whether matches are required to be in-order
-<4> collect payloads or not
 
diff --git a/docs/java-api/query-dsl/span-or-query.asciidoc b/docs/java-api/query-dsl/span-or-query.asciidoc
index 61f72a24fcf3c..54f566c47701f 100644
--- a/docs/java-api/query-dsl/span-or-query.asciidoc
+++ b/docs/java-api/query-dsl/span-or-query.asciidoc
@@ -6,9 +6,9 @@ See {ref}/query-dsl-span-or-query.html[Span Or Query]
 [source,java]
 --------------------------------------------------
 QueryBuilder qb = spanOrQuery(
-    spanTermQuery("field","value1"))            <1>
-    .clause(spanTermQuery("field","value2"))    <1>
-    .clause(spanTermQuery("field","value3"));   <1>
+    spanTermQuery("field","value1"))               <1>
+    .addClause(spanTermQuery("field","value2"))    <1>
+    .addClause(spanTermQuery("field","value3"));   <1>
 --------------------------------------------------
 <1> span term queries
 
diff --git a/docs/java-api/query-dsl/span-within-query.asciidoc b/docs/java-api/query-dsl/span-within-query.asciidoc
index 345dabd8c12c4..0e8752785fb5d 100644
--- a/docs/java-api/query-dsl/span-within-query.asciidoc
+++ b/docs/java-api/query-dsl/span-within-query.asciidoc
@@ -7,7 +7,7 @@ See {ref}/query-dsl-span-within-query.html[Span Within Query]
 --------------------------------------------------
 QueryBuilder qb = spanWithinQuery(
     spanNearQuery(spanTermQuery("field1", "bar"), 5) <1>
-        .clause(spanTermQuery("field1", "baz"))
+        .addClause(spanTermQuery("field1", "baz"))
         .inOrder(true),
     spanTermQuery("field1", "foo"));                 <2>
 --------------------------------------------------
diff --git a/docs/java-api/query-dsl/special-queries.asciidoc b/docs/java-api/query-dsl/special-queries.asciidoc
index 31db47ce6367a..4e4d59a6d4aa5 100644
--- a/docs/java-api/query-dsl/special-queries.asciidoc
+++ b/docs/java-api/query-dsl/special-queries.asciidoc
@@ -9,12 +9,6 @@ This group contains queries which do not fit into the other groups:
 This query finds documents which are similar to the specified text, document,
 or collection of documents.
 
-<<java-query-dsl-template-query,`template` query>>::
-
-The `template` query accepts a Mustache template (either inline, indexed, or
-from a file), and a map of parameters,  and combines the two to generate the
-final query to execute.
-
 <<java-query-dsl-script-query,`script` query>>::
 
 This query allows a script to act as a filter.  Also see the
@@ -26,8 +20,6 @@ This query finds percolator queries based on documents.
 
 include::mlt-query.asciidoc[]
 
-include::template-query.asciidoc[]
-
 include::script-query.asciidoc[]
 
 include::percolate-query.asciidoc[]
diff --git a/docs/java-api/query-dsl/template-query.asciidoc b/docs/java-api/query-dsl/template-query.asciidoc
deleted file mode 100644
index dfba0d63a6315..0000000000000
--- a/docs/java-api/query-dsl/template-query.asciidoc
+++ /dev/null
@@ -1,89 +0,0 @@
-[[java-query-dsl-template-query]]
-==== Template Query
-
-See {ref}/search-template.html[Search Template] documentation
-
-In order to use the `template` query from the Java API
-the lang-mustache module dependency should be on the classpath and
-the transport client should be loaded with the lang-mustache plugin:
-
-[source,java]
---------------------------------------------------
-TransportClient transportClient = TransportClient.builder()
-        .settings(Settings.builder().put("node.name", "node"))
-        .addPlugin(MustachePlugin.class)
-        .build();
-transportClient.addTransportAddress(
-        new InetSocketTransportAddress(new InetSocketAddress(InetAddresses.forString("127.0.0.1"), 9300))
-);
---------------------------------------------------
-
-Define your template parameters as a `Map<String,Object>`:
-
-[source,java]
---------------------------------------------------
-Map<String, Object> template_params = new HashMap<>();
-template_params.put("param_gender", "male");
---------------------------------------------------
-
-You can use your stored search templates in `config/scripts`.
-For example, if you have a file named `config/scripts/template_gender.mustache` containing:
-
-[source,js]
---------------------------------------------------
-{
-    "template" : {
-        "query" : {
-            "match" : {
-                "gender" : "{{param_gender}}"
-            }
-        }
-    }
-}
---------------------------------------------------
-// NOTCONSOLE
-
-Define your template query:
-
-[source,java]
---------------------------------------------------
-QueryBuilder qb = new TemplateQueryBuilder(
-    "gender_template",                  <1>
-    ScriptService.ScriptType.FILE,      <2>
-    template_params);                   <3>
---------------------------------------------------
-<1> template name
-<2> template stored on disk in `gender_template.mustache`
-<3> parameters
-
-You can also store your template in the cluster state:
-
-[source,java]
---------------------------------------------------
-client.admin().cluster().preparePutStoredScript()
-    .setScriptLang("mustache")
-    .setId("template_gender")
-    .setSource(new BytesArray(
-        "{\n" +
-        "    \"template\" : {\n" +
-        "        \"query\" : {\n" +
-        "            \"match\" : {\n" +
-        "                \"gender\" : \"{{param_gender}}\"\n" +
-        "            }\n" +
-        "        }\n" +
-        "    }\n" +
-        "}")).get();
---------------------------------------------------
-
-To execute a stored templates, use `ScriptService.ScriptType.STORED`:
-
-[source,java]
---------------------------------------------------
-QueryBuilder qb = new TemplateQueryBuilder(
-    "gender_template",                  <1>
-    ScriptType.STORED,                  <2>
-    template_params);                   <3>
---------------------------------------------------
-<1> template name
-<2> template stored in the cluster state
-<3> parameters
diff --git a/docs/java-api/search.asciidoc b/docs/java-api/search.asciidoc
index 8fd47762aded8..dfc2767520237 100644
--- a/docs/java-api/search.asciidoc
+++ b/docs/java-api/search.asciidoc
@@ -3,7 +3,7 @@
 
 The search API allows one to execute a search query and get back search hits
 that match the query. It can be executed across one or more indices and
-across one or more types. The query can provided using the <<java-query-dsl,query Java API>>.
+across one or more types. The query can be provided using the <<java-query-dsl,query Java API>>.
 The body of the search request is built using the `SearchSourceBuilder`. Here is an example:
 
 [source,java]
@@ -21,8 +21,7 @@ SearchResponse response = client.prepareSearch("index1", "index2")
         .setQuery(QueryBuilders.termQuery("multi", "test"))                 // Query
         .setPostFilter(QueryBuilders.rangeQuery("age").from(12).to(18))     // Filter
         .setFrom(0).setSize(60).setExplain(true)
-        .execute()
-        .actionGet();
+        .get();
 --------------------------------------------------
 
 Note that all parameters are optional. Here is the smallest search call
@@ -31,7 +30,7 @@ you can write:
 [source,java]
 --------------------------------------------------
 // MatchAll on the whole cluster with all default options
-SearchResponse response = client.prepareSearch().execute().actionGet();
+SearchResponse response = client.prepareSearch().get();
 --------------------------------------------------
 
 NOTE:   Although the Java API defines the additional search types QUERY_AND_FETCH and
@@ -58,7 +57,7 @@ SearchResponse scrollResp = client.prepareSearch(test)
         .addSort(FieldSortBuilder.DOC_FIELD_NAME, SortOrder.ASC)
         .setScroll(new TimeValue(60000))
         .setQuery(qb)
-        .setSize(100).execute().actionGet(); //100 hits per shard will be returned for each scroll
+        .setSize(100).get(); //max of 100 hits will be returned for each scroll
 //Scroll until no hits are returned
 do {
     for (SearchHit hit : scrollResp.getHits().getHits()) {
@@ -68,10 +67,6 @@ do {
     scrollResp = client.prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(60000)).execute().actionGet();
 } while(scrollResp.getHits().getHits().length != 0); // Zero hits mark the end of the scroll and the while loop.
 --------------------------------------------------
-[NOTE]
-====
-The size-parameter is per shard, so if you run a query against multiple indices (leading to many shards being involved in the query) the result might be more documents per execution of the scroll than you would expect!
-====
 
 [[java-search-msearch]]
 === MultiSearch API
@@ -81,15 +76,15 @@ documentation
 
 [source,java]
 --------------------------------------------------
-SearchRequestBuilder srb1 = node.client()
+SearchRequestBuilder srb1 = client
     .prepareSearch().setQuery(QueryBuilders.queryStringQuery("elasticsearch")).setSize(1);
-SearchRequestBuilder srb2 = node.client()
+SearchRequestBuilder srb2 = client
     .prepareSearch().setQuery(QueryBuilders.matchQuery("name", "kimchy")).setSize(1);
 
-MultiSearchResponse sr = node.client().prepareMultiSearch()
+MultiSearchResponse sr = client.prepareMultiSearch()
         .add(srb1)
         .add(srb2)
-        .execute().actionGet();
+        .get();
 
 // You will get all individual responses from MultiSearchResponse#getResponses()
 long nbHits = 0;
@@ -107,7 +102,7 @@ The following code shows how to add two aggregations within your search:
 
 [source,java]
 --------------------------------------------------
-SearchResponse sr = node.client().prepareSearch()
+SearchResponse sr = client.prepareSearch()
     .setQuery(QueryBuilders.matchAllQuery())
     .addAggregation(
             AggregationBuilders.terms("agg1").field("field")
@@ -117,11 +112,11 @@ SearchResponse sr = node.client().prepareSearch()
                     .field("birth")
                     .dateHistogramInterval(DateHistogramInterval.YEAR)
     )
-    .execute().actionGet();
+    .get();
 
 // Get your facet results
 Terms agg1 = sr.getAggregations().get("agg1");
-DateHistogram agg2 = sr.getAggregations().get("agg2");
+Histogram agg2 = sr.getAggregations().get("agg2");
 --------------------------------------------------
 
 See <<java-aggs,Aggregations Java API>>
@@ -146,3 +141,111 @@ if (sr.isTerminatedEarly()) {
 }
 --------------------------------------------------
 <1> Finish after 1000 docs
+
+[[java-search-template]]
+=== Search Template
+
+See {ref}/search-template.html[Search Template] documentation
+
+Define your template parameters as a `Map<String,Object>`:
+
+[source,java]
+--------------------------------------------------
+Map<String, Object> template_params = new HashMap<>();
+template_params.put("param_gender", "male");
+--------------------------------------------------
+
+You can use your stored search templates in `config/scripts`.
+For example, if you have a file named `config/scripts/template_gender.mustache` containing:
+
+[source,js]
+--------------------------------------------------
+{
+    "query" : {
+        "match" : {
+            "gender" : "{{param_gender}}"
+        }
+    }
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+Create your search template request:
+
+[source,java]
+--------------------------------------------------
+SearchResponse sr = new SearchTemplateRequestBuilder(client)
+    .setScript("template_gender")                 <1>
+    .setScriptType(ScriptService.ScriptType.FILE) <2>
+    .setScriptParams(template_params)             <3>
+    .setRequest(new SearchRequest())              <4>
+    .get()                                        <5>
+    .getResponse();                               <6>
+--------------------------------------------------
+<1> template name
+<2> template stored on disk in `gender_template.mustache`
+<3> parameters
+<4> set the execution context (ie. define the index name here)
+<5> execute and get the template response
+<6> get from the template response the search response itself
+
+You can also store your template in the cluster state:
+
+[source,java]
+--------------------------------------------------
+client.admin().cluster().preparePutStoredScript()
+    .setScriptLang("mustache")
+    .setId("template_gender")
+    .setSource(new BytesArray(
+        "{\n" +
+        "    \"query\" : {\n" +
+        "        \"match\" : {\n" +
+        "            \"gender\" : \"{{param_gender}}\"\n" +
+        "        }\n" +
+        "    }\n" +
+        "}")).get();
+--------------------------------------------------
+
+To execute a stored templates, use `ScriptService.ScriptType.STORED`:
+
+[source,java]
+--------------------------------------------------
+SearchResponse sr = new SearchTemplateRequestBuilder(client)
+        .setScript("template_gender")                       <1>
+        .setScriptType(ScriptType.STORED)     <2>
+        .setScriptParams(template_params)                   <3>
+        .setRequest(new SearchRequest())                    <4>
+        .get()                                              <5>
+        .getResponse();                                     <6>
+--------------------------------------------------
+<1> template name
+<2> template stored in the cluster state
+<3> parameters
+<4> set the execution context (ie. define the index name here)
+<5> execute and get the template response
+<6> get from the template response the search response itself
+
+You can also execute inline templates:
+
+[source,java]
+--------------------------------------------------
+sr = new SearchTemplateRequestBuilder(client)
+        .setScript("{\n" +                                  <1>
+                "        \"query\" : {\n" +
+                "            \"match\" : {\n" +
+                "                \"gender\" : \"{{param_gender}}\"\n" +
+                "            }\n" +
+                "        }\n" +
+                "}")
+        .setScriptType(ScriptType.INLINE)    <2>
+        .setScriptParams(template_params)                  <3>
+        .setRequest(new SearchRequest())                   <4>
+        .get()                                             <5>
+        .getResponse();                                    <6>
+--------------------------------------------------
+<1> template name
+<2> template is passed inline
+<3> parameters
+<4> set the execution context (ie. define the index name here)
+<5> execute and get the template response
+<6> get from the template response the search response itself
diff --git a/docs/java-rest/configuration.asciidoc b/docs/java-rest/configuration.asciidoc
deleted file mode 100644
index b3546c0f75b28..0000000000000
--- a/docs/java-rest/configuration.asciidoc
+++ /dev/null
@@ -1,113 +0,0 @@
-== Common configuration
-
-The `RestClientBuilder` supports providing both a `RequestConfigCallback` and
-an `HttpClientConfigCallback` which allow for any customization that the Apache
-Async Http Client exposes. Those callbacks make it possible to modify some
-specific behaviour of the client without overriding every other default
-configuration that the `RestClient` is initialized with. This section
-describes some common scenarios that require additional configuration for the
-low-level Java REST Client.
-
-=== Timeouts
-
-Configuring requests timeouts can be done by providing an instance of
-`RequestConfigCallback` while building the `RestClient` through its builder.
-The interface has one method that receives an instance of
-https://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/client/config/RequestConfig.Builder.html[`org.apache.http.client.config.RequestConfig.Builder`]
- as an argument and has the same return type. The request config builder can
-be modified and then returned. In the following example we increase the
-connect timeout (defaults to 1 second) and the socket timeout (defaults to 10
-seconds). Also we adjust the max retry timeout accordingly (defaults to 10
-seconds too).
-
-[source,java]
---------------------------------------------------
-RestClient restClient = RestClient.builder(new HttpHost("localhost", 9200))
-        .setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
-            @Override
-            public RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
-                return requestConfigBuilder.setConnectTimeout(5000)
-                        .setSocketTimeout(30000);
-            }
-        })
-        .setMaxRetryTimeoutMillis(30000)
-        .build();
---------------------------------------------------
-
-=== Number of threads
-
-The Apache Http Async Client starts by default one dispatcher thread, and a
-number of worker threads used by the connection manager, as many as the number
-of locally detected processors (depending on what
-`Runtime.getRuntime().availableProcessors()` returns). The number of threads
-can be modified as follows:
-
-[source,java]
---------------------------------------------------
-RestClient restClient = RestClient.builder(new HttpHost("localhost", 9200))
-        .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
-            @Override
-            public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
-                return httpClientBuilder.setDefaultIOReactorConfig(
-                        IOReactorConfig.custom().setIoThreadCount(1).build());
-            }
-        })
-        .build();
---------------------------------------------------
-
-=== Basic authentication
-
-Configuring basic authentication can be done by providing an
-`HttpClientConfigCallback` while building the `RestClient` through its builder.
-The interface has one method that receives an instance of
-https://hc.apache.org/httpcomponents-asyncclient-dev/httpasyncclient/apidocs/org/apache/http/impl/nio/client/HttpAsyncClientBuilder.html[`org.apache.http.impl.nio.client.HttpAsyncClientBuilder`]
- as an argument and has the same return type. The http client builder can be
-modified and then returned. In the following example we set a default
-credentials provider that requires basic authentication.
-
-[source,java]
---------------------------------------------------
-final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
-credentialsProvider.setCredentials(AuthScope.ANY,
-        new UsernamePasswordCredentials("user", "password"));
-
-RestClient restClient = RestClient.builder(new HttpHost("localhost", 9200))
-        .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
-            @Override
-            public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
-                return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
-            }
-        })
-        .build();
---------------------------------------------------
-
-=== Encrypted communication
-
-Encrypted communication can also be configured through the
-`HttpClientConfigCallback`. The
-https://hc.apache.org/httpcomponents-asyncclient-dev/httpasyncclient/apidocs/org/apache/http/impl/nio/client/HttpAsyncClientBuilder.html[`org.apache.http.impl.nio.client.HttpAsyncClientBuilder`]
- received as an argument exposes multiple methods to configure encrypted
- communication: `setSSLContext`, `setSSLSessionStrategy` and
- `setConnectionManager`, in order of precedence from the least important.
- The following is an example:
-
-[source,java]
---------------------------------------------------
-KeyStore keyStore = KeyStore.getInstance("jks");
-try (InputStream is = Files.newInputStream(keyStorePath)) {
-    keyStore.load(is, keyStorePass.toCharArray());
-}
-RestClient restClient = RestClient.builder(new HttpHost("localhost", 9200))
-        .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
-            @Override
-            public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
-                return httpClientBuilder.setSSLContext(sslcontext);
-            }
-        })
-        .build();
---------------------------------------------------
-
-=== Others
-
-For any other required configuration needed, the Apache HttpAsyncClient docs
-should be consulted: https://hc.apache.org/httpcomponents-asyncclient-4.1.x/ .
\ No newline at end of file
diff --git a/docs/java-rest/high-level/apis/_index.asciidoc b/docs/java-rest/high-level/apis/_index.asciidoc
new file mode 100644
index 0000000000000..8a9ee69c08c24
--- /dev/null
+++ b/docs/java-rest/high-level/apis/_index.asciidoc
@@ -0,0 +1,149 @@
+[[java-rest-high-document-index]]
+=== Index API
+
+[[java-rest-high-document-index-request]]
+==== Index Request
+
+An `IndexRequest` requires the following arguments:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[index-request-string]
+--------------------------------------------------
+<1> Index
+<2> Type
+<3> Document id
+<4> Document source provided as a `String`
+
+==== Providing the document source
+The document source can be provided in different ways:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[index-request-map]
+--------------------------------------------------
+<1> Document source provided as a `Map` which gets automatically converted
+to JSON format
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[index-request-xcontent]
+--------------------------------------------------
+<1> Document source provided as an `XContentBuilder` object, the Elasticsearch
+built-in helpers to generate JSON content
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[index-request-shortcut]
+--------------------------------------------------
+<1> Document source provided as `Object` key-pairs, which gets converted to
+JSON format
+
+==== Optional arguments
+The following arguments can optionally be provided:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[index-request-routing]
+--------------------------------------------------
+<1> Routing value
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[index-request-parent]
+--------------------------------------------------
+<1> Parent value
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[index-request-timeout]
+--------------------------------------------------
+<1> Timeout to wait for primary shard to become available as a `TimeValue`
+<2> Timeout to wait for primary shard to become available as a `String`
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[index-request-refresh]
+--------------------------------------------------
+<1> Refresh policy as a `WriteRequest.RefreshPolicy` instance
+<2> Refresh policy as a `String`
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[index-request-version]
+--------------------------------------------------
+<1> Version
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[index-request-version-type]
+--------------------------------------------------
+<1> Version type
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[index-request-op-type]
+--------------------------------------------------
+<1> Operation type provided as an `DocWriteRequest.OpType` value
+<2> Operation type provided as a `String`: can be `create` or `update` (default)
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[index-request-pipeline]
+--------------------------------------------------
+<1> The name of the ingest pipeline to be executed before indexing the document
+
+[[java-rest-high-document-index-sync]]
+==== Synchronous Execution
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[index-execute]
+--------------------------------------------------
+
+[[java-rest-high-document-index-async]]
+==== Asynchronous Execution
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[index-execute-async]
+--------------------------------------------------
+<1> Called when the execution is successfully completed. The response is
+provided as an argument
+<2> Called in case of failure. The raised exception is provided as an argument
+
+[[java-rest-high-document-index-response]]
+==== Index Response
+
+The returned `IndexResponse` allows to retrieve information about the executed
+ operation as follows:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[index-response]
+--------------------------------------------------
+<1> Handle (if needed) the case where the document was created for the first
+time
+<2> Handle (if needed) the case where the document was rewritten as it was
+already existing
+<3> Handle the situation where number of successful shards is less than
+total shards
+<4> Handle the potential failures
+
+If there is a version conflict, an `ElasticsearchException` will
+be thrown:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[index-conflict]
+--------------------------------------------------
+<1> The raised exception indicates that a version conflict error was returned
+
+Same will happen in case `opType` was set to `create` and a document with
+same index, type and id already existed:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[index-optype]
+--------------------------------------------------
+<1> The raised exception indicates that a version conflict error was returned
diff --git a/docs/java-rest/high-level/apis/bulk.asciidoc b/docs/java-rest/high-level/apis/bulk.asciidoc
new file mode 100644
index 0000000000000..9bbc0b3106267
--- /dev/null
+++ b/docs/java-rest/high-level/apis/bulk.asciidoc
@@ -0,0 +1,211 @@
+[[java-rest-high-document-bulk]]
+=== Bulk API
+
+NOTE: The Java High Level REST Client provides the <<java-rest-high-document-bulk-processor>> to assist with bulk requests
+
+[[java-rest-high-document-bulk-request]]
+==== Bulk Request
+
+A `BulkRequest` can be used to execute multiple index, update and/or delete
+operations using a single request.
+
+It requires at least one operation to be added to the Bulk request:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[bulk-request]
+--------------------------------------------------
+<1> Creates the `BulkRequest`
+<2> Adds a first `IndexRequest` to the Bulk request. See <<java-rest-high-document-index>>
+for more information on how to build `IndexRequest`.
+<3> Adds a second `IndexRequest`
+<4> Adds a third `IndexRequest`
+
+WARNING: The Bulk API supports only documents encoded in JSON or SMILE. Providing documents
+ in any other format will result in an error.
+
+And different operation types can be added to the same `BulkRequest`:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[bulk-request-with-mixed-operations]
+--------------------------------------------------
+<1> Adds a `DeleteRequest` to the `BulkRequest`. See <<java-rest-high-document-delete>>
+for more information on how to build `DeleteRequest`.
+<2> Adds an `UpdateRequest` to the `BulkRequest`. See <<java-rest-high-document-update>>
+for more information on how to build `UpdateRequest`.
+<3> Adds an `IndexRequest` using the SMILE format
+
+==== Optional arguments
+The following arguments can optionally be provided:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[bulk-request-timeout]
+--------------------------------------------------
+<1> Timeout to wait for the bulk request to be performed as a `TimeValue`
+<2> Timeout to wait for the bulk request to be performed as a `String`
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[bulk-request-refresh]
+--------------------------------------------------
+<1> Refresh policy as a `WriteRequest.RefreshPolicy` instance
+<2> Refresh policy as a `String`
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[bulk-request-active-shards]
+--------------------------------------------------
+<1> Sets the number of shard copies that must be active before proceeding with
+the index/update/delete operations.
+<2> Number of shard copies provided as a `ActiveShardCount`: can be `ActiveShardCount.ALL`,
+`ActiveShardCount.ONE` or `ActiveShardCount.DEFAULT` (default)
+
+
+[[java-rest-high-document-bulk-sync]]
+==== Synchronous Execution
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[bulk-execute]
+--------------------------------------------------
+
+[[java-rest-high-document-bulk-async]]
+==== Asynchronous Execution
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[bulk-execute-async]
+--------------------------------------------------
+<1> Called when the execution is successfully completed. The response is
+provided as an argument and contains a list of individual results for each
+operation that was executed. Note that one or more operations might have
+failed while the others have been successfully executed.
+<2> Called when the whole `BulkRequest` fails. In this case the raised
+exception is provided as an argument and no operation has been executed.
+
+[[java-rest-high-document-bulk-response]]
+==== Bulk Response
+
+The returned `BulkResponse` contains information about the executed operations and
+ allows to iterate over each result as follows:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[bulk-response]
+--------------------------------------------------
+<1> Iterate over the results of all operations
+<2> Retrieve the response of the operation (successful or not), can be `IndexResponse`,
+`UpdateResponse` or `DeleteResponse` which can all be seen as `DocWriteResponse` instances
+<3> Handle the response of an index operation
+<4> Handle the response of a update operation
+<5> Handle the response of a delete operation
+
+The Bulk response provides a method to quickly check if one or more operation has failed:
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[bulk-has-failures]
+--------------------------------------------------
+<1> This method returns `true` if at least one operation failed
+
+In such situation it is necessary to iterate over all operation results in order to check
+ if the operation failed, and if so, retrieve the corresponding failure:
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[bulk-errors]
+--------------------------------------------------
+<1> Indicate if a given operation failed
+<2> Retrieve the failure of the failed operation
+
+[[java-rest-high-document-bulk-processor]]
+==== Bulk Processor
+
+The `BulkProcessor` simplifies the usage of the Bulk API by providing
+a utility class that allows index/update/delete operations to be
+transparently executed as they are added to the processor.
+
+In order to execute the requests, the `BulkProcessor` requires 3 components:
+
+`RestHighLevelClient`:: This client is used to execute the `BulkRequest`
+and to retrieve the `BulkResponse`
+`BulkProcessor.Listener`:: This listener is called before and after
+every `BulkRequest` execution or when a `BulkRequest` failed
+`ThreadPool`:: The `BulkRequest` executions are done using threads from this
+pool, allowing the `BulkProcessor` to work in a non-blocking manner and to
+accept new index/update/delete requests while bulk requests are executing.
+
+Then the `BulkProcessor.Builder` class can be used to build a new `BulkProcessor`:
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[bulk-processor-init]
+--------------------------------------------------
+<1> Create the `ThreadPool` using the given `Settings`
+<2> Create the `BulkProcessor.Listener`
+<3> This method is called before each execution of a `BulkRequest`
+<4> This method is called after each execution of a `BulkRequest`
+<5> This method is called when a `BulkRequest` failed
+<6> Create the `BulkProcessor` by calling the `build()` method from
+the `BulkProcessor.Builder`. The `RestHighLevelClient.bulkAsync()`
+method will be used to execute the `BulkRequest` under the hood.
+
+The `BulkProcessor.Builder` provides methods to configure how the `BulkProcessor`
+should handle requests execution:
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[bulk-processor-options]
+--------------------------------------------------
+<1> Set when to flush a new bulk request based on the number of
+actions currently added (defaults to 1000, use -1 to disable it)
+<2> Set when to flush a new bulk request based on the size of
+actions currently added (defaults to 5Mb, use -1 to disable it)
+<3> Set the number of concurrent requests allowed to be executed
+(default to 1, use 0 to only allow the execution of a single request)
+<4> Set a flush interval flushing any `BulkRequest` pending if the
+interval passes (defaults to not set)
+<5> Set a constant back off policy that initially waits for 1 second
+and retries up to 3 times. See `BackoffPolicy.noBackoff()`,
+`BackoffPolicy.constantBackoff()` and `BackoffPolicy.exponentialBackoff()`
+for more options.
+
+Once the `BulkProcessor` is created requests can be added to it:
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[bulk-processor-add]
+--------------------------------------------------
+
+The requests will be executed by the `BulkProcessor`, which takes care of
+calling the `BulkProcessor.Listener` for every bulk request.
+
+The listener provides methods to access to the `BulkRequest` and the `BulkResponse`:
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[bulk-processor-listener]
+--------------------------------------------------
+<1> Called before each execution of a `BulkRequest`, this method allows
+to know the number of operations that are going to be executed within the `BulkRequest`
+<2> Called after each execution of a `BulkRequest`, this method allows
+to know if the `BulkResponse` contains errors
+<3> Called if the `BulkRequest` failed, this method allows to know
+the failure
+
+Once all requests have been added to the `BulkProcessor`, its instance needs to
+be closed closed using one of the two available closing methods.
+
+The `awaitClose()` method can be used to wait until all requests have been processed
+ or the specified waiting time elapses:
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[bulk-processor-await]
+--------------------------------------------------
+<1> The method returns `true` if all bulk requests completed and `false` if the
+waiting time elapsed before all the bulk requests completed
+
+The `close()` method can be used to immediately close the `BulkProcessor`:
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[bulk-processor-close]
+--------------------------------------------------
+
+Both methods flush the requests added to the processor before closing the processor
+and also forbid any new request to be added to it.
diff --git a/docs/java-rest/high-level/apis/delete.asciidoc b/docs/java-rest/high-level/apis/delete.asciidoc
new file mode 100644
index 0000000000000..5b71f6214a5a7
--- /dev/null
+++ b/docs/java-rest/high-level/apis/delete.asciidoc
@@ -0,0 +1,108 @@
+[[java-rest-high-document-delete]]
+=== Delete API
+
+[[java-rest-high-document-delete-request]]
+==== Delete Request
+
+A `DeleteRequest` requires the following arguments:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[delete-request]
+--------------------------------------------------
+<1> Index
+<2> Type
+<3> Document id
+
+==== Optional arguments
+The following arguments can optionally be provided:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[delete-request-routing]
+--------------------------------------------------
+<1> Routing value
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[delete-request-parent]
+--------------------------------------------------
+<1> Parent value
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[delete-request-timeout]
+--------------------------------------------------
+<1> Timeout to wait for primary shard to become available as a `TimeValue`
+<2> Timeout to wait for primary shard to become available as a `String`
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[delete-request-refresh]
+--------------------------------------------------
+<1> Refresh policy as a `WriteRequest.RefreshPolicy` instance
+<2> Refresh policy as a `String`
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[delete-request-version]
+--------------------------------------------------
+<1> Version
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[delete-request-version-type]
+--------------------------------------------------
+<1> Version type
+
+[[java-rest-high-document-delete-sync]]
+==== Synchronous Execution
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[delete-execute]
+--------------------------------------------------
+
+[[java-rest-high-document-delete-async]]
+==== Asynchronous Execution
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[delete-execute-async]
+--------------------------------------------------
+<1> Called when the execution is successfully completed. The response is
+provided as an argument
+<2> Called in case of failure. The raised exception is provided as an argument
+
+[[java-rest-high-document-delete-response]]
+==== Delete Response
+
+The returned `DeleteResponse` allows to retrieve information about the executed
+ operation as follows:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[delete-response]
+--------------------------------------------------
+<1> Handle the situation where number of successful shards is less than
+total shards
+<2> Handle the potential failures
+
+
+It is also possible to check whether the document was found or not:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[delete-notfound]
+--------------------------------------------------
+<1> Do something if the document to be deleted was not found
+
+If there is a version conflict, an `ElasticsearchException` will
+be thrown:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[delete-conflict]
+--------------------------------------------------
+<1> The raised exception indicates that a version conflict error was returned
+
diff --git a/docs/java-rest/high-level/apis/get.asciidoc b/docs/java-rest/high-level/apis/get.asciidoc
new file mode 100644
index 0000000000000..af1d64fa2a04b
--- /dev/null
+++ b/docs/java-rest/high-level/apis/get.asciidoc
@@ -0,0 +1,143 @@
+[[java-rest-high-document-get]]
+=== Get API
+
+[[java-rest-high-document-get-request]]
+==== Get Request
+
+A `GetRequest` requires the following arguments:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[get-request]
+--------------------------------------------------
+<1> Index
+<2> Type
+<3> Document id
+
+==== Optional arguments
+The following arguments can optionally be provided:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[get-request-no-source]
+--------------------------------------------------
+<1> Disable source retrieval, enabled by default
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[get-request-source-include]
+--------------------------------------------------
+<1> Configure source inclusion for specific fields
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[get-request-source-exclude]
+--------------------------------------------------
+<1> Configure source exclusion for specific fields
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[get-request-stored]
+--------------------------------------------------
+<1> Configure retrieval for specific stored fields (requires fields to be
+stored separately in the mappings)
+<2> Retrieve the `message` stored field (requires the field to be stored
+separately in the mappings)
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[get-request-routing]
+--------------------------------------------------
+<1> Routing value
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[get-request-parent]
+--------------------------------------------------
+<1> Parent value
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[get-request-preference]
+--------------------------------------------------
+<1> Preference value
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[get-request-realtime]
+--------------------------------------------------
+<1> Set realtime flag to `false` (`true` by default)
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[get-request-refresh]
+--------------------------------------------------
+<1> Perform a refresh before retrieving the document (`false` by default)
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[get-request-version]
+--------------------------------------------------
+<1> Version
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[get-request-version-type]
+--------------------------------------------------
+<1> Version type
+
+[[java-rest-high-document-get-sync]]
+==== Synchronous Execution
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[get-execute]
+--------------------------------------------------
+
+[[java-rest-high-document-get-async]]
+==== Asynchronous Execution
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[get-execute-async]
+--------------------------------------------------
+<1> Called when the execution is successfully completed. The response is
+provided as an argument.
+<2> Called in case of failure. The raised exception is provided as an argument.
+
+[[java-rest-high-document-get-response]]
+==== Get Response
+
+The returned `GetResponse` allows to retrieve the requested document along with
+its metadata and eventually stored fields.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[get-response]
+--------------------------------------------------
+<1> Retrieve the document as a `String`
+<2> Retrieve the document as a `Map<String, Object>`
+<3> Retrieve the document as a `byte[]`
+<4> Handle the scenario where the document was not found. Note that although
+the returned response has `404` status code, a valid `GetResponse` is
+returned rather than an exception thrown. Such response does not hold any
+source document and its `isExists` method returns `false`.
+
+When a get request is performed against an index that does not exist, the
+response has `404` status code, an `ElasticsearchException` gets thrown
+which needs to be handled as follows:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[get-indexnotfound]
+--------------------------------------------------
+<1> Handle the exception thrown because the index does not exist
+
+In case a specific document version has been requested, and the existing
+document has a different version number, a version conflict is raised:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[get-conflict]
+--------------------------------------------------
+<1> The raised exception indicates that a version conflict error was returned
diff --git a/docs/java-rest/high-level/apis/index.asciidoc b/docs/java-rest/high-level/apis/index.asciidoc
new file mode 100644
index 0000000000000..b4dcf7e9d8037
--- /dev/null
+++ b/docs/java-rest/high-level/apis/index.asciidoc
@@ -0,0 +1,8 @@
+include::_index.asciidoc[]
+include::get.asciidoc[]
+include::delete.asciidoc[]
+include::update.asciidoc[]
+include::bulk.asciidoc[]
+include::search.asciidoc[]
+include::scroll.asciidoc[]
+include::main.asciidoc[]
diff --git a/docs/java-rest/high-level/apis/main.asciidoc b/docs/java-rest/high-level/apis/main.asciidoc
new file mode 100644
index 0000000000000..b37e85ee8bd7b
--- /dev/null
+++ b/docs/java-rest/high-level/apis/main.asciidoc
@@ -0,0 +1,27 @@
+[[java-rest-high-main]]
+=== Info API
+
+[[java-rest-high-main-request]]
+==== Execution
+
+Cluster information can be retrieved using the `info()` method:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/MainDocumentationIT.java[main-execute]
+--------------------------------------------------
+
+[[java-rest-high-main-response]]
+==== Response
+
+The returned `MainResponse` provides various kinds of information about the cluster:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/MainDocumentationIT.java[main-response]
+--------------------------------------------------
+<1> Retrieve the name of the cluster as a `ClusterName`
+<2> Retrieve the unique identifier of the cluster
+<3> Retrieve the name of the node the request has been executed on
+<4> Retrieve the version of the node the request has been executed on
+<5> Retrieve the build information of the node the request has been executed on
diff --git a/docs/java-rest/high-level/apis/scroll.asciidoc b/docs/java-rest/high-level/apis/scroll.asciidoc
new file mode 100644
index 0000000000000..d96cf569e6064
--- /dev/null
+++ b/docs/java-rest/high-level/apis/scroll.asciidoc
@@ -0,0 +1,184 @@
+[[java-rest-high-search-scroll]]
+=== Search Scroll API
+
+The Scroll API can be used to retrieve a large number of results from
+a search request.
+
+In order to use scrolling, the following steps need to be executed in the
+given order.
+
+
+==== Initialize the search scroll context
+
+An initial search request with a `scroll` parameter must be executed to
+initialize the scroll session through the <<java-rest-high-search>>.
+When processing this `SearchRequest`, Elasticsearch detects the presence of
+the `scroll` parameter and keeps the search context alive for the
+corresponding time interval.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[search-scroll-init]
+--------------------------------------------------
+<1> Create the `SearchRequest` and its corresponding `SearchSourceBuilder`.
+Also optionally set the `size` to control how many results to retrieve at
+a time.
+<2> Set the scroll interval
+<3> Read the returned scroll id, which points to the search context that's
+being kept alive and will be needed in the following search scroll call
+<4> Retrieve the first batch of search hits
+
+==== Retrieve all the relevant documents
+
+As a second step, the received scroll identifier must be set to a
+`SearchScrollRequest` along with a new scroll interval and sent through the
+`searchScroll` method. Elasticsearch returns another batch of results with
+a new scroll identifier. This new scroll identifier can then be used in a
+subsequent `SearchScrollRequest` to retrieve the next batch of results,
+and so on. This process should be repeated in a loop until no more results are
+returned, meaning that the scroll has been exhausted and all the matching
+documents have been retrieved.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[search-scroll2]
+--------------------------------------------------
+<1> Create the `SearchScrollRequest` by setting the required scroll id and
+the scroll interval
+<2> Read the new scroll id, which points to the search context that's
+being kept alive and will be needed in the following search scroll call
+<3> Retrieve another batch of search hits
+<4>
+
+==== Clear the scroll context
+
+Finally, the last scroll identifier can be deleted using the <<java-rest-high-clear-scroll>>
+in order to release the search context. This happens automatically when the
+scroll expires, but it's good practice to do it as soon as the scroll session
+is completed.
+
+==== Optional arguments
+
+The following arguments can optionally be provided when constructing
+the `SearchScrollRequest`:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[scroll-request-arguments]
+--------------------------------------------------
+<1> Scroll interval as a `TimeValue`
+<2> Scroll interval as a `String`
+
+If no `scroll` value is set for the `SearchScrollRequest`, the search context will
+expire once the initial scroll time expired (ie, the scroll time set in the
+initial search request).
+
+[[java-rest-high-search-scroll-sync]]
+==== Synchronous Execution
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[search-scroll-execute-sync]
+--------------------------------------------------
+
+[[java-rest-high-search-scroll-async]]
+==== Asynchronous Execution
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[search-scroll-execute-async]
+--------------------------------------------------
+<1> Called when the execution is successfully completed. The response is
+provided as an argument
+<2> Called in case of failure. The raised exception is provided as an argument
+
+[[java-rest-high-search-scroll-response]]
+==== Response
+
+The search scroll API returns a `SearchResponse` object, same as the
+Search API.
+
+[[java-rest-high-search-scroll-example]]
+==== Full example
+
+The following is a complete example of a scrolled search.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[search-scroll-example]
+--------------------------------------------------
+<1> Initialize the search context by sending the initial `SearchRequest`
+<2> Retrieve all the search hits by calling the Search Scroll api in a loop
+until no documents are returned
+<3> Create a new `SearchScrollRequest` holding the last returned scroll
+identifier and the scroll interval
+<4> Process the returned search results
+<5> Clear the scroll context once the scroll is completed
+
+[[java-rest-high-clear-scroll]]
+=== Clear Scroll API
+
+The search contexts used by the Search Scroll API are automatically deleted when the scroll
+times out. But it is advised to release search contexts as soon as they are not
+necessary anymore using the Clear Scroll API.
+
+[[java-rest-high-clear-scroll-request]]
+==== Clear Scroll Request
+
+A `ClearScrollRequest` can be created as follows:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[clear-scroll-request]
+--------------------------------------------------
+<1> Create a new `ClearScrollRequest`
+<2> Adds a scroll id to the list of scroll identifiers to clear
+
+==== Providing the scroll identifiers
+The `ClearScrollRequest` allows to clear one or more scroll identifiers in a single request.
+
+The scroll identifiers can be added to the request one by one:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[clear-scroll-add-scroll-id]
+--------------------------------------------------
+
+Or all together using:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[clear-scroll-add-scroll-ids]
+--------------------------------------------------
+
+[[java-rest-high-clear-scroll-sync]]
+==== Synchronous Execution
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[clear-scroll-execute]
+--------------------------------------------------
+
+[[java-rest-high-clear-scroll-async]]
+==== Asynchronous Execution
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[clear-scroll-execute-async]
+--------------------------------------------------
+<1> Called when the execution is successfully completed. The response is
+provided as an argument
+<2> Called in case of failure. The raised exception is provided as an argument
+
+[[java-rest-high-clear-scroll-response]]
+==== Clear Scroll Response
+
+The returned `ClearScrollResponse` allows to retrieve information about the released
+ search contexts:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[clear-scroll-response]
+--------------------------------------------------
+<1> Return true if the request succeeded
+<2> Return the number of released search contexts
diff --git a/docs/java-rest/high-level/apis/search.asciidoc b/docs/java-rest/high-level/apis/search.asciidoc
new file mode 100644
index 0000000000000..de336fd60d8e4
--- /dev/null
+++ b/docs/java-rest/high-level/apis/search.asciidoc
@@ -0,0 +1,479 @@
+[[java-rest-high-search]]
+=== Search API
+
+[[java-rest-high-document-search-request]]
+==== Search Request
+
+The `SearchRequest` is used for any operation that has to do with searching
+documents, aggregations, suggestions and also offers ways of requesting
+highlighting on the resulting documents.
+
+In its most basic form, we can add a query to the request:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[search-request-basic]
+--------------------------------------------------
+
+<1> Creates the `SeachRequest`. Without arguments this runs against all indices.
+<2> Most search parameters are added to the `SearchSourceBuilder`. It offers setters for everything that goes into the search request body.
+<3> Add a `match_all` query to the `SearchSourceBuilder`.
+
+===== Optional arguments
+
+Let's first look at some of the optional arguments of a `SearchRequest`:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[search-request-indices-types]
+--------------------------------------------------
+<1> Restricts the request to an index
+<2> Limits the request to a type
+
+There are a couple of other interesting optional parameters:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[search-request-routing]
+--------------------------------------------------
+<1> Set a routing parameter
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[search-request-indicesOptions]
+--------------------------------------------------
+<1> Setting `IndicesOptions` controls how unavailable indices are resolved and
+how wildcard expressions are expanded
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[search-request-preference]
+--------------------------------------------------
+<1> Use the preference parameter e.g. to execute the search to prefer local
+shards. The default is to randomize across shards.
+
+===== Using the SearchSourceBuilder
+
+Most options controlling the search behavior can be set on the
+`SearchSourceBuilder`,
+which contains more or less the equivalent of the options in the search request
+body of the Rest API.
+
+Here are a few examples of some common options:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[search-source-basics]
+--------------------------------------------------
+<1> Create a `SearchSourceBuilder` with default options.
+<2> Set the query. Can be any type of `QueryBuilder`
+<3> Set the `from` option that determines the result index to start searching
+from. Defaults to 0.
+<4> Set the `size` option that determines the number of search hits to return.
+Defaults to 10.
+<5> Set an optional timeout that controls how long the search is allowed to
+take.
+
+After this, the `SearchSourceBuilder` only needs to be added to the
+`SearchRequest`:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[search-source-setter]
+--------------------------------------------------
+
+[[java-rest-high-document-search-request-building-queries]]
+===== Building queries
+
+Search queries are created using `QueryBuilder` objects. A `QueryBuilder` exists
+ for every search query type supported by Elasticsearch's {ref}/query-dsl.html[Query DSL].
+
+A `QueryBuilder` can be created using its constructor:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[search-query-builder-ctor]
+--------------------------------------------------
+<1> Create a full text {ref}/query-dsl-match-query.html[Match Query] that matches
+the text "kimchy" over the field "user".
+
+Once created, the `QueryBuilder` object provides methods to configure the options
+of the search query it creates:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[search-query-builder-options]
+--------------------------------------------------
+<1> Enable fuzzy matching on the match query
+<2> Set the prefix length option on the match query
+<3> Set the max expansion options to control the fuzzy
+    process of the query
+
+`QueryBuilder` objects can also be created using the `QueryBuilders` utility class.
+This class provides helper methods that can be used to create `QueryBuilder` objects
+ using a fluent programming style:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[search-query-builders]
+--------------------------------------------------
+
+Whatever the method used to create it, the `QueryBuilder` object must be added
+to the `SearchSourceBuilder` as follows:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[search-query-setter]
+--------------------------------------------------
+
+The <<java-rest-high-query-builders, Building Queries>> page gives a list of all available search queries with
+their corresponding `QueryBuilder` objects and `QueryBuilders` helper methods.
+
+
+===== Specifying Sorting
+
+The `SearchSourceBuilder` allows to add one or more `SortBuilder` instances. There are four special implementations (Field-, Score-, GeoDistance- and ScriptSortBuilder).
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[search-source-sorting]
+--------------------------------------------------
+<1> Sort descending by `_score` (the default)
+<2> Also sort ascending by `_id` field 
+
+===== Source filtering
+
+By default, search requests return the contents of the document `_source` but like in the Rest API you can overwrite this behavior. For example, you can turn off `_source` retrieval completely:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[search-source-filtering-off]
+--------------------------------------------------
+
+The method also accepts an array of one or more wildcard patterns to control which fields get included or excluded in a more fine grained way:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[search-source-filtering-includes]
+--------------------------------------------------
+
+[[java-rest-high-request-highlighting]]
+===== Requesting Highlighting
+
+Highlighting search results can be achieved by setting a `HighlightBuilder` on the
+`SearchSourceBuilder`. Different highlighting behaviour can be defined for each
+fields by adding one or more `HighlightBuilder.Field` instances to a `HighlightBuilder`.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[search-request-highlighting]
+--------------------------------------------------
+<1> Creates a new `HighlightBuilder`
+<2> Create a field highlighter for the `title` field
+<3> Set the field highlighter type
+<4> Add the field highlighter to the highlight builder
+
+There are many options which are explained in detail in the Rest API documentation. The Rest
+API parameters (e.g. `pre_tags`) are usually changed by
+setters with a similar name (e.g. `#preTags(String ...)`).
+
+Highlighted text fragments can <<java-rest-high-retrieve-highlighting,later be retrieved>> from the `SearchResponse`.
+
+[[java-rest-high-document-search-request-building-aggs]]
+===== Requesting Aggregations
+
+Aggregations can be added to the search by first creating the appropriate
+`AggregationBuilder` and then setting it on the `SearchSourceBuilder`. In the
+following example we create a `terms` aggregation on company names with a
+sub-aggregation on the average age of employees in the company:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[search-request-aggregations]
+--------------------------------------------------
+
+The <<java-rest-high-aggregation-builders, Building Aggregations>> page gives a list of all available aggregations with
+their corresponding `AggregationBuilder` objects and `AggregationBuilders` helper methods.
+
+We will later see how to <<java-rest-high-retrieve-aggs,access aggregations>> in the `SearchResponse`.
+
+===== Requesting Suggestions
+
+To add Suggestions to the search request, use one of the `SuggestionBuilder` implementations
+that are easily accessible from the `SuggestBuilders` factory class. Suggestion builders
+need to be added to the top level `SuggestBuilder`, which itself can be set on the  `SearchSourceBuilder`.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[search-request-suggestion]
+--------------------------------------------------
+<1> Creates a new `TermSuggestionBuilder` for the `user` field and
+the text `kmichy`
+<2> Adds the suggestion builder and names it `suggest_user`
+
+We will later see how to <<java-rest-high-retrieve-suggestions,retrieve suggestions>> from the
+`SearchResponse`.
+
+===== Profiling Queries and Aggregations
+
+The {ref}/search-profile.html[Profile API] can be used to profile the execution of queries and aggregations for
+a specific search request. in order to use it, the profile flag must be set to true on the `SearchSourceBuilder`:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[search-request-profiling]
+--------------------------------------------------
+
+Once the `SearchRequest` is executed the corresponding `SearchResponse` will
+<<java-rest-high-retrieve-profile-results,contain the profiling results>>.
+
+[[java-rest-high-document-search-sync]]
+==== Synchronous Execution
+
+When executing a `SearchRequest` in the following manner, the client waits
+for the `SearchResponse` to be returned before continuing with code execution:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[search-execute]
+--------------------------------------------------
+
+[[java-rest-high-document-search-async]]
+==== Asynchronous Execution
+
+
+Executing a `SearchRequest` can also be done in an asynchronous fashion so that
+the client can return directly. Users need to specify how the response or
+potential failures will be handled by passing in appropriate listeners:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[search-execute-async]
+--------------------------------------------------
+<1> Called when the execution is successfully completed.
+<2> Called when the whole `SearchRequest` fails.
+
+==== SearchResponse
+
+The `SearchResponse` that is returned by executing the search provides details
+about the search execution itself as well as access to the documents returned.
+First, there is useful information about the request execution itself, like the
+HTTP status code, execution time or wether the request terminated early or timed
+out:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[search-response-1]
+--------------------------------------------------
+
+Second, the response also provides information about the execution on the
+shard level by offering statistics about the total number of shards that were
+affected by the search, and the successful vs. unsuccessful shards. Possible
+failures can also be handled by iterating over an array off
+`ShardSearchFailures` like in the following example:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[search-response-2]
+--------------------------------------------------
+
+[[java-rest-high-retrieve-searchHits]]
+===== Retrieving SearchHits
+
+To get access to the returned documents, we need to first get the `SearchHits`
+contained in the response:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[search-hits-get]
+--------------------------------------------------
+
+The `SearchHits` provides global information about all hits, like total number
+of hits or the maximum score:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[search-hits-info]
+--------------------------------------------------
+
+Nested inside the `SearchHits` are the individual search results that can
+be iterated over:
+
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[search-hits-singleHit]
+--------------------------------------------------
+
+The `SearchHit` provides access to basic information like index, type, docId and
+score of each search hit:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[search-hits-singleHit-properties]
+--------------------------------------------------
+
+Furthermore, it lets you get back the document source, either as a simple
+JSON-String or as a map of key/value pairs. In this map, regular fields
+are keyed by the field name and contain the field value. Multi-valued fields are
+returned as lists of objects, nested objects as another key/value map. These
+cases need to be cast accordingly:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[search-hits-singleHit-source]
+--------------------------------------------------
+
+[[java-rest-high-retrieve-highlighting]]
+===== Retrieving Highlighting
+
+If <<java-rest-high-request-highlighting,requested>>, highlighted text fragments can be retrieved from each `SearchHit` in the result. The hit object offers
+access to a map of field names to `HighlightField` instances, each of which contains one
+or many highlighted text fragments:
+
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[search-request-highlighting-get]
+--------------------------------------------------
+<1> Get the highlighting for the `title` field
+<2> Get one or many fragments containing the highlighted field content
+
+[[java-rest-high-retrieve-aggs]]
+===== Retrieving Aggregations
+
+Aggregations can be retrieved from the `SearchResponse` by first getting the
+root of the aggregation tree, the `Aggregations` object, and then getting the
+aggregation by name.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[search-request-aggregations-get]
+--------------------------------------------------
+<1> Get the `by_company` terms aggregation
+<2> Get the buckets that is keyed with `Elastic`
+<3> Get the `average_age` sub-aggregation from that bucket
+
+Note that if you access aggregations by name, you need to specify the
+aggregation interface according to the type of aggregation you requested,
+otherwise a `ClassCastException` will be thrown:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[search-request-aggregations-get-wrongCast]
+--------------------------------------------------
+<1> This will throw an exception because "by_company" is a `terms` aggregation
+but we try to retrieve it as a `range` aggregation
+
+It is also possible to access all aggregations as a map that is keyed by the
+aggregation name. In this case, the cast to the proper aggregation interface
+needs to happen explicitly:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[search-request-aggregations-asMap]
+--------------------------------------------------
+
+There are also getters that return all top level aggregations as a list:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[search-request-aggregations-asList]
+--------------------------------------------------
+
+And last but not least you can iterate over all aggregations and then e.g.
+decide how to further process them based on their type:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[search-request-aggregations-iterator]
+--------------------------------------------------
+
+[[java-rest-high-retrieve-suggestions]]
+===== Retrieving Suggestions
+
+To get back the suggestions from a `SearchResponse`, use the `Suggest` object as an entry point and then retrieve the nested suggestion objects:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[search-request-suggestion-get]
+--------------------------------------------------
+<1> Use the `Suggest` class to access suggestions
+<2> Suggestions can be retrieved by name. You need to assign them to the correct
+type of Suggestion class (here `TermSuggestion`), otherwise a `ClassCastException` is thrown
+<3> Iterate over the suggestion entries
+<4> Iterate over the options in one entry
+
+[[java-rest-high-retrieve-profile-results]]
+===== Retrieving Profiling Results
+
+Profiling results are retrieved from a `SearchResponse` using the `getProfileResults()` method. This
+ method returns a `Map` containing a `ProfileShardResult` object for every shard involved in the
+ `SearchRequest` execution. `ProfileShardResult` are stored in the `Map` using a key that uniquely
+ identifies the shard the profile result corresponds to.
+
+Here is a sample code that shows how to iterate over all the profiling results of every shard:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[search-request-profiling-get]
+--------------------------------------------------
+<1> Retrieve the `Map` of `ProfileShardResult` from the `SearchResponse`
+<2> Profiling results can be retrieved by shard's key if the key is known, otherwise it might be simpler
+ to iterate over all the profiling results
+<3> Retrieve the key that identifies which shard the `ProfileShardResult` belongs to
+<4> Retrieve the `ProfileShardResult` for the given shard
+
+The `ProfileShardResult` object itself contains one or more query profile results, one for each query
+executed against the underlying Lucene index:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[search-request-profiling-queries]
+--------------------------------------------------
+<1> Retrieve the list of `QueryProfileShardResult`
+<2> Iterate over each `QueryProfileShardResult`
+
+Each `QueryProfileShardResult` gives access to the detailed query tree execution, returned as a list of
+`ProfileResult` objects:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[search-request-profiling-queries-results]
+--------------------------------------------------
+<1> Iterate over the profile results
+<2> Retrieve the name of the Lucene query
+<3> Retrieve the time in millis spent executing the Lucene query
+<4> Retrieve the profile results for the sub-queries (if any)
+
+The Rest API documentation contains more information about {ref}/_profiling_queries.html[Profiling Queries] with
+a description of the {ref}/_profiling_queries.html#_literal_query_literal_section[query profiling information]
+
+The `QueryProfileShardResult` also gives access to the profiling information for the Lucene collectors:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[search-request-profiling-queries-collectors]
+--------------------------------------------------
+<1> Retrieve the profiling result of the Lucene collector
+<2> Retrieve the name of the Lucene collector
+<3> Retrieve the time in millis spent executing the Lucene collector
+<4> Retrieve the profile results for the sub-collectors (if any)
+
+The Rest API documentation contains more information about profiling information
+{ref}/_profiling_queries.html#_literal_collectors_literal_section[for Lucene collectors].
+
+In a very similar manner to the query tree execution, the `QueryProfileShardResult` objects gives access
+to the detailed aggregations tree execution:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[search-request-profiling-aggs]
+--------------------------------------------------
+<1> Retrieve the `AggregationProfileShardResult`
+<2> Iterate over the aggregation profile results
+<3> Retrieve the type of the aggregation (corresponds to Java class used to execute the aggregation)
+<4> Retrieve the time in millis spent executing the Lucene collector
+<5> Retrieve the profile results for the sub-aggregations (if any)
+
+The Rest API documentation contains more information about {ref}/_profiling_aggregations.html[Profiling Aggregations]
diff --git a/docs/java-rest/high-level/apis/update.asciidoc b/docs/java-rest/high-level/apis/update.asciidoc
new file mode 100644
index 0000000000000..1689306bb79be
--- /dev/null
+++ b/docs/java-rest/high-level/apis/update.asciidoc
@@ -0,0 +1,254 @@
+[[java-rest-high-document-update]]
+=== Update API
+
+[[java-rest-high-document-update-request]]
+==== Update Request
+
+An `UpdateRequest` requires the following arguments:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[update-request]
+--------------------------------------------------
+<1> Index
+<2> Type
+<3> Document id
+
+The Update API allows to update an existing document by using a script
+or by passing a partial document.
+
+==== Updates with a script
+The script can be provided as an inline script:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[update-request-with-inline-script]
+--------------------------------------------------
+<1> Script parameters provided as a `Map` of objects
+<2> Create an inline script using the `painless` language and the previous parameters
+<3> Sets the script to the update request
+
+Or as a stored script:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[update-request-with-stored-script]
+--------------------------------------------------
+<1> Reference to a script stored under the name `increment-field` in the `painless` language
+<2> Sets the script in the update request
+
+==== Updates with a partial document
+When using updates with a partial document, the partial document will be merged with the
+existing document.
+
+The partial document can be provided in different ways:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[update-request-with-doc-as-string]
+--------------------------------------------------
+<1> Partial document source provided as a `String` in JSON format
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[update-request-with-doc-as-map]
+--------------------------------------------------
+<1> Partial document source provided as a `Map` which gets automatically converted
+to JSON format
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[update-request-with-doc-as-xcontent]
+--------------------------------------------------
+<1> Partial document source provided as an `XContentBuilder` object, the Elasticsearch
+built-in helpers to generate JSON content
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[update-request-shortcut]
+--------------------------------------------------
+<1> Partial document source provided as `Object` key-pairs, which gets converted to
+JSON format
+
+==== Upserts
+If the document does not already exist, it is possible to define some content that
+will be inserted as a new document using the `upsert` method:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[update-request-upsert]
+--------------------------------------------------
+<1> Upsert document source provided as a `String`
+
+Similarly to the partial document updates, the content of the `upsert` document
+can be defined using methods that accept `String`, `Map`, `XContentBuilder` or
+`Object` key-pairs.
+
+==== Optional arguments
+The following arguments can optionally be provided:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[update-request-routing]
+--------------------------------------------------
+<1> Routing value
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[update-request-parent]
+--------------------------------------------------
+<1> Parent value
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[update-request-timeout]
+--------------------------------------------------
+<1> Timeout to wait for primary shard to become available as a `TimeValue`
+<2> Timeout to wait for primary shard to become available as a `String`
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[update-request-refresh]
+--------------------------------------------------
+<1> Refresh policy as a `WriteRequest.RefreshPolicy` instance
+<2> Refresh policy as a `String`
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[update-request-retry]
+--------------------------------------------------
+<1> How many times to retry the update operation if the document to update has
+been changed by another operation between the get and indexing phases of the
+update operation
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[update-request-no-source]
+--------------------------------------------------
+<1> Enable source retrieval, disabled by default
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[update-request-source-include]
+--------------------------------------------------
+<1> Configure source inclusion for specific fields
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[update-request-source-exclude]
+--------------------------------------------------
+<1> Configure source exclusion for specific fields
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[update-request-version]
+--------------------------------------------------
+<1> Version
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[update-request-detect-noop]
+--------------------------------------------------
+<1> Disable the noop detection
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[update-request-scripted-upsert]
+--------------------------------------------------
+<1> Indicate that the script must run regardless of whether the document exists or not,
+ie the script takes care of creating the document if it does not already exist.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[update-request-doc-upsert]
+--------------------------------------------------
+<1> Indicate that the partial document must be used as the upsert document if it
+does not exist yet.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[update-request-active-shards]
+--------------------------------------------------
+<1> Sets the number of shard copies that must be active before proceeding with
+the update operation.
+<2> Number of shard copies provided as a `ActiveShardCount`: can be `ActiveShardCount.ALL`,
+`ActiveShardCount.ONE` or `ActiveShardCount.DEFAULT` (default)
+
+[[java-rest-high-document-update-sync]]
+==== Synchronous Execution
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[update-execute]
+--------------------------------------------------
+
+[[java-rest-high-document-update-async]]
+==== Asynchronous Execution
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[update-execute-async]
+--------------------------------------------------
+<1> Called when the execution is successfully completed. The response is
+provided as an argument.
+<2> Called in case of failure. The raised exception is provided as an argument.
+
+[[java-rest-high-document-update-response]]
+==== Update Response
+
+The returned `UpdateResponse` allows to retrieve information about the executed
+ operation as follows:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[update-response]
+--------------------------------------------------
+<1> Handle the case where the document was created for the first time (upsert)
+<2> Handle the case where the document was updated
+<3> Handle the case where the document was deleted
+<4> Handle the case where the document was not impacted by the update,
+ie no operation (noop) was executed on the document
+
+When the source retrieval is enabled in the `UpdateRequest`
+through the fetchSource method, the response contains the
+source of the updated document:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[update-getresult]
+--------------------------------------------------
+<1> Retrieve the updated document as a `GetResult`
+<2> Retrieve the source of the updated document as a `String`
+<3> Retrieve the source of the updated document as a `Map<String, Object>`
+<4> Retrieve the source of the updated document as a `byte[]`
+<5> Handle the scenario where the source of the document is not present in
+the response (this is the case by default)
+
+It is also possible to check for shard failures:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[update-failure]
+--------------------------------------------------
+<1> Handle the situation where number of successful shards is less than
+total shards
+<2> Handle the potential failures
+
+When a `UpdateRequest` is performed against a document that does not exist,
+the response has `404` status code, an `ElasticsearchException` gets thrown
+which needs to be handled as follows:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[update-docnotfound]
+--------------------------------------------------
+<1> Handle the exception thrown because the document not exist
+
+If there is a version conflict, an `ElasticsearchException` will
+be thrown:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/CRUDDocumentationIT.java[update-conflict]
+--------------------------------------------------
+<1> The raised exception indicates that a version conflict error was returned.
diff --git a/docs/java-rest/high-level/builders/aggs.asciidoc b/docs/java-rest/high-level/builders/aggs.asciidoc
new file mode 100644
index 0000000000000..33d5e15ac45d9
--- /dev/null
+++ b/docs/java-rest/high-level/builders/aggs.asciidoc
@@ -0,0 +1,80 @@
+[[java-rest-high-aggregation-builders]]
+=== Building Aggregations
+
+This page lists all the available aggregations with their corresponding `AggregationBuilder` class name and helper method name in the
+`AggregationBuilders` or `PipelineAggregatorBuilders` utility classes.
+
+:agg-ref:        {elasticsearch-javadoc}/org/elasticsearch/search/aggregations
+:parentjoin-ref: {parent-join-client-javadoc}/org/elasticsearch/join/aggregations
+:matrixstats-ref: {matrixstats-client-javadoc}/org/elasticsearch/search/aggregations
+
+==== Metrics Aggregations
+[options="header"]
+|======
+| Aggregation                                                                                        | AggregationBuilder Class                                                                                | Method in AggregationBuilders
+| {ref}/search-aggregations-metrics-avg-aggregation.html[Avg]                                        | {agg-ref}/metrics/avg/AvgAggregationBuilder.html[AvgAggregationBuilder]                                 | {agg-ref}/AggregationBuilders.html#avg-java.lang.String-[AggregationBuilders.avg()]
+| {ref}/search-aggregations-metrics-cardinality-aggregation.html[Cardinality]                        | {agg-ref}/metrics/cardinality/CardinalityAggregationBuilder.html[CardinalityAggregationBuilder]         | {agg-ref}/AggregationBuilders.html#cardinality-java.lang.String-[AggregationBuilders.cardinality()]
+| {ref}/search-aggregations-metrics-extendedstats-aggregation.html[Extended Stats]                   | {agg-ref}/metrics/stats/extended/ExtendedStatsAggregationBuilder.html[ExtendedStatsAggregationBuilder]  | {agg-ref}/AggregationBuilders.html#extendedStats-java.lang.String-[AggregationBuilders.extendedStats()]
+| {ref}/search-aggregations-metrics-geobounds-aggregation.html[Geo Bounds]                           | {agg-ref}/metrics/geobounds/GeoBoundsAggregationBuilder.html[GeoBoundsAggregationBuilder]               | {agg-ref}/AggregationBuilders.html#geoBounds-java.lang.String-[AggregationBuilders.geoBounds()]
+| {ref}/search-aggregations-metrics-geocentroid-aggregation.html[Geo Centroid]                       | {agg-ref}/metrics/geocentroid/GeoCentroidAggregationBuilder.html[GeoCentroidAggregationBuilder]         | {agg-ref}/AggregationBuilders.html#geoCentroid-java.lang.String-[AggregationBuilders.geoCentroid()]
+| {ref}/search-aggregations-metrics-max-aggregation.html[Max]                                        | {agg-ref}/metrics/max/MaxAggregationBuilder.html[MaxAggregationBuilder]                                 | {agg-ref}/AggregationBuilders.html#max-java.lang.String-[AggregationBuilders.max()]
+| {ref}/search-aggregations-metrics-min-aggregation.html[Min]                                        | {agg-ref}/metrics/min/MinAggregationBuilder.html[MinxAggregationBuilder]                                | {agg-ref}/AggregationBuilders.html#min-java.lang.String-[AggregationBuilders.min()]
+| {ref}/search-aggregations-metrics-percentile-aggregation.html[Percentiles]                         | {agg-ref}/metrics/percentiles/PercentilesAggregationBuilder.html[PercentilesAggregationBuilder]         | {agg-ref}/AggregationBuilders.html#percentiles-java.lang.String-[AggregationBuilders.percentiles()]
+| {ref}/search-aggregations-metrics-percentile-rank-aggregation.html[Percentile Ranks]               | {agg-ref}/metrics/percentiles/PercentileRanksAggregationBuilder.html[PercentileRanksAggregationBuilder] | {agg-ref}/AggregationBuilders.html#percentileRanks-java.lang.String-[AggregationBuilders.percentileRanks()]
+| {ref}/search-aggregations-metrics-scripted-metric-aggregation.html[Scripted Metric]                | {agg-ref}/metrics/scripted/ScriptedMetricAggregationBuilder.html[ScriptedMetricAggregationBuilder]      | {agg-ref}/AggregationBuilders.html#scriptedMetric-java.lang.String-[AggregationBuilders.scriptedMetric()]
+| {ref}/search-aggregations-metrics-stats-aggregation.html[Stats]                                    | {agg-ref}/metrics/stats/StatsAggregationBuilder.html[StatsAggregationBuilder]                           | {agg-ref}/AggregationBuilders.html#stats-java.lang.String-[AggregationBuilders.stats()]
+| {ref}/search-aggregations-metrics-sum-aggregation.html[Sum]                                        | {agg-ref}/metrics/sum/SumAggregationBuilder.html[SumAggregationBuilder]                                 | {agg-ref}/AggregationBuilders.html#sum-java.lang.String-[AggregationBuilders.sum()]
+| {ref}/search-aggregations-metrics-top-hits-aggregation.html[Top hits]                              | {agg-ref}/metrics/tophits/TopHitsAggregationBuilder.html[TopHitsAggregationBuilder]                     | {agg-ref}/AggregationBuilders.html#topHits-java.lang.String-[AggregationBuilders.topHits()]
+| {ref}/search-aggregations-metrics-valuecount-aggregation.html[Value Count]                         | {agg-ref}/metrics/valuecount/ValueCountAggregationBuilder.html[ValueCountAggregationBuilder]            | {agg-ref}/AggregationBuilders.html#count-java.lang.String-[AggregationBuilders.count()]
+|======
+
+==== Bucket Aggregations
+[options="header"]
+|======
+| Aggregation                                                                                        | AggregationBuilder Class                                                                                 | Method in AggregationBuilders
+| {ref}/search-aggregations-bucket-adjacency-matrix-aggregation.html[Adjacency Matrix]               | {agg-ref}/bucket/adjacency/AdjacencyMatrixAggregationBuilder.html[AdjacencyMatrixAggregationBuilder]     | {agg-ref}/AggregationBuilders.html#adjacencyMatrix-java.lang.String-java.util.Map-[AggregationBuilders.adjacencyMatrix()]
+| {ref}/search-aggregations-bucket-children-aggregation.html[Children]                               | {parentjoin-ref}/ChildrenAggregationBuilder.html[ChildrenAggregationBuilder]                             |
+| {ref}/search-aggregations-bucket-datehistogram-aggregation.html[Date Histogram]                    | {agg-ref}/bucket/histogram/DateHistogramAggregationBuilder.html[DateHistogramAggregationBuilder]         | {agg-ref}/AggregationBuilders.html#dateHistogram-java.lang.String-[AggregationBuilders.dateHistogram()]
+| {ref}/search-aggregations-bucket-daterange-aggregation.html[Date Range]                            | {agg-ref}/bucket/range/DateRangeAggregationBuilder.html[DateRangeAggregationBuilder]                     | {agg-ref}/AggregationBuilders.html#dateRange-java.lang.String-[AggregationBuilders.dateRange()]
+| {ref}/search-aggregations-bucket-diversified-sampler-aggregation.html[Diversified Sampler]         | {agg-ref}/bucket/sampler/DiversifiedAggregationBuilder.html[DiversifiedAggregationBuilder]               | {agg-ref}/AggregationBuilders.html#diversifiedSampler-java.lang.String-[AggregationBuilders.diversifiedSampler()]
+| {ref}/search-aggregations-bucket-filter-aggregation.html[Filter]                                   | {agg-ref}/bucket/filter/FilterAggregationBuilder.html[FilterAggregationBuilder]                          | {agg-ref}/AggregationBuilders.html#filter-java.lang.String-org.elasticsearch.index.query.QueryBuilder-[AggregationBuilders.filter()]
+| {ref}/search-aggregations-bucket-filters-aggregation.html[Filters]                                 | {agg-ref}/bucket/filters/FiltersAggregationBuilder.html[FiltersAggregationBuilder]                        | {agg-ref}/AggregationBuilders.html#filters-java.lang.String-org.elasticsearch.index.query.QueryBuilder...-[AggregationBuilders.filters()]
+| {ref}/search-aggregations-bucket-geodistance-aggregation.html[Geo Distance]                        | {agg-ref}/bucket/range/GeoDistanceAggregationBuilder.html[GeoDistanceAggregationBuilder]                 | {agg-ref}/AggregationBuilders.html#geoDistance-java.lang.String-org.elasticsearch.common.geo.GeoPoint-[AggregationBuilders.geoDistance()]
+| {ref}/search-aggregations-bucket-geohashgrid-aggregation.html[GeoHash Grid]                        | {agg-ref}/bucket/geogrid/GeoGridAggregationBuilder.html[GeoGridAggregationBuilder]                       | {agg-ref}/AggregationBuilders.html#geohashGrid-java.lang.String-[AggregationBuilders.geohashGrid()]
+| {ref}/search-aggregations-bucket-global-aggregation.html[Global]                                   | {agg-ref}/bucket/global/GlobalAggregationBuilder.html[GlobalAggregationBuilder]                          | {agg-ref}/AggregationBuilders.html#global-java.lang.String-[AggregationBuilders.global()]
+| {ref}/search-aggregations-bucket-histogram-aggregation.html[Histogram]                             | {agg-ref}/bucket/histogram/HistogramAggregationBuilder.html[HistogramAggregationBuilder]                 | {agg-ref}/AggregationBuilders.html#histogram-java.lang.String-[AggregationBuilders.histogram()]
+| {ref}/search-aggregations-bucket-iprange-aggregation.html[IP Range]                                | {agg-ref}/bucket/range/IpRangeAggregationBuilder.html[IpRangeAggregationBuilder]                         | {agg-ref}/AggregationBuilders.html#ipRange-java.lang.String-[AggregationBuilders.ipRange()]
+| {ref}/search-aggregations-bucket-missing-aggregation.html[Missing]                                 | {agg-ref}/bucket/missing/MissingAggregationBuilder.html[MissingAggregationBuilder]                       | {agg-ref}/AggregationBuilders.html#missing-java.lang.String-[AggregationBuilders.missing()]
+| {ref}/search-aggregations-bucket-nested-aggregation.html[Nested]                                   | {agg-ref}/bucket/nested/NestedAggregationBuilder.html[NestedAggregationBuilder]                          | {agg-ref}/AggregationBuilders.html#nested-java.lang.String-java.lang.String-[AggregationBuilders.nested()]
+| {ref}/search-aggregations-bucket-range-aggregation.html[Range]                                     | {agg-ref}/bucket/range/RangeAggregationBuilder.html[RangeAggregationBuilder]                             | {agg-ref}/AggregationBuilders.html#range-java.lang.String-[AggregationBuilders.range()]
+| {ref}/search-aggregations-bucket-reverse-nested-aggregation.html[Reverse nested]                   | {agg-ref}/bucket/nested/ReverseNestedAggregationBuilder.html[ReverseNestedAggregationBuilder]            | {agg-ref}/AggregationBuilders.html#reverseNested-java.lang.String-[AggregationBuilders.reverseNested()]
+| {ref}/search-aggregations-bucket-sampler-aggregation.html[Sampler]                                 | {agg-ref}/bucket/sampler/SamplerAggregationBuilder.html[SamplerAggregationBuilder]                       | {agg-ref}/AggregationBuilders.html#sampler-java.lang.String-[AggregationBuilders.sampler()]
+| {ref}/search-aggregations-bucket-significantterms-aggregation.html[Significant Terms]              | {agg-ref}/bucket/significant/SignificantTermsAggregationBuilder.html[SignificantTermsAggregationBuilder] | {agg-ref}/AggregationBuilders.html#significantTerms-java.lang.String-[AggregationBuilders.significantTerms()]
+| {ref}/search-aggregations-bucket-terms-aggregation.html[Terms]                                     | {agg-ref}/bucket/terms/TermsAggregationBuilder.html[TermsAggregationBuilder]                             | {agg-ref}/AggregationBuilders.html#terms-java.lang.String-[AggregationBuilders.terms()]
+|======
+
+==== Pipeline Aggregations
+[options="header"]
+|======
+| Pipeline on                                                                                        | PipelineAggregationBuilder Class                                                                                                                  | Method in PipelineAggregatorBuilders
+| {ref}/search-aggregations-pipeline-avg-bucket-aggregation.html[Avg Bucket]                         | {agg-ref}/pipeline/bucketmetrics/avg/AvgBucketPipelineAggregationBuilder.html[AvgBucketPipelineAggregationBuilder]                                | {agg-ref}/pipeline/PipelineAggregatorBuilders.html#avgBucket-java.lang.String-java.lang.String-[PipelineAggregatorBuilders.avgBucket()]
+| {ref}/search-aggregations-pipeline-derivative-aggregation.html[Derivative]                         | {agg-ref}/pipeline/derivative/DerivativePipelineAggregationBuilder.html[DerivativePipelineAggregationBuilder]                                     | {agg-ref}/pipeline/PipelineAggregatorBuilders.html#derivative-java.lang.String-java.lang.String-[PipelineAggregatorBuilders.derivative()]
+| {ref}/search-aggregations-pipeline-max-bucket-aggregation.html[Max Bucket]                         | {agg-ref}/pipeline/bucketmetrics/max/MaxBucketPipelineAggregationBuilder.html[MaxBucketPipelineAggregationBuilder]                                | {agg-ref}/pipeline/PipelineAggregatorBuilders.html#maxBucket-java.lang.String-java.lang.String-[PipelineAggregatorBuilders.maxBucket()]
+| {ref}/search-aggregations-pipeline-min-bucket-aggregation.html[Min Bucket]                         | {agg-ref}/pipeline/bucketmetrics/min/MinBucketPipelineAggregationBuilder.html[MinBucketPipelineAggregationBuilder]                                | {agg-ref}/pipeline/PipelineAggregatorBuilders.html#minBucket-java.lang.String-java.lang.String-[PipelineAggregatorBuilders.minBucket()]
+| {ref}/search-aggregations-pipeline-sum-bucket-aggregation.html[Sum Bucket]                         | {agg-ref}/pipeline/bucketmetrics/sum/SumBucketPipelineAggregationBuilder.html[SumBucketPipelineAggregationBuilder]                                | {agg-ref}/pipeline/PipelineAggregatorBuilders.html#sumBucket-java.lang.String-java.lang.String-[PipelineAggregatorBuilders.sumBucket()]
+| {ref}/search-aggregations-pipeline-stats-bucket-aggregation.html[Stats Bucket]                     | {agg-ref}/pipeline/bucketmetrics/stats/StatsBucketPipelineAggregationBuilder.html[StatsBucketPipelineAggregationBuilder]                          | {agg-ref}/pipeline/PipelineAggregatorBuilders.html#statsBucket-java.lang.String-java.lang.String-[PipelineAggregatorBuilders.statsBucket()]
+| {ref}/search-aggregations-pipeline-extended-stats-bucket-aggregation.html[Extended Stats Bucket]   | {agg-ref}/pipeline/bucketmetrics/stats/extended/ExtendedStatsBucketPipelineAggregationBuilder.html[ExtendedStatsBucketPipelineAggregationBuilder] | {agg-ref}/pipeline/PipelineAggregatorBuilders.html#extendedStatsBucket-java.lang.String-java.lang.String-[PipelineAggregatorBuilders.extendedStatsBucket()]
+| {ref}/search-aggregations-pipeline-percentiles-bucket-aggregation.html[Percentiles Bucket]         | {agg-ref}/pipeline/bucketmetrics/percentile/PercentilesBucketPipelineAggregationBuilder.html[PercentilesBucketPipelineAggregationBuilder]         | {agg-ref}/pipeline/PipelineAggregatorBuilders.html#percentilesBucket-java.lang.String-java.lang.String-[PipelineAggregatorBuilders.percentilesBucket()]
+| {ref}/search-aggregations-pipeline-movavg-aggregation.html[Moving Average]                         | {agg-ref}/pipeline/movavg/MovAvgPipelineAggregationBuilder.html[MovAvgPipelineAggregationBuilder]                                                 | {agg-ref}/pipeline/PipelineAggregatorBuilders.html#movingAvg-java.lang.String-java.lang.String-[PipelineAggregatorBuilders.movingAvg()]
+| {ref}/search-aggregations-pipeline-cumulative-sum-aggregation.html[Cumulative Sum]                 | {agg-ref}/pipeline/cumulativesum/CumulativeSumPipelineAggregationBuilder.html[CumulativeSumPipelineAggregationBuilder]                            | {agg-ref}/pipeline/PipelineAggregatorBuilders.html#cumulativeSum-java.lang.String-java.lang.String-[PipelineAggregatorBuilders.cumulativeSum()]
+| {ref}/search-aggregations-pipeline-bucket-script-aggregation.html[Bucket Script]                   | {agg-ref}/pipeline/bucketscript/BucketScriptPipelineAggregationBuilder.html[BucketScriptPipelineAggregationBuilder]                               | {agg-ref}/pipeline/PipelineAggregatorBuilders.html#bucketScript-java.lang.String-java.util.Map-org.elasticsearch.script.Script-[PipelineAggregatorBuilders.bucketScript()]
+| {ref}/search-aggregations-pipeline-bucket-selector-aggregation.html[Bucket Selector]               | {agg-ref}/pipeline/bucketselector/BucketSelectorPipelineAggregationBuilder.html[BucketSelectorPipelineAggregationBuilder]                         | {agg-ref}/pipeline/PipelineAggregatorBuilders.html#bucketSelector-java.lang.String-java.util.Map-org.elasticsearch.script.Script-[PipelineAggregatorBuilders.bucketSelector()]
+| {ref}/search-aggregations-pipeline-serialdiff-aggregation.html[Serial Differencing]                | {agg-ref}/pipeline/serialdiff/SerialDiffPipelineAggregationBuilder.html[SerialDiffPipelineAggregationBuilder]                                     | {agg-ref}/pipeline/PipelineAggregatorBuilders.html#diff-java.lang.String-java.lang.String-[PipelineAggregatorBuilders.diff()]
+|======
+
+==== Matrix Aggregations
+[options="header"]
+|======
+| Aggregation                                                                                        | AggregationBuilder Class                                                                  | Method in MatrixStatsAggregationBuilders
+| {ref}/search-aggregations-matrix-stats-aggregation.html[Matrix Stats]                              | {matrixstats-ref}/matrix/stats/MatrixStatsAggregationBuilder.html[MatrixStatsAggregationBuilder]  | {matrixstats-ref}/MatrixStatsAggregationBuilders.html#matrixStats-java.lang.String-[MatrixStatsAggregationBuilders.matrixStats()]
+|======
diff --git a/docs/java-rest/high-level/builders/queries.asciidoc b/docs/java-rest/high-level/builders/queries.asciidoc
new file mode 100644
index 0000000000000..88204baa8745d
--- /dev/null
+++ b/docs/java-rest/high-level/builders/queries.asciidoc
@@ -0,0 +1,100 @@
+[[java-rest-high-query-builders]]
+=== Building Queries
+
+This page lists all the available search queries with their corresponding `QueryBuilder` class name and helper method name in the
+`QueryBuilders` utility class.
+
+:query-ref:        {elasticsearch-javadoc}/org/elasticsearch/index/query
+:parentjoin-ref:   {parent-join-client-javadoc}/org/elasticsearch/join/query
+:percolate-ref:    {percolator-client-javadoc}/org/elasticsearch/percolator
+
+==== Match All Query
+[options="header"]
+|======
+| Search Query                                                        | QueryBuilder Class | Method in QueryBuilders
+| {ref}/query-dsl-match-all-query.html[Match All]                     | {query-ref}/MatchAllQueryBuilder.html[MatchAllQueryBuilder]                         | {query-ref}/QueryBuilders.html#matchAllQuery--[QueryBuilders.matchAllQuery()]
+|======
+
+==== Full Text Queries
+[options="header"]
+|======
+| Search Query                                                        | QueryBuilder Class | Method in QueryBuilders
+| {ref}/query-dsl-match-query.html[Match]                             | {query-ref}/MatchQueryBuilder.html[MatchQueryBuilder]                               | {query-ref}/QueryBuilders.html#matchQuery-java.lang.String-java.lang.Object-[QueryBuilders.matchQuery()]
+| {ref}/query-dsl-match-query-phrase.html[Match Phrase]               | {query-ref}/MatchPhraseQueryBuilder.html[MatchPhraseQueryBuilder]                   | {query-ref}/QueryBuilders.html#matchPhraseQuery-java.lang.String-java.lang.Object-[QueryBuilders.matchPhraseQuery()]
+| {ref}/query-dsl-match-query-phrase-prefix.html[Match Phrase Prefix] | {query-ref}/MatchPhrasePrefixQueryBuilder.html[MatchPhrasePrefixQueryBuilder]       | {query-ref}/QueryBuilders.html#matchPhrasePrefixQuery-java.lang.String-java.lang.Object-[QueryBuilders.matchPhrasePrefixQuery()]
+| {ref}/query-dsl-multi-match-query.html[Multi Match]                 | {query-ref}/MultiMatchQueryBuilder.html[MultiMatchQueryBuilder]                     | {query-ref}/QueryBuilders.html#multiMatchQuery-java.lang.Object-java.lang.String\…-[QueryBuilders.multiMatchQuery()]
+| {ref}/query-dsl-common-terms-query.html[Common Terms]               | {query-ref}/CommonTermsQueryBuilder.html[CommonTermsQueryBuilder]                   | {query-ref}/QueryBuilders.html#commonTermsQuery-java.lang.String-java.lang.Object-[QueryBuilders.commonTermsQuery()]
+| {ref}/query-dsl-query-string-query.html[Query String]               | {query-ref}/QueryStringQueryBuilder.html[QueryStringQueryBuilder]                   | {query-ref}/QueryBuilders.html#queryStringQuery-java.lang.String-[QueryBuilders.queryStringQuery()]
+| {ref}/query-dsl-simple-query-string-query.html[Simple Query String] | {query-ref}/SimpleQueryStringBuilder.html[SimpleQueryStringBuilder]                 | {query-ref}/QueryBuilders.html#simpleQueryStringQuery-java.lang.String-[QueryBuilders.simpleQueryStringQuery()]
+|======
+
+==== Term level queries
+[options="header"]
+|======
+| Search Query                                                        | QueryBuilder Class | Method in QueryBuilders
+| {ref}/query-dsl-term-query.html[Term]                               | {query-ref}/TermQueryBuilder.html[TermQueryBuilder]                                 | {query-ref}/QueryBuilders.html#termQuery-java.lang.String-java.lang.String-[QueryBuilders.termQuery()]
+| {ref}/query-dsl-terms-query.html[Terms]                             | {query-ref}/TermsQueryBuilder.html[TermsQueryBuilder]                               | {query-ref}/QueryBuilders.html#termsQuery-java.lang.String-java.util.Collection-[QueryBuilders.termsQuery()]
+| {ref}/query-dsl-range-query.html[Range]                             | {query-ref}/RangeQueryBuilder.html[RangeQueryBuilder]                               | {query-ref}/QueryBuilders.html#rangeQuery-java.lang.String-[QueryBuilders.rangeQuery()]
+| {ref}/query-dsl-exists-query.html[Exists]                           | {query-ref}/ExistsQueryBuilder.html[ExistsQueryBuilder]                             | {query-ref}/QueryBuilders.html#existsQuery-java.lang.String-[QueryBuilders.existsQuery()]
+| {ref}/query-dsl-prefix-query.html[Prefix]                           | {query-ref}/PrefixQueryBuilder.html[PrefixQueryBuilder]                             | {query-ref}/QueryBuilders.html#prefixQuery-java.lang.String-java.lang.String-[QueryBuilders.prefixQuery()]
+| {ref}/query-dsl-wildcard-query.html[Wildcard]                       | {query-ref}/WildcardQueryBuilder.html[WildcardQueryBuilder]                         | {query-ref}/QueryBuilders.html#wildcardQuery-java.lang.String-java.lang.String-[QueryBuilders.wildcardQuery()]
+| {ref}/query-dsl-regexp-query.html[Regexp]                           | {query-ref}/RegexpQueryBuilder.html[RegexpQueryBuilder]                             | {query-ref}/QueryBuilders.html#regexpQuery-java.lang.String-java.lang.String-[QueryBuilders.regexpQuery()]
+| {ref}/query-dsl-fuzzy-query.html[Fuzzy]                             | {query-ref}/FuzzyQueryBuilder.html[FuzzyQueryBuilder]                               | {query-ref}/QueryBuilders.html#fuzzyQuery-java.lang.String-java.lang.String-[QueryBuilders.fuzzyQuery()]
+| {ref}/query-dsl-type-query.html[Type]                               | {query-ref}/TypeQueryBuilder.html[TypeQueryBuilder]                                 | {query-ref}/QueryBuilders.html#typeQuery-java.lang.String-[QueryBuilders.typeQuery()]
+| {ref}/query-dsl-ids-query.html[Ids]                                 | {query-ref}/IdsQueryBuilder.html[IdsQueryBuilder]                                   | {query-ref}/QueryBuilders.html#idsQuery--[QueryBuilders.idsQuery()]
+|======
+
+==== Compound queries
+[options="header"]
+|======
+| Search Query                                                        | QueryBuilder Class | Method in QueryBuilders
+| {ref}/query-dsl-constant-score-query.html[Constant Score]           | {query-ref}/ConstantScoreQueryBuilder.html[ConstantScoreQueryBuilder]               | {query-ref}/QueryBuilders.html#constantScoreQuery-org.elasticsearch.index.query.QueryBuilder-[QueryBuilders.constantScoreQuery()]
+| {ref}/query-dsl-bool-query.html[Bool]                               | {query-ref}/BoolQueryBuilder.html[BoolQueryBuilder]                                 | {query-ref}/QueryBuilders.html#boolQuery--[QueryBuilders.boolQuery()]
+| {ref}/query-dsl-dis-max-query.html[Dis Max]                         | {query-ref}/DisMaxQueryBuilder.html[DisMaxQueryBuilder]                             | {query-ref}/QueryBuilders.html#disMaxQuery--[QueryBuilders.disMaxQuery()]
+| {ref}/query-dsl-function-score-query.html[Function Score]           | {query-ref}/functionscore/FunctionScoreQueryBuilder.html[FunctionScoreQueryBuilder] | {query-ref}/QueryBuilders.html#functionScoreQuery-org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder.FilterFunctionBuilder:A-[QueryBuilders.functionScoreQuery()]
+| {ref}/query-dsl-boosting-query.html[Boosting]                       | {query-ref}/BoostingQueryBuilder.html[BoostingQueryBuilder]                         | {query-ref}/QueryBuilders.html#boostingQuery-org.elasticsearch.index.query.QueryBuilder-org.elasticsearch.index.query.QueryBuilder-[QueryBuilders.boostingQuery()]
+|======
+
+==== Joining queries
+[options="header"]
+|======
+| Search Query                                                        | QueryBuilder Class | Method in QueryBuilders
+| {ref}/query-dsl-nested-query.html[Nested]                           | {query-ref}/NestedQueryBuilder.html[NestedQueryBuilder]                             | {query-ref}/QueryBuilders.html#nestedQuery-java.lang.String-org.elasticsearch.index.query.QueryBuilder-org.apache.lucene.search.join.ScoreMode-[QueryBuilders.nestedQuery()]
+| {ref}/query-dsl-has-child-query.html[Has Child]                     | {parentjoin-ref}/HasChildQueryBuilder.html[HasChildQueryBuilder]                    |
+| {ref}/query-dsl-has-parent-query.html[Has Parent]                   | {parentjoin-ref}/HasParentQueryBuilder.html[HasParentQueryBuilder]                  |
+| {ref}/query-dsl-parent-id-query.html[Parent Id]                     | {parentjoin-ref}/ParentIdQueryBuilder.html[ParentIdQueryBuilder]                    |
+|======
+
+==== Geo queries
+[options="header"]
+|======
+| Search Query                                                        | QueryBuilder Class | Method in QueryBuilders
+| {ref}/query-dsl-geo-shape-query.html[GeoShape]                      | {query-ref}/GeoShapeQueryBuilder.html[GeoShapeQueryBuilder]                         | {query-ref}/QueryBuilders.html#geoShapeQuery-java.lang.String-java.lang.String-java.lang.String-[QueryBuilders.geoShapeQuery()]
+| {ref}/query-dsl-geo-bounding-box-query.html[Geo Bounding Box]       | {query-ref}/GeoBoundingBoxQueryBuilder.html[GeoBoundingBoxQueryBuilder]             | {query-ref}/QueryBuilders.html#geoBoundingBoxQuery-java.lang.String-[QueryBuilders.geoBoundingBoxQuery()]
+| {ref}/query-dsl-geo-distance-query.html[Geo Distance]               | {query-ref}/GeoDistanceQueryBuilder.html[GeoDistanceQueryBuilder]                   | {query-ref}/QueryBuilders.html#geoDistanceQuery-java.lang.String-[QueryBuilders.geoDistanceQuery()]
+| {ref}/query-dsl-geo-polygon-query.html[Geo Polygon]                 | {query-ref}/GeoPolygonQueryBuilder.html[GeoPolygonQueryBuilder]                     | {query-ref}/QueryBuilders.html#geoPolygonQuery-java.lang.String-java.util.List-[QueryBuilders.geoPolygonQuery()]
+|======
+
+==== Specialized queries
+[options="header"]
+|======
+| Search Query                                                        | QueryBuilder Class | Method in QueryBuilders
+| {ref}/query-dsl-mlt-query.html[More Like This]                      | {query-ref}/MoreLikeThisQueryBuilder.html[MoreLikeThisQueryBuilder]                 | {query-ref}/QueryBuilders.html#moreLikeThisQuery-org.elasticsearch.index.query.MoreLikeThisQueryBuilder.Item:A-[QueryBuilders.moreLikeThisQuery()]
+| {ref}/query-dsl-script-query.html[Script]                           | {query-ref}/ScriptQueryBuilder.html[ScriptQueryBuilder]                             | {query-ref}/QueryBuilders.html#scriptQuery-org.elasticsearch.script.Script-[QueryBuilders.scriptQuery()]
+| {ref}/query-dsl-percolate-query.html[Percolate]                     | {percolate-ref}/PercolateQueryBuilder.html[PercolateQueryBuilder]                   |
+|======
+
+==== Span queries
+[options="header"]
+|======
+| Search Query                                                        | QueryBuilder Class | Method in QueryBuilders
+| {ref}/query-dsl-span-term-query.html[Span Term]                     | {query-ref}/SpanTermQueryBuilder.html[SpanTermQueryBuilder]                         | {query-ref}/QueryBuilders.html#spanTermQuery-java.lang.String-double-[QueryBuilders.spanTermQuery()]
+| {ref}/query-dsl-span-multi-term-query.html[Span Multi Term]         | {query-ref}/SpanMultiTermQueryBuilder.html[SpanMultiTermQueryBuilder]               | {query-ref}/QueryBuilders.html#spanMultiTermQueryBuilder-org.elasticsearch.index.query.MultiTermQueryBuilder-[QueryBuilders.spanMultiTermQueryBuilder()]
+| {ref}/query-dsl-span-first-query.html[Span First]                   | {query-ref}/SpanFirstQueryBuilder.html[SpanFirstQueryBuilder]                       | {query-ref}/QueryBuilders.html#spanFirstQuery-org.elasticsearch.index.query.SpanQueryBuilder-int-[QueryBuilders.spanFirstQuery()]
+| {ref}/query-dsl-span-near-query.html[Span Near]                     | {query-ref}/SpanNearQueryBuilder.html[SpanNearQueryBuilder]                         | {query-ref}/QueryBuilders.html#spanNearQuery-org.elasticsearch.index.query.SpanQueryBuilder-int-[QueryBuilders.spanNearQuery()]
+| {ref}/query-dsl-span-or-query.html[Span Or]                         | {query-ref}/SpanOrQueryBuilder.html[SpanOrQueryBuilder]                             | {query-ref}/QueryBuilders.html#spanOrQuery-org.elasticsearch.index.query.SpanQueryBuilder-[QueryBuilders.spanOrQuery()]
+| {ref}/query-dsl-span-not-query.html[Span Not]                       | {query-ref}/SpanNotQueryBuilder.html[SpanNotQueryBuilder]                           | {query-ref}/QueryBuilders.html#spanNotQuery-org.elasticsearch.index.query.SpanQueryBuilder-org.elasticsearch.index.query.SpanQueryBuilder-[QueryBuilders.spanNotQuery()]
+| {ref}/query-dsl-span-containing-query.html[Span Containing]         | {query-ref}/SpanContainingQueryBuilder.html[SpanContainingQueryBuilder]             | {query-ref}/QueryBuilders.html#spanContainingQuery-org.elasticsearch.index.query.SpanQueryBuilder-org.elasticsearch.index.query.SpanQueryBuilder-[QueryBuilders.spanContainingQuery()]
+| {ref}/query-dsl-span-within-query.html[Span Within]                 | {query-ref}/SpanWithinQueryBuilder.html[SpanWithinQueryBuilder]                     | {query-ref}/QueryBuilders.html#spanWithinQuery-org.elasticsearch.index.query.SpanQueryBuilder-org.elasticsearch.index.query.SpanQueryBuilder-[QueryBuilders.spanWithinQuery()]
+| {ref}/query-dsl-span-field-masking-query.html[Span Field Masking]   | {query-ref}/FieldMaskingSpanQueryBuilder.html[FieldMaskingSpanQueryBuilder]         | {query-ref}/QueryBuilders.html#fieldMaskingSpanQuery-org.elasticsearch.index.query.SpanQueryBuilder-java.lang.String-[QueryBuilders.fieldMaskingSpanQuery()]
+|======
diff --git a/docs/java-rest/high-level/getting-started.asciidoc b/docs/java-rest/high-level/getting-started.asciidoc
new file mode 100644
index 0000000000000..fb05953edd7d4
--- /dev/null
+++ b/docs/java-rest/high-level/getting-started.asciidoc
@@ -0,0 +1,104 @@
+[[java-rest-high-getting-started]]
+== Getting started
+
+This section describes how to get started with the high-level REST client from
+getting the artifact to using it in an application.
+
+[[java-rest-high-compatibility]]
+=== Compatibility
+The Java High Level REST Client requires Java 1.8 and depends on the Elasticsearch
+core project. The client version is the same as the Elasticsearch version that the
+client was developed for. It accepts the same request arguments as the `TransportClient`
+and returns the same response objects. See the <<java-rest-high-level-migration>>
+if you need to migrate an application from `TransportClient` to the new REST client.
+
+The High Level Client is guaranteed to be able to communicate with any Elasticsearch
+node running on the same major version and greater or equal minor version. It
+doesn't need to be in the same minor version as the Elasticsearch nodes it
+communicates with, as it is forward compatible meaning that it supports
+communicating with later versions of Elasticsearch than the one it was developed for.
+
+The 5.6 client can communicate with any 5.6.x Elasticsearch node. Previous 5.x minor
+versions like 5.5.x, 5.4.x etc. are not (fully) supported.
+
+The 6.0 client is able to communicate with any 6.x Elasticsearch node while the 6.1
+client is for sure able to communicate with 6.1, 6.2 and any later 6.x version, but
+there may be incompatibility issues when communicating with a previous Elasticsearch
+node version, for instance between 6.1 and 6.0, in case the 6.1 client supports new
+request body fields for some APIs that are not known by the 6.0 node(s).
+
+It is recommended to upgrade the High Level Client when upgrading the Elasticsearch
+cluster to a new major version, as REST API breaking changes may cause unexpected
+results depending on the node that is hit by the request, and newly added APIs will
+only be supported by the newer version of the client. The client should always be
+updated last, once all of the nodes in the cluster have been upgraded to the new
+major version.
+
+[[java-rest-high-javadoc]]
+=== Javadoc
+
+The javadoc for the REST high level client can be found at {rest-high-level-client-javadoc}/index.html.
+
+[[java-rest-high-getting-started-maven]]
+=== Maven Repository
+
+The high-level Java REST client is hosted on
+http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.elasticsearch.client%22[Maven
+Central]. The minimum Java version required is `1.8`.
+
+The High Level REST Client is subject to the same release cycle as
+Elasticsearch. Replace the version with the desired client version.
+
+[[java-rest-high-getting-started-maven-maven]]
+==== Maven configuration
+
+Here is how you can configure the dependency using maven as a dependency manager.
+Add the following to your `pom.xml` file:
+
+["source","xml",subs="attributes"]
+--------------------------------------------------
+<dependency>
+    <groupId>org.elasticsearch.client</groupId>
+    <artifactId>elasticsearch-rest-high-level-client</artifactId>
+    <version>{version}</version>
+</dependency>
+--------------------------------------------------
+
+[[java-rest-high-getting-started-maven-gradle]]
+==== Gradle configuration
+
+Here is how you can configure the dependency using gradle as a dependency manager.
+Add the following to your `build.gradle` file:
+
+["source","groovy",subs="attributes"]
+--------------------------------------------------
+dependencies {
+    compile 'org.elasticsearch.client:elasticsearch-rest-high-level-client:{version}'
+}
+--------------------------------------------------
+
+[[java-rest-high-getting-started-dependencies]]
+=== Dependencies
+
+The High Level Java REST Client depends on the following artifacts and their
+transitive dependencies:
+
+- org.elasticsearch.client:elasticsearch-rest-client
+- org.elasticsearch:elasticsearch
+
+
+[[java-rest-high-getting-started-initialization]]
+=== Initialization
+
+A `RestHighLevelClient` instance needs a <<java-rest-low-usage-initialization,REST low-level client>>
+to be built as follows:
+
+[source,java]
+--------------------------------------------------
+RestHighLevelClient client =
+    new RestHighLevelClient(lowLevelRestClient); <1>
+--------------------------------------------------
+<1> We pass the <<java-rest-low-usage-initialization,REST low-level client>> instance
+
+In the rest of this documentation about the Java High Level Client, the `RestHighLevelClient` instance
+will be referenced as `client`.
diff --git a/docs/java-rest/high-level/index.asciidoc b/docs/java-rest/high-level/index.asciidoc
new file mode 100644
index 0000000000000..46fd451524e26
--- /dev/null
+++ b/docs/java-rest/high-level/index.asciidoc
@@ -0,0 +1,31 @@
+[[java-rest-high]]
+= Java High Level REST Client
+
+[partintro]
+--
+The Java High Level REST Client works on top of the Java Low Level REST client.
+Its main goal is to expose API specific methods, that accept request objects as
+an argument and return response objects, so that request marshalling and
+response un-marshalling is handled by the client itself.
+
+Each API can be called synchronously or asynchronously. The synchronous
+methods return a response object, while the asynchronous methods, whose names
+end with the `async` suffix, require a listener argument that is notified
+(on the thread pool managed by the low level client) once a response or an
+error is received.
+
+The Java High Level REST Client depends on the Elasticsearch core project.
+It accepts the same request arguments as the `TransportClient` and returns
+the same response objects.
+
+--
+
+:doc-tests: {docdir}/../../client/rest-high-level/src/test/java/org/elasticsearch/client/documentation
+
+include::getting-started.asciidoc[]
+include::supported-apis.asciidoc[]
+include::java-builders.asciidoc[]
+include::migration.asciidoc[]
+include::../license.asciidoc[]
+
+:doc-tests!:
diff --git a/docs/java-rest/high-level/java-builders.asciidoc b/docs/java-rest/high-level/java-builders.asciidoc
new file mode 100644
index 0000000000000..be5c19293cd30
--- /dev/null
+++ b/docs/java-rest/high-level/java-builders.asciidoc
@@ -0,0 +1,32 @@
+[[java-rest-high-java-builders]]
+== Using Java Builders
+
+The Java High Level REST Client depends on the Elasticsearch core project which provides
+different types of Java `Builders` objects, including:
+
+Query Builders::
+
+The query builders are used to create the query to execute within a search request. There
+is a query builder for every type of query supported by the Query DSL. Each query builder
+implements the `QueryBuilder` interface and allows to set the specific options for a given
+type of query. Once created, the `QueryBuilder` object can be set as the query parameter of
+`SearchSourceBuilder`. The <<java-rest-high-document-search-request-building-queries, Search Request>>
+page shows an example of how to build a full search request using `SearchSourceBuilder` and
+`QueryBuilder` objects. The <<java-rest-high-query-builders, Building Search Queries>> page
+gives a list of all available search queries with their corresponding `QueryBuilder` objects
+and `QueryBuilders` helper methods.
+
+Aggregation Builders::
+
+Similarly to query builders, the aggregation builders are used to create the aggregations to
+compute during a search request execution. There is an aggregation builder for every type of
+aggregation (or pipeline aggregation) supported by Elasticsearch. All builders extend the
+`AggregationBuilder` class (or `PipelineAggregationBuilder`class). Once created, `AggregationBuilder`
+objects can be set as the aggregation parameter of `SearchSourceBuilder`. There is a example
+of how `AggregationBuilder` objects are used with `SearchSourceBuilder` objects to define the aggregations
+to compute with a search query in <<java-rest-high-document-search-request-building-aggs, Search Request>> page.
+The <<java-rest-high-aggregation-builders, Building Aggregations>> page gives a list of all available
+aggregations with their corresponding `AggregationBuilder` objects and `AggregationBuilders` helper methods.
+
+include::builders/queries.asciidoc[]
+include::builders/aggs.asciidoc[]
diff --git a/docs/java-rest/high-level/migration.asciidoc b/docs/java-rest/high-level/migration.asciidoc
new file mode 100644
index 0000000000000..b65a2923d06c3
--- /dev/null
+++ b/docs/java-rest/high-level/migration.asciidoc
@@ -0,0 +1,362 @@
+[[java-rest-high-level-migration]]
+== Migration Guide
+
+This section describes how to migrate existing code from the `TransportClient`
+to the new Java High Level REST Client released with the version 5.6.0
+of Elasticsearch.
+
+=== Motivations around a new Java client
+
+The existing `TransportClient` has been part of Elasticsearch since https://github.com/elastic/elasticsearch/blob/b3337c312765e51cec7bde5883bbc0a08f56fb65/modules/elasticsearch/src/main/java/org/elasticsearch/client/transport/TransportClient.java[its very first commit].
+ It is a special client as it uses the transport protocol to communicate with Elasticsearch,
+ which causes compatibility problems if the client is not on the same version as the
+ Elasticsearch instances it talks to.
+
+We released a low-level REST client in 2016, which is based on the well known Apache HTTP
+client and it allows to communicate with an Elasticsearch cluster in any version using HTTP.
+On top of that we released the high-level REST client which is based on the low-level client
+but takes care of request marshalling and response un-marshalling.
+
+If you're interested in knowing more about these changes, we wrote a blog post about the
+https://www.elastic.co/blog/state-of-the-official-elasticsearch-java-clients[state of the official Elasticsearch Java clients].
+
+=== Prerequisite
+
+The Java High Level Rest Client requires Java `1.8` and can be used to send requests
+to an <<java-rest-high-compatibility,Elasticsearch cluster in a compatible version>>.
+
+=== How to migrate
+
+Adapting existing code to use the `RestHighLevelClient` instead of the `TransportClient`
+requires the following steps:
+
+- Update dependencies
+- Update client initialization
+- Update application code
+
+=== Updating the dependencies
+
+Java application that uses the `TransportClient` depends on the
+`org.elasticsearch.client:transport` artifact. This dependency
+must be replaced by a new dependency on the high-level client.
+
+The <<java-rest-high-getting-started,Getting Started>> page shows
+ typical configurations for Maven and Gradle and presents the
+ <<java-rest-high-getting-started-dependencies, dependencies>> brought by the
+ high-level client.
+
+=== Changing the client's initialization code
+
+The `TransportClient` is typically initialized as follows:
+[source,java]
+--------------------------------------------------
+Settings settings = Settings.builder()
+        .put("cluster.name", "prod").build();
+
+TransportClient transportClient = new PreBuiltTransportClient(settings)
+        .addTransportAddress(new TransportAddress(InetAddress.getByName("host"), 9300));
+--------------------------------------------------
+
+The initialization of a `RestHighLevelClient` is different. It first requires the initialization
+of a <<java-rest-low-usage-initialization,low-level client>>:
+
+[source,java]
+--------------------------------------------------
+RestClient lowLevelRestClient = RestClient.builder(
+                new HttpHost("host", 9200, "http")).build();
+--------------------------------------------------
+
+NOTE: The `RestClient` uses Elasticsearch's HTTP service which is
+ bounded by default on `9200`. This port is different from the port
+ used to connect to Elasticsearch with a `TransportClient`.
+
+Which is then passed to the constructor of the `RestHighLevelClient`:
+
+[source,java]
+--------------------------------------------------
+RestHighLevelClient client =
+    new RestHighLevelClient(lowLevelRestClient);
+--------------------------------------------------
+
+Both `RestClient` and `RestHighLevelClient` are thread safe. They are
+ typically instantiated by the application at startup time or when the
+ first request is executed.
+
+Once the `RestHighLevelClient` is initialized, it can then be used to
+execute any of the <<java-rest-high-supported-apis,supported APIs>>.
+
+As with the `TransportClient`, the `RestClient` must be closed when it
+is not needed anymore or when the application is stopped.
+
+So the code that closes the `TransportClient`:
+
+[source,java]
+--------------------------------------------------
+transportClient.close();
+--------------------------------------------------
+
+Must be replaced with:
+
+[source,java]
+--------------------------------------------------
+lowLevelRestClient.close();
+--------------------------------------------------
+
+=== Changing the application's code
+
+The `RestHighLevelClient` supports the same request and response objects
+as the `TransportClient`, but exposes slightly different methods to
+send the requests.
+
+More importantly, the high-level client:
+
+- does not support request builders. The legacy methods like
+`client.prepareIndex()` must be changed to use
+ request constructors like `new IndexRequest()` to create requests
+ objects. The requests are then executed using synchronous or
+ asynchronous dedicated methods like `client.index()` or `client.indexAsync()`.
+- does not provide indices or cluster management APIs. Management
+operations can be executed by external scripts or
+<<java-rest-high-level-migration-manage-indices, using the low-level client>>.
+
+==== How to migrate the way requests are built
+
+The Java API provides two ways to build a request: by using the request's constructor or by using
+a request builder. Migrating from the `TransportClient` to the high-level client can be
+straightforward if application's code uses the former, while changing usages of the latter can
+require more work.
+
+[[java-rest-high-level-migration-request-ctor]]
+===== With request constructors
+
+When request constructors are used, like in the following example:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/MigrationDocumentationIT.java[migration-request-ctor]
+--------------------------------------------------
+<1> Create an `IndexRequest` using its constructor
+
+The migration is very simple. The execution using the `TransportClient`:
+
+[source,java]
+--------------------------------------------------
+IndexResponse response = transportClient.index(indexRequest).actionGet();
+--------------------------------------------------
+
+Can be easily replaced to use the `RestHighLevelClient`:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/MigrationDocumentationIT.java[migration-request-ctor-execution]
+--------------------------------------------------
+
+[[java-rest-high-level-migration-request-builder]]
+===== With request builders
+
+The Java API provides a request builder for every type of request. They are exposed by the
+`TransportClient` through the many `prepare()` methods. Here are some examples:
+
+[source,java]
+--------------------------------------------------
+IndexRequestBuilder indexRequestBuilder   = transportClient.prepareIndex();  // <1>
+DeleteRequestBuilder deleteRequestBuilder = transportClient.prepareDelete(); // <2>
+SearchRequestBuilder searchRequestBuilder = transportClient.prepareSearch(); // <3>
+--------------------------------------------------
+<1> Create a `IndexRequestBuilder` using the `prepareIndex()` method from the `TransportClient`. The
+request builder encapsulates the `IndexRequest` to be executed.
+<2> Create a `DeleteRequestBuilder` using the `prepareDelete()` method from the `TransportClient`. The
+request builder encapsulates the `DeleteRequest` to be executed.
+<3> Create a `SearchRequestBuilder` using the `prepareSearch()` method from the `TransportClient`. The
+request builder encapsulates the `SearchRequest` to be executed.
+
+Since the Java High Level REST Client does not support request builders, applications that use
+them must be changed to use <<java-rest-high-level-migration-request-ctor,requests constructors>> instead.
+
+NOTE: While you are incrementally migrating your application and you have both the transport client
+and the high level client available you can always get the `Request` object from the `Builder` object
+by calling `Builder.request()`. We do not advise continuing to depend on the builders in the long run
+but it should be possible to use them during the transition from the transport client to the high
+level rest client.
+
+==== How to migrate the way requests are executed
+
+The `TransportClient` allows to execute requests in both synchronous and asynchronous ways. This is also
+possible using the high-level client.
+
+===== Synchronous execution
+
+The following example shows how a `DeleteRequest` can be synchronously executed using the `TransportClient`:
+
+[source,java]
+--------------------------------------------------
+DeleteRequest request = new DeleteRequest("index", "doc", "id"); // <1>
+DeleteResponse response = transportClient.delete(request).actionGet(); // <2>
+--------------------------------------------------
+<1> Create the `DeleteRequest` using its constructor
+<2> Execute the `DeleteRequest`. The `actionGet()` method blocks until a
+response is returned by the cluster.
+
+The same request synchronously executed using the high-level client is:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/MigrationDocumentationIT.java[migration-request-sync-execution]
+--------------------------------------------------
+<1> Execute the `DeleteRequest`. The `delete()` method blocks until a
+response is returned by the cluster.
+
+===== Asynchronous execution
+
+The following example shows how a `DeleteRequest` can be asynchronously executed using the `TransportClient`:
+
+[source,java]
+--------------------------------------------------
+DeleteRequest request = new DeleteRequest("index", "doc", "id"); // <1>
+transportClient.delete(request, new ActionListener<DeleteResponse>() { // <2>
+    @Override
+    public void onResponse(DeleteResponse deleteResponse) {
+        // <3>
+    }
+
+    @Override
+    public void onFailure(Exception e) {
+        // <4>
+    }
+});
+--------------------------------------------------
+<1> Create the `DeleteRequest` using its constructor
+<2> Execute the `DeleteRequest` by passing the request and a
+`ActionListener` that gets called on execution completion or
+failure. This method does not block and returns immediately.
+<3> The `onResponse()` method is called when the response is
+returned by the cluster.
+<4> The `onFailure()` method is called when an error occurs
+during the execution of the request.
+
+The same request asynchronously executed using the high-level client is:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/MigrationDocumentationIT.java[migration-request-async-execution]
+--------------------------------------------------
+<1> Create the `DeleteRequest` using its constructor
+<2> Execute the `DeleteRequest` by passing the request and a
+`ActionListener` that gets called on execution completion or
+failure. This method does not block and returns immediately.
+<3> The `onResponse()` method is called when the response is
+returned by the cluster.
+<4> The `onFailure()` method is called when an error occurs
+during the execution of the request.
+
+[[java-rest-high-level-migration-manage-indices]]
+==== Manage Indices using the Low-Level REST Client
+
+The low-level client is able to execute any kind of HTTP requests, and can
+therefore be used to call the APIs that are not yet supported by the high level client.
+
+For example, creating a new index with the `TransportClient` may look like this:
+
+[source,java]
+--------------------------------------------------
+Settings settings = Settings.builder() // <1>
+                                .put(SETTING_NUMBER_OF_SHARDS, 1)
+                                .put(SETTING_NUMBER_OF_REPLICAS, 0)
+                                .build();
+
+String mappings = XContentFactory.jsonBuilder()  // <2>
+                                .startObject()
+                                    .startObject("doc")
+                                        .startObject("properties")
+                                            .startObject("time")
+                                                .field("type", "date")
+                                            .endObject()
+                                        .endObject()
+                                    .endObject()
+                                .endObject()
+                                .string();
+
+CreateIndexResponse response = transportClient.admin().indices()  // <3>
+        .prepareCreate("my-index")
+        .setSettings(indexSettings)
+        .addMapping("doc", docMapping, XContentType.JSON)
+        .get();
+
+if (response.isAcknowledged() == false) {
+    // <4>
+}
+--------------------------------------------------
+<1> Define the settings of the index
+<2> Define the mapping for document of type `doc` using a
+`XContentBuilder`
+<3> Create the index with the previous settings and mapping
+using the `prepareCreate()` method. The execution is synchronous
+and blocks on the `get()` method until the remote cluster returns
+a response.
+<4> Handle the situation where the index has not been created
+
+The same operation executed with the low-level client could be:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/MigrationDocumentationIT.java[migration-create-inded]
+--------------------------------------------------
+<1> Define the settings of the index
+<2> Define the body of the HTTP request using a `XContentBuilder` with JSON format
+<3> Include the settings in the request body
+<4> Include the mappings in the request body
+<5> Convert the request body from `String` to a `HttpEntity` and
+set its content type (here, JSON)
+<6> Execute the request using the low-level client. The execution is synchronous
+and blocks on the `performRequest()` method until the remote cluster returns
+a response.
+<7> Handle the situation where the index has not been created
+
+
+[[java-rest-high-level-migration-cluster-health]]
+==== Checking Cluster Health using the Low-Level REST Client
+
+Another common need is to check the cluster's health using the Cluster API. With
+the `TransportClient` it can be done this way:
+
+[source,java]
+--------------------------------------------------
+ClusterHealthResponse response = client.admin().cluster().prepareHealth().get(); // <1>
+
+ClusterHealthStatus healthStatus = response.getStatus(); // <2>
+if (healthStatus != ClusterHealthStatus.GREEN) {
+    // <3>
+}
+--------------------------------------------------
+<1> Execute a `ClusterHealth` with default parameters
+<2> Retrieve the cluster's health status from the response
+<3> Handle the situation where the cluster's health is not green
+
+With the low-level client, the code can be changed to:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/MigrationDocumentationIT.java[migration-cluster-health]
+--------------------------------------------------
+<1> Call the cluster's health REST endpoint using the default paramaters
+and gets back a `Response` object
+<2> Retrieve an `InputStream` object in order to read the response's content
+<3> Parse the response's content using Elasticsearch's helper class `XContentHelper`. This
+ helper requires the content type of the response to be passed as an argument and returns
+ a `Map` of objects. Values in the map can be of any type, including inner `Map` that are
+ used to represent the JSON object hierarchy.
+<4> Retrieve the value of the `status` field in the response map, casts it as a a `String`
+object and use the `ClusterHealthStatus.fromString()` method to convert it as a `ClusterHealthStatus`
+object. This method throws an exception if the value does not corresponds to a valid cluster
+health status.
+<5> Handle the situation where the cluster's health is not green
+
+Note that for convenience this example uses Elasticsearch's helpers to parse the JSON response
+body, but any other JSON parser could have been use instead.
+
+=== Provide feedback
+
+We love to hear from you! Please give us your feedback about your migration
+experience and how to improve the Java High Level Rest Client on https://discuss.elastic.co/[our forum].
+
+
diff --git a/docs/java-rest/high-level/supported-apis.asciidoc b/docs/java-rest/high-level/supported-apis.asciidoc
new file mode 100644
index 0000000000000..51411ea9fcaf0
--- /dev/null
+++ b/docs/java-rest/high-level/supported-apis.asciidoc
@@ -0,0 +1,23 @@
+[[java-rest-high-supported-apis]]
+== Supported APIs
+
+The Java High Level REST Client supports the following APIs:
+
+Single document APIs::
+* <<java-rest-high-document-index>>
+* <<java-rest-high-document-get>>
+* <<java-rest-high-document-delete>>
+* <<java-rest-high-document-update>>
+
+Multi document APIs::
+* <<java-rest-high-document-bulk>>
+
+Search APIs::
+* <<java-rest-high-search>>
+* <<java-rest-high-search-scroll>>
+* <<java-rest-high-clear-scroll>>
+
+Miscellaneous APIs::
+* <<java-rest-high-main>>
+
+include::apis/index.asciidoc[]
\ No newline at end of file
diff --git a/docs/java-rest/index.asciidoc b/docs/java-rest/index.asciidoc
index 2c2435d27187c..d06e312bce7f3 100644
--- a/docs/java-rest/index.asciidoc
+++ b/docs/java-rest/index.asciidoc
@@ -1,12 +1,10 @@
 [[java-rest]]
 = Java REST Client
 
-:version: 5.0.0-alpha5
+include::../Versions.asciidoc[]
 
 include::overview.asciidoc[]
 
-include::usage.asciidoc[]
+include::low-level/index.asciidoc[]
 
-include::configuration.asciidoc[]
-
-include::sniffer.asciidoc[]
+include::high-level/index.asciidoc[]
diff --git a/docs/java-rest/license.asciidoc b/docs/java-rest/license.asciidoc
new file mode 100644
index 0000000000000..3379a96944f20
--- /dev/null
+++ b/docs/java-rest/license.asciidoc
@@ -0,0 +1,15 @@
+== License
+
+Copyright 2013-2017 Elasticsearch
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
\ No newline at end of file
diff --git a/docs/java-rest/low-level/configuration.asciidoc b/docs/java-rest/low-level/configuration.asciidoc
new file mode 100644
index 0000000000000..54f7cd2817354
--- /dev/null
+++ b/docs/java-rest/low-level/configuration.asciidoc
@@ -0,0 +1,88 @@
+== Common configuration
+
+As explained in <<java-rest-low-usage-initialization>>, the `RestClientBuilder`
+supports providing both a `RequestConfigCallback` and an `HttpClientConfigCallback`
+which allow for any customization that the Apache Async Http Client exposes.
+Those callbacks make it possible to modify some specific behaviour of the client
+without overriding every other default configuration that the `RestClient`
+is initialized with. This section describes some common scenarios that require
+additional configuration for the low-level Java REST Client.
+
+=== Timeouts
+
+Configuring requests timeouts can be done by providing an instance of
+`RequestConfigCallback` while building the `RestClient` through its builder.
+The interface has one method that receives an instance of
+https://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/client/config/RequestConfig.Builder.html[`org.apache.http.client.config.RequestConfig.Builder`]
+ as an argument and has the same return type. The request config builder can
+be modified and then returned. In the following example we increase the
+connect timeout (defaults to 1 second) and the socket timeout (defaults to 30
+seconds). Also we adjust the max retry timeout accordingly (defaults to 30
+seconds too).
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/RestClientDocumentation.java[rest-client-config-timeouts]
+--------------------------------------------------
+
+=== Number of threads
+
+The Apache Http Async Client starts by default one dispatcher thread, and a
+number of worker threads used by the connection manager, as many as the number
+of locally detected processors (depending on what
+`Runtime.getRuntime().availableProcessors()` returns). The number of threads
+can be modified as follows:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/RestClientDocumentation.java[rest-client-config-threads]
+--------------------------------------------------
+
+=== Basic authentication
+
+Configuring basic authentication can be done by providing an
+`HttpClientConfigCallback` while building the `RestClient` through its builder.
+The interface has one method that receives an instance of
+https://hc.apache.org/httpcomponents-asyncclient-dev/httpasyncclient/apidocs/org/apache/http/impl/nio/client/HttpAsyncClientBuilder.html[`org.apache.http.impl.nio.client.HttpAsyncClientBuilder`]
+ as an argument and has the same return type. The http client builder can be
+modified and then returned. In the following example we set a default
+credentials provider that requires basic authentication.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/RestClientDocumentation.java[rest-client-config-basic-auth]
+--------------------------------------------------
+
+Preemptive Authentication can be disabled, which means that every request will be sent without
+authorization headers to see if it is accepted and, upon receiving a HTTP 401 response, it will
+resend the exact same request with the basic authentication header. If you wish to do this, then
+you can do so by disabling it via the `HttpAsyncClientBuilder`:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/RestClientDocumentation.java[rest-client-config-disable-preemptive-auth]
+--------------------------------------------------
+<1> Disable preemptive authentication
+
+=== Encrypted communication
+
+Encrypted communication can also be configured through the
+`HttpClientConfigCallback`. The
+https://hc.apache.org/httpcomponents-asyncclient-dev/httpasyncclient/apidocs/org/apache/http/impl/nio/client/HttpAsyncClientBuilder.html[`org.apache.http.impl.nio.client.HttpAsyncClientBuilder`]
+ received as an argument exposes multiple methods to configure encrypted
+ communication: `setSSLContext`, `setSSLSessionStrategy` and
+ `setConnectionManager`, in order of precedence from the least important.
+ The following is an example:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/RestClientDocumentation.java[rest-client-config-encrypted-communication]
+--------------------------------------------------
+
+If no explicit configuration is provided, the http://docs.oracle.com/javase/7/docs/technotes/guides/security/jsse/JSSERefGuide.html#CustomizingStores[system default configuration]
+will be used.
+
+=== Others
+
+For any other required configuration needed, the Apache HttpAsyncClient docs
+should be consulted: https://hc.apache.org/httpcomponents-asyncclient-4.1.x/ .
diff --git a/docs/java-rest/low-level/index.asciidoc b/docs/java-rest/low-level/index.asciidoc
new file mode 100644
index 0000000000000..92230a657a91d
--- /dev/null
+++ b/docs/java-rest/low-level/index.asciidoc
@@ -0,0 +1,36 @@
+[[java-rest-low]]
+= Java Low Level REST Client
+
+[partintro]
+--
+
+The low-level client's features include:
+
+* minimal dependencies
+
+* load balancing across all available nodes
+
+* failover in case of node failures and upon specific response codes
+
+* failed connection penalization (whether a failed node is retried depends on
+ how many consecutive times it failed; the more failed attempts the longer the
+ client will wait before trying that same node again)
+
+* persistent connections
+
+* trace logging of requests and responses
+
+* optional automatic <<sniffer,discovery of cluster nodes>>
+
+--
+
+:doc-tests: {docdir}/../../client/rest/src/test/java/org/elasticsearch/client/documentation
+include::usage.asciidoc[]
+include::configuration.asciidoc[]
+
+:doc-tests: {docdir}/../../client/sniffer/src/test/java/org/elasticsearch/client/sniff/documentation
+include::sniffer.asciidoc[]
+
+include::../license.asciidoc[]
+
+:doc-tests!:
diff --git a/docs/java-rest/low-level/sniffer.asciidoc b/docs/java-rest/low-level/sniffer.asciidoc
new file mode 100644
index 0000000000000..4af61b5d2f359
--- /dev/null
+++ b/docs/java-rest/low-level/sniffer.asciidoc
@@ -0,0 +1,134 @@
+[[sniffer]]
+== Sniffer
+
+Minimal library that allows to automatically discover nodes from a running
+Elasticsearch cluster and set them to an existing `RestClient` instance.
+It retrieves by default the nodes that belong to the cluster using the
+Nodes Info api and uses jackson to parse the obtained json response.
+
+Compatible with Elasticsearch 2.x and onwards.
+
+[[java-rest-sniffer-javadoc]]
+=== Javadoc
+
+The javadoc for the REST client sniffer can be found at {rest-client-sniffer-javadoc}/index.html.
+
+=== Maven Repository
+
+The REST client sniffer is subject to the same release cycle as
+elasticsearch. Replace the version with the desired sniffer version, first
+released with `5.0.0-alpha4`. There is no relation between the sniffer version
+and the elasticsearch version that the client can communicate with. Sniffer
+supports fetching the nodes list from elasticsearch 2.x and onwards.
+
+
+==== Maven configuration
+
+Here is how you can configure the dependency using maven as a dependency manager.
+Add the following to your `pom.xml` file:
+
+["source","xml",subs="attributes"]
+--------------------------------------------------
+<dependency>
+    <groupId>org.elasticsearch.client</groupId>
+    <artifactId>elasticsearch-rest-client-sniffer</artifactId>
+    <version>{version}</version>
+</dependency>
+--------------------------------------------------
+
+==== Gradle configuration
+
+Here is how you can configure the dependency using gradle as a dependency manager.
+Add the following to your `build.gradle` file:
+
+["source","groovy",subs="attributes"]
+--------------------------------------------------
+dependencies {
+    compile 'org.elasticsearch.client:elasticsearch-rest-client-sniffer:{version}'
+}
+--------------------------------------------------
+
+=== Usage
+
+Once a `RestClient` instance has been created as shown in <<java-rest-low-usage-initialization>>,
+a `Sniffer` can be associated to it. The `Sniffer` will make use of the provided `RestClient`
+to periodically (every 5 minutes by default) fetch the list of current nodes from the cluster
+and update them by calling `RestClient#setHosts`.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SnifferDocumentation.java[sniffer-init]
+--------------------------------------------------
+
+It is important to close the `Sniffer` so that its background thread gets
+properly shutdown and all of its resources are released. The `Sniffer`
+object should have the same lifecycle as the `RestClient` and get closed
+right before the client:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SnifferDocumentation.java[sniffer-close]
+--------------------------------------------------
+
+The `Sniffer` updates the nodes by default every 5 minutes. This interval can
+be customized by providing it (in milliseconds) as follows:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SnifferDocumentation.java[sniffer-interval]
+--------------------------------------------------
+
+It is also possible to enable sniffing on failure, meaning that after each
+failure the nodes list gets updated straightaway rather than at the following
+ordinary sniffing round. In this case a `SniffOnFailureListener` needs to
+be created at first and provided at `RestClient` creation. Also once the
+`Sniffer` is later created, it needs to be associated with that same
+`SniffOnFailureListener` instance, which will be notified at each failure
+and use the `Sniffer` to perform the additional sniffing round as described.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SnifferDocumentation.java[sniff-on-failure]
+--------------------------------------------------
+<1> Set the failure listener to the `RestClient` instance
+<2> When sniffing on failure, not only do the nodes get updated after each
+failure, but an additional sniffing round is also scheduled sooner than usual,
+by default one minute after the failure, assuming that things will go back to
+normal and we want to detect that as soon as possible. Said interval can be
+customized  at `Sniffer` creation time through the `setSniffAfterFailureDelayMillis`
+method. Note that this last configuration parameter has no effect in case sniffing
+on failure is not enabled like explained above.
+<3> Set the `Sniffer` instance to the failure listener
+
+The Elasticsearch Nodes Info api doesn't return the protocol to use when
+connecting to the nodes but only their `host:port` key-pair, hence `http`
+is used by default. In case `https` should be used instead, the
+`ElasticsearchHostsSniffer` instance has to be manually created and provided
+as follows:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SnifferDocumentation.java[sniffer-https]
+--------------------------------------------------
+
+In the same way it is also possible to customize the `sniffRequestTimeout`,
+which defaults to one second. That is the `timeout` parameter provided as a
+querystring parameter when calling the Nodes Info api, so that when the
+timeout expires on the server side, a valid response is still returned
+although it may contain only a subset of the nodes that are part of the
+cluster, the ones that have responded until then.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SnifferDocumentation.java[sniff-request-timeout]
+--------------------------------------------------
+
+Also, a custom `HostsSniffer` implementation can be provided for advanced
+use-cases that may require fetching the hosts from external sources rather
+than from Elasticsearch:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SnifferDocumentation.java[custom-hosts-sniffer]
+--------------------------------------------------
+<1> Fetch the hosts from the external source
\ No newline at end of file
diff --git a/docs/java-rest/low-level/usage.asciidoc b/docs/java-rest/low-level/usage.asciidoc
new file mode 100644
index 0000000000000..d39fab38dda09
--- /dev/null
+++ b/docs/java-rest/low-level/usage.asciidoc
@@ -0,0 +1,396 @@
+[[java-rest-low-usage]]
+== Getting started
+
+This section describes how to get started with the low-level REST client from
+getting the artifact to using it in an application.
+
+[[java-rest-low-javadoc]]
+=== Javadoc
+
+The javadoc for the low level REST client can be found at {rest-client-javadoc}/index.html.
+
+[[java-rest-low-usage-maven]]
+=== Maven Repository
+
+The low-level Java REST client is hosted on
+http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.elasticsearch.client%22[Maven
+Central]. The minimum Java version required is `1.7`.
+
+The low-level REST client is subject to the same release cycle as
+elasticsearch. Replace the version with the desired client version, first
+released with `5.0.0-alpha4`. There is no relation between the client version
+and the elasticsearch version that the client can communicate with. The
+low-level REST client is compatible with all elasticsearch versions.
+
+[[java-rest-low-usage-maven-maven]]
+==== Maven configuration
+
+Here is how you can configure the dependency using maven as a dependency manager.
+Add the following to your `pom.xml` file:
+
+["source","xml",subs="attributes"]
+--------------------------------------------------
+<dependency>
+    <groupId>org.elasticsearch.client</groupId>
+    <artifactId>elasticsearch-rest-client</artifactId>
+    <version>{version}</version>
+</dependency>
+--------------------------------------------------
+
+[[java-rest-low-usage-maven-gradle]]
+==== Gradle configuration
+
+Here is how you can configure the dependency using gradle as a dependency manager.
+Add the following to your `build.gradle` file:
+
+["source","groovy",subs="attributes"]
+--------------------------------------------------
+dependencies {
+    compile 'org.elasticsearch.client:elasticsearch-rest-client:{version}'
+}
+--------------------------------------------------
+
+[[java-rest-low-usage-dependencies]]
+=== Dependencies
+
+The low-level Java REST client internally uses the
+http://hc.apache.org/httpcomponents-asyncclient-dev/[Apache Http Async Client]
+ to send http requests. It depends on the following artifacts, namely the async
+ http client and its own transitive dependencies:
+
+- org.apache.httpcomponents:httpasyncclient
+- org.apache.httpcomponents:httpcore-nio
+- org.apache.httpcomponents:httpclient
+- org.apache.httpcomponents:httpcore
+- commons-codec:commons-codec
+- commons-logging:commons-logging
+
+[[java-rest-low-usage-shading]]
+=== Shading
+
+In order to avoid version conflicts, the dependencies can be shaded and packaged
+within the client in a single JAR file (sometimes called an "uber JAR" or "fat
+JAR"). Shading a dependency consists of taking its content (resources files and
+Java class files) and renaming some of its packages before putting them in the
+same JAR file as the low-level Java REST client. Shading a JAR can be
+accomplished by 3rd-party plugins for Gradle and Maven.
+
+Be advised that shading a JAR also has implications. Shading the Commons Logging
+layer, for instance, means that 3rd-party logging backends need to be shaded as
+well.
+
+[[java-rest-low-usage-shading-maven]]
+==== Maven configuration
+
+Here is a configuration using the Maven
+https://maven.apache.org/plugins/maven-shade-plugin/index.html[Shade]
+plugin. Add the following to your `pom.xml` file:
+
+["source","xml",subs="attributes"]
+--------------------------------------------------
+<build>
+    <plugins>
+        <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-shade-plugin</artifactId>
+            <version>3.1.0</version>
+            <executions>
+                <execution>
+                    <phase>package</phase>
+                    <goals><goal>shade</goal></goals>
+                    <configuration>
+                        <relocations>
+                            <relocation>
+                                <pattern>org.apache.http</pattern>
+                                <shadedPattern>hidden.org.apache.http</shadedPattern>
+                            </relocation>
+                            <relocation>
+                                <pattern>org.apache.logging</pattern>
+                                <shadedPattern>hidden.org.apache.logging</shadedPattern>
+                            </relocation>
+                            <relocation>
+                                <pattern>org.apache.commons.codec</pattern>
+                                <shadedPattern>hidden.org.apache.commons.codec</shadedPattern>
+                            </relocation>
+                            <relocation>
+                                <pattern>org.apache.commons.logging</pattern>
+                                <shadedPattern>hidden.org.apache.commons.logging</shadedPattern>
+                            </relocation>
+                        </relocations>
+                    </configuration>
+                </execution>
+            </executions>
+        </plugin>
+    </plugins>
+</build>
+--------------------------------------------------
+
+[[java-rest-low-usage-shading-gradle]]
+==== Gradle configuration
+
+Here is a configuration using the Gradle
+https://github.com/johnrengelman/shadow[ShadowJar] plugin. Add the following to
+your `build.gradle` file:
+
+["source","groovy",subs="attributes"]
+--------------------------------------------------
+shadowJar {
+    relocate 'org.apache.http', 'hidden.org.apache.http'
+    relocate 'org.apache.logging', 'hidden.org.apache.logging'
+    relocate 'org.apache.commons.codec', 'hidden.org.apache.commons.codec'
+    relocate 'org.apache.commons.logging', 'hidden.org.apache.commons.logging'
+}
+--------------------------------------------------
+
+[[java-rest-low-usage-initialization]]
+=== Initialization
+
+A `RestClient` instance can be built through the corresponding
+`RestClientBuilder` class, created via `RestClient#builder(HttpHost...)`
+static method. The only required argument is one or more hosts that the
+client will communicate with, provided as instances of
+https://hc.apache.org/httpcomponents-core-ga/httpcore/apidocs/org/apache/http/HttpHost.html[HttpHost]
+ as follows:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/RestClientDocumentation.java[rest-client-init]
+--------------------------------------------------
+
+The `RestClient` class is thread-safe and ideally has the same lifecycle as
+the application that uses it. It is important that it gets closed when no
+longer needed so that all the resources used by it get properly released,
+as well as the underlying http client instance and its threads:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/RestClientDocumentation.java[rest-client-close]
+--------------------------------------------------
+
+`RestClientBuilder` also allows to optionally set the following configuration
+parameters while building the `RestClient` instance:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/RestClientDocumentation.java[rest-client-init-default-headers]
+--------------------------------------------------
+<1> Set the default headers that need to be sent with each request, to
+prevent having to specify them with each single request
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/RestClientDocumentation.java[rest-client-init-max-retry-timeout]
+--------------------------------------------------
+<1> Set the timeout that should be honoured in case multiple attempts are made
+for the same request. The default value is 30 seconds, same as the default
+socket timeout. In case the socket timeout is customized, the maximum retry
+timeout should be adjusted accordingly
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/RestClientDocumentation.java[rest-client-init-failure-listener]
+--------------------------------------------------
+<1> Set a listener that gets notified every time a node fails,  in case actions
+need to be taken. Used internally when sniffing on failure is enabled.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/RestClientDocumentation.java[rest-client-init-request-config-callback]
+--------------------------------------------------
+<1> Set a callback that allows to modify the default request configuration
+(e.g. request timeouts, authentication, or anything that the
+https://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/client/config/RequestConfig.Builder.html[`org.apache.http.client.config.RequestConfig.Builder`]
+ allows to set)
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/RestClientDocumentation.java[rest-client-init-client-config-callback]
+--------------------------------------------------
+<1> Set a callback that allows to modify the http client configuration
+(e.g. encrypted communication over ssl, or anything that the
+http://hc.apache.org/httpcomponents-asyncclient-dev/httpasyncclient/apidocs/org/apache/http/impl/nio/client/HttpAsyncClientBuilder.html[`org.apache.http.impl.nio.client.HttpAsyncClientBuilder`]
+ allows to set)
+
+
+[[java-rest-low-usage-requests]]
+=== Performing requests
+
+Once the `RestClient` has been created, requests can be sent by calling one of
+the available `performRequest` or `performRequestAsync` method variants.
+The `performRequest` methods are synchronous and return the `Response` directly,
+meaning that the client will block and wait for a response to be returned.
+The `performRequestAsync` variants return `void` and accept an extra
+`ResponseListener` as an argument instead, meaning that they are executed
+asynchronously. The provided listener will be notified upon request completion
+or failure.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/RestClientDocumentation.java[rest-client-verb-endpoint]
+--------------------------------------------------
+<1> Send a request by providing only the verb and the endpoint, minimum set
+of required arguments
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/RestClientDocumentation.java[rest-client-verb-endpoint-params]
+--------------------------------------------------
+<1> Send a request by providing the verb, the endpoint, and some querystring
+parameter
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/RestClientDocumentation.java[rest-client-verb-endpoint-params-body]
+--------------------------------------------------
+<1> Send a request by providing the verb, the endpoint, optional querystring
+parameters and the request body enclosed in an `org.apache.http.HttpEntity`
+object
+
+IMPORTANT: The `ContentType` specified for the `HttpEntity` is important
+because it will be used to set the `Content-Type` header so that Elasticsearch
+can properly parse the content.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/RestClientDocumentation.java[rest-client-response-consumer]
+--------------------------------------------------
+<1> Send a request by providing the verb, the endpoint, optional querystring
+parameters, optional request body and the optional factory that is used to
+create an http://hc.apache.org/httpcomponents-core-ga/httpcore-nio/apidocs/org/apache/http/nio/protocol/HttpAsyncResponseConsumer.html[`org.apache.http.nio.protocol.HttpAsyncResponseConsumer`]
+callback instance per request attempt. Controls how the response body gets
+streamed from a non-blocking HTTP connection on the client side. When not
+provided, the default implementation is used which buffers the whole response
+body in heap memory, up to 100 MB.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/RestClientDocumentation.java[rest-client-verb-endpoint-async]
+--------------------------------------------------
+<1> Define what needs to happen when the request is successfully performed
+<2> Define what needs to happen when the request fails, meaning whenever
+there's a connection error or a response with error status code is returned.
+<3> Send an async request by providing only the verb, the endpoint, and the
+response listener to be notified once the request is completed, minimum set
+of required arguments
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/RestClientDocumentation.java[rest-client-verb-endpoint-params-async]
+--------------------------------------------------
+<1> Send an async request by providing the verb, the endpoint, some querystring
+parameter and the response listener to be notified once the request is completed
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/RestClientDocumentation.java[rest-client-verb-endpoint-params-body-async]
+--------------------------------------------------
+<1> Send an async request by providing the verb, the endpoint, optional
+querystring parameters, the request body enclosed in an
+`org.apache.http.HttpEntity` object and the response listener to be
+notified once the request is completed
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/RestClientDocumentation.java[rest-client-response-consumer-async]
+--------------------------------------------------
+<1> Send an async request by providing the verb, the endpoint, optional
+querystring parameters, optional request body and the optional factory that is
+used to create an http://hc.apache.org/httpcomponents-core-ga/httpcore-nio/apidocs/org/apache/http/nio/protocol/HttpAsyncResponseConsumer.html[`org.apache.http.nio.protocol.HttpAsyncResponseConsumer`]
+callback instance per request attempt. Controls how the response body gets
+streamed from a non-blocking HTTP connection on the client side. When not
+provided, the default implementation is used which buffers the whole response
+body in heap memory, up to 100 MB.
+
+The following is a basic example of how async requests can be sent:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/RestClientDocumentation.java[rest-client-async-example]
+--------------------------------------------------
+<1> Process the returned response
+<2> Handle the returned exception, due to communication error or a response
+with status code that indicates an error
+
+Each of the above listed method supports sending headers along with the
+request through a `Header` varargs argument as in the following examples:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/RestClientDocumentation.java[rest-client-headers]
+--------------------------------------------------
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/RestClientDocumentation.java[rest-client-headers-async]
+--------------------------------------------------
+
+[[java-rest-low-usage-responses]]
+=== Reading responses
+
+The `Response` object, either returned by the synchronous `performRequest` methods or
+received as an argument in `ResponseListener#onSuccess(Response)`, wraps the
+response object returned by the http client and exposes some additional information.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/RestClientDocumentation.java[rest-client-response2]
+--------------------------------------------------
+<1> Information about the performed request
+<2> The host that returned the response
+<3> The response status line, from which you can for instance retrieve the status code
+<4> The response headers, which can also be retrieved by name though `getHeader(String)`
+<5> The response body enclosed in an https://hc.apache.org/httpcomponents-core-ga/httpcore/apidocs/org/apache/http/HttpEntity.html[`org.apache.http.HttpEntity`]
+ object
+
+When performing a request, an exception is thrown (or received as an argument
+ in `ResponseListener#onFailure(Exception)` in the following scenarios:
+
+`IOException`:: communication problem (e.g. SocketTimeoutException)
+`ResponseException`:: a response was returned, but its status code indicated
+an error (not `2xx`). A `ResponseException` originates from a valid
+http response, hence it exposes its corresponding `Response` object which gives
+access to the returned response.
+
+NOTE: A `ResponseException` is **not** thrown for `HEAD` requests that return
+a `404` status code because it is an expected `HEAD` response that simply
+denotes that the resource is not found. All other HTTP methods (e.g., `GET`)
+throw a `ResponseException` for `404` responses unless the `ignore` parameter
+contains `404`. `ignore` is a special client parameter that doesn't get sent
+to Elasticsearch and contains a comma separated list of error status codes.
+It allows to control whether some error status code should be treated as an
+expected response rather than as an exception. This is useful for instance
+with the get api as it can return `404` when the document is missing, in which
+case the response body will not contain an error but rather the usual get api
+response, just without the document as it was not found.
+
+Note that the low-level client doesn't expose any helper for json marshalling
+and un-marshalling. Users are free to use the library that they prefer for that
+purpose.
+
+The underlying Apache Async Http Client ships with different
+https://hc.apache.org/httpcomponents-core-ga/httpcore/apidocs/org/apache/http/HttpEntity.html[`org.apache.http.HttpEntity`]
+ implementations that allow to provide the request body in different formats
+(stream, byte array, string etc.). As for reading the response body, the
+`HttpEntity#getContent` method comes handy which returns an `InputStream`
+reading from the previously buffered response body. As an alternative, it is
+possible to provide a custom
+http://hc.apache.org/httpcomponents-core-ga/httpcore-nio/apidocs/org/apache/http/nio/protocol/HttpAsyncResponseConsumer.html[`org.apache.http.nio.protocol.HttpAsyncResponseConsumer`]
+ that controls how bytes are read and buffered.
+
+[[java-rest-low-usage-logging]]
+=== Logging
+
+The Java REST client uses the same logging library that the Apache Async Http
+Client uses: https://commons.apache.org/proper/commons-logging/[Apache Commons Logging],
+ which comes with support for a number of popular logging implementations. The
+java packages to enable logging for are `org.elasticsearch.client` for the
+client itself and `org.elasticsearch.client.sniffer` for the sniffer.
+
+The request tracer logging can also be enabled to log every request and
+corresponding response in curl format. That comes handy when debugging, for
+instance in case a request needs to be manually executed to check whether it
+still yields the same response as it did. Enable trace logging for the `tracer`
+package to have such log lines printed out. Do note that this type of logging is
+expensive and should not be enabled at all times in production environments,
+but rather temporarily used only when needed.
+
diff --git a/docs/java-rest/overview.asciidoc b/docs/java-rest/overview.asciidoc
index 206fcb931b496..4539406e4c3f4 100644
--- a/docs/java-rest/overview.asciidoc
+++ b/docs/java-rest/overview.asciidoc
@@ -1,42 +1,13 @@
+[[java-rest-overview]]
 == Overview
 
-Official low-level client for Elasticsearch. Allows to communicate with an
-Elasticsearch cluster through http. Compatible with all elasticsearch versions.
+The Java REST Client comes in 2 flavors:
 
-=== Features
-
-The low-level client's features include:
-
-* minimal dependencies
-
-* load balancing across all available nodes
-
-* failover in case of node failures and upon specific response codes
-
-* failed connection penalization (whether a failed node is retried depends on
- how many consecutive times it failed; the more failed attempts the longer the
- client will wait before trying that same node again)
-
-* persistent connections
-
-* trace logging of requests and responses
-
-* optional automatic <<sniffer,discovery of cluster nodes>>
-
-
-=== License
-
-Copyright 2013-2016 Elasticsearch
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
+* <<java-rest-low>>: the official low-level client for Elasticsearch.
+It allows to communicate with an Elasticsearch cluster through http.
+Leaves requests marshalling and responses un-marshalling to users.
+It is compatible with all Elasticsearch versions.
 
+* <<java-rest-high>>: the official high-level client for Elasticsearch.
+Based on the low-level client, it exposes API specific methods and takes care
+of requests marshalling and responses un-marshalling.
\ No newline at end of file
diff --git a/docs/java-rest/sniffer.asciidoc b/docs/java-rest/sniffer.asciidoc
deleted file mode 100644
index 233c4c74a3e57..0000000000000
--- a/docs/java-rest/sniffer.asciidoc
+++ /dev/null
@@ -1,136 +0,0 @@
-[[sniffer]]
-== Sniffer
-
-Minimal library that allows to automatically discover nodes from a running
-Elasticsearch cluster and set them to an existing `RestClient` instance.
-It retrieves by default the nodes that belong to the cluster using the
-Nodes Info api and uses jackson to parse the obtained json response.
-
-Compatible with Elasticsearch 2.x and onwards.
-
-=== Maven Repository
-
-Here is how you can configure the dependency using maven as a dependency manager.
-Add the following to your `pom.xml` file:
-
-["source","xml",subs="attributes"]
---------------------------------------------------
-<dependency>
-    <groupId>org.elasticsearch.client</groupId>
-    <artifactId>sniffer</artifactId>
-    <version>{version}</version>
-</dependency>
---------------------------------------------------
-
-The low-level REST client is subject to the same release cycle as
-elasticsearch. Replace `${es.version}` with the desired sniffer version, first
-released with `5.0.0-alpha4`. There  is no relation between the sniffer version
-and the elasticsearch version that the client can communicate with. Sniffer
-supports fetching the nodes list from elasticsearch 2.x and onwards.
-
-=== Usage
-
-Once a `RestClient` instance has been created, a `Sniffer` can be associated
-to it. The `Sniffer` will make use of the provided `RestClient` to periodically
-(every 5 minutes by default) fetch the list of current nodes from the cluster
-and update them by calling `RestClient#setHosts`.
-
-
-
-[source,java]
---------------------------------------------------
-Sniffer sniffer = Sniffer.builder(restClient).build();
---------------------------------------------------
-
-It is important to close the `Sniffer` so that its background thread gets
-properly shutdown and all of its resources are released. The `Sniffer`
-object should have the same lifecycle as the `RestClient` and get closed
-right before the client:
-
-[source,java]
---------------------------------------------------
-sniffer.close();
-restClient.close();
---------------------------------------------------
-
-The Elasticsearch Nodes Info api doesn't return the protocol to use when
-connecting to the nodes but only their `host:port` key-pair, hence `http`
-is used by default. In case `https` should be used instead, the
-`ElasticsearchHostsSniffer` object has to be manually created and provided
-as follows:
-
-[source,java]
---------------------------------------------------
-HostsSniffer hostsSniffer = new ElasticsearchHostsSniffer(restClient,
-        ElasticsearchHostsSniffer.DEFAULT_SNIFF_REQUEST_TIMEOUT,
-        ElasticsearchHostsSniffer.Scheme.HTTPS);
-Sniffer sniffer = Sniffer.builder(restClient)
-        .setHostsSniffer(hostsSniffer).build();
---------------------------------------------------
-
-In the same way it is also possible to customize the `sniffRequestTimeout`,
-which defaults to one second. That is the `timeout` parameter provided as a
-querystring parameter when calling the Nodes Info api, so that when the
-timeout expires on the server side, a valid response is still returned
-although it may contain only a subset of the nodes that are part of the
-cluster, the ones that have responsed until then.
-Also, a custom `HostsSniffer` implementation can be provided for advanced
-use-cases that may require fetching the hosts from external sources.
-
-The `Sniffer` updates the nodes by default every 5 minutes. This interval can
-be customized by providing it (in milliseconds) as follows:
-
-[source,java]
---------------------------------------------------
-Sniffer sniffer = Sniffer.builder(restClient)
-        .setSniffIntervalMillis(60000).build();
---------------------------------------------------
-
-It is also possible to enable sniffing on failure, meaning that after each
-failure the nodes list gets updated straightaway rather than at the following
-ordinary sniffing round. In this case a `SniffOnFailureListener` needs to
-be created at first and provided at `RestClient` creation. Also once the
-`Sniffer` is later created, it needs to be associated with that same
-`SniffOnFailureListener` instance, which will be notified at each failure
-and use the `Sniffer` to perform the additional sniffing round as described.
-
-[source,java]
---------------------------------------------------
-SniffOnFailureListener sniffOnFailureListener = new SniffOnFailureListener();
-RestClient restClient = RestClient.builder(new HttpHost("localhost", 9200))
-        .setFailureListener(sniffOnFailureListener).build();
-Sniffer sniffer = Sniffer.builder(restClient).build();
-sniffOnFailureListener.setSniffer(sniffer);
---------------------------------------------------
-
-When using sniffing on failure, not only do the nodes get updated after each
-failure, but an additional sniffing round is also scheduled sooner than usual,
-by default one minute after the failure, assuming that things will go back to
-normal and we want  to detect that as soon as possible. Said interval can be
-customized  at `Sniffer` creation time as follows:
-
-[source,java]
---------------------------------------------------
-Sniffer sniffer = Sniffer.builder(restClient)
-        .setSniffAfterFailureDelayMillis(30000).build();
---------------------------------------------------
-
-Note that this last configuration parameter has no effect in case sniffing
-on failure is not enabled like explained above.
-
-=== License
-
-Copyright 2013-2016 Elasticsearch
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
diff --git a/docs/java-rest/usage.asciidoc b/docs/java-rest/usage.asciidoc
deleted file mode 100644
index 55eac8e4668a2..0000000000000
--- a/docs/java-rest/usage.asciidoc
+++ /dev/null
@@ -1,284 +0,0 @@
-== Getting started
-
-=== Maven Repository
-
-The low-level Java REST client is hosted on
-http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.elasticsearch.client%22[Maven
-Central]. The minimum Java version required is `1.7`.
-
-Here is how you can configure the dependency using maven as a dependency manager.
-Add the following to your `pom.xml` file:
-
-["source","xml",subs="attributes"]
---------------------------------------------------
-<dependency>
-    <groupId>org.elasticsearch.client</groupId>
-    <artifactId>rest</artifactId>
-    <version>{version}</version>
-</dependency>
---------------------------------------------------
-
-The low-level REST client is subject to the same release cycle as
-elasticsearch. Replace `${es.version}` with the desired client version, first
-released with `5.0.0-alpha4`. There  is no relation between the client version
-and the elasticsearch version that the client can communicate with. The
-low-level REST client is compatible with all elasticsearch versions.
-
-=== Dependencies
-
-The low-level Java REST client internally uses the
-http://hc.apache.org/httpcomponents-asyncclient-dev/[Apache Http Async Client]
- to send http requests. It depends on the following artifacts, namely the async
- http client and its own transitive dependencies:
-
-- org.apache.httpcomponents:httpasyncclient
-- org.apache.httpcomponents:httpcore-nio
-- org.apache.httpcomponents:httpclient
-- org.apache.httpcomponents:httpcore
-- commons-codec:commons-codec
-- commons-logging:commons-logging
-
-
-=== Initialization
-
-A `RestClient` instance can be built through the corresponding
-`RestClientBuilder` class, created via `RestClient#builder(HttpHost...)`
-static method. The only required argument is one or more hosts that the
-client will communicate with, provided as instances of
-https://hc.apache.org/httpcomponents-core-ga/httpcore/apidocs/org/apache/http/HttpHost.html[HttpHost]
- as follows:
-
-[source,java]
---------------------------------------------------
-RestClient restClient = RestClient.builder(
-        new HttpHost("http", "localhost", 9200),
-        new HttpHost("http", "localhost", 9201)).build();
---------------------------------------------------
-
-The `RestClient` class is thread-safe and ideally has the same lifecycle as
-the application that uses it. It is important that it gets closed when no
-longer needed so that all the resources used by it get properly released,
-as well as the underlying http client instance and its threads:
-
-[source,java]
---------------------------------------------------
-restClient.close();
---------------------------------------------------
-
-`RestClientBuilder` also allows to optionally set the following configuration
-parameters while building the `RestClient` instance:
-
-`setDefaultHeaders`:: default headers that need to be sent with each request,
-to prevent having to specify them with each single request
-`setMaxRetryTimeoutMillis`:: the timeout that should be honoured in case
-multiple attempts are made for the same request. The default value is 10
-seconds, same as the default socket timeout. In case the socket timeout is
-customized, the maximum retry timeout should be adjusted accordingly
-`setFailureListener`:: a listener that gets notified every time a node
-fails,  in case actions need to be taken. Used internally when sniffing on
-failure is enabled
-`setRequestConfigCallback`:: callback that allows to modify the default
-request configuration (e.g. request timeouts, authentication, or anything that
-the https://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/client/config/RequestConfig.Builder.html[`org.apache.http.client.config.RequestConfig.Builder`]
- allows to set)
-`setHttpClientConfigCallback`:: callback that allows to modify the http client
- configuration (e.g. encrypted communication over ssl, or anything that the
-http://hc.apache.org/httpcomponents-asyncclient-dev/httpasyncclient/apidocs/org/apache/http/impl/nio/client/HttpAsyncClientBuilder.html[`org.apache.http.impl.nio.client.HttpAsyncClientBuilder`]
- allows to set)
-
-
-=== Performing requests
-
-Once the `RestClient` has been created, requests can be sent by calling one of
-the available `performRequest` or `performRequestAsync` method variants.
-The `performRequest` methods are synchronous and they return the `Response`
-directly, meaning that the client will block and wait for a response to be returned.
-The `performRequestAsync` variants, which return `void` and accept an extra
-`ResponseListener` as an argument, are executed asynchronously. The provided
-listener will be notified upon completion or failure.
-
-[source,java]
---------------------------------------------------
-// Synchronous variants
-Response performRequest(String method, String endpoint,
-                        Header... headers)
-    throws IOException;
-
-Response performRequest(String method, String endpoint,
-                        Map<String, String> params, Header... headers)
-    throws IOException;
-
-Response performRequest(String method, String endpoint,
-                        Map<String, String> params,
-                        HttpEntity entity,
-                        Header... headers)
-    throws IOException;
-
-Response performRequest(String method, String endpoint,
-                        Map<String, String> params,
-                        HttpEntity entity,
-                        HttpAsyncResponseConsumer<HttpResponse> responseConsumer,
-                        Header... headers)
-    throws IOException;
-
-// Asynchronous variants
-void performRequestAsync(String method, String endpoint,
-                         ResponseListener responseListener,
-                         Header... headers);
-
-void performRequestAsync(String method, String endpoint,
-                         Map<String, String> params,
-                         ResponseListener responseListener,
-                         Header... headers);
-
-void performRequestAsync(String method, String endpoint,
-                         Map<String, String> params,
-                         HttpEntity entity,
-                         ResponseListener responseListener,
-                         Header... headers);
-
-void performRequestAsync(String method, String endpoint,
-                         Map<String, String> params,
-                         HttpEntity entity,
-                         ResponseListener responseListener,
-                         HttpAsyncResponseConsumer<HttpResponse> responseConsumer,
-                         Header... headers);
---------------------------------------------------
-
-==== Request Arguments
-
-The following are the arguments accepted by the different methods:
-
-`method`:: the http method or verb
-`endpoint`:: the request path, which identifies the Elasticsearch API to
-call (e.g. `/_cluster/health`)
-`params`:: the optional parameters to be sent as querystring parameters
-`entity`:: the optional request body enclosed in an
-`org.apache.http.HttpEntity` object
-`responseConsumer`:: the optional
-http://hc.apache.org/httpcomponents-core-ga/httpcore-nio/apidocs/org/apache/http/nio/protocol/HttpAsyncResponseConsumer.html[`org.apache.http.nio.protocol.HttpAsyncResponseConsumer`]
- callback. Controls how the response body gets streamed from a non-blocking
-HTTP connection on the client side. When not provided, the default
-implementation is used which buffers the whole response body in heap memory
-`responseListener`:: the listener to be notified upon asynchronous
-request success or failure
-`headers`:: optional request headers
-
-=== Reading responses
-
-The `Response` object, either returned by the synchronous `performRequest` methods or
-received as an argument in `ResponseListener#onSuccess(Response)`, wraps the
-response object returned by the http client and exposes the following information:
-
-`getRequestLine`:: information about the performed request
-`getHost`:: the host that returned the response
-`getStatusLine`:: the response status line
-`getHeaders`:: the response headers, which can also be retrieved by name
-though `getHeader(String)`
-`getEntity`:: the response body enclosed in an
-https://hc.apache.org/httpcomponents-core-ga/httpcore/apidocs/org/apache/http/HttpEntity.html[`org.apache.http.HttpEntity`]
- object
-
-When performing a request, an exception is thrown (or received as an argument
- in `ResponseListener#onFailure(Exception)` in the following scenarios:
-
-`IOException`:: communication problem (e.g. SocketTimeoutException etc.)
-`ResponseException`:: a response was returned, but its status code indicated
-an error (not `2xx`). A `ResponseException` originates from a valid
-http response, hence it exposes its corresponding `Response` object which gives
-access to the returned response.
-
-NOTE: A `ResponseException` is **not** thrown for `HEAD` requests that return
-a `404` status code because it is an expected `HEAD` response that simply
-denotes that the resource is not found. All other HTTP methods (e.g., `GET`)
-throw a `ResponseException` for `404` responses.
-
-
-=== Example requests
-
-Here are a couple of examples:
-
-[source,java]
---------------------------------------------------
-Response response = restClient.performRequest("GET", "/",
-        Collections.singletonMap("pretty", "true"));
-System.out.println(EntityUtils.toString(response.getEntity()));
-
-//index a document
-HttpEntity entity = new NStringEntity(
-        "{\n" +
-        "    \"user\" : \"kimchy\",\n" +
-        "    \"post_date\" : \"2009-11-15T14:12:12\",\n" +
-        "    \"message\" : \"trying out Elasticsearch\"\n" +
-        "}", ContentType.APPLICATION_JSON);
-Response indexResponse = restClient.performRequest(
-        "PUT",
-        "/twitter/tweet/1",
-        Collections.<String, String>emptyMap(),
-        entity);
-
-
---------------------------------------------------
-
-Note that the low-level client doesn't expose any helper for json marshalling
-and un-marshalling. Users are free to use the library that they prefer for that
-purpose.
-
-The underlying Apache Async Http Client ships with different
-https://hc.apache.org/httpcomponents-core-ga/httpcore/apidocs/org/apache/http/HttpEntity.html[`org.apache.http.HttpEntity`]
- implementations that allow to provide the request body in different formats
-(stream, byte array, string etc.). As for reading the response body, the
-`HttpEntity#getContent` method comes handy which returns an `InputStream`
-reading from the previously buffered response body. As an alternative, it is
-possible to provide a custom
-http://hc.apache.org/httpcomponents-core-ga/httpcore-nio/apidocs/org/apache/http/nio/protocol/HttpAsyncResponseConsumer.html[`org.apache.http.nio.protocol.HttpAsyncResponseConsumer`]
- that controls how bytes are read and buffered.
-
-The following is a basic example of how async requests can be sent:
-
-[source,java]
---------------------------------------------------
-int numRequests = 10;
-final CountDownLatch latch = new CountDownLatch(numRequests);
-for (int i = 0; i < numRequests; i++) {
-    restClient.performRequestAsync(
-        "PUT",
-        "/twitter/tweet/" + i,
-        Collections.<String, String>emptyMap(),
-        //assume that the documents are stored in an entities array
-        entities[i],
-        new ResponseListener() {
-            @Override
-            public void onSuccess(Response response) {
-                System.out.println(response);
-                latch.countDown();
-            }
-
-            @Override
-            public void onFailure(Exception exception) {
-                latch.countDown();
-            }
-        }
-    );
-}
-//wait for all requests to be completed
-latch.await();
-
---------------------------------------------------
-
-=== Logging
-
-The Java REST client uses the same logging library that the Apache Async Http
-Client uses: https://commons.apache.org/proper/commons-logging/[Apache Commons Logging],
- which comes with support for a number of popular logging implementations. The
-java packages to enable logging for are `org.elasticsearch.client` for the
-client itself and `org.elasticsearch.client.sniffer` for the sniffer.
-
-The request tracer logging can also be enabled to log every request and
-corresponding response in curl format. That comes handy when debugging, for
-instance in case a request needs to be manually executed to check whether it
-still yields the same response as it did. Enable trace logging for the `tracer`
-package to have such log lines printed out. Do note that this type of logging is
-expensive and should not be enabled at all times in production environments,
-but rather temporarily used only when needed.
-
diff --git a/docs/painless/index.asciidoc b/docs/painless/index.asciidoc
new file mode 100644
index 0000000000000..4898ed933363b
--- /dev/null
+++ b/docs/painless/index.asciidoc
@@ -0,0 +1,43 @@
+[[painless]]
+= Painless Scripting Language
+
+include::../Versions.asciidoc[]
+
+include::painless-getting-started.asciidoc[]
+
+// include::painless-examples.asciidoc[]
+
+// include::painless-design.asciidoc[]
+
+include::painless-lang-spec.asciidoc[]
+
+include::painless-syntax.asciidoc[]
+
+include::painless-api-reference.asciidoc[]
+
+////
+Proposed Outline (WIP)
+Getting Started with Painless
+  Accessing Doc Values
+  Updating Fields
+  Working with Dates
+  Using Regular Expressions
+  Debugging Painless Scripts
+
+Example Scripts
+  Using Painless in Script Fields
+  Using Painless in Watches
+  Using Painless in Function Score Queries
+  Using Painless in Script Queries
+  Using Painless When Updating Docs
+  Using Painless When Reindexing
+
+How Painless Works
+  Painless Architecture
+  Dispatching Functions
+
+Painless Language Specification
+Painless API
+////
+
+Painless API Reference
diff --git a/docs/painless/painless-api-reference.asciidoc b/docs/painless/painless-api-reference.asciidoc
new file mode 100644
index 0000000000000..1bda6d890c859
--- /dev/null
+++ b/docs/painless/painless-api-reference.asciidoc
@@ -0,0 +1,17 @@
+["appendix",id="painless-api-reference"]
+= Painless API Reference
+
+Painless has a strict whitelist for methods and
+classes to make sure that all painless scripts are secure and fast. Most of
+these methods are exposed directly from the JRE while others are part of
+Elasticsearch or Painless itself. Below is a list of all available methods
+grouped under the classes on which you can call them. Clicking on the method
+name takes you to the documentation for the method.
+
+NOTE: Methods defined in the JRE also have a `(java 9)` link which can be used
+to see the method's documentation in Java 9 while clicking on the method's name
+goes to the Java 8 documentation. Usually these aren't different but it is
+worth going to the version that matches the version of Java you are using to
+run Elasticsearch just in case.
+
+include::painless-api-reference/index.asciidoc[]
diff --git a/docs/painless/painless-api-reference/AbstractChronology.asciidoc b/docs/painless/painless-api-reference/AbstractChronology.asciidoc
new file mode 100644
index 0000000000000..3cc90ee65cde2
--- /dev/null
+++ b/docs/painless/painless-api-reference/AbstractChronology.asciidoc
@@ -0,0 +1,7 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-AbstractChronology]]++AbstractChronology++::
+* Inherits methods from ++<<painless-api-reference-Chronology,Chronology>>++
diff --git a/docs/painless/painless-api-reference/AbstractCollection.asciidoc b/docs/painless/painless-api-reference/AbstractCollection.asciidoc
new file mode 100644
index 0000000000000..313c327ac07cc
--- /dev/null
+++ b/docs/painless/painless-api-reference/AbstractCollection.asciidoc
@@ -0,0 +1,7 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-AbstractCollection]]++AbstractCollection++::
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/AbstractList.asciidoc b/docs/painless/painless-api-reference/AbstractList.asciidoc
new file mode 100644
index 0000000000000..9ce7c9a683001
--- /dev/null
+++ b/docs/painless/painless-api-reference/AbstractList.asciidoc
@@ -0,0 +1,7 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-AbstractList]]++AbstractList++::
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-List,List>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/AbstractMap.SimpleEntry.asciidoc b/docs/painless/painless-api-reference/AbstractMap.SimpleEntry.asciidoc
new file mode 100644
index 0000000000000..bdd784070f4fb
--- /dev/null
+++ b/docs/painless/painless-api-reference/AbstractMap.SimpleEntry.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-AbstractMap-SimpleEntry]]++AbstractMap.SimpleEntry++::
+* ++[[painless-api-reference-AbstractMap-SimpleEntry-AbstractMap.SimpleEntry-1]]link:{java8-javadoc}/java/util/AbstractMap.SimpleEntry.html#AbstractMap.SimpleEntry%2Djava.util.Map$Entry%2D[AbstractMap.SimpleEntry](<<painless-api-reference-Map-Entry,Map.Entry>>)++ (link:{java9-javadoc}/java/util/AbstractMap.SimpleEntry.html#AbstractMap.SimpleEntry%2Djava.util.Map$Entry%2D[java 9])
+* ++[[painless-api-reference-AbstractMap-SimpleEntry-AbstractMap.SimpleEntry-2]]link:{java8-javadoc}/java/util/AbstractMap.SimpleEntry.html#AbstractMap.SimpleEntry%2Djava.lang.Object%2Djava.lang.Object%2D[AbstractMap.SimpleEntry](def, def)++ (link:{java9-javadoc}/java/util/AbstractMap.SimpleEntry.html#AbstractMap.SimpleEntry%2Djava.lang.Object%2Djava.lang.Object%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Map-Entry,Map.Entry>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/AbstractMap.SimpleImmutableEntry.asciidoc b/docs/painless/painless-api-reference/AbstractMap.SimpleImmutableEntry.asciidoc
new file mode 100644
index 0000000000000..5b5605a42ae81
--- /dev/null
+++ b/docs/painless/painless-api-reference/AbstractMap.SimpleImmutableEntry.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-AbstractMap-SimpleImmutableEntry]]++AbstractMap.SimpleImmutableEntry++::
+* ++[[painless-api-reference-AbstractMap-SimpleImmutableEntry-AbstractMap.SimpleImmutableEntry-1]]link:{java8-javadoc}/java/util/AbstractMap.SimpleImmutableEntry.html#AbstractMap.SimpleImmutableEntry%2Djava.util.Map$Entry%2D[AbstractMap.SimpleImmutableEntry](<<painless-api-reference-Map-Entry,Map.Entry>>)++ (link:{java9-javadoc}/java/util/AbstractMap.SimpleImmutableEntry.html#AbstractMap.SimpleImmutableEntry%2Djava.util.Map$Entry%2D[java 9])
+* ++[[painless-api-reference-AbstractMap-SimpleImmutableEntry-AbstractMap.SimpleImmutableEntry-2]]link:{java8-javadoc}/java/util/AbstractMap.SimpleImmutableEntry.html#AbstractMap.SimpleImmutableEntry%2Djava.lang.Object%2Djava.lang.Object%2D[AbstractMap.SimpleImmutableEntry](def, def)++ (link:{java9-javadoc}/java/util/AbstractMap.SimpleImmutableEntry.html#AbstractMap.SimpleImmutableEntry%2Djava.lang.Object%2Djava.lang.Object%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Map-Entry,Map.Entry>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/AbstractMap.asciidoc b/docs/painless/painless-api-reference/AbstractMap.asciidoc
new file mode 100644
index 0000000000000..ea3be0d125a98
--- /dev/null
+++ b/docs/painless/painless-api-reference/AbstractMap.asciidoc
@@ -0,0 +1,7 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-AbstractMap]]++AbstractMap++::
+* Inherits methods from ++<<painless-api-reference-Map,Map>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/AbstractQueue.asciidoc b/docs/painless/painless-api-reference/AbstractQueue.asciidoc
new file mode 100644
index 0000000000000..37fc53bd73d65
--- /dev/null
+++ b/docs/painless/painless-api-reference/AbstractQueue.asciidoc
@@ -0,0 +1,7 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-AbstractQueue]]++AbstractQueue++::
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-Queue,Queue>>++
diff --git a/docs/painless/painless-api-reference/AbstractSequentialList.asciidoc b/docs/painless/painless-api-reference/AbstractSequentialList.asciidoc
new file mode 100644
index 0000000000000..822b203851648
--- /dev/null
+++ b/docs/painless/painless-api-reference/AbstractSequentialList.asciidoc
@@ -0,0 +1,7 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-AbstractSequentialList]]++AbstractSequentialList++::
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-List,List>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/AbstractSet.asciidoc b/docs/painless/painless-api-reference/AbstractSet.asciidoc
new file mode 100644
index 0000000000000..7baa3a58f7995
--- /dev/null
+++ b/docs/painless/painless-api-reference/AbstractSet.asciidoc
@@ -0,0 +1,7 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-AbstractSet]]++AbstractSet++::
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-Set,Set>>++
diff --git a/docs/painless/painless-api-reference/Annotation.asciidoc b/docs/painless/painless-api-reference/Annotation.asciidoc
new file mode 100644
index 0000000000000..33252543951b8
--- /dev/null
+++ b/docs/painless/painless-api-reference/Annotation.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Annotation]]++Annotation++::
+* ++[[painless-api-reference-Annotation-Annotation-1]]link:{java8-javadoc}/java/text/Annotation.html#Annotation%2Djava.lang.Object%2D[Annotation](<<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/text/Annotation.html#Annotation%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Annotation-getValue-0]]def link:{java8-javadoc}/java/text/Annotation.html#getValue%2D%2D[getValue]()++ (link:{java9-javadoc}/java/text/Annotation.html#getValue%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Appendable.asciidoc b/docs/painless/painless-api-reference/Appendable.asciidoc
new file mode 100644
index 0000000000000..61fcbaf5a6052
--- /dev/null
+++ b/docs/painless/painless-api-reference/Appendable.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Appendable]]++Appendable++::
+* ++[[painless-api-reference-Appendable-append-3]]<<painless-api-reference-Appendable,Appendable>> link:{java8-javadoc}/java/lang/Appendable.html#append%2Djava.lang.CharSequence%2Dint%2Dint%2D[append](<<painless-api-reference-CharSequence,CharSequence>>, int, int)++ (link:{java9-javadoc}/java/lang/Appendable.html#append%2Djava.lang.CharSequence%2Dint%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/ArithmeticException.asciidoc b/docs/painless/painless-api-reference/ArithmeticException.asciidoc
new file mode 100644
index 0000000000000..f486289320570
--- /dev/null
+++ b/docs/painless/painless-api-reference/ArithmeticException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ArithmeticException]]++ArithmeticException++::
+* ++[[painless-api-reference-ArithmeticException-ArithmeticException-0]]link:{java8-javadoc}/java/lang/ArithmeticException.html#ArithmeticException%2D%2D[ArithmeticException]()++ (link:{java9-javadoc}/java/lang/ArithmeticException.html#ArithmeticException%2D%2D[java 9])
+* ++[[painless-api-reference-ArithmeticException-ArithmeticException-1]]link:{java8-javadoc}/java/lang/ArithmeticException.html#ArithmeticException%2Djava.lang.String%2D[ArithmeticException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/ArithmeticException.html#ArithmeticException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/ArrayDeque.asciidoc b/docs/painless/painless-api-reference/ArrayDeque.asciidoc
new file mode 100644
index 0000000000000..668a86185e690
--- /dev/null
+++ b/docs/painless/painless-api-reference/ArrayDeque.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ArrayDeque]]++ArrayDeque++::
+* ++[[painless-api-reference-ArrayDeque-ArrayDeque-0]]link:{java8-javadoc}/java/util/ArrayDeque.html#ArrayDeque%2D%2D[ArrayDeque]()++ (link:{java9-javadoc}/java/util/ArrayDeque.html#ArrayDeque%2D%2D[java 9])
+* ++[[painless-api-reference-ArrayDeque-ArrayDeque-1]]link:{java8-javadoc}/java/util/ArrayDeque.html#ArrayDeque%2Djava.util.Collection%2D[ArrayDeque](<<painless-api-reference-Collection,Collection>>)++ (link:{java9-javadoc}/java/util/ArrayDeque.html#ArrayDeque%2Djava.util.Collection%2D[java 9])
+* ++[[painless-api-reference-ArrayDeque-clone-0]]<<painless-api-reference-ArrayDeque,ArrayDeque>> link:{java8-javadoc}/java/util/ArrayDeque.html#clone%2D%2D[clone]()++ (link:{java9-javadoc}/java/util/ArrayDeque.html#clone%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Deque,Deque>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-Queue,Queue>>++
diff --git a/docs/painless/painless-api-reference/ArrayIndexOutOfBoundsException.asciidoc b/docs/painless/painless-api-reference/ArrayIndexOutOfBoundsException.asciidoc
new file mode 100644
index 0000000000000..49b13361163f4
--- /dev/null
+++ b/docs/painless/painless-api-reference/ArrayIndexOutOfBoundsException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ArrayIndexOutOfBoundsException]]++ArrayIndexOutOfBoundsException++::
+* ++[[painless-api-reference-ArrayIndexOutOfBoundsException-ArrayIndexOutOfBoundsException-0]]link:{java8-javadoc}/java/lang/ArrayIndexOutOfBoundsException.html#ArrayIndexOutOfBoundsException%2D%2D[ArrayIndexOutOfBoundsException]()++ (link:{java9-javadoc}/java/lang/ArrayIndexOutOfBoundsException.html#ArrayIndexOutOfBoundsException%2D%2D[java 9])
+* ++[[painless-api-reference-ArrayIndexOutOfBoundsException-ArrayIndexOutOfBoundsException-1]]link:{java8-javadoc}/java/lang/ArrayIndexOutOfBoundsException.html#ArrayIndexOutOfBoundsException%2Djava.lang.String%2D[ArrayIndexOutOfBoundsException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/ArrayIndexOutOfBoundsException.html#ArrayIndexOutOfBoundsException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/ArrayList.asciidoc b/docs/painless/painless-api-reference/ArrayList.asciidoc
new file mode 100644
index 0000000000000..90386a3742cf7
--- /dev/null
+++ b/docs/painless/painless-api-reference/ArrayList.asciidoc
@@ -0,0 +1,11 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ArrayList]]++ArrayList++::
+* ++[[painless-api-reference-ArrayList-ArrayList-0]]link:{java8-javadoc}/java/util/ArrayList.html#ArrayList%2D%2D[ArrayList]()++ (link:{java9-javadoc}/java/util/ArrayList.html#ArrayList%2D%2D[java 9])
+* ++[[painless-api-reference-ArrayList-ArrayList-1]]link:{java8-javadoc}/java/util/ArrayList.html#ArrayList%2Djava.util.Collection%2D[ArrayList](<<painless-api-reference-Collection,Collection>>)++ (link:{java9-javadoc}/java/util/ArrayList.html#ArrayList%2Djava.util.Collection%2D[java 9])
+* ++[[painless-api-reference-ArrayList-clone-0]]def link:{java8-javadoc}/java/util/ArrayList.html#clone%2D%2D[clone]()++ (link:{java9-javadoc}/java/util/ArrayList.html#clone%2D%2D[java 9])
+* ++[[painless-api-reference-ArrayList-trimToSize-0]]void link:{java8-javadoc}/java/util/ArrayList.html#trimToSize%2D%2D[trimToSize]()++ (link:{java9-javadoc}/java/util/ArrayList.html#trimToSize%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-List,List>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/ArrayStoreException.asciidoc b/docs/painless/painless-api-reference/ArrayStoreException.asciidoc
new file mode 100644
index 0000000000000..29f7a536eadf9
--- /dev/null
+++ b/docs/painless/painless-api-reference/ArrayStoreException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ArrayStoreException]]++ArrayStoreException++::
+* ++[[painless-api-reference-ArrayStoreException-ArrayStoreException-0]]link:{java8-javadoc}/java/lang/ArrayStoreException.html#ArrayStoreException%2D%2D[ArrayStoreException]()++ (link:{java9-javadoc}/java/lang/ArrayStoreException.html#ArrayStoreException%2D%2D[java 9])
+* ++[[painless-api-reference-ArrayStoreException-ArrayStoreException-1]]link:{java8-javadoc}/java/lang/ArrayStoreException.html#ArrayStoreException%2Djava.lang.String%2D[ArrayStoreException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/ArrayStoreException.html#ArrayStoreException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Arrays.asciidoc b/docs/painless/painless-api-reference/Arrays.asciidoc
new file mode 100644
index 0000000000000..97598d64d5790
--- /dev/null
+++ b/docs/painless/painless-api-reference/Arrays.asciidoc
@@ -0,0 +1,11 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Arrays]]++Arrays++::
+* ++[[painless-api-reference-Arrays-asList-1]]static <<painless-api-reference-List,List>> link:{java8-javadoc}/java/util/Arrays.html#asList%2Djava.lang.Object:A%2D[asList](<<painless-api-reference-Object,Object>>[])++ (link:{java9-javadoc}/java/util/Arrays.html#asList%2Djava.lang.Object:A%2D[java 9])
+* ++[[painless-api-reference-Arrays-deepEquals-2]]static boolean link:{java8-javadoc}/java/util/Arrays.html#deepEquals%2Djava.lang.Object:A%2Djava.lang.Object:A%2D[deepEquals](<<painless-api-reference-Object,Object>>[], <<painless-api-reference-Object,Object>>[])++ (link:{java9-javadoc}/java/util/Arrays.html#deepEquals%2Djava.lang.Object:A%2Djava.lang.Object:A%2D[java 9])
+* ++[[painless-api-reference-Arrays-deepHashCode-1]]static int link:{java8-javadoc}/java/util/Arrays.html#deepHashCode%2Djava.lang.Object:A%2D[deepHashCode](<<painless-api-reference-Object,Object>>[])++ (link:{java9-javadoc}/java/util/Arrays.html#deepHashCode%2Djava.lang.Object:A%2D[java 9])
+* ++[[painless-api-reference-Arrays-deepToString-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Arrays.html#deepToString%2Djava.lang.Object:A%2D[deepToString](<<painless-api-reference-Object,Object>>[])++ (link:{java9-javadoc}/java/util/Arrays.html#deepToString%2Djava.lang.Object:A%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/AttributedCharacterIterator.Attribute.asciidoc b/docs/painless/painless-api-reference/AttributedCharacterIterator.Attribute.asciidoc
new file mode 100644
index 0000000000000..dbcdd9446f3f9
--- /dev/null
+++ b/docs/painless/painless-api-reference/AttributedCharacterIterator.Attribute.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-AttributedCharacterIterator-Attribute]]++AttributedCharacterIterator.Attribute++::
+** [[painless-api-reference-AttributedCharacterIterator-Attribute-INPUT_METHOD_SEGMENT]]static <<painless-api-reference-AttributedCharacterIterator-Attribute,AttributedCharacterIterator.Attribute>> link:{java8-javadoc}/java/text/AttributedCharacterIterator.Attribute.html#INPUT_METHOD_SEGMENT[INPUT_METHOD_SEGMENT] (link:{java9-javadoc}/java/text/AttributedCharacterIterator.Attribute.html#INPUT_METHOD_SEGMENT[java 9])
+** [[painless-api-reference-AttributedCharacterIterator-Attribute-LANGUAGE]]static <<painless-api-reference-AttributedCharacterIterator-Attribute,AttributedCharacterIterator.Attribute>> link:{java8-javadoc}/java/text/AttributedCharacterIterator.Attribute.html#LANGUAGE[LANGUAGE] (link:{java9-javadoc}/java/text/AttributedCharacterIterator.Attribute.html#LANGUAGE[java 9])
+** [[painless-api-reference-AttributedCharacterIterator-Attribute-READING]]static <<painless-api-reference-AttributedCharacterIterator-Attribute,AttributedCharacterIterator.Attribute>> link:{java8-javadoc}/java/text/AttributedCharacterIterator.Attribute.html#READING[READING] (link:{java9-javadoc}/java/text/AttributedCharacterIterator.Attribute.html#READING[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/AttributedCharacterIterator.asciidoc b/docs/painless/painless-api-reference/AttributedCharacterIterator.asciidoc
new file mode 100644
index 0000000000000..a71d3906fa167
--- /dev/null
+++ b/docs/painless/painless-api-reference/AttributedCharacterIterator.asciidoc
@@ -0,0 +1,14 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-AttributedCharacterIterator]]++AttributedCharacterIterator++::
+* ++[[painless-api-reference-AttributedCharacterIterator-getAllAttributeKeys-0]]<<painless-api-reference-Set,Set>> link:{java8-javadoc}/java/text/AttributedCharacterIterator.html#getAllAttributeKeys%2D%2D[getAllAttributeKeys]()++ (link:{java9-javadoc}/java/text/AttributedCharacterIterator.html#getAllAttributeKeys%2D%2D[java 9])
+* ++[[painless-api-reference-AttributedCharacterIterator-getAttribute-1]]def link:{java8-javadoc}/java/text/AttributedCharacterIterator.html#getAttribute%2Djava.text.AttributedCharacterIterator$Attribute%2D[getAttribute](<<painless-api-reference-AttributedCharacterIterator-Attribute,AttributedCharacterIterator.Attribute>>)++ (link:{java9-javadoc}/java/text/AttributedCharacterIterator.html#getAttribute%2Djava.text.AttributedCharacterIterator$Attribute%2D[java 9])
+* ++[[painless-api-reference-AttributedCharacterIterator-getAttributes-0]]<<painless-api-reference-Map,Map>> link:{java8-javadoc}/java/text/AttributedCharacterIterator.html#getAttributes%2D%2D[getAttributes]()++ (link:{java9-javadoc}/java/text/AttributedCharacterIterator.html#getAttributes%2D%2D[java 9])
+* ++[[painless-api-reference-AttributedCharacterIterator-getRunLimit-0]]int link:{java8-javadoc}/java/text/AttributedCharacterIterator.html#getRunLimit%2D%2D[getRunLimit]()++ (link:{java9-javadoc}/java/text/AttributedCharacterIterator.html#getRunLimit%2D%2D[java 9])
+* ++[[painless-api-reference-AttributedCharacterIterator-getRunLimit-1]]int link:{java8-javadoc}/java/text/AttributedCharacterIterator.html#getRunLimit%2Djava.util.Set%2D[getRunLimit](<<painless-api-reference-Set,Set>>)++ (link:{java9-javadoc}/java/text/AttributedCharacterIterator.html#getRunLimit%2Djava.util.Set%2D[java 9])
+* ++[[painless-api-reference-AttributedCharacterIterator-getRunStart-0]]int link:{java8-javadoc}/java/text/AttributedCharacterIterator.html#getRunStart%2D%2D[getRunStart]()++ (link:{java9-javadoc}/java/text/AttributedCharacterIterator.html#getRunStart%2D%2D[java 9])
+* ++[[painless-api-reference-AttributedCharacterIterator-getRunStart-1]]int link:{java8-javadoc}/java/text/AttributedCharacterIterator.html#getRunStart%2Djava.util.Set%2D[getRunStart](<<painless-api-reference-Set,Set>>)++ (link:{java9-javadoc}/java/text/AttributedCharacterIterator.html#getRunStart%2Djava.util.Set%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-CharacterIterator,CharacterIterator>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/AttributedString.asciidoc b/docs/painless/painless-api-reference/AttributedString.asciidoc
new file mode 100644
index 0000000000000..0393838a73717
--- /dev/null
+++ b/docs/painless/painless-api-reference/AttributedString.asciidoc
@@ -0,0 +1,15 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-AttributedString]]++AttributedString++::
+* ++[[painless-api-reference-AttributedString-AttributedString-1]]link:{java8-javadoc}/java/text/AttributedString.html#AttributedString%2Djava.lang.String%2D[AttributedString](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/AttributedString.html#AttributedString%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-AttributedString-AttributedString-2]]link:{java8-javadoc}/java/text/AttributedString.html#AttributedString%2Djava.lang.String%2Djava.util.Map%2D[AttributedString](<<painless-api-reference-String,String>>, <<painless-api-reference-Map,Map>>)++ (link:{java9-javadoc}/java/text/AttributedString.html#AttributedString%2Djava.lang.String%2Djava.util.Map%2D[java 9])
+* ++[[painless-api-reference-AttributedString-addAttribute-2]]void link:{java8-javadoc}/java/text/AttributedString.html#addAttribute%2Djava.text.AttributedCharacterIterator$Attribute%2Djava.lang.Object%2D[addAttribute](<<painless-api-reference-AttributedCharacterIterator-Attribute,AttributedCharacterIterator.Attribute>>, <<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/text/AttributedString.html#addAttribute%2Djava.text.AttributedCharacterIterator$Attribute%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-AttributedString-addAttribute-4]]void link:{java8-javadoc}/java/text/AttributedString.html#addAttribute%2Djava.text.AttributedCharacterIterator$Attribute%2Djava.lang.Object%2Dint%2Dint%2D[addAttribute](<<painless-api-reference-AttributedCharacterIterator-Attribute,AttributedCharacterIterator.Attribute>>, <<painless-api-reference-Object,Object>>, int, int)++ (link:{java9-javadoc}/java/text/AttributedString.html#addAttribute%2Djava.text.AttributedCharacterIterator$Attribute%2Djava.lang.Object%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-AttributedString-addAttributes-3]]void link:{java8-javadoc}/java/text/AttributedString.html#addAttributes%2Djava.util.Map%2Dint%2Dint%2D[addAttributes](<<painless-api-reference-Map,Map>>, int, int)++ (link:{java9-javadoc}/java/text/AttributedString.html#addAttributes%2Djava.util.Map%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-AttributedString-getIterator-0]]<<painless-api-reference-AttributedCharacterIterator,AttributedCharacterIterator>> link:{java8-javadoc}/java/text/AttributedString.html#getIterator%2D%2D[getIterator]()++ (link:{java9-javadoc}/java/text/AttributedString.html#getIterator%2D%2D[java 9])
+* ++[[painless-api-reference-AttributedString-getIterator-1]]<<painless-api-reference-AttributedCharacterIterator,AttributedCharacterIterator>> link:{java8-javadoc}/java/text/AttributedString.html#getIterator%2Djava.text.AttributedCharacterIterator$Attribute:A%2D[getIterator](<<painless-api-reference-AttributedCharacterIterator-Attribute,AttributedCharacterIterator.Attribute>>[])++ (link:{java9-javadoc}/java/text/AttributedString.html#getIterator%2Djava.text.AttributedCharacterIterator$Attribute:A%2D[java 9])
+* ++[[painless-api-reference-AttributedString-getIterator-3]]<<painless-api-reference-AttributedCharacterIterator,AttributedCharacterIterator>> link:{java8-javadoc}/java/text/AttributedString.html#getIterator%2Djava.text.AttributedCharacterIterator$Attribute:A%2Dint%2Dint%2D[getIterator](<<painless-api-reference-AttributedCharacterIterator-Attribute,AttributedCharacterIterator.Attribute>>[], int, int)++ (link:{java9-javadoc}/java/text/AttributedString.html#getIterator%2Djava.text.AttributedCharacterIterator$Attribute:A%2Dint%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Base64.Decoder.asciidoc b/docs/painless/painless-api-reference/Base64.Decoder.asciidoc
new file mode 100644
index 0000000000000..3324f1a230308
--- /dev/null
+++ b/docs/painless/painless-api-reference/Base64.Decoder.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Base64-Decoder]]++Base64.Decoder++::
+* ++[[painless-api-reference-Base64-Decoder-decode-1]]byte[] link:{java8-javadoc}/java/util/Base64.Decoder.html#decode%2Djava.lang.String%2D[decode](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/Base64.Decoder.html#decode%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Base64-Decoder-decode-2]]int link:{java8-javadoc}/java/util/Base64.Decoder.html#decode%2Dbyte:A%2Dbyte:A%2D[decode](byte[], byte[])++ (link:{java9-javadoc}/java/util/Base64.Decoder.html#decode%2Dbyte:A%2Dbyte:A%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Base64.Encoder.asciidoc b/docs/painless/painless-api-reference/Base64.Encoder.asciidoc
new file mode 100644
index 0000000000000..6cdcd0536e5be
--- /dev/null
+++ b/docs/painless/painless-api-reference/Base64.Encoder.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Base64-Encoder]]++Base64.Encoder++::
+* ++[[painless-api-reference-Base64-Encoder-encode-2]]int link:{java8-javadoc}/java/util/Base64.Encoder.html#encode%2Dbyte:A%2Dbyte:A%2D[encode](byte[], byte[])++ (link:{java9-javadoc}/java/util/Base64.Encoder.html#encode%2Dbyte:A%2Dbyte:A%2D[java 9])
+* ++[[painless-api-reference-Base64-Encoder-encodeToString-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Base64.Encoder.html#encodeToString%2Dbyte:A%2D[encodeToString](byte[])++ (link:{java9-javadoc}/java/util/Base64.Encoder.html#encodeToString%2Dbyte:A%2D[java 9])
+* ++[[painless-api-reference-Base64-Encoder-withoutPadding-0]]<<painless-api-reference-Base64-Encoder,Base64.Encoder>> link:{java8-javadoc}/java/util/Base64.Encoder.html#withoutPadding%2D%2D[withoutPadding]()++ (link:{java9-javadoc}/java/util/Base64.Encoder.html#withoutPadding%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Base64.asciidoc b/docs/painless/painless-api-reference/Base64.asciidoc
new file mode 100644
index 0000000000000..795f538d7d479
--- /dev/null
+++ b/docs/painless/painless-api-reference/Base64.asciidoc
@@ -0,0 +1,14 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Base64]]++Base64++::
+* ++[[painless-api-reference-Base64-getDecoder-0]]static <<painless-api-reference-Base64-Decoder,Base64.Decoder>> link:{java8-javadoc}/java/util/Base64.html#getDecoder%2D%2D[getDecoder]()++ (link:{java9-javadoc}/java/util/Base64.html#getDecoder%2D%2D[java 9])
+* ++[[painless-api-reference-Base64-getEncoder-0]]static <<painless-api-reference-Base64-Encoder,Base64.Encoder>> link:{java8-javadoc}/java/util/Base64.html#getEncoder%2D%2D[getEncoder]()++ (link:{java9-javadoc}/java/util/Base64.html#getEncoder%2D%2D[java 9])
+* ++[[painless-api-reference-Base64-getMimeDecoder-0]]static <<painless-api-reference-Base64-Decoder,Base64.Decoder>> link:{java8-javadoc}/java/util/Base64.html#getMimeDecoder%2D%2D[getMimeDecoder]()++ (link:{java9-javadoc}/java/util/Base64.html#getMimeDecoder%2D%2D[java 9])
+* ++[[painless-api-reference-Base64-getMimeEncoder-0]]static <<painless-api-reference-Base64-Encoder,Base64.Encoder>> link:{java8-javadoc}/java/util/Base64.html#getMimeEncoder%2D%2D[getMimeEncoder]()++ (link:{java9-javadoc}/java/util/Base64.html#getMimeEncoder%2D%2D[java 9])
+* ++[[painless-api-reference-Base64-getMimeEncoder-2]]static <<painless-api-reference-Base64-Encoder,Base64.Encoder>> link:{java8-javadoc}/java/util/Base64.html#getMimeEncoder%2Dint%2Dbyte:A%2D[getMimeEncoder](int, byte[])++ (link:{java9-javadoc}/java/util/Base64.html#getMimeEncoder%2Dint%2Dbyte:A%2D[java 9])
+* ++[[painless-api-reference-Base64-getUrlDecoder-0]]static <<painless-api-reference-Base64-Decoder,Base64.Decoder>> link:{java8-javadoc}/java/util/Base64.html#getUrlDecoder%2D%2D[getUrlDecoder]()++ (link:{java9-javadoc}/java/util/Base64.html#getUrlDecoder%2D%2D[java 9])
+* ++[[painless-api-reference-Base64-getUrlEncoder-0]]static <<painless-api-reference-Base64-Encoder,Base64.Encoder>> link:{java8-javadoc}/java/util/Base64.html#getUrlEncoder%2D%2D[getUrlEncoder]()++ (link:{java9-javadoc}/java/util/Base64.html#getUrlEncoder%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/BaseStream.asciidoc b/docs/painless/painless-api-reference/BaseStream.asciidoc
new file mode 100644
index 0000000000000..680e9fff71d79
--- /dev/null
+++ b/docs/painless/painless-api-reference/BaseStream.asciidoc
@@ -0,0 +1,13 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-BaseStream]]++BaseStream++::
+* ++[[painless-api-reference-BaseStream-close-0]]void link:{java8-javadoc}/java/util/stream/BaseStream.html#close%2D%2D[close]()++ (link:{java9-javadoc}/java/util/stream/BaseStream.html#close%2D%2D[java 9])
+* ++[[painless-api-reference-BaseStream-isParallel-0]]boolean link:{java8-javadoc}/java/util/stream/BaseStream.html#isParallel%2D%2D[isParallel]()++ (link:{java9-javadoc}/java/util/stream/BaseStream.html#isParallel%2D%2D[java 9])
+* ++[[painless-api-reference-BaseStream-iterator-0]]<<painless-api-reference-Iterator,Iterator>> link:{java8-javadoc}/java/util/stream/BaseStream.html#iterator%2D%2D[iterator]()++ (link:{java9-javadoc}/java/util/stream/BaseStream.html#iterator%2D%2D[java 9])
+* ++[[painless-api-reference-BaseStream-sequential-0]]<<painless-api-reference-BaseStream,BaseStream>> link:{java8-javadoc}/java/util/stream/BaseStream.html#sequential%2D%2D[sequential]()++ (link:{java9-javadoc}/java/util/stream/BaseStream.html#sequential%2D%2D[java 9])
+* ++[[painless-api-reference-BaseStream-spliterator-0]]<<painless-api-reference-Spliterator,Spliterator>> link:{java8-javadoc}/java/util/stream/BaseStream.html#spliterator%2D%2D[spliterator]()++ (link:{java9-javadoc}/java/util/stream/BaseStream.html#spliterator%2D%2D[java 9])
+* ++[[painless-api-reference-BaseStream-unordered-0]]<<painless-api-reference-BaseStream,BaseStream>> link:{java8-javadoc}/java/util/stream/BaseStream.html#unordered%2D%2D[unordered]()++ (link:{java9-javadoc}/java/util/stream/BaseStream.html#unordered%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/BiConsumer.asciidoc b/docs/painless/painless-api-reference/BiConsumer.asciidoc
new file mode 100644
index 0000000000000..a13bbdbd300e2
--- /dev/null
+++ b/docs/painless/painless-api-reference/BiConsumer.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-BiConsumer]]++BiConsumer++::
+* ++[[painless-api-reference-BiConsumer-accept-2]]void link:{java8-javadoc}/java/util/function/BiConsumer.html#accept%2Djava.lang.Object%2Djava.lang.Object%2D[accept](def, def)++ (link:{java9-javadoc}/java/util/function/BiConsumer.html#accept%2Djava.lang.Object%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-BiConsumer-andThen-1]]<<painless-api-reference-BiConsumer,BiConsumer>> link:{java8-javadoc}/java/util/function/BiConsumer.html#andThen%2Djava.util.function.BiConsumer%2D[andThen](<<painless-api-reference-BiConsumer,BiConsumer>>)++ (link:{java9-javadoc}/java/util/function/BiConsumer.html#andThen%2Djava.util.function.BiConsumer%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/BiFunction.asciidoc b/docs/painless/painless-api-reference/BiFunction.asciidoc
new file mode 100644
index 0000000000000..88b316300321e
--- /dev/null
+++ b/docs/painless/painless-api-reference/BiFunction.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-BiFunction]]++BiFunction++::
+* ++[[painless-api-reference-BiFunction-andThen-1]]<<painless-api-reference-BiFunction,BiFunction>> link:{java8-javadoc}/java/util/function/BiFunction.html#andThen%2Djava.util.function.Function%2D[andThen](<<painless-api-reference-Function,Function>>)++ (link:{java9-javadoc}/java/util/function/BiFunction.html#andThen%2Djava.util.function.Function%2D[java 9])
+* ++[[painless-api-reference-BiFunction-apply-2]]def link:{java8-javadoc}/java/util/function/BiFunction.html#apply%2Djava.lang.Object%2Djava.lang.Object%2D[apply](def, def)++ (link:{java9-javadoc}/java/util/function/BiFunction.html#apply%2Djava.lang.Object%2Djava.lang.Object%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/BiPredicate.asciidoc b/docs/painless/painless-api-reference/BiPredicate.asciidoc
new file mode 100644
index 0000000000000..7961800814cc6
--- /dev/null
+++ b/docs/painless/painless-api-reference/BiPredicate.asciidoc
@@ -0,0 +1,11 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-BiPredicate]]++BiPredicate++::
+* ++[[painless-api-reference-BiPredicate-and-1]]<<painless-api-reference-BiPredicate,BiPredicate>> link:{java8-javadoc}/java/util/function/BiPredicate.html#and%2Djava.util.function.BiPredicate%2D[and](<<painless-api-reference-BiPredicate,BiPredicate>>)++ (link:{java9-javadoc}/java/util/function/BiPredicate.html#and%2Djava.util.function.BiPredicate%2D[java 9])
+* ++[[painless-api-reference-BiPredicate-negate-0]]<<painless-api-reference-BiPredicate,BiPredicate>> link:{java8-javadoc}/java/util/function/BiPredicate.html#negate%2D%2D[negate]()++ (link:{java9-javadoc}/java/util/function/BiPredicate.html#negate%2D%2D[java 9])
+* ++[[painless-api-reference-BiPredicate-or-1]]<<painless-api-reference-BiPredicate,BiPredicate>> link:{java8-javadoc}/java/util/function/BiPredicate.html#or%2Djava.util.function.BiPredicate%2D[or](<<painless-api-reference-BiPredicate,BiPredicate>>)++ (link:{java9-javadoc}/java/util/function/BiPredicate.html#or%2Djava.util.function.BiPredicate%2D[java 9])
+* ++[[painless-api-reference-BiPredicate-test-2]]boolean link:{java8-javadoc}/java/util/function/BiPredicate.html#test%2Djava.lang.Object%2Djava.lang.Object%2D[test](def, def)++ (link:{java9-javadoc}/java/util/function/BiPredicate.html#test%2Djava.lang.Object%2Djava.lang.Object%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Bidi.asciidoc b/docs/painless/painless-api-reference/Bidi.asciidoc
new file mode 100644
index 0000000000000..cf6272dd49cf5
--- /dev/null
+++ b/docs/painless/painless-api-reference/Bidi.asciidoc
@@ -0,0 +1,28 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Bidi]]++Bidi++::
+** [[painless-api-reference-Bidi-DIRECTION_DEFAULT_LEFT_TO_RIGHT]]static int link:{java8-javadoc}/java/text/Bidi.html#DIRECTION_DEFAULT_LEFT_TO_RIGHT[DIRECTION_DEFAULT_LEFT_TO_RIGHT] (link:{java9-javadoc}/java/text/Bidi.html#DIRECTION_DEFAULT_LEFT_TO_RIGHT[java 9])
+** [[painless-api-reference-Bidi-DIRECTION_DEFAULT_RIGHT_TO_LEFT]]static int link:{java8-javadoc}/java/text/Bidi.html#DIRECTION_DEFAULT_RIGHT_TO_LEFT[DIRECTION_DEFAULT_RIGHT_TO_LEFT] (link:{java9-javadoc}/java/text/Bidi.html#DIRECTION_DEFAULT_RIGHT_TO_LEFT[java 9])
+** [[painless-api-reference-Bidi-DIRECTION_LEFT_TO_RIGHT]]static int link:{java8-javadoc}/java/text/Bidi.html#DIRECTION_LEFT_TO_RIGHT[DIRECTION_LEFT_TO_RIGHT] (link:{java9-javadoc}/java/text/Bidi.html#DIRECTION_LEFT_TO_RIGHT[java 9])
+** [[painless-api-reference-Bidi-DIRECTION_RIGHT_TO_LEFT]]static int link:{java8-javadoc}/java/text/Bidi.html#DIRECTION_RIGHT_TO_LEFT[DIRECTION_RIGHT_TO_LEFT] (link:{java9-javadoc}/java/text/Bidi.html#DIRECTION_RIGHT_TO_LEFT[java 9])
+* ++[[painless-api-reference-Bidi-reorderVisually-5]]static void link:{java8-javadoc}/java/text/Bidi.html#reorderVisually%2Dbyte:A%2Dint%2Djava.lang.Object:A%2Dint%2Dint%2D[reorderVisually](byte[], int, <<painless-api-reference-Object,Object>>[], int, int)++ (link:{java9-javadoc}/java/text/Bidi.html#reorderVisually%2Dbyte:A%2Dint%2Djava.lang.Object:A%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Bidi-requiresBidi-3]]static boolean link:{java8-javadoc}/java/text/Bidi.html#requiresBidi%2Dchar:A%2Dint%2Dint%2D[requiresBidi](char[], int, int)++ (link:{java9-javadoc}/java/text/Bidi.html#requiresBidi%2Dchar:A%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Bidi-Bidi-1]]link:{java8-javadoc}/java/text/Bidi.html#Bidi%2Djava.text.AttributedCharacterIterator%2D[Bidi](<<painless-api-reference-AttributedCharacterIterator,AttributedCharacterIterator>>)++ (link:{java9-javadoc}/java/text/Bidi.html#Bidi%2Djava.text.AttributedCharacterIterator%2D[java 9])
+* ++[[painless-api-reference-Bidi-Bidi-2]]link:{java8-javadoc}/java/text/Bidi.html#Bidi%2Djava.lang.String%2Dint%2D[Bidi](<<painless-api-reference-String,String>>, int)++ (link:{java9-javadoc}/java/text/Bidi.html#Bidi%2Djava.lang.String%2Dint%2D[java 9])
+* ++[[painless-api-reference-Bidi-Bidi-6]]link:{java8-javadoc}/java/text/Bidi.html#Bidi%2Dchar:A%2Dint%2Dbyte:A%2Dint%2Dint%2Dint%2D[Bidi](char[], int, byte[], int, int, int)++ (link:{java9-javadoc}/java/text/Bidi.html#Bidi%2Dchar:A%2Dint%2Dbyte:A%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Bidi-baseIsLeftToRight-0]]boolean link:{java8-javadoc}/java/text/Bidi.html#baseIsLeftToRight%2D%2D[baseIsLeftToRight]()++ (link:{java9-javadoc}/java/text/Bidi.html#baseIsLeftToRight%2D%2D[java 9])
+* ++[[painless-api-reference-Bidi-createLineBidi-2]]<<painless-api-reference-Bidi,Bidi>> link:{java8-javadoc}/java/text/Bidi.html#createLineBidi%2Dint%2Dint%2D[createLineBidi](int, int)++ (link:{java9-javadoc}/java/text/Bidi.html#createLineBidi%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Bidi-getBaseLevel-0]]int link:{java8-javadoc}/java/text/Bidi.html#getBaseLevel%2D%2D[getBaseLevel]()++ (link:{java9-javadoc}/java/text/Bidi.html#getBaseLevel%2D%2D[java 9])
+* ++[[painless-api-reference-Bidi-getLength-0]]int link:{java8-javadoc}/java/text/Bidi.html#getLength%2D%2D[getLength]()++ (link:{java9-javadoc}/java/text/Bidi.html#getLength%2D%2D[java 9])
+* ++[[painless-api-reference-Bidi-getLevelAt-1]]int link:{java8-javadoc}/java/text/Bidi.html#getLevelAt%2Dint%2D[getLevelAt](int)++ (link:{java9-javadoc}/java/text/Bidi.html#getLevelAt%2Dint%2D[java 9])
+* ++[[painless-api-reference-Bidi-getRunCount-0]]int link:{java8-javadoc}/java/text/Bidi.html#getRunCount%2D%2D[getRunCount]()++ (link:{java9-javadoc}/java/text/Bidi.html#getRunCount%2D%2D[java 9])
+* ++[[painless-api-reference-Bidi-getRunLevel-1]]int link:{java8-javadoc}/java/text/Bidi.html#getRunLevel%2Dint%2D[getRunLevel](int)++ (link:{java9-javadoc}/java/text/Bidi.html#getRunLevel%2Dint%2D[java 9])
+* ++[[painless-api-reference-Bidi-getRunLimit-1]]int link:{java8-javadoc}/java/text/Bidi.html#getRunLimit%2Dint%2D[getRunLimit](int)++ (link:{java9-javadoc}/java/text/Bidi.html#getRunLimit%2Dint%2D[java 9])
+* ++[[painless-api-reference-Bidi-getRunStart-1]]int link:{java8-javadoc}/java/text/Bidi.html#getRunStart%2Dint%2D[getRunStart](int)++ (link:{java9-javadoc}/java/text/Bidi.html#getRunStart%2Dint%2D[java 9])
+* ++[[painless-api-reference-Bidi-isLeftToRight-0]]boolean link:{java8-javadoc}/java/text/Bidi.html#isLeftToRight%2D%2D[isLeftToRight]()++ (link:{java9-javadoc}/java/text/Bidi.html#isLeftToRight%2D%2D[java 9])
+* ++[[painless-api-reference-Bidi-isMixed-0]]boolean link:{java8-javadoc}/java/text/Bidi.html#isMixed%2D%2D[isMixed]()++ (link:{java9-javadoc}/java/text/Bidi.html#isMixed%2D%2D[java 9])
+* ++[[painless-api-reference-Bidi-isRightToLeft-0]]boolean link:{java8-javadoc}/java/text/Bidi.html#isRightToLeft%2D%2D[isRightToLeft]()++ (link:{java9-javadoc}/java/text/Bidi.html#isRightToLeft%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/BigDecimal.asciidoc b/docs/painless/painless-api-reference/BigDecimal.asciidoc
new file mode 100644
index 0000000000000..ab8a78f35c15b
--- /dev/null
+++ b/docs/painless/painless-api-reference/BigDecimal.asciidoc
@@ -0,0 +1,57 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-BigDecimal]]++BigDecimal++::
+** [[painless-api-reference-BigDecimal-ONE]]static <<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#ONE[ONE] (link:{java9-javadoc}/java/math/BigDecimal.html#ONE[java 9])
+** [[painless-api-reference-BigDecimal-TEN]]static <<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#TEN[TEN] (link:{java9-javadoc}/java/math/BigDecimal.html#TEN[java 9])
+** [[painless-api-reference-BigDecimal-ZERO]]static <<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#ZERO[ZERO] (link:{java9-javadoc}/java/math/BigDecimal.html#ZERO[java 9])
+* ++[[painless-api-reference-BigDecimal-valueOf-1]]static <<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#valueOf%2Ddouble%2D[valueOf](double)++ (link:{java9-javadoc}/java/math/BigDecimal.html#valueOf%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-BigDecimal-1]]link:{java8-javadoc}/java/math/BigDecimal.html#BigDecimal%2Djava.lang.String%2D[BigDecimal](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#BigDecimal%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-BigDecimal-2]]link:{java8-javadoc}/java/math/BigDecimal.html#BigDecimal%2Djava.lang.String%2Djava.math.MathContext%2D[BigDecimal](<<painless-api-reference-String,String>>, <<painless-api-reference-MathContext,MathContext>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#BigDecimal%2Djava.lang.String%2Djava.math.MathContext%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-abs-0]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#abs%2D%2D[abs]()++ (link:{java9-javadoc}/java/math/BigDecimal.html#abs%2D%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-abs-1]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#abs%2Djava.math.MathContext%2D[abs](<<painless-api-reference-MathContext,MathContext>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#abs%2Djava.math.MathContext%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-add-1]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#add%2Djava.math.BigDecimal%2D[add](<<painless-api-reference-BigDecimal,BigDecimal>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#add%2Djava.math.BigDecimal%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-add-2]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#add%2Djava.math.BigDecimal%2Djava.math.MathContext%2D[add](<<painless-api-reference-BigDecimal,BigDecimal>>, <<painless-api-reference-MathContext,MathContext>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#add%2Djava.math.BigDecimal%2Djava.math.MathContext%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-byteValueExact-0]]byte link:{java8-javadoc}/java/math/BigDecimal.html#byteValueExact%2D%2D[byteValueExact]()++ (link:{java9-javadoc}/java/math/BigDecimal.html#byteValueExact%2D%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-compareTo-1]]int link:{java8-javadoc}/java/math/BigDecimal.html#compareTo%2Djava.math.BigDecimal%2D[compareTo](<<painless-api-reference-BigDecimal,BigDecimal>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#compareTo%2Djava.math.BigDecimal%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-divide-1]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#divide%2Djava.math.BigDecimal%2D[divide](<<painless-api-reference-BigDecimal,BigDecimal>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#divide%2Djava.math.BigDecimal%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-divide-2]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#divide%2Djava.math.BigDecimal%2Djava.math.MathContext%2D[divide](<<painless-api-reference-BigDecimal,BigDecimal>>, <<painless-api-reference-MathContext,MathContext>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#divide%2Djava.math.BigDecimal%2Djava.math.MathContext%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-divideAndRemainder-1]]<<painless-api-reference-BigDecimal,BigDecimal>>[] link:{java8-javadoc}/java/math/BigDecimal.html#divideAndRemainder%2Djava.math.BigDecimal%2D[divideAndRemainder](<<painless-api-reference-BigDecimal,BigDecimal>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#divideAndRemainder%2Djava.math.BigDecimal%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-divideAndRemainder-2]]<<painless-api-reference-BigDecimal,BigDecimal>>[] link:{java8-javadoc}/java/math/BigDecimal.html#divideAndRemainder%2Djava.math.BigDecimal%2Djava.math.MathContext%2D[divideAndRemainder](<<painless-api-reference-BigDecimal,BigDecimal>>, <<painless-api-reference-MathContext,MathContext>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#divideAndRemainder%2Djava.math.BigDecimal%2Djava.math.MathContext%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-divideToIntegralValue-1]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#divideToIntegralValue%2Djava.math.BigDecimal%2D[divideToIntegralValue](<<painless-api-reference-BigDecimal,BigDecimal>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#divideToIntegralValue%2Djava.math.BigDecimal%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-divideToIntegralValue-2]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#divideToIntegralValue%2Djava.math.BigDecimal%2Djava.math.MathContext%2D[divideToIntegralValue](<<painless-api-reference-BigDecimal,BigDecimal>>, <<painless-api-reference-MathContext,MathContext>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#divideToIntegralValue%2Djava.math.BigDecimal%2Djava.math.MathContext%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-intValueExact-0]]int link:{java8-javadoc}/java/math/BigDecimal.html#intValueExact%2D%2D[intValueExact]()++ (link:{java9-javadoc}/java/math/BigDecimal.html#intValueExact%2D%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-longValueExact-0]]long link:{java8-javadoc}/java/math/BigDecimal.html#longValueExact%2D%2D[longValueExact]()++ (link:{java9-javadoc}/java/math/BigDecimal.html#longValueExact%2D%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-max-1]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#max%2Djava.math.BigDecimal%2D[max](<<painless-api-reference-BigDecimal,BigDecimal>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#max%2Djava.math.BigDecimal%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-min-1]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#min%2Djava.math.BigDecimal%2D[min](<<painless-api-reference-BigDecimal,BigDecimal>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#min%2Djava.math.BigDecimal%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-movePointLeft-1]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#movePointLeft%2Dint%2D[movePointLeft](int)++ (link:{java9-javadoc}/java/math/BigDecimal.html#movePointLeft%2Dint%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-movePointRight-1]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#movePointRight%2Dint%2D[movePointRight](int)++ (link:{java9-javadoc}/java/math/BigDecimal.html#movePointRight%2Dint%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-multiply-1]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#multiply%2Djava.math.BigDecimal%2D[multiply](<<painless-api-reference-BigDecimal,BigDecimal>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#multiply%2Djava.math.BigDecimal%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-multiply-2]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#multiply%2Djava.math.BigDecimal%2Djava.math.MathContext%2D[multiply](<<painless-api-reference-BigDecimal,BigDecimal>>, <<painless-api-reference-MathContext,MathContext>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#multiply%2Djava.math.BigDecimal%2Djava.math.MathContext%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-negate-0]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#negate%2D%2D[negate]()++ (link:{java9-javadoc}/java/math/BigDecimal.html#negate%2D%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-negate-1]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#negate%2Djava.math.MathContext%2D[negate](<<painless-api-reference-MathContext,MathContext>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#negate%2Djava.math.MathContext%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-plus-0]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#plus%2D%2D[plus]()++ (link:{java9-javadoc}/java/math/BigDecimal.html#plus%2D%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-plus-1]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#plus%2Djava.math.MathContext%2D[plus](<<painless-api-reference-MathContext,MathContext>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#plus%2Djava.math.MathContext%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-pow-1]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#pow%2Dint%2D[pow](int)++ (link:{java9-javadoc}/java/math/BigDecimal.html#pow%2Dint%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-pow-2]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#pow%2Dint%2Djava.math.MathContext%2D[pow](int, <<painless-api-reference-MathContext,MathContext>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#pow%2Dint%2Djava.math.MathContext%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-precision-0]]int link:{java8-javadoc}/java/math/BigDecimal.html#precision%2D%2D[precision]()++ (link:{java9-javadoc}/java/math/BigDecimal.html#precision%2D%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-remainder-1]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#remainder%2Djava.math.BigDecimal%2D[remainder](<<painless-api-reference-BigDecimal,BigDecimal>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#remainder%2Djava.math.BigDecimal%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-remainder-2]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#remainder%2Djava.math.BigDecimal%2Djava.math.MathContext%2D[remainder](<<painless-api-reference-BigDecimal,BigDecimal>>, <<painless-api-reference-MathContext,MathContext>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#remainder%2Djava.math.BigDecimal%2Djava.math.MathContext%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-round-1]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#round%2Djava.math.MathContext%2D[round](<<painless-api-reference-MathContext,MathContext>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#round%2Djava.math.MathContext%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-scale-0]]int link:{java8-javadoc}/java/math/BigDecimal.html#scale%2D%2D[scale]()++ (link:{java9-javadoc}/java/math/BigDecimal.html#scale%2D%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-scaleByPowerOfTen-1]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#scaleByPowerOfTen%2Dint%2D[scaleByPowerOfTen](int)++ (link:{java9-javadoc}/java/math/BigDecimal.html#scaleByPowerOfTen%2Dint%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-setScale-1]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#setScale%2Dint%2D[setScale](int)++ (link:{java9-javadoc}/java/math/BigDecimal.html#setScale%2Dint%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-setScale-2]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#setScale%2Dint%2Djava.math.RoundingMode%2D[setScale](int, <<painless-api-reference-RoundingMode,RoundingMode>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#setScale%2Dint%2Djava.math.RoundingMode%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-shortValueExact-0]]short link:{java8-javadoc}/java/math/BigDecimal.html#shortValueExact%2D%2D[shortValueExact]()++ (link:{java9-javadoc}/java/math/BigDecimal.html#shortValueExact%2D%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-signum-0]]int link:{java8-javadoc}/java/math/BigDecimal.html#signum%2D%2D[signum]()++ (link:{java9-javadoc}/java/math/BigDecimal.html#signum%2D%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-stripTrailingZeros-0]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#stripTrailingZeros%2D%2D[stripTrailingZeros]()++ (link:{java9-javadoc}/java/math/BigDecimal.html#stripTrailingZeros%2D%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-subtract-1]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#subtract%2Djava.math.BigDecimal%2D[subtract](<<painless-api-reference-BigDecimal,BigDecimal>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#subtract%2Djava.math.BigDecimal%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-subtract-2]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#subtract%2Djava.math.BigDecimal%2Djava.math.MathContext%2D[subtract](<<painless-api-reference-BigDecimal,BigDecimal>>, <<painless-api-reference-MathContext,MathContext>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#subtract%2Djava.math.BigDecimal%2Djava.math.MathContext%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-toBigInteger-0]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigDecimal.html#toBigInteger%2D%2D[toBigInteger]()++ (link:{java9-javadoc}/java/math/BigDecimal.html#toBigInteger%2D%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-toBigIntegerExact-0]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigDecimal.html#toBigIntegerExact%2D%2D[toBigIntegerExact]()++ (link:{java9-javadoc}/java/math/BigDecimal.html#toBigIntegerExact%2D%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-toEngineeringString-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/math/BigDecimal.html#toEngineeringString%2D%2D[toEngineeringString]()++ (link:{java9-javadoc}/java/math/BigDecimal.html#toEngineeringString%2D%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-toPlainString-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/math/BigDecimal.html#toPlainString%2D%2D[toPlainString]()++ (link:{java9-javadoc}/java/math/BigDecimal.html#toPlainString%2D%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-ulp-0]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#ulp%2D%2D[ulp]()++ (link:{java9-javadoc}/java/math/BigDecimal.html#ulp%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Number,Number>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/BigInteger.asciidoc b/docs/painless/painless-api-reference/BigInteger.asciidoc
new file mode 100644
index 0000000000000..796355cadd3b7
--- /dev/null
+++ b/docs/painless/painless-api-reference/BigInteger.asciidoc
@@ -0,0 +1,50 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-BigInteger]]++BigInteger++::
+** [[painless-api-reference-BigInteger-ONE]]static <<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#ONE[ONE] (link:{java9-javadoc}/java/math/BigInteger.html#ONE[java 9])
+** [[painless-api-reference-BigInteger-TEN]]static <<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#TEN[TEN] (link:{java9-javadoc}/java/math/BigInteger.html#TEN[java 9])
+** [[painless-api-reference-BigInteger-ZERO]]static <<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#ZERO[ZERO] (link:{java9-javadoc}/java/math/BigInteger.html#ZERO[java 9])
+* ++[[painless-api-reference-BigInteger-valueOf-1]]static <<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#valueOf%2Dlong%2D[valueOf](long)++ (link:{java9-javadoc}/java/math/BigInteger.html#valueOf%2Dlong%2D[java 9])
+* ++[[painless-api-reference-BigInteger-BigInteger-1]]link:{java8-javadoc}/java/math/BigInteger.html#BigInteger%2Djava.lang.String%2D[BigInteger](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/math/BigInteger.html#BigInteger%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-BigInteger-BigInteger-2]]link:{java8-javadoc}/java/math/BigInteger.html#BigInteger%2Djava.lang.String%2Dint%2D[BigInteger](<<painless-api-reference-String,String>>, int)++ (link:{java9-javadoc}/java/math/BigInteger.html#BigInteger%2Djava.lang.String%2Dint%2D[java 9])
+* ++[[painless-api-reference-BigInteger-abs-0]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#abs%2D%2D[abs]()++ (link:{java9-javadoc}/java/math/BigInteger.html#abs%2D%2D[java 9])
+* ++[[painless-api-reference-BigInteger-add-1]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#add%2Djava.math.BigInteger%2D[add](<<painless-api-reference-BigInteger,BigInteger>>)++ (link:{java9-javadoc}/java/math/BigInteger.html#add%2Djava.math.BigInteger%2D[java 9])
+* ++[[painless-api-reference-BigInteger-and-1]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#and%2Djava.math.BigInteger%2D[and](<<painless-api-reference-BigInteger,BigInteger>>)++ (link:{java9-javadoc}/java/math/BigInteger.html#and%2Djava.math.BigInteger%2D[java 9])
+* ++[[painless-api-reference-BigInteger-andNot-1]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#andNot%2Djava.math.BigInteger%2D[andNot](<<painless-api-reference-BigInteger,BigInteger>>)++ (link:{java9-javadoc}/java/math/BigInteger.html#andNot%2Djava.math.BigInteger%2D[java 9])
+* ++[[painless-api-reference-BigInteger-bitCount-0]]int link:{java8-javadoc}/java/math/BigInteger.html#bitCount%2D%2D[bitCount]()++ (link:{java9-javadoc}/java/math/BigInteger.html#bitCount%2D%2D[java 9])
+* ++[[painless-api-reference-BigInteger-bitLength-0]]int link:{java8-javadoc}/java/math/BigInteger.html#bitLength%2D%2D[bitLength]()++ (link:{java9-javadoc}/java/math/BigInteger.html#bitLength%2D%2D[java 9])
+* ++[[painless-api-reference-BigInteger-byteValueExact-0]]byte link:{java8-javadoc}/java/math/BigInteger.html#byteValueExact%2D%2D[byteValueExact]()++ (link:{java9-javadoc}/java/math/BigInteger.html#byteValueExact%2D%2D[java 9])
+* ++[[painless-api-reference-BigInteger-clearBit-1]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#clearBit%2Dint%2D[clearBit](int)++ (link:{java9-javadoc}/java/math/BigInteger.html#clearBit%2Dint%2D[java 9])
+* ++[[painless-api-reference-BigInteger-compareTo-1]]int link:{java8-javadoc}/java/math/BigInteger.html#compareTo%2Djava.math.BigInteger%2D[compareTo](<<painless-api-reference-BigInteger,BigInteger>>)++ (link:{java9-javadoc}/java/math/BigInteger.html#compareTo%2Djava.math.BigInteger%2D[java 9])
+* ++[[painless-api-reference-BigInteger-divide-1]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#divide%2Djava.math.BigInteger%2D[divide](<<painless-api-reference-BigInteger,BigInteger>>)++ (link:{java9-javadoc}/java/math/BigInteger.html#divide%2Djava.math.BigInteger%2D[java 9])
+* ++[[painless-api-reference-BigInteger-divideAndRemainder-1]]<<painless-api-reference-BigInteger,BigInteger>>[] link:{java8-javadoc}/java/math/BigInteger.html#divideAndRemainder%2Djava.math.BigInteger%2D[divideAndRemainder](<<painless-api-reference-BigInteger,BigInteger>>)++ (link:{java9-javadoc}/java/math/BigInteger.html#divideAndRemainder%2Djava.math.BigInteger%2D[java 9])
+* ++[[painless-api-reference-BigInteger-flipBit-1]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#flipBit%2Dint%2D[flipBit](int)++ (link:{java9-javadoc}/java/math/BigInteger.html#flipBit%2Dint%2D[java 9])
+* ++[[painless-api-reference-BigInteger-gcd-1]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#gcd%2Djava.math.BigInteger%2D[gcd](<<painless-api-reference-BigInteger,BigInteger>>)++ (link:{java9-javadoc}/java/math/BigInteger.html#gcd%2Djava.math.BigInteger%2D[java 9])
+* ++[[painless-api-reference-BigInteger-getLowestSetBit-0]]int link:{java8-javadoc}/java/math/BigInteger.html#getLowestSetBit%2D%2D[getLowestSetBit]()++ (link:{java9-javadoc}/java/math/BigInteger.html#getLowestSetBit%2D%2D[java 9])
+* ++[[painless-api-reference-BigInteger-intValueExact-0]]int link:{java8-javadoc}/java/math/BigInteger.html#intValueExact%2D%2D[intValueExact]()++ (link:{java9-javadoc}/java/math/BigInteger.html#intValueExact%2D%2D[java 9])
+* ++[[painless-api-reference-BigInteger-longValueExact-0]]long link:{java8-javadoc}/java/math/BigInteger.html#longValueExact%2D%2D[longValueExact]()++ (link:{java9-javadoc}/java/math/BigInteger.html#longValueExact%2D%2D[java 9])
+* ++[[painless-api-reference-BigInteger-max-1]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#max%2Djava.math.BigInteger%2D[max](<<painless-api-reference-BigInteger,BigInteger>>)++ (link:{java9-javadoc}/java/math/BigInteger.html#max%2Djava.math.BigInteger%2D[java 9])
+* ++[[painless-api-reference-BigInteger-min-1]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#min%2Djava.math.BigInteger%2D[min](<<painless-api-reference-BigInteger,BigInteger>>)++ (link:{java9-javadoc}/java/math/BigInteger.html#min%2Djava.math.BigInteger%2D[java 9])
+* ++[[painless-api-reference-BigInteger-mod-1]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#mod%2Djava.math.BigInteger%2D[mod](<<painless-api-reference-BigInteger,BigInteger>>)++ (link:{java9-javadoc}/java/math/BigInteger.html#mod%2Djava.math.BigInteger%2D[java 9])
+* ++[[painless-api-reference-BigInteger-modInverse-1]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#modInverse%2Djava.math.BigInteger%2D[modInverse](<<painless-api-reference-BigInteger,BigInteger>>)++ (link:{java9-javadoc}/java/math/BigInteger.html#modInverse%2Djava.math.BigInteger%2D[java 9])
+* ++[[painless-api-reference-BigInteger-modPow-2]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#modPow%2Djava.math.BigInteger%2Djava.math.BigInteger%2D[modPow](<<painless-api-reference-BigInteger,BigInteger>>, <<painless-api-reference-BigInteger,BigInteger>>)++ (link:{java9-javadoc}/java/math/BigInteger.html#modPow%2Djava.math.BigInteger%2Djava.math.BigInteger%2D[java 9])
+* ++[[painless-api-reference-BigInteger-multiply-1]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#multiply%2Djava.math.BigInteger%2D[multiply](<<painless-api-reference-BigInteger,BigInteger>>)++ (link:{java9-javadoc}/java/math/BigInteger.html#multiply%2Djava.math.BigInteger%2D[java 9])
+* ++[[painless-api-reference-BigInteger-negate-0]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#negate%2D%2D[negate]()++ (link:{java9-javadoc}/java/math/BigInteger.html#negate%2D%2D[java 9])
+* ++[[painless-api-reference-BigInteger-not-0]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#not%2D%2D[not]()++ (link:{java9-javadoc}/java/math/BigInteger.html#not%2D%2D[java 9])
+* ++[[painless-api-reference-BigInteger-or-1]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#or%2Djava.math.BigInteger%2D[or](<<painless-api-reference-BigInteger,BigInteger>>)++ (link:{java9-javadoc}/java/math/BigInteger.html#or%2Djava.math.BigInteger%2D[java 9])
+* ++[[painless-api-reference-BigInteger-pow-1]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#pow%2Dint%2D[pow](int)++ (link:{java9-javadoc}/java/math/BigInteger.html#pow%2Dint%2D[java 9])
+* ++[[painless-api-reference-BigInteger-remainder-1]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#remainder%2Djava.math.BigInteger%2D[remainder](<<painless-api-reference-BigInteger,BigInteger>>)++ (link:{java9-javadoc}/java/math/BigInteger.html#remainder%2Djava.math.BigInteger%2D[java 9])
+* ++[[painless-api-reference-BigInteger-setBit-1]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#setBit%2Dint%2D[setBit](int)++ (link:{java9-javadoc}/java/math/BigInteger.html#setBit%2Dint%2D[java 9])
+* ++[[painless-api-reference-BigInteger-shiftLeft-1]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#shiftLeft%2Dint%2D[shiftLeft](int)++ (link:{java9-javadoc}/java/math/BigInteger.html#shiftLeft%2Dint%2D[java 9])
+* ++[[painless-api-reference-BigInteger-shiftRight-1]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#shiftRight%2Dint%2D[shiftRight](int)++ (link:{java9-javadoc}/java/math/BigInteger.html#shiftRight%2Dint%2D[java 9])
+* ++[[painless-api-reference-BigInteger-shortValueExact-0]]short link:{java8-javadoc}/java/math/BigInteger.html#shortValueExact%2D%2D[shortValueExact]()++ (link:{java9-javadoc}/java/math/BigInteger.html#shortValueExact%2D%2D[java 9])
+* ++[[painless-api-reference-BigInteger-signum-0]]int link:{java8-javadoc}/java/math/BigInteger.html#signum%2D%2D[signum]()++ (link:{java9-javadoc}/java/math/BigInteger.html#signum%2D%2D[java 9])
+* ++[[painless-api-reference-BigInteger-subtract-1]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#subtract%2Djava.math.BigInteger%2D[subtract](<<painless-api-reference-BigInteger,BigInteger>>)++ (link:{java9-javadoc}/java/math/BigInteger.html#subtract%2Djava.math.BigInteger%2D[java 9])
+* ++[[painless-api-reference-BigInteger-testBit-1]]boolean link:{java8-javadoc}/java/math/BigInteger.html#testBit%2Dint%2D[testBit](int)++ (link:{java9-javadoc}/java/math/BigInteger.html#testBit%2Dint%2D[java 9])
+* ++[[painless-api-reference-BigInteger-toByteArray-0]]byte[] link:{java8-javadoc}/java/math/BigInteger.html#toByteArray%2D%2D[toByteArray]()++ (link:{java9-javadoc}/java/math/BigInteger.html#toByteArray%2D%2D[java 9])
+* ++[[painless-api-reference-BigInteger-toString-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/math/BigInteger.html#toString%2Dint%2D[toString](int)++ (link:{java9-javadoc}/java/math/BigInteger.html#toString%2Dint%2D[java 9])
+* ++[[painless-api-reference-BigInteger-xor-1]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#xor%2Djava.math.BigInteger%2D[xor](<<painless-api-reference-BigInteger,BigInteger>>)++ (link:{java9-javadoc}/java/math/BigInteger.html#xor%2Djava.math.BigInteger%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Number,Number>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/BinaryOperator.asciidoc b/docs/painless/painless-api-reference/BinaryOperator.asciidoc
new file mode 100644
index 0000000000000..c7e53be47f720
--- /dev/null
+++ b/docs/painless/painless-api-reference/BinaryOperator.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-BinaryOperator]]++BinaryOperator++::
+* ++[[painless-api-reference-BinaryOperator-maxBy-1]]static <<painless-api-reference-BinaryOperator,BinaryOperator>> link:{java8-javadoc}/java/util/function/BinaryOperator.html#maxBy%2Djava.util.Comparator%2D[maxBy](<<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/function/BinaryOperator.html#maxBy%2Djava.util.Comparator%2D[java 9])
+* ++[[painless-api-reference-BinaryOperator-minBy-1]]static <<painless-api-reference-BinaryOperator,BinaryOperator>> link:{java8-javadoc}/java/util/function/BinaryOperator.html#minBy%2Djava.util.Comparator%2D[minBy](<<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/function/BinaryOperator.html#minBy%2Djava.util.Comparator%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-BiFunction,BiFunction>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/BitSet.asciidoc b/docs/painless/painless-api-reference/BitSet.asciidoc
new file mode 100644
index 0000000000000..158f64675d93e
--- /dev/null
+++ b/docs/painless/painless-api-reference/BitSet.asciidoc
@@ -0,0 +1,34 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-BitSet]]++BitSet++::
+* ++[[painless-api-reference-BitSet-valueOf-1]]static <<painless-api-reference-BitSet,BitSet>> link:{java8-javadoc}/java/util/BitSet.html#valueOf%2Dlong:A%2D[valueOf](long[])++ (link:{java9-javadoc}/java/util/BitSet.html#valueOf%2Dlong:A%2D[java 9])
+* ++[[painless-api-reference-BitSet-BitSet-0]]link:{java8-javadoc}/java/util/BitSet.html#BitSet%2D%2D[BitSet]()++ (link:{java9-javadoc}/java/util/BitSet.html#BitSet%2D%2D[java 9])
+* ++[[painless-api-reference-BitSet-BitSet-1]]link:{java8-javadoc}/java/util/BitSet.html#BitSet%2Dint%2D[BitSet](int)++ (link:{java9-javadoc}/java/util/BitSet.html#BitSet%2Dint%2D[java 9])
+* ++[[painless-api-reference-BitSet-and-1]]void link:{java8-javadoc}/java/util/BitSet.html#and%2Djava.util.BitSet%2D[and](<<painless-api-reference-BitSet,BitSet>>)++ (link:{java9-javadoc}/java/util/BitSet.html#and%2Djava.util.BitSet%2D[java 9])
+* ++[[painless-api-reference-BitSet-andNot-1]]void link:{java8-javadoc}/java/util/BitSet.html#andNot%2Djava.util.BitSet%2D[andNot](<<painless-api-reference-BitSet,BitSet>>)++ (link:{java9-javadoc}/java/util/BitSet.html#andNot%2Djava.util.BitSet%2D[java 9])
+* ++[[painless-api-reference-BitSet-cardinality-0]]int link:{java8-javadoc}/java/util/BitSet.html#cardinality%2D%2D[cardinality]()++ (link:{java9-javadoc}/java/util/BitSet.html#cardinality%2D%2D[java 9])
+* ++[[painless-api-reference-BitSet-clear-0]]void link:{java8-javadoc}/java/util/BitSet.html#clear%2D%2D[clear]()++ (link:{java9-javadoc}/java/util/BitSet.html#clear%2D%2D[java 9])
+* ++[[painless-api-reference-BitSet-clear-1]]void link:{java8-javadoc}/java/util/BitSet.html#clear%2Dint%2D[clear](int)++ (link:{java9-javadoc}/java/util/BitSet.html#clear%2Dint%2D[java 9])
+* ++[[painless-api-reference-BitSet-clear-2]]void link:{java8-javadoc}/java/util/BitSet.html#clear%2Dint%2Dint%2D[clear](int, int)++ (link:{java9-javadoc}/java/util/BitSet.html#clear%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-BitSet-clone-0]]def link:{java8-javadoc}/java/util/BitSet.html#clone%2D%2D[clone]()++ (link:{java9-javadoc}/java/util/BitSet.html#clone%2D%2D[java 9])
+* ++[[painless-api-reference-BitSet-flip-1]]void link:{java8-javadoc}/java/util/BitSet.html#flip%2Dint%2D[flip](int)++ (link:{java9-javadoc}/java/util/BitSet.html#flip%2Dint%2D[java 9])
+* ++[[painless-api-reference-BitSet-flip-2]]void link:{java8-javadoc}/java/util/BitSet.html#flip%2Dint%2Dint%2D[flip](int, int)++ (link:{java9-javadoc}/java/util/BitSet.html#flip%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-BitSet-intersects-1]]boolean link:{java8-javadoc}/java/util/BitSet.html#intersects%2Djava.util.BitSet%2D[intersects](<<painless-api-reference-BitSet,BitSet>>)++ (link:{java9-javadoc}/java/util/BitSet.html#intersects%2Djava.util.BitSet%2D[java 9])
+* ++[[painless-api-reference-BitSet-isEmpty-0]]boolean link:{java8-javadoc}/java/util/BitSet.html#isEmpty%2D%2D[isEmpty]()++ (link:{java9-javadoc}/java/util/BitSet.html#isEmpty%2D%2D[java 9])
+* ++[[painless-api-reference-BitSet-length-0]]int link:{java8-javadoc}/java/util/BitSet.html#length%2D%2D[length]()++ (link:{java9-javadoc}/java/util/BitSet.html#length%2D%2D[java 9])
+* ++[[painless-api-reference-BitSet-nextClearBit-1]]int link:{java8-javadoc}/java/util/BitSet.html#nextClearBit%2Dint%2D[nextClearBit](int)++ (link:{java9-javadoc}/java/util/BitSet.html#nextClearBit%2Dint%2D[java 9])
+* ++[[painless-api-reference-BitSet-nextSetBit-1]]int link:{java8-javadoc}/java/util/BitSet.html#nextSetBit%2Dint%2D[nextSetBit](int)++ (link:{java9-javadoc}/java/util/BitSet.html#nextSetBit%2Dint%2D[java 9])
+* ++[[painless-api-reference-BitSet-or-1]]void link:{java8-javadoc}/java/util/BitSet.html#or%2Djava.util.BitSet%2D[or](<<painless-api-reference-BitSet,BitSet>>)++ (link:{java9-javadoc}/java/util/BitSet.html#or%2Djava.util.BitSet%2D[java 9])
+* ++[[painless-api-reference-BitSet-previousClearBit-1]]int link:{java8-javadoc}/java/util/BitSet.html#previousClearBit%2Dint%2D[previousClearBit](int)++ (link:{java9-javadoc}/java/util/BitSet.html#previousClearBit%2Dint%2D[java 9])
+* ++[[painless-api-reference-BitSet-previousSetBit-1]]int link:{java8-javadoc}/java/util/BitSet.html#previousSetBit%2Dint%2D[previousSetBit](int)++ (link:{java9-javadoc}/java/util/BitSet.html#previousSetBit%2Dint%2D[java 9])
+* ++[[painless-api-reference-BitSet-set-1]]void link:{java8-javadoc}/java/util/BitSet.html#set%2Dint%2D[set](int)++ (link:{java9-javadoc}/java/util/BitSet.html#set%2Dint%2D[java 9])
+* ++[[painless-api-reference-BitSet-set-2]]void link:{java8-javadoc}/java/util/BitSet.html#set%2Dint%2Dint%2D[set](int, int)++ (link:{java9-javadoc}/java/util/BitSet.html#set%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-BitSet-set-3]]void link:{java8-javadoc}/java/util/BitSet.html#set%2Dint%2Dint%2Dboolean%2D[set](int, int, boolean)++ (link:{java9-javadoc}/java/util/BitSet.html#set%2Dint%2Dint%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-BitSet-size-0]]int link:{java8-javadoc}/java/util/BitSet.html#size%2D%2D[size]()++ (link:{java9-javadoc}/java/util/BitSet.html#size%2D%2D[java 9])
+* ++[[painless-api-reference-BitSet-toByteArray-0]]byte[] link:{java8-javadoc}/java/util/BitSet.html#toByteArray%2D%2D[toByteArray]()++ (link:{java9-javadoc}/java/util/BitSet.html#toByteArray%2D%2D[java 9])
+* ++[[painless-api-reference-BitSet-toLongArray-0]]long[] link:{java8-javadoc}/java/util/BitSet.html#toLongArray%2D%2D[toLongArray]()++ (link:{java9-javadoc}/java/util/BitSet.html#toLongArray%2D%2D[java 9])
+* ++[[painless-api-reference-BitSet-xor-1]]void link:{java8-javadoc}/java/util/BitSet.html#xor%2Djava.util.BitSet%2D[xor](<<painless-api-reference-BitSet,BitSet>>)++ (link:{java9-javadoc}/java/util/BitSet.html#xor%2Djava.util.BitSet%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Boolean.asciidoc b/docs/painless/painless-api-reference/Boolean.asciidoc
new file mode 100644
index 0000000000000..47000e58fa828
--- /dev/null
+++ b/docs/painless/painless-api-reference/Boolean.asciidoc
@@ -0,0 +1,19 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Boolean]]++Boolean++::
+** [[painless-api-reference-Boolean-FALSE]]static <<painless-api-reference-Boolean,Boolean>> link:{java8-javadoc}/java/lang/Boolean.html#FALSE[FALSE] (link:{java9-javadoc}/java/lang/Boolean.html#FALSE[java 9])
+** [[painless-api-reference-Boolean-TRUE]]static <<painless-api-reference-Boolean,Boolean>> link:{java8-javadoc}/java/lang/Boolean.html#TRUE[TRUE] (link:{java9-javadoc}/java/lang/Boolean.html#TRUE[java 9])
+* ++[[painless-api-reference-Boolean-compare-2]]static int link:{java8-javadoc}/java/lang/Boolean.html#compare%2Dboolean%2Dboolean%2D[compare](boolean, boolean)++ (link:{java9-javadoc}/java/lang/Boolean.html#compare%2Dboolean%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-Boolean-hashCode-1]]static int link:{java8-javadoc}/java/lang/Boolean.html#hashCode%2Dboolean%2D[hashCode](boolean)++ (link:{java9-javadoc}/java/lang/Boolean.html#hashCode%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-Boolean-logicalAnd-2]]static boolean link:{java8-javadoc}/java/lang/Boolean.html#logicalAnd%2Dboolean%2Dboolean%2D[logicalAnd](boolean, boolean)++ (link:{java9-javadoc}/java/lang/Boolean.html#logicalAnd%2Dboolean%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-Boolean-logicalOr-2]]static boolean link:{java8-javadoc}/java/lang/Boolean.html#logicalOr%2Dboolean%2Dboolean%2D[logicalOr](boolean, boolean)++ (link:{java9-javadoc}/java/lang/Boolean.html#logicalOr%2Dboolean%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-Boolean-logicalXor-2]]static boolean link:{java8-javadoc}/java/lang/Boolean.html#logicalXor%2Dboolean%2Dboolean%2D[logicalXor](boolean, boolean)++ (link:{java9-javadoc}/java/lang/Boolean.html#logicalXor%2Dboolean%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-Boolean-parseBoolean-1]]static boolean link:{java8-javadoc}/java/lang/Boolean.html#parseBoolean%2Djava.lang.String%2D[parseBoolean](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/Boolean.html#parseBoolean%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Boolean-toString-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Boolean.html#toString%2Dboolean%2D[toString](boolean)++ (link:{java9-javadoc}/java/lang/Boolean.html#toString%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-Boolean-valueOf-1]]static <<painless-api-reference-Boolean,Boolean>> link:{java8-javadoc}/java/lang/Boolean.html#valueOf%2Dboolean%2D[valueOf](boolean)++ (link:{java9-javadoc}/java/lang/Boolean.html#valueOf%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-Boolean-booleanValue-0]]boolean link:{java8-javadoc}/java/lang/Boolean.html#booleanValue%2D%2D[booleanValue]()++ (link:{java9-javadoc}/java/lang/Boolean.html#booleanValue%2D%2D[java 9])
+* ++[[painless-api-reference-Boolean-compareTo-1]]int link:{java8-javadoc}/java/lang/Boolean.html#compareTo%2Djava.lang.Boolean%2D[compareTo](<<painless-api-reference-Boolean,Boolean>>)++ (link:{java9-javadoc}/java/lang/Boolean.html#compareTo%2Djava.lang.Boolean%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/BooleanSupplier.asciidoc b/docs/painless/painless-api-reference/BooleanSupplier.asciidoc
new file mode 100644
index 0000000000000..9a34d1caf2d99
--- /dev/null
+++ b/docs/painless/painless-api-reference/BooleanSupplier.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-BooleanSupplier]]++BooleanSupplier++::
+* ++[[painless-api-reference-BooleanSupplier-getAsBoolean-0]]boolean link:{java8-javadoc}/java/util/function/BooleanSupplier.html#getAsBoolean%2D%2D[getAsBoolean]()++ (link:{java9-javadoc}/java/util/function/BooleanSupplier.html#getAsBoolean%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/BreakIterator.asciidoc b/docs/painless/painless-api-reference/BreakIterator.asciidoc
new file mode 100644
index 0000000000000..0e830859c457f
--- /dev/null
+++ b/docs/painless/painless-api-reference/BreakIterator.asciidoc
@@ -0,0 +1,29 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-BreakIterator]]++BreakIterator++::
+** [[painless-api-reference-BreakIterator-DONE]]static int link:{java8-javadoc}/java/text/BreakIterator.html#DONE[DONE] (link:{java9-javadoc}/java/text/BreakIterator.html#DONE[java 9])
+* ++[[painless-api-reference-BreakIterator-getAvailableLocales-0]]static <<painless-api-reference-Locale,Locale>>[] link:{java8-javadoc}/java/text/BreakIterator.html#getAvailableLocales%2D%2D[getAvailableLocales]()++ (link:{java9-javadoc}/java/text/BreakIterator.html#getAvailableLocales%2D%2D[java 9])
+* ++[[painless-api-reference-BreakIterator-getCharacterInstance-0]]static <<painless-api-reference-BreakIterator,BreakIterator>> link:{java8-javadoc}/java/text/BreakIterator.html#getCharacterInstance%2D%2D[getCharacterInstance]()++ (link:{java9-javadoc}/java/text/BreakIterator.html#getCharacterInstance%2D%2D[java 9])
+* ++[[painless-api-reference-BreakIterator-getCharacterInstance-1]]static <<painless-api-reference-BreakIterator,BreakIterator>> link:{java8-javadoc}/java/text/BreakIterator.html#getCharacterInstance%2Djava.util.Locale%2D[getCharacterInstance](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/text/BreakIterator.html#getCharacterInstance%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-BreakIterator-getLineInstance-0]]static <<painless-api-reference-BreakIterator,BreakIterator>> link:{java8-javadoc}/java/text/BreakIterator.html#getLineInstance%2D%2D[getLineInstance]()++ (link:{java9-javadoc}/java/text/BreakIterator.html#getLineInstance%2D%2D[java 9])
+* ++[[painless-api-reference-BreakIterator-getLineInstance-1]]static <<painless-api-reference-BreakIterator,BreakIterator>> link:{java8-javadoc}/java/text/BreakIterator.html#getLineInstance%2Djava.util.Locale%2D[getLineInstance](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/text/BreakIterator.html#getLineInstance%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-BreakIterator-getSentenceInstance-0]]static <<painless-api-reference-BreakIterator,BreakIterator>> link:{java8-javadoc}/java/text/BreakIterator.html#getSentenceInstance%2D%2D[getSentenceInstance]()++ (link:{java9-javadoc}/java/text/BreakIterator.html#getSentenceInstance%2D%2D[java 9])
+* ++[[painless-api-reference-BreakIterator-getSentenceInstance-1]]static <<painless-api-reference-BreakIterator,BreakIterator>> link:{java8-javadoc}/java/text/BreakIterator.html#getSentenceInstance%2Djava.util.Locale%2D[getSentenceInstance](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/text/BreakIterator.html#getSentenceInstance%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-BreakIterator-getWordInstance-0]]static <<painless-api-reference-BreakIterator,BreakIterator>> link:{java8-javadoc}/java/text/BreakIterator.html#getWordInstance%2D%2D[getWordInstance]()++ (link:{java9-javadoc}/java/text/BreakIterator.html#getWordInstance%2D%2D[java 9])
+* ++[[painless-api-reference-BreakIterator-getWordInstance-1]]static <<painless-api-reference-BreakIterator,BreakIterator>> link:{java8-javadoc}/java/text/BreakIterator.html#getWordInstance%2Djava.util.Locale%2D[getWordInstance](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/text/BreakIterator.html#getWordInstance%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-BreakIterator-clone-0]]def link:{java8-javadoc}/java/text/BreakIterator.html#clone%2D%2D[clone]()++ (link:{java9-javadoc}/java/text/BreakIterator.html#clone%2D%2D[java 9])
+* ++[[painless-api-reference-BreakIterator-current-0]]int link:{java8-javadoc}/java/text/BreakIterator.html#current%2D%2D[current]()++ (link:{java9-javadoc}/java/text/BreakIterator.html#current%2D%2D[java 9])
+* ++[[painless-api-reference-BreakIterator-first-0]]int link:{java8-javadoc}/java/text/BreakIterator.html#first%2D%2D[first]()++ (link:{java9-javadoc}/java/text/BreakIterator.html#first%2D%2D[java 9])
+* ++[[painless-api-reference-BreakIterator-following-1]]int link:{java8-javadoc}/java/text/BreakIterator.html#following%2Dint%2D[following](int)++ (link:{java9-javadoc}/java/text/BreakIterator.html#following%2Dint%2D[java 9])
+* ++[[painless-api-reference-BreakIterator-getText-0]]<<painless-api-reference-CharacterIterator,CharacterIterator>> link:{java8-javadoc}/java/text/BreakIterator.html#getText%2D%2D[getText]()++ (link:{java9-javadoc}/java/text/BreakIterator.html#getText%2D%2D[java 9])
+* ++[[painless-api-reference-BreakIterator-isBoundary-1]]boolean link:{java8-javadoc}/java/text/BreakIterator.html#isBoundary%2Dint%2D[isBoundary](int)++ (link:{java9-javadoc}/java/text/BreakIterator.html#isBoundary%2Dint%2D[java 9])
+* ++[[painless-api-reference-BreakIterator-last-0]]int link:{java8-javadoc}/java/text/BreakIterator.html#last%2D%2D[last]()++ (link:{java9-javadoc}/java/text/BreakIterator.html#last%2D%2D[java 9])
+* ++[[painless-api-reference-BreakIterator-next-0]]int link:{java8-javadoc}/java/text/BreakIterator.html#next%2D%2D[next]()++ (link:{java9-javadoc}/java/text/BreakIterator.html#next%2D%2D[java 9])
+* ++[[painless-api-reference-BreakIterator-next-1]]int link:{java8-javadoc}/java/text/BreakIterator.html#next%2Dint%2D[next](int)++ (link:{java9-javadoc}/java/text/BreakIterator.html#next%2Dint%2D[java 9])
+* ++[[painless-api-reference-BreakIterator-preceding-1]]int link:{java8-javadoc}/java/text/BreakIterator.html#preceding%2Dint%2D[preceding](int)++ (link:{java9-javadoc}/java/text/BreakIterator.html#preceding%2Dint%2D[java 9])
+* ++[[painless-api-reference-BreakIterator-previous-0]]int link:{java8-javadoc}/java/text/BreakIterator.html#previous%2D%2D[previous]()++ (link:{java9-javadoc}/java/text/BreakIterator.html#previous%2D%2D[java 9])
+* ++[[painless-api-reference-BreakIterator-setText-1]]void link:{java8-javadoc}/java/text/BreakIterator.html#setText%2Djava.lang.String%2D[setText](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/BreakIterator.html#setText%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Byte.asciidoc b/docs/painless/painless-api-reference/Byte.asciidoc
new file mode 100644
index 0000000000000..2fc42b6362540
--- /dev/null
+++ b/docs/painless/painless-api-reference/Byte.asciidoc
@@ -0,0 +1,22 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Byte]]++Byte++::
+** [[painless-api-reference-Byte-BYTES]]static int link:{java8-javadoc}/java/lang/Byte.html#BYTES[BYTES] (link:{java9-javadoc}/java/lang/Byte.html#BYTES[java 9])
+** [[painless-api-reference-Byte-MAX_VALUE]]static byte link:{java8-javadoc}/java/lang/Byte.html#MAX_VALUE[MAX_VALUE] (link:{java9-javadoc}/java/lang/Byte.html#MAX_VALUE[java 9])
+** [[painless-api-reference-Byte-MIN_VALUE]]static byte link:{java8-javadoc}/java/lang/Byte.html#MIN_VALUE[MIN_VALUE] (link:{java9-javadoc}/java/lang/Byte.html#MIN_VALUE[java 9])
+** [[painless-api-reference-Byte-SIZE]]static int link:{java8-javadoc}/java/lang/Byte.html#SIZE[SIZE] (link:{java9-javadoc}/java/lang/Byte.html#SIZE[java 9])
+* ++[[painless-api-reference-Byte-compare-2]]static int link:{java8-javadoc}/java/lang/Byte.html#compare%2Dbyte%2Dbyte%2D[compare](byte, byte)++ (link:{java9-javadoc}/java/lang/Byte.html#compare%2Dbyte%2Dbyte%2D[java 9])
+* ++[[painless-api-reference-Byte-decode-1]]static <<painless-api-reference-Byte,Byte>> link:{java8-javadoc}/java/lang/Byte.html#decode%2Djava.lang.String%2D[decode](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/Byte.html#decode%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Byte-hashCode-1]]static int link:{java8-javadoc}/java/lang/Byte.html#hashCode%2Dbyte%2D[hashCode](byte)++ (link:{java9-javadoc}/java/lang/Byte.html#hashCode%2Dbyte%2D[java 9])
+* ++[[painless-api-reference-Byte-parseByte-1]]static byte link:{java8-javadoc}/java/lang/Byte.html#parseByte%2Djava.lang.String%2D[parseByte](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/Byte.html#parseByte%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Byte-parseByte-2]]static byte link:{java8-javadoc}/java/lang/Byte.html#parseByte%2Djava.lang.String%2Dint%2D[parseByte](<<painless-api-reference-String,String>>, int)++ (link:{java9-javadoc}/java/lang/Byte.html#parseByte%2Djava.lang.String%2Dint%2D[java 9])
+* ++[[painless-api-reference-Byte-toString-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Byte.html#toString%2Dbyte%2D[toString](byte)++ (link:{java9-javadoc}/java/lang/Byte.html#toString%2Dbyte%2D[java 9])
+* ++[[painless-api-reference-Byte-toUnsignedInt-1]]static int link:{java8-javadoc}/java/lang/Byte.html#toUnsignedInt%2Dbyte%2D[toUnsignedInt](byte)++ (link:{java9-javadoc}/java/lang/Byte.html#toUnsignedInt%2Dbyte%2D[java 9])
+* ++[[painless-api-reference-Byte-toUnsignedLong-1]]static long link:{java8-javadoc}/java/lang/Byte.html#toUnsignedLong%2Dbyte%2D[toUnsignedLong](byte)++ (link:{java9-javadoc}/java/lang/Byte.html#toUnsignedLong%2Dbyte%2D[java 9])
+* ++[[painless-api-reference-Byte-valueOf-1]]static <<painless-api-reference-Byte,Byte>> link:{java8-javadoc}/java/lang/Byte.html#valueOf%2Dbyte%2D[valueOf](byte)++ (link:{java9-javadoc}/java/lang/Byte.html#valueOf%2Dbyte%2D[java 9])
+* ++[[painless-api-reference-Byte-valueOf-2]]static <<painless-api-reference-Byte,Byte>> link:{java8-javadoc}/java/lang/Byte.html#valueOf%2Djava.lang.String%2Dint%2D[valueOf](<<painless-api-reference-String,String>>, int)++ (link:{java9-javadoc}/java/lang/Byte.html#valueOf%2Djava.lang.String%2Dint%2D[java 9])
+* ++[[painless-api-reference-Byte-compareTo-1]]int link:{java8-javadoc}/java/lang/Byte.html#compareTo%2Djava.lang.Byte%2D[compareTo](<<painless-api-reference-Byte,Byte>>)++ (link:{java9-javadoc}/java/lang/Byte.html#compareTo%2Djava.lang.Byte%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Number,Number>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/BytesRef.asciidoc b/docs/painless/painless-api-reference/BytesRef.asciidoc
new file mode 100644
index 0000000000000..6114a5625fcae
--- /dev/null
+++ b/docs/painless/painless-api-reference/BytesRef.asciidoc
@@ -0,0 +1,12 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-BytesRef]]++BytesRef++::
+** [[painless-api-reference-BytesRef-bytes]]byte[] link:{lucene-core-javadoc}/org/apache/lucene/util/BytesRef.html#bytes[bytes]
+** [[painless-api-reference-BytesRef-length]]int link:{lucene-core-javadoc}/org/apache/lucene/util/BytesRef.html#length[length]
+** [[painless-api-reference-BytesRef-offset]]int link:{lucene-core-javadoc}/org/apache/lucene/util/BytesRef.html#offset[offset]
+* ++[[painless-api-reference-BytesRef-bytesEquals-1]]boolean link:{lucene-core-javadoc}/org/apache/lucene/util/BytesRef.html#bytesEquals%2Dorg.apache.lucene.util.BytesRef%2D[bytesEquals](<<painless-api-reference-BytesRef,BytesRef>>)++
+* ++[[painless-api-reference-BytesRef-utf8ToString-0]]<<painless-api-reference-String,String>> link:{lucene-core-javadoc}/org/apache/lucene/util/BytesRef.html#utf8ToString%2D%2D[utf8ToString]()++
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Calendar.Builder.asciidoc b/docs/painless/painless-api-reference/Calendar.Builder.asciidoc
new file mode 100644
index 0000000000000..2ab66c5ecac9e
--- /dev/null
+++ b/docs/painless/painless-api-reference/Calendar.Builder.asciidoc
@@ -0,0 +1,21 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Calendar-Builder]]++Calendar.Builder++::
+* ++[[painless-api-reference-Calendar-Builder-Calendar.Builder-0]]link:{java8-javadoc}/java/util/Calendar.Builder.html#Calendar.Builder%2D%2D[Calendar.Builder]()++ (link:{java9-javadoc}/java/util/Calendar.Builder.html#Calendar.Builder%2D%2D[java 9])
+* ++[[painless-api-reference-Calendar-Builder-build-0]]<<painless-api-reference-Calendar,Calendar>> link:{java8-javadoc}/java/util/Calendar.Builder.html#build%2D%2D[build]()++ (link:{java9-javadoc}/java/util/Calendar.Builder.html#build%2D%2D[java 9])
+* ++[[painless-api-reference-Calendar-Builder-set-2]]<<painless-api-reference-Calendar-Builder,Calendar.Builder>> link:{java8-javadoc}/java/util/Calendar.Builder.html#set%2Dint%2Dint%2D[set](int, int)++ (link:{java9-javadoc}/java/util/Calendar.Builder.html#set%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Calendar-Builder-setCalendarType-1]]<<painless-api-reference-Calendar-Builder,Calendar.Builder>> link:{java8-javadoc}/java/util/Calendar.Builder.html#setCalendarType%2Djava.lang.String%2D[setCalendarType](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/Calendar.Builder.html#setCalendarType%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Calendar-Builder-setDate-3]]<<painless-api-reference-Calendar-Builder,Calendar.Builder>> link:{java8-javadoc}/java/util/Calendar.Builder.html#setDate%2Dint%2Dint%2Dint%2D[setDate](int, int, int)++ (link:{java9-javadoc}/java/util/Calendar.Builder.html#setDate%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Calendar-Builder-setFields-1]]<<painless-api-reference-Calendar-Builder,Calendar.Builder>> link:{java8-javadoc}/java/util/Calendar.Builder.html#setFields%2Dint:A%2D[setFields](int[])++ (link:{java9-javadoc}/java/util/Calendar.Builder.html#setFields%2Dint:A%2D[java 9])
+* ++[[painless-api-reference-Calendar-Builder-setInstant-1]]<<painless-api-reference-Calendar-Builder,Calendar.Builder>> link:{java8-javadoc}/java/util/Calendar.Builder.html#setInstant%2Dlong%2D[setInstant](long)++ (link:{java9-javadoc}/java/util/Calendar.Builder.html#setInstant%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Calendar-Builder-setLenient-1]]<<painless-api-reference-Calendar-Builder,Calendar.Builder>> link:{java8-javadoc}/java/util/Calendar.Builder.html#setLenient%2Dboolean%2D[setLenient](boolean)++ (link:{java9-javadoc}/java/util/Calendar.Builder.html#setLenient%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-Calendar-Builder-setLocale-1]]<<painless-api-reference-Calendar-Builder,Calendar.Builder>> link:{java8-javadoc}/java/util/Calendar.Builder.html#setLocale%2Djava.util.Locale%2D[setLocale](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/util/Calendar.Builder.html#setLocale%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-Calendar-Builder-setTimeOfDay-3]]<<painless-api-reference-Calendar-Builder,Calendar.Builder>> link:{java8-javadoc}/java/util/Calendar.Builder.html#setTimeOfDay%2Dint%2Dint%2Dint%2D[setTimeOfDay](int, int, int)++ (link:{java9-javadoc}/java/util/Calendar.Builder.html#setTimeOfDay%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Calendar-Builder-setTimeOfDay-4]]<<painless-api-reference-Calendar-Builder,Calendar.Builder>> link:{java8-javadoc}/java/util/Calendar.Builder.html#setTimeOfDay%2Dint%2Dint%2Dint%2Dint%2D[setTimeOfDay](int, int, int, int)++ (link:{java9-javadoc}/java/util/Calendar.Builder.html#setTimeOfDay%2Dint%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Calendar-Builder-setTimeZone-1]]<<painless-api-reference-Calendar-Builder,Calendar.Builder>> link:{java8-javadoc}/java/util/Calendar.Builder.html#setTimeZone%2Djava.util.TimeZone%2D[setTimeZone](<<painless-api-reference-TimeZone,TimeZone>>)++ (link:{java9-javadoc}/java/util/Calendar.Builder.html#setTimeZone%2Djava.util.TimeZone%2D[java 9])
+* ++[[painless-api-reference-Calendar-Builder-setWeekDate-3]]<<painless-api-reference-Calendar-Builder,Calendar.Builder>> link:{java8-javadoc}/java/util/Calendar.Builder.html#setWeekDate%2Dint%2Dint%2Dint%2D[setWeekDate](int, int, int)++ (link:{java9-javadoc}/java/util/Calendar.Builder.html#setWeekDate%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Calendar-Builder-setWeekDefinition-2]]<<painless-api-reference-Calendar-Builder,Calendar.Builder>> link:{java8-javadoc}/java/util/Calendar.Builder.html#setWeekDefinition%2Dint%2Dint%2D[setWeekDefinition](int, int)++ (link:{java9-javadoc}/java/util/Calendar.Builder.html#setWeekDefinition%2Dint%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Calendar.asciidoc b/docs/painless/painless-api-reference/Calendar.asciidoc
new file mode 100644
index 0000000000000..e1a489d8c87d1
--- /dev/null
+++ b/docs/painless/painless-api-reference/Calendar.asciidoc
@@ -0,0 +1,102 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Calendar]]++Calendar++::
+** [[painless-api-reference-Calendar-ALL_STYLES]]static int link:{java8-javadoc}/java/util/Calendar.html#ALL_STYLES[ALL_STYLES] (link:{java9-javadoc}/java/util/Calendar.html#ALL_STYLES[java 9])
+** [[painless-api-reference-Calendar-AM]]static int link:{java8-javadoc}/java/util/Calendar.html#AM[AM] (link:{java9-javadoc}/java/util/Calendar.html#AM[java 9])
+** [[painless-api-reference-Calendar-AM_PM]]static int link:{java8-javadoc}/java/util/Calendar.html#AM_PM[AM_PM] (link:{java9-javadoc}/java/util/Calendar.html#AM_PM[java 9])
+** [[painless-api-reference-Calendar-APRIL]]static int link:{java8-javadoc}/java/util/Calendar.html#APRIL[APRIL] (link:{java9-javadoc}/java/util/Calendar.html#APRIL[java 9])
+** [[painless-api-reference-Calendar-AUGUST]]static int link:{java8-javadoc}/java/util/Calendar.html#AUGUST[AUGUST] (link:{java9-javadoc}/java/util/Calendar.html#AUGUST[java 9])
+** [[painless-api-reference-Calendar-DATE]]static int link:{java8-javadoc}/java/util/Calendar.html#DATE[DATE] (link:{java9-javadoc}/java/util/Calendar.html#DATE[java 9])
+** [[painless-api-reference-Calendar-DAY_OF_MONTH]]static int link:{java8-javadoc}/java/util/Calendar.html#DAY_OF_MONTH[DAY_OF_MONTH] (link:{java9-javadoc}/java/util/Calendar.html#DAY_OF_MONTH[java 9])
+** [[painless-api-reference-Calendar-DAY_OF_WEEK]]static int link:{java8-javadoc}/java/util/Calendar.html#DAY_OF_WEEK[DAY_OF_WEEK] (link:{java9-javadoc}/java/util/Calendar.html#DAY_OF_WEEK[java 9])
+** [[painless-api-reference-Calendar-DAY_OF_WEEK_IN_MONTH]]static int link:{java8-javadoc}/java/util/Calendar.html#DAY_OF_WEEK_IN_MONTH[DAY_OF_WEEK_IN_MONTH] (link:{java9-javadoc}/java/util/Calendar.html#DAY_OF_WEEK_IN_MONTH[java 9])
+** [[painless-api-reference-Calendar-DAY_OF_YEAR]]static int link:{java8-javadoc}/java/util/Calendar.html#DAY_OF_YEAR[DAY_OF_YEAR] (link:{java9-javadoc}/java/util/Calendar.html#DAY_OF_YEAR[java 9])
+** [[painless-api-reference-Calendar-DECEMBER]]static int link:{java8-javadoc}/java/util/Calendar.html#DECEMBER[DECEMBER] (link:{java9-javadoc}/java/util/Calendar.html#DECEMBER[java 9])
+** [[painless-api-reference-Calendar-DST_OFFSET]]static int link:{java8-javadoc}/java/util/Calendar.html#DST_OFFSET[DST_OFFSET] (link:{java9-javadoc}/java/util/Calendar.html#DST_OFFSET[java 9])
+** [[painless-api-reference-Calendar-ERA]]static int link:{java8-javadoc}/java/util/Calendar.html#ERA[ERA] (link:{java9-javadoc}/java/util/Calendar.html#ERA[java 9])
+** [[painless-api-reference-Calendar-FEBRUARY]]static int link:{java8-javadoc}/java/util/Calendar.html#FEBRUARY[FEBRUARY] (link:{java9-javadoc}/java/util/Calendar.html#FEBRUARY[java 9])
+** [[painless-api-reference-Calendar-FIELD_COUNT]]static int link:{java8-javadoc}/java/util/Calendar.html#FIELD_COUNT[FIELD_COUNT] (link:{java9-javadoc}/java/util/Calendar.html#FIELD_COUNT[java 9])
+** [[painless-api-reference-Calendar-FRIDAY]]static int link:{java8-javadoc}/java/util/Calendar.html#FRIDAY[FRIDAY] (link:{java9-javadoc}/java/util/Calendar.html#FRIDAY[java 9])
+** [[painless-api-reference-Calendar-HOUR]]static int link:{java8-javadoc}/java/util/Calendar.html#HOUR[HOUR] (link:{java9-javadoc}/java/util/Calendar.html#HOUR[java 9])
+** [[painless-api-reference-Calendar-HOUR_OF_DAY]]static int link:{java8-javadoc}/java/util/Calendar.html#HOUR_OF_DAY[HOUR_OF_DAY] (link:{java9-javadoc}/java/util/Calendar.html#HOUR_OF_DAY[java 9])
+** [[painless-api-reference-Calendar-JANUARY]]static int link:{java8-javadoc}/java/util/Calendar.html#JANUARY[JANUARY] (link:{java9-javadoc}/java/util/Calendar.html#JANUARY[java 9])
+** [[painless-api-reference-Calendar-JULY]]static int link:{java8-javadoc}/java/util/Calendar.html#JULY[JULY] (link:{java9-javadoc}/java/util/Calendar.html#JULY[java 9])
+** [[painless-api-reference-Calendar-JUNE]]static int link:{java8-javadoc}/java/util/Calendar.html#JUNE[JUNE] (link:{java9-javadoc}/java/util/Calendar.html#JUNE[java 9])
+** [[painless-api-reference-Calendar-LONG]]static int link:{java8-javadoc}/java/util/Calendar.html#LONG[LONG] (link:{java9-javadoc}/java/util/Calendar.html#LONG[java 9])
+** [[painless-api-reference-Calendar-LONG_FORMAT]]static int link:{java8-javadoc}/java/util/Calendar.html#LONG_FORMAT[LONG_FORMAT] (link:{java9-javadoc}/java/util/Calendar.html#LONG_FORMAT[java 9])
+** [[painless-api-reference-Calendar-LONG_STANDALONE]]static int link:{java8-javadoc}/java/util/Calendar.html#LONG_STANDALONE[LONG_STANDALONE] (link:{java9-javadoc}/java/util/Calendar.html#LONG_STANDALONE[java 9])
+** [[painless-api-reference-Calendar-MARCH]]static int link:{java8-javadoc}/java/util/Calendar.html#MARCH[MARCH] (link:{java9-javadoc}/java/util/Calendar.html#MARCH[java 9])
+** [[painless-api-reference-Calendar-MAY]]static int link:{java8-javadoc}/java/util/Calendar.html#MAY[MAY] (link:{java9-javadoc}/java/util/Calendar.html#MAY[java 9])
+** [[painless-api-reference-Calendar-MILLISECOND]]static int link:{java8-javadoc}/java/util/Calendar.html#MILLISECOND[MILLISECOND] (link:{java9-javadoc}/java/util/Calendar.html#MILLISECOND[java 9])
+** [[painless-api-reference-Calendar-MINUTE]]static int link:{java8-javadoc}/java/util/Calendar.html#MINUTE[MINUTE] (link:{java9-javadoc}/java/util/Calendar.html#MINUTE[java 9])
+** [[painless-api-reference-Calendar-MONDAY]]static int link:{java8-javadoc}/java/util/Calendar.html#MONDAY[MONDAY] (link:{java9-javadoc}/java/util/Calendar.html#MONDAY[java 9])
+** [[painless-api-reference-Calendar-MONTH]]static int link:{java8-javadoc}/java/util/Calendar.html#MONTH[MONTH] (link:{java9-javadoc}/java/util/Calendar.html#MONTH[java 9])
+** [[painless-api-reference-Calendar-NARROW_FORMAT]]static int link:{java8-javadoc}/java/util/Calendar.html#NARROW_FORMAT[NARROW_FORMAT] (link:{java9-javadoc}/java/util/Calendar.html#NARROW_FORMAT[java 9])
+** [[painless-api-reference-Calendar-NARROW_STANDALONE]]static int link:{java8-javadoc}/java/util/Calendar.html#NARROW_STANDALONE[NARROW_STANDALONE] (link:{java9-javadoc}/java/util/Calendar.html#NARROW_STANDALONE[java 9])
+** [[painless-api-reference-Calendar-NOVEMBER]]static int link:{java8-javadoc}/java/util/Calendar.html#NOVEMBER[NOVEMBER] (link:{java9-javadoc}/java/util/Calendar.html#NOVEMBER[java 9])
+** [[painless-api-reference-Calendar-OCTOBER]]static int link:{java8-javadoc}/java/util/Calendar.html#OCTOBER[OCTOBER] (link:{java9-javadoc}/java/util/Calendar.html#OCTOBER[java 9])
+** [[painless-api-reference-Calendar-PM]]static int link:{java8-javadoc}/java/util/Calendar.html#PM[PM] (link:{java9-javadoc}/java/util/Calendar.html#PM[java 9])
+** [[painless-api-reference-Calendar-SATURDAY]]static int link:{java8-javadoc}/java/util/Calendar.html#SATURDAY[SATURDAY] (link:{java9-javadoc}/java/util/Calendar.html#SATURDAY[java 9])
+** [[painless-api-reference-Calendar-SECOND]]static int link:{java8-javadoc}/java/util/Calendar.html#SECOND[SECOND] (link:{java9-javadoc}/java/util/Calendar.html#SECOND[java 9])
+** [[painless-api-reference-Calendar-SEPTEMBER]]static int link:{java8-javadoc}/java/util/Calendar.html#SEPTEMBER[SEPTEMBER] (link:{java9-javadoc}/java/util/Calendar.html#SEPTEMBER[java 9])
+** [[painless-api-reference-Calendar-SHORT]]static int link:{java8-javadoc}/java/util/Calendar.html#SHORT[SHORT] (link:{java9-javadoc}/java/util/Calendar.html#SHORT[java 9])
+** [[painless-api-reference-Calendar-SHORT_FORMAT]]static int link:{java8-javadoc}/java/util/Calendar.html#SHORT_FORMAT[SHORT_FORMAT] (link:{java9-javadoc}/java/util/Calendar.html#SHORT_FORMAT[java 9])
+** [[painless-api-reference-Calendar-SHORT_STANDALONE]]static int link:{java8-javadoc}/java/util/Calendar.html#SHORT_STANDALONE[SHORT_STANDALONE] (link:{java9-javadoc}/java/util/Calendar.html#SHORT_STANDALONE[java 9])
+** [[painless-api-reference-Calendar-SUNDAY]]static int link:{java8-javadoc}/java/util/Calendar.html#SUNDAY[SUNDAY] (link:{java9-javadoc}/java/util/Calendar.html#SUNDAY[java 9])
+** [[painless-api-reference-Calendar-THURSDAY]]static int link:{java8-javadoc}/java/util/Calendar.html#THURSDAY[THURSDAY] (link:{java9-javadoc}/java/util/Calendar.html#THURSDAY[java 9])
+** [[painless-api-reference-Calendar-TUESDAY]]static int link:{java8-javadoc}/java/util/Calendar.html#TUESDAY[TUESDAY] (link:{java9-javadoc}/java/util/Calendar.html#TUESDAY[java 9])
+** [[painless-api-reference-Calendar-UNDECIMBER]]static int link:{java8-javadoc}/java/util/Calendar.html#UNDECIMBER[UNDECIMBER] (link:{java9-javadoc}/java/util/Calendar.html#UNDECIMBER[java 9])
+** [[painless-api-reference-Calendar-WEDNESDAY]]static int link:{java8-javadoc}/java/util/Calendar.html#WEDNESDAY[WEDNESDAY] (link:{java9-javadoc}/java/util/Calendar.html#WEDNESDAY[java 9])
+** [[painless-api-reference-Calendar-WEEK_OF_MONTH]]static int link:{java8-javadoc}/java/util/Calendar.html#WEEK_OF_MONTH[WEEK_OF_MONTH] (link:{java9-javadoc}/java/util/Calendar.html#WEEK_OF_MONTH[java 9])
+** [[painless-api-reference-Calendar-WEEK_OF_YEAR]]static int link:{java8-javadoc}/java/util/Calendar.html#WEEK_OF_YEAR[WEEK_OF_YEAR] (link:{java9-javadoc}/java/util/Calendar.html#WEEK_OF_YEAR[java 9])
+** [[painless-api-reference-Calendar-YEAR]]static int link:{java8-javadoc}/java/util/Calendar.html#YEAR[YEAR] (link:{java9-javadoc}/java/util/Calendar.html#YEAR[java 9])
+** [[painless-api-reference-Calendar-ZONE_OFFSET]]static int link:{java8-javadoc}/java/util/Calendar.html#ZONE_OFFSET[ZONE_OFFSET] (link:{java9-javadoc}/java/util/Calendar.html#ZONE_OFFSET[java 9])
+* ++[[painless-api-reference-Calendar-getAvailableCalendarTypes-0]]static <<painless-api-reference-Set,Set>> link:{java8-javadoc}/java/util/Calendar.html#getAvailableCalendarTypes%2D%2D[getAvailableCalendarTypes]()++ (link:{java9-javadoc}/java/util/Calendar.html#getAvailableCalendarTypes%2D%2D[java 9])
+* ++[[painless-api-reference-Calendar-getAvailableLocales-0]]static <<painless-api-reference-Locale,Locale>>[] link:{java8-javadoc}/java/util/Calendar.html#getAvailableLocales%2D%2D[getAvailableLocales]()++ (link:{java9-javadoc}/java/util/Calendar.html#getAvailableLocales%2D%2D[java 9])
+* ++[[painless-api-reference-Calendar-getInstance-0]]static <<painless-api-reference-Calendar,Calendar>> link:{java8-javadoc}/java/util/Calendar.html#getInstance%2D%2D[getInstance]()++ (link:{java9-javadoc}/java/util/Calendar.html#getInstance%2D%2D[java 9])
+* ++[[painless-api-reference-Calendar-getInstance-1]]static <<painless-api-reference-Calendar,Calendar>> link:{java8-javadoc}/java/util/Calendar.html#getInstance%2Djava.util.TimeZone%2D[getInstance](<<painless-api-reference-TimeZone,TimeZone>>)++ (link:{java9-javadoc}/java/util/Calendar.html#getInstance%2Djava.util.TimeZone%2D[java 9])
+* ++[[painless-api-reference-Calendar-getInstance-2]]static <<painless-api-reference-Calendar,Calendar>> link:{java8-javadoc}/java/util/Calendar.html#getInstance%2Djava.util.TimeZone%2Djava.util.Locale%2D[getInstance](<<painless-api-reference-TimeZone,TimeZone>>, <<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/util/Calendar.html#getInstance%2Djava.util.TimeZone%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-Calendar-add-2]]void link:{java8-javadoc}/java/util/Calendar.html#add%2Dint%2Dint%2D[add](int, int)++ (link:{java9-javadoc}/java/util/Calendar.html#add%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Calendar-after-1]]boolean link:{java8-javadoc}/java/util/Calendar.html#after%2Djava.lang.Object%2D[after](<<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/util/Calendar.html#after%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Calendar-before-1]]boolean link:{java8-javadoc}/java/util/Calendar.html#before%2Djava.lang.Object%2D[before](<<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/util/Calendar.html#before%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Calendar-clear-0]]void link:{java8-javadoc}/java/util/Calendar.html#clear%2D%2D[clear]()++ (link:{java9-javadoc}/java/util/Calendar.html#clear%2D%2D[java 9])
+* ++[[painless-api-reference-Calendar-clear-1]]void link:{java8-javadoc}/java/util/Calendar.html#clear%2Dint%2D[clear](int)++ (link:{java9-javadoc}/java/util/Calendar.html#clear%2Dint%2D[java 9])
+* ++[[painless-api-reference-Calendar-clone-0]]def link:{java8-javadoc}/java/util/Calendar.html#clone%2D%2D[clone]()++ (link:{java9-javadoc}/java/util/Calendar.html#clone%2D%2D[java 9])
+* ++[[painless-api-reference-Calendar-compareTo-1]]int link:{java8-javadoc}/java/util/Calendar.html#compareTo%2Djava.util.Calendar%2D[compareTo](<<painless-api-reference-Calendar,Calendar>>)++ (link:{java9-javadoc}/java/util/Calendar.html#compareTo%2Djava.util.Calendar%2D[java 9])
+* ++[[painless-api-reference-Calendar-get-1]]int link:{java8-javadoc}/java/util/Calendar.html#get%2Dint%2D[get](int)++ (link:{java9-javadoc}/java/util/Calendar.html#get%2Dint%2D[java 9])
+* ++[[painless-api-reference-Calendar-getActualMaximum-1]]int link:{java8-javadoc}/java/util/Calendar.html#getActualMaximum%2Dint%2D[getActualMaximum](int)++ (link:{java9-javadoc}/java/util/Calendar.html#getActualMaximum%2Dint%2D[java 9])
+* ++[[painless-api-reference-Calendar-getActualMinimum-1]]int link:{java8-javadoc}/java/util/Calendar.html#getActualMinimum%2Dint%2D[getActualMinimum](int)++ (link:{java9-javadoc}/java/util/Calendar.html#getActualMinimum%2Dint%2D[java 9])
+* ++[[painless-api-reference-Calendar-getCalendarType-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Calendar.html#getCalendarType%2D%2D[getCalendarType]()++ (link:{java9-javadoc}/java/util/Calendar.html#getCalendarType%2D%2D[java 9])
+* ++[[painless-api-reference-Calendar-getDisplayName-3]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Calendar.html#getDisplayName%2Dint%2Dint%2Djava.util.Locale%2D[getDisplayName](int, int, <<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/util/Calendar.html#getDisplayName%2Dint%2Dint%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-Calendar-getDisplayNames-3]]<<painless-api-reference-Map,Map>> link:{java8-javadoc}/java/util/Calendar.html#getDisplayNames%2Dint%2Dint%2Djava.util.Locale%2D[getDisplayNames](int, int, <<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/util/Calendar.html#getDisplayNames%2Dint%2Dint%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-Calendar-getFirstDayOfWeek-0]]int link:{java8-javadoc}/java/util/Calendar.html#getFirstDayOfWeek%2D%2D[getFirstDayOfWeek]()++ (link:{java9-javadoc}/java/util/Calendar.html#getFirstDayOfWeek%2D%2D[java 9])
+* ++[[painless-api-reference-Calendar-getGreatestMinimum-1]]int link:{java8-javadoc}/java/util/Calendar.html#getGreatestMinimum%2Dint%2D[getGreatestMinimum](int)++ (link:{java9-javadoc}/java/util/Calendar.html#getGreatestMinimum%2Dint%2D[java 9])
+* ++[[painless-api-reference-Calendar-getLeastMaximum-1]]int link:{java8-javadoc}/java/util/Calendar.html#getLeastMaximum%2Dint%2D[getLeastMaximum](int)++ (link:{java9-javadoc}/java/util/Calendar.html#getLeastMaximum%2Dint%2D[java 9])
+* ++[[painless-api-reference-Calendar-getMaximum-1]]int link:{java8-javadoc}/java/util/Calendar.html#getMaximum%2Dint%2D[getMaximum](int)++ (link:{java9-javadoc}/java/util/Calendar.html#getMaximum%2Dint%2D[java 9])
+* ++[[painless-api-reference-Calendar-getMinimalDaysInFirstWeek-0]]int link:{java8-javadoc}/java/util/Calendar.html#getMinimalDaysInFirstWeek%2D%2D[getMinimalDaysInFirstWeek]()++ (link:{java9-javadoc}/java/util/Calendar.html#getMinimalDaysInFirstWeek%2D%2D[java 9])
+* ++[[painless-api-reference-Calendar-getMinimum-1]]int link:{java8-javadoc}/java/util/Calendar.html#getMinimum%2Dint%2D[getMinimum](int)++ (link:{java9-javadoc}/java/util/Calendar.html#getMinimum%2Dint%2D[java 9])
+* ++[[painless-api-reference-Calendar-getTime-0]]<<painless-api-reference-Date,Date>> link:{java8-javadoc}/java/util/Calendar.html#getTime%2D%2D[getTime]()++ (link:{java9-javadoc}/java/util/Calendar.html#getTime%2D%2D[java 9])
+* ++[[painless-api-reference-Calendar-getTimeInMillis-0]]long link:{java8-javadoc}/java/util/Calendar.html#getTimeInMillis%2D%2D[getTimeInMillis]()++ (link:{java9-javadoc}/java/util/Calendar.html#getTimeInMillis%2D%2D[java 9])
+* ++[[painless-api-reference-Calendar-getTimeZone-0]]<<painless-api-reference-TimeZone,TimeZone>> link:{java8-javadoc}/java/util/Calendar.html#getTimeZone%2D%2D[getTimeZone]()++ (link:{java9-javadoc}/java/util/Calendar.html#getTimeZone%2D%2D[java 9])
+* ++[[painless-api-reference-Calendar-getWeekYear-0]]int link:{java8-javadoc}/java/util/Calendar.html#getWeekYear%2D%2D[getWeekYear]()++ (link:{java9-javadoc}/java/util/Calendar.html#getWeekYear%2D%2D[java 9])
+* ++[[painless-api-reference-Calendar-getWeeksInWeekYear-0]]int link:{java8-javadoc}/java/util/Calendar.html#getWeeksInWeekYear%2D%2D[getWeeksInWeekYear]()++ (link:{java9-javadoc}/java/util/Calendar.html#getWeeksInWeekYear%2D%2D[java 9])
+* ++[[painless-api-reference-Calendar-isLenient-0]]boolean link:{java8-javadoc}/java/util/Calendar.html#isLenient%2D%2D[isLenient]()++ (link:{java9-javadoc}/java/util/Calendar.html#isLenient%2D%2D[java 9])
+* ++[[painless-api-reference-Calendar-isSet-1]]boolean link:{java8-javadoc}/java/util/Calendar.html#isSet%2Dint%2D[isSet](int)++ (link:{java9-javadoc}/java/util/Calendar.html#isSet%2Dint%2D[java 9])
+* ++[[painless-api-reference-Calendar-isWeekDateSupported-0]]boolean link:{java8-javadoc}/java/util/Calendar.html#isWeekDateSupported%2D%2D[isWeekDateSupported]()++ (link:{java9-javadoc}/java/util/Calendar.html#isWeekDateSupported%2D%2D[java 9])
+* ++[[painless-api-reference-Calendar-roll-2]]void link:{java8-javadoc}/java/util/Calendar.html#roll%2Dint%2Dint%2D[roll](int, int)++ (link:{java9-javadoc}/java/util/Calendar.html#roll%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Calendar-set-2]]void link:{java8-javadoc}/java/util/Calendar.html#set%2Dint%2Dint%2D[set](int, int)++ (link:{java9-javadoc}/java/util/Calendar.html#set%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Calendar-set-3]]void link:{java8-javadoc}/java/util/Calendar.html#set%2Dint%2Dint%2Dint%2D[set](int, int, int)++ (link:{java9-javadoc}/java/util/Calendar.html#set%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Calendar-set-5]]void link:{java8-javadoc}/java/util/Calendar.html#set%2Dint%2Dint%2Dint%2Dint%2Dint%2D[set](int, int, int, int, int)++ (link:{java9-javadoc}/java/util/Calendar.html#set%2Dint%2Dint%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Calendar-set-6]]void link:{java8-javadoc}/java/util/Calendar.html#set%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2D[set](int, int, int, int, int, int)++ (link:{java9-javadoc}/java/util/Calendar.html#set%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Calendar-setFirstDayOfWeek-1]]void link:{java8-javadoc}/java/util/Calendar.html#setFirstDayOfWeek%2Dint%2D[setFirstDayOfWeek](int)++ (link:{java9-javadoc}/java/util/Calendar.html#setFirstDayOfWeek%2Dint%2D[java 9])
+* ++[[painless-api-reference-Calendar-setLenient-1]]void link:{java8-javadoc}/java/util/Calendar.html#setLenient%2Dboolean%2D[setLenient](boolean)++ (link:{java9-javadoc}/java/util/Calendar.html#setLenient%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-Calendar-setMinimalDaysInFirstWeek-1]]void link:{java8-javadoc}/java/util/Calendar.html#setMinimalDaysInFirstWeek%2Dint%2D[setMinimalDaysInFirstWeek](int)++ (link:{java9-javadoc}/java/util/Calendar.html#setMinimalDaysInFirstWeek%2Dint%2D[java 9])
+* ++[[painless-api-reference-Calendar-setTime-1]]void link:{java8-javadoc}/java/util/Calendar.html#setTime%2Djava.util.Date%2D[setTime](<<painless-api-reference-Date,Date>>)++ (link:{java9-javadoc}/java/util/Calendar.html#setTime%2Djava.util.Date%2D[java 9])
+* ++[[painless-api-reference-Calendar-setTimeInMillis-1]]void link:{java8-javadoc}/java/util/Calendar.html#setTimeInMillis%2Dlong%2D[setTimeInMillis](long)++ (link:{java9-javadoc}/java/util/Calendar.html#setTimeInMillis%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Calendar-setTimeZone-1]]void link:{java8-javadoc}/java/util/Calendar.html#setTimeZone%2Djava.util.TimeZone%2D[setTimeZone](<<painless-api-reference-TimeZone,TimeZone>>)++ (link:{java9-javadoc}/java/util/Calendar.html#setTimeZone%2Djava.util.TimeZone%2D[java 9])
+* ++[[painless-api-reference-Calendar-setWeekDate-3]]void link:{java8-javadoc}/java/util/Calendar.html#setWeekDate%2Dint%2Dint%2Dint%2D[setWeekDate](int, int, int)++ (link:{java9-javadoc}/java/util/Calendar.html#setWeekDate%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Calendar-toInstant-0]]<<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/util/Calendar.html#toInstant%2D%2D[toInstant]()++ (link:{java9-javadoc}/java/util/Calendar.html#toInstant%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/CharSequence.asciidoc b/docs/painless/painless-api-reference/CharSequence.asciidoc
new file mode 100644
index 0000000000000..2618a22881e02
--- /dev/null
+++ b/docs/painless/painless-api-reference/CharSequence.asciidoc
@@ -0,0 +1,15 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-CharSequence]]++CharSequence++::
+* ++[[painless-api-reference-CharSequence-charAt-1]]char link:{java8-javadoc}/java/lang/CharSequence.html#charAt%2Dint%2D[charAt](int)++ (link:{java9-javadoc}/java/lang/CharSequence.html#charAt%2Dint%2D[java 9])
+* ++[[painless-api-reference-CharSequence-chars-0]]<<painless-api-reference-IntStream,IntStream>> link:{java8-javadoc}/java/lang/CharSequence.html#chars%2D%2D[chars]()++ (link:{java9-javadoc}/java/lang/CharSequence.html#chars%2D%2D[java 9])
+* ++[[painless-api-reference-CharSequence-codePoints-0]]<<painless-api-reference-IntStream,IntStream>> link:{java8-javadoc}/java/lang/CharSequence.html#codePoints%2D%2D[codePoints]()++ (link:{java9-javadoc}/java/lang/CharSequence.html#codePoints%2D%2D[java 9])
+* ++[[painless-api-reference-CharSequence-length-0]]int link:{java8-javadoc}/java/lang/CharSequence.html#length%2D%2D[length]()++ (link:{java9-javadoc}/java/lang/CharSequence.html#length%2D%2D[java 9])
+* ++[[painless-api-reference-CharSequence-replaceAll-2]]<<painless-api-reference-String,String>> link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#replaceAll%2Djava.lang.CharSequence%2Djava.util.regex.Pattern%2Djava.util.function.Function%2D[replaceAll](<<painless-api-reference-Pattern,Pattern>>, <<painless-api-reference-Function,Function>>)++
+* ++[[painless-api-reference-CharSequence-replaceFirst-2]]<<painless-api-reference-String,String>> link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#replaceFirst%2Djava.lang.CharSequence%2Djava.util.regex.Pattern%2Djava.util.function.Function%2D[replaceFirst](<<painless-api-reference-Pattern,Pattern>>, <<painless-api-reference-Function,Function>>)++
+* ++[[painless-api-reference-CharSequence-subSequence-2]]<<painless-api-reference-CharSequence,CharSequence>> link:{java8-javadoc}/java/lang/CharSequence.html#subSequence%2Dint%2Dint%2D[subSequence](int, int)++ (link:{java9-javadoc}/java/lang/CharSequence.html#subSequence%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-CharSequence-toString-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/CharSequence.html#toString%2D%2D[toString]()++ (link:{java9-javadoc}/java/lang/CharSequence.html#toString%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Character.Subset.asciidoc b/docs/painless/painless-api-reference/Character.Subset.asciidoc
new file mode 100644
index 0000000000000..e8dff322e6e27
--- /dev/null
+++ b/docs/painless/painless-api-reference/Character.Subset.asciidoc
@@ -0,0 +1,7 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Character-Subset]]++Character.Subset++::
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Character.UnicodeBlock.asciidoc b/docs/painless/painless-api-reference/Character.UnicodeBlock.asciidoc
new file mode 100644
index 0000000000000..6fdf81adfc922
--- /dev/null
+++ b/docs/painless/painless-api-reference/Character.UnicodeBlock.asciidoc
@@ -0,0 +1,229 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Character-UnicodeBlock]]++Character.UnicodeBlock++::
+** [[painless-api-reference-Character-UnicodeBlock-AEGEAN_NUMBERS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#AEGEAN_NUMBERS[AEGEAN_NUMBERS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#AEGEAN_NUMBERS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-ALCHEMICAL_SYMBOLS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#ALCHEMICAL_SYMBOLS[ALCHEMICAL_SYMBOLS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#ALCHEMICAL_SYMBOLS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-ALPHABETIC_PRESENTATION_FORMS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#ALPHABETIC_PRESENTATION_FORMS[ALPHABETIC_PRESENTATION_FORMS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#ALPHABETIC_PRESENTATION_FORMS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-ANCIENT_GREEK_MUSICAL_NOTATION]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#ANCIENT_GREEK_MUSICAL_NOTATION[ANCIENT_GREEK_MUSICAL_NOTATION] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#ANCIENT_GREEK_MUSICAL_NOTATION[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-ANCIENT_GREEK_NUMBERS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#ANCIENT_GREEK_NUMBERS[ANCIENT_GREEK_NUMBERS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#ANCIENT_GREEK_NUMBERS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-ANCIENT_SYMBOLS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#ANCIENT_SYMBOLS[ANCIENT_SYMBOLS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#ANCIENT_SYMBOLS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-ARABIC]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#ARABIC[ARABIC] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#ARABIC[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-ARABIC_EXTENDED_A]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#ARABIC_EXTENDED_A[ARABIC_EXTENDED_A] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#ARABIC_EXTENDED_A[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-ARABIC_MATHEMATICAL_ALPHABETIC_SYMBOLS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#ARABIC_MATHEMATICAL_ALPHABETIC_SYMBOLS[ARABIC_MATHEMATICAL_ALPHABETIC_SYMBOLS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#ARABIC_MATHEMATICAL_ALPHABETIC_SYMBOLS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-ARABIC_PRESENTATION_FORMS_A]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#ARABIC_PRESENTATION_FORMS_A[ARABIC_PRESENTATION_FORMS_A] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#ARABIC_PRESENTATION_FORMS_A[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-ARABIC_PRESENTATION_FORMS_B]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#ARABIC_PRESENTATION_FORMS_B[ARABIC_PRESENTATION_FORMS_B] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#ARABIC_PRESENTATION_FORMS_B[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-ARABIC_SUPPLEMENT]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#ARABIC_SUPPLEMENT[ARABIC_SUPPLEMENT] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#ARABIC_SUPPLEMENT[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-ARMENIAN]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#ARMENIAN[ARMENIAN] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#ARMENIAN[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-ARROWS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#ARROWS[ARROWS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#ARROWS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-AVESTAN]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#AVESTAN[AVESTAN] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#AVESTAN[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-BALINESE]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#BALINESE[BALINESE] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#BALINESE[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-BAMUM]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#BAMUM[BAMUM] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#BAMUM[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-BAMUM_SUPPLEMENT]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#BAMUM_SUPPLEMENT[BAMUM_SUPPLEMENT] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#BAMUM_SUPPLEMENT[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-BASIC_LATIN]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#BASIC_LATIN[BASIC_LATIN] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#BASIC_LATIN[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-BATAK]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#BATAK[BATAK] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#BATAK[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-BENGALI]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#BENGALI[BENGALI] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#BENGALI[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-BLOCK_ELEMENTS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#BLOCK_ELEMENTS[BLOCK_ELEMENTS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#BLOCK_ELEMENTS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-BOPOMOFO]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#BOPOMOFO[BOPOMOFO] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#BOPOMOFO[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-BOPOMOFO_EXTENDED]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#BOPOMOFO_EXTENDED[BOPOMOFO_EXTENDED] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#BOPOMOFO_EXTENDED[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-BOX_DRAWING]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#BOX_DRAWING[BOX_DRAWING] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#BOX_DRAWING[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-BRAHMI]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#BRAHMI[BRAHMI] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#BRAHMI[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-BRAILLE_PATTERNS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#BRAILLE_PATTERNS[BRAILLE_PATTERNS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#BRAILLE_PATTERNS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-BUGINESE]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#BUGINESE[BUGINESE] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#BUGINESE[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-BUHID]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#BUHID[BUHID] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#BUHID[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-BYZANTINE_MUSICAL_SYMBOLS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#BYZANTINE_MUSICAL_SYMBOLS[BYZANTINE_MUSICAL_SYMBOLS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#BYZANTINE_MUSICAL_SYMBOLS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CARIAN]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#CARIAN[CARIAN] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#CARIAN[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CHAKMA]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#CHAKMA[CHAKMA] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#CHAKMA[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CHAM]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#CHAM[CHAM] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#CHAM[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CHEROKEE]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#CHEROKEE[CHEROKEE] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#CHEROKEE[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CJK_COMPATIBILITY]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#CJK_COMPATIBILITY[CJK_COMPATIBILITY] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#CJK_COMPATIBILITY[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CJK_COMPATIBILITY_FORMS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#CJK_COMPATIBILITY_FORMS[CJK_COMPATIBILITY_FORMS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#CJK_COMPATIBILITY_FORMS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CJK_COMPATIBILITY_IDEOGRAPHS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#CJK_COMPATIBILITY_IDEOGRAPHS[CJK_COMPATIBILITY_IDEOGRAPHS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#CJK_COMPATIBILITY_IDEOGRAPHS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CJK_COMPATIBILITY_IDEOGRAPHS_SUPPLEMENT]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#CJK_COMPATIBILITY_IDEOGRAPHS_SUPPLEMENT[CJK_COMPATIBILITY_IDEOGRAPHS_SUPPLEMENT] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#CJK_COMPATIBILITY_IDEOGRAPHS_SUPPLEMENT[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CJK_RADICALS_SUPPLEMENT]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#CJK_RADICALS_SUPPLEMENT[CJK_RADICALS_SUPPLEMENT] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#CJK_RADICALS_SUPPLEMENT[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CJK_STROKES]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#CJK_STROKES[CJK_STROKES] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#CJK_STROKES[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CJK_SYMBOLS_AND_PUNCTUATION]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#CJK_SYMBOLS_AND_PUNCTUATION[CJK_SYMBOLS_AND_PUNCTUATION] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#CJK_SYMBOLS_AND_PUNCTUATION[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CJK_UNIFIED_IDEOGRAPHS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#CJK_UNIFIED_IDEOGRAPHS[CJK_UNIFIED_IDEOGRAPHS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#CJK_UNIFIED_IDEOGRAPHS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CJK_UNIFIED_IDEOGRAPHS_EXTENSION_A]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#CJK_UNIFIED_IDEOGRAPHS_EXTENSION_A[CJK_UNIFIED_IDEOGRAPHS_EXTENSION_A] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#CJK_UNIFIED_IDEOGRAPHS_EXTENSION_A[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CJK_UNIFIED_IDEOGRAPHS_EXTENSION_B]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#CJK_UNIFIED_IDEOGRAPHS_EXTENSION_B[CJK_UNIFIED_IDEOGRAPHS_EXTENSION_B] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#CJK_UNIFIED_IDEOGRAPHS_EXTENSION_B[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CJK_UNIFIED_IDEOGRAPHS_EXTENSION_C]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#CJK_UNIFIED_IDEOGRAPHS_EXTENSION_C[CJK_UNIFIED_IDEOGRAPHS_EXTENSION_C] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#CJK_UNIFIED_IDEOGRAPHS_EXTENSION_C[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CJK_UNIFIED_IDEOGRAPHS_EXTENSION_D]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#CJK_UNIFIED_IDEOGRAPHS_EXTENSION_D[CJK_UNIFIED_IDEOGRAPHS_EXTENSION_D] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#CJK_UNIFIED_IDEOGRAPHS_EXTENSION_D[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-COMBINING_DIACRITICAL_MARKS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#COMBINING_DIACRITICAL_MARKS[COMBINING_DIACRITICAL_MARKS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#COMBINING_DIACRITICAL_MARKS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-COMBINING_DIACRITICAL_MARKS_SUPPLEMENT]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#COMBINING_DIACRITICAL_MARKS_SUPPLEMENT[COMBINING_DIACRITICAL_MARKS_SUPPLEMENT] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#COMBINING_DIACRITICAL_MARKS_SUPPLEMENT[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-COMBINING_HALF_MARKS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#COMBINING_HALF_MARKS[COMBINING_HALF_MARKS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#COMBINING_HALF_MARKS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-COMBINING_MARKS_FOR_SYMBOLS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#COMBINING_MARKS_FOR_SYMBOLS[COMBINING_MARKS_FOR_SYMBOLS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#COMBINING_MARKS_FOR_SYMBOLS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-COMMON_INDIC_NUMBER_FORMS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#COMMON_INDIC_NUMBER_FORMS[COMMON_INDIC_NUMBER_FORMS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#COMMON_INDIC_NUMBER_FORMS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CONTROL_PICTURES]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#CONTROL_PICTURES[CONTROL_PICTURES] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#CONTROL_PICTURES[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-COPTIC]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#COPTIC[COPTIC] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#COPTIC[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-COUNTING_ROD_NUMERALS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#COUNTING_ROD_NUMERALS[COUNTING_ROD_NUMERALS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#COUNTING_ROD_NUMERALS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CUNEIFORM]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#CUNEIFORM[CUNEIFORM] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#CUNEIFORM[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CUNEIFORM_NUMBERS_AND_PUNCTUATION]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#CUNEIFORM_NUMBERS_AND_PUNCTUATION[CUNEIFORM_NUMBERS_AND_PUNCTUATION] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#CUNEIFORM_NUMBERS_AND_PUNCTUATION[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CURRENCY_SYMBOLS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#CURRENCY_SYMBOLS[CURRENCY_SYMBOLS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#CURRENCY_SYMBOLS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CYPRIOT_SYLLABARY]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#CYPRIOT_SYLLABARY[CYPRIOT_SYLLABARY] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#CYPRIOT_SYLLABARY[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CYRILLIC]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#CYRILLIC[CYRILLIC] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#CYRILLIC[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CYRILLIC_EXTENDED_A]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#CYRILLIC_EXTENDED_A[CYRILLIC_EXTENDED_A] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#CYRILLIC_EXTENDED_A[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CYRILLIC_EXTENDED_B]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#CYRILLIC_EXTENDED_B[CYRILLIC_EXTENDED_B] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#CYRILLIC_EXTENDED_B[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CYRILLIC_SUPPLEMENTARY]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#CYRILLIC_SUPPLEMENTARY[CYRILLIC_SUPPLEMENTARY] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#CYRILLIC_SUPPLEMENTARY[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-DESERET]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#DESERET[DESERET] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#DESERET[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-DEVANAGARI]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#DEVANAGARI[DEVANAGARI] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#DEVANAGARI[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-DEVANAGARI_EXTENDED]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#DEVANAGARI_EXTENDED[DEVANAGARI_EXTENDED] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#DEVANAGARI_EXTENDED[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-DINGBATS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#DINGBATS[DINGBATS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#DINGBATS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-DOMINO_TILES]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#DOMINO_TILES[DOMINO_TILES] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#DOMINO_TILES[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-EGYPTIAN_HIEROGLYPHS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#EGYPTIAN_HIEROGLYPHS[EGYPTIAN_HIEROGLYPHS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#EGYPTIAN_HIEROGLYPHS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-EMOTICONS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#EMOTICONS[EMOTICONS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#EMOTICONS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-ENCLOSED_ALPHANUMERICS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#ENCLOSED_ALPHANUMERICS[ENCLOSED_ALPHANUMERICS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#ENCLOSED_ALPHANUMERICS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-ENCLOSED_ALPHANUMERIC_SUPPLEMENT]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#ENCLOSED_ALPHANUMERIC_SUPPLEMENT[ENCLOSED_ALPHANUMERIC_SUPPLEMENT] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#ENCLOSED_ALPHANUMERIC_SUPPLEMENT[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-ENCLOSED_CJK_LETTERS_AND_MONTHS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#ENCLOSED_CJK_LETTERS_AND_MONTHS[ENCLOSED_CJK_LETTERS_AND_MONTHS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#ENCLOSED_CJK_LETTERS_AND_MONTHS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-ENCLOSED_IDEOGRAPHIC_SUPPLEMENT]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#ENCLOSED_IDEOGRAPHIC_SUPPLEMENT[ENCLOSED_IDEOGRAPHIC_SUPPLEMENT] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#ENCLOSED_IDEOGRAPHIC_SUPPLEMENT[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-ETHIOPIC]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#ETHIOPIC[ETHIOPIC] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#ETHIOPIC[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-ETHIOPIC_EXTENDED]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#ETHIOPIC_EXTENDED[ETHIOPIC_EXTENDED] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#ETHIOPIC_EXTENDED[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-ETHIOPIC_EXTENDED_A]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#ETHIOPIC_EXTENDED_A[ETHIOPIC_EXTENDED_A] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#ETHIOPIC_EXTENDED_A[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-ETHIOPIC_SUPPLEMENT]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#ETHIOPIC_SUPPLEMENT[ETHIOPIC_SUPPLEMENT] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#ETHIOPIC_SUPPLEMENT[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-GENERAL_PUNCTUATION]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#GENERAL_PUNCTUATION[GENERAL_PUNCTUATION] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#GENERAL_PUNCTUATION[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-GEOMETRIC_SHAPES]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#GEOMETRIC_SHAPES[GEOMETRIC_SHAPES] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#GEOMETRIC_SHAPES[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-GEORGIAN]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#GEORGIAN[GEORGIAN] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#GEORGIAN[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-GEORGIAN_SUPPLEMENT]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#GEORGIAN_SUPPLEMENT[GEORGIAN_SUPPLEMENT] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#GEORGIAN_SUPPLEMENT[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-GLAGOLITIC]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#GLAGOLITIC[GLAGOLITIC] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#GLAGOLITIC[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-GOTHIC]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#GOTHIC[GOTHIC] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#GOTHIC[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-GREEK]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#GREEK[GREEK] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#GREEK[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-GREEK_EXTENDED]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#GREEK_EXTENDED[GREEK_EXTENDED] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#GREEK_EXTENDED[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-GUJARATI]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#GUJARATI[GUJARATI] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#GUJARATI[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-GURMUKHI]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#GURMUKHI[GURMUKHI] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#GURMUKHI[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-HALFWIDTH_AND_FULLWIDTH_FORMS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#HALFWIDTH_AND_FULLWIDTH_FORMS[HALFWIDTH_AND_FULLWIDTH_FORMS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#HALFWIDTH_AND_FULLWIDTH_FORMS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-HANGUL_COMPATIBILITY_JAMO]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#HANGUL_COMPATIBILITY_JAMO[HANGUL_COMPATIBILITY_JAMO] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#HANGUL_COMPATIBILITY_JAMO[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-HANGUL_JAMO]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#HANGUL_JAMO[HANGUL_JAMO] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#HANGUL_JAMO[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-HANGUL_JAMO_EXTENDED_A]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#HANGUL_JAMO_EXTENDED_A[HANGUL_JAMO_EXTENDED_A] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#HANGUL_JAMO_EXTENDED_A[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-HANGUL_JAMO_EXTENDED_B]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#HANGUL_JAMO_EXTENDED_B[HANGUL_JAMO_EXTENDED_B] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#HANGUL_JAMO_EXTENDED_B[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-HANGUL_SYLLABLES]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#HANGUL_SYLLABLES[HANGUL_SYLLABLES] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#HANGUL_SYLLABLES[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-HANUNOO]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#HANUNOO[HANUNOO] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#HANUNOO[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-HEBREW]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#HEBREW[HEBREW] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#HEBREW[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-HIGH_PRIVATE_USE_SURROGATES]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#HIGH_PRIVATE_USE_SURROGATES[HIGH_PRIVATE_USE_SURROGATES] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#HIGH_PRIVATE_USE_SURROGATES[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-HIGH_SURROGATES]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#HIGH_SURROGATES[HIGH_SURROGATES] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#HIGH_SURROGATES[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-HIRAGANA]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#HIRAGANA[HIRAGANA] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#HIRAGANA[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-IDEOGRAPHIC_DESCRIPTION_CHARACTERS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#IDEOGRAPHIC_DESCRIPTION_CHARACTERS[IDEOGRAPHIC_DESCRIPTION_CHARACTERS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#IDEOGRAPHIC_DESCRIPTION_CHARACTERS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-IMPERIAL_ARAMAIC]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#IMPERIAL_ARAMAIC[IMPERIAL_ARAMAIC] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#IMPERIAL_ARAMAIC[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-INSCRIPTIONAL_PAHLAVI]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#INSCRIPTIONAL_PAHLAVI[INSCRIPTIONAL_PAHLAVI] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#INSCRIPTIONAL_PAHLAVI[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-INSCRIPTIONAL_PARTHIAN]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#INSCRIPTIONAL_PARTHIAN[INSCRIPTIONAL_PARTHIAN] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#INSCRIPTIONAL_PARTHIAN[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-IPA_EXTENSIONS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#IPA_EXTENSIONS[IPA_EXTENSIONS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#IPA_EXTENSIONS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-JAVANESE]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#JAVANESE[JAVANESE] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#JAVANESE[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-KAITHI]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#KAITHI[KAITHI] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#KAITHI[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-KANA_SUPPLEMENT]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#KANA_SUPPLEMENT[KANA_SUPPLEMENT] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#KANA_SUPPLEMENT[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-KANBUN]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#KANBUN[KANBUN] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#KANBUN[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-KANGXI_RADICALS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#KANGXI_RADICALS[KANGXI_RADICALS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#KANGXI_RADICALS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-KANNADA]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#KANNADA[KANNADA] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#KANNADA[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-KATAKANA]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#KATAKANA[KATAKANA] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#KATAKANA[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-KATAKANA_PHONETIC_EXTENSIONS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#KATAKANA_PHONETIC_EXTENSIONS[KATAKANA_PHONETIC_EXTENSIONS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#KATAKANA_PHONETIC_EXTENSIONS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-KAYAH_LI]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#KAYAH_LI[KAYAH_LI] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#KAYAH_LI[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-KHAROSHTHI]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#KHAROSHTHI[KHAROSHTHI] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#KHAROSHTHI[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-KHMER]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#KHMER[KHMER] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#KHMER[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-KHMER_SYMBOLS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#KHMER_SYMBOLS[KHMER_SYMBOLS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#KHMER_SYMBOLS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-LAO]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#LAO[LAO] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#LAO[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-LATIN_1_SUPPLEMENT]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#LATIN_1_SUPPLEMENT[LATIN_1_SUPPLEMENT] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#LATIN_1_SUPPLEMENT[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-LATIN_EXTENDED_A]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#LATIN_EXTENDED_A[LATIN_EXTENDED_A] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#LATIN_EXTENDED_A[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-LATIN_EXTENDED_ADDITIONAL]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#LATIN_EXTENDED_ADDITIONAL[LATIN_EXTENDED_ADDITIONAL] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#LATIN_EXTENDED_ADDITIONAL[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-LATIN_EXTENDED_B]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#LATIN_EXTENDED_B[LATIN_EXTENDED_B] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#LATIN_EXTENDED_B[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-LATIN_EXTENDED_C]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#LATIN_EXTENDED_C[LATIN_EXTENDED_C] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#LATIN_EXTENDED_C[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-LATIN_EXTENDED_D]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#LATIN_EXTENDED_D[LATIN_EXTENDED_D] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#LATIN_EXTENDED_D[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-LEPCHA]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#LEPCHA[LEPCHA] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#LEPCHA[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-LETTERLIKE_SYMBOLS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#LETTERLIKE_SYMBOLS[LETTERLIKE_SYMBOLS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#LETTERLIKE_SYMBOLS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-LIMBU]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#LIMBU[LIMBU] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#LIMBU[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-LINEAR_B_IDEOGRAMS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#LINEAR_B_IDEOGRAMS[LINEAR_B_IDEOGRAMS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#LINEAR_B_IDEOGRAMS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-LINEAR_B_SYLLABARY]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#LINEAR_B_SYLLABARY[LINEAR_B_SYLLABARY] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#LINEAR_B_SYLLABARY[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-LISU]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#LISU[LISU] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#LISU[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-LOW_SURROGATES]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#LOW_SURROGATES[LOW_SURROGATES] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#LOW_SURROGATES[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-LYCIAN]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#LYCIAN[LYCIAN] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#LYCIAN[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-LYDIAN]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#LYDIAN[LYDIAN] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#LYDIAN[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-MAHJONG_TILES]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#MAHJONG_TILES[MAHJONG_TILES] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#MAHJONG_TILES[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-MALAYALAM]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#MALAYALAM[MALAYALAM] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#MALAYALAM[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-MANDAIC]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#MANDAIC[MANDAIC] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#MANDAIC[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-MATHEMATICAL_ALPHANUMERIC_SYMBOLS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#MATHEMATICAL_ALPHANUMERIC_SYMBOLS[MATHEMATICAL_ALPHANUMERIC_SYMBOLS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#MATHEMATICAL_ALPHANUMERIC_SYMBOLS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-MATHEMATICAL_OPERATORS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#MATHEMATICAL_OPERATORS[MATHEMATICAL_OPERATORS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#MATHEMATICAL_OPERATORS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-MEETEI_MAYEK]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#MEETEI_MAYEK[MEETEI_MAYEK] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#MEETEI_MAYEK[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-MEETEI_MAYEK_EXTENSIONS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#MEETEI_MAYEK_EXTENSIONS[MEETEI_MAYEK_EXTENSIONS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#MEETEI_MAYEK_EXTENSIONS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-MEROITIC_CURSIVE]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#MEROITIC_CURSIVE[MEROITIC_CURSIVE] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#MEROITIC_CURSIVE[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-MEROITIC_HIEROGLYPHS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#MEROITIC_HIEROGLYPHS[MEROITIC_HIEROGLYPHS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#MEROITIC_HIEROGLYPHS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-MIAO]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#MIAO[MIAO] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#MIAO[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-MISCELLANEOUS_MATHEMATICAL_SYMBOLS_A]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#MISCELLANEOUS_MATHEMATICAL_SYMBOLS_A[MISCELLANEOUS_MATHEMATICAL_SYMBOLS_A] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#MISCELLANEOUS_MATHEMATICAL_SYMBOLS_A[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-MISCELLANEOUS_MATHEMATICAL_SYMBOLS_B]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#MISCELLANEOUS_MATHEMATICAL_SYMBOLS_B[MISCELLANEOUS_MATHEMATICAL_SYMBOLS_B] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#MISCELLANEOUS_MATHEMATICAL_SYMBOLS_B[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-MISCELLANEOUS_SYMBOLS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#MISCELLANEOUS_SYMBOLS[MISCELLANEOUS_SYMBOLS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#MISCELLANEOUS_SYMBOLS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-MISCELLANEOUS_SYMBOLS_AND_ARROWS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#MISCELLANEOUS_SYMBOLS_AND_ARROWS[MISCELLANEOUS_SYMBOLS_AND_ARROWS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#MISCELLANEOUS_SYMBOLS_AND_ARROWS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-MISCELLANEOUS_SYMBOLS_AND_PICTOGRAPHS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#MISCELLANEOUS_SYMBOLS_AND_PICTOGRAPHS[MISCELLANEOUS_SYMBOLS_AND_PICTOGRAPHS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#MISCELLANEOUS_SYMBOLS_AND_PICTOGRAPHS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-MISCELLANEOUS_TECHNICAL]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#MISCELLANEOUS_TECHNICAL[MISCELLANEOUS_TECHNICAL] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#MISCELLANEOUS_TECHNICAL[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-MODIFIER_TONE_LETTERS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#MODIFIER_TONE_LETTERS[MODIFIER_TONE_LETTERS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#MODIFIER_TONE_LETTERS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-MONGOLIAN]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#MONGOLIAN[MONGOLIAN] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#MONGOLIAN[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-MUSICAL_SYMBOLS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#MUSICAL_SYMBOLS[MUSICAL_SYMBOLS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#MUSICAL_SYMBOLS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-MYANMAR]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#MYANMAR[MYANMAR] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#MYANMAR[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-MYANMAR_EXTENDED_A]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#MYANMAR_EXTENDED_A[MYANMAR_EXTENDED_A] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#MYANMAR_EXTENDED_A[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-NEW_TAI_LUE]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#NEW_TAI_LUE[NEW_TAI_LUE] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#NEW_TAI_LUE[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-NKO]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#NKO[NKO] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#NKO[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-NUMBER_FORMS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#NUMBER_FORMS[NUMBER_FORMS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#NUMBER_FORMS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-OGHAM]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#OGHAM[OGHAM] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#OGHAM[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-OLD_ITALIC]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#OLD_ITALIC[OLD_ITALIC] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#OLD_ITALIC[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-OLD_PERSIAN]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#OLD_PERSIAN[OLD_PERSIAN] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#OLD_PERSIAN[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-OLD_SOUTH_ARABIAN]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#OLD_SOUTH_ARABIAN[OLD_SOUTH_ARABIAN] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#OLD_SOUTH_ARABIAN[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-OLD_TURKIC]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#OLD_TURKIC[OLD_TURKIC] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#OLD_TURKIC[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-OL_CHIKI]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#OL_CHIKI[OL_CHIKI] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#OL_CHIKI[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-OPTICAL_CHARACTER_RECOGNITION]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#OPTICAL_CHARACTER_RECOGNITION[OPTICAL_CHARACTER_RECOGNITION] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#OPTICAL_CHARACTER_RECOGNITION[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-ORIYA]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#ORIYA[ORIYA] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#ORIYA[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-OSMANYA]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#OSMANYA[OSMANYA] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#OSMANYA[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-PHAGS_PA]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#PHAGS_PA[PHAGS_PA] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#PHAGS_PA[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-PHAISTOS_DISC]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#PHAISTOS_DISC[PHAISTOS_DISC] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#PHAISTOS_DISC[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-PHOENICIAN]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#PHOENICIAN[PHOENICIAN] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#PHOENICIAN[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-PHONETIC_EXTENSIONS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#PHONETIC_EXTENSIONS[PHONETIC_EXTENSIONS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#PHONETIC_EXTENSIONS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-PHONETIC_EXTENSIONS_SUPPLEMENT]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#PHONETIC_EXTENSIONS_SUPPLEMENT[PHONETIC_EXTENSIONS_SUPPLEMENT] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#PHONETIC_EXTENSIONS_SUPPLEMENT[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-PLAYING_CARDS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#PLAYING_CARDS[PLAYING_CARDS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#PLAYING_CARDS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-PRIVATE_USE_AREA]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#PRIVATE_USE_AREA[PRIVATE_USE_AREA] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#PRIVATE_USE_AREA[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-REJANG]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#REJANG[REJANG] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#REJANG[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-RUMI_NUMERAL_SYMBOLS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#RUMI_NUMERAL_SYMBOLS[RUMI_NUMERAL_SYMBOLS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#RUMI_NUMERAL_SYMBOLS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-RUNIC]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#RUNIC[RUNIC] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#RUNIC[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-SAMARITAN]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#SAMARITAN[SAMARITAN] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#SAMARITAN[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-SAURASHTRA]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#SAURASHTRA[SAURASHTRA] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#SAURASHTRA[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-SHARADA]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#SHARADA[SHARADA] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#SHARADA[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-SHAVIAN]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#SHAVIAN[SHAVIAN] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#SHAVIAN[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-SINHALA]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#SINHALA[SINHALA] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#SINHALA[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-SMALL_FORM_VARIANTS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#SMALL_FORM_VARIANTS[SMALL_FORM_VARIANTS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#SMALL_FORM_VARIANTS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-SORA_SOMPENG]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#SORA_SOMPENG[SORA_SOMPENG] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#SORA_SOMPENG[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-SPACING_MODIFIER_LETTERS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#SPACING_MODIFIER_LETTERS[SPACING_MODIFIER_LETTERS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#SPACING_MODIFIER_LETTERS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-SPECIALS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#SPECIALS[SPECIALS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#SPECIALS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-SUNDANESE]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#SUNDANESE[SUNDANESE] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#SUNDANESE[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-SUNDANESE_SUPPLEMENT]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#SUNDANESE_SUPPLEMENT[SUNDANESE_SUPPLEMENT] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#SUNDANESE_SUPPLEMENT[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-SUPERSCRIPTS_AND_SUBSCRIPTS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#SUPERSCRIPTS_AND_SUBSCRIPTS[SUPERSCRIPTS_AND_SUBSCRIPTS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#SUPERSCRIPTS_AND_SUBSCRIPTS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-SUPPLEMENTAL_ARROWS_A]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#SUPPLEMENTAL_ARROWS_A[SUPPLEMENTAL_ARROWS_A] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#SUPPLEMENTAL_ARROWS_A[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-SUPPLEMENTAL_ARROWS_B]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#SUPPLEMENTAL_ARROWS_B[SUPPLEMENTAL_ARROWS_B] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#SUPPLEMENTAL_ARROWS_B[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-SUPPLEMENTAL_MATHEMATICAL_OPERATORS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#SUPPLEMENTAL_MATHEMATICAL_OPERATORS[SUPPLEMENTAL_MATHEMATICAL_OPERATORS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#SUPPLEMENTAL_MATHEMATICAL_OPERATORS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-SUPPLEMENTAL_PUNCTUATION]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#SUPPLEMENTAL_PUNCTUATION[SUPPLEMENTAL_PUNCTUATION] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#SUPPLEMENTAL_PUNCTUATION[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-SUPPLEMENTARY_PRIVATE_USE_AREA_A]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#SUPPLEMENTARY_PRIVATE_USE_AREA_A[SUPPLEMENTARY_PRIVATE_USE_AREA_A] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#SUPPLEMENTARY_PRIVATE_USE_AREA_A[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-SUPPLEMENTARY_PRIVATE_USE_AREA_B]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#SUPPLEMENTARY_PRIVATE_USE_AREA_B[SUPPLEMENTARY_PRIVATE_USE_AREA_B] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#SUPPLEMENTARY_PRIVATE_USE_AREA_B[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-SYLOTI_NAGRI]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#SYLOTI_NAGRI[SYLOTI_NAGRI] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#SYLOTI_NAGRI[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-SYRIAC]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#SYRIAC[SYRIAC] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#SYRIAC[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-TAGALOG]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#TAGALOG[TAGALOG] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#TAGALOG[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-TAGBANWA]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#TAGBANWA[TAGBANWA] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#TAGBANWA[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-TAGS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#TAGS[TAGS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#TAGS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-TAI_LE]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#TAI_LE[TAI_LE] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#TAI_LE[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-TAI_THAM]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#TAI_THAM[TAI_THAM] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#TAI_THAM[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-TAI_VIET]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#TAI_VIET[TAI_VIET] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#TAI_VIET[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-TAI_XUAN_JING_SYMBOLS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#TAI_XUAN_JING_SYMBOLS[TAI_XUAN_JING_SYMBOLS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#TAI_XUAN_JING_SYMBOLS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-TAKRI]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#TAKRI[TAKRI] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#TAKRI[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-TAMIL]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#TAMIL[TAMIL] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#TAMIL[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-TELUGU]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#TELUGU[TELUGU] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#TELUGU[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-THAANA]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#THAANA[THAANA] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#THAANA[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-THAI]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#THAI[THAI] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#THAI[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-TIBETAN]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#TIBETAN[TIBETAN] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#TIBETAN[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-TIFINAGH]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#TIFINAGH[TIFINAGH] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#TIFINAGH[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-TRANSPORT_AND_MAP_SYMBOLS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#TRANSPORT_AND_MAP_SYMBOLS[TRANSPORT_AND_MAP_SYMBOLS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#TRANSPORT_AND_MAP_SYMBOLS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-UGARITIC]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#UGARITIC[UGARITIC] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#UGARITIC[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-UNIFIED_CANADIAN_ABORIGINAL_SYLLABICS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#UNIFIED_CANADIAN_ABORIGINAL_SYLLABICS[UNIFIED_CANADIAN_ABORIGINAL_SYLLABICS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#UNIFIED_CANADIAN_ABORIGINAL_SYLLABICS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-UNIFIED_CANADIAN_ABORIGINAL_SYLLABICS_EXTENDED]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#UNIFIED_CANADIAN_ABORIGINAL_SYLLABICS_EXTENDED[UNIFIED_CANADIAN_ABORIGINAL_SYLLABICS_EXTENDED] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#UNIFIED_CANADIAN_ABORIGINAL_SYLLABICS_EXTENDED[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-VAI]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#VAI[VAI] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#VAI[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-VARIATION_SELECTORS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#VARIATION_SELECTORS[VARIATION_SELECTORS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#VARIATION_SELECTORS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-VARIATION_SELECTORS_SUPPLEMENT]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#VARIATION_SELECTORS_SUPPLEMENT[VARIATION_SELECTORS_SUPPLEMENT] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#VARIATION_SELECTORS_SUPPLEMENT[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-VEDIC_EXTENSIONS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#VEDIC_EXTENSIONS[VEDIC_EXTENSIONS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#VEDIC_EXTENSIONS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-VERTICAL_FORMS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#VERTICAL_FORMS[VERTICAL_FORMS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#VERTICAL_FORMS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-YIJING_HEXAGRAM_SYMBOLS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#YIJING_HEXAGRAM_SYMBOLS[YIJING_HEXAGRAM_SYMBOLS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#YIJING_HEXAGRAM_SYMBOLS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-YI_RADICALS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#YI_RADICALS[YI_RADICALS] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#YI_RADICALS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-YI_SYLLABLES]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#YI_SYLLABLES[YI_SYLLABLES] (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#YI_SYLLABLES[java 9])
+* ++[[painless-api-reference-Character-UnicodeBlock-forName-1]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#forName%2Djava.lang.String%2D[forName](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#forName%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Character-UnicodeBlock-of-1]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character.UnicodeBlock.html#of%2Dint%2D[of](int)++ (link:{java9-javadoc}/java/lang/Character.UnicodeBlock.html#of%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Character.UnicodeScript.asciidoc b/docs/painless/painless-api-reference/Character.UnicodeScript.asciidoc
new file mode 100644
index 0000000000000..0e9ff73e1e53d
--- /dev/null
+++ b/docs/painless/painless-api-reference/Character.UnicodeScript.asciidoc
@@ -0,0 +1,114 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Character-UnicodeScript]]++Character.UnicodeScript++::
+** [[painless-api-reference-Character-UnicodeScript-ARABIC]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#ARABIC[ARABIC] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#ARABIC[java 9])
+** [[painless-api-reference-Character-UnicodeScript-ARMENIAN]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#ARMENIAN[ARMENIAN] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#ARMENIAN[java 9])
+** [[painless-api-reference-Character-UnicodeScript-AVESTAN]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#AVESTAN[AVESTAN] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#AVESTAN[java 9])
+** [[painless-api-reference-Character-UnicodeScript-BALINESE]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#BALINESE[BALINESE] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#BALINESE[java 9])
+** [[painless-api-reference-Character-UnicodeScript-BAMUM]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#BAMUM[BAMUM] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#BAMUM[java 9])
+** [[painless-api-reference-Character-UnicodeScript-BATAK]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#BATAK[BATAK] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#BATAK[java 9])
+** [[painless-api-reference-Character-UnicodeScript-BENGALI]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#BENGALI[BENGALI] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#BENGALI[java 9])
+** [[painless-api-reference-Character-UnicodeScript-BOPOMOFO]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#BOPOMOFO[BOPOMOFO] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#BOPOMOFO[java 9])
+** [[painless-api-reference-Character-UnicodeScript-BRAHMI]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#BRAHMI[BRAHMI] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#BRAHMI[java 9])
+** [[painless-api-reference-Character-UnicodeScript-BRAILLE]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#BRAILLE[BRAILLE] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#BRAILLE[java 9])
+** [[painless-api-reference-Character-UnicodeScript-BUGINESE]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#BUGINESE[BUGINESE] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#BUGINESE[java 9])
+** [[painless-api-reference-Character-UnicodeScript-BUHID]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#BUHID[BUHID] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#BUHID[java 9])
+** [[painless-api-reference-Character-UnicodeScript-CANADIAN_ABORIGINAL]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#CANADIAN_ABORIGINAL[CANADIAN_ABORIGINAL] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#CANADIAN_ABORIGINAL[java 9])
+** [[painless-api-reference-Character-UnicodeScript-CARIAN]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#CARIAN[CARIAN] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#CARIAN[java 9])
+** [[painless-api-reference-Character-UnicodeScript-CHAKMA]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#CHAKMA[CHAKMA] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#CHAKMA[java 9])
+** [[painless-api-reference-Character-UnicodeScript-CHAM]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#CHAM[CHAM] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#CHAM[java 9])
+** [[painless-api-reference-Character-UnicodeScript-CHEROKEE]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#CHEROKEE[CHEROKEE] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#CHEROKEE[java 9])
+** [[painless-api-reference-Character-UnicodeScript-COMMON]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#COMMON[COMMON] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#COMMON[java 9])
+** [[painless-api-reference-Character-UnicodeScript-COPTIC]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#COPTIC[COPTIC] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#COPTIC[java 9])
+** [[painless-api-reference-Character-UnicodeScript-CUNEIFORM]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#CUNEIFORM[CUNEIFORM] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#CUNEIFORM[java 9])
+** [[painless-api-reference-Character-UnicodeScript-CYPRIOT]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#CYPRIOT[CYPRIOT] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#CYPRIOT[java 9])
+** [[painless-api-reference-Character-UnicodeScript-CYRILLIC]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#CYRILLIC[CYRILLIC] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#CYRILLIC[java 9])
+** [[painless-api-reference-Character-UnicodeScript-DESERET]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#DESERET[DESERET] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#DESERET[java 9])
+** [[painless-api-reference-Character-UnicodeScript-DEVANAGARI]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#DEVANAGARI[DEVANAGARI] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#DEVANAGARI[java 9])
+** [[painless-api-reference-Character-UnicodeScript-EGYPTIAN_HIEROGLYPHS]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#EGYPTIAN_HIEROGLYPHS[EGYPTIAN_HIEROGLYPHS] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#EGYPTIAN_HIEROGLYPHS[java 9])
+** [[painless-api-reference-Character-UnicodeScript-ETHIOPIC]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#ETHIOPIC[ETHIOPIC] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#ETHIOPIC[java 9])
+** [[painless-api-reference-Character-UnicodeScript-GEORGIAN]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#GEORGIAN[GEORGIAN] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#GEORGIAN[java 9])
+** [[painless-api-reference-Character-UnicodeScript-GLAGOLITIC]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#GLAGOLITIC[GLAGOLITIC] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#GLAGOLITIC[java 9])
+** [[painless-api-reference-Character-UnicodeScript-GOTHIC]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#GOTHIC[GOTHIC] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#GOTHIC[java 9])
+** [[painless-api-reference-Character-UnicodeScript-GREEK]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#GREEK[GREEK] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#GREEK[java 9])
+** [[painless-api-reference-Character-UnicodeScript-GUJARATI]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#GUJARATI[GUJARATI] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#GUJARATI[java 9])
+** [[painless-api-reference-Character-UnicodeScript-GURMUKHI]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#GURMUKHI[GURMUKHI] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#GURMUKHI[java 9])
+** [[painless-api-reference-Character-UnicodeScript-HAN]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#HAN[HAN] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#HAN[java 9])
+** [[painless-api-reference-Character-UnicodeScript-HANGUL]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#HANGUL[HANGUL] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#HANGUL[java 9])
+** [[painless-api-reference-Character-UnicodeScript-HANUNOO]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#HANUNOO[HANUNOO] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#HANUNOO[java 9])
+** [[painless-api-reference-Character-UnicodeScript-HEBREW]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#HEBREW[HEBREW] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#HEBREW[java 9])
+** [[painless-api-reference-Character-UnicodeScript-HIRAGANA]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#HIRAGANA[HIRAGANA] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#HIRAGANA[java 9])
+** [[painless-api-reference-Character-UnicodeScript-IMPERIAL_ARAMAIC]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#IMPERIAL_ARAMAIC[IMPERIAL_ARAMAIC] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#IMPERIAL_ARAMAIC[java 9])
+** [[painless-api-reference-Character-UnicodeScript-INHERITED]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#INHERITED[INHERITED] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#INHERITED[java 9])
+** [[painless-api-reference-Character-UnicodeScript-INSCRIPTIONAL_PAHLAVI]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#INSCRIPTIONAL_PAHLAVI[INSCRIPTIONAL_PAHLAVI] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#INSCRIPTIONAL_PAHLAVI[java 9])
+** [[painless-api-reference-Character-UnicodeScript-INSCRIPTIONAL_PARTHIAN]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#INSCRIPTIONAL_PARTHIAN[INSCRIPTIONAL_PARTHIAN] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#INSCRIPTIONAL_PARTHIAN[java 9])
+** [[painless-api-reference-Character-UnicodeScript-JAVANESE]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#JAVANESE[JAVANESE] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#JAVANESE[java 9])
+** [[painless-api-reference-Character-UnicodeScript-KAITHI]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#KAITHI[KAITHI] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#KAITHI[java 9])
+** [[painless-api-reference-Character-UnicodeScript-KANNADA]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#KANNADA[KANNADA] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#KANNADA[java 9])
+** [[painless-api-reference-Character-UnicodeScript-KATAKANA]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#KATAKANA[KATAKANA] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#KATAKANA[java 9])
+** [[painless-api-reference-Character-UnicodeScript-KAYAH_LI]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#KAYAH_LI[KAYAH_LI] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#KAYAH_LI[java 9])
+** [[painless-api-reference-Character-UnicodeScript-KHAROSHTHI]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#KHAROSHTHI[KHAROSHTHI] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#KHAROSHTHI[java 9])
+** [[painless-api-reference-Character-UnicodeScript-KHMER]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#KHMER[KHMER] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#KHMER[java 9])
+** [[painless-api-reference-Character-UnicodeScript-LAO]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#LAO[LAO] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#LAO[java 9])
+** [[painless-api-reference-Character-UnicodeScript-LATIN]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#LATIN[LATIN] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#LATIN[java 9])
+** [[painless-api-reference-Character-UnicodeScript-LEPCHA]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#LEPCHA[LEPCHA] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#LEPCHA[java 9])
+** [[painless-api-reference-Character-UnicodeScript-LIMBU]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#LIMBU[LIMBU] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#LIMBU[java 9])
+** [[painless-api-reference-Character-UnicodeScript-LINEAR_B]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#LINEAR_B[LINEAR_B] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#LINEAR_B[java 9])
+** [[painless-api-reference-Character-UnicodeScript-LISU]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#LISU[LISU] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#LISU[java 9])
+** [[painless-api-reference-Character-UnicodeScript-LYCIAN]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#LYCIAN[LYCIAN] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#LYCIAN[java 9])
+** [[painless-api-reference-Character-UnicodeScript-LYDIAN]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#LYDIAN[LYDIAN] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#LYDIAN[java 9])
+** [[painless-api-reference-Character-UnicodeScript-MALAYALAM]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#MALAYALAM[MALAYALAM] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#MALAYALAM[java 9])
+** [[painless-api-reference-Character-UnicodeScript-MANDAIC]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#MANDAIC[MANDAIC] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#MANDAIC[java 9])
+** [[painless-api-reference-Character-UnicodeScript-MEETEI_MAYEK]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#MEETEI_MAYEK[MEETEI_MAYEK] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#MEETEI_MAYEK[java 9])
+** [[painless-api-reference-Character-UnicodeScript-MEROITIC_CURSIVE]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#MEROITIC_CURSIVE[MEROITIC_CURSIVE] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#MEROITIC_CURSIVE[java 9])
+** [[painless-api-reference-Character-UnicodeScript-MEROITIC_HIEROGLYPHS]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#MEROITIC_HIEROGLYPHS[MEROITIC_HIEROGLYPHS] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#MEROITIC_HIEROGLYPHS[java 9])
+** [[painless-api-reference-Character-UnicodeScript-MIAO]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#MIAO[MIAO] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#MIAO[java 9])
+** [[painless-api-reference-Character-UnicodeScript-MONGOLIAN]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#MONGOLIAN[MONGOLIAN] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#MONGOLIAN[java 9])
+** [[painless-api-reference-Character-UnicodeScript-MYANMAR]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#MYANMAR[MYANMAR] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#MYANMAR[java 9])
+** [[painless-api-reference-Character-UnicodeScript-NEW_TAI_LUE]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#NEW_TAI_LUE[NEW_TAI_LUE] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#NEW_TAI_LUE[java 9])
+** [[painless-api-reference-Character-UnicodeScript-NKO]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#NKO[NKO] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#NKO[java 9])
+** [[painless-api-reference-Character-UnicodeScript-OGHAM]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#OGHAM[OGHAM] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#OGHAM[java 9])
+** [[painless-api-reference-Character-UnicodeScript-OLD_ITALIC]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#OLD_ITALIC[OLD_ITALIC] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#OLD_ITALIC[java 9])
+** [[painless-api-reference-Character-UnicodeScript-OLD_PERSIAN]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#OLD_PERSIAN[OLD_PERSIAN] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#OLD_PERSIAN[java 9])
+** [[painless-api-reference-Character-UnicodeScript-OLD_SOUTH_ARABIAN]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#OLD_SOUTH_ARABIAN[OLD_SOUTH_ARABIAN] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#OLD_SOUTH_ARABIAN[java 9])
+** [[painless-api-reference-Character-UnicodeScript-OLD_TURKIC]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#OLD_TURKIC[OLD_TURKIC] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#OLD_TURKIC[java 9])
+** [[painless-api-reference-Character-UnicodeScript-OL_CHIKI]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#OL_CHIKI[OL_CHIKI] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#OL_CHIKI[java 9])
+** [[painless-api-reference-Character-UnicodeScript-ORIYA]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#ORIYA[ORIYA] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#ORIYA[java 9])
+** [[painless-api-reference-Character-UnicodeScript-OSMANYA]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#OSMANYA[OSMANYA] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#OSMANYA[java 9])
+** [[painless-api-reference-Character-UnicodeScript-PHAGS_PA]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#PHAGS_PA[PHAGS_PA] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#PHAGS_PA[java 9])
+** [[painless-api-reference-Character-UnicodeScript-PHOENICIAN]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#PHOENICIAN[PHOENICIAN] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#PHOENICIAN[java 9])
+** [[painless-api-reference-Character-UnicodeScript-REJANG]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#REJANG[REJANG] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#REJANG[java 9])
+** [[painless-api-reference-Character-UnicodeScript-RUNIC]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#RUNIC[RUNIC] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#RUNIC[java 9])
+** [[painless-api-reference-Character-UnicodeScript-SAMARITAN]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#SAMARITAN[SAMARITAN] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#SAMARITAN[java 9])
+** [[painless-api-reference-Character-UnicodeScript-SAURASHTRA]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#SAURASHTRA[SAURASHTRA] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#SAURASHTRA[java 9])
+** [[painless-api-reference-Character-UnicodeScript-SHARADA]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#SHARADA[SHARADA] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#SHARADA[java 9])
+** [[painless-api-reference-Character-UnicodeScript-SHAVIAN]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#SHAVIAN[SHAVIAN] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#SHAVIAN[java 9])
+** [[painless-api-reference-Character-UnicodeScript-SINHALA]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#SINHALA[SINHALA] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#SINHALA[java 9])
+** [[painless-api-reference-Character-UnicodeScript-SORA_SOMPENG]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#SORA_SOMPENG[SORA_SOMPENG] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#SORA_SOMPENG[java 9])
+** [[painless-api-reference-Character-UnicodeScript-SUNDANESE]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#SUNDANESE[SUNDANESE] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#SUNDANESE[java 9])
+** [[painless-api-reference-Character-UnicodeScript-SYLOTI_NAGRI]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#SYLOTI_NAGRI[SYLOTI_NAGRI] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#SYLOTI_NAGRI[java 9])
+** [[painless-api-reference-Character-UnicodeScript-SYRIAC]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#SYRIAC[SYRIAC] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#SYRIAC[java 9])
+** [[painless-api-reference-Character-UnicodeScript-TAGALOG]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#TAGALOG[TAGALOG] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#TAGALOG[java 9])
+** [[painless-api-reference-Character-UnicodeScript-TAGBANWA]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#TAGBANWA[TAGBANWA] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#TAGBANWA[java 9])
+** [[painless-api-reference-Character-UnicodeScript-TAI_LE]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#TAI_LE[TAI_LE] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#TAI_LE[java 9])
+** [[painless-api-reference-Character-UnicodeScript-TAI_THAM]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#TAI_THAM[TAI_THAM] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#TAI_THAM[java 9])
+** [[painless-api-reference-Character-UnicodeScript-TAI_VIET]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#TAI_VIET[TAI_VIET] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#TAI_VIET[java 9])
+** [[painless-api-reference-Character-UnicodeScript-TAKRI]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#TAKRI[TAKRI] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#TAKRI[java 9])
+** [[painless-api-reference-Character-UnicodeScript-TAMIL]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#TAMIL[TAMIL] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#TAMIL[java 9])
+** [[painless-api-reference-Character-UnicodeScript-TELUGU]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#TELUGU[TELUGU] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#TELUGU[java 9])
+** [[painless-api-reference-Character-UnicodeScript-THAANA]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#THAANA[THAANA] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#THAANA[java 9])
+** [[painless-api-reference-Character-UnicodeScript-THAI]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#THAI[THAI] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#THAI[java 9])
+** [[painless-api-reference-Character-UnicodeScript-TIBETAN]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#TIBETAN[TIBETAN] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#TIBETAN[java 9])
+** [[painless-api-reference-Character-UnicodeScript-TIFINAGH]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#TIFINAGH[TIFINAGH] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#TIFINAGH[java 9])
+** [[painless-api-reference-Character-UnicodeScript-UGARITIC]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#UGARITIC[UGARITIC] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#UGARITIC[java 9])
+** [[painless-api-reference-Character-UnicodeScript-UNKNOWN]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#UNKNOWN[UNKNOWN] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#UNKNOWN[java 9])
+** [[painless-api-reference-Character-UnicodeScript-VAI]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#VAI[VAI] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#VAI[java 9])
+** [[painless-api-reference-Character-UnicodeScript-YI]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#YI[YI] (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#YI[java 9])
+* ++[[painless-api-reference-Character-UnicodeScript-forName-1]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#forName%2Djava.lang.String%2D[forName](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#forName%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Character-UnicodeScript-of-1]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#of%2Dint%2D[of](int)++ (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#of%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-UnicodeScript-valueOf-1]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#valueOf%2Djava.lang.String%2D[valueOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#valueOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Character-UnicodeScript-values-0]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>>[] link:{java8-javadoc}/java/lang/Character.UnicodeScript.html#values%2D%2D[values]()++ (link:{java9-javadoc}/java/lang/Character.UnicodeScript.html#values%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Enum,Enum>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Character.asciidoc b/docs/painless/painless-api-reference/Character.asciidoc
new file mode 100644
index 0000000000000..929a4ea3c5ec1
--- /dev/null
+++ b/docs/painless/painless-api-reference/Character.asciidoc
@@ -0,0 +1,125 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Character]]++Character++::
+** [[painless-api-reference-Character-BYTES]]static int link:{java8-javadoc}/java/lang/Character.html#BYTES[BYTES] (link:{java9-javadoc}/java/lang/Character.html#BYTES[java 9])
+** [[painless-api-reference-Character-COMBINING_SPACING_MARK]]static byte link:{java8-javadoc}/java/lang/Character.html#COMBINING_SPACING_MARK[COMBINING_SPACING_MARK] (link:{java9-javadoc}/java/lang/Character.html#COMBINING_SPACING_MARK[java 9])
+** [[painless-api-reference-Character-CONNECTOR_PUNCTUATION]]static byte link:{java8-javadoc}/java/lang/Character.html#CONNECTOR_PUNCTUATION[CONNECTOR_PUNCTUATION] (link:{java9-javadoc}/java/lang/Character.html#CONNECTOR_PUNCTUATION[java 9])
+** [[painless-api-reference-Character-CONTROL]]static byte link:{java8-javadoc}/java/lang/Character.html#CONTROL[CONTROL] (link:{java9-javadoc}/java/lang/Character.html#CONTROL[java 9])
+** [[painless-api-reference-Character-CURRENCY_SYMBOL]]static byte link:{java8-javadoc}/java/lang/Character.html#CURRENCY_SYMBOL[CURRENCY_SYMBOL] (link:{java9-javadoc}/java/lang/Character.html#CURRENCY_SYMBOL[java 9])
+** [[painless-api-reference-Character-DASH_PUNCTUATION]]static byte link:{java8-javadoc}/java/lang/Character.html#DASH_PUNCTUATION[DASH_PUNCTUATION] (link:{java9-javadoc}/java/lang/Character.html#DASH_PUNCTUATION[java 9])
+** [[painless-api-reference-Character-DECIMAL_DIGIT_NUMBER]]static byte link:{java8-javadoc}/java/lang/Character.html#DECIMAL_DIGIT_NUMBER[DECIMAL_DIGIT_NUMBER] (link:{java9-javadoc}/java/lang/Character.html#DECIMAL_DIGIT_NUMBER[java 9])
+** [[painless-api-reference-Character-DIRECTIONALITY_ARABIC_NUMBER]]static byte link:{java8-javadoc}/java/lang/Character.html#DIRECTIONALITY_ARABIC_NUMBER[DIRECTIONALITY_ARABIC_NUMBER] (link:{java9-javadoc}/java/lang/Character.html#DIRECTIONALITY_ARABIC_NUMBER[java 9])
+** [[painless-api-reference-Character-DIRECTIONALITY_BOUNDARY_NEUTRAL]]static byte link:{java8-javadoc}/java/lang/Character.html#DIRECTIONALITY_BOUNDARY_NEUTRAL[DIRECTIONALITY_BOUNDARY_NEUTRAL] (link:{java9-javadoc}/java/lang/Character.html#DIRECTIONALITY_BOUNDARY_NEUTRAL[java 9])
+** [[painless-api-reference-Character-DIRECTIONALITY_COMMON_NUMBER_SEPARATOR]]static byte link:{java8-javadoc}/java/lang/Character.html#DIRECTIONALITY_COMMON_NUMBER_SEPARATOR[DIRECTIONALITY_COMMON_NUMBER_SEPARATOR] (link:{java9-javadoc}/java/lang/Character.html#DIRECTIONALITY_COMMON_NUMBER_SEPARATOR[java 9])
+** [[painless-api-reference-Character-DIRECTIONALITY_EUROPEAN_NUMBER]]static byte link:{java8-javadoc}/java/lang/Character.html#DIRECTIONALITY_EUROPEAN_NUMBER[DIRECTIONALITY_EUROPEAN_NUMBER] (link:{java9-javadoc}/java/lang/Character.html#DIRECTIONALITY_EUROPEAN_NUMBER[java 9])
+** [[painless-api-reference-Character-DIRECTIONALITY_EUROPEAN_NUMBER_SEPARATOR]]static byte link:{java8-javadoc}/java/lang/Character.html#DIRECTIONALITY_EUROPEAN_NUMBER_SEPARATOR[DIRECTIONALITY_EUROPEAN_NUMBER_SEPARATOR] (link:{java9-javadoc}/java/lang/Character.html#DIRECTIONALITY_EUROPEAN_NUMBER_SEPARATOR[java 9])
+** [[painless-api-reference-Character-DIRECTIONALITY_EUROPEAN_NUMBER_TERMINATOR]]static byte link:{java8-javadoc}/java/lang/Character.html#DIRECTIONALITY_EUROPEAN_NUMBER_TERMINATOR[DIRECTIONALITY_EUROPEAN_NUMBER_TERMINATOR] (link:{java9-javadoc}/java/lang/Character.html#DIRECTIONALITY_EUROPEAN_NUMBER_TERMINATOR[java 9])
+** [[painless-api-reference-Character-DIRECTIONALITY_LEFT_TO_RIGHT]]static byte link:{java8-javadoc}/java/lang/Character.html#DIRECTIONALITY_LEFT_TO_RIGHT[DIRECTIONALITY_LEFT_TO_RIGHT] (link:{java9-javadoc}/java/lang/Character.html#DIRECTIONALITY_LEFT_TO_RIGHT[java 9])
+** [[painless-api-reference-Character-DIRECTIONALITY_LEFT_TO_RIGHT_EMBEDDING]]static byte link:{java8-javadoc}/java/lang/Character.html#DIRECTIONALITY_LEFT_TO_RIGHT_EMBEDDING[DIRECTIONALITY_LEFT_TO_RIGHT_EMBEDDING] (link:{java9-javadoc}/java/lang/Character.html#DIRECTIONALITY_LEFT_TO_RIGHT_EMBEDDING[java 9])
+** [[painless-api-reference-Character-DIRECTIONALITY_LEFT_TO_RIGHT_OVERRIDE]]static byte link:{java8-javadoc}/java/lang/Character.html#DIRECTIONALITY_LEFT_TO_RIGHT_OVERRIDE[DIRECTIONALITY_LEFT_TO_RIGHT_OVERRIDE] (link:{java9-javadoc}/java/lang/Character.html#DIRECTIONALITY_LEFT_TO_RIGHT_OVERRIDE[java 9])
+** [[painless-api-reference-Character-DIRECTIONALITY_NONSPACING_MARK]]static byte link:{java8-javadoc}/java/lang/Character.html#DIRECTIONALITY_NONSPACING_MARK[DIRECTIONALITY_NONSPACING_MARK] (link:{java9-javadoc}/java/lang/Character.html#DIRECTIONALITY_NONSPACING_MARK[java 9])
+** [[painless-api-reference-Character-DIRECTIONALITY_OTHER_NEUTRALS]]static byte link:{java8-javadoc}/java/lang/Character.html#DIRECTIONALITY_OTHER_NEUTRALS[DIRECTIONALITY_OTHER_NEUTRALS] (link:{java9-javadoc}/java/lang/Character.html#DIRECTIONALITY_OTHER_NEUTRALS[java 9])
+** [[painless-api-reference-Character-DIRECTIONALITY_PARAGRAPH_SEPARATOR]]static byte link:{java8-javadoc}/java/lang/Character.html#DIRECTIONALITY_PARAGRAPH_SEPARATOR[DIRECTIONALITY_PARAGRAPH_SEPARATOR] (link:{java9-javadoc}/java/lang/Character.html#DIRECTIONALITY_PARAGRAPH_SEPARATOR[java 9])
+** [[painless-api-reference-Character-DIRECTIONALITY_POP_DIRECTIONAL_FORMAT]]static byte link:{java8-javadoc}/java/lang/Character.html#DIRECTIONALITY_POP_DIRECTIONAL_FORMAT[DIRECTIONALITY_POP_DIRECTIONAL_FORMAT] (link:{java9-javadoc}/java/lang/Character.html#DIRECTIONALITY_POP_DIRECTIONAL_FORMAT[java 9])
+** [[painless-api-reference-Character-DIRECTIONALITY_RIGHT_TO_LEFT]]static byte link:{java8-javadoc}/java/lang/Character.html#DIRECTIONALITY_RIGHT_TO_LEFT[DIRECTIONALITY_RIGHT_TO_LEFT] (link:{java9-javadoc}/java/lang/Character.html#DIRECTIONALITY_RIGHT_TO_LEFT[java 9])
+** [[painless-api-reference-Character-DIRECTIONALITY_RIGHT_TO_LEFT_ARABIC]]static byte link:{java8-javadoc}/java/lang/Character.html#DIRECTIONALITY_RIGHT_TO_LEFT_ARABIC[DIRECTIONALITY_RIGHT_TO_LEFT_ARABIC] (link:{java9-javadoc}/java/lang/Character.html#DIRECTIONALITY_RIGHT_TO_LEFT_ARABIC[java 9])
+** [[painless-api-reference-Character-DIRECTIONALITY_RIGHT_TO_LEFT_EMBEDDING]]static byte link:{java8-javadoc}/java/lang/Character.html#DIRECTIONALITY_RIGHT_TO_LEFT_EMBEDDING[DIRECTIONALITY_RIGHT_TO_LEFT_EMBEDDING] (link:{java9-javadoc}/java/lang/Character.html#DIRECTIONALITY_RIGHT_TO_LEFT_EMBEDDING[java 9])
+** [[painless-api-reference-Character-DIRECTIONALITY_RIGHT_TO_LEFT_OVERRIDE]]static byte link:{java8-javadoc}/java/lang/Character.html#DIRECTIONALITY_RIGHT_TO_LEFT_OVERRIDE[DIRECTIONALITY_RIGHT_TO_LEFT_OVERRIDE] (link:{java9-javadoc}/java/lang/Character.html#DIRECTIONALITY_RIGHT_TO_LEFT_OVERRIDE[java 9])
+** [[painless-api-reference-Character-DIRECTIONALITY_SEGMENT_SEPARATOR]]static byte link:{java8-javadoc}/java/lang/Character.html#DIRECTIONALITY_SEGMENT_SEPARATOR[DIRECTIONALITY_SEGMENT_SEPARATOR] (link:{java9-javadoc}/java/lang/Character.html#DIRECTIONALITY_SEGMENT_SEPARATOR[java 9])
+** [[painless-api-reference-Character-DIRECTIONALITY_UNDEFINED]]static byte link:{java8-javadoc}/java/lang/Character.html#DIRECTIONALITY_UNDEFINED[DIRECTIONALITY_UNDEFINED] (link:{java9-javadoc}/java/lang/Character.html#DIRECTIONALITY_UNDEFINED[java 9])
+** [[painless-api-reference-Character-DIRECTIONALITY_WHITESPACE]]static byte link:{java8-javadoc}/java/lang/Character.html#DIRECTIONALITY_WHITESPACE[DIRECTIONALITY_WHITESPACE] (link:{java9-javadoc}/java/lang/Character.html#DIRECTIONALITY_WHITESPACE[java 9])
+** [[painless-api-reference-Character-ENCLOSING_MARK]]static byte link:{java8-javadoc}/java/lang/Character.html#ENCLOSING_MARK[ENCLOSING_MARK] (link:{java9-javadoc}/java/lang/Character.html#ENCLOSING_MARK[java 9])
+** [[painless-api-reference-Character-END_PUNCTUATION]]static byte link:{java8-javadoc}/java/lang/Character.html#END_PUNCTUATION[END_PUNCTUATION] (link:{java9-javadoc}/java/lang/Character.html#END_PUNCTUATION[java 9])
+** [[painless-api-reference-Character-FINAL_QUOTE_PUNCTUATION]]static byte link:{java8-javadoc}/java/lang/Character.html#FINAL_QUOTE_PUNCTUATION[FINAL_QUOTE_PUNCTUATION] (link:{java9-javadoc}/java/lang/Character.html#FINAL_QUOTE_PUNCTUATION[java 9])
+** [[painless-api-reference-Character-FORMAT]]static byte link:{java8-javadoc}/java/lang/Character.html#FORMAT[FORMAT] (link:{java9-javadoc}/java/lang/Character.html#FORMAT[java 9])
+** [[painless-api-reference-Character-INITIAL_QUOTE_PUNCTUATION]]static byte link:{java8-javadoc}/java/lang/Character.html#INITIAL_QUOTE_PUNCTUATION[INITIAL_QUOTE_PUNCTUATION] (link:{java9-javadoc}/java/lang/Character.html#INITIAL_QUOTE_PUNCTUATION[java 9])
+** [[painless-api-reference-Character-LETTER_NUMBER]]static byte link:{java8-javadoc}/java/lang/Character.html#LETTER_NUMBER[LETTER_NUMBER] (link:{java9-javadoc}/java/lang/Character.html#LETTER_NUMBER[java 9])
+** [[painless-api-reference-Character-LINE_SEPARATOR]]static byte link:{java8-javadoc}/java/lang/Character.html#LINE_SEPARATOR[LINE_SEPARATOR] (link:{java9-javadoc}/java/lang/Character.html#LINE_SEPARATOR[java 9])
+** [[painless-api-reference-Character-LOWERCASE_LETTER]]static byte link:{java8-javadoc}/java/lang/Character.html#LOWERCASE_LETTER[LOWERCASE_LETTER] (link:{java9-javadoc}/java/lang/Character.html#LOWERCASE_LETTER[java 9])
+** [[painless-api-reference-Character-MATH_SYMBOL]]static byte link:{java8-javadoc}/java/lang/Character.html#MATH_SYMBOL[MATH_SYMBOL] (link:{java9-javadoc}/java/lang/Character.html#MATH_SYMBOL[java 9])
+** [[painless-api-reference-Character-MAX_CODE_POINT]]static int link:{java8-javadoc}/java/lang/Character.html#MAX_CODE_POINT[MAX_CODE_POINT] (link:{java9-javadoc}/java/lang/Character.html#MAX_CODE_POINT[java 9])
+** [[painless-api-reference-Character-MAX_HIGH_SURROGATE]]static char link:{java8-javadoc}/java/lang/Character.html#MAX_HIGH_SURROGATE[MAX_HIGH_SURROGATE] (link:{java9-javadoc}/java/lang/Character.html#MAX_HIGH_SURROGATE[java 9])
+** [[painless-api-reference-Character-MAX_LOW_SURROGATE]]static char link:{java8-javadoc}/java/lang/Character.html#MAX_LOW_SURROGATE[MAX_LOW_SURROGATE] (link:{java9-javadoc}/java/lang/Character.html#MAX_LOW_SURROGATE[java 9])
+** [[painless-api-reference-Character-MAX_RADIX]]static int link:{java8-javadoc}/java/lang/Character.html#MAX_RADIX[MAX_RADIX] (link:{java9-javadoc}/java/lang/Character.html#MAX_RADIX[java 9])
+** [[painless-api-reference-Character-MAX_SURROGATE]]static char link:{java8-javadoc}/java/lang/Character.html#MAX_SURROGATE[MAX_SURROGATE] (link:{java9-javadoc}/java/lang/Character.html#MAX_SURROGATE[java 9])
+** [[painless-api-reference-Character-MAX_VALUE]]static char link:{java8-javadoc}/java/lang/Character.html#MAX_VALUE[MAX_VALUE] (link:{java9-javadoc}/java/lang/Character.html#MAX_VALUE[java 9])
+** [[painless-api-reference-Character-MIN_CODE_POINT]]static char link:{java8-javadoc}/java/lang/Character.html#MIN_CODE_POINT[MIN_CODE_POINT] (link:{java9-javadoc}/java/lang/Character.html#MIN_CODE_POINT[java 9])
+** [[painless-api-reference-Character-MIN_HIGH_SURROGATE]]static char link:{java8-javadoc}/java/lang/Character.html#MIN_HIGH_SURROGATE[MIN_HIGH_SURROGATE] (link:{java9-javadoc}/java/lang/Character.html#MIN_HIGH_SURROGATE[java 9])
+** [[painless-api-reference-Character-MIN_LOW_SURROGATE]]static char link:{java8-javadoc}/java/lang/Character.html#MIN_LOW_SURROGATE[MIN_LOW_SURROGATE] (link:{java9-javadoc}/java/lang/Character.html#MIN_LOW_SURROGATE[java 9])
+** [[painless-api-reference-Character-MIN_RADIX]]static int link:{java8-javadoc}/java/lang/Character.html#MIN_RADIX[MIN_RADIX] (link:{java9-javadoc}/java/lang/Character.html#MIN_RADIX[java 9])
+** [[painless-api-reference-Character-MIN_SUPPLEMENTARY_CODE_POINT]]static int link:{java8-javadoc}/java/lang/Character.html#MIN_SUPPLEMENTARY_CODE_POINT[MIN_SUPPLEMENTARY_CODE_POINT] (link:{java9-javadoc}/java/lang/Character.html#MIN_SUPPLEMENTARY_CODE_POINT[java 9])
+** [[painless-api-reference-Character-MIN_SURROGATE]]static char link:{java8-javadoc}/java/lang/Character.html#MIN_SURROGATE[MIN_SURROGATE] (link:{java9-javadoc}/java/lang/Character.html#MIN_SURROGATE[java 9])
+** [[painless-api-reference-Character-MIN_VALUE]]static char link:{java8-javadoc}/java/lang/Character.html#MIN_VALUE[MIN_VALUE] (link:{java9-javadoc}/java/lang/Character.html#MIN_VALUE[java 9])
+** [[painless-api-reference-Character-MODIFIER_LETTER]]static byte link:{java8-javadoc}/java/lang/Character.html#MODIFIER_LETTER[MODIFIER_LETTER] (link:{java9-javadoc}/java/lang/Character.html#MODIFIER_LETTER[java 9])
+** [[painless-api-reference-Character-MODIFIER_SYMBOL]]static byte link:{java8-javadoc}/java/lang/Character.html#MODIFIER_SYMBOL[MODIFIER_SYMBOL] (link:{java9-javadoc}/java/lang/Character.html#MODIFIER_SYMBOL[java 9])
+** [[painless-api-reference-Character-NON_SPACING_MARK]]static byte link:{java8-javadoc}/java/lang/Character.html#NON_SPACING_MARK[NON_SPACING_MARK] (link:{java9-javadoc}/java/lang/Character.html#NON_SPACING_MARK[java 9])
+** [[painless-api-reference-Character-OTHER_LETTER]]static byte link:{java8-javadoc}/java/lang/Character.html#OTHER_LETTER[OTHER_LETTER] (link:{java9-javadoc}/java/lang/Character.html#OTHER_LETTER[java 9])
+** [[painless-api-reference-Character-OTHER_NUMBER]]static byte link:{java8-javadoc}/java/lang/Character.html#OTHER_NUMBER[OTHER_NUMBER] (link:{java9-javadoc}/java/lang/Character.html#OTHER_NUMBER[java 9])
+** [[painless-api-reference-Character-OTHER_PUNCTUATION]]static byte link:{java8-javadoc}/java/lang/Character.html#OTHER_PUNCTUATION[OTHER_PUNCTUATION] (link:{java9-javadoc}/java/lang/Character.html#OTHER_PUNCTUATION[java 9])
+** [[painless-api-reference-Character-OTHER_SYMBOL]]static byte link:{java8-javadoc}/java/lang/Character.html#OTHER_SYMBOL[OTHER_SYMBOL] (link:{java9-javadoc}/java/lang/Character.html#OTHER_SYMBOL[java 9])
+** [[painless-api-reference-Character-PARAGRAPH_SEPARATOR]]static byte link:{java8-javadoc}/java/lang/Character.html#PARAGRAPH_SEPARATOR[PARAGRAPH_SEPARATOR] (link:{java9-javadoc}/java/lang/Character.html#PARAGRAPH_SEPARATOR[java 9])
+** [[painless-api-reference-Character-PRIVATE_USE]]static byte link:{java8-javadoc}/java/lang/Character.html#PRIVATE_USE[PRIVATE_USE] (link:{java9-javadoc}/java/lang/Character.html#PRIVATE_USE[java 9])
+** [[painless-api-reference-Character-SIZE]]static int link:{java8-javadoc}/java/lang/Character.html#SIZE[SIZE] (link:{java9-javadoc}/java/lang/Character.html#SIZE[java 9])
+** [[painless-api-reference-Character-SPACE_SEPARATOR]]static byte link:{java8-javadoc}/java/lang/Character.html#SPACE_SEPARATOR[SPACE_SEPARATOR] (link:{java9-javadoc}/java/lang/Character.html#SPACE_SEPARATOR[java 9])
+** [[painless-api-reference-Character-START_PUNCTUATION]]static byte link:{java8-javadoc}/java/lang/Character.html#START_PUNCTUATION[START_PUNCTUATION] (link:{java9-javadoc}/java/lang/Character.html#START_PUNCTUATION[java 9])
+** [[painless-api-reference-Character-SURROGATE]]static byte link:{java8-javadoc}/java/lang/Character.html#SURROGATE[SURROGATE] (link:{java9-javadoc}/java/lang/Character.html#SURROGATE[java 9])
+** [[painless-api-reference-Character-TITLECASE_LETTER]]static byte link:{java8-javadoc}/java/lang/Character.html#TITLECASE_LETTER[TITLECASE_LETTER] (link:{java9-javadoc}/java/lang/Character.html#TITLECASE_LETTER[java 9])
+** [[painless-api-reference-Character-UNASSIGNED]]static byte link:{java8-javadoc}/java/lang/Character.html#UNASSIGNED[UNASSIGNED] (link:{java9-javadoc}/java/lang/Character.html#UNASSIGNED[java 9])
+** [[painless-api-reference-Character-UPPERCASE_LETTER]]static byte link:{java8-javadoc}/java/lang/Character.html#UPPERCASE_LETTER[UPPERCASE_LETTER] (link:{java9-javadoc}/java/lang/Character.html#UPPERCASE_LETTER[java 9])
+* ++[[painless-api-reference-Character-charCount-1]]static int link:{java8-javadoc}/java/lang/Character.html#charCount%2Dint%2D[charCount](int)++ (link:{java9-javadoc}/java/lang/Character.html#charCount%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-codePointAt-2]]static int link:{java8-javadoc}/java/lang/Character.html#codePointAt%2Djava.lang.CharSequence%2Dint%2D[codePointAt](<<painless-api-reference-CharSequence,CharSequence>>, int)++ (link:{java9-javadoc}/java/lang/Character.html#codePointAt%2Djava.lang.CharSequence%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-codePointAt-3]]static int link:{java8-javadoc}/java/lang/Character.html#codePointAt%2Dchar:A%2Dint%2Dint%2D[codePointAt](char[], int, int)++ (link:{java9-javadoc}/java/lang/Character.html#codePointAt%2Dchar:A%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-codePointBefore-2]]static int link:{java8-javadoc}/java/lang/Character.html#codePointBefore%2Djava.lang.CharSequence%2Dint%2D[codePointBefore](<<painless-api-reference-CharSequence,CharSequence>>, int)++ (link:{java9-javadoc}/java/lang/Character.html#codePointBefore%2Djava.lang.CharSequence%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-codePointBefore-3]]static int link:{java8-javadoc}/java/lang/Character.html#codePointBefore%2Dchar:A%2Dint%2Dint%2D[codePointBefore](char[], int, int)++ (link:{java9-javadoc}/java/lang/Character.html#codePointBefore%2Dchar:A%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-codePointCount-3]]static int link:{java8-javadoc}/java/lang/Character.html#codePointCount%2Djava.lang.CharSequence%2Dint%2Dint%2D[codePointCount](<<painless-api-reference-CharSequence,CharSequence>>, int, int)++ (link:{java9-javadoc}/java/lang/Character.html#codePointCount%2Djava.lang.CharSequence%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-compare-2]]static int link:{java8-javadoc}/java/lang/Character.html#compare%2Dchar%2Dchar%2D[compare](char, char)++ (link:{java9-javadoc}/java/lang/Character.html#compare%2Dchar%2Dchar%2D[java 9])
+* ++[[painless-api-reference-Character-digit-2]]static int link:{java8-javadoc}/java/lang/Character.html#digit%2Dint%2Dint%2D[digit](int, int)++ (link:{java9-javadoc}/java/lang/Character.html#digit%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-forDigit-2]]static char link:{java8-javadoc}/java/lang/Character.html#forDigit%2Dint%2Dint%2D[forDigit](int, int)++ (link:{java9-javadoc}/java/lang/Character.html#forDigit%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-getDirectionality-1]]static byte link:{java8-javadoc}/java/lang/Character.html#getDirectionality%2Dint%2D[getDirectionality](int)++ (link:{java9-javadoc}/java/lang/Character.html#getDirectionality%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-getName-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Character.html#getName%2Dint%2D[getName](int)++ (link:{java9-javadoc}/java/lang/Character.html#getName%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-getNumericValue-1]]static int link:{java8-javadoc}/java/lang/Character.html#getNumericValue%2Dint%2D[getNumericValue](int)++ (link:{java9-javadoc}/java/lang/Character.html#getNumericValue%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-getType-1]]static int link:{java8-javadoc}/java/lang/Character.html#getType%2Dint%2D[getType](int)++ (link:{java9-javadoc}/java/lang/Character.html#getType%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-hashCode-1]]static int link:{java8-javadoc}/java/lang/Character.html#hashCode%2Dchar%2D[hashCode](char)++ (link:{java9-javadoc}/java/lang/Character.html#hashCode%2Dchar%2D[java 9])
+* ++[[painless-api-reference-Character-highSurrogate-1]]static char link:{java8-javadoc}/java/lang/Character.html#highSurrogate%2Dint%2D[highSurrogate](int)++ (link:{java9-javadoc}/java/lang/Character.html#highSurrogate%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-isAlphabetic-1]]static boolean link:{java8-javadoc}/java/lang/Character.html#isAlphabetic%2Dint%2D[isAlphabetic](int)++ (link:{java9-javadoc}/java/lang/Character.html#isAlphabetic%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-isBmpCodePoint-1]]static boolean link:{java8-javadoc}/java/lang/Character.html#isBmpCodePoint%2Dint%2D[isBmpCodePoint](int)++ (link:{java9-javadoc}/java/lang/Character.html#isBmpCodePoint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-isDefined-1]]static boolean link:{java8-javadoc}/java/lang/Character.html#isDefined%2Dint%2D[isDefined](int)++ (link:{java9-javadoc}/java/lang/Character.html#isDefined%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-isDigit-1]]static boolean link:{java8-javadoc}/java/lang/Character.html#isDigit%2Dint%2D[isDigit](int)++ (link:{java9-javadoc}/java/lang/Character.html#isDigit%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-isHighSurrogate-1]]static boolean link:{java8-javadoc}/java/lang/Character.html#isHighSurrogate%2Dchar%2D[isHighSurrogate](char)++ (link:{java9-javadoc}/java/lang/Character.html#isHighSurrogate%2Dchar%2D[java 9])
+* ++[[painless-api-reference-Character-isISOControl-1]]static boolean link:{java8-javadoc}/java/lang/Character.html#isISOControl%2Dint%2D[isISOControl](int)++ (link:{java9-javadoc}/java/lang/Character.html#isISOControl%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-isIdentifierIgnorable-1]]static boolean link:{java8-javadoc}/java/lang/Character.html#isIdentifierIgnorable%2Dint%2D[isIdentifierIgnorable](int)++ (link:{java9-javadoc}/java/lang/Character.html#isIdentifierIgnorable%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-isIdeographic-1]]static boolean link:{java8-javadoc}/java/lang/Character.html#isIdeographic%2Dint%2D[isIdeographic](int)++ (link:{java9-javadoc}/java/lang/Character.html#isIdeographic%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-isJavaIdentifierPart-1]]static boolean link:{java8-javadoc}/java/lang/Character.html#isJavaIdentifierPart%2Dint%2D[isJavaIdentifierPart](int)++ (link:{java9-javadoc}/java/lang/Character.html#isJavaIdentifierPart%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-isJavaIdentifierStart-1]]static boolean link:{java8-javadoc}/java/lang/Character.html#isJavaIdentifierStart%2Dint%2D[isJavaIdentifierStart](int)++ (link:{java9-javadoc}/java/lang/Character.html#isJavaIdentifierStart%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-isLetter-1]]static boolean link:{java8-javadoc}/java/lang/Character.html#isLetter%2Dint%2D[isLetter](int)++ (link:{java9-javadoc}/java/lang/Character.html#isLetter%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-isLetterOrDigit-1]]static boolean link:{java8-javadoc}/java/lang/Character.html#isLetterOrDigit%2Dint%2D[isLetterOrDigit](int)++ (link:{java9-javadoc}/java/lang/Character.html#isLetterOrDigit%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-isLowerCase-1]]static boolean link:{java8-javadoc}/java/lang/Character.html#isLowerCase%2Dint%2D[isLowerCase](int)++ (link:{java9-javadoc}/java/lang/Character.html#isLowerCase%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-isMirrored-1]]static boolean link:{java8-javadoc}/java/lang/Character.html#isMirrored%2Dint%2D[isMirrored](int)++ (link:{java9-javadoc}/java/lang/Character.html#isMirrored%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-isSpaceChar-1]]static boolean link:{java8-javadoc}/java/lang/Character.html#isSpaceChar%2Dint%2D[isSpaceChar](int)++ (link:{java9-javadoc}/java/lang/Character.html#isSpaceChar%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-isSupplementaryCodePoint-1]]static boolean link:{java8-javadoc}/java/lang/Character.html#isSupplementaryCodePoint%2Dint%2D[isSupplementaryCodePoint](int)++ (link:{java9-javadoc}/java/lang/Character.html#isSupplementaryCodePoint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-isSurrogate-1]]static boolean link:{java8-javadoc}/java/lang/Character.html#isSurrogate%2Dchar%2D[isSurrogate](char)++ (link:{java9-javadoc}/java/lang/Character.html#isSurrogate%2Dchar%2D[java 9])
+* ++[[painless-api-reference-Character-isSurrogatePair-2]]static boolean link:{java8-javadoc}/java/lang/Character.html#isSurrogatePair%2Dchar%2Dchar%2D[isSurrogatePair](char, char)++ (link:{java9-javadoc}/java/lang/Character.html#isSurrogatePair%2Dchar%2Dchar%2D[java 9])
+* ++[[painless-api-reference-Character-isTitleCase-1]]static boolean link:{java8-javadoc}/java/lang/Character.html#isTitleCase%2Dint%2D[isTitleCase](int)++ (link:{java9-javadoc}/java/lang/Character.html#isTitleCase%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-isUnicodeIdentifierPart-1]]static boolean link:{java8-javadoc}/java/lang/Character.html#isUnicodeIdentifierPart%2Dint%2D[isUnicodeIdentifierPart](int)++ (link:{java9-javadoc}/java/lang/Character.html#isUnicodeIdentifierPart%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-isUnicodeIdentifierStart-1]]static boolean link:{java8-javadoc}/java/lang/Character.html#isUnicodeIdentifierStart%2Dint%2D[isUnicodeIdentifierStart](int)++ (link:{java9-javadoc}/java/lang/Character.html#isUnicodeIdentifierStart%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-isUpperCase-1]]static boolean link:{java8-javadoc}/java/lang/Character.html#isUpperCase%2Dint%2D[isUpperCase](int)++ (link:{java9-javadoc}/java/lang/Character.html#isUpperCase%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-isValidCodePoint-1]]static boolean link:{java8-javadoc}/java/lang/Character.html#isValidCodePoint%2Dint%2D[isValidCodePoint](int)++ (link:{java9-javadoc}/java/lang/Character.html#isValidCodePoint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-isWhitespace-1]]static boolean link:{java8-javadoc}/java/lang/Character.html#isWhitespace%2Dint%2D[isWhitespace](int)++ (link:{java9-javadoc}/java/lang/Character.html#isWhitespace%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-lowSurrogate-1]]static char link:{java8-javadoc}/java/lang/Character.html#lowSurrogate%2Dint%2D[lowSurrogate](int)++ (link:{java9-javadoc}/java/lang/Character.html#lowSurrogate%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-offsetByCodePoints-3]]static int link:{java8-javadoc}/java/lang/Character.html#offsetByCodePoints%2Djava.lang.CharSequence%2Dint%2Dint%2D[offsetByCodePoints](<<painless-api-reference-CharSequence,CharSequence>>, int, int)++ (link:{java9-javadoc}/java/lang/Character.html#offsetByCodePoints%2Djava.lang.CharSequence%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-offsetByCodePoints-5]]static int link:{java8-javadoc}/java/lang/Character.html#offsetByCodePoints%2Dchar:A%2Dint%2Dint%2Dint%2Dint%2D[offsetByCodePoints](char[], int, int, int, int)++ (link:{java9-javadoc}/java/lang/Character.html#offsetByCodePoints%2Dchar:A%2Dint%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-reverseBytes-1]]static char link:{java8-javadoc}/java/lang/Character.html#reverseBytes%2Dchar%2D[reverseBytes](char)++ (link:{java9-javadoc}/java/lang/Character.html#reverseBytes%2Dchar%2D[java 9])
+* ++[[painless-api-reference-Character-toChars-1]]static char[] link:{java8-javadoc}/java/lang/Character.html#toChars%2Dint%2D[toChars](int)++ (link:{java9-javadoc}/java/lang/Character.html#toChars%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-toChars-3]]static int link:{java8-javadoc}/java/lang/Character.html#toChars%2Dint%2Dchar:A%2Dint%2D[toChars](int, char[], int)++ (link:{java9-javadoc}/java/lang/Character.html#toChars%2Dint%2Dchar:A%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-toCodePoint-2]]static int link:{java8-javadoc}/java/lang/Character.html#toCodePoint%2Dchar%2Dchar%2D[toCodePoint](char, char)++ (link:{java9-javadoc}/java/lang/Character.html#toCodePoint%2Dchar%2Dchar%2D[java 9])
+* ++[[painless-api-reference-Character-toLowerCase-1]]static char link:{java8-javadoc}/java/lang/Character.html#toLowerCase%2Dchar%2D[toLowerCase](char)++ (link:{java9-javadoc}/java/lang/Character.html#toLowerCase%2Dchar%2D[java 9])
+* ++[[painless-api-reference-Character-toString-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Character.html#toString%2Dchar%2D[toString](char)++ (link:{java9-javadoc}/java/lang/Character.html#toString%2Dchar%2D[java 9])
+* ++[[painless-api-reference-Character-toTitleCase-1]]static char link:{java8-javadoc}/java/lang/Character.html#toTitleCase%2Dchar%2D[toTitleCase](char)++ (link:{java9-javadoc}/java/lang/Character.html#toTitleCase%2Dchar%2D[java 9])
+* ++[[painless-api-reference-Character-toUpperCase-1]]static char link:{java8-javadoc}/java/lang/Character.html#toUpperCase%2Dchar%2D[toUpperCase](char)++ (link:{java9-javadoc}/java/lang/Character.html#toUpperCase%2Dchar%2D[java 9])
+* ++[[painless-api-reference-Character-valueOf-1]]static <<painless-api-reference-Character,Character>> link:{java8-javadoc}/java/lang/Character.html#valueOf%2Dchar%2D[valueOf](char)++ (link:{java9-javadoc}/java/lang/Character.html#valueOf%2Dchar%2D[java 9])
+* ++[[painless-api-reference-Character-charValue-0]]char link:{java8-javadoc}/java/lang/Character.html#charValue%2D%2D[charValue]()++ (link:{java9-javadoc}/java/lang/Character.html#charValue%2D%2D[java 9])
+* ++[[painless-api-reference-Character-compareTo-1]]int link:{java8-javadoc}/java/lang/Character.html#compareTo%2Djava.lang.Character%2D[compareTo](<<painless-api-reference-Character,Character>>)++ (link:{java9-javadoc}/java/lang/Character.html#compareTo%2Djava.lang.Character%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/CharacterIterator.asciidoc b/docs/painless/painless-api-reference/CharacterIterator.asciidoc
new file mode 100644
index 0000000000000..1303e2d47396a
--- /dev/null
+++ b/docs/painless/painless-api-reference/CharacterIterator.asciidoc
@@ -0,0 +1,18 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-CharacterIterator]]++CharacterIterator++::
+** [[painless-api-reference-CharacterIterator-DONE]]static char link:{java8-javadoc}/java/text/CharacterIterator.html#DONE[DONE] (link:{java9-javadoc}/java/text/CharacterIterator.html#DONE[java 9])
+* ++[[painless-api-reference-CharacterIterator-clone-0]]def link:{java8-javadoc}/java/text/CharacterIterator.html#clone%2D%2D[clone]()++ (link:{java9-javadoc}/java/text/CharacterIterator.html#clone%2D%2D[java 9])
+* ++[[painless-api-reference-CharacterIterator-current-0]]char link:{java8-javadoc}/java/text/CharacterIterator.html#current%2D%2D[current]()++ (link:{java9-javadoc}/java/text/CharacterIterator.html#current%2D%2D[java 9])
+* ++[[painless-api-reference-CharacterIterator-first-0]]char link:{java8-javadoc}/java/text/CharacterIterator.html#first%2D%2D[first]()++ (link:{java9-javadoc}/java/text/CharacterIterator.html#first%2D%2D[java 9])
+* ++[[painless-api-reference-CharacterIterator-getBeginIndex-0]]int link:{java8-javadoc}/java/text/CharacterIterator.html#getBeginIndex%2D%2D[getBeginIndex]()++ (link:{java9-javadoc}/java/text/CharacterIterator.html#getBeginIndex%2D%2D[java 9])
+* ++[[painless-api-reference-CharacterIterator-getEndIndex-0]]int link:{java8-javadoc}/java/text/CharacterIterator.html#getEndIndex%2D%2D[getEndIndex]()++ (link:{java9-javadoc}/java/text/CharacterIterator.html#getEndIndex%2D%2D[java 9])
+* ++[[painless-api-reference-CharacterIterator-getIndex-0]]int link:{java8-javadoc}/java/text/CharacterIterator.html#getIndex%2D%2D[getIndex]()++ (link:{java9-javadoc}/java/text/CharacterIterator.html#getIndex%2D%2D[java 9])
+* ++[[painless-api-reference-CharacterIterator-last-0]]char link:{java8-javadoc}/java/text/CharacterIterator.html#last%2D%2D[last]()++ (link:{java9-javadoc}/java/text/CharacterIterator.html#last%2D%2D[java 9])
+* ++[[painless-api-reference-CharacterIterator-next-0]]char link:{java8-javadoc}/java/text/CharacterIterator.html#next%2D%2D[next]()++ (link:{java9-javadoc}/java/text/CharacterIterator.html#next%2D%2D[java 9])
+* ++[[painless-api-reference-CharacterIterator-previous-0]]char link:{java8-javadoc}/java/text/CharacterIterator.html#previous%2D%2D[previous]()++ (link:{java9-javadoc}/java/text/CharacterIterator.html#previous%2D%2D[java 9])
+* ++[[painless-api-reference-CharacterIterator-setIndex-1]]char link:{java8-javadoc}/java/text/CharacterIterator.html#setIndex%2Dint%2D[setIndex](int)++ (link:{java9-javadoc}/java/text/CharacterIterator.html#setIndex%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/ChoiceFormat.asciidoc b/docs/painless/painless-api-reference/ChoiceFormat.asciidoc
new file mode 100644
index 0000000000000..426120a194f73
--- /dev/null
+++ b/docs/painless/painless-api-reference/ChoiceFormat.asciidoc
@@ -0,0 +1,17 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ChoiceFormat]]++ChoiceFormat++::
+* ++[[painless-api-reference-ChoiceFormat-nextDouble-1]]static double link:{java8-javadoc}/java/text/ChoiceFormat.html#nextDouble%2Ddouble%2D[nextDouble](double)++ (link:{java9-javadoc}/java/text/ChoiceFormat.html#nextDouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-ChoiceFormat-nextDouble-2]]static double link:{java8-javadoc}/java/text/ChoiceFormat.html#nextDouble%2Ddouble%2Dboolean%2D[nextDouble](double, boolean)++ (link:{java9-javadoc}/java/text/ChoiceFormat.html#nextDouble%2Ddouble%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-ChoiceFormat-previousDouble-1]]static double link:{java8-javadoc}/java/text/ChoiceFormat.html#previousDouble%2Ddouble%2D[previousDouble](double)++ (link:{java9-javadoc}/java/text/ChoiceFormat.html#previousDouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-ChoiceFormat-ChoiceFormat-1]]link:{java8-javadoc}/java/text/ChoiceFormat.html#ChoiceFormat%2Djava.lang.String%2D[ChoiceFormat](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/ChoiceFormat.html#ChoiceFormat%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-ChoiceFormat-ChoiceFormat-2]]link:{java8-javadoc}/java/text/ChoiceFormat.html#ChoiceFormat%2Ddouble:A%2Djava.lang.String:A%2D[ChoiceFormat](double[], <<painless-api-reference-String,String>>[])++ (link:{java9-javadoc}/java/text/ChoiceFormat.html#ChoiceFormat%2Ddouble:A%2Djava.lang.String:A%2D[java 9])
+* ++[[painless-api-reference-ChoiceFormat-applyPattern-1]]void link:{java8-javadoc}/java/text/ChoiceFormat.html#applyPattern%2Djava.lang.String%2D[applyPattern](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/ChoiceFormat.html#applyPattern%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-ChoiceFormat-getFormats-0]]def[] link:{java8-javadoc}/java/text/ChoiceFormat.html#getFormats%2D%2D[getFormats]()++ (link:{java9-javadoc}/java/text/ChoiceFormat.html#getFormats%2D%2D[java 9])
+* ++[[painless-api-reference-ChoiceFormat-getLimits-0]]double[] link:{java8-javadoc}/java/text/ChoiceFormat.html#getLimits%2D%2D[getLimits]()++ (link:{java9-javadoc}/java/text/ChoiceFormat.html#getLimits%2D%2D[java 9])
+* ++[[painless-api-reference-ChoiceFormat-setChoices-2]]void link:{java8-javadoc}/java/text/ChoiceFormat.html#setChoices%2Ddouble:A%2Djava.lang.String:A%2D[setChoices](double[], <<painless-api-reference-String,String>>[])++ (link:{java9-javadoc}/java/text/ChoiceFormat.html#setChoices%2Ddouble:A%2Djava.lang.String:A%2D[java 9])
+* ++[[painless-api-reference-ChoiceFormat-toPattern-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/text/ChoiceFormat.html#toPattern%2D%2D[toPattern]()++ (link:{java9-javadoc}/java/text/ChoiceFormat.html#toPattern%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Format,Format>>++, ++<<painless-api-reference-NumberFormat,NumberFormat>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/ChronoField.asciidoc b/docs/painless/painless-api-reference/ChronoField.asciidoc
new file mode 100644
index 0000000000000..33143a52eaf63
--- /dev/null
+++ b/docs/painless/painless-api-reference/ChronoField.asciidoc
@@ -0,0 +1,41 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ChronoField]]++ChronoField++::
+** [[painless-api-reference-ChronoField-ALIGNED_DAY_OF_WEEK_IN_MONTH]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#ALIGNED_DAY_OF_WEEK_IN_MONTH[ALIGNED_DAY_OF_WEEK_IN_MONTH] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#ALIGNED_DAY_OF_WEEK_IN_MONTH[java 9])
+** [[painless-api-reference-ChronoField-ALIGNED_DAY_OF_WEEK_IN_YEAR]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#ALIGNED_DAY_OF_WEEK_IN_YEAR[ALIGNED_DAY_OF_WEEK_IN_YEAR] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#ALIGNED_DAY_OF_WEEK_IN_YEAR[java 9])
+** [[painless-api-reference-ChronoField-ALIGNED_WEEK_OF_MONTH]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#ALIGNED_WEEK_OF_MONTH[ALIGNED_WEEK_OF_MONTH] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#ALIGNED_WEEK_OF_MONTH[java 9])
+** [[painless-api-reference-ChronoField-ALIGNED_WEEK_OF_YEAR]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#ALIGNED_WEEK_OF_YEAR[ALIGNED_WEEK_OF_YEAR] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#ALIGNED_WEEK_OF_YEAR[java 9])
+** [[painless-api-reference-ChronoField-AMPM_OF_DAY]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#AMPM_OF_DAY[AMPM_OF_DAY] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#AMPM_OF_DAY[java 9])
+** [[painless-api-reference-ChronoField-CLOCK_HOUR_OF_AMPM]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#CLOCK_HOUR_OF_AMPM[CLOCK_HOUR_OF_AMPM] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#CLOCK_HOUR_OF_AMPM[java 9])
+** [[painless-api-reference-ChronoField-CLOCK_HOUR_OF_DAY]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#CLOCK_HOUR_OF_DAY[CLOCK_HOUR_OF_DAY] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#CLOCK_HOUR_OF_DAY[java 9])
+** [[painless-api-reference-ChronoField-DAY_OF_MONTH]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#DAY_OF_MONTH[DAY_OF_MONTH] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#DAY_OF_MONTH[java 9])
+** [[painless-api-reference-ChronoField-DAY_OF_WEEK]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#DAY_OF_WEEK[DAY_OF_WEEK] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#DAY_OF_WEEK[java 9])
+** [[painless-api-reference-ChronoField-DAY_OF_YEAR]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#DAY_OF_YEAR[DAY_OF_YEAR] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#DAY_OF_YEAR[java 9])
+** [[painless-api-reference-ChronoField-EPOCH_DAY]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#EPOCH_DAY[EPOCH_DAY] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#EPOCH_DAY[java 9])
+** [[painless-api-reference-ChronoField-ERA]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#ERA[ERA] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#ERA[java 9])
+** [[painless-api-reference-ChronoField-HOUR_OF_AMPM]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#HOUR_OF_AMPM[HOUR_OF_AMPM] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#HOUR_OF_AMPM[java 9])
+** [[painless-api-reference-ChronoField-HOUR_OF_DAY]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#HOUR_OF_DAY[HOUR_OF_DAY] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#HOUR_OF_DAY[java 9])
+** [[painless-api-reference-ChronoField-INSTANT_SECONDS]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#INSTANT_SECONDS[INSTANT_SECONDS] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#INSTANT_SECONDS[java 9])
+** [[painless-api-reference-ChronoField-MICRO_OF_DAY]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#MICRO_OF_DAY[MICRO_OF_DAY] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#MICRO_OF_DAY[java 9])
+** [[painless-api-reference-ChronoField-MICRO_OF_SECOND]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#MICRO_OF_SECOND[MICRO_OF_SECOND] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#MICRO_OF_SECOND[java 9])
+** [[painless-api-reference-ChronoField-MILLI_OF_DAY]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#MILLI_OF_DAY[MILLI_OF_DAY] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#MILLI_OF_DAY[java 9])
+** [[painless-api-reference-ChronoField-MILLI_OF_SECOND]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#MILLI_OF_SECOND[MILLI_OF_SECOND] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#MILLI_OF_SECOND[java 9])
+** [[painless-api-reference-ChronoField-MINUTE_OF_DAY]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#MINUTE_OF_DAY[MINUTE_OF_DAY] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#MINUTE_OF_DAY[java 9])
+** [[painless-api-reference-ChronoField-MINUTE_OF_HOUR]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#MINUTE_OF_HOUR[MINUTE_OF_HOUR] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#MINUTE_OF_HOUR[java 9])
+** [[painless-api-reference-ChronoField-MONTH_OF_YEAR]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#MONTH_OF_YEAR[MONTH_OF_YEAR] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#MONTH_OF_YEAR[java 9])
+** [[painless-api-reference-ChronoField-NANO_OF_DAY]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#NANO_OF_DAY[NANO_OF_DAY] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#NANO_OF_DAY[java 9])
+** [[painless-api-reference-ChronoField-NANO_OF_SECOND]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#NANO_OF_SECOND[NANO_OF_SECOND] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#NANO_OF_SECOND[java 9])
+** [[painless-api-reference-ChronoField-OFFSET_SECONDS]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#OFFSET_SECONDS[OFFSET_SECONDS] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#OFFSET_SECONDS[java 9])
+** [[painless-api-reference-ChronoField-PROLEPTIC_MONTH]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#PROLEPTIC_MONTH[PROLEPTIC_MONTH] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#PROLEPTIC_MONTH[java 9])
+** [[painless-api-reference-ChronoField-SECOND_OF_DAY]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#SECOND_OF_DAY[SECOND_OF_DAY] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#SECOND_OF_DAY[java 9])
+** [[painless-api-reference-ChronoField-SECOND_OF_MINUTE]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#SECOND_OF_MINUTE[SECOND_OF_MINUTE] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#SECOND_OF_MINUTE[java 9])
+** [[painless-api-reference-ChronoField-YEAR]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#YEAR[YEAR] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#YEAR[java 9])
+** [[painless-api-reference-ChronoField-YEAR_OF_ERA]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#YEAR_OF_ERA[YEAR_OF_ERA] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#YEAR_OF_ERA[java 9])
+* ++[[painless-api-reference-ChronoField-valueOf-1]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#valueOf%2Djava.lang.String%2D[valueOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/temporal/ChronoField.html#valueOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-ChronoField-values-0]]static <<painless-api-reference-ChronoField,ChronoField>>[] link:{java8-javadoc}/java/time/temporal/ChronoField.html#values%2D%2D[values]()++ (link:{java9-javadoc}/java/time/temporal/ChronoField.html#values%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoField-checkValidIntValue-1]]int link:{java8-javadoc}/java/time/temporal/ChronoField.html#checkValidIntValue%2Dlong%2D[checkValidIntValue](long)++ (link:{java9-javadoc}/java/time/temporal/ChronoField.html#checkValidIntValue%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ChronoField-checkValidValue-1]]long link:{java8-javadoc}/java/time/temporal/ChronoField.html#checkValidValue%2Dlong%2D[checkValidValue](long)++ (link:{java9-javadoc}/java/time/temporal/ChronoField.html#checkValidValue%2Dlong%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Enum,Enum>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-TemporalField,TemporalField>>++
diff --git a/docs/painless/painless-api-reference/ChronoLocalDate.asciidoc b/docs/painless/painless-api-reference/ChronoLocalDate.asciidoc
new file mode 100644
index 0000000000000..2c483c27bd8d9
--- /dev/null
+++ b/docs/painless/painless-api-reference/ChronoLocalDate.asciidoc
@@ -0,0 +1,31 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ChronoLocalDate]]++ChronoLocalDate++::
+* ++[[painless-api-reference-ChronoLocalDate-from-1]]static <<painless-api-reference-ChronoLocalDate,ChronoLocalDate>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#from%2Djava.time.temporal.TemporalAccessor%2D[from](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#from%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDate-timeLineOrder-0]]static <<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#timeLineOrder%2D%2D[timeLineOrder]()++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#timeLineOrder%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDate-atTime-1]]<<painless-api-reference-ChronoLocalDateTime,ChronoLocalDateTime>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#atTime%2Djava.time.LocalTime%2D[atTime](<<painless-api-reference-LocalTime,LocalTime>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#atTime%2Djava.time.LocalTime%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDate-compareTo-1]]int link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#compareTo%2Djava.time.chrono.ChronoLocalDate%2D[compareTo](<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#compareTo%2Djava.time.chrono.ChronoLocalDate%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDate-equals-1]]boolean link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#equals%2Djava.lang.Object%2D[equals](<<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#equals%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDate-format-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#format%2Djava.time.format.DateTimeFormatter%2D[format](<<painless-api-reference-DateTimeFormatter,DateTimeFormatter>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#format%2Djava.time.format.DateTimeFormatter%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDate-getChronology-0]]<<painless-api-reference-Chronology,Chronology>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#getChronology%2D%2D[getChronology]()++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#getChronology%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDate-getEra-0]]<<painless-api-reference-Era,Era>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#getEra%2D%2D[getEra]()++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#getEra%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDate-hashCode-0]]int link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#hashCode%2D%2D[hashCode]()++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#hashCode%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDate-isAfter-1]]boolean link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#isAfter%2Djava.time.chrono.ChronoLocalDate%2D[isAfter](<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#isAfter%2Djava.time.chrono.ChronoLocalDate%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDate-isBefore-1]]boolean link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#isBefore%2Djava.time.chrono.ChronoLocalDate%2D[isBefore](<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#isBefore%2Djava.time.chrono.ChronoLocalDate%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDate-isEqual-1]]boolean link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#isEqual%2Djava.time.chrono.ChronoLocalDate%2D[isEqual](<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#isEqual%2Djava.time.chrono.ChronoLocalDate%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDate-isLeapYear-0]]boolean link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#isLeapYear%2D%2D[isLeapYear]()++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#isLeapYear%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDate-lengthOfMonth-0]]int link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#lengthOfMonth%2D%2D[lengthOfMonth]()++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#lengthOfMonth%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDate-lengthOfYear-0]]int link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#lengthOfYear%2D%2D[lengthOfYear]()++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#lengthOfYear%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDate-minus-1]]<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#minus%2Djava.time.temporal.TemporalAmount%2D[minus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#minus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDate-minus-2]]<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[minus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDate-plus-1]]<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#plus%2Djava.time.temporal.TemporalAmount%2D[plus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#plus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDate-plus-2]]<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[plus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDate-toEpochDay-0]]long link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#toEpochDay%2D%2D[toEpochDay]()++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#toEpochDay%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDate-toString-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#toString%2D%2D[toString]()++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#toString%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDate-until-1]]<<painless-api-reference-ChronoPeriod,ChronoPeriod>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#until%2Djava.time.chrono.ChronoLocalDate%2D[until](<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#until%2Djava.time.chrono.ChronoLocalDate%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDate-with-1]]<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#with%2Djava.time.temporal.TemporalAdjuster%2D[with](<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#with%2Djava.time.temporal.TemporalAdjuster%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDate-with-2]]<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[with](<<painless-api-reference-TemporalField,TemporalField>>, long)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Temporal,Temporal>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++, ++<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>++
diff --git a/docs/painless/painless-api-reference/ChronoLocalDateTime.asciidoc b/docs/painless/painless-api-reference/ChronoLocalDateTime.asciidoc
new file mode 100644
index 0000000000000..7b49712ee0985
--- /dev/null
+++ b/docs/painless/painless-api-reference/ChronoLocalDateTime.asciidoc
@@ -0,0 +1,29 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ChronoLocalDateTime]]++ChronoLocalDateTime++::
+* ++[[painless-api-reference-ChronoLocalDateTime-from-1]]static <<painless-api-reference-ChronoLocalDateTime,ChronoLocalDateTime>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDateTime.html#from%2Djava.time.temporal.TemporalAccessor%2D[from](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDateTime.html#from%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDateTime-timeLineOrder-0]]static <<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDateTime.html#timeLineOrder%2D%2D[timeLineOrder]()++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDateTime.html#timeLineOrder%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDateTime-atZone-1]]<<painless-api-reference-ChronoZonedDateTime,ChronoZonedDateTime>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDateTime.html#atZone%2Djava.time.ZoneId%2D[atZone](<<painless-api-reference-ZoneId,ZoneId>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDateTime.html#atZone%2Djava.time.ZoneId%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDateTime-compareTo-1]]int link:{java8-javadoc}/java/time/chrono/ChronoLocalDateTime.html#compareTo%2Djava.time.chrono.ChronoLocalDateTime%2D[compareTo](<<painless-api-reference-ChronoLocalDateTime,ChronoLocalDateTime>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDateTime.html#compareTo%2Djava.time.chrono.ChronoLocalDateTime%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDateTime-equals-1]]boolean link:{java8-javadoc}/java/time/chrono/ChronoLocalDateTime.html#equals%2Djava.lang.Object%2D[equals](<<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDateTime.html#equals%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDateTime-format-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDateTime.html#format%2Djava.time.format.DateTimeFormatter%2D[format](<<painless-api-reference-DateTimeFormatter,DateTimeFormatter>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDateTime.html#format%2Djava.time.format.DateTimeFormatter%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDateTime-getChronology-0]]<<painless-api-reference-Chronology,Chronology>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDateTime.html#getChronology%2D%2D[getChronology]()++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDateTime.html#getChronology%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDateTime-hashCode-0]]int link:{java8-javadoc}/java/time/chrono/ChronoLocalDateTime.html#hashCode%2D%2D[hashCode]()++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDateTime.html#hashCode%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDateTime-isAfter-1]]boolean link:{java8-javadoc}/java/time/chrono/ChronoLocalDateTime.html#isAfter%2Djava.time.chrono.ChronoLocalDateTime%2D[isAfter](<<painless-api-reference-ChronoLocalDateTime,ChronoLocalDateTime>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDateTime.html#isAfter%2Djava.time.chrono.ChronoLocalDateTime%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDateTime-isBefore-1]]boolean link:{java8-javadoc}/java/time/chrono/ChronoLocalDateTime.html#isBefore%2Djava.time.chrono.ChronoLocalDateTime%2D[isBefore](<<painless-api-reference-ChronoLocalDateTime,ChronoLocalDateTime>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDateTime.html#isBefore%2Djava.time.chrono.ChronoLocalDateTime%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDateTime-isEqual-1]]boolean link:{java8-javadoc}/java/time/chrono/ChronoLocalDateTime.html#isEqual%2Djava.time.chrono.ChronoLocalDateTime%2D[isEqual](<<painless-api-reference-ChronoLocalDateTime,ChronoLocalDateTime>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDateTime.html#isEqual%2Djava.time.chrono.ChronoLocalDateTime%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDateTime-minus-1]]<<painless-api-reference-ChronoLocalDateTime,ChronoLocalDateTime>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDateTime.html#minus%2Djava.time.temporal.TemporalAmount%2D[minus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDateTime.html#minus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDateTime-minus-2]]<<painless-api-reference-ChronoLocalDateTime,ChronoLocalDateTime>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDateTime.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[minus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDateTime.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDateTime-plus-1]]<<painless-api-reference-ChronoLocalDateTime,ChronoLocalDateTime>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDateTime.html#plus%2Djava.time.temporal.TemporalAmount%2D[plus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDateTime.html#plus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDateTime-plus-2]]<<painless-api-reference-ChronoLocalDateTime,ChronoLocalDateTime>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDateTime.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[plus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDateTime.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDateTime-toEpochSecond-1]]long link:{java8-javadoc}/java/time/chrono/ChronoLocalDateTime.html#toEpochSecond%2Djava.time.ZoneOffset%2D[toEpochSecond](<<painless-api-reference-ZoneOffset,ZoneOffset>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDateTime.html#toEpochSecond%2Djava.time.ZoneOffset%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDateTime-toInstant-1]]<<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDateTime.html#toInstant%2Djava.time.ZoneOffset%2D[toInstant](<<painless-api-reference-ZoneOffset,ZoneOffset>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDateTime.html#toInstant%2Djava.time.ZoneOffset%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDateTime-toLocalDate-0]]<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDateTime.html#toLocalDate%2D%2D[toLocalDate]()++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDateTime.html#toLocalDate%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDateTime-toLocalTime-0]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDateTime.html#toLocalTime%2D%2D[toLocalTime]()++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDateTime.html#toLocalTime%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDateTime-toString-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDateTime.html#toString%2D%2D[toString]()++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDateTime.html#toString%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDateTime-with-1]]<<painless-api-reference-ChronoLocalDateTime,ChronoLocalDateTime>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDateTime.html#with%2Djava.time.temporal.TemporalAdjuster%2D[with](<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDateTime.html#with%2Djava.time.temporal.TemporalAdjuster%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDateTime-with-2]]<<painless-api-reference-ChronoLocalDateTime,ChronoLocalDateTime>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDateTime.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[with](<<painless-api-reference-TemporalField,TemporalField>>, long)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDateTime.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Temporal,Temporal>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++, ++<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>++
diff --git a/docs/painless/painless-api-reference/ChronoPeriod.asciidoc b/docs/painless/painless-api-reference/ChronoPeriod.asciidoc
new file mode 100644
index 0000000000000..1b15df31c7667
--- /dev/null
+++ b/docs/painless/painless-api-reference/ChronoPeriod.asciidoc
@@ -0,0 +1,20 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ChronoPeriod]]++ChronoPeriod++::
+* ++[[painless-api-reference-ChronoPeriod-between-2]]static <<painless-api-reference-ChronoPeriod,ChronoPeriod>> link:{java8-javadoc}/java/time/chrono/ChronoPeriod.html#between%2Djava.time.chrono.ChronoLocalDate%2Djava.time.chrono.ChronoLocalDate%2D[between](<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>>, <<painless-api-reference-ChronoLocalDate,ChronoLocalDate>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoPeriod.html#between%2Djava.time.chrono.ChronoLocalDate%2Djava.time.chrono.ChronoLocalDate%2D[java 9])
+* ++[[painless-api-reference-ChronoPeriod-equals-1]]boolean link:{java8-javadoc}/java/time/chrono/ChronoPeriod.html#equals%2Djava.lang.Object%2D[equals](<<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoPeriod.html#equals%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-ChronoPeriod-getChronology-0]]<<painless-api-reference-Chronology,Chronology>> link:{java8-javadoc}/java/time/chrono/ChronoPeriod.html#getChronology%2D%2D[getChronology]()++ (link:{java9-javadoc}/java/time/chrono/ChronoPeriod.html#getChronology%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoPeriod-getUnits-0]]<<painless-api-reference-List,List>> link:{java8-javadoc}/java/time/chrono/ChronoPeriod.html#getUnits%2D%2D[getUnits]()++ (link:{java9-javadoc}/java/time/chrono/ChronoPeriod.html#getUnits%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoPeriod-hashCode-0]]int link:{java8-javadoc}/java/time/chrono/ChronoPeriod.html#hashCode%2D%2D[hashCode]()++ (link:{java9-javadoc}/java/time/chrono/ChronoPeriod.html#hashCode%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoPeriod-isNegative-0]]boolean link:{java8-javadoc}/java/time/chrono/ChronoPeriod.html#isNegative%2D%2D[isNegative]()++ (link:{java9-javadoc}/java/time/chrono/ChronoPeriod.html#isNegative%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoPeriod-isZero-0]]boolean link:{java8-javadoc}/java/time/chrono/ChronoPeriod.html#isZero%2D%2D[isZero]()++ (link:{java9-javadoc}/java/time/chrono/ChronoPeriod.html#isZero%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoPeriod-minus-1]]<<painless-api-reference-ChronoPeriod,ChronoPeriod>> link:{java8-javadoc}/java/time/chrono/ChronoPeriod.html#minus%2Djava.time.temporal.TemporalAmount%2D[minus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoPeriod.html#minus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-ChronoPeriod-multipliedBy-1]]<<painless-api-reference-ChronoPeriod,ChronoPeriod>> link:{java8-javadoc}/java/time/chrono/ChronoPeriod.html#multipliedBy%2Dint%2D[multipliedBy](int)++ (link:{java9-javadoc}/java/time/chrono/ChronoPeriod.html#multipliedBy%2Dint%2D[java 9])
+* ++[[painless-api-reference-ChronoPeriod-negated-0]]<<painless-api-reference-ChronoPeriod,ChronoPeriod>> link:{java8-javadoc}/java/time/chrono/ChronoPeriod.html#negated%2D%2D[negated]()++ (link:{java9-javadoc}/java/time/chrono/ChronoPeriod.html#negated%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoPeriod-normalized-0]]<<painless-api-reference-ChronoPeriod,ChronoPeriod>> link:{java8-javadoc}/java/time/chrono/ChronoPeriod.html#normalized%2D%2D[normalized]()++ (link:{java9-javadoc}/java/time/chrono/ChronoPeriod.html#normalized%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoPeriod-plus-1]]<<painless-api-reference-ChronoPeriod,ChronoPeriod>> link:{java8-javadoc}/java/time/chrono/ChronoPeriod.html#plus%2Djava.time.temporal.TemporalAmount%2D[plus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoPeriod.html#plus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-ChronoPeriod-toString-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/chrono/ChronoPeriod.html#toString%2D%2D[toString]()++ (link:{java9-javadoc}/java/time/chrono/ChronoPeriod.html#toString%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-TemporalAmount,TemporalAmount>>++
diff --git a/docs/painless/painless-api-reference/ChronoUnit.asciidoc b/docs/painless/painless-api-reference/ChronoUnit.asciidoc
new file mode 100644
index 0000000000000..297fbf61835f9
--- /dev/null
+++ b/docs/painless/painless-api-reference/ChronoUnit.asciidoc
@@ -0,0 +1,25 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ChronoUnit]]++ChronoUnit++::
+** [[painless-api-reference-ChronoUnit-CENTURIES]]static <<painless-api-reference-ChronoUnit,ChronoUnit>> link:{java8-javadoc}/java/time/temporal/ChronoUnit.html#CENTURIES[CENTURIES] (link:{java9-javadoc}/java/time/temporal/ChronoUnit.html#CENTURIES[java 9])
+** [[painless-api-reference-ChronoUnit-DAYS]]static <<painless-api-reference-ChronoUnit,ChronoUnit>> link:{java8-javadoc}/java/time/temporal/ChronoUnit.html#DAYS[DAYS] (link:{java9-javadoc}/java/time/temporal/ChronoUnit.html#DAYS[java 9])
+** [[painless-api-reference-ChronoUnit-DECADES]]static <<painless-api-reference-ChronoUnit,ChronoUnit>> link:{java8-javadoc}/java/time/temporal/ChronoUnit.html#DECADES[DECADES] (link:{java9-javadoc}/java/time/temporal/ChronoUnit.html#DECADES[java 9])
+** [[painless-api-reference-ChronoUnit-ERAS]]static <<painless-api-reference-ChronoUnit,ChronoUnit>> link:{java8-javadoc}/java/time/temporal/ChronoUnit.html#ERAS[ERAS] (link:{java9-javadoc}/java/time/temporal/ChronoUnit.html#ERAS[java 9])
+** [[painless-api-reference-ChronoUnit-FOREVER]]static <<painless-api-reference-ChronoUnit,ChronoUnit>> link:{java8-javadoc}/java/time/temporal/ChronoUnit.html#FOREVER[FOREVER] (link:{java9-javadoc}/java/time/temporal/ChronoUnit.html#FOREVER[java 9])
+** [[painless-api-reference-ChronoUnit-HALF_DAYS]]static <<painless-api-reference-ChronoUnit,ChronoUnit>> link:{java8-javadoc}/java/time/temporal/ChronoUnit.html#HALF_DAYS[HALF_DAYS] (link:{java9-javadoc}/java/time/temporal/ChronoUnit.html#HALF_DAYS[java 9])
+** [[painless-api-reference-ChronoUnit-HOURS]]static <<painless-api-reference-ChronoUnit,ChronoUnit>> link:{java8-javadoc}/java/time/temporal/ChronoUnit.html#HOURS[HOURS] (link:{java9-javadoc}/java/time/temporal/ChronoUnit.html#HOURS[java 9])
+** [[painless-api-reference-ChronoUnit-MICROS]]static <<painless-api-reference-ChronoUnit,ChronoUnit>> link:{java8-javadoc}/java/time/temporal/ChronoUnit.html#MICROS[MICROS] (link:{java9-javadoc}/java/time/temporal/ChronoUnit.html#MICROS[java 9])
+** [[painless-api-reference-ChronoUnit-MILLENNIA]]static <<painless-api-reference-ChronoUnit,ChronoUnit>> link:{java8-javadoc}/java/time/temporal/ChronoUnit.html#MILLENNIA[MILLENNIA] (link:{java9-javadoc}/java/time/temporal/ChronoUnit.html#MILLENNIA[java 9])
+** [[painless-api-reference-ChronoUnit-MILLIS]]static <<painless-api-reference-ChronoUnit,ChronoUnit>> link:{java8-javadoc}/java/time/temporal/ChronoUnit.html#MILLIS[MILLIS] (link:{java9-javadoc}/java/time/temporal/ChronoUnit.html#MILLIS[java 9])
+** [[painless-api-reference-ChronoUnit-MINUTES]]static <<painless-api-reference-ChronoUnit,ChronoUnit>> link:{java8-javadoc}/java/time/temporal/ChronoUnit.html#MINUTES[MINUTES] (link:{java9-javadoc}/java/time/temporal/ChronoUnit.html#MINUTES[java 9])
+** [[painless-api-reference-ChronoUnit-MONTHS]]static <<painless-api-reference-ChronoUnit,ChronoUnit>> link:{java8-javadoc}/java/time/temporal/ChronoUnit.html#MONTHS[MONTHS] (link:{java9-javadoc}/java/time/temporal/ChronoUnit.html#MONTHS[java 9])
+** [[painless-api-reference-ChronoUnit-NANOS]]static <<painless-api-reference-ChronoUnit,ChronoUnit>> link:{java8-javadoc}/java/time/temporal/ChronoUnit.html#NANOS[NANOS] (link:{java9-javadoc}/java/time/temporal/ChronoUnit.html#NANOS[java 9])
+** [[painless-api-reference-ChronoUnit-SECONDS]]static <<painless-api-reference-ChronoUnit,ChronoUnit>> link:{java8-javadoc}/java/time/temporal/ChronoUnit.html#SECONDS[SECONDS] (link:{java9-javadoc}/java/time/temporal/ChronoUnit.html#SECONDS[java 9])
+** [[painless-api-reference-ChronoUnit-WEEKS]]static <<painless-api-reference-ChronoUnit,ChronoUnit>> link:{java8-javadoc}/java/time/temporal/ChronoUnit.html#WEEKS[WEEKS] (link:{java9-javadoc}/java/time/temporal/ChronoUnit.html#WEEKS[java 9])
+** [[painless-api-reference-ChronoUnit-YEARS]]static <<painless-api-reference-ChronoUnit,ChronoUnit>> link:{java8-javadoc}/java/time/temporal/ChronoUnit.html#YEARS[YEARS] (link:{java9-javadoc}/java/time/temporal/ChronoUnit.html#YEARS[java 9])
+* ++[[painless-api-reference-ChronoUnit-valueOf-1]]static <<painless-api-reference-ChronoUnit,ChronoUnit>> link:{java8-javadoc}/java/time/temporal/ChronoUnit.html#valueOf%2Djava.lang.String%2D[valueOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/temporal/ChronoUnit.html#valueOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-ChronoUnit-values-0]]static <<painless-api-reference-ChronoUnit,ChronoUnit>>[] link:{java8-javadoc}/java/time/temporal/ChronoUnit.html#values%2D%2D[values]()++ (link:{java9-javadoc}/java/time/temporal/ChronoUnit.html#values%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Enum,Enum>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-TemporalUnit,TemporalUnit>>++
diff --git a/docs/painless/painless-api-reference/ChronoZonedDateTime.asciidoc b/docs/painless/painless-api-reference/ChronoZonedDateTime.asciidoc
new file mode 100644
index 0000000000000..0684b8b541e02
--- /dev/null
+++ b/docs/painless/painless-api-reference/ChronoZonedDateTime.asciidoc
@@ -0,0 +1,35 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ChronoZonedDateTime]]++ChronoZonedDateTime++::
+* ++[[painless-api-reference-ChronoZonedDateTime-from-1]]static <<painless-api-reference-ChronoZonedDateTime,ChronoZonedDateTime>> link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#from%2Djava.time.temporal.TemporalAccessor%2D[from](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#from%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-timeLineOrder-0]]static <<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#timeLineOrder%2D%2D[timeLineOrder]()++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#timeLineOrder%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-compareTo-1]]int link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#compareTo%2Djava.time.chrono.ChronoZonedDateTime%2D[compareTo](<<painless-api-reference-ChronoZonedDateTime,ChronoZonedDateTime>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#compareTo%2Djava.time.chrono.ChronoZonedDateTime%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-equals-1]]boolean link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#equals%2Djava.lang.Object%2D[equals](<<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#equals%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-format-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#format%2Djava.time.format.DateTimeFormatter%2D[format](<<painless-api-reference-DateTimeFormatter,DateTimeFormatter>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#format%2Djava.time.format.DateTimeFormatter%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-getChronology-0]]<<painless-api-reference-Chronology,Chronology>> link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#getChronology%2D%2D[getChronology]()++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#getChronology%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-getOffset-0]]<<painless-api-reference-ZoneOffset,ZoneOffset>> link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#getOffset%2D%2D[getOffset]()++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#getOffset%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-getZone-0]]<<painless-api-reference-ZoneId,ZoneId>> link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#getZone%2D%2D[getZone]()++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#getZone%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-hashCode-0]]int link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#hashCode%2D%2D[hashCode]()++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#hashCode%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-isAfter-1]]boolean link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#isAfter%2Djava.time.chrono.ChronoZonedDateTime%2D[isAfter](<<painless-api-reference-ChronoZonedDateTime,ChronoZonedDateTime>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#isAfter%2Djava.time.chrono.ChronoZonedDateTime%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-isBefore-1]]boolean link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#isBefore%2Djava.time.chrono.ChronoZonedDateTime%2D[isBefore](<<painless-api-reference-ChronoZonedDateTime,ChronoZonedDateTime>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#isBefore%2Djava.time.chrono.ChronoZonedDateTime%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-isEqual-1]]boolean link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#isEqual%2Djava.time.chrono.ChronoZonedDateTime%2D[isEqual](<<painless-api-reference-ChronoZonedDateTime,ChronoZonedDateTime>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#isEqual%2Djava.time.chrono.ChronoZonedDateTime%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-minus-1]]<<painless-api-reference-ChronoZonedDateTime,ChronoZonedDateTime>> link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#minus%2Djava.time.temporal.TemporalAmount%2D[minus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#minus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-minus-2]]<<painless-api-reference-ChronoZonedDateTime,ChronoZonedDateTime>> link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[minus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-plus-1]]<<painless-api-reference-ChronoZonedDateTime,ChronoZonedDateTime>> link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#plus%2Djava.time.temporal.TemporalAmount%2D[plus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#plus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-plus-2]]<<painless-api-reference-ChronoZonedDateTime,ChronoZonedDateTime>> link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[plus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-toEpochSecond-0]]long link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#toEpochSecond%2D%2D[toEpochSecond]()++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#toEpochSecond%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-toInstant-0]]<<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#toInstant%2D%2D[toInstant]()++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#toInstant%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-toLocalDate-0]]<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>> link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#toLocalDate%2D%2D[toLocalDate]()++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#toLocalDate%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-toLocalDateTime-0]]<<painless-api-reference-ChronoLocalDateTime,ChronoLocalDateTime>> link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#toLocalDateTime%2D%2D[toLocalDateTime]()++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#toLocalDateTime%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-toLocalTime-0]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#toLocalTime%2D%2D[toLocalTime]()++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#toLocalTime%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-toString-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#toString%2D%2D[toString]()++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#toString%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-with-1]]<<painless-api-reference-ChronoZonedDateTime,ChronoZonedDateTime>> link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#with%2Djava.time.temporal.TemporalAdjuster%2D[with](<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#with%2Djava.time.temporal.TemporalAdjuster%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-with-2]]<<painless-api-reference-ChronoZonedDateTime,ChronoZonedDateTime>> link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[with](<<painless-api-reference-TemporalField,TemporalField>>, long)++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-withEarlierOffsetAtOverlap-0]]<<painless-api-reference-ChronoZonedDateTime,ChronoZonedDateTime>> link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#withEarlierOffsetAtOverlap%2D%2D[withEarlierOffsetAtOverlap]()++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#withEarlierOffsetAtOverlap%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-withLaterOffsetAtOverlap-0]]<<painless-api-reference-ChronoZonedDateTime,ChronoZonedDateTime>> link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#withLaterOffsetAtOverlap%2D%2D[withLaterOffsetAtOverlap]()++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#withLaterOffsetAtOverlap%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-withZoneSameInstant-1]]<<painless-api-reference-ChronoZonedDateTime,ChronoZonedDateTime>> link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#withZoneSameInstant%2Djava.time.ZoneId%2D[withZoneSameInstant](<<painless-api-reference-ZoneId,ZoneId>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#withZoneSameInstant%2Djava.time.ZoneId%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-withZoneSameLocal-1]]<<painless-api-reference-ChronoZonedDateTime,ChronoZonedDateTime>> link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#withZoneSameLocal%2Djava.time.ZoneId%2D[withZoneSameLocal](<<painless-api-reference-ZoneId,ZoneId>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#withZoneSameLocal%2Djava.time.ZoneId%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Temporal,Temporal>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++
diff --git a/docs/painless/painless-api-reference/Chronology.asciidoc b/docs/painless/painless-api-reference/Chronology.asciidoc
new file mode 100644
index 0000000000000..42b395d930a40
--- /dev/null
+++ b/docs/painless/painless-api-reference/Chronology.asciidoc
@@ -0,0 +1,33 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Chronology]]++Chronology++::
+* ++[[painless-api-reference-Chronology-from-1]]static <<painless-api-reference-Chronology,Chronology>> link:{java8-javadoc}/java/time/chrono/Chronology.html#from%2Djava.time.temporal.TemporalAccessor%2D[from](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#from%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-Chronology-getAvailableChronologies-0]]static <<painless-api-reference-Set,Set>> link:{java8-javadoc}/java/time/chrono/Chronology.html#getAvailableChronologies%2D%2D[getAvailableChronologies]()++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#getAvailableChronologies%2D%2D[java 9])
+* ++[[painless-api-reference-Chronology-of-1]]static <<painless-api-reference-Chronology,Chronology>> link:{java8-javadoc}/java/time/chrono/Chronology.html#of%2Djava.lang.String%2D[of](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#of%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Chronology-ofLocale-1]]static <<painless-api-reference-Chronology,Chronology>> link:{java8-javadoc}/java/time/chrono/Chronology.html#ofLocale%2Djava.util.Locale%2D[ofLocale](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#ofLocale%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-Chronology-compareTo-1]]int link:{java8-javadoc}/java/time/chrono/Chronology.html#compareTo%2Djava.time.chrono.Chronology%2D[compareTo](<<painless-api-reference-Chronology,Chronology>>)++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#compareTo%2Djava.time.chrono.Chronology%2D[java 9])
+* ++[[painless-api-reference-Chronology-date-1]]<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>> link:{java8-javadoc}/java/time/chrono/Chronology.html#date%2Djava.time.temporal.TemporalAccessor%2D[date](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#date%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-Chronology-date-3]]<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>> link:{java8-javadoc}/java/time/chrono/Chronology.html#date%2Dint%2Dint%2Dint%2D[date](int, int, int)++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#date%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Chronology-date-4]]<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>> link:{java8-javadoc}/java/time/chrono/Chronology.html#date%2Djava.time.chrono.Era%2Dint%2Dint%2Dint%2D[date](<<painless-api-reference-Era,Era>>, int, int, int)++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#date%2Djava.time.chrono.Era%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Chronology-dateEpochDay-1]]<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>> link:{java8-javadoc}/java/time/chrono/Chronology.html#dateEpochDay%2Dlong%2D[dateEpochDay](long)++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#dateEpochDay%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Chronology-dateYearDay-2]]<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>> link:{java8-javadoc}/java/time/chrono/Chronology.html#dateYearDay%2Dint%2Dint%2D[dateYearDay](int, int)++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#dateYearDay%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Chronology-dateYearDay-3]]<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>> link:{java8-javadoc}/java/time/chrono/Chronology.html#dateYearDay%2Djava.time.chrono.Era%2Dint%2Dint%2D[dateYearDay](<<painless-api-reference-Era,Era>>, int, int)++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#dateYearDay%2Djava.time.chrono.Era%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Chronology-equals-1]]boolean link:{java8-javadoc}/java/time/chrono/Chronology.html#equals%2Djava.lang.Object%2D[equals](<<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#equals%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Chronology-eraOf-1]]<<painless-api-reference-Era,Era>> link:{java8-javadoc}/java/time/chrono/Chronology.html#eraOf%2Dint%2D[eraOf](int)++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#eraOf%2Dint%2D[java 9])
+* ++[[painless-api-reference-Chronology-eras-0]]<<painless-api-reference-List,List>> link:{java8-javadoc}/java/time/chrono/Chronology.html#eras%2D%2D[eras]()++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#eras%2D%2D[java 9])
+* ++[[painless-api-reference-Chronology-getCalendarType-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/chrono/Chronology.html#getCalendarType%2D%2D[getCalendarType]()++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#getCalendarType%2D%2D[java 9])
+* ++[[painless-api-reference-Chronology-getDisplayName-2]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/chrono/Chronology.html#getDisplayName%2Djava.time.format.TextStyle%2Djava.util.Locale%2D[getDisplayName](<<painless-api-reference-TextStyle,TextStyle>>, <<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#getDisplayName%2Djava.time.format.TextStyle%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-Chronology-getId-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/chrono/Chronology.html#getId%2D%2D[getId]()++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#getId%2D%2D[java 9])
+* ++[[painless-api-reference-Chronology-hashCode-0]]int link:{java8-javadoc}/java/time/chrono/Chronology.html#hashCode%2D%2D[hashCode]()++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#hashCode%2D%2D[java 9])
+* ++[[painless-api-reference-Chronology-isLeapYear-1]]boolean link:{java8-javadoc}/java/time/chrono/Chronology.html#isLeapYear%2Dlong%2D[isLeapYear](long)++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#isLeapYear%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Chronology-localDateTime-1]]<<painless-api-reference-ChronoLocalDateTime,ChronoLocalDateTime>> link:{java8-javadoc}/java/time/chrono/Chronology.html#localDateTime%2Djava.time.temporal.TemporalAccessor%2D[localDateTime](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#localDateTime%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-Chronology-period-3]]<<painless-api-reference-ChronoPeriod,ChronoPeriod>> link:{java8-javadoc}/java/time/chrono/Chronology.html#period%2Dint%2Dint%2Dint%2D[period](int, int, int)++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#period%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Chronology-prolepticYear-2]]int link:{java8-javadoc}/java/time/chrono/Chronology.html#prolepticYear%2Djava.time.chrono.Era%2Dint%2D[prolepticYear](<<painless-api-reference-Era,Era>>, int)++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#prolepticYear%2Djava.time.chrono.Era%2Dint%2D[java 9])
+* ++[[painless-api-reference-Chronology-range-1]]<<painless-api-reference-ValueRange,ValueRange>> link:{java8-javadoc}/java/time/chrono/Chronology.html#range%2Djava.time.temporal.ChronoField%2D[range](<<painless-api-reference-ChronoField,ChronoField>>)++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#range%2Djava.time.temporal.ChronoField%2D[java 9])
+* ++[[painless-api-reference-Chronology-resolveDate-2]]<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>> link:{java8-javadoc}/java/time/chrono/Chronology.html#resolveDate%2Djava.util.Map%2Djava.time.format.ResolverStyle%2D[resolveDate](<<painless-api-reference-Map,Map>>, <<painless-api-reference-ResolverStyle,ResolverStyle>>)++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#resolveDate%2Djava.util.Map%2Djava.time.format.ResolverStyle%2D[java 9])
+* ++[[painless-api-reference-Chronology-toString-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/chrono/Chronology.html#toString%2D%2D[toString]()++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#toString%2D%2D[java 9])
+* ++[[painless-api-reference-Chronology-zonedDateTime-1]]<<painless-api-reference-ChronoZonedDateTime,ChronoZonedDateTime>> link:{java8-javadoc}/java/time/chrono/Chronology.html#zonedDateTime%2Djava.time.temporal.TemporalAccessor%2D[zonedDateTime](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#zonedDateTime%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-Chronology-zonedDateTime-2]]<<painless-api-reference-ChronoZonedDateTime,ChronoZonedDateTime>> link:{java8-javadoc}/java/time/chrono/Chronology.html#zonedDateTime%2Djava.time.Instant%2Djava.time.ZoneId%2D[zonedDateTime](<<painless-api-reference-Instant,Instant>>, <<painless-api-reference-ZoneId,ZoneId>>)++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#zonedDateTime%2Djava.time.Instant%2Djava.time.ZoneId%2D[java 9])
diff --git a/docs/painless/painless-api-reference/ClassCastException.asciidoc b/docs/painless/painless-api-reference/ClassCastException.asciidoc
new file mode 100644
index 0000000000000..f837c66f9db11
--- /dev/null
+++ b/docs/painless/painless-api-reference/ClassCastException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ClassCastException]]++ClassCastException++::
+* ++[[painless-api-reference-ClassCastException-ClassCastException-0]]link:{java8-javadoc}/java/lang/ClassCastException.html#ClassCastException%2D%2D[ClassCastException]()++ (link:{java9-javadoc}/java/lang/ClassCastException.html#ClassCastException%2D%2D[java 9])
+* ++[[painless-api-reference-ClassCastException-ClassCastException-1]]link:{java8-javadoc}/java/lang/ClassCastException.html#ClassCastException%2Djava.lang.String%2D[ClassCastException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/ClassCastException.html#ClassCastException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/ClassNotFoundException.asciidoc b/docs/painless/painless-api-reference/ClassNotFoundException.asciidoc
new file mode 100644
index 0000000000000..f4d0d63aa0605
--- /dev/null
+++ b/docs/painless/painless-api-reference/ClassNotFoundException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ClassNotFoundException]]++ClassNotFoundException++::
+* ++[[painless-api-reference-ClassNotFoundException-ClassNotFoundException-0]]link:{java8-javadoc}/java/lang/ClassNotFoundException.html#ClassNotFoundException%2D%2D[ClassNotFoundException]()++ (link:{java9-javadoc}/java/lang/ClassNotFoundException.html#ClassNotFoundException%2D%2D[java 9])
+* ++[[painless-api-reference-ClassNotFoundException-ClassNotFoundException-1]]link:{java8-javadoc}/java/lang/ClassNotFoundException.html#ClassNotFoundException%2Djava.lang.String%2D[ClassNotFoundException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/ClassNotFoundException.html#ClassNotFoundException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Clock.asciidoc b/docs/painless/painless-api-reference/Clock.asciidoc
new file mode 100644
index 0000000000000..c40a09a2fe92a
--- /dev/null
+++ b/docs/painless/painless-api-reference/Clock.asciidoc
@@ -0,0 +1,13 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Clock]]++Clock++::
+* ++[[painless-api-reference-Clock-fixed-2]]static <<painless-api-reference-Clock,Clock>> link:{java8-javadoc}/java/time/Clock.html#fixed%2Djava.time.Instant%2Djava.time.ZoneId%2D[fixed](<<painless-api-reference-Instant,Instant>>, <<painless-api-reference-ZoneId,ZoneId>>)++ (link:{java9-javadoc}/java/time/Clock.html#fixed%2Djava.time.Instant%2Djava.time.ZoneId%2D[java 9])
+* ++[[painless-api-reference-Clock-offset-2]]static <<painless-api-reference-Clock,Clock>> link:{java8-javadoc}/java/time/Clock.html#offset%2Djava.time.Clock%2Djava.time.Duration%2D[offset](<<painless-api-reference-Clock,Clock>>, <<painless-api-reference-Duration,Duration>>)++ (link:{java9-javadoc}/java/time/Clock.html#offset%2Djava.time.Clock%2Djava.time.Duration%2D[java 9])
+* ++[[painless-api-reference-Clock-tick-2]]static <<painless-api-reference-Clock,Clock>> link:{java8-javadoc}/java/time/Clock.html#tick%2Djava.time.Clock%2Djava.time.Duration%2D[tick](<<painless-api-reference-Clock,Clock>>, <<painless-api-reference-Duration,Duration>>)++ (link:{java9-javadoc}/java/time/Clock.html#tick%2Djava.time.Clock%2Djava.time.Duration%2D[java 9])
+* ++[[painless-api-reference-Clock-getZone-0]]<<painless-api-reference-ZoneId,ZoneId>> link:{java8-javadoc}/java/time/Clock.html#getZone%2D%2D[getZone]()++ (link:{java9-javadoc}/java/time/Clock.html#getZone%2D%2D[java 9])
+* ++[[painless-api-reference-Clock-instant-0]]<<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/Clock.html#instant%2D%2D[instant]()++ (link:{java9-javadoc}/java/time/Clock.html#instant%2D%2D[java 9])
+* ++[[painless-api-reference-Clock-millis-0]]long link:{java8-javadoc}/java/time/Clock.html#millis%2D%2D[millis]()++ (link:{java9-javadoc}/java/time/Clock.html#millis%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/CloneNotSupportedException.asciidoc b/docs/painless/painless-api-reference/CloneNotSupportedException.asciidoc
new file mode 100644
index 0000000000000..deeb87d599b5c
--- /dev/null
+++ b/docs/painless/painless-api-reference/CloneNotSupportedException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-CloneNotSupportedException]]++CloneNotSupportedException++::
+* ++[[painless-api-reference-CloneNotSupportedException-CloneNotSupportedException-0]]link:{java8-javadoc}/java/lang/CloneNotSupportedException.html#CloneNotSupportedException%2D%2D[CloneNotSupportedException]()++ (link:{java9-javadoc}/java/lang/CloneNotSupportedException.html#CloneNotSupportedException%2D%2D[java 9])
+* ++[[painless-api-reference-CloneNotSupportedException-CloneNotSupportedException-1]]link:{java8-javadoc}/java/lang/CloneNotSupportedException.html#CloneNotSupportedException%2Djava.lang.String%2D[CloneNotSupportedException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/CloneNotSupportedException.html#CloneNotSupportedException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/CollationElementIterator.asciidoc b/docs/painless/painless-api-reference/CollationElementIterator.asciidoc
new file mode 100644
index 0000000000000..f277709123d8a
--- /dev/null
+++ b/docs/painless/painless-api-reference/CollationElementIterator.asciidoc
@@ -0,0 +1,18 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-CollationElementIterator]]++CollationElementIterator++::
+** [[painless-api-reference-CollationElementIterator-NULLORDER]]static int link:{java8-javadoc}/java/text/CollationElementIterator.html#NULLORDER[NULLORDER] (link:{java9-javadoc}/java/text/CollationElementIterator.html#NULLORDER[java 9])
+* ++[[painless-api-reference-CollationElementIterator-primaryOrder-1]]static int link:{java8-javadoc}/java/text/CollationElementIterator.html#primaryOrder%2Dint%2D[primaryOrder](int)++ (link:{java9-javadoc}/java/text/CollationElementIterator.html#primaryOrder%2Dint%2D[java 9])
+* ++[[painless-api-reference-CollationElementIterator-secondaryOrder-1]]static short link:{java8-javadoc}/java/text/CollationElementIterator.html#secondaryOrder%2Dint%2D[secondaryOrder](int)++ (link:{java9-javadoc}/java/text/CollationElementIterator.html#secondaryOrder%2Dint%2D[java 9])
+* ++[[painless-api-reference-CollationElementIterator-tertiaryOrder-1]]static short link:{java8-javadoc}/java/text/CollationElementIterator.html#tertiaryOrder%2Dint%2D[tertiaryOrder](int)++ (link:{java9-javadoc}/java/text/CollationElementIterator.html#tertiaryOrder%2Dint%2D[java 9])
+* ++[[painless-api-reference-CollationElementIterator-getMaxExpansion-1]]int link:{java8-javadoc}/java/text/CollationElementIterator.html#getMaxExpansion%2Dint%2D[getMaxExpansion](int)++ (link:{java9-javadoc}/java/text/CollationElementIterator.html#getMaxExpansion%2Dint%2D[java 9])
+* ++[[painless-api-reference-CollationElementIterator-getOffset-0]]int link:{java8-javadoc}/java/text/CollationElementIterator.html#getOffset%2D%2D[getOffset]()++ (link:{java9-javadoc}/java/text/CollationElementIterator.html#getOffset%2D%2D[java 9])
+* ++[[painless-api-reference-CollationElementIterator-next-0]]int link:{java8-javadoc}/java/text/CollationElementIterator.html#next%2D%2D[next]()++ (link:{java9-javadoc}/java/text/CollationElementIterator.html#next%2D%2D[java 9])
+* ++[[painless-api-reference-CollationElementIterator-previous-0]]int link:{java8-javadoc}/java/text/CollationElementIterator.html#previous%2D%2D[previous]()++ (link:{java9-javadoc}/java/text/CollationElementIterator.html#previous%2D%2D[java 9])
+* ++[[painless-api-reference-CollationElementIterator-reset-0]]void link:{java8-javadoc}/java/text/CollationElementIterator.html#reset%2D%2D[reset]()++ (link:{java9-javadoc}/java/text/CollationElementIterator.html#reset%2D%2D[java 9])
+* ++[[painless-api-reference-CollationElementIterator-setOffset-1]]void link:{java8-javadoc}/java/text/CollationElementIterator.html#setOffset%2Dint%2D[setOffset](int)++ (link:{java9-javadoc}/java/text/CollationElementIterator.html#setOffset%2Dint%2D[java 9])
+* ++[[painless-api-reference-CollationElementIterator-setText-1]]void link:{java8-javadoc}/java/text/CollationElementIterator.html#setText%2Djava.lang.String%2D[setText](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/CollationElementIterator.html#setText%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/CollationKey.asciidoc b/docs/painless/painless-api-reference/CollationKey.asciidoc
new file mode 100644
index 0000000000000..adb6d29cf81bc
--- /dev/null
+++ b/docs/painless/painless-api-reference/CollationKey.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-CollationKey]]++CollationKey++::
+* ++[[painless-api-reference-CollationKey-compareTo-1]]int link:{java8-javadoc}/java/text/CollationKey.html#compareTo%2Djava.text.CollationKey%2D[compareTo](<<painless-api-reference-CollationKey,CollationKey>>)++ (link:{java9-javadoc}/java/text/CollationKey.html#compareTo%2Djava.text.CollationKey%2D[java 9])
+* ++[[painless-api-reference-CollationKey-getSourceString-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/text/CollationKey.html#getSourceString%2D%2D[getSourceString]()++ (link:{java9-javadoc}/java/text/CollationKey.html#getSourceString%2D%2D[java 9])
+* ++[[painless-api-reference-CollationKey-toByteArray-0]]byte[] link:{java8-javadoc}/java/text/CollationKey.html#toByteArray%2D%2D[toByteArray]()++ (link:{java9-javadoc}/java/text/CollationKey.html#toByteArray%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Collator.asciidoc b/docs/painless/painless-api-reference/Collator.asciidoc
new file mode 100644
index 0000000000000..a14f29d986851
--- /dev/null
+++ b/docs/painless/painless-api-reference/Collator.asciidoc
@@ -0,0 +1,24 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Collator]]++Collator++::
+** [[painless-api-reference-Collator-CANONICAL_DECOMPOSITION]]static int link:{java8-javadoc}/java/text/Collator.html#CANONICAL_DECOMPOSITION[CANONICAL_DECOMPOSITION] (link:{java9-javadoc}/java/text/Collator.html#CANONICAL_DECOMPOSITION[java 9])
+** [[painless-api-reference-Collator-FULL_DECOMPOSITION]]static int link:{java8-javadoc}/java/text/Collator.html#FULL_DECOMPOSITION[FULL_DECOMPOSITION] (link:{java9-javadoc}/java/text/Collator.html#FULL_DECOMPOSITION[java 9])
+** [[painless-api-reference-Collator-IDENTICAL]]static int link:{java8-javadoc}/java/text/Collator.html#IDENTICAL[IDENTICAL] (link:{java9-javadoc}/java/text/Collator.html#IDENTICAL[java 9])
+** [[painless-api-reference-Collator-NO_DECOMPOSITION]]static int link:{java8-javadoc}/java/text/Collator.html#NO_DECOMPOSITION[NO_DECOMPOSITION] (link:{java9-javadoc}/java/text/Collator.html#NO_DECOMPOSITION[java 9])
+** [[painless-api-reference-Collator-PRIMARY]]static int link:{java8-javadoc}/java/text/Collator.html#PRIMARY[PRIMARY] (link:{java9-javadoc}/java/text/Collator.html#PRIMARY[java 9])
+** [[painless-api-reference-Collator-SECONDARY]]static int link:{java8-javadoc}/java/text/Collator.html#SECONDARY[SECONDARY] (link:{java9-javadoc}/java/text/Collator.html#SECONDARY[java 9])
+** [[painless-api-reference-Collator-TERTIARY]]static int link:{java8-javadoc}/java/text/Collator.html#TERTIARY[TERTIARY] (link:{java9-javadoc}/java/text/Collator.html#TERTIARY[java 9])
+* ++[[painless-api-reference-Collator-getAvailableLocales-0]]static <<painless-api-reference-Locale,Locale>>[] link:{java8-javadoc}/java/text/Collator.html#getAvailableLocales%2D%2D[getAvailableLocales]()++ (link:{java9-javadoc}/java/text/Collator.html#getAvailableLocales%2D%2D[java 9])
+* ++[[painless-api-reference-Collator-getInstance-0]]static <<painless-api-reference-Collator,Collator>> link:{java8-javadoc}/java/text/Collator.html#getInstance%2D%2D[getInstance]()++ (link:{java9-javadoc}/java/text/Collator.html#getInstance%2D%2D[java 9])
+* ++[[painless-api-reference-Collator-getInstance-1]]static <<painless-api-reference-Collator,Collator>> link:{java8-javadoc}/java/text/Collator.html#getInstance%2Djava.util.Locale%2D[getInstance](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/text/Collator.html#getInstance%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-Collator-clone-0]]def link:{java8-javadoc}/java/text/Collator.html#clone%2D%2D[clone]()++ (link:{java9-javadoc}/java/text/Collator.html#clone%2D%2D[java 9])
+* ++[[painless-api-reference-Collator-equals-2]]boolean link:{java8-javadoc}/java/text/Collator.html#equals%2Djava.lang.String%2Djava.lang.String%2D[equals](<<painless-api-reference-String,String>>, <<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/Collator.html#equals%2Djava.lang.String%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Collator-getCollationKey-1]]<<painless-api-reference-CollationKey,CollationKey>> link:{java8-javadoc}/java/text/Collator.html#getCollationKey%2Djava.lang.String%2D[getCollationKey](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/Collator.html#getCollationKey%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Collator-getDecomposition-0]]int link:{java8-javadoc}/java/text/Collator.html#getDecomposition%2D%2D[getDecomposition]()++ (link:{java9-javadoc}/java/text/Collator.html#getDecomposition%2D%2D[java 9])
+* ++[[painless-api-reference-Collator-getStrength-0]]int link:{java8-javadoc}/java/text/Collator.html#getStrength%2D%2D[getStrength]()++ (link:{java9-javadoc}/java/text/Collator.html#getStrength%2D%2D[java 9])
+* ++[[painless-api-reference-Collator-setDecomposition-1]]void link:{java8-javadoc}/java/text/Collator.html#setDecomposition%2Dint%2D[setDecomposition](int)++ (link:{java9-javadoc}/java/text/Collator.html#setDecomposition%2Dint%2D[java 9])
+* ++[[painless-api-reference-Collator-setStrength-1]]void link:{java8-javadoc}/java/text/Collator.html#setStrength%2Dint%2D[setStrength](int)++ (link:{java9-javadoc}/java/text/Collator.html#setStrength%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Comparator,Comparator>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Collection.asciidoc b/docs/painless/painless-api-reference/Collection.asciidoc
new file mode 100644
index 0000000000000..e8faba23a6109
--- /dev/null
+++ b/docs/painless/painless-api-reference/Collection.asciidoc
@@ -0,0 +1,28 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Collection]]++Collection++::
+* ++[[painless-api-reference-Collection-add-1]]boolean link:{java8-javadoc}/java/util/Collection.html#add%2Djava.lang.Object%2D[add](def)++ (link:{java9-javadoc}/java/util/Collection.html#add%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Collection-addAll-1]]boolean link:{java8-javadoc}/java/util/Collection.html#addAll%2Djava.util.Collection%2D[addAll](<<painless-api-reference-Collection,Collection>>)++ (link:{java9-javadoc}/java/util/Collection.html#addAll%2Djava.util.Collection%2D[java 9])
+* ++[[painless-api-reference-Collection-clear-0]]void link:{java8-javadoc}/java/util/Collection.html#clear%2D%2D[clear]()++ (link:{java9-javadoc}/java/util/Collection.html#clear%2D%2D[java 9])
+* ++[[painless-api-reference-Collection-collect-1]]<<painless-api-reference-List,List>> link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#collect%2Djava.util.Collection%2Djava.util.function.Function%2D[collect](<<painless-api-reference-Function,Function>>)++
+* ++[[painless-api-reference-Collection-collect-2]]def link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#collect%2Djava.util.Collection%2Djava.util.Collection%2Djava.util.function.Function%2D[collect](<<painless-api-reference-Collection,Collection>>, <<painless-api-reference-Function,Function>>)++
+* ++[[painless-api-reference-Collection-contains-1]]boolean link:{java8-javadoc}/java/util/Collection.html#contains%2Djava.lang.Object%2D[contains](def)++ (link:{java9-javadoc}/java/util/Collection.html#contains%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Collection-containsAll-1]]boolean link:{java8-javadoc}/java/util/Collection.html#containsAll%2Djava.util.Collection%2D[containsAll](<<painless-api-reference-Collection,Collection>>)++ (link:{java9-javadoc}/java/util/Collection.html#containsAll%2Djava.util.Collection%2D[java 9])
+* ++[[painless-api-reference-Collection-find-1]]def link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#find%2Djava.util.Collection%2Djava.util.function.Predicate%2D[find](<<painless-api-reference-Predicate,Predicate>>)++
+* ++[[painless-api-reference-Collection-findAll-1]]<<painless-api-reference-List,List>> link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#findAll%2Djava.util.Collection%2Djava.util.function.Predicate%2D[findAll](<<painless-api-reference-Predicate,Predicate>>)++
+* ++[[painless-api-reference-Collection-findResult-1]]def link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#findResult%2Djava.util.Collection%2Djava.util.function.Function%2D[findResult](<<painless-api-reference-Function,Function>>)++
+* ++[[painless-api-reference-Collection-findResult-2]]def link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#findResult%2Djava.util.Collection%2Djava.lang.Object%2Djava.util.function.Function%2D[findResult](def, <<painless-api-reference-Function,Function>>)++
+* ++[[painless-api-reference-Collection-isEmpty-0]]boolean link:{java8-javadoc}/java/util/Collection.html#isEmpty%2D%2D[isEmpty]()++ (link:{java9-javadoc}/java/util/Collection.html#isEmpty%2D%2D[java 9])
+* ++[[painless-api-reference-Collection-removeAll-1]]boolean link:{java8-javadoc}/java/util/Collection.html#removeAll%2Djava.util.Collection%2D[removeAll](<<painless-api-reference-Collection,Collection>>)++ (link:{java9-javadoc}/java/util/Collection.html#removeAll%2Djava.util.Collection%2D[java 9])
+* ++[[painless-api-reference-Collection-removeIf-1]]boolean link:{java8-javadoc}/java/util/Collection.html#removeIf%2Djava.util.function.Predicate%2D[removeIf](<<painless-api-reference-Predicate,Predicate>>)++ (link:{java9-javadoc}/java/util/Collection.html#removeIf%2Djava.util.function.Predicate%2D[java 9])
+* ++[[painless-api-reference-Collection-retainAll-1]]boolean link:{java8-javadoc}/java/util/Collection.html#retainAll%2Djava.util.Collection%2D[retainAll](<<painless-api-reference-Collection,Collection>>)++ (link:{java9-javadoc}/java/util/Collection.html#retainAll%2Djava.util.Collection%2D[java 9])
+* ++[[painless-api-reference-Collection-size-0]]int link:{java8-javadoc}/java/util/Collection.html#size%2D%2D[size]()++ (link:{java9-javadoc}/java/util/Collection.html#size%2D%2D[java 9])
+* ++[[painless-api-reference-Collection-split-1]]<<painless-api-reference-List,List>> link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#split%2Djava.util.Collection%2Djava.util.function.Predicate%2D[split](<<painless-api-reference-Predicate,Predicate>>)++
+* ++[[painless-api-reference-Collection-spliterator-0]]<<painless-api-reference-Spliterator,Spliterator>> link:{java8-javadoc}/java/util/Collection.html#spliterator%2D%2D[spliterator]()++ (link:{java9-javadoc}/java/util/Collection.html#spliterator%2D%2D[java 9])
+* ++[[painless-api-reference-Collection-stream-0]]<<painless-api-reference-Stream,Stream>> link:{java8-javadoc}/java/util/Collection.html#stream%2D%2D[stream]()++ (link:{java9-javadoc}/java/util/Collection.html#stream%2D%2D[java 9])
+* ++[[painless-api-reference-Collection-toArray-0]]def[] link:{java8-javadoc}/java/util/Collection.html#toArray%2D%2D[toArray]()++ (link:{java9-javadoc}/java/util/Collection.html#toArray%2D%2D[java 9])
+* ++[[painless-api-reference-Collection-toArray-1]]def[] link:{java8-javadoc}/java/util/Collection.html#toArray%2Djava.lang.Object:A%2D[toArray](def[])++ (link:{java9-javadoc}/java/util/Collection.html#toArray%2Djava.lang.Object:A%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Collections.asciidoc b/docs/painless/painless-api-reference/Collections.asciidoc
new file mode 100644
index 0000000000000..db6d87e85210e
--- /dev/null
+++ b/docs/painless/painless-api-reference/Collections.asciidoc
@@ -0,0 +1,59 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Collections]]++Collections++::
+** [[painless-api-reference-Collections-EMPTY_LIST]]static <<painless-api-reference-List,List>> link:{java8-javadoc}/java/util/Collections.html#EMPTY_LIST[EMPTY_LIST] (link:{java9-javadoc}/java/util/Collections.html#EMPTY_LIST[java 9])
+** [[painless-api-reference-Collections-EMPTY_MAP]]static <<painless-api-reference-Map,Map>> link:{java8-javadoc}/java/util/Collections.html#EMPTY_MAP[EMPTY_MAP] (link:{java9-javadoc}/java/util/Collections.html#EMPTY_MAP[java 9])
+** [[painless-api-reference-Collections-EMPTY_SET]]static <<painless-api-reference-Set,Set>> link:{java8-javadoc}/java/util/Collections.html#EMPTY_SET[EMPTY_SET] (link:{java9-javadoc}/java/util/Collections.html#EMPTY_SET[java 9])
+* ++[[painless-api-reference-Collections-addAll-2]]static boolean link:{java8-javadoc}/java/util/Collections.html#addAll%2Djava.util.Collection%2Djava.lang.Object:A%2D[addAll](<<painless-api-reference-Collection,Collection>>, def[])++ (link:{java9-javadoc}/java/util/Collections.html#addAll%2Djava.util.Collection%2Djava.lang.Object:A%2D[java 9])
+* ++[[painless-api-reference-Collections-asLifoQueue-1]]static <<painless-api-reference-Queue,Queue>> link:{java8-javadoc}/java/util/Collections.html#asLifoQueue%2Djava.util.Deque%2D[asLifoQueue](<<painless-api-reference-Deque,Deque>>)++ (link:{java9-javadoc}/java/util/Collections.html#asLifoQueue%2Djava.util.Deque%2D[java 9])
+* ++[[painless-api-reference-Collections-binarySearch-2]]static int link:{java8-javadoc}/java/util/Collections.html#binarySearch%2Djava.util.List%2Djava.lang.Object%2D[binarySearch](<<painless-api-reference-List,List>>, def)++ (link:{java9-javadoc}/java/util/Collections.html#binarySearch%2Djava.util.List%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Collections-binarySearch-3]]static int link:{java8-javadoc}/java/util/Collections.html#binarySearch%2Djava.util.List%2Djava.lang.Object%2Djava.util.Comparator%2D[binarySearch](<<painless-api-reference-List,List>>, def, <<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/Collections.html#binarySearch%2Djava.util.List%2Djava.lang.Object%2Djava.util.Comparator%2D[java 9])
+* ++[[painless-api-reference-Collections-copy-2]]static void link:{java8-javadoc}/java/util/Collections.html#copy%2Djava.util.List%2Djava.util.List%2D[copy](<<painless-api-reference-List,List>>, <<painless-api-reference-List,List>>)++ (link:{java9-javadoc}/java/util/Collections.html#copy%2Djava.util.List%2Djava.util.List%2D[java 9])
+* ++[[painless-api-reference-Collections-disjoint-2]]static boolean link:{java8-javadoc}/java/util/Collections.html#disjoint%2Djava.util.Collection%2Djava.util.Collection%2D[disjoint](<<painless-api-reference-Collection,Collection>>, <<painless-api-reference-Collection,Collection>>)++ (link:{java9-javadoc}/java/util/Collections.html#disjoint%2Djava.util.Collection%2Djava.util.Collection%2D[java 9])
+* ++[[painless-api-reference-Collections-emptyEnumeration-0]]static <<painless-api-reference-Enumeration,Enumeration>> link:{java8-javadoc}/java/util/Collections.html#emptyEnumeration%2D%2D[emptyEnumeration]()++ (link:{java9-javadoc}/java/util/Collections.html#emptyEnumeration%2D%2D[java 9])
+* ++[[painless-api-reference-Collections-emptyIterator-0]]static <<painless-api-reference-Iterator,Iterator>> link:{java8-javadoc}/java/util/Collections.html#emptyIterator%2D%2D[emptyIterator]()++ (link:{java9-javadoc}/java/util/Collections.html#emptyIterator%2D%2D[java 9])
+* ++[[painless-api-reference-Collections-emptyList-0]]static <<painless-api-reference-List,List>> link:{java8-javadoc}/java/util/Collections.html#emptyList%2D%2D[emptyList]()++ (link:{java9-javadoc}/java/util/Collections.html#emptyList%2D%2D[java 9])
+* ++[[painless-api-reference-Collections-emptyListIterator-0]]static <<painless-api-reference-ListIterator,ListIterator>> link:{java8-javadoc}/java/util/Collections.html#emptyListIterator%2D%2D[emptyListIterator]()++ (link:{java9-javadoc}/java/util/Collections.html#emptyListIterator%2D%2D[java 9])
+* ++[[painless-api-reference-Collections-emptyMap-0]]static <<painless-api-reference-Map,Map>> link:{java8-javadoc}/java/util/Collections.html#emptyMap%2D%2D[emptyMap]()++ (link:{java9-javadoc}/java/util/Collections.html#emptyMap%2D%2D[java 9])
+* ++[[painless-api-reference-Collections-emptyNavigableMap-0]]static <<painless-api-reference-NavigableMap,NavigableMap>> link:{java8-javadoc}/java/util/Collections.html#emptyNavigableMap%2D%2D[emptyNavigableMap]()++ (link:{java9-javadoc}/java/util/Collections.html#emptyNavigableMap%2D%2D[java 9])
+* ++[[painless-api-reference-Collections-emptyNavigableSet-0]]static <<painless-api-reference-NavigableSet,NavigableSet>> link:{java8-javadoc}/java/util/Collections.html#emptyNavigableSet%2D%2D[emptyNavigableSet]()++ (link:{java9-javadoc}/java/util/Collections.html#emptyNavigableSet%2D%2D[java 9])
+* ++[[painless-api-reference-Collections-emptySet-0]]static <<painless-api-reference-Set,Set>> link:{java8-javadoc}/java/util/Collections.html#emptySet%2D%2D[emptySet]()++ (link:{java9-javadoc}/java/util/Collections.html#emptySet%2D%2D[java 9])
+* ++[[painless-api-reference-Collections-emptySortedMap-0]]static <<painless-api-reference-SortedMap,SortedMap>> link:{java8-javadoc}/java/util/Collections.html#emptySortedMap%2D%2D[emptySortedMap]()++ (link:{java9-javadoc}/java/util/Collections.html#emptySortedMap%2D%2D[java 9])
+* ++[[painless-api-reference-Collections-emptySortedSet-0]]static <<painless-api-reference-SortedSet,SortedSet>> link:{java8-javadoc}/java/util/Collections.html#emptySortedSet%2D%2D[emptySortedSet]()++ (link:{java9-javadoc}/java/util/Collections.html#emptySortedSet%2D%2D[java 9])
+* ++[[painless-api-reference-Collections-enumeration-1]]static <<painless-api-reference-Enumeration,Enumeration>> link:{java8-javadoc}/java/util/Collections.html#enumeration%2Djava.util.Collection%2D[enumeration](<<painless-api-reference-Collection,Collection>>)++ (link:{java9-javadoc}/java/util/Collections.html#enumeration%2Djava.util.Collection%2D[java 9])
+* ++[[painless-api-reference-Collections-fill-2]]static void link:{java8-javadoc}/java/util/Collections.html#fill%2Djava.util.List%2Djava.lang.Object%2D[fill](<<painless-api-reference-List,List>>, def)++ (link:{java9-javadoc}/java/util/Collections.html#fill%2Djava.util.List%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Collections-frequency-2]]static int link:{java8-javadoc}/java/util/Collections.html#frequency%2Djava.util.Collection%2Djava.lang.Object%2D[frequency](<<painless-api-reference-Collection,Collection>>, def)++ (link:{java9-javadoc}/java/util/Collections.html#frequency%2Djava.util.Collection%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Collections-indexOfSubList-2]]static int link:{java8-javadoc}/java/util/Collections.html#indexOfSubList%2Djava.util.List%2Djava.util.List%2D[indexOfSubList](<<painless-api-reference-List,List>>, <<painless-api-reference-List,List>>)++ (link:{java9-javadoc}/java/util/Collections.html#indexOfSubList%2Djava.util.List%2Djava.util.List%2D[java 9])
+* ++[[painless-api-reference-Collections-lastIndexOfSubList-2]]static int link:{java8-javadoc}/java/util/Collections.html#lastIndexOfSubList%2Djava.util.List%2Djava.util.List%2D[lastIndexOfSubList](<<painless-api-reference-List,List>>, <<painless-api-reference-List,List>>)++ (link:{java9-javadoc}/java/util/Collections.html#lastIndexOfSubList%2Djava.util.List%2Djava.util.List%2D[java 9])
+* ++[[painless-api-reference-Collections-list-1]]static <<painless-api-reference-ArrayList,ArrayList>> link:{java8-javadoc}/java/util/Collections.html#list%2Djava.util.Enumeration%2D[list](<<painless-api-reference-Enumeration,Enumeration>>)++ (link:{java9-javadoc}/java/util/Collections.html#list%2Djava.util.Enumeration%2D[java 9])
+* ++[[painless-api-reference-Collections-max-1]]static def link:{java8-javadoc}/java/util/Collections.html#max%2Djava.util.Collection%2D[max](<<painless-api-reference-Collection,Collection>>)++ (link:{java9-javadoc}/java/util/Collections.html#max%2Djava.util.Collection%2D[java 9])
+* ++[[painless-api-reference-Collections-max-2]]static def link:{java8-javadoc}/java/util/Collections.html#max%2Djava.util.Collection%2Djava.util.Comparator%2D[max](<<painless-api-reference-Collection,Collection>>, <<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/Collections.html#max%2Djava.util.Collection%2Djava.util.Comparator%2D[java 9])
+* ++[[painless-api-reference-Collections-min-1]]static def link:{java8-javadoc}/java/util/Collections.html#min%2Djava.util.Collection%2D[min](<<painless-api-reference-Collection,Collection>>)++ (link:{java9-javadoc}/java/util/Collections.html#min%2Djava.util.Collection%2D[java 9])
+* ++[[painless-api-reference-Collections-min-2]]static def link:{java8-javadoc}/java/util/Collections.html#min%2Djava.util.Collection%2Djava.util.Comparator%2D[min](<<painless-api-reference-Collection,Collection>>, <<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/Collections.html#min%2Djava.util.Collection%2Djava.util.Comparator%2D[java 9])
+* ++[[painless-api-reference-Collections-nCopies-2]]static <<painless-api-reference-List,List>> link:{java8-javadoc}/java/util/Collections.html#nCopies%2Dint%2Djava.lang.Object%2D[nCopies](int, def)++ (link:{java9-javadoc}/java/util/Collections.html#nCopies%2Dint%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Collections-newSetFromMap-1]]static <<painless-api-reference-Set,Set>> link:{java8-javadoc}/java/util/Collections.html#newSetFromMap%2Djava.util.Map%2D[newSetFromMap](<<painless-api-reference-Map,Map>>)++ (link:{java9-javadoc}/java/util/Collections.html#newSetFromMap%2Djava.util.Map%2D[java 9])
+* ++[[painless-api-reference-Collections-replaceAll-3]]static boolean link:{java8-javadoc}/java/util/Collections.html#replaceAll%2Djava.util.List%2Djava.lang.Object%2Djava.lang.Object%2D[replaceAll](<<painless-api-reference-List,List>>, def, def)++ (link:{java9-javadoc}/java/util/Collections.html#replaceAll%2Djava.util.List%2Djava.lang.Object%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Collections-reverse-1]]static void link:{java8-javadoc}/java/util/Collections.html#reverse%2Djava.util.List%2D[reverse](<<painless-api-reference-List,List>>)++ (link:{java9-javadoc}/java/util/Collections.html#reverse%2Djava.util.List%2D[java 9])
+* ++[[painless-api-reference-Collections-reverseOrder-0]]static <<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/Collections.html#reverseOrder%2D%2D[reverseOrder]()++ (link:{java9-javadoc}/java/util/Collections.html#reverseOrder%2D%2D[java 9])
+* ++[[painless-api-reference-Collections-reverseOrder-1]]static <<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/Collections.html#reverseOrder%2Djava.util.Comparator%2D[reverseOrder](<<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/Collections.html#reverseOrder%2Djava.util.Comparator%2D[java 9])
+* ++[[painless-api-reference-Collections-rotate-2]]static void link:{java8-javadoc}/java/util/Collections.html#rotate%2Djava.util.List%2Dint%2D[rotate](<<painless-api-reference-List,List>>, int)++ (link:{java9-javadoc}/java/util/Collections.html#rotate%2Djava.util.List%2Dint%2D[java 9])
+* ++[[painless-api-reference-Collections-shuffle-1]]static void link:{java8-javadoc}/java/util/Collections.html#shuffle%2Djava.util.List%2D[shuffle](<<painless-api-reference-List,List>>)++ (link:{java9-javadoc}/java/util/Collections.html#shuffle%2Djava.util.List%2D[java 9])
+* ++[[painless-api-reference-Collections-shuffle-2]]static void link:{java8-javadoc}/java/util/Collections.html#shuffle%2Djava.util.List%2Djava.util.Random%2D[shuffle](<<painless-api-reference-List,List>>, <<painless-api-reference-Random,Random>>)++ (link:{java9-javadoc}/java/util/Collections.html#shuffle%2Djava.util.List%2Djava.util.Random%2D[java 9])
+* ++[[painless-api-reference-Collections-singleton-1]]static <<painless-api-reference-Set,Set>> link:{java8-javadoc}/java/util/Collections.html#singleton%2Djava.lang.Object%2D[singleton](def)++ (link:{java9-javadoc}/java/util/Collections.html#singleton%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Collections-singletonList-1]]static <<painless-api-reference-List,List>> link:{java8-javadoc}/java/util/Collections.html#singletonList%2Djava.lang.Object%2D[singletonList](def)++ (link:{java9-javadoc}/java/util/Collections.html#singletonList%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Collections-singletonMap-2]]static <<painless-api-reference-Map,Map>> link:{java8-javadoc}/java/util/Collections.html#singletonMap%2Djava.lang.Object%2Djava.lang.Object%2D[singletonMap](def, def)++ (link:{java9-javadoc}/java/util/Collections.html#singletonMap%2Djava.lang.Object%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Collections-sort-1]]static void link:{java8-javadoc}/java/util/Collections.html#sort%2Djava.util.List%2D[sort](<<painless-api-reference-List,List>>)++ (link:{java9-javadoc}/java/util/Collections.html#sort%2Djava.util.List%2D[java 9])
+* ++[[painless-api-reference-Collections-sort-2]]static void link:{java8-javadoc}/java/util/Collections.html#sort%2Djava.util.List%2Djava.util.Comparator%2D[sort](<<painless-api-reference-List,List>>, <<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/Collections.html#sort%2Djava.util.List%2Djava.util.Comparator%2D[java 9])
+* ++[[painless-api-reference-Collections-swap-3]]static void link:{java8-javadoc}/java/util/Collections.html#swap%2Djava.util.List%2Dint%2Dint%2D[swap](<<painless-api-reference-List,List>>, int, int)++ (link:{java9-javadoc}/java/util/Collections.html#swap%2Djava.util.List%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Collections-unmodifiableCollection-1]]static <<painless-api-reference-Collection,Collection>> link:{java8-javadoc}/java/util/Collections.html#unmodifiableCollection%2Djava.util.Collection%2D[unmodifiableCollection](<<painless-api-reference-Collection,Collection>>)++ (link:{java9-javadoc}/java/util/Collections.html#unmodifiableCollection%2Djava.util.Collection%2D[java 9])
+* ++[[painless-api-reference-Collections-unmodifiableList-1]]static <<painless-api-reference-List,List>> link:{java8-javadoc}/java/util/Collections.html#unmodifiableList%2Djava.util.List%2D[unmodifiableList](<<painless-api-reference-List,List>>)++ (link:{java9-javadoc}/java/util/Collections.html#unmodifiableList%2Djava.util.List%2D[java 9])
+* ++[[painless-api-reference-Collections-unmodifiableMap-1]]static <<painless-api-reference-Map,Map>> link:{java8-javadoc}/java/util/Collections.html#unmodifiableMap%2Djava.util.Map%2D[unmodifiableMap](<<painless-api-reference-Map,Map>>)++ (link:{java9-javadoc}/java/util/Collections.html#unmodifiableMap%2Djava.util.Map%2D[java 9])
+* ++[[painless-api-reference-Collections-unmodifiableNavigableMap-1]]static <<painless-api-reference-NavigableMap,NavigableMap>> link:{java8-javadoc}/java/util/Collections.html#unmodifiableNavigableMap%2Djava.util.NavigableMap%2D[unmodifiableNavigableMap](<<painless-api-reference-NavigableMap,NavigableMap>>)++ (link:{java9-javadoc}/java/util/Collections.html#unmodifiableNavigableMap%2Djava.util.NavigableMap%2D[java 9])
+* ++[[painless-api-reference-Collections-unmodifiableNavigableSet-1]]static <<painless-api-reference-NavigableSet,NavigableSet>> link:{java8-javadoc}/java/util/Collections.html#unmodifiableNavigableSet%2Djava.util.NavigableSet%2D[unmodifiableNavigableSet](<<painless-api-reference-NavigableSet,NavigableSet>>)++ (link:{java9-javadoc}/java/util/Collections.html#unmodifiableNavigableSet%2Djava.util.NavigableSet%2D[java 9])
+* ++[[painless-api-reference-Collections-unmodifiableSet-1]]static <<painless-api-reference-Set,Set>> link:{java8-javadoc}/java/util/Collections.html#unmodifiableSet%2Djava.util.Set%2D[unmodifiableSet](<<painless-api-reference-Set,Set>>)++ (link:{java9-javadoc}/java/util/Collections.html#unmodifiableSet%2Djava.util.Set%2D[java 9])
+* ++[[painless-api-reference-Collections-unmodifiableSortedMap-1]]static <<painless-api-reference-SortedMap,SortedMap>> link:{java8-javadoc}/java/util/Collections.html#unmodifiableSortedMap%2Djava.util.SortedMap%2D[unmodifiableSortedMap](<<painless-api-reference-SortedMap,SortedMap>>)++ (link:{java9-javadoc}/java/util/Collections.html#unmodifiableSortedMap%2Djava.util.SortedMap%2D[java 9])
+* ++[[painless-api-reference-Collections-unmodifiableSortedSet-1]]static <<painless-api-reference-SortedSet,SortedSet>> link:{java8-javadoc}/java/util/Collections.html#unmodifiableSortedSet%2Djava.util.SortedSet%2D[unmodifiableSortedSet](<<painless-api-reference-SortedSet,SortedSet>>)++ (link:{java9-javadoc}/java/util/Collections.html#unmodifiableSortedSet%2Djava.util.SortedSet%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Collector.Characteristics.asciidoc b/docs/painless/painless-api-reference/Collector.Characteristics.asciidoc
new file mode 100644
index 0000000000000..a0c95223d57d0
--- /dev/null
+++ b/docs/painless/painless-api-reference/Collector.Characteristics.asciidoc
@@ -0,0 +1,12 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Collector-Characteristics]]++Collector.Characteristics++::
+** [[painless-api-reference-Collector-Characteristics-CONCURRENT]]static <<painless-api-reference-Collector-Characteristics,Collector.Characteristics>> link:{java8-javadoc}/java/util/stream/Collector.Characteristics.html#CONCURRENT[CONCURRENT] (link:{java9-javadoc}/java/util/stream/Collector.Characteristics.html#CONCURRENT[java 9])
+** [[painless-api-reference-Collector-Characteristics-IDENTITY_FINISH]]static <<painless-api-reference-Collector-Characteristics,Collector.Characteristics>> link:{java8-javadoc}/java/util/stream/Collector.Characteristics.html#IDENTITY_FINISH[IDENTITY_FINISH] (link:{java9-javadoc}/java/util/stream/Collector.Characteristics.html#IDENTITY_FINISH[java 9])
+** [[painless-api-reference-Collector-Characteristics-UNORDERED]]static <<painless-api-reference-Collector-Characteristics,Collector.Characteristics>> link:{java8-javadoc}/java/util/stream/Collector.Characteristics.html#UNORDERED[UNORDERED] (link:{java9-javadoc}/java/util/stream/Collector.Characteristics.html#UNORDERED[java 9])
+* ++[[painless-api-reference-Collector-Characteristics-valueOf-1]]static <<painless-api-reference-Collector-Characteristics,Collector.Characteristics>> link:{java8-javadoc}/java/util/stream/Collector.Characteristics.html#valueOf%2Djava.lang.String%2D[valueOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/stream/Collector.Characteristics.html#valueOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Collector-Characteristics-values-0]]static <<painless-api-reference-Collector-Characteristics,Collector.Characteristics>>[] link:{java8-javadoc}/java/util/stream/Collector.Characteristics.html#values%2D%2D[values]()++ (link:{java9-javadoc}/java/util/stream/Collector.Characteristics.html#values%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Enum,Enum>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Collector.asciidoc b/docs/painless/painless-api-reference/Collector.asciidoc
new file mode 100644
index 0000000000000..1d1b92f1e0911
--- /dev/null
+++ b/docs/painless/painless-api-reference/Collector.asciidoc
@@ -0,0 +1,14 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Collector]]++Collector++::
+* ++[[painless-api-reference-Collector-of-4]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collector.html#of%2Djava.util.function.Supplier%2Djava.util.function.BiConsumer%2Djava.util.function.BinaryOperator%2Djava.util.stream.Collector$Characteristics:A%2D[of](<<painless-api-reference-Supplier,Supplier>>, <<painless-api-reference-BiConsumer,BiConsumer>>, <<painless-api-reference-BinaryOperator,BinaryOperator>>, <<painless-api-reference-Collector-Characteristics,Collector.Characteristics>>[])++ (link:{java9-javadoc}/java/util/stream/Collector.html#of%2Djava.util.function.Supplier%2Djava.util.function.BiConsumer%2Djava.util.function.BinaryOperator%2Djava.util.stream.Collector$Characteristics:A%2D[java 9])
+* ++[[painless-api-reference-Collector-of-5]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collector.html#of%2Djava.util.function.Supplier%2Djava.util.function.BiConsumer%2Djava.util.function.BinaryOperator%2Djava.util.function.Function%2Djava.util.stream.Collector$Characteristics:A%2D[of](<<painless-api-reference-Supplier,Supplier>>, <<painless-api-reference-BiConsumer,BiConsumer>>, <<painless-api-reference-BinaryOperator,BinaryOperator>>, <<painless-api-reference-Function,Function>>, <<painless-api-reference-Collector-Characteristics,Collector.Characteristics>>[])++ (link:{java9-javadoc}/java/util/stream/Collector.html#of%2Djava.util.function.Supplier%2Djava.util.function.BiConsumer%2Djava.util.function.BinaryOperator%2Djava.util.function.Function%2Djava.util.stream.Collector$Characteristics:A%2D[java 9])
+* ++[[painless-api-reference-Collector-accumulator-0]]<<painless-api-reference-BiConsumer,BiConsumer>> link:{java8-javadoc}/java/util/stream/Collector.html#accumulator%2D%2D[accumulator]()++ (link:{java9-javadoc}/java/util/stream/Collector.html#accumulator%2D%2D[java 9])
+* ++[[painless-api-reference-Collector-characteristics-0]]<<painless-api-reference-Set,Set>> link:{java8-javadoc}/java/util/stream/Collector.html#characteristics%2D%2D[characteristics]()++ (link:{java9-javadoc}/java/util/stream/Collector.html#characteristics%2D%2D[java 9])
+* ++[[painless-api-reference-Collector-combiner-0]]<<painless-api-reference-BinaryOperator,BinaryOperator>> link:{java8-javadoc}/java/util/stream/Collector.html#combiner%2D%2D[combiner]()++ (link:{java9-javadoc}/java/util/stream/Collector.html#combiner%2D%2D[java 9])
+* ++[[painless-api-reference-Collector-finisher-0]]<<painless-api-reference-Function,Function>> link:{java8-javadoc}/java/util/stream/Collector.html#finisher%2D%2D[finisher]()++ (link:{java9-javadoc}/java/util/stream/Collector.html#finisher%2D%2D[java 9])
+* ++[[painless-api-reference-Collector-supplier-0]]<<painless-api-reference-Supplier,Supplier>> link:{java8-javadoc}/java/util/stream/Collector.html#supplier%2D%2D[supplier]()++ (link:{java9-javadoc}/java/util/stream/Collector.html#supplier%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Collectors.asciidoc b/docs/painless/painless-api-reference/Collectors.asciidoc
new file mode 100644
index 0000000000000..ef4d0199386e1
--- /dev/null
+++ b/docs/painless/painless-api-reference/Collectors.asciidoc
@@ -0,0 +1,38 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Collectors]]++Collectors++::
+* ++[[painless-api-reference-Collectors-averagingDouble-1]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#averagingDouble%2Djava.util.function.ToDoubleFunction%2D[averagingDouble](<<painless-api-reference-ToDoubleFunction,ToDoubleFunction>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#averagingDouble%2Djava.util.function.ToDoubleFunction%2D[java 9])
+* ++[[painless-api-reference-Collectors-averagingInt-1]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#averagingInt%2Djava.util.function.ToIntFunction%2D[averagingInt](<<painless-api-reference-ToIntFunction,ToIntFunction>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#averagingInt%2Djava.util.function.ToIntFunction%2D[java 9])
+* ++[[painless-api-reference-Collectors-averagingLong-1]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#averagingLong%2Djava.util.function.ToLongFunction%2D[averagingLong](<<painless-api-reference-ToLongFunction,ToLongFunction>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#averagingLong%2Djava.util.function.ToLongFunction%2D[java 9])
+* ++[[painless-api-reference-Collectors-collectingAndThen-2]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#collectingAndThen%2Djava.util.stream.Collector%2Djava.util.function.Function%2D[collectingAndThen](<<painless-api-reference-Collector,Collector>>, <<painless-api-reference-Function,Function>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#collectingAndThen%2Djava.util.stream.Collector%2Djava.util.function.Function%2D[java 9])
+* ++[[painless-api-reference-Collectors-counting-0]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#counting%2D%2D[counting]()++ (link:{java9-javadoc}/java/util/stream/Collectors.html#counting%2D%2D[java 9])
+* ++[[painless-api-reference-Collectors-groupingBy-1]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#groupingBy%2Djava.util.function.Function%2D[groupingBy](<<painless-api-reference-Function,Function>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#groupingBy%2Djava.util.function.Function%2D[java 9])
+* ++[[painless-api-reference-Collectors-groupingBy-2]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#groupingBy%2Djava.util.function.Function%2Djava.util.stream.Collector%2D[groupingBy](<<painless-api-reference-Function,Function>>, <<painless-api-reference-Collector,Collector>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#groupingBy%2Djava.util.function.Function%2Djava.util.stream.Collector%2D[java 9])
+* ++[[painless-api-reference-Collectors-groupingBy-3]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#groupingBy%2Djava.util.function.Function%2Djava.util.function.Supplier%2Djava.util.stream.Collector%2D[groupingBy](<<painless-api-reference-Function,Function>>, <<painless-api-reference-Supplier,Supplier>>, <<painless-api-reference-Collector,Collector>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#groupingBy%2Djava.util.function.Function%2Djava.util.function.Supplier%2Djava.util.stream.Collector%2D[java 9])
+* ++[[painless-api-reference-Collectors-joining-0]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#joining%2D%2D[joining]()++ (link:{java9-javadoc}/java/util/stream/Collectors.html#joining%2D%2D[java 9])
+* ++[[painless-api-reference-Collectors-joining-1]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#joining%2Djava.lang.CharSequence%2D[joining](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#joining%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-Collectors-joining-3]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#joining%2Djava.lang.CharSequence%2Djava.lang.CharSequence%2Djava.lang.CharSequence%2D[joining](<<painless-api-reference-CharSequence,CharSequence>>, <<painless-api-reference-CharSequence,CharSequence>>, <<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#joining%2Djava.lang.CharSequence%2Djava.lang.CharSequence%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-Collectors-mapping-2]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#mapping%2Djava.util.function.Function%2Djava.util.stream.Collector%2D[mapping](<<painless-api-reference-Function,Function>>, <<painless-api-reference-Collector,Collector>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#mapping%2Djava.util.function.Function%2Djava.util.stream.Collector%2D[java 9])
+* ++[[painless-api-reference-Collectors-maxBy-1]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#maxBy%2Djava.util.Comparator%2D[maxBy](<<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#maxBy%2Djava.util.Comparator%2D[java 9])
+* ++[[painless-api-reference-Collectors-minBy-1]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#minBy%2Djava.util.Comparator%2D[minBy](<<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#minBy%2Djava.util.Comparator%2D[java 9])
+* ++[[painless-api-reference-Collectors-partitioningBy-1]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#partitioningBy%2Djava.util.function.Predicate%2D[partitioningBy](<<painless-api-reference-Predicate,Predicate>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#partitioningBy%2Djava.util.function.Predicate%2D[java 9])
+* ++[[painless-api-reference-Collectors-partitioningBy-2]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#partitioningBy%2Djava.util.function.Predicate%2Djava.util.stream.Collector%2D[partitioningBy](<<painless-api-reference-Predicate,Predicate>>, <<painless-api-reference-Collector,Collector>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#partitioningBy%2Djava.util.function.Predicate%2Djava.util.stream.Collector%2D[java 9])
+* ++[[painless-api-reference-Collectors-reducing-1]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#reducing%2Djava.util.function.BinaryOperator%2D[reducing](<<painless-api-reference-BinaryOperator,BinaryOperator>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#reducing%2Djava.util.function.BinaryOperator%2D[java 9])
+* ++[[painless-api-reference-Collectors-reducing-2]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#reducing%2Djava.lang.Object%2Djava.util.function.BinaryOperator%2D[reducing](def, <<painless-api-reference-BinaryOperator,BinaryOperator>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#reducing%2Djava.lang.Object%2Djava.util.function.BinaryOperator%2D[java 9])
+* ++[[painless-api-reference-Collectors-reducing-3]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#reducing%2Djava.lang.Object%2Djava.util.function.Function%2Djava.util.function.BinaryOperator%2D[reducing](def, <<painless-api-reference-Function,Function>>, <<painless-api-reference-BinaryOperator,BinaryOperator>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#reducing%2Djava.lang.Object%2Djava.util.function.Function%2Djava.util.function.BinaryOperator%2D[java 9])
+* ++[[painless-api-reference-Collectors-summarizingDouble-1]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#summarizingDouble%2Djava.util.function.ToDoubleFunction%2D[summarizingDouble](<<painless-api-reference-ToDoubleFunction,ToDoubleFunction>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#summarizingDouble%2Djava.util.function.ToDoubleFunction%2D[java 9])
+* ++[[painless-api-reference-Collectors-summarizingInt-1]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#summarizingInt%2Djava.util.function.ToIntFunction%2D[summarizingInt](<<painless-api-reference-ToIntFunction,ToIntFunction>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#summarizingInt%2Djava.util.function.ToIntFunction%2D[java 9])
+* ++[[painless-api-reference-Collectors-summarizingLong-1]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#summarizingLong%2Djava.util.function.ToLongFunction%2D[summarizingLong](<<painless-api-reference-ToLongFunction,ToLongFunction>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#summarizingLong%2Djava.util.function.ToLongFunction%2D[java 9])
+* ++[[painless-api-reference-Collectors-summingDouble-1]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#summingDouble%2Djava.util.function.ToDoubleFunction%2D[summingDouble](<<painless-api-reference-ToDoubleFunction,ToDoubleFunction>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#summingDouble%2Djava.util.function.ToDoubleFunction%2D[java 9])
+* ++[[painless-api-reference-Collectors-summingInt-1]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#summingInt%2Djava.util.function.ToIntFunction%2D[summingInt](<<painless-api-reference-ToIntFunction,ToIntFunction>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#summingInt%2Djava.util.function.ToIntFunction%2D[java 9])
+* ++[[painless-api-reference-Collectors-summingLong-1]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#summingLong%2Djava.util.function.ToLongFunction%2D[summingLong](<<painless-api-reference-ToLongFunction,ToLongFunction>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#summingLong%2Djava.util.function.ToLongFunction%2D[java 9])
+* ++[[painless-api-reference-Collectors-toCollection-1]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#toCollection%2Djava.util.function.Supplier%2D[toCollection](<<painless-api-reference-Supplier,Supplier>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#toCollection%2Djava.util.function.Supplier%2D[java 9])
+* ++[[painless-api-reference-Collectors-toList-0]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#toList%2D%2D[toList]()++ (link:{java9-javadoc}/java/util/stream/Collectors.html#toList%2D%2D[java 9])
+* ++[[painless-api-reference-Collectors-toMap-2]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#toMap%2Djava.util.function.Function%2Djava.util.function.Function%2D[toMap](<<painless-api-reference-Function,Function>>, <<painless-api-reference-Function,Function>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#toMap%2Djava.util.function.Function%2Djava.util.function.Function%2D[java 9])
+* ++[[painless-api-reference-Collectors-toMap-3]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#toMap%2Djava.util.function.Function%2Djava.util.function.Function%2Djava.util.function.BinaryOperator%2D[toMap](<<painless-api-reference-Function,Function>>, <<painless-api-reference-Function,Function>>, <<painless-api-reference-BinaryOperator,BinaryOperator>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#toMap%2Djava.util.function.Function%2Djava.util.function.Function%2Djava.util.function.BinaryOperator%2D[java 9])
+* ++[[painless-api-reference-Collectors-toMap-4]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#toMap%2Djava.util.function.Function%2Djava.util.function.Function%2Djava.util.function.BinaryOperator%2Djava.util.function.Supplier%2D[toMap](<<painless-api-reference-Function,Function>>, <<painless-api-reference-Function,Function>>, <<painless-api-reference-BinaryOperator,BinaryOperator>>, <<painless-api-reference-Supplier,Supplier>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#toMap%2Djava.util.function.Function%2Djava.util.function.Function%2Djava.util.function.BinaryOperator%2Djava.util.function.Supplier%2D[java 9])
+* ++[[painless-api-reference-Collectors-toSet-0]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#toSet%2D%2D[toSet]()++ (link:{java9-javadoc}/java/util/stream/Collectors.html#toSet%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Comparable.asciidoc b/docs/painless/painless-api-reference/Comparable.asciidoc
new file mode 100644
index 0000000000000..1550d93b51144
--- /dev/null
+++ b/docs/painless/painless-api-reference/Comparable.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Comparable]]++Comparable++::
+* ++[[painless-api-reference-Comparable-compareTo-1]]int link:{java8-javadoc}/java/lang/Comparable.html#compareTo%2Djava.lang.Object%2D[compareTo](def)++ (link:{java9-javadoc}/java/lang/Comparable.html#compareTo%2Djava.lang.Object%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Comparator.asciidoc b/docs/painless/painless-api-reference/Comparator.asciidoc
new file mode 100644
index 0000000000000..25545376942cc
--- /dev/null
+++ b/docs/painless/painless-api-reference/Comparator.asciidoc
@@ -0,0 +1,24 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Comparator]]++Comparator++::
+* ++[[painless-api-reference-Comparator-comparing-1]]static <<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/Comparator.html#comparing%2Djava.util.function.Function%2D[comparing](<<painless-api-reference-Function,Function>>)++ (link:{java9-javadoc}/java/util/Comparator.html#comparing%2Djava.util.function.Function%2D[java 9])
+* ++[[painless-api-reference-Comparator-comparing-2]]static <<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/Comparator.html#comparing%2Djava.util.function.Function%2Djava.util.Comparator%2D[comparing](<<painless-api-reference-Function,Function>>, <<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/Comparator.html#comparing%2Djava.util.function.Function%2Djava.util.Comparator%2D[java 9])
+* ++[[painless-api-reference-Comparator-comparingDouble-1]]static <<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/Comparator.html#comparingDouble%2Djava.util.function.ToDoubleFunction%2D[comparingDouble](<<painless-api-reference-ToDoubleFunction,ToDoubleFunction>>)++ (link:{java9-javadoc}/java/util/Comparator.html#comparingDouble%2Djava.util.function.ToDoubleFunction%2D[java 9])
+* ++[[painless-api-reference-Comparator-comparingInt-1]]static <<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/Comparator.html#comparingInt%2Djava.util.function.ToIntFunction%2D[comparingInt](<<painless-api-reference-ToIntFunction,ToIntFunction>>)++ (link:{java9-javadoc}/java/util/Comparator.html#comparingInt%2Djava.util.function.ToIntFunction%2D[java 9])
+* ++[[painless-api-reference-Comparator-comparingLong-1]]static <<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/Comparator.html#comparingLong%2Djava.util.function.ToLongFunction%2D[comparingLong](<<painless-api-reference-ToLongFunction,ToLongFunction>>)++ (link:{java9-javadoc}/java/util/Comparator.html#comparingLong%2Djava.util.function.ToLongFunction%2D[java 9])
+* ++[[painless-api-reference-Comparator-naturalOrder-0]]static <<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/Comparator.html#naturalOrder%2D%2D[naturalOrder]()++ (link:{java9-javadoc}/java/util/Comparator.html#naturalOrder%2D%2D[java 9])
+* ++[[painless-api-reference-Comparator-nullsFirst-1]]static <<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/Comparator.html#nullsFirst%2Djava.util.Comparator%2D[nullsFirst](<<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/Comparator.html#nullsFirst%2Djava.util.Comparator%2D[java 9])
+* ++[[painless-api-reference-Comparator-nullsLast-1]]static <<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/Comparator.html#nullsLast%2Djava.util.Comparator%2D[nullsLast](<<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/Comparator.html#nullsLast%2Djava.util.Comparator%2D[java 9])
+* ++[[painless-api-reference-Comparator-reverseOrder-0]]static <<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/Comparator.html#reverseOrder%2D%2D[reverseOrder]()++ (link:{java9-javadoc}/java/util/Comparator.html#reverseOrder%2D%2D[java 9])
+* ++[[painless-api-reference-Comparator-compare-2]]int link:{java8-javadoc}/java/util/Comparator.html#compare%2Djava.lang.Object%2Djava.lang.Object%2D[compare](def, def)++ (link:{java9-javadoc}/java/util/Comparator.html#compare%2Djava.lang.Object%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Comparator-equals-1]]boolean link:{java8-javadoc}/java/util/Comparator.html#equals%2Djava.lang.Object%2D[equals](<<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/util/Comparator.html#equals%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Comparator-reversed-0]]<<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/Comparator.html#reversed%2D%2D[reversed]()++ (link:{java9-javadoc}/java/util/Comparator.html#reversed%2D%2D[java 9])
+* ++[[painless-api-reference-Comparator-thenComparing-1]]<<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/Comparator.html#thenComparing%2Djava.util.Comparator%2D[thenComparing](<<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/Comparator.html#thenComparing%2Djava.util.Comparator%2D[java 9])
+* ++[[painless-api-reference-Comparator-thenComparing-2]]<<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/Comparator.html#thenComparing%2Djava.util.function.Function%2Djava.util.Comparator%2D[thenComparing](<<painless-api-reference-Function,Function>>, <<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/Comparator.html#thenComparing%2Djava.util.function.Function%2Djava.util.Comparator%2D[java 9])
+* ++[[painless-api-reference-Comparator-thenComparingDouble-1]]<<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/Comparator.html#thenComparingDouble%2Djava.util.function.ToDoubleFunction%2D[thenComparingDouble](<<painless-api-reference-ToDoubleFunction,ToDoubleFunction>>)++ (link:{java9-javadoc}/java/util/Comparator.html#thenComparingDouble%2Djava.util.function.ToDoubleFunction%2D[java 9])
+* ++[[painless-api-reference-Comparator-thenComparingInt-1]]<<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/Comparator.html#thenComparingInt%2Djava.util.function.ToIntFunction%2D[thenComparingInt](<<painless-api-reference-ToIntFunction,ToIntFunction>>)++ (link:{java9-javadoc}/java/util/Comparator.html#thenComparingInt%2Djava.util.function.ToIntFunction%2D[java 9])
+* ++[[painless-api-reference-Comparator-thenComparingLong-1]]<<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/Comparator.html#thenComparingLong%2Djava.util.function.ToLongFunction%2D[thenComparingLong](<<painless-api-reference-ToLongFunction,ToLongFunction>>)++ (link:{java9-javadoc}/java/util/Comparator.html#thenComparingLong%2Djava.util.function.ToLongFunction%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/ConcurrentModificationException.asciidoc b/docs/painless/painless-api-reference/ConcurrentModificationException.asciidoc
new file mode 100644
index 0000000000000..b33eefee1e5ad
--- /dev/null
+++ b/docs/painless/painless-api-reference/ConcurrentModificationException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ConcurrentModificationException]]++ConcurrentModificationException++::
+* ++[[painless-api-reference-ConcurrentModificationException-ConcurrentModificationException-0]]link:{java8-javadoc}/java/util/ConcurrentModificationException.html#ConcurrentModificationException%2D%2D[ConcurrentModificationException]()++ (link:{java9-javadoc}/java/util/ConcurrentModificationException.html#ConcurrentModificationException%2D%2D[java 9])
+* ++[[painless-api-reference-ConcurrentModificationException-ConcurrentModificationException-1]]link:{java8-javadoc}/java/util/ConcurrentModificationException.html#ConcurrentModificationException%2Djava.lang.String%2D[ConcurrentModificationException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/ConcurrentModificationException.html#ConcurrentModificationException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Consumer.asciidoc b/docs/painless/painless-api-reference/Consumer.asciidoc
new file mode 100644
index 0000000000000..a93f47e1a1a7c
--- /dev/null
+++ b/docs/painless/painless-api-reference/Consumer.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Consumer]]++Consumer++::
+* ++[[painless-api-reference-Consumer-accept-1]]void link:{java8-javadoc}/java/util/function/Consumer.html#accept%2Djava.lang.Object%2D[accept](def)++ (link:{java9-javadoc}/java/util/function/Consumer.html#accept%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Consumer-andThen-1]]<<painless-api-reference-Consumer,Consumer>> link:{java8-javadoc}/java/util/function/Consumer.html#andThen%2Djava.util.function.Consumer%2D[andThen](<<painless-api-reference-Consumer,Consumer>>)++ (link:{java9-javadoc}/java/util/function/Consumer.html#andThen%2Djava.util.function.Consumer%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Currency.asciidoc b/docs/painless/painless-api-reference/Currency.asciidoc
new file mode 100644
index 0000000000000..12e172e01eeea
--- /dev/null
+++ b/docs/painless/painless-api-reference/Currency.asciidoc
@@ -0,0 +1,16 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Currency]]++Currency++::
+* ++[[painless-api-reference-Currency-getAvailableCurrencies-0]]static <<painless-api-reference-Set,Set>> link:{java8-javadoc}/java/util/Currency.html#getAvailableCurrencies%2D%2D[getAvailableCurrencies]()++ (link:{java9-javadoc}/java/util/Currency.html#getAvailableCurrencies%2D%2D[java 9])
+* ++[[painless-api-reference-Currency-getInstance-1]]static <<painless-api-reference-Currency,Currency>> link:{java8-javadoc}/java/util/Currency.html#getInstance%2Djava.lang.String%2D[getInstance](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/Currency.html#getInstance%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Currency-getCurrencyCode-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Currency.html#getCurrencyCode%2D%2D[getCurrencyCode]()++ (link:{java9-javadoc}/java/util/Currency.html#getCurrencyCode%2D%2D[java 9])
+* ++[[painless-api-reference-Currency-getDefaultFractionDigits-0]]int link:{java8-javadoc}/java/util/Currency.html#getDefaultFractionDigits%2D%2D[getDefaultFractionDigits]()++ (link:{java9-javadoc}/java/util/Currency.html#getDefaultFractionDigits%2D%2D[java 9])
+* ++[[painless-api-reference-Currency-getDisplayName-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Currency.html#getDisplayName%2D%2D[getDisplayName]()++ (link:{java9-javadoc}/java/util/Currency.html#getDisplayName%2D%2D[java 9])
+* ++[[painless-api-reference-Currency-getDisplayName-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Currency.html#getDisplayName%2Djava.util.Locale%2D[getDisplayName](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/util/Currency.html#getDisplayName%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-Currency-getNumericCode-0]]int link:{java8-javadoc}/java/util/Currency.html#getNumericCode%2D%2D[getNumericCode]()++ (link:{java9-javadoc}/java/util/Currency.html#getNumericCode%2D%2D[java 9])
+* ++[[painless-api-reference-Currency-getSymbol-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Currency.html#getSymbol%2D%2D[getSymbol]()++ (link:{java9-javadoc}/java/util/Currency.html#getSymbol%2D%2D[java 9])
+* ++[[painless-api-reference-Currency-getSymbol-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Currency.html#getSymbol%2Djava.util.Locale%2D[getSymbol](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/util/Currency.html#getSymbol%2Djava.util.Locale%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Date.asciidoc b/docs/painless/painless-api-reference/Date.asciidoc
new file mode 100644
index 0000000000000..5b83f70a0eaae
--- /dev/null
+++ b/docs/painless/painless-api-reference/Date.asciidoc
@@ -0,0 +1,16 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Date]]++Date++::
+* ++[[painless-api-reference-Date-from-1]]static <<painless-api-reference-Date,Date>> link:{java8-javadoc}/java/util/Date.html#from%2Djava.time.Instant%2D[from](<<painless-api-reference-Instant,Instant>>)++ (link:{java9-javadoc}/java/util/Date.html#from%2Djava.time.Instant%2D[java 9])
+* ++[[painless-api-reference-Date-Date-0]]link:{java8-javadoc}/java/util/Date.html#Date%2D%2D[Date]()++ (link:{java9-javadoc}/java/util/Date.html#Date%2D%2D[java 9])
+* ++[[painless-api-reference-Date-Date-1]]link:{java8-javadoc}/java/util/Date.html#Date%2Dlong%2D[Date](long)++ (link:{java9-javadoc}/java/util/Date.html#Date%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Date-after-1]]boolean link:{java8-javadoc}/java/util/Date.html#after%2Djava.util.Date%2D[after](<<painless-api-reference-Date,Date>>)++ (link:{java9-javadoc}/java/util/Date.html#after%2Djava.util.Date%2D[java 9])
+* ++[[painless-api-reference-Date-before-1]]boolean link:{java8-javadoc}/java/util/Date.html#before%2Djava.util.Date%2D[before](<<painless-api-reference-Date,Date>>)++ (link:{java9-javadoc}/java/util/Date.html#before%2Djava.util.Date%2D[java 9])
+* ++[[painless-api-reference-Date-clone-0]]def link:{java8-javadoc}/java/util/Date.html#clone%2D%2D[clone]()++ (link:{java9-javadoc}/java/util/Date.html#clone%2D%2D[java 9])
+* ++[[painless-api-reference-Date-compareTo-1]]int link:{java8-javadoc}/java/util/Date.html#compareTo%2Djava.util.Date%2D[compareTo](<<painless-api-reference-Date,Date>>)++ (link:{java9-javadoc}/java/util/Date.html#compareTo%2Djava.util.Date%2D[java 9])
+* ++[[painless-api-reference-Date-getTime-0]]long link:{java8-javadoc}/java/util/Date.html#getTime%2D%2D[getTime]()++ (link:{java9-javadoc}/java/util/Date.html#getTime%2D%2D[java 9])
+* ++[[painless-api-reference-Date-setTime-1]]void link:{java8-javadoc}/java/util/Date.html#setTime%2Dlong%2D[setTime](long)++ (link:{java9-javadoc}/java/util/Date.html#setTime%2Dlong%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/DateFormat.Field.asciidoc b/docs/painless/painless-api-reference/DateFormat.Field.asciidoc
new file mode 100644
index 0000000000000..b949cfe830cdc
--- /dev/null
+++ b/docs/painless/painless-api-reference/DateFormat.Field.asciidoc
@@ -0,0 +1,27 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-DateFormat-Field]]++DateFormat.Field++::
+** [[painless-api-reference-DateFormat-Field-AM_PM]]static <<painless-api-reference-DateFormat-Field,DateFormat.Field>> link:{java8-javadoc}/java/text/DateFormat.Field.html#AM_PM[AM_PM] (link:{java9-javadoc}/java/text/DateFormat.Field.html#AM_PM[java 9])
+** [[painless-api-reference-DateFormat-Field-DAY_OF_MONTH]]static <<painless-api-reference-DateFormat-Field,DateFormat.Field>> link:{java8-javadoc}/java/text/DateFormat.Field.html#DAY_OF_MONTH[DAY_OF_MONTH] (link:{java9-javadoc}/java/text/DateFormat.Field.html#DAY_OF_MONTH[java 9])
+** [[painless-api-reference-DateFormat-Field-DAY_OF_WEEK]]static <<painless-api-reference-DateFormat-Field,DateFormat.Field>> link:{java8-javadoc}/java/text/DateFormat.Field.html#DAY_OF_WEEK[DAY_OF_WEEK] (link:{java9-javadoc}/java/text/DateFormat.Field.html#DAY_OF_WEEK[java 9])
+** [[painless-api-reference-DateFormat-Field-DAY_OF_WEEK_IN_MONTH]]static <<painless-api-reference-DateFormat-Field,DateFormat.Field>> link:{java8-javadoc}/java/text/DateFormat.Field.html#DAY_OF_WEEK_IN_MONTH[DAY_OF_WEEK_IN_MONTH] (link:{java9-javadoc}/java/text/DateFormat.Field.html#DAY_OF_WEEK_IN_MONTH[java 9])
+** [[painless-api-reference-DateFormat-Field-DAY_OF_YEAR]]static <<painless-api-reference-DateFormat-Field,DateFormat.Field>> link:{java8-javadoc}/java/text/DateFormat.Field.html#DAY_OF_YEAR[DAY_OF_YEAR] (link:{java9-javadoc}/java/text/DateFormat.Field.html#DAY_OF_YEAR[java 9])
+** [[painless-api-reference-DateFormat-Field-ERA]]static <<painless-api-reference-DateFormat-Field,DateFormat.Field>> link:{java8-javadoc}/java/text/DateFormat.Field.html#ERA[ERA] (link:{java9-javadoc}/java/text/DateFormat.Field.html#ERA[java 9])
+** [[painless-api-reference-DateFormat-Field-HOUR0]]static <<painless-api-reference-DateFormat-Field,DateFormat.Field>> link:{java8-javadoc}/java/text/DateFormat.Field.html#HOUR0[HOUR0] (link:{java9-javadoc}/java/text/DateFormat.Field.html#HOUR0[java 9])
+** [[painless-api-reference-DateFormat-Field-HOUR1]]static <<painless-api-reference-DateFormat-Field,DateFormat.Field>> link:{java8-javadoc}/java/text/DateFormat.Field.html#HOUR1[HOUR1] (link:{java9-javadoc}/java/text/DateFormat.Field.html#HOUR1[java 9])
+** [[painless-api-reference-DateFormat-Field-HOUR_OF_DAY0]]static <<painless-api-reference-DateFormat-Field,DateFormat.Field>> link:{java8-javadoc}/java/text/DateFormat.Field.html#HOUR_OF_DAY0[HOUR_OF_DAY0] (link:{java9-javadoc}/java/text/DateFormat.Field.html#HOUR_OF_DAY0[java 9])
+** [[painless-api-reference-DateFormat-Field-HOUR_OF_DAY1]]static <<painless-api-reference-DateFormat-Field,DateFormat.Field>> link:{java8-javadoc}/java/text/DateFormat.Field.html#HOUR_OF_DAY1[HOUR_OF_DAY1] (link:{java9-javadoc}/java/text/DateFormat.Field.html#HOUR_OF_DAY1[java 9])
+** [[painless-api-reference-DateFormat-Field-MILLISECOND]]static <<painless-api-reference-DateFormat-Field,DateFormat.Field>> link:{java8-javadoc}/java/text/DateFormat.Field.html#MILLISECOND[MILLISECOND] (link:{java9-javadoc}/java/text/DateFormat.Field.html#MILLISECOND[java 9])
+** [[painless-api-reference-DateFormat-Field-MINUTE]]static <<painless-api-reference-DateFormat-Field,DateFormat.Field>> link:{java8-javadoc}/java/text/DateFormat.Field.html#MINUTE[MINUTE] (link:{java9-javadoc}/java/text/DateFormat.Field.html#MINUTE[java 9])
+** [[painless-api-reference-DateFormat-Field-MONTH]]static <<painless-api-reference-DateFormat-Field,DateFormat.Field>> link:{java8-javadoc}/java/text/DateFormat.Field.html#MONTH[MONTH] (link:{java9-javadoc}/java/text/DateFormat.Field.html#MONTH[java 9])
+** [[painless-api-reference-DateFormat-Field-SECOND]]static <<painless-api-reference-DateFormat-Field,DateFormat.Field>> link:{java8-javadoc}/java/text/DateFormat.Field.html#SECOND[SECOND] (link:{java9-javadoc}/java/text/DateFormat.Field.html#SECOND[java 9])
+** [[painless-api-reference-DateFormat-Field-TIME_ZONE]]static <<painless-api-reference-DateFormat-Field,DateFormat.Field>> link:{java8-javadoc}/java/text/DateFormat.Field.html#TIME_ZONE[TIME_ZONE] (link:{java9-javadoc}/java/text/DateFormat.Field.html#TIME_ZONE[java 9])
+** [[painless-api-reference-DateFormat-Field-WEEK_OF_MONTH]]static <<painless-api-reference-DateFormat-Field,DateFormat.Field>> link:{java8-javadoc}/java/text/DateFormat.Field.html#WEEK_OF_MONTH[WEEK_OF_MONTH] (link:{java9-javadoc}/java/text/DateFormat.Field.html#WEEK_OF_MONTH[java 9])
+** [[painless-api-reference-DateFormat-Field-WEEK_OF_YEAR]]static <<painless-api-reference-DateFormat-Field,DateFormat.Field>> link:{java8-javadoc}/java/text/DateFormat.Field.html#WEEK_OF_YEAR[WEEK_OF_YEAR] (link:{java9-javadoc}/java/text/DateFormat.Field.html#WEEK_OF_YEAR[java 9])
+** [[painless-api-reference-DateFormat-Field-YEAR]]static <<painless-api-reference-DateFormat-Field,DateFormat.Field>> link:{java8-javadoc}/java/text/DateFormat.Field.html#YEAR[YEAR] (link:{java9-javadoc}/java/text/DateFormat.Field.html#YEAR[java 9])
+* ++[[painless-api-reference-DateFormat-Field-ofCalendarField-1]]static <<painless-api-reference-DateFormat-Field,DateFormat.Field>> link:{java8-javadoc}/java/text/DateFormat.Field.html#ofCalendarField%2Dint%2D[ofCalendarField](int)++ (link:{java9-javadoc}/java/text/DateFormat.Field.html#ofCalendarField%2Dint%2D[java 9])
+* ++[[painless-api-reference-DateFormat-Field-getCalendarField-0]]int link:{java8-javadoc}/java/text/DateFormat.Field.html#getCalendarField%2D%2D[getCalendarField]()++ (link:{java9-javadoc}/java/text/DateFormat.Field.html#getCalendarField%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/DateFormat.asciidoc b/docs/painless/painless-api-reference/DateFormat.asciidoc
new file mode 100644
index 0000000000000..8bb4fdf2eff71
--- /dev/null
+++ b/docs/painless/painless-api-reference/DateFormat.asciidoc
@@ -0,0 +1,51 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-DateFormat]]++DateFormat++::
+** [[painless-api-reference-DateFormat-AM_PM_FIELD]]static int link:{java8-javadoc}/java/text/DateFormat.html#AM_PM_FIELD[AM_PM_FIELD] (link:{java9-javadoc}/java/text/DateFormat.html#AM_PM_FIELD[java 9])
+** [[painless-api-reference-DateFormat-DATE_FIELD]]static int link:{java8-javadoc}/java/text/DateFormat.html#DATE_FIELD[DATE_FIELD] (link:{java9-javadoc}/java/text/DateFormat.html#DATE_FIELD[java 9])
+** [[painless-api-reference-DateFormat-DAY_OF_WEEK_FIELD]]static int link:{java8-javadoc}/java/text/DateFormat.html#DAY_OF_WEEK_FIELD[DAY_OF_WEEK_FIELD] (link:{java9-javadoc}/java/text/DateFormat.html#DAY_OF_WEEK_FIELD[java 9])
+** [[painless-api-reference-DateFormat-DAY_OF_WEEK_IN_MONTH_FIELD]]static int link:{java8-javadoc}/java/text/DateFormat.html#DAY_OF_WEEK_IN_MONTH_FIELD[DAY_OF_WEEK_IN_MONTH_FIELD] (link:{java9-javadoc}/java/text/DateFormat.html#DAY_OF_WEEK_IN_MONTH_FIELD[java 9])
+** [[painless-api-reference-DateFormat-DAY_OF_YEAR_FIELD]]static int link:{java8-javadoc}/java/text/DateFormat.html#DAY_OF_YEAR_FIELD[DAY_OF_YEAR_FIELD] (link:{java9-javadoc}/java/text/DateFormat.html#DAY_OF_YEAR_FIELD[java 9])
+** [[painless-api-reference-DateFormat-DEFAULT]]static int link:{java8-javadoc}/java/text/DateFormat.html#DEFAULT[DEFAULT] (link:{java9-javadoc}/java/text/DateFormat.html#DEFAULT[java 9])
+** [[painless-api-reference-DateFormat-ERA_FIELD]]static int link:{java8-javadoc}/java/text/DateFormat.html#ERA_FIELD[ERA_FIELD] (link:{java9-javadoc}/java/text/DateFormat.html#ERA_FIELD[java 9])
+** [[painless-api-reference-DateFormat-FULL]]static int link:{java8-javadoc}/java/text/DateFormat.html#FULL[FULL] (link:{java9-javadoc}/java/text/DateFormat.html#FULL[java 9])
+** [[painless-api-reference-DateFormat-HOUR0_FIELD]]static int link:{java8-javadoc}/java/text/DateFormat.html#HOUR0_FIELD[HOUR0_FIELD] (link:{java9-javadoc}/java/text/DateFormat.html#HOUR0_FIELD[java 9])
+** [[painless-api-reference-DateFormat-HOUR1_FIELD]]static int link:{java8-javadoc}/java/text/DateFormat.html#HOUR1_FIELD[HOUR1_FIELD] (link:{java9-javadoc}/java/text/DateFormat.html#HOUR1_FIELD[java 9])
+** [[painless-api-reference-DateFormat-HOUR_OF_DAY0_FIELD]]static int link:{java8-javadoc}/java/text/DateFormat.html#HOUR_OF_DAY0_FIELD[HOUR_OF_DAY0_FIELD] (link:{java9-javadoc}/java/text/DateFormat.html#HOUR_OF_DAY0_FIELD[java 9])
+** [[painless-api-reference-DateFormat-HOUR_OF_DAY1_FIELD]]static int link:{java8-javadoc}/java/text/DateFormat.html#HOUR_OF_DAY1_FIELD[HOUR_OF_DAY1_FIELD] (link:{java9-javadoc}/java/text/DateFormat.html#HOUR_OF_DAY1_FIELD[java 9])
+** [[painless-api-reference-DateFormat-LONG]]static int link:{java8-javadoc}/java/text/DateFormat.html#LONG[LONG] (link:{java9-javadoc}/java/text/DateFormat.html#LONG[java 9])
+** [[painless-api-reference-DateFormat-MEDIUM]]static int link:{java8-javadoc}/java/text/DateFormat.html#MEDIUM[MEDIUM] (link:{java9-javadoc}/java/text/DateFormat.html#MEDIUM[java 9])
+** [[painless-api-reference-DateFormat-MILLISECOND_FIELD]]static int link:{java8-javadoc}/java/text/DateFormat.html#MILLISECOND_FIELD[MILLISECOND_FIELD] (link:{java9-javadoc}/java/text/DateFormat.html#MILLISECOND_FIELD[java 9])
+** [[painless-api-reference-DateFormat-MINUTE_FIELD]]static int link:{java8-javadoc}/java/text/DateFormat.html#MINUTE_FIELD[MINUTE_FIELD] (link:{java9-javadoc}/java/text/DateFormat.html#MINUTE_FIELD[java 9])
+** [[painless-api-reference-DateFormat-MONTH_FIELD]]static int link:{java8-javadoc}/java/text/DateFormat.html#MONTH_FIELD[MONTH_FIELD] (link:{java9-javadoc}/java/text/DateFormat.html#MONTH_FIELD[java 9])
+** [[painless-api-reference-DateFormat-SECOND_FIELD]]static int link:{java8-javadoc}/java/text/DateFormat.html#SECOND_FIELD[SECOND_FIELD] (link:{java9-javadoc}/java/text/DateFormat.html#SECOND_FIELD[java 9])
+** [[painless-api-reference-DateFormat-SHORT]]static int link:{java8-javadoc}/java/text/DateFormat.html#SHORT[SHORT] (link:{java9-javadoc}/java/text/DateFormat.html#SHORT[java 9])
+** [[painless-api-reference-DateFormat-TIMEZONE_FIELD]]static int link:{java8-javadoc}/java/text/DateFormat.html#TIMEZONE_FIELD[TIMEZONE_FIELD] (link:{java9-javadoc}/java/text/DateFormat.html#TIMEZONE_FIELD[java 9])
+** [[painless-api-reference-DateFormat-WEEK_OF_MONTH_FIELD]]static int link:{java8-javadoc}/java/text/DateFormat.html#WEEK_OF_MONTH_FIELD[WEEK_OF_MONTH_FIELD] (link:{java9-javadoc}/java/text/DateFormat.html#WEEK_OF_MONTH_FIELD[java 9])
+** [[painless-api-reference-DateFormat-WEEK_OF_YEAR_FIELD]]static int link:{java8-javadoc}/java/text/DateFormat.html#WEEK_OF_YEAR_FIELD[WEEK_OF_YEAR_FIELD] (link:{java9-javadoc}/java/text/DateFormat.html#WEEK_OF_YEAR_FIELD[java 9])
+** [[painless-api-reference-DateFormat-YEAR_FIELD]]static int link:{java8-javadoc}/java/text/DateFormat.html#YEAR_FIELD[YEAR_FIELD] (link:{java9-javadoc}/java/text/DateFormat.html#YEAR_FIELD[java 9])
+* ++[[painless-api-reference-DateFormat-getAvailableLocales-0]]static <<painless-api-reference-Locale,Locale>>[] link:{java8-javadoc}/java/text/DateFormat.html#getAvailableLocales%2D%2D[getAvailableLocales]()++ (link:{java9-javadoc}/java/text/DateFormat.html#getAvailableLocales%2D%2D[java 9])
+* ++[[painless-api-reference-DateFormat-getDateInstance-0]]static <<painless-api-reference-DateFormat,DateFormat>> link:{java8-javadoc}/java/text/DateFormat.html#getDateInstance%2D%2D[getDateInstance]()++ (link:{java9-javadoc}/java/text/DateFormat.html#getDateInstance%2D%2D[java 9])
+* ++[[painless-api-reference-DateFormat-getDateInstance-1]]static <<painless-api-reference-DateFormat,DateFormat>> link:{java8-javadoc}/java/text/DateFormat.html#getDateInstance%2Dint%2D[getDateInstance](int)++ (link:{java9-javadoc}/java/text/DateFormat.html#getDateInstance%2Dint%2D[java 9])
+* ++[[painless-api-reference-DateFormat-getDateInstance-2]]static <<painless-api-reference-DateFormat,DateFormat>> link:{java8-javadoc}/java/text/DateFormat.html#getDateInstance%2Dint%2Djava.util.Locale%2D[getDateInstance](int, <<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/text/DateFormat.html#getDateInstance%2Dint%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-DateFormat-getDateTimeInstance-0]]static <<painless-api-reference-DateFormat,DateFormat>> link:{java8-javadoc}/java/text/DateFormat.html#getDateTimeInstance%2D%2D[getDateTimeInstance]()++ (link:{java9-javadoc}/java/text/DateFormat.html#getDateTimeInstance%2D%2D[java 9])
+* ++[[painless-api-reference-DateFormat-getDateTimeInstance-2]]static <<painless-api-reference-DateFormat,DateFormat>> link:{java8-javadoc}/java/text/DateFormat.html#getDateTimeInstance%2Dint%2Dint%2D[getDateTimeInstance](int, int)++ (link:{java9-javadoc}/java/text/DateFormat.html#getDateTimeInstance%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-DateFormat-getDateTimeInstance-3]]static <<painless-api-reference-DateFormat,DateFormat>> link:{java8-javadoc}/java/text/DateFormat.html#getDateTimeInstance%2Dint%2Dint%2Djava.util.Locale%2D[getDateTimeInstance](int, int, <<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/text/DateFormat.html#getDateTimeInstance%2Dint%2Dint%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-DateFormat-getInstance-0]]static <<painless-api-reference-DateFormat,DateFormat>> link:{java8-javadoc}/java/text/DateFormat.html#getInstance%2D%2D[getInstance]()++ (link:{java9-javadoc}/java/text/DateFormat.html#getInstance%2D%2D[java 9])
+* ++[[painless-api-reference-DateFormat-getTimeInstance-0]]static <<painless-api-reference-DateFormat,DateFormat>> link:{java8-javadoc}/java/text/DateFormat.html#getTimeInstance%2D%2D[getTimeInstance]()++ (link:{java9-javadoc}/java/text/DateFormat.html#getTimeInstance%2D%2D[java 9])
+* ++[[painless-api-reference-DateFormat-getTimeInstance-1]]static <<painless-api-reference-DateFormat,DateFormat>> link:{java8-javadoc}/java/text/DateFormat.html#getTimeInstance%2Dint%2D[getTimeInstance](int)++ (link:{java9-javadoc}/java/text/DateFormat.html#getTimeInstance%2Dint%2D[java 9])
+* ++[[painless-api-reference-DateFormat-getTimeInstance-2]]static <<painless-api-reference-DateFormat,DateFormat>> link:{java8-javadoc}/java/text/DateFormat.html#getTimeInstance%2Dint%2Djava.util.Locale%2D[getTimeInstance](int, <<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/text/DateFormat.html#getTimeInstance%2Dint%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-DateFormat-getCalendar-0]]<<painless-api-reference-Calendar,Calendar>> link:{java8-javadoc}/java/text/DateFormat.html#getCalendar%2D%2D[getCalendar]()++ (link:{java9-javadoc}/java/text/DateFormat.html#getCalendar%2D%2D[java 9])
+* ++[[painless-api-reference-DateFormat-getNumberFormat-0]]<<painless-api-reference-NumberFormat,NumberFormat>> link:{java8-javadoc}/java/text/DateFormat.html#getNumberFormat%2D%2D[getNumberFormat]()++ (link:{java9-javadoc}/java/text/DateFormat.html#getNumberFormat%2D%2D[java 9])
+* ++[[painless-api-reference-DateFormat-getTimeZone-0]]<<painless-api-reference-TimeZone,TimeZone>> link:{java8-javadoc}/java/text/DateFormat.html#getTimeZone%2D%2D[getTimeZone]()++ (link:{java9-javadoc}/java/text/DateFormat.html#getTimeZone%2D%2D[java 9])
+* ++[[painless-api-reference-DateFormat-isLenient-0]]boolean link:{java8-javadoc}/java/text/DateFormat.html#isLenient%2D%2D[isLenient]()++ (link:{java9-javadoc}/java/text/DateFormat.html#isLenient%2D%2D[java 9])
+* ++[[painless-api-reference-DateFormat-parse-1]]<<painless-api-reference-Date,Date>> link:{java8-javadoc}/java/text/DateFormat.html#parse%2Djava.lang.String%2D[parse](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/DateFormat.html#parse%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-DateFormat-parse-2]]<<painless-api-reference-Date,Date>> link:{java8-javadoc}/java/text/DateFormat.html#parse%2Djava.lang.String%2Djava.text.ParsePosition%2D[parse](<<painless-api-reference-String,String>>, <<painless-api-reference-ParsePosition,ParsePosition>>)++ (link:{java9-javadoc}/java/text/DateFormat.html#parse%2Djava.lang.String%2Djava.text.ParsePosition%2D[java 9])
+* ++[[painless-api-reference-DateFormat-setCalendar-1]]void link:{java8-javadoc}/java/text/DateFormat.html#setCalendar%2Djava.util.Calendar%2D[setCalendar](<<painless-api-reference-Calendar,Calendar>>)++ (link:{java9-javadoc}/java/text/DateFormat.html#setCalendar%2Djava.util.Calendar%2D[java 9])
+* ++[[painless-api-reference-DateFormat-setLenient-1]]void link:{java8-javadoc}/java/text/DateFormat.html#setLenient%2Dboolean%2D[setLenient](boolean)++ (link:{java9-javadoc}/java/text/DateFormat.html#setLenient%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-DateFormat-setNumberFormat-1]]void link:{java8-javadoc}/java/text/DateFormat.html#setNumberFormat%2Djava.text.NumberFormat%2D[setNumberFormat](<<painless-api-reference-NumberFormat,NumberFormat>>)++ (link:{java9-javadoc}/java/text/DateFormat.html#setNumberFormat%2Djava.text.NumberFormat%2D[java 9])
+* ++[[painless-api-reference-DateFormat-setTimeZone-1]]void link:{java8-javadoc}/java/text/DateFormat.html#setTimeZone%2Djava.util.TimeZone%2D[setTimeZone](<<painless-api-reference-TimeZone,TimeZone>>)++ (link:{java9-javadoc}/java/text/DateFormat.html#setTimeZone%2Djava.util.TimeZone%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Format,Format>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/DateFormatSymbols.asciidoc b/docs/painless/painless-api-reference/DateFormatSymbols.asciidoc
new file mode 100644
index 0000000000000..31cf39bbc4586
--- /dev/null
+++ b/docs/painless/painless-api-reference/DateFormatSymbols.asciidoc
@@ -0,0 +1,30 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-DateFormatSymbols]]++DateFormatSymbols++::
+* ++[[painless-api-reference-DateFormatSymbols-getAvailableLocales-0]]static <<painless-api-reference-Locale,Locale>>[] link:{java8-javadoc}/java/text/DateFormatSymbols.html#getAvailableLocales%2D%2D[getAvailableLocales]()++ (link:{java9-javadoc}/java/text/DateFormatSymbols.html#getAvailableLocales%2D%2D[java 9])
+* ++[[painless-api-reference-DateFormatSymbols-getInstance-0]]static <<painless-api-reference-DateFormatSymbols,DateFormatSymbols>> link:{java8-javadoc}/java/text/DateFormatSymbols.html#getInstance%2D%2D[getInstance]()++ (link:{java9-javadoc}/java/text/DateFormatSymbols.html#getInstance%2D%2D[java 9])
+* ++[[painless-api-reference-DateFormatSymbols-getInstance-1]]static <<painless-api-reference-DateFormatSymbols,DateFormatSymbols>> link:{java8-javadoc}/java/text/DateFormatSymbols.html#getInstance%2Djava.util.Locale%2D[getInstance](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/text/DateFormatSymbols.html#getInstance%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-DateFormatSymbols-DateFormatSymbols-0]]link:{java8-javadoc}/java/text/DateFormatSymbols.html#DateFormatSymbols%2D%2D[DateFormatSymbols]()++ (link:{java9-javadoc}/java/text/DateFormatSymbols.html#DateFormatSymbols%2D%2D[java 9])
+* ++[[painless-api-reference-DateFormatSymbols-DateFormatSymbols-1]]link:{java8-javadoc}/java/text/DateFormatSymbols.html#DateFormatSymbols%2Djava.util.Locale%2D[DateFormatSymbols](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/text/DateFormatSymbols.html#DateFormatSymbols%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-DateFormatSymbols-clone-0]]def link:{java8-javadoc}/java/text/DateFormatSymbols.html#clone%2D%2D[clone]()++ (link:{java9-javadoc}/java/text/DateFormatSymbols.html#clone%2D%2D[java 9])
+* ++[[painless-api-reference-DateFormatSymbols-getAmPmStrings-0]]<<painless-api-reference-String,String>>[] link:{java8-javadoc}/java/text/DateFormatSymbols.html#getAmPmStrings%2D%2D[getAmPmStrings]()++ (link:{java9-javadoc}/java/text/DateFormatSymbols.html#getAmPmStrings%2D%2D[java 9])
+* ++[[painless-api-reference-DateFormatSymbols-getEras-0]]<<painless-api-reference-String,String>>[] link:{java8-javadoc}/java/text/DateFormatSymbols.html#getEras%2D%2D[getEras]()++ (link:{java9-javadoc}/java/text/DateFormatSymbols.html#getEras%2D%2D[java 9])
+* ++[[painless-api-reference-DateFormatSymbols-getLocalPatternChars-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/text/DateFormatSymbols.html#getLocalPatternChars%2D%2D[getLocalPatternChars]()++ (link:{java9-javadoc}/java/text/DateFormatSymbols.html#getLocalPatternChars%2D%2D[java 9])
+* ++[[painless-api-reference-DateFormatSymbols-getMonths-0]]<<painless-api-reference-String,String>>[] link:{java8-javadoc}/java/text/DateFormatSymbols.html#getMonths%2D%2D[getMonths]()++ (link:{java9-javadoc}/java/text/DateFormatSymbols.html#getMonths%2D%2D[java 9])
+* ++[[painless-api-reference-DateFormatSymbols-getShortMonths-0]]<<painless-api-reference-String,String>>[] link:{java8-javadoc}/java/text/DateFormatSymbols.html#getShortMonths%2D%2D[getShortMonths]()++ (link:{java9-javadoc}/java/text/DateFormatSymbols.html#getShortMonths%2D%2D[java 9])
+* ++[[painless-api-reference-DateFormatSymbols-getShortWeekdays-0]]<<painless-api-reference-String,String>>[] link:{java8-javadoc}/java/text/DateFormatSymbols.html#getShortWeekdays%2D%2D[getShortWeekdays]()++ (link:{java9-javadoc}/java/text/DateFormatSymbols.html#getShortWeekdays%2D%2D[java 9])
+* ++[[painless-api-reference-DateFormatSymbols-getWeekdays-0]]<<painless-api-reference-String,String>>[] link:{java8-javadoc}/java/text/DateFormatSymbols.html#getWeekdays%2D%2D[getWeekdays]()++ (link:{java9-javadoc}/java/text/DateFormatSymbols.html#getWeekdays%2D%2D[java 9])
+* ++[[painless-api-reference-DateFormatSymbols-getZoneStrings-0]]<<painless-api-reference-String,String>>[][] link:{java8-javadoc}/java/text/DateFormatSymbols.html#getZoneStrings%2D%2D[getZoneStrings]()++ (link:{java9-javadoc}/java/text/DateFormatSymbols.html#getZoneStrings%2D%2D[java 9])
+* ++[[painless-api-reference-DateFormatSymbols-hashCode-0]]int link:{java8-javadoc}/java/text/DateFormatSymbols.html#hashCode%2D%2D[hashCode]()++ (link:{java9-javadoc}/java/text/DateFormatSymbols.html#hashCode%2D%2D[java 9])
+* ++[[painless-api-reference-DateFormatSymbols-setAmPmStrings-1]]void link:{java8-javadoc}/java/text/DateFormatSymbols.html#setAmPmStrings%2Djava.lang.String:A%2D[setAmPmStrings](<<painless-api-reference-String,String>>[])++ (link:{java9-javadoc}/java/text/DateFormatSymbols.html#setAmPmStrings%2Djava.lang.String:A%2D[java 9])
+* ++[[painless-api-reference-DateFormatSymbols-setEras-1]]void link:{java8-javadoc}/java/text/DateFormatSymbols.html#setEras%2Djava.lang.String:A%2D[setEras](<<painless-api-reference-String,String>>[])++ (link:{java9-javadoc}/java/text/DateFormatSymbols.html#setEras%2Djava.lang.String:A%2D[java 9])
+* ++[[painless-api-reference-DateFormatSymbols-setLocalPatternChars-1]]void link:{java8-javadoc}/java/text/DateFormatSymbols.html#setLocalPatternChars%2Djava.lang.String%2D[setLocalPatternChars](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/DateFormatSymbols.html#setLocalPatternChars%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-DateFormatSymbols-setMonths-1]]void link:{java8-javadoc}/java/text/DateFormatSymbols.html#setMonths%2Djava.lang.String:A%2D[setMonths](<<painless-api-reference-String,String>>[])++ (link:{java9-javadoc}/java/text/DateFormatSymbols.html#setMonths%2Djava.lang.String:A%2D[java 9])
+* ++[[painless-api-reference-DateFormatSymbols-setShortMonths-1]]void link:{java8-javadoc}/java/text/DateFormatSymbols.html#setShortMonths%2Djava.lang.String:A%2D[setShortMonths](<<painless-api-reference-String,String>>[])++ (link:{java9-javadoc}/java/text/DateFormatSymbols.html#setShortMonths%2Djava.lang.String:A%2D[java 9])
+* ++[[painless-api-reference-DateFormatSymbols-setShortWeekdays-1]]void link:{java8-javadoc}/java/text/DateFormatSymbols.html#setShortWeekdays%2Djava.lang.String:A%2D[setShortWeekdays](<<painless-api-reference-String,String>>[])++ (link:{java9-javadoc}/java/text/DateFormatSymbols.html#setShortWeekdays%2Djava.lang.String:A%2D[java 9])
+* ++[[painless-api-reference-DateFormatSymbols-setWeekdays-1]]void link:{java8-javadoc}/java/text/DateFormatSymbols.html#setWeekdays%2Djava.lang.String:A%2D[setWeekdays](<<painless-api-reference-String,String>>[])++ (link:{java9-javadoc}/java/text/DateFormatSymbols.html#setWeekdays%2Djava.lang.String:A%2D[java 9])
+* ++[[painless-api-reference-DateFormatSymbols-setZoneStrings-1]]void link:{java8-javadoc}/java/text/DateFormatSymbols.html#setZoneStrings%2Djava.lang.String:A%2D[setZoneStrings](<<painless-api-reference-String,String>>[][])++ (link:{java9-javadoc}/java/text/DateFormatSymbols.html#setZoneStrings%2Djava.lang.String:A%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/DateTimeException.asciidoc b/docs/painless/painless-api-reference/DateTimeException.asciidoc
new file mode 100644
index 0000000000000..5a9953c5bd582
--- /dev/null
+++ b/docs/painless/painless-api-reference/DateTimeException.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-DateTimeException]]++DateTimeException++::
+* ++[[painless-api-reference-DateTimeException-DateTimeException-1]]link:{java8-javadoc}/java/time/DateTimeException.html#DateTimeException%2Djava.lang.String%2D[DateTimeException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/DateTimeException.html#DateTimeException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/DateTimeFormatter.asciidoc b/docs/painless/painless-api-reference/DateTimeFormatter.asciidoc
new file mode 100644
index 0000000000000..ad104dfe5d6d2
--- /dev/null
+++ b/docs/painless/painless-api-reference/DateTimeFormatter.asciidoc
@@ -0,0 +1,50 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-DateTimeFormatter]]++DateTimeFormatter++::
+** [[painless-api-reference-DateTimeFormatter-BASIC_ISO_DATE]]static <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#BASIC_ISO_DATE[BASIC_ISO_DATE] (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#BASIC_ISO_DATE[java 9])
+** [[painless-api-reference-DateTimeFormatter-ISO_DATE]]static <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#ISO_DATE[ISO_DATE] (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#ISO_DATE[java 9])
+** [[painless-api-reference-DateTimeFormatter-ISO_DATE_TIME]]static <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#ISO_DATE_TIME[ISO_DATE_TIME] (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#ISO_DATE_TIME[java 9])
+** [[painless-api-reference-DateTimeFormatter-ISO_INSTANT]]static <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#ISO_INSTANT[ISO_INSTANT] (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#ISO_INSTANT[java 9])
+** [[painless-api-reference-DateTimeFormatter-ISO_LOCAL_DATE]]static <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#ISO_LOCAL_DATE[ISO_LOCAL_DATE] (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#ISO_LOCAL_DATE[java 9])
+** [[painless-api-reference-DateTimeFormatter-ISO_LOCAL_DATE_TIME]]static <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#ISO_LOCAL_DATE_TIME[ISO_LOCAL_DATE_TIME] (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#ISO_LOCAL_DATE_TIME[java 9])
+** [[painless-api-reference-DateTimeFormatter-ISO_LOCAL_TIME]]static <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#ISO_LOCAL_TIME[ISO_LOCAL_TIME] (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#ISO_LOCAL_TIME[java 9])
+** [[painless-api-reference-DateTimeFormatter-ISO_OFFSET_DATE]]static <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#ISO_OFFSET_DATE[ISO_OFFSET_DATE] (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#ISO_OFFSET_DATE[java 9])
+** [[painless-api-reference-DateTimeFormatter-ISO_OFFSET_DATE_TIME]]static <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#ISO_OFFSET_DATE_TIME[ISO_OFFSET_DATE_TIME] (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#ISO_OFFSET_DATE_TIME[java 9])
+** [[painless-api-reference-DateTimeFormatter-ISO_OFFSET_TIME]]static <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#ISO_OFFSET_TIME[ISO_OFFSET_TIME] (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#ISO_OFFSET_TIME[java 9])
+** [[painless-api-reference-DateTimeFormatter-ISO_ORDINAL_DATE]]static <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#ISO_ORDINAL_DATE[ISO_ORDINAL_DATE] (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#ISO_ORDINAL_DATE[java 9])
+** [[painless-api-reference-DateTimeFormatter-ISO_TIME]]static <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#ISO_TIME[ISO_TIME] (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#ISO_TIME[java 9])
+** [[painless-api-reference-DateTimeFormatter-ISO_WEEK_DATE]]static <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#ISO_WEEK_DATE[ISO_WEEK_DATE] (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#ISO_WEEK_DATE[java 9])
+** [[painless-api-reference-DateTimeFormatter-ISO_ZONED_DATE_TIME]]static <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#ISO_ZONED_DATE_TIME[ISO_ZONED_DATE_TIME] (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#ISO_ZONED_DATE_TIME[java 9])
+** [[painless-api-reference-DateTimeFormatter-RFC_1123_DATE_TIME]]static <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#RFC_1123_DATE_TIME[RFC_1123_DATE_TIME] (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#RFC_1123_DATE_TIME[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-ofLocalizedDate-1]]static <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#ofLocalizedDate%2Djava.time.format.FormatStyle%2D[ofLocalizedDate](<<painless-api-reference-FormatStyle,FormatStyle>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#ofLocalizedDate%2Djava.time.format.FormatStyle%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-ofLocalizedDateTime-1]]static <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#ofLocalizedDateTime%2Djava.time.format.FormatStyle%2D[ofLocalizedDateTime](<<painless-api-reference-FormatStyle,FormatStyle>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#ofLocalizedDateTime%2Djava.time.format.FormatStyle%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-ofLocalizedDateTime-2]]static <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#ofLocalizedDateTime%2Djava.time.format.FormatStyle%2Djava.time.format.FormatStyle%2D[ofLocalizedDateTime](<<painless-api-reference-FormatStyle,FormatStyle>>, <<painless-api-reference-FormatStyle,FormatStyle>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#ofLocalizedDateTime%2Djava.time.format.FormatStyle%2Djava.time.format.FormatStyle%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-ofLocalizedTime-1]]static <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#ofLocalizedTime%2Djava.time.format.FormatStyle%2D[ofLocalizedTime](<<painless-api-reference-FormatStyle,FormatStyle>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#ofLocalizedTime%2Djava.time.format.FormatStyle%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-ofPattern-1]]static <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#ofPattern%2Djava.lang.String%2D[ofPattern](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#ofPattern%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-ofPattern-2]]static <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#ofPattern%2Djava.lang.String%2Djava.util.Locale%2D[ofPattern](<<painless-api-reference-String,String>>, <<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#ofPattern%2Djava.lang.String%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-parsedExcessDays-0]]static <<painless-api-reference-TemporalQuery,TemporalQuery>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#parsedExcessDays%2D%2D[parsedExcessDays]()++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#parsedExcessDays%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-parsedLeapSecond-0]]static <<painless-api-reference-TemporalQuery,TemporalQuery>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#parsedLeapSecond%2D%2D[parsedLeapSecond]()++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#parsedLeapSecond%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-format-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#format%2Djava.time.temporal.TemporalAccessor%2D[format](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#format%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-formatTo-2]]void link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#formatTo%2Djava.time.temporal.TemporalAccessor%2Djava.lang.Appendable%2D[formatTo](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>, <<painless-api-reference-Appendable,Appendable>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#formatTo%2Djava.time.temporal.TemporalAccessor%2Djava.lang.Appendable%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-getChronology-0]]<<painless-api-reference-Chronology,Chronology>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#getChronology%2D%2D[getChronology]()++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#getChronology%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-getDecimalStyle-0]]<<painless-api-reference-DecimalStyle,DecimalStyle>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#getDecimalStyle%2D%2D[getDecimalStyle]()++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#getDecimalStyle%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-getLocale-0]]<<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#getLocale%2D%2D[getLocale]()++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#getLocale%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-getResolverFields-0]]<<painless-api-reference-Set,Set>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#getResolverFields%2D%2D[getResolverFields]()++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#getResolverFields%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-getResolverStyle-0]]<<painless-api-reference-ResolverStyle,ResolverStyle>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#getResolverStyle%2D%2D[getResolverStyle]()++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#getResolverStyle%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-getZone-0]]<<painless-api-reference-ZoneId,ZoneId>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#getZone%2D%2D[getZone]()++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#getZone%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-parse-1]]<<painless-api-reference-TemporalAccessor,TemporalAccessor>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#parse%2Djava.lang.CharSequence%2D[parse](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#parse%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-parse-2]]def link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#parse%2Djava.lang.CharSequence%2Djava.time.temporal.TemporalQuery%2D[parse](<<painless-api-reference-CharSequence,CharSequence>>, <<painless-api-reference-TemporalQuery,TemporalQuery>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#parse%2Djava.lang.CharSequence%2Djava.time.temporal.TemporalQuery%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-parseBest-2]]<<painless-api-reference-TemporalAccessor,TemporalAccessor>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#parseBest%2Djava.lang.CharSequence%2Djava.time.temporal.TemporalQuery:A%2D[parseBest](<<painless-api-reference-CharSequence,CharSequence>>, <<painless-api-reference-TemporalQuery,TemporalQuery>>[])++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#parseBest%2Djava.lang.CharSequence%2Djava.time.temporal.TemporalQuery:A%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-parseUnresolved-2]]<<painless-api-reference-TemporalAccessor,TemporalAccessor>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#parseUnresolved%2Djava.lang.CharSequence%2Djava.text.ParsePosition%2D[parseUnresolved](<<painless-api-reference-CharSequence,CharSequence>>, <<painless-api-reference-ParsePosition,ParsePosition>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#parseUnresolved%2Djava.lang.CharSequence%2Djava.text.ParsePosition%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-toFormat-0]]<<painless-api-reference-Format,Format>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#toFormat%2D%2D[toFormat]()++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#toFormat%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-toFormat-1]]<<painless-api-reference-Format,Format>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#toFormat%2Djava.time.temporal.TemporalQuery%2D[toFormat](<<painless-api-reference-TemporalQuery,TemporalQuery>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#toFormat%2Djava.time.temporal.TemporalQuery%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-withChronology-1]]<<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#withChronology%2Djava.time.chrono.Chronology%2D[withChronology](<<painless-api-reference-Chronology,Chronology>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#withChronology%2Djava.time.chrono.Chronology%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-withDecimalStyle-1]]<<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#withDecimalStyle%2Djava.time.format.DecimalStyle%2D[withDecimalStyle](<<painless-api-reference-DecimalStyle,DecimalStyle>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#withDecimalStyle%2Djava.time.format.DecimalStyle%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-withLocale-1]]<<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#withLocale%2Djava.util.Locale%2D[withLocale](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#withLocale%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-withResolverFields-1]]<<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#withResolverFields%2Djava.util.Set%2D[withResolverFields](<<painless-api-reference-Set,Set>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#withResolverFields%2Djava.util.Set%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-withResolverStyle-1]]<<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#withResolverStyle%2Djava.time.format.ResolverStyle%2D[withResolverStyle](<<painless-api-reference-ResolverStyle,ResolverStyle>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#withResolverStyle%2Djava.time.format.ResolverStyle%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-withZone-1]]<<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#withZone%2Djava.time.ZoneId%2D[withZone](<<painless-api-reference-ZoneId,ZoneId>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#withZone%2Djava.time.ZoneId%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/DateTimeFormatterBuilder.asciidoc b/docs/painless/painless-api-reference/DateTimeFormatterBuilder.asciidoc
new file mode 100644
index 0000000000000..cc352560f41a9
--- /dev/null
+++ b/docs/painless/painless-api-reference/DateTimeFormatterBuilder.asciidoc
@@ -0,0 +1,44 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-DateTimeFormatterBuilder]]++DateTimeFormatterBuilder++::
+* ++[[painless-api-reference-DateTimeFormatterBuilder-getLocalizedDateTimePattern-4]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#getLocalizedDateTimePattern%2Djava.time.format.FormatStyle%2Djava.time.format.FormatStyle%2Djava.time.chrono.Chronology%2Djava.util.Locale%2D[getLocalizedDateTimePattern](<<painless-api-reference-FormatStyle,FormatStyle>>, <<painless-api-reference-FormatStyle,FormatStyle>>, <<painless-api-reference-Chronology,Chronology>>, <<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#getLocalizedDateTimePattern%2Djava.time.format.FormatStyle%2Djava.time.format.FormatStyle%2Djava.time.chrono.Chronology%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-DateTimeFormatterBuilder-0]]link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#DateTimeFormatterBuilder%2D%2D[DateTimeFormatterBuilder]()++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#DateTimeFormatterBuilder%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-append-1]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#append%2Djava.time.format.DateTimeFormatter%2D[append](<<painless-api-reference-DateTimeFormatter,DateTimeFormatter>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#append%2Djava.time.format.DateTimeFormatter%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-appendChronologyId-0]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendChronologyId%2D%2D[appendChronologyId]()++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendChronologyId%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-appendChronologyText-1]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendChronologyText%2Djava.time.format.TextStyle%2D[appendChronologyText](<<painless-api-reference-TextStyle,TextStyle>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendChronologyText%2Djava.time.format.TextStyle%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-appendFraction-4]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendFraction%2Djava.time.temporal.TemporalField%2Dint%2Dint%2Dboolean%2D[appendFraction](<<painless-api-reference-TemporalField,TemporalField>>, int, int, boolean)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendFraction%2Djava.time.temporal.TemporalField%2Dint%2Dint%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-appendInstant-0]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendInstant%2D%2D[appendInstant]()++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendInstant%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-appendInstant-1]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendInstant%2Dint%2D[appendInstant](int)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendInstant%2Dint%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-appendLiteral-1]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendLiteral%2Djava.lang.String%2D[appendLiteral](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendLiteral%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-appendLocalized-2]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendLocalized%2Djava.time.format.FormatStyle%2Djava.time.format.FormatStyle%2D[appendLocalized](<<painless-api-reference-FormatStyle,FormatStyle>>, <<painless-api-reference-FormatStyle,FormatStyle>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendLocalized%2Djava.time.format.FormatStyle%2Djava.time.format.FormatStyle%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-appendLocalizedOffset-1]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendLocalizedOffset%2Djava.time.format.TextStyle%2D[appendLocalizedOffset](<<painless-api-reference-TextStyle,TextStyle>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendLocalizedOffset%2Djava.time.format.TextStyle%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-appendOffset-2]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendOffset%2Djava.lang.String%2Djava.lang.String%2D[appendOffset](<<painless-api-reference-String,String>>, <<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendOffset%2Djava.lang.String%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-appendOffsetId-0]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendOffsetId%2D%2D[appendOffsetId]()++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendOffsetId%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-appendOptional-1]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendOptional%2Djava.time.format.DateTimeFormatter%2D[appendOptional](<<painless-api-reference-DateTimeFormatter,DateTimeFormatter>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendOptional%2Djava.time.format.DateTimeFormatter%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-appendPattern-1]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendPattern%2Djava.lang.String%2D[appendPattern](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendPattern%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-appendText-1]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendText%2Djava.time.temporal.TemporalField%2D[appendText](<<painless-api-reference-TemporalField,TemporalField>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendText%2Djava.time.temporal.TemporalField%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-appendText-2]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendText%2Djava.time.temporal.TemporalField%2Djava.time.format.TextStyle%2D[appendText](<<painless-api-reference-TemporalField,TemporalField>>, <<painless-api-reference-TextStyle,TextStyle>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendText%2Djava.time.temporal.TemporalField%2Djava.time.format.TextStyle%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-appendValue-1]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendValue%2Djava.time.temporal.TemporalField%2D[appendValue](<<painless-api-reference-TemporalField,TemporalField>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendValue%2Djava.time.temporal.TemporalField%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-appendValue-2]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendValue%2Djava.time.temporal.TemporalField%2Dint%2D[appendValue](<<painless-api-reference-TemporalField,TemporalField>>, int)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendValue%2Djava.time.temporal.TemporalField%2Dint%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-appendValue-4]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendValue%2Djava.time.temporal.TemporalField%2Dint%2Dint%2Djava.time.format.SignStyle%2D[appendValue](<<painless-api-reference-TemporalField,TemporalField>>, int, int, <<painless-api-reference-SignStyle,SignStyle>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendValue%2Djava.time.temporal.TemporalField%2Dint%2Dint%2Djava.time.format.SignStyle%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-appendValueReduced-4]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendValueReduced%2Djava.time.temporal.TemporalField%2Dint%2Dint%2Dint%2D[appendValueReduced](<<painless-api-reference-TemporalField,TemporalField>>, int, int, int)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendValueReduced%2Djava.time.temporal.TemporalField%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-appendZoneId-0]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendZoneId%2D%2D[appendZoneId]()++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendZoneId%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-appendZoneOrOffsetId-0]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendZoneOrOffsetId%2D%2D[appendZoneOrOffsetId]()++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendZoneOrOffsetId%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-appendZoneRegionId-0]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendZoneRegionId%2D%2D[appendZoneRegionId]()++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendZoneRegionId%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-appendZoneText-1]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendZoneText%2Djava.time.format.TextStyle%2D[appendZoneText](<<painless-api-reference-TextStyle,TextStyle>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendZoneText%2Djava.time.format.TextStyle%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-appendZoneText-2]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendZoneText%2Djava.time.format.TextStyle%2Djava.util.Set%2D[appendZoneText](<<painless-api-reference-TextStyle,TextStyle>>, <<painless-api-reference-Set,Set>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendZoneText%2Djava.time.format.TextStyle%2Djava.util.Set%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-optionalEnd-0]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#optionalEnd%2D%2D[optionalEnd]()++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#optionalEnd%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-optionalStart-0]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#optionalStart%2D%2D[optionalStart]()++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#optionalStart%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-padNext-1]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#padNext%2Dint%2D[padNext](int)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#padNext%2Dint%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-padNext-2]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#padNext%2Dint%2Dchar%2D[padNext](int, char)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#padNext%2Dint%2Dchar%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-parseCaseInsensitive-0]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#parseCaseInsensitive%2D%2D[parseCaseInsensitive]()++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#parseCaseInsensitive%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-parseCaseSensitive-0]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#parseCaseSensitive%2D%2D[parseCaseSensitive]()++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#parseCaseSensitive%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-parseDefaulting-2]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#parseDefaulting%2Djava.time.temporal.TemporalField%2Dlong%2D[parseDefaulting](<<painless-api-reference-TemporalField,TemporalField>>, long)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#parseDefaulting%2Djava.time.temporal.TemporalField%2Dlong%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-parseLenient-0]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#parseLenient%2D%2D[parseLenient]()++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#parseLenient%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-parseStrict-0]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#parseStrict%2D%2D[parseStrict]()++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#parseStrict%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-toFormatter-0]]<<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#toFormatter%2D%2D[toFormatter]()++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#toFormatter%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-toFormatter-1]]<<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#toFormatter%2Djava.util.Locale%2D[toFormatter](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#toFormatter%2Djava.util.Locale%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/DateTimeParseException.asciidoc b/docs/painless/painless-api-reference/DateTimeParseException.asciidoc
new file mode 100644
index 0000000000000..8a9626b64052a
--- /dev/null
+++ b/docs/painless/painless-api-reference/DateTimeParseException.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-DateTimeParseException]]++DateTimeParseException++::
+* ++[[painless-api-reference-DateTimeParseException-DateTimeParseException-3]]link:{java8-javadoc}/java/time/format/DateTimeParseException.html#DateTimeParseException%2Djava.lang.String%2Djava.lang.CharSequence%2Dint%2D[DateTimeParseException](<<painless-api-reference-String,String>>, <<painless-api-reference-CharSequence,CharSequence>>, int)++ (link:{java9-javadoc}/java/time/format/DateTimeParseException.html#DateTimeParseException%2Djava.lang.String%2Djava.lang.CharSequence%2Dint%2D[java 9])
+* ++[[painless-api-reference-DateTimeParseException-getErrorIndex-0]]int link:{java8-javadoc}/java/time/format/DateTimeParseException.html#getErrorIndex%2D%2D[getErrorIndex]()++ (link:{java9-javadoc}/java/time/format/DateTimeParseException.html#getErrorIndex%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeParseException-getParsedString-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/format/DateTimeParseException.html#getParsedString%2D%2D[getParsedString]()++ (link:{java9-javadoc}/java/time/format/DateTimeParseException.html#getParsedString%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/DayOfWeek.asciidoc b/docs/painless/painless-api-reference/DayOfWeek.asciidoc
new file mode 100644
index 0000000000000..fd847f3d019ab
--- /dev/null
+++ b/docs/painless/painless-api-reference/DayOfWeek.asciidoc
@@ -0,0 +1,22 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-DayOfWeek]]++DayOfWeek++::
+** [[painless-api-reference-DayOfWeek-FRIDAY]]static <<painless-api-reference-DayOfWeek,DayOfWeek>> link:{java8-javadoc}/java/time/DayOfWeek.html#FRIDAY[FRIDAY] (link:{java9-javadoc}/java/time/DayOfWeek.html#FRIDAY[java 9])
+** [[painless-api-reference-DayOfWeek-MONDAY]]static <<painless-api-reference-DayOfWeek,DayOfWeek>> link:{java8-javadoc}/java/time/DayOfWeek.html#MONDAY[MONDAY] (link:{java9-javadoc}/java/time/DayOfWeek.html#MONDAY[java 9])
+** [[painless-api-reference-DayOfWeek-SATURDAY]]static <<painless-api-reference-DayOfWeek,DayOfWeek>> link:{java8-javadoc}/java/time/DayOfWeek.html#SATURDAY[SATURDAY] (link:{java9-javadoc}/java/time/DayOfWeek.html#SATURDAY[java 9])
+** [[painless-api-reference-DayOfWeek-SUNDAY]]static <<painless-api-reference-DayOfWeek,DayOfWeek>> link:{java8-javadoc}/java/time/DayOfWeek.html#SUNDAY[SUNDAY] (link:{java9-javadoc}/java/time/DayOfWeek.html#SUNDAY[java 9])
+** [[painless-api-reference-DayOfWeek-THURSDAY]]static <<painless-api-reference-DayOfWeek,DayOfWeek>> link:{java8-javadoc}/java/time/DayOfWeek.html#THURSDAY[THURSDAY] (link:{java9-javadoc}/java/time/DayOfWeek.html#THURSDAY[java 9])
+** [[painless-api-reference-DayOfWeek-TUESDAY]]static <<painless-api-reference-DayOfWeek,DayOfWeek>> link:{java8-javadoc}/java/time/DayOfWeek.html#TUESDAY[TUESDAY] (link:{java9-javadoc}/java/time/DayOfWeek.html#TUESDAY[java 9])
+** [[painless-api-reference-DayOfWeek-WEDNESDAY]]static <<painless-api-reference-DayOfWeek,DayOfWeek>> link:{java8-javadoc}/java/time/DayOfWeek.html#WEDNESDAY[WEDNESDAY] (link:{java9-javadoc}/java/time/DayOfWeek.html#WEDNESDAY[java 9])
+* ++[[painless-api-reference-DayOfWeek-from-1]]static <<painless-api-reference-DayOfWeek,DayOfWeek>> link:{java8-javadoc}/java/time/DayOfWeek.html#from%2Djava.time.temporal.TemporalAccessor%2D[from](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/DayOfWeek.html#from%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-DayOfWeek-of-1]]static <<painless-api-reference-DayOfWeek,DayOfWeek>> link:{java8-javadoc}/java/time/DayOfWeek.html#of%2Dint%2D[of](int)++ (link:{java9-javadoc}/java/time/DayOfWeek.html#of%2Dint%2D[java 9])
+* ++[[painless-api-reference-DayOfWeek-valueOf-1]]static <<painless-api-reference-DayOfWeek,DayOfWeek>> link:{java8-javadoc}/java/time/DayOfWeek.html#valueOf%2Djava.lang.String%2D[valueOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/DayOfWeek.html#valueOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-DayOfWeek-values-0]]static <<painless-api-reference-DayOfWeek,DayOfWeek>>[] link:{java8-javadoc}/java/time/DayOfWeek.html#values%2D%2D[values]()++ (link:{java9-javadoc}/java/time/DayOfWeek.html#values%2D%2D[java 9])
+* ++[[painless-api-reference-DayOfWeek-getDisplayName-2]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/DayOfWeek.html#getDisplayName%2Djava.time.format.TextStyle%2Djava.util.Locale%2D[getDisplayName](<<painless-api-reference-TextStyle,TextStyle>>, <<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/time/DayOfWeek.html#getDisplayName%2Djava.time.format.TextStyle%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-DayOfWeek-getValue-0]]int link:{java8-javadoc}/java/time/DayOfWeek.html#getValue%2D%2D[getValue]()++ (link:{java9-javadoc}/java/time/DayOfWeek.html#getValue%2D%2D[java 9])
+* ++[[painless-api-reference-DayOfWeek-minus-1]]<<painless-api-reference-DayOfWeek,DayOfWeek>> link:{java8-javadoc}/java/time/DayOfWeek.html#minus%2Dlong%2D[minus](long)++ (link:{java9-javadoc}/java/time/DayOfWeek.html#minus%2Dlong%2D[java 9])
+* ++[[painless-api-reference-DayOfWeek-plus-1]]<<painless-api-reference-DayOfWeek,DayOfWeek>> link:{java8-javadoc}/java/time/DayOfWeek.html#plus%2Dlong%2D[plus](long)++ (link:{java9-javadoc}/java/time/DayOfWeek.html#plus%2Dlong%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Enum,Enum>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++, ++<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>++
diff --git a/docs/painless/painless-api-reference/Debug.asciidoc b/docs/painless/painless-api-reference/Debug.asciidoc
new file mode 100644
index 0000000000000..8c64ed6bcc4ad
--- /dev/null
+++ b/docs/painless/painless-api-reference/Debug.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Debug]]++Debug++::
+* ++[[painless-api-reference-Debug-explain-1]]static void link:{painless-javadoc}/org/elasticsearch/painless/api/Debug.html#explain%2Djava.lang.Object%2D[explain](<<painless-api-reference-Object,Object>>)++
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/DecimalFormat.asciidoc b/docs/painless/painless-api-reference/DecimalFormat.asciidoc
new file mode 100644
index 0000000000000..96768b77273f8
--- /dev/null
+++ b/docs/painless/painless-api-reference/DecimalFormat.asciidoc
@@ -0,0 +1,32 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-DecimalFormat]]++DecimalFormat++::
+* ++[[painless-api-reference-DecimalFormat-DecimalFormat-0]]link:{java8-javadoc}/java/text/DecimalFormat.html#DecimalFormat%2D%2D[DecimalFormat]()++ (link:{java9-javadoc}/java/text/DecimalFormat.html#DecimalFormat%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-DecimalFormat-1]]link:{java8-javadoc}/java/text/DecimalFormat.html#DecimalFormat%2Djava.lang.String%2D[DecimalFormat](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/DecimalFormat.html#DecimalFormat%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-DecimalFormat-2]]link:{java8-javadoc}/java/text/DecimalFormat.html#DecimalFormat%2Djava.lang.String%2Djava.text.DecimalFormatSymbols%2D[DecimalFormat](<<painless-api-reference-String,String>>, <<painless-api-reference-DecimalFormatSymbols,DecimalFormatSymbols>>)++ (link:{java9-javadoc}/java/text/DecimalFormat.html#DecimalFormat%2Djava.lang.String%2Djava.text.DecimalFormatSymbols%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-applyLocalizedPattern-1]]void link:{java8-javadoc}/java/text/DecimalFormat.html#applyLocalizedPattern%2Djava.lang.String%2D[applyLocalizedPattern](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/DecimalFormat.html#applyLocalizedPattern%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-applyPattern-1]]void link:{java8-javadoc}/java/text/DecimalFormat.html#applyPattern%2Djava.lang.String%2D[applyPattern](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/DecimalFormat.html#applyPattern%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-getDecimalFormatSymbols-0]]<<painless-api-reference-DecimalFormatSymbols,DecimalFormatSymbols>> link:{java8-javadoc}/java/text/DecimalFormat.html#getDecimalFormatSymbols%2D%2D[getDecimalFormatSymbols]()++ (link:{java9-javadoc}/java/text/DecimalFormat.html#getDecimalFormatSymbols%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-getGroupingSize-0]]int link:{java8-javadoc}/java/text/DecimalFormat.html#getGroupingSize%2D%2D[getGroupingSize]()++ (link:{java9-javadoc}/java/text/DecimalFormat.html#getGroupingSize%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-getMultiplier-0]]int link:{java8-javadoc}/java/text/DecimalFormat.html#getMultiplier%2D%2D[getMultiplier]()++ (link:{java9-javadoc}/java/text/DecimalFormat.html#getMultiplier%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-getNegativePrefix-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/text/DecimalFormat.html#getNegativePrefix%2D%2D[getNegativePrefix]()++ (link:{java9-javadoc}/java/text/DecimalFormat.html#getNegativePrefix%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-getNegativeSuffix-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/text/DecimalFormat.html#getNegativeSuffix%2D%2D[getNegativeSuffix]()++ (link:{java9-javadoc}/java/text/DecimalFormat.html#getNegativeSuffix%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-getPositivePrefix-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/text/DecimalFormat.html#getPositivePrefix%2D%2D[getPositivePrefix]()++ (link:{java9-javadoc}/java/text/DecimalFormat.html#getPositivePrefix%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-getPositiveSuffix-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/text/DecimalFormat.html#getPositiveSuffix%2D%2D[getPositiveSuffix]()++ (link:{java9-javadoc}/java/text/DecimalFormat.html#getPositiveSuffix%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-isDecimalSeparatorAlwaysShown-0]]boolean link:{java8-javadoc}/java/text/DecimalFormat.html#isDecimalSeparatorAlwaysShown%2D%2D[isDecimalSeparatorAlwaysShown]()++ (link:{java9-javadoc}/java/text/DecimalFormat.html#isDecimalSeparatorAlwaysShown%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-isParseBigDecimal-0]]boolean link:{java8-javadoc}/java/text/DecimalFormat.html#isParseBigDecimal%2D%2D[isParseBigDecimal]()++ (link:{java9-javadoc}/java/text/DecimalFormat.html#isParseBigDecimal%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-setDecimalFormatSymbols-1]]void link:{java8-javadoc}/java/text/DecimalFormat.html#setDecimalFormatSymbols%2Djava.text.DecimalFormatSymbols%2D[setDecimalFormatSymbols](<<painless-api-reference-DecimalFormatSymbols,DecimalFormatSymbols>>)++ (link:{java9-javadoc}/java/text/DecimalFormat.html#setDecimalFormatSymbols%2Djava.text.DecimalFormatSymbols%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-setDecimalSeparatorAlwaysShown-1]]void link:{java8-javadoc}/java/text/DecimalFormat.html#setDecimalSeparatorAlwaysShown%2Dboolean%2D[setDecimalSeparatorAlwaysShown](boolean)++ (link:{java9-javadoc}/java/text/DecimalFormat.html#setDecimalSeparatorAlwaysShown%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-setGroupingSize-1]]void link:{java8-javadoc}/java/text/DecimalFormat.html#setGroupingSize%2Dint%2D[setGroupingSize](int)++ (link:{java9-javadoc}/java/text/DecimalFormat.html#setGroupingSize%2Dint%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-setMultiplier-1]]void link:{java8-javadoc}/java/text/DecimalFormat.html#setMultiplier%2Dint%2D[setMultiplier](int)++ (link:{java9-javadoc}/java/text/DecimalFormat.html#setMultiplier%2Dint%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-setNegativePrefix-1]]void link:{java8-javadoc}/java/text/DecimalFormat.html#setNegativePrefix%2Djava.lang.String%2D[setNegativePrefix](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/DecimalFormat.html#setNegativePrefix%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-setNegativeSuffix-1]]void link:{java8-javadoc}/java/text/DecimalFormat.html#setNegativeSuffix%2Djava.lang.String%2D[setNegativeSuffix](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/DecimalFormat.html#setNegativeSuffix%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-setParseBigDecimal-1]]void link:{java8-javadoc}/java/text/DecimalFormat.html#setParseBigDecimal%2Dboolean%2D[setParseBigDecimal](boolean)++ (link:{java9-javadoc}/java/text/DecimalFormat.html#setParseBigDecimal%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-setPositivePrefix-1]]void link:{java8-javadoc}/java/text/DecimalFormat.html#setPositivePrefix%2Djava.lang.String%2D[setPositivePrefix](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/DecimalFormat.html#setPositivePrefix%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-setPositiveSuffix-1]]void link:{java8-javadoc}/java/text/DecimalFormat.html#setPositiveSuffix%2Djava.lang.String%2D[setPositiveSuffix](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/DecimalFormat.html#setPositiveSuffix%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-toLocalizedPattern-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/text/DecimalFormat.html#toLocalizedPattern%2D%2D[toLocalizedPattern]()++ (link:{java9-javadoc}/java/text/DecimalFormat.html#toLocalizedPattern%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-toPattern-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/text/DecimalFormat.html#toPattern%2D%2D[toPattern]()++ (link:{java9-javadoc}/java/text/DecimalFormat.html#toPattern%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Format,Format>>++, ++<<painless-api-reference-NumberFormat,NumberFormat>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/DecimalFormatSymbols.asciidoc b/docs/painless/painless-api-reference/DecimalFormatSymbols.asciidoc
new file mode 100644
index 0000000000000..37f6fc090d105
--- /dev/null
+++ b/docs/painless/painless-api-reference/DecimalFormatSymbols.asciidoc
@@ -0,0 +1,43 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-DecimalFormatSymbols]]++DecimalFormatSymbols++::
+* ++[[painless-api-reference-DecimalFormatSymbols-getAvailableLocales-0]]static <<painless-api-reference-Locale,Locale>>[] link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#getAvailableLocales%2D%2D[getAvailableLocales]()++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#getAvailableLocales%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-getInstance-0]]static <<painless-api-reference-DecimalFormatSymbols,DecimalFormatSymbols>> link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#getInstance%2D%2D[getInstance]()++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#getInstance%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-getInstance-1]]static <<painless-api-reference-DecimalFormatSymbols,DecimalFormatSymbols>> link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#getInstance%2Djava.util.Locale%2D[getInstance](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#getInstance%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-DecimalFormatSymbols-0]]link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#DecimalFormatSymbols%2D%2D[DecimalFormatSymbols]()++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#DecimalFormatSymbols%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-DecimalFormatSymbols-1]]link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#DecimalFormatSymbols%2Djava.util.Locale%2D[DecimalFormatSymbols](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#DecimalFormatSymbols%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-clone-0]]def link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#clone%2D%2D[clone]()++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#clone%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-getCurrency-0]]<<painless-api-reference-Currency,Currency>> link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#getCurrency%2D%2D[getCurrency]()++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#getCurrency%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-getCurrencySymbol-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#getCurrencySymbol%2D%2D[getCurrencySymbol]()++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#getCurrencySymbol%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-getDecimalSeparator-0]]char link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#getDecimalSeparator%2D%2D[getDecimalSeparator]()++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#getDecimalSeparator%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-getDigit-0]]char link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#getDigit%2D%2D[getDigit]()++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#getDigit%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-getExponentSeparator-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#getExponentSeparator%2D%2D[getExponentSeparator]()++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#getExponentSeparator%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-getGroupingSeparator-0]]char link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#getGroupingSeparator%2D%2D[getGroupingSeparator]()++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#getGroupingSeparator%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-getInfinity-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#getInfinity%2D%2D[getInfinity]()++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#getInfinity%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-getInternationalCurrencySymbol-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#getInternationalCurrencySymbol%2D%2D[getInternationalCurrencySymbol]()++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#getInternationalCurrencySymbol%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-getMinusSign-0]]char link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#getMinusSign%2D%2D[getMinusSign]()++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#getMinusSign%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-getMonetaryDecimalSeparator-0]]char link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#getMonetaryDecimalSeparator%2D%2D[getMonetaryDecimalSeparator]()++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#getMonetaryDecimalSeparator%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-getNaN-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#getNaN%2D%2D[getNaN]()++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#getNaN%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-getPatternSeparator-0]]char link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#getPatternSeparator%2D%2D[getPatternSeparator]()++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#getPatternSeparator%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-getPerMill-0]]char link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#getPerMill%2D%2D[getPerMill]()++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#getPerMill%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-getPercent-0]]char link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#getPercent%2D%2D[getPercent]()++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#getPercent%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-getZeroDigit-0]]char link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#getZeroDigit%2D%2D[getZeroDigit]()++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#getZeroDigit%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-setCurrency-1]]void link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#setCurrency%2Djava.util.Currency%2D[setCurrency](<<painless-api-reference-Currency,Currency>>)++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#setCurrency%2Djava.util.Currency%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-setCurrencySymbol-1]]void link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#setCurrencySymbol%2Djava.lang.String%2D[setCurrencySymbol](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#setCurrencySymbol%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-setDecimalSeparator-1]]void link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#setDecimalSeparator%2Dchar%2D[setDecimalSeparator](char)++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#setDecimalSeparator%2Dchar%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-setDigit-1]]void link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#setDigit%2Dchar%2D[setDigit](char)++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#setDigit%2Dchar%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-setExponentSeparator-1]]void link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#setExponentSeparator%2Djava.lang.String%2D[setExponentSeparator](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#setExponentSeparator%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-setGroupingSeparator-1]]void link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#setGroupingSeparator%2Dchar%2D[setGroupingSeparator](char)++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#setGroupingSeparator%2Dchar%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-setInfinity-1]]void link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#setInfinity%2Djava.lang.String%2D[setInfinity](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#setInfinity%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-setInternationalCurrencySymbol-1]]void link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#setInternationalCurrencySymbol%2Djava.lang.String%2D[setInternationalCurrencySymbol](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#setInternationalCurrencySymbol%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-setMinusSign-1]]void link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#setMinusSign%2Dchar%2D[setMinusSign](char)++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#setMinusSign%2Dchar%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-setMonetaryDecimalSeparator-1]]void link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#setMonetaryDecimalSeparator%2Dchar%2D[setMonetaryDecimalSeparator](char)++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#setMonetaryDecimalSeparator%2Dchar%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-setNaN-1]]void link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#setNaN%2Djava.lang.String%2D[setNaN](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#setNaN%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-setPatternSeparator-1]]void link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#setPatternSeparator%2Dchar%2D[setPatternSeparator](char)++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#setPatternSeparator%2Dchar%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-setPerMill-1]]void link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#setPerMill%2Dchar%2D[setPerMill](char)++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#setPerMill%2Dchar%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-setPercent-1]]void link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#setPercent%2Dchar%2D[setPercent](char)++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#setPercent%2Dchar%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-setZeroDigit-1]]void link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#setZeroDigit%2Dchar%2D[setZeroDigit](char)++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#setZeroDigit%2Dchar%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/DecimalStyle.asciidoc b/docs/painless/painless-api-reference/DecimalStyle.asciidoc
new file mode 100644
index 0000000000000..8dac3a7ae2440
--- /dev/null
+++ b/docs/painless/painless-api-reference/DecimalStyle.asciidoc
@@ -0,0 +1,19 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-DecimalStyle]]++DecimalStyle++::
+** [[painless-api-reference-DecimalStyle-STANDARD]]static <<painless-api-reference-DecimalStyle,DecimalStyle>> link:{java8-javadoc}/java/time/format/DecimalStyle.html#STANDARD[STANDARD] (link:{java9-javadoc}/java/time/format/DecimalStyle.html#STANDARD[java 9])
+* ++[[painless-api-reference-DecimalStyle-getAvailableLocales-0]]static <<painless-api-reference-Set,Set>> link:{java8-javadoc}/java/time/format/DecimalStyle.html#getAvailableLocales%2D%2D[getAvailableLocales]()++ (link:{java9-javadoc}/java/time/format/DecimalStyle.html#getAvailableLocales%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalStyle-of-1]]static <<painless-api-reference-DecimalStyle,DecimalStyle>> link:{java8-javadoc}/java/time/format/DecimalStyle.html#of%2Djava.util.Locale%2D[of](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/time/format/DecimalStyle.html#of%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-DecimalStyle-ofDefaultLocale-0]]static <<painless-api-reference-DecimalStyle,DecimalStyle>> link:{java8-javadoc}/java/time/format/DecimalStyle.html#ofDefaultLocale%2D%2D[ofDefaultLocale]()++ (link:{java9-javadoc}/java/time/format/DecimalStyle.html#ofDefaultLocale%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalStyle-getDecimalSeparator-0]]char link:{java8-javadoc}/java/time/format/DecimalStyle.html#getDecimalSeparator%2D%2D[getDecimalSeparator]()++ (link:{java9-javadoc}/java/time/format/DecimalStyle.html#getDecimalSeparator%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalStyle-getNegativeSign-0]]char link:{java8-javadoc}/java/time/format/DecimalStyle.html#getNegativeSign%2D%2D[getNegativeSign]()++ (link:{java9-javadoc}/java/time/format/DecimalStyle.html#getNegativeSign%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalStyle-getPositiveSign-0]]char link:{java8-javadoc}/java/time/format/DecimalStyle.html#getPositiveSign%2D%2D[getPositiveSign]()++ (link:{java9-javadoc}/java/time/format/DecimalStyle.html#getPositiveSign%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalStyle-getZeroDigit-0]]char link:{java8-javadoc}/java/time/format/DecimalStyle.html#getZeroDigit%2D%2D[getZeroDigit]()++ (link:{java9-javadoc}/java/time/format/DecimalStyle.html#getZeroDigit%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalStyle-withDecimalSeparator-1]]<<painless-api-reference-DecimalStyle,DecimalStyle>> link:{java8-javadoc}/java/time/format/DecimalStyle.html#withDecimalSeparator%2Dchar%2D[withDecimalSeparator](char)++ (link:{java9-javadoc}/java/time/format/DecimalStyle.html#withDecimalSeparator%2Dchar%2D[java 9])
+* ++[[painless-api-reference-DecimalStyle-withNegativeSign-1]]<<painless-api-reference-DecimalStyle,DecimalStyle>> link:{java8-javadoc}/java/time/format/DecimalStyle.html#withNegativeSign%2Dchar%2D[withNegativeSign](char)++ (link:{java9-javadoc}/java/time/format/DecimalStyle.html#withNegativeSign%2Dchar%2D[java 9])
+* ++[[painless-api-reference-DecimalStyle-withPositiveSign-1]]<<painless-api-reference-DecimalStyle,DecimalStyle>> link:{java8-javadoc}/java/time/format/DecimalStyle.html#withPositiveSign%2Dchar%2D[withPositiveSign](char)++ (link:{java9-javadoc}/java/time/format/DecimalStyle.html#withPositiveSign%2Dchar%2D[java 9])
+* ++[[painless-api-reference-DecimalStyle-withZeroDigit-1]]<<painless-api-reference-DecimalStyle,DecimalStyle>> link:{java8-javadoc}/java/time/format/DecimalStyle.html#withZeroDigit%2Dchar%2D[withZeroDigit](char)++ (link:{java9-javadoc}/java/time/format/DecimalStyle.html#withZeroDigit%2Dchar%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Deque.asciidoc b/docs/painless/painless-api-reference/Deque.asciidoc
new file mode 100644
index 0000000000000..4dc6b5c06fc84
--- /dev/null
+++ b/docs/painless/painless-api-reference/Deque.asciidoc
@@ -0,0 +1,25 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Deque]]++Deque++::
+* ++[[painless-api-reference-Deque-addFirst-1]]void link:{java8-javadoc}/java/util/Deque.html#addFirst%2Djava.lang.Object%2D[addFirst](def)++ (link:{java9-javadoc}/java/util/Deque.html#addFirst%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Deque-addLast-1]]void link:{java8-javadoc}/java/util/Deque.html#addLast%2Djava.lang.Object%2D[addLast](def)++ (link:{java9-javadoc}/java/util/Deque.html#addLast%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Deque-descendingIterator-0]]<<painless-api-reference-Iterator,Iterator>> link:{java8-javadoc}/java/util/Deque.html#descendingIterator%2D%2D[descendingIterator]()++ (link:{java9-javadoc}/java/util/Deque.html#descendingIterator%2D%2D[java 9])
+* ++[[painless-api-reference-Deque-getFirst-0]]def link:{java8-javadoc}/java/util/Deque.html#getFirst%2D%2D[getFirst]()++ (link:{java9-javadoc}/java/util/Deque.html#getFirst%2D%2D[java 9])
+* ++[[painless-api-reference-Deque-getLast-0]]def link:{java8-javadoc}/java/util/Deque.html#getLast%2D%2D[getLast]()++ (link:{java9-javadoc}/java/util/Deque.html#getLast%2D%2D[java 9])
+* ++[[painless-api-reference-Deque-offerFirst-1]]boolean link:{java8-javadoc}/java/util/Deque.html#offerFirst%2Djava.lang.Object%2D[offerFirst](def)++ (link:{java9-javadoc}/java/util/Deque.html#offerFirst%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Deque-offerLast-1]]boolean link:{java8-javadoc}/java/util/Deque.html#offerLast%2Djava.lang.Object%2D[offerLast](def)++ (link:{java9-javadoc}/java/util/Deque.html#offerLast%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Deque-peekFirst-0]]def link:{java8-javadoc}/java/util/Deque.html#peekFirst%2D%2D[peekFirst]()++ (link:{java9-javadoc}/java/util/Deque.html#peekFirst%2D%2D[java 9])
+* ++[[painless-api-reference-Deque-peekLast-0]]def link:{java8-javadoc}/java/util/Deque.html#peekLast%2D%2D[peekLast]()++ (link:{java9-javadoc}/java/util/Deque.html#peekLast%2D%2D[java 9])
+* ++[[painless-api-reference-Deque-pollFirst-0]]def link:{java8-javadoc}/java/util/Deque.html#pollFirst%2D%2D[pollFirst]()++ (link:{java9-javadoc}/java/util/Deque.html#pollFirst%2D%2D[java 9])
+* ++[[painless-api-reference-Deque-pollLast-0]]def link:{java8-javadoc}/java/util/Deque.html#pollLast%2D%2D[pollLast]()++ (link:{java9-javadoc}/java/util/Deque.html#pollLast%2D%2D[java 9])
+* ++[[painless-api-reference-Deque-pop-0]]def link:{java8-javadoc}/java/util/Deque.html#pop%2D%2D[pop]()++ (link:{java9-javadoc}/java/util/Deque.html#pop%2D%2D[java 9])
+* ++[[painless-api-reference-Deque-push-1]]void link:{java8-javadoc}/java/util/Deque.html#push%2Djava.lang.Object%2D[push](def)++ (link:{java9-javadoc}/java/util/Deque.html#push%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Deque-remove-1]]boolean link:{java8-javadoc}/java/util/Deque.html#remove%2Djava.lang.Object%2D[remove](def)++ (link:{java9-javadoc}/java/util/Deque.html#remove%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Deque-removeFirst-0]]def link:{java8-javadoc}/java/util/Deque.html#removeFirst%2D%2D[removeFirst]()++ (link:{java9-javadoc}/java/util/Deque.html#removeFirst%2D%2D[java 9])
+* ++[[painless-api-reference-Deque-removeFirstOccurrence-1]]boolean link:{java8-javadoc}/java/util/Deque.html#removeFirstOccurrence%2Djava.lang.Object%2D[removeFirstOccurrence](def)++ (link:{java9-javadoc}/java/util/Deque.html#removeFirstOccurrence%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Deque-removeLast-0]]def link:{java8-javadoc}/java/util/Deque.html#removeLast%2D%2D[removeLast]()++ (link:{java9-javadoc}/java/util/Deque.html#removeLast%2D%2D[java 9])
+* ++[[painless-api-reference-Deque-removeLastOccurrence-1]]boolean link:{java8-javadoc}/java/util/Deque.html#removeLastOccurrence%2Djava.lang.Object%2D[removeLastOccurrence](def)++ (link:{java9-javadoc}/java/util/Deque.html#removeLastOccurrence%2Djava.lang.Object%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-Queue,Queue>>++
diff --git a/docs/painless/painless-api-reference/Dictionary.asciidoc b/docs/painless/painless-api-reference/Dictionary.asciidoc
new file mode 100644
index 0000000000000..bfa8f2c495e19
--- /dev/null
+++ b/docs/painless/painless-api-reference/Dictionary.asciidoc
@@ -0,0 +1,14 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Dictionary]]++Dictionary++::
+* ++[[painless-api-reference-Dictionary-elements-0]]<<painless-api-reference-Enumeration,Enumeration>> link:{java8-javadoc}/java/util/Dictionary.html#elements%2D%2D[elements]()++ (link:{java9-javadoc}/java/util/Dictionary.html#elements%2D%2D[java 9])
+* ++[[painless-api-reference-Dictionary-get-1]]def link:{java8-javadoc}/java/util/Dictionary.html#get%2Djava.lang.Object%2D[get](def)++ (link:{java9-javadoc}/java/util/Dictionary.html#get%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Dictionary-isEmpty-0]]boolean link:{java8-javadoc}/java/util/Dictionary.html#isEmpty%2D%2D[isEmpty]()++ (link:{java9-javadoc}/java/util/Dictionary.html#isEmpty%2D%2D[java 9])
+* ++[[painless-api-reference-Dictionary-keys-0]]<<painless-api-reference-Enumeration,Enumeration>> link:{java8-javadoc}/java/util/Dictionary.html#keys%2D%2D[keys]()++ (link:{java9-javadoc}/java/util/Dictionary.html#keys%2D%2D[java 9])
+* ++[[painless-api-reference-Dictionary-put-2]]def link:{java8-javadoc}/java/util/Dictionary.html#put%2Djava.lang.Object%2Djava.lang.Object%2D[put](def, def)++ (link:{java9-javadoc}/java/util/Dictionary.html#put%2Djava.lang.Object%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Dictionary-remove-1]]def link:{java8-javadoc}/java/util/Dictionary.html#remove%2Djava.lang.Object%2D[remove](def)++ (link:{java9-javadoc}/java/util/Dictionary.html#remove%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Dictionary-size-0]]int link:{java8-javadoc}/java/util/Dictionary.html#size%2D%2D[size]()++ (link:{java9-javadoc}/java/util/Dictionary.html#size%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Double.asciidoc b/docs/painless/painless-api-reference/Double.asciidoc
new file mode 100644
index 0000000000000..50d1d8c217cba
--- /dev/null
+++ b/docs/painless/painless-api-reference/Double.asciidoc
@@ -0,0 +1,35 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Double]]++Double++::
+** [[painless-api-reference-Double-BYTES]]static int link:{java8-javadoc}/java/lang/Double.html#BYTES[BYTES] (link:{java9-javadoc}/java/lang/Double.html#BYTES[java 9])
+** [[painless-api-reference-Double-MAX_EXPONENT]]static int link:{java8-javadoc}/java/lang/Double.html#MAX_EXPONENT[MAX_EXPONENT] (link:{java9-javadoc}/java/lang/Double.html#MAX_EXPONENT[java 9])
+** [[painless-api-reference-Double-MAX_VALUE]]static double link:{java8-javadoc}/java/lang/Double.html#MAX_VALUE[MAX_VALUE] (link:{java9-javadoc}/java/lang/Double.html#MAX_VALUE[java 9])
+** [[painless-api-reference-Double-MIN_EXPONENT]]static int link:{java8-javadoc}/java/lang/Double.html#MIN_EXPONENT[MIN_EXPONENT] (link:{java9-javadoc}/java/lang/Double.html#MIN_EXPONENT[java 9])
+** [[painless-api-reference-Double-MIN_NORMAL]]static double link:{java8-javadoc}/java/lang/Double.html#MIN_NORMAL[MIN_NORMAL] (link:{java9-javadoc}/java/lang/Double.html#MIN_NORMAL[java 9])
+** [[painless-api-reference-Double-MIN_VALUE]]static double link:{java8-javadoc}/java/lang/Double.html#MIN_VALUE[MIN_VALUE] (link:{java9-javadoc}/java/lang/Double.html#MIN_VALUE[java 9])
+** [[painless-api-reference-Double-NEGATIVE_INFINITY]]static double link:{java8-javadoc}/java/lang/Double.html#NEGATIVE_INFINITY[NEGATIVE_INFINITY] (link:{java9-javadoc}/java/lang/Double.html#NEGATIVE_INFINITY[java 9])
+** [[painless-api-reference-Double-NaN]]static double link:{java8-javadoc}/java/lang/Double.html#NaN[NaN] (link:{java9-javadoc}/java/lang/Double.html#NaN[java 9])
+** [[painless-api-reference-Double-POSITIVE_INFINITY]]static double link:{java8-javadoc}/java/lang/Double.html#POSITIVE_INFINITY[POSITIVE_INFINITY] (link:{java9-javadoc}/java/lang/Double.html#POSITIVE_INFINITY[java 9])
+** [[painless-api-reference-Double-SIZE]]static int link:{java8-javadoc}/java/lang/Double.html#SIZE[SIZE] (link:{java9-javadoc}/java/lang/Double.html#SIZE[java 9])
+* ++[[painless-api-reference-Double-compare-2]]static int link:{java8-javadoc}/java/lang/Double.html#compare%2Ddouble%2Ddouble%2D[compare](double, double)++ (link:{java9-javadoc}/java/lang/Double.html#compare%2Ddouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Double-doubleToLongBits-1]]static long link:{java8-javadoc}/java/lang/Double.html#doubleToLongBits%2Ddouble%2D[doubleToLongBits](double)++ (link:{java9-javadoc}/java/lang/Double.html#doubleToLongBits%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Double-doubleToRawLongBits-1]]static long link:{java8-javadoc}/java/lang/Double.html#doubleToRawLongBits%2Ddouble%2D[doubleToRawLongBits](double)++ (link:{java9-javadoc}/java/lang/Double.html#doubleToRawLongBits%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Double-hashCode-1]]static int link:{java8-javadoc}/java/lang/Double.html#hashCode%2Ddouble%2D[hashCode](double)++ (link:{java9-javadoc}/java/lang/Double.html#hashCode%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Double-isFinite-1]]static boolean link:{java8-javadoc}/java/lang/Double.html#isFinite%2Ddouble%2D[isFinite](double)++ (link:{java9-javadoc}/java/lang/Double.html#isFinite%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Double-isInfinite-1]]static boolean link:{java8-javadoc}/java/lang/Double.html#isInfinite%2Ddouble%2D[isInfinite](double)++ (link:{java9-javadoc}/java/lang/Double.html#isInfinite%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Double-isNaN-1]]static boolean link:{java8-javadoc}/java/lang/Double.html#isNaN%2Ddouble%2D[isNaN](double)++ (link:{java9-javadoc}/java/lang/Double.html#isNaN%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Double-longBitsToDouble-1]]static double link:{java8-javadoc}/java/lang/Double.html#longBitsToDouble%2Dlong%2D[longBitsToDouble](long)++ (link:{java9-javadoc}/java/lang/Double.html#longBitsToDouble%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Double-max-2]]static double link:{java8-javadoc}/java/lang/Double.html#max%2Ddouble%2Ddouble%2D[max](double, double)++ (link:{java9-javadoc}/java/lang/Double.html#max%2Ddouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Double-min-2]]static double link:{java8-javadoc}/java/lang/Double.html#min%2Ddouble%2Ddouble%2D[min](double, double)++ (link:{java9-javadoc}/java/lang/Double.html#min%2Ddouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Double-parseDouble-1]]static double link:{java8-javadoc}/java/lang/Double.html#parseDouble%2Djava.lang.String%2D[parseDouble](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/Double.html#parseDouble%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Double-sum-2]]static double link:{java8-javadoc}/java/lang/Double.html#sum%2Ddouble%2Ddouble%2D[sum](double, double)++ (link:{java9-javadoc}/java/lang/Double.html#sum%2Ddouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Double-toHexString-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Double.html#toHexString%2Ddouble%2D[toHexString](double)++ (link:{java9-javadoc}/java/lang/Double.html#toHexString%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Double-toString-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Double.html#toString%2Ddouble%2D[toString](double)++ (link:{java9-javadoc}/java/lang/Double.html#toString%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Double-valueOf-1]]static <<painless-api-reference-Double,Double>> link:{java8-javadoc}/java/lang/Double.html#valueOf%2Ddouble%2D[valueOf](double)++ (link:{java9-javadoc}/java/lang/Double.html#valueOf%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Double-compareTo-1]]int link:{java8-javadoc}/java/lang/Double.html#compareTo%2Djava.lang.Double%2D[compareTo](<<painless-api-reference-Double,Double>>)++ (link:{java9-javadoc}/java/lang/Double.html#compareTo%2Djava.lang.Double%2D[java 9])
+* ++[[painless-api-reference-Double-isInfinite-0]]boolean link:{java8-javadoc}/java/lang/Double.html#isInfinite%2D%2D[isInfinite]()++ (link:{java9-javadoc}/java/lang/Double.html#isInfinite%2D%2D[java 9])
+* ++[[painless-api-reference-Double-isNaN-0]]boolean link:{java8-javadoc}/java/lang/Double.html#isNaN%2D%2D[isNaN]()++ (link:{java9-javadoc}/java/lang/Double.html#isNaN%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Number,Number>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/DoubleBinaryOperator.asciidoc b/docs/painless/painless-api-reference/DoubleBinaryOperator.asciidoc
new file mode 100644
index 0000000000000..deec8c5337c81
--- /dev/null
+++ b/docs/painless/painless-api-reference/DoubleBinaryOperator.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-DoubleBinaryOperator]]++DoubleBinaryOperator++::
+* ++[[painless-api-reference-DoubleBinaryOperator-applyAsDouble-2]]double link:{java8-javadoc}/java/util/function/DoubleBinaryOperator.html#applyAsDouble%2Ddouble%2Ddouble%2D[applyAsDouble](double, double)++ (link:{java9-javadoc}/java/util/function/DoubleBinaryOperator.html#applyAsDouble%2Ddouble%2Ddouble%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/DoubleConsumer.asciidoc b/docs/painless/painless-api-reference/DoubleConsumer.asciidoc
new file mode 100644
index 0000000000000..3c078a4d4fb1c
--- /dev/null
+++ b/docs/painless/painless-api-reference/DoubleConsumer.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-DoubleConsumer]]++DoubleConsumer++::
+* ++[[painless-api-reference-DoubleConsumer-accept-1]]void link:{java8-javadoc}/java/util/function/DoubleConsumer.html#accept%2Ddouble%2D[accept](double)++ (link:{java9-javadoc}/java/util/function/DoubleConsumer.html#accept%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-DoubleConsumer-andThen-1]]<<painless-api-reference-DoubleConsumer,DoubleConsumer>> link:{java8-javadoc}/java/util/function/DoubleConsumer.html#andThen%2Djava.util.function.DoubleConsumer%2D[andThen](<<painless-api-reference-DoubleConsumer,DoubleConsumer>>)++ (link:{java9-javadoc}/java/util/function/DoubleConsumer.html#andThen%2Djava.util.function.DoubleConsumer%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/DoubleFunction.asciidoc b/docs/painless/painless-api-reference/DoubleFunction.asciidoc
new file mode 100644
index 0000000000000..082a70001328e
--- /dev/null
+++ b/docs/painless/painless-api-reference/DoubleFunction.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-DoubleFunction]]++DoubleFunction++::
+* ++[[painless-api-reference-DoubleFunction-apply-1]]def link:{java8-javadoc}/java/util/function/DoubleFunction.html#apply%2Ddouble%2D[apply](double)++ (link:{java9-javadoc}/java/util/function/DoubleFunction.html#apply%2Ddouble%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/DoublePredicate.asciidoc b/docs/painless/painless-api-reference/DoublePredicate.asciidoc
new file mode 100644
index 0000000000000..48cfd3c59296c
--- /dev/null
+++ b/docs/painless/painless-api-reference/DoublePredicate.asciidoc
@@ -0,0 +1,11 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-DoublePredicate]]++DoublePredicate++::
+* ++[[painless-api-reference-DoublePredicate-and-1]]<<painless-api-reference-DoublePredicate,DoublePredicate>> link:{java8-javadoc}/java/util/function/DoublePredicate.html#and%2Djava.util.function.DoublePredicate%2D[and](<<painless-api-reference-DoublePredicate,DoublePredicate>>)++ (link:{java9-javadoc}/java/util/function/DoublePredicate.html#and%2Djava.util.function.DoublePredicate%2D[java 9])
+* ++[[painless-api-reference-DoublePredicate-negate-0]]<<painless-api-reference-DoublePredicate,DoublePredicate>> link:{java8-javadoc}/java/util/function/DoublePredicate.html#negate%2D%2D[negate]()++ (link:{java9-javadoc}/java/util/function/DoublePredicate.html#negate%2D%2D[java 9])
+* ++[[painless-api-reference-DoublePredicate-or-1]]<<painless-api-reference-DoublePredicate,DoublePredicate>> link:{java8-javadoc}/java/util/function/DoublePredicate.html#or%2Djava.util.function.DoublePredicate%2D[or](<<painless-api-reference-DoublePredicate,DoublePredicate>>)++ (link:{java9-javadoc}/java/util/function/DoublePredicate.html#or%2Djava.util.function.DoublePredicate%2D[java 9])
+* ++[[painless-api-reference-DoublePredicate-test-1]]boolean link:{java8-javadoc}/java/util/function/DoublePredicate.html#test%2Ddouble%2D[test](double)++ (link:{java9-javadoc}/java/util/function/DoublePredicate.html#test%2Ddouble%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/DoubleStream.Builder.asciidoc b/docs/painless/painless-api-reference/DoubleStream.Builder.asciidoc
new file mode 100644
index 0000000000000..8f59249115f8f
--- /dev/null
+++ b/docs/painless/painless-api-reference/DoubleStream.Builder.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-DoubleStream-Builder]]++DoubleStream.Builder++::
+* ++[[painless-api-reference-DoubleStream-Builder-add-1]]<<painless-api-reference-DoubleStream-Builder,DoubleStream.Builder>> link:{java8-javadoc}/java/util/stream/DoubleStream.Builder.html#add%2Ddouble%2D[add](double)++ (link:{java9-javadoc}/java/util/stream/DoubleStream.Builder.html#add%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-Builder-build-0]]<<painless-api-reference-DoubleStream,DoubleStream>> link:{java8-javadoc}/java/util/stream/DoubleStream.Builder.html#build%2D%2D[build]()++ (link:{java9-javadoc}/java/util/stream/DoubleStream.Builder.html#build%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-DoubleConsumer,DoubleConsumer>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/DoubleStream.asciidoc b/docs/painless/painless-api-reference/DoubleStream.asciidoc
new file mode 100644
index 0000000000000..d3f517e29e40b
--- /dev/null
+++ b/docs/painless/painless-api-reference/DoubleStream.asciidoc
@@ -0,0 +1,43 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-DoubleStream]]++DoubleStream++::
+* ++[[painless-api-reference-DoubleStream-builder-0]]static <<painless-api-reference-DoubleStream-Builder,DoubleStream.Builder>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#builder%2D%2D[builder]()++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#builder%2D%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-concat-2]]static <<painless-api-reference-DoubleStream,DoubleStream>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#concat%2Djava.util.stream.DoubleStream%2Djava.util.stream.DoubleStream%2D[concat](<<painless-api-reference-DoubleStream,DoubleStream>>, <<painless-api-reference-DoubleStream,DoubleStream>>)++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#concat%2Djava.util.stream.DoubleStream%2Djava.util.stream.DoubleStream%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-empty-0]]static <<painless-api-reference-DoubleStream,DoubleStream>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#empty%2D%2D[empty]()++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#empty%2D%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-of-1]]static <<painless-api-reference-DoubleStream,DoubleStream>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#of%2Ddouble:A%2D[of](double[])++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#of%2Ddouble:A%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-allMatch-1]]boolean link:{java8-javadoc}/java/util/stream/DoubleStream.html#allMatch%2Djava.util.function.DoublePredicate%2D[allMatch](<<painless-api-reference-DoublePredicate,DoublePredicate>>)++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#allMatch%2Djava.util.function.DoublePredicate%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-anyMatch-1]]boolean link:{java8-javadoc}/java/util/stream/DoubleStream.html#anyMatch%2Djava.util.function.DoublePredicate%2D[anyMatch](<<painless-api-reference-DoublePredicate,DoublePredicate>>)++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#anyMatch%2Djava.util.function.DoublePredicate%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-average-0]]<<painless-api-reference-OptionalDouble,OptionalDouble>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#average%2D%2D[average]()++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#average%2D%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-boxed-0]]<<painless-api-reference-Stream,Stream>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#boxed%2D%2D[boxed]()++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#boxed%2D%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-collect-3]]def link:{java8-javadoc}/java/util/stream/DoubleStream.html#collect%2Djava.util.function.Supplier%2Djava.util.function.ObjDoubleConsumer%2Djava.util.function.BiConsumer%2D[collect](<<painless-api-reference-Supplier,Supplier>>, <<painless-api-reference-ObjDoubleConsumer,ObjDoubleConsumer>>, <<painless-api-reference-BiConsumer,BiConsumer>>)++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#collect%2Djava.util.function.Supplier%2Djava.util.function.ObjDoubleConsumer%2Djava.util.function.BiConsumer%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-count-0]]long link:{java8-javadoc}/java/util/stream/DoubleStream.html#count%2D%2D[count]()++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#count%2D%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-distinct-0]]<<painless-api-reference-DoubleStream,DoubleStream>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#distinct%2D%2D[distinct]()++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#distinct%2D%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-filter-1]]<<painless-api-reference-DoubleStream,DoubleStream>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#filter%2Djava.util.function.DoublePredicate%2D[filter](<<painless-api-reference-DoublePredicate,DoublePredicate>>)++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#filter%2Djava.util.function.DoublePredicate%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-findAny-0]]<<painless-api-reference-OptionalDouble,OptionalDouble>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#findAny%2D%2D[findAny]()++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#findAny%2D%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-findFirst-0]]<<painless-api-reference-OptionalDouble,OptionalDouble>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#findFirst%2D%2D[findFirst]()++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#findFirst%2D%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-flatMap-1]]<<painless-api-reference-DoubleStream,DoubleStream>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#flatMap%2Djava.util.function.DoubleFunction%2D[flatMap](<<painless-api-reference-DoubleFunction,DoubleFunction>>)++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#flatMap%2Djava.util.function.DoubleFunction%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-forEach-1]]void link:{java8-javadoc}/java/util/stream/DoubleStream.html#forEach%2Djava.util.function.DoubleConsumer%2D[forEach](<<painless-api-reference-DoubleConsumer,DoubleConsumer>>)++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#forEach%2Djava.util.function.DoubleConsumer%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-forEachOrdered-1]]void link:{java8-javadoc}/java/util/stream/DoubleStream.html#forEachOrdered%2Djava.util.function.DoubleConsumer%2D[forEachOrdered](<<painless-api-reference-DoubleConsumer,DoubleConsumer>>)++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#forEachOrdered%2Djava.util.function.DoubleConsumer%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-iterator-0]]<<painless-api-reference-PrimitiveIterator-OfDouble,PrimitiveIterator.OfDouble>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#iterator%2D%2D[iterator]()++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#iterator%2D%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-limit-1]]<<painless-api-reference-DoubleStream,DoubleStream>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#limit%2Dlong%2D[limit](long)++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#limit%2Dlong%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-map-1]]<<painless-api-reference-DoubleStream,DoubleStream>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#map%2Djava.util.function.DoubleUnaryOperator%2D[map](<<painless-api-reference-DoubleUnaryOperator,DoubleUnaryOperator>>)++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#map%2Djava.util.function.DoubleUnaryOperator%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-mapToInt-1]]<<painless-api-reference-IntStream,IntStream>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#mapToInt%2Djava.util.function.DoubleToIntFunction%2D[mapToInt](<<painless-api-reference-DoubleToIntFunction,DoubleToIntFunction>>)++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#mapToInt%2Djava.util.function.DoubleToIntFunction%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-mapToLong-1]]<<painless-api-reference-LongStream,LongStream>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#mapToLong%2Djava.util.function.DoubleToLongFunction%2D[mapToLong](<<painless-api-reference-DoubleToLongFunction,DoubleToLongFunction>>)++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#mapToLong%2Djava.util.function.DoubleToLongFunction%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-mapToObj-1]]<<painless-api-reference-Stream,Stream>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#mapToObj%2Djava.util.function.DoubleFunction%2D[mapToObj](<<painless-api-reference-DoubleFunction,DoubleFunction>>)++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#mapToObj%2Djava.util.function.DoubleFunction%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-max-0]]<<painless-api-reference-OptionalDouble,OptionalDouble>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#max%2D%2D[max]()++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#max%2D%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-min-0]]<<painless-api-reference-OptionalDouble,OptionalDouble>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#min%2D%2D[min]()++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#min%2D%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-noneMatch-1]]boolean link:{java8-javadoc}/java/util/stream/DoubleStream.html#noneMatch%2Djava.util.function.DoublePredicate%2D[noneMatch](<<painless-api-reference-DoublePredicate,DoublePredicate>>)++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#noneMatch%2Djava.util.function.DoublePredicate%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-peek-1]]<<painless-api-reference-DoubleStream,DoubleStream>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#peek%2Djava.util.function.DoubleConsumer%2D[peek](<<painless-api-reference-DoubleConsumer,DoubleConsumer>>)++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#peek%2Djava.util.function.DoubleConsumer%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-reduce-1]]<<painless-api-reference-OptionalDouble,OptionalDouble>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#reduce%2Djava.util.function.DoubleBinaryOperator%2D[reduce](<<painless-api-reference-DoubleBinaryOperator,DoubleBinaryOperator>>)++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#reduce%2Djava.util.function.DoubleBinaryOperator%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-reduce-2]]double link:{java8-javadoc}/java/util/stream/DoubleStream.html#reduce%2Ddouble%2Djava.util.function.DoubleBinaryOperator%2D[reduce](double, <<painless-api-reference-DoubleBinaryOperator,DoubleBinaryOperator>>)++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#reduce%2Ddouble%2Djava.util.function.DoubleBinaryOperator%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-sequential-0]]<<painless-api-reference-DoubleStream,DoubleStream>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#sequential%2D%2D[sequential]()++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#sequential%2D%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-skip-1]]<<painless-api-reference-DoubleStream,DoubleStream>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#skip%2Dlong%2D[skip](long)++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#skip%2Dlong%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-sorted-0]]<<painless-api-reference-DoubleStream,DoubleStream>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#sorted%2D%2D[sorted]()++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#sorted%2D%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-spliterator-0]]<<painless-api-reference-Spliterator-OfDouble,Spliterator.OfDouble>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#spliterator%2D%2D[spliterator]()++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#spliterator%2D%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-sum-0]]double link:{java8-javadoc}/java/util/stream/DoubleStream.html#sum%2D%2D[sum]()++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#sum%2D%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-summaryStatistics-0]]<<painless-api-reference-DoubleSummaryStatistics,DoubleSummaryStatistics>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#summaryStatistics%2D%2D[summaryStatistics]()++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#summaryStatistics%2D%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-toArray-0]]double[] link:{java8-javadoc}/java/util/stream/DoubleStream.html#toArray%2D%2D[toArray]()++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#toArray%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-BaseStream,BaseStream>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/DoubleSummaryStatistics.asciidoc b/docs/painless/painless-api-reference/DoubleSummaryStatistics.asciidoc
new file mode 100644
index 0000000000000..976eb98440410
--- /dev/null
+++ b/docs/painless/painless-api-reference/DoubleSummaryStatistics.asciidoc
@@ -0,0 +1,14 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-DoubleSummaryStatistics]]++DoubleSummaryStatistics++::
+* ++[[painless-api-reference-DoubleSummaryStatistics-DoubleSummaryStatistics-0]]link:{java8-javadoc}/java/util/DoubleSummaryStatistics.html#DoubleSummaryStatistics%2D%2D[DoubleSummaryStatistics]()++ (link:{java9-javadoc}/java/util/DoubleSummaryStatistics.html#DoubleSummaryStatistics%2D%2D[java 9])
+* ++[[painless-api-reference-DoubleSummaryStatistics-combine-1]]void link:{java8-javadoc}/java/util/DoubleSummaryStatistics.html#combine%2Djava.util.DoubleSummaryStatistics%2D[combine](<<painless-api-reference-DoubleSummaryStatistics,DoubleSummaryStatistics>>)++ (link:{java9-javadoc}/java/util/DoubleSummaryStatistics.html#combine%2Djava.util.DoubleSummaryStatistics%2D[java 9])
+* ++[[painless-api-reference-DoubleSummaryStatistics-getAverage-0]]double link:{java8-javadoc}/java/util/DoubleSummaryStatistics.html#getAverage%2D%2D[getAverage]()++ (link:{java9-javadoc}/java/util/DoubleSummaryStatistics.html#getAverage%2D%2D[java 9])
+* ++[[painless-api-reference-DoubleSummaryStatistics-getCount-0]]long link:{java8-javadoc}/java/util/DoubleSummaryStatistics.html#getCount%2D%2D[getCount]()++ (link:{java9-javadoc}/java/util/DoubleSummaryStatistics.html#getCount%2D%2D[java 9])
+* ++[[painless-api-reference-DoubleSummaryStatistics-getMax-0]]double link:{java8-javadoc}/java/util/DoubleSummaryStatistics.html#getMax%2D%2D[getMax]()++ (link:{java9-javadoc}/java/util/DoubleSummaryStatistics.html#getMax%2D%2D[java 9])
+* ++[[painless-api-reference-DoubleSummaryStatistics-getMin-0]]double link:{java8-javadoc}/java/util/DoubleSummaryStatistics.html#getMin%2D%2D[getMin]()++ (link:{java9-javadoc}/java/util/DoubleSummaryStatistics.html#getMin%2D%2D[java 9])
+* ++[[painless-api-reference-DoubleSummaryStatistics-getSum-0]]double link:{java8-javadoc}/java/util/DoubleSummaryStatistics.html#getSum%2D%2D[getSum]()++ (link:{java9-javadoc}/java/util/DoubleSummaryStatistics.html#getSum%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-DoubleConsumer,DoubleConsumer>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/DoubleSupplier.asciidoc b/docs/painless/painless-api-reference/DoubleSupplier.asciidoc
new file mode 100644
index 0000000000000..1de478ab50f95
--- /dev/null
+++ b/docs/painless/painless-api-reference/DoubleSupplier.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-DoubleSupplier]]++DoubleSupplier++::
+* ++[[painless-api-reference-DoubleSupplier-getAsDouble-0]]double link:{java8-javadoc}/java/util/function/DoubleSupplier.html#getAsDouble%2D%2D[getAsDouble]()++ (link:{java9-javadoc}/java/util/function/DoubleSupplier.html#getAsDouble%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/DoubleToIntFunction.asciidoc b/docs/painless/painless-api-reference/DoubleToIntFunction.asciidoc
new file mode 100644
index 0000000000000..9448a88faf35a
--- /dev/null
+++ b/docs/painless/painless-api-reference/DoubleToIntFunction.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-DoubleToIntFunction]]++DoubleToIntFunction++::
+* ++[[painless-api-reference-DoubleToIntFunction-applyAsInt-1]]int link:{java8-javadoc}/java/util/function/DoubleToIntFunction.html#applyAsInt%2Ddouble%2D[applyAsInt](double)++ (link:{java9-javadoc}/java/util/function/DoubleToIntFunction.html#applyAsInt%2Ddouble%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/DoubleToLongFunction.asciidoc b/docs/painless/painless-api-reference/DoubleToLongFunction.asciidoc
new file mode 100644
index 0000000000000..62a3f841b3391
--- /dev/null
+++ b/docs/painless/painless-api-reference/DoubleToLongFunction.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-DoubleToLongFunction]]++DoubleToLongFunction++::
+* ++[[painless-api-reference-DoubleToLongFunction-applyAsLong-1]]long link:{java8-javadoc}/java/util/function/DoubleToLongFunction.html#applyAsLong%2Ddouble%2D[applyAsLong](double)++ (link:{java9-javadoc}/java/util/function/DoubleToLongFunction.html#applyAsLong%2Ddouble%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/DoubleUnaryOperator.asciidoc b/docs/painless/painless-api-reference/DoubleUnaryOperator.asciidoc
new file mode 100644
index 0000000000000..a78b8ac5d0b1c
--- /dev/null
+++ b/docs/painless/painless-api-reference/DoubleUnaryOperator.asciidoc
@@ -0,0 +1,11 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-DoubleUnaryOperator]]++DoubleUnaryOperator++::
+* ++[[painless-api-reference-DoubleUnaryOperator-identity-0]]static <<painless-api-reference-DoubleUnaryOperator,DoubleUnaryOperator>> link:{java8-javadoc}/java/util/function/DoubleUnaryOperator.html#identity%2D%2D[identity]()++ (link:{java9-javadoc}/java/util/function/DoubleUnaryOperator.html#identity%2D%2D[java 9])
+* ++[[painless-api-reference-DoubleUnaryOperator-andThen-1]]<<painless-api-reference-DoubleUnaryOperator,DoubleUnaryOperator>> link:{java8-javadoc}/java/util/function/DoubleUnaryOperator.html#andThen%2Djava.util.function.DoubleUnaryOperator%2D[andThen](<<painless-api-reference-DoubleUnaryOperator,DoubleUnaryOperator>>)++ (link:{java9-javadoc}/java/util/function/DoubleUnaryOperator.html#andThen%2Djava.util.function.DoubleUnaryOperator%2D[java 9])
+* ++[[painless-api-reference-DoubleUnaryOperator-applyAsDouble-1]]double link:{java8-javadoc}/java/util/function/DoubleUnaryOperator.html#applyAsDouble%2Ddouble%2D[applyAsDouble](double)++ (link:{java9-javadoc}/java/util/function/DoubleUnaryOperator.html#applyAsDouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-DoubleUnaryOperator-compose-1]]<<painless-api-reference-DoubleUnaryOperator,DoubleUnaryOperator>> link:{java8-javadoc}/java/util/function/DoubleUnaryOperator.html#compose%2Djava.util.function.DoubleUnaryOperator%2D[compose](<<painless-api-reference-DoubleUnaryOperator,DoubleUnaryOperator>>)++ (link:{java9-javadoc}/java/util/function/DoubleUnaryOperator.html#compose%2Djava.util.function.DoubleUnaryOperator%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/DuplicateFormatFlagsException.asciidoc b/docs/painless/painless-api-reference/DuplicateFormatFlagsException.asciidoc
new file mode 100644
index 0000000000000..f26103fdc198f
--- /dev/null
+++ b/docs/painless/painless-api-reference/DuplicateFormatFlagsException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-DuplicateFormatFlagsException]]++DuplicateFormatFlagsException++::
+* ++[[painless-api-reference-DuplicateFormatFlagsException-DuplicateFormatFlagsException-1]]link:{java8-javadoc}/java/util/DuplicateFormatFlagsException.html#DuplicateFormatFlagsException%2Djava.lang.String%2D[DuplicateFormatFlagsException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/DuplicateFormatFlagsException.html#DuplicateFormatFlagsException%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-DuplicateFormatFlagsException-getFlags-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/DuplicateFormatFlagsException.html#getFlags%2D%2D[getFlags]()++ (link:{java9-javadoc}/java/util/DuplicateFormatFlagsException.html#getFlags%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Duration.asciidoc b/docs/painless/painless-api-reference/Duration.asciidoc
new file mode 100644
index 0000000000000..ef9ecf720a0f2
--- /dev/null
+++ b/docs/painless/painless-api-reference/Duration.asciidoc
@@ -0,0 +1,51 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Duration]]++Duration++::
+** [[painless-api-reference-Duration-ZERO]]static <<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#ZERO[ZERO] (link:{java9-javadoc}/java/time/Duration.html#ZERO[java 9])
+* ++[[painless-api-reference-Duration-between-2]]static <<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#between%2Djava.time.temporal.Temporal%2Djava.time.temporal.Temporal%2D[between](<<painless-api-reference-Temporal,Temporal>>, <<painless-api-reference-Temporal,Temporal>>)++ (link:{java9-javadoc}/java/time/Duration.html#between%2Djava.time.temporal.Temporal%2Djava.time.temporal.Temporal%2D[java 9])
+* ++[[painless-api-reference-Duration-from-1]]static <<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#from%2Djava.time.temporal.TemporalAmount%2D[from](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/Duration.html#from%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-Duration-of-2]]static <<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#of%2Dlong%2Djava.time.temporal.TemporalUnit%2D[of](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/Duration.html#of%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-Duration-ofDays-1]]static <<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#ofDays%2Dlong%2D[ofDays](long)++ (link:{java9-javadoc}/java/time/Duration.html#ofDays%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Duration-ofHours-1]]static <<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#ofHours%2Dlong%2D[ofHours](long)++ (link:{java9-javadoc}/java/time/Duration.html#ofHours%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Duration-ofMillis-1]]static <<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#ofMillis%2Dlong%2D[ofMillis](long)++ (link:{java9-javadoc}/java/time/Duration.html#ofMillis%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Duration-ofMinutes-1]]static <<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#ofMinutes%2Dlong%2D[ofMinutes](long)++ (link:{java9-javadoc}/java/time/Duration.html#ofMinutes%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Duration-ofNanos-1]]static <<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#ofNanos%2Dlong%2D[ofNanos](long)++ (link:{java9-javadoc}/java/time/Duration.html#ofNanos%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Duration-ofSeconds-1]]static <<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#ofSeconds%2Dlong%2D[ofSeconds](long)++ (link:{java9-javadoc}/java/time/Duration.html#ofSeconds%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Duration-ofSeconds-2]]static <<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#ofSeconds%2Dlong%2Dlong%2D[ofSeconds](long, long)++ (link:{java9-javadoc}/java/time/Duration.html#ofSeconds%2Dlong%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Duration-parse-1]]static <<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#parse%2Djava.lang.CharSequence%2D[parse](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/time/Duration.html#parse%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-Duration-abs-0]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#abs%2D%2D[abs]()++ (link:{java9-javadoc}/java/time/Duration.html#abs%2D%2D[java 9])
+* ++[[painless-api-reference-Duration-compareTo-1]]int link:{java8-javadoc}/java/time/Duration.html#compareTo%2Djava.time.Duration%2D[compareTo](<<painless-api-reference-Duration,Duration>>)++ (link:{java9-javadoc}/java/time/Duration.html#compareTo%2Djava.time.Duration%2D[java 9])
+* ++[[painless-api-reference-Duration-dividedBy-1]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#dividedBy%2Dlong%2D[dividedBy](long)++ (link:{java9-javadoc}/java/time/Duration.html#dividedBy%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Duration-getNano-0]]int link:{java8-javadoc}/java/time/Duration.html#getNano%2D%2D[getNano]()++ (link:{java9-javadoc}/java/time/Duration.html#getNano%2D%2D[java 9])
+* ++[[painless-api-reference-Duration-getSeconds-0]]long link:{java8-javadoc}/java/time/Duration.html#getSeconds%2D%2D[getSeconds]()++ (link:{java9-javadoc}/java/time/Duration.html#getSeconds%2D%2D[java 9])
+* ++[[painless-api-reference-Duration-isNegative-0]]boolean link:{java8-javadoc}/java/time/Duration.html#isNegative%2D%2D[isNegative]()++ (link:{java9-javadoc}/java/time/Duration.html#isNegative%2D%2D[java 9])
+* ++[[painless-api-reference-Duration-isZero-0]]boolean link:{java8-javadoc}/java/time/Duration.html#isZero%2D%2D[isZero]()++ (link:{java9-javadoc}/java/time/Duration.html#isZero%2D%2D[java 9])
+* ++[[painless-api-reference-Duration-minus-1]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#minus%2Djava.time.Duration%2D[minus](<<painless-api-reference-Duration,Duration>>)++ (link:{java9-javadoc}/java/time/Duration.html#minus%2Djava.time.Duration%2D[java 9])
+* ++[[painless-api-reference-Duration-minus-2]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[minus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/Duration.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-Duration-minusDays-1]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#minusDays%2Dlong%2D[minusDays](long)++ (link:{java9-javadoc}/java/time/Duration.html#minusDays%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Duration-minusHours-1]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#minusHours%2Dlong%2D[minusHours](long)++ (link:{java9-javadoc}/java/time/Duration.html#minusHours%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Duration-minusMillis-1]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#minusMillis%2Dlong%2D[minusMillis](long)++ (link:{java9-javadoc}/java/time/Duration.html#minusMillis%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Duration-minusMinutes-1]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#minusMinutes%2Dlong%2D[minusMinutes](long)++ (link:{java9-javadoc}/java/time/Duration.html#minusMinutes%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Duration-minusNanos-1]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#minusNanos%2Dlong%2D[minusNanos](long)++ (link:{java9-javadoc}/java/time/Duration.html#minusNanos%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Duration-minusSeconds-1]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#minusSeconds%2Dlong%2D[minusSeconds](long)++ (link:{java9-javadoc}/java/time/Duration.html#minusSeconds%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Duration-multipliedBy-1]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#multipliedBy%2Dlong%2D[multipliedBy](long)++ (link:{java9-javadoc}/java/time/Duration.html#multipliedBy%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Duration-negated-0]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#negated%2D%2D[negated]()++ (link:{java9-javadoc}/java/time/Duration.html#negated%2D%2D[java 9])
+* ++[[painless-api-reference-Duration-plus-1]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#plus%2Djava.time.Duration%2D[plus](<<painless-api-reference-Duration,Duration>>)++ (link:{java9-javadoc}/java/time/Duration.html#plus%2Djava.time.Duration%2D[java 9])
+* ++[[painless-api-reference-Duration-plus-2]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[plus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/Duration.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-Duration-plusDays-1]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#plusDays%2Dlong%2D[plusDays](long)++ (link:{java9-javadoc}/java/time/Duration.html#plusDays%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Duration-plusHours-1]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#plusHours%2Dlong%2D[plusHours](long)++ (link:{java9-javadoc}/java/time/Duration.html#plusHours%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Duration-plusMillis-1]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#plusMillis%2Dlong%2D[plusMillis](long)++ (link:{java9-javadoc}/java/time/Duration.html#plusMillis%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Duration-plusMinutes-1]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#plusMinutes%2Dlong%2D[plusMinutes](long)++ (link:{java9-javadoc}/java/time/Duration.html#plusMinutes%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Duration-plusNanos-1]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#plusNanos%2Dlong%2D[plusNanos](long)++ (link:{java9-javadoc}/java/time/Duration.html#plusNanos%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Duration-plusSeconds-1]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#plusSeconds%2Dlong%2D[plusSeconds](long)++ (link:{java9-javadoc}/java/time/Duration.html#plusSeconds%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Duration-toDays-0]]long link:{java8-javadoc}/java/time/Duration.html#toDays%2D%2D[toDays]()++ (link:{java9-javadoc}/java/time/Duration.html#toDays%2D%2D[java 9])
+* ++[[painless-api-reference-Duration-toHours-0]]long link:{java8-javadoc}/java/time/Duration.html#toHours%2D%2D[toHours]()++ (link:{java9-javadoc}/java/time/Duration.html#toHours%2D%2D[java 9])
+* ++[[painless-api-reference-Duration-toMillis-0]]long link:{java8-javadoc}/java/time/Duration.html#toMillis%2D%2D[toMillis]()++ (link:{java9-javadoc}/java/time/Duration.html#toMillis%2D%2D[java 9])
+* ++[[painless-api-reference-Duration-toMinutes-0]]long link:{java8-javadoc}/java/time/Duration.html#toMinutes%2D%2D[toMinutes]()++ (link:{java9-javadoc}/java/time/Duration.html#toMinutes%2D%2D[java 9])
+* ++[[painless-api-reference-Duration-toNanos-0]]long link:{java8-javadoc}/java/time/Duration.html#toNanos%2D%2D[toNanos]()++ (link:{java9-javadoc}/java/time/Duration.html#toNanos%2D%2D[java 9])
+* ++[[painless-api-reference-Duration-withNanos-1]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#withNanos%2Dint%2D[withNanos](int)++ (link:{java9-javadoc}/java/time/Duration.html#withNanos%2Dint%2D[java 9])
+* ++[[painless-api-reference-Duration-withSeconds-1]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#withSeconds%2Dlong%2D[withSeconds](long)++ (link:{java9-javadoc}/java/time/Duration.html#withSeconds%2Dlong%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-TemporalAmount,TemporalAmount>>++
diff --git a/docs/painless/painless-api-reference/EmptyStackException.asciidoc b/docs/painless/painless-api-reference/EmptyStackException.asciidoc
new file mode 100644
index 0000000000000..09ca4ca96be20
--- /dev/null
+++ b/docs/painless/painless-api-reference/EmptyStackException.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-EmptyStackException]]++EmptyStackException++::
+* ++[[painless-api-reference-EmptyStackException-EmptyStackException-0]]link:{java8-javadoc}/java/util/EmptyStackException.html#EmptyStackException%2D%2D[EmptyStackException]()++ (link:{java9-javadoc}/java/util/EmptyStackException.html#EmptyStackException%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Enum.asciidoc b/docs/painless/painless-api-reference/Enum.asciidoc
new file mode 100644
index 0000000000000..077044dbfb9cd
--- /dev/null
+++ b/docs/painless/painless-api-reference/Enum.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Enum]]++Enum++::
+* ++[[painless-api-reference-Enum-compareTo-1]]int link:{java8-javadoc}/java/lang/Enum.html#compareTo%2Djava.lang.Enum%2D[compareTo](<<painless-api-reference-Enum,Enum>>)++ (link:{java9-javadoc}/java/lang/Enum.html#compareTo%2Djava.lang.Enum%2D[java 9])
+* ++[[painless-api-reference-Enum-name-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Enum.html#name%2D%2D[name]()++ (link:{java9-javadoc}/java/lang/Enum.html#name%2D%2D[java 9])
+* ++[[painless-api-reference-Enum-ordinal-0]]int link:{java8-javadoc}/java/lang/Enum.html#ordinal%2D%2D[ordinal]()++ (link:{java9-javadoc}/java/lang/Enum.html#ordinal%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/EnumConstantNotPresentException.asciidoc b/docs/painless/painless-api-reference/EnumConstantNotPresentException.asciidoc
new file mode 100644
index 0000000000000..66a709379d797
--- /dev/null
+++ b/docs/painless/painless-api-reference/EnumConstantNotPresentException.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-EnumConstantNotPresentException]]++EnumConstantNotPresentException++::
+* ++[[painless-api-reference-EnumConstantNotPresentException-constantName-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/EnumConstantNotPresentException.html#constantName%2D%2D[constantName]()++ (link:{java9-javadoc}/java/lang/EnumConstantNotPresentException.html#constantName%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Enumeration.asciidoc b/docs/painless/painless-api-reference/Enumeration.asciidoc
new file mode 100644
index 0000000000000..51849ec27de0f
--- /dev/null
+++ b/docs/painless/painless-api-reference/Enumeration.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Enumeration]]++Enumeration++::
+* ++[[painless-api-reference-Enumeration-hasMoreElements-0]]boolean link:{java8-javadoc}/java/util/Enumeration.html#hasMoreElements%2D%2D[hasMoreElements]()++ (link:{java9-javadoc}/java/util/Enumeration.html#hasMoreElements%2D%2D[java 9])
+* ++[[painless-api-reference-Enumeration-nextElement-0]]def link:{java8-javadoc}/java/util/Enumeration.html#nextElement%2D%2D[nextElement]()++ (link:{java9-javadoc}/java/util/Enumeration.html#nextElement%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Era.asciidoc b/docs/painless/painless-api-reference/Era.asciidoc
new file mode 100644
index 0000000000000..db5ee6aa86b0d
--- /dev/null
+++ b/docs/painless/painless-api-reference/Era.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Era]]++Era++::
+* ++[[painless-api-reference-Era-getDisplayName-2]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/chrono/Era.html#getDisplayName%2Djava.time.format.TextStyle%2Djava.util.Locale%2D[getDisplayName](<<painless-api-reference-TextStyle,TextStyle>>, <<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/time/chrono/Era.html#getDisplayName%2Djava.time.format.TextStyle%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-Era-getValue-0]]int link:{java8-javadoc}/java/time/chrono/Era.html#getValue%2D%2D[getValue]()++ (link:{java9-javadoc}/java/time/chrono/Era.html#getValue%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++, ++<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>++
diff --git a/docs/painless/painless-api-reference/EventListener.asciidoc b/docs/painless/painless-api-reference/EventListener.asciidoc
new file mode 100644
index 0000000000000..499095ae020a0
--- /dev/null
+++ b/docs/painless/painless-api-reference/EventListener.asciidoc
@@ -0,0 +1,7 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-EventListener]]++EventListener++::
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/EventListenerProxy.asciidoc b/docs/painless/painless-api-reference/EventListenerProxy.asciidoc
new file mode 100644
index 0000000000000..a3a4958ec7115
--- /dev/null
+++ b/docs/painless/painless-api-reference/EventListenerProxy.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-EventListenerProxy]]++EventListenerProxy++::
+* ++[[painless-api-reference-EventListenerProxy-getListener-0]]<<painless-api-reference-EventListener,EventListener>> link:{java8-javadoc}/java/util/EventListenerProxy.html#getListener%2D%2D[getListener]()++ (link:{java9-javadoc}/java/util/EventListenerProxy.html#getListener%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/EventObject.asciidoc b/docs/painless/painless-api-reference/EventObject.asciidoc
new file mode 100644
index 0000000000000..4ac6b14150cb1
--- /dev/null
+++ b/docs/painless/painless-api-reference/EventObject.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-EventObject]]++EventObject++::
+* ++[[painless-api-reference-EventObject-EventObject-1]]link:{java8-javadoc}/java/util/EventObject.html#EventObject%2Djava.lang.Object%2D[EventObject](<<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/util/EventObject.html#EventObject%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-EventObject-getSource-0]]<<painless-api-reference-Object,Object>> link:{java8-javadoc}/java/util/EventObject.html#getSource%2D%2D[getSource]()++ (link:{java9-javadoc}/java/util/EventObject.html#getSource%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Exception.asciidoc b/docs/painless/painless-api-reference/Exception.asciidoc
new file mode 100644
index 0000000000000..d6ff7b0a7e53e
--- /dev/null
+++ b/docs/painless/painless-api-reference/Exception.asciidoc
@@ -0,0 +1,12 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Exception]]++Exception++::
+* ++[[painless-api-reference-Exception-Exception-0]]link:{java8-javadoc}/java/lang/Exception.html#Exception%2D%2D[Exception]()++ (link:{java9-javadoc}/java/lang/Exception.html#Exception%2D%2D[java 9])
+* ++[[painless-api-reference-Exception-Exception-1]]link:{java8-javadoc}/java/lang/Exception.html#Exception%2Djava.lang.String%2D[Exception](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/Exception.html#Exception%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Exception-getLocalizedMessage-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Exception.html#getLocalizedMessage%2D%2D[getLocalizedMessage]()++ (link:{java9-javadoc}/java/lang/Exception.html#getLocalizedMessage%2D%2D[java 9])
+* ++[[painless-api-reference-Exception-getMessage-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Exception.html#getMessage%2D%2D[getMessage]()++ (link:{java9-javadoc}/java/lang/Exception.html#getMessage%2D%2D[java 9])
+* ++[[painless-api-reference-Exception-getStackTrace-0]]<<painless-api-reference-StackTraceElement,StackTraceElement>>[] link:{java8-javadoc}/java/lang/Exception.html#getStackTrace%2D%2D[getStackTrace]()++ (link:{java9-javadoc}/java/lang/Exception.html#getStackTrace%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/FieldPosition.asciidoc b/docs/painless/painless-api-reference/FieldPosition.asciidoc
new file mode 100644
index 0000000000000..7a4b5d78407ae
--- /dev/null
+++ b/docs/painless/painless-api-reference/FieldPosition.asciidoc
@@ -0,0 +1,15 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-FieldPosition]]++FieldPosition++::
+* ++[[painless-api-reference-FieldPosition-FieldPosition-1]]link:{java8-javadoc}/java/text/FieldPosition.html#FieldPosition%2Dint%2D[FieldPosition](int)++ (link:{java9-javadoc}/java/text/FieldPosition.html#FieldPosition%2Dint%2D[java 9])
+* ++[[painless-api-reference-FieldPosition-FieldPosition-2]]link:{java8-javadoc}/java/text/FieldPosition.html#FieldPosition%2Djava.text.Format$Field%2Dint%2D[FieldPosition](<<painless-api-reference-Format-Field,Format.Field>>, int)++ (link:{java9-javadoc}/java/text/FieldPosition.html#FieldPosition%2Djava.text.Format$Field%2Dint%2D[java 9])
+* ++[[painless-api-reference-FieldPosition-getBeginIndex-0]]int link:{java8-javadoc}/java/text/FieldPosition.html#getBeginIndex%2D%2D[getBeginIndex]()++ (link:{java9-javadoc}/java/text/FieldPosition.html#getBeginIndex%2D%2D[java 9])
+* ++[[painless-api-reference-FieldPosition-getEndIndex-0]]int link:{java8-javadoc}/java/text/FieldPosition.html#getEndIndex%2D%2D[getEndIndex]()++ (link:{java9-javadoc}/java/text/FieldPosition.html#getEndIndex%2D%2D[java 9])
+* ++[[painless-api-reference-FieldPosition-getField-0]]int link:{java8-javadoc}/java/text/FieldPosition.html#getField%2D%2D[getField]()++ (link:{java9-javadoc}/java/text/FieldPosition.html#getField%2D%2D[java 9])
+* ++[[painless-api-reference-FieldPosition-getFieldAttribute-0]]<<painless-api-reference-Format-Field,Format.Field>> link:{java8-javadoc}/java/text/FieldPosition.html#getFieldAttribute%2D%2D[getFieldAttribute]()++ (link:{java9-javadoc}/java/text/FieldPosition.html#getFieldAttribute%2D%2D[java 9])
+* ++[[painless-api-reference-FieldPosition-setBeginIndex-1]]void link:{java8-javadoc}/java/text/FieldPosition.html#setBeginIndex%2Dint%2D[setBeginIndex](int)++ (link:{java9-javadoc}/java/text/FieldPosition.html#setBeginIndex%2Dint%2D[java 9])
+* ++[[painless-api-reference-FieldPosition-setEndIndex-1]]void link:{java8-javadoc}/java/text/FieldPosition.html#setEndIndex%2Dint%2D[setEndIndex](int)++ (link:{java9-javadoc}/java/text/FieldPosition.html#setEndIndex%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Float.asciidoc b/docs/painless/painless-api-reference/Float.asciidoc
new file mode 100644
index 0000000000000..1d20357842cfc
--- /dev/null
+++ b/docs/painless/painless-api-reference/Float.asciidoc
@@ -0,0 +1,35 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Float]]++Float++::
+** [[painless-api-reference-Float-BYTES]]static int link:{java8-javadoc}/java/lang/Float.html#BYTES[BYTES] (link:{java9-javadoc}/java/lang/Float.html#BYTES[java 9])
+** [[painless-api-reference-Float-MAX_EXPONENT]]static int link:{java8-javadoc}/java/lang/Float.html#MAX_EXPONENT[MAX_EXPONENT] (link:{java9-javadoc}/java/lang/Float.html#MAX_EXPONENT[java 9])
+** [[painless-api-reference-Float-MAX_VALUE]]static float link:{java8-javadoc}/java/lang/Float.html#MAX_VALUE[MAX_VALUE] (link:{java9-javadoc}/java/lang/Float.html#MAX_VALUE[java 9])
+** [[painless-api-reference-Float-MIN_EXPONENT]]static int link:{java8-javadoc}/java/lang/Float.html#MIN_EXPONENT[MIN_EXPONENT] (link:{java9-javadoc}/java/lang/Float.html#MIN_EXPONENT[java 9])
+** [[painless-api-reference-Float-MIN_NORMAL]]static float link:{java8-javadoc}/java/lang/Float.html#MIN_NORMAL[MIN_NORMAL] (link:{java9-javadoc}/java/lang/Float.html#MIN_NORMAL[java 9])
+** [[painless-api-reference-Float-MIN_VALUE]]static float link:{java8-javadoc}/java/lang/Float.html#MIN_VALUE[MIN_VALUE] (link:{java9-javadoc}/java/lang/Float.html#MIN_VALUE[java 9])
+** [[painless-api-reference-Float-NEGATIVE_INFINITY]]static float link:{java8-javadoc}/java/lang/Float.html#NEGATIVE_INFINITY[NEGATIVE_INFINITY] (link:{java9-javadoc}/java/lang/Float.html#NEGATIVE_INFINITY[java 9])
+** [[painless-api-reference-Float-NaN]]static float link:{java8-javadoc}/java/lang/Float.html#NaN[NaN] (link:{java9-javadoc}/java/lang/Float.html#NaN[java 9])
+** [[painless-api-reference-Float-POSITIVE_INFINITY]]static float link:{java8-javadoc}/java/lang/Float.html#POSITIVE_INFINITY[POSITIVE_INFINITY] (link:{java9-javadoc}/java/lang/Float.html#POSITIVE_INFINITY[java 9])
+** [[painless-api-reference-Float-SIZE]]static int link:{java8-javadoc}/java/lang/Float.html#SIZE[SIZE] (link:{java9-javadoc}/java/lang/Float.html#SIZE[java 9])
+* ++[[painless-api-reference-Float-compare-2]]static int link:{java8-javadoc}/java/lang/Float.html#compare%2Dfloat%2Dfloat%2D[compare](float, float)++ (link:{java9-javadoc}/java/lang/Float.html#compare%2Dfloat%2Dfloat%2D[java 9])
+* ++[[painless-api-reference-Float-floatToIntBits-1]]static int link:{java8-javadoc}/java/lang/Float.html#floatToIntBits%2Dfloat%2D[floatToIntBits](float)++ (link:{java9-javadoc}/java/lang/Float.html#floatToIntBits%2Dfloat%2D[java 9])
+* ++[[painless-api-reference-Float-floatToRawIntBits-1]]static int link:{java8-javadoc}/java/lang/Float.html#floatToRawIntBits%2Dfloat%2D[floatToRawIntBits](float)++ (link:{java9-javadoc}/java/lang/Float.html#floatToRawIntBits%2Dfloat%2D[java 9])
+* ++[[painless-api-reference-Float-hashCode-1]]static int link:{java8-javadoc}/java/lang/Float.html#hashCode%2Dfloat%2D[hashCode](float)++ (link:{java9-javadoc}/java/lang/Float.html#hashCode%2Dfloat%2D[java 9])
+* ++[[painless-api-reference-Float-intBitsToFloat-1]]static float link:{java8-javadoc}/java/lang/Float.html#intBitsToFloat%2Dint%2D[intBitsToFloat](int)++ (link:{java9-javadoc}/java/lang/Float.html#intBitsToFloat%2Dint%2D[java 9])
+* ++[[painless-api-reference-Float-isFinite-1]]static boolean link:{java8-javadoc}/java/lang/Float.html#isFinite%2Dfloat%2D[isFinite](float)++ (link:{java9-javadoc}/java/lang/Float.html#isFinite%2Dfloat%2D[java 9])
+* ++[[painless-api-reference-Float-isInfinite-1]]static boolean link:{java8-javadoc}/java/lang/Float.html#isInfinite%2Dfloat%2D[isInfinite](float)++ (link:{java9-javadoc}/java/lang/Float.html#isInfinite%2Dfloat%2D[java 9])
+* ++[[painless-api-reference-Float-isNaN-1]]static boolean link:{java8-javadoc}/java/lang/Float.html#isNaN%2Dfloat%2D[isNaN](float)++ (link:{java9-javadoc}/java/lang/Float.html#isNaN%2Dfloat%2D[java 9])
+* ++[[painless-api-reference-Float-max-2]]static float link:{java8-javadoc}/java/lang/Float.html#max%2Dfloat%2Dfloat%2D[max](float, float)++ (link:{java9-javadoc}/java/lang/Float.html#max%2Dfloat%2Dfloat%2D[java 9])
+* ++[[painless-api-reference-Float-min-2]]static float link:{java8-javadoc}/java/lang/Float.html#min%2Dfloat%2Dfloat%2D[min](float, float)++ (link:{java9-javadoc}/java/lang/Float.html#min%2Dfloat%2Dfloat%2D[java 9])
+* ++[[painless-api-reference-Float-parseFloat-1]]static float link:{java8-javadoc}/java/lang/Float.html#parseFloat%2Djava.lang.String%2D[parseFloat](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/Float.html#parseFloat%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Float-sum-2]]static float link:{java8-javadoc}/java/lang/Float.html#sum%2Dfloat%2Dfloat%2D[sum](float, float)++ (link:{java9-javadoc}/java/lang/Float.html#sum%2Dfloat%2Dfloat%2D[java 9])
+* ++[[painless-api-reference-Float-toHexString-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Float.html#toHexString%2Dfloat%2D[toHexString](float)++ (link:{java9-javadoc}/java/lang/Float.html#toHexString%2Dfloat%2D[java 9])
+* ++[[painless-api-reference-Float-toString-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Float.html#toString%2Dfloat%2D[toString](float)++ (link:{java9-javadoc}/java/lang/Float.html#toString%2Dfloat%2D[java 9])
+* ++[[painless-api-reference-Float-valueOf-1]]static <<painless-api-reference-Float,Float>> link:{java8-javadoc}/java/lang/Float.html#valueOf%2Dfloat%2D[valueOf](float)++ (link:{java9-javadoc}/java/lang/Float.html#valueOf%2Dfloat%2D[java 9])
+* ++[[painless-api-reference-Float-compareTo-1]]int link:{java8-javadoc}/java/lang/Float.html#compareTo%2Djava.lang.Float%2D[compareTo](<<painless-api-reference-Float,Float>>)++ (link:{java9-javadoc}/java/lang/Float.html#compareTo%2Djava.lang.Float%2D[java 9])
+* ++[[painless-api-reference-Float-isInfinite-0]]boolean link:{java8-javadoc}/java/lang/Float.html#isInfinite%2D%2D[isInfinite]()++ (link:{java9-javadoc}/java/lang/Float.html#isInfinite%2D%2D[java 9])
+* ++[[painless-api-reference-Float-isNaN-0]]boolean link:{java8-javadoc}/java/lang/Float.html#isNaN%2D%2D[isNaN]()++ (link:{java9-javadoc}/java/lang/Float.html#isNaN%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Number,Number>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Format.Field.asciidoc b/docs/painless/painless-api-reference/Format.Field.asciidoc
new file mode 100644
index 0000000000000..3ddf6ec6d2f15
--- /dev/null
+++ b/docs/painless/painless-api-reference/Format.Field.asciidoc
@@ -0,0 +1,7 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Format-Field]]++Format.Field++::
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Format.asciidoc b/docs/painless/painless-api-reference/Format.asciidoc
new file mode 100644
index 0000000000000..afa42a0598dce
--- /dev/null
+++ b/docs/painless/painless-api-reference/Format.asciidoc
@@ -0,0 +1,13 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Format]]++Format++::
+* ++[[painless-api-reference-Format-clone-0]]def link:{java8-javadoc}/java/text/Format.html#clone%2D%2D[clone]()++ (link:{java9-javadoc}/java/text/Format.html#clone%2D%2D[java 9])
+* ++[[painless-api-reference-Format-format-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/text/Format.html#format%2Djava.lang.Object%2D[format](<<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/text/Format.html#format%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Format-format-3]]<<painless-api-reference-StringBuffer,StringBuffer>> link:{java8-javadoc}/java/text/Format.html#format%2Djava.lang.Object%2Djava.lang.StringBuffer%2Djava.text.FieldPosition%2D[format](<<painless-api-reference-Object,Object>>, <<painless-api-reference-StringBuffer,StringBuffer>>, <<painless-api-reference-FieldPosition,FieldPosition>>)++ (link:{java9-javadoc}/java/text/Format.html#format%2Djava.lang.Object%2Djava.lang.StringBuffer%2Djava.text.FieldPosition%2D[java 9])
+* ++[[painless-api-reference-Format-formatToCharacterIterator-1]]<<painless-api-reference-AttributedCharacterIterator,AttributedCharacterIterator>> link:{java8-javadoc}/java/text/Format.html#formatToCharacterIterator%2Djava.lang.Object%2D[formatToCharacterIterator](<<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/text/Format.html#formatToCharacterIterator%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Format-parseObject-1]]<<painless-api-reference-Object,Object>> link:{java8-javadoc}/java/text/Format.html#parseObject%2Djava.lang.String%2D[parseObject](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/Format.html#parseObject%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Format-parseObject-2]]<<painless-api-reference-Object,Object>> link:{java8-javadoc}/java/text/Format.html#parseObject%2Djava.lang.String%2Djava.text.ParsePosition%2D[parseObject](<<painless-api-reference-String,String>>, <<painless-api-reference-ParsePosition,ParsePosition>>)++ (link:{java9-javadoc}/java/text/Format.html#parseObject%2Djava.lang.String%2Djava.text.ParsePosition%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/FormatFlagsConversionMismatchException.asciidoc b/docs/painless/painless-api-reference/FormatFlagsConversionMismatchException.asciidoc
new file mode 100644
index 0000000000000..b72d827b196e8
--- /dev/null
+++ b/docs/painless/painless-api-reference/FormatFlagsConversionMismatchException.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-FormatFlagsConversionMismatchException]]++FormatFlagsConversionMismatchException++::
+* ++[[painless-api-reference-FormatFlagsConversionMismatchException-FormatFlagsConversionMismatchException-2]]link:{java8-javadoc}/java/util/FormatFlagsConversionMismatchException.html#FormatFlagsConversionMismatchException%2Djava.lang.String%2Dchar%2D[FormatFlagsConversionMismatchException](<<painless-api-reference-String,String>>, char)++ (link:{java9-javadoc}/java/util/FormatFlagsConversionMismatchException.html#FormatFlagsConversionMismatchException%2Djava.lang.String%2Dchar%2D[java 9])
+* ++[[painless-api-reference-FormatFlagsConversionMismatchException-getConversion-0]]char link:{java8-javadoc}/java/util/FormatFlagsConversionMismatchException.html#getConversion%2D%2D[getConversion]()++ (link:{java9-javadoc}/java/util/FormatFlagsConversionMismatchException.html#getConversion%2D%2D[java 9])
+* ++[[painless-api-reference-FormatFlagsConversionMismatchException-getFlags-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/FormatFlagsConversionMismatchException.html#getFlags%2D%2D[getFlags]()++ (link:{java9-javadoc}/java/util/FormatFlagsConversionMismatchException.html#getFlags%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/FormatStyle.asciidoc b/docs/painless/painless-api-reference/FormatStyle.asciidoc
new file mode 100644
index 0000000000000..4db787c051ae6
--- /dev/null
+++ b/docs/painless/painless-api-reference/FormatStyle.asciidoc
@@ -0,0 +1,13 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-FormatStyle]]++FormatStyle++::
+** [[painless-api-reference-FormatStyle-FULL]]static <<painless-api-reference-FormatStyle,FormatStyle>> link:{java8-javadoc}/java/time/format/FormatStyle.html#FULL[FULL] (link:{java9-javadoc}/java/time/format/FormatStyle.html#FULL[java 9])
+** [[painless-api-reference-FormatStyle-LONG]]static <<painless-api-reference-FormatStyle,FormatStyle>> link:{java8-javadoc}/java/time/format/FormatStyle.html#LONG[LONG] (link:{java9-javadoc}/java/time/format/FormatStyle.html#LONG[java 9])
+** [[painless-api-reference-FormatStyle-MEDIUM]]static <<painless-api-reference-FormatStyle,FormatStyle>> link:{java8-javadoc}/java/time/format/FormatStyle.html#MEDIUM[MEDIUM] (link:{java9-javadoc}/java/time/format/FormatStyle.html#MEDIUM[java 9])
+** [[painless-api-reference-FormatStyle-SHORT]]static <<painless-api-reference-FormatStyle,FormatStyle>> link:{java8-javadoc}/java/time/format/FormatStyle.html#SHORT[SHORT] (link:{java9-javadoc}/java/time/format/FormatStyle.html#SHORT[java 9])
+* ++[[painless-api-reference-FormatStyle-valueOf-1]]static <<painless-api-reference-FormatStyle,FormatStyle>> link:{java8-javadoc}/java/time/format/FormatStyle.html#valueOf%2Djava.lang.String%2D[valueOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/format/FormatStyle.html#valueOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-FormatStyle-values-0]]static <<painless-api-reference-FormatStyle,FormatStyle>>[] link:{java8-javadoc}/java/time/format/FormatStyle.html#values%2D%2D[values]()++ (link:{java9-javadoc}/java/time/format/FormatStyle.html#values%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Enum,Enum>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Formattable.asciidoc b/docs/painless/painless-api-reference/Formattable.asciidoc
new file mode 100644
index 0000000000000..ab70d1319e939
--- /dev/null
+++ b/docs/painless/painless-api-reference/Formattable.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Formattable]]++Formattable++::
+* ++[[painless-api-reference-Formattable-formatTo-4]]void link:{java8-javadoc}/java/util/Formattable.html#formatTo%2Djava.util.Formatter%2Dint%2Dint%2Dint%2D[formatTo](<<painless-api-reference-Formatter,Formatter>>, int, int, int)++ (link:{java9-javadoc}/java/util/Formattable.html#formatTo%2Djava.util.Formatter%2Dint%2Dint%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/FormattableFlags.asciidoc b/docs/painless/painless-api-reference/FormattableFlags.asciidoc
new file mode 100644
index 0000000000000..dec639fe70b3e
--- /dev/null
+++ b/docs/painless/painless-api-reference/FormattableFlags.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-FormattableFlags]]++FormattableFlags++::
+** [[painless-api-reference-FormattableFlags-ALTERNATE]]static int link:{java8-javadoc}/java/util/FormattableFlags.html#ALTERNATE[ALTERNATE] (link:{java9-javadoc}/java/util/FormattableFlags.html#ALTERNATE[java 9])
+** [[painless-api-reference-FormattableFlags-LEFT_JUSTIFY]]static int link:{java8-javadoc}/java/util/FormattableFlags.html#LEFT_JUSTIFY[LEFT_JUSTIFY] (link:{java9-javadoc}/java/util/FormattableFlags.html#LEFT_JUSTIFY[java 9])
+** [[painless-api-reference-FormattableFlags-UPPERCASE]]static int link:{java8-javadoc}/java/util/FormattableFlags.html#UPPERCASE[UPPERCASE] (link:{java9-javadoc}/java/util/FormattableFlags.html#UPPERCASE[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Formatter.BigDecimalLayoutForm.asciidoc b/docs/painless/painless-api-reference/Formatter.BigDecimalLayoutForm.asciidoc
new file mode 100644
index 0000000000000..4fb9ce48062e6
--- /dev/null
+++ b/docs/painless/painless-api-reference/Formatter.BigDecimalLayoutForm.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Formatter-BigDecimalLayoutForm]]++Formatter.BigDecimalLayoutForm++::
+** [[painless-api-reference-Formatter-BigDecimalLayoutForm-DECIMAL_FLOAT]]static <<painless-api-reference-Formatter-BigDecimalLayoutForm,Formatter.BigDecimalLayoutForm>> link:{java8-javadoc}/java/util/Formatter.BigDecimalLayoutForm.html#DECIMAL_FLOAT[DECIMAL_FLOAT] (link:{java9-javadoc}/java/util/Formatter.BigDecimalLayoutForm.html#DECIMAL_FLOAT[java 9])
+** [[painless-api-reference-Formatter-BigDecimalLayoutForm-SCIENTIFIC]]static <<painless-api-reference-Formatter-BigDecimalLayoutForm,Formatter.BigDecimalLayoutForm>> link:{java8-javadoc}/java/util/Formatter.BigDecimalLayoutForm.html#SCIENTIFIC[SCIENTIFIC] (link:{java9-javadoc}/java/util/Formatter.BigDecimalLayoutForm.html#SCIENTIFIC[java 9])
+* Inherits methods from ++<<painless-api-reference-Enum,Enum>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Formatter.asciidoc b/docs/painless/painless-api-reference/Formatter.asciidoc
new file mode 100644
index 0000000000000..4cad434f1308f
--- /dev/null
+++ b/docs/painless/painless-api-reference/Formatter.asciidoc
@@ -0,0 +1,14 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Formatter]]++Formatter++::
+* ++[[painless-api-reference-Formatter-Formatter-0]]link:{java8-javadoc}/java/util/Formatter.html#Formatter%2D%2D[Formatter]()++ (link:{java9-javadoc}/java/util/Formatter.html#Formatter%2D%2D[java 9])
+* ++[[painless-api-reference-Formatter-Formatter-1]]link:{java8-javadoc}/java/util/Formatter.html#Formatter%2Djava.lang.Appendable%2D[Formatter](<<painless-api-reference-Appendable,Appendable>>)++ (link:{java9-javadoc}/java/util/Formatter.html#Formatter%2Djava.lang.Appendable%2D[java 9])
+* ++[[painless-api-reference-Formatter-Formatter-2]]link:{java8-javadoc}/java/util/Formatter.html#Formatter%2Djava.lang.Appendable%2Djava.util.Locale%2D[Formatter](<<painless-api-reference-Appendable,Appendable>>, <<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/util/Formatter.html#Formatter%2Djava.lang.Appendable%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-Formatter-format-2]]<<painless-api-reference-Formatter,Formatter>> link:{java8-javadoc}/java/util/Formatter.html#format%2Djava.lang.String%2Djava.lang.Object:A%2D[format](<<painless-api-reference-String,String>>, def[])++ (link:{java9-javadoc}/java/util/Formatter.html#format%2Djava.lang.String%2Djava.lang.Object:A%2D[java 9])
+* ++[[painless-api-reference-Formatter-format-3]]<<painless-api-reference-Formatter,Formatter>> link:{java8-javadoc}/java/util/Formatter.html#format%2Djava.util.Locale%2Djava.lang.String%2Djava.lang.Object:A%2D[format](<<painless-api-reference-Locale,Locale>>, <<painless-api-reference-String,String>>, def[])++ (link:{java9-javadoc}/java/util/Formatter.html#format%2Djava.util.Locale%2Djava.lang.String%2Djava.lang.Object:A%2D[java 9])
+* ++[[painless-api-reference-Formatter-locale-0]]<<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Formatter.html#locale%2D%2D[locale]()++ (link:{java9-javadoc}/java/util/Formatter.html#locale%2D%2D[java 9])
+* ++[[painless-api-reference-Formatter-out-0]]<<painless-api-reference-Appendable,Appendable>> link:{java8-javadoc}/java/util/Formatter.html#out%2D%2D[out]()++ (link:{java9-javadoc}/java/util/Formatter.html#out%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/FormatterClosedException.asciidoc b/docs/painless/painless-api-reference/FormatterClosedException.asciidoc
new file mode 100644
index 0000000000000..00c4f5971d335
--- /dev/null
+++ b/docs/painless/painless-api-reference/FormatterClosedException.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-FormatterClosedException]]++FormatterClosedException++::
+* ++[[painless-api-reference-FormatterClosedException-FormatterClosedException-0]]link:{java8-javadoc}/java/util/FormatterClosedException.html#FormatterClosedException%2D%2D[FormatterClosedException]()++ (link:{java9-javadoc}/java/util/FormatterClosedException.html#FormatterClosedException%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Function.asciidoc b/docs/painless/painless-api-reference/Function.asciidoc
new file mode 100644
index 0000000000000..61fe3b6b5da70
--- /dev/null
+++ b/docs/painless/painless-api-reference/Function.asciidoc
@@ -0,0 +1,11 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Function]]++Function++::
+* ++[[painless-api-reference-Function-identity-0]]static <<painless-api-reference-Function,Function>> link:{java8-javadoc}/java/util/function/Function.html#identity%2D%2D[identity]()++ (link:{java9-javadoc}/java/util/function/Function.html#identity%2D%2D[java 9])
+* ++[[painless-api-reference-Function-andThen-1]]<<painless-api-reference-Function,Function>> link:{java8-javadoc}/java/util/function/Function.html#andThen%2Djava.util.function.Function%2D[andThen](<<painless-api-reference-Function,Function>>)++ (link:{java9-javadoc}/java/util/function/Function.html#andThen%2Djava.util.function.Function%2D[java 9])
+* ++[[painless-api-reference-Function-apply-1]]def link:{java8-javadoc}/java/util/function/Function.html#apply%2Djava.lang.Object%2D[apply](def)++ (link:{java9-javadoc}/java/util/function/Function.html#apply%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Function-compose-1]]<<painless-api-reference-Function,Function>> link:{java8-javadoc}/java/util/function/Function.html#compose%2Djava.util.function.Function%2D[compose](<<painless-api-reference-Function,Function>>)++ (link:{java9-javadoc}/java/util/function/Function.html#compose%2Djava.util.function.Function%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/GregorianCalendar.asciidoc b/docs/painless/painless-api-reference/GregorianCalendar.asciidoc
new file mode 100644
index 0000000000000..8f7b9ccecc7e6
--- /dev/null
+++ b/docs/painless/painless-api-reference/GregorianCalendar.asciidoc
@@ -0,0 +1,20 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-GregorianCalendar]]++GregorianCalendar++::
+** [[painless-api-reference-GregorianCalendar-AD]]static int link:{java8-javadoc}/java/util/GregorianCalendar.html#AD[AD] (link:{java9-javadoc}/java/util/GregorianCalendar.html#AD[java 9])
+** [[painless-api-reference-GregorianCalendar-BC]]static int link:{java8-javadoc}/java/util/GregorianCalendar.html#BC[BC] (link:{java9-javadoc}/java/util/GregorianCalendar.html#BC[java 9])
+* ++[[painless-api-reference-GregorianCalendar-from-1]]static <<painless-api-reference-GregorianCalendar,GregorianCalendar>> link:{java8-javadoc}/java/util/GregorianCalendar.html#from%2Djava.time.ZonedDateTime%2D[from](<<painless-api-reference-ZonedDateTime,ZonedDateTime>>)++ (link:{java9-javadoc}/java/util/GregorianCalendar.html#from%2Djava.time.ZonedDateTime%2D[java 9])
+* ++[[painless-api-reference-GregorianCalendar-GregorianCalendar-0]]link:{java8-javadoc}/java/util/GregorianCalendar.html#GregorianCalendar%2D%2D[GregorianCalendar]()++ (link:{java9-javadoc}/java/util/GregorianCalendar.html#GregorianCalendar%2D%2D[java 9])
+* ++[[painless-api-reference-GregorianCalendar-GregorianCalendar-1]]link:{java8-javadoc}/java/util/GregorianCalendar.html#GregorianCalendar%2Djava.util.TimeZone%2D[GregorianCalendar](<<painless-api-reference-TimeZone,TimeZone>>)++ (link:{java9-javadoc}/java/util/GregorianCalendar.html#GregorianCalendar%2Djava.util.TimeZone%2D[java 9])
+* ++[[painless-api-reference-GregorianCalendar-GregorianCalendar-2]]link:{java8-javadoc}/java/util/GregorianCalendar.html#GregorianCalendar%2Djava.util.TimeZone%2Djava.util.Locale%2D[GregorianCalendar](<<painless-api-reference-TimeZone,TimeZone>>, <<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/util/GregorianCalendar.html#GregorianCalendar%2Djava.util.TimeZone%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-GregorianCalendar-GregorianCalendar-3]]link:{java8-javadoc}/java/util/GregorianCalendar.html#GregorianCalendar%2Dint%2Dint%2Dint%2D[GregorianCalendar](int, int, int)++ (link:{java9-javadoc}/java/util/GregorianCalendar.html#GregorianCalendar%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-GregorianCalendar-GregorianCalendar-5]]link:{java8-javadoc}/java/util/GregorianCalendar.html#GregorianCalendar%2Dint%2Dint%2Dint%2Dint%2Dint%2D[GregorianCalendar](int, int, int, int, int)++ (link:{java9-javadoc}/java/util/GregorianCalendar.html#GregorianCalendar%2Dint%2Dint%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-GregorianCalendar-GregorianCalendar-6]]link:{java8-javadoc}/java/util/GregorianCalendar.html#GregorianCalendar%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2D[GregorianCalendar](int, int, int, int, int, int)++ (link:{java9-javadoc}/java/util/GregorianCalendar.html#GregorianCalendar%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-GregorianCalendar-getGregorianChange-0]]<<painless-api-reference-Date,Date>> link:{java8-javadoc}/java/util/GregorianCalendar.html#getGregorianChange%2D%2D[getGregorianChange]()++ (link:{java9-javadoc}/java/util/GregorianCalendar.html#getGregorianChange%2D%2D[java 9])
+* ++[[painless-api-reference-GregorianCalendar-isLeapYear-1]]boolean link:{java8-javadoc}/java/util/GregorianCalendar.html#isLeapYear%2Dint%2D[isLeapYear](int)++ (link:{java9-javadoc}/java/util/GregorianCalendar.html#isLeapYear%2Dint%2D[java 9])
+* ++[[painless-api-reference-GregorianCalendar-setGregorianChange-1]]void link:{java8-javadoc}/java/util/GregorianCalendar.html#setGregorianChange%2Djava.util.Date%2D[setGregorianChange](<<painless-api-reference-Date,Date>>)++ (link:{java9-javadoc}/java/util/GregorianCalendar.html#setGregorianChange%2Djava.util.Date%2D[java 9])
+* ++[[painless-api-reference-GregorianCalendar-toZonedDateTime-0]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/util/GregorianCalendar.html#toZonedDateTime%2D%2D[toZonedDateTime]()++ (link:{java9-javadoc}/java/util/GregorianCalendar.html#toZonedDateTime%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Calendar,Calendar>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/HashMap.asciidoc b/docs/painless/painless-api-reference/HashMap.asciidoc
new file mode 100644
index 0000000000000..f67e75d702f0c
--- /dev/null
+++ b/docs/painless/painless-api-reference/HashMap.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-HashMap]]++HashMap++::
+* ++[[painless-api-reference-HashMap-HashMap-0]]link:{java8-javadoc}/java/util/HashMap.html#HashMap%2D%2D[HashMap]()++ (link:{java9-javadoc}/java/util/HashMap.html#HashMap%2D%2D[java 9])
+* ++[[painless-api-reference-HashMap-HashMap-1]]link:{java8-javadoc}/java/util/HashMap.html#HashMap%2Djava.util.Map%2D[HashMap](<<painless-api-reference-Map,Map>>)++ (link:{java9-javadoc}/java/util/HashMap.html#HashMap%2Djava.util.Map%2D[java 9])
+* ++[[painless-api-reference-HashMap-clone-0]]def link:{java8-javadoc}/java/util/HashMap.html#clone%2D%2D[clone]()++ (link:{java9-javadoc}/java/util/HashMap.html#clone%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Map,Map>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/HashSet.asciidoc b/docs/painless/painless-api-reference/HashSet.asciidoc
new file mode 100644
index 0000000000000..af4171def2262
--- /dev/null
+++ b/docs/painless/painless-api-reference/HashSet.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-HashSet]]++HashSet++::
+* ++[[painless-api-reference-HashSet-HashSet-0]]link:{java8-javadoc}/java/util/HashSet.html#HashSet%2D%2D[HashSet]()++ (link:{java9-javadoc}/java/util/HashSet.html#HashSet%2D%2D[java 9])
+* ++[[painless-api-reference-HashSet-HashSet-1]]link:{java8-javadoc}/java/util/HashSet.html#HashSet%2Djava.util.Collection%2D[HashSet](<<painless-api-reference-Collection,Collection>>)++ (link:{java9-javadoc}/java/util/HashSet.html#HashSet%2Djava.util.Collection%2D[java 9])
+* ++[[painless-api-reference-HashSet-clone-0]]def link:{java8-javadoc}/java/util/HashSet.html#clone%2D%2D[clone]()++ (link:{java9-javadoc}/java/util/HashSet.html#clone%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-Set,Set>>++
diff --git a/docs/painless/painless-api-reference/Hashtable.asciidoc b/docs/painless/painless-api-reference/Hashtable.asciidoc
new file mode 100644
index 0000000000000..34702e8466065
--- /dev/null
+++ b/docs/painless/painless-api-reference/Hashtable.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Hashtable]]++Hashtable++::
+* ++[[painless-api-reference-Hashtable-Hashtable-0]]link:{java8-javadoc}/java/util/Hashtable.html#Hashtable%2D%2D[Hashtable]()++ (link:{java9-javadoc}/java/util/Hashtable.html#Hashtable%2D%2D[java 9])
+* ++[[painless-api-reference-Hashtable-Hashtable-1]]link:{java8-javadoc}/java/util/Hashtable.html#Hashtable%2Djava.util.Map%2D[Hashtable](<<painless-api-reference-Map,Map>>)++ (link:{java9-javadoc}/java/util/Hashtable.html#Hashtable%2Djava.util.Map%2D[java 9])
+* ++[[painless-api-reference-Hashtable-clone-0]]def link:{java8-javadoc}/java/util/Hashtable.html#clone%2D%2D[clone]()++ (link:{java9-javadoc}/java/util/Hashtable.html#clone%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Dictionary,Dictionary>>++, ++<<painless-api-reference-Map,Map>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/HijrahChronology.asciidoc b/docs/painless/painless-api-reference/HijrahChronology.asciidoc
new file mode 100644
index 0000000000000..b7577fbdfd720
--- /dev/null
+++ b/docs/painless/painless-api-reference/HijrahChronology.asciidoc
@@ -0,0 +1,16 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-HijrahChronology]]++HijrahChronology++::
+** [[painless-api-reference-HijrahChronology-INSTANCE]]static <<painless-api-reference-HijrahChronology,HijrahChronology>> link:{java8-javadoc}/java/time/chrono/HijrahChronology.html#INSTANCE[INSTANCE] (link:{java9-javadoc}/java/time/chrono/HijrahChronology.html#INSTANCE[java 9])
+* ++[[painless-api-reference-HijrahChronology-date-1]]<<painless-api-reference-HijrahDate,HijrahDate>> link:{java8-javadoc}/java/time/chrono/HijrahChronology.html#date%2Djava.time.temporal.TemporalAccessor%2D[date](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/chrono/HijrahChronology.html#date%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-HijrahChronology-date-3]]<<painless-api-reference-HijrahDate,HijrahDate>> link:{java8-javadoc}/java/time/chrono/HijrahChronology.html#date%2Dint%2Dint%2Dint%2D[date](int, int, int)++ (link:{java9-javadoc}/java/time/chrono/HijrahChronology.html#date%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-HijrahChronology-date-4]]<<painless-api-reference-HijrahDate,HijrahDate>> link:{java8-javadoc}/java/time/chrono/HijrahChronology.html#date%2Djava.time.chrono.Era%2Dint%2Dint%2Dint%2D[date](<<painless-api-reference-Era,Era>>, int, int, int)++ (link:{java9-javadoc}/java/time/chrono/HijrahChronology.html#date%2Djava.time.chrono.Era%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-HijrahChronology-dateEpochDay-1]]<<painless-api-reference-HijrahDate,HijrahDate>> link:{java8-javadoc}/java/time/chrono/HijrahChronology.html#dateEpochDay%2Dlong%2D[dateEpochDay](long)++ (link:{java9-javadoc}/java/time/chrono/HijrahChronology.html#dateEpochDay%2Dlong%2D[java 9])
+* ++[[painless-api-reference-HijrahChronology-dateYearDay-2]]<<painless-api-reference-HijrahDate,HijrahDate>> link:{java8-javadoc}/java/time/chrono/HijrahChronology.html#dateYearDay%2Dint%2Dint%2D[dateYearDay](int, int)++ (link:{java9-javadoc}/java/time/chrono/HijrahChronology.html#dateYearDay%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-HijrahChronology-dateYearDay-3]]<<painless-api-reference-HijrahDate,HijrahDate>> link:{java8-javadoc}/java/time/chrono/HijrahChronology.html#dateYearDay%2Djava.time.chrono.Era%2Dint%2Dint%2D[dateYearDay](<<painless-api-reference-Era,Era>>, int, int)++ (link:{java9-javadoc}/java/time/chrono/HijrahChronology.html#dateYearDay%2Djava.time.chrono.Era%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-HijrahChronology-eraOf-1]]<<painless-api-reference-HijrahEra,HijrahEra>> link:{java8-javadoc}/java/time/chrono/HijrahChronology.html#eraOf%2Dint%2D[eraOf](int)++ (link:{java9-javadoc}/java/time/chrono/HijrahChronology.html#eraOf%2Dint%2D[java 9])
+* ++[[painless-api-reference-HijrahChronology-resolveDate-2]]<<painless-api-reference-HijrahDate,HijrahDate>> link:{java8-javadoc}/java/time/chrono/HijrahChronology.html#resolveDate%2Djava.util.Map%2Djava.time.format.ResolverStyle%2D[resolveDate](<<painless-api-reference-Map,Map>>, <<painless-api-reference-ResolverStyle,ResolverStyle>>)++ (link:{java9-javadoc}/java/time/chrono/HijrahChronology.html#resolveDate%2Djava.util.Map%2Djava.time.format.ResolverStyle%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Chronology,Chronology>>++
diff --git a/docs/painless/painless-api-reference/HijrahDate.asciidoc b/docs/painless/painless-api-reference/HijrahDate.asciidoc
new file mode 100644
index 0000000000000..09ebf31682a13
--- /dev/null
+++ b/docs/painless/painless-api-reference/HijrahDate.asciidoc
@@ -0,0 +1,18 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-HijrahDate]]++HijrahDate++::
+* ++[[painless-api-reference-HijrahDate-from-1]]static <<painless-api-reference-HijrahDate,HijrahDate>> link:{java8-javadoc}/java/time/chrono/HijrahDate.html#from%2Djava.time.temporal.TemporalAccessor%2D[from](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/chrono/HijrahDate.html#from%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-HijrahDate-of-3]]static <<painless-api-reference-HijrahDate,HijrahDate>> link:{java8-javadoc}/java/time/chrono/HijrahDate.html#of%2Dint%2Dint%2Dint%2D[of](int, int, int)++ (link:{java9-javadoc}/java/time/chrono/HijrahDate.html#of%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-HijrahDate-getChronology-0]]<<painless-api-reference-HijrahChronology,HijrahChronology>> link:{java8-javadoc}/java/time/chrono/HijrahDate.html#getChronology%2D%2D[getChronology]()++ (link:{java9-javadoc}/java/time/chrono/HijrahDate.html#getChronology%2D%2D[java 9])
+* ++[[painless-api-reference-HijrahDate-getEra-0]]<<painless-api-reference-HijrahEra,HijrahEra>> link:{java8-javadoc}/java/time/chrono/HijrahDate.html#getEra%2D%2D[getEra]()++ (link:{java9-javadoc}/java/time/chrono/HijrahDate.html#getEra%2D%2D[java 9])
+* ++[[painless-api-reference-HijrahDate-minus-1]]<<painless-api-reference-HijrahDate,HijrahDate>> link:{java8-javadoc}/java/time/chrono/HijrahDate.html#minus%2Djava.time.temporal.TemporalAmount%2D[minus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/chrono/HijrahDate.html#minus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-HijrahDate-minus-2]]<<painless-api-reference-HijrahDate,HijrahDate>> link:{java8-javadoc}/java/time/chrono/HijrahDate.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[minus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/chrono/HijrahDate.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-HijrahDate-plus-1]]<<painless-api-reference-HijrahDate,HijrahDate>> link:{java8-javadoc}/java/time/chrono/HijrahDate.html#plus%2Djava.time.temporal.TemporalAmount%2D[plus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/chrono/HijrahDate.html#plus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-HijrahDate-plus-2]]<<painless-api-reference-HijrahDate,HijrahDate>> link:{java8-javadoc}/java/time/chrono/HijrahDate.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[plus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/chrono/HijrahDate.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-HijrahDate-with-1]]<<painless-api-reference-HijrahDate,HijrahDate>> link:{java8-javadoc}/java/time/chrono/HijrahDate.html#with%2Djava.time.temporal.TemporalAdjuster%2D[with](<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>)++ (link:{java9-javadoc}/java/time/chrono/HijrahDate.html#with%2Djava.time.temporal.TemporalAdjuster%2D[java 9])
+* ++[[painless-api-reference-HijrahDate-with-2]]<<painless-api-reference-HijrahDate,HijrahDate>> link:{java8-javadoc}/java/time/chrono/HijrahDate.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[with](<<painless-api-reference-TemporalField,TemporalField>>, long)++ (link:{java9-javadoc}/java/time/chrono/HijrahDate.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[java 9])
+* ++[[painless-api-reference-HijrahDate-withVariant-1]]<<painless-api-reference-HijrahDate,HijrahDate>> link:{java8-javadoc}/java/time/chrono/HijrahDate.html#withVariant%2Djava.time.chrono.HijrahChronology%2D[withVariant](<<painless-api-reference-HijrahChronology,HijrahChronology>>)++ (link:{java9-javadoc}/java/time/chrono/HijrahDate.html#withVariant%2Djava.time.chrono.HijrahChronology%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>>++, ++<<painless-api-reference-Temporal,Temporal>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++, ++<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>++
diff --git a/docs/painless/painless-api-reference/HijrahEra.asciidoc b/docs/painless/painless-api-reference/HijrahEra.asciidoc
new file mode 100644
index 0000000000000..9bfe561d2fa58
--- /dev/null
+++ b/docs/painless/painless-api-reference/HijrahEra.asciidoc
@@ -0,0 +1,12 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-HijrahEra]]++HijrahEra++::
+** [[painless-api-reference-HijrahEra-AH]]static <<painless-api-reference-HijrahEra,HijrahEra>> link:{java8-javadoc}/java/time/chrono/HijrahEra.html#AH[AH] (link:{java9-javadoc}/java/time/chrono/HijrahEra.html#AH[java 9])
+* ++[[painless-api-reference-HijrahEra-of-1]]static <<painless-api-reference-HijrahEra,HijrahEra>> link:{java8-javadoc}/java/time/chrono/HijrahEra.html#of%2Dint%2D[of](int)++ (link:{java9-javadoc}/java/time/chrono/HijrahEra.html#of%2Dint%2D[java 9])
+* ++[[painless-api-reference-HijrahEra-valueOf-1]]static <<painless-api-reference-HijrahEra,HijrahEra>> link:{java8-javadoc}/java/time/chrono/HijrahEra.html#valueOf%2Djava.lang.String%2D[valueOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/chrono/HijrahEra.html#valueOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-HijrahEra-values-0]]static <<painless-api-reference-HijrahEra,HijrahEra>>[] link:{java8-javadoc}/java/time/chrono/HijrahEra.html#values%2D%2D[values]()++ (link:{java9-javadoc}/java/time/chrono/HijrahEra.html#values%2D%2D[java 9])
+* ++[[painless-api-reference-HijrahEra-getValue-0]]int link:{java8-javadoc}/java/time/chrono/HijrahEra.html#getValue%2D%2D[getValue]()++ (link:{java9-javadoc}/java/time/chrono/HijrahEra.html#getValue%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Enum,Enum>>++, ++<<painless-api-reference-Era,Era>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++, ++<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>++
diff --git a/docs/painless/painless-api-reference/IdentityHashMap.asciidoc b/docs/painless/painless-api-reference/IdentityHashMap.asciidoc
new file mode 100644
index 0000000000000..050d0eaeb1c8e
--- /dev/null
+++ b/docs/painless/painless-api-reference/IdentityHashMap.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IdentityHashMap]]++IdentityHashMap++::
+* ++[[painless-api-reference-IdentityHashMap-IdentityHashMap-0]]link:{java8-javadoc}/java/util/IdentityHashMap.html#IdentityHashMap%2D%2D[IdentityHashMap]()++ (link:{java9-javadoc}/java/util/IdentityHashMap.html#IdentityHashMap%2D%2D[java 9])
+* ++[[painless-api-reference-IdentityHashMap-IdentityHashMap-1]]link:{java8-javadoc}/java/util/IdentityHashMap.html#IdentityHashMap%2Djava.util.Map%2D[IdentityHashMap](<<painless-api-reference-Map,Map>>)++ (link:{java9-javadoc}/java/util/IdentityHashMap.html#IdentityHashMap%2Djava.util.Map%2D[java 9])
+* ++[[painless-api-reference-IdentityHashMap-clone-0]]def link:{java8-javadoc}/java/util/IdentityHashMap.html#clone%2D%2D[clone]()++ (link:{java9-javadoc}/java/util/IdentityHashMap.html#clone%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Map,Map>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/IllegalAccessException.asciidoc b/docs/painless/painless-api-reference/IllegalAccessException.asciidoc
new file mode 100644
index 0000000000000..30c8ba4536d02
--- /dev/null
+++ b/docs/painless/painless-api-reference/IllegalAccessException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IllegalAccessException]]++IllegalAccessException++::
+* ++[[painless-api-reference-IllegalAccessException-IllegalAccessException-0]]link:{java8-javadoc}/java/lang/IllegalAccessException.html#IllegalAccessException%2D%2D[IllegalAccessException]()++ (link:{java9-javadoc}/java/lang/IllegalAccessException.html#IllegalAccessException%2D%2D[java 9])
+* ++[[painless-api-reference-IllegalAccessException-IllegalAccessException-1]]link:{java8-javadoc}/java/lang/IllegalAccessException.html#IllegalAccessException%2Djava.lang.String%2D[IllegalAccessException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/IllegalAccessException.html#IllegalAccessException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/IllegalArgumentException.asciidoc b/docs/painless/painless-api-reference/IllegalArgumentException.asciidoc
new file mode 100644
index 0000000000000..538fa46a656e2
--- /dev/null
+++ b/docs/painless/painless-api-reference/IllegalArgumentException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IllegalArgumentException]]++IllegalArgumentException++::
+* ++[[painless-api-reference-IllegalArgumentException-IllegalArgumentException-0]]link:{java8-javadoc}/java/lang/IllegalArgumentException.html#IllegalArgumentException%2D%2D[IllegalArgumentException]()++ (link:{java9-javadoc}/java/lang/IllegalArgumentException.html#IllegalArgumentException%2D%2D[java 9])
+* ++[[painless-api-reference-IllegalArgumentException-IllegalArgumentException-1]]link:{java8-javadoc}/java/lang/IllegalArgumentException.html#IllegalArgumentException%2Djava.lang.String%2D[IllegalArgumentException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/IllegalArgumentException.html#IllegalArgumentException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/IllegalFormatCodePointException.asciidoc b/docs/painless/painless-api-reference/IllegalFormatCodePointException.asciidoc
new file mode 100644
index 0000000000000..2a533384198d1
--- /dev/null
+++ b/docs/painless/painless-api-reference/IllegalFormatCodePointException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IllegalFormatCodePointException]]++IllegalFormatCodePointException++::
+* ++[[painless-api-reference-IllegalFormatCodePointException-IllegalFormatCodePointException-1]]link:{java8-javadoc}/java/util/IllegalFormatCodePointException.html#IllegalFormatCodePointException%2Dint%2D[IllegalFormatCodePointException](int)++ (link:{java9-javadoc}/java/util/IllegalFormatCodePointException.html#IllegalFormatCodePointException%2Dint%2D[java 9])
+* ++[[painless-api-reference-IllegalFormatCodePointException-getCodePoint-0]]int link:{java8-javadoc}/java/util/IllegalFormatCodePointException.html#getCodePoint%2D%2D[getCodePoint]()++ (link:{java9-javadoc}/java/util/IllegalFormatCodePointException.html#getCodePoint%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/IllegalFormatConversionException.asciidoc b/docs/painless/painless-api-reference/IllegalFormatConversionException.asciidoc
new file mode 100644
index 0000000000000..2a36b5367a339
--- /dev/null
+++ b/docs/painless/painless-api-reference/IllegalFormatConversionException.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IllegalFormatConversionException]]++IllegalFormatConversionException++::
+* ++[[painless-api-reference-IllegalFormatConversionException-getConversion-0]]char link:{java8-javadoc}/java/util/IllegalFormatConversionException.html#getConversion%2D%2D[getConversion]()++ (link:{java9-javadoc}/java/util/IllegalFormatConversionException.html#getConversion%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/IllegalFormatException.asciidoc b/docs/painless/painless-api-reference/IllegalFormatException.asciidoc
new file mode 100644
index 0000000000000..98a58ceddd265
--- /dev/null
+++ b/docs/painless/painless-api-reference/IllegalFormatException.asciidoc
@@ -0,0 +1,7 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IllegalFormatException]]++IllegalFormatException++::
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/IllegalFormatFlagsException.asciidoc b/docs/painless/painless-api-reference/IllegalFormatFlagsException.asciidoc
new file mode 100644
index 0000000000000..9ed7810f0e60f
--- /dev/null
+++ b/docs/painless/painless-api-reference/IllegalFormatFlagsException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IllegalFormatFlagsException]]++IllegalFormatFlagsException++::
+* ++[[painless-api-reference-IllegalFormatFlagsException-IllegalFormatFlagsException-1]]link:{java8-javadoc}/java/util/IllegalFormatFlagsException.html#IllegalFormatFlagsException%2Djava.lang.String%2D[IllegalFormatFlagsException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/IllegalFormatFlagsException.html#IllegalFormatFlagsException%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-IllegalFormatFlagsException-getFlags-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/IllegalFormatFlagsException.html#getFlags%2D%2D[getFlags]()++ (link:{java9-javadoc}/java/util/IllegalFormatFlagsException.html#getFlags%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/IllegalFormatPrecisionException.asciidoc b/docs/painless/painless-api-reference/IllegalFormatPrecisionException.asciidoc
new file mode 100644
index 0000000000000..318b80dcfaa6c
--- /dev/null
+++ b/docs/painless/painless-api-reference/IllegalFormatPrecisionException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IllegalFormatPrecisionException]]++IllegalFormatPrecisionException++::
+* ++[[painless-api-reference-IllegalFormatPrecisionException-IllegalFormatPrecisionException-1]]link:{java8-javadoc}/java/util/IllegalFormatPrecisionException.html#IllegalFormatPrecisionException%2Dint%2D[IllegalFormatPrecisionException](int)++ (link:{java9-javadoc}/java/util/IllegalFormatPrecisionException.html#IllegalFormatPrecisionException%2Dint%2D[java 9])
+* ++[[painless-api-reference-IllegalFormatPrecisionException-getPrecision-0]]int link:{java8-javadoc}/java/util/IllegalFormatPrecisionException.html#getPrecision%2D%2D[getPrecision]()++ (link:{java9-javadoc}/java/util/IllegalFormatPrecisionException.html#getPrecision%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/IllegalFormatWidthException.asciidoc b/docs/painless/painless-api-reference/IllegalFormatWidthException.asciidoc
new file mode 100644
index 0000000000000..96f008b4a5334
--- /dev/null
+++ b/docs/painless/painless-api-reference/IllegalFormatWidthException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IllegalFormatWidthException]]++IllegalFormatWidthException++::
+* ++[[painless-api-reference-IllegalFormatWidthException-IllegalFormatWidthException-1]]link:{java8-javadoc}/java/util/IllegalFormatWidthException.html#IllegalFormatWidthException%2Dint%2D[IllegalFormatWidthException](int)++ (link:{java9-javadoc}/java/util/IllegalFormatWidthException.html#IllegalFormatWidthException%2Dint%2D[java 9])
+* ++[[painless-api-reference-IllegalFormatWidthException-getWidth-0]]int link:{java8-javadoc}/java/util/IllegalFormatWidthException.html#getWidth%2D%2D[getWidth]()++ (link:{java9-javadoc}/java/util/IllegalFormatWidthException.html#getWidth%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/IllegalMonitorStateException.asciidoc b/docs/painless/painless-api-reference/IllegalMonitorStateException.asciidoc
new file mode 100644
index 0000000000000..057c58f948b73
--- /dev/null
+++ b/docs/painless/painless-api-reference/IllegalMonitorStateException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IllegalMonitorStateException]]++IllegalMonitorStateException++::
+* ++[[painless-api-reference-IllegalMonitorStateException-IllegalMonitorStateException-0]]link:{java8-javadoc}/java/lang/IllegalMonitorStateException.html#IllegalMonitorStateException%2D%2D[IllegalMonitorStateException]()++ (link:{java9-javadoc}/java/lang/IllegalMonitorStateException.html#IllegalMonitorStateException%2D%2D[java 9])
+* ++[[painless-api-reference-IllegalMonitorStateException-IllegalMonitorStateException-1]]link:{java8-javadoc}/java/lang/IllegalMonitorStateException.html#IllegalMonitorStateException%2Djava.lang.String%2D[IllegalMonitorStateException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/IllegalMonitorStateException.html#IllegalMonitorStateException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/IllegalStateException.asciidoc b/docs/painless/painless-api-reference/IllegalStateException.asciidoc
new file mode 100644
index 0000000000000..fa8ddf72fa5bd
--- /dev/null
+++ b/docs/painless/painless-api-reference/IllegalStateException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IllegalStateException]]++IllegalStateException++::
+* ++[[painless-api-reference-IllegalStateException-IllegalStateException-0]]link:{java8-javadoc}/java/lang/IllegalStateException.html#IllegalStateException%2D%2D[IllegalStateException]()++ (link:{java9-javadoc}/java/lang/IllegalStateException.html#IllegalStateException%2D%2D[java 9])
+* ++[[painless-api-reference-IllegalStateException-IllegalStateException-1]]link:{java8-javadoc}/java/lang/IllegalStateException.html#IllegalStateException%2Djava.lang.String%2D[IllegalStateException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/IllegalStateException.html#IllegalStateException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/IllegalThreadStateException.asciidoc b/docs/painless/painless-api-reference/IllegalThreadStateException.asciidoc
new file mode 100644
index 0000000000000..5fc8293fe0933
--- /dev/null
+++ b/docs/painless/painless-api-reference/IllegalThreadStateException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IllegalThreadStateException]]++IllegalThreadStateException++::
+* ++[[painless-api-reference-IllegalThreadStateException-IllegalThreadStateException-0]]link:{java8-javadoc}/java/lang/IllegalThreadStateException.html#IllegalThreadStateException%2D%2D[IllegalThreadStateException]()++ (link:{java9-javadoc}/java/lang/IllegalThreadStateException.html#IllegalThreadStateException%2D%2D[java 9])
+* ++[[painless-api-reference-IllegalThreadStateException-IllegalThreadStateException-1]]link:{java8-javadoc}/java/lang/IllegalThreadStateException.html#IllegalThreadStateException%2Djava.lang.String%2D[IllegalThreadStateException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/IllegalThreadStateException.html#IllegalThreadStateException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/IllformedLocaleException.asciidoc b/docs/painless/painless-api-reference/IllformedLocaleException.asciidoc
new file mode 100644
index 0000000000000..59a814d9520dd
--- /dev/null
+++ b/docs/painless/painless-api-reference/IllformedLocaleException.asciidoc
@@ -0,0 +1,11 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IllformedLocaleException]]++IllformedLocaleException++::
+* ++[[painless-api-reference-IllformedLocaleException-IllformedLocaleException-0]]link:{java8-javadoc}/java/util/IllformedLocaleException.html#IllformedLocaleException%2D%2D[IllformedLocaleException]()++ (link:{java9-javadoc}/java/util/IllformedLocaleException.html#IllformedLocaleException%2D%2D[java 9])
+* ++[[painless-api-reference-IllformedLocaleException-IllformedLocaleException-1]]link:{java8-javadoc}/java/util/IllformedLocaleException.html#IllformedLocaleException%2Djava.lang.String%2D[IllformedLocaleException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/IllformedLocaleException.html#IllformedLocaleException%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-IllformedLocaleException-IllformedLocaleException-2]]link:{java8-javadoc}/java/util/IllformedLocaleException.html#IllformedLocaleException%2Djava.lang.String%2Dint%2D[IllformedLocaleException](<<painless-api-reference-String,String>>, int)++ (link:{java9-javadoc}/java/util/IllformedLocaleException.html#IllformedLocaleException%2Djava.lang.String%2Dint%2D[java 9])
+* ++[[painless-api-reference-IllformedLocaleException-getErrorIndex-0]]int link:{java8-javadoc}/java/util/IllformedLocaleException.html#getErrorIndex%2D%2D[getErrorIndex]()++ (link:{java9-javadoc}/java/util/IllformedLocaleException.html#getErrorIndex%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/IndexOutOfBoundsException.asciidoc b/docs/painless/painless-api-reference/IndexOutOfBoundsException.asciidoc
new file mode 100644
index 0000000000000..a141ec6b04152
--- /dev/null
+++ b/docs/painless/painless-api-reference/IndexOutOfBoundsException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IndexOutOfBoundsException]]++IndexOutOfBoundsException++::
+* ++[[painless-api-reference-IndexOutOfBoundsException-IndexOutOfBoundsException-0]]link:{java8-javadoc}/java/lang/IndexOutOfBoundsException.html#IndexOutOfBoundsException%2D%2D[IndexOutOfBoundsException]()++ (link:{java9-javadoc}/java/lang/IndexOutOfBoundsException.html#IndexOutOfBoundsException%2D%2D[java 9])
+* ++[[painless-api-reference-IndexOutOfBoundsException-IndexOutOfBoundsException-1]]link:{java8-javadoc}/java/lang/IndexOutOfBoundsException.html#IndexOutOfBoundsException%2Djava.lang.String%2D[IndexOutOfBoundsException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/IndexOutOfBoundsException.html#IndexOutOfBoundsException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/InputMismatchException.asciidoc b/docs/painless/painless-api-reference/InputMismatchException.asciidoc
new file mode 100644
index 0000000000000..9497cf9486e03
--- /dev/null
+++ b/docs/painless/painless-api-reference/InputMismatchException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-InputMismatchException]]++InputMismatchException++::
+* ++[[painless-api-reference-InputMismatchException-InputMismatchException-0]]link:{java8-javadoc}/java/util/InputMismatchException.html#InputMismatchException%2D%2D[InputMismatchException]()++ (link:{java9-javadoc}/java/util/InputMismatchException.html#InputMismatchException%2D%2D[java 9])
+* ++[[painless-api-reference-InputMismatchException-InputMismatchException-1]]link:{java8-javadoc}/java/util/InputMismatchException.html#InputMismatchException%2Djava.lang.String%2D[InputMismatchException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/InputMismatchException.html#InputMismatchException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Instant.asciidoc b/docs/painless/painless-api-reference/Instant.asciidoc
new file mode 100644
index 0000000000000..4a7ffe45d991c
--- /dev/null
+++ b/docs/painless/painless-api-reference/Instant.asciidoc
@@ -0,0 +1,36 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Instant]]++Instant++::
+** [[painless-api-reference-Instant-EPOCH]]static <<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/Instant.html#EPOCH[EPOCH] (link:{java9-javadoc}/java/time/Instant.html#EPOCH[java 9])
+** [[painless-api-reference-Instant-MAX]]static <<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/Instant.html#MAX[MAX] (link:{java9-javadoc}/java/time/Instant.html#MAX[java 9])
+** [[painless-api-reference-Instant-MIN]]static <<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/Instant.html#MIN[MIN] (link:{java9-javadoc}/java/time/Instant.html#MIN[java 9])
+* ++[[painless-api-reference-Instant-from-1]]static <<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/Instant.html#from%2Djava.time.temporal.TemporalAccessor%2D[from](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/Instant.html#from%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-Instant-ofEpochMilli-1]]static <<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/Instant.html#ofEpochMilli%2Dlong%2D[ofEpochMilli](long)++ (link:{java9-javadoc}/java/time/Instant.html#ofEpochMilli%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Instant-ofEpochSecond-1]]static <<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/Instant.html#ofEpochSecond%2Dlong%2D[ofEpochSecond](long)++ (link:{java9-javadoc}/java/time/Instant.html#ofEpochSecond%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Instant-ofEpochSecond-2]]static <<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/Instant.html#ofEpochSecond%2Dlong%2Dlong%2D[ofEpochSecond](long, long)++ (link:{java9-javadoc}/java/time/Instant.html#ofEpochSecond%2Dlong%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Instant-parse-1]]static <<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/Instant.html#parse%2Djava.lang.CharSequence%2D[parse](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/time/Instant.html#parse%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-Instant-atOffset-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/Instant.html#atOffset%2Djava.time.ZoneOffset%2D[atOffset](<<painless-api-reference-ZoneOffset,ZoneOffset>>)++ (link:{java9-javadoc}/java/time/Instant.html#atOffset%2Djava.time.ZoneOffset%2D[java 9])
+* ++[[painless-api-reference-Instant-atZone-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/Instant.html#atZone%2Djava.time.ZoneId%2D[atZone](<<painless-api-reference-ZoneId,ZoneId>>)++ (link:{java9-javadoc}/java/time/Instant.html#atZone%2Djava.time.ZoneId%2D[java 9])
+* ++[[painless-api-reference-Instant-compareTo-1]]int link:{java8-javadoc}/java/time/Instant.html#compareTo%2Djava.time.Instant%2D[compareTo](<<painless-api-reference-Instant,Instant>>)++ (link:{java9-javadoc}/java/time/Instant.html#compareTo%2Djava.time.Instant%2D[java 9])
+* ++[[painless-api-reference-Instant-getEpochSecond-0]]long link:{java8-javadoc}/java/time/Instant.html#getEpochSecond%2D%2D[getEpochSecond]()++ (link:{java9-javadoc}/java/time/Instant.html#getEpochSecond%2D%2D[java 9])
+* ++[[painless-api-reference-Instant-getNano-0]]int link:{java8-javadoc}/java/time/Instant.html#getNano%2D%2D[getNano]()++ (link:{java9-javadoc}/java/time/Instant.html#getNano%2D%2D[java 9])
+* ++[[painless-api-reference-Instant-isAfter-1]]boolean link:{java8-javadoc}/java/time/Instant.html#isAfter%2Djava.time.Instant%2D[isAfter](<<painless-api-reference-Instant,Instant>>)++ (link:{java9-javadoc}/java/time/Instant.html#isAfter%2Djava.time.Instant%2D[java 9])
+* ++[[painless-api-reference-Instant-isBefore-1]]boolean link:{java8-javadoc}/java/time/Instant.html#isBefore%2Djava.time.Instant%2D[isBefore](<<painless-api-reference-Instant,Instant>>)++ (link:{java9-javadoc}/java/time/Instant.html#isBefore%2Djava.time.Instant%2D[java 9])
+* ++[[painless-api-reference-Instant-minus-1]]<<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/Instant.html#minus%2Djava.time.temporal.TemporalAmount%2D[minus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/Instant.html#minus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-Instant-minus-2]]<<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/Instant.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[minus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/Instant.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-Instant-minusMillis-1]]<<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/Instant.html#minusMillis%2Dlong%2D[minusMillis](long)++ (link:{java9-javadoc}/java/time/Instant.html#minusMillis%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Instant-minusNanos-1]]<<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/Instant.html#minusNanos%2Dlong%2D[minusNanos](long)++ (link:{java9-javadoc}/java/time/Instant.html#minusNanos%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Instant-minusSeconds-1]]<<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/Instant.html#minusSeconds%2Dlong%2D[minusSeconds](long)++ (link:{java9-javadoc}/java/time/Instant.html#minusSeconds%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Instant-plus-1]]<<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/Instant.html#plus%2Djava.time.temporal.TemporalAmount%2D[plus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/Instant.html#plus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-Instant-plus-2]]<<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/Instant.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[plus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/Instant.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-Instant-plusMillis-1]]<<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/Instant.html#plusMillis%2Dlong%2D[plusMillis](long)++ (link:{java9-javadoc}/java/time/Instant.html#plusMillis%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Instant-plusNanos-1]]<<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/Instant.html#plusNanos%2Dlong%2D[plusNanos](long)++ (link:{java9-javadoc}/java/time/Instant.html#plusNanos%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Instant-plusSeconds-1]]<<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/Instant.html#plusSeconds%2Dlong%2D[plusSeconds](long)++ (link:{java9-javadoc}/java/time/Instant.html#plusSeconds%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Instant-toEpochMilli-0]]long link:{java8-javadoc}/java/time/Instant.html#toEpochMilli%2D%2D[toEpochMilli]()++ (link:{java9-javadoc}/java/time/Instant.html#toEpochMilli%2D%2D[java 9])
+* ++[[painless-api-reference-Instant-truncatedTo-1]]<<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/Instant.html#truncatedTo%2Djava.time.temporal.TemporalUnit%2D[truncatedTo](<<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/Instant.html#truncatedTo%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-Instant-with-1]]<<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/Instant.html#with%2Djava.time.temporal.TemporalAdjuster%2D[with](<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>)++ (link:{java9-javadoc}/java/time/Instant.html#with%2Djava.time.temporal.TemporalAdjuster%2D[java 9])
+* ++[[painless-api-reference-Instant-with-2]]<<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/Instant.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[with](<<painless-api-reference-TemporalField,TemporalField>>, long)++ (link:{java9-javadoc}/java/time/Instant.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-Temporal,Temporal>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++, ++<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>++
diff --git a/docs/painless/painless-api-reference/InstantiationException.asciidoc b/docs/painless/painless-api-reference/InstantiationException.asciidoc
new file mode 100644
index 0000000000000..06d88eb654f00
--- /dev/null
+++ b/docs/painless/painless-api-reference/InstantiationException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-InstantiationException]]++InstantiationException++::
+* ++[[painless-api-reference-InstantiationException-InstantiationException-0]]link:{java8-javadoc}/java/lang/InstantiationException.html#InstantiationException%2D%2D[InstantiationException]()++ (link:{java9-javadoc}/java/lang/InstantiationException.html#InstantiationException%2D%2D[java 9])
+* ++[[painless-api-reference-InstantiationException-InstantiationException-1]]link:{java8-javadoc}/java/lang/InstantiationException.html#InstantiationException%2Djava.lang.String%2D[InstantiationException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/InstantiationException.html#InstantiationException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/IntBinaryOperator.asciidoc b/docs/painless/painless-api-reference/IntBinaryOperator.asciidoc
new file mode 100644
index 0000000000000..ca3c64ab3d898
--- /dev/null
+++ b/docs/painless/painless-api-reference/IntBinaryOperator.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IntBinaryOperator]]++IntBinaryOperator++::
+* ++[[painless-api-reference-IntBinaryOperator-applyAsInt-2]]int link:{java8-javadoc}/java/util/function/IntBinaryOperator.html#applyAsInt%2Dint%2Dint%2D[applyAsInt](int, int)++ (link:{java9-javadoc}/java/util/function/IntBinaryOperator.html#applyAsInt%2Dint%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/IntConsumer.asciidoc b/docs/painless/painless-api-reference/IntConsumer.asciidoc
new file mode 100644
index 0000000000000..e8325838dac51
--- /dev/null
+++ b/docs/painless/painless-api-reference/IntConsumer.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IntConsumer]]++IntConsumer++::
+* ++[[painless-api-reference-IntConsumer-accept-1]]void link:{java8-javadoc}/java/util/function/IntConsumer.html#accept%2Dint%2D[accept](int)++ (link:{java9-javadoc}/java/util/function/IntConsumer.html#accept%2Dint%2D[java 9])
+* ++[[painless-api-reference-IntConsumer-andThen-1]]<<painless-api-reference-IntConsumer,IntConsumer>> link:{java8-javadoc}/java/util/function/IntConsumer.html#andThen%2Djava.util.function.IntConsumer%2D[andThen](<<painless-api-reference-IntConsumer,IntConsumer>>)++ (link:{java9-javadoc}/java/util/function/IntConsumer.html#andThen%2Djava.util.function.IntConsumer%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/IntFunction.asciidoc b/docs/painless/painless-api-reference/IntFunction.asciidoc
new file mode 100644
index 0000000000000..336c1f7c3cb70
--- /dev/null
+++ b/docs/painless/painless-api-reference/IntFunction.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IntFunction]]++IntFunction++::
+* ++[[painless-api-reference-IntFunction-apply-1]]def link:{java8-javadoc}/java/util/function/IntFunction.html#apply%2Dint%2D[apply](int)++ (link:{java9-javadoc}/java/util/function/IntFunction.html#apply%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/IntPredicate.asciidoc b/docs/painless/painless-api-reference/IntPredicate.asciidoc
new file mode 100644
index 0000000000000..a9c45c95af855
--- /dev/null
+++ b/docs/painless/painless-api-reference/IntPredicate.asciidoc
@@ -0,0 +1,11 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IntPredicate]]++IntPredicate++::
+* ++[[painless-api-reference-IntPredicate-and-1]]<<painless-api-reference-IntPredicate,IntPredicate>> link:{java8-javadoc}/java/util/function/IntPredicate.html#and%2Djava.util.function.IntPredicate%2D[and](<<painless-api-reference-IntPredicate,IntPredicate>>)++ (link:{java9-javadoc}/java/util/function/IntPredicate.html#and%2Djava.util.function.IntPredicate%2D[java 9])
+* ++[[painless-api-reference-IntPredicate-negate-0]]<<painless-api-reference-IntPredicate,IntPredicate>> link:{java8-javadoc}/java/util/function/IntPredicate.html#negate%2D%2D[negate]()++ (link:{java9-javadoc}/java/util/function/IntPredicate.html#negate%2D%2D[java 9])
+* ++[[painless-api-reference-IntPredicate-or-1]]<<painless-api-reference-IntPredicate,IntPredicate>> link:{java8-javadoc}/java/util/function/IntPredicate.html#or%2Djava.util.function.IntPredicate%2D[or](<<painless-api-reference-IntPredicate,IntPredicate>>)++ (link:{java9-javadoc}/java/util/function/IntPredicate.html#or%2Djava.util.function.IntPredicate%2D[java 9])
+* ++[[painless-api-reference-IntPredicate-test-1]]boolean link:{java8-javadoc}/java/util/function/IntPredicate.html#test%2Dint%2D[test](int)++ (link:{java9-javadoc}/java/util/function/IntPredicate.html#test%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/IntStream.Builder.asciidoc b/docs/painless/painless-api-reference/IntStream.Builder.asciidoc
new file mode 100644
index 0000000000000..0db51e7164b22
--- /dev/null
+++ b/docs/painless/painless-api-reference/IntStream.Builder.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IntStream-Builder]]++IntStream.Builder++::
+* ++[[painless-api-reference-IntStream-Builder-add-1]]<<painless-api-reference-IntStream-Builder,IntStream.Builder>> link:{java8-javadoc}/java/util/stream/IntStream.Builder.html#add%2Dint%2D[add](int)++ (link:{java9-javadoc}/java/util/stream/IntStream.Builder.html#add%2Dint%2D[java 9])
+* ++[[painless-api-reference-IntStream-Builder-build-0]]<<painless-api-reference-IntStream,IntStream>> link:{java8-javadoc}/java/util/stream/IntStream.Builder.html#build%2D%2D[build]()++ (link:{java9-javadoc}/java/util/stream/IntStream.Builder.html#build%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-IntConsumer,IntConsumer>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/IntStream.asciidoc b/docs/painless/painless-api-reference/IntStream.asciidoc
new file mode 100644
index 0000000000000..10e1149268986
--- /dev/null
+++ b/docs/painless/painless-api-reference/IntStream.asciidoc
@@ -0,0 +1,47 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IntStream]]++IntStream++::
+* ++[[painless-api-reference-IntStream-builder-0]]static <<painless-api-reference-IntStream-Builder,IntStream.Builder>> link:{java8-javadoc}/java/util/stream/IntStream.html#builder%2D%2D[builder]()++ (link:{java9-javadoc}/java/util/stream/IntStream.html#builder%2D%2D[java 9])
+* ++[[painless-api-reference-IntStream-concat-2]]static <<painless-api-reference-IntStream,IntStream>> link:{java8-javadoc}/java/util/stream/IntStream.html#concat%2Djava.util.stream.IntStream%2Djava.util.stream.IntStream%2D[concat](<<painless-api-reference-IntStream,IntStream>>, <<painless-api-reference-IntStream,IntStream>>)++ (link:{java9-javadoc}/java/util/stream/IntStream.html#concat%2Djava.util.stream.IntStream%2Djava.util.stream.IntStream%2D[java 9])
+* ++[[painless-api-reference-IntStream-empty-0]]static <<painless-api-reference-IntStream,IntStream>> link:{java8-javadoc}/java/util/stream/IntStream.html#empty%2D%2D[empty]()++ (link:{java9-javadoc}/java/util/stream/IntStream.html#empty%2D%2D[java 9])
+* ++[[painless-api-reference-IntStream-of-1]]static <<painless-api-reference-IntStream,IntStream>> link:{java8-javadoc}/java/util/stream/IntStream.html#of%2Dint:A%2D[of](int[])++ (link:{java9-javadoc}/java/util/stream/IntStream.html#of%2Dint:A%2D[java 9])
+* ++[[painless-api-reference-IntStream-range-2]]static <<painless-api-reference-IntStream,IntStream>> link:{java8-javadoc}/java/util/stream/IntStream.html#range%2Dint%2Dint%2D[range](int, int)++ (link:{java9-javadoc}/java/util/stream/IntStream.html#range%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-IntStream-rangeClosed-2]]static <<painless-api-reference-IntStream,IntStream>> link:{java8-javadoc}/java/util/stream/IntStream.html#rangeClosed%2Dint%2Dint%2D[rangeClosed](int, int)++ (link:{java9-javadoc}/java/util/stream/IntStream.html#rangeClosed%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-IntStream-allMatch-1]]boolean link:{java8-javadoc}/java/util/stream/IntStream.html#allMatch%2Djava.util.function.IntPredicate%2D[allMatch](<<painless-api-reference-IntPredicate,IntPredicate>>)++ (link:{java9-javadoc}/java/util/stream/IntStream.html#allMatch%2Djava.util.function.IntPredicate%2D[java 9])
+* ++[[painless-api-reference-IntStream-anyMatch-1]]boolean link:{java8-javadoc}/java/util/stream/IntStream.html#anyMatch%2Djava.util.function.IntPredicate%2D[anyMatch](<<painless-api-reference-IntPredicate,IntPredicate>>)++ (link:{java9-javadoc}/java/util/stream/IntStream.html#anyMatch%2Djava.util.function.IntPredicate%2D[java 9])
+* ++[[painless-api-reference-IntStream-asDoubleStream-0]]<<painless-api-reference-DoubleStream,DoubleStream>> link:{java8-javadoc}/java/util/stream/IntStream.html#asDoubleStream%2D%2D[asDoubleStream]()++ (link:{java9-javadoc}/java/util/stream/IntStream.html#asDoubleStream%2D%2D[java 9])
+* ++[[painless-api-reference-IntStream-asLongStream-0]]<<painless-api-reference-LongStream,LongStream>> link:{java8-javadoc}/java/util/stream/IntStream.html#asLongStream%2D%2D[asLongStream]()++ (link:{java9-javadoc}/java/util/stream/IntStream.html#asLongStream%2D%2D[java 9])
+* ++[[painless-api-reference-IntStream-average-0]]<<painless-api-reference-OptionalDouble,OptionalDouble>> link:{java8-javadoc}/java/util/stream/IntStream.html#average%2D%2D[average]()++ (link:{java9-javadoc}/java/util/stream/IntStream.html#average%2D%2D[java 9])
+* ++[[painless-api-reference-IntStream-boxed-0]]<<painless-api-reference-Stream,Stream>> link:{java8-javadoc}/java/util/stream/IntStream.html#boxed%2D%2D[boxed]()++ (link:{java9-javadoc}/java/util/stream/IntStream.html#boxed%2D%2D[java 9])
+* ++[[painless-api-reference-IntStream-collect-3]]def link:{java8-javadoc}/java/util/stream/IntStream.html#collect%2Djava.util.function.Supplier%2Djava.util.function.ObjIntConsumer%2Djava.util.function.BiConsumer%2D[collect](<<painless-api-reference-Supplier,Supplier>>, <<painless-api-reference-ObjIntConsumer,ObjIntConsumer>>, <<painless-api-reference-BiConsumer,BiConsumer>>)++ (link:{java9-javadoc}/java/util/stream/IntStream.html#collect%2Djava.util.function.Supplier%2Djava.util.function.ObjIntConsumer%2Djava.util.function.BiConsumer%2D[java 9])
+* ++[[painless-api-reference-IntStream-count-0]]long link:{java8-javadoc}/java/util/stream/IntStream.html#count%2D%2D[count]()++ (link:{java9-javadoc}/java/util/stream/IntStream.html#count%2D%2D[java 9])
+* ++[[painless-api-reference-IntStream-distinct-0]]<<painless-api-reference-IntStream,IntStream>> link:{java8-javadoc}/java/util/stream/IntStream.html#distinct%2D%2D[distinct]()++ (link:{java9-javadoc}/java/util/stream/IntStream.html#distinct%2D%2D[java 9])
+* ++[[painless-api-reference-IntStream-filter-1]]<<painless-api-reference-IntStream,IntStream>> link:{java8-javadoc}/java/util/stream/IntStream.html#filter%2Djava.util.function.IntPredicate%2D[filter](<<painless-api-reference-IntPredicate,IntPredicate>>)++ (link:{java9-javadoc}/java/util/stream/IntStream.html#filter%2Djava.util.function.IntPredicate%2D[java 9])
+* ++[[painless-api-reference-IntStream-findAny-0]]<<painless-api-reference-OptionalInt,OptionalInt>> link:{java8-javadoc}/java/util/stream/IntStream.html#findAny%2D%2D[findAny]()++ (link:{java9-javadoc}/java/util/stream/IntStream.html#findAny%2D%2D[java 9])
+* ++[[painless-api-reference-IntStream-findFirst-0]]<<painless-api-reference-OptionalInt,OptionalInt>> link:{java8-javadoc}/java/util/stream/IntStream.html#findFirst%2D%2D[findFirst]()++ (link:{java9-javadoc}/java/util/stream/IntStream.html#findFirst%2D%2D[java 9])
+* ++[[painless-api-reference-IntStream-flatMap-1]]<<painless-api-reference-IntStream,IntStream>> link:{java8-javadoc}/java/util/stream/IntStream.html#flatMap%2Djava.util.function.IntFunction%2D[flatMap](<<painless-api-reference-IntFunction,IntFunction>>)++ (link:{java9-javadoc}/java/util/stream/IntStream.html#flatMap%2Djava.util.function.IntFunction%2D[java 9])
+* ++[[painless-api-reference-IntStream-forEach-1]]void link:{java8-javadoc}/java/util/stream/IntStream.html#forEach%2Djava.util.function.IntConsumer%2D[forEach](<<painless-api-reference-IntConsumer,IntConsumer>>)++ (link:{java9-javadoc}/java/util/stream/IntStream.html#forEach%2Djava.util.function.IntConsumer%2D[java 9])
+* ++[[painless-api-reference-IntStream-forEachOrdered-1]]void link:{java8-javadoc}/java/util/stream/IntStream.html#forEachOrdered%2Djava.util.function.IntConsumer%2D[forEachOrdered](<<painless-api-reference-IntConsumer,IntConsumer>>)++ (link:{java9-javadoc}/java/util/stream/IntStream.html#forEachOrdered%2Djava.util.function.IntConsumer%2D[java 9])
+* ++[[painless-api-reference-IntStream-iterator-0]]<<painless-api-reference-PrimitiveIterator-OfInt,PrimitiveIterator.OfInt>> link:{java8-javadoc}/java/util/stream/IntStream.html#iterator%2D%2D[iterator]()++ (link:{java9-javadoc}/java/util/stream/IntStream.html#iterator%2D%2D[java 9])
+* ++[[painless-api-reference-IntStream-limit-1]]<<painless-api-reference-IntStream,IntStream>> link:{java8-javadoc}/java/util/stream/IntStream.html#limit%2Dlong%2D[limit](long)++ (link:{java9-javadoc}/java/util/stream/IntStream.html#limit%2Dlong%2D[java 9])
+* ++[[painless-api-reference-IntStream-map-1]]<<painless-api-reference-IntStream,IntStream>> link:{java8-javadoc}/java/util/stream/IntStream.html#map%2Djava.util.function.IntUnaryOperator%2D[map](<<painless-api-reference-IntUnaryOperator,IntUnaryOperator>>)++ (link:{java9-javadoc}/java/util/stream/IntStream.html#map%2Djava.util.function.IntUnaryOperator%2D[java 9])
+* ++[[painless-api-reference-IntStream-mapToDouble-1]]<<painless-api-reference-DoubleStream,DoubleStream>> link:{java8-javadoc}/java/util/stream/IntStream.html#mapToDouble%2Djava.util.function.IntToDoubleFunction%2D[mapToDouble](<<painless-api-reference-IntToDoubleFunction,IntToDoubleFunction>>)++ (link:{java9-javadoc}/java/util/stream/IntStream.html#mapToDouble%2Djava.util.function.IntToDoubleFunction%2D[java 9])
+* ++[[painless-api-reference-IntStream-mapToLong-1]]<<painless-api-reference-LongStream,LongStream>> link:{java8-javadoc}/java/util/stream/IntStream.html#mapToLong%2Djava.util.function.IntToLongFunction%2D[mapToLong](<<painless-api-reference-IntToLongFunction,IntToLongFunction>>)++ (link:{java9-javadoc}/java/util/stream/IntStream.html#mapToLong%2Djava.util.function.IntToLongFunction%2D[java 9])
+* ++[[painless-api-reference-IntStream-mapToObj-1]]<<painless-api-reference-Stream,Stream>> link:{java8-javadoc}/java/util/stream/IntStream.html#mapToObj%2Djava.util.function.IntFunction%2D[mapToObj](<<painless-api-reference-IntFunction,IntFunction>>)++ (link:{java9-javadoc}/java/util/stream/IntStream.html#mapToObj%2Djava.util.function.IntFunction%2D[java 9])
+* ++[[painless-api-reference-IntStream-max-0]]<<painless-api-reference-OptionalInt,OptionalInt>> link:{java8-javadoc}/java/util/stream/IntStream.html#max%2D%2D[max]()++ (link:{java9-javadoc}/java/util/stream/IntStream.html#max%2D%2D[java 9])
+* ++[[painless-api-reference-IntStream-min-0]]<<painless-api-reference-OptionalInt,OptionalInt>> link:{java8-javadoc}/java/util/stream/IntStream.html#min%2D%2D[min]()++ (link:{java9-javadoc}/java/util/stream/IntStream.html#min%2D%2D[java 9])
+* ++[[painless-api-reference-IntStream-noneMatch-1]]boolean link:{java8-javadoc}/java/util/stream/IntStream.html#noneMatch%2Djava.util.function.IntPredicate%2D[noneMatch](<<painless-api-reference-IntPredicate,IntPredicate>>)++ (link:{java9-javadoc}/java/util/stream/IntStream.html#noneMatch%2Djava.util.function.IntPredicate%2D[java 9])
+* ++[[painless-api-reference-IntStream-peek-1]]<<painless-api-reference-IntStream,IntStream>> link:{java8-javadoc}/java/util/stream/IntStream.html#peek%2Djava.util.function.IntConsumer%2D[peek](<<painless-api-reference-IntConsumer,IntConsumer>>)++ (link:{java9-javadoc}/java/util/stream/IntStream.html#peek%2Djava.util.function.IntConsumer%2D[java 9])
+* ++[[painless-api-reference-IntStream-reduce-1]]<<painless-api-reference-OptionalInt,OptionalInt>> link:{java8-javadoc}/java/util/stream/IntStream.html#reduce%2Djava.util.function.IntBinaryOperator%2D[reduce](<<painless-api-reference-IntBinaryOperator,IntBinaryOperator>>)++ (link:{java9-javadoc}/java/util/stream/IntStream.html#reduce%2Djava.util.function.IntBinaryOperator%2D[java 9])
+* ++[[painless-api-reference-IntStream-reduce-2]]int link:{java8-javadoc}/java/util/stream/IntStream.html#reduce%2Dint%2Djava.util.function.IntBinaryOperator%2D[reduce](int, <<painless-api-reference-IntBinaryOperator,IntBinaryOperator>>)++ (link:{java9-javadoc}/java/util/stream/IntStream.html#reduce%2Dint%2Djava.util.function.IntBinaryOperator%2D[java 9])
+* ++[[painless-api-reference-IntStream-sequential-0]]<<painless-api-reference-IntStream,IntStream>> link:{java8-javadoc}/java/util/stream/IntStream.html#sequential%2D%2D[sequential]()++ (link:{java9-javadoc}/java/util/stream/IntStream.html#sequential%2D%2D[java 9])
+* ++[[painless-api-reference-IntStream-skip-1]]<<painless-api-reference-IntStream,IntStream>> link:{java8-javadoc}/java/util/stream/IntStream.html#skip%2Dlong%2D[skip](long)++ (link:{java9-javadoc}/java/util/stream/IntStream.html#skip%2Dlong%2D[java 9])
+* ++[[painless-api-reference-IntStream-sorted-0]]<<painless-api-reference-IntStream,IntStream>> link:{java8-javadoc}/java/util/stream/IntStream.html#sorted%2D%2D[sorted]()++ (link:{java9-javadoc}/java/util/stream/IntStream.html#sorted%2D%2D[java 9])
+* ++[[painless-api-reference-IntStream-spliterator-0]]<<painless-api-reference-Spliterator-OfInt,Spliterator.OfInt>> link:{java8-javadoc}/java/util/stream/IntStream.html#spliterator%2D%2D[spliterator]()++ (link:{java9-javadoc}/java/util/stream/IntStream.html#spliterator%2D%2D[java 9])
+* ++[[painless-api-reference-IntStream-sum-0]]int link:{java8-javadoc}/java/util/stream/IntStream.html#sum%2D%2D[sum]()++ (link:{java9-javadoc}/java/util/stream/IntStream.html#sum%2D%2D[java 9])
+* ++[[painless-api-reference-IntStream-summaryStatistics-0]]<<painless-api-reference-IntSummaryStatistics,IntSummaryStatistics>> link:{java8-javadoc}/java/util/stream/IntStream.html#summaryStatistics%2D%2D[summaryStatistics]()++ (link:{java9-javadoc}/java/util/stream/IntStream.html#summaryStatistics%2D%2D[java 9])
+* ++[[painless-api-reference-IntStream-toArray-0]]int[] link:{java8-javadoc}/java/util/stream/IntStream.html#toArray%2D%2D[toArray]()++ (link:{java9-javadoc}/java/util/stream/IntStream.html#toArray%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-BaseStream,BaseStream>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/IntSummaryStatistics.asciidoc b/docs/painless/painless-api-reference/IntSummaryStatistics.asciidoc
new file mode 100644
index 0000000000000..e13d47ad7c4aa
--- /dev/null
+++ b/docs/painless/painless-api-reference/IntSummaryStatistics.asciidoc
@@ -0,0 +1,14 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IntSummaryStatistics]]++IntSummaryStatistics++::
+* ++[[painless-api-reference-IntSummaryStatistics-IntSummaryStatistics-0]]link:{java8-javadoc}/java/util/IntSummaryStatistics.html#IntSummaryStatistics%2D%2D[IntSummaryStatistics]()++ (link:{java9-javadoc}/java/util/IntSummaryStatistics.html#IntSummaryStatistics%2D%2D[java 9])
+* ++[[painless-api-reference-IntSummaryStatistics-combine-1]]void link:{java8-javadoc}/java/util/IntSummaryStatistics.html#combine%2Djava.util.IntSummaryStatistics%2D[combine](<<painless-api-reference-IntSummaryStatistics,IntSummaryStatistics>>)++ (link:{java9-javadoc}/java/util/IntSummaryStatistics.html#combine%2Djava.util.IntSummaryStatistics%2D[java 9])
+* ++[[painless-api-reference-IntSummaryStatistics-getAverage-0]]double link:{java8-javadoc}/java/util/IntSummaryStatistics.html#getAverage%2D%2D[getAverage]()++ (link:{java9-javadoc}/java/util/IntSummaryStatistics.html#getAverage%2D%2D[java 9])
+* ++[[painless-api-reference-IntSummaryStatistics-getCount-0]]long link:{java8-javadoc}/java/util/IntSummaryStatistics.html#getCount%2D%2D[getCount]()++ (link:{java9-javadoc}/java/util/IntSummaryStatistics.html#getCount%2D%2D[java 9])
+* ++[[painless-api-reference-IntSummaryStatistics-getMax-0]]int link:{java8-javadoc}/java/util/IntSummaryStatistics.html#getMax%2D%2D[getMax]()++ (link:{java9-javadoc}/java/util/IntSummaryStatistics.html#getMax%2D%2D[java 9])
+* ++[[painless-api-reference-IntSummaryStatistics-getMin-0]]int link:{java8-javadoc}/java/util/IntSummaryStatistics.html#getMin%2D%2D[getMin]()++ (link:{java9-javadoc}/java/util/IntSummaryStatistics.html#getMin%2D%2D[java 9])
+* ++[[painless-api-reference-IntSummaryStatistics-getSum-0]]long link:{java8-javadoc}/java/util/IntSummaryStatistics.html#getSum%2D%2D[getSum]()++ (link:{java9-javadoc}/java/util/IntSummaryStatistics.html#getSum%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-IntConsumer,IntConsumer>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/IntSupplier.asciidoc b/docs/painless/painless-api-reference/IntSupplier.asciidoc
new file mode 100644
index 0000000000000..bca332f7e6cf4
--- /dev/null
+++ b/docs/painless/painless-api-reference/IntSupplier.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IntSupplier]]++IntSupplier++::
+* ++[[painless-api-reference-IntSupplier-getAsInt-0]]int link:{java8-javadoc}/java/util/function/IntSupplier.html#getAsInt%2D%2D[getAsInt]()++ (link:{java9-javadoc}/java/util/function/IntSupplier.html#getAsInt%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/IntToDoubleFunction.asciidoc b/docs/painless/painless-api-reference/IntToDoubleFunction.asciidoc
new file mode 100644
index 0000000000000..1cfadec16b2d8
--- /dev/null
+++ b/docs/painless/painless-api-reference/IntToDoubleFunction.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IntToDoubleFunction]]++IntToDoubleFunction++::
+* ++[[painless-api-reference-IntToDoubleFunction-applyAsDouble-1]]double link:{java8-javadoc}/java/util/function/IntToDoubleFunction.html#applyAsDouble%2Dint%2D[applyAsDouble](int)++ (link:{java9-javadoc}/java/util/function/IntToDoubleFunction.html#applyAsDouble%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/IntToLongFunction.asciidoc b/docs/painless/painless-api-reference/IntToLongFunction.asciidoc
new file mode 100644
index 0000000000000..3cffe6cddea16
--- /dev/null
+++ b/docs/painless/painless-api-reference/IntToLongFunction.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IntToLongFunction]]++IntToLongFunction++::
+* ++[[painless-api-reference-IntToLongFunction-applyAsLong-1]]long link:{java8-javadoc}/java/util/function/IntToLongFunction.html#applyAsLong%2Dint%2D[applyAsLong](int)++ (link:{java9-javadoc}/java/util/function/IntToLongFunction.html#applyAsLong%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/IntUnaryOperator.asciidoc b/docs/painless/painless-api-reference/IntUnaryOperator.asciidoc
new file mode 100644
index 0000000000000..a1bb1651fee56
--- /dev/null
+++ b/docs/painless/painless-api-reference/IntUnaryOperator.asciidoc
@@ -0,0 +1,11 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IntUnaryOperator]]++IntUnaryOperator++::
+* ++[[painless-api-reference-IntUnaryOperator-identity-0]]static <<painless-api-reference-IntUnaryOperator,IntUnaryOperator>> link:{java8-javadoc}/java/util/function/IntUnaryOperator.html#identity%2D%2D[identity]()++ (link:{java9-javadoc}/java/util/function/IntUnaryOperator.html#identity%2D%2D[java 9])
+* ++[[painless-api-reference-IntUnaryOperator-andThen-1]]<<painless-api-reference-IntUnaryOperator,IntUnaryOperator>> link:{java8-javadoc}/java/util/function/IntUnaryOperator.html#andThen%2Djava.util.function.IntUnaryOperator%2D[andThen](<<painless-api-reference-IntUnaryOperator,IntUnaryOperator>>)++ (link:{java9-javadoc}/java/util/function/IntUnaryOperator.html#andThen%2Djava.util.function.IntUnaryOperator%2D[java 9])
+* ++[[painless-api-reference-IntUnaryOperator-applyAsInt-1]]int link:{java8-javadoc}/java/util/function/IntUnaryOperator.html#applyAsInt%2Dint%2D[applyAsInt](int)++ (link:{java9-javadoc}/java/util/function/IntUnaryOperator.html#applyAsInt%2Dint%2D[java 9])
+* ++[[painless-api-reference-IntUnaryOperator-compose-1]]<<painless-api-reference-IntUnaryOperator,IntUnaryOperator>> link:{java8-javadoc}/java/util/function/IntUnaryOperator.html#compose%2Djava.util.function.IntUnaryOperator%2D[compose](<<painless-api-reference-IntUnaryOperator,IntUnaryOperator>>)++ (link:{java9-javadoc}/java/util/function/IntUnaryOperator.html#compose%2Djava.util.function.IntUnaryOperator%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Integer.asciidoc b/docs/painless/painless-api-reference/Integer.asciidoc
new file mode 100644
index 0000000000000..2f608bcc662f7
--- /dev/null
+++ b/docs/painless/painless-api-reference/Integer.asciidoc
@@ -0,0 +1,44 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Integer]]++Integer++::
+** [[painless-api-reference-Integer-BYTES]]static int link:{java8-javadoc}/java/lang/Integer.html#BYTES[BYTES] (link:{java9-javadoc}/java/lang/Integer.html#BYTES[java 9])
+** [[painless-api-reference-Integer-MAX_VALUE]]static int link:{java8-javadoc}/java/lang/Integer.html#MAX_VALUE[MAX_VALUE] (link:{java9-javadoc}/java/lang/Integer.html#MAX_VALUE[java 9])
+** [[painless-api-reference-Integer-MIN_VALUE]]static int link:{java8-javadoc}/java/lang/Integer.html#MIN_VALUE[MIN_VALUE] (link:{java9-javadoc}/java/lang/Integer.html#MIN_VALUE[java 9])
+** [[painless-api-reference-Integer-SIZE]]static int link:{java8-javadoc}/java/lang/Integer.html#SIZE[SIZE] (link:{java9-javadoc}/java/lang/Integer.html#SIZE[java 9])
+* ++[[painless-api-reference-Integer-bitCount-1]]static int link:{java8-javadoc}/java/lang/Integer.html#bitCount%2Dint%2D[bitCount](int)++ (link:{java9-javadoc}/java/lang/Integer.html#bitCount%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-compare-2]]static int link:{java8-javadoc}/java/lang/Integer.html#compare%2Dint%2Dint%2D[compare](int, int)++ (link:{java9-javadoc}/java/lang/Integer.html#compare%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-compareUnsigned-2]]static int link:{java8-javadoc}/java/lang/Integer.html#compareUnsigned%2Dint%2Dint%2D[compareUnsigned](int, int)++ (link:{java9-javadoc}/java/lang/Integer.html#compareUnsigned%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-decode-1]]static <<painless-api-reference-Integer,Integer>> link:{java8-javadoc}/java/lang/Integer.html#decode%2Djava.lang.String%2D[decode](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/Integer.html#decode%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Integer-divideUnsigned-2]]static int link:{java8-javadoc}/java/lang/Integer.html#divideUnsigned%2Dint%2Dint%2D[divideUnsigned](int, int)++ (link:{java9-javadoc}/java/lang/Integer.html#divideUnsigned%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-hashCode-1]]static int link:{java8-javadoc}/java/lang/Integer.html#hashCode%2Dint%2D[hashCode](int)++ (link:{java9-javadoc}/java/lang/Integer.html#hashCode%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-highestOneBit-1]]static int link:{java8-javadoc}/java/lang/Integer.html#highestOneBit%2Dint%2D[highestOneBit](int)++ (link:{java9-javadoc}/java/lang/Integer.html#highestOneBit%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-lowestOneBit-1]]static int link:{java8-javadoc}/java/lang/Integer.html#lowestOneBit%2Dint%2D[lowestOneBit](int)++ (link:{java9-javadoc}/java/lang/Integer.html#lowestOneBit%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-max-2]]static int link:{java8-javadoc}/java/lang/Integer.html#max%2Dint%2Dint%2D[max](int, int)++ (link:{java9-javadoc}/java/lang/Integer.html#max%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-min-2]]static int link:{java8-javadoc}/java/lang/Integer.html#min%2Dint%2Dint%2D[min](int, int)++ (link:{java9-javadoc}/java/lang/Integer.html#min%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-numberOfLeadingZeros-1]]static int link:{java8-javadoc}/java/lang/Integer.html#numberOfLeadingZeros%2Dint%2D[numberOfLeadingZeros](int)++ (link:{java9-javadoc}/java/lang/Integer.html#numberOfLeadingZeros%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-numberOfTrailingZeros-1]]static int link:{java8-javadoc}/java/lang/Integer.html#numberOfTrailingZeros%2Dint%2D[numberOfTrailingZeros](int)++ (link:{java9-javadoc}/java/lang/Integer.html#numberOfTrailingZeros%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-parseInt-1]]static int link:{java8-javadoc}/java/lang/Integer.html#parseInt%2Djava.lang.String%2D[parseInt](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/Integer.html#parseInt%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Integer-parseInt-2]]static int link:{java8-javadoc}/java/lang/Integer.html#parseInt%2Djava.lang.String%2Dint%2D[parseInt](<<painless-api-reference-String,String>>, int)++ (link:{java9-javadoc}/java/lang/Integer.html#parseInt%2Djava.lang.String%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-parseUnsignedInt-1]]static int link:{java8-javadoc}/java/lang/Integer.html#parseUnsignedInt%2Djava.lang.String%2D[parseUnsignedInt](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/Integer.html#parseUnsignedInt%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Integer-parseUnsignedInt-2]]static int link:{java8-javadoc}/java/lang/Integer.html#parseUnsignedInt%2Djava.lang.String%2Dint%2D[parseUnsignedInt](<<painless-api-reference-String,String>>, int)++ (link:{java9-javadoc}/java/lang/Integer.html#parseUnsignedInt%2Djava.lang.String%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-remainderUnsigned-2]]static int link:{java8-javadoc}/java/lang/Integer.html#remainderUnsigned%2Dint%2Dint%2D[remainderUnsigned](int, int)++ (link:{java9-javadoc}/java/lang/Integer.html#remainderUnsigned%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-reverse-1]]static int link:{java8-javadoc}/java/lang/Integer.html#reverse%2Dint%2D[reverse](int)++ (link:{java9-javadoc}/java/lang/Integer.html#reverse%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-reverseBytes-1]]static int link:{java8-javadoc}/java/lang/Integer.html#reverseBytes%2Dint%2D[reverseBytes](int)++ (link:{java9-javadoc}/java/lang/Integer.html#reverseBytes%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-rotateLeft-2]]static int link:{java8-javadoc}/java/lang/Integer.html#rotateLeft%2Dint%2Dint%2D[rotateLeft](int, int)++ (link:{java9-javadoc}/java/lang/Integer.html#rotateLeft%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-rotateRight-2]]static int link:{java8-javadoc}/java/lang/Integer.html#rotateRight%2Dint%2Dint%2D[rotateRight](int, int)++ (link:{java9-javadoc}/java/lang/Integer.html#rotateRight%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-signum-1]]static int link:{java8-javadoc}/java/lang/Integer.html#signum%2Dint%2D[signum](int)++ (link:{java9-javadoc}/java/lang/Integer.html#signum%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-toBinaryString-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Integer.html#toBinaryString%2Dint%2D[toBinaryString](int)++ (link:{java9-javadoc}/java/lang/Integer.html#toBinaryString%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-toHexString-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Integer.html#toHexString%2Dint%2D[toHexString](int)++ (link:{java9-javadoc}/java/lang/Integer.html#toHexString%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-toOctalString-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Integer.html#toOctalString%2Dint%2D[toOctalString](int)++ (link:{java9-javadoc}/java/lang/Integer.html#toOctalString%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-toString-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Integer.html#toString%2Dint%2D[toString](int)++ (link:{java9-javadoc}/java/lang/Integer.html#toString%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-toString-2]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Integer.html#toString%2Dint%2Dint%2D[toString](int, int)++ (link:{java9-javadoc}/java/lang/Integer.html#toString%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-toUnsignedLong-1]]static long link:{java8-javadoc}/java/lang/Integer.html#toUnsignedLong%2Dint%2D[toUnsignedLong](int)++ (link:{java9-javadoc}/java/lang/Integer.html#toUnsignedLong%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-toUnsignedString-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Integer.html#toUnsignedString%2Dint%2D[toUnsignedString](int)++ (link:{java9-javadoc}/java/lang/Integer.html#toUnsignedString%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-toUnsignedString-2]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Integer.html#toUnsignedString%2Dint%2Dint%2D[toUnsignedString](int, int)++ (link:{java9-javadoc}/java/lang/Integer.html#toUnsignedString%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-valueOf-1]]static <<painless-api-reference-Integer,Integer>> link:{java8-javadoc}/java/lang/Integer.html#valueOf%2Dint%2D[valueOf](int)++ (link:{java9-javadoc}/java/lang/Integer.html#valueOf%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-valueOf-2]]static <<painless-api-reference-Integer,Integer>> link:{java8-javadoc}/java/lang/Integer.html#valueOf%2Djava.lang.String%2Dint%2D[valueOf](<<painless-api-reference-String,String>>, int)++ (link:{java9-javadoc}/java/lang/Integer.html#valueOf%2Djava.lang.String%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-compareTo-1]]int link:{java8-javadoc}/java/lang/Integer.html#compareTo%2Djava.lang.Integer%2D[compareTo](<<painless-api-reference-Integer,Integer>>)++ (link:{java9-javadoc}/java/lang/Integer.html#compareTo%2Djava.lang.Integer%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Number,Number>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/InterruptedException.asciidoc b/docs/painless/painless-api-reference/InterruptedException.asciidoc
new file mode 100644
index 0000000000000..cd0ef457f76ab
--- /dev/null
+++ b/docs/painless/painless-api-reference/InterruptedException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-InterruptedException]]++InterruptedException++::
+* ++[[painless-api-reference-InterruptedException-InterruptedException-0]]link:{java8-javadoc}/java/lang/InterruptedException.html#InterruptedException%2D%2D[InterruptedException]()++ (link:{java9-javadoc}/java/lang/InterruptedException.html#InterruptedException%2D%2D[java 9])
+* ++[[painless-api-reference-InterruptedException-InterruptedException-1]]link:{java8-javadoc}/java/lang/InterruptedException.html#InterruptedException%2Djava.lang.String%2D[InterruptedException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/InterruptedException.html#InterruptedException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/IsoChronology.asciidoc b/docs/painless/painless-api-reference/IsoChronology.asciidoc
new file mode 100644
index 0000000000000..3990839b041fc
--- /dev/null
+++ b/docs/painless/painless-api-reference/IsoChronology.asciidoc
@@ -0,0 +1,20 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IsoChronology]]++IsoChronology++::
+** [[painless-api-reference-IsoChronology-INSTANCE]]static <<painless-api-reference-IsoChronology,IsoChronology>> link:{java8-javadoc}/java/time/chrono/IsoChronology.html#INSTANCE[INSTANCE] (link:{java9-javadoc}/java/time/chrono/IsoChronology.html#INSTANCE[java 9])
+* ++[[painless-api-reference-IsoChronology-date-1]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/chrono/IsoChronology.html#date%2Djava.time.temporal.TemporalAccessor%2D[date](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/chrono/IsoChronology.html#date%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-IsoChronology-date-3]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/chrono/IsoChronology.html#date%2Dint%2Dint%2Dint%2D[date](int, int, int)++ (link:{java9-javadoc}/java/time/chrono/IsoChronology.html#date%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-IsoChronology-date-4]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/chrono/IsoChronology.html#date%2Djava.time.chrono.Era%2Dint%2Dint%2Dint%2D[date](<<painless-api-reference-Era,Era>>, int, int, int)++ (link:{java9-javadoc}/java/time/chrono/IsoChronology.html#date%2Djava.time.chrono.Era%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-IsoChronology-dateEpochDay-1]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/chrono/IsoChronology.html#dateEpochDay%2Dlong%2D[dateEpochDay](long)++ (link:{java9-javadoc}/java/time/chrono/IsoChronology.html#dateEpochDay%2Dlong%2D[java 9])
+* ++[[painless-api-reference-IsoChronology-dateYearDay-2]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/chrono/IsoChronology.html#dateYearDay%2Dint%2Dint%2D[dateYearDay](int, int)++ (link:{java9-javadoc}/java/time/chrono/IsoChronology.html#dateYearDay%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-IsoChronology-dateYearDay-3]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/chrono/IsoChronology.html#dateYearDay%2Djava.time.chrono.Era%2Dint%2Dint%2D[dateYearDay](<<painless-api-reference-Era,Era>>, int, int)++ (link:{java9-javadoc}/java/time/chrono/IsoChronology.html#dateYearDay%2Djava.time.chrono.Era%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-IsoChronology-eraOf-1]]<<painless-api-reference-IsoEra,IsoEra>> link:{java8-javadoc}/java/time/chrono/IsoChronology.html#eraOf%2Dint%2D[eraOf](int)++ (link:{java9-javadoc}/java/time/chrono/IsoChronology.html#eraOf%2Dint%2D[java 9])
+* ++[[painless-api-reference-IsoChronology-localDateTime-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/chrono/IsoChronology.html#localDateTime%2Djava.time.temporal.TemporalAccessor%2D[localDateTime](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/chrono/IsoChronology.html#localDateTime%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-IsoChronology-period-3]]<<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/chrono/IsoChronology.html#period%2Dint%2Dint%2Dint%2D[period](int, int, int)++ (link:{java9-javadoc}/java/time/chrono/IsoChronology.html#period%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-IsoChronology-resolveDate-2]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/chrono/IsoChronology.html#resolveDate%2Djava.util.Map%2Djava.time.format.ResolverStyle%2D[resolveDate](<<painless-api-reference-Map,Map>>, <<painless-api-reference-ResolverStyle,ResolverStyle>>)++ (link:{java9-javadoc}/java/time/chrono/IsoChronology.html#resolveDate%2Djava.util.Map%2Djava.time.format.ResolverStyle%2D[java 9])
+* ++[[painless-api-reference-IsoChronology-zonedDateTime-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/chrono/IsoChronology.html#zonedDateTime%2Djava.time.temporal.TemporalAccessor%2D[zonedDateTime](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/chrono/IsoChronology.html#zonedDateTime%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-IsoChronology-zonedDateTime-2]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/chrono/IsoChronology.html#zonedDateTime%2Djava.time.Instant%2Djava.time.ZoneId%2D[zonedDateTime](<<painless-api-reference-Instant,Instant>>, <<painless-api-reference-ZoneId,ZoneId>>)++ (link:{java9-javadoc}/java/time/chrono/IsoChronology.html#zonedDateTime%2Djava.time.Instant%2Djava.time.ZoneId%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Chronology,Chronology>>++
diff --git a/docs/painless/painless-api-reference/IsoEra.asciidoc b/docs/painless/painless-api-reference/IsoEra.asciidoc
new file mode 100644
index 0000000000000..975d7d1153aef
--- /dev/null
+++ b/docs/painless/painless-api-reference/IsoEra.asciidoc
@@ -0,0 +1,13 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IsoEra]]++IsoEra++::
+** [[painless-api-reference-IsoEra-BCE]]static <<painless-api-reference-IsoEra,IsoEra>> link:{java8-javadoc}/java/time/chrono/IsoEra.html#BCE[BCE] (link:{java9-javadoc}/java/time/chrono/IsoEra.html#BCE[java 9])
+** [[painless-api-reference-IsoEra-CE]]static <<painless-api-reference-IsoEra,IsoEra>> link:{java8-javadoc}/java/time/chrono/IsoEra.html#CE[CE] (link:{java9-javadoc}/java/time/chrono/IsoEra.html#CE[java 9])
+* ++[[painless-api-reference-IsoEra-of-1]]static <<painless-api-reference-IsoEra,IsoEra>> link:{java8-javadoc}/java/time/chrono/IsoEra.html#of%2Dint%2D[of](int)++ (link:{java9-javadoc}/java/time/chrono/IsoEra.html#of%2Dint%2D[java 9])
+* ++[[painless-api-reference-IsoEra-valueOf-1]]static <<painless-api-reference-IsoEra,IsoEra>> link:{java8-javadoc}/java/time/chrono/IsoEra.html#valueOf%2Djava.lang.String%2D[valueOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/chrono/IsoEra.html#valueOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-IsoEra-values-0]]static <<painless-api-reference-IsoEra,IsoEra>>[] link:{java8-javadoc}/java/time/chrono/IsoEra.html#values%2D%2D[values]()++ (link:{java9-javadoc}/java/time/chrono/IsoEra.html#values%2D%2D[java 9])
+* ++[[painless-api-reference-IsoEra-getValue-0]]int link:{java8-javadoc}/java/time/chrono/IsoEra.html#getValue%2D%2D[getValue]()++ (link:{java9-javadoc}/java/time/chrono/IsoEra.html#getValue%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Enum,Enum>>++, ++<<painless-api-reference-Era,Era>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++, ++<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>++
diff --git a/docs/painless/painless-api-reference/IsoFields.asciidoc b/docs/painless/painless-api-reference/IsoFields.asciidoc
new file mode 100644
index 0000000000000..db3f5bbefbb5e
--- /dev/null
+++ b/docs/painless/painless-api-reference/IsoFields.asciidoc
@@ -0,0 +1,13 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IsoFields]]++IsoFields++::
+** [[painless-api-reference-IsoFields-DAY_OF_QUARTER]]static <<painless-api-reference-TemporalField,TemporalField>> link:{java8-javadoc}/java/time/temporal/IsoFields.html#DAY_OF_QUARTER[DAY_OF_QUARTER] (link:{java9-javadoc}/java/time/temporal/IsoFields.html#DAY_OF_QUARTER[java 9])
+** [[painless-api-reference-IsoFields-QUARTER_OF_YEAR]]static <<painless-api-reference-TemporalField,TemporalField>> link:{java8-javadoc}/java/time/temporal/IsoFields.html#QUARTER_OF_YEAR[QUARTER_OF_YEAR] (link:{java9-javadoc}/java/time/temporal/IsoFields.html#QUARTER_OF_YEAR[java 9])
+** [[painless-api-reference-IsoFields-QUARTER_YEARS]]static <<painless-api-reference-TemporalUnit,TemporalUnit>> link:{java8-javadoc}/java/time/temporal/IsoFields.html#QUARTER_YEARS[QUARTER_YEARS] (link:{java9-javadoc}/java/time/temporal/IsoFields.html#QUARTER_YEARS[java 9])
+** [[painless-api-reference-IsoFields-WEEK_BASED_YEAR]]static <<painless-api-reference-TemporalField,TemporalField>> link:{java8-javadoc}/java/time/temporal/IsoFields.html#WEEK_BASED_YEAR[WEEK_BASED_YEAR] (link:{java9-javadoc}/java/time/temporal/IsoFields.html#WEEK_BASED_YEAR[java 9])
+** [[painless-api-reference-IsoFields-WEEK_BASED_YEARS]]static <<painless-api-reference-TemporalUnit,TemporalUnit>> link:{java8-javadoc}/java/time/temporal/IsoFields.html#WEEK_BASED_YEARS[WEEK_BASED_YEARS] (link:{java9-javadoc}/java/time/temporal/IsoFields.html#WEEK_BASED_YEARS[java 9])
+** [[painless-api-reference-IsoFields-WEEK_OF_WEEK_BASED_YEAR]]static <<painless-api-reference-TemporalField,TemporalField>> link:{java8-javadoc}/java/time/temporal/IsoFields.html#WEEK_OF_WEEK_BASED_YEAR[WEEK_OF_WEEK_BASED_YEAR] (link:{java9-javadoc}/java/time/temporal/IsoFields.html#WEEK_OF_WEEK_BASED_YEAR[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Iterable.asciidoc b/docs/painless/painless-api-reference/Iterable.asciidoc
new file mode 100644
index 0000000000000..b7d626767c710
--- /dev/null
+++ b/docs/painless/painless-api-reference/Iterable.asciidoc
@@ -0,0 +1,21 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Iterable]]++Iterable++::
+* ++[[painless-api-reference-Iterable-any-1]]boolean link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#any%2Djava.lang.Iterable%2Djava.util.function.Predicate%2D[any](<<painless-api-reference-Predicate,Predicate>>)++
+* ++[[painless-api-reference-Iterable-asCollection-0]]<<painless-api-reference-Collection,Collection>> link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#asCollection%2Djava.lang.Iterable%2D[asCollection]()++
+* ++[[painless-api-reference-Iterable-asList-0]]<<painless-api-reference-List,List>> link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#asList%2Djava.lang.Iterable%2D[asList]()++
+* ++[[painless-api-reference-Iterable-each-1]]def link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#each%2Djava.lang.Iterable%2Djava.util.function.Consumer%2D[each](<<painless-api-reference-Consumer,Consumer>>)++
+* ++[[painless-api-reference-Iterable-eachWithIndex-1]]def link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#eachWithIndex%2Djava.lang.Iterable%2Djava.util.function.ObjIntConsumer%2D[eachWithIndex](<<painless-api-reference-ObjIntConsumer,ObjIntConsumer>>)++
+* ++[[painless-api-reference-Iterable-every-1]]boolean link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#every%2Djava.lang.Iterable%2Djava.util.function.Predicate%2D[every](<<painless-api-reference-Predicate,Predicate>>)++
+* ++[[painless-api-reference-Iterable-findResults-1]]<<painless-api-reference-List,List>> link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#findResults%2Djava.lang.Iterable%2Djava.util.function.Function%2D[findResults](<<painless-api-reference-Function,Function>>)++
+* ++[[painless-api-reference-Iterable-forEach-1]]void link:{java8-javadoc}/java/lang/Iterable.html#forEach%2Djava.util.function.Consumer%2D[forEach](<<painless-api-reference-Consumer,Consumer>>)++ (link:{java9-javadoc}/java/lang/Iterable.html#forEach%2Djava.util.function.Consumer%2D[java 9])
+* ++[[painless-api-reference-Iterable-groupBy-1]]<<painless-api-reference-Map,Map>> link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#groupBy%2Djava.lang.Iterable%2Djava.util.function.Function%2D[groupBy](<<painless-api-reference-Function,Function>>)++
+* ++[[painless-api-reference-Iterable-iterator-0]]<<painless-api-reference-Iterator,Iterator>> link:{java8-javadoc}/java/lang/Iterable.html#iterator%2D%2D[iterator]()++ (link:{java9-javadoc}/java/lang/Iterable.html#iterator%2D%2D[java 9])
+* ++[[painless-api-reference-Iterable-join-1]]<<painless-api-reference-String,String>> link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#join%2Djava.lang.Iterable%2Djava.lang.String%2D[join](<<painless-api-reference-String,String>>)++
+* ++[[painless-api-reference-Iterable-spliterator-0]]<<painless-api-reference-Spliterator,Spliterator>> link:{java8-javadoc}/java/lang/Iterable.html#spliterator%2D%2D[spliterator]()++ (link:{java9-javadoc}/java/lang/Iterable.html#spliterator%2D%2D[java 9])
+* ++[[painless-api-reference-Iterable-sum-0]]double link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#sum%2Djava.lang.Iterable%2D[sum]()++
+* ++[[painless-api-reference-Iterable-sum-1]]double link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#sum%2Djava.lang.Iterable%2Djava.util.function.ToDoubleFunction%2D[sum](<<painless-api-reference-ToDoubleFunction,ToDoubleFunction>>)++
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Iterator.asciidoc b/docs/painless/painless-api-reference/Iterator.asciidoc
new file mode 100644
index 0000000000000..9dd3f5601d54b
--- /dev/null
+++ b/docs/painless/painless-api-reference/Iterator.asciidoc
@@ -0,0 +1,11 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Iterator]]++Iterator++::
+* ++[[painless-api-reference-Iterator-forEachRemaining-1]]void link:{java8-javadoc}/java/util/Iterator.html#forEachRemaining%2Djava.util.function.Consumer%2D[forEachRemaining](<<painless-api-reference-Consumer,Consumer>>)++ (link:{java9-javadoc}/java/util/Iterator.html#forEachRemaining%2Djava.util.function.Consumer%2D[java 9])
+* ++[[painless-api-reference-Iterator-hasNext-0]]boolean link:{java8-javadoc}/java/util/Iterator.html#hasNext%2D%2D[hasNext]()++ (link:{java9-javadoc}/java/util/Iterator.html#hasNext%2D%2D[java 9])
+* ++[[painless-api-reference-Iterator-next-0]]def link:{java8-javadoc}/java/util/Iterator.html#next%2D%2D[next]()++ (link:{java9-javadoc}/java/util/Iterator.html#next%2D%2D[java 9])
+* ++[[painless-api-reference-Iterator-remove-0]]void link:{java8-javadoc}/java/util/Iterator.html#remove%2D%2D[remove]()++ (link:{java9-javadoc}/java/util/Iterator.html#remove%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/JapaneseChronology.asciidoc b/docs/painless/painless-api-reference/JapaneseChronology.asciidoc
new file mode 100644
index 0000000000000..166f3ca176d09
--- /dev/null
+++ b/docs/painless/painless-api-reference/JapaneseChronology.asciidoc
@@ -0,0 +1,16 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-JapaneseChronology]]++JapaneseChronology++::
+** [[painless-api-reference-JapaneseChronology-INSTANCE]]static <<painless-api-reference-JapaneseChronology,JapaneseChronology>> link:{java8-javadoc}/java/time/chrono/JapaneseChronology.html#INSTANCE[INSTANCE] (link:{java9-javadoc}/java/time/chrono/JapaneseChronology.html#INSTANCE[java 9])
+* ++[[painless-api-reference-JapaneseChronology-date-1]]<<painless-api-reference-JapaneseDate,JapaneseDate>> link:{java8-javadoc}/java/time/chrono/JapaneseChronology.html#date%2Djava.time.temporal.TemporalAccessor%2D[date](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/chrono/JapaneseChronology.html#date%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-JapaneseChronology-date-3]]<<painless-api-reference-JapaneseDate,JapaneseDate>> link:{java8-javadoc}/java/time/chrono/JapaneseChronology.html#date%2Dint%2Dint%2Dint%2D[date](int, int, int)++ (link:{java9-javadoc}/java/time/chrono/JapaneseChronology.html#date%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-JapaneseChronology-date-4]]<<painless-api-reference-JapaneseDate,JapaneseDate>> link:{java8-javadoc}/java/time/chrono/JapaneseChronology.html#date%2Djava.time.chrono.Era%2Dint%2Dint%2Dint%2D[date](<<painless-api-reference-Era,Era>>, int, int, int)++ (link:{java9-javadoc}/java/time/chrono/JapaneseChronology.html#date%2Djava.time.chrono.Era%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-JapaneseChronology-dateEpochDay-1]]<<painless-api-reference-JapaneseDate,JapaneseDate>> link:{java8-javadoc}/java/time/chrono/JapaneseChronology.html#dateEpochDay%2Dlong%2D[dateEpochDay](long)++ (link:{java9-javadoc}/java/time/chrono/JapaneseChronology.html#dateEpochDay%2Dlong%2D[java 9])
+* ++[[painless-api-reference-JapaneseChronology-dateYearDay-2]]<<painless-api-reference-JapaneseDate,JapaneseDate>> link:{java8-javadoc}/java/time/chrono/JapaneseChronology.html#dateYearDay%2Dint%2Dint%2D[dateYearDay](int, int)++ (link:{java9-javadoc}/java/time/chrono/JapaneseChronology.html#dateYearDay%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-JapaneseChronology-dateYearDay-3]]<<painless-api-reference-JapaneseDate,JapaneseDate>> link:{java8-javadoc}/java/time/chrono/JapaneseChronology.html#dateYearDay%2Djava.time.chrono.Era%2Dint%2Dint%2D[dateYearDay](<<painless-api-reference-Era,Era>>, int, int)++ (link:{java9-javadoc}/java/time/chrono/JapaneseChronology.html#dateYearDay%2Djava.time.chrono.Era%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-JapaneseChronology-eraOf-1]]<<painless-api-reference-JapaneseEra,JapaneseEra>> link:{java8-javadoc}/java/time/chrono/JapaneseChronology.html#eraOf%2Dint%2D[eraOf](int)++ (link:{java9-javadoc}/java/time/chrono/JapaneseChronology.html#eraOf%2Dint%2D[java 9])
+* ++[[painless-api-reference-JapaneseChronology-resolveDate-2]]<<painless-api-reference-JapaneseDate,JapaneseDate>> link:{java8-javadoc}/java/time/chrono/JapaneseChronology.html#resolveDate%2Djava.util.Map%2Djava.time.format.ResolverStyle%2D[resolveDate](<<painless-api-reference-Map,Map>>, <<painless-api-reference-ResolverStyle,ResolverStyle>>)++ (link:{java9-javadoc}/java/time/chrono/JapaneseChronology.html#resolveDate%2Djava.util.Map%2Djava.time.format.ResolverStyle%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Chronology,Chronology>>++
diff --git a/docs/painless/painless-api-reference/JapaneseDate.asciidoc b/docs/painless/painless-api-reference/JapaneseDate.asciidoc
new file mode 100644
index 0000000000000..560ebc7a664bc
--- /dev/null
+++ b/docs/painless/painless-api-reference/JapaneseDate.asciidoc
@@ -0,0 +1,17 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-JapaneseDate]]++JapaneseDate++::
+* ++[[painless-api-reference-JapaneseDate-from-1]]static <<painless-api-reference-JapaneseDate,JapaneseDate>> link:{java8-javadoc}/java/time/chrono/JapaneseDate.html#from%2Djava.time.temporal.TemporalAccessor%2D[from](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/chrono/JapaneseDate.html#from%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-JapaneseDate-of-3]]static <<painless-api-reference-JapaneseDate,JapaneseDate>> link:{java8-javadoc}/java/time/chrono/JapaneseDate.html#of%2Dint%2Dint%2Dint%2D[of](int, int, int)++ (link:{java9-javadoc}/java/time/chrono/JapaneseDate.html#of%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-JapaneseDate-getChronology-0]]<<painless-api-reference-JapaneseChronology,JapaneseChronology>> link:{java8-javadoc}/java/time/chrono/JapaneseDate.html#getChronology%2D%2D[getChronology]()++ (link:{java9-javadoc}/java/time/chrono/JapaneseDate.html#getChronology%2D%2D[java 9])
+* ++[[painless-api-reference-JapaneseDate-getEra-0]]<<painless-api-reference-JapaneseEra,JapaneseEra>> link:{java8-javadoc}/java/time/chrono/JapaneseDate.html#getEra%2D%2D[getEra]()++ (link:{java9-javadoc}/java/time/chrono/JapaneseDate.html#getEra%2D%2D[java 9])
+* ++[[painless-api-reference-JapaneseDate-minus-1]]<<painless-api-reference-JapaneseDate,JapaneseDate>> link:{java8-javadoc}/java/time/chrono/JapaneseDate.html#minus%2Djava.time.temporal.TemporalAmount%2D[minus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/chrono/JapaneseDate.html#minus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-JapaneseDate-minus-2]]<<painless-api-reference-JapaneseDate,JapaneseDate>> link:{java8-javadoc}/java/time/chrono/JapaneseDate.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[minus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/chrono/JapaneseDate.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-JapaneseDate-plus-1]]<<painless-api-reference-JapaneseDate,JapaneseDate>> link:{java8-javadoc}/java/time/chrono/JapaneseDate.html#plus%2Djava.time.temporal.TemporalAmount%2D[plus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/chrono/JapaneseDate.html#plus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-JapaneseDate-plus-2]]<<painless-api-reference-JapaneseDate,JapaneseDate>> link:{java8-javadoc}/java/time/chrono/JapaneseDate.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[plus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/chrono/JapaneseDate.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-JapaneseDate-with-1]]<<painless-api-reference-JapaneseDate,JapaneseDate>> link:{java8-javadoc}/java/time/chrono/JapaneseDate.html#with%2Djava.time.temporal.TemporalAdjuster%2D[with](<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>)++ (link:{java9-javadoc}/java/time/chrono/JapaneseDate.html#with%2Djava.time.temporal.TemporalAdjuster%2D[java 9])
+* ++[[painless-api-reference-JapaneseDate-with-2]]<<painless-api-reference-JapaneseDate,JapaneseDate>> link:{java8-javadoc}/java/time/chrono/JapaneseDate.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[with](<<painless-api-reference-TemporalField,TemporalField>>, long)++ (link:{java9-javadoc}/java/time/chrono/JapaneseDate.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>>++, ++<<painless-api-reference-Temporal,Temporal>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++, ++<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>++
diff --git a/docs/painless/painless-api-reference/JapaneseEra.asciidoc b/docs/painless/painless-api-reference/JapaneseEra.asciidoc
new file mode 100644
index 0000000000000..b5a9c7f63bc04
--- /dev/null
+++ b/docs/painless/painless-api-reference/JapaneseEra.asciidoc
@@ -0,0 +1,15 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-JapaneseEra]]++JapaneseEra++::
+** [[painless-api-reference-JapaneseEra-HEISEI]]static <<painless-api-reference-JapaneseEra,JapaneseEra>> link:{java8-javadoc}/java/time/chrono/JapaneseEra.html#HEISEI[HEISEI] (link:{java9-javadoc}/java/time/chrono/JapaneseEra.html#HEISEI[java 9])
+** [[painless-api-reference-JapaneseEra-MEIJI]]static <<painless-api-reference-JapaneseEra,JapaneseEra>> link:{java8-javadoc}/java/time/chrono/JapaneseEra.html#MEIJI[MEIJI] (link:{java9-javadoc}/java/time/chrono/JapaneseEra.html#MEIJI[java 9])
+** [[painless-api-reference-JapaneseEra-SHOWA]]static <<painless-api-reference-JapaneseEra,JapaneseEra>> link:{java8-javadoc}/java/time/chrono/JapaneseEra.html#SHOWA[SHOWA] (link:{java9-javadoc}/java/time/chrono/JapaneseEra.html#SHOWA[java 9])
+** [[painless-api-reference-JapaneseEra-TAISHO]]static <<painless-api-reference-JapaneseEra,JapaneseEra>> link:{java8-javadoc}/java/time/chrono/JapaneseEra.html#TAISHO[TAISHO] (link:{java9-javadoc}/java/time/chrono/JapaneseEra.html#TAISHO[java 9])
+* ++[[painless-api-reference-JapaneseEra-of-1]]static <<painless-api-reference-JapaneseEra,JapaneseEra>> link:{java8-javadoc}/java/time/chrono/JapaneseEra.html#of%2Dint%2D[of](int)++ (link:{java9-javadoc}/java/time/chrono/JapaneseEra.html#of%2Dint%2D[java 9])
+* ++[[painless-api-reference-JapaneseEra-valueOf-1]]static <<painless-api-reference-JapaneseEra,JapaneseEra>> link:{java8-javadoc}/java/time/chrono/JapaneseEra.html#valueOf%2Djava.lang.String%2D[valueOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/chrono/JapaneseEra.html#valueOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-JapaneseEra-values-0]]static <<painless-api-reference-JapaneseEra,JapaneseEra>>[] link:{java8-javadoc}/java/time/chrono/JapaneseEra.html#values%2D%2D[values]()++ (link:{java9-javadoc}/java/time/chrono/JapaneseEra.html#values%2D%2D[java 9])
+* ++[[painless-api-reference-JapaneseEra-getValue-0]]int link:{java8-javadoc}/java/time/chrono/JapaneseEra.html#getValue%2D%2D[getValue]()++ (link:{java9-javadoc}/java/time/chrono/JapaneseEra.html#getValue%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Era,Era>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++, ++<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>++
diff --git a/docs/painless/painless-api-reference/JulianFields.asciidoc b/docs/painless/painless-api-reference/JulianFields.asciidoc
new file mode 100644
index 0000000000000..19ee6c5e2ebd6
--- /dev/null
+++ b/docs/painless/painless-api-reference/JulianFields.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-JulianFields]]++JulianFields++::
+** [[painless-api-reference-JulianFields-JULIAN_DAY]]static <<painless-api-reference-TemporalField,TemporalField>> link:{java8-javadoc}/java/time/temporal/JulianFields.html#JULIAN_DAY[JULIAN_DAY] (link:{java9-javadoc}/java/time/temporal/JulianFields.html#JULIAN_DAY[java 9])
+** [[painless-api-reference-JulianFields-MODIFIED_JULIAN_DAY]]static <<painless-api-reference-TemporalField,TemporalField>> link:{java8-javadoc}/java/time/temporal/JulianFields.html#MODIFIED_JULIAN_DAY[MODIFIED_JULIAN_DAY] (link:{java9-javadoc}/java/time/temporal/JulianFields.html#MODIFIED_JULIAN_DAY[java 9])
+** [[painless-api-reference-JulianFields-RATA_DIE]]static <<painless-api-reference-TemporalField,TemporalField>> link:{java8-javadoc}/java/time/temporal/JulianFields.html#RATA_DIE[RATA_DIE] (link:{java9-javadoc}/java/time/temporal/JulianFields.html#RATA_DIE[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/LinkedHashMap.asciidoc b/docs/painless/painless-api-reference/LinkedHashMap.asciidoc
new file mode 100644
index 0000000000000..bda2d45f22cb6
--- /dev/null
+++ b/docs/painless/painless-api-reference/LinkedHashMap.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-LinkedHashMap]]++LinkedHashMap++::
+* ++[[painless-api-reference-LinkedHashMap-LinkedHashMap-0]]link:{java8-javadoc}/java/util/LinkedHashMap.html#LinkedHashMap%2D%2D[LinkedHashMap]()++ (link:{java9-javadoc}/java/util/LinkedHashMap.html#LinkedHashMap%2D%2D[java 9])
+* ++[[painless-api-reference-LinkedHashMap-LinkedHashMap-1]]link:{java8-javadoc}/java/util/LinkedHashMap.html#LinkedHashMap%2Djava.util.Map%2D[LinkedHashMap](<<painless-api-reference-Map,Map>>)++ (link:{java9-javadoc}/java/util/LinkedHashMap.html#LinkedHashMap%2Djava.util.Map%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-HashMap,HashMap>>++, ++<<painless-api-reference-Map,Map>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/LinkedHashSet.asciidoc b/docs/painless/painless-api-reference/LinkedHashSet.asciidoc
new file mode 100644
index 0000000000000..3a7710771e8ba
--- /dev/null
+++ b/docs/painless/painless-api-reference/LinkedHashSet.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-LinkedHashSet]]++LinkedHashSet++::
+* ++[[painless-api-reference-LinkedHashSet-LinkedHashSet-0]]link:{java8-javadoc}/java/util/LinkedHashSet.html#LinkedHashSet%2D%2D[LinkedHashSet]()++ (link:{java9-javadoc}/java/util/LinkedHashSet.html#LinkedHashSet%2D%2D[java 9])
+* ++[[painless-api-reference-LinkedHashSet-LinkedHashSet-1]]link:{java8-javadoc}/java/util/LinkedHashSet.html#LinkedHashSet%2Djava.util.Collection%2D[LinkedHashSet](<<painless-api-reference-Collection,Collection>>)++ (link:{java9-javadoc}/java/util/LinkedHashSet.html#LinkedHashSet%2Djava.util.Collection%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-HashSet,HashSet>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-Set,Set>>++
diff --git a/docs/painless/painless-api-reference/LinkedList.asciidoc b/docs/painless/painless-api-reference/LinkedList.asciidoc
new file mode 100644
index 0000000000000..5d18b373ba360
--- /dev/null
+++ b/docs/painless/painless-api-reference/LinkedList.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-LinkedList]]++LinkedList++::
+* ++[[painless-api-reference-LinkedList-LinkedList-0]]link:{java8-javadoc}/java/util/LinkedList.html#LinkedList%2D%2D[LinkedList]()++ (link:{java9-javadoc}/java/util/LinkedList.html#LinkedList%2D%2D[java 9])
+* ++[[painless-api-reference-LinkedList-LinkedList-1]]link:{java8-javadoc}/java/util/LinkedList.html#LinkedList%2Djava.util.Collection%2D[LinkedList](<<painless-api-reference-Collection,Collection>>)++ (link:{java9-javadoc}/java/util/LinkedList.html#LinkedList%2Djava.util.Collection%2D[java 9])
+* ++[[painless-api-reference-LinkedList-clone-0]]def link:{java8-javadoc}/java/util/LinkedList.html#clone%2D%2D[clone]()++ (link:{java9-javadoc}/java/util/LinkedList.html#clone%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Deque,Deque>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-List,List>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-Queue,Queue>>++
diff --git a/docs/painless/painless-api-reference/List.asciidoc b/docs/painless/painless-api-reference/List.asciidoc
new file mode 100644
index 0000000000000..abe438451b86b
--- /dev/null
+++ b/docs/painless/painless-api-reference/List.asciidoc
@@ -0,0 +1,22 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-List]]++List++::
+* ++[[painless-api-reference-List-add-2]]void link:{java8-javadoc}/java/util/List.html#add%2Dint%2Djava.lang.Object%2D[add](int, def)++ (link:{java9-javadoc}/java/util/List.html#add%2Dint%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-List-addAll-2]]boolean link:{java8-javadoc}/java/util/List.html#addAll%2Dint%2Djava.util.Collection%2D[addAll](int, <<painless-api-reference-Collection,Collection>>)++ (link:{java9-javadoc}/java/util/List.html#addAll%2Dint%2Djava.util.Collection%2D[java 9])
+* ++[[painless-api-reference-List-equals-1]]boolean link:{java8-javadoc}/java/util/List.html#equals%2Djava.lang.Object%2D[equals](<<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/util/List.html#equals%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-List-get-1]]def link:{java8-javadoc}/java/util/List.html#get%2Dint%2D[get](int)++ (link:{java9-javadoc}/java/util/List.html#get%2Dint%2D[java 9])
+* ++[[painless-api-reference-List-getLength-0]]int link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#getLength%2Djava.util.List%2D[getLength]()++
+* ++[[painless-api-reference-List-hashCode-0]]int link:{java8-javadoc}/java/util/List.html#hashCode%2D%2D[hashCode]()++ (link:{java9-javadoc}/java/util/List.html#hashCode%2D%2D[java 9])
+* ++[[painless-api-reference-List-indexOf-1]]int link:{java8-javadoc}/java/util/List.html#indexOf%2Djava.lang.Object%2D[indexOf](def)++ (link:{java9-javadoc}/java/util/List.html#indexOf%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-List-lastIndexOf-1]]int link:{java8-javadoc}/java/util/List.html#lastIndexOf%2Djava.lang.Object%2D[lastIndexOf](def)++ (link:{java9-javadoc}/java/util/List.html#lastIndexOf%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-List-listIterator-0]]<<painless-api-reference-ListIterator,ListIterator>> link:{java8-javadoc}/java/util/List.html#listIterator%2D%2D[listIterator]()++ (link:{java9-javadoc}/java/util/List.html#listIterator%2D%2D[java 9])
+* ++[[painless-api-reference-List-listIterator-1]]<<painless-api-reference-ListIterator,ListIterator>> link:{java8-javadoc}/java/util/List.html#listIterator%2Dint%2D[listIterator](int)++ (link:{java9-javadoc}/java/util/List.html#listIterator%2Dint%2D[java 9])
+* ++[[painless-api-reference-List-remove-1]]def link:{java8-javadoc}/java/util/List.html#remove%2Dint%2D[remove](int)++ (link:{java9-javadoc}/java/util/List.html#remove%2Dint%2D[java 9])
+* ++[[painless-api-reference-List-replaceAll-1]]void link:{java8-javadoc}/java/util/List.html#replaceAll%2Djava.util.function.UnaryOperator%2D[replaceAll](<<painless-api-reference-UnaryOperator,UnaryOperator>>)++ (link:{java9-javadoc}/java/util/List.html#replaceAll%2Djava.util.function.UnaryOperator%2D[java 9])
+* ++[[painless-api-reference-List-set-2]]def link:{java8-javadoc}/java/util/List.html#set%2Dint%2Djava.lang.Object%2D[set](int, def)++ (link:{java9-javadoc}/java/util/List.html#set%2Dint%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-List-sort-1]]void link:{java8-javadoc}/java/util/List.html#sort%2Djava.util.Comparator%2D[sort](<<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/List.html#sort%2Djava.util.Comparator%2D[java 9])
+* ++[[painless-api-reference-List-subList-2]]<<painless-api-reference-List,List>> link:{java8-javadoc}/java/util/List.html#subList%2Dint%2Dint%2D[subList](int, int)++ (link:{java9-javadoc}/java/util/List.html#subList%2Dint%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/ListIterator.asciidoc b/docs/painless/painless-api-reference/ListIterator.asciidoc
new file mode 100644
index 0000000000000..37d04b2a755cf
--- /dev/null
+++ b/docs/painless/painless-api-reference/ListIterator.asciidoc
@@ -0,0 +1,12 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ListIterator]]++ListIterator++::
+* ++[[painless-api-reference-ListIterator-add-1]]void link:{java8-javadoc}/java/util/ListIterator.html#add%2Djava.lang.Object%2D[add](def)++ (link:{java9-javadoc}/java/util/ListIterator.html#add%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-ListIterator-hasPrevious-0]]boolean link:{java8-javadoc}/java/util/ListIterator.html#hasPrevious%2D%2D[hasPrevious]()++ (link:{java9-javadoc}/java/util/ListIterator.html#hasPrevious%2D%2D[java 9])
+* ++[[painless-api-reference-ListIterator-nextIndex-0]]int link:{java8-javadoc}/java/util/ListIterator.html#nextIndex%2D%2D[nextIndex]()++ (link:{java9-javadoc}/java/util/ListIterator.html#nextIndex%2D%2D[java 9])
+* ++[[painless-api-reference-ListIterator-previousIndex-0]]int link:{java8-javadoc}/java/util/ListIterator.html#previousIndex%2D%2D[previousIndex]()++ (link:{java9-javadoc}/java/util/ListIterator.html#previousIndex%2D%2D[java 9])
+* ++[[painless-api-reference-ListIterator-set-1]]void link:{java8-javadoc}/java/util/ListIterator.html#set%2Djava.lang.Object%2D[set](def)++ (link:{java9-javadoc}/java/util/ListIterator.html#set%2Djava.lang.Object%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Iterator,Iterator>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/LocalDate.asciidoc b/docs/painless/painless-api-reference/LocalDate.asciidoc
new file mode 100644
index 0000000000000..8498474d02a0f
--- /dev/null
+++ b/docs/painless/painless-api-reference/LocalDate.asciidoc
@@ -0,0 +1,47 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-LocalDate]]++LocalDate++::
+** [[painless-api-reference-LocalDate-MAX]]static <<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#MAX[MAX] (link:{java9-javadoc}/java/time/LocalDate.html#MAX[java 9])
+** [[painless-api-reference-LocalDate-MIN]]static <<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#MIN[MIN] (link:{java9-javadoc}/java/time/LocalDate.html#MIN[java 9])
+* ++[[painless-api-reference-LocalDate-from-1]]static <<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#from%2Djava.time.temporal.TemporalAccessor%2D[from](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/LocalDate.html#from%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-LocalDate-of-3]]static <<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#of%2Dint%2Dint%2Dint%2D[of](int, int, int)++ (link:{java9-javadoc}/java/time/LocalDate.html#of%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-LocalDate-ofEpochDay-1]]static <<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#ofEpochDay%2Dlong%2D[ofEpochDay](long)++ (link:{java9-javadoc}/java/time/LocalDate.html#ofEpochDay%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDate-ofYearDay-2]]static <<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#ofYearDay%2Dint%2Dint%2D[ofYearDay](int, int)++ (link:{java9-javadoc}/java/time/LocalDate.html#ofYearDay%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-LocalDate-parse-1]]static <<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#parse%2Djava.lang.CharSequence%2D[parse](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/time/LocalDate.html#parse%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-LocalDate-parse-2]]static <<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#parse%2Djava.lang.CharSequence%2Djava.time.format.DateTimeFormatter%2D[parse](<<painless-api-reference-CharSequence,CharSequence>>, <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>>)++ (link:{java9-javadoc}/java/time/LocalDate.html#parse%2Djava.lang.CharSequence%2Djava.time.format.DateTimeFormatter%2D[java 9])
+* ++[[painless-api-reference-LocalDate-atStartOfDay-0]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDate.html#atStartOfDay%2D%2D[atStartOfDay]()++ (link:{java9-javadoc}/java/time/LocalDate.html#atStartOfDay%2D%2D[java 9])
+* ++[[painless-api-reference-LocalDate-atStartOfDay-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/LocalDate.html#atStartOfDay%2Djava.time.ZoneId%2D[atStartOfDay](<<painless-api-reference-ZoneId,ZoneId>>)++ (link:{java9-javadoc}/java/time/LocalDate.html#atStartOfDay%2Djava.time.ZoneId%2D[java 9])
+* ++[[painless-api-reference-LocalDate-atTime-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDate.html#atTime%2Djava.time.LocalTime%2D[atTime](<<painless-api-reference-LocalTime,LocalTime>>)++ (link:{java9-javadoc}/java/time/LocalDate.html#atTime%2Djava.time.LocalTime%2D[java 9])
+* ++[[painless-api-reference-LocalDate-atTime-2]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDate.html#atTime%2Dint%2Dint%2D[atTime](int, int)++ (link:{java9-javadoc}/java/time/LocalDate.html#atTime%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-LocalDate-atTime-3]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDate.html#atTime%2Dint%2Dint%2Dint%2D[atTime](int, int, int)++ (link:{java9-javadoc}/java/time/LocalDate.html#atTime%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-LocalDate-atTime-4]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDate.html#atTime%2Dint%2Dint%2Dint%2Dint%2D[atTime](int, int, int, int)++ (link:{java9-javadoc}/java/time/LocalDate.html#atTime%2Dint%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-LocalDate-getChronology-0]]<<painless-api-reference-IsoChronology,IsoChronology>> link:{java8-javadoc}/java/time/LocalDate.html#getChronology%2D%2D[getChronology]()++ (link:{java9-javadoc}/java/time/LocalDate.html#getChronology%2D%2D[java 9])
+* ++[[painless-api-reference-LocalDate-getDayOfMonth-0]]int link:{java8-javadoc}/java/time/LocalDate.html#getDayOfMonth%2D%2D[getDayOfMonth]()++ (link:{java9-javadoc}/java/time/LocalDate.html#getDayOfMonth%2D%2D[java 9])
+* ++[[painless-api-reference-LocalDate-getDayOfWeek-0]]<<painless-api-reference-DayOfWeek,DayOfWeek>> link:{java8-javadoc}/java/time/LocalDate.html#getDayOfWeek%2D%2D[getDayOfWeek]()++ (link:{java9-javadoc}/java/time/LocalDate.html#getDayOfWeek%2D%2D[java 9])
+* ++[[painless-api-reference-LocalDate-getDayOfYear-0]]int link:{java8-javadoc}/java/time/LocalDate.html#getDayOfYear%2D%2D[getDayOfYear]()++ (link:{java9-javadoc}/java/time/LocalDate.html#getDayOfYear%2D%2D[java 9])
+* ++[[painless-api-reference-LocalDate-getMonth-0]]<<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/LocalDate.html#getMonth%2D%2D[getMonth]()++ (link:{java9-javadoc}/java/time/LocalDate.html#getMonth%2D%2D[java 9])
+* ++[[painless-api-reference-LocalDate-getMonthValue-0]]int link:{java8-javadoc}/java/time/LocalDate.html#getMonthValue%2D%2D[getMonthValue]()++ (link:{java9-javadoc}/java/time/LocalDate.html#getMonthValue%2D%2D[java 9])
+* ++[[painless-api-reference-LocalDate-getYear-0]]int link:{java8-javadoc}/java/time/LocalDate.html#getYear%2D%2D[getYear]()++ (link:{java9-javadoc}/java/time/LocalDate.html#getYear%2D%2D[java 9])
+* ++[[painless-api-reference-LocalDate-minus-1]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#minus%2Djava.time.temporal.TemporalAmount%2D[minus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/LocalDate.html#minus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-LocalDate-minus-2]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[minus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/LocalDate.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-LocalDate-minusDays-1]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#minusDays%2Dlong%2D[minusDays](long)++ (link:{java9-javadoc}/java/time/LocalDate.html#minusDays%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDate-minusMonths-1]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#minusMonths%2Dlong%2D[minusMonths](long)++ (link:{java9-javadoc}/java/time/LocalDate.html#minusMonths%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDate-minusWeeks-1]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#minusWeeks%2Dlong%2D[minusWeeks](long)++ (link:{java9-javadoc}/java/time/LocalDate.html#minusWeeks%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDate-minusYears-1]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#minusYears%2Dlong%2D[minusYears](long)++ (link:{java9-javadoc}/java/time/LocalDate.html#minusYears%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDate-plus-1]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#plus%2Djava.time.temporal.TemporalAmount%2D[plus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/LocalDate.html#plus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-LocalDate-plus-2]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[plus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/LocalDate.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-LocalDate-plusDays-1]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#plusDays%2Dlong%2D[plusDays](long)++ (link:{java9-javadoc}/java/time/LocalDate.html#plusDays%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDate-plusMonths-1]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#plusMonths%2Dlong%2D[plusMonths](long)++ (link:{java9-javadoc}/java/time/LocalDate.html#plusMonths%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDate-plusWeeks-1]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#plusWeeks%2Dlong%2D[plusWeeks](long)++ (link:{java9-javadoc}/java/time/LocalDate.html#plusWeeks%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDate-plusYears-1]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#plusYears%2Dlong%2D[plusYears](long)++ (link:{java9-javadoc}/java/time/LocalDate.html#plusYears%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDate-until-1]]<<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/LocalDate.html#until%2Djava.time.chrono.ChronoLocalDate%2D[until](<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>>)++ (link:{java9-javadoc}/java/time/LocalDate.html#until%2Djava.time.chrono.ChronoLocalDate%2D[java 9])
+* ++[[painless-api-reference-LocalDate-with-1]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#with%2Djava.time.temporal.TemporalAdjuster%2D[with](<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>)++ (link:{java9-javadoc}/java/time/LocalDate.html#with%2Djava.time.temporal.TemporalAdjuster%2D[java 9])
+* ++[[painless-api-reference-LocalDate-with-2]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[with](<<painless-api-reference-TemporalField,TemporalField>>, long)++ (link:{java9-javadoc}/java/time/LocalDate.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDate-withDayOfMonth-1]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#withDayOfMonth%2Dint%2D[withDayOfMonth](int)++ (link:{java9-javadoc}/java/time/LocalDate.html#withDayOfMonth%2Dint%2D[java 9])
+* ++[[painless-api-reference-LocalDate-withDayOfYear-1]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#withDayOfYear%2Dint%2D[withDayOfYear](int)++ (link:{java9-javadoc}/java/time/LocalDate.html#withDayOfYear%2Dint%2D[java 9])
+* ++[[painless-api-reference-LocalDate-withMonth-1]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#withMonth%2Dint%2D[withMonth](int)++ (link:{java9-javadoc}/java/time/LocalDate.html#withMonth%2Dint%2D[java 9])
+* ++[[painless-api-reference-LocalDate-withYear-1]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#withYear%2Dint%2D[withYear](int)++ (link:{java9-javadoc}/java/time/LocalDate.html#withYear%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>>++, ++<<painless-api-reference-Temporal,Temporal>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++, ++<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>++
diff --git a/docs/painless/painless-api-reference/LocalDateTime.asciidoc b/docs/painless/painless-api-reference/LocalDateTime.asciidoc
new file mode 100644
index 0000000000000..9507ff1dd5231
--- /dev/null
+++ b/docs/painless/painless-api-reference/LocalDateTime.asciidoc
@@ -0,0 +1,61 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-LocalDateTime]]++LocalDateTime++::
+** [[painless-api-reference-LocalDateTime-MAX]]static <<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#MAX[MAX] (link:{java9-javadoc}/java/time/LocalDateTime.html#MAX[java 9])
+** [[painless-api-reference-LocalDateTime-MIN]]static <<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#MIN[MIN] (link:{java9-javadoc}/java/time/LocalDateTime.html#MIN[java 9])
+* ++[[painless-api-reference-LocalDateTime-from-1]]static <<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#from%2Djava.time.temporal.TemporalAccessor%2D[from](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#from%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-of-2]]static <<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#of%2Djava.time.LocalDate%2Djava.time.LocalTime%2D[of](<<painless-api-reference-LocalDate,LocalDate>>, <<painless-api-reference-LocalTime,LocalTime>>)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#of%2Djava.time.LocalDate%2Djava.time.LocalTime%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-of-5]]static <<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#of%2Dint%2Dint%2Dint%2Dint%2Dint%2D[of](int, int, int, int, int)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#of%2Dint%2Dint%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-of-6]]static <<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#of%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2D[of](int, int, int, int, int, int)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#of%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-of-7]]static <<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#of%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2D[of](int, int, int, int, int, int, int)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#of%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-ofEpochSecond-3]]static <<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#ofEpochSecond%2Dlong%2Dint%2Djava.time.ZoneOffset%2D[ofEpochSecond](long, int, <<painless-api-reference-ZoneOffset,ZoneOffset>>)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#ofEpochSecond%2Dlong%2Dint%2Djava.time.ZoneOffset%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-ofInstant-2]]static <<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#ofInstant%2Djava.time.Instant%2Djava.time.ZoneId%2D[ofInstant](<<painless-api-reference-Instant,Instant>>, <<painless-api-reference-ZoneId,ZoneId>>)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#ofInstant%2Djava.time.Instant%2Djava.time.ZoneId%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-parse-1]]static <<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#parse%2Djava.lang.CharSequence%2D[parse](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#parse%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-parse-2]]static <<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#parse%2Djava.lang.CharSequence%2Djava.time.format.DateTimeFormatter%2D[parse](<<painless-api-reference-CharSequence,CharSequence>>, <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>>)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#parse%2Djava.lang.CharSequence%2Djava.time.format.DateTimeFormatter%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-atOffset-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#atOffset%2Djava.time.ZoneOffset%2D[atOffset](<<painless-api-reference-ZoneOffset,ZoneOffset>>)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#atOffset%2Djava.time.ZoneOffset%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-atZone-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#atZone%2Djava.time.ZoneId%2D[atZone](<<painless-api-reference-ZoneId,ZoneId>>)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#atZone%2Djava.time.ZoneId%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-getDayOfMonth-0]]int link:{java8-javadoc}/java/time/LocalDateTime.html#getDayOfMonth%2D%2D[getDayOfMonth]()++ (link:{java9-javadoc}/java/time/LocalDateTime.html#getDayOfMonth%2D%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-getDayOfWeek-0]]<<painless-api-reference-DayOfWeek,DayOfWeek>> link:{java8-javadoc}/java/time/LocalDateTime.html#getDayOfWeek%2D%2D[getDayOfWeek]()++ (link:{java9-javadoc}/java/time/LocalDateTime.html#getDayOfWeek%2D%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-getDayOfYear-0]]int link:{java8-javadoc}/java/time/LocalDateTime.html#getDayOfYear%2D%2D[getDayOfYear]()++ (link:{java9-javadoc}/java/time/LocalDateTime.html#getDayOfYear%2D%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-getHour-0]]int link:{java8-javadoc}/java/time/LocalDateTime.html#getHour%2D%2D[getHour]()++ (link:{java9-javadoc}/java/time/LocalDateTime.html#getHour%2D%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-getMinute-0]]int link:{java8-javadoc}/java/time/LocalDateTime.html#getMinute%2D%2D[getMinute]()++ (link:{java9-javadoc}/java/time/LocalDateTime.html#getMinute%2D%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-getMonth-0]]<<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/LocalDateTime.html#getMonth%2D%2D[getMonth]()++ (link:{java9-javadoc}/java/time/LocalDateTime.html#getMonth%2D%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-getMonthValue-0]]int link:{java8-javadoc}/java/time/LocalDateTime.html#getMonthValue%2D%2D[getMonthValue]()++ (link:{java9-javadoc}/java/time/LocalDateTime.html#getMonthValue%2D%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-getNano-0]]int link:{java8-javadoc}/java/time/LocalDateTime.html#getNano%2D%2D[getNano]()++ (link:{java9-javadoc}/java/time/LocalDateTime.html#getNano%2D%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-getSecond-0]]int link:{java8-javadoc}/java/time/LocalDateTime.html#getSecond%2D%2D[getSecond]()++ (link:{java9-javadoc}/java/time/LocalDateTime.html#getSecond%2D%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-getYear-0]]int link:{java8-javadoc}/java/time/LocalDateTime.html#getYear%2D%2D[getYear]()++ (link:{java9-javadoc}/java/time/LocalDateTime.html#getYear%2D%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-minus-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#minus%2Djava.time.temporal.TemporalAmount%2D[minus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#minus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-minus-2]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[minus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-minusDays-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#minusDays%2Dlong%2D[minusDays](long)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#minusDays%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-minusHours-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#minusHours%2Dlong%2D[minusHours](long)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#minusHours%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-minusMinutes-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#minusMinutes%2Dlong%2D[minusMinutes](long)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#minusMinutes%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-minusMonths-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#minusMonths%2Dlong%2D[minusMonths](long)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#minusMonths%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-minusNanos-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#minusNanos%2Dlong%2D[minusNanos](long)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#minusNanos%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-minusSeconds-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#minusSeconds%2Dlong%2D[minusSeconds](long)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#minusSeconds%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-minusWeeks-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#minusWeeks%2Dlong%2D[minusWeeks](long)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#minusWeeks%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-minusYears-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#minusYears%2Dlong%2D[minusYears](long)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#minusYears%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-plus-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#plus%2Djava.time.temporal.TemporalAmount%2D[plus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#plus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-plus-2]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[plus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-plusDays-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#plusDays%2Dlong%2D[plusDays](long)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#plusDays%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-plusHours-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#plusHours%2Dlong%2D[plusHours](long)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#plusHours%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-plusMinutes-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#plusMinutes%2Dlong%2D[plusMinutes](long)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#plusMinutes%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-plusMonths-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#plusMonths%2Dlong%2D[plusMonths](long)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#plusMonths%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-plusNanos-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#plusNanos%2Dlong%2D[plusNanos](long)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#plusNanos%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-plusSeconds-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#plusSeconds%2Dlong%2D[plusSeconds](long)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#plusSeconds%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-plusWeeks-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#plusWeeks%2Dlong%2D[plusWeeks](long)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#plusWeeks%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-plusYears-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#plusYears%2Dlong%2D[plusYears](long)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#plusYears%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-toLocalDate-0]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDateTime.html#toLocalDate%2D%2D[toLocalDate]()++ (link:{java9-javadoc}/java/time/LocalDateTime.html#toLocalDate%2D%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-truncatedTo-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#truncatedTo%2Djava.time.temporal.TemporalUnit%2D[truncatedTo](<<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#truncatedTo%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-with-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#with%2Djava.time.temporal.TemporalAdjuster%2D[with](<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#with%2Djava.time.temporal.TemporalAdjuster%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-with-2]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[with](<<painless-api-reference-TemporalField,TemporalField>>, long)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-withDayOfMonth-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#withDayOfMonth%2Dint%2D[withDayOfMonth](int)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#withDayOfMonth%2Dint%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-withDayOfYear-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#withDayOfYear%2Dint%2D[withDayOfYear](int)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#withDayOfYear%2Dint%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-withHour-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#withHour%2Dint%2D[withHour](int)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#withHour%2Dint%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-withMinute-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#withMinute%2Dint%2D[withMinute](int)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#withMinute%2Dint%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-withMonth-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#withMonth%2Dint%2D[withMonth](int)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#withMonth%2Dint%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-withSecond-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#withSecond%2Dint%2D[withSecond](int)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#withSecond%2Dint%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-withYear-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#withYear%2Dint%2D[withYear](int)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#withYear%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-ChronoLocalDateTime,ChronoLocalDateTime>>++, ++<<painless-api-reference-Temporal,Temporal>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++, ++<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>++
diff --git a/docs/painless/painless-api-reference/LocalTime.asciidoc b/docs/painless/painless-api-reference/LocalTime.asciidoc
new file mode 100644
index 0000000000000..f12e13f4e7854
--- /dev/null
+++ b/docs/painless/painless-api-reference/LocalTime.asciidoc
@@ -0,0 +1,50 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-LocalTime]]++LocalTime++::
+** [[painless-api-reference-LocalTime-MAX]]static <<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#MAX[MAX] (link:{java9-javadoc}/java/time/LocalTime.html#MAX[java 9])
+** [[painless-api-reference-LocalTime-MIDNIGHT]]static <<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#MIDNIGHT[MIDNIGHT] (link:{java9-javadoc}/java/time/LocalTime.html#MIDNIGHT[java 9])
+** [[painless-api-reference-LocalTime-MIN]]static <<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#MIN[MIN] (link:{java9-javadoc}/java/time/LocalTime.html#MIN[java 9])
+** [[painless-api-reference-LocalTime-NOON]]static <<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#NOON[NOON] (link:{java9-javadoc}/java/time/LocalTime.html#NOON[java 9])
+* ++[[painless-api-reference-LocalTime-from-1]]static <<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#from%2Djava.time.temporal.TemporalAccessor%2D[from](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/LocalTime.html#from%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-LocalTime-of-2]]static <<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#of%2Dint%2Dint%2D[of](int, int)++ (link:{java9-javadoc}/java/time/LocalTime.html#of%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-LocalTime-of-3]]static <<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#of%2Dint%2Dint%2Dint%2D[of](int, int, int)++ (link:{java9-javadoc}/java/time/LocalTime.html#of%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-LocalTime-of-4]]static <<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#of%2Dint%2Dint%2Dint%2Dint%2D[of](int, int, int, int)++ (link:{java9-javadoc}/java/time/LocalTime.html#of%2Dint%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-LocalTime-ofNanoOfDay-1]]static <<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#ofNanoOfDay%2Dlong%2D[ofNanoOfDay](long)++ (link:{java9-javadoc}/java/time/LocalTime.html#ofNanoOfDay%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalTime-ofSecondOfDay-1]]static <<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#ofSecondOfDay%2Dlong%2D[ofSecondOfDay](long)++ (link:{java9-javadoc}/java/time/LocalTime.html#ofSecondOfDay%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalTime-parse-1]]static <<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#parse%2Djava.lang.CharSequence%2D[parse](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/time/LocalTime.html#parse%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-LocalTime-parse-2]]static <<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#parse%2Djava.lang.CharSequence%2Djava.time.format.DateTimeFormatter%2D[parse](<<painless-api-reference-CharSequence,CharSequence>>, <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>>)++ (link:{java9-javadoc}/java/time/LocalTime.html#parse%2Djava.lang.CharSequence%2Djava.time.format.DateTimeFormatter%2D[java 9])
+* ++[[painless-api-reference-LocalTime-atDate-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalTime.html#atDate%2Djava.time.LocalDate%2D[atDate](<<painless-api-reference-LocalDate,LocalDate>>)++ (link:{java9-javadoc}/java/time/LocalTime.html#atDate%2Djava.time.LocalDate%2D[java 9])
+* ++[[painless-api-reference-LocalTime-atOffset-1]]<<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/LocalTime.html#atOffset%2Djava.time.ZoneOffset%2D[atOffset](<<painless-api-reference-ZoneOffset,ZoneOffset>>)++ (link:{java9-javadoc}/java/time/LocalTime.html#atOffset%2Djava.time.ZoneOffset%2D[java 9])
+* ++[[painless-api-reference-LocalTime-compareTo-1]]int link:{java8-javadoc}/java/time/LocalTime.html#compareTo%2Djava.time.LocalTime%2D[compareTo](<<painless-api-reference-LocalTime,LocalTime>>)++ (link:{java9-javadoc}/java/time/LocalTime.html#compareTo%2Djava.time.LocalTime%2D[java 9])
+* ++[[painless-api-reference-LocalTime-format-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/LocalTime.html#format%2Djava.time.format.DateTimeFormatter%2D[format](<<painless-api-reference-DateTimeFormatter,DateTimeFormatter>>)++ (link:{java9-javadoc}/java/time/LocalTime.html#format%2Djava.time.format.DateTimeFormatter%2D[java 9])
+* ++[[painless-api-reference-LocalTime-getHour-0]]int link:{java8-javadoc}/java/time/LocalTime.html#getHour%2D%2D[getHour]()++ (link:{java9-javadoc}/java/time/LocalTime.html#getHour%2D%2D[java 9])
+* ++[[painless-api-reference-LocalTime-getMinute-0]]int link:{java8-javadoc}/java/time/LocalTime.html#getMinute%2D%2D[getMinute]()++ (link:{java9-javadoc}/java/time/LocalTime.html#getMinute%2D%2D[java 9])
+* ++[[painless-api-reference-LocalTime-getNano-0]]int link:{java8-javadoc}/java/time/LocalTime.html#getNano%2D%2D[getNano]()++ (link:{java9-javadoc}/java/time/LocalTime.html#getNano%2D%2D[java 9])
+* ++[[painless-api-reference-LocalTime-getSecond-0]]int link:{java8-javadoc}/java/time/LocalTime.html#getSecond%2D%2D[getSecond]()++ (link:{java9-javadoc}/java/time/LocalTime.html#getSecond%2D%2D[java 9])
+* ++[[painless-api-reference-LocalTime-isAfter-1]]boolean link:{java8-javadoc}/java/time/LocalTime.html#isAfter%2Djava.time.LocalTime%2D[isAfter](<<painless-api-reference-LocalTime,LocalTime>>)++ (link:{java9-javadoc}/java/time/LocalTime.html#isAfter%2Djava.time.LocalTime%2D[java 9])
+* ++[[painless-api-reference-LocalTime-isBefore-1]]boolean link:{java8-javadoc}/java/time/LocalTime.html#isBefore%2Djava.time.LocalTime%2D[isBefore](<<painless-api-reference-LocalTime,LocalTime>>)++ (link:{java9-javadoc}/java/time/LocalTime.html#isBefore%2Djava.time.LocalTime%2D[java 9])
+* ++[[painless-api-reference-LocalTime-minus-1]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#minus%2Djava.time.temporal.TemporalAmount%2D[minus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/LocalTime.html#minus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-LocalTime-minus-2]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[minus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/LocalTime.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-LocalTime-minusHours-1]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#minusHours%2Dlong%2D[minusHours](long)++ (link:{java9-javadoc}/java/time/LocalTime.html#minusHours%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalTime-minusMinutes-1]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#minusMinutes%2Dlong%2D[minusMinutes](long)++ (link:{java9-javadoc}/java/time/LocalTime.html#minusMinutes%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalTime-minusNanos-1]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#minusNanos%2Dlong%2D[minusNanos](long)++ (link:{java9-javadoc}/java/time/LocalTime.html#minusNanos%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalTime-minusSeconds-1]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#minusSeconds%2Dlong%2D[minusSeconds](long)++ (link:{java9-javadoc}/java/time/LocalTime.html#minusSeconds%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalTime-plus-1]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#plus%2Djava.time.temporal.TemporalAmount%2D[plus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/LocalTime.html#plus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-LocalTime-plus-2]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[plus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/LocalTime.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-LocalTime-plusHours-1]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#plusHours%2Dlong%2D[plusHours](long)++ (link:{java9-javadoc}/java/time/LocalTime.html#plusHours%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalTime-plusMinutes-1]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#plusMinutes%2Dlong%2D[plusMinutes](long)++ (link:{java9-javadoc}/java/time/LocalTime.html#plusMinutes%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalTime-plusNanos-1]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#plusNanos%2Dlong%2D[plusNanos](long)++ (link:{java9-javadoc}/java/time/LocalTime.html#plusNanos%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalTime-plusSeconds-1]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#plusSeconds%2Dlong%2D[plusSeconds](long)++ (link:{java9-javadoc}/java/time/LocalTime.html#plusSeconds%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalTime-toNanoOfDay-0]]long link:{java8-javadoc}/java/time/LocalTime.html#toNanoOfDay%2D%2D[toNanoOfDay]()++ (link:{java9-javadoc}/java/time/LocalTime.html#toNanoOfDay%2D%2D[java 9])
+* ++[[painless-api-reference-LocalTime-toSecondOfDay-0]]int link:{java8-javadoc}/java/time/LocalTime.html#toSecondOfDay%2D%2D[toSecondOfDay]()++ (link:{java9-javadoc}/java/time/LocalTime.html#toSecondOfDay%2D%2D[java 9])
+* ++[[painless-api-reference-LocalTime-truncatedTo-1]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#truncatedTo%2Djava.time.temporal.TemporalUnit%2D[truncatedTo](<<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/LocalTime.html#truncatedTo%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-LocalTime-with-1]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#with%2Djava.time.temporal.TemporalAdjuster%2D[with](<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>)++ (link:{java9-javadoc}/java/time/LocalTime.html#with%2Djava.time.temporal.TemporalAdjuster%2D[java 9])
+* ++[[painless-api-reference-LocalTime-with-2]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[with](<<painless-api-reference-TemporalField,TemporalField>>, long)++ (link:{java9-javadoc}/java/time/LocalTime.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalTime-withHour-1]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#withHour%2Dint%2D[withHour](int)++ (link:{java9-javadoc}/java/time/LocalTime.html#withHour%2Dint%2D[java 9])
+* ++[[painless-api-reference-LocalTime-withMinute-1]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#withMinute%2Dint%2D[withMinute](int)++ (link:{java9-javadoc}/java/time/LocalTime.html#withMinute%2Dint%2D[java 9])
+* ++[[painless-api-reference-LocalTime-withNano-1]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#withNano%2Dint%2D[withNano](int)++ (link:{java9-javadoc}/java/time/LocalTime.html#withNano%2Dint%2D[java 9])
+* ++[[painless-api-reference-LocalTime-withSecond-1]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#withSecond%2Dint%2D[withSecond](int)++ (link:{java9-javadoc}/java/time/LocalTime.html#withSecond%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-Temporal,Temporal>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++, ++<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>++
diff --git a/docs/painless/painless-api-reference/Locale.Builder.asciidoc b/docs/painless/painless-api-reference/Locale.Builder.asciidoc
new file mode 100644
index 0000000000000..3677c81e60411
--- /dev/null
+++ b/docs/painless/painless-api-reference/Locale.Builder.asciidoc
@@ -0,0 +1,21 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Locale-Builder]]++Locale.Builder++::
+* ++[[painless-api-reference-Locale-Builder-Locale.Builder-0]]link:{java8-javadoc}/java/util/Locale.Builder.html#Locale.Builder%2D%2D[Locale.Builder]()++ (link:{java9-javadoc}/java/util/Locale.Builder.html#Locale.Builder%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-Builder-addUnicodeLocaleAttribute-1]]<<painless-api-reference-Locale-Builder,Locale.Builder>> link:{java8-javadoc}/java/util/Locale.Builder.html#addUnicodeLocaleAttribute%2Djava.lang.String%2D[addUnicodeLocaleAttribute](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/Locale.Builder.html#addUnicodeLocaleAttribute%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Locale-Builder-build-0]]<<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.Builder.html#build%2D%2D[build]()++ (link:{java9-javadoc}/java/util/Locale.Builder.html#build%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-Builder-clear-0]]<<painless-api-reference-Locale-Builder,Locale.Builder>> link:{java8-javadoc}/java/util/Locale.Builder.html#clear%2D%2D[clear]()++ (link:{java9-javadoc}/java/util/Locale.Builder.html#clear%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-Builder-clearExtensions-0]]<<painless-api-reference-Locale-Builder,Locale.Builder>> link:{java8-javadoc}/java/util/Locale.Builder.html#clearExtensions%2D%2D[clearExtensions]()++ (link:{java9-javadoc}/java/util/Locale.Builder.html#clearExtensions%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-Builder-removeUnicodeLocaleAttribute-1]]<<painless-api-reference-Locale-Builder,Locale.Builder>> link:{java8-javadoc}/java/util/Locale.Builder.html#removeUnicodeLocaleAttribute%2Djava.lang.String%2D[removeUnicodeLocaleAttribute](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/Locale.Builder.html#removeUnicodeLocaleAttribute%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Locale-Builder-setExtension-2]]<<painless-api-reference-Locale-Builder,Locale.Builder>> link:{java8-javadoc}/java/util/Locale.Builder.html#setExtension%2Dchar%2Djava.lang.String%2D[setExtension](char, <<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/Locale.Builder.html#setExtension%2Dchar%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Locale-Builder-setLanguage-1]]<<painless-api-reference-Locale-Builder,Locale.Builder>> link:{java8-javadoc}/java/util/Locale.Builder.html#setLanguage%2Djava.lang.String%2D[setLanguage](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/Locale.Builder.html#setLanguage%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Locale-Builder-setLanguageTag-1]]<<painless-api-reference-Locale-Builder,Locale.Builder>> link:{java8-javadoc}/java/util/Locale.Builder.html#setLanguageTag%2Djava.lang.String%2D[setLanguageTag](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/Locale.Builder.html#setLanguageTag%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Locale-Builder-setLocale-1]]<<painless-api-reference-Locale-Builder,Locale.Builder>> link:{java8-javadoc}/java/util/Locale.Builder.html#setLocale%2Djava.util.Locale%2D[setLocale](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/util/Locale.Builder.html#setLocale%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-Locale-Builder-setRegion-1]]<<painless-api-reference-Locale-Builder,Locale.Builder>> link:{java8-javadoc}/java/util/Locale.Builder.html#setRegion%2Djava.lang.String%2D[setRegion](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/Locale.Builder.html#setRegion%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Locale-Builder-setScript-1]]<<painless-api-reference-Locale-Builder,Locale.Builder>> link:{java8-javadoc}/java/util/Locale.Builder.html#setScript%2Djava.lang.String%2D[setScript](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/Locale.Builder.html#setScript%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Locale-Builder-setUnicodeLocaleKeyword-2]]<<painless-api-reference-Locale-Builder,Locale.Builder>> link:{java8-javadoc}/java/util/Locale.Builder.html#setUnicodeLocaleKeyword%2Djava.lang.String%2Djava.lang.String%2D[setUnicodeLocaleKeyword](<<painless-api-reference-String,String>>, <<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/Locale.Builder.html#setUnicodeLocaleKeyword%2Djava.lang.String%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Locale-Builder-setVariant-1]]<<painless-api-reference-Locale-Builder,Locale.Builder>> link:{java8-javadoc}/java/util/Locale.Builder.html#setVariant%2Djava.lang.String%2D[setVariant](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/Locale.Builder.html#setVariant%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Locale.Category.asciidoc b/docs/painless/painless-api-reference/Locale.Category.asciidoc
new file mode 100644
index 0000000000000..96b1cd3fd1011
--- /dev/null
+++ b/docs/painless/painless-api-reference/Locale.Category.asciidoc
@@ -0,0 +1,11 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Locale-Category]]++Locale.Category++::
+** [[painless-api-reference-Locale-Category-DISPLAY]]static <<painless-api-reference-Locale-Category,Locale.Category>> link:{java8-javadoc}/java/util/Locale.Category.html#DISPLAY[DISPLAY] (link:{java9-javadoc}/java/util/Locale.Category.html#DISPLAY[java 9])
+** [[painless-api-reference-Locale-Category-FORMAT]]static <<painless-api-reference-Locale-Category,Locale.Category>> link:{java8-javadoc}/java/util/Locale.Category.html#FORMAT[FORMAT] (link:{java9-javadoc}/java/util/Locale.Category.html#FORMAT[java 9])
+* ++[[painless-api-reference-Locale-Category-valueOf-1]]static <<painless-api-reference-Locale-Category,Locale.Category>> link:{java8-javadoc}/java/util/Locale.Category.html#valueOf%2Djava.lang.String%2D[valueOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/Locale.Category.html#valueOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Locale-Category-values-0]]static <<painless-api-reference-Locale-Category,Locale.Category>>[] link:{java8-javadoc}/java/util/Locale.Category.html#values%2D%2D[values]()++ (link:{java9-javadoc}/java/util/Locale.Category.html#values%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Enum,Enum>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Locale.FilteringMode.asciidoc b/docs/painless/painless-api-reference/Locale.FilteringMode.asciidoc
new file mode 100644
index 0000000000000..f513e4c239668
--- /dev/null
+++ b/docs/painless/painless-api-reference/Locale.FilteringMode.asciidoc
@@ -0,0 +1,14 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Locale-FilteringMode]]++Locale.FilteringMode++::
+** [[painless-api-reference-Locale-FilteringMode-AUTOSELECT_FILTERING]]static <<painless-api-reference-Locale-FilteringMode,Locale.FilteringMode>> link:{java8-javadoc}/java/util/Locale.FilteringMode.html#AUTOSELECT_FILTERING[AUTOSELECT_FILTERING] (link:{java9-javadoc}/java/util/Locale.FilteringMode.html#AUTOSELECT_FILTERING[java 9])
+** [[painless-api-reference-Locale-FilteringMode-EXTENDED_FILTERING]]static <<painless-api-reference-Locale-FilteringMode,Locale.FilteringMode>> link:{java8-javadoc}/java/util/Locale.FilteringMode.html#EXTENDED_FILTERING[EXTENDED_FILTERING] (link:{java9-javadoc}/java/util/Locale.FilteringMode.html#EXTENDED_FILTERING[java 9])
+** [[painless-api-reference-Locale-FilteringMode-IGNORE_EXTENDED_RANGES]]static <<painless-api-reference-Locale-FilteringMode,Locale.FilteringMode>> link:{java8-javadoc}/java/util/Locale.FilteringMode.html#IGNORE_EXTENDED_RANGES[IGNORE_EXTENDED_RANGES] (link:{java9-javadoc}/java/util/Locale.FilteringMode.html#IGNORE_EXTENDED_RANGES[java 9])
+** [[painless-api-reference-Locale-FilteringMode-MAP_EXTENDED_RANGES]]static <<painless-api-reference-Locale-FilteringMode,Locale.FilteringMode>> link:{java8-javadoc}/java/util/Locale.FilteringMode.html#MAP_EXTENDED_RANGES[MAP_EXTENDED_RANGES] (link:{java9-javadoc}/java/util/Locale.FilteringMode.html#MAP_EXTENDED_RANGES[java 9])
+** [[painless-api-reference-Locale-FilteringMode-REJECT_EXTENDED_RANGES]]static <<painless-api-reference-Locale-FilteringMode,Locale.FilteringMode>> link:{java8-javadoc}/java/util/Locale.FilteringMode.html#REJECT_EXTENDED_RANGES[REJECT_EXTENDED_RANGES] (link:{java9-javadoc}/java/util/Locale.FilteringMode.html#REJECT_EXTENDED_RANGES[java 9])
+* ++[[painless-api-reference-Locale-FilteringMode-valueOf-1]]static <<painless-api-reference-Locale-FilteringMode,Locale.FilteringMode>> link:{java8-javadoc}/java/util/Locale.FilteringMode.html#valueOf%2Djava.lang.String%2D[valueOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/Locale.FilteringMode.html#valueOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Locale-FilteringMode-values-0]]static <<painless-api-reference-Locale-FilteringMode,Locale.FilteringMode>>[] link:{java8-javadoc}/java/util/Locale.FilteringMode.html#values%2D%2D[values]()++ (link:{java9-javadoc}/java/util/Locale.FilteringMode.html#values%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Enum,Enum>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Locale.LanguageRange.asciidoc b/docs/painless/painless-api-reference/Locale.LanguageRange.asciidoc
new file mode 100644
index 0000000000000..a0a76b3282021
--- /dev/null
+++ b/docs/painless/painless-api-reference/Locale.LanguageRange.asciidoc
@@ -0,0 +1,16 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Locale-LanguageRange]]++Locale.LanguageRange++::
+** [[painless-api-reference-Locale-LanguageRange-MAX_WEIGHT]]static double link:{java8-javadoc}/java/util/Locale.LanguageRange.html#MAX_WEIGHT[MAX_WEIGHT] (link:{java9-javadoc}/java/util/Locale.LanguageRange.html#MAX_WEIGHT[java 9])
+** [[painless-api-reference-Locale-LanguageRange-MIN_WEIGHT]]static double link:{java8-javadoc}/java/util/Locale.LanguageRange.html#MIN_WEIGHT[MIN_WEIGHT] (link:{java9-javadoc}/java/util/Locale.LanguageRange.html#MIN_WEIGHT[java 9])
+* ++[[painless-api-reference-Locale-LanguageRange-mapEquivalents-2]]static <<painless-api-reference-List,List>> link:{java8-javadoc}/java/util/Locale.LanguageRange.html#mapEquivalents%2Djava.util.List%2Djava.util.Map%2D[mapEquivalents](<<painless-api-reference-List,List>>, <<painless-api-reference-Map,Map>>)++ (link:{java9-javadoc}/java/util/Locale.LanguageRange.html#mapEquivalents%2Djava.util.List%2Djava.util.Map%2D[java 9])
+* ++[[painless-api-reference-Locale-LanguageRange-parse-1]]static <<painless-api-reference-List,List>> link:{java8-javadoc}/java/util/Locale.LanguageRange.html#parse%2Djava.lang.String%2D[parse](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/Locale.LanguageRange.html#parse%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Locale-LanguageRange-parse-2]]static <<painless-api-reference-List,List>> link:{java8-javadoc}/java/util/Locale.LanguageRange.html#parse%2Djava.lang.String%2Djava.util.Map%2D[parse](<<painless-api-reference-String,String>>, <<painless-api-reference-Map,Map>>)++ (link:{java9-javadoc}/java/util/Locale.LanguageRange.html#parse%2Djava.lang.String%2Djava.util.Map%2D[java 9])
+* ++[[painless-api-reference-Locale-LanguageRange-Locale.LanguageRange-1]]link:{java8-javadoc}/java/util/Locale.LanguageRange.html#Locale.LanguageRange%2Djava.lang.String%2D[Locale.LanguageRange](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/Locale.LanguageRange.html#Locale.LanguageRange%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Locale-LanguageRange-Locale.LanguageRange-2]]link:{java8-javadoc}/java/util/Locale.LanguageRange.html#Locale.LanguageRange%2Djava.lang.String%2Ddouble%2D[Locale.LanguageRange](<<painless-api-reference-String,String>>, double)++ (link:{java9-javadoc}/java/util/Locale.LanguageRange.html#Locale.LanguageRange%2Djava.lang.String%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Locale-LanguageRange-getRange-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Locale.LanguageRange.html#getRange%2D%2D[getRange]()++ (link:{java9-javadoc}/java/util/Locale.LanguageRange.html#getRange%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-LanguageRange-getWeight-0]]double link:{java8-javadoc}/java/util/Locale.LanguageRange.html#getWeight%2D%2D[getWeight]()++ (link:{java9-javadoc}/java/util/Locale.LanguageRange.html#getWeight%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Locale.asciidoc b/docs/painless/painless-api-reference/Locale.asciidoc
new file mode 100644
index 0000000000000..d041c6c1f7fb3
--- /dev/null
+++ b/docs/painless/painless-api-reference/Locale.asciidoc
@@ -0,0 +1,69 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Locale]]++Locale++::
+** [[painless-api-reference-Locale-CANADA]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#CANADA[CANADA] (link:{java9-javadoc}/java/util/Locale.html#CANADA[java 9])
+** [[painless-api-reference-Locale-CANADA_FRENCH]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#CANADA_FRENCH[CANADA_FRENCH] (link:{java9-javadoc}/java/util/Locale.html#CANADA_FRENCH[java 9])
+** [[painless-api-reference-Locale-CHINA]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#CHINA[CHINA] (link:{java9-javadoc}/java/util/Locale.html#CHINA[java 9])
+** [[painless-api-reference-Locale-CHINESE]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#CHINESE[CHINESE] (link:{java9-javadoc}/java/util/Locale.html#CHINESE[java 9])
+** [[painless-api-reference-Locale-ENGLISH]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#ENGLISH[ENGLISH] (link:{java9-javadoc}/java/util/Locale.html#ENGLISH[java 9])
+** [[painless-api-reference-Locale-FRANCE]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#FRANCE[FRANCE] (link:{java9-javadoc}/java/util/Locale.html#FRANCE[java 9])
+** [[painless-api-reference-Locale-FRENCH]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#FRENCH[FRENCH] (link:{java9-javadoc}/java/util/Locale.html#FRENCH[java 9])
+** [[painless-api-reference-Locale-GERMAN]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#GERMAN[GERMAN] (link:{java9-javadoc}/java/util/Locale.html#GERMAN[java 9])
+** [[painless-api-reference-Locale-GERMANY]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#GERMANY[GERMANY] (link:{java9-javadoc}/java/util/Locale.html#GERMANY[java 9])
+** [[painless-api-reference-Locale-ITALIAN]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#ITALIAN[ITALIAN] (link:{java9-javadoc}/java/util/Locale.html#ITALIAN[java 9])
+** [[painless-api-reference-Locale-ITALY]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#ITALY[ITALY] (link:{java9-javadoc}/java/util/Locale.html#ITALY[java 9])
+** [[painless-api-reference-Locale-JAPAN]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#JAPAN[JAPAN] (link:{java9-javadoc}/java/util/Locale.html#JAPAN[java 9])
+** [[painless-api-reference-Locale-JAPANESE]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#JAPANESE[JAPANESE] (link:{java9-javadoc}/java/util/Locale.html#JAPANESE[java 9])
+** [[painless-api-reference-Locale-KOREA]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#KOREA[KOREA] (link:{java9-javadoc}/java/util/Locale.html#KOREA[java 9])
+** [[painless-api-reference-Locale-KOREAN]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#KOREAN[KOREAN] (link:{java9-javadoc}/java/util/Locale.html#KOREAN[java 9])
+** [[painless-api-reference-Locale-PRC]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#PRC[PRC] (link:{java9-javadoc}/java/util/Locale.html#PRC[java 9])
+** [[painless-api-reference-Locale-PRIVATE_USE_EXTENSION]]static char link:{java8-javadoc}/java/util/Locale.html#PRIVATE_USE_EXTENSION[PRIVATE_USE_EXTENSION] (link:{java9-javadoc}/java/util/Locale.html#PRIVATE_USE_EXTENSION[java 9])
+** [[painless-api-reference-Locale-ROOT]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#ROOT[ROOT] (link:{java9-javadoc}/java/util/Locale.html#ROOT[java 9])
+** [[painless-api-reference-Locale-SIMPLIFIED_CHINESE]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#SIMPLIFIED_CHINESE[SIMPLIFIED_CHINESE] (link:{java9-javadoc}/java/util/Locale.html#SIMPLIFIED_CHINESE[java 9])
+** [[painless-api-reference-Locale-TAIWAN]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#TAIWAN[TAIWAN] (link:{java9-javadoc}/java/util/Locale.html#TAIWAN[java 9])
+** [[painless-api-reference-Locale-TRADITIONAL_CHINESE]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#TRADITIONAL_CHINESE[TRADITIONAL_CHINESE] (link:{java9-javadoc}/java/util/Locale.html#TRADITIONAL_CHINESE[java 9])
+** [[painless-api-reference-Locale-UK]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#UK[UK] (link:{java9-javadoc}/java/util/Locale.html#UK[java 9])
+** [[painless-api-reference-Locale-UNICODE_LOCALE_EXTENSION]]static char link:{java8-javadoc}/java/util/Locale.html#UNICODE_LOCALE_EXTENSION[UNICODE_LOCALE_EXTENSION] (link:{java9-javadoc}/java/util/Locale.html#UNICODE_LOCALE_EXTENSION[java 9])
+** [[painless-api-reference-Locale-US]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#US[US] (link:{java9-javadoc}/java/util/Locale.html#US[java 9])
+* ++[[painless-api-reference-Locale-filter-2]]static <<painless-api-reference-List,List>> link:{java8-javadoc}/java/util/Locale.html#filter%2Djava.util.List%2Djava.util.Collection%2D[filter](<<painless-api-reference-List,List>>, <<painless-api-reference-Collection,Collection>>)++ (link:{java9-javadoc}/java/util/Locale.html#filter%2Djava.util.List%2Djava.util.Collection%2D[java 9])
+* ++[[painless-api-reference-Locale-filterTags-2]]static <<painless-api-reference-List,List>> link:{java8-javadoc}/java/util/Locale.html#filterTags%2Djava.util.List%2Djava.util.Collection%2D[filterTags](<<painless-api-reference-List,List>>, <<painless-api-reference-Collection,Collection>>)++ (link:{java9-javadoc}/java/util/Locale.html#filterTags%2Djava.util.List%2Djava.util.Collection%2D[java 9])
+* ++[[painless-api-reference-Locale-forLanguageTag-1]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#forLanguageTag%2Djava.lang.String%2D[forLanguageTag](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/Locale.html#forLanguageTag%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Locale-getAvailableLocales-0]]static <<painless-api-reference-Locale,Locale>>[] link:{java8-javadoc}/java/util/Locale.html#getAvailableLocales%2D%2D[getAvailableLocales]()++ (link:{java9-javadoc}/java/util/Locale.html#getAvailableLocales%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-getDefault-0]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#getDefault%2D%2D[getDefault]()++ (link:{java9-javadoc}/java/util/Locale.html#getDefault%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-getDefault-1]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#getDefault%2Djava.util.Locale$Category%2D[getDefault](<<painless-api-reference-Locale-Category,Locale.Category>>)++ (link:{java9-javadoc}/java/util/Locale.html#getDefault%2Djava.util.Locale$Category%2D[java 9])
+* ++[[painless-api-reference-Locale-getISOCountries-0]]static <<painless-api-reference-String,String>>[] link:{java8-javadoc}/java/util/Locale.html#getISOCountries%2D%2D[getISOCountries]()++ (link:{java9-javadoc}/java/util/Locale.html#getISOCountries%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-getISOLanguages-0]]static <<painless-api-reference-String,String>>[] link:{java8-javadoc}/java/util/Locale.html#getISOLanguages%2D%2D[getISOLanguages]()++ (link:{java9-javadoc}/java/util/Locale.html#getISOLanguages%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-lookup-2]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#lookup%2Djava.util.List%2Djava.util.Collection%2D[lookup](<<painless-api-reference-List,List>>, <<painless-api-reference-Collection,Collection>>)++ (link:{java9-javadoc}/java/util/Locale.html#lookup%2Djava.util.List%2Djava.util.Collection%2D[java 9])
+* ++[[painless-api-reference-Locale-lookupTag-2]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Locale.html#lookupTag%2Djava.util.List%2Djava.util.Collection%2D[lookupTag](<<painless-api-reference-List,List>>, <<painless-api-reference-Collection,Collection>>)++ (link:{java9-javadoc}/java/util/Locale.html#lookupTag%2Djava.util.List%2Djava.util.Collection%2D[java 9])
+* ++[[painless-api-reference-Locale-Locale-1]]link:{java8-javadoc}/java/util/Locale.html#Locale%2Djava.lang.String%2D[Locale](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/Locale.html#Locale%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Locale-Locale-2]]link:{java8-javadoc}/java/util/Locale.html#Locale%2Djava.lang.String%2Djava.lang.String%2D[Locale](<<painless-api-reference-String,String>>, <<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/Locale.html#Locale%2Djava.lang.String%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Locale-Locale-3]]link:{java8-javadoc}/java/util/Locale.html#Locale%2Djava.lang.String%2Djava.lang.String%2Djava.lang.String%2D[Locale](<<painless-api-reference-String,String>>, <<painless-api-reference-String,String>>, <<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/Locale.html#Locale%2Djava.lang.String%2Djava.lang.String%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Locale-clone-0]]def link:{java8-javadoc}/java/util/Locale.html#clone%2D%2D[clone]()++ (link:{java9-javadoc}/java/util/Locale.html#clone%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-getCountry-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Locale.html#getCountry%2D%2D[getCountry]()++ (link:{java9-javadoc}/java/util/Locale.html#getCountry%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-getDisplayCountry-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Locale.html#getDisplayCountry%2D%2D[getDisplayCountry]()++ (link:{java9-javadoc}/java/util/Locale.html#getDisplayCountry%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-getDisplayCountry-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Locale.html#getDisplayCountry%2Djava.util.Locale%2D[getDisplayCountry](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/util/Locale.html#getDisplayCountry%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-Locale-getDisplayLanguage-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Locale.html#getDisplayLanguage%2D%2D[getDisplayLanguage]()++ (link:{java9-javadoc}/java/util/Locale.html#getDisplayLanguage%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-getDisplayLanguage-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Locale.html#getDisplayLanguage%2Djava.util.Locale%2D[getDisplayLanguage](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/util/Locale.html#getDisplayLanguage%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-Locale-getDisplayName-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Locale.html#getDisplayName%2D%2D[getDisplayName]()++ (link:{java9-javadoc}/java/util/Locale.html#getDisplayName%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-getDisplayName-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Locale.html#getDisplayName%2Djava.util.Locale%2D[getDisplayName](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/util/Locale.html#getDisplayName%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-Locale-getDisplayScript-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Locale.html#getDisplayScript%2D%2D[getDisplayScript]()++ (link:{java9-javadoc}/java/util/Locale.html#getDisplayScript%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-getDisplayScript-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Locale.html#getDisplayScript%2Djava.util.Locale%2D[getDisplayScript](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/util/Locale.html#getDisplayScript%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-Locale-getDisplayVariant-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Locale.html#getDisplayVariant%2D%2D[getDisplayVariant]()++ (link:{java9-javadoc}/java/util/Locale.html#getDisplayVariant%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-getDisplayVariant-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Locale.html#getDisplayVariant%2Djava.util.Locale%2D[getDisplayVariant](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/util/Locale.html#getDisplayVariant%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-Locale-getExtension-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Locale.html#getExtension%2Dchar%2D[getExtension](char)++ (link:{java9-javadoc}/java/util/Locale.html#getExtension%2Dchar%2D[java 9])
+* ++[[painless-api-reference-Locale-getExtensionKeys-0]]<<painless-api-reference-Set,Set>> link:{java8-javadoc}/java/util/Locale.html#getExtensionKeys%2D%2D[getExtensionKeys]()++ (link:{java9-javadoc}/java/util/Locale.html#getExtensionKeys%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-getISO3Country-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Locale.html#getISO3Country%2D%2D[getISO3Country]()++ (link:{java9-javadoc}/java/util/Locale.html#getISO3Country%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-getISO3Language-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Locale.html#getISO3Language%2D%2D[getISO3Language]()++ (link:{java9-javadoc}/java/util/Locale.html#getISO3Language%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-getLanguage-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Locale.html#getLanguage%2D%2D[getLanguage]()++ (link:{java9-javadoc}/java/util/Locale.html#getLanguage%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-getScript-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Locale.html#getScript%2D%2D[getScript]()++ (link:{java9-javadoc}/java/util/Locale.html#getScript%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-getUnicodeLocaleAttributes-0]]<<painless-api-reference-Set,Set>> link:{java8-javadoc}/java/util/Locale.html#getUnicodeLocaleAttributes%2D%2D[getUnicodeLocaleAttributes]()++ (link:{java9-javadoc}/java/util/Locale.html#getUnicodeLocaleAttributes%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-getUnicodeLocaleKeys-0]]<<painless-api-reference-Set,Set>> link:{java8-javadoc}/java/util/Locale.html#getUnicodeLocaleKeys%2D%2D[getUnicodeLocaleKeys]()++ (link:{java9-javadoc}/java/util/Locale.html#getUnicodeLocaleKeys%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-getUnicodeLocaleType-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Locale.html#getUnicodeLocaleType%2Djava.lang.String%2D[getUnicodeLocaleType](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/Locale.html#getUnicodeLocaleType%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Locale-getVariant-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Locale.html#getVariant%2D%2D[getVariant]()++ (link:{java9-javadoc}/java/util/Locale.html#getVariant%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-hasExtensions-0]]boolean link:{java8-javadoc}/java/util/Locale.html#hasExtensions%2D%2D[hasExtensions]()++ (link:{java9-javadoc}/java/util/Locale.html#hasExtensions%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-stripExtensions-0]]<<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#stripExtensions%2D%2D[stripExtensions]()++ (link:{java9-javadoc}/java/util/Locale.html#stripExtensions%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-toLanguageTag-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Locale.html#toLanguageTag%2D%2D[toLanguageTag]()++ (link:{java9-javadoc}/java/util/Locale.html#toLanguageTag%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Long.asciidoc b/docs/painless/painless-api-reference/Long.asciidoc
new file mode 100644
index 0000000000000..e35c80f63b96d
--- /dev/null
+++ b/docs/painless/painless-api-reference/Long.asciidoc
@@ -0,0 +1,44 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Long]]++Long++::
+** [[painless-api-reference-Long-BYTES]]static int link:{java8-javadoc}/java/lang/Long.html#BYTES[BYTES] (link:{java9-javadoc}/java/lang/Long.html#BYTES[java 9])
+** [[painless-api-reference-Long-MAX_VALUE]]static long link:{java8-javadoc}/java/lang/Long.html#MAX_VALUE[MAX_VALUE] (link:{java9-javadoc}/java/lang/Long.html#MAX_VALUE[java 9])
+** [[painless-api-reference-Long-MIN_VALUE]]static long link:{java8-javadoc}/java/lang/Long.html#MIN_VALUE[MIN_VALUE] (link:{java9-javadoc}/java/lang/Long.html#MIN_VALUE[java 9])
+** [[painless-api-reference-Long-SIZE]]static int link:{java8-javadoc}/java/lang/Long.html#SIZE[SIZE] (link:{java9-javadoc}/java/lang/Long.html#SIZE[java 9])
+* ++[[painless-api-reference-Long-bitCount-1]]static int link:{java8-javadoc}/java/lang/Long.html#bitCount%2Dlong%2D[bitCount](long)++ (link:{java9-javadoc}/java/lang/Long.html#bitCount%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Long-compare-2]]static int link:{java8-javadoc}/java/lang/Long.html#compare%2Dlong%2Dlong%2D[compare](long, long)++ (link:{java9-javadoc}/java/lang/Long.html#compare%2Dlong%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Long-compareUnsigned-2]]static int link:{java8-javadoc}/java/lang/Long.html#compareUnsigned%2Dlong%2Dlong%2D[compareUnsigned](long, long)++ (link:{java9-javadoc}/java/lang/Long.html#compareUnsigned%2Dlong%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Long-decode-1]]static <<painless-api-reference-Long,Long>> link:{java8-javadoc}/java/lang/Long.html#decode%2Djava.lang.String%2D[decode](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/Long.html#decode%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Long-divideUnsigned-2]]static long link:{java8-javadoc}/java/lang/Long.html#divideUnsigned%2Dlong%2Dlong%2D[divideUnsigned](long, long)++ (link:{java9-javadoc}/java/lang/Long.html#divideUnsigned%2Dlong%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Long-hashCode-1]]static int link:{java8-javadoc}/java/lang/Long.html#hashCode%2Dlong%2D[hashCode](long)++ (link:{java9-javadoc}/java/lang/Long.html#hashCode%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Long-highestOneBit-1]]static long link:{java8-javadoc}/java/lang/Long.html#highestOneBit%2Dlong%2D[highestOneBit](long)++ (link:{java9-javadoc}/java/lang/Long.html#highestOneBit%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Long-lowestOneBit-1]]static long link:{java8-javadoc}/java/lang/Long.html#lowestOneBit%2Dlong%2D[lowestOneBit](long)++ (link:{java9-javadoc}/java/lang/Long.html#lowestOneBit%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Long-max-2]]static long link:{java8-javadoc}/java/lang/Long.html#max%2Dlong%2Dlong%2D[max](long, long)++ (link:{java9-javadoc}/java/lang/Long.html#max%2Dlong%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Long-min-2]]static long link:{java8-javadoc}/java/lang/Long.html#min%2Dlong%2Dlong%2D[min](long, long)++ (link:{java9-javadoc}/java/lang/Long.html#min%2Dlong%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Long-numberOfLeadingZeros-1]]static int link:{java8-javadoc}/java/lang/Long.html#numberOfLeadingZeros%2Dlong%2D[numberOfLeadingZeros](long)++ (link:{java9-javadoc}/java/lang/Long.html#numberOfLeadingZeros%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Long-numberOfTrailingZeros-1]]static int link:{java8-javadoc}/java/lang/Long.html#numberOfTrailingZeros%2Dlong%2D[numberOfTrailingZeros](long)++ (link:{java9-javadoc}/java/lang/Long.html#numberOfTrailingZeros%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Long-parseLong-1]]static long link:{java8-javadoc}/java/lang/Long.html#parseLong%2Djava.lang.String%2D[parseLong](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/Long.html#parseLong%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Long-parseLong-2]]static long link:{java8-javadoc}/java/lang/Long.html#parseLong%2Djava.lang.String%2Dint%2D[parseLong](<<painless-api-reference-String,String>>, int)++ (link:{java9-javadoc}/java/lang/Long.html#parseLong%2Djava.lang.String%2Dint%2D[java 9])
+* ++[[painless-api-reference-Long-parseUnsignedLong-1]]static long link:{java8-javadoc}/java/lang/Long.html#parseUnsignedLong%2Djava.lang.String%2D[parseUnsignedLong](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/Long.html#parseUnsignedLong%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Long-parseUnsignedLong-2]]static long link:{java8-javadoc}/java/lang/Long.html#parseUnsignedLong%2Djava.lang.String%2Dint%2D[parseUnsignedLong](<<painless-api-reference-String,String>>, int)++ (link:{java9-javadoc}/java/lang/Long.html#parseUnsignedLong%2Djava.lang.String%2Dint%2D[java 9])
+* ++[[painless-api-reference-Long-remainderUnsigned-2]]static long link:{java8-javadoc}/java/lang/Long.html#remainderUnsigned%2Dlong%2Dlong%2D[remainderUnsigned](long, long)++ (link:{java9-javadoc}/java/lang/Long.html#remainderUnsigned%2Dlong%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Long-reverse-1]]static long link:{java8-javadoc}/java/lang/Long.html#reverse%2Dlong%2D[reverse](long)++ (link:{java9-javadoc}/java/lang/Long.html#reverse%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Long-reverseBytes-1]]static long link:{java8-javadoc}/java/lang/Long.html#reverseBytes%2Dlong%2D[reverseBytes](long)++ (link:{java9-javadoc}/java/lang/Long.html#reverseBytes%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Long-rotateLeft-2]]static long link:{java8-javadoc}/java/lang/Long.html#rotateLeft%2Dlong%2Dint%2D[rotateLeft](long, int)++ (link:{java9-javadoc}/java/lang/Long.html#rotateLeft%2Dlong%2Dint%2D[java 9])
+* ++[[painless-api-reference-Long-rotateRight-2]]static long link:{java8-javadoc}/java/lang/Long.html#rotateRight%2Dlong%2Dint%2D[rotateRight](long, int)++ (link:{java9-javadoc}/java/lang/Long.html#rotateRight%2Dlong%2Dint%2D[java 9])
+* ++[[painless-api-reference-Long-signum-1]]static int link:{java8-javadoc}/java/lang/Long.html#signum%2Dlong%2D[signum](long)++ (link:{java9-javadoc}/java/lang/Long.html#signum%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Long-sum-2]]static long link:{java8-javadoc}/java/lang/Long.html#sum%2Dlong%2Dlong%2D[sum](long, long)++ (link:{java9-javadoc}/java/lang/Long.html#sum%2Dlong%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Long-toBinaryString-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Long.html#toBinaryString%2Dlong%2D[toBinaryString](long)++ (link:{java9-javadoc}/java/lang/Long.html#toBinaryString%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Long-toHexString-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Long.html#toHexString%2Dlong%2D[toHexString](long)++ (link:{java9-javadoc}/java/lang/Long.html#toHexString%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Long-toOctalString-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Long.html#toOctalString%2Dlong%2D[toOctalString](long)++ (link:{java9-javadoc}/java/lang/Long.html#toOctalString%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Long-toString-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Long.html#toString%2Dlong%2D[toString](long)++ (link:{java9-javadoc}/java/lang/Long.html#toString%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Long-toString-2]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Long.html#toString%2Dlong%2Dint%2D[toString](long, int)++ (link:{java9-javadoc}/java/lang/Long.html#toString%2Dlong%2Dint%2D[java 9])
+* ++[[painless-api-reference-Long-toUnsignedString-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Long.html#toUnsignedString%2Dlong%2D[toUnsignedString](long)++ (link:{java9-javadoc}/java/lang/Long.html#toUnsignedString%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Long-toUnsignedString-2]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Long.html#toUnsignedString%2Dlong%2Dint%2D[toUnsignedString](long, int)++ (link:{java9-javadoc}/java/lang/Long.html#toUnsignedString%2Dlong%2Dint%2D[java 9])
+* ++[[painless-api-reference-Long-valueOf-1]]static <<painless-api-reference-Long,Long>> link:{java8-javadoc}/java/lang/Long.html#valueOf%2Dlong%2D[valueOf](long)++ (link:{java9-javadoc}/java/lang/Long.html#valueOf%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Long-valueOf-2]]static <<painless-api-reference-Long,Long>> link:{java8-javadoc}/java/lang/Long.html#valueOf%2Djava.lang.String%2Dint%2D[valueOf](<<painless-api-reference-String,String>>, int)++ (link:{java9-javadoc}/java/lang/Long.html#valueOf%2Djava.lang.String%2Dint%2D[java 9])
+* ++[[painless-api-reference-Long-compareTo-1]]int link:{java8-javadoc}/java/lang/Long.html#compareTo%2Djava.lang.Long%2D[compareTo](<<painless-api-reference-Long,Long>>)++ (link:{java9-javadoc}/java/lang/Long.html#compareTo%2Djava.lang.Long%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Number,Number>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/LongBinaryOperator.asciidoc b/docs/painless/painless-api-reference/LongBinaryOperator.asciidoc
new file mode 100644
index 0000000000000..45a777a69715b
--- /dev/null
+++ b/docs/painless/painless-api-reference/LongBinaryOperator.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-LongBinaryOperator]]++LongBinaryOperator++::
+* ++[[painless-api-reference-LongBinaryOperator-applyAsLong-2]]long link:{java8-javadoc}/java/util/function/LongBinaryOperator.html#applyAsLong%2Dlong%2Dlong%2D[applyAsLong](long, long)++ (link:{java9-javadoc}/java/util/function/LongBinaryOperator.html#applyAsLong%2Dlong%2Dlong%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/LongConsumer.asciidoc b/docs/painless/painless-api-reference/LongConsumer.asciidoc
new file mode 100644
index 0000000000000..28bbc92c03121
--- /dev/null
+++ b/docs/painless/painless-api-reference/LongConsumer.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-LongConsumer]]++LongConsumer++::
+* ++[[painless-api-reference-LongConsumer-accept-1]]void link:{java8-javadoc}/java/util/function/LongConsumer.html#accept%2Dlong%2D[accept](long)++ (link:{java9-javadoc}/java/util/function/LongConsumer.html#accept%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LongConsumer-andThen-1]]<<painless-api-reference-LongConsumer,LongConsumer>> link:{java8-javadoc}/java/util/function/LongConsumer.html#andThen%2Djava.util.function.LongConsumer%2D[andThen](<<painless-api-reference-LongConsumer,LongConsumer>>)++ (link:{java9-javadoc}/java/util/function/LongConsumer.html#andThen%2Djava.util.function.LongConsumer%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/LongFunction.asciidoc b/docs/painless/painless-api-reference/LongFunction.asciidoc
new file mode 100644
index 0000000000000..033392914a5e3
--- /dev/null
+++ b/docs/painless/painless-api-reference/LongFunction.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-LongFunction]]++LongFunction++::
+* ++[[painless-api-reference-LongFunction-apply-1]]def link:{java8-javadoc}/java/util/function/LongFunction.html#apply%2Dlong%2D[apply](long)++ (link:{java9-javadoc}/java/util/function/LongFunction.html#apply%2Dlong%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/LongPredicate.asciidoc b/docs/painless/painless-api-reference/LongPredicate.asciidoc
new file mode 100644
index 0000000000000..59b719572e3d1
--- /dev/null
+++ b/docs/painless/painless-api-reference/LongPredicate.asciidoc
@@ -0,0 +1,11 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-LongPredicate]]++LongPredicate++::
+* ++[[painless-api-reference-LongPredicate-and-1]]<<painless-api-reference-LongPredicate,LongPredicate>> link:{java8-javadoc}/java/util/function/LongPredicate.html#and%2Djava.util.function.LongPredicate%2D[and](<<painless-api-reference-LongPredicate,LongPredicate>>)++ (link:{java9-javadoc}/java/util/function/LongPredicate.html#and%2Djava.util.function.LongPredicate%2D[java 9])
+* ++[[painless-api-reference-LongPredicate-negate-0]]<<painless-api-reference-LongPredicate,LongPredicate>> link:{java8-javadoc}/java/util/function/LongPredicate.html#negate%2D%2D[negate]()++ (link:{java9-javadoc}/java/util/function/LongPredicate.html#negate%2D%2D[java 9])
+* ++[[painless-api-reference-LongPredicate-or-1]]<<painless-api-reference-LongPredicate,LongPredicate>> link:{java8-javadoc}/java/util/function/LongPredicate.html#or%2Djava.util.function.LongPredicate%2D[or](<<painless-api-reference-LongPredicate,LongPredicate>>)++ (link:{java9-javadoc}/java/util/function/LongPredicate.html#or%2Djava.util.function.LongPredicate%2D[java 9])
+* ++[[painless-api-reference-LongPredicate-test-1]]boolean link:{java8-javadoc}/java/util/function/LongPredicate.html#test%2Dlong%2D[test](long)++ (link:{java9-javadoc}/java/util/function/LongPredicate.html#test%2Dlong%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/LongStream.Builder.asciidoc b/docs/painless/painless-api-reference/LongStream.Builder.asciidoc
new file mode 100644
index 0000000000000..a5d8d0a874cb0
--- /dev/null
+++ b/docs/painless/painless-api-reference/LongStream.Builder.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-LongStream-Builder]]++LongStream.Builder++::
+* ++[[painless-api-reference-LongStream-Builder-add-1]]<<painless-api-reference-LongStream-Builder,LongStream.Builder>> link:{java8-javadoc}/java/util/stream/LongStream.Builder.html#add%2Dlong%2D[add](long)++ (link:{java9-javadoc}/java/util/stream/LongStream.Builder.html#add%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LongStream-Builder-build-0]]<<painless-api-reference-LongStream,LongStream>> link:{java8-javadoc}/java/util/stream/LongStream.Builder.html#build%2D%2D[build]()++ (link:{java9-javadoc}/java/util/stream/LongStream.Builder.html#build%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-LongConsumer,LongConsumer>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/LongStream.asciidoc b/docs/painless/painless-api-reference/LongStream.asciidoc
new file mode 100644
index 0000000000000..08ea178451a0a
--- /dev/null
+++ b/docs/painless/painless-api-reference/LongStream.asciidoc
@@ -0,0 +1,46 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-LongStream]]++LongStream++::
+* ++[[painless-api-reference-LongStream-builder-0]]static <<painless-api-reference-LongStream-Builder,LongStream.Builder>> link:{java8-javadoc}/java/util/stream/LongStream.html#builder%2D%2D[builder]()++ (link:{java9-javadoc}/java/util/stream/LongStream.html#builder%2D%2D[java 9])
+* ++[[painless-api-reference-LongStream-concat-2]]static <<painless-api-reference-LongStream,LongStream>> link:{java8-javadoc}/java/util/stream/LongStream.html#concat%2Djava.util.stream.LongStream%2Djava.util.stream.LongStream%2D[concat](<<painless-api-reference-LongStream,LongStream>>, <<painless-api-reference-LongStream,LongStream>>)++ (link:{java9-javadoc}/java/util/stream/LongStream.html#concat%2Djava.util.stream.LongStream%2Djava.util.stream.LongStream%2D[java 9])
+* ++[[painless-api-reference-LongStream-empty-0]]static <<painless-api-reference-LongStream,LongStream>> link:{java8-javadoc}/java/util/stream/LongStream.html#empty%2D%2D[empty]()++ (link:{java9-javadoc}/java/util/stream/LongStream.html#empty%2D%2D[java 9])
+* ++[[painless-api-reference-LongStream-of-1]]static <<painless-api-reference-LongStream,LongStream>> link:{java8-javadoc}/java/util/stream/LongStream.html#of%2Dlong:A%2D[of](long[])++ (link:{java9-javadoc}/java/util/stream/LongStream.html#of%2Dlong:A%2D[java 9])
+* ++[[painless-api-reference-LongStream-range-2]]static <<painless-api-reference-LongStream,LongStream>> link:{java8-javadoc}/java/util/stream/LongStream.html#range%2Dlong%2Dlong%2D[range](long, long)++ (link:{java9-javadoc}/java/util/stream/LongStream.html#range%2Dlong%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LongStream-rangeClosed-2]]static <<painless-api-reference-LongStream,LongStream>> link:{java8-javadoc}/java/util/stream/LongStream.html#rangeClosed%2Dlong%2Dlong%2D[rangeClosed](long, long)++ (link:{java9-javadoc}/java/util/stream/LongStream.html#rangeClosed%2Dlong%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LongStream-allMatch-1]]boolean link:{java8-javadoc}/java/util/stream/LongStream.html#allMatch%2Djava.util.function.LongPredicate%2D[allMatch](<<painless-api-reference-LongPredicate,LongPredicate>>)++ (link:{java9-javadoc}/java/util/stream/LongStream.html#allMatch%2Djava.util.function.LongPredicate%2D[java 9])
+* ++[[painless-api-reference-LongStream-anyMatch-1]]boolean link:{java8-javadoc}/java/util/stream/LongStream.html#anyMatch%2Djava.util.function.LongPredicate%2D[anyMatch](<<painless-api-reference-LongPredicate,LongPredicate>>)++ (link:{java9-javadoc}/java/util/stream/LongStream.html#anyMatch%2Djava.util.function.LongPredicate%2D[java 9])
+* ++[[painless-api-reference-LongStream-asDoubleStream-0]]<<painless-api-reference-DoubleStream,DoubleStream>> link:{java8-javadoc}/java/util/stream/LongStream.html#asDoubleStream%2D%2D[asDoubleStream]()++ (link:{java9-javadoc}/java/util/stream/LongStream.html#asDoubleStream%2D%2D[java 9])
+* ++[[painless-api-reference-LongStream-average-0]]<<painless-api-reference-OptionalDouble,OptionalDouble>> link:{java8-javadoc}/java/util/stream/LongStream.html#average%2D%2D[average]()++ (link:{java9-javadoc}/java/util/stream/LongStream.html#average%2D%2D[java 9])
+* ++[[painless-api-reference-LongStream-boxed-0]]<<painless-api-reference-Stream,Stream>> link:{java8-javadoc}/java/util/stream/LongStream.html#boxed%2D%2D[boxed]()++ (link:{java9-javadoc}/java/util/stream/LongStream.html#boxed%2D%2D[java 9])
+* ++[[painless-api-reference-LongStream-collect-3]]def link:{java8-javadoc}/java/util/stream/LongStream.html#collect%2Djava.util.function.Supplier%2Djava.util.function.ObjLongConsumer%2Djava.util.function.BiConsumer%2D[collect](<<painless-api-reference-Supplier,Supplier>>, <<painless-api-reference-ObjLongConsumer,ObjLongConsumer>>, <<painless-api-reference-BiConsumer,BiConsumer>>)++ (link:{java9-javadoc}/java/util/stream/LongStream.html#collect%2Djava.util.function.Supplier%2Djava.util.function.ObjLongConsumer%2Djava.util.function.BiConsumer%2D[java 9])
+* ++[[painless-api-reference-LongStream-count-0]]long link:{java8-javadoc}/java/util/stream/LongStream.html#count%2D%2D[count]()++ (link:{java9-javadoc}/java/util/stream/LongStream.html#count%2D%2D[java 9])
+* ++[[painless-api-reference-LongStream-distinct-0]]<<painless-api-reference-LongStream,LongStream>> link:{java8-javadoc}/java/util/stream/LongStream.html#distinct%2D%2D[distinct]()++ (link:{java9-javadoc}/java/util/stream/LongStream.html#distinct%2D%2D[java 9])
+* ++[[painless-api-reference-LongStream-filter-1]]<<painless-api-reference-LongStream,LongStream>> link:{java8-javadoc}/java/util/stream/LongStream.html#filter%2Djava.util.function.LongPredicate%2D[filter](<<painless-api-reference-LongPredicate,LongPredicate>>)++ (link:{java9-javadoc}/java/util/stream/LongStream.html#filter%2Djava.util.function.LongPredicate%2D[java 9])
+* ++[[painless-api-reference-LongStream-findAny-0]]<<painless-api-reference-OptionalLong,OptionalLong>> link:{java8-javadoc}/java/util/stream/LongStream.html#findAny%2D%2D[findAny]()++ (link:{java9-javadoc}/java/util/stream/LongStream.html#findAny%2D%2D[java 9])
+* ++[[painless-api-reference-LongStream-findFirst-0]]<<painless-api-reference-OptionalLong,OptionalLong>> link:{java8-javadoc}/java/util/stream/LongStream.html#findFirst%2D%2D[findFirst]()++ (link:{java9-javadoc}/java/util/stream/LongStream.html#findFirst%2D%2D[java 9])
+* ++[[painless-api-reference-LongStream-flatMap-1]]<<painless-api-reference-LongStream,LongStream>> link:{java8-javadoc}/java/util/stream/LongStream.html#flatMap%2Djava.util.function.LongFunction%2D[flatMap](<<painless-api-reference-LongFunction,LongFunction>>)++ (link:{java9-javadoc}/java/util/stream/LongStream.html#flatMap%2Djava.util.function.LongFunction%2D[java 9])
+* ++[[painless-api-reference-LongStream-forEach-1]]void link:{java8-javadoc}/java/util/stream/LongStream.html#forEach%2Djava.util.function.LongConsumer%2D[forEach](<<painless-api-reference-LongConsumer,LongConsumer>>)++ (link:{java9-javadoc}/java/util/stream/LongStream.html#forEach%2Djava.util.function.LongConsumer%2D[java 9])
+* ++[[painless-api-reference-LongStream-forEachOrdered-1]]void link:{java8-javadoc}/java/util/stream/LongStream.html#forEachOrdered%2Djava.util.function.LongConsumer%2D[forEachOrdered](<<painless-api-reference-LongConsumer,LongConsumer>>)++ (link:{java9-javadoc}/java/util/stream/LongStream.html#forEachOrdered%2Djava.util.function.LongConsumer%2D[java 9])
+* ++[[painless-api-reference-LongStream-iterator-0]]<<painless-api-reference-PrimitiveIterator-OfLong,PrimitiveIterator.OfLong>> link:{java8-javadoc}/java/util/stream/LongStream.html#iterator%2D%2D[iterator]()++ (link:{java9-javadoc}/java/util/stream/LongStream.html#iterator%2D%2D[java 9])
+* ++[[painless-api-reference-LongStream-limit-1]]<<painless-api-reference-LongStream,LongStream>> link:{java8-javadoc}/java/util/stream/LongStream.html#limit%2Dlong%2D[limit](long)++ (link:{java9-javadoc}/java/util/stream/LongStream.html#limit%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LongStream-map-1]]<<painless-api-reference-LongStream,LongStream>> link:{java8-javadoc}/java/util/stream/LongStream.html#map%2Djava.util.function.LongUnaryOperator%2D[map](<<painless-api-reference-LongUnaryOperator,LongUnaryOperator>>)++ (link:{java9-javadoc}/java/util/stream/LongStream.html#map%2Djava.util.function.LongUnaryOperator%2D[java 9])
+* ++[[painless-api-reference-LongStream-mapToDouble-1]]<<painless-api-reference-DoubleStream,DoubleStream>> link:{java8-javadoc}/java/util/stream/LongStream.html#mapToDouble%2Djava.util.function.LongToDoubleFunction%2D[mapToDouble](<<painless-api-reference-LongToDoubleFunction,LongToDoubleFunction>>)++ (link:{java9-javadoc}/java/util/stream/LongStream.html#mapToDouble%2Djava.util.function.LongToDoubleFunction%2D[java 9])
+* ++[[painless-api-reference-LongStream-mapToInt-1]]<<painless-api-reference-IntStream,IntStream>> link:{java8-javadoc}/java/util/stream/LongStream.html#mapToInt%2Djava.util.function.LongToIntFunction%2D[mapToInt](<<painless-api-reference-LongToIntFunction,LongToIntFunction>>)++ (link:{java9-javadoc}/java/util/stream/LongStream.html#mapToInt%2Djava.util.function.LongToIntFunction%2D[java 9])
+* ++[[painless-api-reference-LongStream-mapToObj-1]]<<painless-api-reference-Stream,Stream>> link:{java8-javadoc}/java/util/stream/LongStream.html#mapToObj%2Djava.util.function.LongFunction%2D[mapToObj](<<painless-api-reference-LongFunction,LongFunction>>)++ (link:{java9-javadoc}/java/util/stream/LongStream.html#mapToObj%2Djava.util.function.LongFunction%2D[java 9])
+* ++[[painless-api-reference-LongStream-max-0]]<<painless-api-reference-OptionalLong,OptionalLong>> link:{java8-javadoc}/java/util/stream/LongStream.html#max%2D%2D[max]()++ (link:{java9-javadoc}/java/util/stream/LongStream.html#max%2D%2D[java 9])
+* ++[[painless-api-reference-LongStream-min-0]]<<painless-api-reference-OptionalLong,OptionalLong>> link:{java8-javadoc}/java/util/stream/LongStream.html#min%2D%2D[min]()++ (link:{java9-javadoc}/java/util/stream/LongStream.html#min%2D%2D[java 9])
+* ++[[painless-api-reference-LongStream-noneMatch-1]]boolean link:{java8-javadoc}/java/util/stream/LongStream.html#noneMatch%2Djava.util.function.LongPredicate%2D[noneMatch](<<painless-api-reference-LongPredicate,LongPredicate>>)++ (link:{java9-javadoc}/java/util/stream/LongStream.html#noneMatch%2Djava.util.function.LongPredicate%2D[java 9])
+* ++[[painless-api-reference-LongStream-peek-1]]<<painless-api-reference-LongStream,LongStream>> link:{java8-javadoc}/java/util/stream/LongStream.html#peek%2Djava.util.function.LongConsumer%2D[peek](<<painless-api-reference-LongConsumer,LongConsumer>>)++ (link:{java9-javadoc}/java/util/stream/LongStream.html#peek%2Djava.util.function.LongConsumer%2D[java 9])
+* ++[[painless-api-reference-LongStream-reduce-1]]<<painless-api-reference-OptionalLong,OptionalLong>> link:{java8-javadoc}/java/util/stream/LongStream.html#reduce%2Djava.util.function.LongBinaryOperator%2D[reduce](<<painless-api-reference-LongBinaryOperator,LongBinaryOperator>>)++ (link:{java9-javadoc}/java/util/stream/LongStream.html#reduce%2Djava.util.function.LongBinaryOperator%2D[java 9])
+* ++[[painless-api-reference-LongStream-reduce-2]]long link:{java8-javadoc}/java/util/stream/LongStream.html#reduce%2Dlong%2Djava.util.function.LongBinaryOperator%2D[reduce](long, <<painless-api-reference-LongBinaryOperator,LongBinaryOperator>>)++ (link:{java9-javadoc}/java/util/stream/LongStream.html#reduce%2Dlong%2Djava.util.function.LongBinaryOperator%2D[java 9])
+* ++[[painless-api-reference-LongStream-sequential-0]]<<painless-api-reference-LongStream,LongStream>> link:{java8-javadoc}/java/util/stream/LongStream.html#sequential%2D%2D[sequential]()++ (link:{java9-javadoc}/java/util/stream/LongStream.html#sequential%2D%2D[java 9])
+* ++[[painless-api-reference-LongStream-skip-1]]<<painless-api-reference-LongStream,LongStream>> link:{java8-javadoc}/java/util/stream/LongStream.html#skip%2Dlong%2D[skip](long)++ (link:{java9-javadoc}/java/util/stream/LongStream.html#skip%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LongStream-sorted-0]]<<painless-api-reference-LongStream,LongStream>> link:{java8-javadoc}/java/util/stream/LongStream.html#sorted%2D%2D[sorted]()++ (link:{java9-javadoc}/java/util/stream/LongStream.html#sorted%2D%2D[java 9])
+* ++[[painless-api-reference-LongStream-spliterator-0]]<<painless-api-reference-Spliterator-OfLong,Spliterator.OfLong>> link:{java8-javadoc}/java/util/stream/LongStream.html#spliterator%2D%2D[spliterator]()++ (link:{java9-javadoc}/java/util/stream/LongStream.html#spliterator%2D%2D[java 9])
+* ++[[painless-api-reference-LongStream-sum-0]]long link:{java8-javadoc}/java/util/stream/LongStream.html#sum%2D%2D[sum]()++ (link:{java9-javadoc}/java/util/stream/LongStream.html#sum%2D%2D[java 9])
+* ++[[painless-api-reference-LongStream-summaryStatistics-0]]<<painless-api-reference-LongSummaryStatistics,LongSummaryStatistics>> link:{java8-javadoc}/java/util/stream/LongStream.html#summaryStatistics%2D%2D[summaryStatistics]()++ (link:{java9-javadoc}/java/util/stream/LongStream.html#summaryStatistics%2D%2D[java 9])
+* ++[[painless-api-reference-LongStream-toArray-0]]long[] link:{java8-javadoc}/java/util/stream/LongStream.html#toArray%2D%2D[toArray]()++ (link:{java9-javadoc}/java/util/stream/LongStream.html#toArray%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-BaseStream,BaseStream>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/LongSummaryStatistics.asciidoc b/docs/painless/painless-api-reference/LongSummaryStatistics.asciidoc
new file mode 100644
index 0000000000000..b678d51efa4ac
--- /dev/null
+++ b/docs/painless/painless-api-reference/LongSummaryStatistics.asciidoc
@@ -0,0 +1,14 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-LongSummaryStatistics]]++LongSummaryStatistics++::
+* ++[[painless-api-reference-LongSummaryStatistics-LongSummaryStatistics-0]]link:{java8-javadoc}/java/util/LongSummaryStatistics.html#LongSummaryStatistics%2D%2D[LongSummaryStatistics]()++ (link:{java9-javadoc}/java/util/LongSummaryStatistics.html#LongSummaryStatistics%2D%2D[java 9])
+* ++[[painless-api-reference-LongSummaryStatistics-combine-1]]void link:{java8-javadoc}/java/util/LongSummaryStatistics.html#combine%2Djava.util.LongSummaryStatistics%2D[combine](<<painless-api-reference-LongSummaryStatistics,LongSummaryStatistics>>)++ (link:{java9-javadoc}/java/util/LongSummaryStatistics.html#combine%2Djava.util.LongSummaryStatistics%2D[java 9])
+* ++[[painless-api-reference-LongSummaryStatistics-getAverage-0]]double link:{java8-javadoc}/java/util/LongSummaryStatistics.html#getAverage%2D%2D[getAverage]()++ (link:{java9-javadoc}/java/util/LongSummaryStatistics.html#getAverage%2D%2D[java 9])
+* ++[[painless-api-reference-LongSummaryStatistics-getCount-0]]long link:{java8-javadoc}/java/util/LongSummaryStatistics.html#getCount%2D%2D[getCount]()++ (link:{java9-javadoc}/java/util/LongSummaryStatistics.html#getCount%2D%2D[java 9])
+* ++[[painless-api-reference-LongSummaryStatistics-getMax-0]]long link:{java8-javadoc}/java/util/LongSummaryStatistics.html#getMax%2D%2D[getMax]()++ (link:{java9-javadoc}/java/util/LongSummaryStatistics.html#getMax%2D%2D[java 9])
+* ++[[painless-api-reference-LongSummaryStatistics-getMin-0]]long link:{java8-javadoc}/java/util/LongSummaryStatistics.html#getMin%2D%2D[getMin]()++ (link:{java9-javadoc}/java/util/LongSummaryStatistics.html#getMin%2D%2D[java 9])
+* ++[[painless-api-reference-LongSummaryStatistics-getSum-0]]long link:{java8-javadoc}/java/util/LongSummaryStatistics.html#getSum%2D%2D[getSum]()++ (link:{java9-javadoc}/java/util/LongSummaryStatistics.html#getSum%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-LongConsumer,LongConsumer>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/LongSupplier.asciidoc b/docs/painless/painless-api-reference/LongSupplier.asciidoc
new file mode 100644
index 0000000000000..6693e51f17a47
--- /dev/null
+++ b/docs/painless/painless-api-reference/LongSupplier.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-LongSupplier]]++LongSupplier++::
+* ++[[painless-api-reference-LongSupplier-getAsLong-0]]long link:{java8-javadoc}/java/util/function/LongSupplier.html#getAsLong%2D%2D[getAsLong]()++ (link:{java9-javadoc}/java/util/function/LongSupplier.html#getAsLong%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/LongToDoubleFunction.asciidoc b/docs/painless/painless-api-reference/LongToDoubleFunction.asciidoc
new file mode 100644
index 0000000000000..0bed3833371c7
--- /dev/null
+++ b/docs/painless/painless-api-reference/LongToDoubleFunction.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-LongToDoubleFunction]]++LongToDoubleFunction++::
+* ++[[painless-api-reference-LongToDoubleFunction-applyAsDouble-1]]double link:{java8-javadoc}/java/util/function/LongToDoubleFunction.html#applyAsDouble%2Dlong%2D[applyAsDouble](long)++ (link:{java9-javadoc}/java/util/function/LongToDoubleFunction.html#applyAsDouble%2Dlong%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/LongToIntFunction.asciidoc b/docs/painless/painless-api-reference/LongToIntFunction.asciidoc
new file mode 100644
index 0000000000000..2586ba85511b0
--- /dev/null
+++ b/docs/painless/painless-api-reference/LongToIntFunction.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-LongToIntFunction]]++LongToIntFunction++::
+* ++[[painless-api-reference-LongToIntFunction-applyAsInt-1]]int link:{java8-javadoc}/java/util/function/LongToIntFunction.html#applyAsInt%2Dlong%2D[applyAsInt](long)++ (link:{java9-javadoc}/java/util/function/LongToIntFunction.html#applyAsInt%2Dlong%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/LongUnaryOperator.asciidoc b/docs/painless/painless-api-reference/LongUnaryOperator.asciidoc
new file mode 100644
index 0000000000000..eb84a3688bdeb
--- /dev/null
+++ b/docs/painless/painless-api-reference/LongUnaryOperator.asciidoc
@@ -0,0 +1,11 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-LongUnaryOperator]]++LongUnaryOperator++::
+* ++[[painless-api-reference-LongUnaryOperator-identity-0]]static <<painless-api-reference-LongUnaryOperator,LongUnaryOperator>> link:{java8-javadoc}/java/util/function/LongUnaryOperator.html#identity%2D%2D[identity]()++ (link:{java9-javadoc}/java/util/function/LongUnaryOperator.html#identity%2D%2D[java 9])
+* ++[[painless-api-reference-LongUnaryOperator-andThen-1]]<<painless-api-reference-LongUnaryOperator,LongUnaryOperator>> link:{java8-javadoc}/java/util/function/LongUnaryOperator.html#andThen%2Djava.util.function.LongUnaryOperator%2D[andThen](<<painless-api-reference-LongUnaryOperator,LongUnaryOperator>>)++ (link:{java9-javadoc}/java/util/function/LongUnaryOperator.html#andThen%2Djava.util.function.LongUnaryOperator%2D[java 9])
+* ++[[painless-api-reference-LongUnaryOperator-applyAsLong-1]]long link:{java8-javadoc}/java/util/function/LongUnaryOperator.html#applyAsLong%2Dlong%2D[applyAsLong](long)++ (link:{java9-javadoc}/java/util/function/LongUnaryOperator.html#applyAsLong%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LongUnaryOperator-compose-1]]<<painless-api-reference-LongUnaryOperator,LongUnaryOperator>> link:{java8-javadoc}/java/util/function/LongUnaryOperator.html#compose%2Djava.util.function.LongUnaryOperator%2D[compose](<<painless-api-reference-LongUnaryOperator,LongUnaryOperator>>)++ (link:{java9-javadoc}/java/util/function/LongUnaryOperator.html#compose%2Djava.util.function.LongUnaryOperator%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Map.Entry.asciidoc b/docs/painless/painless-api-reference/Map.Entry.asciidoc
new file mode 100644
index 0000000000000..bf40736759eea
--- /dev/null
+++ b/docs/painless/painless-api-reference/Map.Entry.asciidoc
@@ -0,0 +1,16 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Map-Entry]]++Map.Entry++::
+* ++[[painless-api-reference-Map-Entry-comparingByKey-0]]static <<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/Map.Entry.html#comparingByKey%2D%2D[comparingByKey]()++ (link:{java9-javadoc}/java/util/Map.Entry.html#comparingByKey%2D%2D[java 9])
+* ++[[painless-api-reference-Map-Entry-comparingByKey-1]]static <<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/Map.Entry.html#comparingByKey%2Djava.util.Comparator%2D[comparingByKey](<<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/Map.Entry.html#comparingByKey%2Djava.util.Comparator%2D[java 9])
+* ++[[painless-api-reference-Map-Entry-comparingByValue-0]]static <<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/Map.Entry.html#comparingByValue%2D%2D[comparingByValue]()++ (link:{java9-javadoc}/java/util/Map.Entry.html#comparingByValue%2D%2D[java 9])
+* ++[[painless-api-reference-Map-Entry-comparingByValue-1]]static <<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/Map.Entry.html#comparingByValue%2Djava.util.Comparator%2D[comparingByValue](<<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/Map.Entry.html#comparingByValue%2Djava.util.Comparator%2D[java 9])
+* ++[[painless-api-reference-Map-Entry-equals-1]]boolean link:{java8-javadoc}/java/util/Map.Entry.html#equals%2Djava.lang.Object%2D[equals](<<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/util/Map.Entry.html#equals%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Map-Entry-getKey-0]]def link:{java8-javadoc}/java/util/Map.Entry.html#getKey%2D%2D[getKey]()++ (link:{java9-javadoc}/java/util/Map.Entry.html#getKey%2D%2D[java 9])
+* ++[[painless-api-reference-Map-Entry-getValue-0]]def link:{java8-javadoc}/java/util/Map.Entry.html#getValue%2D%2D[getValue]()++ (link:{java9-javadoc}/java/util/Map.Entry.html#getValue%2D%2D[java 9])
+* ++[[painless-api-reference-Map-Entry-hashCode-0]]int link:{java8-javadoc}/java/util/Map.Entry.html#hashCode%2D%2D[hashCode]()++ (link:{java9-javadoc}/java/util/Map.Entry.html#hashCode%2D%2D[java 9])
+* ++[[painless-api-reference-Map-Entry-setValue-1]]def link:{java8-javadoc}/java/util/Map.Entry.html#setValue%2Djava.lang.Object%2D[setValue](def)++ (link:{java9-javadoc}/java/util/Map.Entry.html#setValue%2Djava.lang.Object%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Map.asciidoc b/docs/painless/painless-api-reference/Map.asciidoc
new file mode 100644
index 0000000000000..a32c9c137d8cd
--- /dev/null
+++ b/docs/painless/painless-api-reference/Map.asciidoc
@@ -0,0 +1,42 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Map]]++Map++::
+* ++[[painless-api-reference-Map-clear-0]]void link:{java8-javadoc}/java/util/Map.html#clear%2D%2D[clear]()++ (link:{java9-javadoc}/java/util/Map.html#clear%2D%2D[java 9])
+* ++[[painless-api-reference-Map-collect-1]]<<painless-api-reference-List,List>> link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#collect%2Djava.util.Map%2Djava.util.function.BiFunction%2D[collect](<<painless-api-reference-BiFunction,BiFunction>>)++
+* ++[[painless-api-reference-Map-collect-2]]def link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#collect%2Djava.util.Map%2Djava.util.Collection%2Djava.util.function.BiFunction%2D[collect](<<painless-api-reference-Collection,Collection>>, <<painless-api-reference-BiFunction,BiFunction>>)++
+* ++[[painless-api-reference-Map-compute-2]]def link:{java8-javadoc}/java/util/Map.html#compute%2Djava.lang.Object%2Djava.util.function.BiFunction%2D[compute](def, <<painless-api-reference-BiFunction,BiFunction>>)++ (link:{java9-javadoc}/java/util/Map.html#compute%2Djava.lang.Object%2Djava.util.function.BiFunction%2D[java 9])
+* ++[[painless-api-reference-Map-computeIfAbsent-2]]def link:{java8-javadoc}/java/util/Map.html#computeIfAbsent%2Djava.lang.Object%2Djava.util.function.Function%2D[computeIfAbsent](def, <<painless-api-reference-Function,Function>>)++ (link:{java9-javadoc}/java/util/Map.html#computeIfAbsent%2Djava.lang.Object%2Djava.util.function.Function%2D[java 9])
+* ++[[painless-api-reference-Map-computeIfPresent-2]]def link:{java8-javadoc}/java/util/Map.html#computeIfPresent%2Djava.lang.Object%2Djava.util.function.BiFunction%2D[computeIfPresent](def, <<painless-api-reference-BiFunction,BiFunction>>)++ (link:{java9-javadoc}/java/util/Map.html#computeIfPresent%2Djava.lang.Object%2Djava.util.function.BiFunction%2D[java 9])
+* ++[[painless-api-reference-Map-containsKey-1]]boolean link:{java8-javadoc}/java/util/Map.html#containsKey%2Djava.lang.Object%2D[containsKey](def)++ (link:{java9-javadoc}/java/util/Map.html#containsKey%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Map-containsValue-1]]boolean link:{java8-javadoc}/java/util/Map.html#containsValue%2Djava.lang.Object%2D[containsValue](def)++ (link:{java9-javadoc}/java/util/Map.html#containsValue%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Map-count-1]]int link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#count%2Djava.util.Map%2Djava.util.function.BiPredicate%2D[count](<<painless-api-reference-BiPredicate,BiPredicate>>)++
+* ++[[painless-api-reference-Map-each-1]]def link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#each%2Djava.util.Map%2Djava.util.function.BiConsumer%2D[each](<<painless-api-reference-BiConsumer,BiConsumer>>)++
+* ++[[painless-api-reference-Map-entrySet-0]]<<painless-api-reference-Set,Set>> link:{java8-javadoc}/java/util/Map.html#entrySet%2D%2D[entrySet]()++ (link:{java9-javadoc}/java/util/Map.html#entrySet%2D%2D[java 9])
+* ++[[painless-api-reference-Map-equals-1]]boolean link:{java8-javadoc}/java/util/Map.html#equals%2Djava.lang.Object%2D[equals](<<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/util/Map.html#equals%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Map-every-1]]boolean link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#every%2Djava.util.Map%2Djava.util.function.BiPredicate%2D[every](<<painless-api-reference-BiPredicate,BiPredicate>>)++
+* ++[[painless-api-reference-Map-find-1]]<<painless-api-reference-Map-Entry,Map.Entry>> link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#find%2Djava.util.Map%2Djava.util.function.BiPredicate%2D[find](<<painless-api-reference-BiPredicate,BiPredicate>>)++
+* ++[[painless-api-reference-Map-findAll-1]]<<painless-api-reference-Map,Map>> link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#findAll%2Djava.util.Map%2Djava.util.function.BiPredicate%2D[findAll](<<painless-api-reference-BiPredicate,BiPredicate>>)++
+* ++[[painless-api-reference-Map-findResult-1]]def link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#findResult%2Djava.util.Map%2Djava.util.function.BiFunction%2D[findResult](<<painless-api-reference-BiFunction,BiFunction>>)++
+* ++[[painless-api-reference-Map-findResult-2]]def link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#findResult%2Djava.util.Map%2Djava.lang.Object%2Djava.util.function.BiFunction%2D[findResult](def, <<painless-api-reference-BiFunction,BiFunction>>)++
+* ++[[painless-api-reference-Map-findResults-1]]<<painless-api-reference-List,List>> link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#findResults%2Djava.util.Map%2Djava.util.function.BiFunction%2D[findResults](<<painless-api-reference-BiFunction,BiFunction>>)++
+* ++[[painless-api-reference-Map-forEach-1]]void link:{java8-javadoc}/java/util/Map.html#forEach%2Djava.util.function.BiConsumer%2D[forEach](<<painless-api-reference-BiConsumer,BiConsumer>>)++ (link:{java9-javadoc}/java/util/Map.html#forEach%2Djava.util.function.BiConsumer%2D[java 9])
+* ++[[painless-api-reference-Map-get-1]]def link:{java8-javadoc}/java/util/Map.html#get%2Djava.lang.Object%2D[get](def)++ (link:{java9-javadoc}/java/util/Map.html#get%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Map-getOrDefault-2]]def link:{java8-javadoc}/java/util/Map.html#getOrDefault%2Djava.lang.Object%2Djava.lang.Object%2D[getOrDefault](def, def)++ (link:{java9-javadoc}/java/util/Map.html#getOrDefault%2Djava.lang.Object%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Map-groupBy-1]]<<painless-api-reference-Map,Map>> link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#groupBy%2Djava.util.Map%2Djava.util.function.BiFunction%2D[groupBy](<<painless-api-reference-BiFunction,BiFunction>>)++
+* ++[[painless-api-reference-Map-isEmpty-0]]boolean link:{java8-javadoc}/java/util/Map.html#isEmpty%2D%2D[isEmpty]()++ (link:{java9-javadoc}/java/util/Map.html#isEmpty%2D%2D[java 9])
+* ++[[painless-api-reference-Map-keySet-0]]<<painless-api-reference-Set,Set>> link:{java8-javadoc}/java/util/Map.html#keySet%2D%2D[keySet]()++ (link:{java9-javadoc}/java/util/Map.html#keySet%2D%2D[java 9])
+* ++[[painless-api-reference-Map-merge-3]]def link:{java8-javadoc}/java/util/Map.html#merge%2Djava.lang.Object%2Djava.lang.Object%2Djava.util.function.BiFunction%2D[merge](def, def, <<painless-api-reference-BiFunction,BiFunction>>)++ (link:{java9-javadoc}/java/util/Map.html#merge%2Djava.lang.Object%2Djava.lang.Object%2Djava.util.function.BiFunction%2D[java 9])
+* ++[[painless-api-reference-Map-put-2]]def link:{java8-javadoc}/java/util/Map.html#put%2Djava.lang.Object%2Djava.lang.Object%2D[put](def, def)++ (link:{java9-javadoc}/java/util/Map.html#put%2Djava.lang.Object%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Map-putAll-1]]void link:{java8-javadoc}/java/util/Map.html#putAll%2Djava.util.Map%2D[putAll](<<painless-api-reference-Map,Map>>)++ (link:{java9-javadoc}/java/util/Map.html#putAll%2Djava.util.Map%2D[java 9])
+* ++[[painless-api-reference-Map-putIfAbsent-2]]def link:{java8-javadoc}/java/util/Map.html#putIfAbsent%2Djava.lang.Object%2Djava.lang.Object%2D[putIfAbsent](def, def)++ (link:{java9-javadoc}/java/util/Map.html#putIfAbsent%2Djava.lang.Object%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Map-remove-1]]def link:{java8-javadoc}/java/util/Map.html#remove%2Djava.lang.Object%2D[remove](def)++ (link:{java9-javadoc}/java/util/Map.html#remove%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Map-remove-2]]boolean link:{java8-javadoc}/java/util/Map.html#remove%2Djava.lang.Object%2Djava.lang.Object%2D[remove](def, def)++ (link:{java9-javadoc}/java/util/Map.html#remove%2Djava.lang.Object%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Map-replace-2]]def link:{java8-javadoc}/java/util/Map.html#replace%2Djava.lang.Object%2Djava.lang.Object%2D[replace](def, def)++ (link:{java9-javadoc}/java/util/Map.html#replace%2Djava.lang.Object%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Map-replace-3]]boolean link:{java8-javadoc}/java/util/Map.html#replace%2Djava.lang.Object%2Djava.lang.Object%2Djava.lang.Object%2D[replace](def, def, def)++ (link:{java9-javadoc}/java/util/Map.html#replace%2Djava.lang.Object%2Djava.lang.Object%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Map-replaceAll-1]]void link:{java8-javadoc}/java/util/Map.html#replaceAll%2Djava.util.function.BiFunction%2D[replaceAll](<<painless-api-reference-BiFunction,BiFunction>>)++ (link:{java9-javadoc}/java/util/Map.html#replaceAll%2Djava.util.function.BiFunction%2D[java 9])
+* ++[[painless-api-reference-Map-size-0]]int link:{java8-javadoc}/java/util/Map.html#size%2D%2D[size]()++ (link:{java9-javadoc}/java/util/Map.html#size%2D%2D[java 9])
+* ++[[painless-api-reference-Map-values-0]]<<painless-api-reference-Collection,Collection>> link:{java8-javadoc}/java/util/Map.html#values%2D%2D[values]()++ (link:{java9-javadoc}/java/util/Map.html#values%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Matcher.asciidoc b/docs/painless/painless-api-reference/Matcher.asciidoc
new file mode 100644
index 0000000000000..85198e0bc8db1
--- /dev/null
+++ b/docs/painless/painless-api-reference/Matcher.asciidoc
@@ -0,0 +1,34 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Matcher]]++Matcher++::
+* ++[[painless-api-reference-Matcher-quoteReplacement-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/regex/Matcher.html#quoteReplacement%2Djava.lang.String%2D[quoteReplacement](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/regex/Matcher.html#quoteReplacement%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Matcher-end-0]]int link:{java8-javadoc}/java/util/regex/Matcher.html#end%2D%2D[end]()++ (link:{java9-javadoc}/java/util/regex/Matcher.html#end%2D%2D[java 9])
+* ++[[painless-api-reference-Matcher-end-1]]int link:{java8-javadoc}/java/util/regex/Matcher.html#end%2Dint%2D[end](int)++ (link:{java9-javadoc}/java/util/regex/Matcher.html#end%2Dint%2D[java 9])
+* ++[[painless-api-reference-Matcher-find-0]]boolean link:{java8-javadoc}/java/util/regex/Matcher.html#find%2D%2D[find]()++ (link:{java9-javadoc}/java/util/regex/Matcher.html#find%2D%2D[java 9])
+* ++[[painless-api-reference-Matcher-find-1]]boolean link:{java8-javadoc}/java/util/regex/Matcher.html#find%2Dint%2D[find](int)++ (link:{java9-javadoc}/java/util/regex/Matcher.html#find%2Dint%2D[java 9])
+* ++[[painless-api-reference-Matcher-group-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/regex/Matcher.html#group%2D%2D[group]()++ (link:{java9-javadoc}/java/util/regex/Matcher.html#group%2D%2D[java 9])
+* ++[[painless-api-reference-Matcher-group-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/regex/Matcher.html#group%2Dint%2D[group](int)++ (link:{java9-javadoc}/java/util/regex/Matcher.html#group%2Dint%2D[java 9])
+* ++[[painless-api-reference-Matcher-groupCount-0]]int link:{java8-javadoc}/java/util/regex/Matcher.html#groupCount%2D%2D[groupCount]()++ (link:{java9-javadoc}/java/util/regex/Matcher.html#groupCount%2D%2D[java 9])
+* ++[[painless-api-reference-Matcher-hasAnchoringBounds-0]]boolean link:{java8-javadoc}/java/util/regex/Matcher.html#hasAnchoringBounds%2D%2D[hasAnchoringBounds]()++ (link:{java9-javadoc}/java/util/regex/Matcher.html#hasAnchoringBounds%2D%2D[java 9])
+* ++[[painless-api-reference-Matcher-hasTransparentBounds-0]]boolean link:{java8-javadoc}/java/util/regex/Matcher.html#hasTransparentBounds%2D%2D[hasTransparentBounds]()++ (link:{java9-javadoc}/java/util/regex/Matcher.html#hasTransparentBounds%2D%2D[java 9])
+* ++[[painless-api-reference-Matcher-hitEnd-0]]boolean link:{java8-javadoc}/java/util/regex/Matcher.html#hitEnd%2D%2D[hitEnd]()++ (link:{java9-javadoc}/java/util/regex/Matcher.html#hitEnd%2D%2D[java 9])
+* ++[[painless-api-reference-Matcher-lookingAt-0]]boolean link:{java8-javadoc}/java/util/regex/Matcher.html#lookingAt%2D%2D[lookingAt]()++ (link:{java9-javadoc}/java/util/regex/Matcher.html#lookingAt%2D%2D[java 9])
+* ++[[painless-api-reference-Matcher-matches-0]]boolean link:{java8-javadoc}/java/util/regex/Matcher.html#matches%2D%2D[matches]()++ (link:{java9-javadoc}/java/util/regex/Matcher.html#matches%2D%2D[java 9])
+* ++[[painless-api-reference-Matcher-namedGroup-1]]<<painless-api-reference-String,String>> link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#namedGroup%2Djava.util.regex.Matcher%2Djava.lang.String%2D[namedGroup](<<painless-api-reference-String,String>>)++
+* ++[[painless-api-reference-Matcher-pattern-0]]<<painless-api-reference-Pattern,Pattern>> link:{java8-javadoc}/java/util/regex/Matcher.html#pattern%2D%2D[pattern]()++ (link:{java9-javadoc}/java/util/regex/Matcher.html#pattern%2D%2D[java 9])
+* ++[[painless-api-reference-Matcher-region-2]]<<painless-api-reference-Matcher,Matcher>> link:{java8-javadoc}/java/util/regex/Matcher.html#region%2Dint%2Dint%2D[region](int, int)++ (link:{java9-javadoc}/java/util/regex/Matcher.html#region%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Matcher-regionEnd-0]]int link:{java8-javadoc}/java/util/regex/Matcher.html#regionEnd%2D%2D[regionEnd]()++ (link:{java9-javadoc}/java/util/regex/Matcher.html#regionEnd%2D%2D[java 9])
+* ++[[painless-api-reference-Matcher-regionStart-0]]int link:{java8-javadoc}/java/util/regex/Matcher.html#regionStart%2D%2D[regionStart]()++ (link:{java9-javadoc}/java/util/regex/Matcher.html#regionStart%2D%2D[java 9])
+* ++[[painless-api-reference-Matcher-replaceAll-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/regex/Matcher.html#replaceAll%2Djava.lang.String%2D[replaceAll](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/regex/Matcher.html#replaceAll%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Matcher-replaceFirst-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/regex/Matcher.html#replaceFirst%2Djava.lang.String%2D[replaceFirst](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/regex/Matcher.html#replaceFirst%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Matcher-requireEnd-0]]boolean link:{java8-javadoc}/java/util/regex/Matcher.html#requireEnd%2D%2D[requireEnd]()++ (link:{java9-javadoc}/java/util/regex/Matcher.html#requireEnd%2D%2D[java 9])
+* ++[[painless-api-reference-Matcher-reset-0]]<<painless-api-reference-Matcher,Matcher>> link:{java8-javadoc}/java/util/regex/Matcher.html#reset%2D%2D[reset]()++ (link:{java9-javadoc}/java/util/regex/Matcher.html#reset%2D%2D[java 9])
+* ++[[painless-api-reference-Matcher-start-0]]int link:{java8-javadoc}/java/util/regex/Matcher.html#start%2D%2D[start]()++ (link:{java9-javadoc}/java/util/regex/Matcher.html#start%2D%2D[java 9])
+* ++[[painless-api-reference-Matcher-start-1]]int link:{java8-javadoc}/java/util/regex/Matcher.html#start%2Dint%2D[start](int)++ (link:{java9-javadoc}/java/util/regex/Matcher.html#start%2Dint%2D[java 9])
+* ++[[painless-api-reference-Matcher-useAnchoringBounds-1]]<<painless-api-reference-Matcher,Matcher>> link:{java8-javadoc}/java/util/regex/Matcher.html#useAnchoringBounds%2Dboolean%2D[useAnchoringBounds](boolean)++ (link:{java9-javadoc}/java/util/regex/Matcher.html#useAnchoringBounds%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-Matcher-usePattern-1]]<<painless-api-reference-Matcher,Matcher>> link:{java8-javadoc}/java/util/regex/Matcher.html#usePattern%2Djava.util.regex.Pattern%2D[usePattern](<<painless-api-reference-Pattern,Pattern>>)++ (link:{java9-javadoc}/java/util/regex/Matcher.html#usePattern%2Djava.util.regex.Pattern%2D[java 9])
+* ++[[painless-api-reference-Matcher-useTransparentBounds-1]]<<painless-api-reference-Matcher,Matcher>> link:{java8-javadoc}/java/util/regex/Matcher.html#useTransparentBounds%2Dboolean%2D[useTransparentBounds](boolean)++ (link:{java9-javadoc}/java/util/regex/Matcher.html#useTransparentBounds%2Dboolean%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Math.asciidoc b/docs/painless/painless-api-reference/Math.asciidoc
new file mode 100644
index 0000000000000..4b8322c15fc50
--- /dev/null
+++ b/docs/painless/painless-api-reference/Math.asciidoc
@@ -0,0 +1,46 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Math]]++Math++::
+** [[painless-api-reference-Math-E]]static double link:{java8-javadoc}/java/lang/Math.html#E[E] (link:{java9-javadoc}/java/lang/Math.html#E[java 9])
+** [[painless-api-reference-Math-PI]]static double link:{java8-javadoc}/java/lang/Math.html#PI[PI] (link:{java9-javadoc}/java/lang/Math.html#PI[java 9])
+* ++[[painless-api-reference-Math-IEEEremainder-2]]static double link:{java8-javadoc}/java/lang/Math.html#IEEEremainder%2Ddouble%2Ddouble%2D[IEEEremainder](double, double)++ (link:{java9-javadoc}/java/lang/Math.html#IEEEremainder%2Ddouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-abs-1]]static double link:{java8-javadoc}/java/lang/Math.html#abs%2Ddouble%2D[abs](double)++ (link:{java9-javadoc}/java/lang/Math.html#abs%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-acos-1]]static double link:{java8-javadoc}/java/lang/Math.html#acos%2Ddouble%2D[acos](double)++ (link:{java9-javadoc}/java/lang/Math.html#acos%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-asin-1]]static double link:{java8-javadoc}/java/lang/Math.html#asin%2Ddouble%2D[asin](double)++ (link:{java9-javadoc}/java/lang/Math.html#asin%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-atan-1]]static double link:{java8-javadoc}/java/lang/Math.html#atan%2Ddouble%2D[atan](double)++ (link:{java9-javadoc}/java/lang/Math.html#atan%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-atan2-2]]static double link:{java8-javadoc}/java/lang/Math.html#atan2%2Ddouble%2Ddouble%2D[atan2](double, double)++ (link:{java9-javadoc}/java/lang/Math.html#atan2%2Ddouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-cbrt-1]]static double link:{java8-javadoc}/java/lang/Math.html#cbrt%2Ddouble%2D[cbrt](double)++ (link:{java9-javadoc}/java/lang/Math.html#cbrt%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-ceil-1]]static double link:{java8-javadoc}/java/lang/Math.html#ceil%2Ddouble%2D[ceil](double)++ (link:{java9-javadoc}/java/lang/Math.html#ceil%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-copySign-2]]static double link:{java8-javadoc}/java/lang/Math.html#copySign%2Ddouble%2Ddouble%2D[copySign](double, double)++ (link:{java9-javadoc}/java/lang/Math.html#copySign%2Ddouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-cos-1]]static double link:{java8-javadoc}/java/lang/Math.html#cos%2Ddouble%2D[cos](double)++ (link:{java9-javadoc}/java/lang/Math.html#cos%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-cosh-1]]static double link:{java8-javadoc}/java/lang/Math.html#cosh%2Ddouble%2D[cosh](double)++ (link:{java9-javadoc}/java/lang/Math.html#cosh%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-exp-1]]static double link:{java8-javadoc}/java/lang/Math.html#exp%2Ddouble%2D[exp](double)++ (link:{java9-javadoc}/java/lang/Math.html#exp%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-expm1-1]]static double link:{java8-javadoc}/java/lang/Math.html#expm1%2Ddouble%2D[expm1](double)++ (link:{java9-javadoc}/java/lang/Math.html#expm1%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-floor-1]]static double link:{java8-javadoc}/java/lang/Math.html#floor%2Ddouble%2D[floor](double)++ (link:{java9-javadoc}/java/lang/Math.html#floor%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-hypot-2]]static double link:{java8-javadoc}/java/lang/Math.html#hypot%2Ddouble%2Ddouble%2D[hypot](double, double)++ (link:{java9-javadoc}/java/lang/Math.html#hypot%2Ddouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-log-1]]static double link:{java8-javadoc}/java/lang/Math.html#log%2Ddouble%2D[log](double)++ (link:{java9-javadoc}/java/lang/Math.html#log%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-log10-1]]static double link:{java8-javadoc}/java/lang/Math.html#log10%2Ddouble%2D[log10](double)++ (link:{java9-javadoc}/java/lang/Math.html#log10%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-log1p-1]]static double link:{java8-javadoc}/java/lang/Math.html#log1p%2Ddouble%2D[log1p](double)++ (link:{java9-javadoc}/java/lang/Math.html#log1p%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-max-2]]static double link:{java8-javadoc}/java/lang/Math.html#max%2Ddouble%2Ddouble%2D[max](double, double)++ (link:{java9-javadoc}/java/lang/Math.html#max%2Ddouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-min-2]]static double link:{java8-javadoc}/java/lang/Math.html#min%2Ddouble%2Ddouble%2D[min](double, double)++ (link:{java9-javadoc}/java/lang/Math.html#min%2Ddouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-nextAfter-2]]static double link:{java8-javadoc}/java/lang/Math.html#nextAfter%2Ddouble%2Ddouble%2D[nextAfter](double, double)++ (link:{java9-javadoc}/java/lang/Math.html#nextAfter%2Ddouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-nextDown-1]]static double link:{java8-javadoc}/java/lang/Math.html#nextDown%2Ddouble%2D[nextDown](double)++ (link:{java9-javadoc}/java/lang/Math.html#nextDown%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-nextUp-1]]static double link:{java8-javadoc}/java/lang/Math.html#nextUp%2Ddouble%2D[nextUp](double)++ (link:{java9-javadoc}/java/lang/Math.html#nextUp%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-pow-2]]static double link:{java8-javadoc}/java/lang/Math.html#pow%2Ddouble%2Ddouble%2D[pow](double, double)++ (link:{java9-javadoc}/java/lang/Math.html#pow%2Ddouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-random-0]]static double link:{java8-javadoc}/java/lang/Math.html#random%2D%2D[random]()++ (link:{java9-javadoc}/java/lang/Math.html#random%2D%2D[java 9])
+* ++[[painless-api-reference-Math-rint-1]]static double link:{java8-javadoc}/java/lang/Math.html#rint%2Ddouble%2D[rint](double)++ (link:{java9-javadoc}/java/lang/Math.html#rint%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-round-1]]static long link:{java8-javadoc}/java/lang/Math.html#round%2Ddouble%2D[round](double)++ (link:{java9-javadoc}/java/lang/Math.html#round%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-scalb-2]]static double link:{java8-javadoc}/java/lang/Math.html#scalb%2Ddouble%2Dint%2D[scalb](double, int)++ (link:{java9-javadoc}/java/lang/Math.html#scalb%2Ddouble%2Dint%2D[java 9])
+* ++[[painless-api-reference-Math-signum-1]]static double link:{java8-javadoc}/java/lang/Math.html#signum%2Ddouble%2D[signum](double)++ (link:{java9-javadoc}/java/lang/Math.html#signum%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-sin-1]]static double link:{java8-javadoc}/java/lang/Math.html#sin%2Ddouble%2D[sin](double)++ (link:{java9-javadoc}/java/lang/Math.html#sin%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-sinh-1]]static double link:{java8-javadoc}/java/lang/Math.html#sinh%2Ddouble%2D[sinh](double)++ (link:{java9-javadoc}/java/lang/Math.html#sinh%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-sqrt-1]]static double link:{java8-javadoc}/java/lang/Math.html#sqrt%2Ddouble%2D[sqrt](double)++ (link:{java9-javadoc}/java/lang/Math.html#sqrt%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-tan-1]]static double link:{java8-javadoc}/java/lang/Math.html#tan%2Ddouble%2D[tan](double)++ (link:{java9-javadoc}/java/lang/Math.html#tan%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-tanh-1]]static double link:{java8-javadoc}/java/lang/Math.html#tanh%2Ddouble%2D[tanh](double)++ (link:{java9-javadoc}/java/lang/Math.html#tanh%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-toDegrees-1]]static double link:{java8-javadoc}/java/lang/Math.html#toDegrees%2Ddouble%2D[toDegrees](double)++ (link:{java9-javadoc}/java/lang/Math.html#toDegrees%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-toRadians-1]]static double link:{java8-javadoc}/java/lang/Math.html#toRadians%2Ddouble%2D[toRadians](double)++ (link:{java9-javadoc}/java/lang/Math.html#toRadians%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-ulp-1]]static double link:{java8-javadoc}/java/lang/Math.html#ulp%2Ddouble%2D[ulp](double)++ (link:{java9-javadoc}/java/lang/Math.html#ulp%2Ddouble%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/MathContext.asciidoc b/docs/painless/painless-api-reference/MathContext.asciidoc
new file mode 100644
index 0000000000000..c8167367fd59b
--- /dev/null
+++ b/docs/painless/painless-api-reference/MathContext.asciidoc
@@ -0,0 +1,15 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-MathContext]]++MathContext++::
+** [[painless-api-reference-MathContext-DECIMAL128]]static <<painless-api-reference-MathContext,MathContext>> link:{java8-javadoc}/java/math/MathContext.html#DECIMAL128[DECIMAL128] (link:{java9-javadoc}/java/math/MathContext.html#DECIMAL128[java 9])
+** [[painless-api-reference-MathContext-DECIMAL32]]static <<painless-api-reference-MathContext,MathContext>> link:{java8-javadoc}/java/math/MathContext.html#DECIMAL32[DECIMAL32] (link:{java9-javadoc}/java/math/MathContext.html#DECIMAL32[java 9])
+** [[painless-api-reference-MathContext-DECIMAL64]]static <<painless-api-reference-MathContext,MathContext>> link:{java8-javadoc}/java/math/MathContext.html#DECIMAL64[DECIMAL64] (link:{java9-javadoc}/java/math/MathContext.html#DECIMAL64[java 9])
+** [[painless-api-reference-MathContext-UNLIMITED]]static <<painless-api-reference-MathContext,MathContext>> link:{java8-javadoc}/java/math/MathContext.html#UNLIMITED[UNLIMITED] (link:{java9-javadoc}/java/math/MathContext.html#UNLIMITED[java 9])
+* ++[[painless-api-reference-MathContext-MathContext-1]]link:{java8-javadoc}/java/math/MathContext.html#MathContext%2Dint%2D[MathContext](int)++ (link:{java9-javadoc}/java/math/MathContext.html#MathContext%2Dint%2D[java 9])
+* ++[[painless-api-reference-MathContext-MathContext-2]]link:{java8-javadoc}/java/math/MathContext.html#MathContext%2Dint%2Djava.math.RoundingMode%2D[MathContext](int, <<painless-api-reference-RoundingMode,RoundingMode>>)++ (link:{java9-javadoc}/java/math/MathContext.html#MathContext%2Dint%2Djava.math.RoundingMode%2D[java 9])
+* ++[[painless-api-reference-MathContext-getPrecision-0]]int link:{java8-javadoc}/java/math/MathContext.html#getPrecision%2D%2D[getPrecision]()++ (link:{java9-javadoc}/java/math/MathContext.html#getPrecision%2D%2D[java 9])
+* ++[[painless-api-reference-MathContext-getRoundingMode-0]]<<painless-api-reference-RoundingMode,RoundingMode>> link:{java8-javadoc}/java/math/MathContext.html#getRoundingMode%2D%2D[getRoundingMode]()++ (link:{java9-javadoc}/java/math/MathContext.html#getRoundingMode%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/MessageFormat.Field.asciidoc b/docs/painless/painless-api-reference/MessageFormat.Field.asciidoc
new file mode 100644
index 0000000000000..200d59df17de9
--- /dev/null
+++ b/docs/painless/painless-api-reference/MessageFormat.Field.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-MessageFormat-Field]]++MessageFormat.Field++::
+** [[painless-api-reference-MessageFormat-Field-ARGUMENT]]static <<painless-api-reference-MessageFormat-Field,MessageFormat.Field>> link:{java8-javadoc}/java/text/MessageFormat.Field.html#ARGUMENT[ARGUMENT] (link:{java9-javadoc}/java/text/MessageFormat.Field.html#ARGUMENT[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/MessageFormat.asciidoc b/docs/painless/painless-api-reference/MessageFormat.asciidoc
new file mode 100644
index 0000000000000..eaed777d9f0f7
--- /dev/null
+++ b/docs/painless/painless-api-reference/MessageFormat.asciidoc
@@ -0,0 +1,20 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-MessageFormat]]++MessageFormat++::
+* ++[[painless-api-reference-MessageFormat-format-2]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/text/MessageFormat.html#format%2Djava.lang.String%2Djava.lang.Object:A%2D[format](<<painless-api-reference-String,String>>, <<painless-api-reference-Object,Object>>[])++ (link:{java9-javadoc}/java/text/MessageFormat.html#format%2Djava.lang.String%2Djava.lang.Object:A%2D[java 9])
+* ++[[painless-api-reference-MessageFormat-applyPattern-1]]void link:{java8-javadoc}/java/text/MessageFormat.html#applyPattern%2Djava.lang.String%2D[applyPattern](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/MessageFormat.html#applyPattern%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-MessageFormat-getFormats-0]]<<painless-api-reference-Format,Format>>[] link:{java8-javadoc}/java/text/MessageFormat.html#getFormats%2D%2D[getFormats]()++ (link:{java9-javadoc}/java/text/MessageFormat.html#getFormats%2D%2D[java 9])
+* ++[[painless-api-reference-MessageFormat-getFormatsByArgumentIndex-0]]<<painless-api-reference-Format,Format>>[] link:{java8-javadoc}/java/text/MessageFormat.html#getFormatsByArgumentIndex%2D%2D[getFormatsByArgumentIndex]()++ (link:{java9-javadoc}/java/text/MessageFormat.html#getFormatsByArgumentIndex%2D%2D[java 9])
+* ++[[painless-api-reference-MessageFormat-getLocale-0]]<<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/text/MessageFormat.html#getLocale%2D%2D[getLocale]()++ (link:{java9-javadoc}/java/text/MessageFormat.html#getLocale%2D%2D[java 9])
+* ++[[painless-api-reference-MessageFormat-parse-1]]<<painless-api-reference-Object,Object>>[] link:{java8-javadoc}/java/text/MessageFormat.html#parse%2Djava.lang.String%2D[parse](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/MessageFormat.html#parse%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-MessageFormat-parse-2]]<<painless-api-reference-Object,Object>>[] link:{java8-javadoc}/java/text/MessageFormat.html#parse%2Djava.lang.String%2Djava.text.ParsePosition%2D[parse](<<painless-api-reference-String,String>>, <<painless-api-reference-ParsePosition,ParsePosition>>)++ (link:{java9-javadoc}/java/text/MessageFormat.html#parse%2Djava.lang.String%2Djava.text.ParsePosition%2D[java 9])
+* ++[[painless-api-reference-MessageFormat-setFormat-2]]void link:{java8-javadoc}/java/text/MessageFormat.html#setFormat%2Dint%2Djava.text.Format%2D[setFormat](int, <<painless-api-reference-Format,Format>>)++ (link:{java9-javadoc}/java/text/MessageFormat.html#setFormat%2Dint%2Djava.text.Format%2D[java 9])
+* ++[[painless-api-reference-MessageFormat-setFormatByArgumentIndex-2]]void link:{java8-javadoc}/java/text/MessageFormat.html#setFormatByArgumentIndex%2Dint%2Djava.text.Format%2D[setFormatByArgumentIndex](int, <<painless-api-reference-Format,Format>>)++ (link:{java9-javadoc}/java/text/MessageFormat.html#setFormatByArgumentIndex%2Dint%2Djava.text.Format%2D[java 9])
+* ++[[painless-api-reference-MessageFormat-setFormats-1]]void link:{java8-javadoc}/java/text/MessageFormat.html#setFormats%2Djava.text.Format:A%2D[setFormats](<<painless-api-reference-Format,Format>>[])++ (link:{java9-javadoc}/java/text/MessageFormat.html#setFormats%2Djava.text.Format:A%2D[java 9])
+* ++[[painless-api-reference-MessageFormat-setFormatsByArgumentIndex-1]]void link:{java8-javadoc}/java/text/MessageFormat.html#setFormatsByArgumentIndex%2Djava.text.Format:A%2D[setFormatsByArgumentIndex](<<painless-api-reference-Format,Format>>[])++ (link:{java9-javadoc}/java/text/MessageFormat.html#setFormatsByArgumentIndex%2Djava.text.Format:A%2D[java 9])
+* ++[[painless-api-reference-MessageFormat-setLocale-1]]void link:{java8-javadoc}/java/text/MessageFormat.html#setLocale%2Djava.util.Locale%2D[setLocale](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/text/MessageFormat.html#setLocale%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-MessageFormat-toPattern-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/text/MessageFormat.html#toPattern%2D%2D[toPattern]()++ (link:{java9-javadoc}/java/text/MessageFormat.html#toPattern%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Format,Format>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/MinguoChronology.asciidoc b/docs/painless/painless-api-reference/MinguoChronology.asciidoc
new file mode 100644
index 0000000000000..9e9afb2cedfa2
--- /dev/null
+++ b/docs/painless/painless-api-reference/MinguoChronology.asciidoc
@@ -0,0 +1,16 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-MinguoChronology]]++MinguoChronology++::
+** [[painless-api-reference-MinguoChronology-INSTANCE]]static <<painless-api-reference-MinguoChronology,MinguoChronology>> link:{java8-javadoc}/java/time/chrono/MinguoChronology.html#INSTANCE[INSTANCE] (link:{java9-javadoc}/java/time/chrono/MinguoChronology.html#INSTANCE[java 9])
+* ++[[painless-api-reference-MinguoChronology-date-1]]<<painless-api-reference-MinguoDate,MinguoDate>> link:{java8-javadoc}/java/time/chrono/MinguoChronology.html#date%2Djava.time.temporal.TemporalAccessor%2D[date](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/chrono/MinguoChronology.html#date%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-MinguoChronology-date-3]]<<painless-api-reference-MinguoDate,MinguoDate>> link:{java8-javadoc}/java/time/chrono/MinguoChronology.html#date%2Dint%2Dint%2Dint%2D[date](int, int, int)++ (link:{java9-javadoc}/java/time/chrono/MinguoChronology.html#date%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-MinguoChronology-date-4]]<<painless-api-reference-MinguoDate,MinguoDate>> link:{java8-javadoc}/java/time/chrono/MinguoChronology.html#date%2Djava.time.chrono.Era%2Dint%2Dint%2Dint%2D[date](<<painless-api-reference-Era,Era>>, int, int, int)++ (link:{java9-javadoc}/java/time/chrono/MinguoChronology.html#date%2Djava.time.chrono.Era%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-MinguoChronology-dateEpochDay-1]]<<painless-api-reference-MinguoDate,MinguoDate>> link:{java8-javadoc}/java/time/chrono/MinguoChronology.html#dateEpochDay%2Dlong%2D[dateEpochDay](long)++ (link:{java9-javadoc}/java/time/chrono/MinguoChronology.html#dateEpochDay%2Dlong%2D[java 9])
+* ++[[painless-api-reference-MinguoChronology-dateYearDay-2]]<<painless-api-reference-MinguoDate,MinguoDate>> link:{java8-javadoc}/java/time/chrono/MinguoChronology.html#dateYearDay%2Dint%2Dint%2D[dateYearDay](int, int)++ (link:{java9-javadoc}/java/time/chrono/MinguoChronology.html#dateYearDay%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-MinguoChronology-dateYearDay-3]]<<painless-api-reference-MinguoDate,MinguoDate>> link:{java8-javadoc}/java/time/chrono/MinguoChronology.html#dateYearDay%2Djava.time.chrono.Era%2Dint%2Dint%2D[dateYearDay](<<painless-api-reference-Era,Era>>, int, int)++ (link:{java9-javadoc}/java/time/chrono/MinguoChronology.html#dateYearDay%2Djava.time.chrono.Era%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-MinguoChronology-eraOf-1]]<<painless-api-reference-MinguoEra,MinguoEra>> link:{java8-javadoc}/java/time/chrono/MinguoChronology.html#eraOf%2Dint%2D[eraOf](int)++ (link:{java9-javadoc}/java/time/chrono/MinguoChronology.html#eraOf%2Dint%2D[java 9])
+* ++[[painless-api-reference-MinguoChronology-resolveDate-2]]<<painless-api-reference-MinguoDate,MinguoDate>> link:{java8-javadoc}/java/time/chrono/MinguoChronology.html#resolveDate%2Djava.util.Map%2Djava.time.format.ResolverStyle%2D[resolveDate](<<painless-api-reference-Map,Map>>, <<painless-api-reference-ResolverStyle,ResolverStyle>>)++ (link:{java9-javadoc}/java/time/chrono/MinguoChronology.html#resolveDate%2Djava.util.Map%2Djava.time.format.ResolverStyle%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Chronology,Chronology>>++
diff --git a/docs/painless/painless-api-reference/MinguoDate.asciidoc b/docs/painless/painless-api-reference/MinguoDate.asciidoc
new file mode 100644
index 0000000000000..38d0b87da762d
--- /dev/null
+++ b/docs/painless/painless-api-reference/MinguoDate.asciidoc
@@ -0,0 +1,17 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-MinguoDate]]++MinguoDate++::
+* ++[[painless-api-reference-MinguoDate-from-1]]static <<painless-api-reference-MinguoDate,MinguoDate>> link:{java8-javadoc}/java/time/chrono/MinguoDate.html#from%2Djava.time.temporal.TemporalAccessor%2D[from](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/chrono/MinguoDate.html#from%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-MinguoDate-of-3]]static <<painless-api-reference-MinguoDate,MinguoDate>> link:{java8-javadoc}/java/time/chrono/MinguoDate.html#of%2Dint%2Dint%2Dint%2D[of](int, int, int)++ (link:{java9-javadoc}/java/time/chrono/MinguoDate.html#of%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-MinguoDate-getChronology-0]]<<painless-api-reference-MinguoChronology,MinguoChronology>> link:{java8-javadoc}/java/time/chrono/MinguoDate.html#getChronology%2D%2D[getChronology]()++ (link:{java9-javadoc}/java/time/chrono/MinguoDate.html#getChronology%2D%2D[java 9])
+* ++[[painless-api-reference-MinguoDate-getEra-0]]<<painless-api-reference-MinguoEra,MinguoEra>> link:{java8-javadoc}/java/time/chrono/MinguoDate.html#getEra%2D%2D[getEra]()++ (link:{java9-javadoc}/java/time/chrono/MinguoDate.html#getEra%2D%2D[java 9])
+* ++[[painless-api-reference-MinguoDate-minus-1]]<<painless-api-reference-MinguoDate,MinguoDate>> link:{java8-javadoc}/java/time/chrono/MinguoDate.html#minus%2Djava.time.temporal.TemporalAmount%2D[minus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/chrono/MinguoDate.html#minus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-MinguoDate-minus-2]]<<painless-api-reference-MinguoDate,MinguoDate>> link:{java8-javadoc}/java/time/chrono/MinguoDate.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[minus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/chrono/MinguoDate.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-MinguoDate-plus-1]]<<painless-api-reference-MinguoDate,MinguoDate>> link:{java8-javadoc}/java/time/chrono/MinguoDate.html#plus%2Djava.time.temporal.TemporalAmount%2D[plus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/chrono/MinguoDate.html#plus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-MinguoDate-plus-2]]<<painless-api-reference-MinguoDate,MinguoDate>> link:{java8-javadoc}/java/time/chrono/MinguoDate.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[plus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/chrono/MinguoDate.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-MinguoDate-with-1]]<<painless-api-reference-MinguoDate,MinguoDate>> link:{java8-javadoc}/java/time/chrono/MinguoDate.html#with%2Djava.time.temporal.TemporalAdjuster%2D[with](<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>)++ (link:{java9-javadoc}/java/time/chrono/MinguoDate.html#with%2Djava.time.temporal.TemporalAdjuster%2D[java 9])
+* ++[[painless-api-reference-MinguoDate-with-2]]<<painless-api-reference-MinguoDate,MinguoDate>> link:{java8-javadoc}/java/time/chrono/MinguoDate.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[with](<<painless-api-reference-TemporalField,TemporalField>>, long)++ (link:{java9-javadoc}/java/time/chrono/MinguoDate.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>>++, ++<<painless-api-reference-Temporal,Temporal>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++, ++<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>++
diff --git a/docs/painless/painless-api-reference/MinguoEra.asciidoc b/docs/painless/painless-api-reference/MinguoEra.asciidoc
new file mode 100644
index 0000000000000..0b32509913d91
--- /dev/null
+++ b/docs/painless/painless-api-reference/MinguoEra.asciidoc
@@ -0,0 +1,13 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-MinguoEra]]++MinguoEra++::
+** [[painless-api-reference-MinguoEra-BEFORE_ROC]]static <<painless-api-reference-MinguoEra,MinguoEra>> link:{java8-javadoc}/java/time/chrono/MinguoEra.html#BEFORE_ROC[BEFORE_ROC] (link:{java9-javadoc}/java/time/chrono/MinguoEra.html#BEFORE_ROC[java 9])
+** [[painless-api-reference-MinguoEra-ROC]]static <<painless-api-reference-MinguoEra,MinguoEra>> link:{java8-javadoc}/java/time/chrono/MinguoEra.html#ROC[ROC] (link:{java9-javadoc}/java/time/chrono/MinguoEra.html#ROC[java 9])
+* ++[[painless-api-reference-MinguoEra-of-1]]static <<painless-api-reference-MinguoEra,MinguoEra>> link:{java8-javadoc}/java/time/chrono/MinguoEra.html#of%2Dint%2D[of](int)++ (link:{java9-javadoc}/java/time/chrono/MinguoEra.html#of%2Dint%2D[java 9])
+* ++[[painless-api-reference-MinguoEra-valueOf-1]]static <<painless-api-reference-MinguoEra,MinguoEra>> link:{java8-javadoc}/java/time/chrono/MinguoEra.html#valueOf%2Djava.lang.String%2D[valueOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/chrono/MinguoEra.html#valueOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-MinguoEra-values-0]]static <<painless-api-reference-MinguoEra,MinguoEra>>[] link:{java8-javadoc}/java/time/chrono/MinguoEra.html#values%2D%2D[values]()++ (link:{java9-javadoc}/java/time/chrono/MinguoEra.html#values%2D%2D[java 9])
+* ++[[painless-api-reference-MinguoEra-getValue-0]]int link:{java8-javadoc}/java/time/chrono/MinguoEra.html#getValue%2D%2D[getValue]()++ (link:{java9-javadoc}/java/time/chrono/MinguoEra.html#getValue%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Enum,Enum>>++, ++<<painless-api-reference-Era,Era>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++, ++<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>++
diff --git a/docs/painless/painless-api-reference/MissingFormatArgumentException.asciidoc b/docs/painless/painless-api-reference/MissingFormatArgumentException.asciidoc
new file mode 100644
index 0000000000000..7c09ad6ba5cb7
--- /dev/null
+++ b/docs/painless/painless-api-reference/MissingFormatArgumentException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-MissingFormatArgumentException]]++MissingFormatArgumentException++::
+* ++[[painless-api-reference-MissingFormatArgumentException-MissingFormatArgumentException-1]]link:{java8-javadoc}/java/util/MissingFormatArgumentException.html#MissingFormatArgumentException%2Djava.lang.String%2D[MissingFormatArgumentException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/MissingFormatArgumentException.html#MissingFormatArgumentException%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-MissingFormatArgumentException-getFormatSpecifier-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/MissingFormatArgumentException.html#getFormatSpecifier%2D%2D[getFormatSpecifier]()++ (link:{java9-javadoc}/java/util/MissingFormatArgumentException.html#getFormatSpecifier%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/MissingFormatWidthException.asciidoc b/docs/painless/painless-api-reference/MissingFormatWidthException.asciidoc
new file mode 100644
index 0000000000000..155637b48dd2e
--- /dev/null
+++ b/docs/painless/painless-api-reference/MissingFormatWidthException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-MissingFormatWidthException]]++MissingFormatWidthException++::
+* ++[[painless-api-reference-MissingFormatWidthException-MissingFormatWidthException-1]]link:{java8-javadoc}/java/util/MissingFormatWidthException.html#MissingFormatWidthException%2Djava.lang.String%2D[MissingFormatWidthException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/MissingFormatWidthException.html#MissingFormatWidthException%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-MissingFormatWidthException-getFormatSpecifier-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/MissingFormatWidthException.html#getFormatSpecifier%2D%2D[getFormatSpecifier]()++ (link:{java9-javadoc}/java/util/MissingFormatWidthException.html#getFormatSpecifier%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/MissingResourceException.asciidoc b/docs/painless/painless-api-reference/MissingResourceException.asciidoc
new file mode 100644
index 0000000000000..a415708a728a6
--- /dev/null
+++ b/docs/painless/painless-api-reference/MissingResourceException.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-MissingResourceException]]++MissingResourceException++::
+* ++[[painless-api-reference-MissingResourceException-MissingResourceException-3]]link:{java8-javadoc}/java/util/MissingResourceException.html#MissingResourceException%2Djava.lang.String%2Djava.lang.String%2Djava.lang.String%2D[MissingResourceException](<<painless-api-reference-String,String>>, <<painless-api-reference-String,String>>, <<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/MissingResourceException.html#MissingResourceException%2Djava.lang.String%2Djava.lang.String%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-MissingResourceException-getClassName-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/MissingResourceException.html#getClassName%2D%2D[getClassName]()++ (link:{java9-javadoc}/java/util/MissingResourceException.html#getClassName%2D%2D[java 9])
+* ++[[painless-api-reference-MissingResourceException-getKey-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/MissingResourceException.html#getKey%2D%2D[getKey]()++ (link:{java9-javadoc}/java/util/MissingResourceException.html#getKey%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Month.asciidoc b/docs/painless/painless-api-reference/Month.asciidoc
new file mode 100644
index 0000000000000..f4eaa9cc12944
--- /dev/null
+++ b/docs/painless/painless-api-reference/Month.asciidoc
@@ -0,0 +1,32 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Month]]++Month++::
+** [[painless-api-reference-Month-APRIL]]static <<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/Month.html#APRIL[APRIL] (link:{java9-javadoc}/java/time/Month.html#APRIL[java 9])
+** [[painless-api-reference-Month-AUGUST]]static <<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/Month.html#AUGUST[AUGUST] (link:{java9-javadoc}/java/time/Month.html#AUGUST[java 9])
+** [[painless-api-reference-Month-DECEMBER]]static <<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/Month.html#DECEMBER[DECEMBER] (link:{java9-javadoc}/java/time/Month.html#DECEMBER[java 9])
+** [[painless-api-reference-Month-FEBRUARY]]static <<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/Month.html#FEBRUARY[FEBRUARY] (link:{java9-javadoc}/java/time/Month.html#FEBRUARY[java 9])
+** [[painless-api-reference-Month-JANUARY]]static <<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/Month.html#JANUARY[JANUARY] (link:{java9-javadoc}/java/time/Month.html#JANUARY[java 9])
+** [[painless-api-reference-Month-JULY]]static <<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/Month.html#JULY[JULY] (link:{java9-javadoc}/java/time/Month.html#JULY[java 9])
+** [[painless-api-reference-Month-JUNE]]static <<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/Month.html#JUNE[JUNE] (link:{java9-javadoc}/java/time/Month.html#JUNE[java 9])
+** [[painless-api-reference-Month-MARCH]]static <<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/Month.html#MARCH[MARCH] (link:{java9-javadoc}/java/time/Month.html#MARCH[java 9])
+** [[painless-api-reference-Month-MAY]]static <<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/Month.html#MAY[MAY] (link:{java9-javadoc}/java/time/Month.html#MAY[java 9])
+** [[painless-api-reference-Month-NOVEMBER]]static <<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/Month.html#NOVEMBER[NOVEMBER] (link:{java9-javadoc}/java/time/Month.html#NOVEMBER[java 9])
+** [[painless-api-reference-Month-OCTOBER]]static <<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/Month.html#OCTOBER[OCTOBER] (link:{java9-javadoc}/java/time/Month.html#OCTOBER[java 9])
+** [[painless-api-reference-Month-SEPTEMBER]]static <<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/Month.html#SEPTEMBER[SEPTEMBER] (link:{java9-javadoc}/java/time/Month.html#SEPTEMBER[java 9])
+* ++[[painless-api-reference-Month-from-1]]static <<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/Month.html#from%2Djava.time.temporal.TemporalAccessor%2D[from](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/Month.html#from%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-Month-of-1]]static <<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/Month.html#of%2Dint%2D[of](int)++ (link:{java9-javadoc}/java/time/Month.html#of%2Dint%2D[java 9])
+* ++[[painless-api-reference-Month-valueOf-1]]static <<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/Month.html#valueOf%2Djava.lang.String%2D[valueOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/Month.html#valueOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Month-values-0]]static <<painless-api-reference-Month,Month>>[] link:{java8-javadoc}/java/time/Month.html#values%2D%2D[values]()++ (link:{java9-javadoc}/java/time/Month.html#values%2D%2D[java 9])
+* ++[[painless-api-reference-Month-firstDayOfYear-1]]int link:{java8-javadoc}/java/time/Month.html#firstDayOfYear%2Dboolean%2D[firstDayOfYear](boolean)++ (link:{java9-javadoc}/java/time/Month.html#firstDayOfYear%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-Month-firstMonthOfQuarter-0]]<<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/Month.html#firstMonthOfQuarter%2D%2D[firstMonthOfQuarter]()++ (link:{java9-javadoc}/java/time/Month.html#firstMonthOfQuarter%2D%2D[java 9])
+* ++[[painless-api-reference-Month-getDisplayName-2]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/Month.html#getDisplayName%2Djava.time.format.TextStyle%2Djava.util.Locale%2D[getDisplayName](<<painless-api-reference-TextStyle,TextStyle>>, <<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/time/Month.html#getDisplayName%2Djava.time.format.TextStyle%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-Month-getValue-0]]int link:{java8-javadoc}/java/time/Month.html#getValue%2D%2D[getValue]()++ (link:{java9-javadoc}/java/time/Month.html#getValue%2D%2D[java 9])
+* ++[[painless-api-reference-Month-length-1]]int link:{java8-javadoc}/java/time/Month.html#length%2Dboolean%2D[length](boolean)++ (link:{java9-javadoc}/java/time/Month.html#length%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-Month-maxLength-0]]int link:{java8-javadoc}/java/time/Month.html#maxLength%2D%2D[maxLength]()++ (link:{java9-javadoc}/java/time/Month.html#maxLength%2D%2D[java 9])
+* ++[[painless-api-reference-Month-minLength-0]]int link:{java8-javadoc}/java/time/Month.html#minLength%2D%2D[minLength]()++ (link:{java9-javadoc}/java/time/Month.html#minLength%2D%2D[java 9])
+* ++[[painless-api-reference-Month-minus-1]]<<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/Month.html#minus%2Dlong%2D[minus](long)++ (link:{java9-javadoc}/java/time/Month.html#minus%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Month-plus-1]]<<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/Month.html#plus%2Dlong%2D[plus](long)++ (link:{java9-javadoc}/java/time/Month.html#plus%2Dlong%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Enum,Enum>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++, ++<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>++
diff --git a/docs/painless/painless-api-reference/MonthDay.asciidoc b/docs/painless/painless-api-reference/MonthDay.asciidoc
new file mode 100644
index 0000000000000..4bb7c675d70da
--- /dev/null
+++ b/docs/painless/painless-api-reference/MonthDay.asciidoc
@@ -0,0 +1,23 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-MonthDay]]++MonthDay++::
+* ++[[painless-api-reference-MonthDay-from-1]]static <<painless-api-reference-MonthDay,MonthDay>> link:{java8-javadoc}/java/time/MonthDay.html#from%2Djava.time.temporal.TemporalAccessor%2D[from](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/MonthDay.html#from%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-MonthDay-of-2]]static <<painless-api-reference-MonthDay,MonthDay>> link:{java8-javadoc}/java/time/MonthDay.html#of%2Dint%2Dint%2D[of](int, int)++ (link:{java9-javadoc}/java/time/MonthDay.html#of%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-MonthDay-parse-1]]static <<painless-api-reference-MonthDay,MonthDay>> link:{java8-javadoc}/java/time/MonthDay.html#parse%2Djava.lang.CharSequence%2D[parse](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/time/MonthDay.html#parse%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-MonthDay-parse-2]]static <<painless-api-reference-MonthDay,MonthDay>> link:{java8-javadoc}/java/time/MonthDay.html#parse%2Djava.lang.CharSequence%2Djava.time.format.DateTimeFormatter%2D[parse](<<painless-api-reference-CharSequence,CharSequence>>, <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>>)++ (link:{java9-javadoc}/java/time/MonthDay.html#parse%2Djava.lang.CharSequence%2Djava.time.format.DateTimeFormatter%2D[java 9])
+* ++[[painless-api-reference-MonthDay-atYear-1]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/MonthDay.html#atYear%2Dint%2D[atYear](int)++ (link:{java9-javadoc}/java/time/MonthDay.html#atYear%2Dint%2D[java 9])
+* ++[[painless-api-reference-MonthDay-compareTo-1]]int link:{java8-javadoc}/java/time/MonthDay.html#compareTo%2Djava.time.MonthDay%2D[compareTo](<<painless-api-reference-MonthDay,MonthDay>>)++ (link:{java9-javadoc}/java/time/MonthDay.html#compareTo%2Djava.time.MonthDay%2D[java 9])
+* ++[[painless-api-reference-MonthDay-format-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/MonthDay.html#format%2Djava.time.format.DateTimeFormatter%2D[format](<<painless-api-reference-DateTimeFormatter,DateTimeFormatter>>)++ (link:{java9-javadoc}/java/time/MonthDay.html#format%2Djava.time.format.DateTimeFormatter%2D[java 9])
+* ++[[painless-api-reference-MonthDay-getDayOfMonth-0]]int link:{java8-javadoc}/java/time/MonthDay.html#getDayOfMonth%2D%2D[getDayOfMonth]()++ (link:{java9-javadoc}/java/time/MonthDay.html#getDayOfMonth%2D%2D[java 9])
+* ++[[painless-api-reference-MonthDay-getMonth-0]]<<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/MonthDay.html#getMonth%2D%2D[getMonth]()++ (link:{java9-javadoc}/java/time/MonthDay.html#getMonth%2D%2D[java 9])
+* ++[[painless-api-reference-MonthDay-getMonthValue-0]]int link:{java8-javadoc}/java/time/MonthDay.html#getMonthValue%2D%2D[getMonthValue]()++ (link:{java9-javadoc}/java/time/MonthDay.html#getMonthValue%2D%2D[java 9])
+* ++[[painless-api-reference-MonthDay-isAfter-1]]boolean link:{java8-javadoc}/java/time/MonthDay.html#isAfter%2Djava.time.MonthDay%2D[isAfter](<<painless-api-reference-MonthDay,MonthDay>>)++ (link:{java9-javadoc}/java/time/MonthDay.html#isAfter%2Djava.time.MonthDay%2D[java 9])
+* ++[[painless-api-reference-MonthDay-isBefore-1]]boolean link:{java8-javadoc}/java/time/MonthDay.html#isBefore%2Djava.time.MonthDay%2D[isBefore](<<painless-api-reference-MonthDay,MonthDay>>)++ (link:{java9-javadoc}/java/time/MonthDay.html#isBefore%2Djava.time.MonthDay%2D[java 9])
+* ++[[painless-api-reference-MonthDay-isValidYear-1]]boolean link:{java8-javadoc}/java/time/MonthDay.html#isValidYear%2Dint%2D[isValidYear](int)++ (link:{java9-javadoc}/java/time/MonthDay.html#isValidYear%2Dint%2D[java 9])
+* ++[[painless-api-reference-MonthDay-with-1]]<<painless-api-reference-MonthDay,MonthDay>> link:{java8-javadoc}/java/time/MonthDay.html#with%2Djava.time.Month%2D[with](<<painless-api-reference-Month,Month>>)++ (link:{java9-javadoc}/java/time/MonthDay.html#with%2Djava.time.Month%2D[java 9])
+* ++[[painless-api-reference-MonthDay-withDayOfMonth-1]]<<painless-api-reference-MonthDay,MonthDay>> link:{java8-javadoc}/java/time/MonthDay.html#withDayOfMonth%2Dint%2D[withDayOfMonth](int)++ (link:{java9-javadoc}/java/time/MonthDay.html#withDayOfMonth%2Dint%2D[java 9])
+* ++[[painless-api-reference-MonthDay-withMonth-1]]<<painless-api-reference-MonthDay,MonthDay>> link:{java8-javadoc}/java/time/MonthDay.html#withMonth%2Dint%2D[withMonth](int)++ (link:{java9-javadoc}/java/time/MonthDay.html#withMonth%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++, ++<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>++
diff --git a/docs/painless/painless-api-reference/NavigableMap.asciidoc b/docs/painless/painless-api-reference/NavigableMap.asciidoc
new file mode 100644
index 0000000000000..507ab8acd61f3
--- /dev/null
+++ b/docs/painless/painless-api-reference/NavigableMap.asciidoc
@@ -0,0 +1,24 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-NavigableMap]]++NavigableMap++::
+* ++[[painless-api-reference-NavigableMap-ceilingEntry-1]]<<painless-api-reference-Map-Entry,Map.Entry>> link:{java8-javadoc}/java/util/NavigableMap.html#ceilingEntry%2Djava.lang.Object%2D[ceilingEntry](def)++ (link:{java9-javadoc}/java/util/NavigableMap.html#ceilingEntry%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-NavigableMap-ceilingKey-1]]def link:{java8-javadoc}/java/util/NavigableMap.html#ceilingKey%2Djava.lang.Object%2D[ceilingKey](def)++ (link:{java9-javadoc}/java/util/NavigableMap.html#ceilingKey%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-NavigableMap-descendingKeySet-0]]<<painless-api-reference-NavigableSet,NavigableSet>> link:{java8-javadoc}/java/util/NavigableMap.html#descendingKeySet%2D%2D[descendingKeySet]()++ (link:{java9-javadoc}/java/util/NavigableMap.html#descendingKeySet%2D%2D[java 9])
+* ++[[painless-api-reference-NavigableMap-descendingMap-0]]<<painless-api-reference-NavigableMap,NavigableMap>> link:{java8-javadoc}/java/util/NavigableMap.html#descendingMap%2D%2D[descendingMap]()++ (link:{java9-javadoc}/java/util/NavigableMap.html#descendingMap%2D%2D[java 9])
+* ++[[painless-api-reference-NavigableMap-firstEntry-0]]<<painless-api-reference-Map-Entry,Map.Entry>> link:{java8-javadoc}/java/util/NavigableMap.html#firstEntry%2D%2D[firstEntry]()++ (link:{java9-javadoc}/java/util/NavigableMap.html#firstEntry%2D%2D[java 9])
+* ++[[painless-api-reference-NavigableMap-floorEntry-1]]<<painless-api-reference-Map-Entry,Map.Entry>> link:{java8-javadoc}/java/util/NavigableMap.html#floorEntry%2Djava.lang.Object%2D[floorEntry](def)++ (link:{java9-javadoc}/java/util/NavigableMap.html#floorEntry%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-NavigableMap-floorKey-1]]def link:{java8-javadoc}/java/util/NavigableMap.html#floorKey%2Djava.lang.Object%2D[floorKey](def)++ (link:{java9-javadoc}/java/util/NavigableMap.html#floorKey%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-NavigableMap-headMap-2]]<<painless-api-reference-NavigableMap,NavigableMap>> link:{java8-javadoc}/java/util/NavigableMap.html#headMap%2Djava.lang.Object%2Dboolean%2D[headMap](def, boolean)++ (link:{java9-javadoc}/java/util/NavigableMap.html#headMap%2Djava.lang.Object%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-NavigableMap-higherEntry-1]]<<painless-api-reference-Map-Entry,Map.Entry>> link:{java8-javadoc}/java/util/NavigableMap.html#higherEntry%2Djava.lang.Object%2D[higherEntry](def)++ (link:{java9-javadoc}/java/util/NavigableMap.html#higherEntry%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-NavigableMap-higherKey-1]]def link:{java8-javadoc}/java/util/NavigableMap.html#higherKey%2Djava.lang.Object%2D[higherKey](def)++ (link:{java9-javadoc}/java/util/NavigableMap.html#higherKey%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-NavigableMap-lastEntry-0]]<<painless-api-reference-Map-Entry,Map.Entry>> link:{java8-javadoc}/java/util/NavigableMap.html#lastEntry%2D%2D[lastEntry]()++ (link:{java9-javadoc}/java/util/NavigableMap.html#lastEntry%2D%2D[java 9])
+* ++[[painless-api-reference-NavigableMap-lowerEntry-1]]<<painless-api-reference-Map-Entry,Map.Entry>> link:{java8-javadoc}/java/util/NavigableMap.html#lowerEntry%2Djava.lang.Object%2D[lowerEntry](def)++ (link:{java9-javadoc}/java/util/NavigableMap.html#lowerEntry%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-NavigableMap-navigableKeySet-0]]<<painless-api-reference-NavigableSet,NavigableSet>> link:{java8-javadoc}/java/util/NavigableMap.html#navigableKeySet%2D%2D[navigableKeySet]()++ (link:{java9-javadoc}/java/util/NavigableMap.html#navigableKeySet%2D%2D[java 9])
+* ++[[painless-api-reference-NavigableMap-pollFirstEntry-0]]<<painless-api-reference-Map-Entry,Map.Entry>> link:{java8-javadoc}/java/util/NavigableMap.html#pollFirstEntry%2D%2D[pollFirstEntry]()++ (link:{java9-javadoc}/java/util/NavigableMap.html#pollFirstEntry%2D%2D[java 9])
+* ++[[painless-api-reference-NavigableMap-pollLastEntry-0]]<<painless-api-reference-Map-Entry,Map.Entry>> link:{java8-javadoc}/java/util/NavigableMap.html#pollLastEntry%2D%2D[pollLastEntry]()++ (link:{java9-javadoc}/java/util/NavigableMap.html#pollLastEntry%2D%2D[java 9])
+* ++[[painless-api-reference-NavigableMap-subMap-4]]<<painless-api-reference-NavigableMap,NavigableMap>> link:{java8-javadoc}/java/util/NavigableMap.html#subMap%2Djava.lang.Object%2Dboolean%2Djava.lang.Object%2Dboolean%2D[subMap](def, boolean, def, boolean)++ (link:{java9-javadoc}/java/util/NavigableMap.html#subMap%2Djava.lang.Object%2Dboolean%2Djava.lang.Object%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-NavigableMap-tailMap-2]]<<painless-api-reference-NavigableMap,NavigableMap>> link:{java8-javadoc}/java/util/NavigableMap.html#tailMap%2Djava.lang.Object%2Dboolean%2D[tailMap](def, boolean)++ (link:{java9-javadoc}/java/util/NavigableMap.html#tailMap%2Djava.lang.Object%2Dboolean%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Map,Map>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-SortedMap,SortedMap>>++
diff --git a/docs/painless/painless-api-reference/NavigableSet.asciidoc b/docs/painless/painless-api-reference/NavigableSet.asciidoc
new file mode 100644
index 0000000000000..15fff25f44d5b
--- /dev/null
+++ b/docs/painless/painless-api-reference/NavigableSet.asciidoc
@@ -0,0 +1,18 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-NavigableSet]]++NavigableSet++::
+* ++[[painless-api-reference-NavigableSet-ceiling-1]]def link:{java8-javadoc}/java/util/NavigableSet.html#ceiling%2Djava.lang.Object%2D[ceiling](def)++ (link:{java9-javadoc}/java/util/NavigableSet.html#ceiling%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-NavigableSet-descendingIterator-0]]<<painless-api-reference-Iterator,Iterator>> link:{java8-javadoc}/java/util/NavigableSet.html#descendingIterator%2D%2D[descendingIterator]()++ (link:{java9-javadoc}/java/util/NavigableSet.html#descendingIterator%2D%2D[java 9])
+* ++[[painless-api-reference-NavigableSet-descendingSet-0]]<<painless-api-reference-NavigableSet,NavigableSet>> link:{java8-javadoc}/java/util/NavigableSet.html#descendingSet%2D%2D[descendingSet]()++ (link:{java9-javadoc}/java/util/NavigableSet.html#descendingSet%2D%2D[java 9])
+* ++[[painless-api-reference-NavigableSet-floor-1]]def link:{java8-javadoc}/java/util/NavigableSet.html#floor%2Djava.lang.Object%2D[floor](def)++ (link:{java9-javadoc}/java/util/NavigableSet.html#floor%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-NavigableSet-headSet-2]]<<painless-api-reference-NavigableSet,NavigableSet>> link:{java8-javadoc}/java/util/NavigableSet.html#headSet%2Djava.lang.Object%2Dboolean%2D[headSet](def, boolean)++ (link:{java9-javadoc}/java/util/NavigableSet.html#headSet%2Djava.lang.Object%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-NavigableSet-higher-1]]def link:{java8-javadoc}/java/util/NavigableSet.html#higher%2Djava.lang.Object%2D[higher](def)++ (link:{java9-javadoc}/java/util/NavigableSet.html#higher%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-NavigableSet-lower-1]]def link:{java8-javadoc}/java/util/NavigableSet.html#lower%2Djava.lang.Object%2D[lower](def)++ (link:{java9-javadoc}/java/util/NavigableSet.html#lower%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-NavigableSet-pollFirst-0]]def link:{java8-javadoc}/java/util/NavigableSet.html#pollFirst%2D%2D[pollFirst]()++ (link:{java9-javadoc}/java/util/NavigableSet.html#pollFirst%2D%2D[java 9])
+* ++[[painless-api-reference-NavigableSet-pollLast-0]]def link:{java8-javadoc}/java/util/NavigableSet.html#pollLast%2D%2D[pollLast]()++ (link:{java9-javadoc}/java/util/NavigableSet.html#pollLast%2D%2D[java 9])
+* ++[[painless-api-reference-NavigableSet-subSet-4]]<<painless-api-reference-NavigableSet,NavigableSet>> link:{java8-javadoc}/java/util/NavigableSet.html#subSet%2Djava.lang.Object%2Dboolean%2Djava.lang.Object%2Dboolean%2D[subSet](def, boolean, def, boolean)++ (link:{java9-javadoc}/java/util/NavigableSet.html#subSet%2Djava.lang.Object%2Dboolean%2Djava.lang.Object%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-NavigableSet-tailSet-2]]<<painless-api-reference-NavigableSet,NavigableSet>> link:{java8-javadoc}/java/util/NavigableSet.html#tailSet%2Djava.lang.Object%2Dboolean%2D[tailSet](def, boolean)++ (link:{java9-javadoc}/java/util/NavigableSet.html#tailSet%2Djava.lang.Object%2Dboolean%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-Set,Set>>++, ++<<painless-api-reference-SortedSet,SortedSet>>++
diff --git a/docs/painless/painless-api-reference/NegativeArraySizeException.asciidoc b/docs/painless/painless-api-reference/NegativeArraySizeException.asciidoc
new file mode 100644
index 0000000000000..78ee0006eca71
--- /dev/null
+++ b/docs/painless/painless-api-reference/NegativeArraySizeException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-NegativeArraySizeException]]++NegativeArraySizeException++::
+* ++[[painless-api-reference-NegativeArraySizeException-NegativeArraySizeException-0]]link:{java8-javadoc}/java/lang/NegativeArraySizeException.html#NegativeArraySizeException%2D%2D[NegativeArraySizeException]()++ (link:{java9-javadoc}/java/lang/NegativeArraySizeException.html#NegativeArraySizeException%2D%2D[java 9])
+* ++[[painless-api-reference-NegativeArraySizeException-NegativeArraySizeException-1]]link:{java8-javadoc}/java/lang/NegativeArraySizeException.html#NegativeArraySizeException%2Djava.lang.String%2D[NegativeArraySizeException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/NegativeArraySizeException.html#NegativeArraySizeException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/NoSuchElementException.asciidoc b/docs/painless/painless-api-reference/NoSuchElementException.asciidoc
new file mode 100644
index 0000000000000..7a28bdfb5dd24
--- /dev/null
+++ b/docs/painless/painless-api-reference/NoSuchElementException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-NoSuchElementException]]++NoSuchElementException++::
+* ++[[painless-api-reference-NoSuchElementException-NoSuchElementException-0]]link:{java8-javadoc}/java/util/NoSuchElementException.html#NoSuchElementException%2D%2D[NoSuchElementException]()++ (link:{java9-javadoc}/java/util/NoSuchElementException.html#NoSuchElementException%2D%2D[java 9])
+* ++[[painless-api-reference-NoSuchElementException-NoSuchElementException-1]]link:{java8-javadoc}/java/util/NoSuchElementException.html#NoSuchElementException%2Djava.lang.String%2D[NoSuchElementException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/NoSuchElementException.html#NoSuchElementException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/NoSuchFieldException.asciidoc b/docs/painless/painless-api-reference/NoSuchFieldException.asciidoc
new file mode 100644
index 0000000000000..6f3e12d16d7d0
--- /dev/null
+++ b/docs/painless/painless-api-reference/NoSuchFieldException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-NoSuchFieldException]]++NoSuchFieldException++::
+* ++[[painless-api-reference-NoSuchFieldException-NoSuchFieldException-0]]link:{java8-javadoc}/java/lang/NoSuchFieldException.html#NoSuchFieldException%2D%2D[NoSuchFieldException]()++ (link:{java9-javadoc}/java/lang/NoSuchFieldException.html#NoSuchFieldException%2D%2D[java 9])
+* ++[[painless-api-reference-NoSuchFieldException-NoSuchFieldException-1]]link:{java8-javadoc}/java/lang/NoSuchFieldException.html#NoSuchFieldException%2Djava.lang.String%2D[NoSuchFieldException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/NoSuchFieldException.html#NoSuchFieldException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/NoSuchMethodException.asciidoc b/docs/painless/painless-api-reference/NoSuchMethodException.asciidoc
new file mode 100644
index 0000000000000..0fec067e9d281
--- /dev/null
+++ b/docs/painless/painless-api-reference/NoSuchMethodException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-NoSuchMethodException]]++NoSuchMethodException++::
+* ++[[painless-api-reference-NoSuchMethodException-NoSuchMethodException-0]]link:{java8-javadoc}/java/lang/NoSuchMethodException.html#NoSuchMethodException%2D%2D[NoSuchMethodException]()++ (link:{java9-javadoc}/java/lang/NoSuchMethodException.html#NoSuchMethodException%2D%2D[java 9])
+* ++[[painless-api-reference-NoSuchMethodException-NoSuchMethodException-1]]link:{java8-javadoc}/java/lang/NoSuchMethodException.html#NoSuchMethodException%2Djava.lang.String%2D[NoSuchMethodException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/NoSuchMethodException.html#NoSuchMethodException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Normalizer.Form.asciidoc b/docs/painless/painless-api-reference/Normalizer.Form.asciidoc
new file mode 100644
index 0000000000000..5255435473004
--- /dev/null
+++ b/docs/painless/painless-api-reference/Normalizer.Form.asciidoc
@@ -0,0 +1,13 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Normalizer-Form]]++Normalizer.Form++::
+** [[painless-api-reference-Normalizer-Form-NFC]]static <<painless-api-reference-Normalizer-Form,Normalizer.Form>> link:{java8-javadoc}/java/text/Normalizer.Form.html#NFC[NFC] (link:{java9-javadoc}/java/text/Normalizer.Form.html#NFC[java 9])
+** [[painless-api-reference-Normalizer-Form-NFD]]static <<painless-api-reference-Normalizer-Form,Normalizer.Form>> link:{java8-javadoc}/java/text/Normalizer.Form.html#NFD[NFD] (link:{java9-javadoc}/java/text/Normalizer.Form.html#NFD[java 9])
+** [[painless-api-reference-Normalizer-Form-NFKC]]static <<painless-api-reference-Normalizer-Form,Normalizer.Form>> link:{java8-javadoc}/java/text/Normalizer.Form.html#NFKC[NFKC] (link:{java9-javadoc}/java/text/Normalizer.Form.html#NFKC[java 9])
+** [[painless-api-reference-Normalizer-Form-NFKD]]static <<painless-api-reference-Normalizer-Form,Normalizer.Form>> link:{java8-javadoc}/java/text/Normalizer.Form.html#NFKD[NFKD] (link:{java9-javadoc}/java/text/Normalizer.Form.html#NFKD[java 9])
+* ++[[painless-api-reference-Normalizer-Form-valueOf-1]]static <<painless-api-reference-Normalizer-Form,Normalizer.Form>> link:{java8-javadoc}/java/text/Normalizer.Form.html#valueOf%2Djava.lang.String%2D[valueOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/Normalizer.Form.html#valueOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Normalizer-Form-values-0]]static <<painless-api-reference-Normalizer-Form,Normalizer.Form>>[] link:{java8-javadoc}/java/text/Normalizer.Form.html#values%2D%2D[values]()++ (link:{java9-javadoc}/java/text/Normalizer.Form.html#values%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Enum,Enum>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Normalizer.asciidoc b/docs/painless/painless-api-reference/Normalizer.asciidoc
new file mode 100644
index 0000000000000..ecfe03c7fb4bd
--- /dev/null
+++ b/docs/painless/painless-api-reference/Normalizer.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Normalizer]]++Normalizer++::
+* ++[[painless-api-reference-Normalizer-isNormalized-2]]static boolean link:{java8-javadoc}/java/text/Normalizer.html#isNormalized%2Djava.lang.CharSequence%2Djava.text.Normalizer$Form%2D[isNormalized](<<painless-api-reference-CharSequence,CharSequence>>, <<painless-api-reference-Normalizer-Form,Normalizer.Form>>)++ (link:{java9-javadoc}/java/text/Normalizer.html#isNormalized%2Djava.lang.CharSequence%2Djava.text.Normalizer$Form%2D[java 9])
+* ++[[painless-api-reference-Normalizer-normalize-2]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/text/Normalizer.html#normalize%2Djava.lang.CharSequence%2Djava.text.Normalizer$Form%2D[normalize](<<painless-api-reference-CharSequence,CharSequence>>, <<painless-api-reference-Normalizer-Form,Normalizer.Form>>)++ (link:{java9-javadoc}/java/text/Normalizer.html#normalize%2Djava.lang.CharSequence%2Djava.text.Normalizer$Form%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/NullPointerException.asciidoc b/docs/painless/painless-api-reference/NullPointerException.asciidoc
new file mode 100644
index 0000000000000..72795c7399060
--- /dev/null
+++ b/docs/painless/painless-api-reference/NullPointerException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-NullPointerException]]++NullPointerException++::
+* ++[[painless-api-reference-NullPointerException-NullPointerException-0]]link:{java8-javadoc}/java/lang/NullPointerException.html#NullPointerException%2D%2D[NullPointerException]()++ (link:{java9-javadoc}/java/lang/NullPointerException.html#NullPointerException%2D%2D[java 9])
+* ++[[painless-api-reference-NullPointerException-NullPointerException-1]]link:{java8-javadoc}/java/lang/NullPointerException.html#NullPointerException%2Djava.lang.String%2D[NullPointerException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/NullPointerException.html#NullPointerException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Number.asciidoc b/docs/painless/painless-api-reference/Number.asciidoc
new file mode 100644
index 0000000000000..a8c588f33704e
--- /dev/null
+++ b/docs/painless/painless-api-reference/Number.asciidoc
@@ -0,0 +1,13 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Number]]++Number++::
+* ++[[painless-api-reference-Number-byteValue-0]]byte link:{java8-javadoc}/java/lang/Number.html#byteValue%2D%2D[byteValue]()++ (link:{java9-javadoc}/java/lang/Number.html#byteValue%2D%2D[java 9])
+* ++[[painless-api-reference-Number-doubleValue-0]]double link:{java8-javadoc}/java/lang/Number.html#doubleValue%2D%2D[doubleValue]()++ (link:{java9-javadoc}/java/lang/Number.html#doubleValue%2D%2D[java 9])
+* ++[[painless-api-reference-Number-floatValue-0]]float link:{java8-javadoc}/java/lang/Number.html#floatValue%2D%2D[floatValue]()++ (link:{java9-javadoc}/java/lang/Number.html#floatValue%2D%2D[java 9])
+* ++[[painless-api-reference-Number-intValue-0]]int link:{java8-javadoc}/java/lang/Number.html#intValue%2D%2D[intValue]()++ (link:{java9-javadoc}/java/lang/Number.html#intValue%2D%2D[java 9])
+* ++[[painless-api-reference-Number-longValue-0]]long link:{java8-javadoc}/java/lang/Number.html#longValue%2D%2D[longValue]()++ (link:{java9-javadoc}/java/lang/Number.html#longValue%2D%2D[java 9])
+* ++[[painless-api-reference-Number-shortValue-0]]short link:{java8-javadoc}/java/lang/Number.html#shortValue%2D%2D[shortValue]()++ (link:{java9-javadoc}/java/lang/Number.html#shortValue%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/NumberFormat.Field.asciidoc b/docs/painless/painless-api-reference/NumberFormat.Field.asciidoc
new file mode 100644
index 0000000000000..279b0dbaaf5b7
--- /dev/null
+++ b/docs/painless/painless-api-reference/NumberFormat.Field.asciidoc
@@ -0,0 +1,18 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-NumberFormat-Field]]++NumberFormat.Field++::
+** [[painless-api-reference-NumberFormat-Field-CURRENCY]]static <<painless-api-reference-NumberFormat-Field,NumberFormat.Field>> link:{java8-javadoc}/java/text/NumberFormat.Field.html#CURRENCY[CURRENCY] (link:{java9-javadoc}/java/text/NumberFormat.Field.html#CURRENCY[java 9])
+** [[painless-api-reference-NumberFormat-Field-DECIMAL_SEPARATOR]]static <<painless-api-reference-NumberFormat-Field,NumberFormat.Field>> link:{java8-javadoc}/java/text/NumberFormat.Field.html#DECIMAL_SEPARATOR[DECIMAL_SEPARATOR] (link:{java9-javadoc}/java/text/NumberFormat.Field.html#DECIMAL_SEPARATOR[java 9])
+** [[painless-api-reference-NumberFormat-Field-EXPONENT]]static <<painless-api-reference-NumberFormat-Field,NumberFormat.Field>> link:{java8-javadoc}/java/text/NumberFormat.Field.html#EXPONENT[EXPONENT] (link:{java9-javadoc}/java/text/NumberFormat.Field.html#EXPONENT[java 9])
+** [[painless-api-reference-NumberFormat-Field-EXPONENT_SIGN]]static <<painless-api-reference-NumberFormat-Field,NumberFormat.Field>> link:{java8-javadoc}/java/text/NumberFormat.Field.html#EXPONENT_SIGN[EXPONENT_SIGN] (link:{java9-javadoc}/java/text/NumberFormat.Field.html#EXPONENT_SIGN[java 9])
+** [[painless-api-reference-NumberFormat-Field-EXPONENT_SYMBOL]]static <<painless-api-reference-NumberFormat-Field,NumberFormat.Field>> link:{java8-javadoc}/java/text/NumberFormat.Field.html#EXPONENT_SYMBOL[EXPONENT_SYMBOL] (link:{java9-javadoc}/java/text/NumberFormat.Field.html#EXPONENT_SYMBOL[java 9])
+** [[painless-api-reference-NumberFormat-Field-FRACTION]]static <<painless-api-reference-NumberFormat-Field,NumberFormat.Field>> link:{java8-javadoc}/java/text/NumberFormat.Field.html#FRACTION[FRACTION] (link:{java9-javadoc}/java/text/NumberFormat.Field.html#FRACTION[java 9])
+** [[painless-api-reference-NumberFormat-Field-GROUPING_SEPARATOR]]static <<painless-api-reference-NumberFormat-Field,NumberFormat.Field>> link:{java8-javadoc}/java/text/NumberFormat.Field.html#GROUPING_SEPARATOR[GROUPING_SEPARATOR] (link:{java9-javadoc}/java/text/NumberFormat.Field.html#GROUPING_SEPARATOR[java 9])
+** [[painless-api-reference-NumberFormat-Field-INTEGER]]static <<painless-api-reference-NumberFormat-Field,NumberFormat.Field>> link:{java8-javadoc}/java/text/NumberFormat.Field.html#INTEGER[INTEGER] (link:{java9-javadoc}/java/text/NumberFormat.Field.html#INTEGER[java 9])
+** [[painless-api-reference-NumberFormat-Field-PERCENT]]static <<painless-api-reference-NumberFormat-Field,NumberFormat.Field>> link:{java8-javadoc}/java/text/NumberFormat.Field.html#PERCENT[PERCENT] (link:{java9-javadoc}/java/text/NumberFormat.Field.html#PERCENT[java 9])
+** [[painless-api-reference-NumberFormat-Field-PERMILLE]]static <<painless-api-reference-NumberFormat-Field,NumberFormat.Field>> link:{java8-javadoc}/java/text/NumberFormat.Field.html#PERMILLE[PERMILLE] (link:{java9-javadoc}/java/text/NumberFormat.Field.html#PERMILLE[java 9])
+** [[painless-api-reference-NumberFormat-Field-SIGN]]static <<painless-api-reference-NumberFormat-Field,NumberFormat.Field>> link:{java8-javadoc}/java/text/NumberFormat.Field.html#SIGN[SIGN] (link:{java9-javadoc}/java/text/NumberFormat.Field.html#SIGN[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/NumberFormat.asciidoc b/docs/painless/painless-api-reference/NumberFormat.asciidoc
new file mode 100644
index 0000000000000..ddca8ccd6826f
--- /dev/null
+++ b/docs/painless/painless-api-reference/NumberFormat.asciidoc
@@ -0,0 +1,38 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-NumberFormat]]++NumberFormat++::
+** [[painless-api-reference-NumberFormat-FRACTION_FIELD]]static int link:{java8-javadoc}/java/text/NumberFormat.html#FRACTION_FIELD[FRACTION_FIELD] (link:{java9-javadoc}/java/text/NumberFormat.html#FRACTION_FIELD[java 9])
+** [[painless-api-reference-NumberFormat-INTEGER_FIELD]]static int link:{java8-javadoc}/java/text/NumberFormat.html#INTEGER_FIELD[INTEGER_FIELD] (link:{java9-javadoc}/java/text/NumberFormat.html#INTEGER_FIELD[java 9])
+* ++[[painless-api-reference-NumberFormat-getAvailableLocales-0]]static <<painless-api-reference-Locale,Locale>>[] link:{java8-javadoc}/java/text/NumberFormat.html#getAvailableLocales%2D%2D[getAvailableLocales]()++ (link:{java9-javadoc}/java/text/NumberFormat.html#getAvailableLocales%2D%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-getCurrencyInstance-0]]static <<painless-api-reference-NumberFormat,NumberFormat>> link:{java8-javadoc}/java/text/NumberFormat.html#getCurrencyInstance%2D%2D[getCurrencyInstance]()++ (link:{java9-javadoc}/java/text/NumberFormat.html#getCurrencyInstance%2D%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-getCurrencyInstance-1]]static <<painless-api-reference-NumberFormat,NumberFormat>> link:{java8-javadoc}/java/text/NumberFormat.html#getCurrencyInstance%2Djava.util.Locale%2D[getCurrencyInstance](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/text/NumberFormat.html#getCurrencyInstance%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-getInstance-0]]static <<painless-api-reference-NumberFormat,NumberFormat>> link:{java8-javadoc}/java/text/NumberFormat.html#getInstance%2D%2D[getInstance]()++ (link:{java9-javadoc}/java/text/NumberFormat.html#getInstance%2D%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-getInstance-1]]static <<painless-api-reference-NumberFormat,NumberFormat>> link:{java8-javadoc}/java/text/NumberFormat.html#getInstance%2Djava.util.Locale%2D[getInstance](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/text/NumberFormat.html#getInstance%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-getIntegerInstance-0]]static <<painless-api-reference-NumberFormat,NumberFormat>> link:{java8-javadoc}/java/text/NumberFormat.html#getIntegerInstance%2D%2D[getIntegerInstance]()++ (link:{java9-javadoc}/java/text/NumberFormat.html#getIntegerInstance%2D%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-getIntegerInstance-1]]static <<painless-api-reference-NumberFormat,NumberFormat>> link:{java8-javadoc}/java/text/NumberFormat.html#getIntegerInstance%2Djava.util.Locale%2D[getIntegerInstance](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/text/NumberFormat.html#getIntegerInstance%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-getNumberInstance-0]]static <<painless-api-reference-NumberFormat,NumberFormat>> link:{java8-javadoc}/java/text/NumberFormat.html#getNumberInstance%2D%2D[getNumberInstance]()++ (link:{java9-javadoc}/java/text/NumberFormat.html#getNumberInstance%2D%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-getNumberInstance-1]]static <<painless-api-reference-NumberFormat,NumberFormat>> link:{java8-javadoc}/java/text/NumberFormat.html#getNumberInstance%2Djava.util.Locale%2D[getNumberInstance](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/text/NumberFormat.html#getNumberInstance%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-getPercentInstance-0]]static <<painless-api-reference-NumberFormat,NumberFormat>> link:{java8-javadoc}/java/text/NumberFormat.html#getPercentInstance%2D%2D[getPercentInstance]()++ (link:{java9-javadoc}/java/text/NumberFormat.html#getPercentInstance%2D%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-getPercentInstance-1]]static <<painless-api-reference-NumberFormat,NumberFormat>> link:{java8-javadoc}/java/text/NumberFormat.html#getPercentInstance%2Djava.util.Locale%2D[getPercentInstance](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/text/NumberFormat.html#getPercentInstance%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-getCurrency-0]]<<painless-api-reference-Currency,Currency>> link:{java8-javadoc}/java/text/NumberFormat.html#getCurrency%2D%2D[getCurrency]()++ (link:{java9-javadoc}/java/text/NumberFormat.html#getCurrency%2D%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-getMaximumFractionDigits-0]]int link:{java8-javadoc}/java/text/NumberFormat.html#getMaximumFractionDigits%2D%2D[getMaximumFractionDigits]()++ (link:{java9-javadoc}/java/text/NumberFormat.html#getMaximumFractionDigits%2D%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-getMaximumIntegerDigits-0]]int link:{java8-javadoc}/java/text/NumberFormat.html#getMaximumIntegerDigits%2D%2D[getMaximumIntegerDigits]()++ (link:{java9-javadoc}/java/text/NumberFormat.html#getMaximumIntegerDigits%2D%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-getMinimumFractionDigits-0]]int link:{java8-javadoc}/java/text/NumberFormat.html#getMinimumFractionDigits%2D%2D[getMinimumFractionDigits]()++ (link:{java9-javadoc}/java/text/NumberFormat.html#getMinimumFractionDigits%2D%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-getMinimumIntegerDigits-0]]int link:{java8-javadoc}/java/text/NumberFormat.html#getMinimumIntegerDigits%2D%2D[getMinimumIntegerDigits]()++ (link:{java9-javadoc}/java/text/NumberFormat.html#getMinimumIntegerDigits%2D%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-getRoundingMode-0]]<<painless-api-reference-RoundingMode,RoundingMode>> link:{java8-javadoc}/java/text/NumberFormat.html#getRoundingMode%2D%2D[getRoundingMode]()++ (link:{java9-javadoc}/java/text/NumberFormat.html#getRoundingMode%2D%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-isGroupingUsed-0]]boolean link:{java8-javadoc}/java/text/NumberFormat.html#isGroupingUsed%2D%2D[isGroupingUsed]()++ (link:{java9-javadoc}/java/text/NumberFormat.html#isGroupingUsed%2D%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-isParseIntegerOnly-0]]boolean link:{java8-javadoc}/java/text/NumberFormat.html#isParseIntegerOnly%2D%2D[isParseIntegerOnly]()++ (link:{java9-javadoc}/java/text/NumberFormat.html#isParseIntegerOnly%2D%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-parse-1]]<<painless-api-reference-Number,Number>> link:{java8-javadoc}/java/text/NumberFormat.html#parse%2Djava.lang.String%2D[parse](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/NumberFormat.html#parse%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-parse-2]]<<painless-api-reference-Number,Number>> link:{java8-javadoc}/java/text/NumberFormat.html#parse%2Djava.lang.String%2Djava.text.ParsePosition%2D[parse](<<painless-api-reference-String,String>>, <<painless-api-reference-ParsePosition,ParsePosition>>)++ (link:{java9-javadoc}/java/text/NumberFormat.html#parse%2Djava.lang.String%2Djava.text.ParsePosition%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-setCurrency-1]]void link:{java8-javadoc}/java/text/NumberFormat.html#setCurrency%2Djava.util.Currency%2D[setCurrency](<<painless-api-reference-Currency,Currency>>)++ (link:{java9-javadoc}/java/text/NumberFormat.html#setCurrency%2Djava.util.Currency%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-setGroupingUsed-1]]void link:{java8-javadoc}/java/text/NumberFormat.html#setGroupingUsed%2Dboolean%2D[setGroupingUsed](boolean)++ (link:{java9-javadoc}/java/text/NumberFormat.html#setGroupingUsed%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-setMaximumFractionDigits-1]]void link:{java8-javadoc}/java/text/NumberFormat.html#setMaximumFractionDigits%2Dint%2D[setMaximumFractionDigits](int)++ (link:{java9-javadoc}/java/text/NumberFormat.html#setMaximumFractionDigits%2Dint%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-setMaximumIntegerDigits-1]]void link:{java8-javadoc}/java/text/NumberFormat.html#setMaximumIntegerDigits%2Dint%2D[setMaximumIntegerDigits](int)++ (link:{java9-javadoc}/java/text/NumberFormat.html#setMaximumIntegerDigits%2Dint%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-setMinimumFractionDigits-1]]void link:{java8-javadoc}/java/text/NumberFormat.html#setMinimumFractionDigits%2Dint%2D[setMinimumFractionDigits](int)++ (link:{java9-javadoc}/java/text/NumberFormat.html#setMinimumFractionDigits%2Dint%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-setMinimumIntegerDigits-1]]void link:{java8-javadoc}/java/text/NumberFormat.html#setMinimumIntegerDigits%2Dint%2D[setMinimumIntegerDigits](int)++ (link:{java9-javadoc}/java/text/NumberFormat.html#setMinimumIntegerDigits%2Dint%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-setParseIntegerOnly-1]]void link:{java8-javadoc}/java/text/NumberFormat.html#setParseIntegerOnly%2Dboolean%2D[setParseIntegerOnly](boolean)++ (link:{java9-javadoc}/java/text/NumberFormat.html#setParseIntegerOnly%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-setRoundingMode-1]]void link:{java8-javadoc}/java/text/NumberFormat.html#setRoundingMode%2Djava.math.RoundingMode%2D[setRoundingMode](<<painless-api-reference-RoundingMode,RoundingMode>>)++ (link:{java9-javadoc}/java/text/NumberFormat.html#setRoundingMode%2Djava.math.RoundingMode%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Format,Format>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/NumberFormatException.asciidoc b/docs/painless/painless-api-reference/NumberFormatException.asciidoc
new file mode 100644
index 0000000000000..4066147650c33
--- /dev/null
+++ b/docs/painless/painless-api-reference/NumberFormatException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-NumberFormatException]]++NumberFormatException++::
+* ++[[painless-api-reference-NumberFormatException-NumberFormatException-0]]link:{java8-javadoc}/java/lang/NumberFormatException.html#NumberFormatException%2D%2D[NumberFormatException]()++ (link:{java9-javadoc}/java/lang/NumberFormatException.html#NumberFormatException%2D%2D[java 9])
+* ++[[painless-api-reference-NumberFormatException-NumberFormatException-1]]link:{java8-javadoc}/java/lang/NumberFormatException.html#NumberFormatException%2Djava.lang.String%2D[NumberFormatException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/NumberFormatException.html#NumberFormatException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/ObjDoubleConsumer.asciidoc b/docs/painless/painless-api-reference/ObjDoubleConsumer.asciidoc
new file mode 100644
index 0000000000000..2fc41f9ce0fb5
--- /dev/null
+++ b/docs/painless/painless-api-reference/ObjDoubleConsumer.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ObjDoubleConsumer]]++ObjDoubleConsumer++::
+* ++[[painless-api-reference-ObjDoubleConsumer-accept-2]]void link:{java8-javadoc}/java/util/function/ObjDoubleConsumer.html#accept%2Djava.lang.Object%2Ddouble%2D[accept](def, double)++ (link:{java9-javadoc}/java/util/function/ObjDoubleConsumer.html#accept%2Djava.lang.Object%2Ddouble%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/ObjIntConsumer.asciidoc b/docs/painless/painless-api-reference/ObjIntConsumer.asciidoc
new file mode 100644
index 0000000000000..8779b333df92c
--- /dev/null
+++ b/docs/painless/painless-api-reference/ObjIntConsumer.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ObjIntConsumer]]++ObjIntConsumer++::
+* ++[[painless-api-reference-ObjIntConsumer-accept-2]]void link:{java8-javadoc}/java/util/function/ObjIntConsumer.html#accept%2Djava.lang.Object%2Dint%2D[accept](def, int)++ (link:{java9-javadoc}/java/util/function/ObjIntConsumer.html#accept%2Djava.lang.Object%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/ObjLongConsumer.asciidoc b/docs/painless/painless-api-reference/ObjLongConsumer.asciidoc
new file mode 100644
index 0000000000000..cdb0ed6546c4f
--- /dev/null
+++ b/docs/painless/painless-api-reference/ObjLongConsumer.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ObjLongConsumer]]++ObjLongConsumer++::
+* ++[[painless-api-reference-ObjLongConsumer-accept-2]]void link:{java8-javadoc}/java/util/function/ObjLongConsumer.html#accept%2Djava.lang.Object%2Dlong%2D[accept](def, long)++ (link:{java9-javadoc}/java/util/function/ObjLongConsumer.html#accept%2Djava.lang.Object%2Dlong%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Object.asciidoc b/docs/painless/painless-api-reference/Object.asciidoc
new file mode 100644
index 0000000000000..99c4ae5cffdb6
--- /dev/null
+++ b/docs/painless/painless-api-reference/Object.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Object]]++Object++::
+* ++[[painless-api-reference-Object-equals-1]]boolean link:{java8-javadoc}/java/lang/Object.html#equals%2Djava.lang.Object%2D[equals](<<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/lang/Object.html#equals%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Object-hashCode-0]]int link:{java8-javadoc}/java/lang/Object.html#hashCode%2D%2D[hashCode]()++ (link:{java9-javadoc}/java/lang/Object.html#hashCode%2D%2D[java 9])
+* ++[[painless-api-reference-Object-toString-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Object.html#toString%2D%2D[toString]()++ (link:{java9-javadoc}/java/lang/Object.html#toString%2D%2D[java 9])
diff --git a/docs/painless/painless-api-reference/Objects.asciidoc b/docs/painless/painless-api-reference/Objects.asciidoc
new file mode 100644
index 0000000000000..9fdbad2e5b71d
--- /dev/null
+++ b/docs/painless/painless-api-reference/Objects.asciidoc
@@ -0,0 +1,18 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Objects]]++Objects++::
+* ++[[painless-api-reference-Objects-compare-3]]static int link:{java8-javadoc}/java/util/Objects.html#compare%2Djava.lang.Object%2Djava.lang.Object%2Djava.util.Comparator%2D[compare](def, def, <<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/Objects.html#compare%2Djava.lang.Object%2Djava.lang.Object%2Djava.util.Comparator%2D[java 9])
+* ++[[painless-api-reference-Objects-deepEquals-2]]static boolean link:{java8-javadoc}/java/util/Objects.html#deepEquals%2Djava.lang.Object%2Djava.lang.Object%2D[deepEquals](<<painless-api-reference-Object,Object>>, <<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/util/Objects.html#deepEquals%2Djava.lang.Object%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Objects-equals-2]]static boolean link:{java8-javadoc}/java/util/Objects.html#equals%2Djava.lang.Object%2Djava.lang.Object%2D[equals](<<painless-api-reference-Object,Object>>, <<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/util/Objects.html#equals%2Djava.lang.Object%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Objects-hash-1]]static int link:{java8-javadoc}/java/util/Objects.html#hash%2Djava.lang.Object:A%2D[hash](<<painless-api-reference-Object,Object>>[])++ (link:{java9-javadoc}/java/util/Objects.html#hash%2Djava.lang.Object:A%2D[java 9])
+* ++[[painless-api-reference-Objects-hashCode-1]]static int link:{java8-javadoc}/java/util/Objects.html#hashCode%2Djava.lang.Object%2D[hashCode](<<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/util/Objects.html#hashCode%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Objects-isNull-1]]static boolean link:{java8-javadoc}/java/util/Objects.html#isNull%2Djava.lang.Object%2D[isNull](<<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/util/Objects.html#isNull%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Objects-nonNull-1]]static boolean link:{java8-javadoc}/java/util/Objects.html#nonNull%2Djava.lang.Object%2D[nonNull](<<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/util/Objects.html#nonNull%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Objects-requireNonNull-1]]static def link:{java8-javadoc}/java/util/Objects.html#requireNonNull%2Djava.lang.Object%2D[requireNonNull](def)++ (link:{java9-javadoc}/java/util/Objects.html#requireNonNull%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Objects-requireNonNull-2]]static def link:{java8-javadoc}/java/util/Objects.html#requireNonNull%2Djava.lang.Object%2Djava.lang.String%2D[requireNonNull](def, <<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/Objects.html#requireNonNull%2Djava.lang.Object%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Objects-toString-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Objects.html#toString%2Djava.lang.Object%2D[toString](<<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/util/Objects.html#toString%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Objects-toString-2]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Objects.html#toString%2Djava.lang.Object%2Djava.lang.String%2D[toString](<<painless-api-reference-Object,Object>>, <<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/Objects.html#toString%2Djava.lang.Object%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Observable.asciidoc b/docs/painless/painless-api-reference/Observable.asciidoc
new file mode 100644
index 0000000000000..65d8fae45ae36
--- /dev/null
+++ b/docs/painless/painless-api-reference/Observable.asciidoc
@@ -0,0 +1,15 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Observable]]++Observable++::
+* ++[[painless-api-reference-Observable-Observable-0]]link:{java8-javadoc}/java/util/Observable.html#Observable%2D%2D[Observable]()++ (link:{java9-javadoc}/java/util/Observable.html#Observable%2D%2D[java 9])
+* ++[[painless-api-reference-Observable-addObserver-1]]void link:{java8-javadoc}/java/util/Observable.html#addObserver%2Djava.util.Observer%2D[addObserver](<<painless-api-reference-Observer,Observer>>)++ (link:{java9-javadoc}/java/util/Observable.html#addObserver%2Djava.util.Observer%2D[java 9])
+* ++[[painless-api-reference-Observable-countObservers-0]]int link:{java8-javadoc}/java/util/Observable.html#countObservers%2D%2D[countObservers]()++ (link:{java9-javadoc}/java/util/Observable.html#countObservers%2D%2D[java 9])
+* ++[[painless-api-reference-Observable-deleteObserver-1]]void link:{java8-javadoc}/java/util/Observable.html#deleteObserver%2Djava.util.Observer%2D[deleteObserver](<<painless-api-reference-Observer,Observer>>)++ (link:{java9-javadoc}/java/util/Observable.html#deleteObserver%2Djava.util.Observer%2D[java 9])
+* ++[[painless-api-reference-Observable-deleteObservers-0]]void link:{java8-javadoc}/java/util/Observable.html#deleteObservers%2D%2D[deleteObservers]()++ (link:{java9-javadoc}/java/util/Observable.html#deleteObservers%2D%2D[java 9])
+* ++[[painless-api-reference-Observable-hasChanged-0]]boolean link:{java8-javadoc}/java/util/Observable.html#hasChanged%2D%2D[hasChanged]()++ (link:{java9-javadoc}/java/util/Observable.html#hasChanged%2D%2D[java 9])
+* ++[[painless-api-reference-Observable-notifyObservers-0]]void link:{java8-javadoc}/java/util/Observable.html#notifyObservers%2D%2D[notifyObservers]()++ (link:{java9-javadoc}/java/util/Observable.html#notifyObservers%2D%2D[java 9])
+* ++[[painless-api-reference-Observable-notifyObservers-1]]void link:{java8-javadoc}/java/util/Observable.html#notifyObservers%2Djava.lang.Object%2D[notifyObservers](<<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/util/Observable.html#notifyObservers%2Djava.lang.Object%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Observer.asciidoc b/docs/painless/painless-api-reference/Observer.asciidoc
new file mode 100644
index 0000000000000..9277663dd6232
--- /dev/null
+++ b/docs/painless/painless-api-reference/Observer.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Observer]]++Observer++::
+* ++[[painless-api-reference-Observer-update-2]]void link:{java8-javadoc}/java/util/Observer.html#update%2Djava.util.Observable%2Djava.lang.Object%2D[update](<<painless-api-reference-Observable,Observable>>, <<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/util/Observer.html#update%2Djava.util.Observable%2Djava.lang.Object%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/OffsetDateTime.asciidoc b/docs/painless/painless-api-reference/OffsetDateTime.asciidoc
new file mode 100644
index 0000000000000..d476c181c55f0
--- /dev/null
+++ b/docs/painless/painless-api-reference/OffsetDateTime.asciidoc
@@ -0,0 +1,75 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-OffsetDateTime]]++OffsetDateTime++::
+** [[painless-api-reference-OffsetDateTime-MAX]]static <<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#MAX[MAX] (link:{java9-javadoc}/java/time/OffsetDateTime.html#MAX[java 9])
+** [[painless-api-reference-OffsetDateTime-MIN]]static <<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#MIN[MIN] (link:{java9-javadoc}/java/time/OffsetDateTime.html#MIN[java 9])
+* ++[[painless-api-reference-OffsetDateTime-from-1]]static <<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#from%2Djava.time.temporal.TemporalAccessor%2D[from](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#from%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-of-2]]static <<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#of%2Djava.time.LocalDateTime%2Djava.time.ZoneOffset%2D[of](<<painless-api-reference-LocalDateTime,LocalDateTime>>, <<painless-api-reference-ZoneOffset,ZoneOffset>>)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#of%2Djava.time.LocalDateTime%2Djava.time.ZoneOffset%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-of-3]]static <<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#of%2Djava.time.LocalDate%2Djava.time.LocalTime%2Djava.time.ZoneOffset%2D[of](<<painless-api-reference-LocalDate,LocalDate>>, <<painless-api-reference-LocalTime,LocalTime>>, <<painless-api-reference-ZoneOffset,ZoneOffset>>)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#of%2Djava.time.LocalDate%2Djava.time.LocalTime%2Djava.time.ZoneOffset%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-of-8]]static <<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#of%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Djava.time.ZoneOffset%2D[of](int, int, int, int, int, int, int, <<painless-api-reference-ZoneOffset,ZoneOffset>>)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#of%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Djava.time.ZoneOffset%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-ofInstant-2]]static <<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#ofInstant%2Djava.time.Instant%2Djava.time.ZoneId%2D[ofInstant](<<painless-api-reference-Instant,Instant>>, <<painless-api-reference-ZoneId,ZoneId>>)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#ofInstant%2Djava.time.Instant%2Djava.time.ZoneId%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-parse-1]]static <<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#parse%2Djava.lang.CharSequence%2D[parse](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#parse%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-parse-2]]static <<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#parse%2Djava.lang.CharSequence%2Djava.time.format.DateTimeFormatter%2D[parse](<<painless-api-reference-CharSequence,CharSequence>>, <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>>)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#parse%2Djava.lang.CharSequence%2Djava.time.format.DateTimeFormatter%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-timeLineOrder-0]]static <<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/time/OffsetDateTime.html#timeLineOrder%2D%2D[timeLineOrder]()++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#timeLineOrder%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-atZoneSameInstant-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#atZoneSameInstant%2Djava.time.ZoneId%2D[atZoneSameInstant](<<painless-api-reference-ZoneId,ZoneId>>)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#atZoneSameInstant%2Djava.time.ZoneId%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-atZoneSimilarLocal-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#atZoneSimilarLocal%2Djava.time.ZoneId%2D[atZoneSimilarLocal](<<painless-api-reference-ZoneId,ZoneId>>)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#atZoneSimilarLocal%2Djava.time.ZoneId%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-compareTo-1]]int link:{java8-javadoc}/java/time/OffsetDateTime.html#compareTo%2Djava.time.OffsetDateTime%2D[compareTo](<<painless-api-reference-OffsetDateTime,OffsetDateTime>>)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#compareTo%2Djava.time.OffsetDateTime%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-format-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/OffsetDateTime.html#format%2Djava.time.format.DateTimeFormatter%2D[format](<<painless-api-reference-DateTimeFormatter,DateTimeFormatter>>)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#format%2Djava.time.format.DateTimeFormatter%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-getDayOfMonth-0]]int link:{java8-javadoc}/java/time/OffsetDateTime.html#getDayOfMonth%2D%2D[getDayOfMonth]()++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#getDayOfMonth%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-getDayOfWeek-0]]<<painless-api-reference-DayOfWeek,DayOfWeek>> link:{java8-javadoc}/java/time/OffsetDateTime.html#getDayOfWeek%2D%2D[getDayOfWeek]()++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#getDayOfWeek%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-getDayOfYear-0]]int link:{java8-javadoc}/java/time/OffsetDateTime.html#getDayOfYear%2D%2D[getDayOfYear]()++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#getDayOfYear%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-getHour-0]]int link:{java8-javadoc}/java/time/OffsetDateTime.html#getHour%2D%2D[getHour]()++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#getHour%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-getMinute-0]]int link:{java8-javadoc}/java/time/OffsetDateTime.html#getMinute%2D%2D[getMinute]()++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#getMinute%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-getMonth-0]]<<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/OffsetDateTime.html#getMonth%2D%2D[getMonth]()++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#getMonth%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-getMonthValue-0]]int link:{java8-javadoc}/java/time/OffsetDateTime.html#getMonthValue%2D%2D[getMonthValue]()++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#getMonthValue%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-getNano-0]]int link:{java8-javadoc}/java/time/OffsetDateTime.html#getNano%2D%2D[getNano]()++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#getNano%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-getOffset-0]]<<painless-api-reference-ZoneOffset,ZoneOffset>> link:{java8-javadoc}/java/time/OffsetDateTime.html#getOffset%2D%2D[getOffset]()++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#getOffset%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-getSecond-0]]int link:{java8-javadoc}/java/time/OffsetDateTime.html#getSecond%2D%2D[getSecond]()++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#getSecond%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-getYear-0]]int link:{java8-javadoc}/java/time/OffsetDateTime.html#getYear%2D%2D[getYear]()++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#getYear%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-isAfter-1]]boolean link:{java8-javadoc}/java/time/OffsetDateTime.html#isAfter%2Djava.time.OffsetDateTime%2D[isAfter](<<painless-api-reference-OffsetDateTime,OffsetDateTime>>)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#isAfter%2Djava.time.OffsetDateTime%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-isBefore-1]]boolean link:{java8-javadoc}/java/time/OffsetDateTime.html#isBefore%2Djava.time.OffsetDateTime%2D[isBefore](<<painless-api-reference-OffsetDateTime,OffsetDateTime>>)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#isBefore%2Djava.time.OffsetDateTime%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-isEqual-1]]boolean link:{java8-javadoc}/java/time/OffsetDateTime.html#isEqual%2Djava.time.OffsetDateTime%2D[isEqual](<<painless-api-reference-OffsetDateTime,OffsetDateTime>>)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#isEqual%2Djava.time.OffsetDateTime%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-minus-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#minus%2Djava.time.temporal.TemporalAmount%2D[minus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#minus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-minus-2]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[minus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-minusDays-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#minusDays%2Dlong%2D[minusDays](long)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#minusDays%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-minusHours-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#minusHours%2Dlong%2D[minusHours](long)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#minusHours%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-minusMinutes-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#minusMinutes%2Dlong%2D[minusMinutes](long)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#minusMinutes%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-minusMonths-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#minusMonths%2Dlong%2D[minusMonths](long)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#minusMonths%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-minusNanos-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#minusNanos%2Dlong%2D[minusNanos](long)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#minusNanos%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-minusSeconds-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#minusSeconds%2Dlong%2D[minusSeconds](long)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#minusSeconds%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-minusWeeks-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#minusWeeks%2Dlong%2D[minusWeeks](long)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#minusWeeks%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-minusYears-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#minusYears%2Dlong%2D[minusYears](long)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#minusYears%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-plus-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#plus%2Djava.time.temporal.TemporalAmount%2D[plus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#plus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-plus-2]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[plus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-plusDays-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#plusDays%2Dlong%2D[plusDays](long)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#plusDays%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-plusHours-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#plusHours%2Dlong%2D[plusHours](long)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#plusHours%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-plusMinutes-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#plusMinutes%2Dlong%2D[plusMinutes](long)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#plusMinutes%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-plusMonths-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#plusMonths%2Dlong%2D[plusMonths](long)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#plusMonths%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-plusNanos-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#plusNanos%2Dlong%2D[plusNanos](long)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#plusNanos%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-plusSeconds-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#plusSeconds%2Dlong%2D[plusSeconds](long)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#plusSeconds%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-plusWeeks-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#plusWeeks%2Dlong%2D[plusWeeks](long)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#plusWeeks%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-plusYears-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#plusYears%2Dlong%2D[plusYears](long)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#plusYears%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-toEpochSecond-0]]long link:{java8-javadoc}/java/time/OffsetDateTime.html#toEpochSecond%2D%2D[toEpochSecond]()++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#toEpochSecond%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-toInstant-0]]<<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/OffsetDateTime.html#toInstant%2D%2D[toInstant]()++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#toInstant%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-toLocalDate-0]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/OffsetDateTime.html#toLocalDate%2D%2D[toLocalDate]()++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#toLocalDate%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-toLocalDateTime-0]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#toLocalDateTime%2D%2D[toLocalDateTime]()++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#toLocalDateTime%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-toLocalTime-0]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#toLocalTime%2D%2D[toLocalTime]()++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#toLocalTime%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-toOffsetTime-0]]<<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#toOffsetTime%2D%2D[toOffsetTime]()++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#toOffsetTime%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-toZonedDateTime-0]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#toZonedDateTime%2D%2D[toZonedDateTime]()++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#toZonedDateTime%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-truncatedTo-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#truncatedTo%2Djava.time.temporal.TemporalUnit%2D[truncatedTo](<<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#truncatedTo%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-with-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#with%2Djava.time.temporal.TemporalAdjuster%2D[with](<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#with%2Djava.time.temporal.TemporalAdjuster%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-with-2]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[with](<<painless-api-reference-TemporalField,TemporalField>>, long)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-withDayOfMonth-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#withDayOfMonth%2Dint%2D[withDayOfMonth](int)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#withDayOfMonth%2Dint%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-withDayOfYear-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#withDayOfYear%2Dint%2D[withDayOfYear](int)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#withDayOfYear%2Dint%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-withHour-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#withHour%2Dint%2D[withHour](int)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#withHour%2Dint%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-withMinute-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#withMinute%2Dint%2D[withMinute](int)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#withMinute%2Dint%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-withMonth-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#withMonth%2Dint%2D[withMonth](int)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#withMonth%2Dint%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-withNano-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#withNano%2Dint%2D[withNano](int)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#withNano%2Dint%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-withOffsetSameInstant-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#withOffsetSameInstant%2Djava.time.ZoneOffset%2D[withOffsetSameInstant](<<painless-api-reference-ZoneOffset,ZoneOffset>>)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#withOffsetSameInstant%2Djava.time.ZoneOffset%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-withOffsetSameLocal-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#withOffsetSameLocal%2Djava.time.ZoneOffset%2D[withOffsetSameLocal](<<painless-api-reference-ZoneOffset,ZoneOffset>>)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#withOffsetSameLocal%2Djava.time.ZoneOffset%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-withSecond-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#withSecond%2Dint%2D[withSecond](int)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#withSecond%2Dint%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-withYear-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#withYear%2Dint%2D[withYear](int)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#withYear%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-Temporal,Temporal>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++, ++<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>++
diff --git a/docs/painless/painless-api-reference/OffsetTime.asciidoc b/docs/painless/painless-api-reference/OffsetTime.asciidoc
new file mode 100644
index 0000000000000..b23e9bb9577d0
--- /dev/null
+++ b/docs/painless/painless-api-reference/OffsetTime.asciidoc
@@ -0,0 +1,47 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-OffsetTime]]++OffsetTime++::
+** [[painless-api-reference-OffsetTime-MAX]]static <<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#MAX[MAX] (link:{java9-javadoc}/java/time/OffsetTime.html#MAX[java 9])
+** [[painless-api-reference-OffsetTime-MIN]]static <<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#MIN[MIN] (link:{java9-javadoc}/java/time/OffsetTime.html#MIN[java 9])
+* ++[[painless-api-reference-OffsetTime-from-1]]static <<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#from%2Djava.time.temporal.TemporalAccessor%2D[from](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/OffsetTime.html#from%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-of-2]]static <<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#of%2Djava.time.LocalTime%2Djava.time.ZoneOffset%2D[of](<<painless-api-reference-LocalTime,LocalTime>>, <<painless-api-reference-ZoneOffset,ZoneOffset>>)++ (link:{java9-javadoc}/java/time/OffsetTime.html#of%2Djava.time.LocalTime%2Djava.time.ZoneOffset%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-of-5]]static <<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#of%2Dint%2Dint%2Dint%2Dint%2Djava.time.ZoneOffset%2D[of](int, int, int, int, <<painless-api-reference-ZoneOffset,ZoneOffset>>)++ (link:{java9-javadoc}/java/time/OffsetTime.html#of%2Dint%2Dint%2Dint%2Dint%2Djava.time.ZoneOffset%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-ofInstant-2]]static <<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#ofInstant%2Djava.time.Instant%2Djava.time.ZoneId%2D[ofInstant](<<painless-api-reference-Instant,Instant>>, <<painless-api-reference-ZoneId,ZoneId>>)++ (link:{java9-javadoc}/java/time/OffsetTime.html#ofInstant%2Djava.time.Instant%2Djava.time.ZoneId%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-parse-1]]static <<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#parse%2Djava.lang.CharSequence%2D[parse](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/time/OffsetTime.html#parse%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-parse-2]]static <<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#parse%2Djava.lang.CharSequence%2Djava.time.format.DateTimeFormatter%2D[parse](<<painless-api-reference-CharSequence,CharSequence>>, <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>>)++ (link:{java9-javadoc}/java/time/OffsetTime.html#parse%2Djava.lang.CharSequence%2Djava.time.format.DateTimeFormatter%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-compareTo-1]]int link:{java8-javadoc}/java/time/OffsetTime.html#compareTo%2Djava.time.OffsetTime%2D[compareTo](<<painless-api-reference-OffsetTime,OffsetTime>>)++ (link:{java9-javadoc}/java/time/OffsetTime.html#compareTo%2Djava.time.OffsetTime%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-format-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/OffsetTime.html#format%2Djava.time.format.DateTimeFormatter%2D[format](<<painless-api-reference-DateTimeFormatter,DateTimeFormatter>>)++ (link:{java9-javadoc}/java/time/OffsetTime.html#format%2Djava.time.format.DateTimeFormatter%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-getHour-0]]int link:{java8-javadoc}/java/time/OffsetTime.html#getHour%2D%2D[getHour]()++ (link:{java9-javadoc}/java/time/OffsetTime.html#getHour%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-getMinute-0]]int link:{java8-javadoc}/java/time/OffsetTime.html#getMinute%2D%2D[getMinute]()++ (link:{java9-javadoc}/java/time/OffsetTime.html#getMinute%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-getNano-0]]int link:{java8-javadoc}/java/time/OffsetTime.html#getNano%2D%2D[getNano]()++ (link:{java9-javadoc}/java/time/OffsetTime.html#getNano%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-getOffset-0]]<<painless-api-reference-ZoneOffset,ZoneOffset>> link:{java8-javadoc}/java/time/OffsetTime.html#getOffset%2D%2D[getOffset]()++ (link:{java9-javadoc}/java/time/OffsetTime.html#getOffset%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-getSecond-0]]int link:{java8-javadoc}/java/time/OffsetTime.html#getSecond%2D%2D[getSecond]()++ (link:{java9-javadoc}/java/time/OffsetTime.html#getSecond%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-isAfter-1]]boolean link:{java8-javadoc}/java/time/OffsetTime.html#isAfter%2Djava.time.OffsetTime%2D[isAfter](<<painless-api-reference-OffsetTime,OffsetTime>>)++ (link:{java9-javadoc}/java/time/OffsetTime.html#isAfter%2Djava.time.OffsetTime%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-isBefore-1]]boolean link:{java8-javadoc}/java/time/OffsetTime.html#isBefore%2Djava.time.OffsetTime%2D[isBefore](<<painless-api-reference-OffsetTime,OffsetTime>>)++ (link:{java9-javadoc}/java/time/OffsetTime.html#isBefore%2Djava.time.OffsetTime%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-isEqual-1]]boolean link:{java8-javadoc}/java/time/OffsetTime.html#isEqual%2Djava.time.OffsetTime%2D[isEqual](<<painless-api-reference-OffsetTime,OffsetTime>>)++ (link:{java9-javadoc}/java/time/OffsetTime.html#isEqual%2Djava.time.OffsetTime%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-minus-1]]<<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#minus%2Djava.time.temporal.TemporalAmount%2D[minus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/OffsetTime.html#minus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-minus-2]]<<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[minus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/OffsetTime.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-minusHours-1]]<<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#minusHours%2Dlong%2D[minusHours](long)++ (link:{java9-javadoc}/java/time/OffsetTime.html#minusHours%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-minusMinutes-1]]<<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#minusMinutes%2Dlong%2D[minusMinutes](long)++ (link:{java9-javadoc}/java/time/OffsetTime.html#minusMinutes%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-minusNanos-1]]<<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#minusNanos%2Dlong%2D[minusNanos](long)++ (link:{java9-javadoc}/java/time/OffsetTime.html#minusNanos%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-minusSeconds-1]]<<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#minusSeconds%2Dlong%2D[minusSeconds](long)++ (link:{java9-javadoc}/java/time/OffsetTime.html#minusSeconds%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-plus-1]]<<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#plus%2Djava.time.temporal.TemporalAmount%2D[plus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/OffsetTime.html#plus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-plus-2]]<<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[plus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/OffsetTime.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-plusHours-1]]<<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#plusHours%2Dlong%2D[plusHours](long)++ (link:{java9-javadoc}/java/time/OffsetTime.html#plusHours%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-plusMinutes-1]]<<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#plusMinutes%2Dlong%2D[plusMinutes](long)++ (link:{java9-javadoc}/java/time/OffsetTime.html#plusMinutes%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-plusNanos-1]]<<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#plusNanos%2Dlong%2D[plusNanos](long)++ (link:{java9-javadoc}/java/time/OffsetTime.html#plusNanos%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-plusSeconds-1]]<<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#plusSeconds%2Dlong%2D[plusSeconds](long)++ (link:{java9-javadoc}/java/time/OffsetTime.html#plusSeconds%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-toLocalTime-0]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/OffsetTime.html#toLocalTime%2D%2D[toLocalTime]()++ (link:{java9-javadoc}/java/time/OffsetTime.html#toLocalTime%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-truncatedTo-1]]<<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#truncatedTo%2Djava.time.temporal.TemporalUnit%2D[truncatedTo](<<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/OffsetTime.html#truncatedTo%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-with-1]]<<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#with%2Djava.time.temporal.TemporalAdjuster%2D[with](<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>)++ (link:{java9-javadoc}/java/time/OffsetTime.html#with%2Djava.time.temporal.TemporalAdjuster%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-with-2]]<<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[with](<<painless-api-reference-TemporalField,TemporalField>>, long)++ (link:{java9-javadoc}/java/time/OffsetTime.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-withHour-1]]<<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#withHour%2Dint%2D[withHour](int)++ (link:{java9-javadoc}/java/time/OffsetTime.html#withHour%2Dint%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-withMinute-1]]<<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#withMinute%2Dint%2D[withMinute](int)++ (link:{java9-javadoc}/java/time/OffsetTime.html#withMinute%2Dint%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-withNano-1]]<<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#withNano%2Dint%2D[withNano](int)++ (link:{java9-javadoc}/java/time/OffsetTime.html#withNano%2Dint%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-withOffsetSameInstant-1]]<<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#withOffsetSameInstant%2Djava.time.ZoneOffset%2D[withOffsetSameInstant](<<painless-api-reference-ZoneOffset,ZoneOffset>>)++ (link:{java9-javadoc}/java/time/OffsetTime.html#withOffsetSameInstant%2Djava.time.ZoneOffset%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-withOffsetSameLocal-1]]<<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#withOffsetSameLocal%2Djava.time.ZoneOffset%2D[withOffsetSameLocal](<<painless-api-reference-ZoneOffset,ZoneOffset>>)++ (link:{java9-javadoc}/java/time/OffsetTime.html#withOffsetSameLocal%2Djava.time.ZoneOffset%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-withSecond-1]]<<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#withSecond%2Dint%2D[withSecond](int)++ (link:{java9-javadoc}/java/time/OffsetTime.html#withSecond%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-Temporal,Temporal>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++, ++<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>++
diff --git a/docs/painless/painless-api-reference/Optional.asciidoc b/docs/painless/painless-api-reference/Optional.asciidoc
new file mode 100644
index 0000000000000..a67616f006fea
--- /dev/null
+++ b/docs/painless/painless-api-reference/Optional.asciidoc
@@ -0,0 +1,19 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Optional]]++Optional++::
+* ++[[painless-api-reference-Optional-empty-0]]static <<painless-api-reference-Optional,Optional>> link:{java8-javadoc}/java/util/Optional.html#empty%2D%2D[empty]()++ (link:{java9-javadoc}/java/util/Optional.html#empty%2D%2D[java 9])
+* ++[[painless-api-reference-Optional-of-1]]static <<painless-api-reference-Optional,Optional>> link:{java8-javadoc}/java/util/Optional.html#of%2Djava.lang.Object%2D[of](def)++ (link:{java9-javadoc}/java/util/Optional.html#of%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Optional-ofNullable-1]]static <<painless-api-reference-Optional,Optional>> link:{java8-javadoc}/java/util/Optional.html#ofNullable%2Djava.lang.Object%2D[ofNullable](def)++ (link:{java9-javadoc}/java/util/Optional.html#ofNullable%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Optional-filter-1]]<<painless-api-reference-Optional,Optional>> link:{java8-javadoc}/java/util/Optional.html#filter%2Djava.util.function.Predicate%2D[filter](<<painless-api-reference-Predicate,Predicate>>)++ (link:{java9-javadoc}/java/util/Optional.html#filter%2Djava.util.function.Predicate%2D[java 9])
+* ++[[painless-api-reference-Optional-flatMap-1]]<<painless-api-reference-Optional,Optional>> link:{java8-javadoc}/java/util/Optional.html#flatMap%2Djava.util.function.Function%2D[flatMap](<<painless-api-reference-Function,Function>>)++ (link:{java9-javadoc}/java/util/Optional.html#flatMap%2Djava.util.function.Function%2D[java 9])
+* ++[[painless-api-reference-Optional-get-0]]def link:{java8-javadoc}/java/util/Optional.html#get%2D%2D[get]()++ (link:{java9-javadoc}/java/util/Optional.html#get%2D%2D[java 9])
+* ++[[painless-api-reference-Optional-ifPresent-1]]void link:{java8-javadoc}/java/util/Optional.html#ifPresent%2Djava.util.function.Consumer%2D[ifPresent](<<painless-api-reference-Consumer,Consumer>>)++ (link:{java9-javadoc}/java/util/Optional.html#ifPresent%2Djava.util.function.Consumer%2D[java 9])
+* ++[[painless-api-reference-Optional-isPresent-0]]boolean link:{java8-javadoc}/java/util/Optional.html#isPresent%2D%2D[isPresent]()++ (link:{java9-javadoc}/java/util/Optional.html#isPresent%2D%2D[java 9])
+* ++[[painless-api-reference-Optional-map-1]]<<painless-api-reference-Optional,Optional>> link:{java8-javadoc}/java/util/Optional.html#map%2Djava.util.function.Function%2D[map](<<painless-api-reference-Function,Function>>)++ (link:{java9-javadoc}/java/util/Optional.html#map%2Djava.util.function.Function%2D[java 9])
+* ++[[painless-api-reference-Optional-orElse-1]]def link:{java8-javadoc}/java/util/Optional.html#orElse%2Djava.lang.Object%2D[orElse](def)++ (link:{java9-javadoc}/java/util/Optional.html#orElse%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Optional-orElseGet-1]]def link:{java8-javadoc}/java/util/Optional.html#orElseGet%2Djava.util.function.Supplier%2D[orElseGet](<<painless-api-reference-Supplier,Supplier>>)++ (link:{java9-javadoc}/java/util/Optional.html#orElseGet%2Djava.util.function.Supplier%2D[java 9])
+* ++[[painless-api-reference-Optional-orElseThrow-1]]def link:{java8-javadoc}/java/util/Optional.html#orElseThrow%2Djava.util.function.Supplier%2D[orElseThrow](<<painless-api-reference-Supplier,Supplier>>)++ (link:{java9-javadoc}/java/util/Optional.html#orElseThrow%2Djava.util.function.Supplier%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/OptionalDouble.asciidoc b/docs/painless/painless-api-reference/OptionalDouble.asciidoc
new file mode 100644
index 0000000000000..56ac37deaa336
--- /dev/null
+++ b/docs/painless/painless-api-reference/OptionalDouble.asciidoc
@@ -0,0 +1,15 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-OptionalDouble]]++OptionalDouble++::
+* ++[[painless-api-reference-OptionalDouble-empty-0]]static <<painless-api-reference-OptionalDouble,OptionalDouble>> link:{java8-javadoc}/java/util/OptionalDouble.html#empty%2D%2D[empty]()++ (link:{java9-javadoc}/java/util/OptionalDouble.html#empty%2D%2D[java 9])
+* ++[[painless-api-reference-OptionalDouble-of-1]]static <<painless-api-reference-OptionalDouble,OptionalDouble>> link:{java8-javadoc}/java/util/OptionalDouble.html#of%2Ddouble%2D[of](double)++ (link:{java9-javadoc}/java/util/OptionalDouble.html#of%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-OptionalDouble-getAsDouble-0]]double link:{java8-javadoc}/java/util/OptionalDouble.html#getAsDouble%2D%2D[getAsDouble]()++ (link:{java9-javadoc}/java/util/OptionalDouble.html#getAsDouble%2D%2D[java 9])
+* ++[[painless-api-reference-OptionalDouble-ifPresent-1]]void link:{java8-javadoc}/java/util/OptionalDouble.html#ifPresent%2Djava.util.function.DoubleConsumer%2D[ifPresent](<<painless-api-reference-DoubleConsumer,DoubleConsumer>>)++ (link:{java9-javadoc}/java/util/OptionalDouble.html#ifPresent%2Djava.util.function.DoubleConsumer%2D[java 9])
+* ++[[painless-api-reference-OptionalDouble-isPresent-0]]boolean link:{java8-javadoc}/java/util/OptionalDouble.html#isPresent%2D%2D[isPresent]()++ (link:{java9-javadoc}/java/util/OptionalDouble.html#isPresent%2D%2D[java 9])
+* ++[[painless-api-reference-OptionalDouble-orElse-1]]double link:{java8-javadoc}/java/util/OptionalDouble.html#orElse%2Ddouble%2D[orElse](double)++ (link:{java9-javadoc}/java/util/OptionalDouble.html#orElse%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-OptionalDouble-orElseGet-1]]double link:{java8-javadoc}/java/util/OptionalDouble.html#orElseGet%2Djava.util.function.DoubleSupplier%2D[orElseGet](<<painless-api-reference-DoubleSupplier,DoubleSupplier>>)++ (link:{java9-javadoc}/java/util/OptionalDouble.html#orElseGet%2Djava.util.function.DoubleSupplier%2D[java 9])
+* ++[[painless-api-reference-OptionalDouble-orElseThrow-1]]double link:{java8-javadoc}/java/util/OptionalDouble.html#orElseThrow%2Djava.util.function.Supplier%2D[orElseThrow](<<painless-api-reference-Supplier,Supplier>>)++ (link:{java9-javadoc}/java/util/OptionalDouble.html#orElseThrow%2Djava.util.function.Supplier%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/OptionalInt.asciidoc b/docs/painless/painless-api-reference/OptionalInt.asciidoc
new file mode 100644
index 0000000000000..69f0059b85f9c
--- /dev/null
+++ b/docs/painless/painless-api-reference/OptionalInt.asciidoc
@@ -0,0 +1,15 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-OptionalInt]]++OptionalInt++::
+* ++[[painless-api-reference-OptionalInt-empty-0]]static <<painless-api-reference-OptionalInt,OptionalInt>> link:{java8-javadoc}/java/util/OptionalInt.html#empty%2D%2D[empty]()++ (link:{java9-javadoc}/java/util/OptionalInt.html#empty%2D%2D[java 9])
+* ++[[painless-api-reference-OptionalInt-of-1]]static <<painless-api-reference-OptionalInt,OptionalInt>> link:{java8-javadoc}/java/util/OptionalInt.html#of%2Dint%2D[of](int)++ (link:{java9-javadoc}/java/util/OptionalInt.html#of%2Dint%2D[java 9])
+* ++[[painless-api-reference-OptionalInt-getAsInt-0]]int link:{java8-javadoc}/java/util/OptionalInt.html#getAsInt%2D%2D[getAsInt]()++ (link:{java9-javadoc}/java/util/OptionalInt.html#getAsInt%2D%2D[java 9])
+* ++[[painless-api-reference-OptionalInt-ifPresent-1]]void link:{java8-javadoc}/java/util/OptionalInt.html#ifPresent%2Djava.util.function.IntConsumer%2D[ifPresent](<<painless-api-reference-IntConsumer,IntConsumer>>)++ (link:{java9-javadoc}/java/util/OptionalInt.html#ifPresent%2Djava.util.function.IntConsumer%2D[java 9])
+* ++[[painless-api-reference-OptionalInt-isPresent-0]]boolean link:{java8-javadoc}/java/util/OptionalInt.html#isPresent%2D%2D[isPresent]()++ (link:{java9-javadoc}/java/util/OptionalInt.html#isPresent%2D%2D[java 9])
+* ++[[painless-api-reference-OptionalInt-orElse-1]]int link:{java8-javadoc}/java/util/OptionalInt.html#orElse%2Dint%2D[orElse](int)++ (link:{java9-javadoc}/java/util/OptionalInt.html#orElse%2Dint%2D[java 9])
+* ++[[painless-api-reference-OptionalInt-orElseGet-1]]int link:{java8-javadoc}/java/util/OptionalInt.html#orElseGet%2Djava.util.function.IntSupplier%2D[orElseGet](<<painless-api-reference-IntSupplier,IntSupplier>>)++ (link:{java9-javadoc}/java/util/OptionalInt.html#orElseGet%2Djava.util.function.IntSupplier%2D[java 9])
+* ++[[painless-api-reference-OptionalInt-orElseThrow-1]]int link:{java8-javadoc}/java/util/OptionalInt.html#orElseThrow%2Djava.util.function.Supplier%2D[orElseThrow](<<painless-api-reference-Supplier,Supplier>>)++ (link:{java9-javadoc}/java/util/OptionalInt.html#orElseThrow%2Djava.util.function.Supplier%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/OptionalLong.asciidoc b/docs/painless/painless-api-reference/OptionalLong.asciidoc
new file mode 100644
index 0000000000000..a4b135e55115a
--- /dev/null
+++ b/docs/painless/painless-api-reference/OptionalLong.asciidoc
@@ -0,0 +1,15 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-OptionalLong]]++OptionalLong++::
+* ++[[painless-api-reference-OptionalLong-empty-0]]static <<painless-api-reference-OptionalLong,OptionalLong>> link:{java8-javadoc}/java/util/OptionalLong.html#empty%2D%2D[empty]()++ (link:{java9-javadoc}/java/util/OptionalLong.html#empty%2D%2D[java 9])
+* ++[[painless-api-reference-OptionalLong-of-1]]static <<painless-api-reference-OptionalLong,OptionalLong>> link:{java8-javadoc}/java/util/OptionalLong.html#of%2Dlong%2D[of](long)++ (link:{java9-javadoc}/java/util/OptionalLong.html#of%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OptionalLong-getAsLong-0]]long link:{java8-javadoc}/java/util/OptionalLong.html#getAsLong%2D%2D[getAsLong]()++ (link:{java9-javadoc}/java/util/OptionalLong.html#getAsLong%2D%2D[java 9])
+* ++[[painless-api-reference-OptionalLong-ifPresent-1]]void link:{java8-javadoc}/java/util/OptionalLong.html#ifPresent%2Djava.util.function.LongConsumer%2D[ifPresent](<<painless-api-reference-LongConsumer,LongConsumer>>)++ (link:{java9-javadoc}/java/util/OptionalLong.html#ifPresent%2Djava.util.function.LongConsumer%2D[java 9])
+* ++[[painless-api-reference-OptionalLong-isPresent-0]]boolean link:{java8-javadoc}/java/util/OptionalLong.html#isPresent%2D%2D[isPresent]()++ (link:{java9-javadoc}/java/util/OptionalLong.html#isPresent%2D%2D[java 9])
+* ++[[painless-api-reference-OptionalLong-orElse-1]]long link:{java8-javadoc}/java/util/OptionalLong.html#orElse%2Dlong%2D[orElse](long)++ (link:{java9-javadoc}/java/util/OptionalLong.html#orElse%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OptionalLong-orElseGet-1]]long link:{java8-javadoc}/java/util/OptionalLong.html#orElseGet%2Djava.util.function.LongSupplier%2D[orElseGet](<<painless-api-reference-LongSupplier,LongSupplier>>)++ (link:{java9-javadoc}/java/util/OptionalLong.html#orElseGet%2Djava.util.function.LongSupplier%2D[java 9])
+* ++[[painless-api-reference-OptionalLong-orElseThrow-1]]long link:{java8-javadoc}/java/util/OptionalLong.html#orElseThrow%2Djava.util.function.Supplier%2D[orElseThrow](<<painless-api-reference-Supplier,Supplier>>)++ (link:{java9-javadoc}/java/util/OptionalLong.html#orElseThrow%2Djava.util.function.Supplier%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/ParseException.asciidoc b/docs/painless/painless-api-reference/ParseException.asciidoc
new file mode 100644
index 0000000000000..086bcd0b7aaed
--- /dev/null
+++ b/docs/painless/painless-api-reference/ParseException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ParseException]]++ParseException++::
+* ++[[painless-api-reference-ParseException-ParseException-2]]link:{java8-javadoc}/java/text/ParseException.html#ParseException%2Djava.lang.String%2Dint%2D[ParseException](<<painless-api-reference-String,String>>, int)++ (link:{java9-javadoc}/java/text/ParseException.html#ParseException%2Djava.lang.String%2Dint%2D[java 9])
+* ++[[painless-api-reference-ParseException-getErrorOffset-0]]int link:{java8-javadoc}/java/text/ParseException.html#getErrorOffset%2D%2D[getErrorOffset]()++ (link:{java9-javadoc}/java/text/ParseException.html#getErrorOffset%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/ParsePosition.asciidoc b/docs/painless/painless-api-reference/ParsePosition.asciidoc
new file mode 100644
index 0000000000000..88fe6369657bd
--- /dev/null
+++ b/docs/painless/painless-api-reference/ParsePosition.asciidoc
@@ -0,0 +1,12 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ParsePosition]]++ParsePosition++::
+* ++[[painless-api-reference-ParsePosition-ParsePosition-1]]link:{java8-javadoc}/java/text/ParsePosition.html#ParsePosition%2Dint%2D[ParsePosition](int)++ (link:{java9-javadoc}/java/text/ParsePosition.html#ParsePosition%2Dint%2D[java 9])
+* ++[[painless-api-reference-ParsePosition-getErrorIndex-0]]int link:{java8-javadoc}/java/text/ParsePosition.html#getErrorIndex%2D%2D[getErrorIndex]()++ (link:{java9-javadoc}/java/text/ParsePosition.html#getErrorIndex%2D%2D[java 9])
+* ++[[painless-api-reference-ParsePosition-getIndex-0]]int link:{java8-javadoc}/java/text/ParsePosition.html#getIndex%2D%2D[getIndex]()++ (link:{java9-javadoc}/java/text/ParsePosition.html#getIndex%2D%2D[java 9])
+* ++[[painless-api-reference-ParsePosition-setErrorIndex-1]]void link:{java8-javadoc}/java/text/ParsePosition.html#setErrorIndex%2Dint%2D[setErrorIndex](int)++ (link:{java9-javadoc}/java/text/ParsePosition.html#setErrorIndex%2Dint%2D[java 9])
+* ++[[painless-api-reference-ParsePosition-setIndex-1]]void link:{java8-javadoc}/java/text/ParsePosition.html#setIndex%2Dint%2D[setIndex](int)++ (link:{java9-javadoc}/java/text/ParsePosition.html#setIndex%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Pattern.asciidoc b/docs/painless/painless-api-reference/Pattern.asciidoc
new file mode 100644
index 0000000000000..ad7b3603036b1
--- /dev/null
+++ b/docs/painless/painless-api-reference/Pattern.asciidoc
@@ -0,0 +1,15 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Pattern]]++Pattern++::
+* ++[[painless-api-reference-Pattern-quote-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/regex/Pattern.html#quote%2Djava.lang.String%2D[quote](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/regex/Pattern.html#quote%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Pattern-asPredicate-0]]<<painless-api-reference-Predicate,Predicate>> link:{java8-javadoc}/java/util/regex/Pattern.html#asPredicate%2D%2D[asPredicate]()++ (link:{java9-javadoc}/java/util/regex/Pattern.html#asPredicate%2D%2D[java 9])
+* ++[[painless-api-reference-Pattern-flags-0]]int link:{java8-javadoc}/java/util/regex/Pattern.html#flags%2D%2D[flags]()++ (link:{java9-javadoc}/java/util/regex/Pattern.html#flags%2D%2D[java 9])
+* ++[[painless-api-reference-Pattern-matcher-1]]<<painless-api-reference-Matcher,Matcher>> link:{java8-javadoc}/java/util/regex/Pattern.html#matcher%2Djava.lang.CharSequence%2D[matcher](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/util/regex/Pattern.html#matcher%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-Pattern-pattern-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/regex/Pattern.html#pattern%2D%2D[pattern]()++ (link:{java9-javadoc}/java/util/regex/Pattern.html#pattern%2D%2D[java 9])
+* ++[[painless-api-reference-Pattern-split-1]]<<painless-api-reference-String,String>>[] link:{java8-javadoc}/java/util/regex/Pattern.html#split%2Djava.lang.CharSequence%2D[split](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/util/regex/Pattern.html#split%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-Pattern-split-2]]<<painless-api-reference-String,String>>[] link:{java8-javadoc}/java/util/regex/Pattern.html#split%2Djava.lang.CharSequence%2Dint%2D[split](<<painless-api-reference-CharSequence,CharSequence>>, int)++ (link:{java9-javadoc}/java/util/regex/Pattern.html#split%2Djava.lang.CharSequence%2Dint%2D[java 9])
+* ++[[painless-api-reference-Pattern-splitAsStream-1]]<<painless-api-reference-Stream,Stream>> link:{java8-javadoc}/java/util/regex/Pattern.html#splitAsStream%2Djava.lang.CharSequence%2D[splitAsStream](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/util/regex/Pattern.html#splitAsStream%2Djava.lang.CharSequence%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Period.asciidoc b/docs/painless/painless-api-reference/Period.asciidoc
new file mode 100644
index 0000000000000..bb20dddae60a8
--- /dev/null
+++ b/docs/painless/painless-api-reference/Period.asciidoc
@@ -0,0 +1,35 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Period]]++Period++::
+** [[painless-api-reference-Period-ZERO]]static <<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/Period.html#ZERO[ZERO] (link:{java9-javadoc}/java/time/Period.html#ZERO[java 9])
+* ++[[painless-api-reference-Period-between-2]]static <<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/Period.html#between%2Djava.time.LocalDate%2Djava.time.LocalDate%2D[between](<<painless-api-reference-LocalDate,LocalDate>>, <<painless-api-reference-LocalDate,LocalDate>>)++ (link:{java9-javadoc}/java/time/Period.html#between%2Djava.time.LocalDate%2Djava.time.LocalDate%2D[java 9])
+* ++[[painless-api-reference-Period-from-1]]static <<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/Period.html#from%2Djava.time.temporal.TemporalAmount%2D[from](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/Period.html#from%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-Period-of-3]]static <<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/Period.html#of%2Dint%2Dint%2Dint%2D[of](int, int, int)++ (link:{java9-javadoc}/java/time/Period.html#of%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Period-ofDays-1]]static <<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/Period.html#ofDays%2Dint%2D[ofDays](int)++ (link:{java9-javadoc}/java/time/Period.html#ofDays%2Dint%2D[java 9])
+* ++[[painless-api-reference-Period-ofMonths-1]]static <<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/Period.html#ofMonths%2Dint%2D[ofMonths](int)++ (link:{java9-javadoc}/java/time/Period.html#ofMonths%2Dint%2D[java 9])
+* ++[[painless-api-reference-Period-ofWeeks-1]]static <<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/Period.html#ofWeeks%2Dint%2D[ofWeeks](int)++ (link:{java9-javadoc}/java/time/Period.html#ofWeeks%2Dint%2D[java 9])
+* ++[[painless-api-reference-Period-ofYears-1]]static <<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/Period.html#ofYears%2Dint%2D[ofYears](int)++ (link:{java9-javadoc}/java/time/Period.html#ofYears%2Dint%2D[java 9])
+* ++[[painless-api-reference-Period-parse-1]]static <<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/Period.html#parse%2Djava.lang.CharSequence%2D[parse](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/time/Period.html#parse%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-Period-getChronology-0]]<<painless-api-reference-IsoChronology,IsoChronology>> link:{java8-javadoc}/java/time/Period.html#getChronology%2D%2D[getChronology]()++ (link:{java9-javadoc}/java/time/Period.html#getChronology%2D%2D[java 9])
+* ++[[painless-api-reference-Period-getDays-0]]int link:{java8-javadoc}/java/time/Period.html#getDays%2D%2D[getDays]()++ (link:{java9-javadoc}/java/time/Period.html#getDays%2D%2D[java 9])
+* ++[[painless-api-reference-Period-getMonths-0]]int link:{java8-javadoc}/java/time/Period.html#getMonths%2D%2D[getMonths]()++ (link:{java9-javadoc}/java/time/Period.html#getMonths%2D%2D[java 9])
+* ++[[painless-api-reference-Period-getYears-0]]int link:{java8-javadoc}/java/time/Period.html#getYears%2D%2D[getYears]()++ (link:{java9-javadoc}/java/time/Period.html#getYears%2D%2D[java 9])
+* ++[[painless-api-reference-Period-minus-1]]<<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/Period.html#minus%2Djava.time.temporal.TemporalAmount%2D[minus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/Period.html#minus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-Period-minusDays-1]]<<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/Period.html#minusDays%2Dlong%2D[minusDays](long)++ (link:{java9-javadoc}/java/time/Period.html#minusDays%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Period-minusMonths-1]]<<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/Period.html#minusMonths%2Dlong%2D[minusMonths](long)++ (link:{java9-javadoc}/java/time/Period.html#minusMonths%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Period-minusYears-1]]<<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/Period.html#minusYears%2Dlong%2D[minusYears](long)++ (link:{java9-javadoc}/java/time/Period.html#minusYears%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Period-multipliedBy-1]]<<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/Period.html#multipliedBy%2Dint%2D[multipliedBy](int)++ (link:{java9-javadoc}/java/time/Period.html#multipliedBy%2Dint%2D[java 9])
+* ++[[painless-api-reference-Period-negated-0]]<<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/Period.html#negated%2D%2D[negated]()++ (link:{java9-javadoc}/java/time/Period.html#negated%2D%2D[java 9])
+* ++[[painless-api-reference-Period-normalized-0]]<<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/Period.html#normalized%2D%2D[normalized]()++ (link:{java9-javadoc}/java/time/Period.html#normalized%2D%2D[java 9])
+* ++[[painless-api-reference-Period-plus-1]]<<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/Period.html#plus%2Djava.time.temporal.TemporalAmount%2D[plus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/Period.html#plus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-Period-plusDays-1]]<<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/Period.html#plusDays%2Dlong%2D[plusDays](long)++ (link:{java9-javadoc}/java/time/Period.html#plusDays%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Period-plusMonths-1]]<<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/Period.html#plusMonths%2Dlong%2D[plusMonths](long)++ (link:{java9-javadoc}/java/time/Period.html#plusMonths%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Period-plusYears-1]]<<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/Period.html#plusYears%2Dlong%2D[plusYears](long)++ (link:{java9-javadoc}/java/time/Period.html#plusYears%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Period-toTotalMonths-0]]long link:{java8-javadoc}/java/time/Period.html#toTotalMonths%2D%2D[toTotalMonths]()++ (link:{java9-javadoc}/java/time/Period.html#toTotalMonths%2D%2D[java 9])
+* ++[[painless-api-reference-Period-withDays-1]]<<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/Period.html#withDays%2Dint%2D[withDays](int)++ (link:{java9-javadoc}/java/time/Period.html#withDays%2Dint%2D[java 9])
+* ++[[painless-api-reference-Period-withMonths-1]]<<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/Period.html#withMonths%2Dint%2D[withMonths](int)++ (link:{java9-javadoc}/java/time/Period.html#withMonths%2Dint%2D[java 9])
+* ++[[painless-api-reference-Period-withYears-1]]<<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/Period.html#withYears%2Dint%2D[withYears](int)++ (link:{java9-javadoc}/java/time/Period.html#withYears%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-ChronoPeriod,ChronoPeriod>>++, ++<<painless-api-reference-TemporalAmount,TemporalAmount>>++
diff --git a/docs/painless/painless-api-reference/Predicate.asciidoc b/docs/painless/painless-api-reference/Predicate.asciidoc
new file mode 100644
index 0000000000000..6eef93ea27c02
--- /dev/null
+++ b/docs/painless/painless-api-reference/Predicate.asciidoc
@@ -0,0 +1,12 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Predicate]]++Predicate++::
+* ++[[painless-api-reference-Predicate-isEqual-1]]static <<painless-api-reference-Predicate,Predicate>> link:{java8-javadoc}/java/util/function/Predicate.html#isEqual%2Djava.lang.Object%2D[isEqual](def)++ (link:{java9-javadoc}/java/util/function/Predicate.html#isEqual%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Predicate-and-1]]<<painless-api-reference-Predicate,Predicate>> link:{java8-javadoc}/java/util/function/Predicate.html#and%2Djava.util.function.Predicate%2D[and](<<painless-api-reference-Predicate,Predicate>>)++ (link:{java9-javadoc}/java/util/function/Predicate.html#and%2Djava.util.function.Predicate%2D[java 9])
+* ++[[painless-api-reference-Predicate-negate-0]]<<painless-api-reference-Predicate,Predicate>> link:{java8-javadoc}/java/util/function/Predicate.html#negate%2D%2D[negate]()++ (link:{java9-javadoc}/java/util/function/Predicate.html#negate%2D%2D[java 9])
+* ++[[painless-api-reference-Predicate-or-1]]<<painless-api-reference-Predicate,Predicate>> link:{java8-javadoc}/java/util/function/Predicate.html#or%2Djava.util.function.Predicate%2D[or](<<painless-api-reference-Predicate,Predicate>>)++ (link:{java9-javadoc}/java/util/function/Predicate.html#or%2Djava.util.function.Predicate%2D[java 9])
+* ++[[painless-api-reference-Predicate-test-1]]boolean link:{java8-javadoc}/java/util/function/Predicate.html#test%2Djava.lang.Object%2D[test](def)++ (link:{java9-javadoc}/java/util/function/Predicate.html#test%2Djava.lang.Object%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/PrimitiveIterator.OfDouble.asciidoc b/docs/painless/painless-api-reference/PrimitiveIterator.OfDouble.asciidoc
new file mode 100644
index 0000000000000..678c1265f04c9
--- /dev/null
+++ b/docs/painless/painless-api-reference/PrimitiveIterator.OfDouble.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-PrimitiveIterator-OfDouble]]++PrimitiveIterator.OfDouble++::
+* ++[[painless-api-reference-PrimitiveIterator-OfDouble-next-0]]<<painless-api-reference-Double,Double>> link:{java8-javadoc}/java/util/PrimitiveIterator.OfDouble.html#next%2D%2D[next]()++ (link:{java9-javadoc}/java/util/PrimitiveIterator.OfDouble.html#next%2D%2D[java 9])
+* ++[[painless-api-reference-PrimitiveIterator-OfDouble-nextDouble-0]]double link:{java8-javadoc}/java/util/PrimitiveIterator.OfDouble.html#nextDouble%2D%2D[nextDouble]()++ (link:{java9-javadoc}/java/util/PrimitiveIterator.OfDouble.html#nextDouble%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Iterator,Iterator>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-PrimitiveIterator,PrimitiveIterator>>++
diff --git a/docs/painless/painless-api-reference/PrimitiveIterator.OfInt.asciidoc b/docs/painless/painless-api-reference/PrimitiveIterator.OfInt.asciidoc
new file mode 100644
index 0000000000000..0f7c1f31f0366
--- /dev/null
+++ b/docs/painless/painless-api-reference/PrimitiveIterator.OfInt.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-PrimitiveIterator-OfInt]]++PrimitiveIterator.OfInt++::
+* ++[[painless-api-reference-PrimitiveIterator-OfInt-next-0]]<<painless-api-reference-Integer,Integer>> link:{java8-javadoc}/java/util/PrimitiveIterator.OfInt.html#next%2D%2D[next]()++ (link:{java9-javadoc}/java/util/PrimitiveIterator.OfInt.html#next%2D%2D[java 9])
+* ++[[painless-api-reference-PrimitiveIterator-OfInt-nextInt-0]]int link:{java8-javadoc}/java/util/PrimitiveIterator.OfInt.html#nextInt%2D%2D[nextInt]()++ (link:{java9-javadoc}/java/util/PrimitiveIterator.OfInt.html#nextInt%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Iterator,Iterator>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-PrimitiveIterator,PrimitiveIterator>>++
diff --git a/docs/painless/painless-api-reference/PrimitiveIterator.OfLong.asciidoc b/docs/painless/painless-api-reference/PrimitiveIterator.OfLong.asciidoc
new file mode 100644
index 0000000000000..ce5b3f5b3a72a
--- /dev/null
+++ b/docs/painless/painless-api-reference/PrimitiveIterator.OfLong.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-PrimitiveIterator-OfLong]]++PrimitiveIterator.OfLong++::
+* ++[[painless-api-reference-PrimitiveIterator-OfLong-next-0]]<<painless-api-reference-Long,Long>> link:{java8-javadoc}/java/util/PrimitiveIterator.OfLong.html#next%2D%2D[next]()++ (link:{java9-javadoc}/java/util/PrimitiveIterator.OfLong.html#next%2D%2D[java 9])
+* ++[[painless-api-reference-PrimitiveIterator-OfLong-nextLong-0]]long link:{java8-javadoc}/java/util/PrimitiveIterator.OfLong.html#nextLong%2D%2D[nextLong]()++ (link:{java9-javadoc}/java/util/PrimitiveIterator.OfLong.html#nextLong%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Iterator,Iterator>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-PrimitiveIterator,PrimitiveIterator>>++
diff --git a/docs/painless/painless-api-reference/PrimitiveIterator.asciidoc b/docs/painless/painless-api-reference/PrimitiveIterator.asciidoc
new file mode 100644
index 0000000000000..42742f79d3d40
--- /dev/null
+++ b/docs/painless/painless-api-reference/PrimitiveIterator.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-PrimitiveIterator]]++PrimitiveIterator++::
+* ++[[painless-api-reference-PrimitiveIterator-forEachRemaining-1]]void link:{java8-javadoc}/java/util/PrimitiveIterator.html#forEachRemaining%2Djava.lang.Object%2D[forEachRemaining](def)++ (link:{java9-javadoc}/java/util/PrimitiveIterator.html#forEachRemaining%2Djava.lang.Object%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Iterator,Iterator>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/PriorityQueue.asciidoc b/docs/painless/painless-api-reference/PriorityQueue.asciidoc
new file mode 100644
index 0000000000000..88334cd8b9c63
--- /dev/null
+++ b/docs/painless/painless-api-reference/PriorityQueue.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-PriorityQueue]]++PriorityQueue++::
+* ++[[painless-api-reference-PriorityQueue-PriorityQueue-0]]link:{java8-javadoc}/java/util/PriorityQueue.html#PriorityQueue%2D%2D[PriorityQueue]()++ (link:{java9-javadoc}/java/util/PriorityQueue.html#PriorityQueue%2D%2D[java 9])
+* ++[[painless-api-reference-PriorityQueue-PriorityQueue-1]]link:{java8-javadoc}/java/util/PriorityQueue.html#PriorityQueue%2Djava.util.Comparator%2D[PriorityQueue](<<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/PriorityQueue.html#PriorityQueue%2Djava.util.Comparator%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-Queue,Queue>>++
diff --git a/docs/painless/painless-api-reference/Queue.asciidoc b/docs/painless/painless-api-reference/Queue.asciidoc
new file mode 100644
index 0000000000000..7d323457591ff
--- /dev/null
+++ b/docs/painless/painless-api-reference/Queue.asciidoc
@@ -0,0 +1,12 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Queue]]++Queue++::
+* ++[[painless-api-reference-Queue-element-0]]def link:{java8-javadoc}/java/util/Queue.html#element%2D%2D[element]()++ (link:{java9-javadoc}/java/util/Queue.html#element%2D%2D[java 9])
+* ++[[painless-api-reference-Queue-offer-1]]boolean link:{java8-javadoc}/java/util/Queue.html#offer%2Djava.lang.Object%2D[offer](def)++ (link:{java9-javadoc}/java/util/Queue.html#offer%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Queue-peek-0]]def link:{java8-javadoc}/java/util/Queue.html#peek%2D%2D[peek]()++ (link:{java9-javadoc}/java/util/Queue.html#peek%2D%2D[java 9])
+* ++[[painless-api-reference-Queue-poll-0]]def link:{java8-javadoc}/java/util/Queue.html#poll%2D%2D[poll]()++ (link:{java9-javadoc}/java/util/Queue.html#poll%2D%2D[java 9])
+* ++[[painless-api-reference-Queue-remove-0]]def link:{java8-javadoc}/java/util/Queue.html#remove%2D%2D[remove]()++ (link:{java9-javadoc}/java/util/Queue.html#remove%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Random.asciidoc b/docs/painless/painless-api-reference/Random.asciidoc
new file mode 100644
index 0000000000000..4288c330850da
--- /dev/null
+++ b/docs/painless/painless-api-reference/Random.asciidoc
@@ -0,0 +1,24 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Random]]++Random++::
+* ++[[painless-api-reference-Random-Random-0]]link:{java8-javadoc}/java/util/Random.html#Random%2D%2D[Random]()++ (link:{java9-javadoc}/java/util/Random.html#Random%2D%2D[java 9])
+* ++[[painless-api-reference-Random-Random-1]]link:{java8-javadoc}/java/util/Random.html#Random%2Dlong%2D[Random](long)++ (link:{java9-javadoc}/java/util/Random.html#Random%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Random-doubles-1]]<<painless-api-reference-DoubleStream,DoubleStream>> link:{java8-javadoc}/java/util/Random.html#doubles%2Dlong%2D[doubles](long)++ (link:{java9-javadoc}/java/util/Random.html#doubles%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Random-doubles-3]]<<painless-api-reference-DoubleStream,DoubleStream>> link:{java8-javadoc}/java/util/Random.html#doubles%2Dlong%2Ddouble%2Ddouble%2D[doubles](long, double, double)++ (link:{java9-javadoc}/java/util/Random.html#doubles%2Dlong%2Ddouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Random-ints-1]]<<painless-api-reference-IntStream,IntStream>> link:{java8-javadoc}/java/util/Random.html#ints%2Dlong%2D[ints](long)++ (link:{java9-javadoc}/java/util/Random.html#ints%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Random-ints-3]]<<painless-api-reference-IntStream,IntStream>> link:{java8-javadoc}/java/util/Random.html#ints%2Dlong%2Dint%2Dint%2D[ints](long, int, int)++ (link:{java9-javadoc}/java/util/Random.html#ints%2Dlong%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Random-longs-1]]<<painless-api-reference-LongStream,LongStream>> link:{java8-javadoc}/java/util/Random.html#longs%2Dlong%2D[longs](long)++ (link:{java9-javadoc}/java/util/Random.html#longs%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Random-longs-3]]<<painless-api-reference-LongStream,LongStream>> link:{java8-javadoc}/java/util/Random.html#longs%2Dlong%2Dlong%2Dlong%2D[longs](long, long, long)++ (link:{java9-javadoc}/java/util/Random.html#longs%2Dlong%2Dlong%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Random-nextBoolean-0]]boolean link:{java8-javadoc}/java/util/Random.html#nextBoolean%2D%2D[nextBoolean]()++ (link:{java9-javadoc}/java/util/Random.html#nextBoolean%2D%2D[java 9])
+* ++[[painless-api-reference-Random-nextBytes-1]]void link:{java8-javadoc}/java/util/Random.html#nextBytes%2Dbyte:A%2D[nextBytes](byte[])++ (link:{java9-javadoc}/java/util/Random.html#nextBytes%2Dbyte:A%2D[java 9])
+* ++[[painless-api-reference-Random-nextDouble-0]]double link:{java8-javadoc}/java/util/Random.html#nextDouble%2D%2D[nextDouble]()++ (link:{java9-javadoc}/java/util/Random.html#nextDouble%2D%2D[java 9])
+* ++[[painless-api-reference-Random-nextFloat-0]]float link:{java8-javadoc}/java/util/Random.html#nextFloat%2D%2D[nextFloat]()++ (link:{java9-javadoc}/java/util/Random.html#nextFloat%2D%2D[java 9])
+* ++[[painless-api-reference-Random-nextGaussian-0]]double link:{java8-javadoc}/java/util/Random.html#nextGaussian%2D%2D[nextGaussian]()++ (link:{java9-javadoc}/java/util/Random.html#nextGaussian%2D%2D[java 9])
+* ++[[painless-api-reference-Random-nextInt-0]]int link:{java8-javadoc}/java/util/Random.html#nextInt%2D%2D[nextInt]()++ (link:{java9-javadoc}/java/util/Random.html#nextInt%2D%2D[java 9])
+* ++[[painless-api-reference-Random-nextInt-1]]int link:{java8-javadoc}/java/util/Random.html#nextInt%2Dint%2D[nextInt](int)++ (link:{java9-javadoc}/java/util/Random.html#nextInt%2Dint%2D[java 9])
+* ++[[painless-api-reference-Random-nextLong-0]]long link:{java8-javadoc}/java/util/Random.html#nextLong%2D%2D[nextLong]()++ (link:{java9-javadoc}/java/util/Random.html#nextLong%2D%2D[java 9])
+* ++[[painless-api-reference-Random-setSeed-1]]void link:{java8-javadoc}/java/util/Random.html#setSeed%2Dlong%2D[setSeed](long)++ (link:{java9-javadoc}/java/util/Random.html#setSeed%2Dlong%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/RandomAccess.asciidoc b/docs/painless/painless-api-reference/RandomAccess.asciidoc
new file mode 100644
index 0000000000000..b7afb205f0c6c
--- /dev/null
+++ b/docs/painless/painless-api-reference/RandomAccess.asciidoc
@@ -0,0 +1,7 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-RandomAccess]]++RandomAccess++::
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/ReflectiveOperationException.asciidoc b/docs/painless/painless-api-reference/ReflectiveOperationException.asciidoc
new file mode 100644
index 0000000000000..eee134d373a93
--- /dev/null
+++ b/docs/painless/painless-api-reference/ReflectiveOperationException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ReflectiveOperationException]]++ReflectiveOperationException++::
+* ++[[painless-api-reference-ReflectiveOperationException-ReflectiveOperationException-0]]link:{java8-javadoc}/java/lang/ReflectiveOperationException.html#ReflectiveOperationException%2D%2D[ReflectiveOperationException]()++ (link:{java9-javadoc}/java/lang/ReflectiveOperationException.html#ReflectiveOperationException%2D%2D[java 9])
+* ++[[painless-api-reference-ReflectiveOperationException-ReflectiveOperationException-1]]link:{java8-javadoc}/java/lang/ReflectiveOperationException.html#ReflectiveOperationException%2Djava.lang.String%2D[ReflectiveOperationException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/ReflectiveOperationException.html#ReflectiveOperationException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/ResolverStyle.asciidoc b/docs/painless/painless-api-reference/ResolverStyle.asciidoc
new file mode 100644
index 0000000000000..00576a2d93a0e
--- /dev/null
+++ b/docs/painless/painless-api-reference/ResolverStyle.asciidoc
@@ -0,0 +1,12 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ResolverStyle]]++ResolverStyle++::
+** [[painless-api-reference-ResolverStyle-LENIENT]]static <<painless-api-reference-ResolverStyle,ResolverStyle>> link:{java8-javadoc}/java/time/format/ResolverStyle.html#LENIENT[LENIENT] (link:{java9-javadoc}/java/time/format/ResolverStyle.html#LENIENT[java 9])
+** [[painless-api-reference-ResolverStyle-SMART]]static <<painless-api-reference-ResolverStyle,ResolverStyle>> link:{java8-javadoc}/java/time/format/ResolverStyle.html#SMART[SMART] (link:{java9-javadoc}/java/time/format/ResolverStyle.html#SMART[java 9])
+** [[painless-api-reference-ResolverStyle-STRICT]]static <<painless-api-reference-ResolverStyle,ResolverStyle>> link:{java8-javadoc}/java/time/format/ResolverStyle.html#STRICT[STRICT] (link:{java9-javadoc}/java/time/format/ResolverStyle.html#STRICT[java 9])
+* ++[[painless-api-reference-ResolverStyle-valueOf-1]]static <<painless-api-reference-ResolverStyle,ResolverStyle>> link:{java8-javadoc}/java/time/format/ResolverStyle.html#valueOf%2Djava.lang.String%2D[valueOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/format/ResolverStyle.html#valueOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-ResolverStyle-values-0]]static <<painless-api-reference-ResolverStyle,ResolverStyle>>[] link:{java8-javadoc}/java/time/format/ResolverStyle.html#values%2D%2D[values]()++ (link:{java9-javadoc}/java/time/format/ResolverStyle.html#values%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Enum,Enum>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/RoundingMode.asciidoc b/docs/painless/painless-api-reference/RoundingMode.asciidoc
new file mode 100644
index 0000000000000..e0b4d579835e9
--- /dev/null
+++ b/docs/painless/painless-api-reference/RoundingMode.asciidoc
@@ -0,0 +1,17 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-RoundingMode]]++RoundingMode++::
+** [[painless-api-reference-RoundingMode-CEILING]]static <<painless-api-reference-RoundingMode,RoundingMode>> link:{java8-javadoc}/java/math/RoundingMode.html#CEILING[CEILING] (link:{java9-javadoc}/java/math/RoundingMode.html#CEILING[java 9])
+** [[painless-api-reference-RoundingMode-DOWN]]static <<painless-api-reference-RoundingMode,RoundingMode>> link:{java8-javadoc}/java/math/RoundingMode.html#DOWN[DOWN] (link:{java9-javadoc}/java/math/RoundingMode.html#DOWN[java 9])
+** [[painless-api-reference-RoundingMode-FLOOR]]static <<painless-api-reference-RoundingMode,RoundingMode>> link:{java8-javadoc}/java/math/RoundingMode.html#FLOOR[FLOOR] (link:{java9-javadoc}/java/math/RoundingMode.html#FLOOR[java 9])
+** [[painless-api-reference-RoundingMode-HALF_DOWN]]static <<painless-api-reference-RoundingMode,RoundingMode>> link:{java8-javadoc}/java/math/RoundingMode.html#HALF_DOWN[HALF_DOWN] (link:{java9-javadoc}/java/math/RoundingMode.html#HALF_DOWN[java 9])
+** [[painless-api-reference-RoundingMode-HALF_EVEN]]static <<painless-api-reference-RoundingMode,RoundingMode>> link:{java8-javadoc}/java/math/RoundingMode.html#HALF_EVEN[HALF_EVEN] (link:{java9-javadoc}/java/math/RoundingMode.html#HALF_EVEN[java 9])
+** [[painless-api-reference-RoundingMode-HALF_UP]]static <<painless-api-reference-RoundingMode,RoundingMode>> link:{java8-javadoc}/java/math/RoundingMode.html#HALF_UP[HALF_UP] (link:{java9-javadoc}/java/math/RoundingMode.html#HALF_UP[java 9])
+** [[painless-api-reference-RoundingMode-UNNECESSARY]]static <<painless-api-reference-RoundingMode,RoundingMode>> link:{java8-javadoc}/java/math/RoundingMode.html#UNNECESSARY[UNNECESSARY] (link:{java9-javadoc}/java/math/RoundingMode.html#UNNECESSARY[java 9])
+** [[painless-api-reference-RoundingMode-UP]]static <<painless-api-reference-RoundingMode,RoundingMode>> link:{java8-javadoc}/java/math/RoundingMode.html#UP[UP] (link:{java9-javadoc}/java/math/RoundingMode.html#UP[java 9])
+* ++[[painless-api-reference-RoundingMode-valueOf-1]]static <<painless-api-reference-RoundingMode,RoundingMode>> link:{java8-javadoc}/java/math/RoundingMode.html#valueOf%2Djava.lang.String%2D[valueOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/math/RoundingMode.html#valueOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-RoundingMode-values-0]]static <<painless-api-reference-RoundingMode,RoundingMode>>[] link:{java8-javadoc}/java/math/RoundingMode.html#values%2D%2D[values]()++ (link:{java9-javadoc}/java/math/RoundingMode.html#values%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Enum,Enum>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/RuleBasedCollator.asciidoc b/docs/painless/painless-api-reference/RuleBasedCollator.asciidoc
new file mode 100644
index 0000000000000..5581768e2f300
--- /dev/null
+++ b/docs/painless/painless-api-reference/RuleBasedCollator.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-RuleBasedCollator]]++RuleBasedCollator++::
+* ++[[painless-api-reference-RuleBasedCollator-RuleBasedCollator-1]]link:{java8-javadoc}/java/text/RuleBasedCollator.html#RuleBasedCollator%2Djava.lang.String%2D[RuleBasedCollator](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/RuleBasedCollator.html#RuleBasedCollator%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-RuleBasedCollator-getCollationElementIterator-1]]<<painless-api-reference-CollationElementIterator,CollationElementIterator>> link:{java8-javadoc}/java/text/RuleBasedCollator.html#getCollationElementIterator%2Djava.lang.String%2D[getCollationElementIterator](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/RuleBasedCollator.html#getCollationElementIterator%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-RuleBasedCollator-getRules-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/text/RuleBasedCollator.html#getRules%2D%2D[getRules]()++ (link:{java9-javadoc}/java/text/RuleBasedCollator.html#getRules%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Collator,Collator>>++, ++<<painless-api-reference-Comparator,Comparator>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/RuntimeException.asciidoc b/docs/painless/painless-api-reference/RuntimeException.asciidoc
new file mode 100644
index 0000000000000..fc8523169e33d
--- /dev/null
+++ b/docs/painless/painless-api-reference/RuntimeException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-RuntimeException]]++RuntimeException++::
+* ++[[painless-api-reference-RuntimeException-RuntimeException-0]]link:{java8-javadoc}/java/lang/RuntimeException.html#RuntimeException%2D%2D[RuntimeException]()++ (link:{java9-javadoc}/java/lang/RuntimeException.html#RuntimeException%2D%2D[java 9])
+* ++[[painless-api-reference-RuntimeException-RuntimeException-1]]link:{java8-javadoc}/java/lang/RuntimeException.html#RuntimeException%2Djava.lang.String%2D[RuntimeException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/RuntimeException.html#RuntimeException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/SecurityException.asciidoc b/docs/painless/painless-api-reference/SecurityException.asciidoc
new file mode 100644
index 0000000000000..ec6ea157e5587
--- /dev/null
+++ b/docs/painless/painless-api-reference/SecurityException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-SecurityException]]++SecurityException++::
+* ++[[painless-api-reference-SecurityException-SecurityException-0]]link:{java8-javadoc}/java/lang/SecurityException.html#SecurityException%2D%2D[SecurityException]()++ (link:{java9-javadoc}/java/lang/SecurityException.html#SecurityException%2D%2D[java 9])
+* ++[[painless-api-reference-SecurityException-SecurityException-1]]link:{java8-javadoc}/java/lang/SecurityException.html#SecurityException%2Djava.lang.String%2D[SecurityException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/SecurityException.html#SecurityException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Set.asciidoc b/docs/painless/painless-api-reference/Set.asciidoc
new file mode 100644
index 0000000000000..23d8a352f231f
--- /dev/null
+++ b/docs/painless/painless-api-reference/Set.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Set]]++Set++::
+* ++[[painless-api-reference-Set-equals-1]]boolean link:{java8-javadoc}/java/util/Set.html#equals%2Djava.lang.Object%2D[equals](<<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/util/Set.html#equals%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Set-hashCode-0]]int link:{java8-javadoc}/java/util/Set.html#hashCode%2D%2D[hashCode]()++ (link:{java9-javadoc}/java/util/Set.html#hashCode%2D%2D[java 9])
+* ++[[painless-api-reference-Set-remove-1]]boolean link:{java8-javadoc}/java/util/Set.html#remove%2Djava.lang.Object%2D[remove](def)++ (link:{java9-javadoc}/java/util/Set.html#remove%2Djava.lang.Object%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Short.asciidoc b/docs/painless/painless-api-reference/Short.asciidoc
new file mode 100644
index 0000000000000..12f0336e8fd40
--- /dev/null
+++ b/docs/painless/painless-api-reference/Short.asciidoc
@@ -0,0 +1,23 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Short]]++Short++::
+** [[painless-api-reference-Short-BYTES]]static int link:{java8-javadoc}/java/lang/Short.html#BYTES[BYTES] (link:{java9-javadoc}/java/lang/Short.html#BYTES[java 9])
+** [[painless-api-reference-Short-MAX_VALUE]]static short link:{java8-javadoc}/java/lang/Short.html#MAX_VALUE[MAX_VALUE] (link:{java9-javadoc}/java/lang/Short.html#MAX_VALUE[java 9])
+** [[painless-api-reference-Short-MIN_VALUE]]static short link:{java8-javadoc}/java/lang/Short.html#MIN_VALUE[MIN_VALUE] (link:{java9-javadoc}/java/lang/Short.html#MIN_VALUE[java 9])
+** [[painless-api-reference-Short-SIZE]]static int link:{java8-javadoc}/java/lang/Short.html#SIZE[SIZE] (link:{java9-javadoc}/java/lang/Short.html#SIZE[java 9])
+* ++[[painless-api-reference-Short-compare-2]]static int link:{java8-javadoc}/java/lang/Short.html#compare%2Dshort%2Dshort%2D[compare](short, short)++ (link:{java9-javadoc}/java/lang/Short.html#compare%2Dshort%2Dshort%2D[java 9])
+* ++[[painless-api-reference-Short-decode-1]]static <<painless-api-reference-Short,Short>> link:{java8-javadoc}/java/lang/Short.html#decode%2Djava.lang.String%2D[decode](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/Short.html#decode%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Short-hashCode-1]]static int link:{java8-javadoc}/java/lang/Short.html#hashCode%2Dshort%2D[hashCode](short)++ (link:{java9-javadoc}/java/lang/Short.html#hashCode%2Dshort%2D[java 9])
+* ++[[painless-api-reference-Short-parseShort-1]]static short link:{java8-javadoc}/java/lang/Short.html#parseShort%2Djava.lang.String%2D[parseShort](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/Short.html#parseShort%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Short-parseShort-2]]static short link:{java8-javadoc}/java/lang/Short.html#parseShort%2Djava.lang.String%2Dint%2D[parseShort](<<painless-api-reference-String,String>>, int)++ (link:{java9-javadoc}/java/lang/Short.html#parseShort%2Djava.lang.String%2Dint%2D[java 9])
+* ++[[painless-api-reference-Short-reverseBytes-1]]static short link:{java8-javadoc}/java/lang/Short.html#reverseBytes%2Dshort%2D[reverseBytes](short)++ (link:{java9-javadoc}/java/lang/Short.html#reverseBytes%2Dshort%2D[java 9])
+* ++[[painless-api-reference-Short-toString-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Short.html#toString%2Dshort%2D[toString](short)++ (link:{java9-javadoc}/java/lang/Short.html#toString%2Dshort%2D[java 9])
+* ++[[painless-api-reference-Short-toUnsignedInt-1]]static int link:{java8-javadoc}/java/lang/Short.html#toUnsignedInt%2Dshort%2D[toUnsignedInt](short)++ (link:{java9-javadoc}/java/lang/Short.html#toUnsignedInt%2Dshort%2D[java 9])
+* ++[[painless-api-reference-Short-toUnsignedLong-1]]static long link:{java8-javadoc}/java/lang/Short.html#toUnsignedLong%2Dshort%2D[toUnsignedLong](short)++ (link:{java9-javadoc}/java/lang/Short.html#toUnsignedLong%2Dshort%2D[java 9])
+* ++[[painless-api-reference-Short-valueOf-1]]static <<painless-api-reference-Short,Short>> link:{java8-javadoc}/java/lang/Short.html#valueOf%2Dshort%2D[valueOf](short)++ (link:{java9-javadoc}/java/lang/Short.html#valueOf%2Dshort%2D[java 9])
+* ++[[painless-api-reference-Short-valueOf-2]]static <<painless-api-reference-Short,Short>> link:{java8-javadoc}/java/lang/Short.html#valueOf%2Djava.lang.String%2Dint%2D[valueOf](<<painless-api-reference-String,String>>, int)++ (link:{java9-javadoc}/java/lang/Short.html#valueOf%2Djava.lang.String%2Dint%2D[java 9])
+* ++[[painless-api-reference-Short-compareTo-1]]int link:{java8-javadoc}/java/lang/Short.html#compareTo%2Djava.lang.Short%2D[compareTo](<<painless-api-reference-Short,Short>>)++ (link:{java9-javadoc}/java/lang/Short.html#compareTo%2Djava.lang.Short%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Number,Number>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/SignStyle.asciidoc b/docs/painless/painless-api-reference/SignStyle.asciidoc
new file mode 100644
index 0000000000000..a42ce94ebfa33
--- /dev/null
+++ b/docs/painless/painless-api-reference/SignStyle.asciidoc
@@ -0,0 +1,14 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-SignStyle]]++SignStyle++::
+** [[painless-api-reference-SignStyle-ALWAYS]]static <<painless-api-reference-SignStyle,SignStyle>> link:{java8-javadoc}/java/time/format/SignStyle.html#ALWAYS[ALWAYS] (link:{java9-javadoc}/java/time/format/SignStyle.html#ALWAYS[java 9])
+** [[painless-api-reference-SignStyle-EXCEEDS_PAD]]static <<painless-api-reference-SignStyle,SignStyle>> link:{java8-javadoc}/java/time/format/SignStyle.html#EXCEEDS_PAD[EXCEEDS_PAD] (link:{java9-javadoc}/java/time/format/SignStyle.html#EXCEEDS_PAD[java 9])
+** [[painless-api-reference-SignStyle-NEVER]]static <<painless-api-reference-SignStyle,SignStyle>> link:{java8-javadoc}/java/time/format/SignStyle.html#NEVER[NEVER] (link:{java9-javadoc}/java/time/format/SignStyle.html#NEVER[java 9])
+** [[painless-api-reference-SignStyle-NORMAL]]static <<painless-api-reference-SignStyle,SignStyle>> link:{java8-javadoc}/java/time/format/SignStyle.html#NORMAL[NORMAL] (link:{java9-javadoc}/java/time/format/SignStyle.html#NORMAL[java 9])
+** [[painless-api-reference-SignStyle-NOT_NEGATIVE]]static <<painless-api-reference-SignStyle,SignStyle>> link:{java8-javadoc}/java/time/format/SignStyle.html#NOT_NEGATIVE[NOT_NEGATIVE] (link:{java9-javadoc}/java/time/format/SignStyle.html#NOT_NEGATIVE[java 9])
+* ++[[painless-api-reference-SignStyle-valueOf-1]]static <<painless-api-reference-SignStyle,SignStyle>> link:{java8-javadoc}/java/time/format/SignStyle.html#valueOf%2Djava.lang.String%2D[valueOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/format/SignStyle.html#valueOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-SignStyle-values-0]]static <<painless-api-reference-SignStyle,SignStyle>>[] link:{java8-javadoc}/java/time/format/SignStyle.html#values%2D%2D[values]()++ (link:{java9-javadoc}/java/time/format/SignStyle.html#values%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Enum,Enum>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/SimpleDateFormat.asciidoc b/docs/painless/painless-api-reference/SimpleDateFormat.asciidoc
new file mode 100644
index 0000000000000..b7a8ffc191c45
--- /dev/null
+++ b/docs/painless/painless-api-reference/SimpleDateFormat.asciidoc
@@ -0,0 +1,18 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-SimpleDateFormat]]++SimpleDateFormat++::
+* ++[[painless-api-reference-SimpleDateFormat-SimpleDateFormat-0]]link:{java8-javadoc}/java/text/SimpleDateFormat.html#SimpleDateFormat%2D%2D[SimpleDateFormat]()++ (link:{java9-javadoc}/java/text/SimpleDateFormat.html#SimpleDateFormat%2D%2D[java 9])
+* ++[[painless-api-reference-SimpleDateFormat-SimpleDateFormat-1]]link:{java8-javadoc}/java/text/SimpleDateFormat.html#SimpleDateFormat%2Djava.lang.String%2D[SimpleDateFormat](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/SimpleDateFormat.html#SimpleDateFormat%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-SimpleDateFormat-SimpleDateFormat-2]]link:{java8-javadoc}/java/text/SimpleDateFormat.html#SimpleDateFormat%2Djava.lang.String%2Djava.util.Locale%2D[SimpleDateFormat](<<painless-api-reference-String,String>>, <<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/text/SimpleDateFormat.html#SimpleDateFormat%2Djava.lang.String%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-SimpleDateFormat-applyLocalizedPattern-1]]void link:{java8-javadoc}/java/text/SimpleDateFormat.html#applyLocalizedPattern%2Djava.lang.String%2D[applyLocalizedPattern](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/SimpleDateFormat.html#applyLocalizedPattern%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-SimpleDateFormat-applyPattern-1]]void link:{java8-javadoc}/java/text/SimpleDateFormat.html#applyPattern%2Djava.lang.String%2D[applyPattern](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/SimpleDateFormat.html#applyPattern%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-SimpleDateFormat-get2DigitYearStart-0]]<<painless-api-reference-Date,Date>> link:{java8-javadoc}/java/text/SimpleDateFormat.html#get2DigitYearStart%2D%2D[get2DigitYearStart]()++ (link:{java9-javadoc}/java/text/SimpleDateFormat.html#get2DigitYearStart%2D%2D[java 9])
+* ++[[painless-api-reference-SimpleDateFormat-getDateFormatSymbols-0]]<<painless-api-reference-DateFormatSymbols,DateFormatSymbols>> link:{java8-javadoc}/java/text/SimpleDateFormat.html#getDateFormatSymbols%2D%2D[getDateFormatSymbols]()++ (link:{java9-javadoc}/java/text/SimpleDateFormat.html#getDateFormatSymbols%2D%2D[java 9])
+* ++[[painless-api-reference-SimpleDateFormat-set2DigitYearStart-1]]void link:{java8-javadoc}/java/text/SimpleDateFormat.html#set2DigitYearStart%2Djava.util.Date%2D[set2DigitYearStart](<<painless-api-reference-Date,Date>>)++ (link:{java9-javadoc}/java/text/SimpleDateFormat.html#set2DigitYearStart%2Djava.util.Date%2D[java 9])
+* ++[[painless-api-reference-SimpleDateFormat-setDateFormatSymbols-1]]void link:{java8-javadoc}/java/text/SimpleDateFormat.html#setDateFormatSymbols%2Djava.text.DateFormatSymbols%2D[setDateFormatSymbols](<<painless-api-reference-DateFormatSymbols,DateFormatSymbols>>)++ (link:{java9-javadoc}/java/text/SimpleDateFormat.html#setDateFormatSymbols%2Djava.text.DateFormatSymbols%2D[java 9])
+* ++[[painless-api-reference-SimpleDateFormat-toLocalizedPattern-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/text/SimpleDateFormat.html#toLocalizedPattern%2D%2D[toLocalizedPattern]()++ (link:{java9-javadoc}/java/text/SimpleDateFormat.html#toLocalizedPattern%2D%2D[java 9])
+* ++[[painless-api-reference-SimpleDateFormat-toPattern-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/text/SimpleDateFormat.html#toPattern%2D%2D[toPattern]()++ (link:{java9-javadoc}/java/text/SimpleDateFormat.html#toPattern%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-DateFormat,DateFormat>>++, ++<<painless-api-reference-Format,Format>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/SimpleTimeZone.asciidoc b/docs/painless/painless-api-reference/SimpleTimeZone.asciidoc
new file mode 100644
index 0000000000000..eff23a40723a3
--- /dev/null
+++ b/docs/painless/painless-api-reference/SimpleTimeZone.asciidoc
@@ -0,0 +1,23 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-SimpleTimeZone]]++SimpleTimeZone++::
+** [[painless-api-reference-SimpleTimeZone-STANDARD_TIME]]static int link:{java8-javadoc}/java/util/SimpleTimeZone.html#STANDARD_TIME[STANDARD_TIME] (link:{java9-javadoc}/java/util/SimpleTimeZone.html#STANDARD_TIME[java 9])
+** [[painless-api-reference-SimpleTimeZone-UTC_TIME]]static int link:{java8-javadoc}/java/util/SimpleTimeZone.html#UTC_TIME[UTC_TIME] (link:{java9-javadoc}/java/util/SimpleTimeZone.html#UTC_TIME[java 9])
+** [[painless-api-reference-SimpleTimeZone-WALL_TIME]]static int link:{java8-javadoc}/java/util/SimpleTimeZone.html#WALL_TIME[WALL_TIME] (link:{java9-javadoc}/java/util/SimpleTimeZone.html#WALL_TIME[java 9])
+* ++[[painless-api-reference-SimpleTimeZone-SimpleTimeZone-2]]link:{java8-javadoc}/java/util/SimpleTimeZone.html#SimpleTimeZone%2Dint%2Djava.lang.String%2D[SimpleTimeZone](int, <<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/SimpleTimeZone.html#SimpleTimeZone%2Dint%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-SimpleTimeZone-SimpleTimeZone-10]]link:{java8-javadoc}/java/util/SimpleTimeZone.html#SimpleTimeZone%2Dint%2Djava.lang.String%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2D[SimpleTimeZone](int, <<painless-api-reference-String,String>>, int, int, int, int, int, int, int, int)++ (link:{java9-javadoc}/java/util/SimpleTimeZone.html#SimpleTimeZone%2Dint%2Djava.lang.String%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-SimpleTimeZone-SimpleTimeZone-11]]link:{java8-javadoc}/java/util/SimpleTimeZone.html#SimpleTimeZone%2Dint%2Djava.lang.String%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2D[SimpleTimeZone](int, <<painless-api-reference-String,String>>, int, int, int, int, int, int, int, int, int)++ (link:{java9-javadoc}/java/util/SimpleTimeZone.html#SimpleTimeZone%2Dint%2Djava.lang.String%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-SimpleTimeZone-SimpleTimeZone-13]]link:{java8-javadoc}/java/util/SimpleTimeZone.html#SimpleTimeZone%2Dint%2Djava.lang.String%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2D[SimpleTimeZone](int, <<painless-api-reference-String,String>>, int, int, int, int, int, int, int, int, int, int, int)++ (link:{java9-javadoc}/java/util/SimpleTimeZone.html#SimpleTimeZone%2Dint%2Djava.lang.String%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-SimpleTimeZone-getDSTSavings-0]]int link:{java8-javadoc}/java/util/SimpleTimeZone.html#getDSTSavings%2D%2D[getDSTSavings]()++ (link:{java9-javadoc}/java/util/SimpleTimeZone.html#getDSTSavings%2D%2D[java 9])
+* ++[[painless-api-reference-SimpleTimeZone-setDSTSavings-1]]void link:{java8-javadoc}/java/util/SimpleTimeZone.html#setDSTSavings%2Dint%2D[setDSTSavings](int)++ (link:{java9-javadoc}/java/util/SimpleTimeZone.html#setDSTSavings%2Dint%2D[java 9])
+* ++[[painless-api-reference-SimpleTimeZone-setEndRule-3]]void link:{java8-javadoc}/java/util/SimpleTimeZone.html#setEndRule%2Dint%2Dint%2Dint%2D[setEndRule](int, int, int)++ (link:{java9-javadoc}/java/util/SimpleTimeZone.html#setEndRule%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-SimpleTimeZone-setEndRule-4]]void link:{java8-javadoc}/java/util/SimpleTimeZone.html#setEndRule%2Dint%2Dint%2Dint%2Dint%2D[setEndRule](int, int, int, int)++ (link:{java9-javadoc}/java/util/SimpleTimeZone.html#setEndRule%2Dint%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-SimpleTimeZone-setEndRule-5]]void link:{java8-javadoc}/java/util/SimpleTimeZone.html#setEndRule%2Dint%2Dint%2Dint%2Dint%2Dboolean%2D[setEndRule](int, int, int, int, boolean)++ (link:{java9-javadoc}/java/util/SimpleTimeZone.html#setEndRule%2Dint%2Dint%2Dint%2Dint%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-SimpleTimeZone-setStartRule-3]]void link:{java8-javadoc}/java/util/SimpleTimeZone.html#setStartRule%2Dint%2Dint%2Dint%2D[setStartRule](int, int, int)++ (link:{java9-javadoc}/java/util/SimpleTimeZone.html#setStartRule%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-SimpleTimeZone-setStartRule-4]]void link:{java8-javadoc}/java/util/SimpleTimeZone.html#setStartRule%2Dint%2Dint%2Dint%2Dint%2D[setStartRule](int, int, int, int)++ (link:{java9-javadoc}/java/util/SimpleTimeZone.html#setStartRule%2Dint%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-SimpleTimeZone-setStartRule-5]]void link:{java8-javadoc}/java/util/SimpleTimeZone.html#setStartRule%2Dint%2Dint%2Dint%2Dint%2Dboolean%2D[setStartRule](int, int, int, int, boolean)++ (link:{java9-javadoc}/java/util/SimpleTimeZone.html#setStartRule%2Dint%2Dint%2Dint%2Dint%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-SimpleTimeZone-setStartYear-1]]void link:{java8-javadoc}/java/util/SimpleTimeZone.html#setStartYear%2Dint%2D[setStartYear](int)++ (link:{java9-javadoc}/java/util/SimpleTimeZone.html#setStartYear%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-TimeZone,TimeZone>>++
diff --git a/docs/painless/painless-api-reference/SortedMap.asciidoc b/docs/painless/painless-api-reference/SortedMap.asciidoc
new file mode 100644
index 0000000000000..d4e31e6c8fa37
--- /dev/null
+++ b/docs/painless/painless-api-reference/SortedMap.asciidoc
@@ -0,0 +1,13 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-SortedMap]]++SortedMap++::
+* ++[[painless-api-reference-SortedMap-comparator-0]]<<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/SortedMap.html#comparator%2D%2D[comparator]()++ (link:{java9-javadoc}/java/util/SortedMap.html#comparator%2D%2D[java 9])
+* ++[[painless-api-reference-SortedMap-firstKey-0]]def link:{java8-javadoc}/java/util/SortedMap.html#firstKey%2D%2D[firstKey]()++ (link:{java9-javadoc}/java/util/SortedMap.html#firstKey%2D%2D[java 9])
+* ++[[painless-api-reference-SortedMap-headMap-1]]<<painless-api-reference-SortedMap,SortedMap>> link:{java8-javadoc}/java/util/SortedMap.html#headMap%2Djava.lang.Object%2D[headMap](def)++ (link:{java9-javadoc}/java/util/SortedMap.html#headMap%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-SortedMap-lastKey-0]]def link:{java8-javadoc}/java/util/SortedMap.html#lastKey%2D%2D[lastKey]()++ (link:{java9-javadoc}/java/util/SortedMap.html#lastKey%2D%2D[java 9])
+* ++[[painless-api-reference-SortedMap-subMap-2]]<<painless-api-reference-SortedMap,SortedMap>> link:{java8-javadoc}/java/util/SortedMap.html#subMap%2Djava.lang.Object%2Djava.lang.Object%2D[subMap](def, def)++ (link:{java9-javadoc}/java/util/SortedMap.html#subMap%2Djava.lang.Object%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-SortedMap-tailMap-1]]<<painless-api-reference-SortedMap,SortedMap>> link:{java8-javadoc}/java/util/SortedMap.html#tailMap%2Djava.lang.Object%2D[tailMap](def)++ (link:{java9-javadoc}/java/util/SortedMap.html#tailMap%2Djava.lang.Object%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Map,Map>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/SortedSet.asciidoc b/docs/painless/painless-api-reference/SortedSet.asciidoc
new file mode 100644
index 0000000000000..2a7160aa2ba44
--- /dev/null
+++ b/docs/painless/painless-api-reference/SortedSet.asciidoc
@@ -0,0 +1,13 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-SortedSet]]++SortedSet++::
+* ++[[painless-api-reference-SortedSet-comparator-0]]<<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/SortedSet.html#comparator%2D%2D[comparator]()++ (link:{java9-javadoc}/java/util/SortedSet.html#comparator%2D%2D[java 9])
+* ++[[painless-api-reference-SortedSet-first-0]]def link:{java8-javadoc}/java/util/SortedSet.html#first%2D%2D[first]()++ (link:{java9-javadoc}/java/util/SortedSet.html#first%2D%2D[java 9])
+* ++[[painless-api-reference-SortedSet-headSet-1]]<<painless-api-reference-SortedSet,SortedSet>> link:{java8-javadoc}/java/util/SortedSet.html#headSet%2Djava.lang.Object%2D[headSet](def)++ (link:{java9-javadoc}/java/util/SortedSet.html#headSet%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-SortedSet-last-0]]def link:{java8-javadoc}/java/util/SortedSet.html#last%2D%2D[last]()++ (link:{java9-javadoc}/java/util/SortedSet.html#last%2D%2D[java 9])
+* ++[[painless-api-reference-SortedSet-subSet-2]]<<painless-api-reference-SortedSet,SortedSet>> link:{java8-javadoc}/java/util/SortedSet.html#subSet%2Djava.lang.Object%2Djava.lang.Object%2D[subSet](def, def)++ (link:{java9-javadoc}/java/util/SortedSet.html#subSet%2Djava.lang.Object%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-SortedSet-tailSet-1]]<<painless-api-reference-SortedSet,SortedSet>> link:{java8-javadoc}/java/util/SortedSet.html#tailSet%2Djava.lang.Object%2D[tailSet](def)++ (link:{java9-javadoc}/java/util/SortedSet.html#tailSet%2Djava.lang.Object%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-Set,Set>>++
diff --git a/docs/painless/painless-api-reference/Spliterator.OfDouble.asciidoc b/docs/painless/painless-api-reference/Spliterator.OfDouble.asciidoc
new file mode 100644
index 0000000000000..4722a8e6e0005
--- /dev/null
+++ b/docs/painless/painless-api-reference/Spliterator.OfDouble.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Spliterator-OfDouble]]++Spliterator.OfDouble++::
+* ++[[painless-api-reference-Spliterator-OfDouble-trySplit-0]]<<painless-api-reference-Spliterator-OfDouble,Spliterator.OfDouble>> link:{java8-javadoc}/java/util/Spliterator.OfDouble.html#trySplit%2D%2D[trySplit]()++ (link:{java9-javadoc}/java/util/Spliterator.OfDouble.html#trySplit%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-Spliterator,Spliterator>>++, ++<<painless-api-reference-Spliterator-OfPrimitive,Spliterator.OfPrimitive>>++
diff --git a/docs/painless/painless-api-reference/Spliterator.OfInt.asciidoc b/docs/painless/painless-api-reference/Spliterator.OfInt.asciidoc
new file mode 100644
index 0000000000000..e70221d6bf1c7
--- /dev/null
+++ b/docs/painless/painless-api-reference/Spliterator.OfInt.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Spliterator-OfInt]]++Spliterator.OfInt++::
+* ++[[painless-api-reference-Spliterator-OfInt-trySplit-0]]<<painless-api-reference-Spliterator-OfInt,Spliterator.OfInt>> link:{java8-javadoc}/java/util/Spliterator.OfInt.html#trySplit%2D%2D[trySplit]()++ (link:{java9-javadoc}/java/util/Spliterator.OfInt.html#trySplit%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-Spliterator,Spliterator>>++, ++<<painless-api-reference-Spliterator-OfPrimitive,Spliterator.OfPrimitive>>++
diff --git a/docs/painless/painless-api-reference/Spliterator.OfLong.asciidoc b/docs/painless/painless-api-reference/Spliterator.OfLong.asciidoc
new file mode 100644
index 0000000000000..8f709add01e17
--- /dev/null
+++ b/docs/painless/painless-api-reference/Spliterator.OfLong.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Spliterator-OfLong]]++Spliterator.OfLong++::
+* ++[[painless-api-reference-Spliterator-OfLong-trySplit-0]]<<painless-api-reference-Spliterator-OfLong,Spliterator.OfLong>> link:{java8-javadoc}/java/util/Spliterator.OfLong.html#trySplit%2D%2D[trySplit]()++ (link:{java9-javadoc}/java/util/Spliterator.OfLong.html#trySplit%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-Spliterator,Spliterator>>++, ++<<painless-api-reference-Spliterator-OfPrimitive,Spliterator.OfPrimitive>>++
diff --git a/docs/painless/painless-api-reference/Spliterator.OfPrimitive.asciidoc b/docs/painless/painless-api-reference/Spliterator.OfPrimitive.asciidoc
new file mode 100644
index 0000000000000..297c213b44f50
--- /dev/null
+++ b/docs/painless/painless-api-reference/Spliterator.OfPrimitive.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Spliterator-OfPrimitive]]++Spliterator.OfPrimitive++::
+* ++[[painless-api-reference-Spliterator-OfPrimitive-forEachRemaining-1]]void link:{java8-javadoc}/java/util/Spliterator.OfPrimitive.html#forEachRemaining%2Djava.lang.Object%2D[forEachRemaining](def)++ (link:{java9-javadoc}/java/util/Spliterator.OfPrimitive.html#forEachRemaining%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Spliterator-OfPrimitive-tryAdvance-1]]boolean link:{java8-javadoc}/java/util/Spliterator.OfPrimitive.html#tryAdvance%2Djava.lang.Object%2D[tryAdvance](def)++ (link:{java9-javadoc}/java/util/Spliterator.OfPrimitive.html#tryAdvance%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Spliterator-OfPrimitive-trySplit-0]]<<painless-api-reference-Spliterator-OfPrimitive,Spliterator.OfPrimitive>> link:{java8-javadoc}/java/util/Spliterator.OfPrimitive.html#trySplit%2D%2D[trySplit]()++ (link:{java9-javadoc}/java/util/Spliterator.OfPrimitive.html#trySplit%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-Spliterator,Spliterator>>++
diff --git a/docs/painless/painless-api-reference/Spliterator.asciidoc b/docs/painless/painless-api-reference/Spliterator.asciidoc
new file mode 100644
index 0000000000000..fcf919409858d
--- /dev/null
+++ b/docs/painless/painless-api-reference/Spliterator.asciidoc
@@ -0,0 +1,23 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Spliterator]]++Spliterator++::
+** [[painless-api-reference-Spliterator-CONCURRENT]]static int link:{java8-javadoc}/java/util/Spliterator.html#CONCURRENT[CONCURRENT] (link:{java9-javadoc}/java/util/Spliterator.html#CONCURRENT[java 9])
+** [[painless-api-reference-Spliterator-DISTINCT]]static int link:{java8-javadoc}/java/util/Spliterator.html#DISTINCT[DISTINCT] (link:{java9-javadoc}/java/util/Spliterator.html#DISTINCT[java 9])
+** [[painless-api-reference-Spliterator-IMMUTABLE]]static int link:{java8-javadoc}/java/util/Spliterator.html#IMMUTABLE[IMMUTABLE] (link:{java9-javadoc}/java/util/Spliterator.html#IMMUTABLE[java 9])
+** [[painless-api-reference-Spliterator-NONNULL]]static int link:{java8-javadoc}/java/util/Spliterator.html#NONNULL[NONNULL] (link:{java9-javadoc}/java/util/Spliterator.html#NONNULL[java 9])
+** [[painless-api-reference-Spliterator-ORDERED]]static int link:{java8-javadoc}/java/util/Spliterator.html#ORDERED[ORDERED] (link:{java9-javadoc}/java/util/Spliterator.html#ORDERED[java 9])
+** [[painless-api-reference-Spliterator-SIZED]]static int link:{java8-javadoc}/java/util/Spliterator.html#SIZED[SIZED] (link:{java9-javadoc}/java/util/Spliterator.html#SIZED[java 9])
+** [[painless-api-reference-Spliterator-SORTED]]static int link:{java8-javadoc}/java/util/Spliterator.html#SORTED[SORTED] (link:{java9-javadoc}/java/util/Spliterator.html#SORTED[java 9])
+** [[painless-api-reference-Spliterator-SUBSIZED]]static int link:{java8-javadoc}/java/util/Spliterator.html#SUBSIZED[SUBSIZED] (link:{java9-javadoc}/java/util/Spliterator.html#SUBSIZED[java 9])
+* ++[[painless-api-reference-Spliterator-characteristics-0]]int link:{java8-javadoc}/java/util/Spliterator.html#characteristics%2D%2D[characteristics]()++ (link:{java9-javadoc}/java/util/Spliterator.html#characteristics%2D%2D[java 9])
+* ++[[painless-api-reference-Spliterator-estimateSize-0]]long link:{java8-javadoc}/java/util/Spliterator.html#estimateSize%2D%2D[estimateSize]()++ (link:{java9-javadoc}/java/util/Spliterator.html#estimateSize%2D%2D[java 9])
+* ++[[painless-api-reference-Spliterator-forEachRemaining-1]]void link:{java8-javadoc}/java/util/Spliterator.html#forEachRemaining%2Djava.util.function.Consumer%2D[forEachRemaining](<<painless-api-reference-Consumer,Consumer>>)++ (link:{java9-javadoc}/java/util/Spliterator.html#forEachRemaining%2Djava.util.function.Consumer%2D[java 9])
+* ++[[painless-api-reference-Spliterator-getComparator-0]]<<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/Spliterator.html#getComparator%2D%2D[getComparator]()++ (link:{java9-javadoc}/java/util/Spliterator.html#getComparator%2D%2D[java 9])
+* ++[[painless-api-reference-Spliterator-getExactSizeIfKnown-0]]long link:{java8-javadoc}/java/util/Spliterator.html#getExactSizeIfKnown%2D%2D[getExactSizeIfKnown]()++ (link:{java9-javadoc}/java/util/Spliterator.html#getExactSizeIfKnown%2D%2D[java 9])
+* ++[[painless-api-reference-Spliterator-hasCharacteristics-1]]boolean link:{java8-javadoc}/java/util/Spliterator.html#hasCharacteristics%2Dint%2D[hasCharacteristics](int)++ (link:{java9-javadoc}/java/util/Spliterator.html#hasCharacteristics%2Dint%2D[java 9])
+* ++[[painless-api-reference-Spliterator-tryAdvance-1]]boolean link:{java8-javadoc}/java/util/Spliterator.html#tryAdvance%2Djava.util.function.Consumer%2D[tryAdvance](<<painless-api-reference-Consumer,Consumer>>)++ (link:{java9-javadoc}/java/util/Spliterator.html#tryAdvance%2Djava.util.function.Consumer%2D[java 9])
+* ++[[painless-api-reference-Spliterator-trySplit-0]]<<painless-api-reference-Spliterator,Spliterator>> link:{java8-javadoc}/java/util/Spliterator.html#trySplit%2D%2D[trySplit]()++ (link:{java9-javadoc}/java/util/Spliterator.html#trySplit%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Spliterators.asciidoc b/docs/painless/painless-api-reference/Spliterators.asciidoc
new file mode 100644
index 0000000000000..6d8202a567411
--- /dev/null
+++ b/docs/painless/painless-api-reference/Spliterators.asciidoc
@@ -0,0 +1,15 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Spliterators]]++Spliterators++::
+* ++[[painless-api-reference-Spliterators-emptyDoubleSpliterator-0]]static <<painless-api-reference-Spliterator-OfDouble,Spliterator.OfDouble>> link:{java8-javadoc}/java/util/Spliterators.html#emptyDoubleSpliterator%2D%2D[emptyDoubleSpliterator]()++ (link:{java9-javadoc}/java/util/Spliterators.html#emptyDoubleSpliterator%2D%2D[java 9])
+* ++[[painless-api-reference-Spliterators-emptyIntSpliterator-0]]static <<painless-api-reference-Spliterator-OfInt,Spliterator.OfInt>> link:{java8-javadoc}/java/util/Spliterators.html#emptyIntSpliterator%2D%2D[emptyIntSpliterator]()++ (link:{java9-javadoc}/java/util/Spliterators.html#emptyIntSpliterator%2D%2D[java 9])
+* ++[[painless-api-reference-Spliterators-emptyLongSpliterator-0]]static <<painless-api-reference-Spliterator-OfLong,Spliterator.OfLong>> link:{java8-javadoc}/java/util/Spliterators.html#emptyLongSpliterator%2D%2D[emptyLongSpliterator]()++ (link:{java9-javadoc}/java/util/Spliterators.html#emptyLongSpliterator%2D%2D[java 9])
+* ++[[painless-api-reference-Spliterators-emptySpliterator-0]]static <<painless-api-reference-Spliterator,Spliterator>> link:{java8-javadoc}/java/util/Spliterators.html#emptySpliterator%2D%2D[emptySpliterator]()++ (link:{java9-javadoc}/java/util/Spliterators.html#emptySpliterator%2D%2D[java 9])
+* ++[[painless-api-reference-Spliterators-iterator-1]]static <<painless-api-reference-Iterator,Iterator>> link:{java8-javadoc}/java/util/Spliterators.html#iterator%2Djava.util.Spliterator%2D[iterator](<<painless-api-reference-Spliterator,Spliterator>>)++ (link:{java9-javadoc}/java/util/Spliterators.html#iterator%2Djava.util.Spliterator%2D[java 9])
+* ++[[painless-api-reference-Spliterators-spliterator-2]]static <<painless-api-reference-Spliterator,Spliterator>> link:{java8-javadoc}/java/util/Spliterators.html#spliterator%2Djava.util.Collection%2Dint%2D[spliterator](<<painless-api-reference-Collection,Collection>>, int)++ (link:{java9-javadoc}/java/util/Spliterators.html#spliterator%2Djava.util.Collection%2Dint%2D[java 9])
+* ++[[painless-api-reference-Spliterators-spliterator-3]]static <<painless-api-reference-Spliterator,Spliterator>> link:{java8-javadoc}/java/util/Spliterators.html#spliterator%2Djava.util.Iterator%2Dlong%2Dint%2D[spliterator](<<painless-api-reference-Iterator,Iterator>>, long, int)++ (link:{java9-javadoc}/java/util/Spliterators.html#spliterator%2Djava.util.Iterator%2Dlong%2Dint%2D[java 9])
+* ++[[painless-api-reference-Spliterators-spliteratorUnknownSize-2]]static <<painless-api-reference-Spliterator,Spliterator>> link:{java8-javadoc}/java/util/Spliterators.html#spliteratorUnknownSize%2Djava.util.Iterator%2Dint%2D[spliteratorUnknownSize](<<painless-api-reference-Iterator,Iterator>>, int)++ (link:{java9-javadoc}/java/util/Spliterators.html#spliteratorUnknownSize%2Djava.util.Iterator%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Stack.asciidoc b/docs/painless/painless-api-reference/Stack.asciidoc
new file mode 100644
index 0000000000000..ad1520d9d885d
--- /dev/null
+++ b/docs/painless/painless-api-reference/Stack.asciidoc
@@ -0,0 +1,13 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Stack]]++Stack++::
+* ++[[painless-api-reference-Stack-Stack-0]]link:{java8-javadoc}/java/util/Stack.html#Stack%2D%2D[Stack]()++ (link:{java9-javadoc}/java/util/Stack.html#Stack%2D%2D[java 9])
+* ++[[painless-api-reference-Stack-empty-0]]boolean link:{java8-javadoc}/java/util/Stack.html#empty%2D%2D[empty]()++ (link:{java9-javadoc}/java/util/Stack.html#empty%2D%2D[java 9])
+* ++[[painless-api-reference-Stack-peek-0]]def link:{java8-javadoc}/java/util/Stack.html#peek%2D%2D[peek]()++ (link:{java9-javadoc}/java/util/Stack.html#peek%2D%2D[java 9])
+* ++[[painless-api-reference-Stack-pop-0]]def link:{java8-javadoc}/java/util/Stack.html#pop%2D%2D[pop]()++ (link:{java9-javadoc}/java/util/Stack.html#pop%2D%2D[java 9])
+* ++[[painless-api-reference-Stack-push-1]]def link:{java8-javadoc}/java/util/Stack.html#push%2Djava.lang.Object%2D[push](def)++ (link:{java9-javadoc}/java/util/Stack.html#push%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Stack-search-1]]int link:{java8-javadoc}/java/util/Stack.html#search%2Djava.lang.Object%2D[search](def)++ (link:{java9-javadoc}/java/util/Stack.html#search%2Djava.lang.Object%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-List,List>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-Vector,Vector>>++
diff --git a/docs/painless/painless-api-reference/StackTraceElement.asciidoc b/docs/painless/painless-api-reference/StackTraceElement.asciidoc
new file mode 100644
index 0000000000000..046e56ef1f016
--- /dev/null
+++ b/docs/painless/painless-api-reference/StackTraceElement.asciidoc
@@ -0,0 +1,13 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-StackTraceElement]]++StackTraceElement++::
+* ++[[painless-api-reference-StackTraceElement-StackTraceElement-4]]link:{java8-javadoc}/java/lang/StackTraceElement.html#StackTraceElement%2Djava.lang.String%2Djava.lang.String%2Djava.lang.String%2Dint%2D[StackTraceElement](<<painless-api-reference-String,String>>, <<painless-api-reference-String,String>>, <<painless-api-reference-String,String>>, int)++ (link:{java9-javadoc}/java/lang/StackTraceElement.html#StackTraceElement%2Djava.lang.String%2Djava.lang.String%2Djava.lang.String%2Dint%2D[java 9])
+* ++[[painless-api-reference-StackTraceElement-getClassName-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/StackTraceElement.html#getClassName%2D%2D[getClassName]()++ (link:{java9-javadoc}/java/lang/StackTraceElement.html#getClassName%2D%2D[java 9])
+* ++[[painless-api-reference-StackTraceElement-getFileName-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/StackTraceElement.html#getFileName%2D%2D[getFileName]()++ (link:{java9-javadoc}/java/lang/StackTraceElement.html#getFileName%2D%2D[java 9])
+* ++[[painless-api-reference-StackTraceElement-getLineNumber-0]]int link:{java8-javadoc}/java/lang/StackTraceElement.html#getLineNumber%2D%2D[getLineNumber]()++ (link:{java9-javadoc}/java/lang/StackTraceElement.html#getLineNumber%2D%2D[java 9])
+* ++[[painless-api-reference-StackTraceElement-getMethodName-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/StackTraceElement.html#getMethodName%2D%2D[getMethodName]()++ (link:{java9-javadoc}/java/lang/StackTraceElement.html#getMethodName%2D%2D[java 9])
+* ++[[painless-api-reference-StackTraceElement-isNativeMethod-0]]boolean link:{java8-javadoc}/java/lang/StackTraceElement.html#isNativeMethod%2D%2D[isNativeMethod]()++ (link:{java9-javadoc}/java/lang/StackTraceElement.html#isNativeMethod%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Stream.Builder.asciidoc b/docs/painless/painless-api-reference/Stream.Builder.asciidoc
new file mode 100644
index 0000000000000..42d91e01a2703
--- /dev/null
+++ b/docs/painless/painless-api-reference/Stream.Builder.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Stream-Builder]]++Stream.Builder++::
+* ++[[painless-api-reference-Stream-Builder-add-1]]<<painless-api-reference-Stream-Builder,Stream.Builder>> link:{java8-javadoc}/java/util/stream/Stream.Builder.html#add%2Djava.lang.Object%2D[add](def)++ (link:{java9-javadoc}/java/util/stream/Stream.Builder.html#add%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Stream-Builder-build-0]]<<painless-api-reference-Stream,Stream>> link:{java8-javadoc}/java/util/stream/Stream.Builder.html#build%2D%2D[build]()++ (link:{java9-javadoc}/java/util/stream/Stream.Builder.html#build%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Consumer,Consumer>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Stream.asciidoc b/docs/painless/painless-api-reference/Stream.asciidoc
new file mode 100644
index 0000000000000..0713ad1ce9d90
--- /dev/null
+++ b/docs/painless/painless-api-reference/Stream.asciidoc
@@ -0,0 +1,43 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Stream]]++Stream++::
+* ++[[painless-api-reference-Stream-builder-0]]static <<painless-api-reference-Stream-Builder,Stream.Builder>> link:{java8-javadoc}/java/util/stream/Stream.html#builder%2D%2D[builder]()++ (link:{java9-javadoc}/java/util/stream/Stream.html#builder%2D%2D[java 9])
+* ++[[painless-api-reference-Stream-concat-2]]static <<painless-api-reference-Stream,Stream>> link:{java8-javadoc}/java/util/stream/Stream.html#concat%2Djava.util.stream.Stream%2Djava.util.stream.Stream%2D[concat](<<painless-api-reference-Stream,Stream>>, <<painless-api-reference-Stream,Stream>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#concat%2Djava.util.stream.Stream%2Djava.util.stream.Stream%2D[java 9])
+* ++[[painless-api-reference-Stream-empty-0]]static <<painless-api-reference-Stream,Stream>> link:{java8-javadoc}/java/util/stream/Stream.html#empty%2D%2D[empty]()++ (link:{java9-javadoc}/java/util/stream/Stream.html#empty%2D%2D[java 9])
+* ++[[painless-api-reference-Stream-of-1]]static <<painless-api-reference-Stream,Stream>> link:{java8-javadoc}/java/util/stream/Stream.html#of%2Djava.lang.Object:A%2D[of](def[])++ (link:{java9-javadoc}/java/util/stream/Stream.html#of%2Djava.lang.Object:A%2D[java 9])
+* ++[[painless-api-reference-Stream-allMatch-1]]boolean link:{java8-javadoc}/java/util/stream/Stream.html#allMatch%2Djava.util.function.Predicate%2D[allMatch](<<painless-api-reference-Predicate,Predicate>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#allMatch%2Djava.util.function.Predicate%2D[java 9])
+* ++[[painless-api-reference-Stream-anyMatch-1]]boolean link:{java8-javadoc}/java/util/stream/Stream.html#anyMatch%2Djava.util.function.Predicate%2D[anyMatch](<<painless-api-reference-Predicate,Predicate>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#anyMatch%2Djava.util.function.Predicate%2D[java 9])
+* ++[[painless-api-reference-Stream-collect-1]]def link:{java8-javadoc}/java/util/stream/Stream.html#collect%2Djava.util.stream.Collector%2D[collect](<<painless-api-reference-Collector,Collector>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#collect%2Djava.util.stream.Collector%2D[java 9])
+* ++[[painless-api-reference-Stream-collect-3]]def link:{java8-javadoc}/java/util/stream/Stream.html#collect%2Djava.util.function.Supplier%2Djava.util.function.BiConsumer%2Djava.util.function.BiConsumer%2D[collect](<<painless-api-reference-Supplier,Supplier>>, <<painless-api-reference-BiConsumer,BiConsumer>>, <<painless-api-reference-BiConsumer,BiConsumer>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#collect%2Djava.util.function.Supplier%2Djava.util.function.BiConsumer%2Djava.util.function.BiConsumer%2D[java 9])
+* ++[[painless-api-reference-Stream-count-0]]long link:{java8-javadoc}/java/util/stream/Stream.html#count%2D%2D[count]()++ (link:{java9-javadoc}/java/util/stream/Stream.html#count%2D%2D[java 9])
+* ++[[painless-api-reference-Stream-distinct-0]]<<painless-api-reference-Stream,Stream>> link:{java8-javadoc}/java/util/stream/Stream.html#distinct%2D%2D[distinct]()++ (link:{java9-javadoc}/java/util/stream/Stream.html#distinct%2D%2D[java 9])
+* ++[[painless-api-reference-Stream-filter-1]]<<painless-api-reference-Stream,Stream>> link:{java8-javadoc}/java/util/stream/Stream.html#filter%2Djava.util.function.Predicate%2D[filter](<<painless-api-reference-Predicate,Predicate>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#filter%2Djava.util.function.Predicate%2D[java 9])
+* ++[[painless-api-reference-Stream-findAny-0]]<<painless-api-reference-Optional,Optional>> link:{java8-javadoc}/java/util/stream/Stream.html#findAny%2D%2D[findAny]()++ (link:{java9-javadoc}/java/util/stream/Stream.html#findAny%2D%2D[java 9])
+* ++[[painless-api-reference-Stream-findFirst-0]]<<painless-api-reference-Optional,Optional>> link:{java8-javadoc}/java/util/stream/Stream.html#findFirst%2D%2D[findFirst]()++ (link:{java9-javadoc}/java/util/stream/Stream.html#findFirst%2D%2D[java 9])
+* ++[[painless-api-reference-Stream-flatMap-1]]<<painless-api-reference-Stream,Stream>> link:{java8-javadoc}/java/util/stream/Stream.html#flatMap%2Djava.util.function.Function%2D[flatMap](<<painless-api-reference-Function,Function>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#flatMap%2Djava.util.function.Function%2D[java 9])
+* ++[[painless-api-reference-Stream-flatMapToDouble-1]]<<painless-api-reference-DoubleStream,DoubleStream>> link:{java8-javadoc}/java/util/stream/Stream.html#flatMapToDouble%2Djava.util.function.Function%2D[flatMapToDouble](<<painless-api-reference-Function,Function>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#flatMapToDouble%2Djava.util.function.Function%2D[java 9])
+* ++[[painless-api-reference-Stream-flatMapToInt-1]]<<painless-api-reference-IntStream,IntStream>> link:{java8-javadoc}/java/util/stream/Stream.html#flatMapToInt%2Djava.util.function.Function%2D[flatMapToInt](<<painless-api-reference-Function,Function>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#flatMapToInt%2Djava.util.function.Function%2D[java 9])
+* ++[[painless-api-reference-Stream-flatMapToLong-1]]<<painless-api-reference-LongStream,LongStream>> link:{java8-javadoc}/java/util/stream/Stream.html#flatMapToLong%2Djava.util.function.Function%2D[flatMapToLong](<<painless-api-reference-Function,Function>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#flatMapToLong%2Djava.util.function.Function%2D[java 9])
+* ++[[painless-api-reference-Stream-forEach-1]]void link:{java8-javadoc}/java/util/stream/Stream.html#forEach%2Djava.util.function.Consumer%2D[forEach](<<painless-api-reference-Consumer,Consumer>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#forEach%2Djava.util.function.Consumer%2D[java 9])
+* ++[[painless-api-reference-Stream-forEachOrdered-1]]void link:{java8-javadoc}/java/util/stream/Stream.html#forEachOrdered%2Djava.util.function.Consumer%2D[forEachOrdered](<<painless-api-reference-Consumer,Consumer>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#forEachOrdered%2Djava.util.function.Consumer%2D[java 9])
+* ++[[painless-api-reference-Stream-limit-1]]<<painless-api-reference-Stream,Stream>> link:{java8-javadoc}/java/util/stream/Stream.html#limit%2Dlong%2D[limit](long)++ (link:{java9-javadoc}/java/util/stream/Stream.html#limit%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Stream-map-1]]<<painless-api-reference-Stream,Stream>> link:{java8-javadoc}/java/util/stream/Stream.html#map%2Djava.util.function.Function%2D[map](<<painless-api-reference-Function,Function>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#map%2Djava.util.function.Function%2D[java 9])
+* ++[[painless-api-reference-Stream-mapToDouble-1]]<<painless-api-reference-DoubleStream,DoubleStream>> link:{java8-javadoc}/java/util/stream/Stream.html#mapToDouble%2Djava.util.function.ToDoubleFunction%2D[mapToDouble](<<painless-api-reference-ToDoubleFunction,ToDoubleFunction>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#mapToDouble%2Djava.util.function.ToDoubleFunction%2D[java 9])
+* ++[[painless-api-reference-Stream-mapToInt-1]]<<painless-api-reference-IntStream,IntStream>> link:{java8-javadoc}/java/util/stream/Stream.html#mapToInt%2Djava.util.function.ToIntFunction%2D[mapToInt](<<painless-api-reference-ToIntFunction,ToIntFunction>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#mapToInt%2Djava.util.function.ToIntFunction%2D[java 9])
+* ++[[painless-api-reference-Stream-mapToLong-1]]<<painless-api-reference-LongStream,LongStream>> link:{java8-javadoc}/java/util/stream/Stream.html#mapToLong%2Djava.util.function.ToLongFunction%2D[mapToLong](<<painless-api-reference-ToLongFunction,ToLongFunction>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#mapToLong%2Djava.util.function.ToLongFunction%2D[java 9])
+* ++[[painless-api-reference-Stream-max-1]]<<painless-api-reference-Optional,Optional>> link:{java8-javadoc}/java/util/stream/Stream.html#max%2Djava.util.Comparator%2D[max](<<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#max%2Djava.util.Comparator%2D[java 9])
+* ++[[painless-api-reference-Stream-min-1]]<<painless-api-reference-Optional,Optional>> link:{java8-javadoc}/java/util/stream/Stream.html#min%2Djava.util.Comparator%2D[min](<<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#min%2Djava.util.Comparator%2D[java 9])
+* ++[[painless-api-reference-Stream-noneMatch-1]]boolean link:{java8-javadoc}/java/util/stream/Stream.html#noneMatch%2Djava.util.function.Predicate%2D[noneMatch](<<painless-api-reference-Predicate,Predicate>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#noneMatch%2Djava.util.function.Predicate%2D[java 9])
+* ++[[painless-api-reference-Stream-peek-1]]<<painless-api-reference-Stream,Stream>> link:{java8-javadoc}/java/util/stream/Stream.html#peek%2Djava.util.function.Consumer%2D[peek](<<painless-api-reference-Consumer,Consumer>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#peek%2Djava.util.function.Consumer%2D[java 9])
+* ++[[painless-api-reference-Stream-reduce-1]]<<painless-api-reference-Optional,Optional>> link:{java8-javadoc}/java/util/stream/Stream.html#reduce%2Djava.util.function.BinaryOperator%2D[reduce](<<painless-api-reference-BinaryOperator,BinaryOperator>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#reduce%2Djava.util.function.BinaryOperator%2D[java 9])
+* ++[[painless-api-reference-Stream-reduce-2]]def link:{java8-javadoc}/java/util/stream/Stream.html#reduce%2Djava.lang.Object%2Djava.util.function.BinaryOperator%2D[reduce](def, <<painless-api-reference-BinaryOperator,BinaryOperator>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#reduce%2Djava.lang.Object%2Djava.util.function.BinaryOperator%2D[java 9])
+* ++[[painless-api-reference-Stream-reduce-3]]def link:{java8-javadoc}/java/util/stream/Stream.html#reduce%2Djava.lang.Object%2Djava.util.function.BiFunction%2Djava.util.function.BinaryOperator%2D[reduce](def, <<painless-api-reference-BiFunction,BiFunction>>, <<painless-api-reference-BinaryOperator,BinaryOperator>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#reduce%2Djava.lang.Object%2Djava.util.function.BiFunction%2Djava.util.function.BinaryOperator%2D[java 9])
+* ++[[painless-api-reference-Stream-skip-1]]<<painless-api-reference-Stream,Stream>> link:{java8-javadoc}/java/util/stream/Stream.html#skip%2Dlong%2D[skip](long)++ (link:{java9-javadoc}/java/util/stream/Stream.html#skip%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Stream-sorted-0]]<<painless-api-reference-Stream,Stream>> link:{java8-javadoc}/java/util/stream/Stream.html#sorted%2D%2D[sorted]()++ (link:{java9-javadoc}/java/util/stream/Stream.html#sorted%2D%2D[java 9])
+* ++[[painless-api-reference-Stream-sorted-1]]<<painless-api-reference-Stream,Stream>> link:{java8-javadoc}/java/util/stream/Stream.html#sorted%2Djava.util.Comparator%2D[sorted](<<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#sorted%2Djava.util.Comparator%2D[java 9])
+* ++[[painless-api-reference-Stream-toArray-0]]def[] link:{java8-javadoc}/java/util/stream/Stream.html#toArray%2D%2D[toArray]()++ (link:{java9-javadoc}/java/util/stream/Stream.html#toArray%2D%2D[java 9])
+* ++[[painless-api-reference-Stream-toArray-1]]def[] link:{java8-javadoc}/java/util/stream/Stream.html#toArray%2Djava.util.function.IntFunction%2D[toArray](<<painless-api-reference-IntFunction,IntFunction>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#toArray%2Djava.util.function.IntFunction%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-BaseStream,BaseStream>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/StrictMath.asciidoc b/docs/painless/painless-api-reference/StrictMath.asciidoc
new file mode 100644
index 0000000000000..17346c33ba3f9
--- /dev/null
+++ b/docs/painless/painless-api-reference/StrictMath.asciidoc
@@ -0,0 +1,46 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-StrictMath]]++StrictMath++::
+** [[painless-api-reference-StrictMath-E]]static double link:{java8-javadoc}/java/lang/StrictMath.html#E[E] (link:{java9-javadoc}/java/lang/StrictMath.html#E[java 9])
+** [[painless-api-reference-StrictMath-PI]]static double link:{java8-javadoc}/java/lang/StrictMath.html#PI[PI] (link:{java9-javadoc}/java/lang/StrictMath.html#PI[java 9])
+* ++[[painless-api-reference-StrictMath-IEEEremainder-2]]static double link:{java8-javadoc}/java/lang/StrictMath.html#IEEEremainder%2Ddouble%2Ddouble%2D[IEEEremainder](double, double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#IEEEremainder%2Ddouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-abs-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#abs%2Ddouble%2D[abs](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#abs%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-acos-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#acos%2Ddouble%2D[acos](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#acos%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-asin-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#asin%2Ddouble%2D[asin](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#asin%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-atan-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#atan%2Ddouble%2D[atan](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#atan%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-atan2-2]]static double link:{java8-javadoc}/java/lang/StrictMath.html#atan2%2Ddouble%2Ddouble%2D[atan2](double, double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#atan2%2Ddouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-cbrt-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#cbrt%2Ddouble%2D[cbrt](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#cbrt%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-ceil-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#ceil%2Ddouble%2D[ceil](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#ceil%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-copySign-2]]static double link:{java8-javadoc}/java/lang/StrictMath.html#copySign%2Ddouble%2Ddouble%2D[copySign](double, double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#copySign%2Ddouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-cos-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#cos%2Ddouble%2D[cos](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#cos%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-cosh-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#cosh%2Ddouble%2D[cosh](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#cosh%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-exp-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#exp%2Ddouble%2D[exp](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#exp%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-expm1-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#expm1%2Ddouble%2D[expm1](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#expm1%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-floor-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#floor%2Ddouble%2D[floor](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#floor%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-hypot-2]]static double link:{java8-javadoc}/java/lang/StrictMath.html#hypot%2Ddouble%2Ddouble%2D[hypot](double, double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#hypot%2Ddouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-log-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#log%2Ddouble%2D[log](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#log%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-log10-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#log10%2Ddouble%2D[log10](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#log10%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-log1p-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#log1p%2Ddouble%2D[log1p](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#log1p%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-max-2]]static double link:{java8-javadoc}/java/lang/StrictMath.html#max%2Ddouble%2Ddouble%2D[max](double, double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#max%2Ddouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-min-2]]static double link:{java8-javadoc}/java/lang/StrictMath.html#min%2Ddouble%2Ddouble%2D[min](double, double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#min%2Ddouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-nextAfter-2]]static double link:{java8-javadoc}/java/lang/StrictMath.html#nextAfter%2Ddouble%2Ddouble%2D[nextAfter](double, double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#nextAfter%2Ddouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-nextDown-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#nextDown%2Ddouble%2D[nextDown](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#nextDown%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-nextUp-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#nextUp%2Ddouble%2D[nextUp](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#nextUp%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-pow-2]]static double link:{java8-javadoc}/java/lang/StrictMath.html#pow%2Ddouble%2Ddouble%2D[pow](double, double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#pow%2Ddouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-random-0]]static double link:{java8-javadoc}/java/lang/StrictMath.html#random%2D%2D[random]()++ (link:{java9-javadoc}/java/lang/StrictMath.html#random%2D%2D[java 9])
+* ++[[painless-api-reference-StrictMath-rint-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#rint%2Ddouble%2D[rint](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#rint%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-round-1]]static long link:{java8-javadoc}/java/lang/StrictMath.html#round%2Ddouble%2D[round](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#round%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-scalb-2]]static double link:{java8-javadoc}/java/lang/StrictMath.html#scalb%2Ddouble%2Dint%2D[scalb](double, int)++ (link:{java9-javadoc}/java/lang/StrictMath.html#scalb%2Ddouble%2Dint%2D[java 9])
+* ++[[painless-api-reference-StrictMath-signum-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#signum%2Ddouble%2D[signum](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#signum%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-sin-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#sin%2Ddouble%2D[sin](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#sin%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-sinh-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#sinh%2Ddouble%2D[sinh](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#sinh%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-sqrt-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#sqrt%2Ddouble%2D[sqrt](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#sqrt%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-tan-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#tan%2Ddouble%2D[tan](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#tan%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-tanh-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#tanh%2Ddouble%2D[tanh](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#tanh%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-toDegrees-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#toDegrees%2Ddouble%2D[toDegrees](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#toDegrees%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-toRadians-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#toRadians%2Ddouble%2D[toRadians](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#toRadians%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-ulp-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#ulp%2Ddouble%2D[ulp](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#ulp%2Ddouble%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/String.asciidoc b/docs/painless/painless-api-reference/String.asciidoc
new file mode 100644
index 0000000000000..a40e1811183d5
--- /dev/null
+++ b/docs/painless/painless-api-reference/String.asciidoc
@@ -0,0 +1,46 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-String]]++String++::
+* ++[[painless-api-reference-String-copyValueOf-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/String.html#copyValueOf%2Dchar:A%2D[copyValueOf](char[])++ (link:{java9-javadoc}/java/lang/String.html#copyValueOf%2Dchar:A%2D[java 9])
+* ++[[painless-api-reference-String-copyValueOf-3]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/String.html#copyValueOf%2Dchar:A%2Dint%2Dint%2D[copyValueOf](char[], int, int)++ (link:{java9-javadoc}/java/lang/String.html#copyValueOf%2Dchar:A%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-String-format-2]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/String.html#format%2Djava.lang.String%2Djava.lang.Object:A%2D[format](<<painless-api-reference-String,String>>, def[])++ (link:{java9-javadoc}/java/lang/String.html#format%2Djava.lang.String%2Djava.lang.Object:A%2D[java 9])
+* ++[[painless-api-reference-String-format-3]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/String.html#format%2Djava.util.Locale%2Djava.lang.String%2Djava.lang.Object:A%2D[format](<<painless-api-reference-Locale,Locale>>, <<painless-api-reference-String,String>>, def[])++ (link:{java9-javadoc}/java/lang/String.html#format%2Djava.util.Locale%2Djava.lang.String%2Djava.lang.Object:A%2D[java 9])
+* ++[[painless-api-reference-String-join-2]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/String.html#join%2Djava.lang.CharSequence%2Djava.lang.Iterable%2D[join](<<painless-api-reference-CharSequence,CharSequence>>, <<painless-api-reference-Iterable,Iterable>>)++ (link:{java9-javadoc}/java/lang/String.html#join%2Djava.lang.CharSequence%2Djava.lang.Iterable%2D[java 9])
+* ++[[painless-api-reference-String-valueOf-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/String.html#valueOf%2Djava.lang.Object%2D[valueOf](def)++ (link:{java9-javadoc}/java/lang/String.html#valueOf%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-String-String-0]]link:{java8-javadoc}/java/lang/String.html#String%2D%2D[String]()++ (link:{java9-javadoc}/java/lang/String.html#String%2D%2D[java 9])
+* ++[[painless-api-reference-String-codePointAt-1]]int link:{java8-javadoc}/java/lang/String.html#codePointAt%2Dint%2D[codePointAt](int)++ (link:{java9-javadoc}/java/lang/String.html#codePointAt%2Dint%2D[java 9])
+* ++[[painless-api-reference-String-codePointBefore-1]]int link:{java8-javadoc}/java/lang/String.html#codePointBefore%2Dint%2D[codePointBefore](int)++ (link:{java9-javadoc}/java/lang/String.html#codePointBefore%2Dint%2D[java 9])
+* ++[[painless-api-reference-String-codePointCount-2]]int link:{java8-javadoc}/java/lang/String.html#codePointCount%2Dint%2Dint%2D[codePointCount](int, int)++ (link:{java9-javadoc}/java/lang/String.html#codePointCount%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-String-compareTo-1]]int link:{java8-javadoc}/java/lang/String.html#compareTo%2Djava.lang.String%2D[compareTo](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/String.html#compareTo%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-String-compareToIgnoreCase-1]]int link:{java8-javadoc}/java/lang/String.html#compareToIgnoreCase%2Djava.lang.String%2D[compareToIgnoreCase](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/String.html#compareToIgnoreCase%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-String-concat-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/String.html#concat%2Djava.lang.String%2D[concat](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/String.html#concat%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-String-contains-1]]boolean link:{java8-javadoc}/java/lang/String.html#contains%2Djava.lang.CharSequence%2D[contains](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/lang/String.html#contains%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-String-contentEquals-1]]boolean link:{java8-javadoc}/java/lang/String.html#contentEquals%2Djava.lang.CharSequence%2D[contentEquals](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/lang/String.html#contentEquals%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-String-decodeBase64-0]]<<painless-api-reference-String,String>> link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#decodeBase64%2Djava.lang.String%2D[decodeBase64]()++
+* ++[[painless-api-reference-String-encodeBase64-0]]<<painless-api-reference-String,String>> link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#encodeBase64%2Djava.lang.String%2D[encodeBase64]()++
+* ++[[painless-api-reference-String-endsWith-1]]boolean link:{java8-javadoc}/java/lang/String.html#endsWith%2Djava.lang.String%2D[endsWith](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/String.html#endsWith%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-String-equalsIgnoreCase-1]]boolean link:{java8-javadoc}/java/lang/String.html#equalsIgnoreCase%2Djava.lang.String%2D[equalsIgnoreCase](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/String.html#equalsIgnoreCase%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-String-getChars-4]]void link:{java8-javadoc}/java/lang/String.html#getChars%2Dint%2Dint%2Dchar:A%2Dint%2D[getChars](int, int, char[], int)++ (link:{java9-javadoc}/java/lang/String.html#getChars%2Dint%2Dint%2Dchar:A%2Dint%2D[java 9])
+* ++[[painless-api-reference-String-indexOf-1]]int link:{java8-javadoc}/java/lang/String.html#indexOf%2Djava.lang.String%2D[indexOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/String.html#indexOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-String-indexOf-2]]int link:{java8-javadoc}/java/lang/String.html#indexOf%2Djava.lang.String%2Dint%2D[indexOf](<<painless-api-reference-String,String>>, int)++ (link:{java9-javadoc}/java/lang/String.html#indexOf%2Djava.lang.String%2Dint%2D[java 9])
+* ++[[painless-api-reference-String-isEmpty-0]]boolean link:{java8-javadoc}/java/lang/String.html#isEmpty%2D%2D[isEmpty]()++ (link:{java9-javadoc}/java/lang/String.html#isEmpty%2D%2D[java 9])
+* ++[[painless-api-reference-String-lastIndexOf-1]]int link:{java8-javadoc}/java/lang/String.html#lastIndexOf%2Djava.lang.String%2D[lastIndexOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/String.html#lastIndexOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-String-lastIndexOf-2]]int link:{java8-javadoc}/java/lang/String.html#lastIndexOf%2Djava.lang.String%2Dint%2D[lastIndexOf](<<painless-api-reference-String,String>>, int)++ (link:{java9-javadoc}/java/lang/String.html#lastIndexOf%2Djava.lang.String%2Dint%2D[java 9])
+* ++[[painless-api-reference-String-offsetByCodePoints-2]]int link:{java8-javadoc}/java/lang/String.html#offsetByCodePoints%2Dint%2Dint%2D[offsetByCodePoints](int, int)++ (link:{java9-javadoc}/java/lang/String.html#offsetByCodePoints%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-String-regionMatches-4]]boolean link:{java8-javadoc}/java/lang/String.html#regionMatches%2Dint%2Djava.lang.String%2Dint%2Dint%2D[regionMatches](int, <<painless-api-reference-String,String>>, int, int)++ (link:{java9-javadoc}/java/lang/String.html#regionMatches%2Dint%2Djava.lang.String%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-String-regionMatches-5]]boolean link:{java8-javadoc}/java/lang/String.html#regionMatches%2Dboolean%2Dint%2Djava.lang.String%2Dint%2Dint%2D[regionMatches](boolean, int, <<painless-api-reference-String,String>>, int, int)++ (link:{java9-javadoc}/java/lang/String.html#regionMatches%2Dboolean%2Dint%2Djava.lang.String%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-String-replace-2]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/String.html#replace%2Djava.lang.CharSequence%2Djava.lang.CharSequence%2D[replace](<<painless-api-reference-CharSequence,CharSequence>>, <<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/lang/String.html#replace%2Djava.lang.CharSequence%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-String-startsWith-1]]boolean link:{java8-javadoc}/java/lang/String.html#startsWith%2Djava.lang.String%2D[startsWith](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/String.html#startsWith%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-String-startsWith-2]]boolean link:{java8-javadoc}/java/lang/String.html#startsWith%2Djava.lang.String%2Dint%2D[startsWith](<<painless-api-reference-String,String>>, int)++ (link:{java9-javadoc}/java/lang/String.html#startsWith%2Djava.lang.String%2Dint%2D[java 9])
+* ++[[painless-api-reference-String-substring-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/String.html#substring%2Dint%2D[substring](int)++ (link:{java9-javadoc}/java/lang/String.html#substring%2Dint%2D[java 9])
+* ++[[painless-api-reference-String-substring-2]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/String.html#substring%2Dint%2Dint%2D[substring](int, int)++ (link:{java9-javadoc}/java/lang/String.html#substring%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-String-toCharArray-0]]char[] link:{java8-javadoc}/java/lang/String.html#toCharArray%2D%2D[toCharArray]()++ (link:{java9-javadoc}/java/lang/String.html#toCharArray%2D%2D[java 9])
+* ++[[painless-api-reference-String-toLowerCase-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/String.html#toLowerCase%2D%2D[toLowerCase]()++ (link:{java9-javadoc}/java/lang/String.html#toLowerCase%2D%2D[java 9])
+* ++[[painless-api-reference-String-toLowerCase-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/String.html#toLowerCase%2Djava.util.Locale%2D[toLowerCase](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/lang/String.html#toLowerCase%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-String-toUpperCase-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/String.html#toUpperCase%2D%2D[toUpperCase]()++ (link:{java9-javadoc}/java/lang/String.html#toUpperCase%2D%2D[java 9])
+* ++[[painless-api-reference-String-toUpperCase-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/String.html#toUpperCase%2Djava.util.Locale%2D[toUpperCase](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/lang/String.html#toUpperCase%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-String-trim-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/String.html#trim%2D%2D[trim]()++ (link:{java9-javadoc}/java/lang/String.html#trim%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-CharSequence,CharSequence>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/StringBuffer.asciidoc b/docs/painless/painless-api-reference/StringBuffer.asciidoc
new file mode 100644
index 0000000000000..5993832a983da
--- /dev/null
+++ b/docs/painless/painless-api-reference/StringBuffer.asciidoc
@@ -0,0 +1,31 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-StringBuffer]]++StringBuffer++::
+* ++[[painless-api-reference-StringBuffer-StringBuffer-0]]link:{java8-javadoc}/java/lang/StringBuffer.html#StringBuffer%2D%2D[StringBuffer]()++ (link:{java9-javadoc}/java/lang/StringBuffer.html#StringBuffer%2D%2D[java 9])
+* ++[[painless-api-reference-StringBuffer-StringBuffer-1]]link:{java8-javadoc}/java/lang/StringBuffer.html#StringBuffer%2Djava.lang.CharSequence%2D[StringBuffer](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/lang/StringBuffer.html#StringBuffer%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-StringBuffer-append-1]]<<painless-api-reference-StringBuffer,StringBuffer>> link:{java8-javadoc}/java/lang/StringBuffer.html#append%2Djava.lang.Object%2D[append](def)++ (link:{java9-javadoc}/java/lang/StringBuffer.html#append%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-StringBuffer-append-3]]<<painless-api-reference-StringBuffer,StringBuffer>> link:{java8-javadoc}/java/lang/StringBuffer.html#append%2Djava.lang.CharSequence%2Dint%2Dint%2D[append](<<painless-api-reference-CharSequence,CharSequence>>, int, int)++ (link:{java9-javadoc}/java/lang/StringBuffer.html#append%2Djava.lang.CharSequence%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuffer-appendCodePoint-1]]<<painless-api-reference-StringBuffer,StringBuffer>> link:{java8-javadoc}/java/lang/StringBuffer.html#appendCodePoint%2Dint%2D[appendCodePoint](int)++ (link:{java9-javadoc}/java/lang/StringBuffer.html#appendCodePoint%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuffer-capacity-0]]int link:{java8-javadoc}/java/lang/StringBuffer.html#capacity%2D%2D[capacity]()++ (link:{java9-javadoc}/java/lang/StringBuffer.html#capacity%2D%2D[java 9])
+* ++[[painless-api-reference-StringBuffer-codePointAt-1]]int link:{java8-javadoc}/java/lang/StringBuffer.html#codePointAt%2Dint%2D[codePointAt](int)++ (link:{java9-javadoc}/java/lang/StringBuffer.html#codePointAt%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuffer-codePointBefore-1]]int link:{java8-javadoc}/java/lang/StringBuffer.html#codePointBefore%2Dint%2D[codePointBefore](int)++ (link:{java9-javadoc}/java/lang/StringBuffer.html#codePointBefore%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuffer-codePointCount-2]]int link:{java8-javadoc}/java/lang/StringBuffer.html#codePointCount%2Dint%2Dint%2D[codePointCount](int, int)++ (link:{java9-javadoc}/java/lang/StringBuffer.html#codePointCount%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuffer-delete-2]]<<painless-api-reference-StringBuffer,StringBuffer>> link:{java8-javadoc}/java/lang/StringBuffer.html#delete%2Dint%2Dint%2D[delete](int, int)++ (link:{java9-javadoc}/java/lang/StringBuffer.html#delete%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuffer-deleteCharAt-1]]<<painless-api-reference-StringBuffer,StringBuffer>> link:{java8-javadoc}/java/lang/StringBuffer.html#deleteCharAt%2Dint%2D[deleteCharAt](int)++ (link:{java9-javadoc}/java/lang/StringBuffer.html#deleteCharAt%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuffer-getChars-4]]void link:{java8-javadoc}/java/lang/StringBuffer.html#getChars%2Dint%2Dint%2Dchar:A%2Dint%2D[getChars](int, int, char[], int)++ (link:{java9-javadoc}/java/lang/StringBuffer.html#getChars%2Dint%2Dint%2Dchar:A%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuffer-indexOf-1]]int link:{java8-javadoc}/java/lang/StringBuffer.html#indexOf%2Djava.lang.String%2D[indexOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/StringBuffer.html#indexOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-StringBuffer-indexOf-2]]int link:{java8-javadoc}/java/lang/StringBuffer.html#indexOf%2Djava.lang.String%2Dint%2D[indexOf](<<painless-api-reference-String,String>>, int)++ (link:{java9-javadoc}/java/lang/StringBuffer.html#indexOf%2Djava.lang.String%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuffer-insert-2]]<<painless-api-reference-StringBuffer,StringBuffer>> link:{java8-javadoc}/java/lang/StringBuffer.html#insert%2Dint%2Djava.lang.Object%2D[insert](int, def)++ (link:{java9-javadoc}/java/lang/StringBuffer.html#insert%2Dint%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-StringBuffer-lastIndexOf-1]]int link:{java8-javadoc}/java/lang/StringBuffer.html#lastIndexOf%2Djava.lang.String%2D[lastIndexOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/StringBuffer.html#lastIndexOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-StringBuffer-lastIndexOf-2]]int link:{java8-javadoc}/java/lang/StringBuffer.html#lastIndexOf%2Djava.lang.String%2Dint%2D[lastIndexOf](<<painless-api-reference-String,String>>, int)++ (link:{java9-javadoc}/java/lang/StringBuffer.html#lastIndexOf%2Djava.lang.String%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuffer-offsetByCodePoints-2]]int link:{java8-javadoc}/java/lang/StringBuffer.html#offsetByCodePoints%2Dint%2Dint%2D[offsetByCodePoints](int, int)++ (link:{java9-javadoc}/java/lang/StringBuffer.html#offsetByCodePoints%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuffer-replace-3]]<<painless-api-reference-StringBuffer,StringBuffer>> link:{java8-javadoc}/java/lang/StringBuffer.html#replace%2Dint%2Dint%2Djava.lang.String%2D[replace](int, int, <<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/StringBuffer.html#replace%2Dint%2Dint%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-StringBuffer-reverse-0]]<<painless-api-reference-StringBuffer,StringBuffer>> link:{java8-javadoc}/java/lang/StringBuffer.html#reverse%2D%2D[reverse]()++ (link:{java9-javadoc}/java/lang/StringBuffer.html#reverse%2D%2D[java 9])
+* ++[[painless-api-reference-StringBuffer-setCharAt-2]]void link:{java8-javadoc}/java/lang/StringBuffer.html#setCharAt%2Dint%2Dchar%2D[setCharAt](int, char)++ (link:{java9-javadoc}/java/lang/StringBuffer.html#setCharAt%2Dint%2Dchar%2D[java 9])
+* ++[[painless-api-reference-StringBuffer-setLength-1]]void link:{java8-javadoc}/java/lang/StringBuffer.html#setLength%2Dint%2D[setLength](int)++ (link:{java9-javadoc}/java/lang/StringBuffer.html#setLength%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuffer-substring-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/StringBuffer.html#substring%2Dint%2D[substring](int)++ (link:{java9-javadoc}/java/lang/StringBuffer.html#substring%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuffer-substring-2]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/StringBuffer.html#substring%2Dint%2Dint%2D[substring](int, int)++ (link:{java9-javadoc}/java/lang/StringBuffer.html#substring%2Dint%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-CharSequence,CharSequence>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/StringBuilder.asciidoc b/docs/painless/painless-api-reference/StringBuilder.asciidoc
new file mode 100644
index 0000000000000..fd9c0c097cc03
--- /dev/null
+++ b/docs/painless/painless-api-reference/StringBuilder.asciidoc
@@ -0,0 +1,31 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-StringBuilder]]++StringBuilder++::
+* ++[[painless-api-reference-StringBuilder-StringBuilder-0]]link:{java8-javadoc}/java/lang/StringBuilder.html#StringBuilder%2D%2D[StringBuilder]()++ (link:{java9-javadoc}/java/lang/StringBuilder.html#StringBuilder%2D%2D[java 9])
+* ++[[painless-api-reference-StringBuilder-StringBuilder-1]]link:{java8-javadoc}/java/lang/StringBuilder.html#StringBuilder%2Djava.lang.CharSequence%2D[StringBuilder](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/lang/StringBuilder.html#StringBuilder%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-StringBuilder-append-1]]<<painless-api-reference-StringBuilder,StringBuilder>> link:{java8-javadoc}/java/lang/StringBuilder.html#append%2Djava.lang.Object%2D[append](def)++ (link:{java9-javadoc}/java/lang/StringBuilder.html#append%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-StringBuilder-append-3]]<<painless-api-reference-StringBuilder,StringBuilder>> link:{java8-javadoc}/java/lang/StringBuilder.html#append%2Djava.lang.CharSequence%2Dint%2Dint%2D[append](<<painless-api-reference-CharSequence,CharSequence>>, int, int)++ (link:{java9-javadoc}/java/lang/StringBuilder.html#append%2Djava.lang.CharSequence%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuilder-appendCodePoint-1]]<<painless-api-reference-StringBuilder,StringBuilder>> link:{java8-javadoc}/java/lang/StringBuilder.html#appendCodePoint%2Dint%2D[appendCodePoint](int)++ (link:{java9-javadoc}/java/lang/StringBuilder.html#appendCodePoint%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuilder-capacity-0]]int link:{java8-javadoc}/java/lang/StringBuilder.html#capacity%2D%2D[capacity]()++ (link:{java9-javadoc}/java/lang/StringBuilder.html#capacity%2D%2D[java 9])
+* ++[[painless-api-reference-StringBuilder-codePointAt-1]]int link:{java8-javadoc}/java/lang/StringBuilder.html#codePointAt%2Dint%2D[codePointAt](int)++ (link:{java9-javadoc}/java/lang/StringBuilder.html#codePointAt%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuilder-codePointBefore-1]]int link:{java8-javadoc}/java/lang/StringBuilder.html#codePointBefore%2Dint%2D[codePointBefore](int)++ (link:{java9-javadoc}/java/lang/StringBuilder.html#codePointBefore%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuilder-codePointCount-2]]int link:{java8-javadoc}/java/lang/StringBuilder.html#codePointCount%2Dint%2Dint%2D[codePointCount](int, int)++ (link:{java9-javadoc}/java/lang/StringBuilder.html#codePointCount%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuilder-delete-2]]<<painless-api-reference-StringBuilder,StringBuilder>> link:{java8-javadoc}/java/lang/StringBuilder.html#delete%2Dint%2Dint%2D[delete](int, int)++ (link:{java9-javadoc}/java/lang/StringBuilder.html#delete%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuilder-deleteCharAt-1]]<<painless-api-reference-StringBuilder,StringBuilder>> link:{java8-javadoc}/java/lang/StringBuilder.html#deleteCharAt%2Dint%2D[deleteCharAt](int)++ (link:{java9-javadoc}/java/lang/StringBuilder.html#deleteCharAt%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuilder-getChars-4]]void link:{java8-javadoc}/java/lang/StringBuilder.html#getChars%2Dint%2Dint%2Dchar:A%2Dint%2D[getChars](int, int, char[], int)++ (link:{java9-javadoc}/java/lang/StringBuilder.html#getChars%2Dint%2Dint%2Dchar:A%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuilder-indexOf-1]]int link:{java8-javadoc}/java/lang/StringBuilder.html#indexOf%2Djava.lang.String%2D[indexOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/StringBuilder.html#indexOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-StringBuilder-indexOf-2]]int link:{java8-javadoc}/java/lang/StringBuilder.html#indexOf%2Djava.lang.String%2Dint%2D[indexOf](<<painless-api-reference-String,String>>, int)++ (link:{java9-javadoc}/java/lang/StringBuilder.html#indexOf%2Djava.lang.String%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuilder-insert-2]]<<painless-api-reference-StringBuilder,StringBuilder>> link:{java8-javadoc}/java/lang/StringBuilder.html#insert%2Dint%2Djava.lang.Object%2D[insert](int, def)++ (link:{java9-javadoc}/java/lang/StringBuilder.html#insert%2Dint%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-StringBuilder-lastIndexOf-1]]int link:{java8-javadoc}/java/lang/StringBuilder.html#lastIndexOf%2Djava.lang.String%2D[lastIndexOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/StringBuilder.html#lastIndexOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-StringBuilder-lastIndexOf-2]]int link:{java8-javadoc}/java/lang/StringBuilder.html#lastIndexOf%2Djava.lang.String%2Dint%2D[lastIndexOf](<<painless-api-reference-String,String>>, int)++ (link:{java9-javadoc}/java/lang/StringBuilder.html#lastIndexOf%2Djava.lang.String%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuilder-offsetByCodePoints-2]]int link:{java8-javadoc}/java/lang/StringBuilder.html#offsetByCodePoints%2Dint%2Dint%2D[offsetByCodePoints](int, int)++ (link:{java9-javadoc}/java/lang/StringBuilder.html#offsetByCodePoints%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuilder-replace-3]]<<painless-api-reference-StringBuilder,StringBuilder>> link:{java8-javadoc}/java/lang/StringBuilder.html#replace%2Dint%2Dint%2Djava.lang.String%2D[replace](int, int, <<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/StringBuilder.html#replace%2Dint%2Dint%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-StringBuilder-reverse-0]]<<painless-api-reference-StringBuilder,StringBuilder>> link:{java8-javadoc}/java/lang/StringBuilder.html#reverse%2D%2D[reverse]()++ (link:{java9-javadoc}/java/lang/StringBuilder.html#reverse%2D%2D[java 9])
+* ++[[painless-api-reference-StringBuilder-setCharAt-2]]void link:{java8-javadoc}/java/lang/StringBuilder.html#setCharAt%2Dint%2Dchar%2D[setCharAt](int, char)++ (link:{java9-javadoc}/java/lang/StringBuilder.html#setCharAt%2Dint%2Dchar%2D[java 9])
+* ++[[painless-api-reference-StringBuilder-setLength-1]]void link:{java8-javadoc}/java/lang/StringBuilder.html#setLength%2Dint%2D[setLength](int)++ (link:{java9-javadoc}/java/lang/StringBuilder.html#setLength%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuilder-substring-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/StringBuilder.html#substring%2Dint%2D[substring](int)++ (link:{java9-javadoc}/java/lang/StringBuilder.html#substring%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuilder-substring-2]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/StringBuilder.html#substring%2Dint%2Dint%2D[substring](int, int)++ (link:{java9-javadoc}/java/lang/StringBuilder.html#substring%2Dint%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-CharSequence,CharSequence>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/StringCharacterIterator.asciidoc b/docs/painless/painless-api-reference/StringCharacterIterator.asciidoc
new file mode 100644
index 0000000000000..8f65f2b6c5e53
--- /dev/null
+++ b/docs/painless/painless-api-reference/StringCharacterIterator.asciidoc
@@ -0,0 +1,11 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-StringCharacterIterator]]++StringCharacterIterator++::
+* ++[[painless-api-reference-StringCharacterIterator-StringCharacterIterator-1]]link:{java8-javadoc}/java/text/StringCharacterIterator.html#StringCharacterIterator%2Djava.lang.String%2D[StringCharacterIterator](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/StringCharacterIterator.html#StringCharacterIterator%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-StringCharacterIterator-StringCharacterIterator-2]]link:{java8-javadoc}/java/text/StringCharacterIterator.html#StringCharacterIterator%2Djava.lang.String%2Dint%2D[StringCharacterIterator](<<painless-api-reference-String,String>>, int)++ (link:{java9-javadoc}/java/text/StringCharacterIterator.html#StringCharacterIterator%2Djava.lang.String%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringCharacterIterator-StringCharacterIterator-4]]link:{java8-javadoc}/java/text/StringCharacterIterator.html#StringCharacterIterator%2Djava.lang.String%2Dint%2Dint%2Dint%2D[StringCharacterIterator](<<painless-api-reference-String,String>>, int, int, int)++ (link:{java9-javadoc}/java/text/StringCharacterIterator.html#StringCharacterIterator%2Djava.lang.String%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringCharacterIterator-setText-1]]void link:{java8-javadoc}/java/text/StringCharacterIterator.html#setText%2Djava.lang.String%2D[setText](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/StringCharacterIterator.html#setText%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-CharacterIterator,CharacterIterator>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/StringIndexOutOfBoundsException.asciidoc b/docs/painless/painless-api-reference/StringIndexOutOfBoundsException.asciidoc
new file mode 100644
index 0000000000000..a29cae3f857f0
--- /dev/null
+++ b/docs/painless/painless-api-reference/StringIndexOutOfBoundsException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-StringIndexOutOfBoundsException]]++StringIndexOutOfBoundsException++::
+* ++[[painless-api-reference-StringIndexOutOfBoundsException-StringIndexOutOfBoundsException-0]]link:{java8-javadoc}/java/lang/StringIndexOutOfBoundsException.html#StringIndexOutOfBoundsException%2D%2D[StringIndexOutOfBoundsException]()++ (link:{java9-javadoc}/java/lang/StringIndexOutOfBoundsException.html#StringIndexOutOfBoundsException%2D%2D[java 9])
+* ++[[painless-api-reference-StringIndexOutOfBoundsException-StringIndexOutOfBoundsException-1]]link:{java8-javadoc}/java/lang/StringIndexOutOfBoundsException.html#StringIndexOutOfBoundsException%2Djava.lang.String%2D[StringIndexOutOfBoundsException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/StringIndexOutOfBoundsException.html#StringIndexOutOfBoundsException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/StringJoiner.asciidoc b/docs/painless/painless-api-reference/StringJoiner.asciidoc
new file mode 100644
index 0000000000000..7bd9baa4460ac
--- /dev/null
+++ b/docs/painless/painless-api-reference/StringJoiner.asciidoc
@@ -0,0 +1,13 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-StringJoiner]]++StringJoiner++::
+* ++[[painless-api-reference-StringJoiner-StringJoiner-1]]link:{java8-javadoc}/java/util/StringJoiner.html#StringJoiner%2Djava.lang.CharSequence%2D[StringJoiner](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/util/StringJoiner.html#StringJoiner%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-StringJoiner-StringJoiner-3]]link:{java8-javadoc}/java/util/StringJoiner.html#StringJoiner%2Djava.lang.CharSequence%2Djava.lang.CharSequence%2Djava.lang.CharSequence%2D[StringJoiner](<<painless-api-reference-CharSequence,CharSequence>>, <<painless-api-reference-CharSequence,CharSequence>>, <<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/util/StringJoiner.html#StringJoiner%2Djava.lang.CharSequence%2Djava.lang.CharSequence%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-StringJoiner-add-1]]<<painless-api-reference-StringJoiner,StringJoiner>> link:{java8-javadoc}/java/util/StringJoiner.html#add%2Djava.lang.CharSequence%2D[add](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/util/StringJoiner.html#add%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-StringJoiner-length-0]]int link:{java8-javadoc}/java/util/StringJoiner.html#length%2D%2D[length]()++ (link:{java9-javadoc}/java/util/StringJoiner.html#length%2D%2D[java 9])
+* ++[[painless-api-reference-StringJoiner-merge-1]]<<painless-api-reference-StringJoiner,StringJoiner>> link:{java8-javadoc}/java/util/StringJoiner.html#merge%2Djava.util.StringJoiner%2D[merge](<<painless-api-reference-StringJoiner,StringJoiner>>)++ (link:{java9-javadoc}/java/util/StringJoiner.html#merge%2Djava.util.StringJoiner%2D[java 9])
+* ++[[painless-api-reference-StringJoiner-setEmptyValue-1]]<<painless-api-reference-StringJoiner,StringJoiner>> link:{java8-javadoc}/java/util/StringJoiner.html#setEmptyValue%2Djava.lang.CharSequence%2D[setEmptyValue](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/util/StringJoiner.html#setEmptyValue%2Djava.lang.CharSequence%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/StringTokenizer.asciidoc b/docs/painless/painless-api-reference/StringTokenizer.asciidoc
new file mode 100644
index 0000000000000..9462cecc33b4e
--- /dev/null
+++ b/docs/painless/painless-api-reference/StringTokenizer.asciidoc
@@ -0,0 +1,14 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-StringTokenizer]]++StringTokenizer++::
+* ++[[painless-api-reference-StringTokenizer-StringTokenizer-1]]link:{java8-javadoc}/java/util/StringTokenizer.html#StringTokenizer%2Djava.lang.String%2D[StringTokenizer](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/StringTokenizer.html#StringTokenizer%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-StringTokenizer-StringTokenizer-2]]link:{java8-javadoc}/java/util/StringTokenizer.html#StringTokenizer%2Djava.lang.String%2Djava.lang.String%2D[StringTokenizer](<<painless-api-reference-String,String>>, <<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/StringTokenizer.html#StringTokenizer%2Djava.lang.String%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-StringTokenizer-StringTokenizer-3]]link:{java8-javadoc}/java/util/StringTokenizer.html#StringTokenizer%2Djava.lang.String%2Djava.lang.String%2Dboolean%2D[StringTokenizer](<<painless-api-reference-String,String>>, <<painless-api-reference-String,String>>, boolean)++ (link:{java9-javadoc}/java/util/StringTokenizer.html#StringTokenizer%2Djava.lang.String%2Djava.lang.String%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-StringTokenizer-countTokens-0]]int link:{java8-javadoc}/java/util/StringTokenizer.html#countTokens%2D%2D[countTokens]()++ (link:{java9-javadoc}/java/util/StringTokenizer.html#countTokens%2D%2D[java 9])
+* ++[[painless-api-reference-StringTokenizer-hasMoreTokens-0]]boolean link:{java8-javadoc}/java/util/StringTokenizer.html#hasMoreTokens%2D%2D[hasMoreTokens]()++ (link:{java9-javadoc}/java/util/StringTokenizer.html#hasMoreTokens%2D%2D[java 9])
+* ++[[painless-api-reference-StringTokenizer-nextToken-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/StringTokenizer.html#nextToken%2D%2D[nextToken]()++ (link:{java9-javadoc}/java/util/StringTokenizer.html#nextToken%2D%2D[java 9])
+* ++[[painless-api-reference-StringTokenizer-nextToken-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/StringTokenizer.html#nextToken%2Djava.lang.String%2D[nextToken](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/StringTokenizer.html#nextToken%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Enumeration,Enumeration>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Supplier.asciidoc b/docs/painless/painless-api-reference/Supplier.asciidoc
new file mode 100644
index 0000000000000..ba3cd9da951ec
--- /dev/null
+++ b/docs/painless/painless-api-reference/Supplier.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Supplier]]++Supplier++::
+* ++[[painless-api-reference-Supplier-get-0]]def link:{java8-javadoc}/java/util/function/Supplier.html#get%2D%2D[get]()++ (link:{java9-javadoc}/java/util/function/Supplier.html#get%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/System.asciidoc b/docs/painless/painless-api-reference/System.asciidoc
new file mode 100644
index 0000000000000..a791d5dc15966
--- /dev/null
+++ b/docs/painless/painless-api-reference/System.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-System]]++System++::
+* ++[[painless-api-reference-System-arraycopy-5]]static void link:{java8-javadoc}/java/lang/System.html#arraycopy%2Djava.lang.Object%2Dint%2Djava.lang.Object%2Dint%2Dint%2D[arraycopy](<<painless-api-reference-Object,Object>>, int, <<painless-api-reference-Object,Object>>, int, int)++ (link:{java9-javadoc}/java/lang/System.html#arraycopy%2Djava.lang.Object%2Dint%2Djava.lang.Object%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-System-currentTimeMillis-0]]static long link:{java8-javadoc}/java/lang/System.html#currentTimeMillis%2D%2D[currentTimeMillis]()++ (link:{java9-javadoc}/java/lang/System.html#currentTimeMillis%2D%2D[java 9])
+* ++[[painless-api-reference-System-nanoTime-0]]static long link:{java8-javadoc}/java/lang/System.html#nanoTime%2D%2D[nanoTime]()++ (link:{java9-javadoc}/java/lang/System.html#nanoTime%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Temporal.asciidoc b/docs/painless/painless-api-reference/Temporal.asciidoc
new file mode 100644
index 0000000000000..ce693cfde51fa
--- /dev/null
+++ b/docs/painless/painless-api-reference/Temporal.asciidoc
@@ -0,0 +1,14 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Temporal]]++Temporal++::
+* ++[[painless-api-reference-Temporal-minus-1]]<<painless-api-reference-Temporal,Temporal>> link:{java8-javadoc}/java/time/temporal/Temporal.html#minus%2Djava.time.temporal.TemporalAmount%2D[minus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/temporal/Temporal.html#minus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-Temporal-minus-2]]<<painless-api-reference-Temporal,Temporal>> link:{java8-javadoc}/java/time/temporal/Temporal.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[minus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/temporal/Temporal.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-Temporal-plus-1]]<<painless-api-reference-Temporal,Temporal>> link:{java8-javadoc}/java/time/temporal/Temporal.html#plus%2Djava.time.temporal.TemporalAmount%2D[plus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/temporal/Temporal.html#plus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-Temporal-plus-2]]<<painless-api-reference-Temporal,Temporal>> link:{java8-javadoc}/java/time/temporal/Temporal.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[plus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/temporal/Temporal.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-Temporal-until-2]]long link:{java8-javadoc}/java/time/temporal/Temporal.html#until%2Djava.time.temporal.Temporal%2Djava.time.temporal.TemporalUnit%2D[until](<<painless-api-reference-Temporal,Temporal>>, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/temporal/Temporal.html#until%2Djava.time.temporal.Temporal%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-Temporal-with-1]]<<painless-api-reference-Temporal,Temporal>> link:{java8-javadoc}/java/time/temporal/Temporal.html#with%2Djava.time.temporal.TemporalAdjuster%2D[with](<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>)++ (link:{java9-javadoc}/java/time/temporal/Temporal.html#with%2Djava.time.temporal.TemporalAdjuster%2D[java 9])
+* ++[[painless-api-reference-Temporal-with-2]]<<painless-api-reference-Temporal,Temporal>> link:{java8-javadoc}/java/time/temporal/Temporal.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[with](<<painless-api-reference-TemporalField,TemporalField>>, long)++ (link:{java9-javadoc}/java/time/temporal/Temporal.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++
diff --git a/docs/painless/painless-api-reference/TemporalAccessor.asciidoc b/docs/painless/painless-api-reference/TemporalAccessor.asciidoc
new file mode 100644
index 0000000000000..7583b2180d940
--- /dev/null
+++ b/docs/painless/painless-api-reference/TemporalAccessor.asciidoc
@@ -0,0 +1,12 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-TemporalAccessor]]++TemporalAccessor++::
+* ++[[painless-api-reference-TemporalAccessor-get-1]]int link:{java8-javadoc}/java/time/temporal/TemporalAccessor.html#get%2Djava.time.temporal.TemporalField%2D[get](<<painless-api-reference-TemporalField,TemporalField>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalAccessor.html#get%2Djava.time.temporal.TemporalField%2D[java 9])
+* ++[[painless-api-reference-TemporalAccessor-getLong-1]]long link:{java8-javadoc}/java/time/temporal/TemporalAccessor.html#getLong%2Djava.time.temporal.TemporalField%2D[getLong](<<painless-api-reference-TemporalField,TemporalField>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalAccessor.html#getLong%2Djava.time.temporal.TemporalField%2D[java 9])
+* ++[[painless-api-reference-TemporalAccessor-isSupported-1]]boolean link:{java8-javadoc}/java/time/temporal/TemporalAccessor.html#isSupported%2Djava.time.temporal.TemporalField%2D[isSupported](<<painless-api-reference-TemporalField,TemporalField>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalAccessor.html#isSupported%2Djava.time.temporal.TemporalField%2D[java 9])
+* ++[[painless-api-reference-TemporalAccessor-query-1]]def link:{java8-javadoc}/java/time/temporal/TemporalAccessor.html#query%2Djava.time.temporal.TemporalQuery%2D[query](<<painless-api-reference-TemporalQuery,TemporalQuery>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalAccessor.html#query%2Djava.time.temporal.TemporalQuery%2D[java 9])
+* ++[[painless-api-reference-TemporalAccessor-range-1]]<<painless-api-reference-ValueRange,ValueRange>> link:{java8-javadoc}/java/time/temporal/TemporalAccessor.html#range%2Djava.time.temporal.TemporalField%2D[range](<<painless-api-reference-TemporalField,TemporalField>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalAccessor.html#range%2Djava.time.temporal.TemporalField%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/TemporalAdjuster.asciidoc b/docs/painless/painless-api-reference/TemporalAdjuster.asciidoc
new file mode 100644
index 0000000000000..b80f0d7afbfae
--- /dev/null
+++ b/docs/painless/painless-api-reference/TemporalAdjuster.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-TemporalAdjuster]]++TemporalAdjuster++::
+* ++[[painless-api-reference-TemporalAdjuster-adjustInto-1]]<<painless-api-reference-Temporal,Temporal>> link:{java8-javadoc}/java/time/temporal/TemporalAdjuster.html#adjustInto%2Djava.time.temporal.Temporal%2D[adjustInto](<<painless-api-reference-Temporal,Temporal>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalAdjuster.html#adjustInto%2Djava.time.temporal.Temporal%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/TemporalAdjusters.asciidoc b/docs/painless/painless-api-reference/TemporalAdjusters.asciidoc
new file mode 100644
index 0000000000000..7299ae9256abc
--- /dev/null
+++ b/docs/painless/painless-api-reference/TemporalAdjusters.asciidoc
@@ -0,0 +1,21 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-TemporalAdjusters]]++TemporalAdjusters++::
+* ++[[painless-api-reference-TemporalAdjusters-dayOfWeekInMonth-2]]static <<painless-api-reference-TemporalAdjuster,TemporalAdjuster>> link:{java8-javadoc}/java/time/temporal/TemporalAdjusters.html#dayOfWeekInMonth%2Dint%2Djava.time.DayOfWeek%2D[dayOfWeekInMonth](int, <<painless-api-reference-DayOfWeek,DayOfWeek>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalAdjusters.html#dayOfWeekInMonth%2Dint%2Djava.time.DayOfWeek%2D[java 9])
+* ++[[painless-api-reference-TemporalAdjusters-firstDayOfMonth-0]]static <<painless-api-reference-TemporalAdjuster,TemporalAdjuster>> link:{java8-javadoc}/java/time/temporal/TemporalAdjusters.html#firstDayOfMonth%2D%2D[firstDayOfMonth]()++ (link:{java9-javadoc}/java/time/temporal/TemporalAdjusters.html#firstDayOfMonth%2D%2D[java 9])
+* ++[[painless-api-reference-TemporalAdjusters-firstDayOfNextMonth-0]]static <<painless-api-reference-TemporalAdjuster,TemporalAdjuster>> link:{java8-javadoc}/java/time/temporal/TemporalAdjusters.html#firstDayOfNextMonth%2D%2D[firstDayOfNextMonth]()++ (link:{java9-javadoc}/java/time/temporal/TemporalAdjusters.html#firstDayOfNextMonth%2D%2D[java 9])
+* ++[[painless-api-reference-TemporalAdjusters-firstDayOfNextYear-0]]static <<painless-api-reference-TemporalAdjuster,TemporalAdjuster>> link:{java8-javadoc}/java/time/temporal/TemporalAdjusters.html#firstDayOfNextYear%2D%2D[firstDayOfNextYear]()++ (link:{java9-javadoc}/java/time/temporal/TemporalAdjusters.html#firstDayOfNextYear%2D%2D[java 9])
+* ++[[painless-api-reference-TemporalAdjusters-firstDayOfYear-0]]static <<painless-api-reference-TemporalAdjuster,TemporalAdjuster>> link:{java8-javadoc}/java/time/temporal/TemporalAdjusters.html#firstDayOfYear%2D%2D[firstDayOfYear]()++ (link:{java9-javadoc}/java/time/temporal/TemporalAdjusters.html#firstDayOfYear%2D%2D[java 9])
+* ++[[painless-api-reference-TemporalAdjusters-firstInMonth-1]]static <<painless-api-reference-TemporalAdjuster,TemporalAdjuster>> link:{java8-javadoc}/java/time/temporal/TemporalAdjusters.html#firstInMonth%2Djava.time.DayOfWeek%2D[firstInMonth](<<painless-api-reference-DayOfWeek,DayOfWeek>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalAdjusters.html#firstInMonth%2Djava.time.DayOfWeek%2D[java 9])
+* ++[[painless-api-reference-TemporalAdjusters-lastDayOfMonth-0]]static <<painless-api-reference-TemporalAdjuster,TemporalAdjuster>> link:{java8-javadoc}/java/time/temporal/TemporalAdjusters.html#lastDayOfMonth%2D%2D[lastDayOfMonth]()++ (link:{java9-javadoc}/java/time/temporal/TemporalAdjusters.html#lastDayOfMonth%2D%2D[java 9])
+* ++[[painless-api-reference-TemporalAdjusters-lastDayOfYear-0]]static <<painless-api-reference-TemporalAdjuster,TemporalAdjuster>> link:{java8-javadoc}/java/time/temporal/TemporalAdjusters.html#lastDayOfYear%2D%2D[lastDayOfYear]()++ (link:{java9-javadoc}/java/time/temporal/TemporalAdjusters.html#lastDayOfYear%2D%2D[java 9])
+* ++[[painless-api-reference-TemporalAdjusters-lastInMonth-1]]static <<painless-api-reference-TemporalAdjuster,TemporalAdjuster>> link:{java8-javadoc}/java/time/temporal/TemporalAdjusters.html#lastInMonth%2Djava.time.DayOfWeek%2D[lastInMonth](<<painless-api-reference-DayOfWeek,DayOfWeek>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalAdjusters.html#lastInMonth%2Djava.time.DayOfWeek%2D[java 9])
+* ++[[painless-api-reference-TemporalAdjusters-next-1]]static <<painless-api-reference-TemporalAdjuster,TemporalAdjuster>> link:{java8-javadoc}/java/time/temporal/TemporalAdjusters.html#next%2Djava.time.DayOfWeek%2D[next](<<painless-api-reference-DayOfWeek,DayOfWeek>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalAdjusters.html#next%2Djava.time.DayOfWeek%2D[java 9])
+* ++[[painless-api-reference-TemporalAdjusters-nextOrSame-1]]static <<painless-api-reference-TemporalAdjuster,TemporalAdjuster>> link:{java8-javadoc}/java/time/temporal/TemporalAdjusters.html#nextOrSame%2Djava.time.DayOfWeek%2D[nextOrSame](<<painless-api-reference-DayOfWeek,DayOfWeek>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalAdjusters.html#nextOrSame%2Djava.time.DayOfWeek%2D[java 9])
+* ++[[painless-api-reference-TemporalAdjusters-ofDateAdjuster-1]]static <<painless-api-reference-TemporalAdjuster,TemporalAdjuster>> link:{java8-javadoc}/java/time/temporal/TemporalAdjusters.html#ofDateAdjuster%2Djava.util.function.UnaryOperator%2D[ofDateAdjuster](<<painless-api-reference-UnaryOperator,UnaryOperator>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalAdjusters.html#ofDateAdjuster%2Djava.util.function.UnaryOperator%2D[java 9])
+* ++[[painless-api-reference-TemporalAdjusters-previous-1]]static <<painless-api-reference-TemporalAdjuster,TemporalAdjuster>> link:{java8-javadoc}/java/time/temporal/TemporalAdjusters.html#previous%2Djava.time.DayOfWeek%2D[previous](<<painless-api-reference-DayOfWeek,DayOfWeek>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalAdjusters.html#previous%2Djava.time.DayOfWeek%2D[java 9])
+* ++[[painless-api-reference-TemporalAdjusters-previousOrSame-1]]static <<painless-api-reference-TemporalAdjuster,TemporalAdjuster>> link:{java8-javadoc}/java/time/temporal/TemporalAdjusters.html#previousOrSame%2Djava.time.DayOfWeek%2D[previousOrSame](<<painless-api-reference-DayOfWeek,DayOfWeek>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalAdjusters.html#previousOrSame%2Djava.time.DayOfWeek%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/TemporalAmount.asciidoc b/docs/painless/painless-api-reference/TemporalAmount.asciidoc
new file mode 100644
index 0000000000000..e27954b4db3e4
--- /dev/null
+++ b/docs/painless/painless-api-reference/TemporalAmount.asciidoc
@@ -0,0 +1,11 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-TemporalAmount]]++TemporalAmount++::
+* ++[[painless-api-reference-TemporalAmount-addTo-1]]<<painless-api-reference-Temporal,Temporal>> link:{java8-javadoc}/java/time/temporal/TemporalAmount.html#addTo%2Djava.time.temporal.Temporal%2D[addTo](<<painless-api-reference-Temporal,Temporal>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalAmount.html#addTo%2Djava.time.temporal.Temporal%2D[java 9])
+* ++[[painless-api-reference-TemporalAmount-get-1]]long link:{java8-javadoc}/java/time/temporal/TemporalAmount.html#get%2Djava.time.temporal.TemporalUnit%2D[get](<<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalAmount.html#get%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-TemporalAmount-getUnits-0]]<<painless-api-reference-List,List>> link:{java8-javadoc}/java/time/temporal/TemporalAmount.html#getUnits%2D%2D[getUnits]()++ (link:{java9-javadoc}/java/time/temporal/TemporalAmount.html#getUnits%2D%2D[java 9])
+* ++[[painless-api-reference-TemporalAmount-subtractFrom-1]]<<painless-api-reference-Temporal,Temporal>> link:{java8-javadoc}/java/time/temporal/TemporalAmount.html#subtractFrom%2Djava.time.temporal.Temporal%2D[subtractFrom](<<painless-api-reference-Temporal,Temporal>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalAmount.html#subtractFrom%2Djava.time.temporal.Temporal%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/TemporalField.asciidoc b/docs/painless/painless-api-reference/TemporalField.asciidoc
new file mode 100644
index 0000000000000..dd38544d46e4c
--- /dev/null
+++ b/docs/painless/painless-api-reference/TemporalField.asciidoc
@@ -0,0 +1,19 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-TemporalField]]++TemporalField++::
+* ++[[painless-api-reference-TemporalField-adjustInto-2]]<<painless-api-reference-Temporal,Temporal>> link:{java8-javadoc}/java/time/temporal/TemporalField.html#adjustInto%2Djava.time.temporal.Temporal%2Dlong%2D[adjustInto](<<painless-api-reference-Temporal,Temporal>>, long)++ (link:{java9-javadoc}/java/time/temporal/TemporalField.html#adjustInto%2Djava.time.temporal.Temporal%2Dlong%2D[java 9])
+* ++[[painless-api-reference-TemporalField-getBaseUnit-0]]<<painless-api-reference-TemporalUnit,TemporalUnit>> link:{java8-javadoc}/java/time/temporal/TemporalField.html#getBaseUnit%2D%2D[getBaseUnit]()++ (link:{java9-javadoc}/java/time/temporal/TemporalField.html#getBaseUnit%2D%2D[java 9])
+* ++[[painless-api-reference-TemporalField-getDisplayName-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/temporal/TemporalField.html#getDisplayName%2Djava.util.Locale%2D[getDisplayName](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalField.html#getDisplayName%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-TemporalField-getFrom-1]]long link:{java8-javadoc}/java/time/temporal/TemporalField.html#getFrom%2Djava.time.temporal.TemporalAccessor%2D[getFrom](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalField.html#getFrom%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-TemporalField-getRangeUnit-0]]<<painless-api-reference-TemporalUnit,TemporalUnit>> link:{java8-javadoc}/java/time/temporal/TemporalField.html#getRangeUnit%2D%2D[getRangeUnit]()++ (link:{java9-javadoc}/java/time/temporal/TemporalField.html#getRangeUnit%2D%2D[java 9])
+* ++[[painless-api-reference-TemporalField-isDateBased-0]]boolean link:{java8-javadoc}/java/time/temporal/TemporalField.html#isDateBased%2D%2D[isDateBased]()++ (link:{java9-javadoc}/java/time/temporal/TemporalField.html#isDateBased%2D%2D[java 9])
+* ++[[painless-api-reference-TemporalField-isSupportedBy-1]]boolean link:{java8-javadoc}/java/time/temporal/TemporalField.html#isSupportedBy%2Djava.time.temporal.TemporalAccessor%2D[isSupportedBy](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalField.html#isSupportedBy%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-TemporalField-isTimeBased-0]]boolean link:{java8-javadoc}/java/time/temporal/TemporalField.html#isTimeBased%2D%2D[isTimeBased]()++ (link:{java9-javadoc}/java/time/temporal/TemporalField.html#isTimeBased%2D%2D[java 9])
+* ++[[painless-api-reference-TemporalField-range-0]]<<painless-api-reference-ValueRange,ValueRange>> link:{java8-javadoc}/java/time/temporal/TemporalField.html#range%2D%2D[range]()++ (link:{java9-javadoc}/java/time/temporal/TemporalField.html#range%2D%2D[java 9])
+* ++[[painless-api-reference-TemporalField-rangeRefinedBy-1]]<<painless-api-reference-ValueRange,ValueRange>> link:{java8-javadoc}/java/time/temporal/TemporalField.html#rangeRefinedBy%2Djava.time.temporal.TemporalAccessor%2D[rangeRefinedBy](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalField.html#rangeRefinedBy%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-TemporalField-resolve-3]]<<painless-api-reference-TemporalAccessor,TemporalAccessor>> link:{java8-javadoc}/java/time/temporal/TemporalField.html#resolve%2Djava.util.Map%2Djava.time.temporal.TemporalAccessor%2Djava.time.format.ResolverStyle%2D[resolve](<<painless-api-reference-Map,Map>>, <<painless-api-reference-TemporalAccessor,TemporalAccessor>>, <<painless-api-reference-ResolverStyle,ResolverStyle>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalField.html#resolve%2Djava.util.Map%2Djava.time.temporal.TemporalAccessor%2Djava.time.format.ResolverStyle%2D[java 9])
+* ++[[painless-api-reference-TemporalField-toString-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/temporal/TemporalField.html#toString%2D%2D[toString]()++ (link:{java9-javadoc}/java/time/temporal/TemporalField.html#toString%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/TemporalQueries.asciidoc b/docs/painless/painless-api-reference/TemporalQueries.asciidoc
new file mode 100644
index 0000000000000..bd01ba6c2e0ed
--- /dev/null
+++ b/docs/painless/painless-api-reference/TemporalQueries.asciidoc
@@ -0,0 +1,14 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-TemporalQueries]]++TemporalQueries++::
+* ++[[painless-api-reference-TemporalQueries-chronology-0]]static <<painless-api-reference-TemporalQuery,TemporalQuery>> link:{java8-javadoc}/java/time/temporal/TemporalQueries.html#chronology%2D%2D[chronology]()++ (link:{java9-javadoc}/java/time/temporal/TemporalQueries.html#chronology%2D%2D[java 9])
+* ++[[painless-api-reference-TemporalQueries-localDate-0]]static <<painless-api-reference-TemporalQuery,TemporalQuery>> link:{java8-javadoc}/java/time/temporal/TemporalQueries.html#localDate%2D%2D[localDate]()++ (link:{java9-javadoc}/java/time/temporal/TemporalQueries.html#localDate%2D%2D[java 9])
+* ++[[painless-api-reference-TemporalQueries-localTime-0]]static <<painless-api-reference-TemporalQuery,TemporalQuery>> link:{java8-javadoc}/java/time/temporal/TemporalQueries.html#localTime%2D%2D[localTime]()++ (link:{java9-javadoc}/java/time/temporal/TemporalQueries.html#localTime%2D%2D[java 9])
+* ++[[painless-api-reference-TemporalQueries-offset-0]]static <<painless-api-reference-TemporalQuery,TemporalQuery>> link:{java8-javadoc}/java/time/temporal/TemporalQueries.html#offset%2D%2D[offset]()++ (link:{java9-javadoc}/java/time/temporal/TemporalQueries.html#offset%2D%2D[java 9])
+* ++[[painless-api-reference-TemporalQueries-precision-0]]static <<painless-api-reference-TemporalQuery,TemporalQuery>> link:{java8-javadoc}/java/time/temporal/TemporalQueries.html#precision%2D%2D[precision]()++ (link:{java9-javadoc}/java/time/temporal/TemporalQueries.html#precision%2D%2D[java 9])
+* ++[[painless-api-reference-TemporalQueries-zone-0]]static <<painless-api-reference-TemporalQuery,TemporalQuery>> link:{java8-javadoc}/java/time/temporal/TemporalQueries.html#zone%2D%2D[zone]()++ (link:{java9-javadoc}/java/time/temporal/TemporalQueries.html#zone%2D%2D[java 9])
+* ++[[painless-api-reference-TemporalQueries-zoneId-0]]static <<painless-api-reference-TemporalQuery,TemporalQuery>> link:{java8-javadoc}/java/time/temporal/TemporalQueries.html#zoneId%2D%2D[zoneId]()++ (link:{java9-javadoc}/java/time/temporal/TemporalQueries.html#zoneId%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/TemporalQuery.asciidoc b/docs/painless/painless-api-reference/TemporalQuery.asciidoc
new file mode 100644
index 0000000000000..ca09eff368de3
--- /dev/null
+++ b/docs/painless/painless-api-reference/TemporalQuery.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-TemporalQuery]]++TemporalQuery++::
+* ++[[painless-api-reference-TemporalQuery-queryFrom-1]]def link:{java8-javadoc}/java/time/temporal/TemporalQuery.html#queryFrom%2Djava.time.temporal.TemporalAccessor%2D[queryFrom](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalQuery.html#queryFrom%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/TemporalUnit.asciidoc b/docs/painless/painless-api-reference/TemporalUnit.asciidoc
new file mode 100644
index 0000000000000..cc940d838efc2
--- /dev/null
+++ b/docs/painless/painless-api-reference/TemporalUnit.asciidoc
@@ -0,0 +1,15 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-TemporalUnit]]++TemporalUnit++::
+* ++[[painless-api-reference-TemporalUnit-addTo-2]]<<painless-api-reference-Temporal,Temporal>> link:{java8-javadoc}/java/time/temporal/TemporalUnit.html#addTo%2Djava.time.temporal.Temporal%2Dlong%2D[addTo](<<painless-api-reference-Temporal,Temporal>>, long)++ (link:{java9-javadoc}/java/time/temporal/TemporalUnit.html#addTo%2Djava.time.temporal.Temporal%2Dlong%2D[java 9])
+* ++[[painless-api-reference-TemporalUnit-between-2]]long link:{java8-javadoc}/java/time/temporal/TemporalUnit.html#between%2Djava.time.temporal.Temporal%2Djava.time.temporal.Temporal%2D[between](<<painless-api-reference-Temporal,Temporal>>, <<painless-api-reference-Temporal,Temporal>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalUnit.html#between%2Djava.time.temporal.Temporal%2Djava.time.temporal.Temporal%2D[java 9])
+* ++[[painless-api-reference-TemporalUnit-getDuration-0]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/temporal/TemporalUnit.html#getDuration%2D%2D[getDuration]()++ (link:{java9-javadoc}/java/time/temporal/TemporalUnit.html#getDuration%2D%2D[java 9])
+* ++[[painless-api-reference-TemporalUnit-isDateBased-0]]boolean link:{java8-javadoc}/java/time/temporal/TemporalUnit.html#isDateBased%2D%2D[isDateBased]()++ (link:{java9-javadoc}/java/time/temporal/TemporalUnit.html#isDateBased%2D%2D[java 9])
+* ++[[painless-api-reference-TemporalUnit-isDurationEstimated-0]]boolean link:{java8-javadoc}/java/time/temporal/TemporalUnit.html#isDurationEstimated%2D%2D[isDurationEstimated]()++ (link:{java9-javadoc}/java/time/temporal/TemporalUnit.html#isDurationEstimated%2D%2D[java 9])
+* ++[[painless-api-reference-TemporalUnit-isSupportedBy-1]]boolean link:{java8-javadoc}/java/time/temporal/TemporalUnit.html#isSupportedBy%2Djava.time.temporal.Temporal%2D[isSupportedBy](<<painless-api-reference-Temporal,Temporal>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalUnit.html#isSupportedBy%2Djava.time.temporal.Temporal%2D[java 9])
+* ++[[painless-api-reference-TemporalUnit-isTimeBased-0]]boolean link:{java8-javadoc}/java/time/temporal/TemporalUnit.html#isTimeBased%2D%2D[isTimeBased]()++ (link:{java9-javadoc}/java/time/temporal/TemporalUnit.html#isTimeBased%2D%2D[java 9])
+* ++[[painless-api-reference-TemporalUnit-toString-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/temporal/TemporalUnit.html#toString%2D%2D[toString]()++ (link:{java9-javadoc}/java/time/temporal/TemporalUnit.html#toString%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/TextStyle.asciidoc b/docs/painless/painless-api-reference/TextStyle.asciidoc
new file mode 100644
index 0000000000000..c7b122f934592
--- /dev/null
+++ b/docs/painless/painless-api-reference/TextStyle.asciidoc
@@ -0,0 +1,18 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-TextStyle]]++TextStyle++::
+** [[painless-api-reference-TextStyle-FULL]]static <<painless-api-reference-TextStyle,TextStyle>> link:{java8-javadoc}/java/time/format/TextStyle.html#FULL[FULL] (link:{java9-javadoc}/java/time/format/TextStyle.html#FULL[java 9])
+** [[painless-api-reference-TextStyle-FULL_STANDALONE]]static <<painless-api-reference-TextStyle,TextStyle>> link:{java8-javadoc}/java/time/format/TextStyle.html#FULL_STANDALONE[FULL_STANDALONE] (link:{java9-javadoc}/java/time/format/TextStyle.html#FULL_STANDALONE[java 9])
+** [[painless-api-reference-TextStyle-NARROW]]static <<painless-api-reference-TextStyle,TextStyle>> link:{java8-javadoc}/java/time/format/TextStyle.html#NARROW[NARROW] (link:{java9-javadoc}/java/time/format/TextStyle.html#NARROW[java 9])
+** [[painless-api-reference-TextStyle-NARROW_STANDALONE]]static <<painless-api-reference-TextStyle,TextStyle>> link:{java8-javadoc}/java/time/format/TextStyle.html#NARROW_STANDALONE[NARROW_STANDALONE] (link:{java9-javadoc}/java/time/format/TextStyle.html#NARROW_STANDALONE[java 9])
+** [[painless-api-reference-TextStyle-SHORT]]static <<painless-api-reference-TextStyle,TextStyle>> link:{java8-javadoc}/java/time/format/TextStyle.html#SHORT[SHORT] (link:{java9-javadoc}/java/time/format/TextStyle.html#SHORT[java 9])
+** [[painless-api-reference-TextStyle-SHORT_STANDALONE]]static <<painless-api-reference-TextStyle,TextStyle>> link:{java8-javadoc}/java/time/format/TextStyle.html#SHORT_STANDALONE[SHORT_STANDALONE] (link:{java9-javadoc}/java/time/format/TextStyle.html#SHORT_STANDALONE[java 9])
+* ++[[painless-api-reference-TextStyle-valueOf-1]]static <<painless-api-reference-TextStyle,TextStyle>> link:{java8-javadoc}/java/time/format/TextStyle.html#valueOf%2Djava.lang.String%2D[valueOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/format/TextStyle.html#valueOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-TextStyle-values-0]]static <<painless-api-reference-TextStyle,TextStyle>>[] link:{java8-javadoc}/java/time/format/TextStyle.html#values%2D%2D[values]()++ (link:{java9-javadoc}/java/time/format/TextStyle.html#values%2D%2D[java 9])
+* ++[[painless-api-reference-TextStyle-asNormal-0]]<<painless-api-reference-TextStyle,TextStyle>> link:{java8-javadoc}/java/time/format/TextStyle.html#asNormal%2D%2D[asNormal]()++ (link:{java9-javadoc}/java/time/format/TextStyle.html#asNormal%2D%2D[java 9])
+* ++[[painless-api-reference-TextStyle-asStandalone-0]]<<painless-api-reference-TextStyle,TextStyle>> link:{java8-javadoc}/java/time/format/TextStyle.html#asStandalone%2D%2D[asStandalone]()++ (link:{java9-javadoc}/java/time/format/TextStyle.html#asStandalone%2D%2D[java 9])
+* ++[[painless-api-reference-TextStyle-isStandalone-0]]boolean link:{java8-javadoc}/java/time/format/TextStyle.html#isStandalone%2D%2D[isStandalone]()++ (link:{java9-javadoc}/java/time/format/TextStyle.html#isStandalone%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Enum,Enum>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/ThaiBuddhistChronology.asciidoc b/docs/painless/painless-api-reference/ThaiBuddhistChronology.asciidoc
new file mode 100644
index 0000000000000..86c721fda7ada
--- /dev/null
+++ b/docs/painless/painless-api-reference/ThaiBuddhistChronology.asciidoc
@@ -0,0 +1,16 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ThaiBuddhistChronology]]++ThaiBuddhistChronology++::
+** [[painless-api-reference-ThaiBuddhistChronology-INSTANCE]]static <<painless-api-reference-ThaiBuddhistChronology,ThaiBuddhistChronology>> link:{java8-javadoc}/java/time/chrono/ThaiBuddhistChronology.html#INSTANCE[INSTANCE] (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistChronology.html#INSTANCE[java 9])
+* ++[[painless-api-reference-ThaiBuddhistChronology-date-1]]<<painless-api-reference-ThaiBuddhistDate,ThaiBuddhistDate>> link:{java8-javadoc}/java/time/chrono/ThaiBuddhistChronology.html#date%2Djava.time.temporal.TemporalAccessor%2D[date](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistChronology.html#date%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-ThaiBuddhistChronology-date-3]]<<painless-api-reference-ThaiBuddhistDate,ThaiBuddhistDate>> link:{java8-javadoc}/java/time/chrono/ThaiBuddhistChronology.html#date%2Dint%2Dint%2Dint%2D[date](int, int, int)++ (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistChronology.html#date%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-ThaiBuddhistChronology-date-4]]<<painless-api-reference-ThaiBuddhistDate,ThaiBuddhistDate>> link:{java8-javadoc}/java/time/chrono/ThaiBuddhistChronology.html#date%2Djava.time.chrono.Era%2Dint%2Dint%2Dint%2D[date](<<painless-api-reference-Era,Era>>, int, int, int)++ (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistChronology.html#date%2Djava.time.chrono.Era%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-ThaiBuddhistChronology-dateEpochDay-1]]<<painless-api-reference-ThaiBuddhistDate,ThaiBuddhistDate>> link:{java8-javadoc}/java/time/chrono/ThaiBuddhistChronology.html#dateEpochDay%2Dlong%2D[dateEpochDay](long)++ (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistChronology.html#dateEpochDay%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ThaiBuddhistChronology-dateYearDay-2]]<<painless-api-reference-ThaiBuddhistDate,ThaiBuddhistDate>> link:{java8-javadoc}/java/time/chrono/ThaiBuddhistChronology.html#dateYearDay%2Dint%2Dint%2D[dateYearDay](int, int)++ (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistChronology.html#dateYearDay%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-ThaiBuddhistChronology-dateYearDay-3]]<<painless-api-reference-ThaiBuddhistDate,ThaiBuddhistDate>> link:{java8-javadoc}/java/time/chrono/ThaiBuddhistChronology.html#dateYearDay%2Djava.time.chrono.Era%2Dint%2Dint%2D[dateYearDay](<<painless-api-reference-Era,Era>>, int, int)++ (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistChronology.html#dateYearDay%2Djava.time.chrono.Era%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-ThaiBuddhistChronology-eraOf-1]]<<painless-api-reference-ThaiBuddhistEra,ThaiBuddhistEra>> link:{java8-javadoc}/java/time/chrono/ThaiBuddhistChronology.html#eraOf%2Dint%2D[eraOf](int)++ (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistChronology.html#eraOf%2Dint%2D[java 9])
+* ++[[painless-api-reference-ThaiBuddhistChronology-resolveDate-2]]<<painless-api-reference-ThaiBuddhistDate,ThaiBuddhistDate>> link:{java8-javadoc}/java/time/chrono/ThaiBuddhistChronology.html#resolveDate%2Djava.util.Map%2Djava.time.format.ResolverStyle%2D[resolveDate](<<painless-api-reference-Map,Map>>, <<painless-api-reference-ResolverStyle,ResolverStyle>>)++ (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistChronology.html#resolveDate%2Djava.util.Map%2Djava.time.format.ResolverStyle%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Chronology,Chronology>>++
diff --git a/docs/painless/painless-api-reference/ThaiBuddhistDate.asciidoc b/docs/painless/painless-api-reference/ThaiBuddhistDate.asciidoc
new file mode 100644
index 0000000000000..94d620182dff4
--- /dev/null
+++ b/docs/painless/painless-api-reference/ThaiBuddhistDate.asciidoc
@@ -0,0 +1,17 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ThaiBuddhistDate]]++ThaiBuddhistDate++::
+* ++[[painless-api-reference-ThaiBuddhistDate-from-1]]static <<painless-api-reference-ThaiBuddhistDate,ThaiBuddhistDate>> link:{java8-javadoc}/java/time/chrono/ThaiBuddhistDate.html#from%2Djava.time.temporal.TemporalAccessor%2D[from](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistDate.html#from%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-ThaiBuddhistDate-of-3]]static <<painless-api-reference-ThaiBuddhistDate,ThaiBuddhistDate>> link:{java8-javadoc}/java/time/chrono/ThaiBuddhistDate.html#of%2Dint%2Dint%2Dint%2D[of](int, int, int)++ (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistDate.html#of%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-ThaiBuddhistDate-getChronology-0]]<<painless-api-reference-ThaiBuddhistChronology,ThaiBuddhistChronology>> link:{java8-javadoc}/java/time/chrono/ThaiBuddhistDate.html#getChronology%2D%2D[getChronology]()++ (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistDate.html#getChronology%2D%2D[java 9])
+* ++[[painless-api-reference-ThaiBuddhistDate-getEra-0]]<<painless-api-reference-ThaiBuddhistEra,ThaiBuddhistEra>> link:{java8-javadoc}/java/time/chrono/ThaiBuddhistDate.html#getEra%2D%2D[getEra]()++ (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistDate.html#getEra%2D%2D[java 9])
+* ++[[painless-api-reference-ThaiBuddhistDate-minus-1]]<<painless-api-reference-ThaiBuddhistDate,ThaiBuddhistDate>> link:{java8-javadoc}/java/time/chrono/ThaiBuddhistDate.html#minus%2Djava.time.temporal.TemporalAmount%2D[minus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistDate.html#minus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-ThaiBuddhistDate-minus-2]]<<painless-api-reference-ThaiBuddhistDate,ThaiBuddhistDate>> link:{java8-javadoc}/java/time/chrono/ThaiBuddhistDate.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[minus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistDate.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-ThaiBuddhistDate-plus-1]]<<painless-api-reference-ThaiBuddhistDate,ThaiBuddhistDate>> link:{java8-javadoc}/java/time/chrono/ThaiBuddhistDate.html#plus%2Djava.time.temporal.TemporalAmount%2D[plus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistDate.html#plus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-ThaiBuddhistDate-plus-2]]<<painless-api-reference-ThaiBuddhistDate,ThaiBuddhistDate>> link:{java8-javadoc}/java/time/chrono/ThaiBuddhistDate.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[plus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistDate.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-ThaiBuddhistDate-with-1]]<<painless-api-reference-ThaiBuddhistDate,ThaiBuddhistDate>> link:{java8-javadoc}/java/time/chrono/ThaiBuddhistDate.html#with%2Djava.time.temporal.TemporalAdjuster%2D[with](<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>)++ (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistDate.html#with%2Djava.time.temporal.TemporalAdjuster%2D[java 9])
+* ++[[painless-api-reference-ThaiBuddhistDate-with-2]]<<painless-api-reference-ThaiBuddhistDate,ThaiBuddhistDate>> link:{java8-javadoc}/java/time/chrono/ThaiBuddhistDate.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[with](<<painless-api-reference-TemporalField,TemporalField>>, long)++ (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistDate.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>>++, ++<<painless-api-reference-Temporal,Temporal>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++, ++<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>++
diff --git a/docs/painless/painless-api-reference/ThaiBuddhistEra.asciidoc b/docs/painless/painless-api-reference/ThaiBuddhistEra.asciidoc
new file mode 100644
index 0000000000000..b6c070503a9ee
--- /dev/null
+++ b/docs/painless/painless-api-reference/ThaiBuddhistEra.asciidoc
@@ -0,0 +1,13 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ThaiBuddhistEra]]++ThaiBuddhistEra++::
+** [[painless-api-reference-ThaiBuddhistEra-BE]]static <<painless-api-reference-ThaiBuddhistEra,ThaiBuddhistEra>> link:{java8-javadoc}/java/time/chrono/ThaiBuddhistEra.html#BE[BE] (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistEra.html#BE[java 9])
+** [[painless-api-reference-ThaiBuddhistEra-BEFORE_BE]]static <<painless-api-reference-ThaiBuddhistEra,ThaiBuddhistEra>> link:{java8-javadoc}/java/time/chrono/ThaiBuddhistEra.html#BEFORE_BE[BEFORE_BE] (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistEra.html#BEFORE_BE[java 9])
+* ++[[painless-api-reference-ThaiBuddhistEra-of-1]]static <<painless-api-reference-ThaiBuddhistEra,ThaiBuddhistEra>> link:{java8-javadoc}/java/time/chrono/ThaiBuddhistEra.html#of%2Dint%2D[of](int)++ (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistEra.html#of%2Dint%2D[java 9])
+* ++[[painless-api-reference-ThaiBuddhistEra-valueOf-1]]static <<painless-api-reference-ThaiBuddhistEra,ThaiBuddhistEra>> link:{java8-javadoc}/java/time/chrono/ThaiBuddhistEra.html#valueOf%2Djava.lang.String%2D[valueOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistEra.html#valueOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-ThaiBuddhistEra-values-0]]static <<painless-api-reference-ThaiBuddhistEra,ThaiBuddhistEra>>[] link:{java8-javadoc}/java/time/chrono/ThaiBuddhistEra.html#values%2D%2D[values]()++ (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistEra.html#values%2D%2D[java 9])
+* ++[[painless-api-reference-ThaiBuddhistEra-getValue-0]]int link:{java8-javadoc}/java/time/chrono/ThaiBuddhistEra.html#getValue%2D%2D[getValue]()++ (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistEra.html#getValue%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Enum,Enum>>++, ++<<painless-api-reference-Era,Era>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++, ++<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>++
diff --git a/docs/painless/painless-api-reference/TimeZone.asciidoc b/docs/painless/painless-api-reference/TimeZone.asciidoc
new file mode 100644
index 0000000000000..5618e388a4429
--- /dev/null
+++ b/docs/painless/painless-api-reference/TimeZone.asciidoc
@@ -0,0 +1,29 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-TimeZone]]++TimeZone++::
+** [[painless-api-reference-TimeZone-LONG]]static int link:{java8-javadoc}/java/util/TimeZone.html#LONG[LONG] (link:{java9-javadoc}/java/util/TimeZone.html#LONG[java 9])
+** [[painless-api-reference-TimeZone-SHORT]]static int link:{java8-javadoc}/java/util/TimeZone.html#SHORT[SHORT] (link:{java9-javadoc}/java/util/TimeZone.html#SHORT[java 9])
+* ++[[painless-api-reference-TimeZone-getAvailableIDs-0]]static <<painless-api-reference-String,String>>[] link:{java8-javadoc}/java/util/TimeZone.html#getAvailableIDs%2D%2D[getAvailableIDs]()++ (link:{java9-javadoc}/java/util/TimeZone.html#getAvailableIDs%2D%2D[java 9])
+* ++[[painless-api-reference-TimeZone-getAvailableIDs-1]]static <<painless-api-reference-String,String>>[] link:{java8-javadoc}/java/util/TimeZone.html#getAvailableIDs%2Dint%2D[getAvailableIDs](int)++ (link:{java9-javadoc}/java/util/TimeZone.html#getAvailableIDs%2Dint%2D[java 9])
+* ++[[painless-api-reference-TimeZone-getDefault-0]]static <<painless-api-reference-TimeZone,TimeZone>> link:{java8-javadoc}/java/util/TimeZone.html#getDefault%2D%2D[getDefault]()++ (link:{java9-javadoc}/java/util/TimeZone.html#getDefault%2D%2D[java 9])
+* ++[[painless-api-reference-TimeZone-getTimeZone-1]]static <<painless-api-reference-TimeZone,TimeZone>> link:{java8-javadoc}/java/util/TimeZone.html#getTimeZone%2Djava.lang.String%2D[getTimeZone](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/TimeZone.html#getTimeZone%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-TimeZone-clone-0]]def link:{java8-javadoc}/java/util/TimeZone.html#clone%2D%2D[clone]()++ (link:{java9-javadoc}/java/util/TimeZone.html#clone%2D%2D[java 9])
+* ++[[painless-api-reference-TimeZone-getDSTSavings-0]]int link:{java8-javadoc}/java/util/TimeZone.html#getDSTSavings%2D%2D[getDSTSavings]()++ (link:{java9-javadoc}/java/util/TimeZone.html#getDSTSavings%2D%2D[java 9])
+* ++[[painless-api-reference-TimeZone-getDisplayName-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/TimeZone.html#getDisplayName%2D%2D[getDisplayName]()++ (link:{java9-javadoc}/java/util/TimeZone.html#getDisplayName%2D%2D[java 9])
+* ++[[painless-api-reference-TimeZone-getDisplayName-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/TimeZone.html#getDisplayName%2Djava.util.Locale%2D[getDisplayName](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/util/TimeZone.html#getDisplayName%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-TimeZone-getDisplayName-2]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/TimeZone.html#getDisplayName%2Dboolean%2Dint%2D[getDisplayName](boolean, int)++ (link:{java9-javadoc}/java/util/TimeZone.html#getDisplayName%2Dboolean%2Dint%2D[java 9])
+* ++[[painless-api-reference-TimeZone-getDisplayName-3]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/TimeZone.html#getDisplayName%2Dboolean%2Dint%2Djava.util.Locale%2D[getDisplayName](boolean, int, <<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/util/TimeZone.html#getDisplayName%2Dboolean%2Dint%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-TimeZone-getID-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/TimeZone.html#getID%2D%2D[getID]()++ (link:{java9-javadoc}/java/util/TimeZone.html#getID%2D%2D[java 9])
+* ++[[painless-api-reference-TimeZone-getOffset-1]]int link:{java8-javadoc}/java/util/TimeZone.html#getOffset%2Dlong%2D[getOffset](long)++ (link:{java9-javadoc}/java/util/TimeZone.html#getOffset%2Dlong%2D[java 9])
+* ++[[painless-api-reference-TimeZone-getOffset-6]]int link:{java8-javadoc}/java/util/TimeZone.html#getOffset%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2D[getOffset](int, int, int, int, int, int)++ (link:{java9-javadoc}/java/util/TimeZone.html#getOffset%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-TimeZone-getRawOffset-0]]int link:{java8-javadoc}/java/util/TimeZone.html#getRawOffset%2D%2D[getRawOffset]()++ (link:{java9-javadoc}/java/util/TimeZone.html#getRawOffset%2D%2D[java 9])
+* ++[[painless-api-reference-TimeZone-hasSameRules-1]]boolean link:{java8-javadoc}/java/util/TimeZone.html#hasSameRules%2Djava.util.TimeZone%2D[hasSameRules](<<painless-api-reference-TimeZone,TimeZone>>)++ (link:{java9-javadoc}/java/util/TimeZone.html#hasSameRules%2Djava.util.TimeZone%2D[java 9])
+* ++[[painless-api-reference-TimeZone-inDaylightTime-1]]boolean link:{java8-javadoc}/java/util/TimeZone.html#inDaylightTime%2Djava.util.Date%2D[inDaylightTime](<<painless-api-reference-Date,Date>>)++ (link:{java9-javadoc}/java/util/TimeZone.html#inDaylightTime%2Djava.util.Date%2D[java 9])
+* ++[[painless-api-reference-TimeZone-observesDaylightTime-0]]boolean link:{java8-javadoc}/java/util/TimeZone.html#observesDaylightTime%2D%2D[observesDaylightTime]()++ (link:{java9-javadoc}/java/util/TimeZone.html#observesDaylightTime%2D%2D[java 9])
+* ++[[painless-api-reference-TimeZone-setRawOffset-1]]void link:{java8-javadoc}/java/util/TimeZone.html#setRawOffset%2Dint%2D[setRawOffset](int)++ (link:{java9-javadoc}/java/util/TimeZone.html#setRawOffset%2Dint%2D[java 9])
+* ++[[painless-api-reference-TimeZone-toZoneId-0]]<<painless-api-reference-ZoneId,ZoneId>> link:{java8-javadoc}/java/util/TimeZone.html#toZoneId%2D%2D[toZoneId]()++ (link:{java9-javadoc}/java/util/TimeZone.html#toZoneId%2D%2D[java 9])
+* ++[[painless-api-reference-TimeZone-useDaylightTime-0]]boolean link:{java8-javadoc}/java/util/TimeZone.html#useDaylightTime%2D%2D[useDaylightTime]()++ (link:{java9-javadoc}/java/util/TimeZone.html#useDaylightTime%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/ToDoubleBiFunction.asciidoc b/docs/painless/painless-api-reference/ToDoubleBiFunction.asciidoc
new file mode 100644
index 0000000000000..8ef0873fb5e15
--- /dev/null
+++ b/docs/painless/painless-api-reference/ToDoubleBiFunction.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ToDoubleBiFunction]]++ToDoubleBiFunction++::
+* ++[[painless-api-reference-ToDoubleBiFunction-applyAsDouble-2]]double link:{java8-javadoc}/java/util/function/ToDoubleBiFunction.html#applyAsDouble%2Djava.lang.Object%2Djava.lang.Object%2D[applyAsDouble](def, def)++ (link:{java9-javadoc}/java/util/function/ToDoubleBiFunction.html#applyAsDouble%2Djava.lang.Object%2Djava.lang.Object%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/ToDoubleFunction.asciidoc b/docs/painless/painless-api-reference/ToDoubleFunction.asciidoc
new file mode 100644
index 0000000000000..b0d66be9dc09e
--- /dev/null
+++ b/docs/painless/painless-api-reference/ToDoubleFunction.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ToDoubleFunction]]++ToDoubleFunction++::
+* ++[[painless-api-reference-ToDoubleFunction-applyAsDouble-1]]double link:{java8-javadoc}/java/util/function/ToDoubleFunction.html#applyAsDouble%2Djava.lang.Object%2D[applyAsDouble](def)++ (link:{java9-javadoc}/java/util/function/ToDoubleFunction.html#applyAsDouble%2Djava.lang.Object%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/ToIntBiFunction.asciidoc b/docs/painless/painless-api-reference/ToIntBiFunction.asciidoc
new file mode 100644
index 0000000000000..42cc396a1a989
--- /dev/null
+++ b/docs/painless/painless-api-reference/ToIntBiFunction.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ToIntBiFunction]]++ToIntBiFunction++::
+* ++[[painless-api-reference-ToIntBiFunction-applyAsInt-2]]int link:{java8-javadoc}/java/util/function/ToIntBiFunction.html#applyAsInt%2Djava.lang.Object%2Djava.lang.Object%2D[applyAsInt](def, def)++ (link:{java9-javadoc}/java/util/function/ToIntBiFunction.html#applyAsInt%2Djava.lang.Object%2Djava.lang.Object%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/ToIntFunction.asciidoc b/docs/painless/painless-api-reference/ToIntFunction.asciidoc
new file mode 100644
index 0000000000000..c471de6b77dc4
--- /dev/null
+++ b/docs/painless/painless-api-reference/ToIntFunction.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ToIntFunction]]++ToIntFunction++::
+* ++[[painless-api-reference-ToIntFunction-applyAsInt-1]]int link:{java8-javadoc}/java/util/function/ToIntFunction.html#applyAsInt%2Djava.lang.Object%2D[applyAsInt](def)++ (link:{java9-javadoc}/java/util/function/ToIntFunction.html#applyAsInt%2Djava.lang.Object%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/ToLongBiFunction.asciidoc b/docs/painless/painless-api-reference/ToLongBiFunction.asciidoc
new file mode 100644
index 0000000000000..ffda1353210a8
--- /dev/null
+++ b/docs/painless/painless-api-reference/ToLongBiFunction.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ToLongBiFunction]]++ToLongBiFunction++::
+* ++[[painless-api-reference-ToLongBiFunction-applyAsLong-2]]long link:{java8-javadoc}/java/util/function/ToLongBiFunction.html#applyAsLong%2Djava.lang.Object%2Djava.lang.Object%2D[applyAsLong](def, def)++ (link:{java9-javadoc}/java/util/function/ToLongBiFunction.html#applyAsLong%2Djava.lang.Object%2Djava.lang.Object%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/ToLongFunction.asciidoc b/docs/painless/painless-api-reference/ToLongFunction.asciidoc
new file mode 100644
index 0000000000000..785c414f8c906
--- /dev/null
+++ b/docs/painless/painless-api-reference/ToLongFunction.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ToLongFunction]]++ToLongFunction++::
+* ++[[painless-api-reference-ToLongFunction-applyAsLong-1]]long link:{java8-javadoc}/java/util/function/ToLongFunction.html#applyAsLong%2Djava.lang.Object%2D[applyAsLong](def)++ (link:{java9-javadoc}/java/util/function/ToLongFunction.html#applyAsLong%2Djava.lang.Object%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/TooManyListenersException.asciidoc b/docs/painless/painless-api-reference/TooManyListenersException.asciidoc
new file mode 100644
index 0000000000000..5a2d3d57b314d
--- /dev/null
+++ b/docs/painless/painless-api-reference/TooManyListenersException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-TooManyListenersException]]++TooManyListenersException++::
+* ++[[painless-api-reference-TooManyListenersException-TooManyListenersException-0]]link:{java8-javadoc}/java/util/TooManyListenersException.html#TooManyListenersException%2D%2D[TooManyListenersException]()++ (link:{java9-javadoc}/java/util/TooManyListenersException.html#TooManyListenersException%2D%2D[java 9])
+* ++[[painless-api-reference-TooManyListenersException-TooManyListenersException-1]]link:{java8-javadoc}/java/util/TooManyListenersException.html#TooManyListenersException%2Djava.lang.String%2D[TooManyListenersException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/TooManyListenersException.html#TooManyListenersException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/TreeMap.asciidoc b/docs/painless/painless-api-reference/TreeMap.asciidoc
new file mode 100644
index 0000000000000..f91bae69162f6
--- /dev/null
+++ b/docs/painless/painless-api-reference/TreeMap.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-TreeMap]]++TreeMap++::
+* ++[[painless-api-reference-TreeMap-TreeMap-0]]link:{java8-javadoc}/java/util/TreeMap.html#TreeMap%2D%2D[TreeMap]()++ (link:{java9-javadoc}/java/util/TreeMap.html#TreeMap%2D%2D[java 9])
+* ++[[painless-api-reference-TreeMap-TreeMap-1]]link:{java8-javadoc}/java/util/TreeMap.html#TreeMap%2Djava.util.Comparator%2D[TreeMap](<<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/TreeMap.html#TreeMap%2Djava.util.Comparator%2D[java 9])
+* ++[[painless-api-reference-TreeMap-clone-0]]def link:{java8-javadoc}/java/util/TreeMap.html#clone%2D%2D[clone]()++ (link:{java9-javadoc}/java/util/TreeMap.html#clone%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Map,Map>>++, ++<<painless-api-reference-NavigableMap,NavigableMap>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-SortedMap,SortedMap>>++
diff --git a/docs/painless/painless-api-reference/TreeSet.asciidoc b/docs/painless/painless-api-reference/TreeSet.asciidoc
new file mode 100644
index 0000000000000..84e8a339fe3bf
--- /dev/null
+++ b/docs/painless/painless-api-reference/TreeSet.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-TreeSet]]++TreeSet++::
+* ++[[painless-api-reference-TreeSet-TreeSet-0]]link:{java8-javadoc}/java/util/TreeSet.html#TreeSet%2D%2D[TreeSet]()++ (link:{java9-javadoc}/java/util/TreeSet.html#TreeSet%2D%2D[java 9])
+* ++[[painless-api-reference-TreeSet-TreeSet-1]]link:{java8-javadoc}/java/util/TreeSet.html#TreeSet%2Djava.util.Comparator%2D[TreeSet](<<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/TreeSet.html#TreeSet%2Djava.util.Comparator%2D[java 9])
+* ++[[painless-api-reference-TreeSet-clone-0]]def link:{java8-javadoc}/java/util/TreeSet.html#clone%2D%2D[clone]()++ (link:{java9-javadoc}/java/util/TreeSet.html#clone%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-NavigableSet,NavigableSet>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-Set,Set>>++, ++<<painless-api-reference-SortedSet,SortedSet>>++
diff --git a/docs/painless/painless-api-reference/TypeNotPresentException.asciidoc b/docs/painless/painless-api-reference/TypeNotPresentException.asciidoc
new file mode 100644
index 0000000000000..cecb588d83b17
--- /dev/null
+++ b/docs/painless/painless-api-reference/TypeNotPresentException.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-TypeNotPresentException]]++TypeNotPresentException++::
+* ++[[painless-api-reference-TypeNotPresentException-typeName-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/TypeNotPresentException.html#typeName%2D%2D[typeName]()++ (link:{java9-javadoc}/java/lang/TypeNotPresentException.html#typeName%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/UUID.asciidoc b/docs/painless/painless-api-reference/UUID.asciidoc
new file mode 100644
index 0000000000000..a47620f9bc1d6
--- /dev/null
+++ b/docs/painless/painless-api-reference/UUID.asciidoc
@@ -0,0 +1,18 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-UUID]]++UUID++::
+* ++[[painless-api-reference-UUID-fromString-1]]static <<painless-api-reference-UUID,UUID>> link:{java8-javadoc}/java/util/UUID.html#fromString%2Djava.lang.String%2D[fromString](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/UUID.html#fromString%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-UUID-nameUUIDFromBytes-1]]static <<painless-api-reference-UUID,UUID>> link:{java8-javadoc}/java/util/UUID.html#nameUUIDFromBytes%2Dbyte:A%2D[nameUUIDFromBytes](byte[])++ (link:{java9-javadoc}/java/util/UUID.html#nameUUIDFromBytes%2Dbyte:A%2D[java 9])
+* ++[[painless-api-reference-UUID-UUID-2]]link:{java8-javadoc}/java/util/UUID.html#UUID%2Dlong%2Dlong%2D[UUID](long, long)++ (link:{java9-javadoc}/java/util/UUID.html#UUID%2Dlong%2Dlong%2D[java 9])
+* ++[[painless-api-reference-UUID-clockSequence-0]]int link:{java8-javadoc}/java/util/UUID.html#clockSequence%2D%2D[clockSequence]()++ (link:{java9-javadoc}/java/util/UUID.html#clockSequence%2D%2D[java 9])
+* ++[[painless-api-reference-UUID-compareTo-1]]int link:{java8-javadoc}/java/util/UUID.html#compareTo%2Djava.util.UUID%2D[compareTo](<<painless-api-reference-UUID,UUID>>)++ (link:{java9-javadoc}/java/util/UUID.html#compareTo%2Djava.util.UUID%2D[java 9])
+* ++[[painless-api-reference-UUID-getLeastSignificantBits-0]]long link:{java8-javadoc}/java/util/UUID.html#getLeastSignificantBits%2D%2D[getLeastSignificantBits]()++ (link:{java9-javadoc}/java/util/UUID.html#getLeastSignificantBits%2D%2D[java 9])
+* ++[[painless-api-reference-UUID-getMostSignificantBits-0]]long link:{java8-javadoc}/java/util/UUID.html#getMostSignificantBits%2D%2D[getMostSignificantBits]()++ (link:{java9-javadoc}/java/util/UUID.html#getMostSignificantBits%2D%2D[java 9])
+* ++[[painless-api-reference-UUID-node-0]]long link:{java8-javadoc}/java/util/UUID.html#node%2D%2D[node]()++ (link:{java9-javadoc}/java/util/UUID.html#node%2D%2D[java 9])
+* ++[[painless-api-reference-UUID-timestamp-0]]long link:{java8-javadoc}/java/util/UUID.html#timestamp%2D%2D[timestamp]()++ (link:{java9-javadoc}/java/util/UUID.html#timestamp%2D%2D[java 9])
+* ++[[painless-api-reference-UUID-variant-0]]int link:{java8-javadoc}/java/util/UUID.html#variant%2D%2D[variant]()++ (link:{java9-javadoc}/java/util/UUID.html#variant%2D%2D[java 9])
+* ++[[painless-api-reference-UUID-version-0]]int link:{java8-javadoc}/java/util/UUID.html#version%2D%2D[version]()++ (link:{java9-javadoc}/java/util/UUID.html#version%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/UnaryOperator.asciidoc b/docs/painless/painless-api-reference/UnaryOperator.asciidoc
new file mode 100644
index 0000000000000..fb7c6ed70d25d
--- /dev/null
+++ b/docs/painless/painless-api-reference/UnaryOperator.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-UnaryOperator]]++UnaryOperator++::
+* ++[[painless-api-reference-UnaryOperator-identity-0]]static <<painless-api-reference-UnaryOperator,UnaryOperator>> link:{java8-javadoc}/java/util/function/UnaryOperator.html#identity%2D%2D[identity]()++ (link:{java9-javadoc}/java/util/function/UnaryOperator.html#identity%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Function,Function>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/UnknownFormatConversionException.asciidoc b/docs/painless/painless-api-reference/UnknownFormatConversionException.asciidoc
new file mode 100644
index 0000000000000..95dd348f422c1
--- /dev/null
+++ b/docs/painless/painless-api-reference/UnknownFormatConversionException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-UnknownFormatConversionException]]++UnknownFormatConversionException++::
+* ++[[painless-api-reference-UnknownFormatConversionException-UnknownFormatConversionException-1]]link:{java8-javadoc}/java/util/UnknownFormatConversionException.html#UnknownFormatConversionException%2Djava.lang.String%2D[UnknownFormatConversionException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/UnknownFormatConversionException.html#UnknownFormatConversionException%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-UnknownFormatConversionException-getConversion-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/UnknownFormatConversionException.html#getConversion%2D%2D[getConversion]()++ (link:{java9-javadoc}/java/util/UnknownFormatConversionException.html#getConversion%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/UnknownFormatFlagsException.asciidoc b/docs/painless/painless-api-reference/UnknownFormatFlagsException.asciidoc
new file mode 100644
index 0000000000000..f14d986c05b38
--- /dev/null
+++ b/docs/painless/painless-api-reference/UnknownFormatFlagsException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-UnknownFormatFlagsException]]++UnknownFormatFlagsException++::
+* ++[[painless-api-reference-UnknownFormatFlagsException-UnknownFormatFlagsException-1]]link:{java8-javadoc}/java/util/UnknownFormatFlagsException.html#UnknownFormatFlagsException%2Djava.lang.String%2D[UnknownFormatFlagsException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/UnknownFormatFlagsException.html#UnknownFormatFlagsException%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-UnknownFormatFlagsException-getFlags-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/UnknownFormatFlagsException.html#getFlags%2D%2D[getFlags]()++ (link:{java9-javadoc}/java/util/UnknownFormatFlagsException.html#getFlags%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/UnsupportedOperationException.asciidoc b/docs/painless/painless-api-reference/UnsupportedOperationException.asciidoc
new file mode 100644
index 0000000000000..ad9cbdb1478bf
--- /dev/null
+++ b/docs/painless/painless-api-reference/UnsupportedOperationException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-UnsupportedOperationException]]++UnsupportedOperationException++::
+* ++[[painless-api-reference-UnsupportedOperationException-UnsupportedOperationException-0]]link:{java8-javadoc}/java/lang/UnsupportedOperationException.html#UnsupportedOperationException%2D%2D[UnsupportedOperationException]()++ (link:{java9-javadoc}/java/lang/UnsupportedOperationException.html#UnsupportedOperationException%2D%2D[java 9])
+* ++[[painless-api-reference-UnsupportedOperationException-UnsupportedOperationException-1]]link:{java8-javadoc}/java/lang/UnsupportedOperationException.html#UnsupportedOperationException%2Djava.lang.String%2D[UnsupportedOperationException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/UnsupportedOperationException.html#UnsupportedOperationException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/UnsupportedTemporalTypeException.asciidoc b/docs/painless/painless-api-reference/UnsupportedTemporalTypeException.asciidoc
new file mode 100644
index 0000000000000..96e4c1d840220
--- /dev/null
+++ b/docs/painless/painless-api-reference/UnsupportedTemporalTypeException.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-UnsupportedTemporalTypeException]]++UnsupportedTemporalTypeException++::
+* ++[[painless-api-reference-UnsupportedTemporalTypeException-UnsupportedTemporalTypeException-1]]link:{java8-javadoc}/java/time/temporal/UnsupportedTemporalTypeException.html#UnsupportedTemporalTypeException%2Djava.lang.String%2D[UnsupportedTemporalTypeException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/temporal/UnsupportedTemporalTypeException.html#UnsupportedTemporalTypeException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/ValueRange.asciidoc b/docs/painless/painless-api-reference/ValueRange.asciidoc
new file mode 100644
index 0000000000000..409d155f9546a
--- /dev/null
+++ b/docs/painless/painless-api-reference/ValueRange.asciidoc
@@ -0,0 +1,20 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ValueRange]]++ValueRange++::
+* ++[[painless-api-reference-ValueRange-of-2]]static <<painless-api-reference-ValueRange,ValueRange>> link:{java8-javadoc}/java/time/temporal/ValueRange.html#of%2Dlong%2Dlong%2D[of](long, long)++ (link:{java9-javadoc}/java/time/temporal/ValueRange.html#of%2Dlong%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ValueRange-of-3]]static <<painless-api-reference-ValueRange,ValueRange>> link:{java8-javadoc}/java/time/temporal/ValueRange.html#of%2Dlong%2Dlong%2Dlong%2D[of](long, long, long)++ (link:{java9-javadoc}/java/time/temporal/ValueRange.html#of%2Dlong%2Dlong%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ValueRange-of-4]]static <<painless-api-reference-ValueRange,ValueRange>> link:{java8-javadoc}/java/time/temporal/ValueRange.html#of%2Dlong%2Dlong%2Dlong%2Dlong%2D[of](long, long, long, long)++ (link:{java9-javadoc}/java/time/temporal/ValueRange.html#of%2Dlong%2Dlong%2Dlong%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ValueRange-checkValidIntValue-2]]int link:{java8-javadoc}/java/time/temporal/ValueRange.html#checkValidIntValue%2Dlong%2Djava.time.temporal.TemporalField%2D[checkValidIntValue](long, <<painless-api-reference-TemporalField,TemporalField>>)++ (link:{java9-javadoc}/java/time/temporal/ValueRange.html#checkValidIntValue%2Dlong%2Djava.time.temporal.TemporalField%2D[java 9])
+* ++[[painless-api-reference-ValueRange-checkValidValue-2]]long link:{java8-javadoc}/java/time/temporal/ValueRange.html#checkValidValue%2Dlong%2Djava.time.temporal.TemporalField%2D[checkValidValue](long, <<painless-api-reference-TemporalField,TemporalField>>)++ (link:{java9-javadoc}/java/time/temporal/ValueRange.html#checkValidValue%2Dlong%2Djava.time.temporal.TemporalField%2D[java 9])
+* ++[[painless-api-reference-ValueRange-getLargestMinimum-0]]long link:{java8-javadoc}/java/time/temporal/ValueRange.html#getLargestMinimum%2D%2D[getLargestMinimum]()++ (link:{java9-javadoc}/java/time/temporal/ValueRange.html#getLargestMinimum%2D%2D[java 9])
+* ++[[painless-api-reference-ValueRange-getMaximum-0]]long link:{java8-javadoc}/java/time/temporal/ValueRange.html#getMaximum%2D%2D[getMaximum]()++ (link:{java9-javadoc}/java/time/temporal/ValueRange.html#getMaximum%2D%2D[java 9])
+* ++[[painless-api-reference-ValueRange-getMinimum-0]]long link:{java8-javadoc}/java/time/temporal/ValueRange.html#getMinimum%2D%2D[getMinimum]()++ (link:{java9-javadoc}/java/time/temporal/ValueRange.html#getMinimum%2D%2D[java 9])
+* ++[[painless-api-reference-ValueRange-getSmallestMaximum-0]]long link:{java8-javadoc}/java/time/temporal/ValueRange.html#getSmallestMaximum%2D%2D[getSmallestMaximum]()++ (link:{java9-javadoc}/java/time/temporal/ValueRange.html#getSmallestMaximum%2D%2D[java 9])
+* ++[[painless-api-reference-ValueRange-isFixed-0]]boolean link:{java8-javadoc}/java/time/temporal/ValueRange.html#isFixed%2D%2D[isFixed]()++ (link:{java9-javadoc}/java/time/temporal/ValueRange.html#isFixed%2D%2D[java 9])
+* ++[[painless-api-reference-ValueRange-isIntValue-0]]boolean link:{java8-javadoc}/java/time/temporal/ValueRange.html#isIntValue%2D%2D[isIntValue]()++ (link:{java9-javadoc}/java/time/temporal/ValueRange.html#isIntValue%2D%2D[java 9])
+* ++[[painless-api-reference-ValueRange-isValidIntValue-1]]boolean link:{java8-javadoc}/java/time/temporal/ValueRange.html#isValidIntValue%2Dlong%2D[isValidIntValue](long)++ (link:{java9-javadoc}/java/time/temporal/ValueRange.html#isValidIntValue%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ValueRange-isValidValue-1]]boolean link:{java8-javadoc}/java/time/temporal/ValueRange.html#isValidValue%2Dlong%2D[isValidValue](long)++ (link:{java9-javadoc}/java/time/temporal/ValueRange.html#isValidValue%2Dlong%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Vector.asciidoc b/docs/painless/painless-api-reference/Vector.asciidoc
new file mode 100644
index 0000000000000..b1d9ed88b6f9a
--- /dev/null
+++ b/docs/painless/painless-api-reference/Vector.asciidoc
@@ -0,0 +1,22 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Vector]]++Vector++::
+* ++[[painless-api-reference-Vector-Vector-0]]link:{java8-javadoc}/java/util/Vector.html#Vector%2D%2D[Vector]()++ (link:{java9-javadoc}/java/util/Vector.html#Vector%2D%2D[java 9])
+* ++[[painless-api-reference-Vector-Vector-1]]link:{java8-javadoc}/java/util/Vector.html#Vector%2Djava.util.Collection%2D[Vector](<<painless-api-reference-Collection,Collection>>)++ (link:{java9-javadoc}/java/util/Vector.html#Vector%2Djava.util.Collection%2D[java 9])
+* ++[[painless-api-reference-Vector-addElement-1]]void link:{java8-javadoc}/java/util/Vector.html#addElement%2Djava.lang.Object%2D[addElement](def)++ (link:{java9-javadoc}/java/util/Vector.html#addElement%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Vector-clone-0]]def link:{java8-javadoc}/java/util/Vector.html#clone%2D%2D[clone]()++ (link:{java9-javadoc}/java/util/Vector.html#clone%2D%2D[java 9])
+* ++[[painless-api-reference-Vector-copyInto-1]]void link:{java8-javadoc}/java/util/Vector.html#copyInto%2Djava.lang.Object:A%2D[copyInto](<<painless-api-reference-Object,Object>>[])++ (link:{java9-javadoc}/java/util/Vector.html#copyInto%2Djava.lang.Object:A%2D[java 9])
+* ++[[painless-api-reference-Vector-elementAt-1]]def link:{java8-javadoc}/java/util/Vector.html#elementAt%2Dint%2D[elementAt](int)++ (link:{java9-javadoc}/java/util/Vector.html#elementAt%2Dint%2D[java 9])
+* ++[[painless-api-reference-Vector-elements-0]]<<painless-api-reference-Enumeration,Enumeration>> link:{java8-javadoc}/java/util/Vector.html#elements%2D%2D[elements]()++ (link:{java9-javadoc}/java/util/Vector.html#elements%2D%2D[java 9])
+* ++[[painless-api-reference-Vector-firstElement-0]]def link:{java8-javadoc}/java/util/Vector.html#firstElement%2D%2D[firstElement]()++ (link:{java9-javadoc}/java/util/Vector.html#firstElement%2D%2D[java 9])
+* ++[[painless-api-reference-Vector-insertElementAt-2]]void link:{java8-javadoc}/java/util/Vector.html#insertElementAt%2Djava.lang.Object%2Dint%2D[insertElementAt](def, int)++ (link:{java9-javadoc}/java/util/Vector.html#insertElementAt%2Djava.lang.Object%2Dint%2D[java 9])
+* ++[[painless-api-reference-Vector-lastElement-0]]def link:{java8-javadoc}/java/util/Vector.html#lastElement%2D%2D[lastElement]()++ (link:{java9-javadoc}/java/util/Vector.html#lastElement%2D%2D[java 9])
+* ++[[painless-api-reference-Vector-lastIndexOf-2]]int link:{java8-javadoc}/java/util/Vector.html#lastIndexOf%2Djava.lang.Object%2Dint%2D[lastIndexOf](def, int)++ (link:{java9-javadoc}/java/util/Vector.html#lastIndexOf%2Djava.lang.Object%2Dint%2D[java 9])
+* ++[[painless-api-reference-Vector-removeAllElements-0]]void link:{java8-javadoc}/java/util/Vector.html#removeAllElements%2D%2D[removeAllElements]()++ (link:{java9-javadoc}/java/util/Vector.html#removeAllElements%2D%2D[java 9])
+* ++[[painless-api-reference-Vector-removeElement-1]]boolean link:{java8-javadoc}/java/util/Vector.html#removeElement%2Djava.lang.Object%2D[removeElement](def)++ (link:{java9-javadoc}/java/util/Vector.html#removeElement%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Vector-removeElementAt-1]]void link:{java8-javadoc}/java/util/Vector.html#removeElementAt%2Dint%2D[removeElementAt](int)++ (link:{java9-javadoc}/java/util/Vector.html#removeElementAt%2Dint%2D[java 9])
+* ++[[painless-api-reference-Vector-setElementAt-2]]void link:{java8-javadoc}/java/util/Vector.html#setElementAt%2Djava.lang.Object%2Dint%2D[setElementAt](def, int)++ (link:{java9-javadoc}/java/util/Vector.html#setElementAt%2Djava.lang.Object%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-List,List>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/WeekFields.asciidoc b/docs/painless/painless-api-reference/WeekFields.asciidoc
new file mode 100644
index 0000000000000..330b2e98d2e7a
--- /dev/null
+++ b/docs/painless/painless-api-reference/WeekFields.asciidoc
@@ -0,0 +1,19 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-WeekFields]]++WeekFields++::
+** [[painless-api-reference-WeekFields-ISO]]static <<painless-api-reference-WeekFields,WeekFields>> link:{java8-javadoc}/java/time/temporal/WeekFields.html#ISO[ISO] (link:{java9-javadoc}/java/time/temporal/WeekFields.html#ISO[java 9])
+** [[painless-api-reference-WeekFields-SUNDAY_START]]static <<painless-api-reference-WeekFields,WeekFields>> link:{java8-javadoc}/java/time/temporal/WeekFields.html#SUNDAY_START[SUNDAY_START] (link:{java9-javadoc}/java/time/temporal/WeekFields.html#SUNDAY_START[java 9])
+** [[painless-api-reference-WeekFields-WEEK_BASED_YEARS]]static <<painless-api-reference-TemporalUnit,TemporalUnit>> link:{java8-javadoc}/java/time/temporal/WeekFields.html#WEEK_BASED_YEARS[WEEK_BASED_YEARS] (link:{java9-javadoc}/java/time/temporal/WeekFields.html#WEEK_BASED_YEARS[java 9])
+* ++[[painless-api-reference-WeekFields-of-1]]static <<painless-api-reference-WeekFields,WeekFields>> link:{java8-javadoc}/java/time/temporal/WeekFields.html#of%2Djava.util.Locale%2D[of](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/time/temporal/WeekFields.html#of%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-WeekFields-of-2]]static <<painless-api-reference-WeekFields,WeekFields>> link:{java8-javadoc}/java/time/temporal/WeekFields.html#of%2Djava.time.DayOfWeek%2Dint%2D[of](<<painless-api-reference-DayOfWeek,DayOfWeek>>, int)++ (link:{java9-javadoc}/java/time/temporal/WeekFields.html#of%2Djava.time.DayOfWeek%2Dint%2D[java 9])
+* ++[[painless-api-reference-WeekFields-dayOfWeek-0]]<<painless-api-reference-TemporalField,TemporalField>> link:{java8-javadoc}/java/time/temporal/WeekFields.html#dayOfWeek%2D%2D[dayOfWeek]()++ (link:{java9-javadoc}/java/time/temporal/WeekFields.html#dayOfWeek%2D%2D[java 9])
+* ++[[painless-api-reference-WeekFields-getFirstDayOfWeek-0]]<<painless-api-reference-DayOfWeek,DayOfWeek>> link:{java8-javadoc}/java/time/temporal/WeekFields.html#getFirstDayOfWeek%2D%2D[getFirstDayOfWeek]()++ (link:{java9-javadoc}/java/time/temporal/WeekFields.html#getFirstDayOfWeek%2D%2D[java 9])
+* ++[[painless-api-reference-WeekFields-getMinimalDaysInFirstWeek-0]]int link:{java8-javadoc}/java/time/temporal/WeekFields.html#getMinimalDaysInFirstWeek%2D%2D[getMinimalDaysInFirstWeek]()++ (link:{java9-javadoc}/java/time/temporal/WeekFields.html#getMinimalDaysInFirstWeek%2D%2D[java 9])
+* ++[[painless-api-reference-WeekFields-weekBasedYear-0]]<<painless-api-reference-TemporalField,TemporalField>> link:{java8-javadoc}/java/time/temporal/WeekFields.html#weekBasedYear%2D%2D[weekBasedYear]()++ (link:{java9-javadoc}/java/time/temporal/WeekFields.html#weekBasedYear%2D%2D[java 9])
+* ++[[painless-api-reference-WeekFields-weekOfMonth-0]]<<painless-api-reference-TemporalField,TemporalField>> link:{java8-javadoc}/java/time/temporal/WeekFields.html#weekOfMonth%2D%2D[weekOfMonth]()++ (link:{java9-javadoc}/java/time/temporal/WeekFields.html#weekOfMonth%2D%2D[java 9])
+* ++[[painless-api-reference-WeekFields-weekOfWeekBasedYear-0]]<<painless-api-reference-TemporalField,TemporalField>> link:{java8-javadoc}/java/time/temporal/WeekFields.html#weekOfWeekBasedYear%2D%2D[weekOfWeekBasedYear]()++ (link:{java9-javadoc}/java/time/temporal/WeekFields.html#weekOfWeekBasedYear%2D%2D[java 9])
+* ++[[painless-api-reference-WeekFields-weekOfYear-0]]<<painless-api-reference-TemporalField,TemporalField>> link:{java8-javadoc}/java/time/temporal/WeekFields.html#weekOfYear%2D%2D[weekOfYear]()++ (link:{java9-javadoc}/java/time/temporal/WeekFields.html#weekOfYear%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/Year.asciidoc b/docs/painless/painless-api-reference/Year.asciidoc
new file mode 100644
index 0000000000000..e3800991039d6
--- /dev/null
+++ b/docs/painless/painless-api-reference/Year.asciidoc
@@ -0,0 +1,32 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Year]]++Year++::
+** [[painless-api-reference-Year-MAX_VALUE]]static int link:{java8-javadoc}/java/time/Year.html#MAX_VALUE[MAX_VALUE] (link:{java9-javadoc}/java/time/Year.html#MAX_VALUE[java 9])
+** [[painless-api-reference-Year-MIN_VALUE]]static int link:{java8-javadoc}/java/time/Year.html#MIN_VALUE[MIN_VALUE] (link:{java9-javadoc}/java/time/Year.html#MIN_VALUE[java 9])
+* ++[[painless-api-reference-Year-from-1]]static <<painless-api-reference-Year,Year>> link:{java8-javadoc}/java/time/Year.html#from%2Djava.time.temporal.TemporalAccessor%2D[from](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/Year.html#from%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-Year-isLeap-1]]static boolean link:{java8-javadoc}/java/time/Year.html#isLeap%2Dlong%2D[isLeap](long)++ (link:{java9-javadoc}/java/time/Year.html#isLeap%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Year-of-1]]static <<painless-api-reference-Year,Year>> link:{java8-javadoc}/java/time/Year.html#of%2Dint%2D[of](int)++ (link:{java9-javadoc}/java/time/Year.html#of%2Dint%2D[java 9])
+* ++[[painless-api-reference-Year-parse-1]]static <<painless-api-reference-Year,Year>> link:{java8-javadoc}/java/time/Year.html#parse%2Djava.lang.CharSequence%2D[parse](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/time/Year.html#parse%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-Year-parse-2]]static <<painless-api-reference-Year,Year>> link:{java8-javadoc}/java/time/Year.html#parse%2Djava.lang.CharSequence%2Djava.time.format.DateTimeFormatter%2D[parse](<<painless-api-reference-CharSequence,CharSequence>>, <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>>)++ (link:{java9-javadoc}/java/time/Year.html#parse%2Djava.lang.CharSequence%2Djava.time.format.DateTimeFormatter%2D[java 9])
+* ++[[painless-api-reference-Year-atDay-1]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/Year.html#atDay%2Dint%2D[atDay](int)++ (link:{java9-javadoc}/java/time/Year.html#atDay%2Dint%2D[java 9])
+* ++[[painless-api-reference-Year-atMonth-1]]<<painless-api-reference-YearMonth,YearMonth>> link:{java8-javadoc}/java/time/Year.html#atMonth%2Dint%2D[atMonth](int)++ (link:{java9-javadoc}/java/time/Year.html#atMonth%2Dint%2D[java 9])
+* ++[[painless-api-reference-Year-atMonthDay-1]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/Year.html#atMonthDay%2Djava.time.MonthDay%2D[atMonthDay](<<painless-api-reference-MonthDay,MonthDay>>)++ (link:{java9-javadoc}/java/time/Year.html#atMonthDay%2Djava.time.MonthDay%2D[java 9])
+* ++[[painless-api-reference-Year-compareTo-1]]int link:{java8-javadoc}/java/time/Year.html#compareTo%2Djava.time.Year%2D[compareTo](<<painless-api-reference-Year,Year>>)++ (link:{java9-javadoc}/java/time/Year.html#compareTo%2Djava.time.Year%2D[java 9])
+* ++[[painless-api-reference-Year-format-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/Year.html#format%2Djava.time.format.DateTimeFormatter%2D[format](<<painless-api-reference-DateTimeFormatter,DateTimeFormatter>>)++ (link:{java9-javadoc}/java/time/Year.html#format%2Djava.time.format.DateTimeFormatter%2D[java 9])
+* ++[[painless-api-reference-Year-getValue-0]]int link:{java8-javadoc}/java/time/Year.html#getValue%2D%2D[getValue]()++ (link:{java9-javadoc}/java/time/Year.html#getValue%2D%2D[java 9])
+* ++[[painless-api-reference-Year-isAfter-1]]boolean link:{java8-javadoc}/java/time/Year.html#isAfter%2Djava.time.Year%2D[isAfter](<<painless-api-reference-Year,Year>>)++ (link:{java9-javadoc}/java/time/Year.html#isAfter%2Djava.time.Year%2D[java 9])
+* ++[[painless-api-reference-Year-isLeap-0]]boolean link:{java8-javadoc}/java/time/Year.html#isLeap%2D%2D[isLeap]()++ (link:{java9-javadoc}/java/time/Year.html#isLeap%2D%2D[java 9])
+* ++[[painless-api-reference-Year-isValidMonthDay-1]]boolean link:{java8-javadoc}/java/time/Year.html#isValidMonthDay%2Djava.time.MonthDay%2D[isValidMonthDay](<<painless-api-reference-MonthDay,MonthDay>>)++ (link:{java9-javadoc}/java/time/Year.html#isValidMonthDay%2Djava.time.MonthDay%2D[java 9])
+* ++[[painless-api-reference-Year-length-0]]int link:{java8-javadoc}/java/time/Year.html#length%2D%2D[length]()++ (link:{java9-javadoc}/java/time/Year.html#length%2D%2D[java 9])
+* ++[[painless-api-reference-Year-minus-1]]<<painless-api-reference-Year,Year>> link:{java8-javadoc}/java/time/Year.html#minus%2Djava.time.temporal.TemporalAmount%2D[minus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/Year.html#minus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-Year-minus-2]]<<painless-api-reference-Year,Year>> link:{java8-javadoc}/java/time/Year.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[minus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/Year.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-Year-minusYears-1]]<<painless-api-reference-Year,Year>> link:{java8-javadoc}/java/time/Year.html#minusYears%2Dlong%2D[minusYears](long)++ (link:{java9-javadoc}/java/time/Year.html#minusYears%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Year-plus-1]]<<painless-api-reference-Year,Year>> link:{java8-javadoc}/java/time/Year.html#plus%2Djava.time.temporal.TemporalAmount%2D[plus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/Year.html#plus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-Year-plus-2]]<<painless-api-reference-Year,Year>> link:{java8-javadoc}/java/time/Year.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[plus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/Year.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-Year-plusYears-1]]<<painless-api-reference-Year,Year>> link:{java8-javadoc}/java/time/Year.html#plusYears%2Dlong%2D[plusYears](long)++ (link:{java9-javadoc}/java/time/Year.html#plusYears%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Year-with-1]]<<painless-api-reference-Year,Year>> link:{java8-javadoc}/java/time/Year.html#with%2Djava.time.temporal.TemporalAdjuster%2D[with](<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>)++ (link:{java9-javadoc}/java/time/Year.html#with%2Djava.time.temporal.TemporalAdjuster%2D[java 9])
+* ++[[painless-api-reference-Year-with-2]]<<painless-api-reference-Year,Year>> link:{java8-javadoc}/java/time/Year.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[with](<<painless-api-reference-TemporalField,TemporalField>>, long)++ (link:{java9-javadoc}/java/time/Year.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-Temporal,Temporal>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++, ++<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>++
diff --git a/docs/painless/painless-api-reference/YearMonth.asciidoc b/docs/painless/painless-api-reference/YearMonth.asciidoc
new file mode 100644
index 0000000000000..5573b16c0640c
--- /dev/null
+++ b/docs/painless/painless-api-reference/YearMonth.asciidoc
@@ -0,0 +1,36 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-YearMonth]]++YearMonth++::
+* ++[[painless-api-reference-YearMonth-from-1]]static <<painless-api-reference-YearMonth,YearMonth>> link:{java8-javadoc}/java/time/YearMonth.html#from%2Djava.time.temporal.TemporalAccessor%2D[from](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/YearMonth.html#from%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-YearMonth-of-2]]static <<painless-api-reference-YearMonth,YearMonth>> link:{java8-javadoc}/java/time/YearMonth.html#of%2Dint%2Dint%2D[of](int, int)++ (link:{java9-javadoc}/java/time/YearMonth.html#of%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-YearMonth-parse-1]]static <<painless-api-reference-YearMonth,YearMonth>> link:{java8-javadoc}/java/time/YearMonth.html#parse%2Djava.lang.CharSequence%2D[parse](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/time/YearMonth.html#parse%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-YearMonth-parse-2]]static <<painless-api-reference-YearMonth,YearMonth>> link:{java8-javadoc}/java/time/YearMonth.html#parse%2Djava.lang.CharSequence%2Djava.time.format.DateTimeFormatter%2D[parse](<<painless-api-reference-CharSequence,CharSequence>>, <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>>)++ (link:{java9-javadoc}/java/time/YearMonth.html#parse%2Djava.lang.CharSequence%2Djava.time.format.DateTimeFormatter%2D[java 9])
+* ++[[painless-api-reference-YearMonth-atDay-1]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/YearMonth.html#atDay%2Dint%2D[atDay](int)++ (link:{java9-javadoc}/java/time/YearMonth.html#atDay%2Dint%2D[java 9])
+* ++[[painless-api-reference-YearMonth-atEndOfMonth-0]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/YearMonth.html#atEndOfMonth%2D%2D[atEndOfMonth]()++ (link:{java9-javadoc}/java/time/YearMonth.html#atEndOfMonth%2D%2D[java 9])
+* ++[[painless-api-reference-YearMonth-compareTo-1]]int link:{java8-javadoc}/java/time/YearMonth.html#compareTo%2Djava.time.YearMonth%2D[compareTo](<<painless-api-reference-YearMonth,YearMonth>>)++ (link:{java9-javadoc}/java/time/YearMonth.html#compareTo%2Djava.time.YearMonth%2D[java 9])
+* ++[[painless-api-reference-YearMonth-format-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/YearMonth.html#format%2Djava.time.format.DateTimeFormatter%2D[format](<<painless-api-reference-DateTimeFormatter,DateTimeFormatter>>)++ (link:{java9-javadoc}/java/time/YearMonth.html#format%2Djava.time.format.DateTimeFormatter%2D[java 9])
+* ++[[painless-api-reference-YearMonth-getMonth-0]]<<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/YearMonth.html#getMonth%2D%2D[getMonth]()++ (link:{java9-javadoc}/java/time/YearMonth.html#getMonth%2D%2D[java 9])
+* ++[[painless-api-reference-YearMonth-getMonthValue-0]]int link:{java8-javadoc}/java/time/YearMonth.html#getMonthValue%2D%2D[getMonthValue]()++ (link:{java9-javadoc}/java/time/YearMonth.html#getMonthValue%2D%2D[java 9])
+* ++[[painless-api-reference-YearMonth-getYear-0]]int link:{java8-javadoc}/java/time/YearMonth.html#getYear%2D%2D[getYear]()++ (link:{java9-javadoc}/java/time/YearMonth.html#getYear%2D%2D[java 9])
+* ++[[painless-api-reference-YearMonth-isAfter-1]]boolean link:{java8-javadoc}/java/time/YearMonth.html#isAfter%2Djava.time.YearMonth%2D[isAfter](<<painless-api-reference-YearMonth,YearMonth>>)++ (link:{java9-javadoc}/java/time/YearMonth.html#isAfter%2Djava.time.YearMonth%2D[java 9])
+* ++[[painless-api-reference-YearMonth-isBefore-1]]boolean link:{java8-javadoc}/java/time/YearMonth.html#isBefore%2Djava.time.YearMonth%2D[isBefore](<<painless-api-reference-YearMonth,YearMonth>>)++ (link:{java9-javadoc}/java/time/YearMonth.html#isBefore%2Djava.time.YearMonth%2D[java 9])
+* ++[[painless-api-reference-YearMonth-isLeapYear-0]]boolean link:{java8-javadoc}/java/time/YearMonth.html#isLeapYear%2D%2D[isLeapYear]()++ (link:{java9-javadoc}/java/time/YearMonth.html#isLeapYear%2D%2D[java 9])
+* ++[[painless-api-reference-YearMonth-isValidDay-1]]boolean link:{java8-javadoc}/java/time/YearMonth.html#isValidDay%2Dint%2D[isValidDay](int)++ (link:{java9-javadoc}/java/time/YearMonth.html#isValidDay%2Dint%2D[java 9])
+* ++[[painless-api-reference-YearMonth-lengthOfMonth-0]]int link:{java8-javadoc}/java/time/YearMonth.html#lengthOfMonth%2D%2D[lengthOfMonth]()++ (link:{java9-javadoc}/java/time/YearMonth.html#lengthOfMonth%2D%2D[java 9])
+* ++[[painless-api-reference-YearMonth-lengthOfYear-0]]int link:{java8-javadoc}/java/time/YearMonth.html#lengthOfYear%2D%2D[lengthOfYear]()++ (link:{java9-javadoc}/java/time/YearMonth.html#lengthOfYear%2D%2D[java 9])
+* ++[[painless-api-reference-YearMonth-minus-1]]<<painless-api-reference-YearMonth,YearMonth>> link:{java8-javadoc}/java/time/YearMonth.html#minus%2Djava.time.temporal.TemporalAmount%2D[minus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/YearMonth.html#minus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-YearMonth-minus-2]]<<painless-api-reference-YearMonth,YearMonth>> link:{java8-javadoc}/java/time/YearMonth.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[minus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/YearMonth.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-YearMonth-minusMonths-1]]<<painless-api-reference-YearMonth,YearMonth>> link:{java8-javadoc}/java/time/YearMonth.html#minusMonths%2Dlong%2D[minusMonths](long)++ (link:{java9-javadoc}/java/time/YearMonth.html#minusMonths%2Dlong%2D[java 9])
+* ++[[painless-api-reference-YearMonth-minusYears-1]]<<painless-api-reference-YearMonth,YearMonth>> link:{java8-javadoc}/java/time/YearMonth.html#minusYears%2Dlong%2D[minusYears](long)++ (link:{java9-javadoc}/java/time/YearMonth.html#minusYears%2Dlong%2D[java 9])
+* ++[[painless-api-reference-YearMonth-plus-1]]<<painless-api-reference-YearMonth,YearMonth>> link:{java8-javadoc}/java/time/YearMonth.html#plus%2Djava.time.temporal.TemporalAmount%2D[plus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/YearMonth.html#plus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-YearMonth-plus-2]]<<painless-api-reference-YearMonth,YearMonth>> link:{java8-javadoc}/java/time/YearMonth.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[plus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/YearMonth.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-YearMonth-plusMonths-1]]<<painless-api-reference-YearMonth,YearMonth>> link:{java8-javadoc}/java/time/YearMonth.html#plusMonths%2Dlong%2D[plusMonths](long)++ (link:{java9-javadoc}/java/time/YearMonth.html#plusMonths%2Dlong%2D[java 9])
+* ++[[painless-api-reference-YearMonth-plusYears-1]]<<painless-api-reference-YearMonth,YearMonth>> link:{java8-javadoc}/java/time/YearMonth.html#plusYears%2Dlong%2D[plusYears](long)++ (link:{java9-javadoc}/java/time/YearMonth.html#plusYears%2Dlong%2D[java 9])
+* ++[[painless-api-reference-YearMonth-with-1]]<<painless-api-reference-YearMonth,YearMonth>> link:{java8-javadoc}/java/time/YearMonth.html#with%2Djava.time.temporal.TemporalAdjuster%2D[with](<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>)++ (link:{java9-javadoc}/java/time/YearMonth.html#with%2Djava.time.temporal.TemporalAdjuster%2D[java 9])
+* ++[[painless-api-reference-YearMonth-with-2]]<<painless-api-reference-YearMonth,YearMonth>> link:{java8-javadoc}/java/time/YearMonth.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[with](<<painless-api-reference-TemporalField,TemporalField>>, long)++ (link:{java9-javadoc}/java/time/YearMonth.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[java 9])
+* ++[[painless-api-reference-YearMonth-withMonth-1]]<<painless-api-reference-YearMonth,YearMonth>> link:{java8-javadoc}/java/time/YearMonth.html#withMonth%2Dint%2D[withMonth](int)++ (link:{java9-javadoc}/java/time/YearMonth.html#withMonth%2Dint%2D[java 9])
+* ++[[painless-api-reference-YearMonth-withYear-1]]<<painless-api-reference-YearMonth,YearMonth>> link:{java8-javadoc}/java/time/YearMonth.html#withYear%2Dint%2D[withYear](int)++ (link:{java9-javadoc}/java/time/YearMonth.html#withYear%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-Temporal,Temporal>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++, ++<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>++
diff --git a/docs/painless/painless-api-reference/ZoneId.asciidoc b/docs/painless/painless-api-reference/ZoneId.asciidoc
new file mode 100644
index 0000000000000..fe31c9df00bb1
--- /dev/null
+++ b/docs/painless/painless-api-reference/ZoneId.asciidoc
@@ -0,0 +1,18 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ZoneId]]++ZoneId++::
+** [[painless-api-reference-ZoneId-SHORT_IDS]]static <<painless-api-reference-Map,Map>> link:{java8-javadoc}/java/time/ZoneId.html#SHORT_IDS[SHORT_IDS] (link:{java9-javadoc}/java/time/ZoneId.html#SHORT_IDS[java 9])
+* ++[[painless-api-reference-ZoneId-from-1]]static <<painless-api-reference-ZoneId,ZoneId>> link:{java8-javadoc}/java/time/ZoneId.html#from%2Djava.time.temporal.TemporalAccessor%2D[from](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/ZoneId.html#from%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-ZoneId-getAvailableZoneIds-0]]static <<painless-api-reference-Set,Set>> link:{java8-javadoc}/java/time/ZoneId.html#getAvailableZoneIds%2D%2D[getAvailableZoneIds]()++ (link:{java9-javadoc}/java/time/ZoneId.html#getAvailableZoneIds%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneId-of-1]]static <<painless-api-reference-ZoneId,ZoneId>> link:{java8-javadoc}/java/time/ZoneId.html#of%2Djava.lang.String%2D[of](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/ZoneId.html#of%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-ZoneId-of-2]]static <<painless-api-reference-ZoneId,ZoneId>> link:{java8-javadoc}/java/time/ZoneId.html#of%2Djava.lang.String%2Djava.util.Map%2D[of](<<painless-api-reference-String,String>>, <<painless-api-reference-Map,Map>>)++ (link:{java9-javadoc}/java/time/ZoneId.html#of%2Djava.lang.String%2Djava.util.Map%2D[java 9])
+* ++[[painless-api-reference-ZoneId-ofOffset-2]]static <<painless-api-reference-ZoneId,ZoneId>> link:{java8-javadoc}/java/time/ZoneId.html#ofOffset%2Djava.lang.String%2Djava.time.ZoneOffset%2D[ofOffset](<<painless-api-reference-String,String>>, <<painless-api-reference-ZoneOffset,ZoneOffset>>)++ (link:{java9-javadoc}/java/time/ZoneId.html#ofOffset%2Djava.lang.String%2Djava.time.ZoneOffset%2D[java 9])
+* ++[[painless-api-reference-ZoneId-systemDefault-0]]static <<painless-api-reference-ZoneId,ZoneId>> link:{java8-javadoc}/java/time/ZoneId.html#systemDefault%2D%2D[systemDefault]()++ (link:{java9-javadoc}/java/time/ZoneId.html#systemDefault%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneId-getDisplayName-2]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/ZoneId.html#getDisplayName%2Djava.time.format.TextStyle%2Djava.util.Locale%2D[getDisplayName](<<painless-api-reference-TextStyle,TextStyle>>, <<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/time/ZoneId.html#getDisplayName%2Djava.time.format.TextStyle%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-ZoneId-getId-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/ZoneId.html#getId%2D%2D[getId]()++ (link:{java9-javadoc}/java/time/ZoneId.html#getId%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneId-getRules-0]]<<painless-api-reference-ZoneRules,ZoneRules>> link:{java8-javadoc}/java/time/ZoneId.html#getRules%2D%2D[getRules]()++ (link:{java9-javadoc}/java/time/ZoneId.html#getRules%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneId-normalized-0]]<<painless-api-reference-ZoneId,ZoneId>> link:{java8-javadoc}/java/time/ZoneId.html#normalized%2D%2D[normalized]()++ (link:{java9-javadoc}/java/time/ZoneId.html#normalized%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/ZoneOffset.asciidoc b/docs/painless/painless-api-reference/ZoneOffset.asciidoc
new file mode 100644
index 0000000000000..84e8530f5938b
--- /dev/null
+++ b/docs/painless/painless-api-reference/ZoneOffset.asciidoc
@@ -0,0 +1,17 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ZoneOffset]]++ZoneOffset++::
+** [[painless-api-reference-ZoneOffset-MAX]]static <<painless-api-reference-ZoneOffset,ZoneOffset>> link:{java8-javadoc}/java/time/ZoneOffset.html#MAX[MAX] (link:{java9-javadoc}/java/time/ZoneOffset.html#MAX[java 9])
+** [[painless-api-reference-ZoneOffset-MIN]]static <<painless-api-reference-ZoneOffset,ZoneOffset>> link:{java8-javadoc}/java/time/ZoneOffset.html#MIN[MIN] (link:{java9-javadoc}/java/time/ZoneOffset.html#MIN[java 9])
+** [[painless-api-reference-ZoneOffset-UTC]]static <<painless-api-reference-ZoneOffset,ZoneOffset>> link:{java8-javadoc}/java/time/ZoneOffset.html#UTC[UTC] (link:{java9-javadoc}/java/time/ZoneOffset.html#UTC[java 9])
+* ++[[painless-api-reference-ZoneOffset-from-1]]static <<painless-api-reference-ZoneOffset,ZoneOffset>> link:{java8-javadoc}/java/time/ZoneOffset.html#from%2Djava.time.temporal.TemporalAccessor%2D[from](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/ZoneOffset.html#from%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-ZoneOffset-of-1]]static <<painless-api-reference-ZoneOffset,ZoneOffset>> link:{java8-javadoc}/java/time/ZoneOffset.html#of%2Djava.lang.String%2D[of](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/ZoneOffset.html#of%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-ZoneOffset-ofHours-1]]static <<painless-api-reference-ZoneOffset,ZoneOffset>> link:{java8-javadoc}/java/time/ZoneOffset.html#ofHours%2Dint%2D[ofHours](int)++ (link:{java9-javadoc}/java/time/ZoneOffset.html#ofHours%2Dint%2D[java 9])
+* ++[[painless-api-reference-ZoneOffset-ofHoursMinutes-2]]static <<painless-api-reference-ZoneOffset,ZoneOffset>> link:{java8-javadoc}/java/time/ZoneOffset.html#ofHoursMinutes%2Dint%2Dint%2D[ofHoursMinutes](int, int)++ (link:{java9-javadoc}/java/time/ZoneOffset.html#ofHoursMinutes%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-ZoneOffset-ofHoursMinutesSeconds-3]]static <<painless-api-reference-ZoneOffset,ZoneOffset>> link:{java8-javadoc}/java/time/ZoneOffset.html#ofHoursMinutesSeconds%2Dint%2Dint%2Dint%2D[ofHoursMinutesSeconds](int, int, int)++ (link:{java9-javadoc}/java/time/ZoneOffset.html#ofHoursMinutesSeconds%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-ZoneOffset-ofTotalSeconds-1]]static <<painless-api-reference-ZoneOffset,ZoneOffset>> link:{java8-javadoc}/java/time/ZoneOffset.html#ofTotalSeconds%2Dint%2D[ofTotalSeconds](int)++ (link:{java9-javadoc}/java/time/ZoneOffset.html#ofTotalSeconds%2Dint%2D[java 9])
+* ++[[painless-api-reference-ZoneOffset-getTotalSeconds-0]]int link:{java8-javadoc}/java/time/ZoneOffset.html#getTotalSeconds%2D%2D[getTotalSeconds]()++ (link:{java9-javadoc}/java/time/ZoneOffset.html#getTotalSeconds%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-ZoneId,ZoneId>>++
diff --git a/docs/painless/painless-api-reference/ZoneOffsetTransition.asciidoc b/docs/painless/painless-api-reference/ZoneOffsetTransition.asciidoc
new file mode 100644
index 0000000000000..43be4f56ee4b3
--- /dev/null
+++ b/docs/painless/painless-api-reference/ZoneOffsetTransition.asciidoc
@@ -0,0 +1,19 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ZoneOffsetTransition]]++ZoneOffsetTransition++::
+* ++[[painless-api-reference-ZoneOffsetTransition-of-3]]static <<painless-api-reference-ZoneOffsetTransition,ZoneOffsetTransition>> link:{java8-javadoc}/java/time/zone/ZoneOffsetTransition.html#of%2Djava.time.LocalDateTime%2Djava.time.ZoneOffset%2Djava.time.ZoneOffset%2D[of](<<painless-api-reference-LocalDateTime,LocalDateTime>>, <<painless-api-reference-ZoneOffset,ZoneOffset>>, <<painless-api-reference-ZoneOffset,ZoneOffset>>)++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransition.html#of%2Djava.time.LocalDateTime%2Djava.time.ZoneOffset%2Djava.time.ZoneOffset%2D[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransition-compareTo-1]]int link:{java8-javadoc}/java/time/zone/ZoneOffsetTransition.html#compareTo%2Djava.time.zone.ZoneOffsetTransition%2D[compareTo](<<painless-api-reference-ZoneOffsetTransition,ZoneOffsetTransition>>)++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransition.html#compareTo%2Djava.time.zone.ZoneOffsetTransition%2D[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransition-getDateTimeAfter-0]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/zone/ZoneOffsetTransition.html#getDateTimeAfter%2D%2D[getDateTimeAfter]()++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransition.html#getDateTimeAfter%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransition-getDateTimeBefore-0]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/zone/ZoneOffsetTransition.html#getDateTimeBefore%2D%2D[getDateTimeBefore]()++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransition.html#getDateTimeBefore%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransition-getDuration-0]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/zone/ZoneOffsetTransition.html#getDuration%2D%2D[getDuration]()++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransition.html#getDuration%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransition-getInstant-0]]<<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/zone/ZoneOffsetTransition.html#getInstant%2D%2D[getInstant]()++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransition.html#getInstant%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransition-getOffsetAfter-0]]<<painless-api-reference-ZoneOffset,ZoneOffset>> link:{java8-javadoc}/java/time/zone/ZoneOffsetTransition.html#getOffsetAfter%2D%2D[getOffsetAfter]()++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransition.html#getOffsetAfter%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransition-getOffsetBefore-0]]<<painless-api-reference-ZoneOffset,ZoneOffset>> link:{java8-javadoc}/java/time/zone/ZoneOffsetTransition.html#getOffsetBefore%2D%2D[getOffsetBefore]()++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransition.html#getOffsetBefore%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransition-isGap-0]]boolean link:{java8-javadoc}/java/time/zone/ZoneOffsetTransition.html#isGap%2D%2D[isGap]()++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransition.html#isGap%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransition-isOverlap-0]]boolean link:{java8-javadoc}/java/time/zone/ZoneOffsetTransition.html#isOverlap%2D%2D[isOverlap]()++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransition.html#isOverlap%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransition-isValidOffset-1]]boolean link:{java8-javadoc}/java/time/zone/ZoneOffsetTransition.html#isValidOffset%2Djava.time.ZoneOffset%2D[isValidOffset](<<painless-api-reference-ZoneOffset,ZoneOffset>>)++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransition.html#isValidOffset%2Djava.time.ZoneOffset%2D[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransition-toEpochSecond-0]]long link:{java8-javadoc}/java/time/zone/ZoneOffsetTransition.html#toEpochSecond%2D%2D[toEpochSecond]()++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransition.html#toEpochSecond%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/ZoneOffsetTransitionRule.TimeDefinition.asciidoc b/docs/painless/painless-api-reference/ZoneOffsetTransitionRule.TimeDefinition.asciidoc
new file mode 100644
index 0000000000000..045f2a0188948
--- /dev/null
+++ b/docs/painless/painless-api-reference/ZoneOffsetTransitionRule.TimeDefinition.asciidoc
@@ -0,0 +1,13 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ZoneOffsetTransitionRule-TimeDefinition]]++ZoneOffsetTransitionRule.TimeDefinition++::
+** [[painless-api-reference-ZoneOffsetTransitionRule-TimeDefinition-STANDARD]]static <<painless-api-reference-ZoneOffsetTransitionRule-TimeDefinition,ZoneOffsetTransitionRule.TimeDefinition>> link:{java8-javadoc}/java/time/zone/ZoneOffsetTransitionRule.TimeDefinition.html#STANDARD[STANDARD] (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransitionRule.TimeDefinition.html#STANDARD[java 9])
+** [[painless-api-reference-ZoneOffsetTransitionRule-TimeDefinition-UTC]]static <<painless-api-reference-ZoneOffsetTransitionRule-TimeDefinition,ZoneOffsetTransitionRule.TimeDefinition>> link:{java8-javadoc}/java/time/zone/ZoneOffsetTransitionRule.TimeDefinition.html#UTC[UTC] (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransitionRule.TimeDefinition.html#UTC[java 9])
+** [[painless-api-reference-ZoneOffsetTransitionRule-TimeDefinition-WALL]]static <<painless-api-reference-ZoneOffsetTransitionRule-TimeDefinition,ZoneOffsetTransitionRule.TimeDefinition>> link:{java8-javadoc}/java/time/zone/ZoneOffsetTransitionRule.TimeDefinition.html#WALL[WALL] (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransitionRule.TimeDefinition.html#WALL[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransitionRule-TimeDefinition-valueOf-1]]static <<painless-api-reference-ZoneOffsetTransitionRule-TimeDefinition,ZoneOffsetTransitionRule.TimeDefinition>> link:{java8-javadoc}/java/time/zone/ZoneOffsetTransitionRule.TimeDefinition.html#valueOf%2Djava.lang.String%2D[valueOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransitionRule.TimeDefinition.html#valueOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransitionRule-TimeDefinition-values-0]]static <<painless-api-reference-ZoneOffsetTransitionRule-TimeDefinition,ZoneOffsetTransitionRule.TimeDefinition>>[] link:{java8-javadoc}/java/time/zone/ZoneOffsetTransitionRule.TimeDefinition.html#values%2D%2D[values]()++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransitionRule.TimeDefinition.html#values%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransitionRule-TimeDefinition-createDateTime-3]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/zone/ZoneOffsetTransitionRule.TimeDefinition.html#createDateTime%2Djava.time.LocalDateTime%2Djava.time.ZoneOffset%2Djava.time.ZoneOffset%2D[createDateTime](<<painless-api-reference-LocalDateTime,LocalDateTime>>, <<painless-api-reference-ZoneOffset,ZoneOffset>>, <<painless-api-reference-ZoneOffset,ZoneOffset>>)++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransitionRule.TimeDefinition.html#createDateTime%2Djava.time.LocalDateTime%2Djava.time.ZoneOffset%2Djava.time.ZoneOffset%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Enum,Enum>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/ZoneOffsetTransitionRule.asciidoc b/docs/painless/painless-api-reference/ZoneOffsetTransitionRule.asciidoc
new file mode 100644
index 0000000000000..8e84bff01722b
--- /dev/null
+++ b/docs/painless/painless-api-reference/ZoneOffsetTransitionRule.asciidoc
@@ -0,0 +1,18 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ZoneOffsetTransitionRule]]++ZoneOffsetTransitionRule++::
+* ++[[painless-api-reference-ZoneOffsetTransitionRule-of-9]]static <<painless-api-reference-ZoneOffsetTransitionRule,ZoneOffsetTransitionRule>> link:{java8-javadoc}/java/time/zone/ZoneOffsetTransitionRule.html#of%2Djava.time.Month%2Dint%2Djava.time.DayOfWeek%2Djava.time.LocalTime%2Dboolean%2Djava.time.zone.ZoneOffsetTransitionRule$TimeDefinition%2Djava.time.ZoneOffset%2Djava.time.ZoneOffset%2Djava.time.ZoneOffset%2D[of](<<painless-api-reference-Month,Month>>, int, <<painless-api-reference-DayOfWeek,DayOfWeek>>, <<painless-api-reference-LocalTime,LocalTime>>, boolean, <<painless-api-reference-ZoneOffsetTransitionRule-TimeDefinition,ZoneOffsetTransitionRule.TimeDefinition>>, <<painless-api-reference-ZoneOffset,ZoneOffset>>, <<painless-api-reference-ZoneOffset,ZoneOffset>>, <<painless-api-reference-ZoneOffset,ZoneOffset>>)++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransitionRule.html#of%2Djava.time.Month%2Dint%2Djava.time.DayOfWeek%2Djava.time.LocalTime%2Dboolean%2Djava.time.zone.ZoneOffsetTransitionRule$TimeDefinition%2Djava.time.ZoneOffset%2Djava.time.ZoneOffset%2Djava.time.ZoneOffset%2D[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransitionRule-createTransition-1]]<<painless-api-reference-ZoneOffsetTransition,ZoneOffsetTransition>> link:{java8-javadoc}/java/time/zone/ZoneOffsetTransitionRule.html#createTransition%2Dint%2D[createTransition](int)++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransitionRule.html#createTransition%2Dint%2D[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransitionRule-getDayOfMonthIndicator-0]]int link:{java8-javadoc}/java/time/zone/ZoneOffsetTransitionRule.html#getDayOfMonthIndicator%2D%2D[getDayOfMonthIndicator]()++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransitionRule.html#getDayOfMonthIndicator%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransitionRule-getDayOfWeek-0]]<<painless-api-reference-DayOfWeek,DayOfWeek>> link:{java8-javadoc}/java/time/zone/ZoneOffsetTransitionRule.html#getDayOfWeek%2D%2D[getDayOfWeek]()++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransitionRule.html#getDayOfWeek%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransitionRule-getLocalTime-0]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/zone/ZoneOffsetTransitionRule.html#getLocalTime%2D%2D[getLocalTime]()++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransitionRule.html#getLocalTime%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransitionRule-getMonth-0]]<<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/zone/ZoneOffsetTransitionRule.html#getMonth%2D%2D[getMonth]()++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransitionRule.html#getMonth%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransitionRule-getOffsetAfter-0]]<<painless-api-reference-ZoneOffset,ZoneOffset>> link:{java8-javadoc}/java/time/zone/ZoneOffsetTransitionRule.html#getOffsetAfter%2D%2D[getOffsetAfter]()++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransitionRule.html#getOffsetAfter%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransitionRule-getOffsetBefore-0]]<<painless-api-reference-ZoneOffset,ZoneOffset>> link:{java8-javadoc}/java/time/zone/ZoneOffsetTransitionRule.html#getOffsetBefore%2D%2D[getOffsetBefore]()++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransitionRule.html#getOffsetBefore%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransitionRule-getStandardOffset-0]]<<painless-api-reference-ZoneOffset,ZoneOffset>> link:{java8-javadoc}/java/time/zone/ZoneOffsetTransitionRule.html#getStandardOffset%2D%2D[getStandardOffset]()++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransitionRule.html#getStandardOffset%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransitionRule-getTimeDefinition-0]]<<painless-api-reference-ZoneOffsetTransitionRule-TimeDefinition,ZoneOffsetTransitionRule.TimeDefinition>> link:{java8-javadoc}/java/time/zone/ZoneOffsetTransitionRule.html#getTimeDefinition%2D%2D[getTimeDefinition]()++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransitionRule.html#getTimeDefinition%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransitionRule-isMidnightEndOfDay-0]]boolean link:{java8-javadoc}/java/time/zone/ZoneOffsetTransitionRule.html#isMidnightEndOfDay%2D%2D[isMidnightEndOfDay]()++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransitionRule.html#isMidnightEndOfDay%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/ZoneRules.asciidoc b/docs/painless/painless-api-reference/ZoneRules.asciidoc
new file mode 100644
index 0000000000000..6c3104a7e02ab
--- /dev/null
+++ b/docs/painless/painless-api-reference/ZoneRules.asciidoc
@@ -0,0 +1,21 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ZoneRules]]++ZoneRules++::
+* ++[[painless-api-reference-ZoneRules-of-1]]static <<painless-api-reference-ZoneRules,ZoneRules>> link:{java8-javadoc}/java/time/zone/ZoneRules.html#of%2Djava.time.ZoneOffset%2D[of](<<painless-api-reference-ZoneOffset,ZoneOffset>>)++ (link:{java9-javadoc}/java/time/zone/ZoneRules.html#of%2Djava.time.ZoneOffset%2D[java 9])
+* ++[[painless-api-reference-ZoneRules-of-5]]static <<painless-api-reference-ZoneRules,ZoneRules>> link:{java8-javadoc}/java/time/zone/ZoneRules.html#of%2Djava.time.ZoneOffset%2Djava.time.ZoneOffset%2Djava.util.List%2Djava.util.List%2Djava.util.List%2D[of](<<painless-api-reference-ZoneOffset,ZoneOffset>>, <<painless-api-reference-ZoneOffset,ZoneOffset>>, <<painless-api-reference-List,List>>, <<painless-api-reference-List,List>>, <<painless-api-reference-List,List>>)++ (link:{java9-javadoc}/java/time/zone/ZoneRules.html#of%2Djava.time.ZoneOffset%2Djava.time.ZoneOffset%2Djava.util.List%2Djava.util.List%2Djava.util.List%2D[java 9])
+* ++[[painless-api-reference-ZoneRules-getDaylightSavings-1]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/zone/ZoneRules.html#getDaylightSavings%2Djava.time.Instant%2D[getDaylightSavings](<<painless-api-reference-Instant,Instant>>)++ (link:{java9-javadoc}/java/time/zone/ZoneRules.html#getDaylightSavings%2Djava.time.Instant%2D[java 9])
+* ++[[painless-api-reference-ZoneRules-getOffset-1]]<<painless-api-reference-ZoneOffset,ZoneOffset>> link:{java8-javadoc}/java/time/zone/ZoneRules.html#getOffset%2Djava.time.Instant%2D[getOffset](<<painless-api-reference-Instant,Instant>>)++ (link:{java9-javadoc}/java/time/zone/ZoneRules.html#getOffset%2Djava.time.Instant%2D[java 9])
+* ++[[painless-api-reference-ZoneRules-getStandardOffset-1]]<<painless-api-reference-ZoneOffset,ZoneOffset>> link:{java8-javadoc}/java/time/zone/ZoneRules.html#getStandardOffset%2Djava.time.Instant%2D[getStandardOffset](<<painless-api-reference-Instant,Instant>>)++ (link:{java9-javadoc}/java/time/zone/ZoneRules.html#getStandardOffset%2Djava.time.Instant%2D[java 9])
+* ++[[painless-api-reference-ZoneRules-getTransition-1]]<<painless-api-reference-ZoneOffsetTransition,ZoneOffsetTransition>> link:{java8-javadoc}/java/time/zone/ZoneRules.html#getTransition%2Djava.time.LocalDateTime%2D[getTransition](<<painless-api-reference-LocalDateTime,LocalDateTime>>)++ (link:{java9-javadoc}/java/time/zone/ZoneRules.html#getTransition%2Djava.time.LocalDateTime%2D[java 9])
+* ++[[painless-api-reference-ZoneRules-getTransitionRules-0]]<<painless-api-reference-List,List>> link:{java8-javadoc}/java/time/zone/ZoneRules.html#getTransitionRules%2D%2D[getTransitionRules]()++ (link:{java9-javadoc}/java/time/zone/ZoneRules.html#getTransitionRules%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneRules-getTransitions-0]]<<painless-api-reference-List,List>> link:{java8-javadoc}/java/time/zone/ZoneRules.html#getTransitions%2D%2D[getTransitions]()++ (link:{java9-javadoc}/java/time/zone/ZoneRules.html#getTransitions%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneRules-getValidOffsets-1]]<<painless-api-reference-List,List>> link:{java8-javadoc}/java/time/zone/ZoneRules.html#getValidOffsets%2Djava.time.LocalDateTime%2D[getValidOffsets](<<painless-api-reference-LocalDateTime,LocalDateTime>>)++ (link:{java9-javadoc}/java/time/zone/ZoneRules.html#getValidOffsets%2Djava.time.LocalDateTime%2D[java 9])
+* ++[[painless-api-reference-ZoneRules-isDaylightSavings-1]]boolean link:{java8-javadoc}/java/time/zone/ZoneRules.html#isDaylightSavings%2Djava.time.Instant%2D[isDaylightSavings](<<painless-api-reference-Instant,Instant>>)++ (link:{java9-javadoc}/java/time/zone/ZoneRules.html#isDaylightSavings%2Djava.time.Instant%2D[java 9])
+* ++[[painless-api-reference-ZoneRules-isFixedOffset-0]]boolean link:{java8-javadoc}/java/time/zone/ZoneRules.html#isFixedOffset%2D%2D[isFixedOffset]()++ (link:{java9-javadoc}/java/time/zone/ZoneRules.html#isFixedOffset%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneRules-isValidOffset-2]]boolean link:{java8-javadoc}/java/time/zone/ZoneRules.html#isValidOffset%2Djava.time.LocalDateTime%2Djava.time.ZoneOffset%2D[isValidOffset](<<painless-api-reference-LocalDateTime,LocalDateTime>>, <<painless-api-reference-ZoneOffset,ZoneOffset>>)++ (link:{java9-javadoc}/java/time/zone/ZoneRules.html#isValidOffset%2Djava.time.LocalDateTime%2Djava.time.ZoneOffset%2D[java 9])
+* ++[[painless-api-reference-ZoneRules-nextTransition-1]]<<painless-api-reference-ZoneOffsetTransition,ZoneOffsetTransition>> link:{java8-javadoc}/java/time/zone/ZoneRules.html#nextTransition%2Djava.time.Instant%2D[nextTransition](<<painless-api-reference-Instant,Instant>>)++ (link:{java9-javadoc}/java/time/zone/ZoneRules.html#nextTransition%2Djava.time.Instant%2D[java 9])
+* ++[[painless-api-reference-ZoneRules-previousTransition-1]]<<painless-api-reference-ZoneOffsetTransition,ZoneOffsetTransition>> link:{java8-javadoc}/java/time/zone/ZoneRules.html#previousTransition%2Djava.time.Instant%2D[previousTransition](<<painless-api-reference-Instant,Instant>>)++ (link:{java9-javadoc}/java/time/zone/ZoneRules.html#previousTransition%2Djava.time.Instant%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/ZoneRulesException.asciidoc b/docs/painless/painless-api-reference/ZoneRulesException.asciidoc
new file mode 100644
index 0000000000000..8bb3ef0cc98b0
--- /dev/null
+++ b/docs/painless/painless-api-reference/ZoneRulesException.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ZoneRulesException]]++ZoneRulesException++::
+* ++[[painless-api-reference-ZoneRulesException-ZoneRulesException-1]]link:{java8-javadoc}/java/time/zone/ZoneRulesException.html#ZoneRulesException%2Djava.lang.String%2D[ZoneRulesException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/zone/ZoneRulesException.html#ZoneRulesException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/ZoneRulesProvider.asciidoc b/docs/painless/painless-api-reference/ZoneRulesProvider.asciidoc
new file mode 100644
index 0000000000000..be33f691f370e
--- /dev/null
+++ b/docs/painless/painless-api-reference/ZoneRulesProvider.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ZoneRulesProvider]]++ZoneRulesProvider++::
+* ++[[painless-api-reference-ZoneRulesProvider-getAvailableZoneIds-0]]static <<painless-api-reference-Set,Set>> link:{java8-javadoc}/java/time/zone/ZoneRulesProvider.html#getAvailableZoneIds%2D%2D[getAvailableZoneIds]()++ (link:{java9-javadoc}/java/time/zone/ZoneRulesProvider.html#getAvailableZoneIds%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneRulesProvider-getRules-2]]static <<painless-api-reference-ZoneRules,ZoneRules>> link:{java8-javadoc}/java/time/zone/ZoneRulesProvider.html#getRules%2Djava.lang.String%2Dboolean%2D[getRules](<<painless-api-reference-String,String>>, boolean)++ (link:{java9-javadoc}/java/time/zone/ZoneRulesProvider.html#getRules%2Djava.lang.String%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-ZoneRulesProvider-getVersions-1]]static <<painless-api-reference-NavigableMap,NavigableMap>> link:{java8-javadoc}/java/time/zone/ZoneRulesProvider.html#getVersions%2Djava.lang.String%2D[getVersions](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/zone/ZoneRulesProvider.html#getVersions%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/ZonedDateTime.asciidoc b/docs/painless/painless-api-reference/ZonedDateTime.asciidoc
new file mode 100644
index 0000000000000..e8ba7501c6c20
--- /dev/null
+++ b/docs/painless/painless-api-reference/ZonedDateTime.asciidoc
@@ -0,0 +1,66 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ZonedDateTime]]++ZonedDateTime++::
+* ++[[painless-api-reference-ZonedDateTime-from-1]]static <<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#from%2Djava.time.temporal.TemporalAccessor%2D[from](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#from%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-of-2]]static <<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#of%2Djava.time.LocalDateTime%2Djava.time.ZoneId%2D[of](<<painless-api-reference-LocalDateTime,LocalDateTime>>, <<painless-api-reference-ZoneId,ZoneId>>)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#of%2Djava.time.LocalDateTime%2Djava.time.ZoneId%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-of-3]]static <<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#of%2Djava.time.LocalDate%2Djava.time.LocalTime%2Djava.time.ZoneId%2D[of](<<painless-api-reference-LocalDate,LocalDate>>, <<painless-api-reference-LocalTime,LocalTime>>, <<painless-api-reference-ZoneId,ZoneId>>)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#of%2Djava.time.LocalDate%2Djava.time.LocalTime%2Djava.time.ZoneId%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-of-8]]static <<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#of%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Djava.time.ZoneId%2D[of](int, int, int, int, int, int, int, <<painless-api-reference-ZoneId,ZoneId>>)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#of%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Djava.time.ZoneId%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-ofInstant-2]]static <<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#ofInstant%2Djava.time.Instant%2Djava.time.ZoneId%2D[ofInstant](<<painless-api-reference-Instant,Instant>>, <<painless-api-reference-ZoneId,ZoneId>>)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#ofInstant%2Djava.time.Instant%2Djava.time.ZoneId%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-ofInstant-3]]static <<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#ofInstant%2Djava.time.LocalDateTime%2Djava.time.ZoneOffset%2Djava.time.ZoneId%2D[ofInstant](<<painless-api-reference-LocalDateTime,LocalDateTime>>, <<painless-api-reference-ZoneOffset,ZoneOffset>>, <<painless-api-reference-ZoneId,ZoneId>>)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#ofInstant%2Djava.time.LocalDateTime%2Djava.time.ZoneOffset%2Djava.time.ZoneId%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-ofLocal-3]]static <<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#ofLocal%2Djava.time.LocalDateTime%2Djava.time.ZoneId%2Djava.time.ZoneOffset%2D[ofLocal](<<painless-api-reference-LocalDateTime,LocalDateTime>>, <<painless-api-reference-ZoneId,ZoneId>>, <<painless-api-reference-ZoneOffset,ZoneOffset>>)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#ofLocal%2Djava.time.LocalDateTime%2Djava.time.ZoneId%2Djava.time.ZoneOffset%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-ofStrict-3]]static <<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#ofStrict%2Djava.time.LocalDateTime%2Djava.time.ZoneOffset%2Djava.time.ZoneId%2D[ofStrict](<<painless-api-reference-LocalDateTime,LocalDateTime>>, <<painless-api-reference-ZoneOffset,ZoneOffset>>, <<painless-api-reference-ZoneId,ZoneId>>)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#ofStrict%2Djava.time.LocalDateTime%2Djava.time.ZoneOffset%2Djava.time.ZoneId%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-parse-1]]static <<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#parse%2Djava.lang.CharSequence%2D[parse](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#parse%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-parse-2]]static <<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#parse%2Djava.lang.CharSequence%2Djava.time.format.DateTimeFormatter%2D[parse](<<painless-api-reference-CharSequence,CharSequence>>, <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>>)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#parse%2Djava.lang.CharSequence%2Djava.time.format.DateTimeFormatter%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-getDayOfMonth-0]]int link:{java8-javadoc}/java/time/ZonedDateTime.html#getDayOfMonth%2D%2D[getDayOfMonth]()++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#getDayOfMonth%2D%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-getDayOfWeek-0]]<<painless-api-reference-DayOfWeek,DayOfWeek>> link:{java8-javadoc}/java/time/ZonedDateTime.html#getDayOfWeek%2D%2D[getDayOfWeek]()++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#getDayOfWeek%2D%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-getDayOfYear-0]]int link:{java8-javadoc}/java/time/ZonedDateTime.html#getDayOfYear%2D%2D[getDayOfYear]()++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#getDayOfYear%2D%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-getHour-0]]int link:{java8-javadoc}/java/time/ZonedDateTime.html#getHour%2D%2D[getHour]()++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#getHour%2D%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-getMinute-0]]int link:{java8-javadoc}/java/time/ZonedDateTime.html#getMinute%2D%2D[getMinute]()++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#getMinute%2D%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-getMonth-0]]<<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/ZonedDateTime.html#getMonth%2D%2D[getMonth]()++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#getMonth%2D%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-getMonthValue-0]]int link:{java8-javadoc}/java/time/ZonedDateTime.html#getMonthValue%2D%2D[getMonthValue]()++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#getMonthValue%2D%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-getNano-0]]int link:{java8-javadoc}/java/time/ZonedDateTime.html#getNano%2D%2D[getNano]()++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#getNano%2D%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-getSecond-0]]int link:{java8-javadoc}/java/time/ZonedDateTime.html#getSecond%2D%2D[getSecond]()++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#getSecond%2D%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-getYear-0]]int link:{java8-javadoc}/java/time/ZonedDateTime.html#getYear%2D%2D[getYear]()++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#getYear%2D%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-minus-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#minus%2Djava.time.temporal.TemporalAmount%2D[minus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#minus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-minus-2]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[minus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-minusDays-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#minusDays%2Dlong%2D[minusDays](long)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#minusDays%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-minusHours-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#minusHours%2Dlong%2D[minusHours](long)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#minusHours%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-minusMinutes-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#minusMinutes%2Dlong%2D[minusMinutes](long)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#minusMinutes%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-minusMonths-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#minusMonths%2Dlong%2D[minusMonths](long)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#minusMonths%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-minusNanos-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#minusNanos%2Dlong%2D[minusNanos](long)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#minusNanos%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-minusSeconds-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#minusSeconds%2Dlong%2D[minusSeconds](long)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#minusSeconds%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-minusWeeks-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#minusWeeks%2Dlong%2D[minusWeeks](long)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#minusWeeks%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-minusYears-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#minusYears%2Dlong%2D[minusYears](long)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#minusYears%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-plus-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#plus%2Djava.time.temporal.TemporalAmount%2D[plus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#plus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-plus-2]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[plus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-plusDays-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#plusDays%2Dlong%2D[plusDays](long)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#plusDays%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-plusHours-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#plusHours%2Dlong%2D[plusHours](long)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#plusHours%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-plusMinutes-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#plusMinutes%2Dlong%2D[plusMinutes](long)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#plusMinutes%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-plusMonths-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#plusMonths%2Dlong%2D[plusMonths](long)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#plusMonths%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-plusNanos-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#plusNanos%2Dlong%2D[plusNanos](long)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#plusNanos%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-plusSeconds-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#plusSeconds%2Dlong%2D[plusSeconds](long)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#plusSeconds%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-plusWeeks-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#plusWeeks%2Dlong%2D[plusWeeks](long)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#plusWeeks%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-plusYears-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#plusYears%2Dlong%2D[plusYears](long)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#plusYears%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-toLocalDate-0]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/ZonedDateTime.html#toLocalDate%2D%2D[toLocalDate]()++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#toLocalDate%2D%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-toLocalDateTime-0]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#toLocalDateTime%2D%2D[toLocalDateTime]()++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#toLocalDateTime%2D%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-toOffsetDateTime-0]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#toOffsetDateTime%2D%2D[toOffsetDateTime]()++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#toOffsetDateTime%2D%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-truncatedTo-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#truncatedTo%2Djava.time.temporal.TemporalUnit%2D[truncatedTo](<<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#truncatedTo%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-with-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#with%2Djava.time.temporal.TemporalAdjuster%2D[with](<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#with%2Djava.time.temporal.TemporalAdjuster%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-with-2]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[with](<<painless-api-reference-TemporalField,TemporalField>>, long)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-withDayOfMonth-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#withDayOfMonth%2Dint%2D[withDayOfMonth](int)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#withDayOfMonth%2Dint%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-withDayOfYear-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#withDayOfYear%2Dint%2D[withDayOfYear](int)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#withDayOfYear%2Dint%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-withEarlierOffsetAtOverlap-0]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#withEarlierOffsetAtOverlap%2D%2D[withEarlierOffsetAtOverlap]()++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#withEarlierOffsetAtOverlap%2D%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-withFixedOffsetZone-0]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#withFixedOffsetZone%2D%2D[withFixedOffsetZone]()++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#withFixedOffsetZone%2D%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-withHour-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#withHour%2Dint%2D[withHour](int)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#withHour%2Dint%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-withLaterOffsetAtOverlap-0]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#withLaterOffsetAtOverlap%2D%2D[withLaterOffsetAtOverlap]()++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#withLaterOffsetAtOverlap%2D%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-withMinute-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#withMinute%2Dint%2D[withMinute](int)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#withMinute%2Dint%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-withMonth-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#withMonth%2Dint%2D[withMonth](int)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#withMonth%2Dint%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-withNano-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#withNano%2Dint%2D[withNano](int)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#withNano%2Dint%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-withSecond-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#withSecond%2Dint%2D[withSecond](int)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#withSecond%2Dint%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-withYear-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#withYear%2Dint%2D[withYear](int)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#withYear%2Dint%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-withZoneSameInstant-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#withZoneSameInstant%2Djava.time.ZoneId%2D[withZoneSameInstant](<<painless-api-reference-ZoneId,ZoneId>>)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#withZoneSameInstant%2Djava.time.ZoneId%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-withZoneSameLocal-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#withZoneSameLocal%2Djava.time.ZoneId%2D[withZoneSameLocal](<<painless-api-reference-ZoneId,ZoneId>>)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#withZoneSameLocal%2Djava.time.ZoneId%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-ChronoZonedDateTime,ChronoZonedDateTime>>++, ++<<painless-api-reference-Temporal,Temporal>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++
diff --git a/docs/painless/painless-api-reference/index.asciidoc b/docs/painless/painless-api-reference/index.asciidoc
new file mode 100644
index 0000000000000..e1c5795e3dd37
--- /dev/null
+++ b/docs/painless/painless-api-reference/index.asciidoc
@@ -0,0 +1,339 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+include::AbstractChronology.asciidoc[]
+include::AbstractCollection.asciidoc[]
+include::AbstractList.asciidoc[]
+include::AbstractMap.asciidoc[]
+include::AbstractMap.SimpleEntry.asciidoc[]
+include::AbstractMap.SimpleImmutableEntry.asciidoc[]
+include::AbstractQueue.asciidoc[]
+include::AbstractSequentialList.asciidoc[]
+include::AbstractSet.asciidoc[]
+include::Annotation.asciidoc[]
+include::Appendable.asciidoc[]
+include::ArithmeticException.asciidoc[]
+include::ArrayDeque.asciidoc[]
+include::ArrayIndexOutOfBoundsException.asciidoc[]
+include::ArrayList.asciidoc[]
+include::ArrayStoreException.asciidoc[]
+include::Arrays.asciidoc[]
+include::AttributedCharacterIterator.asciidoc[]
+include::AttributedCharacterIterator.Attribute.asciidoc[]
+include::AttributedString.asciidoc[]
+include::Base64.asciidoc[]
+include::Base64.Decoder.asciidoc[]
+include::Base64.Encoder.asciidoc[]
+include::BaseStream.asciidoc[]
+include::BiConsumer.asciidoc[]
+include::BiFunction.asciidoc[]
+include::BiPredicate.asciidoc[]
+include::Bidi.asciidoc[]
+include::BigDecimal.asciidoc[]
+include::BigInteger.asciidoc[]
+include::BinaryOperator.asciidoc[]
+include::BitSet.asciidoc[]
+include::Boolean.asciidoc[]
+include::BooleanSupplier.asciidoc[]
+include::BreakIterator.asciidoc[]
+include::Byte.asciidoc[]
+include::BytesRef.asciidoc[]
+include::Calendar.asciidoc[]
+include::Calendar.Builder.asciidoc[]
+include::CharSequence.asciidoc[]
+include::Character.asciidoc[]
+include::Character.Subset.asciidoc[]
+include::Character.UnicodeBlock.asciidoc[]
+include::Character.UnicodeScript.asciidoc[]
+include::CharacterIterator.asciidoc[]
+include::ChoiceFormat.asciidoc[]
+include::ChronoField.asciidoc[]
+include::ChronoLocalDate.asciidoc[]
+include::ChronoLocalDateTime.asciidoc[]
+include::ChronoPeriod.asciidoc[]
+include::ChronoUnit.asciidoc[]
+include::ChronoZonedDateTime.asciidoc[]
+include::Chronology.asciidoc[]
+include::ClassCastException.asciidoc[]
+include::ClassNotFoundException.asciidoc[]
+include::Clock.asciidoc[]
+include::CloneNotSupportedException.asciidoc[]
+include::CollationElementIterator.asciidoc[]
+include::CollationKey.asciidoc[]
+include::Collator.asciidoc[]
+include::Collection.asciidoc[]
+include::Collections.asciidoc[]
+include::Collector.asciidoc[]
+include::Collector.Characteristics.asciidoc[]
+include::Collectors.asciidoc[]
+include::Comparable.asciidoc[]
+include::Comparator.asciidoc[]
+include::ConcurrentModificationException.asciidoc[]
+include::Consumer.asciidoc[]
+include::Currency.asciidoc[]
+include::Date.asciidoc[]
+include::DateFormat.asciidoc[]
+include::DateFormat.Field.asciidoc[]
+include::DateFormatSymbols.asciidoc[]
+include::DateTimeException.asciidoc[]
+include::DateTimeFormatter.asciidoc[]
+include::DateTimeFormatterBuilder.asciidoc[]
+include::DateTimeParseException.asciidoc[]
+include::DayOfWeek.asciidoc[]
+include::Debug.asciidoc[]
+include::DecimalFormat.asciidoc[]
+include::DecimalFormatSymbols.asciidoc[]
+include::DecimalStyle.asciidoc[]
+include::Deque.asciidoc[]
+include::Dictionary.asciidoc[]
+include::Double.asciidoc[]
+include::DoubleBinaryOperator.asciidoc[]
+include::DoubleConsumer.asciidoc[]
+include::DoubleFunction.asciidoc[]
+include::DoublePredicate.asciidoc[]
+include::DoubleStream.asciidoc[]
+include::DoubleStream.Builder.asciidoc[]
+include::DoubleSummaryStatistics.asciidoc[]
+include::DoubleSupplier.asciidoc[]
+include::DoubleToIntFunction.asciidoc[]
+include::DoubleToLongFunction.asciidoc[]
+include::DoubleUnaryOperator.asciidoc[]
+include::DuplicateFormatFlagsException.asciidoc[]
+include::Duration.asciidoc[]
+include::EmptyStackException.asciidoc[]
+include::Enum.asciidoc[]
+include::EnumConstantNotPresentException.asciidoc[]
+include::Enumeration.asciidoc[]
+include::Era.asciidoc[]
+include::EventListener.asciidoc[]
+include::EventListenerProxy.asciidoc[]
+include::EventObject.asciidoc[]
+include::Exception.asciidoc[]
+include::FieldPosition.asciidoc[]
+include::Float.asciidoc[]
+include::Format.asciidoc[]
+include::Format.Field.asciidoc[]
+include::FormatFlagsConversionMismatchException.asciidoc[]
+include::FormatStyle.asciidoc[]
+include::Formattable.asciidoc[]
+include::FormattableFlags.asciidoc[]
+include::Formatter.asciidoc[]
+include::Formatter.BigDecimalLayoutForm.asciidoc[]
+include::FormatterClosedException.asciidoc[]
+include::Function.asciidoc[]
+include::GregorianCalendar.asciidoc[]
+include::HashMap.asciidoc[]
+include::HashSet.asciidoc[]
+include::Hashtable.asciidoc[]
+include::HijrahChronology.asciidoc[]
+include::HijrahDate.asciidoc[]
+include::HijrahEra.asciidoc[]
+include::IdentityHashMap.asciidoc[]
+include::IllegalAccessException.asciidoc[]
+include::IllegalArgumentException.asciidoc[]
+include::IllegalFormatCodePointException.asciidoc[]
+include::IllegalFormatConversionException.asciidoc[]
+include::IllegalFormatException.asciidoc[]
+include::IllegalFormatFlagsException.asciidoc[]
+include::IllegalFormatPrecisionException.asciidoc[]
+include::IllegalFormatWidthException.asciidoc[]
+include::IllegalMonitorStateException.asciidoc[]
+include::IllegalStateException.asciidoc[]
+include::IllegalThreadStateException.asciidoc[]
+include::IllformedLocaleException.asciidoc[]
+include::IndexOutOfBoundsException.asciidoc[]
+include::InputMismatchException.asciidoc[]
+include::Instant.asciidoc[]
+include::InstantiationException.asciidoc[]
+include::IntBinaryOperator.asciidoc[]
+include::IntConsumer.asciidoc[]
+include::IntFunction.asciidoc[]
+include::IntPredicate.asciidoc[]
+include::IntStream.asciidoc[]
+include::IntStream.Builder.asciidoc[]
+include::IntSummaryStatistics.asciidoc[]
+include::IntSupplier.asciidoc[]
+include::IntToDoubleFunction.asciidoc[]
+include::IntToLongFunction.asciidoc[]
+include::IntUnaryOperator.asciidoc[]
+include::Integer.asciidoc[]
+include::InterruptedException.asciidoc[]
+include::IsoChronology.asciidoc[]
+include::IsoEra.asciidoc[]
+include::IsoFields.asciidoc[]
+include::Iterable.asciidoc[]
+include::Iterator.asciidoc[]
+include::JapaneseChronology.asciidoc[]
+include::JapaneseDate.asciidoc[]
+include::JapaneseEra.asciidoc[]
+include::JulianFields.asciidoc[]
+include::LinkedHashMap.asciidoc[]
+include::LinkedHashSet.asciidoc[]
+include::LinkedList.asciidoc[]
+include::List.asciidoc[]
+include::ListIterator.asciidoc[]
+include::LocalDate.asciidoc[]
+include::LocalDateTime.asciidoc[]
+include::LocalTime.asciidoc[]
+include::Locale.asciidoc[]
+include::Locale.Builder.asciidoc[]
+include::Locale.Category.asciidoc[]
+include::Locale.FilteringMode.asciidoc[]
+include::Locale.LanguageRange.asciidoc[]
+include::Long.asciidoc[]
+include::LongBinaryOperator.asciidoc[]
+include::LongConsumer.asciidoc[]
+include::LongFunction.asciidoc[]
+include::LongPredicate.asciidoc[]
+include::LongStream.asciidoc[]
+include::LongStream.Builder.asciidoc[]
+include::LongSummaryStatistics.asciidoc[]
+include::LongSupplier.asciidoc[]
+include::LongToDoubleFunction.asciidoc[]
+include::LongToIntFunction.asciidoc[]
+include::LongUnaryOperator.asciidoc[]
+include::Map.asciidoc[]
+include::Map.Entry.asciidoc[]
+include::Matcher.asciidoc[]
+include::Math.asciidoc[]
+include::MathContext.asciidoc[]
+include::MessageFormat.asciidoc[]
+include::MessageFormat.Field.asciidoc[]
+include::MinguoChronology.asciidoc[]
+include::MinguoDate.asciidoc[]
+include::MinguoEra.asciidoc[]
+include::MissingFormatArgumentException.asciidoc[]
+include::MissingFormatWidthException.asciidoc[]
+include::MissingResourceException.asciidoc[]
+include::Month.asciidoc[]
+include::MonthDay.asciidoc[]
+include::NavigableMap.asciidoc[]
+include::NavigableSet.asciidoc[]
+include::NegativeArraySizeException.asciidoc[]
+include::NoSuchElementException.asciidoc[]
+include::NoSuchFieldException.asciidoc[]
+include::NoSuchMethodException.asciidoc[]
+include::Normalizer.asciidoc[]
+include::Normalizer.Form.asciidoc[]
+include::NullPointerException.asciidoc[]
+include::Number.asciidoc[]
+include::NumberFormat.asciidoc[]
+include::NumberFormat.Field.asciidoc[]
+include::NumberFormatException.asciidoc[]
+include::ObjDoubleConsumer.asciidoc[]
+include::ObjIntConsumer.asciidoc[]
+include::ObjLongConsumer.asciidoc[]
+include::Object.asciidoc[]
+include::Objects.asciidoc[]
+include::Observable.asciidoc[]
+include::Observer.asciidoc[]
+include::OffsetDateTime.asciidoc[]
+include::OffsetTime.asciidoc[]
+include::Optional.asciidoc[]
+include::OptionalDouble.asciidoc[]
+include::OptionalInt.asciidoc[]
+include::OptionalLong.asciidoc[]
+include::ParseException.asciidoc[]
+include::ParsePosition.asciidoc[]
+include::Pattern.asciidoc[]
+include::Period.asciidoc[]
+include::Predicate.asciidoc[]
+include::PrimitiveIterator.asciidoc[]
+include::PrimitiveIterator.OfDouble.asciidoc[]
+include::PrimitiveIterator.OfInt.asciidoc[]
+include::PrimitiveIterator.OfLong.asciidoc[]
+include::PriorityQueue.asciidoc[]
+include::Queue.asciidoc[]
+include::Random.asciidoc[]
+include::RandomAccess.asciidoc[]
+include::ReflectiveOperationException.asciidoc[]
+include::ResolverStyle.asciidoc[]
+include::RoundingMode.asciidoc[]
+include::RuleBasedCollator.asciidoc[]
+include::RuntimeException.asciidoc[]
+include::SecurityException.asciidoc[]
+include::Set.asciidoc[]
+include::Short.asciidoc[]
+include::SignStyle.asciidoc[]
+include::SimpleDateFormat.asciidoc[]
+include::SimpleTimeZone.asciidoc[]
+include::SortedMap.asciidoc[]
+include::SortedSet.asciidoc[]
+include::Spliterator.asciidoc[]
+include::Spliterator.OfDouble.asciidoc[]
+include::Spliterator.OfInt.asciidoc[]
+include::Spliterator.OfLong.asciidoc[]
+include::Spliterator.OfPrimitive.asciidoc[]
+include::Spliterators.asciidoc[]
+include::Stack.asciidoc[]
+include::StackTraceElement.asciidoc[]
+include::Stream.asciidoc[]
+include::Stream.Builder.asciidoc[]
+include::StrictMath.asciidoc[]
+include::String.asciidoc[]
+include::StringBuffer.asciidoc[]
+include::StringBuilder.asciidoc[]
+include::StringCharacterIterator.asciidoc[]
+include::StringIndexOutOfBoundsException.asciidoc[]
+include::StringJoiner.asciidoc[]
+include::StringTokenizer.asciidoc[]
+include::Supplier.asciidoc[]
+include::System.asciidoc[]
+include::Temporal.asciidoc[]
+include::TemporalAccessor.asciidoc[]
+include::TemporalAdjuster.asciidoc[]
+include::TemporalAdjusters.asciidoc[]
+include::TemporalAmount.asciidoc[]
+include::TemporalField.asciidoc[]
+include::TemporalQueries.asciidoc[]
+include::TemporalQuery.asciidoc[]
+include::TemporalUnit.asciidoc[]
+include::TextStyle.asciidoc[]
+include::ThaiBuddhistChronology.asciidoc[]
+include::ThaiBuddhistDate.asciidoc[]
+include::ThaiBuddhistEra.asciidoc[]
+include::TimeZone.asciidoc[]
+include::ToDoubleBiFunction.asciidoc[]
+include::ToDoubleFunction.asciidoc[]
+include::ToIntBiFunction.asciidoc[]
+include::ToIntFunction.asciidoc[]
+include::ToLongBiFunction.asciidoc[]
+include::ToLongFunction.asciidoc[]
+include::TooManyListenersException.asciidoc[]
+include::TreeMap.asciidoc[]
+include::TreeSet.asciidoc[]
+include::TypeNotPresentException.asciidoc[]
+include::UUID.asciidoc[]
+include::UnaryOperator.asciidoc[]
+include::UnknownFormatConversionException.asciidoc[]
+include::UnknownFormatFlagsException.asciidoc[]
+include::UnsupportedOperationException.asciidoc[]
+include::UnsupportedTemporalTypeException.asciidoc[]
+include::ValueRange.asciidoc[]
+include::Vector.asciidoc[]
+include::WeekFields.asciidoc[]
+include::Year.asciidoc[]
+include::YearMonth.asciidoc[]
+include::ZoneId.asciidoc[]
+include::ZoneOffset.asciidoc[]
+include::ZoneOffsetTransition.asciidoc[]
+include::ZoneOffsetTransitionRule.asciidoc[]
+include::ZoneOffsetTransitionRule.TimeDefinition.asciidoc[]
+include::ZoneRules.asciidoc[]
+include::ZoneRulesException.asciidoc[]
+include::ZoneRulesProvider.asciidoc[]
+include::ZonedDateTime.asciidoc[]
+include::org.elasticsearch.common.geo.GeoPoint.asciidoc[]
+include::org.elasticsearch.index.fielddata.ScriptDocValues.Booleans.asciidoc[]
+include::org.elasticsearch.index.fielddata.ScriptDocValues.BytesRefs.asciidoc[]
+include::org.elasticsearch.index.fielddata.ScriptDocValues.Doubles.asciidoc[]
+include::org.elasticsearch.index.fielddata.ScriptDocValues.GeoPoints.asciidoc[]
+include::org.elasticsearch.index.fielddata.ScriptDocValues.Longs.asciidoc[]
+include::org.elasticsearch.index.fielddata.ScriptDocValues.Strings.asciidoc[]
+include::org.elasticsearch.index.mapper.IpFieldMapper.IpFieldType.IpScriptDocValues.asciidoc[]
+include::org.elasticsearch.painless.FeatureTest.asciidoc[]
+include::org.elasticsearch.search.lookup.FieldLookup.asciidoc[]
+include::org.joda.time.ReadableDateTime.asciidoc[]
+include::org.joda.time.ReadableInstant.asciidoc[]
diff --git a/docs/painless/painless-api-reference/org.elasticsearch.common.geo.GeoPoint.asciidoc b/docs/painless/painless-api-reference/org.elasticsearch.common.geo.GeoPoint.asciidoc
new file mode 100644
index 0000000000000..2a593d0ec6314
--- /dev/null
+++ b/docs/painless/painless-api-reference/org.elasticsearch.common.geo.GeoPoint.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-org-elasticsearch-common-geo-GeoPoint]]++org.elasticsearch.common.geo.GeoPoint++::
+* ++[[painless-api-reference-org-elasticsearch-common-geo-GeoPoint-getLat-0]]double link:{elasticsearch-javadoc}/org/elasticsearch/common/geo/GeoPoint.html#getLat%2D%2D[getLat]()++
+* ++[[painless-api-reference-org-elasticsearch-common-geo-GeoPoint-getLon-0]]double link:{elasticsearch-javadoc}/org/elasticsearch/common/geo/GeoPoint.html#getLon%2D%2D[getLon]()++
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/org.elasticsearch.index.fielddata.ScriptDocValues.Booleans.asciidoc b/docs/painless/painless-api-reference/org.elasticsearch.index.fielddata.ScriptDocValues.Booleans.asciidoc
new file mode 100644
index 0000000000000..b3e77c6fbc5cb
--- /dev/null
+++ b/docs/painless/painless-api-reference/org.elasticsearch.index.fielddata.ScriptDocValues.Booleans.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-Booleans]]++org.elasticsearch.index.fielddata.ScriptDocValues.Booleans++::
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-Booleans-get-1]]<<painless-api-reference-Boolean,Boolean>> link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues.Booleans.html#get%2Dint%2D[get](int)++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-Booleans-getValue-0]]boolean link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues.Booleans.html#getValue%2D%2D[getValue]()++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-Booleans-getValues-0]]<<painless-api-reference-List,List>> link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues.Booleans.html#getValues%2D%2D[getValues]()++
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-List,List>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/org.elasticsearch.index.fielddata.ScriptDocValues.BytesRefs.asciidoc b/docs/painless/painless-api-reference/org.elasticsearch.index.fielddata.ScriptDocValues.BytesRefs.asciidoc
new file mode 100644
index 0000000000000..3b61905bd0ebf
--- /dev/null
+++ b/docs/painless/painless-api-reference/org.elasticsearch.index.fielddata.ScriptDocValues.BytesRefs.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-BytesRefs]]++org.elasticsearch.index.fielddata.ScriptDocValues.BytesRefs++::
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-BytesRefs-get-1]]<<painless-api-reference-BytesRef,BytesRef>> link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues.BytesRefs.html#get%2Dint%2D[get](int)++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-BytesRefs-getValue-0]]<<painless-api-reference-BytesRef,BytesRef>> link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues.BytesRefs.html#getValue%2D%2D[getValue]()++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-BytesRefs-getValues-0]]<<painless-api-reference-List,List>> link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues.BytesRefs.html#getValues%2D%2D[getValues]()++
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-List,List>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/org.elasticsearch.index.fielddata.ScriptDocValues.Doubles.asciidoc b/docs/painless/painless-api-reference/org.elasticsearch.index.fielddata.ScriptDocValues.Doubles.asciidoc
new file mode 100644
index 0000000000000..53c43a9963514
--- /dev/null
+++ b/docs/painless/painless-api-reference/org.elasticsearch.index.fielddata.ScriptDocValues.Doubles.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-Doubles]]++org.elasticsearch.index.fielddata.ScriptDocValues.Doubles++::
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-Doubles-get-1]]<<painless-api-reference-Double,Double>> link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues.Doubles.html#get%2Dint%2D[get](int)++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-Doubles-getValue-0]]double link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues.Doubles.html#getValue%2D%2D[getValue]()++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-Doubles-getValues-0]]<<painless-api-reference-List,List>> link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues.Doubles.html#getValues%2D%2D[getValues]()++
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-List,List>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/org.elasticsearch.index.fielddata.ScriptDocValues.GeoPoints.asciidoc b/docs/painless/painless-api-reference/org.elasticsearch.index.fielddata.ScriptDocValues.GeoPoints.asciidoc
new file mode 100644
index 0000000000000..a345cd712cf0d
--- /dev/null
+++ b/docs/painless/painless-api-reference/org.elasticsearch.index.fielddata.ScriptDocValues.GeoPoints.asciidoc
@@ -0,0 +1,20 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-GeoPoints]]++org.elasticsearch.index.fielddata.ScriptDocValues.GeoPoints++::
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-GeoPoints-arcDistance-2]]double link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues.GeoPoints.html#arcDistance%2Ddouble%2Ddouble%2D[arcDistance](double, double)++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-GeoPoints-arcDistanceWithDefault-3]]double link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues.GeoPoints.html#arcDistanceWithDefault%2Ddouble%2Ddouble%2Ddouble%2D[arcDistanceWithDefault](double, double, double)++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-GeoPoints-geohashDistance-1]]double link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues.GeoPoints.html#geohashDistance%2Djava.lang.String%2D[geohashDistance](<<painless-api-reference-String,String>>)++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-GeoPoints-geohashDistanceWithDefault-2]]double link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues.GeoPoints.html#geohashDistanceWithDefault%2Djava.lang.String%2Ddouble%2D[geohashDistanceWithDefault](<<painless-api-reference-String,String>>, double)++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-GeoPoints-get-1]]<<painless-api-reference-org-elasticsearch-common-geo-GeoPoint,org.elasticsearch.common.geo.GeoPoint>> link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues.GeoPoints.html#get%2Dint%2D[get](int)++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-GeoPoints-getLat-0]]double link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues.GeoPoints.html#getLat%2D%2D[getLat]()++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-GeoPoints-getLats-0]]double[] link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues.GeoPoints.html#getLats%2D%2D[getLats]()++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-GeoPoints-getLon-0]]double link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues.GeoPoints.html#getLon%2D%2D[getLon]()++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-GeoPoints-getLons-0]]double[] link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues.GeoPoints.html#getLons%2D%2D[getLons]()++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-GeoPoints-getValue-0]]<<painless-api-reference-org-elasticsearch-common-geo-GeoPoint,org.elasticsearch.common.geo.GeoPoint>> link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues.GeoPoints.html#getValue%2D%2D[getValue]()++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-GeoPoints-getValues-0]]<<painless-api-reference-List,List>> link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues.GeoPoints.html#getValues%2D%2D[getValues]()++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-GeoPoints-planeDistance-2]]double link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues.GeoPoints.html#planeDistance%2Ddouble%2Ddouble%2D[planeDistance](double, double)++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-GeoPoints-planeDistanceWithDefault-3]]double link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues.GeoPoints.html#planeDistanceWithDefault%2Ddouble%2Ddouble%2Ddouble%2D[planeDistanceWithDefault](double, double, double)++
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-List,List>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/org.elasticsearch.index.fielddata.ScriptDocValues.Longs.asciidoc b/docs/painless/painless-api-reference/org.elasticsearch.index.fielddata.ScriptDocValues.Longs.asciidoc
new file mode 100644
index 0000000000000..f8a55344f35ee
--- /dev/null
+++ b/docs/painless/painless-api-reference/org.elasticsearch.index.fielddata.ScriptDocValues.Longs.asciidoc
@@ -0,0 +1,12 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-Longs]]++org.elasticsearch.index.fielddata.ScriptDocValues.Longs++::
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-Longs-get-1]]<<painless-api-reference-Long,Long>> link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues.Longs.html#get%2Dint%2D[get](int)++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-Longs-getDate-0]]<<painless-api-reference-org-joda-time-ReadableDateTime,org.joda.time.ReadableDateTime>> link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues.Longs.html#getDate%2D%2D[getDate]()++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-Longs-getDates-0]]<<painless-api-reference-List,List>> link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues.Longs.html#getDates%2D%2D[getDates]()++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-Longs-getValue-0]]long link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues.Longs.html#getValue%2D%2D[getValue]()++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-Longs-getValues-0]]<<painless-api-reference-List,List>> link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues.Longs.html#getValues%2D%2D[getValues]()++
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-List,List>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/org.elasticsearch.index.fielddata.ScriptDocValues.Strings.asciidoc b/docs/painless/painless-api-reference/org.elasticsearch.index.fielddata.ScriptDocValues.Strings.asciidoc
new file mode 100644
index 0000000000000..3284662cb9008
--- /dev/null
+++ b/docs/painless/painless-api-reference/org.elasticsearch.index.fielddata.ScriptDocValues.Strings.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-Strings]]++org.elasticsearch.index.fielddata.ScriptDocValues.Strings++::
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-Strings-get-1]]<<painless-api-reference-String,String>> link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues.Strings.html#get%2Dint%2D[get](int)++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-Strings-getValue-0]]<<painless-api-reference-String,String>> link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues.Strings.html#getValue%2D%2D[getValue]()++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-Strings-getValues-0]]<<painless-api-reference-List,List>> link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues.Strings.html#getValues%2D%2D[getValues]()++
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-List,List>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/org.elasticsearch.index.mapper.IpFieldMapper.IpFieldType.IpScriptDocValues.asciidoc b/docs/painless/painless-api-reference/org.elasticsearch.index.mapper.IpFieldMapper.IpFieldType.IpScriptDocValues.asciidoc
new file mode 100644
index 0000000000000..872f355f308c8
--- /dev/null
+++ b/docs/painless/painless-api-reference/org.elasticsearch.index.mapper.IpFieldMapper.IpFieldType.IpScriptDocValues.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-org-elasticsearch-index-mapper-IpFieldMapper-IpFieldType-IpScriptDocValues]]++org.elasticsearch.index.mapper.IpFieldMapper.IpFieldType.IpScriptDocValues++::
+* ++[[painless-api-reference-org-elasticsearch-index-mapper-IpFieldMapper-IpFieldType-IpScriptDocValues-get-1]]<<painless-api-reference-String,String>> link:{elasticsearch-javadoc}/org/elasticsearch/index/mapper/IpFieldMapper.IpFieldType.IpScriptDocValues.html#get%2Dint%2D[get](int)++
+* ++[[painless-api-reference-org-elasticsearch-index-mapper-IpFieldMapper-IpFieldType-IpScriptDocValues-getValue-0]]<<painless-api-reference-String,String>> link:{elasticsearch-javadoc}/org/elasticsearch/index/mapper/IpFieldMapper.IpFieldType.IpScriptDocValues.html#getValue%2D%2D[getValue]()++
+* ++[[painless-api-reference-org-elasticsearch-index-mapper-IpFieldMapper-IpFieldType-IpScriptDocValues-getValues-0]]<<painless-api-reference-List,List>> link:{elasticsearch-javadoc}/org/elasticsearch/index/mapper/IpFieldMapper.IpFieldType.IpScriptDocValues.html#getValues%2D%2D[getValues]()++
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-List,List>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/org.elasticsearch.painless.FeatureTest.asciidoc b/docs/painless/painless-api-reference/org.elasticsearch.painless.FeatureTest.asciidoc
new file mode 100644
index 0000000000000..4718ca7933250
--- /dev/null
+++ b/docs/painless/painless-api-reference/org.elasticsearch.painless.FeatureTest.asciidoc
@@ -0,0 +1,17 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-org-elasticsearch-painless-FeatureTest]]++org.elasticsearch.painless.FeatureTest++::
+* ++[[painless-api-reference-org-elasticsearch-painless-FeatureTest-overloadedStatic-0]]static boolean link:{painless-javadoc}/org/elasticsearch/painless/FeatureTest.html#overloadedStatic%2D%2D[overloadedStatic]()++
+* ++[[painless-api-reference-org-elasticsearch-painless-FeatureTest-overloadedStatic-1]]static boolean link:{painless-javadoc}/org/elasticsearch/painless/FeatureTest.html#overloadedStatic%2Dboolean%2D[overloadedStatic](boolean)++
+* ++[[painless-api-reference-org-elasticsearch-painless-FeatureTest-org.elasticsearch.painless.FeatureTest-0]]link:{painless-javadoc}/org/elasticsearch/painless/FeatureTest.html#org.elasticsearch.painless.FeatureTest%2D%2D[org.elasticsearch.painless.FeatureTest]()++
+* ++[[painless-api-reference-org-elasticsearch-painless-FeatureTest-org.elasticsearch.painless.FeatureTest-2]]link:{painless-javadoc}/org/elasticsearch/painless/FeatureTest.html#org.elasticsearch.painless.FeatureTest%2Dint%2Dint%2D[org.elasticsearch.painless.FeatureTest](int, int)++
+* ++[[painless-api-reference-org-elasticsearch-painless-FeatureTest-getX-0]]int link:{painless-javadoc}/org/elasticsearch/painless/FeatureTest.html#getX%2D%2D[getX]()++
+* ++[[painless-api-reference-org-elasticsearch-painless-FeatureTest-getY-0]]int link:{painless-javadoc}/org/elasticsearch/painless/FeatureTest.html#getY%2D%2D[getY]()++
+* ++[[painless-api-reference-org-elasticsearch-painless-FeatureTest-listInput-1]]void link:{painless-javadoc}/org/elasticsearch/painless/FeatureTest.html#listInput%2Djava.util.List%2D[listInput](<<painless-api-reference-List,List>>)++
+* ++[[painless-api-reference-org-elasticsearch-painless-FeatureTest-setX-1]]void link:{painless-javadoc}/org/elasticsearch/painless/FeatureTest.html#setX%2Dint%2D[setX](int)++
+* ++[[painless-api-reference-org-elasticsearch-painless-FeatureTest-setY-1]]void link:{painless-javadoc}/org/elasticsearch/painless/FeatureTest.html#setY%2Dint%2D[setY](int)++
+* ++[[painless-api-reference-org-elasticsearch-painless-FeatureTest-twoFunctionsOfX-2]]<<painless-api-reference-Object,Object>> link:{painless-javadoc}/org/elasticsearch/painless/FeatureTest.html#twoFunctionsOfX%2Djava.util.function.Function%2Djava.util.function.Function%2D[twoFunctionsOfX](<<painless-api-reference-Function,Function>>, <<painless-api-reference-Function,Function>>)++
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/org.elasticsearch.search.lookup.FieldLookup.asciidoc b/docs/painless/painless-api-reference/org.elasticsearch.search.lookup.FieldLookup.asciidoc
new file mode 100644
index 0000000000000..a9735010ed6f6
--- /dev/null
+++ b/docs/painless/painless-api-reference/org.elasticsearch.search.lookup.FieldLookup.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-org-elasticsearch-search-lookup-FieldLookup]]++org.elasticsearch.search.lookup.FieldLookup++::
+* ++[[painless-api-reference-org-elasticsearch-search-lookup-FieldLookup-getValue-0]]def link:{elasticsearch-javadoc}/org/elasticsearch/search/lookup/FieldLookup.html#getValue%2D%2D[getValue]()++
+* ++[[painless-api-reference-org-elasticsearch-search-lookup-FieldLookup-getValues-0]]<<painless-api-reference-List,List>> link:{elasticsearch-javadoc}/org/elasticsearch/search/lookup/FieldLookup.html#getValues%2D%2D[getValues]()++
+* ++[[painless-api-reference-org-elasticsearch-search-lookup-FieldLookup-isEmpty-0]]boolean link:{elasticsearch-javadoc}/org/elasticsearch/search/lookup/FieldLookup.html#isEmpty%2D%2D[isEmpty]()++
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/painless/painless-api-reference/org.joda.time.ReadableDateTime.asciidoc b/docs/painless/painless-api-reference/org.joda.time.ReadableDateTime.asciidoc
new file mode 100644
index 0000000000000..a7a0a17c1adb8
--- /dev/null
+++ b/docs/painless/painless-api-reference/org.joda.time.ReadableDateTime.asciidoc
@@ -0,0 +1,27 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-org-joda-time-ReadableDateTime]]++org.joda.time.ReadableDateTime++::
+* ++[[painless-api-reference-org-joda-time-ReadableDateTime-getCenturyOfEra-0]]int link:{joda-time-javadoc}/org/joda/time/ReadableDateTime.html#getCenturyOfEra%2D%2D[getCenturyOfEra]()++
+* ++[[painless-api-reference-org-joda-time-ReadableDateTime-getDayOfMonth-0]]int link:{joda-time-javadoc}/org/joda/time/ReadableDateTime.html#getDayOfMonth%2D%2D[getDayOfMonth]()++
+* ++[[painless-api-reference-org-joda-time-ReadableDateTime-getDayOfWeek-0]]int link:{joda-time-javadoc}/org/joda/time/ReadableDateTime.html#getDayOfWeek%2D%2D[getDayOfWeek]()++
+* ++[[painless-api-reference-org-joda-time-ReadableDateTime-getDayOfYear-0]]int link:{joda-time-javadoc}/org/joda/time/ReadableDateTime.html#getDayOfYear%2D%2D[getDayOfYear]()++
+* ++[[painless-api-reference-org-joda-time-ReadableDateTime-getEra-0]]int link:{joda-time-javadoc}/org/joda/time/ReadableDateTime.html#getEra%2D%2D[getEra]()++
+* ++[[painless-api-reference-org-joda-time-ReadableDateTime-getHourOfDay-0]]int link:{joda-time-javadoc}/org/joda/time/ReadableDateTime.html#getHourOfDay%2D%2D[getHourOfDay]()++
+* ++[[painless-api-reference-org-joda-time-ReadableDateTime-getMillisOfDay-0]]int link:{joda-time-javadoc}/org/joda/time/ReadableDateTime.html#getMillisOfDay%2D%2D[getMillisOfDay]()++
+* ++[[painless-api-reference-org-joda-time-ReadableDateTime-getMillisOfSecond-0]]int link:{joda-time-javadoc}/org/joda/time/ReadableDateTime.html#getMillisOfSecond%2D%2D[getMillisOfSecond]()++
+* ++[[painless-api-reference-org-joda-time-ReadableDateTime-getMinuteOfDay-0]]int link:{joda-time-javadoc}/org/joda/time/ReadableDateTime.html#getMinuteOfDay%2D%2D[getMinuteOfDay]()++
+* ++[[painless-api-reference-org-joda-time-ReadableDateTime-getMinuteOfHour-0]]int link:{joda-time-javadoc}/org/joda/time/ReadableDateTime.html#getMinuteOfHour%2D%2D[getMinuteOfHour]()++
+* ++[[painless-api-reference-org-joda-time-ReadableDateTime-getMonthOfYear-0]]int link:{joda-time-javadoc}/org/joda/time/ReadableDateTime.html#getMonthOfYear%2D%2D[getMonthOfYear]()++
+* ++[[painless-api-reference-org-joda-time-ReadableDateTime-getSecondOfDay-0]]int link:{joda-time-javadoc}/org/joda/time/ReadableDateTime.html#getSecondOfDay%2D%2D[getSecondOfDay]()++
+* ++[[painless-api-reference-org-joda-time-ReadableDateTime-getSecondOfMinute-0]]int link:{joda-time-javadoc}/org/joda/time/ReadableDateTime.html#getSecondOfMinute%2D%2D[getSecondOfMinute]()++
+* ++[[painless-api-reference-org-joda-time-ReadableDateTime-getWeekOfWeekyear-0]]int link:{joda-time-javadoc}/org/joda/time/ReadableDateTime.html#getWeekOfWeekyear%2D%2D[getWeekOfWeekyear]()++
+* ++[[painless-api-reference-org-joda-time-ReadableDateTime-getWeekyear-0]]int link:{joda-time-javadoc}/org/joda/time/ReadableDateTime.html#getWeekyear%2D%2D[getWeekyear]()++
+* ++[[painless-api-reference-org-joda-time-ReadableDateTime-getYear-0]]int link:{joda-time-javadoc}/org/joda/time/ReadableDateTime.html#getYear%2D%2D[getYear]()++
+* ++[[painless-api-reference-org-joda-time-ReadableDateTime-getYearOfCentury-0]]int link:{joda-time-javadoc}/org/joda/time/ReadableDateTime.html#getYearOfCentury%2D%2D[getYearOfCentury]()++
+* ++[[painless-api-reference-org-joda-time-ReadableDateTime-getYearOfEra-0]]int link:{joda-time-javadoc}/org/joda/time/ReadableDateTime.html#getYearOfEra%2D%2D[getYearOfEra]()++
+* ++[[painless-api-reference-org-joda-time-ReadableDateTime-toString-1]]<<painless-api-reference-String,String>> link:{joda-time-javadoc}/org/joda/time/ReadableDateTime.html#toString%2Djava.lang.String%2D[toString](<<painless-api-reference-String,String>>)++
+* ++[[painless-api-reference-org-joda-time-ReadableDateTime-toString-2]]<<painless-api-reference-String,String>> link:{joda-time-javadoc}/org/joda/time/ReadableDateTime.html#toString%2Djava.lang.String%2Djava.util.Locale%2D[toString](<<painless-api-reference-String,String>>, <<painless-api-reference-Locale,Locale>>)++
+* Inherits methods from ++<<painless-api-reference-Comparable,Comparable>>++, ++<<painless-api-reference-org-joda-time-ReadableInstant,org.joda.time.ReadableInstant>>++
diff --git a/docs/painless/painless-api-reference/org.joda.time.ReadableInstant.asciidoc b/docs/painless/painless-api-reference/org.joda.time.ReadableInstant.asciidoc
new file mode 100644
index 0000000000000..96d1f61a81f55
--- /dev/null
+++ b/docs/painless/painless-api-reference/org.joda.time.ReadableInstant.asciidoc
@@ -0,0 +1,14 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-org-joda-time-ReadableInstant]]++org.joda.time.ReadableInstant++::
+* ++[[painless-api-reference-org-joda-time-ReadableInstant-equals-1]]boolean link:{joda-time-javadoc}/org/joda/time/ReadableInstant.html#equals%2Djava.lang.Object%2D[equals](<<painless-api-reference-Object,Object>>)++
+* ++[[painless-api-reference-org-joda-time-ReadableInstant-getMillis-0]]long link:{joda-time-javadoc}/org/joda/time/ReadableInstant.html#getMillis%2D%2D[getMillis]()++
+* ++[[painless-api-reference-org-joda-time-ReadableInstant-hashCode-0]]int link:{joda-time-javadoc}/org/joda/time/ReadableInstant.html#hashCode%2D%2D[hashCode]()++
+* ++[[painless-api-reference-org-joda-time-ReadableInstant-isAfter-1]]boolean link:{joda-time-javadoc}/org/joda/time/ReadableInstant.html#isAfter%2Dorg.joda.time.ReadableInstant%2D[isAfter](<<painless-api-reference-org-joda-time-ReadableInstant,org.joda.time.ReadableInstant>>)++
+* ++[[painless-api-reference-org-joda-time-ReadableInstant-isBefore-1]]boolean link:{joda-time-javadoc}/org/joda/time/ReadableInstant.html#isBefore%2Dorg.joda.time.ReadableInstant%2D[isBefore](<<painless-api-reference-org-joda-time-ReadableInstant,org.joda.time.ReadableInstant>>)++
+* ++[[painless-api-reference-org-joda-time-ReadableInstant-isEqual-1]]boolean link:{joda-time-javadoc}/org/joda/time/ReadableInstant.html#isEqual%2Dorg.joda.time.ReadableInstant%2D[isEqual](<<painless-api-reference-org-joda-time-ReadableInstant,org.joda.time.ReadableInstant>>)++
+* ++[[painless-api-reference-org-joda-time-ReadableInstant-toString-0]]<<painless-api-reference-String,String>> link:{joda-time-javadoc}/org/joda/time/ReadableInstant.html#toString%2D%2D[toString]()++
+* Inherits methods from ++<<painless-api-reference-Comparable,Comparable>>++
diff --git a/docs/painless/painless-debugging.asciidoc b/docs/painless/painless-debugging.asciidoc
new file mode 100644
index 0000000000000..7c4484938b03e
--- /dev/null
+++ b/docs/painless/painless-debugging.asciidoc
@@ -0,0 +1,96 @@
+[[painless-debugging]]
+=== Painless Debugging
+
+experimental[The Painless scripting language is new and is still marked as experimental. The syntax or API may be changed in the future in non-backwards compatible ways if required.]
+
+==== Debug.Explain
+
+Painless doesn't have a
+https://en.wikipedia.org/wiki/Read%E2%80%93eval%E2%80%93print_loop[REPL]
+and while it'd be nice for it to have one one day, it wouldn't tell you the
+whole story around debugging painless scripts embedded in Elasticsearch because
+the data that the scripts have access to or "context" is so important. For now
+the best way to debug embedded scripts is by throwing exceptions at choice
+places. While you can throw your own exceptions
+(`throw new Exception('whatever')`), Painless's sandbox prevents you from
+accessing useful information like the type of an object. So Painless has a
+utility method, `Debug.explain` which throws the exception for you. For
+example, you can use {ref}/search-explain.html[`_explain`] to explore the
+context available to a {ref}/query-dsl-script-query.html[script query].
+
+[source,js]
+---------------------------------------------------------
+PUT /hockey/player/1?refresh
+{"first":"johnny","last":"gaudreau","goals":[9,27,1],"assists":[17,46,0],"gp":[26,82,1]}
+
+POST /hockey/player/1/_explain
+{
+  "query": {
+    "script": {
+      "script": "Debug.explain(doc.goals)"
+    }
+  }
+}
+---------------------------------------------------------
+// CONSOLE
+// TEST[s/_explain/_explain?error_trace=false/ catch:/painless_explain_error/]
+// The test system sends error_trace=true by default for easier debugging so
+// we have to override it to get a normal shaped response
+
+Which shows that the class of `doc.first` is
+`org.elasticsearch.index.fielddata.ScriptDocValues.Longs` by responding with:
+
+[source,js]
+---------------------------------------------------------
+{
+   "error": {
+      "type": "script_exception",
+      "to_string": "[1, 9, 27]",
+      "painless_class": "org.elasticsearch.index.fielddata.ScriptDocValues.Longs",
+      "java_class": "org.elasticsearch.index.fielddata.ScriptDocValues$Longs",
+      ...
+   },
+   "status": 500
+}
+---------------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"script_stack": $body.error.script_stack, "script": $body.error.script, "lang": $body.error.lang, "caused_by": $body.error.caused_by, "root_cause": $body.error.root_cause, "reason": $body.error.reason/]
+
+You can use the same trick to see that `_source` is a `LinkedHashMap`
+in the `_update` API:
+
+[source,js]
+---------------------------------------------------------
+POST /hockey/player/1/_update
+{
+  "script": "Debug.explain(ctx._source)"
+}
+---------------------------------------------------------
+// CONSOLE
+// TEST[continued s/_update/_update?error_trace=false/ catch:/painless_explain_error/]
+
+The response looks like:
+
+[source,js]
+---------------------------------------------------------
+{
+  "error" : {
+    "root_cause": ...,
+    "type": "illegal_argument_exception",
+    "reason": "failed to execute script",
+    "caused_by": {
+      "type": "script_exception",
+      "to_string": "{gp=[26, 82, 1], last=gaudreau, assists=[17, 46, 0], first=johnny, goals=[9, 27, 1]}",
+      "painless_class": "LinkedHashMap",
+      "java_class": "java.util.LinkedHashMap",
+      ...
+    }
+  },
+  "status": 400
+}
+---------------------------------------------------------
+// TESTRESPONSE[s/"root_cause": \.\.\./"root_cause": $body.error.root_cause/]
+// TESTRESPONSE[s/\.\.\./"script_stack": $body.error.caused_by.script_stack, "script": $body.error.caused_by.script, "lang": $body.error.caused_by.lang, "caused_by": $body.error.caused_by.caused_by, "reason": $body.error.caused_by.reason/]
+// TESTRESPONSE[s/"to_string": ".+"/"to_string": $body.error.caused_by.to_string/]
+
+Once you have a class you can go to <<painless-api-reference>> to see a list of
+available methods.
diff --git a/docs/painless/painless-description.asciidoc b/docs/painless/painless-description.asciidoc
new file mode 100644
index 0000000000000..874eab5632cfb
--- /dev/null
+++ b/docs/painless/painless-description.asciidoc
@@ -0,0 +1,24 @@
+_Painless_ is a simple, secure scripting language designed specifically for use
+with Elasticsearch. It is the default scripting language for Elasticsearch and
+can safely be used for inline and stored scripts. For a detailed description of
+the Painless syntax and language features, see the
+{painless}/painless-specification.html[Painless Language Specification].
+
+[[painless-features]]
+You can use Painless anywhere scripts can be used in Elasticsearch. Painless
+provides:
+
+* Fast performance: Painless scripts https://benchmarks.elastic.co/index.html#search_qps_scripts[
+run several times faster] than the alternatives.
+
+* Safety: Fine-grained whitelist with method call/field granularity. See the
+{painless}/painless-api-reference.html[Painless API Reference] for a
+complete list of available classes and methods.
+
+* Optional typing: Variables and parameters can use explicit types or the
+dynamic `def` type.
+
+* Syntax: Extends Java's syntax to provide http://groovy-lang.org/index.html[
+Groovy-style] scripting language features that make scripts easier to write.
+
+* Optimizations: Designed specifically for Elasticsearch scripting.
diff --git a/docs/painless/painless-getting-started.asciidoc b/docs/painless/painless-getting-started.asciidoc
new file mode 100644
index 0000000000000..5cb0a75916c3f
--- /dev/null
+++ b/docs/painless/painless-getting-started.asciidoc
@@ -0,0 +1,360 @@
+[[painless-getting-started]]
+== Getting Started with Painless
+
+experimental[The Painless scripting language is new and is still marked as experimental. The syntax or API may be changed in the future in non-backwards compatible ways if required.]
+
+include::painless-description.asciidoc[]
+
+[[painless-examples]]
+=== Painless Examples
+
+To illustrate how Painless works, let's load some hockey stats into an Elasticsearch index:
+
+[source,js]
+----------------------------------------------------------------
+PUT hockey/player/_bulk?refresh
+{"index":{"_id":1}}
+{"first":"johnny","last":"gaudreau","goals":[9,27,1],"assists":[17,46,0],"gp":[26,82,1],"born":"1993/08/13"}
+{"index":{"_id":2}}
+{"first":"sean","last":"monohan","goals":[7,54,26],"assists":[11,26,13],"gp":[26,82,82],"born":"1994/10/12"}
+{"index":{"_id":3}}
+{"first":"jiri","last":"hudler","goals":[5,34,36],"assists":[11,62,42],"gp":[24,80,79],"born":"1984/01/04"}
+{"index":{"_id":4}}
+{"first":"micheal","last":"frolik","goals":[4,6,15],"assists":[8,23,15],"gp":[26,82,82],"born":"1988/02/17"}
+{"index":{"_id":5}}
+{"first":"sam","last":"bennett","goals":[5,0,0],"assists":[8,1,0],"gp":[26,1,0],"born":"1996/06/20"}
+{"index":{"_id":6}}
+{"first":"dennis","last":"wideman","goals":[0,26,15],"assists":[11,30,24],"gp":[26,81,82],"born":"1983/03/20"}
+{"index":{"_id":7}}
+{"first":"david","last":"jones","goals":[7,19,5],"assists":[3,17,4],"gp":[26,45,34],"born":"1984/08/10"}
+{"index":{"_id":8}}
+{"first":"tj","last":"brodie","goals":[2,14,7],"assists":[8,42,30],"gp":[26,82,82],"born":"1990/06/07"}
+{"index":{"_id":39}}
+{"first":"mark","last":"giordano","goals":[6,30,15],"assists":[3,30,24],"gp":[26,60,63],"born":"1983/10/03"}
+{"index":{"_id":10}}
+{"first":"mikael","last":"backlund","goals":[3,15,13],"assists":[6,24,18],"gp":[26,82,82],"born":"1989/03/17"}
+{"index":{"_id":11}}
+{"first":"joe","last":"colborne","goals":[3,18,13],"assists":[6,20,24],"gp":[26,67,82],"born":"1990/01/30"}
+----------------------------------------------------------------
+// CONSOLE
+// TESTSETUP
+
+[float]
+==== Accessing Doc Values from Painless
+
+Document values can be accessed from a `Map` named `doc`.
+
+For example, the following script calculates a player's total goals. This example uses a strongly typed `int` and a `for` loop.
+
+[source,js]
+----------------------------------------------------------------
+GET hockey/_search
+{
+  "query": {
+    "function_score": {
+      "script_score": {
+        "script": {
+          "lang": "painless",
+          "source": "int total = 0; for (int i = 0; i < doc['goals'].length; ++i) { total += doc['goals'][i]; } return total;"
+        }
+      }
+    }
+  }
+}
+----------------------------------------------------------------
+// CONSOLE
+
+Alternatively, you could do the same thing using a script field instead of a function score:
+
+[source,js]
+----------------------------------------------------------------
+GET hockey/_search
+{
+  "query": {
+    "match_all": {}
+  },
+  "script_fields": {
+    "total_goals": {
+      "script": {
+        "lang": "painless",
+        "source": "int total = 0; for (int i = 0; i < doc['goals'].length; ++i) { total += doc['goals'][i]; } return total;"
+      }
+    }
+  }
+}
+----------------------------------------------------------------
+// CONSOLE
+
+The following example uses a Painless script to sort the players by their combined first and last names. The names are accessed using
+`doc['first'].value` and `doc['last'].value`.
+
+[source,js]
+----------------------------------------------------------------
+GET hockey/_search
+{
+  "query": {
+    "match_all": {}
+  },
+  "sort": {
+    "_script": {
+      "type": "string",
+      "order": "asc",
+      "script": {
+        "lang": "painless",
+        "source": "doc['first.keyword'].value + ' ' + doc['last.keyword'].value"
+      }
+    }
+  }
+}
+----------------------------------------------------------------
+// CONSOLE
+
+[float]
+==== Updating Fields with Painless
+
+You can also easily update fields. You access the original source for a field as `ctx._source.<field-name>`.
+
+First, let's look at the source data for a player by submitting the following request:
+
+[source,js]
+----------------------------------------------------------------
+GET hockey/_search
+{
+  "stored_fields": [
+    "_id",
+    "_source"
+  ],
+  "query": {
+    "term": {
+      "_id": 1
+    }
+  }
+}
+----------------------------------------------------------------
+// CONSOLE
+
+To change player 1's last name to `hockey`, simply set `ctx._source.last` to the new value:
+
+[source,js]
+----------------------------------------------------------------
+POST hockey/player/1/_update
+{
+  "script": {
+    "lang": "painless",
+    "source": "ctx._source.last = params.last",
+    "params": {
+      "last": "hockey"
+    }
+  }
+}
+----------------------------------------------------------------
+// CONSOLE
+
+You can also add fields to a document. For example, this script adds a new field that contains
+the player's nickname,  _hockey_.
+
+[source,js]
+----------------------------------------------------------------
+POST hockey/player/1/_update
+{
+  "script": {
+    "lang": "painless",
+    "source": "ctx._source.last = params.last; ctx._source.nick = params.nick",
+    "params": {
+      "last": "gaudreau",
+      "nick": "hockey"
+    }
+  }
+}
+----------------------------------------------------------------
+// CONSOLE
+
+[float]
+[[modules-scripting-painless-dates]]
+==== Dates
+
+Date fields are exposed as
+`ReadableDateTime`
+so they support methods like
+`getYear`,
+and `getDayOfWeek`.
+To get milliseconds since epoch call
+`getMillis`.
+For example, the following returns every hockey player's birth year:
+
+[source,js]
+----------------------------------------------------------------
+GET hockey/_search
+{
+  "script_fields": {
+    "birth_year": {
+      "script": {
+        "source": "doc.born.date.year"
+      }
+    }
+  }
+}
+----------------------------------------------------------------
+// CONSOLE
+
+[float]
+[[modules-scripting-painless-regex]]
+==== Regular expressions
+
+NOTE: Regexes are disabled by default because they circumvent Painless's
+protection against long running and memory hungry scripts. To make matters
+worse even innocuous looking regexes can have staggering performance and stack
+depth behavior. They remain an amazing powerful tool but are too scary to enable
+by default. To enable them yourself set `script.painless.regex.enabled: true` in
+`elasticsearch.yml`. We'd like very much to have a safe alternative
+implementation that can be enabled by default so check this space for later
+developments!
+
+Painless's native support for regular expressions has syntax constructs:
+
+* `/pattern/`: Pattern literals create patterns. This is the only way to create
+a pattern in painless. The pattern inside the ++/++'s are just
+http://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html[Java regular expressions].
+See <<pattern-flags>> for more.
+* `=~`: The find operator return a `boolean`, `true` if a subsequence of the
+text matches, `false` otherwise.
+* `==~`: The match operator returns a `boolean`, `true` if the text matches,
+`false` if it doesn't.
+
+Using the find operator (`=~`) you can update all hockey players with "b" in
+their last name:
+
+[source,js]
+----------------------------------------------------------------
+POST hockey/player/_update_by_query
+{
+  "script": {
+    "lang": "painless",
+    "source": "if (ctx._source.last =~ /b/) {ctx._source.last += \"matched\"} else {ctx.op = 'noop'}"
+  }
+}
+----------------------------------------------------------------
+// CONSOLE
+
+Using the match operator (`==~`) you can update all the hockey players who's
+names start with a consonant and end with a vowel:
+
+[source,js]
+----------------------------------------------------------------
+POST hockey/player/_update_by_query
+{
+  "script": {
+    "lang": "painless",
+    "source": "if (ctx._source.last ==~ /[^aeiou].*[aeiou]/) {ctx._source.last += \"matched\"} else {ctx.op = 'noop'}"
+  }
+}
+----------------------------------------------------------------
+// CONSOLE
+
+You can use the `Pattern.matcher` directly to get a `Matcher` instance and
+remove all of the vowels in all of their last names:
+
+[source,js]
+----------------------------------------------------------------
+POST hockey/player/_update_by_query
+{
+  "script": {
+    "lang": "painless",
+    "source": "ctx._source.last = /[aeiou]/.matcher(ctx._source.last).replaceAll('')"
+  }
+}
+----------------------------------------------------------------
+// CONSOLE
+
+`Matcher.replaceAll` is just a call to Java's `Matcher`'s
+http://docs.oracle.com/javase/8/docs/api/java/util/regex/Matcher.html#replaceAll-java.lang.String-[replaceAll]
+method so it supports `$1` and `\1` for replacements:
+
+[source,js]
+----------------------------------------------------------------
+POST hockey/player/_update_by_query
+{
+  "script": {
+    "lang": "painless",
+    "source": "ctx._source.last = /n([aeiou])/.matcher(ctx._source.last).replaceAll('$1')"
+  }
+}
+----------------------------------------------------------------
+// CONSOLE
+
+If you need more control over replacements you can call `replaceAll` on a
+`CharSequence` with a `Function<Matcher, String>` that builds the replacement.
+This does not support `$1` or `\1` to access replacements because you already
+have a reference to the matcher and can get them with `m.group(1)`.
+
+IMPORTANT: Calling `Matcher.find` inside of the function that builds the
+replacement is rude and will likely break the replacement process.
+
+This will make all of the vowels in the hockey player's last names upper case:
+
+[source,js]
+----------------------------------------------------------------
+POST hockey/player/_update_by_query
+{
+  "script": {
+    "lang": "painless",
+    "source": "ctx._source.last = ctx._source.last.replaceAll(/[aeiou]/, m -> m.group().toUpperCase(Locale.ROOT))"
+  }
+}
+----------------------------------------------------------------
+// CONSOLE
+
+Or you can use the `CharSequence.replaceFirst` to make the first vowel in their
+last names upper case:
+
+[source,js]
+----------------------------------------------------------------
+POST hockey/player/_update_by_query
+{
+  "script": {
+    "lang": "painless",
+    "source": "ctx._source.last = ctx._source.last.replaceFirst(/[aeiou]/, m -> m.group().toUpperCase(Locale.ROOT))"
+  }
+}
+----------------------------------------------------------------
+// CONSOLE
+
+
+Note: all of the `_update_by_query` examples above could really do with a
+`query` to limit the data that they pull back. While you *could* use a
+See {ref}/query-dsl-script-query.html[script query] it wouldn't be as efficient
+as using any other query because script queries aren't able to use the inverted
+index to limit the documents that they have to check.
+
+[[modules-scripting-painless-dispatch]]
+=== How painless dispatches functions
+
+Painless uses receiver, name, and https://en.wikipedia.org/wiki/Arity[arity]
+for method dispatch. For example, `s.foo(a, b)` is resolved by first getting
+the class of `s` and then looking up the method `foo` with two parameters. This
+is different from Groovy which uses the
+https://en.wikipedia.org/wiki/Multiple_dispatch[runtime types] of the
+parameters and Java which uses the compile time types of the parameters.
+
+The consequence of this that Painless doesn't support overloaded methods like
+Java, leading to some trouble when it whitelists classes from the Java
+standard library. For example, in Java and Groovy, `Matcher` has two methods:
+`group(int)` and `group(String)`. Painless can't whitelist both of these methods
+because they have the same name and the same number of parameters. So instead it
+has `group(int)` and `namedGroup(String)`.
+
+We have a few justifications for this different way of dispatching methods:
+
+1. It makes operating on `def` types simpler and, presumably, faster. Using
+receiver, name, and arity means that when Painless sees a call on a `def` object it
+can dispatch the appropriate method without having to do expensive comparisons
+of the types of the parameters. The same is true for invocations with `def`
+typed parameters.
+2. It keeps things consistent. It would be genuinely weird for Painless to
+behave like Groovy if any `def` typed parameters were involved and Java
+otherwise. It'd be slow for it to behave like Groovy all the time.
+3. It keeps Painless maintainable. Adding the Java or Groovy like method
+dispatch *feels* like it'd add a ton of complexity which'd make maintenance and
+other improvements much more difficult.
+
+include::painless-debugging.asciidoc[]
diff --git a/docs/painless/painless-lang-spec.asciidoc b/docs/painless/painless-lang-spec.asciidoc
new file mode 100644
index 0000000000000..dbad00931af20
--- /dev/null
+++ b/docs/painless/painless-lang-spec.asciidoc
@@ -0,0 +1,73 @@
+[[painless-specification]]
+== Painless Language Specification
+
+Painless uses a Java-style syntax that is similar to Groovy. In fact, most
+Painless scripts are also valid Groovy, and simple Groovy scripts are typically
+valid Painless. This specification assumes you have at least a passing
+familiarity with Java and related languages.
+
+Painless is essentially a subset of Java with some additional scripting
+language features that make scripts easier to write. However, there are some
+important differences, particularly with the casting model. For more detailed
+conceptual information about the basic constructs that Java and Painless share,
+refer to the corresponding topics in the
+https://docs.oracle.com/javase/specs/jls/se8/html/index.html[Java Language
+Specification].
+
+Painless scripts are parsed and compiled using the http://www.antlr.org/[ANTLR4]
+and http://asm.ow2.org/[ASM] libraries. Painless scripts are compiled directly
+into Java byte code and executed against a standard Java Virtual Machine. This
+specification uses ANTLR4 grammar notation to describe the allowed syntax.
+However, the actual Painless grammar is more compact than what is shown here.
+
+[float]
+[[comments]]
+==== Comments
+
+Painless supports both single-line and multi-line comments. You can include
+comments anywhere within a script.
+
+Single-line comments are preceded by two slashes: `// comment`. They can be
+placed anywhere on a line. All characters from the two slashes to the end of
+the line are ignored.
+
+Multi-line comments are preceded by a slash-star `/*` and closed by
+star-slash `*/`. Multi-line comments can start anywhere on a line. All
+characters from the opening `/*` to the closing `*/` are ignored.
+
+*Examples:*
+
+[source,Java]
+----
+// single-line comment
+
+<code> // single-line comment
+
+/* multi-
+   line
+   comment */
+
+<code> /* multi-line
+          comment */ <code>
+
+<code> /* multi-line comment */ <code>
+----
+
+[float]
+[[keywords]]
+==== Keywords
+
+Painless reserves the following keywords for built-in language features.
+These keywords cannot be used used in other contexts, such as identifiers.
+
+[cols="^1,^1,^1,^1,^1"]
+|====
+| if | else | while | do | for
+| in | continue | break | return | new
+| try | catch | throw | this | instanceof
+|====
+
+include::painless-literals.asciidoc[]
+include::painless-variables.asciidoc[]
+include::painless-types.asciidoc[]
+include::painless-operators.asciidoc[]
diff --git a/docs/painless/painless-literals.asciidoc b/docs/painless/painless-literals.asciidoc
new file mode 100644
index 0000000000000..43c5eb82f96a2
--- /dev/null
+++ b/docs/painless/painless-literals.asciidoc
@@ -0,0 +1,94 @@
+[[literals]]
+=== Literals
+
+Literals are values that you can specify directly in Painless scripts.
+
+[[integers]]
+==== Integers
+
+Specify integer literals in decimal, octal, or hex notation. Use the following
+single letter designations to specify the primitive type: `l` for `long`, `f`
+for `float`, and `d` for `double`.  If not specified, the type defaults to
+`int` (with the exception of certain assignments described later).
+
+*Grammar:*
+[source,ANTLR4]
+----
+INTEGER: '-'? ( '0' | [1-9] [0-9]* ) [lLfFdD]?;
+OCTAL: '-'? '0' [0-7]+ [lL]?;
+HEX: '-'? '0' [xX] [0-9a-fA-F]+ [lL]?;
+----
+
+*Examples:*
+[source,Java]
+----
+0     // integer literal of 0
+0D    // double literal of 0.0
+1234L // long literal of 1234
+-90F  // float literal of -90.0
+-022  // integer literal of -18 specified in octal
+0xF2A // integer literal of 3882
+----
+
+[[floating-point-values]]
+==== Floating Point Values
+
+Specify floating point literals using the following single letter designations
+for the primitive type: `f` for `float` and `d` for `double`.
+If not specified, the type defaults to `double`.
+
+*Grammar:*
+[source,ANTLR4]
+----
+DECIMAL: '-'? ( '0' | [1-9] [0-9]* ) (DOT [0-9]+)? ( [eE] [+\-]? [0-9]+ )? [fFdD]?;
+----
+
+*Examples:*
+[source,Java]
+----
+0.0      // double value of 0.0
+1E6      // double value of 1000000
+0.977777 // double value of 0.97777
+-126.34  // double value of -126.34
+89.9F    // float value of 89.9
+----
+
+[[strings]]
+==== Strings
+
+Specify literal string with either single or double quotes. In double-quoted
+literal strings, you can escape double-quotes with a backslash to include them
+in the string. Similarly, you escape single quotes with a backslash in
+single-quoted literal strings. Backslashes themselves also need to be
+escaped with a backslash.
+
+*Grammar:*
+[source,ANTLR4]
+----
+STRING: ( '"' ( '\\"' | '\\\\' | ~[\\"] )*? '"' ) | ( '\'' ( '\\\'' | '\\\\' | ~[\\'] )*? '\'' );
+----
+
+*Examples:*
+[source,Java]
+----
+"double-quoted String literal"
+'single-quoted String literal'
+"\"double-quoted String with escaped double-quotes\" and backslash: \\"
+'\'single-quoted String with escaped single-quotes\' and backslash \\'
+"double-quoted String with non-escaped 'single-quotes'"
+'single-quoted String with non-escaped "double-quotes"'
+----
+
+[[char]]
+===== Char
+
+You cannot directly specify character literals in Painless. However, you can
+cast single-character strings to char. Attempting to cast a multi-character
+string to a char throws an error.
+
+*Examples:*
+[source,Java]
+----
+(char)"C"
+(char)'c'
+----
\ No newline at end of file
diff --git a/docs/painless/painless-operators.asciidoc b/docs/painless/painless-operators.asciidoc
new file mode 100644
index 0000000000000..2ed0fb6d4be50
--- /dev/null
+++ b/docs/painless/painless-operators.asciidoc
@@ -0,0 +1,1774 @@
+=== Operators
+
+The following is a table of the available operators in Painless.  Each operator will have further information and examples outside of the table.  Many operators will have a promotion table as described by the documentation on promotion [MARK].
+
+[options="header",cols="6,3,2,4"]
+|====
+|Operator|Symbol(s)|Precedence|Associativity
+|Precedence|()|0|left-to-right
+|Field Access|.|1|left-to-right
+|Method Call|. ()|1|left-to-right
+|Null Safe|?.|1|left-to-right
+|Function Call|()|1|left-to-right
+|Array Initialization|[] {}|1|left-to-right
+|Array Access|[]|1|left-to-right
+|Array Length|.|1|left-to-right
+|List Initialization|[]|1|left-to-right
+|List Access|[]|1|left-to-right
+|Map Initialization|[:]|1|left-to-right
+|Map Access|[]|1|left-to-right
+|Post Increment|++|1|left-to-right
+|Post Decrement|--|1|left-to-right
+|Pre Increment|++|2|right-to-left
+|Pre Decrement|--|2|right-to-left
+|Unary Positive|+|2|right-to-left
+|Unary Negative|-|2|right-to-left
+|Boolean Not|!|2|right-to-left
+|Bitwise Not|~|2|right-to-left
+|Cast|()|3|right-to-left
+|Constructor Call|new ()|3|right-to-left
+|New Array|new|3|right-to-left
+|Multiplication|*|4|left-to-right
+|Division|/|4|left-to-right
+|Remainder|%|4|left-to-right
+|String Concatenation|+|5|left-to-right
+|Addition|+|5|left-to-right
+|Subtraction|-|5|left-to-right
+|Left Shift|<<|6|left-to-right
+|Right Shift|>>|6|left-to-right
+|Unsigned Right Shift|>>>|6|left-to-right
+|Greater Than|>|7|left-to-right
+|Greater Than Or Equal|>=|7|left-to-right
+|Less Than|<|7|left-to-right
+|Less Than Or Equal|<=|7|left-to-right
+|Instance Of|instanceof|8|left-to-right
+|Equality Equals|==|9|left-to-right
+|Equality Not Equals|!=|9|left-to-right
+|Identity Equals|===|9|left-to-right
+|Identity Not Equals|!==|9|left-to-right
+|Bitwise And|&|10|left-to-right
+|Boolean Xor|^|11|left-to-right
+|Bitwise Xor|^|11|left-to-right
+|Bitwise Or|\||12|left-to-right
+|Boolean And|&&|13|left-to-right
+|Boolean Or|\|\||14|left-to-right
+|Conditional|? :|15|right-to-left
+|Elvis|?:|16|right-to-left
+|Assignment|=|17|right-to-left
+|Compound Assignment|$=|17|right-to-left
+|====
+
+[[precedence-operator]]
+==== Precedence
+
+You group expressions using the precedence operator to guarantee
+the order of evaluation and override existing precedence relationships between operators. The format is an opening parenthesis, one or more expressions, and
+a closing parenthesis. For example, `(20+1)*2`.
+
+*Grammar:*
+[source,ANTLR4]
+----
+precedence: '(' expression ')';
+----
+
+*Examples:*
+[source,Java]
+----
+int x = (5+4)*6; // declares the variable int x and sets it to (5+4)*6
+                 // where 5+4 is evaluated first due to the precedence operator
+int y = 2*(x-4); // declares the variable int y and sets it to 2*(x-4)
+                 // where x-4 is evaluated first due to the precedence operator
+----
+
+
+[[dot-operator]]
+==== Dot
+You use the dot operator `.` to access a type's <<field-access,
+fields>> and <<method-access, methods>>.
+
+[[field-access]]
+===== Accessing Fields
+You access primitive and reference type members in a reference type using the
+dot operator '.' followed by the id of the member. The accessed member behaves
+the same way as the type it represents with one exception: if the reference
+type is of type `def`, the member is also considered to be of type `def` and
+resolved at runtime.
+
+*Grammar:*
+[source,ANTLR4]
+----
+field_access: ID '.' ID;
+----
+
+*Examples:*
+[source,Java]
+----
+FeatureTest ft = new FeatureTest(); // Declare FeatureTest variable ft and
+                                    //   set it to a newly allocated FeatureTest
+ft.x = 5;                           // Access int member x from ft and assign
+                                    //   it the literal int value 5
+ft.y = ft.x;                        // Access int member y from ft and assign
+                                    //   it the value of ft member x
+int value = ft.x + ft.y;            // Declare variable value as an int,
+                                    //   add ft members x and y together,
+                                    //   assign the sum to the variable value
+----
+
+[[method-access]]
+===== Calling Methods
+
+You call reference type methods using the dot operator and the method id:
+`.method_id(arg1,...,argn)`. The parentheses are required even if there are no
+arguments.
+
+If the reference type is not type `def`, the argument types for the method
+can be resolved at compile time. An error occurs if appropriate type
+conversions (casting) cannot be performed. If the reference type is type `def`, the argument types for the method are all considered to be the type `def`. The
+appropriate type conversions are performed at run-time.
+
+Automatic <<boxing-unboxing,boxing and unboxing>> is performed when you pass in
+arguments to a method.
+
+Method calls can be overloaded based on arity in Painless. The same method
+name can be re-used for different methods as long as the number of arguments
+differs. This differs from Java method overloading, where only the types must
+differ. This has an effect on some of the provided reference type methods in
+the <<painless-api-reference,Painless API>>. Where there are overloaded methods with
+the same arity for a reference type in Java, Painless chooses a single method
+to be provided.
+
+*Grammar:*
+[source,ANTLR4]
+----
+method_call: ID '.' ID '(' (expression (',' expression)*)? ')';
+----
+
+*Examples:*
+[source,Java]
+----
+Map m = new HashMap();         // Declare Map variable m and set it a newly
+                               //   allocated HashMap
+x.put(1, 2);                   // Call the put method on variable x to add key 1
+                               //   with the value 2 to the Map
+int z = x.get(1);              // Declare int variable z, call the get method to
+                               //   retrieve the value of key 1, and assign the
+                               //   return value of the method call to variable z
+def d = new ArrayList();       // Declare def variable m and set it a newly
+                               //   allocated ArrayList
+d.add(1);                      // Call the add method on variable d and add the
+                               //   literal int 1 to the ArrayList. Note that
+                               //   the argument type is considered to be of
+                               //   type def since the reference type is also def
+int i = Integer.parseInt('2'); // Declare int variable i and set it to the
+                               //   value returned by the static method parseInt
+----
+
+**************************
+Painless describes the Map method arguments using the `def` type:
+
+[source,Java]
+----
+put(def, def)
+get(def)
+----
+
+When you call `x.put(1, 2)`, the key and value are implicitly converted from
+the int type to the def type.
+
+Assume for a minute that the Map method arguments were described as Integers:
+
+[source,Java]
+----
+put(Integer, Integer)
+get(Integer)
+----
+
+In this case, the key and value would implicitly be _boxed_ from the primitive
+int type to the Integer reference type. For more information about how Painless
+casts between primitive types and reference types, see <<boxing-unboxing, Boxing
+and Unboxing>>.
+**************************
+
+==== Null Safe
+
+The null safe operator `?.` can be used in place of the dot operator
+to check if a reference type instance is `null` before attempting to access
+a field or make a method call against it. When using the null safe operator,
+if the instance is `null`, the returned value is `null`. If the reference
+type instance is non-null, it returns the value of the field or result of
+the method call normally.
+
+// REVIEWER NOTE: The following paragraph doesn't make sense to me. Do you
+All resultant types must be a reference type or be able to be implicitly cast
+to a reference type or an error will occur.
+
+*Grammar:*
+[source,ANTLR4]
+----
+null_safe: null_safe_field_access
+         | null_safe_method_call;
+null_safe_field_access: ID '?.' ID;
+null_safe_method_call: ID '?.' ID '(' (expression (',' expression)*)? ')';
+----
+
+*Examples:*
+[source,Java]
+----
+Map x = new HashMap(); // Declare the Map variable x and set it to a newly
+                       //   allocated HashMap
+Map y = null;          // Declare the Map variable y and set it to null
+def z = new HashMap(); // Declares the def variable z and set it to a newly
+                       // allocated HashMap
+
+x.put(1, 2);           // Put the key-value pair 1 and 2 into x
+z.put(5, 6);           // Put the key-value pair 5 and 6 into z
+
+def value = x?.get(1); // Declare the def variable value and set it to the
+                       //   result of .get(1) since x is not null
+value = y?.get(3);     // Sets value to null since y is null
+value = z?.get(5);     // Sets value to the result of .get(5) since z is not null
+----
+
+==== Parenthesis
+
+User-defined function calls can be made in Painless using the parenthesis
+operator.  See Function Calls [MARK] for more information.
+
+==== Brackets and Braces
+
+The brackets operator `[]` is used to create and access arrays, lists, and maps.
+The braces operator `{}` is used to intialize arrays.
+
+===== Creating and Initializing Arrays
+
+You create and initialize arrays using the brackets `[]` and braces `{}`
+operators. Each set of brackets represents a dimension. The values you want to
+initialize each dimension with are specified as a comma-separated list enclosed
+in braces. For example, `new int[] {1, 2, 3}` creates a one dimensional `int`
+array with a size of 3 and the values 1, 2, and 3.
+
+For more information about allocating and initializing arrays, see <<array-type,
+Array Type>>.
+
+===== Accessing Array Elements
+
+Elements in an array are stored and accessed using the brackets `[]` operator.
+Elements are referenced by an expression enclosed in brackets. An error
+occurs if the expression used to reference an element cannot be implicitly
+cast to an `int`.
+
+The range of elements within an array that can be accessed is `[0, size)` where
+size is the originally allocated size of the array. To access elements relative
+to the last element in an array, you can use a negative numeric value from
+`[-size, -1]`. An error occurs if you attempt to reference an element outside
+of the array's range.
+
+*Grammar:*
+[source,ANTLR4]
+----
+brace_access: '[' expression ']'
+----
+
+*Examples:*
+[source,Java]
+----
+
+int[] x = new int[2];     // Declare int array x and set it to a newly allocated
+                          //   array with a size of 2
+x[0] = 2;                 // Set the 0th element of array x to 2
+x[1] = 5;                 // Set the 1st element of array x to 5
+int y = x[0] + x[1];      // Declare the int variable y and set it to the sum
+                          //   of the first two elements of array x
+int z = 1;                // Declare the int variable z and set it to 1
+return x[z];              // Access the 1st element of array x using the
+                          //   variable z as an expression and return the value
+
+def d = new int[2];       // Declare def variable d and set it to a newly
+                          //   allocated array with a size of 2
+d[0] = 2;                 // Set the 0th element of array d to 2
+d[1] = 5;                 // Set the 1st element of array d to 2
+def y = d[0] + d[1];      // Declare def variable y and set it to the sum
+                          //   of the first two elements of array d
+def z = 1;                // Declare def variable z and set it to 1
+return d[z];              // Access the 1st element of array d using the
+                          //   variable z as an expression and return the value
+----
+
+NOTE: The use of the `def` type in the second example means that the types
+cannot be resolved until runtime.
+
+===== Array Length
+
+Arrays contain a special member known as 'length' that is a read-only value that contains the size of the array.  This member can be accessed from an array using the dot operator.
+
+*Examples:*
+[source,Java]
+----
+int[] x = new int[10]; // declares an int array variable x and sets it to a newly allocated array with a size of 10
+int l = x.length;      // declares and int variable l and sets it to the field length of variable x
+----
+
+===== Creating and Initializing Lists
+
+You create and initialize lists using the brackets `[]` operator. The values
+you want to initialize the list with are specified as a comma-separated list
+of expressions enclosed in brackets. For example, `List l = [1, 2, 3]` creates
+a new three item list. Each expression used to initialize the list is converted
+a `def` type when the value is inserted into the list. The order of the
+expressions is maintained.
+
+*Grammar:*
+[source,ANTLR4]
+----
+list_initialization: '[' expression (',' expression)* ']'
+                   | '[' ']';
+----
+
+*Examples:*
+[source,Java]
+----
+List empty = [];          // declares the List variable empty and sets it to a newly initialized empty List
+List l0 = [1, 2, 3];      // declares the List variable l0 and sets it to a newly initialized List with the values 1, 2, and 3
+
+int i = 1;
+long l = 2L;
+float f = 3.0F;
+double d = 4.0;
+String s = "5";
+List l1 = [i, l, f*d, s]; // declares the List variable l1 and sets it to a newly initialized List with the values of i, l, and f*d and s
+----
+
+===== Accessing List Elements
+
+Elements in a List are stored or accessed using the brackets operator.  The format begins with an opening bracket, followed by an expression, and finishes with a closing bracket.  Storing elements in a List is equivalent to invoking a List's set method.  Accessing elements in a List is equivalent to invoking a List's get method.  Using this operator is strictly a shortcut for the previously mentioned methods.  The range of elements within a List that can be accessed is [0, size) where size is the number of elements currently in the List.  Elements may also be accessed from the last element in a List using a negative numeric value from [-size, -1].  The expression used to determine which element is accessed must be able to be implicitly cast to an int.  An error will occur if the expression is outside of the legal range or is not of type int.
+
+*Grammar:*
+[source,ANTLR4]
+----
+list_access: '[' expression ']'
+----
+
+*Examples:*
+[source,Java]
+----
+List x = new ArrayList(); // declares a List variable x and sets it to a newly allocated ArrayList
+x.add(1);                 // invokes the add method on the variable x and adds the constant int 1 to the List
+x.add(2);                 // invokes the add method on the variable x and adds the constant int 2 to the List
+x.add(3);                 // invokes the add method on the variable x and adds the constant int 3 to the List
+x[0] = 2;                 // sets the 0th element of the variable x to the constant int 2
+x[1] = 5;                 // sets the 1st element of the variable x to the constant int 2
+int y = x[0] + x[1];      // declares the int variable y and sets it to the sum of the first two elements of the variable x
+int z = 1;                // declares the int variable z and sets it to the constant int 1
+return x[z];              // accesses the 1st element of the variable x using the variable z as an expression and returns the value
+
+def d = new ArrayList(); // declares a def variable d and sets it to a newly allocated ArrayList
+d.add(1);                // invokes the add method on the variable d and adds the constant int 1 to the List
+d.add(2);                // invokes the add method on the variable d and adds the constant int 2 to the List
+d.add(3);                // invokes the add method on the variable d and adds the constant int 3 to the List
+d[0] = 2;                // sets the 0th element of the variable d to the constant int 2
+d[1] = 5;                // sets the 1st element of the variable d to the constant int 2
+def y = d[0] + d[1];     // declares the def variable y and sets it to the sum of the first two elements of the variable d
+def z = 1;               // declares the def variable z and sets it to the constant int 1
+return d[z];             // accesses the 1st element of the variable d using the variable z as an expression and returns the value
+----
+
+Note in the first example above all types can be resolved at compile-time, while in the second example all types must wait to be resolved until run-time.
+
+===== Creating and Initializing Maps
+
+A Map can be created and initialized using the brackets operator.  The format begins with a bracket, followed by an arbitrary number of key-value pairs delimited with commas (except the last), and ends with a closing bracket.  Each key-value pair is a set of two expressions separate by a colon.  If there is only a single colon with no expressions, a new empty Map is created.
+
+*Grammar:*
+[source,ANTLR4]
+----
+map_initialization: '[' key_pair (',' key_pair)* ']'
+                  | '[' ':' ']';
+key_pair: expression ':' expression
+----
+
+Each expression used as part of the initialization is converted to a `def` type
+for insertion into the map.
+
+*Examples:*
+[source,Java]
+----
+Map empty = [:];            // declares the Map variable empty and sets it to a newly initialized empty Map
+Map m0 = [1:2, 3:4, 5:6];   // declares the Map variable m0 and sets it to a newly initialized Map with the keys 1, 3, 5 and values 2, 4, 6, respectively
+
+byte b = 0;
+int i = 1;
+long l = 2L;
+float f = 3.0F;
+double d = 4.0;
+String s = "5";
+Map m1 = [b:i, l:f*d, d:s]; // declares the Map variable m1 and sets it to a newly initialized Map with the keys b, l, d and values i, f*d, s, respectively
+----
+
+===== Accessing Map Elements
+
+Elements in a Map can be stored or accessed using the brackets operator.  The format begins with an opening bracket, followed by an expression, and finishes with a closing bracket.  Storing values in a Map is equivalent to invoking a Map's put method.  Accessing values in a Map is equivalent to invoking a Map's get method.  Using this operator is strictly a shortcut for the previously mentioned methods.  Any element from a Map can be stored/accessed where the expression is the key.  If a key has no corresponding value when accessing a Map then the value will be null.
+
+*Grammar:*
+[source,ANTLR4]
+----
+map_access: '[' expression ']'
+----
+
+*Examples:*
+[source,Java]
+----
+Map x = new HashMap();             // declares a Map variable x and sets it to a newly allocated HashMap
+x['value2'] = 2;                   // puts the value of the key constant String value2 of the variable x to the constant int 2
+x['value5'] = 5;                   // puts the value of the key constant String value5 of the variable x to the constant int 5
+int y = x['value2'] + x['value5']; // declares the int variable y and sets it to the sum of the two values of the variable x
+String z = 'value5';               // declares the String variable z and sets it to the constant String value5
+return x[z];                       // accesses the value for the key value5 of the variable x using the variable z as an expression and returns the value
+
+def d = new HashMap();             // declares a def variable d and sets it to a newly allocated HashMap
+d['value2'] = 2;                   // puts the value of the key constant String value2 of the variable d to the constant int 2
+d['value5'] = 5;                   // puts the value of the key constant String value5 of the variable d to the constant int 5
+int y = d['value2'] + d['value5']; // declares the int variable y and sets it to the sum of the two values of the variable d
+String z = 'value5';               // declares the String variable z and sets it to the constant String value5
+return d[z];                       // accesses the value for the key value5 of the variable x using the variable z as an expression and returns the value
+----
+
+Note in the first example above all types can be resolved at compile-time, while in the second example all types must wait to be resolved until run-time.
+
+==== Post Increment
+
+A variable/field representing a numerical value can be possibly evaluated as part of an expression, and then increased by 1 for its respective type.  The format starts with a variable name followed by a plus and ends with a plus.
+
+*Grammar:*
+[source,ANTLR4]
+----
+post_increment: ( variable | member ) '++'
+----
+
+A numeric promotion may occur during a post-increment followed by a downcast if necessary.  A def type evaluated at run-time will follow the same promotion table at run-time following whatever type def represents.  A downcast may be required after the type promotion to assign the appropriate value back into the variable/field.  Non-numeric variables/members will result in an error.
+
+Promotion Table:
+
+|====
+|from|to|downcast
+|byte|int|byte
+|short|int|short
+|char|int|char
+|int|int|
+|long|long|
+|float|float|
+|double|double|
+|def|def|
+|====
+
+Examples(s):
+[source,Java]
+----
+int i = 0;    // declares the int variable i and sets it to the constant 0
+i++;          // increments the int variable i by 1 to a value of 1
+long l = 1;   // declares the long variable l and set it the constant 1
+long k;       // declares the long variable k
+k = l++;      // sets the long variable k to the value of l (1), and then increments the long variable l by 1 to a value of 2
+----
+
+==== Post Decrement
+
+A variable/field representing a numerical value can be possibly evaluated as part of an expression, and then increased by 1 for its respective type.  The format starts with a variable name followed by a minus and ends with a minus.
+
+*Grammar:*
+[source,ANTLR4]
+----
+post_increment: ( variable | member ) '--'
+----
+
+A numeric promotion may occur during a post-decrement followed by a downcast if necessary.  A def type evaluated at run-time will follow the same promotion table at run-time following whatever type def represents.  A downcast may be required after the type promotion to assign the appropriate value back into the variable/field.  Non-numeric variables/members will result in an error.
+
+Promotion Table:
+
+|====
+|from|to|downcast
+|byte|int|byte
+|short|int|short
+|char|int|char
+|int|int|
+|long|long|
+|float|float|
+|double|double|
+|def|def|
+|====
+
+Examples(s):
+[source,Java]
+----
+short i = 0;    // declares the short variable i and sets it to the constant short 0
+i--;            // decrements the short variable i by 1 to a value of -1 (promoted to int and downcast to short)
+float l = 1.0f; // declares the float variable l and sets it the constant float 1.0f
+float k;        // declares the float variable k
+k = l--;        // sets the float variable k to the value of l (1.0f), and then decrements the float variable l by 1.0 to a value of 0.0
+----
+
+==== Pre Increment
+
+A variable/field representing a numerical value can be increased by 1 for its respective type, and then possibly evaluated as part of an expression.  The format starts with a plus followed by a plus and ends with a variable name.
+
+*Grammar:*
+[source,ANTLR4]
+----
+pre_increment: '++' ( variable | member )
+----
+
+A numeric promotion may occur during a pre-increment followed by a downcast if necessary.  A def type evaluated at run-time will follow the same promotion table at run-time following whatever type def represents.  A downcast may be required after the type promotion to assign the appropriate value back into the variable/field.  Non-numeric variables/members will result in an error.
+
+Promotion Table:
+
+|====
+|from|to|downcast
+|byte|int|byte
+|short|int|short
+|char|int|char
+|int|int|
+|long|long|
+|float|float|
+|double|double|
+|def|def|
+|====
+
+Examples(s):
+[source,Java]
+----
+int i = 0;    // declares the int variable i and sets it to the constant int 0
+++i;          // increments the int variable i by 1 to a value of 1
+long l = 1;   // declares the long variable l and sets it to the constant long 1
+long k;       // declares the long variable k
+k = ++l;      // increments the long variable l by 1 to a value of 2, and then sets the long variable k to the value of l (2)
+----
+
+==== Pre Decrement
+
+A variable/field representing a numerical value can be decreased by 1 for its respective type, and then possibly evaluated as part of an expression.  The format starts with a minus followed by a minus and ends with a variable name.
+
+*Grammar:*
+[source,ANTLR4]
+----
+pre_decrement: '--' ( variable | member )
+----
+
+A numeric promotion may occur during a pre-decrement followed by a downcast if necessary.  A def type evaluated at run-time will follow the same promotion table at run-time following whatever type def represents.  A downcast may be required after the type promotion to assign the appropriate value back into the variable/field.  Non-numeric variables/members will result in an error.
+
+Promotion Table:
+|====
+|from|to|downcast
+|byte|int|byte
+|short|int|short
+|char|int|char
+|int|int|
+|long|long|
+|float|float|
+|double|double|
+|def|def|
+|====
+
+Examples(s):
+[source,Java]
+----
+byte i = 1;      // declares the byte variable i and sets it to the constant int 1
+--i;             // decrements the byte variable i by 1 to a value of 0 (promoted to int and downcast to byte)
+double l = 1.0;  // declares the double variable l and sets it to the constant double 1.0
+double k;        // declares the double variable k
+k = --l;         // decrements the double variable l by 1.0 to a value of 0.0, and then sets the double variable k to the value of l (0.0)
+----
+
+==== Unary Positive
+
+Unary positive gives the identity of a numerical value using the plus operator.  In practice this is usually a no-op, but will cause some numeric types to be promoted.  Format starts with a plus operator followed by a numerical expression.
+
+*Grammar:*
+[source,ANTLR4]
+----
+unary_positive: '+' expression
+----
+
+A numeric promotion may occur during a unary positive operation.  A def type evaluated at run-time will follow the same promotion table at run-time following whatever type def represents.  Non-numeric expressions will result in an error.
+
+Promotion Table:
+|====
+|from|to
+|byte|int
+|short|int
+|char|int
+|int|int
+|long|long
+|float|float
+|double|double
+|def|def
+|====
+
+*Examples:*
+[source,Java]
+----
+int x = +1;  // declares the int variable x and sets it to positive 1
+long y = +x; // declares the long variable y and sets it to positive x (promoted to long from int)
+def z = +y;  // declares the def variable z and sets it to positive y
+byte z = +2; //ERROR: cannot implicitly downcast an int to a byte
+----
+
+==== Unary Negative
+
+Unary negative negates a numeric value using the minus operator.  Format starts with a minus followed by a numerical expression.
+
+*Grammar:*
+[source,ANTLR4]
+----
+unary_negative: '-' expression
+----
+
+A numeric promotion may occur during a unary negative operation.  A def type evaluated at run-time will follow the same promotion table at run-time following whatever type def represents.  Non-numeric expressions will result in an error.
+
+Promotion Table:
+|====
+|from|to
+|byte|int
+|short|int
+|char|int
+|int|int
+|long|long
+|float|float
+|double|double
+|def|def
+|====
+
+*Examples:*
+[source,Java]
+----
+int x = -1;  // declares the int variable x and sets it to negative 1
+long y = -x; // declares the long variable y and sets it to negative x (promoted to long from int)
+def z = -y;  // declares the def variable z and sets it to negative y
+byte z = -2; //ERROR: cannot implicitly downcast an int to a byte
+----
+
+==== Boolean Not
+
+Boolean not will flip a boolean value from true to false or false to true using the bang operator.  The format is a bang operator followed by an expression.
+
+*Grammar:*
+[source,ANTLR4]
+----
+boolean_not: '!' expression;
+----
+
+Note that def types will be assumed to be of the boolean type.  Any def type evaluated at run-time that does not represent a boolean will result in an error.  Non-boolean expressions will result in an error.
+
+*Examples:*
+[source,Java]
+----
+boolean x = !false; // declares the boolean variable x and sets it to the the opposite of the false value
+boolean y = !x;     // declares the boolean variable y and sets it to the opposite of the boolean variable x
+def z = !y;         // declares the def variable z and sets it to the opposite of the boolean variable y
+----
+
+==== Bitwise Not
+
+Bitwise not will flip each bit of an integer type expression.  The format is the tilde operator followed by an expression.
+
+*Grammar:*
+[source,ANTLR4]
+----
+bitwise_not: '~' expression;
+----
+
+A numeric promotion may occur during unary positive operation.  A def type evaluated at run-time will follow the same promotion table at run-time following whatever type def represents.  Non-integer expressions will result in an error.
+
+Promotion Table:
+|====
+|from|to
+|byte|int
+|short|int
+|char|int
+|int|int
+|long|long
+|def|def
+|====
+
+*Examples:*
+[source,Java]
+----
+byte x = 1;  // declares the byte variable x and sets it to a constant int 1
+int y = ~x;  // declares the int variable y and sets it to the negation of x
+long z = ~y; // declares the long variable z and sets it the negation of y
+def d = ~z;  // declares the def variable d and sets it the negation of z
+def e;       // declares the def variable e
+e = ~d;      // sets e the negation of d
+----
+
+==== Cast
+
+The cast operator can be used to explicitly convert one type to another.  See casting [MARK] for more information.
+
+==== Constructor Call
+
+A constructor call is a special type of method call [MARK] used to allocate a reference type instance using the new operator.  The format is the new operator followed by a type, an opening parenthesis, arguments if any, and a closing parenthesis.  Arguments are a series of zero-to-many expressions delimited by commas.  Auto-boxing and auto-unboxing will be applied automatically for arguments passed into a constructor call.  See boxing and unboxing [MARK] for more information on this topic.  Constructor argument types can always be resolved at run-time; if appropriate type conversions (casting) cannot be applied an error will occur.  Once a reference type instance has been allocated, its members may be used as part of other expressions.
+
+Constructor calls may be overloaded based on arity in Painless.  This means the same reference type may have multiple constructors as long as the number of arguments differs for each one.  This does have an effect on some of the provided reference type constructors in the Painless API [MARK].  When there are overloaded constructors with the same arity for a reference type in Java a single constructor must be chosen to be provided in Painless.
+
+*Grammar:*
+[source,ANTLR4]
+----
+constructor_call: 'new' TYPE '(' (expression (',' expression)*)? ')';
+----
+
+*Examples:*
+[source,Java]
+----
+Map m = new HashMap();   // declares the Map variable m and sets it to a newly allocated HashMap using an empty constructor
+m.put(3, 3);             // invokes the method call member put and adds the key-value pair of 3 to Map variable m
+def d = new ArrayList(); // declares the def variable d and sets it to a newly allocated ArrayList using an empty constructor
+def e;                   // declares the def variable e
+e = new HashMap(m);      // sets e to a newly allocated HashMap using the constructor with a single argument m
+----
+
+==== New Array
+
+An array type instance can be allocated using the new operator. The format starts with the new operator followed by the type followed by a series of opening and closing braces each containing an expression for the size of the dimension.
+
+*Grammar:*
+[source,ANTLR4]
+----
+new_array: 'new' TYPE ('[' expression ']')+;
+----
+
+*Examples:*
+[source,Java]
+----
+int[] x = new int[5];      // declares an int array variable x and sets it to a newly allocated array with a size of 5
+x = new int[10];           // sets the int array variable x to a newly allocated array with a size of 10
+def[][] y = new def[5][5]; // declares a 2-dimensional def array variable y and set it to a newly
+                           // allocated 2-dimensional array where both dimensions have a size of 5
+----
+
+==== Multiplication
+
+Multiplies two numerical expressions.  Rules for resultant overflow and NaN values follow the Java specification.  The format is an expression, followed by the star operator, and a closing expression.
+
+*Grammar:*
+[source,ANTLR4]
+----
+multiplication: expression '*' expression;
+----
+
+A numeric promotion may occur during a multiplication operation.  A def type evaluated at run-time will follow the same promotion table at run-time following whatever type def represents. Non-numeric numbers will result in an error.
+
+Promotion Table:
+|====
+||byte|short|char|int|long|float|double|def
+|byte|int|int|int|int|long|float|double|def
+|short|int|int|int|int|long|float|double|def
+|char|int|int|int|int|long|float|double|def
+|int|int|int|int|int|long|float|double|def
+|long|long|long|long|long|long|float|double|def
+|float|float|float|float|float|float|float|double|def
+|double|double|double|double|double|double|double|double|def
+|def|def|def|def|def|def|def|def|def
+|====
+
+*Examples:*
+[source,Java]
+----
+int x = 5*4;      // declares the int variable x and sets it to the result of 5 multiplied by 4
+double y = x*7.0; // declares the double variable y and sets it to the result of x multiplied by 7.0 (x is promoted to a double)
+def z = x*y;      // declares the def variable z and sets it to the result of x multiplied by y (x is promoted to a double)
+def a = z*x;      // declares the def variable a and sets it to the result of z multiplied by x (x is promoted to def at compile-time and double at run-time)
+----
+
+==== Division
+
+Divides two numerical expressions.  Rules for NaN values and division by zero follow the Java specification.  Integer division will drop the remainder of the resultant value.  The format is an expression, followed by the slash operator, and a closing expression.
+
+*Grammar:*
+[source,ANTLR4]
+----
+division: expression '/' expression;
+----
+
+A numeric promotion may occur during a division operation.  A def type evaluated at run-time will follow the same promotion table at run-time following whatever type def represents. Non-numeric expressions will result in an error.
+
+Promotion Table:
+|====
+||byte|short|char|int|long|float|double|def
+|byte|int|int|int|int|long|float|double|def
+|short|int|int|int|int|long|float|double|def
+|char|int|int|int|int|long|float|double|def
+|int|int|int|int|int|long|float|double|def
+|long|long|long|long|long|long|float|double|def
+|float|float|float|float|float|float|float|double|def
+|double|double|double|double|double|double|double|double|def
+|def|def|def|def|def|def|def|def|def
+|====
+
+*Examples:*
+[source,Java]
+----
+int x = 5/4;      // declares the int variable x and sets it to the result of 5 divided by 4
+double y = x/7.0; // declares the double variable y and sets it to the result of x divided by 7.0 (x is promoted to a double)
+def z = x/y;      // declares the def variable z and sets it to the result of x divided by y (x is promoted to a double)
+def a = z/x;      // declares the def variable a and sets it to the result of z divided by x (x is promoted to def at compile-time and double at run-time)
+----
+
+==== Remainder
+
+Calculates the remainder for division between two numerical expressions.  Rules for NaN values and division by zero follow the Java specification.  The format is an expression, followed by the percent operator, and a closing expression.
+
+*Grammar:*
+[source,ANTLR4]
+----
+remainder: expression '%' expression;
+----
+
+A numeric promotion may occur during a remainder operation.  A def type evaluated at run-time will follow the same promotion table at run-time following whatever type def represents. Non-numeric expressions will result in an error.
+
+Promotion Table:
+|====
+||byte|short|char|int|long|float|double|def
+|byte|int|int|int|int|long|float|double|def
+|short|int|int|int|int|long|float|double|def
+|char|int|int|int|int|long|float|double|def
+|int|int|int|int|int|long|float|double|def
+|long|long|long|long|long|long|float|double|def
+|float|float|float|float|float|float|float|double|def
+|double|double|double|double|double|double|double|double|def
+|def|def|def|def|def|def|def|def|def
+|====
+
+*Examples:*
+[source,Java]
+----
+int x = 5%4;      // declares the int variable x and sets it to the remainder of 5 divided by 4
+double y = x%7.0; // declares the double variable y and sets it to the remainder of x divided by 7.0 (x is promoted to a double)
+def z = x%y;      // declares the def variable z and sets it to the remainder of x divided by y (x is promoted to a double)
+def a = z%x;      // declares the def variable a and sets it to the remainder of z divided by x (x is promoted to def at compile-time and double at run-time)
+----
+
+==== String Concatenation
+
+Concatenates two expressions together as a single String where at least of one of the expressions is a String to begin with.  The format is an expression, followed by a plus operator, and a closing expression.
+
+*Grammar:*
+[source,ANTLR4]
+----
+concatenate: expression '+' expression;
+----
+
+*Examples:*
+[source,Java]
+----
+String x = "con";          // declares the String variable x and sets it to the String constant "con"
+String y = x + "cat";      // declares the String variable y and sets it to the concatenation of the String variable x and the String constant "cat"
+String z = 4 + x;          // declares the String variable z and sets it to the concatenation of the int constant 4 and the String variable x (4 is implicitly cast to a String)
+def d = 2;                 // declares the def variable d and sets it to the int constant 2
+z = z + d;                 // sets the String variable z to the concatenation of the String variable z
+d = "con" + x + y + "cat"; // sets the def variable d to the concatenation of String constant "con", x, y, and the String constant "cat"
+----
+
+==== Addition
+
+Adds two numerical expressions.  Rules for resultant overflow and NaN values follow the Java specification.  The format is an expression, followed by the plus operator, and a closing expression.
+
+*Grammar:*
+[source,ANTLR4]
+----
+addition: expression '+' expression;
+----
+
+A numeric promotion may occur during a addition operation.  A def type evaluated at run-time will follow the same promotion table at run-time following whatever type def represents. Non-numeric expressions will result in an error, except in the case of String which then implies the operation is string concatenation [MARK] rather than addition.
+
+Promotion Table:
+|====
+||byte|short|char|int|long|float|double|def
+|byte|int|int|int|int|long|float|double|def
+|short|int|int|int|int|long|float|double|def
+|char|int|int|int|int|long|float|double|def
+|int|int|int|int|int|long|float|double|def
+|long|long|long|long|long|long|float|double|def
+|float|float|float|float|float|float|float|double|def
+|double|double|double|double|double|double|double|double|def
+|def|def|def|def|def|def|def|def|def
+|====
+
+*Examples:*
+[source,Java]
+----
+int x = 5 + 4;      // declares the int variable x and sets it to the result of 5 added to 4
+double y = x + 7.0; // declares the double variable y and sets it to the result of x added to 7.0 (x is promoted to a double)
+def z = x + y;      // declares the def variable z and sets it to the result of x added to y (x is promoted to a double)
+def a = z + x;      // declares the def variable a and sets it to the result of z added to x (x is promoted to def at compile-time and double at run-time)
+----
+
+==== Subtraction
+
+Subtracts two numerical expressions.  Rules for resultant overflow and NaN values follow the Java specification.  The format is an expression, followed by the minus operator, and a closing expression.
+
+*Grammar:*
+[source,ANTLR4]
+----
+subtraction: expression '-' expression;
+----
+
+A numeric promotion may occur during a subtraction operation.  A def type evaluated at run-time will follow the same promotion table at run-time following whatever type def represents. Non-numeric expressions will result in an error.
+
+Promotion Table:
+|====
+||byte|short|char|int|long|float|double|def
+|byte|int|int|int|int|long|float|double|def
+|short|int|int|int|int|long|float|double|def
+|char|int|int|int|int|long|float|double|def
+|int|int|int|int|int|long|float|double|def
+|long|long|long|long|long|long|float|double|def
+|float|float|float|float|float|float|float|double|def
+|double|double|double|double|double|double|double|double|def
+|def|def|def|def|def|def|def|def|def
+|====
+
+*Examples:*
+[source,Java]
+----
+int x = 5-4;      // declares the int variable x and sets it to the result of 4 subtracted from 5
+double y = x-7.0; // declares the double variable y and sets it to the result of 7.0 subtracted from x (x is promoted to a double)
+def z = x-y;      // declares the def variable z and sets it to the result of y subtracted from x (x is promoted to a double)
+def a = z-x;      // declares the def variable a and sets it to the result of x subtracted from z (x is promoted to def at compile-time and double at run-time)
+----
+
+==== Left Shift
+
+Shifts lower order bits to higher order bits in the left-side expression by the distance specified in the right-side expression.  The format is an expression followed by two left-carrots, and a closing expression.
+
+*Grammar:*
+[source,ANTLR4]
+----
+left_shift: expression '<<' expression;
+----
+
+A numeric promotion may occur during a left shift operation to the left-side expression.  A def type evaluated at run-time will follow the same promotion table at run-time following whatever type def represents. Non-numeric and floating point expressions will result in an error.
+
+Promotion Table:
+|====
+|from|to
+|byte|int
+|short|int
+|char|int
+|int|int
+|long|long
+|def|def
+|====
+
+The right-side expression will be explicitly cast to an int value and truncated based on the promoted type of the left-side expression.  If the left-side expression is of type int then the lowest order 5-bits will be taken as the distance to shift from the right-side expression (0-31).  If the left-side expression is of type long then the lowest order 6-bits will be taken as the distance to shift from the right-side expression (0-63). Non-numeric and floating point expressions will result in an error.
+
+*Examples:*
+[source,Java]
+----
+int x = 5 << 4;  // declares the int variable x and sets it to the result of 5 left shifted by 4
+long y = x << 7; // declares the long variable y and sets it to the result of x left shifted by 7 (x is promoted to a long)
+def z = x << y;  // declares the def variable z and sets it to the result of x left shifted by y
+def a = z << x;  // declares the def variable a and sets it to the result of z left shifted by x
+----
+
+==== Right Shift
+
+Shifts higher order bits to lower order bits in the left-side expression by the distance specified in the right-side expression.  Right shift will preserve the signed bit (highest order bit) as part of the result.  The format is an expression followed by two right-carrots, and a closing expression.
+
+*Grammar:*
+[source,ANTLR4]
+----
+right_shift: expression '>>' expression;
+----
+
+A numeric promotion may occur during a right shift operation to the left-side expression.  A def type evaluated at run-time will follow the same promotion table at run-time following whatever type def represents. Non-numeric and floating point expressions will result in an error.
+
+Promotion Table:
+|====
+|from|to
+|byte|int
+|short|int
+|char|int
+|int|int
+|long|long
+|def|def
+|====
+
+The right-side expression will be explicitly cast to an int value and truncated based on the promoted type of the left-side expression.  If the left-side expression is of type int then the lowest order 5-bits will be taken as the distance to shift from the right-side expression (0-31).  If the left-side expression is of type long then the lowest order 6-bits will be taken as the distance to shift from the right-side expression (0-63). Non-numeric and floating point expressions will result in an error.
+
+*Examples:*
+[source,Java]
+----
+int x = 5 >> 4;  // declares the int variable x and sets it to the result of 5 right shifted by 4
+long y = x >> 7; // declares the long variable y and sets it to the result of x right shifted by 7 (x is promoted to a long)
+def z = x >> y;  // declares the def variable z and sets it to the result of x right shifted by y
+def a = z >> x;  // declares the def variable a and sets it to the result of z right shifted by x
+----
+
+==== Unsigned Right Shift
+
+Shifts higher order bits to lower order bits in the left-side expression by the distance specified in the right-side expression.  Unsigned right shift will not preserve the signed bit (highest order bit) as part of the result.  The format is an expression followed by three right-carrots, and a closing expression.
+
+*Grammar:*
+[source,ANTLR4]
+----
+unsigned_right_shift: expression '>>>' expression;
+----
+
+A numeric promotion may occur during an unsigned right shift operation to the left-side expression.  A def type evaluated at run-time will follow the same promotion table at run-time following whatever type def represents. Non-numeric and floating point expressions will result in an error.
+
+Promotion Table:
+|====
+|from|to
+|byte|int
+|short|int
+|char|int
+|int|int
+|long|long
+|def|def
+|====
+
+The right-side expression will be explicitly cast to an int value and truncated based on the promoted type of the left-side expression.  If the left-side expression is of type int then the lowest order 5-bits will be taken as the distance to shift from the right-side expression (0-31).  If the left-side expression is of type long then the lowest order 6-bits will be taken as the distance to shift from the right-side expression (0-63). Non-numeric and floating point expressions will result in an error.
+
+*Examples:*
+[source,Java]
+----
+int x = 5 >> 4;  // declares the int variable x and sets it to the result of 5 unsigned right shifted by 4
+long y = x >> 7; // declares the long variable y and sets it to the result of x unsigned right shifted by 7 (x is promoted to a long)
+def z = x >> y;  // declares the def variable z and sets it to the result of x unsigned right shifted by y
+def a = z >> x;  // declares the def variable a and sets it to the result of z unsigned right shifted by x
+----
+
+==== Greater Than
+
+Greater than compares two numerical expressions where a resultant boolean value will be true if the left-side expression is a larger value than the right-side expression otherwise false.  The format is an expression, followed by the right angle operator, and a closing expression.
+
+*Grammar:*
+[source,ANTLR4]
+----
+greater_than: expression '>' expression;
+----
+
+A numeric promotion may occur during a greater than operation.  A def type evaluated at run-time will follow the same promotion table at run-time following whatever type def represents. Non-numeric expressions will result in an error.
+
+Promotion Table:
+|====
+||byte|short|char|int|long|float|double|def
+|byte|int|int|int|int|long|float|double|def
+|short|int|int|int|int|long|float|double|def
+|char|int|int|int|int|long|float|double|def
+|int|int|int|int|int|long|float|double|def
+|long|long|long|long|long|long|float|double|def
+|float|float|float|float|float|float|float|double|def
+|double|double|double|double|double|double|double|double|def
+|def|def|def|def|def|def|def|def|def
+|====
+
+*Examples:*
+[source,Java]
+----
+boolean x = 5 > 4; // declares the int variable x and sets it to the result of 5 greater than 4
+double y = 7.0;    // declares the double variable y and sets it to the double constant 7.0
+def z = y > 6.5;   // declares the def variable z and sets it to the result of y greater than 6.5
+def a = y > x;     // declares the def variable a and sets it to the result of y greater than z (x is promoted to double at compile-time)
+----
+
+==== Greater Than Or Equal
+
+Greater than or equal compares two numerical expressions where a resultant boolean value will be true if the left-side expression is a larger value than or equal to the right-side expression otherwise false.  The format is an expression, followed by the right angle and equals operator, and a closing expression.
+
+*Grammar:*
+[source,ANTLR4]
+----
+greater_than_or_equal: expression '>=' expression;
+----
+
+A numeric promotion may occur during a greater than or equal operation.  A def type evaluated at run-time will follow the same promotion table at run-time following whatever type def represents. Non-numeric expressions will result in an error.
+
+Promotion Table:
+|====
+||byte|short|char|int|long|float|double|def
+|byte|int|int|int|int|long|float|double|def
+|short|int|int|int|int|long|float|double|def
+|char|int|int|int|int|long|float|double|def
+|int|int|int|int|int|long|float|double|def
+|long|long|long|long|long|long|float|double|def
+|float|float|float|float|float|float|float|double|def
+|double|double|double|double|double|double|double|double|def
+|def|def|def|def|def|def|def|def|def
+|====
+
+*Examples:*
+[source,Java]
+----
+boolean x = 5 >= 4; // declares the int variable x and sets it to the result of 5 greater than or equal to 4
+double y = 7.0;     // declares the double variable y and sets it to the double constant 7.0
+def z = y >= 6.5;   // declares the def variable z and sets it to the result of y greater than or equal to 6.5
+def a = y >= x;     // declares the def variable a and sets it to the result of y greater than or equal to z (x is promoted to double at compile-time)
+----
+
+==== Less Than
+
+Less than compares two numerical expressions where a resultant boolean value will be true if the left-side expression is a smaller value than the right-side expression otherwise false.  The format is an expression, followed by the left angle operator, and a closing expression.
+
+*Grammar:*
+[source,ANTLR4]
+----
+less_than: expression '<' expression;
+----
+
+A numeric promotion may occur during a less than operation.  A def type evaluated at run-time will follow the same promotion table at run-time following whatever type def represents. Non-numeric expressions will result in an error.
+
+Promotion Table:
+|====
+||byte|short|char|int|long|float|double|def
+|byte|int|int|int|int|long|float|double|def
+|short|int|int|int|int|long|float|double|def
+|char|int|int|int|int|long|float|double|def
+|int|int|int|int|int|long|float|double|def
+|long|long|long|long|long|long|float|double|def
+|float|float|float|float|float|float|float|double|def
+|double|double|double|double|double|double|double|double|def
+|def|def|def|def|def|def|def|def|def
+|====
+
+*Examples:*
+[source,Java]
+----
+boolean x = 5 < 4; // declares the int variable x and sets it to the result of 5 less than 4
+double y = 7.0;    // declares the double variable y and sets it to the double constant 7.0
+def z = y < 6.5;   // declares the def variable z and sets it to the result of y less than 6.5
+def a = y < x;     // declares the def variable a and sets it to the result of y less than z (x is promoted to double at compile-time)
+----
+
+==== Less Than Or Equal
+
+Less than or equal compares two numerical expressions where a resultant boolean value will be true if the left-side expression is a larger value than or equal to the right-side expression otherwise false.  The format is an expression, followed by the left angle and equals operator, and a closing expression.
+
+*Grammar:*
+[source,ANTLR4]
+----
+less_than_or_equal: expression '<=' expression;
+----
+
+A numeric promotion may occur during a less than or equal operation.  A def type evaluated at run-time will follow the same promotion table at run-time following whatever type def represents. Non-numeric expressions will result in an error.
+
+Promotion Table:
+|====
+||byte|short|char|int|long|float|double|def
+|byte|int|int|int|int|long|float|double|def
+|short|int|int|int|int|long|float|double|def
+|char|int|int|int|int|long|float|double|def
+|int|int|int|int|int|long|float|double|def
+|long|long|long|long|long|long|float|double|def
+|float|float|float|float|float|float|float|double|def
+|double|double|double|double|double|double|double|double|def
+|def|def|def|def|def|def|def|def|def
+|====
+
+*Examples:*
+[source,Java]
+----
+boolean x = 5 <= 4; // declares the int variable x and sets it to the result of 5 less than or equal to 4
+double y = 7.0;     // declares the double variable y and sets it to the double constant 7.0
+def z = y <= 6.5;   // declares the def variable z and sets it to the result of y less than or equal to 6.5
+def a = y <= x;     // declares the def variable a and sets it to the result of y less than or equal to z (x is promoted to double at compile-time)
+----
+
+==== Instance Of
+
+The instanceof operator can be used to compare a variable's type to a specified reference type where a resultant boolean value is true if the variable type is the same as or a descendant of the specified reference type and false otherwise.  The format is an id, followed by the instanceof operator, and finished with a type.
+
+*Grammar:*
+[source,ANTLR4]
+----
+instance_of: ID 'instanceof' TYPE;
+----
+
+*Examples:*
+[source,Java]
+----
+Map x = new HashMap();            // declares the Map variable x and sets it to a newly allocated HashMap
+List y = new ArrayList();         // declares the List variable y and sets it to a newly allocated ArrayList
+def z = y;                        // declares the def variable z and sets it to y
+boolean a = x instanceof HashMap; // declares the boolean variable a and sets it to true since x's type is the same type as HashMap
+boolean b = y instanceof Map;     // declares the boolean variable b and sets it to false since y's type is not the same type as Map or a descendant of Map
+boolean c = z instanceof List;    // declares the boolean variable c and sets it to true since z's type is a descendant of the type List
+----
+
+==== Equality Equals
+
+Equality equals compares two expressions where a resultant boolean value is true if the two expressions are equal and false otherwise.  When reference types are compared using this operator the equivalent of the equals member method will be called against the first expression, where the second expression is the argument.  Though the equals member method is used for reference types, this operation will always be null-safe.  Valid comparisons are between boolean types, primitive numeric types, and reference types.  If a comparison is made that is not listed as one of the valid comparisons an error will occur.  The format is an expression, followed by the equals-equals operator, and finished with an expression.
+
+*Grammar:*
+[source,ANTLR4]
+----
+equality_equals: expression '==' expression;
+----
+
+A numeric type promotion may occur during a primitive numeric comparison.  A def type evaluated at run-time will follow the same promotion table at run-time following whatever type def represents.
+
+Promotion Table:
+|====
+||byte|short|char|int|long|float|double|def
+|byte|int|int|int|int|long|float|double|def
+|short|int|int|int|int|long|float|double|def
+|char|int|int|int|int|long|float|double|def
+|int|int|int|int|int|long|float|double|def
+|long|long|long|long|long|long|float|double|def
+|float|float|float|float|float|float|float|double|def
+|double|double|double|double|double|double|double|double|def
+|def|def|def|def|def|def|def|def|def
+|====
+
+*Examples:*
+[source,Java]
+----
+boolean b0 = true;              // declares the boolean variable b0 and sets it the constant boolean true
+boolean b1 = false;             // declares the boolean variable b1 and sets it the constant boolean false
+int i = 2;                      // declares the int variable i and sets it the constant int 2
+float f = 2.0f;                 // declares the float variable f and sets it the constant float 2.0
+List l0 = new ArrayList();      // declares the List variable l0 and sets it to a newly allocated ArrayList
+ArrayList l1 = new ArrayList(); // declares the ArrayList variable l1 and sets it to a newly allocated ArrayList
+def di0 = 2;                    // declares the def variable di0 and sets it the constant int 2
+def di1 = 3;                    // declares the def variable di1 and sets it the constant int 3
+def dl = new ArrayList();       // declares the def variable dl and sets it to a newly allocated ArrayList
+boolean result;                 // declares the boolean variable result
+
+result = b0 == b1;              // compares b0 to b1 and has a boolean result of false
+result = i == f;                // compares i to f where i is promoted to float and has a boolean result of true
+result = b0 == i;               // ERROR: a comparison between a boolean and a primitive numeric type is illegal
+result = i == l0;               // ERROR: a comparison between a primitive numeric type and a reference type is illegal
+
+l0.add(1);                      // adds a constant int 1 to the List l0
+l1.add(1);                      // adds a constant int 1 to the ArrayList l1
+result = l0 == l1;              // compares l0 to l1 using l0.equals(l1) and has a boolean result of true
+l0.add(1);                      // adds a constant int 1 to the List l0
+result = l0 == l1;              // compares l0 to l1 using l0.equals(l1) and has a boolean result of false
+
+result = di0 == di1;            // compares di0 to di1 and has a boolean result of false
+result = di0 == i;              // compares di0 to i where i is promoted to def and has a boolean result of true
+
+dl.add(1);                      // adds a constant int 1 to the def ArrayList dl
+result = dl == l0;              // compares dl to l0 using dl.equals(l0) with a boolean result of true
+
+result = null == dl;            // compares null to dl with a boolean result of false
+result = l1 == null;            // compares l1 to null with a boolean result of false
+----
+
+==== Equality Not Equals
+
+Equality not equals compares two expressions where a resultant boolean value is true if the two expressions are not equal and false otherwise.  When reference types are compared using this operator the equivalent of the equals member method will be called against the first expression, where the second expression is the argument, with the resultant boolean being reversed.  Though the equals member method is used for reference types, this operation will always be null-safe.  Valid comparisons are between boolean types, primitive numeric types, and reference types.  If a comparison is made that is not listed as one of the valid comparisons an error will occur.  The format is an expression, followed by the bang-equals operator, and finished with an expression.
+
+*Grammar:*
+[source,ANTLR4]
+----
+equality_not_equals: expression '!=' expression;
+----
+
+A numeric type promotion may occur during a primitive numeric comparison.  A def type evaluated at run-time will follow the same promotion table at run-time following whatever type def represents.
+
+Promotion Table:
+|====
+||byte|short|char|int|long|float|double|def
+|byte|int|int|int|int|long|float|double|def
+|short|int|int|int|int|long|float|double|def
+|char|int|int|int|int|long|float|double|def
+|int|int|int|int|int|long|float|double|def
+|long|long|long|long|long|long|float|double|def
+|float|float|float|float|float|float|float|double|def
+|double|double|double|double|double|double|double|double|def
+|def|def|def|def|def|def|def|def|def
+|====
+
+*Examples:*
+[source,Java]
+----
+boolean b0 = true;              // declares the boolean variable b0 and sets it the constant boolean true
+boolean b1 = false;             // declares the boolean variable b1 and sets it the constant boolean false
+int i = 2;                      // declares the int variable i and sets it the constant int 2
+float f = 2.0f;                 // declares the float variable f and sets it the constant float 2.0
+List l0 = new ArrayList();      // declares the List variable l0 and sets it to a newly allocated ArrayList
+ArrayList l1 = new ArrayList(); // declares the ArrayList variable l1 and sets it to a newly allocated ArrayList
+def di0 = 2;                    // declares the def variable di0 and sets it the constant int 2
+def di1 = 3;                    // declares the def variable di1 and sets it the constant int 3
+def dl = new ArrayList();       // declares the def variable dl and sets it to a newly allocated ArrayList
+boolean result;                 // declares the boolean variable result
+
+result = b0 != b1;              // compares b0 to b1 and has a boolean result of true
+result = i != f;                // compares i to f where i is promoted to float and has a boolean result of false
+result = b0 != i;               // ERROR: a comparison between a boolean and a primitive numeric type is illegal
+result = i != l0;               // ERROR: a comparison between a primitive numeric type and a reference type is illegal
+
+l0.add(1);                      // adds a constant int 1 to the List l0
+l1.add(1);                      // adds a constant int 1 to the ArrayList l1
+result = l0 != l1;              // compares l0 to l1 using l0.equals(l1) and has a boolean result of false
+l0.add(1);                      // adds a constant int 1 to the List l0
+result = l0 != l1;              // compares l0 to l1 using l0.equals(l1) and has a boolean result of true
+
+result = di0 != di1;            // compares di0 to di1 and has a boolean result of true
+result = di0 != i;              // compares di0 to i where i is promoted to def and has a boolean result of false
+
+dl.add(1);                      // adds a constant int 1 to the def ArrayList dl
+result = dl != l0;              // compares dl to l0 using dl.equals(l0) with a boolean result of false
+
+result = null != dl;            // compares null to dl with a boolean result of true
+result = l1 != null;            // compares null to l1 with a boolean result of true
+----
+
+==== Identity Equals
+
+Identity equals compares two expressions where a resultant boolean value is true if the two expressions are equal and false otherwise.  Two primitive types are considered to be equal if they have the same value.  Two reference types are considered to be equal if they refer to the exact same instance in memory or are both null.  Valid comparisons are between boolean types, primitive numeric types, and reference types.  If a comparison is made that is not listed as one of the valid comparisons an error will occur.  The format is an expression, followed by the equals-equals-equals operator, and finished with an expression.
+
+*Grammar:*
+[source,ANTLR4]
+----
+identity_equals: expression '===' expression;
+----
+
+A numeric type promotion may occur during a primitive numeric comparison.  A def type evaluated at run-time will follow the same promotion table at run-time following whatever type def represents.
+
+Promotion Table:
+|====
+||byte|short|char|int|long|float|double|def
+|byte|int|int|int|int|long|float|double|def
+|short|int|int|int|int|long|float|double|def
+|char|int|int|int|int|long|float|double|def
+|int|int|int|int|int|long|float|double|def
+|long|long|long|long|long|long|float|double|def
+|float|float|float|float|float|float|float|double|def
+|double|double|double|double|double|double|double|double|def
+|def|def|def|def|def|def|def|def|def
+|====
+
+*Examples:*
+[source,Java]
+----
+boolean b0 = true;              // declares the boolean variable b0 and sets it the constant boolean true
+boolean b1 = false;             // declares the boolean variable b1 and sets it the constant boolean false
+int i = 2;                      // declares the int variable i and sets it the constant int 2
+float f = 2.0f;                 // declares the float variable f and sets it the constant float 2.0
+List l0 = new ArrayList();      // declares the List variable l0 and sets it to a newly allocated ArrayList
+ArrayList l1 = new ArrayList(); // declares the ArrayList variable l1 and sets it to a newly allocated ArrayList
+List l2 = l1;                   // declares the List variable l2 and sets it to l1
+def di0 = 2;                    // declares the def variable di0 and sets it the constant int 2
+def di1 = 3;                    // declares the def variable di1 and sets it the constant int 3
+def dl = l0;                    // declares the def variable dl and sets it to l0
+boolean result;                 // declares the boolean variable result
+
+result = b0 === b1;             // compares b0 to b1 and has a boolean result of false
+result = i === f;               // compares i to f where i is promoted to float and has a boolean result of true
+result = b0 === i;              // ERROR: a comparison between a boolean and a primitive numeric type is illegal
+result = i === l0;              // ERROR: a comparison between a primitive numeric type and a reference type is illegal
+
+l0.add(1);                      // adds a constant int 1 to the List l0
+l1.add(1);                      // adds a constant int 1 to the ArrayList l1
+result = l0 === l1;             // compares l0 to l1 and has a boolean result of false
+l0.add(1);                      // adds a constant int 1 to the List l0
+result = l0 === l1;             // compares l0 to l1 and has a boolean result of false
+result = l1 === l2;             // compares l1 to l2 and has a boolean result of true
+
+result = di0 === di1;           // compares di0 to di1 and has a boolean result of false
+result = di0 === i;             // compares di0 to i where i is promoted to def and has a boolean result of true
+
+result = dl === l0;             // compares dl to l0 with a boolean result of true
+
+result = null === dl;           // compares null to dl with a boolean result of false
+result = l1 === null;           // compares null to l1 with a boolean result of false
+----
+
+==== Identity Not Equals
+
+Identity not equals compares two expressions where a resultant boolean value is true if the two expressions are not equal and false otherwise.  Two primitive types are considered to be not equal if they have different values.  Two reference types are considered to be not equal if they refer to the different instances in memory or one is null and the other is not.  Valid comparisons are between boolean types, primitive numeric types, and reference types.  If a comparison is made that is not listed as one of the valid comparisons an error will occur.  The format is an expression, followed by the bang-equals-equals operator, and finished with an expression.
+
+*Grammar:*
+[source,ANTLR4]
+----
+identity_not_equals: expression '!==' expression;
+----
+
+A numeric type promotion may occur during a primitive numeric comparison.  A def type evaluated at run-time will follow the same promotion table at run-time following whatever type def represents.
+
+Promotion Table:
+|====
+||byte|short|char|int|long|float|double|def
+|byte|int|int|int|int|long|float|double|def
+|short|int|int|int|int|long|float|double|def
+|char|int|int|int|int|long|float|double|def
+|int|int|int|int|int|long|float|double|def
+|long|long|long|long|long|long|float|double|def
+|float|float|float|float|float|float|float|double|def
+|double|double|double|double|double|double|double|double|def
+|def|def|def|def|def|def|def|def|def
+|====
+
+*Examples:*
+[source,Java]
+----
+boolean b0 = true;              // declares the boolean variable b0 and sets it the constant boolean true
+boolean b1 = false;             // declares the boolean variable b1 and sets it the constant boolean false
+int i = 2;                      // declares the int variable i and sets it the constant int 2
+float f = 2.0f;                 // declares the float variable f and sets it the constant float 2.0
+List l0 = new ArrayList();      // declares the List variable l0 and sets it to a newly allocated ArrayList
+ArrayList l1 = new ArrayList(); // declares the ArrayList variable l1 and sets it to a newly allocated ArrayList
+List l2 = l1;                   // declares the List variable l2 and sets it to l1
+def di0 = 2;                    // declares the def variable di0 and sets it the constant int 2
+def di1 = 3;                    // declares the def variable di1 and sets it the constant int 3
+def dl = l0;                    // declares the def variable dl and sets it to l0
+boolean result;                 // declares the boolean variable result
+
+result = b0 !== b1;             // compares b0 to b1 and has a boolean result of true
+result = i !== f;               // compares i to f where i is promoted to float and has a boolean result of false
+result = b0 !== i;              // ERROR: a comparison between a boolean and a primitive numeric type is illegal
+result = i !== l0;              // ERROR: a comparison between a primitive numeric type and a reference type is illegal
+
+l0.add(1);                      // adds a constant int 1 to the List l0
+l1.add(1);                      // adds a constant int 1 to the ArrayList l1
+result = l0 !== l1;             // compares l0 to l1 and has a boolean result of true
+l0.add(1);                      // adds a constant int 1 to the List l0
+result = l0 !== l1;             // compares l0 to l1 and has a boolean result of true
+result = l1 !== l2;             // compares l1 to l2 and has a boolean result of false
+
+result = di0 !== di1;           // compares di0 to di1 and has a boolean result of true
+result = di0 !== i;             // compares di0 to i where i is promoted to def and has a boolean result of false
+
+result = dl !== l0;             // compares dl to l0 with a boolean result of false
+
+result = null !== dl;           // compares null to dl with a boolean result of true
+result = l1 !== null;           // compares null to l1 with a boolean result of true
+----
+
+==== Bitwise And
+
+Bitwise and will and together two integer type expressions.  The table below shows what each resultant bit will in the resultant integer type value be based on the corresponding bit in each integer type expression.
+
+|====
+||1|0
+|1|1|0
+|0|0|0
+|====
+
+The format starts with an expression, follows with the ampersand operator, and finishes with an expression.
+
+*Grammar:*
+[source,ANTLR4]
+----
+bitwise_and: expression '&' expression;
+----
+
+A numeric promotion may occur during a bitwise and operation.  A def type evaluated at run-time will follow the same promotion table at run-time following whatever type def represents.  Non-integer expressions will result in an error.
+
+Promotion Table:
+|====
+||byte|short|char|int|long|def
+|byte|int|int|int|int|long|def
+|short|int|int|int|int|long|def
+|char|int|int|int|int|long|def
+|int|int|int|int|int|long|def
+|long|long|long|long|long|long|def
+|def|def|def|def|def|def|def|def|def
+|====
+
+*Examples:*
+[source,Java]
+----
+byte x = 16;    // declares the byte variable x and sets it to a constant int 1
+int y = x & 4;  // declares the int variable y and sets it to the result of x and 4
+long z = y & x; // declares the long variable z and sets it the result of y and x
+def d = z & 2;  // declares the def variable d and sets it the result of z and 2
+def e;          // declares the def variable e
+e = d & z;      // sets e to the result of d and z
+----
+
+==== Boolean Xor
+
+Boolean xor will xor together two boolean expressions.  The table below shows what the resultant boolean value will be based on the two boolean expressions.
+
+|====
+||true|false
+|true|false|true
+|false|true|false
+|====
+
+The format starts with an expression, follows with the carrot operator, and finishes with an expression.
+
+*Grammar:*
+[source,ANTLR4]
+----
+boolean_xor: expression '^' expression;
+----
+
+Note that def types will be assumed to be of the boolean type.  Any def type evaluated at run-time that does not represent a boolean will result in an error.  Non-boolean expressions will result in an error.
+
+*Examples:*
+[source,Java]
+----
+boolean x = false;    // declares the boolean variable x and sets the constant boolean false
+boolean y = x ^ true; // declares the boolean variable y and sets it the result of x xor true
+def z = y ^ x;        // declares the def variable z and sets it to the result of y xor x
+----
+
+==== Bitwise Xor
+
+Bitwise xor will xor together two integer type expressions.  The table below shows what each resultant bit will in the resultant integer type value be based on the corresponding bit in each integer type expression.
+
+|====
+||1|0
+|1|0|1
+|0|1|0
+|====
+
+The format starts with an expression, follows with the carrot operator, and finishes with an expression.
+
+*Grammar:*
+[source,ANTLR4]
+----
+bitwise_xor: expression '^' expression;
+----
+
+A numeric promotion may occur during a bitwise xor operation.  A def type evaluated at run-time will follow the same promotion table at run-time following whatever type def represents.  Non-integer expressions will result in an error.
+
+Promotion Table:
+|====
+||byte|short|char|int|long|def
+|byte|int|int|int|int|long|def
+|short|int|int|int|int|long|def
+|char|int|int|int|int|long|def
+|int|int|int|int|int|long|def
+|long|long|long|long|long|long|def
+|def|def|def|def|def|def|def|def|def
+|====
+
+*Examples:*
+[source,Java]
+----
+byte x = 16;    // declares the byte variable x and sets it to a constant int 1
+int y = x ^ 4;  // declares the int variable y and sets it to the result of x xor 4
+long z = y ^ x; // declares the long variable z and sets it the result of y xor x
+def d = z ^ 2;  // declares the def variable d and sets it the result of z xor 2
+def e;          // declares the def variable e
+e = d ^ z;      // sets e to the result of d xor z
+----
+
+==== Bitwise Or
+
+Bitwise or will or together two integer type expressions.  The table below shows what each resultant bit will in the resultant integer type value be based on the corresponding bit in each integer type expression.
+
+|====
+||1|0
+|1|1|1
+|0|1|0
+|====
+
+The format starts with an expression, follows with the pipe operator, and finishes with an expression.
+
+*Grammar:*
+[source,ANTLR4]
+----
+bitwise_or: expression '|' expression;
+----
+
+A numeric promotion may occur during a bitwise xor operation.  A def type evaluated at run-time will follow the same promotion table at run-time following whatever type def represents.  Non-integer expressions will result in an error.
+
+Promotion Table:
+|====
+||byte|short|char|int|long|def
+|byte|int|int|int|int|long|def
+|short|int|int|int|int|long|def
+|char|int|int|int|int|long|def
+|int|int|int|int|int|long|def
+|long|long|long|long|long|long|def
+|def|def|def|def|def|def|def|def|def
+|====
+
+*Examples:*
+[source,Java]
+----
+byte x = 16;    // declares the byte variable x and sets it to a constant int 1
+int y = x | 4;  // declares the int variable y and sets it to the result of x or 4
+long z = y | x; // declares the long variable z and sets it the result of y or x
+def d = z | 2;  // declares the def variable d and sets it the result of z or 2
+def e;          // declares the def variable e
+e = d | z;      // sets e to the result of d or z
+----
+
+==== Boolean And
+
+Boolean and will and together two boolean expressions.  If the first expression is found to be false then it is known that the result will also be false, so evaluation of the second expression will be skipped.  The table below shows what the resultant boolean value will be based on the two boolean expressions.
+
+||true|false
+|true|true|false
+|false|false|false
+
+The format starts with an expression, follows with the ampersand-ampersand operator, and finishes with an expression.
+
+*Grammar:*
+[source,ANTLR4]
+----
+boolean_and: expression '&&' expression;
+----
+
+Note that def types will be assumed to be of the boolean type.  Any def type evaluated at run-time that does not represent a boolean will result in an error.  Non-boolean expressions will result in an error.
+
+*Examples:*
+[source,Java]
+----
+boolean x = false;     // declares the boolean variable x and sets the constant boolean false
+boolean y = x && true; // declares the boolean variable y and sets it the result of x and true
+def z = y && x;        // declares the def variable z and sets it to the result of y and x
+----
+
+==== Boolean Or
+
+Boolean or will or together two boolean expressions.  If the first expression is found to be true then it is known that the result will also be true, so evaluation of the second expression will be skipped.  The table below shows what the resultant boolean value will be based on the two boolean expressions.
+
+|====
+||true|false
+|true|true|true
+|false|true|false
+|====
+
+The format starts with an expression, follows with the pipe-pipe operator, and finishes with an expression.
+
+*Grammar:*
+[source,ANTLR4]
+----
+boolean_and: expression '||' expression;
+----
+
+Note that def types will be assumed to be of the boolean type.  Any def type evaluated at run-time that does not represent a boolean will result in an error.  Non-boolean expressions will result in an error.
+
+*Examples:*
+[source,Java]
+----
+boolean x = false;     // declares the boolean variable x and sets the constant boolean false
+boolean y = x || true; // declares the boolean variable y and sets it the result of x or true
+def z = y || x;        // declares the def variable z and sets it to the result of y or x
+----
+
+==== Conditional
+
+A conditional operation consists of three expressions.  The first expression is evaluated with an expected boolean result type.  If the first expression evaluates to true then the second expression will be evaluated.  If the first expression evaluates to false then the third expression will be evaluated.  This can be used as a shortcut many different operations without requiring a full if/else branch.  Errors will occur if the first expression does not evaluate to a boolean type or if one of the second or third expression cannot be converted to a type appropriate for the expected result.  The format is an expression followed by a question-mark operator, another expression, a colon operator, and finishes with a final expression.
+
+*Grammar:*
+[source,ANTLR4]
+----
+conditional: expression '?' expression ':' expression;
+----
+
+A numeric type promotion may occur during the evaluation of a conditional with the second and third expressions if the expected result is a numeric type.  A def type evaluated at run-time will follow the same promotion table at run-time following whatever type def represents.
+
+Promotion Table:
+|====
+||byte|short|char|int|long|float|double|def
+|byte|int|int|int|int|long|float|double|def
+|short|int|int|int|int|long|float|double|def
+|char|int|int|int|int|long|float|double|def
+|int|int|int|int|int|long|float|double|def
+|long|long|long|long|long|long|float|double|def
+|float|float|float|float|float|float|float|double|def
+|double|double|double|double|double|double|double|double|def
+|def|def|def|def|def|def|def|def|def
+|====
+
+*Examples:*
+[source,Java]
+----
+boolean b = true;                        // declares the boolean variable b and sets it the constant boolean true
+
+int x = b ? 1 : 2;                       // declares the int variable x and sets it to the int constant 1
+                                         // since the first expression of the conditional evaluates to true
+                                         // so the second expression is evaluated for a result
+
+List y = x > 1 ? new ArrayList() : null; // declares the List variable y and sets it to null
+                                         // since the first expression of the conditional evaluates to false
+                                         // so the third expression is evaluated for a result
+
+def z = x < 2 ? true : false;            // declares the def variable z and sets it to the boolean constant true
+                                         // since the first expression of the conditional evaluates to true
+                                         // so the second expression is evaluated for a result
+----
+
+==== Elvis
+
+The elvis operator consists of two expressions.  If the first expression is a non-null value then the resultant value will be the evaluated first expression otherwise the resultant value will be the evaluated second expression.  This is typically used as a shortcut for a null check in a conditional.  An error will occur if the expected result is a primitive type.  The format is an expression, followed by the question-mark-colon operator, and finishes with an expression.
+
+*Grammar:*
+[source,ANTLR4]
+----
+elvis: expression '?:' expression;
+----
+
+*Examples:*
+[source,Java]
+----
+List l = new ArrayList();      // declares the List variable l and sets it to a newly allocated ArrayList
+List y = l ?: new ArrayList(); // declares the List variable y and sets it to l since l is not null
+y = null;                      // sets y to null
+def z = y ?: new HashMap();    // declares the def variable z and sets it to a newly allocated HashMap since y is null
+----
+
+==== Assignment
+
+Assignment can be used to assign a value to a variable.  See Variable Assignment [MARK] for more information.
+
+==== Compound Assignment
+
+Compound assignment can be used as a shortcut for an assignment where a binary operation would occur between the variable/field as the left-side expression and a separate right-side expression.  The variable/field and right-side expression must be of appropriate types for the specific operation or an error will occur.  A downcast may be necessary for certain operations to be able to assign the result back into the variable/field and will happen implicitly.  The format is a variable/field, followed by one of the compound assignment operators, finished with an expression.
+
+*Grammar:*
+[source,ANTLR4]
+----
+compund_assignment: ID (. ID)? '$=' expression; // $ is a placeholder for the operation symbol
+----
+
+A compound assignment is equivalent to the expression below where V is the variable/field and T is the type of variable/member.
+
+[source,Java]
+----
+V = (T)(V op expression);
+----
+
+The table below shows all available operators for compound assignment.  All operators follow any casting/promotion rules according to their regular definition.
+
+|====
+|Operator|Compound Symbol
+|Multiplication|*=
+|Division|/=
+|Remainder|%=
+|String Concatenation|+=
+|Addition|+=
+|Subtraction|-=
+|Left Shift|<<=
+|Right Shift|>>=
+|Unsigned Right Shift|>>>=
+|Bitwise And|&=
+|Boolean And|&=
+|Bitwise Xor|^=
+|Boolean Xor|^=
+|Bitwise Or|\|=
+|Boolean Or|\|=
+|====
+
+*Examples:*
+[source,Java]
+----
+int i = 10;         // declares the variable i and sets it to constant int 10
+i *= 2;             // multiplies i by 2 -- i = (int)(i * 2)
+i /= 5;             // divides i by 5 -- i = (int)(i / 5)
+i %= 3;             // gives the remainder for i/3 -- i = (int)(i % 3)
+i += 5;             // adds 5 to i -- i = (int)(i + 5)
+i -= 5;             // subtracts 5 from i -- i = (int)(i - 5)
+i <<= 2;            // left shifts i by 2 -- i = (int)(i << 2)
+i >>= 1;            // right shifts i by 1 -- i = (int)(i >> 1)
+i >>>= 1;           // unsigned right shifts i by 1 -- i = (int)(i >>> 1)
+i &= 15;            // ands i with 15 -- i = (int)(i & 15)
+i ^= 12;            // xors i with 12 -- i = (int)(i ^ 2)
+i |= 4;             // ors i with 4 -- i = (int)(i | 4)
+
+boolean b = true;   // declares the boolean variable b and sets it to the constant boolean true
+b &= false;         // ands b with false -- b = (boolean)(b & false)
+b ^= false;         // xors b with false -- b = (boolean)(b & false)
+b |= true;          // ors be with true -- b = (boolean)(b & false)
+
+def x = 'compound'; // declares the def variable x and sets it to the constant String 'compound'
+x += ' assignment'; // string concatenates ' assignment' to x -- x = (String)(x + ' assignment')
+----
diff --git a/docs/painless/painless-syntax.asciidoc b/docs/painless/painless-syntax.asciidoc
new file mode 100644
index 0000000000000..79e830c05d21c
--- /dev/null
+++ b/docs/painless/painless-syntax.asciidoc
@@ -0,0 +1,119 @@
+[[painless-syntax]]
+=== Painless Syntax
+
+experimental[The Painless scripting language is new and is still marked as experimental. The syntax or API may be changed in the future in non-backwards compatible ways if required.]
+
+[float]
+[[control-flow]]
+==== Control flow
+
+Painless supports all of Java's https://docs.oracle.com/javase/tutorial/java/nutsandbolts/flow.html[
+control flow statements] except the `switch` statement.
+
+Painless also supports the `for in` syntax from Groovy:
+
+[source,painless]
+---------------------------------------------------------
+for (item : list) {
+  ...
+}
+---------------------------------------------------------
+
+[float]
+[[functions]]
+==== Functions
+
+You can declare functions at the beginning of a Painless script, for example:
+
+[source,painless]
+---------------------------------------------------------
+boolean isNegative(def x) { x < 0 }
+...
+if (isNegative(someVar)) {
+  ...
+}
+---------------------------------------------------------
+
+[float]
+[[lambda-expressions]]
+==== Lambda expressions
+Lambda expressions and method references work the same as in https://docs.oracle.com/javase/tutorial/java/javaOO/lambdaexpressions.html[Java].
+
+[source,painless]
+---------------------------------------------------------
+list.removeIf(item -> item == 2);
+list.removeIf((int item) -> item == 2);
+list.removeIf((int item) -> { item == 2 });
+list.sort((x, y) -> x - y);
+list.sort(Integer::compare);
+---------------------------------------------------------
+
+You can make method references to functions within the script with `this`,
+for example `list.sort(this::mycompare)`.
+
+[float]
+[[patterns]]
+==== Patterns
+
+Regular expression constants are directly supported. To ensure fast performance,
+this is the only mechanism for creating patterns. Regular expressions
+are always constants and compiled efficiently a single time.
+
+[source,painless]
+---------------------------------------------------------
+Pattern p = /[aeiou]/
+---------------------------------------------------------
+
+[float]
+[[pattern-flags]]
+===== Pattern flags
+
+You can define flags on patterns in Painless by adding characters after the
+trailing `/` like `/foo/i` or `/foo \w #comment/iUx`. Painless exposes all of
+the flags from Java's
+https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html[
+Pattern class] using these characters:
+
+[cols="<,<,<",options="header",]
+|=======================================================================
+| Character | Java Constant | Example
+|`c` | CANON_EQ                  | `'å' ==~ /å/c` (open in hex editor to see)
+|`i` | CASE_INSENSITIVE          | `'A' ==~ /a/i`
+|`l` | LITERAL                   | `'[a]' ==~ /[a]/l`
+|`m` | MULTILINE                 | `'a\nb\nc' =~ /^b$/m`
+|`s` | DOTALL (aka single line)  | `'a\nb\nc' =~ /.b./s`
+|`U` | UNICODE_CHARACTER_CLASS   | `'Ɛ' ==~ /\\w/U`
+|`u` | UNICODE_CASE              | `'Ɛ' ==~ /ɛ/iu`
+|`x` | COMMENTS (aka extended)   | `'a' ==~ /a #comment/x`
+|=======================================================================
+
+[float]
+[[painless-deref]]
+==== Dereferences
+
+Like lots of languages, Painless uses `.` to reference fields and call methods:
+
+[source,painless]
+---------------------------------------------------------
+String foo = 'foo';
+TypeWithGetterOrPublicField bar = new TypeWithGetterOrPublicField()
+return foo.length() + bar.x
+---------------------------------------------------------
+
+Like Groovy, Painless uses `?.` to perform null-safe references, with the
+result being `null` if the left hand side is `null`:
+
+[source,painless]
+---------------------------------------------------------
+String foo = null;
+return foo?.length()  // Returns null
+---------------------------------------------------------
+
+Unlike Groovy, Painless doesn't support writing to `null` values with this
+operator:
+
+[source,painless]
+---------------------------------------------------------
+TypeWithSetterOrPublicField foo = null;
+foo?.x = 'bar'  // Compile error
+---------------------------------------------------------
diff --git a/docs/painless/painless-types.asciidoc b/docs/painless/painless-types.asciidoc
new file mode 100644
index 0000000000000..36cf78312ea26
--- /dev/null
+++ b/docs/painless/painless-types.asciidoc
@@ -0,0 +1,442 @@
+[[types]]
+=== Data Types
+
+Painless supports both dynamic and static types. Static types are split into
+_primitive types_ and _reference types_.
+
+[[dynamic-types]]
+==== Dynamic Types
+
+Painless supports one dynamic type: `def`.  The `def` type can represent any
+primitive or reference type. When you use the `def` type, it mimics the exact
+behavior of whatever type it represents at runtime. The default value for the
+def type is `null.`
+
+Internally, if the `def` type represents a primitive type, it is converted to the
+corresponding reference type. It still behaves like the primitive type, however,
+including within the casting model. The `def` type can be assigned to different
+types during the course of script execution.
+
+IMPORTANT: Because a `def` type variable can be assigned to different types
+during execution, type conversion errors that occur when using the `def` type
+happen at runtime.
+
+Using the `def` type can have a slight impact on performance. If performance is
+critical, it's better to declare static types.
+
+*Examples:*
+[source,Java]
+----
+def x = 1;               // Declare def variable x and set it to the
+                         //   literal int 1
+def l = new ArrayList(); // Declare def variable l and set it a newly 
+                         //   allocated ArrayList
+----
+
+[[primitive-types]]
+==== Primitive Types
+
+Primitive types are allocated directly onto the stack according to the standard
+Java memory model.
+
+Primitive types can behave as their corresponding (<<boxing-unboxing, boxed>>)
+reference type. This means any piece of a reference type can be accessed or
+called through the primitive type. Operations performed in this manner convert
+the primitive type to its corresponding reference type at runtime and perform
+the field access  or method call without needing to perform any other
+operations.
+
+Painless supports the following primitive types.
+
+byte::
+An 8-bit, signed, two's complement integer.
+Range: [-128, 127].
+Default value: 0.
+Reference type: Byte.
+
+short:: 
+A 16-bit, signed, two's complement integer.
+Range: [-32768, 32767].
+Default value: 0.
+Reference type: Short.
+
+char::
+A 16-bit Unicode character.
+Range: [0, 65535].
+Default value: 0 or `\u0000`.
+Reference type: Character.
+
+int::
+A 32-bit, signed, two's complement integer.
+Range: [-2^32, 2^32-1].
+Default value: 0.
+Reference type: Integer.
+
+long::
+A 64-bit, signed, two's complement integer.
+Range: [-2^64, 2^64-1].
+Default value: 0.
+Reference type: Long.
+
+float::
+A 32-bit, single-precision, IEEE 754 floating point number.
+Range:  Depends on multiple factors.
+Default value: 0.0.
+Reference type: Float.
+
+double::
+A 64-bit, double-precision, IEEE 754 floating point number.
+Range: Depends on multiple factors.
+Default value: 0.0.
+Reference type: Double.
+
+boolean::
+A logical quanity with two possible values: true and false.
+Range: true/false.
+Default value: false.
+Reference type: Boolean.
+
+
+*Examples:*
+[source,Java]
+----
+int i = 1;        // Declare variable i as an int and set it to the
+                  //   literal 1
+double d;         // Declare variable d as a double and set it to the
+                  //   default value of 0.0
+boolean b = true; // Declare variable b as a boolean and set it to true
+----
+
+Using methods from the corresponding reference type on a primitive type.
+
+[source,Java]
+----
+int i = 1;    // Declare variable i as an int and set it to the
+              //   literal 1
+i.toString(); // Invokes the Integer method toString on variable i
+----
+
+[[reference-types]]
+==== Reference Types
+
+Reference types are similar to Java classes and can contain multiple pieces
+known as _members_. However, reference types do not support access modifiers.
+You allocate reference type instances on the heap using the `new` operator.
+
+Reference types can have both static and non-static members:
+
+* Static members are shared by all instances of the same reference type and
+can be accessed without allocating an instance of the reference type. For
+example `Integer.MAX_VALUE`.
+* Non-static members are specific to an instance of the reference type
+and can only be accessed through the allocated instance.
+
+The default value for a reference type is `null`, indicating that no memory has
+been allocated for it. When you assign `null` to a reference type, its previous
+value is discarded and garbage collected in accordance with the Java memory
+model as long as there are no other references to that value.
+
+A reference type can contain:
+
+* Zero to many primitive types. Primitive type members can be static or
+non-static and read-only or read-write.
+* Zero to many reference types. Reference type members can be static or
+non-static and read-only or read-write.
+* Methods that call an internal function to return a value and/or manipulate
+the primitive or reference type members. Method members can be static or
+non-static.
+* Constructors that call an internal function to return a newly-allocated
+reference type instance. Constructors are non-static methods that can
+optionally manipulate the primitive and reference type members.
+
+Reference types support a Java-style inheritance model. Consider types A and B.
+Type A is considered to be a parent of B, and B a child of A, if B inherits
+(is able to access as its own) all of A's fields and methods. Type B is
+considered a descendant of A if there exists a recursive parent-child
+relationship from B to A with none to many types in between. In this case, B
+inherits all of A's fields and methods along with all of the fields and
+methods of the types in between. Type B is also considered to be a type A
+in both relationships.
+
+For the complete list of Painless reference types and their supported methods,
+see the https://www.elastic.co/guide/en/elasticsearch/reference/current/painless-api-reference.html[Painless API Reference].
+
+For more information about working with reference types, see
+<<field-access, Accessing Fields>> and <<method-access, Calling Methods>>.
+
+*Examples:*
+[source,Java]
+----
+ArrayList al = new ArrayList();  // Declare variable al as an ArrayList and
+                                 //   set it to a newly allocated ArrayList
+List l = new ArrayList();        // Declare variable l as a List and set
+                                 //   it to a newly allocated ArrayList, which is
+                                 //   allowed because ArrayList inherits from List
+Map m;                           // Declare variable m as a Map and set it
+                                 //   to the default value of null
+----
+
+Directly accessing static pieces of a reference type.
+
+[source,Java]
+----
+Integer.MAX_VALUE      // a static field access
+Long.parseLong("123L") // a static function call
+----
+
+[[string-type]]
+==== String Type
+
+A `String` is a specialized reference type that is immutable and does not have
+to be explicitly allocated. You can directly assign to a `String` without first
+allocating it with the `new` keyword. (Strings can be allocated with the `new`
+keyword, but it's not required.)
+
+When assigning a value to a `String`, you must enclose the text in single or
+double quotes. Strings are allocated according to the standard Java Memory Model.
+The default value for a `String` is `null.`
+
+*Examples:*
+[source,Java]
+----
+String r = "some text";             // Declare String r and set it to the
+                                    //   String "some text"
+String s = 'some text';             // Declare String s and set it to the
+                                    //   String 'some text'
+String t = new String("some text"); // Declare String t and set it to the
+                                    //   String "some text"
+String u;                           // Declare String u and set it to the
+                                    //   default value null
+----
+
+[[void-type]]
+==== void Type
+
+The `void` type represents the concept of no type. In Painless, `void` declares
+that a function has no return value.
+
+[[array-type]]
+==== Array Type
+
+Arrays contain a series of elements of the same type that can be allocated
+simultaneously. Painless supports both single and multi-dimensional arrays for
+all types except void (including `def`).
+
+You declare an array by specifying a type followed by a series of empty brackets,
+where each set of brackets represents a dimension. Declared arrays have a default
+value of `null` and are themselves a reference type.
+
+To allocate an array, you use the `new` keyword followed by the type and a
+set of brackets for each dimension. You can explicitly define the size of each dimension by specifying an expression within the brackets, or initialize each
+dimension with the desired number of values. The allocated size of each
+dimension is its permanent size.
+
+To initialize an array, specify the values you want to initialize
+each dimension with as a comma-separated list of expressions enclosed in braces.
+For example, `new int[] {1, 2, 3}` creates a one-dimensional `int` array with a
+size of 3 and the values 1, 2, and 3.
+
+When you initialize an array, the order of the expressions is maintained. Each expression used as part of the initialization is converted to the
+array's type. An error occurs if the types do not match.
+
+*Grammar:*
+[source,ANTLR4]
+----
+declare_array: TYPE ('[' ']')+;
+
+array_initialization: 'new' TYPE '[' ']' '{' expression (',' expression) '}'
+                    | 'new' TYPE '[' ']' '{' '}';
+----
+
+*Examples:*
+[source,Java]
+----
+int[] x = new int[5];          // Declare int array x and assign it a newly
+                               //   allocated int array with a size of 5
+def[][] y = new def[5][5];     // Declare the 2-dimensional def array y and
+                               //   assign it a newly allocated 2-dimensional
+                               //   array where both dimensions have a size of 5
+int[] x = new int[] {1, 2, 3}; // Declare int array x and set it to an int
+                               //   array with values 1, 2, 3 and a size of 3
+int i = 1;
+long l = 2L;
+float f = 3.0F;
+double d = 4.0;
+String s = "5";
+def[] da = new def[] {i, l, f*d, s}; // Declare def array da and set it to
+                                     // a def array with a size of 4 and the
+                                     // values i, l, f*d, and s
+----
+
+[[casting]]
+=== Casting
+
+Casting is the conversion of one type to another. Implicit casts are casts that
+occur automatically, such as during an assignment operation. Explicit casts are
+casts where you use the casting operator to explicitly convert one type to
+another. This is necessary during operations where the cast cannot be inferred.
+
+To cast to a new type, precede the expression by the new type enclosed in
+parentheses, for example
+`(int)x`.
+
+The following sections specify the implicit casts that can be performed and the
+explicit casts that are allowed. The only other permitted cast is casting
+a single character `String` to a `char`.
+
+*Grammar:*
+[source,ANTLR4]
+----
+cast: '(' TYPE ')' expression
+----
+
+[[numeric-casting]]
+==== Numeric Casting
+
+The following table shows the allowed implicit and explicit casts between
+numeric types. Read the table by row. To find out if you need to explicitly
+cast from type A to type B, find the row for type A and scan across to the
+column for type B.
+
+IMPORTANT: Explicit casts between numeric types can result in some data loss. A
+smaller numeric type cannot necessarily accommodate the value from a larger
+numeric type. You might also lose precision when casting from integer types
+to floating point types.
+
+|====
+|       | byte     | short    | char     | int      | long     | float    | double
+| byte  |          | implicit | implicit | implicit | implicit | implicit | implicit
+| short | explicit |          | explicit | implicit | implicit | implicit | implicit
+| char  | explicit | explicit |          | implicit | implicit | implicit | implicit
+| int   | explicit | explicit | explicit |          | implicit | implicit | implicit
+| long  | explicit | explicit | explicit | explicit |          | implicit | implicit
+| float | explicit | explicit | explicit | explicit | explicit |          | implicit
+| float | explicit | explicit | explicit | explicit | explicit | explicit |
+|====
+
+
+Example(s)
+[source,Java]
+----
+int a = 1;            // Declare int variable a and set it to the literal
+                      //   value 1
+long b = a;           // Declare long variable b and set it to int variable
+                      //    a with an implicit cast to convert from int to long
+short c = (short)b;   // Declare short variable c, explicitly cast b to a
+                      //   short, and assign b to c
+byte d = a;           // ERROR: Casting an int to a byte requires an explicit
+                      //   cast
+double e = (double)a; // Explicitly cast int variable a to a double and assign
+                      //   it to the double variable e. The explicit cast is
+                      //   allowed, but it is not necessary.
+----
+
+[[reference-casting]]
+==== Reference Casting
+
+A reference type can be implicitly cast to another reference type as long as
+the type being cast _from_ is a descendant of the type being cast _to_.  A
+reference type can be explicitly cast _to_ if the type being cast to is a
+descendant of the type being cast _from_.
+
+*Examples:*
+[source,Java]
+----
+List x;                        // Declare List variable x
+ArrayList y = new ArrayList(); // Declare ArrayList variable y and assign it a
+                               //   newly allocated ArrayList [1]
+x = y;                         // Assign Arraylist y to List x using an
+                               //   implicit cast
+y = (ArrayList)x;              // Explicitly cast List x to an ArrayList and
+                               //  assign it to ArrayList y
+x = (List)y;                   // Set List x to ArrayList y using an explicit
+                               //   cast (the explicit cast is not necessary)
+y = x;                         // ERROR: List x cannot be implicitly cast to
+                               //   an ArrayList, an explicit cast is required
+Map m = y;                     // ERROR: Cannot implicitly or explicitly cast [2]
+                               //   an ArrayList to a Map, no relationship
+                               //   exists between the two types. 
+----
+[1] `ArrayList` is a descendant of the `List` type.
+[2] `Map` is unrelated to the `List` and `ArrayList` types.
+
+[[def-type-casting]]
+==== def Type Casting
+All primitive and reference types can always be implicitly cast to
+`def`. While it is possible to explicitly cast to `def`, it is not necessary.
+
+However, it is not always possible to implicitly cast a `def` to other
+primitive and reference types. An explicit cast is required if an explicit
+cast would normally be required between the non-def types.
+
+
+*Examples:*
+[source,Java]
+----
+def x;          // Declare def variable x and set it to null
+x = 3;          // Set the def variable x to the literal 3 with an implicit
+                //   cast from int to def
+double a = x;   // Declare double variable y and set it to def variable x,
+                //   which contains a double
+int b = x;      // ERROR: Results in a run-time error because an explicit cast is
+                //   required to cast from a double to an int
+int c = (int)x; // Declare int variable c, explicitly cast def variable x to an
+                //   int, and assign x to c
+----
+
+[[boxing-unboxing]]
+==== Boxing and Unboxing
+
+Boxing is where a cast is used to convert a primitive type to its corresponding
+reference type. Unboxing is the reverse, converting a reference type to the
+corresponding primitive type.
+
+There are two places Painless performs implicit boxing and unboxing:
+
+* When you call methods, Painless automatically boxes and unboxes arguments
+so you can specify either primitive types or their corresponding reference
+types.
+* When you use the `def` type, Painless automatically boxes and unboxes as
+needed when converting to and from `def`.
+
+The casting operator does not support any way to explicitly box a primitive
+type or unbox a reference type.
+
+If a primitive type needs to be converted to a reference type, the Painless
+reference type API supports methods that can do that. However, under normal
+circumstances this should not be necessary.
+
+*Examples:*
+[source,Java]
+----
+Integer x = 1;               // ERROR: not a legal implicit cast
+Integer y = (Integer)1;      // ERROR: not a legal explicit cast
+int a = new Integer(1);      // ERROR: not a legal implicit cast
+int b = (int)new Integer(1); // ERROR: not a legal explicit cast
+----
+
+[[promotion]]
+==== Promotion
+
+Promotion is where certain operations require types to be either a minimum
+numerical type or for two (or more) types to be equivalent.
+The documentation for each operation that has these requirements
+includes promotion tables that describe how this is handled.
+
+When an operation promotes a type or types, the resultant type
+of the operation is the promoted type.  Types can be promoted to def
+at compile-time; however, at run-time, the resultant type will be the
+promotion of the types the `def` is representing.
+
+*Examples:*
+[source,Java]
+----
+2 + 2.0     // Add the literal int 2 and the literal double 2.0. The literal
+            //   2 is promoted to a double and the resulting value is a double.
+
+def x = 1;  // Declare def variable x and set it to the literal int 1 through
+            //   an implicit cast
+x + 2.0F    // Add def variable x and the literal float 2.0.
+            // At compile-time the types are promoted to def.
+            // At run-time the types are promoted to float.
+----
diff --git a/docs/painless/painless-variables.asciidoc b/docs/painless/painless-variables.asciidoc
new file mode 100644
index 0000000000000..2177b0bb91ba8
--- /dev/null
+++ b/docs/painless/painless-variables.asciidoc
@@ -0,0 +1,123 @@
+[[variables]]
+=== Variables
+
+Variables in Painless must be declared and can be statically or <<dynamic-types,
+dynamically typed>>.
+
+[[variable-identifiers]]
+==== Variable Identifiers
+
+Specify variable identifiers using the following grammar. Variable identifiers
+must start with a letter or underscore. You cannot use <<keywords, keywords>> or
+<<types, types>> as identifiers.
+
+*Grammar:*
+[source,ANTLR4]
+----
+ID: [_a-zA-Z] [_a-zA-Z-0-9]*;
+----
+
+*Examples:*
+[source,Java]
+----
+_
+a
+Z
+id
+list
+list0
+MAP25
+_map25
+----
+
+[[variable-declaration]]
+==== Variable Declaration
+
+Variables must be declared before you use them. The format is `type-name
+identifier-name`. To declare multiple variables of the same type, specify a
+comma-separated list of identifier names. You can immediately assign a value to
+a variable when you declare it.
+
+*Grammar:*
+[source,ANTLR4]
+----
+type: ID ('[' ']')*;
+declaration : type ID (',' ID)*;
+----
+
+*Examples:*
+[source,Java]
+----
+int x;        // Declare a variable with type int and id x
+List y;       // Declare a variable with type List and id y
+int x, y, z;  // Declare variables with type int and ids x, y, and z
+def[] d;      // Declare the variable d with type def[]
+int i = 10;   // Declare the int variable i and set it to the int literal 10
+----
+
+[[variable-assignment]]
+==== Variable Assignment
+
+Use the equals operator (`=`) to assign a value to a variable. The format is
+`identifier-name = value`. Any value expression can be assigned to any variable
+as long as the types match or the expression's type can be implicitly cast to
+the variable's type. An error occurs if the types do not match.
+
+*Grammar:*
+[source,ANTLR4]
+----
+assignment: ID '=' expression
+----
+
+
+*Examples:*
+
+Assigning a literal of the appropriate type directly to a declared variable.
+
+[source,Java]
+----
+int i;   // Declare an int i
+i = 10;  // Set the int i to the int literal 10
+----
+
+Immediately assigning a value when declaring a variable. 
+
+[source,Java]
+----
+int i = 10;     // Declare the int variable i and set it the int literal 1
+double j = 2.0; // Declare the double variable j and set it to the double
+                //   literal 2.0
+----
+
+Assigning a variable of one primitive type to another variable of the same
+type.
+
+[source,Java]
+----
+int i = 10; // Declare the int variable i and set it to the int literal 10
+int j = i;  // Declare the int variable j and set it to the int variable i
+----
+
+Assigning a reference type to a new heap allocation with the `new` operator.
+
+[source,Java]
+----
+ArrayList l = new ArrayList();  // Declare an ArrayList variable l and set it
+                                //   to a newly allocated ArrayList
+Map m = new HashMap();          // Declare a Map variable m and set it
+                                //   to a newly allocated HashMap
+----
+
+Assigning a variable of one reference type to another variable of the same type.
+
+[source,Java]
+----
+List l = new ArrayList(); // Declare List variable l and set it a newly
+                          //    allocated ArrayList
+List k = l;               // Declare List variable k and set it to the
+                          //    value of the List variable l
+List m;                   // Declare List variable m and set it the
+                          //    default value null
+m = k;                    // Set the value of List variable m to the value
+                          //    of List variable k
+----
diff --git a/docs/painless/painless-xref.asciidoc b/docs/painless/painless-xref.asciidoc
new file mode 100644
index 0000000000000..86407b3e697d6
--- /dev/null
+++ b/docs/painless/painless-xref.asciidoc
@@ -0,0 +1,2 @@
+Ready to start scripting with Painless? See {painless}/painless-getting-started.html[Getting Started with Painless] in the guide to the
+{painless}/painless.html[Painless Scripting Language].
\ No newline at end of file
diff --git a/docs/perl/index.asciidoc b/docs/perl/index.asciidoc
deleted file mode 100644
index 734447d0dd103..0000000000000
--- a/docs/perl/index.asciidoc
+++ /dev/null
@@ -1,124 +0,0 @@
-= Elasticsearch.pm
-
-== Overview
-
-Search::Elasticsearch is the official Perl API for Elasticsearch. The full
-documentation is available on https://metacpan.org/module/Search::Elasticsearch.
-
-It can be installed with:
-
-[source,sh]
-------------------------------------
-cpanm Search::Elasticsearch
-------------------------------------
-
-=== Features
-
-This client provides:
-
-* Full support for all Elasticsearch APIs
-
-* HTTP backend (blocking and asynchronous with https://metacpan.org/module/Search::Elasticsearch::Async)
-
-* Robust networking support which handles load balancing, failure detection and failover
-
-* Good defaults
-
-* Helper utilities for more complex operations, such as bulk indexing, scrolled searches and reindexing.
-
-* Logging support via Log::Any
-
-* Compatibility with the official clients for Python, Ruby, PHP and Javascript
-
-* Easy extensibility
-
-== Synopsis
-
-[source,perl]
-------------------------------------
-use Search::Elasticsearch;
-
-# Connect to localhost:9200:
-my $e = Search::Elasticsearch->new();
-
-# Round-robin between two nodes:
-my $e = Search::Elasticsearch->new(
-    nodes => [
-        'search1:9200',
-        'search2:9200'
-    ]
-);
-
-# Connect to cluster at search1:9200, sniff all nodes and round-robin between them:
-my $e = Search::Elasticsearch->new(
-    nodes    => 'search1:9200',
-    cxn_pool => 'Sniff'
-);
-
-# Index a document:
-$e->index(
-    index   => 'my_app',
-    type    => 'blog_post',
-    id      => 1,
-    body    => {
-        title   => 'Elasticsearch clients',
-        content => 'Interesting content...',
-        date    => '2014-09-24'
-    }
-);
-
-# Get the document:
-my $doc = $e->get(
-    index   => 'my_app',
-    type    => 'blog_post',
-    id      => 1
-);
-
-# Search:
-my $results = $e->search(
-    index => 'my_app',
-    body  => {
-        query => {
-            match => { title => 'elasticsearch' }
-        }
-    }
-);
-------------------------------------
-
-[[v0_90]]
-== Elasticsearch 0.90.* and earlier
-
-The current version of the client supports the Elasticsearch 1.0 branch by
-default, which is not backwards compatible with the 0.90 branch.
-
-If you need to talk to a version of Elasticsearch before 1.0.0,
-please use `Search::Elasticsearch::Client::0_90::Direct` as follows:
-
-[source,perl]
-------------------------------------
-    $es = Search::Elasticsearch->new(
-        client => '0_90::Direct'
-    );
-------------------------------------
-
-
-== Reporting issues
-
-The GitHub repository is https://github.com/elastic/elasticsearch-perl
-and any issues can be reported on the issues list at
-https://github.com/elastic/elasticsearch-perl/issues.
-
-== Contributing
-
-Open source contributions are welcome. Please read our
-https://github.com/elastic/elasticsearch-perl/blob/master/CONTRIBUTING.asciidoc[guide to contributing].
-
-== Copyright and License
-
-This software is Copyright (c) 2013-2016 by Elasticsearch BV.
-
-This is free software, licensed under:
-https://github.com/elastic/elasticsearch-perl/blob/master/LICENSE.txt[The Apache License Version 2.0].
-
-
-
diff --git a/docs/plugins/alerting.asciidoc b/docs/plugins/alerting.asciidoc
index 9472dbb638222..1e365306a84ea 100644
--- a/docs/plugins/alerting.asciidoc
+++ b/docs/plugins/alerting.asciidoc
@@ -8,11 +8,10 @@ Alerting plugins allow Elasticsearch to monitor indices and to trigger alerts wh
 
 The core alerting plugins are:
 
-link:/products/watcher[Watcher]::
+link:/products/x-pack/alerting[X-Pack]::
 
-Watcher is the alerting and notification product for Elasticsearch that lets
-you take action based on changes in your data. It is designed around the
+X-Pack contains the alerting and notification product for Elasticsearch that
+lets you take action based on changes in your data. It is designed around the
 principle that if you can query something in Elasticsearch, you can alert on
 it. Simply define a query, condition, schedule, and the actions to take, and
-Watcher will do the rest.
-
+X-Pack will do the rest.
diff --git a/docs/plugins/analysis-icu.asciidoc b/docs/plugins/analysis-icu.asciidoc
index a21e0c5c82f5c..0322511a5b905 100644
--- a/docs/plugins/analysis-icu.asciidoc
+++ b/docs/plugins/analysis-icu.asciidoc
@@ -7,32 +7,24 @@ libraries, including better analysis of Asian languages, Unicode
 normalization, Unicode-aware case folding, collation support, and
 transliteration.
 
-[[analysis-icu-install]]
-[float]
-==== Installation
+[IMPORTANT]
+.ICU analysis and backwards compatibility
+================================================
 
-This plugin can be installed using the plugin manager:
+From time to time, the ICU library receives updates such as adding new
+characters and emojis, and improving collation (sort) orders.  These changes
+may or may not affect search and sort orders, depending on which characters
+sets you are using.
 
-[source,sh]
-----------------------------------------------------------------
-sudo bin/elasticsearch-plugin install analysis-icu
-----------------------------------------------------------------
+While we restrict ICU upgrades to major versions, you may find that an index
+created in the previous major version will need to be reindexed in order to
+return correct (and correctly ordered) results, and to take advantage of new
+characters.
 
-The plugin must be installed on every node in the cluster, and each node must
-be restarted after installation.
+================================================
 
-[[analysis-icu-remove]]
-[float]
-==== Removal
-
-The plugin can be removed with the following command:
-
-[source,sh]
-----------------------------------------------------------------
-sudo bin/elasticsearch-plugin remove analysis-icu
-----------------------------------------------------------------
-
-The node must be stopped before removing the plugin.
+:plugin_name: analysis-icu
+include::install_remove.asciidoc[]
 
 [[analysis-icu-normalization-charfilter]]
 ==== ICU Normalization Character Filter
@@ -45,6 +37,10 @@ normalization can be specified with the `name` parameter, which accepts `nfc`,
 `nfkc`, and `nfkc_cf` (default).  Set the `mode` parameter to `decompose` to
 convert `nfc` to `nfd` or `nfkc` to `nfkd` respectively:
 
+Which letters are normalized can be controlled by specifying the
+`unicodeSetFilter` parameter, which accepts a
+http://icu-project.org/apiref/icu4j/com/ibm/icu/text/UnicodeSet.html[UnicodeSet].
+
 Here are two examples, the default usage and a customised character filter:
 
 
@@ -161,7 +157,11 @@ PUT icu_sample
     }
 }
 
-POST icu_sample/_analyze?analyzer=my_analyzer&text=Elasticsearch. Wow!
+POST icu_sample/_analyze
+{
+  "analyzer": "my_analyzer",
+  "text": "Elasticsearch. Wow!"
+}
 --------------------------------------------------
 // CONSOLE
 
@@ -193,6 +193,10 @@ without any further configuration. The type of normalization can be specified
 with the `name` parameter, which accepts `nfc`, `nfkc`, and `nfkc_cf`
 (default).
 
+Which letters are normalized can be controlled by specifying the
+`unicodeSetFilter` parameter, which accepts a
+http://icu-project.org/apiref/icu4j/com/ibm/icu/text/UnicodeSet.html[UnicodeSet].
+
 You should probably prefer the <<analysis-icu-normalization-charfilter,Normalization character filter>>.
 
 Here are two examples, the default usage and a customised token filter:
@@ -306,50 +310,46 @@ PUT icu_sample
 --------------------------------------------------
 // CONSOLE
 
+
 [[analysis-icu-collation]]
 ==== ICU Collation Token Filter
 
+[WARNING]
+======
+This token filter has been deprecated since Lucene 5.0.  Please use
+<<analysis-icu-collation-keyword-field, ICU Collation Keyword Field>>.
+======
+
+[[analysis-icu-collation-keyword-field]]
+==== ICU Collation Keyword Field
+
 Collations are used for sorting documents in a language-specific word order.
-The `icu_collation` token filter is available to all indices and defaults to
-using the
-https://www.elastic.co/guide/en/elasticsearch/guide/current/sorting-collations.html#uca[DUCET collation],
+The `icu_collation_keyword` field type is available to all indices and will encode
+the terms directly as bytes in a doc values field and a single indexed token just
+like a standard {ref}/keyword.html[Keyword Field].
+
+Defaults to using {defguide}/sorting-collations.html#uca[DUCET collation],
 which is a best-effort attempt at language-neutral sorting.
 
 Below is an example of how to set up a field for sorting German names in
 ``phonebook'' order:
 
 [source,js]
---------------------------------------------------
-PUT /my_index
+--------------------------
+PUT my_index
 {
-  "settings": {
-    "analysis": {
-      "filter": {
-        "german_phonebook": {
-          "type":     "icu_collation",
-          "language": "de",
-          "country":  "DE",
-          "variant":  "@collation=phonebook"
-        }
-      },
-      "analyzer": {
-        "german_phonebook": {
-          "tokenizer": "keyword",
-          "filter":  [ "german_phonebook" ]
-        }
-      }
-    }
-  },
   "mappings": {
     "user": {
       "properties": {
-        "name": { <1>
+        "name": {   <1>
           "type": "text",
           "fields": {
-            "sort": { <2>
-              "type":      "text",
-              "fielddata": true,
-              "analyzer":  "german_phonebook"
+            "sort": {  <2>
+              "type": "icu_collation_keyword",
+              "index": false,
+              "language": "de",
+              "country": "DE",
+              "variant": "@collation=phonebook"
             }
           }
         }
@@ -368,15 +368,47 @@ GET _search <3>
   "sort": "name.sort"
 }
 
---------------------------------------------------
+--------------------------
 // CONSOLE
 
 <1> The `name` field uses the `standard` analyzer, and so support full text queries.
-<2> The `name.sort` field uses the `keyword` analyzer to preserve the name as
-    a single token, and applies the `german_phonebook` token filter to index
-    the value in German phonebook sort order.
+<2> The `name.sort` field is an `icu_collation_keyword` field that will preserve the name as
+    a single token doc values, and applies the German ``phonebook'' order.
 <3> An example query which searches the `name` field and sorts on the `name.sort` field.
 
+===== Parameters for ICU Collation Keyword Fields
+
+The following parameters are accepted by `icu_collation_keyword` fields:
+
+[horizontal]
+
+`doc_values`::
+
+    Should the field be stored on disk in a column-stride fashion, so that it
+    can later be used for sorting, aggregations, or scripting? Accepts `true`
+    (default) or `false`.
+
+`index`::
+
+    Should the field be searchable? Accepts `true` (default) or `false`.
+
+`null_value`::
+
+    Accepts a string value which is substituted for any explicit `null`
+    values.  Defaults to `null`, which means the field is treated as missing.
+
+`store`::
+
+    Whether the field value should be stored and retrievable separately from
+    the {ref}/mapping-source-field.html[`_source`] field. Accepts `true` or `false`
+    (default).
+
+`fields`::
+
+    Multi-fields allow the same string value to be indexed in multiple ways for
+    different purposes, such as one field for search and a multi-field for
+    sorting and aggregations.
+
 ===== Collation options
 
 `strength`::
@@ -408,14 +440,14 @@ Possible values: `shifted` or `non-ignorable`. Sets the alternate handling for
 strength `quaternary` to be either shifted or non-ignorable. Which boils down
 to ignoring punctuation and whitespace.
 
-`caseLevel`::
+`case_level`::
 
 Possible values: `true` or `false` (default). Whether case level sorting is
 required. When strength is set to `primary` this will ignore accent
 differences.
 
 
-`caseFirst`::
+`case_first`::
 
 Possible values: `lower` or `upper`. Useful to control which case is sorted
 first when case is not ignored for strength `tertiary`. The default depends on
@@ -428,11 +460,11 @@ according to their numeric representation. For example the value `egg-9` is
 sorted before the value `egg-21`.
 
 
-`variableTop`::
+`variable_top`::
 
 Single character or contraction. Controls what is variable for `alternate`.
 
-`hiraganaQuaternaryMode`::
+`hiragana_quaternary_mode`::
 
 Possible values: `true` or `false`.  Distinguishing between Katakana and
 Hiragana characters in `quaternary` strength.
@@ -477,18 +509,21 @@ PUT icu_sample
   }
 }
 
-GET icu_sample/_analyze?analyzer=latin
+GET icu_sample/_analyze
 {
+  "analyzer": "latin",
   "text": "你好" <2>
 }
 
-GET icu_sample/_analyze?analyzer=latin
+GET icu_sample/_analyze
 {
+  "analyzer": "latin",
   "text": "здравствуйте" <3>
 }
 
-GET icu_sample/_analyze?analyzer=latin
+GET icu_sample/_analyze
 {
+  "analyzer": "latin",
   "text": "こんにちは" <4>
 }
 
diff --git a/docs/plugins/analysis-kuromoji.asciidoc b/docs/plugins/analysis-kuromoji.asciidoc
index 90584645bbb20..2240ab0ea96a0 100644
--- a/docs/plugins/analysis-kuromoji.asciidoc
+++ b/docs/plugins/analysis-kuromoji.asciidoc
@@ -4,32 +4,8 @@
 The Japanese (kuromoji) Analysis plugin integrates Lucene kuromoji analysis
 module into elasticsearch.
 
-[[analysis-kuromoji-install]]
-[float]
-==== Installation
-
-This plugin can be installed using the plugin manager:
-
-[source,sh]
-----------------------------------------------------------------
-sudo bin/elasticsearch-plugin install analysis-kuromoji
-----------------------------------------------------------------
-
-The plugin must be installed on every node in the cluster, and each node must
-be restarted after installation.
-
-[[analysis-kuromoji-remove]]
-[float]
-==== Removal
-
-The plugin can be removed with the following command:
-
-[source,sh]
-----------------------------------------------------------------
-sudo bin/elasticsearch-plugin remove analysis-kuromoji
-----------------------------------------------------------------
-
-The node must be stopped before removing the plugin.
+:plugin_name: analysis-kuromoji
+include::install_remove.asciidoc[]
 
 [[analysis-kuromoji-analyzer]]
 ==== `kuromoji` analyzer
@@ -172,7 +148,11 @@ PUT kuromoji_sample
   }
 }
 
-POST kuromoji_sample/_analyze?analyzer=my_analyzer&text=東京スカイツリー
+POST kuromoji_sample/_analyze
+{
+  "analyzer": "my_analyzer",
+  "text": "東京スカイツリー"
+}
 --------------------------------------------------
 // CONSOLE
 
@@ -225,7 +205,11 @@ PUT kuromoji_sample
   }
 }
 
-POST kuromoji_sample/_analyze?analyzer=my_analyzer&text=飲み
+POST kuromoji_sample/_analyze
+{
+  "analyzer": "my_analyzer",
+  "text": "飲み"
+}
 --------------------------------------------------
 // CONSOLE
 
@@ -287,7 +271,11 @@ PUT kuromoji_sample
   }
 }
 
-POST kuromoji_sample/_analyze?analyzer=my_analyzer&text=寿司がおいしいね
+POST kuromoji_sample/_analyze
+{
+  "analyzer": "my_analyzer",
+  "text": "寿司がおいしいね"
+}
 --------------------------------------------------
 // CONSOLE
 
@@ -360,9 +348,17 @@ PUT kuromoji_sample
     }
 }
 
-POST kuromoji_sample/_analyze?analyzer=katakana_analyzer&text=寿司 <1>
+POST kuromoji_sample/_analyze
+{
+  "analyzer": "katakana_analyzer",
+  "text": "寿司" <1>
+}
 
-POST kuromoji_sample/_analyze?analyzer=romaji_analyzer&text=寿司 <2>
+POST kuromoji_sample/_analyze
+{
+  "analyzer": "romaji_analyzer",
+  "text": "寿司" <2>
+}
 --------------------------------------------------
 // CONSOLE
 
@@ -410,9 +406,17 @@ PUT kuromoji_sample
   }
 }
 
-POST kuromoji_sample/_analyze?analyzer=my_analyzer&text=コピー <1>
+POST kuromoji_sample/_analyze
+{
+  "analyzer": "my_analyzer",
+  "text": "コピー" <1>
+}
 
-POST kuromoji_sample/_analyze?analyzer=my_analyzer&text=サーバー <2>
+POST kuromoji_sample/_analyze
+{
+  "analyzer": "my_analyzer",
+  "text": "サーバー" <2>
+}
 --------------------------------------------------
 // CONSOLE
 
@@ -458,7 +462,11 @@ PUT kuromoji_sample
   }
 }
 
-POST kuromoji_sample/_analyze?analyzer=analyzer_with_ja_stop&text=ストップは消える
+POST kuromoji_sample/_analyze
+{
+  "analyzer": "analyzer_with_ja_stop",
+  "text": "ストップは消える"
+}
 --------------------------------------------------
 // CONSOLE
 
@@ -504,7 +512,11 @@ PUT kuromoji_sample
   }
 }
 
-POST kuromoji_sample/_analyze?analyzer=my_analyzer&text=一〇〇〇
+POST kuromoji_sample/_analyze
+{
+  "analyzer": "my_analyzer",
+  "text": "一〇〇〇"
+}
 --------------------------------------------------
 // CONSOLE
 
diff --git a/docs/plugins/analysis-phonetic.asciidoc b/docs/plugins/analysis-phonetic.asciidoc
index 34f14abe3c5ec..e32e91eaca8a6 100644
--- a/docs/plugins/analysis-phonetic.asciidoc
+++ b/docs/plugins/analysis-phonetic.asciidoc
@@ -5,32 +5,9 @@ The Phonetic Analysis plugin provides token filters which convert tokens to
 their phonetic representation using Soundex, Metaphone, and a variety of other
 algorithms.
 
-[[analysis-phonetic-install]]
-[float]
-==== Installation
-
-This plugin can be installed using the plugin manager:
-
-[source,sh]
-----------------------------------------------------------------
-sudo bin/elasticsearch-plugin install analysis-phonetic
-----------------------------------------------------------------
-
-The plugin must be installed on every node in the cluster, and each node must
-be restarted after installation.
-
-[[analysis-phonetic-remove]]
-[float]
-==== Removal
-
-The plugin can be removed with the following command:
-
-[source,sh]
-----------------------------------------------------------------
-sudo bin/elasticsearch-plugin remove analysis-phonetic
-----------------------------------------------------------------
+:plugin_name: analysis-phonetic
+include::install_remove.asciidoc[]
 
-The node must be stopped before removing the plugin.
 
 [[analysis-phonetic-token-filter]]
 ==== `phonetic` token filter
@@ -79,7 +56,11 @@ PUT phonetic_sample
   }
 }
 
-POST phonetic_sample/_analyze?analyzer=my_analyzer&text=Joe Bloggs <1>
+POST phonetic_sample/_analyze
+{
+  "analyzer": "my_analyzer",
+  "text": "Joe Bloggs" <1>
+}
 --------------------------------------------------
 // CONSOLE
 
diff --git a/docs/plugins/analysis-smartcn.asciidoc b/docs/plugins/analysis-smartcn.asciidoc
index 18a020cf40de9..4c019ebf2732c 100644
--- a/docs/plugins/analysis-smartcn.asciidoc
+++ b/docs/plugins/analysis-smartcn.asciidoc
@@ -9,33 +9,9 @@ analyzer uses probabilistic knowledge to find the optimal word segmentation
 for Simplified Chinese text. The text is first broken into sentences, then
 each sentence is segmented into words.
 
+:plugin_name: analysis-smartcn
+include::install_remove.asciidoc[]
 
-[[analysis-smartcn-install]]
-[float]
-==== Installation
-
-This plugin can be installed using the plugin manager:
-
-[source,sh]
-----------------------------------------------------------------
-sudo bin/elasticsearch-plugin install analysis-smartcn
-----------------------------------------------------------------
-
-The plugin must be installed on every node in the cluster, and each node must
-be restarted after installation.
-
-[[analysis-smartcn-remove]]
-[float]
-==== Removal
-
-The plugin can be removed with the following command:
-
-[source,sh]
-----------------------------------------------------------------
-sudo bin/elasticsearch-plugin remove analysis-smartcn
-----------------------------------------------------------------
-
-The node must be stopped before removing the plugin.
 
 [[analysis-smartcn-tokenizer]]
 [float]
diff --git a/docs/plugins/analysis-stempel.asciidoc b/docs/plugins/analysis-stempel.asciidoc
index 88b43a1a80558..a5526129a97d3 100644
--- a/docs/plugins/analysis-stempel.asciidoc
+++ b/docs/plugins/analysis-stempel.asciidoc
@@ -7,32 +7,8 @@ module for Polish into elasticsearch.
 It provides high quality stemming for Polish, based on the
 http://www.egothor.org/[Egothor project].
 
-[[analysis-stempel-install]]
-[float]
-==== Installation
-
-This plugin can be installed using the plugin manager:
-
-[source,sh]
-----------------------------------------------------------------
-sudo bin/elasticsearch-plugin install analysis-stempel
-----------------------------------------------------------------
-
-The plugin must be installed on every node in the cluster, and each node must
-be restarted after installation.
-
-[[analysis-stempel-remove]]
-[float]
-==== Removal
-
-The plugin can be removed with the following command:
-
-[source,sh]
-----------------------------------------------------------------
-sudo bin/elasticsearch-plugin remove analysis-stempel
-----------------------------------------------------------------
-
-The node must be stopped before removing the plugin.
+:plugin_name: analysis-stempel
+include::install_remove.asciidoc[]
 
 [[analysis-stempel-tokenizer]]
 [float]
diff --git a/docs/plugins/analysis-ukrainian.asciidoc b/docs/plugins/analysis-ukrainian.asciidoc
new file mode 100644
index 0000000000000..a86c1d18eea97
--- /dev/null
+++ b/docs/plugins/analysis-ukrainian.asciidoc
@@ -0,0 +1,15 @@
+[[analysis-ukrainian]]
+=== Ukrainian Analysis Plugin
+
+The Ukrainian Analysis plugin integrates Lucene's UkrainianMorfologikAnalyzer into elasticsearch.
+
+It provides stemming for Ukrainian using the http://github.com/morfologik/morfologik-stemming[Morfologik project].
+
+:plugin_name: analysis-ukrainian
+include::install_remove.asciidoc[]
+
+[[analysis-ukrainian-analyzer]]
+[float]
+==== `ukrainian` analyzer
+
+The plugin provides the `ukrainian` analyzer.
diff --git a/docs/plugins/analysis.asciidoc b/docs/plugins/analysis.asciidoc
index 884dc2aebae1a..3c3df021de5cb 100644
--- a/docs/plugins/analysis.asciidoc
+++ b/docs/plugins/analysis.asciidoc
@@ -36,16 +36,18 @@ segmented into words.
 
 Provides high quality stemming for Polish.
 
+<<analysis-ukrainian,Ukrainian>>::
+
+Provides stemming for Ukrainian.
+
 [float]
 ==== Community contributed analysis plugins
 
 A number of analysis plugins have been contributed by our community:
 
-* https://github.com/yakaz/elasticsearch-analysis-combo/[Combo Analysis Plugin] (by Olivier Favre, Yakaz)
 * https://github.com/synhershko/elasticsearch-analysis-hebrew[Hebrew Analysis Plugin] (by Itamar Syn-Hershko)
 * https://github.com/medcl/elasticsearch-analysis-ik[IK Analysis Plugin] (by Medcl)
 * https://github.com/medcl/elasticsearch-analysis-mmseg[Mmseg Analysis Plugin] (by Medcl)
-* https://github.com/chytreg/elasticsearch-analysis-morfologik[Morfologik (Polish) Analysis plugin] (by chytreg)
 * https://github.com/imotov/elasticsearch-analysis-morphology[Russian and English Morphological Analysis Plugin] (by Igor Motov)
 * https://github.com/medcl/elasticsearch-analysis-pinyin[Pinyin Analysis Plugin] (by Medcl)
 * https://github.com/duydo/elasticsearch-analysis-vietnamese[Vietnamese Analysis Plugin] (by Duy Do)
@@ -62,5 +64,4 @@ include::analysis-smartcn.asciidoc[]
 
 include::analysis-stempel.asciidoc[]
 
-
-
+include::analysis-ukrainian.asciidoc[]
diff --git a/docs/plugins/api.asciidoc b/docs/plugins/api.asciidoc
index 54edcbc7f0ef3..a2fbc5165ac94 100644
--- a/docs/plugins/api.asciidoc
+++ b/docs/plugins/api.asciidoc
@@ -14,10 +14,6 @@ A number of plugins have been contributed by our community:
 * https://github.com/wikimedia/search-extra[Elasticsearch Trigram Accelerated Regular Expression Filter]:
   (by Wikimedia Foundation/Nik Everett)
 
-* https://github.com/kzwang/elasticsearch-image[Elasticsearch Image Plugin]:
-  Uses https://code.google.com/p/lire/[Lire (Lucene Image Retrieval)] to allow users
-  to index images and search for similar images (by Kevin Wang)
-
 * https://github.com/wikimedia/search-highlighter[Elasticsearch Experimental Highlighter]:
   (by Wikimedia Foundation/Nik Everett)
 
@@ -30,7 +26,4 @@ A number of plugins have been contributed by our community:
 * https://github.com/codelibs/elasticsearch-taste[Elasticsearch Taste Plugin]:
   Mahout Taste-based Collaborative Filtering implementation (by CodeLibs Project)
 
-* https://github.com/hadashiA/elasticsearch-flavor[Elasticsearch Flavor Plugin] using
-  http://mahout.apache.org/[Mahout] Collaboration filtering (by hadashiA)
 * https://github.com/jurgc11/es-change-feed-plugin[WebSocket Change Feed Plugin] (by ForgeRock/Chris Clifton)
-
diff --git a/docs/plugins/authors.asciidoc b/docs/plugins/authors.asciidoc
index b7fd43f71c7ca..3b91fb76f6694 100644
--- a/docs/plugins/authors.asciidoc
+++ b/docs/plugins/authors.asciidoc
@@ -76,6 +76,7 @@ Read more in {ref}/integration-tests.html#changing-node-configuration[Changing N
 
 
 [float]
+[[plugin-authors-jsm]]
 === Java Security permissions
 
 Some plugins may need additional security permissions. A plugin can include
@@ -111,4 +112,3 @@ AccessController.doPrivileged(
 
 See http://www.oracle.com/technetwork/java/seccodeguide-139067.html[Secure Coding Guidelines for Java SE]
 for more information.
-
diff --git a/docs/plugins/discovery-azure-classic.asciidoc b/docs/plugins/discovery-azure-classic.asciidoc
index 0feb5f7f8e53c..784bbd69ee9ee 100644
--- a/docs/plugins/discovery-azure-classic.asciidoc
+++ b/docs/plugins/discovery-azure-classic.asciidoc
@@ -7,32 +7,9 @@ The Azure Classic Discovery plugin uses the Azure Classic API for unicast discov
 // See issue https://github.com/elastic/elasticsearch/issues/19146
 deprecated[5.0.0, Use coming Azure ARM Discovery plugin instead]
 
-[[discovery-azure-classic-install]]
-[float]
-==== Installation
+:plugin_name: discovery-azure-classic
+include::install_remove.asciidoc[]
 
-This plugin can be installed using the plugin manager:
-
-[source,sh]
-----------------------------------------------------------------
-sudo bin/elasticsearch-plugin install discovery-azure-classic
-----------------------------------------------------------------
-
-The plugin must be installed on every node in the cluster, and each node must
-be restarted after installation.
-
-[[discovery-azure-classic-remove]]
-[float]
-==== Removal
-
-The plugin can be removed with the following command:
-
-[source,sh]
-----------------------------------------------------------------
-sudo bin/elasticsearch-plugin remove discovery-azure-classic
-----------------------------------------------------------------
-
-The node must be stopped before removing the plugin.
 
 [[discovery-azure-classic-usage]]
 ==== Azure Virtual Machine Discovery
@@ -53,7 +30,7 @@ cloud:
                    type: pkcs12
 
 discovery:
-    type: azure
+    zen.hosts_provider: azure
 ----
 
 [IMPORTANT]
@@ -166,7 +143,7 @@ Before starting, you need to have:
 * A http://www.windowsazure.com/[Windows Azure account]
 * OpenSSL that isn't from MacPorts, specifically `OpenSSL 1.0.1f 6 Jan
   2014` doesn't seem to create a valid keypair for ssh. FWIW,
- `OpenSSL 1.0.1c 10 May 2012` on Ubuntu 12.04 LTS is known to work.
+ `OpenSSL 1.0.1c 10 May 2012` on Ubuntu 14.04 LTS is known to work.
 * SSH keys and certificate
 +
 --
@@ -375,6 +352,7 @@ curl -s https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticse
 # Prepare Elasticsearch installation
 sudo dpkg -i elasticsearch-{version}.deb
 ----
+// NOTCONSOLE
 
 Check that elasticsearch is running:
 
@@ -391,6 +369,7 @@ This command should give you a JSON result:
 {
   "name" : "Cp8oag6",
   "cluster_name" : "elasticsearch",
+  "cluster_uuid" : "AT69_T_DTp-1qgIJlatQqA",
   "version" : {
     "number" : "{version}",
     "build_hash" : "f27399d",
@@ -403,6 +382,7 @@ This command should give you a JSON result:
 --------------------------------------------
 // TESTRESPONSE[s/"name" : "Cp8oag6",/"name" : "$body.name",/]
 // TESTRESPONSE[s/"cluster_name" : "elasticsearch",/"cluster_name" : "$body.cluster_name",/]
+// TESTRESPONSE[s/"cluster_uuid" : "AT69_T_DTp-1qgIJlatQqA",/"cluster_uuid" : "$body.cluster_uuid",/]
 // TESTRESPONSE[s/"build_hash" : "f27399d",/"build_hash" : "$body.version.build_hash",/]
 // TESTRESPONSE[s/"build_date" : "2016-03-30T09:51:41.449Z",/"build_date" : $body.version.build_date,/]
 // TESTRESPONSE[s/"build_snapshot" : false,/"build_snapshot" : $body.version.build_snapshot,/]
diff --git a/docs/plugins/discovery-ec2.asciidoc b/docs/plugins/discovery-ec2.asciidoc
index 0803d0a4fcda6..c60d90b42ceab 100644
--- a/docs/plugins/discovery-ec2.asciidoc
+++ b/docs/plugins/discovery-ec2.asciidoc
@@ -5,32 +5,8 @@ The EC2 discovery plugin uses the https://github.com/aws/aws-sdk-java[AWS API] f
 
 *If you are looking for a hosted solution of Elasticsearch on AWS, please visit http://www.elastic.co/cloud.*
 
-[[discovery-ec2-install]]
-[float]
-==== Installation
-
-This plugin can be installed using the plugin manager:
-
-[source,sh]
-----------------------------------------------------------------
-sudo bin/elasticsearch-plugin install discovery-ec2
-----------------------------------------------------------------
-
-The plugin must be installed on every node in the cluster, and each node must
-be restarted after installation.
-
-[[discovery-ec2-remove]]
-[float]
-==== Removal
-
-The plugin can be removed with the following command:
-
-[source,sh]
-----------------------------------------------------------------
-sudo bin/elasticsearch-plugin remove discovery-ec2
-----------------------------------------------------------------
-
-The node must be stopped before removing the plugin.
+:plugin_name: discovery-ec2
+include::install_remove.asciidoc[]
 
 [[discovery-ec2-usage]]
 ==== Getting started with AWS
@@ -108,17 +84,20 @@ The `cloud.aws.region` can be set to a region and will automatically use the rel
 The available values are:
 
 * `us-east` (`us-east-1`) for US East (N. Virginia)
+* `us-east-2` for US East (Ohio)
 * `us-west` (`us-west-1`) for US West (N. California)
 * `us-west-2` for US West (Oregon)
-* `ap-south-1` for Asia Pacific (Mumbai)
+* `ap-south` (`ap-south-1`) for Asia Pacific (Mumbai)
 * `ap-southeast` (`ap-southeast-1`) for Asia Pacific (Singapore)
 * `ap-southeast-2` for Asia Pacific (Sydney)
 * `ap-northeast` (`ap-northeast-1`) for Asia Pacific (Tokyo)
 * `ap-northeast-2` (`ap-northeast-2`) for Asia Pacific (Seoul)
 * `eu-west` (`eu-west-1`) for EU (Ireland)
+* `eu-west-2` (`eu-west-2`) for EU (London)
 * `eu-central` (`eu-central-1`) for EU (Frankfurt)
 * `sa-east` (`sa-east-1`) for South America (São Paulo)
 * `cn-north` (`cn-north-1`) for China (Beijing)
+* `ca-central` (`ca-central-1`) for Canada (Central)
 
 [[discovery-ec2-usage-signer]]
 ===== EC2 Signer API
@@ -127,6 +106,17 @@ If you are using a compatible EC2 service, they might be using an older API to s
 You can set your compatible signer API using `cloud.aws.signer` (or `cloud.aws.ec2.signer`)
 with the right signer to use.
 
+===== Read timeout
+
+Read timeout determines the amount of time to wait for data to be transferred over an established,
+open connection before the connection is timed out. Defaults to AWS SDK default value (`50 s`).
+It can be configured with `cloud.aws.read_timeout` (or `cloud.aws.ec2.read_timeout`) setting:
+
+[source, yaml]
+----
+cloud.aws.read_timeout: 30s
+----
+
 [[discovery-ec2-discovery]]
 ==== EC2 Discovery
 
@@ -136,7 +126,7 @@ environments). Here is a simple sample configuration:
 [source,yaml]
 ----
 discovery:
-    type: ec2
+    zen.hosts_provider: ec2
 ----
 
 You must also set `cloud.aws.region` if you are not using default AWS region. See <<discovery-ec2-usage-region>> for details.
@@ -155,9 +145,20 @@ The following are a list of settings (prefixed with `discovery.ec2`) that can fu
 
 `host_type`::
 
-    The type of host type to use to communicate with other instances. Can be
-    one of `private_ip`, `public_ip`, `private_dns`, `public_dns`. Defaults to
-    `private_ip`.
++
+--
+The type of host type to use to communicate with other instances. Can be
+one of `private_ip`, `public_ip`, `private_dns`, `public_dns` or `tag:TAGNAME` where
+`TAGNAME` refers to a name of a tag configured for all EC2 instances. Instances which don't
+have this tag set will be ignored by the discovery process.
+
+For example if you defined a tag `my-elasticsearch-host` in ec2 and set it to `myhostname1.mydomain.com`, then
+setting `host_type: tag:my-elasticsearch-host` will tell Discovery Ec2 plugin to read the host name from the
+`my-elasticsearch-host` tag. In this case, it will be resolved to `myhostname1.mydomain.com`.
+http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html[Read more about EC2 Tags].
+
+Defaults to `private_ip`.
+--
 
 `availability_zones`::
 
@@ -175,7 +176,6 @@ The following are a list of settings (prefixed with `discovery.ec2`) that can fu
     How long the list of hosts is cached to prevent further requests to the AWS API.
     Defaults to `10s`.
 
-
 [IMPORTANT]
 .Binding the network host
 ==============================================
@@ -247,9 +247,20 @@ name - highly undesirable). Tagging elasticsearch ec2 nodes and then filtering b
 [[discovery-ec2-attributes]]
 ===== Automatic Node Attributes
 
-Though not dependent on actually using `ec2` as discovery (but still requires the cloud aws plugin installed), the
-plugin can automatically add node attributes relating to ec2 (for example, availability zone, that can be used with
-the awareness allocation feature). In order to enable it, set `cloud.node.auto_attributes` to `true` in the settings.
+Though not dependent on actually using `ec2` as discovery (but still requires the `discovery-ec2` plugin installed), the
+plugin can automatically add node attributes relating to ec2. In the future this may support other attributes, but this will
+currently only add an `aws_availability_zone` node attribute, which is the availability zone of the current node. Attributes
+can be used to isolate primary and replica shards across availability zones by using the
+{ref}/allocation-awareness.html[Allocation Awareness] feature.
+
+In order to enable it, set `cloud.node.auto_attributes` to `true` in the settings. For example:
+
+[source,yaml]
+----
+cloud.node.auto_attributes: true
+
+cluster.routing.allocation.awareness.attributes: aws_availability_zone
+----
 
 [[discovery-ec2-endpoint]]
 ===== Using other EC2 endpoint
@@ -275,7 +286,7 @@ Prefer https://aws.amazon.com/amazon-linux-ami/[Amazon Linux AMIs] as since Elas
 ===== Networking
 * Networking throttling takes place on smaller instance types in both the form of https://lab.getbase.com/how-we-discovered-limitations-on-the-aws-tcp-stack/[bandwidth and number of connections]. Therefore if large number of connections are needed and networking is becoming a bottleneck, avoid https://aws.amazon.com/ec2/instance-types/[instance types] with networking labeled as `Moderate` or `Low`.
 * Multicast is not supported, even when in an VPC; the aws cloud plugin which joins by performing a security group lookup.
-* When running in multiple http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html[availability zones] be sure to leverage https://www.elastic.co/guide/en/elasticsearch/reference/master/allocation-awareness.html[shard allocation awareness] so that not all copies of shard data reside in the same availability zone.
+* When running in multiple http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html[availability zones] be sure to leverage {ref}/allocation-awareness.html[shard allocation awareness] so that not all copies of shard data reside in the same availability zone.
 * Do not span a cluster across regions.  If necessary, use a tribe node.
 
 ===== Misc
diff --git a/docs/plugins/discovery-file.asciidoc b/docs/plugins/discovery-file.asciidoc
new file mode 100644
index 0000000000000..ad06cfc0cc5f1
--- /dev/null
+++ b/docs/plugins/discovery-file.asciidoc
@@ -0,0 +1,71 @@
+[[discovery-file]]
+=== File-Based Discovery Plugin
+
+The file-based discovery plugin uses a list of hosts/ports in a `unicast_hosts.txt` file
+in the `config/discovery-file` directory for unicast discovery.
+
+:plugin_name: discovery-file
+include::install_remove.asciidoc[]
+
+[[discovery-file-usage]]
+[float]
+==== Using the file-based discovery plugin
+
+The file-based discovery plugin provides the ability to specify the
+unicast hosts list through a simple `unicast_hosts.txt` file that can
+be dynamically updated at any time. To enable, add the following in `elasticsearch.yml`:
+
+[source,yaml]
+----
+discovery.zen.hosts_provider: file
+----
+
+This plugin simply provides a facility to supply the unicast hosts list for
+zen discovery through an external file that can be updated at any time by a side process.
+
+For example, this gives a convenient mechanism for an Elasticsearch instance
+that is  run in docker containers to be dynamically supplied a list of IP
+addresses to connect to for zen discovery when those IP addresses may not be
+known at node startup.
+
+Note that the file-based discovery plugin is meant to augment the unicast
+hosts list in `elasticsearch.yml` (if specified), not replace it.  Therefore,
+if there are valid unicast host entries in `discovery.zen.ping.unicast.hosts`,
+they will be used in addition to those supplied in `unicast_hosts.txt`.
+
+Anytime a change is made to the `unicast_hosts.txt` file, even as Elasticsearch
+continues to run, the new changes will be picked up by the plugin and the
+new hosts list will be used for the next pinging round for master election.
+
+Upon installation of the plugin, a default `unicast_hosts.txt` file will
+be found in the `$CONFIG_DIR/discovery-file` directory.  This default file
+will contain some comments about what the file should contain.  All comments
+for this file must appear on their lines starting with `#` (i.e. comments
+cannot start in the middle of a line).
+
+[[discovery-file-format]]
+[float]
+==== unicast_hosts.txt file format
+
+The format of the file is to specify one unicast host entry per line.
+Each unicast host entry consists of the host (host name or IP address) and
+an optional transport port number.  If the port number is specified, is must
+come immediately after the host (on the same line) separated by a `:`.
+If the port number is not specified, a default value of 9300 is used.
+
+For example, this is an example of `unicast_hosts.txt` for a cluster with
+four nodes that participate in unicast discovery, some of which are not
+running on the default port:
+
+[source,txt]
+----------------------------------------------------------------
+10.10.10.5
+10.10.10.6:9305
+10.10.10.5:10005
+# an IPv6 address
+[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:9301
+----------------------------------------------------------------
+
+Host names are allowed instead of IP addresses (similar to
+`discovery.zen.ping.unicast.hosts`), and IPv6 addresses must be
+specified in brackets with the port coming after the brackets.
diff --git a/docs/plugins/discovery-gce.asciidoc b/docs/plugins/discovery-gce.asciidoc
index f615fc9810ece..16c84f9033d39 100644
--- a/docs/plugins/discovery-gce.asciidoc
+++ b/docs/plugins/discovery-gce.asciidoc
@@ -3,32 +3,8 @@
 
 The Google Compute Engine Discovery plugin uses the GCE API for unicast discovery.
 
-[[discovery-gce-install]]
-[float]
-==== Installation
-
-This plugin can be installed using the plugin manager:
-
-[source,sh]
-----------------------------------------------------------------
-sudo bin/elasticsearch-plugin install discovery-gce
-----------------------------------------------------------------
-
-The plugin must be installed on every node in the cluster, and each node must
-be restarted after installation.
-
-[[discovery-gce-remove]]
-[float]
-==== Removal
-
-The plugin can be removed with the following command:
-
-[source,sh]
-----------------------------------------------------------------
-sudo bin/elasticsearch-plugin remove discovery-gce
-----------------------------------------------------------------
-
-The node must be stopped before removing the plugin.
+:plugin_name: discovery-gce
+include::install_remove.asciidoc[]
 
 [[discovery-gce-usage]]
 ==== GCE Virtual Machine Discovery
@@ -43,7 +19,7 @@ cloud:
       project_id: <your-google-project-id>
       zone: <your-zone>
 discovery:
-      type: gce
+      zen.hosts_provider: gce
 --------------------------------------------------
 
 The following gce settings (prefixed with `cloud.gce`) are supported:
diff --git a/docs/plugins/discovery.asciidoc b/docs/plugins/discovery.asciidoc
index 999bf9c0e1f9e..39afbea96dc0a 100644
--- a/docs/plugins/discovery.asciidoc
+++ b/docs/plugins/discovery.asciidoc
@@ -21,14 +21,16 @@ The Azure Classic discovery plugin uses the Azure Classic API for unicast discov
 
 The Google Compute Engine discovery plugin uses the GCE API for unicast discovery.
 
+<<discovery-file,File-based discovery>>::
+
+The File-based discovery plugin allows providing the unicast hosts list through a dynamically updatable file.
+
 [float]
 ==== Community contributed discovery plugins
 
 A number of discovery plugins have been contributed by our community:
 
-* https://github.com/grantr/elasticsearch-srv-discovery[DNS SRV Discovery Plugin] (by Grant Rodgers)
 * https://github.com/shikhar/eskka[eskka Discovery Plugin] (by Shikhar Bhushan)
-* https://github.com/grmblfrz/elasticsearch-zookeeper[ZooKeeper Discovery Plugin] (by Sonian Inc.)
 * https://github.com/fabric8io/elasticsearch-cloud-kubernetes[Kubernetes Discovery Plugin] (by Jimmi Dyson, http://fabric8.io[fabric8])
 
 include::discovery-ec2.asciidoc[]
@@ -37,3 +39,4 @@ include::discovery-azure-classic.asciidoc[]
 
 include::discovery-gce.asciidoc[]
 
+include::discovery-file.asciidoc[]
diff --git a/docs/plugins/index.asciidoc b/docs/plugins/index.asciidoc
index 128b12f5dbaab..841a0777af08e 100644
--- a/docs/plugins/index.asciidoc
+++ b/docs/plugins/index.asciidoc
@@ -1,9 +1,6 @@
 = Elasticsearch Plugins and Integrations
 
-:ref:            https://www.elastic.co/guide/en/elasticsearch/reference/master
-:guide:          https://www.elastic.co/guide
-:version:        5.0.0-alpha5
-:lucene_version: 6.2.0
+include::../Versions.asciidoc[]
 
 [[intro]]
 == Introduction to plugins
@@ -16,6 +13,10 @@ Plugins contain JAR files, but may also contain scripts and config files, and
 must be installed on every node in the cluster. After installation, each
 node must be restarted before the plugin becomes visible.
 
+NOTE: A full cluster restart is required for installing plugins that have
+custom cluster state metadata, such as X-Pack. It is still possible to upgrade
+such plugins with a rolling restart.
+
 This documentation distinguishes two categories of plugins:
 
 Core Plugins::    This category identifies plugins that are part of Elasticsearch
diff --git a/docs/plugins/ingest-attachment.asciidoc b/docs/plugins/ingest-attachment.asciidoc
index 65add6ac9274c..443d1fb578a6d 100644
--- a/docs/plugins/ingest-attachment.asciidoc
+++ b/docs/plugins/ingest-attachment.asciidoc
@@ -11,32 +11,8 @@ the overhead of converting back and forth between base64, you can use the CBOR
 format instead of JSON and specify the field as a bytes array instead of a string
 representation. The processor will skip the base64 decoding then.
 
-[[ingest-attachment-install]]
-[float]
-==== Installation
-
-This plugin can be installed using the plugin manager:
-
-[source,sh]
-----------------------------------------------------------------
-sudo bin/elasticsearch-plugin install ingest-attachment
-----------------------------------------------------------------
-
-The plugin must be installed on every node in the cluster, and each node must
-be restarted after installation.
-
-[[ingest-attachment-remove]]
-[float]
-==== Removal
-
-The plugin can be removed with the following command:
-
-[source,sh]
-----------------------------------------------------------------
-sudo bin/elasticsearch-plugin remove ingest-attachment
-----------------------------------------------------------------
-
-The node must be stopped before removing the plugin.
+:plugin_name: ingest-attachment
+include::install_remove.asciidoc[]
 
 [[using-ingest-attachment]]
 ==== Using the Attachment Processor in a Pipeline
@@ -49,7 +25,8 @@ The node must be stopped before removing the plugin.
 | `field`                | yes       | -                | The field to get the base64 encoded field from
 | `target_field`         | no        | attachment       | The field that will hold the attachment information
 | `indexed_chars`        | no        | 100000           | The number of chars being used for extraction to prevent huge fields. Use `-1` for no limit.
-| `properties`           | no        | all              | Properties to select to be stored. Can be `content`, `title`, `name`, `author`, `keywords`, `date`, `content_type`, `content_length`, `language`
+| `properties`           | no        | all properties   | Array of properties to select to be stored. Can be `content`, `title`, `name`, `author`, `keywords`, `date`, `content_type`, `content_length`, `language`
+| `ignore_missing`       | no        | `false`          | If `true` and `field` does not exist, the processor quietly exits without modifying the document
 |======
 
 For example, this:
@@ -98,6 +75,139 @@ Returns this:
 --------------------------------------------------
 // TESTRESPONSE
 
+
+To specify only some fields to be extracted:
+
+[source,js]
+--------------------------------------------------
+PUT _ingest/pipeline/attachment
+{
+  "description" : "Extract attachment information",
+  "processors" : [
+    {
+      "attachment" : {
+        "field" : "data",
+        "properties": [ "content", "title" ]
+      }
+    }
+  ]
+}
+--------------------------------------------------
+// CONSOLE
+
 NOTE: Extracting contents from binary data is a resource intensive operation and
       consumes a lot of resources. It is highly recommended to run pipelines
       using this processor in a dedicated ingest node.
+
+[[ingest-attachment-with-arrays]]
+==== Using the Attachment Processor with arrays
+
+To use the attachment processor within an array of attachments the
+{ref}/foreach-processor.html[foreach processor] is required. This
+enables the attachment processor to be run on the individual elements
+of the array.
+
+For example, given the following source:
+
+[source,js]
+--------------------------------------------------
+{
+  "attachments" : [
+    {
+      "filename" : "ipsum.txt",
+      "data" : "dGhpcyBpcwpqdXN0IHNvbWUgdGV4dAo="
+    },
+    {
+      "filename" : "test.txt",
+      "data" : "VGhpcyBpcyBhIHRlc3QK"
+    }
+  ]
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+In this case, we want to process the data field in each element
+of the attachments field and insert
+the properties into the document so the following `foreach`
+processor is used:
+
+[source,js]
+--------------------------------------------------
+PUT _ingest/pipeline/attachment
+{
+  "description" : "Extract attachment information from arrays",
+  "processors" : [
+    {
+      "foreach": {
+        "field": "attachments",
+        "processor": {
+          "attachment": {
+            "target_field": "_ingest._value.attachment",
+            "field": "_ingest._value.data"
+          }
+        }
+      }
+    }
+  ]
+}
+PUT my_index/my_type/my_id?pipeline=attachment
+{
+  "attachments" : [
+    {
+      "filename" : "ipsum.txt",
+      "data" : "dGhpcyBpcwpqdXN0IHNvbWUgdGV4dAo="
+    },
+    {
+      "filename" : "test.txt",
+      "data" : "VGhpcyBpcyBhIHRlc3QK"
+    }
+  ]
+}
+GET my_index/my_type/my_id
+--------------------------------------------------
+// CONSOLE
+
+Returns this:
+[source,js]
+--------------------------------------------------
+{
+  "_index" : "my_index",
+  "_type" : "my_type",
+  "_id" : "my_id",
+  "_version" : 1,
+  "found" : true,
+  "_source" : {
+    "attachments" : [
+      {
+        "filename" : "ipsum.txt",
+        "data" : "dGhpcyBpcwpqdXN0IHNvbWUgdGV4dAo=",
+        "attachment" : {
+          "content_type" : "text/plain; charset=ISO-8859-1",
+          "language" : "en",
+          "content" : "this is\njust some text",
+          "content_length" : 24
+        }
+      },
+      {
+        "filename" : "test.txt",
+        "data" : "VGhpcyBpcyBhIHRlc3QK",
+        "attachment" : {
+          "content_type" : "text/plain; charset=ISO-8859-1",
+          "language" : "en",
+          "content" : "This is a test",
+          "content_length" : 16
+        }
+      }
+    ]
+  }
+}
+--------------------------------------------------
+// TESTRESPONSE
+
+
+Note that the `target_field` needs to be set, otherwise the
+default value is used which is a top level field `attachment`. The
+properties on this top level field will contain the value of the
+first attachment only. However, by specifying the
+`target_field` on to a value on `_ingest._value` it will correctly
+associate the properties with the correct attachment.
diff --git a/docs/plugins/ingest-geoip.asciidoc b/docs/plugins/ingest-geoip.asciidoc
index 1626be6c8e64b..51274b109ad7a 100644
--- a/docs/plugins/ingest-geoip.asciidoc
+++ b/docs/plugins/ingest-geoip.asciidoc
@@ -2,7 +2,8 @@
 === Ingest Geoip Processor Plugin
 
 The GeoIP processor adds information about the geographical location of IP addresses, based on data from the Maxmind databases.
-This processor adds this information by default under the `geoip` field.
+This processor adds this information by default under the `geoip` field. The `geoip` processor can resolve both IPv4 and
+IPv6 addresses.
 
 The ingest-geoip plugin ships by default with the GeoLite2 City and GeoLite2 Country geoip2 databases from Maxmind made available
 under the CCA-ShareAlike 3.0 license. For more details see, http://dev.maxmind.com/geoip/geoip2/geolite2/
@@ -11,32 +12,8 @@ The GeoIP processor can run with other geoip2 databases from Maxmind. The files
 and the `database_file` option should be used to specify the filename of the custom database. Custom database files must be compressed
 with gzip. The geoip config directory is located at `$ES_HOME/config/ingest/geoip` and holds the shipped databases too.
 
-[[ingest-geoip-install]]
-[float]
-==== Installation
-
-This plugin can be installed using the plugin manager:
-
-[source,sh]
-----------------------------------------------------------------
-sudo bin/elasticsearch-plugin install ingest-geoip
-----------------------------------------------------------------
-
-The plugin must be installed on every node in the cluster, and each node must
-be restarted after installation.
-
-[[ingest-geoip-remove]]
-[float]
-==== Removal
-
-The plugin can be removed with the following command:
-
-[source,sh]
-----------------------------------------------------------------
-sudo bin/elasticsearch-plugin remove ingest-geoip
-----------------------------------------------------------------
-
-The node must be stopped before removing the plugin.
+:plugin_name: ingest-geoip
+include::install_remove.asciidoc[]
 
 [[using-ingest-geoip]]
 ==== Using the Geoip Processor in a Pipeline
@@ -50,6 +27,7 @@ The node must be stopped before removing the plugin.
 | `target_field`         | no        | geoip                                                                              | The field that will hold the geographical information looked up from the Maxmind database.
 | `database_file`        | no        | GeoLite2-City.mmdb                                                                 | The database filename in the geoip config directory. The ingest-geoip plugin ships with the GeoLite2-City.mmdb.gz and GeoLite2-Country.mmdb.gz files.
 | `properties`           | no        | [`continent_name`, `country_iso_code`, `region_name`, `city_name`, `location`] *   | Controls what properties are added to the `target_field` based on the geoip lookup.
+| `ignore_missing`       | no        | `false`                                                                            | If `true` and `field` does not exist, the processor quietly exits without modifying the document
 |======
 
 *Depends on what is available in `database_field`:
@@ -154,3 +132,59 @@ returns this:
 }
 --------------------------------------------------
 // TESTRESPONSE
+
+
+Not all IP addresses find geo information from the database, When this
+occurs, no `target_field` is inserted into the document.
+
+Here is an example of what documents will be indexed as when information for "93.114.45.13"
+cannot be found:
+
+[source,js]
+--------------------------------------------------
+PUT _ingest/pipeline/geoip
+{
+  "description" : "Add geoip info",
+  "processors" : [
+    {
+      "geoip" : {
+        "field" : "ip"
+      }
+    }
+  ]
+}
+PUT my_index/my_type/my_id?pipeline=geoip
+{
+  "ip": "93.114.45.13"
+}
+GET my_index/my_type/my_id
+--------------------------------------------------
+// CONSOLE
+
+Which returns:
+
+[source,js]
+--------------------------------------------------
+{
+  "found": true,
+  "_index": "my_index",
+  "_type": "my_type",
+  "_id": "my_id",
+  "_version": 1,
+  "_source": {
+    "ip": "93.114.45.13"
+  }
+}
+--------------------------------------------------
+// TESTRESPONSE
+
+[[ingest-geoip-settings]]
+===== Node Settings
+
+The geoip processor supports the following setting:
+
+`ingest.geoip.cache_size`::
+
+    The maximum number of results that should be cached. Defaults to `1000`.
+
+Note that these settings are node settings and apply to all geoip processors, i.e. there is one cache for all defined geoip processors.
\ No newline at end of file
diff --git a/docs/plugins/ingest-user-agent.asciidoc b/docs/plugins/ingest-user-agent.asciidoc
index 29903224f39be..fadb0ad1138f0 100644
--- a/docs/plugins/ingest-user-agent.asciidoc
+++ b/docs/plugins/ingest-user-agent.asciidoc
@@ -6,32 +6,8 @@ This processor adds this information by default under the `user_agent` field.
 
 The ingest-user-agent plugin ships by default with the regexes.yaml made available by uap-java with an Apache 2.0 license. For more details see https://github.com/ua-parser/uap-core.
 
-[[ingest-user-agent-install]]
-[float]
-==== Installation
-
-This plugin can be installed using the plugin manager:
-
-[source,sh]
-----------------------------------------------------------------
-sudo bin/elasticsearch-plugin install ingest-user-agent
-----------------------------------------------------------------
-
-The plugin must be installed on every node in the cluster, and each node must
-be restarted after installation.
-
-[[ingest-user-agent-remove]]
-[float]
-==== Removal
-
-The plugin can be removed with the following command:
-
-[source,sh]
-----------------------------------------------------------------
-sudo bin/elasticsearch-plugin remove ingest-user-agent
-----------------------------------------------------------------
-
-The node must be stopped before removing the plugin.
+:plugin_name: ingest-user-agent
+include::install_remove.asciidoc[]
 
 [[using-ingest-user-agent]]
 ==== Using the user_agent Processor in a Pipeline
@@ -40,11 +16,12 @@ The node must be stopped before removing the plugin.
 .User-agent options
 [options="header"]
 |======
-| Name                   | Required  | Default                                                                            | Description
-| `field`                | yes       | -                                                                                  | The field containing the user agent string.
-| `target_field`         | no        | user_agent                                                                          | The field that will be filled with the user agent details.
-| `regex_file`           | no        | -                                                                                  | The name of the file in the `config/ingest-user-agent` directory containing the regular expressions for parsing the user agent string. Both the directory and the file have to be created before starting Elasticsearch. If not specified, ingest-user-agent will use the regexes.yaml from uap-core it ships with (see below).
+| Name                   | Required  | Default                                                                                         | Description
+| `field`                | yes       | -                                                                                               | The field containing the user agent string.
+| `target_field`         | no        | user_agent                                                                                      | The field that will be filled with the user agent details.
+| `regex_file`           | no        | -                                                                                               | The name of the file in the `config/ingest-user-agent` directory containing the regular expressions for parsing the user agent string. Both the directory and the file have to be created before starting Elasticsearch. If not specified, ingest-user-agent will use the regexes.yaml from uap-core it ships with (see below).
 | `properties`           | no        | [`name`, `major`, `minor`, `patch`, `build`, `os`, `os_name`, `os_major`, `os_minor`, `device`] | Controls what properties are added to `target_field`.
+| `ignore_missing`       | no        | `false`                                                                                         | If `true` and `field` does not exist, the processor quietly exits without modifying the document
 |======
 
 Here is an example that adds the user agent details to the `user_agent` field based on the `agent` field:
diff --git a/docs/plugins/install_remove.asciidoc b/docs/plugins/install_remove.asciidoc
new file mode 100644
index 0000000000000..f41bea39bbc4b
--- /dev/null
+++ b/docs/plugins/install_remove.asciidoc
@@ -0,0 +1,40 @@
+[float]
+[id="{plugin_name}-install"]
+==== Installation
+
+ifeval::["{release-state}"=="unreleased"]
+
+Version {version} of the Elastic Stack has not yet been released.
+
+endif::[]
+
+ifeval::["{release-state}"!="unreleased"]
+
+This plugin can be installed using the plugin manager:
+
+["source","sh",subs="attributes,callouts"]
+----------------------------------------------------------------
+sudo bin/elasticsearch-plugin install {plugin_name}
+----------------------------------------------------------------
+
+The plugin must be installed on every node in the cluster, and each node must
+be restarted after installation.
+
+This plugin can be downloaded for <<plugin-management-custom-url,offline install>> from
+{plugin_url}/{plugin_name}/{plugin_name}-{version}.zip.
+
+endif::[]
+
+[float]
+[id="{plugin_name}-remove"]
+==== Removal
+
+The plugin can be removed with the following command:
+
+["source","sh",subs="attributes,callouts"]
+----------------------------------------------------------------
+sudo bin/elasticsearch-plugin remove {plugin_name}
+----------------------------------------------------------------
+
+The node must be stopped before removing the plugin.
+
diff --git a/docs/plugins/integrations.asciidoc b/docs/plugins/integrations.asciidoc
index d39d02efeac71..4cfc5ab7539a8 100644
--- a/docs/plugins/integrations.asciidoc
+++ b/docs/plugins/integrations.asciidoc
@@ -28,6 +28,9 @@ Integrations are not plugins, but are external tools or modules that make it eas
   search (facets, etc), along with some Natural Language Processing features
   (ex.: More like this)
 
+* http://extensions.xwiki.org/xwiki/bin/view/Extension/Elastic+Search+Macro/[XWiki Next Generation Wiki]:
+  XWiki has an Elasticsearch and Kibana macro allowing to run Elasticsearch queries and display the results in XWiki pages using XWiki's scripting language as well as include Kibana Widgets in XWiki pages
+
 [float]
 [[data-integrations]]
 === Data import/export and validation
@@ -38,13 +41,13 @@ releases 2.0 and later do not support rivers.
 [float]
 ==== Supported by Elasticsearch:
 
-* https://www.elastic.co/guide/en/logstash/current/plugins-outputs-elasticsearch.html[Logstash output to Elasticsearch]:
+* {logstash-ref}/plugins-outputs-elasticsearch.html[Logstash output to Elasticsearch]:
   The Logstash `elasticsearch` output plugin.
-* https://www.elastic.co/guide/en/logstash/current/plugins-inputs-elasticsearch.html[Elasticsearch input to Logstash]
+* {logstash-ref}/plugins-inputs-elasticsearch.html[Elasticsearch input to Logstash]
   The Logstash `elasticsearch` input plugin.
-* https://www.elastic.co/guide/en/logstash/current/plugins-filters-elasticsearch.html[Elasticsearch event filtering in Logstash]
+* {logstash-ref}/plugins-filters-elasticsearch.html[Elasticsearch event filtering in Logstash]
   The Logstash `elasticsearch` filter plugin.
-* https://www.elastic.co/guide/en/logstash/current/plugins-codecs-es_bulk.html[Elasticsearch bulk codec]
+* {logstash-ref}/plugins-codecs-es_bulk.html[Elasticsearch bulk codec]
   The Logstash `es_bulk` plugin decodes the Elasticsearch bulk format into individual events.
 
 [float]
@@ -118,6 +121,9 @@ releases 2.0 and later do not support rivers.
 * http://haystacksearch.org/[Haystack]:
   Modular search for Django
 
+* http://hibernate.org/search/[Hibernate Search]
+  Integration with Hibernate ORM, from the Hibernate team. Automatic synchronization of write operations, yet exposes full Elasticsearch capabilities for queries. Can return either Elasticsearch native or re-map queries back into managed entities loaded within transaction from the reference database.
+
 * https://github.com/cleverage/play2-elasticsearch[play2-elasticsearch]:
   Elasticsearch module for Play Framework 2.x
 
@@ -201,4 +207,3 @@ These projects appear to have been abandoned:
   D3.
 * https://github.com/OlegKunitsyn/eslogd[eslogd]:
   Linux daemon that replicates events to a central Elasticsearch server in realtime
->>>>>>> 02602a3... Update integrations.asciidoc (#18915)
diff --git a/docs/plugins/lang-javascript.asciidoc b/docs/plugins/lang-javascript.asciidoc
index 0b8346f2a906d..6c049ee13d8af 100644
--- a/docs/plugins/lang-javascript.asciidoc
+++ b/docs/plugins/lang-javascript.asciidoc
@@ -1,38 +1,14 @@
 [[lang-javascript]]
 === JavaScript Language Plugin
 
-deprecated[5.0.0,JavaScript will be replaced by the new scripting language {ref}/modules-scripting-painless.html[`Painless`]]
+deprecated[5.0.0,JavaScript will be replaced by the new default scripting language `Painless`]
 
 The JavaScript language plugin enables the use of JavaScript in Elasticsearch
 scripts, via Mozilla's
 https://developer.mozilla.org/en-US/docs/Mozilla/Projects/Rhino[Rhino JavaScript] engine.
 
-[[lang-javascript-install]]
-[float]
-==== Installation
-
-This plugin can be installed using the plugin manager:
-
-[source,sh]
-----------------------------------------------------------------
-sudo bin/elasticsearch-plugin install lang-javascript
-----------------------------------------------------------------
-
-The plugin must be installed on every node in the cluster, and each node must
-be restarted after installation.
-
-[[lang-javascript-remove]]
-[float]
-==== Removal
-
-The plugin can be removed with the following command:
-
-[source,sh]
-----------------------------------------------------------------
-sudo bin/elasticsearch-plugin remove lang-javascript
-----------------------------------------------------------------
-
-The node must be stopped before removing the plugin.
+:plugin_name: lang-javascript
+include::install_remove.asciidoc[]
 
 [[lang-javascript-usage]]
 ==== Using JavaScript in Elasticsearch
@@ -50,8 +26,7 @@ Scripting is available in many APIs, but we will use an example with the
 WARNING: Enabling inline scripting on an unprotected Elasticsearch cluster is dangerous.
 See <<lang-javascript-file>> for a safer option.
 
-If you have enabled {ref}/modules-scripting-security.html#enable-dynamic-scripting[inline scripts],
-you can use JavaScript as follows:
+If you have set `script.inline: true` then you can use JavaScript as follows:
 
 [source,js]
 ----
@@ -71,7 +46,7 @@ GET test/_search
     "function_score": {
       "script_score": {
         "script": {
-          "inline": "doc[\"num\"].value * factor",
+          "source": "doc[\"num\"].value * factor",
           "lang": "javascript",
           "params": {
             "factor": 2
@@ -83,6 +58,7 @@ GET test/_search
 }
 ----
 // CONSOLE
+// TEST[skip:deprecated and no easy way in the docs test to mark warnings properly]
 
 [[lang-javascript-stored]]
 [float]
@@ -91,8 +67,7 @@ GET test/_search
 WARNING: Enabling stored scripts on an unprotected Elasticsearch cluster is dangerous.
 See <<lang-javascript-file>> for a safer option.
 
-If you have enabled {ref}/modules-scripting-security.html#enable-dynamic-scripting[stored scripts],
-you can use JavaScript as follows:
+If you have set `script.stored: true` you can use JavaScript as follows:
 
 [source,js]
 ----
@@ -117,7 +92,7 @@ GET test/_search
     "function_score": {
       "script_score": {
         "script": {
-          "id": "my_script", <2>
+          "stored": "my_script", <2>
           "lang": "javascript",
           "params": {
             "factor": 2
@@ -129,6 +104,7 @@ GET test/_search
 }
 ----
 // CONSOLE
+// TEST[skip:deprecated and no easy way in the docs test to mark warnings properly]
 
 <1> We store the script under the id `my_script`.
 <2> The function score query retrieves the script with id `my_script`.
@@ -182,5 +158,6 @@ GET test/_search
 }
 ----
 // CONSOLE
+// TEST[skip:deprecated and no easy way in the docs test to mark warnings properly]
 
 <1> The function score query retrieves the script with filename `my_script.javascript`.
diff --git a/docs/plugins/lang-python.asciidoc b/docs/plugins/lang-python.asciidoc
index a642e7f65cca5..b996570c0fac4 100644
--- a/docs/plugins/lang-python.asciidoc
+++ b/docs/plugins/lang-python.asciidoc
@@ -1,37 +1,13 @@
 [[lang-python]]
 === Python Language Plugin
 
-deprecated[5.0.0,Python will be replaced by the new scripting language {ref}/modules-scripting-painless.html[`Painless`]]
+deprecated[5.0.0,Python will be replaced by the new default scripting language `Painless`]
 
 The Python language plugin enables the use of Python in Elasticsearch
 scripts, via the http://www.jython.org/[Jython] Java implementation of Python.
 
-[[lang-python-install]]
-[float]
-==== Installation
-
-This plugin can be installed using the plugin manager:
-
-[source,sh]
-----------------------------------------------------------------
-sudo bin/elasticsearch-plugin install lang-python
-----------------------------------------------------------------
-
-The plugin must be installed on every node in the cluster, and each node must
-be restarted after installation.
-
-[[lang-python-remove]]
-[float]
-==== Removal
-
-The plugin can be removed with the following command:
-
-[source,sh]
-----------------------------------------------------------------
-sudo bin/elasticsearch-plugin remove lang-python
-----------------------------------------------------------------
-
-The node must be stopped before removing the plugin.
+:plugin_name: lang-python
+include::install_remove.asciidoc[]
 
 [[lang-python-usage]]
 ==== Using Python in Elasticsearch
@@ -49,8 +25,7 @@ Scripting is available in many APIs, but we will use an example with the
 WARNING: Enabling inline scripting on an unprotected Elasticsearch cluster is dangerous.
 See <<lang-python-file>> for a safer option.
 
-If you have enabled {ref}/modules-scripting-security.html#enable-dynamic-scripting[inline scripts],
-you can use Python as follows:
+If you have set `script.inline: true` then you can use Python as follows:
 
 [source,js]
 ----
@@ -70,7 +45,7 @@ GET test/_search
     "function_score": {
       "script_score": {
         "script": {
-          "inline": "doc[\"num\"].value * factor",
+          "source": "doc[\"num\"].value * factor",
           "lang": "python",
           "params": {
             "factor": 2
@@ -82,6 +57,7 @@ GET test/_search
 }
 ----
 // CONSOLE
+// TEST[skip:deprecated and no easy way in the docs test to mark warnings properly]
 
 [[lang-python-stored]]
 [float]
@@ -90,8 +66,7 @@ GET test/_search
 WARNING: Enabling stored scripts on an unprotected Elasticsearch cluster is dangerous.
 See <<lang-python-file>> for a safer option.
 
-If you have enabled {ref}/modules-scripting-security.html#enable-dynamic-scripting[stored scripts],
-you can use Python as follows:
+If you have set `script.stored: true` then you can use Python as follows:
 
 [source,js]
 ----
@@ -116,7 +91,7 @@ GET test/_search
     "function_score": {
       "script_score": {
         "script": {
-          "id": "my_script", <2>
+          "stored": "my_script", <2>
           "lang": "python",
           "params": {
             "factor": 2
@@ -129,6 +104,7 @@ GET test/_search
 
 ----
 // CONSOLE
+// TEST[skip:deprecated and no easy way in the docs test to mark warnings properly]
 
 <1> We store the script under the id `my_script`.
 <2> The function score query retrieves the script with id `my_script`.
@@ -182,5 +158,6 @@ GET test/_search
 }
 ----
 // CONSOLE
+// TEST[skip:deprecated and no easy way in the docs test to mark warnings properly]
 
 <1> The function score query retrieves the script with filename `my_script.py`.
diff --git a/docs/plugins/management.asciidoc b/docs/plugins/management.asciidoc
index b48d29da3abea..243aaf47e33c9 100644
--- a/docs/plugins/management.asciidoc
+++ b/docs/plugins/management.asciidoc
@@ -1,19 +1,16 @@
 [[management]]
-== Management and Site Plugins
+== Management Plugins
 
-Management and site plugins offer UIs for managing and interacting with
-Elasticsearch.
+Management plugins offer UIs for managing and interacting with Elasticsearch.
 
 [float]
 === Core management plugins
 
 The core management plugins are:
 
-link:/products/marvel[Marvel]::
+link:/products/x-pack/monitoring[X-Pack]::
 
-Marvel is a management and monitoring product for Elasticsearch. Marvel
+X-Pack contains the management and monitoring features for Elasticsearch. It
 aggregates cluster wide statistics and events and offers a single interface to
-view and analyze them. Marvel is free for development use but requires a
-license to run in production.
-
-
+view and analyze them. You can get a link:/subscriptions[free license] for basic
+monitoring or a higher level license for more advanced needs.
diff --git a/docs/plugins/mapper-attachments.asciidoc b/docs/plugins/mapper-attachments.asciidoc
index 0e21c6ab7051b..5ad086a301c71 100644
--- a/docs/plugins/mapper-attachments.asciidoc
+++ b/docs/plugins/mapper-attachments.asciidoc
@@ -9,32 +9,8 @@ using the Apache text extraction library http://lucene.apache.org/tika/[Tika].
 In practice, the plugin adds the `attachment` type when mapping properties so that documents can be populated with
 file attachment contents (encoded as `base64`).
 
-[[mapper-attachments-install]]
-[float]
-==== Installation
-
-This plugin can be installed using the plugin manager:
-
-[source,sh]
-----------------------------------------------------------------
-sudo bin/elasticsearch-plugin install mapper-attachments
-----------------------------------------------------------------
-
-The plugin must be installed on every node in the cluster, and each node must
-be restarted after installation.
-
-[[mapper-attachments-remove]]
-[float]
-==== Removal
-
-The plugin can be removed with the following command:
-
-[source,sh]
-----------------------------------------------------------------
-sudo bin/elasticsearch-plugin remove mapper-attachments
-----------------------------------------------------------------
-
-The node must be stopped before removing the plugin.
+:plugin_name: mapper-attachments
+include::install_remove.asciidoc[]
 
 [[mapper-attachments-helloworld]]
 ==== Hello, world
@@ -266,6 +242,7 @@ Will give you:
    "_shards": {
       "total": 5,
       "successful": 5,
+      "skipped": 0,
       "failed": 0
    },
    "hits": {
@@ -396,6 +373,7 @@ It gives back:
    "_shards": {
       "total": 5,
       "successful": 5,
+      "skipped": 0,
       "failed": 0
    },
    "hits": {
diff --git a/docs/plugins/mapper-murmur3.asciidoc b/docs/plugins/mapper-murmur3.asciidoc
index 28b7a2387ef0b..a714347ced743 100644
--- a/docs/plugins/mapper-murmur3.asciidoc
+++ b/docs/plugins/mapper-murmur3.asciidoc
@@ -5,32 +5,8 @@ The mapper-murmur3 plugin provides the ability to compute hash of field values
 at index-time and store them in the index. This can sometimes be helpful when
 running cardinality aggregations on high-cardinality and large string fields.
 
-[[mapper-murmur3-install]]
-[float]
-==== Installation
-
-This plugin can be installed using the plugin manager:
-
-[source,sh]
-----------------------------------------------------------------
-sudo bin/elasticsearch-plugin install mapper-murmur3
-----------------------------------------------------------------
-
-The plugin must be installed on every node in the cluster, and each node must
-be restarted after installation.
-
-[[mapper-murmur3-remove]]
-[float]
-==== Removal
-
-The plugin can be removed with the following command:
-
-[source,sh]
-----------------------------------------------------------------
-sudo bin/elasticsearch-plugin remove mapper-murmur3
-----------------------------------------------------------------
-
-The node must be stopped before removing the plugin.
+:plugin_name: mapper-murmur3
+include::install_remove.asciidoc[]
 
 [[mapper-murmur3-usage]]
 ==== Using the `murmur3` field
diff --git a/docs/plugins/mapper-size.asciidoc b/docs/plugins/mapper-size.asciidoc
index 3c0277895d2ea..27353414f063a 100644
--- a/docs/plugins/mapper-size.asciidoc
+++ b/docs/plugins/mapper-size.asciidoc
@@ -5,32 +5,8 @@ The mapper-size plugin provides the `_size` meta field which, when enabled,
 indexes the size in bytes of the original
 {ref}/mapping-source-field.html[`_source`] field.
 
-[[mapper-size-install]]
-[float]
-==== Installation
-
-This plugin can be installed using the plugin manager:
-
-[source,sh]
-----------------------------------------------------------------
-sudo bin/elasticsearch-plugin install mapper-size
-----------------------------------------------------------------
-
-The plugin must be installed on every node in the cluster, and each node must
-be restarted after installation.
-
-[[mapper-size-remove]]
-[float]
-==== Removal
-
-The plugin can be removed with the following command:
-
-[source,sh]
-----------------------------------------------------------------
-sudo bin/elasticsearch-plugin remove mapper-size
-----------------------------------------------------------------
-
-The node must be stopped before removing the plugin.
+:plugin_name: mapper-size
+include::install_remove.asciidoc[]
 
 [[mapper-size-usage]]
 ==== Using the `_size` field
diff --git a/docs/plugins/plugin-script.asciidoc b/docs/plugins/plugin-script.asciidoc
index f8fb8814129d9..1916d2669a1f0 100644
--- a/docs/plugins/plugin-script.asciidoc
+++ b/docs/plugins/plugin-script.asciidoc
@@ -20,7 +20,7 @@ sudo bin/elasticsearch-plugin -h
 .Running as root
 =====================
 If Elasticsearch was installed using the deb or rpm package then run
-`/usr/share/elasticsearch-plugin` as `root` so it can write to the appropriate files on disk.
+`/usr/share/elasticsearch/bin/elasticsearch-plugin` as `root` so it can write to the appropriate files on disk.
 Otherwise run `bin/elasticsearch-plugin` as the user that owns all of the Elasticsearch
 files.
 =====================
@@ -53,7 +53,7 @@ sudo bin/elasticsearch-plugin install analysis-icu
 This command will install the version of the plugin that matches your
 Elasticsearch version and also show a progress bar while downloading.
 
-[float]
+[[plugin-management-custom-url]]
 === Custom URL or file system
 
 A plugin can also be downloaded directly from a custom location by specifying the URL:
@@ -64,24 +64,45 @@ sudo bin/elasticsearch-plugin install [url] <1>
 -----------------------------------
 <1> must be a valid URL, the plugin name is determined from its descriptor.
 
-For instance, to install a plugin from your local file system, you could run:
-
+--
+Unix::
+To install a plugin from your local file system at `/path/to/plugin.zip`, you could run:
++
 [source,shell]
 -----------------------------------
 sudo bin/elasticsearch-plugin install file:///path/to/plugin.zip
 -----------------------------------
 
+Windows::
+To install a plugin from your local file system at `C:\path\to\plugin.zip`, you could run:
++
+[source,shell]
+-----------------------------------
+bin\elasticsearch-plugin install file:///C:/path/to/plugin.zip
+-----------------------------------
++
+NOTE: Any path that contains spaces must be wrapped in quotes!
+
+HTTP::
+To install a plugin from a HTTP URL:
++
+[source,shell]
+-----------------------------------
+sudo bin/elasticsearch-plugin install http://some.domain/path/to/plugin.zip
+-----------------------------------
++
 The plugin script will refuse to talk to an HTTPS URL with an untrusted
 certificate. To use a self-signed HTTPS cert, you will need to add the CA cert
 to a local Java truststore and pass the location to the script as follows:
-
++
 [source,shell]
 -----------------------------------
-sudo ES_JAVA_OPTS="-Djavax.net.ssl.trustStore=/path/to/trustStore.jks" bin/elasticsearch-plugin install https://....
+sudo ES_JAVA_OPTS="-Djavax.net.ssl.trustStore=/path/to/trustStore.jks" bin/elasticsearch-plugin install https://host/plugin.zip
 -----------------------------------
+--
 
-[[listing-removing]]
-=== Listing and Removing Installed Plugins
+[[listing-removing-updating]]
+=== Listing, Removing and Updating Installed Plugins
 
 [float]
 === Listing plugins
@@ -107,7 +128,26 @@ Plugins can be removed manually, by deleting the appropriate directory under
 sudo bin/elasticsearch-plugin remove [pluginname]
 -----------------------------------
 
-After a Java plugin has been removed, you will need to restart the node to complete the removal process.
+After a Java plugin has been removed, you will need to restart the node to
+complete the removal process.
+
+By default, plugin configuration files (if any) are preserved on disk; this is
+so that configuration is not lost while upgrading a plugin. If you wish to
+purge the configuration files while removing a plugin, use `-p` or `--purge`.
+This can option can be used after a plugin is removed to remove any lingering
+configuration files.
+
+[float]
+=== Updating plugins
+
+Plugins are built for a specific version of Elasticsearch, and therefore must be reinstalled
+each time Elasticsearch is updated.
+
+[source,shell]
+-----------------------------------
+sudo bin/elasticsearch-plugin remove [pluginname]
+sudo bin/elasticsearch-plugin install [pluginname]
+-----------------------------------
 
 === Other command line parameters
 
@@ -126,6 +166,24 @@ return the following exit codes:
 `74`:: IO error
 `70`:: any other error
 
+[float]
+=== Batch mode
+
+Certain plugins require more privileges than those provided by default in core
+Elasticsearch. These plugins will list the required privileges and ask the
+user for confirmation before continuing with installation.
+
+When running the plugin install script from another program (e.g. install
+automation scripts), the plugin script should detect that it is not being
+called from the console and skip the confirmation response, automatically
+granting all requested permissions.  If console detection fails, then batch
+mode can be forced by specifying `-b` or `--batch` as follows:
+
+[source,shell]
+-----------------------------------
+sudo bin/elasticsearch-plugin install --batch [pluginname]
+-----------------------------------
+
 [float]
 === Custom config directory
 
@@ -135,31 +193,9 @@ can do this as follows:
 
 [source,sh]
 ---------------------
-sudo bin/elasticsearch-plugin -Epath.conf=/path/to/custom/config/dir install <plugin name>
+sudo CONF_DIR=/path/to/conf/dir bin/elasticsearch-plugin install <plugin name>
 ---------------------
 
-You can also set the `CONF_DIR` environment variable to the custom config
-directory path.
-
-[float]
-=== Timeout settings
-
-By default, the `plugin` script will wait indefinitely when downloading before
-failing. The timeout parameter can be used to explicitly specify how long it
-waits. Here is some examples of setting it to different values:
-
-[source,shell]
------------------------------------
-# Wait for 30 seconds before failing
-sudo bin/elasticsearch-plugin install analysis-icu --timeout 30s
-
-# Wait for 1 minute before failing
-sudo bin/elasticsearch-plugin install analysis-icu --timeout 1m
-
-# Wait forever (default)
-sudo bin/elasticsearch-plugin install analysis-icu --timeout 0
------------------------------------
-
 [float]
 === Proxy settings
 
@@ -176,8 +212,8 @@ Or on Windows:
 
 [source,shell]
 ------------------------------------
-set ES_JAVA_OPTS="-DproxyHost=host_name -DproxyPort=port_number -Dhttps.proxyHost=host_name -Dhttps.proxyPort=https_port_number"
-bin/elasticsearch-plugin install analysis-icu
+set ES_JAVA_OPTS="-Dhttp.proxyHost=host_name -Dhttp.proxyPort=port_number -Dhttps.proxyHost=host_name -Dhttps.proxyPort=https_port_number"
+bin\elasticsearch-plugin install analysis-icu
 ------------------------------------
 
 === Plugins directory
diff --git a/docs/plugins/redirects.asciidoc b/docs/plugins/redirects.asciidoc
index 0f9c0b40f2c8c..ccda0d9ff06b9 100644
--- a/docs/plugins/redirects.asciidoc
+++ b/docs/plugins/redirects.asciidoc
@@ -13,12 +13,13 @@ one of the <<discovery,cloud discovery plugins>>.
 [role="exclude",id="cloud-aws"]
 === AWS Cloud Plugin
 
-The `cloud-aws` plugin has been split into two separate plugins:
+Looking for a hosted solution for Elasticsearch on AWS? Check out http://www.elastic.co/cloud.
+
+The Elasticsearch `cloud-aws` plugin has been split into two separate plugins:
 
 *  <<discovery-ec2>> (`discovery-ec2`)
 *  <<repository-s3>> (`repository-s3`)
 
-
 [role="exclude",id="cloud-azure"]
 === Azure Cloud Plugin
 
diff --git a/docs/plugins/repository-azure.asciidoc b/docs/plugins/repository-azure.asciidoc
index 03466f0c6432a..e643ad90312ab 100644
--- a/docs/plugins/repository-azure.asciidoc
+++ b/docs/plugins/repository-azure.asciidoc
@@ -4,32 +4,8 @@
 The Azure Repository plugin adds support for using Azure as a repository for
 {ref}/modules-snapshots.html[Snapshot/Restore].
 
-[[repository-azure-install]]
-[float]
-==== Installation
-
-This plugin can be installed using the plugin manager:
-
-[source,sh]
-----------------------------------------------------------------
-sudo bin/elasticsearch-plugin install repository-azure
-----------------------------------------------------------------
-
-The plugin must be installed on every node in the cluster, and each node must
-be restarted after installation.
-
-[[repository-azure-remove]]
-[float]
-==== Removal
-
-The plugin can be removed with the following command:
-
-[source,sh]
-----------------------------------------------------------------
-sudo bin/elasticsearch-plugin remove repository-azure
-----------------------------------------------------------------
-
-The node must be stopped before removing the plugin.
+:plugin_name: repository-azure
+include::install_remove.asciidoc[]
 
 [[repository-azure-usage]]
 ==== Azure Repository
@@ -69,6 +45,12 @@ It's not set by default which means that elasticsearch is using the
 http://azure.github.io/azure-storage-java/com/microsoft/azure/storage/RequestOptions.html#setTimeoutIntervalInMs(java.lang.Integer)[default value]
 set by the azure client (known as 5 minutes).
 
+`max_retries` can help to control the exponential backoff policy. It will fix the number of retries
+in case of failures before considering the snapshot is failing. Defaults to `3` retries.
+The initial backoff period is defined by Azure SDK as `30s`. Which means `30s` of wait time
+before retrying after a first timeout or failure. The maximum backoff period is defined by Azure SDK as
+`90s`.
+
 [source,yaml]
 ----
 cloud:
@@ -79,13 +61,28 @@ cloud:
                 account: your_azure_storage_account1
                 key: your_azure_storage_key1
                 default: true
+                max_retries: 7
             my_account2:
                 account: your_azure_storage_account2
                 key: your_azure_storage_key2
                 timeout: 30s
 ----
 
-In this example, timeout will be 10s for `my_account1` and 30s for `my_account2`.
+In this example, timeout will be `10s` per try for `my_account1` with `7` retries before failing
+and `30s` per try for `my_account2` with `3` retries.
+
+[IMPORTANT]
+.Supported Azure Storage Account types
+===============================================
+The Azure Repository plugin works with all Standard storage accounts
+
+* Standard Locally Redundant Storage - `Standard_LRS`
+* Standard Zone-Redundant Storage - `Standard_ZRS`
+* Standard Geo-Redundant Storage - `Standard_GRS`
+* Standard Read Access Geo-Redundant Storage - `Standard_RAGRS`
+
+https://azure.microsoft.com/en-gb/documentation/articles/storage-premium-storage[Premium Locally Redundant Storage] (`Premium_LRS`) is **not supported** as it is only usable as VM disk storage, not as general storage.
+===============================================
 
 [[repository-azure-repository-settings]]
 ===== Repository settings
@@ -99,7 +96,8 @@ The Azure repository supports following settings:
 
 `container`::
 
-    Container name. Defaults to `elasticsearch-snapshots`
+    Container name. You must create the azure container before creating the repository.
+    Defaults to `elasticsearch-snapshots`.
 
 `base_path`::
 
@@ -118,14 +116,14 @@ The Azure repository supports following settings:
     setting doesn't affect index files that are already compressed by default.
     Defaults to `false`.
 
-`read_only`::
+`readonly`::
 
-    Makes repository read-only. coming[2.1.0]  Defaults to `false`.
+    Makes repository read-only. Defaults to `false`.
 
 `location_mode`::
 
     `primary_only` or `secondary_only`. Defaults to `primary_only`. Note that if you set it
-    to `secondary_only`, it will force `read_only` to true.
+    to `secondary_only`, it will force `readonly` to true.
 
 Some examples, using scripts:
 
@@ -175,28 +173,12 @@ Example using Java:
 [source,java]
 ----
 client.admin().cluster().preparePutRepository("my_backup_java1")
-    .setType("azure").setSettings(Settings.settingsBuilder()
+    .setType("azure").setSettings(Settings.builder()
         .put(Storage.CONTAINER, "backup-container")
         .put(Storage.CHUNK_SIZE, new ByteSizeValue(32, ByteSizeUnit.MB))
     ).get();
 ----
 
-[[repository-azure-global-settings]]
-===== Global repositories settings
-
-All those repository settings can also be defined globally in `elasticsearch.yml` file using prefix
-`repositories.azure.`. For example:
-
-[source,yaml]
-----
-repositories.azure:
-    container: backup-container
-    base_path: backups
-    chunk_size: 32m
-    compress": true
-----
-
-
 [[repository-azure-validation]]
 ===== Repository validation rules
 
diff --git a/docs/plugins/repository-gcs.asciidoc b/docs/plugins/repository-gcs.asciidoc
index d0a5c74843908..90c6e8f0ba4eb 100644
--- a/docs/plugins/repository-gcs.asciidoc
+++ b/docs/plugins/repository-gcs.asciidoc
@@ -4,34 +4,8 @@
 The GCS repository plugin adds support for using the https://cloud.google.com/storage/[Google Cloud Storage]
 service as a repository for {ref}/modules-snapshots.html[Snapshot/Restore].
 
-[[repository-gcs-install]]
-[float]
-==== Installation
-
-This plugin can be installed using the plugin manager:
-
-[source,sh]
-----------------------------------------------------------------
-sudo bin/elasticsearch-plugin install repository-gcs
-----------------------------------------------------------------
-
-NOTE: The plugin requires new permission to be installed in order to work
-
-The plugin must be installed on every node in the cluster, and each node must
-be restarted after installation.
-
-[[repository-gcs-remove]]
-[float]
-==== Removal
-
-The plugin can be removed with the following command:
-
-[source,sh]
-----------------------------------------------------------------
-sudo bin/elasticsearch-plugin remove repository-gcs
-----------------------------------------------------------------
-
-The node must be stopped before removing the plugin.
+:plugin_name: repository-gcs
+include::install_remove.asciidoc[]
 
 [[repository-gcs-usage]]
 ==== Getting started
@@ -67,34 +41,19 @@ The bucket should now be created.
 
 The plugin supports two authentication modes:
 
-* the built-in <<repository-gcs-using-compute-engine, Compute Engine authentication>>. This mode is
+* The built-in <<repository-gcs-using-compute-engine, Compute Engine authentication>>. This mode is
 recommended if your elasticsearch node is running on a Compute Engine virtual machine.
 
-* the <<repository-gcs-using-service-account, Service Account>> authentication mode.
+* Specifying <<repository-gcs-using-service-account, Service Account>> credentials.
 
 [[repository-gcs-using-compute-engine]]
 ===== Using Compute Engine
-When running on Compute Engine, the plugin use the Google's built-in authentication mechanism to
+When running on Compute Engine, the plugin use Google's built-in authentication mechanism to
 authenticate on the Storage service. Compute Engine virtual machines are usually associated to a
 default service account. This service account can be found in the VM instance details in the
 https://console.cloud.google.com/compute/[Compute Engine console].
 
-To indicate that a repository should use the built-in authentication,
-the repository `service_account` setting must be set to `_default_`:
-
-[source,js]
-----
-PUT _snapshot/my_gcs_repository_on_compute_engine
-{
-  "type": "gcs",
-  "settings": {
-    "bucket": "my_bucket",
-    "service_account": "_default_"
-  }
-}
-----
-// CONSOLE
-// TEST[skip:we don't have gcs setup while testing this]
+This is the default authentication mode and requires no configuration.
 
 NOTE: The Compute Engine VM must be allowed to use the Storage service. This can be done only at VM
 creation time, when "Storage" access can be configured to "Read/Write" permission. Check your
@@ -102,7 +61,7 @@ instance details at the section "Cloud API access scopes".
 
 [[repository-gcs-using-service-account]]
 ===== Using a Service Account
-If your elasticsearch node is not running on Compute Engine, or if you don't want to use Google
+If your elasticsearch node is not running on Compute Engine, or if you don't want to use Google's
 built-in authentication mechanism, you can authenticate on the Storage service using a
 https://cloud.google.com/iam/docs/overview#service_account[Service Account] file.
 
@@ -133,10 +92,14 @@ A service account file looks like this:
 ----
 // NOTCONSOLE
 
-This file must be copied in the `config` directory of the elasticsearch installation and on
-every node of the cluster.
+This file must be stored in the {ref}/secure-settings.html[elasticsearch keystore], under a setting name
+of the form `gcs.client.NAME.credentials_file`, where `NAME` is the name of the client configuration.
+The default client name is `default`, but a different client name can be specified in repository
+settings using `client`.
 
-To indicate that a repository should use a service account file:
+For example, if specifying the credentials file in the keystore under
+`gcs.client.my_alternate_client.credentials_file`, you can configure a repository to use these
+credentials like this:
 
 [source,js]
 ----
@@ -145,7 +108,7 @@ PUT _snapshot/my_gcs_repository
   "type": "gcs",
   "settings": {
     "bucket": "my_bucket",
-    "service_account": "service_account.json"
+    "client": "my_alternate_client"
   }
 }
 ----
@@ -176,8 +139,7 @@ PUT _snapshot/my_gcs_repository
 {
   "type": "gcs",
   "settings": {
-    "bucket": "my_bucket",
-    "service_account": "service_account.json"
+    "bucket": "my_bucket"
   }
 }
 ----
@@ -190,10 +152,10 @@ The following settings are supported:
 
     The name of the bucket to be used for snapshots. (Mandatory)
 
-`service_account`::
+`client`::
 
-    The service account to use. It can be a relative path to a service account JSON file
-    or the value `_default_` that indicate to use built-in Compute Engine service account.
+    The client congfiguration to use. This controls which credentials are used to connect
+    to Compute Engine.
 
 `base_path`::
 
diff --git a/docs/plugins/repository-hdfs.asciidoc b/docs/plugins/repository-hdfs.asciidoc
index 62b1d2a95ca13..20c62a5861a4e 100644
--- a/docs/plugins/repository-hdfs.asciidoc
+++ b/docs/plugins/repository-hdfs.asciidoc
@@ -4,32 +4,8 @@
 The HDFS repository plugin adds support for using HDFS File System as a repository for
 {ref}/modules-snapshots.html[Snapshot/Restore].
 
-[[repository-hdfs-install]]
-[float]
-==== Installation
-
-This plugin can be installed through the plugin manager:
-
-[source,sh]
-----------------------------------------------------------------
-sudo bin/elasticsearch-plugin install repository-hdfs
-----------------------------------------------------------------
-
-The plugin must be installed on _every_ node in the cluster, and each node must
-be restarted after installation.
-
-[[repository-hdfs-remove]]
-[float]
-==== Removal
-
-The plugin can be removed by specifying the _installed_ package:
-
-[source,sh]
-----------------------------------------------------------------
-sudo bin/elasticsearch-plugin remove repository-hdfs
-----------------------------------------------------------------
-
-The node must be stopped before removing the plugin.
+:plugin_name: repository-hdfs
+include::install_remove.asciidoc[]
 
 [[repository-hdfs-usage]]
 ==== Getting started with HDFS
@@ -92,16 +68,113 @@ The following settings are supported:
 
     Override the chunk size. (Disabled by default)
 
+`security.principal`::
+
+    Kerberos principal to use when connecting to a secured HDFS cluster.
+    If you are using a service principal for your elasticsearch node, you may
+    use the `_HOST` pattern in the principal name and the plugin will replace
+    the pattern with the hostname of the node at runtime (see
+    link:repository-hdfs-security-runtime[Creating the Secure Repository]).
+
+[[repository-hdfs-security]]
+==== Hadoop Security
+
+The HDFS Repository Plugin integrates seamlessly with Hadoop's authentication model. The following authentication
+methods are supported by the plugin:
+
+[horizontal]
+`simple`::
 
-Alternatively, you can define the `hdfs` repository and its settings in your `elasticsearch.yml`:
-[source,yaml]
+    Also means "no security" and is enabled by default. Uses information from underlying operating system account
+    running elasticsearch to inform Hadoop of the name of the current user. Hadoop makes no attempts to verify this
+    information.
+
+`kerberos`::
+
+    Authenticates to Hadoop through the usage of a Kerberos principal and keytab. Interfacing with HDFS clusters
+    secured with Kerberos requires a few additional steps to enable (See <<repository-hdfs-security-keytabs>> and
+    <<repository-hdfs-security-runtime>> for more info)
+
+[[repository-hdfs-security-keytabs]]
+[float]
+===== Principals and Keytabs
+Before attempting to connect to a secured HDFS cluster, provision the Kerberos principals and keytabs that the
+Elasticsearch nodes will use for authenticating to Kerberos. For maximum security and to avoid tripping up the Kerberos
+replay protection, you should create a service principal per node, following the pattern of
+`elasticsearch/hostname@REALM`.
+
+WARNING: In some cases, if the same principal is authenticating from multiple clients at once, services may reject
+authentication for those principals under the assumption that they could be replay attacks. If you are running the
+plugin in production with multiple nodes you should be using a unique service principal for each node.
+
+On each Elasticsearch node, place the appropriate keytab file in the node's configuration location under the
+`repository-hdfs` directory using the name `krb5.keytab`:
+
+[source, bash]
 ----
-repositories:
-  hdfs:
-    uri: "hdfs://<host>:<port>/"    \# required - HDFS address only
-    path: "some/path"               \# required - path within the file-system where data is stored/loaded
-    load_defaults: "true"           \# optional - whether to load the default Hadoop configuration (default) or not
-    conf.<key> : "<value>"          \# optional - 'inlined' key=value added to the Hadoop configuration
-    compress: "false"               \# optional - whether to compress the metadata or not (default)
-    chunk_size: "10mb"              \# optional - chunk size (disabled by default)
+$> cd elasticsearch/config
+$> ls
+elasticsearch.yml  jvm.options        log4j2.properties  repository-hdfs/   scripts/
+$> cd repository-hdfs
+$> ls
+krb5.keytab
+----
+// TEST[skip:this is for demonstration purposes only
+
+NOTE: Make sure you have the correct keytabs! If you are using a service principal per node (like
+`elasticsearch/hostname@REALM`) then each node will need its own unique keytab file for the principal assigned to that
+host!
+
+// Setup at runtime (principal name)
+[[repository-hdfs-security-runtime]]
+[float]
+===== Creating the Secure Repository
+Once your keytab files are in place and your cluster is started, creating a secured HDFS repository is simple. Just
+add the name of the principal that you will be authenticating as in the repository settings under the
+`security.principal` option:
+
+[source,js]
 ----
+PUT _snapshot/my_hdfs_repository
+{
+  "type": "hdfs",
+  "settings": {
+    "uri": "hdfs://namenode:8020/",
+    "path": "/user/elasticsearch/respositories/my_hdfs_repository",
+    "security.principal": "elasticsearch@REALM"
+  }
+}
+----
+// CONSOLE
+// TEST[skip:we don't have hdfs set up while testing this]
+
+If you are using different service principals for each node, you can use the `_HOST` pattern in your principal
+name. Elasticsearch will automatically replace the pattern with the hostname of the node at runtime:
+
+[source,js]
+----
+PUT _snapshot/my_hdfs_repository
+{
+  "type": "hdfs",
+  "settings": {
+    "uri": "hdfs://namenode:8020/",
+    "path": "/user/elasticsearch/respositories/my_hdfs_repository",
+    "security.principal": "elasticsearch/_HOST@REALM"
+  }
+}
+----
+// CONSOLE
+// TEST[skip:we don't have hdfs set up while testing this]
+
+[[repository-hdfs-security-authorization]]
+[float]
+===== Authorization
+Once Elasticsearch is connected and authenticated to HDFS, HDFS will infer a username to use for
+authorizing file access for the client. By default, it picks this username from the primary part of
+the kerberos principal used to authenticate to the service. For example, in the case of a principal
+like `elasticsearch@REALM` or `elasticsearch/hostname@REALM` then the username that HDFS
+extracts for file access checks will be `elasticsearch`.
+
+NOTE: The repository plugin makes no assumptions of what Elasticsearch's principal name is. The main fragment of the
+Kerberos principal is not required to be `elasticsearch`. If you have a principal or service name that works better
+for you or your organization then feel free to use it instead!
\ No newline at end of file
diff --git a/docs/plugins/repository-s3.asciidoc b/docs/plugins/repository-s3.asciidoc
index 5848b827c63d7..861335f1e0d6b 100644
--- a/docs/plugins/repository-s3.asciidoc
+++ b/docs/plugins/repository-s3.asciidoc
@@ -6,32 +6,8 @@ The S3 repository plugin adds support for using S3 as a repository for
 
 *If you are looking for a hosted solution of Elasticsearch on AWS, please visit http://www.elastic.co/cloud.*
 
-[[repository-s3-install]]
-[float]
-==== Installation
-
-This plugin can be installed using the plugin manager:
-
-[source,sh]
-----------------------------------------------------------------
-sudo bin/elasticsearch-plugin install repository-s3
-----------------------------------------------------------------
-
-The plugin must be installed on every node in the cluster, and each node must
-be restarted after installation.
-
-[[repository-s3-remove]]
-[float]
-==== Removal
-
-The plugin can be removed with the following command:
-
-[source,sh]
-----------------------------------------------------------------
-sudo bin/elasticsearch-plugin remove repository-s3
-----------------------------------------------------------------
-
-The node must be stopped before removing the plugin.
+:plugin_name: repository-s3
+include::install_remove.asciidoc[]
 
 [[repository-s3-usage]]
 ==== Getting started with AWS
@@ -113,17 +89,20 @@ You can specifically set it for s3 only using `cloud.aws.s3.region`.
 The available values are:
 
 * `us-east` (`us-east-1`) for US East (N. Virginia)
+* `us-east-2` for US East (Ohio)
 * `us-west` (`us-west-1`) for US West (N. California)
 * `us-west-2` for US West (Oregon)
-* `ap-south-1` for Asia Pacific (Mumbai)
+* `ap-south` (`ap-south-1`) for Asia Pacific (Mumbai)
 * `ap-southeast` (`ap-southeast-1`) for Asia Pacific (Singapore)
 * `ap-southeast-2` for Asia Pacific (Sydney)
 * `ap-northeast` (`ap-northeast-1`) for Asia Pacific (Tokyo)
 * `ap-northeast-2` (`ap-northeast-2`) for Asia Pacific (Seoul)
 * `eu-west` (`eu-west-1`) for EU (Ireland)
+* `eu-west-2` (`eu-west-2`) for EU (London)
 * `eu-central` (`eu-central-1`) for EU (Frankfurt)
 * `sa-east` (`sa-east-1`) for South America (São Paulo)
 * `cn-north` (`cn-north-1`) for China (Beijing)
+* `ca-central` (`ca-central-1`) for Canada (Central)
 
 [[repository-s3-usage-signer]]
 ===== S3 Signer API
@@ -135,6 +114,17 @@ signer to use.
 If you are using a compatible S3 service which do not support Version 4 signing process, you may need to
 use `S3SignerType`, which is Signature Version 2.
 
+===== Read timeout
+
+Read timeout determines the amount of time to wait for data to be transferred over an established,
+open connection before the connection is timed out. Defaults to AWS SDK default value (`50s`).
+It can be configured with `cloud.aws.read_timeout` (or `cloud.aws.s3.read_timeout`) setting:
+
+[source, yaml]
+----
+cloud.aws.read_timeout: 30s
+----
+
 [[repository-s3-repository]]
 ==== S3 Repository
 
@@ -178,6 +168,9 @@ The following settings are supported:
 
     Specifies the path within bucket to repository data. Defaults to
     value of `repositories.s3.base_path` or to root directory if not set.
+    Previously, the base_path could take a leading `/` (forward slash).
+    However, this has been deprecated and setting the base_path now should
+    omit the leading `/`.
 
 `access_key`::
 
@@ -214,7 +207,8 @@ The following settings are supported:
     to split the chunk into several parts, each of `buffer_size` length, and
     to upload each part in its own request. Note that setting a buffer
     size lower than `5mb` is not allowed since it will prevents the use of the
-    Multipart API and may result in upload errors. Defaults to `100mb`.
+    Multipart API and may result in upload errors. Defaults to the minimum
+    between `100mb` and `5%` of the heap size.
 
 `max_retries`::
 
@@ -224,7 +218,7 @@ The following settings are supported:
 
     Set to `true` if you want to throttle retries. Defaults to AWS SDK default value (`false`).
 
-`read_only`::
+`readonly`::
 
     Makes repository read-only. Defaults to `false`.
 
@@ -246,14 +240,10 @@ The following settings are supported:
 
 `path_style_access`::
 
-    Activate path style access for [virtual hosting of buckets](http://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html).
+    Activate path style access for http://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html[virtual hosting of buckets].
     The default behaviour is to detect which access style to use based on the configured endpoint (an IP will result
     in path-style access) and the bucket being accessed (some buckets are not valid DNS names).
 
-Note that you can define S3 repository settings for all S3 repositories in `elasticsearch.yml` configuration file.
-They are all prefixed with `repositories.s3.`. For example, you can define compression for all S3 repositories
-by setting `repositories.s3.compress: true` in `elasticsearch.yml`.
-
 The S3 repositories use the same credentials as the rest of the AWS services
 provided by this plugin (`discovery`). See <<repository-s3-usage>> for details.
 
@@ -381,48 +371,3 @@ See <<repository-s3-repository>> for details.
 AWS instances resolve S3 endpoints to a public IP. If the elasticsearch instances reside in a private subnet in an AWS VPC then all traffic to S3 will go through that VPC's NAT instance. If your VPC's NAT instance is a smaller instance size (e.g. a t1.micro) or is handling a high volume of network traffic your bandwidth to S3 may be limited by that NAT instance's networking bandwidth limitations.
 
 Instances residing in a public subnet in an AWS VPC will connect to S3 via the VPC's internet gateway and not be bandwidth limited by the VPC's NAT instance.
-
-[[repository-s3-testing]]
-==== Testing AWS
-
-Integrations tests in this plugin require working AWS configuration and therefore disabled by default. Three buckets
-and two iam users have to be created. The first iam user needs access to two buckets in different regions and the final
-bucket is exclusive for the other iam user. To enable tests prepare a config file elasticsearch.yml with the following
-content:
-
-[source,yaml]
-----
-cloud:
-    aws:
-        access_key: AKVAIQBF2RECL7FJWGJQ
-        secret_key: vExyMThREXeRMm/b/LRzEB8jWwvzQeXgjqMX+6br
-
-repositories:
-    s3:
-        bucket: "bucket_name"
-        region: "us-west-2"
-        private-bucket:
-            bucket: <bucket not accessible by default key>
-            access_key: <access key>
-            secret_key: <secret key>
-        remote-bucket:
-            bucket: <bucket in other region>
-            region: <region>
-	external-bucket:
-	    bucket: <bucket>
-	    access_key: <access key>
-	    secret_key: <secret key>
-	    endpoint: <endpoint>
-	    protocol: <protocol>
-
-----
-
-Replace all occurrences of `access_key`, `secret_key`, `endpoint`, `protocol`, `bucket` and `region` with your settings.
-Please, note that the test will delete all snapshot/restore related files in the specified buckets.
-
-To run test:
-
-[source,sh]
-----
-mvn -Dtests.aws=true -Dtests.config=/path/to/config/file/elasticsearch.yml clean test
-----
diff --git a/docs/plugins/security.asciidoc b/docs/plugins/security.asciidoc
index 95ba68a6f0514..d113c12bfc25a 100644
--- a/docs/plugins/security.asciidoc
+++ b/docs/plugins/security.asciidoc
@@ -8,12 +8,12 @@ Security plugins add a security layer to  Elasticsearch.
 
 The core security plugins are:
 
-link:/products/shield[Shield]::
+link:/products/x-pack/security[X-Pack]::
 
-Shield is the Elastic product that makes it easy for anyone to add
-enterprise-grade security to their ELK stack. Designed to address the growing security
-needs of thousands of enterprises using ELK today, Shield provides peace of
-mind when it comes to protecting your data.
+X-Pack is the Elastic product that makes it easy for anyone to add
+enterprise-grade security to their Elastic Stack. Designed to address the
+growing security needs of thousands of enterprises using the Elastic Stack
+today, X-Pack provides peace of mind when it comes to protecting your data.
 
 [float]
 === Community contributed security plugins
@@ -25,4 +25,3 @@ The following plugins have been contributed by our community:
 
 * https://github.com/sscarduzio/elasticsearch-readonlyrest-plugin[Readonly REST]:
   High performance access control for Elasticsearch native REST API (by Simone Scarduzio)
-
diff --git a/docs/plugins/store-smb.asciidoc b/docs/plugins/store-smb.asciidoc
index 731894ae0a884..4f713568655cb 100644
--- a/docs/plugins/store-smb.asciidoc
+++ b/docs/plugins/store-smb.asciidoc
@@ -3,32 +3,8 @@
 
 The Store SMB plugin works around for a bug in Windows SMB and Java on windows.
 
-[[store-smb-install]]
-[float]
-==== Installation
-
-This plugin can be installed using the plugin manager:
-
-[source,sh]
-----------------------------------------------------------------
-sudo bin/elasticsearch-plugin install store-smb
-----------------------------------------------------------------
-
-The plugin must be installed on every node in the cluster, and each node must
-be restarted after installation.
-
-[[store-smb-remove]]
-[float]
-==== Removal
-
-The plugin can be removed with the following command:
-
-[source,sh]
-----------------------------------------------------------------
-sudo bin/elasticsearch-plugin remove store-smb
-----------------------------------------------------------------
-
-The node must be stopped before removing the plugin.
+:plugin_name: store-smb
+include::install_remove.asciidoc[]
 
 [[store-smb-usage]]
 ==== Working around a bug in Windows SMB and Java on windows
diff --git a/docs/python/index.asciidoc b/docs/python/index.asciidoc
deleted file mode 100644
index 8440ed1cd0dd5..0000000000000
--- a/docs/python/index.asciidoc
+++ /dev/null
@@ -1,127 +0,0 @@
-= elasticsearch-py
-
-== Overview
-
-Official low-level client for Elasticsearch. Its goal is to provide common
-ground for all Elasticsearch-related code in Python; because of this it tries
-to be opinion-free and very extendable. The full documentation is available at
-http://elasticsearch-py.rtfd.org/
-
-.Elasticsearch DSL
-************************************************************************************
-For a more high level client library with more limited scope, have a look at
-http://elasticsearch-dsl.rtfd.org/[elasticsearch-dsl] - a more pythonic library
-sitting on top of `elasticsearch-py`.
-
-It provides a more convenient and idiomatic way to write and manipulate
-http://elasticsearch-dsl.readthedocs.org/en/latest/search_dsl.html[queries]. It
-stays close to the Elasticsearch JSON DSL, mirroring its terminology and
-structure while exposing the whole range of the DSL from Python either directly
-using defined classes or a queryset-like expressions.
-
-It also provides an optional
-http://elasticsearch-dsl.readthedocs.org/en/latest/persistence.html#doctype[persistence
-layer] for working with documents as Python objects in an ORM-like fashion:
-defining mappings, retrieving and saving documents, wrapping the document data
-in user-defined classes.
-************************************************************************************
-
-
-=== Installation
-
-It can be installed with pip:
-
-[source,sh]
-------------------------------------
-pip install elasticsearch
-------------------------------------
-
-=== Versioning
-
-There are two branches for development - `master` and `1.x`. Master branch is
-used to track all the changes for Elasticsearch 2.0 and beyond whereas 1.x
-tracks Elasticsearch 1.*.
-
-Releases with major version 1 (1.X.Y) are to be used with Elasticsearch 1.* and
-later, 0.4 releases are meant to work with Elasticsearch 0.90.*.
-
-The recommended way to set your requirements in your `setup.py` or
-`requirements.txt` is:
-
-[source,txt]
-------------------------------------
-    # Elasticsearch 2.x
-    elasticsearch>=2.0.0,<3.0.0
-
-    # Elasticsearch 1.x
-    elasticsearch>=1.0.0,<2.0.0
-------------------------------------
-
-=== Example use
-
-Simple use-case:
-
-[source,python]
-------------------------------------
->>> from datetime import datetime
->>> from elasticsearch import Elasticsearch
-
-# by default we connect to localhost:9200
->>> es = Elasticsearch()
-
-# datetimes will be serialized
->>> es.index(index="my-index", doc_type="test-type", id=42, body={"any": "data", "timestamp": datetime.now()})
-{u'_id': u'42', u'_index': u'my-index', u'_type': u'test-type', u'_version': 1, u'ok': True}
-
-# but not deserialized
->>> es.get(index="my-index", doc_type="test-type", id=42)['_source']
-{u'any': u'data', u'timestamp': u'2013-05-12T19:45:31.804229'}
-------------------------------------
-
-[NOTE]
-All the API calls map the raw REST api as closely as possible, including
-the distinction between required and optional arguments to the calls. This
-means that the code makes distinction between positional and keyword arguments;
-we, however, recommend that people use keyword arguments for all calls for
-consistency and safety.
-
-=== Features
-
-The client's features include:
-
-* translating basic Python data types to and from json (datetimes are not
-  decoded for performance reasons)
-
-* configurable automatic discovery of cluster nodes
-
-* persistent connections
-
-* load balancing (with pluggable selection strategy) across all available nodes
-
-* failed connection penalization (time based - failed connections won't be
-  retried until a timeout is reached)
-
-* thread safety
-
-* pluggable architecture
-
-The client also contains a convenient set of
-http://elasticsearch-py.readthedocs.org/en/master/helpers.html[helpers] for
-some of the more engaging tasks like bulk indexing and reindexing.
-
-
-=== License
-
-Copyright 2013-2016 Elasticsearch
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
diff --git a/docs/reference/aggregations.asciidoc b/docs/reference/aggregations.asciidoc
index ada134238bbf4..f2fdd9a16de82 100644
--- a/docs/reference/aggregations.asciidoc
+++ b/docs/reference/aggregations.asciidoc
@@ -11,7 +11,7 @@ the execution defines what this document set is (e.g. a top-level aggregation ex
 query/filters of the search request).
 
 There are many different types of aggregations, each with its own purpose and output. To better understand these types,
-it is often easier to break them into three main families:
+it is often easier to break them into four main families:
 
 <<search-aggregations-bucket, _Bucketing_>>::
 				A family of aggregations that build buckets, where each bucket is associated with a _key_ and a document
@@ -58,6 +58,7 @@ The following snippet captures the basic structure of aggregations:
     [,"<aggregation_name_2>" : { ... } ]*
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 The `aggregations` object (the key `aggs` can also be used) in the JSON holds the aggregations to be computed. Each aggregation
 is associated with a logical name that the user defines (e.g. if the aggregation computes the average price, then it would
@@ -91,11 +92,15 @@ enables defining all the "dynamic" expressions in the script as parameters, whic
 between calls (this will ensure the use of the cached compiled scripts in Elasticsearch).
 ===============================
 
-Scripts can generate a single value or multiple values per document. When generating multiple values, one can use the
-`script_values_sorted` settings to indicate whether these values are sorted or not. Internally, Elasticsearch can
-perform optimizations when dealing with sorted values (for example, with the `min` aggregations, knowing the values are
-sorted, Elasticsearch will skip the iterations over all the values and rely on the first value in the list to be the
-minimum value among all other values associated with the same document).
+Elasticsearch uses the type of the field in the mapping in order to figure out
+how to run the aggregation and format the response. However there are two cases
+in which Elasticsearch cannot figure out this information: unmapped fields (for
+instance in the case of a search request across multiple indices, and only some
+of them have a mapping for the field) and pure scripts. For those cases, it is
+possible to give Elasticsearch a hint using the `value_type` option, which
+accepts the following values: `string`, `long` (works for all integer types),
+`double` (works for all decimal types like `float` or `scaled_float`), `date`,
+`ip` and `boolean`.
 
 --
 
diff --git a/docs/reference/aggregations/bucket.asciidoc b/docs/reference/aggregations/bucket.asciidoc
index 851511257cfc1..f98fd52febaf3 100644
--- a/docs/reference/aggregations/bucket.asciidoc
+++ b/docs/reference/aggregations/bucket.asciidoc
@@ -13,6 +13,8 @@ aggregated for the buckets created by their "parent" bucket aggregation.
 There are different bucket aggregators, each with a different "bucketing" strategy. Some define a single bucket, some
 define fixed number of multiple buckets, and others dynamically create the buckets during the aggregation process.
 
+include::bucket/adjacency-matrix-aggregation.asciidoc[]
+
 include::bucket/children-aggregation.asciidoc[]
 
 include::bucket/datehistogram-aggregation.asciidoc[]
diff --git a/docs/reference/aggregations/bucket/adjacency-matrix-aggregation.asciidoc b/docs/reference/aggregations/bucket/adjacency-matrix-aggregation.asciidoc
new file mode 100644
index 0000000000000..8e4102683f147
--- /dev/null
+++ b/docs/reference/aggregations/bucket/adjacency-matrix-aggregation.asciidoc
@@ -0,0 +1,116 @@
+[[search-aggregations-bucket-adjacency-matrix-aggregation]]
+=== Adjacency Matrix Aggregation
+
+A bucket aggregation returning a form of https://en.wikipedia.org/wiki/Adjacency_matrix[adjacency matrix].
+The request provides a collection of named filter expressions, similar to the `filters` aggregation
+request. 
+Each bucket in the response represents a non-empty cell in the matrix of intersecting filters.
+
+experimental[The `adjacency_matrix` aggregation is a new feature and we may evolve its design as we get feedback on its use.  As a result, the API for this feature may change in non-backwards compatible ways]
+
+
+Given filters named `A`, `B` and `C` the response would return buckets with the following names:
+
+
+[options="header"]
+|=======================
+|  h|A   h|B  h|C   
+h|A |A   |A&B |A&C 
+h|B |    |B   |B&C 
+h|C |    |    |C  
+|=======================
+
+The intersecting buckets e.g `A&C` are labelled using a combination of the two filter names separated by
+the ampersand character. Note that the response does not also include a "C&A" bucket as this would be the
+same set of documents as "A&C". The matrix is said to be _symmetric_ so we only return half of it. To do this we sort 
+the filter name strings and always use the lowest of a pair as the value to the left of the "&" separator. 
+
+An alternative `separator` parameter can be passed in the request if clients wish to use a separator string 
+other than the default of the ampersand.
+
+
+Example:
+
+[source,js]
+--------------------------------------------------
+PUT /emails/message/_bulk?refresh
+{ "index" : { "_id" : 1 } }
+{ "accounts" : ["hillary", "sidney"]}
+{ "index" : { "_id" : 2 } }
+{ "accounts" : ["hillary", "donald"]}
+{ "index" : { "_id" : 3 } }
+{ "accounts" : ["vladimir", "donald"]}
+
+GET emails/message/_search
+{
+  "size": 0,
+  "aggs" : {
+    "interactions" : {
+      "adjacency_matrix" : {
+        "filters" : {
+          "grpA" : { "terms" : { "accounts" : ["hillary", "sidney"] }},
+          "grpB" : { "terms" : { "accounts" : ["donald", "mitt"] }},
+          "grpC" : { "terms" : { "accounts" : ["vladimir", "nigel"] }}
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+
+In the above example, we analyse email messages to see which groups of individuals 
+have exchanged messages.
+We will get counts for each group individually and also a count of messages for pairs
+of groups that have recorded interactions.
+
+Response:
+
+[source,js]
+--------------------------------------------------
+{
+  "took": 9,
+  "timed_out": false,
+  "_shards": ...,
+  "hits": ...,
+  "aggregations": {
+    "interactions": {
+      "buckets": [
+        {
+          "key":"grpA",
+          "doc_count": 2
+        },
+        {
+          "key":"grpA&grpB",
+          "doc_count": 1
+        },
+        {
+          "key":"grpB",
+          "doc_count": 2
+        },
+        {
+          "key":"grpB&grpC",
+          "doc_count": 1
+        },
+        {
+          "key":"grpC",
+          "doc_count": 1
+        }
+      ]
+    }
+  }
+}
+--------------------------------------------------
+// TESTRESPONSE[s/"took": 9/"took": $body.took/]
+// TESTRESPONSE[s/"_shards": \.\.\./"_shards": $body._shards/]
+// TESTRESPONSE[s/"hits": \.\.\./"hits": $body.hits/]
+
+==== Usage
+On its own this aggregation can provide all of the data required to create an undirected weighted graph.
+However, when used with child aggregations such as a `date_histogram` the results can provide the
+additional levels of data required to perform https://en.wikipedia.org/wiki/Dynamic_network_analysis[dynamic network analysis]
+where examining interactions _over time_ becomes important.
+
+==== Limitations
+For N filters the matrix of buckets produced can be N²/2 and so there is a default maximum 
+imposed of 100 filters . This setting can be changed using the `index.max_adjacency_matrix_filters` index-level setting.
diff --git a/docs/reference/aggregations/bucket/children-aggregation.asciidoc b/docs/reference/aggregations/bucket/children-aggregation.asciidoc
index e69877d97f289..36262f7953c78 100644
--- a/docs/reference/aggregations/bucket/children-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/children-aggregation.asciidoc
@@ -5,19 +5,23 @@ A special single bucket aggregation that enables aggregating from buckets on par
 
 This aggregation relies on the <<mapping-parent-field,_parent field>> in the mapping. This aggregation has a single option:
 
-* `type` - The what child type the buckets in the parent space should be mapped to.
+* `type` - The child type that the buckets in the parent space should be mapped to.
 
 For example, let's say we have an index of questions and answers. The answer type has the following `_parent` field in the mapping:
 [source,js]
 --------------------------------------------------
+PUT child_example
 {
-    "answer" : {
-        "_parent" : {
-            "type" : "question"
+    "mappings": {
+        "answer" : {
+            "_parent" : {
+                "type" : "question"
+            }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 The question typed document contain a tag field and the answer typed documents contain an owner field. With the `children`
 aggregation the tag buckets can be mapped to the owner buckets in a single request even though the two fields exist in
@@ -26,6 +30,7 @@ two different kinds of documents.
 An example of a question typed document:
 [source,js]
 --------------------------------------------------
+PUT child_example/question/1
 {
     "body": "<p>I have Windows 2003 server and i bought a new Windows 2008 server...",
     "title": "Whats the best way to file transfer my site from server to a newer one?",
@@ -33,33 +38,49 @@ An example of a question typed document:
         "windows-server-2003",
         "windows-server-2008",
         "file-transfer"
-    ],
+    ]
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
-An example of an answer typed document:
+Examples of `answer` typed documents:
 [source,js]
 --------------------------------------------------
+PUT child_example/answer/1?parent=1&refresh
 {
     "owner": {
         "location": "Norfolk, United Kingdom",
         "display_name": "Sam",
         "id": 48
     },
-    "body": "<p>Unfortunately your pretty much limited to FTP...",
+    "body": "<p>Unfortunately you're pretty much limited to FTP...",
     "creation_date": "2009-05-04T13:45:37.030"
 }
+PUT child_example/answer/2?parent=1&refresh
+{
+    "owner": {
+        "location": "Norfolk, United Kingdom",
+        "display_name": "Troll",
+        "id": 49
+    },
+    "body": "<p>Use Linux...",
+    "creation_date": "2009-05-05T13:45:37.030"
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 The following request can be built that connects the two together:
 
 [source,js]
 --------------------------------------------------
+POST child_example/_search?size=0
 {
   "aggs": {
     "top-tags": {
       "terms": {
-        "field": "tags",
+        "field": "tags.keyword",
         "size": 10
       },
       "aggs": {
@@ -70,7 +91,7 @@ The following request can be built that connects the two together:
           "aggs": {
             "top-names": {
               "terms": {
-                "field": "owner.display_name",
+                "field": "owner.display_name.keyword",
                 "size": 10
               }
             }
@@ -81,6 +102,8 @@ The following request can be built that connects the two together:
   }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 <1> The `type` points to type / mapping with the name `answer`.
 
@@ -91,245 +114,83 @@ Possible response:
 [source,js]
 --------------------------------------------------
 {
+  "timed_out": false,
+  "took": 25,
+  "_shards": {
+    "total": 5,
+    "successful": 5,
+    "skipped" : 0,
+    "failed": 0
+  },
+  "hits": {
+    "total": 3,
+    "max_score": 0.0,
+    "hits": []
+  },
   "aggregations": {
     "top-tags": {
+      "doc_count_error_upper_bound": 0,
+      "sum_other_doc_count": 0,
       "buckets": [
         {
-          "key": "windows-server-2003",
-          "doc_count": 25365, <1>
+          "key": "file-transfer",
+          "doc_count": 1, <1>
           "to-answers": {
-            "doc_count": 36004, <2>
+            "doc_count": 2, <2>
             "top-names": {
+              "doc_count_error_upper_bound": 0,
+              "sum_other_doc_count": 0,
               "buckets": [
                 {
                   "key": "Sam",
-                  "doc_count": 274
-                },
-                {
-                  "key": "chris",
-                  "doc_count": 19
-                },
-                {
-                  "key": "david",
-                  "doc_count": 14
-                },
-                ...
-              ]
-            }
-          }
-        },
-        {
-          "key": "linux",
-          "doc_count": 18342,
-          "to-answers": {
-            "doc_count": 6655,
-            "top-names": {
-              "buckets": [
-                {
-                  "key": "abrams",
-                  "doc_count": 25
-                },
-                {
-                  "key": "ignacio",
-                  "doc_count": 25
+                  "doc_count": 1
                 },
                 {
-                  "key": "vazquez",
-                  "doc_count": 25
-                },
-                ...
+                  "key": "Troll",
+                  "doc_count": 1
+                }
               ]
             }
           }
         },
         {
-          "key": "windows",
-          "doc_count": 18119,
-          "to-answers": {
-            "doc_count": 24051,
-            "top-names": {
-              "buckets": [
-                {
-                  "key": "molly7244",
-                  "doc_count": 265
-                },
-                {
-                  "key": "david",
-                  "doc_count": 27
-                },
-                {
-                  "key": "chris",
-                  "doc_count": 26
-                },
-                ...
-              ]
-            }
-          }
-        },
-        {
-          "key": "osx",
-          "doc_count": 10971,
-          "to-answers": {
-            "doc_count": 5902,
-            "top-names": {
-              "buckets": [
-                {
-                  "key": "diago",
-                  "doc_count": 4
-                },
-                {
-                  "key": "albert",
-                  "doc_count": 3
-                },
-                {
-                  "key": "asmus",
-                  "doc_count": 3
-                },
-                ...
-              ]
-            }
-          }
-        },
-        {
-          "key": "ubuntu",
-          "doc_count": 8743,
-          "to-answers": {
-            "doc_count": 8784,
-            "top-names": {
-              "buckets": [
-                {
-                  "key": "ignacio",
-                  "doc_count": 9
-                },
-                {
-                  "key": "abrams",
-                  "doc_count": 8
-                },
-                {
-                  "key": "molly7244",
-                  "doc_count": 8
-                },
-                ...
-              ]
-            }
-          }
-        },
-        {
-          "key": "windows-xp",
-          "doc_count": 7517,
-          "to-answers": {
-            "doc_count": 13610,
-            "top-names": {
-              "buckets": [
-                {
-                  "key": "molly7244",
-                  "doc_count": 232
-                },
-                {
-                  "key": "chris",
-                  "doc_count": 9
-                },
-                {
-                  "key": "john",
-                  "doc_count": 9
-                },
-                ...
-              ]
-            }
-          }
-        },
-        {
-          "key": "networking",
-          "doc_count": 6739,
-          "to-answers": {
-            "doc_count": 2076,
-            "top-names": {
-              "buckets": [
-                {
-                  "key": "molly7244",
-                  "doc_count": 6
-                },
-                {
-                  "key": "alnitak",
-                  "doc_count": 5
-                },
-                {
-                  "key": "chris",
-                  "doc_count": 3
-                },
-                ...
-              ]
-            }
-          }
-        },
-        {
-          "key": "mac",
-          "doc_count": 5590,
-          "to-answers": {
-            "doc_count": 999,
-            "top-names": {
-              "buckets": [
-                {
-                  "key": "abrams",
-                  "doc_count": 2
-                },
-                {
-                  "key": "ignacio",
-                  "doc_count": 2
-                },
-                {
-                  "key": "vazquez",
-                  "doc_count": 2
-                },
-                ...
-              ]
-            }
-          }
-        },
-        {
-          "key": "wireless-networking",
-          "doc_count": 4409,
+          "key": "windows-server-2003",
+          "doc_count": 1, <1>
           "to-answers": {
-            "doc_count": 6497,
+            "doc_count": 2, <2>
             "top-names": {
+              "doc_count_error_upper_bound": 0,
+              "sum_other_doc_count": 0,
               "buckets": [
                 {
-                  "key": "molly7244",
-                  "doc_count": 61
-                },
-                {
-                  "key": "chris",
-                  "doc_count": 5
+                  "key": "Sam",
+                  "doc_count": 1
                 },
                 {
-                  "key": "mike",
-                  "doc_count": 5
-                },
-                ...
+                  "key": "Troll",
+                  "doc_count": 1
+                }
               ]
             }
           }
         },
         {
-          "key": "windows-8",
-          "doc_count": 3601,
+          "key": "windows-server-2008",
+          "doc_count": 1, <1>
           "to-answers": {
-            "doc_count": 4263,
+            "doc_count": 2, <2>
             "top-names": {
+              "doc_count_error_upper_bound": 0,
+              "sum_other_doc_count": 0,
               "buckets": [
                 {
-                  "key": "molly7244",
-                  "doc_count": 3
-                },
-                {
-                  "key": "msft",
-                  "doc_count": 2
+                  "key": "Sam",
+                  "doc_count": 1
                 },
                 {
-                  "key": "user172132",
-                  "doc_count": 2
-                },
-                ...
+                  "key": "Troll",
+                  "doc_count": 1
+                }
               ]
             }
           }
@@ -339,6 +200,7 @@ Possible response:
   }
 }
 --------------------------------------------------
+// TESTRESPONSE[s/"took": 25/"took": $body.took/]
 
-<1> The number of question documents with the tag `windows-server-2003`.
-<2> The number of answer documents that are related to question documents with the tag `windows-server-2003`.
+<1> The number of question documents with the tag `file-transfer`, `windows-server-2003`, etc.
+<2> The number of answer documents that are related to question documents with the tag `file-transfer`, `windows-server-2003`, etc.
diff --git a/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc b/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc
index 4d9147067ec50..57f8fc3059052 100644
--- a/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc
@@ -12,9 +12,10 @@ Requesting bucket intervals of a month.
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
     "aggs" : {
-        "articles_over_time" : {
+        "sales_over_time" : {
             "date_histogram" : {
                 "field" : "date",
                 "interval" : "month"
@@ -23,6 +24,8 @@ Requesting bucket intervals of a month.
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
 Available expressions for interval: `year`, `quarter`, `month`, `week`, `day`, `hour`, `minute`, `second`
 
@@ -32,9 +35,10 @@ time unit (e.g., `1.5h` could instead be specified as `90m`).
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
     "aggs" : {
-        "articles_over_time" : {
+        "sales_over_time" : {
             "date_histogram" : {
                 "field" : "date",
                 "interval" : "90m"
@@ -43,6 +47,8 @@ time unit (e.g., `1.5h` could instead be specified as `90m`).
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
 ==== Keys
 
@@ -56,9 +62,10 @@ TIP: If no `format` is specified, then it will use the first date
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
     "aggs" : {
-        "articles_over_time" : {
+        "sales_over_time" : {
             "date_histogram" : {
                 "field" : "date",
                 "interval" : "1M",
@@ -68,6 +75,8 @@ TIP: If no `format` is specified, then it will use the first date
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
 <1> Supports expressive date <<date-format-pattern,format pattern>>
 
@@ -76,25 +85,31 @@ Response:
 [source,js]
 --------------------------------------------------
 {
+    ...
     "aggregations": {
-        "articles_over_time": {
+        "sales_over_time": {
             "buckets": [
                 {
-                    "key_as_string": "2013-02-02",
-                    "key": 1328140800000,
-                    "doc_count": 1
+                    "key_as_string": "2015-01-01",
+                    "key": 1420070400000,
+                    "doc_count": 3
                 },
                 {
-                    "key_as_string": "2013-03-02",
-                    "key": 1330646400000,
+                    "key_as_string": "2015-02-01",
+                    "key": 1422748800000,
                     "doc_count": 2
                 },
-                ...
+                {
+                    "key_as_string": "2015-03-01",
+                    "key": 1425168000000,
+                    "doc_count": 2
+                }
             ]
         }
     }
 }
 --------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
 
 ==== Time Zone
 
@@ -110,12 +125,12 @@ Consider the following example:
 
 [source,js]
 ---------------------------------
-PUT my_index/log/1
+PUT my_index/log/1?refresh
 {
   "date": "2015-10-01T00:30:00Z"
 }
 
-PUT my_index/log/2
+PUT my_index/log/2?refresh
 {
   "date": "2015-10-01T01:30:00Z"
 }
@@ -132,6 +147,7 @@ GET my_index/_search?size=0
   }
 }
 ---------------------------------
+// CONSOLE
 
 UTC is used if no time zone is specified, which would result in both of these
 documents being placed into the same day bucket, which starts at midnight UTC
@@ -139,18 +155,22 @@ on 1 October 2015:
 
 [source,js]
 ---------------------------------
-"aggregations": {
-  "by_day": {
-    "buckets": [
-      {
-        "key_as_string": "2015-10-01T00:00:00.000Z",
-        "key":           1443657600000,
-        "doc_count":     2
-      }
-    ]
+{
+  ...
+  "aggregations": {
+    "by_day": {
+      "buckets": [
+        {
+          "key_as_string": "2015-10-01T00:00:00.000Z",
+          "key":           1443657600000,
+          "doc_count":     2
+        }
+      ]
+    }
   }
 }
 ---------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
 
 If a `time_zone` of `-01:00` is specified, then midnight starts at one hour before
 midnight UTC:
@@ -170,37 +190,55 @@ GET my_index/_search?size=0
   }
 }
 ---------------------------------
+// CONSOLE
+// TEST[continued]
 
 Now the first document falls into the bucket for 30 September 2015, while the
 second document falls into the bucket for 1 October 2015:
 
 [source,js]
 ---------------------------------
-"aggregations": {
-  "by_day": {
-    "buckets": [
-      {
-        "key_as_string": "2015-09-30T00:00:00.000-01:00", <1>
-        "key": 1443571200000,
-        "doc_count": 1
-      },
-      {
-        "key_as_string": "2015-10-01T00:00:00.000-01:00", <1>
-        "key": 1443657600000,
-        "doc_count": 1
-      }
-    ]
+{
+  ...
+  "aggregations": {
+    "by_day": {
+      "buckets": [
+        {
+          "key_as_string": "2015-09-30T00:00:00.000-01:00", <1>
+          "key": 1443574800000,
+          "doc_count": 1
+        },
+        {
+          "key_as_string": "2015-10-01T00:00:00.000-01:00", <1>
+          "key": 1443661200000,
+          "doc_count": 1
+        }
+      ]
+    }
   }
 }
 ---------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
+
 <1> The `key_as_string` value represents midnight on each day
     in the specified time zone.
 
+WARNING: When using time zones that follow DST (daylight savings time) changes,
+buckets close to the moment when those changes happen can have slightly different
+sizes than would be expected from the used `interval`.
+For example, consider a DST start in the `CET` time zone: on 27 March 2016 at 2am,
+clocks were turned forward 1 hour to 3am local time. When using `day` as `interval`,
+the bucket covering that day will only hold data for 23 hours instead of the usual
+24 hours for other buckets. The same is true for shorter intervals like e.g. 12h.
+Here, we will have only a 11h bucket on the morning of 27 March when the DST shift
+happens.
+
+
 ==== Offset
 
 The `offset` parameter is used to change the start value of each bucket by the
 specified positive (`+`) or negative offset (`-`) duration, such as `1h` for
-an hour, or `1M` for a month. See <<time-units>> for more possible time
+an hour, or `1d` for a day. See <<time-units>> for more possible time
 duration options.
 
 For instance, when using an interval of `day`, each bucket runs from midnight
@@ -209,12 +247,12 @@ to run from 6am to 6am:
 
 [source,js]
 -----------------------------
-PUT my_index/log/1
+PUT my_index/log/1?refresh
 {
   "date": "2015-10-01T05:30:00Z"
 }
 
-PUT my_index/log/2
+PUT my_index/log/2?refresh
 {
   "date": "2015-10-01T06:30:00Z"
 }
@@ -232,33 +270,92 @@ GET my_index/_search?size=0
   }
 }
 -----------------------------
+// CONSOLE
 
 Instead of a single bucket starting at midnight, the above request groups the
 documents into buckets starting at 6am:
 
 [source,js]
 -----------------------------
-"aggregations": {
-  "by_day": {
-    "buckets": [
-      {
-        "key_as_string": "2015-09-30T06:00:00.000Z",
-        "key": 1443592800000,
-        "doc_count": 1
-      },
-      {
-        "key_as_string": "2015-10-01T06:00:00.000Z",
-        "key": 1443679200000,
-        "doc_count": 1
-      }
-    ]
+{
+  ...
+  "aggregations": {
+    "by_day": {
+      "buckets": [
+        {
+          "key_as_string": "2015-09-30T06:00:00.000Z",
+          "key": 1443592800000,
+          "doc_count": 1
+        },
+        {
+          "key_as_string": "2015-10-01T06:00:00.000Z",
+          "key": 1443679200000,
+          "doc_count": 1
+        }
+      ]
+    }
   }
 }
 -----------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
 
 NOTE: The start `offset` of each bucket is calculated after the `time_zone`
 adjustments have been made.
 
+==== Keyed Response
+
+Setting the `keyed` flag to `true` will associate a unique string key with each bucket and return the ranges as a hash rather than an array:
+
+[source,js]
+--------------------------------------------------
+POST /sales/_search?size=0
+{
+    "aggs" : {
+        "sales_over_time" : {
+            "date_histogram" : {
+                "field" : "date",
+                "interval" : "1M",
+                "format" : "yyyy-MM-dd",
+                "keyed": true
+            }
+        }
+    }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
+
+Response:
+
+[source,js]
+--------------------------------------------------
+{
+    ...
+    "aggregations": {
+        "sales_over_time": {
+            "buckets": {
+                "2015-01-01": {
+                    "key_as_string": "2015-01-01",
+                    "key": 1420070400000,
+                    "doc_count": 3
+                },
+                "2015-02-01": {
+                    "key_as_string": "2015-02-01",
+                    "key": 1422748800000,
+                    "doc_count": 2
+                },
+                "2015-03-01": {
+                    "key_as_string": "2015-03-01",
+                    "key": 1425168000000,
+                    "doc_count": 2
+                }
+            }
+        }
+    }
+}
+--------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
+
 ==== Scripts
 
 Like with the normal <<search-aggregations-bucket-histogram-aggregation,histogram>>, both document level scripts and
@@ -276,17 +373,75 @@ had a value.
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
     "aggs" : {
-        "publish_date" : {
+        "sale_date" : {
              "date_histogram" : {
-                 "field" : "publish_date",
+                 "field" : "date",
                  "interval": "year",
-                 "missing": "2000-01-01" <1>
+                 "missing": "2000/01/01" <1>
              }
          }
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
 <1> Documents without a value in the `publish_date` field will fall into the same bucket as documents that have the value `2000-01-01`.
+
+==== Use of a script to aggregate by day of the week
+
+There are some cases where date histogram can't help us, like for example, when we need
+to aggregate the results by day of the week.
+In this case to overcome the problem, we can use a script that returns the day of the week:
+
+
+[source,js]
+--------------------------------------------------
+POST /sales/_search?size=0
+{
+    "aggs": {
+        "dayOfWeek": {
+            "terms": {
+                "script": {
+                    "lang": "painless",
+                    "source": "doc['date'].date.dayOfWeek"
+                }
+            }
+        }
+    }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
+
+Response:
+
+[source,js]
+--------------------------------------------------
+{
+  ...
+  "aggregations": {
+    "dayOfWeek": {
+      "doc_count_error_upper_bound": 0,
+      "sum_other_doc_count": 0,
+      "buckets": [
+        {
+          "key": "7",
+          "doc_count": 4
+        },
+        {
+          "key": "4",
+          "doc_count": 3
+        }
+      ]
+    }
+  }
+}
+--------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
+
+The response will contain all the buckets having as key the relative day of
+the week: 1 for Monday, 2 for Tuesday... 7 for Sunday.
diff --git a/docs/reference/aggregations/bucket/daterange-aggregation.asciidoc b/docs/reference/aggregations/bucket/daterange-aggregation.asciidoc
index a69b2b3cb1138..42c64f23cd335 100644
--- a/docs/reference/aggregations/bucket/daterange-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/daterange-aggregation.asciidoc
@@ -8,6 +8,7 @@ Example:
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
     "aggs": {
         "range": {
@@ -23,6 +24,9 @@ Example:
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales s/now-10M\/M/10-2015/]
+
 <1> < now minus 10 months, rounded down to the start of the month.
 <2> >= now minus 10 months, rounded down to the start of the month.
 
@@ -35,25 +39,27 @@ Response:
 --------------------------------------------------
 {
     ...
-
     "aggregations": {
         "range": {
             "buckets": [
                 {
-                    "to": 1.3437792E+12,
-                    "to_as_string": "08-2012",
-                    "doc_count": 7
+                    "to": 1.4436576E12,
+                    "to_as_string": "10-2015",
+                    "doc_count": 7,
+                    "key": "*-10-2015"
                 },
                 {
-                    "from": 1.3437792E+12,
-                    "from_as_string": "08-2012",
-                    "doc_count": 2
+                    "from": 1.4436576E12,
+                    "from_as_string": "10-2015",
+                    "doc_count": 0,
+                    "key": "10-2015-*"
                 }
             ]
         }
     }
 }
 --------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
 
 [[date-format-pattern]]
 ==== Date Format/Pattern
@@ -125,21 +131,131 @@ to round to the beginning of the day in the CET time zone, you can do the follow
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
    "aggs": {
-           "range": {
-               "date_range": {
-                   "field": "date",
-                   "time_zone": "CET",
-                   "ranges": [
-                      { "to": "2016-02-15/d" }, <1>
-                      { "from": "2016-02-15/d", "to" : "now/d" <2>},
-                      { "from": "now/d" },
-                  ]
-              }
+       "range": {
+           "date_range": {
+               "field": "date",
+               "time_zone": "CET",
+               "ranges": [
+                  { "to": "2016/02/01" }, <1>
+                  { "from": "2016/02/01", "to" : "now/d" <2>},
+                  { "from": "now/d" }
+              ]
           }
       }
-  }
+   }
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
+
 <1> This date will be converted to `2016-02-15T00:00:00.000+01:00`.
 <2> `now/d` will be rounded to the beginning of the day in the CET time zone.
+
+==== Keyed Response
+
+Setting the `keyed` flag to `true` will associate a unique string key with each bucket and return the ranges as a hash rather than an array:
+
+[source,js]
+--------------------------------------------------
+POST /sales/_search?size=0
+{
+    "aggs": {
+        "range": {
+            "date_range": {
+                "field": "date",
+                "format": "MM-yyy",
+                "ranges": [
+                    { "to": "now-10M/M" },
+                    { "from": "now-10M/M" }
+                ],
+                "keyed": true
+            }
+        }
+    }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:sales s/now-10M\/M/10-2015/]
+
+Response:
+
+[source,js]
+--------------------------------------------------
+{
+    ...
+    "aggregations": {
+        "range": {
+            "buckets": {
+                "*-10-2015": {
+                    "to": 1.4436576E12,
+                    "to_as_string": "10-2015",
+                    "doc_count": 7
+                },
+                "10-2015-*": {
+                    "from": 1.4436576E12,
+                    "from_as_string": "10-2015",
+                    "doc_count": 0
+                }
+            }
+        }
+    }
+}
+--------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
+
+It is also possible to customize the key for each range:
+
+[source,js]
+--------------------------------------------------
+POST /sales/_search?size=0
+{
+    "aggs": {
+        "range": {
+            "date_range": {
+                "field": "date",
+                "format": "MM-yyy",
+                "ranges": [
+                    { "from": "01-2015",  "to": "03-2015", "key": "quarter_01" },
+                    { "from": "03-2015", "to": "06-2015", "key": "quarter_02" }
+                ],
+                "keyed": true
+            }
+        }
+    }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
+
+Response:
+
+[source,js]
+--------------------------------------------------
+{
+    ...
+    "aggregations": {
+        "range": {
+            "buckets": {
+                "quarter_01": {
+                    "from": 1.4200704E12,
+                    "from_as_string": "01-2015",
+                    "to": 1.425168E12,
+                    "to_as_string": "03-2015",
+                    "doc_count": 5
+                },
+                "quarter_02": {
+                    "from": 1.425168E12,
+                    "from_as_string": "03-2015",
+                    "to": 1.4331168E12,
+                    "to_as_string": "06-2015",
+                    "doc_count": 2
+                }
+            }
+        }
+    }
+}
+--------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
diff --git a/docs/reference/aggregations/bucket/diversified-sampler-aggregation.asciidoc b/docs/reference/aggregations/bucket/diversified-sampler-aggregation.asciidoc
index 70412d2680a15..8366e79787691 100644
--- a/docs/reference/aggregations/bucket/diversified-sampler-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/diversified-sampler-aggregation.asciidoc
@@ -1,37 +1,53 @@
 [[search-aggregations-bucket-diversified-sampler-aggregation]]
-=== Sampler Aggregation
+=== Diversified Sampler Aggregation
 
 experimental[]
 
-A filtering aggregation used to limit any sub aggregations' processing to a sample of the top-scoring documents. Diversity settings are 
-used to limit the number of matches that share a common value such as an "author".
+Like the `sampler` aggregation this is a filtering aggregation used to limit any sub aggregations' processing to a sample of the top-scoring documents.
+The `diversified_sampler` aggregation adds the ability to limit the number of matches that share a common value such as an "author".
+
+NOTE: Any good market researcher will tell you that when working with samples of data it is important
+that the sample represents a healthy variety of opinions rather than being skewed by any single voice.
+The same is true with aggregations and sampling with these diversify settings can offer a way to remove the bias in your content (an over-populated geography, 
+a large spike in a timeline or an over-active forum spammer).  
+
 
 .Example use cases:
 * Tightening the focus of analytics to high-relevance matches rather than the potentially very long tail of low-quality matches
 * Removing bias from analytics by ensuring fair representation of content from different sources
 * Reducing the running cost of aggregations that can produce useful results using only samples e.g. `significant_terms`
  
+A choice of `field` or `script` setting is used to provide values used for de-duplication and the `max_docs_per_value` setting controls the maximum 
+number of documents collected on any one shard which share a common value. The default setting for `max_docs_per_value` is 1.
+
+The aggregation will throw an error if the choice of `field` or `script` produces multiple values for a single document (de-duplication using multi-valued fields is not supported due to efficiency concerns).
+
 
 Example:
 
+We might want to see which tags are strongly associated with `#elasticsearch` on StackOverflow
+forum posts but ignoring the effects of some prolific users with a tendency to misspell #Kibana as #Cabana.
+
 [source,js]
 --------------------------------------------------
+POST /stackoverflow/_search?size=0
 {
     "query": {
-        "match": {
-            "text": "iphone"
+        "query_string": {
+            "query": "tags:elasticsearch"
         }
     },
     "aggs": {
-        "sample": {
-            "sampler": {
+        "my_unbiased_sample": {
+            "diversified_sampler": {
                 "shard_size": 200,
-                "field" : "user.id"   
+                "field" : "author"   
             },
             "aggs": {
                 "keywords": {
                     "significant_terms": {
-                        "field": "text"
+                        "field": "tags",
+                        "exclude": ["elasticsearch"]
                     }
                 }
             }
@@ -39,6 +55,8 @@ Example:
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:stackoverflow]
 
 Response:
 
@@ -46,92 +64,118 @@ Response:
 --------------------------------------------------
 {
     ...
-        "aggregations": {
-        "sample": {
-            "doc_count": 1000,<1>
+    "aggregations": {
+        "my_unbiased_sample": {
+            "doc_count": 151,<1>
             "keywords": {<2>
-                "doc_count": 1000,
+                "doc_count": 151,
+                "bg_count": 650,
                 "buckets": [
-                    ...
                     {
-                        "key": "bend",
-                        "doc_count": 58,
-                        "score": 37.982536582524276,
-                        "bg_count": 103
-                    },
-                    ....
+                        "key": "kibana",
+                        "doc_count": 150,
+                        "score": 2.213,
+                        "bg_count": 200
+                    }
+                ]
+            }
+        }
+    }
 }
 --------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
+// TESTRESPONSE[s/2.213/$body.aggregations.my_unbiased_sample.keywords.buckets.0.score/]
 
-<1> 1000 documents were sampled in total becase we asked for a maximum of 200 from an index with 5 shards. The cost of performing the nested significant_terms aggregation was therefore limited rather than unbounded.
-<2> The results of the significant_terms aggregation are not skewed by any single over-active Twitter user because we asked for a maximum of one tweet from any one user in our sample.
-
+<1> 151 documents were sampled in total.
+<2> The results of the significant_terms aggregation are not skewed by any single author's quirks because we asked for a maximum of one post from any one author in our sample.
 
-==== shard_size
-
-The `shard_size` parameter limits how many top-scoring documents are collected in the sample processed on each shard.
-The default value is 100.
+==== Scripted example:
 
-==== Controlling diversity
-=`field` or `script` and `max_docs_per_value` settings are used to control the maximum number of documents collected on any one shard which share a common value.
-The choice of value (e.g. `author`) is loaded from a regular `field` or derived dynamically by a `script`.
-
-The aggregation will throw an error if the choice of field or script produces multiple values for a document.
-It is currently not possible to offer this form of de-duplication using many values, primarily due to concerns over efficiency.
-
-NOTE: Any good market researcher will tell you that when working with samples of data it is important
-that the sample represents a healthy variety of opinions rather than being skewed by any single voice.
-The same is true with aggregations and sampling with these diversify settings can offer a way to remove the bias in your content (an over-populated geography, a large spike in a timeline or an over-active forum spammer).  
-
-==== Field
-
-Controlling diversity using a field:
+In this scenario we might want to diversify on a combination of field values. We can use a `script` to produce a hash of the 
+multiple values in a tags field to ensure we don't have a sample that consists of the same repeated combinations of tags.
 
 [source,js]
 --------------------------------------------------
+POST /stackoverflow/_search?size=0
 {
-    "aggs" : {
-        "sample" : {
-            "diverisfied_sampler" : {
-                "field" : "author",
-                "max_docs_per_value" : 3
+    "query": {
+        "query_string": {
+            "query": "tags:kibana"
+        }
+    },
+    "aggs": {
+        "my_unbiased_sample": {
+            "diversified_sampler": {
+                "shard_size": 200,
+                "max_docs_per_value" : 3,
+                "script" : {
+                    "lang": "painless",
+                    "source": "doc['tags'].values.hashCode()"
+                }   
+            },
+            "aggs": {
+                "keywords": {
+                    "significant_terms": {
+                        "field": "tags",
+                        "exclude": ["kibana"]
+                    }
+                }
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:stackoverflow]
 
-Note that the `max_docs_per_value` setting applies on a per-shard basis only for the purposes of shard-local sampling.
-It is not intended as a way of providing a global de-duplication feature on search results.
-
-
-
-==== Script
-
-Controlling diversity using a script:
+Response:
 
 [source,js]
 --------------------------------------------------
 {
-    "aggs" : {
-        "sample" : {
-            "diversified_sampler" : {
-                "script" : { 
-                    "lang" : "painless",
-                    "inline" : "doc['author'].value + '/' + doc['genre'].value"
-                }
+    ...
+    "aggregations": {
+        "my_unbiased_sample": {
+            "doc_count": 6,
+            "keywords": {
+                "doc_count": 6,
+                "bg_count": 650,
+                "buckets": [
+                    {
+                        "key": "logstash",
+                        "doc_count": 3,
+                        "score": 2.213,
+                        "bg_count": 50
+                    },
+                    {
+                        "key": "elasticsearch",
+                        "doc_count": 3,
+                        "score": 1.34,
+                        "bg_count": 200
+                    },
+                ]
             }
         }
     }
 }
 --------------------------------------------------
-Note in the above example we chose to use the default `max_docs_per_value` setting of 1 and combine author and genre fields to ensure 
-each shard sample has, at most, one match for an author/genre pair.
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
+// TESTRESPONSE[s/2.213/$body.aggregations.my_unbiased_sample.keywords.buckets.0.score/]
+// TESTRESPONSE[s/1.34/$body.aggregations.my_unbiased_sample.keywords.buckets.1.score/]
+
+==== shard_size
+
+The `shard_size` parameter limits how many top-scoring documents are collected in the sample processed on each shard.
+The default value is 100.
+
+==== max_docs_per_value
+The `max_docs_per_value` is an optional parameter and limits how many documents are permitted per choice of de-duplicating value.
+The default setting is "1".
 
 
 ==== execution_hint
 
-When using the settings to control diversity, the optional `execution_hint` setting can influence the management of the values used for de-duplication.
+The optional `execution_hint` setting can influence the management of the values used for de-duplication.
 Each option will hold up to `shard_size` values in memory while performing de-duplication but the type of value held can be controlled as follows:
  
  - hold field values directly (`map`)
@@ -145,12 +189,12 @@ Please note that Elasticsearch will ignore the choice of execution hint if it is
 ==== Limitations
 
 ===== Cannot be nested under `breadth_first` aggregations
-Being a quality-based filter the sampler aggregation needs access to the relevance score produced for each document.
+Being a quality-based filter the diversified_sampler aggregation needs access to the relevance score produced for each document.
 It therefore cannot be nested under a `terms` aggregation which has the `collect_mode` switched from the default `depth_first` mode to `breadth_first` as this discards scores.
 In this situation an error will be thrown.
 
 ===== Limited de-dup logic.
-The de-duplication logic in the diversify settings applies only at a shard level so will not apply across shards.
+The de-duplication logic applies only at a shard level so will not apply across shards.
 
 ===== No specialized syntax for geo/date fields
 Currently the syntax for defining the diversifying values is defined by a choice of `field` or
diff --git a/docs/reference/aggregations/bucket/filter-aggregation.asciidoc b/docs/reference/aggregations/bucket/filter-aggregation.asciidoc
index be5c6ac58f251..bff4096e2dd3a 100644
--- a/docs/reference/aggregations/bucket/filter-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/filter-aggregation.asciidoc
@@ -7,10 +7,11 @@ Example:
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
     "aggs" : {
-        "red_products" : {
-            "filter" : { "term": { "color": "red" } },
+        "t_shirts" : {
+            "filter" : { "term": { "type": "t-shirt" } },
             "aggs" : {
                 "avg_price" : { "avg" : { "field" : "price" } }
             }
@@ -18,8 +19,10 @@ Example:
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
-In the above example, we calculate the average price of all the products that are red.
+In the above example, we calculate the average price of all the products that are of type t-shirt.
 
 Response:
 
@@ -27,12 +30,12 @@ Response:
 --------------------------------------------------
 {
     ...
-
-    "aggs" : {
-        "red_products" : {
-            "doc_count" : 100,
-            "avg_price" : { "value" : 56.3 }
+    "aggregations" : {
+        "t_shirts" : {
+            "doc_count" : 3,
+            "avg_price" : { "value" : 128.33333333333334 }
         }
     }
 }
 --------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
diff --git a/docs/reference/aggregations/bucket/filters-aggregation.asciidoc b/docs/reference/aggregations/bucket/filters-aggregation.asciidoc
index 5f5d6c54b6af2..97b7f2a50e1cc 100644
--- a/docs/reference/aggregations/bucket/filters-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/filters-aggregation.asciidoc
@@ -9,62 +9,61 @@ Example:
 
 [source,js]
 --------------------------------------------------
+PUT /logs/message/_bulk?refresh
+{ "index" : { "_id" : 1 } }
+{ "body" : "warning: page could not be rendered" }
+{ "index" : { "_id" : 2 } }
+{ "body" : "authentication error" }
+{ "index" : { "_id" : 3 } }
+{ "body" : "warning: connection timed out" }
+
+GET logs/_search
 {
+  "size": 0,
   "aggs" : {
     "messages" : {
       "filters" : {
         "filters" : {
-          "errors" :   { "term" : { "body" : "error"   }},
-          "warnings" : { "term" : { "body" : "warning" }}
-        }
-      },
-      "aggs" : {
-        "monthly" : {
-          "histogram" : {
-            "field" : "timestamp",
-            "interval" : "1M"
-          }
+          "errors" :   { "match" : { "body" : "error"   }},
+          "warnings" : { "match" : { "body" : "warning" }}
         }
       }
     }
   }
 }
 --------------------------------------------------
+// CONSOLE
 
 In the above example, we analyze log messages. The aggregation will build two
 collection (buckets) of log messages - one for all those containing an error,
-and another for all those containing a warning. And for each of these buckets
-it will break them down by month.
+and another for all those containing a warning.
 
 Response:
 
 [source,js]
 --------------------------------------------------
-...
-  "aggs" : {
-    "messages" : {
-      "buckets" : {
-        "errors" : {
-          "doc_count" : 34,
-          "monthly" : {
-            "buckets" : [
-              ... // the histogram monthly breakdown
-            ]
-          }
+{
+  "took": 9,
+  "timed_out": false,
+  "_shards": ...,
+  "hits": ...,
+  "aggregations": {
+    "messages": {
+      "buckets": {
+        "errors": {
+          "doc_count": 1
         },
-        "warnings" : {
-          "doc_count" : 439,
-          "monthly" : {
-            "buckets" : [
-               ... // the histogram monthly breakdown
-            ]
-          }
+        "warnings": {
+          "doc_count": 2
         }
       }
     }
   }
-...
+}
 --------------------------------------------------
+// TESTRESPONSE[s/"took": 9/"took": $body.took/]
+// TESTRESPONSE[s/"_shards": \.\.\./"_shards": $body._shards/]
+// TESTRESPONSE[s/"hits": \.\.\./"hits": $body.hits/]
 
 ==== Anonymous filters
 
@@ -73,58 +72,51 @@ following request:
 
 [source,js]
 --------------------------------------------------
+GET logs/_search
 {
+  "size": 0,
   "aggs" : {
     "messages" : {
       "filters" : {
         "filters" : [
-          { "term" : { "body" : "error"   }},
-          { "term" : { "body" : "warning" }}
+          { "match" : { "body" : "error"   }},
+          { "match" : { "body" : "warning" }}
         ]
-      },
-      "aggs" : {
-        "monthly" : {
-          "histogram" : {
-            "field" : "timestamp",
-            "interval" : "1M"
-          }
-        }
       }
     }
   }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 The filtered buckets are returned in the same order as provided in the
 request.  The response for this example would be:
 
 [source,js]
 --------------------------------------------------
-...
-  "aggs" : {
-    "messages" : {
-      "buckets" : [
+{
+  "took": 4,
+  "timed_out": false,
+  "_shards": ...,
+  "hits": ...,
+  "aggregations": {
+    "messages": {
+      "buckets": [
         {
-          "doc_count" : 34,
-          "monthly" : {
-            "buckets : [
-              ... // the histogram monthly breakdown
-            ]
-          }
+          "doc_count": 1
         },
         {
-          "doc_count" : 439,
-          "monthly" : {
-            "buckets : [
-              ... // the histogram monthly breakdown
-            ]
-          }
+          "doc_count": 2
         }
       ]
     }
   }
-...
+}
 --------------------------------------------------
+// TESTRESPONSE[s/"took": 4/"took": $body.took/]
+// TESTRESPONSE[s/"_shards": \.\.\./"_shards": $body._shards/]
+// TESTRESPONSE[s/"hits": \.\.\./"hits": $body.hits/]
 
 ==== `Other` Bucket
 
@@ -142,64 +134,56 @@ The following snippet shows a response where the `other` bucket is requested to
 
 [source,js]
 --------------------------------------------------
+PUT logs/message/4?refresh
 {
+  "body": "info: user Bob logged out"
+}
+
+GET logs/_search
+{
+  "size": 0,
   "aggs" : {
     "messages" : {
       "filters" : {
         "other_bucket_key": "other_messages",
         "filters" : {
-          "errors" :   { "term" : { "body" : "error"   }},
-          "warnings" : { "term" : { "body" : "warning" }}
-        }
-      },
-      "aggs" : {
-        "monthly" : {
-          "histogram" : {
-            "field" : "timestamp",
-            "interval" : "1M"
-          }
+          "errors" :   { "match" : { "body" : "error"   }},
+          "warnings" : { "match" : { "body" : "warning" }}
         }
       }
     }
   }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 The response would be something like the following:
 
 [source,js]
 --------------------------------------------------
-...
-  "aggs" : {
-    "messages" : {
-      "buckets" : {
-        "errors" : {
-          "doc_count" : 34,
-            "monthly" : {
-              "buckets" : [
-                ... // the histogram monthly breakdown
-              ]
-            }
-          },
-          "warnings" : {
-            "doc_count" : 439,
-            "monthly" : {
-              "buckets" : [
-                 ... // the histogram monthly breakdown
-              ]
-            }
-          },
-          "other_messages" : {
-            "doc_count" : 237,
-            "monthly" : {
-              "buckets" : [
-                 ... // the histogram monthly breakdown
-              ]
-            }
-          }
+{
+  "took": 3,
+  "timed_out": false,
+  "_shards": ...,
+  "hits": ...,
+  "aggregations": {
+    "messages": {
+      "buckets": {
+        "errors": {
+          "doc_count": 1
+        },
+        "warnings": {
+          "doc_count": 2
+        },
+        "other_messages": {
+          "doc_count": 1
         }
       }
     }
   }
-...
+}
 --------------------------------------------------
+// TESTRESPONSE[s/"took": 3/"took": $body.took/]
+// TESTRESPONSE[s/"_shards": \.\.\./"_shards": $body._shards/]
+// TESTRESPONSE[s/"hits": \.\.\./"hits": $body.hits/]
diff --git a/docs/reference/aggregations/bucket/geodistance-aggregation.asciidoc b/docs/reference/aggregations/bucket/geodistance-aggregation.asciidoc
index 4890f9dca9da4..ede1d4c28eb2b 100644
--- a/docs/reference/aggregations/bucket/geodistance-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/geodistance-aggregation.asciidoc
@@ -5,6 +5,34 @@ A multi-bucket aggregation that works on `geo_point` fields and conceptually wor
 
 [source,js]
 --------------------------------------------------
+PUT /museums
+{
+    "mappings": {
+        "doc": {
+            "properties": {
+                "location": {
+                    "type": "geo_point"
+                }
+            }
+        }
+    }
+}
+
+POST /museums/doc/_bulk?refresh
+{"index":{"_id":1}}
+{"location": "52.374081,4.912350", "name": "NEMO Science Museum"}
+{"index":{"_id":2}}
+{"location": "52.369219,4.901618", "name": "Museum Het Rembrandthuis"}
+{"index":{"_id":3}}
+{"location": "52.371667,4.914722", "name": "Nederlands Scheepvaartmuseum"}
+{"index":{"_id":4}}
+{"location": "51.222900,4.405200", "name": "Letterenhuis"}
+{"index":{"_id":5}}
+{"location": "48.861111,2.336389", "name": "Musée du Louvre"}
+{"index":{"_id":6}}
+{"location": "48.860000,2.327000", "name": "Musée d'Orsay"}
+
+POST /museums/_search?size=0
 {
     "aggs" : {
         "rings_around_amsterdam" : {
@@ -12,46 +40,49 @@ A multi-bucket aggregation that works on `geo_point` fields and conceptually wor
                 "field" : "location",
                 "origin" : "52.3760, 4.894",
                 "ranges" : [
-                    { "to" : 100 },
-                    { "from" : 100, "to" : 300 },
-                    { "from" : 300 }
+                    { "to" : 100000 },
+                    { "from" : 100000, "to" : 300000 },
+                    { "from" : 300000 }
                 ]
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 Response:
 
 [source,js]
 --------------------------------------------------
 {
+    ...
     "aggregations": {
-        "rings" : {
+        "rings_around_amsterdam" : {
             "buckets": [
                 {
-                    "key": "*-100.0",
-                    "from": 0,
-                    "to": 100.0,
+                    "key": "*-100000.0",
+                    "from": 0.0,
+                    "to": 100000.0,
                     "doc_count": 3
                 },
                 {
-                    "key": "100.0-300.0",
-                    "from": 100.0,
-                    "to": 300.0,
+                    "key": "100000.0-300000.0",
+                    "from": 100000.0,
+                    "to": 300000.0,
                     "doc_count": 1
                 },
                 {
-                    "key": "300.0-*",
-                    "from": 300.0,
-                    "doc_count": 7
+                    "key": "300000.0-*",
+                    "from": 300000.0,
+                    "doc_count": 2
                 }
             ]
         }
     }
 }
 --------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"_shards": $body._shards,"hits":$body.hits,"timed_out":false,/]
 
 The specified field must be of type `geo_point` (which can only be set explicitly in the mappings). And it can also hold an array of `geo_point` fields, in which case all will be taken into account during aggregation. The origin point can accept all formats supported by the <<geo-point,`geo_point` type>>:
 
@@ -59,17 +90,18 @@ The specified field must be of type `geo_point` (which can only be set explicitl
 * String format: `"52.3760, 4.894"` - where the first number is the `lat` and the second is the `lon`
 * Array format: `[4.894, 52.3760]` - which is based on the `GeoJson` standard and where the first number is the `lon` and the second one is the `lat`
 
-By default, the distance unit is `m` (metres) but it can also accept: `mi` (miles), `in` (inches), `yd` (yards), `km` (kilometers), `cm` (centimeters), `mm` (millimeters).
+By default, the distance unit is `m` (meters) but it can also accept: `mi` (miles), `in` (inches), `yd` (yards), `km` (kilometers), `cm` (centimeters), `mm` (millimeters).
 
 [source,js]
 --------------------------------------------------
+POST /museums/_search?size=0
 {
     "aggs" : {
         "rings" : {
             "geo_distance" : {
                 "field" : "location",
                 "origin" : "52.3760, 4.894",
-                "unit" : "mi", <1>
+                "unit" : "km", <1>
                 "ranges" : [
                     { "to" : 100 },
                     { "from" : 100, "to" : 300 },
@@ -80,19 +112,26 @@ By default, the distance unit is `m` (metres) but it can also accept: `mi` (mile
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
-<1> The distances will be computed as miles
+<1> The distances will be computed in kilometers
 
-There are three distance calculation modes: `sloppy_arc` (the default), `arc` (most accurate) and `plane` (fastest). The `arc` calculation is the most accurate one but also the more expensive one in terms of performance. The `sloppy_arc` is faster but less accurate. The `plane` is the fastest but least accurate distance function. Consider using `plane` when your search context is "narrow" and spans smaller geographical areas (like cities or even countries). `plane` may return higher error mergins for searches across very large areas (e.g. cross continent search). The distance calculation type can be set using the `distance_type` parameter:
+There are two distance calculation modes: `arc` (the default), and `plane`. The `arc` calculation is the most accurate.
+The `plane` is the fastest but least accurate. Consider using `plane` when your search context is "narrow", and spans
+smaller geographical areas (~5km). `plane` will return higher error margins for searches across very large areas
+(e.g. cross continent search). The distance calculation type can be set using the `distance_type` parameter:
 
 [source,js]
 --------------------------------------------------
+POST /museums/_search?size=0
 {
     "aggs" : {
         "rings" : {
             "geo_distance" : {
                 "field" : "location",
                 "origin" : "52.3760, 4.894",
+                "unit" : "km",
                 "distance_type" : "plane",
                 "ranges" : [
                     { "to" : 100 },
@@ -104,3 +143,118 @@ There are three distance calculation modes: `sloppy_arc` (the default), `arc` (m
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+==== Keyed Response
+
+Setting the `keyed` flag to `true` will associate a unique string key with each bucket and return the ranges as a hash rather than an array:
+
+[source,js]
+--------------------------------------------------
+POST /museums/_search?size=0
+{
+    "aggs" : {
+        "rings_around_amsterdam" : {
+            "geo_distance" : {
+                "field" : "location",
+                "origin" : "52.3760, 4.894",
+                "ranges" : [
+                    { "to" : 100000 },
+                    { "from" : 100000, "to" : 300000 },
+                    { "from" : 300000 }
+                ],
+                "keyed": true
+            }
+        }
+    }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+Response:
+
+[source,js]
+--------------------------------------------------
+{
+    ...
+    "aggregations": {
+        "rings_around_amsterdam" : {
+            "buckets": {
+                "*-100000.0": {
+                    "from": 0.0,
+                    "to": 100000.0,
+                    "doc_count": 3
+                },
+                "100000.0-300000.0": {
+                    "from": 100000.0,
+                    "to": 300000.0,
+                    "doc_count": 1
+                },
+                "300000.0-*": {
+                    "from": 300000.0,
+                    "doc_count": 2
+                }
+            }
+        }
+    }
+}
+--------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"_shards": $body._shards,"hits":$body.hits,"timed_out":false,/]
+
+It is also possible to customize the key for each range:
+
+[source,js]
+--------------------------------------------------
+POST /museums/_search?size=0
+{
+    "aggs" : {
+        "rings_around_amsterdam" : {
+            "geo_distance" : {
+                "field" : "location",
+                "origin" : "52.3760, 4.894",
+                "ranges" : [
+                    { "to" : 100000, "key": "first_ring" },
+                    { "from" : 100000, "to" : 300000, "key": "second_ring" },
+                    { "from" : 300000, "key": "third_ring" }
+                ],
+                "keyed": true
+            }
+        }
+    }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+Response:
+
+[source,js]
+--------------------------------------------------
+{
+    ...
+    "aggregations": {
+        "rings_around_amsterdam" : {
+            "buckets": {
+                "first_ring": {
+                    "from": 0.0,
+                    "to": 100000.0,
+                    "doc_count": 3
+                },
+                "second_ring": {
+                    "from": 100000.0,
+                    "to": 300000.0,
+                    "doc_count": 1
+                },
+                "third_ring": {
+                    "from": 300000.0,
+                    "doc_count": 2
+                }
+            }
+        }
+    }
+}
+--------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"_shards": $body._shards,"hits":$body.hits,"timed_out":false,/]
+
diff --git a/docs/reference/aggregations/bucket/geohashgrid-aggregation.asciidoc b/docs/reference/aggregations/bucket/geohashgrid-aggregation.asciidoc
index 9963b48fb045e..84f70185aa97f 100644
--- a/docs/reference/aggregations/bucket/geohashgrid-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/geohashgrid-aggregation.asciidoc
@@ -19,9 +19,37 @@ The specified field must be of type `geo_point` (which can only be set explicitl
 
 [source,js]
 --------------------------------------------------
+PUT /museums
+{
+    "mappings": {
+        "doc": {
+            "properties": {
+                "location": {
+                    "type": "geo_point"
+                }
+            }
+        }
+    }
+}
+
+POST /museums/doc/_bulk?refresh
+{"index":{"_id":1}}
+{"location": "52.374081,4.912350", "name": "NEMO Science Museum"}
+{"index":{"_id":2}}
+{"location": "52.369219,4.901618", "name": "Museum Het Rembrandthuis"}
+{"index":{"_id":3}}
+{"location": "52.371667,4.914722", "name": "Nederlands Scheepvaartmuseum"}
+{"index":{"_id":4}}
+{"location": "51.222900,4.405200", "name": "Letterenhuis"}
+{"index":{"_id":5}}
+{"location": "48.861111,2.336389", "name": "Musée du Louvre"}
+{"index":{"_id":6}}
+{"location": "48.860000,2.327000", "name": "Musée d'Orsay"}
+
+POST /museums/_search?size=0
 {
     "aggregations" : {
-        "myLarge-GrainGeoHashGrid" : {
+        "large-grid" : {
             "geohash_grid" : {
                 "field" : "location",
                 "precision" : 3
@@ -30,30 +58,35 @@ The specified field must be of type `geo_point` (which can only be set explicitl
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 Response:
 
 [source,js]
 --------------------------------------------------
 {
+    ...
     "aggregations": {
-        "myLarge-GrainGeoHashGrid": {
+        "large-grid": {
             "buckets": [
                 {
-                    "key": "svz",
-                    "doc_count": 10964
+                    "key": "u17",
+                    "doc_count": 3
+                },
+                {
+                    "key": "u09",
+                    "doc_count": 2
                 },
                 {
-                    "key": "sv8",
-                    "doc_count": 3198
+                    "key": "u15",
+                    "doc_count": 1
                 }
             ]
         }
     }
 }
 --------------------------------------------------
-
-
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"_shards": $body._shards,"hits":$body.hits,"timed_out":false,/]
 
 ==== High-precision requests
 
@@ -61,29 +94,32 @@ When requesting detailed buckets (typically for displaying a "zoomed in" map) a
 
 [source,js]
 --------------------------------------------------
+POST /museums/_search?size=0
 {
     "aggregations" : {
-        "zoomedInView" : {
+        "zoomed-in" : {
             "filter" : {
                 "geo_bounding_box" : {
                     "location" : {
-                        "top_left" : "51.73, 0.9",
-                        "bottom_right" : "51.55, 1.1"
+                        "top_left" : "52.4, 4.9",
+                        "bottom_right" : "52.3, 5.0"
                     }
                 }
             },
             "aggregations":{
                 "zoom1":{
                     "geohash_grid" : {
-                        "field":"location",
-                        "precision":8,
+                        "field": "location",
+                        "precision": 8
                     }
                 }
             }
         }
     }
- }
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 ==== Cell dimensions at the equator
 The table below shows the metric dimensions for cells covered by various string lengths of geohash.
diff --git a/docs/reference/aggregations/bucket/global-aggregation.asciidoc b/docs/reference/aggregations/bucket/global-aggregation.asciidoc
index 4e5addb46c176..affabc5067e66 100644
--- a/docs/reference/aggregations/bucket/global-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/global-aggregation.asciidoc
@@ -1,18 +1,22 @@
 [[search-aggregations-bucket-global-aggregation]]
 === Global Aggregation
 
-Defines a single bucket of all the documents within the search execution context. This context is defined by the indices and the document types you're searching on, but is *not* influenced by the search query itself.
+Defines a single bucket of all the documents within the search execution
+context. This context is defined by the indices and the document types you're
+searching on, but is *not* influenced by the search query itself.
 
-NOTE:   Global aggregators can only be placed as top level aggregators (it makes no sense to embed a global aggregator
-        within another bucket aggregator)
+NOTE:   Global aggregators can only be placed as top level aggregators because
+        it doesn't make sense to embed a global aggregator within another
+        bucket aggregator.
 
 Example:
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
     "query" : {
-        "match" : { "title" : "shirt" }
+        "match" : { "type" : "t-shirt" }
     },
     "aggs" : {
         "all_products" : {
@@ -20,15 +24,21 @@ Example:
             "aggs" : { <2>
                 "avg_price" : { "avg" : { "field" : "price" } }
             }
-        }
+        },
+        "t_shirts": { "avg" : { "field" : "price" } }
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
 <1> The `global` aggregation has an empty body
 <2> The sub-aggregations that are registered for this `global` aggregation
 
-The above aggregation demonstrates how one would compute aggregations (`avg_price` in this example) on all the documents in the search context, regardless of the query (in our example, it will compute the average price over all products in our catalog, not just on the "shirts").
+The above aggregation demonstrates how one would compute aggregations
+(`avg_price` in this example) on all the documents in the search context,
+regardless of the query (in our example, it will compute the average price over
+all products in our catalog, not just on the "shirts").
 
 The response for the above aggregation:
 
@@ -36,16 +46,22 @@ The response for the above aggregation:
 --------------------------------------------------
 {
     ...
-
     "aggregations" : {
         "all_products" : {
-            "doc_count" : 100, <1>
+            "doc_count" : 7, <1>
             "avg_price" : {
-                "value" : 56.3
+                "value" : 140.71428571428572 <2>
             }
+        },
+        "t_shirts": {
+            "value" : 128.33333333333334 <3>
         }
     }
 }
 --------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
 
-<1> The number of documents that were aggregated (in our case, all documents within the search context)
+<1> The number of documents that were aggregated (in our case, all documents
+within the search context)
+<2> The average price of all products in the index
+<3> The average price of all t-shirts
diff --git a/docs/reference/aggregations/bucket/histogram-aggregation.asciidoc b/docs/reference/aggregations/bucket/histogram-aggregation.asciidoc
index 0336e21c2eaf6..de828c62aa9a7 100644
--- a/docs/reference/aggregations/bucket/histogram-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/histogram-aggregation.asciidoc
@@ -20,6 +20,7 @@ The following snippet "buckets" the products based on their `price` by interval
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
     "aggs" : {
         "prices" : {
@@ -31,29 +32,36 @@ The following snippet "buckets" the products based on their `price` by interval
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
 And the following may be the response:
 
 [source,js]
 --------------------------------------------------
 {
+    ...
     "aggregations": {
         "prices" : {
             "buckets": [
                 {
-                    "key": 0,
-                    "doc_count": 2
+                    "key": 0.0,
+                    "doc_count": 1
                 },
                 {
-                    "key": 50,
-                    "doc_count": 4
+                    "key": 50.0,
+                    "doc_count": 1
                 },
                 {
-                    "key": 100,
+                    "key": 100.0,
                     "doc_count": 0
                 },
                 {
-                    "key": 150,
+                    "key": 150.0,
+                    "doc_count": 2
+                },
+                {
+                    "key": 200.0,
                     "doc_count": 3
                 }
             ]
@@ -61,6 +69,7 @@ And the following may be the response:
     }
 }
 --------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
 
 ==== Minimum document count
 
@@ -70,6 +79,7 @@ a higher minimum count thanks to the `min_doc_count` setting:
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
     "aggs" : {
         "prices" : {
@@ -82,25 +92,32 @@ a higher minimum count thanks to the `min_doc_count` setting:
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
 Response:
 
 [source,js]
 --------------------------------------------------
 {
+    ...
     "aggregations": {
         "prices" : {
             "buckets": [
                 {
-                    "key": 0,
-                    "doc_count": 2
+                    "key": 0.0,
+                    "doc_count": 1
+                },
+                {
+                    "key": 50.0,
+                    "doc_count": 1
                 },
                 {
-                    "key": 50,
-                    "doc_count": 4
+                    "key": 150.0,
+                    "doc_count": 2
                 },
                 {
-                    "key": 150,
+                    "key": 200.0,
                     "doc_count": 3
                 }
             ]
@@ -108,6 +125,7 @@ Response:
     }
 }
 --------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
 
 [[search-aggregations-bucket-histogram-aggregation-extended-bounds]]
 By default the `histogram` returns all the buckets within the range of the data itself, that is, the documents with
@@ -137,6 +155,7 @@ Example:
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
     "query" : {
         "constant_score" : { "filter": { "range" : { "price" : { "to" : "500" } } } }
@@ -155,6 +174,8 @@ Example:
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
 ==== Order
 
@@ -165,6 +186,7 @@ Ordering the buckets by their key - descending:
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
     "aggs" : {
         "prices" : {
@@ -177,11 +199,14 @@ Ordering the buckets by their key - descending:
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
 Ordering the buckets by their `doc_count` - ascending:
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
     "aggs" : {
         "prices" : {
@@ -194,11 +219,14 @@ Ordering the buckets by their `doc_count` - ascending:
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
 If the histogram aggregation has a direct metrics sub-aggregation, the latter can determine the order of the buckets:
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
     "aggs" : {
         "prices" : {
@@ -208,17 +236,17 @@ If the histogram aggregation has a direct metrics sub-aggregation, the latter ca
                 "order" : { "price_stats.min" : "asc" } <1>
             },
             "aggs" : {
-                "price_stats" : { "stats" : {} } <2>
+                "price_stats" : { "stats" : {"field" : "price"} }
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
 <1> The `{ "price_stats.min" : asc" }` will sort the buckets based on `min` value of their `price_stats` sub-aggregation.
 
-<2> There is no need to configure the `price` field for the `price_stats` aggregation as it will inherit it by default from its parent histogram aggregation.
-
 It is also possible to order the buckets based on a "deeper" aggregation in the hierarchy. This is supported as long
 as the aggregations path are of a single-bucket type, where the last aggregation in the path may either by a single-bucket
 one or a metrics one. If it's a single-bucket type, the order will be defined by the number of docs in the bucket (i.e. `doc_count`),
@@ -239,6 +267,7 @@ PATH                =  <AGG_NAME> [ <AGG_SEPARATOR>, <AGG_NAME> ]* [ <METRIC_SEP
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
     "aggs" : {
         "prices" : {
@@ -259,6 +288,8 @@ PATH                =  <AGG_NAME> [ <AGG_SEPARATOR>, <AGG_NAME> ]* [ <METRIC_SEP
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
 The above will sort the buckets based on the avg rating among the promoted products
 
@@ -279,6 +310,7 @@ instead keyed by the buckets keys:
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
     "aggs" : {
         "prices" : {
@@ -291,25 +323,36 @@ instead keyed by the buckets keys:
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
 Response:
 
 [source,js]
 --------------------------------------------------
 {
+    ...
     "aggregations": {
         "prices": {
             "buckets": {
-                "0": {
-                    "key": 0,
-                    "doc_count": 2
+                "0.0": {
+                    "key": 0.0,
+                    "doc_count": 1
+                },
+                "50.0": {
+                    "key": 50.0,
+                    "doc_count": 1
                 },
-                "50": {
-                    "key": 50,
-                    "doc_count": 4
+                "100.0": {
+                    "key": 100.0,
+                    "doc_count": 0
+                },
+                "150.0": {
+                    "key": 150.0,
+                    "doc_count": 2
                 },
-                "150": {
-                    "key": 150,
+                "200.0": {
+                    "key": 200.0,
                     "doc_count": 3
                 }
             }
@@ -317,6 +360,7 @@ Response:
     }
 }
 --------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
 
 ==== Missing value
 
@@ -326,6 +370,7 @@ had a value.
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
     "aggs" : {
         "quantity" : {
@@ -338,5 +383,7 @@ had a value.
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
 <1> Documents without a value in the `quantity` field will fall into the same bucket as documents that have the value `0`.
diff --git a/docs/reference/aggregations/bucket/iprange-aggregation.asciidoc b/docs/reference/aggregations/bucket/iprange-aggregation.asciidoc
index bb20b18663e03..ee77d57a0b476 100644
--- a/docs/reference/aggregations/bucket/iprange-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/iprange-aggregation.asciidoc
@@ -90,3 +90,93 @@ Response:
     }
 }
 --------------------------------------------------
+
+==== Keyed Response
+
+Setting the `keyed` flag to `true` will associate a unique string key with each bucket and return the ranges as a hash rather than an array:
+
+[source,js]
+--------------------------------------------------
+{
+    "aggs": {
+        "ip_ranges": {
+            "ip_range": {
+                "field": "remote_ip",
+                "ranges": [
+                    { "to" : "10.0.0.5" },
+                    { "from" : "10.0.0.5" }
+                ],
+                "keyed": true
+            }
+        }
+    }
+}
+--------------------------------------------------
+
+Response:
+
+[source,js]
+--------------------------------------------------
+{
+    ...
+
+    "aggregations": {
+        "ip_ranges": {
+            "buckets": {
+                "*-10.0.0.5": {
+                    "to": "10.0.0.5",
+                    "doc_count": 1462
+                },
+                "10.0.0.5-*": {
+                    "from": "10.0.0.5",
+                    "doc_count": 50000
+                }
+            }
+        }
+    }
+}
+--------------------------------------------------
+
+It is also possible to customize the key for each range:
+
+[source,js]
+--------------------------------------------------
+{
+    "aggs": {
+        "ip_ranges": {
+            "ip_range": {
+                "field": "remote_ip",
+                "ranges": [
+                    { "key": "infinity", "to" : "10.0.0.5" },
+                    { "key": "and-beyond", "from" : "10.0.0.5" }
+                ],
+                "keyed": true
+            }
+        }
+    }
+}
+--------------------------------------------------
+
+Response:
+
+[source,js]
+--------------------------------------------------
+{
+    ...
+
+    "aggregations": {
+        "ip_ranges": {
+            "buckets": {
+                "infinity": {
+                    "to": "10.0.0.5",
+                    "doc_count": 1462
+                },
+                "and-beyond": {
+                    "from": "10.0.0.5",
+                    "doc_count": 50000
+                }
+            }
+        }
+    }
+}
+--------------------------------------------------
diff --git a/docs/reference/aggregations/bucket/missing-aggregation.asciidoc b/docs/reference/aggregations/bucket/missing-aggregation.asciidoc
index f0b8fb4ac347f..b670529119673 100644
--- a/docs/reference/aggregations/bucket/missing-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/missing-aggregation.asciidoc
@@ -7,6 +7,7 @@ Example:
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
     "aggs" : {
         "products_without_a_price" : {
@@ -15,6 +16,8 @@ Example:
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
 In the above example, we get the total number of products that do not have a price.
 
@@ -24,11 +27,11 @@ Response:
 --------------------------------------------------
 {
     ...
-
-    "aggs" : {
+    "aggregations" : {
         "products_without_a_price" : {
-            "doc_count" : 10
+            "doc_count" : 00
         }
     }
 }
 --------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
\ No newline at end of file
diff --git a/docs/reference/aggregations/bucket/nested-aggregation.asciidoc b/docs/reference/aggregations/bucket/nested-aggregation.asciidoc
index 89142df13e786..3b4510919bb5b 100644
--- a/docs/reference/aggregations/bucket/nested-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/nested-aggregation.asciidoc
@@ -3,7 +3,7 @@
 
 A special single bucket aggregation that enables aggregating nested documents.
 
-For example, lets say we have a index of products, and each product holds the list of resellers - each having its own
+For example, lets say we have an index of products, and each product holds the list of resellers - each having its own
 price for the product. The mapping could look like:
 
 [source,js]
diff --git a/docs/reference/aggregations/bucket/range-aggregation.asciidoc b/docs/reference/aggregations/bucket/range-aggregation.asciidoc
index efcbd9715e1ac..e18edbe75e524 100644
--- a/docs/reference/aggregations/bucket/range-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/range-aggregation.asciidoc
@@ -136,7 +136,7 @@ It is also possible to customize the key for each range:
             "range" : {
                 "script" : {
                     "lang": "painless",
-                    "inline": "doc['price'].value"
+                    "source": "doc['price'].value"
                 },
                 "ranges" : [
                     { "to" : 50 },
@@ -189,7 +189,7 @@ Lets say the product prices are in USD but we would like to get the price ranges
                 "field" : "price",
                 "script" : {
                     "lang": "painless",
-                    "inline": "_value * params.conversion_rate",
+                    "source": "_value * params.conversion_rate",
                     "params" : {
                         "conversion_rate" : 0.8
                     }
diff --git a/docs/reference/aggregations/bucket/reverse-nested-aggregation.asciidoc b/docs/reference/aggregations/bucket/reverse-nested-aggregation.asciidoc
index 9dba1f2adf000..b6074298e1c03 100644
--- a/docs/reference/aggregations/bucket/reverse-nested-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/reverse-nested-aggregation.asciidoc
@@ -22,9 +22,9 @@ the issue documents as nested documents. The mapping could look like:
 
     "issue" : {
         "properties" : {
-            "tags" : { "type" : "text" }
+            "tags" : { "type" : "text" },
             "comments" : { <1>
-                "type" : "nested"
+                "type" : "nested",
                 "properties" : {
                     "username" : { "type" : "keyword" },
                     "comment" : { "type" : "text" }
diff --git a/docs/reference/aggregations/bucket/sampler-aggregation.asciidoc b/docs/reference/aggregations/bucket/sampler-aggregation.asciidoc
index e1f3e708aa6c7..4957901920e23 100644
--- a/docs/reference/aggregations/bucket/sampler-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/sampler-aggregation.asciidoc
@@ -12,12 +12,18 @@ A filtering aggregation used to limit any sub aggregations' processing to a samp
 
 Example:
 
+A query on StackOverflow data for the popular term `javascript` OR the rarer term
+`kibana` will match many documents - most of them missing the word Kibana. To focus
+the `significant_terms` aggregation on top-scoring documents that are more likely to match 
+the most interesting parts of our query we use a sample.
+
 [source,js]
 --------------------------------------------------
+POST /stackoverflow/_search?size=0
 {
     "query": {
-        "match": {
-            "text": "iphone"
+        "query_string": {
+            "query": "tags:kibana OR tags:javascript"
         }
     },
     "aggs": {
@@ -28,7 +34,8 @@ Example:
             "aggs": {
                 "keywords": {
                     "significant_terms": {
-                        "field": "text"
+                        "field": "tags",
+                        "exclude": ["kibana", "javascript"]
                     }
                 }
             }
@@ -36,6 +43,8 @@ Example:
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:stackoverflow]
 
 Response:
 
@@ -43,24 +52,102 @@ Response:
 --------------------------------------------------
 {
     ...
-        "aggregations": {
+    "aggregations": {
         "sample": {
-            "doc_count": 1000,<1>
+            "doc_count": 200,<1>
             "keywords": {
-                "doc_count": 1000,
+                "doc_count": 200,
+                "bg_count": 650,
                 "buckets": [
-                    ...
                     {
-                        "key": "bend",
-                        "doc_count": 58,
-                        "score": 37.982536582524276,
-                        "bg_count": 103
+                        "key": "elasticsearch",
+                        "doc_count": 150,
+                        "score": 1.078125,
+                        "bg_count": 200
                     },
-                    ....
+                    {
+                        "key": "logstash",
+                        "doc_count": 50,
+                        "score": 0.5625,
+                        "bg_count": 50
+                    }
+                ]
+            }
+        }
+    }
+}
+--------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
+
+<1> 200 documents were sampled in total. The cost of performing the nested significant_terms aggregation was
+therefore limited rather than unbounded.
+
+
+Without the `sampler` aggregation the request query considers the full "long tail" of low-quality matches and therefore identifies
+less significant terms such as `jquery` and `angular` rather than focusing on the more insightful Kibana-related terms.
+
+
+[source,js]
+--------------------------------------------------
+POST /stackoverflow/_search?size=0
+{
+    "query": {
+        "query_string": {
+            "query": "tags:kibana OR tags:javascript"
+        }
+    },
+    "aggs": {
+             "low_quality_keywords": {
+                "significant_terms": {
+                    "field": "tags",
+                    "size": 3,
+                    "exclude":["kibana", "javascript"]
+                }
+        }
+    }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:stackoverflow]
+
+Response:
+
+[source,js]
+--------------------------------------------------
+{
+    ...
+    "aggregations": {
+        "low_quality_keywords": {
+            "doc_count": 600,
+            "bg_count": 650,
+            "buckets": [
+                {
+                    "key": "angular",
+                    "doc_count": 200,
+                    "score": 0.02777,
+                    "bg_count": 200
+                },
+                {
+                    "key": "jquery",
+                    "doc_count": 200,
+                    "score": 0.02777,
+                    "bg_count": 200
+                },
+                {
+                    "key": "logstash",
+                    "doc_count": 50,
+                    "score": 0.0069,
+                    "bg_count": 50
+                }
+            ]
+        }
+    }
 }
 --------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
+// TESTRESPONSE[s/0.02777/$body.aggregations.low_quality_keywords.buckets.0.score/]
+// TESTRESPONSE[s/0.0069/$body.aggregations.low_quality_keywords.buckets.2.score/]
 
-<1> 1000 documents were sampled in total because we asked for a maximum of 200 from an index with 5 shards. The cost of performing the nested significant_terms aggregation was therefore limited rather than unbounded.
 
 
 ==== shard_size
diff --git a/docs/reference/aggregations/bucket/significantterms-aggregation.asciidoc b/docs/reference/aggregations/bucket/significantterms-aggregation.asciidoc
index 61deffeccd229..ba8bce6736995 100644
--- a/docs/reference/aggregations/bucket/significantterms-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/significantterms-aggregation.asciidoc
@@ -48,6 +48,7 @@ Response:
     "aggregations" : {
         "significantCrimeTypes" : {
             "doc_count": 47347,
+            "bg_count": 5064554,
             "buckets" : [
                 {
                     "key": "Bicycle theft",
@@ -111,6 +112,7 @@ Response:
                 "doc_count": 894038,
                 "significantCrimeTypes": {
                     "doc_count": 894038,
+                    "bg_count": 5064554,
                     "buckets": [
                         {
                             "key": "Robbery",
@@ -127,6 +129,7 @@ Response:
                 "doc_count": 47347,
                 "significantCrimeTypes": {
                     "doc_count": 47347,
+                    "bg_count": 5064554,
                     "buckets": [
                         {
                             "key": "Bicycle theft",
@@ -353,7 +356,10 @@ Customized scores can be implemented via a script:
 --------------------------------------------------
 
 	    "script_heuristic": {
-              "script": "_subset_freq/(_superset_freq - _subset_freq + 1)"
+              "script": {
+	        "lang": "painless",
+	        "source": "params._subset_freq/(params._superset_freq - params._subset_freq + 1)"
+	      }
             }
 --------------------------------------------------
 
diff --git a/docs/reference/aggregations/bucket/terms-aggregation.asciidoc b/docs/reference/aggregations/bucket/terms-aggregation.asciidoc
index 68b2e8511f92b..d22a748db8f58 100644
--- a/docs/reference/aggregations/bucket/terms-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/terms-aggregation.asciidoc
@@ -297,10 +297,10 @@ Ordering the buckets by single value metrics sub-aggregation (identified by the
         "genres" : {
             "terms" : {
                 "field" : "genre",
-                "order" : { "avg_play_count" : "desc" }
+                "order" : { "max_play_count" : "desc" }
             },
             "aggs" : {
-                "avg_play_count" : { "avg" : { "field" : "play_count" } }
+                "max_play_count" : { "max" : { "field" : "play_count" } }
             }
         }
     }
@@ -316,7 +316,7 @@ Ordering the buckets by multi value metrics sub-aggregation (identified by the a
         "genres" : {
             "terms" : {
                 "field" : "genre",
-                "order" : { "playback_stats.avg" : "desc" }
+                "order" : { "playback_stats.max" : "desc" }
             },
             "aggs" : {
                 "playback_stats" : { "stats" : { "field" : "play_count" } }
@@ -459,7 +459,7 @@ Generating the terms using a script:
         "genres" : {
             "terms" : {
                 "script" : {
-                    "inline": "doc['genre'].value",
+                    "source": "doc['genre'].value",
                     "lang": "painless"
                 }
             }
@@ -499,9 +499,9 @@ TIP: for indexed scripts replace the `file` parameter with an `id` parameter.
     "aggs" : {
         "genres" : {
             "terms" : {
-                "field" : "gendre",
+                "field" : "gender",
                 "script" : {
-                    "inline" : "'Genre: ' +_value"
+                    "source" : "'Genre: ' +_value"
                     "lang" : "painless"
                 }
             }
@@ -514,7 +514,10 @@ TIP: for indexed scripts replace the `file` parameter with an `id` parameter.
 ==== Filtering Values
 
 It is possible to filter the values for which buckets will be created. This can be done using the `include` and
-`exclude` parameters which are based on regular expression strings or arrays of exact values.
+`exclude` parameters which are based on regular expression strings or arrays of exact values. Additionally,
+`include` clauses can filter using `partition` expressions.
+
+===== Filtering Values with regular expressions
 
 [source,js]
 --------------------------------------------------
@@ -538,6 +541,8 @@ both are defined, the `exclude` has precedence, meaning, the `include` is evalua
 
 The syntax is the same as <<regexp-syntax,regexp queries>>.
 
+===== Filtering Values with exact values
+
 For matching based on exact values the `include` and `exclude` parameters can simply take an array of
 strings that represent the terms as they are found in the index:
 
@@ -561,6 +566,67 @@ strings that represent the terms as they are found in the index:
 }
 --------------------------------------------------
 
+===== Filtering Values with partitions
+
+Sometimes there are too many unique terms to process in a single request/response pair so 
+it can be useful to break the analysis up into multiple requests.
+This can be achieved by grouping the field's values into a number of partitions at query-time and processing
+only one partition in each request.
+Consider this request which is looking for accounts that have not logged any access recently:
+
+[source,js]
+--------------------------------------------------
+{
+   "size": 0,
+   "aggs": {
+      "expired_sessions": {
+         "terms": {
+            "field": "account_id",
+            "include": {
+               "partition": 0,
+               "num_partitions": 20
+            },
+            "size": 10000,
+            "order": {
+               "last_access": "asc"
+            }
+         },
+         "aggs": {
+            "last_access": {
+               "max": {
+                  "field": "access_date"
+               }
+            }
+         }
+      }
+   }
+}
+--------------------------------------------------
+
+This request is finding the last logged access date for a subset of customer accounts because we
+might want to expire some customer accounts who haven't been seen for a long while.
+The `num_partitions` setting has requested that the unique account_ids are organized evenly into twenty
+partitions (0 to 19). and the `partition` setting in this request filters to only consider account_ids falling 
+into partition 0. Subsequent requests should ask for partitions 1 then 2 etc to complete the expired-account analysis.
+
+Note that the `size` setting for the number of results returned needs to be tuned with the `num_partitions`. 
+For this particular account-expiration example the process for balancing values for `size` and `num_partitions` would be as follows:
+
+1. Use the `cardinality` aggregation to estimate the total number of unique account_id values
+2. Pick a value for `num_partitions` to break the number from 1) up into more manageable chunks
+3. Pick a `size` value for the number of responses we want from each partition
+4. Run a test request
+
+If we have a circuit-breaker error we are trying to do too much in one request and must increase `num_partitions`.
+If the request was successful but the last account ID in the date-sorted test response was still an account we might want to 
+expire then we may be missing accounts of interest and have set our numbers too low. We must either 
+
+* increase the `size` parameter to return more results per partition (could be heavy on memory) or
+* increase the `num_partitions` to consider less accounts per request (could increase overall processing time as we need to make more requests)
+
+Ultimately this is a balancing act between managing the elasticsearch resources required to process a single request and the volume
+of requests that the client application must issue to complete a task.
+
 ==== Multi-field terms aggregation
 
 The `terms` aggregation does not support collecting terms from multiple fields
diff --git a/docs/reference/aggregations/matrix/stats-aggregation.asciidoc b/docs/reference/aggregations/matrix/stats-aggregation.asciidoc
index 8dafb252f08bd..bb66115ecd571 100644
--- a/docs/reference/aggregations/matrix/stats-aggregation.asciidoc
+++ b/docs/reference/aggregations/matrix/stats-aggregation.asciidoc
@@ -37,6 +37,7 @@ the statistics. The above request returns the following response:
     ...
     "aggregations": {
         "matrixstats": {
+            "doc_count": 50,
             "fields": [{
                 "name": "income",
                 "count": 50,
@@ -73,6 +74,8 @@ the statistics. The above request returns the following response:
 }
 --------------------------------------------------
 
+The `doc_count` field indicates the number of documents involved in the computation of the statistics.
+
 ==== Multi Value Fields
 
 The `matrix_stats` aggregation treats each document field as an independent sample. The `mode` parameter controls what
diff --git a/docs/reference/aggregations/metrics/avg-aggregation.asciidoc b/docs/reference/aggregations/metrics/avg-aggregation.asciidoc
index 9967f4c9d9ea1..66aba126f8a39 100644
--- a/docs/reference/aggregations/metrics/avg-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/avg-aggregation.asciidoc
@@ -3,32 +3,35 @@
 
 A `single-value` metrics aggregation that computes the average of numeric values that are extracted from the aggregated documents. These values can be extracted either from specific numeric fields in the documents, or be generated by a provided script.
 
-Assuming the data consists of documents representing exams grades (between 0 and 100) of students
+Assuming the data consists of documents representing exams grades (between 0
+and 100) of students we can average their scores with:
 
 [source,js]
 --------------------------------------------------
+POST /exams/_search?size=0
 {
     "aggs" : {
         "avg_grade" : { "avg" : { "field" : "grade" } }
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:exams]
 
 The above aggregation computes the average grade over all documents. The aggregation type is `avg` and the `field` setting defines the numeric field of the documents the average will be computed on. The above will return the following:
 
-
 [source,js]
 --------------------------------------------------
 {
     ...
-
     "aggregations": {
         "avg_grade": {
-            "value": 75
+            "value": 75.0
         }
     }
 }
 --------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
 
 The name of the aggregation (`avg_grade` above) also serves as the key by which the aggregation result can be retrieved from the returned response.
 
@@ -38,32 +41,31 @@ Computing the average grade based on a script:
 
 [source,js]
 --------------------------------------------------
+POST /exams/_search?size=0
 {
-    ...,
-
     "aggs" : {
-        "avg_grade" : { 
-            "avg" : { 
+        "avg_grade" : {
+            "avg" : {
                 "script" : {
-                    "inline" : "doc['grade'].value",
-                    "lang" : "painless"
+                    "source" : "doc.grade.value"
                 }
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:exams]
 
 This will interpret the `script` parameter as an `inline` script with the `painless` script language and no script parameters. To use a file script use the following syntax:
 
 [source,js]
 --------------------------------------------------
+POST /exams/_search?size=0
 {
-    ...,
-
     "aggs" : {
-        "avg_grade" : { 
-            "avg" : { 
+        "avg_grade" : {
+            "avg" : {
                 "script" : {
                     "file": "my_script",
                     "params": {
@@ -75,6 +77,8 @@ This will interpret the `script` parameter as an `inline` script with the `painl
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:exams]
 
 TIP: for indexed scripts replace the `file` parameter with an `id` parameter.
 
@@ -84,20 +88,17 @@ It turned out that the exam was way above the level of the students and a grade
 
 [source,js]
 --------------------------------------------------
+POST /exams/_search?size=0
 {
     "aggs" : {
-        ...
-
-        "aggs" : {
-            "avg_corrected_grade" : {
-                "avg" : {
-                    "field" : "grade",
-                    "script" : {
-                        "lang": "painless",
-                        "inline": "_value * params.correction",
-                        "params" : {
-                            "correction" : 1.2
-                        }
+        "avg_corrected_grade" : {
+            "avg" : {
+                "field" : "grade",
+                "script" : {
+                    "lang": "painless",
+                    "source": "_value * params.correction",
+                    "params" : {
+                        "correction" : 1.2
                     }
                 }
             }
@@ -105,6 +106,8 @@ It turned out that the exam was way above the level of the students and a grade
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:exams]
 
 ==== Missing value
 
@@ -114,6 +117,7 @@ had a value.
 
 [source,js]
 --------------------------------------------------
+POST /exams/_search?size=0
 {
     "aggs" : {
         "grade_avg" : {
@@ -125,5 +129,7 @@ had a value.
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:exams]
 
 <1> Documents without a value in the `grade` field will fall into the same bucket as documents that have the value `10`.
diff --git a/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc b/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc
index 77fc7dfcd5a01..c820d175d6f3f 100644
--- a/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc
@@ -10,16 +10,34 @@ match a query:
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
     "aggs" : {
-        "author_count" : {
+        "type_count" : {
             "cardinality" : {
-                "field" : "author"
+                "field" : "type"
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
+
+Response:
+
+[source,js]
+--------------------------------------------------
+{
+    ...
+    "aggregations" : {
+        "type_count" : {
+            "value" : 3
+        }
+    }
+}
+--------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
 
 ==== Precision control
 
@@ -29,17 +47,20 @@ experimental[The `precision_threshold` option is specific to the current interna
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
     "aggs" : {
-        "author_count" : {
+        "type_count" : {
             "cardinality" : {
-                "field" : "author_hash",
+                "field" : "type",
                 "precision_threshold": 100 <1>
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
 <1> The `precision_threshold` options allows to trade memory for accuracy, and
 defines a unique count below which counts are expected to be close to
@@ -159,33 +180,37 @@ however since hashes need to be computed on the fly.
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
     "aggs" : {
-        "author_count" : {
+        "type_promoted_count" : {
             "cardinality" : {
                 "script": {
                     "lang": "painless",
-                    "inline": "doc['author.first_name'].value + ' ' + doc['author.last_name'].value"
+                    "source": "doc['type'].value + ' ' + doc['promoted'].value"
                 }
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
 This will interpret the `script` parameter as an `inline` script with the `painless` script language and no script parameters. To use a file script use the following syntax:
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
     "aggs" : {
-        "author_count" : {
+        "type_promoted_count" : {
             "cardinality" : {
                 "script" : {
                     "file": "my_script",
                     "params": {
-                        "first_name_field": "author.first_name",
-                        "last_name_field": "author.last_name"
+                        "type_field": "type",
+                        "promoted_field": "promoted"
                     }
                 }
             }
@@ -193,6 +218,8 @@ This will interpret the `script` parameter as an `inline` script with the `painl
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[skip:no script]
 
 TIP: for indexed scripts replace the `file` parameter with an `id` parameter.
 
@@ -204,6 +231,7 @@ had a value.
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
     "aggs" : {
         "tag_cardinality" : {
@@ -215,5 +243,6 @@ had a value.
     }
 }
 --------------------------------------------------
-
+// CONSOLE
+// TEST[setup:sales]
 <1> Documents without a value in the `tag` field will fall into the same bucket as documents that have the value `N/A`.
diff --git a/docs/reference/aggregations/metrics/extendedstats-aggregation.asciidoc b/docs/reference/aggregations/metrics/extendedstats-aggregation.asciidoc
index 0e324089dc799..8a7676e9f6346 100644
--- a/docs/reference/aggregations/metrics/extendedstats-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/extendedstats-aggregation.asciidoc
@@ -89,7 +89,7 @@ Computing the grades stats based on a script:
         "grades_stats" : { 
             "extended_stats" : { 
                 "script" : { 
-                    "inline" : "doc['grade'].value",
+                    "source" : "doc['grade'].value",
                     "lang" : "painless"
                  }
              }
@@ -138,7 +138,7 @@ It turned out that the exam was way above the level of the students and a grade
                     "field" : "grade",
                     "script" : {
                         "lang" : "painless",
-                        "inline": "_value * params.correction",
+                        "source": "_value * params.correction",
                         "params" : {
                             "correction" : 1.2
                         }
diff --git a/docs/reference/aggregations/metrics/geobounds-aggregation.asciidoc b/docs/reference/aggregations/metrics/geobounds-aggregation.asciidoc
index ade59477ee372..4d78e0c30821f 100644
--- a/docs/reference/aggregations/metrics/geobounds-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/geobounds-aggregation.asciidoc
@@ -8,9 +8,37 @@ Example:
 
 [source,js]
 --------------------------------------------------
+PUT /museums
+{
+    "mappings": {
+        "doc": {
+            "properties": {
+                "location": {
+                    "type": "geo_point"
+                }
+            }
+        }
+    }
+}
+
+POST /museums/doc/_bulk?refresh
+{"index":{"_id":1}}
+{"location": "52.374081,4.912350", "name": "NEMO Science Museum"}
+{"index":{"_id":2}}
+{"location": "52.369219,4.901618", "name": "Museum Het Rembrandthuis"}
+{"index":{"_id":3}}
+{"location": "52.371667,4.914722", "name": "Nederlands Scheepvaartmuseum"}
+{"index":{"_id":4}}
+{"location": "51.222900,4.405200", "name": "Letterenhuis"}
+{"index":{"_id":5}}
+{"location": "48.861111,2.336389", "name": "Musée du Louvre"}
+{"index":{"_id":6}}
+{"location": "48.860000,2.327000", "name": "Musée d'Orsay"}
+
+POST /museums/_search?size=0
 {
     "query" : {
-        "match" : { "business_type" : "shop" }
+        "match" : { "name" : "musée" }
     },
     "aggs" : {
         "viewport" : {
@@ -22,6 +50,7 @@ Example:
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 <1> The `geo_bounds` aggregation specifies the field to use to obtain the bounds
 <2> `wrap_longitude` is an optional parameter which specifies whether the bounding box should be allowed to overlap the international date line. The default value is `true`
@@ -34,20 +63,20 @@ The response for the above aggregation:
 --------------------------------------------------
 {
     ...
-
     "aggregations": {
         "viewport": {
             "bounds": {
                 "top_left": {
-                    "lat": 80.45,
-                    "lon": -160.22
+                    "lat": 48.86111099738628,
+                    "lon": 2.3269999679178
                 },
                 "bottom_right": {
-                    "lat": 40.65,
-                    "lon": 42.57
+                    "lat": 48.85999997612089,
+                    "lon": 2.3363889567553997
                 }
             }
         }
     }
 }
 --------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"_shards": $body._shards,"hits":$body.hits,"timed_out":false,/]
diff --git a/docs/reference/aggregations/metrics/geocentroid-aggregation.asciidoc b/docs/reference/aggregations/metrics/geocentroid-aggregation.asciidoc
index 8f871dc8dbcbc..42358587d41d0 100644
--- a/docs/reference/aggregations/metrics/geocentroid-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/geocentroid-aggregation.asciidoc
@@ -1,17 +1,41 @@
 [[search-aggregations-metrics-geocentroid-aggregation]]
 === Geo Centroid Aggregation
 
-A metric aggregation that computes the weighted centroid from all coordinate values for a <<geo-point>> field.
-
+A metric aggregation that computes the weighted https://en.wikipedia.org/wiki/Centroid[centroid] from all coordinate values for a <<geo-point>> field.
 
 Example:
 
 [source,js]
 --------------------------------------------------
+PUT /museums
+{
+    "mappings": {
+        "doc": {
+            "properties": {
+                "location": {
+                    "type": "geo_point"
+                }
+            }
+        }
+    }
+}
+
+POST /museums/doc/_bulk?refresh
+{"index":{"_id":1}}
+{"location": "52.374081,4.912350", "city": "Amsterdam", "name": "NEMO Science Museum"}
+{"index":{"_id":2}}
+{"location": "52.369219,4.901618", "city": "Amsterdam", "name": "Museum Het Rembrandthuis"}
+{"index":{"_id":3}}
+{"location": "52.371667,4.914722", "city": "Amsterdam", "name": "Nederlands Scheepvaartmuseum"}
+{"index":{"_id":4}}
+{"location": "51.222900,4.405200", "city": "Antwerp", "name": "Letterenhuis"}
+{"index":{"_id":5}}
+{"location": "48.861111,2.336389", "city": "Paris", "name": "Musée du Louvre"}
+{"index":{"_id":6}}
+{"location": "48.860000,2.327000", "city": "Paris", "name": "Musée d'Orsay"}
+
+POST /museums/_search?size=0
 {
-    "query" : {
-        "match" : { "crime" : "burglary" }
-    },
     "aggs" : {
         "centroid" : {
             "geo_centroid" : {
@@ -21,6 +45,7 @@ Example:
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 <1> The `geo_centroid` aggregation specifies the field to use for computing the centroid. (NOTE: field must be a <<geo-point>> type)
 
@@ -32,18 +57,18 @@ The response for the above aggregation:
 --------------------------------------------------
 {
     ...
-
     "aggregations": {
         "centroid": {
             "location": {
-                "lat": 80.45,
-                "lon": -160.22
-            }
+                "lat": 51.009829603135586,
+                "lon": 3.966213036328554
+            },
+            "count": 6
         }
     }
 }
 --------------------------------------------------
-
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"_shards": $body._shards,"hits":$body.hits,"timed_out":false,/]
 
 The `geo_centroid` aggregation is more interesting when combined as a sub-aggregation to other bucket aggregations.
 
@@ -51,13 +76,11 @@ Example:
 
 [source,js]
 --------------------------------------------------
+POST /museums/_search?size=0
 {
-    "query" : {
-        "match" : { "crime" : "burglary" }
-    },
     "aggs" : {
-        "towns" : {
-            "terms" : { "field" : "town" },
+        "cities" : {
+            "terms" : { "field" : "city.keyword" },
             "aggs" : {
                 "centroid" : {
                     "geo_centroid" : { "field" : "location" }
@@ -67,9 +90,12 @@ Example:
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
-The above example uses `geo_centroid` as a sub-aggregation to a <<search-aggregations-bucket-terms-aggregation, terms>> bucket aggregation
-for finding the central location for all crimes of type burglary in each town.
+The above example uses `geo_centroid` as a sub-aggregation to a
+<<search-aggregations-bucket-terms-aggregation, terms>> bucket aggregation
+for finding the central location for museums in each city.
 
 The response for the above aggregation:
 
@@ -77,28 +103,47 @@ The response for the above aggregation:
 --------------------------------------------------
 {
     ...
-
-    "buckets": [
-       {
-           "key": "Los Altos",
-           "doc_count": 113,
-           "centroid": {
-              "location": {
-                 "lat": 37.3924582824111,
-                 "lon": -122.12104808539152
-              }
-           }
-       },
-       {
-           "key": "Mountain View",
-           "doc_count": 92,
-           "centroid": {
-              "location": {
-                 "lat": 37.382152481004596,
-                 "lon": -122.08116559311748
-              }
-           }
+    "aggregations": {
+        "cities": {
+            "sum_other_doc_count": 0,
+            "doc_count_error_upper_bound": 0,
+            "buckets": [
+               {
+                   "key": "Amsterdam",
+                   "doc_count": 3,
+                   "centroid": {
+                      "location": {
+                         "lat": 52.371655656024814,
+                         "lon": 4.909563269466162
+                      },
+                      "count": 3
+                   }
+               },
+               {
+                   "key": "Paris",
+                   "doc_count": 2,
+                   "centroid": {
+                      "location": {
+                         "lat": 48.86055544484407,
+                         "lon": 2.331694420427084
+                      },
+                      "count": 2
+                   }
+                },
+                {
+                    "key": "Antwerp",
+                    "doc_count": 1,
+                    "centroid": {
+                       "location": {
+                          "lat": 51.222899928689,
+                          "lon": 4.405199903994799
+                       },
+                       "count": 1
+                    }
+                 }
+            ]
         }
-    ]
+    }
 }
---------------------------------------------------
\ No newline at end of file
+--------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"_shards": $body._shards,"hits":$body.hits,"timed_out":false,/]
diff --git a/docs/reference/aggregations/metrics/max-aggregation.asciidoc b/docs/reference/aggregations/metrics/max-aggregation.asciidoc
index 8cfc0bd998ee6..357384a4145bc 100644
--- a/docs/reference/aggregations/metrics/max-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/max-aggregation.asciidoc
@@ -1,18 +1,28 @@
 [[search-aggregations-metrics-max-aggregation]]
 === Max Aggregation
 
-A `single-value` metrics aggregation that keeps track and returns the maximum value among the numeric values extracted from the aggregated documents. These values can be extracted either from specific numeric fields in the documents, or be generated by a provided script.
+A `single-value` metrics aggregation that keeps track and returns the maximum
+value among the numeric values extracted from the aggregated documents. These
+values can be extracted either from specific numeric fields in the documents,
+or be generated by a provided script.
+
+NOTE: The `min` and `max` aggregation operate on the `double` representation of
+the data. As a consequence, the result may be approximate when running on longs
+whose absolute value is greater than +2^53+.
 
 Computing the max price value across all documents
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
     "aggs" : {
         "max_price" : { "max" : { "field" : "price" } }
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
 Response:
 
@@ -20,45 +30,52 @@ Response:
 --------------------------------------------------
 {
     ...
-
     "aggregations": {
         "max_price": {
-            "value": 35
+            "value": 200.0
         }
     }
 }
 --------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
 
-As can be seen, the name of the aggregation (`max_price` above) also serves as the key by which the aggregation result can be retrieved from the returned response.
+As can be seen, the name of the aggregation (`max_price` above) also serves as
+the key by which the aggregation result can be retrieved from the returned
+response.
 
 ==== Script
 
-Computing the max price value across all document, this time using a script:
+The `max` aggregation can also calculate the maximum of a script. The example
+below computes the maximum price:
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search
 {
     "aggs" : {
-        "max_price" : { 
-            "max" : { 
+        "max_price" : {
+            "max" : {
                 "script" : {
-                    "inline" : "doc['price'].value",
-                    "lang" : "painless"
+                    "source" : "doc.price.value"
                 }
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
-This will interpret the `script` parameter as an `inline` script with the `painless` script language and no script parameters. To use a file script use the following syntax:
+This will use the <<modules-scripting-painless, Painless>> scripting language
+and no script parameters. To use a file script use the following syntax:
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search
 {
     "aggs" : {
-        "max_price" : { 
-            "max" : { 
+        "max_price" : {
+            "max" : {
                 "script" : {
                     "file": "my_script",
                     "params": {
@@ -70,23 +87,28 @@ This will interpret the `script` parameter as an `inline` script with the `painl
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
-TIP: for indexed scripts replace the `file` parameter with an `id` parameter.
+TIP: For indexed scripts replace the `file` parameter with an `id` parameter.
 
 ==== Value Script
 
-Let's say that the prices of the documents in our index are in USD, but we would like to compute the max in EURO (and for the sake of this example, lets say the conversion rate is 1.2). We can use a value script to apply the conversion rate to every value before it is aggregated:
+Let's say that the prices of the documents in our index are in USD, but we
+would like to compute the max in EURO (and for the sake of this example, let's
+say the conversion rate is 1.2). We can use a value script to apply the
+conversion rate to every value before it is aggregated:
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search
 {
     "aggs" : {
         "max_price_in_euros" : {
             "max" : {
                 "field" : "price",
                 "script" : {
-                    "lang": "painless",
-                    "inline": "_value * params.conversion_rate",
+                    "source" : "_value * params.conversion_rate",
                     "params" : {
                         "conversion_rate" : 1.2
                     }
@@ -96,15 +118,18 @@ Let's say that the prices of the documents in our index are in USD, but we would
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
 ==== Missing value
 
-The `missing` parameter defines how documents that are missing a value should be treated.
-By default they will be ignored but it is also possible to treat them as if they
-had a value.
+The `missing` parameter defines how documents that are missing a value should
+be treated. By default they will be ignored but it is also possible to treat
+them as if they had a value.
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search
 {
     "aggs" : {
         "grade_max" : {
@@ -116,5 +141,8 @@ had a value.
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
-<1> Documents without a value in the `grade` field will fall into the same bucket as documents that have the value `10`.
+<1> Documents without a value in the `grade` field will fall into the same
+bucket as documents that have the value `10`.
diff --git a/docs/reference/aggregations/metrics/min-aggregation.asciidoc b/docs/reference/aggregations/metrics/min-aggregation.asciidoc
index 819d70343fb5e..2a3897eb9637d 100644
--- a/docs/reference/aggregations/metrics/min-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/min-aggregation.asciidoc
@@ -1,18 +1,28 @@
 [[search-aggregations-metrics-min-aggregation]]
 === Min Aggregation
 
-A `single-value` metrics aggregation that keeps track and returns the minimum value among numeric values extracted from the aggregated documents. These values can be extracted either from specific numeric fields in the documents, or be generated by a provided script.
+A `single-value` metrics aggregation that keeps track and returns the minimum
+value among numeric values extracted from the aggregated documents. These
+values can be extracted either from specific numeric fields in the documents,
+or be generated by a provided script.
+
+NOTE: The `min` and `max` aggregation operate on the `double` representation of
+the data. As a consequence, the result may be approximate when running on longs
+whose absolute value is greater than +2^53+.
 
 Computing the min price value across all documents:
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
     "aggs" : {
         "min_price" : { "min" : { "field" : "price" } }
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
 Response:
 
@@ -23,42 +33,50 @@ Response:
 
     "aggregations": {
         "min_price": {
-            "value": 10
+            "value": 10.0
         }
     }
 }
 --------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
 
-As can be seen, the name of the aggregation (`min_price` above) also serves as the key by which the aggregation result can be retrieved from the returned response.
+As can be seen, the name of the aggregation (`min_price` above) also serves as
+the key by which the aggregation result can be retrieved from the returned
+response.
 
 ==== Script
 
-Computing the min price value across all document, this time using a script:
+The `min` aggregation can also calculate the maximum of a script. The example
+below computes the minimum price:
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search
 {
     "aggs" : {
-        "min_price" : { 
-            "min" : { 
+        "min_price" : {
+            "min" : {
                 "script" : {
-                    "inline" : "doc['price'].value",
-                    "lang" : "painless"
+                    "source" : "doc.price.value"
                 }
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
-This will interpret the `script` parameter as an `inline` script with the `painless` script language and no script parameters. To use a file script use the following syntax:
+This will use the <<modules-scripting-painless, Painless>> scripting language
+and no script parameters. To use a file script use the following syntax:
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search
 {
     "aggs" : {
-        "min_price" : { 
-            "min" : { 
+        "min_price" : {
+            "min" : {
                 "script" : {
                     "file": "my_script",
                     "params": {
@@ -70,23 +88,28 @@ This will interpret the `script` parameter as an `inline` script with the `painl
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
-TIP: for indexed scripts replace the `file` parameter with an `id` parameter.
+TIP: For indexed scripts replace the `file` parameter with an `id` parameter.
 
 ==== Value Script
 
-Let's say that the prices of the documents in our index are in USD, but we would like to compute the min in EURO (and for the sake of this example, lets say the conversion rate is 1.2). We can use a value script to apply the conversion rate to every value before it is aggregated:
+Let's say that the prices of the documents in our index are in USD, but we
+would like to compute the min in EURO (and for the sake of this example, let's
+say the conversion rate is 1.2). We can use a value script to apply the
+conversion rate to every value before it is aggregated:
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search
 {
     "aggs" : {
         "min_price_in_euros" : {
             "min" : {
                 "field" : "price",
-                "script" : 
-                    "lang" : "painless",
-                    "inline": "_value * params.conversion_rate",
+                "script" : {
+                    "source" : "_value * params.conversion_rate",
                     "params" : {
                         "conversion_rate" : 1.2
                     }
@@ -96,15 +119,18 @@ Let's say that the prices of the documents in our index are in USD, but we would
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
 ==== Missing value
 
-The `missing` parameter defines how documents that are missing a value should be treated.
-By default they will be ignored but it is also possible to treat them as if they
-had a value.
+The `missing` parameter defines how documents that are missing a value should
+be treated. By default they will be ignored but it is also possible to treat
+them as if they had a value.
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search
 {
     "aggs" : {
         "grade_min" : {
@@ -116,5 +142,8 @@ had a value.
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
-<1> Documents without a value in the `grade` field will fall into the same bucket as documents that have the value `10`. 
+<1> Documents without a value in the `grade` field will fall into the same
+bucket as documents that have the value `10`.
diff --git a/docs/reference/aggregations/metrics/percentile-aggregation.asciidoc b/docs/reference/aggregations/metrics/percentile-aggregation.asciidoc
index db15d0a6a6639..f19f1f586cb30 100644
--- a/docs/reference/aggregations/metrics/percentile-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/percentile-aggregation.asciidoc
@@ -86,7 +86,78 @@ must be a value between 0-100 inclusive):
 --------------------------------------------------
 <1> Use the `percents` parameter to specify particular percentiles to calculate
 
+==== Keyed Response
 
+By default the `keyed` flag is set to `true` which associates a unique string key with each bucket and returns the ranges as a hash rather than an array. Setting the `keyed` flag to `false` will disable this behavior:
+
+[source,js]
+--------------------------------------------------
+POST bank/account/_search?size=0
+{
+    "aggs": {
+        "balance_outlier": {
+            "percentiles": {
+                "field": "balance",
+                "keyed": false
+            }
+        }
+    }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:bank]
+
+Response:
+
+[source,js]
+--------------------------------------------------
+{
+    ...
+
+    "aggregations": {
+        "balance_outlier": {
+            "values": [
+                {
+                    "key": 1.0,
+                    "value": 1462.8400000000001
+                },
+                {
+                    "key": 5.0,
+                    "value": 3591.85
+                },
+                {
+                    "key": 25.0,
+                    "value": 13709.333333333334
+                },
+                {
+                    "key": 50.0,
+                    "value": 26020.11666666667
+                },
+                {
+                    "key": 75.0,
+                    "value": 38139.648148148146
+                },
+                {
+                    "key": 95.0,
+                    "value": 47551.549999999996
+                },
+                {
+                    "key": 99.0,
+                    "value": 49339.16
+                }
+            ]
+        }
+    }
+}
+--------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
+// TESTRESPONSE[s/1462.8400000000001/$body.aggregations.balance_outlier.values.0.value/]
+// TESTRESPONSE[s/3591.85/$body.aggregations.balance_outlier.values.1.value/]
+// TESTRESPONSE[s/13709.333333333334/$body.aggregations.balance_outlier.values.2.value/]
+// TESTRESPONSE[s/26020.11666666667/$body.aggregations.balance_outlier.values.3.value/]
+// TESTRESPONSE[s/38139.648148148146/$body.aggregations.balance_outlier.values.4.value/]
+// TESTRESPONSE[s/47551.549999999996/$body.aggregations.balance_outlier.values.5.value/]
+// TESTRESPONSE[s/49339.16/$body.aggregations.balance_outlier.values.6.value/]
 
 ==== Script
 
@@ -102,7 +173,7 @@ a script to convert them on-the-fly:
             "percentiles" : {
                 "script" : {
                     "lang": "painless",
-                    "inline": "doc['load_time'].value / params.timeUnit", <1>
+                    "source": "doc['load_time'].value / params.timeUnit", <1>
                     "params" : {
                         "timeUnit" : 1000   <2>
                     }
diff --git a/docs/reference/aggregations/metrics/percentile-rank-aggregation.asciidoc b/docs/reference/aggregations/metrics/percentile-rank-aggregation.asciidoc
index d4df92105defc..610933638d3d5 100644
--- a/docs/reference/aggregations/metrics/percentile-rank-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/percentile-rank-aggregation.asciidoc
@@ -58,6 +58,54 @@ The response will look like this:
 From this information you can determine you are hitting the 99% load time target but not quite
 hitting the 95% load time target
 
+==== Keyed Response
+
+By default the `keyed` flag is set to `true` associates a unique string key with each bucket and returns the ranges as a hash rather than an array. Setting the `keyed` flag to `false` will disable this behavior:
+
+[source,js]
+--------------------------------------------------
+POST bank/account/_search?size=0
+{
+    "aggs": {
+        "balance_outlier": {
+            "percentile_ranks": {
+                "field": "balance",
+                "values": [25000, 50000],
+                "keyed": false
+            }
+        }
+    }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:bank]
+
+Response:
+
+[source,js]
+--------------------------------------------------
+{
+    ...
+
+    "aggregations": {
+        "balance_outlier": {
+            "values": [
+                {
+                    "key": 25000.0,
+                    "value": 48.537724935732655
+                },
+                {
+                    "key": 50000.0,
+                    "value": 99.85567010309278
+                }
+            ]
+        }
+    }
+}
+--------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
+// TESTRESPONSE[s/48.537724935732655/$body.aggregations.balance_outlier.values.0.value/]
+// TESTRESPONSE[s/99.85567010309278/$body.aggregations.balance_outlier.values.1.value/]
 
 ==== Script
 
@@ -74,7 +122,7 @@ a script to convert them on-the-fly:
                 "values" : [3, 5],
                 "script" : {
                     "lang": "painless",
-                    "inline": "doc['load_time'].value / params.timeUnit", <1>
+                    "source": "doc['load_time'].value / params.timeUnit", <1>
                     "params" : {
                         "timeUnit" : 1000   <2>
                     }
diff --git a/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc b/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc
index ba9899f9d687d..c53ced8a71921 100644
--- a/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc
@@ -9,6 +9,7 @@ Example:
 
 [source,js]
 --------------------------------------------------
+POST ledger/_search?size=0
 {
     "query" : {
         "match_all" : {}
@@ -16,15 +17,17 @@ Example:
     "aggs": {
         "profit": {
             "scripted_metric": {
-                "init_script" : "_agg['transactions'] = []",
-                "map_script" : "if (doc['type'].value == \"sale\") { _agg.transactions.add(doc['amount'].value) } else { _agg.transactions.add(-1 * doc['amount'].value) }", <1>
-                "combine_script" : "profit = 0; for (t in _agg.transactions) { profit += t }; return profit",
-                "reduce_script" : "profit = 0; for (a in _aggs) { profit += a }; return profit"
+                "init_script" : "params._agg.transactions = []",
+                "map_script" : "params._agg.transactions.add(doc.type.value == 'sale' ? doc.amount.value : -1 * doc.amount.value)", <1>
+                "combine_script" : "double profit = 0; for (t in params._agg.transactions) { profit += t } return profit",
+                "reduce_script" : "double profit = 0; for (a in params._aggs) { profit += a } return profit"
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:ledger]
 
 <1> `map_script` is the only required  parameter
 
@@ -35,24 +38,24 @@ The response for the above aggregation:
 [source,js]
 --------------------------------------------------
 {
+    "took": 218,
     ...
-
     "aggregations": {
         "profit": {
-            "value": 170
+            "value": 240.0
         }
    }
 }
 --------------------------------------------------
+// TESTRESPONSE[s/"took": 218/"took": $body.took/]
+// TESTRESPONSE[s/\.\.\./"_shards": $body._shards, "hits": $body.hits, "timed_out": false,/]
 
 The above example can also be specified using file scripts as follows:
 
 [source,js]
 --------------------------------------------------
+POST ledger/_search?size=0
 {
-    "query" : {
-        "match_all" : {}
-    },
     "aggs": {
         "profit": {
             "scripted_metric": {
@@ -66,29 +69,53 @@ The above example can also be specified using file scripts as follows:
                     "file": "my_combine_script"
                 },
                 "params": {
-                    "field": "amount" <1>
+                    "field": "amount", <1>
+                    "_agg": {}        <2>
                 },
                 "reduce_script" : {
                     "file": "my_reduce_script"
-                },
+                }
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:ledger]
+
+<1> script parameters for `init`, `map` and `combine` scripts must be specified
+in a global `params` object so that it can be share between the scripts.
+<2> if you specify script parameters then you must specify `"_agg": {}`.
+
+////
+Verify this response as well but in a hidden block.
 
-<1> script parameters for init, map and combine scripts must be specified in a global `params` object so that it can be share between the scripts
+[source,js]
+--------------------------------------------------
+{
+    "took": 218,
+    ...
+    "aggregations": {
+        "profit": {
+            "value": 240.0
+        }
+   }
+}
+--------------------------------------------------
+// TESTRESPONSE[s/"took": 218/"took": $body.took/]
+// TESTRESPONSE[s/\.\.\./"_shards": $body._shards, "hits": $body.hits, "timed_out": false,/]
+////
 
 For more details on specifying scripts see <<modules-scripting, script documentation>>.
 
 ==== Allowed return types
 
-Whilst and valid script object can be used within a single script. the scripts must return or store in the `_agg` object only the following types:
+Whilst any valid script object can be used within a single script, the scripts must return or store in the `_agg` object only the following types:
 
 * primitive types
 * String
 * Map (containing only keys and values of the types listed here)
-* Array (containing elements of only the types listed here) 
+* Array (containing elements of only the types listed here)
 
 ==== Scope of scripts
 
@@ -98,62 +125,45 @@ init_script::       Executed prior to any collection of documents. Allows the ag
 +
 In the above example, the `init_script` creates an array `transactions` in the `_agg` object.
 
-map_script::        Executed once per document collected. This is the only required script. If no combine_script is specified, the resulting state 
+map_script::        Executed once per document collected. This is the only required script. If no combine_script is specified, the resulting state
                     needs to be stored in an object named `_agg`.
 +
-In the above example, the `map_script` checks the value of the type field. If the value is 'sale' the value of the amount field 
-is added to the transactions array. If the value of the type field is not 'sale' the negated value of the amount field is added 
+In the above example, the `map_script` checks the value of the type field. If the value is 'sale' the value of the amount field
+is added to the transactions array. If the value of the type field is not 'sale' the negated value of the amount field is added
 to transactions.
 
-combine_script::    Executed once on each shard after document collection is complete. Allows the aggregation to consolidate the state returned from 
+combine_script::    Executed once on each shard after document collection is complete. Allows the aggregation to consolidate the state returned from
                     each shard. If a combine_script is not provided the combine phase will return the aggregation variable.
 +
-In the above example, the `combine_script` iterates through all the stored transactions, summing the values in the `profit` variable 
+In the above example, the `combine_script` iterates through all the stored transactions, summing the values in the `profit` variable
 and finally returns `profit`.
 
-reduce_script::     Executed once on the coordinating node after all shards have returned their results. The script is provided with access to a 
-                    variable `_aggs` which is an array of the result of the combine_script on each shard. If a reduce_script is not provided 
+reduce_script::     Executed once on the coordinating node after all shards have returned their results. The script is provided with access to a
+                    variable `_aggs` which is an array of the result of the combine_script on each shard. If a reduce_script is not provided
                     the reduce phase will return the `_aggs` variable.
 +
-In the above example, the `reduce_script` iterates through the `profit` returned by each shard summing the values before returning the 
+In the above example, the `reduce_script` iterates through the `profit` returned by each shard summing the values before returning the
 final combined profit which will be returned in the response of the aggregation.
 
 ==== Worked Example
 
-Imagine a situation where you index the following documents into and index with 2 shards:
+Imagine a situation where you index the following documents into an index with 2 shards:
 
 [source,js]
 --------------------------------------------------
-$ curl -XPUT 'http://localhost:9200/transactions/stock/1' -d '
-{
-    "type": "sale",
-    "amount": 80
-}
-'
-
-$ curl -XPUT 'http://localhost:9200/transactions/stock/2' -d '
-{
-    "type": "cost",
-    "amount": 10
-}
-'
-
-$ curl -XPUT 'http://localhost:9200/transactions/stock/3' -d '
-{
-    "type": "cost",
-    "amount": 30
-}
-'
-
-$ curl -XPUT 'http://localhost:9200/transactions/stock/4' -d '
-{
-    "type": "sale",
-    "amount": 130
-}
-'
+PUT /transactions/stock/_bulk?refresh
+{"index":{"_id":1}}
+{"type": "sale","amount": 80}
+{"index":{"_id":2}}
+{"type": "cost","amount": 10}
+{"index":{"_id":3}}
+{"type": "cost","amount": 30}
+{"index":{"_id":4}}
+{"type": "sale","amount": 130}
 --------------------------------------------------
+// CONSOLE
 
-Lets say that documents 1 and 3 end up on shard A and documents 2 and 4 end up on shard B. The following is a breakdown of what the aggregation result is 
+Lets say that documents 1 and 3 end up on shard A and documents 2 and 4 end up on shard B. The following is a breakdown of what the aggregation result is
 at each stage of the example above.
 
 ===== Before init_script
@@ -221,7 +231,7 @@ Shard B::
 
 ===== After combine_script
 
-The combine_script is executed on each shard after document collection is complete and reduces all the transactions down to a single profit figure for each 
+The combine_script is executed on each shard after document collection is complete and reduces all the transactions down to a single profit figure for each
 shard (by summing the values in the transactions array) which is passed back to the coordinating node:
 
 Shard A::        50
@@ -239,7 +249,7 @@ The reduce_script receives an `_aggs` array containing the result of the combine
 ]
 --------------------------------------------------
 
-It reduces the responses for the shards down to a final overall profit figure (by summing the values) and returns this as the result of the aggregation to 
+It reduces the responses for the shards down to a final overall profit figure (by summing the values) and returns this as the result of the aggregation to
 produce the response:
 
 [source,js]
@@ -258,8 +268,8 @@ produce the response:
 ==== Other Parameters
 
 [horizontal]
-params::           Optional. An object whose contents will be passed as variables to the  `init_script`, `map_script` and `combine_script`. This can be 
-                   useful to allow the user to control the behavior of the aggregation and for storing state between the scripts. If this is not specified, 
+params::           Optional. An object whose contents will be passed as variables to the  `init_script`, `map_script` and `combine_script`. This can be
+                   useful to allow the user to control the behavior of the aggregation and for storing state between the scripts. If this is not specified,
                    the default is the equivalent of providing:
 +
 [source,js]
@@ -268,6 +278,9 @@ params::           Optional. An object whose contents will be passed as variable
     "_agg" : {}
 }
 --------------------------------------------------
-reduce_params::    Optional. An object whose contents will be passed as variables to the `reduce_script`. This can be useful to allow the user to control 
-                   the behavior of the reduce phase. If this is not specified the variable will be undefined in the reduce_script execution.
 
+==== Empty Buckets
+
+If a parent bucket of the scripted metric aggregation does not collect any documents an empty aggregation response will be returned from the
+shard with a `null` value. In this case the `reduce_script`'s `_aggs` variable will contain `null` as a response from that shard.  
+`reduce_script`'s should therefore expect and deal with `null` responses from shards.  
diff --git a/docs/reference/aggregations/metrics/stats-aggregation.asciidoc b/docs/reference/aggregations/metrics/stats-aggregation.asciidoc
index a442fb1215011..8c862e3d9d30b 100644
--- a/docs/reference/aggregations/metrics/stats-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/stats-aggregation.asciidoc
@@ -52,7 +52,7 @@ Computing the grades stats based on a script:
              "stats" : {
                  "script" : { 
                      "lang": "painless",
-                     "inline": "doc['grade'].value" 
+                     "source": "doc['grade'].value"
                  }
              } 
          }
@@ -100,7 +100,7 @@ It turned out that the exam was way above the level of the students and a grade
                     "field" : "grade",
                     "script" : 
                         "lang": "painless",
-                        "inline": "_value * params.correction",
+                        "source": "_value * params.correction",
                         "params" : {
                             "correction" : 1.2
                         }
diff --git a/docs/reference/aggregations/metrics/sum-aggregation.asciidoc b/docs/reference/aggregations/metrics/sum-aggregation.asciidoc
index b9aee74b32057..cd6936dbe9459 100644
--- a/docs/reference/aggregations/metrics/sum-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/sum-aggregation.asciidoc
@@ -3,78 +3,94 @@
 
 A `single-value` metrics aggregation that sums up numeric values that are extracted from the aggregated documents. These values can be extracted either from specific numeric fields in the documents, or be generated by a provided script.
 
-Assuming the data consists of documents representing stock ticks, where each tick holds the change in the stock price from the previous tick.
+Assuming the data consists of documents representing sales records we can sum
+the sale price of all hats with:
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
     "query" : {
         "constant_score" : {
             "filter" : {
-                "range" : { "timestamp" : { "from" : "now/1d+9.5h", "to" : "now/1d+16h" }}
+                "match" : { "type" : "hat" }
             }
         }
     },
     "aggs" : {
-        "intraday_return" : { "sum" : { "field" : "change" } }
+        "hat_prices" : { "sum" : { "field" : "price" } }
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
-The above aggregation sums up all changes in the today's trading stock ticks which accounts for the intraday return. The aggregation type is `sum` and the `field` setting defines the numeric field of the documents of which values will be summed up. The above will return the following:
-
+Resulting in:
 
 [source,js]
 --------------------------------------------------
 {
     ...
-
     "aggregations": {
-        "intraday_return": {
-           "value": 2.18
+        "hat_prices": {
+           "value": 450.0
         }
     }
 }
 --------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
 
 The name of the aggregation (`intraday_return` above) also serves as the key by which the aggregation result can be retrieved from the returned response.
 
 ==== Script
 
-Computing the intraday return based on a script:
+We could also use a script to fetch the sales price:
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
-    ...,
-
+    "query" : {
+        "constant_score" : {
+            "filter" : {
+                "match" : { "type" : "hat" }
+            }
+        }
+    },
     "aggs" : {
-        "intraday_return" : { 
-            "sum" : { 
+        "hat_prices" : {
+            "sum" : {
                 "script" : {
-                   "lang": "painless",
-                   "inline": "doc['change'].value" 
-                } 
+                   "source": "doc.price.value"
+                }
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
 This will interpret the `script` parameter as an `inline` script with the `painless` script language and no script parameters. To use a file script use the following syntax:
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
-    ...,
-
+    "query" : {
+        "constant_score" : {
+            "filter" : {
+                "match" : { "type" : "hat" }
+            }
+        }
+    },
     "aggs" : {
-        "intraday_return" : { 
-            "sum" : { 
+        "hat_prices" : {
+            "sum" : {
                 "script" : {
                     "file": "my_script",
                     "params" : {
-                        "field" : "change"
+                        "field" : "price"
                     }
                 }
             }
@@ -82,52 +98,69 @@ This will interpret the `script` parameter as an `inline` script with the `painl
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
 TIP: for indexed scripts replace the `file` parameter with an `id` parameter.
 
 ===== Value Script
 
-Computing the sum of squares over all stock tick changes:
+It is also possible to access the field value from the script using `_value`.
+For example, this will sum the square of the prices for all hats:
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
+    "query" : {
+        "constant_score" : {
+            "filter" : {
+                "match" : { "type" : "hat" }
+            }
+        }
+    },
     "aggs" : {
-        ...
-
-        "aggs" : {
-            "daytime_return" : {
-                "sum" : {
-                    "field" : "change",
-                    "script" : {
-                        "lang": "painless",
-                        "inline": "_value * _value"
-                    }
+        "square_hats" : {
+            "sum" : {
+                "field" : "price",
+                "script" : {
+                    "source": "_value * _value"
                 }
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
 ==== Missing value
 
-The `missing` parameter defines how documents that are missing a value should be treated.
-By default they will be ignored but it is also possible to treat them as if they
-had a value.
+The `missing` parameter defines how documents that are missing a value should
+be treated. By default documents missing the value will be ignored but it is
+also possible to treat them as if they had a value. For example, this treats
+all hat sales without a price as being `100`.
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
+    "query" : {
+        "constant_score" : {
+            "filter" : {
+                "match" : { "type" : "hat" }
+            }
+        }
+    },
     "aggs" : {
-        "total_time" : {
+        "hat_prices" : {
             "sum" : {
-                "field" : "took",
+                "field" : "price",
                 "missing": 100 <1>
             }
         }
     }
 }
 --------------------------------------------------
-
-<1> Documents without a value in the `took` field will fall into the same bucket as documents that have the value `100`.
+// CONSOLE
+// TEST[setup:sales]
diff --git a/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc b/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc
index 83607b18df3bc..05fb51e6d8530 100644
--- a/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc
@@ -33,27 +33,26 @@ only the title field is being included in the source.
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
     "aggs": {
-        "top-tags": {
+        "top_tags": {
             "terms": {
-                "field": "tags",
+                "field": "type",
                 "size": 3
             },
             "aggs": {
-                "top_tag_hits": {
+                "top_sales_hits": {
                     "top_hits": {
                         "sort": [
                             {
-                                "last_activity_date": {
+                                "date": {
                                     "order": "desc"
                                 }
                             }
                         ],
                         "_source": {
-                            "include": [
-                                "title"
-                            ]
+                            "includes": [ "date", "price" ]
                         },
                         "size" : 1
                     }
@@ -63,90 +62,105 @@ only the title field is being included in the source.
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
-Possible response snippet:
+Possible response:
 
 [source,js]
 --------------------------------------------------
-"aggregations": {
-  "top-tags": {
-     "buckets": [
-        {
-           "key": "windows-7",
-           "doc_count": 25365,
-           "top_tags_hits": {
-              "hits": {
-                 "total": 25365,
-                 "max_score": 1,
-                 "hits": [
-                    {
-                       "_index": "stack",
-                       "_type": "question",
-                       "_id": "602679",
-                       "_score": 1,
-                       "_source": {
-                          "title": "Windows port opening"
-                       },
-                       "sort": [
-                          1370143231177
-                       ]
-                    }
-                 ]
-              }
-           }
-        },
-        {
-           "key": "linux",
-           "doc_count": 18342,
-           "top_tags_hits": {
-              "hits": {
-                 "total": 18342,
-                 "max_score": 1,
-                 "hits": [
-                    {
-                       "_index": "stack",
-                       "_type": "question",
-                       "_id": "602672",
-                       "_score": 1,
-                       "_source": {
-                          "title": "Ubuntu RFID Screensaver lock-unlock"
-                       },
-                       "sort": [
-                          1370143379747
-                       ]
-                    }
-                 ]
-              }
-           }
-        },
-        {
-           "key": "windows",
-           "doc_count": 18119,
-           "top_tags_hits": {
-              "hits": {
-                 "total": 18119,
-                 "max_score": 1,
-                 "hits": [
-                    {
-                       "_index": "stack",
-                       "_type": "question",
-                       "_id": "602678",
-                       "_score": 1,
-                       "_source": {
-                          "title": "If I change my computers date / time, what could be affected?"
-                       },
-                       "sort": [
-                          1370142868283
-                       ]
-                    }
-                 ]
-              }
-           }
-        }
-     ]
+{
+  ...
+  "aggregations": {
+    "top_tags": {
+       "doc_count_error_upper_bound": 0,
+       "sum_other_doc_count": 0,
+       "buckets": [
+          {
+             "key": "hat",
+             "doc_count": 3,
+             "top_sales_hits": {
+                "hits": {
+                   "total": 3,
+                   "max_score": null,
+                   "hits": [
+                      {
+                         "_index": "sales",
+                         "_type": "sale",
+                         "_id": "AVnNBmauCQpcRyxw6ChK",
+                         "_source": {
+                            "date": "2015/03/01 00:00:00",
+                            "price": 200
+                         },
+                         "sort": [
+                            1425168000000
+                         ],
+                         "_score": null
+                      }
+                   ]
+                }
+             }
+          },
+          {
+             "key": "t-shirt",
+             "doc_count": 3,
+             "top_sales_hits": {
+                "hits": {
+                   "total": 3,
+                   "max_score": null,
+                   "hits": [
+                      {
+                         "_index": "sales",
+                         "_type": "sale",
+                         "_id": "AVnNBmauCQpcRyxw6ChL",
+                         "_source": {
+                            "date": "2015/03/01 00:00:00",
+                            "price": 175
+                         },
+                         "sort": [
+                            1425168000000
+                         ],
+                         "_score": null
+                      }
+                   ]
+                }
+             }
+          },
+          {
+             "key": "bag",
+             "doc_count": 1,
+             "top_sales_hits": {
+                "hits": {
+                   "total": 1,
+                   "max_score": null,
+                   "hits": [
+                      {
+                         "_index": "sales",
+                         "_type": "sale",
+                         "_id": "AVnNBmatCQpcRyxw6ChH",
+                         "_source": {
+                            "date": "2015/01/01 00:00:00",
+                            "price": 150
+                         },
+                         "sort": [
+                            1420070400000
+                         ],
+                         "_score": null
+                      }
+                   ]
+                }
+             }
+          }
+       ]
+    }
   }
 }
 --------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
+// TESTRESPONSE[s/AVnNBmauCQpcRyxw6ChK/$body.aggregations.top_tags.buckets.0.top_sales_hits.hits.hits.0._id/]
+// TESTRESPONSE[s/AVnNBmauCQpcRyxw6ChL/$body.aggregations.top_tags.buckets.1.top_sales_hits.hits.hits.0._id/]
+// TESTRESPONSE[s/AVnNBmatCQpcRyxw6ChH/$body.aggregations.top_tags.buckets.2.top_sales_hits.hits.hits.0._id/]
+
 
 ==== Field collapse example
 
@@ -184,8 +198,7 @@ relevancy order of the most relevant document in a bucket.
         "top_hit" : {
           "max": {
             "script": {
-              "lang": "painless",
-              "inline": "_score"
+              "source": "_score"
             }
           }
         }
diff --git a/docs/reference/aggregations/metrics/valuecount-aggregation.asciidoc b/docs/reference/aggregations/metrics/valuecount-aggregation.asciidoc
index 925f5426187b8..d7752f636f38e 100644
--- a/docs/reference/aggregations/metrics/valuecount-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/valuecount-aggregation.asciidoc
@@ -8,12 +8,15 @@ one might be interested in the number of values the average is computed over.
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
     "aggs" : {
-        "grades_count" : { "value_count" : { "field" : "grade" } }
+        "types_count" : { "value_count" : { "field" : "type" } }
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
 Response:
 
@@ -21,16 +24,16 @@ Response:
 --------------------------------------------------
 {
     ...
-
     "aggregations": {
-        "grades_count": {
-            "value": 10
+        "types_count": {
+            "value": 7
         }
     }
 }
 --------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
 
-The name of the aggregation (`grades_count` above) also serves as the key by which the aggregation result can be
+The name of the aggregation (`types_count` above) also serves as the key by which the aggregation result can be
 retrieved from the returned response.
 
 ==== Script
@@ -39,36 +42,35 @@ Counting the values generated by a script:
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
-    ...,
-
     "aggs" : {
-        "grades_count" : { 
-            "value_count" : { 
+        "type_count" : {
+            "value_count" : {
                 "script" : {
-                    "inline" : "doc['grade'].value",
-                    "lang" : "painless"
+                    "source" : "doc['type'].value"
                 }
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
 This will interpret the `script` parameter as an `inline` script with the `painless` script language and no script parameters. To use a file script use the following syntax:
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
-    ...,
-
     "aggs" : {
-        "grades_count" : { 
-            "value_count" : { 
+        "types_count" : {
+            "value_count" : {
                 "script" : {
                     "file": "my_script",
                     "params" : {
-                        "field" : "grade"
+                        "field" : "type"
                     }
                 }
             }
@@ -76,5 +78,7 @@ This will interpret the `script` parameter as an `inline` script with the `painl
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
 TIP: for indexed scripts replace the `file` parameter with an `id` parameter.
diff --git a/docs/reference/aggregations/misc.asciidoc b/docs/reference/aggregations/misc.asciidoc
index 7210daf141726..5998c0c12cff1 100644
--- a/docs/reference/aggregations/misc.asciidoc
+++ b/docs/reference/aggregations/misc.asciidoc
@@ -82,3 +82,89 @@ Then that piece of metadata will be returned in place for our `titles` terms agg
 }
 --------------------------------------------------
 // TESTRESPONSE[s/\.\.\./"took": "$body.took", "timed_out": false, "_shards": "$body._shards", "hits": "$body.hits"/]
+
+
+[[returning-aggregation-type]]
+== Returning the type of the aggregation
+
+Sometimes you need to know the exact type of an aggregation in order to parse its results. The `typed_keys` parameter
+ can be used to change the aggregation's name in the response so that it will be prefixed by its internal type.
+
+Considering the following <<search-aggregations-bucket-datehistogram-aggregation,`date_histogram` aggregation>> named
+`tweets_over_time` which has a sub <<search-aggregations-metrics-top-hits-aggregation, 'top_hits` aggregation>> named
+ `top_users`:
+
+[source,js]
+--------------------------------------------------
+GET /twitter/tweet/_search?typed_keys
+{
+  "aggregations": {
+    "tweets_over_time": {
+      "date_histogram": {
+        "field": "date",
+        "interval": "year"
+      },
+      "aggregations": {
+        "top_users": {
+            "top_hits": {
+                "size": 1
+            }
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
+
+In the response, the aggregations names will be changed to respectively `date_histogram#tweets_over_time` and
+`top_hits#top_users`, reflecting the internal types of each aggregation:
+
+[source,js]
+--------------------------------------------------
+{
+    "aggregations": {
+        "date_histogram#tweets_over_time": { <1>
+            "buckets" : [
+                {
+                    "key_as_string" : "2009-01-01T00:00:00.000Z",
+                    "key" : 1230768000000,
+                    "doc_count" : 5,
+                    "top_hits#top_users" : {  <2>
+                        "hits" : {
+                            "total" : 5,
+                            "max_score" : 1.0,
+                            "hits" : [
+                                {
+                                  "_index": "twitter",
+                                  "_type": "tweet",
+                                  "_id": "0",
+                                  "_score": 1.0,
+                                  "_source": {
+                                    "date": "2009-11-15T14:12:12",
+                                    "message": "trying out Elasticsearch",
+                                    "user": "kimchy",
+                                    "likes": 0
+                                  }
+                                }
+                            ]
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    ...
+}
+--------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": "$body.took", "timed_out": false, "_shards": "$body._shards", "hits": "$body.hits"/]
+
+<1> The name `tweets_over_time` now contains the `date_histogram` prefix.
+<2> The name `top_users` now contains the `top_hits` prefix.
+
+NOTE: For some aggregations, it is possible that the returned type is not the same as the one provided with the
+request. This is the case for Terms, Significant Terms and Percentiles aggregations, where the returned type
+also contains information about the type of the targeted field: `lterms` (for a terms aggregation on a Long field),
+ `sigsterms` (for a significant terms aggregation on a String field), `tdigest_percentiles` (for a percentile
+ aggregation based on the TDigest algorithm).
diff --git a/docs/reference/aggregations/pipeline.asciidoc b/docs/reference/aggregations/pipeline.asciidoc
index c9cd14d62039f..0bb2f646fffc7 100644
--- a/docs/reference/aggregations/pipeline.asciidoc
+++ b/docs/reference/aggregations/pipeline.asciidoc
@@ -167,7 +167,7 @@ POST /sales/_search
               "count": "categories._bucket_count" <1>
             },
             "script": {
-              "inline": "params.count != 0"
+              "source": "params.count != 0"
             }
           }
         }
diff --git a/docs/reference/aggregations/pipeline/avg-bucket-aggregation.asciidoc b/docs/reference/aggregations/pipeline/avg-bucket-aggregation.asciidoc
index dc1aa78baea84..fbbee906f446e 100644
--- a/docs/reference/aggregations/pipeline/avg-bucket-aggregation.asciidoc
+++ b/docs/reference/aggregations/pipeline/avg-bucket-aggregation.asciidoc
@@ -25,8 +25,8 @@ An `avg_bucket` aggregation looks like this in isolation:
 |`buckets_path` |The path to the buckets we wish to find the average for (see <<buckets-path-syntax>> for more
  details) |Required |
  |`gap_policy` |The policy to apply when gaps are found in the data (see <<gap-policy>> for more
- details)|Optional, defaults to `skip` ||
- |`format` |format to apply to the output value of this aggregation |Optional, defaults to `null` |
+ details) |Optional |`skip`
+ |`format` |format to apply to the output value of this aggregation |Optional | `null`
 |===
 
 The following snippet calculates the average of the total monthly `sales`:
diff --git a/docs/reference/aggregations/pipeline/bucket-script-aggregation.asciidoc b/docs/reference/aggregations/pipeline/bucket-script-aggregation.asciidoc
index 6a010650e5c03..737942bc6e2a4 100644
--- a/docs/reference/aggregations/pipeline/bucket-script-aggregation.asciidoc
+++ b/docs/reference/aggregations/pipeline/bucket-script-aggregation.asciidoc
@@ -18,7 +18,7 @@ A `bucket_script` aggregation looks like this in isolation:
             "my_var1": "the_sum", <1>
             "my_var2": "the_value_count"
         },
-        "script": "my_var1 / my_var2"
+        "script": "params.my_var1 / params.my_var2"
     }
 }
 --------------------------------------------------
@@ -33,9 +33,9 @@ the metrics to use for that variable.
 for more details) |Required |
 |`buckets_path` |A map of script variables and their associated path to the buckets we wish to use for the variable
 (see <<buckets-path-syntax>> for more details) |Required |
- |`gap_policy` |The policy to apply when gaps are found in the data (see <<gap-policy>> for more
- details)|Optional, defaults to `skip` |
- |`format` |format to apply to the output value of this aggregation |Optional, defaults to `null` |
+|`gap_policy` |The policy to apply when gaps are found in the data (see <<gap-policy>> for more
+ details)|Optional |`skip`
+|`format` |format to apply to the output value of this aggregation |Optional |`null`
 |===
 
 The following snippet calculates the ratio percentage of t-shirt sales compared to total sales each month:
diff --git a/docs/reference/aggregations/pipeline/bucket-selector-aggregation.asciidoc b/docs/reference/aggregations/pipeline/bucket-selector-aggregation.asciidoc
index 685c40f48308d..34018fc9556e3 100644
--- a/docs/reference/aggregations/pipeline/bucket-selector-aggregation.asciidoc
+++ b/docs/reference/aggregations/pipeline/bucket-selector-aggregation.asciidoc
@@ -38,8 +38,8 @@ the metrics to use for that variable.
 for more details) |Required |
 |`buckets_path` |A map of script variables and their associated path to the buckets we wish to use for the variable
 (see <<buckets-path-syntax>> for more details) |Required |
- |`gap_policy` |The policy to apply when gaps are found in the data (see <<gap-policy>> for more
- details)|Optional, defaults to `skip` |
+|`gap_policy` |The policy to apply when gaps are found in the data (see <<gap-policy>> for more
+ details)|Optional |`skip`
 |===
 
 The following snippet only retains buckets where the total sales for the month is more than 400:
diff --git a/docs/reference/aggregations/pipeline/cumulative-sum-aggregation.asciidoc b/docs/reference/aggregations/pipeline/cumulative-sum-aggregation.asciidoc
index 6039813351cc2..47cbb1b3f3c75 100644
--- a/docs/reference/aggregations/pipeline/cumulative-sum-aggregation.asciidoc
+++ b/docs/reference/aggregations/pipeline/cumulative-sum-aggregation.asciidoc
@@ -25,7 +25,7 @@ A `cumulative_sum` aggregation looks like this in isolation:
 |Parameter Name |Description |Required |Default Value
 |`buckets_path` |The path to the buckets we wish to find the cumulative sum for (see <<buckets-path-syntax>> for more
  details) |Required |
- |`format` |format to apply to the output value of this aggregation |Optional, defaults to `null` |
+|`format` |format to apply to the output value of this aggregation |Optional |`null` 
 |===
 
 The following snippet calculates the cumulative sum of the total monthly `sales`:
diff --git a/docs/reference/aggregations/pipeline/derivative-aggregation.asciidoc b/docs/reference/aggregations/pipeline/derivative-aggregation.asciidoc
index d7844cd6a3ed7..7db32caac929d 100644
--- a/docs/reference/aggregations/pipeline/derivative-aggregation.asciidoc
+++ b/docs/reference/aggregations/pipeline/derivative-aggregation.asciidoc
@@ -24,8 +24,8 @@ A `derivative` aggregation looks like this in isolation:
 |`buckets_path` |The path to the buckets we wish to find the derivative for (see <<buckets-path-syntax>> for more
  details) |Required |
  |`gap_policy` |The policy to apply when gaps are found in the data (see <<gap-policy>> for more
- details)|Optional, defaults to `skip` |
- |`format` |format to apply to the output value of this aggregation |Optional, defaults to `null` |
+ details)|Optional |`skip` 
+ |`format` |format to apply to the output value of this aggregation |Optional | `null` 
 |===
 
 
@@ -119,7 +119,7 @@ And the following may be the response:
 <1> No derivative for the first bucket since we need at least 2 data points to calculate the derivative
 <2> Derivative value units are implicitly defined by the `sales` aggregation and the parent histogram so in this case the units
 would be $/month assuming the `price` field has units of $.
-<3> The number of documents in the bucket are represented by the `doc_count` f
+<3> The number of documents in the bucket are represented by the `doc_count`
 
 ==== Second Order Derivative
 
diff --git a/docs/reference/aggregations/pipeline/extended-stats-bucket-aggregation.asciidoc b/docs/reference/aggregations/pipeline/extended-stats-bucket-aggregation.asciidoc
index 11d9c2906b90e..e6a69ccc2792e 100644
--- a/docs/reference/aggregations/pipeline/extended-stats-bucket-aggregation.asciidoc
+++ b/docs/reference/aggregations/pipeline/extended-stats-bucket-aggregation.asciidoc
@@ -32,7 +32,7 @@ A `extended_stats_bucket` aggregation looks like this in isolation:
 |`sigma` |The number of standard deviations above/below the mean to display |Optional | 2
 |===
 
-The following snippet calculates the sum of all the total monthly `sales` buckets:
+The following snippet calculates the extended stats for monthly `sales` bucket:
 
 [source,js]
 --------------------------------------------------
diff --git a/docs/reference/aggregations/pipeline/max-bucket-aggregation.asciidoc b/docs/reference/aggregations/pipeline/max-bucket-aggregation.asciidoc
index 4c11d7afc4159..96c2d7e0ee613 100644
--- a/docs/reference/aggregations/pipeline/max-bucket-aggregation.asciidoc
+++ b/docs/reference/aggregations/pipeline/max-bucket-aggregation.asciidoc
@@ -25,9 +25,9 @@ A `max_bucket` aggregation looks like this in isolation:
 |Parameter Name |Description |Required |Default Value
 |`buckets_path` |The path to the buckets we wish to find the maximum for (see <<buckets-path-syntax>> for more
  details) |Required |
- |`gap_policy` |The policy to apply when gaps are found in the data (see <<gap-policy>> for more
- details)|Optional, defaults to `skip` |
- |`format` |format to apply to the output value of this aggregation |Optional, defaults to `null` |
+|`gap_policy` |The policy to apply when gaps are found in the data (see <<gap-policy>> for more
+ details)|Optional | `skip`
+ |`format` |format to apply to the output value of this aggregation |Optional |`null` 
 |===
 
 The following snippet calculates the maximum of the total monthly `sales`:
diff --git a/docs/reference/aggregations/pipeline/min-bucket-aggregation.asciidoc b/docs/reference/aggregations/pipeline/min-bucket-aggregation.asciidoc
index 9056707ee46eb..031354fadb4c6 100644
--- a/docs/reference/aggregations/pipeline/min-bucket-aggregation.asciidoc
+++ b/docs/reference/aggregations/pipeline/min-bucket-aggregation.asciidoc
@@ -9,7 +9,7 @@ be a multi-bucket aggregation.
 
 ==== Syntax
 
-A `max_bucket` aggregation looks like this in isolation:
+A `min_bucket` aggregation looks like this in isolation:
 
 [source,js]
 --------------------------------------------------
@@ -26,11 +26,10 @@ A `max_bucket` aggregation looks like this in isolation:
 |`buckets_path` |The path to the buckets we wish to find the minimum for (see <<buckets-path-syntax>> for more
  details) |Required |
  |`gap_policy` |The policy to apply when gaps are found in the data (see <<gap-policy>> for more
- details)|Optional, defaults to `skip` |
- |`format` |format to apply to the output value of this aggregation |Optional, defaults to `null` |
+ details)|Optional | `skip`
+ |`format` |format to apply to the output value of this aggregation |Optional |`null`
 |===
 
-
 The following snippet calculates the minimum of the total monthly `sales`:
 
 [source,js]
@@ -63,7 +62,7 @@ POST /sales/_search
 // CONSOLE
 // TEST[setup:sales]
 
-<1> `buckets_path` instructs this max_bucket aggregation that we want the minimum value of the `sales` aggregation in the
+<1> `buckets_path` instructs this min_bucket aggregation that we want the minimum value of the `sales` aggregation in the
 `sales_per_month` date histogram.
 
 And the following may be the response:
diff --git a/docs/reference/aggregations/pipeline/movavg-aggregation.asciidoc b/docs/reference/aggregations/pipeline/movavg-aggregation.asciidoc
index bbe4170e2e7c2..9025b7d9c8bb0 100644
--- a/docs/reference/aggregations/pipeline/movavg-aggregation.asciidoc
+++ b/docs/reference/aggregations/pipeline/movavg-aggregation.asciidoc
@@ -27,7 +27,7 @@ A `moving_avg` aggregation looks like this in isolation:
         "buckets_path": "the_sum",
         "model": "holt",
         "window": 5,
-        "gap_policy": "insert_zero",
+        "gap_policy": "insert_zeros",
         "settings": {
             "alpha": 0.8
         }
@@ -40,7 +40,7 @@ A `moving_avg` aggregation looks like this in isolation:
 |Parameter Name |Description |Required |Default Value
 |`buckets_path` |Path to the metric of interest (see <<buckets-path-syntax, `buckets_path` Syntax>> for more details |Required |
 |`model` |The moving average weighting model that we wish to use |Optional |`simple`
-|`gap_policy` |Determines what should happen when a gap in the data is encountered. |Optional |`insert_zero`
+|`gap_policy` |Determines what should happen when a gap in the data is encountered. |Optional |`insert_zeros`
 |`window` |The size of window to "slide" across the histogram. |Optional |`5`
 |`minimize` |If the model should be algorithmically minimized.  See <<movavg-minimizer, Minimization>> for more
  details |Optional |`false` for most models
diff --git a/docs/reference/aggregations/pipeline/percentiles-bucket-aggregation.asciidoc b/docs/reference/aggregations/pipeline/percentiles-bucket-aggregation.asciidoc
index fcb29104d67b7..e83b46cb2a97d 100644
--- a/docs/reference/aggregations/pipeline/percentiles-bucket-aggregation.asciidoc
+++ b/docs/reference/aggregations/pipeline/percentiles-bucket-aggregation.asciidoc
@@ -19,10 +19,10 @@ A `percentiles_bucket` aggregation looks like this in isolation:
 }
 --------------------------------------------------
 
-.`sum_bucket` Parameters
+.`percentiles_bucket` Parameters
 |===
 |Parameter Name |Description |Required |Default Value
-|`buckets_path` |The path to the buckets we wish to find the sum for (see <<buckets-path-syntax>> for more
+|`buckets_path` |The path to the buckets we wish to find the percentiles for (see <<buckets-path-syntax>> for more
  details) |Required |
 |`gap_policy` |The policy to apply when gaps are found in the data (see <<gap-policy>> for more
  details)|Optional | `skip`
@@ -30,7 +30,7 @@ A `percentiles_bucket` aggregation looks like this in isolation:
 |`percents` |The list of percentiles to calculate |Optional | `[ 1, 5, 25, 50, 75, 95, 99 ]`
 |===
 
-The following snippet calculates the sum of all the total monthly `sales` buckets:
+The following snippet calculates the percentiles for the total monthly `sales` buckets:
 
 [source,js]
 --------------------------------------------------
@@ -65,7 +65,7 @@ POST /sales/_search
 
 <1> `buckets_path` instructs this percentiles_bucket aggregation that we want to calculate percentiles for
 the `sales` aggregation in the `sales_per_month` date histogram.
-<2> `percents` specifies which percentiles we wish to calculate, in this case, the 25th, 50th and 75th percentil
+<2> `percents` specifies which percentiles we wish to calculate, in this case, the 25th, 50th and 75th percentiles.
 
 And the following may be the response:
 
@@ -107,7 +107,7 @@ And the following may be the response:
       },
       "percentiles_monthly_sales": {
         "values" : {
-            "25.0": 60.0,
+            "25.0": 375.0,
             "50.0": 375.0,
             "75.0": 550.0
          }
diff --git a/docs/reference/aggregations/pipeline/stats-bucket-aggregation.asciidoc b/docs/reference/aggregations/pipeline/stats-bucket-aggregation.asciidoc
index 6c08fe5cd2fe3..9e9d4c8aa7a9a 100644
--- a/docs/reference/aggregations/pipeline/stats-bucket-aggregation.asciidoc
+++ b/docs/reference/aggregations/pipeline/stats-bucket-aggregation.asciidoc
@@ -29,7 +29,7 @@ A `stats_bucket` aggregation looks like this in isolation:
 |`format` |format to apply to the output value of this aggregation |Optional | `null`
 |===
 
-The following snippet calculates the sum of all the total monthly `sales` buckets:
+The following snippet calculates the stats for monthly `sales`:
 
 [source,js]
 --------------------------------------------------
diff --git a/docs/reference/aggregations/pipeline/sum-bucket-aggregation.asciidoc b/docs/reference/aggregations/pipeline/sum-bucket-aggregation.asciidoc
index 142b6b6f777bd..3c249959f9dfd 100644
--- a/docs/reference/aggregations/pipeline/sum-bucket-aggregation.asciidoc
+++ b/docs/reference/aggregations/pipeline/sum-bucket-aggregation.asciidoc
@@ -25,8 +25,8 @@ A `sum_bucket` aggregation looks like this in isolation:
 |`buckets_path` |The path to the buckets we wish to find the sum for (see <<buckets-path-syntax>> for more
  details) |Required |
  |`gap_policy` |The policy to apply when gaps are found in the data (see <<gap-policy>> for more
- details)|Optional, defaults to `skip` ||
- |`format` |format to apply to the output value of this aggregation |Optional, defaults to `null` |
+ details)|Optional | `skip`
+ |`format` |format to apply to the output value of this aggregation |Optional |`null` 
 |===
 
 The following snippet calculates the sum of all the total monthly `sales` buckets:
diff --git a/docs/reference/analysis.asciidoc b/docs/reference/analysis.asciidoc
index 42b1ef65d2489..a8299f45e19fc 100644
--- a/docs/reference/analysis.asciidoc
+++ b/docs/reference/analysis.asciidoc
@@ -112,6 +112,8 @@ include::analysis/testing.asciidoc[]
 
 include::analysis/analyzers.asciidoc[]
 
+include::analysis/normalizers.asciidoc[]
+
 include::analysis/tokenizers.asciidoc[]
 
 include::analysis/tokenfilters.asciidoc[]
diff --git a/docs/reference/analysis/analyzers/custom-analyzer.asciidoc b/docs/reference/analysis/analyzers/custom-analyzer.asciidoc
index f14759856dd05..34572acaa9650 100644
--- a/docs/reference/analysis/analyzers/custom-analyzer.asciidoc
+++ b/docs/reference/analysis/analyzers/custom-analyzer.asciidoc
@@ -202,7 +202,7 @@ POST my_index/_analyze
 --------------------------------------------------
 // CONSOLE
 
-<1> The `emoticon` character filter, `punctuation` tokenizer and
+<1> The `emoticons` character filter, `punctuation` tokenizer and
     `english_stop` token filter are custom implementations which are defined
     in the same index settings.
 
diff --git a/docs/reference/analysis/analyzers/lang-analyzer.asciidoc b/docs/reference/analysis/analyzers/lang-analyzer.asciidoc
index 983a719e4f477..65cc30780b1a7 100644
--- a/docs/reference/analysis/analyzers/lang-analyzer.asciidoc
+++ b/docs/reference/analysis/analyzers/lang-analyzer.asciidoc
@@ -55,9 +55,9 @@ functionality is implemented by adding the
 with the `keywords` set to the value of the `stem_exclusion` parameter.
 
 The following analyzers support setting custom `stem_exclusion` list:
-`arabic`, `armenian`, `basque`, `catalan`, `bulgarian`, `catalan`,
-`czech`, `finnish`, `dutch`, `english`, `finnish`, `french`, `galician`,
-`german`, `irish`, `hindi`, `hungarian`, `indonesian`, `italian`, `latvian`,
+`arabic`, `armenian`, `basque`, `bulgarian`, `catalan`, `czech`,
+`dutch`, `english`, `finnish`, `french`, `galician`,
+`german`, `hindi`, `hungarian`, `indonesian`, `irish`, `italian`, `latvian`,
 `lithuanian`, `norwegian`, `portuguese`, `romanian`, `russian`, `sorani`,
 `spanish`, `swedish`, `turkish`.
 
@@ -77,6 +77,7 @@ The `arabic` analyzer could be reimplemented as a `custom` analyzer as follows:
 
 [source,js]
 ----------------------------------------------------
+PUT /arabic_example
 {
   "settings": {
     "analysis": {
@@ -87,7 +88,7 @@ The `arabic` analyzer could be reimplemented as a `custom` analyzer as follows:
         },
         "arabic_keywords": {
           "type":       "keyword_marker",
-          "keywords":   [] <2>
+          "keywords":   ["مثال"] <2>
         },
         "arabic_stemmer": {
           "type":       "stemmer",
@@ -110,6 +111,7 @@ The `arabic` analyzer could be reimplemented as a `custom` analyzer as follows:
   }
 }
 ----------------------------------------------------
+// CONSOLE
 <1> The default stopwords can be overridden with the `stopwords`
     or `stopwords_path` parameters.
 <2> This filter should be removed unless there are words which should
@@ -122,6 +124,7 @@ The `armenian` analyzer could be reimplemented as a `custom` analyzer as follows
 
 [source,js]
 ----------------------------------------------------
+PUT /armenian_example
 {
   "settings": {
     "analysis": {
@@ -132,7 +135,7 @@ The `armenian` analyzer could be reimplemented as a `custom` analyzer as follows
         },
         "armenian_keywords": {
           "type":       "keyword_marker",
-          "keywords":   [] <2>
+          "keywords":   ["օրինակ"] <2>
         },
         "armenian_stemmer": {
           "type":       "stemmer",
@@ -154,6 +157,7 @@ The `armenian` analyzer could be reimplemented as a `custom` analyzer as follows
   }
 }
 ----------------------------------------------------
+// CONSOLE
 <1> The default stopwords can be overridden with the `stopwords`
     or `stopwords_path` parameters.
 <2> This filter should be removed unless there are words which should
@@ -166,6 +170,7 @@ The `basque` analyzer could be reimplemented as a `custom` analyzer as follows:
 
 [source,js]
 ----------------------------------------------------
+PUT /armenian_example
 {
   "settings": {
     "analysis": {
@@ -176,7 +181,7 @@ The `basque` analyzer could be reimplemented as a `custom` analyzer as follows:
         },
         "basque_keywords": {
           "type":       "keyword_marker",
-          "keywords":   [] <2>
+          "keywords":   ["Adibidez"] <2>
         },
         "basque_stemmer": {
           "type":       "stemmer",
@@ -198,6 +203,7 @@ The `basque` analyzer could be reimplemented as a `custom` analyzer as follows:
   }
 }
 ----------------------------------------------------
+// CONSOLE
 <1> The default stopwords can be overridden with the `stopwords`
     or `stopwords_path` parameters.
 <2> This filter should be removed unless there are words which should
@@ -210,6 +216,7 @@ The `brazilian` analyzer could be reimplemented as a `custom` analyzer as follow
 
 [source,js]
 ----------------------------------------------------
+PUT /brazilian_example
 {
   "settings": {
     "analysis": {
@@ -220,7 +227,7 @@ The `brazilian` analyzer could be reimplemented as a `custom` analyzer as follow
         },
         "brazilian_keywords": {
           "type":       "keyword_marker",
-          "keywords":   [] <2>
+          "keywords":   ["exemplo"] <2>
         },
         "brazilian_stemmer": {
           "type":       "stemmer",
@@ -242,6 +249,7 @@ The `brazilian` analyzer could be reimplemented as a `custom` analyzer as follow
   }
 }
 ----------------------------------------------------
+// CONSOLE
 <1> The default stopwords can be overridden with the `stopwords`
     or `stopwords_path` parameters.
 <2> This filter should be removed unless there are words which should
@@ -254,6 +262,7 @@ The `bulgarian` analyzer could be reimplemented as a `custom` analyzer as follow
 
 [source,js]
 ----------------------------------------------------
+PUT /bulgarian_example
 {
   "settings": {
     "analysis": {
@@ -264,7 +273,7 @@ The `bulgarian` analyzer could be reimplemented as a `custom` analyzer as follow
         },
         "bulgarian_keywords": {
           "type":       "keyword_marker",
-          "keywords":   [] <2>
+          "keywords":   ["пример"] <2>
         },
         "bulgarian_stemmer": {
           "type":       "stemmer",
@@ -286,6 +295,7 @@ The `bulgarian` analyzer could be reimplemented as a `custom` analyzer as follow
   }
 }
 ----------------------------------------------------
+// CONSOLE
 <1> The default stopwords can be overridden with the `stopwords`
     or `stopwords_path` parameters.
 <2> This filter should be removed unless there are words which should
@@ -298,6 +308,7 @@ The `catalan` analyzer could be reimplemented as a `custom` analyzer as follows:
 
 [source,js]
 ----------------------------------------------------
+PUT /catalan_example
 {
   "settings": {
     "analysis": {
@@ -312,7 +323,7 @@ The `catalan` analyzer could be reimplemented as a `custom` analyzer as follows:
         },
         "catalan_keywords": {
           "type":       "keyword_marker",
-          "keywords":   [] <2>
+          "keywords":   ["exemple"] <2>
         },
         "catalan_stemmer": {
           "type":       "stemmer",
@@ -335,6 +346,7 @@ The `catalan` analyzer could be reimplemented as a `custom` analyzer as follows:
   }
 }
 ----------------------------------------------------
+// CONSOLE
 <1> The default stopwords can be overridden with the `stopwords`
     or `stopwords_path` parameters.
 <2> This filter should be removed unless there are words which should
@@ -347,6 +359,7 @@ The `cjk` analyzer could be reimplemented as a `custom` analyzer as follows:
 
 [source,js]
 ----------------------------------------------------
+PUT /cjk_example
 {
   "settings": {
     "analysis": {
@@ -371,6 +384,7 @@ The `cjk` analyzer could be reimplemented as a `custom` analyzer as follows:
   }
 }
 ----------------------------------------------------
+// CONSOLE
 <1> The default stopwords can be overridden with the `stopwords`
     or `stopwords_path` parameters.
 
@@ -381,6 +395,7 @@ The `czech` analyzer could be reimplemented as a `custom` analyzer as follows:
 
 [source,js]
 ----------------------------------------------------
+PUT /czech_example
 {
   "settings": {
     "analysis": {
@@ -391,7 +406,7 @@ The `czech` analyzer could be reimplemented as a `custom` analyzer as follows:
         },
         "czech_keywords": {
           "type":       "keyword_marker",
-          "keywords":   [] <2>
+          "keywords":   ["příklad"] <2>
         },
         "czech_stemmer": {
           "type":       "stemmer",
@@ -413,6 +428,7 @@ The `czech` analyzer could be reimplemented as a `custom` analyzer as follows:
   }
 }
 ----------------------------------------------------
+// CONSOLE
 <1> The default stopwords can be overridden with the `stopwords`
     or `stopwords_path` parameters.
 <2> This filter should be removed unless there are words which should
@@ -425,6 +441,7 @@ The `danish` analyzer could be reimplemented as a `custom` analyzer as follows:
 
 [source,js]
 ----------------------------------------------------
+PUT /danish_example
 {
   "settings": {
     "analysis": {
@@ -435,7 +452,7 @@ The `danish` analyzer could be reimplemented as a `custom` analyzer as follows:
         },
         "danish_keywords": {
           "type":       "keyword_marker",
-          "keywords":   [] <2>
+          "keywords":   ["eksempel"] <2>
         },
         "danish_stemmer": {
           "type":       "stemmer",
@@ -457,6 +474,7 @@ The `danish` analyzer could be reimplemented as a `custom` analyzer as follows:
   }
 }
 ----------------------------------------------------
+// CONSOLE
 <1> The default stopwords can be overridden with the `stopwords`
     or `stopwords_path` parameters.
 <2> This filter should be removed unless there are words which should
@@ -469,6 +487,7 @@ The `dutch` analyzer could be reimplemented as a `custom` analyzer as follows:
 
 [source,js]
 ----------------------------------------------------
+PUT /detch_example
 {
   "settings": {
     "analysis": {
@@ -479,7 +498,7 @@ The `dutch` analyzer could be reimplemented as a `custom` analyzer as follows:
         },
         "dutch_keywords": {
           "type":       "keyword_marker",
-          "keywords":   [] <2>
+          "keywords":   ["voorbeeld"] <2>
         },
         "dutch_stemmer": {
           "type":       "stemmer",
@@ -511,6 +530,7 @@ The `dutch` analyzer could be reimplemented as a `custom` analyzer as follows:
   }
 }
 ----------------------------------------------------
+// CONSOLE
 <1> The default stopwords can be overridden with the `stopwords`
     or `stopwords_path` parameters.
 <2> This filter should be removed unless there are words which should
@@ -523,6 +543,7 @@ The `english` analyzer could be reimplemented as a `custom` analyzer as follows:
 
 [source,js]
 ----------------------------------------------------
+PUT /english_example
 {
   "settings": {
     "analysis": {
@@ -533,7 +554,7 @@ The `english` analyzer could be reimplemented as a `custom` analyzer as follows:
         },
         "english_keywords": {
           "type":       "keyword_marker",
-          "keywords":   [] <2>
+          "keywords":   ["example"] <2>
         },
         "english_stemmer": {
           "type":       "stemmer",
@@ -560,6 +581,7 @@ The `english` analyzer could be reimplemented as a `custom` analyzer as follows:
   }
 }
 ----------------------------------------------------
+// CONSOLE
 <1> The default stopwords can be overridden with the `stopwords`
     or `stopwords_path` parameters.
 <2> This filter should be removed unless there are words which should
@@ -572,6 +594,7 @@ The `finnish` analyzer could be reimplemented as a `custom` analyzer as follows:
 
 [source,js]
 ----------------------------------------------------
+PUT /finnish_example
 {
   "settings": {
     "analysis": {
@@ -582,7 +605,7 @@ The `finnish` analyzer could be reimplemented as a `custom` analyzer as follows:
         },
         "finnish_keywords": {
           "type":       "keyword_marker",
-          "keywords":   [] <2>
+          "keywords":   ["esimerkki"] <2>
         },
         "finnish_stemmer": {
           "type":       "stemmer",
@@ -604,6 +627,7 @@ The `finnish` analyzer could be reimplemented as a `custom` analyzer as follows:
   }
 }
 ----------------------------------------------------
+// CONSOLE
 <1> The default stopwords can be overridden with the `stopwords`
     or `stopwords_path` parameters.
 <2> This filter should be removed unless there are words which should
@@ -616,6 +640,7 @@ The `french` analyzer could be reimplemented as a `custom` analyzer as follows:
 
 [source,js]
 ----------------------------------------------------
+PUT /french_example
 {
   "settings": {
     "analysis": {
@@ -635,7 +660,7 @@ The `french` analyzer could be reimplemented as a `custom` analyzer as follows:
         },
         "french_keywords": {
           "type":       "keyword_marker",
-          "keywords":   [] <2>
+          "keywords":   ["Exemple"] <2>
         },
         "french_stemmer": {
           "type":       "stemmer",
@@ -658,6 +683,7 @@ The `french` analyzer could be reimplemented as a `custom` analyzer as follows:
   }
 }
 ----------------------------------------------------
+// CONSOLE
 <1> The default stopwords can be overridden with the `stopwords`
     or `stopwords_path` parameters.
 <2> This filter should be removed unless there are words which should
@@ -670,6 +696,7 @@ The `galician` analyzer could be reimplemented as a `custom` analyzer as follows
 
 [source,js]
 ----------------------------------------------------
+PUT /galician_example
 {
   "settings": {
     "analysis": {
@@ -680,7 +707,7 @@ The `galician` analyzer could be reimplemented as a `custom` analyzer as follows
         },
         "galician_keywords": {
           "type":       "keyword_marker",
-          "keywords":   [] <2>
+          "keywords":   ["exemplo"] <2>
         },
         "galician_stemmer": {
           "type":       "stemmer",
@@ -702,6 +729,7 @@ The `galician` analyzer could be reimplemented as a `custom` analyzer as follows
   }
 }
 ----------------------------------------------------
+// CONSOLE
 <1> The default stopwords can be overridden with the `stopwords`
     or `stopwords_path` parameters.
 <2> This filter should be removed unless there are words which should
@@ -714,6 +742,7 @@ The `german` analyzer could be reimplemented as a `custom` analyzer as follows:
 
 [source,js]
 ----------------------------------------------------
+PUT /german_example
 {
   "settings": {
     "analysis": {
@@ -724,7 +753,7 @@ The `german` analyzer could be reimplemented as a `custom` analyzer as follows:
         },
         "german_keywords": {
           "type":       "keyword_marker",
-          "keywords":   [] <2>
+          "keywords":   ["Beispiel"] <2>
         },
         "german_stemmer": {
           "type":       "stemmer",
@@ -747,6 +776,7 @@ The `german` analyzer could be reimplemented as a `custom` analyzer as follows:
   }
 }
 ----------------------------------------------------
+// CONSOLE
 <1> The default stopwords can be overridden with the `stopwords`
     or `stopwords_path` parameters.
 <2> This filter should be removed unless there are words which should
@@ -759,6 +789,7 @@ The `greek` analyzer could be reimplemented as a `custom` analyzer as follows:
 
 [source,js]
 ----------------------------------------------------
+PUT /greek_example
 {
   "settings": {
     "analysis": {
@@ -773,7 +804,7 @@ The `greek` analyzer could be reimplemented as a `custom` analyzer as follows:
         },
         "greek_keywords": {
           "type":       "keyword_marker",
-          "keywords":   [] <2>
+          "keywords":   ["παράδειγμα"] <2>
         },
         "greek_stemmer": {
           "type":       "stemmer",
@@ -795,6 +826,7 @@ The `greek` analyzer could be reimplemented as a `custom` analyzer as follows:
   }
 }
 ----------------------------------------------------
+// CONSOLE
 <1> The default stopwords can be overridden with the `stopwords`
     or `stopwords_path` parameters.
 <2> This filter should be removed unless there are words which should
@@ -807,6 +839,7 @@ The `hindi` analyzer could be reimplemented as a `custom` analyzer as follows:
 
 [source,js]
 ----------------------------------------------------
+PUT /hindi_example
 {
   "settings": {
     "analysis": {
@@ -817,7 +850,7 @@ The `hindi` analyzer could be reimplemented as a `custom` analyzer as follows:
         },
         "hindi_keywords": {
           "type":       "keyword_marker",
-          "keywords":   [] <2>
+          "keywords":   ["उदाहरण"] <2>
         },
         "hindi_stemmer": {
           "type":       "stemmer",
@@ -841,6 +874,7 @@ The `hindi` analyzer could be reimplemented as a `custom` analyzer as follows:
   }
 }
 ----------------------------------------------------
+// CONSOLE
 <1> The default stopwords can be overridden with the `stopwords`
     or `stopwords_path` parameters.
 <2> This filter should be removed unless there are words which should
@@ -853,6 +887,7 @@ The `hungarian` analyzer could be reimplemented as a `custom` analyzer as follow
 
 [source,js]
 ----------------------------------------------------
+PUT /hungarian_example
 {
   "settings": {
     "analysis": {
@@ -863,7 +898,7 @@ The `hungarian` analyzer could be reimplemented as a `custom` analyzer as follow
         },
         "hungarian_keywords": {
           "type":       "keyword_marker",
-          "keywords":   [] <2>
+          "keywords":   ["példa"] <2>
         },
         "hungarian_stemmer": {
           "type":       "stemmer",
@@ -885,6 +920,7 @@ The `hungarian` analyzer could be reimplemented as a `custom` analyzer as follow
   }
 }
 ----------------------------------------------------
+// CONSOLE
 <1> The default stopwords can be overridden with the `stopwords`
     or `stopwords_path` parameters.
 <2> This filter should be removed unless there are words which should
@@ -898,6 +934,7 @@ The `indonesian` analyzer could be reimplemented as a `custom` analyzer as follo
 
 [source,js]
 ----------------------------------------------------
+PUT /indonesian_example
 {
   "settings": {
     "analysis": {
@@ -908,7 +945,7 @@ The `indonesian` analyzer could be reimplemented as a `custom` analyzer as follo
         },
         "indonesian_keywords": {
           "type":       "keyword_marker",
-          "keywords":   [] <2>
+          "keywords":   ["contoh"] <2>
         },
         "indonesian_stemmer": {
           "type":       "stemmer",
@@ -930,6 +967,7 @@ The `indonesian` analyzer could be reimplemented as a `custom` analyzer as follo
   }
 }
 ----------------------------------------------------
+// CONSOLE
 <1> The default stopwords can be overridden with the `stopwords`
     or `stopwords_path` parameters.
 <2> This filter should be removed unless there are words which should
@@ -942,6 +980,7 @@ The `irish` analyzer could be reimplemented as a `custom` analyzer as follows:
 
 [source,js]
 ----------------------------------------------------
+PUT /irish_example
 {
   "settings": {
     "analysis": {
@@ -960,7 +999,7 @@ The `irish` analyzer could be reimplemented as a `custom` analyzer as follows:
         },
         "irish_keywords": {
           "type":       "keyword_marker",
-          "keywords":   [] <2>
+          "keywords":   ["sampla"] <2>
         },
         "irish_stemmer": {
           "type":       "stemmer",
@@ -983,6 +1022,7 @@ The `irish` analyzer could be reimplemented as a `custom` analyzer as follows:
   }
 }
 ----------------------------------------------------
+// CONSOLE
 <1> The default stopwords can be overridden with the `stopwords`
     or `stopwords_path` parameters.
 <2> This filter should be removed unless there are words which should
@@ -995,6 +1035,7 @@ The `italian` analyzer could be reimplemented as a `custom` analyzer as follows:
 
 [source,js]
 ----------------------------------------------------
+PUT /italian_example
 {
   "settings": {
     "analysis": {
@@ -1014,7 +1055,7 @@ The `italian` analyzer could be reimplemented as a `custom` analyzer as follows:
         },
         "italian_keywords": {
           "type":       "keyword_marker",
-          "keywords":   [] <2>
+          "keywords":   ["esempio"] <2>
         },
         "italian_stemmer": {
           "type":       "stemmer",
@@ -1037,6 +1078,7 @@ The `italian` analyzer could be reimplemented as a `custom` analyzer as follows:
   }
 }
 ----------------------------------------------------
+// CONSOLE
 <1> The default stopwords can be overridden with the `stopwords`
     or `stopwords_path` parameters.
 <2> This filter should be removed unless there are words which should
@@ -1049,6 +1091,7 @@ The `latvian` analyzer could be reimplemented as a `custom` analyzer as follows:
 
 [source,js]
 ----------------------------------------------------
+PUT /latvian_example
 {
   "settings": {
     "analysis": {
@@ -1059,7 +1102,7 @@ The `latvian` analyzer could be reimplemented as a `custom` analyzer as follows:
         },
         "latvian_keywords": {
           "type":       "keyword_marker",
-          "keywords":   [] <2>
+          "keywords":   ["piemērs"] <2>
         },
         "latvian_stemmer": {
           "type":       "stemmer",
@@ -1081,6 +1124,7 @@ The `latvian` analyzer could be reimplemented as a `custom` analyzer as follows:
   }
 }
 ----------------------------------------------------
+// CONSOLE
 <1> The default stopwords can be overridden with the `stopwords`
     or `stopwords_path` parameters.
 <2> This filter should be removed unless there are words which should
@@ -1093,6 +1137,7 @@ The `lithuanian` analyzer could be reimplemented as a `custom` analyzer as follo
 
 [source,js]
 ----------------------------------------------------
+PUT /lithuanian_example
 {
   "settings": {
     "analysis": {
@@ -1103,7 +1148,7 @@ The `lithuanian` analyzer could be reimplemented as a `custom` analyzer as follo
         },
         "lithuanian_keywords": {
           "type":       "keyword_marker",
-          "keywords":   [] <2>
+          "keywords":   ["pavyzdys"] <2>
         },
         "lithuanian_stemmer": {
           "type":       "stemmer",
@@ -1125,6 +1170,7 @@ The `lithuanian` analyzer could be reimplemented as a `custom` analyzer as follo
   }
 }
 ----------------------------------------------------
+// CONSOLE
 <1> The default stopwords can be overridden with the `stopwords`
     or `stopwords_path` parameters.
 <2> This filter should be removed unless there are words which should
@@ -1137,6 +1183,7 @@ The `norwegian` analyzer could be reimplemented as a `custom` analyzer as follow
 
 [source,js]
 ----------------------------------------------------
+PUT /norwegian_example
 {
   "settings": {
     "analysis": {
@@ -1147,7 +1194,7 @@ The `norwegian` analyzer could be reimplemented as a `custom` analyzer as follow
         },
         "norwegian_keywords": {
           "type":       "keyword_marker",
-          "keywords":   [] <2>
+          "keywords":   ["eksempel"] <2>
         },
         "norwegian_stemmer": {
           "type":       "stemmer",
@@ -1169,6 +1216,7 @@ The `norwegian` analyzer could be reimplemented as a `custom` analyzer as follow
   }
 }
 ----------------------------------------------------
+// CONSOLE
 <1> The default stopwords can be overridden with the `stopwords`
     or `stopwords_path` parameters.
 <2> This filter should be removed unless there are words which should
@@ -1181,6 +1229,7 @@ The `persian` analyzer could be reimplemented as a `custom` analyzer as follows:
 
 [source,js]
 ----------------------------------------------------
+PUT /persian_example
 {
   "settings": {
     "analysis": {
@@ -1212,6 +1261,7 @@ The `persian` analyzer could be reimplemented as a `custom` analyzer as follows:
   }
 }
 ----------------------------------------------------
+// CONSOLE
 <1> Replaces zero-width non-joiners with an ASCII space.
 <2> The default stopwords can be overridden with the `stopwords`
     or `stopwords_path` parameters.
@@ -1223,6 +1273,7 @@ The `portuguese` analyzer could be reimplemented as a `custom` analyzer as follo
 
 [source,js]
 ----------------------------------------------------
+PUT /portuguese_example
 {
   "settings": {
     "analysis": {
@@ -1233,7 +1284,7 @@ The `portuguese` analyzer could be reimplemented as a `custom` analyzer as follo
         },
         "portuguese_keywords": {
           "type":       "keyword_marker",
-          "keywords":   [] <2>
+          "keywords":   ["exemplo"] <2>
         },
         "portuguese_stemmer": {
           "type":       "stemmer",
@@ -1255,6 +1306,7 @@ The `portuguese` analyzer could be reimplemented as a `custom` analyzer as follo
   }
 }
 ----------------------------------------------------
+// CONSOLE
 <1> The default stopwords can be overridden with the `stopwords`
     or `stopwords_path` parameters.
 <2> This filter should be removed unless there are words which should
@@ -1267,6 +1319,7 @@ The `romanian` analyzer could be reimplemented as a `custom` analyzer as follows
 
 [source,js]
 ----------------------------------------------------
+PUT /romanian_example
 {
   "settings": {
     "analysis": {
@@ -1277,7 +1330,7 @@ The `romanian` analyzer could be reimplemented as a `custom` analyzer as follows
         },
         "romanian_keywords": {
           "type":       "keyword_marker",
-          "keywords":   [] <2>
+          "keywords":   ["exemplu"] <2>
         },
         "romanian_stemmer": {
           "type":       "stemmer",
@@ -1299,6 +1352,7 @@ The `romanian` analyzer could be reimplemented as a `custom` analyzer as follows
   }
 }
 ----------------------------------------------------
+// CONSOLE
 <1> The default stopwords can be overridden with the `stopwords`
     or `stopwords_path` parameters.
 <2> This filter should be removed unless there are words which should
@@ -1312,6 +1366,7 @@ The `russian` analyzer could be reimplemented as a `custom` analyzer as follows:
 
 [source,js]
 ----------------------------------------------------
+PUT /russian_example
 {
   "settings": {
     "analysis": {
@@ -1322,7 +1377,7 @@ The `russian` analyzer could be reimplemented as a `custom` analyzer as follows:
         },
         "russian_keywords": {
           "type":       "keyword_marker",
-          "keywords":   [] <2>
+          "keywords":   ["пример"] <2>
         },
         "russian_stemmer": {
           "type":       "stemmer",
@@ -1344,6 +1399,7 @@ The `russian` analyzer could be reimplemented as a `custom` analyzer as follows:
   }
 }
 ----------------------------------------------------
+// CONSOLE
 <1> The default stopwords can be overridden with the `stopwords`
     or `stopwords_path` parameters.
 <2> This filter should be removed unless there are words which should
@@ -1356,6 +1412,7 @@ The `sorani` analyzer could be reimplemented as a `custom` analyzer as follows:
 
 [source,js]
 ----------------------------------------------------
+PUT /sorani_example
 {
   "settings": {
     "analysis": {
@@ -1366,7 +1423,7 @@ The `sorani` analyzer could be reimplemented as a `custom` analyzer as follows:
         },
         "sorani_keywords": {
           "type":       "keyword_marker",
-          "keywords":   [] <2>
+          "keywords":   ["mînak"] <2>
         },
         "sorani_stemmer": {
           "type":       "stemmer",
@@ -1389,6 +1446,7 @@ The `sorani` analyzer could be reimplemented as a `custom` analyzer as follows:
   }
 }
 ----------------------------------------------------
+// CONSOLE
 <1> The default stopwords can be overridden with the `stopwords`
     or `stopwords_path` parameters.
 <2> This filter should be removed unless there are words which should
@@ -1401,6 +1459,7 @@ The `spanish` analyzer could be reimplemented as a `custom` analyzer as follows:
 
 [source,js]
 ----------------------------------------------------
+PUT /spanish_example
 {
   "settings": {
     "analysis": {
@@ -1411,7 +1470,7 @@ The `spanish` analyzer could be reimplemented as a `custom` analyzer as follows:
         },
         "spanish_keywords": {
           "type":       "keyword_marker",
-          "keywords":   [] <2>
+          "keywords":   ["ejemplo"] <2>
         },
         "spanish_stemmer": {
           "type":       "stemmer",
@@ -1433,6 +1492,7 @@ The `spanish` analyzer could be reimplemented as a `custom` analyzer as follows:
   }
 }
 ----------------------------------------------------
+// CONSOLE
 <1> The default stopwords can be overridden with the `stopwords`
     or `stopwords_path` parameters.
 <2> This filter should be removed unless there are words which should
@@ -1445,6 +1505,7 @@ The `swedish` analyzer could be reimplemented as a `custom` analyzer as follows:
 
 [source,js]
 ----------------------------------------------------
+PUT /swidish_example
 {
   "settings": {
     "analysis": {
@@ -1455,7 +1516,7 @@ The `swedish` analyzer could be reimplemented as a `custom` analyzer as follows:
         },
         "swedish_keywords": {
           "type":       "keyword_marker",
-          "keywords":   [] <2>
+          "keywords":   ["exempel"] <2>
         },
         "swedish_stemmer": {
           "type":       "stemmer",
@@ -1477,6 +1538,7 @@ The `swedish` analyzer could be reimplemented as a `custom` analyzer as follows:
   }
 }
 ----------------------------------------------------
+// CONSOLE
 <1> The default stopwords can be overridden with the `stopwords`
     or `stopwords_path` parameters.
 <2> This filter should be removed unless there are words which should
@@ -1489,6 +1551,7 @@ The `turkish` analyzer could be reimplemented as a `custom` analyzer as follows:
 
 [source,js]
 ----------------------------------------------------
+PUT /turkish_example
 {
   "settings": {
     "analysis": {
@@ -1503,7 +1566,7 @@ The `turkish` analyzer could be reimplemented as a `custom` analyzer as follows:
         },
         "turkish_keywords": {
           "type":       "keyword_marker",
-          "keywords":   [] <2>
+          "keywords":   ["örnek"] <2>
         },
         "turkish_stemmer": {
           "type":       "stemmer",
@@ -1526,6 +1589,7 @@ The `turkish` analyzer could be reimplemented as a `custom` analyzer as follows:
   }
 }
 ----------------------------------------------------
+// CONSOLE
 <1> The default stopwords can be overridden with the `stopwords`
     or `stopwords_path` parameters.
 <2> This filter should be removed unless there are words which should
@@ -1538,6 +1602,7 @@ The `thai` analyzer could be reimplemented as a `custom` analyzer as follows:
 
 [source,js]
 ----------------------------------------------------
+PUT /thai_example
 {
   "settings": {
     "analysis": {
@@ -1560,5 +1625,6 @@ The `thai` analyzer could be reimplemented as a `custom` analyzer as follows:
   }
 }
 ----------------------------------------------------
+// CONSOLE
 <1> The default stopwords can be overridden with the `stopwords`
     or `stopwords_path` parameters.
diff --git a/docs/reference/analysis/analyzers/pattern-analyzer.asciidoc b/docs/reference/analysis/analyzers/pattern-analyzer.asciidoc
index 36901de0d15df..64ab3999ef9a9 100644
--- a/docs/reference/analysis/analyzers/pattern-analyzer.asciidoc
+++ b/docs/reference/analysis/analyzers/pattern-analyzer.asciidoc
@@ -5,6 +5,21 @@ The `pattern` analyzer uses a regular expression to split the text into terms.
 The regular expression should match the *token separators*  not the tokens
 themselves. The regular expression defaults to `\W+` (or all non-word characters).
 
+[WARNING]
+.Beware of Pathological Regular Expressions
+========================================
+
+The pattern analyzer uses
+http://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html[Java Regular Expressions].
+
+A badly written regular expression could run very slowly or even throw a
+StackOverflowError and cause the node it is running on to exit suddenly.
+
+Read more about http://www.regular-expressions.info/catastrophic.html[pathological regular expressions and how to avoid them].
+
+========================================
+
+
 [float]
 === Definition
 
@@ -154,11 +169,6 @@ The `pattern` analyzer accepts the following parameters:
 
     Should terms be lowercased or not. Defaults to `true`.
 
-`max_token_length`::
-
-    The maximum token length. If a token is seen that exceeds this length then
-    it is split at `max_token_length` intervals. Defaults to `255`.
-
 `stopwords`::
 
     A pre-defined stop words list like `_english_` or an array  containing a
@@ -356,9 +366,8 @@ The above example produces the following terms:
 
 The regex above is easier to understand as:
 
-[source,js]
+[source,regex]
 --------------------------------------------------
-
   ([^\p{L}\d]+)                 # swallow non letters and numbers,
 | (?<=\D)(?=\d)                 # or non-number followed by number,
 | (?<=\d)(?=\D)                 # or number followed by non-number,
diff --git a/docs/reference/analysis/analyzers/stop-analyzer.asciidoc b/docs/reference/analysis/analyzers/stop-analyzer.asciidoc
index 31cb07ce70606..eacc7e106e799 100644
--- a/docs/reference/analysis/analyzers/stop-analyzer.asciidoc
+++ b/docs/reference/analysis/analyzers/stop-analyzer.asciidoc
@@ -126,7 +126,8 @@ The `stop` analyzer accepts the following parameters:
 
 `stopwords_path`::
 
-    The path to a file containing stop words.
+    The path to a file containing stop words. This path is relative to the
+    Elasticsearch `config` directory.
 
 
 See the <<analysis-stop-tokenfilter,Stop Token Filter>> for more information
diff --git a/docs/reference/analysis/anatomy.asciidoc b/docs/reference/analysis/anatomy.asciidoc
index 87739f80f60d8..729e0f7a27ef8 100644
--- a/docs/reference/analysis/anatomy.asciidoc
+++ b/docs/reference/analysis/anatomy.asciidoc
@@ -15,8 +15,8 @@ combined to define new <<analysis-custom-analyzer,`custom`>> analyzers.
 
 A _character filter_ receives the original text as a stream of characters and
 can transform the stream by adding, removing, or changing characters.  For
-instance, a character filter could be used to convert Arabic numerals
-(٠‎١٢٣٤٥٦٧٨‎٩‎) into their Latin equivalents (0123456789), or to strip HTML
+instance, a character filter could be used to convert Hindu-Arabic numerals
+(٠‎١٢٣٤٥٦٧٨‎٩‎) into their Arabic-Latin equivalents (0123456789), or to strip HTML
 elements like `<b>` from the stream.
 
 An analyzer may have *zero or more* <<analysis-charfilters,character filters>>,
diff --git a/docs/reference/analysis/charfilters.asciidoc b/docs/reference/analysis/charfilters.asciidoc
index cd24f5bf57176..47390fe0ae837 100644
--- a/docs/reference/analysis/charfilters.asciidoc
+++ b/docs/reference/analysis/charfilters.asciidoc
@@ -6,8 +6,8 @@ is passed to the <<analysis-tokenizers,tokenizer>>.
 
 A character filter receives the original text as a stream of characters and
 can transform the stream by adding, removing, or changing characters.  For
-instance, a character filter could be used to convert Arabic numerals
-(٠‎١٢٣٤٥٦٧٨‎٩‎) into their Latin equivalents (0123456789), or to strip HTML
+instance, a character filter could be used to convert Hindu-Arabic numerals
+(٠‎١٢٣٤٥٦٧٨‎٩‎) into their Arabic-Latin equivalents (0123456789), or to strip HTML
 elements like `<b>` from the stream.
 
 
diff --git a/docs/reference/analysis/charfilters/htmlstrip-charfilter.asciidoc b/docs/reference/analysis/charfilters/htmlstrip-charfilter.asciidoc
index 217b618c9c227..6c1a1875d67ca 100644
--- a/docs/reference/analysis/charfilters/htmlstrip-charfilter.asciidoc
+++ b/docs/reference/analysis/charfilters/htmlstrip-charfilter.asciidoc
@@ -43,14 +43,14 @@ POST _analyze
 
 The above example returns the term:
 
-[source,js]
+[source,text]
 ---------------------------
 [ \nI'm so happy!\n ]
 ---------------------------
 
 The same example with the `standard` tokenizer would return the following terms:
 
-[source,js]
+[source,text]
 ---------------------------
 [ I'm, so, happy ]
 ---------------------------
diff --git a/docs/reference/analysis/charfilters/pattern-replace-charfilter.asciidoc b/docs/reference/analysis/charfilters/pattern-replace-charfilter.asciidoc
index 3f4bf9aa05adf..f3e7ae0f7a9dc 100644
--- a/docs/reference/analysis/charfilters/pattern-replace-charfilter.asciidoc
+++ b/docs/reference/analysis/charfilters/pattern-replace-charfilter.asciidoc
@@ -5,6 +5,20 @@ The `pattern_replace` character filter uses a regular expression to match
 characters which should be replaced with the specified replacement string.
 The replacement string can refer to capture groups in the regular expression.
 
+[WARNING]
+.Beware of Pathological Regular Expressions
+========================================
+
+The pattern replace character filter uses
+http://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html[Java Regular Expressions].
+
+A badly written regular expression could run very slowly or even throw a
+StackOverflowError and cause the node it is running on to exit suddenly.
+
+Read more about http://www.regular-expressions.info/catastrophic.html[pathological regular expressions and how to avoid them].
+
+========================================
+
 [float]
 === Configuration
 
@@ -65,7 +79,9 @@ POST my_index/_analyze
 }
 ----------------------------
 // CONSOLE
-// TEST[skip:Test interprets $1 as a stashed variable]
+// TEST[s/\$1//]
+// the test framework doesn't like the $1 so we just throw it away rather than
+// try to get it to work properly. At least we are still testing the charfilter.
 
 The above example produces the following term:
 
@@ -74,7 +90,6 @@ The above example produces the following term:
 [ My, credit, card, is 123_456_789 ]
 ---------------------------
 
-
 WARNING: Using a replacement string that changes the length of the original
 text will work for search purposes, but will result in incorrect highlighting,
 as can be seen in the following example.
@@ -179,7 +194,7 @@ POST my_index/_analyze
 
 The above returns the following terms:
 
-[source,js]
+[source,text]
 ----------------------------
 [ the, foo, bar, baz, method ]
 ----------------------------
@@ -222,6 +237,7 @@ The output from the above is:
   "_shards": {
     "total": 5,
     "successful": 5,
+    "skipped" : 0,
     "failed": 0
   },
   "hits": {
diff --git a/docs/reference/analysis/normalizers.asciidoc b/docs/reference/analysis/normalizers.asciidoc
new file mode 100644
index 0000000000000..313f26b7c9515
--- /dev/null
+++ b/docs/reference/analysis/normalizers.asciidoc
@@ -0,0 +1,57 @@
+[[analysis-normalizers]]
+== Normalizers
+
+experimental[]
+
+Normalizers are similar to analyzers except that they may only emit a single
+token. As a consequence, they do not have a tokenizer and only accept a subset
+of the available char filters and token filters. Only the filters that work on
+a per-character basis are allowed. For instance a lowercasing filter would be
+allowed, but not a stemming filter, which needs to look at the keyword as a
+whole.
+
+[float]
+=== Custom normalizers
+
+Elasticsearch does not ship with built-in normalizers so far, so the only way
+to get one is by building a custom one. Custom normalizers take a list of char
+<<analysis-charfilters, character filters>> and a list of
+<<analysis-tokenfilters,token filters>>.
+
+[source,js]
+--------------------------------
+PUT index
+{
+  "settings": {
+    "analysis": {
+      "char_filter": {
+        "quote": {
+          "type": "mapping",
+          "mappings": [
+            "« => \"",
+            "» => \""
+          ]
+        }
+      },
+      "normalizer": {
+        "my_normalizer": {
+          "type": "custom",
+          "char_filter": ["quote"],
+          "filter": ["lowercase", "asciifolding"]
+        }
+      }
+    }
+  },
+  "mappings": {
+    "type": {
+      "properties": {
+        "foo": {
+          "type": "keyword",
+          "normalizer": "my_normalizer"
+        }
+      }
+    }
+  }
+}
+--------------------------------
+// CONSOLE
diff --git a/docs/reference/analysis/tokenfilters.asciidoc b/docs/reference/analysis/tokenfilters.asciidoc
index 89cce11a615c1..6e77b4498650d 100644
--- a/docs/reference/analysis/tokenfilters.asciidoc
+++ b/docs/reference/analysis/tokenfilters.asciidoc
@@ -13,6 +13,8 @@ include::tokenfilters/standard-tokenfilter.asciidoc[]
 
 include::tokenfilters/asciifolding-tokenfilter.asciidoc[]
 
+include::tokenfilters/flatten-graph-tokenfilter.asciidoc[]
+
 include::tokenfilters/length-tokenfilter.asciidoc[]
 
 include::tokenfilters/lowercase-tokenfilter.asciidoc[]
@@ -31,6 +33,8 @@ include::tokenfilters/stop-tokenfilter.asciidoc[]
 
 include::tokenfilters/word-delimiter-tokenfilter.asciidoc[]
 
+include::tokenfilters/word-delimiter-graph-tokenfilter.asciidoc[]
+
 include::tokenfilters/stemmer-tokenfilter.asciidoc[]
 
 include::tokenfilters/stemmer-override-tokenfilter.asciidoc[]
@@ -47,6 +51,8 @@ include::tokenfilters/phonetic-tokenfilter.asciidoc[]
 
 include::tokenfilters/synonym-tokenfilter.asciidoc[]
 
+include::tokenfilters/synonym-graph-tokenfilter.asciidoc[]
+
 include::tokenfilters/compound-word-tokenfilter.asciidoc[]
 
 include::tokenfilters/reverse-tokenfilter.asciidoc[]
diff --git a/docs/reference/analysis/tokenfilters/asciifolding-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/asciifolding-tokenfilter.asciidoc
index 68891c18e2365..73d35549da8b6 100644
--- a/docs/reference/analysis/tokenfilters/asciifolding-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/asciifolding-tokenfilter.asciidoc
@@ -8,17 +8,21 @@ equivalents, if one exists.  Example:
 
 [source,js]
 --------------------------------------------------
-"index" : {
-    "analysis" : {
-        "analyzer" : {
-            "default" : {
-                "tokenizer" : "standard",
-                "filter" : ["standard", "asciifolding"]
+PUT /asciifold_example
+{
+    "settings" : {
+        "analysis" : {
+            "analyzer" : {
+                "default" : {
+                    "tokenizer" : "standard",
+                    "filter" : ["standard", "asciifolding"]
+                }
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 Accepts `preserve_original` setting which defaults to false but if true
 will keep the original token as well as emit the folded token.  For
@@ -26,20 +30,24 @@ example:
 
 [source,js]
 --------------------------------------------------
-"index" : {
-    "analysis" : {
-        "analyzer" : {
-            "default" : {
-                "tokenizer" : "standard",
-                "filter" : ["standard", "my_ascii_folding"]
-            }
-        },
-        "filter" : {
-            "my_ascii_folding" : {
-                "type" : "asciifolding",
-                "preserve_original" : true
+PUT /asciifold_example
+{
+    "settings" : {
+        "analysis" : {
+            "analyzer" : {
+                "default" : {
+                    "tokenizer" : "standard",
+                    "filter" : ["standard", "my_ascii_folding"]
+                }
+            },
+            "filter" : {
+                "my_ascii_folding" : {
+                    "type" : "asciifolding",
+                    "preserve_original" : true
+                }
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
diff --git a/docs/reference/analysis/tokenfilters/cjk-bigram-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/cjk-bigram-tokenfilter.asciidoc
index c1e278b2183b3..cc26d025f04f9 100644
--- a/docs/reference/analysis/tokenfilters/cjk-bigram-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/cjk-bigram-tokenfilter.asciidoc
@@ -16,8 +16,9 @@ Bigrams are generated for characters in `han`, `hiragana`, `katakana` and
 
 [source,js]
 --------------------------------------------------
+PUT /cjk_bigram_example
 {
-    "index" : {
+    "settings" : {
         "analysis" : {
             "analyzer" : {
                 "han_bigrams" : {
@@ -40,3 +41,4 @@ Bigrams are generated for characters in `han`, `hiragana`, `katakana` and
     }
 }
 --------------------------------------------------
+// CONSOLE
diff --git a/docs/reference/analysis/tokenfilters/common-grams-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/common-grams-tokenfilter.asciidoc
index eb1469af80344..c7d8ff660d347 100644
--- a/docs/reference/analysis/tokenfilters/common-grams-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/common-grams-tokenfilter.asciidoc
@@ -41,21 +41,33 @@ Here is an example:
 
 [source,js]
 --------------------------------------------------
-index :
-    analysis :
-        analyzer :
-            index_grams :
-                tokenizer : whitespace
-                filter : [common_grams]
-            search_grams :
-                tokenizer : whitespace
-                filter : [common_grams_query]
-        filter :
-            common_grams :
-                type : common_grams
-                common_words: [a, an, the]                
-            common_grams_query :
-                type : common_grams
-                query_mode: true
-                common_words: [a, an, the]                
+PUT /common_grams_example
+{
+    "settings": {
+        "analysis": {
+            "my_analyzer": {
+                "index_grams": {
+                    "tokenizer": "whitespace",
+                    "filter": ["common_grams"]
+                },
+                "search_grams": {
+                    "tokenizer": "whitespace",
+                    "filter": ["common_grams_query"]
+                }
+            },
+            "filter": {
+                "common_grams": {
+                    "type": "common_grams",
+                    "common_words": ["a", "an", "the"]
+                },
+                "common_grams_query": {
+                    "type": "common_grams",
+                    "query_mode": true,
+                    "common_words": ["a", "an", "the"]
+                }
+            }
+        }
+    }
+}
 --------------------------------------------------
+// CONSOLE
diff --git a/docs/reference/analysis/tokenfilters/compound-word-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/compound-word-tokenfilter.asciidoc
index 1268727b2efd8..e790ed4c4b5b1 100644
--- a/docs/reference/analysis/tokenfilters/compound-word-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/compound-word-tokenfilter.asciidoc
@@ -1,5 +1,5 @@
 [[analysis-compound-word-tokenfilter]]
-=== Compound Word Token Filter
+=== Compound Word Token Filters
 
 The `hyphenation_decompounder` and `dictionary_decompounder` token filters can
 decompose compound words found in many German languages into word parts.
@@ -26,7 +26,7 @@ output tokens is directly connected to the quality of the grammar file you
 use. For languages like German they are quite good.
 
 XML based hyphenation grammar files can be found in the
-http://offo.sourceforge.net/hyphenation/#FOP+XML+Hyphenation+Patterns[Objects For Formatting Objects]
+http://offo.sourceforge.net/#FOP+XML+Hyphenation+Patterns[Objects For Formatting Objects]
 (OFFO) Sourceforge project. Currently only FOP v1.2 compatible hyphenation files
 are supported. You can download https://sourceforge.net/projects/offo/files/offo-hyphenation/1.2/offo-hyphenation_v1.2.zip/download[offo-hyphenation_v1.2.zip]
 directly and look in the `offo-hyphenation/hyph/` directory.
@@ -84,20 +84,31 @@ Here is an example:
 
 [source,js]
 --------------------------------------------------
-index :
-    analysis :
-        analyzer :
-            myAnalyzer2 :
-                type : custom
-                tokenizer : standard
-                filter : [myTokenFilter1, myTokenFilter2]
-        filter :
-            myTokenFilter1 :
-                type : dictionary_decompounder
-                word_list: [one, two, three]
-            myTokenFilter2 :
-                type : hyphenation_decompounder
-                word_list_path: path/to/words.txt
-                hyphenation_patterns_path: path/to/fop.xml
-                max_subword_size : 22
+PUT /compound_word_example
+{
+    "index": {
+        "analysis": {
+            "analyzer": {
+                "my_analyzer": {
+                    "type": "custom",
+                    "tokenizer": "standard",
+                    "filter": ["dictionary_decompounder", "hyphenation_decompounder"]
+                }
+            },
+            "filter": {
+                "dictionary_decompounder": {
+                    "type": "dictionary_decompounder",
+                    "word_list": ["one", "two", "three"]
+                },
+                "hyphenation_decompounder": {
+                    "type" : "hyphenation_decompounder",
+                    "word_list_path": "analysis/example_word_list.txt",
+                    "hyphenation_patterns_path": "analysis/hyphenation_patterns.xml",
+                    "max_subword_size": 22
+                }
+            }
+        }
+    }
+}
 --------------------------------------------------
+// CONSOLE
diff --git a/docs/reference/analysis/tokenfilters/elision-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/elision-tokenfilter.asciidoc
index c44ccffd51e61..956c5ad13d034 100644
--- a/docs/reference/analysis/tokenfilters/elision-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/elision-tokenfilter.asciidoc
@@ -9,20 +9,24 @@ example:
 
 [source,js]
 --------------------------------------------------
-"index" : {
-    "analysis" : {
-        "analyzer" : {
-            "default" : {
-                "tokenizer" : "standard",
-                "filter" : ["standard", "elision"]
-            }
-        },
-        "filter" : {
-            "elision" : {
-                "type" : "elision",
-                "articles" : ["l", "m", "t", "qu", "n", "s", "j"]
+PUT /elision_example
+{
+    "settings" : {
+        "analysis" : {
+            "analyzer" : {
+                "default" : {
+                    "tokenizer" : "standard",
+                    "filter" : ["standard", "elision"]
+                }
+            },
+            "filter" : {
+                "elision" : {
+                    "type" : "elision",
+                    "articles" : ["l", "m", "t", "qu", "n", "s", "j"]
+                }
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
diff --git a/docs/reference/analysis/tokenfilters/flatten-graph-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/flatten-graph-tokenfilter.asciidoc
new file mode 100644
index 0000000000000..90b6136898306
--- /dev/null
+++ b/docs/reference/analysis/tokenfilters/flatten-graph-tokenfilter.asciidoc
@@ -0,0 +1,20 @@
+[[analysis-flatten-graph-tokenfilter]]
+=== Flatten Graph Token Filter
+
+experimental[]
+
+The `flatten_graph` token filter accepts an arbitrary graph token
+stream, such as that produced by
+<<analysis-synonym-graph-tokenfilter>>, and flattens it into a single
+linear chain of tokens suitable for indexing.
+
+This is a lossy process, as separate side paths are squashed on top of
+one another, but it is necessary if you use a graph token stream
+during indexing because a Lucene index cannot currently represent a
+graph.  For this reason, it's best to apply graph analyzers only at
+search time because that preserves the full graph structure and gives
+correct matches for proximity queries.
+
+For more information on this topic and its various complexities,
+please read the http://blog.mikemccandless.com/2012/04/lucenes-tokenstreams-are-actually.html[Lucene's
+TokenStreams are actually graphs] blog post.
diff --git a/docs/reference/analysis/tokenfilters/hunspell-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/hunspell-tokenfilter.asciidoc
index 9b3f188d95145..cef687f761905 100644
--- a/docs/reference/analysis/tokenfilters/hunspell-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/hunspell-tokenfilter.asciidoc
@@ -10,7 +10,7 @@ one or more `*.dic` files (all of which will automatically be picked up).
 For example, assuming the default hunspell location is used, the
 following directory layout will define the `en_US` dictionary:
 
-[source,js]
+[source,txt]
 --------------------------------------------------
 - conf
     |-- hunspell
@@ -42,24 +42,28 @@ settings:
 
 [source,js]
 --------------------------------------------------
+PUT /hunspell_example
 {
-    "analysis" : {
-        "analyzer" : {
-            "en" : {
-                "tokenizer" : "standard",
-                "filter" : [ "lowercase", "en_US" ]
-            }
-        },
-        "filter" : {
-            "en_US" : {
-                "type" : "hunspell",
-                "locale" : "en_US",
-                "dedup" : true
+    "settings": {
+        "analysis" : {
+            "analyzer" : {
+                "en" : {
+                    "tokenizer" : "standard",
+                    "filter" : [ "lowercase", "en_US" ]
+                }
+            },
+            "filter" : {
+                "en_US" : {
+                    "type" : "hunspell",
+                    "locale" : "en_US",
+                    "dedup" : true
+                }
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 The hunspell token filter accepts four options:
 
diff --git a/docs/reference/analysis/tokenfilters/keep-types-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/keep-types-tokenfilter.asciidoc
index bb1103dff8a2d..afaf4f8fa8c46 100644
--- a/docs/reference/analysis/tokenfilters/keep-types-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/keep-types-tokenfilter.asciidoc
@@ -1,7 +1,7 @@
 [[analysis-keep-types-tokenfilter]]
 === Keep Types Token Filter
 
-A token filter of type `keep_types` that only keeps tokens with a token type 
+A token filter of type `keep_types` that only keeps tokens with a token type
 contained in a predefined set.
 
 
@@ -14,24 +14,61 @@ types:: a list of types to keep
 [float]
 === Settings example
 
+You can set it up like:
+
 [source,js]
 --------------------------------------------------
+PUT /keep_types_example
 {
-    "index" : {
+    "settings" : {
         "analysis" : {
             "analyzer" : {
                 "my_analyzer" : {
                     "tokenizer" : "standard",
                     "filter" : ["standard", "lowercase", "extract_numbers"]
-                },
+                }
             },
             "filter" : {
                 "extract_numbers" : {
                     "type" : "keep_types",
                     "types" : [ "<NUM>" ]
-                },
+                }
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
+
+And test it like:
+
+[source,js]
+--------------------------------------------------
+POST /keep_types_example/_analyze
+{
+  "analyzer" : "my_analyzer",
+  "text" : "this is just 1 a test"
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+And it'd respond:
+
+[source,js]
+--------------------------------------------------
+{
+  "tokens": [
+    {
+      "token": "1",
+      "start_offset": 13,
+      "end_offset": 14,
+      "type": "<NUM>",
+      "position": 3
+    }
+  ]
+}
+--------------------------------------------------
+// TESTRESPONSE
+
+Note how only the `<NUM>` token is in the output.
diff --git a/docs/reference/analysis/tokenfilters/keep-words-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/keep-words-tokenfilter.asciidoc
index e4abbeff15dea..50c74942a0101 100644
--- a/docs/reference/analysis/tokenfilters/keep-words-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/keep-words-tokenfilter.asciidoc
@@ -20,17 +20,18 @@ keep_words_case:: a boolean indicating whether to lower case the words (defaults
 
 [source,js]
 --------------------------------------------------
+PUT /keep_words_example
 {
-    "index" : {
+    "settings" : {
         "analysis" : {
             "analyzer" : {
-                "my_analyzer" : {
+                "example_1" : {
                     "tokenizer" : "standard",
                     "filter" : ["standard", "lowercase", "words_till_three"]
                 },
-                "my_analyzer1" : {
+                "example_2" : {
                     "tokenizer" : "standard",
-                    "filter" : ["standard", "lowercase", "words_on_file"]
+                    "filter" : ["standard", "lowercase", "words_in_file"]
                 }
             },
             "filter" : {
@@ -38,12 +39,13 @@ keep_words_case:: a boolean indicating whether to lower case the words (defaults
                     "type" : "keep",
                     "keep_words" : [ "one", "two", "three"]
                 },
-                "words_on_file" : {
+                "words_in_file" : {
                     "type" : "keep",
-                    "keep_words_path" : "/path/to/word/file"
+                    "keep_words_path" : "analysis/example_word_list.txt"
                 }
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
diff --git a/docs/reference/analysis/tokenfilters/keyword-marker-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/keyword-marker-tokenfilter.asciidoc
index e43687e29dc1c..1f1e4e655c55e 100644
--- a/docs/reference/analysis/tokenfilters/keyword-marker-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/keyword-marker-tokenfilter.asciidoc
@@ -12,23 +12,131 @@ any stemming filters.
 |`keywords_path` |A path (either relative to `config` location, or
 absolute) to a list of words.
 
+|`keywords_pattern` |A regular expression pattern to match against words
+in the text.
+
 |`ignore_case` |Set to `true` to lower case all words first. Defaults to
 `false`.
 |=======================================================================
 
-Here is an example:
+You can configure it like:
+
+[source,js]
+--------------------------------------------------
+PUT /keyword_marker_example
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "protect_cats": {
+          "type": "custom",
+          "tokenizer": "standard",
+          "filter": ["lowercase", "protect_cats", "porter_stem"]
+        },
+        "normal": {
+          "type": "custom",
+          "tokenizer": "standard",
+          "filter": ["lowercase", "porter_stem"]
+        }
+      },
+      "filter": {
+        "protect_cats": {
+          "type": "keyword_marker",
+          "keywords": ["cats"]
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+
+And test it with:
+
+[source,js]
+--------------------------------------------------
+POST /keyword_marker_example/_analyze
+{
+  "analyzer" : "protect_cats",
+  "text" : "I like cats"
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+And it'd respond:
+
+[source,js]
+--------------------------------------------------
+{
+  "tokens": [
+    {
+      "token": "i",
+      "start_offset": 0,
+      "end_offset": 1,
+      "type": "<ALPHANUM>",
+      "position": 0
+    },
+    {
+      "token": "like",
+      "start_offset": 2,
+      "end_offset": 6,
+      "type": "<ALPHANUM>",
+      "position": 1
+    },
+    {
+      "token": "cats",
+      "start_offset": 7,
+      "end_offset": 11,
+      "type": "<ALPHANUM>",
+      "position": 2
+    }
+  ]
+}
+--------------------------------------------------
+// TESTRESPONSE
+
+As compared to the `normal` analyzer which has `cats` stemmed to `cat`:
+
+[source,js]
+--------------------------------------------------
+POST /keyword_marker_example/_analyze
+{
+  "analyzer" : "normal",
+  "text" : "I like cats"
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+Response:
 
 [source,js]
 --------------------------------------------------
-index :
-    analysis :
-        analyzer :
-            myAnalyzer :
-                type : custom
-                tokenizer : standard
-                filter : [lowercase, protwords, porter_stem]    
-        filter :
-            protwords :
-                type : keyword_marker
-                keywords_path : analysis/protwords.txt
+{
+  "tokens": [
+    {
+      "token": "i",
+      "start_offset": 0,
+      "end_offset": 1,
+      "type": "<ALPHANUM>",
+      "position": 0
+    },
+    {
+      "token": "like",
+      "start_offset": 2,
+      "end_offset": 6,
+      "type": "<ALPHANUM>",
+      "position": 1
+    },
+    {
+      "token": "cat",
+      "start_offset": 7,
+      "end_offset": 11,
+      "type": "<ALPHANUM>",
+      "position": 2
+    }
+  ]
+}
 --------------------------------------------------
+// TESTRESPONSE
diff --git a/docs/reference/analysis/tokenfilters/keyword-repeat-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/keyword-repeat-tokenfilter.asciidoc
index aa8c7a9b75285..044e8c1476951 100644
--- a/docs/reference/analysis/tokenfilters/keyword-repeat-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/keyword-repeat-tokenfilter.asciidoc
@@ -9,18 +9,85 @@ subsequent stemmer will be indexed twice. Therefore, consider adding a
 `unique` filter with `only_on_same_position` set to `true` to drop
 unnecessary duplicates.
 
-Here is an example:
+Here is an example of using the `keyword_repeat` token filter to
+preserve both the stemmed and unstemmed version of tokens:
 
 [source,js]
 --------------------------------------------------
-index :
-    analysis :
-        analyzer :
-            myAnalyzer :
-                type : custom
-                tokenizer : standard
-                filter : [lowercase, keyword_repeat, porter_stem, unique_stem]    
-            unique_stem:
-                type: unique
-                only_on_same_position : true
+PUT /keyword_repeat_example
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "stemmed_and_unstemmed": {
+          "type": "custom",
+          "tokenizer": "standard",
+          "filter": ["lowercase", "keyword_repeat", "porter_stem", "unique_stem"]
+        }
+      },
+      "filter": {
+        "unique_stem": {
+          "type": "unique",
+          "only_on_same_position": true
+        }
+      }
+    }
+  }
+}
 --------------------------------------------------
+// CONSOLE
+
+And you can test it with:
+
+[source,js]
+--------------------------------------------------
+POST /keyword_repeat_example/_analyze
+{
+  "analyzer" : "stemmed_and_unstemmed",
+  "text" : "I like cats"
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+And it'd respond:
+
+[source,js]
+--------------------------------------------------
+{
+  "tokens": [
+    {
+      "token": "i",
+      "start_offset": 0,
+      "end_offset": 1,
+      "type": "<ALPHANUM>",
+      "position": 0
+    },
+    {
+      "token": "like",
+      "start_offset": 2,
+      "end_offset": 6,
+      "type": "<ALPHANUM>",
+      "position": 1
+    },
+    {
+      "token": "cats",
+      "start_offset": 7,
+      "end_offset": 11,
+      "type": "<ALPHANUM>",
+      "position": 2
+    },
+    {
+      "token": "cat",
+      "start_offset": 7,
+      "end_offset": 11,
+      "type": "<ALPHANUM>",
+      "position": 2
+    }
+  ]
+}
+--------------------------------------------------
+// TESTRESPONSE
+
+Which preserves both the `cat` and `cats` tokens. Compare this to the example
+on the <<analysis-keyword-marker-tokenfilter>>.
diff --git a/docs/reference/analysis/tokenfilters/length-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/length-tokenfilter.asciidoc
index 2651980966e95..e53a198df5570 100644
--- a/docs/reference/analysis/tokenfilters/length-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/length-tokenfilter.asciidoc
@@ -11,6 +11,6 @@ type:
 |===========================================================
 |Setting |Description
 |`min` |The minimum number. Defaults to `0`.
-|`max` |The maximum number. Defaults to `Integer.MAX_VALUE`.
+|`max` |The maximum number. Defaults to `Integer.MAX_VALUE`, which is `2^31-1` or 2147483647.
 |===========================================================
 
diff --git a/docs/reference/analysis/tokenfilters/limit-token-count-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/limit-token-count-tokenfilter.asciidoc
index a6598be6095ec..ba2018c107626 100644
--- a/docs/reference/analysis/tokenfilters/limit-token-count-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/limit-token-count-tokenfilter.asciidoc
@@ -18,15 +18,25 @@ Here is an example:
 
 [source,js]
 --------------------------------------------------
-index :
-    analysis :
-        analyzer :
-            myAnalyzer :
-                type : custom
-                tokenizer : standard
-                filter : [lowercase, five_token_limit]
-        filter :
-            five_token_limit :
-                type : limit
-                max_token_count : 5
+PUT /limit_example
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "limit_example": {
+          "type": "custom",
+          "tokenizer": "standard",
+          "filter": ["lowercase", "five_token_limit"]
+        }
+      },
+      "filter": {
+        "five_token_limit": {
+          "type": "limit",
+          "max_token_count": 5
+        }
+      }
+    }
+  }
+}
 --------------------------------------------------
+// CONSOLE
diff --git a/docs/reference/analysis/tokenfilters/lowercase-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/lowercase-tokenfilter.asciidoc
index 674dfe541c9cf..519fd77ba2afd 100644
--- a/docs/reference/analysis/tokenfilters/lowercase-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/lowercase-tokenfilter.asciidoc
@@ -10,28 +10,30 @@ custom analyzer
 
 [source,js]
 --------------------------------------------------
-index :
-    analysis :
-        analyzer : 
-            myAnalyzer2 :
-                type : custom
-                tokenizer : myTokenizer1
-                filter : [myTokenFilter1, myGreekLowerCaseFilter]
-                char_filter : [my_html]
-        tokenizer :
-            myTokenizer1 :
-                type : standard
-                max_token_length : 900
-        filter :
-            myTokenFilter1 :
-                type : stop
-                stopwords : [stop1, stop2, stop3, stop4]
-            myGreekLowerCaseFilter :
-                type : lowercase
-                language : greek
-        char_filter :
-              my_html :
-                type : html_strip
-                escaped_tags : [xxx, yyy]
-                read_ahead : 1024
+PUT /lowercase_example
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "standard_lowercase_example": {
+          "type": "custom",
+          "tokenizer": "standard",
+          "filter": ["lowercase"]
+        },
+        "greek_lowercase_example": {
+          "type": "custom",
+          "tokenizer": "standard",
+          "filter": ["greek_lowercase"]
+        }
+      },
+      "filter": {
+        "greek_lowercase": {
+          "type": "lowercase",
+          "language": "greek"
+        }
+      }
+    }
+  }
+}
 --------------------------------------------------
+// CONSOLE
diff --git a/docs/reference/analysis/tokenfilters/minhash-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/minhash-tokenfilter.asciidoc
index 0660b0b0e906b..eb6a4d820ef1b 100644
--- a/docs/reference/analysis/tokenfilters/minhash-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/minhash-tokenfilter.asciidoc
@@ -1,5 +1,5 @@
 [[analysis-minhash-tokenfilter]]
-== Minhash Token Filter
+=== Minhash Token Filter
 
 A token filter of type `min_hash` hashes each token of the token stream and divides
 the resulting hashes into buckets, keeping the lowest-valued hashes per
diff --git a/docs/reference/analysis/tokenfilters/pattern-capture-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/pattern-capture-tokenfilter.asciidoc
index 7c919b56b9810..5b935d31f1289 100644
--- a/docs/reference/analysis/tokenfilters/pattern-capture-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/pattern-capture-tokenfilter.asciidoc
@@ -7,16 +7,30 @@ Patterns are not anchored to the beginning and end of the string, so
 each pattern can match multiple times, and matches are allowed to
 overlap.
 
+[WARNING]
+.Beware of Pathological Regular Expressions
+========================================
+
+The pattern capture token filter uses
+http://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html[Java Regular Expressions].
+
+A badly written regular expression could run very slowly or even throw a
+StackOverflowError and cause the node it is running on to exit suddenly.
+
+Read more about http://www.regular-expressions.info/catastrophic.html[pathological regular expressions and how to avoid them].
+
+========================================
+
 For instance a pattern like :
 
-[source,js]
+[source,text]
 --------------------------------------------------
 "(([a-z]+)(\d*))"
 --------------------------------------------------
 
 when matched against:
 
-[source,js]
+[source,text]
 --------------------------------------------------
 "abc123def456"
 --------------------------------------------------
@@ -32,14 +46,14 @@ This is particularly useful for indexing text like camel-case code, eg
 
 [source,js]
 --------------------------------------------------
-curl -XPUT localhost:9200/test/  -d '
+PUT test
 {
    "settings" : {
       "analysis" : {
          "filter" : {
             "code" : {
                "type" : "pattern_capture",
-               "preserve_original" : 1,
+               "preserve_original" : true,
                "patterns" : [
                   "(\\p{Ll}+|\\p{Lu}\\p{Ll}+|\\p{Lu}+)",
                   "(\\d+)"
@@ -55,12 +69,12 @@ curl -XPUT localhost:9200/test/  -d '
       }
    }
 }
-'
 --------------------------------------------------
+// CONSOLE
 
 When used to analyze the text
 
-[source,js]
+[source,java]
 --------------------------------------------------
 import static org.apache.commons.lang.StringEscapeUtils.escapeHtml
 --------------------------------------------------
@@ -73,14 +87,14 @@ Another example is analyzing email addresses:
 
 [source,js]
 --------------------------------------------------
-curl -XPUT localhost:9200/test/  -d '
+PUT test
 {
    "settings" : {
       "analysis" : {
          "filter" : {
             "email" : {
                "type" : "pattern_capture",
-               "preserve_original" : 1,
+               "preserve_original" : true,
                "patterns" : [
                   "([^@]+)",
                   "(\\p{L}+)",
@@ -98,12 +112,12 @@ curl -XPUT localhost:9200/test/  -d '
       }
    }
 }
-'
 --------------------------------------------------
+// CONSOLE
 
 When the above analyzer is used on an email address like:
 
-[source,js]
+[source,text]
 --------------------------------------------------
 john-smith_123@foo-bar.com
 --------------------------------------------------
@@ -117,19 +131,21 @@ Multiple patterns are required to allow overlapping captures, but also
 means that patterns are less dense and easier to understand.
 
 *Note:* All tokens are emitted in the same position, and with the same
-character offsets, so when combined with highlighting, the whole
-original token will be highlighted, not just the matching subset. For
-instance, querying the above email address for `"smith"` would
-highlight:
+character offsets. This means, for example, that a `match` query for
+`john-smith_123@foo-bar.com` that uses this analyzer will return documents
+containing any of these tokens, even when using the `and` operator.
+Also, when combined with highlighting, the whole original token will 
+be highlighted, not just the matching subset. For instance, querying 
+the above email address for `"smith"` would highlight:
 
-[source,js]
+[source,html]
 --------------------------------------------------
   <em>john-smith_123@foo-bar.com</em>
 --------------------------------------------------
 
 not:
 
-[source,js]
+[source,html]
 --------------------------------------------------
   john-<em>smith</em>_123@foo-bar.com
 --------------------------------------------------
diff --git a/docs/reference/analysis/tokenfilters/pattern_replace-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/pattern_replace-tokenfilter.asciidoc
index 54e08426e8bd0..bc8cdc385bf56 100644
--- a/docs/reference/analysis/tokenfilters/pattern_replace-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/pattern_replace-tokenfilter.asciidoc
@@ -7,3 +7,17 @@ defined using the `pattern` parameter, and the replacement string can be
 provided using the `replacement` parameter (supporting referencing the
 original text, as explained
 http://docs.oracle.com/javase/6/docs/api/java/util/regex/Matcher.html#appendReplacement(java.lang.StringBuffer,%20java.lang.String)[here]).
+
+[WARNING]
+.Beware of Pathological Regular Expressions
+========================================
+
+The pattern replace token filter uses
+http://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html[Java Regular Expressions].
+
+A badly written regular expression could run very slowly or even throw a
+StackOverflowError and cause the node it is running on to exit suddenly.
+
+Read more about http://www.regular-expressions.info/catastrophic.html[pathological regular expressions and how to avoid them].
+
+========================================
diff --git a/docs/reference/analysis/tokenfilters/snowball-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/snowball-tokenfilter.asciidoc
index 6042642027cd9..93e1eed26b4b2 100644
--- a/docs/reference/analysis/tokenfilters/snowball-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/snowball-tokenfilter.asciidoc
@@ -12,8 +12,9 @@ For example:
 
 [source,js]
 --------------------------------------------------
+PUT /my_index
 {
-    "index" : {
+    "settings": {
         "analysis" : {
             "analyzer" : {
                 "my_analyzer" : {
@@ -31,3 +32,4 @@ For example:
     }
 }
 --------------------------------------------------
+// CONSOLE
diff --git a/docs/reference/analysis/tokenfilters/stemmer-override-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/stemmer-override-tokenfilter.asciidoc
index 6e010894f419b..e178181d1474a 100644
--- a/docs/reference/analysis/tokenfilters/stemmer-override-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/stemmer-override-tokenfilter.asciidoc
@@ -20,15 +20,60 @@ Here is an example:
 
 [source,js]
 --------------------------------------------------
-index :
-    analysis :
-        analyzer :
-            myAnalyzer :
-                type : custom
-                tokenizer : standard
-                filter : [lowercase, custom_stems, porter_stem]    
-        filter:
-            custom_stems:
-                type: stemmer_override
-                rules_path : analysis/custom_stems.txt
+PUT /my_index
+{
+    "settings": {
+        "analysis" : {
+            "analyzer" : {
+                "my_analyzer" : {
+                    "tokenizer" : "standard",
+                    "filter" : ["lowercase", "custom_stems", "porter_stem"]
+                }
+            },
+            "filter" : {
+                "custom_stems" : {
+                    "type" : "stemmer_override",
+                    "rules_path" : "analysis/stemmer_override.txt"
+                }
+            }
+        }
+    }
+}
 --------------------------------------------------
+// CONSOLE
+
+Where the file looks like:
+
+[source,stemmer_override]
+--------------------------------------------------
+include::{es-test-dir}/cluster/config/analysis/stemmer_override.txt[]
+--------------------------------------------------
+
+You can also define the overrides rules inline:
+
+[source,js]
+--------------------------------------------------
+PUT /my_index
+{
+    "settings": {
+        "analysis" : {
+            "analyzer" : {
+                "my_analyzer" : {
+                    "tokenizer" : "standard",
+                    "filter" : ["lowercase", "custom_stems", "porter_stem"]
+                }
+            },
+            "filter" : {
+                "custom_stems" : {
+                    "type" : "stemmer_override",
+                    "rules" : [
+                        "running => run",
+                        "stemmer => stemmer"
+                    ]
+                }
+            }
+        }
+    }
+}
+--------------------------------------------------
+// CONSOLE
diff --git a/docs/reference/analysis/tokenfilters/stemmer-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/stemmer-tokenfilter.asciidoc
index 548342c521bdf..a052a4a7a5877 100644
--- a/docs/reference/analysis/tokenfilters/stemmer-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/stemmer-tokenfilter.asciidoc
@@ -6,8 +6,9 @@ filters through a single unified interface. For example:
 
 [source,js]
 --------------------------------------------------
+PUT /my_index
 {
-    "index" : {
+    "settings": {
         "analysis" : {
             "analyzer" : {
                 "my_analyzer" : {
@@ -25,6 +26,7 @@ filters through a single unified interface. For example:
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 The `language`/`name` parameter controls the stemmer with the following
 available values (the preferred filters are marked in *bold*):
@@ -177,4 +179,3 @@ http://clef.isti.cnr.it/2003/WN_web/22.pdf[`light_swedish`]
 Turkish::
 
 http://snowball.tartarus.org/algorithms/turkish/stemmer.html[*`turkish`*]
-
diff --git a/docs/reference/analysis/tokenfilters/stop-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/stop-tokenfilter.asciidoc
index f3b5a195662f3..b20f9c9418dc7 100644
--- a/docs/reference/analysis/tokenfilters/stop-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/stop-tokenfilter.asciidoc
@@ -47,6 +47,7 @@ PUT /my_index
     }
 }
 ------------------------------------
+// CONSOLE
 
 or a predefined language-specific list:
 
@@ -66,6 +67,7 @@ PUT /my_index
     }
 }
 ------------------------------------
+// CONSOLE
 
 Elasticsearch provides the following predefined list of languages:
 
diff --git a/docs/reference/analysis/tokenfilters/synonym-graph-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/synonym-graph-tokenfilter.asciidoc
new file mode 100644
index 0000000000000..4b145053e8168
--- /dev/null
+++ b/docs/reference/analysis/tokenfilters/synonym-graph-tokenfilter.asciidoc
@@ -0,0 +1,132 @@
+[[analysis-synonym-graph-tokenfilter]]
+=== Synonym Graph Token Filter
+
+experimental[]
+
+The `synonym_graph` token filter allows to easily handle synonyms,
+including multi-word synonyms correctly during the analysis process.
+
+In order to properly handle multi-word synonyms this token filter
+creates a "graph token stream" during processing.  For more information
+on this topic and its various complexities, please read the
+http://blog.mikemccandless.com/2012/04/lucenes-tokenstreams-are-actually.html[Lucene's TokenStreams are actually graphs] blog post.
+
+["NOTE",id="synonym-graph-index-note"]
+===============================
+This token filter is designed to be used as part of a search analyzer
+only.  If you want to apply synonyms during indexing please use the
+standard <<analysis-synonym-tokenfilter,synonym token filter>>.
+===============================
+
+Synonyms are configured using a configuration file.
+Here is an example:
+
+[source,js]
+--------------------------------------------------
+PUT /test_index
+{
+    "settings": {
+        "index" : {
+            "analysis" : {
+                "filter" : {
+                    "graph_synonyms" : { <1>
+                        "type" : "synonym_graph",
+                        "synonyms_path" : "analysis/synonym.txt",
+                        "tokenizer": "whitespace" <2>
+                    }
+                },
+                "analyzer" : {
+                    "search_synonyms" : {
+                        "tokenizer" : "whitespace",
+                        "filter" : ["graph_synonyms"] <3>
+                    }
+                }
+            }
+        }
+    }
+}
+--------------------------------------------------
+// CONSOLE
+
+<1> We configure a `graph_synonyms` filter, with a path of `analysis/synonym.txt`
+    (relative to the `config` location).
+<2> The `tokenizer` parameter controls the tokenizer that will be used to
+    tokenize the synonym, and defaults to the `whitespace` tokenizer.
+    Additional settings are: `ignore_case` (defaults to `false`), and
+    `expand` (defaults to `true`).
+<3> The `search_synonyms` analyzer is then configured with the filter.
+
+Two synonym formats are supported: Solr, WordNet.
+
+[float]
+==== Solr synonyms
+
+The following is a sample format of the file:
+
+[source,synonyms]
+--------------------------------------------------
+include::{es-test-dir}/cluster/config/analysis/synonym.txt[]
+--------------------------------------------------
+
+You can also define synonyms for the filter directly in the
+configuration file (note use of `synonyms` instead of `synonyms_path`):
+
+[source,js]
+--------------------------------------------------
+PUT /test_index
+{
+    "settings": {
+        "index" : {
+            "analysis" : {
+                "filter" : {
+                    "synonym" : {
+                        "type" : "synonym_graph",
+                        "synonyms" : [
+                            "lol, laughing out loud",
+                            "universe, cosmos"
+                        ]
+                    }
+                }
+            }
+        }
+    }
+}
+--------------------------------------------------
+// CONSOLE
+
+However, it is recommended to define large synonyms set in a file using
+`synonyms_path`, because specifying them inline increases cluster size unnecessarily.
+
+[float]
+==== WordNet synonyms
+
+Synonyms based on http://wordnet.princeton.edu/[WordNet] format can be
+declared using `format`:
+
+[source,js]
+--------------------------------------------------
+PUT /test_index
+{
+    "settings": {
+        "index" : {
+            "analysis" : {
+                "filter" : {
+                    "synonym" : {
+                        "type" : "synonym_graph",
+                        "format" : "wordnet",
+                        "synonyms" : [
+                            "s(100000001,1,'abstain',v,1,0).",
+                            "s(100000001,2,'refrain',v,1,0).",
+                            "s(100000001,3,'desist',v,1,0)."
+                        ]
+                    }
+                }
+            }
+        }
+    }
+}
+--------------------------------------------------
+// CONSOLE
+
+Using `synonyms_path` to define WordNet synonyms in a file is supported
+as well.
diff --git a/docs/reference/analysis/tokenfilters/synonym-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/synonym-tokenfilter.asciidoc
index 12da352b51cfe..1aff4525fb50e 100644
--- a/docs/reference/analysis/tokenfilters/synonym-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/synonym-tokenfilter.asciidoc
@@ -7,34 +7,38 @@ Here is an example:
 
 [source,js]
 --------------------------------------------------
+PUT /test_index
 {
-    "index" : {
-        "analysis" : {
-            "analyzer" : {
-                "synonym" : {
-                    "tokenizer" : "whitespace",
-                    "filter" : ["synonym"]
-                }
-            },
-            "filter" : {
-                "synonym" : {
-                    "type" : "synonym",
-                    "synonyms_path" : "analysis/synonym.txt"
+    "settings": {
+        "index" : {
+            "analysis" : {
+                "filter" : {
+                    "synonym" : {
+                        "type" : "synonym", <1>
+                        "synonyms_path" : "analysis/synonym.txt",
+                        "tokenizer" : "whitespace" <2>
+                    }
+                },
+                "analyzer" : {
+                    "synonym" : {
+                        "tokenizer" : "whitespace",
+                        "filter" : ["synonym"] <3>
+                    }
                 }
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
-The above configures a `synonym` filter, with a path of
-`analysis/synonym.txt` (relative to the `config` location). The
-`synonym` analyzer is then configured with the filter. Additional
-settings are: `ignore_case` (defaults to `false`), and `expand`
-(defaults to `true`).
-
-The `tokenizer` parameter controls the tokenizers that will be used to
-tokenize the synonym, and defaults to the `whitespace` tokenizer.
+<1> We configure a `synonym` filter, with a path of `analysis/synonym.txt`
+    (relative to the `config` location).
+<2> The `tokenizer` parameter controls the tokenizers that will be used to
+    tokenize the synonym, and defaults to the `whitespace` tokenizer.
+    Additional settings are: `ignore_case` (defaults to `false`), and
+    `expand` (defaults to `true`).
+<3> The `synonym` analyzer is then configured with the filter.
 
 Two synonym formats are supported: Solr, WordNet.
 
@@ -43,38 +47,9 @@ Two synonym formats are supported: Solr, WordNet.
 
 The following is a sample format of the file:
 
-[source,js]
+[source,synonyms]
 --------------------------------------------------
-# Blank lines and lines starting with pound are comments.
-
-# Explicit mappings match any token sequence on the LHS of "=>"
-# and replace with all alternatives on the RHS.  These types of mappings
-# ignore the expand parameter in the schema.
-# Examples:
-i-pod, i pod => ipod,
-sea biscuit, sea biscit => seabiscuit
-
-# Equivalent synonyms may be separated with commas and give
-# no explicit mapping.  In this case the mapping behavior will
-# be taken from the expand parameter in the schema.  This allows
-# the same synonym file to be used in different synonym handling strategies.
-# Examples:
-ipod, i-pod, i pod
-foozball , foosball
-universe , cosmos
-
-# If expand==true, "ipod, i-pod, i pod" is equivalent
-# to the explicit mapping:
-ipod, i-pod, i pod => ipod, i-pod, i pod
-# If expand==false, "ipod, i-pod, i pod" is equivalent
-# to the explicit mapping:
-ipod, i-pod, i pod => ipod
-
-# Multiple synonym mapping entries are merged.
-foo => foo bar
-foo => baz
-# is equivalent to
-foo => foo bar, baz
+include::{es-test-dir}/cluster/config/analysis/synonym.txt[]
 --------------------------------------------------
 
 You can also define synonyms for the filter directly in the
@@ -82,18 +57,26 @@ configuration file (note use of `synonyms` instead of `synonyms_path`):
 
 [source,js]
 --------------------------------------------------
+PUT /test_index
 {
-    "filter" : {
-        "synonym" : {
-            "type" : "synonym",
-            "synonyms" : [
-                "i-pod, i pod => ipod",
-                "universe, cosmos"
-            ] 
+    "settings": {
+        "index" : {
+            "analysis" : {
+                "filter" : {
+                    "synonym" : {
+                        "type" : "synonym",
+                        "synonyms" : [
+                            "i-pod, i pod => ipod",
+                            "universe, cosmos"
+                        ]
+                    }
+                }
+            }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 However, it is recommended to define large synonyms set in a file using
 `synonyms_path`, because specifying them inline increases cluster size unnecessarily.
@@ -106,20 +89,28 @@ declared using `format`:
 
 [source,js]
 --------------------------------------------------
+PUT /test_index
 {
-    "filter" : {
-        "synonym" : {
-            "type" : "synonym",
-            "format" : "wordnet",
-            "synonyms" : [
-                "s(100000001,1,'abstain',v,1,0).",
-                "s(100000001,2,'refrain',v,1,0).",
-                "s(100000001,3,'desist',v,1,0)."
-            ]
+    "settings": {
+        "index" : {
+            "analysis" : {
+                "filter" : {
+                    "synonym" : {
+                        "type" : "synonym",
+                        "format" : "wordnet",
+                        "synonyms" : [
+                            "s(100000001,1,'abstain',v,1,0).",
+                            "s(100000001,2,'refrain',v,1,0).",
+                            "s(100000001,3,'desist',v,1,0)."
+                        ]
+                    }
+                }
+            }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 Using `synonyms_path` to define WordNet synonyms in a file is supported
 as well.
diff --git a/docs/reference/analysis/tokenfilters/truncate-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/truncate-tokenfilter.asciidoc
index 14652f4634283..4c28ddba38146 100644
--- a/docs/reference/analysis/tokenfilters/truncate-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/truncate-tokenfilter.asciidoc
@@ -2,9 +2,7 @@
 === Truncate Token Filter
 
 The `truncate` token filter can be used to truncate tokens into a
-specific length. This can come in handy with keyword (single token)
-based mapped fields that are used for sorting in order to reduce memory
-usage.
+specific length.
 
 It accepts a `length` parameter which control the number of characters
 to truncate to, defaults to `10`.
diff --git a/docs/reference/analysis/tokenfilters/word-delimiter-graph-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/word-delimiter-graph-tokenfilter.asciidoc
new file mode 100644
index 0000000000000..c221075b49f1f
--- /dev/null
+++ b/docs/reference/analysis/tokenfilters/word-delimiter-graph-tokenfilter.asciidoc
@@ -0,0 +1,96 @@
+[[analysis-word-delimiter-graph-tokenfilter]]
+=== Word Delimiter Graph Token Filter
+
+experimental[]
+
+Named `word_delimiter_graph`, it splits words into subwords and performs
+optional transformations on subword groups. Words are split into
+subwords with the following rules:
+
+* split on intra-word delimiters (by default, all non alpha-numeric
+characters).
+* "Wi-Fi" -> "Wi", "Fi"
+* split on case transitions: "PowerShot" -> "Power", "Shot"
+* split on letter-number transitions: "SD500" -> "SD", "500"
+* leading and trailing intra-word delimiters on each subword are
+ignored: "//hello---there, 'dude'" -> "hello", "there", "dude"
+* trailing "'s" are removed for each subword: "O'Neil's" -> "O", "Neil"
+
+Unlike the `word_delimiter`, this token filter correctly handles positions for
+multi terms expansion at search-time when any of the following options
+are set to true:
+
+ * `preserve_original`
+ * `catenate_numbers`
+ * `catenate_words`
+ * `catenate_all`
+
+Parameters include:
+
+`generate_word_parts`::
+    If `true` causes parts of words to be
+    generated: "PowerShot" => "Power" "Shot". Defaults to `true`.
+
+`generate_number_parts`::
+    If `true` causes number subwords to be
+    generated: "500-42" => "500" "42". Defaults to `true`.
+
+`catenate_words`::
+    If `true` causes maximum runs of word parts to be
+    catenated: "wi-fi" => "wifi". Defaults to `false`.
+
+`catenate_numbers`::
+    If `true` causes maximum runs of number parts to
+    be catenated: "500-42" => "50042". Defaults to `false`.
+
+`catenate_all`::
+    If `true` causes all subword parts to be catenated:
+    "wi-fi-4000" => "wifi4000". Defaults to `false`.
+
+`split_on_case_change`::
+    If `true` causes "PowerShot" to be two tokens;
+    ("Power-Shot" remains two parts regards). Defaults to `true`.
+
+`preserve_original`::
+    If `true` includes original words in subwords:
+    "500-42" => "500-42" "500" "42". Defaults to `false`.
+
+`split_on_numerics`::
+    If `true` causes "j2se" to be three tokens; "j"
+    "2" "se". Defaults to `true`.
+
+`stem_english_possessive`::
+    If `true` causes trailing "'s" to be
+    removed for each subword: "O'Neil's" => "O", "Neil". Defaults to `true`.
+
+Advance settings include:
+
+`protected_words`::
+    A list of protected words from being delimiter.
+    Either an array, or also can set `protected_words_path` which resolved
+    to a file configured with protected words (one on each line).
+    Automatically resolves to `config/` based location if exists.
+
+`type_table`::
+    A custom type mapping table, for example (when configured
+    using `type_table_path`):
+
+[source,type_table]
+--------------------------------------------------
+    # Map the $, %, '.', and ',' characters to DIGIT
+    # This might be useful for financial data.
+    $ => DIGIT
+    % => DIGIT
+    . => DIGIT
+    \\u002C => DIGIT
+
+    # in some cases you might not want to split on ZWJ
+    # this also tests the case where we need a bigger byte[]
+    # see http://en.wikipedia.org/wiki/Zero-width_joiner
+    \\u200D => ALPHANUM
+--------------------------------------------------
+
+NOTE: Using a tokenizer like the `standard` tokenizer may interfere with
+the `catenate_*` and `preserve_original` parameters, as the original
+string may already have lost punctuation during tokenization.  Instead,
+you may want to use the `whitespace` tokenizer.
diff --git a/docs/reference/analysis/tokenfilters/word-delimiter-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/word-delimiter-tokenfilter.asciidoc
index edb3f3b5590e6..009b027b9ef2d 100644
--- a/docs/reference/analysis/tokenfilters/word-delimiter-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/word-delimiter-tokenfilter.asciidoc
@@ -64,7 +64,7 @@ Advance settings include:
     A custom type mapping table, for example (when configured
     using `type_table_path`):
 
-[source,js]
+[source,type_table]
 --------------------------------------------------
     # Map the $, %, '.', and ',' characters to DIGIT
     # This might be useful for financial data.
@@ -83,4 +83,3 @@ NOTE: Using a tokenizer like the `standard` tokenizer may interfere with
 the `catenate_*` and `preserve_original` parameters, as the original
 string may already have lost punctuation during tokenization.  Instead,
 you may want to use the `whitespace` tokenizer.
-
diff --git a/docs/reference/analysis/tokenizers.asciidoc b/docs/reference/analysis/tokenizers.asciidoc
index b30822b6a0b8c..e042268a72f11 100644
--- a/docs/reference/analysis/tokenizers.asciidoc
+++ b/docs/reference/analysis/tokenizers.asciidoc
@@ -32,7 +32,7 @@ punctuation symbols. It is the best choice for most languages.
 The `letter` tokenizer divides text into terms whenever it encounters a
 character which is not a letter.
 
-<<analysis-letter-tokenizer,Lowercase Tokenizer>>::
+<<analysis-lowercase-tokenizer,Lowercase Tokenizer>>::
 
 The `lowercase` tokenizer, like the `letter` tokenizer,  divides text into
 terms whenever it encounters a character which is not a letter, but it also
diff --git a/docs/reference/analysis/tokenizers/edgengram-tokenizer.asciidoc b/docs/reference/analysis/tokenizers/edgengram-tokenizer.asciidoc
index 3ef526325e770..c22aa1c5bf3cf 100644
--- a/docs/reference/analysis/tokenizers/edgengram-tokenizer.asciidoc
+++ b/docs/reference/analysis/tokenizers/edgengram-tokenizer.asciidoc
@@ -296,6 +296,7 @@ GET my_index/_search
   "_shards": {
     "total": 5,
     "successful": 5,
+    "skipped" : 0,
     "failed": 0
   },
   "hits": {
diff --git a/docs/reference/analysis/tokenizers/lowercase-tokenizer.asciidoc b/docs/reference/analysis/tokenizers/lowercase-tokenizer.asciidoc
index 5aad28b439481..15620dbfef4eb 100644
--- a/docs/reference/analysis/tokenizers/lowercase-tokenizer.asciidoc
+++ b/docs/reference/analysis/tokenizers/lowercase-tokenizer.asciidoc
@@ -2,10 +2,10 @@
 === Lowercase Tokenizer
 
 
-The `lowercase` toknenizer, like the
+The `lowercase` tokenizer, like the
 <<analysis-letter-tokenizer, `letter` tokenizer>> breaks text into terms
 whenever it encounters a character which is not a letter, but it also
-lowecases all terms.  It is functionally equivalent to the
+lowercases all terms.  It is functionally equivalent to the
 <<analysis-letter-tokenizer, `letter` tokenizer>> combined with the
 <<analysis-lowercase-tokenfilter, `lowercase` token filter>>, but is more
 efficient as it performs both steps in a single pass.
diff --git a/docs/reference/analysis/tokenizers/pattern-tokenizer.asciidoc b/docs/reference/analysis/tokenizers/pattern-tokenizer.asciidoc
index c96fd08c95268..5c19fcf59cc92 100644
--- a/docs/reference/analysis/tokenizers/pattern-tokenizer.asciidoc
+++ b/docs/reference/analysis/tokenizers/pattern-tokenizer.asciidoc
@@ -8,6 +8,20 @@ terms.
 The default pattern is `\W+`, which splits text whenever it encounters
 non-word characters.
 
+[WARNING]
+.Beware of Pathological Regular Expressions
+========================================
+
+The pattern tokenizer uses
+http://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html[Java Regular Expressions].
+
+A badly written regular expression could run very slowly or even throw a
+StackOverflowError and cause the node it is running on to exit suddenly.
+
+Read more about http://www.regular-expressions.info/catastrophic.html[pathological regular expressions and how to avoid them].
+
+========================================
+
 [float]
 === Example output
 
@@ -97,7 +111,7 @@ The `pattern` tokenizer accepts the following parameters:
 `flags`::
 
     Java regular expression http://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html#field.summary[flags].
-    lags should be pipe-separated, eg `"CASE_INSENSITIVE|COMMENTS"`.
+    Flags should be pipe-separated, eg `"CASE_INSENSITIVE|COMMENTS"`.
 
 `group`::
 
diff --git a/docs/reference/analysis/tokenizers/whitespace-tokenizer.asciidoc b/docs/reference/analysis/tokenizers/whitespace-tokenizer.asciidoc
index 9d06ea28d55ab..24b27237f29f3 100644
--- a/docs/reference/analysis/tokenizers/whitespace-tokenizer.asciidoc
+++ b/docs/reference/analysis/tokenizers/whitespace-tokenizer.asciidoc
@@ -1,5 +1,5 @@
 [[analysis-whitespace-tokenizer]]
-=== Whitespace Analyzer
+=== Whitespace Tokenizer
 
 The `whitespace` tokenizer breaks text into terms whenever it encounters a
 whitespace character.
@@ -17,6 +17,11 @@ POST _analyze
 ---------------------------
 // CONSOLE
 
+[NOTE]
+=============================================
+The maximum token length is 255. If a token is seen that exceeds this length, then it is split at 255 characters.
+=============================================
+
 /////////////////////
 
 [source,js]
diff --git a/docs/reference/api-conventions.asciidoc b/docs/reference/api-conventions.asciidoc
index ea2e28a92ec34..202174f496f53 100644
--- a/docs/reference/api-conventions.asciidoc
+++ b/docs/reference/api-conventions.asciidoc
@@ -11,6 +11,7 @@ API, unless otherwise specified.
 * <<multi-index>>
 * <<date-math-index-names>>
 * <<common-options>>
+* <<url-access-control>>
 
 --
 
@@ -84,29 +85,52 @@ Where:
 `date_format`:: is the optional format in which the computed date should be rendered. Defaults to `YYYY.MM.dd`.
 `time_zone`:: is the optional time zone . Defaults to `utc`.
 
-You must enclose date math index name expressions within angle brackets. For example:
+You must enclose date math index name expressions within angle brackets, and
+all special characters should be URI encoded. For example:
 
 [source,js]
 ----------------------------------------------------------------------
-curl -XGET 'localhost:9200/<logstash-{now%2Fd-2d}>/_search' {
+# GET /<logstash-{now/d}>/_search
+GET /%3Clogstash-%7Bnow%2Fd%7D%3E/_search
+{
   "query" : {
-    ...
+    "match": {
+      "test": "data"
+    }
   }
 }
 ----------------------------------------------------------------------
+// CONSOLE
+// TEST[s/^/PUT logstash-2016.09.20\n/]
+// TEST[s/now/2016.09.20||/]
 
-NOTE: The `/` used for date rounding must be url encoded as `%2F` in any url.
+[NOTE]
+.Percent encoding of date math characters
+======================================================
+The special characters used for date rounding must be URI encoded as follows:
+
+[horizontal]
+`<`:: `%3C`
+`>`:: `%3E`
+`/`:: `%2F`
+`{`:: `%7B`
+`}`:: `%7D`
+`|`:: `%7C`
+`+`:: `%2B`
+`:`:: `%3A`
+`,`:: `%2C`
+======================================================
 
 The following example shows different forms of date math index names and the final index names
 they resolve to given the current time is 22rd March 2024 noon utc.
 
 [options="header"]
 |======
-| Expression                		      |Resolves to
-| `<logstash-{now/d}>`      		      | `logstash-2024.03.22`
-| `<logstash-{now/M}>`      		      | `logstash-2024.03.01`
-| `<logstash-{now/M{YYYY.MM}}>`           | `logstash-2024.03`
-| `<logstash-{now/M-1M{YYYY.MM}}>`        | `logstash-2024.02`
+| Expression                                |Resolves to
+| `<logstash-{now/d}>`                      | `logstash-2024.03.22`
+| `<logstash-{now/M}>`                      | `logstash-2024.03.01`
+| `<logstash-{now/M{YYYY.MM}}>`             | `logstash-2024.03`
+| `<logstash-{now/M-1M{YYYY.MM}}>`          | `logstash-2024.02`
 | `<logstash-{now/d{YYYY.MM.dd\|+12:00}}>`  | `logstash-2024.03.23`
 |======
 
@@ -121,12 +145,19 @@ three days, assuming the indices use the default Logstash index name format,
 
 [source,js]
 ----------------------------------------------------------------------
-curl -XGET 'localhost:9200/<logstash-{now%2Fd-2d}>,<logstash-{now%2Fd-1d}>,<logstash-{now%2Fd}>/_search' {
+# GET /<logstash-{now/d-2d}>,<logstash-{now/d-1d}>,<logstash-{now/d}>/_search
+GET /%3Clogstash-%7Bnow%2Fd-2d%7D%3E%2C%3Clogstash-%7Bnow%2Fd-1d%7D%3E%2C%3Clogstash-%7Bnow%2Fd%7D%3E/_search
+{
   "query" : {
-    ...
+    "match": {
+      "test": "data"
+    }
   }
 }
 ----------------------------------------------------------------------
+// CONSOLE
+// TEST[s/^/PUT logstash-2016.09.20\nPUT logstash-2016.09.19\nPUT logstash-2016.09.18\n/]
+// TEST[s/now/2016.09.20||/]
 
 [[common-options]]
 == Common options
@@ -171,7 +202,7 @@ one or more maths expressions:
 * `-1d` - subtract one day
 * `/d`  - round down to the nearest day
 
-The supported time units differ than those supported by <<time-units, time units>> for durations.
+The supported time units differ from those supported by <<time-units, time units>> for durations.
 The supported units are:
 
 [horizontal]
@@ -193,123 +224,141 @@ Some examples are:
 `2015-01-01||+1M/d`::   `2015-01-01` plus one month, rounded down to the nearest day.
 
 [float]
+[[common-options-response-filtering]]
 === Response Filtering
 
 All REST APIs accept a `filter_path` parameter that can be used to reduce
 the response returned by elasticsearch. This parameter takes a comma
 separated list of filters expressed with the dot notation:
 
-[source,sh]
+[source,js]
+--------------------------------------------------
+GET /_search?q=elasticsearch&filter_path=took,hits.hits._id,hits.hits._score
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
+
+Responds:
+
+[source,js]
 --------------------------------------------------
-curl -XGET 'localhost:9200/_search?pretty&filter_path=took,hits.hits._id,hits.hits._score'
 {
   "took" : 3,
   "hits" : {
     "hits" : [
       {
-        "_id" : "3640",
-        "_score" : 1.0
-      },
-      {
-        "_id" : "3642",
-        "_score" : 1.0
+        "_id" : "0",
+        "_score" : 1.6375021
       }
     ]
   }
 }
 --------------------------------------------------
+// TESTRESPONSE[s/"took" : 3/"took" : $body.took/]
+// TESTRESPONSE[s/1.6375021/$body.hits.hits.0._score/]
 
 It also supports the `*` wildcard character to match any field or part
 of a field's name:
 
 [source,sh]
 --------------------------------------------------
-curl -XGET 'localhost:9200/_nodes/stats?filter_path=nodes.*.ho*'
+GET /_cluster/state?filter_path=metadata.indices.*.stat*
+--------------------------------------------------
+// CONSOLE
+// TEST[s/^/PUT twitter\n/]
+
+Responds:
+
+[source,sh]
+--------------------------------------------------
 {
-  "nodes" : {
-    "lvJHed8uQQu4brS-SXKsNA" : {
-      "host" : "portable"
+  "metadata" : {
+    "indices" : {
+      "twitter": {"state": "open"}
     }
   }
 }
 --------------------------------------------------
+// TESTRESPONSE
 
 And the `**` wildcard can be used to include fields without knowing the
 exact path of the field. For example, we can return the Lucene version
 of every segment with this request:
 
-[source,sh]
+[source,js]
+--------------------------------------------------
+GET /_cluster/state?filter_path=routing_table.indices.**.state
+--------------------------------------------------
+// CONSOLE
+// TEST[s/^/PUT twitter\n/]
+
+Responds:
+
+[source,js]
 --------------------------------------------------
-curl 'localhost:9200/_segments?pretty&filter_path=indices.**.version'
 {
-  "indices" : {
-    "movies" : {
-      "shards" : {
-        "0" : [ {
-          "segments" : {
-            "_0" : {
-              "version" : "5.2.0"
-            }
-          }
-        } ],
-        "2" : [ {
-          "segments" : {
-            "_0" : {
-              "version" : "5.2.0"
-            }
-          }
-        } ]
-      }
-    },
-    "books" : {
-      "shards" : {
-        "0" : [ {
-          "segments" : {
-            "_0" : {
-              "version" : "5.2.0"
-            }
-          }
-        } ]
+  "routing_table": {
+    "indices": {
+      "twitter": {
+        "shards": {
+          "0": [{"state": "STARTED"}, {"state": "UNASSIGNED"}],
+          "1": [{"state": "STARTED"}, {"state": "UNASSIGNED"}],
+          "2": [{"state": "STARTED"}, {"state": "UNASSIGNED"}],
+          "3": [{"state": "STARTED"}, {"state": "UNASSIGNED"}],
+          "4": [{"state": "STARTED"}, {"state": "UNASSIGNED"}]
+        }
       }
     }
   }
 }
 --------------------------------------------------
+// TESTRESPONSE
 
 It is also possible to exclude one or more fields by prefixing the filter with the char `-`:
 
-[source,sh]
+[source,js]
+--------------------------------------------------
+GET /_count?filter_path=-_shards
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
+
+Responds:
+
+[source,js]
 --------------------------------------------------
-curl -XGET 'localhost:9200/_count?filter_path=-_shards'
 {
-  "count" : 1
+  "count" : 5
 }
-%
 --------------------------------------------------
+// TESTRESPONSE
 
 And for more control, both inclusive and exclusive filters can be combined in the same expression. In
 this case, the exclusive filters will be applied first and the result will be filtered again using the
 inclusive filters:
 
-[source,sh]
+[source,js]
+--------------------------------------------------
+GET /_cluster/state?filter_path=metadata.indices.*.state,-metadata.indices.logstash-*
+--------------------------------------------------
+// CONSOLE
+// TEST[s/^/PUT index-1\nPUT index-2\nPUT index-3\nPUT logstash-2016.01\n/]
+
+Responds:
+
+[source,js]
 --------------------------------------------------
-curl -XGET 'localhost:9200/_cluster/state?filter_path=metadata.indices.*.state,-metadata.indices.logs-*'
 {
   "metadata" : {
     "indices" : {
-      "index-1" : {
-        "state" : "open"
-      },
-      "index-3" : {
-        "state" : "open"
-      },
-      "index-2" : {
-        "state" : "open"
-      }
+      "index-1" : {"state" : "open"},
+      "index-2" : {"state" : "open"},
+      "index-3" : {"state" : "open"}
     }
   }
-}%
+}
 --------------------------------------------------
+// TESTRESPONSE
 
 Note that elasticsearch sometimes returns directly the raw value of a field,
 like the `_source` field. If you want to filter `_source` fields, you should
@@ -317,21 +366,33 @@ consider combining the already existing `_source` parameter (see
 <<get-source-filtering,Get API>> for more details) with the `filter_path`
 parameter like this:
 
-[source,sh]
+[source,js]
+--------------------------------------------------
+POST /library/book?refresh
+{"title": "Book #1", "rating": 200.1}
+POST /library/book?refresh
+{"title": "Book #2", "rating": 1.7}
+POST /library/book?refresh
+{"title": "Book #3", "rating": 0.1}
+GET /_search?filter_path=hits.hits._source&_source=title&sort=rating:desc
+--------------------------------------------------
+// CONSOLE
+
+[source,js]
 --------------------------------------------------
-curl -XGET 'localhost:9200/_search?pretty&filter_path=hits.hits._source&_source=title'
 {
   "hits" : {
     "hits" : [ {
-      "_source":{"title":"Book #2"}
-    }, {
       "_source":{"title":"Book #1"}
+    }, {
+      "_source":{"title":"Book #2"}
     }, {
       "_source":{"title":"Book #3"}
     } ]
   }
 }
 --------------------------------------------------
+// TESTRESPONSE
 
 
 [float]
@@ -340,32 +401,68 @@ curl -XGET 'localhost:9200/_search?pretty&filter_path=hits.hits._source&_source=
 The `flat_settings` flag affects rendering of the lists of settings. When
 `flat_settings` flag is `true` settings are returned in a flat format:
 
+[source,js]
+--------------------------------------------------
+GET twitter/_settings?flat_settings=true
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
+
+Returns:
+
 [source,js]
 --------------------------------------------------
 {
-  "persistent" : { },
-  "transient" : {
-    "discovery.zen.minimum_master_nodes" : "1"
+  "twitter" : {
+    "settings": {
+      "index.number_of_replicas": "1",
+      "index.number_of_shards": "1",
+      "index.creation_date": "1474389951325",
+      "index.uuid": "n6gzFZTgS664GUfx0Xrpjw",
+      "index.version.created": ...,
+      "index.provided_name" : "twitter"
+    }
   }
 }
 --------------------------------------------------
+// TESTRESPONSE[s/1474389951325/$body.twitter.settings.index\\\\.creation_date/]
+// TESTRESPONSE[s/n6gzFZTgS664GUfx0Xrpjw/$body.twitter.settings.index\\\\.uuid/]
+// TESTRESPONSE[s/"index.version.created": \.\.\./"index.version.created": $body.twitter.settings.index\\\\.version\\\\.created/]
 
 When the `flat_settings` flag is `false` settings are returned in a more
 human readable structured format:
 
+[source,js]
+--------------------------------------------------
+GET twitter/_settings?flat_settings=false
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
+
+Returns:
+
 [source,js]
 --------------------------------------------------
 {
-  "persistent" : { },
-  "transient" : {
-    "discovery" : {
-      "zen" : {
-        "minimum_master_nodes" : "1"
+  "twitter" : {
+    "settings" : {
+      "index" : {
+        "number_of_replicas": "1",
+        "number_of_shards": "1",
+        "creation_date": "1474389951325",
+        "uuid": "n6gzFZTgS664GUfx0Xrpjw",
+        "version": {
+          "created": ...
+        },
+        "provided_name" : "twitter"
       }
     }
   }
 }
 --------------------------------------------------
+// TESTRESPONSE[s/1474389951325/$body.twitter.settings.index.creation_date/]
+// TESTRESPONSE[s/n6gzFZTgS664GUfx0Xrpjw/$body.twitter.settings.index.uuid/]
+// TESTRESPONSE[s/"created": \.\.\./"created": $body.twitter.settings.index.version.created/]
 
 By default the `flat_settings` is set to `false`.
 
@@ -380,8 +477,9 @@ convention of using underscore casing.
 
 All REST APIs parameters (both request parameters and JSON body) support
 providing boolean "false" as the values: `false`, `0`, `no` and `off`.
-All other values are considered "true". Note, this is not related to
-fields within a document indexed treated as boolean fields.
+All other values are considered "true".
+
+deprecated[5.3.0,Usage of any value other than "false" and "true" is deprecated.]
 
 [float]
 === Number Values
@@ -410,8 +508,8 @@ the unit, like `2d` for 2 days.  The supported units are:
 === Byte size units
 
 Whenever the byte size of data needs to be specified, eg when setting a buffer size
-parameter, the value must specify the unit, like `10kb` for 10 kilobytes.  The
-supported units are:
+parameter, the value must specify the unit, like `10kb` for 10 kilobytes. Note that
+these units use powers of 1024, so `1kb` means 1024 bytes. The supported units are:
 
 [horizontal]
 `b`::   Bytes
@@ -460,10 +558,6 @@ Centimeter::    `cm` or `centimeters`
 Millimeter::    `mm` or `millimeters`
 Nautical mile:: `NM`, `nmi` or `nauticalmiles`
 
-The `precision` parameter in the <<query-dsl-geohash-cell-query>> accepts
-distances with the above units, but if no unit is specified, then the
-precision is interpreted as the length of the geohash.
-
 [[fuzziness]]
 [float]
 === Fuzziness
@@ -495,19 +589,110 @@ generates an edit distance based on the length of the term. For lengths:
 --
 
 [float]
-=== Result Casing
+[[common-options-error-options]]
+=== Enabling stack traces
+
+By default when a request returns an error Elasticsearch doesn't include the
+stack trace of the error. You can enable that behavior by setting the
+`error_trace` url parameter to `true`. For example, by default when you send an
+invalid `size` parameter to the `_search` API:
+
+[source,js]
+----------------------------------------------------------------------
+POST /twitter/_search?size=surprise_me
+----------------------------------------------------------------------
+// CONSOLE
+// TEST[s/surprise_me/surprise_me&error_trace=false/ catch:request]
+// Since the test system sends error_trace=true by default we have to override
+
+The response looks like:
+
+[source,js]
+----------------------------------------------------------------------
+{
+  "error" : {
+    "root_cause" : [
+      {
+        "type" : "illegal_argument_exception",
+        "reason" : "Failed to parse int parameter [size] with value [surprise_me]"
+      }
+    ],
+    "type" : "illegal_argument_exception",
+    "reason" : "Failed to parse int parameter [size] with value [surprise_me]",
+    "caused_by" : {
+      "type" : "number_format_exception",
+      "reason" : "For input string: \"surprise_me\""
+    }
+  },
+  "status" : 400
+}
+----------------------------------------------------------------------
+// TESTRESPONSE
+
+But if you set `error_trace=true`:
 
-All REST APIs accept the `case` parameter. When set to `camelCase`, all
-field names in the result will be returned in camel casing, otherwise,
-underscore casing will be used. Note, this does not apply to the source
-document indexed.
+[source,js]
+----------------------------------------------------------------------
+POST /twitter/_search?size=surprise_me&error_trace=true
+----------------------------------------------------------------------
+// CONSOLE
+// TEST[catch:request]
+
+The response looks like:
+
+[source,js]
+----------------------------------------------------------------------
+{
+  "error": {
+    "root_cause": [
+      {
+        "type": "illegal_argument_exception",
+        "reason": "Failed to parse int parameter [size] with value [surprise_me]",
+        "stack_trace": "Failed to parse int parameter [size] with value [surprise_me]]; nested: IllegalArgumentException..."
+      }
+    ],
+    "type": "illegal_argument_exception",
+    "reason": "Failed to parse int parameter [size] with value [surprise_me]",
+    "stack_trace": "java.lang.IllegalArgumentException: Failed to parse int parameter [size] with value [surprise_me]\n    at org.elasticsearch.rest.RestRequest.paramAsInt(RestRequest.java:175)...",
+    "caused_by": {
+      "type": "number_format_exception",
+      "reason": "For input string: \"surprise_me\"",
+      "stack_trace": "java.lang.NumberFormatException: For input string: \"surprise_me\"\n    at java.lang.NumberFormatException.forInputString(NumberFormatException.java:65)..."
+    }
+  },
+  "status": 400
+}
+----------------------------------------------------------------------
+// TESTRESPONSE[s/"stack_trace": "Failed to parse int parameter.+\.\.\."/"stack_trace": $body.error.root_cause.0.stack_trace/]
+// TESTRESPONSE[s/"stack_trace": "java.lang.IllegalArgum.+\.\.\."/"stack_trace": $body.error.stack_trace/]
+// TESTRESPONSE[s/"stack_trace": "java.lang.Number.+\.\.\."/"stack_trace": $body.error.caused_by.stack_trace/]
 
 [float]
 === Request body in query string
 
 For libraries that don't accept a request body for non-POST requests,
 you can pass the request body as the `source` query string parameter
-instead.
+instead. When using this method, the `source_content_type` parameter
+should also be passed with a media type value that indicates the format
+of the source, such as `application/json`.
+
+[float]
+=== Content-Type auto-detection
+
+deprecated[5.3.0, Provide the proper Content-Type header]
+
+The content sent in a request body or using the `source` query string
+parameter is inspected to automatically determine the content type
+(JSON, YAML, SMILE, or CBOR).
+
+A strict mode can be enabled which disables auto-detection and requires
+that all requests with a body have a Content-Type header that maps to
+a supported format. To enabled this strict mode, add the following
+setting to the `elasticsearch.yml` file:
+
+    http.content_type.required: true
+
+The default value is `false`.
 
 [[url-access-control]]
 == URL-based access control
@@ -519,7 +704,7 @@ the choice of specifying an index in the URL and on each individual request
 within the request body. This can make URL-based access control challenging.
 
 To prevent the user from overriding the index which has been specified in the
-URL, add this setting to the `config.yml` file:
+URL, add this setting to the `elasticsearch.yml` file:
 
     rest.action.multi.allow_explicit_index: false
 
diff --git a/docs/reference/cat.asciidoc b/docs/reference/cat.asciidoc
index 9a0bc0f5cc051..37db4a10982ce 100644
--- a/docs/reference/cat.asciidoc
+++ b/docs/reference/cat.asciidoc
@@ -25,46 +25,67 @@ the available commands.
 === Verbose
 
 Each of the commands accepts a query string parameter `v` to turn on
-verbose output.
+verbose output. For example:
 
-[source,sh]
+[source,js]
+--------------------------------------------------
+GET /_cat/master?v
+--------------------------------------------------
+// CONSOLE
+
+Might respond with:
+
+[source,js]
 --------------------------------------------------
-% curl 'localhost:9200/_cat/master?v'
-id                     ip        node
-EGtKWZlWQYWDmX29fUnp3Q 127.0.0.1 Grey, Sara
+id                     host      ip        node
+u_n93zwxThWHi1PDBJAGAg 127.0.0.1 127.0.0.1 u_n93zw
 --------------------------------------------------
+// TESTRESPONSE[s/u_n93zw(xThWHi1PDBJAGAg)?/.+/ _cat]
 
 [float]
 [[help]]
 === Help
 
 Each of the commands accepts a query string parameter `help` which will
-output its available columns.
+output its available columns. For example:
 
-[source,sh]
+[source,js]
 --------------------------------------------------
-% curl 'localhost:9200/_cat/master?help'
-id   | node id
-ip   | node transport ip address
-node | node name
+GET /_cat/master?help
 --------------------------------------------------
+// CONSOLE
+
+Might respond respond with:
+
+[source,js]
+--------------------------------------------------
+id   |   | node id
+host | h | host name
+ip   |   | ip address
+node | n | node name
+--------------------------------------------------
+// TESTRESPONSE[s/[|]/[|]/ _cat]
 
 [float]
 [[headers]]
 === Headers
 
 Each of the commands accepts a query string parameter `h` which forces
-only those columns to appear.
+only those columns to appear. For example:
 
-[source,sh]
+[source,js]
+--------------------------------------------------
+GET /_cat/nodes?h=ip,port,heapPercent,name
+--------------------------------------------------
+// CONSOLE
+
+Responds with:
+
+[source,js]
 --------------------------------------------------
-% curl 'n1:9200/_cat/nodes?h=ip,port,heapPercent,name'
-192.168.56.40 9300 40.3 Captain Universe
-192.168.56.20 9300 15.3 Kaluu
-192.168.56.50 9300 17.0 Yellowjacket
-192.168.56.10 9300 12.3 Remy LeBeau
-192.168.56.30 9300 43.9 Ramsey, Doug
+127.0.0.1 9300 27 sLBaIGK
 --------------------------------------------------
+// TESTRESPONSE[s/9300 27 sLBaIGK/\\d+ \\d+ .+/ _cat]
 
 You can also request multiple columns using simple wildcards like
 `/_cat/thread_pool?h=ip,bulk.*` to get all headers (or aliases) starting
@@ -85,7 +106,7 @@ by all the shards, not number of documents).  The `/_cat/indices` API
 is ideal.  We only need to tweak two things.  First, we want to turn
 off human mode.  We'll use a byte-level resolution.  Then we'll pipe
 our output into `sort` using the appropriate column, which in this
-case is the eight one.
+case is the eighth one.
 
 [source,sh]
 --------------------------------------------------
@@ -94,6 +115,7 @@ green wiki2 3 0 10000   0 105274918 105274918
 green wiki1 3 0 10000 413 103776272 103776272
 green foo   1 0   227   0   2065131   2065131
 --------------------------------------------------
+// NOTCONSOLE
 
 If you want to change the <<time-units,time units>>, use `time` parameter.
 
@@ -106,7 +128,7 @@ If you want to change the <<byte-units,byte units>>, use `bytes` parameter.
 
 [source,sh]
 --------------------------------------------------
-% curl '192.168.56.10:9200/_cat/indices?format=json' | jq .
+% curl 'localhost:9200/_cat/indices?format=json&pretty'
 [
   {
     "pri.store.size": "650b",
@@ -121,6 +143,7 @@ If you want to change the <<byte-units,byte units>>, use `bytes` parameter.
   }
 ]
 --------------------------------------------------
+// NOTCONSOLE
 
 Currently supported formats (for the `?format=` parameter):
 - text (default)
@@ -135,7 +158,7 @@ For example:
 
 [source,sh]
 --------------------------------------------------
-% curl '192.168.56.10:9200/_cat/indices' -H "Accept: application/json" | jq .
+% curl '192.168.56.10:9200/_cat/indices?pretty' -H "Accept: application/json"
 [
   {
     "pri.store.size": "650b",
@@ -150,6 +173,37 @@ For example:
   }
 ]
 --------------------------------------------------
+// NOTCONSOLE
+
+[float]
+[[sort]]
+=== Sort
+
+Each of the commands accepts a query string parameter `s` which sorts the table by
+the columns specified as the parameter value. Columns are specified either by name or by
+alias, and are provided as a comma separated string. By default, sorting is done in
+ascending fashion. Appending `:desc` to a column will invert the ordering for
+that column. `:asc` is also accepted but exhibits the same behavior as the default sort order.
+
+For example, with a sort string `s=column1,column2:desc,column3`, the table will be
+sorted in ascending order by column1, in descending order by column2, and in ascending
+order by column3.
+
+[source,sh]
+--------------------------------------------------
+GET _cat/templates?v&s=order:desc,template
+--------------------------------------------------
+//CONSOLE
+
+returns:
+
+[source,sh]
+--------------------------------------------------
+name                  template     order version
+pizza_pepperoni       *pepperoni*  2
+sushi_california_roll *avocado*    1     1
+pizza_hawaiian        *pineapples* 1
+--------------------------------------------------
 
 --
 
@@ -186,3 +240,5 @@ include::cat/shards.asciidoc[]
 include::cat/segments.asciidoc[]
 
 include::cat/snapshots.asciidoc[]
+
+include::cat/templates.asciidoc[]
diff --git a/docs/reference/cat/alias.asciidoc b/docs/reference/cat/alias.asciidoc
index a790b3d15cd45..84d567d110a60 100644
--- a/docs/reference/cat/alias.asciidoc
+++ b/docs/reference/cat/alias.asciidoc
@@ -4,19 +4,56 @@
 `aliases` shows information about currently configured aliases to indices
 including filter and routing infos.
 
-[source,sh]
+////
+Hidden setup for example:
+[source,js]
+--------------------------------------------------
+PUT test1
+{
+  "aliases": {
+    "alias1": {},
+    "alias2": {
+      "filter": {
+        "match": {
+          "user": "kimchy"
+        }
+      }
+    },
+    "alias3": {
+      "routing": "1"
+    },
+    "alias4": {
+      "index_routing": "2",
+      "search_routing": "1,2"
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+////
+
+[source,js]
+--------------------------------------------------
+GET /_cat/aliases?v
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+Might respond with:
+
+[source,txt]
 --------------------------------------------------
-% curl '192.168.56.10:9200/_cat/aliases?v'
 alias  index filter routing.index routing.search
-alias2 test1 *      -            -
-alias4 test1 -      2            1,2
 alias1 test1 -      -            -
+alias2 test1 *      -            -
 alias3 test1 -      1            1
+alias4 test1 -      2            1,2
 --------------------------------------------------
+// TESTRESPONSE[s/[*]/[*]/ _cat]
 
-The output shows that `alias` has configured a filter, and specific routing
+The output shows that `alias2` has configured a filter, and specific routing
 configurations in `alias3` and `alias4`.
 
-If you only want to get information about a single alias, you can specify
-the alias in the URL, for example `/_cat/aliases/alias1`.
-
+If you only want to get information about specific aliases, you can specify 
+the aliases in comma-delimited format as a URL parameter, e.g., 
+/_cat/aliases/aliases/alias1,alias2.
\ No newline at end of file
diff --git a/docs/reference/cat/allocation.asciidoc b/docs/reference/cat/allocation.asciidoc
index 4c354a814d930..ba702080e581a 100644
--- a/docs/reference/cat/allocation.asciidoc
+++ b/docs/reference/cat/allocation.asciidoc
@@ -4,14 +4,22 @@
 `allocation` provides a snapshot of how many shards are allocated to each data node
 and how much disk space they are using.
 
-[source,sh]
+[source,js]
 --------------------------------------------------
-% curl '192.168.56.10:9200/_cat/allocation?v'
-shards disk.indices disk.used disk.avail disk.total disk.percent host          ip            node
-     1        3.1gb     5.6gb     72.2gb     77.8gb          7.8 192.168.56.10 192.168.56.10 Jarella
-     1        3.1gb     5.6gb     72.2gb     77.8gb          7.8 192.168.56.30 192.168.56.30 Solarr
-     1        3.0gb     5.5gb     72.3gb     77.8gb          7.6 192.168.56.20 192.168.56.20 Adam II
+GET /_cat/allocation?v
 --------------------------------------------------
+// CONSOLE
+// TEST[s/^/PUT test\n{"settings": {"number_of_replicas": 0}}\n/]
+
+Might respond with:
+
+[source,txt]
+--------------------------------------------------
+shards disk.indices disk.used disk.avail disk.total disk.percent host      ip        node
+     5         260b    47.3gb     43.4gb    100.7gb           46 127.0.0.1 127.0.0.1 CSUXak2
+--------------------------------------------------
+// TESTRESPONSE[s/260b/\\d+b/ s/\d+(\.\d+)?[tgmk]?b/\\d+(\\.\\d+)?[tgmk]?b/ s/46/\\d+/]
+// TESTRESPONSE[s/CSUXak2/.+/ _cat]
 
 Here we can see that each node has been allocated a single shard and
 that they're all using about the same amount of space.
diff --git a/docs/reference/cat/count.asciidoc b/docs/reference/cat/count.asciidoc
index ff64f2bc73bf1..28dc39adc8dff 100644
--- a/docs/reference/cat/count.asciidoc
+++ b/docs/reference/cat/count.asciidoc
@@ -4,17 +4,38 @@
 `count` provides quick access to the document count of the entire
 cluster, or individual indices.
 
-[source,sh]
+[source,js]
 --------------------------------------------------
-% curl 192.168.56.10:9200/_cat/indices
-green wiki1 3 0 10000 331 168.5mb 168.5mb
-green wiki2 3 0   428   0     8mb     8mb
+GET /_cat/count?v
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:big_twitter]
+// TEST[s/^/POST test\/test\?refresh\n{"test": "test"}\n/]
+
+Looks like:
+
+[source,txt]
+--------------------------------------------------
+epoch      timestamp count
+1475868259 15:24:19  121
+--------------------------------------------------
+// TESTRESPONSE[s/1475868259 15:24:19/\\d+ \\d+:\\d+:\\d+/ _cat]
 
-% curl 192.168.56.10:9200/_cat/count
-1384314124582 19:42:04 10428
+Or for a single index:
 
-% curl 192.168.56.10:9200/_cat/count/wiki2
-1384314139815 19:42:19 428
+[source,js]
 --------------------------------------------------
+GET /_cat/count/twitter?v
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+[source,txt]
+--------------------------------------------------
+epoch      timestamp count
+1475868259 15:24:20  120
+--------------------------------------------------
+// TESTRESPONSE[s/1475868259 15:24:20/\\d+ \\d+:\\d+:\\d+/ _cat]
+
 
 NOTE: The document count indicates the number of live documents and does not include deleted documents which have not yet been cleaned up by the merge process.
diff --git a/docs/reference/cat/fielddata.asciidoc b/docs/reference/cat/fielddata.asciidoc
index 854853e4d39b8..19b08a5792942 100644
--- a/docs/reference/cat/fielddata.asciidoc
+++ b/docs/reference/cat/fielddata.asciidoc
@@ -1,39 +1,98 @@
 [[cat-fielddata]]
 == cat fielddata
 
-`fielddata` shows how much heap memory is currently being used by fielddata 
+`fielddata` shows how much heap memory is currently being used by fielddata
 on every data node in the cluster.
 
-[source,sh]
+
+////
+Hidden setup snippet to build an index with fielddata so our results are real:
+[source,js]
+--------------------------------------------------
+PUT test
+{
+  "mappings": {
+    "test": {
+      "properties": {
+        "body": {
+          "type": "text",
+          "fielddata":true
+        },
+        "soul": {
+          "type": "text",
+          "fielddata":true
+        }
+      }
+    }
+  }
+}
+POST test/test?refresh
+{
+  "body": "some words so there is a little field data",
+  "soul": "some more words"
+}
+
+# Perform a search to load the field data
+POST test/_search?sort=body,soul
+--------------------------------------------------
+// CONSOLE
+////
+
+[source,js]
 --------------------------------------------------
-% curl '192.168.56.10:9200/_cat/fielddata?v'
-id                     host    ip            node          field   size
-c223lARiSGeezlbrcugAYQ myhost1 10.20.100.200 Jessica Jones body    159.8kb
-c223lARiSGeezlbrcugAYQ myhost1 10.20.100.200 Jessica Jones text    225.7kb
-waPCbitNQaCL6xC8VxjAwg myhost2 10.20.100.201 Adversary     body    159.8kb
-waPCbitNQaCL6xC8VxjAwg myhost2 10.20.100.201 Adversary     text    275.3kb
-yaDkp-G3R0q1AJ-HUEvkSQ myhost3 10.20.100.202 Microchip     body    109.2kb
-yaDkp-G3R0q1AJ-HUEvkSQ myhost3 10.20.100.202 Microchip     text    175.3kb
+GET /_cat/fielddata?v
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+Looks like:
+
+[source,txt]
+--------------------------------------------------
+id                     host      ip        node    field   size
+Nqk-6inXQq-OxUfOUI8jNQ 127.0.0.1 127.0.0.1 Nqk-6in body    544b
+Nqk-6inXQq-OxUfOUI8jNQ 127.0.0.1 127.0.0.1 Nqk-6in soul    480b
+--------------------------------------------------
+// TESTRESPONSE[s/544b|480b/\\d+(\\.\\d+)?[tgmk]?b/]
+// TESTRESPONSE[s/Nqk-6in[^ ]*/.+/ s/soul|body/\\w+/ _cat]
 
 Fields can be specified either as a query parameter, or in the URL path:
 
-[source,sh]
+[source,js]
+--------------------------------------------------
+GET /_cat/fielddata?v&fields=body
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+Which looks like:
+
+[source,txt]
+--------------------------------------------------
+id                     host      ip        node    field   size
+Nqk-6inXQq-OxUfOUI8jNQ 127.0.0.1 127.0.0.1 Nqk-6in body    544b
+--------------------------------------------------
+// TESTRESPONSE[s/544b|480b/\\d+(\\.\\d+)?[tgmk]?b/]
+// TESTRESPONSE[s/Nqk-6in[^ ]*/.+/ _cat]
+
+And it accepts a comma delimited list:
+
+[source,js]
+--------------------------------------------------
+GET /_cat/fielddata/body,soul?v
 --------------------------------------------------
-% curl '192.168.56.10:9200/_cat/fielddata?v&fields=body'
-id                     host    ip            node          field   size
-c223lARiSGeezlbrcugAYQ myhost1 10.20.100.200 Jessica Jones body    159.8kb
-waPCbitNQaCL6xC8VxjAwg myhost2 10.20.100.201 Adversary     body    159.8kb
-yaDkp-G3R0q1AJ-HUEvkSQ myhost3 10.20.100.202 Microchip     body    109.2kb
+// CONSOLE
+// TEST[continued]
 
-% curl '192.168.56.10:9200/_cat/fielddata/body,text?v'
-id                     host    ip            node          field   size
-c223lARiSGeezlbrcugAYQ myhost1 10.20.100.200 Jessica Jones body    159.8kb
-c223lARiSGeezlbrcugAYQ myhost1 10.20.100.200 Jessica Jones text    225.7kb
-waPCbitNQaCL6xC8VxjAwg myhost2 10.20.100.201 Adversary     body    159.8kb
-waPCbitNQaCL6xC8VxjAwg myhost2 10.20.100.201 Adversary     text    275.3kb
-yaDkp-G3R0q1AJ-HUEvkSQ myhost3 10.20.100.202 Microchip     body    109.2kb
-yaDkp-G3R0q1AJ-HUEvkSQ myhost3 10.20.100.202 Microchip     text    175.3kb
+Which produces the same output as the first snippet:
+
+[source,txt]
+--------------------------------------------------
+id                     host      ip        node    field   size
+Nqk-6inXQq-OxUfOUI8jNQ 127.0.0.1 127.0.0.1 Nqk-6in body    544b
+Nqk-6inXQq-OxUfOUI8jNQ 127.0.0.1 127.0.0.1 Nqk-6in soul    480b
 --------------------------------------------------
+// TESTRESPONSE[s/544b|480b/\\d+(\\.\\d+)?[tgmk]?b/]
+// TESTRESPONSE[s/Nqk-6in[^ ]*/.+/ s/soul|body/\\w+/ _cat]
 
-The output shows the individual fielddata for the`body` and `text` fields, one row per field per node.
+The output shows the individual fielddata for the `body` and `soul` fields, one row per field per node.
diff --git a/docs/reference/cat/health.asciidoc b/docs/reference/cat/health.asciidoc
index bf9b3f1744316..ca2a1838adb02 100644
--- a/docs/reference/cat/health.asciidoc
+++ b/docs/reference/cat/health.asciidoc
@@ -2,17 +2,40 @@
 == cat health
 
 `health` is a terse, one-line representation of the same information
-from `/_cluster/health`. It has one option `ts` to disable the
-timestamping.
+from `/_cluster/health`.
 
-[source,sh]
+[source,js]
+--------------------------------------------------
+GET /_cat/health?v
+--------------------------------------------------
+// CONSOLE
+// TEST[s/^/PUT twitter\n{"settings":{"number_of_replicas": 0}}\n/]
+
+[source,txt]
+--------------------------------------------------
+epoch      timestamp cluster       status node.total node.data shards pri relo init unassign pending_tasks max_task_wait_time active_shards_percent
+1475871424 16:17:04  elasticsearch green           1         1      5   5    0    0        0             0                  -                100.0%
+--------------------------------------------------
+// TESTRESPONSE[s/1475871424 16:17:04/\\d+ \\d+:\\d+:\\d+/]
+// TESTRESPONSE[s/elasticsearch/[^ ]+/ s/0                  -/\\d+ (-|\\d+(\\.\\d+)?[ms]+)/ _cat]
+
+It has one option `ts` to disable the timestamping:
+
+[source,js]
+--------------------------------------------------
+GET /_cat/health?v&ts=false
+--------------------------------------------------
+// CONSOLE
+// TEST[s/^/PUT twitter\n{"settings":{"number_of_replicas": 0}}\n/]
+
+which looks like:
+
+[source,txt]
 --------------------------------------------------
-% curl localhost:9200/_cat/health
-1384308967 18:16:07 foo green 3 3 3 3 0 0 0
-% curl 'localhost:9200/_cat/health?v&ts=0'
-cluster status nodeTotal nodeData shards pri relo init unassign tasks
-foo     green          3        3      3   3    0    0        0     0
+cluster       status node.total node.data shards pri relo init unassign pending_tasks max_task_wait_time active_shards_percent
+elasticsearch green           1         1      5   5    0    0        0             0                  -                100.0%
 --------------------------------------------------
+// TESTRESPONSE[s/elasticsearch/[^ ]+/ s/0                  -/\\d+ (-|\\d+(\\.\\d+)?[ms]+)/ _cat]
 
 A common use of this command is to verify the health is consistent
 across nodes:
@@ -27,6 +50,7 @@ across nodes:
 [3] 20:20:52 [SUCCESS] es2.vm
 1384309218 18:20:18 foo green 3 3 3 3 0 0 0 0
 --------------------------------------------------
+// NOTCONSOLE
 
 A less obvious use is to track recovery of a large cluster over
 time. With enough shards, starting a cluster, or even recovering after
@@ -42,6 +66,7 @@ to track its progress is by using this command in a delayed loop:
 1384309806 18:30:06 foo green 3 3 1832 916 4 0 0
 ^C
 --------------------------------------------------
+// NOTCONSOLE
 
 In this scenario, we can tell that recovery took roughly four minutes.
 If this were going on for hours, we would be able to watch the
diff --git a/docs/reference/cat/indices.asciidoc b/docs/reference/cat/indices.asciidoc
index 3e8938faab7ac..0c840071bb9b1 100644
--- a/docs/reference/cat/indices.asciidoc
+++ b/docs/reference/cat/indices.asciidoc
@@ -2,21 +2,39 @@
 == cat indices
 
 The `indices` command provides a cross-section of each index.  This
-information *spans nodes*.
+information *spans nodes*. For example:
 
-[source,sh]
+[source,js]
 --------------------------------------------------
-% curl 'localhost:9200/_cat/indices/twi*?v'
-health status index    pri rep docs.count docs.deleted store.size pri.store.size
-green  open   twitter    5   1      11434            0       64mb           32mb
-green  open   twitter2   2   0       2030            0      5.8mb          5.8mb
+GET /_cat/indices/twi*?v&s=index
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:huge_twitter]
+// TEST[s/^/PUT twitter2\n{"settings": {"number_of_replicas": 0}}\n/]
+
+Might respond with:
+
+[source,txt]
+--------------------------------------------------
+health status index    uuid                   pri rep docs.count docs.deleted store.size pri.store.size
+yellow open   twitter  u8FNjxh8Rfy_awN11oDKYQ   1   1       1200            0     88.1kb         88.1kb
+green  open   twitter2 nYFWZEO7TUiOjLQXBaYJpA   5   0          0            0       260b           260b
+--------------------------------------------------
+// TESTRESPONSE[s/\d+(\.\d+)?[tgmk]?b/\\d+(\\.\\d+)?[tgmk]?b/]
+// TESTRESPONSE[s/u8FNjxh8Rfy_awN11oDKYQ|nYFWZEO7TUiOjLQXBaYJpA/.+/ _cat]
 
 We can tell quickly how many shards make up an index, the number of
-docs at the Lucene level, including hidden docs (e.g., from nested types), 
-deleted docs, primary store size, and total store size (all shards including replicas). 
+docs, deleted docs, primary store size, and total store size (all shards including replicas).
 All these exposed metrics come directly from Lucene APIs.
 
+*Notes:*
+
+1. As the number of documents and deleted documents shown in this are at the lucene level,
+it includes all the hidden documents (e.g. from nested documents) as well.
+
+2. To get actual count of documents at the elasticsearch level, the recommended way
+is to use either the <<cat-count>> or the <<search-count>>
+
 [float]
 [[pri-flag]]
 === Primaries
@@ -31,39 +49,77 @@ the view of relevant stats in the context of only the primaries.
 
 Which indices are yellow?
 
-[source,sh]
+[source,js]
+--------------------------------------------------
+GET /_cat/indices?v&health=yellow
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+Which looks like:
+
+[source,txt]
 --------------------------------------------------
-% curl localhost:9200/_cat/indices | grep ^yell
-yellow open  wiki     2 1  6401 1115 151.4mb 151.4mb
-yellow open  twitter  5 1 11434    0    32mb    32mb
+health status index    uuid                   pri rep docs.count docs.deleted store.size pri.store.size
+yellow open   twitter  u8FNjxh8Rfy_awN11oDKYQ   1   1       1200            0     88.1kb         88.1kb
 --------------------------------------------------
+// TESTRESPONSE[s/\d+(\.\d+)?[tgmk]?b/\\d+(\\.\\d+)?[tgmk]?b/]
+// TESTRESPONSE[s/u8FNjxh8Rfy_awN11oDKYQ/.+/ _cat]
 
-What's my largest index by disk usage not including replicas?
+Which index has the largest number of documents?
 
-[source,sh]
+[source,js]
 --------------------------------------------------
-% curl 'localhost:9200/_cat/indices?bytes=b' | sort -rnk8
-green open  wiki     2 0  6401 1115 158843725 158843725
-green open  twitter  5 1 11434    0  67155614  33577857
-green open  twitter2 2 0  2030    0   6125085   6125085
+GET /_cat/indices?v&s=docs.count:desc
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
-How many merge operations have the shards for the `wiki` completed?
+Which looks like:
 
-[source,sh]
+[source,txt]
 --------------------------------------------------
-% curl 'localhost:9200/_cat/indices/wiki?pri&v&h=health,index,prirep,docs.count,mt'
-health index docs.count mt pri.mt
-green  wiki        9646 16     16
+health status index    uuid                   pri rep docs.count docs.deleted store.size pri.store.size
+yellow open   twitter  u8FNjxh8Rfy_awN11oDKYQ   1   1       1200            0     88.1kb         88.1kb
+green  open   twitter2 nYFWZEO7TUiOjLQXBaYJpA   5   0          0            0       260b           260b
 --------------------------------------------------
+// TESTRESPONSE[s/\d+(\.\d+)?[tgmk]?b/\\d+(\\.\\d+)?[tgmk]?b/]
+// TESTRESPONSE[s/u8FNjxh8Rfy_awN11oDKYQ|nYFWZEO7TUiOjLQXBaYJpA/.+/ _cat]
+
+How many merge operations have the shards for the `twitter` completed?
+
+[source,js]
+--------------------------------------------------
+GET /_cat/indices/twitter?pri&v&h=health,index,pri,rep,docs.count,mt
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+Might look like:
+
+[source,js]
+--------------------------------------------------
+health index   pri rep docs.count mt pri.mt
+yellow twitter   1   1 1200       16     16
+--------------------------------------------------
+// TESTRESPONSE[s/16/\\d+/ _cat]
 
 How much memory is used per index?
 
-[source,sh]
+[source,js]
+--------------------------------------------------
+GET /_cat/indices?v&h=i,tm&s=tm:desc
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+Might look like:
+
+[source,js]
 --------------------------------------------------
-% curl 'localhost:9200/_cat/indices?v&h=i,tm'
-i     tm
-wiki  8.1gb
-test  30.5kb
-user  1.9mb
+i         tm
+twitter   8.1gb
+twitter2  30.5kb
 --------------------------------------------------
+// TESTRESPONSE[s/\d+(\.\d+)?[tgmk]?b/\\d+(\\.\\d+)?[tgmk]?b/]
+// TESTRESPONSE[_cat]
diff --git a/docs/reference/cat/master.asciidoc b/docs/reference/cat/master.asciidoc
index 6189948d912df..dfa10d6e3a4e6 100644
--- a/docs/reference/cat/master.asciidoc
+++ b/docs/reference/cat/master.asciidoc
@@ -2,14 +2,22 @@
 == cat master
 
 `master` doesn't have any extra options. It simply displays the
-master's node ID, bound IP address, and node name.
+master's node ID, bound IP address, and node name. For example:
 
-[source,sh]
+[source,js]
+--------------------------------------------------
+GET /_cat/master?v
+--------------------------------------------------
+// CONSOLE
+
+might respond:
+
+[source,txt]
 --------------------------------------------------
-% curl 'localhost:9200/_cat/master?v'
-id                     ip            node
-Ntgn2DcuTjGuXlhKDUD4vA 192.168.56.30 Solarr
+id                     host      ip        node
+YzWoH_2BT-6UjVGDyPdqYg 127.0.0.1 127.0.0.1 YzWoH_2
 --------------------------------------------------
+// TESTRESPONSE[s/YzWoH_2.+/.+/ _cat]
 
 This information is also available via the `nodes` command, but this
 is slightly shorter when all you want to do, for example, is verify
@@ -19,9 +27,10 @@ all nodes agree on the master:
 --------------------------------------------------
 % pssh -i -h list.of.cluster.hosts curl -s localhost:9200/_cat/master
 [1] 19:16:37 [SUCCESS] es3.vm
-Ntgn2DcuTjGuXlhKDUD4vA 192.168.56.30 Solarr
+Ntgn2DcuTjGuXlhKDUD4vA 192.168.56.30 H5dfFeA
 [2] 19:16:37 [SUCCESS] es2.vm
-Ntgn2DcuTjGuXlhKDUD4vA 192.168.56.30 Solarr
+Ntgn2DcuTjGuXlhKDUD4vA 192.168.56.30 H5dfFeA
 [3] 19:16:37 [SUCCESS] es1.vm
-Ntgn2DcuTjGuXlhKDUD4vA 192.168.56.30 Solarr
+Ntgn2DcuTjGuXlhKDUD4vA 192.168.56.30 H5dfFeA
 --------------------------------------------------
+// NOTCONSOLE
diff --git a/docs/reference/cat/nodeattrs.asciidoc b/docs/reference/cat/nodeattrs.asciidoc
index 2c73d715e3424..18feeba8d03f3 100644
--- a/docs/reference/cat/nodeattrs.asciidoc
+++ b/docs/reference/cat/nodeattrs.asciidoc
@@ -2,34 +2,26 @@
 == cat nodeattrs
 
 The `nodeattrs` command shows custom node attributes.
+For example:
 
-["source","sh",subs="attributes,callouts"]
+[source,js]
 --------------------------------------------------
-% curl 192.168.56.10:9200/_cat/nodeattrs
-node       host    ip          attr  value
-Black Bolt epsilon 192.168.1.8 rack  rack314
-Black Bolt epsilon 192.168.1.8 azone us-east-1
+GET /_cat/nodeattrs?v
 --------------------------------------------------
+// CONSOLE
 
-The first few columns give you basic info per node.
+Could look like:
 
-
-["source","sh",subs="attributes,callouts"]
+[source,txt]
 --------------------------------------------------
-node       host    ip
-Black Bolt epsilon 192.168.1.8
-Black Bolt epsilon 192.168.1.8
+node    host      ip        attr     value
+EK_AsJb 127.0.0.1 127.0.0.1 testattr test
 --------------------------------------------------
+// TESTRESPONSE[s/EK_AsJb/.+/ _cat]
 
-
-The attr and value columns can give you a picture of custom node attributes.
-
-[source,sh]
---------------------------------------------------
-attr  value
-rack  rack314
-azone us-east-1
---------------------------------------------------
+The first few columns (`node`, `host`, `ip`) give you basic info per node
+and the `attr` and `value` columns give you the custom node attributes,
+one per line.
 
 [float]
 === Columns
@@ -49,18 +41,25 @@ by default.  To have the headers appear in the output, use verbose
 mode (`v`). The header name will match the supplied value (e.g.,
 `pid` versus `p`).  For example:
 
-["source","sh",subs="attributes,callouts"]
+[source,js]
+--------------------------------------------------
+GET /_cat/nodeattrs?v&h=name,pid,attr,value
+--------------------------------------------------
+// CONSOLE
+
+Might look like:
+
+[source,js]
 --------------------------------------------------
-% curl 192.168.56.10:9200/_cat/nodeattrs?v&h=name,pid,attr,value
-name       pid   attr  value
-Black Bolt 28000 rack  rack314
-Black Bolt 28000 azone us-east-1
+name    pid   attr     value
+EK_AsJb 19566 testattr test
 --------------------------------------------------
+// TESTRESPONSE[s/EK_AsJb/.+/ s/19566/\\d*/ _cat]
 
 [cols="<,<,<,<,<",options="header",subs="normal"]
 |=======================================================================
 |Header |Alias |Appear by Default |Description |Example
-|`node`|`name`|Yes|Name of the node|Black Bolt
+|`node`|`name`|Yes|Name of the node|DKDM97B
 |`id` |`nodeId` |No |Unique node ID |k0zy
 |`pid` |`p` |No |Process ID |13061
 |`host` |`h` |Yes |Host name |n1
diff --git a/docs/reference/cat/nodes.asciidoc b/docs/reference/cat/nodes.asciidoc
index 288a28f3642d2..95621110916f0 100644
--- a/docs/reference/cat/nodes.asciidoc
+++ b/docs/reference/cat/nodes.asciidoc
@@ -1,51 +1,31 @@
 [[cat-nodes]]
 == cat nodes
 
-The `nodes` command shows the cluster topology.
+The `nodes` command shows the cluster topology. For example
 
-["source","sh",subs="attributes,callouts"]
+[source,js]
 --------------------------------------------------
-% curl 192.168.56.10:9200/_cat/nodes
-SP4H 4727 192.168.56.30 9300 {version} {jdk} 72.1gb 35.4 93.9mb 79 239.1mb 0.45 3.4h mdi - Boneyard
-_uhJ 5134 192.168.56.10 9300 {version} {jdk} 72.1gb 33.3 93.9mb 85 239.1mb 0.06 3.4h mdi * Athena
-HfDp 4562 192.168.56.20 9300 {version} {jdk} 72.2gb 74.5 93.9mb 83 239.1mb 0.12 3.4h mdi - Zarek
+GET /_cat/nodes?v
 --------------------------------------------------
+// CONSOLE
 
-The first few columns tell you where your nodes live.  For sanity it
-also tells you what version of ES and the JVM each one runs.
+Might look like:
 
-["source","sh",subs="attributes,callouts"]
+[source,txt]
 --------------------------------------------------
-nodeId pid  ip            port version     jdk
-u2PZ   4234 192.168.56.30 9300 {version} {jdk}
-URzf   5443 192.168.56.10 9300 {version} {jdk}
-ActN   3806 192.168.56.20 9300 {version} {jdk}
+ip        heap.percent ram.percent cpu load_1m load_5m load_15m node.role master name
+127.0.0.1           65          99  42    3.07                  mdi       *      mJw06l1
 --------------------------------------------------
+// TESTRESPONSE[s/3.07/(\\d+\\.\\d+( \\d+\\.\\d+ (\\d+\\.\\d+)?)?)?/]
+// TESTRESPONSE[s/65          99  42/\\d+ \\d+ \\d+/]
+// TESTRESPONSE[s/[*]/[*]/ s/mJw06l1/.+/ _cat]
 
+The first few columns (`ip`, `heap.percent`, `ram.percent`, `cpu`, `load_*`) tell
+you where your nodes live and give a quick picture of performance stats.
 
-The next few give a picture of your heap, memory, and load.
-
-[source,sh]
---------------------------------------------------
-diskAvail heapPercent heapMax ramPercent  ramMax load
-   72.1gb        31.3  93.9mb         81 239.1mb 0.24
-   72.1gb        19.6  93.9mb         82 239.1mb 0.05
-   72.2gb        64.9  93.9mb         84 239.1mb 0.12
---------------------------------------------------
-
-The last columns provide ancillary information that can often be
-useful when looking at the cluster as a whole, particularly large
-ones.  How many master-eligible nodes do I have?  How many client
-nodes?  It looks like someone restarted a node recently; which one was
-it?
-
-[source,sh]
---------------------------------------------------
-uptime node.role master name
-  3.5h di        -      Boneyard
-  3.5h md        *      Athena
-  3.5h i         -      Zarek
---------------------------------------------------
+The last (`node.role`, `master`, and `name`) columns provide ancillary
+information that can often be useful when looking at the cluster as a whole,
+particularly large ones.  How many master-eligible nodes do I have?
 
 [float]
 === Columns
@@ -65,18 +45,20 @@ by default.  To have the headers appear in the output, use verbose
 mode (`v`). The header name will match the supplied value (e.g.,
 `pid` versus `p`).  For example:
 
-["source","sh",subs="attributes,callouts"]
+[source,js]
+--------------------------------------------------
+GET /_cat/nodes?v&h=id,ip,port,v,m
+--------------------------------------------------
+// CONSOLE
+
+Might look like:
+
+["source","js",subs="attributes,callouts"]
 --------------------------------------------------
-% curl 192.168.56.10:9200/_cat/nodes?v&h=id,ip,port,v,m
-id   ip            port v           m
-pLSN 192.168.56.30 9300 {version} -
-k0zy 192.168.56.10 9300 {version} -
-6Tyi 192.168.56.20 9300 {version} *
-% curl 192.168.56.10:9200/_cat/nodes?h=id,ip,port,v,m
-pLSN 192.168.56.30 9300 {version} -
-k0zy 192.168.56.10 9300 {version} -
-6Tyi 192.168.56.20 9300 {version} *
+id   ip        port  v         m
+veJR 127.0.0.1 59938 {version} *
 --------------------------------------------------
+// TESTRESPONSE[s/veJR/.+/ s/59938/\\d+/ s/[*]/[*]/ _cat]
 
 [cols="<,<,<,<,<",options="header",subs="normal"]
 |=======================================================================
@@ -85,11 +67,14 @@ k0zy 192.168.56.10 9300 {version} -
 |`pid` |`p` |No |Process ID |13061
 |`ip` |`i` |Yes |IP address |127.0.1.1
 |`port` |`po` |No |Bound transport port |9300
-|`http_address` |`http`| No | bound http address | 127.0.0.1:9200
+|`http_address` |`http`| No |Bound http address | 127.0.0.1:9200
 |`version` |`v` |No |Elasticsearch version |{version}
 |`build` |`b` |No |Elasticsearch Build hash |5c03844
 |`jdk` |`j` |No |Running Java version |1.8.0
-|`disk.avail` |`d`, `disk`, `diskAvail` |No |Available disk space |1.8gb
+|`disk.total` |`dt`, `diskTotal` |No |Total disk space| 458.3gb
+|`disk.used` |`du`, `diskUsed` |No |Used disk space| 259.8gb
+|`disk.avail` |`d`, `disk`, `diskAvail` |No |Available disk space |198.4gb
+|`disk.used_percent` |`dup`, `diskUsedPercent` |No |Used disk space percentage |56.71
 |`heap.current` |`hc`, `heapCurrent` |No |Used heap |311.2mb
 |`heap.percent` |`hp`, `heapPercent` |Yes |Used heap percentage |7
 |`heap.max` |`hm`, `heapMax` |No |Maximum configured heap |1015.6mb
@@ -102,13 +87,15 @@ descriptors |123
 descriptors percentage |1
 |`file_desc.max` |`fdm`, `fileDescriptorMax` |No |Maximum number of file
 descriptors |1024
-|`load` |`l` |No |Most recent load average |0.22
 |`cpu` | |No |Recent system CPU usage as percent |12
+|`load_1m` |`l` |No |Most recent load average |0.22
+|`load_5m` |`l` |No |Load average for the last five minutes |0.78
+|`load_15m` |`l` |No |Load average for the last fifteen minutes |1.24
 |`uptime` |`u` |No |Node uptime |17.3m
 |`node.role` |`r`, `role`, `nodeRole` |Yes |Master eligible node (m);
 Data node (d); Ingest node (i); Coordinating node only (-) |mdi
 |`master` |`m` |Yes |Elected master (*); Not elected master (-) |*
-|`name` |`n` |Yes |Node name |Venom
+|`name` |`n` |Yes |Node name |I8hydUG
 |`completion.size` |`cs`, `completionSize` |No |Size of completion |0b
 |`fielddata.memory_size` |`fm`, `fielddataMemory` |No |Used fielddata
 cache memory |0b
@@ -152,6 +139,8 @@ of current indexing operations |0
 indexing |134ms
 |`indexing.index_total` |`iito`, `indexingIndexTotal` |No |Number of
 indexing operations |1
+|`indexing.index_failed` |`iif`, `indexingIndexFailed` |No |Number of
+failed indexing operations |0
 |`merges.current` |`mc`, `mergesCurrent` |No |Number of current
 merge operations |0
 |`merges.current_docs` |`mcd`, `mergesCurrentDocs` |No |Number of
@@ -166,15 +155,6 @@ documents |0
 merges |0b
 |`merges.total_time` |`mtt`, `mergesTotalTime` |No |Time spent merging
 documents |0s
-|`percolate.current` |`pc`, `percolateCurrent` |No |Number of current
-percolations |0
-|`percolate.memory_size` |`pm`, `percolateMemory` |No |Memory used by
-current percolations |0b
-|`percolate.queries` |`pq`, `percolateQueries` |No |Number of
-registered percolation queries |0
-|`percolate.time` |`pti`, `percolateTime` |No |Time spent
-percolating |0s
-|`percolate.total` |`pto`, `percolateTotal` |No |Total percolations |0
 |`refresh.total` |`rto`, `refreshTotal` |No |Number of refreshes |16
 |`refresh.time` |`rti`, `refreshTime` |No |Time spent in refreshes |91ms
 |`script.compilations` |`scrcc`, `scriptCompilations` |No |Total script compilations |17
@@ -187,11 +167,11 @@ phase |37ms
 operations |7
 |`search.open_contexts` |`so`, `searchOpenContexts` |No |Open search
 contexts |0
-|`search.query_current` |`sqc`, `searchFetchCurrent` |No |Current query
+|`search.query_current` |`sqc`, `searchQueryCurrent` |No |Current query
 phase operations |0
-|`search.query_time` |`sqti`, `searchFetchTime` |No |Time spent in query
+|`search.query_time` |`sqti`, `searchQueryTime` |No |Time spent in query
 phase |43ms
-|`search.query_total` |`sqto`, `searchFetchTotal` |No |Number of query
+|`search.query_total` |`sqto`, `searchQueryTotal` |No |Number of query
 operations |9
 |`search.scroll_current` |`scc`, `searchScrollCurrent` |No |Open scroll contexts |2
 |`search.scroll_time` |`scti`, `searchScrollTime` |No |Time scroll contexts held open|2m
@@ -203,4 +183,9 @@ segments |1.4kb
 |Memory used by index writer |18mb
 |`segments.version_map_memory` |`svmm`, `segmentsVersionMapMemory` |No
 |Memory used by version map |1.0kb
+|`segments.fixed_bitset_memory` |`sfbm`, `fixedBitsetMemory` |No
+|Memory used by fixed bit sets for nested object field types and type filters for types referred in _parent fields |1.0kb
+|`suggest.current` |`suc`, `suggestCurrent` |No |Number of current suggest operations |0
+|`suggest.time` |`suti`, `suggestTime` |No |Time spent in suggest |0
+|`suggest.total` |`suto`, `suggestTotal` |No |Number of suggest operations |0
 |=======================================================================
diff --git a/docs/reference/cat/pending_tasks.asciidoc b/docs/reference/cat/pending_tasks.asciidoc
index 5452052669c72..d5216c1eb0023 100644
--- a/docs/reference/cat/pending_tasks.asciidoc
+++ b/docs/reference/cat/pending_tasks.asciidoc
@@ -3,11 +3,18 @@
 
 `pending_tasks` provides the same information as the
 <<cluster-pending,`/_cluster/pending_tasks`>> API in a
-convenient tabular format.
+convenient tabular format. For example:
 
-[source,sh]
+[source,js]
+--------------------------------------------------
+GET /_cat/pending_tasks?v
+--------------------------------------------------
+// CONSOLE
+
+Might look like:
+
+[source,txt]
 --------------------------------------------------
-% curl 'localhost:9200/_cat/pending_tasks?v'
 insertOrder timeInQueue priority source
        1685       855ms HIGH     update-mapping [foo][t]
        1686       843ms HIGH     update-mapping [foo][t]
@@ -17,3 +24,6 @@ insertOrder timeInQueue priority source
        1690       787ms HIGH     update-mapping [foo][t]
        1691       773ms HIGH     update-mapping [foo][t]
 --------------------------------------------------
+// TESTRESPONSE[s/(\n.+)+/(\\n.+)*/ _cat]
+// We can't assert anything about the tasks in progress here because we don't
+// know what might be in progress....
diff --git a/docs/reference/cat/plugins.asciidoc b/docs/reference/cat/plugins.asciidoc
index 0af1faa5c9c3a..75ed7e26d88e6 100644
--- a/docs/reference/cat/plugins.asciidoc
+++ b/docs/reference/cat/plugins.asciidoc
@@ -3,12 +3,38 @@
 
 The `plugins` command provides a view per node of running plugins. This information *spans nodes*.
 
-[source,sh]
+[source,js]
 ------------------------------------------------------------------------------
-% curl 'localhost:9200/_cat/plugins?v'
-name    component       version        description
-Abraxas discovery-gce   5.0.0          The Google Compute Engine (GCE) Discovery plugin allows to use GCE API for the unicast discovery mechanism. 
-Abraxas lang-javascript 5.0.0          The JavaScript language plugin allows to have javascript as the language of scripts to execute.
--------------------------------------------------------------------------------
+GET /_cat/plugins?v&s=component&h=name,component,version,description
+------------------------------------------------------------------------------
+// CONSOLE
+
+Might look like:
+
+["source","txt",subs="attributes,callouts"]
+------------------------------------------------------------------------------
+name    component               version   description
+U7321H6 analysis-icu            {version} The ICU Analysis plugin integrates Lucene ICU module into elasticsearch, adding ICU relates analysis components.
+U7321H6 analysis-kuromoji       {version} The Japanese (kuromoji) Analysis plugin integrates Lucene kuromoji analysis module into elasticsearch.
+U7321H6 analysis-phonetic       {version} The Phonetic Analysis plugin integrates phonetic token filter analysis with elasticsearch.
+U7321H6 analysis-smartcn        {version} Smart Chinese Analysis plugin integrates Lucene Smart Chinese analysis module into elasticsearch.
+U7321H6 analysis-stempel        {version} The Stempel (Polish) Analysis plugin integrates Lucene stempel (polish) analysis module into elasticsearch.
+U7321H6 analysis-ukrainian        {version} The Ukrainian Analysis plugin integrates the Lucene UkrainianMorfologikAnalyzer into elasticsearch.
+U7321H6 discovery-azure-classic {version} The Azure Classic Discovery plugin allows to use Azure Classic API for the unicast discovery mechanism
+U7321H6 discovery-ec2           {version} The EC2 discovery plugin allows to use AWS API for the unicast discovery mechanism.
+U7321H6 discovery-file          {version} Discovery file plugin enables unicast discovery from hosts stored in a file.
+U7321H6 discovery-gce           {version} The Google Compute Engine (GCE) Discovery plugin allows to use GCE API for the unicast discovery mechanism.
+U7321H6 ingest-attachment       {version} Ingest processor that uses Apache Tika to extract contents
+U7321H6 ingest-geoip            {version} Ingest processor that uses looksup geo data based on ip adresses using the Maxmind geo database
+U7321H6 ingest-user-agent       {version} Ingest processor that extracts information from a user agent
+U7321H6 jvm-example             {version} Demonstrates all the pluggable Java entry points in Elasticsearch
+U7321H6 lang-javascript         {version} The JavaScript language plugin allows to have javascript as the language of scripts to execute.
+U7321H6 lang-python             {version} The Python language plugin allows to have python as the language of scripts to execute.
+U7321H6 mapper-attachments      {version} The mapper attachments plugin adds the attachment type to Elasticsearch using Apache Tika.
+U7321H6 mapper-murmur3          {version} The Mapper Murmur3 plugin allows to compute hashes of a field's values at index-time and to store them in the index.
+U7321H6 mapper-size             {version} The Mapper Size plugin allows document to record their uncompressed size at index time.
+U7321H6 store-smb               {version} The Store SMB plugin adds support for SMB stores.
+------------------------------------------------------------------------------
+// TESTRESPONSE[s/([.()])/\\$1/ s/U7321H6/.+/ _cat]
 
 We can tell quickly how many plugins per node we have and which versions.
diff --git a/docs/reference/cat/recovery.asciidoc b/docs/reference/cat/recovery.asciidoc
index 6fe748096d14f..d6cfc7842a2b7 100644
--- a/docs/reference/cat/recovery.asciidoc
+++ b/docs/reference/cat/recovery.asciidoc
@@ -12,16 +12,27 @@ way for shards to be loaded from disk when a node starts up.
 As an example, here is what the recovery state of a cluster may look like when there
 are no shards in transit from one node to another:
 
-[source,sh]
+[source,js]
 ----------------------------------------------------------------------------
-> curl -XGET 'localhost:9200/_cat/recovery?v'
-index shard time type  stage source_host source_node target_host target_node repository snapshot files files_percent bytes bytes_percent total_files total_bytes translog translog_percent total_translog
-index 0     87ms store done  127.0.0.1        Athena      127.0.0.1        Athena      n/a        n/a      0     0.0%          0     0.0%          0           0           0        100.0%           0
-index 1     97ms store done  127.0.0.1        Athena      127.0.0.1        Athena      n/a        n/a      0     0.0%          0     0.0%          0           0           0        100.0%           0
-index 2     93ms store done  127.0.0.1        Athena      127.0.0.1        Athena      n/a        n/a      0     0.0%          0     0.0%          0           0           0        100.0%           0
-index 3     90ms store done  127.0.0.1        Athena      127.0.0.1        Athena      n/a        n/a      0     0.0%          0     0.0%          0           0           0        100.0%           0
-index 4     9ms  store done  127.0.0.1        Athena      127.0.0.1        Athena      n/a        n/a      0     0.0%          0     0.0%          0           0           0        100.0%           0
+GET _cat/recovery?v
+----------------------------------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
+
+The response of this request will be something like:
+
+[source,js]
+---------------------------------------------------------------------------
+index   shard time type  stage source_host source_node target_host target_node repository snapshot files files_recovered files_percent files_total bytes bytes_recovered bytes_percent bytes_total translog_ops translog_ops_recovered translog_ops_percent
+twitter 0     13ms store done  n/a         n/a         node0       node-0      n/a        n/a      0     0               100%          13          0     0               100%          9928        0            0                      100.0%
 ---------------------------------------------------------------------------
+// TESTRESPONSE[s/store/empty_store/]
+// TESTRESPONSE[s/100%/0.0%/]
+// TESTRESPONSE[s/9928/0/]
+// TESTRESPONSE[s/node0/127.0.0.1/]
+// TESTRESPONSE[s/node-0/.*/]
+// TESTRESPONSE[s/13ms/\\d+m?s/]
+// TESTRESPONSE[s/13/\\d+/ _cat]
 
 In the above case, the source and target nodes are the same because the recovery
 type was store, i.e. they were read from local storage on node start.
@@ -30,43 +41,46 @@ Now let's see what a live recovery looks like. By increasing the replica count
 of our index and bringing another node online to host the replicas, we can see
 what a live shard recovery looks like.
 
-[source,sh]
+[source,js]
 ----------------------------------------------------------------------------
-> curl -XPUT 'localhost:9200/wiki/_settings' -d'{"number_of_replicas":1}'
-{"acknowledged":true}
-
-> curl -XGET 'localhost:9200/_cat/recovery?v&h=i,s,t,ty,st,shost,thost,f,fp,b,bp'
-i     s t      ty      st    shost  thost  f     fp      b        bp
-wiki  0 1252ms store   done  hostA  hostA  4     100.0%  23638870 100.0%
-wiki  0 1672ms replica index hostA  hostB  4     75.0%   23638870 48.8%
-wiki  1 1698ms replica index hostA  hostB  4     75.0%   23348540 49.4%
-wiki  1 4812ms store   done  hostA  hostA  33    100.0%  24501912 100.0%
-wiki  2 1689ms replica index hostA  hostB  4     75.0%   28681851 40.2%
-wiki  2 5317ms store   done  hostA  hostA  36    100.0%  30267222 100.0%
+GET _cat/recovery?v&h=i,s,t,ty,st,shost,thost,f,fp,b,bp
 ----------------------------------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
 
-We can see in the above listing that our 3 initial shards are in various stages
-of being replicated from one node to another. Notice that the recovery type is
-shown as `replica`. The files and bytes copied are real-time measurements.
+This will return a line like:
+
+[source,js]
+----------------------------------------------------------------------------
+i       s t      ty   st    shost       thost       f     fp      b bp
+twitter 0 1252ms peer done  192.168.1.1 192.168.1.2 0     100.0%  0 100.0%
+----------------------------------------------------------------------------
+// TESTRESPONSE[s/peer/empty_store/]
+// TESTRESPONSE[s/192.168.1.2/127.0.0.1/]
+// TESTRESPONSE[s/192.168.1.1/n\/a/]
+// TESTRESPONSE[s/100.0%/0.0%/]
+// TESTRESPONSE[s/1252/\\d+/ _cat]
+
+We can see in the above listing that our thw twitter shard was recovered from another node.
+Notice that the recovery type is shown as `peer`. The files and bytes copied are
+real-time measurements.
 
 Finally, let's see what a snapshot recovery looks like. Assuming I have previously
 made a backup of my index, I can restore it using the <<modules-snapshots,snapshot and restore>>
 API.
 
-[source,sh]
+[source,js]
 --------------------------------------------------------------------------------
-> curl -XPOST 'localhost:9200/_snapshot/imdb/snapshot_2/_restore'
-{"acknowledged":true}
-> curl -XGET 'localhost:9200/_cat/recovery?v&h=i,s,t,ty,st,rep,snap,f,fp,b,bp'
-i     s t      ty       st    rep        snap     f     fp      b     bp
-imdb  0 1978ms snapshot done  imdb       snap_1   79    8.0%    12086 9.0%
-imdb  1 2790ms snapshot index imdb       snap_1   88    7.7%    11025 8.1%
-imdb  2 2790ms snapshot index imdb       snap_1   85    0.0%    12072 0.0%
-imdb  3 2796ms snapshot index imdb       snap_1   85    2.4%    12048 7.2%
-imdb  4  819ms snapshot init  imdb       snap_1   0     0.0%    0     0.0%
+GET _cat/recovery?v&h=i,s,t,ty,st,rep,snap,f,fp,b,bp
 --------------------------------------------------------------------------------
+// CONSOLE
+// TEST[skip:no need to execute snapshot/restore here]
 
+This will show a recovery of type snapshot in the response
 
-
-
-
+[source,js]
+--------------------------------------------------------------------------------
+i       s t      ty       st    rep     snap   f  fp   b     bp
+twitter 0 1978ms snapshot done  twitter snap_1 79 8.0% 12086 9.0%
+--------------------------------------------------------------------------------
+// TESTRESPONSE[_cat]
diff --git a/docs/reference/cat/repositories.asciidoc b/docs/reference/cat/repositories.asciidoc
index 5fb68a8929c99..8caf3c5fd6f73 100644
--- a/docs/reference/cat/repositories.asciidoc
+++ b/docs/reference/cat/repositories.asciidoc
@@ -1,14 +1,24 @@
 [[cat-repositories]]
 == cat repositories
 
-The `repositories` command shows the snapshot repositories registered in the cluster.
+The `repositories` command shows the snapshot repositories registered in the
+cluster. For example:
 
-[source,sh]
+[source,js]
+--------------------------------------------------
+GET /_cat/repositories?v
+--------------------------------------------------
+// CONSOLE
+// TEST[s/^/PUT \/_snapshot\/repo1\n{"type": "fs", "settings": {"location": "repo\/1"}}\n/]
+
+might looks like:
+
+[source,txt]
 --------------------------------------------------
-% curl 'localhost:9200/_cat/repositories?v'
 id    type
 repo1   fs
 repo2   s3
 --------------------------------------------------
+// TESTRESPONSE[s/\nrepo2   s3// _cat]
 
 We can quickly see which repositories are registered and their type.
diff --git a/docs/reference/cat/segments.asciidoc b/docs/reference/cat/segments.asciidoc
index 84456e3a3137f..05d86758a8c74 100644
--- a/docs/reference/cat/segments.asciidoc
+++ b/docs/reference/cat/segments.asciidoc
@@ -3,24 +3,24 @@
 
 The `segments` command provides low level information about the segments
 in the shards of an index. It provides information similar to the
-link:indices-segments.html[_segments] endpoint.
+link:indices-segments.html[_segments] endpoint. For example:
 
-[source,sh]
+[source,js]
 --------------------------------------------------
-% curl 'http://localhost:9200/_cat/segments?v'
-index shard prirep ip            segment generation docs.count [...]
-test  4     p      192.168.2.105 _0               0          1
-test1 2     p      192.168.2.105 _0               0          1
-test1 3     p      192.168.2.105 _2               2          1
+GET /_cat/segments?v
 --------------------------------------------------
+// CONSOLE
+// TEST[s/^/PUT \/test\/test\/1?refresh\n{"test":"test"}\nPUT \/test1\/test\/1?refresh\n{"test":"test"}\n/]
 
-[source,sh]
+might look like:
+
+["source","txt",subs="attributes,callouts"]
 --------------------------------------------------
-[...] docs.deleted  size size.memory committed searchable version compound
-                 0 2.9kb        7818 false     true       4.10.2  true
-                 0 2.9kb        7818 false     true       4.10.2  true
-                 0 2.9kb        7818 false     true       4.10.2  true
+index shard prirep ip        segment generation docs.count docs.deleted size size.memory committed searchable version compound
+test  3     p      127.0.0.1 _0               0          1            0  3kb        2042 false     true       {lucene_version}   true
+test1 3     p      127.0.0.1 _0               0          1            0  3kb        2042 false     true       {lucene_version}   true
 --------------------------------------------------
+// TESTRESPONSE[s/3kb/\\d+(\\.\\d+)?[mk]?b/ s/2042/\\d+/ _cat]
 
 The output shows information about index names and shard numbers in the first
 two columns.
@@ -34,7 +34,7 @@ The following columns provide additional monitoring information:
 
 prirep::        Whether this segment belongs to a primary or replica shard.
 
-ip::            The ip address of the segments shard.
+ip::            The ip address of the segment's shard.
 
 segment::       A segment name, derived from the segment generation. The name
                 is internally used to generate the file names in the directory
diff --git a/docs/reference/cat/shards.asciidoc b/docs/reference/cat/shards.asciidoc
index 8eece0702872a..9876d83ffeeca 100644
--- a/docs/reference/cat/shards.asciidoc
+++ b/docs/reference/cat/shards.asciidoc
@@ -5,15 +5,26 @@ The `shards` command is the detailed view of what nodes contain which
 shards.  It will tell you if it's a primary or replica, the number of
 docs, the bytes it takes on disk, and the node where it's located.
 
-Here we see a single index, with three primary shards and no replicas:
-
-[source,sh]
---------------------------------------------------
-% curl 192.168.56.20:9200/_cat/shards
-wiki1 0 p STARTED 3014 31.1mb 192.168.56.10 Stiletto
-wiki1 1 p STARTED 3013 29.6mb 192.168.56.30 Frankie Raye
-wiki1 2 p STARTED 3973 38.1mb 192.168.56.20 Commander Kraken
---------------------------------------------------
+Here we see a single index, with one primary shard and no replicas:
+
+[source,js]
+---------------------------------------------------------------------------
+GET _cat/shards
+---------------------------------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
+
+This will return
+
+[source,js]
+---------------------------------------------------------------------------
+twitter 0 p STARTED 3014 31.1mb 192.168.56.10 H5dfFeA
+twitter 0 r UNASSIGNED
+---------------------------------------------------------------------------
+// TESTRESPONSE[s/3014/\\d+/]
+// TESTRESPONSE[s/31.1mb/\\d+(\.\\d+)?[kmg]?b/]
+// TESTRESPONSE[s/192.168.56.10/.*/]
+// TESTRESPONSE[s/H5dfFeA/.*/ _cat]
 
 [float]
 [[index-pattern]]
@@ -23,30 +34,47 @@ If you have many shards, you may wish to limit which indices show up
 in the output.  You can always do this with `grep`, but you can save
 some bandwidth by supplying an index pattern to the end.
 
-[source,sh]
---------------------------------------------------
-% curl 192.168.56.20:9200/_cat/shards/wiki*
-wiki2 0 p STARTED 197 3.2mb 192.168.56.10 Stiletto
-wiki2 1 p STARTED 205 5.9mb 192.168.56.30 Frankie Raye
-wiki2 2 p STARTED 275 7.8mb 192.168.56.20 Commander Kraken
---------------------------------------------------
+[source,js]
+---------------------------------------------------------------------------
+GET _cat/shards/twitt*
+---------------------------------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
+
+Which will return the following
+
+[source,js]
+---------------------------------------------------------------------------
+twitter 0 p STARTED 3014 31.1mb 192.168.56.10 H5dfFeA
+twitter 0 r UNASSIGNED
+---------------------------------------------------------------------------
+// TESTRESPONSE[s/3014/\\d+/]
+// TESTRESPONSE[s/31.1mb/\\d+(\.\\d+)?[kmg]?b/]
+// TESTRESPONSE[s/192.168.56.10/.*/]
+// TESTRESPONSE[s/H5dfFeA/.*/ _cat]
 
 
 [float]
 [[relocation]]
 === Relocation
 
-Let's say you've checked your health and you see two relocating
+Let's say you've checked your health and you see a relocating
 shards.  Where are they from and where are they going?
 
-[source,sh]
---------------------------------------------------
-% curl 192.168.56.10:9200/_cat/health
-1384315316 20:01:56 foo green 3 3 12 6 2 0 0
-% curl 192.168.56.10:9200/_cat/shards | fgrep RELO
-wiki1 0 r RELOCATING 3014 31.1mb 192.168.56.20 Commander Kraken -> 192.168.56.30 Frankie Raye
-wiki1 1 r RELOCATING 3013 29.6mb 192.168.56.10 Stiletto -> 192.168.56.30 Frankie Raye
---------------------------------------------------
+[source,js]
+---------------------------------------------------------------------------
+GET _cat/shards
+---------------------------------------------------------------------------
+// CONSOLE
+// TEST[skip:for now, relocation cannot be recreated]
+
+A relocating shard will be shown as follows
+
+[source,js]
+---------------------------------------------------------------------------
+twitter 0 p RELOCATING 3014 31.1mb 192.168.56.10 H5dfFeA -> -> 192.168.56.30 bGG90GE
+---------------------------------------------------------------------------
+// TESTRESPONSE[_cat]
 
 [float]
 [[states]]
@@ -55,48 +83,51 @@ wiki1 1 r RELOCATING 3013 29.6mb 192.168.56.10 Stiletto -> 192.168.56.30 Frankie
 Before a shard can be used, it goes through an `INITIALIZING` state.
 `shards` can show you which ones.
 
-[source,sh]
---------------------------------------------------
-% curl -XPUT 192.168.56.20:9200/_settings -d'{"number_of_replicas":1}'
-{"acknowledged":true}
-% curl 192.168.56.20:9200/_cat/shards
-wiki1 0 p STARTED      3014 31.1mb 192.168.56.10 Stiletto
-wiki1 0 r INITIALIZING    0 14.3mb 192.168.56.30 Frankie Raye
-wiki1 1 p STARTED      3013 29.6mb 192.168.56.30 Frankie Raye
-wiki1 1 r INITIALIZING    0 13.1mb 192.168.56.20 Commander Kraken
-wiki1 2 r INITIALIZING    0   14mb 192.168.56.10 Stiletto
-wiki1 2 p STARTED      3973 38.1mb 192.168.56.20 Commander Kraken
---------------------------------------------------
+[source,js]
+---------------------------------------------------------------------------
+GET _cat/shards
+---------------------------------------------------------------------------
+// CONSOLE
+// TEST[skip:there is no guarantee to test for shards in initializing state]
+
+You can get the initializing state in the response like this
+
+[source,js]
+---------------------------------------------------------------------------
+twitter 0 p STARTED      3014 31.1mb 192.168.56.10 H5dfFeA
+twitter 0 r INITIALIZING    0 14.3mb 192.168.56.30 bGG90GE
+---------------------------------------------------------------------------
+// TESTRESPONSE[_cat]
 
 If a shard cannot be assigned, for example you've overallocated the
 number of replicas for the number of nodes in the cluster, the shard
 will remain `UNASSIGNED` with the <<reason-unassigned,reason code>> `ALLOCATION_FAILED`.
 
-[source,sh]
---------------------------------------------------
-% curl -XPUT 192.168.56.20:9200/_settings -d'{"number_of_replicas":3}'
-% curl 192.168.56.20:9200/_cat/health
-1384316325 20:18:45 foo yellow 3 3 9 3 0 0 3
-% curl 192.168.56.20:9200/_cat/shards
-wiki1 0 p STARTED    3014 31.1mb 192.168.56.10 Stiletto
-wiki1 0 r STARTED    3014 31.1mb 192.168.56.30 Frankie Raye
-wiki1 0 r STARTED    3014 31.1mb 192.168.56.20 Commander Kraken
-wiki1 0 r UNASSIGNED ALLOCATION_FAILED
-wiki1 1 r STARTED    3013 29.6mb 192.168.56.10 Stiletto
-wiki1 1 p STARTED    3013 29.6mb 192.168.56.30 Frankie Raye
-wiki1 1 r STARTED    3013 29.6mb 192.168.56.20 Commander Kraken
-wiki1 1 r UNASSIGNED ALLOCATION_FAILED
-wiki1 2 r STARTED    3973 38.1mb 192.168.56.10 Stiletto
-wiki1 2 r STARTED    3973 38.1mb 192.168.56.30 Frankie Raye
-wiki1 2 p STARTED    3973 38.1mb 192.168.56.20 Commander Kraken
-wiki1 2 r UNASSIGNED ALLOCATION_FAILED
---------------------------------------------------
+You can use the shards API to find out that reason.
+
+[source,js]
+---------------------------------------------------------------------------
+GET _cat/shards?h=index,shard,prirep,state,unassigned.reason
+---------------------------------------------------------------------------
+// CONSOLE
+// TEST[skip:for now]
+
+The reason for an unassigned shard will be listed as the last field
+
+[source,js]
+---------------------------------------------------------------------------
+twitter 0 p STARTED    3014 31.1mb 192.168.56.10 H5dfFeA
+twitter 0 r STARTED    3014 31.1mb 192.168.56.30 bGG90GE
+twitter 0 r STARTED    3014 31.1mb 192.168.56.20 I8hydUG
+twitter 0 r UNASSIGNED ALLOCATION_FAILED
+---------------------------------------------------------------------------
+// TESTRESPONSE[_cat]
 
 [float]
 [[reason-unassigned]]
 === Reasons for unassigned shard
 
-These are the possible reasons for a shard be in a unassigned state:
+These are the possible reasons for a shard to be in a unassigned state:
 
 [horizontal]
 `INDEX_CREATED`::           Unassigned as a result of an API creation of an index.
diff --git a/docs/reference/cat/snapshots.asciidoc b/docs/reference/cat/snapshots.asciidoc
index 3d34cd51e6d9e..5677a0f2a7cd4 100644
--- a/docs/reference/cat/snapshots.asciidoc
+++ b/docs/reference/cat/snapshots.asciidoc
@@ -5,15 +5,30 @@ The `snapshots` command shows all snapshots that belong to a specific repository
 To find a list of available repositories to query, the command `/_cat/repositories` can be used.
 Querying the snapshots of a repository named `repo1` then looks as follows.
 
-[source,sh]
+[source,js]
+--------------------------------------------------
+GET /_cat/snapshots/repo1?v&s=id
+--------------------------------------------------
+// CONSOLE
+// TEST[s/^/PUT \/_snapshot\/repo1\/snap1?wait_for_completion=true\n/]
+// TEST[s/^/PUT \/_snapshot\/repo1\/snap2?wait_for_completion=true\n/]
+// TEST[s/^/PUT \/_snapshot\/repo1\n{"type": "fs", "settings": {"location": "repo\/1"}}\n/]
+
+Which looks like:
+
+[source,txt]
 --------------------------------------------------
-% curl 'localhost:9200/_cat/snapshots/repo1?v'
 id     status start_epoch start_time end_epoch  end_time duration indices successful_shards failed_shards total_shards
 snap1  FAILED 1445616705  18:11:45   1445616978 18:16:18     4.6m       1                 4             1            5
 snap2 SUCCESS 1445634298  23:04:58   1445634672 23:11:12     6.2m       2                10             0           10
 --------------------------------------------------
+// TESTRESPONSE[s/FAILED/SUCCESS/ s/14456\d+/\\d+/ s/\d+(\.\d+)?(m|s|ms)/\\d+(\\.\\d+)?(m|s|ms)/]
+// TESTRESPONSE[s/\d+:\d+:\d+/\\d+:\\d+:\\d+/]
+// TESTRESPONSE[s/1                 4             1            5/\\d+ \\d+ \\d+ \\d+/]
+// TESTRESPONSE[s/2                10             0           10/\\d+ \\d+ \\d+ \\d+/]
+// TESTRESPONSE[_cat]
 
 Each snapshot contains information about when it was started and stopped.
 Start and stop timestamps are available in two formats.
 The `HH:MM:SS` output is simply for quick human consumption.
-The epoch time retains more information, including date, and is machine sortable if the snapshot process spans days.
\ No newline at end of file
+The epoch time retains more information, including date, and is machine sortable if the snapshot process spans days.
diff --git a/docs/reference/cat/templates.asciidoc b/docs/reference/cat/templates.asciidoc
new file mode 100644
index 0000000000000..a2420e010fad3
--- /dev/null
+++ b/docs/reference/cat/templates.asciidoc
@@ -0,0 +1,31 @@
+[[cat-templates]]
+== cat templates
+
+The `templates` command provides information about existing templates.
+
+[source,js]
+--------------------------------------------------
+GET /_cat/templates?v&s=name
+--------------------------------------------------
+// CONSOLE
+// TEST[s/^/PUT _template\/template0\n{"template": "te*", "order": 0}\n/]
+// TEST[s/^/PUT _template\/template1\n{"template": "tea*", "order": 1}\n/]
+// TEST[s/^/PUT _template\/template2\n{"template": "teak*", "order": 2, "version": 7}\n/]
+
+which looks like
+
+[source,txt]
+--------------------------------------------------
+name      template order version
+template0 te*      0
+template1 tea*     1
+template2 teak*    2     7
+--------------------------------------------------
+// TESTRESPONSE[s/\*/\\*/ _cat]
+
+The output shows that there are three existing templates,
+with template2 having a version value.
+
+The endpoint also supports giving a template name or pattern in the url
+to filter the results, for example `/_cat/templates/template*` or
+`/_cat/templates/template0`.
diff --git a/docs/reference/cat/thread_pool.asciidoc b/docs/reference/cat/thread_pool.asciidoc
index 300767c256b9f..0512eafaa74cd 100644
--- a/docs/reference/cat/thread_pool.asciidoc
+++ b/docs/reference/cat/thread_pool.asciidoc
@@ -4,35 +4,43 @@
 The `thread_pool` command shows cluster wide thread pool statistics per node. By default the active, queue and rejected
 statistics are returned for all thread pools.
 
-[source,sh]
+[source,js]
 --------------------------------------------------
-% curl 192.168.56.10:9200/_cat/thread_pool
-0EWUhXe bulk                0 0 0
-0EWUhXe fetch_shard_started 0 0 0
-0EWUhXe fetch_shard_store   0 0 0
-0EWUhXe flush               0 0 0
-0EWUhXe force_merge         0 0 0
-0EWUhXe generic             0 0 0
-0EWUhXe get                 0 0 0
-0EWUhXe index               0 0 0
-0EWUhXe listener            0 0 0
-0EWUhXe management          1 0 0
-0EWUhXe refresh             0 0 0
-0EWUhXe search              0 0 0
-0EWUhXe snapshot            0 0 0
-0EWUhXe warmer              0 0 0
+GET /_cat/thread_pool
 --------------------------------------------------
+// CONSOLE
+
+Which looks like:
+
+[source,txt]
+--------------------------------------------------
+Z6MkIvC bulk                0 0 0
+Z6MkIvC fetch_shard_started 0 0 0
+Z6MkIvC fetch_shard_store   0 0 0
+Z6MkIvC flush               0 0 0
+Z6MkIvC force_merge         0 0 0
+Z6MkIvC generic             0 0 0
+Z6MkIvC get                 0 0 0
+Z6MkIvC index               0 0 0
+Z6MkIvC listener            0 0 0
+Z6MkIvC management          1 0 0
+Z6MkIvC refresh             0 0 0
+Z6MkIvC search              0 0 0
+Z6MkIvC snapshot            0 0 0
+Z6MkIvC warmer              0 0 0
+--------------------------------------------------
+// TESTRESPONSE[s/Z6MkIvC/[\\w-]+/ s/\d+/\\d+/ _cat]
 
 The first column is the node name
 
-[source,sh]
+[source,txt]
 --------------------------------------------------
 node_name
-0EWUhXe
+Z6MkIvC
 --------------------------------------------------
 
 The second column is the thread pool name
-[source,sh]
+[source,txt]
 --------------------------------------------------
 name
 bulk
@@ -54,7 +62,7 @@ warmer
 
 The next three columns show the active, queue, and rejected statistics for each thread pool
 
-[source,sh]
+[source,txt]
 --------------------------------------------------
 active queue rejected
      0     0        0
@@ -76,14 +84,22 @@ active queue rejected
 The cat thread pool API accepts a `thread_pool_patterns` URL parameter for specifying a
 comma-separated list of regular expressions to match thread pool names.
 
-[source,sh]
+[source,js]
+--------------------------------------------------
+GET /_cat/thread_pool/generic?v&h=id,name,active,rejected,completed
+--------------------------------------------------
+// CONSOLE
+
+which looks like:
+
+[source,js]
 --------------------------------------------------
-% curl 'localhost:9200/_cat/thread_pool/generic?v&h=id,name,active,rejected,completed'
 id                     name    active rejected completed
 0EWUhXeBQtaVGlexUeVwMg generic      0        0        70
 --------------------------------------------------
+// TESTRESPONSE[s/0EWUhXeBQtaVGlexUeVwMg/[\\w-]+/ s/\d+/\\d+/ _cat]
 
-Here the host columns and the active, rejected and completed suggest thread pool statistic are displayed.
+Here the host columns and the active, rejected and completed suggest thread pool statistics are displayed.
 
 All <<modules-threadpool,built-in thread pools>> and custom thread pools are available.
 [float]
diff --git a/docs/reference/cluster.asciidoc b/docs/reference/cluster.asciidoc
index 7955239fc5dc4..2b244ffcd3bf2 100644
--- a/docs/reference/cluster.asciidoc
+++ b/docs/reference/cluster.asciidoc
@@ -15,18 +15,19 @@ example, here are some sample executions of nodes info:
 [source,js]
 --------------------------------------------------
 # Local
-curl localhost:9200/_nodes/_local
+GET /_nodes/_local
 # Address
-curl localhost:9200/_nodes/10.0.0.3,10.0.0.4
-curl localhost:9200/_nodes/10.0.0.*
+GET /_nodes/10.0.0.3,10.0.0.4
+GET /_nodes/10.0.0.*
 # Names
-curl localhost:9200/_nodes/node_name_goes_here
-curl localhost:9200/_nodes/node_name_goes_*
-# Attributes (set something like node.rack: 2 in the config)
-curl localhost:9200/_nodes/rack:2
-curl localhost:9200/_nodes/ra*:2
-curl localhost:9200/_nodes/ra*:2*
+GET /_nodes/node_name_goes_here
+GET /_nodes/node_name_goes_*
+# Attributes (set something like node.attr.rack: 2 in the config)
+GET /_nodes/rack:2
+GET /_nodes/ra*:2
+GET /_nodes/ra*:2*
 --------------------------------------------------
+// CONSOLE
 --
 
 include::cluster/health.asciidoc[]
@@ -45,6 +46,8 @@ include::cluster/nodes-stats.asciidoc[]
 
 include::cluster/nodes-info.asciidoc[]
 
+include::cluster/remote-info.asciidoc[]
+
 include::cluster/tasks.asciidoc[]
 
 include::cluster/nodes-hot-threads.asciidoc[]
diff --git a/docs/reference/cluster/allocation-explain.asciidoc b/docs/reference/cluster/allocation-explain.asciidoc
index 769a8a4e10d47..8749970aeb27c 100644
--- a/docs/reference/cluster/allocation-explain.asciidoc
+++ b/docs/reference/cluster/allocation-explain.asciidoc
@@ -1,203 +1,307 @@
 [[cluster-allocation-explain]]
 == Cluster Allocation Explain API
 
-The cluster allocation explanation API is designed to assist in answering the
-question "why is this shard unassigned?". To explain the allocation (on
-unassigned state) of a shard, issue a request like:
+The purpose of the cluster allocation explain API is to provide
+explanations for shard allocations in the cluster.  For unassigned shards,
+the explain API provides an explanation for why the shard is unassigned.
+For assigned shards, the explain API provides an explanation for why the
+shard is remaining on its current node and has not moved or rebalanced to
+another node.  This API can be very useful when attempting to diagnose why
+a shard is unassigned or why a shard continues to remain on its current node
+when you might expect otherwise.
 
-experimental[The cluster allocation explain API is new and should still be considered experimental. The API may change in ways that are not backwards compatible]
+[float]
+=== Explain API Request
+
+To explain the allocation of a shard, first an index should exist:
 
 [source,js]
 --------------------------------------------------
-$ curl -XGET 'http://localhost:9200/_cluster/allocation/explain' -d'{
+PUT /myindex
+--------------------------------------------------
+// TESTSETUP
+
+And then the allocation for shards of that index can be explained:
+
+[source,js]
+--------------------------------------------------
+GET /_cluster/allocation/explain
+{
   "index": "myindex",
   "shard": 0,
-  "primary": false
-}'
+  "primary": true
+}
 --------------------------------------------------
+// CONSOLE
 
 Specify the `index` and `shard` id of the shard you would like an explanation
-for, as well as the `primary` flag to indicate whether to explain a primary or
-replica shard.
+for, as well as the `primary` flag to indicate whether to explain the primary
+shard for the given shard id or one of its replica shards.  These three request
+parameters are required.
 
-The response looks like:
+You may also specify an optional `current_node` request parameter to only explain
+a shard that is currently located on `current_node`.  The `current_node` can be
+specified as either the node id or node name.
 
 [source,js]
 --------------------------------------------------
+GET /_cluster/allocation/explain
 {
-  "shard" : {
-    "index" : "myindex",
-    "index_uuid" : "KnW0-zELRs6PK84l0r38ZA",
-    "id" : 0,
-    "primary" : false
-  },
-  "assigned" : false,                             <1>
-  "shard_state_fetch_pending": false,             <2>
+  "index": "myindex",
+  "shard": 0,
+  "primary": false,
+  "current_node": "nodeA"                         <1>
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[skip:no way of knowing the current_node]
+<1> The node where shard 0 currently has a replica on
+
+You can also have Elasticsearch explain the allocation of the first unassigned
+shard that it finds by sending an empty body for the request:
+
+[source,js]
+--------------------------------------------------
+GET /_cluster/allocation/explain
+--------------------------------------------------
+// CONSOLE
+
+[float]
+=== Explain API Response
+
+This section includes examples of the cluster allocation explain API response output
+under various scenarios.
+
+The API response for an unassigned shard:
+
+[source,js]
+--------------------------------------------------
+{
+  "index" : "idx",
+  "shard" : 0,
+  "primary" : true,
+  "current_state" : "unassigned",                 <1>
   "unassigned_info" : {
-    "reason" : "INDEX_CREATED",                   <3>
-    "at" : "2016-03-22T20:04:23.620Z"
+    "reason" : "INDEX_CREATED",                   <2>
+    "at" : "2017-01-04T18:08:16.600Z",
+    "last_allocation_status" : "no"
   },
-  "allocation_delay_ms" : 0,                      <4>
-  "remaining_delay_ms" : 0,                       <5>
-  "nodes" : {
-    "V-Spi0AyRZ6ZvKbaI3691w" : {
-      "node_name" : "node1",
-      "node_attributes" : {                       <6>
-        "bar" : "baz"
-      },
-      "store" : {
-        "shard_copy" : "NONE"                     <7>
-      },
-      "final_decision" : "NO",                    <8>
-      "final_explanation" : "the shard cannot be assigned because one or more allocation decider returns a 'NO' decision",
-      "weight" : 0.06666675,                      <9>
-      "decisions" : [ {                           <10>
-        "decider" : "filter",
-        "decision" : "NO",
-        "explanation" : "node does not match index include filters [foo:\"bar\"]"
-      } ]
-    },
-    "Qc6VL8c5RWaw1qXZ0Rg57g" : {
-      "node_name" : "node2",
-      "node_attributes" : {
-        "bar" : "baz",
-        "foo" : "bar"
-      },
-      "store" : {
-        "shard_copy" : "AVAILABLE"
-      },
-      "final_decision" : "NO",
-      "final_explanation" : "the shard cannot be assigned because one or more allocation decider returns a 'NO' decision",
-      "weight" : -1.3833332,
-      "decisions" : [ {
-        "decider" : "same_shard",
-        "decision" : "NO",
-        "explanation" : "the shard cannot be allocated on the same node id [Qc6VL8c5RWaw1qXZ0Rg57g] on which it already exists"
-      } ]
+  "can_allocate" : "no",                          <3>
+  "allocate_explanation" : "cannot allocate because allocation is not permitted to any of the nodes",
+  "node_allocation_decisions" : [
+    {
+      "node_id" : "8qt2rY-pT6KNZB3-hGfLnw",
+      "node_name" : "node_t1",
+      "transport_address" : "127.0.0.1:9401",
+      "node_decision" : "no",                     <4>
+      "weight_ranking" : 1,
+      "deciders" : [
+        {
+          "decider" : "filter",                   <5>
+          "decision" : "NO",
+          "explanation" : "node does not match index setting [index.routing.allocation.include] filters [_name:\"non_existent_node\"]"  <6>
+        }
+      ]
     },
-    "PzdyMZGXQdGhqTJHF_hGgA" : {
-      "node_name" : "node3",
-      "node_attributes" : { },
-      "store" : {
-        "shard_copy" : "NONE"
-      },
-      "final_decision" : "NO",
-      "final_explanation" : "the shard cannot be assigned because one or more allocation decider returns a 'NO' decision",
-      "weight" : 2.3166666,
-      "decisions" : [ {
-        "decider" : "filter",
-        "decision" : "NO",
-        "explanation" : "node does not match index include filters [foo:\"bar\"]"
-      } ]
+    {
+      "node_id" : "7Wr-QxLXRLKDxhzNm50pFA",
+      "node_name" : "node_t0",
+      "transport_address" : "127.0.0.1:9400",
+      "node_decision" : "no",
+      "weight_ranking" : 2,
+      "deciders" : [
+        {
+          "decider" : "filter",
+          "decision" : "NO",
+          "explanation" : "node does not match index setting [index.routing.allocation.include] filters [_name:\"non_existent_node\"]"
+        }
+      ]
     }
-  }
+  ]
 }
 --------------------------------------------------
-<1> Whether the shard is assigned or unassigned
-<2> Whether information about the shard is still being fetched
-<3> Reason for the shard originally becoming unassigned
-<4> Configured delay before the shard can be allocated
-<5> Remaining delay before the shard can be allocated
-<6> User-added attributes the node has
-<7> The shard copy information for this node and error (if applicable)
-<8> Final decision and explanation of whether the shard can be allocated to this node
-<9> Weight for how much the allocator would like to allocate the shard to this node
-<10> List of node decisions factoring into final decision about the shard
+<1> The current state of the shard
+<2> The reason for the shard originally becoming unassigned
+<3> Whether to allocate the shard
+<4> Whether to allocate the shard to the particular node
+<5> The decider which led to the `no` decision for the node
+<6> An explanation as to why the decider returned a `no` decision, with a helpful hint pointing to the setting that led to the decision
 
-For a shard that is already assigned, the output looks similar to:
+You can return information gathered by the cluster info service about disk usage
+and shard sizes by setting the `include_disk_info` parameter to `true`:
+
+[source,js]
+--------------------------------------------------
+GET /_cluster/allocation/explain?include_disk_info=true
+--------------------------------------------------
+// CONSOLE
+
+Additionally, if you would like to include all decisions that were factored into the final
+decision, the `include_yes_decisions` parameter will return all decisions for each node:
+
+[source,js]
+--------------------------------------------------
+GET /_cluster/allocation/explain?include_yes_decisions=true
+--------------------------------------------------
+// CONSOLE
+
+The default value for `include_yes_decisions` is `false`, which will only
+include the `no` decisions in the response.  This is generally what you would
+want, as the `no` decisions indicate why a shard is unassigned or cannot be moved,
+and including all decisions include the `yes` ones adds a lot of verbosity to the
+API's response output.
+
+The API response output for an unassigned primary shard that had previously been
+allocated to a node in the cluster:
 
 [source,js]
 --------------------------------------------------
 {
-  "shard" : {
-    "index" : "only-foo",
-    "index_uuid" : "KnW0-zELRs6PK84l0r38ZA",
-    "id" : 0,
-    "primary" : true
+  "index" : "idx",
+  "shard" : 0,
+  "primary" : true,
+  "current_state" : "unassigned",
+  "unassigned_info" : {
+    "reason" : "NODE_LEFT",
+    "at" : "2017-01-04T18:03:28.464Z",
+    "details" : "node_left[OIWe8UhhThCK0V5XfmdrmQ]",
+    "last_allocation_status" : "no_valid_shard_copy"
   },
-  "assigned" : true,
-  "assigned_node_id" : "Qc6VL8c5RWaw1qXZ0Rg57g",      <1>
-  "shard_state_fetch_pending": false,
-  "allocation_delay_ms" : 0,
-  "remaining_delay_ms" : 0,
-  "nodes" : {
-    "V-Spi0AyRZ6ZvKbaI3691w" : {
-      "node_name" : "Susan Storm",
-      "node_attributes" : {
-        "bar" : "baz"
-      },
-      "store" : {
-        "shard_copy" : "NONE"
-      },
-      "final_decision" : "NO",
-      "final_explanation" : "the shard cannot be assigned because one or more allocation decider returns a 'NO' decision",
-      "weight" : 1.4499999,
-      "decisions" : [ {
-        "decider" : "filter",
-        "decision" : "NO",
-        "explanation" : "node does not match index include filters [foo:\"bar\"]"
-      } ]
-    },
-    "Qc6VL8c5RWaw1qXZ0Rg57g" : {
-      "node_name" : "Slipstream",
-      "node_attributes" : {
-        "bar" : "baz",
-        "foo" : "bar"
-      },
-      "store" : {
-        "shard_copy" : "AVAILABLE"
-      },
-      "final_decision" : "ALREADY_ASSIGNED",        <2>
-      "final_explanation" : "the shard is already assigned to this node",
-      "weight" : 0.0,
-      "decisions" : [ {
-        "decider" : "same_shard",
-        "decision" : "NO",
-        "explanation" : "the shard cannot be allocated on the same node id [Qc6VL8c5RWaw1qXZ0Rg57g] on which it already exists"
-      } ]
-    },
-    "PzdyMZGXQdGhqTJHF_hGgA" : {
-      "node_name" : "The Symbiote",
-      "node_attributes" : { },
-      "store" : {
-        "shard_copy" : "NONE"
-      },
-      "final_decision" : "NO",
-      "final_explanation" : "the shard cannot be assigned because one or more allocation decider returns a 'NO' decision",
-      "weight" : 3.6999998,
-      "decisions" : [ {
-        "decider" : "filter",
-        "decision" : "NO",
-        "explanation" : "node does not match index include filters [foo:\"bar\"]"
-      } ]
-    }
-  }
+  "can_allocate" : "no_valid_shard_copy",
+  "allocate_explanation" : "cannot allocate because a previous copy of the primary shard existed but can no longer be found on the nodes in the cluster"
 }
 --------------------------------------------------
-<1> Node the shard is currently assigned to
-<2> The decision is "ALREADY_ASSIGNED" because the shard is currently assigned to this node
 
-You can also have Elasticsearch explain the allocation of the first unassigned
-shard it finds by sending an empty body, such as:
+The API response output for a replica that is unassigned due to delayed allocation:
 
 [source,js]
 --------------------------------------------------
-$ curl -XGET 'http://localhost:9200/_cluster/allocation/explain'
+{
+  "index" : "idx",
+  "shard" : 0,
+  "primary" : false,
+  "current_state" : "unassigned",
+  "unassigned_info" : {
+    "reason" : "NODE_LEFT",
+    "at" : "2017-01-04T18:53:59.498Z",
+    "details" : "node_left[G92ZwuuaRY-9n8_tc-IzEg]",
+    "last_allocation_status" : "no_attempt"
+  },
+  "can_allocate" : "allocation_delayed",
+  "allocate_explanation" : "cannot allocate because the cluster is still waiting 59.8s for the departed node holding a replica to rejoin, despite being allowed to allocate the shard to at least one other node",
+  "configured_delay" : "1m",                      <1>
+  "configured_delay_in_millis" : 60000,
+  "remaining_delay" : "59.8s",                    <2>
+  "remaining_delay_in_millis" : 59824,
+  "node_allocation_decisions" : [
+    {
+      "node_id" : "pmnHu_ooQWCPEFobZGbpWw",
+      "node_name" : "node_t2",
+      "transport_address" : "127.0.0.1:9402",
+      "node_decision" : "yes"
+    },
+    {
+      "node_id" : "3sULLVJrRneSg0EfBB-2Ew",
+      "node_name" : "node_t0",
+      "transport_address" : "127.0.0.1:9400",
+      "node_decision" : "no",
+      "store" : {                                 <3>
+        "matching_size" : "4.2kb",
+        "matching_size_in_bytes" : 4325
+      },
+      "deciders" : [
+        {
+          "decider" : "same_shard",
+          "decision" : "NO",
+          "explanation" : "the shard cannot be allocated to the same node on which a copy of the shard already exists [[idx][0], node[3sULLVJrRneSg0EfBB-2Ew], [P], s[STARTED], a[id=eV9P8BN1QPqRc3B4PLx6cg]]"
+        }
+      ]
+    }
+  ]
+}
 --------------------------------------------------
+<1> The configured delay before allocating a replica shard that does not exist due to the node holding it leaving the cluster
+<2> The remaining delay before allocating the replica shard
+<3> Information about the shard data found on a node
 
-If you would like to include all decisions that were factored into the final
-decision, the `include_yes_decisions` parameter will return all decisions:
+The API response output for an assigned shard that is not allowed to
+remain on its current node and is required to move:
 
 [source,js]
 --------------------------------------------------
-$ curl -XGET 'http://localhost:9200/_cluster/allocation/explain?include_yes_decisions=true'
+{
+  "index" : "idx",
+  "shard" : 0,
+  "primary" : true,
+  "current_state" : "started",
+  "current_node" : {
+    "id" : "8lWJeJ7tSoui0bxrwuNhTA",
+    "name" : "node_t1",
+    "transport_address" : "127.0.0.1:9401"
+  },
+  "can_remain_on_current_node" : "no",            <1>
+  "can_remain_decisions" : [                      <2>
+    {
+      "decider" : "filter",
+      "decision" : "NO",
+      "explanation" : "node does not match index setting [index.routing.allocation.include] filters [_name:\"non_existent_node\"]"
+    }
+  ],
+  "can_move_to_other_node" : "no",                <3>
+  "move_explanation" : "cannot move shard to another node, even though it is not allowed to remain on its current node",
+  "node_allocation_decisions" : [
+    {
+      "node_id" : "_P8olZS8Twax9u6ioN-GGA",
+      "node_name" : "node_t0",
+      "transport_address" : "127.0.0.1:9400",
+      "node_decision" : "no",
+      "weight_ranking" : 1,
+      "deciders" : [
+        {
+          "decider" : "filter",
+          "decision" : "NO",
+          "explanation" : "node does not match index setting [index.routing.allocation.include] filters [_name:\"non_existent_node\"]"
+        }
+      ]
+    }
+  ]
+}
 --------------------------------------------------
+<1> Whether the shard is allowed to remain on its current node
+<2> The deciders that factored into the decision of why the shard is not allowed to remain on its current node
+<3> Whether the shard is allowed to be allocated to another node
 
-Additionally, you can return information gathered by the cluster info service
-about disk usage and shard sizes by setting the `include_disk_info` parameter to
-`true`:
+The API response output for an assigned shard that remains on its current node
+because moving the shard to another node does not form a better cluster balance:
 
 [source,js]
 --------------------------------------------------
-$ curl -XGET 'http://localhost:9200/_cluster/allocation/explain?include_disk_info=true'
+{
+  "index" : "idx",
+  "shard" : 0,
+  "primary" : true,
+  "current_state" : "started",
+  "current_node" : {
+    "id" : "wLzJm4N4RymDkBYxwWoJsg",
+    "name" : "node_t0",
+    "transport_address" : "127.0.0.1:9400",
+    "weight_ranking" : 1
+  },
+  "can_remain_on_current_node" : "yes",
+  "can_rebalance_cluster" : "yes",                <1>
+  "can_rebalance_to_other_node" : "no",           <2>
+  "rebalance_explanation" : "cannot rebalance as no target node exists that can both allocate this shard and improve the cluster balance",
+  "node_allocation_decisions" : [
+    {
+      "node_id" : "oE3EGFc8QN-Tdi5FFEprIA",
+      "node_name" : "node_t1",
+      "transport_address" : "127.0.0.1:9401",
+      "node_decision" : "worse_balance",          <3>
+      "weight_ranking" : 1
+    }
+  ]
+}
 --------------------------------------------------
+<1> Whether rebalancing is allowed on the cluster
+<2> Whether the shard can be rebalanced to another node
+<3> The reason the shard cannot be rebalanced to the node, in this case indicating that it offers no better balance than the current node
diff --git a/docs/reference/cluster/health.asciidoc b/docs/reference/cluster/health.asciidoc
index dc73b4408e621..f6fb84ea9297c 100644
--- a/docs/reference/cluster/health.asciidoc
+++ b/docs/reference/cluster/health.asciidoc
@@ -34,7 +34,7 @@ Returns this:
   "active_shards_percent_as_number": 50.0
 }
 --------------------------------------------------
-// TESTRESPONSE[s/testcluster/docs_integTest/]
+// TESTRESPONSE[s/testcluster/docs_integTestCluster/]
 // TESTRESPONSE[s/"number_of_pending_tasks" : 0,/"number_of_pending_tasks" : $body.number_of_pending_tasks,/]
 // TESTRESPONSE[s/"task_max_waiting_in_queue_millis": 0/"task_max_waiting_in_queue_millis": $body.task_max_waiting_in_queue_millis/]
 
diff --git a/docs/reference/cluster/nodes-info.asciidoc b/docs/reference/cluster/nodes-info.asciidoc
index 77f77da3c897a..1d572eb31fdd1 100644
--- a/docs/reference/cluster/nodes-info.asciidoc
+++ b/docs/reference/cluster/nodes-info.asciidoc
@@ -26,9 +26,6 @@ By default, it just returns all attributes and core settings for a node:
 `host`::
 	The node's host name.
 
-`http_address`::
-	Host and port where primary HTTP connections are accepted.
-
 `ip`::
 	The node's IP address.
 
@@ -113,45 +110,46 @@ the current running process:
 [[plugins-info]]
 ==== Plugins information
 
-`plugins` - if set, the result will contain details about the loaded
-plugins per node:
+`plugins` - if set, the result will contain details about the installed plugins
+ per node:
 
 * `name`: plugin name
-* `description`: plugin description if any
-* `site`: `true` if the plugin is a site plugin
-* `jvm`: `true` if the plugin is a plugin running in the JVM
-* `url`: URL if the plugin is a site plugin
+* `version`: version of Elasticsearch the plugin was built for
+* `description`: short description of the plugin's purpose
+* `classname`: fully-qualified class name of the plugin's entry point
+* `has_native_controller`: whether or not the plugin has a native controller process
 
 The result will look similar to:
 
 [source,js]
 --------------------------------------------------
 {
-  "cluster_name" : "test-cluster-MacBook-Air-de-David.local",
-  "nodes" : {
-    "hJLXmY_NTrCytiIMbX4_1g" : {
-      "name" : "node4",
-      "transport_address" : "inet[/172.18.58.139:9303]",
-      "hostname" : "MacBook-Air-de-David.local",
-      "version" : "0.90.0.Beta2-SNAPSHOT",
-      "http_address" : "inet[/172.18.58.139:9203]",
-      "plugins" : [ {
-        "name" : "test-plugin",
-        "description" : "test-plugin description",
-        "site" : true,
-        "jvm" : false
-      }, {
-        "name" : "test-no-version-plugin",
-        "description" : "test-no-version-plugin description",
-        "site" : true,
-        "jvm" : false
-      }, {
-        "name" : "dummy",
-        "description" : "No description found for dummy.",
-        "url" : "/_plugin/dummy/",
-        "site" : false,
-        "jvm" : true
-      } ]
+  "cluster_name": "elasticsearch",
+  "nodes": {
+    "O70_wBv6S9aPPcAKdSUBtw": {
+      "plugins": [
+        {
+          "name": "analysis-icu",
+          "version": "{version}",
+          "description": "The ICU Analysis plugin integrates Lucene ICU module into elasticsearch, adding ICU relates analysis components.",
+          "classname": "org.elasticsearch.plugin.analysis.icu.AnalysisICUPlugin",
+          "has_native_controller": false
+        },
+        {
+          "name": "ingest-geoip",
+          "version": "{version}",
+          "description": "Ingest processor that uses looksup geo data based on ip adresses using the Maxmind geo database",
+          "classname": "org.elasticsearch.ingest.geoip.IngestGeoIpPlugin",
+          "has_native_controller": false
+        },
+        {
+          "name": "ingest-user-agent",
+          "version": "{version}",
+          "description": "Ingest processor that extracts information from a user agent",
+          "classname": "org.elasticsearch.ingest.useragent.IngestUserAgentPlugin",
+          "has_native_controller": false
+        }
+      ]
     }
   }
 }
diff --git a/docs/reference/cluster/nodes-stats.asciidoc b/docs/reference/cluster/nodes-stats.asciidoc
index e49e2f6b8aadc..793a056b3ebb9 100644
--- a/docs/reference/cluster/nodes-stats.asciidoc
+++ b/docs/reference/cluster/nodes-stats.asciidoc
@@ -9,9 +9,10 @@ the cluster nodes statistics.
 
 [source,js]
 --------------------------------------------------
-curl -XGET 'http://localhost:9200/_nodes/stats'
-curl -XGET 'http://localhost:9200/_nodes/nodeId1,nodeId2/stats'
+GET /_nodes/stats
+GET /_nodes/nodeId1,nodeId2/stats
 --------------------------------------------------
+// CONSOLE
 
 The first command retrieves stats of all the nodes in the cluster. The
 second command selectively retrieves nodes stats of only `nodeId1` and
@@ -24,57 +25,59 @@ of `indices`, `os`, `process`, `jvm`, `transport`, `http`,
 
 [horizontal]
 `indices`::
-	Indices stats about size, document count, indexing and
-	deletion times, search times, field cache size, merges and flushes
+  Indices stats about size, document count, indexing and
+  deletion times, search times, field cache size, merges and flushes
 
 `fs`::
-	File system information, data path, free disk space, read/write
-	stats (see <<fs-info,FS information>>)
+  File system information, data path, free disk space, read/write
+  stats (see <<fs-info,FS information>>)
 
 `http`::
-	HTTP connection information
+  HTTP connection information
 
 `jvm`::
-	JVM stats, memory pool information, garbage collection, buffer
-	pools, number of loaded/unloaded classes
+  JVM stats, memory pool information, garbage collection, buffer
+  pools, number of loaded/unloaded classes
 
 `os`::
-	Operating system stats, load average, mem, swap
-	(see <<os-stats,OS statistics>>)
+  Operating system stats, load average, mem, swap
+  (see <<os-stats,OS statistics>>)
 
 `process`::
-	Process statistics, memory consumption, cpu usage, open
-	file descriptors (see <<process-stats,Process statistics>>)
+  Process statistics, memory consumption, cpu usage, open
+  file descriptors (see <<process-stats,Process statistics>>)
 
 `thread_pool`::
-	Statistics about each thread pool, including current
-	size, queue and rejected tasks
+  Statistics about each thread pool, including current
+  size, queue and rejected tasks
 
 `transport`::
-	Transport statistics about sent and received bytes in
-	cluster communication
+  Transport statistics about sent and received bytes in
+  cluster communication
 
 `breaker`::
-	Statistics about the field data circuit breaker
+  Statistics about the field data circuit breaker
 
 `discovery`::
-	Statistics about the discovery
+  Statistics about the discovery
 
 `ingest`::
     Statistics about ingest preprocessing
 
 [source,js]
 --------------------------------------------------
-# return indices and os
-curl -XGET 'http://localhost:9200/_nodes/stats/os'
+# return just indices
+GET /_nodes/stats/indices
+
 # return just os and process
-curl -XGET 'http://localhost:9200/_nodes/stats/os,process'
-# specific type endpoint
-curl -XGET 'http://localhost:9200/_nodes/stats/process'
-curl -XGET 'http://localhost:9200/_nodes/10.0.0.1/stats/process'
+GET /_nodes/stats/os,process
+
+# return just process for node with IP address 10.0.0.1
+GET /_nodes/10.0.0.1/stats/process
 --------------------------------------------------
+// CONSOLE
 
-The `all` flag can be set to return all the stats.
+All stats can be explicitly requested via `/_nodes/stats/_all` or `/_nodes/stats?metric=_all`.
 
 [float]
 [[fs-info]]
@@ -84,42 +87,42 @@ The `fs` flag can be set to retrieve
 information that concern the file system:
 
 `fs.timestamp`::
-	Last time the file stores statistics have been refreshed
+  Last time the file stores statistics have been refreshed
 
 `fs.total.total_in_bytes`::
-	Total size (in bytes) of all file stores
+  Total size (in bytes) of all file stores
 
 `fs.total.free_in_bytes`::
-	Total number of unallocated bytes in all file stores
+  Total number of unallocated bytes in all file stores
 
 `fs.total.available_in_bytes`::
-	Total number of bytes available to this Java virtual machine on all file stores
+  Total number of bytes available to this Java virtual machine on all file stores
 
 `fs.data`::
-	List of all file stores
+  List of all file stores
 
 `fs.data.path`::
-	Path to the file store
+  Path to the file store
 
 `fs.data.mount`::
-	Mount point of the file store (ex: /dev/sda2)
+  Mount point of the file store (ex: /dev/sda2)
 
 `fs.data.type`::
-	Type of the file store (ex: ext4)
+  Type of the file store (ex: ext4)
 
 `fs.data.total_in_bytes`::
-	Total size (in bytes) of the file store
+  Total size (in bytes) of the file store
 
 `fs.data.free_in_bytes`::
-	Total number of unallocated bytes in the file store
+  Total number of unallocated bytes in the file store
 
 `fs.data.available_in_bytes`::
-	Total number of bytes available to this Java virtual machine on this file store
+  Total number of bytes available to this Java virtual machine on this file store
 
 `fs.data.spins` (Linux only)::
-	Indicates if the file store is backed by spinning storage.
-	`null` means we could not determine it, `true` means the device possibly spins
-	 and `false` means it does not (ex: solid-state disks).
+  Indicates if the file store is backed by spinning storage.
+  `null` means we could not determine it, `true` means the device possibly spins
+   and `false` means it does not (ex: solid-state disks).
 
 `fs.io_stats.devices` (Linux only)::
     Array of disk metrics for each device that is backing an
@@ -178,7 +181,7 @@ The `os` flag can be set to retrieve statistics that concern
 the operating system:
 
 `os.timestamp`::
-	Last time the operating system statistics have been refreshed
+  Last time the operating system statistics have been refreshed
 
 `os.cpu.percent`::
     Recent CPU usage for the whole system, or -1 if not supported
@@ -194,29 +197,66 @@ the operating system:
     fifteen-minute load average is not available)
 
 `os.mem.total_in_bytes`::
-	Total amount of physical memory in bytes
+  Total amount of physical memory in bytes
 
 `os.mem.free_in_bytes`::
-	Amount of free physical memory in bytes
+  Amount of free physical memory in bytes
 
 `os.mem.free_percent`::
-	Percentage of free memory
+  Percentage of free memory
 
 `os.mem.used_in_bytes`::
-	Amount of used physical memory in bytes
+  Amount of used physical memory in bytes
 
 `os.mem.used_percent`::
-	Percentage of used memory
+  Percentage of used memory
 
 `os.swap.total_in_bytes`::
-	Total amount of swap space in bytes
+  Total amount of swap space in bytes
 
 `os.swap.free_in_bytes`::
-	Amount of free swap space in bytes
+  Amount of free swap space in bytes
 
 `os.swap.used_in_bytes`::
-	Amount of used swap space in bytes
+  Amount of used swap space in bytes
+
+`os.cgroup.cpuacct.control_group` (Linux only)::
+    The `cpuacct` control group to which the Elasticsearch process
+    belongs
+
+`os.cgroup.cpuacct.usage_nanos` (Linux only)::
+    The total CPU time (in nanoseconds) consumed by all tasks in the
+    same cgroup as the Elasticsearch process
+
+`os.cgroup.cpu.control_group` (Linux only)::
+    The `cpu` control group to which the Elasticsearch process belongs
+
+`os.cgroup.cpu.cfs_period_micros` (Linux only)::
+    The period of time (in microseconds) for how regularly all tasks in
+    the same cgroup as the Elasticsearch process should have their
+    access to CPU resources reallocated.
 
+`os.cgroup.cpu.cfs_quota_micros` (Linux only)::
+    The total amount of time (in microseconds) for which all tasks in
+    the same cgroup as the Elasticsearch process can run during one
+    period `os.cgroup.cpu.cfs_period_micros`
+
+`os.cgroup.cpu.stat.number_of_elapsed_periods` (Linux only)::
+    The number of reporting periods (as specified by
+    `os.cgroup.cpu.cfs_period_micros`) that have elapsed
+
+`os.cgroup.cpu.stat.number_of_times_throttled` (Linux only)::
+    The number of times all tasks in the same cgroup as the
+    Elasticsearch process have been throttled.
+
+`os.cgroup.cpu.stat.time_throttled_nanos` (Linux only)::
+    The total amount of time (in nanoseconds) for which all tasks in
+    the same cgroup as the Elasticsearch process have been throttled.
+
+NOTE: For the cgroup stats to be visible, cgroups must be compiled into
+the kernal, the `cpu` and `cpuacct` cgroup subsystems must be
+configured and stats must be readable from `/sys/fs/cgroup/cpu`
+and `/sys/fs/cgroup/cpuacct`.
 
 [float]
 [[process-stats]]
@@ -226,43 +266,64 @@ The `process` flag can be set to retrieve statistics that concern
 the current running process:
 
 `process.timestamp`::
-	Last time the process statistics have been refreshed
+  Last time the process statistics have been refreshed
 
 `process.open_file_descriptors`::
-	Number of opened file descriptors associated with the current process, or -1 if not supported
+  Number of opened file descriptors associated with the current process, or -1 if not supported
 
 `process.max_file_descriptors`::
-	Maximum number of file descriptors allowed on the system, or -1 if not supported
+  Maximum number of file descriptors allowed on the system, or -1 if not supported
 
 `process.cpu.percent`::
-	CPU usage in percent, or -1 if not known at the time the stats are computed
+  CPU usage in percent, or -1 if not known at the time the stats are computed
 
 `process.cpu.total_in_millis`::
-	CPU time (in milliseconds) used by the process on which the Java virtual machine is running, or -1 if not supported
+  CPU time (in milliseconds) used by the process on which the Java virtual machine is running, or -1 if not supported
 
 `process.mem.total_virtual_in_bytes`::
-	Size in bytes of virtual memory that is guaranteed to be available to the running process
-
+  Size in bytes of virtual memory that is guaranteed to be available to the running process
 
 [float]
-[[field-data]]
-=== Field data statistics
+[[node-indices-stats]]
+=== Indices statistics
 
-You can get information about field data memory usage on node
-level or on index level.
+You can get information about indices stats on `node`, `indices`, or `shards` level.
 
 [source,js]
 --------------------------------------------------
-# Node Stats
-curl -XGET 'http://localhost:9200/_nodes/stats/indices/?fields=field1,field2&pretty'
+# Fielddata summarised by node
+GET /_nodes/stats/indices/fielddata?fields=field1,field2
+
+# Fielddata summarised by node and index
+GET /_nodes/stats/indices/fielddata?level=indices&fields=field1,field2
 
-# Indices Stat
-curl -XGET 'http://localhost:9200/_stats/fielddata/?fields=field1,field2&pretty'
+# Fielddata summarised by node, index, and shard
+GET /_nodes/stats/indices/fielddata?level=shards&fields=field1,field2
 
 # You can use wildcards for field names
-curl -XGET 'http://localhost:9200/_stats/fielddata/?fields=field*&pretty'
-curl -XGET 'http://localhost:9200/_nodes/stats/indices/?fields=field*&pretty'
+GET /_nodes/stats/indices/fielddata?fields=field*
 --------------------------------------------------
+// CONSOLE
+
+Supported metrics are:
+
+* `completion`
+* `docs`
+* `fielddata`
+* `flush`
+* `get`
+* `indexing`
+* `merge`
+* `query_cache`
+* `recovery`
+* `refresh`
+* `request_cache`
+* `search`
+* `segments`
+* `store`
+* `suggest`
+* `translog`
+* `warmer`
 
 [float]
 [[search-groups]]
@@ -274,11 +335,12 @@ on this node.
 [source,js]
 --------------------------------------------------
 # All groups with all stats
-curl -XGET 'http://localhost:9200/_nodes/stats?pretty&groups=_all'
+GET /_nodes/stats?groups=_all
 
 # Some groups from just the indices stats
-curl -XGET 'http://localhost:9200/_nodes/stats/indices?pretty&groups=foo,bar'
+GET /_nodes/stats/indices?groups=foo,bar
 --------------------------------------------------
+// CONSOLE
 
 [float]
 [[ingest-stats]]
diff --git a/docs/reference/cluster/remote-info.asciidoc b/docs/reference/cluster/remote-info.asciidoc
new file mode 100644
index 0000000000000..304e7c8de0b4c
--- /dev/null
+++ b/docs/reference/cluster/remote-info.asciidoc
@@ -0,0 +1,35 @@
+[[cluster-remote-info]]
+== Remote Cluster Info
+
+The cluster remote info API allows to retrieve all of the configured
+remote cluster information.
+
+[source,js]
+----------------------------------
+GET /_remote/info
+----------------------------------
+// CONSOLE
+
+This command returns returns connection and endpoint information keyed by
+the configured remote cluster alias.
+
+[float]
+[[connection-info]]
+
+`seeds`::
+	The configured initial seed transport addresses of the remote cluster.
+
+`http_addresses`::
+	The published http addresses of all connected remote nodes.
+
+`connected`::
+	True if there is at least one connection to the remote cluster.
+
+`num_nodes_connected`::
+    The number of connected nodes in the remote cluster.
+
+`max_connection_per_cluster`::
+	The maximum number of connections maintained for the remote cluster.
+
+`initial_connect_timeout`::
+	The initial connect timeout for remote cluster connections.
diff --git a/docs/reference/cluster/reroute.asciidoc b/docs/reference/cluster/reroute.asciidoc
index bb48a00fbe53d..aba51f4893f24 100644
--- a/docs/reference/cluster/reroute.asciidoc
+++ b/docs/reference/cluster/reroute.asciidoc
@@ -11,22 +11,26 @@ Here is a short example of how a simple reroute API call:
 
 [source,js]
 --------------------------------------------------
-curl -XPOST 'localhost:9200/_cluster/reroute' -d '{
-    "commands" : [ {
-        "move" :
-            {
-              "index" : "test", "shard" : 0,
-              "from_node" : "node1", "to_node" : "node2"
+POST /_cluster/reroute
+{
+    "commands" : [
+        {
+            "move" : {
+                "index" : "test", "shard" : 0,
+                "from_node" : "node1", "to_node" : "node2"
             }
         },
         {
           "allocate_replica" : {
-              "index" : "test", "shard" : 1, "node" : "node3"
+                "index" : "test", "shard" : 1,
+                "node" : "node3"
           }
         }
     ]
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[skip:doc tests run with only a single node]
 
 An important aspect to remember is the fact that once when an allocation
 occurs, the cluster will aim at re-balancing its state back to an even
@@ -114,5 +118,5 @@ structural problems such as having an analyzer which refers to a stopwords
 file which doesn't exist on all nodes.
 
 Once the problem has been corrected, allocation can be manually retried by
-calling the <<cluster-reroute,`_reroute`>> API with `?retry_failed`, which
-will attempt a single retry round for these shards.
\ No newline at end of file
+calling the <<cluster-reroute,`reroute`>> API with `?retry_failed`, which
+will attempt a single retry round for these shards.
diff --git a/docs/reference/cluster/stats.asciidoc b/docs/reference/cluster/stats.asciidoc
index a95273f85f9bc..e4bed585ead5e 100644
--- a/docs/reference/cluster/stats.asciidoc
+++ b/docs/reference/cluster/stats.asciidoc
@@ -167,13 +167,27 @@ Will return, for example:
          "available_in_bytes": 34802409472
       },
       "plugins": [
-         // all plugins installed on nodes
-         {
-            "name": "analysis-stempel",
-            "version": "{version}",
-            "description": "The Stempel (Polish) Analysis plugin integrates Lucene stempel (polish) analysis module into elasticsearch.",
-            "classname": "org.elasticsearch.plugin.analysis.stempel.AnalysisStempelPlugin"
-         }
+        {
+          "name": "analysis-icu",
+          "version": "{version}",
+          "description": "The ICU Analysis plugin integrates Lucene ICU module into elasticsearch, adding ICU relates analysis components.",
+          "classname": "org.elasticsearch.plugin.analysis.icu.AnalysisICUPlugin",
+          "has_native_controller": false
+        },
+        {
+          "name": "ingest-geoip",
+          "version": "{version}",
+          "description": "Ingest processor that uses looksup geo data based on ip adresses using the Maxmind geo database",
+          "classname": "org.elasticsearch.ingest.geoip.IngestGeoIpPlugin",
+          "has_native_controller": false
+        },
+        {
+          "name": "ingest-user-agent",
+          "version": "{version}",
+          "description": "Ingest processor that extracts information from a user agent",
+          "classname": "org.elasticsearch.ingest.useragent.IngestUserAgentPlugin",
+          "has_native_controller": false
+        }
       ]
    }
 }
diff --git a/docs/reference/cluster/tasks.asciidoc b/docs/reference/cluster/tasks.asciidoc
index 5720227f9f049..f1f9a66931285 100644
--- a/docs/reference/cluster/tasks.asciidoc
+++ b/docs/reference/cluster/tasks.asciidoc
@@ -28,7 +28,7 @@ The result will look similar to the following:
 {
   "nodes" : {
     "oTUltX4IQMOUUVeiohTt8A" : {
-      "name" : "Tamara Rahn",
+      "name" : "H5dfFeA",
       "transport_address" : "127.0.0.1:9300",
       "host" : "127.0.0.1",
       "ip" : "127.0.0.1:9300",
@@ -62,7 +62,7 @@ It is also possible to retrieve information for a particular task:
 
 [source,js]
 --------------------------------------------------
-GET _tasks/taskId:1 <1>
+GET _tasks/task_id:1 <1>
 --------------------------------------------------
 // CONSOLE
 // TEST[catch:missing]
@@ -79,8 +79,57 @@ GET _tasks?parent_task_id=parentTaskId:1 <1>
 
 <1> This won't return a 404 if the parent isn't found.
 
-The task API can be also used to wait for completion of a particular task. The following call will
-block for 10 seconds or until the task with id `oTUltX4IQMOUUVeiohTt8A:12345` is completed.
+You can also use the `detailed` request parameter to get more information about
+the running tasks. This is useful for telling one task from another but is more
+costly to execute. For example, fetching all searches using the `detailed`
+request parameter:
+
+[source,js]
+--------------------------------------------------
+GET _tasks?actions=*search&detailed
+--------------------------------------------------
+// CONSOLE
+
+might look like:
+
+[source,js]
+--------------------------------------------------
+{
+  "nodes" : {
+    "oTUltX4IQMOUUVeiohTt8A" : {
+      "name" : "H5dfFeA",
+      "transport_address" : "127.0.0.1:9300",
+      "host" : "127.0.0.1",
+      "ip" : "127.0.0.1:9300",
+      "tasks" : {
+        "oTUltX4IQMOUUVeiohTt8A:464" : {
+          "node" : "oTUltX4IQMOUUVeiohTt8A",
+          "id" : 464,
+          "type" : "transport",
+          "action" : "indices:data/read/search",
+          "description" : "indices[test], types[test], search_type[QUERY_THEN_FETCH], source[{\"query\":...}]",
+          "start_time_in_millis" : 1483478610008,
+          "running_time_in_nanos" : 13991383,
+          "cancellable" : true
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+
+The new `description` field contains human readable text that identifies the
+particular request that the task is performing such as identifying the search
+request being performed by a search task like the example above. Other kinds of
+task have have different descriptions, like <<docs-reindex,`_reindex`>> which
+has the search and the destination, or <<docs-bulk,`_bulk`>> which just has the
+number of requests and the destination indices. Many requests will only have an
+empty description because more detailed information about the request is not
+easily available or particularly helpful in identifying the request.
+
+The task API can also be used to wait for completion of a particular task. The
+following call will block for 10 seconds or until the task with id
+`oTUltX4IQMOUUVeiohTt8A:12345` is completed.
 
 [source,js]
 --------------------------------------------------
@@ -98,25 +147,28 @@ GET _tasks?actions=*reindex&wait_for_completion=true&timeout=10s
 --------------------------------------------------
 // CONSOLE
 
-Tasks can be also listed using _cat version of the list tasks command, which accepts the same arguments
-as the standard list tasks command.
+Tasks can be also listed using _cat version of the list tasks command, which
+accepts the same arguments as the standard list tasks command.
 
 [source,js]
 --------------------------------------------------
 GET _cat/tasks
+GET _cat/tasks?detailed
 --------------------------------------------------
 // CONSOLE
 
 [float]
+[[task-cancellation]]
 === Task Cancellation
 
 If a long-running task supports cancellation, it can be cancelled by the following command:
 
 [source,js]
 --------------------------------------------------
-POST _tasks/taskId:1/_cancel
+POST _tasks/node_id:task_id/_cancel
 --------------------------------------------------
 // CONSOLE
+// TEST[s/task_id/1/]
 
 The task cancellation command supports the same task selection parameters as the list tasks command, so multiple tasks
 can be cancelled at the same time. For example, the following command will cancel all reindex tasks running on the
@@ -124,7 +176,7 @@ nodes `nodeId1` and `nodeId2`.
 
 [source,js]
 --------------------------------------------------
-POST _tasks/_cancel?node_id=nodeId1,nodeId2&actions=*reindex
+POST _tasks/_cancel?nodes=nodeId1,nodeId2&actions=*reindex
 --------------------------------------------------
 // CONSOLE
 
diff --git a/docs/reference/cluster/update-settings.asciidoc b/docs/reference/cluster/update-settings.asciidoc
index d0b127cd35225..c8c970bdbedca 100644
--- a/docs/reference/cluster/update-settings.asciidoc
+++ b/docs/reference/cluster/update-settings.asciidoc
@@ -7,23 +7,29 @@ survive a full cluster restart). Here is an example:
 
 [source,js]
 --------------------------------------------------
-curl -XPUT localhost:9200/_cluster/settings -d '{
+PUT /_cluster/settings
+{
     "persistent" : {
-        "discovery.zen.minimum_master_nodes" : 2
+        "indices.recovery.max_bytes_per_sec" : "50mb"
     }
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[warning:[script.max_compilations_per_minute] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version.]
 
 Or:
 
 [source,js]
 --------------------------------------------------
-curl -XPUT localhost:9200/_cluster/settings -d '{
+PUT /_cluster/settings?flat_settings=true
+{
     "transient" : {
-        "discovery.zen.minimum_master_nodes" : 2
+        "indices.recovery.max_bytes_per_sec" : "20mb"
     }
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[warning:[script.max_compilations_per_minute] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version.]
 
 The cluster responds with the settings updated. So the response for the
 last example will be:
@@ -31,12 +37,14 @@ last example will be:
 [source,js]
 --------------------------------------------------
 {
-    "persistent" : {},
+    ...
+    "persistent" : { },
     "transient" : {
-        "discovery.zen.minimum_master_nodes" : "2"
+        "indices.recovery.max_bytes_per_sec" : "20mb"
     }
-}'
+}
 --------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"acknowledged": true,/]
 
 Resetting persistent or transient settings can be done by assigning a
 `null` value. If a transient setting is reset, the persistent setting
@@ -46,12 +54,16 @@ value. Here is an example:
 
 [source,js]
 --------------------------------------------------
-curl -XPUT localhost:9200/_cluster/settings -d '{
+PUT /_cluster/settings
+{
     "transient" : {
-        "discovery.zen.minimum_master_nodes" : null
+        "indices.recovery.max_bytes_per_sec" : null
     }
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[s/^/PUT _cluster\/settings\n{ "transient" : { "indices.recovery.max_bytes_per_sec" : "20mb" } }\n/]
+// TEST[warning:[script.max_compilations_per_minute] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version.]
 
 Reset settings will not be included in the cluster response. So
 the response for the last example will be:
@@ -59,29 +71,36 @@ the response for the last example will be:
 [source,js]
 --------------------------------------------------
 {
+    ...
     "persistent" : {},
     "transient" : {}
 }
 --------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"acknowledged": true,/]
 
 Settings can also be reset using simple wildcards. For instance to reset
-all dynamic `discovery.zen` setting a prefix can be used:
+all dynamic `indices.recovery` setting a prefix can be used:
 
 [source,js]
 --------------------------------------------------
-curl -XPUT localhost:9200/_cluster/settings -d '{
+PUT /_cluster/settings
+{
     "transient" : {
-        "discovery.zen.*" : null
+        "indices.recovery.*" : null
     }
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[s/^/PUT _cluster\/settings\n{ "transient" : { "indices.recovery.max_bytes_per_sec" : "20mb" } }\n/]
+// TEST[warning:[script.max_compilations_per_minute] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version.]
 
 Cluster wide settings can be returned using:
 
 [source,js]
 --------------------------------------------------
-curl -XGET localhost:9200/_cluster/settings
+GET /_cluster/settings
 --------------------------------------------------
+// CONSOLE
 
 [float]
 === Precedence of settings
diff --git a/docs/reference/docs.asciidoc b/docs/reference/docs.asciidoc
index 04049663e848f..a8ab282853c8f 100644
--- a/docs/reference/docs.asciidoc
+++ b/docs/reference/docs.asciidoc
@@ -4,7 +4,8 @@
 [partintro]
 --
 
-This section describes the following CRUD APIs:
+This section starts with a short introduction to Elasticsearch's <<docs-replication,data replication model>>, followed by a
+detailed description of the following CRUD APIs:
 
 .Single document APIs
 * <<docs-index_>>
@@ -15,6 +16,7 @@ This section describes the following CRUD APIs:
 .Multi-document APIs
 * <<docs-multi-get>>
 * <<docs-bulk>>
+* <<docs-delete-by-query>>
 * <<docs-update-by-query>>
 * <<docs-reindex>>
 
@@ -23,6 +25,8 @@ index name, or an `alias` which points to a single index.
 
 --
 
+include::docs/data-replication.asciidoc[]
+
 include::docs/index_.asciidoc[]
 
 include::docs/get.asciidoc[]
diff --git a/docs/reference/docs/bulk.asciidoc b/docs/reference/docs/bulk.asciidoc
index b139e76149002..d17ec21ff7006 100644
--- a/docs/reference/docs/bulk.asciidoc
+++ b/docs/reference/docs/bulk.asciidoc
@@ -12,8 +12,8 @@ bulk requests and reindexing of documents from one index to another:
 
 Perl::
 
-    See https://metacpan.org/pod/Search::Elasticsearch::Bulk[Search::Elasticsearch::Bulk]
-    and https://metacpan.org/pod/Search::Elasticsearch::Scroll[Search::Elasticsearch::Scroll]
+    See https://metacpan.org/pod/Search::Elasticsearch::Client::5_0::Bulk[Search::Elasticsearch::Client::5_0::Bulk]
+    and https://metacpan.org/pod/Search::Elasticsearch::Client::5_0::Scroll[Search::Elasticsearch::Client::5_0::Scroll]
 
 Python::
 
@@ -21,8 +21,8 @@ Python::
 
 *********************************************
 
-The REST API endpoint is `/_bulk`, and it expects the following JSON
-structure:
+The REST API endpoint is `/_bulk`, and it expects the following newline delimited JSON
+(NDJSON) structure:
 
 [source,js]
 --------------------------------------------------
@@ -34,8 +34,11 @@ optional_source\n
 action_and_meta_data\n
 optional_source\n
 --------------------------------------------------
+// NOTCONSOLE
 
-*NOTE*: the final line of data must end with a newline character `\n`.
+*NOTE*: the final line of data must end with a newline character `\n`. Each newline character
+may be preceded by a carriage return `\r`. When sending requests to this endpoint the
+`Content-Type` header should be set to `application/x-ndjson`.
 
 The possible actions are `index`, `create`, `delete` and `update`.
 `index` and `create` expect a source on the next
@@ -56,9 +59,11 @@ newlines. Example:
 $ cat requests
 { "index" : { "_index" : "test", "_type" : "type1", "_id" : "1" } }
 { "field1" : "value1" }
-$ curl -s -XPOST localhost:9200/_bulk --data-binary "@requests"; echo
+$ curl -s -H "Content-Type: application/x-ndjson" -XPOST localhost:9200/_bulk --data-binary "@requests"; echo
 {"took":7, "errors": false, "items":[{"index":{"_index":"test","_type":"type1","_id":"1","_version":1,"result":"created","forced_refresh":false}}]}
 --------------------------------------------------
+// NOTCONSOLE
+// Not converting to console because this shows how curl works
 
 Because this format uses literal `\n`'s as delimiters, please be sure
 that the JSON actions and sources are not pretty printed. Here is an
@@ -66,17 +71,92 @@ example of a correct sequence of bulk commands:
 
 [source,js]
 --------------------------------------------------
+POST _bulk
 { "index" : { "_index" : "test", "_type" : "type1", "_id" : "1" } }
 { "field1" : "value1" }
 { "delete" : { "_index" : "test", "_type" : "type1", "_id" : "2" } }
 { "create" : { "_index" : "test", "_type" : "type1", "_id" : "3" } }
 { "field1" : "value3" }
-{ "update" : {"_id" : "1", "_type" : "type1", "_index" : "index1"} }
+{ "update" : {"_id" : "1", "_type" : "type1", "_index" : "test"} }
 { "doc" : {"field2" : "value2"} }
 --------------------------------------------------
+// CONSOLE
 
-In the above example `doc` for the `update` action is a partial
-document, that will be merged with the already stored document.
+The result of this bulk operation is:
+
+[source,js]
+--------------------------------------------------
+{
+   "took": 30,
+   "errors": false,
+   "items": [
+      {
+         "index": {
+            "_index": "test",
+            "_type": "type1",
+            "_id": "1",
+            "_version": 1,
+            "result": "created",
+            "_shards": {
+               "total": 2,
+               "successful": 1,
+               "failed": 0
+            },
+            "created": true,
+            "status": 201
+         }
+      },
+      {
+         "delete": {
+            "found": false,
+            "_index": "test",
+            "_type": "type1",
+            "_id": "2",
+            "_version": 1,
+            "result": "not_found",
+            "_shards": {
+               "total": 2,
+               "successful": 1,
+               "failed": 0
+            },
+            "status": 404
+         }
+      },
+      {
+         "create": {
+            "_index": "test",
+            "_type": "type1",
+            "_id": "3",
+            "_version": 1,
+            "result": "created",
+            "_shards": {
+               "total": 2,
+               "successful": 1,
+               "failed": 0
+            },
+            "created": true,
+            "status": 201
+         }
+      },
+      {
+         "update": {
+            "_index": "test",
+            "_type": "type1",
+            "_id": "1",
+            "_version": 2,
+            "result": "updated",
+            "_shards": {
+                "total": 2,
+                "successful": 1,
+                "failed": 0
+            },
+            "status": 200
+         }
+      }
+   ]
+}
+--------------------------------------------------
+// TESTRESPONSE[s/"took": 30/"took": $body.took/ s/"index_uuid": .../"index_uuid": $body.items.3.update.error.index_uuid/]
 
 The endpoints are `/_bulk`, `/{index}/_bulk`, and `{index}/{type}/_bulk`.
 When the index or the index/type are provided, they will be used by
@@ -153,23 +233,25 @@ the action itself (not in the extra payload line), to specify how many
 times an update should be retried in the case of a version conflict.
 
 The `update` action payload, supports the following options: `doc`
-(partial document), `upsert`, `doc_as_upsert`, `script`, `params` (for
-script), `lang` (for script) and `fields`. See update documentation for details on
-the options. Curl example with update actions:
+(partial document), `upsert`, `doc_as_upsert`, `script` and `_source`. See <<docs-update,update>> documentation for details on
+the options. Example with update actions:
 
 [source,js]
 --------------------------------------------------
+POST _bulk
 { "update" : {"_id" : "1", "_type" : "type1", "_index" : "index1", "_retry_on_conflict" : 3} }
 { "doc" : {"field" : "value"} }
 { "update" : { "_id" : "0", "_type" : "type1", "_index" : "index1", "_retry_on_conflict" : 3} }
-{ "script" : { "inline": "ctx._source.counter += params.param1", "lang" : "painless", "params" : {"param1" : 1}}, "upsert" : {"counter" : 1}}
+{ "script" : { "source": "ctx._source.counter += params.param1", "lang" : "painless", "params" : {"param1" : 1}}, "upsert" : {"counter" : 1}}
 { "update" : {"_id" : "2", "_type" : "type1", "_index" : "index1", "_retry_on_conflict" : 3} }
 { "doc" : {"field" : "value"}, "doc_as_upsert" : true }
-{ "update" : {"_id" : "3", "_type" : "type1", "_index" : "index1", "fields" : ["_source"]} }
+{ "update" : {"_id" : "3", "_type" : "type1", "_index" : "index1", "_source" : true} }
 { "doc" : {"field" : "value"} }
 { "update" : {"_id" : "4", "_type" : "type1", "_index" : "index1"} }
-{ "doc" : {"field" : "value"}, "fields": ["_source"]}
+{ "doc" : {"field" : "value"}, "_source": true}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 [float]
 [[bulk-security]]
diff --git a/docs/reference/docs/data-replication.asciidoc b/docs/reference/docs/data-replication.asciidoc
new file mode 100644
index 0000000000000..47af258204fdb
--- /dev/null
+++ b/docs/reference/docs/data-replication.asciidoc
@@ -0,0 +1,149 @@
+
+[[docs-replication]]
+== Reading and Writing documents
+
+[float]
+=== Introduction
+
+Each index in Elasticsearch is <<getting-started-shards-and-replicas,divided into shards>>
+and each shard can have multiple copies. These copies are known as a _replication group_ and must be kept in sync when documents
+are added or removed. If we fail to do so, reading from one copy will result in very different results than reading from another.
+The process of keeping the shard copies in sync and serving reads from them is what we call the _data replication model_.
+
+Elasticsearch’s data replication model is based on the _primary-backup model_ and is described very well in the 
+https://www.microsoft.com/en-us/research/publication/pacifica-replication-in-log-based-distributed-storage-systems/[PacificA paper] of
+Microsoft Research. That model is based on having a single copy from the replication group that acts as the primary shard.
+The other copies are called _replica shards_. The primary serves as the main entry point for all indexing operations. It is in charge of
+validating them and making sure they are correct. Once an index operation has been accepted by the primary, the primary is also
+responsible for replicating the operation to the other copies.
+
+This purpose of this section is to give a high level overview of the Elasticsearch replication model and discuss the implications
+it has for various interactions between write and read operations.
+
+[float]
+=== Basic write model
+
+Every indexing operation in Elasticsearch is first resolved to a replication group using <<index-routing,routing>>, 
+typically based on the document ID. Once the replication group has been determined,
+the operation is forwarded internally to the current _primary shard_ of the group. The primary shard is responsible
+for validating the operation and forwarding it to the other replicas. Since replicas can be offline, the primary
+is not required to replicate to all replicas. Instead, Elasticsearch maintains a list of shard copies that should
+receive the operation. This list is called the _in-sync copies_ and is maintained by the master node. As the name implies,
+these are the set of "good" shard copies that are guaranteed to have processed all of the index and delete operations that
+have been acknowledged to the user. The primary is responsible for maintaining this invariant and thus has to replicate all
+operations to each copy in this set.
+
+The primary shard follows this basic flow:
+
+. Validate incoming operation and reject it if structurally invalid (Example: have an object field where a number is expected)
+. Execute the operation locally i.e. indexing or deleting the relevant document. This will also validate the content of fields
+   and reject if needed (Example: a keyword value is too long for indexing in Lucene).
+. Forward the operation to each replica in the current in-sync copies set. If there are multiple replicas, this is done in parallel.
+. Once all replicas have successfully performed the operation and responded to the primary, the primary acknowledges the successful
+   completion of the request to the client.
+
+[float]
+==== Failure handling
+
+Many things can go wrong during indexing -- disks can get corrupted, nodes can be disconnected from each other, or some
+configuration mistake could cause an operation to fail on a replica despite it being successful on the primary. These
+are infrequent but the primary has to respond to them.
+
+In the case that the primary itself fails, the node hosting the primary will send a message to the master about it. The indexing
+operation will wait (up to 1 minute, by <<dynamic-index-settings,default>>) for the master to promote one of the replicas to be a 
+new primary. The operation will then be forwarded to the new primary for processing. Note that the master also monitors the
+health of the nodes and may decide to proactively demote a primary. This typically happens when the node holding the primary
+is isolated from the cluster by a networking issue. See <<demoted-primary,here>> for more details.
+
+Once the operation has been successfully performed on the primary, the primary has to deal with potential failures
+when executing it on the replica shards. This may be caused by an actual failure on the replica or due to a network
+issue preventing the operation from reaching the replica (or preventing the replica from responding). All of these
+share the same end result: a replica which is part of the in-sync replica set misses an operation that is about to
+be acknowledged. In order to avoid violating the invariant, the primary sends a message to the master requesting
+that the problematic shard be removed from the in-sync replica set. Only once removal of the shard has been acknowledged 
+by the master does the primary acknowledge the operation. Note that the master will also instruct another node to start 
+building a new shard copy in order to restore the system to a healthy state.
+
+[[demoted-primary]]
+While forwarding an operation to the replicas, the primary will use the replicas to validate that it is still the
+active primary. If the primary has been isolated due to a network partition (or a long GC) it may continue to process
+incoming indexing operations before realising that it has been demoted. Operations that come from a stale primary
+will be rejected by the replicas. When the primary receives a response from the replica rejecting its request because
+it is no longer the primary then it will reach out to the master and will learn that it has been replaced. The
+operation is then routed to the new primary.
+
+.What happens if there are no replicas? 
+************
+This is a valid scenario that can happen due to index configuration or simply
+because all the replicas have failed. In that case the primary is processing operations without any external validation,
+which may seem problematic. On the other hand, the primary cannot fail other shards on its own but request the master to do
+so on its behalf. This means that the master knows that the primary is the only single good copy. We are therefore guaranteed 
+that the master will not promote any other (out-of-date) shard copy to be a  new primary and that any operation indexed 
+into the primary will not be lost. Of course, since at that point we are running with only single copy of the data, physical hardware
+issues can cause data loss. See <<index-wait-for-active-shards>> for some mitigation options.
+************
+
+[float]
+=== Basic read model
+
+Reads in Elasticsearch can be very lightweight lookups by ID or a heavy search request with complex aggregations that
+take non-trivial CPU power. One of the beauties of the primary-backup model is that it keeps all shard copies identical
+(with the exception of in-flight operations). As such, a single in-sync copy is sufficient to serve read requests.
+
+When a read request is received by a node, that node is responsible for forwarding it to the nodes that hold the relevant shards,
+collating the responses, and responding to the client. We call that node the _coordinating node_ for that request. The basic flow 
+is as follows:
+
+. Resolve the read requests to the relevant shards. Note that since most searches will be sent to one or more indices,
+   they typically need to read from multiple shards, each representing a different subset of the data.
+. Select an active copy of each relevant shard, from the shard replication group. This can be either the primary or
+   a replica. By default, Elasticsearch will simply round robin between the shard copies.
+. Send shard level read requests to the selected copies.
+. Combine the results and respond. Note that in the case of get by ID look up, only one shard is relevant and this step can be skipped.
+
+[float]
+==== Failure handling
+
+When a shard fails to respond to a read request, the coordinating node will select another copy from the same replication group
+and send the shard level search request to that copy instead. Repetitive failures can result in no shard copies being available.
+In some cases, such as `_search`, Elasticsearch will prefer to respond fast, albeit with partial results, instead of waiting 
+for the issue to be resolved (partial results are indicated in the `_shards` header of the response).
+
+[float]
+=== A few simple implications
+
+Each of these basic flows determines how Elasticsearch behaves as a system for both reads and writes. Furthermore, since read
+and write requests can be executed concurrently, these two basic flows interact with each other. This has a few inherent implications:
+
+Efficient reads:: Under normal operation each read operation is performed once for each relevant replication group.
+   Only under failure conditions do multiple copies of the same shard execute the same search.
+
+Read unacknowledged:: Since the primary first indexes locally and then replicates the request, it is possible for a
+  concurrent read to already see the change before it has been acknowledged.
+
+Two copies by default:: This model can be fault tolerant while maintaining only two copies of the data. This is in contrast to
+  quorum-based system where the minimum number of copies for fault tolerance is 3.
+
+[float]
+=== Failures
+
+Under failures, the following is possible:
+
+A single shard can slow down indexing:: Because the primary waits for all replicas in the in-sync copies set during each operation,
+  a single slow shard can slow down the entire replication group. This is the price we pay for the read efficiency mentioned above.
+  Of course a single slow shard will also slow down unlucky searches that have been routed to it.
+
+Dirty reads:: An isolated primary can expose writes that will not be acknowledged. This is caused by the fact that an isolated
+  primary will only realize that it is isolated once it sends requests to its replicas or when reaching out to the master.
+  At that point the operation is already indexed into the primary and can be read by a concurrent read. Elasticsearch mitigates
+  this risk by pinging the master every second (by default) and rejecting indexing operations if no master is known.
+
+[float]
+=== The Tip of the Iceberg
+
+This document provides a high level overview of how Elasticsearch deals with data. Of course, there is much much more 
+going on under the hood. Things like primary terms, cluster state publishing and master election all play a role in 
+keeping this system behaving correctly. This document also doesn't cover known and important
+bugs (both closed and open). We recognize that https://github.com/elastic/elasticsearch/issues?q=label%3Aresiliency[GitHub is hard to keep up with].
+To help people stay on top of those and we maintain a dedicated https://www.elastic.co/guide/en/elasticsearch/resiliency/current/index.html[resiliency page]
+on our website. We strongly advise reading it.
diff --git a/docs/reference/docs/delete-by-query.asciidoc b/docs/reference/docs/delete-by-query.asciidoc
index 08bfa3318f12e..d3a1b00b6513a 100644
--- a/docs/reference/docs/delete-by-query.asciidoc
+++ b/docs/reference/docs/delete-by-query.asciidoc
@@ -1,8 +1,6 @@
 [[docs-delete-by-query]]
 == Delete By Query API
 
-experimental[The delete-by-query API is new and should still be considered experimental.  The API may change in ways that are not backwards compatible]
-
 The simplest usage of `_delete_by_query` just performs a deletion on every
 document that match a query. Here is the API:
 
@@ -66,9 +64,9 @@ these documents. In case a search or bulk request got rejected, `_delete_by_quer
  exponential back off). Reaching the maximum retries limit causes the `_delete_by_query`
  to abort and all failures are returned in the `failures` of the response.
  The deletions that have been performed still stick. In other words, the process
- is not rolled back, only aborted. While the first failure causes the abort all
+ is not rolled back, only aborted. While the first failure causes the abort, all
  failures that are returned by the failing bulk request are returned in the `failures`
- element so it's possible for there to be quite a few.
+ element; therefore it's possible for there to be quite a few failed entities.
 
 If you'd like to count version conflicts rather than cause them to abort then
 set `conflicts=proceed` on the url or `"conflicts": "proceed"` in the request body.
@@ -160,20 +158,31 @@ to keep or remove as you see fit. When you are done with it, delete it so
 Elasticsearch can reclaim the space it uses.
 
 `wait_for_active_shards` controls how many copies of a shard must be active
-before proceeding with the request. See <<index-wait-for-active-shards,here>> 
+before proceeding with the request. See <<index-wait-for-active-shards,here>>
 for details. `timeout` controls how long each write request waits for unavailable
 shards to become available. Both work exactly how they work in the
 <<docs-bulk,Bulk API>>.
 
 `requests_per_second` can be set to any positive decimal number (`1.4`, `6`,
-`1000`, etc) and throttles the number of requests per second that the delete-by-query
-issues or it can be set to `-1` to disabled throttling. The throttling is done
-waiting between bulk batches so that it can manipulate the scroll timeout. The
-wait time is the difference between the time it took the batch to complete and
-the time `requests_per_second * requests_in_the_batch`. Since the batch isn't
-broken into multiple bulk requests large batch sizes will cause Elasticsearch
-to create many requests and then wait for a while before starting the next set.
-This is "bursty" instead of "smooth". The default is `-1`.
+`1000`, etc) and throttles rate at which `_delete_by_query` issues batches of
+delete operations by padding each batch with a wait time. The throttling can be
+disabled by setting `requests_per_second` to `-1`.
+
+The throttling is done by waiting between batches so that scroll that
+`_delete_by_query` uses internally can be given a timeout that takes into
+account the padding. The padding time is the difference between the batch size
+divided by the `requests_per_second` and the time spent writing. By default the
+batch size is `1000`, so if the `requests_per_second` is set to `500`:
+
+[source,txt]
+--------------------------------------------------
+target_time = 1000 / 500 per second = 2 seconds
+wait_time = target_time - write_time = 2 seconds - .5 seconds = 1.5 seconds
+--------------------------------------------------
+
+Since the batch is issued as a single `_bulk` request large batch sizes will
+cause Elasticsearch to create many requests and then wait for a while before
+starting the next set. This is "bursty" instead of "smooth". The default is `-1`.
 
 [float]
 === Response body
@@ -233,7 +242,7 @@ You can fetch the status of any running delete-by-query requests with the
 
 [source,js]
 --------------------------------------------------
-GET _tasks?detailed=true&action=*/delete/byquery
+GET _tasks?detailed=true&actions=*/delete/byquery
 --------------------------------------------------
 // CONSOLE
 
@@ -244,7 +253,7 @@ The responses looks like:
 {
   "nodes" : {
     "r1A2WoRbTwKZ516z6NEs5A" : {
-      "name" : "Tyrannus",
+      "name" : "r1A2WoR",
       "transport_address" : "127.0.0.1:9300",
       "host" : "127.0.0.1",
       "ip" : "127.0.0.1:9300",
@@ -294,7 +303,7 @@ GET /_tasks/taskId:1
 
 The advantage of this API is that it integrates with `wait_for_completion=false`
 to transparently return the status of completed tasks. If the task is completed
-and `wait_for_completion=false` was set on it them it'll come back with a
+and `wait_for_completion=false` was set on it then it'll come back with
 `results` or an `error` field. The cost of this feature is the document that
 `wait_for_completion=false` creates at `.tasks/task/${taskId}`. It is up to
 you to delete that document.
@@ -308,13 +317,13 @@ Any Delete By Query can be canceled using the <<tasks,Task Cancel API>>:
 
 [source,js]
 --------------------------------------------------
-POST _tasks/taskid:1/_cancel
+POST _tasks/task_id:1/_cancel
 --------------------------------------------------
 // CONSOLE
 
 The `task_id` can be found using the tasks API above.
 
-Cancelation should happen quickly but might take a few seconds. The task status
+Cancellation should happen quickly but might take a few seconds. The task status
 API above will continue to list the task until it is wakes to cancel itself.
 
 
@@ -327,7 +336,7 @@ using the `_rethrottle` API:
 
 [source,js]
 --------------------------------------------------
-POST _delete_by_query/taskid:1/_rethrottle?requests_per_second=-1
+POST _delete_by_query/task_id:1/_rethrottle?requests_per_second=-1
 --------------------------------------------------
 // CONSOLE
 
@@ -339,3 +348,170 @@ like `1.7` or `12` to throttle to that level. Rethrottling that speeds up the
 query takes effect immediately but rethrotting that slows down the query will
 take effect on after completing the current batch. This prevents scroll
 timeouts.
+
+[float]
+[[docs-delete-by-query-manual-slice]]
+=== Manually slicing
+
+Delete-by-query supports <<sliced-scroll>> allowing you to manually parallelize
+the process relatively easily:
+
+[source,js]
+----------------------------------------------------------------
+POST twitter/_delete_by_query
+{
+  "slice": {
+    "id": 0,
+    "max": 2
+  },
+  "query": {
+    "range": {
+      "likes": {
+        "lt": 10
+      }
+    }
+  }
+}
+POST twitter/_delete_by_query
+{
+  "slice": {
+    "id": 1,
+    "max": 2
+  },
+  "query": {
+    "range": {
+      "likes": {
+        "lt": 10
+      }
+    }
+  }
+}
+----------------------------------------------------------------
+// CONSOLE
+// TEST[setup:big_twitter]
+
+Which you can verify works with:
+
+[source,js]
+----------------------------------------------------------------
+GET _refresh
+POST twitter/_search?size=0&filter_path=hits.total
+{
+  "query": {
+    "range": {
+      "likes": {
+        "lt": 10
+      }
+    }
+  }
+}
+----------------------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+Which results in a sensible `total` like this one:
+
+[source,js]
+----------------------------------------------------------------
+{
+  "hits": {
+    "total": 0
+  }
+}
+----------------------------------------------------------------
+// TESTRESPONSE
+
+[float]
+[[docs-delete-by-query-automatic-slice]]
+=== Automatic slicing
+
+You can also let delete-by-query automatically parallelize using
+<<sliced-scroll>> to slice on `_uid`:
+
+[source,js]
+----------------------------------------------------------------
+POST twitter/_delete_by_query?refresh&slices=5
+{
+  "query": {
+    "range": {
+      "likes": {
+        "lt": 10
+      }
+    }
+  }
+}
+----------------------------------------------------------------
+// CONSOLE
+// TEST[setup:big_twitter]
+
+Which you also can verify works with:
+
+[source,js]
+----------------------------------------------------------------
+POST twitter/_search?size=0&filter_path=hits.total
+{
+  "query": {
+    "range": {
+      "likes": {
+        "lt": 10
+      }
+    }
+  }
+}
+----------------------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+Which results in a sensible `total` like this one:
+
+[source,js]
+----------------------------------------------------------------
+{
+  "hits": {
+    "total": 0
+  }
+}
+----------------------------------------------------------------
+// TESTRESPONSE
+
+Adding `slices` to `_delete_by_query` just automates the manual process used in
+the section above, creating sub-requests which means it has some quirks:
+
+* You can see these requests in the
+<<docs-delete-by-query-task-api,Tasks APIs>>. These sub-requests are "child"
+tasks of the task for the request with `slices`.
+* Fetching the status of the task for the request with `slices` only contains
+the status of completed slices.
+* These sub-requests are individually addressable for things like cancellation
+and rethrottling.
+* Rethrottling the request with `slices` will rethrottle the unfinished
+sub-request proportionally.
+* Canceling the request with `slices` will cancel each sub-request.
+* Due to the nature of `slices` each sub-request won't get a perfectly even
+portion of the documents. All documents will be addressed, but some slices may
+be larger than others. Expect larger slices to have a more even distribution.
+* Parameters like `requests_per_second` and `size` on a request with `slices`
+are distributed proportionally to each sub-request. Combine that with the point
+above about distribution being uneven and you should conclude that the using
+`size` with `slices` might not result in exactly `size` documents being
+`_delete_by_query`ed.
+* Each sub-requests gets a slightly different snapshot of the source index
+though these are all taken at approximately the same time.
+
+[float]
+[[docs-delete-by-query-picking-slices]]
+=== Picking the number of slices
+
+At this point we have a few recommendations around the number of `slices` to
+use (the `max` parameter in the slice API if manually parallelizing):
+
+* Don't use large numbers. `500` creates fairly massive CPU thrash.
+* It is more efficient from a query performance standpoint to use some multiple
+of the number of shards in the source index.
+* Using exactly as many shards as are in the source index is the most efficient
+from a query performance standpoint.
+* Indexing performance should scale linearly across available resources with
+the number of `slices`.
+* Whether indexing or query performance dominates that process depends on lots
+of factors like the documents being reindexed and the cluster doing the
+reindexing.
diff --git a/docs/reference/docs/delete.asciidoc b/docs/reference/docs/delete.asciidoc
index a130e66a191d7..a9af68e5bcbd9 100644
--- a/docs/reference/docs/delete.asciidoc
+++ b/docs/reference/docs/delete.asciidoc
@@ -8,8 +8,10 @@ from an index called twitter, under a type called tweet, with id valued
 
 [source,js]
 --------------------------------------------------
-$ curl -XDELETE 'http://localhost:9200/twitter/tweet/1'
+DELETE /twitter/tweet/1
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
 
 The result of the above delete operation is:
 
@@ -17,18 +19,19 @@ The result of the above delete operation is:
 --------------------------------------------------
 {
     "_shards" : {
-        "total" : 10,
+        "total" : 2,
         "failed" : 0,
-        "successful" : 10
+        "successful" : 2
     },
     "found" : true,
     "_index" : "twitter",
     "_type" : "tweet",
     "_id" : "1",
     "_version" : 2,
-    "result: deleted"
+    "result": "deleted"
 }
 --------------------------------------------------
+// TESTRESPONSE[s/"successful" : 2/"successful" : 1/]
 
 [float]
 [[delete-versioning]]
@@ -48,10 +51,26 @@ When indexing using the ability to control the routing, in order to
 delete a document, the routing value should also be provided. For
 example:
 
+////
+Example to delete with routing
+
+[source,js]
+--------------------------------------------------
+PUT /twitter/tweet/1?routing=kimhcy
+{
+    "test": "test"
+}
+--------------------------------------------------
+// CONSOLE
+////
+
+
 [source,js]
 --------------------------------------------------
-$ curl -XDELETE 'http://localhost:9200/twitter/tweet/1?routing=kimchy'
+DELETE /twitter/tweet/1?routing=kimchy
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 The above will delete a tweet with id 1, but will be routed based on the
 user. Note, issuing a delete without the correct routing, will cause the
@@ -130,5 +149,7 @@ to 5 minutes:
 
 [source,js]
 --------------------------------------------------
-$ curl -XDELETE 'http://localhost:9200/twitter/tweet/1?timeout=5m'
+DELETE /twitter/tweet/1?timeout=5m
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
diff --git a/docs/reference/docs/get.asciidoc b/docs/reference/docs/get.asciidoc
index 4f504f2132078..fef929cdbd537 100644
--- a/docs/reference/docs/get.asciidoc
+++ b/docs/reference/docs/get.asciidoc
@@ -3,12 +3,14 @@
 
 The get API allows to get a typed JSON document from the index based on
 its id. The following example gets a JSON document from an index called
-twitter, under a type called tweet, with id valued 1:
+twitter, under a type called tweet, with id valued 0:
 
 [source,js]
 --------------------------------------------------
-curl -XGET 'http://localhost:9200/twitter/tweet/1'
+GET twitter/tweet/0
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
 
 The result of the above get operation is:
 
@@ -17,16 +19,18 @@ The result of the above get operation is:
 {
     "_index" : "twitter",
     "_type" : "tweet",
-    "_id" : "1",
+    "_id" : "0",
     "_version" : 1,
     "found": true,
     "_source" : {
         "user" : "kimchy",
-        "postDate" : "2009-11-15T14:12:12",
+        "date" : "2009-11-15T14:12:12",
+        "likes": 0,
         "message" : "trying out Elasticsearch"
     }
 }
 --------------------------------------------------
+// TESTRESPONSE
 
 The above result includes the `_index`, `_type`, `_id` and `_version`
 of the document we wish to retrieve, including the actual `_source`
@@ -38,8 +42,10 @@ The API also allows to check for the existence of a document using
 
 [source,js]
 --------------------------------------------------
-curl -XHEAD -i 'http://localhost:9200/twitter/tweet/1'
+HEAD twitter/tweet/0
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
 
 [float]
 [[realtime]]
@@ -52,10 +58,6 @@ call in-place to make the document visible. This will also make other documents
 changed since the last refresh visible. In order to disable realtime GET,
 one can set the `realtime` parameter to `false`.
 
-When getting a document, one can specify `fields` to fetch from it. They
-will, when possible, be fetched as stored fields (fields mapped as
-<<mapping-store,stored>> in the mapping).
-
 [float]
 [[type]]
 === Optional Type
@@ -69,13 +71,15 @@ to fetch the first document matching the id across all types.
 === Source filtering
 
 By default, the get operation returns the contents of the `_source` field unless
-you have used the `fields` parameter or if the `_source` field is disabled. 
+you have used the `stored_fields` parameter or if the `_source` field is disabled.
 You can turn off `_source` retrieval by using the `_source` parameter:
 
 [source,js]
 --------------------------------------------------
-curl -XGET 'http://localhost:9200/twitter/tweet/1?_source=false'
+GET twitter/tweet/0?_source=false
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
 
 If you only need one or two fields from the complete `_source`, you can use the `_source_include`
 & `_source_exclude` parameters to include or filter out that parts you need. This can be especially helpful
@@ -84,46 +88,139 @@ of fields or wildcard expressions. Example:
 
 [source,js]
 --------------------------------------------------
-curl -XGET 'http://localhost:9200/twitter/tweet/1?_source_include=*.id&_source_exclude=entities'
+GET twitter/tweet/0?_source_include=*.id&_source_exclude=entities
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
 
 If you only want to specify includes, you can use a shorter notation:
 
 [source,js]
 --------------------------------------------------
-curl -XGET 'http://localhost:9200/twitter/tweet/1?_source=*.id,retweeted'
+GET twitter/tweet/0?_source=*.id,retweeted
 --------------------------------------------------
-
+// CONSOLE
+// TEST[setup:twitter]
 
 [float]
-[[get-fields]]
-=== Fields
+[[get-stored-fields]]
+=== Stored Fields
 
 The get operation allows specifying a set of stored fields that will be
-returned by passing the `fields` parameter. For example:
+returned by passing the `stored_fields` parameter.
+If the requested fields are not stored, they will be ignored.
+Consider for instance the following mapping:
 
 [source,js]
 --------------------------------------------------
-curl -XGET 'http://localhost:9200/twitter/tweet/1?fields=title,content'
+PUT twitter
+{
+   "mappings": {
+      "tweet": {
+         "properties": {
+            "counter": {
+               "type": "integer",
+               "store": false
+            },
+            "tags": {
+               "type": "keyword",
+               "store": true
+            }
+         }
+      }
+   }
+}
 --------------------------------------------------
+// CONSOLE
 
-For backward compatibility, if the requested fields are not stored, they will be fetched
-from the `_source` (parsed and extracted). This functionality has been replaced by the
-<<get-source-filtering,source filtering>> parameter.
+Now we can add a document:
 
-Field values fetched from the document it self are always returned as an array. Metadata fields like `_routing` and
-`_parent` fields are never returned as an array.
+[source,js]
+--------------------------------------------------
+PUT twitter/tweet/1
+{
+    "counter" : 1,
+    "tags" : ["red"]
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
-Also only leaf fields can be returned via the `field` option. So object fields can't be returned and such requests
-will fail.
+... and try to retrieve it:
 
-[float]
-[[generated-fields]]
-=== Generated fields
-If no refresh occurred between indexing and refresh, GET will access the transaction log to fetch the document. However, some fields are generated only when indexing. 
-If you try to access a field that is only generated when indexing, you will get an exception (default). You can choose to ignore field that are generated if the transaction log is accessed by setting `ignore_errors_on_generated_fields=true`.
+[source,js]
+--------------------------------------------------
+GET twitter/tweet/1?stored_fields=tags,counter
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+The result of the above get operation is:
+
+[source,js]
+--------------------------------------------------
+{
+   "_index": "twitter",
+   "_type": "tweet",
+   "_id": "1",
+   "_version": 1,
+   "found": true,
+   "fields": {
+      "tags": [
+         "red"
+      ]
+   }
+}
+--------------------------------------------------
+// TESTRESPONSE
 
 
+Field values fetched from the document it self are always returned as an array.
+Since the `counter` field is not stored the get request simply ignores it when trying to get the `stored_fields.`
+
+It is also possible to retrieve metadata fields like `_routing` and `_parent` fields:
+
+[source,js]
+--------------------------------------------------
+PUT twitter/tweet/2?routing=user1
+{
+    "counter" : 1,
+    "tags" : ["white"]
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+[source,js]
+--------------------------------------------------
+GET twitter/tweet/2?routing=user1&stored_fields=tags,counter
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+The result of the above get operation is:
+
+[source,js]
+--------------------------------------------------
+{
+   "_index": "twitter",
+   "_type": "tweet",
+   "_id": "2",
+   "_version": 1,
+   "_routing": "user1",
+   "found": true,
+   "fields": {
+      "tags": [
+         "white"
+      ]
+   }
+}
+--------------------------------------------------
+// TESTRESPONSE
+
+Also only leaf fields can be returned via the `stored_field` option. So object fields can't be returned and such requests
+will fail.
+
 [float]
 [[_source]]
 === Getting the _source directly
@@ -134,24 +231,29 @@ without any additional content around it. For example:
 
 [source,js]
 --------------------------------------------------
-curl -XGET 'http://localhost:9200/twitter/tweet/1/_source'
+GET twitter/tweet/1/_source
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 You can also use the same source filtering parameters to control which parts of the `_source` will be returned:
 
 [source,js]
 --------------------------------------------------
-curl -XGET 'http://localhost:9200/twitter/tweet/1/_source?_source_include=*.id&_source_exclude=entities'
+GET twitter/tweet/1/_source?_source_include=*.id&_source_exclude=entities'
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 Note, there is also a HEAD variant for the _source endpoint to efficiently test for document _source existence.
 An existing document will not have a _source if it is disabled in the <<mapping-source-field,mapping>>.
-Curl example:
 
 [source,js]
 --------------------------------------------------
-curl -XHEAD -i 'http://localhost:9200/twitter/tweet/1/_source'
+HEAD twitter/tweet/1/_source
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 [float]
 [[get-routing]]
@@ -162,10 +264,12 @@ a document, the routing value should also be provided. For example:
 
 [source,js]
 --------------------------------------------------
-curl -XGET 'http://localhost:9200/twitter/tweet/1?routing=kimchy'
+GET twitter/tweet/2?routing=user1
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
-The above will get a tweet with id 1, but will be routed based on the
+The above will get a tweet with id 2, but will be routed based on the
 user. Note, issuing a get without the correct routing, will cause the
 document not to be fetched.
 
@@ -222,7 +326,7 @@ better GET scaling we will have.
 You can use the `version` parameter to retrieve the document only if
 its current version is equal to the specified one. This behavior is the same
 for all version types with the exception of version type `FORCE` which always
-retrieves the document.
+retrieves the document. Note that `FORCE` version type is deprecated.
 
 Internally, Elasticsearch has marked the old document as deleted and added an
 entirely new document. The old version of the document doesn’t disappear
diff --git a/docs/reference/docs/index_.asciidoc b/docs/reference/docs/index_.asciidoc
index 57453b546ac23..37c0443dabf3e 100644
--- a/docs/reference/docs/index_.asciidoc
+++ b/docs/reference/docs/index_.asciidoc
@@ -1,6 +1,8 @@
 [[docs-index_]]
 == Index API
 
+IMPORTANT: See <<removal-of-types>>.
+
 The index API adds or updates a typed JSON document in a specific index,
 making it searchable. The following example inserts the JSON document
 into the "twitter" index, under a type called "tweet" with an id of 1:
@@ -44,9 +46,9 @@ The `_shards` header provides information about the replication process of the i
 
 The index operation is successful in the case `successful` is at least 1.
 
-NOTE:   Replica shards may not all be started when an indexing operation successfully returns (by default, only the 
-        primary is required, but this behavior can be <<index-wait-for-active-shards,changed>>). In that case, 
-        `total` will be equal to the total shards based on the `number_of_replicas` setting and `successful` will be 
+NOTE:   Replica shards may not all be started when an indexing operation successfully returns (by default, only the
+        primary is required, but this behavior can be <<index-wait-for-active-shards,changed>>). In that case,
+        `total` will be equal to the total shards based on the `number_of_replicas` setting and `successful` will be
         equal to the number of shards started (primary plus replicas). If there were no failures, the `failed` will be 0.
 
 [float]
@@ -69,8 +71,7 @@ section for more information on mapping definitions.
 Automatic index creation can be disabled by setting
 `action.auto_create_index` to `false` in the config file of all nodes.
 Automatic mapping creation can be disabled by setting
-`index.mapper.dynamic` to `false` in the config files of all nodes (or
-on the specific index settings).
+`index.mapper.dynamic` to `false` per-index as an index setting.
 
 Automatic index creation can include a pattern based white/black list,
 for example, set `action.auto_create_index` to `+aaa*,-bbb*,+ccc*,-*` (+
@@ -100,6 +101,7 @@ PUT twitter/tweet/1?version=2
 }
 --------------------------------------------------
 // CONSOLE
+// TEST[continued]
 // TEST[catch: conflict]
 
 *NOTE:* versioning is completely real time, and is not affected by the
@@ -155,12 +157,10 @@ than the version of the stored document. If there is no existing document
 the operation will succeed as well. The given version will be used as the new version
 and will be stored with the new document. The supplied version must be a non-negative long number.
 
-`force`:: the document will be indexed regardless of the version of the stored document or if there
-is no existing document. The given version will be used as the new version and will be stored
-with the new document. This version type is typically used for correcting errors.
-
-*NOTE*: The `external_gte` & `force` version types are meant for special use cases and should be used
-with care. If used incorrectly, they can result in loss of data.
+*NOTE*: The `external_gte` version type is meant for special use cases and
+should be used with care. If used incorrectly, it can result in loss of data.
+There is another option, `force`, which is deprecated because it can cause
+primary and replica shards to diverge.
 
 [float]
 [[operation-type]]
@@ -311,46 +311,46 @@ if needed, the update is distributed to applicable replicas.
 [[index-wait-for-active-shards]]
 === Wait For Active Shards
 
-To improve the resiliency of writes to the system, indexing operations 
-can be configured to wait for a certain number of active shard copies 
+To improve the resiliency of writes to the system, indexing operations
+can be configured to wait for a certain number of active shard copies
 before proceeding with the operation. If the requisite number of active
-shard copies are not available, then the write operation must wait and 
-retry, until either the requisite shard copies have started or a timeout 
-occurs. By default, write operations only wait for the primary shards 
+shard copies are not available, then the write operation must wait and
+retry, until either the requisite shard copies have started or a timeout
+occurs. By default, write operations only wait for the primary shards
 to be active before proceeding (i.e. `wait_for_active_shards=1`).
 This default can be overridden in the index settings dynamically
-by setting `index.write.wait_for_active_shards`. To alter this behavior 
+by setting `index.write.wait_for_active_shards`. To alter this behavior
 per operation, the `wait_for_active_shards` request parameter can be used.
 
 Valid values are `all` or any positive integer up to the total number
 of configured copies per shard in the index (which is `number_of_replicas+1`).
-Specifying a negative value or a number greater than the number of 
+Specifying a negative value or a number greater than the number of
 shard copies will throw an error.
 
-For example, suppose we have a cluster of three nodes, `A, `B`, and `C` and
-we create an index `index` with the number of replicas set to 3 (resulting in 
-4 shard copies, one more copy than there are nodes). If we 
+For example, suppose we have a cluster of three nodes, `A`, `B`, and `C` and
+we create an index `index` with the number of replicas set to 3 (resulting in
+4 shard copies, one more copy than there are nodes). If we
 attempt an indexing operation, by default the operation will only ensure
 the primary copy of each shard is available before proceeding. This means
 that even if `B` and `C` went down, and `A` hosted the primary shard copies,
-the indexing operation would still proceed with only one copy of the data. 
+the indexing operation would still proceed with only one copy of the data.
 If `wait_for_active_shards` is set on the request to `3` (and all 3 nodes
-are up), then the indexing operation will require 3 active shard copies 
+are up), then the indexing operation will require 3 active shard copies
 before proceeding, a requirement which should be met because there are 3
 active nodes in the cluster, each one holding a copy of the shard. However,
-if we set `wait_for_active_shards` to `all` (or to `4`, which is the same), 
-the indexing operation will not proceed as we do not have all 4 copies of 
-each shard active in the index. The operation will timeout 
+if we set `wait_for_active_shards` to `all` (or to `4`, which is the same),
+the indexing operation will not proceed as we do not have all 4 copies of
+each shard active in the index. The operation will timeout
 unless a new node is brought up in the cluster to host the fourth copy of
 the shard.
 
-It is important to note that this setting greatly reduces the chances of 
-the write operation not writing to the requisite number of shard copies, 
+It is important to note that this setting greatly reduces the chances of
+the write operation not writing to the requisite number of shard copies,
 but it does not completely eliminate the possibility, because this check
 occurs before the write operation commences. Once the write operation
-is underway, it is still possible for replication to fail on any number of 
+is underway, it is still possible for replication to fail on any number of
 shard copies but still succeed on the primary. The `_shards` section of the
-write operation's response reveals the number of shard copies on which 
+write operation's response reveals the number of shard copies on which
 replication succeeded/failed.
 
 [source,js]
@@ -363,6 +363,7 @@ replication succeeded/failed.
     }
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 [float]
 [[index-refresh]]
diff --git a/docs/reference/docs/multi-get.asciidoc b/docs/reference/docs/multi-get.asciidoc
index 15dec9aea2578..b594793a4a2c2 100644
--- a/docs/reference/docs/multi-get.asciidoc
+++ b/docs/reference/docs/multi-get.asciidoc
@@ -9,7 +9,8 @@ API. Here is an example:
 
 [source,js]
 --------------------------------------------------
-curl 'localhost:9200/_mget' -d '{
+GET _mget
+{
     "docs" : [
         {
             "_index" : "test",
@@ -22,15 +23,17 @@ curl 'localhost:9200/_mget' -d '{
             "_id" : "2"
         }
     ]
-}'
+}
 --------------------------------------------------
+// CONSOLE
 
 The `mget` endpoint can also be used against an index (in which case it
 is not required in the body):
 
 [source,js]
 --------------------------------------------------
-curl 'localhost:9200/test/_mget' -d '{
+GET test/_mget
+{
     "docs" : [
         {
             "_type" : "type",
@@ -41,14 +44,16 @@ curl 'localhost:9200/test/_mget' -d '{
             "_id" : "2"
         }
     ]
-}'
+}
 --------------------------------------------------
+// CONSOLE
 
 And type:
 
 [source,js]
 --------------------------------------------------
-curl 'localhost:9200/test/type/_mget' -d '{
+GET test/type/_mget
+{
     "docs" : [
         {
             "_id" : "1"
@@ -57,18 +62,21 @@ curl 'localhost:9200/test/type/_mget' -d '{
             "_id" : "2"
         }
     ]
-}'
+}
 --------------------------------------------------
+//CONSOLE
 
 In which case, the `ids` element can directly be used to simplify the
 request:
 
 [source,js]
 --------------------------------------------------
-curl 'localhost:9200/test/type/_mget' -d '{
+GET test/type/_mget
+{
     "ids" : ["1", "2"]
-}'
+}
 --------------------------------------------------
+// CONSOLE
 
 [float]
 [[mget-type]]
@@ -85,16 +93,18 @@ will give you back only the same document twice:
 
 [source,js]
 --------------------------------------------------
-curl 'localhost:9200/test/_mget' -d '{
+GET test/_mget
+{
     "ids" : ["1", "1"]
-}'
+}
 --------------------------------------------------
+// CONSOLE
 
 You need in that case to explicitly set the `_type`:
 
 [source,js]
 --------------------------------------------------
-GET /test/_mget/
+GET test/_mget/
 {
   "docs" : [
         {
@@ -108,6 +118,7 @@ GET /test/_mget/
     ]
 }
 --------------------------------------------------
+// CONSOLE
 
 [float]
 [[mget-source-filtering]]
@@ -123,7 +134,8 @@ For example:
 
 [source,js]
 --------------------------------------------------
-curl 'localhost:9200/_mget' -d '{
+GET _mget
+{
     "docs" : [
         {
             "_index" : "test",
@@ -147,62 +159,62 @@ curl 'localhost:9200/_mget' -d '{
             }
         }
     ]
-}'
+}
 --------------------------------------------------
+// CONSOLE
 
 
 [float]
 [[mget-fields]]
 === Fields
 
-Specific stored fields can be specified to be retrieved per document to get, similar to the <<get-fields,fields>> parameter of the Get API.
+Specific stored fields can be specified to be retrieved per document to get, similar to the <<get-stored-fields,stored_fields>> parameter of the Get API.
 For example:
 
 [source,js]
 --------------------------------------------------
-curl 'localhost:9200/_mget' -d '{
+GET _mget
+{
     "docs" : [
         {
             "_index" : "test",
             "_type" : "type",
             "_id" : "1",
-            "fields" : ["field1", "field2"]
+            "stored_fields" : ["field1", "field2"]
         },
         {
             "_index" : "test",
             "_type" : "type",
             "_id" : "2",
-            "fields" : ["field3", "field4"]
+            "stored_fields" : ["field3", "field4"]
         }
     ]
-}'
+}
 --------------------------------------------------
+// CONSOLE
 
-Alternatively, you can specify the `fields` parameter in the query string
+Alternatively, you can specify the `stored_fields` parameter in the query string
 as a default to be applied to all documents.
 
 [source,js]
 --------------------------------------------------
-curl 'localhost:9200/test/type/_mget?fields=field1,field2' -d '{
+GET test/type/_mget?stored_fields=field1,field2
+{
     "docs" : [
         {
             "_id" : "1" <1>
         },
         {
             "_id" : "2",
-            "fields" : ["field3", "field4"] <2>
+            "stored_fields" : ["field3", "field4"] <2>
         }
     ]
-}'
+}
 --------------------------------------------------
+// CONSOLE
 <1> Returns `field1` and `field2`
 <2> Returns `field3` and `field4`
 
-[float]
-=== Generated fields
-
-See <<generated-fields>> for fields are generated only when indexing. 
-
 [float]
 [[mget-routing]]
 === Routing
@@ -211,7 +223,8 @@ You can also specify routing value as a parameter:
 
 [source,js]
 --------------------------------------------------
-curl 'localhost:9200/_mget?routing=key1' -d '{
+GET _mget?routing=key1
+{
     "docs" : [
         {
             "_index" : "test",
@@ -225,8 +238,9 @@ curl 'localhost:9200/_mget?routing=key1' -d '{
             "_id" : "2"
         }
     ]
-}'
+}
 --------------------------------------------------
+// CONSOLE
 
 In this example, document `test/type/2` will be fetch from shard corresponding to routing key `key1` but
 document `test/type/1` will be fetch from shard corresponding to routing key `key2`.
diff --git a/docs/reference/docs/multi-termvectors.asciidoc b/docs/reference/docs/multi-termvectors.asciidoc
index 1c0ff7f9bb7c3..19867478084a6 100644
--- a/docs/reference/docs/multi-termvectors.asciidoc
+++ b/docs/reference/docs/multi-termvectors.asciidoc
@@ -3,7 +3,8 @@
 
 Multi termvectors API allows to get multiple termvectors at once. The
 documents from which to retrieve the term vectors are specified by an index,
-type and id. But the documents could also be artificially provided
+type and id. But the documents could also be artificially provided in the request itself.
+
 The response includes a `docs`
 array with all the fetched termvectors, each element having the structure
 provided by the <<docs-termvectors,termvectors>>
@@ -11,25 +12,28 @@ API. Here is an example:
 
 [source,js]
 --------------------------------------------------
-curl 'localhost:9200/_mtermvectors' -d '{
+POST /_mtermvectors
+{
    "docs": [
       {
-         "_index": "testidx",
-         "_type": "test",
+         "_index": "twitter",
+         "_type": "tweet",
          "_id": "2",
          "term_statistics": true
       },
       {
-         "_index": "testidx",
-         "_type": "test",
+         "_index": "twitter",
+         "_type": "tweet",
          "_id": "1",
          "fields": [
-            "text"
+            "message"
          ]
       }
    ]
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
 
 See the <<docs-termvectors,termvectors>> API for a description of possible parameters.
 
@@ -38,34 +42,38 @@ is not required in the body):
 
 [source,js]
 --------------------------------------------------
-curl 'localhost:9200/testidx/_mtermvectors' -d '{
+POST /twitter/_mtermvectors
+{
    "docs": [
       {
-         "_type": "test",
+         "_type": "tweet",
          "_id": "2",
          "fields": [
-            "text"
+            "message"
          ],
          "term_statistics": true
       },
       {
-         "_type": "test",
+         "_type": "tweet",
          "_id": "1"
       }
    ]
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
 
 And type:
 
 [source,js]
 --------------------------------------------------
-curl 'localhost:9200/testidx/test/_mtermvectors' -d '{
+POST /twitter/tweet/_mtermvectors
+{
    "docs": [
       {
          "_id": "2",
          "fields": [
-            "text"
+            "message"
          ],
          "term_statistics": true
       },
@@ -73,24 +81,28 @@ curl 'localhost:9200/testidx/test/_mtermvectors' -d '{
          "_id": "1"
       }
    ]
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
 
 If all requested documents are on same index and have same type and also the parameters are the same, the request can be simplified:
 
 [source,js]
 --------------------------------------------------
-curl 'localhost:9200/testidx/test/_mtermvectors' -d '{
+POST /twitter/tweet/_mtermvectors
+{
     "ids" : ["1", "2"],
     "parameters": {
     	"fields": [
-         	"text"
+         	"message"
       	],
-      	"term_statistics": true,
-      	…
+      	"term_statistics": true
     }
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
 
 Additionally, just like for the <<docs-termvectors,termvectors>>
 API, term vectors could be generated for user provided documents.  The mapping used is
@@ -98,24 +110,27 @@ determined by `_index` and `_type`.
 
 [source,js]
 --------------------------------------------------
-curl 'localhost:9200/_mtermvectors' -d '{
+POST /_mtermvectors
+{
    "docs": [
       {
-         "_index": "testidx",
-         "_type": "test",
+         "_index": "twitter",
+         "_type": "tweet",
          "doc" : {
-            "fullname" : "John Doe",
-            "text" : "twitter test test test"
+            "user" : "John Doe",
+            "message" : "twitter test test test"
          }
       },
       {
-         "_index": "testidx",
+         "_index": "twitter",
          "_type": "test",
          "doc" : {
-           "fullname" : "Jane Doe",
-           "text" : "Another twitter test ..."
+           "user" : "Jane Doe",
+           "message" : "Another twitter test ..."
          }
       }
    ]
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
diff --git a/docs/reference/docs/reindex.asciidoc b/docs/reference/docs/reindex.asciidoc
index a53ddc34dacaf..a326ef6b1b7d4 100644
--- a/docs/reference/docs/reindex.asciidoc
+++ b/docs/reference/docs/reindex.asciidoc
@@ -1,7 +1,10 @@
 [[docs-reindex]]
 == Reindex API
 
-experimental[The reindex API is new and should still be considered experimental.  The API may change in ways that are not backwards compatible]
+IMPORTANT: Reindex does not attempt to set up the destination index.  It does
+not copy the settings of the source index.  You should set up the destination
+index prior to running a `_reindex` action, including setting up mappings, shard
+counts, replicas, etc.
 
 The most basic form of `_reindex` just copies documents from one index to another.
 This will copy documents from the `twitter` index into the `new_twitter` index:
@@ -222,6 +225,28 @@ POST _reindex
 // CONSOLE
 // TEST[setup:twitter]
 
+The `source` section supports all the elements that are supported in a
+<<search-request-body,search request>>. For instance only a subset of the
+fields from the original documents can be reindexed using source filtering
+as follows:
+
+[source,js]
+--------------------------------------------------
+POST _reindex
+{
+  "source": {
+    "index": "twitter",
+    "_source": ["user", "tweet"]
+  },
+  "dest": {
+    "index": "new_twitter"
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
+
+
 Like `_update_by_query`, `_reindex` supports a script that modifies the
 document. Unlike `_update_by_query`, the script is allowed to modify the
 document's metadata. This example bumps the version of the source document:
@@ -238,7 +263,7 @@ POST _reindex
     "version_type": "external"
   },
   "script": {
-    "inline": "if (ctx._source.foo == 'bar') {ctx._version++; ctx._source.remove('foo')}",
+    "source": "if (ctx._source.foo == 'bar') {ctx._version++; ctx._source.remove('foo')}",
     "lang": "painless"
   }
 }
@@ -362,6 +387,7 @@ POST _reindex
 // TEST[s/^/PUT source\n/]
 
 [float]
+[[reindex-from-remote]]
 === Reindex from Remote
 
 Reindex supports reindexing from a remote Elasticsearch cluster:
@@ -398,14 +424,14 @@ POST _reindex
 The `host` parameter must contain a scheme, host, and port (e.g.
 `https://otherhost:9200`). The `username` and `password` parameters are
 optional and when they are present reindex will connect to the remote
-Elasticsearch node using using basic auth. Be sure to use `https` when using
+Elasticsearch node using basic auth. Be sure to use `https` when using
 basic auth or the password will be sent in plain text.
 
 Remote hosts have to be explicitly whitelisted in elasticsearch.yaml using the
 `reindex.remote.whitelist` property. It can be set to a comma delimited list
 of allowed remote `host` and `port` combinations (e.g.
-`otherhost:9200, another:9200`). Scheme is ignored by the whitelist - only host
-and port are used.
+`otherhost:9200, another:9200, 127.0.10.*:9200, localhost:*`). Scheme is
+ignored by the whitelist - only host and port are used.
 
 This feature should work with remote clusters of any version of Elasticsearch
 you are likely to find. This should allow you to upgrade from any version of
@@ -415,6 +441,74 @@ version.
 To enable queries sent to older versions of Elasticsearch the `query` parameter
 is sent directly to the remote host without validation or modification.
 
+NOTE: Reindexing from remote clusters does not support
+<<docs-reindex-manual-slice, manual>> or
+<<docs-reindex-automatic-slice, automatic slicing>>.
+
+Reindexing from a remote server uses an on-heap buffer that defaults to a
+maximum size of 100mb. If the remote index includes very large documents you'll
+need to use a smaller batch size. The example below sets the batch size `10`
+which is very, very small.
+
+[source,js]
+--------------------------------------------------
+POST _reindex
+{
+  "source": {
+    "remote": {
+      "host": "http://otherhost:9200"
+    },
+    "index": "source",
+    "size": 10,
+    "query": {
+      "match": {
+        "test": "data"
+      }
+    }
+  },
+  "dest": {
+    "index": "dest"
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:host]
+// TEST[s/^/PUT source\n/]
+// TEST[s/otherhost:9200/\${host}/]
+
+It is also possible to set the socket read timeout on the remote connection
+with the `socket_timeout` field and the connection timeout with the
+`connect_timeout` field. Both default to thirty seconds. This example
+sets the socket read timeout to one minute and the connection timeout to ten
+seconds:
+
+[source,js]
+--------------------------------------------------
+POST _reindex
+{
+  "source": {
+    "remote": {
+      "host": "http://otherhost:9200",
+      "socket_timeout": "1m",
+      "connect_timeout": "10s"
+    },
+    "index": "source",
+    "query": {
+      "match": {
+        "test": "data"
+      }
+    }
+  },
+  "dest": {
+    "index": "dest"
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:host]
+// TEST[s/^/PUT source\n/]
+// TEST[s/otherhost:9200/\${host}/]
+
 [float]
 === URL Parameters
 
@@ -441,14 +535,25 @@ shards to become available. Both work exactly how they work in the
 <<docs-bulk,Bulk API>>.
 
 `requests_per_second` can be set to any positive decimal number (`1.4`, `6`,
-`1000`, etc) and throttles the number of requests per second that the reindex
-issues or it can be set to `-1` to disabled throttling. The throttling is done
-waiting between bulk batches so that it can manipulate the scroll timeout. The
-wait time is the difference between the time it took the batch to complete and
-the time `requests_per_second * requests_in_the_batch`. Since the batch isn't
-broken into multiple bulk requests large batch sizes will cause Elasticsearch
-to create many requests and then wait for a while before starting the next set.
-This is "bursty" instead of "smooth". The default is `-1`.
+`1000`, etc) and throttles rate at which reindex issues batches of index
+operations by padding each batch with a wait time. The throttling can be
+disabled by setting `requests_per_second` to `-1`.
+
+The throttling is done by waiting between batches so that scroll that reindex
+uses internally can be given a timeout that takes into account the padding.
+The padding time is the difference between the batch size divided by the
+`requests_per_second` and the time spent writing. By default the batch size is
+`1000`, so if the `requests_per_second` is set to `500`:
+
+[source,txt]
+--------------------------------------------------
+target_time = 1000 / 500 per second = 2 seconds
+wait_time = target_time - write_time = 2 seconds - .5 seconds = 1.5 seconds
+--------------------------------------------------
+
+Since the batch is issued as a single `_bulk` request large batch sizes will
+cause Elasticsearch to create many requests and then wait for a while before
+starting the next set. This is "bursty" instead of "smooth". The default is `-1`.
 
 [float]
 [[docs-reindex-response-body]]
@@ -528,7 +633,7 @@ The responses looks like:
 {
   "nodes" : {
     "r1A2WoRbTwKZ516z6NEs5A" : {
-      "name" : "Tyrannus",
+      "name" : "r1A2WoR",
       "transport_address" : "127.0.0.1:9300",
       "host" : "127.0.0.1",
       "ip" : "127.0.0.1:9300",
@@ -595,7 +700,7 @@ Any Reindex can be canceled using the <<tasks,Task Cancel API>>:
 
 [source,js]
 --------------------------------------------------
-POST _tasks/taskid:1/_cancel
+POST _tasks/task_id:1/_cancel
 --------------------------------------------------
 // CONSOLE
 
@@ -614,7 +719,7 @@ the `_rethrottle` API:
 
 [source,js]
 --------------------------------------------------
-POST _reindex/taskid:1/_rethrottle?requests_per_second=-1
+POST _reindex/task_id:1/_rethrottle?requests_per_second=-1
 --------------------------------------------------
 // CONSOLE
 
@@ -658,7 +763,7 @@ POST _reindex
     "index": "test2"
   },
   "script": {
-    "inline": "ctx._source.tag = ctx._source.remove(\"flag\")"
+    "source": "ctx._source.tag = ctx._source.remove(\"flag\")"
   }
 }
 --------------------------------------------------
@@ -694,6 +799,149 @@ and it'll look like:
 
 Or you can search by `tag` or whatever you want.
 
+[float]
+[[docs-reindex-manual-slice]]
+==== Manual slicing
+Reindex supports <<sliced-scroll>>, allowing you to manually parallelize the
+process relatively easily:
+
+[source,js]
+----------------------------------------------------------------
+POST _reindex
+{
+  "source": {
+    "index": "twitter",
+    "slice": {
+      "id": 0,
+      "max": 2
+    }
+  },
+  "dest": {
+    "index": "new_twitter"
+  }
+}
+POST _reindex
+{
+  "source": {
+    "index": "twitter",
+    "slice": {
+      "id": 1,
+      "max": 2
+    }
+  },
+  "dest": {
+    "index": "new_twitter"
+  }
+}
+----------------------------------------------------------------
+// CONSOLE
+// TEST[setup:big_twitter]
+
+Which you can verify works with:
+
+[source,js]
+----------------------------------------------------------------
+GET _refresh
+POST new_twitter/_search?size=0&filter_path=hits.total
+----------------------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+Which results in a sensible `total` like this one:
+
+[source,js]
+----------------------------------------------------------------
+{
+  "hits": {
+    "total": 120
+  }
+}
+----------------------------------------------------------------
+// TESTRESPONSE
+
+[float]
+[[docs-reindex-automatic-slice]]
+=== Automatic slicing
+
+You can also let reindex automatically parallelize using <<sliced-scroll>> to
+slice on `_uid`:
+
+[source,js]
+----------------------------------------------------------------
+POST _reindex?slices=5&refresh
+{
+  "source": {
+    "index": "twitter"
+  },
+  "dest": {
+    "index": "new_twitter"
+  }
+}
+----------------------------------------------------------------
+// CONSOLE
+// TEST[setup:big_twitter]
+
+Which you also can verify works with:
+
+[source,js]
+----------------------------------------------------------------
+POST new_twitter/_search?size=0&filter_path=hits.total
+----------------------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+Which results in a sensible `total` like this one:
+
+[source,js]
+----------------------------------------------------------------
+{
+  "hits": {
+    "total": 120
+  }
+}
+----------------------------------------------------------------
+// TESTRESPONSE
+
+Adding `slices` to `_reindex` just automates the manual process used in the
+section above, creating sub-requests which means it has some quirks:
+
+* You can see these requests in the <<docs-reindex-task-api,Tasks APIs>>. These
+sub-requests are "child" tasks of the task for the request with `slices`.
+* Fetching the status of the task for the request with `slices` only contains
+the status of completed slices.
+* These sub-requests are individually addressable for things like cancellation
+and rethrottling.
+* Rethrottling the request with `slices` will rethrottle the unfinished
+sub-request proportionally.
+* Canceling the request with `slices` will cancel each sub-request.
+* Due to the nature of `slices` each sub-request won't get a perfectly even
+portion of the documents. All documents will be addressed, but some slices may
+be larger than others. Expect larger slices to have a more even distribution.
+* Parameters like `requests_per_second` and `size` on a request with `slices`
+are distributed proportionally to each sub-request. Combine that with the point
+above about distribution being uneven and you should conclude that the using
+`size` with `slices` might not result in exactly `size` documents being
+`_reindex`ed.
+* Each sub-requests gets a slightly different snapshot of the source index
+though these are all taken at approximately the same time.
+
+[float]
+[[docs-reindex-picking-slices]]
+=== Picking the number of slices
+
+At this point we have a few recommendations around the number of `slices` to
+use (the `max` parameter in the slice API if manually parallelizing):
+
+* Don't use large numbers. `500` creates fairly massive CPU thrash.
+* It is more efficient from a query performance standpoint to use some multiple
+of the number of shards in the source index.
+* Using exactly as many shards as are in the source index is the most efficient
+from a query performance standpoint.
+* Indexing performance should scale linearly across available resources with
+the number of `slices`.
+* Whether indexing or query performance dominates that process depends on lots
+of factors like the documents being reindexed and the cluster doing the
+reindexing.
 
 [float]
 === Reindex daily indices
@@ -732,7 +980,7 @@ POST _reindex
   },
   "script": {
     "lang": "painless",
-    "inline": "ctx._index = 'metricbeat-' + (ctx._index.substring('metricbeat-'.length(), ctx._index.length())) + '-1'"
+    "source": "ctx._index = 'metricbeat-' + (ctx._index.substring('metricbeat-'.length(), ctx._index.length())) + '-1'"
   }
 }
 ----------------------------------------------------------------
@@ -751,3 +999,34 @@ GET metricbeat-2016.05.31-1/beat/1
 
 The previous method can also be used in combination with <<docs-reindex-change-name, change the name of a field>>
 to only load the existing data into the new index, but also rename fields if needed.
+
+[float]
+=== Extracting a random subset of an index
+
+Reindex can be used to extract a random subset of an index for testing:
+
+[source,js]
+----------------------------------------------------------------
+POST _reindex
+{
+  "size": 10,
+  "source": {
+    "index": "twitter",
+    "query": {
+      "function_score" : {
+        "query" : { "match_all": {} },
+        "random_score" : {}
+      }
+    },
+    "sort": "_score"    <1>
+  },
+  "dest": {
+    "index": "random_twitter"
+  }
+}
+----------------------------------------------------------------
+// CONSOLE
+// TEST[setup:big_twitter]
+
+<1> Reindex defaults to sorting by `_doc` so `random_score` won't have any
+effect unless you override the sort to `_score`.
diff --git a/docs/reference/docs/termvectors.asciidoc b/docs/reference/docs/termvectors.asciidoc
index 586f40432b3c2..05372d13f30b7 100644
--- a/docs/reference/docs/termvectors.asciidoc
+++ b/docs/reference/docs/termvectors.asciidoc
@@ -8,16 +8,20 @@ realtime. This can be changed by setting `realtime` parameter to `false`.
 
 [source,js]
 --------------------------------------------------
-curl -XGET 'http://localhost:9200/twitter/tweet/1/_termvectors?pretty=true'
+GET /twitter/tweet/1/_termvectors
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
 
 Optionally, you can specify the fields for which the information is
 retrieved either with a parameter in the url
 
 [source,js]
 --------------------------------------------------
-curl -XGET 'http://localhost:9200/twitter/tweet/1/_termvectors?fields=text,...'
+GET /twitter/tweet/1/_termvectors?fields=message
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
 
 or by adding the requested fields in the request body (see
 example below). Fields can also be specified with wildcards
@@ -117,15 +121,15 @@ whereas the absolute numbers have no meaning in this context. By default,
 when requesting term vectors of artificial documents, a shard to get the statistics
 from is randomly selected. Use `routing` only to hit a particular shard.
 
-.Returning stored term vectors
-==================================================
+[float]
+==== Example: Returning stored term vectors
 
 First, we create an index that stores term vectors, payloads etc. :
 
 [source,js]
 --------------------------------------------------
-curl -s -XPUT 'http://localhost:9200/twitter/' -d '{
-  "mappings": {
+PUT /twitter/
+{ "mappings": {
     "tweet": {
       "properties": {
         "text": {
@@ -160,51 +164,58 @@ curl -s -XPUT 'http://localhost:9200/twitter/' -d '{
       }
     }
   }
-}'
+}
 --------------------------------------------------
+// CONSOLE
 
 Second, we add some documents:
 
 [source,js]
 --------------------------------------------------
-curl -XPUT 'http://localhost:9200/twitter/tweet/1?pretty=true' -d '{
+PUT /twitter/tweet/1
+{
   "fullname" : "John Doe",
   "text" : "twitter test test test "
-}'
+}
 
-curl -XPUT 'http://localhost:9200/twitter/tweet/2?pretty=true' -d '{
+PUT /twitter/tweet/2
+{
   "fullname" : "Jane Doe",
   "text" : "Another twitter test ..."
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 The following request returns all information and statistics for field
 `text` in document `1` (John Doe):
 
 [source,js]
 --------------------------------------------------
-
-curl -XGET 'http://localhost:9200/twitter/tweet/1/_termvectors?pretty=true' -d '{
+GET /twitter/tweet/1/_termvectors
+{
   "fields" : ["text"],
   "offsets" : true,
   "payloads" : true,
   "positions" : true,
   "term_statistics" : true,
   "field_statistics" : true
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 Response:
 
 [source,js]
 --------------------------------------------------
-
 {
     "_id": "1",
     "_index": "twitter",
     "_type": "tweet",
     "_version": 1,
     "found": true,
+    "took": 6,
     "term_vectors": {
         "text": {
             "field_statistics": {
@@ -256,10 +267,11 @@ Response:
     }
 }
 --------------------------------------------------
-==================================================
+// TEST[continued]
+// TESTRESPONSE[s/"took": 6/"took": "$body.took"/]
 
-.Generating term vectors on the fly
-==================================================
+[float]
+==== Example: Generating term vectors on the fly
 
 Term vectors which are not explicitly stored in the index are automatically
 computed on the fly. The following request returns all information and statistics for the
@@ -268,20 +280,21 @@ Note that for the field `text`, the terms are not re-generated.
 
 [source,js]
 --------------------------------------------------
-curl -XGET 'http://localhost:9200/twitter/tweet/1/_termvectors?pretty=true' -d '{
+GET /twitter/tweet/1/_termvectors
+{
   "fields" : ["text", "some_field_without_term_vectors"],
   "offsets" : true,
   "positions" : true,
   "term_statistics" : true,
   "field_statistics" : true
-}'
+}
 --------------------------------------------------
-==================================================
+// CONSOLE
+// TEST[continued]
 
 [[docs-termvectors-artificial-doc]]
-[example]
-.Artificial documents
---
+[float]
+==== Example: Artificial documents
 
 Term vectors can also be generated for artificial documents,
 that is for documents not present in the index.  For example, the following request would
@@ -293,18 +306,20 @@ mapping will be dynamically created.*
 
 [source,js]
 --------------------------------------------------
-curl -XGET 'http://localhost:9200/twitter/tweet/_termvectors' -d '{
+GET /twitter/tweet/_termvectors
+{
   "doc" : {
     "fullname" : "John Doe",
     "text" : "twitter test test test"
   }
-}'
+}
 --------------------------------------------------
---
+// CONSOLE
+// TEST[continued]
 
 [[docs-termvectors-per-field-analyzer]]
-.Per-field analyzer
-==================================================
+[float]
+===== Per-field analyzer
 
 Additionally, a different analyzer than the one at the field may be provided
 by using the `per_field_analyzer` parameter. This is useful in order to
@@ -314,7 +329,8 @@ vectors, the term vectors will be re-generated.
 
 [source,js]
 --------------------------------------------------
-curl -XGET 'http://localhost:9200/twitter/tweet/_termvectors' -d '{
+GET /twitter/tweet/_termvectors
+{
   "doc" : {
     "fullname" : "John Doe",
     "text" : "twitter test test test"
@@ -323,8 +339,10 @@ curl -XGET 'http://localhost:9200/twitter/tweet/_termvectors' -d '{
   "per_field_analyzer" : {
     "fullname": "keyword"
   }
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 Response:
 
@@ -335,12 +353,13 @@ Response:
   "_type": "tweet",
   "_version": 0,
   "found": true,
+  "took": 6,
   "term_vectors": {
     "fullname": {
        "field_statistics": {
-          "sum_doc_freq": 1,
-          "doc_count": 1,
-          "sum_ttf": 1
+          "sum_doc_freq": 2,
+          "doc_count": 4,
+          "sum_ttf": 4
        },
        "terms": {
           "John Doe": {
@@ -358,11 +377,16 @@ Response:
   }
 }
 --------------------------------------------------
-==================================================
+// TEST[continued]
+// TESTRESPONSE[s/"took": 6/"took": "$body.took"/]
+// TESTRESPONSE[s/"sum_doc_freq": 2/"sum_doc_freq": "$body.term_vectors.fullname.field_statistics.sum_doc_freq"/]
+// TESTRESPONSE[s/"doc_count": 4/"doc_count": "$body.term_vectors.fullname.field_statistics.doc_count"/]
+// TESTRESPONSE[s/"sum_ttf": 4/"sum_ttf": "$body.term_vectors.fullname.field_statistics.sum_ttf"/]
+
 
 [[docs-termvectors-terms-filtering]]
-.Terms filtering
-==================================================
+[float]
+==== Example: Terms filtering
 
 Finally, the terms returned could be filtered based on their tf-idf scores. In
 the example below we obtain the three most "interesting" keywords from the
@@ -388,6 +412,8 @@ GET /imdb/movies/_termvectors
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[skip:no imdb test index]
 
 Response:
 
@@ -429,4 +455,4 @@ Response:
    }
 }
 --------------------------------------------------
-==================================================
+// TESTRESPONSE
diff --git a/docs/reference/docs/update-by-query.asciidoc b/docs/reference/docs/update-by-query.asciidoc
index 7d1f74be83521..a14647441f499 100644
--- a/docs/reference/docs/update-by-query.asciidoc
+++ b/docs/reference/docs/update-by-query.asciidoc
@@ -1,8 +1,6 @@
 [[docs-update-by-query]]
 == Update By Query API
 
-experimental[The update-by-query API is new and should still be considered experimental.  The API may change in ways that are not backwards compatible]
-
 The simplest usage of `_update_by_query` just performs an update on every
 document in the index without changing the source. This is useful to
 <<picking-up-a-new-property,pick up a new property>> or some other online
@@ -53,9 +51,9 @@ version number, documents with version equal to zero cannot be updated using
 All update and query failures cause the `_update_by_query` to abort and are
 returned in the `failures` of the response. The updates that have been
 performed still stick. In other words, the process is not rolled back, only
-aborted. While the first failure causes the abort all failures that are
-returned by the failing bulk request are returned in the `failures` element so
-it's possible for there to be quite a few.
+aborted. While the first failure causes the abort, all failures that are
+returned by the failing bulk request are returned in the `failures` element; therefore
+it's possible for there to be quite a few failed entities.
 
 If you want to simply count version conflicts not cause the `_update_by_query`
 to abort you can set `conflicts=proceed` on the url or `"conflicts": "proceed"`
@@ -108,7 +106,7 @@ will increment the `likes` field on all of kimchy's tweets:
 POST twitter/_update_by_query
 {
   "script": {
-    "inline": "ctx._source.likes++",
+    "source": "ctx._source.likes++",
     "lang": "painless"
   },
   "query": {
@@ -217,20 +215,31 @@ to keep or remove as you see fit. When you are done with it, delete it so
 Elasticsearch can reclaim the space it uses.
 
 `wait_for_active_shards` controls how many copies of a shard must be active
-before proceeding with the request. See <<index-wait-for-active-shards,here>> 
+before proceeding with the request. See <<index-wait-for-active-shards,here>>
 for details. `timeout` controls how long each write request waits for unavailable
 shards to become available. Both work exactly how they work in the
 <<docs-bulk,Bulk API>>.
 
 `requests_per_second` can be set to any positive decimal number (`1.4`, `6`,
-`1000`, etc) and throttles the number of requests per second that the update-by-query
-issues or it can be set to `-1` to disabled throttling. The throttling is done
-waiting between bulk batches so that it can manipulate the scroll timeout. The
-wait time is the difference between the time it took the batch to complete and
-the time `requests_per_second * requests_in_the_batch`. Since the batch isn't
-broken into multiple bulk requests large batch sizes will cause Elasticsearch
-to create many requests and then wait for a while before starting the next set.
-This is "bursty" instead of "smooth". The default is `-1`.
+`1000`, etc) and throttles rate at which `_update_by_query` issues batches of
+index operations by padding each batch with a wait time. The throttling can be
+disabled by setting `requests_per_second` to `-1`.
+
+The throttling is done by waiting between batches so that scroll that
+`_update_by_query` uses internally can be given a timeout that takes into
+account the padding. The padding time is the difference between the batch size
+divided by the `requests_per_second` and the time spent writing. By default the
+batch size is `1000`, so if the `requests_per_second` is set to `500`:
+
+[source,txt]
+--------------------------------------------------
+target_time = 1000 / 500 per second = 2 seconds
+wait_time = target_time - delete_time = 2 seconds - .5 seconds = 1.5 seconds
+--------------------------------------------------
+
+Since the batch is issued as a single `_bulk` request large batch sizes will
+cause Elasticsearch to create many requests and then wait for a while before
+starting the next set. This is "bursty" instead of "smooth". The default is `-1`.
 
 [float]
 [[docs-update-by-query-response-body]]
@@ -295,7 +304,7 @@ You can fetch the status of all running update-by-query requests with the
 
 [source,js]
 --------------------------------------------------
-GET _tasks?detailed=true&action=*byquery
+GET _tasks?detailed=true&actions=*byquery
 --------------------------------------------------
 // CONSOLE
 
@@ -306,7 +315,7 @@ The responses looks like:
 {
   "nodes" : {
     "r1A2WoRbTwKZ516z6NEs5A" : {
-      "name" : "Tyrannus",
+      "name" : "r1A2WoR",
       "transport_address" : "127.0.0.1:9300",
       "host" : "127.0.0.1",
       "ip" : "127.0.0.1:9300",
@@ -373,13 +382,13 @@ Any Update By Query can be canceled using the <<tasks,Task Cancel API>>:
 
 [source,js]
 --------------------------------------------------
-POST _tasks/taskid:1/_cancel
+POST _tasks/task_id:1/_cancel
 --------------------------------------------------
 // CONSOLE
 
 The `task_id` can be found using the tasks API above.
 
-Cancelation should happen quickly but might take a few seconds. The task status
+Cancellation should happen quickly but might take a few seconds. The task status
 API above will continue to list the task until it is wakes to cancel itself.
 
 
@@ -392,7 +401,7 @@ using the `_rethrottle` API:
 
 [source,js]
 --------------------------------------------------
-POST _update_by_query/taskid:1/_rethrottle?requests_per_second=-1
+POST _update_by_query/task_id:1/_rethrottle?requests_per_second=-1
 --------------------------------------------------
 // CONSOLE
 
@@ -405,6 +414,143 @@ query takes effect immediately but rethrotting that slows down the query will
 take effect on after completing the current batch. This prevents scroll
 timeouts.
 
+[float]
+[[docs-update-by-query-manual-slice]]
+==== Manual slicing
+Update-by-query supports <<sliced-scroll>> allowing you to manually parallelize
+the process relatively easily:
+
+[source,js]
+----------------------------------------------------------------
+POST twitter/_update_by_query
+{
+  "slice": {
+    "id": 0,
+    "max": 2
+  },
+  "script": {
+    "source": "ctx._source['extra'] = 'test'"
+  }
+}
+POST twitter/_update_by_query
+{
+  "slice": {
+    "id": 1,
+    "max": 2
+  },
+  "script": {
+    "source": "ctx._source['extra'] = 'test'"
+  }
+}
+----------------------------------------------------------------
+// CONSOLE
+// TEST[setup:big_twitter]
+
+Which you can verify works with:
+
+[source,js]
+----------------------------------------------------------------
+GET _refresh
+POST twitter/_search?size=0&q=extra:test&filter_path=hits.total
+----------------------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+Which results in a sensible `total` like this one:
+
+[source,js]
+----------------------------------------------------------------
+{
+  "hits": {
+    "total": 120
+  }
+}
+----------------------------------------------------------------
+// TESTRESPONSE
+
+[float]
+[[docs-update-by-query-automatic-slice]]
+=== Automatic slicing
+
+You can also let update-by-query automatically parallelize using
+<<sliced-scroll>> to slice on `_uid`:
+
+[source,js]
+----------------------------------------------------------------
+POST twitter/_update_by_query?refresh&slices=5
+{
+  "script": {
+    "source": "ctx._source['extra'] = 'test'"
+  }
+}
+----------------------------------------------------------------
+// CONSOLE
+// TEST[setup:big_twitter]
+
+Which you also can verify works with:
+
+[source,js]
+----------------------------------------------------------------
+POST twitter/_search?size=0&q=extra:test&filter_path=hits.total
+----------------------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+Which results in a sensible `total` like this one:
+
+[source,js]
+----------------------------------------------------------------
+{
+  "hits": {
+    "total": 120
+  }
+}
+----------------------------------------------------------------
+// TESTRESPONSE
+
+Adding `slices` to `_update_by_query` just automates the manual process used in
+the section above, creating sub-requests which means it has some quirks:
+
+* You can see these requests in the
+<<docs-update-by-query-task-api,Tasks APIs>>. These sub-requests are "child"
+tasks of the task for the request with `slices`.
+* Fetching the status of the task for the request with `slices` only contains
+the status of completed slices.
+* These sub-requests are individually addressable for things like cancellation
+and rethrottling.
+* Rethrottling the request with `slices` will rethrottle the unfinished
+sub-request proportionally.
+* Canceling the request with `slices` will cancel each sub-request.
+* Due to the nature of `slices` each sub-request won't get a perfectly even
+portion of the documents. All documents will be addressed, but some slices may
+be larger than others. Expect larger slices to have a more even distribution.
+* Parameters like `requests_per_second` and `size` on a request with `slices`
+are distributed proportionally to each sub-request. Combine that with the point
+above about distribution being uneven and you should conclude that the using
+`size` with `slices` might not result in exactly `size` documents being
+`_update_by_query`ed.
+* Each sub-requests gets a slightly different snapshot of the source index
+though these are all taken at approximately the same time.
+
+[float]
+[[docs-update-by-query-picking-slices]]
+=== Picking the number of slices
+
+At this point we have a few recommendations around the number of `slices` to
+use (the `max` parameter in the slice API if manually parallelizing):
+
+* Don't use large numbers. `500` creates fairly massive CPU thrash.
+* It is more efficient from a query performance standpoint to use some multiple
+of the number of shards in the source index.
+* Using exactly as many shards as are in the source index is the most efficient
+from a query performance standpoint.
+* Indexing performance should scale linearly across available resources with
+the number of `slices`.
+* Whether indexing or query performance dominates that process depends on lots
+of factors like the documents being reindexed and the cluster doing the
+reindexing.
+
+
 [float]
 [[picking-up-a-new-property]]
 === Pick up a new property
diff --git a/docs/reference/docs/update.asciidoc b/docs/reference/docs/update.asciidoc
index 0cf942121afea..d19e693944672 100644
--- a/docs/reference/docs/update.asciidoc
+++ b/docs/reference/docs/update.asciidoc
@@ -17,11 +17,13 @@ For example, lets index a simple doc:
 
 [source,js]
 --------------------------------------------------
-curl -XPUT localhost:9200/test/type1/1 -d '{
+PUT test/type1/1
+{
     "counter" : 1,
     "tags" : ["red"]
-}'
+}
 --------------------------------------------------
+// CONSOLE
 
 [float]
 === Scripted updates
@@ -30,71 +32,86 @@ Now, we can execute a script that would increment the counter:
 
 [source,js]
 --------------------------------------------------
-curl -XPOST 'localhost:9200/test/type1/1/_update' -d '{
+POST test/type1/1/_update
+{
     "script" : {
-        "inline": "ctx._source.counter += params.count",
+        "source": "ctx._source.counter += params.count",
         "lang": "painless",
         "params" : {
             "count" : 4
         }
     }
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 We can add a tag to the list of tags (note, if the tag exists, it
 will still add it, since its a list):
 
 [source,js]
 --------------------------------------------------
-curl -XPOST 'localhost:9200/test/type1/1/_update' -d '{
+POST test/type1/1/_update
+{
     "script" : {
-        "inline": "ctx._source.tags.add(params.tag)",
+        "source": "ctx._source.tags.add(params.tag)",
         "lang": "painless",
         "params" : {
             "tag" : "blue"
         }
     }
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 In addition to `_source`, the following variables are available through
 the `ctx` map: `_index`, `_type`, `_id`, `_version`, `_routing`,
-`_parent`.
+`_parent`, and `_now` (the current timestamp).
 
 We can also add a new field to the document:
 
 [source,js]
 --------------------------------------------------
-curl -XPOST 'localhost:9200/test/type1/1/_update' -d '{
-    "script" : "ctx._source.name_of_new_field = \"value_of_new_field\""
-}'
+POST test/type1/1/_update
+{
+    "script" : "ctx._source.new_field = 'value_of_new_field'"
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 Or remove a field from the document:
 
 [source,js]
 --------------------------------------------------
-curl -XPOST 'localhost:9200/test/type1/1/_update' -d '{
-    "script" : "ctx._source.remove(\"name_of_field\")"
-}'
+POST test/type1/1/_update
+{
+    "script" : "ctx._source.remove('new_field')"
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 And, we can even change the operation that is executed.  This example deletes
-the doc if the `tags` field contain `blue`, otherwise it does nothing
+the doc if the `tags` field contain `green`, otherwise it does nothing
 (`noop`):
 
 [source,js]
 --------------------------------------------------
-curl -XPOST 'localhost:9200/test/type1/1/_update' -d '{
+POST test/type1/1/_update
+{
     "script" : {
-        "inline": "ctx._source.tags.contains(params.tag) ? ctx.op = \"delete\" : ctx.op = \"none\"",
+        "source": "if (ctx._source.tags.contains(params.tag)) { ctx.op = 'delete' } else { ctx.op = 'none' }",
         "lang": "painless",
         "params" : {
-            "tag" : "blue"
+            "tag" : "green"
         }
     }
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 [float]
 === Updates with a partial document
@@ -106,31 +123,36 @@ example:
 
 [source,js]
 --------------------------------------------------
-curl -XPOST 'localhost:9200/test/type1/1/_update' -d '{
+POST test/type1/1/_update
+{
     "doc" : {
         "name" : "new_name"
     }
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 If both `doc` and `script` are specified, then `doc` is ignored. Best is
 to put your field pairs of the partial document in the script itself.
 
 [float]
 === Detecting noop updates
-If `doc` is specified its value is merged with the existing `_source`. By
-default the document is only reindexed if the new `_source` field differs from
-the old. Setting `detect_noop` to `false` will cause Elasticsearch to always
-update the document even if it hasn't changed. For example:
+
+If `doc` is specified its value is merged with the existing `_source`.
+By default updates that don't change anything detect that they don't change anything and return "result": "noop" like this:
+
 [source,js]
 --------------------------------------------------
-curl -XPOST 'localhost:9200/test/type1/1/_update' -d '{
+POST test/type1/1/_update
+{
     "doc" : {
         "name" : "new_name"
-    },
-    "detect_noop": false
-}'
+    }
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 If `name` was `new_name` before the request was sent then the entire update
 request is ignored. The `result` element in the response returns `noop` if
@@ -139,13 +161,34 @@ the request was ignored.
 [source,js]
 --------------------------------------------------
 {
+   "_shards": {
+        "total": 0,
+        "successful": 0,
+        "failed": 0
+   },
    "_index": "test",
    "_type": "type1",
    "_id": "1",
-   "_version": 1,
+   "_version": 6,
    "result": noop
 }
 --------------------------------------------------
+// TESTRESPONSE
+
+You can disable this behavior by setting "detect_noop": false like this:
+
+[source,js]
+--------------------------------------------------
+POST test/type1/1/_update
+{
+    "doc" : {
+        "name" : "new_name"
+    },
+    "detect_noop": false
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 [[upserts]]
 [float]
@@ -157,9 +200,10 @@ will be inserted as a new document.  If the document does exist, then the
 
 [source,js]
 --------------------------------------------------
-curl -XPOST 'localhost:9200/test/type1/1/_update' -d '{
+POST test/type1/1/_update
+{
     "script" : {
-        "inline": "ctx._source.counter += params.count",
+        "source": "ctx._source.counter += params.count",
         "lang": "painless",
         "params" : {
             "count" : 4
@@ -168,8 +212,10 @@ curl -XPOST 'localhost:9200/test/type1/1/_update' -d '{
     "upsert" : {
         "counter" : 1
     }
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 [float]
 ==== `scripted_upsert`
@@ -180,7 +226,8 @@ or not -- i.e. the script handles initializing the document instead of the
 
 [source,js]
 --------------------------------------------------
-curl -XPOST 'localhost:9200/sessions/session/dh3sgudg8gsrgl/_update' -d '{
+POST sessions/session/dh3sgudg8gsrgl/_update
+{
     "scripted_upsert":true,
     "script" : {
         "id": "my_web_session_summariser",
@@ -193,8 +240,11 @@ curl -XPOST 'localhost:9200/sessions/session/dh3sgudg8gsrgl/_update' -d '{
         }
     },
     "upsert" : {}
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[s/"id": "my_web_session_summariser"/"source": "ctx._source.page_view_event = params.pageViewEvent"/]
+// TEST[continued]
 
 [float]
 ==== `doc_as_upsert`
@@ -205,14 +255,16 @@ value:
 
 [source,js]
 --------------------------------------------------
-curl -XPOST 'localhost:9200/test/type1/1/_update' -d '{
+POST test/type1/1/_update
+{
     "doc" : {
         "name" : "new_name"
     },
     "doc_as_upsert" : true
-}'
+}
 --------------------------------------------------
-
+// CONSOLE
+// TEST[continued]
 
 [float]
 === Parameters
@@ -255,10 +307,12 @@ See <<index-wait-for-active-shards,here>> for details.
 Control when the changes made by this request are visible to search. See
 <<docs-refresh>>.
 
-`fields`::
+`_source`::
+
+Allows to control if and how the updated source should be returned in the response.
+By default the updated source is not returned.
+See <<search-request-source-filtering, `source filtering`>> for details.
 
-Return the relevant fields from the updated document. Specify `_source` to
-return the full updated source.
 
 `version` & `version_type`::
 
diff --git a/docs/reference/getting-started.asciidoc b/docs/reference/getting-started.asciidoc
index fa257742bbea3..e120b0f8d40d2 100755
--- a/docs/reference/getting-started.asciidoc
+++ b/docs/reference/getting-started.asciidoc
@@ -40,7 +40,8 @@ Note that it is valid and perfectly fine to have a cluster with only a single no
 [float]
 === Node
 
-A node is a single server that is part of your cluster, stores your data, and participates in the cluster's indexing and search capabilities. Just like a cluster, a node is identified by a name which by default is a random Marvel character name that is assigned to the node at startup. You can define any node name you want if you do not want the default.  This name is important for administration purposes where you want to identify which servers in your network correspond to which nodes in your Elasticsearch cluster.
+A node is a single server that is part of your cluster, stores your data, and participates in the cluster's indexing and search
+capabilities. Just like a cluster, a node is identified by a name which by default is a random Universally Unique IDentifier (UUID) that is assigned to the node at startup. You can define any node name you want if you do not want the default.  This name is important for administration purposes where you want to identify which servers in your network correspond to which nodes in your Elasticsearch cluster.
 
 A node can be configured to join a specific cluster by the cluster name. By default, each node is set up to join a cluster named `elasticsearch` which means that if you start up a number of nodes on your network and--assuming they can discover each other--they will all automatically form and join a single cluster named `elasticsearch`.
 
@@ -66,6 +67,7 @@ A document is a basic unit of information that can be indexed. For example, you
 
 Within an index/type, you can store as many documents as you want. Note that although a document physically resides in an index, a document actually must be indexed/assigned to a type inside an index.
 
+[[getting-started-shards-and-replicas]]
 [float]
 === Shards & Replicas
 
@@ -90,12 +92,12 @@ Replication is important for two primary reasons:
 
 
 To summarize, each index can be split into multiple shards. An index can also be replicated zero (meaning no replicas) or more times. Once replicated, each index will have primary shards (the original shards that were replicated from) and replica shards (the copies of the primary shards).
-The number of shards and replicas can be defined per index at the time the index is created. After the index is created, you may change the number of replicas dynamically anytime but you cannot change the number shards after-the-fact.
+The number of shards and replicas can be defined per index at the time the index is created. After the index is created, you may change the number of replicas dynamically anytime but you cannot change the number of shards after-the-fact.
 
 By default, each index in Elasticsearch is allocated 5 primary shards and 1 replica which means that if you have at least two nodes in your cluster, your index will have 5 primary shards and another 5 replica shards (1 complete replica) for a total of 10 shards per index.
 
 NOTE: Each Elasticsearch shard is a Lucene index.  There is a maximum number of documents you can have in a single Lucene index.  As of https://issues.apache.org/jira/browse/LUCENE-5843[`LUCENE-5843`], the limit is `2,147,483,519` (= Integer.MAX_VALUE - 128) documents.
-You can monitor shard sizes using the <<cat-shards,`_cat/shards`>> api.
+You can monitor shard sizes using the {ref}/cat-shards.html[`_cat/shards`] API.
 
 With that out of the way, let's get started with the fun part...
 
@@ -109,16 +111,22 @@ java -version
 echo $JAVA_HOME
 --------------------------------------------------
 
-Once we have Java set up, we can then download and run Elasticsearch. The binaries are available from http://www.elastic.co/downloads[`www.elastic.co/downloads`] along with all the releases that have been made in the past. For each release, you have a choice among a `zip` or `tar` archive, or a `DEB` or `RPM` package. For simplicity, let's use the tar file.
+Once we have Java set up, we can then download and run Elasticsearch. The binaries are available from http://www.elastic.co/downloads[`www.elastic.co/downloads`] along with all the releases that have been made in the past. For each release, you have a choice among a `zip` or `tar` archive, a `DEB` or `RPM` package, or a Windows `MSI` installation package.
 
-Let's download the Elasticsearch {version} tar as follows (Windows users should download the zip package):
+[float]
+=== Installation example with tar
+
+For simplicity, let's use the {ref}/zip-targz.html[tar] file.
+
+Let's download the Elasticsearch {version} tar as follows:
 
 ["source","sh",subs="attributes,callouts"]
 --------------------------------------------------
-curl -L -O https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/{version}/elasticsearch-{version}.tar.gz
+curl -L -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.tar.gz
 --------------------------------------------------
+// NOTCONSOLE
 
-Then extract it as follows (Windows users should unzip the zip package):
+Then extract it as follows:
 
 ["source","sh",subs="attributes,callouts"]
 --------------------------------------------------
@@ -132,32 +140,114 @@ It will then create a bunch of files and folders in your current directory. We t
 cd elasticsearch-{version}/bin
 --------------------------------------------------
 
-And now we are ready to start our node and single cluster (Windows users should run the elasticsearch.bat file):
+And now we are ready to start our node and single cluster:
 
 [source,sh]
 --------------------------------------------------
 ./elasticsearch
 --------------------------------------------------
 
-If everything goes well, you should see a bunch of messages that look like below:
+[float]
+=== Instalation with Homebrew
 
-["source","sh",subs="attributes,callouts"]
+On macOS, Elasticsearch can also be installed via https://brew.sh[Homebrew]:
+
+["source","sh"]
 --------------------------------------------------
-./elasticsearch
-[2014-03-13 13:42:17,218][INFO ][node           ] [New Goblin] version[{version}], pid[2085], build[5c03844/2014-02-25T15:52:53Z]
-[2014-03-13 13:42:17,219][INFO ][node           ] [New Goblin] initializing ...
-[2014-03-13 13:42:17,223][INFO ][plugins        ] [New Goblin] loaded [], sites []
-[2014-03-13 13:42:19,831][INFO ][node           ] [New Goblin] initialized
-[2014-03-13 13:42:19,832][INFO ][node           ] [New Goblin] starting ...
-[2014-03-13 13:42:19,958][INFO ][transport      ] [New Goblin] bound_address {inet[/0:0:0:0:0:0:0:0:9300]}, publish_address {inet[/192.168.8.112:9300]}
-[2014-03-13 13:42:23,030][INFO ][cluster.service] [New Goblin] new_master [New Goblin][rWMtGj3dQouz2r6ZFL9v4g][mwubuntu1][inet[/192.168.8.112:9300]], reason: zen-disco-join (elected_as_master)
-[2014-03-13 13:42:23,100][INFO ][discovery      ] [New Goblin] elasticsearch/rWMtGj3dQouz2r6ZFL9v4g
-[2014-03-13 13:42:23,125][INFO ][http           ] [New Goblin] bound_address {inet[/0:0:0:0:0:0:0:0:9200]}, publish_address {inet[/192.168.8.112:9200]}
-[2014-03-13 13:42:23,629][INFO ][gateway        ] [New Goblin] recovered [1] indices into cluster_state
-[2014-03-13 13:42:23,630][INFO ][node           ] [New Goblin] started
+brew install elasticsearch
+--------------------------------------------------
+
+[float]
+=== Installation example with MSI Windows Installer
+
+For Windows users, we recommend using the {ref}/windows.html[MSI Installer package]. The package contains a graphical user interface (GUI) that guides you through the installation process.
+
+First, download the Elasticsearch {version} MSI from
+https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.msi.
+
+Then double-click the downloaded file to launch the GUI. Within the first screen, select the deployment directories:
+
+[[getting-started-msi-installer-locations]]
+image::images/msi_installer/msi_installer_locations.png[]
+
+Then select whether to install as a service or start Elasticsearch manually as needed.
+To align with the tar example, choose not to install as a service:
+
+[[getting-started-msi-installer-service]]
+image::images/msi_installer/msi_installer_no_service.png[]
+
+For configuration, simply leave the default values:
+
+[[getting-started-msi-installer-configuration]]
+image::images/msi_installer/msi_installer_configuration.png[]
+
+Again, to align with the tar example, uncheck all plugins to not install any plugins:
+
+[[getting-started-msi-installer-plugins]]
+image::images/msi_installer/msi_installer_plugins.png[]
+
+After clicking the install button, Elasticsearch will be installed:
+
+[[getting-started-msi-installer-success]]
+image::images/msi_installer/msi_installer_success.png[]
+
+By default, Elasticsearch will be installed at `%PROGRAMFILES%\Elastic\Elasticsearch`. Navigate here and go into the bin directory as follows:
+
+**with Command Prompt:**
+
+[source,sh]
+--------------------------------------------------
+cd %PROGRAMFILES%\Elastic\Elasticsearch\bin
 --------------------------------------------------
 
-Without going too much into detail, we can see that our node named "New Goblin" (which will be a different Marvel character in your case) has started and elected itself as a master in a single cluster. Don't worry yet at the moment what master means. The main thing that is important here is that we have started one node within one cluster.
+**with PowerShell:**
+
+[source,powershell]
+--------------------------------------------------
+cd $env:PROGRAMFILES\Elastic\Elasticsearch\bin
+--------------------------------------------------
+
+And now we are ready to start our node and single cluster:
+
+[source,sh]
+--------------------------------------------------
+.\elasticsearch.exe
+--------------------------------------------------
+
+[float]
+=== Successfully running node
+
+If everything goes well with installation, you should see a bunch of messages that look like below:
+
+["source","sh",subs="attributes,callouts"]
+--------------------------------------------------
+[2016-09-16T14:17:51,251][INFO ][o.e.n.Node               ] [] initializing ...
+[2016-09-16T14:17:51,329][INFO ][o.e.e.NodeEnvironment    ] [6-bjhwl] using [1] data paths, mounts [[/ (/dev/sda1)]], net usable_space [317.7gb], net total_space [453.6gb], spins? [no], types [ext4]
+[2016-09-16T14:17:51,330][INFO ][o.e.e.NodeEnvironment    ] [6-bjhwl] heap size [1.9gb], compressed ordinary object pointers [true]
+[2016-09-16T14:17:51,333][INFO ][o.e.n.Node               ] [6-bjhwl] node name [6-bjhwl] derived from node ID; set [node.name] to override
+[2016-09-16T14:17:51,334][INFO ][o.e.n.Node               ] [6-bjhwl] version[{version}], pid[21261], build[f5daa16/2016-09-16T09:12:24.346Z], OS[Linux/4.4.0-36-generic/amd64], JVM[Oracle Corporation/Java HotSpot(TM) 64-Bit Server VM/1.8.0_60/25.60-b23]
+[2016-09-16T14:17:51,967][INFO ][o.e.p.PluginsService     ] [6-bjhwl] loaded module [aggs-matrix-stats]
+[2016-09-16T14:17:51,967][INFO ][o.e.p.PluginsService     ] [6-bjhwl] loaded module [ingest-common]
+[2016-09-16T14:17:51,967][INFO ][o.e.p.PluginsService     ] [6-bjhwl] loaded module [lang-expression]
+[2016-09-16T14:17:51,967][INFO ][o.e.p.PluginsService     ] [6-bjhwl] loaded module [lang-groovy]
+[2016-09-16T14:17:51,967][INFO ][o.e.p.PluginsService     ] [6-bjhwl] loaded module [lang-mustache]
+[2016-09-16T14:17:51,967][INFO ][o.e.p.PluginsService     ] [6-bjhwl] loaded module [lang-painless]
+[2016-09-16T14:17:51,967][INFO ][o.e.p.PluginsService     ] [6-bjhwl] loaded module [percolator]
+[2016-09-16T14:17:51,968][INFO ][o.e.p.PluginsService     ] [6-bjhwl] loaded module [reindex]
+[2016-09-16T14:17:51,968][INFO ][o.e.p.PluginsService     ] [6-bjhwl] loaded module [transport-netty3]
+[2016-09-16T14:17:51,968][INFO ][o.e.p.PluginsService     ] [6-bjhwl] loaded module [transport-netty4]
+[2016-09-16T14:17:51,968][INFO ][o.e.p.PluginsService     ] [6-bjhwl] loaded plugin [mapper-murmur3]
+[2016-09-16T14:17:53,521][INFO ][o.e.n.Node               ] [6-bjhwl] initialized
+[2016-09-16T14:17:53,521][INFO ][o.e.n.Node               ] [6-bjhwl] starting ...
+[2016-09-16T14:17:53,671][INFO ][o.e.t.TransportService   ] [6-bjhwl] publish_address {192.168.8.112:9300}, bound_addresses {{192.168.8.112:9300}
+[2016-09-16T14:17:53,676][WARN ][o.e.b.BootstrapCheck     ] [6-bjhwl] max virtual memory areas vm.max_map_count [65530] likely too low, increase to at least [262144]
+[2016-09-16T14:17:56,718][INFO ][o.e.c.s.ClusterService   ] [6-bjhwl] new_master {6-bjhwl}{6-bjhwl4TkajjoD2oEipnQ}{8m3SNKoFR6yQl1I0JUfPig}{192.168.8.112}{192.168.8.112:9300}, reason: zen-disco-elected-as-master ([0] nodes joined)
+[2016-09-16T14:17:56,731][INFO ][o.e.h.HttpServer         ] [6-bjhwl] publish_address {192.168.8.112:9200}, bound_addresses {[::1]:9200}, {192.168.8.112:9200}
+[2016-09-16T14:17:56,732][INFO ][o.e.g.GatewayService     ] [6-bjhwl] recovered [0] indices into cluster_state
+[2016-09-16T14:17:56,748][INFO ][o.e.n.Node               ] [6-bjhwl] started
+--------------------------------------------------
+
+Without going too much into detail, we can see that our node named "6-bjhwl" (which will be a different set of characters in your case) has started and elected itself as a master in a single cluster. Don't worry yet at the moment what master means. The main thing that is important here is that we have started one node within one cluster.
 
 As mentioned previously, we can override either the cluster or node name. This can be done from the command line when starting Elasticsearch as follows:
 
@@ -184,61 +274,71 @@ Now that we have our node (and cluster) up and running, the next step is to unde
 
 Let's start with a basic health check, which we can use to see how our cluster is doing. We'll be using curl to do this but you can use any tool that allows you to make HTTP/REST calls. Let's assume that we are still on the same node where we started Elasticsearch on and open another command shell window.
 
-To check the cluster health, we will be using the <<cat,`_cat` API>>. Remember previously that our node HTTP endpoint is available at port `9200`:
+To check the cluster health, we will be using the {ref}/cat.html[`_cat` API]. You can
+run the command below in {kibana-ref}/console-kibana.html[Kibana's Console]
+by clicking "VIEW IN CONSOLE" or with `curl` by clicking the "COPY AS CURL"
+link below and pasting it into a terminal.
 
-[source,sh]
+[source,js]
 --------------------------------------------------
-curl 'localhost:9200/_cat/health?v'
+GET /_cat/health?v
 --------------------------------------------------
+// CONSOLE
 
 And the response:
 
-[source,sh]
+[source,txt]
 --------------------------------------------------
-epoch      timestamp cluster       status node.total node.data shards pri relo init unassign
-1394735289 14:28:09  elasticsearch green           1         1      0   0    0    0        0
+epoch      timestamp cluster       status node.total node.data shards pri relo init unassign pending_tasks max_task_wait_time active_shards_percent
+1475247709 17:01:49  elasticsearch green           1         1      0   0    0    0        0             0                  -                100.0%
 --------------------------------------------------
+// TESTRESPONSE[s/1475247709 17:01:49  elasticsearch/\\d+ \\d+:\\d+:\\d+ docs_integTestCluster/]
+// TESTRESPONSE[s/0             0                  -/0             \\d+                  -/]
+// TESTRESPONSE[_cat]
 
 We can see that our cluster named "elasticsearch" is up with a green status.
 
 Whenever we ask for the cluster health, we either get green, yellow, or red. Green means everything is good (cluster is fully functional), yellow means all data is available but some replicas are not yet allocated (cluster is fully functional), and red means some data is not available for whatever reason. Note that even if a cluster is red, it still is partially functional (i.e. it will continue to serve search requests from the available shards) but you will likely need to fix it ASAP since you have missing data.
 
-Also from the above response, we can see and total of 1 node and that we have 0 shards since we have no data in it yet. Note that since we are using the default cluster name (elasticsearch) and since Elasticsearch uses unicast network discovery by default to find other nodes on the same machine, it is possible that you could accidentally start up more than one node on your computer and have them all join a single cluster. In this scenario, you may see more than 1 node in the above response.
+Also from the above response, we can see a total of 1 node and that we have 0 shards since we have no data in it yet. Note that since we are using the default cluster name (elasticsearch) and since Elasticsearch uses unicast network discovery by default to find other nodes on the same machine, it is possible that you could accidentally start up more than one node on your computer and have them all join a single cluster. In this scenario, you may see more than 1 node in the above response.
 
 We can also get a list of nodes in our cluster as follows:
 
-[source,sh]
+[source,js]
 --------------------------------------------------
-curl 'localhost:9200/_cat/nodes?v'
+GET /_cat/nodes?v
 --------------------------------------------------
+// CONSOLE
 
 And the response:
 
-[source,sh]
+[source,txt]
 --------------------------------------------------
-curl 'localhost:9200/_cat/nodes?v'
-host         ip        heap.percent ram.percent load node.role master name
-mwubuntu1    127.0.1.1            8           4 0.00 d         *      New Goblin
+ip        heap.percent ram.percent cpu load_1m load_5m load_15m node.role master name
+127.0.0.1           10           5   5    4.46                        mdi      *      PB2SGZY
 --------------------------------------------------
+// TESTRESPONSE[s/10           5   5    4.46/\\d+ \\d+ \\d+ (\\d+\\.\\d+)? (\\d+\\.\\d+)? (\\d+\.\\d+)?/]
+// TESTRESPONSE[s/[*]/[*]/ s/PB2SGZY/.+/ _cat]
 
-Here, we can see our one node named "New Goblin", which is the single node that is currently in our cluster.
+Here, we can see our one node named "PB2SGZY", which is the single node that is currently in our cluster.
 
 === List All Indices
 
 Now let's take a peek at our indices:
 
-[source,sh]
+[source,js]
 --------------------------------------------------
-curl 'localhost:9200/_cat/indices?v'
+GET /_cat/indices?v
 --------------------------------------------------
+// CONSOLE
 
 And the response:
 
-[source,sh]
+[source,txt]
 --------------------------------------------------
-curl 'localhost:9200/_cat/indices?v'
-health index pri rep docs.count docs.deleted store.size pri.store.size
+health status index uuid pri rep docs.count docs.deleted store.size pri.store.size
 --------------------------------------------------
+// TESTRESPONSE[_cat]
 
 Which simply means we have no indices yet in the cluster.
 
@@ -246,28 +346,23 @@ Which simply means we have no indices yet in the cluster.
 
 Now let's create an index named "customer" and then list all the indexes again:
 
-[source,sh]
+[source,js]
 --------------------------------------------------
-curl -XPUT 'localhost:9200/customer?pretty'
-curl 'localhost:9200/_cat/indices?v'
+PUT /customer?pretty
+GET /_cat/indices?v
 --------------------------------------------------
+// CONSOLE
 
 The first command creates the index named "customer" using the PUT verb. We simply append `pretty` to the end of the call to tell it to pretty-print the JSON response (if any).
 
 And the response:
 
-[source,sh]
+[source,txt]
 --------------------------------------------------
-curl -XPUT 'localhost:9200/customer?pretty'
-{
-  "acknowledged" : true,
-  "shards_acknowledged": true
-}
-
-curl 'localhost:9200/_cat/indices?v'
-health index    pri rep docs.count docs.deleted store.size pri.store.size
-yellow customer   5   1          0            0       495b           495b
+health status index    uuid                   pri rep docs.count docs.deleted store.size pri.store.size
+yellow open   customer 95SQ4TSUT7mWBT7VNHH67A   5   1          0            0       260b           260b
 --------------------------------------------------
+// TESTRESPONSE[s/95SQ4TSUT7mWBT7VNHH67A/.+/ s/260b/\\d+b/ _cat]
 
 The results of the second command tells us that we now have 1 index named customer and it has 5 primary shards and 1 replica (the defaults) and it contains 0 documents in it.
 
@@ -279,32 +374,34 @@ Let's now put something into our customer index. Remember previously that in ord
 
 Let's index a simple customer document into the customer index, "external" type, with an ID of 1 as follows:
 
-Our JSON document: { "name": "John Doe" }
-
-[source,sh]
+[source,js]
 --------------------------------------------------
-curl -XPUT 'localhost:9200/customer/external/1?pretty' -d '
+PUT /customer/external/1?pretty
 {
   "name": "John Doe"
-}'
+}
 --------------------------------------------------
+// CONSOLE
 
 And the response:
 
 [source,sh]
 --------------------------------------------------
-curl -XPUT 'localhost:9200/customer/external/1?pretty' -d '
-{
-  "name": "John Doe"
-}'
 {
   "_index" : "customer",
   "_type" : "external",
   "_id" : "1",
   "_version" : 1,
+  "result" : "created",
+  "_shards" : {
+    "total" : 2,
+    "successful" : 1,
+    "failed" : 0
+  },
   "created" : true
 }
 --------------------------------------------------
+// TESTRESPONSE
 
 From the above, we can see that a new customer document was successfully created inside the customer index and the external type. The document also has an internal id of 1 which we specified at index time.
 
@@ -312,16 +409,17 @@ It is important to note that Elasticsearch does not require you to explicitly cr
 
 Let's now retrieve that document that we just indexed:
 
-[source,sh]
+[source,js]
 --------------------------------------------------
-curl -XGET 'localhost:9200/customer/external/1?pretty'
+GET /customer/external/1?pretty
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 And the response:
 
-[source,sh]
+[source,js]
 --------------------------------------------------
-curl -XGET 'localhost:9200/customer/external/1?pretty'
 {
   "_index" : "customer",
   "_type" : "external",
@@ -331,6 +429,7 @@ curl -XGET 'localhost:9200/customer/external/1?pretty'
   "_source" : { "name": "John Doe" }
 }
 --------------------------------------------------
+// TESTRESPONSE
 
 Nothing out of the ordinary here other than a field, `found`, stating that we found a document with the requested ID 1 and another field, `_source`, which returns the full JSON document that we indexed from the previous step.
 
@@ -338,47 +437,47 @@ Nothing out of the ordinary here other than a field, `found`, stating that we fo
 
 Now let's delete the index that we just created and then list all the indexes again:
 
-[source,sh]
+[source,js]
 --------------------------------------------------
-curl -XDELETE 'localhost:9200/customer?pretty'
-curl 'localhost:9200/_cat/indices?v'
+DELETE /customer?pretty
+GET /_cat/indices?v
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 And the response:
 
-[source,sh]
+[source,txt]
 --------------------------------------------------
-curl -XDELETE 'localhost:9200/customer?pretty'
-{
-  "acknowledged" : true
-}
-curl 'localhost:9200/_cat/indices?v'
-health index pri rep docs.count docs.deleted store.size pri.store.size
+health status index uuid pri rep docs.count docs.deleted store.size pri.store.size
 --------------------------------------------------
+// TESTRESPONSE[_cat]
 
 Which means that the index was deleted successfully and we are now back to where we started with nothing in our cluster.
 
 Before we move on, let's take a closer look again at some of the API commands that we have learned so far:
 
-[source,sh]
+[source,js]
 --------------------------------------------------
-curl -XPUT 'localhost:9200/customer'
-curl -XPUT 'localhost:9200/customer/external/1' -d '
+PUT /customer
+PUT /customer/external/1
 {
   "name": "John Doe"
-}'
-curl 'localhost:9200/customer/external/1'
-curl -XDELETE 'localhost:9200/customer'
+}
+GET /customer/external/1
+DELETE /customer
 --------------------------------------------------
+// CONSOLE
 
 If we study the above commands carefully, we can actually see a pattern of how we access data in Elasticsearch. That pattern can be summarized as follows:
 
-[source,sh]
+[source,js]
 --------------------------------------------------
-curl -X<REST Verb> <Node>:<Port>/<Index>/<Type>/<ID>
+<REST Verb> /<Index>/<Type>/<ID>
 --------------------------------------------------
+// NOTCONSOLE
 
-This REST access pattern is pervasive throughout all the API commands that if you can simply remember it, you will have a good head start at mastering Elasticsearch.
+This REST access pattern is so pervasive throughout all the API commands that if you can simply remember it, you will have a good head start at mastering Elasticsearch.
 
 == Modifying Your Data
 
@@ -389,33 +488,38 @@ Elasticsearch provides data manipulation and search capabilities in near real ti
 
 We've previously seen how we can index a single document. Let's recall that command again:
 
-[source,sh]
+[source,js]
 --------------------------------------------------
-curl -XPUT 'localhost:9200/customer/external/1?pretty' -d '
+PUT /customer/external/1?pretty
 {
   "name": "John Doe"
-}'
+}
 --------------------------------------------------
+// CONSOLE
 
 Again, the above will index the specified document into the customer index, external type, with the ID of 1. If we then executed the above command again with a different (or same) document, Elasticsearch will replace (i.e. reindex) a new document on top of the existing one with the ID of 1:
 
-[source,sh]
+[source,js]
 --------------------------------------------------
-curl -XPUT 'localhost:9200/customer/external/1?pretty' -d '
+PUT /customer/external/1?pretty
 {
   "name": "Jane Doe"
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 The above changes the name of the document with the ID of 1 from "John Doe" to "Jane Doe". If, on the other hand, we use a different ID, a new document will be indexed and the existing document(s) already in the index remains untouched.
 
-[source,sh]
+[source,js]
 --------------------------------------------------
-curl -XPUT 'localhost:9200/customer/external/2?pretty' -d '
+PUT /customer/external/2?pretty
 {
   "name": "Jane Doe"
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 The above indexes a new document with an ID of 2.
 
@@ -423,15 +527,17 @@ When indexing, the ID part is optional. If not specified, Elasticsearch will gen
 
 This example shows how to index a document without an explicit ID:
 
-[source,sh]
+[source,js]
 --------------------------------------------------
-curl -XPOST 'localhost:9200/customer/external?pretty' -d '
+POST /customer/external?pretty
 {
   "name": "Jane Doe"
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
-Note that in the above case, we are using the POST verb instead of PUT since we didn't specify an ID.
+Note that in the above case, we are using the `POST` verb instead of PUT since we didn't specify an ID.
 
 === Updating Documents
 
@@ -439,33 +545,39 @@ In addition to being able to index and replace documents, we can also update doc
 
 This example shows how to update our previous document (ID of 1) by changing the name field to "Jane Doe":
 
-[source,sh]
+[source,js]
 --------------------------------------------------
-curl -XPOST 'localhost:9200/customer/external/1/_update?pretty' -d '
+POST /customer/external/1/_update?pretty
 {
   "doc": { "name": "Jane Doe" }
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 This example shows how to update our previous document (ID of 1) by changing the name field to "Jane Doe" and at the same time add an age field to it:
 
-[source,sh]
+[source,js]
 --------------------------------------------------
-curl -XPOST 'localhost:9200/customer/external/1/_update?pretty' -d '
+POST /customer/external/1/_update?pretty
 {
   "doc": { "name": "Jane Doe", "age": 20 }
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
-Updates can also be performed by using simple scripts. Note that dynamic scripts like the following are disabled by default as of `1.4.3`, have a look at the <<modules-scripting,scripting docs>> for more details. This example uses a script to increment the age by 5:
+Updates can also be performed by using simple scripts. This example uses a script to increment the age by 5:
 
-[source,sh]
+[source,js]
 --------------------------------------------------
-curl -XPOST 'localhost:9200/customer/external/1/_update?pretty' -d '
+POST /customer/external/1/_update?pretty
 {
   "script" : "ctx._source.age += 5"
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 In the above example, `ctx._source` refers to the current source document that is about to be updated.
 
@@ -475,43 +587,48 @@ Note that as of this writing, updates can only be performed on a single document
 
 Deleting a document is fairly straightforward. This example shows how to delete our previous customer with the ID of 2:
 
-[source,sh]
+[source,js]
 --------------------------------------------------
-curl -XDELETE 'localhost:9200/customer/external/2?pretty'
+DELETE /customer/external/2?pretty
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
-The `delete-by-query` plugin can delete all documents matching a specific query.
+See the {ref}/docs-delete-by-query.html[`_delete_by_query` API] to delete all documents matching a specific query.
+It is worth noting that it is much more efficient to delete a whole index
+instead of deleting all documents with the Delete By Query API.
 
 === Batch Processing
 
-In addition to being able to index, update, and delete individual documents, Elasticsearch also provides the ability to perform any of the above operations in batches using the <<docs-bulk,`_bulk` API>>. This functionality is important in that it provides a very efficient mechanism to do multiple operations as fast as possible with as little network roundtrips as possible.
+In addition to being able to index, update, and delete individual documents, Elasticsearch also provides the ability to perform any of the above operations in batches using the {ref}/docs-bulk.html[`_bulk` API]. This functionality is important in that it provides a very efficient mechanism to do multiple operations as fast as possible with as few network roundtrips as possible.
 
 As a quick example, the following call indexes two documents (ID 1 - John Doe and ID 2 - Jane Doe) in one bulk operation:
 
-[source,sh]
+[source,js]
 --------------------------------------------------
-curl -XPOST 'localhost:9200/customer/external/_bulk?pretty' -d '
+POST /customer/external/_bulk?pretty
 {"index":{"_id":"1"}}
 {"name": "John Doe" }
 {"index":{"_id":"2"}}
 {"name": "Jane Doe" }
-'
 --------------------------------------------------
+// CONSOLE
 
 This example updates the first document (ID of 1) and then deletes the second document (ID of 2) in one bulk operation:
 
 [source,sh]
 --------------------------------------------------
-curl -XPOST 'localhost:9200/customer/external/_bulk?pretty' -d '
+POST /customer/external/_bulk?pretty
 {"update":{"_id":"1"}}
 {"doc": { "name": "John Doe becomes Jane Doe" } }
 {"delete":{"_id":"2"}}
-'
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 Note above that for the delete action, there is no corresponding source document after it since deletes only require the ID of the document to be deleted.
 
-The bulk API executes all the actions sequentially and in order. If a single action fails for whatever reason, it will continue to process the remainder of the actions after it. When the bulk API returns, it will provide a status for each action (in the same order it was sent in) so that you can check if a specific action failed or not.
+The Bulk API does not fail due to failures in one of the actions. If a single action fails for whatever reason, it will continue to process the remainder of the actions after it. When the bulk API returns, it will provide a status for each action (in the same order it was sent in) so that you can check if a specific action failed or not.
 
 == Exploring Your Data
 
@@ -520,7 +637,7 @@ The bulk API executes all the actions sequentially and in order. If a single act
 
 Now that we've gotten a glimpse of the basics, let's try to work on a more realistic dataset. I've prepared a sample of fictitious JSON documents of customer bank account information. Each document has the following schema:
 
-[source,sh]
+[source,js]
 --------------------------------------------------
 {
     "account_number": 0,
@@ -536,74 +653,98 @@ Now that we've gotten a glimpse of the basics, let's try to work on a more reali
     "state": "CO"
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 For the curious, I generated this data from http://www.json-generator.com/[`www.json-generator.com/`] so please ignore the actual values and semantics of the data as these are all randomly generated.
 
 [float]
 === Loading the Sample Dataset
 
-You can download the sample dataset (accounts.json) from https://github.com/bly2k/files/blob/master/accounts.zip?raw=true[here]. Extract it to our current directory and let's load it into our cluster as follows:
+You can download the sample dataset (accounts.json) from https://github.com/elastic/elasticsearch/blob/master/docs/src/test/resources/accounts.json?raw=true[here]. Extract it to our current directory and let's load it into our cluster as follows:
 
 [source,sh]
 --------------------------------------------------
-curl -XPOST 'localhost:9200/bank/account/_bulk?pretty' --data-binary "@accounts.json"
+curl -H "Content-Type: application/json" -XPOST 'localhost:9200/bank/account/_bulk?pretty&refresh' --data-binary "@accounts.json"
 curl 'localhost:9200/_cat/indices?v'
 --------------------------------------------------
+// NOTCONSOLE
+
+////
+This replicates the above in a document-testing friendly way but isn't visible
+in the docs:
+
+[source,js]
+--------------------------------------------------
+GET /_cat/indices?v
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:bank]
+////
 
 And the response:
 
-[source,sh]
+[source,js]
 --------------------------------------------------
-curl 'localhost:9200/_cat/indices?v'
-health index pri rep docs.count docs.deleted store.size pri.store.size
-yellow bank    5   1       1000            0    424.4kb        424.4kb
+health status index uuid                   pri rep docs.count docs.deleted store.size pri.store.size
+yellow open   bank  l7sSYV2cQXmu6_4rJWVIww   5   1       1000            0    128.6kb        128.6kb
 --------------------------------------------------
+// TESTRESPONSE[s/128.6kb/\\d+(\\.\\d+)?[mk]?b/]
+// TESTRESPONSE[s/l7sSYV2cQXmu6_4rJWVIww/.+/ _cat]
 
 Which means that we just successfully bulk indexed 1000 documents into the bank index (under the account type).
 
 === The Search API
 
-Now let's start with some simple searches. There are two basic ways to run searches: one is by sending search parameters through the <<search-uri-request,REST request URI>> and the other by sending them through the <<search-request-body,REST request body>>. The request body method allows you to be more expressive and also to define your searches in a more readable JSON format. We'll try one example of the request URI method but for the remainder of this tutorial, we will exclusively be using the request body method.
+Now let's start with some simple searches. There are two basic ways to run searches: one is by sending search parameters through the {ref}/search-uri-request.html[REST request URI] and the other by sending them through the {ref}/search-request-body.html[REST request body]. The request body method allows you to be more expressive and also to define your searches in a more readable JSON format. We'll try one example of the request URI method but for the remainder of this tutorial, we will exclusively be using the request body method.
 
 The REST API for search is accessible from the `_search` endpoint. This example returns all documents in the bank index:
 
-[source,sh]
+[source,js]
 --------------------------------------------------
-curl 'localhost:9200/bank/_search?q=*&pretty'
+GET /bank/_search?q=*&sort=account_number:asc&pretty
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
-Let's first dissect the search call. We are searching (`_search` endpoint) in the bank index, and the `q=*` parameter instructs Elasticsearch to match all documents in the index. The `pretty` parameter, again, just tells Elasticsearch to return pretty-printed JSON results.
+Let's first dissect the search call. We are searching (`_search` endpoint) in the bank index, and the `q=*` parameter instructs Elasticsearch to match all documents in the index. The `sort=account_number:asc` parameter indicates to sort the results using the `account_number` field of each document in an ascending order. The `pretty` parameter, again, just tells Elasticsearch to return pretty-printed JSON results.
 
 And the response (partially shown):
 
-[source,sh]
+[source,js]
 --------------------------------------------------
-curl 'localhost:9200/bank/_search?q=*&pretty'
 {
   "took" : 63,
   "timed_out" : false,
   "_shards" : {
     "total" : 5,
     "successful" : 5,
+    "skipped" : 0,
     "failed" : 0
   },
   "hits" : {
     "total" : 1000,
-    "max_score" : 1.0,
+    "max_score" : null,
     "hits" : [ {
       "_index" : "bank",
       "_type" : "account",
-      "_id" : "1",
-      "_score" : 1.0, "_source" : {"account_number":1,"balance":39225,"firstname":"Amber","lastname":"Duke","age":32,"gender":"M","address":"880 Holmes Lane","employer":"Pyrami","email":"amberduke@pyrami.com","city":"Brogan","state":"IL"}
-    }, {
-      "_index" : "bank",
-      "_type" : "account",
-      "_id" : "6",
-      "_score" : 1.0, "_source" : {"account_number":6,"balance":5686,"firstname":"Hattie","lastname":"Bond","age":36,"gender":"M","address":"671 Bristol Street","employer":"Netagy","email":"hattiebond@netagy.com","city":"Dante","state":"TN"}
+      "_id" : "0",
+      "sort": [0],
+      "_score" : null,
+      "_source" : {"account_number":0,"balance":16623,"firstname":"Bradshaw","lastname":"Mckenzie","age":29,"gender":"F","address":"244 Columbus Place","employer":"Euron","email":"bradshawmckenzie@euron.com","city":"Hobucken","state":"CO"}
     }, {
       "_index" : "bank",
       "_type" : "account",
+      "_id" : "1",
+      "sort": [1],
+      "_score" : null,
+      "_source" : {"account_number":1,"balance":39225,"firstname":"Amber","lastname":"Duke","age":32,"gender":"M","address":"880 Holmes Lane","employer":"Pyrami","email":"amberduke@pyrami.com","city":"Brogan","state":"IL"}
+    }, ...
+    ]
+  }
+}
 --------------------------------------------------
+// TESTRESPONSE[s/"took" : 63/"took" : $body.took/]
+// TESTRESPONSE[s/\.\.\./$body.hits.hits.2, $body.hits.hits.3, $body.hits.hits.4, $body.hits.hits.5, $body.hits.hits.6, $body.hits.hits.7, $body.hits.hits.8, $body.hits.hits.9/]
 
 As for the response, we see the following parts:
 
@@ -613,109 +754,133 @@ As for the response, we see the following parts:
 * `hits` – search results
 * `hits.total` – total number of documents matching our search criteria
 * `hits.hits` – actual array of search results (defaults to first 10 documents)
-* `_score` and `max_score` - ignore these fields for now
+* `hits.sort` - sort key for results (missing if sorting by score)
+* `hits._score` and `max_score` - ignore these fields for now
 
 Here is the same exact search above using the alternative request body method:
 
-[source,sh]
+[source,js]
 --------------------------------------------------
-curl -XPOST 'localhost:9200/bank/_search?pretty' -d '
+GET /bank/_search
 {
-  "query": { "match_all": {} }
-}'
+  "query": { "match_all": {} },
+  "sort": [
+    { "account_number": "asc" }
+  ]
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 The difference here is that instead of passing `q=*` in the URI, we POST a JSON-style query request body to the `_search` API. We'll discuss this JSON query in the next section.
 
-And the response (partially shown):
+////
+Hidden response just so we can assert that it is indeed the same but don't have
+to clutter the docs with it:
 
-[source,sh]
+[source,js]
 --------------------------------------------------
-curl -XPOST 'localhost:9200/bank/_search?pretty' -d '
-{
-  "query": { "match_all": {} }
-}'
 {
-  "took" : 26,
+  "took" : 63,
   "timed_out" : false,
   "_shards" : {
     "total" : 5,
     "successful" : 5,
+    "skipped" : 0,
     "failed" : 0
   },
   "hits" : {
     "total" : 1000,
-    "max_score" : 1.0,
+    "max_score": null,
     "hits" : [ {
       "_index" : "bank",
       "_type" : "account",
-      "_id" : "1",
-      "_score" : 1.0, "_source" : {"account_number":1,"balance":39225,"firstname":"Amber","lastname":"Duke","age":32,"gender":"M","address":"880 Holmes Lane","employer":"Pyrami","email":"amberduke@pyrami.com","city":"Brogan","state":"IL"}
-    }, {
-      "_index" : "bank",
-      "_type" : "account",
-      "_id" : "6",
-      "_score" : 1.0, "_source" : {"account_number":6,"balance":5686,"firstname":"Hattie","lastname":"Bond","age":36,"gender":"M","address":"671 Bristol Street","employer":"Netagy","email":"hattiebond@netagy.com","city":"Dante","state":"TN"}
+      "_id" : "0",
+      "sort": [0],
+      "_score": null,
+      "_source" : {"account_number":0,"balance":16623,"firstname":"Bradshaw","lastname":"Mckenzie","age":29,"gender":"F","address":"244 Columbus Place","employer":"Euron","email":"bradshawmckenzie@euron.com","city":"Hobucken","state":"CO"}
     }, {
       "_index" : "bank",
       "_type" : "account",
-      "_id" : "13",
+      "_id" : "1",
+      "sort": [1],
+      "_score": null,
+      "_source" : {"account_number":1,"balance":39225,"firstname":"Amber","lastname":"Duke","age":32,"gender":"M","address":"880 Holmes Lane","employer":"Pyrami","email":"amberduke@pyrami.com","city":"Brogan","state":"IL"}
+    }, ...
+    ]
+  }
+}
 --------------------------------------------------
+// TESTRESPONSE[s/"took" : 63/"took" : $body.took/]
+// TESTRESPONSE[s/\.\.\./$body.hits.hits.2, $body.hits.hits.3, $body.hits.hits.4, $body.hits.hits.5, $body.hits.hits.6, $body.hits.hits.7, $body.hits.hits.8, $body.hits.hits.9/]
+
+////
 
 It is important to understand that once you get your search results back, Elasticsearch is completely done with the request and does not maintain any kind of server-side resources or open cursors into your results. This is in stark contrast to many other platforms such as SQL wherein you may initially get a partial subset of your query results up-front and then you have to continuously go back to the server if you want to fetch (or page through) the rest of the results using some kind of stateful server-side cursor.
 
 === Introducing the Query Language
 
-Elasticsearch provides a JSON-style domain-specific language that you can use to execute queries. This is referred to as the <<query-dsl,Query DSL>>. The query language is quite comprehensive and can be intimidating at first glance but the best way to actually learn it is to start with a few basic examples.
+Elasticsearch provides a JSON-style domain-specific language that you can use to execute queries. This is referred to as the {ref}/query-dsl.html[Query DSL]. The query language is quite comprehensive and can be intimidating at first glance but the best way to actually learn it is to start with a few basic examples.
 
 Going back to our last example, we executed this query:
 
-[source,sh]
+[source,js]
 --------------------------------------------------
+GET /bank/_search
 {
   "query": { "match_all": {} }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 Dissecting the above, the `query` part tells us what our query definition is and the `match_all` part is simply the type of query that we want to run. The `match_all` query is simply a search for all documents in the specified index.
 
-In addition to the `query` parameter, we also can pass other parameters to influence the search results. For example, the following does a `match_all` and returns only the first document:
+In addition to the `query` parameter, we also can pass other parameters to
+influence the search results. In the example in the section above we passed in
+`sort`, here we pass in `size`:
 
-[source,sh]
+[source,js]
 --------------------------------------------------
-curl -XPOST 'localhost:9200/bank/_search?pretty' -d '
+GET /bank/_search
 {
   "query": { "match_all": {} },
   "size": 1
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 Note that if `size` is not specified, it defaults to 10.
 
 This example does a `match_all` and returns documents 11 through 20:
 
-[source,sh]
+[source,js]
 --------------------------------------------------
-curl -XPOST 'localhost:9200/bank/_search?pretty' -d '
+GET /bank/_search
 {
   "query": { "match_all": {} },
   "from": 10,
   "size": 10
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 The `from` parameter (0-based) specifies which document index to start from and the `size` parameter specifies how many documents to return starting at the from parameter. This feature is useful when implementing paging of search results. Note that if `from` is not specified, it defaults to 0.
 
 This example does a `match_all` and sorts the results by account balance in descending order and returns the top 10 (default size) documents.
 
-[source,sh]
+[source,js]
 --------------------------------------------------
-curl -XPOST 'localhost:9200/bank/_search?pretty' -d '
+GET /bank/_search
 {
   "query": { "match_all": {} },
   "sort": { "balance": { "order": "desc" } }
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 === Executing Searches
 
@@ -723,68 +888,78 @@ Now that we have seen a few of the basic search parameters, let's dig in some mo
 
 This example shows how to return two fields, `account_number` and `balance` (inside of `_source`), from the search:
 
-[source,sh]
+[source,js]
 --------------------------------------------------
-curl -XPOST 'localhost:9200/bank/_search?pretty' -d '
+GET /bank/_search
 {
   "query": { "match_all": {} },
   "_source": ["account_number", "balance"]
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 Note that the above example simply reduces the `_source` field. It will still only return one field named `_source` but within it, only the fields `account_number` and `balance` are included.
 
 If you come from a SQL background, the above is somewhat similar in concept to the `SQL SELECT FROM` field list.
 
-Now let's move on to the query part. Previously, we've seen how the `match_all` query is used to match all documents. Let's now introduce a new query called the <<query-dsl-match-query,`match` query>>, which can be thought of as a basic fielded search query (i.e. a search done against a specific field or set of fields).
+Now let's move on to the query part. Previously, we've seen how the `match_all` query is used to match all documents. Let's now introduce a new query called the {ref}/query-dsl-match-query.html[`match` query], which can be thought of as a basic fielded search query (i.e. a search done against a specific field or set of fields).
 
 This example returns the account numbered 20:
 
-[source,sh]
+[source,js]
 --------------------------------------------------
-curl -XPOST 'localhost:9200/bank/_search?pretty' -d '
+GET /bank/_search
 {
   "query": { "match": { "account_number": 20 } }
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 This example returns all accounts containing the term "mill" in the address:
 
-[source,sh]
+[source,js]
 --------------------------------------------------
-curl -XPOST 'localhost:9200/bank/_search?pretty' -d '
+GET /bank/_search
 {
   "query": { "match": { "address": "mill" } }
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 This example returns all accounts containing the term "mill" or "lane" in the address:
 
-[source,sh]
+[source,js]
 --------------------------------------------------
-curl -XPOST 'localhost:9200/bank/_search?pretty' -d '
+GET /bank/_search
 {
   "query": { "match": { "address": "mill lane" } }
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 This example is a variant of `match` (`match_phrase`) that returns all accounts containing the phrase "mill lane" in the address:
 
-[source,sh]
+[source,js]
 --------------------------------------------------
-curl -XPOST 'localhost:9200/bank/_search?pretty' -d '
+GET /bank/_search
 {
   "query": { "match_phrase": { "address": "mill lane" } }
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
-Let's now introduce the <<query-dsl-bool-query,`bool`(ean) query>>. The `bool` query allows us to compose smaller queries into bigger queries using boolean logic.
+Let's now introduce the {ref}/query-dsl-bool-query.html[`bool` query]. The `bool` query allows us to compose smaller queries into bigger queries using boolean logic.
 
 This example composes two `match` queries and returns all accounts containing "mill" and "lane" in the address:
 
-[source,sh]
+[source,js]
 --------------------------------------------------
-curl -XPOST 'localhost:9200/bank/_search?pretty' -d '
+GET /bank/_search
 {
   "query": {
     "bool": {
@@ -794,16 +969,18 @@ curl -XPOST 'localhost:9200/bank/_search?pretty' -d '
       ]
     }
   }
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 In the above example, the `bool must` clause specifies all the queries that must be true for a document to be considered a match.
 
 In contrast, this example composes two `match` queries and returns all accounts containing "mill" or "lane" in the address:
 
-[source,sh]
+[source,js]
 --------------------------------------------------
-curl -XPOST 'localhost:9200/bank/_search?pretty' -d '
+GET /bank/_search
 {
   "query": {
     "bool": {
@@ -813,16 +990,18 @@ curl -XPOST 'localhost:9200/bank/_search?pretty' -d '
       ]
     }
   }
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 In the above example, the `bool should` clause specifies a list of queries either of which must be true for a document to be considered a match.
 
 This example composes two `match` queries and returns all accounts that contain neither "mill" nor "lane" in the address:
 
-[source,sh]
+[source,js]
 --------------------------------------------------
-curl -XPOST 'localhost:9200/bank/_search?pretty' -d '
+GET /bank/_search
 {
   "query": {
     "bool": {
@@ -832,18 +1011,20 @@ curl -XPOST 'localhost:9200/bank/_search?pretty' -d '
       ]
     }
   }
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 In the above example, the `bool must_not` clause specifies a list of queries none of which must be true for a document to be considered a match.
 
 We can combine `must`, `should`, and `must_not` clauses simultaneously inside a `bool` query. Furthermore, we can compose `bool` queries inside any of these `bool` clauses to mimic any complex multi-level boolean logic.
 
-This example returns all accounts of anybody who is 40 years old but don't live in ID(aho):
+This example returns all accounts of anybody who is 40 years old but doesn't live in ID(aho):
 
-[source,sh]
+[source,js]
 --------------------------------------------------
-curl -XPOST 'localhost:9200/bank/_search?pretty' -d '
+GET /bank/_search
 {
   "query": {
     "bool": {
@@ -855,8 +1036,10 @@ curl -XPOST 'localhost:9200/bank/_search?pretty' -d '
       ]
     }
   }
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 === Executing Filters
 
@@ -864,13 +1047,13 @@ In the previous section, we skipped over a little detail called the document sco
 
 But queries do not always need to produce scores, in particular when they are only used for "filtering" the document set. Elasticsearch detects these situations and automatically optimizes query execution in order not to compute useless scores.
 
-The <<query-dsl-bool-query,`bool` query>> that we introduced in the previous section also supports `filter` clauses which allow to use a query to restrict the documents that will be matched by other clauses, without changing how scores are computed. As an example, let's introduce the <<query-dsl-range-query,`range` query>>, which allows us to filter documents by a range of values. This is generally used for numeric or date filtering.
+The {ref}/query-dsl-bool-query.html[`bool` query] that we introduced in the previous section also supports `filter` clauses which allow to use a query to restrict the documents that will be matched by other clauses, without changing how scores are computed. As an example, let's introduce the {ref}/query-dsl-range-query.html[`range` query], which allows us to filter documents by a range of values. This is generally used for numeric or date filtering.
 
 This example uses a bool query to return all accounts with balances between 20000 and 30000, inclusive. In other words, we want to find accounts with a balance that is greater than or equal to 20000 and less than or equal to 30000.
 
-[source,sh]
+[source,js]
 --------------------------------------------------
-curl -XPOST 'localhost:9200/bank/_search?pretty' -d '
+GET /bank/_search
 {
   "query": {
     "bool": {
@@ -885,8 +1068,10 @@ curl -XPOST 'localhost:9200/bank/_search?pretty' -d '
       }
     }
   }
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 Dissecting the above, the bool query contains a `match_all` query (the query part) and a `range` query (the filter part). We can substitute any other queries into the query and the filter parts. In the above case, the range query makes perfect sense since documents falling into the range all match "equally", i.e., no document is more relevant than another.
 
@@ -898,9 +1083,9 @@ Aggregations provide the ability to group and extract statistics from your data.
 
 To start with, this example groups all the accounts by state, and then returns the top 10 (default) states sorted by count descending (also default):
 
-[source,sh]
+[source,js]
 --------------------------------------------------
-curl -XPOST 'localhost:9200/bank/_search?pretty' -d '
+GET /bank/_search
 {
   "size": 0,
   "aggs": {
@@ -910,8 +1095,10 @@ curl -XPOST 'localhost:9200/bank/_search?pretty' -d '
       }
     }
   }
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 In SQL, the above aggregation is similar in concept to:
 
@@ -922,8 +1109,17 @@ SELECT state, COUNT(*) FROM bank GROUP BY state ORDER BY COUNT(*) DESC
 
 And the response (partially shown):
 
-[source,sh]
+[source,js]
 --------------------------------------------------
+{
+  "took": 29,
+  "timed_out": false,
+  "_shards": {
+    "total": 5,
+    "successful": 5,
+    "skipped" : 0,
+    "failed": 0
+  },
   "hits" : {
     "total" : 1000,
     "max_score" : 0.0,
@@ -931,51 +1127,55 @@ And the response (partially shown):
   },
   "aggregations" : {
     "group_by_state" : {
+      "doc_count_error_upper_bound": 20,
+      "sum_other_doc_count": 770,
       "buckets" : [ {
-        "key" : "al",
-        "doc_count" : 21
+        "key" : "ID",
+        "doc_count" : 27
       }, {
-        "key" : "tx",
-        "doc_count" : 17
+        "key" : "TX",
+        "doc_count" : 27
       }, {
-        "key" : "id",
-        "doc_count" : 15
+        "key" : "AL",
+        "doc_count" : 25
       }, {
-        "key" : "ma",
-        "doc_count" : 15
+        "key" : "MD",
+        "doc_count" : 25
       }, {
-        "key" : "md",
-        "doc_count" : 15
+        "key" : "TN",
+        "doc_count" : 23
       }, {
-        "key" : "pa",
-        "doc_count" : 15
+        "key" : "MA",
+        "doc_count" : 21
       }, {
-        "key" : "dc",
-        "doc_count" : 14
+        "key" : "NC",
+        "doc_count" : 21
       }, {
-        "key" : "me",
-        "doc_count" : 14
+        "key" : "ND",
+        "doc_count" : 21
       }, {
-        "key" : "mo",
-        "doc_count" : 14
+        "key" : "ME",
+        "doc_count" : 20
       }, {
-        "key" : "nd",
-        "doc_count" : 14
+        "key" : "MO",
+        "doc_count" : 20
       } ]
     }
   }
 }
 --------------------------------------------------
+// TESTRESPONSE[s/"took": 29/"took": $body.took/]
 
-We can see that there are 21 accounts in AL(abama), followed by 17 accounts in TX, followed by 15 accounts in ID(aho), and so forth.
+We can see that there are 27 accounts in `ID` (Idaho), followed by 27 accounts
+in `TX` (Texas), followed by 25 accounts in `AL` (Alabama), and so forth.
 
 Note that we set `size=0` to not show search hits because we only want to see the aggregation results in the response.
 
 Building on the previous aggregation, this example calculates the average account balance by state (again only for the top 10 states sorted by count in descending order):
 
-[source,sh]
+[source,js]
 --------------------------------------------------
-curl -XPOST 'localhost:9200/bank/_search?pretty' -d '
+GET /bank/_search
 {
   "size": 0,
   "aggs": {
@@ -992,16 +1192,18 @@ curl -XPOST 'localhost:9200/bank/_search?pretty' -d '
       }
     }
   }
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 Notice how we nested the `average_balance` aggregation inside the `group_by_state` aggregation. This is a common pattern for all the aggregations. You can nest aggregations inside aggregations arbitrarily to extract pivoted summarizations that you require from your data.
 
 Building on the previous aggregation, let's now sort on the average balance in descending order:
 
-[source,sh]
+[source,js]
 --------------------------------------------------
-curl -XPOST 'localhost:9200/bank/_search?pretty' -d '
+GET /bank/_search
 {
   "size": 0,
   "aggs": {
@@ -1021,14 +1223,16 @@ curl -XPOST 'localhost:9200/bank/_search?pretty' -d '
       }
     }
   }
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 This example demonstrates how we can group by age brackets (ages 20-29, 30-39, and 40-49), then by gender, and then finally get the average account balance, per age bracket, per gender:
 
-[source,sh]
+[source,js]
 --------------------------------------------------
-curl -XPOST 'localhost:9200/bank/_search?pretty' -d '
+GET /bank/_search
 {
   "size": 0,
   "aggs": {
@@ -1053,7 +1257,7 @@ curl -XPOST 'localhost:9200/bank/_search?pretty' -d '
       "aggs": {
         "group_by_gender": {
           "terms": {
-            "field": "gender"
+            "field": "gender.keyword"
           },
           "aggs": {
             "average_balance": {
@@ -1066,10 +1270,12 @@ curl -XPOST 'localhost:9200/bank/_search?pretty' -d '
       }
     }
   }
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
-There are a many other aggregations capabilities that we won't go into detail here. The <<search-aggregations,aggregations reference guide>> is a great starting point if you want to do further experimentation.
+There are many other aggregations capabilities that we won't go into detail here. The {ref}/search-aggregations.html[aggregations reference guide] is a great starting point if you want to do further experimentation.
 
 == Conclusion
 
diff --git a/docs/reference/glossary.asciidoc b/docs/reference/glossary.asciidoc
index edf84f3b4c837..3559cb8985b2b 100644
--- a/docs/reference/glossary.asciidoc
+++ b/docs/reference/glossary.asciidoc
@@ -45,7 +45,7 @@
 [[glossary-id]] id ::
 
   The ID of a <<glossary-document,document>> identifies a document. The
-  `index/type/id` of a document must be unique. If no ID is provided,
+  `index/id` of a document must be unique. If no ID is provided,
   then it will be auto-generated. (also see <<glossary-routing,routing>>)
 
 [[glossary-field]] field ::
@@ -64,8 +64,8 @@
 [[glossary-index]] index ::
 
   An index is like a _table_ in a relational database. It has a
-  <<glossary-mapping,mapping>> which defines the <<glossary-field,fields>> in the index,
-  which are grouped by multiple <<glossary-type,type>>.
+  <<glossary-mapping,mapping>> which contains a <<glossary-type,type>>,
+  which contains the <<glossary-field,fields>> in the index.
   +
   An index is a logical namespace which maps to one or more
   <<glossary-primary-shard,primary shards>> and can have zero or more
@@ -74,8 +74,8 @@
 [[glossary-mapping]] mapping ::
 
   A mapping is like a _schema definition_ in a relational database. Each
-  <<glossary-index,index>> has a mapping, which defines each <<glossary-type,type>>
-  within the index, plus a number of index-wide settings.
+  <<glossary-index,index>> has a mapping, which defines a <<glossary-type,type>>,
+  plus a number of index-wide settings.
   +
   A mapping can either be defined explicitly, or it will be generated
   automatically when a document is indexed.
@@ -179,11 +179,6 @@
 
 [[glossary-type]] type ::
 
-  A type represents the _type_ of document, e.g. an `email`, a `user`, or a `tweet`.
-  The search API can filter documents by type.
-  An <<glossary-index,index>> can contain multiple types, and each type has a
-  list of <<glossary-field,fields>> that can be specified for
-  <<glossary-document,documents>> of that type.  Fields with the same
-  name in different types in the same index must have the same <<glossary-mapping,mapping>>
-  (which defines how each field in the document is indexed and made searchable).
+  A type used to represent the _type_ of document, e.g. an `email`, a `user`, or a `tweet`.
+  Types are deprecated and are in the process of being removed.  See <<removal-of-types>>.
 
diff --git a/docs/reference/gs-index.asciidoc b/docs/reference/gs-index.asciidoc
new file mode 100644
index 0000000000000..30beb16b0dfb5
--- /dev/null
+++ b/docs/reference/gs-index.asciidoc
@@ -0,0 +1,6 @@
+[[elasticsearch-reference]]
+= Elasticsearchリファレンス
+
+include::../Versions.asciidoc[]
+
+include::getting-started.asciidoc[]
diff --git a/docs/reference/how-to.asciidoc b/docs/reference/how-to.asciidoc
index f41c3a3bb9ca9..accdb8e8915ed 100644
--- a/docs/reference/how-to.asciidoc
+++ b/docs/reference/how-to.asciidoc
@@ -4,7 +4,7 @@
 [partintro]
 --
 Elasticsearch ships with defaults which are intended to give a good out of
-the box experience. Full text search, highlighting, aggregations, indexing
+the box experience. Full text search, highlighting, aggregations, and indexing
 should all just work without the user having to change anything.
 
 Once you better understand how you want to use Elasticsearch, however,
@@ -17,6 +17,8 @@ made.
 
 include::how-to/general.asciidoc[]
 
+include::how-to/recipes.asciidoc[]
+
 include::how-to/indexing-speed.asciidoc[]
 
 include::how-to/search-speed.asciidoc[]
diff --git a/docs/reference/how-to/general.asciidoc b/docs/reference/how-to/general.asciidoc
index 60f0181b2bb3c..0900c49ce0640 100644
--- a/docs/reference/how-to/general.asciidoc
+++ b/docs/reference/how-to/general.asciidoc
@@ -11,6 +11,35 @@ for workloads that fall into the database domain, such as retrieving all
 documents that match a particular query. If you need to do this, make sure to
 use the <<search-request-scroll,Scroll>> API.
 
+[float]
+[[maximum-document-size]]
+=== Avoid large documents
+
+Given that the default <<modules-http,`http.max_context_length`>> is set to
+100MB, Elasticsearch will refuse to index any document that is larger than
+that. You might decide to increase that particular setting, but Lucene still
+has a limit of about 2GB.
+
+Even without considering hard limits, large documents are usually not
+practical. Large documents put more stress on network, memory usage and disk,
+even for search requests that do not request the `_source` since Elasticsearch
+needs to fetch the `_id` of the document in all cases, and the cost of getting
+this field is bigger for large documents due to how the filesystem cache works.
+Indexing this document can use an amount of memory that is a multiplier of the
+original size of the document. Proximity search (phrase queries for instance)
+and <<search-request-highlighting,highlighting>> also become more expensive
+since their cost directly depends on the size of the original document.
+
+It is sometimes useful to reconsider what the unit of information should be.
+For instance, the fact you want to make books searchable doesn't necesarily
+mean that a document should consist of a whole book. It might be a better idea
+to use chapters or even paragraphs as documents, and then have a property in
+these documents that identifies which book they belong to. This does not only
+avoid the issues with large documents, it also makes the search experience
+better. For instance if a user searches for two words `foo` and `bar`, a match
+across different chapters is probably very poor, while a match within the same
+paragraph is likely good.
+
 [float]
 [[sparsity]]
 === Avoid sparsity
diff --git a/docs/reference/how-to/indexing-speed.asciidoc b/docs/reference/how-to/indexing-speed.asciidoc
index 50187af5b28a1..668857ed41e4d 100644
--- a/docs/reference/how-to/indexing-speed.asciidoc
+++ b/docs/reference/how-to/indexing-speed.asciidoc
@@ -5,7 +5,7 @@
 === Use bulk requests
 
 Bulk requests will yield much better performance than single-document index
-requests. In order to know the optimal size of a bulk request, you shoud run
+requests. In order to know the optimal size of a bulk request, you should run
 a benchmark on a single node with a single shard. First try to index 100
 documents at once, then 200, then 400, etc. doubling the number of documents
 in a bulk request in every benchmark run. When the indexing speed starts to
@@ -17,29 +17,29 @@ it is advisable to avoid going beyond a couple tens of megabytes per request
 even if larger requests seem to perform better.
 
 [float]
-=== Use multiple workers/threads to send data to elasticsearch
+=== Use multiple workers/threads to send data to Elasticsearch
 
 A single thread sending bulk requests is unlikely to be able to max out the
-indexing capacity of an elasticsearch cluster. In order to use all resources
+indexing capacity of an Elasticsearch cluster. In order to use all resources
 of the cluster, you should send data from multiple threads or processes. In
 addition to making better use of the resources of the cluster, this should
 help reduce the cost of each fsync.
 
 Make sure to watch for `TOO_MANY_REQUESTS (429)` response codes
 (`EsRejectedExecutionException` with the Java client), which is the way that
-elasticsearch tells you that it cannot keep up with the current indexing rate.
+Elasticsearch tells you that it cannot keep up with the current indexing rate.
 When it happens, you should pause indexing a bit before trying again, ideally
 with randomized exponential backoff.
 
 Similarly to sizing bulk requests, only testing can tell what the optimal
-number of workers is. This can be tested by progressivily increasing the
+number of workers is. This can be tested by progressively increasing the
 number of workers until either I/O or CPU is saturated on the cluster.
 
 [float]
 === Increase the refresh interval
 
 The default <<dynamic-index-settings,`index.refresh_interval`>> is `1s`, which
-forces elasticsearch to create a new segment every second.
+forces Elasticsearch to create a new segment every second.
 Increasing this value (to say, `30s`) will allow larger segments to flush and
 decreases future merge pressure.
 
@@ -58,7 +58,7 @@ original values.
 === Disable swapping
 
 You should make sure that the operating system is not swapping out the java
-process by <<setup-configuration-memory,disabling swappping>>.
+process by <<setup-configuration-memory,disabling swapping>>.
 
 [float]
 === Give memory to the filesystem cache
@@ -113,3 +113,9 @@ indexing.
 The default is `10%` which is often plenty: for example, if you give the JVM
 10GB of memory, it will give 1GB to the index buffer, which is enough to host
 two shards that are heavily indexing.
+
+[float]
+=== Additional optimizations
+
+Many of the strategies outlined in <<tune-for-disk-usage>> also
+provide an improvement in the speed of indexing.
diff --git a/docs/reference/how-to/recipes.asciidoc b/docs/reference/how-to/recipes.asciidoc
new file mode 100644
index 0000000000000..d12f8a080f270
--- /dev/null
+++ b/docs/reference/how-to/recipes.asciidoc
@@ -0,0 +1,307 @@
+[[recipes]]
+== Recipes
+
+[float]
+[[mixing-exact-search-with-stemming]]
+=== Mixing exact search with stemming
+
+When building a search application, stemming is often a must as it is desirable
+for a query on `skiing` to match documents that contain `ski` or `skis`. But
+what if a user wants to search for `skiing` specifically? The typical way to do
+this would be to use a <<multi-fields,multi-field>> in order to have the same
+content indexed in two different ways:
+
+[source,js]
+--------------------------------------------------
+PUT index
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "english_exact": {
+          "tokenizer": "standard",
+          "filter": [
+            "lowercase"
+          ]
+        }
+      }
+    }
+  },
+  "mappings": {
+    "type": {
+      "properties": {
+        "body": {
+          "type": "text",
+          "analyzer": "english",
+          "fields": {
+            "exact": {
+              "type": "text",
+              "analyzer": "english_exact"
+            }
+          }
+        }
+      }
+    }
+  }
+}
+
+PUT index/type/1
+{
+  "body": "Ski resort"
+}
+
+PUT index/type/2
+{
+  "body": "A pair of skis"
+}
+
+POST index/_refresh
+--------------------------------------------------
+// CONSOLE
+
+With such a setup, searching for `ski` on `body` would return both documents:
+
+[source,js]
+--------------------------------------------------
+GET index/_search
+{
+  "query": {
+    "simple_query_string": {
+      "fields": [ "body" ],
+      "query": "ski"
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+[source,js]
+--------------------------------------------------
+{
+  "took": 2,
+  "timed_out": false,
+  "_shards": {
+    "total": 5,
+    "successful": 5,
+    "skipped" : 0,
+    "failed": 0
+  },
+  "hits": {
+    "total": 2,
+    "max_score": 0.25811607,
+    "hits": [
+      {
+        "_index": "index",
+        "_type": "type",
+        "_id": "2",
+        "_score": 0.25811607,
+        "_source": {
+          "body": "A pair of skis"
+        }
+      },
+      {
+        "_index": "index",
+        "_type": "type",
+        "_id": "1",
+        "_score": 0.25811607,
+        "_source": {
+          "body": "Ski resort"
+        }
+      }
+    ]
+  }
+}
+--------------------------------------------------
+// TESTRESPONSE[s/"took": 2,/"took": "$body.took",/]
+
+On the other hand, searching for `ski` on `body.exact` would only return
+document `1` since the analysis chain of `body.exact` does not perform
+stemming.
+
+[source,js]
+--------------------------------------------------
+GET index/_search
+{
+  "query": {
+    "simple_query_string": {
+      "fields": [ "body.exact" ],
+      "query": "ski"
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+[source,js]
+--------------------------------------------------
+{
+  "took": 1,
+  "timed_out": false,
+  "_shards": {
+    "total": 5,
+    "successful": 5,
+    "skipped" : 0,
+    "failed": 0
+  },
+  "hits": {
+    "total": 1,
+    "max_score": 0.25811607,
+    "hits": [
+      {
+        "_index": "index",
+        "_type": "type",
+        "_id": "1",
+        "_score": 0.25811607,
+        "_source": {
+          "body": "Ski resort"
+        }
+      }
+    ]
+  }
+}
+--------------------------------------------------
+// TESTRESPONSE[s/"took": 1,/"took": "$body.took",/]
+
+This is not something that is easy to expose to end users, as we would need to
+have a way to figure out whether they are looking for an exact match or not and
+redirect to the appropriate field accordingly. Also what to do if only parts of
+the query need to be matched exactly while other parts should still take
+stemming into account?
+
+Fortunately, the `query_string` and `simple_query_string` queries have a feature
+that solve this exact problem: `quote_field_suffix`. This tell Elasticsearch
+that the words that appear in between quotes are to be redirected to a different
+field, see below:
+
+[source,js]
+--------------------------------------------------
+GET index/_search
+{
+  "query": {
+    "simple_query_string": {
+      "fields": [ "body" ],
+      "quote_field_suffix": ".exact",
+      "query": "\"ski\""
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+[source,js]
+--------------------------------------------------
+{
+  "took": 2,
+  "timed_out": false,
+  "_shards": {
+    "total": 5,
+    "successful": 5,
+    "skipped" : 0,
+    "failed": 0
+  },
+  "hits": {
+    "total": 1,
+    "max_score": 0.25811607,
+    "hits": [
+      {
+        "_index": "index",
+        "_type": "type",
+        "_id": "1",
+        "_score": 0.25811607,
+        "_source": {
+          "body": "Ski resort"
+        }
+      }
+    ]
+  }
+}
+--------------------------------------------------
+// TESTRESPONSE[s/"took": 2,/"took": "$body.took",/]
+
+In the above case, since `ski` was in-between quotes, it was searched on the
+`body.exact` field due to the `quote_field_suffix` parameter, so only document
+`1` matched. This allows users to mix exact search with stemmed search as they
+like.
+
+[float]
+[[consistent-scoring]]
+=== Getting consistent scoring
+
+The fact that Elasticsearch operates with shards and replicas adds challenges
+when it comes to having good scoring.
+
+[float]
+==== Scores are not reproducible
+
+Say the same user runs the same request twice in a row and documents do not come
+back in the same order both times, this is a pretty bad experience isn't it?
+Unfortunately this is something that can happen if you have replicas
+(`index.number_of_replicas` is greater than 0). The reason is that Elasticsearch
+selects the shards that the query should go to in a round-robin fashion, so it
+is quite likely if you run the same query twice in a row that it will go to
+different copies of the same shard.
+
+Now why is it a problem? Index statistics are an important part of the score.
+And these index statistics may be different across copies of the same shard
+due to deleted documents. As you may know when documents are deleted or updated,
+the old document is not immediately removed from the index, it is just marked
+as deleted and it will only be removed from disk on the next time that the
+segment this old document belongs to is merged. However for practical reasons,
+those deleted documents are taken into account for index statistics. So imagine
+that the primary shard just finished a large merge that removed lots of deleted
+documents, then it might have index statistics that are sufficiently different
+from the replica (which still have plenty of deleted documents) so that scores
+are different too.
+
+The recommended way to work around this issue is to use a string that identifies
+the user that is logged is (a user id or session id for instance) as a
+<<search-request-preference,preference>>. This ensures that all queries of a
+given user are always going to hit the same shards, so scores remain more
+consistent across queries.
+
+This work around has another benefit: when two documents have the same score,
+they will be sorted by their internal Lucene doc id (which is unrelated to the
+`_id` or `_uid`) by default. However these doc ids could be different across
+copies of the same shard. So by always hitting the same shard, we would get
+more consistent ordering of documents that have the same scores.
+
+[float]
+==== Relevancy looks wrong
+
+If you notice that two documents with the same content get different scores or
+that an exact match is not ranked first, then the issue might be related to
+sharding. By default, Elasticsearch makes each shard responsible for producing
+its own scores. However since index statistics are an important contributor to
+the scores, this only works well if shards have similar index statistics. The
+assumption is that since documents are routed evenly to shards by default, then
+index statistics should be very similar and scoring would work as expected.
+However in the event that you either
+ - use routing at index time,
+ - query multiple _indices_,
+ - or have too little data in your index
+then there are good chances that all shards that are involved in the search
+request do not have similar index statistics and relevancy could be bad.
+
+If you have a small dataset, the easiest way to work around this issue is to
+index everything into an index that has a single shard
+(`index.number_of_shards: 1`). Then index statistics will be the same for all
+documents and scores will be consistent.
+
+Otherwise the recommended way to work around this issue is to use the
+<<dfs-query-then-fetch,`dfs_query_then_fetch`>> search type. This will make
+Elasticsearch perform an inital round trip to all involved shards, asking
+them for their index statistics relatively to the query, then the coordinating
+node will merge those statistics and send the merged statistics alongside the
+request when asking shards to perform the `query` phase, so that shards can
+use these global statistics rather than their own statistics in order to do the
+scoring.
+
+In most cases, this additional round trip should be very cheap. However in the
+event that your query contains a very large number of fields/terms or fuzzy
+queries, beware that gathering statistics alone might not be cheap since all
+terms have to be looked up in the terms dictionaries in order to look up
+statistics.
+
diff --git a/docs/reference/images/msi_installer/elasticsearch_exe.png b/docs/reference/images/msi_installer/elasticsearch_exe.png
new file mode 100644
index 0000000000000..1c0aacb58c096
Binary files /dev/null and b/docs/reference/images/msi_installer/elasticsearch_exe.png differ
diff --git a/docs/reference/images/msi_installer/msi_installer_configuration.png b/docs/reference/images/msi_installer/msi_installer_configuration.png
new file mode 100644
index 0000000000000..d5502dc7cca9f
Binary files /dev/null and b/docs/reference/images/msi_installer/msi_installer_configuration.png differ
diff --git a/docs/reference/images/msi_installer/msi_installer_help.png b/docs/reference/images/msi_installer/msi_installer_help.png
new file mode 100644
index 0000000000000..9b63c512100de
Binary files /dev/null and b/docs/reference/images/msi_installer/msi_installer_help.png differ
diff --git a/docs/reference/images/msi_installer/msi_installer_installed_service.png b/docs/reference/images/msi_installer/msi_installer_installed_service.png
new file mode 100644
index 0000000000000..34585377a9113
Binary files /dev/null and b/docs/reference/images/msi_installer/msi_installer_installed_service.png differ
diff --git a/docs/reference/images/msi_installer/msi_installer_locations.png b/docs/reference/images/msi_installer/msi_installer_locations.png
new file mode 100644
index 0000000000000..6ccb3dcb23fbb
Binary files /dev/null and b/docs/reference/images/msi_installer/msi_installer_locations.png differ
diff --git a/docs/reference/images/msi_installer/msi_installer_no_service.png b/docs/reference/images/msi_installer/msi_installer_no_service.png
new file mode 100644
index 0000000000000..d26bb75f265f6
Binary files /dev/null and b/docs/reference/images/msi_installer/msi_installer_no_service.png differ
diff --git a/docs/reference/images/msi_installer/msi_installer_plugins.png b/docs/reference/images/msi_installer/msi_installer_plugins.png
new file mode 100644
index 0000000000000..fc3b29677dd53
Binary files /dev/null and b/docs/reference/images/msi_installer/msi_installer_plugins.png differ
diff --git a/docs/reference/images/msi_installer/msi_installer_selected_plugins.png b/docs/reference/images/msi_installer/msi_installer_selected_plugins.png
new file mode 100644
index 0000000000000..c39631eb6cc07
Binary files /dev/null and b/docs/reference/images/msi_installer/msi_installer_selected_plugins.png differ
diff --git a/docs/reference/images/msi_installer/msi_installer_service.png b/docs/reference/images/msi_installer/msi_installer_service.png
new file mode 100644
index 0000000000000..a964abf6ed53d
Binary files /dev/null and b/docs/reference/images/msi_installer/msi_installer_service.png differ
diff --git a/docs/reference/images/msi_installer/msi_installer_success.png b/docs/reference/images/msi_installer/msi_installer_success.png
new file mode 100644
index 0000000000000..1c391e3dfd1ad
Binary files /dev/null and b/docs/reference/images/msi_installer/msi_installer_success.png differ
diff --git a/docs/reference/images/msi_installer/msi_installer_uninstall.png b/docs/reference/images/msi_installer/msi_installer_uninstall.png
new file mode 100644
index 0000000000000..d612b51d7a184
Binary files /dev/null and b/docs/reference/images/msi_installer/msi_installer_uninstall.png differ
diff --git a/docs/reference/images/msi_installer/msi_installer_upgrade_configuration.png b/docs/reference/images/msi_installer/msi_installer_upgrade_configuration.png
new file mode 100644
index 0000000000000..04a5e92cc39f0
Binary files /dev/null and b/docs/reference/images/msi_installer/msi_installer_upgrade_configuration.png differ
diff --git a/docs/reference/images/msi_installer/msi_installer_upgrade_notice.png b/docs/reference/images/msi_installer/msi_installer_upgrade_notice.png
new file mode 100644
index 0000000000000..2b3ac7ea118e8
Binary files /dev/null and b/docs/reference/images/msi_installer/msi_installer_upgrade_notice.png differ
diff --git a/docs/reference/images/msi_installer/msi_installer_upgrade_plugins.png b/docs/reference/images/msi_installer/msi_installer_upgrade_plugins.png
new file mode 100644
index 0000000000000..ef6e93bb84991
Binary files /dev/null and b/docs/reference/images/msi_installer/msi_installer_upgrade_plugins.png differ
diff --git a/docs/reference/index-modules.asciidoc b/docs/reference/index-modules.asciidoc
index ff1a3c62c7fd6..604dd24c5f624 100644
--- a/docs/reference/index-modules.asciidoc
+++ b/docs/reference/index-modules.asciidoc
@@ -38,7 +38,11 @@ specific index module:
 
     The number of primary shards that an index should have.  Defaults to 5.
     This setting can only be set at index creation time.  It cannot be
-    changed on a closed index.
+    changed on a closed index. Note: the number of shards are limited to `1024` per
+    index. This limitation is a safety limit to prevent accidental creation of indices
+    that can destabilize a cluster due to resource allocation. The limit can be modified
+    by specifying `export ES_JAVA_OPTS="-Des.index.max_number_of_shards=128"` system property on every node that is
+    part of the cluster.
 
 `index.shard.check_on_startup`::
 +
@@ -74,6 +78,16 @@ Checking shards may take a lot of time on large indices.
     compression, but this can be set to +best_compression+
     which uses https://en.wikipedia.org/wiki/DEFLATE[DEFLATE] for a higher
     compression ratio, at the expense of slower stored fields performance.
+    If you are updating the compression type, the new one will be applied
+    after segments are merged. Segment merging can be forced using
+    <<indices-forcemerge,force merge>>.
+
+[[routing-partition-size]] `index.routing_partition_size`::
+
+    The number of shards a custom <<mapping-routing-field,routing>> value can go to.
+    Defaults to 1 and can only be set at index creation time. This value must be less
+    than the `index.number_of_shards` unless the `index.number_of_shards` value is also 1.
+    See <<routing-index-partition>> for more details about how this setting is used.
 
 [float]
 [[dynamic-index-settings]]
@@ -109,7 +123,7 @@ specific index module:
 
 `index.max_rescore_window`::
 
-    The maximum value of `window_size` for `rescore`s in searches of this index.
+    The maximum value of `window_size` for `rescore` requests in searches of this index.
     Defaults to `index.max_result_window` which defaults to `10000`. Search
     requests take heap memory and time proportional to
     `max(window_size, from + size)` and this limits that memory.
@@ -119,6 +133,10 @@ specific index module:
     Set to `true` to make the index and index metadata read only, `false` to
     allow writes and metadata changes.
 
+`index.blocks.read_only_allow_delete`::
+    Identical to `index.blocks.read_only` but allows deleting the index to free
+     up resources.
+
 `index.blocks.read`::
 
     Set to `true` to disable read operations against the index.
diff --git a/docs/reference/index-modules/similarity.asciidoc b/docs/reference/index-modules/similarity.asciidoc
index 1833b45a9baab..01a8241bc2eb3 100644
--- a/docs/reference/index-modules/similarity.asciidoc
+++ b/docs/reference/index-modules/similarity.asciidoc
@@ -5,7 +5,7 @@ A similarity (scoring / ranking model) defines how matching documents
 are scored. Similarity is per field, meaning that via the mapping one
 can define a different similarity per field.
 
-Configuring a custom similarity is considered a expert feature and the
+Configuring a custom similarity is considered an expert feature and the
 builtin similarities are most likely sufficient as is described in
 <<similarity>>.
 
@@ -174,9 +174,41 @@ implementation used for these two methods, while not changing the
 `default`, it is possible to configure a similarity with the name
 `base`. This similarity will then be used for the two methods.
 
-You can change the default similarity for all fields by putting the following setting into `elasticsearch.yml`:
+You can change the default similarity for all fields in an index when
+it is <<indices-create-index,created>>:
 
 [source,js]
 --------------------------------------------------
-index.similarity.default.type: classic
+PUT /my_index
+{
+  "settings": {
+    "index": {
+      "similarity": {
+        "default": {
+          "type": "classic"
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+
+If you want to change the default similarity after creating the index
+you must <<indices-open-close,close>> your index, send the follwing
+request and <<indices-open-close,open>> it again afterwards:
+
+[source,js]
+--------------------------------------------------
+PUT /my_index/_settings
+{
+  "settings": {
+    "index": {
+      "similarity": {
+        "default": {
+          "type": "classic"
+        }
+      }
+    }
+  }
+}
 --------------------------------------------------
diff --git a/docs/reference/index-modules/slowlog.asciidoc b/docs/reference/index-modules/slowlog.asciidoc
index 423968bb07181..1ee49b1c04dbc 100644
--- a/docs/reference/index-modules/slowlog.asciidoc
+++ b/docs/reference/index-modules/slowlog.asciidoc
@@ -24,7 +24,7 @@ index.search.slowlog.threshold.fetch.debug: 500ms
 index.search.slowlog.threshold.fetch.trace: 200ms
 --------------------------------------------------
 
-All of the above settings are _dynamic_ and can be set per-index.
+All of the above settings are _dynamic_ and are set per-index.
 
 By default, none are enabled (set to `-1`). Levels (`warn`, `info`,
 `debug`, `trace`) allow to control under which logging level the log
@@ -42,7 +42,7 @@ level.
 The logging file is configured by default using the following
 configuration (found in `log4j2.properties`):
 
-[source,yaml]
+[source,properties]
 --------------------------------------------------
 appender.index_search_slowlog_rolling.type = RollingFile
 appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling
@@ -67,8 +67,8 @@ logger.index_search_slowlog_rolling.additivity = false
 
 The indexing slow log, similar in functionality to the search slow
 log. The log file name ends with `_index_indexing_slowlog.log`. Log and
-the thresholds are configured in the elasticsearch.yml file in the same
-way as the search slowlog. Index slowlog sample:
+the thresholds are configured in the same way as the search slowlog.
+Index slowlog sample:
 
 [source,yaml]
 --------------------------------------------------
@@ -80,23 +80,35 @@ index.indexing.slowlog.level: info
 index.indexing.slowlog.source: 1000
 --------------------------------------------------
 
-All of the above settings are _dynamic_ and can be set per-index.
+All of the above settings are _dynamic_ and are set per-index.
 
 By default Elasticsearch will log the first 1000 characters of the _source in
 the slowlog. You can change that with `index.indexing.slowlog.source`. Setting
 it to `false` or `0` will skip logging the source entirely an setting it to
-`true` will log the entire source regardless of size.
+`true` will log the entire source regardless of size. The original `_source` is
+reformatted by default to make sure that it fits on a single log line. If preserving
+the original document format is important, you can turn off reformatting by setting
+`index.indexing.slowlog.reformat` to `false`, which will cause the source to be
+logged "as is" and can potentially span multiple log lines.
 
-The index slow log file is configured by default in the `logging.yml`
+The index slow log file is configured by default in the `log4j2.properties`
 file:
 
-[source,yaml]
+[source,properties]
 --------------------------------------------------
-index_indexing_slow_log_file:
-    type: dailyRollingFile
-    file: ${path.logs}/${cluster.name}_index_indexing_slowlog.log
-    datePattern: "'.'yyyy-MM-dd"
-    layout:
-      type: pattern
-      conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"
+appender.index_indexing_slowlog_rolling.type = RollingFile
+appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling
+appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs}_index_indexing_slowlog.log
+appender.index_indexing_slowlog_rolling.layout.type = PatternLayout
+appender.index_indexing_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.10000m%n
+appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs}_index_indexing_slowlog-%d{yyyy-MM-dd}.log
+appender.index_indexing_slowlog_rolling.policies.type = Policies
+appender.index_indexing_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy
+appender.index_indexing_slowlog_rolling.policies.time.interval = 1
+appender.index_indexing_slowlog_rolling.policies.time.modulate = true
+
+logger.index_indexing_slowlog.name = index.indexing.slowlog.index
+logger.index_indexing_slowlog.level = trace
+logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling.ref = index_indexing_slowlog_rolling
+logger.index_indexing_slowlog.additivity = false
 --------------------------------------------------
diff --git a/docs/reference/index-modules/store.asciidoc b/docs/reference/index-modules/store.asciidoc
index ee7fd8766fdc9..e33378ddd70be 100644
--- a/docs/reference/index-modules/store.asciidoc
+++ b/docs/reference/index-modules/store.asciidoc
@@ -79,7 +79,7 @@ By default, elasticsearch completely relies on the operating system file system
 cache for caching I/O operations. It is possible to set `index.store.preload`
 in order to tell the operating system to load the content of hot index
 files into memory upon opening. This setting accept a comma-separated list of
-files extensions: all files whose extenion is in the list will be pre-loaded
+files extensions: all files whose extension is in the list will be pre-loaded
 upon opening. This can be useful to improve search performance of an index,
 especially when the host operating system is restarted, since this causes the
 file system cache to be trashed. However note that this may slow down the
@@ -89,12 +89,12 @@ loaded into physical memory.
 This setting is best-effort only and may not work at all depending on the store
 type and host operating system.
 
-The `index.store.pre_load` is a static setting that can either be set in the
+The `index.store.preload` is a static setting that can either be set in the
 `config/elasticsearch.yml`:
 
 [source,yaml]
 ---------------------------------
-index.store.pre_load: ["nvd", "dvd"]
+index.store.preload: ["nvd", "dvd"]
 ---------------------------------
 
 or in the index settings at index creation time:
@@ -104,7 +104,7 @@ or in the index settings at index creation time:
 PUT /my_index
 {
   "settings": {
-    "index.store.pre_load": ["nvd", "dvd"]
+    "index.store.preload": ["nvd", "dvd"]
   }
 }
 ---------------------------------
@@ -116,7 +116,7 @@ values to be loaded eagerly into physical memory. These are the two first
 extensions to look at since elasticsearch performs random access on them.
 
 A wildcard can be used in order to indicate that all files should be preloaded:
-`index.store.pre_load: ["*"]`. Note however that it is generally not useful to
+`index.store.preload: ["*"]`. Note however that it is generally not useful to
 load all files into memory, in particular those for stored fields and term
 vectors, so a better option might be to set it to
 `["nvd", "dvd", "tim", "doc", "dim"]`, which will preload norms, doc values,
diff --git a/docs/reference/index-shared1.asciidoc b/docs/reference/index-shared1.asciidoc
new file mode 100644
index 0000000000000..9325bd6e73e29
--- /dev/null
+++ b/docs/reference/index-shared1.asciidoc
@@ -0,0 +1,4 @@
+
+include::getting-started.asciidoc[]
+
+include::setup.asciidoc[]
diff --git a/docs/reference/index-shared2.asciidoc b/docs/reference/index-shared2.asciidoc
new file mode 100644
index 0000000000000..e48948079cc9f
--- /dev/null
+++ b/docs/reference/index-shared2.asciidoc
@@ -0,0 +1,2 @@
+
+include::migration/index.asciidoc[]
diff --git a/docs/reference/index-shared3.asciidoc b/docs/reference/index-shared3.asciidoc
new file mode 100644
index 0000000000000..4da338186b0c8
--- /dev/null
+++ b/docs/reference/index-shared3.asciidoc
@@ -0,0 +1,26 @@
+
+include::api-conventions.asciidoc[]
+
+include::docs.asciidoc[]
+
+include::search.asciidoc[]
+
+include::aggregations.asciidoc[]
+
+include::indices.asciidoc[]
+
+include::cat.asciidoc[]
+
+include::cluster.asciidoc[]
+
+include::query-dsl.asciidoc[]
+
+include::mapping.asciidoc[]
+
+include::analysis.asciidoc[]
+
+include::modules.asciidoc[]
+
+include::index-modules.asciidoc[]
+
+include::ingest.asciidoc[]
diff --git a/docs/reference/index-shared4.asciidoc b/docs/reference/index-shared4.asciidoc
new file mode 100644
index 0000000000000..3d807dd98d39c
--- /dev/null
+++ b/docs/reference/index-shared4.asciidoc
@@ -0,0 +1,8 @@
+
+include::how-to.asciidoc[]
+
+include::testing.asciidoc[]
+
+include::glossary.asciidoc[]
+
+include::release-notes.asciidoc[]
diff --git a/docs/reference/index-shared5.asciidoc b/docs/reference/index-shared5.asciidoc
new file mode 100644
index 0000000000000..572522f6c8e74
--- /dev/null
+++ b/docs/reference/index-shared5.asciidoc
@@ -0,0 +1,2 @@
+
+include::redirects.asciidoc[]
diff --git a/docs/reference/index.asciidoc b/docs/reference/index.asciidoc
index 427ef0a84dcbf..8aa9eef32f8bf 100644
--- a/docs/reference/index.asciidoc
+++ b/docs/reference/index.asciidoc
@@ -1,55 +1,12 @@
 [[elasticsearch-reference]]
 = Elasticsearch Reference
 
-:version:        5.0.0-alpha5
-:major-version:  5.x
-:lucene_version: 6.2.0
-:branch:         master
-:jdk:            1.8.0_73
-:defguide:       https://www.elastic.co/guide/en/elasticsearch/guide/master
-:plugins:        https://www.elastic.co/guide/en/elasticsearch/plugins/master
-:javaclient:     https://www.elastic.co/guide/en/elasticsearch/client/java-api/master/
-:issue:          https://github.com/elastic/elasticsearch/issues/
-:pull:           https://github.com/elastic/elasticsearch/pull/
-
-include::getting-started.asciidoc[]
-
-include::setup.asciidoc[]
-
-include::migration/index.asciidoc[]
-
-include::api-conventions.asciidoc[]
-
-include::docs.asciidoc[]
-
-include::search.asciidoc[]
-
-include::aggregations.asciidoc[]
-
-include::indices.asciidoc[]
-
-include::cat.asciidoc[]
-
-include::cluster.asciidoc[]
-
-include::query-dsl.asciidoc[]
-
-include::mapping.asciidoc[]
-
-include::analysis.asciidoc[]
-
-include::modules.asciidoc[]
-
-include::index-modules.asciidoc[]
-
-include::ingest.asciidoc[]
-
-include::how-to.asciidoc[]
-
-include::testing.asciidoc[]
-
-include::glossary.asciidoc[]
-
-include::release-notes.asciidoc[]
-
-include::redirects.asciidoc[]
+:es-test-dir:           {docdir}/../src/test
+:plugins-examples-dir:  {docdir}/../../plugins/examples
+
+include::../Versions.asciidoc[]
+include::index-shared1.asciidoc[]
+include::index-shared2.asciidoc[]
+include::index-shared3.asciidoc[]
+include::index-shared4.asciidoc[]
+include::index-shared5.asciidoc[]
diff --git a/docs/reference/indices.asciidoc b/docs/reference/indices.asciidoc
index 3c1eadfa07e4c..afb5088d85e0a 100644
--- a/docs/reference/indices.asciidoc
+++ b/docs/reference/indices.asciidoc
@@ -60,7 +60,6 @@ index settings, aliases, mappings, and index templates.
 * <<indices-refresh>>
 * <<indices-flush>>
 * <<indices-forcemerge>>
-* <<indices-upgrade>>
 
 --
 
@@ -114,5 +113,3 @@ include::indices/refresh.asciidoc[]
 
 include::indices/forcemerge.asciidoc[]
 
-include::indices/upgrade.asciidoc[]
-
diff --git a/docs/reference/indices/aliases.asciidoc b/docs/reference/indices/aliases.asciidoc
index 16d318ee97439..3f839465e8ba2 100644
--- a/docs/reference/indices/aliases.asciidoc
+++ b/docs/reference/indices/aliases.asciidoc
@@ -1,12 +1,12 @@
 [[indices-aliases]]
 == Index Aliases
 
-APIs in elasticsearch accept an index name when working against a
+APIs in Elasticsearch accept an index name when working against a
 specific index, and several indices when applicable. The index aliases
-API allow to alias an index with a name, with all APIs automatically
+API allows aliasing an index with a name, with all APIs automatically
 converting the alias name to the actual index name. An alias can also be
 mapped to more than one index, and when specifying it, the alias will
-automatically expand to the aliases indices. An alias can also be
+automatically expand to the aliased indices. An alias can also be
 associated with a filter that will automatically be applied when
 searching, and routing values. An alias cannot have the same name as an index.
 
@@ -55,7 +55,7 @@ POST /_aliases
 // CONSOLE
 // TEST[continued]
 
-Associating an alias with more than one index are simply several `add`
+Associating an alias with more than one index is simply several `add`
 actions:
 
 [source,js]
@@ -374,7 +374,7 @@ DELETE /logs_20162801/_alias/current_day
 [[alias-retrieving]]
 === Retrieving existing aliases
 
-The get index alias api allows to filter by
+The get index alias API allows to filter by
 alias name and index name. This api redirects to the master and fetches
 the requested index aliases, if available. This api only serialises the
 found index aliases.
@@ -382,7 +382,7 @@ found index aliases.
 Possible options:
 [horizontal]
 `index`::
-    The index name to get aliases for. Partially names are
+    The index name to get aliases for. Partial names are
     supported via wildcards, also multiple index names can be specified
     separated with a comma. Also the alias name for an index can be used.
 
diff --git a/docs/reference/indices/analyze.asciidoc b/docs/reference/indices/analyze.asciidoc
index dbb2c8f101adb..31e68fa8fbb52 100644
--- a/docs/reference/indices/analyze.asciidoc
+++ b/docs/reference/indices/analyze.asciidoc
@@ -9,23 +9,25 @@ analyzers:
 
 [source,js]
 --------------------------------------------------
-curl -XGET 'localhost:9200/_analyze' -d '
+GET _analyze
 {
   "analyzer" : "standard",
   "text" : "this is a test"
-}'
+}
 --------------------------------------------------
+// CONSOLE
 
 If text parameter is provided as array of strings, it is analyzed as a multi-valued field.
 
 [source,js]
 --------------------------------------------------
-curl -XGET 'localhost:9200/_analyze' -d '
+GET _analyze
 {
   "analyzer" : "standard",
   "text" : ["this is a test", "the second text"]
-}'
+}
 --------------------------------------------------
+// CONSOLE
 
 Or by building a custom transient analyzer out of tokenizers,
 token filters and char filters. Token filters can use the shorter 'filter'
@@ -33,21 +35,26 @@ parameter name:
 
 [source,js]
 --------------------------------------------------
-curl -XGET 'localhost:9200/_analyze' -d '
+GET _analyze
 {
   "tokenizer" : "keyword",
   "filter" : ["lowercase"],
   "text" : "this is a test"
-}'
+}
+--------------------------------------------------
+// CONSOLE
 
-curl -XGET 'localhost:9200/_analyze' -d '
+[source,js]
+--------------------------------------------------
+GET _analyze
 {
   "tokenizer" : "keyword",
   "filter" : ["lowercase"],
   "char_filter" : ["html_strip"],
   "text" : "this is a <b>test</b>"
-}'
+}
 --------------------------------------------------
+// CONSOLE
 
 deprecated[5.0.0, Use `filter`/`char_filter` instead of `filters`/`char_filters` and `token_filters` has been removed]
 
@@ -55,23 +62,26 @@ Custom tokenizers, token filters, and character filters can be specified in the
 
 [source,js]
 --------------------------------------------------
-curl -XGET 'localhost:9200/_analyze' -d '
+GET _analyze
 {
   "tokenizer" : "whitespace",
   "filter" : ["lowercase", {"type": "stop", "stopwords": ["a", "is", "this"]}],
   "text" : "this is a test"
-}'
+}
 --------------------------------------------------
+// CONSOLE
 
 It can also run against a specific index:
 
 [source,js]
 --------------------------------------------------
-curl -XGET 'localhost:9200/test/_analyze' -d '
+GET twitter/_analyze
 {
   "text" : "this is a test"
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
 
 The above will run an analysis on the "this is a test" text, using the
 default index analyzer associated with the `test` index. An `analyzer`
@@ -79,41 +89,53 @@ can also be provided to use a different analyzer:
 
 [source,js]
 --------------------------------------------------
-curl -XGET 'localhost:9200/test/_analyze' -d '
+GET twitter/_analyze
 {
   "analyzer" : "whitespace",
   "text" : "this is a test"
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
 
 Also, the analyzer can be derived based on a field mapping, for example:
 
 [source,js]
 --------------------------------------------------
-curl -XGET 'localhost:9200/test/_analyze' -d '
+GET twitter/_analyze
 {
   "field" : "obj1.field1",
   "text" : "this is a test"
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
 
 Will cause the analysis to happen based on the analyzer configured in the
 mapping for `obj1.field1` (and if not, the default index analyzer).
 
+deprecated[5.1.0 request parameters are deprecated and will be removed in the next major release. please use JSON params instead of request params]
 All parameters can also supplied as request parameters. For example:
 
 [source,js]
 --------------------------------------------------
-curl -XGET 'localhost:9200/_analyze?tokenizer=keyword&filter=lowercase&text=this+is+a+test'
+GET /_analyze?tokenizer=keyword&filter=lowercase&text=this+is+a+test
 --------------------------------------------------
+// CONSOLE
+// TEST[warning:text request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param]
+// TEST[warning:tokenizer request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param]
+// TEST[warning:filter request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param]
 
 For backwards compatibility, we also accept the text parameter as the body of the request,
 provided it doesn't start with `{` :
 
 [source,js]
 --------------------------------------------------
-curl -XGET 'localhost:9200/_analyze?tokenizer=keyword&filter=lowercase&char_filter=html_strip' -d 'this is a <b>test</b>'
+curl -XGET 'localhost:9200/_analyze?tokenizer=keyword&filter=lowercase&char_filter=reverse' -d 'this is a test' -H 'Content-Type: text/plain'
 --------------------------------------------------
+// NOTCONSOLE
+
+deprecated[5.1.0 the text parameter as the body of the request are deprecated and this feature will be removed in the next major release. please use JSON text param]
 
 === Explain Analyze
 
diff --git a/docs/reference/indices/create-index.asciidoc b/docs/reference/indices/create-index.asciidoc
index a04fe0ca3f695..6537fa74cb79f 100644
--- a/docs/reference/indices/create-index.asciidoc
+++ b/docs/reference/indices/create-index.asciidoc
@@ -129,13 +129,14 @@ what happened:
 --------------------------------------------------
 {
     "acknowledged": true,
-    "shards_acknowledged": true
+    "shards_acknowledged": true,
+    "index": "test"
 }
 --------------------------------------------------
 // TESTRESPONSE
 
 `acknowledged` indicates whether the index was successfully created in the cluster, while 
-`shards_acknowledged` indices whether the requisite number of shard copies were started for
+`shards_acknowledged` indicates whether the requisite number of shard copies were started for
 each shard in the index before timing out. Note that it is still possible for either 
 `acknowledged` or `shards_acknowledged` to be `false`, but the index creation was successful. 
 These values simply indicate whether the operation completed before the timeout. If 
diff --git a/docs/reference/indices/flush.asciidoc b/docs/reference/indices/flush.asciidoc
index 5864c16d4c1d6..2b9f519e17c7c 100644
--- a/docs/reference/indices/flush.asciidoc
+++ b/docs/reference/indices/flush.asciidoc
@@ -74,7 +74,7 @@ the <<indices-stats,indices stats>> API:
 
 [source,sh]
 --------------------------------------------------
-GET twitter/_stats/commit?level=shards
+GET twitter/_stats?level=shards
 --------------------------------------------------
 // CONSOLE
 // TEST[s/^/PUT twitter\n/]
diff --git a/docs/reference/indices/get-field-mapping.asciidoc b/docs/reference/indices/get-field-mapping.asciidoc
index 224e74605f486..d2f0d5a750815 100644
--- a/docs/reference/indices/get-field-mapping.asciidoc
+++ b/docs/reference/indices/get-field-mapping.asciidoc
@@ -5,34 +5,53 @@ The get field mapping API allows you to retrieve mapping definitions for one or
 This is useful when you do not need the complete type mapping returned by
 the <<indices-get-mapping>> API.
 
-The following returns the mapping of the field `text` only:
+For example, consider the following mapping:
 
 [source,js]
 --------------------------------------------------
-GET /twitter/_mapping/tweet/field/message
+PUT publications
+{
+    "mappings": {
+        "article": {
+            "properties": {
+                "id": { "type": "text" },
+                "title":  { "type": "text"},
+                "abstract": { "type": "text"},
+                "author": {
+                    "properties": {
+                        "id": { "type": "text" },
+                        "name": { "type": "text" }
+                    }
+                }
+            }
+        }
+    }
+}
 --------------------------------------------------
+// TESTSETUP
 // CONSOLE
-// TEST[setup:twitter]
 
-For which the response is (assuming `text` is a default string field):
+The following returns the mapping of the field `title` only:
+
+[source,js]
+--------------------------------------------------
+GET publications/_mapping/article/field/title
+--------------------------------------------------
+// CONSOLE
+
+For which the response is:
 
 [source,js]
 --------------------------------------------------
 {
-   "twitter": {
+   "publications": {
       "mappings": {
-         "tweet": {
-            "message": {
-               "full_name": "message",
+         "article": {
+            "title": {
+               "full_name": "title",
                "mapping": {
-                  "message": {
-                     "type": "text",
-                     "fields": {
-                        "keyword": {
-                           "type": "keyword",
-                           "ignore_above": 256
-                        }
-                     }
+                  "title": {
+                     "type": "text"
                   }
                }
             }
@@ -43,7 +62,6 @@ For which the response is (assuming `text` is a default string field):
 --------------------------------------------------
 // TESTRESPONSE
 
-
 [float]
 === Multiple Indices, Types and Fields
 
@@ -69,46 +87,54 @@ GET /_all/_mapping/tw*/field/*.id
 [float]
 === Specifying fields
 
-The get mapping api allows you to specify one or more fields separated with by a comma.
-You can also use wildcards. The field names can be any of the following:
+The get mapping api allows you to specify a comma-separated list of fields.
 
-[horizontal]
-Full names:: the full path, including any parent object name the field is
-   part of (ex. `user.id`).
-Field names:: the name of the field without the path to it (ex. `id` for `{ "user" : { "id" : 1 } }`).
+For instance to select the `id` of the `author` field, you must use its full name `author.id`.
 
-The above options are specified in the order the `field` parameter is resolved.
-The first field found which matches is returned. This is especially important
-if index names or field names are used as those can be ambiguous.
+[source,js]
+--------------------------------------------------
+GET publications/_mapping/article/field/author.id,abstract,name
+--------------------------------------------------
+// CONSOLE
 
-For example, consider the following mapping:
+returns:
 
 [source,js]
 --------------------------------------------------
- {
-     "article": {
-         "properties": {
-             "id": { "type": "text" },
-             "title":  { "type": "text"},
-             "abstract": { "type": "text"},
-             "author": {
-                 "properties": {
-                     "id": { "type": "text" },
-                     "name": { "type": "text" }
-                 }
-             }
+{
+   "publications": {
+      "mappings": {
+         "article": {
+            "author.id": {
+               "full_name": "author.id",
+               "mapping": {
+                  "id": {
+                     "type": "text"
+                  }
+               }
+            },
+            "abstract": {
+               "full_name": "abstract",
+               "mapping": {
+                  "abstract": {
+                     "type": "text"
+                  }
+               }
+            }
          }
-     }
- }
+      }
+   }
+}
 --------------------------------------------------
+// TESTRESPONSE
 
-To select the `id` of the `author` field, you can use its full name `author.id`. `name` will return
-the field `author.name`:
+The get field mapping API also supports wildcard notation.
 
 [source,js]
 --------------------------------------------------
-curl -XGET "http://localhost:9200/publications/_mapping/article/field/author.id,abstract,name"
+GET publications/_mapping/article/field/a*
 --------------------------------------------------
+// CONSOLE
 
 returns:
 
@@ -116,33 +142,38 @@ returns:
 --------------------------------------------------
 {
    "publications": {
-      "article": {
-         "abstract": {
-            "full_name": "abstract",
-            "mapping": {
-               "abstract": { "type": "text" }
-            }
-         },
-         "author.id": {
-            "full_name": "author.id",
-            "mapping": {
-               "id": { "type": "text" }
-            }
-         },
-         "name": {
-            "full_name": "author.name",
-            "mapping": {
-               "name": { "type": "text" }
+      "mappings": {
+         "article": {
+            "author.name": {
+               "full_name": "author.name",
+               "mapping": {
+                  "name": {
+                     "type": "text"
+                  }
+               }
+            },
+            "abstract": {
+               "full_name": "abstract",
+               "mapping": {
+                  "abstract": {
+                     "type": "text"
+                  }
+               }
+            },
+            "author.id": {
+               "full_name": "author.id",
+               "mapping": {
+                  "id": {
+                     "type": "text"
+                  }
+               }
             }
          }
       }
    }
 }
 --------------------------------------------------
-
-Note how the response always use the same fields specified in the request as keys.
-The `full_name` in every entry contains the full name of the field whose mapping were returned.
-This is useful when the request can refer to to multiple fields.
+// TESTRESPONSE
 
 [float]
 === Other options
diff --git a/docs/reference/indices/get-index.asciidoc b/docs/reference/indices/get-index.asciidoc
index 772318c71d87a..dabceb22994f9 100644
--- a/docs/reference/indices/get-index.asciidoc
+++ b/docs/reference/indices/get-index.asciidoc
@@ -19,6 +19,8 @@ all indices by using `_all` or `*` as index.
 [float]
 === Filtering index information
 
+deprecated[5.5.0, This comma-separated format is deprecated and will be removed. You can retrieve either a single feature (ie _alias) or all features]
+
 The information returned by the get API can be filtered to include only specific features
 by specifying a comma delimited list of features in the URL:
 
@@ -27,7 +29,7 @@ by specifying a comma delimited list of features in the URL:
 GET twitter/_settings,_mappings
 --------------------------------------------------
 // CONSOLE
-// TEST[setup:twitter]
+// TEST[setup:twitter warning:Requesting comma-separated features is deprecated and will be removed in 6.0+, retrieve all features instead.]
 
 The above command will only return the settings and mappings for the index called `twitter`.
 
diff --git a/docs/reference/indices/get-mapping.asciidoc b/docs/reference/indices/get-mapping.asciidoc
index c3580917d9ad5..d1e45b2dbb09b 100644
--- a/docs/reference/indices/get-mapping.asciidoc
+++ b/docs/reference/indices/get-mapping.asciidoc
@@ -23,9 +23,9 @@ following are some examples:
 
 [source,js]
 --------------------------------------------------
-GET /_mapping/tweet,kimchy
+GET /_mapping/tweet
 
-GET /_all/_mapping/tweet,book
+GET /_all/_mapping/tweet
 --------------------------------------------------
 // CONSOLE
 // TEST[setup:twitter]
diff --git a/docs/reference/indices/get-settings.asciidoc b/docs/reference/indices/get-settings.asciidoc
index 60d7a75a86142..5fb488f49c3e9 100644
--- a/docs/reference/indices/get-settings.asciidoc
+++ b/docs/reference/indices/get-settings.asciidoc
@@ -40,5 +40,5 @@ as follows:
 
 [source,js]
 --------------------------------------------------
-curl -XGET 'http://localhost:9200/2013-*/_settings/name=index.number_*'
+curl -XGET 'http://localhost:9200/2013-*/_settings/index.number_*'
 --------------------------------------------------
diff --git a/docs/reference/indices/put-mapping.asciidoc b/docs/reference/indices/put-mapping.asciidoc
index 5105f82b0912d..6bc623b95e251 100644
--- a/docs/reference/indices/put-mapping.asciidoc
+++ b/docs/reference/indices/put-mapping.asciidoc
@@ -1,7 +1,7 @@
 [[indices-put-mapping]]
 == Put Mapping
 
-The PUT mapping API allows you to provide type mappings while creating a new index, add a new type to an existing index, or add new
+The PUT mapping API allows you to add a new type to an existing index, or add new
 fields to an existing type:
 
 [source,js]
@@ -62,6 +62,11 @@ PUT /{index}/_mapping/{type}
 * `{body}` contains the mapping changes that should be applied.
 
 
+NOTE: When updating the `_default_` mapping with the
+<<indices-put-mapping,PUT mapping>> API, the new mapping is not merged with
+the existing mapping.  Instead, the new `_default_` mapping replaces the
+existing one.
+
 [[updating-field-mappings]]
 [float]
 === Updating field mappings
@@ -71,7 +76,6 @@ exceptions to this rule. For instance:
 
 * new <<properties>> can be added to <<object>> fields.
 * new <<multi-fields,multi-fields>> can be added to existing fields.
-* <<doc-values>> can be disabled, but not enabled.
 * the <<ignore-above>> parameter can be updated.
 
 For example:
diff --git a/docs/reference/indices/rollover-index.asciidoc b/docs/reference/indices/rollover-index.asciidoc
index 6ee28e7b2f453..9aec8243af3f9 100644
--- a/docs/reference/indices/rollover-index.asciidoc
+++ b/docs/reference/indices/rollover-index.asciidoc
@@ -7,7 +7,7 @@ index is considered to be too large or too old.
 The API accepts a single alias name and a list of `conditions`.  The alias
 must point to a single index only.  If the index satisfies the specified
 conditions then a new index is created and the alias is switched to point to
-the new alias.
+the new index.
 
 
 [source,js]
@@ -34,7 +34,7 @@ POST /logs_write/_rollover <2>
 // TEST[s/# Add > 1000 documents to logs-000001/POST _reindex?refresh\n{"source":{"index":"twitter"},"dest":{"index":"logs-000001"}}/]
 <1> Creates an index called `logs-0000001` with the alias `logs_write`.
 <2> If the index pointed to by `logs_write` was created 7 or more days ago, or
-    contains 1,000 or more documents, then the `logs-0002` index is created
+    contains 1,000 or more documents, then the `logs-000002` index is created
     and the `logs_write` alias is updated to point to `logs-000002`.
 
 The above request might return the following response:
@@ -83,6 +83,86 @@ POST /my_alias/_rollover/my_new_index_name
 // CONSOLE
 // TEST[s/^/PUT my_old_index_name\nPUT my_old_index_name\/_alias\/my_alias\n/]
 
+[float]
+=== Using date math with the rollover API
+
+It can be useful to use <<date-math-index-names,date math>> to name the
+rollover index according to the date that the index rolled over, e.g.
+`logstash-2016.02.03`.  The rollover API supports date math, but requires the
+index name to end with a dash followed by a number, e.g.
+`logstash-2016.02.03-1` which is incremented every time the index is rolled
+over. For instance:
+
+[source,js]
+--------------------------------------------------
+# PUT /<logs-{now/d}-1> with URI encoding:
+PUT /%3Clogs-%7Bnow%2Fd%7D-1%3E <1>
+{
+  "aliases": {
+    "logs_write": {}
+  }
+}
+
+PUT logs_write/log/1
+{
+  "message": "a dummy log"
+}
+
+POST logs_write/_refresh
+
+# Wait for a day to pass
+
+POST /logs_write/_rollover <2>
+{
+  "conditions": {
+    "max_docs":   "1"
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[s/now/2016.10.31||/]
+<1> Creates an index named with today's date (e.g.) `logs-2016.10.31-1`
+<2> Rolls over to a new index with today's date, e.g. `logs-2016.10.31-000002` if run immediately, or `logs-2016.11.01-000002` if run after 24 hours
+
+//////////////////////////
+
+[source,js]
+--------------------------------------------------
+GET _alias
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+[source,js]
+--------------------------------------------------
+{
+  "logs-2016.10.31-000002": {
+    "aliases": {
+      "logs_write": {}
+    }
+  },
+  "logs-2016.10.31-1": {
+    "aliases": {}
+  }
+}
+--------------------------------------------------
+// TESTRESPONSE
+
+//////////////////////////
+
+These indices can then be referenced as described in the
+<<date-math-index-names,date math documentation>>.  For example, to search
+over indices created in the last three days, you could do the following:
+
+[source,js]
+--------------------------------------------------
+# GET /<logs-{now/d}-*>,<logs-{now/d-1d}-*>,<logs-{now/d-2d}-*>/_search
+GET /%3Clogs-%7Bnow%2Fd%7D-*%3E%2C%3Clogs-%7Bnow%2Fd-1d%7D-*%3E%2C%3Clogs-%7Bnow%2Fd-2d%7D-*%3E/_search
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+// TEST[s/now/2016.10.31||/]
+
 [float]
 === Defining the new index
 
@@ -144,5 +224,5 @@ POST /logs_write/_rollover?dry_run
 === Wait For Active Shards
 
 Because the rollover operation creates a new index to rollover to, the
-<<create-index-wait-for-active-shards,wait for active shards>> setting on
+<<create-index-wait-for-active-shards,`wait_for_active_shards`>> setting on
 index creation applies to the rollover action as well.
diff --git a/docs/reference/indices/shadow-replicas.asciidoc b/docs/reference/indices/shadow-replicas.asciidoc
index 3a0b23852b0cc..dd255a0e644ed 100644
--- a/docs/reference/indices/shadow-replicas.asciidoc
+++ b/docs/reference/indices/shadow-replicas.asciidoc
@@ -1,7 +1,7 @@
 [[indices-shadow-replicas]]
 == Shadow replica indices
 
-experimental[]
+deprecated[5.2.0, Shadow replicas don't see much usage and we are planning to remove them]
 
 If you would like to use a shared filesystem, you can use the shadow replicas
 settings to choose where on disk the data for an index should be kept, as well
@@ -44,7 +44,7 @@ mappings when using shadow replicas.
 
 [source,js]
 --------------------------------------------------
-curl -XPUT 'localhost:9200/my_index' -d '
+PUT /my_index
 {
     "index" : {
         "number_of_shards" : 1,
@@ -52,8 +52,10 @@ curl -XPUT 'localhost:9200/my_index' -d '
         "data_path": "/opt/data/my_index",
         "shadow_replicas": true
     }
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[skip:no way to configure path.shared_data for /opt/data]
 
 [WARNING]
 ========================
diff --git a/docs/reference/indices/templates.asciidoc b/docs/reference/indices/templates.asciidoc
index 6e2f7ce91f1d2..614ba5c869ec7 100644
--- a/docs/reference/indices/templates.asciidoc
+++ b/docs/reference/indices/templates.asciidoc
@@ -2,12 +2,15 @@
 == Index Templates
 
 Index templates allow you to define templates that will automatically be
-applied when new indices are created. The templates include both settings and
-mappings, and a simple pattern template that controls whether the template
-should be applied to the new index.
+applied when new indices are created. The templates include both 
+<<index-modules-settings,settings>> and <<mapping,mappings>>
+and a simple pattern template that controls whether the template should be 
+applied to the new index.
 
-NOTE: Templates are only applied at index creation time.  Changing a template
-will have no impact on existing indices.
+NOTE: Templates are only applied at index creation time. Changing a template
+will have no impact on existing indices. When using the create index API, the
+settings/mappings defined as part of the create index call will take precedence
+over any matching settings/mappings defined in the template.
 
 For example:
 
@@ -40,9 +43,12 @@ PUT _template/template_1
 // CONSOLE
 // TESTSETUP
 
-Defines a template named template_1, with a template pattern of `te*`.
+NOTE: Index templates provide C-style /* */ block comments. Comments are allowed
+everywhere in the JSON document except before the initial opening curly bracket. 
+
+Defines a template named `template_1`, with a template pattern of `te*`.
 The settings and mappings will be applied to any index name that matches
-the `te*` template.
+the `te*` pattern.
 
 It is also possible to include aliases in an index template as follows:
 
@@ -69,8 +75,8 @@ PUT _template/template_1
 // CONSOLE
 // TEST[s/^/DELETE _template\/template_1\n/]
 
-<1> the `{index}` placeholder within the alias name will be replaced with the
-actual index name that the template gets applied to during index creation.
+<1> the `{index}` placeholder in the alias name will be replaced with the
+actual index name that the template gets applied to, during index creation.
 
 [float]
 [[delete]]
@@ -117,7 +123,7 @@ GET /_template
 
 [float]
 [[indices-templates-exists]]
-=== Templates exists
+=== Template exists
 
 Used to check if the template exists or not. For example:
 
@@ -128,11 +134,12 @@ HEAD _template/template_1
 // CONSOLE
 
 The HTTP status code indicates if the template with the given name
-exists or not. A status code `200` means it exists, a `404` it does not.
+exists or not. Status code `200` means it exists and `404` means
+it does not.
 
 [float]
 [[multiple-templates]]
-=== Multiple Template Matching
+=== Multiple Templates Matching
 
 Multiple index templates can potentially match an index, in this case,
 both the settings and mappings are merged into the final configuration
@@ -174,7 +181,54 @@ PUT /_template/template_2
 // TEST[s/^/DELETE _template\/template_1\n/]
 
 The above will disable storing the `_source` on all `type1` types, but
-for indices of that start with `te*`, source will still be enabled.
+for indices that start with `te*`, `_source` will still be enabled.
 Note, for mappings, the merging is "deep", meaning that specific
 object/property based mappings can easily be added/overridden on higher
 order templates, with lower order templates providing the basis.
+
+[float]
+[[versioning-templates]]
+=== Template Versioning
+
+Templates can optionally add a `version` number, which can be any integer value,
+in order to simplify template management by external systems. The `version`
+field is completely optional and it is meant solely for external management of
+templates. To unset a `version`, simply replace the template without specifying
+one.
+
+[source,js]
+--------------------------------------------------
+PUT /_template/template_1
+{
+    "template" : "*",
+    "order" : 0,
+    "settings" : {
+        "number_of_shards" : 1
+    },
+    "version": 123
+}
+--------------------------------------------------
+// CONSOLE
+
+To check the `version`, you can
+<<common-options-response-filtering, filter responses>>
+using `filter_path` to limit the response to just the `version`:
+
+[source,js]
+--------------------------------------------------
+GET /_template/template_1?filter_path=*.version
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+This should give a small response that makes it both easy and inexpensive to parse:
+
+[source,js]
+--------------------------------------------------
+{
+  "template_1" : {
+    "version" : 123
+  }
+}
+--------------------------------------------------
+// TESTRESPONSE
diff --git a/docs/reference/indices/update-settings.asciidoc b/docs/reference/indices/update-settings.asciidoc
index 574565192e4dc..15d0cdaaead96 100644
--- a/docs/reference/indices/update-settings.asciidoc
+++ b/docs/reference/indices/update-settings.asciidoc
@@ -4,33 +4,25 @@
 Change specific index level settings in real time.
 
 The REST endpoint is `/_settings` (to update all indices) or
-`{index}/_settings` to update one (or more) indices settings. The body
-of the request includes the updated settings, for example:
+`{index}/_settings` to update one (or more) indices settings.
+The body of the request includes the updated settings, for example:
 
 [source,js]
 --------------------------------------------------
+PUT /twitter/_settings
 {
     "index" : {
-        "number_of_replicas" : 4
+        "number_of_replicas" : 2
     }
 }
 --------------------------------------------------
-
-The above will change the number of replicas to 4 from the current
-number of replicas. Here is a curl example:
-
-[source,js]
---------------------------------------------------
-curl -XPUT 'localhost:9200/my_index/_settings' -d '
-{
-    "index" : {
-        "number_of_replicas" : 4
-    }
-}'
---------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
 
 The list of per-index settings which can be updated dynamically on live
 indices can be found in <<index-modules>>.
+To preserve existing settings from being updated, the `preserve_existing`
+request parameter can be set to `true`.
 
 [float]
 [[bulk]]
@@ -43,11 +35,15 @@ use:
 
 [source,js]
 --------------------------------------------------
-curl -XPUT localhost:9200/test/_settings -d '{
+PUT /twitter/_settings
+{
     "index" : {
         "refresh_interval" : "-1"
-    } }'
+    }
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
 
 (Another optimization option is to start the index without any replicas,
 and only later adding them, but that really depends on the use case).
@@ -57,18 +53,24 @@ the defaults for example):
 
 [source,js]
 --------------------------------------------------
-curl -XPUT localhost:9200/test/_settings -d '{
+PUT /twitter/_settings
+{
     "index" : {
         "refresh_interval" : "1s"
-    } }'
+    }
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 And, a force merge should be called:
 
 [source,js]
 --------------------------------------------------
-curl -XPOST 'http://localhost:9200/test/_forcemerge?max_num_segments=5'
+POST /twitter/_forcemerge?max_num_segments=5
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 [float]
 [[update-settings-analysis]]
@@ -83,9 +85,10 @@ you can use the following commands to add it:
 
 [source,js]
 --------------------------------------------------
-curl -XPOST 'localhost:9200/myindex/_close'
+POST /twitter/_close
 
-curl -XPUT 'localhost:9200/myindex/_settings' -d '{
+PUT /twitter/_settings
+{
   "analysis" : {
     "analyzer":{
       "content":{
@@ -94,7 +97,9 @@ curl -XPUT 'localhost:9200/myindex/_settings' -d '{
       }
     }
   }
-}'
+}
 
-curl -XPOST 'localhost:9200/myindex/_open'
+POST /twitter/_open
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
diff --git a/docs/reference/indices/upgrade.asciidoc b/docs/reference/indices/upgrade.asciidoc
deleted file mode 100644
index 1b8e99f212571..0000000000000
--- a/docs/reference/indices/upgrade.asciidoc
+++ /dev/null
@@ -1,123 +0,0 @@
-[[indices-upgrade]]
-== Upgrade
-
-The upgrade API allows to upgrade one or more indices to the latest Lucene
-format through an API. The upgrade process converts any segments written with
-older formats.
-
-[IMPORTANT]
-===================================================
-
-**The upgrade API in its current form will not help you to migrate indices
-created in Elasticsearch 1.x to 5.x.**
-
-The upgrade API rewrites an index in the latest Lucene format, but it still
-retains the original data structures that were used when the index was first
-created.  For instance:
-
-* Doc-values on numeric fields used to use BinaryDocValues, but now use dedicated NumericDocValues.
-* The parent-child feature has been completely rewritten to use a new data structure.
-* Geo-point fields now require doc values and the Lucene index where, previously, they relied on in-memory calculations.
-
-**Migrating 1.x indices to 5.x**
-
-The only way to prepare an index created in 1.x for use in 5.x is to **reindex
-your data** in a cluster running Elasticsearch 2.3.x, which you can do with
-the new <<docs-reindex,reindex API>>.
-
-The steps to do this are as follows:
-
-1. Create a new index (e.g. `new_index`) with the correct settings and
-   mappings.  These can be retrieved from the old index with the
-   <<indices-get-index,get-index>> API.
-
-2. Reindex from `old_index` to `new_index` with the
-   <<docs-reindex,reindex API>>.
-
-3. Retrieve a list of any aliases associated with the `old_index` using the
-   <<alias-retrieving,get-alias API>>.
-
-4. Delete the `old_index` using the <<indices-delete-index,delete index API>>.
-
-5. Add an alias called `old_index` to the `new_index` along with any aliases
-   returned in step 3, using the <<indices-aliases,update aliases API>>.
-
-In the future, we plan to change the upgrade API to perform a reindex-in-
-place.  In other words, it would reindex data from `old_index` to `.old_index`
-then atomically delete `old_index` and rename `.old_index` to `old_index`.
-
-===================================================
-
-
-[float]
-=== Start an upgrade
-
-[source,sh]
---------------------------------------------------
-$ curl -XPOST 'http://localhost:9200/twitter/_upgrade'
---------------------------------------------------
-
-NOTE: Upgrading is an I/O intensive operation, and is limited to processing a
-single shard per node at a time.  It also is not allowed to run at the same
-time as an optimize/force-merge.
-
-This call will block until the upgrade is complete. If the http connection
-is lost, the request will continue in the background, and
-any new requests will block until the previous upgrade is complete.
-
-[float]
-[[upgrade-parameters]]
-==== Request Parameters
-
-The `upgrade` API accepts the following request parameters:
-
-[horizontal]
-`only_ancient_segments`:: If true, only very old segments (from a
-previous Lucene major release) will be upgraded.  While this will do
-the minimal work to ensure the next major release of Elasticsearch can
-read the segments, it's dangerous because it can leave other very old
-segments in sub-optimal formats.  Defaults to `false`.
-
-[float]
-=== Check upgrade status
-
-Use a `GET` request to monitor how much of an index is upgraded.  This
-can also be used prior to starting an upgrade to identify which
-indices you want to upgrade at the same time.
-
-The `ancient` byte values that are returned indicate total bytes of
-segments whose version is extremely old (Lucene major version is
-different from the current version), showing how much upgrading is
-necessary when you run with `only_ancient_segments=true`.
-
-[source,sh]
---------------------------------------------------
-curl 'http://localhost:9200/twitter/_upgrade?pretty&human'
---------------------------------------------------
-
-[source,js]
---------------------------------------------------
-{
-  "size": "21gb",
-  "size_in_bytes": "21000000000",
-  "size_to_upgrade": "10gb",
-  "size_to_upgrade_in_bytes": "10000000000"
-  "size_to_upgrade_ancient": "1gb",
-  "size_to_upgrade_ancient_in_bytes": "1000000000"
-  "indices": {
-    "twitter": {
-      "size": "21gb",
-      "size_in_bytes": "21000000000",
-      "size_to_upgrade": "10gb",
-      "size_to_upgrade_in_bytes": "10000000000"
-      "size_to_upgrade_ancient": "1gb",
-      "size_to_upgrade_ancient_in_bytes": "1000000000"
-    }
-  }
-}
---------------------------------------------------
-
-The level of details in the upgrade status command can be controlled by
-setting `level` parameter to `cluster`, `index` (default) or `shard` levels.
-For example, you can run the upgrade status command with `level=shard` to
-get detailed upgrade information of each individual shard.
diff --git a/docs/reference/ingest.asciidoc b/docs/reference/ingest.asciidoc
index 7150bd32739c1..2710b56288574 100644
--- a/docs/reference/ingest.asciidoc
+++ b/docs/reference/ingest.asciidoc
@@ -18,7 +18,8 @@ node.ingest: false
 To pre-process documents before indexing, you <<pipeline,define a pipeline>> that specifies
 a series of <<ingest-processors,processors>>. Each processor transforms the document in some way.
 For example, you may have a pipeline that consists of one processor that removes a field from
-the document followed by another processor that renames a field.
+the document followed by another processor that renames a field. Configured pipelines are then stored
+in the <<cluster-state,cluster state>>.
 
 To use a pipeline, you simply specify the `pipeline` parameter on an index or bulk request to
 tell the ingest node which pipeline to use. For example:
diff --git a/docs/reference/ingest/ingest-node.asciidoc b/docs/reference/ingest/ingest-node.asciidoc
index a06e3b9e1df49..adabd82fa55ab 100644
--- a/docs/reference/ingest/ingest-node.asciidoc
+++ b/docs/reference/ingest/ingest-node.asciidoc
@@ -12,6 +12,7 @@ and a list of `processors`:
   "processors" : [ ... ]
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 The `description` is a special field to store a helpful description of
 what the pipeline does.
@@ -90,10 +91,61 @@ For each returned pipeline, the source and the version are returned.
 The version is useful for knowing which version of the pipeline the node has.
 You can specify multiple IDs to return more than one pipeline. Wildcards are also supported.
 
+[float]
+[[versioning-pipelines]]
+==== Pipeline Versioning
+
+Pipelines can optionally add a `version` number, which can be any integer value,
+in order to simplify pipeline management by external systems. The `version`
+field is completely optional and it is meant solely for external management of
+pipelines. To unset a `version`, simply replace the pipeline without specifying
+one.
+
+[source,js]
+--------------------------------------------------
+PUT _ingest/pipeline/my-pipeline-id
+{
+  "description" : "describe pipeline",
+  "version" : 123,
+  "processors" : [
+    {
+      "set" : {
+        "field": "foo",
+        "value": "bar"
+      }
+    }
+  ]
+}
+--------------------------------------------------
+// CONSOLE
+
+To check for the `version`, you can
+<<common-options-response-filtering, filter responses>>
+using `filter_path` to limit the response to just the `version`:
+
+[source,js]
+--------------------------------------------------
+GET /_ingest/pipeline/my-pipeline-id?filter_path=*.version
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+This should give a small response that makes it both easy and inexpensive to parse:
+
+[source,js]
+--------------------------------------------------
+{
+  "my-pipeline-id" : {
+    "version" : 123
+  }
+}
+--------------------------------------------------
+// TESTRESPONSE
+
 [[delete-pipeline-api]]
 === Delete Pipeline API
 
-The delete pipeline API deletes pipelines by ID.
+The delete pipeline API deletes pipelines by ID or wildcard match (`my-*`, `*`).
 
 [source,js]
 --------------------------------------------------
@@ -102,6 +154,36 @@ DELETE _ingest/pipeline/my-pipeline-id
 // CONSOLE
 // TEST[continued]
 
+////
+Hidden setup for wildcard test:
+[source,js]
+--------------------------------------------------
+PUT _ingest/pipeline/wild-one
+{
+  "description" : "first pipeline to be wildcard deleted",
+  "processors" : [ ]
+}
+
+PUT _ingest/pipeline/wild-two
+{
+  "description" : "second pipeline to be wildcard deleted",
+  "processors" : [ ]
+}
+
+DELETE _ingest/pipeline/*
+--------------------------------------------------
+// CONSOLE
+
+Hidden expected response:
+[source,js]
+--------------------------------------------------
+{
+"acknowledged": true
+}
+--------------------------------------------------
+// TESTRESPONSE
+////
+
 [[simulate-pipeline-api]]
 === Simulate Pipeline API
 
@@ -123,12 +205,13 @@ POST _ingest/pipeline/_simulate
     // pipeline definition here
   },
   "docs" : [
-    { /** first document **/ },
-    { /** second document **/ },
+    { "_source": {/** first document **/} },
+    { "_source": {/** second document **/} },
     // ...
   ]
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 Here is the structure of a simulate request against an existing pipeline:
 
@@ -137,13 +220,13 @@ Here is the structure of a simulate request against an existing pipeline:
 POST _ingest/pipeline/my-pipeline-id/_simulate
 {
   "docs" : [
-    { /** first document **/ },
-    { /** second document **/ },
+    { "_source": {/** first document **/} },
+    { "_source": {/** second document **/} },
     // ...
   ]
 }
 --------------------------------------------------
-
+// NOTCONSOLE
 
 Here is an example of a simulate request with a pipeline defined in the request
 and its response:
@@ -195,42 +278,36 @@ Response:
       {
          "doc": {
             "_id": "id",
-            "_ttl": null,
-            "_parent": null,
             "_index": "index",
-            "_routing": null,
             "_type": "type",
-            "_timestamp": null,
             "_source": {
                "field2": "_value",
                "foo": "bar"
             },
             "_ingest": {
-               "timestamp": "2016-01-04T23:53:27.186+0000"
+               "timestamp": "2017-05-04T22:30:03.187Z"
             }
          }
       },
       {
          "doc": {
             "_id": "id",
-            "_ttl": null,
-            "_parent": null,
             "_index": "index",
-            "_routing": null,
             "_type": "type",
-            "_timestamp": null,
             "_source": {
                "field2": "_value",
                "foo": "rab"
             },
             "_ingest": {
-               "timestamp": "2016-01-04T23:53:27.186+0000"
+               "timestamp": "2017-05-04T22:30:03.188Z"
             }
          }
       }
    ]
 }
 --------------------------------------------------
+// TESTRESPONSE[s/"2017-05-04T22:30:03.187Z"/$body.docs.0.doc._ingest.timestamp/]
+// TESTRESPONSE[s/"2017-05-04T22:30:03.188Z"/$body.docs.1.doc._ingest.timestamp/]
 
 [[ingest-verbose-param]]
 ==== Viewing Verbose Results
@@ -294,41 +371,31 @@ Response:
       {
          "processor_results": [
             {
-               "tag": "processor[set]-0",
                "doc": {
                   "_id": "id",
-                  "_ttl": null,
-                  "_parent": null,
                   "_index": "index",
-                  "_routing": null,
                   "_type": "type",
-                  "_timestamp": null,
                   "_source": {
                      "field2": "_value2",
                      "foo": "bar"
                   },
                   "_ingest": {
-                     "timestamp": "2016-01-05T00:02:51.383+0000"
+                     "timestamp": "2017-05-04T22:46:09.674Z"
                   }
                }
             },
             {
-               "tag": "processor[set]-1",
                "doc": {
                   "_id": "id",
-                  "_ttl": null,
-                  "_parent": null,
                   "_index": "index",
-                  "_routing": null,
                   "_type": "type",
-                  "_timestamp": null,
                   "_source": {
                      "field3": "_value3",
                      "field2": "_value2",
                      "foo": "bar"
                   },
                   "_ingest": {
-                     "timestamp": "2016-01-05T00:02:51.383+0000"
+                     "timestamp": "2017-05-04T22:46:09.675Z"
                   }
                }
             }
@@ -337,41 +404,31 @@ Response:
       {
          "processor_results": [
             {
-               "tag": "processor[set]-0",
                "doc": {
                   "_id": "id",
-                  "_ttl": null,
-                  "_parent": null,
                   "_index": "index",
-                  "_routing": null,
                   "_type": "type",
-                  "_timestamp": null,
                   "_source": {
                      "field2": "_value2",
                      "foo": "rab"
                   },
                   "_ingest": {
-                     "timestamp": "2016-01-05T00:02:51.384+0000"
+                     "timestamp": "2017-05-04T22:46:09.676Z"
                   }
                }
             },
             {
-               "tag": "processor[set]-1",
                "doc": {
                   "_id": "id",
-                  "_ttl": null,
-                  "_parent": null,
                   "_index": "index",
-                  "_routing": null,
                   "_type": "type",
-                  "_timestamp": null,
                   "_source": {
                      "field3": "_value3",
                      "field2": "_value2",
                      "foo": "rab"
                   },
                   "_ingest": {
-                     "timestamp": "2016-01-05T00:02:51.384+0000"
+                     "timestamp": "2017-05-04T22:46:09.677Z"
                   }
                }
             }
@@ -380,6 +437,10 @@ Response:
    ]
 }
 --------------------------------------------------
+// TESTRESPONSE[s/"2017-05-04T22:46:09.674Z"/$body.docs.0.processor_results.0.doc._ingest.timestamp/]
+// TESTRESPONSE[s/"2017-05-04T22:46:09.675Z"/$body.docs.0.processor_results.1.doc._ingest.timestamp/]
+// TESTRESPONSE[s/"2017-05-04T22:46:09.676Z"/$body.docs.1.processor_results.0.doc._ingest.timestamp/]
+// TESTRESPONSE[s/"2017-05-04T22:46:09.677Z"/$body.docs.1.processor_results.1.doc._ingest.timestamp/]
 
 [[accessing-data-in-pipelines]]
 == Accessing Data in Pipelines
@@ -402,6 +463,7 @@ their name. For example:
   }
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 On top of this, fields from the source are always accessible via the `_source` prefix:
 
@@ -414,6 +476,7 @@ On top of this, fields from the source are always accessible via the `_source` p
   }
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 [float]
 [[accessing-metadata-fields]]
@@ -433,6 +496,7 @@ The following example sets the `_id` metadata field of a document to `1`:
   }
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 The following metadata fields are accessible by a processor: `_index`, `_type`, `_id`, `_routing`, `_parent`.
 
@@ -458,6 +522,7 @@ The following example adds a field with the name `received`. The value is the in
   }
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 Unlike Elasticsearch metadata fields, the ingest metadata field name `_ingest` can be used as a valid field name
 in the source of a document. Use `_source._ingest` to refer to the field in the source document. Otherwise, `_ingest`
@@ -482,6 +547,7 @@ the values of `field_a` and `field_b`.
   }
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 The following example uses the value of the `geoip.country_iso_code` field in the source
 to set the index that the document will be indexed into:
@@ -495,6 +561,7 @@ to set the index that the document will be indexed into:
   }
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 [[handling-failure-in-pipelines]]
 == Handling Failures in Pipelines
@@ -540,6 +607,7 @@ Elasticsearch.
   ]
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 The following example defines an `on_failure` block on a whole pipeline to change
 the index to which failed documents get sent.
@@ -559,6 +627,7 @@ the index to which failed documents get sent.
   ]
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 Alternatively instead of defining behaviour in case of processor failure, it is also possible
 to ignore a failure and continue with the next processor by specifying the `ignore_failure` setting.
@@ -581,6 +650,7 @@ continues to execute, which in this case means that the pipeline does nothing.
   ]
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 The `ignore_failure` can be set on any processor and defaults to `false`.
 
@@ -619,6 +689,7 @@ metadata field to provide the error message.
   ]
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 [[ingest-processors]]
 == Processors
@@ -633,6 +704,7 @@ All processors are defined in the following way within a pipeline definition:
   }
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 Each processor defines its own configuration parameters, but all processors have
 the ability to declare `tag` and `on_failure` fields. These fields are optional.
@@ -660,7 +732,7 @@ A node will not start if either of these plugins are not available.
 The <<ingest-stats,node stats API>> can be used to fetch ingest usage statistics, globally and on a per
 pipeline basis. Useful to find out which pipelines are used the most or spent the most time on preprocessing.
 
-[[append-procesesor]]
+[[append-processor]]
 === Append Processor
 Appends one or more values to an existing array if the field already exists and it is an array.
 Converts a scalar to an array and appends one or more values to it if the field exists and it is a scalar.
@@ -680,11 +752,12 @@ Accepts a single value or an array of values.
 --------------------------------------------------
 {
   "append": {
-    "field": "field1"
+    "field": "field1",
     "value": ["item2", "item3", "item4"]
   }
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 [[convert-processor]]
 === Convert Processor
@@ -706,21 +779,23 @@ such a case, `target_field` will still be updated with the unconverted field val
 .Convert Options
 [options="header"]
 |======
-| Name           | Required  | Default  | Description
-| `field`        | yes       | -        | The field whose value is to be converted
-| `target_field` | no        | `field`  | The field to assign the converted value to, by default `field` is updated in-place
-| `type`         | yes       | -        | The type to convert the existing value to
+| Name             | Required  | Default  | Description
+| `field`          | yes       | -        | The field whose value is to be converted
+| `target_field`   | no        | `field`  | The field to assign the converted value to, by default `field` is updated in-place
+| `type`           | yes       | -        | The type to convert the existing value to
+| `ignore_missing` | no        | `false`  | If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document
 |======
 
 [source,js]
 --------------------------------------------------
 {
   "convert": {
-    "field" : "foo"
+    "field" : "foo",
     "type": "integer"
   }
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 [[date-processor]]
 === Date Processor
@@ -761,6 +836,7 @@ Here is an example that adds the parsed date to the `timestamp` field based on t
   ]
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 [[date-index-name-processor]]
 === Date Index Name Processor
@@ -880,14 +956,14 @@ and the result:
           "date1" : "2016-04-25T12:02:01.789Z"
         },
         "_ingest" : {
-          "timestamp" : "2016-08-11T12:00:01.222Z"
+          "timestamp" : "2016-11-08T19:43:03.850+0000"
         }
       }
     }
   ]
 }
 --------------------------------------------------
-// TESTRESPONSE[s/2016-08-11T12:00:01.222Z/$body.docs.0.doc._ingest.timestamp/]
+// TESTRESPONSE[s/2016-11-08T19:43:03.850\+0000/$body.docs.0.doc._ingest.timestamp/]
 
 The above example shows that `_index` was set to `<myindex-{2016-04-25||/M{yyyy-MM-dd|UTC}}>`. Elasticsearch
 understands this to mean `2016-04-01` as is explained in the <<date-math-index-names, date math index name documentation>>
@@ -928,9 +1004,16 @@ to the requester.
   }
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 [[foreach-processor]]
 === Foreach Processor
+
+experimental[This processor may change or be replaced by something else that provides similar functionality. This
+processor executes in its own context, which makes it different compared to all other processors and for features like
+verbose simulation the subprocessor isn't visible. The reason we still expose this processor, is that it is the only
+processor that can operate on an array]
+
 Processes elements in an array of unknown length.
 
 All processors can operate on elements inside an array, but if all elements of an array need to
@@ -970,6 +1053,7 @@ Assume the following document:
   "values" : ["foo", "bar", "baz"]
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 When this `foreach` processor operates on this sample document:
 
@@ -986,6 +1070,7 @@ When this `foreach` processor operates on this sample document:
   }
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 Then the document will look like this after preprocessing:
 
@@ -995,6 +1080,7 @@ Then the document will look like this after preprocessing:
   "values" : ["FOO", "BAR", "BAZ"]
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 Let's take a look at another example:
 
@@ -1013,6 +1099,7 @@ Let's take a look at another example:
   ]
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 In this case, the `id` field needs to be removed,
 so the following `foreach` processor is used:
@@ -1030,6 +1117,7 @@ so the following `foreach` processor is used:
   }
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 After preprocessing the result is:
 
@@ -1046,6 +1134,7 @@ After preprocessing the result is:
   ]
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 The wrapped processor can have a `on_failure` definition.
 For example, the `id` field may not exist on all person objects.
@@ -1073,11 +1162,16 @@ block to send the document to the 'failure_index' index for later inspection:
   }
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 In this example, if the `remove` processor does fail, then
 the array elements that have been processed thus far will
 be updated.
 
+Another advanced example can be found in the {plugins}/ingest-attachment-with-arrays.html[attachment processor documentation].
+
+
+
 [[grok-processor]]
 === Grok Processor
 
@@ -1117,7 +1211,7 @@ The `TYPE` is the type you wish to cast your named field. `int` and `float` are
 
 For example, you might want to match the following text:
 
-[source,js]
+[source,txt]
 --------------------------------------------------
 3.44 55.3.244.1
 --------------------------------------------------
@@ -1125,7 +1219,7 @@ For example, you might want to match the following text:
 You may know that the message in the example is a number followed by an IP address. You can match this text by using the following
 Grok expression.
 
-[source,js]
+[source,txt]
 --------------------------------------------------
 %{NUMBER:duration} %{IP:client}
 --------------------------------------------------
@@ -1142,6 +1236,7 @@ Grok expression.
 | `patterns`             | yes       | -                   | An ordered list of grok expression to match and extract named captures with. Returns on the first expression in the list that matches.
 | `pattern_definitions`  | no        | -                   | A map of pattern-name and pattern tuples defining custom patterns to be used by the current processor. Patterns matching existing names will override the pre-existing definition.
 | `trace_match`          | no        | false               | when true, `_ingest._grok_match_index` will be inserted into your matched document's metadata with the index into the pattern found in `patterns` that matched.
+| `ignore_missing`       | no        | false               | If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document
 |======
 
 Here is an example of using the provided patterns to extract out and name structured fields from a string field in
@@ -1153,10 +1248,11 @@ a document.
   "message": "55.3.244.1 GET /index.html 15824 0.043"
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 The pattern for this could be:
 
-[source,js]
+[source,txt]
 --------------------------------------------------
 %{IP:client} %{WORD:method} %{URIPATHPARAM:request} %{NUMBER:bytes} %{NUMBER:duration}
 --------------------------------------------------
@@ -1177,6 +1273,7 @@ Here is an example pipeline for processing the above document by using Grok:
   ]
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 This pipeline will insert these named captures as new fields within the document, like so:
 
@@ -1191,6 +1288,7 @@ This pipeline will insert these named captures as new fields within the document
   "duration": "0.043"
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 [[custom-patterns]]
 ==== Custom Patterns and Pattern Files
@@ -1219,6 +1317,166 @@ Here is an example of a pipeline specifying custom pattern definitions:
   ]
 }
 --------------------------------------------------
+// NOTCONSOLE
+
+[[trace-match]]
+==== Providing Multiple Match Patterns
+
+Sometimes one pattern is not enough to capture the potential structure of a field. Let's assume we
+want to match all messages that contain your favorite pet breeds of either cats or dogs. One way to accomplish
+this is to provide two distinct patterns that can be matched, instead of one really complicated expression capturing
+the same `or` behavior.
+
+Here is an example of such a configuration executed against the simulate API:
+
+[source,js]
+--------------------------------------------------
+POST _ingest/pipeline/_simulate
+{
+  "pipeline": {
+  "description" : "parse multiple patterns",
+  "processors": [
+    {
+      "grok": {
+        "field": "message",
+        "patterns": ["%{FAVORITE_DOG:pet}", "%{FAVORITE_CAT:pet}"],
+        "pattern_definitions" : {
+          "FAVORITE_DOG" : "beagle",
+          "FAVORITE_CAT" : "burmese"
+        }
+      }
+    }
+  ]
+},
+"docs":[
+  {
+    "_source": {
+      "message": "I love burmese cats!"
+    }
+  }
+  ]
+}
+--------------------------------------------------
+// CONSOLE
+
+response:
+
+[source,js]
+--------------------------------------------------
+{
+  "docs": [
+    {
+      "doc": {
+        "_type": "_type",
+        "_index": "_index",
+        "_id": "_id",
+        "_source": {
+          "message": "I love burmese cats!",
+          "pet": "burmese"
+        },
+        "_ingest": {
+          "timestamp": "2016-11-08T19:43:03.850+0000"
+        }
+      }
+    }
+  ]
+}
+--------------------------------------------------
+// TESTRESPONSE[s/2016-11-08T19:43:03.850\+0000/$body.docs.0.doc._ingest.timestamp/]
+
+Both patterns will set the field `pet` with the appropriate match, but what if we want to trace which of our
+patterns matched and populated our fields? We can do this with the `trace_match` parameter. Here is the output of
+that same pipeline, but with `"trace_match": true` configured:
+
+////
+Hidden setup for example:
+[source,js]
+--------------------------------------------------
+POST _ingest/pipeline/_simulate
+{
+  "pipeline": {
+  "description" : "parse multiple patterns",
+  "processors": [
+    {
+      "grok": {
+        "field": "message",
+        "patterns": ["%{FAVORITE_DOG:pet}", "%{FAVORITE_CAT:pet}"],
+        "trace_match": true,
+        "pattern_definitions" : {
+          "FAVORITE_DOG" : "beagle",
+          "FAVORITE_CAT" : "burmese"
+        }
+      }
+    }
+  ]
+},
+"docs":[
+  {
+    "_source": {
+      "message": "I love burmese cats!"
+    }
+  }
+  ]
+}
+--------------------------------------------------
+// CONSOLE
+////
+
+[source,js]
+--------------------------------------------------
+{
+  "docs": [
+    {
+      "doc": {
+        "_type": "_type",
+        "_index": "_index",
+        "_id": "_id",
+        "_source": {
+          "message": "I love burmese cats!",
+          "pet": "burmese"
+        },
+        "_ingest": {
+          "_grok_match_index": "1",
+          "timestamp": "2016-11-08T19:43:03.850+0000"
+        }
+      }
+    }
+  ]
+}
+--------------------------------------------------
+// TESTRESPONSE[s/2016-11-08T19:43:03.850\+0000/$body.docs.0.doc._ingest.timestamp/]
+
+In the above response, you can see that the index of the pattern that matched was `"1"`. This is to say that it was the
+second (index starts at zero) pattern in `patterns` to match.
+
+This trace metadata enables debugging which of the patterns matched. This information is stored in the ingest
+metadata and will not be indexed.
+
+[[grok-processor-rest-get]]
+==== Retrieving patterns from REST endpoint
+
+The Grok Processor comes packaged with its own REST endpoint for retrieving which patterns the processor is packaged with.
+
+[source,js]
+--------------------------------------------------
+GET _ingest/processor/grok
+--------------------------------------------------
+// CONSOLE
+
+The above request will return a response body containing a key-value representation of the built-in patterns dictionary.
+
+[source,js]
+--------------------------------------------------
+{
+  "patterns" : {
+    "BACULA_CAPACITY" : "%{INT}{1,3}(,%{INT}{3})*",
+    "PATH" : "(?:%{UNIXPATH}|%{WINPATH})",
+    ...
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+This can be useful to reference as the built-in patterns change across versions.
 
 [[gsub-processor]]
 === Gsub Processor
@@ -1229,10 +1487,12 @@ If the field is not a string, the processor will throw an exception.
 .Gsub Options
 [options="header"]
 |======
-| Name          | Required  | Default  | Description
-| `field`       | yes       | -        | The field to apply the replacement to
-| `pattern`     | yes       | -        | The pattern to be replaced
-| `replacement` | yes       | -        | The string to replace the matching patterns with
+| Name              | Required  | Default  | Description
+| `field`           | yes       | -        | The field to apply the replacement to
+| `pattern`         | yes       | -        | The pattern to be replaced
+| `replacement`     | yes       | -        | The string to replace the matching patterns with
+| `target_field`    | no        | `field`  | The field to assign the converted value to, by default `field` is updated in-place
+| `ignore_missing`  | no        | `false`  | If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document
 |======
 
 [source,js]
@@ -1245,6 +1505,7 @@ If the field is not a string, the processor will throw an exception.
   }
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 [[join-processor]]
 === Join Processor
@@ -1255,9 +1516,10 @@ Throws an error when the field is not an array.
 .Join Options
 [options="header"]
 |======
-| Name          | Required  | Default  | Description
-| `field`       | yes       | -        | The field to be separated
-| `separator`   | yes       | -        | The separator character
+| Name              | Required  | Default  | Description
+| `field`           | yes       | -        | The field to be separated
+| `separator`       | yes       | -        | The separator character
+| `target_field`    | no        | `field`  | The field to assign the joined value to, by default `field` is updated in-place
 |======
 
 [source,js]
@@ -1269,6 +1531,127 @@ Throws an error when the field is not an array.
   }
 }
 --------------------------------------------------
+// NOTCONSOLE
+
+[[json-processor]]
+=== JSON Processor
+Converts a JSON string into a structured JSON object.
+
+[[json-options]]
+.Json Options
+[options="header"]
+|======
+| Name           | Required  | Default  | Description
+| `field`        | yes       | -        | The field to be parsed
+| `target_field` | no        | `field`  | The field to insert the converted structured object into
+| `add_to_root`  | no        | false    | Flag that forces the serialized json to be injected into the top level of the document. `target_field` must not be set when this option is chosen.
+|======
+
+Suppose you provide this configuration of the `json` processor:
+
+[source,js]
+--------------------------------------------------
+{
+  "json" : {
+    "field" : "string_source",
+    "target_field" : "json_target"
+  }
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+If the following document is processed:
+
+[source,js]
+--------------------------------------------------
+{
+  "string_source": "{\"foo\": 2000}"
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+after the `json` processor operates on it, it will look like:
+
+[source,js]
+--------------------------------------------------
+{
+  "string_source": "{\"foo\": 2000}",
+  "json_target": {
+    "foo": 2000
+  }
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+If the following configuration is provided, omitting the optional `target_field` setting:
+[source,js]
+--------------------------------------------------
+{
+  "json" : {
+    "field" : "source_and_target"
+  }
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+then after the `json` processor operates on this document:
+
+[source,js]
+--------------------------------------------------
+{
+  "source_and_target": "{\"foo\": 2000}"
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+it will look like:
+
+[source,js]
+--------------------------------------------------
+{
+  "source_and_target": {
+    "foo": 2000
+  }
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+This illustrates that, unless it is explicitly named in the processor configuration, the `target_field`
+is the same field provided in the required `field` configuration.
+
+[[kv-processor]]
+=== KV Processor
+This processor helps automatically parse messages (or specific event fields) which are of the foo=bar variety.
+
+For example, if you have a log message which contains `ip=1.2.3.4 error=REFUSED`, you can parse those automatically by configuring:
+
+
+[source,js]
+--------------------------------------------------
+{
+  "kv": {
+    "field": "message",
+    "field_split": " ",
+    "value_split": "="
+  }
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+[[kv-options]]
+.Kv Options
+[options="header"]
+|======
+| Name             | Required  | Default  | Description
+| `field`          | yes       | -        | The field to be parsed
+| `field_split`    | yes       | -        | Regex pattern to use for splitting key-value pairs
+| `value_split`    | yes       | -        | Regex pattern to use for splitting the key from the value within a key-value pair
+| `target_field`   | no        | `null`   | The field to insert the extracted keys into. Defaults to the root of the document
+| `include_keys`   | no        | `null`   | List of keys to filter and insert into document. Defaults to including all keys
+| `exclude_keys`   | no        | `null`   | List of keys to exclude from document
+| `ignore_missing` | no        | `false`  | If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document
+|======
+
 
 [[lowercase-processor]]
 === Lowercase Processor
@@ -1278,8 +1661,10 @@ Converts a string to its lowercase equivalent.
 .Lowercase Options
 [options="header"]
 |======
-| Name     | Required  | Default  | Description
-| `field`  | yes       | -        | The field to make lowercase
+| Name             | Required  | Default  | Description
+| `field`          | yes       | -        | The field to make lowercase
+| `target_field`   | no        | `field`  | The field to assign the converted value to, by default `field` is updated in-place
+| `ignore_missing` | no        | `false`  | If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document
 |======
 
 [source,js]
@@ -1290,19 +1675,22 @@ Converts a string to its lowercase equivalent.
   }
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 [[remove-processor]]
 === Remove Processor
-Removes an existing field. If the field doesn't exist, an exception will be thrown.
+Removes existing fields. If one field doesn't exist, an exception will be thrown.
 
 [[remove-options]]
 .Remove Options
 [options="header"]
 |======
 | Name      | Required  | Default  | Description
-| `field`   | yes       | -        | The field to be removed
+| `field`   | yes       | -        | Fields to be removed
 |======
 
+Here is an example to remove a single field:
+
 [source,js]
 --------------------------------------------------
 {
@@ -1311,6 +1699,19 @@ Removes an existing field. If the field doesn't exist, an exception will be thro
   }
 }
 --------------------------------------------------
+// NOTCONSOLE
+
+To remove multiple fields, you can use the following query:
+
+[source,js]
+--------------------------------------------------
+{
+  "remove": {
+    "field": ["foo", "bar"]
+  }
+}
+--------------------------------------------------
+// NOTCONSOLE
 
 [[rename-processor]]
 === Rename Processor
@@ -1320,9 +1721,10 @@ Renames an existing field. If the field doesn't exist or the new name is already
 .Rename Options
 [options="header"]
 |======
-| Name           | Required  | Default  | Description
-| `field`        | yes       | -        | The field to be renamed
-| `target_field` | yes       | -        | The new name of the field
+| Name             | Required  | Default  | Description
+| `field`          | yes       | -        | The field to be renamed
+| `target_field`   | yes       | -        | The new name of the field
+| `ignore_missing` | no        | `false`  | If `true` and `field` does not exist, the processor quietly exits without modifying the document
 |======
 
 [source,js]
@@ -1334,6 +1736,7 @@ Renames an existing field. If the field doesn't exist or the new name is already
   }
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 [[script-processor]]
 === Script Processor
@@ -1350,15 +1753,16 @@ caching see <<modules-scripting-using-caching, Script Caching>>.
 .Script Options
 [options="header"]
 |======
-| Name                   | Required  | Default | Description
-| `field`                | yes       | -       | The field to set
-| `lang`                 | no        | -       | The scripting language
-| `file`                 | no        | -       | The script file to refer to
-| `id`                   | no        | -       | The stored script id to refer to
-| `inline`               | no        | -       | An inline script to be executed
-| `params`               | no        | -       | Script Parameters
+| Name                   | Required  | Default    | Description
+| `lang`                 | no        | "painless" | The scripting language
+| `file`                 | no        | -          | The script file to refer to
+| `id`                   | no        | -          | The stored script id to refer to
+| `source`               | no        | -          | An inline script to be executed
+| `params`               | no        | -          | Script Parameters
 |======
 
+One of `file`, `id`, or `source` options must be provided in order to properly reference a script to execute.
+
 You can access the current ingest document from within the script context by using the `ctx` variable.
 
 The following example sets a new field called `field_a_plus_b_times_c` to be the sum of two existing
@@ -1369,13 +1773,14 @@ numeric fields `field_a` and `field_b` multiplied by the parameter param_c:
 {
   "script": {
     "lang": "painless",
-    "inline": "ctx.field_a_plus_b_times_c = (ctx.field_a + ctx.field_b) * params.param_c",
+    "source": "ctx.field_a_plus_b_times_c = (ctx.field_a + ctx.field_b) * params.param_c",
     "params": {
       "param_c": 10
     }
   }
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 
 [[set-processor]]
@@ -1402,6 +1807,7 @@ its value will be replaced with the provided one.
   }
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 [[split-processor]]
 === Split Processor
@@ -1411,9 +1817,11 @@ Splits a field into an array using a separator character. Only works on string f
 .Split Options
 [options="header"]
 |======
-| Name        | Required  | Default  | Description
-| `field`     | yes       | -        | The field to split
-| `separator` | yes       | -        | A regex which matches the separator, eg `,` or `\s+`
+| Name              | Required  | Default  | Description
+| `field`           | yes       | -        | The field to split
+| `separator`       | yes       | -        | A regex which matches the separator, eg `,` or `\s+`
+| `target_field`    | no        | `field`  | The field to assign the split value to, by default `field` is updated in-place
+| `ignore_missing`  | no        | `false`  | If `true` and `field` does not exist, the processor quietly exits without modifying the document
 |======
 
 [source,js]
@@ -1425,6 +1833,7 @@ Splits a field into an array using a separator character. Only works on string f
   }
 }
 --------------------------------------------------
+// NOTCONSOLE
 <1> Treat all consecutive whitespace characters as a single separator
 
 [[sort-processor]]
@@ -1437,9 +1846,10 @@ Throws an error when the field is not an array.
 .Sort Options
 [options="header"]
 |======
-| Name    | Required  | Default  | Description
-| `field` | yes       | -        | The field to be sorted
-| `order` | no        | `"asc"`  | The sort order to use. Accepts `"asc"` or `"desc"`.
+| Name              | Required  | Default  | Description
+| `field`           | yes       | -        | The field to be sorted
+| `order`           | no        | `"asc"`  | The sort order to use. Accepts `"asc"` or `"desc"`.
+| `target_field`    | no        | `field`  | The field to assign the sorted value to, by default `field` is updated in-place
 |======
 
 [source,js]
@@ -1451,6 +1861,7 @@ Throws an error when the field is not an array.
   }
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 [[trim-processor]]
 === Trim Processor
@@ -1462,8 +1873,10 @@ NOTE: This only works on leading and trailing whitespace.
 .Trim Options
 [options="header"]
 |======
-| Name     | Required  | Default  | Description
-| `field`  | yes       | -        | The string-valued field to trim whitespace from
+| Name              | Required  | Default  | Description
+| `field`           | yes       | -        | The string-valued field to trim whitespace from
+| `target_field`    | no        | `field`  | The field to assign the trimmed value to, by default `field` is updated in-place
+| `ignore_missing`  | no        | `false`  | If `true` and `field` does not exist, the processor quietly exits without modifying the document
 |======
 
 [source,js]
@@ -1474,6 +1887,7 @@ NOTE: This only works on leading and trailing whitespace.
   }
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 [[uppercase-processor]]
 === Uppercase Processor
@@ -1483,8 +1897,10 @@ Converts a string to its uppercase equivalent.
 .Uppercase Options
 [options="header"]
 |======
-| Name     | Required  | Default  | Description
-| `field`  | yes       | -        | The field to make uppercase
+| Name             | Required  | Default  | Description
+| `field`          | yes       | -        | The field to make uppercase
+| `target_field`   | no        | `field`  | The field to assign the converted value to, by default `field` is updated in-place
+| `ignore_missing` | no        | `false`  | If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document
 |======
 
 [source,js]
@@ -1495,6 +1911,7 @@ Converts a string to its uppercase equivalent.
   }
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 [[dot-expand-processor]]
 === Dot Expander Processor
@@ -1520,6 +1937,7 @@ Otherwise these <<accessing-data-in-pipelines,fields> can't be accessed by any p
   }
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 For example the dot expand processor would turn this document:
 
@@ -1529,6 +1947,7 @@ For example the dot expand processor would turn this document:
   "foo.bar" : "value"
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 into:
 
@@ -1540,6 +1959,7 @@ into:
   }
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 If there is already a `bar` field nested under `foo` then
 this processor merges the the `foo.bar` field into it. If the field is
@@ -1556,6 +1976,7 @@ For example, the following document:
   }
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 is transformed by the `dot_expander` processor into:
 
@@ -1567,6 +1988,7 @@ is transformed by the `dot_expander` processor into:
   }
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 If any field outside of the leaf field conflicts with a pre-existing field of the same name,
 then that field needs to be renamed first.
@@ -1580,6 +2002,7 @@ Consider the following document:
   "foo.bar": "value2"
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 Then the the `foo` needs to be renamed first before the `dot_expander`
 processor is applied. So in order for the `foo.bar` field to properly
@@ -1604,6 +2027,7 @@ pipeline should be used:
   ]
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 The reason for this is that Ingest doesn't know how to automatically cast
-a scalar field to an object field.
\ No newline at end of file
+a scalar field to an object field.
diff --git a/docs/reference/mapping.asciidoc b/docs/reference/mapping.asciidoc
index 5507ba97ff65a..2e694e477f496 100644
--- a/docs/reference/mapping.asciidoc
+++ b/docs/reference/mapping.asciidoc
@@ -23,7 +23,9 @@ Each index has one or more _mapping types_, which are used to divide the
 documents in an index into logical groups. User documents might be stored in a
 `user` type, and blog posts in a `blogpost` type.
 
-Each mapping type has:
+deprecated[6.0.0,See <<removal-of-types>>].
+
+A mapping type has:
 
 <<mapping-fields,Meta-fields>>::
 
@@ -64,6 +66,37 @@ the <<analysis-standard-analyzer,`standard` analyzer>>, the
 This is the purpose of _multi-fields_.  Most datatypes support multi-fields
 via the <<multi-fields>> parameter.
 
+[[mapping-limit-settings]]
+[float]
+=== Settings to prevent mappings explosion
+
+Defining too many fields in an index is a condition that can lead to a
+mapping explosion, which can cause out of memory errors and difficult
+situations to recover from. This problem may be more common than expected.
+As an example, consider a situation in which every new document inserted
+introduces new fields. This is quite common with dynamic mappings.
+Every time a document contains new fields, those will end up in the index's
+mappings. This isn't worrying for a small amount of data, but it can become a
+problem as the mapping grows.
+The following settings allow you to limit the number of field mappings that
+can be created manually or dynamically, in order to prevent bad documents from
+causing a mapping explosion:
+
+`index.mapping.total_fields.limit`::
+    The maximum number of fields in an index. The default value is `1000`.
+
+`index.mapping.depth.limit`::
+    The maximum depth for a field, which is measured as the number of inner
+    objects. For instance, if all fields are defined at the root object level,
+    then the depth is `1`. If there is one object mapping, then the depth is
+    `2`, etc. The default is `20`.
+
+`index.mapping.nested_fields.limit`::
+    The maximum number of `nested` fields in an index, defaults to `50`.
+    Indexing 1 document with 100 nested fields actually indexes 101 documents
+    as each nested document is indexed as a separate hidden document.
+
+
 [float]
 == Dynamic mapping
 
@@ -94,7 +127,7 @@ fields to an existing index with the <<indices-put-mapping,PUT mapping API>>.
 Other than where documented, *existing type and field mappings cannot be
 updated*. Changing the mapping would mean invalidating already indexed
 documents.  Instead, you should create a new index with the correct mappings
-and reindex your data into that index.
+and <<docs-reindex,reindex>> your data into that index.
 
 [[field-conflicts]]
 [float]
@@ -166,6 +199,8 @@ PUT my_index <1>
 
 --
 
+include::mapping/removal_of_types.asciidoc[]
+
 include::mapping/types.asciidoc[]
 
 include::mapping/fields.asciidoc[]
diff --git a/docs/reference/mapping/dynamic/default-mapping.asciidoc b/docs/reference/mapping/dynamic/default-mapping.asciidoc
index 268ae4f29c7bd..3ca8403eeaca3 100644
--- a/docs/reference/mapping/dynamic/default-mapping.asciidoc
+++ b/docs/reference/mapping/dynamic/default-mapping.asciidoc
@@ -32,6 +32,11 @@ PUT my_index
 <2> The `user` type inherits the settings from `_default_`.
 <3> The `blogpost` type overrides the defaults and enables the <<mapping-all-field,`_all`>> field.
 
+NOTE: When updating the `_default_` mapping with the
+<<indices-put-mapping,PUT mapping>> API, the new mapping is not merged with
+the existing mapping.  Instead, the new `_default_` mapping replaces the
+existing one.
+
 While the `_default_` mapping can be updated after an index has been created,
 the new defaults will only affect mapping types that are created afterwards.
 
diff --git a/docs/reference/mapping/dynamic/field-mapping.asciidoc b/docs/reference/mapping/dynamic/field-mapping.asciidoc
index 020c5d24552d7..2b0afa1562d24 100644
--- a/docs/reference/mapping/dynamic/field-mapping.asciidoc
+++ b/docs/reference/mapping/dynamic/field-mapping.asciidoc
@@ -4,7 +4,8 @@
 By default, when a previously unseen field is found in a document,
 Elasticsearch will add the new field to the type mapping.   This behaviour can
 be disabled, both at the document and at the <<object,`object`>>  level, by
-setting the <<dynamic,`dynamic`>> parameter to `false` or to `strict`.
+setting the <<dynamic,`dynamic`>> parameter to `false` (to ignore new fields) or to `strict` (to throw
+an exception if an unknown field is encountered).
 
 Assuming `dynamic` field mapping is enabled, some simple rules are used to
 determine which datatype the field should have:
@@ -30,20 +31,6 @@ detected.  All other datatypes must be mapped explicitly.
 Besides the options listed below, dynamic field mapping rules can be further
 customised with <<dynamic-templates,`dynamic_templates`>>.
 
-[[mapping-limit-settings]]
-==== Settings to prevent mappings explosion
-
-Two settings allow to control mapping explosion, in order to prevent adversary
-documents to create huge mappings through dynamic mappings for instance:
-
-`index.mapping.total_fields.limit`::
-    The maximum number of fields in an index. The default value is `1000`.
-`index.mapping.depth.limit`::
-    The maximum depth for a field, which is measured as the number of nested
-    objects. For instance, if all fields are defined at the root object level,
-    then the depth is `1`. If there is one object mapping, then the depth is
-    `2`, etc. The default is `20`.
-
 [[date-detection]]
 ==== Date detection
 
diff --git a/docs/reference/mapping/dynamic/templates.asciidoc b/docs/reference/mapping/dynamic/templates.asciidoc
index 21b0eadf683ab..a839e761db9e4 100644
--- a/docs/reference/mapping/dynamic/templates.asciidoc
+++ b/docs/reference/mapping/dynamic/templates.asciidoc
@@ -32,6 +32,7 @@ Dynamic templates are specified as an array of named objects:
     ...
   ]
 --------------------------------------------------
+// NOTCONSOLE
 <1> The template name can be any string value.
 <2> The match conditions can include any of : `match_mapping_type`, `match`, `match_pattern`, `unmatch`, `path_match`, `path_unmatch`.
 <3> The mapping that the matched field should use.
@@ -94,7 +95,6 @@ PUT my_index/my_type/1
   "my_integer": 5, <1>
   "my_string": "Some string" <2>
 }
-
 --------------------------------------------------
 // CONSOLE
 <1> The `my_integer` field is mapped as an `integer`.
@@ -156,6 +156,7 @@ instead of simple wildcards, for instance:
   "match_pattern": "regex",
   "match": "^profit_\d+$"
 --------------------------------------------------
+// NOTCONSOLE
 
 [[path-match-unmatch]]
 ==== `path_match` and `path_unmatch`
@@ -282,6 +283,7 @@ PUT my_index
   }
 }
 --------------------------------------------------
+// CONSOLE
 
 ===== `text`-only mappings for strings
 
@@ -311,6 +313,7 @@ PUT my_index
   }
 }
 --------------------------------------------------
+// CONSOLE
 
 ===== Disabled norms
 
@@ -345,6 +348,7 @@ PUT my_index
   }
 }
 --------------------------------------------------
+// CONSOLE
 
 The sub `keyword` field appears in this template to be consistent with the
 default rules of dynamic mappings. Of course if you do not need them because
@@ -353,7 +357,7 @@ remove it as described in the previous section.
 
 ===== Time-series
 
-When doing time series analysis with elastisearch, it is common to have many
+When doing time series analysis with elasticsearch, it is common to have many
 numeric fields that you will often aggregate on but never filter on. In such a
 case, you could disable indexing on those fields to save disk space and also
 maybe gain some indexing speed:
@@ -388,6 +392,7 @@ PUT my_index
   }
 }
 --------------------------------------------------
+// CONSOLE
 <1> Like the default dynamic mapping rules, doubles are mapped as floats, which
     are usually accurate enough, yet require half the disk space.
 
diff --git a/docs/reference/mapping/fields/all-field.asciidoc b/docs/reference/mapping/fields/all-field.asciidoc
index 70fac4ed06800..c001d5aa428a7 100644
--- a/docs/reference/mapping/fields/all-field.asciidoc
+++ b/docs/reference/mapping/fields/all-field.asciidoc
@@ -86,6 +86,8 @@ requests>> (which is rewritten to a `query_string` query internally):
 --------------------------------
 GET _search?q=john+smith+1970
 --------------------------------
+// TEST[skip:_all is no longer allowed]
+// CONSOLE
 
 Other queries, such as the <<query-dsl-match-query,`match`>> and
 <<query-dsl-term-query,`term`>> queries require you to specify
diff --git a/docs/reference/mapping/fields/field-names-field.asciidoc b/docs/reference/mapping/fields/field-names-field.asciidoc
index 815606fb7bd3d..45839ac55d950 100644
--- a/docs/reference/mapping/fields/field-names-field.asciidoc
+++ b/docs/reference/mapping/fields/field-names-field.asciidoc
@@ -6,7 +6,7 @@ contains any value other than `null`.  This field is used by the
 <<query-dsl-exists-query,`exists`>> query to find documents that
 either have or don't have any non-+null+ value for a particular field.
 
-The value of the `_field_name` field is accessible in queries:
+The value of the `_field_names` field is accessible in queries:
 
 [source,js]
 --------------------------
@@ -34,4 +34,4 @@ GET my_index/_search
 --------------------------
 // CONSOLE
 
-<1> Querying on the `_field_names` field (also see the <<query-dsl-exists-query,`exists`>> query)
\ No newline at end of file
+<1> Querying on the `_field_names` field (also see the <<query-dsl-exists-query,`exists`>> query)
diff --git a/docs/reference/mapping/fields/index-field.asciidoc b/docs/reference/mapping/fields/index-field.asciidoc
index cd285c4255718..1cd6b05a93d7a 100644
--- a/docs/reference/mapping/fields/index-field.asciidoc
+++ b/docs/reference/mapping/fields/index-field.asciidoc
@@ -52,7 +52,7 @@ GET index_1,index_2/_search
     "index_name": {
       "script": {
         "lang": "painless",
-        "inline": "doc['_index']" <4>
+        "source": "doc['_index']" <4>
       }
     }
   }
diff --git a/docs/reference/mapping/fields/parent-field.asciidoc b/docs/reference/mapping/fields/parent-field.asciidoc
index eddf9ef3a25f9..8556842062eb5 100644
--- a/docs/reference/mapping/fields/parent-field.asciidoc
+++ b/docs/reference/mapping/fields/parent-field.asciidoc
@@ -59,16 +59,18 @@ See the <<query-dsl-has-child-query,`has_child`>> and
 the <<search-aggregations-bucket-children-aggregation,`children`>> aggregation,
 and <<parent-child-inner-hits,inner hits>> for more information.
 
-The value of the `_parent` field is accessible in queries, aggregations,
-and scripts:
+The value of the `_parent` field is accessible in aggregations
+and scripts, and may be queried with the
+<<query-dsl-parent-id-query, `parent_id` query>>:
 
 [source,js]
 --------------------------
 GET my_index/_search
 {
   "query": {
-    "terms": {
-      "_parent": [ "1" ] <1>
+    "parent_id": { <1>
+      "type": "my_child",
+      "id": "1"
     }
   },
   "aggs": {
@@ -82,8 +84,7 @@ GET my_index/_search
   "script_fields": {
     "parent": {
       "script": {
-         "lang": "painless",
-         "inline": "doc['_parent']" <3>
+         "source": "doc['_parent']" <3>
       }
     }
   }
@@ -92,7 +93,7 @@ GET my_index/_search
 // CONSOLE
 // TEST[continued]
 
-<1> Querying on the `_parent` field (also see the <<query-dsl-has-parent-query,`has_parent` query>> and the <<query-dsl-has-child-query,`has_child` query>>)
+<1> Querying the id of the `_parent` field (also see the <<query-dsl-has-parent-query,`has_parent` query>> and the <<query-dsl-has-child-query,`has_child` query>>)
 <2> Aggregating on the `_parent` field (also see the <<search-aggregations-bucket-children-aggregation,`children`>> aggregation)
 <3> Accessing the `_parent` field in scripts
 
@@ -120,11 +121,17 @@ Global ordinals need to be rebuilt after any change to a shard. The more
 parent id values are stored in a shard, the longer it takes to rebuild the
 global ordinals for the `_parent` field.
 
-Global ordinals, by default, are built lazily: the first parent-child query or
-aggregation after a refresh will trigger building of global ordinals. This can
-introduce a significant latency spike for your users. You can use
-<<global-ordinals,eager_global_ordinals>> to shift the cost of building global
-ordinals from query time to refresh time, by mapping the `_parent` field as follows:
+Global ordinals, by default, are built eagerly: if the index has changed,
+global ordinals for the `_parent` field will be rebuilt as part of the refresh.
+This can add significant time the refresh. However most of the times this is the
+right trade-off, otherwise global ordinals are rebuilt when the first parent-child
+query or aggregation is used. This can introduce a significant latency spike for
+your users and usually this is worse as multiple global ordinals for the `_parent`
+field may be attempt rebuilt within a single refresh interval when many writes
+are occurring.
+
+When the parent/child is used infrequently and writes occur frequently it may
+make sense to disable eager loading:
 
 [source,js]
 --------------------------------------------------
@@ -135,7 +142,7 @@ PUT my_index
     "my_child": {
       "_parent": {
         "type": "my_parent",
-        "eager_global_ordinals": true
+        "eager_global_ordinals": false
       }
     }
   }
diff --git a/docs/reference/mapping/fields/routing-field.asciidoc b/docs/reference/mapping/fields/routing-field.asciidoc
index bbdef3370c4ca..ebb659af344a0 100644
--- a/docs/reference/mapping/fields/routing-field.asciidoc
+++ b/docs/reference/mapping/fields/routing-field.asciidoc
@@ -109,3 +109,29 @@ documents with the same `_id` might end up on different shards if indexed with
 different `_routing` values.
 
 It is up to the user to ensure that IDs are unique across the index.
+
+[[routing-index-partition]]
+==== Routing to an index partition
+
+An index can be configured such that custom routing values will go to a subset of the shards rather
+than a single shard. This helps mitigate the risk of ending up with an imbalanced cluster while still
+reducing the impact of searches.
+
+This is done by providing the index level setting <<routing-partition-size,`index.routing_partition_size`>> at index creation.
+As the partition size increases, the more evenly distributed the data will become at the
+expense of having to search more shards per request.
+
+When this setting is present, the formula for calculating the shard becomes:
+
+    shard_num = (hash(_routing) + hash(_id) % routing_partition_size) % num_primary_shards
+
+That is, the `_routing` field is used to calculate a set of shards within the index and then the
+`_id` is used to pick a shard within that set.
+
+To enable this feature, the `index.routing_partition_size` should have a value greater than 1 and
+less than `index.number_of_shards`.
+
+Once enabled, the partitioned index will have the following limitations:
+
+*   Mappings with parent-child relationships cannot be created within it.
+*   All mappings within the index must have the `_routing` field marked as required.
\ No newline at end of file
diff --git a/docs/reference/mapping/fields/type-field.asciidoc b/docs/reference/mapping/fields/type-field.asciidoc
index a84b2508de478..453df4af5e4e4 100644
--- a/docs/reference/mapping/fields/type-field.asciidoc
+++ b/docs/reference/mapping/fields/type-field.asciidoc
@@ -1,6 +1,8 @@
 [[mapping-type-field]]
 === `_type` field
 
+deprecated[6.0.0,See <<removal-of-types>>]
+
 Each document indexed is associated with a <<mapping-type-field,`_type`>> (see
 <<mapping-type>>) and an <<mapping-id-field,`_id`>>.  The `_type` field is
 indexed in order to make searching by type name fast.
@@ -21,7 +23,7 @@ PUT my_index/type_2/2?refresh=true
   "text": "Document with type 2"
 }
 
-GET my_index/type_*/_search
+GET my_index/_search
 {
   "query": {
     "terms": {
@@ -47,7 +49,7 @@ GET my_index/type_*/_search
     "type": {
       "script": {
         "lang": "painless",
-        "inline": "doc['_type']" <4>
+        "source": "doc['_type']" <4>
       }
     }
   }
diff --git a/docs/reference/mapping/fields/uid-field.asciidoc b/docs/reference/mapping/fields/uid-field.asciidoc
index bad9eb4768b47..c6fe88067fc71 100644
--- a/docs/reference/mapping/fields/uid-field.asciidoc
+++ b/docs/reference/mapping/fields/uid-field.asciidoc
@@ -47,7 +47,7 @@ GET my_index/_search
     "UID": {
       "script": {
          "lang": "painless",
-         "inline": "doc['_uid']" <4>
+         "source": "doc['_uid']" <4>
       }
     }
   }
diff --git a/docs/reference/mapping/params.asciidoc b/docs/reference/mapping/params.asciidoc
index 5134ab6733a00..2422035623370 100644
--- a/docs/reference/mapping/params.asciidoc
+++ b/docs/reference/mapping/params.asciidoc
@@ -8,6 +8,7 @@ parameters that are used by <<mapping-types,field mappings>>:
 The following mapping parameters are common to some or all field datatypes:
 
 * <<analyzer,`analyzer`>>
+* <<normalizer, `normalizer`>>
 * <<mapping-boost,`boost`>>
 * <<coerce,`coerce`>>
 * <<copy-to,`copy_to`>>
@@ -15,15 +16,11 @@ The following mapping parameters are common to some or all field datatypes:
 * <<dynamic,`dynamic`>>
 * <<enabled,`enabled`>>
 * <<fielddata,`fielddata`>>
-* <<geohash,`geohash`>>
-* <<geohash-precision,`geohash_precision`>>
-* <<geohash-prefix,`geohash_prefix`>>
 * <<mapping-date-format,`format`>>
 * <<ignore-above,`ignore_above`>>
 * <<ignore-malformed,`ignore_malformed`>>
 * <<include-in-all,`include_in_all`>>
 * <<index-options,`index_options`>>
-* <<lat-lon,`lat_lon`>>
 * <<mapping-index,`index`>>
 * <<multi-fields,`fields`>>
 * <<norms,`norms`>>
@@ -38,6 +35,8 @@ The following mapping parameters are common to some or all field datatypes:
 
 include::params/analyzer.asciidoc[]
 
+include::params/normalizer.asciidoc[]
+
 include::params/boost.asciidoc[]
 
 include::params/coerce.asciidoc[]
@@ -54,12 +53,6 @@ include::params/fielddata.asciidoc[]
 
 include::params/format.asciidoc[]
 
-include::params/geohash.asciidoc[]
-
-include::params/geohash-precision.asciidoc[]
-
-include::params/geohash-prefix.asciidoc[]
-
 include::params/ignore-above.asciidoc[]
 
 include::params/ignore-malformed.asciidoc[]
@@ -70,8 +63,6 @@ include::params/index.asciidoc[]
 
 include::params/index-options.asciidoc[]
 
-include::params/lat-lon.asciidoc[]
-
 include::params/multi-fields.asciidoc[]
 
 include::params/norms.asciidoc[]
diff --git a/docs/reference/mapping/params/analyzer.asciidoc b/docs/reference/mapping/params/analyzer.asciidoc
index c075b6628055e..358be142373ca 100644
--- a/docs/reference/mapping/params/analyzer.asciidoc
+++ b/docs/reference/mapping/params/analyzer.asciidoc
@@ -60,13 +60,15 @@ PUT /my_index
   }
 }
 
-GET my_index/_analyze?field=text <3>
+GET my_index/_analyze <3>
 {
+  "field": "text",
   "text": "The quick Brown Foxes."
 }
 
-GET my_index/_analyze?field=text.english <4>
+GET my_index/_analyze <4>
 {
+  "field": "text.english",
   "text": "The quick Brown Foxes."
 }
 --------------------------------------------------
@@ -133,11 +135,7 @@ PUT my_index
       }
    }
 }
---------------------------------------------------
-// CONSOLE
 
-[source,js]
---------------------------------------------------
 PUT my_index/my_type/1
 {
    "title":"The Quick Brown Fox"
@@ -157,6 +155,7 @@ GET my_index/my_type/_search
    }
 }
 --------------------------------------------------
+// CONSOLE
 <1> `my_analyzer` analyzer which tokens all terms including stop words
 <2> `my_stop_analyzer` analyzer which removes stop words
 <3> `analyzer` setting that points to the `my_analyzer` analyzer which will be used at index time
diff --git a/docs/reference/mapping/params/doc-values.asciidoc b/docs/reference/mapping/params/doc-values.asciidoc
index d9d6bbc9afca3..afe325310c78c 100644
--- a/docs/reference/mapping/params/doc-values.asciidoc
+++ b/docs/reference/mapping/params/doc-values.asciidoc
@@ -44,8 +44,3 @@ PUT my_index
 <1> The `status_code` field has `doc_values` enabled by default.
 <2> The `session_id` has `doc_values` disabled, but can still be queried.
 
-TIP: The `doc_values` setting is allowed to have different settings for fields
-of the same name in the same index.  It can be disabled (set to `false`) on
-existing fields using the <<indices-put-mapping,PUT mapping API>>.
-
-
diff --git a/docs/reference/mapping/params/dynamic.asciidoc b/docs/reference/mapping/params/dynamic.asciidoc
index 985e5a63df3fe..f5a30773c7275 100644
--- a/docs/reference/mapping/params/dynamic.asciidoc
+++ b/docs/reference/mapping/params/dynamic.asciidoc
@@ -46,8 +46,11 @@ not.  It accepts three settings:
 
 [horizontal]
 `true`::    Newly detected fields are added to the mapping. (default)
-`false`::   Newly detected fields are ignored.  New fields must be added explicitly.
-`strict`::  If new fields are detected, an exception is thrown and the document is rejected.
+`false`::   Newly detected fields are ignored. These fields will not be indexed so will not be searchable 
+            but will still appear in the `_source` field of returned hits. These fields will not be added 
+            to the mapping,  new fields must be added explicitly.
+`strict`::  If new fields are detected, an exception is thrown and the document is rejected. New fields 
+            must be explicitly added to the mapping.
 
 The `dynamic` setting may be set at the mapping type level, and on each
 <<object,inner object>>.  Inner objects inherit the setting from their parent
diff --git a/docs/reference/mapping/params/fielddata.asciidoc b/docs/reference/mapping/params/fielddata.asciidoc
index 6795b0f5b9b4f..baca3c426d630 100644
--- a/docs/reference/mapping/params/fielddata.asciidoc
+++ b/docs/reference/mapping/params/fielddata.asciidoc
@@ -135,8 +135,7 @@ whenever a once new segment becomes visible.
 The loading time of global ordinals depends on the number of terms in a field,
 but in general it is low, since it source field data has already been loaded.
 The memory overhead of global ordinals is a small because it is very
-efficiently compressed. Eager loading of global ordinals can move the loading
-time from the first search request, to the refresh itself.
+efficiently compressed.
 
 *****************************************
 
diff --git a/docs/reference/mapping/params/format.asciidoc b/docs/reference/mapping/params/format.asciidoc
index 905a7ffe90a72..3619017141f63 100644
--- a/docs/reference/mapping/params/format.asciidoc
+++ b/docs/reference/mapping/params/format.asciidoc
@@ -147,17 +147,18 @@ The following tables lists all the defaults ISO formats supported:
 
 `date_hour` or `strict_date_hour`::
 
-    A formatter that combines a full date and two digit hour of day.
+    A formatter that combines a full date and two digit hour of day:
+    `yyyy-MM-dd'T'HH`.
 
 `date_hour_minute` or `strict_date_hour_minute`::
 
     A formatter that combines a full date, two digit hour of day, and two
-    digit minute of hour.
+    digit minute of hour: `yyyy-MM-dd'T'HH:mm`.
 
 `date_hour_minute_second` or `strict_date_hour_minute_second`::
 
     A formatter that combines a full date, two digit hour of day, two digit
-    minute of hour, and two digit second of minute.
+    minute of hour, and two digit second of minute: `yyyy-MM-dd'T'HH:mm:ss`.
 
 `date_hour_minute_second_fraction` or `strict_date_hour_minute_second_fraction`::
 
@@ -183,16 +184,17 @@ The following tables lists all the defaults ISO formats supported:
 
 `hour` or `strict_hour`::
 
-    A formatter for a two digit hour of day.
+    A formatter for a two digit hour of day: `HH`
 
 `hour_minute` or `strict_hour_minute`::
 
-    A formatter for a two digit hour of day and two digit minute of hour.
+    A formatter for a two digit hour of day and two digit minute of hour:
+    `HH:mm`.
 
 `hour_minute_second` or `strict_hour_minute_second`::
 
     A formatter for a two digit hour of day, two digit minute of hour, and two
-    digit second of minute.
+    digit second of minute: `HH:mm:ss`.
 
 `hour_minute_second_fraction` or `strict_hour_minute_second_fraction`::
 
@@ -258,27 +260,28 @@ The following tables lists all the defaults ISO formats supported:
 
 `weekyear` or `strict_weekyear`::
 
-    A formatter for a four digit weekyear.
+    A formatter for a four digit weekyear: `xxxx`.
 
 `weekyear_week` or `strict_weekyear_week`::
 
-    A formatter for a four digit weekyear and two digit week of weekyear.
+    A formatter for a four digit weekyear and two digit week of weekyear:
+    `xxxx-'W'ww`.
 
 `weekyear_week_day` or `strict_weekyear_week_day`::
 
     A formatter for a four digit weekyear, two digit week of weekyear, and one
-    digit day of week.
+    digit day of week: `xxxx-'W'ww-e`.
 
 `year` or `strict_year`::
 
-    A formatter for a four digit year.
+    A formatter for a four digit year: `yyyy`.
 
 `year_month` or `strict_year_month`::
 
-    A formatter for a four digit year and two digit month of year.
+    A formatter for a four digit year and two digit month of year: `yyyy-MM`.
 
 `year_month_day` or `strict_year_month_day`::
 
     A formatter for a four digit year, two digit month of year, and two digit
-    day of month.
+    day of month: `yyyy-MM-dd`.
 
diff --git a/docs/reference/mapping/params/geohash-precision.asciidoc b/docs/reference/mapping/params/geohash-precision.asciidoc
deleted file mode 100644
index 35020feea5f36..0000000000000
--- a/docs/reference/mapping/params/geohash-precision.asciidoc
+++ /dev/null
@@ -1,72 +0,0 @@
-[[geohash-precision]]
-=== `geohash_precision`
-
-deprecated[5.0.0, Will be removed in the next major version.]
-
-Geohashes are a form of lat/lon encoding which divides the earth up into
-a grid. Each cell in this grid is represented by a geohash string. Each
-cell in turn can be further subdivided into smaller cells which are
-represented by a longer string. So the longer the geohash, the smaller
-(and thus more accurate) the cell is.
-
-The `geohash_precision` setting controls the length of the geohash that is
-indexed when the <<geohash,`geohash`>> option is enabled, and the maximum
-geohash length when the <<geohash-prefix,`geohash_prefix`>> option is enabled.
-
-It accepts:
-
-* a number between 1 and 12 (default), which represents the length of the geohash.
-* a <<distance-units,distance>>, e.g. `1km`.
-
-If a distance is specified, it will be translated to the smallest
-geohash-length that will provide the requested resolution.
-
-For example, using this mapping:
-
-[source,js]
---------------------------------------------------
-PUT my_index
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "location": {
-          "type": "geo_point",
-          "geohash_prefix": true,
-          "geohash_precision": 6 <1>
-        }
-      }
-    }
-  }
-}
---------------------------------------------------
-// CONSOLE
-// TEST[warning:geo_point geohash_precision parameter is deprecated and will be removed in the next major release]
-// TEST[warning:geo_point geohash_prefix parameter is deprecated and will be removed in the next major release]
-// TEST[warning:geo_point geohash parameter is deprecated and will be removed in the next major release]
-
-You can issue this index and query:
-[source,js]
---------------------------------------------------
-PUT my_index/my_type/1?refresh
-{
-  "location": {
-    "lat": 41.12,
-    "lon": -71.34
-  }
-}
-
-GET my_index/_search?fielddata_fields=location.geohash
-{
-  "query": {
-    "term": {
-      "location.geohash": "drm3bt"
-    }
-  }
-}
-
---------------------------------------------------
-// CONSOLE
-// TEST[continued]
-
-<1> A `geohash_precision` of 6 equates to geohash cells of approximately 1.26km x 0.6km
diff --git a/docs/reference/mapping/params/geohash-prefix.asciidoc b/docs/reference/mapping/params/geohash-prefix.asciidoc
deleted file mode 100644
index 51dfc829947dc..0000000000000
--- a/docs/reference/mapping/params/geohash-prefix.asciidoc
+++ /dev/null
@@ -1,74 +0,0 @@
-[[geohash-prefix]]
-=== `geohash_prefix`
-
-deprecated[5.0.0, Will be removed in the next major version.]
-
-
-Geohashes are a form of lat/lon encoding which divides the earth up into
-a grid. Each cell in this grid is represented by a geohash string. Each
-cell in turn can be further subdivided into smaller cells which are
-represented by a longer string. So the longer the geohash, the smaller
-(and thus more accurate) the cell is.
-
-While the <<geohash,`geohash`>> option enables indexing the geohash that
-corresponds to the lat/lon point, at the specified
-<<geohash-precision,precision>>, the `geohash_prefix` option will also
-index all the enclosing cells as well.
-
-For instance, a geohash of `drm3btev3e86` will index all of the following
-terms: [ `d`, `dr`, `drm`, `drm3`, `drm3b`, `drm3bt`, `drm3bte`, `drm3btev`,
-`drm3btev3`, `drm3btev3e`, `drm3btev3e8`, `drm3btev3e86` ].
-
-The geohash prefixes can be used with the
-<<query-dsl-geohash-cell-query,`geohash_cell` query>> to find points within a
-particular geohash, or its neighbours:
-
-
-[source,js]
---------------------------------------------------
-PUT my_index
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "location": {
-          "type": "geo_point",
-          "geohash_prefix": true,
-          "geohash_precision": 6
-        }
-      }
-    }
-  }
-}
---------------------------------------------------
-// CONSOLE
-// TEST[warning:geo_point geohash_precision parameter is deprecated and will be removed in the next major release]
-// TEST[warning:geo_point geohash_prefix parameter is deprecated and will be removed in the next major release]
-// TEST[warning:geo_point geohash parameter is deprecated and will be removed in the next major release]
-
-[source,js]
---------------------------------------------------
-PUT my_index/my_type/1
-{
-  "location": {
-    "lat": 41.12,
-    "lon": -71.34
-  }
-}
-
-GET my_index/_search?fielddata_fields=location.geohash
-{
-  "query": {
-    "geohash_cell": {
-      "location": {
-        "lat": 41.02,
-        "lon": -71.48
-      },
-      "precision": 4, <1>
-      "neighbors": true <1>
-    }
-  }
-}
---------------------------------------------------
-// CONSOLE
-// TEST[continued]
diff --git a/docs/reference/mapping/params/geohash.asciidoc b/docs/reference/mapping/params/geohash.asciidoc
deleted file mode 100644
index 95f91fc6e86de..0000000000000
--- a/docs/reference/mapping/params/geohash.asciidoc
+++ /dev/null
@@ -1,78 +0,0 @@
-[[geohash]]
-=== `geohash`
-
-deprecated[5.0.0, Will be removed in the next major version.]
-
-Geohashes are a form of lat/lon encoding which divides the earth up into
-a grid. Each cell in this grid is represented by a geohash string. Each
-cell in turn can be further subdivided into smaller cells which are
-represented by a longer string. So the longer the geohash, the smaller
-(and thus more accurate) the cell is.
-
-Because geohashes are just strings, they can be stored in an inverted
-index like any other string, which makes querying them very efficient.
-
-If you enable the `geohash` option, a `geohash` ``sub-field'' will be indexed
-as, eg `.geohash`. The length of the geohash is controlled by the
-<<geohash-precision,`geohash_precision`>> parameter.
-
-If the <<geohash-prefix,`geohash_prefix`>> option is enabled, the `geohash`
-option will be enabled automatically.
-
-For example, with this mapping:
-
-[source,js]
---------------------------------------------------
-PUT my_index
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "location": {
-          "type": "geo_point", <1>
-          "geohash": true
-        }
-      }
-    }
-  }
-}
---------------------------------------------------
-// CONSOLE
-// TEST[warning:geo_point geohash parameter is deprecated and will be removed in the next major release]
-
-You can index a document and use this query:
-
-[source,js]
---------------------------------------------------
-PUT my_index/my_type/1?refresh
-{
-  "location": {
-    "lat": 41.12,
-    "lon": -71.34
-  }
-}
-
-GET my_index/_search?fielddata_fields=location.geohash <2>
-{
-  "query": {
-    "prefix": {
-      "location.geohash": "drm3b" <3>
-    }
-  }
-}
---------------------------------------------------
-// CONSOLE
-// TEST[continued]
-
-<1> A `location.geohash` field will be indexed for each geo-point.
-<2> The geohash can be retrieved with <<doc-values,`doc_values`>>.
-<3> A <<query-dsl-prefix-query,`prefix`>> query can find all geohashes which start with a particular prefix.
-
-[WARNING]
-============================================
-
-A `prefix` query on geohashes is expensive.  Instead, consider using the
-<<geohash-prefix,`geohash_prefix`>> to pay the expense once at index time
-instead of on every query.
-
-============================================
diff --git a/docs/reference/mapping/params/include-in-all.asciidoc b/docs/reference/mapping/params/include-in-all.asciidoc
index ade2db80d2cb0..87f580fb70721 100644
--- a/docs/reference/mapping/params/include-in-all.asciidoc
+++ b/docs/reference/mapping/params/include-in-all.asciidoc
@@ -2,7 +2,7 @@
 === `include_in_all`
 
 The `include_in_all` parameter provides per-field control over which fields
-are included in the <<mapping-all-field,`_all`>> field.  It defaults to `true`, unless <<mapping-index,`index`>> is set to `no`.
+are included in the <<mapping-all-field,`_all`>> field.  It defaults to `true`, unless <<mapping-index,`index`>> is set to `false`.
 
 This example demonstrates how to exclude the `date` field from the `_all` field:
 
@@ -29,6 +29,7 @@ PUT my_index
 }
 --------------------------------
 // CONSOLE
+// TEST[warning:field [include_in_all] is deprecated, as [_all] is deprecated, and will be disallowed in 6.0, use [copy_to] instead.]
 
 <1> The `title` and `content` fields will be included in the `_all` field.
 <2> The `date` field will not be included in the `_all` field.
@@ -70,6 +71,7 @@ PUT my_index
 }
 --------------------------------
 // CONSOLE
+// TEST[warning:field [include_in_all] is deprecated, as [_all] is deprecated, and will be disallowed in 6.0, use [copy_to] instead.]
 
 <1> All fields in `my_type` are excluded from `_all`.
 <2> The `author.first_name` and `author.last_name` fields are included in `_all`.
diff --git a/docs/reference/mapping/params/index.asciidoc b/docs/reference/mapping/params/index.asciidoc
index e097293d1422b..270c930e28444 100644
--- a/docs/reference/mapping/params/index.asciidoc
+++ b/docs/reference/mapping/params/index.asciidoc
@@ -4,3 +4,4 @@
 The `index` option controls whether field values are indexed. It accepts `true`
 or `false`. Fields that are not indexed are not queryable.
 
+NOTE: For the legacy mapping type <<string,`string`>> the `index` option only accepts legacy values `analyzed` (default, treat as full-text field), `not_analyzed` (treat as keyword field) and `no`.
diff --git a/docs/reference/mapping/params/lat-lon.asciidoc b/docs/reference/mapping/params/lat-lon.asciidoc
deleted file mode 100644
index 234c652c93272..0000000000000
--- a/docs/reference/mapping/params/lat-lon.asciidoc
+++ /dev/null
@@ -1,80 +0,0 @@
-[[lat-lon]]
-=== `lat_lon`
-
-deprecated[5.0.0, ????????]
-// https://github.com/elastic/elasticsearch/issues/19792
-
-<<geo-queries,Geo-queries>> are usually performed by plugging the value of
-each <<geo-point,`geo_point`>> field into a formula to determine whether it
-falls into the required area or not. Unlike most queries, the inverted index
-is not involved.
-
-Setting `lat_lon` to `true` causes the latitude and longitude values to be
-indexed as numeric fields (called `.lat` and `.lon`). These fields can be used
-by the <<query-dsl-geo-bounding-box-query,`geo_bounding_box`>> and
-<<query-dsl-geo-distance-query,`geo_distance`>> queries instead of
-performing in-memory calculations. So this mapping:
-
-[source,js]
---------------------------------------------------
-PUT my_index
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "location": {
-          "type": "geo_point",
-          "lat_lon": true <1>
-        }
-      }
-    }
-  }
-}
---------------------------------------------------
-// CONSOLE
-// TEST[warning:geo_point lat_lon parameter is deprecated and will be removed in the next major release]
-<1> Setting `lat_lon` to true indexes the geo-point in the `location.lat` and `location.lon` fields.
-
-With this document:
-
-[source,js]
---------------------------------------------------
-PUT my_index/my_type/1?refresh
-{
-  "location": {
-    "lat": 41.12,
-    "lon": -71.34
-  }
-}
---------------------------------------------------
-// CONSOLE
-// TEST[continued]
-
-Allows this query:
-
-[source,js]
---------------------------------------------------
-GET my_index/_search
-{
-  "query": {
-    "geo_distance": {
-      "location": {
-        "lat": 41,
-        "lon": -71
-      },
-      "distance": "50km",
-      "optimize_bbox": "indexed" <1>
-    }
-  }
-}
---------------------------------------------------
-// CONSOLE
-// TEST[continued]
-// TEST[warning:Deprecated field [optimize_bbox] used, replaced by [no replacement: `optimize_bbox` is no longer supported due to recent improvements]]
-<1> The `indexed` option tells the geo-distance query to use the inverted index instead of the in-memory calculation.
-
-Whether the in-memory or indexed operation performs better depends both on
-your dataset and on the types of queries that you are running.
-
-NOTE: The `lat_lon` option only makes sense for single-value `geo_point`
-fields. It will not work with arrays of geo-points.
diff --git a/docs/reference/mapping/params/normalizer.asciidoc b/docs/reference/mapping/params/normalizer.asciidoc
new file mode 100644
index 0000000000000..44a27dab7589d
--- /dev/null
+++ b/docs/reference/mapping/params/normalizer.asciidoc
@@ -0,0 +1,165 @@
+[[normalizer]]
+=== `normalizer`
+
+The `normalizer` property of <<keyword,`keyword`>> fields is similar to
+<<analyzer,`analyzer`>> except that it guarantees that the analysis chain
+produces a single token.
+
+The `normalizer` is applied prior to indexing the keyword, as well as at
+search-time when the `keyword` field is searched via a query parser such as
+the <<query-dsl-match-query,`match`>> query.
+
+[source,js]
+--------------------------------
+PUT index
+{
+  "settings": {
+    "analysis": {
+      "normalizer": {
+        "my_normalizer": {
+          "type": "custom",
+          "char_filter": [],
+          "filter": ["lowercase", "asciifolding"]
+        }
+      }
+    }
+  },
+  "mappings": {
+    "type": {
+      "properties": {
+        "foo": {
+          "type": "keyword",
+          "normalizer": "my_normalizer"
+        }
+      }
+    }
+  }
+}
+
+PUT index/type/1
+{
+  "foo": "BÀR"
+}
+
+PUT index/type/2
+{
+  "foo": "bar"
+}
+
+PUT index/type/3
+{
+  "foo": "baz"
+}
+
+POST index/_refresh
+
+GET index/_search
+{
+  "query": {
+    "match": {
+      "foo": "BAR"
+    }
+  }
+}
+--------------------------------
+// CONSOLE
+
+The above query matches documents 1 and 2 since `BÀR` is converted to `bar` at
+both index and query time.
+
+[source,js]
+----------------------------
+{
+  "took": $body.took,
+  "timed_out": false,
+  "_shards": {
+    "total": 5,
+    "successful": 5,
+    "skipped" : 0,
+    "failed": 0
+  },
+  "hits": {
+    "total": 2,
+    "max_score": 0.2876821,
+    "hits": [
+      {
+        "_index": "index",
+        "_type": "type",
+        "_id": "2",
+        "_score": 0.2876821,
+        "_source": {
+          "foo": "bar"
+        }
+      },
+      {
+        "_index": "index",
+        "_type": "type",
+        "_id": "1",
+        "_score": 0.2876821,
+        "_source": {
+          "foo": "BÀR"
+        }
+      }
+    ]
+  }
+}
+----------------------------
+// TESTRESPONSE[s/"took".*/"took": "$body.took",/]
+
+Also, the fact that keywords are converted prior to indexing also means that
+aggregations return normalized values:
+
+[source,js]
+----------------------------
+GET index/_search
+{
+  "size": 0,
+  "aggs": {
+    "foo_terms": {
+      "terms": {
+        "field": "foo"
+      }
+    }
+  }
+}
+----------------------------
+// CONSOLE
+// TEST[continued]
+
+returns
+
+[source,js]
+----------------------------
+{
+  "took": 43,
+  "timed_out": false,
+  "_shards": {
+    "total": 5,
+    "successful": 5,
+    "skipped" : 0,
+    "failed": 0
+  },
+  "hits": {
+    "total": 3,
+    "max_score": 0.0,
+    "hits": []
+  },
+  "aggregations": {
+    "foo_terms": {
+      "doc_count_error_upper_bound": 0,
+      "sum_other_doc_count": 0,
+      "buckets": [
+        {
+          "key": "bar",
+          "doc_count": 2
+        },
+        {
+          "key": "baz",
+          "doc_count": 1
+        }
+      ]
+    }
+  }
+}
+----------------------------
+// TESTRESPONSE[s/"took".*/"took": "$body.took",/]
diff --git a/docs/reference/mapping/params/similarity.asciidoc b/docs/reference/mapping/params/similarity.asciidoc
index 527e961960b72..0a5979c9d3272 100644
--- a/docs/reference/mapping/params/similarity.asciidoc
+++ b/docs/reference/mapping/params/similarity.asciidoc
@@ -3,7 +3,7 @@
 
 Elasticsearch allows you to configure a scoring algorithm or _similarity_ per
 field. The `similarity` setting provides a simple way of choosing a similarity
-algorithm other than the default TF/IDF, such as `BM25`.
+algorithm other than the default `BM25`, such as `TF/IDF`.
 
 Similarities are mostly useful for <<text,`text`>> fields, but can also apply
 to other field types.
@@ -15,15 +15,20 @@ similarities. For more details about this expert options, see the
 The only similarities which can be used out of the box, without any further
 configuration are:
 
+`BM25`::
+        The Okapi BM25 algorithm. The algorithm used by default in Elasticsearch and Lucene.
+        See {defguide}/pluggable-similarites.html[Pluggable Similarity Algorithms]
+        for more information.
+
 `classic`::
-        The Default TF/IDF algorithm used by Elasticsearch and
+        The TF/IDF algorithm which used to be the default in Elasticsearch and
         Lucene. See {defguide}/practical-scoring-function.html[Lucene’s Practical Scoring Function]
         for more information.
 
-`BM25`::
-        The Okapi BM25 algorithm.
-        See {defguide}/pluggable-similarites.html[Pluggable Similarity Algorithms]
-        for more information.
+`boolean`::
+        A simple boolean similarity, which is used when full-text ranking is not needed
+        and the score should only be based on whether the query terms match or not.
+        Boolean similarity gives terms a score equal to their query boost.
 
 
 The `similarity` can be set on the field level when a field is first created,
@@ -39,9 +44,13 @@ PUT my_index
         "default_field": { <1>
           "type": "text"
         },
-        "bm25_field": {
+        "classic_field": {
           "type": "text",
-          "similarity": "BM25" <2>
+          "similarity": "classic" <2>
+        },
+        "boolean_sim_field": {
+          "type": "text",
+          "similarity": "boolean" <3>
         }
       }
     }
@@ -49,6 +58,6 @@ PUT my_index
 }
 --------------------------------------------------
 // CONSOLE
-<1> The `default_field` uses the `classic` similarity (ie TF/IDF).
-<2> The `bm25_field` uses the `BM25` similarity.
-
+<1> The `default_field` uses the `BM25` similarity.
+<2> The `classic_field` uses the `classic` similarity (ie TF/IDF).
+<3> The `boolean_sim_field` uses the `boolean` similarity.
diff --git a/docs/reference/mapping/removal_of_types.asciidoc b/docs/reference/mapping/removal_of_types.asciidoc
new file mode 100644
index 0000000000000..006bc789f3084
--- /dev/null
+++ b/docs/reference/mapping/removal_of_types.asciidoc
@@ -0,0 +1,419 @@
+[[removal-of-types]]
+== Removal of mapping types
+
+IMPORTANT: Indices created in Elasticsearch 6.0.0 or later may only contain a
+single <<mapping-type,mapping type>>.  Indices created in 5.x with multiple
+mapping types will continue to function as before in Elasticsearch 6.x.
+Mapping types will be completely removed in Elasticsearch 7.0.0.
+
+[float]
+=== What are mapping types?
+
+Since the first release of Elasticsearch, each document has been stored in a
+single index and assigned a single mapping type.  A mapping type was used to
+represent the type of document or entity being indexed, for instance a
+`twitter` index might have a `user` type and a `tweet` type.
+
+Each mapping type could have its own fields, so the `user` type might have a
+`full_name` field, a `user_name` field, and an `email` field, while the
+`tweet` type could have a `content` field, a `tweeted_at` field and, like the
+`user` type, a `user_name` field.
+
+Each document had a `_type` meta-field containing the type name, and searches
+could be limited to one or more types by specifying the type name(s) in the
+URL:
+
+[source,js]
+----
+GET twitter/user,tweet/_search
+{
+  "query": {
+    "match": {
+      "user_name": "kimchy"
+    }
+  }
+}
+----
+// NOTCONSOLE
+
+The `_type` field was combined with the document's `_id` to generate a `_uid`
+field, so documents of different types with the same `_id` could exist in a
+single index.
+
+Mapping types were also used to establish a
+<<mapping-parent-field,parent-child relationship>>
+between documents, so documents of type `question` could be parents to
+documents of type `answer`.
+
+[float]
+=== Why are mapping types being removed?
+
+Initially, we spoke about an ``index'' being similar to a ``database'' in an
+SQL database, and a ``type'' being equivalent to a
+``table''.
+
+This was a bad analogy that led to incorrect assumptions. In an SQL database,
+tables are independent of each other.  The columns in one table have no
+bearing on columns with the same name in another table.  This is not the case
+for fields in a mapping type.
+
+In an Elasticsearch index, fields that have the same name in different mapping
+types are backed by the same Lucene field internally.  In other words, using
+the example above, the `user_name` field in the `user` type is stored in
+exactly the same field as the `user_name` field in the `tweet` type, and both
+`user_name` fields must have the same mapping (definition) in both types.
+
+This can lead to frustration when, for example, you want `deleted` to be a
+`date` field in one type and a `boolean` field in another type in the same
+index.
+
+On top of that, storing different entities that have few or no fields in
+common in the same index leads to sparse data and interferes with Lucene's
+ability to compress documents efficiently.
+
+For these reasons, we have decided to remove the concept of mapping types from
+Elasticsearch.
+
+[float]
+=== Alternatives to mapping types
+
+[float]
+==== Index per document type
+
+The first alternative is to have an index per document type.  Instead of
+storing tweets and users in a single `twitter` index, you could store tweets
+in the `tweets` index and users in the `user` index. Indices are completely
+independent of each other and so there will be no conflict of field types
+between indices.
+
+This approach has two benefits:
+
+* Data is more likely to be dense and so benefit from compression techniques
+  used in Lucene.
+
+* The term statistics used for scoring in full text search are more likely to
+  be accurate because all documents in the same index represent a single
+  entity.
+
+Each index can be sized appropriately for the number of documents it will
+contain: you can use a smaller number of primary shards for `users` and a
+larger number of primary shards for `tweets`.
+
+[float]
+==== Custom type field
+
+Of course, there is a limit to how many primary shards can exist in a cluster
+so you may not want to waste an entire shard for a collection of only a few
+thousand documents.  In this case, you can implement your own custom `type`
+field which will work in a similar way to the old `_type`.
+
+Let's take the `user`/`tweet` example above.  Originally, the workflow would
+have looked something like this:
+
+[source,js]
+----
+PUT twitter
+{
+  "mappings": {
+    "user": {
+      "properties": {
+        "name": { "type": "text" },
+        "user_name": { "type": "keyword" },
+        "email": { "type": "keyword" }
+      }
+    },
+    "tweet": {
+      "properties": {
+        "content": { "type": "text" },
+        "user_name": { "type": "keyword" },
+        "tweeted_at": { "type": "date" }
+      }
+    }
+  }
+}
+
+PUT twitter/user/kimchy
+{
+  "name": "Shay Banon",
+  "user_name": "kimchy",
+  "email": "shay@kimchy.com"
+}
+
+PUT twitter/tweet/1
+{
+  "user_name": "kimchy",
+  "tweeted_at": "2017-10-24T09:00:00Z",
+  "content": "Types are going away"
+}
+
+GET twitter/tweet/_search
+{
+  "query": {
+    "match": {
+      "user_name": "kimchy"
+    }
+  }
+}
+----
+// NOTCONSOLE
+
+You could achieve the same thing by adding a custom `type` field as follows:
+
+[source,js]
+----
+PUT twitter
+{
+  "mappings": {
+    "doc": {
+      "properties": {
+        "type": { "type": "keyword" }, <1>
+        "name": { "type": "text" },
+        "user_name": { "type": "keyword" },
+        "email": { "type": "keyword" },
+        "content": { "type": "text" },
+        "tweeted_at": { "type": "date" }
+      }
+    }
+  }
+}
+
+PUT twitter/doc/user-kimchy
+{
+  "type": "user", <1>
+  "name": "Shay Banon",
+  "user_name": "kimchy",
+  "email": "shay@kimchy.com"
+}
+
+PUT twitter/doc/tweet-1
+{
+  "type": "tweet", <1>
+  "user_name": "kimchy",
+  "tweeted_at": "2017-10-24T09:00:00Z",
+  "content": "Types are going away"
+}
+
+GET twitter/_search
+{
+  "query": {
+    "bool": {
+      "must": {
+        "match": {
+          "user_name": "kimchy"
+        }
+      },
+      "filter": {
+        "match": {
+          "type": "tweet" <1>
+        }
+      }
+    }
+  }
+}
+----
+// NOTCONSOLE
+<1> The explicit `type` field takes the place of the implicit `_type` field.
+
+[float]
+==== Parent/Child without mapping types
+
+Previously, a parent-child relationship was represented by making one mapping
+type the parent, and one or more other mapping types the children.  Without
+types, we can no longer use this syntax.  The parent-child feature will
+continue to function as before, except that the way of expressing the
+relationship between documents has been changed to use the new
+<<parent-join,`join` field>>.
+
+
+[float]
+=== Schedule for removal of mapping types
+
+This is a big change for our users, so we have tried to make it as painless as
+possible.  The change will roll out as follows:
+
+Elasticsearch 5.6.0::
+
+* Setting `index.mapping.single_type: true` on an index will enable the
+  single-type-per-index behaviour which will be enforced in 6.0.
+
+* The <<parent-join,`join` field>> replacement for parent-child is available
+  on indices created in 5.6.
+
+Elasticsearch 6.x::
+
+* Indices created in 5.x will continue to function in 6.x as they did in 5.x.
+
+* Indices created in 6.x only allow a single-type per index.  Any name
+  can be used for the type, but there can be only one.
+
+* The `_type` name can no longer be combined with the `_id` to form the `_uid`
+  field. The `_uid` field has become an alias for the `_id` field.
+
+* New indices no longer support the old-style of parent/child and should
+  use the <<parent-join,`join` field>> instead.
+
+* The `_default_` mapping type is deprecated.
+
+Elasticsearch 7.x::
+
+* The `type` parameter in URLs are optional.  For instance, indexing
+  a document no longer requires a document `type`.
+
+* The `GET|PUT _mapping` APIs support a query string parameter
+  (`include_type_name`) which indicates whether the body should include
+  a layer for the type name. It defaults to `true`. 7.x indices which
+  don't have an explicit type will use the dummy type name `_doc`.
+
+* The `_default_` mapping type is removed.
+
+Elasticsearch 8.x::
+
+* The `type` parameter is no longer supported in URLs.
+
+* The `include_type_name` parameter defaults to `false`.
+
+Elasticsearch 9.x::
+
+* The `include_in_type` parameter is removed.
+
+[float]
+=== Migrating multi-type indices to single-type
+
+The <<docs-reindex,Reindex API>> can be used to convert multi-type indices to
+single-type indices. The following examples can be used in Elasticsearch 5.6
+or Elasticsearch 6.x.  In 6.x, there is no need to specify
+`index.mapping.single_type` as that is the default.
+
+[float]
+==== Index per document type
+
+This first example splits our `twitter` index into a `tweets` index and a
+`users` index:
+
+[source,js]
+----
+PUT users
+{
+  "settings": {
+    "index.mapping.single_type": true
+  },
+  "mappings": {
+    "user": {
+      "properties": {
+        "name": {
+          "type": "text"
+        },
+        "user_name": {
+          "type": "keyword"
+        },
+        "email": {
+          "type": "keyword"
+        }
+      }
+    }
+  }
+}
+
+PUT tweets
+{
+  "settings": {
+    "index.mapping.single_type": true
+  },
+  "mappings": {
+    "tweet": {
+      "properties": {
+        "content": {
+          "type": "text"
+        },
+        "user_name": {
+          "type": "keyword"
+        },
+        "tweeted_at": {
+          "type": "date"
+        }
+      }
+    }
+  }
+}
+
+POST _reindex
+{
+  "source": {
+    "index": "twitter",
+    "type": "user"
+  },
+  "dest": {
+    "index": "users"
+  }
+}
+
+POST _reindex
+{
+  "source": {
+    "index": "twitter",
+    "type": "tweet"
+  },
+  "dest": {
+    "index": "tweets"
+  }
+}
+----
+// NOTCONSOLE
+
+[float]
+==== Custom type field
+
+This next example adds a custom `type` field and sets it to the value of the
+original `_type`.  It also adds the type to the `_id` in case there are any
+documents of different types which have conflicting IDs:
+
+[source,js]
+----
+PUT new_twitter
+{
+  "mappings": {
+    "doc": {
+      "properties": {
+        "type": {
+          "type": "keyword"
+        },
+        "name": {
+          "type": "text"
+        },
+        "user_name": {
+          "type": "keyword"
+        },
+        "email": {
+          "type": "keyword"
+        },
+        "content": {
+          "type": "text"
+        },
+        "tweeted_at": {
+          "type": "date"
+        }
+      }
+    }
+  }
+}
+
+
+POST _reindex
+{
+  "source": {
+    "index": "twitter"
+  },
+  "dest": {
+    "index": "new_twitter"
+  },
+  "script": {
+    "source": """
+      ctx._source.type = ctx._type;
+      ctx._id = ctx._type + '-' + ctx._id;
+      ctx._type = 'doc';
+    """
+  }
+}
+----
+// NOTCONSOLE
+
diff --git a/docs/reference/mapping/types.asciidoc b/docs/reference/mapping/types.asciidoc
index a05723cd1963c..a8465d1ba3cbd 100644
--- a/docs/reference/mapping/types.asciidoc
+++ b/docs/reference/mapping/types.asciidoc
@@ -8,10 +8,11 @@ document:
 === Core datatypes
 
 string::        <<text,`text`>> and <<keyword,`keyword`>>
-<<number>>::    `long`, `integer`, `short`, `byte`, `double`, `float`
+<<number>>::    `long`, `integer`, `short`, `byte`, `double`, `float`, `half_float`, `scaled_float`
 <<date>>::      `date`
 <<boolean>>::   `boolean`
 <<binary>>::    `binary`
+<<range>>::     `integer_range`, `float_range`, `long_range`, `double_range`, `date_range`
 
 [float]
 === Complex datatypes
@@ -43,6 +44,8 @@ Attachment datatype::
 
 <<percolator>>::    Accepts queries from the query-dsl
 
+<<parent-join>>::   Defines parent/child relation for documents within the same index
+
 [float]
 === Multi-fields
 
@@ -61,6 +64,8 @@ include::types/array.asciidoc[]
 
 include::types/binary.asciidoc[]
 
+include::types/range.asciidoc[]
+
 include::types/boolean.asciidoc[]
 
 include::types/date.asciidoc[]
@@ -87,7 +92,7 @@ include::types/token-count.asciidoc[]
 
 include::types/percolator.asciidoc[]
 
-
+include::types/parent-join.asciidoc[]
 
 
 
diff --git a/docs/reference/mapping/types/binary.asciidoc b/docs/reference/mapping/types/binary.asciidoc
index 7f82523416f14..556fc55b7f0f0 100644
--- a/docs/reference/mapping/types/binary.asciidoc
+++ b/docs/reference/mapping/types/binary.asciidoc
@@ -29,6 +29,7 @@ PUT my_index/my_type/1
   "blob": "U29tZSBiaW5hcnkgYmxvYg==" <1>
 }
 --------------------------------------------------
+// CONSOLE
 <1> The Base64 encoded binary value must not have embedded newlines `\n`.
 
 [[binary-params]]
@@ -49,5 +50,3 @@ The following parameters are accepted by `binary` fields:
     Whether the field value should be stored and retrievable separately from
     the <<mapping-source-field,`_source`>> field. Accepts `true` or `false`
     (default).
-
-
diff --git a/docs/reference/mapping/types/boolean.asciidoc b/docs/reference/mapping/types/boolean.asciidoc
index a5f559facc21d..62c12f0159e78 100644
--- a/docs/reference/mapping/types/boolean.asciidoc
+++ b/docs/reference/mapping/types/boolean.asciidoc
@@ -13,6 +13,10 @@ True values::
 
     Anything that isn't false.
 
+deprecated[5.1.0,While Elasticsearch will currently accept the above values during index time. Searching a boolean field using these pseudo-boolean values is deprecated. Please use "true" or "false" instead.]
+
+deprecated[5.3.0,Usage of any value other than `false`, `"false"`, `true` and `"true"` is deprecated.]
+
 For example:
 
 [source,js]
@@ -32,21 +36,21 @@ PUT my_index
 
 POST my_index/my_type/1
 {
-  "is_published": true <1>
+  "is_published": "true" <1>
 }
 
 GET my_index/_search
 {
   "query": {
     "term": {
-      "is_published": 1 <2>
+      "is_published": true <2>
     }
   }
 }
 --------------------------------------------------
 // CONSOLE
-<1> Indexing a document with a JSON `true`.
-<2> Querying for the document with `1`, which is interpreted as `true`.
+<1> Indexing a document with `"true"`, which is interpreted as `true`.
+<2> Searching for documents with a JSON `true`.
 
 Aggregations like the <<search-aggregations-bucket-terms-aggregation,`terms`
 aggregation>>  use `1` and `0` for the `key`, and the strings `"true"` and
@@ -78,7 +82,7 @@ GET my_index/_search
     "is_published": {
       "script": {
         "lang": "painless",
-        "inline": "doc['is_published'].value"
+        "source": "doc['is_published'].value"
       }
     }
   }
diff --git a/docs/reference/mapping/types/date.asciidoc b/docs/reference/mapping/types/date.asciidoc
index 0f3c8a7862ca5..691387ba8ab3a 100644
--- a/docs/reference/mapping/types/date.asciidoc
+++ b/docs/reference/mapping/types/date.asciidoc
@@ -74,7 +74,7 @@ PUT my_index
       "properties": {
         "date": {
           "type":   "date",
-          "format": "yyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis"
+          "format": "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis"
         }
       }
     }
@@ -106,6 +106,12 @@ The following parameters are accepted by `date` fields:
     The date format(s) that can be parsed.  Defaults to
     `strict_date_optional_time||epoch_millis`.
 
+`locale`::
+
+    The locale to use when parsing dates since months do not have the same names
+    and/or abbreviations in all languages. The default is the
+    https://docs.oracle.com/javase/8/docs/api/java/util/Locale.html#ROOT[`ROOT` locale],
+
 <<ignore-malformed,`ignore_malformed`>>::
 
     If `true`, malformed numbers are ignored. If `false` (default), malformed
diff --git a/docs/reference/mapping/types/geo-point.asciidoc b/docs/reference/mapping/types/geo-point.asciidoc
index 35001e5675f37..ccfc93e34c58e 100644
--- a/docs/reference/mapping/types/geo-point.asciidoc
+++ b/docs/reference/mapping/types/geo-point.asciidoc
@@ -5,8 +5,7 @@ Fields of type `geo_point` accept latitude-longitude pairs, which can be used:
 
 * to find geo-points within a <<query-dsl-geo-bounding-box-query,bounding box>>,
   within a certain <<query-dsl-geo-distance-query,distance>> of a central point,
-  within a <<query-dsl-geo-polygon-query,polygon>>, or within a
-  <<query-dsl-geohash-cell-query,geohash>> cell.
+  or within a <<query-dsl-geo-polygon-query,polygon>>.
 * to aggregate documents by <<search-aggregations-bucket-geohashgrid-aggregation,geographically>>
   or by <<search-aggregations-bucket-geodistance-aggregation,distance>> from a central point.
 * to integrate distance into a document's <<query-dsl-function-score-query,relevance score>>.
@@ -101,52 +100,28 @@ The following parameters are accepted by `geo_point` fields:
 
 [horizontal]
 
-<<geohash,`geohash`>>::
-
-    Should the geo-point also be indexed as a geohash in the `.geohash`
-    sub-field? Defaults to `false`, unless `geohash_prefix` is `true`.
-    deprecated[2.4]
-
-<<geohash-precision,`geohash_precision`>>::
-
-    The maximum length of the geohash to use for the `geohash` and
-    `geohash_prefix` options. deprecated[2.4]
-
-<<geohash-prefix,`geohash_prefix`>>::
-
-    Should the geo-point also be indexed as a geohash plus all its prefixes?
-    Defaults to `false`. deprecated[2.4]
-
 <<ignore-malformed,`ignore_malformed`>>::
 
     If `true`, malformed geo-points are ignored. If `false` (default),
     malformed geo-points throw an exception and reject the whole document.
 
-<<lat-lon,`lat_lon`>>::
-
-    Should the geo-point also be indexed as `.lat` and `.lon` sub-fields?
-    Accepts `true` and `false` (default). deprecated[2.3]
-
 ==== Using geo-points in scripts
 
 When accessing the value of a geo-point in a script, the value is returned as
 a `GeoPoint` object, which allows access to the `.lat` and `.lon` values
 respectively:
 
-
-[source,js]
+[source,painless]
 --------------------------------------------------
-geopoint = doc['location'].value;
-lat      = geopoint.lat;
-lon      = geopoint.lon;
+def geopoint = doc['location'].value;
+def lat      = geopoint.lat;
+def lon      = geopoint.lon;
 --------------------------------------------------
 
 For performance reasons, it is better to access the lat/lon values directly:
 
-[source,js]
+[source,painless]
 --------------------------------------------------
-lat      = doc['location'].lat;
-lon      = doc['location'].lon;
+def lat      = doc['location'].lat;
+def lon      = doc['location'].lon;
 --------------------------------------------------
-
-
diff --git a/docs/reference/mapping/types/geo-shape.asciidoc b/docs/reference/mapping/types/geo-shape.asciidoc
index 1f0c76e1b93ef..18ffdbcbc6363 100644
--- a/docs/reference/mapping/types/geo-shape.asciidoc
+++ b/docs/reference/mapping/types/geo-shape.asciidoc
@@ -156,16 +156,23 @@ cell right next to it -- even though the shape is very close to the point.
 
 [source,js]
 --------------------------------------------------
+PUT /example
 {
-    "properties": {
-        "location": {
-            "type": "geo_shape",
-            "tree": "quadtree",
-            "precision": "1m"
+    "mappings": {
+        "doc": {
+            "properties": {
+                "location": {
+                    "type": "geo_shape",
+                    "tree": "quadtree",
+                    "precision": "1m"
+                }
+            }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TESTSETUP
 
 This mapping maps the location field to the geo_shape type using the
 quad_tree implementation and a precision of 1m. Elasticsearch translates
@@ -175,7 +182,7 @@ this into a tree_levels setting of 26.
 ===== Performance considerations
 
 Elasticsearch uses the paths in the prefix tree as terms in the index
-and in queries. The higher the levels is (and thus the precision), the
+and in queries. The higher the level is (and thus the precision), the
 more terms are generated. Of course, calculating the terms, keeping them in
 memory, and storing them on disk all have a price. Especially with higher
 tree levels, indices can become extremely large even with a modest
@@ -240,6 +247,7 @@ API.
 
 [source,js]
 --------------------------------------------------
+POST /example/doc
 {
     "location" : {
         "type" : "point",
@@ -247,6 +255,7 @@ API.
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 [float]
 ===== http://geojson.org/geojson-spec.html#id3[LineString]
@@ -257,6 +266,7 @@ line.  Specifying more than two points creates an arbitrary path.
 
 [source,js]
 --------------------------------------------------
+POST /example/doc
 {
     "location" : {
         "type" : "linestring",
@@ -264,6 +274,7 @@ line.  Specifying more than two points creates an arbitrary path.
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 The above `linestring` would draw a straight line starting at the White
 House to the US Capitol Building.
@@ -277,6 +288,7 @@ closed).
 
 [source,js]
 --------------------------------------------------
+POST /example/doc
 {
     "location" : {
         "type" : "polygon",
@@ -286,12 +298,14 @@ closed).
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 The first array represents the outer boundary of the polygon, the other
 arrays represent the interior shapes ("holes"):
 
 [source,js]
 --------------------------------------------------
+POST /example/doc
 {
     "location" : {
         "type" : "polygon",
@@ -302,6 +316,8 @@ arrays represent the interior shapes ("holes"):
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[skip:https://github.com/elastic/elasticsearch/issues/23836]
 
 *IMPORTANT NOTE:* GeoJSON does not mandate a specific order for vertices thus ambiguous
 polygons around the dateline and poles are possible. To alleviate ambiguity
@@ -322,6 +338,7 @@ OGC standards to eliminate ambiguity resulting in a polygon that crosses the dat
 
 [source,js]
 --------------------------------------------------
+POST /example/doc
 {
     "location" : {
         "type" : "polygon",
@@ -332,6 +349,8 @@ OGC standards to eliminate ambiguity resulting in a polygon that crosses the dat
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[skip:https://github.com/elastic/elasticsearch/issues/23836]
 
 An `orientation` parameter can be defined when setting the geo_shape mapping (see <<geo-shape-mapping-options>>). This will define vertex
 order for the coordinate list on the mapped geo_shape field. It can also be overridden on each document.  The following is an example for
@@ -339,6 +358,7 @@ overriding the orientation on a document:
 
 [source,js]
 --------------------------------------------------
+POST /example/doc
 {
     "location" : {
         "type" : "polygon",
@@ -350,6 +370,8 @@ overriding the orientation on a document:
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[skip:https://github.com/elastic/elasticsearch/issues/23836]
 
 [float]
 ===== http://www.geojson.org/geojson-spec.html#id5[MultiPoint]
@@ -358,6 +380,7 @@ A list of geojson points.
 
 [source,js]
 --------------------------------------------------
+POST /example/doc
 {
     "location" : {
         "type" : "multipoint",
@@ -367,6 +390,7 @@ A list of geojson points.
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 [float]
 ===== http://www.geojson.org/geojson-spec.html#id6[MultiLineString]
@@ -375,6 +399,7 @@ A list of geojson linestrings.
 
 [source,js]
 --------------------------------------------------
+POST /example/doc
 {
     "location" : {
         "type" : "multilinestring",
@@ -386,6 +411,8 @@ A list of geojson linestrings.
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[skip:https://github.com/elastic/elasticsearch/issues/23836]
 
 [float]
 ===== http://www.geojson.org/geojson-spec.html#id7[MultiPolygon]
@@ -394,18 +421,20 @@ A list of geojson polygons.
 
 [source,js]
 --------------------------------------------------
+POST /example/doc
 {
     "location" : {
         "type" : "multipolygon",
         "coordinates" : [
             [ [[102.0, 2.0], [103.0, 2.0], [103.0, 3.0], [102.0, 3.0], [102.0, 2.0]] ],
-
             [ [[100.0, 0.0], [101.0, 0.0], [101.0, 1.0], [100.0, 1.0], [100.0, 0.0]],
               [[100.2, 0.2], [100.8, 0.2], [100.8, 0.8], [100.2, 0.8], [100.2, 0.2]] ]
         ]
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[skip:https://github.com/elastic/elasticsearch/issues/23836]
 
 [float]
 ===== http://geojson.org/geojson-spec.html#geometrycollection[Geometry Collection]
@@ -414,6 +443,7 @@ A collection of geojson geometry objects.
 
 [source,js]
 --------------------------------------------------
+POST /example/doc
 {
     "location" : {
         "type": "geometrycollection",
@@ -430,7 +460,8 @@ A collection of geojson geometry objects.
     }
 }
 --------------------------------------------------
-
+// CONSOLE
+// TEST[skip:https://github.com/elastic/elasticsearch/issues/23836]
 
 [float]
 ===== Envelope
@@ -441,6 +472,7 @@ bounding rectangle:
 
 [source,js]
 --------------------------------------------------
+POST /example/doc
 {
     "location" : {
         "type" : "envelope",
@@ -448,6 +480,8 @@ bounding rectangle:
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[skip:https://github.com/elastic/elasticsearch/issues/23836]
 
 [float]
 ===== Circle
@@ -457,6 +491,7 @@ point with a radius:
 
 [source,js]
 --------------------------------------------------
+POST /example/doc
 {
     "location" : {
         "type" : "circle",
@@ -465,6 +500,7 @@ point with a radius:
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 Note: The inner `radius` field is required. If not specified, then
 the units of the `radius` will default to `METERS`.
diff --git a/docs/reference/mapping/types/ip.asciidoc b/docs/reference/mapping/types/ip.asciidoc
index 4bcfe553c5c0a..77b892350fd28 100644
--- a/docs/reference/mapping/types/ip.asciidoc
+++ b/docs/reference/mapping/types/ip.asciidoc
@@ -34,7 +34,7 @@ GET my_index/_search
 }
 --------------------------------------------------
 // CONSOLE
-
+// TESTSETUP
 
 [[ip-params]]
 ==== Parameters for `ip` fields
@@ -94,6 +94,7 @@ GET my_index/_search
   }
 }
 --------------------------------------------------
+// CONSOLE
 
 or
 
@@ -108,6 +109,7 @@ GET my_index/_search
   }
 }
 --------------------------------------------------
+// CONSOLE
 
 Also beware that colons are special characters to the
 <<query-dsl-query-string-query,`query_string`>> query, so ipv6 addresses will
@@ -116,7 +118,7 @@ searched value:
 
 [source,js]
 --------------------------------------------------
-GET t/_search
+GET my_index/_search
 {
   "query": {
     "query_string" : {
@@ -125,3 +127,4 @@ GET t/_search
   }
 }
 --------------------------------------------------
+// CONSOLE
diff --git a/docs/reference/mapping/types/keyword.asciidoc b/docs/reference/mapping/types/keyword.asciidoc
index fa260bbeff64d..c5cb1b8641654 100644
--- a/docs/reference/mapping/types/keyword.asciidoc
+++ b/docs/reference/mapping/types/keyword.asciidoc
@@ -6,7 +6,7 @@ codes, zip codes or tags.
 
 They are typically used for filtering (_Find me all blog posts where
 ++status++ is ++published++_), for sorting, and for aggregations. Keyword
-fields are ony searchable by their exact value.
+fields are only searchable by their exact value.
 
 If you need to index full text content such as email bodies or product
 descriptions, it is likely that you should rather use a <<text,`text`>> field.
@@ -99,15 +99,16 @@ The following parameters are accepted by `keyword` fields:
     the <<mapping-source-field,`_source`>> field. Accepts `true` or `false`
     (default).
 
-<<search-analyzer,`search_analyzer`>>::
-
-    The <<analyzer,`analyzer`>> that should be used at search time on
-    <<mapping-index,`analyzed`>> fields. Defaults to the `analyzer` setting.
-
 <<similarity,`similarity`>>::
 
     Which scoring algorithm or _similarity_ should be used. Defaults
-    to `classic`, which uses TF/IDF.
+    to `BM25`.
+
+<<normalizer,`normalizer`>>::
+
+    experimental[]
+    How to pre-process the keyword prior to indexing. Defaults to `null`,
+    meaning the keyword is kept as-is.
 
 NOTE: Indexes imported from 2.x do not support `keyword`. Instead they will
 attempt to downgrade `keyword` into `string`. This allows you to merge modern
diff --git a/docs/reference/mapping/types/nested.asciidoc b/docs/reference/mapping/types/nested.asciidoc
index 8850763cdc844..6f93634f53f2f 100644
--- a/docs/reference/mapping/types/nested.asciidoc
+++ b/docs/reference/mapping/types/nested.asciidoc
@@ -42,6 +42,7 @@ would be transformed internally into a document that looks more like this:
   "user.last" :  [ "smith", "white" ]
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 The `user.first` and `user.last` fields are flattened into multi-value fields,
 and the association between `alice` and `white` is lost.  This document would
@@ -205,5 +206,5 @@ phase.  Instead, highlighting needs to be performed via
 
 Indexing a document with 100 nested fields actually indexes 101 documents as each nested
 document is indexed as a separate document. To safeguard against ill-defined mappings
-the number of nested fields that can be defined per index has been limited to 50. This
-default limit can be changed with the index setting `index.mapping.nested_fields.limit`.
+the number of nested fields that can be defined per index has been limited to 50. See
+<<mapping-limit-settings>>.
diff --git a/docs/reference/mapping/types/numeric.asciidoc b/docs/reference/mapping/types/numeric.asciidoc
index 6fdc0c806a4d8..48a74cd176034 100644
--- a/docs/reference/mapping/types/numeric.asciidoc
+++ b/docs/reference/mapping/types/numeric.asciidoc
@@ -39,6 +39,12 @@ PUT my_index
 --------------------------------------------------
 // CONSOLE
 
+NOTE: The `double`, `float` and `half_float` types consider that `-0.0` and
+`+0.0` are different values. As a consequence, doing a `term` query on
+`-0.0` will not match `+0.0` and vice-versa. Same is true for range queries:
+if the upper bound is `-0.0` then `+0.0` will not match, and if the lower
+bound is `+0.0` then `-0.0` will not match.
+
 ==== Which type should I use?
 
 As far as integer types (`byte`, `short`, `integer` and `long`) are concerned,
diff --git a/docs/reference/mapping/types/object.asciidoc b/docs/reference/mapping/types/object.asciidoc
index 202c05512fe14..5e55c0e0388e1 100644
--- a/docs/reference/mapping/types/object.asciidoc
+++ b/docs/reference/mapping/types/object.asciidoc
@@ -35,6 +35,7 @@ pairs, something like this:
   "manager.name.last":  "Smith"
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 An explicit mapping for the above document could look like this:
 
@@ -101,4 +102,3 @@ The following parameters are accepted by `object` fields:
 
 IMPORTANT: If you need to index arrays of objects instead of single objects,
 read <<nested>> first.
-
diff --git a/docs/reference/mapping/types/parent-join.asciidoc b/docs/reference/mapping/types/parent-join.asciidoc
new file mode 100644
index 0000000000000..26936c3f06e5c
--- /dev/null
+++ b/docs/reference/mapping/types/parent-join.asciidoc
@@ -0,0 +1,438 @@
+[[parent-join]]
+=== `join` datatype
+
+WARNING: `mapping.single_type` must be set to true in the index settings
+ in order to be able to use a `join` field. The `join` field is agnostic to types
+ and works only with single typed mapping.
+
+The `join` datatype is a special field that creates
+parent/child relation within documents of the same index.
+The `relations` section defines a set of possible relations within the documents,
+each relation being a parent name and a child name.
+A parent/child relation can be defined as follows:
+
+[source,js]
+--------------------------------------------------
+PUT my_index
+{
+  "settings": {
+    "mapping.single_type": true
+  },
+  "mappings": {
+    "doc": {
+      "properties": {
+        "my_join_field": { <1>
+          "type": "join",
+          "relations": {
+            "question": "answer" <2>
+          }
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+
+<1> The name for the field
+<2> Defines a single relation where `question` is parent of `answer`.
+
+To index a document with a join, the name of the relation and the optional parent
+of the document must be provided in the `source`.
+For instance the following example creates two `parent` documents in the `question` context:
+
+[source,js]
+--------------------------------------------------
+PUT my_index/doc/1?refresh
+{
+  "text": "This is a question",
+  "my_join_field": {
+    "name": "question" <1>
+  }
+}
+
+PUT my_index/doc/2?refresh
+{
+  "text": "This is a another question",
+  "my_join_field": {
+    "name": "question"
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+<1> This document is a `question` document.
+
+When indexing parent documents, you can choose to specify just the name of the relation
+as a shortcut instead of encapsulating it in the normal object notation:
+
+[source,js]
+--------------------------------------------------
+PUT my_index/doc/1?refresh
+{
+  "text": "This is a question",
+  "my_join_field": "question" <1>
+}
+
+PUT my_index/doc/2?refresh
+{
+  "text": "This is another question",
+  "my_join_field": "question"
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+<1> This document is a `question` document.
+
+When indexing a child, the name of the relation as well as the parent id of the document
+must be added in the `_source`.
+
+WARNING: It is required to index the lineage of a parent in the same shard so you must
+always route child documents using their greater parent id.
+
+For instance the following example shows how to index two `child` documents:
+
+[source,js]
+--------------------------------------------------
+PUT my_index/doc/3?routing=1&refresh <1>
+{
+  "text": "This is an answer",
+  "my_join_field": {
+    "name": "answer", <2>
+    "parent": "1" <3>
+  }
+}
+
+PUT my_index/doc/4?routing=1&refresh
+{
+  "text": "This is another answer",
+  "my_join_field": {
+    "name": "answer",
+    "parent": "1"
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+<1> The routing value is mandatory because parent and child documents must be indexed on the same shard
+<2> `answer` is the name of the join for this document
+<3> The parent id of this child document
+
+==== Parent-join restrictions
+
+* Only one `join` field mapping is allowed per index.
+* Parent and child documents must be indexed on the same shard.
+  This means that the same `routing` value needs to be provided when
+  <<docs-get,getting>>, <<docs-delete,deleting>>, or <<docs-update,updating>>
+  a child document.
+* An element can have multiple children but only one parent.
+* It is possible to add a new relation to an existing `join` field.
+* It is also possible to add a child to an existing element
+  but only if the element is already a parent.
+
+==== Searching with parent-join
+
+The parent-join creates one field to index the name of the relation
+within the document (`my_parent`, `my_child`, ...).
+
+It also creates one field per parent/child relation.
+The name of this field is the name of the `join` field followed by `#` and the
+name of the parent in the relation.
+So for instance for the `my_parent` => [`my_child`, `another_child`] relation,
+the `join` field creates an additional field named `my_join_field#my_parent`.
+
+This field contains the parent `_id` that the document links to
+if the document is a child (`my_child` or `another_child`) and the `_id` of
+document if it's a parent (`my_parent`).
+
+When searching an index that contains a `join` field, these two fields are always
+returned in the search response:
+
+[source,js]
+--------------------------
+GET my_index/_search
+{
+  "query": {
+    "match_all": {}
+  },
+  "sort": ["_id"]
+}
+--------------------------
+// CONSOLE
+// TEST[continued]
+
+Will return:
+
+[source,js]
+--------------------------------------------------
+{
+    ...,
+    "hits": {
+        "total": 4,
+        "max_score": null,
+        "hits": [
+            {
+                "_index": "my_index",
+                "_type": "doc",
+                "_id": "1",
+                "_score": null,
+                "_source": {
+                    "text": "This is a question",
+                    "my_join_field": "question" <1>
+                },
+                "sort": [
+                    "1"
+                ]
+            },
+            {
+                "_index": "my_index",
+                "_type": "doc",
+                "_id": "2",
+                "_score": null,
+                "_source": {
+                    "text": "This is another question",
+                    "my_join_field": "question" <2>
+                },
+                "sort": [
+                    "2"
+                ]
+            },
+            {
+                "_index": "my_index",
+                "_type": "doc",
+                "_id": "3",
+                "_score": null,
+                "_routing": "1",
+                "_source": {
+                    "text": "This is an answer",
+                    "my_join_field": {
+                        "name": "answer", <3>
+                        "parent": "1"  <4>
+                    }
+                },
+                "sort": [
+                    "3"
+                ]
+            },
+            {
+                "_index": "my_index",
+                "_type": "doc",
+                "_id": "4",
+                "_score": null,
+                "_routing": "1",
+                "_source": {
+                    "text": "This is another answer",
+                    "my_join_field": {
+                        "name": "answer",
+                        "parent": "1"
+                    }
+                },
+                "sort": [
+                    "4"
+                ]
+            }
+        ]
+    }
+}
+--------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"timed_out": false, "took": $body.took, "_shards": $body._shards/]
+
+<1> This document belongs to the `question` join
+<2> This document belongs to the `question` join
+<3> This document belongs to the `answer` join
+<4> The linked parent id for the child document
+
+==== Parent-join queries and aggregations
+
+See the <<query-dsl-has-child-query,`has_child`>> and
+<<query-dsl-has-parent-query,`has_parent`>> queries,
+the <<search-aggregations-bucket-children-aggregation,`children`>> aggregation,
+and <<parent-child-inner-hits,inner hits>> for more information.
+
+The value of the `join` field is accessible in aggregations
+and scripts, and may be queried with the
+<<query-dsl-parent-id-query, `parent_id` query>>:
+
+[source,js]
+--------------------------
+GET my_index/_search
+{
+  "query": {
+    "parent_id": { <1>
+      "type": "answer",
+      "id": "1"
+    }
+  },
+  "aggs": {
+    "parents": {
+      "terms": {
+        "field": "my_join_field#question", <2>
+        "size": 10
+      }
+    }
+  },
+  "script_fields": {
+    "parent": {
+      "script": {
+         "source": "doc['my_join_field#question']" <3>
+      }
+    }
+  }
+}
+--------------------------
+// CONSOLE
+// TEST[continued]
+
+<1> Querying the `parent id` field (also see the <<query-dsl-has-parent-query,`has_parent` query>> and the <<query-dsl-has-child-query,`has_child` query>>)
+<2> Aggregating on the `parent id` field (also see the <<search-aggregations-bucket-children-aggregation,`children`>> aggregation)
+<3> Accessing the parent id` field in scripts
+
+
+==== Global ordinals
+
+The `join` field uses <<global-ordinals,global ordinals>> to speed up joins.
+Global ordinals need to be rebuilt after any change to a shard. The more
+parent id values are stored in a shard, the longer it takes to rebuild the
+global ordinals for the `join` field.
+
+Global ordinals, by default, are built eagerly: if the index has changed,
+global ordinals for the `join` field will be rebuilt as part of the refresh.
+This can add significant time to the refresh. However most of the times this is the
+right trade-off, otherwise global ordinals are rebuilt when the first parent-join
+query or aggregation is used. This can introduce a significant latency spike for
+your users and usually this is worse as multiple global ordinals for the `join`
+field may be attempt rebuilt within a single refresh interval when many writes
+are occurring.
+
+When the `join` field is used infrequently and writes occur frequently it may
+make sense to disable eager loading:
+
+[source,js]
+--------------------------------------------------
+PUT my_index
+{
+  "settings": {
+    "mapping.single_type": true
+  },
+  "mappings": {
+    "doc": {
+      "properties": {
+        "my_join_field": {
+          "type": "join",
+          "relations": {
+             "question": "answer"
+          },
+          "eager_global_ordinals": false
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+
+The amount of heap used by global ordinals can be checked per parent relation
+as follows:
+
+[source,sh]
+--------------------------------------------------
+# Per-index
+GET _stats/fielddata?human&fields=my_join_field#question
+
+# Per-node per-index
+GET _nodes/stats/indices/fielddata?human&fields=my_join_field#question
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+==== Multiple levels of parent join
+
+It is also possible to define multiple children for a single parent:
+
+[source,js]
+--------------------------------------------------
+PUT my_index
+{
+  "settings": {
+    "mapping.single_type": true
+  },
+  "mappings": {
+    "doc": {
+      "properties": {
+        "my_join_field": {
+          "type": "join",
+          "relations": {
+            "question": ["answer", "comment"]  <1>
+          }
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+
+<1> `question` is parent of `answer` and `comment`.
+
+And multiple levels of parent/child:
+
+[source,js]
+--------------------------------------------------
+PUT my_index
+{
+  "settings": {
+    "mapping.single_type": true
+  },
+  "mappings": {
+    "doc": {
+      "properties": {
+        "my_join_field": {
+          "type": "join",
+          "relations": {
+            "question": ["answer", "comment"],  <1>
+            "answer": "vote" <2>
+          }
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+
+<1> `question` is parent of `answer` and `comment`
+<2> `answer` is parent of `vote`
+
+The mapping above represents the following tree:
+
+                         question
+                          /    \
+                         /      \
+                      comment  answer
+                                 |
+                                 |
+                                vote
+
+Indexing a grand child document requires a `routing` value equals
+to the grand-parent (the greater parent of the lineage):
+
+
+[source,js]
+--------------------------------------------------
+PUT my_index/doc/3?routing=1&refresh <1>
+{
+  "text": "This is a vote",
+  "my_join_field": {
+    "name": "vote",
+    "parent": "2" <2>
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+<1> This child document must be on the same shard than its grandparent and parent
+<2> The parent id of this document (must points to an `answer` document)
diff --git a/docs/reference/mapping/types/percolator.asciidoc b/docs/reference/mapping/types/percolator.asciidoc
index ca8c8386e9bc8..4915a1970c947 100644
--- a/docs/reference/mapping/types/percolator.asciidoc
+++ b/docs/reference/mapping/types/percolator.asciidoc
@@ -15,27 +15,39 @@ If the following mapping configures the `percolator` field type for the
 
 [source,js]
 --------------------------------------------------
+PUT my_index
 {
-    "properties": {
-        "query": {
-            "type": "percolator"
+    "mappings": {
+        "doc": {
+            "properties": {
+                "query": {
+                    "type": "percolator"
+                },
+                "field": {
+                    "type": "text"
+                }
+            }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TESTSETUP
 
-Then the following json snippet can be indexed as a native query:
+Then you can index a query:
 
 [source,js]
 --------------------------------------------------
+PUT my_index/doc/match_value
 {
     "query" : {
-		"match" : {
-			"field" : "value"
-		}
-	}
+        "match" : {
+            "field" : "value"
+        }
+    }
 }
 --------------------------------------------------
+// CONSOLE
 
 [IMPORTANT]
 =====================================
@@ -45,9 +57,6 @@ associated with the index used for percolation. In order to make sure these fiel
 add or update a mapping via the <<indices-create-index,create index>> or <<indices-put-mapping,put mapping>> APIs.
 Fields referred in a percolator query may exist in any type of the index containing the `percolator` field type.
 
-Also an index can only contain up to one percolator field mapping. Multiple percolator fields will be rejected by the
-put index and put mapping APIs.
-
 =====================================
 
 [float]
@@ -73,13 +82,27 @@ fail.
 [float]
 ==== Limitations
 
+[float]
+===== Parent/child
+
 Because the `percolate` query is processing one document at a time, it doesn't support queries and filters that run
 against child documents such as `has_child` and `has_parent`.
 
-The percolator doesn't accepts percolator queries containing `range` queries with ranges that are based on current
-time (using `now`).
+[float]
+===== Fetching queries
 
 There are a number of queries that fetch data via a get call during query parsing. For example the `terms` query when
 using terms lookup, `template` query when using indexed scripts and `geo_shape` when using pre-indexed shapes. When these
 queries are indexed by the `percolator` field type then the get call is executed once. So each time the `percolator`
-query evaluates these queries, the fetches terms, shapes etc. as the were upon index time will be used.
\ No newline at end of file
+query evaluates these queries, the fetches terms, shapes etc. as the were upon index time will be used. Important to note
+is that fetching of terms that these queries do, happens both each time the percolator query gets indexed on both primary
+and replica shards, so the terms that are actually indexed can be different between shard copies, if the source index
+changed while indexing.
+
+[float]
+===== Script query
+
+The script inside a `script` query can only access doc values fields. The `percolate` query indexes the provided document
+into an in-memory index. This in-memory index doesn't support stored fields and because of that the `_source` field and
+other stored fields are not stored. This is the reason why in the `script` query the `_source` and other stored fields
+aren't available.
diff --git a/docs/reference/mapping/types/range.asciidoc b/docs/reference/mapping/types/range.asciidoc
new file mode 100644
index 0000000000000..69720dd930d1f
--- /dev/null
+++ b/docs/reference/mapping/types/range.asciidoc
@@ -0,0 +1,148 @@
+[[range]]
+=== Range datatypes
+
+The following range types are supported:
+
+[horizontal]
+`integer_range`::   A range of signed 32-bit integers with a minimum value of +-2^31^+ and maximum of +2^31^-1+.
+`float_range`::     A range of single-precision 32-bit IEEE 754 floating point values.
+`long_range`::      A range of signed 64-bit integers with a minimum value of +-2^63^+ and maximum of +2^63^-1+.
+`double_range`::    A range of double-precision 64-bit IEEE 754 floating point values.
+`date_range`::      A range of date values represented as unsigned 64-bit integer milliseconds elapsed since system epoch.
+`ip_range` ::       A range of ip values supporting either https://en.wikipedia.org/wiki/IPv4[IPv4] or
+                    https://en.wikipedia.org/wiki/IPv6[IPv6] (or mixed) addresses.
+
+Below is an example of configuring a mapping with various range fields followed by an example that indexes several range types.
+
+[source,js]
+--------------------------------------------------
+PUT range_index
+{
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "expected_attendees": {
+          "type": "integer_range"
+        },
+        "time_frame": {
+          "type": "date_range", <1>
+          "format": "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis"
+        }
+      }
+    }
+  }
+}
+
+PUT range_index/my_type/1
+{
+  "expected_attendees" : { <2>
+    "gte" : 10,
+    "lte" : 20
+  },
+  "time_frame" : { <3>
+    "gte" : "2015-10-31 12:00:00", <4>
+    "lte" : "2015-11-01"
+  }
+}
+--------------------------------------------------
+//CONSOLE
+
+The following is an example of a `date_range` query over the `date_range` field named "time_frame".
+
+[source,js]
+--------------------------------------------------
+POST range_index/_search
+{
+  "query" : {
+    "range" : {
+      "time_frame" : { <5>
+        "gte" : "2015-10-31",
+        "lte" : "2015-11-01",
+        "relation" : "within" <6>
+      }
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:range_index]
+
+The result produced by the above query.
+
+[source,js]
+--------------------------------------------------
+{
+  "took": 13,
+  "timed_out": false,
+  "_shards" : {
+    "total": 2,
+    "successful": 2,
+    "skipped" : 0,
+    "failed": 0
+  },
+  "hits" : {
+    "total" : 1,
+    "max_score" : 1.0,
+    "hits" : [
+      {
+        "_index" : "range_index",
+        "_type" : "my_type",
+        "_id" : "1",
+        "_score" : 1.0,
+        "_source" : {
+          "expected_attendees" : {
+            "gte" : 10, "lte" : 20
+          },
+          "time_frame" : {
+            "gte" : "2015-10-31 12:00:00", "lte" : "2015-11-01"
+          }
+        }
+      }
+    ]
+  }
+}
+--------------------------------------------------
+// TESTRESPONSE[s/"took": 13/"took" : $body.took/]
+
+<1> `date_range` types accept the same field parameters defined by the <<date, `date`>> type.
+<2> Example indexing a meeting with 10 to 20 attendees.
+<3> Date ranges accept the same format as described in <<ranges-on-dates, date range queries>>.
+<4> Example date range using date time stamp. This also accepts <<date-math, date math>> formatting, or "now" for system time.
+<5> Range queries work the same as described in <<query-dsl-range-query, range query>>.
+<6> Range queries over range <<mapping-types, fields>> support a `relation` parameter which can be one of `WITHIN`, `CONTAINS`,
+    `INTERSECTS` (default).
+
+[[range-params]]
+==== Parameters for range fields
+
+The following parameters are accepted by range types:
+
+[horizontal]
+
+<<coerce,`coerce`>>::
+
+    Try to convert strings to numbers and truncate fractions for integers.
+    Accepts `true` (default) and `false`.
+
+<<mapping-boost,`boost`>>::
+
+    Mapping field-level query time boosting. Accepts a floating point number, defaults
+    to `1.0`.
+
+<<include-in-all,`include_in_all`>>::
+
+    Whether or not the field value should be included in the
+    <<mapping-all-field,`_all`>> field? Accepts `true` or `false`.  Defaults
+    to `false` if <<mapping-index,`index`>> is set to `false`, or if a parent
+    <<object,`object`>> field sets `include_in_all` to `false`.
+    Otherwise defaults to `true`.
+
+<<mapping-index,`index`>>::
+
+    Should the field be searchable? Accepts `true` (default) and `false`.
+
+<<mapping-store,`store`>>::
+
+    Whether the field value should be stored and retrievable separately from
+    the <<mapping-source-field,`_source`>> field. Accepts `true` or `false`
+    (default).
diff --git a/docs/reference/mapping/types/string.asciidoc b/docs/reference/mapping/types/string.asciidoc
index b6a5fef86eb54..c4be060cc40b5 100644
--- a/docs/reference/mapping/types/string.asciidoc
+++ b/docs/reference/mapping/types/string.asciidoc
@@ -12,7 +12,7 @@ to use `text` or `keyword`.
 
 Indexes imported from 2.x *only* support `string` and not `text` or `keyword`.
 To ease the migration from 2.x Elasticsearch will downgrade `text` and `keyword`
-mappings applied to indexes imported to 2.x into `string`. While long lived
+mappings applied to indexes imported from 2.x into `string`. While long lived
 indexes will eventually need to be recreated against 5.x before eventually
 upgrading to 6.x, this downgrading smooths the process before you find time for
 it.
diff --git a/docs/reference/mapping/types/text.asciidoc b/docs/reference/mapping/types/text.asciidoc
index 49d7618ae862c..f12c28deb13cc 100644
--- a/docs/reference/mapping/types/text.asciidoc
+++ b/docs/reference/mapping/types/text.asciidoc
@@ -129,7 +129,7 @@ The following parameters are accepted by `text` fields:
 <<similarity,`similarity`>>::
 
     Which scoring algorithm or _similarity_ should be used. Defaults
-    to `classic`, which uses TF/IDF.
+    to `BM25`.
 
 <<term-vector,`term_vector`>>::
 
diff --git a/docs/reference/mapping/types/token-count.asciidoc b/docs/reference/mapping/types/token-count.asciidoc
index 49e4e3b424f59..d89a3504bf8bc 100644
--- a/docs/reference/mapping/types/token-count.asciidoc
+++ b/docs/reference/mapping/types/token-count.asciidoc
@@ -48,12 +48,6 @@ GET my_index/_search
 <2> The `name.length` field is a `token_count` <<multi-fields,multi-field>> which will index the number of tokens in the `name` field.
 <3> This query matches only the document containing `Rachel Alice Williams`, as it contains three tokens.
 
-[NOTE]
-===================================================================
-Technically the `token_count` type sums position increments rather than
-counting tokens. This means that even if the analyzer filters out stop
-words they are included in the count.
-===================================================================
 
 [[token-count-params]]
 ==== Parameters for `token_count` fields
@@ -68,6 +62,12 @@ The following parameters are accepted by `token_count` fields:
     value. Required. For best performance, use an analyzer without token
     filters.
 
+`enable_position_increments`:: 
+
+Indicates if position increments should be counted. 
+Set to `false` if you don't want to count tokens removed by analyzer filters (like <<analysis-stop-tokenfilter,`stop`>>). 
+Defaults to `true`.
+
 <<mapping-boost,`boost`>>::
 
     Mapping field-level query time boosting. Accepts a floating point number, defaults
diff --git a/docs/reference/migration/index.asciidoc b/docs/reference/migration/index.asciidoc
index 0dda43da71326..428d5e9127a84 100644
--- a/docs/reference/migration/index.asciidoc
+++ b/docs/reference/migration/index.asciidoc
@@ -19,4 +19,17 @@ As a general rule:
 
 See <<setup-upgrade>> for more info.
 --
+
+include::migrate_5_6.asciidoc[]
+
+include::migrate_5_5.asciidoc[]
+
+include::migrate_5_4.asciidoc[]
+
+include::migrate_5_3.asciidoc[]
+
+include::migrate_5_2.asciidoc[]
+
+include::migrate_5_1.asciidoc[]
+
 include::migrate_5_0.asciidoc[]
diff --git a/docs/reference/migration/migrate_5_0.asciidoc b/docs/reference/migration/migrate_5_0.asciidoc
index 2449406928034..b0c08bd86d6b4 100644
--- a/docs/reference/migration/migrate_5_0.asciidoc
+++ b/docs/reference/migration/migrate_5_0.asciidoc
@@ -4,41 +4,6 @@
 This section discusses the changes that you need to be aware of when migrating
 your application to Elasticsearch 5.0.
 
-[IMPORTANT]
-.Known networking bug in 5.0.0-alpha5
-======================================================
-
-There is a bug in the new Netty4 implementation in this release which affects any REST request with
-a body that is sent in two requests, the first with an `Expect: 100-continue` header. This bug will
-manifest with an exception similar to the following:
-
-[source,txt]
-----
-[WARN ][http.netty4] [wtOV9Vb] caught exception while handling client http traffic, closing connection [id: 0x1320b717, L:/0:0:0:0:0:0:0:1:9200 - R:/0:0:0:0:0:0:0:1:54732]
-java.lang.UnsupportedOperationException: unsupported message type: DefaultFullHttpResponse (expected: ByteBuf, FileRegion)
-----
-
-This is due to incorrect handling of the `Expect` HTTP header, and it can be
-worked around in one of three ways:
-
-* Use a client which does not add `Expect` headers (including the official clients).
-
-* Pass a blank `Except` header, e.g.
-+
-[source,sh]
-----
-curl -H 'Expect:' ...
-----
-
-* Use Netty3 for the HTTP layer by passing the following setting at startup:
-+
-[source,sh]
-----
-./bin/elasticsearch -Ehttp.type=netty3
-----
-
-======================================================
-
 [float]
 [[migration-plugin]]
 === Migration Plugin
@@ -59,10 +24,8 @@ version of Elasticsearch before 2.0.
 =========================================
 
 Indices created in Elasticsearch 1.x or before will need to be reindexed with
-Elasticsearch 2.x in order to be readable by Elasticsearch 5.x. It is not
-sufficient to use the <<indices-upgrade,`upgrade`>> API.  The easiest
-way to reindex old indices is to upgrade to Elasticsearch 2.3 or later and to use the
-`reindex` API, or the reindex UI provided by the <<migration-plugin,Migration Plugin>>.
+Elasticsearch 2.x or 5.x in order to be readable by Elasticsearch 5.x. It is not
+sufficient to use the `upgrade` API.  See <<reindex-upgrade>> for more details.
 
 =========================================
 
diff --git a/docs/reference/migration/migrate_5_0/docs.asciidoc b/docs/reference/migration/migrate_5_0/docs.asciidoc
index 2c2da15d0cade..d3e60d5ef5a89 100644
--- a/docs/reference/migration/migrate_5_0/docs.asciidoc
+++ b/docs/reference/migration/migrate_5_0/docs.asciidoc
@@ -14,16 +14,16 @@ immediate refresh related action. See <<docs-refresh>>.
 ==== `created` field deprecated in the Index API
 
 The `created` field has been deprecated in the Index API. It now returns
-`operation`, returning `"operation": "create"` when it created a document and
-`"operation": "index"` when it updated the document. This is also true for
+`result`, returning `"result": "created"` when it created a document and
+`"result": "updated"` when it updated the document. This is also true for
 `index` bulk operations.
 
 ==== `found` field deprecated in the Delete API
 
 The `found` field has been deprecated in the Delete API. It now returns
-`operation`, returning `"operation": "deleted"` when it deleted a document and
-`"operation": "noop"` when it didn't found the document. This is also true for
-`index` bulk operations.
+`result`, returning `"result": "deleted"` when it deleted a document and
+`"result": "not_found"` when it didn't find the document. This is also true for
+`delete` bulk operations.
 
 ==== Reindex and Update By Query
 Before 5.0.0 `_reindex` and `_update_by_query` only retried bulk failures so
@@ -65,7 +65,16 @@ performance if the same document is updated very frequently using a read modify
 pattern since it might create many small segments. This behavior can be disabled by
 passing `realtime=false` to the get request.
 
-==== version type 'force' removed
+The `fields` field has been renamed to `stored_fields`
+
+==== mget API
+
+The `fields` field has been renamed to `stored_fields`
+
+==== update API
+
+The `fields` field has been deprecated. You should use `_source` to load the field from _source.
+
+==== bulk API
 
-In 5.0.0 document modification operations may no longer specify the
-`version_type` of `force` to override any previous version checks.
+The `fields` field has been deprecated. You should use `_source` to load the field from _source.
diff --git a/docs/reference/migration/migrate_5_0/index-apis.asciidoc b/docs/reference/migration/migrate_5_0/index-apis.asciidoc
index c8f939c259d76..91c0a4ca27ca0 100644
--- a/docs/reference/migration/migrate_5_0/index-apis.asciidoc
+++ b/docs/reference/migration/migrate_5_0/index-apis.asciidoc
@@ -69,3 +69,10 @@ prefer there to be only one (obvious) way to do things like this.
 
 As of 5.0 indexing a document with `op_type=create` without specifying an ID is not
 supported anymore.
+
+==== Flush API
+
+The `wait_if_ongoing` flag default has changed to `true` causing `_flush` calls to wait and block
+if another flush operation is currently running on the same shard. In turn, if `wait_if_ongoing` is set to
+`false` and another flush operation is already running the flush is skipped and the shards flush call will return
+immediately without any error. In previous versions `flush_not_allowed` exceptions where reported for each skipped shard.
\ No newline at end of file
diff --git a/docs/reference/migration/migrate_5_0/java.asciidoc b/docs/reference/migration/migrate_5_0/java.asciidoc
index e0f7fba26b00a..59caf0c6018ac 100644
--- a/docs/reference/migration/migrate_5_0/java.asciidoc
+++ b/docs/reference/migration/migrate_5_0/java.asciidoc
@@ -13,7 +13,7 @@ The Java transport client has been moved to its own module which can be referenc
 <dependency>
     <groupId>org.elasticsearch.client</groupId>
     <artifactId>transport</artifactId>
-    <version>5.0.0-alpha5</version>
+    <version>5.0.0</version>
 </dependency>
 -----
 
@@ -26,7 +26,7 @@ TransportClient client = new PreBuiltTransportClient(Settings.EMPTY)
         .addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName("host2"), 9300));
 -----
 
-For more information please see the {javaclient}java-api.html[Java client documentation]
+For more information please see the {javaclient}/java-api.html[Java client documentation]
 
 ==== Count api has been removed
 
@@ -88,7 +88,7 @@ language `org.codehaus.groovy:groovy` artifact.
 
 ==== DocumentAlreadyExistsException removed
 
-`DocumentAlreadyExistsException` is removed and a `VersionConflictException` is thrown instead (with a better
+`DocumentAlreadyExistsException` is removed and a `VersionConflictEngineException` is thrown instead (with a better
 error description). This will influence code that use the `IndexRequest.opType()` or `IndexRequest.create()`
 to index a document only if it doesn't already exist.
 
@@ -210,7 +210,7 @@ Removing setters for to(Object ...) and from(Object ...) in favour of the only t
 arguments (String, Number). Removing setter for center point (point(), geohash()) because parameter
 is mandatory and should already be set in constructor.
 Also removing setters for lt(), lte(), gt(), gte() since they can all be replaced by equivalent
-calls to to/from() and inludeLower()/includeUpper().
+calls to to/from() and includeLower()/includeUpper().
 
 ===== GeoPolygonQueryBuilder
 
@@ -228,7 +228,7 @@ Use the `field(String, float)` method instead.
 ===== MissingQueryBuilder
 
 The MissingQueryBuilder which was deprecated in 2.2.0 is removed. As a replacement use ExistsQueryBuilder
-inside a mustNot() clause. So instead of using `new ExistsQueryBuilder(name)` now use
+inside a mustNot() clause. So instead of using `new MissingQueryBuilder(name)` now use
 `new BoolQueryBuilder().mustNot(new ExistsQueryBuilder(name))`.
 
 ===== NotQueryBuilder
@@ -306,7 +306,7 @@ The `setExtraSource(SearchSourceBuilder)` method has been removed.
 
 The `setTemplateSource(String)` and `setTemplateSource(BytesReference)` methods have been removed. Use `setTemplate(Template)` instead.
 
-`setRescorer(Rescorer)` and `setRescorer(Rescorer, int)` have been removed infavor of `setRescorer(RescoreBuilder)` and `setRescorer(RescoreBuilder, int)`
+`setRescorer(Rescorer)` and `setRescorer(Rescorer, int)` have been removed in favor of `setRescorer(RescoreBuilder)` and `setRescorer(RescoreBuilder, int)`
 
 ==== SearchRequest
 
@@ -415,3 +415,7 @@ try (InputStream stream = bytesRef.streamInput()) {
 
 For any custom implementation of the `BlobContainer` interface, these three methods must be removed.
 
+==== NodeBuilder removed
+
+NodeBuilder has been removed. While using Node directly within an application is not officially supported, it can still be constructed with the `Node(Settings)` constructor.
+
diff --git a/docs/reference/migration/migrate_5_0/mapping.asciidoc b/docs/reference/migration/migrate_5_0/mapping.asciidoc
index 9d446d48341e7..a57cb5ea2972b 100644
--- a/docs/reference/migration/migrate_5_0/mapping.asciidoc
+++ b/docs/reference/migration/migrate_5_0/mapping.asciidoc
@@ -76,6 +76,15 @@ PUT my_index
 Also the `precision_step` parameter is now irrelevant and will be rejected on
 indices that are created on or after 5.0.
 
+==== `geo_point` fields
+
+Like Numeric fields the Geo point field now uses the new BKD tree structure. Since
+this structure is fundamentally designed for multi-dimension spatial data, the
+following field parameters are no longer needed or supported: `geohash`,
+`geohash_prefix`, `geohash_precision`, `lat_lon`. Geohashes are still supported from
+an API perspective, and can still be accessed using the `.geohash` field extension,
+but they are no longer used to index geo point data.
+
 ==== `_timestamp` and `_ttl`
 
 The `_timestamp` and `_ttl` fields were deprecated and are now removed. As a
@@ -93,8 +102,8 @@ property now only accepts `true`/`false` instead of `not_analyzed`/`no`. The
 ==== Doc values on unindexed fields
 
 Previously, setting a field to `index:no` would also disable doc-values.  Now,
-doc-values are always enabled on numeric and boolean fields unless
-`doc_values` is set to `false`.
+doc-values are enabled by default on all types but `text` and `binary`,
+regardless of the value of the `index` property.
 
 ==== Floating points use `float` instead of `double`
 
@@ -121,7 +130,20 @@ Regex filters are not supported anymore and will be dropped on upgrade.
 
 ==== Source-transform removed
 
-The source `transform` feature has been removed. Instead, use an ingest pipeline
+The source `transform` feature has been removed. Instead, use an ingest pipeline.
+
+
+==== Field mapping limits
+
+To prevent mapping explosions, the following limits are applied to indices
+created in 5.x:
+
+* The maximum number of fields in an index is limited to 1000.
+* The maximum depth for a field (1 plus the number of `object` or `nested` parents) is limited to 20.
+* The maximum number of `nested` fields in an index is limited to 50.
+
+See <<mapping-limit-settings>> for more.
+
 
 ==== `_parent` field no longer indexed
 
@@ -219,7 +241,7 @@ POST _reindex
   },
   "script": {
     "lang": "painless",
-    "inline": "ctx._source.timestamp = ctx._timestamp; ctx._timestamp = null"
+    "source": "ctx._source.timestamp = ctx._timestamp; ctx._timestamp = null"
   }
 }
 ---------------
diff --git a/docs/reference/migration/migrate_5_0/packaging.asciidoc b/docs/reference/migration/migrate_5_0/packaging.asciidoc
index 74faf3bb7d9a3..13a27751409ac 100644
--- a/docs/reference/migration/migrate_5_0/packaging.asciidoc
+++ b/docs/reference/migration/migrate_5_0/packaging.asciidoc
@@ -1,6 +1,13 @@
 [[breaking_50_packaging]]
 === Packaging
 
+==== APT/YUM repository URL changes
+
+The repository for apt and yum packages has changed from
+`https://packages.elastic.co` to `https://artifacts.elastic.co/`.
+
+Full details can be found in <<install-elasticsearch>>.
+
 ==== Default logging using systemd (since Elasticsearch 2.2.0)
 
 In previous versions of Elasticsearch, the default logging
@@ -37,6 +44,16 @@ from the tar or zip distributions, and /etc/elasticsearch/jvm.options if install
 from the Debian or RPM packages. You can specify an alternative location by setting
 the environment variable `ES_JVM_OPTIONS` to the path to the file.
 
+==== Thread stack size for the Windows service
+
+Previously when installing the Windows service, the installation script
+would configure the thread stack size (this is required for the service
+daemon). As a result of moving all JVM configuration to the
+<<jvm-options,jvm.options file>>, the service installation script no
+longer configures the thread stack size. When installing the Windows
+service, you must configure thread stack size. For additional details,
+see the <<windows-service,installation docs>>.
+
 ==== /bin/bash is now required
 
 Previously, the scripts used to start Elasticsearch and run plugin
diff --git a/docs/reference/migration/migrate_5_0/plugins.asciidoc b/docs/reference/migration/migrate_5_0/plugins.asciidoc
index 82c0e4f6b9ac7..15861c5e60d64 100644
--- a/docs/reference/migration/migrate_5_0/plugins.asciidoc
+++ b/docs/reference/migration/migrate_5_0/plugins.asciidoc
@@ -25,23 +25,17 @@ discovery plugins.
 Plugins implementing custom queries need to implement the `fromXContent(QueryParseContext)` method in their
 `QueryParser` subclass rather than `parse`. This method will take care of parsing the query from `XContent` format
 into an intermediate query representation that can be streamed between the nodes in binary format, effectively the
-query object used in the java api. Also, the query parser needs to implement the `getBuilderPrototype` method that
-returns a prototype of the `NamedWriteable` query, which allows to deserialize an incoming query by calling
-`readFrom(StreamInput)` against it, which will create a new object, see usages of `Writeable`. The `QueryParser`
-also needs to declare the generic type of the query that it supports and it's able to parse.
+query object used in the java api. Also, the query builder needs to be registered as a `NamedWriteable`. This is
+all done by implementing the `SearchPlugin` interface and overriding the `getQueries` method.
 The query object can then transform itself into a lucene query through the new `toQuery(QueryShardContext)` method,
 which returns a lucene query to be executed on the data node.
 
 Similarly, plugins implementing custom score functions need to implement the `fromXContent(QueryParseContext)`
 method in their `ScoreFunctionParser` subclass rather than `parse`. This method will take care of parsing
 the function from `XContent` format into an intermediate function representation that can be streamed between
-the nodes in binary format, effectively the function object used in the java api. Also, the query parser needs
-to implement the `getBuilderPrototype` method that returns a prototype of the `NamedWriteable` function, which
-allows to deserialize an incoming function by calling `readFrom(StreamInput)` against it, which will create a
-new object, see usages of `Writeable`. The `ScoreFunctionParser` also needs to declare the generic type of the
-function that it supports and it's able to parse. The function object can then transform itself into a lucene
-function through the new `toFunction(QueryShardContext)` method, which returns a lucene function to be executed
-on the data node.
+the nodes in binary format, effectively the function object used in the java api. The function object can then
+transform itself into a lucene function through the new `toFunction(QueryShardContext)` method, which returns
+a lucene function to be executed on the data node.
 
 ==== Cloud AWS plugin changes
 
@@ -147,6 +141,18 @@ Plugins that register custom search time behavior (`Query`, `Suggester`,
 `ScoreFunction`, `FetchSubPhase`, `Highlighter`, etc) should implement
 `SearchPlugin` and remove their `onModule(SearchModule)` implementation.
 
+==== SearchParseElement
+
+The `SearchParseElement` interface has been removed. Custom search request
+ sections can only be provided under the `ext` element. Plugins can
+ plug in custom parsers for those additional sections by providing a
+ `SearchPlugin.SearchExtSpec`, which consists of a `SearchExtParser`
+ implementation that can parse`XContent` into a `SearchExtBuilder`
+ implementation. The parsing happens now in the coordinating node. The
+ result of parsing is serialized to the data nodes through transport layer
+ together with the rest of the search request and stored in the search
+ context for later retrieval.
+
 ==== Testing Custom Plugins
 
 `ESIntegTestCase#pluginList` has been removed. Use `Arrays.asList` instead. It
@@ -155,7 +161,5 @@ isn't needed now that all plugins require Java 1.8.
 ==== Mapper-Size plugin
 
 The metadata field `_size` is not accessible in aggregations, scripts and when
-sorting for indices created in 2.x even if the index has been upgraded using the <<indices-upgrade,`upgrade`>> API.
+sorting for indices created in 2.x.
 If these features are needed in your application it is required to reindex the data with Elasticsearch 5.x.
-The easiest way to reindex old indices is to use the `reindex` API, or the reindex UI provided by
-the <<migration-plugin,Migration Plugin>>.
diff --git a/docs/reference/migration/migrate_5_0/rest.asciidoc b/docs/reference/migration/migrate_5_0/rest.asciidoc
index afba8819ed348..ecbed731d6a72 100644
--- a/docs/reference/migration/migrate_5_0/rest.asciidoc
+++ b/docs/reference/migration/migrate_5_0/rest.asciidoc
@@ -1,7 +1,16 @@
-
 [[breaking_50_rest_api_changes]]
 === REST API changes
 
+==== Strict REST query string parameter parsing
+
+Previous versions of Elasticsearch ignored unrecognized URL query
+string parameters. This means that extraneous parameters or parameters
+containing typographical errors would be silently accepted by
+Elasticsearch. This is dangerous from an end-user perspective because it
+means a submitted request will silently execute not as intended. This
+leniency has been removed and Elasticsearch will now fail any request
+that contains unrecognized query string parameters.
+
 ==== id values longer than 512 bytes are rejected
 
 When specifying an `_id` value longer than 512 bytes, the request will be
@@ -57,7 +66,7 @@ empty, that means that the node is a coordinating only node.
 
 Previously, JSON documents were allowed with unquoted field names, which isn't
 strictly JSON and broke some Elasticsearch clients. If documents were already
-indexed with unquoted fields in a previous vesrion of Elasticsearch, some
+indexed with unquoted fields in a previous version of Elasticsearch, some
 operations may throw errors. To accompany this, a commented out JVM option has
 been added to the `jvm.options` file:
 `-Delasticsearch.json.allow_unquoted_field_names`.
diff --git a/docs/reference/migration/migrate_5_0/scripting.asciidoc b/docs/reference/migration/migrate_5_0/scripting.asciidoc
index 3e0db9e1cbd88..d918e65570f02 100644
--- a/docs/reference/migration/migrate_5_0/scripting.asciidoc
+++ b/docs/reference/migration/migrate_5_0/scripting.asciidoc
@@ -20,7 +20,7 @@ Groovy:
   "script_score": {
     "script": {
       "lang": "groovy",
-      "inline": "Math.log(_score * 2) + my_modifier",
+      "source": "Math.log(_score * 2) + my_modifier",
       "params": {
         "my_modifier": 8
       }
@@ -37,7 +37,7 @@ Painless (`my_modifer` is prefixed with `params`):
   "script_score": {
     "script": {
       "lang": "painless",
-      "inline": "Math.log(_score * 2) + params.my_modifier",
+      "source": "Math.log(_score * 2) + params.my_modifier",
       "params": {
         "my_modifier": 8
       }
@@ -84,7 +84,7 @@ and instead this syntax must be used:
   "script_score": {
     "script": {
       "lang": "groovy",
-      "inline": "Math.log(_score * 2) + my_modifier",
+      "source": "Math.log(_score * 2) + my_modifier",
       "params": {
         "my_modifier": 8
       }
@@ -187,7 +187,7 @@ and instead the following syntax should be used:
 {
     "query": {
         "template": {
-            "inline": {"match_{{template}}": {}},
+            "source": {"match_{{template}}": {}},
             "params" : {
                 "template" : "all"
             }
@@ -221,7 +221,7 @@ and instead the following syntax should be used:
 [source,js]
 -----------------------------------
 {
-    "inline" : {
+    "source" : {
         "query": { "match" : { "{{my_field}}" : "{{my_value}}" } },
         "size" : "{{my_size}}"
     },
@@ -389,4 +389,17 @@ The following helper methods have been removed from GeoPoint scripting:
 
 Instead use `arcDistance`, `arcDistanceWithDefault`, `planeDistance`, `planeDistanceWithDefault`, `geohashDistance`,
 `geohashDistanceWithDefault` and convert from default units (meters) to desired units using the appropriate constance
-(e.g., multiply by `0.001` to convert to Km).
\ No newline at end of file
+(e.g., multiply by `0.001` to convert to Km).
+
+==== Only 15 unique scripts can be compiled per minute by default
+
+If you compile too many unique scripts within a small amount of time,
+Elasticsearch will reject the new dynamic scripts with a
+`circuit_breaking_exception` error. By default, up to 15 inline scripts per
+minute will be compiled. You can change this setting dynamically by setting
+`script.max_compilations_per_minute`.
+
+You should watch out for this if you are hard-coding values into your scripts.
+
+Elasticsearch recommends the usage of parameters for efficient script handling.
+See details <<prefer-params,here>>.
diff --git a/docs/reference/migration/migrate_5_0/search.asciidoc b/docs/reference/migration/migrate_5_0/search.asciidoc
index 09478ee325195..6fb844dadd25b 100644
--- a/docs/reference/migration/migrate_5_0/search.asciidoc
+++ b/docs/reference/migration/migrate_5_0/search.asciidoc
@@ -62,6 +62,16 @@ Scroll requests sorted by `_doc` have been optimized to more efficiently resume
 from where the previous request stopped, so this will have the same performance
 characteristics as the former `scan` search type.
 
+==== Search shard limit
+
+In 5.0, Elasticsearch rejects requests that would query more than 1000 shard
+copies (primaries or replicas).  The reason is that such large numbers of
+shards make the job of the coordinating node very CPU and memory intensive. It
+is usually a better idea to organize data in such a way that there are fewer
+larger shards. In case you would like to bypass this limit, which is
+discouraged, you can update the `action.search.shard_count.limit` cluster
+setting to a greater value.
+
 ==== `fields` parameter
 
 The `fields` parameter has been replaced by `stored_fields`.
@@ -86,6 +96,7 @@ The following deprecated queries have been removed:
 `filtered`::      Use `bool` query instead, which supports `filter` clauses too.
 `and`::           Use `must` clauses in a `bool` query instead.
 `or`::            Use `should` clauses in a `bool` query instead.
+`missing`::       Use a negated `exists` query instead.  (Also removed `_missing_` from the `query_string` query)
 `limit`::         Use the `terminate_after` parameter instead.
 `fquery`::        Is obsolete after filters and queries have been merged.
 `query`::         Is obsolete after filters and queries have been merged.
@@ -145,12 +156,14 @@ This parameter was undocumented and silently ignored before for these types of `
 
 * Deprecated support for the coerce, normalize, ignore_malformed parameters in GeoPolygonQuery. Use parameter validation_method instead.
 
-* Deprecated support for the coerce, normalize, ignore_malformed parameters in GeoDistanceRangeQuery. Use parameter validation_method instead.
-
 * Deprecated support for the coerce, normalize, ignore_malformed parameters in GeoDistanceQuery. Use parameter validation_method instead.
 
 * Deprecated support for the coerce, normalize, ignore_malformed parameters in GeoBoundingBoxQuery. Use parameter validation_method instead.
 
+* The `geo_distance_range` query is deprecated and should be replaced by either the `geo_distance` bucket aggregation, or geo_distance sort.
+
+* For `geo_distance` query, aggregation, and sort the `sloppy_arc` option for the `distance_type` parameter has been deprecated.
+
 ==== Top level `filter` parameter
 
 Removed support for the deprecated top level `filter` in the search api,
@@ -207,6 +220,30 @@ been superseded by `_prefer_nodes`. By specifying a single node,
 `_prefer_nodes` provides the same functionality as `_prefer_node` but
 also supports specifying multiple nodes.
 
-==== Default similarity
+The <<search-request-preference,search preference>> `_shards` accepts a
+secondary preference, for example `_primary` to specify the primary copy
+of the specified shards. The separator previously used to separate the
+`_shards` portion of the parameter from the secondary preference was
+`;`. However, this is also an acceptable separator between query string
+parameters which means that unless the `;` was escaped, the secondary
+preference was never observed. The separator has been changed to `|` and
+does not need to be escaped.
+
+
+==== Scoring changes
+
+===== Default similarity
 
 The default similarity has been changed to `BM25`.
+
+===== DF formula
+
+Document frequency (which is for instance used to compute inverse document
+frequency - IDF) is now based on the number of documents that have a value
+for the considered field rather than the total number of documents in the
+index. This change affects most similarities. See
+https://issues.apache.org/jira/browse/LUCENE-6711[`LUCENE-6711`] for more information.
+
+==== explain API
+
+The `fields` field has been renamed to `stored_fields`
diff --git a/docs/reference/migration/migrate_5_0/settings.asciidoc b/docs/reference/migration/migrate_5_0/settings.asciidoc
index f6875f60c60c3..d40a0e1b309f9 100644
--- a/docs/reference/migration/migrate_5_0/settings.asciidoc
+++ b/docs/reference/migration/migrate_5_0/settings.asciidoc
@@ -28,11 +28,11 @@ The `name` setting has been removed and is replaced by `node.name`. Usage of
 
 The `node.add_id_to_custom_path` was renamed to `add_lock_id_to_custom_path`.
 
-The default for the `node.name` settings is now the first 7 charachters of the node id,
+The default for the `node.name` settings is now the first 7 characters of the node id,
 which is in turn a randomly generated UUID.
 
-The settings `node.mode` and `node.local` are removed. Local mode should be configured via
-`discovery.type: local` and `transport.type:local`. In order to disable _http_ please use `http.enabled: false`
+The settings `node.mode` and `node.local` are removed. Local mode should be configured via `transport.type: local`. In
+order to disable HTTP please use `http.enabled: false`
 
 ==== Node attribute settings
 
@@ -73,7 +73,7 @@ scope (e.g. `_local_,_site_` for all loopback and private network addresses)
 or by explicit interface names, hostnames, or addresses.
 
 The `netty.epollBugWorkaround` settings is removed. This settings allow people to enable
-a netty work around for https://github.com/netty/netty/issues/327[a high CPU usage issue] with early JVM versions. 
+a netty work around for https://github.com/netty/netty/issues/327[a high CPU usage issue] with early JVM versions.
 This bug was http://bugs.java.com/view_bug.do?bug_id=6403933[fixed in Java 7]. Since Elasticsearch 5.0 requires Java 8 the settings is removed. Note that if the workaround needs to be reintroduced you can still set the `org.jboss.netty.epollBugWorkaround` system property to control Netty directly.
 
 ==== Forbid changing of thread pool types
@@ -133,7 +133,7 @@ Recovery settings deprecated in 1.x have been removed:
  * `index.shard.recovery.translog_size` is superseded by `indices.recovery.translog_size`
  * `index.shard.recovery.translog_ops` is superseded by `indices.recovery.translog_ops`
  * `index.shard.recovery.file_chunk_size` is superseded by `indices.recovery.file_chunk_size`
- * `index.shard.recovery.concurrent_streams` is superseded by `indices.recovery.concurrent_streams`
+ * `indices.recovery.concurrent_streams` is superseded by `cluster.routing.allocation.node_concurrent_recoveries`
  * `index.shard.recovery.concurrent_small_file_streams` is superseded by `indices.recovery.concurrent_small_file_streams`
  * `indices.recovery.max_size_per_sec` is superseded by `indices.recovery.max_bytes_per_sec`
 
@@ -230,7 +230,12 @@ the cache implementation used for the request cache and the field data cache.
 
 ==== Using system properties to configure Elasticsearch
 
-Elasticsearch can no longer be configured by setting system properties.
+Elasticsearch can no longer be configured by setting system properties. This means that support for all of the following has been removed:
+
+ * setting via command line arguments to `elasticsearch` as `-Des.name.of.setting=value.of.setting`
+ * setting via the JAVA_OPTS environment variable `JAVA_OPTS=$JAVA_OPTS -Des.name.of.setting=value.of.setting`
+ * setting via the ES_JAVA_OPTS environment variable `ES_JAVA_OPTS=$ES_JAVA_OPTS -Des.name.of.setting=value.of.setting`
+
 Instead, use `-Ename.of.setting=value.of.setting`.
 
 ==== Removed using double-dashes to configure Elasticsearch
@@ -259,6 +264,36 @@ a fallback realtime setting for the get and mget APIs when realtime
 wasn't specified. Now if the parameter isn't specified we always
 default to true.
 
+==== Memory lock settings
+
+The setting `bootstrap.mlockall` has been renamed to
+`bootstrap.memory_lock`.
+
+==== Snapshot settings
+
+The default setting `include_global_state` for restoring snapshots has been
+changed from `true` to `false`. It has not been changed for taking snapshots and
+still defaults to `true` in that case.
+
+==== Time value parsing
+
+The unit 'w' representing weeks is no longer supported.
+
+Fractional time values (e.g., 0.5s) are no longer supported. For example, this means when setting
+timeouts "0.5s" will be rejected and should instead be input as "500ms".
+
+==== Node max local storage nodes
+
+Previous versions of Elasticsearch defaulted to allowing multiple nodes to share the same data
+directory (up to 50). This can be confusing where users accidentally startup multiple nodes and end
+up thinking that they've lost data because the second node will start with an empty data directory.
+While the default of allowing multiple nodes is friendly to playing with forming a small cluster on
+a laptop, and end-users do sometimes run multiple nodes on the same host, this tends to be the
+exception. Keeping with Elasticsearch's continual movement towards safer out-of-the-box defaults,
+and optimizing for the norm instead of the exception, the default for
+`node.max_local_storage_nodes` is now one.
+
+[float]
 === Script settings
 
 ==== Indexed script settings
@@ -284,8 +319,8 @@ Previously script mode settings (e.g., "script.inline: true",
 ==== Script sandbox settings removed
 
 Prior to 5.0 a third option could be specified for the `script.inline` and
-`script.stored` settings ("sandbox"). This has been removed, You can now only
-set `script.line: true` or `script.stored: true`.
+`script.stored` settings ("sandbox"). This has been removed, you can now only
+set `script.inline: true` or `script.stored: true`.
 
 ==== Search settings
 
@@ -293,31 +328,3 @@ The setting `index.query.bool.max_clause_count` has been removed. In order to
 set the maximum number of boolean clauses `indices.query.bool.max_clause_count`
 should be used instead.
 
-==== Memory lock settings
-
-The setting `bootstrap.mlockall` has been renamed to
-`bootstrap.memory_lock`.
-
-==== Snapshot settings
-
-The default setting `include_global_state` for restoring snapshots has been
-changed from `true` to `false`. It has not been changed for taking snapshots and
-still defaults to `true` in that case.
-
-==== Time value parsing
-
-The unit 'w' representing weeks is no longer supported.
-
-Fractional time values (e.g., 0.5s) are no longer supported. For example, this means when setting
-timeouts "0.5s" will be rejected and should instead be input as "500ms".
-
-==== Node max local storage nodes
-
-Previous versions of Elasticsearch defaulted to allowing multiple nodes to share the same data
-directory (up to 50). This can be confusing where users accidentally startup multiple nodes and end
-up thinking that they've lost data because the second node will start with an empty data directory.
-While the default of allowing multiple nodes is friendly to playing with forming a small cluster on
-a laptop, and end-users do sometimes run multiple nodes on the same host, this tends to be the
-exception. Keeping with Elasticsearch's continual movement towards safer out-of-the-box defaults,
-and optimizing for the norm instead of the exception, the default for
-`node.max_local_storage_nodes` is now one.
diff --git a/docs/reference/migration/migrate_5_1.asciidoc b/docs/reference/migration/migrate_5_1.asciidoc
new file mode 100644
index 0000000000000..5c2a8210c7816
--- /dev/null
+++ b/docs/reference/migration/migrate_5_1.asciidoc
@@ -0,0 +1,73 @@
+[[breaking-changes-5.1]]
+== Breaking changes in 5.1
+
+[[breaking_51_index_api_changes]]
+[float]
+=== Indices API changes
+
+[float]
+==== Alias names are validated against (most of) the rules for index names
+
+Alias names are now validated against almost the same set of rules that validate
+index names. The only difference is that aliases are allowed to have uppercase
+characters. That means that aliases may not:
+
+* Start with `_`, `-`, or `+`
+* Contain `#`, `\`, `/`, `*`, `?`, `"`, `<`, `>`, `|`, ` `, `,`
+* Be longer than 100 UTF-8 encoded bytes
+* Be exactly `.` or `..`
+
+Aliases created in versions before 5.1.0 are still supported but no new aliases
+can be added that violate those rules. Since modifying an alias in elasticsearch
+is removing it and recreating it atomically using the `_aliases` API, modifying
+aliases with invalid names is also no longer supported.
+
+[[breaking_51_java_api_changes]]
+[float]
+=== Java API changes
+
+[float]
+==== Log4j dependency has been upgraded
+
+The Log4j dependency has been upgraded from version 2.6.2 to version 2.7. If you're using the transport client in your
+application, you should update your Log4j dependencies accordingly.
+
+[float]
+==== Local discovery has been removed
+
+Local discovery has been removed; this discovery implementation was used internally in the tribe service and for tests
+that ran multiple nodes inside the same JVM. This means that setting `discovery.type` to `local` will fail on startup.
+
+[[breaking_51_plugin_api]]
+[float]
+=== Plugin API changes
+
+[float]
+==== UnicastHostsProvider now pull based
+
+Plugging in a `UnicastHostsProvider` for zen discovery is now pull based. Implement a `DiscoveryPlugin` and override the `getZenHostsProviders` method. Selecting a hosts provider is also now done with a separate setting, `discovery.zen.hosts_provider`.
+
+[float]
+==== ZenPing and MasterElectService pluggability removed
+
+These classes are no longer pluggable. Either implement your own discovery, or extend from ZenDiscovery and customize as necessary.
+
+[float]
+==== onModule support removed
+
+Plugins could formerly implement methods of the name `onModule` which took a single
+Guice module. All the uses of onModule for plugging in custom behavior have now been
+converted to pull based plugins, and hooks for onModule have been removed.
+
+[[breaking_51_other_api_changes]]
+[float]
+=== Other API changes
+
+[float]
+==== Indices stats and node stats API unrecognized metrics
+
+The indices stats and node stats APIs allow querying Elasticsearch for a variety of metrics. Previous versions of
+Elasticsearch would silently accept unrecognized metrics (e.g., typos like "transprot"). In 5.1.0, this is no longer
+the case; unrecognized metrics will cause the request to fail. There is one exception to this which is the percolate
+metric which was removed in 5.0.0 but requests for these will only produce a warning in the 5.x series starting with
+5.1.0 and will fail like any other unrecognized metric in 6.0.0.
diff --git a/docs/reference/migration/migrate_5_2.asciidoc b/docs/reference/migration/migrate_5_2.asciidoc
new file mode 100644
index 0000000000000..0de3d78cc864e
--- /dev/null
+++ b/docs/reference/migration/migrate_5_2.asciidoc
@@ -0,0 +1,68 @@
+[[breaking-changes-5.2]]
+== Breaking changes in 5.2
+
+[[breaking_52_packaging_changes]]
+[float]
+=== Packaging changes
+
+[float]
+==== System call bootstrap check
+
+Elasticsearch has attempted to install a system call filter since version 2.1.0. On some systems, installing this
+system call filter could fail. Previous versions of Elasticsearch would log a warning, but would otherwise continue
+executing potentially leaving the end-user unaware of this situation. Starting in Elasticsearch 5.2.0, there is now a
+<<bootstrap-checks,bootstrap check>> for success of installing the system call filter. If you encounter an issue
+starting Elasticsearch due to this bootstrap check, you need to either fix your configuration so that the system call
+filter can be installed, or *at your own risk* disable the <<system-call-filter-check,system call filter check>>.
+
+[[breaking_52_settings_changes]]
+[float]
+=== Settings changes
+
+[float]
+==== System call filter setting
+
+Elasticsearch has attempted to install a system call filter since version 2.1.0. These are enabled by default and
+could be disabled via `bootstrap.seccomp`. The naming of this setting is poor since seccomp is specific to Linux but
+Elasticsearch attempts to install a system call filter on various operating systems. Starting in Elasticsearch 5.2.0,
+this setting has been renamed to `bootstrap.system_call_filter`. The previous setting is still support but will be
+removed in Elasticsearch 6.0.0.
+
+[[breaking_52_java_api_changes]]
+[float]
+=== Java API changes
+
+[float]
+==== Removed some of the `source` overrides
+
+In an effort to clean up internals we've removed the following methods:
+
+* `PutRepositoryRequest#source(XContentBuilder)`
+* `PutRepositoryRequest#source(String)`
+* `PutRepositoryRequest#source(byte[])`
+* `PutRepositoryRequest#source(byte[], int, int)`
+* `PutRepositoryRequest#source(BytesReference)`
+* `CreateSnapshotRequest#source(XContentBuilder)`
+* `CreateSnapshotRequest#source(String)`
+* `CreateSnapshotRequest#source(byte[])`
+* `CreateSnapshotRequest#source(byte[], int, int)`
+* `CreateSnapshotRequest#source(BytesReference)`
+* `RestoreSnapshotRequest#source(XContentBuilder)`
+* `RestoreSnapshotRequest#source(String)`
+* `RestoreSnapshotRequest#source(byte[])`
+* `RestoreSnapshotRequest#source(byte[], int, int)`
+* `RestoreSnapshotRequest#source(BytesReference)`
+* `RolloverRequest#source(BytesReference)`
+* `ShrinkRequest#source(BytesReference)`
+* `UpdateRequest#fromXContent(BytesReference)`
+
+Please use the non-`source` methods instead (like `settings` and `type`).
+
+[float]
+==== Timestamp meta-data field type for ingest processors has changed
+
+The type of the "timestamp" meta-data field for ingest processors has changed from `java.lang.String` to `java.util.Date`.
+
+=== Shadow Replicas are deprecated
+
+<<indices-shadow-replicas,Shadow Replicas>> don't see much usage and we are planning to remove them.
diff --git a/docs/reference/migration/migrate_5_3.asciidoc b/docs/reference/migration/migrate_5_3.asciidoc
new file mode 100644
index 0000000000000..d65fef37f3b7f
--- /dev/null
+++ b/docs/reference/migration/migrate_5_3.asciidoc
@@ -0,0 +1,61 @@
+[[breaking-changes-5.3]]
+== Breaking changes in 5.3
+
+[[breaking_53_packaging_changes]]
+[float]
+=== Packaging changes
+
+[float]
+==== Logging configuration
+
+Previously Elasticsearch exposed a single system property (`es.logs`) that
+included the absolute path to the configured logs directory, and the prefix of
+the filenames used for the various logging files (the main log file, the
+deprecation log, and the slow logs). This property has been replaced in favor of
+three properties:
+
+ * `es.logs.base_path`: the absolute path to the configured logs directory
+ * `es.logs.cluster_name`: the default prefix of the filenames used for the
+   various logging files
+ * `es.logs.node_name`: exposed if `node.name` is configured for inclusion in
+   the filenames of the various logging files (if you prefer)
+
+The property `es.logs` is deprecated and will be removed in Elasticsearch 6.0.0.
+
+[float]
+==== Use of Netty 3 is deprecated
+
+Usage of Netty 3 for transport (`transport.type=netty3`) or HTTP
+(`http.type=netty3`) is deprecated and will be removed in Elasticsearch
+6.0.0.
+
+[[breaking_53_settings_changes]]
+[float]
+=== Settings changes
+
+[float]
+==== Lenient boolean representations are deprecated
+
+Usage of any value other than `false`, `"false"`, `true` and `"true"` in boolean
+settings deprecated.
+
+[[breaking_53_rest_api_changes]]
+[float]
+=== REST API changes
+
+[float]
+==== Lenient boolean representations are deprecated
+
+Usage of any value other than `false`, `"false"`, `true` and `"true"` for
+boolean request parameters and boolean properties in the body of REST API calls
+is deprecated.
+
+[[breaking_53_mapping_changes]]
+[float]
+=== Mapping changes
+
+[float]
+==== Lenient boolean representations are deprecated
+
+Usage of any value other than `false`, `"false"`, `true` and `"true"` for
+boolean values in mappings is deprecated.
diff --git a/docs/reference/migration/migrate_5_4.asciidoc b/docs/reference/migration/migrate_5_4.asciidoc
new file mode 100644
index 0000000000000..2f1d8b6ab4da9
--- /dev/null
+++ b/docs/reference/migration/migrate_5_4.asciidoc
@@ -0,0 +1,8 @@
+[[breaking-changes-5.4]]
+== Breaking changes in 5.4
+
+[[breaking_54_delete_by_query_changes]]
+[float]
+=== Delete By Query API changes
+
+Requests without an explicit query are deprecated and will be invalid in Elasticsearch 6.0.0.
diff --git a/docs/reference/migration/migrate_5_5.asciidoc b/docs/reference/migration/migrate_5_5.asciidoc
new file mode 100644
index 0000000000000..2bd9a89b97948
--- /dev/null
+++ b/docs/reference/migration/migrate_5_5.asciidoc
@@ -0,0 +1,30 @@
+[[breaking-changes-5.5]]
+== Breaking changes in 5.5
+
+[[breaking_55_plugins]]
+[float]
+=== Ignoring hidden folders
+
+Previous versions of Elasticsearch would skip hidden files and directories when
+scanning the plugins folder. This allows broken plugin installs to remain
+undetected so this leniency has been removed.
+
+[[breaking_55_packaging_changes]]
+[float]
+=== Skip setting kernel parameters
+
+Elasticsearch attempts to set the kernel parameter `vm.max_map_count` during
+installation. In some environments this can fail. Previous versions of
+Elasticsearch accepted a flag `ES_SKIP_SET_KERNEL_PARAMTERS` to skip this
+writing. Starting in Elasticsearch 5.5.0, this flag has been removed and
+instead the same functionality can be achieved by masking the `systemd-sysctl`
+service.
+
+[[breaking_55_java_api_changes]]
+[float]
+=== Java API changes
+
+The `has_child`, `has_parent` and `parent_id` queries have moved to parent-join module. Also the
+`children` aggregation has been moved to the parent-join module.
+The parent-join module should be loaded when using the transport client. Alternatively use the `PreBuiltTransportClient`
+client, which load the module automatically.
diff --git a/docs/reference/migration/migrate_5_6.asciidoc b/docs/reference/migration/migrate_5_6.asciidoc
new file mode 100644
index 0000000000000..a21bc76a0ea18
--- /dev/null
+++ b/docs/reference/migration/migrate_5_6.asciidoc
@@ -0,0 +1,23 @@
+[[breaking-changes-5.6]]
+== Breaking changes in 5.6
+
+[[breaking_56_clients]]
+[float]
+=== Java High Level REST Client
+
+Starting from version 5.6.0 a new Java client has been released: the Java High Level REST Client.
+This official high-level client (named like this to differentiate it from the existing low-level client) for
+Elasticsearch can be used to execute search, index, delete, update and bulk operations using the same Core
+Java classes as the `TransportClient` uses.
+
+This Java High Level REST Client is designed to replace the `TransportClient` in a near future.
+
+[[breaking_56_packaging_changes]]
+[float]
+=== RPM and Debian packages `setgid` on `/etc/elasticsearch`
+
+The RPM and Debian packages now set the `setgid` flag on `/etc/elasticsearch` so
+that any files and subdirectories created under `/etc/elasticsearch` have the
+same ownership as `/etc/elasticsearch` (`root:elasticsearch`). This is used to
+ensure that files and subdirectories created under `/etc/elasticsearch` (e.g., a
+keystore) are readable by the `elasticsearch` user.
diff --git a/docs/reference/modules.asciidoc b/docs/reference/modules.asciidoc
index 5a39cdfd7905c..32477e7251bbc 100644
--- a/docs/reference/modules.asciidoc
+++ b/docs/reference/modules.asciidoc
@@ -45,10 +45,10 @@ The modules in this section are:
 <<modules-node,Node client>>::
 
     A Java node client joins the cluster, but doesn't hold data or act as a master node.
-    
+
 <<modules-scripting-painless,Painless>>::
 
-    A built-in scripting language for Elasticsearch that's designed to be as secure as possible. 
+    A built-in scripting language for Elasticsearch that's designed to be as secure as possible.
 
 <<modules-plugins,Plugins>>::
 
@@ -57,7 +57,7 @@ The modules in this section are:
 <<modules-scripting,Scripting>>::
 
     Custom scripting available in Lucene Expressions, Groovy, Python, and
-    Javascript. You can also write scripts in the built-in scripting language, 
+    Javascript. You can also write scripts in the built-in scripting language,
     <<modules-scripting-painless, Painless>>.
 
 <<modules-snapshots,Snapshot/Restore>>::
@@ -77,6 +77,11 @@ The modules in this section are:
 
     A tribe node joins one or more clusters and acts as a federated
     client across them.
+
+<<modules-cross-cluster-search, Cross cluster Search>>::
+
+    Cross cluster search enables executing search requests across more than one cluster without joining them and acts
+    as a federated client across them.
 --
 
 
@@ -94,6 +99,7 @@ include::modules/network.asciidoc[]
 
 include::modules/node.asciidoc[]
 
+:edit_url:
 include::modules/plugins.asciidoc[]
 
 include::modules/scripting.asciidoc[]
@@ -106,6 +112,4 @@ include::modules/transport.asciidoc[]
 
 include::modules/tribe.asciidoc[]
 
-
-
-
+include::modules/cross-cluster-search.asciidoc[]
diff --git a/docs/reference/modules/cluster/allocation_awareness.asciidoc b/docs/reference/modules/cluster/allocation_awareness.asciidoc
index f4e61fb0da107..8ffa4e6b06d2a 100644
--- a/docs/reference/modules/cluster/allocation_awareness.asciidoc
+++ b/docs/reference/modules/cluster/allocation_awareness.asciidoc
@@ -104,7 +104,7 @@ cluster.routing.allocation.awareness.attributes: zone
 
 Now, if we start 2 nodes with `node.attr.zone` set to `zone1` and create an index
 with 5 shards and 1 replica. The index will be created, but only the 5 primary
-shards will be allocated (with no replicas). Only when we start more shards
+shards will be allocated (with no replicas). Only when we start more nodes
 with `node.attr.zone` set to `zone2` will the replicas be allocated.
 
 The `cluster.routing.allocation.awareness.*` settings can all be updated
diff --git a/docs/reference/modules/cluster/allocation_filtering.asciidoc b/docs/reference/modules/cluster/allocation_filtering.asciidoc
index 437f243c01810..bcaecbf0434f2 100644
--- a/docs/reference/modules/cluster/allocation_filtering.asciidoc
+++ b/docs/reference/modules/cluster/allocation_filtering.asciidoc
@@ -22,6 +22,7 @@ PUT _cluster/settings
 }
 --------------------------------------------------
 // CONSOLE
+// TEST[warning:[script.max_compilations_per_minute] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version.]
 
 NOTE: Shards will only be relocated if it is possible to do so without
 breaking another routing constraint, such as never allocating a primary and
diff --git a/docs/reference/modules/cluster/disk_allocator.asciidoc b/docs/reference/modules/cluster/disk_allocator.asciidoc
index b6289e37c731f..4f45136ac61d1 100644
--- a/docs/reference/modules/cluster/disk_allocator.asciidoc
+++ b/docs/reference/modules/cluster/disk_allocator.asciidoc
@@ -66,6 +66,7 @@ PUT _cluster/settings
 }
 --------------------------------------------------
 // CONSOLE
+// TEST[warning:[script.max_compilations_per_minute] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version.]
 
 NOTE: Prior to 2.0.0, when using multiple data paths, the disk threshold
 decider only factored in the usage across all data paths (if you had two
diff --git a/docs/reference/modules/cluster/misc.asciidoc b/docs/reference/modules/cluster/misc.asciidoc
index 57b6e8ea5c91e..25d9131860e67 100644
--- a/docs/reference/modules/cluster/misc.asciidoc
+++ b/docs/reference/modules/cluster/misc.asciidoc
@@ -12,6 +12,11 @@ An entire cluster may be set to read-only with the following _dynamic_ setting:
       operations), metadata is not allowed to be modified (create or delete
       indices).
 
+`cluster.blocks.read_only_allow_delete`::
+
+      Identical to `cluster.blocks.read_only` but allows to delete indices
+      to free up resources.
+
 WARNING: Don't rely on this setting to prevent changes to your cluster. Any
 user with access to the <<cluster-update-settings,cluster-update-settings>>
 API can make the cluster read-write again.
@@ -20,18 +25,18 @@ API can make the cluster read-write again.
 [[cluster-max-tombstones]]
 ==== Index Tombstones
 
-The cluster state maintains index tombstones to explicitly denote indices that 
-have been deleted.  The number of tombstones maintained in the cluster state is 
+The cluster state maintains index tombstones to explicitly denote indices that
+have been deleted.  The number of tombstones maintained in the cluster state is
 controlled by the following property, which cannot be updated dynamically:
 
 `cluster.indices.tombstones.size`::
 
-Index tombstones prevent nodes that are not part of the cluster when a delete 
-occurs from joining the cluster and reimporting the index as though the delete 
-was never issued. To keep the cluster state from growing huge we only keep the 
-last `cluster.indices.tombstones.size` deletes, which defaults to 500. You can 
-increase it if you expect nodes to be absent from the cluster and miss more 
-than 500 deletes. We think that is rare, thus the default. Tombstones don't take 
+Index tombstones prevent nodes that are not part of the cluster when a delete
+occurs from joining the cluster and reimporting the index as though the delete
+was never issued. To keep the cluster state from growing huge we only keep the
+last `cluster.indices.tombstones.size` deletes, which defaults to 500. You can
+increase it if you expect nodes to be absent from the cluster and miss more
+than 500 deletes. We think that is rare, thus the default. Tombstones don't take
 up much space, but we also think that a number like 50,000 is probably too big.
 
 [[cluster-logger]]
@@ -46,8 +51,9 @@ The settings which control logging can be updated dynamically with the
 PUT /_cluster/settings
 {
   "transient": {
-    "logger.indices.recovery": "DEBUG"
+    "logger.org.elasticsearch.indices.recovery": "DEBUG"
   }
 }
 -------------------------------
-
+// CONSOLE
+// TEST[warning:[script.max_compilations_per_minute] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version.]
diff --git a/docs/reference/modules/cluster/shards_allocation.asciidoc b/docs/reference/modules/cluster/shards_allocation.asciidoc
index c5b2197903021..4274764119b0b 100644
--- a/docs/reference/modules/cluster/shards_allocation.asciidoc
+++ b/docs/reference/modules/cluster/shards_allocation.asciidoc
@@ -37,6 +37,12 @@ one of the active allocation ids in the cluster state.
      How many concurrent outgoing shard recoveries are allowed to happen on a node. Outgoing recoveries are the recoveries
      where the source shard (most likely the primary unless a shard is relocating) is allocated on the node. Defaults to `2`.
 
+`cluster.routing.allocation.node_concurrent_recoveries`::
+
+     A shortcut to set both `cluster.routing.allocation.node_concurrent_incoming_recoveries` and
+     `cluster.routing.allocation.node_concurrent_outgoing_recoveries`.
+
+
 `cluster.routing.allocation.node_initial_primaries_recoveries`::
 
     While the recovery of replicas happens over the network, the recovery of
@@ -85,24 +91,28 @@ Specify when shard rebalancing is allowed:
 `cluster.routing.allocation.cluster_concurrent_rebalance`::
 
       Allow to control how many concurrent shard rebalances are
-      allowed cluster wide. Defaults to `2`.
+      allowed cluster wide. Defaults to `2`. Note that this setting
+      only controls the number of concurrent shard relocations due
+      to imbalances in the cluster. This setting does not limit shard
+      relocations due to <<allocation-filtering,allocation filtering>>
+      or <<forced-awareness,forced awareness>>.
 
 [float]
 === Shard Balancing Heuristics
 
 The following settings are used together to determine where to place each
-shard.  The cluster is balanced when no allowed action can bring the weights
-of each node closer together by more then the `balance.threshold`.
+shard.  The cluster is balanced when no allowed rebalancing operation can bring the weight
+of any node closer to the weight of any other node by more than the `balance.threshold`.
 
 `cluster.routing.allocation.balance.shard`::
 
-     Defines the weight factor for shards allocated on a node
+     Defines the weight factor for the total number of shards allocated on a node
      (float). Defaults to `0.45f`.  Raising this raises the tendency to
      equalize the number of shards across all nodes in the cluster.
 
 `cluster.routing.allocation.balance.index`::
 
-     Defines a factor to the number of shards per index allocated
+     Defines the weight factor for the number of shards per index allocated
       on a specific node (float). Defaults to `0.55f`.  Raising this raises the
       tendency to equalize the number of shards per index across all nodes in
       the cluster.
diff --git a/docs/reference/modules/cross-cluster-search.asciidoc b/docs/reference/modules/cross-cluster-search.asciidoc
new file mode 100644
index 0000000000000..8bd1946eace6b
--- /dev/null
+++ b/docs/reference/modules/cross-cluster-search.asciidoc
@@ -0,0 +1,209 @@
+[[modules-cross-cluster-search]]
+== Cross Cluster Search
+
+beta[]
+
+The _cross cluster search_ feature allows any node to act as a federated client across
+multiple clusters. In contrast to the <<modules-tribe,tribe node>> feature, a  cross cluster search node won't
+join the remote cluster, instead it connects to a remote cluster in a light fashion in order to execute
+federated search requests.
+
+Cross cluster search works by configuring a remote cluster in the cluster state and connecting only to a
+limited number of nodes in the remote cluster. Each remote cluster is referenced by a name and a list of seed nodes.
+When a remote cluster is registered, its cluster state is retrieved from one of the seed nodes so that up to 3
+_gateway nodes_ are selected to be connected to as part of upcoming cross cluster search requests.
+Cross cluster search requests consist of uni-directional connections from the coordinating node to the previously
+selected remote nodes only. It is possible to tag which nodes should be selected through
+node attributes (see <<cross-cluster-search-settings>>).
+
+Each node in a cluster that has remote clusters configured connects to one or more _gateway nodes_ and uses
+them to federate search requests to the remote cluster.
+
+[float]
+=== Configuring Cross Cluster Search
+
+Remote clusters can be specified globally using <<cluster-update-settings,cluster settings>>
+(which can be updated dynamically), or local to individual nodes using the
+`elasticsearch.yml` file.
+
+If a remote cluster is configured via `elasticsearch.yml` only the nodes with
+that configuration will be able to connect to the remote cluster. In other
+words, federated search requests will have to be sent specifically to those
+nodes. Remote clusters set via the <<cluster-update-settings,cluster settings API>>
+will be available on every node in the cluster.
+
+[WARNING]
+This feature was added as Beta in Elasticsearch `v5.3` with further improvements made in 5.4 and 5.5. It requires gateway eligible nodes to be on `v5.5` onwards.
+
+The `elasticsearch.yml` config file for a _cross cluster search_ node just needs to list the
+remote clusters that should be connected to, for instance:
+
+[source,yaml]
+--------------------------------
+search:
+    remote:
+        cluster_one: <1>
+            seeds: 127.0.0.1:9300
+        cluster_two: <1>
+            seeds: 127.0.0.1:9301
+
+--------------------------------
+<1> `cluster_one` and `cluster_two` are arbitrary cluster aliases representing the connection to each cluster.
+These names are subsequently used to distinguish between local and remote indices.
+
+The equivalent example using the <<cluster-update-settings,cluster settings API>>
+to add remote clusters to all nodes in the cluster would look like the
+following:
+
+[source,js]
+--------------------------------
+PUT _cluster/settings
+{
+  "persistent": {
+    "search": {
+      "remote": {
+        "cluster_one": {
+          "seeds": [
+            "127.0.0.1:9300"
+          ]
+        },
+        "cluster_two": {
+          "seeds": [
+            "127.0.0.1:9301"
+          ]
+        }
+      }
+    }
+  }
+}
+--------------------------------
+// CONSOLE
+// TEST[warning:[script.max_compilations_per_minute] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version.]
+
+A remote cluster can be deleted from the cluster settings by setting its seeds to `null`:
+
+[source,js]
+--------------------------------
+PUT _cluster/settings
+{
+  "persistent": {
+    "search": {
+      "remote": {
+        "cluster_one": {
+          "seeds": null <1>
+        }
+      }
+    }
+  }
+}
+--------------------------------
+// CONSOLE
+<1> `cluster_one` would be removed from the cluster settings, leaving `cluster_two` intact.
+
+
+[float]
+=== Using cross cluster search
+
+To search the `twitter` index on remote cluster `cluster_1` the index name must be prefixed with the cluster alias
+separated by a `:` character:
+
+[source,js]
+--------------------------------------------------
+POST /cluster_one:twitter/tweet/_search
+{
+  "query": {
+    "match_all": {}
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[skip:we don't have two clusters set up during docs testing]
+
+In contrast to the `tribe` feature cross cluster search can also search indices with the same name on different
+clusters:
+
+[source,js]
+--------------------------------------------------
+POST /cluster_one:twitter,twitter/tweet/_search
+{
+  "query": {
+    "match_all": {}
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[skip:we don't have two clusters set up during docs testing]
+
+Search results are disambiguated the same way as the indices are disambiguated in the request. Even if index names are
+identical these indices will be treated as different indices when results are merged. All results retrieved from a
+remote index
+will be prefixed with their remote cluster name:
+
+[source,js]
+--------------------------------------------------
+{
+  "took" : 89,
+  "timed_out" : false,
+  "_shards" : {
+    "total" : 10,
+    "successful" : 10,
+    "failed" : 0
+  },
+  "hits" : {
+    "total" : 2,
+    "max_score" : 1.0,
+    "hits" : [
+      {
+        "_index" : "cluster_one:twitter",
+        "_type" : "tweet",
+        "_id" : "1",
+        "_score" : 1.0,
+        "_source" : {
+          "user" : "kimchy",
+          "postDate" : "2009-11-15T14:12:12",
+          "message" : "trying out Elasticsearch"
+        }
+      },
+      {
+        "_index" : "twitter",
+        "_type" : "tweet",
+        "_id" : "1",
+        "_score" : 1.0,
+        "_source" : {
+          "user" : "kimchy",
+          "postDate" : "2009-11-15T14:12:12",
+          "message" : "trying out Elasticsearch"
+        }
+      }
+    ]
+  }
+}
+--------------------------------------------------
+// TESTRESPONSE
+
+[float]
+[[cross-cluster-search-settings]]
+=== Cross cluster search settings
+
+`search.remote.connections_per_cluster`::
+
+  The number of nodes to connect to per remote cluster. The default is `3`.
+
+`search.remote.initial_connect_timeout`::
+
+  The time to wait for remote connections to be established when the node starts. The default is `30s`.
+
+`search.remote.node.attr`::
+
+  A node attribute to filter out nodes that are eligible as a gateway node in
+  the remote cluster. For instance a node can have a node attribute
+  `node.attr.gateway: true` such that only nodes with this attribute will be
+  connected to if `search.remote.node.attr` is set to `gateway`.
+
+`search.remote.connect`::
+
+  By default, any node in the cluster can act as a cross-cluster client and
+  connect to remote clusters. The `search.remote.connect` setting can be set
+  to `false` (defaults to `true`) to prevent certain nodes from connecting to
+  remote clusters. Cross-cluster search requests must be sent to a node that
+  is allowed to act as a cross-cluster client.
diff --git a/docs/reference/modules/discovery/zen.asciidoc b/docs/reference/modules/discovery/zen.asciidoc
index f1439adc8b6e0..29fdc7aa10257 100644
--- a/docs/reference/modules/discovery/zen.asciidoc
+++ b/docs/reference/modules/discovery/zen.asciidoc
@@ -22,21 +22,33 @@ other nodes.
 [[unicast]]
 ===== Unicast
 
-The unicast discovery requires a list of hosts to use that will act
-as gossip routers. It provides the following settings with the
-`discovery.zen.ping.unicast` prefix:
+Unicast discovery requires a list of hosts to use that will act as gossip routers. These hosts can be specified as
+hostnames or IP addresses; hosts specified as hostnames are resolved to IP addresses during each round of pinging. Note
+that with the Java security manager in place, the JVM defaults to caching positive hostname resolutions indefinitely.
+This can be modified by adding
+http://docs.oracle.com/javase/8/docs/technotes/guides/net/properties.html[`networkaddress.cache.ttl=<timeout>`] to your
+http://docs.oracle.com/javase/8/docs/technotes/guides/security/PolicyFiles.html[Java security policy]. Any hosts that
+fail to resolve will be logged. Note also that with the Java security manager in place, the JVM defaults to caching
+negative hostname resolutions for ten seconds. This can be modified by adding
+http://docs.oracle.com/javase/8/docs/technotes/guides/net/properties.html[`networkaddress.cache.negative.ttl=<timeout>`]
+to your http://docs.oracle.com/javase/8/docs/technotes/guides/security/PolicyFiles.html[Java security policy].
+
+It is recommended that the unicast hosts list be maintained as the list of
+master-eligible nodes in the cluster.
+
+Unicast discovery provides the following settings with the `discovery.zen.ping.unicast` prefix:
 
 [cols="<,<",options="header",]
 |=======================================================================
 |Setting |Description
 |`hosts` |Either an array setting or a comma delimited setting. Each
-value should be in the form of `host:port` or `host` (where `port` defaults to `9300`). Note that IPv6 hosts must be bracketed. Defaults to
-`127.0.0.1, [::1]`
+          value should be in the form of `host:port` or `host` (where `port` defaults to the setting `transport.profiles.default.port`
+          falling back to `transport.tcp.port` if not set). Note that IPv6 hosts must be bracketed. Defaults to `127.0.0.1, [::1]`
+|`hosts.resolve_timeout` |The amount of time to wait for DNS lookups on each round of pinging. Specified as
+<<time-units, time units>>. Defaults to 5s.
 |=======================================================================
 
-The unicast discovery uses the
-<<modules-transport,transport>> module to
-perform the discovery.
+The unicast discovery uses the <<modules-transport,transport>> module to perform the discovery.
 
 [float]
 [[master-election]]
@@ -68,9 +80,10 @@ complete and for the elected node to accept its mastership. The same setting con
 active master eligible nodes that should be a part of any active cluster. If this requirement is not met the
 active master node will step down and a new master election will be begin.
 
-This setting must be set to a quorum of your master eligible nodes. It is recommended to avoid
-having only two master eligible nodes, since a quorum of two is two. Therefore, a loss
-of either master node will result in an inoperable cluster.
+This setting must be set to a <<minimum_master_nodes,quorum>> of your master
+eligible nodes. It is recommended to avoid having only two master eligible
+nodes, since a quorum of two is two. Therefore, a loss of either master
+eligible node will result in an inoperable cluster.
 
 [float]
 [[fault-detection]]
@@ -131,7 +144,7 @@ The `discovery.zen.no_master_block` setting has two valid options:
 `all`:: All operations on the node--i.e. both read & writes--will be rejected. This also applies for api cluster state
 read or write operations, like the get index settings, put mapping and cluster state api.
 `write`:: (default) Write operations will be rejected. Read operations will succeed, based on the last known cluster configuration.
-This may result in partial reads of stale data as this node may be isolated from the rest of the cluster. 
+This may result in partial reads of stale data as this node may be isolated from the rest of the cluster.
 
 The `discovery.zen.no_master_block` setting doesn't apply to nodes-based apis (for example cluster stats, node info and
 node stats apis).  Requests to these apis will not be blocked and can run on any available node.
diff --git a/docs/reference/modules/gateway.asciidoc b/docs/reference/modules/gateway.asciidoc
index cad05d1baaa0c..4231bff60e3d6 100644
--- a/docs/reference/modules/gateway.asciidoc
+++ b/docs/reference/modules/gateway.asciidoc
@@ -4,9 +4,9 @@
 The local gateway module stores the cluster state and shard data across full
 cluster restarts.
 
- The following _static_ settings, which must be set on every master node,
-  control how long a freshly elected master should wait before it tries to
-  recover the cluster state and the cluster's data:
+The following _static_ settings, which must be set on every master node,
+control how long a freshly elected master should wait before it tries to
+recover the cluster state and the cluster's data:
 
 `gateway.expected_nodes`::
 
@@ -48,3 +48,12 @@ as long as the following conditions are met:
     Recover as long as this many data nodes have joined the cluster.
 
 NOTE: These settings only take effect on a full cluster restart.
+
+=== Dangling indices
+
+When a node joins the cluster, any shards/indices stored in its local `data/` 
+directory which do not already exist in the cluster will be imported into the 
+cluster. This functionality is intended as a best effort to help users who 
+lost all master nodes. If a new master node is started which is unaware of 
+the other indices in the cluster, adding the old nodes will cause the old 
+indices to be imported, instead of being deleted.
diff --git a/docs/reference/modules/http.asciidoc b/docs/reference/modules/http.asciidoc
index 065c91349c25a..3682234f5e6fb 100644
--- a/docs/reference/modules/http.asciidoc
+++ b/docs/reference/modules/http.asciidoc
@@ -100,6 +100,9 @@ simple message will be returned. Defaults to `true`
 
 |`http.pipelining.max_events` |The maximum number of events to be queued up in memory before a HTTP connection is closed, defaults to `10000`.
 
+|`http.content_type.required`|Enables or disables strict checking and usage of
+the `Content-Type` header for all requests with content, defaults to `false`.
+
 |=======================================================================
 
 It also uses the common
diff --git a/docs/reference/modules/indices/circuit_breaker.asciidoc b/docs/reference/modules/indices/circuit_breaker.asciidoc
index 04ffaf7a0cd34..63b5896ea6496 100644
--- a/docs/reference/modules/indices/circuit_breaker.asciidoc
+++ b/docs/reference/modules/indices/circuit_breaker.asciidoc
@@ -87,3 +87,5 @@ documentation for more information.
 
     Limit for the number of unique dynamic scripts within a minute that are
     allowed to be compiled. Defaults to 15.
+
+deprecated[5.6.0 This parameter is deprecated and will be replaced with `script.max_compilation_rate` in Elasticsearch 6.0]
diff --git a/docs/reference/modules/indices/recovery.asciidoc b/docs/reference/modules/indices/recovery.asciidoc
index 8de3309347c79..84352cfe6898e 100644
--- a/docs/reference/modules/indices/recovery.asciidoc
+++ b/docs/reference/modules/indices/recovery.asciidoc
@@ -1,22 +1,10 @@
 [[recovery]]
 === Indices Recovery
 
-The following _expert_ settings can be set to manage the recovery policy.
-
-`indices.recovery.file_chunk_size`::
-    Defaults to `512kb`.
-
-`indices.recovery.translog_ops`::
-    Defaults to `1000`.
-
-`indices.recovery.translog_size`::
-    Defaults to `512kb`.
-
-`indices.recovery.compress`::
-    Defaults to `true`.
+The following _expert_ setting can be set to manage the recovery policy.
 
 `indices.recovery.max_bytes_per_sec`::
     Defaults to `40mb`.
 
-These settings can be dynamically updated on a live cluster with the
+This setting can be dynamically updated on a live cluster with the
 <<cluster-update-settings,cluster-update-settings>> API:
diff --git a/docs/reference/modules/indices/request_cache.asciidoc b/docs/reference/modules/indices/request_cache.asciidoc
index 4969224bd2224..fc04c5e9c63f6 100644
--- a/docs/reference/modules/indices/request_cache.asciidoc
+++ b/docs/reference/modules/indices/request_cache.asciidoc
@@ -42,8 +42,10 @@ The cache can be expired manually with the <<indices-clearcache,`clear-cache` AP
 
 [source,js]
 ------------------------
-curl -XPOST 'localhost:9200/kimchy,elasticsearch/_cache/clear?request_cache=true'
+POST /kimchy,elasticsearch/_cache/clear?request=true
 ------------------------
+// CONSOLE
+// TEST[s/^/PUT kimchy\nPUT elasticsearch\n/]
 
 [float]
 ==== Enabling and disabling caching
@@ -53,24 +55,26 @@ index as follows:
 
 [source,js]
 -----------------------------
-curl -XPUT localhost:9200/my_index -d'
+PUT /my_index
 {
   "settings": {
     "index.requests.cache.enable": false
   }
 }
-'
 -----------------------------
+// CONSOLE
 
 It can also be enabled or disabled dynamically on an existing index with the
 <<indices-update-settings,`update-settings`>> API:
 
 [source,js]
 -----------------------------
-curl -XPUT localhost:9200/my_index/_settings -d'
+PUT /my_index/_settings
 { "index.requests.cache.enable": true }
-'
 -----------------------------
+// CONSOLE
+// TEST[continued]
+
 
 [float]
 ==== Enabling and disabling caching per request
@@ -80,7 +84,7 @@ caching on a *per-request* basis.  If set, it overrides the index-level setting:
 
 [source,js]
 -----------------------------
-curl 'localhost:9200/my_index/_search?request_cache=true' -d'
+GET /my_index/_search?request_cache=true
 {
   "size": 0,
   "aggs": {
@@ -91,14 +95,15 @@ curl 'localhost:9200/my_index/_search?request_cache=true' -d'
     }
   }
 }
-'
 -----------------------------
+// CONSOLE
+// TEST[continued]
 
 IMPORTANT: If your query uses a script whose result is not deterministic (e.g.
 it uses a random function or references the current time) you should set the
 `request_cache` flag to `false` to disable caching for that request.
 
-Requests `size` is greater than 0 will not be cached even if the request cache is
+Requests where `size` is greater than 0 will not be cached even if the request cache is
 enabled in the index settings. To cache these requests you will need to use the
 query-string parameter detailed here.
 
@@ -137,12 +142,14 @@ by index, with the <<indices-stats,`indices-stats`>> API:
 
 [source,js]
 ------------------------
-curl 'localhost:9200/_stats/request_cache?pretty&human'
+GET /_stats/request_cache?human
 ------------------------
+// CONSOLE
 
 or by node with the <<cluster-nodes-stats,`nodes-stats`>> API:
 
 [source,js]
 ------------------------
-curl 'localhost:9200/_nodes/stats/indices/request_cache?pretty&human'
+GET /_nodes/stats/indices/request_cache?human
 ------------------------
+// CONSOLE
diff --git a/docs/reference/modules/network.asciidoc b/docs/reference/modules/network.asciidoc
index d6118246b5076..8cbca1c058fe8 100644
--- a/docs/reference/modules/network.asciidoc
+++ b/docs/reference/modules/network.asciidoc
@@ -31,7 +31,12 @@ Defaults to `_local_`.
 In order to join a cluster, a node needs to know the hostname or IP address of
 at least some of the other nodes in the cluster.  This setting provides the
 initial list of other nodes that this node will try to contact. Accepts IP
-addresses or hostnames.
+addresses or hostnames.  If a hostname lookup resolves to multiple IP
+addresses then each IP address will be used for discovery.
+https://en.wikipedia.org/wiki/Round-robin_DNS[Round robin DNS] -- returning a
+different IP from a list on each lookup -- can be used for discovery; non-
+existent IP addresses will throw exceptions and cause another DNS lookup
+on the next round of pinging (subject to JVM DNS caching).
 +
 Defaults to `["127.0.0.1", "[::1]"]`.
 
@@ -111,12 +116,14 @@ two network cards, or a site-local address and a local address. Defaults to
 
 `network.publish_host`::
 
-The publish host is the single interface that the node advertises to other
-nodes in the cluster, so that those nodes can connect to it.   Currently an
-elasticsearch node may be bound to multiple addresses, but only publishes one.
-If not specified, this defaults to the ``best'' address from
-`network.host`, sorted by IPv4/IPv6 stack preference, then by
-reachability.
+The publish host is the single interface that the node advertises to other nodes
+in the cluster, so that those nodes can connect to it. Currently an
+Elasticsearch node may be bound to multiple addresses, but only publishes one.
+If not specified, this defaults to the ``best'' address from `network.host`,
+sorted by IPv4/IPv6 stack preference, then by reachability. If you set a
+`network.host` that results in multiple bind addresses yet rely on a specific
+address for node-to-node communication, you should explicitly set
+`network.publish_host`.
 
 Both of the above settings can be configured just like `network.host` -- they
 accept IP addresses, host names, and
diff --git a/docs/reference/modules/node.asciidoc b/docs/reference/modules/node.asciidoc
index e1cc3e0b8663f..5e1871adf25bf 100644
--- a/docs/reference/modules/node.asciidoc
+++ b/docs/reference/modules/node.asciidoc
@@ -8,7 +8,7 @@ then you have a cluster of one node.
 
 Every node in the cluster can handle <<modules-http,HTTP>> and
 <<modules-transport,Transport>> traffic by default. The transport layer
-is used exclusively for communication between nodes and between nodes and the
+is used exclusively for communication between nodes and the
 {javaclient}/transport-client.html[Java `TransportClient`]; the HTTP layer is
 used only by external REST clients.
 
@@ -82,7 +82,7 @@ cluster health to have a stable master node.
 Any master-eligible node (all nodes by default) may be elected to become the
 master node by the <<modules-discovery-zen,master election process>>.
 
-IMPORTANT: Master nodes must have access to the `data/` directory (just like 
+IMPORTANT: Master nodes must have access to the `data/` directory (just like
 `data` nodes) as this is where the cluster state is persisted between node restarts.
 
 Indexing and searching your data is CPU-, memory-, and I/O-intensive work
@@ -97,7 +97,7 @@ better _not_ to use dedicated master nodes for this purpose. It is important
 for the stability of the cluster that master-eligible nodes do as little work
 as possible.
 
-To create a standalone master-eligible node, set:
+To create a dedicated master-eligible node, set:
 
 [source,yaml]
 -------------------
@@ -109,6 +109,12 @@ node.ingest: false <3>
 <2> Disable the `node.data` role (enabled by default).
 <3> Disable the `node.ingest` role (enabled by default).
 
+ifdef::include-xpack[]
+NOTE: These settings apply only when {xpack} is not installed. To create a
+dedicated master-eligible node when {xpack} is installed, see <<modules-node-xpack,{xpack} node settings>>.
+endif::include-xpack[]
+
+
 [float]
 [[split-brain]]
 ==== Avoiding split brain with `minimum_master_nodes`
@@ -162,6 +168,7 @@ PUT _cluster/settings
 ----------------------------
 // CONSOLE
 // TEST[catch:/cannot set discovery.zen.minimum_master_nodes to more than the current master nodes/]
+// TEST[warning:[script.max_compilations_per_minute] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version.]
 
 TIP: An advantage of splitting the master and data roles between dedicated
 nodes is that you can have just three master-eligible nodes and set
@@ -193,6 +200,11 @@ node.ingest: false <3>
 <2> The `node.data` role is enabled by default.
 <3> Disable the `node.ingest` role (enabled by default).
 
+ifdef::include-xpack[]
+NOTE: These settings apply only when {xpack} is not installed. To create a
+dedicated data node when {xpack} is installed, see <<modules-node-xpack,{xpack} node settings>>.
+endif::include-xpack[]
+
 [float]
 [[node-ingest-node]]
 === Ingest Node
@@ -209,10 +221,17 @@ To create a dedicated ingest node, set:
 node.master: false <1>
 node.data: false <2>
 node.ingest: true <3>
+search.remote.connect: false <4>
 -------------------
 <1> Disable the `node.master` role (enabled by default).
 <2> Disable the `node.data` role (enabled by default).
 <3> The `node.ingest` role is enabled by default.
+<4> Disable cross-cluster search (enabled by default).
+
+ifdef::include-xpack[]
+NOTE: These settings apply only when {xpack} is not installed. To create a
+dedicated ingest node when {xpack} is installed, see <<modules-node-xpack,{xpack} node settings>>.
+endif::include-xpack[]
 
 [float]
 [[coordinating-only-node]]
@@ -235,17 +254,24 @@ acknowledgement of cluster state updates from every node! The benefit of
 coordinating only nodes should not be overstated -- data nodes can happily
 serve the same purpose.
 
-To create a coordinating only node, set:
+To create a dedicated coordinating node, set:
 
 [source,yaml]
 -------------------
 node.master: false <1>
 node.data: false <2>
 node.ingest: false <3>
+search.remote.connect: false <4>
 -------------------
 <1> Disable the `node.master` role (enabled by default).
 <2> Disable the `node.data` role (enabled by default).
 <3> Disable the `node.ingest` role (enabled by default).
+<4> Disable cross-cluster search (enabled by default).
+
+ifdef::include-xpack[]
+NOTE: These settings apply only when {xpack} is not installed. To create a
+dedicated coordinating node when {xpack} is installed, see <<modules-node-xpack,{xpack} node settings>>.
+endif::include-xpack[]
 
 [float]
 == Node data path settings
@@ -287,7 +313,7 @@ machine. In production, however, it is recommended to run only one node of Elast
 
 By default, Elasticsearch is configured to prevent more than one node from sharing the same data
 path. To allow for more than one node (e.g., on your development machine), use the setting
-`node.max_local_storage_nodes` and set this to a positve integer larger than one.
+`node.max_local_storage_nodes` and set this to a positive integer larger than one.
 
 WARNING: Never run different node types (i.e. master, data) from the same data directory. This can
 lead to unexpected data loss.
@@ -298,3 +324,8 @@ lead to unexpected data loss.
 More node settings can be found in <<modules,Modules>>.  Of particular note are
 the <<cluster.name,`cluster.name`>>, the <<node.name,`node.name`>> and the
 <<modules-network,network settings>>.
+
+ifdef::include-xpack[]
+:edit_url!:
+include::{xes-repo-dir}/node.asciidoc[]
+endif::include-xpack[]
diff --git a/docs/reference/modules/scripting.asciidoc b/docs/reference/modules/scripting.asciidoc
index 253fab3941b98..d9965f067fdf2 100644
--- a/docs/reference/modules/scripting.asciidoc
+++ b/docs/reference/modules/scripting.asciidoc
@@ -63,7 +63,7 @@ certain tasks.
     |built-in
     |templates
 
-|<<modules-scripting-native, `java`>>
+|<<modules-scripting-engine, `java`>>
     |n/a
     |you write it!
     |expert API
@@ -91,11 +91,6 @@ include::scripting/groovy.asciidoc[]
 
 include::scripting/painless.asciidoc[]
 
-include::scripting/painless-syntax.asciidoc[]
-
 include::scripting/expression.asciidoc[]
 
-include::scripting/native.asciidoc[]
-
-include::scripting/advanced-scripting.asciidoc[]
-
+include::scripting/engine.asciidoc[]
diff --git a/docs/reference/modules/scripting/advanced-scripting.asciidoc b/docs/reference/modules/scripting/advanced-scripting.asciidoc
deleted file mode 100644
index b4053d765980a..0000000000000
--- a/docs/reference/modules/scripting/advanced-scripting.asciidoc
+++ /dev/null
@@ -1,187 +0,0 @@
-[[modules-advanced-scripting]]
-=== Advanced text scoring in scripts
-
-experimental[The functionality described on this page is considered experimental and may be changed or removed in a future release]
-
-Text features, such as term or document frequency for a specific term can be
-accessed in scripts with the `_index` variable. This can be useful if, for
-example, you want to implement your own scoring model using for example a
-script inside a <<query-dsl-function-score-query,function score query>>.
-Statistics over the document collection are computed *per shard*, not per
-index.
-
-[float]
-=== Nomenclature:
-
-
-[horizontal]
-`df`::
-
-    document frequency. The number of documents a term appears in. Computed
-    per field.
-
-
-`tf`::
-
-    term frequency. The number times a term appears in a field in one specific
-    document.
-
-`ttf`::
-
-    total term frequency. The number of times this term appears in all
-    documents, that is, the sum of `tf` over all documents.  Computed per
-    field.
-
-`df` and `ttf` are computed per shard and therefore these numbers can vary
-depending on the shard the current document resides in.
-
-
-[float]
-=== Shard statistics:
-
-`_index.numDocs()`::
-
-    Number of documents in shard.
-
-`_index.maxDoc()`::
-
-    Maximal document number in shard.
-
-`_index.numDeletedDocs()`::
-
-    Number of deleted documents in shard.
-
-
-[float]
-=== Field statistics:
-
-Field statistics can be accessed with a subscript operator like this:
-`_index['FIELD']`.
-
-
-`_index['FIELD'].docCount()`::
-
-    Number of documents containing the field `FIELD`. Does not take deleted documents into account.
-
-`_index['FIELD'].sumttf()`::
-
-    Sum of `ttf` over all terms that appear in field `FIELD` in all documents.
-
-`_index['FIELD'].sumdf()`::
-
-    The sum of `df` s over all terms that appear in field `FIELD` in all
-    documents.
-
-
-Field statistics are computed per shard and therefore these numbers can vary
-depending on the shard the current document resides in.
-The number of terms in a field cannot be accessed using the `_index` variable. See <<token-count>> for how to do that.
-
-[float]
-=== Term statistics:
-
-Term statistics for a field can be accessed with a subscript operator like
-this: `_index['FIELD']['TERM']`. This will never return null, even if term or field does not exist.
-If you do not need the term frequency, call `_index['FIELD'].get('TERM', 0)`
-to avoid unnecessary initialization of the frequencies. The flag will have only
-affect is your set the <<index-options,`index_options`>> to `docs`.
-
-
-`_index['FIELD']['TERM'].df()`::
-
-    `df` of term `TERM` in field `FIELD`. Will be returned, even if the term
-    is not present in the current document.
-
-`_index['FIELD']['TERM'].ttf()`::
-
-    The sum of term frequencies of term `TERM` in field `FIELD` over all
-    documents. Will be returned, even if the term is not present in the
-    current document.
-
-`_index['FIELD']['TERM'].tf()`::
-
-    `tf` of term `TERM` in field `FIELD`. Will be 0 if the term is not present
-    in the current document.
-
-
-[float]
-=== Term positions, offsets and payloads:
-
-If you need information on the positions of terms in a field, call
-`_index['FIELD'].get('TERM', flag)` where flag can be
-
-[horizontal]
-`_POSITIONS`::      if you need the positions of the term
-`_OFFSETS`::        if you need the offsets of the term
-`_PAYLOADS`::       if you need the payloads of the term
-`_CACHE`::          if you need to iterate over all positions several times
-
-The iterator uses the underlying lucene classes to iterate over positions. For efficiency reasons, you can only iterate over positions once. If you need to iterate over the positions several times, set the `_CACHE` flag.
-
-You can combine the operators with a `|` if you need more than one info. For
-example, the following will return an object holding the positions and payloads,
-as well as all statistics:
-
-
-    `_index['FIELD'].get('TERM', _POSITIONS | _PAYLOADS)`
-
-
-Positions can be accessed with an iterator that returns an object
-(`POS_OBJECT`) holding position, offsets and payload for each term position.
-
-`POS_OBJECT.position`::
-
-    The position of the term.
-
-`POS_OBJECT.startOffset`::
-
-    The start offset of the term.
-
-`POS_OBJECT.endOffset`::
-
-    The end offset of the term.
-
-`POS_OBJECT.payload`::
-
-    The payload of the term.
-
-`POS_OBJECT.payloadAsInt(missingValue)`::
-
-    The payload of the term converted to integer. If the current position has
-    no payload, the `missingValue` will be returned. Call this only if you
-    know that your payloads are integers.
-
-`POS_OBJECT.payloadAsFloat(missingValue)`::
-
-    The payload of the term converted to float. If the current position has no
-    payload, the `missingValue` will be returned. Call this only if you know
-    that your payloads are floats.
-
-`POS_OBJECT.payloadAsString()`::
-
-    The payload of the term converted to string. If the current position has
-    no payload, `null` will be returned. Call this only if you know that your
-    payloads are strings.
-
-
-Example: sums up all payloads for the term `foo`.
-
-[source,groovy]
----------------------------------------------------------
-termInfo = _index['my_field'].get('foo',_PAYLOADS);
-score = 0;
-for (pos in termInfo) {
-    score = score + pos.payloadAsInt(0);
-}
-return score;
----------------------------------------------------------
-
-
-[float]
-=== Term vectors:
-
-The `_index` variable can only be used to gather statistics for single terms. If you want to use information on all terms in a field, you must store the term vectors (see <<term-vector>>). To access them, call
-`_index.termVectors()` to get a
-https://lucene.apache.org/core/4_0_0/core/org/apache/lucene/index/Fields.html[Fields]
-instance. This object can then be used as described in https://lucene.apache.org/core/4_0_0/core/org/apache/lucene/index/Fields.html[lucene doc] to iterate over fields and then for each field iterate over each term in the field.
-The method will return null if the term vectors were not stored.
diff --git a/docs/reference/modules/scripting/engine.asciidoc b/docs/reference/modules/scripting/engine.asciidoc
new file mode 100644
index 0000000000000..37baa0801c9aa
--- /dev/null
+++ b/docs/reference/modules/scripting/engine.asciidoc
@@ -0,0 +1,57 @@
+[[modules-scripting-engine]]
+=== Advanced scripts using script engines
+
+A `ScriptEngine` is a backend for implementing a scripting language. It may also
+be used to write scripts that need to use advanced internals of scripting. For example,
+a script that wants to use term frequencies while scoring.
+
+The plugin {plugins}/plugin-authors.html[documentation] has more information on
+how to write a plugin so that Elasticsearch will properly load it. To register
+the `ScriptEngine`, your plugin should implement the `ScriptPlugin` interface
+and override the `getScriptEngine(Settings settings)` method.
+
+The following is an example of a custom `ScriptEngine` which uses the language
+name `expert_scripts`. It implements a single script called `pure_df` which
+may be used as a search script to override each document's score as
+the document frequency of a provided term.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{plugins-examples-dir}/script-expert-scoring/src/main/java/org/elasticsearch/example/expertscript/ExpertScriptPlugin.java[expert_engine]
+--------------------------------------------------
+
+You can execute the script by specifying its `lang` as `expert_scripts`, and the name
+of the script as the the script source:
+
+
+[source,js]
+--------------------------------------------------
+POST /_search
+{
+  "query": {
+    "function_score": {
+      "query": {
+        "match": {
+          "body": "foo"
+        }
+      },
+      "functions": [
+        {
+          "script_score": {
+            "script": {
+                "source": "pure_df",
+                "lang" : "expert_scripts",
+                "params": {
+                    "field": "body",
+                    "term": "foo"
+                }
+            }
+          }
+        }
+      ]
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[skip:we don't have an expert script plugin installed to test this]
diff --git a/docs/reference/modules/scripting/fields.asciidoc b/docs/reference/modules/scripting/fields.asciidoc
index 35e1488fc28b5..306fb81107d65 100644
--- a/docs/reference/modules/scripting/fields.asciidoc
+++ b/docs/reference/modules/scripting/fields.asciidoc
@@ -48,13 +48,13 @@ relevance `_score` of each document:
 
 [source,js]
 -------------------------------------
-PUT my_index/my_type/1
+PUT my_index/my_type/1?refresh
 {
   "text": "quick brown fox",
   "popularity": 1
 }
 
-PUT my_index/my_type/2
+PUT my_index/my_type/2?refresh
 {
   "text": "quick fox",
   "popularity": 5
@@ -72,7 +72,7 @@ GET my_index/_search
       "script_score": {
         "script": {
           "lang": "expression",
-          "inline": "_score * doc['popularity']"
+          "source": "_score * doc['popularity']"
         }
       }
     }
@@ -93,7 +93,7 @@ store, enabled by default on all fields except for <<text,analyzed `text` fields
 
 [source,js]
 -------------------------------
-PUT my_index/my_type/1
+PUT my_index/my_type/1?refresh
 {
   "cost_price": 100
 }
@@ -104,7 +104,7 @@ GET my_index/_search
     "sales_price": {
       "script": {
         "lang":   "expression",
-        "inline": "doc['cost_price'] * markup",
+        "source": "doc['cost_price'] * markup",
         "params": {
           "markup": 0.2
         }
@@ -186,7 +186,7 @@ PUT my_index
   }
 }
 
-PUT my_index/my_type/1
+PUT my_index/my_type/1?refresh
 {
   "title": "Mr",
   "first_name": "Barry",
@@ -198,14 +198,12 @@ GET my_index/_search
   "script_fields": {
     "source": {
       "script": {
-        "lang": "groovy",
-        "inline": "_source.title + ' ' + _source.first_name + ' ' + _source.last_name" <2>
+        "source": "params._source.title + ' ' + params._source.first_name + ' ' + params._source.last_name" <2>
       }
     },
     "stored_fields": {
       "script": {
-        "lang": "groovy",
-        "inline": "_fields['first_name'].value + ' ' + _fields['last_name'].value"
+        "source": "params._fields['first_name'].value + ' ' + params._fields['last_name'].value"
       }
     }
   }
@@ -229,4 +227,3 @@ The only time it really makes sense to use stored fields instead of the
 access a few small stored fields instead of the entire `_source`.
 
 =======================================================
-
diff --git a/docs/reference/modules/scripting/groovy.asciidoc b/docs/reference/modules/scripting/groovy.asciidoc
index aaacd85f24377..24974e5733f28 100644
--- a/docs/reference/modules/scripting/groovy.asciidoc
+++ b/docs/reference/modules/scripting/groovy.asciidoc
@@ -3,7 +3,7 @@
 
 deprecated[5.0.0,Groovy will be replaced by the new scripting language <<modules-scripting-painless, `Painless`>>]
 
-Groovy is the default scripting language available in Elasticsearch.  Although
+Groovy is available in Elasticsearch by default.  Although
 limited by the <<java-security-manager,Java Security Manager>>, it is not a
 sandboxed language and only `file` scripts may be used by default.
 
diff --git a/docs/reference/modules/scripting/native.asciidoc b/docs/reference/modules/scripting/native.asciidoc
deleted file mode 100644
index 9580720d9675a..0000000000000
--- a/docs/reference/modules/scripting/native.asciidoc
+++ /dev/null
@@ -1,84 +0,0 @@
-[[modules-scripting-native]]
-=== Native (Java) Scripts
-
-Sometimes `groovy` and <<modules-scripting-expression, expression>> aren't enough. For those times you can
-implement a native script.
-
-The best way to implement a native script is to write a plugin and install it.
-The plugin {plugins}/plugin-authors.html[documentation] has more information on
-how to write a plugin so that Elasticsearch will properly load it.
-
-To register the actual script you'll need to implement `NativeScriptFactory`
-to construct the script. The actual script will extend either
-`AbstractExecutableScript` or `AbstractSearchScript`. The second one is likely
-the most useful and has several helpful subclasses you can extend like
-`AbstractLongSearchScript` and `AbstractDoubleSearchScript`.
-Finally, your plugin should register the native
-script by declaring the `onModule(ScriptModule)` method.
-
-If you squashed the whole thing into one class it'd look like:
-
-[source,java]
---------------------------------------------------
-public class MyNativeScriptPlugin extends Plugin {
-    @Override
-    public String name() {
-        return "my-native-script";
-    }
-    @Override
-    public String description() {
-        return "my native script that does something great";
-    }
-    public void onModule(ScriptModule scriptModule) {
-        scriptModule.registerScript("my_script", MyNativeScriptFactory.class);
-    }
-
-    public static class MyNativeScriptFactory implements NativeScriptFactory {
-        @Override
-        public ExecutableScript newScript(@Nullable Map<String, Object> params) {
-            return new MyNativeScript();
-        }
-        @Override
-        public boolean needsScores() {
-            return false;
-        }
-    }
-
-    public static class MyNativeScript extends AbstractDoubleSearchScript {
-        @Override
-        public double runAsDouble() {
-            double a = (double) source().get("a");
-            double b = (double) source().get("b");
-            return a * b;
-        }
-    }
-}
---------------------------------------------------
-
-You can execute the script by specifying its `lang` as `native`, and the name
-of the script as the `id`:
-
-[source,js]
---------------------------------------------------
-curl -XPOST localhost:9200/_search -d '{
-  "query": {
-    "function_score": {
-      "query": {
-        "match": {
-          "body": "foo"
-        }
-      },
-      "functions": [
-        {
-          "script_score": {
-            "script": {
-                "id": "my_script",
-                "lang" : "native"
-            }
-          }
-        }
-      ]
-    }
-  }
-}'
---------------------------------------------------
diff --git a/docs/reference/modules/scripting/painless-syntax.asciidoc b/docs/reference/modules/scripting/painless-syntax.asciidoc
deleted file mode 100644
index 1191facc369c8..0000000000000
--- a/docs/reference/modules/scripting/painless-syntax.asciidoc
+++ /dev/null
@@ -1,173 +0,0 @@
-[[modules-scripting-painless-syntax]]
-=== Painless Syntax
-
-experimental[The Painless scripting language is new and is still marked as experimental. The syntax or API may be changed in the future in non-backwards compatible ways if required.]
-
-[float]
-[[painless-types]]
-=== Variable types
-
-Painless supports all of https://docs.oracle.com/javase/tutorial/java/nutsandbolts/variables.html[Java's types],
-including array types, but adds some additional built-in types.
-
-[float]
-[[painless-def]]
-==== Def
-
-The dynamic type `def` serves as a placeholder for any other type. It adopts the behavior
-of whatever runtime type it represents.
-
-[float]
-[[painless-strings]]
-==== String
-
-String constants can be declared with single quotes, to avoid escaping horrors with JSON:
-
-[source,painless]
----------------------------------------------------------
-def mystring = 'foo';
----------------------------------------------------------
-
-[float]
-[[painless-lists]]
-==== List
-
-Lists can be created explicitly (e.g. `new ArrayList()`) or initialized similar to Groovy:
-
-[source,painless]
----------------------------------------------------------
-def list = [1,2,3];
----------------------------------------------------------
-
-Lists can also be accessed similar to arrays: they support subscript and `.length`:
-
-[source,painless]
----------------------------------------------------------
-def list = [1,2,3];
-return list[0]
----------------------------------------------------------
-
-[float]
-[[painless-maps]]
-==== Map
-
-Maps can be created explicitly (e.g. `new HashMap()`) or initialized similar to Groovy:
-
-[source,painless]
----------------------------------------------------------
-def person = ['name': 'Joe', 'age': 63];
----------------------------------------------------------
-
-Map keys can also be accessed as properties.
-
-[source,painless]
----------------------------------------------------------
-def person = ['name': 'Joe', 'age': 63];
-person.retired = true;
-return person.name
----------------------------------------------------------
-
-Map keys can also be accessed via subscript (for keys containing special characters):
-
-[source,painless]
----------------------------------------------------------
-return map['something-absurd!']
----------------------------------------------------------
-
-[float]
-[[painless-pattern]]
-==== Pattern
-
-Regular expression constants are directly supported:
-
-[source,painless]
----------------------------------------------------------
-Pattern p = /[aeiou]/
----------------------------------------------------------
-
-Patterns can only be created via this mechanism. This ensures fast performance, regular expressions
-are always constants and compiled efficiently a single time.
-
-[float]
-[[modules-scripting-painless-regex-flags]]
-==== Pattern flags
-
-You can define flags on patterns in Painless by adding characters after the
-trailing `/` like `/foo/i` or `/foo \w #comment/iUx`. Painless exposes all the
-flags from
-https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html[Java's Pattern class]
-using these characters:
-
-[cols="<,<,<",options="header",]
-|=======================================================================
-| Character | Java Constant | Example
-|`c` | CANON_EQ                  | `'å' ==~ /å/c` (open in hex editor to see)
-|`i` | CASE_INSENSITIVE          | `'A' ==~ /a/i`
-|`l` | LITERAL                   | `'[a]' ==~ /[a]/l`
-|`m` | MULTILINE                 | `'a\nb\nc' =~ /^b$/m`
-|`s` | DOTALL (aka single line)  | `'a\nb\nc' =~ /.b./s`
-|`U` | UNICODE_CHARACTER_CLASS   | `'Ɛ' ==~ /\\w/U`
-|`u` | UNICODE_CASE              | `'Ɛ' ==~ /ɛ/iu`
-|`x` | COMMENTS (aka extended)   | `'a' ==~ /a #comment/x`
-|=======================================================================
-
-[float]
-[[painless-operators]]
-=== Operators
-
-All of Java's https://docs.oracle.com/javase/tutorial/java/nutsandbolts/operators.html[operators] are
-supported with the same precedence, promotion, and semantics.
-
-There are only a few minor differences and add-ons:
-
-* `==` behaves as Java's for numeric types, but for non-numeric types acts as https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html#equals-java.lang.Object-[`Object.equals()`]
-* `===` and `!==` support exact reference comparison (e.g. `x === y`)
-* `=~` true if a portion of the text matches a pattern (e.g. `x =~ /b/`)
-* `==~` true if the entire text matches a pattern (e.g. `x ==~ /[Bb]ob/`)
-
-[float]
-[[painless-control-flow]]
-=== Control flow
-
-Java's https://docs.oracle.com/javase/tutorial/java/nutsandbolts/flow.html[control flow statements] are supported, with the exception
-of the `switch` statement.
-
-In addition to Java's `enhanced for` loop, the `for in` syntax from groovy can also be used:
-
-[source,painless]
----------------------------------------------------------
-for (item : list) {
-  ...
-}
----------------------------------------------------------
-
-[float]
-[[painless-functions]]
-=== Functions
-
-Functions can be declared at the beginning of the script, for example:
-
-[source,painless]
----------------------------------------------------------
-boolean isNegative(def x) { x < 0 }
-...
-if (isNegative(someVar)) {
-  ...
-}
----------------------------------------------------------
-
-[float]
-[[painless-lambda-expressions]]
-=== Lambda expressions
-Lambda expressions and method references work the same as https://docs.oracle.com/javase/tutorial/java/javaOO/lambdaexpressions.html[Java's].
-
-[source,painless]
----------------------------------------------------------
-list.removeIf(item -> item == 2);
-list.removeIf((int item) -> item == 2);
-list.removeIf((int item) -> { item == 2 });
-list.sort((x, y) -> x - y);
-list.sort(Integer::compare);
----------------------------------------------------------
-
-Method references to functions within the script can be accomplished using `this`, e.g. `list.sort(this::mycompare)`.
diff --git a/docs/reference/modules/scripting/painless.asciidoc b/docs/reference/modules/scripting/painless.asciidoc
index ad36cdd6df4a5..0993701033b4b 100644
--- a/docs/reference/modules/scripting/painless.asciidoc
+++ b/docs/reference/modules/scripting/painless.asciidoc
@@ -3,336 +3,7 @@
 
 experimental[The Painless scripting language is new and is still marked as experimental. The syntax or API may be changed in the future in non-backwards compatible ways if required.]
 
-_Painless_ is a simple, secure scripting language available in Elasticsearch
-by default. It is designed specifically for use with Elasticsearch and can
-safely be used with `inline` and `stored` scripting, which is enabled by
-default.
+include::../../../painless/painless-description.asciidoc[]
 
-The Painless syntax is similar to http://groovy-lang.org/index.html[Groovy].
-
-You can use Painless anywhere a script can be used in Elasticsearch--simply set the `lang` parameter
-to `painless`.
-
-[[painless-features]]
-[float]
-== Painless Features
-
-* Fast performance: https://benchmarks.elastic.co/index.html#search_qps_scripts[several times faster] than the alternatives.
-
-* Safety: Fine-grained <<painless-api, whitelist>> with method call/field granularity.
-
-* Optional typing: Variables and parameters can use explicit types or the dynamic `def` type.
-
-* Syntax: Extends Java's syntax with a subset of Groovy for ease of use. See the <<modules-scripting-painless-syntax, Syntax Overview>>.
-
-* Optimizations: Designed specifically for Elasticsearch scripting.
-
-[[painless-examples]]
-[float]
-== Painless Examples
-
-To illustrate how Painless works, let's load some hockey stats into an Elasticsearch index:
-
-[source,js]
-----------------------------------------------------------------
-PUT hockey/player/_bulk?refresh
-{"index":{"_id":1}}
-{"first":"johnny","last":"gaudreau","goals":[9,27,1],"assists":[17,46,0],"gp":[26,82,1]}
-{"index":{"_id":2}}
-{"first":"sean","last":"monohan","goals":[7,54,26],"assists":[11,26,13],"gp":[26,82,82]}
-{"index":{"_id":3}}
-{"first":"jiri","last":"hudler","goals":[5,34,36],"assists":[11,62,42],"gp":[24,80,79]}
-{"index":{"_id":4}}
-{"first":"micheal","last":"frolik","goals":[4,6,15],"assists":[8,23,15],"gp":[26,82,82]}
-{"index":{"_id":5}}
-{"first":"sam","last":"bennett","goals":[5,0,0],"assists":[8,1,0],"gp":[26,1,0]}
-{"index":{"_id":6}}
-{"first":"dennis","last":"wideman","goals":[0,26,15],"assists":[11,30,24],"gp":[26,81,82]}
-{"index":{"_id":7}}
-{"first":"david","last":"jones","goals":[7,19,5],"assists":[3,17,4],"gp":[26,45,34]}
-{"index":{"_id":8}}
-{"first":"tj","last":"brodie","goals":[2,14,7],"assists":[8,42,30],"gp":[26,82,82]}
-{"index":{"_id":39}}
-{"first":"mark","last":"giordano","goals":[6,30,15],"assists":[3,30,24],"gp":[26,60,63]}
-{"index":{"_id":10}}
-{"first":"mikael","last":"backlund","goals":[3,15,13],"assists":[6,24,18],"gp":[26,82,82]}
-{"index":{"_id":11}}
-{"first":"joe","last":"colborne","goals":[3,18,13],"assists":[6,20,24],"gp":[26,67,82]}
-----------------------------------------------------------------
-// CONSOLE
-// TESTSETUP
-
-[float]
-=== Accessing Doc Values from Painless
-
-Document values can be accessed from a `Map` named `doc`.
-
-For example, the following script calculates a player's total goals. This example uses a strongly typed `int` and a `for` loop.
-
-[source,js]
-----------------------------------------------------------------
-GET hockey/_search
-{
-  "query": {
-    "function_score": {
-      "script_score": {
-        "script": {
-          "lang": "painless",
-          "inline": "int total = 0; for (int i = 0; i < doc['goals'].length; ++i) { total += doc['goals'][i]; } return total;"
-        }
-      }
-    }
-  }
-}
-----------------------------------------------------------------
-// CONSOLE
-
-Alternatively, you could do the same thing using a script field instead of a function score:
-
-[source,js]
-----------------------------------------------------------------
-GET hockey/_search
-{
-  "query": {
-    "match_all": {}
-  },
-  "script_fields": {
-    "total_goals": {
-      "script": {
-        "lang": "painless",
-        "inline": "int total = 0; for (int i = 0; i < doc['goals'].length; ++i) { total += doc['goals'][i]; } return total;"
-      }
-    }
-  }
-}
-----------------------------------------------------------------
-// CONSOLE
-
-The following example uses a Painless script to sort the players by their combined first and last names. The names are accessed using
-`doc['first'].value` and `doc['last'].value`.
-
-[source,js]
-----------------------------------------------------------------
-GET hockey/_search
-{
-  "query": {
-    "match_all": {}
-  },
-  "sort": {
-    "_script": {
-      "type": "string",
-      "order": "asc",
-      "script": {
-        "lang": "painless",
-        "inline": "doc['first.keyword'].value + ' ' + doc['last.keyword'].value"
-      }
-    }
-  }
-}
-----------------------------------------------------------------
-// CONSOLE
-
-[float]
-=== Updating Fields with Painless
-
-You can also easily update fields. You access the original source for a field as `ctx._source.<field-name>`.
-
-First, let's look at the source data for a player by submitting the following request:
-
-[source,js]
-----------------------------------------------------------------
-GET hockey/_search
-{
-  "stored_fields": [
-    "_id",
-    "_source"
-  ],
-  "query": {
-    "term": {
-      "_id": 1
-    }
-  }
-}
-----------------------------------------------------------------
-// CONSOLE
-
-To change player 1's last name to `hockey`, simply set `ctx._source.last` to the new value:
-
-[source,js]
-----------------------------------------------------------------
-POST hockey/player/1/_update
-{
-  "script": {
-    "lang": "painless",
-    "inline": "ctx._source.last = params.last",
-    "params": {
-      "last": "hockey"
-    }
-  }
-}
-----------------------------------------------------------------
-// CONSOLE
-
-You can also add fields to a document. For example, this script adds a new field that contains
-the player's nickname,  _hockey_.
-
-[source,js]
-----------------------------------------------------------------
-POST hockey/player/1/_update
-{
-  "script": {
-    "lang": "painless",
-    "inline": "ctx._source.last = params.last; ctx._source.nick = params.nick",
-    "params": {
-      "last": "gaudreau",
-      "nick": "hockey"
-    }
-  }
-}
-----------------------------------------------------------------
-// CONSOLE
-
-[float]
-[[modules-scripting-painless-regex]]
-=== Regular expressions
-
-Painless's native support for regular expressions has syntax constructs:
-
-* `/pattern/`: Pattern literals create patterns. This is the only way to create
-a pattern in painless. The pattern inside the `/`s are just
-http://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html[Java regular expressions].
-See <<modules-scripting-painless-regex-flags>> for more.
-* `=~`: The find operator return a `boolean`, `true` if a subsequence of the
-text matches, `false` otherwise.
-* `==~`: The match operator returns a `boolean`, `true` if the text matches,
-`false` if it doesn't.
-
-Using the find operator (`=~`) you can update all hockey players with "b" in
-their last name:
-
-[source,js]
-----------------------------------------------------------------
-POST hockey/player/_update_by_query
-{
-  "script": {
-    "lang": "painless",
-    "inline": "if (ctx._source.last =~ /b/) {ctx._source.last += \"matched\"} else {ctx.op = 'noop'}"
-  }
-}
-----------------------------------------------------------------
-// CONSOLE
-
-Using the match operator (`==~`) you can update all the hockey players who's
-names start with a consonant and end with a vowel:
-
-[source,js]
-----------------------------------------------------------------
-POST hockey/player/_update_by_query
-{
-  "script": {
-    "lang": "painless",
-    "inline": "if (ctx._source.last ==~ /[^aeiou].*[aeiou]/) {ctx._source.last += \"matched\"} else {ctx.op = 'noop'}"
-  }
-}
-----------------------------------------------------------------
-// CONSOLE
-
-You can use the `Pattern.matcher` directly to get a `Matcher` instance and
-remove all of the vowels in all of their last names:
-
-[source,js]
-----------------------------------------------------------------
-POST hockey/player/_update_by_query
-{
-  "script": {
-    "lang": "painless",
-    "inline": "ctx._source.last = /[aeiou]/.matcher(ctx._source.last).replaceAll('')"
-  }
-}
-----------------------------------------------------------------
-// CONSOLE
-
-`Matcher.replaceAll` is just a call to Java's `Matcher`'s
-http://docs.oracle.com/javase/8/docs/api/java/util/regex/Matcher.html#replaceAll-java.lang.String-[replaceAll]
-method so it supports `$1` and `\1` for replacements:
-
-[source,js]
-----------------------------------------------------------------
-POST hockey/player/_update_by_query
-{
-  "script": {
-    "lang": "painless",
-    "inline": "ctx._source.last = /n([aeiou])/.matcher(ctx._source.last).replaceAll('$1')"
-  }
-}
-----------------------------------------------------------------
-// CONSOLE
-
-If you need more control over replacements you can call `replaceAll` on a
-`CharSequence` with a `Function<Matcher, String>` that builds the replacement.
-This does not support `$1` or `\1` to access replacements because you already
-have a reference to the matcher and can get them with `m.group(1)`.
-
-IMPORTANT: Calling `Matcher.find` inside of the function that builds the
-replacement is rude and will likely break the replacement process.
-
-This will make all of the vowels in the hockey player's last names upper case:
-
-[source,js]
-----------------------------------------------------------------
-POST hockey/player/_update_by_query
-{
-  "script": {
-    "lang": "painless",
-    "inline": "ctx._source.last = ctx._source.last.replaceAll(/[aeiou]/, m -> m.group().toUpperCase(Locale.ROOT))"
-  }
-}
-----------------------------------------------------------------
-// CONSOLE
-
-Or you can use the `CharSequence.replaceFirst` to make the first vowel in their
-last names upper case:
-
-[source,js]
-----------------------------------------------------------------
-POST hockey/player/_update_by_query
-{
-  "script": {
-    "lang": "painless",
-    "inline": "ctx._source.last = ctx._source.last.replaceFirst(/[aeiou]/, m -> m.group().toUpperCase(Locale.ROOT))"
-  }
-}
-----------------------------------------------------------------
-// CONSOLE
-
-
-Note: all of the `_update_by_query` examples above could really do with a
-`query` to limit the data that they pull back. While you *could* use a
-<<query-dsl-script-query>> it wouldn't be as efficient as using any other query
-because script queries aren't able to use the inverted index to limit the
-documents that they have to check.
-
-[float]
-[[painless-api]]
-== Painless API
-
-The following Java packages are available for use in the Painless language:
-
-* https://docs.oracle.com/javase/8/docs/api/java/lang/package-summary.html[java.lang]
-* https://docs.oracle.com/javase/8/docs/api/java/math/package-summary.html[java.math]
-* https://docs.oracle.com/javase/8/docs/api/java/text/package-summary.html[java.text]
-* https://docs.oracle.com/javase/8/docs/api/java/time/package-summary.html[java.time]
-* https://docs.oracle.com/javase/8/docs/api/java/time/chrono/package-summary.html[java.time.chrono]
-* https://docs.oracle.com/javase/8/docs/api/java/time/format/package-summary.html[java.time.format]
-* https://docs.oracle.com/javase/8/docs/api/java/time/temporal/package-summary.html[java.time.temporal]
-* https://docs.oracle.com/javase/8/docs/api/java/time/zone/package-summary.html[java.time.zone]
-* https://docs.oracle.com/javase/8/docs/api/java/util/package-summary.html[java.util]
-* https://docs.oracle.com/javase/8/docs/api/java/util/function/package-summary.html[java.util.function]
-* https://docs.oracle.com/javase/8/docs/api/java/util/regex/package-summary.html[java.util.regex]
-* https://docs.oracle.com/javase/8/docs/api/java/util/stream/package-summary.html[java.util.stream]
-
-Note that unsafe classes and methods are not included, there is no support for:
-
-* Manipulation of processes and threads
-* Input/Output
-* Reflection
+Ready to start scripting with Painless? See {painless}/painless-getting-started.html[Getting Started with Painless] in the guide to the
+{painless}/index.html[Painless Scripting Language].
\ No newline at end of file
diff --git a/docs/reference/modules/scripting/security.asciidoc b/docs/reference/modules/scripting/security.asciidoc
index 4d35b0da250b4..3ca5d3ac35a29 100644
--- a/docs/reference/modules/scripting/security.asciidoc
+++ b/docs/reference/modules/scripting/security.asciidoc
@@ -1,73 +1,162 @@
 [[modules-scripting-security]]
 === Scripting and security
 
-You should never run Elasticsearch as the `root` user, as this would allow a
-script to access or do *anything* on your server, without limitations.
+While Elasticsearch contributors make every effort to prevent scripts from
+running amok, security is something best done in
+https://en.wikipedia.org/wiki/Defense_in_depth_(computing)[layers] because
+all software has bugs and it is important to minimize the risk of failure in
+any security layer. Find below rules of thumb for how to keep Elasticsearch
+from being a vulnerability.
 
-You should not expose Elasticsearch directly to users, but instead have a
-proxy application inbetween. If you *do* intend to expose Elasticsearch
-directly to your users, then you have to decide whether you trust them enough
-to run scripts on your box or not, and apply the appropriate safety measures.
+[float]
+=== Do not run as root
+First and foremost, never run Elasticsearch as the `root` user as this would
+allow any successful effort to circumvent the other security layers to do
+*anything* on your server. Elasticsearch will refuse to start if it detects
+that it is running as `root` but this is so important that it is worth double
+and triple checking.
+
+[float]
+=== Do not expose Elasticsearch directly to users
+Do not expose Elasticsearch directly to users, instead have an application
+make requests on behalf of users. If this is not possible, have an application
+to sanitize requests from users. If *that* is not possible then have some
+mechanism to track which users did what. Understand that it is quite possible
+to write a <<search, `_search`>> that overwhelms Elasticsearch and brings down
+the cluster. All such searches should be considered bugs and the Elasticsearch
+contributors make an effort to prevent this but they are still possible.
 
-[[enable-dynamic-scripting]]
 [float]
-=== Enabling dynamic scripting
+=== Do not expose Elasticsearch directly to the Internet
+Do not expose Elasticsearch to the Internet, instead have an application
+make requests on behalf of the Internet. Do not entertain the thought of having
+an application "sanitize" requests to Elasticsearch. Understand that it is
+possible for a sufficiently determined malicious user to write searches that
+overwhelm the Elasticsearch cluster and bring it down. For example:
+
+Good:
+* Users type text into a search box and the text is sent directly to a
+<<query-dsl-match-query>>, <<query-dsl-match-query-phrase>>,
+<<query-dsl-simple-query-string-query>>, or any of the <<search-suggesters>>.
+* Running a script with any of the above queries that was written as part of
+the application development process.
+* Running a script with `params` provided by users.
+* User actions makes documents with a fixed structure.
+
+Bad:
+* Users can write arbitrary scripts, queries, `_search` requests.
+* User actions make documents with structure defined by users.
 
-The `script.*` settings allow for <<security-script-fine,fine-grained>>
-control of which script languages (e.g `groovy`, `painless`) are allowed to
-run in which context ( e.g. `search`, `aggs`, `update`), and where the script
-source is allowed to come from (i.e. `inline`, `stored`, `file`).
+[float]
+[[modules-scripting-security-do-no-weaken]]
+=== Do not weaken script security settings
+By default Elasticsearch will run inline, stored, and filesystem scripts for
+sandboxed languages, namely the scripting language Painless, the template
+language Mustache, and the expression language Expressions. These *ought* to be
+safe to expose to trusted users and to your application servers because they
+have strong security sandboxes. By default Elasticsearch will only run
+filesystem scripts for non-sandboxed languages and enabling them is a poor
+choice because:
+1. This drops a layer of security, leaving only Elasticsearch's builtin
+<<modules-scripting-other-layers, security layers>>.
+2. Non-sandboxed scripts have unchecked access to Elasticsearch's internals and
+can cause all kinds of trouble if misused.
 
-For instance, the following setting enables `stored` `update` scripts for
-`groovy`:
 
-[source,yaml]
-----------------
-script.engine.groovy.inline.update: true
-----------------
+[float]
+[[modules-scripting-other-layers]]
+=== Other security layers
+In addition to user privileges and script sandboxing Elasticsearch uses the
+http://www.oracle.com/technetwork/java/seccodeguide-139067.html[Java Security Manager]
+and native security tools as additional layers of security.
+
+As part of its startup sequence Elasticsearch enables the Java Security Manager
+which limits the actions that can be taken by portions of the code. Painless
+uses this to limit the actions that generated Painless scripts can take,
+preventing them from being able to do things like write files and listen to
+sockets.
+
+Elasticsearch uses
+https://en.wikipedia.org/wiki/Seccomp[seccomp] in Linux,
+https://www.chromium.org/developers/design-documents/sandbox/osx-sandboxing-design[Seatbelt]
+in macOS, and
+https://msdn.microsoft.com/en-us/library/windows/desktop/ms684147[ActiveProcessLimit]
+on Windows to prevent Elasticsearch from forking or executing other processes.
+
+Below this we describe the security settings for scripts and how you can
+change from the defaults described above. You should be very, very careful
+when allowing more than the defaults. Any extra permissions weakens the total
+security of the Elasticsearch deployment.
+
+[[allowed-script-types-setting]]
+[float]
+=== Allowed script types setting
 
-Less fine-grained settings exist which allow you to enable or disable scripts
-for all sources, all languages, or all contexts.  The following settings
-enable `inline` and `stored` scripts for all languages in all contexts:
+By default all script types are allowed to be executed.  This can be modified using the
+setting `script.allowed_types`.  Only the types specified as part of the setting will be
+allowed to be executed.  To specify no types are allowed, set `script.allowed_types` to
+be `none`.
 
 [source,yaml]
------------------------------------
-script.inline: true
-script.stored: true
------------------------------------
+----
+script.allowed_types: inline <1>
+----
+<1> This will allow only inline scripts to be executed but not stored scripts
+(or any other types).
 
-WARNING:  The above settings mean that anybody who can send requests to your
-Elasticsearch instance can run whatever scripts they choose! This is a
-security risk and may well lead to your Elasticsearch cluster being
-compromised.
+[[allowed-script-contexts-setting]]
+[float]
+=== Allowed script contexts setting
 
-[[security-script-source]]
+By default all script contexts are allowed to be executed.  This can be modified using the
+setting `script.allowed_contexts`.  Only the contexts specified as part of the setting will
+be allowed to be executed.  To specify no contexts are allowed, set `script.allowed_contexts`
+to be `none`.
+
+[source,yaml]
+----
+script.allowed_contexts: search, update <1>
+----
+<1> This will allow only search and update scripts to be executed but not
+aggs or plugin scripts (or any other contexts).
+
+[[deprecated-script=settings]]
 [float]
-=== Script source settings
+=== Deprecated script settings
 
-Scripts may be enabled or disabled depending on their source: `inline`,
-`stored` in the cluster state, or from a `file` on each node in the cluster.
-Each of these settings takes one of these values:
+The following settings have all been deprecated and will be removed in 6.0:
 
+ * <<security-script-source>>
+ * <<security-script-context>>
+ * <<security-script-fine>>
 
-[horizontal]
-`false`::   Scripting is enabled.
-`true`::    Scripting is disabled.
+Use the following instead:
 
-The default values are the following:
+ * <<allowed-script-types-setting>>
+ * <<allowed-script-contexts-setting>>
+
+[[security-script-source]]
+[float]
+=== Script source settings
+
+Which scripts Elasticsearch will execute where is controlled by settings
+starting with `scripts.`. The simplest settings allow scripts to be enabled
+or disabled based on where they are stored. For example:
 
 [source,yaml]
 -----------------------------------
-script.inline: false
-script.stored: false
-script.file:   true
+script.inline: false  <1>
+script.stored: false  <2>
+script.file:   true   <3>
 -----------------------------------
+<1> Refuse to run scripts provided inline in the API.
+<2> Refuse to run scripts stored using the API.
+<3> Run scripts found on the filesystem in `/etc/elasticsearch/scripts`
+(rpm or deb) or `config/scripts` (zip or tar).
 
-NOTE: Global scripting settings affect the `mustache` scripting language.
-<<search-template,Search templates>> internally use the `mustache` language,
-and will still be enabled by default as the `mustache` engine is sandboxed,
-but they will be enabled/disabled according to fine-grained settings
-specified in `elasticsearch.yml`.
+NOTE: These settings override the defaults mentioned
+<<modules-scripting-security-do-no-weaken, above>>. Recreating the defaults
+requires more fine grained settings described <<security-script-fine, below>>.
 
 [[security-script-context]]
 [float]
@@ -102,15 +191,13 @@ script.plugin: false
 === Fine-grained script settings
 
 First, the high-level script settings described above are applied in order
-(context settings have precedence over source settings).  Then,  fine-grained
+(context settings have precedence over source settings). Then fine-grained
 settings which include the script language take precedence over any high-level
-settings.
-
-Fine-grained settings have the form:
+settings. They have two forms:
 
 [source,yaml]
 ------------------------
-script.engine.{lang}.{source}.{context}: true|false
+script.engine.{lang}.{inline|file|stored}.{context}: true|false
 ------------------------
 
 And
@@ -128,36 +215,36 @@ script.inline: false <1>
 script.stored: false <1>
 script.file:   false <1>
 
-script.engine.groovy.inline:          true <2>
-script.engine.groovy.stored.search:   true <3>
-script.engine.groovy.stored.aggs:     true <3>
+script.engine.painless.inline:          true <2>
+script.engine.painless.stored.search:   true <3>
+script.engine.painless.stored.aggs:     true <3>
 
-script.engine.mustache.stored.search: true <4>
+script.engine.mustache.stored.search:   true <4>
 -----------------------------------
 <1> Disable all scripting from any source.
-<2> Allow inline Groovy scripts for all operations
-<3> Allow stored Groovy scripts to be used for search and aggregations.
+<2> Allow inline Painless scripts for all operations.
+<3> Allow stored Painless scripts to be used for search and aggregations.
 <4> Allow stored Mustache templates to be used for search.
 
 [[java-security-manager]]
 [float]
 === Java Security Manager
 
-Elasticsearch runs with the https://docs.oracle.com/javase/tutorial/essential/environment/security.html[Java Security Manager]
-enabled by default.  The security policy in Elasticsearch locks down the
+As mentioned above, Elasticsearch runs with the https://docs.oracle.com/javase/tutorial/essential/environment/security.html[Java Security Manager]
+enabled by default. The security policy in Elasticsearch locks down the
 permissions granted to each class to the bare minimum required to operate.
 The benefit of doing this is that it severely limits the attack vectors
 available to a hacker.
 
-Restricting permissions is particularly important with scripting languages
-like Groovy and Javascript which are designed to do anything that can be done
-in Java itself, including writing to the file system, opening sockets to
-remote servers, etc.
+Restricting permissions is particularly important for non-sandboxed scripting
+languages like Groovy and Javascript which are designed to do anything that can
+be done in Java itself, including writing to the file system, opening sockets
+to remote servers, etc.
 
 [float]
 === Script Classloader Whitelist
 
-Scripting languages are only allowed to load classes which appear in a
+Groovy makes an effort to prevent loading classes which do not appear in a
 hardcoded whitelist that can be found in
 https://github.com/elastic/elasticsearch/blob/{branch}/core/src/main/java/org/elasticsearch/script/ClassPermission.java[`org.elasticsearch.script.ClassPermission`].
 
@@ -176,6 +263,9 @@ GET _search
   }
 }
 ------------------------------
+// CONSOLE
+// TEST[setup:twitter]
+// TEST[catch:request]
 
 will return the following exception:
 
@@ -196,6 +286,9 @@ will return the following exception:
   }
 }
 ------------------------------
+// NOTCONSOLE
+// This isn't exactly the error we get but it isn't worth fixing it because
+// this is being dropped in master anyway.
 
 However, classloader issues may also result in more difficult to interpret
 exceptions.  For instance, this script:
@@ -220,6 +313,7 @@ Returns the following exception:
   }
 }
 ------------------------------
+// NOTCONSOLE
 
 [float]
 == Dealing with Java Security Manager issues
@@ -242,7 +336,7 @@ Security Policy either:
 
 * system wide: `$JAVA_HOME/lib/security/java.policy`,
 * for just the `elasticsearch` user: `/home/elasticsearch/.java.policy`
-* by adding a system property to the <<sysconfig,es-java-opts>> configuration: `-Djava.security.policy=someURL`, or
+* by adding a system property to the <<jvm-options,jvm.options>> configuration: `-Djava.security.policy=someURL`, or
 * via the `ES_JAVA_OPTS` environment variable with `-Djava.security.policy=someURL`:
 +
 [source,js]
@@ -250,6 +344,7 @@ Security Policy either:
 export ES_JAVA_OPTS="${ES_JAVA_OPTS} -Djava.security.policy=file:///path/to/my.policy`
 ./bin/elasticsearch
 ---------------------------------
+// NOTCONSOLE
 
 Permissions may be granted at the class, package, or global level.  For instance:
 
@@ -261,6 +356,7 @@ grant {
     permission org.elasticsearch.script.ClassPermission "*"; // allow all (disables filtering basically)
 };
 ----------------------------------
+// NOTCONSOLE
 
 Here is an example of how to enable the `groovy.time.TimeCategory` class:
 
@@ -271,6 +367,7 @@ grant {
     permission org.elasticsearch.script.ClassPermission "groovy.time.TimeCategory";
 };
 ----------------------------------
+// NOTCONSOLE
 
 [TIP]
 ======================================
@@ -287,4 +384,3 @@ doing so.
 ======================================
 
 See http://docs.oracle.com/javase/7/docs/technotes/guides/security/PolicyFiles.html for more information.
-
diff --git a/docs/reference/modules/scripting/using.asciidoc b/docs/reference/modules/scripting/using.asciidoc
index 17ab4a8180ae9..18181ce301e9f 100644
--- a/docs/reference/modules/scripting/using.asciidoc
+++ b/docs/reference/modules/scripting/using.asciidoc
@@ -8,12 +8,13 @@ the same pattern:
 -------------------------------------
   "script": {
     "lang":   "...",  <1>
-    "inline" | "id" | "file": "...", <2>
+    "source" | "id" | "file": "...", <2>
     "params": { ... } <3>
   }
 -------------------------------------
+// NOTCONSOLE
 <1> The language the script is written in, which defaults to `painless`.
-<2> The script itself which may be specfied as `inline`, `id`, or `file`.
+<2> The script itself which may be specified as `source`, `id`, or `file`.
 <3> Any named parameters that should be passed into the script.
 
 For example, the following script is used in a search request to return a
@@ -32,7 +33,7 @@ GET my_index/_search
     "my_doubled_field": {
       "script": {
         "lang":   "expression",
-        "inline": "doc['my_field'] * multiplier",
+        "source": "doc['my_field'] * multiplier",
         "params": {
           "multiplier": 2
         }
@@ -48,17 +49,17 @@ GET my_index/_search
 
 `lang`::
 
-    Specifies the language the script is written in.  Defaults to `groovy` but
+    Specifies the language the script is written in.  Defaults to `painless` but
     may be set to any of languages listed in <<modules-scripting>>. The
     default language may be changed in the `elasticsearch.yml` config file by
     setting `script.default_lang` to the appropriate language.
 
 
-`inline`, `id`, `file`::
+`source`, `id`, `file`::
 
-    Specifies the source of the script.  An `inline` script is specified
-    `inline` as in the example above, a stored script with the specified `id`
-    is retrieved from the cluster state (see <<modules-scripting-stored-scripts,Stored Scripts>>),
+    Specifies the source of the script.  An `inline` script is specified with
+    `source` as in the example above, a `stored` script is specified with `id`
+    and is retrieved from the cluster state (see <<modules-scripting-stored-scripts,Stored Scripts>>),
     and a `file` script is retrieved from a file in the `config/scripts`
     directory (see <<modules-scripting-file-scripts, File Scripts>>).
 +
@@ -87,18 +88,20 @@ multipliers, don't hard-code the multiplier into the script:
 
 [source,js]
 ----------------------
-  "inline": "doc['my_field'] * 2"
+  "source": "doc['my_field'] * 2"
 ----------------------
+// NOTCONSOLE
 
 Instead, pass it in as a named parameter:
 
 [source,js]
 ----------------------
-  "inline": "doc['my_field'] * multiplier",
+  "source": "doc['my_field'] * multiplier",
   "params": {
     "multiplier": 2
   }
 ----------------------
+// NOTCONSOLE
 
 The first version has to be recompiled every time the multiplier changes.  The
 second version is only compiled once.
@@ -134,7 +137,7 @@ the following example creates a Groovy script called `calculate-score`:
 
 [source,sh]
 --------------------------------------------------
-cat "log(_score * 2) + my_modifier" > config/scripts/calculate-score.groovy
+cat "Math.log(_score * 2) + params.my_modifier" > config/scripts/calculate_score.painless
 --------------------------------------------------
 
 This script can be used as follows:
@@ -146,8 +149,8 @@ GET my_index/_search
   "query": {
     "script": {
       "script": {
-        "lang":   "groovy", <1>
-        "file":   "calculate-score", <2>
+        "lang":   "painless", <1>
+        "file":   "calculate_score", <2>
         "params": {
           "my_modifier": 2
         }
@@ -156,6 +159,8 @@ GET my_index/_search
   }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 <1> The language of the script, which should correspond with the script file suffix.
 <2> The name of the script, which should be the name of the file.
 
@@ -164,7 +169,6 @@ hierarchy of directories is flattened and concatenated with underscores.  A
 script in `group1/group2/my_script.groovy` should use `group1_group2_myscript`
 as the `file` name.
 
-
 [[reload-scripts]]
 [float]
 ==== Automatic script reloading
@@ -181,23 +185,45 @@ Script reloading can be completely disabled by setting
 === Stored Scripts
 
 Scripts may be stored in and retrieved from the cluster state using the
-`_scripts` end-point:
+`_scripts` end-point.
 
-[source,js]
------------------------------------
-/_scripts/{lang}/{id} <1> <2>
------------------------------------
-<1> The `lang` represents the script language.
-<2> The `id` is a unique identifier or script name.
+==== Deprecated Namespace
+
+The namespace for stored scripts using both `lang` and `id` as a unique
+identifier has been deprecated.  The new namespace for stored scripts will
+only use `id`. Stored scripts with the same `id`, but different `lang`'s
+will no longer be allowed in 6.0.  To comply with the new namespace for
+stored scripts, existing stored scripts should be deleted and put again.
+Any scripts that share an `id` but have different `lang`s will need to
+be re-named.  For example, take the following:
 
-This example stores a Groovy script called `calculate-score` in the cluster
-state:
+"id": "example", "lang": "painless"
+"id": "example", "lang": "expressions"
+
+The above scripts will conflict under the new namespace since the id's are
+the same.  At least one will have to be re-named to comply with the new
+namespace of only `id`.
+
+As a final caveat, stored search templates and stored scripts share
+the same namespace, so if a search template has the same `id` as a
+stored script, one of the two will have to be re-named as well using
+delete and put requests.
+
+==== Request Examples
+
+The following are examples of using a stored script that lives at
+`/_scripts/{id}`.
+
+First, create the script called `calculate-score` in the cluster state:
 
 [source,js]
 -----------------------------------
-POST _scripts/groovy/calculate-score
+POST _scripts/calculate-score
 {
-  "script": "log(_score * 2) + my_modifier"
+  "script": {
+    "lang": "painless",
+    "source": "Math.log(_score * 2) + params.my_modifier"
+  }
 }
 -----------------------------------
 // CONSOLE
@@ -206,12 +232,12 @@ This same script can be retrieved with:
 
 [source,js]
 -----------------------------------
-GET _scripts/groovy/calculate-score
+GET _scripts/calculate-score
 -----------------------------------
 // CONSOLE
 // TEST[continued]
 
-Stored scripts can be used by specifying the `lang` and `id` parameters as follows:
+Stored scripts can be used by specifying the `stored` parameters as follows:
 
 [source,js]
 --------------------------------------------------
@@ -220,8 +246,7 @@ GET _search
   "query": {
     "script": {
       "script": {
-        "lang": "groovy",
-        "id":   "calculate-score",
+        "id": "calculate-score",
         "params": {
           "my_modifier": 2
         }
@@ -237,7 +262,7 @@ And deleted with:
 
 [source,js]
 -----------------------------------
-DELETE _scripts/groovy/calculate-score
+DELETE _scripts/calculate-score
 -----------------------------------
 // CONSOLE
 // TEST[continued]
diff --git a/docs/reference/modules/snapshots.asciidoc b/docs/reference/modules/snapshots.asciidoc
index 9fd8e069480be..6a0bc5a2c56a0 100644
--- a/docs/reference/modules/snapshots.asciidoc
+++ b/docs/reference/modules/snapshots.asciidoc
@@ -1,9 +1,33 @@
 [[modules-snapshots]]
 == Snapshot And Restore
 
-The snapshot and restore module allows to create snapshots of individual indices or an entire cluster into a remote
-repository. At the time of the initial release only shared file system repository was supported, but now a range of
-backends are available via officially supported repository plugins.
+The snapshot and restore module allows to create snapshots of individual
+indices or an entire cluster into a remote repository like shared file system,
+S3, or HDFS. These snapshots are great for backups because they can be restored
+relatively quickly but they are not archival because they can only be restored
+to versions of Elasticsearch that can read the index. That means that:
+
+* A snapshot of an index created in 2.x can be restored to 5.x.
+* A snapshot of an index created in 1.x can be restored to 2.x.
+* A snapshot of an index created in 1.x can **not** be restored to 5.x.
+
+To restore a snapshot of an index created in 1.x to 5.x you can restore it to
+a 2.x cluster and use <<reindex-from-remote,reindex-from-remote>> to rebuild
+the index in a 5.x cluster. This is as time consuming as restoring from
+archival copies of the original data.
+
+Note: If a repository is connected to a 2.x cluster, and you want to connect
+a 5.x cluster to the same repository, you will have to either first set the 2.x 
+repository to `readonly` mode (see below for details on `readonly` mode) or create
+the 5.x repository in `readonly` mode.  A 5.x cluster will update the repository 
+to conform to 5.x specific formats, which will mean that any new snapshots written 
+via the 2.x cluster will not be visible to the 5.x cluster, and vice versa.  
+In fact, as a general rule, only one cluster should connect to the same repository
+location with write access; all other clusters connected to the same repository
+should be set to `readonly` mode.  While setting all but one repositories to 
+`readonly` should work with multiple clusters differing by one major version, 
+it is not a supported configuration.
+
 
 [float]
 === Repositories
@@ -98,7 +122,7 @@ the name `my_backup`:
 
 [source,js]
 -----------------------------------
-$ curl -XPUT 'http://localhost:9200/_snapshot/my_backup' -d '{
+$ curl -XPUT 'http://localhost:9200/_snapshot/my_backup' -H 'Content-Type: application/json' -d '{
     "type": "fs",
     "settings": {
         "location": "/mount/backups/my_backup",
@@ -112,7 +136,7 @@ in `path.repo`:
 
 [source,js]
 -----------------------------------
-$ curl -XPUT 'http://localhost:9200/_snapshot/my_backup' -d '{
+$ curl -XPUT 'http://localhost:9200/_snapshot/my_backup' -H 'Content-Type: application/json' -d '{
     "type": "fs",
     "settings": {
         "location": "my_backup",
@@ -261,6 +285,35 @@ GET /_snapshot/my_backup/snapshot_1
 -----------------------------------
 // CONSOLE
 
+This command returns basic information about the snapshot including start and end time, version of
+elasticsearch that created the snapshot, the list of included indices, the current state of the
+snapshot and the list of failures that occurred during the snapshot. The snapshot `state` can be
+
+[horizontal]
+`IN_PROGRESS`::
+
+  The snapshot is currently running.
+
+`SUCCESS`::
+
+  The snapshot finished and all shards were stored successfully.
+
+`FAILED`::
+
+  The snapshot finished with an error and failed to store any data.
+
+`PARTIAL`::
+
+  The global cluster state was stored, but data of at least one shard wasn't stored successfully.
+  The `failure` section in this case should contain more detailed information about shards
+  that were not processed correctly.
+
+`INCOMPATIBLE`::
+
+  The snapshot was created with an old version of elasticsearch and therefore is incompatible with
+  the current version of the cluster.
+
+
 Similar as for repositories, information about multiple snapshots can be queried in one go, supporting wildcards as well:
 
 [source,sh]
@@ -277,9 +330,18 @@ GET /_snapshot/my_backup/_all
 -----------------------------------
 // CONSOLE
 
-The command fails if some of the snapshots are unavailable. The boolean parameter `ignore_unvailable` can be used to
+The command fails if some of the snapshots are unavailable. The boolean parameter `ignore_unavailable` can be used to
 return all snapshots that are currently available.
 
+Getting all snapshots in the repository can be costly on cloud-based repositories,
+both from a cost and performance perspective.  If the only information required is
+the snapshot names/uuids in the repository and the indices in each snapshot, then
+the optional boolean parameter `verbose` can be set to `false` to execute a more
+performant and cost-effective retrieval of the snapshots in the repository.  Note
+that setting `verbose` to `false` will omit all other information about the snapshot
+such as status information, the number of snapshotted shards, etc.  The default
+value of the `verbose` parameter is `true`.
+
 A currently running snapshot can be retrieved using the following command:
 
 [source,sh]
@@ -353,7 +415,7 @@ existing index can be only restored if it's <<indices-open-close,closed>> and
 has the same number of shards as the index in the snapshot. The restore
 operation automatically opens restored indices if they were closed and creates
 new indices if they didn't exist in the cluster. If cluster state is restored
-with `include_cluster_state` (defaults to `false`), the restored templates that
+with `include_global_state` (defaults to `false`), the restored templates that
 don't currently exist in the cluster are added and existing templates with the
 same name are replaced by the restored templates. The restored persistent
 settings are added to the existing persistent settings.
@@ -516,5 +578,3 @@ well as the global metadata were readable. The restore operation requires the gl
 the index level blocks are ignored during restore because indices are essentially recreated during restore.
 Please note that a repository content is not part of the cluster and therefore cluster blocks don't affect internal
 repository operations such as listing or deleting snapshots from an already registered repository.
-
-
diff --git a/docs/reference/modules/threadpool.asciidoc b/docs/reference/modules/threadpool.asciidoc
index a333312e0fbc2..93b6a28710f58 100644
--- a/docs/reference/modules/threadpool.asciidoc
+++ b/docs/reference/modules/threadpool.asciidoc
@@ -32,14 +32,9 @@ There are several thread pools, but the important ones include:
 `bulk`::
     For bulk operations. Thread pool type is `fixed`
     with a size of `# of available processors`,
-    queue_size of `50`.  The maximum size for this pool
+    queue_size of `200`.  The maximum size for this pool
     is `1 + # of available processors`.
 
-`percolate`::
-    For percolate operations. Thread pool type is `fixed`
-    with a size of `# of available processors`,
-    queue_size of `1000`.
-
 `snapshot`::
     For snapshot/restore operations. Thread pool type is `scaling` with a
     keep-alive of `5m` and a max of `min(5, (# of available processors)/2)`.
@@ -118,14 +113,37 @@ thread_pool:
 [[processors]]
 === Processors setting
 The number of processors is automatically detected, and the thread pool
-settings are automatically set based on it. Sometimes, the number of processors
-are wrongly detected, in such cases, the number of processors can be
-explicitly set using the `processors` setting.
+settings are automatically set based on it. In some cases it can be
+useful to override the number of detected processors. This can be done
+by explicitly setting the `processors` setting.
 
 [source,yaml]
 --------------------------------------------------
 processors: 2
 --------------------------------------------------
 
+There are a few use-cases for explicitly overriding the `processors`
+setting:
+
+. If you are running multiple instances of Elasticsearch on the same
+host but want Elasticsearch to size its thread pools as if it only has a
+fraction of the CPU, you should override the `processors` setting to the
+desired fraction (e.g., if you're running two instances of Elasticsearch
+on a 16-core machine, set `processors` to 8). Note that this is an
+expert-level use-case and there's a lot more involved than just setting
+the `processors` setting as there are other considerations like changing
+the number of garbage collector threads, pinning processes to cores,
+etc.
+. The number of processors is by default bounded to 32. This means that
+on systems that have more than 32 processors, Elasticsearch will size
+its thread pools as if there are only 32 processors present. This
+limitation was added to avoid creating too many threads on systems that
+have not properly adjusted the `ulimit` for max number of processes. In
+cases where you've adjusted the `ulimit` appropriately, you can override
+this bound by explicitly setting the `processors` setting.
+. Sometimes the number of processors is wrongly detected and in such
+cases explicitly setting the `processors` setting will workaround such
+issues.
+
 In order to check the number of processors detected, use the nodes info
 API with the `os` flag.
diff --git a/docs/reference/modules/transport.asciidoc b/docs/reference/modules/transport.asciidoc
index 1c92eb24e4c96..19a0a746223f5 100644
--- a/docs/reference/modules/transport.asciidoc
+++ b/docs/reference/modules/transport.asciidoc
@@ -80,39 +80,38 @@ The following parameters can be configured like that
 * `tcp_send_buffer_size`: Configures the send buffer size of the socket
 * `tcp_receive_buffer_size`: Configures the receive buffer size of the socket
 
-[float]
-=== Local Transport
-
-This is a handy transport to use when running integration tests within
-the JVM. It is automatically enabled when using
-`NodeBuilder#local(true)`.
-
 [float]
 === Transport Tracer
 
 The transport module has a dedicated tracer logger which, when activated, logs incoming and out going requests. The log can be dynamically activated
-by settings the level of the `transport.tracer` logger to `TRACE`:
+by settings the level of the `org.elasticsearch.transport.TransportService.tracer` logger to `TRACE`:
 
 [source,js]
 --------------------------------------------------
-curl -XPUT localhost:9200/_cluster/settings -d '{
-    "transient" : {
-        "logger.transport.tracer" : "TRACE"
-    }
-}'
+PUT _cluster/settings
+{
+   "transient" : {
+      "logger.org.elasticsearch.transport.TransportService.tracer" : "TRACE"
+   }
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[warning:[script.max_compilations_per_minute] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version.]
 
 You can also control which actions will be traced, using a set of include and exclude wildcard patterns. By default every request will be traced
 except for fault detection pings:
 
 [source,js]
 --------------------------------------------------
-curl -XPUT localhost:9200/_cluster/settings -d '{
-    "transient" : {
-        "transport.tracer.include" : "*"
-        "transport.tracer.exclude" : "internal:discovery/zen/fd*"
-    }
-}'
+PUT _cluster/settings
+{
+   "transient" : {
+      "transport.tracer.include" : "*",
+      "transport.tracer.exclude" : "internal:discovery/zen/fd*"
+   }
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[warning:[script.max_compilations_per_minute] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version.]
 
 
diff --git a/docs/reference/modules/tribe.asciidoc b/docs/reference/modules/tribe.asciidoc
index a1240a28f84f9..ded8bc43a076e 100644
--- a/docs/reference/modules/tribe.asciidoc
+++ b/docs/reference/modules/tribe.asciidoc
@@ -1,6 +1,8 @@
 [[modules-tribe]]
 == Tribe node
 
+deprecated[5.4.0,The `tribe` node is deprecated in favour of <<modules-cross-cluster-search>> and will be removed in Elasticsearch 7.0.]
+
 The _tribes_ feature allows a _tribe node_ to act as a federated client across
 multiple clusters.
 
diff --git a/docs/reference/query-dsl/bool-query.asciidoc b/docs/reference/query-dsl/bool-query.asciidoc
index 2a289910c43e2..fa0430672ee4d 100644
--- a/docs/reference/query-dsl/bool-query.asciidoc
+++ b/docs/reference/query-dsl/bool-query.asciidoc
@@ -13,17 +13,24 @@ occurrence types are:
 contribute to the score.
 
 |`filter` |The clause (query) must appear in matching documents. However unlike
-`must` the score of the query will be ignored.
-
-|`should` |The clause (query) should appear in the matching document. In
-a boolean query with no `must` or `filter` clauses, one or more `should` clauses
-must match a document. The minimum number of should clauses to match can
-be set using the
-<<query-dsl-minimum-should-match,`minimum_should_match`>>
-parameter.
+`must` the score of the query will be ignored. Filter clauses are executed
+in <<query-filter-context,filter context>>, meaning that scoring is ignored
+and clauses are considered for caching.
+
+|`should` |The clause (query) should appear in the matching document. If the
+`bool` query is in a <<query-filter-context,query context>> and has a `must` or
+`filter` clause then a document will match the `bool` query even if none of the
+`should` queries match. In this case these clauses are only used to influence
+the score. If the `bool` query is a <<query-filter-context,filter context>>
+or has neither `must` or `filter` then at least one of the `should` queries
+must match a document for it to match the `bool` query. This behavior may be
+explicitly controlled by settings the
+<<query-dsl-minimum-should-match,`minimum_should_match`>> parameter.
 
 |`must_not` |The clause (query) must not appear in the matching
-documents.
+documents.  Clauses are executed in <<query-filter-context,filter context>> meaning
+that scoring is ignored and clauses are considered for caching. Because scoring is
+ignored, a score of `0` for all documents is returned.
 |=======================================================================
 
 [IMPORTANT]
@@ -34,7 +41,8 @@ clauses then at least one `should` clause is required to match.
 ========================================================================
 
 The bool query also supports `disable_coord` parameter (defaults to
-`false`). Basically the coord similarity computes a score factor based
+`false`) which changes how the `classic` similarity calculates the `bool`
+query's score. Basically the coord similarity computes a score factor based
 on the fraction of all query terms that a document contains. See Lucene
 `BooleanQuery` for more details.
 
@@ -56,7 +64,7 @@ POST _search
       },
       "must_not" : {
         "range" : {
-          "age" : { "from" : 10, "to" : 20 }
+          "age" : { "gte" : 10, "lte" : 20 }
         }
       },
       "should" : [
diff --git a/docs/reference/query-dsl/constant-score-query.asciidoc b/docs/reference/query-dsl/constant-score-query.asciidoc
index bced9fc9fbee7..aa7ee60aa5c6f 100644
--- a/docs/reference/query-dsl/constant-score-query.asciidoc
+++ b/docs/reference/query-dsl/constant-score-query.asciidoc
@@ -20,3 +20,6 @@ GET /_search
 }
 --------------------------------------------------
 // CONSOLE
+
+Filter clauses are executed in <<query-filter-context,filter context>>,
+meaning that scoring is ignored and clauses are considered for caching.
diff --git a/docs/reference/query-dsl/dis-max-query.asciidoc b/docs/reference/query-dsl/dis-max-query.asciidoc
index 2f82f2294b3d0..1f9fc53d66d9f 100644
--- a/docs/reference/query-dsl/dis-max-query.asciidoc
+++ b/docs/reference/query-dsl/dis-max-query.asciidoc
@@ -32,15 +32,15 @@ GET /_search
     "query": {
         "dis_max" : {
             "tie_breaker" : 0.7,
-                "boost" : 1.2,
-                "queries" : [
+            "boost" : 1.2,
+            "queries" : [
                 {
                     "term" : { "age" : 34 }
                 },
                 {
                     "term" : { "age" : 35 }
                 }
-                ]
+            ]
         }
     }
 }    
diff --git a/docs/reference/query-dsl/exists-query.asciidoc b/docs/reference/query-dsl/exists-query.asciidoc
index 4971219366f94..fc851435c35da 100644
--- a/docs/reference/query-dsl/exists-query.asciidoc
+++ b/docs/reference/query-dsl/exists-query.asciidoc
@@ -51,7 +51,7 @@ instance, if the `user` field were mapped as follows:
 [source,js]
 --------------------------------------------------
   "user": {
-    "type": "text",
+    "type": "keyword",
     "null_value": "_null_"
   }
 --------------------------------------------------
diff --git a/docs/reference/query-dsl/function-score-query.asciidoc b/docs/reference/query-dsl/function-score-query.asciidoc
index 3072372d17924..76d2becbdbc44 100644
--- a/docs/reference/query-dsl/function-score-query.asciidoc
+++ b/docs/reference/query-dsl/function-score-query.asciidoc
@@ -27,6 +27,7 @@ GET /_search
 }
 --------------------------------------------------
 // CONSOLE
+// TEST[setup:twitter]
 
 <1> See <<score-functions>> for a list of supported functions.
 
@@ -62,6 +63,7 @@ GET /_search
 }
 --------------------------------------------------
 // CONSOLE
+// TEST[setup:twitter]
 
 <1> Boost for the whole query.
 <2> See <<score-functions>> for a list of supported functions.
@@ -110,6 +112,8 @@ query. The parameter `boost_mode` defines how:
 By default, modifying the score does not change which documents match. To exclude
 documents that do not meet a certain score threshold the `min_score` parameter can be set to the desired score threshold.
 
+NOTE: For `min_score` to work, **all** documents returned by the query need to be scored and then filtered out one by one.
+
 [[score-functions]]
 
 The `function_score` query provides several types of score functions.
@@ -130,35 +134,57 @@ simple sample:
 
 [source,js]
 --------------------------------------------------
-"script_score" : {
-    "script" : {
-      "lang": "painless",
-      "inline": "_score * doc['my_numeric_field'].value"
+GET /_search
+{
+    "query": {
+        "function_score": {
+            "query": {
+                "match": { "message": "elasticsearch" }
+            },
+            "script_score" : {
+                "script" : {
+                  "source": "Math.log(2 + doc['likes'].value)"
+                }
+            }
+        }
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
 
 On top of the different scripting field values and expression, the
 `_score` script parameter can be used to retrieve the score based on the
 wrapped query.
 
-Scripts are cached for faster execution. If the script has parameters
-that it needs to take into account, it is preferable to reuse the same
-script, and provide parameters to it:
+Scripts compilation is cached for faster execution. If the script has
+parameters that it needs to take into account, it is preferable to reuse the
+same script, and provide parameters to it:
 
 [source,js]
 --------------------------------------------------
-"script_score": {
-    "script": {
-        "lang": "painless",
-        "params": {
-            "param1": value1,
-            "param2": value2
-        },
-        "inline": "_score * doc['my_numeric_field'].value / Math.pow(params.param1, params.param2)"
+GET /_search
+{
+    "query": {
+        "function_score": {
+            "query": {
+                "match": { "message": "elasticsearch" }
+            },
+            "script_score" : {
+                "script" : {
+                    "params": {
+                        "a": 5,
+                        "b": 1.2
+                    },
+                    "source": "params.a / Math.pow(params.b, doc['likes'].value)"
+                }
+            }
+        }
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
 
 Note that unlike the `custom_score` query, the
 score of the query is multiplied with the result of the script scoring. If
@@ -176,6 +202,8 @@ not. The number value is of type float.
 --------------------------------------------------
 "weight" : number
 --------------------------------------------------
+// NOTCONSOLE
+// I couldn't come up with a good example for this one.
 
 [[function-random]]
 ==== Random
@@ -189,10 +217,19 @@ be a memory intensive operation since the values are unique.
 
 [source,js]
 --------------------------------------------------
-"random_score": {
-    "seed" : number
+GET /_search
+{
+    "query": {
+        "function_score": {
+            "random_score": {
+                "seed": 10
+            }
+        }
+    }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
 
 [[function-field-value-factor]]
 ==== Field Value factor
@@ -202,23 +239,32 @@ influence the score. It's similar to using the `script_score` function, however,
 it avoids the overhead of scripting. If used on a multi-valued field, only the
 first value of the field is used in calculations.
 
-As an example, imagine you have a document indexed with a numeric `popularity`
+As an example, imagine you have a document indexed with a numeric `likes`
 field and wish to influence the score of a document with this field, an example
 doing so would look like:
 
 [source,js]
 --------------------------------------------------
-"field_value_factor": {
-  "field": "popularity",
-  "factor": 1.2,
-  "modifier": "sqrt",
-  "missing": 1
+GET /_search
+{
+    "query": {
+        "function_score": {
+            "field_value_factor": {
+                "field": "likes",
+                "factor": 1.2,
+                "modifier": "sqrt",
+                "missing": 1
+            }
+        }
+    }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
 
 Which will translate into the following formula for scoring:
 
-`sqrt(1.2 * doc['popularity'].value)`
+`sqrt(1.2 * doc['likes'].value)`
 
 There are a number of options for the `field_value_factor` function:
 
@@ -289,23 +335,36 @@ decay function is specified as
     }
 }
 --------------------------------------------------
+// NOTCONSOLE
 <1> The `DECAY_FUNCTION` should be one of `linear`, `exp`, or `gauss`.
 <2> The specified field must be a numeric, date, or geo-point field.
 
-In the above example, the field is a <<geo-point,`geo_point`>> and origin can be provided in geo format. `scale` and `offset` must be given with a unit in this case. If your field is a date field, you can set `scale` and `offset` as days, weeks, and so on. Example:
+In the above example, the field is a <<geo-point,`geo_point`>> and origin can
+be provided in geo format. `scale` and `offset` must be given with a unit in
+this case. If your field is a date field, you can set `scale` and `offset` as
+days, weeks, and so on. Example:
 
 
 [source,js]
 --------------------------------------------------
-    "gauss": {
-        "date": {
-              "origin": "2013-09-17", <1>
-              "scale": "10d",
-              "offset": "5d", <2>
-              "decay" : 0.5 <2>
+GET /_search
+{
+    "query": {
+        "function_score": {
+            "gauss": {
+                "date": {
+                      "origin": "2013-09-17", <1>
+                      "scale": "10d",
+                      "offset": "5d", <2>
+                      "decay" : 0.5 <2>
+                }
+            }
         }
     }
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
 <1> The date format of the origin depends on the <<mapping-date-format,`format`>> defined in
     your mapping. If you do not define the origin, the current time is used.
 <2> The `offset` and `decay` parameters are optional.
@@ -318,7 +377,7 @@ In the above example, the field is a <<geo-point,`geo_point`>> and origin can be
     math (for example `now-1h`) is supported for origin.
 
 `scale`::
-    Required for all types. Defines the distance from origin + offest at which the computed
+    Required for all types. Defines the distance from origin + offset at which the computed
     score will equal `decay` parameter. For geo fields: Can be defined as number+unit (1km, 12m,...).
     Default unit is meters. For date fields: Can to be defined as a number+unit ("1h", "10d",...).
     Default unit is milliseconds. For numeric field: Any number.
@@ -426,6 +485,7 @@ Example:
         "multi_value_mode": "avg"
     }
 --------------------------------------------------
+// NOTCONSOLE
 
 
 ==== Detailed example
@@ -462,6 +522,7 @@ The function for `price` in this case would be
     }
 }
 --------------------------------------------------
+// NOTCONSOLE
 <1> This decay function could also be `linear` or `exp`.
 
 and for `location`:
@@ -476,6 +537,7 @@ and for `location`:
     }
 }
 --------------------------------------------------
+// NOTCONSOLE
 <1> This decay function could also be `linear` or `exp`.
 
 Suppose you want to multiply these two functions on the original score,
diff --git a/docs/reference/query-dsl/fuzzy-query.asciidoc b/docs/reference/query-dsl/fuzzy-query.asciidoc
index f320e81b579af..70f7eb48ada7f 100644
--- a/docs/reference/query-dsl/fuzzy-query.asciidoc
+++ b/docs/reference/query-dsl/fuzzy-query.asciidoc
@@ -1,8 +1,6 @@
 [[query-dsl-fuzzy-query]]
 === Fuzzy Query
 
-deprecated[5.0.0, Will be removed in 6.0. Use match queries with fuzziness instead]
-
 The fuzzy query uses similarity based on Levenshtein edit distance.
 
 ==== String fields
diff --git a/docs/reference/query-dsl/geo-bounding-box-query.asciidoc b/docs/reference/query-dsl/geo-bounding-box-query.asciidoc
index 40debb5710527..1307d0184f143 100644
--- a/docs/reference/query-dsl/geo-bounding-box-query.asciidoc
+++ b/docs/reference/query-dsl/geo-bounding-box-query.asciidoc
@@ -92,7 +92,7 @@ Default is `memory`.
 ==== Accepted Formats
 
 In much the same way the geo_point type can accept different
-representation of the geo point, the filter can accept it as well:
+representations of the geo point, the filter can accept it as well:
 
 [float]
 ===== Lat Lon As Properties
diff --git a/docs/reference/query-dsl/geo-distance-query.asciidoc b/docs/reference/query-dsl/geo-distance-query.asciidoc
index c6496eac39d24..eb419077a8165 100644
--- a/docs/reference/query-dsl/geo-distance-query.asciidoc
+++ b/docs/reference/query-dsl/geo-distance-query.asciidoc
@@ -69,7 +69,7 @@ GET /my_locations/location/_search
 ==== Accepted Formats
 
 In much the same way the `geo_point` type can accept different
-representation of the geo point, the filter can accept it as well:
+representations of the geo point, the filter can accept it as well:
 
 [float]
 ===== Lat Lon As Properties
@@ -191,7 +191,7 @@ The following are options allowed on the filter:
 
 `distance_type`::
 
-    How to compute the distance. Can either be `sloppy_arc` (default), `arc` (slightly more precise but significantly slower) or `plane` (faster, but inaccurate on long distances and close to the poles).
+    How to compute the distance. Can either be `arc` (default), or `plane` (faster, but inaccurate on long distances and close to the poles).
 
 `optimize_bbox`::
 
diff --git a/docs/reference/query-dsl/geo-distance-range-query.asciidoc b/docs/reference/query-dsl/geo-distance-range-query.asciidoc
index c54c50638bc08..4eec3bf543cb0 100644
--- a/docs/reference/query-dsl/geo-distance-range-query.asciidoc
+++ b/docs/reference/query-dsl/geo-distance-range-query.asciidoc
@@ -1,6 +1,10 @@
 [[query-dsl-geo-distance-range-query]]
 === Geo Distance Range Query
 
+deprecated[5.0.0,Elasticsearch will continue to support geo_distance_range queries on indexes created
+prior to 5.0.0. Range searches on indexes created in 5.0.0 and later will no longer be supported.
+Distance aggregations or sorting should be used instead.]
+
 Filters documents that exists within a range from a specific point:
 
 [source,js]
diff --git a/docs/reference/query-dsl/geo-polygon-query.asciidoc b/docs/reference/query-dsl/geo-polygon-query.asciidoc
index 35278fda4a55e..4f01da2548213 100644
--- a/docs/reference/query-dsl/geo-polygon-query.asciidoc
+++ b/docs/reference/query-dsl/geo-polygon-query.asciidoc
@@ -69,7 +69,7 @@ GET /_search
                     "person.location" : {
                         "points" : [
                             [-70, 40],
-                        [-80, 30],
+                            [-80, 30],
                             [-90, 20]
                         ]
                     }
diff --git a/docs/reference/query-dsl/geo-queries.asciidoc b/docs/reference/query-dsl/geo-queries.asciidoc
index 9aaf286526dee..5fdd6114459aa 100644
--- a/docs/reference/query-dsl/geo-queries.asciidoc
+++ b/docs/reference/query-dsl/geo-queries.asciidoc
@@ -24,18 +24,13 @@ The queries in this group are:
 
 <<query-dsl-geo-distance-range-query,`geo_distance_range`>> query::
 
-    Like the `geo_point` query, but the range starts at a specified distance
+    Like the `geo_distance` query, but the range starts at a specified distance
     from the central point.
 
 <<query-dsl-geo-polygon-query,`geo_polygon`>> query::
 
     Find documents with geo-points within the specified polygon.
 
-<<query-dsl-geohash-cell-query,`geohash_cell`>> query::
-
-    Find geo-points whose geohash intersects with the geohash of the specified
-    point.
-
 
 include::geo-shape-query.asciidoc[]
 
@@ -46,5 +41,3 @@ include::geo-distance-query.asciidoc[]
 include::geo-distance-range-query.asciidoc[]
 
 include::geo-polygon-query.asciidoc[]
-
-include::geohash-cell-query.asciidoc[]
diff --git a/docs/reference/query-dsl/geo-shape-query.asciidoc b/docs/reference/query-dsl/geo-shape-query.asciidoc
index 1220306133670..00fd3b5609b54 100644
--- a/docs/reference/query-dsl/geo-shape-query.asciidoc
+++ b/docs/reference/query-dsl/geo-shape-query.asciidoc
@@ -6,7 +6,7 @@ Filter documents indexed using the `geo_shape` type.
 Requires the <<geo-shape,`geo_shape` Mapping>>.
 
 The `geo_shape` query uses the same grid square representation as the
-geo_shape mapping to find documents that have a shape that intersects
+`geo_shape` mapping to find documents that have a shape that intersects
 with the query shape. It will also use the same PrefixTree configuration
 as defined for the field mapping.
 
@@ -17,28 +17,44 @@ examples.
 
 ==== Inline Shape Definition
 
-Similar to the `geo_shape` type, the `geo_shape` Filter uses
+Similar to the `geo_shape` type, the `geo_shape` query uses
 http://www.geojson.org[GeoJSON] to represent shapes.
 
-Given a document that looks like this:
+Given the following index:
 
 [source,js]
 --------------------------------------------------
+PUT /example
 {
-    "name": "Wind & Wetter, Berlin, Germany",
-        "location": {
-            "type": "Point",
-            "coordinates": [13.400544, 52.530286]
+    "mappings": {
+        "doc": {
+            "properties": {
+                "location": {
+                    "type": "geo_shape"
+                }
+            }
         }
+    }
+}
+
+POST /example/doc?refresh
+{
+    "name": "Wind & Wetter, Berlin, Germany",
+    "location": {
+        "type": "point",
+        "coordinates": [13.400544, 52.530286]
+    }
 }
 --------------------------------------------------
+// CONSOLE
+// TESTSETUP
 
 The following query will find the point using the Elasticsearch's
 `envelope` GeoJSON extension:
 
 [source,js]
 --------------------------------------------------
-GET /_search
+GET /example/_search
 {
     "query":{
         "bool": {
@@ -83,25 +99,43 @@ shape:
 
 [source,js]
 --------------------------------------------------
-GET /_search
+PUT /shapes
+{
+    "mappings": {
+        "doc": {
+            "properties": {
+                "location": {
+                    "type": "geo_shape"
+                }
+            }
+        }
+    }
+}
+
+PUT /shapes/doc/deu
+{
+    "location": {
+        "type": "envelope",
+        "coordinates" : [[13.0, 53.0], [14.0, 52.0]]
+    }
+}
+
+GET /example/_search
 {
     "query": {
         "bool": {
-            "must": {
-                "match_all": {}
-            },
-                "filter": {
-                    "geo_shape": {
-                        "location": {
-                            "indexed_shape": {
-                                "id": "DEU",
-                                "type": "countries",
-                                "index": "shapes",
-                                "path": "location"
-                            }
+            "filter": {
+                "geo_shape": {
+                    "location": {
+                        "indexed_shape": {
+                            "index": "shapes",
+                            "type": "doc",
+                            "id": "deu",
+                            "path": "location"
                         }
                     }
                 }
+            }
         }
     }
 }
diff --git a/docs/reference/query-dsl/geohash-cell-query.asciidoc b/docs/reference/query-dsl/geohash-cell-query.asciidoc
deleted file mode 100644
index e6bcfea028879..0000000000000
--- a/docs/reference/query-dsl/geohash-cell-query.asciidoc
+++ /dev/null
@@ -1,81 +0,0 @@
-[[query-dsl-geohash-cell-query]]
-=== Geohash Cell Query
-
-The `geohash_cell` query provides access to a hierarchy of geohashes.
-By defining a geohash cell, only <<geo-point,geopoints>>
-within this cell will match this filter.
-
-To get this filter work all prefixes of a geohash need to be indexed. In
-example a geohash `u30` needs to be decomposed into three terms: `u30`,
-`u3` and `u`. This decomposition must be enabled in the mapping of the
-<<geo-point,geopoint>> field that's going to be filtered by
-setting the `geohash_prefix` option:
-
-[source,js]
---------------------------------------------------
-PUT /my_index
-{
-    "mappings" : {
-        "location": {
-            "properties": {
-                "pin": {
-                    "type": "geo_point",
-                    "geohash": true,
-                    "geohash_prefix": true,
-                    "geohash_precision": 10
-                }
-            }
-        }
-    }
-}
---------------------------------------------------
-// CONSOLE
-// TEST[warning:geo_point geohash parameter is deprecated and will be removed in the next major release]
-// TEST[warning:geo_point geohash_precision parameter is deprecated and will be removed in the next major release]
-// TEST[warning:geo_point geohash_prefix parameter is deprecated and will be removed in the next major release]
-
-The geohash cell can defined by all formats of `geo_points`. If such a cell is
-defined by a latitude and longitude pair the size of the cell needs to be
-setup. This can be done by the `precision` parameter of the filter. This
-parameter can be set to an integer value which sets the length of the geohash
-prefix. Instead of setting a geohash length directly it is also possible to
-define the precision as distance, in example `"precision": "50m"`. (See
-<<distance-units>>.)
-
-The `neighbor` option of the filter offers the possibility to filter cells
-next to the given cell.
-
-[source,js]
---------------------------------------------------
-GET /_search
-{
-    "query": {
-        "bool" : {
-            "must" : {
-                "match_all" : {}
-            },
-            "filter" : {
-                "geohash_cell": {
-                    "pin": {
-                        "lat": 13.4080,
-                        "lon": 52.5186
-                    },
-                    "precision": 3,
-                    "neighbors": true
-                }
-            }
-        }
-    }
-}
---------------------------------------------------
-// CONSOLE
-// TEST[continued]
-
-[float]
-==== Ignore Unmapped
-
-When set to `true` the `ignore_unmapped` option will ignore an unmapped field
-and will not match any documents for this query. This can be useful when
-querying multiple indexes which might have different mappings. When set to
-`false` (the default value) the query will throw an exception if the field
-is not mapped.
diff --git a/docs/reference/query-dsl/has-child-query.asciidoc b/docs/reference/query-dsl/has-child-query.asciidoc
index 1d648a5d29407..bfe7eff4c2f73 100644
--- a/docs/reference/query-dsl/has-child-query.asciidoc
+++ b/docs/reference/query-dsl/has-child-query.asciidoc
@@ -12,11 +12,11 @@ GET /_search
     "query": {
         "has_child" : {
             "type" : "blog_tag",
-                "query" : {
-                    "term" : {
-                        "tag" : "something"
-                    }
+            "query" : {
+                "term" : {
+                    "tag" : "something"
                 }
+            }
         }
     }
 }
@@ -41,12 +41,12 @@ GET /_search
     "query": {
         "has_child" : {
             "type" : "blog_tag",
-                "score_mode" : "min",
-                "query" : {
-                    "term" : {
-                        "tag" : "something"
-                    }
+            "score_mode" : "min",
+            "query" : {
+                "term" : {
+                    "tag" : "something"
                 }
+            }
         }
     }
 }
@@ -123,4 +123,4 @@ GET /_search
     }
 }
 --------------------------------------------------
-// CONSOLE
\ No newline at end of file
+// CONSOLE
diff --git a/docs/reference/query-dsl/has-parent-query.asciidoc b/docs/reference/query-dsl/has-parent-query.asciidoc
index 09ec3383cc246..a1dcf605ddd4a 100644
--- a/docs/reference/query-dsl/has-parent-query.asciidoc
+++ b/docs/reference/query-dsl/has-parent-query.asciidoc
@@ -14,11 +14,11 @@ GET /_search
     "query": {
         "has_parent" : {
             "parent_type" : "blog",
-                "query" : {
-                    "term" : {
-                        "tag" : "something"
-                    }
+            "query" : {
+                "term" : {
+                    "tag" : "something"
                 }
+            }
         }
     }
 }
@@ -43,12 +43,12 @@ GET /_search
     "query": {
         "has_parent" : {
             "parent_type" : "blog",
-                "score" : true,
-                "query" : {
-                    "term" : {
-                        "tag" : "something"
-                    }
+            "score" : true,
+            "query" : {
+                "term" : {
+                    "tag" : "something"
                 }
+            }
         }
     }
 }
@@ -69,7 +69,7 @@ is not mapped.
 
 Child documents can't be sorted by fields in matching parent documents via the
 regular sort options. If you need to sort child documents by field in the parent
-documents then you can should use the `function_score` query and then just sort
+documents then you should use the `function_score` query and then just sort
 by `_score`.
 
 Sorting tags by parent document' `view_count` field:
@@ -93,4 +93,4 @@ GET /_search
     }
 }
 --------------------------------------------------
-// CONSOLE
\ No newline at end of file
+// CONSOLE
diff --git a/docs/reference/query-dsl/joining-queries.asciidoc b/docs/reference/query-dsl/joining-queries.asciidoc
index cfbbf5360b687..6a467fe539a2f 100644
--- a/docs/reference/query-dsl/joining-queries.asciidoc
+++ b/docs/reference/query-dsl/joining-queries.asciidoc
@@ -7,7 +7,7 @@ which are designed to scale horizontally.
 
 <<query-dsl-nested-query,`nested` query>>::
 
-Documents may contains fields of type <<nested,`nested`>>. These
+Documents may contain fields of type <<nested,`nested`>>. These
 fields are used to index arrays of objects, where each object can be queried
 (with the `nested` query) as an independent document.
 
diff --git a/docs/reference/query-dsl/match-phrase-prefix-query.asciidoc b/docs/reference/query-dsl/match-phrase-prefix-query.asciidoc
index 40cfabdc96ff1..73f1be9143cf2 100644
--- a/docs/reference/query-dsl/match-phrase-prefix-query.asciidoc
+++ b/docs/reference/query-dsl/match-phrase-prefix-query.asciidoc
@@ -19,7 +19,7 @@ GET /_search
 
 It accepts the same parameters as the phrase type. In addition, it also
 accepts a `max_expansions` parameter (default `50`) that can control to how
-many prefixes the last term will be expanded. It is highly recommended to set
+many suffixes the last term will be expanded. It is highly recommended to set
 it to an acceptable value to control the execution time of the query. For
 example:
 
@@ -43,7 +43,7 @@ GET /_search
 ===================================================
 
 The `match_phrase_prefix` query is a poor-man's autocomplete.  It is very easy
-to use, which let's you get started quickly with _search-as-you-type_ but it's
+to use, which lets you get started quickly with _search-as-you-type_ but its
 results, which usually are good enough,  can sometimes be confusing.
 
 Consider the query string `quick brown f`.  This query works by creating a
@@ -53,7 +53,7 @@ dictionary to find the first 50 terms that begin with `f`, and
 adds these terms to the phrase query.
 
 The problem is that the first 50 terms may not include the term `fox` so the
-phase `quick brown fox` will not be found.  This usually isn't a problem as
+phrase `quick brown fox` will not be found.  This usually isn't a problem as
 the user will continue to type more letters until the word they are looking
 for appears.
 
diff --git a/docs/reference/query-dsl/mlt-query.asciidoc b/docs/reference/query-dsl/mlt-query.asciidoc
index 8e23afdbb8623..c80b71c613503 100644
--- a/docs/reference/query-dsl/mlt-query.asciidoc
+++ b/docs/reference/query-dsl/mlt-query.asciidoc
@@ -112,9 +112,10 @@ analyzes it, usually using the same analyzer at the field, then selects the
 top K terms with highest tf-idf to form a disjunctive query of these terms.
 
 IMPORTANT: The fields on which to perform MLT must be indexed and of type
-`string`. Additionally, when using `like` with documents, either `_source`
-must be enabled or the fields must be `stored` or store `term_vector`. In
-order to speed up analysis, it could help to store term vectors at index time.
+`text` or `keyword``. Additionally, when using `like` with documents, either
+`_source` must be enabled or the fields must be `stored` or store
+`term_vector`. In order to speed up analysis, it could help to store term
+vectors at index time.
 
 For example, if we wish to perform MLT on the "title" and "tags.raw" fields,
 we can explicitly store their `term_vector` at index time. We can still
diff --git a/docs/reference/query-dsl/multi-match-query.asciidoc b/docs/reference/query-dsl/multi-match-query.asciidoc
index 068156ff70fd6..4ad75c6ca75d7 100644
--- a/docs/reference/query-dsl/multi-match-query.asciidoc
+++ b/docs/reference/query-dsl/multi-match-query.asciidoc
@@ -136,7 +136,7 @@ follows:
   * plus `tie_breaker * _score` for all other matching fields
 
 Also, accepts `analyzer`, `boost`, `operator`, `minimum_should_match`,
-`fuzziness`, `prefix_length`, `max_expansions`, `rewrite`, `zero_terms_query`
+`fuzziness`, `lenient`, `prefix_length`, `max_expansions`, `rewrite`, `zero_terms_query`
 and `cutoff_frequency`, as explained in <<query-dsl-match-query, match query>>.
 
 [IMPORTANT]
@@ -232,7 +232,7 @@ The score from each `match` clause is added together, then divided by the
 number of `match` clauses.
 
 Also, accepts `analyzer`, `boost`, `operator`, `minimum_should_match`,
-`fuzziness`, `prefix_length`, `max_expansions`, `rewrite`, `zero_terms_query`
+`fuzziness`, `lenient`, `prefix_length`, `max_expansions`, `rewrite`, `zero_terms_query`
 and `cutoff_frequency`, as explained in <<query-dsl-match-query,match query>>, but
 *see <<operator-min>>*.
 
@@ -277,7 +277,7 @@ GET /_search
 --------------------------------------------------
 // CONSOLE
 
-Also, accepts `analyzer`, `boost`, `slop` and `zero_terms_query`  as explained
+Also, accepts `analyzer`, `boost`, `lenient`, `slop` and `zero_terms_query`  as explained
 in <<query-dsl-match-query>>.  Type `phrase_prefix` additionally accepts
 `max_expansions`.
 
@@ -374,7 +374,7 @@ explanation:
     +blended("smith", fields: [first_name, last_name])
 
 Also, accepts `analyzer`, `boost`, `operator`, `minimum_should_match`,
-`zero_terms_query` and `cutoff_frequency`, as explained in
+`lenient`, `zero_terms_query` and `cutoff_frequency`, as explained in
 <<query-dsl-match-query, match query>>.
 
 ===== `cross_field` and analysis
diff --git a/docs/reference/query-dsl/nested-query.asciidoc b/docs/reference/query-dsl/nested-query.asciidoc
index 176473174ddc9..0634af86a1977 100644
--- a/docs/reference/query-dsl/nested-query.asciidoc
+++ b/docs/reference/query-dsl/nested-query.asciidoc
@@ -40,8 +40,8 @@ GET /_search
             "query" : {
                 "bool" : {
                     "must" : [
-                    { "match" : {"obj1.name" : "blue"} },
-                    { "range" : {"obj1.count" : {"gt" : 5}} }
+                        { "match" : {"obj1.name" : "blue"} },
+                        { "range" : {"obj1.count" : {"gt" : 5}} }
                     ]
                 }
             }
diff --git a/docs/reference/query-dsl/parent-id-query.asciidoc b/docs/reference/query-dsl/parent-id-query.asciidoc
index f662dc825c001..c46d1a8085da2 100644
--- a/docs/reference/query-dsl/parent-id-query.asciidoc
+++ b/docs/reference/query-dsl/parent-id-query.asciidoc
@@ -29,7 +29,7 @@ PUT /my_index
     }
 }
 
-------------------------------------------
+--------------------------------------------
 // CONSOLE
 // TESTSETUP
 
@@ -40,7 +40,7 @@ GET /my_index/_search
     "query": {
         "parent_id" : {
             "type" : "blog_tag",
-                "id" : "1"
+            "id" : "1"
         }
     }
 }
diff --git a/docs/reference/query-dsl/percolate-query.asciidoc b/docs/reference/query-dsl/percolate-query.asciidoc
index e53e468f540d8..6949e5f3a94b7 100644
--- a/docs/reference/query-dsl/percolate-query.asciidoc
+++ b/docs/reference/query-dsl/percolate-query.asciidoc
@@ -93,6 +93,7 @@ The above request will yield the following response:
   "_shards": {
     "total": 5,
     "successful": 5,
+    "skipped" : 0,
     "failed": 0
   },
   "hits": {
@@ -126,11 +127,11 @@ The above request will yield the following response:
 The following parameters are required when percolating a document:
 
 [horizontal]
-`field`:: The field of type `percolator` and that holds the indexed queries. This is a required parameter.
+`field`:: The field of type `percolator` that holds the indexed queries. This is a required parameter.
 `document_type`:: The type / mapping of the document being percolated. This is a required parameter.
 `document`:: The source of the document being percolated.
 
-Instead of specifying a the source of the document being percolated, the source can also be retrieved from an already
+Instead of specifying the source of the document being percolated, the source can also be retrieved from an already
 stored document. The `percolate` query will then internally execute a get request to fetch that document.
 
 In that case the `document` parameter can be substituted with the following parameters:
@@ -207,7 +208,7 @@ GET /my-index/_search
 // CONSOLE
 // TEST[continued]
 
-<1> The version is optional, but useful in certain cases. We can then ensure that we are try to percolate
+<1> The version is optional, but useful in certain cases. We can ensure that we are trying to percolate
 the document we just have indexed. A change may be made after we have indexed, and if that is the
 case the then the search request would fail with a version conflict error.
 
@@ -292,6 +293,7 @@ This will yield the following response.
   "_shards": {
     "total": 5,
     "successful": 5,
+    "skipped" : 0,
     "failed": 0
   },
   "hits": {
@@ -340,6 +342,8 @@ This will yield the following response.
 --------------------------------------------------
 // TESTRESPONSE[s/"took": 7,/"took": "$body.took",/]
 
+<1> The terms from each query have been highlighted in the document.
+
 Instead of the query in the search request highlighting the percolator hits, the percolator queries are highlighting
 the document defined in the `percolate` query.
 
@@ -347,12 +351,12 @@ the document defined in the `percolate` query.
 ==== How it Works Under the Hood
 
 When indexing a document into an index that has the <<percolator,percolator field type>> mapping configured, the query
-part of the documents gets parsed into a Lucene query and are stored into the Lucene index. A binary representation
+part of the document gets parsed into a Lucene query and is stored into the Lucene index. A binary representation
 of the query gets stored, but also the query's terms are analyzed and stored into an indexed field.
 
 At search time, the document specified in the request gets parsed into a Lucene document and is stored in a in-memory
 temporary Lucene index. This in-memory index can just hold this one document and it is optimized for that. After this
-a special query is build based on the terms in the in-memory index that select candidate percolator queries based on
+a special query is built based on the terms in the in-memory index that select candidate percolator queries based on
 their indexed query terms. These queries are then evaluated by the in-memory index if they actually match.
 
 The selecting of candidate percolator queries matches is an important performance optimization during the execution
diff --git a/docs/reference/query-dsl/query-string-query.asciidoc b/docs/reference/query-dsl/query-string-query.asciidoc
index 60477d6e28a1a..c4a845acc7444 100644
--- a/docs/reference/query-dsl/query-string-query.asciidoc
+++ b/docs/reference/query-dsl/query-string-query.asciidoc
@@ -40,10 +40,6 @@ with default operator of `AND`, the same query is translated to
 |`allow_leading_wildcard` |When set, `*` or `?` are allowed as the first
 character. Defaults to `true`.
 
-|`lowercase_expanded_terms` |Whether terms of wildcard, prefix, fuzzy,
-and range queries are to be automatically lower-cased or not (since they
-are not analyzed). Default it `true`.
-
 |`enable_position_increments` |Set to `true` to enable position
 increments in result queries. Defaults to `true`.
 
@@ -61,12 +57,12 @@ phrase matches are required. Default value is `0`.
 
 |`boost` |Sets the boost value of the query. Defaults to `1.0`.
 
+|`auto_generate_phrase_queries` |Defaults to `false`.
+
 |`analyze_wildcard` |By default, wildcards terms in a query string are
 not analyzed. By setting this value to `true`, a best effort will be
 made to analyze those as well.
 
-|`auto_generate_phrase_queries` |Defaults to `false`.
-
 |`max_determinized_states` |Limit on how many automaton states regexp
 queries are allowed to create.  This protects against too-difficult
 (e.g. exponentially hard) regexps.  Defaults to 10000.
@@ -80,11 +76,23 @@ both>>.
 |`lenient` |If set to `true` will cause format based failures (like
 providing text to a numeric field) to be ignored.
 
-|`locale` | Locale that should be used for string conversions.
-Defaults to `ROOT`.
-
 |`time_zone` | Time Zone to be applied to any range query related to dates. See also
 http://www.joda.org/joda-time/apidocs/org/joda/time/DateTimeZone.html[JODA timezone].
+
+|`quote_field_suffix` | A suffix to append to fields for quoted parts of
+the query string. This allows to use a field that has a different analysis chain
+for exact matching. Look <<mixing-exact-search-with-stemming,here>> for a
+comprehensive example.
+
+|`split_on_whitespace` |Whether query text should be split on whitespace prior to analysis.
+Instead  the queryparser would parse around only real 'operators'. Defaults to `true`.
+It is not allowed to set this option to `false` if `auto_generate_phrase_queries` is already set to `true`.
+
+|`all_fields` | Perform the query on all fields detected in the mapping that can
+be queried. Will be used by default when the `_all` field is disabled and no
+`default_field` is specified (either in the index settings or in the request
+body) and no `fields` are specified.
+
 |=======================================================================
 
 When a multi term query is being generated, one can control how it gets
@@ -99,8 +107,10 @@ When not explicitly specifying the field to search on in the query
 string syntax, the `index.query.default_field` will be used to derive
 which field to search on. It defaults to `_all` field.
 
-So, if `_all` field is disabled, it might make sense to change it to set
-a different default field.
+If the `_all` field is disabled, the `query_string` query will automatically
+attempt to determine the existing fields in the index's mapping that are
+queryable, and perform the search on those fields. Note that this will not
+include nested documents, use a nested query to search those documents.
 
 [float]
 ==== Multi Field
@@ -187,7 +197,24 @@ GET /_search
 
 Another option is to provide the wildcard fields search in the query
 string itself (properly escaping the `*` sign), for example:
-`city.\*:something`.
+`city.\*:something`:
+
+[source,js]
+--------------------------------------------------
+GET /_search
+{
+    "query": {
+        "query_string" : {
+            "query" : "city.\\*:(this AND that OR thus)",
+            "use_dis_max" : true
+        }
+    }
+}
+--------------------------------------------------
+// CONSOLE
+
+NOTE: Since `\` (backslash) is a special character in json strings, it needs to
+be escaped, hence the two backslashes in the above `query_string`.
 
 When running the `query_string` query against multiple fields, the
 following additional parameters are allowed:
diff --git a/docs/reference/query-dsl/query-string-syntax.asciidoc b/docs/reference/query-dsl/query-string-syntax.asciidoc
index 9e847102469cf..b0692c3b5cdf7 100644
--- a/docs/reference/query-dsl/query-string-syntax.asciidoc
+++ b/docs/reference/query-dsl/query-string-syntax.asciidoc
@@ -53,6 +53,25 @@ Be aware that wildcard queries can use an enormous amount of memory and
 perform very badly -- just think how many terms need to be queried to
 match the query string `"a* b* c*"`.
 
+[WARNING]
+=======
+Pure wildcards `\*` are rewritten to <<query-dsl-exists-query,`exists`>> queries for efficiency.
+As a consequence, the wildcard `"field:*"` would match documents with an emtpy value
+ like the following:
+```
+{
+  "field": ""
+}
+```
+\... and would **not** match if the field is missing or set with an explicit null
+value like the following:
+```
+{
+  "field": null
+}
+```
+=======
+
 [WARNING]
 =======
 Allowing a wildcard at the beginning of a word (eg `"*ing"`) is particularly
@@ -61,12 +80,15 @@ they match.  Leading wildcards can be disabled by setting
 `allow_leading_wildcard` to `false`.
 =======
 
-Wildcarded terms are not analyzed by default -- they are lowercased
-(`lowercase_expanded_terms` defaults to `true`) but no further analysis
-is done, mainly because it is impossible to accurately analyze a word that
-is missing some of its letters.  However, by setting `analyze_wildcard` to
-`true`, an attempt will be made to analyze wildcarded words before searching
-the term list for matching terms.
+Only parts of the analysis chain that operate at the character level are
+applied. So for instance, if the analyzer performs both lowercasing and
+stemming, only the lowercasing will be applied: it would be wrong to perform
+stemming on a word that is missing some of its letters.
+
+By setting `analyze_wildcard` to true, queries that end with a `*` will be
+analyzed and a boolean query will be built out of the different tokens, by
+ensuring exact matches on the first N-1 tokens, and prefix match on the last
+token.
 
 ===== Regular expressions
 
@@ -276,14 +298,18 @@ The reserved characters are:  `+ - = && || > < ! ( ) { } [ ] ^ " ~ * ? : \ /`
 Failing to escape these special characters correctly could lead to a syntax
 error which prevents your query from running.
 
+NOTE: `<` and `>` can't be escaped at all. The only way to prevent them from
+attempting to create a range query is to remove them from the query string
+entirely.
+
 .Watch this space
 ****
 A space may also be a reserved character.  For instance, if you have a
 synonym list which converts `"wi fi"` to `"wifi"`, a `query_string` search
 for `"wi fi"` would fail. The query string parser would interpret your
 query as a search for `"wi OR fi"`, while the token stored in your
-index is actually `"wifi"`.  Escaping the space will protect it from
-being touched by the query string parser: `"wi\ fi"`.
+index is actually `"wifi"`.  The option `split_on_whitespace=false` will protect it from
+being touched by the query string parser and will let the analysis run on the entire input (`"wi fi"`).
 ****
 
 ===== Empty Query
diff --git a/docs/reference/query-dsl/regexp-syntax.asciidoc b/docs/reference/query-dsl/regexp-syntax.asciidoc
index 68ca5912458d7..6a929ba98d502 100644
--- a/docs/reference/query-dsl/regexp-syntax.asciidoc
+++ b/docs/reference/query-dsl/regexp-syntax.asciidoc
@@ -203,7 +203,7 @@ For string `"abcd"`:
 ===== Optional operators
 
 These operators are available by default as the `flags` parameter defaults to `ALL`.
-Different flag combinations (concatened with `"\"`) can be used to enable/disable
+Different flag combinations (concatenated with `"|"`) can be used to enable/disable
 specific operators:
 
     {
diff --git a/docs/reference/query-dsl/script-query.asciidoc b/docs/reference/query-dsl/script-query.asciidoc
index db82375abe09e..a2bb49212034e 100644
--- a/docs/reference/query-dsl/script-query.asciidoc
+++ b/docs/reference/query-dsl/script-query.asciidoc
@@ -14,7 +14,7 @@ GET /_search
             "must" : {
                 "script" : {
                     "script" : {
-                        "inline": "doc['num1'].value > 1",
+                        "source": "doc['num1'].value > 1",
                         "lang": "painless"
                      }
                 }
@@ -41,7 +41,7 @@ GET /_search
             "must" : {
                 "script" : {
                     "script" : {
-                        "inline" : "doc['num1'].value > params.param1",
+                        "source" : "doc['num1'].value > params.param1",
                         "lang"   : "painless",
                         "params" : {
                             "param1" : 5
diff --git a/docs/reference/query-dsl/simple-query-string-query.asciidoc b/docs/reference/query-dsl/simple-query-string-query.asciidoc
index 796f2517fea6c..7ef7ba60cc5cd 100644
--- a/docs/reference/query-dsl/simple-query-string-query.asciidoc
+++ b/docs/reference/query-dsl/simple-query-string-query.asciidoc
@@ -44,18 +44,11 @@ creating composite queries.
 |`flags` |Flags specifying which features of the `simple_query_string` to
 enable. Defaults to `ALL`.
 
-|`lowercase_expanded_terms` | Whether terms of prefix and fuzzy queries should
-be automatically lower-cased or not (since they are not analyzed). Defaults to
-`true`.
-
 |`analyze_wildcard` | Whether terms of prefix queries should be automatically
 analyzed or not. If `true` a best effort will be made to analyze the prefix. However,
 some analyzers will be not able to provide a meaningful results
 based just on the prefix of a term. Defaults to `false`.
 
-|`locale` | Locale that should be used for string conversions.
-Defaults to `ROOT`.
-
 |`lenient` | If set to `true` will cause format based failures
 (like providing text to a numeric field) to be ignored.
 
@@ -63,6 +56,15 @@ Defaults to `ROOT`.
  document to be returned. See the
  <<query-dsl-minimum-should-match,`minimum_should_match`>> documentation for the
  full list of options.
+
+|`quote_field_suffix` | A suffix to append to fields for quoted parts of
+the query string. This allows to use a field that has a different analysis chain
+for exact matching. Look <<mixing-exact-search-with-stemming,here>> for a
+comprehensive example.
+
+|`all_fields` | Perform the query on all fields detected in the mapping that can
+be queried. Will be used by default when the `_all` field is disabled and no
+`default_field` is specified index settings, and no `fields` are specified.
 |=======================================================================
 
 [float]
@@ -81,14 +83,40 @@ The `simple_query_string` supports the following special characters:
 In order to search for any of these special characters, they will need to
 be escaped with `\`.
 
+Be aware that this syntax may have a different behavior depending on the
+`default_operator` value. For example, consider the following query:
+
+[source,js]
+--------------------------------------------------
+GET /_search
+{
+    "query": {
+        "simple_query_string" : {
+            "fields" : ["content"],
+            "query" : "foo bar -baz"
+        }
+    }
+}
+--------------------------------------------------
+// CONSOLE
+
+You may expect that documents containing only "foo" or "bar" will be returned,
+as long as they do not contain "baz", however, due to the `default_operator`
+being OR, this really means "match documents that contain "foo" or documents
+that contain "bar", or documents that don't contain "baz". If this is unintended
+then the query can be switched to `"foo bar +-baz"` which will not return
+documents that contain "baz".
+
 [float]
 ==== Default Field
 When not explicitly specifying the field to search on in the query
 string syntax, the `index.query.default_field` will be used to derive
 which field to search on. It defaults to `_all` field.
 
-So, if `_all` field is disabled, it might make sense to change it to set
-a different default field.
+If the `_all` field is disabled and no `fields` are specified in the request`,
+the `simple_query_string` query will automatically attempt to determine the
+existing fields in the index's mapping that are queryable, and perform the
+search on those fields.
 
 [float]
 ==== Multi Field
diff --git a/docs/reference/query-dsl/span-field-masking-query.asciidoc b/docs/reference/query-dsl/span-field-masking-query.asciidoc
new file mode 100644
index 0000000000000..d9e96635a29c4
--- /dev/null
+++ b/docs/reference/query-dsl/span-field-masking-query.asciidoc
@@ -0,0 +1,43 @@
+[[query-dsl-span-field-masking-query]]
+=== Span Field Masking Query
+
+Wrapper to allow span queries to participate in composite single-field span queries by 'lying' about their search field. The span field masking query maps to Lucene's `SpanFieldMaskingQuery`
+
+This can be used to support queries like `span-near` or `span-or` across different fields, which is not ordinarily permitted.
+
+Span field masking query is invaluable in conjunction with *multi-fields* when same content is indexed with multiple analyzers. For instance we could index a field with the standard analyzer which breaks text up into words, and again with the english analyzer which stems words into their root form.
+
+Example:
+
+[source,js]
+--------------------------------------------------
+GET /_search
+{
+  "query": {
+    "span_near": {
+      "clauses": [
+        {
+          "span_term": {
+            "text": "quick brown"
+          }
+        },
+        {
+          "field_masking_span": {
+            "query": {
+              "span_term": {
+                "text.stems": "fox"
+              }
+            },
+            "field": "text"
+          }
+        }
+      ],
+      "slop": 5,
+      "in_order": false
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+
+Note: as span field masking query returns the masked field, scoring will be done using the norms of the field name supplied. This may lead to unexpected scoring behaviour.
\ No newline at end of file
diff --git a/docs/reference/query-dsl/span-multi-term-query.asciidoc b/docs/reference/query-dsl/span-multi-term-query.asciidoc
index 3d1672753d2b7..b41906b565070 100644
--- a/docs/reference/query-dsl/span-multi-term-query.asciidoc
+++ b/docs/reference/query-dsl/span-multi-term-query.asciidoc
@@ -2,7 +2,7 @@
 === Span Multi Term Query
 
 The `span_multi` query allows you to wrap a `multi term query` (one of wildcard,
-fuzzy, prefix, term, range or regexp query) as a `span query`, so
+fuzzy, prefix, range or regexp query) as a `span query`, so
 it can be nested. Example:
 
 [source,js]
diff --git a/docs/reference/query-dsl/span-queries.asciidoc b/docs/reference/query-dsl/span-queries.asciidoc
index 63aad48d987f4..4a1a019574e92 100644
--- a/docs/reference/query-dsl/span-queries.asciidoc
+++ b/docs/reference/query-dsl/span-queries.asciidoc
@@ -47,6 +47,9 @@ Accepts a list of span queries, but only returns those spans which also match a
 The result from a single span query is returned as long is its span falls
 within the spans returned by a list of other span queries.
 
+<<query-dsl-span-field-masking-query,`field_masking_span` query>>::
+
+Allows queries like `span-near` or `span-or` across different fields.
 
 include::span-term-query.asciidoc[]
 
@@ -63,3 +66,5 @@ include::span-not-query.asciidoc[]
 include::span-containing-query.asciidoc[]
 
 include::span-within-query.asciidoc[]
+
+include::span-field-masking-query.asciidoc[]
\ No newline at end of file
diff --git a/docs/reference/query-dsl/template-query.asciidoc b/docs/reference/query-dsl/template-query.asciidoc
index b4b00e5babdcb..01fcb5a15eb5b 100644
--- a/docs/reference/query-dsl/template-query.asciidoc
+++ b/docs/reference/query-dsl/template-query.asciidoc
@@ -14,7 +14,7 @@ GET /_search
 {
     "query": {
         "template": {
-            "inline": { "match": { "text": "{{query_string}}" }},
+            "source": { "match": { "text": "{{query_string}}" }},
             "params" : {
                 "query_string" : "all about search"
             }
@@ -48,7 +48,7 @@ GET /_search
 {
     "query": {
         "template": {
-            "inline": "{ \"match\": { \"text\": \"{{query_string}}\" }}", <1>
+            "source": "{ \"match\": { \"text\": \"{{query_string}}\" }}", <1>
             "params" : {
                 "query_string" : "all about search"
             }
@@ -98,6 +98,7 @@ PUT /_search/template/my_template
 }
 ------------------------------------------
 // CONSOLE
+// TEST[warning:The stored search template API is deprecated. Use stored scripts instead.]
 
 and refer to it in the `template` query with the `id` parameter:
 
diff --git a/docs/reference/query-dsl/terms-query.asciidoc b/docs/reference/query-dsl/terms-query.asciidoc
index e00c18bb56fca..121617c434b12 100644
--- a/docs/reference/query-dsl/terms-query.asciidoc
+++ b/docs/reference/query-dsl/terms-query.asciidoc
@@ -38,7 +38,8 @@ The terms lookup mechanism supports the following options:
 [horizontal]
 `index`::
     The index to fetch the term values from. Defaults to the
-    current index.
+    current index. deprecated[5.6.0, omitting the index is deprecated
+    future version will require the index to be set explicitly]
 
 `type`::
     The type to fetch the term values from.
@@ -68,7 +69,7 @@ possible, reducing the need for networking.
 [float]
 ===== Terms lookup twitter example
 At first we index the information for user with id 2, specifically, its
-followers, than index a tweet from user with id 1. Finally we search on
+followers, then index a tweet from user with id 1. Finally we search on
 all the tweets that match the followers of user 2.
 
 [source,js]
@@ -99,12 +100,13 @@ GET /tweets/_search
 --------------------------------------------------
 // CONSOLE
 
-The structure of the external terms document can also include array of
+The structure of the external terms document can also include an array of
 inner objects, for example:
 
 [source,js]
 --------------------------------------------------
-curl -XPUT localhost:9200/users/user/2 -d '{
+PUT /users/user/2
+{
  "followers" : [
    {
      "id" : "1"
@@ -113,7 +115,8 @@ curl -XPUT localhost:9200/users/user/2 -d '{
      "id" : "2"
    }
  ]
-}'
+}
 --------------------------------------------------
+// CONSOLE
 
 In which case, the lookup path will be `followers.id`.
diff --git a/docs/reference/redirects.asciidoc b/docs/reference/redirects.asciidoc
index 29a9986bc8f61..536623900ed89 100644
--- a/docs/reference/redirects.asciidoc
+++ b/docs/reference/redirects.asciidoc
@@ -10,6 +10,12 @@ The `_shutdown` API has been removed.  Instead, setup Elasticsearch to run as
 a service (see <<rpm>>, <<deb>>,  or <<windows>>) or use the `-p`
 command line option to <<setup-installation-daemon,write the PID to a file>>.
 
+[role="exclude",id="indices-upgrade"]
+=== Upgrade API
+
+The `_upgrade` API is no longer useful and will be removed.  Instead, see
+<<reindex-upgrade>>.
+
 [role="exclude",id="docs-bulk-udp"]
 === Bulk UDP API
 
@@ -54,14 +60,14 @@ directory.  Instead, mappings should be created using the API with:
 
 The `memcached` transport is no longer supported.  Instead use the REST
 interface over <<modules-http,HTTP>> or the
-https://www.elastic.co/guide/en/elasticsearch/client/java-api/current/index.html[Java API].
+{javaclient}/index.html[Java API].
 
 [role="exclude",id="modules-thrift"]
 === Thrift
 
 The `thrift` transport is no longer supported.  Instead use the REST
 interface over <<modules-http,HTTP>> or the
-https://www.elastic.co/guide/en/elasticsearch/client/java-api/current/index.html[Java API].
+{javaclient}/index.html[Java API].
 
 // QUERY DSL
 
@@ -80,7 +86,7 @@ in ``query context'' and as a filter in ``filter context'' (see <<query-dsl>>).
 [role="exclude",id="query-dsl-not-filter"]
 === Not Filter
 
-The `not` query has been replaced by using a `mustNot` clause in a Boolean query.
+The `not` query has been replaced by using a `must_not` clause in a `bool` query (see <<query-dsl-bool-query>>).
 
 [role="exclude",id="query-dsl-bool-filter"]
 === Bool Filter
@@ -131,13 +137,6 @@ The `geo_shape` filter has been replaced by the <<query-dsl-geo-shape-query>>.
 It behaves as a query in ``query context'' and as a filter in ``filter
 context'' (see <<query-dsl>>).
 
-[role="exclude",id="query-dsl-geohash-cell-filter"]
-=== Geohash Cell Filter
-
-The `geohash_cell` filter has been replaced by the <<query-dsl-geohash-cell-query>>.
-It behaves as a query in ``query context'' and as a filter in ``filter
-context'' (see <<query-dsl>>).
-
 [role="exclude",id="query-dsl-has-child-filter"]
 === Has Child Filter
 
@@ -359,6 +358,7 @@ the following:
 
 [source,js]
 -------------------------
+## INCORRECT - DEPRECATED SYNTAX, DO NOT USE
 GET _search
 {
   "query": {
@@ -377,6 +377,7 @@ GET _search
   }
 }
 -------------------------
+// NOTCONSOLE
 
 move the query and filter to the `must` and `filter` parameters in the `bool`
 query:
@@ -401,6 +402,7 @@ GET _search
   }
 }
 -------------------------
+// CONSOLE
 
 [role="exclude",id="query-dsl-or-query"]
 === Or query
@@ -454,3 +456,32 @@ index that make warmers not necessary anymore.
 === Index time boosting
 
 The index time boost mapping has been replaced with query time boost (see <<mapping-boost>>).
+
+[role="exclude",id="modules-scripting-native"]
+=== Native scripting 
+
+Native scripts have been replaced with writing custom `ScriptEngine` backends (see <<modules-scripting-engine>>).
+
+[role="exclude",id="modules-advanced-scripting"]
+=== Advanced scripting
+
+Using `_index` in scripts has been replaced with writing `ScriptEngine` backends (see <<modules-scripting-engine>>).
+
+[role="exclude",id="modules-scripting-painless-syntax"]
+=== Painless Syntax
+
+See the
+{painless}/painless-specification.html[Painless Language Specification]
+in the guide to the {painless}/index.html[Painless Scripting Language].
+
+[role="exclude",id="modules-scripting-painless-debugging"]
+=== Painless Debugging
+
+See {painless}/painless-debugging.html[Painless Debugging] in the
+guide to the {painless}/index.html[Painless Scripting Language].
+
+[role="exclude",id="painless-api-reference"]
+=== Painless API Reference
+
+See the {painless}/painless-api-reference.html[Painless API Reference] in
+the guide to the {painless}/index.html[Painless Scripting Language].
diff --git a/docs/reference/release-notes.asciidoc b/docs/reference/release-notes.asciidoc
index d8daa87754a85..517c30d042e92 100644
--- a/docs/reference/release-notes.asciidoc
+++ b/docs/reference/release-notes.asciidoc
@@ -5,6 +5,34 @@
 --
 This section summarizes the changes in each release.
 
+* <<release-notes-5.6.3>>
+* <<release-notes-5.6.2>>
+* <<release-notes-5.6.1>>
+* <<release-notes-5.6.0>>
+* <<release-notes-5.5.3>>
+* <<release-notes-5.5.2>>
+* <<release-notes-5.5.1>>
+* <<release-notes-5.5.0>>
+* <<release-notes-5.4.3>>
+* <<release-notes-5.4.2>>
+* <<release-notes-5.4.1>>
+* <<release-notes-5.4.0>>
+* <<release-notes-5.3.3>>
+* <<release-notes-5.3.2>>
+* <<release-notes-5.3.1>>
+* <<release-notes-5.3.0>>
+* <<release-notes-5.2.2>>
+* <<release-notes-5.2.1>>
+* <<release-notes-5.2.0>>
+* <<release-notes-5.1.2>>
+* <<release-notes-5.1.1>>
+* <<release-notes-5.1.0>>
+* <<release-notes-5.0.2>>
+* <<release-notes-5.0.1>>
+* <<release-notes-5.0.0>>
+* <<release-notes-5.0.0-GA>>
+* <<release-notes-5.0.0-rc1>>
+* <<release-notes-5.0.0-beta1>>
 * <<release-notes-5.0.0-alpha5>>
 * <<release-notes-5.0.0-alpha4>>
 * <<release-notes-5.0.0-alpha3>>
@@ -13,6 +41,34 @@ This section summarizes the changes in each release.
 * <<release-notes-5.0.0-alpha1-2x>>
 
 --
+include::release-notes/5.6.3.asciidoc[]
+include::release-notes/5.6.2.asciidoc[]
+include::release-notes/5.6.1.asciidoc[]
+include::release-notes/5.6.0.asciidoc[]
+include::release-notes/5.5.3.asciidoc[]
+include::release-notes/5.5.2.asciidoc[]
+include::release-notes/5.5.1.asciidoc[]
+include::release-notes/5.5.0.asciidoc[]
+include::release-notes/5.4.3.asciidoc[]
+include::release-notes/5.4.2.asciidoc[]
+include::release-notes/5.4.1.asciidoc[]
+include::release-notes/5.4.0.asciidoc[]
+include::release-notes/5.3.3.asciidoc[]
+include::release-notes/5.3.2.asciidoc[]
+include::release-notes/5.3.1.asciidoc[]
+include::release-notes/5.3.0.asciidoc[]
+include::release-notes/5.2.2.asciidoc[]
+include::release-notes/5.2.1.asciidoc[]
+include::release-notes/5.2.0.asciidoc[]
+include::release-notes/5.1.2.asciidoc[]
+include::release-notes/5.1.1.asciidoc[]
+include::release-notes/5.1.0.asciidoc[]
+include::release-notes/5.0.2.asciidoc[]
+include::release-notes/5.0.1.asciidoc[]
+include::release-notes/5.0.0-all.asciidoc[]
+include::release-notes/5.0.0.asciidoc[]
+include::release-notes/5.0.0-rc1.asciidoc[]
+include::release-notes/5.0.0-beta1.asciidoc[]
 include::release-notes/5.0.0-alpha5.asciidoc[]
 include::release-notes/5.0.0-alpha4.asciidoc[]
 include::release-notes/5.0.0-alpha3.asciidoc[]
diff --git a/docs/reference/release-notes/5.0.0-all.asciidoc b/docs/reference/release-notes/5.0.0-all.asciidoc
new file mode 100644
index 0000000000000..13e0b958d3f6b
--- /dev/null
+++ b/docs/reference/release-notes/5.0.0-all.asciidoc
@@ -0,0 +1,1804 @@
+[[release-notes-5.0.0]]
+== 5.0.0 Combined Release Notes
+
+The list below covers all changes from 5.0.0-alpha1 to 5.0.0 GA excluding changes which were already released in the 2.x series, which can be found in <<release-notes-5.0.0-alpha1-2x>>.
+
+Also see <<breaking-changes-5.0>>.
+
+[[breaking-5.0.0]]
+[float]
+=== Breaking changes
+
+Aggregations::
+* Remove size 0 options in aggregations {pull}18854[#18854] (issue: {issue}18838[#18838])
+
+Aliases::
+* make get alias expand to open and closed indices by default {pull}15954[#15954] (issue: {issue}14982[#14982])
+* Remove deprecated indices.get_aliases {pull}13906[#13906]
+
+Allocation::
+* Remove DisableAllocationDecider {pull}13313[#13313]
+
+Analysis::
+* Remove `token_filter` in _analyze API {pull}20285[#20285] (issue: {issue}20283[#20283])
+* Removes support for adding aliases to analyzers {pull}19994[#19994] (issue: {issue}18244[#18244])
+* Analyze API : Rename filters/token_filters/char_filter in Analyze API in master {pull}17843[#17843] (issue: {issue}15189[#15189])
+
+CAT API::
+* Improve cat thread pool API {pull}19721[#19721] (issue: {issue}19590[#19590])
+* Row-centric output for _cat/fielddata {pull}18068[#18068] (issue: {issue}10249[#10249])
+* Add raw recovery progress to cat recovery API {pull}17064[#17064] (issue: {issue}17022[#17022])
+* Remove host from cat nodes API {pull}16656[#16656] (issues: {issue}12959[#12959], {issue}16575[#16575])
+* Using the accept header in the request instead of content-type in _cat API. {pull}14421[#14421] (issue: {issue}14195[#14195])
+
+CRUD::
+* Fixed naming inconsistency for fields/stored_fields in the APIs {pull}20166[#20166] (issues: {issue}18943[#18943], {issue}20155[#20155])
+* Disallow creating indices starting with '-' or '+' {pull}20033[#20033] (issue: {issue}19800[#19800])
+* Wait for changes to be visible by search {pull}17986[#17986] (issue: {issue}1063[#1063])
+* Remove object notation for core types. {pull}15684[#15684] (issue: {issue}15388[#15388])
+
+Cache::
+* Remove deprecated query cache settings {pull}15592[#15592]
+
+Cluster::
+* Persistent Node Ids {pull}19140[#19140] (issue: {issue}17811[#17811])
+* Remove validation errors from cluster health response {pull}17773[#17773] (issue: {issue}16979[#16979])
+* Remove memory section {pull}17278[#17278] (issues: {issue}12049[#12049], {issue}16756[#16756])
+
+Core::
+* Remove ignore system bootstrap checks {pull}20511[#20511]
+* Remove minimum master nodes bootstrap check {pull}20082[#20082]
+* Keep input time unit when parsing TimeValues {pull}19102[#19102]
+* Remove cluster name from data path {pull}18554[#18554] (issue: {issue}17810[#17810])
+* Add max number of processes check {pull}16919[#16919]
+* Add mlockall bootstrap check {pull}16909[#16909]
+* One log {pull}16703[#16703] (issue: {issue}16585[#16585])
+
+Engine::
+* Optimize indexing for the autogenerated ID append-only case {pull}20211[#20211] (issue: {issue}19813[#19813])
+* Remove `index.compound_on_flush` setting and default to `true` {pull}15594[#15594] (issue: {issue}10778[#10778])
+
+Exceptions::
+* Die with dignity {pull}19272[#19272] (issue: {issue}19231[#19231])
+
+Fielddata::
+* Remove "uninverted" and "binary" fielddata support for numeric and boolean fields. {pull}14082[#14082] (issue: {issue}14113[#14113])
+
+Geo::
+* Deprecate GeoDistance enums and remove geo distance script helpers {pull}19783[#19783]
+
+Index APIs::
+* Removes write consistency level across replication action APIs in favor of wait_for_active_shards {pull}19454[#19454] (issue: {issue}18985[#18985])
+* Remove `GET` option for /_forcemerge {pull}15223[#15223] (issue: {issue}15165[#15165])
+* Remove /_optimize REST API endpoint {pull}14226[#14226] (issue: {issue}13778[#13778])
+
+Indexed Scripts/Templates::
+* Store indexed scripts in the cluster state instead of the `.scripts` index {pull}17650[#17650] (issue: {issue}16651[#16651])
+
+Inner Hits::
+* Also do not serialize `_index` key in search response for parent/child inner hits {pull}19011[#19011]
+* Don't include `_id`, `_type` and `_index` keys in search response for inner hits {pull}18995[#18995] (issue: {issue}18091[#18091])
+* Nested inner hits shouldn't use relative paths {pull}18567[#18567] (issue: {issue}16653[#16653])
+* Drop top level inner hits in favour of inner hits defined in the query dsl {pull}17816[#17816] (issue: {issue}11118[#11118])
+
+Internal::
+* `_flush` should block by default {pull}20597[#20597] (issue: {issue}20569[#20569])
+* Actually bound the generic thread pool {pull}17017[#17017]
+* Remove support for pre 2.0 indices {pull}13799[#13799]
+
+Logging::
+* Introduce Log4j 2 {pull}20235[#20235] (issues: {issue}16030[#16030], {issue}17697[#17697])
+
+Mapping::
+* Remove `_timestamp` and `_ttl` on 5.x indices. {pull}18980[#18980] (issue: {issue}18280[#18280])
+* Add a soft limit on the mapping depth. {pull}17400[#17400]
+* Disable fielddata on text fields by defaults. {pull}17386[#17386]
+* Add limit to total number of fields in mapping {pull}17357[#17357]
+* Change the field mapping index time boost into a query time boost. {pull}16900[#16900]
+* Deprecate string in favor of text/keyword. {pull}16877[#16877]
+* Term vector APIs should no longer update mappings {pull}16285[#16285]
+* Remove the `format` option of the `_source` field. {pull}15398[#15398]
+* Remove transform {pull}13657[#13657] (issue: {issue}12674[#12674])
+
+Packaging::
+* Rename service.bat to elasticsearch-service.bat {pull}20496[#20496] (issue: {issue}17528[#17528])
+* Remove -D handling in args for windows plugin script {pull}20378[#20378] (issue: {issue}18207[#18207])
+* Set default min heap equal to default max heap {pull}20080[#20080] (issues: {issue}16334[#16334], {issue}17728[#17728], {issue}18311[#18311])
+* Remove allow running as root {pull}18694[#18694] (issue: {issue}18688[#18688])
+* Require /bin/bash in packaging {pull}18259[#18259] (issue: {issue}18251[#18251])
+* Remove plugin script parsing of system properties {pull}18207[#18207] (issue: {issue}18140[#18140])
+* Add JVM options configuration file {pull}17675[#17675] (issue: {issue}17121[#17121])
+
+Parent/Child::
+* Removed `total` score mode in favour for `sum` score mode. {pull}17174[#17174] (issues: {issue}13470[#13470], {issue}17083[#17083])
+* Removed pre 2.x parent child implementation {pull}13376[#13376]
+
+Percolator::
+* Remove `.percolator` type in favour of `percolator` field type {pull}17560[#17560]
+* Change the percolate api to not dynamically add fields to mapping {pull}16077[#16077] (issue: {issue}15751[#15751])
+
+Plugin Delete By Query::
+* Remove Delete-By-Query plugin {pull}18516[#18516] (issue: {issue}18469[#18469])
+
+Plugin Lang Painless::
+* Remove all date 'now' methods from Painless {pull}20766[#20766] (issue: {issue}20762[#20762])
+* Make Painless the Default Language {pull}20017[#20017] (issue: {issue}19960[#19960])
+
+Plugins::
+* Plugins cleanup {pull}18594[#18594] (issue: {issue}18588[#18588])
+* Rename bin/plugin in bin/elasticsearch-plugin {pull}16454[#16454]
+* Change the inner structure of the plugins zip {pull}16453[#16453]
+* Remove multicast plugin {pull}16326[#16326] (issue: {issue}16310[#16310])
+* Plugins: Remove site plugins {pull}16038[#16038]
+
+Query DSL::
+* Lessen leniency of the query dsl. {pull}18276[#18276]
+* Function score query: remove deprecated support for boost_factor {pull}13510[#13510]
+* Remove support for deprecated queries. {pull}13418[#13418] (issue: {issue}13326[#13326])
+
+REST::
+* Change separator for shards preference {pull}20786[#20786] (issues: {issue}20722[#20722], {issue}20769[#20769])
+* Parameter improvements to Cluster Health API wait for shards {pull}20223[#20223] (issue: {issue}20216[#20216])
+* Switch indices.exists_type from `{index}/{type}` to `{index}/_mapping/{type}`. {pull}20055[#20055] (issue: {issue}15613[#15613])
+* Only use `PUT` for index creation, not POST. {pull}20001[#20001] (issues: {issue}15613[#15613], {issue}18160[#18160])
+* Remove camelCase support {pull}17933[#17933] (issue: {issue}8988[#8988])
+* Remove 'case' parameter from rest apis {pull}17774[#17774] (issue: {issue}8988[#8988])
+* Disallow unquoted field names {pull}15351[#15351] (issue: {issue}9800[#9800])
+* Limit the accepted length of the _id {pull}16036[#16036] (issue: {issue}16034[#16034])
+
+Scripting::
+* Hardcode painless as the default scripting lang and add legacy script default for stored scripts {pull}20310[#20310] (issue: {issue}20122[#20122])
+* Remove deprecated 1.x script and template syntax {pull}19387[#19387] (issue: {issue}13729[#13729])
+* Allow only a single extension for a scripting engine {pull}18332[#18332] (issue: {issue}10598[#10598])
+* Remove 'sandbox' option for script settings, allow only registering a single language. {pull}18226[#18226] (issues: {issue}10598[#10598], {issue}17114[#17114])
+
+Search::
+* Rename `fields` to `stored_fields` and add `docvalue_fields` {pull}18992[#18992] (issue: {issue}18943[#18943])
+* Remove only node preference {pull}18875[#18875] (issue: {issue}18822[#18822])
+* Add search preference to prefer multiple nodes {pull}18872[#18872] (issue: {issue}18822[#18822])
+* Add a soft limit on the number of shards that can be queried in a single search request. {pull}17396[#17396]
+* Remove deprecated reverse option from sorting {pull}17282[#17282] (issue: {issue}17047[#17047])
+* Remove some deprecations {pull}14331[#14331]
+* Remove search exists api {pull}13911[#13911] (issues: {issue}13682[#13682], {issue}13910[#13910])
+* Remove the scan and count search types. {pull}13310[#13310]
+
+Search Refactoring::
+* Remove deprecated parameter from field sort builder. {pull}16573[#16573] (issue: {issue}16127[#16127])
+* Remove "query" query and fix related parsing bugs {pull}14304[#14304] (issue: {issue}13326[#13326])
+
+Settings::
+* Default max local storage nodes to one {pull}19964[#19964] (issues: {issue}19679[#19679], {issue}19748[#19748])
+* Persistent Node Names {pull}19456[#19456] (issue: {issue}19140[#19140])
+* Remove support for properties {pull}19398[#19398] (issues: {issue}19388[#19388], {issue}19391[#19391])
+* Rename boostrap.mlockall to bootstrap.memory_lock {pull}18669[#18669]
+* Register `indices.query.bool.max_clause_count` setting {pull}18341[#18341] (issue: {issue}18336[#18336])
+* Remove settings and system properties entanglement {pull}18198[#18198] (issue: {issue}18197[#18197])
+* Remove `action.get.realtime` setting {pull}17857[#17857] (issue: {issue}12543[#12543])
+* Remove ability to specify arbitrary node attributes with `node.` prefix {pull}17402[#17402] (issue: {issue}17280[#17280])
+* Enforce `discovery.zen.minimum_master_nodes` is set when bound to a public ip {pull}17288[#17288]
+* Prevent index level setting from being configured on a node level {pull}17144[#17144] (issue: {issue}16799[#16799])
+* Remove support for node.client setting {pull}16963[#16963] (issue: {issue}16565[#16565])
+* Remove es.max-open-files flag {pull}16757[#16757] (issues: {issue}16506[#16506], {issue}483[#483])
+* Enforce node level limits if node is started in production env {pull}16733[#16733] (issue: {issue}16727[#16727])
+* Make settings validation strict {pull}16365[#16365]
+* Remove the ability to fsync on every operation and only schedule fsync task if really needed {pull}16257[#16257] (issue: {issue}16152[#16152])
+* Script settings {pull}16197[#16197]
+* Remove index.flush_on_close entirely {pull}15977[#15977]
+* Restore chunksize of 512kb on recovery and remove configurability {pull}15235[#15235] (issue: {issue}15161[#15161])
+* Remove ancient deprecated and alternative recovery settings {pull}15234[#15234]
+
+Similarities::
+* Renames `default` similarity into `classic` {pull}15446[#15446] (issue: {issue}15102[#15102])
+
+Snapshot/Restore::
+* Change the default of `include_global_state` from true to false for snapshot restores {pull}18773[#18773] (issue: {issue}18569[#18569])
+* Fail closing or deleting indices during a full snapshot {pull}17021[#17021] (issue: {issue}16321[#16321])
+
+Stats::
+* Modify load average format {pull}15932[#15932] (issue: {issue}15907[#15907])
+* Reintroduce five-minute and fifteen-minute load averages on Linux {pull}15907[#15907] (issues: {issue}12049[#12049], {issue}14741[#14741])
+* Add system CPU percent to OS stats {pull}14741[#14741]
+
+Term Vectors::
+* Remove DFS support from TermVector API {pull}16452[#16452]
+
+Translog::
+* Drop support for simple translog and hard-wire buffer to 8kb {pull}15574[#15574]
+* Simplify translog-based flush settings {pull}15573[#15573]
+
+Warmers::
+* Remove query warmers and the warmer API. {pull}15614[#15614] (issue: {issue}15607[#15607])
+
+
+
+[[breaking-java-5.0.0]]
+[float]
+=== Breaking Java changes
+
+Aggregations::
+* getKeyAsString and key_as_string should be the same for terms aggregation on boolean field {pull}15393[#15393]
+
+Allocation::
+* Move parsing of allocation commands into REST and remove support for plugins to register allocation commands {pull}17802[#17802] (issue: {issue}17894[#17894])
+* Simplify shard balancer interface {pull}17028[#17028] (issue: {issue}8954[#8954])
+
+Analysis::
+* Simplify Analysis registration and configuration {pull}14355[#14355]
+
+CRUD::
+* Removing isCreated and isFound from the Java API {pull}19645[#19645] (issues: {issue}19566[#19566], {issue}19631[#19631])
+
+Cache::
+* Refactor IndicesRequestCache to make it testable. {pull}16610[#16610]
+* Fold IndexCacheModule into IndexModule {pull}14293[#14293]
+
+Core::
+* Remove ability to plug-in TransportService {pull}20505[#20505]
+* Register thread pool settings {pull}18674[#18674] (issues: {issue}18613[#18613], {issue}9216[#9216])
+* Bootstrap does not set system properties {pull}17088[#17088] (issues: {issue}16579[#16579], {issue}16791[#16791])
+* Remove es.useLinkedTransferQueue {pull}16786[#16786]
+
+Discovery::
+* Introduce node handshake {pull}15971[#15971] (issue: {issue}9061[#9061])
+* Include pings from client nodes in master election {pull}17329[#17329] (issue: {issue}17325[#17325])
+
+Highlighting::
+* Register Highlighter instances instead of classes {pull}18859[#18859]
+
+Internal::
+* Remove TransportService#registerRequestHandler leniency {pull}20469[#20469] (issue: {issue}20468[#20468])
+* Consolidate search parser registries {pull}20000[#20000]
+* Move all FetchSubPhases to o.e.search.fetch.subphase {pull}19702[#19702]
+* Squash the rest of o.e.rest.action {pull}19698[#19698]
+* Clean up BytesReference {pull}19196[#19196]
+* Cleanup ClusterService dependencies and detached from Guice {pull}18941[#18941]
+* Simplify SubFetchPhase interface {pull}18881[#18881]
+* Simplify FetchSubPhase registration and detach it from Guice {pull}18862[#18862]
+* Remove duplicate getters from DiscoveryNode and DiscoveryNodes {pull}17410[#17410] (issue: {issue}16963[#16963])
+* Cli: Switch to jopt-simple {pull}17024[#17024] (issue: {issue}11564[#11564])
+* Replace ContextAndHeaders with a ThreadPool based ThreadLocal implementation {pull}15776[#15776]
+* Remove NodeBuilder {pull}15354[#15354]
+* Fix IndexSearcherWrapper interface to not depend on the EngineConfig {pull}14654[#14654]
+* Cleanup query parsing and remove IndexQueryParserService {pull}14452[#14452]
+* Remove circular dependency between IndicesService and IndicesStore {pull}14285[#14285]
+* Remove guice injection from IndexStore and friends {pull}14279[#14279]
+* Replace IndicesLifecycle with a per-index IndexEventListener {pull}14217[#14217] (issue: {issue}13259[#13259])
+* Simplify similarity module and friends {pull}13942[#13942]
+* Refactor SearchRequest to be parsed on the coordinating node {pull}13859[#13859]
+
+Java API::
+* Add a dedicated client/transport project for transport-client {pull}19435[#19435] (issue: {issue}19412[#19412])
+* Remove setRefresh {pull}18752[#18752] (issue: {issue}1063[#1063])
+* Remove the count api {pull}14166[#14166] (issue: {issue}13928[#13928])
+* IdsQueryBuilder to accept only non null ids and types {pull}13937[#13937]
+
+Mapping::
+* [Mapping] Several MappingService cleanups {pull}16133[#16133] (issue: {issue}15924[#15924])
+
+Network::
+* Factor out abstract TCPTransport* classes to reduce the netty footprint {pull}19096[#19096]
+* Remove ability to disable Netty gathering writes {pull}16774[#16774] (issue: {issue}7811[#7811])
+
+Parent/Child::
+* Cleanup ParentFieldMapper {pull}16045[#16045]
+* Several other parent/child cleanups {pull}13470[#13470]
+
+Percolator::
+* Move the percolator from core to its own module {pull}18511[#18511]
+* Remove percolator cache {pull}18434[#18434]
+
+Plugins::
+* Cleanup sub fetch phase extension point {pull}20382[#20382]
+* Remove IndexTemplateFilter {pull}20072[#20072]
+* Switch custom ShardsAllocators to pull based model {pull}20071[#20071]
+* Make custom allocation deciders use pull based extensions {pull}20040[#20040]
+* Migrate query registration from push to pull {pull}19376[#19376]
+* Add components getter as bridge between guice and new plugin init world {pull}19371[#19371]
+* Remove CustomNodeAttributes extension point {pull}19348[#19348]
+* Add RepositoryPlugin interface for registering snapshot repositories {pull}19324[#19324]
+* Simplified repository api for snapshot/restore {pull}19292[#19292]
+* Switch most search extensions from push to pull {pull}19238[#19238]
+* Move RestHandler registration to ActionModule and ActionPlugin {pull}19165[#19165]
+* Pull actions from plugins {pull}19108[#19108]
+* Switch analysis from push to pull {pull}19073[#19073]
+* Remove guice from Mapper plugins {pull}19018[#19018]
+* Fail to start if plugin tries broken onModule {pull}19025[#19025]
+* Simplify ScriptModule and script registration {pull}18903[#18903]
+* Cut over settings registration to a pull model {pull}18890[#18890]
+* Enforce isolated mode for all plugins {pull}17276[#17276]
+* Don't use guice for QueryParsers {pull}15761[#15761]
+* Remove guice from the index level {pull}14518[#14518]
+* Remove shard-level injector {pull}13881[#13881]
+
+Query DSL::
+* Remove the MissingQueryBuilder which was deprecated in 2.2.0. {pull}15364[#15364] (issue: {issue}14112[#14112])
+* Remove NotQueryBuilder {pull}14204[#14204] (issue: {issue}13761[#13761])
+
+Scripting::
+* Remove o.e.script.Template class and move template query to lang-mustache module {pull}19425[#19425] (issue: {issue}16314[#16314])
+* Move search template to lang-mustache module {pull}18765[#18765] (issue: {issue}17906[#17906])
+* Remove LeafSearchScript.runAsFloat(): Nothing calls it. {pull}18364[#18364]
+
+Search::
+* Remove FetchSubPhaseParseElement {pull}20350[#20350]
+* Refactor of query profile classes to make way for other profile implementations {pull}18370[#18370] (issue: {issue}10538[#10538])
+* Query refactoring: split parse phase into fromXContent and toQuery for all queries {pull}13788[#13788] (issue: {issue}10217[#10217])
+
+Search Refactoring::
+* Refactored inner hits parsing and intoduced InnerHitBuilder {pull}17291[#17291]
+* Remove support for query_binary and filter_binary {pull}14433[#14433] (issue: {issue}14308[#14308])
+* Validate query api: move query parsing to the coordinating node {pull}14384[#14384]
+
+Settings::
+* Remove `node.mode` and `node.local` settings {pull}19428[#19428]
+* Remove Settings.settingsBuilder. {pull}17619[#17619]
+* Move remaining settings in NettyHttpServerTransport to the new infra {pull}16531[#16531]
+* Replace IndexSettings annotation with a full-fledged class {pull}14251[#14251]
+* Fix ping timeout settings inconsistencies {pull}13701[#13701] (issue: {issue}6579[#6579])
+
+Snapshot/Restore::
+* Removes extra writeBlob method in BlobContainer {pull}19727[#19727] (issue: {issue}18528[#18528])
+
+Store::
+* Standardize state format type for global and index level metadata {pull}17123[#17123]
+
+Suggesters::
+* Remove suggest threadpool {pull}17304[#17304] (issue: {issue}17198[#17198])
+* Remove suggest transport action {pull}17198[#17198] (issue: {issue}10217[#10217])
+
+
+
+[[deprecation-5.0.0]]
+[float]
+=== Deprecations
+
+CRUD::
+* Deprecate found and created in delete and index rest responses {pull}19633[#19633]
+
+Plugin Discovery Azure Classic::
+* Deprecate discovery-azure and rename it to discovery-azure-classic {pull}19186[#19186] (issue: {issue}19144[#19144])
+
+Plugin Mapper Attachment::
+* Deprecate mapper-attachments plugin {pull}16948[#16948] (issue: {issue}16910[#16910])
+
+Query DSL::
+* Deprecate Indices query {pull}17710[#17710] (issue: {issue}12017[#12017])
+* Deprecate mlt, in and geo_bbox query name shortcuts {pull}17507[#17507]
+
+Query Refactoring::
+* Splits `phrase` and `phrase_prefix` in match query into `MatchPhraseQueryBuilder` and `MatchPhrasePrefixQueryBuilder` {pull}17508[#17508]
+
+Scripting::
+* Deprecate Groovy, Python, and Javascript {pull}20244[#20244] (issue: {issue}20129[#20129])
+
+Search::
+* Deprecate fuzzy query {pull}16211[#16211] (issues: {issue}15760[#15760], {issue}16121[#16121])
+
+Templates::
+* Deprecate template query {pull}19607[#19607] (issue: {issue}19390[#19390])
+
+
+
+[[feature-5.0.0]]
+[float]
+=== New features
+
+Aggregations::
+* Split regular histograms from date histograms. {pull}19551[#19551] (issues: {issue}4847[#4847], {issue}8082[#8082])
+* Adds aggregation profiling to the profile API {pull}18414[#18414] (issue: {issue}10538[#10538])
+* New Matrix Stats Aggregation module {pull}18300[#18300] (issue: {issue}16826[#16826])
+
+Aliases::
+* Add an alias action to delete an index {pull}20184[#20184] (issue: {issue}20064[#20064])
+
+Allocation::
+* Add API to explain why a shard is or isn't assigned {pull}17305[#17305] (issue: {issue}14593[#14593])
+
+Analysis::
+* Exposing lucene 6.x minhash filter. {pull}20206[#20206] (issue: {issue}20149[#20149])
+* Add `fingerprint` token filter and `fingerprint` analyzer {pull}17873[#17873] (issue: {issue}13325[#13325])
+
+Circuit Breakers::
+* Circuit break on aggregation bucket numbers with request breaker {pull}19394[#19394] (issue: {issue}14046[#14046])
+
+Discovery::
+* Add two phased commit to Cluster State publishing {pull}13062[#13062]
+
+Geo::
+* Cut over geo_point field and queries to new LatLonPoint type {pull}20315[#20315] (issue: {issue}20314[#20314])
+
+Index APIs::
+* Add rollover API to switch index aliases given some predicates {pull}18732[#18732] (issue: {issue}18647[#18647])
+
+Ingest::
+* ingest-useragent plugin {pull}19074[#19074]
+* Add a Sort ingest processor {pull}17999[#17999]
+* Add date_index_name processor {pull}17973[#17973] (issue: {issue}17814[#17814])
+* Merge feature/ingest branch into master branch {pull}16049[#16049] (issue: {issue}14049[#14049])
+
+Java REST Client::
+* Introduce async performRequest method {pull}19400[#19400]
+* Low level Rest Client {pull}18735[#18735] (issue: {issue}7743[#7743])
+
+Mapping::
+* Add `scaled_float`. {pull}19264[#19264] (issues: {issue}15939[#15939], {issue}1941[#1941])
+* Expose half-floats. {pull}18887[#18887]
+* Add a text field. {pull}16637[#16637]
+* Add a new `keyword` field. {pull}16589[#16589] (issue: {issue}14113[#14113])
+
+Percolator::
+* index the query terms from the percolator query {pull}13646[#13646] (issue: {issue}12664[#12664])
+
+Plugin Analysis ICU::
+* Adding support for customizing the rule file in ICU tokenizer {pull}13651[#13651] (issue: {issue}13146[#13146])
+
+Plugin Discovery File::
+* File-based discovery plugin {pull}20394[#20394] (issue: {issue}20323[#20323])
+
+Plugin Ingest Attachment::
+* Ingest: Add attachment processor {pull}16490[#16490] (issue: {issue}16303[#16303])
+
+Plugin Mapper Attachment::
+* Migrate mapper attachments plugin to main repository {pull}14605[#14605]
+
+Plugin Repository HDFS::
+* HDFS Snapshot/Restore plugin {pull}15192[#15192] (issue: {issue}15191[#15191])
+
+Plugin Repository S3::
+* Add support for path_style_access {pull}15114[#15114]
+
+Query DSL::
+* Adds a rewrite phase to queries on the shard level {pull}16870[#16870] (issue: {issue}9526[#9526])
+
+Reindex API::
+* Reindex from remote {pull}18585[#18585] (issue: {issue}17447[#17447])
+* Port Delete By Query to Reindex infrastructure {pull}18329[#18329] (issue: {issue}16883[#16883])
+* Merge reindex to master {pull}16861[#16861]
+
+Scripting::
+* Exceptions and Infinite Loop Checking {pull}15936[#15936]
+* Added a new scripting language (PlanA) {pull}15136[#15136] (issue: {issue}13084[#13084])
+
+Scroll::
+* Add the ability to partition a scroll in multiple slices. {pull}18237[#18237] (issue: {issue}13494[#13494])
+
+Search::
+* Add the ability to disable the retrieval of the stored fields entirely {pull}20026[#20026]
+* Add `search_after` parameter in the SearchAPI {pull}16125[#16125] (issue: {issue}8192[#8192])
+
+Settings::
+* Add infrastructure to transactionally apply and reset dynamic settings {pull}15278[#15278]
+
+Snapshot/Restore::
+* Add Google Cloud Storage repository plugin {pull}13578[#13578] (issue: {issue}12880[#12880])
+
+Stats::
+* Extend field stats to report searchable/aggregatable fields {pull}17980[#17980] (issue: {issue}17750[#17750])
+* API for listing index file sizes {pull}16661[#16661] (issue: {issue}16131[#16131])
+
+Store::
+* Expose MMapDirectory.preLoad(). {pull}18880[#18880]
+* Add primitive to shrink an index into a single shard {pull}18270[#18270]
+
+Suggesters::
+* Add support for returning documents with completion suggester {pull}19536[#19536] (issue: {issue}10746[#10746])
+* Add document-oriented completion suggester {pull}14410[#14410] (issue: {issue}10746[#10746])
+
+Task Manager::
+* Add task cancellation mechanism {pull}16320[#16320]
+* Make the Task object available to the action caller {pull}16033[#16033]
+* Task Management: Add framework for registering and communicating with tasks {pull}15347[#15347] (issue: {issue}15117[#15117])
+
+Translog::
+* Add `elasticsearch-translog` CLI tool with `truncate` command {pull}19342[#19342] (issue: {issue}19123[#19123])
+
+
+
+[[enhancement-5.0.0]]
+[float]
+=== Enhancements
+
+Aggregations::
+* Make the heuristic to compute the default shard size less aggressive. {pull}19659[#19659]
+* Add _bucket_count option to buckets_path {pull}19571[#19571] (issue: {issue}19553[#19553])
+* Remove AggregationStreams {pull}19507[#19507]
+* Migrate serial_diff aggregation to NamedWriteable {pull}19483[#19483]
+* Migrate most remaining pipeline aggregations to NamedWriteable {pull}19480[#19480]
+* Migrate moving_avg pipeline aggregation to NamedWriteable {pull}19420[#19420]
+* Migrate matrix_stats to NamedWriteable {pull}19418[#19418]
+* Migrate derivative pipeline aggregation to NamedWriteable {pull}19407[#19407]
+* Migrate top_hits, histogram, and ip_range aggregations to NamedWriteable {pull}19375[#19375]
+* Migrate nested, reverse_nested, and children aggregations to NamedWriteable {pull}19374[#19374]
+* Migrate geohash_grid and geo_bounds aggregations to NamedWriteable {pull}19372[#19372]
+* Clean up significant terms aggregation results {pull}19365[#19365]
+* Migrate range, date_range, and geo_distance aggregations to NamedWriteable {pull}19321[#19321]
+* Migrate terms aggregation to NamedWriteable {pull}19277[#19277]
+* Migrate sampler and missing aggregations to NamedWriteable {pull}19259[#19259]
+* Migrate global, filter, and filters aggregation to NamedWriteable {pull}19220[#19220]
+* Migrate the cardinality, scripted_metric, and geo_centroid aggregations to NamedWriteable {pull}19219[#19219]
+* Use a static default precision for the cardinality aggregation. {pull}19215[#19215]
+* Migrate more aggregations to NamedWriteable {pull}19199[#19199]
+* Migrate stats and extended stats to NamedWriteable {pull}19198[#19198]
+* Migrate sum, min, and max aggregations over to NamedWriteable {pull}19194[#19194]
+* Start migration away from aggregation streams {pull}19097[#19097]
+* Automatically set the collection mode to breadth_first in the terms aggregation when the cardinality of the field is unknown or smaller than the requested size. {pull}18779[#18779] (issue: {issue}9825[#9825])
+* Rename PipelineAggregatorBuilder to PipelineAggregationBuilder. {pull}18677[#18677] (issue: {issue}18377[#18377])
+* AggregatorBuilder and PipelineAggregatorBuilder do not need generics. {pull}18368[#18368] (issue: {issue}18133[#18133])
+* Rename AggregatorBuilder to AggregationBuilder {pull}18377[#18377] (issue: {issue}18367[#18367])
+* Add the ability to use the breadth_first mode with nested aggregations (such as `top_hits`) which require access to score information. {pull}18127[#18127] (issue: {issue}9825[#9825])
+* Make significant terms work on fields that are indexed with points. {pull}18031[#18031]
+* Add tests and documentation for using `time_zone` in date range aggregation {pull}16955[#16955] (issue: {issue}10130[#10130])
+* Fixes serialisation of Ranges {pull}16674[#16674]
+
+Allocation::
+* Verify AllocationIDs in replication actions {pull}20320[#20320]
+* Mark shard as stale on non-replicated write, not on node shutdown {pull}20023[#20023] (issue: {issue}18919[#18919])
+* Add routing changes API to RoutingAllocation {pull}19992[#19992]
+* Primary shard allocator observes limits in forcing allocation {pull}19811[#19811] (issue: {issue}19446[#19446])
+* Use primary terms as authority to fail shards {pull}19715[#19715]
+* Add recovery source to ShardRouting {pull}19516[#19516]
+* Allow `_shrink` to N shards if source shards is a multiple of N {pull}18699[#18699]
+* Only filter intial recovery (post API) when shrinking an index {pull}18661[#18661]
+* Estimate shard size for shrinked indices {pull}18659[#18659]
+* Only fail relocation target shard if failing source shard is a primary {pull}18574[#18574] (issue: {issue}16144[#16144])
+* Simplify delayed shard allocation {pull}18351[#18351] (issue: {issue}18293[#18293])
+* Limit retries of failed allocations per index {pull}18467[#18467] (issue: {issue}18417[#18417])
+* Immutable ShardRouting {pull}17821[#17821]
+* Add the shard's store status to the explain API {pull}17689[#17689] (issue: {issue}17372[#17372])
+* Write shard state metadata as soon as shard is created / initializing {pull}16625[#16625] (issue: {issue}14739[#14739])
+* Reuse existing allocation id for primary shard allocation {pull}16530[#16530] (issue: {issue}14739[#14739])
+* Remove version in ShardRouting (now obsolete) {pull}16243[#16243] (issue: {issue}14739[#14739])
+* Prefer nodes that previously held primary shard for primary shard allocation {pull}16096[#16096] (issue: {issue}14739[#14739])
+* Extend reroute with an option to force assign stale primary shard copies {pull}15708[#15708] (issue: {issue}14739[#14739])
+* Allocate primary shards based on allocation IDs {pull}15281[#15281] (issue: {issue}14739[#14739])
+* Persist currently started allocation IDs to index metadata {pull}14964[#14964] (issue: {issue}14739[#14739])
+* Use ObjectParser to parse AllocationID {pull}14962[#14962] (issue: {issue}14831[#14831])
+* Persist allocation ID with shard state metadata on nodes {pull}14831[#14831] (issue: {issue}14739[#14739])
+
+Analysis::
+* Stop using cached component in _analyze API {pull}19929[#19929] (issue: {issue}19827[#19827])
+* Specify custom char_filters/tokenizer/token_filters in the analyze API {pull}15671[#15671] (issue: {issue}8878[#8878])
+* Add a MultiTermAwareComponent marker interface to analysis factories. {pull}19028[#19028] (issues: {issue}18064[#18064], {issue}9978[#9978])
+* Add Flags Parameter for Char Filter {pull}18363[#18363] (issue: {issue}18362[#18362])
+* Core: better error message when analyzer created without tokenizer or… {pull}18455[#18455] (issue: {issue}15492[#15492])
+* Move AsciiFolding earlier in FingerprintAnalyzer filter chain {pull}18281[#18281] (issue: {issue}18266[#18266])
+* Improve error message if resource files have illegal encoding {pull}17237[#17237] (issue: {issue}17212[#17212])
+
+Benchmark::
+* Add client-benchmark-noop-api-plugin to stress clients even more in benchmarks {pull}20103[#20103]
+
+CAT API::
+* Add health status parameter to cat indices API {pull}20393[#20393]
+* Includes the index UUID in the _cat/indices API {pull}19204[#19204] (issue: {issue}19132[#19132])
+* Add node name to Cat Recovery {pull}18187[#18187] (issue: {issue}8041[#8041])
+* Add support for documented byte/size units and for micros as a time unit in _cat API {pull}17779[#17779]
+* Add _cat/tasks {pull}17551[#17551]
+* Cat health supports ts=0 option {pull}13508[#13508] (issue: {issue}10109[#10109])
+* Expose http address in cat/nodes {pull}16770[#16770]
+* [cat/recovery] Make recovery time a TimeValue() {pull}16743[#16743] (issue: {issue}9209[#9209])
+* :CAT API: remove space at the end of a line {pull}15250[#15250] (issue: {issue}9464[#9464])
+
+CRUD::
+* Renaming operation to result and reworking responses  {pull}19704[#19704] (issue: {issue}19664[#19664])
+* Adding _operation field to index, update, delete response. {pull}19566[#19566] (issues: {issue}19267[#19267], {issue}9642[#9642], {issue}9736[#9736])
+* CRUD: Allow to get and set ttl as a time value/string {pull}15047[#15047]
+
+Cache::
+* Enable option to use request cache for size > 0 {pull}19472[#19472]
+* Cache FieldStats in the request cache {pull}18768[#18768] (issue: {issue}18717[#18717])
+* Allow the query cache to be disabled. {pull}16268[#16268] (issue: {issue}15802[#15802])
+* Enable the indices request cache by default {pull}17162[#17162] (issues: {issue}16870[#16870], {issue}17134[#17134])
+
+Circuit Breakers::
+* Cluster Settings Updates should not trigger circuit breakers. {pull}20827[#20827]
+* Circuit break the number of inline scripts compiled per minute {pull}19694[#19694] (issue: {issue}19396[#19396])
+
+Cluster::
+* Add clusterUUID to RestMainAction output {pull}20503[#20503]
+* Batch process node left and node failure {pull}19289[#19289] (issue: {issue}19282[#19282])
+* Index creation waits for write consistency shards {pull}18985[#18985]
+* Inline reroute with process of node join/master election {pull}18938[#18938] (issue: {issue}17270[#17270])
+* Index creation does not cause the cluster health to go RED {pull}18737[#18737] (issues: {issue}9106[#9106], {issue}9126[#9126])
+* Cluster Health class improvements {pull}18673[#18673]
+* Adds tombstones to cluster state for index deletions {pull}17265[#17265] (issues: {issue}16358[#16358], {issue}17435[#17435])
+* Enable acked indexing {pull}17038[#17038] (issue: {issue}7572[#7572])
+* Cluster Health should run on applied states, even if waitFor=0 {pull}17440[#17440]
+* Resolve index names to Index instances early {pull}17048[#17048]
+* Remove DiscoveryNode#shouldConnectTo method {pull}16898[#16898] (issue: {issue}16815[#16815])
+* Fail demoted primary shards and retry request {pull}16415[#16415] (issue: {issue}14252[#14252])
+* Illegal shard failure requests {pull}16275[#16275]
+* Shard failure requests for non-existent shards {pull}16089[#16089] (issue: {issue}14252[#14252])
+* Add handling of channel failures when starting a shard {pull}16041[#16041] (issue: {issue}15895[#15895])
+* Wait for new master when failing shard {pull}15748[#15748] (issue: {issue}14252[#14252])
+* Master should wait on cluster state publication when failing a shard {pull}15468[#15468] (issue: {issue}14252[#14252])
+* Split cluster state update tasks into roles {pull}14899[#14899] (issue: {issue}13627[#13627])
+* Add timeout mechanism for sending shard failures {pull}14707[#14707] (issue: {issue}14252[#14252])
+* Add listener mechanism for failures to send shard failed {pull}14295[#14295] (issue: {issue}14252[#14252])
+
+Core::
+* Add production warning for pre-release builds {pull}20674[#20674]
+* Add serial collector bootstrap check {pull}20558[#20558]
+* Rename Netty TCP transports thread factories from http_* to transport_* {pull}20207[#20207]
+* Do not log full bootstrap checks exception {pull}19989[#19989]
+* Mark halting the virtual machine as privileged {pull}19923[#19923] (issues: {issue}19272[#19272], {issue}19806[#19806])
+* Makes index creation more friendly {pull}19450[#19450] (issue: {issue}9126[#9126])
+* Clearer error when handling fractional time values {pull}19158[#19158] (issue: {issue}19102[#19102])
+* Read Elasticsearch manifest via URL {pull}18999[#18999] (issue: {issue}18996[#18996])
+* Throw if the local node is not set {pull}18963[#18963] (issue: {issue}18962[#18962])
+* Bootstrap check for OnOutOfMemoryError and seccomp {pull}18756[#18756] (issue: {issue}18736[#18736])
+* Log OS and JVM on startup {pull}18557[#18557]
+* Add GC overhead logging {pull}18419[#18419]
+* Refactor JvmGcMonitorService for testing {pull}18378[#18378]
+* Default to server VM and add client VM check {pull}18155[#18155]
+* Add system bootstrap checks escape hatch {pull}18088[#18088]
+* Avoid sliced locked contention in internal engine {pull}18060[#18060] (issue: {issue}18053[#18053])
+* Add heap size bootstrap check {pull}17728[#17728] (issue: {issue}17490[#17490])
+* Remove hostname from NetworkAddress.format {pull}17601[#17601] (issue: {issue}17604[#17604])
+* Bootstrapping bootstrap checks {pull}17595[#17595] (issues: {issue}17474[#17474], {issue}17570[#17570])
+* Add max map count check {pull}16944[#16944]
+* Remove PROTOTYPE from BulkItemResponse.Failure {pull}17433[#17433] (issue: {issue}17086[#17086])
+* Throw an exception if Writeable.Reader reads null {pull}17332[#17332]
+* Remove PROTOTYPE from RescorerBuilders {pull}17330[#17330]
+* Port Primary Terms to master {pull}17044[#17044] (issues: {issue}14062[#14062], {issue}14651[#14651], {issue}17038[#17038])
+* Use index UUID to lookup indices on IndicesService {pull}17001[#17001]
+* Add -XX+AlwaysPreTouch JVM flag {pull}16937[#16937]
+* Add max size virtual memory check {pull}16935[#16935]
+* Use and test relative time in TransportBulkAction {pull}16916[#16916]
+* Bump Elasticsearch version to 5.0.0-SNAPSHOT {pull}16862[#16862]
+* Assert that we can write in all data-path on startup {pull}16745[#16745]
+* Add G1GC check on startup {pull}16737[#16737] (issue: {issue}10740[#10740])
+* Shards with heavy indexing should get more of the indexing buffer {pull}14121[#14121]
+* Remove and ban ImmutableMap {pull}13939[#13939] (issue: {issue}13224[#13224])
+* Finish banning ImmutableSet {pull}13820[#13820] (issue: {issue}13224[#13224])
+* Removes and bans ImmutableSet {pull}13754[#13754] (issue: {issue}13224[#13224])
+* Remove and ban ImmutableMap#entrySet {pull}13724[#13724]
+* Forbid ForwardingSet {pull}13720[#13720] (issue: {issue}13224[#13224])
+
+Dates::
+* Improve TimeZoneRoundingTests error messages {pull}18895[#18895]
+* Support full range of Java Long for epoch DateTime {pull}18509[#18509] (issue: {issue}17936[#17936])
+
+Discovery::
+* Do not log cluster service errors at after joining a master {pull}19705[#19705]
+* Log warning if minimum_master_nodes set to less than quorum {pull}15625[#15625]
+* Add a dedicate queue for incoming ClusterStates {pull}13303[#13303] (issue: {issue}13062[#13062])
+
+Engine::
+* Only try to read new segments info if we really flushed the index {pull}20474[#20474]
+* Use _refresh instead of reading from Translog in the RT GET case {pull}20102[#20102]
+* Remove writeLockTimeout from InternalEngine {pull}16930[#16930]
+* Don't guard IndexShard#refresh calls by a check to isRefreshNeeded {pull}16118[#16118]
+* Never call a listerner under lock in InternalEngine {pull}15786[#15786]
+* Use System.nanoTime() to initialize Engine.lastWriteNanos {pull}14321[#14321]
+* Flush big merges automatically if shard is inactive {pull}14275[#14275]
+* Remove Engine.Create {pull}13955[#13955]
+* Remove the disabled autogenerated id optimization from InternalEngine {pull}13857[#13857]
+
+Exceptions::
+* Improve startup exception {pull}20083[#20083]
+* Make NotMasterException a first class citizen {pull}19385[#19385]
+* Do not catch throwable {pull}19231[#19231]
+* Make the index-too-old exception more explicit {pull}18438[#18438]
+* Add index name in IndexAlreadyExistsException default message {pull}18274[#18274]
+* Fix typos in exception/assert/log messages in core module. {pull}16649[#16649]
+* Add field names to several mapping errors {pull}16508[#16508] (issue: {issue}16378[#16378])
+* Add serialization support for more important IOExceptions {pull}15766[#15766]
+* Adds exception objects to log messages. {pull}14827[#14827] (issue: {issue}10021[#10021])
+* Add stack traces to logged exceptions where missing {pull}13825[#13825] (issue: {issue}10021[#10021])
+* Remove reflection hacks from ElasticsearchException {pull}13796[#13796]
+* Rename QueryParsingException to a more generic ParsingException {pull}13631[#13631]
+* Add *Exception(Throwable cause) constructors/ call where appropriate {pull}13544[#13544] (issue: {issue}10021[#10021])
+
+Expressions::
+* improve date api for expressions/painless fields {pull}18658[#18658]
+* Support geo_point fields in lucene expressions {pull}18096[#18096]
+* Add support for .empty to expressions, and some docs improvements {pull}18077[#18077]
+
+Geo::
+* GeoBoundingBoxQueryBuilder should throw IAE when topLeft and bottomRight are the same coordinate {pull}18668[#18668] (issues: {issue}18458[#18458], {issue}18631[#18631])
+* Enhanced lat/long error handling {pull}16833[#16833] (issue: {issue}16137[#16137])
+* Fix a potential parsing problem in GeoDistanceSortParser {pull}17111[#17111]
+* Geo: Add validation of shapes to ShapeBuilders {pull}15551[#15551] (issue: {issue}14416[#14416])
+* Make remaining ShapeBuilders implement Writeable {pull}15010[#15010] (issue: {issue}14416[#14416])
+* Geo: Remove internal `translated` flag from LineStringBuilder {pull}14969[#14969]
+* Make PointBuilder, CircleBuilder & EnvelopeBuilder implement Writable  {pull}14933[#14933] (issue: {issue}14416[#14416])
+* Merging BaseLineString and BasePolygonBuilder with subclass {pull}14887[#14887] (issue: {issue}14482[#14482])
+* Moving static factory methods to ShapeBuilders {pull}14529[#14529]
+* Remove InternalLineStringBuilder and InternalPolygonBuilder {pull}14482[#14482] (issue: {issue}14416[#14416])
+
+Highlighting::
+* Switch Highlighting to ObjectParser  {pull}17363[#17363]
+* Use HighlightBuilder in SearchSourceBuilder {pull}15376[#15376] (issue: {issue}15044[#15044])
+* Joint parsing of common global Hightlighter and subfield parameters {pull}15368[#15368] (issue: {issue}15285[#15285])
+* Enable HighlightBuilder to create SearchContextHighlight {pull}15324[#15324]
+*  Add fromXContent method to HighlightBuilder {pull}15157[#15157]
+
+Index APIs::
+* Add date-math support to `_rollover` {pull}20709[#20709]
+* Add Shrink request source parser to parse create index request body {pull}18802[#18802]
+* Fail hot_threads in a better way if unsupported by JDK {pull}15909[#15909]
+
+Index Templates::
+* Add "version" field to Templates {pull}20353[#20353] (issue: {issue}20171[#20171])
+* Parse and validate mappings on index template creation {pull}8802[#8802] (issue: {issue}2415[#2415])
+
+Ingest::
+* Add "version" field to Pipelines {pull}20343[#20343] (issue: {issue}20171[#20171])
+* Make it possible for Ingest Processors to access AnalysisRegistry {pull}20233[#20233]
+* add ignore_missing option to convert,trim,lowercase,uppercase,grok,rename {pull}20194[#20194] (issue: {issue}19995[#19995])
+* Add support for parameters to the script ingest processor {pull}20136[#20136]
+* introduce the JSON Processor {pull}20128[#20128] (issue: {issue}20052[#20052])
+* Allow rename processor to turn leaf fields into branch fields {pull}20053[#20053] (issue: {issue}19892[#19892])
+* remove ability to set field value in script-processor configuration {pull}19981[#19981]
+* Add REST _ingest/pipeline to get all pipelines {pull}19603[#19603] (issue: {issue}19585[#19585])
+* Show ignored errors in verbose simulate result {pull}19404[#19404] (issue: {issue}19319[#19319])
+* update foreach processor to only support one applied processor. {pull}19402[#19402] (issue: {issue}19345[#19345])
+* Skip the execution of an empty pipeline {pull}19200[#19200] (issue: {issue}19192[#19192])
+* Add `ignore_failure` option to all ingest processors {pull}18650[#18650] (issue: {issue}18493[#18493])
+* new ScriptProcessor for Ingest {pull}18193[#18193]
+* Expose underlying processor to blame for thrown exception within CompoundProcessor {pull}18342[#18342] (issue: {issue}17823[#17823])
+* Avoid string concatentation in IngestDocument.FieldPath {pull}18108[#18108]
+* add ability to specify multiple grok patterns {pull}18074[#18074] (issue: {issue}17903[#17903])
+* add ability to disable ability to override values of existing fields in set processor {pull}17902[#17902] (issue: {issue}17659[#17659])
+* Streamline option naming for several processors {pull}17892[#17892] (issue: {issue}17835[#17835])
+* add automatic type conversion support to ConvertProcessor {pull}17263[#17263] (issue: {issue}17139[#17139])
+* Give the foreach processor access to the rest of the document {pull}17172[#17172] (issue: {issue}17147[#17147])
+* Added ingest statistics to node stats API {pull}16915[#16915]
+* Add `ingest_took` to bulk response {pull}16876[#16876]
+* Add ingest info to node info API, which contains a list of available processors {pull}16865[#16865]
+* Use diffs for ingest metadata in cluster state {pull}16847[#16847]
+* hide null-valued metadata fields from WriteableIngestDocument#toXContent {pull}16557[#16557]
+* Ingest: use bulk thread pool for bulk request processing (was index before) {pull}16539[#16539] (issue: {issue}16503[#16503])
+* Add foreach processor {pull}16432[#16432]
+* revert PipelineFactoryError handling with throwing ElasticsearchParseException in ingest pipeline creation {pull}16355[#16355]
+* Add processor tags to on_failure metadata in ingest pipeline {pull}16324[#16324] (issue: {issue}16202[#16202])
+* catch processor/pipeline factory exceptions and return structured error responses {pull}16276[#16276] (issue: {issue}16010[#16010])
+* Ingest: move get/put/delete pipeline methods to ClusterAdminClient {pull}16242[#16242]
+* Geoip processor: remove redundant latitude and longitude fields and make location an object with lat and lon subfields {pull}16173[#16173]
+
+Inner Hits::
+* Change scriptFields member in InnerHitBuilder to set {pull}18092[#18092] (issue: {issue}5831[#5831])
+
+Internal::
+* Remove poor-mans compression in InternalSearchHit and friends {pull}20472[#20472]
+* Don't register SearchTransportService handlers more than once {pull}20468[#20468]
+* Unguice SearchModule {pull}20456[#20456]
+* Deguice SearchService and friends {pull}20423[#20423]
+* NodeStats classes to implement Writeable rather then Streamable {pull}20327[#20327]
+* More info classes to implement Writeable rather than Streamable {pull}20288[#20288] (issue: {issue}20255[#20255])
+* Internal: Split disk threshold monitoring from decider {pull}20018[#20018]
+* Switching LockObtainFailedException over to ShardLockObtainFailedException {pull}19991[#19991] (issue: {issue}19978[#19978])
+* update and delete by query requests to implement IndicesRequest.Replaceable {pull}19961[#19961]
+* VersionFetchSubPhase should not use Versions#loadDocIdAndVersion {pull}19944[#19944]
+* Remove useless PK lookup in IndicesTTLService {pull}19945[#19945]
+* ignore some docker craziness in seccomp environment checks {pull}19754[#19754]
+* Make Priority an enum {pull}19448[#19448]
+* Snapshot UUIDs in blob names {pull}19421[#19421] (issues: {issue}18156[#18156], {issue}18815[#18815], {issue}19002[#19002], {issue}7540[#7540])
+* Add RestController method for deprecating in one step {pull}19343[#19343]
+* Tighten ensure atomic move cleanup {pull}19309[#19309] (issue: {issue}19036[#19036])
+* Enable checkstyle ModifierOrder {pull}19214[#19214]
+* Expose task information from NodeClient {pull}19189[#19189]
+* Changed rest handler interface to take NodeClient {pull}19170[#19170]
+* Deprecate ExceptionsHelper.detailedMessage {pull}19160[#19160] (issue: {issue}19069[#19069])
+* Factor out ChannelBuffer from BytesReference {pull}19129[#19129]
+* Cleanup Compressor interface {pull}19125[#19125]
+* Hot methods redux {pull}19016[#19016] (issue: {issue}16725[#16725])
+* Remove forked joda time BaseDateTime class {pull}18953[#18953]
+* Support optional ctor args in ConstructingObjectParser {pull}18725[#18725]
+* Remove thread pool from page cache recycler {pull}18664[#18664] (issue: {issue}18613[#18613])
+* Do not automatically close XContent objects/arrays {pull}18549[#18549] (issue: {issue}18433[#18433])
+* Remove use of a Fields class in snapshot responses {pull}18497[#18497]
+* Removes multiple toXContent entry points for SnapshotInfo {pull}18494[#18494]
+* Removes unused methods in the o/e/common/Strings class {pull}18346[#18346]
+* Determine content length eagerly in HttpServer {pull}18203[#18203]
+* Consolidate query generation in QueryShardContext {pull}18129[#18129]
+* Make reset in QueryShardContext private {pull}18113[#18113]
+* Remove Strings#splitStringToArray {pull}18110[#18110]
+* Add toString() to GetResponse {pull}18102[#18102]
+* ConstructingObjectParser adapts ObjectParser for ctor args {pull}17596[#17596] (issue: {issue}17352[#17352])
+* Makes Script type writeable {pull}17908[#17908] (issue: {issue}17753[#17753])
+* FiltersAggregatorBuilder: Don't create new context for inner parsing {pull}17851[#17851]
+* Clean up serialization on some stats {pull}17832[#17832] (issue: {issue}17085[#17085])
+* 	Normalize registration for SignificanceHeuristics {pull}17830[#17830] (issue: {issue}17085[#17085])
+* Make (read|write)NamedWriteable public {pull}17829[#17829] (issue: {issue}17682[#17682])
+* Use try-with-resource when creating new parser instances where possible {pull}17822[#17822]
+* Don't pass XContentParser to ParseFieldRegistry#lookup {pull}17794[#17794]
+* Internal: Remove threadlocal from document parser {pull}17764[#17764]
+* Cut range aggregations to registerAggregation {pull}17757[#17757] (issue: {issue}17085[#17085])
+* Remove ParseFieldMatcher from AbstractXContentParser {pull}17756[#17756] (issue: {issue}17417[#17417])
+* Remove parser argument from methods where we already pass in a parse context {pull}17738[#17738]
+* Switch SearchAfterBuilder to writeGenericValue {pull}17735[#17735] (issue: {issue}17085[#17085])
+* Remove StreamableReader {pull}17729[#17729] (issue: {issue}17085[#17085])
+* Cleanup nested, has_child & has_parent query builders for inner hits construction {pull}17719[#17719] (issue: {issue}11118[#11118])
+* Make AllocationCommands NamedWriteables {pull}17661[#17661]
+* Isolate StreamableReader {pull}17656[#17656] (issue: {issue}17085[#17085])
+* Create registration methods for aggregations similar to those for queries {pull}17653[#17653] (issues: {issue}17085[#17085], {issue}17389[#17389])
+* Remove PROTOTYPEs from QueryBuilders {pull}17632[#17632] (issue: {issue}17085[#17085])
+* Remove registerQueryParser {pull}17608[#17608]
+* ParseField#getAllNamesIncludedDeprecated to not return duplicate names {pull}17504[#17504]
+* Rework a query parser and improve registration {pull}17458[#17458]
+* Clean up QueryParseContext and don't hold it inside QueryRewrite/ShardContext {pull}17417[#17417]
+* Remove PROTOTYPE from MLT.Item {pull}17481[#17481] (issue: {issue}17085[#17085])
+* Remove PROTOTYPE from VersionType {pull}17480[#17480] (issue: {issue}17085[#17085])
+* Remove PROTOTYPEs from highlighting {pull}17466[#17466] (issue: {issue}17085[#17085])
+* Remove PROTOTYPEs from ingest {pull}17434[#17434] (issue: {issue}17085[#17085])
+* Start to rework query registration {pull}17424[#17424]
+* Factor out slow logs into Search and IndexingOperationListeners {pull}17398[#17398]
+* Remove PROTOTYPE from Suggesters {pull}17370[#17370]
+* Remove PROTOTYPE from SortBuilders {pull}17337[#17337] (issue: {issue}17085[#17085])
+* Remove PROTOTYPE from ShapeBuilders {pull}17336[#17336] (issue: {issue}17085[#17085])
+* Replace FieldStatsProvider with a method on MappedFieldType. {pull}17334[#17334]
+* Stop using PROTOTYPE in NamedWriteableRegistry {pull}17284[#17284] (issue: {issue}17085[#17085])
+* Support scheduled commands in current context {pull}17077[#17077]
+* Thread limits {pull}17003[#17003]
+* Remove leniency from segments info integrity checks {pull}16985[#16985] (issue: {issue}16973[#16973])
+* Rename SearchServiceTransportAction to SearchTransportService {pull}16880[#16880]
+* Decouple the TransportService and ClusterService {pull}16872[#16872] (issue: {issue}16788[#16788])
+* Refactor bootstrap checks {pull}16844[#16844] (issues: {issue}16733[#16733], {issue}16835[#16835])
+* Add LifecycleRunnable {pull}16752[#16752]
+* Hot inlined methods in your area {pull}16725[#16725]
+* Move IndicesQueryCache and IndicesRequestCache into IndicesService {pull}16603[#16603]
+* Forbid use of java.security.MessageDigest#clone() {pull}16543[#16543] (issue: {issue}16479[#16479])
+* Make IndicesWarmer a private class of IndexService {pull}16470[#16470]
+* Simplify IndicesFieldDataCache and detach from guice {pull}16469[#16469]
+* Uppercase ells ('L') in long literals {pull}16329[#16329] (issue: {issue}16279[#16279])
+* ShardId equality and hash code inconsistency {pull}16319[#16319] (issue: {issue}16217[#16217])
+* Ensure all resources are closed on Node#close() {pull}16316[#16316] (issue: {issue}13685[#13685])
+* Make index uuid available in Index, ShardRouting & ShardId {pull}16217[#16217]
+* Move RefreshTask into IndexService and use since task per index {pull}15933[#15933]
+* Make IndexingMemoryController private to IndicesService {pull}15877[#15877]
+* Cleanup IndexingOperationListeners infrastructure {pull}15875[#15875]
+* Remove and forbid use of j.u.c.ThreadLocalRandom {pull}15862[#15862] (issue: {issue}15294[#15294])
+* Fix IntelliJ query builder type inference issues {pull}15429[#15429]
+* Remove and forbid use of Collections#shuffle(List) and Random#<init>() {pull}15299[#15299] (issue: {issue}15287[#15287])
+* Remove and forbid use of the type-unsafe empty Collections fields {pull}15187[#15187]
+* Move IndicesService.canDeleteShardContent to use IndexSettings {pull}15150[#15150] (issue: {issue}15059[#15059])
+* Simplify MonitorService construction and detach from guice {pull}15035[#15035]
+* Use Supplier for StreamInput#readOptionalStreamable {pull}14806[#14806]
+* Add variable-length long encoding {pull}14780[#14780]
+* Extend usage of IndexSetting class {pull}14731[#14731] (issue: {issue}14251[#14251])
+* Fold SimilarityModule into IndexModule {pull}14284[#14284]
+* Move to lucene BoostQuery {pull}14264[#14264]
+* Use built-in method for computing hash code of longs {pull}14213[#14213]
+* Refactor ShardFailure listener infrastructure {pull}14206[#14206]
+* Add methods for variable-length encoding integral arrays {pull}14087[#14087]
+* Fold IndexAliasesService into IndexService {pull}14044[#14044]
+* Remove unneeded Module abstractions {pull}13944[#13944]
+* Query refactoring: simplify IndexQueryParserService parse methods {pull}13938[#13938] (issue: {issue}13859[#13859])
+* Remove and forbid use of com.google.common.collect.Iterators {pull}13916[#13916] (issue: {issue}13224[#13224])
+* Remove and forbid use of com.google.common.collect.ImmutableCollection {pull}13909[#13909] (issue: {issue}13224[#13224])
+* Remove and forbid use of com.google.common.io.Resources {pull}13908[#13908] (issue: {issue}13224[#13224])
+* Remove and forbid use of com.google.common.hash.* {pull}13907[#13907] (issue: {issue}13224[#13224])
+* Remove and forbid use of com.google.common.net.InetAddresses {pull}13905[#13905] (issue: {issue}13224[#13224])
+* Remove and forbid use of com.google.common.collect.EvictingQueue {pull}13903[#13903] (issue: {issue}13224[#13224])
+* Replace Guava cache with simple concurrent LRU cache {pull}13879[#13879]
+* Remove ClusterSerivce and IndexSettingsService dependency from IndexShard {pull}13853[#13853]
+* Start making RecoverySourceHandler unittestable {pull}13840[#13840]
+* Remove IndexService dep. from IndexShard {pull}13797[#13797]
+* Remove ES internal deletion policies in favour of Lucenes implementations {pull}13794[#13794]
+* Move ShardTermVectorService to be on indices level as TermVectorService {pull}13786[#13786]
+* Move ShardPercolateService creation into IndexShard {pull}13777[#13777]
+* Remove `ExpressionScriptCompilationException` and `ExpressionScriptExecutionException` {pull}13742[#13742]
+* Reduced the number of ClusterStateUpdateTask variants {pull}13735[#13735]
+* Add a BaseParser helper for stream parsing {pull}13615[#13615]
+* Remove and forbid use of com.google.common.primitives.Ints {pull}13596[#13596] (issue: {issue}13224[#13224])
+* Remove and forbid use of com.google.common.math.LongMath {pull}13575[#13575] (issue: {issue}13224[#13224])
+* Remove and forbid use of com.google.common.base.Joiner {pull}13572[#13572] (issue: {issue}13224[#13224])
+* Replace and ban next batch of Guava classes {pull}13562[#13562] (issue: {issue}13224[#13224])
+* Remove and forbid use of com.google.common.collect.Iterables {pull}13559[#13559] (issue: {issue}13224[#13224])
+* Replace LoadingCache usage with a simple ConcurrentHashMap {pull}13552[#13552] (issue: {issue}13224[#13224])
+* Use Supplier instead of Reflection {pull}13545[#13545]
+* Remove and forbid use of com.google.common.base.Preconditions {pull}13540[#13540] (issue: {issue}13224[#13224])
+* Remove and forbid use of guava Function, Charsets, Collections2 {pull}13533[#13533] (issue: {issue}13224[#13224])
+* Remove and forbid use of com.google.common.collect.ImmutableSortedMap {pull}13525[#13525] (issue: {issue}13224[#13224])
+* Remove and forbid use of several com.google.common.util. classes {pull}13524[#13524] (issue: {issue}13224[#13224])
+* Cleanup SearchRequest & SearchRequestBuilder {pull}13518[#13518]
+* Remove and forbid use of com.google.common.collect.Queues {pull}13498[#13498] (issue: {issue}13224[#13224])
+* Remove and forbid use of com.google.common.base.Preconditions#checkNotNull {pull}13493[#13493] (issue: {issue}13224[#13224])
+* Remove and forbid use of com.google.common.collect.Sets {pull}13463[#13463] (issue: {issue}13224[#13224])
+* Remove and forbid use of com.google.common.collect.Maps {pull}13438[#13438] (issue: {issue}13224[#13224])
+* Remove use of underscore as an identifier {pull}13353[#13353]
+* Remove and forbid the use of com.google.common.base.Predicate(s)? {pull}13349[#13349] (issues: {issue}13224[#13224], {issue}13314[#13314])
+* This commit removes com.google.common.io {pull}13302[#13302] (issue: {issue}13224[#13224])
+
+Java API::
+* Ensure PutMappingRequest.buildFromSimplifiedDef input are pairs {pull}19837[#19837] (issue: {issue}19836[#19836])
+* Start from a random node number so that clients do not overload the first node configured {pull}14143[#14143]
+* Switch QueryBuilders to new MatchPhraseQueryBuilder {pull}18753[#18753]
+* Improve adding clauses to `span_near` and `span_or` query {pull}18485[#18485] (issue: {issue}18478[#18478])
+* QueryBuilder does not need generics. {pull}18133[#18133]
+* Remove copy constructors from request classes and TransportMessage type {pull}16640[#16640] (issue: {issue}15776[#15776])
+
+Java REST Client::
+* Add support for a RestClient path prefix {pull}20190[#20190]
+* Add "Async" to the end of each Async RestClient method {pull}20172[#20172] (issue: {issue}20168[#20168])
+* Allow RestClient to send array-based headers {pull}20151[#20151]
+* Add response body to ResponseException error message {pull}19653[#19653] (issue: {issue}19653[#19653])
+* Simplify Sniffer initialization and automatically create the default HostsSniffer {pull}19599[#19599]
+* Remove duplicate dependency declaration for http client {pull}19580[#19580] (issue: {issue}19281[#19281])
+* Add callback to customize http client settings {pull}19373[#19373]
+* Rest Client: add short performRequest method variants without params and/or body {pull}19340[#19340] (issue: {issue}19312[#19312])
+
+Logging::
+* Ensure logging is initialized in CLI tools {pull}20575[#20575]
+* Give useful error message if log config is missing {pull}20493[#20493]
+* Complete Elasticsearch logger names {pull}20457[#20457] (issue: {issue}20326[#20326])
+* Add node name to decider trace logging  {pull}20437[#20437] (issue: {issue}20379[#20379])
+* Logging shutdown hack {pull}20389[#20389] (issue: {issue}20304[#20304])
+* Disable console logging {pull}20387[#20387]
+* Skip loading of jansi from log4j2 {pull}20334[#20334]
+* Configure AWS SDK logging configuration {pull}20313[#20313] (issue: {issue}20294[#20294])
+* Warn if unsupported logging configuration present {pull}20309[#20309]
+* Size limit deprecation logs {pull}20287[#20287] (issues: {issue}20235[#20235], {issue}20254[#20254])
+* Increase visibility of deprecation logger {pull}20254[#20254] (issue: {issue}11033[#11033])
+* Add log message about enforcing bootstrap checks {pull}19451[#19451]
+* Improve logging for batched cluster state updates {pull}19255[#19255]
+* Send HTTP Warning Header(s) for any Deprecation Usage from a REST request {pull}17804[#17804] (issue: {issue}17687[#17687])
+* Throw IllegalStateException when handshake fails due to version or cluster mismatch {pull}18676[#18676]
+
+Mapping::
+* Automatically downgrade text and keyword to string on indexes imported from 2.x {pull}20177[#20177]
+* Do not parse numbers as both strings and numbers when not included in `_all`. {pull}20167[#20167]
+* Don't index the `_version` field {pull}20132[#20132]
+* The root object mapper should support updating `numeric_detection`, `date_detection` and `dynamic_date_formats`. {pull}20119[#20119] (issue: {issue}20111[#20111])
+* Automatically upgrade analyzed string fields that have `index_options` or `position_increment_gap` set. {pull}20002[#20002] (issue: {issue}19974[#19974])
+* Mappings: Support dots in field names in mapping parsing {pull}19899[#19899] (issue: {issue}19443[#19443])
+* Save one utf8 conversion in KeywordFieldMapper. {pull}19867[#19867]
+* Do not parse the created version from the settings every time a field is parsed. {pull}19824[#19824]
+* Elasticsearch should reject dynamic templates with unknown `match_mapping_type`. {pull}17285[#17285] (issue: {issue}16945[#16945])
+* Upgrade `string` fields to `text`/`keyword` even if `include_in_all` is set. {pull}19004[#19004] (issue: {issue}18974[#18974])
+* Adds a methods to find (and dynamically create) the mappers for the parents of a field with dots in the field name {pull}18106[#18106] (issue: {issue}15951[#15951])
+* Automatically upgrade analyzed strings with an analyzer to `text`. {pull}17861[#17861]
+* Support dots in field names when mapping already exists {pull}17759[#17759] (issue: {issue}15951[#15951])
+* Use the new points API to index numeric fields. {pull}17746[#17746] (issues: {issue}11513[#11513], {issue}16751[#16751], {issue}17007[#17007], {issue}17700[#17700])
+* Simplify AllEntries, AllField and AllFieldMapper: {pull}17613[#17613]
+* Make `parseMultiField` part of `parseField`. {pull}17313[#17313]
+* Automatically add a sub keyword field to string dynamic mappings. {pull}17188[#17188]
+* Remove friction from the mapping changes in 5.0. {pull}16991[#16991]
+* Rework norms parameters for 5.0. {pull}16987[#16987]
+* Moved dynamic field handling in doc parsing to end of parsing {pull}16798[#16798]
+* Remove the MapperBuilders utility class. {pull}16609[#16609]
+* Make the `index` property a boolean. {pull}16161[#16161]
+* Remove the ability to enable doc values with the `fielddata.format` setting. {pull}16147[#16147]
+* Be stricter about parsing boolean values in mappings. {pull}16146[#16146]
+* Fix default doc values to be enabled when a field is not indexed. {pull}16141[#16141]
+* Dynamically map floating-point numbers as floats instead of doubles. {pull}15319[#15319] (issue: {issue}13851[#13851])
+* Simplify MetaDataMappingService. {pull}15217[#15217]
+* Remove MergeMappingException. {pull}15177[#15177]
+
+Network::
+* Avoid early initializing Netty {pull}19819[#19819] (issue: {issue}5644[#5644])
+* Network: Allow to listen on virtual interfaces. {pull}19568[#19568] (issues: {issue}17473[#17473], {issue}19537[#19537])
+* Explicitly tell Netty to not use unsafe {pull}19786[#19786] (issues: {issue}19562[#19562], {issue}5624[#5624])
+* Enable Netty 4 extensions {pull}19767[#19767] (issue: {issue}19526[#19526])
+* Modularize netty {pull}19392[#19392]
+* Simplify TcpTransport interface by reducing send code to a single send method {pull}19223[#19223]
+* Do not start scheduled pings until transport start {pull}18702[#18702]
+
+Packaging::
+* Add quiet option to disable console logging {pull}20422[#20422] (issues: {issue}15315[#15315], {issue}16159[#16159], {issue}17220[#17220])
+* Explicitly disable Netty key set replacement {pull}20249[#20249]
+* Remove explicit parallel new GC flag {pull}18767[#18767]
+* Use JAVA_HOME or java.exe in PATH like the Linux scripts do {pull}18685[#18685] (issue: {issue}4913[#4913])
+* Don't mkdir directly in deb init script {pull}18503[#18503] (issue: {issue}18307[#18307])
+* Increase default heap size to 2g {pull}18311[#18311] (issues: {issue}16334[#16334], {issue}17686[#17686], {issue}18309[#18309])
+* Switch init.d scripts to use bash {pull}18308[#18308] (issue: {issue}18259[#18259])
+* Switch scripts to use bash {pull}18251[#18251] (issue: {issue}14002[#14002])
+* Further simplifications of plugin script {pull}18239[#18239] (issue: {issue}18207[#18207])
+* Pass ES_JAVA_OPTS to JVM for plugins script {pull}18140[#18140] (issue: {issue}16790[#16790])
+* Remove unnecessary sleep from init script restart {pull}17966[#17966]
+* Explicitly set packaging permissions {pull}17912[#17912] (issue: {issue}17634[#17634])
+* rpm uses non-portable `--system` flag to `useradd` {pull}14596[#14596] (issue: {issue}14211[#14211])
+* Adding JAVA_HOME to documents and env config file {pull}11338[#11338] (issue: {issue}11291[#11291])
+* Added RPM metadata {pull}17477[#17477]
+* Elasticsearch ownership for data, logs, and configs {pull}17197[#17197] (issue: {issue}12688[#12688])
+* Fail early on JDK with compiler bug {pull}16418[#16418] (issues: {issue}16097[#16097], {issue}16362[#16362])
+* Make security non-optional {pull}16176[#16176]
+* Remove RuntimePermission("accessDeclaredMembers") {pull}15378[#15378]
+* Remove Guava as a dependency {pull}14055[#14055] (issue: {issue}13224[#13224])
+* Remove Guava as a dependency {pull}14054[#14054] (issue: {issue}13224[#13224])
+
+Percolator::
+* Also support query term extract for queries wrapped inside a FunctionScoreQuery {pull}19184[#19184]
+* Add support for synonym query to percolator query term extraction {pull}19066[#19066]
+* Add percolator query extraction support for dismax query {pull}18845[#18845]
+* Improve percolate query performance by not verifying certain candidate matches {pull}18696[#18696]
+* Improve percolator query term extraction {pull}18610[#18610]
+* PercolatorQueryBuilder cleanup by using MemoryIndex#fromDocument(...) helper {pull}17669[#17669] (issue: {issue}9386[#9386])
+* Add scoring support to the percolator query {pull}17385[#17385] (issue: {issue}13827[#13827])
+* Add query extract support for the blended term query and the common terms query {pull}17347[#17347]
+* Add support for several span queries in ExtractQueryTermsService {pull}17323[#17323]
+* Add support for TermsQuery in ExtractQueryTermsService {pull}17316[#17316]
+* Replace percolate APIs with a percolator query {pull}16349[#16349] (issues: {issue}10741[#10741], {issue}11264[#11264], {issue}13176[#13176], {issue}13978[#13978], {issue}4317[#4317], {issue}7297[#7297])
+
+Plugin Analysis Kuromoji::
+* Add nbest options and NumberFilter {pull}17173[#17173]
+
+Plugin Discovery EC2::
+* Use `DefaultAWSCredentialsProviderChain` AWS SDK class for credentials {pull}19561[#19561] (issue: {issue}19556[#19556])
+* Support new Asia Pacific (Mumbai) ap-south-1 AWS region {pull}19112[#19112] (issue: {issue}19110[#19110])
+* Add support for proxy authentication for s3 and ec2 {pull}15293[#15293] (issue: {issue}15268[#15268])
+
+Plugin Discovery GCE::
+* Allow `_gce_` network when not using discovery gce {pull}15765[#15765] (issue: {issue}15724[#15724])
+
+Plugin Ingest Attachment::
+* Minor attachment processor improvements {pull}16574[#16574]
+
+Plugin Lang Painless::
+* Disable regexes by default in painless {pull}20427[#20427] (issue: {issue}20397[#20397])
+* Catch OutOfMemory and StackOverflow errors in Painless {pull}19936[#19936]
+* Change Painless Tree Structure for Variable/Method Chains {pull}19459[#19459]
+* Add replaceAll and replaceFirst {pull}19070[#19070]
+* Painless Initializers {pull}19012[#19012]
+* Add augmentation {pull}19003[#19003]
+* Infer lambda arguments/return type {pull}18983[#18983]
+* Fix explicit casts and improve tests. {pull}18958[#18958]
+* Add lambda captures {pull}18954[#18954]
+* improve Debugger to print code even if it hits exception {pull}18932[#18932] (issue: {issue}1[#1])
+* Move semicolon hack into lexer {pull}18931[#18931]
+* Add flag support to regexes {pull}18927[#18927]
+* improve lambda syntax (allow single expression) {pull}18924[#18924]
+* Remove useless dropArguments in megamorphic cache {pull}18913[#18913]
+* non-capturing lambda support {pull}18911[#18911] (issue: {issue}18824[#18824])
+* fix bugs in operators and more improvements for the dynamic case {pull}18899[#18899]
+* improve unary operators and cleanup tests {pull}18867[#18867] (issue: {issue}18849[#18849])
+* Add support for the find operator (=~) and the match operator (==~) {pull}18858[#18858]
+* Remove casts and boxing for dynamic math {pull}18849[#18849] (issue: {issue}18847[#18847])
+* Refactor def math {pull}18847[#18847]
+* Add support for /regex/ {pull}18842[#18842]
+* Array constructor references {pull}18831[#18831]
+* Method references to user functions {pull}18828[#18828]
+* Add } as a delimiter.  {pull}18827[#18827] (issue: {issue}18821[#18821])
+* Add Lambda Stub Node {pull}18824[#18824]
+* Add capturing method references {pull}18818[#18818] (issue: {issue}18748[#18748])
+* Add Functions to Painless {pull}18810[#18810]
+* Add Method to Get New MethodWriters {pull}18771[#18771]
+* Static For Each {pull}18757[#18757]
+* Method reference support {pull}18748[#18748] (issue: {issue}18578[#18578])
+* Add support for the new Java 9 MethodHandles#arrayLength() factory {pull}18734[#18734]
+* Improve painless compile-time exceptions {pull}18711[#18711] (issue: {issue}18600[#18600])
+* add java.time packages to painless whitelist {pull}18621[#18621]
+* Add Function Reference Stub to Painless {pull}18578[#18578]
+* improve painless whitelist coverage of java api {pull}18533[#18533]
+* Definition cleanup {pull}18463[#18463]
+* Made def variable casting consistent with invokedynamic rules {pull}18425[#18425]
+* Use Java 9 Indy String Concats, if available {pull}18400[#18400] (issue: {issue}18398[#18398])
+* Add method overloading based on arity {pull}18385[#18385]
+* Refactor WriterUtils to extend ASM GeneratorAdapter {pull}18382[#18382]
+* Whitelist expansion {pull}18372[#18372]
+* Remove boxing when loading and storing values in "def" fields/arrays, remove boxing onsimple method calls of "def" methods {pull}18359[#18359]
+* Some cleanups {pull}18352[#18352]
+* Use isAssignableFrom instead of relying on ClassCastException {pull}18350[#18350]
+* Build descriptor of array and field load/store in code {pull}18338[#18338]
+* Rename the dynamic call site factory to DefBootstrap {pull}18335[#18335]
+* Cleanup of DynamicCallSite {pull}18323[#18323]
+* Improve exception stacktraces {pull}18319[#18319]
+* Make Line Number Available in Painless {pull}18298[#18298]
+* Remove input, support params instead {pull}18287[#18287]
+* Decouple ANTLR AST from Painless {pull}18286[#18286]
+* _value support in painess? {pull}18284[#18284]
+* Long priority over Float {pull}18282[#18282]
+* _score as double, not float {pull}18277[#18277]
+* Add 'ctx' keyword to painless. {pull}18264[#18264]
+* Painless doc access {pull}18262[#18262]
+* Retrieve _score directly from Scorer {pull}18258[#18258]
+* Implement needsScore() correctly. {pull}18247[#18247]
+* Add synthetic length property as alias to Lists, so they can be used like arrays {pull}18241[#18241]
+* Use better typing for dynamic method calls {pull}18234[#18234]
+* Array load/store and length with invokedynamic {pull}18232[#18232] (issue: {issue}18201[#18201])
+* Switch painless dynamic calls to invokedynamic, remove perf hack/cheat {pull}18201[#18201]
+* Add fielddata accessors (.value/.values/.distance()/etc) {pull}18169[#18169]
+* painless: optimize/simplify dynamic field and method access {pull}18151[#18151]
+* Painless: Single-Quoted Strings {pull}18150[#18150]
+* Painless Clean Up {pull}17428[#17428]
+* Make Painless a Module {pull}16755[#16755]
+* Minor Clean up {pull}16457[#16457]
+* Remove Extra String Concat Token {pull}16382[#16382]
+
+Plugin Mapper Attachment::
+* minor attachments cleanups: IDE test support and EPUB format {pull}14626[#14626]
+
+Plugin Mapper Size::
+* Add doc values support to the _size field in the mapper-size plugin {pull}19217[#19217] (issue: {issue}18334[#18334])
+
+Plugin Repository Azure::
+* Support global `repositories.azure.` settings {pull}15141[#15141] (issue: {issue}13776[#13776])
+* Add timeout settings (default to 5 minutes) {pull}15080[#15080] (issue: {issue}14277[#14277])
+* Remove AbstractLegacyBlobContainer {pull}14650[#14650] (issue: {issue}13434[#13434])
+
+Plugin Repository HDFS::
+* merge current hdfs improvements to master {pull}15588[#15588]
+
+Plugin Repository S3::
+* Extract AWS Key from KeyChain instead of using potential null value {pull}19560[#19560] (issues: {issue}18703[#18703], {issue}19557[#19557])
+* Check that S3 setting `buffer_size` is always lower than `chunk_size` {pull}17274[#17274] (issue: {issue}17244[#17244])
+
+Plugins::
+* Revert "Display plugins versions" {pull}20807[#20807] (issues: {issue}18683[#18683], {issue}20668[#20668])
+* Provide error message when plugin id is missing {pull}20660[#20660]
+* Print message when removing plugin with config {pull}20338[#20338]
+* Plugins: Update official plugin location with unified release {pull}19996[#19996]
+* Allow plugins to upgrade global custom metadata on startup {pull}19962[#19962]
+* Switch aggregations from push to pull {pull}19839[#19839]
+* Display plugins versions {pull}18683[#18683]
+* Add ScriptService to dependencies available for plugin components {pull}19770[#19770]
+* Make NamedWriteableRegistry immutable and add extension point for named writeables {pull}19764[#19764]
+* Log one plugin info per line {pull}19441[#19441]
+* Make rest headers registration pull based {pull}19440[#19440]
+* Add resource watcher to services available for plugin components {pull}19401[#19401]
+* Add some basic services to createComponents for plugins {pull}19380[#19380]
+* Make plugins closeable {pull}19137[#19137]
+* Plugins: Add status bar on download {pull}18695[#18695]
+* Add did-you-mean for plugin cli {pull}18942[#18942] (issue: {issue}18896[#18896])
+* Plugins: Remove name() and description() from api {pull}18906[#18906]
+* Emit nicer error message when trying to install unknown plugin {pull}18876[#18876] (issue: {issue}17226[#17226])
+* Add plugin information for Verbose mode {pull}18051[#18051] (issue: {issue}16375[#16375])
+* Cli: Improve output for usage errors {pull}17938[#17938]
+* Cli: Add verbose output with zip url when installing plugin {pull}17662[#17662] (issue: {issue}17529[#17529])
+* PluginManager: Add xpack as official plugin {pull}17227[#17227]
+* CliTool: Cleanup and document Terminal {pull}16443[#16443]
+* Plugin cli: Improve maven coordinates detection {pull}16384[#16384] (issue: {issue}16376[#16376])
+* Enforce plugin zip does not contain zip entries outside of the plugin dir {pull}16361[#16361]
+* CliTool: Allow unexpected exceptions to propagate {pull}16359[#16359]
+* Reduce complexity of plugin cli {pull}16336[#16336]
+* Remove Plugin.onIndexService. {pull}15029[#15029] (issue: {issue}14896[#14896])
+* Open up QueryCache and SearcherWrapper extension points {pull}14303[#14303]
+
+Query DSL::
+* Throw exception when multiple field names are provided as part of query short syntax {pull}19871[#19871] (issue: {issue}19791[#19791])
+* Query parsers to throw exception when multiple field names are provided {pull}19791[#19791] (issue: {issue}19547[#19547])
+* Allow empty json object in request body in `_count` API {pull}19595[#19595] (issue: {issue}19422[#19422])
+* Treat zero token in `common` terms query as MatchNoDocsQuery {pull}18656[#18656]
+* Handle empty query bodies at parse time and remove EmptyQueryBuilder {pull}17624[#17624] (issues: {issue}17540[#17540], {issue}17541[#17541])
+* Enforce MatchQueryBuilder#maxExpansions() to be strictly positive {pull}18464[#18464]
+* Don't allow `fuzziness` for `multi_match` types `cross_fields`, `phrase` and `phrase_prefix` {pull}18322[#18322] (issues: {issue}6866[#6866], {issue}7764[#7764])
+* Add MatchNoDocsQuery, a query that matches no documents and prints the reason why in the toString method. {pull}17780[#17780]
+* Adds `ignore_unmapped` option to geo queries {pull}17751[#17751]
+* Adds `ignore_unmapped` option to nested and P/C queries {pull}17748[#17748]
+* SimpleQueryParser should call MappedFieldType.termQuery when appropriate. {pull}17678[#17678]
+* An `exists` query on an object should query a single term. {pull}17186[#17186] (issue: {issue}17131[#17131])
+* Function Score Query: make parsing stricter {pull}16617[#16617] (issue: {issue}16583[#16583])
+* Parsers should throw exception on unknown objects {pull}14255[#14255] (issue: {issue}10974[#10974])
+* UNICODE_CHARACTER_CLASS fix {pull}11598[#11598] (issue: {issue}10146[#10146])
+
+Query Refactoring::
+* Add infrastructure to rewrite query builders {pull}16599[#16599]
+* Switch geo validation to enum {pull}13672[#13672] (issue: {issue}13608[#13608])
+
+REST::
+* Add a REST spec for the create API {pull}20924[#20924]
+* Add response params to REST params did you mean {pull}20753[#20753] (issues: {issue}20722[#20722], {issue}20747[#20747])
+* Add did you mean to strict REST params {pull}20747[#20747] (issue: {issue}20722[#20722])
+* Add exclusion support to response filtering {pull}19865[#19865]
+* Only write forced_refresh if we forced a refresh {pull}19669[#19669] (issue: {issue}19629[#19629])
+* Add Location header to the index, update, and create APIs {pull}19509[#19509] (issue: {issue}19079[#19079])
+* Add support for `wait_for_events` to the `_cluster/health` REST endpoint {pull}19432[#19432] (issue: {issue}19419[#19419])
+* Rename Search Template REST spec names {pull}19178[#19178]
+* Adding status field in _msearch error request bodies {pull}18586[#18586] (issue: {issue}18013[#18013])
+* Add semicolon query string parameter delimiter {pull}18186[#18186] (issue: {issue}18175[#18175])
+* Enable HTTP compression by default with compression level 3 {pull}18066[#18066] (issue: {issue}7309[#7309])
+* Allow JSON with unquoted field names by enabling system property {pull}17801[#17801] (issue: {issue}17674[#17674])
+* More robust handling of CORS HTTP Access Control {pull}16092[#16092]
+* Add option to exclude based on paths in XContent {pull}16017[#16017]
+
+Recovery::
+* Pass on maxUnsafeAutoIdTimestamp on recovery / relocation {pull}20300[#20300]
+* Non-blocking primary relocation hand-off {pull}19013[#19013] (issues: {issue}15900[#15900], {issue}18553[#18553])
+* index shard should be able to cancel check index on close. {pull}18839[#18839] (issue: {issue}12011[#12011])
+* TransportNodesListGatewayStartedShards should fall back to disk based index metadata if not found in cluster state {pull}17663[#17663] (issue: {issue}17630[#17630])
+* Recover broken IndexMetaData as closed {pull}17187[#17187]
+* Relocation source should be marked as relocating before starting recovery to primary relocation target {pull}16500[#16500]
+* Operation counter for IndexShard {pull}15956[#15956] (issue: {issue}15900[#15900])
+* Primary relocation handoff {pull}15900[#15900] (issues: {issue}15532[#15532], {issue}16274[#16274], {issue}19013[#19013])
+* Remove recovery threadpools and throttle outgoing recoveries on the master {pull}15372[#15372]
+* Refactor StoreRecoveryService to be a simple package private util class {pull}13766[#13766]
+
+Reindex API::
+* Only ask for `_version` we need it {pull}19693[#19693] (issue: {issue}19135[#19135])
+* Use fewer threads when reindexing-from-remote {pull}19636[#19636]
+* Support authentication with reindex-from-remote {pull}19310[#19310]
+* Support requests_per_second=-1 to mean no throttling in reindex {pull}19101[#19101] (issue: {issue}19089[#19089])
+* Implement ctx.op = "delete" on _update_by_query and _reindex {pull}18614[#18614] (issue: {issue}18043[#18043])
+* Make Reindex cancellation tests more uniform {pull}18498[#18498]
+* Makes DeleteByQueryRequest implements IndicesRequest {pull}18466[#18466]
+* Teach reindex to retry on search failures {pull}18331[#18331] (issue: {issue}18059[#18059])
+* Remove ReindexResponse in favor of BulkIndexByScrollResponse {pull}18205[#18205]
+* Stricter validation of Reindex's requests_per_second {pull}18028[#18028]
+* Properly mark reindex's child tasks as child tasks {pull}17770[#17770]
+* Make reindex throttling dynamic {pull}17262[#17262]
+* Throttling support for reindex {pull}17039[#17039]
+* Add ingest pipeline support to reindex {pull}16932[#16932]
+
+Scripting::
+* Parse script on storage instead of on retrieval {pull}20356[#20356]
+* Migrate elasticsearch native script examples to the main repo {pull}19334[#19334] (issue: {issue}14662[#14662])
+* Remove ClusterState from compile api {pull}19136[#19136]
+* Mustache: Render Map as JSON {pull}18856[#18856] (issue: {issue}18970[#18970])
+*  Compile each Groovy script in its own classloader {pull}18918[#18918] (issue: {issue}18572[#18572])
+* Include script field even if it value is null {pull}18384[#18384] (issue: {issue}16408[#16408])
+* Skipping hidden files compilation for script service {pull}16286[#16286] (issue: {issue}15269[#15269])
+* Rename Plan A to Painless {pull}16245[#16245]
+* Add plumbing for script compile-time parameters {pull}15464[#15464]
+* Factor mustache -> modules/lang-mustache {pull}15328[#15328]
+
+Scroll::
+* Add an index setting to limit the maximum number of slices allowed in a scroll request. {pull}18782[#18782]
+
+Search::
+* Limit batch size when scrolling {pull}19367[#19367] (issue: {issue}19249[#19249])
+* Record method counts while profiling query components {pull}18302[#18302]
+* Change default similarity to BM25 {pull}18948[#18948] (issue: {issue}18944[#18944])
+* Add a parameter to cap the number of searches the msearch api will concurrently execute {pull}18721[#18721]
+* Introduces GeoValidationMethod to GeoDistanceSortBuilder {pull}18036[#18036]
+* Switches from empty boolean query to matchNoDocs {pull}18007[#18007] (issue: {issue}17981[#17981])
+* Allow binary sort values. {pull}17959[#17959] (issues: {issue}17971[#17971], {issue}6077[#6077])
+* Fail query if it contains very large rescores {pull}17917[#17917] (issue: {issue}17522[#17522])
+* Type filters should not have a performance impact when there is a single type. {pull}17350[#17350]
+* Store _all payloads on 1 byte instead of 4. {pull}16899[#16899]
+* Refuse to load fields from _source when using the `fields` option and support wildcards.  {pull}15017[#15017] (issues: {issue}10783[#10783], {issue}14489[#14489])
+* Add response into ClearScrollResponse {pull}13835[#13835] (issue: {issue}13817[#13817])
+* Shuffle shards for _only_nodes + support multiple specifications like cluster API  {pull}12575[#12575] (issues: {issue}12546[#12546], {issue}12700[#12700])
+
+Search Refactoring::
+* Removes the now obsolete SearchParseElement implementations {pull}18233[#18233]
+* Remove RescoreParseElement {pull}17441[#17441]
+* Remove HighlighterParseElement {pull}17303[#17303]
+* Move top level parsing of sort element to SortBuilder {pull}17248[#17248]
+* Switch to using refactored SortBuilder instead of using BytesReference in serialization {pull}17205[#17205] (issues: {issue}17146[#17146], {issue}17257[#17257])
+* Add build() method to SortBuilder implementations {pull}17146[#17146] (issue: {issue}10217[#10217])
+* Refactoring of Suggestions {pull}17096[#17096] (issue: {issue}10217[#10217])
+* Move sort `order` field up into SortBuilder {pull}17035[#17035]
+* Moves SortParser:parse(...) to only require QueryShardContext {pull}16999[#16999] (issue: {issue}15178[#15178])
+* Change internal representation of suggesters  {pull}16873[#16873]
+* Make GeoDistanceSortBuilder serializable, 2nd try {pull}16572[#16572] (issues: {issue}15178[#15178], {issue}16151[#16151])
+* Move missing() from SortBuilder interface to class {pull}16225[#16225] (issues: {issue}15178[#15178], {issue}16151[#16151])
+* Remove deprecated parameters from ScriptSortBuilder {pull}16153[#16153] (issue: {issue}15178[#15178])
+* Refactor GeoSortBuilder {pull}16151[#16151] (issue: {issue}15178[#15178])
+* Refactor FieldSortBuilder {pull}16127[#16127] (issue: {issue}15178[#15178])
+* Make sort order enum writable. {pull}16124[#16124] (issue: {issue}15178[#15178])
+* Make DistanceUnit writable. {pull}16122[#16122] (issue: {issue}15178[#15178])
+* RescoreBuilder: Add parsing and creating of RescoreSearchContext {pull}16014[#16014] (issue: {issue}15559[#15559])
+* Make RescoreBuilder and nested QueryRescorer Writable {pull}15953[#15953] (issue: {issue}15559[#15559])
+* Explain api: move query parsing to the coordinating node {pull}14270[#14270]
+* Switch query parsers to use ParseField  {pull}14249[#14249] (issue: {issue}8964[#8964])
+* Refactoring of Aggregations {pull}14136[#14136]
+
+Sequence IDs::
+* Persist sequence number checkpoints {pull}18949[#18949] (issue: {issue}10708[#10708])
+* Add sequence numbers to cat shards API {pull}18772[#18772]
+
+Settings::
+* Add precise logging on unknown or invalid settings {pull}20951[#20951] (issue: {issue}20946[#20946])
+* Make `action.auto_create_index` setting a dynamic cluster setting {pull}20274[#20274] (issue: {issue}7513[#7513])
+* Removes space between # and the setting in elasticsearch.yml {pull}20094[#20094] (issue: {issue}20090[#20090])
+* Validates new dynamic settings from the current state {pull}19122[#19122] (issue: {issue}19046[#19046])
+* Improve error message if a setting is not found {pull}18920[#18920] (issue: {issue}18663[#18663])
+* Cleanup placeholder replacement {pull}17335[#17335]
+* Switch to registered Settings for all IndexingMemoryController settings {pull}17778[#17778] (issue: {issue}17442[#17442])
+* Add guard against null-valued settings {pull}17310[#17310] (issue: {issue}17292[#17292])
+* Useful error message for null property placeholder {pull}17293[#17293] (issue: {issue}17292[#17292])
+* Archive cluster level settings if unknown or broken {pull}17246[#17246]
+* Improve error message if setting is not found {pull}17230[#17230]
+* Improve upgrade experience of node level index settings {pull}17223[#17223] (issue: {issue}17187[#17187])
+* Settings with complex matchers should not overlap {pull}16754[#16754]
+* Moves GCE settings to the new infra {pull}16722[#16722] (issue: {issue}16720[#16720])
+* Add filtering support within Setting class {pull}16629[#16629] (issue: {issue}16598[#16598])
+* Migrate AWS settings to new settings infrastructure {pull}16602[#16602] (issue: {issue}16293[#16293])
+* Remove `gateway.initial_meta` and always rely on min master nodes {pull}16446[#16446]
+* Rewrite SettingsFilter to be immutable {pull}16425[#16425]
+* Simplify azure settings {pull}16363[#16363]
+* Convert PageCacheRecycler settings {pull}16341[#16341]
+* Monitor settings {pull}16313[#16313]
+* Cut over tribe node settings to new settings infra {pull}16311[#16311]
+* Convert multcast plugin settings to the new infra {pull}16295[#16295]
+* Convert `request.headers.*` to the new settings infra {pull}16292[#16292]
+* Migrate Azure settings to new settings infrastructure {pull}16291[#16291]
+* Validate logger settings and allow them to be reset via API {pull}16289[#16289]
+* Switch NodeEnvironment's settings to new settings {pull}16273[#16273]
+* Simplify AutoCreateIndex and add more tests {pull}16270[#16270]
+* Convert several pending settings {pull}16269[#16269]
+* Migrate query caching settings to the new settings infra. {pull}16267[#16267]
+* Convert `action.auto_create_index` and `action.master.force_local` to the new settings infra {pull}16263[#16263]
+* Convert `cluster.routing.allocation.type` and `processors` to the new settings infra. {pull}16238[#16238]
+* Validate tribe node settings on startup {pull}16237[#16237]
+* Move node.client, node.data, node.master, node.local and node.mode to new settings infra {pull}16230[#16230]
+* Moved http settings to the new settings infrastructure {pull}16188[#16188]
+* Migrate network service to the new infra {pull}16187[#16187]
+* Convert client.transport settings to new infra {pull}16183[#16183]
+* Move discovery.* settings to new Setting infrastructure {pull}16182[#16182]
+* Change over to o.e.common.settings.Setting for http settings {pull}16181[#16181]
+* Convert "path.*" and "pidfile" to new settings infra {pull}16180[#16180]
+* Migrate repository settings to the new settings API {pull}16178[#16178]
+* Convert "indices.*" settings to new infra. {pull}16177[#16177]
+* Migrate gateway settings to the new settings API. {pull}16175[#16175]
+* Convert several node and test level settings {pull}16172[#16172]
+* Run Metadata upgrade tool on every version {pull}16168[#16168]
+* Check for invalid index settings on metadata upgrade {pull}16156[#16156]
+* Validate the settings key if it's simple chars separated by `.` {pull}16120[#16120]
+* Validate known global settings on startup {pull}16091[#16091]
+* Cut over all index scope settings to the new setting infrastrucuture {pull}16054[#16054] (issues: {issue}12790[#12790], {issue}12854[#12854], {issue}16032[#16032], {issue}6732[#6732])
+* Remove updatability of `index.flush_on_close` {pull}15964[#15964] (issue: {issue}15955[#15955])
+* Move all dynamic settings and their config classes to the index level {pull}15955[#15955] (issue: {issue}6732[#6732])
+* Always require units for bytes and time settings {pull}15948[#15948] (issue: {issue}11437[#11437])
+* Make MetaData parsing less lenient. {pull}15828[#15828]
+* Move async translog sync logic into IndexService {pull}15584[#15584]
+* Remove `index.merge.scheduler.notify_on_failure` and default to `true` {pull}15572[#15572] (issue: {issue}15570[#15570])
+* Remove cache concurrency level settings that no longer apply {pull}14210[#14210] (issues: {issue}13224[#13224], {issue}13717[#13717], {issue}7836[#7836])
+
+Similarities::
+* Defining a global default similarity {pull}16682[#16682] (issue: {issue}16594[#16594])
+
+Snapshot/Restore::
+* Delete differing files in the store before restoring {pull}20220[#20220] (issue: {issue}20148[#20148])
+* Adds ignoreUnavailable option to the snapshot status API {pull}20066[#20066] (issue: {issue}18522[#18522])
+* Check restores in progress before deleting a snapshot {pull}19853[#19853]
+* Snapshot repository cleans up empty index folders {pull}19751[#19751]
+* BlobContainer#writeBlob no longer can overwrite a blob {pull}19749[#19749] (issue: {issue}15579[#15579])
+* More resilient blob handling in snapshot repositories {pull}19706[#19706] (issues: {issue}18156[#18156], {issue}18815[#18815], {issue}19421[#19421], {issue}7540[#7540])
+* Adding repository index generational files {pull}19002[#19002] (issue: {issue}18156[#18156])
+* Raised IOException on deleteBlob {pull}18815[#18815] (issue: {issue}18530[#18530])
+* Adds UUIDs to snapshots {pull}18228[#18228] (issue: {issue}18156[#18156])
+* Clarify the semantics of the BlobContainer interface {pull}18157[#18157] (issue: {issue}15580[#15580])
+* Change BlobPath.buildAsString() method {pull}18461[#18461]
+* Remove the Snapshot class in favor of using SnapshotInfo {pull}18167[#18167] (issue: {issue}18156[#18156])
+
+Stats::
+* Add mem section back to cluster stats {pull}20255[#20255] (issue: {issue}17278[#17278])
+* Add network types to cluster stats {pull}20144[#20144]
+* Add missing field type in the FieldStats response. {pull}19241[#19241] (issue: {issue}17750[#17750])
+* Expose the ClusterInfo object in the allocation explain output {pull}19106[#19106] (issue: {issue}14405[#14405])
+* Add total_indexing_buffer/_in_bytes to nodes info API {pull}18914[#18914] (issue: {issue}18651[#18651])
+* Allow FieldStatsRequest to disable cache {pull}18900[#18900]
+* Remove index_writer_max_memory stat from segment stats {pull}18651[#18651] (issues: {issue}14121[#14121], {issue}7440[#7440])
+* Move DocStats under Engine to get more accurate numbers {pull}18587[#18587]
+* Do not return fieldstats information for fields that exist in the mapping but not in the index. {pull}18212[#18212] (issue: {issue}17980[#17980])
+* Add whether the shard state fetch is pending to the allocation explain API {pull}18119[#18119] (issue: {issue}17372[#17372])
+* Add Failure Details to every NodesResponse {pull}17964[#17964] (issue: {issue}3740[#3740])
+* Add I/O statistics on Linux {pull}15915[#15915] (issue: {issue}15296[#15296])
+* Add points to SegmentStats. {pull}17775[#17775] (issue: {issue}16974[#16974])
+* Remove FieldStats.Float. {pull}17749[#17749]
+* Show configured and remaining delay for an unassigned shard. {pull}17515[#17515] (issue: {issue}17372[#17372])
+* indexing stats now contain indexing ops from recovery  [ISSUE] {pull}17412[#17412]
+* Normalize unavailable load average {pull}16061[#16061] (issues: {issue}12049[#12049], {issue}14741[#14741], {issue}15907[#15907], {issue}15932[#15932], {issue}15934[#15934])
+* Add load averages to OS stats on FreeBSD {pull}15934[#15934] (issue: {issue}15917[#15917])
+* Expose pending cluster state queue size in node stats {pull}14040[#14040] (issue: {issue}13610[#13610])
+
+Store::
+* Use `mmapfs` by default. {pull}17616[#17616] (issue: {issue}16983[#16983])
+* Remove support for legacy checksums {pull}16931[#16931]
+* Rename index folder to index_uuid {pull}16442[#16442] (issues: {issue}13264[#13264], {issue}13265[#13265], {issue}14512[#14512], {issue}14932[#14932], {issue}15853[#15853])
+
+Suggesters::
+* Move SuggestUtils methods to their respective caller classes {pull}19914[#19914] (issue: {issue}19906[#19906])
+* Remove payload option from completion suggester {pull}19877[#19877] (issue: {issue}19536[#19536])
+* Add bwc support for reading  pre-5.0 completion index {pull}17602[#17602]
+
+Task Manager::
+* Rename Task Persistence into Storing Task Results {pull}19982[#19982]
+* Fetch result when wait_for_completion {pull}18905[#18905]
+* Create get task API that falls back to the .tasks index {pull}18682[#18682]
+* Add ability to store results for long running tasks {pull}17928[#17928]
+* Move parentTaskId into TransportRequest  {pull}17872[#17872]
+* Shorten the serialization of the empty TaskId {pull}17870[#17870]
+* Expose whether a task is cancellable in the _tasks list API {pull}17464[#17464] (issue: {issue}17369[#17369])
+* Add ability to group tasks by common parent {pull}17341[#17341]
+* Add start time and duration to tasks {pull}16829[#16829]
+* Combine node name and task id into single string task id {pull}16744[#16744]
+* Add task status {pull}16356[#16356] (issue: {issue}16344[#16344])
+* Extend tracking of parent tasks to master node, replication and broadcast actions {pull}15931[#15931]
+
+Translog::
+* Fsync documents in an async fashion {pull}20145[#20145]
+* Add checksumming and versions to the Translog's Checkpoint files {pull}19797[#19797]
+* Beef up Translog testing with random channel exceptions {pull}18997[#18997]
+* Do not replay into translog on local recovery {pull}18547[#18547]
+* FSync translog outside of the writers global lock {pull}18360[#18360]
+* Remove ChannelReference and simplify Views {pull}15898[#15898]
+* Simplify TranslogWriter to always write to a stream {pull}15771[#15771]
+* Remove TranslogService and fold it into synchronous IndexShard API {pull}13707[#13707]
+
+
+
+[[bug-5.0.0]]
+[float]
+=== Bug fixes
+
+Aggregations::
+* Fixed writeable name from range to geo_distance {pull}20860[#20860]
+* Fix date_range aggregation to not cache if now is used {pull}20740[#20740]
+* The `top_hits` aggregation should compile scripts only once. {pull}20738[#20738]
+* Fix agg profiling when using breadth_first collect mode {pull}20156[#20156]
+* Throw exception when maxBounds greater than minBounds {pull}19855[#19855] (issue: {issue}19833[#19833])
+* Undeprecates `aggs` in the search request {pull}19674[#19674] (issue: {issue}19504[#19504])
+* Change how `nested` and `reverse_nested` aggs know about their nested depth level {pull}19550[#19550] (issues: {issue}11749[#11749], {issue}12410[#12410])
+* Make ExtendedBounds immutable {pull}19490[#19490] (issue: {issue}19481[#19481])
+* Aggregations fix: support include/exclude strings for IP and dates {pull}18408[#18408] (issue: {issue}17705[#17705])
+* Fix xcontent rendering of ip terms aggs. {pull}18003[#18003] (issue: {issue}17971[#17971])
+* Improving parsing of sigma param for Extended Stats Bucket Aggregation {pull}17562[#17562] (issue: {issue}17499[#17499])
+* Fixes NPE when no window is specified in moving average request {pull}17556[#17556] (issue: {issue}17516[#17516])
+* Fixes Filter and FiltersAggregation to work with empty query {pull}17542[#17542] (issue: {issue}17518[#17518])
+* Fixes the defaults for `keyed` in the percentiles aggregations {pull}17217[#17217]
+* Correct typo in class name of StatsAggregator {pull}15264[#15264] (issue: {issue}14730[#14730])
+
+Allocation::
+* Keep a shadow replicas' allocation id when it is promoted to primary {pull}20863[#20863] (issue: {issue}20650[#20650])
+* IndicesClusterStateService should clean local started when re-assigns an initializing shard with the same aid {pull}20687[#20687]
+* IndexRoutingTable.initializeEmpty shouldn't override supplied primary RecoverySource {pull}20638[#20638] (issue: {issue}20637[#20637])
+* Update incoming recoveries stats when shadow replica is reinitialized {pull}20612[#20612]
+* `index.routing.allocation.initial_recovery` limits replica allocation {pull}20589[#20589]
+* Upon being elected as master, prefer joins' node info to existing cluster state {pull}19743[#19743]
+* Fix NPE when initializing replica shard has no UnassignedInfo {pull}19491[#19491] (issue: {issue}19488[#19488])
+* Make shard store fetch less dependent on the current cluster state, both on master and non data nodes {pull}19044[#19044] (issue: {issue}18938[#18938])
+* Fix recovery throttling to properly handle relocating non-primary shards {pull}18701[#18701] (issue: {issue}18640[#18640])
+* Replica shards must be failed before primary shards {pull}15686[#15686]
+
+Analysis::
+* Named analyzer should close the analyzer that it wraps {pull}20197[#20197]
+* Can load non-PreBuiltTokenFilter in Analyze API {pull}20396[#20396]
+* Fix analyzer alias processing {pull}19506[#19506] (issue: {issue}19163[#19163])
+
+Bulk::
+* Add not-null precondition check in BulkRequest {pull}18347[#18347] (issue: {issue}12038[#12038])
+
+CAT API::
+* Fixes cat tasks operation in detailed mode {pull}19759[#19759] (issue: {issue}19755[#19755])
+* Add index pattern wildcards support to _cat/shards {pull}19655[#19655] (issue: {issue}19634[#19634])
+
+CRUD::
+* GET operations should not extract fields from `_source`. {pull}20158[#20158] (issues: {issue}15017[#15017], {issue}20102[#20102])
+* Squash a race condition in RefreshListeners {pull}18806[#18806]
+* Prevent TransportReplicationAction to route request based on stale local routing table {pull}16274[#16274] (issues: {issue}12573[#12573], {issue}12574[#12574])
+* Resolves the conflict between alias routing and parent routing by applying the alias routing and ignoring the parent routing. {pull}15371[#15371] (issue: {issue}3068[#3068])
+
+Cache::
+* Prevent requests that use scripts or now() from being cached {pull}20750[#20750] (issue: {issue}20645[#20645])
+* Serialize index boost and phrase suggest collation keys in a consistent order {pull}20081[#20081] (issue: {issue}19986[#19986])
+
+Circuit Breakers::
+* Never trip circuit breaker in liveness request {pull}18627[#18627] (issue: {issue}17951[#17951])
+* Free bytes reserved on request breaker {pull}18204[#18204] (issue: {issue}18144[#18144])
+
+Cluster::
+* Fixes issue with dangling index being deleted instead of re-imported {pull}19666[#19666]
+* Allow routing table to be filtered by index pattern {pull}19688[#19688]
+* Use executor's describeTasks method to log task information in cluster service {pull}19531[#19531]
+* Acknowledge index deletion requests based on standard cluster state acknowledgment {pull}18602[#18602] (issues: {issue}16442[#16442], {issue}18558[#18558])
+* Dangling indices are not imported if a tombstone for the index exists {pull}18250[#18250] (issue: {issue}18249[#18249])
+* Fix issue with tombstones matching active indices in cluster state {pull}18058[#18058] (issue: {issue}18054[#18054])
+* Shard state action channel exceptions {pull}16057[#16057] (issue: {issue}15748[#15748])
+
+Core::
+* Makes `m` case sensitive in TimeValue {pull}19649[#19649] (issue: {issue}19619[#19619])
+* Guard against negative result from FileStore.getUsableSpace when picking data path for a new shard {pull}19554[#19554]
+* Handle rejected execution exception on reschedule {pull}19505[#19505]
+* Fix concurrency bug in IMC that could cause it to check too infrequently {pull}18357[#18357]
+* Iterables.flatten should not pre-cache the first iterator {pull}18355[#18355] (issue: {issue}18353[#18353])
+* Avoid race while retiring executors {pull}18333[#18333]
+* Refactor UUID-generating methods out of Strings {pull}17837[#17837] (issue: {issue}17819[#17819])
+* Node names cleanup {pull}17723[#17723] (issue: {issue}17718[#17718])
+* NullPointerException from IndexingMemoryController when a version conflict happens during recovery {pull}17569[#17569]
+* Handle RejectedExecution gracefully in TransportService during shutdown {pull}16965[#16965]
+
+Discovery::
+* Update discovery nodes after cluster state is published {pull}20409[#20409]
+* Add current cluster state version to zen pings and use them in master election {pull}20384[#20384] (issue: {issue}20348[#20348])
+
+Engine::
+* Take refresh IOExceptions into account when catching ACE in InternalEngine {pull}20546[#20546] (issue: {issue}19975[#19975])
+* Don't suppress AlreadyClosedException {pull}19975[#19975] (issue: {issue}19861[#19861])
+
+Expressions::
+* replace ScriptException with a better one {pull}18600[#18600]
+
+Geo::
+* Incomplete results when using geo_distance for large distances [ISSUE] {pull}17578[#17578]
+* Fix multi-field support for GeoPoint types {pull}15702[#15702] (issue: {issue}15701[#15701])
+* Enforce distance in distance query is > 0 [ISSUE] {pull}15135[#15135]
+
+Highlighting::
+* Enable BoostingQuery with FVH highlighter {pull}19984[#19984] (issue: {issue}19985[#19985])
+
+Index APIs::
+* Fixes active shard count check in the case of `all` shards {pull}19760[#19760]
+* Add zero-padding to auto-generated rollover index name increment {pull}19610[#19610] (issue: {issue}19484[#19484])
+
+Ingest::
+* no null values in ingest configuration error messages {pull}20616[#20616]
+* JSON Processor was not properly added {pull}20613[#20613]
+* Don't rebuild pipeline on every cluster state update {pull}20189[#20189]
+* Add dotexpander processor {pull}20078[#20078]
+* Fix NPE when simulating a pipeline with no id {pull}19650[#19650]
+* Change foreach processor to use ingest metadata for array element {pull}19609[#19609] (issue: {issue}19592[#19592])
+* No other processors should be executed after on_failure is called {pull}19545[#19545]
+* rethrow script compilation exceptions into ingest configuration exceptions {pull}19318[#19318]
+* Rename from `ingest-useragent` plugin to `ingest-user-agent` and its processor from `useragent` to `user_agent` {pull}19261[#19261]
+* Fix ignore_failure behavior in _simulate?verbose and more cleanup {pull}18987[#18987]
+* Pipeline Stats: Fix concurrent modification exception {pull}18177[#18177] (issue: {issue}18126[#18126])
+* Validate properties values according to database type {pull}17940[#17940] (issue: {issue}17683[#17683])
+* Ingest does not close its factories {pull}17626[#17626] (issue: {issue}17625[#17625])
+* Handle regex parsing errors in Gsub and Grok Processors {pull}17260[#17260]
+* add on_failure exception metadata to ingest document for verbose simulate {pull}16562[#16562]
+* The IngestDocument copy constructor should make a deep copy {pull}16248[#16248] (issue: {issue}16246[#16246])
+
+Inner Hits::
+* Ensure that that InnerHitBuilder uses rewritten queries {pull}19360[#19360] (issue: {issue}19353[#19353])
+
+Internal::
+* Prevent AbstractArrays from release bytes more than once {pull}20819[#20819]
+* IndicesAliasesRequest should not implement CompositeIndicesRequest {pull}20726[#20726]
+* Ensure elasticsearch doesn't start with unuspported indices {pull}20514[#20514] (issue: {issue}20512[#20512])
+* Remove ListTasksResponse#setDiscoveryNodes() {pull}19773[#19773] (issue: {issue}19772[#19772])
+* Priority values should be unmodifiable {pull}19447[#19447]
+* Extract AbstractBytesReferenceTestCase {pull}19141[#19141]
+* Add XPointValues {pull}18011[#18011] (issue: {issue}18010[#18010])
+* Fix BulkItemResponse.Failure.toString {pull}17871[#17871]
+* Enable unmap hack for java 9 {pull}16986[#16986] (issue: {issue}1[#1])
+* Fix issues with failed cache loads {pull}14315[#14315]
+* Allow parser to move on the START_OBJECT token when parsing search source {pull}14145[#14145]
+* Ensure searcher is release if wrapping fails {pull}14107[#14107]
+* Avoid deadlocks in Cache#computeIfAbsent {pull}14091[#14091] (issue: {issue}14090[#14090])
+
+Java API::
+* fix IndexResponse#toString to print out shards info {pull}20562[#20562]
+* Add NamedWriteables from plugins to TransportClient {pull}19825[#19825] (issue: {issue}19764[#19764])
+* Fix potential NPE in SearchSourceBuilder {pull}16905[#16905] (issue: {issue}16902[#16902])
+
+Java REST Client::
+* Rest Client: add slash to log line when missing between host and uri {pull}19325[#19325] (issue: {issue}19314[#19314])
+* Rest Client: HostsSniffer to set http as default scheme {pull}19306[#19306]
+
+Logging::
+* Fix logger when you can not create an azure storage client {pull}20670[#20670] (issues: {issue}20633[#20633], {issue}20669[#20669])
+* Avoid unnecessary creation of prefix loggers {pull}20571[#20571] (issue: {issue}20570[#20570])
+* Fix logging hierarchy configs {pull}20463[#20463]
+* Fix prefix logging {pull}20429[#20429]
+* Hack around Log4j bug rendering exceptions {pull}20306[#20306] (issue: {issue}20304[#20304])
+* Avoid prematurely triggering logger initialization {pull}20170[#20170]
+* Only log running out of slots when out of slots {pull}19637[#19637]
+
+Mapping::
+* Allow position_gap_increment for fields in indices created prior to 5.0 {pull}20806[#20806] (issues: {issue}19510[#19510], {issue}20413[#20413])
+* Validate blank field name {pull}19860[#19860] (issue: {issue}19251[#19251])
+* Better error message when mapping configures null {pull}18809[#18809] (issue: {issue}18803[#18803])
+* Make doc_values accessible for _type {pull}18220[#18220]
+* Fix and test handling of `null_value`. {pull}18090[#18090] (issue: {issue}18085[#18085])
+* Fail automatic string upgrade if the value of `index` is not recognized. {pull}18082[#18082] (issue: {issue}18062[#18062])
+* Fix dynamic check to properly handle parents {pull}17864[#17864] (issues: {issue}17644[#17644], {issue}17854[#17854])
+* Fix array parsing to remove its context when finished parsing {pull}17768[#17768]
+* Disallow fielddata loading on text fields that are not indexed. {pull}17747[#17747]
+* Make dynamic template parsing less lenient. {pull}17249[#17249]
+* Fix dynamic mapper when its parent already has an update {pull}17065[#17065]
+* Fix copy_to when the target is a dynamic object field. {pull}15216[#15216] (issues: {issue}111237[#111237], {issue}11237[#11237])
+* Preserve existing mappings on batch mapping updates {pull}15130[#15130] (issues: {issue}14899[#14899], {issue}15129[#15129])
+
+Network::
+* Fix connection close header handling {pull}20956[#20956] (issue: {issue}20938[#20938])
+* Ensure port range is readable in the exception message {pull}20893[#20893]
+* Fix expect 100 continue header handling {pull}19904[#19904] (issue: {issue}19834[#19834])
+* Fixes netty4 module's CORS config to use defaults  {pull}19874[#19874]
+* Fix various concurrency issues in transport {pull}19675[#19675]
+* Verify lower level transport exceptions don't bubble up on disconnects {pull}19518[#19518] (issue: {issue}19096[#19096])
+
+Packaging::
+* [Packaging] Do not remove scripts directory on upgrade {pull}20452[#20452]
+* [Package] Remove bin/lib/modules directories on RPM uninstall/upgrade {pull}20448[#20448]
+* Fix handling of spaces for jvm.options on Windows {pull}19951[#19951] (issue: {issue}19941[#19941])
+* Disable service in pre-uninstall {pull}19328[#19328]
+* Remove extra bin/ directory in bin folder {pull}18630[#18630]
+* Filter client/server VM options from jvm.options {pull}18473[#18473]
+* Preserve config files from RPM install {pull}18188[#18188] (issue: {issue}18158[#18158])
+* Fix typo in message for variable setup ES_MAX_MEM {pull}18168[#18168]
+* Don't run `mkdir` when $DATA_DIR contains a comma-separated list {pull}17419[#17419] (issue: {issue}16992[#16992])
+* Fix exit code {pull}17082[#17082]
+* Set MAX_OPEN_FILES to 65536 {pull}17431[#17431] (issue: {issue}17430[#17430])
+* [windows] Service command still had positional start command {pull}17391[#17391]
+* Do not pass double-dash arguments on startup {pull}17087[#17087] (issue: {issue}17084[#17084])
+
+Parent/Child::
+*  Make sure that no `_parent#null` gets introduces as default _parent mapping {pull}19470[#19470] (issue: {issue}19389[#19389])
+
+Percolator::
+* Fail indexing percolator queries containing either a has_child or has_parent query {pull}20229[#20229] (issue: {issue}2960[#2960])
+* Add support for MatchNoDocsQuery in percolator's query terms extract service {pull}18492[#18492]
+* Let PercolatorQuery's explain use the two phase iterator {pull}17315[#17315] (issue: {issue}17314[#17314])
+
+Plugin Discovery Azure Classic::
+* Make discovery-azure plugin work again {pull}19062[#19062] (issues: {issue}15630[#15630], {issue}18637[#18637])
+
+Plugin Discovery EC2::
+* Fix EC2 discovery settings {pull}18690[#18690] (issues: {issue}18652[#18652], {issue}18662[#18662])
+* Add TAG_SETTING to list of allowed tags for the ec2 discovery plugin. {pull}18257[#18257]
+* Fix EC2 Discovery settings {pull}17651[#17651] (issue: {issue}16602[#16602])
+
+Plugin Discovery GCE::
+* Fix NPE when GCE region is empty {pull}19176[#19176] (issue: {issue}16967[#16967])
+
+Plugin Ingest Attachment::
+* Adds content-length as number {pull}19927[#19927] (issue: {issue}19924[#19924])
+
+Plugin Ingest GeoIp::
+* [ingest-geoip] update geoip to not include null-valued results from  {pull}20455[#20455]
+
+Plugin Lang Painless::
+* Fix String Concatenation Bug In Painless {pull}20623[#20623]
+* Fix break bug in for/foreach loops. {pull}20146[#20146]
+* Fix compound assignment with string concats {pull}18933[#18933] (issue: {issue}18929[#18929])
+* Fix horrible capture {pull}18907[#18907] (issue: {issue}18899[#18899])
+* Fix Casting Bug {pull}18871[#18871]
+* Remove Grammar Ambiguities {pull}18531[#18531]
+* Remove if/else ANTLR ambiguity. {pull}18428[#18428]
+* Fix insanely slow compilation {pull}18410[#18410] (issue: {issue}18398[#18398])
+* Fix Bug in Painless Assignment {pull}18379[#18379]
+* Fix bracket shortcuts {pull}18263[#18263]
+
+Plugin Repository Azure::
+* Register group setting for repository-azure accounts {pull}19086[#19086]
+* Fix azure files removal {pull}18451[#18451] (issues: {issue}16472[#16472], {issue}18436[#18436])
+
+Plugin Repository S3::
+* Fixes leading forward slash in S3 repository base_path {pull}20861[#20861]
+* Add missing permission to repository-s3 {pull}19128[#19128] (issues: {issue}18539[#18539], {issue}19121[#19121])
+* Fix repository S3 Settings and add more tests {pull}18703[#18703] (issues: {issue}18662[#18662], {issue}18690[#18690])
+
+Plugin Store SMB::
+* Fix calling ensureOpen() on the wrong directory (master forwardport) {pull}16395[#16395] (issue: {issue}16383[#16383])
+
+Plugins::
+* Use sysprop like with es.path.home to pass conf dir {pull}18870[#18870] (issue: {issue}18689[#18689])
+* Quote path to java binary {pull}17496[#17496] (issue: {issue}17495[#17495])
+* CliTool: Messages printed in Terminal should have percent char escaped {pull}16367[#16367]
+
+Query DSL::
+* Fixes MultiMatchQuery so that it doesn't provide a null context {pull}20882[#20882]
+* Fix silently accepting malformed queries {pull}20515[#20515] (issue: {issue}20500[#20500])
+* query_string_query should take term length into consideration when fuzziness is auto {pull}20299[#20299] (issue: {issue}15972[#15972])
+* Throw ParsingException if a query is wrapped in an array {pull}19750[#19750] (issue: {issue}12887[#12887])
+* Restore parameter name auto_generate_phrase_queries {pull}19514[#19514] (issue: {issue}19512[#19512])
+* Resolve string dates and date math to millis before evaluating for rewrite in range query {pull}17239[#17239]
+* `constant_score` query should throw error on more than one filter {pull}17135[#17135] (issue: {issue}17126[#17126])
+* Single IPv4 addresses in IP field term queries {pull}16068[#16068] (issue: {issue}16058[#16058])
+* Make strategy optional in GeoShapeQueryBuilder readFrom and writeTo {pull}13963[#13963]
+
+Query Refactoring::
+* Query refactoring: set has_parent & has_child types context properly {pull}13863[#13863]
+* Make sure equivalent geohashCellQueries are equal after toQuery called {pull}13792[#13792]
+
+REST::
+* Remove lenient URL parameter parsing {pull}20722[#20722] (issue: {issue}14719[#14719])
+* Fixes CORS handling so that it uses the defaults {pull}19522[#19522] (issue: {issue}19520[#19520])
+* Get XContent params from request in Nodes rest actions {pull}18860[#18860] (issue: {issue}18794[#18794])
+* Fixes reading of CORS pre-flight headers and methods {pull}17523[#17523] (issue: {issue}17483[#17483])
+
+Recovery::
+* Fix concurrency issues between cancelling a relocation and marking shard as relocated {pull}20443[#20443]
+* Move `reset recovery` into RecoveriesCollection {pull}19466[#19466] (issue: {issue}19473[#19473])
+* Fix replica-primary inconsistencies when indexing during primary relocation with ongoing replica recoveries {pull}19287[#19287] (issue: {issue}19248[#19248])
+* Invoke `IndexingOperationListeners` also when recovering from store or remote {pull}17406[#17406]
+* Prevent interruption while store checks lucene files for consistency {pull}16308[#16308]
+* Mark shard as recovering on the cluster state thread {pull}14276[#14276] (issues: {issue}13766[#13766], {issue}14115[#14115])
+
+Reindex API::
+* Fix reindex with transport client {pull}19997[#19997] (issues: {issue}19773[#19773], {issue}19979[#19979])
+* Fix a race condition in reindex's rethrottle {pull}18731[#18731] (issue: {issue}18744[#18744])
+* Reindex should never report negative throttled_until {pull}17799[#17799] (issue: {issue}17783[#17783])
+* Reindex should gracefully handle when _source is disabled {pull}17667[#17667] (issue: {issue}17666[#17666])
+
+Scripting::
+* Add support for booleans in scripts {pull}20950[#20950] (issue: {issue}20949[#20949])
+* Fix Javascript OOM build Failure {pull}20307[#20307]
+* Fix propagating the default value for script settings {pull}20183[#20183] (issue: {issue}20159[#20159])
+* Catch and wrap AssertionError and NoClassDefFoundError in groovy scripts {pull}19958[#19958] (issues: {issue}19806[#19806], {issue}19923[#19923])
+
+Search::
+* Do not cache script queries. {pull}20799[#20799] (issue: {issue}20763[#20763])
+* Throw error when trying to fetch fields from source and source is disabled {pull}20424[#20424] (issues: {issue}20093[#20093], {issue}20408[#20408])
+* Source filtering should keep working when the source contains numbers greater than `Long.MAX_VALUE`. {pull}20278[#20278] (issue: {issue}11508[#11508])
+* Fix NPE when running a range query on a `scaled_float` with no upper bound. {pull}20253[#20253]
+* Fix NPE during search with source filtering if the source is disabled. {pull}20093[#20093] (issue: {issue}7758[#7758])
+* Restore assignment of time value when deserializing a scroll instance {pull}19977[#19977] (issue: {issue}18820[#18820])
+* Fix explain output for dfs query {pull}19972[#19972] (issue: {issue}15369[#15369])
+* Don't recursively count children profile timings {pull}19397[#19397] (issue: {issue}18693[#18693])
+* fix explain in function_score if no function filter matches {pull}19185[#19185]
+* Fix NPEs due to disabled source {pull}18957[#18957]
+* Require timeout units when parsing query body {pull}19077[#19077] (issue: {issue}19075[#19075])
+* Close SearchContext if query rewrite failed {pull}18727[#18727]
+* Fix parsing single `rescore` element in SearchSourceBuilder {pull}18440[#18440] (issue: {issue}18439[#18439])
+* Fail queries on not indexed fields. {pull}18014[#18014]
+* Fix for search after {pull}16271[#16271]
+* Do not be lenient when parsing CIDRs {pull}14874[#14874] (issue: {issue}14862[#14862])
+
+Settings::
+* Fix Setting.timeValue() method {pull}20696[#20696] (issue: {issue}20662[#20662])
+* Add a hard limit for `index.number_of_shard` {pull}20682[#20682]
+* Include complex settings in settings requests {pull}20622[#20622]
+* Fix filter cache setting to allow percentages {pull}20335[#20335] (issue: {issue}20330[#20330])
+* Move cluster.routing.allocation.same_shard.host setting to new settings infrastructure {pull}20046[#20046] (issue: {issue}20045[#20045])
+* Validate settings against dynamic updaters on the master {pull}19088[#19088] (issue: {issue}19046[#19046])
+* Register "cloud.node.auto_attributes" setting in EC2 discovery plugin {pull}18678[#18678]
+* Use object equality to compare versions in IndexSettings {pull}18103[#18103]
+* fix exists method for list settings when using numbered setting format {pull}17949[#17949]
+* convert settings for ResourceWatcherService to new infrastructure {pull}17948[#17948]
+* Register bootstrap settings {pull}16513[#16513]
+* Add settings filtering to node info requests {pull}16445[#16445]
+* Ban write access to system properties {pull}14914[#14914]
+
+Snapshot/Restore::
+* Better handling of an empty shard's segments_N file {pull}18784[#18784] (issue: {issue}18707[#18707])
+* Fix race condition in snapshot initialization {pull}18426[#18426] (issue: {issue}18121[#18121])
+* Fix the semantics for the BlobContainer interface {pull}17878[#17878] (issues: {issue}15579[#15579], {issue}15580[#15580])
+
+Stats::
+* Fix FieldStats deserialization of `ip` field {pull}20522[#20522] (issue: {issue}20516[#20516])
+* Fix serialization bug in allocation explain API. {pull}19494[#19494]
+* Allocation explain: Also serialize `includeDiskInfo` field {pull}19492[#19492]
+* Add missing builder.endObject() in FsInfo {pull}18443[#18443] (issues: {issue}15915[#15915], {issue}18433[#18433])
+
+Store::
+* Tighten up concurrent store metadata listing and engine writes {pull}19684[#19684] (issue: {issue}19416[#19416])
+* Make static Store access shard lock aware {pull}19416[#19416] (issue: {issue}18938[#18938])
+* Catch assertion errors on commit and turn it into a real exception {pull}19357[#19357] (issue: {issue}19356[#19356])
+
+Task Manager::
+* Shard level tasks in Bulk Action lose reference to their parent tasks {pull}17743[#17743]
+* Take filterNodeIds into consideration while sending task requests to nodes {pull}17081[#17081]
+
+Term Vectors::
+* Fix calculation of took time of term vectors request {pull}17817[#17817] (issue: {issue}12565[#12565])
+
+Translog::
+* Fix RAM usage estimation of LiveVersionMap. {pull}20123[#20123] (issue: {issue}19787[#19787])
+* Fix translog replay multiple operations same doc {pull}18611[#18611] (issues: {issue}18547[#18547], {issue}18623[#18623])
+* Snapshotting and sync could cause a dead lock TranslogWriter {pull}18481[#18481] (issues: {issue}1[#1], {issue}18360[#18360], {issue}2[#2])
+* Move translog recover outside of the engine {pull}17422[#17422]
+* Mark shard active during recovery; push settings after engine finally inits {pull}16250[#16250] (issues: {issue}14121[#14121], {issue}16209[#16209])
+
+
+
+[[regression-5.0.0]]
+[float]
+=== Regressions
+
+Highlighting::
+* Handle SynonymQuery extraction for the FastVectorHighlighter {pull}20829[#20829] (issue: {issue}20781[#20781])
+
+
+[[upgrade-5.0.0]]
+[float]
+=== Upgrades
+
+Core::
+* Upgrade to Lucene 6.2.0 {pull}20147[#20147] (issue: {issue}20092[#20092])
+* Update to jackson 2.8.1 {pull}18939[#18939] (issue: {issue}18076[#18076])
+* Upgrade to Lucene 6.1.0. {pull}18926[#18926]
+* Upgrade to lucene-6.1.0-snapshot-3a57bea. {pull}18786[#18786]
+* Upgrade to Lucene 6.0.1. {pull}18648[#18648] (issues: {issue}17535[#17535], {issue}28[#28])
+* Upgrade to lucene 6 release {pull}17657[#17657]
+* Upgrade to lucene-6.0.0-f0aa4fc. {pull}17075[#17075]
+* upgrade to lucene 6.0.0-snapshot-bea235f {pull}16964[#16964]
+* Upgrade to Jackson 2.7.1 {pull}16801[#16801] (issue: {issue}16294[#16294])
+
+Ingest::
+* Update MaxMind geoip2 version to 2.6 {pull}16837[#16837] (issue: {issue}16801[#16801])
+
+Internal::
+* Bump master (3.0-snapshot) to java 8 {pull}13314[#13314]
+
+Network::
+* Upgrade to Netty 4.1.5 {pull}20222[#20222] (issue: {issue}19786[#19786])
+* Dependencies: Upgrade to netty 4.1.4 {pull}19689[#19689]
+* Introduce Netty 4 {pull}19526[#19526] (issue: {issue}3226[#3226])
+
+Packaging::
+* Upgrade JNA to 4.2.2 and remove optionality {pull}19045[#19045] (issue: {issue}13245[#13245])
+
+Plugin Discovery EC2::
+* Update aws sdk to 1.10.69 and add use_throttle_retries repository setting {pull}17784[#17784] (issues: {issue}538[#538], {issue}586[#586], {issue}589[#589])
+
+Scripting::
+* Dependencies: Updates to mustache 0.9.3 {pull}20337[#20337]
+
+Search Templates::
+* Update mustache.java to version 0.9.1 {pull}14053[#14053] (issue: {issue}13224[#13224])
+
+
diff --git a/docs/reference/release-notes/5.0.0-beta1.asciidoc b/docs/reference/release-notes/5.0.0-beta1.asciidoc
new file mode 100644
index 0000000000000..61dd0abcaf712
--- /dev/null
+++ b/docs/reference/release-notes/5.0.0-beta1.asciidoc
@@ -0,0 +1,413 @@
+[[release-notes-5.0.0-beta1]]
+== 5.0.0-beta1 Release Notes
+
+Also see <<breaking-changes-5.0>>.
+
+IMPORTANT: This is a beta release and is intended for _testing purposes only_. Indices created in this version will *not be compatible with Elasticsearch 5.0.0 GA*. Upgrading 5.0.0-beta1 to any other version is not supported.
+
+[[breaking-5.0.0-beta1]]
+[float]
+=== Breaking changes
+
+Analysis::
+* Remove `token_filter` in _analyze API {pull}20285[#20285] (issue: {issue}20283[#20283])
+* Removes support for adding aliases to analyzers {pull}19994[#19994] (issue: {issue}18244[#18244])
+
+CRUD::
+* Fixed naming inconsistency for fields/stored_fields in the APIs {pull}20166[#20166] (issues: {issue}18943[#18943], {issue}20155[#20155])
+* Disallow creating indices starting with '-' or '+' {pull}20033[#20033] (issue: {issue}19800[#19800])
+
+Core::
+* Remove minimum master nodes bootstrap check {pull}20082[#20082]
+
+Engine::
+* Optimize indexing for the autogenerated ID append-only case {pull}20211[#20211] (issue: {issue}19813[#19813])
+
+Geo::
+* Deprecate GeoDistance enums and remove geo distance script helpers {pull}19783[#19783]
+
+Logging::
+* Introduce Log4j 2 {pull}20235[#20235] (issues: {issue}16030[#16030], {issue}17697[#17697])
+
+Packaging::
+* Rename service.bat to elasticsearch-service.bat {pull}20496[#20496] (issue: {issue}17528[#17528])
+* Remove -D handling in args for windows plugin script {pull}20378[#20378] (issue: {issue}18207[#18207])
+* Set default min heap equal to default max heap {pull}20080[#20080] (issues: {issue}16334[#16334], {issue}17728[#17728], {issue}18311[#18311])
+
+Plugin Lang Painless::
+* Make Painless the Default Language {pull}20017[#20017] (issue: {issue}19960[#19960])
+
+REST::
+* Parameter improvements to Cluster Health API wait for shards {pull}20223[#20223] (issue: {issue}20216[#20216])
+* Switch indices.exists_type from `{index}/{type}` to `{index}/_mapping/{type}`. {pull}20055[#20055] (issue: {issue}15613[#15613])
+* Only use `PUT` for index creation, not POST. {pull}20001[#20001] (issues: {issue}15613[#15613], {issue}18160[#18160])
+
+Scripting::
+* Hardcode painless as the default scripting lang and add legacy script default for stored scripts {pull}20310[#20310] (issue: {issue}20122[#20122])
+
+Settings::
+* Default max local storage nodes to one {pull}19964[#19964] (issues: {issue}19679[#19679], {issue}19748[#19748])
+
+
+
+[[breaking-java-5.0.0-beta1]]
+[float]
+=== Breaking Java changes
+
+Core::
+* Remove ability to plug-in TransportService {pull}20505[#20505]
+
+Internal::
+* Remove TransportService#registerRequestHandler leniency {pull}20469[#20469] (issue: {issue}20468[#20468])
+* Consolidate search parser registries {pull}20000[#20000]
+* Move all FetchSubPhases to o.e.search.fetch.subphase {pull}19702[#19702]
+* Squash the rest of o.e.rest.action {pull}19698[#19698]
+
+Plugins::
+* Cleanup sub fetch phase extension point {pull}20382[#20382]
+* Remove IndexTemplateFilter {pull}20072[#20072]
+* Switch custom ShardsAllocators to pull based model {pull}20071[#20071]
+* Make custom allocation deciders use pull based extensions {pull}20040[#20040]
+
+Search::
+* Remove FetchSubPhaseParseElement {pull}20350[#20350]
+
+
+
+[[deprecation-5.0.0-beta1]]
+[float]
+=== Deprecations
+
+Geo::
+* Deprecate optimize_bbox on geodistance queries {pull}20032[#20032] (issue: {issue}20014[#20014])
+* Deprecate geohash parameters for geo_point parser {pull}20030[#20030] (issue: {issue}20009[#20009])
+
+Scripting::
+* Deprecate Groovy, Python, and Javascript {pull}20244[#20244] (issue: {issue}20129[#20129])
+
+
+
+[[feature-5.0.0-beta1]]
+[float]
+=== New features
+
+Aliases::
+* Add an alias action to delete an index {pull}20184[#20184] (issue: {issue}20064[#20064])
+
+Analysis::
+* Exposing lucene 6.x minhash filter. {pull}20206[#20206] (issue: {issue}20149[#20149])
+
+Geo::
+* Cut over geo_point field and queries to new LatLonPoint type {pull}20315[#20315] (issue: {issue}20314[#20314])
+
+Plugin Discovery File::
+* File-based discovery plugin {pull}20394[#20394] (issue: {issue}20323[#20323])
+
+Search::
+* Add the ability to disable the retrieval of the stored fields entirely {pull}20026[#20026]
+
+Suggesters::
+* Add support for returning documents with completion suggester {pull}19536[#19536] (issue: {issue}10746[#10746])
+
+
+
+[[enhancement-5.0.0-beta1]]
+[float]
+=== Enhancements
+
+Allocation::
+* Change DiskThresholdDecider's behavior when factoring in leaving shards {pull}20349[#20349]
+* Verify AllocationIDs in replication actions {pull}20320[#20320]
+* Mark shard as stale on non-replicated write, not on node shutdown {pull}20023[#20023] (issue: {issue}18919[#18919])
+* Add routing changes API to RoutingAllocation {pull}19992[#19992]
+* Primary shard allocator observes limits in forcing allocation {pull}19811[#19811] (issue: {issue}19446[#19446])
+* Use primary terms as authority to fail shards {pull}19715[#19715]
+* Add recovery source to ShardRouting {pull}19516[#19516]
+
+Analysis::
+* Stop using cached component in _analyze API {pull}19929[#19929] (issue: {issue}19827[#19827])
+
+Benchmark::
+* Add client-benchmark-noop-api-plugin to stress clients even more in benchmarks {pull}20103[#20103]
+
+CAT API::
+* Add health status parameter to cat indices API {pull}20393[#20393]
+
+Circuit Breakers::
+* Circuit break the number of inline scripts compiled per minute {pull}19694[#19694] (issue: {issue}19396[#19396])
+
+Cluster::
+* Add clusterUUID to RestMainAction output {pull}20503[#20503]
+
+Core::
+* Add serial collector bootstrap check {pull}20558[#20558]
+* Rename Netty TCP transports thread factories from http_* to transport_* {pull}20207[#20207]
+* Do not log full bootstrap checks exception {pull}19989[#19989]
+* Mark halting the virtual machine as privileged {pull}19923[#19923] (issues: {issue}19272[#19272], {issue}19806[#19806])
+
+Engine::
+* Only try to read new segments info if we really flushed the index {pull}20474[#20474]
+* Use _refresh instead of reading from Translog in the RT GET case {pull}20102[#20102]
+
+Exceptions::
+* Improve startup exception {pull}20083[#20083]
+
+Index Templates::
+* Add "version" field to Templates {pull}20353[#20353] (issue: {issue}20171[#20171])
+
+Ingest::
+* Add "version" field to Pipelines {pull}20343[#20343] (issue: {issue}20171[#20171])
+* Make it possible for Ingest Processors to access AnalysisRegistry {pull}20233[#20233]
+* add ignore_missing option to convert,trim,lowercase,uppercase,grok,rename {pull}20194[#20194] (issue: {issue}19995[#19995])
+* Add support for parameters to the script ingest processor {pull}20136[#20136]
+* introduce the JSON Processor {pull}20128[#20128] (issue: {issue}20052[#20052])
+* Allow rename processor to turn leaf fields into branch fields {pull}20053[#20053] (issue: {issue}19892[#19892])
+* remove ability to set field value in script-processor configuration {pull}19981[#19981]
+
+Internal::
+* Remove poor-mans compression in InternalSearchHit and friends {pull}20472[#20472]
+* Don't register SearchTransportService handlers more than once {pull}20468[#20468]
+* Unguice SearchModule {pull}20456[#20456]
+* Deguice SearchService and friends {pull}20423[#20423]
+* NodeStats classes to implement Writeable rather then Streamable {pull}20327[#20327]
+* More info classes to implement Writeable rather than Streamable {pull}20288[#20288] (issue: {issue}20255[#20255])
+* Internal: Split disk threshold monitoring from decider {pull}20018[#20018]
+* Switching LockObtainFailedException over to ShardLockObtainFailedException {pull}19991[#19991] (issue: {issue}19978[#19978])
+* update and delete by query requests to implement IndicesRequest.Replaceable {pull}19961[#19961]
+* VersionFetchSubPhase should not use Versions#loadDocIdAndVersion {pull}19944[#19944]
+
+Java API::
+* Ensure PutMappingRequest.buildFromSimplifiedDef input are pairs {pull}19837[#19837] (issue: {issue}19836[#19836])
+
+Java REST Client::
+* Add support for a RestClient path prefix {pull}20190[#20190]
+* Add "Async" to the end of each Async RestClient method {pull}20172[#20172] (issue: {issue}20168[#20168])
+* Allow RestClient to send array-based headers {pull}20151[#20151]
+
+Logging::
+* Ensure logging is initialized in CLI tools {pull}20575[#20575]
+* Give useful error message if log config is missing {pull}20493[#20493]
+* Complete Elasticsearch logger names {pull}20457[#20457] (issue: {issue}20326[#20326])
+* Add node name to decider trace logging  {pull}20437[#20437] (issue: {issue}20379[#20379])
+* Logging shutdown hack {pull}20389[#20389] (issue: {issue}20304[#20304])
+* Disable console logging {pull}20387[#20387]
+* Skip loading of jansi from log4j2 {pull}20334[#20334]
+* Configure AWS SDK logging configuration {pull}20313[#20313] (issue: {issue}20294[#20294])
+* Warn if unsupported logging configuration present {pull}20309[#20309]
+* Size limit deprecation logs {pull}20287[#20287] (issues: {issue}20235[#20235], {issue}20254[#20254])
+* Increase visibility of deprecation logger {pull}20254[#20254] (issue: {issue}11033[#11033])
+* Add shardId and node name in search slow log {pull}19949[#19949] (issue: {issue}19735[#19735])
+
+Mapping::
+* Automatically downgrade text and keyword to string on indexes imported from 2.x {pull}20177[#20177]
+* Do not parse numbers as both strings and numbers when not included in `_all`. {pull}20167[#20167]
+* Don't index the `_version` field {pull}20132[#20132]
+* The root object mapper should support updating `numeric_detection`, `date_detection` and `dynamic_date_formats`. {pull}20119[#20119] (issue: {issue}20111[#20111])
+* Automatically upgrade analyzed string fields that have `index_options` or `position_increment_gap` set. {pull}20002[#20002] (issue: {issue}19974[#19974])
+* Mappings: Support dots in field names in mapping parsing {pull}19899[#19899] (issue: {issue}19443[#19443])
+* Save one utf8 conversion in KeywordFieldMapper. {pull}19867[#19867]
+* Do not parse the created version from the settings every time a field is parsed. {pull}19824[#19824]
+
+Network::
+* Avoid early initializing Netty {pull}19819[#19819] (issue: {issue}5644[#5644])
+* Network: Allow to listen on virtual interfaces. {pull}19568[#19568] (issues: {issue}17473[#17473], {issue}19537[#19537])
+
+Packaging::
+* Add quiet option to disable console logging {pull}20422[#20422] (issues: {issue}15315[#15315], {issue}16159[#16159], {issue}17220[#17220])
+* Explicitly disable Netty key set replacement {pull}20249[#20249]
+
+Plugin Lang Painless::
+* Disable regexes by default in painless {pull}20427[#20427] (issue: {issue}20397[#20397])
+* Catch OutOfMemory and StackOverflow errors in Painless {pull}19936[#19936]
+
+Plugin Repository S3::
+* Extract AWS Key from KeyChain instead of using potential null value {pull}19560[#19560] (issues: {issue}18703[#18703], {issue}19557[#19557])
+
+Plugins::
+* Print message when removing plugin with config {pull}20338[#20338]
+* Plugins: Update official plugin location with unified release {pull}19996[#19996]
+* Allow plugins to upgrade global custom metadata on startup {pull}19962[#19962]
+* Switch aggregations from push to pull {pull}19839[#19839]
+* Display plugins versions {pull}18683[#18683]
+
+Query DSL::
+* Throw exception when multiple field names are provided as part of query short syntax {pull}19871[#19871] (issue: {issue}19791[#19791])
+* Query parsers to throw exception when multiple field names are provided {pull}19791[#19791] (issue: {issue}19547[#19547])
+
+REST::
+* Add exclusion support to response filtering {pull}19865[#19865]
+
+Recovery::
+* Pass on maxUnsafeAutoIdTimestamp on recovery / relocation {pull}20300[#20300]
+
+Scripting::
+* Parse script on storage instead of on retrieval {pull}20356[#20356]
+* Migrate elasticsearch native script examples to the main repo {pull}19334[#19334] (issue: {issue}14662[#14662])
+
+Settings::
+* Make `action.auto_create_index` setting a dynamic cluster setting {pull}20274[#20274] (issue: {issue}7513[#7513])
+* Removes space between # and the setting in elasticsearch.yml {pull}20094[#20094] (issue: {issue}20090[#20090])
+
+Snapshot/Restore::
+* Delete differing files in the store before restoring {pull}20220[#20220] (issue: {issue}20148[#20148])
+* Adds ignoreUnavailable option to the snapshot status API {pull}20066[#20066] (issue: {issue}18522[#18522])
+* Check restores in progress before deleting a snapshot {pull}19853[#19853]
+* Snapshot repository cleans up empty index folders {pull}19751[#19751]
+
+Stats::
+* Add mem section back to cluster stats {pull}20255[#20255] (issue: {issue}17278[#17278])
+* Add network types to cluster stats {pull}20144[#20144]
+
+Suggesters::
+* Move SuggestUtils methods to their respective caller classes {pull}19914[#19914] (issue: {issue}19906[#19906])
+* Remove payload option from completion suggester {pull}19877[#19877] (issue: {issue}19536[#19536])
+
+Task Manager::
+* Rename Task Persistence into Storing Task Results {pull}19982[#19982]
+
+Translog::
+* Fsync documents in an async fashion {pull}20145[#20145]
+* Add checksumming and versions to the Translog's Checkpoint files {pull}19797[#19797]
+
+
+
+[[bug-5.0.0-beta1]]
+[float]
+=== Bug fixes
+
+Aggregations::
+* Fix agg profiling when using breadth_first collect mode {pull}20156[#20156]
+* Throw exception when maxBounds greater than minBounds {pull}19855[#19855] (issue: {issue}19833[#19833])
+
+Allocation::
+* Upon being elected as master, prefer joins' node info to existing cluster state {pull}19743[#19743]
+
+Analysis::
+* Can load non-PreBuiltTokenFilter in Analyze API {pull}20396[#20396]
+
+CRUD::
+* GET operations should not extract fields from `_source`. {pull}20158[#20158] (issues: {issue}15017[#15017], {issue}20102[#20102])
+
+Cache::
+* Serialize index boost and phrase suggest collation keys in a consistent order {pull}20081[#20081] (issue: {issue}19986[#19986])
+
+Cluster::
+* Fixes issue with dangling index being deleted instead of re-imported {pull}19666[#19666]
+
+Discovery::
+* Update discovery nodes after cluster state is published {pull}20409[#20409]
+* Add current cluster state version to zen pings and use them in master election {pull}20384[#20384] (issue: {issue}20348[#20348])
+
+Engine::
+* Take refresh IOExceptions into account when catching ACE in InternalEngine {pull}20546[#20546] (issue: {issue}19975[#19975])
+* Don't suppress AlreadyClosedException {pull}19975[#19975] (issue: {issue}19861[#19861])
+
+Highlighting::
+* Fix highlighting of MultiTermQuery within a FunctionScoreQuery {pull}20400[#20400] (issue: {issue}20392[#20392])
+* Enable BoostingQuery with FVH highlighter {pull}19984[#19984] (issue: {issue}19985[#19985])
+
+Index APIs::
+* Fix IndexNotFoundException in multi index search request. {pull}20188[#20188] (issue: {issue}3839[#3839])
+
+Ingest::
+* Don't rebuild pipeline on every cluster state update {pull}20189[#20189]
+* Add dotexpander processor {pull}20078[#20078]
+
+Internal::
+* Ensure elasticsearch doesn't start with unuspported indices {pull}20514[#20514] (issue: {issue}20512[#20512])
+* Remove ListTasksResponse#setDiscoveryNodes() {pull}19773[#19773] (issue: {issue}19772[#19772])
+
+Java API::
+* fix IndexResponse#toString to print out shards info {pull}20562[#20562]
+* Add NamedWriteables from plugins to TransportClient {pull}19825[#19825] (issue: {issue}19764[#19764])
+
+Logging::
+* Avoid unnecessary creation of prefix loggers {pull}20571[#20571] (issue: {issue}20570[#20570])
+* Fix logging hierarchy configs {pull}20463[#20463]
+* Fix prefix logging {pull}20429[#20429]
+* Hack around Log4j bug rendering exceptions {pull}20306[#20306] (issue: {issue}20304[#20304])
+* Avoid prematurely triggering logger initialization {pull}20170[#20170]
+
+Mapping::
+* Mapping updates on objects should propagate `include_an_all`. {pull}20051[#20051]
+* Validate blank field name {pull}19860[#19860] (issue: {issue}19251[#19251])
+
+Network::
+* Fix expect 100 continue header handling {pull}19904[#19904] (issue: {issue}19834[#19834])
+* Fixes netty4 module's CORS config to use defaults  {pull}19874[#19874]
+* Fix various concurrency issues in transport {pull}19675[#19675]
+
+Packaging::
+* [Packaging] Do not remove scripts directory on upgrade {pull}20452[#20452]
+* [Package] Remove bin/lib/modules directories on RPM uninstall/upgrade {pull}20448[#20448]
+* RESTART_ON_UPGRADE incorrectly named ES_RESTART_ON_UPGRADE in sysconfig {pull}19976[#19976] (issue: {issue}19950[#19950])
+* Fix handling of spaces for jvm.options on Windows {pull}19951[#19951] (issue: {issue}19941[#19941])
+
+Percolator::
+* Fail indexing percolator queries containing either a has_child or has_parent query {pull}20229[#20229] (issue: {issue}2960[#2960])
+
+Plugin Ingest Attachment::
+* Adds content-length as number {pull}19927[#19927] (issue: {issue}19924[#19924])
+
+Plugin Ingest GeoIp::
+* [ingest-geoip] update geoip to not include null-valued results from  {pull}20455[#20455]
+
+Plugin Lang Painless::
+* Fix break bug in for/foreach loops. {pull}20146[#20146]
+
+Plugins::
+* Use sysprop like with es.path.home to pass conf dir {pull}18870[#18870] (issue: {issue}18689[#18689])
+
+Query DSL::
+* Fix silently accepting malformed queries {pull}20515[#20515] (issue: {issue}20500[#20500])
+* query_string_query should take term length into consideration when fuzziness is auto {pull}20299[#20299] (issue: {issue}15972[#15972])
+
+Recovery::
+* Fix concurrency issues between cancelling a relocation and marking shard as relocated {pull}20443[#20443]
+
+Reindex API::
+* Fix reindex with transport client {pull}19997[#19997] (issues: {issue}19773[#19773], {issue}19979[#19979])
+* Make reindex and lang-javascript compatible {pull}19933[#19933]
+
+Scripting::
+* Fix Javascript OOM build Failure {pull}20307[#20307]
+* Fix propagating the default value for script settings {pull}20183[#20183] (issue: {issue}20159[#20159])
+* Catch and wrap AssertionError and NoClassDefFoundError in groovy scripts {pull}19958[#19958] (issues: {issue}19806[#19806], {issue}19923[#19923])
+
+Search::
+* Throw error when trying to fetch fields from source and source is disabled {pull}20424[#20424] (issues: {issue}20093[#20093], {issue}20408[#20408])
+* Source filtering should keep working when the source contains numbers greater than `Long.MAX_VALUE`. {pull}20278[#20278] (issue: {issue}11508[#11508])
+* Fix NPE when running a range query on a `scaled_float` with no upper bound. {pull}20253[#20253]
+* Add ref-counting to SearchContext to prevent accessing already closed readers {pull}20095[#20095] (issue: {issue}20008[#20008])
+* Fix NPE during search with source filtering if the source is disabled. {pull}20093[#20093] (issue: {issue}7758[#7758])
+* Restore assignment of time value when deserializing a scroll instance {pull}19977[#19977] (issue: {issue}18820[#18820])
+* Fix explain output for dfs query {pull}19972[#19972] (issue: {issue}15369[#15369])
+
+Settings::
+* Validate max thread/merge settings {pull}20383[#20383] (issue: {issue}20380[#20380])
+* Fix filter cache setting to allow percentages {pull}20335[#20335] (issue: {issue}20330[#20330])
+* Move cluster.routing.allocation.same_shard.host setting to new settings infrastructure {pull}20046[#20046] (issue: {issue}20045[#20045])
+
+Stats::
+* Fix FieldStats deserialization of `ip` field {pull}20522[#20522] (issue: {issue}20516[#20516])
+
+Translog::
+* Fix RAM usage estimation of LiveVersionMap. {pull}20123[#20123] (issue: {issue}19787[#19787])
+
+
+
+[[upgrade-5.0.0-beta1]]
+[float]
+=== Upgrades
+
+Core::
+* Upgrade to Lucene 6.2.0 {pull}20147[#20147] (issue: {issue}20092[#20092])
+* Update to jackson 2.8.1 {pull}18939[#18939] (issue: {issue}18076[#18076])
+
+Network::
+* Upgrade to Netty 4.1.5 {pull}20222[#20222] (issue: {issue}19786[#19786])
+
+Scripting::
+* Dependencies: Updates to mustache 0.9.3 {pull}20337[#20337]
+
+
diff --git a/docs/reference/release-notes/5.0.0-rc1.asciidoc b/docs/reference/release-notes/5.0.0-rc1.asciidoc
new file mode 100644
index 0000000000000..5fcf7f3cbd674
--- /dev/null
+++ b/docs/reference/release-notes/5.0.0-rc1.asciidoc
@@ -0,0 +1,99 @@
+[[release-notes-5.0.0-rc1]]
+== 5.0.0-rc1 Release Notes
+
+Also see <<breaking-changes-5.0>>.
+
+IMPORTANT: This is a release candidate and is intended for _testing purposes only_ and should not be deployed in production.
+
+[[breaking-5.0.0-rc1]]
+[float]
+=== Breaking changes
+
+Core::
+* Remove ignore system bootstrap checks {pull}20511[#20511]
+
+Internal::
+* `_flush` should block by default {pull}20597[#20597] (issue: {issue}20569[#20569])
+
+Plugin Lang Painless::
+* Remove all date 'now' methods from Painless {pull}20766[#20766] (issue: {issue}20762[#20762])
+
+
+
+[[enhancement-5.0.0-rc1]]
+[float]
+=== Enhancements
+
+Cluster::
+* Skip shard management code when updating cluster state on client/tribe nodes {pull}20731[#20731]
+
+Core::
+* Add production warning for pre-release builds {pull}20674[#20674]
+
+Index APIs::
+* Add date-math support to `_rollover` {pull}20709[#20709]
+
+Plugins::
+* Provide error message when plugin id is missing {pull}20660[#20660]
+
+REST::
+* Add response params to REST params did you mean {pull}20753[#20753] (issues: {issue}20722[#20722], {issue}20747[#20747])
+* Add did you mean to strict REST params {pull}20747[#20747] (issue: {issue}20722[#20722])
+
+Tribe Node::
+* Fail fast when executing master level write operations via a tribe node {pull}15441[#15441] (issue: {issue}13290[#13290])
+
+
+
+[[bug-5.0.0-rc1]]
+[float]
+=== Bug fixes
+
+Aggregations::
+* Fix date_range aggregation to not cache if now is used {pull}20740[#20740]
+* The `top_hits` aggregation should compile scripts only once. {pull}20738[#20738]
+
+Allocation::
+* IndicesClusterStateService should clean local started when re-assigns an initializing shard with the same aid {pull}20687[#20687]
+* IndexRoutingTable.initializeEmpty shouldn't override supplied primary RecoverySource {pull}20638[#20638] (issue: {issue}20637[#20637])
+* Update incoming recoveries stats when shadow replica is reinitialized {pull}20612[#20612]
+* `index.routing.allocation.initial_recovery` limits replica allocation {pull}20589[#20589]
+
+Analysis::
+* Named analyzer should close the analyzer that it wraps {pull}20197[#20197]
+
+Cache::
+* Prevent requests that use scripts or now() from being cached {pull}20750[#20750] (issue: {issue}20645[#20645])
+
+Core::
+* StoreStatsCache should also ignore AccessDeniedException when checking file size {pull}20790[#20790] (issue: {issue}17580[#17580])
+
+Ingest::
+* no null values in ingest configuration error messages {pull}20616[#20616]
+* JSON Processor was not properly added {pull}20613[#20613]
+
+Internal::
+* IndicesAliasesRequest should not implement CompositeIndicesRequest {pull}20726[#20726]
+
+Logging::
+* Fix logger when you can not create an azure storage client {pull}20670[#20670] (issues: {issue}20633[#20633], {issue}20669[#20669])
+
+Plugin Lang Painless::
+* Fix String Concatenation Bug In Painless {pull}20623[#20623]
+
+REST::
+* Change separator for shards preference {pull}20786[#20786] (issues: {issue}20722[#20722], {issue}20769[#20769])
+* Remove lenient URL parameter parsing {pull}20722[#20722] (issue: {issue}14719[#14719])
+
+Scripting::
+* Native scripts should be created once per index, not per segment. {pull}20609[#20609]
+
+Search::
+* Make range queries round up upper bounds again. {pull}20582[#20582] (issues: {issue}20579[#20579], {issue}8889[#8889])
+
+Settings::
+* Fix Setting.timeValue() method {pull}20696[#20696] (issue: {issue}20662[#20662])
+* Add a hard limit for `index.number_of_shard` {pull}20682[#20682]
+* Include complex settings in settings requests {pull}20622[#20622]
+
+
diff --git a/docs/reference/release-notes/5.0.0.asciidoc b/docs/reference/release-notes/5.0.0.asciidoc
new file mode 100644
index 0000000000000..38195d5a18c22
--- /dev/null
+++ b/docs/reference/release-notes/5.0.0.asciidoc
@@ -0,0 +1,71 @@
+[[release-notes-5.0.0-GA]]
+== 5.0.0 GA Release Notes
+
+The list below cover changes between 5.0.0-rc1 and 5.0.0 GA only.
+
+Also see <<breaking-changes-5.0>>.
+
+[[enhancement-5.0.0-GA]]
+[float]
+=== Enhancements
+
+Circuit Breakers::
+* Cluster Settings Updates should not trigger circuit breakers. {pull}20827[#20827]
+
+Plugins::
+* Revert "Display plugins versions" {pull}20807[#20807] (issues: {issue}18683[#18683], {issue}20668[#20668])
+
+REST::
+* Add a REST spec for the create API {pull}20924[#20924]
+
+Settings::
+* Add precise logging on unknown or invalid settings {pull}20951[#20951] (issue: {issue}20946[#20946])
+
+
+
+[[bug-5.0.0-GA]]
+[float]
+=== Bug fixes
+
+Aggregations::
+* Fixed writeable name from range to geo_distance {pull}20860[#20860]
+
+Allocation::
+* Keep a shadow replicas' allocation id when it is promoted to primary {pull}20863[#20863] (issue: {issue}20650[#20650])
+
+Index APIs::
+* Only negate index expression on all indices with preceding wildcard {pull}20898[#20898] (issues: {issue}19800[#19800], {issue}20033[#20033])
+
+Internal::
+* Prevent AbstractArrays from release bytes more than once {pull}20819[#20819]
+
+Mapping::
+* Allow position_gap_increment for fields in indices created prior to 5.0 {pull}20806[#20806] (issues: {issue}19510[#19510], {issue}20413[#20413])
+
+Network::
+* Fix connection close header handling {pull}20956[#20956] (issue: {issue}20938[#20938])
+* Ensure port range is readable in the exception message {pull}20893[#20893]
+
+Plugin Repository S3::
+* Fixes leading forward slash in S3 repository base_path {pull}20861[#20861]
+
+Query DSL::
+* Fixes MultiMatchQuery so that it doesn't provide a null context {pull}20882[#20882]
+* Fix match_phrase_prefix query with single term on _all field {pull}20471[#20471] (issue: {issue}20470[#20470])
+
+Scripting::
+* Add support for booleans in scripts {pull}20950[#20950] (issue: {issue}20949[#20949])
+
+Search::
+* Do not cache script queries. {pull}20799[#20799] (issue: {issue}20763[#20763])
+
+
+
+[[regression-5.0.0-GA]]
+[float]
+=== Regressions
+
+Highlighting::
+* Handle SynonymQuery extraction for the FastVectorHighlighter {pull}20829[#20829] (issue: {issue}20781[#20781])
+
+
diff --git a/docs/reference/release-notes/5.0.1.asciidoc b/docs/reference/release-notes/5.0.1.asciidoc
new file mode 100644
index 0000000000000..2eebe032cbf60
--- /dev/null
+++ b/docs/reference/release-notes/5.0.1.asciidoc
@@ -0,0 +1,124 @@
+[[release-notes-5.0.1]]
+== 5.0.1 Release Notes
+
+Also see <<breaking-changes-5.0>>.
+
+[[deprecation-5.0.1]]
+[float]
+=== Deprecations
+
+CRUD::
+* Deprecate VersionType.FORCE {pull}21078[#21078] (issue: {issue}20995[#20995])
+
+
+
+[[enhancement-5.0.1]]
+[float]
+=== Enhancements
+
+Aggregations::
+* Rescorer should be applied in the TopHits aggregation {pull}20978[#20978] (issue: {issue}19317[#19317])
+
+Internal::
+* ShardActiveResponseHandler shouldn't hold to an entire cluster state {pull}21470[#21470] (issue: {issue}21394[#21394])
+
+Network::
+* Fix handler name on message not fully read {pull}21478[#21478]
+
+Packaging::
+* Add empty plugins dir for archive distributions {pull}21204[#21204] (issue: {issue}20342[#20342])
+* Make explicit missing settings for Windows service {pull}21200[#21200] (issue: {issue}18317[#18317])
+* Change permissions on config files {pull}20966[#20966]
+
+Search::
+* Optimize query with types filter in the URL (t/t/_search) {pull}20979[#20979]
+
+
+
+[[bug-5.0.1]]
+[float]
+=== Bug fixes
+
+Aggregations::
+* Rewrite Queries/Filter in FilterAggregationBuilder and ensure client usage marks query as non-cachable {pull}21303[#21303] (issue: {issue}21301[#21301])
+* Thread safety for scripted significance heuristics {pull}21113[#21113] (issue: {issue}18120[#18120])
+* `ip_range` aggregation should accept null bounds. {pull}21043[#21043] (issue: {issue}21006[#21006])
+
+CAT API::
+* Consume `full_id` request parameter early {pull}21270[#21270] (issue: {issue}21266[#21266])
+
+Cache::
+* Fix the request cache keys to not hold references to the SearchContext. {pull}21284[#21284]
+
+Circuit Breakers::
+* ClusterState publishing shouldn't trigger circuit breakers {pull}20986[#20986] (issues: {issue}20827[#20827], {issue}20960[#20960])
+
+Core::
+* Fix ShardInfo#toString {pull}21319[#21319]
+* Protect BytesStreamOutput against overflows of the current number of written bytes. {pull}21174[#21174] (issue: {issue}21159[#21159])
+* Return target index name even if _rollover conditions are not met {pull}21138[#21138]
+
+Engine::
+* Retrying replication requests on replica doesn't call `onRetry` {pull}21189[#21189] (issue: {issue}20211[#20211])
+
+Index APIs::
+* Validate the `_rollover` target index name early to also fail if dry_run=true {pull}21330[#21330] (issue: {issue}21149[#21149])
+
+Ingest::
+* Stored scripts and ingest node configurations should be included into a snapshot {pull}21227[#21227] (issue: {issue}21184[#21184])
+
+Internal::
+* Restore thread's original context before returning to the ThreadPool {pull}21411[#21411]
+
+Java API::
+* Fix InternalSearchHit#hasSource to return the proper boolean value {pull}21441[#21441] (issue: {issue}21419[#21419])
+* Null checked for source when calling sourceRef {pull}21431[#21431] (issue: {issue}19279[#19279])
+* ClusterAdminClient.prepareDeletePipeline method should accept pipeline id to delete {pull}21228[#21228]
+
+Java REST Client::
+* Rest client: don't reuse the same HttpAsyncResponseConsumer across multiple retries {pull}21378[#21378]
+
+Logging::
+* Assert status logger does not warn on Log4j usage {pull}21339[#21339]
+* Fix logger names for Netty {pull}21223[#21223] (issue: {issue}20457[#20457])
+
+Packaging::
+* Set vm.max_map_count on systemd package install {pull}21507[#21507]
+* Export ES_JVM_OPTIONS for SysV init {pull}21445[#21445] (issue: {issue}21255[#21255])
+* Debian: configure start-stop-daemon to not go into background {pull}21343[#21343] (issues: {issue}12716[#12716], {issue}21300[#21300])
+* Generate POM files with non-wildcard excludes {pull}21234[#21234] (issue: {issue}21170[#21170])
+
+Query DSL::
+* Max score should be updated when a rescorer is used {pull}20977[#20977] (issue: {issue}20651[#20651])
+
+REST::
+* The routing query string param is supported by mget but was missing from the rest spec {pull}21357[#21357]
+* fix thread_pool_patterns path variable definition {pull}21332[#21332]
+* ensure the XContentBuilder is always closed in RestBuilderListener {pull}21124[#21124]
+
+Reindex API::
+* Bump reindex-from-remote's buffer to 200mb {pull}21222[#21222] (issue: {issue}21185[#21185])
+* Fix reindex-from-remote for parent/child from <2.0 {pull}21070[#21070] (issue: {issue}21044[#21044])
+
+Search::
+* Fixes cachability problems with fetching TTL values when searching {pull}21493[#21493] (issue: {issue}21457[#21457])
+* Remove LateParsingQuery to prevent timestamp access after context is frozen {pull}21328[#21328] (issue: {issue}21295[#21295])
+
+Snapshot/Restore::
+* Ensures cleanup of temporary index-* generational blobs during snapshotting {pull}21469[#21469] (issue: {issue}21462[#21462])
+* Fixes get snapshot duplicates when asking for _all {pull}21340[#21340] (issue: {issue}21335[#21335])
+
+
+
+[[upgrade-5.0.1]]
+[float]
+=== Upgrades
+
+Core::
+* Upgrade to Lucene 6.2.1 {pull}21207[#21207]
+
+Dates::
+* Update Joda Time to version 2.9.5 {pull}21468[#21468] (issue: {issue}20911[#20911])
+
+
+
diff --git a/docs/reference/release-notes/5.0.2.asciidoc b/docs/reference/release-notes/5.0.2.asciidoc
new file mode 100644
index 0000000000000..6a80783e158e0
--- /dev/null
+++ b/docs/reference/release-notes/5.0.2.asciidoc
@@ -0,0 +1,79 @@
+[[release-notes-5.0.2]]
+== 5.0.2 Release Notes
+
+Also see <<breaking-changes-5.0>>.
+
+[[enhancement-5.0.2]]
+[float]
+=== Enhancements
+
+Core::
+* Install a security manager on startup {pull}21716[#21716]
+
+Exceptions::
+* Add BWC layer for Exceptions {pull}21694[#21694] (issue: {issue}21656[#21656])
+
+Logging::
+* Truncate log messages from the end {pull}21609[#21609] (issue: {issue}21602[#21602])
+
+Scripting::
+* Wrap VerifyError in ScriptException {pull}21769[#21769]
+
+Snapshot/Restore::
+* Abort snapshots on a node that leaves the cluster {pull}21084[#21084] (issue: {issue}20876[#20876])
+
+
+
+[[bug-5.0.2]]
+[float]
+=== Bug fixes
+
+Allocation::
+* Allow master to assign primary shard to node that has shard store locked during shard state fetching {pull}21656[#21656] (issue: {issue}19416[#19416])
+
+Cluster::
+* Remove cluster update task when task times out {pull}21578[#21578] (issue: {issue}21568[#21568])
+
+Core::
+* Add a StreamInput#readArraySize method that ensures sane array sizes {pull}21697[#21697]
+* Use a buffer to do character to byte conversion in StreamOutput#writeString {pull}21680[#21680] (issue: {issue}21660[#21660])
+
+Engine::
+* Die with dignity on the Lucene layer {pull}21721[#21721] (issue: {issue}19272[#19272])
+* Fix `InternalEngine#isThrottled` to not always return `false`. {pull}21592[#21592]
+
+Index Templates::
+* Fix integer overflows when dealing with templates. {pull}21628[#21628] (issue: {issue}21622[#21622])
+
+Ingest::
+* fix trace_match behavior for when there is only one grok pattern {pull}21413[#21413] (issue: {issue}21371[#21371])
+
+Internal::
+* Rethrow ExecutionException from the loader to concurrent callers of Cache#computeIfAbsent {pull}21549[#21549]
+* Fixes potential NullPointerException on shard closing {pull}21515[#21515] (issue: {issue}21084[#21084])
+
+Java API::
+* Transport client: Fix remove address to actually work {pull}21743[#21743]
+* Add a HostFailureListener to notify client code if a node got disconnected {pull}21709[#21709] (issue: {issue}21424[#21424])
+
+Logging::
+* Do not prematurely shutdown Log4j {pull}21519[#21519] (issue: {issue}21514[#21514])
+
+Network::
+* Die with dignity on the network layer {pull}21720[#21720] (issue: {issue}19272[#19272])
+
+Plugin Lang Painless::
+* Fix a VerifyError bug in Painless {pull}21765[#21765]
+* Fix Lambdas in Painless to be Able to Use Top-Level Variables Such as params and doc {pull}21635[#21635] (issues: {issue}20869[#20869], {issue}21479[#21479])
+
+Search::
+* Respect default search timeout {pull}21599[#21599] (issues: {issue}12211[#12211], {issue}21595[#21595])
+
+Settings::
+* Don't reset non-dynamic settings unless explicitly requested {pull}21646[#21646] (issue: {issue}21593[#21593])
+
+Tribe Node::
+* Add socket permissions for tribe nodes {pull}21546[#21546] (issues: {issue}16392[#16392], {issue}21122[#21122])
+
+
+
diff --git a/docs/reference/release-notes/5.1.0.asciidoc b/docs/reference/release-notes/5.1.0.asciidoc
new file mode 100644
index 0000000000000..5abdf03b1a56a
--- /dev/null
+++ b/docs/reference/release-notes/5.1.0.asciidoc
@@ -0,0 +1,7 @@
+[[release-notes-5.1.0]]
+== 5.1.0 Release Notes
+
+Version 5.1.0 doesn't exist because, for a short period of time, the Elastic
+Yum and Apt repositories included unreleased binaries labeled 5.1.0. To avoid
+confusion and upgrade issues for the people that have installed these without
+realizing, we decided to skip the 5.1.0 version and release 5.1.1 instead.
diff --git a/docs/reference/release-notes/5.1.1.asciidoc b/docs/reference/release-notes/5.1.1.asciidoc
new file mode 100644
index 0000000000000..3ef9426efe61c
--- /dev/null
+++ b/docs/reference/release-notes/5.1.1.asciidoc
@@ -0,0 +1,388 @@
+[[release-notes-5.1.1]]
+== 5.1.1 Release Notes
+
+Also see <<breaking-changes-5.1>>.
+
+[[breaking-5.1.1]]
+[float]
+=== Breaking changes
+
+Aliases::
+* Validate alias names the same as index names {pull}20771[#20771] (issue: {issue}20748[#20748])
+
+REST::
+* Remove lenient stats parsing 5.x {pull}21576[#21576] (issues: {issue}20722[#20722], {issue}21410[#21410], {issue}21417[#21417])
+
+
+
+[[breaking-java-5.1.1]]
+[float]
+=== Breaking Java changes
+
+Discovery::
+* Remove pluggability of ElectMasterService {pull}21031[#21031]
+
+Exceptions::
+* Remove `IndexTemplateAlreadyExistsException` and `IndexShardAlreadyExistsException` {pull}21539[#21539] (issue: {issue}21494[#21494])
+* Replace IndexAlreadyExistsException with ResourceAlreadyExistsException {pull}21494[#21494]
+* Backport: Replace IndexAlreadyExistsException with ResourceAlreadyExistsException {pull}21601[#21601] (issue: {issue}21494[#21494])
+
+Internal::
+* Pass executor name to request interceptor to support async intercept calls {pull}21089[#21089]
+
+Network::
+* Unguice Transport and friends {pull}20526[#20526]
+
+Plugins::
+* Plugins: Remove support for onModule {pull}21416[#21416]
+
+
+
+[[deprecation-5.1.1]]
+[float]
+=== Deprecations
+
+Analysis::
+* Deprecating request parameters of _analyze API in 5.x {pull}20686[#20686] (issue: {issue}20246[#20246])
+
+CRUD::
+* Deprecate VersionType.FORCE {pull}21078[#21078] (issue: {issue}20995[#20995])
+
+Core::
+* Add deprecation logging for users that explicitly opt in for the `default` fs type. {pull}21617[#21617]
+
+Mapping::
+* Deprecate `timestamp` and `ttl` on index requests. {pull}21826[#21826] (issue: {issue}21670[#21670])
+
+Query DSL::
+* Add deprecation logging for lenient boolean queries {pull}21570[#21570] (issue: {issue}21555[#21555])
+* Add deprecation logging message for 'fuzzy' query {pull}20993[#20993] (issue: {issue}15760[#15760])
+
+Search::
+* Deprecate ignored type parameter in search_shards api {pull}21730[#21730] (issue: {issue}21688[#21688])
+
+Settings::
+* Add deprecation logging for the case that store throttling is used. {pull}21618[#21618]
+
+
+
+[[feature-5.1.1]]
+[float]
+=== New features
+
+Analysis::
+* Expose Lucenes Ukrainian analyzer {pull}21176[#21176] (issue: {issue}19433[#19433])
+
+CAT API::
+* Provides a cat api endpoint for templates. {pull}20545[#20545] (issue: {issue}20467[#20467])
+
+Query DSL::
+* Add "all fields" execution mode to simple_query_string query {pull}21341[#21341] (issues: {issue}19784[#19784], {issue}20925[#20925])
+* Add support for `quote_field_suffix` to `simple_query_string`. {pull}21060[#21060] (issue: {issue}18641[#18641])
+* Add "all field" execution mode to query_string query {pull}20925[#20925] (issue: {issue}19784[#19784])
+
+Reindex API::
+* Add automatic parallelization support to reindex and friends {pull}20767[#20767] (issue: {issue}20624[#20624])
+
+
+
+[[enhancement-5.1.1]]
+[float]
+=== Enhancements
+
+Aggregations::
+* Rescorer should be applied in the TopHits aggregation {pull}20978[#20978] (issue: {issue}19317[#19317])
+
+Allocation::
+* Balance step in BalancedShardsAllocator for a single shard {pull}21103[#21103]
+* Process more expensive allocation deciders last {pull}20724[#20724] (issue: {issue}12815[#12815])
+* Separates decision making from decision application in BalancedShardsAllocator  {pull}20634[#20634]
+* Split allocator decision making from decision application {pull}20431[#20431] (issue: {issue}20347[#20347])
+
+Analysis::
+* Remove AnalysisService and reduce it to a simple name to analyzer mapping {pull}20627[#20627] (issues: {issue}19827[#19827], {issue}19828[#19828])
+
+CAT API::
+* Adding built-in sorting capability to _cat apis. {pull}20658[#20658] (issue: {issue}16975[#16975])
+* Add health status parameter to cat indices API {pull}20393[#20393]
+
+Cache::
+* Do not cache term queries. {pull}21566[#21566] (issues: {issue}16031[#16031], {issue}20116[#20116])
+* Parse alias filters on the coordinating node {pull}20916[#20916]
+
+Cluster::
+* Cache successful shard deletion checks {pull}21438[#21438]
+
+Core::
+* Reduce memory pressure when sending large terms queries. {pull}21776[#21776]
+* Install a security manager on startup {pull}21716[#21716]
+* Log node ID on startup {pull}21673[#21673]
+* Ensure source filtering automatons are only compiled once {pull}20857[#20857] (issue: {issue}20839[#20839])
+* Improve scheduling fairness when batching cluster state changes with equal priority {pull}20775[#20775] (issue: {issue}20768[#20768])
+* Do not log full bootstrap checks exception {pull}19989[#19989]
+
+Exceptions::
+* Add BWC layer for Exceptions {pull}21694[#21694] (issue: {issue}21656[#21656])
+
+Geo::
+* Optimize geo-distance sorting. {pull}20596[#20596] (issue: {issue}20450[#20450])
+
+Ingest::
+* add `ignore_missing` option to SplitProcessor {pull}20982[#20982] (issues: {issue}19995[#19995], {issue}20840[#20840])
+
+Internal::
+* Rename ClusterState#lookupPrototypeSafe to `lookupPrototype` and remove "unsafe" unused variant {pull}21686[#21686]
+* ShardActiveResponseHandler shouldn't hold to an entire cluster state {pull}21470[#21470] (issue: {issue}21394[#21394])
+* Remove unused ClusterService dependency from SearchPhaseController {pull}21421[#21421]
+* Remove special case in case no action filters are registered {pull}21251[#21251]
+* Use TimveValue instead of long for CacheBuilder methods {pull}20887[#20887]
+* Remove SearchContext#current and all it's threadlocals {pull}20778[#20778] (issue: {issue}19341[#19341])
+
+Java REST Client::
+* Provide error message when rest request path is null {pull}21233[#21233] (issue: {issue}21232[#21232])
+
+Logging::
+* Log failure to connect to node at info instead of debug {pull}21809[#21809] (issue: {issue}6468[#6468])
+* Truncate log messages from the end {pull}21609[#21609] (issue: {issue}21602[#21602])
+* Logging shutdown hack {pull}20389[#20389] (issue: {issue}20304[#20304])
+* Disable console logging {pull}20387[#20387]
+
+Mapping::
+* Create the QueryShardContext lazily in DocumentMapperParser. {pull}21287[#21287]
+
+Network::
+* Grant Netty permission to read system somaxconn {pull}21840[#21840]
+* Lazy resolve unicast hosts {pull}21630[#21630] (issues: {issue}14441[#14441], {issue}16412[#16412])
+* Fix handler name on message not fully read {pull}21478[#21478]
+* Handle rejected pings on shutdown gracefully {pull}20842[#20842]
+
+Packaging::
+* Add empty plugins dir for archive distributions {pull}21204[#21204] (issue: {issue}20342[#20342])
+* Make explicit missing settings for Windows service {pull}21200[#21200] (issue: {issue}18317[#18317])
+* Change permissions on config files {pull}20966[#20966]
+
+Plugin Lang Painless::
+* Add Debug.explain to painless {pull}21723[#21723] (issue: {issue}20263[#20263])
+* Implement the ?: operator in painless {pull}21506[#21506]
+* In painless suggest a long constant if int won't do {pull}21415[#21415] (issue: {issue}21313[#21313])
+* Support decimal constants with trailing [dD] in painless {pull}21412[#21412] (issue: {issue}21116[#21116])
+* Implement reading from null safe dereferences {pull}21239[#21239]
+* Painless negative offsets {pull}21080[#21080] (issue: {issue}20870[#20870])
+
+Plugin Repository S3::
+* Make the default S3 buffer size depend on the available memory. {pull}21299[#21299]
+
+Plugins::
+* Clarify that plugins can be closed {pull}21669[#21669]
+* Plugins: Convert custom discovery to pull based plugin {pull}21398[#21398]
+* Removing plugin that isn't installed shouldn't trigger usage information {pull}21272[#21272] (issue: {issue}21250[#21250])
+* Remove pluggability of ZenPing {pull}21049[#21049]
+* Make UnicastHostsProvider extension pull based {pull}21036[#21036]
+
+Query DSL::
+* Using ObjectParser in MatchAllQueryBuilder and IdsQueryBuilder {pull}21273[#21273]
+* Expose splitOnWhitespace in `Query String Query` {pull}20965[#20965] (issue: {issue}20841[#20841])
+* Throw error if query element doesn't end with END_OBJECT {pull}20528[#20528] (issue: {issue}20515[#20515])
+* Remove `lowercase_expanded_terms` and `locale` from query-parser options. {pull}20208[#20208] (issue: {issue}9978[#9978])
+
+Reindex API::
+* Make reindex-from-remote ignore unknown fields {pull}20591[#20591] (issue: {issue}20504[#20504])
+
+Scripting::
+* Wrap VerifyError in ScriptException {pull}21769[#21769]
+* Support binary field type in script values {pull}21484[#21484] (issue: {issue}14469[#14469])
+* Mustache: Add {{#url}}{{/url}} function to URL encode strings {pull}20838[#20838]
+* Expose `ctx._now` in update scripts {pull}20835[#20835] (issue: {issue}17895[#17895])
+
+Search::
+* Add indices and filter information to search shards api output {pull}21738[#21738] (issue: {issue}20916[#20916])
+* remove pointless catch exception in TransportSearchAction {pull}21689[#21689]
+* Optimize query with types filter in the URL (t/t/_search) {pull}20979[#20979]
+* Makes search action cancelable by task management API {pull}20405[#20405]
+
+Search Templates::
+* Add profile and explain parameters to template API {pull}20451[#20451]
+
+Snapshot/Restore::
+* Abort snapshots on a node that leaves the cluster {pull}21084[#21084] (issue: {issue}20876[#20876])
+
+Stats::
+* Remove load average leniency {pull}21380[#21380]
+* Strengthen handling of unavailable cgroup stats {pull}21094[#21094] (issue: {issue}21029[#21029])
+* Add basic cgroup CPU metrics {pull}21029[#21029]
+
+Task Manager::
+* Add search task descriptions {pull}21740[#21740]
+
+Tribe Node::
+* Add support for merging custom meta data in tribe node {pull}21552[#21552] (issues: {issue}20544[#20544], {issue}20791[#20791], {issue}9372[#9372])
+
+
+
+[[bug-5.1.1]]
+[float]
+=== Bug fixes
+
+Aggregations::
+* Rewrite Queries/Filter in FilterAggregationBuilder and ensure client usage marks query as non-cachable {pull}21303[#21303] (issue: {issue}21301[#21301])
+* Percentiles bucket fails for 100th percentile {pull}21218[#21218]
+* Thread safety for scripted significance heuristics {pull}21113[#21113] (issue: {issue}18120[#18120])
+
+Allocation::
+* Allow master to assign primary shard to node that has shard store locked during shard state fetching {pull}21656[#21656] (issue: {issue}19416[#19416])
+
+Analysis::
+* Can load non-PreBuiltTokenFilter in Analyze API {pull}20396[#20396]
+
+CAT API::
+* Consume `full_id` request parameter early {pull}21270[#21270] (issue: {issue}21266[#21266])
+
+Cache::
+* Fix the request cache keys to not hold references to the SearchContext. {pull}21284[#21284]
+
+Circuit Breakers::
+* ClusterState publishing shouldn't trigger circuit breakers {pull}20986[#20986] (issues: {issue}20827[#20827], {issue}20960[#20960])
+
+Cluster::
+* Remove cluster update task when task times out {pull}21578[#21578] (issue: {issue}21568[#21568])
+
+Core::
+* Add a StreamInput#readArraySize method that ensures sane array sizes {pull}21697[#21697]
+* Use a buffer to do character to byte conversion in StreamOutput#writeString {pull}21680[#21680] (issue: {issue}21660[#21660])
+* Fix ShardInfo#toString {pull}21319[#21319]
+* Protect BytesStreamOutput against overflows of the current number of written bytes. {pull}21174[#21174] (issue: {issue}21159[#21159])
+* Return target index name even if _rollover conditions are not met {pull}21138[#21138]
+* .es_temp_file remains after system crash, causing it not to start again {pull}21007[#21007] (issue: {issue}20992[#20992])
+
+Dates::
+* Fix time zone rounding edge case for DST overlaps {pull}21550[#21550] (issue: {issue}20833[#20833])
+
+Engine::
+* Die with dignity on the Lucene layer {pull}21721[#21721] (issue: {issue}19272[#19272])
+* Fix `InternalEngine#isThrottled` to not always return `false`. {pull}21592[#21592]
+* Retrying replication requests on replica doesn't call `onRetry` {pull}21189[#21189] (issue: {issue}20211[#20211])
+
+Highlighting::
+* Fix FiltersFunctionScoreQuery highlighting {pull}21827[#21827]
+* Fix highlighting on a stored keyword field {pull}21645[#21645] (issue: {issue}21636[#21636])
+
+Index APIs::
+* Validate the `_rollover` target index name early to also fail if dry_run=true {pull}21330[#21330] (issue: {issue}21149[#21149])
+
+Index Templates::
+* Fix integer overflows when dealing with templates. {pull}21628[#21628] (issue: {issue}21622[#21622])
+
+Ingest::
+* fix trace_match behavior for when there is only one grok pattern {pull}21413[#21413] (issue: {issue}21371[#21371])
+* Stored scripts and ingest node configurations should be included into a snapshot {pull}21227[#21227] (issue: {issue}21184[#21184])
+
+Inner Hits::
+* Skip adding a parent field to nested documents. {pull}21522[#21522] (issue: {issue}21503[#21503])
+
+Internal::
+* Rethrow ExecutionException from the loader to concurrent callers of Cache#computeIfAbsent {pull}21549[#21549]
+* Restore thread's original context before returning to the ThreadPool {pull}21411[#21411]
+* Fix NPE in SearchContext.toString() {pull}21069[#21069]
+* Source filtering should treat dots in field names as sub objects. {pull}20736[#20736] (issue: {issue}20719[#20719])
+
+Java API::
+* Transport client: Fix remove address to actually work {pull}21743[#21743]
+* Add a HostFailureListener to notify client code if a node got disconnected {pull}21709[#21709] (issue: {issue}21424[#21424])
+* Fix InternalSearchHit#hasSource to return the proper boolean value {pull}21441[#21441] (issue: {issue}21419[#21419])
+* Null checked for source when calling sourceRef {pull}21431[#21431] (issue: {issue}19279[#19279])
+* ClusterAdminClient.prepareDeletePipeline method should accept pipeline id to delete {pull}21228[#21228]
+
+Java REST Client::
+* Rest client: don't reuse the same HttpAsyncResponseConsumer across multiple retries {pull}21378[#21378]
+
+Logging::
+* Do not prematurely shutdown Log4j {pull}21519[#21519] (issue: {issue}21514[#21514])
+* Assert status logger does not warn on Log4j usage {pull}21339[#21339]
+* Fix logger names for Netty {pull}21223[#21223] (issue: {issue}20457[#20457])
+
+Mapping::
+* Fail to index fields with dots in field names when one of the intermediate objects is nested. {pull}21787[#21787] (issue: {issue}21726[#21726])
+* Uncommitted mapping updates should not efect existing indices {pull}21306[#21306] (issue: {issue}21189[#21189])
+
+Network::
+* DiscoveryNode and TransportAddress should preserve host information {pull}21828[#21828]
+* Die with dignity on the network layer {pull}21720[#21720] (issue: {issue}19272[#19272])
+* Prevent double release in TcpTransport if send listener throws an exception {pull}20880[#20880]
+
+Packaging::
+* Set vm.max_map_count on systemd package install {pull}21507[#21507]
+* Export ES_JVM_OPTIONS for SysV init {pull}21445[#21445] (issue: {issue}21255[#21255])
+* Debian: configure start-stop-daemon to not go into background {pull}21343[#21343] (issues: {issue}12716[#12716], {issue}21300[#21300])
+* Generate POM files with non-wildcard excludes {pull}21234[#21234] (issue: {issue}21170[#21170])
+
+Plugin Lang Painless::
+* Test fix for def equals in Painless {pull}21945[#21945] (issue: {issue}21801[#21801])
+* Fix a VerifyError bug in Painless {pull}21765[#21765]
+* Fix Lambdas in Painless to be Able to Use Top-Level Variables Such as params and doc {pull}21635[#21635] (issues: {issue}20869[#20869], {issue}21479[#21479])
+
+Plugin Mapper Attachment::
+* NPE is raised when defining a non existing type within attachments type {pull}21848[#21848]
+
+Query DSL::
+* Fixes date range query using epoch with timezone {pull}21542[#21542] (issue: {issue}21501[#21501])
+* Allow overriding all-field leniency when `lenient` option is specified {pull}21504[#21504] (issues: {issue}20925[#20925], {issue}21341[#21341])
+* Max score should be updated when a rescorer is used {pull}20977[#20977] (issue: {issue}20651[#20651])
+
+REST::
+* Strict level parsing for indices stats {pull}21577[#21577] (issue: {issue}21024[#21024])
+* The routing query string param is supported by mget but was missing from the rest spec {pull}21357[#21357]
+* fix thread_pool_patterns path variable definition {pull}21332[#21332]
+* ensure the XContentBuilder is always closed in RestBuilderListener {pull}21124[#21124]
+* XContentBuilder: Avoid building self-referencing objects {pull}20550[#20550] (issues: {issue}19475[#19475], {issue}20540[#20540])
+
+Reindex API::
+* Ignore IllegalArgumentException with assertVersionSerializable {pull}21409[#21409] (issues: {issue}20767[#20767], {issue}21350[#21350])
+* Bump reindex-from-remote's buffer to 200mb {pull}21222[#21222] (issue: {issue}21185[#21185])
+* Fix reindex-from-remote for parent/child from <2.0 {pull}21070[#21070] (issue: {issue}21044[#21044])
+
+Search::
+* Fix match_phrase_prefix on boosted fields {pull}21623[#21623] (issue: {issue}21613[#21613])
+* Respect default search timeout {pull}21599[#21599] (issues: {issue}12211[#12211], {issue}21595[#21595])
+* Remove LateParsingQuery to prevent timestamp access after context is frozen {pull}21328[#21328] (issue: {issue}21295[#21295])
+
+Search Templates::
+* SearchTemplateRequest to implement CompositeIndicesRequest {pull}21865[#21865] (issue: {issue}21747[#21747])
+
+Settings::
+* Handle spaces in `action.auto_create_index` gracefully {pull}21790[#21790] (issue: {issue}21449[#21449])
+* Fix settings diff generation for affix and group settings {pull}21788[#21788]
+* Don't reset non-dynamic settings unless explicitly requested {pull}21646[#21646] (issue: {issue}21593[#21593])
+
+Snapshot/Restore::
+* Fixes shard level snapshot metadata loading when index-N file is missing {pull}21813[#21813]
+* Ensures cleanup of temporary index-* generational blobs during snapshotting {pull}21469[#21469] (issue: {issue}21462[#21462])
+* Fixes get snapshot duplicates when asking for _all {pull}21340[#21340] (issue: {issue}21335[#21335])
+* Keep snapshot restore state and routing table in sync (5.x backport) {pull}21131[#21131] (issue: {issue}20836[#20836])
+
+Task Manager::
+* Task cancellation command should wait for all child nodes to receive cancellation request before returning {pull}21397[#21397] (issue: {issue}21126[#21126])
+
+Tribe Node::
+* Add socket permissions for tribe nodes {pull}21546[#21546] (issues: {issue}16392[#16392], {issue}21122[#21122])
+
+
+
+[[upgrade-5.1.1]]
+[float]
+=== Upgrades
+
+Core::
+* Upgrade to lucene-6.3.0. {pull}21464[#21464]
+
+Dates::
+* Update Joda Time to version 2.9.5 {pull}21468[#21468] (issues: {issue}20911[#20911], {issue}332[#332], {issue}373[#373], {issue}378[#378], {issue}379[#379], {issue}386[#386], {issue}394[#394], {issue}396[#396], {issue}397[#397], {issue}404[#404], {issue}69[#69])
+
+Logging::
+* Upgrade Log4j 2 to version 2.7 {pull}20805[#20805] (issue: {issue}20304[#20304])
+
+Network::
+* Upgrade to Netty 4.1.6 {pull}21051[#21051]
+
+Plugin Ingest Attachment::
+* Update to Tika 1.14 {pull}21663[#21663] (issues: {issue}20710[#20710], {issue}21591[#21591])
+
diff --git a/docs/reference/release-notes/5.1.2.asciidoc b/docs/reference/release-notes/5.1.2.asciidoc
new file mode 100644
index 0000000000000..f7ebc3b6ed442
--- /dev/null
+++ b/docs/reference/release-notes/5.1.2.asciidoc
@@ -0,0 +1,108 @@
+[[release-notes-5.1.2]]
+== 5.1.2 Release Notes
+
+Also see <<breaking-changes-5.1>>.
+
+[[enhancement-5.1.2]]
+[float]
+=== Enhancements
+
+Internal::
+* Don't output null source node in RecoveryFailedException {pull}21963[#21963]
+
+Mapping::
+* Only update DocumentMapper if field type changes {pull}22165[#22165]
+
+Network::
+* Disable the Netty recycler {pull}22452[#22452] (issues: {issue}22189[#22189], {issue}22360[#22360], {issue}22406[#22406], {issue}5904[#5904])
+* Tell Netty not to be unsafe in transport client {pull}22284[#22284]
+
+Plugins::
+* Provide helpful error message if a plugin exists {pull}22305[#22305] (issue: {issue}22084[#22084])
+* Add shutdown hook for closing CLI commands {pull}22126[#22126] (issue: {issue}22111[#22111])
+
+
+
+[[bug-5.1.2]]
+[float]
+=== Bug fixes
+
+Aggregations::
+* Allow terms aggregations on pure boolean scripts. {pull}22201[#22201] (issue: {issue}20941[#20941])
+* IP range masks exclude the maximum address of the range. {pull}22018[#22018] (issue: {issue}22005[#22005])
+* Fix `other_bucket` on the `filters` agg to be enabled if a key is set. {pull}21994[#21994] (issue: {issue}21951[#21951])
+
+Allocation::
+* Promote shadow replica to primary when initializing primary fails {pull}22021[#22021]
+* Trim in-sync allocations set only when it grows {pull}21976[#21976] (issue: {issue}21719[#21719])
+
+Analysis::
+* Pre-built analysis factories do not implement MultiTermAware correctly. {pull}21981[#21981]
+
+CRUD::
+* Reject external versioning and explicit version numbers on create {pull}21998[#21998]
+
+Core::
+* Support negative numbers in readVLong {pull}22314[#22314]
+
+Exceptions::
+* Fixing shard recovery error message to report the number of docs correctly for each node {pull}22515[#22515] (issue: {issue}21893[#21893])
+
+Index APIs::
+* Ensure shrunk indices carry over version information from its source {pull}22469[#22469] (issue: {issue}22373[#22373])
+
+Java REST Client::
+* Don't use null charset in RequestLogger {pull}22197[#22197] (issue: {issue}22190[#22190])
+
+Mapping::
+* The `_all` default mapper is not completely configured. {pull}22236[#22236]
+* Fix MapperService.allEnabled(). {pull}22227[#22227]
+
+Nested Docs::
+* Fix bug in query builder rewrite that ignores the ignore_unmapped option {pull}22456[#22456]
+
+Packaging::
+* Another fix for handling of paths on Windows {pull}22132[#22132] (issue: {issue}21921[#21921])
+
+Percolator::
+* Fix NPE in percolator's 'now' range check for percolator queries with range queries {pull}22356[#22356] (issue: {issue}22355[#22355])
+
+Plugin Lang Painless::
+* Update Painless Loop Counter to be Higher {pull}22560[#22560] (issue: {issue}22508[#22508])
+
+Plugin Repository Azure::
+* readonly on azure repository must be taken into account {pull}22055[#22055] (issues: {issue}22007[#22007], {issue}22053[#22053])
+
+Recovery::
+* Fix primary relocation for shadow replicas {pull}22474[#22474] (issue: {issue}20300[#20300])
+* Don't close store under CancellableThreads {pull}22434[#22434] (issue: {issue}22325[#22325])
+* Use a fresh recovery id when retrying recoveries {pull}22325[#22325] (issue: {issue}22043[#22043])
+* Allow flush/force_merge/upgrade on shard marked as relocated {pull}22078[#22078] (issue: {issue}22043[#22043])
+
+Reindex API::
+* Don't close rest client from its callback {pull}22061[#22061] (issue: {issue}22027[#22027])
+
+Search::
+* Improve concurrency of ShardCoreKeyMap. {pull}22316[#22316]
+* Make `-0` compare less than `+0` consistently. {pull}22173[#22173] (issue: {issue}22167[#22167])
+* FiltersAggregationBuilder: rewriting filter queries, the same way as in FilterAggregationBuilder {pull}22076[#22076]
+
+Stats::
+* Implement stats for geo_point and geo_shape field {pull}22391[#22391] (issue: {issue}22384[#22384])
+* Use reader for doc stats {pull}22317[#22317] (issue: {issue}22285[#22285])
+* Avoid NPE in NodeService#stats if HTTP is disabled {pull}22060[#22060] (issue: {issue}22058[#22058])
+* Add support for "include_segment_file_sizes" in indices stats REST handler {pull}21879[#21879] (issue: {issue}21878[#21878])
+
+Term Vectors::
+* Fix _termvectors with preference to not hit NPE {pull}21959[#21959]
+* Return correct term statistics when a field is not found in a shard {pull}21922[#21922] (issue: {issue}21906[#21906])
+
+
+
+[[regression-5.1.2]]
+[float]
+=== Regressions
+
+Logging::
+* Restores the original default format of search slow log {pull}21770[#21770] (issue: {issue}21711[#21711])
+
diff --git a/docs/reference/release-notes/5.2.0.asciidoc b/docs/reference/release-notes/5.2.0.asciidoc
new file mode 100644
index 0000000000000..8cf1f089e5b35
--- /dev/null
+++ b/docs/reference/release-notes/5.2.0.asciidoc
@@ -0,0 +1,396 @@
+[[release-notes-5.2.0]]
+== 5.2.0 Release Notes
+
+Also see <<breaking-changes-5.2>>.
+
+[[breaking-5.2.0]]
+[float]
+=== Breaking changes
+
+Core::
+* Add system call filter bootstrap check {pull}21940[#21940]
+
+
+
+[[breaking-java-5.2.0]]
+[float]
+=== Breaking Java changes
+
+Allocation::
+* Cluster Explain API uses the allocation process to explain shard allocation decisions {pull}22182[#22182] (issues: {issue}20347[#20347], {issue}20634[#20634], {issue}21103[#21103], {issue}21662[#21662], {issue}21691[#21691])
+
+Cluster::
+* Remove PROTO-based custom cluster state components {pull}22336[#22336] (issue: {issue}21868[#21868])
+
+Ingest::
+* Change type of ingest doc meta-data field 'TIMESTAMP' to `Date` {pull}22234[#22234] (issue: {issue}22074[#22074])
+
+Internal::
+* Consolidate the last easy parser construction {pull}22095[#22095]
+* Introduce XContentParser#namedObject {pull}22003[#22003]
+
+Plugins::
+* Plugins: Replace Rest filters with RestHandler wrapper {pull}21905[#21905]
+
+Query DSL::
+* Resolve index names in indices_boost {pull}21393[#21393] (issue: {issue}4756[#4756])
+
+Search::
+* Cluster search shards improvements: expose ShardId, adjust visibility of some members {pull}21752[#21752]
+
+
+
+[[deprecation-5.2.0]]
+[float]
+=== Deprecations
+
+Java API::
+* Deprecate and remove "minimumNumberShouldMatch" in BoolQueryBuilder {pull}22403[#22403]
+
+Shadow Replicas::
+* Add a deprecation notice to shadow replicas  {pull}22647[#22647] (issue: {issue}22024[#22024])
+
+
+
+[[feature-5.2.0]]
+[float]
+=== New features
+
+Analysis::
+* Synonym Graph Support (LUCENE-6664) {pull}21517[#21517]
+
+Ingest::
+* introduce KV Processor in Ingest Node {pull}22272[#22272] (issue: {issue}22222[#22222])
+
+Mapping::
+* Add the ability to set a normalizer on keyword fields. {pull}21919[#21919] (issue: {issue}18064[#18064])
+* Add RangeFieldMapper for numeric and date range types {pull}21002[#21002] (issue: {issue}20999[#20999])
+
+
+
+[[enhancement-5.2.0]]
+[float]
+=== Enhancements
+
+Aggregations::
+* `value_type` is useful regardless of scripting. {pull}22160[#22160] (issue: {issue}20163[#20163])
+* Support for partitioning set of terms  {pull}21626[#21626] (issue: {issue}21487[#21487])
+
+Allocation::
+* Adds setting level to allocation decider explanations {pull}22268[#22268] (issue: {issue}21771[#21771])
+* Improves allocation decider decision explanation messages {pull}21771[#21771]
+* Prepares allocator decision objects for use with the allocation explain API {pull}21691[#21691]
+
+Analysis::
+* Expose FlattenGraphTokenFilter {pull}22643[#22643]
+
+CRUD::
+* Use correct block levels for TRA subclasses {pull}22224[#22224]
+
+Cluster::
+* Keep NodeConnectionsService in sync with current nodes in the cluster state {pull}22509[#22509]
+* Add a generic way of checking version before serializing custom cluster object {pull}22376[#22376] (issue: {issue}22313[#22313])
+* Add validation for supported index version on node join, restore, upgrade & open index {pull}21830[#21830] (issue: {issue}21670[#21670])
+* Let ClusterStateObserver only hold onto state that's needed for change detection {pull}21631[#21631] (issue: {issue}21568[#21568])
+* Remove mutable status field from cluster state {pull}21379[#21379]
+
+Core::
+* Rename bootstrap.seccomp to bootstrap.system_call_filter {pull}22226[#22226] (issue: {issue}21940[#21940])
+* Cleanup random stats serialization code {pull}22223[#22223]
+* Avoid corruption when deserializing booleans {pull}22152[#22152]
+* Reduce memory pressure when sending large terms queries. {pull}21776[#21776]
+
+Discovery::
+* Simplify Unicast Zen Ping {pull}22277[#22277] (issues: {issue}19370[#19370], {issue}21739[#21739], {issue}22120[#22120], {issue}22194[#22194])
+* Prefer joining node with conflicting transport address when becoming master {pull}22134[#22134] (issues: {issue}22049[#22049], {issue}22120[#22120])
+
+Ingest::
+* add `ignore_missing` flag to ingest plugins {pull}22273[#22273]
+* Added ability to remove pipelines via wildcards (#22149) {pull}22191[#22191] (issue: {issue}22149[#22149])
+* Enables the ability to inject serialized json fields into root of document {pull}22179[#22179] (issue: {issue}21898[#21898])
+* compile ScriptProcessor inline scripts when creating ingest pipelines {pull}21858[#21858] (issue: {issue}21842[#21842])
+
+Internal::
+* Remove some more usages of ParseFieldMatcher {pull}22398[#22398] (issues: {issue}19552[#19552], {issue}22130[#22130])
+* Remove some more usages of ParseFieldMatcher {pull}22395[#22395] (issues: {issue}19552[#19552], {issue}22130[#22130])
+* Remove some ParseFieldMatcher usages {pull}22389[#22389] (issues: {issue}19552[#19552], {issue}22130[#22130])
+* Add infrastructure to manage network connections outside of Transport/TransportService {pull}22194[#22194]
+* Replace strict parsing mode with response headers assertions {pull}22130[#22130] (issues: {issue}11859[#11859], {issue}19552[#19552], {issue}20993[#20993])
+* Start using `ObjectParser` for aggs. {pull}22048[#22048] (issue: {issue}22009[#22009])
+* Don't output null source node in RecoveryFailedException {pull}21963[#21963]
+* ClusterService should expose "applied" cluster states (i.e., remove ClusterStateStatus) {pull}21817[#21817]
+
+Java High Level REST Client::
+* Add fromxcontent methods to index response {pull}22229[#22229]
+* Add fromXContent() methods for ReplicationResponse {pull}22196[#22196] (issue: {issue}22082[#22082])
+* Add parsing method for ElasticsearchException {pull}22143[#22143]
+* Add fromXContent method to GetResponse {pull}22082[#22082]
+
+Java REST Client::
+* Warn log deprecation warnings received from server {pull}21895[#21895]
+
+Logging::
+* Log deleting indices at info level {pull}22627[#22627] (issue: {issue}22605[#22605])
+* Log failure to connect to node at info instead of debug {pull}21809[#21809] (issue: {issue}6468[#6468])
+
+Mapping::
+* Atomic mapping updates across types {pull}22220[#22220]
+* Only update DocumentMapper if field type changes {pull}22165[#22165]
+* Better error message when `_parent` isn't an object {pull}21987[#21987]
+
+Network::
+* Disable the Netty recycler {pull}22452[#22452] (issues: {issue}22189[#22189], {issue}22360[#22360], {issue}22406[#22406], {issue}5904[#5904])
+* Tell Netty not to be unsafe in transport client {pull}22284[#22284]
+* Introduce a low level protocol handshake {pull}22094[#22094]
+* Detach handshake from connect to node {pull}22037[#22037]
+* Reduce number of connections per node depending on the nodes role {pull}21849[#21849]
+* Add a connect timeout to the ConnectionProfile to allow per node connect timeouts {pull}21847[#21847] (issue: {issue}19719[#19719])
+* Grant Netty permission to read system somaxconn {pull}21840[#21840]
+* Remove connectToNodeLight and replace it with a connection profile {pull}21799[#21799]
+
+Packaging::
+* Improve the out-of-the-box experience {pull}21920[#21920] (issues: {issue}18317[#18317], {issue}21783[#21783])
+
+Plugin Discovery EC2::
+* Add support for ca-central-1 region to EC2 and S3 plugins {pull}22458[#22458] (issue: {issue}22454[#22454])
+* Support for eu-west-2 (London) cloud-aws plugin {pull}22308[#22308] (issue: {issue}22306[#22306])
+* Add us-east-2 AWS region {pull}21961[#21961] (issue: {issue}21881[#21881])
+* Add setting to set read timeout for EC2 discovery and S3 repository plugins {pull}21956[#21956] (issue: {issue}19078[#19078])
+
+Plugin Ingest GeoIp::
+* Cache results of geoip lookups {pull}22231[#22231] (issue: {issue}22074[#22074])
+
+Plugin Lang Painless::
+* Improve painless's ScriptException generation {pull}21762[#21762] (issue: {issue}21733[#21733])
+
+Plugins::
+* Pass ThreadContext to transport interceptors to allow header modification {pull}22618[#22618] (issue: {issue}22585[#22585])
+* Provide helpful error message if a plugin exists {pull}22305[#22305] (issue: {issue}22084[#22084])
+* Add shutdown hook for closing CLI commands {pull}22126[#22126] (issue: {issue}22111[#22111])
+* Allow plugins to install bootstrap checks {pull}22110[#22110]
+
+Query DSL::
+* Un-deprecate fuzzy query {pull}22088[#22088] (issue: {issue}15760[#15760])
+* support numeric bounds with decimal parts for long/integer/short/byte datatypes {pull}21972[#21972] (issue: {issue}21600[#21600])
+
+Reindex API::
+* Reindex: Better error message for pipeline in wrong place {pull}21985[#21985]
+* Timeout improvements for rest client and reindex {pull}21741[#21741] (issue: {issue}21707[#21707])
+
+Scripting::
+* Wrap VerifyError in ScriptException {pull}21769[#21769]
+* Log ScriptException's xcontent if file script compilation fails {pull}21767[#21767] (issue: {issue}21733[#21733])
+
+Search::
+* Move SearchTransportService and SearchPhaseController creation outside of TransportSearchAction constructor {pull}21754[#21754]
+* Don't carry ShardRouting around when not needed in AbstractSearchAsyncAction {pull}21753[#21753]
+* ShardSearchRequest to take ShardId constructor argument rather than the whole ShardRouting {pull}21750[#21750]
+* Use index uuid as key in the alias filter map rather than the index name {pull}21749[#21749]
+
+Settings::
+* Speed up filter and prefix settings operations {pull}22249[#22249]
+
+Snapshot/Restore::
+* Fixes retrieval of the latest snapshot index blob {pull}22700[#22700]
+* Synchronize snapshot deletions on the cluster state {pull}22313[#22313] (issue: {issue}19957[#19957])
+
+Stats::
+* Include unindexed field in FieldStats response {pull}21821[#21821] (issue: {issue}21952[#21952])
+
+Task Manager::
+* Improve the error message if task and node isn't found {pull}22062[#22062] (issue: {issue}22027[#22027])
+* Add descriptions to create snapshot and restore snapshot tasks. {pull}21901[#21901] (issue: {issue}21768[#21768])
+* Add proper descriptions to reindex, update-by-query and delete-by-query tasks. {pull}21841[#21841] (issue: {issue}21768[#21768])
+
+
+
+[[bug-5.2.0]]
+[float]
+=== Bug fixes
+
+Aggregations::
+* Fix scaled_float numeric type in aggregations {pull}22351[#22351] (issue: {issue}22350[#22350])
+* Allow terms aggregations on pure boolean scripts. {pull}22201[#22201] (issue: {issue}20941[#20941])
+* Fix numeric terms aggregations with includes/excludes and minDocCount=0 {pull}22141[#22141] (issue: {issue}22140[#22140])
+* Fix `missing` on aggs on `boolean` fields. {pull}22135[#22135] (issue: {issue}22009[#22009])
+* IP range masks exclude the maximum address of the range. {pull}22018[#22018] (issue: {issue}22005[#22005])
+* Fix `other_bucket` on the `filters` agg to be enabled if a key is set. {pull}21994[#21994] (issue: {issue}21951[#21951])
+
+Allocation::
+* Cannot force allocate primary to a node where the shard already exists {pull}22031[#22031] (issue: {issue}22021[#22021])
+* Promote shadow replica to primary when initializing primary fails {pull}22021[#22021]
+* Trim in-sync allocations set only when it grows {pull}21976[#21976] (issue: {issue}21719[#21719])
+
+Analysis::
+* AsciiFoldingFilter's multi-term component should never preserve the original token. {pull}21982[#21982]
+* Pre-built analysis factories do not implement MultiTermAware correctly. {pull}21981[#21981]
+
+CRUD::
+* Reject external versioning and explicit version numbers on create {pull}21998[#21998]
+
+Cluster::
+* IndicesService handles all exceptions during index deletion {pull}22433[#22433]
+
+Core::
+* Streamline foreign stored context restore and allow to perserve response headers {pull}22677[#22677] (issue: {issue}22647[#22647])
+* Support negative numbers in readVLong {pull}22314[#22314]
+
+Discovery::
+* Close InputStream when receiving cluster state in PublishClusterStateAction {pull}22711[#22711]
+* Do not reply to pings from another cluster {pull}21894[#21894] (issue: {issue}21874[#21874])
+
+Engine::
+* Close and flush refresh listeners on shard close {pull}22342[#22342]
+
+Exceptions::
+* Fixing shard recovery error message to report the number of docs correctly for each node {pull}22515[#22515] (issue: {issue}21893[#21893])
+
+Highlighting::
+* Fix FiltersFunctionScoreQuery highlighting {pull}21827[#21827]
+
+Index APIs::
+* Ensure shrunk indices carry over version information from its source {pull}22469[#22469] (issue: {issue}22373[#22373])
+
+Ingest::
+* fix index out of bounds error in KV Processor {pull}22288[#22288] (issue: {issue}22272[#22272])
+* Fixes GrokProcessor's ignorance of named-captures with same name. {pull}22131[#22131] (issue: {issue}22117[#22117])
+
+Inner Hits::
+* Inner hits and ignore unmapped {pull}21693[#21693] (issue: {issue}21620[#21620])
+
+Internal::
+* Index creation and setting update may not return deprecation logging {pull}22702[#22702]
+
+Java API::
+* Don't output empty ext object in SearchSourceBuilder#toXContent {pull}22093[#22093] (issue: {issue}20969[#20969])
+
+Java REST Client::
+* Don't use null charset in RequestLogger {pull}22197[#22197] (issue: {issue}22190[#22190])
+
+Mapping::
+* The `_all` default mapper is not completely configured. {pull}22236[#22236]
+* Fix MapperService.allEnabled(). {pull}22227[#22227]
+* Dynamic `date` fields should use the `format` that was used to detect it is a date. {pull}22174[#22174] (issue: {issue}9410[#9410])
+* Sub-fields should not accept `include_in_all` parameter {pull}21971[#21971] (issue: {issue}21710[#21710])
+* Mappings: Fix get mapping when no indexes exist to not fail in response generation {pull}21924[#21924] (issue: {issue}21916[#21916])
+
+Nested Docs::
+* Fix bug in query builder rewrite that ignores the ignore_unmapped option {pull}22456[#22456]
+
+Network::
+* Pass `forceExecution` flag to transport interceptor {pull}22739[#22739]
+* Ensure new connections won't be opened if transport is closed or closing {pull}22589[#22589] (issue: {issue}22554[#22554])
+* Prevent open channel leaks if handshake times out or is interrupted {pull}22554[#22554]
+* Execute low level handshake in #openConnection {pull}22440[#22440]
+* Properly configure Netty 3 ClientBootstrap when using custom connection profile {pull}22363[#22363] (issue: {issue}22277[#22277])
+* Handle connection close / reset events gracefully during handshake {pull}22178[#22178]
+* Do not lose host information when pinging {pull}21939[#21939] (issue: {issue}21828[#21828])
+* DiscoveryNode and TransportAddress should preserve host information {pull}21828[#21828]
+
+Packaging::
+* Another fix for handling of paths on Windows {pull}22132[#22132] (issue: {issue}21921[#21921])
+* Fix handling of spaces in Windows paths {pull}21921[#21921] (issues: {issue}20809[#20809], {issue}21525[#21525])
+* Add option to skip kernel parameters on install {pull}21899[#21899] (issue: {issue}21877[#21877])
+
+Percolator::
+* Fix NPE in percolator's 'now' range check for percolator queries with range queries {pull}22356[#22356] (issue: {issue}22355[#22355])
+
+Plugin Analysis Stempel::
+* Fix thread safety of Stempel's token filter factory {pull}22610[#22610] (issue: {issue}21911[#21911])
+
+Plugin Lang Painless::
+* Whitelist some ScriptDocValues in painless {pull}22600[#22600] (issue: {issue}22584[#22584])
+* Update Painless Loop Counter to be Higher {pull}22560[#22560] (issue: {issue}22508[#22508])
+* Fix some issues with painless's strings {pull}22393[#22393] (issue: {issue}22372[#22372])
+* Test fix for def equals in Painless {pull}21945[#21945] (issue: {issue}21801[#21801])
+* Fix a VerifyError bug in Painless {pull}21765[#21765]
+
+Plugin Mapper Attachment::
+* NPE is raised when defining a non existing type within attachments type {pull}21848[#21848]
+
+Plugin Repository Azure::
+* Fixes default chunk size for Azure repositories {pull}22577[#22577] (issue: {issue}22513[#22513])
+* readonly on azure repository must be taken into account {pull}22055[#22055] (issues: {issue}22007[#22007], {issue}22053[#22053])
+
+Query DSL::
+* Fix script score function that combines `_score` and weight {pull}22713[#22713] (issue: {issue}21483[#21483])
+
+REST::
+* Adds percent-encoding for Location headers {pull}21057[#21057] (issue: {issue}21016[#21016])
+
+Recovery::
+* Fix primary relocation for shadow replicas {pull}22474[#22474] (issue: {issue}20300[#20300])
+* Don't close store under CancellableThreads {pull}22434[#22434] (issue: {issue}22325[#22325])
+* Use a fresh recovery id when retrying recoveries {pull}22325[#22325] (issue: {issue}22043[#22043])
+* Allow flush/force_merge/upgrade on shard marked as relocated {pull}22078[#22078] (issue: {issue}22043[#22043])
+
+Reindex API::
+* Fix reindex from remote clearing scroll {pull}22525[#22525] (issue: {issue}22514[#22514])
+* Fix source filtering in reindex-from-remote {pull}22514[#22514] (issue: {issue}22507[#22507])
+* Remove content type detection from reindex-from-remote {pull}22504[#22504] (issue: {issue}22329[#22329])
+* Don't close rest client from its callback {pull}22061[#22061] (issue: {issue}22027[#22027])
+* Keep context during reindex's retries {pull}21941[#21941]
+
+Scripting::
+* Expose `ip` fields as strings in scripts. {pull}21997[#21997] (issue: {issue}21977[#21977])
+
+Search::
+* Improve concurrency of ShardCoreKeyMap. {pull}22316[#22316]
+* Make `-0` compare less than `+0` consistently. {pull}22173[#22173] (issue: {issue}22167[#22167])
+* Fix boost_mode propagation when the function score query builder is rewritten {pull}22172[#22172] (issue: {issue}22138[#22138])
+* FiltersAggregationBuilder: rewriting filter queries, the same way as in FilterAggregationBuilder {pull}22076[#22076]
+* Fix cross_fields type on multi_match query with synonyms {pull}21638[#21638] (issue: {issue}21633[#21633])
+
+Search Templates::
+* SearchTemplateRequest to implement CompositeIndicesRequest {pull}21865[#21865] (issue: {issue}21747[#21747])
+
+Settings::
+* Handle spaces in `action.auto_create_index` gracefully {pull}21790[#21790] (issue: {issue}21449[#21449])
+* Fix settings diff generation for affix and group settings {pull}21788[#21788]
+
+Snapshot/Restore::
+* Gracefully handles pre 2.x compressed snapshots {pull}22267[#22267]
+* URLRepository should throw NoSuchFileException to correctly adhere to readBlob contract {pull}22069[#22069] (issue: {issue}22004[#22004])
+* Fixes shard level snapshot metadata loading when index-N file is missing {pull}21813[#21813]
+
+Stats::
+* Implement stats for geo_point and geo_shape field {pull}22391[#22391] (issue: {issue}22384[#22384])
+* Use reader for doc stats {pull}22317[#22317] (issue: {issue}22285[#22285])
+* Avoid NPE in NodeService#stats if HTTP is disabled {pull}22060[#22060] (issue: {issue}22058[#22058])
+* Add support for "include_segment_file_sizes" in indices stats REST handler {pull}21879[#21879] (issue: {issue}21878[#21878])
+
+Term Vectors::
+* Fix `_termvectors` with preference to not hit NPE {pull}21959[#21959]
+* Return correct term statistics when a field is not found in a shard {pull}21922[#21922] (issue: {issue}21906[#21906])
+
+
+
+[[regression-5.2.0]]
+[float]
+=== Regressions
+
+Core::
+* Source filtering: only accept array items if the previous include pattern matches {pull}22593[#22593] (issue: {issue}22557[#22557])
+
+Logging::
+* Restores the original default format of search slow log {pull}21770[#21770] (issue: {issue}21711[#21711])
+
+Search::
+* Handle specialized term queries in MappedFieldType.extractTerm(Query) {pull}21889[#21889] (issue: {issue}21882[#21882])
+
+
+
+[[upgrade-5.2.0]]
+[float]
+=== Upgrades
+
+Core::
+* Upgrade to Lucene 6.4.0 {pull}22724[#22724]
+* Update Jackson to 2.8.6 {pull}22596[#22596] (issue: {issue}22266[#22266])
+* Upgrade to lucene-6.4.0-snapshot-084f7a0. {pull}22413[#22413]
+* Upgrade to lucene-6.4.0-snapshot-ec38570 {pull}21853[#21853]
+
+Network::
+* Upgrade to Netty 4.1.7 {pull}22587[#22587]
+
+
diff --git a/docs/reference/release-notes/5.2.1.asciidoc b/docs/reference/release-notes/5.2.1.asciidoc
new file mode 100644
index 0000000000000..ca2d20393d803
--- /dev/null
+++ b/docs/reference/release-notes/5.2.1.asciidoc
@@ -0,0 +1,67 @@
+[[release-notes-5.2.1]]
+== 5.2.1 Release Notes
+
+Also see <<breaking-changes-5.2>>.
+
+[[deprecation-5.2.1]]
+[float]
+=== Deprecations
+
+Geo::
+* Geo distance range deprecation {pull}22835[#22835]
+
+
+
+[[enhancement-5.2.1]]
+[float]
+=== Enhancements
+
+Allocation::
+* Include stale replica shard info when explaining an unassigned primary {pull}22826[#22826]
+
+
+
+[[bug-5.2.1]]
+[float]
+=== Bug fixes
+
+Cache::
+* Invalidate cached query results if query timed out {pull}22807[#22807] (issue: {issue}22789[#22789])
+
+Mapping::
+* Range types causing `GetFieldMappingsIndexRequest` to fail due to `NullPointerException` in `RangeFieldMapper.doXContentBody` when `include_defaults=true` is on the query string {pull}22925[#22925]
+* Disallow introducing illegal object mappings (double '..') {pull}22891[#22891] (issue: {issue}22794[#22794])
+
+Plugin Ingest Attachment::
+* Add missing mime4j library {pull}22799[#22799] (issue: {issue}22764[#22764])
+
+Plugin Lang Painless::
+* Don't allow casting from void to def in painless {pull}22969[#22969] (issue: {issue}22908[#22908])
+* Fix def invoked qualified method refs {pull}22918[#22918]
+
+Reindex API::
+* Reindex: do not log when can't clear old scroll {pull}22942[#22942] (issue: {issue}22937[#22937])
+* Fix reindex-from-remote from <2.0 {pull}22931[#22931] (issue: {issue}22893[#22893])
+
+Settings::
+* Expose `search.highlight.term_vector_multi_value` as a node level setting {pull}22999[#22999]
+
+
+
+[[regression-5.2.1]]
+[float]
+=== Regressions
+
+Core::
+* Source filtering: only accept array items if the previous include pattern matches {pull}22593[#22593] (issue: {issue}22557[#22557])
+
+
+
+[[upgrade-5.2.1]]
+[float]
+=== Upgrades
+
+Internal::
+* Upgrade to Lucene 6.4.1. {pull}22978[#22978]
+
+
diff --git a/docs/reference/release-notes/5.2.2.asciidoc b/docs/reference/release-notes/5.2.2.asciidoc
new file mode 100644
index 0000000000000..db8cb75c9cbe0
--- /dev/null
+++ b/docs/reference/release-notes/5.2.2.asciidoc
@@ -0,0 +1,38 @@
+[[release-notes-5.2.2]]
+== 5.2.2 Release Notes
+
+Also see <<breaking-changes-5.2>>.
+
+[[bug-5.2.2]]
+[float]
+=== Bug fixes
+
+Aggregations::
+* Restore support for the `include/pattern` syntax. {pull}23140[#23140] (issue: {issue}22933[#22933])
+
+Cluster::
+* Don't set local node on cluster state used for node join validation {pull}23311[#23311] (issue: {issue}21830[#21830])
+* Cluster allocation explain to never return empty response body {pull}23054[#23054]
+
+Index APIs::
+* Ensure shrunk indices carry over version information from its source {pull}22469[#22469] (issue: {issue}22373[#22373])
+
+Internal::
+* Always restore the ThreadContext for operations delayed due to a block {pull}23349[#23349]
+
+Java REST Client::
+* RestClient asynchronous execution should not throw exceptions {pull}23307[#23307]
+
+Network::
+* Respect promises on pipelined responses {pull}23317[#23317] (issues: {issue}23310[#23310], {issue}23322[#23322])
+* Ensure that releasing listener is called {pull}23310[#23310]
+
+Plugin Ingest Attachment::
+* Remove support for Visio and potm files {pull}23214[#23214] (issues: {issue}22077[#22077], {issue}22079[#22079], {issue}22963[#22963])
+
+REST::
+* Properly encode location header {pull}23133[#23133] (issues: {issue}21057[#21057], {issue}23115[#23115])
+
+Stats::
+* Fix control group pattern {pull}23219[#23219] (issue: {issue}23218[#23218])
+
diff --git a/docs/reference/release-notes/5.3.0.asciidoc b/docs/reference/release-notes/5.3.0.asciidoc
new file mode 100644
index 0000000000000..303eb46183dc2
--- /dev/null
+++ b/docs/reference/release-notes/5.3.0.asciidoc
@@ -0,0 +1,338 @@
+[[release-notes-5.3.0]]
+== 5.3.0 Release Notes
+
+See https://www.elastic.co/blog/multi-data-path-bug-in-elasticsearch-5-3-0[Multi data path bug in Elasticsearch 5.3.0].
+
+Also see <<breaking-changes-5.3>>.
+
+[[breaking-5.3.0]]
+[float]
+=== Breaking changes
+
+Query DSL::
+* Fix name of `enabled_position_increments` {pull}22895[#22895]
+
+Search::
+* Remove DFS_QUERY_AND_FETCH as a search type {pull}22787[#22787]
+
+
+
+[[breaking-java-5.3.0]]
+[float]
+=== Breaking Java changes
+
+CRUD::
+* Make index and delete operation execute as single bulk item (backport of #21964) {pull}22812[#22812] (issue: {issue}21964[#21964])
+
+Internal::
+* Replace SearchExtRegistry with namedObject {pull}22492[#22492]
+* Replace Suggesters with namedObject {pull}22491[#22491]
+
+Network::
+* Remove HttpServer and HttpServerAdapter in favor of a simple dispatch method {pull}22636[#22636] (issue: {issue}18482[#18482])
+
+Plugins::
+* Deguice rest handlers {pull}22575[#22575]
+
+Search::
+* Remove QUERY_AND_FETCH search type {pull}22996[#22996]
+
+
+
+[[deprecation-5.3.0]]
+[float]
+=== Deprecations
+
+Core::
+* Deprecate lenient booleans {pull}22716[#22716] (issues: {issue}22200[#22200], {issue}22696[#22696])
+
+Geo::
+* Geo distance range deprecation {pull}22835[#22835]
+
+Java API::
+* Add BulkProcessor methods with XContentType parameter {pull}23078[#23078] (issue: {issue}22691[#22691])
+
+Mapping::
+* Deprecate the include_in_all mapping parameter {pull}22972[#22972] (issue: {issue}22970[#22970])
+
+Network::
+* Deprecate Netty 3 {pull}23459[#23459] (issue: {issue}23411[#23411])
+* Deprecate Netty 3 {pull}23411[#23411]
+
+Plugin Repository Azure::
+* Deprecate auto creation of container for azure repository {pull}22857[#22857] (issues: {issue}22843[#22843], {issue}22856[#22856])
+
+Plugin Repository S3::
+* S3 Repository: Deprecate specifying region {pull}22848[#22848] (issue: {issue}22758[#22758])
+* S3 Repository: Deprecate auto creation of s3 bucket for repository {pull}22843[#22843] (issue: {issue}22761[#22761])
+* Deprecate specifying credentials through env vars, sys props, and remove profile files {pull}22567[#22567] (issues: {issue}21041[#21041], {issue}22479[#22479])
+
+REST::
+* Deprecate ldjson support and document ndjson for bulk/msearch {pull}23051[#23051] (issues: {issue}23025[#23025], {issue}23049[#23049])
+* Optionally require a valid content type for all rest requests with content {pull}22691[#22691] (issue: {issue}19388[#19388])
+
+Scripting::
+* Change Namespace for Stored Script to Only Use Id {pull}22206[#22206]
+
+
+
+[[feature-5.3.0]]
+[float]
+=== New features
+
+Aggregations::
+* Initial version of an adjacency matrix using the Filters aggregation {pull}22239[#22239] (issue: {issue}22169[#22169])
+
+CRUD::
+* Allow an index to be partitioned with custom routing {pull}22274[#22274] (issue: {issue}21585[#21585])
+
+Highlighting::
+* Integrate UnifiedHighlighter {pull}21621[#21621] (issue: {issue}21376[#21376])
+
+Search::
+* Add federated cross cluster search capabilities {pull}22502[#22502] (issue: {issue}21473[#21473])
+* Add field collapsing for search request {pull}22337[#22337] (issue: {issue}21833[#21833])
+
+Settings::
+* Add infrastructure for elasticsearch keystore {pull}22335[#22335]
+
+
+
+[[enhancement-5.3.0]]
+[float]
+=== Enhancements
+
+Allocation::
+* Include stale replica shard info when explaining an unassigned primary {pull}22826[#22826]
+
+Analysis::
+* Analyze API Position Length Support {pull}22574[#22574]
+
+CRUD::
+* Make index and delete operation execute as a single bulk item {pull}21964[#21964]
+
+Circuit Breakers::
+* Add used memory amount to CircuitBreakingException message (#22521) {pull}22693[#22693] (issue: {issue}22521[#22521])
+
+Cluster::
+* Connect to new nodes concurrently {pull}22984[#22984] (issue: {issue}22828[#22828])
+
+Core::
+* Simplify write failure handling (backport of #19105) {pull}22778[#22778] (issue: {issue}19105[#19105])
+* Simplify ElasticsearchException rendering as a XContent {pull}22611[#22611]
+* Remove setLocalNode from ClusterService and TransportService {pull}22608[#22608]
+
+Engine::
+* Replace EngineClosedException with AlreadyClosedExcpetion {pull}22631[#22631]
+
+Index APIs::
+* Indexing: Add shard id to indexing operation listener {pull}22606[#22606]
+* Better error when can't auto create index  {pull}22488[#22488] (issues: {issue}21448[#21448], {issue}22435[#22435])
+
+Ingest::
+* Lazy load the geoip databases {pull}23337[#23337]
+
+Internal::
+* Improve connection closing in `RemoteClusterConnection` {pull}22804[#22804] (issue: {issue}22803[#22803])
+* Remove some more usages of ParseFieldMatcher {pull}22437[#22437] (issues: {issue}19552[#19552], {issue}22130[#22130])
+* Introduce ToXContentObject interface {pull}22387[#22387] (issue: {issue}16347[#16347])
+
+Java API::
+* prevent NPE when trying to uncompress a null BytesReference {pull}22386[#22386]
+
+Java High Level REST Client::
+* Add parsing from xContent to MainResponse {pull}22934[#22934]
+* Parse elasticsearch exception's root causes {pull}22924[#22924]
+* Add parsing method to BytesRestResponse's error {pull}22873[#22873]
+* Add parsing method for ElasticsearchException.generateThrowableXContent() {pull}22783[#22783]
+* Add fromxcontent methods to delete response {pull}22712[#22712] (issue: {issue}22680[#22680])
+* Add parsing methods for UpdateResponse {pull}22586[#22586]
+* Add parsing from xContent to InternalSearchHit and InternalSearchHits {pull}22429[#22429]
+* Add fromxcontent methods to index response {pull}22229[#22229]
+
+Java REST Client::
+* move ignore parameter support from yaml test client to low level rest client {pull}22637[#22637]
+* Support Preemptive Authentication with RestClient {pull}21336[#21336]
+
+Logging::
+* Expose logs base path {pull}22625[#22625]
+
+Mapping::
+* Improve error message for ipv6 on legacy ip fields {pull}23136[#23136] (issue: {issue}23126[#23126])
+
+Network::
+* Keep the pipeline handler queue small initially {pull}23335[#23335]
+
+Packaging::
+* Introduce Java version check {pull}23194[#23194] (issue: {issue}21102[#21102])
+
+Plugin Discovery EC2::
+* Read ec2 discovery address from aws instance tags {pull}22819[#22819] (issues: {issue}22566[#22566], {issue}22743[#22743])
+
+Plugin Lang Painless::
+* Generate reference links for painless API {pull}22775[#22775]
+* Painless: Add augmentation to String for base 64 {pull}22665[#22665] (issue: {issue}22648[#22648])
+
+Plugin Repository S3::
+* S3 repository: Add named configurations {pull}22762[#22762] (issues: {issue}22479[#22479], {issue}22520[#22520])
+
+Plugins::
+* Add the ability to define search response listeners in search plugin {pull}22682[#22682]
+
+Query DSL::
+* QueryString and SimpleQueryString Graph Support {pull}22541[#22541]
+* Additional Graph Support in Match Query {pull}22503[#22503] (issue: {issue}22490[#22490])
+* RangeQuery WITHIN case now normalises query {pull}22431[#22431] (issue: {issue}22412[#22412])
+
+Reindex API::
+* Increase visibility of doExecute so it can be used directly {pull}22614[#22614]
+* Improve error message when reindex-from-remote gets bad json {pull}22536[#22536] (issue: {issue}22330[#22330])
+
+Scripting::
+* Expose multi-valued dates to scripts and document painless's date functions {pull}22875[#22875] (issue: {issue}22162[#22162])
+
+Search::
+* Add a setting to disable remote cluster connections on a node {pull}23005[#23005]
+* First step towards separating individual search phases {pull}22802[#22802]
+* Add parsing from xContent to SearchProfileShardResults and nested classes {pull}22649[#22649]
+* ProfileResult and CollectorResult should print machine readable timing information {pull}22638[#22638]
+
+Settings::
+* Improve setting deprecation message {pull}23156[#23156] (issue: {issue}22849[#22849])
+* Add secure settings validation on startup {pull}22894[#22894]
+* Allow comma delimited array settings to have a space after each entry {pull}22591[#22591] (issue: {issue}22297[#22297])
+* Allow affix settings to be dynamic / updatable {pull}22526[#22526]
+* Allow affix settings to delegate to actual settings {pull}22523[#22523]
+* Make s3 repository sensitive settings use secure settings {pull}22479[#22479]
+
+Snapshot/Restore::
+* Duplicate snapshot name throws InvalidSnapshotNameException {pull}22921[#22921] (issue: {issue}18228[#18228])
+* Use general cluster state batching mechanism for snapshot state updates {pull}22528[#22528] (issue: {issue}14899[#14899])
+
+Stats::
+* Add geo_point to FieldStats {pull}21947[#21947] (issue: {issue}20707[#20707])
+
+
+
+[[bug-5.3.0]]
+[float]
+=== Bug fixes
+
+CRUD::
+* Fix backport executing ops as single item bulk {pull}23083[#23083] (issues: {issue}21964[#21964], {issue}23069[#23069])
+
+Cache::
+* Invalidate cached query results if query timed out {pull}22807[#22807] (issue: {issue}22789[#22789])
+
+Cluster::
+* Don't set local node on cluster state used for node join validation {pull}23311[#23311] (issues: {issue}21830[#21830], {issue}3[#3], {issue}4[#4], {issue}6[#6], {issue}9[#9])
+* Allow a cluster state applier to create an observer and wait for a better state {pull}23132[#23132] (issue: {issue}21817[#21817])
+* Cluster allocation explain to never return empty response body {pull}23054[#23054]
+
+Exceptions::
+* Stop returning "es." internal exception headers as http response headers {pull}22703[#22703] (issue: {issue}17593[#17593])
+
+Ingest::
+* Improve missing ingest processor error {pull}23379[#23379] (issue: {issue}23392[#23392])
+* fix date-processor to a new default year for every new pipeline execution {pull}22601[#22601] (issue: {issue}22547[#22547])
+
+Internal::
+* Always restore the ThreadContext for operations delayed due to a block {pull}23349[#23349]
+* Fix handling of document failure exception in InternalEngine (backport #22718) {pull}22910[#22910]
+
+Java REST Client::
+* RestClient asynchronous execution should not throw exceptions {pull}23307[#23307]
+
+Mapping::
+* Switch include_in_all in multifield to warning {pull}23656[#23656] (issues: {issue}21971[#21971], {issue}23654[#23654])
+* Fix MapperService StackOverflowError {pull}23605[#23605] (issue: {issue}23604[#23604])
+* Range types causing `GetFieldMappingsIndexRequest` to fail due to `NullPointerException` in `RangeFieldMapper.doXContentBody` when `include_defaults=true` is on the query string {pull}22925[#22925]
+* Disallow introducing illegal object mappings (double '..') {pull}22891[#22891] (issue: {issue}22794[#22794])
+
+Network::
+* Respect promises on pipelined responses {pull}23317[#23317] (issues: {issue}23310[#23310], {issue}23322[#23322])
+* Ensure that releasing listener is called {pull}23310[#23310]
+
+Packaging::
+* Fall back to non-atomic move when removing plugins {pull}23548[#23548] (issue: {issue}35[#35])
+
+Parent/Child::
+* Add null check in case of orphan child document {pull}22772[#22772] (issue: {issue}22770[#22770])
+
+Plugin Ingest Attachment::
+* Remove support for Visio and potm files {pull}23214[#23214] (issues: {issue}22077[#22077], {issue}22079[#22079], {issue}22963[#22963])
+* Add missing mime4j library {pull}22799[#22799] (issue: {issue}22764[#22764])
+
+Plugin Lang Painless::
+* Fix Bad Casts In Painless {pull}23282[#23282] (issue: {issue}23238[#23238])
+* Don't allow casting from void to def in painless {pull}22969[#22969] (issue: {issue}22908[#22908])
+* Fix def invoked qualified method refs {pull}22918[#22918]
+
+Plugins::
+* Add check for null pluginName in remove command {pull}22930[#22930] (issue: {issue}22922[#22922])
+
+Query DSL::
+* Fix parsing for `max_determinized_states` {pull}22749[#22749] (issue: {issue}22722[#22722])
+
+REST::
+* [API] change wait_for_completion default according to docs {pull}23672[#23672]
+* HTTP transport stashes the ThreadContext instead of the RestController {pull}23456[#23456]
+* Ensure we try to autodetect content type for handlers that support plain text {pull}23452[#23452]
+* Fix date format in warning headers {pull}23418[#23418] (issue: {issue}23275[#23275])
+* Correct warning header to be compliant {pull}23275[#23275] (issue: {issue}22986[#22986])
+* Fix search scroll request with a plain text body {pull}23183[#23183] (issue: {issue}22691[#22691])
+* Handle bad HTTP requests {pull}23153[#23153] (issue: {issue}23034[#23034])
+* Properly encode location header {pull}23133[#23133] (issues: {issue}21057[#21057], {issue}23115[#23115])
+
+Reindex API::
+* Reindex: do not log when can't clear old scroll {pull}22942[#22942] (issue: {issue}22937[#22937])
+* Fix reindex-from-remote from <2.0 {pull}22931[#22931] (issue: {issue}22893[#22893])
+
+Scripting::
+* Remove unnecessary Groovy deprecation logging {pull}23410[#23410] (issue: {issue}23401[#23401])
+* Script: Fix value of `ctx._now` to be current epoch time in milliseconds {pull}23175[#23175] (issue: {issue}23169[#23169])
+
+Search::
+* Honor max concurrent searches in multi-search {pull}23538[#23538] (issue: {issue}23527[#23527])
+* Avoid stack overflow in multi-search {pull}23527[#23527] (issue: {issue}23523[#23523])
+* Replace blocking calls in ExpandCollapseSearchResponseListener by asynchronous requests {pull}23053[#23053] (issue: {issue}23048[#23048])
+* Ensure fixed serialization order of InnerHitBuilder {pull}22820[#22820] (issue: {issue}22808[#22808])
+
+Settings::
+* Fix merge scheduler config settings {pull}23391[#23391]
+* Settings: Fix keystore cli prompting for yes/no to handle console returning null {pull}23320[#23320]
+* Expose `search.highlight.term_vector_multi_value` as a node level setting {pull}22999[#22999]
+* NPE when no setting name passed to elasticsearch-keystore {pull}22609[#22609]
+
+Similarities::
+* Fix similarity upgrade when "default" similarity is overridden {pull}23163[#23163]
+
+Stats::
+* Avoid overflow when computing total FS stats {pull}23641[#23641]
+* Handle long overflow when adding paths' totals {pull}23293[#23293] (issue: {issue}23093[#23093])
+* Fix control group pattern {pull}23219[#23219] (issue: {issue}23218[#23218])
+* Fix total disk bytes returning negative value {pull}23093[#23093]
+
+Task Manager::
+* Fix hanging cancelling task with no children {pull}22796[#22796]
+* Fix broken TaskInfo.toString() {pull}22698[#22698] (issue: {issue}22387[#22387])
+
+
+
+[[regression-5.3.0]]
+[float]
+=== Regressions
+
+Core::
+* Source filtering: only accept array items if the previous include pattern matches {pull}22593[#22593] (issue: {issue}22557[#22557])
+
+
+
+[[upgrade-5.3.0]]
+[float]
+=== Upgrades
+
+Internal::
+* Upgrade to Lucene 6.4.1. {pull}22978[#22978]
+
+
diff --git a/docs/reference/release-notes/5.3.1.asciidoc b/docs/reference/release-notes/5.3.1.asciidoc
new file mode 100644
index 0000000000000..440baf7664408
--- /dev/null
+++ b/docs/reference/release-notes/5.3.1.asciidoc
@@ -0,0 +1,96 @@
+[[release-notes-5.3.1]]
+== 5.3.1 Release Notes
+
+See https://www.elastic.co/blog/multi-data-path-bug-in-elasticsearch-5-3-0[Multi data path bug in Elasticsearch 5.3.0].
+
+Also see <<breaking-changes-5.3>>.
+
+[[breaking-5.3.1]]
+[float]
+=== Breaking changes
+
+Settings::
+* Remove support for default settings {pull}24093[#24093] (issues: {issue}23981[#23981], {issue}24052[#24052], {issue}24074[#24074])
+
+
+
+[[enhancement-5.3.1]]
+[float]
+=== Enhancements
+
+Core::
+* Detect remnants of path.data/default.path.data bug {pull}24099[#24099] (issues: {issue}23981[#23981], {issue}24052[#24052], {issue}24074[#24074], {issue}24093[#24093])
+
+Network::
+* Adjust default Netty receive predictor size to 64k {pull}23542[#23542] (issue: {issue}23185[#23185])
+
+REST::
+* Cluster stats should not render empty http/transport types {pull}23735[#23735]
+
+
+
+[[bug-5.3.1]]
+[float]
+=== Bug fixes
+
+Aggregations::
+* The `filter` and `significant_terms` aggregations should parse the `filter` as a filter, not a query. {pull}23797[#23797]
+* Restore support for the `include/pattern` syntax. {pull}23140[#23140] (issue: {issue}22933[#22933])
+
+Bulk::
+* Reject empty IDs {pull}24118[#24118] (issue: {issue}24116[#24116])
+
+Core::
+* Improve performance of extracting warning value {pull}24114[#24114] (issue: {issue}24018[#24018])
+* Restrict build info loading to ES jar, not any jar {pull}24049[#24049] (issue: {issue}21955[#21955])
+
+Discovery::
+* ZenDiscovery - only validate min_master_nodes values if local node is master {pull}23915[#23915] (issue: {issue}23695[#23695])
+
+Index APIs::
+* Honor update request timeout {pull}23825[#23825]
+
+Inner Hits::
+* Replace `NestedChildrenQuery` with `ParentChildrenBlockJoinQuery` {pull}24039[#24039] (issue: {issue}24016[#24016])
+
+Internal::
+* Add infrastructure to mark contexts as system contexts {pull}23830[#23830]
+
+Java REST Client::
+* Make buffer limit configurable in HeapBufferedConsumerFactory {pull}23970[#23970] (issue: {issue}23958[#23958])
+
+Mapping::
+* Preserve response headers when creating an index {pull}23950[#23950] (issue: {issue}23947[#23947])
+* Fix NPE with scaled floats stats when field is not indexed {pull}23528[#23528] (issue: {issue}23487[#23487])
+
+Network::
+* Fix possible hang in local transport when nodes get concurrently disconnected {pull}23962[#23962] (issue: {issue}23942[#23942])
+
+Plugin Repository Azure::
+* Azure blob store's readBlob() method first checks if the blob exists {pull}23483[#23483] (issue: {issue}23480[#23480])
+
+Reindex API::
+* Fix reindex with a remote source on a version before 2.0.0 {pull}23805[#23805]
+
+Search::
+* Disable graph analysis at query time for shingle and cjk filters producing tokens of different size {pull}23920[#23920] (issue: {issue}23918[#23918])
+* Fix cross-cluster remote node gateway attributes {pull}23863[#23863]
+* Use a fixed seed for computing term hashCode in TermsSliceQuery {pull}23795[#23795]
+
+Settings::
+* Do not set path.data in environment if not set {pull}24132[#24132] (issue: {issue}24099[#24099])
+* Correct handling of default and array settings {pull}24074[#24074] (issues: {issue}23981[#23981], {issue}24052[#24052])
+
+Stats::
+* Handle existence of cgroup version 2 hierarchy {pull}23493[#23493] (issue: {issue}23486[#23486])
+
+
+
+[[upgrade-5.3.1]]
+[float]
+=== Upgrades
+
+Internal::
+* upgrade to Lucene 6.4.2 {pull}23933[#23933]
+
+
diff --git a/docs/reference/release-notes/5.3.2.asciidoc b/docs/reference/release-notes/5.3.2.asciidoc
new file mode 100644
index 0000000000000..871499b6d521a
--- /dev/null
+++ b/docs/reference/release-notes/5.3.2.asciidoc
@@ -0,0 +1,38 @@
+[[release-notes-5.3.2]]
+== 5.3.2 Release Notes
+
+See https://www.elastic.co/blog/multi-data-path-bug-in-elasticsearch-5-3-0[Multi data path bug in Elasticsearch 5.3.0].
+
+Also see <<breaking-changes-5.3>>.
+
+[[bug-5.3.2]]
+[float]
+=== Bug fixes
+
+Aggregations::
+* Align behavior HDR percentiles iterator with percentile() method {pull}24206[#24206]
+
+Core::
+* Check for default.path.data included in path.data {pull}24285[#24285] (issue: {issue}24283[#24283])
+
+Plugin Ingest Attachment::
+* Add Fix for java.lang.NoClassDefFoundError: com/graphbuilder/curve/Point fatal error in 5.3 [ISSUE] {pull}24273[#24273]
+
+Plugins::
+* Fix delete of plugin directory on remove plugin {pull}24266[#24266] (issue: {issue}24252[#24252])
+* Use a marker file when removing a plugin {pull}24252[#24252] (issue: {issue}24231[#24231])
+
+Search Templates::
+* No longer add illegal content type option to stored search templates {pull}24251[#24251] (issue: {issue}24227[#24227])
+
+
+
+[[regression-5.3.2]]
+[float]
+=== Regressions
+
+Bulk::
+* Fix _bulk response when it can't create an index {pull}24048[#24048] (issues: {issue}22488[#22488], {issue}24028[#24028])
+
+
+
diff --git a/docs/reference/release-notes/5.3.3.asciidoc b/docs/reference/release-notes/5.3.3.asciidoc
new file mode 100644
index 0000000000000..665c825aca739
--- /dev/null
+++ b/docs/reference/release-notes/5.3.3.asciidoc
@@ -0,0 +1,29 @@
+[[release-notes-5.3.3]]
+== 5.3.3 Release Notes
+
+See https://www.elastic.co/blog/multi-data-path-bug-in-elasticsearch-5-3-0[Multi data path bug in Elasticsearch 5.3.0].
+
+Also see <<breaking-changes-5.3>>.
+
+[[enhancement-5.3.3]]
+[float]
+=== Enhancements
+
+Network::
+* Disable the Netty recycler in the client {pull}24793[#24793] (issues: {issue}22452[#22452], {issue}24721[#24721])
+
+
+
+[[bug-5.3.3]]
+[float]
+=== Bug fixes
+
+Index APIs::
+* Fix legacy GeoPointField decoding in FieldStats {pull}24534[#24534] (issue: {issue}24275[#24275])
+
+Packaging::
+* Handle parentheses in batch file path {pull}24731[#24731] (issue: {issue}24712[#24712])
+
+Search::
+* Fix RangeFieldMapper rangeQuery to properly handle relations {pull}24808[#24808] (issue: {issue}24744[#24744])
+
diff --git a/docs/reference/release-notes/5.4.0.asciidoc b/docs/reference/release-notes/5.4.0.asciidoc
new file mode 100644
index 0000000000000..6d9d1d79c60f3
--- /dev/null
+++ b/docs/reference/release-notes/5.4.0.asciidoc
@@ -0,0 +1,439 @@
+[[release-notes-5.4.0]]
+== 5.4.0 Release Notes
+
+See https://www.elastic.co/blog/multi-data-path-bug-in-elasticsearch-5-3-0[Multi data path bug in Elasticsearch 5.3.0].
+
+Also see <<breaking-changes-5.4>>.
+
+
+[[breaking-5.4.0]]
+[float]
+=== Breaking changes
+
+Settings::
+* Remove support for default settings {pull}24093[#24093] (issues: {issue}23981[#23981], {issue}24052[#24052], {issue}24074[#24074])
+
+
+
+[[breaking-java-5.4.0]]
+[float]
+=== Breaking Java changes
+
+Aggregations::
+* Move getProperty method out of MultiBucketsAggregation.Bucket interface {pull}23988[#23988]
+* Remove getProperty method from Aggregations interface and impl {pull}23972[#23972]
+* Move getProperty method out of Aggregation interface {pull}23949[#23949]
+
+Java API::
+* Fold InternalSearchHits and friends into their interfaces {pull}23042[#23042]
+
+
+
+[[deprecation-5.4.0]]
+[float]
+=== Deprecations
+
+Aggregations::
+* Deprecate Stats#getCountAsString {pull}24292[#24292]
+
+Java API::
+* Add BulkProcessor methods with XContentType parameter {pull}23078[#23078] (issue: {issue}22691[#22691])
+
+Network::
+* Deprecate Netty 3 {pull}23459[#23459] (issue: {issue}23411[#23411])
+
+Packaging::
+* Add deprecation warnings for $ES_USER and $ES_GROUP {pull}24190[#24190] (issue: {issue}23989[#23989])
+
+Plugin Delete By Query::
+* Deprecate delete_by_query requests without an explicit query {pull}23635[#23635]
+
+Plugin Repository Azure::
+* Deprecate global `repositories.azure` settings {pull}22856[#22856] (issue: {issue}22800[#22800])
+
+Plugin Repository S3::
+* Deprecate repositories.s3 settings {pull}23278[#23278] (issues: {issue}22800[#22800], {issue}23276[#23276])
+
+REST::
+* Deprecate ldjson support and document ndjson for bulk/msearch {pull}23051[#23051] (issues: {issue}23025[#23025], {issue}23049[#23049])
+
+Stats::
+* Deprecate `_field_stats` endpoint {pull}23914[#23914]
+
+
+
+[[feature-5.4.0]]
+[float]
+=== New features
+
+Analysis::
+* Adds pattern keyword marker filter support {pull}23600[#23600] (issue: {issue}4877[#4877])
+* Expose WordDelimiterGraphTokenFilter {pull}23327[#23327] (issue: {issue}23104[#23104])
+
+Index APIs::
+* Add FieldCapabilities (`_field_caps`) API {pull}23007[#23007] (issue: {issue}22438[#22438])
+
+Search::
+* Introduce incremental reduction of TopDocs {pull}23946[#23946]
+
+Similarities::
+* Adds boolean similarity to Elasticsearch {pull}23637[#23637] (issue: {issue}6731[#6731])
+
+
+
+[[enhancement-5.4.0]]
+[float]
+=== Enhancements
+
+Aggregations::
+* Add BucketMetricValue interface {pull}24188[#24188]
+* Move aggs CommonFields and TYPED_KEYS_DELIMITER from InternalAggregation to Aggregation {pull}23987[#23987]
+* Use ParseField for aggs CommonFields rather than String {pull}23717[#23717]
+* Share XContent rendering code in terms aggs {pull}23680[#23680]
+* Add unit tests for ParentToChildAggregator {pull}23305[#23305] (issue: {issue}22278[#22278])
+* First step towards incremental reduction of query responses {pull}23253[#23253]
+
+Allocation::
+* Trigger replica recovery restarts by master when primary relocation completes {pull}23926[#23926] (issue: {issue}23904[#23904])
+* Makes the same_shard host dynamically updatable {pull}23397[#23397] (issue: {issue}22992[#22992])
+
+Analysis::
+* Support Keyword type in Analyze API {pull}23161[#23161]
+
+Cluster::
+* Prevent nodes from joining if newer indices exist in the cluster {pull}23843[#23843]
+
+Core::
+* Detect remnants of path.data/default.path.data bug {pull}24099[#24099] (issues: {issue}23981[#23981], {issue}24052[#24052], {issue}24074[#24074], {issue}24093[#24093])
+* Await termination after shutting down executors {pull}23889[#23889]
+* Add early-access check {pull}23743[#23743] (issue: {issue}23668[#23668])
+* Adapter action future should restore interrupts {pull}23618[#23618] (issue: {issue}23617[#23617])
+* Disable bootstrap checks for single-node discovery {pull}23598[#23598] (issues: {issue}23585[#23585], {issue}23595[#23595])
+* Enable explicitly enforcing bootstrap checks {pull}23585[#23585] (issue: {issue}21864[#21864])
+* Add equals/hashcode method to ReplicationResponse {pull}23215[#23215]
+
+Dates::
+* Improve error handling for epoch format parser with time zone (#22621) {pull}23689[#23689]
+
+Discovery::
+* Introduce single-node discovery {pull}23595[#23595]
+* UnicastZenPing shouldn't ping the address of the local node {pull}23567[#23567]
+* MasterFaultDetection can start after the initial cluster state has been processed {pull}23037[#23037] (issue: {issue}22828[#22828])
+
+Highlighting::
+* Add support for fragment_length in the unified highlighter {pull}23431[#23431]
+* Add BreakIteratorBoundaryScanner support {pull}23248[#23248]
+
+Index APIs::
+* Wildcard cluster names for cross cluster search {pull}23985[#23985] (issue: {issue}23893[#23893])
+
+Ingest::
+* Lazy load the geoip databases {pull}23337[#23337]
+
+Internal::
+* Add a dedicated TransportRemoteInfoAction for consistency {pull}24040[#24040] (issue: {issue}23969[#23969])
+* Simplify sorted top docs merging in SearchPhaseController {pull}23881[#23881]
+* Synchronized CollapseTopFieldDocs with lucenes relatives {pull}23854[#23854]
+* Cleanup SearchPhaseController interface {pull}23844[#23844]
+* Do not create String instances in 'Strings' methods accepting StringBuilder {pull}22907[#22907]
+
+Java API::
+* Added types options to DeleteByQueryRequest {pull}23265[#23265] (issue: {issue}21984[#21984])
+
+Java High Level REST Client::
+* Convert suggestion response parsing to use NamedXContentRegistry {pull}23355[#23355]
+* UpdateRequest implements ToXContent {pull}23289[#23289]
+* Add javadoc for DocWriteResponse.Builders {pull}23267[#23267]
+* Expose WriteRequest.RefreshPolicy string representation {pull}23106[#23106]
+* Use `typed_keys` parameter to prefix suggester names by type in search responses {pull}23080[#23080] (issue: {issue}22965[#22965])
+* Add parsing methods to BulkItemResponse {pull}22859[#22859]
+
+Logging::
+* Warn on not enough masters during election {pull}20063[#20063] (issue: {issue}8362[#8362])
+
+Mapping::
+* Improve error message for ipv6 on legacy ip fields {pull}23136[#23136] (issue: {issue}23126[#23126])
+
+Nested Docs::
+* Avoid adding unnecessary nested filters when ranges are used. {pull}23427[#23427]
+
+Network::
+* Adjust default Netty receive predictor size to 64k {pull}23542[#23542] (issue: {issue}23185[#23185])
+* Keep the pipeline handler queue small initially {pull}23335[#23335]
+* Set network receive predictor size to 32kb {pull}23284[#23284] (issue: {issue}23185[#23185])
+* TransportService.connectToNode should validate remote node ID {pull}22828[#22828] (issue: {issue}22194[#22194])
+
+Packaging::
+* Introduce Java version check {pull}23194[#23194] (issue: {issue}21102[#21102])
+* Cleanup some things after removal of joda-time hack {pull}18959[#18959]
+
+Percolator::
+* Allowing range queries with now ranges inside percolator queries {pull}23921[#23921] (issue: {issue}23859[#23859])
+* Add term extraction support for MultiPhraseQuery {pull}23176[#23176]
+
+Plugin Discovery EC2::
+* Settings: Migrate ec2 discovery sensitive settings to elasticsearch keystore {pull}23961[#23961] (issue: {issue}22475[#22475])
+
+Plugin Lang Painless::
+* Allow painless to load stored fields {pull}24290[#24290]
+* Start on custom whitelists for Painless {pull}23563[#23563]
+* Fix Painless's implementation of interfaces returning primitives {pull}23298[#23298] (issue: {issue}22983[#22983])
+* Allow painless to implement more interfaces {pull}22983[#22983]
+
+Plugin Repository Azure::
+* Add Backoff policy to azure repository {pull}23387[#23387] (issue: {issue}22728[#22728])
+
+Plugin Repository S3::
+* Removes the retry mechanism from the S3 blob store {pull}23952[#23952] (issue: {issue}22845[#22845])
+* S3 Repository: Eagerly load static settings {pull}23910[#23910]
+
+Plugins::
+* Modify permissions dialog for plugins {pull}23742[#23742]
+* Plugins: Add plugin cli specific exit codes {pull}23599[#23599] (issue: {issue}15295[#15295])
+* Plugins: Output better error message when existing plugin is incompatible {pull}23562[#23562] (issue: {issue}20691[#20691])
+
+Query DSL::
+* Make it possible to validate a query on all shards instead of a single random shard {pull}23697[#23697] (issue: {issue}18254[#18254])
+
+REST::
+* Validate top-level keys when parsing mget requests {pull}23746[#23746] (issue: {issue}23720[#23720])
+* Cluster stats should not render empty http/transport types {pull}23735[#23735]
+* Add parameter to prefix aggs name with type in search responses {pull}22965[#22965]
+
+Search::
+* Set shard count limit to unlimited {pull}24012[#24012]
+* Streamline shard index availability in all SearchPhaseResults {pull}23788[#23788]
+* Search took time should use a relative clock {pull}23662[#23662]
+* Prevent negative `from` parameter in SearchSourceBuilder {pull}23358[#23358] (issue: {issue}23324[#23324])
+* Remove unnecessary result sorting in SearchPhaseController {pull}23321[#23321]
+* Expose `batched_reduce_size` via `_search` {pull}23288[#23288] (issue: {issue}23253[#23253])
+* Adding fromXContent to Suggest and Suggestion class {pull}23226[#23226] (issue: {issue}23202[#23202])
+* Adding fromXContent to Suggestion.Entry and subclasses {pull}23202[#23202]
+* Add CollapseSearchPhase as a successor for the FetchSearchPhase {pull}23165[#23165]
+* Integrate IndexOrDocValuesQuery. {pull}23119[#23119]
+* Detach SearchPhases from AbstractSearchAsyncAction {pull}23118[#23118]
+* Fix GraphQuery expectation after Lucene upgrade to 6.5 {pull}23117[#23117] (issue: {issue}23102[#23102])
+* Nested queries should avoid adding unnecessary filters when possible. {pull}23079[#23079] (issue: {issue}20797[#20797])
+* Add xcontent parsing to completion suggestion option {pull}23071[#23071]
+* Add xcontent parsing to suggestion options {pull}23018[#23018]
+* Separate reduce (aggs, suggest and profile) from merging fetched hits {pull}23017[#23017]
+
+Settings::
+* Add secure file setting to keystore {pull}24001[#24001]
+* Add a setting which specifies a list of setting {pull}23883[#23883]
+* Add a property to mark setting as final {pull}23872[#23872]
+* Remove obsolete index setting `index.version.minimum_compatible`. {pull}23593[#23593]
+* Provide a method to retrieve a closeable char[] from a SecureString {pull}23389[#23389]
+* Update indices settings api to support CBOR and SMILE format {pull}23309[#23309] (issues: {issue}23242[#23242], {issue}23245[#23245])
+* Improve setting deprecation message {pull}23156[#23156] (issue: {issue}22849[#22849])
+
+Snapshot/Restore::
+* Change snapshot status error to use generic SnapshotException {pull}24355[#24355] (issue: {issue}24225[#24225])
+
+Stats::
+* Add cross-cluster search remote cluster info API {pull}23969[#23969] (issue: {issue}23925[#23925])
+
+Task Manager::
+* Allow task to be unregistered by ClusterStateApplier {pull}23931[#23931]
+* Limit IndexRequest toString() length {pull}22832[#22832]
+
+
+
+[[bug-5.4.0]]
+[float]
+=== Bug fixes
+
+Aggregations::
+* Align behavior HDR percentiles iterator with percentile() method {pull}24206[#24206]
+* The `filter` and `significant_terms` aggregations should parse the `filter` as a filter, not a query. {pull}23797[#23797]
+* Completion suggestion should also consider text if prefix/regex is missing {pull}23451[#23451] (issue: {issue}23340[#23340])
+* Fixes the per term error in the terms aggregation {pull}23399[#23399]
+* Fixes terms error count for multiple reduce phases {pull}23291[#23291] (issue: {issue}23286[#23286])
+* Restore support for the `include/pattern` syntax. {pull}23140[#23140] (issue: {issue}22933[#22933])
+
+Bulk::
+* Reject empty IDs {pull}24118[#24118] (issue: {issue}24116[#24116])
+
+CRUD::
+* Fix backport executing ops as single item bulk {pull}23083[#23083] (issues: {issue}21964[#21964], {issue}23069[#23069])
+
+Cluster::
+* Don't set local node on cluster state used for node join validation {pull}23311[#23311] (issues: {issue}21830[#21830], {issue}3[#3], {issue}4[#4], {issue}6[#6], {issue}9[#9])
+* Allow a cluster state applier to create an observer and wait for a better state {pull}23132[#23132] (issue: {issue}21817[#21817])
+* Cluster allocation explain to never return empty response body {pull}23054[#23054]
+
+Core::
+* Check for default.path.data included in path.data {pull}24285[#24285] (issue: {issue}24283[#24283])
+* Improve performance of extracting warning value {pull}24114[#24114] (issue: {issue}24018[#24018])
+* Reject duplicate settings on the command line {pull}24053[#24053]
+* Restrict build info loading to ES jar, not any jar {pull}24049[#24049] (issue: {issue}21955[#21955])
+
+Discovery::
+* ZenDiscovery - only validate min_master_nodes values if local node is master {pull}23915[#23915] (issue: {issue}23695[#23695])
+
+Index APIs::
+* Fixes restore of a shrunken index when initial recovery node is gone {pull}24322[#24322] (issue: {issue}24257[#24257])
+* Honor update request timeout {pull}23825[#23825]
+
+Ingest::
+* Improve missing ingest processor error {pull}23379[#23379] (issue: {issue}23392[#23392])
+* update _ingest.timestamp to use new ZonedDateTime {pull}23174[#23174] (issue: {issue}23168[#23168])
+
+Inner Hits::
+* Replace NestedChildrenQuery with ParentChildrenBlockJoinQuery {pull}24016[#24016] (issue: {issue}24009[#24009])
+* Changed DisMaxQueryBuilder to extract inner hits from leaf queries {pull}23512[#23512] (issue: {issue}23482[#23482])
+
+Internal::
+* Add infrastructure to mark contexts as system contexts {pull}23830[#23830]
+* Always restore the ThreadContext for operations delayed due to a block {pull}23349[#23349]
+
+Java High Level REST Client::
+* Correctly parse BulkItemResponse.Failure's status {pull}23432[#23432]
+
+Java REST Client::
+* Make buffer limit configurable in HeapBufferedConsumerFactory {pull}23970[#23970] (issue: {issue}23958[#23958])
+* RestClient asynchronous execution should not throw exceptions {pull}23307[#23307]
+
+Mapping::
+* Preserve response headers when creating an index {pull}23950[#23950] (issue: {issue}23947[#23947])
+* Improves disabled fielddata error message {pull}23841[#23841] (issue: {issue}22768[#22768])
+* Switch include_in_all in multifield to warning {pull}23656[#23656] (issues: {issue}21971[#21971], {issue}23654[#23654])
+* Fix MapperService StackOverflowError {pull}23605[#23605] (issue: {issue}23604[#23604])
+* Fix NPE with scaled floats stats when field is not indexed {pull}23528[#23528] (issue: {issue}23487[#23487])
+
+Network::
+* Fix possible hang in local transport when nodes get concurrently disconnected {pull}23962[#23962] (issue: {issue}23942[#23942])
+* Respect promises on pipelined responses {pull}23317[#23317] (issues: {issue}23310[#23310], {issue}23322[#23322])
+* Ensure that releasing listener is called {pull}23310[#23310]
+
+Packaging::
+* Fall back to non-atomic move when removing plugins {pull}23548[#23548] (issue: {issue}35[#35])
+
+Percolator::
+* Fix memory leak when percolator uses bitset or field data cache {pull}24115[#24115] (issue: {issue}24108[#24108])
+
+Plugin Ingest Attachment::
+* Remove support for Visio and potm files {pull}23214[#23214] (issues: {issue}22077[#22077], {issue}22079[#22079], {issue}22963[#22963])
+
+Plugin Lang Painless::
+* Fix painless's regex lexer and error messages {pull}23634[#23634]
+* Replace Painless's Cast with casting strategies {pull}23369[#23369]
+* Fix Bad Casts In Painless {pull}23282[#23282] (issue: {issue}23238[#23238])
+
+Plugin Repository Azure::
+* Azure blob store's readBlob() method first checks if the blob exists {pull}23483[#23483] (issue: {issue}23480[#23480])
+
+Plugin Repository S3::
+* Handle BlobPath's trailing separator case. Add test cases to BlobPathTests.java {pull}23091[#23091]
+
+Plugins::
+* Fix delete of plugin directory on remove plugin {pull}24266[#24266] (issue: {issue}24252[#24252])
+* Use a marker file when removing a plugin {pull}24252[#24252] (issue: {issue}24231[#24231])
+
+Query DSL::
+* FuzzyQueryBuilder should error when parsing array of values {pull}23762[#23762] (issue: {issue}23759[#23759])
+
+REST::
+* [API] change wait_for_completion default according to docs {pull}23672[#23672]
+* Deprecate request_cache for clear-cache {pull}23638[#23638] (issue: {issue}22748[#22748])
+* HTTP transport stashes the ThreadContext instead of the RestController {pull}23456[#23456]
+* Ensure we try to autodetect content type for handlers that support plain text {pull}23452[#23452]
+* Fix date format in warning headers {pull}23418[#23418] (issue: {issue}23275[#23275])
+* Align REST specs for HEAD requests {pull}23313[#23313] (issue: {issue}21125[#21125])
+* Correct warning header to be compliant {pull}23275[#23275] (issue: {issue}22986[#22986])
+* Fix get HEAD requests {pull}23186[#23186] (issue: {issue}21125[#21125])
+* Fix search scroll request with a plain text body {pull}23183[#23183] (issue: {issue}22691[#22691])
+* Handle bad HTTP requests {pull}23153[#23153] (issue: {issue}23034[#23034])
+* Fix get source HEAD requests {pull}23151[#23151] (issue: {issue}21125[#21125])
+* Properly encode location header {pull}23133[#23133] (issues: {issue}21057[#21057], {issue}23115[#23115])
+* Fix template HEAD requests {pull}23130[#23130] (issue: {issue}21125[#21125])
+* Fix index HEAD requests {pull}23112[#23112] (issue: {issue}21125[#21125])
+* Fix alias HEAD requests {pull}23094[#23094] (issue: {issue}21125[#21125])
+
+Recovery::
+* Provide target allocation id as part of start recovery request {pull}24333[#24333] (issue: {issue}24167[#24167])
+
+Reindex API::
+* Fix throttled reindex_from_remote {pull}23953[#23953] (issues: {issue}23828[#23828], {issue}23945[#23945])
+* Fix reindex with a remote source on a version before 2.0.0 {pull}23805[#23805]
+* Make reindex wait for cleanup before responding {pull}23677[#23677] (issue: {issue}23653[#23653])
+
+Scripting::
+* Remove unnecessary Groovy deprecation logging {pull}23410[#23410] (issue: {issue}23401[#23401])
+* Convert script/template objects to json format internally {pull}23308[#23308] (issue: {issue}23245[#23245])
+* Script: Fix value of `ctx._now` to be current epoch time in milliseconds {pull}23175[#23175] (issue: {issue}23169[#23169])
+
+Search::
+* Cross Cluster Search: propagate original indices per cluster {pull}24328[#24328]
+* Query string default field {pull}24214[#24214]
+* Speed up parsing of large `terms` queries. {pull}24210[#24210]
+* IndicesQueryCache should delegate the scorerSupplier method. {pull}24209[#24209]
+* Fork LRUQueryCache from Lucene to work around LUCENE-7749 {pull}24207[#24207]
+* Disable graph analysis at query time for shingle and cjk filters producing tokens of different size {pull}23920[#23920] (issue: {issue}23918[#23918])
+* Fix cross-cluster remote node gateway attributes {pull}23863[#23863]
+* Use a fixed seed for computing term hashCode in TermsSliceQuery {pull}23795[#23795]
+* Honor max concurrent searches in multi-search {pull}23538[#23538] (issue: {issue}23527[#23527])
+* Avoid stack overflow in multi-search {pull}23527[#23527] (issue: {issue}23523[#23523])
+* Fix query_string_query to transform "foo:*" in an exists query on the field name {pull}23433[#23433] (issue: {issue}23356[#23356])
+* Factor out filling of TopDocs in SearchPhaseController {pull}23380[#23380] (issues: {issue}19356[#19356], {issue}23357[#23357])
+* Replace blocking calls in ExpandCollapseSearchResponseListener by asynchronous requests {pull}23053[#23053] (issue: {issue}23048[#23048])
+
+Search Templates::
+* No longer add illegal content type option to stored search templates {pull}24251[#24251] (issue: {issue}24227[#24227])
+
+Settings::
+* Do not set path.data in environment if not set {pull}24132[#24132] (issue: {issue}24099[#24099])
+* Correct handling of default and array settings {pull}24074[#24074] (issues: {issue}23981[#23981], {issue}24052[#24052])
+* Fix merge scheduler config settings {pull}23391[#23391]
+* Settings: Fix keystore cli prompting for yes/no to handle console returning null {pull}23320[#23320]
+
+Similarities::
+* Fix similarity upgrade when "default" similarity is overridden {pull}23163[#23163]
+
+Snapshot/Restore::
+* Fixes maintaining the shards a snapshot is waiting on {pull}24289[#24289]
+* Fixes snapshot status on failed snapshots {pull}23833[#23833] (issue: {issue}23716[#23716])
+* Fixes snapshot deletion handling on in-progress snapshot failure {pull}23703[#23703] (issue: {issue}23663[#23663])
+* Prioritize listing index-N blobs over index.latest in reading snapshots {pull}23333[#23333]
+
+Stats::
+* Avoid overflow when computing total FS stats {pull}23641[#23641]
+* Handle existence of cgroup version 2 hierarchy {pull}23493[#23493] (issue: {issue}23486[#23486])
+* Handle long overflow when adding paths' totals {pull}23293[#23293] (issue: {issue}23093[#23093])
+* Fix control group pattern {pull}23219[#23219] (issue: {issue}23218[#23218])
+* Fix total disk bytes returning negative value {pull}23093[#23093]
+
+
+
+[[regression-5.4.0]]
+[float]
+=== Regressions
+
+Bulk::
+* Fix _bulk response when it can't create an index {pull}24048[#24048] (issues: {issue}22488[#22488], {issue}24028[#24028])
+
+
+
+[[upgrade-5.4.0]]
+[float]
+=== Upgrades
+
+Aggregations::
+* Upgrade HDRHistogram to 2.1.9 {pull}23254[#23254] (issue: {issue}23239[#23239])
+
+Core::
+* Upgrade to Lucene 6.5.0 {pull}23750[#23750]
+* Upgrade from JNA 4.2.2 to JNA 4.4.0 {pull}23636[#23636]
+* Upgrade to lucene-6.5.0-snapshot-d00c5ca {pull}23385[#23385]
+* Upgrade to lucene-6.5.0-snapshot-f919485. {pull}23087[#23087]
+
+Logging::
+* Upgrade to Log4j 2.8.2 {pull}23995[#23995]
+
+Network::
+* Upgrade to Netty 4.1.9 {pull}23540[#23540] (issues: {issue}23172[#23172], {issue}6308[#6308], {issue}6374[#6374])
+* Upgrade to Netty 4.1.8 {pull}23055[#23055]
+
+Plugin Repository Azure::
+* Update to Azure Storage 5.0.0 {pull}23517[#23517] (issue: {issue}23448[#23448])
+
diff --git a/docs/reference/release-notes/5.4.1.asciidoc b/docs/reference/release-notes/5.4.1.asciidoc
new file mode 100644
index 0000000000000..25f44dec8d0ae
--- /dev/null
+++ b/docs/reference/release-notes/5.4.1.asciidoc
@@ -0,0 +1,106 @@
+[[release-notes-5.4.1]]
+== 5.4.1 Release Notes
+
+See https://www.elastic.co/blog/multi-data-path-bug-in-elasticsearch-5-3-0[Multi data path bug in Elasticsearch 5.3.0].
+
+Also see <<breaking-changes-5.4>>.
+
+
+[[enhancement-5.4.1]]
+[float]
+=== Enhancements
+
+Aggregations::
+* Allow scripted metric agg to access `_score` {pull}24295[#24295]
+
+Core::
+* Improve bootstrap checks error messages {pull}24548[#24548]
+
+Internal::
+* Log JVM arguments on startup {pull}24451[#24451]
+
+Network::
+* Disable the Netty recycler in the client {pull}24793[#24793] (issues: {issue}22452[#22452], {issue}24721[#24721])
+* Remove Netty logging hack {pull}24653[#24653] (issues: {issue}24469[#24469], {issue}5624[#5624], {issue}6568[#6568], {issue}6696[#6696])
+* Set available processors for Netty {pull}24420[#24420] (issue: {issue}6224[#6224])
+
+Snapshot/Restore::
+* Ensure every repository has an incompatible-snapshots blob {pull}24403[#24403] (issue: {issue}22267[#22267])
+
+Stats::
+* Show JVM arguments {pull}24450[#24450]
+
+
+
+[[bug-5.4.1]]
+[float]
+=== Bug fixes
+
+Aggregations::
+* Fix ArrayIndexOutOfBoundsException when no ranges are specified in the query {pull}23241[#23241] (issue: {issue}22881[#22881])
+
+Allocation::
+* Discard stale node responses from async shard fetching {pull}24434[#24434] (issue: {issue}24007[#24007])
+
+Core::
+* Fix cache expire after access {pull}24546[#24546]
+
+Index APIs::
+* Fix legacy GeoPointField decoding in FieldStats {pull}24534[#24534] (issue: {issue}24275[#24275])
+* Validates updated settings on closed indices {pull}24487[#24487] (issue: {issue}23787[#23787])
+
+Internal::
+* Remove `_UNRELEASED` from Version constants for released versions {pull}24764[#24764]
+* Avoid race when shutting down controller processes {pull}24579[#24579]
+* Fix NPE if field caps request has a field that exists not in all indices {pull}24504[#24504]
+
+More Like This::
+* Pass over _routing value with more_like_this items to be retrieved {pull}24679[#24679] (issue: {issue}23699[#23699])
+
+Network::
+* Notify onConnectionClosed rather than onNodeDisconnect to prune transport handlers {pull}24639[#24639] (issues: {issue}24557[#24557], {issue}24575[#24575], {issue}24632[#24632])
+* SniffNodesSampler should close connection after handling responses {pull}24632[#24632] (issues: {issue}22828[#22828], {issue}24557[#24557], {issue}24575[#24575])
+
+Packaging::
+* Handle parentheses in batch file path {pull}24731[#24731] (issue: {issue}24712[#24712])
+
+Percolator::
+* Fix range queries with date range based on current time in percolator queries. {pull}24666[#24666] (issue: {issue}23921[#23921])
+
+Plugin Repository HDFS::
+* Fixing permission errors for `KERBEROS` security mode for HDFS Repository {pull}23439[#23439] (issue: {issue}22156[#22156])
+
+Plugins::
+* X-Pack plugin download fails on Windows desktop [ISSUE] {pull}24570[#24570]
+* Fix plugin installation permissions {pull}24527[#24527] (issue: {issue}24480[#24480])
+
+Reindex API::
+* Reindex: don't duplicate _source parameter {pull}24629[#24629] (issue: {issue}24628[#24628])
+* Add qa module that tests reindex-from-remote against pre-5.0 versions of Elasticsearch {pull}24561[#24561] (issues: {issue}23828[#23828], {issue}24520[#24520])
+
+Scroll::
+* Fix single shard scroll within a cluster with nodes in version `>= 5.3` and `<= 5.3` {pull}24512[#24512]
+
+Search::
+* Fix ExpandSearchPhase when response contains no hits {pull}24688[#24688] (issue: {issue}24672[#24672])
+* Include all aliases including non-filtering in  `_search_shards` response {pull}24489[#24489]
+
+Settings::
+* Allow resetting settings that use an IP validator {pull}24713[#24713] (issue: {issue}24709[#24709])
+
+Snapshot/Restore::
+* Removes completed snapshot from cluster state on master change {pull}24605[#24605] (issue: {issue}24452[#24452])
+* Fix inefficient (worst case exponential) loading of snapshot repository {pull}24510[#24510] (issue: {issue}24509[#24509])
+
+
+
+[[upgrade-5.4.1]]
+[float]
+=== Upgrades
+
+Core::
+* Upgrade to Lucene 6.5.1 {pull}24399[#24399]
+
+Network::
+* Upgrade to Netty 4.1.11.Final {pull}24652[#24652]
+* Upgrade Netty to 4.1.10.Final {pull}24414[#24414]
diff --git a/docs/reference/release-notes/5.4.2.asciidoc b/docs/reference/release-notes/5.4.2.asciidoc
new file mode 100644
index 0000000000000..203ede912f734
--- /dev/null
+++ b/docs/reference/release-notes/5.4.2.asciidoc
@@ -0,0 +1,52 @@
+[[release-notes-5.4.2]]
+== 5.4.2 Release Notes
+
+See https://www.elastic.co/blog/multi-data-path-bug-in-elasticsearch-5-3-0[Multi data path bug in Elasticsearch 5.3.0].
+
+Also see <<breaking-changes-5.4>>.
+
+[[enhancement-5.4.2]]
+[float]
+=== Enhancements
+
+Core::
+* Explicitly reject duplicate data paths {pull}25178[#25178]
+
+Packaging::
+* Set number of processes in systemd unit file {pull}24970[#24970] (issue: {issue}20874[#20874])
+
+
+
+[[bug-5.4.2]]
+[float]
+=== Bug fixes
+
+Aggregations::
+* Terms aggregation should remap global ordinal buckets when a sub-aggregator is used to sort the terms {pull}24941[#24941] (issue: {issue}24788[#24788])
+
+Analysis::
+* PatternAnalyzer should lowercase wildcard queries when `lowercase` is true. {pull}24967[#24967]
+
+Circuit Breakers::
+* Checks the circuit breaker before allocating bytes for a new big array {pull}25010[#25010] (issue: {issue}24790[#24790])
+
+Geo::
+* Fix GeoPoint FieldStats ternary logic bug {pull}25211[#25211] (issue: {issue}24275[#24275])
+* Fix GeoDistance Ordinal for BWC {pull}25033[#25033] (issue: {issue}24816[#24816])
+
+Index APIs::
+* Rollover max docs should only count primaries {pull}24977[#24977] (issue: {issue}24217[#24217])
+
+Nested Docs::
+* Fix the `max` score mode. {pull}25216[#25216] (issue: {issue}24647[#24647])
+
+Network::
+* Ensure pending transport handlers are invoked for all channel failures {pull}25150[#25150]
+
+Packaging::
+* When stopping via systemd only kill the JVM, not its control group {pull}25195[#25195]
+
+Suggesters::
+* Fix context suggester to read values from keyword type field {pull}24200[#24200] (issue: {issue}24129[#24129])
+
+
diff --git a/docs/reference/release-notes/5.4.3.asciidoc b/docs/reference/release-notes/5.4.3.asciidoc
new file mode 100644
index 0000000000000..7a8cff66be1c6
--- /dev/null
+++ b/docs/reference/release-notes/5.4.3.asciidoc
@@ -0,0 +1,21 @@
+[[release-notes-5.4.3]]
+== 5.4.3 Release Notes
+
+[[enhancement-5.4.3]]
+[float]
+=== Enhancements
+
+Snapshot/Restore::
+* Improves snapshot logging and snapshot deletion error handling {pull}25264[#25264]
+
+
+
+[[bug-5.4.3]]
+[float]
+=== Bug fixes
+
+Snapshot/Restore::
+* Remove redundant and broken MD5 checksum from repository-s3 {pull}25270[#25270] (issue: {issue}25269[#25269])
+
+
+
diff --git a/docs/reference/release-notes/5.5.0.asciidoc b/docs/reference/release-notes/5.5.0.asciidoc
new file mode 100644
index 0000000000000..37e05c26d56d5
--- /dev/null
+++ b/docs/reference/release-notes/5.5.0.asciidoc
@@ -0,0 +1,383 @@
+[[release-notes-5.5.0]]
+== 5.5.0 Release Notes
+
+See https://www.elastic.co/blog/multi-data-path-bug-in-elasticsearch-5-3-0[Multi data path bug in Elasticsearch 5.3.0].
+
+Also see <<breaking-changes-5.5>>.
+
+
+[[breaking-5.5.0]]
+[float]
+=== Breaking changes
+
+Plugins::
+* Remove hidden file leniency from spawner {pull}24079[#24079] (issue: {issue}23980[#23980])
+
+Settings::
+* Remove support for default settings {pull}24093[#24093] (issues: {issue}23981[#23981], {issue}24052[#24052], {issue}24074[#24074])
+
+
+
+[[breaking-java-5.5.0]]
+[float]
+=== Breaking Java changes
+
+Aggregations::
+* Remove the unused SignificantTerms.compareTerm() method {pull}24714[#24714]
+* Make SignificantTerms.Bucket an interface rather than an abstract class {pull}24670[#24670] (issue: {issue}24492[#24492])
+* Make Terms.Bucket an interface rather than an abstract class {pull}24492[#24492]
+
+Plugin Delete By Query::
+* Move DeleteByQuery and Reindex requests into core {pull}24578[#24578]
+
+
+
+[[deprecation-5.5.0]]
+[float]
+=== Deprecations
+
+Index APIs::
+* Deprecated use of + in index expressions {pull}24585[#24585] (issue: {issue}24515[#24515])
+
+Packaging::
+* Add deprecation warnings for $ES_USER and $ES_GROUP {pull}24190[#24190] (issue: {issue}23989[#23989])
+
+Plugin Repository S3::
+* S3 Repository: Deprecate remaining `repositories.s3.*` settings {pull}24144[#24144] (issue: {issue}24143[#24143])
+
+REST::
+* Add deprecation logging for comma-separated feature parsing {pull}25040[#25040] (issue: {issue}24723[#24723])
+
+Scripting::
+* Scripting: Deprecate native scripts {pull}24692[#24692] (issue: {issue}19966[#19966])
+* Scripting: Deprecate index lookup {pull}24691[#24691] (issue: {issue}19359[#19359])
+* Deprecate Fine Grain Settings for Scripts {pull}24573[#24573] (issue: {issue}24532[#24532])
+* Scripting: Deprecate file script settings {pull}24555[#24555] (issue: {issue}21798[#21798])
+* Scripting: Deprecate file scripts {pull}24552[#24552] (issue: {issue}21798[#21798])
+
+Settings::
+* Settings: Update settings deprecation from yml to yaml {pull}24663[#24663] (issue: {issue}19391[#19391])
+* Deprecate settings in .yml and .json {pull}24059[#24059] (issue: {issue}19391[#19391])
+
+Tribe Node::
+* Deprecate tribe service {pull}24598[#24598] (issue: {issue}24581[#24581])
+
+
+
+[[feature-5.5.0]]
+[float]
+=== New features
+
+Mapping::
+* Add new ip_range field type {pull}24433[#24433]
+
+Plugin Analysis ICU::
+* Add ICUCollationFieldMapper {pull}24126[#24126]
+
+
+
+[[enhancement-5.5.0]]
+[float]
+=== Enhancements
+
+Aggregations::
+* Add superset size to Significant Term REST response {pull}24865[#24865]
+* Add document count to Matrix Stats aggregation response {pull}24776[#24776]
+*  Share XContent rendering code in significant terms aggregations {pull}24677[#24677]
+* Add `count` to rest output of `geo_centroid` {pull}24387[#24387] (issue: {issue}24366[#24366])
+* Allow scripted metric agg to access `_score` {pull}24295[#24295]
+
+Aliases::
+* Handle multiple aliases in _cat/aliases api {pull}23698[#23698] (issue: {issue}23661[#23661])
+
+Circuit Breakers::
+* Closing a ReleasableBytesStreamOutput closes the underlying BigArray {pull}23941[#23941]
+
+Cluster::
+* Add a cluster block that allows to delete indices that are read-only {pull}24678[#24678]
+* Extract a common base class to allow services to listen to remote cluster config updates {pull}24367[#24367]
+
+Core::
+* Explicitly reject duplicate data paths {pull}25178[#25178]
+* Use SPI in High Level Rest Client to load XContent parsers {pull}25097[#25097]
+* Improve bootstrap checks error messages {pull}24548[#24548]
+* Regex upgrades {pull}24316[#24316] (issue: {issue}24226[#24226])
+* Detect remnants of path.data/default.path.data bug {pull}24099[#24099] (issues: {issue}23981[#23981], {issue}24052[#24052], {issue}24074[#24074], {issue}24093[#24093])
+
+Engine::
+* Add refresh stats tracking for realtime get {pull}25052[#25052] (issue: {issue}24806[#24806])
+* Engine: store maxUnsafeAutoIdTimestamp in commit {pull}24149[#24149]
+
+Exceptions::
+* IllegalStateException: Only duplicated jar instead of classpath {pull}24953[#24953]
+
+Index APIs::
+* Open and close index to honour allow_no_indices option {pull}24222[#24222] (issue: {issue}24031[#24031])
+
+Ingest::
+* add `exclude_keys` option to KeyValueProcessor {pull}24876[#24876] (issue: {issue}23856[#23856])
+
+Inner Hits::
+* Reuse inner hit query weight {pull}24571[#24571] (issue: {issue}23917[#23917])
+
+Internal::
+* "shard started" should show index and shard ID {pull}25157[#25157]
+* Add the ability to set eager_global_ordinals in the new parent-join field {pull}25019[#25019]
+* Add assertions enabled helper {pull}24834[#24834]
+* Add cross cluster support to `_field_caps` {pull}24463[#24463] (issue: {issue}24334[#24334])
+* Log JVM arguments on startup {pull}24451[#24451]
+* Preserve cluster alias throughout search execution to lookup nodes by cluster and ID {pull}24438[#24438]
+* Move RemoteClusterService into TransportService {pull}24424[#24424]
+* Enum related performance additions. {pull}24274[#24274] (issue: {issue}24226[#24226])
+
+Java API::
+* Always Accumulate Transport Exceptions {pull}25017[#25017] (issue: {issue}23099[#23099])
+
+Java High Level REST Client::
+* Add doc_count to ParsedMatrixStats {pull}24952[#24952] (issue: {issue}24776[#24776])
+* Add fromXContent method to ClearScrollResponse {pull}24909[#24909]
+* ClearScrollRequest to implement ToXContentObject {pull}24907[#24907]
+* SearchScrollRequest to implement ToXContentObject {pull}24906[#24906] (issue: {issue}3889[#3889])
+* Backport aggs parsers for high level REST Client {pull}24844[#24844] (issues: {issue}22278[#22278], {issue}22668[#22668], {issue}23826[#23826], {issue}23965[#23965], {issue}23973[#23973], {issue}23974[#23974], {issue}24085[#24085], {issue}24155[#24155], {issue}24160[#24160], {issue}24162[#24162], {issue}24182[#24182], {issue}24183[#24183], {issue}24208[#24208], {issue}24213[#24213], {issue}24239[#24239], {issue}24284[#24284], {issue}24291[#24291], {issue}24312[#24312], {issue}24330[#24330], {issue}24365[#24365], {issue}24371[#24371], {issue}24442[#24442], {issue}24521[#24521], {issue}24524[#24524], {issue}24559[#24559], {issue}24564[#24564], {issue}24583[#24583], {issue}24589[#24589], {issue}24648[#24648], {issue}24667[#24667], {issue}24675[#24675], {issue}24682[#24682], {issue}24700[#24700], {issue}24706[#24706], {issue}24717[#24717], {issue}24720[#24720], {issue}24738[#24738], {issue}24746[#24746], {issue}24789[#24789], {issue}24791[#24791], {issue}24794[#24794], {issue}24822[#24822], {issue}24824[#24824])
+* Add utility method to parse named XContent objects with typed prefix {pull}24240[#24240] (issue: {issue}22965[#22965])
+
+Logging::
+* Warn on not enough masters during election {pull}20063[#20063] (issue: {issue}8362[#8362])
+
+Mapping::
+* Do not index `_type` when there is at most one type. {pull}24363[#24363]
+* Only allow one type on 6.0 indices {pull}24317[#24317] (issue: {issue}15613[#15613])
+* token_count type : add an option to count tokens (fix #23227) {pull}24175[#24175] (issue: {issue}23227[#23227])
+
+Network::
+* Disable the Netty recycler in the client {pull}24793[#24793] (issues: {issue}22452[#22452], {issue}24721[#24721])
+* Remove Netty logging hack {pull}24653[#24653] (issues: {issue}24469[#24469], {issue}5624[#5624], {issue}6568[#6568], {issue}6696[#6696])
+* Set available processors for Netty {pull}24420[#24420] (issue: {issue}6224[#6224])
+
+Packaging::
+* Set number of processes in systemd unit file {pull}24970[#24970] (issue: {issue}20874[#20874])
+
+Plugin Lang Painless::
+* Optimize instance creation in LambdaBootstrap {pull}24618[#24618]
+* Allow painless to load stored fields {pull}24290[#24290]
+
+Plugin Repository GCS::
+* GCS Repository: Add secure storage of credentials {pull}24697[#24697]
+
+Plugins::
+* Add purge option to remove plugin CLI {pull}24981[#24981]
+* Move ReindexAction class to core {pull}24684[#24684] (issue: {issue}24578[#24578])
+* Plugins: Add support for platform specific plugins {pull}24265[#24265]
+
+REST::
+* Make ObjectParser support string to boolean conversion {pull}24668[#24668] (issue: {issue}21802[#21802])
+* Allow passing single scrollID in clear scroll API body {pull}24242[#24242] (issue: {issue}24233[#24233])
+
+Scripting::
+* Add Ability to Specify No Types/Contexts Allowed For Scripts {pull}24782[#24782]
+* Add New Security Script Settings {pull}24637[#24637] (issue: {issue}24532[#24532])
+
+Search::
+* Eliminate array access in tight loops when profiling is enabled. {pull}24959[#24959]
+* Support Multiple Inner Hits on a Field Collapse Request {pull}24517[#24517]
+* Expand cross cluster search indices for search requests to the concrete index or to it's aliases {pull}24502[#24502]
+* Remove leniency when merging fetched hits in a search response phase {pull}24158[#24158]
+
+Search Templates::
+* Add max concurrent searches to multi template search {pull}24255[#24255] (issues: {issue}20912[#20912], {issue}21907[#21907])
+
+Settings::
+* Emit settings deprecation logging at most once {pull}25457[#25457]
+
+Snapshot/Restore::
+* Improves snapshot logging and snapshot deletion error handling {pull}25264[#25264]
+* Enhances get snapshots API to allow retrieving repository index only {pull}24477[#24477] (issue: {issue}24288[#24288])
+* Ensure every repository has an incompatible-snapshots blob {pull}24403[#24403] (issue: {issue}22267[#22267])
+* Change snapshot status error to use generic SnapshotException {pull}24355[#24355] (issue: {issue}24225[#24225])
+
+Stats::
+* Show JVM arguments {pull}24450[#24450]
+
+Suggesters::
+* Provide informative error message in case of unknown suggestion context. {pull}24241[#24241]
+* Allow different data types for category in Context suggester {pull}23491[#23491] (issue: {issue}22358[#22358])
+
+Upgrade API::
+* Allow plugins to upgrade templates and index metadata on startup {pull}24379[#24379]
+
+
+
+[[bug-5.5.0]]
+[float]
+=== Bug fixes
+
+Aggregations::
+* Terms aggregation should remap global ordinal buckets when a sub-aggregator is used to sort the terms {pull}24941[#24941] (issue: {issue}24788[#24788])
+* Correctly set doc_count when MovAvg "predicts" values on existing buckets {pull}24892[#24892] (issue: {issue}24327[#24327])
+* InternalPercentilesBucket should not rely on ordered percents array {pull}24336[#24336] (issue: {issue}24331[#24331])
+* Align behavior HDR percentiles iterator with percentile() method {pull}24206[#24206]
+* DateHistogram: Fix `extended_bounds` with `offset` {pull}23789[#23789] (issue: {issue}23776[#23776])
+* Fix ArrayIndexOutOfBoundsException when no ranges are specified in the query {pull}23241[#23241] (issue: {issue}22881[#22881])
+
+Aliases::
+* GET aliases should 404 if aliases are missing {pull}25043[#25043] (issue: {issue}24644[#24644])
+
+Allocation::
+* Discard stale node responses from async shard fetching {pull}24434[#24434] (issue: {issue}24007[#24007])
+
+Analysis::
+* PatternAnalyzer should lowercase wildcard queries when `lowercase` is true. {pull}24967[#24967]
+
+Bulk::
+* Reject empty IDs {pull}24118[#24118] (issue: {issue}24116[#24116])
+
+Cache::
+* fix bug of weight computation {pull}24856[#24856]
+
+Circuit Breakers::
+* Checks the circuit breaker before allocating bytes for a new big array {pull}25010[#25010] (issue: {issue}24790[#24790])
+
+Core::
+* Fix cache expire after access {pull}24546[#24546]
+* Check for default.path.data included in path.data {pull}24285[#24285] (issue: {issue}24283[#24283])
+
+Geo::
+* Fix GeoPoint FieldStats ternary logic bug {pull}25211[#25211] (issue: {issue}24275[#24275])
+* Fix GeoDistance Ordinal for BWC {pull}25033[#25033] (issue: {issue}24816[#24816])
+
+Highlighting::
+* Higlighters: Fix MultiPhrasePrefixQuery rewriting {pull}25103[#25103] (issue: {issue}25088[#25088])
+
+Index APIs::
+* Rollover max docs should only count primaries {pull}24977[#24977] (issue: {issue}24217[#24217])
+* Fix legacy GeoPointField decoding in FieldStats {pull}24534[#24534] (issue: {issue}24275[#24275])
+* Validates updated settings on closed indices {pull}24487[#24487] (issue: {issue}23787[#23787])
+* Fixes restore of a shrunken index when initial recovery node is gone {pull}24322[#24322] (issue: {issue}24257[#24257])
+
+Ingest::
+* fix grok's pattern parsing to validate pattern names in expression {pull}25063[#25063] (issue: {issue}22831[#22831])
+* Fix floating-point error when DateProcessor parses UNIX {pull}24947[#24947]
+* add option for _ingest.timestamp to use new ZonedDateTime (5.x backport) {pull}24030[#24030] (issues: {issue}23168[#23168], {issue}23174[#23174])
+
+Inner Hits::
+* Fix Source filtering in new field collapsing feature {pull}24068[#24068] (issue: {issue}24063[#24063])
+
+Internal::
+* Fix race condition in RemoteClusterConnection node supplier {pull}25432[#25432]
+* Initialise empty lists in BaseTaskResponse constructor {pull}25290[#25290]
+* Extract a common base class for scroll executions {pull}24979[#24979] (issue: {issue}16555[#16555])
+* Ensure remote cluster is connected before fetching `_field_caps` {pull}24845[#24845] (issue: {issue}24763[#24763])
+* Obey lock order if working with store to get metadata snapshots {pull}24787[#24787] (issue: {issue}24481[#24481])
+* Remove `_UNRELEASED` from Version constants for released versions {pull}24764[#24764]
+* Fix `_field_caps` serialization in order to support cross cluster search {pull}24722[#24722]
+* Avoid race when shutting down controller processes {pull}24579[#24579]
+* Fix NPE if field caps request has a field that exists not in all indices {pull}24504[#24504]
+
+More Like This::
+* Pass over _routing value with more_like_this items to be retrieved {pull}24679[#24679] (issue: {issue}23699[#23699])
+
+Nested Docs::
+* In case of a single type the _id field should be added to the nested document instead of _uid field {pull}25149[#25149]
+
+Network::
+* Ensure pending transport handlers are invoked for all channel failures {pull}25150[#25150]
+* Notify onConnectionClosed rather than onNodeDisconnect to prune transport handlers {pull}24639[#24639] (issues: {issue}24557[#24557], {issue}24575[#24575], {issue}24632[#24632])
+* SniffNodesSampler should close connection after handling responses {pull}24632[#24632] (issues: {issue}22828[#22828], {issue}24557[#24557], {issue}24575[#24575])
+
+Packaging::
+* When stopping via systemd only kill the JVM, not its control group {pull}25195[#25195]
+* Handle parentheses in batch file path {pull}24731[#24731] (issue: {issue}24712[#24712])
+
+Percolator::
+* Fix range queries with date range based on current time in percolator queries. {pull}24666[#24666] (issue: {issue}23921[#23921])
+* For legacy indices rewrite percolator query upon percolation time {pull}24617[#24617] (issue: {issue}24485[#24485])
+* Fix memory leak when percolator uses bitset or field data cache {pull}24115[#24115] (issue: {issue}24108[#24108])
+
+Plugin Lang Painless::
+* painless: Fix method references to ctor with the new LambdaBootstrap and cleanup code {pull}24406[#24406]
+* Fix Painless Lambdas for Java 9 {pull}24070[#24070] (issue: {issue}23473[#23473])
+
+Plugins::
+* X-Pack plugin download fails on Windows desktop {pull}24570[#24570]
+* Fix plugin installation permissions {pull}24527[#24527] (issue: {issue}24480[#24480])
+* Fix delete of plugin directory on remove plugin {pull}24266[#24266] (issue: {issue}24252[#24252])
+* Use a marker file when removing a plugin {pull}24252[#24252] (issue: {issue}24231[#24231])
+* Remove hidden file leniency from plugin service {pull}23982[#23982] (issue: {issue}12465[#12465])
+
+Query DSL::
+* Thread falls into infinite loop when processing Indices query {pull}24736[#24736] (issue: {issue}24735[#24735])
+
+REST::
+* Fix handling of exceptions thrown on HEAD requests {pull}25172[#25172] (issue: {issue}21125[#21125])
+* Fixed NPEs caused by requests without content. {pull}23497[#23497] (issue: {issue}24701[#24701])
+* Fix get mappings HEAD requests {pull}23192[#23192] (issue: {issue}21125[#21125])
+
+Recovery::
+* Provide target allocation id as part of start recovery request {pull}24333[#24333] (issue: {issue}24167[#24167])
+
+Reindex API::
+* Reindex: don't duplicate _source parameter {pull}24629[#24629] (issue: {issue}24628[#24628])
+* Add qa module that tests reindex-from-remote against pre-5.0 versions of Elasticsearch {pull}24561[#24561] (issues: {issue}23828[#23828], {issue}24520[#24520])
+
+Scroll::
+* Fix single shard scroll within a cluster with nodes in version `>= 5.3` and `<= 5.3` {pull}24512[#24512]
+
+Search::
+* Adds check for negative search request size {pull}25397[#25397] (issue: {issue}22530[#22530])
+* Do not search locally if remote index pattern resolves to no indices {pull}25436[#25436] (issue: {issue}25426[#25426])
+* Make sure range queries are correctly profiled. {pull}25108[#25108]
+* Compute the took time of the query after the expand phase of field collapsing {pull}24902[#24902] (issue: {issue}24900[#24900])
+* Fix RangeFieldMapper rangeQuery to properly handle relations {pull}24808[#24808] (issue: {issue}24744[#24744])
+* Fix ExpandSearchPhase when response contains no hits {pull}24688[#24688] (issue: {issue}24672[#24672])
+* Include all aliases including non-filtering in  `_search_shards` response {pull}24489[#24489]
+* Cross Cluster Search: propagate original indices per cluster {pull}24328[#24328]
+
+Search Templates::
+* No longer add illegal content type option to stored search templates {pull}24251[#24251] (issue: {issue}24227[#24227])
+
+Settings::
+* Keystore CLI should use the AddFileKeyStoreCommand for files {pull}25298[#25298]
+* Settings: Fix setting groups to include secure settings {pull}25076[#25076] (issue: {issue}25069[#25069])
+* Allow resetting settings that use an IP validator {pull}24713[#24713] (issue: {issue}24709[#24709])
+* Do not set path.data in environment if not set {pull}24132[#24132] (issue: {issue}24099[#24099])
+
+Snapshot/Restore::
+* Output all empty snapshot info fields if in verbose mode {pull}25455[#25455] (issue: {issue}24477[#24477])
+* Remove redundant and broken MD5 checksum from repository-s3 {pull}25270[#25270] (issue: {issue}25269[#25269])
+* Consolidates the logic for cleaning up snapshots on master election {pull}24894[#24894] (issue: {issue}24605[#24605])
+* Removes completed snapshot from cluster state on master change {pull}24605[#24605] (issue: {issue}24452[#24452])
+* Fix inefficient (worst case exponential) loading of snapshot repository {pull}24510[#24510] (issue: {issue}24509[#24509])
+* Fixes maintaining the shards a snapshot is waiting on {pull}24289[#24289]
+
+Stats::
+* _nodes/stats should not fail due to concurrent AlreadyClosedException {pull}25016[#25016] (issue: {issue}23099[#23099])
+* Avoid double decrement on current query counter {pull}24922[#24922] (issues: {issue}22996[#22996], {issue}24872[#24872])
+
+Suggesters::
+* Fix context suggester to read values from keyword type field {pull}24200[#24200] (issue: {issue}24129[#24129])
+
+Task Manager::
+* Task Management: Make TaskInfo parsing forwards compatible {pull}24073[#24073] (issue: {issue}23250[#23250])
+
+
+
+[[regression-5.5.0]]
+[float]
+=== Regressions
+
+Highlighting::
+* Fix Fast Vector Highlighter NPE on match phrase prefix {pull}25116[#25116] (issue: {issue}25088[#25088])
+
+
+
+[[upgrade-5.5.0]]
+[float]
+=== Upgrades
+
+Core::
+* Upgrade to Lucene 6.5.1 {pull}24399[#24399]
+
+Network::
+* Upgrade to Netty 4.1.11.Final {pull}24652[#24652]
+* Upgrade Netty to 4.1.10.Final {pull}24414[#24414]
+
diff --git a/docs/reference/release-notes/5.5.1.asciidoc b/docs/reference/release-notes/5.5.1.asciidoc
new file mode 100644
index 0000000000000..c6dbfc9d5dbb8
--- /dev/null
+++ b/docs/reference/release-notes/5.5.1.asciidoc
@@ -0,0 +1,30 @@
+[[release-notes-5.5.1]]
+== 5.5.1 Release Notes
+
+See https://www.elastic.co/blog/multi-data-path-bug-in-elasticsearch-5-3-0[Multi data path bug in Elasticsearch 5.3.0].
+
+Also see <<breaking-changes-5.5>>.
+
+
+[[bug-5.5.1]]
+[float]
+=== Bug fixes
+
+Aliases::
+* mget with an alias shouldn't ignore alias routing {pull}25697[#25697] (issue: {issue}25696[#25696])
+
+Engine::
+* Engine - Do not store operations that are not index into lucene in the translog (5.x only) {pull}25592[#25592]
+
+Internal::
+* Fix BytesReferenceStreamInput#skip with offset {pull}25634[#25634]
+
+Packaging::
+* Stop disabling explicit GC {pull}25759[#25759]
+* Avoid failing install if system-sysctl is masked {pull}25657[#25657] (issue: {issue}24234[#24234])
+* Get short path name for native controllers {pull}25344[#25344]
+
+Query DSL::
+* WrapperQueryBuilder should also rewrite the parsed query {pull}25480[#25480]
+
+
diff --git a/docs/reference/release-notes/5.5.2.asciidoc b/docs/reference/release-notes/5.5.2.asciidoc
new file mode 100644
index 0000000000000..2658d9fe21479
--- /dev/null
+++ b/docs/reference/release-notes/5.5.2.asciidoc
@@ -0,0 +1,26 @@
+[[release-notes-5.5.2]]
+== 5.5.2 Release Notes
+
+See https://www.elastic.co/blog/multi-data-path-bug-in-elasticsearch-5-3-0[Multi data path bug in Elasticsearch 5.3.0].
+
+Also see <<breaking-changes-5.5>>.
+
+[[bug-5.5.2]]
+[float]
+=== Bug fixes
+
+Aggregations::
+* Fixes array out of bounds for value count agg {pull}26038[#26038] (issue: {issue}17379[#17379])
+
+Core::
+* Release operation permit on thread-pool rejection {pull}25930[#25930] (issue: {issue}25863[#25863])
+
+Inner Hits::
+* Fix inner hits to work with queries wrapped in an indices query {pull}26138[#26138] (issue: {issue}26133[#26133])
+* When fetching nested inner hits only access stored fields when needed {pull}25864[#25864] (issue: {issue}6[#6])
+
+Logging::
+* Declare XContent deprecation logger as static {pull}25881[#25881] (issue: {issue}25879[#25879])
+
+Query DSL::
+* Parse "*" in query_string_query as MatchAllDocsQuery {pull}25872[#25872] (issues: {issue}25556[#25556], {issue}25726[#25726])
diff --git a/docs/reference/release-notes/5.5.3.asciidoc b/docs/reference/release-notes/5.5.3.asciidoc
new file mode 100644
index 0000000000000..9b4e0f6404325
--- /dev/null
+++ b/docs/reference/release-notes/5.5.3.asciidoc
@@ -0,0 +1,30 @@
+[[release-notes-5.5.3]]
+== 5.5.3 Release Notes
+
+See https://www.elastic.co/blog/multi-data-path-bug-in-elasticsearch-5-3-0[Multi data path bug in Elasticsearch 5.3.0].
+
+Also see <<breaking-changes-5.5>>.
+
+[[breaking-msi-5.5.3]]
+[float]
+=== Breaking MSI Windows Installer changes
+
+Upgrades from 5.5.0, 5.5.1, 5.5.2. See https://github.com/elastic/windows-installers/releases/tag/v5.5.3[release notes]::
+* `ES_HOME` and `ES_CONFIG` environment variables unset after successful upgrade
+* Windows Service stopped after successful upgrade
+
+[[deprecation-5.5.3]]
+[float]
+=== Deprecations
+
+Mapping::
+* Add deprecation logging when _all is enabled {pull}26228[#26228] (issue: {issue}22144[#22144])
+
+
+
+[[bug-5.5.3]]
+[float]
+=== Bug fixes
+
+Packaging::
+* setgid on /etc/elasticearch on package install {pull}26412[#26412] (issue: {issue}26410[#26410])
diff --git a/docs/reference/release-notes/5.6.0.asciidoc b/docs/reference/release-notes/5.6.0.asciidoc
new file mode 100644
index 0000000000000..4412248be01dd
--- /dev/null
+++ b/docs/reference/release-notes/5.6.0.asciidoc
@@ -0,0 +1,303 @@
+[[release-notes-5.6.0]]
+== 5.6.0 Release Notes
+
+Also see <<breaking-changes-5.6>>.
+
+[[breaking-msi-5.6.0]]
+[float]
+=== Breaking MSI Windows Installer changes
+
+Upgrades from 5.5.0, 5.5.1, 5.5.2. See https://github.com/elastic/windows-installers/releases/tag/v5.6.0[release notes]::
+* `ES_HOME` and `ES_CONFIG` environment variables unset after successful upgrade
+* Windows Service stopped after successful upgrade
+
+[[breaking-java-5.6.0]]
+[float]
+=== Breaking Java changes
+
+Java High Level REST Client::
+* Allow RestHighLevelClient to use plugins {pull}25024[#25024]
+
+Java REST Client::
+* Rename client artifacts {pull}25693[#25693] (issue: {issue}20248[#20248])
+
+
+
+[[deprecation-5.6.0]]
+[float]
+=== Deprecations
+
+Highlighting::
+* Postings highlighter deprecation {pull}25073[#25073]
+
+Indexed Scripts/Templates::
+* Scripting: Deprecate stored search template apis {pull}25437[#25437] (issue: {issue}24596[#24596])
+
+Mapping::
+* Add deprecation logging when _all is enabled {pull}26228[#26228] (issue: {issue}22144[#22144])
+
+Query DSL::
+* Deprecate omitting terms lookup index name in 5.6 {pull}25755[#25755] (issue: {issue}25750[#25750])
+
+Scripting::
+* Scripting: Deprecate scripts.max_compilation_per_minute setting {pull}26402[#26402]
+* Scripting: Change keys for inline/stored scripts to source/id {pull}25127[#25127]
+
+Tribe Node::
+* Deprecate tribe service {pull}24598[#24598] (issue: {issue}24581[#24581])
+
+
+
+[[feature-5.6.0]]
+[float]
+=== New features
+
+Java High Level REST Client::
+* Backport high level REST client {pull}25086[#25086] (issues: {issue}22371[#22371], {issue}22680[#22680], {issue}23040[#23040], {issue}23187[#23187], {issue}23266[#23266], {issue}23312[#23312], {issue}23328[#23328], {issue}23350[#23350], {issue}23351[#23351], {issue}23373[#23373], {issue}24796[#24796], {issue}24938[#24938], {issue}25038[#25038])
+
+Parent/Child::
+* Typeless parent child backport {pull}25104[#25104]
+* Move parent_id query to the parent-join module {pull}25072[#25072] (issue: {issue}20257[#20257])
+* Introduce ParentJoinFieldMapper, a field mapper that creates parent/child relation within documents of the same index {pull}24978[#24978] (issue: {issue}20257[#20257])
+
+Upgrade API::
+* TemplateUpgraders should be called during rolling restart {pull}25263[#25263] (issues: {issue}24379[#24379], {issue}24680[#24680])
+
+
+
+[[enhancement-5.6.0]]
+[float]
+=== Enhancements
+
+Allocation::
+* Adjust status on bad allocation explain requests {pull}25503[#25503] (issue: {issue}25458[#25458])
+* Promote replica on the highest version node {pull}25277[#25277] (issue: {issue}10708[#10708])
+
+Analysis::
+* [analysis-icu] Allow setting unicodeSetFilter {pull}20814[#20814] (issue: {issue}20820[#20820])
+
+CAT API::
+* expand `/_cat/nodes` to return information about hard drive {pull}21775[#21775] (issue: {issue}21679[#21679])
+
+Cluster::
+* Validate a joining node's version with version of existing cluster nodes {pull}25808[#25808]
+
+Core::
+* Add max file size bootstrap check {pull}25974[#25974]
+* Explicitly reject duplicate data paths {pull}25178[#25178]
+
+Highlighting::
+* Picks offset source for the unified highlighter directly from the es mapping {pull}25747[#25747] (issue: {issue}25699[#25699])
+
+Index APIs::
+* Create index request should return the index name {pull}25139[#25139] (issue: {issue}23044[#23044])
+
+Ingest::
+* Add Ingest-Processor specific Rest Endpoints & Add Grok endpoint {pull}25059[#25059] (issue: {issue}24725[#24725])
+* Port support for commercial GeoIP2 databases from Logstash. {pull}24889[#24889]
+* Allow removing multiple fields in ingest processor {pull}24750[#24750] (issue: {issue}24622[#24622])
+* Add target_field parameter to ingest processors {pull}24133[#24133] (issues: {issue}23228[#23228], {issue}23682[#23682])
+
+Internal::
+* Use holder pattern for lazy deprecation loggers {pull}26218[#26218] (issue: {issue}26210[#26210])
+* "shard started" should show index and shard ID {pull}25157[#25157]
+* Break out clear scroll logic from TransportClearScrollAction {pull}25125[#25125] (issue: {issue}25094[#25094])
+* Add helper methods to TransportActionProxy to identify proxy actions and requests {pull}25124[#25124]
+* Add remote cluster infrastructure to fetch discovery nodes. {pull}25123[#25123] (issue: {issue}25094[#25094])
+* Disallow multiple parent-join fields per mapping {pull}25002[#25002]
+
+Java High Level REST Client::
+* [DOCS] restructure java clients docs pages {pull}25517[#25517]
+* Use SPI in High Level Rest Client to load XContent parsers {pull}25098[#25098] (issues: {issue}25024[#25024], {issue}25097[#25097])
+
+Java REST Client::
+* Shade external dependencies in the rest client jar {pull}25780[#25780] (issue: {issue}25208[#25208])
+* RestClient uses system properties and system default SSLContext {pull}25757[#25757] (issue: {issue}23231[#23231])
+
+Logging::
+* Use LRU set to reduce repeat deprecation messages {pull}25474[#25474] (issue: {issue}25457[#25457])
+
+Mapping::
+* Optimize `terms` queries on `ip` addresses to use a `PointInSetQuery` whenever possible. {pull}25669[#25669] (issue: {issue}25667[#25667])
+
+Network::
+* Simplify connection closing and cleanups in TcpTransport {pull}25250[#25250]
+
+Packaging::
+* Remove memlock suggestion from systemd service {pull}25979[#25979]
+* Set address space limit in systemd service file {pull}25975[#25975]
+
+Parent/Child::
+* Remove ParentJoinFieldSubFetchPhase {pull}25550[#25550] (issue: {issue}25363[#25363])
+* Support parent id being specified as number in the _source {pull}25547[#25547]
+
+Query DSL::
+* Make slop optional when parsing `span_near` query {pull}25677[#25677] (issue: {issue}25642[#25642])
+
+Search::
+* Reduce the overhead of timeouts and low-level search cancellation. {pull}25776[#25776]
+* Reduce profiling overhead. {pull}25772[#25772] (issue: {issue}24799[#24799])
+* Backport `can_match` endpoint to 5.6 to allow 6.0 to use the optimization in mixed version {pull}25704[#25704] (issue: {issue}25658[#25658])
+* Add a shard filter search phase to pre-filter shards based on query rewriting {pull}25658[#25658]
+* Ensure we rewrite common queries to `match_none` if possible {pull}25650[#25650]
+* Limit the number of concurrent shard requests per search request {pull}25632[#25632]
+* Add cluster name validation to RemoteClusterConnection {pull}25568[#25568]
+* Leverage scorerSupplier when applicable. {pull}25109[#25109]
+* Add Cross Cluster Search support for scroll searches {pull}25094[#25094]
+
+Sequence IDs::
+* 5.x - Move primary term from ReplicationRequest to ConcreteShardRequest {pull}25824[#25824] (issue: {issue}25822[#25822])
+* Forward compatibility for primary context handoff on 6.x {pull}25545[#25545] (issue: {issue}25468[#25468])
+
+Settings::
+* Emit settings deprecation logging at most once {pull}25457[#25457]
+* IndexMetaData: Introduce internal format index setting {pull}25292[#25292]
+
+Snapshot/Restore::
+* Improves snapshot logging and snapshot deletion error handling {pull}25264[#25264]
+
+Stats::
+* Update `IndexShard#refreshMetric` via a `ReferenceManager.RefreshListener` {pull}25083[#25083] (issues: {issue}24806[#24806], {issue}25052[#25052])
+
+
+
+[[bug-5.6.0]]
+[float]
+=== Bug fixes
+
+Aggregations::
+* Fixes array out of bounds for value count agg {pull}26038[#26038] (issue: {issue}17379[#17379])
+
+Aliases::
+* mget with an alias shouldn't ignore alias routing {pull}25697[#25697] (issue: {issue}25696[#25696])
+
+Allocation::
+* Allow wildcards for shard IP filtering {pull}26187[#26187] (issues: {issue}22591[#22591], {issue}26184[#26184])
+
+Analysis::
+* Pre-configured shingle filter should disable graph analysis {pull}25853[#25853] (issue: {issue}25555[#25555])
+
+CRUD::
+* Serialize and expose timeout of acknowledged requests in REST layer (ES 5.6) {pull}26213[#26213] (issue: {issue}26189[#26189])
+
+Cluster::
+* Register setting `cluster.indices.tombstones.size` {pull}26193[#26193] (issue: {issue}26191[#26191])
+
+Core::
+* Release operation permit on thread-pool rejection {pull}25930[#25930] (issue: {issue}25863[#25863])
+* Node should start up despite of a lingering `.es_temp_file` {pull}21210[#21210] (issue: {issue}21007[#21007])
+
+Discovery::
+* MasterNodeChangePredicate should use the node instance to detect master change {pull}25877[#25877] (issue: {issue}25471[#25471])
+
+Engine::
+* Engine - Do not store operations that are not index into lucene in the translog (5.x only) {pull}25592[#25592]
+
+Geo::
+* Fix GeoPoint FieldStats ternary logic bug {pull}25211[#25211] (issue: {issue}24275[#24275])
+
+Highlighting::
+* Fix nested query highlighting {pull}26305[#26305] (issue: {issue}26230[#26230])
+* FastVectorHighlighter should not cache the field query globally {pull}25197[#25197] (issue: {issue}25171[#25171])
+
+Index APIs::
+* Shrink API should ignore templates {pull}25380[#25380] (issue: {issue}25035[#25035])
+* Rollover max docs should only count primaries {pull}24977[#24977] (issue: {issue}24217[#24217])
+
+Ingest::
+* Sort Processor does not have proper behavior with targetField {pull}25237[#25237] (issue: {issue}24133[#24133])
+
+Inner Hits::
+* Fix inner hits to work with queries wrapped in an indices query {pull}26138[#26138] (issue: {issue}26133[#26133])
+* When fetching nested inner hits only access stored fields when needed {pull}25864[#25864] (issue: {issue}6[#6])
+
+Internal::
+* Fix BytesReferenceStreamInput#skip with offset {pull}25634[#25634]
+* Fix race condition in RemoteClusterConnection node supplier {pull}25432[#25432]
+* Initialise empty lists in BaseTaskResponse constructor {pull}25290[#25290]
+
+Java High Level REST Client::
+* Register ip_range aggregation with the high level client {pull}26383[#26383]
+* add top hits as a parsed aggregation to the rest high level client {pull}26370[#26370]
+
+Logging::
+* Lazy initialize deprecation logger in parser {pull}26210[#26210] (issue: {issue}25879[#25879])
+* Allow not configure logging without config {pull}26209[#26209] (issues: {issue}20575[#20575], {issue}24076[#24076])
+* Declare XContent deprecation logger as static {pull}25881[#25881] (issue: {issue}25879[#25879])
+
+Mapping::
+* ICUCollationKeywordFieldMapper use SortedSetDocValuesField {pull}26267[#26267]
+* Fixed bug that mapper_parsing_exception is thrown for numeric field with ignore_malformed=true when inserting "NaN" {pull}25967[#25967] (issue: {issue}25289[#25289])
+* Coerce decimal strings for whole number types by truncating the decimal part {pull}25835[#25835] (issue: {issue}25819[#25819])
+* Fix parsing of ip range queries. {pull}25768[#25768] (issue: {issue}25636[#25636])
+* Remove `fielddata:false` from mappings of non-analyzed strings. {pull}25542[#25542]
+* Disable date field mapping changing {pull}25285[#25285] (issue: {issue}25271[#25271])
+* token_count datatype should handle null value {pull}25046[#25046] (issue: {issue}24928[#24928])
+* keep _parent field while updating child type mapping {pull}24407[#24407] (issue: {issue}23381[#23381])
+
+Nested Docs::
+* In case of a single type the _id field should be added to the nested document instead of _uid field {pull}25149[#25149]
+
+Network::
+* Release pipelined http responses on close {pull}26226[#26226]
+* Ensure pending transport handlers are invoked for all channel failures {pull}25150[#25150]
+
+Packaging::
+* setgid on /etc/elasticearch on package install {pull}26412[#26412] (issue: {issue}26410[#26410])
+* Stop disabling explicit GC {pull}25759[#25759]
+* Avoid failing install if system-sysctl is masked {pull}25657[#25657] (issue: {issue}24234[#24234])
+* Get short path name for native controllers {pull}25344[#25344]
+* When stopping via systemd only kill the JVM, not its control group {pull}25195[#25195]
+
+Parent/Child::
+* The default _parent field should not try to load global ordinals {pull}25851[#25851] (issue: {issue}25849[#25849])
+
+Plugin Lang Painless::
+* Painless: allow doubles to be casted to longs. {pull}25936[#25936]
+
+Plugin Repository HDFS::
+* Upgrading HDFS Repository Plugin to use HDFS 2.8.1 Client {pull}25497[#25497] (issue: {issue}25450[#25450])
+
+Query DSL::
+* Parse "*" in query_string_query as MatchAllDocsQuery {pull}25872[#25872] (issues: {issue}25556[#25556], {issue}25726[#25726])
+* SpanNearQueryBuilder should return the inner clause when a single clause is provided {pull}25856[#25856] (issue: {issue}25630[#25630])
+* WrapperQueryBuilder should also rewrite the parsed query {pull}25480[#25480]
+
+REST::
+* Fix handling of exceptions thrown on HEAD requests {pull}25172[#25172] (issue: {issue}21125[#21125])
+
+Search::
+* Fix term(s) query for range field {pull}25918[#25918]
+* Caching a MinDocQuery can lead to wrong results. {pull}25909[#25909]
+* First increment shard stats before notifying and potentially sending response {pull}25818[#25818]
+* Remove assertion about deviation when casting to a float. {pull}25806[#25806] (issue: {issue}25330[#25330])
+* Prevent skipping shards if a suggest builder is present {pull}25739[#25739] (issue: {issue}25658[#25658])
+* Ensure remote cluster alias is preserved in inner hits aggs {pull}25627[#25627] (issue: {issue}25606[#25606])
+* Do not search locally if remote index pattern resolves to no indices {pull}25436[#25436] (issue: {issue}25426[#25426])
+
+Settings::
+* Fix settings serialization to not serialize secure settings or not take the total size into account {pull}25323[#25323]
+* Keystore CLI should use the AddFileKeyStoreCommand for files {pull}25298[#25298]
+
+Snapshot/Restore::
+* Snapshot/Restore: fix NPE while handling null failure reasons {pull}26127[#26127] (issues: {issue}25878[#25878], {issue}25941[#25941])
+* Snapshot/Restore: Ensure that shard failure reasons are correctly stored in CS {pull}25941[#25941] (issue: {issue}25878[#25878])
+* Output all empty snapshot info fields if in verbose mode {pull}25455[#25455] (issue: {issue}24477[#24477])
+
+Stats::
+* _nodes/stats should not fail due to concurrent AlreadyClosedException {pull}25016[#25016] (issue: {issue}23099[#23099])
+
+Suggesters::
+* Context suggester should filter doc values field {pull}25858[#25858] (issue: {issue}25404[#25404])
+
+
+
+[[upgrade-5.6.0]]
+[float]
+=== Upgrades
+
+Network::
+* Upgrade to Netty 4.1.13.Final {pull}25581[#25581] (issues: {issue}24729[#24729], {issue}6866[#6866])
+
+Upgrade API::
+* Improve stability and logging of TemplateUpgradeServiceIT tests {pull}25386[#25386] (issue: {issue}25382[#25382])
diff --git a/docs/reference/release-notes/5.6.1.asciidoc b/docs/reference/release-notes/5.6.1.asciidoc
new file mode 100644
index 0000000000000..e80ed6ff4b4f6
--- /dev/null
+++ b/docs/reference/release-notes/5.6.1.asciidoc
@@ -0,0 +1,26 @@
+[[release-notes-5.6.1]]
+== 5.6.1 Release Notes
+
+Also see <<breaking-changes-5.6>>.
+
+[[bug-5.6.1]]
+[float]
+=== Bug fixes
+
+Highlighting::
+* Fix percolator highlight sub fetch phase to not highlight query twice {pull}26622[#26622]
+
+Mapping::
+* Fix unmapped_type creation for indices created in 2.x  {pull}26602[#26602] (issue: {issue}26162[#26162])
+
+Similarities::
+* Add boolean similarity to built in similarity types {pull}26613[#26613]
+
+
+
+[[upgrade-5.6.1]]
+[float]
+=== Upgrades
+
+Core::
+* Upgrade to Lucene-6.6.1. {pull}26535[#26535]
diff --git a/docs/reference/release-notes/5.6.2.asciidoc b/docs/reference/release-notes/5.6.2.asciidoc
new file mode 100644
index 0000000000000..6aa5d64ee957b
--- /dev/null
+++ b/docs/reference/release-notes/5.6.2.asciidoc
@@ -0,0 +1,47 @@
+[[release-notes-5.6.2]]
+== 5.6.2 Release Notes
+
+Also see <<breaking-changes-5.6>>.
+
+[[bug-5.6.2]]
+[float]
+=== Bug fixes
+
+Aggregations::
+* Do not delegate a null scorer to LeafBucketCollectors {pull}26747[#26747] (issue: {issue}26611[#26611])
+
+Inner Hits::
+* Do not allow inner hits that fetch _source and have a non nested object field as parent {pull}25749[#25749] (issue: {issue}25315[#25315])
+
+Internal::
+* Catch exceptions and inform handler in RemoteClusterConnection#collectNodes {pull}26725[#26725] (issue: {issue}26700[#26700])
+
+Java API::
+* BulkProcessor flush runnable preserves the thread context from creation time {pull}26718[#26718] (issue: {issue}26596[#26596])
+
+Java High Level REST Client::
+* Make RestHighLevelClient's Request class public {pull}26627[#26627] (issue: {issue}26455[#26455])
+
+Plugin Analysis Kuromoji::
+* Fix kuromoji default stoptags {pull}26600[#26600] (issue: {issue}26519[#26519])
+
+Plugin Repository HDFS::
+* Fix permission errors when using Read Only HDFS Repository {pull}26714[#26714] (issues: {issue}22793[#22793], {issue}26513[#26513])
+
+Query DSL::
+*  #26083 Fixed incomplete JSON body on count request making org.elasticsearch.rest.action.RestActions#parseTopLevelQueryBuilder go into endless loop {pull}26680[#26680] (issue: {issue}26083[#26083])
+
+Scripting::
+* Add missing script contexts and types allowed settings {pull}26703[#26703] (issue: {issue}26651[#26651])
+
+Upgrade API::
+* Upgrade API: fix excessive logging and unnecessary template updates {pull}26698[#26698] (issue: {issue}26673[#26673])
+
+
+
+[[upgrade-5.6.2]]
+[float]
+=== Upgrades
+
+Logging::
+* Upgrade to Log4j 2.9.1 {pull}26750[#26750] (issues: {issue}109[#109], {issue}26464[#26464], {issue}26467[#26467])
diff --git a/docs/reference/release-notes/5.6.3.asciidoc b/docs/reference/release-notes/5.6.3.asciidoc
new file mode 100644
index 0000000000000..d926f950f6e46
--- /dev/null
+++ b/docs/reference/release-notes/5.6.3.asciidoc
@@ -0,0 +1,33 @@
+[[release-notes-5.6.3]]
+== 5.6.3 Release Notes
+
+Also see <<breaking-changes-5.6>>.
+
+[[enhancement-5.6.3]]
+[float]
+=== Enhancements
+
+Network::
+* Add additional low-level logging handler {pull}26887[#26887]
+* Unwrap exception causes when checking if  dying {pull}26884[#26884]
+
+
+[[bug-5.6.3]]
+[float]
+=== Bug fixes
+
+Aggregations::
+* Fix IndexOutOfBoundsException in histograms for NaN doubles (#26787) {pull}26856[#26856] (issue: {issue}26787[#26787])
+
+Core::
+* MetaData Builder doesn't properly prevent an alias with the same name as an index {pull}26804[#26804]
+
+Packaging::
+* Fix default paths for Windows service {pull}26875[#26875] (issue: {issue}26873[#26873])
+
+Plugin Repository Azure::
+* Use Azure upload method instead of our own implementation (#26751) {pull}26839[#26839] (issue: {issue}26751[#26751])
+
+Search::
+* Fix serialization errors when cross cluster search goes to a single shard {pull}26881[#26881] (issue: {issue}26833[#26833])
+
diff --git a/docs/reference/search.asciidoc b/docs/reference/search.asciidoc
index a4de20ee21392..b6da09b26bb89 100644
--- a/docs/reference/search.asciidoc
+++ b/docs/reference/search.asciidoc
@@ -18,13 +18,14 @@ when indexing tweets, the routing value can be the user name:
 
 [source,js]
 --------------------------------------------------
-$ curl -XPOST 'http://localhost:9200/twitter/tweet?routing=kimchy' -d '{
+POST /twitter/tweet?routing=kimchy
+{
     "user" : "kimchy",
     "postDate" : "2009-11-15T14:12:12",
     "message" : "trying out Elasticsearch"
 }
-'
 --------------------------------------------------
+// CONSOLE
 
 In such a case, if we want to search only on the tweets for a specific
 user, we can specify it as the routing, resulting in the search hitting
@@ -32,7 +33,8 @@ only the relevant shard:
 
 [source,js]
 --------------------------------------------------
-$ curl -XGET 'http://localhost:9200/twitter/tweet/_search?routing=kimchy' -d '{
+POST /twitter/tweet/_search?routing=kimchy
+{
     "query": {
         "bool" : {
             "must" : {
@@ -46,8 +48,9 @@ $ curl -XGET 'http://localhost:9200/twitter/tweet/_search?routing=kimchy' -d '{
         }
     }
 }
-'
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 The routing parameter can be multi valued represented as a comma
 separated string. This will result in hitting the relevant shards where
@@ -65,6 +68,7 @@ the request with two different groups:
 
 [source,js]
 --------------------------------------------------
+POST /_search
 {
     "query" : {
         "match_all" : {}
@@ -72,6 +76,8 @@ the request with two different groups:
     "stats" : ["group1", "group2"]
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
 
 [float]
 [[global-search-timeout]]
@@ -86,6 +92,19 @@ timeout. The setting key is `search.default_search_timeout` and can be
 set using the <<cluster-update-settings>> endpoints. Setting this value
 to `-1` resets the global search timeout to no timeout.
 
+[float]
+[[global-search-cancellation]]
+== Search Cancellation
+
+Searches can be cancelled using standard <<task-cancellation,task cancellation>>
+mechanism. By default, a running search only checks if it is cancelled or
+not on segment boundaries, therefore the cancellation can be delayed by large
+segments. The search cancellation responsiveness can be improved by setting
+the dynamic cluster-level setting `search.low_level_cancellation` to `true`.
+However, it comes with an additional overhead of more frequent cancellation
+checks that can be noticeable on large fast running search queries. Changing this
+setting only affects the searches that start after the change is made.
+
 --
 
 include::search/search.asciidoc[]
@@ -113,3 +132,5 @@ include::search/profile.asciidoc[]
 include::search/percolate.asciidoc[]
 
 include::search/field-stats.asciidoc[]
+
+include::search/field-caps.asciidoc[]
diff --git a/docs/reference/search/count.asciidoc b/docs/reference/search/count.asciidoc
index 859455e89b72c..dbab691867f54 100644
--- a/docs/reference/search/count.asciidoc
+++ b/docs/reference/search/count.asciidoc
@@ -39,6 +39,7 @@ tweets from the twitter index for a certain user. The result is:
     "_shards" : {
         "total" : 5,
         "successful" : 5,
+        "skipped" : 0,
         "failed" : 0
     }
 }
@@ -74,9 +75,6 @@ query.
 |`lenient` |If set to true will cause format based failures (like
 providing text to a numeric field) to be ignored. Defaults to false.
 
-|`lowercase_expanded_terms` |Should terms be automatically lowercased or
-not. Defaults to `true`.
-
 |`analyze_wildcard` |Should wildcard and prefix queries be analyzed or
 not. Defaults to `false`.
 
diff --git a/docs/reference/search/explain.asciidoc b/docs/reference/search/explain.asciidoc
index 125f3124bff4d..61a8ac641c072 100644
--- a/docs/reference/search/explain.asciidoc
+++ b/docs/reference/search/explain.asciidoc
@@ -15,35 +15,87 @@ Full query example:
 
 [source,js]
 --------------------------------------------------
-curl -XGET 'localhost:9200/twitter/tweet/1/_explain' -d '{
+GET /twitter/tweet/0/_explain
+{
       "query" : {
-        "term" : { "message" : "search" }
+        "match" : { "message" : "elasticsearch" }
       }
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
 
 This will yield the following result:
 
 [source,js]
 --------------------------------------------------
 {
-  "matches" : true,
-  "explanation" : {
-    "value" : 0.15342641,
-    "description" : "fieldWeight(message:search in 0), product of:",
-    "details" : [ {
-      "value" : 1.0,
-      "description" : "tf(termFreq(message:search)=1)"
-    }, {
-      "value" : 0.30685282,
-      "description" : "idf(docFreq=1, maxDocs=1)"
-    }, {
-      "value" : 0.5,
-      "description" : "fieldNorm(field=message, doc=0)"
-    } ]
-  }
+   "_index": "twitter",
+   "_type": "tweet",
+   "_id": "0",
+   "matched": true,
+   "explanation": {
+      "value": 1.55077,
+      "description": "weight(message:elasticsearch in 0) [PerFieldSimilarity], result of:",
+      "details": [
+         {
+            "value": 1.55077,
+            "description": "score(doc=0,freq=1.0 = termFreq=1.0\n), product of:",
+            "details": [
+               {
+                  "value": 1.3862944,
+                  "description": "idf, computed as log(1 + (docCount - docFreq + 0.5) / (docFreq + 0.5)) from:",
+                  "details": [
+                     {
+                        "value": 1.0,
+                        "description": "docFreq",
+                        "details": []
+                     },
+                     {
+                        "value": 5.0,
+                        "description": "docCount",
+                        "details": []
+                      }
+                   ]
+               },
+                {
+                  "value": 1.1186441,
+                  "description": "tfNorm, computed as (freq * (k1 + 1)) / (freq + k1 * (1 - b + b * fieldLength / avgFieldLength)) from:",
+                  "details": [
+                     {
+                        "value": 1.0,
+                        "description": "termFreq=1.0",
+                        "details": []
+                     },
+                     {
+                        "value": 1.2,
+                        "description": "parameter k1",
+                        "details": []
+                     },
+                     {
+                        "value": 0.75,
+                        "description": "parameter b",
+                        "details": []
+                     },
+                     {
+                        "value": 5.4,
+                        "description": "avgFieldLength",
+                        "details": []
+                     },
+                     {
+                        "value": 4.0,
+                        "description": "fieldLength",
+                        "details": []
+                     }
+                  ]
+               }
+            ]
+         }
+      ]
+   }
 }
 --------------------------------------------------
+// TESTRESPONSE
 
 There is also a simpler way of specifying the query via the `q`
 parameter. The specified `q` parameter value is then parsed as if the
@@ -52,8 +104,10 @@ explain api:
 
 [source,js]
 --------------------------------------------------
-curl -XGET 'localhost:9200/twitter/tweet/1/_explain?q=message:search'
+GET /twitter/tweet/0/_explain?q=message:search
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
 
 This will yield the same result as the previous request.
 
@@ -66,7 +120,7 @@ This will yield the same result as the previous request.
     Set to `true` to retrieve the `_source` of the document explained. You can also
     retrieve part of the document by using `_source_include` & `_source_exclude` (see <<get-source-filtering,Get API>> for more details)
 
-`fields`::
+`stored_fields`::
     Allows to control which stored fields to return as part of the
     document explained.
 
@@ -99,10 +153,6 @@ This will yield the same result as the previous request.
     Should wildcard and prefix queries be analyzed or
     not. Defaults to false.
 
-`lowercase_expanded_terms`::
-    Should terms be automatically lowercased
-    or not. Defaults to true.
-
 `lenient`::
     If set to true will cause format based failures (like
     providing text to a numeric field) to be ignored. Defaults to false.
diff --git a/docs/reference/search/field-caps.asciidoc b/docs/reference/search/field-caps.asciidoc
new file mode 100644
index 0000000000000..42211342682d4
--- /dev/null
+++ b/docs/reference/search/field-caps.asciidoc
@@ -0,0 +1,126 @@
+[[search-field-caps]]
+== Field Capabilities API
+
+experimental[]
+
+The field capabilities API allows to retrieve the capabilities of fields among multiple indices.
+
+The field capabilities api by default executes on all indices:
+
+[source,js]
+--------------------------------------------------
+GET _field_caps?fields=rating
+--------------------------------------------------
+// CONSOLE
+
+... but the request can also be restricted to specific indices:
+
+[source,js]
+--------------------------------------------------
+GET twitter/_field_caps?fields=rating
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
+
+Alternatively the `fields` option can also be defined in the request body:
+
+[source,js]
+--------------------------------------------------
+POST _field_caps
+{
+   "fields" : ["rating"]
+}
+--------------------------------------------------
+// CONSOLE
+
+This is equivalent to the previous request.
+
+Supported request options:
+
+[horizontal]
+`fields`::  A list of fields to compute stats for. The field name supports wildcard notation. For example, using `text_*`
+            will cause all fields that match the expression to be returned.
+
+[float]
+=== Field Capabilities
+
+The field capabilities api returns the following information per field:
+
+[horizontal]
+`searchable`::
+
+Whether this field is indexed for search on all indices.
+
+`aggregatable`::
+
+Whether this field can be aggregated on all indices.
+
+`indices`::
+
+The list of indices where this field has the same type,
+or null if all indices have the same type for the field.
+
+`non_searchable_indices`::
+
+The list of indices where this field is not searchable,
+or null if all indices have the same definition for the field.
+
+`non_aggregatable_indices`::
+
+The list of indices where this field is not aggregatable,
+or null if all indices have the same definition for the field.
+
+
+[float]
+=== Response format
+
+Request:
+
+[source,js]
+--------------------------------------------------
+GET _field_caps?fields=rating,title
+--------------------------------------------------
+// CONSOLE
+
+[source,js]
+--------------------------------------------------
+{
+    "fields": {
+        "rating": { <1>
+            "long": {
+                "searchable": true,
+                "aggregatable": false,
+                "indices": ["index1", "index2"],
+                "non_aggregatable_indices": ["index1"] <2>
+            },
+            "keyword": {
+                "searchable": false,
+                "aggregatable": true,
+                "indices": ["index3", "index4"],
+                "non_searchable_indices": ["index4"] <3>
+            }
+        },
+        "title": { <4>
+            "text": {
+                "searchable": true,
+                "aggregatable": false
+
+            }
+        }
+    }
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+<1> The field `rating` is defined as a long in `index1` and `index2`
+and as a `keyword` in `index3` and `index4`.
+<2> The field `rating` is not aggregatable in `index1`.
+<3> The field `rating` is not searchable in `index4`.
+<4> The field `title` is defined as `text` in all indices.
+
+
+
+
+
+
+
diff --git a/docs/reference/search/field-stats.asciidoc b/docs/reference/search/field-stats.asciidoc
index 90dbd539d659d..2b4a48651293e 100644
--- a/docs/reference/search/field-stats.asciidoc
+++ b/docs/reference/search/field-stats.asciidoc
@@ -3,6 +3,8 @@
 
 experimental[]
 
+deprecated[5.4.0, `_field_stats` is deprecated, use `_field_caps` instead or run an min/max aggregation on the desired fields]
+
 The field stats api allows one to find statistical properties of a field
 without executing a search, but looking up measurements that are natively
 available in the Lucene index. This can be useful to explore a dataset which
@@ -16,15 +18,20 @@ All indices:
 
 [source,js]
 --------------------------------------------------
-curl -XGET "http://localhost:9200/_field_stats?fields=rating"
+GET _field_stats?fields=rating
 --------------------------------------------------
+// CONSOLE
+// TEST[warning:[_field_stats] endpoint is deprecated! Use [_field_caps] instead or run a min/max aggregations on the desired fields.]
 
 Specific indices:
 
 [source,js]
 --------------------------------------------------
-curl -XGET "http://localhost:9200/index1,index2/_field_stats?fields=rating"
+GET twitter/_field_stats?fields=rating
 --------------------------------------------------
+// CONSOLE
+// TEST[warning:[_field_stats] endpoint is deprecated! Use [_field_caps] instead or run a min/max aggregations on the desired fields.]
+// TEST[setup:twitter]
 
 Supported request options:
 
@@ -38,10 +45,13 @@ Alternatively the `fields` option can also be defined in the request body:
 
 [source,js]
 --------------------------------------------------
-curl -XPOST "http://localhost:9200/_field_stats?level=indices" -d '{
+POST _field_stats?level=indices
+{
    "fields" : ["rating"]
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[warning:[_field_stats] endpoint is deprecated! Use [_field_caps] instead or run a min/max aggregations on the desired fields.]
 
 This is equivalent to the previous request.
 
@@ -114,8 +124,10 @@ Request:
 
 [source,js]
 --------------------------------------------------
-curl -XGET "http://localhost:9200/_field_stats?fields=rating,answer_count,creation_date,display_name"
+GET _field_stats?fields=rating,answer_count,creation_date,display_name
 --------------------------------------------------
+// CONSOLE
+// TEST[warning:[_field_stats] endpoint is deprecated! Use [_field_caps] instead or run a min/max aggregations on the desired fields.]
 
 Response:
 
@@ -226,8 +238,10 @@ Request:
 
 [source,js]
 --------------------------------------------------
-curl -XGET "http://localhost:9200/_field_stats?fields=rating,answer_count,creation_date,display_name&level=indices"
+GET _field_stats?fields=rating,answer_count,creation_date,display_name&level=indices
 --------------------------------------------------
+// CONSOLE
+// TEST[warning:[_field_stats] endpoint is deprecated! Use [_field_caps] instead or run a min/max aggregations on the desired fields.]
 
 Response:
 
@@ -299,7 +313,7 @@ Response:
 
 Field stats index constraints allows to omit all field stats for indices that don't match with the constraint. An index
 constraint can exclude indices' field stats based on the `min_value` and `max_value` statistic. This option is only
-useful if the `level` option is set to `indices`.
+useful if the `level` option is set to `indices`. Fields that are not indexed (not searchable) are always omitted when an index constraint is defined.
 
 For example index constraints can be useful to find out the min and max value of a particular property of your data in
 a time based scenario. The following request only returns field stats for the `answer_count` property for indices
@@ -307,8 +321,9 @@ holding questions created in the year 2014:
 
 [source,js]
 --------------------------------------------------
-curl -XPOST "http://localhost:9200/_field_stats?level=indices" -d '{
-   "fields" : ["answer_count"] <1>
+POST _field_stats?level=indices
+{
+   "fields" : ["answer_count"], <1>
    "index_constraints" : { <2>
       "creation_date" : { <3>
          "max_value" : { <4>
@@ -319,8 +334,10 @@ curl -XPOST "http://localhost:9200/_field_stats?level=indices" -d '{
          }
       }
    }
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[warning:[_field_stats] endpoint is deprecated! Use [_field_caps] instead or run a min/max aggregations on the desired fields.]
 
 <1> The fields to compute and return field stats for.
 <2> The set index constraints. Note that index constrains can be defined for fields that aren't defined in the `fields` option.
@@ -341,8 +358,9 @@ If missing, the format configured in the field's mapping is used.
 
 [source,js]
 --------------------------------------------------
-curl -XPOST "http://localhost:9200/_field_stats?level=indices" -d '{
-   "fields" : ["answer_count"]
+POST _field_stats?level=indices
+{
+   "fields" : ["answer_count"],
    "index_constraints" : {
       "creation_date" : {
          "max_value" : {
@@ -355,7 +373,9 @@ curl -XPOST "http://localhost:9200/_field_stats?level=indices" -d '{
          }
       }
    }
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[warning:[_field_stats] endpoint is deprecated! Use [_field_caps] instead or run a min/max aggregations on the desired fields.]
 
 <1> Custom date format
diff --git a/docs/reference/search/multi-search.asciidoc b/docs/reference/search/multi-search.asciidoc
index d32cd45d87d8f..c68cf0daaf55a 100644
--- a/docs/reference/search/multi-search.asciidoc
+++ b/docs/reference/search/multi-search.asciidoc
@@ -4,9 +4,10 @@
 The multi search API allows to execute several search requests within
 the same API. The endpoint for it is `_msearch`.
 
-The format of the request is similar to the bulk API format, and the
-structure is as follows (the structure is specifically optimized to
-reduce parsing if a specific search ends up redirected to another node):
+The format of the request is similar to the bulk API format and makes
+use of the newline delimited JSON (NDJSON) format. the structure is as
+follows (the structure is specifically optimized to reduce parsing if
+a specific search ends up redirected to another node):
 
 [source,js]
 --------------------------------------------------
@@ -15,6 +16,11 @@ body\n
 header\n
 body\n
 --------------------------------------------------
+// NOTCONSOLE
+
+*NOTE*: the final line of data must end with a newline character `\n`. Each newline character
+may be preceded by a carriage return `\r`. When sending requests to this endpoint the
+`Content-Type` header should be set to `application/x-ndjson`.
 
 The header part includes which index / indices to search on, optional
 (mapping) types to search on, the `search_type`, `preference`, and
@@ -34,9 +40,14 @@ $ cat requests
 {"query" : {"match_all" : {}}}
 {"search_type" : "dfs_query_then_fetch"}
 {"query" : {"match_all" : {}}}
+--------------------------------------------------
+// NOTCONSOLE
 
-$ curl -XGET localhost:9200/_msearch --data-binary "@requests"; echo
+[source,js]
 --------------------------------------------------
+$ curl -H "Content-Type: application/x-ndjson" -XGET localhost:9200/_msearch --data-binary "@requests"; echo
+--------------------------------------------------
+// NOTCONSOLE
 
 Note, the above includes an example of an empty header (can also be just
 without any content) which is supported as well.
@@ -53,20 +64,20 @@ default unless explicitly defined otherwise in the header. For example:
 
 [source,js]
 --------------------------------------------------
-$ cat requests
+GET twitter/_msearch
 {}
 {"query" : {"match_all" : {}}, "from" : 0, "size" : 10}
 {}
 {"query" : {"match_all" : {}}}
-{"index" : "test2"}
+{"index" : "twitter2"}
 {"query" : {"match_all" : {}}}
-
-$ curl -XGET localhost:9200/test/_msearch --data-binary @requests; echo
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
 
-The above will execute the search against the `test` index for all the
+The above will execute the search against the `twitter` index for all the
 requests that don't define an index, and the last one will be executed
-against the `test2` index.
+against the `twitter2` index.
 
 The `search_type` can be set in a similar manner to globally apply to
 all search requests.
@@ -80,3 +91,76 @@ This default is based on the number of data nodes and the default search thread
 === Security
 
 See <<url-access-control>>
+
+[float]
+[[template-msearch]]
+=== Template support
+
+Much like described in <<search-template>> for the _search resource, _msearch
+also provides support for templates. Submit them like follows:
+
+[source,js]
+-----------------------------------------------
+GET _msearch/template
+{"index" : "twitter"}
+{ "source" : "{ \"query\": { \"match\": { \"message\" : \"{{keywords}}\" } } } }", "params": { "query_type": "match", "keywords": "some message" } }
+{"index" : "twitter"}
+{ "source" : "{ \"query\": { \"match_{{template}}\": {} } }", "params": { "template": "all" } }
+-----------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
+
+for inline templates.
+
+You can also create search templates:
+
+[source,js]
+------------------------------------------
+POST /_scripts/my_template_1
+{
+    "script": {
+        "lang": "mustache",
+        "source": {
+            "query": {
+                "match": {
+                    "message": "{{query_string}}"
+                }
+            }
+        }
+    }
+}
+------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
+
+[source,js]
+------------------------------------------
+POST /_scripts/my_template_2
+{
+    "script": {
+        "lang": "mustache",
+        "source": {
+            "query": {
+                "term": {
+                    "{{field}}": "{{value}}"
+                }
+            }
+        }
+    }
+}
+------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+and later use them in a _msearch:
+
+[source,js]
+-----------------------------------------------
+GET _msearch/template
+{"index" : "main"}
+{ "id": "my_template_1", "params": { "query_string": "some message" } }
+{"index" : "main"}
+{ "id": "my_template_2", "params": { "field": "user", "value": "test" } }
+-----------------------------------------------
+// CONSOLE
+// TEST[continued]
diff --git a/docs/reference/search/profile.asciidoc b/docs/reference/search/profile.asciidoc
index e2c22caf6f422..1c85314803fcf 100644
--- a/docs/reference/search/profile.asciidoc
+++ b/docs/reference/search/profile.asciidoc
@@ -17,13 +17,17 @@ Any `_search` request can be profiled by adding a top-level `profile` parameter:
 
 [source,js]
 --------------------------------------------------
-curl -XGET 'localhost:9200/_search' -d '{
+GET /_search
+{
   "profile": true,<1>
   "query" : {
-    "match" : { "message" : "search test" }
+    "match" : { "message" : "message number" }
   }
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
+
 <1> Setting the top-level `profile` parameter to `true` will enable profiling
 for the search
 
@@ -37,73 +41,77 @@ This will yield the following result:
    "_shards": {
       "total": 1,
       "successful": 1,
+      "skipped" : 0,
       "failed": 0
    },
    "hits": {
-      "total": 3,
-      "max_score": 1.078072,
-      "hits": [ ... ] <1>
+      "total": 4,
+      "max_score": 0.5093388,
+      "hits": [...] <1>
    },
    "profile": {
      "shards": [
         {
-           "id": "[2aE02wS1R8q_QFnYu6vDVQ][test][1]",
+           "id": "[2aE02wS1R8q_QFnYu6vDVQ][twitter][0]",
            "searches": [
               {
                  "query": [
                     {
                        "type": "BooleanQuery",
-                       "description": "message:search message:test",
-                       "time": "15.52889800ms",
+                       "description": "message:message message:number",
+                       "time": "1.873811000ms",
+                       "time_in_nanos": "1873811",
                        "breakdown": {
-                          "score": 6352,
-                          "score_count": 1,
-                          "build_scorer": 1800776,
+                          "score": 51306,
+                          "score_count": 4,
+                          "build_scorer": 2935582,
                           "build_scorer_count": 1,
                           "match": 0,
                           "match_count": 0,
-                          "create_weight": 667400,
+                          "create_weight": 919297,
                           "create_weight_count": 1,
-                          "next_doc": 10563,
-                          "next_doc_count": 2,
+                          "next_doc": 53876,
+                          "next_doc_count": 5,
                           "advance": 0,
                           "advance_count": 0
                        },
                        "children": [
                           {
                              "type": "TermQuery",
-                             "description": "message:search",
-                             "time": "4.938855000ms",
+                             "description": "message:message",
+                             "time": "0.3919430000ms",
+                             "time_in_nanos": "391943",
                              "breakdown": {
-                                "score": 0,
-                                "score_count": 0,
-                                "build_scorer": 3230,
+                                "score": 28776,
+                                "score_count": 4,
+                                "build_scorer": 784451,
                                 "build_scorer_count": 1,
                                 "match": 0,
                                 "match_count": 0,
-                                "create_weight": 415612,
+                                "create_weight": 1669564,
                                 "create_weight_count": 1,
-                                "next_doc": 0,
-                                "next_doc_count": 0,
+                                "next_doc": 10111,
+                                "next_doc_count": 5,
                                 "advance": 0,
                                 "advance_count": 0
                              }
                           },
                           {
                              "type": "TermQuery",
-                             "description": "message:test",
-                             "time": "0.5016660000ms",
+                             "description": "message:number",
+                             "time": "0.2106820000ms",
+                             "time_in_nanos": "210682",
                              "breakdown": {
-                                "score": 5014,
-                                "score_count": 1,
-                                "build_scorer": 1689333,
+                                "score": 4552,
+                                "score_count": 4,
+                                "build_scorer": 42602,
                                 "build_scorer_count": 1,
                                 "match": 0,
                                 "match_count": 0,
-                                "create_weight": 166587,
+                                "create_weight": 89323,
                                 "create_weight_count": 1,
-                                "next_doc": 5542,
-                                "next_doc_count": 2,
+                                "next_doc": 2852,
+                                "next_doc_count": 5,
                                 "advance": 0,
                                 "advance_count": 0
                              }
@@ -111,21 +119,36 @@ This will yield the following result:
                        ]
                     }
                  ],
-                 "rewrite_time": 870954,
+                 "rewrite_time": 51443,
                  "collector": [
                     {
-                       "name": "SimpleTopScoreDocCollector",
-                       "reason": "search_top_hits",
-                       "time": "0.009783000000ms"
+                       "name": "CancellableCollector",
+                       "reason": "search_cancelled",
+                       "time": "0.3043110000ms",
+                       "time_in_nanos": "304311",
+                       "children": [
+                         {
+                           "name": "SimpleTopScoreDocCollector",
+                           "reason": "search_top_hits",
+                           "time": "0.03227300000ms",
+                           "time_in_nanos": "32273"
+                         }
+                       ]
                     }
                  ]
               }
-           ]
+           ],
+           "aggregations": []
         }
      ]
    }
 }
 --------------------------------------------------
+// TESTRESPONSE[s/"took": 25/"took": $body.took/]
+// TESTRESPONSE[s/"hits": \[...\]/"hits": $body.$_path/]
+// TESTRESPONSE[s/(?<=[" ])\d+(\.\d+)?[ms]*/$body.$_path/]
+// TESTRESPONSE[s/\[2aE02wS1R8q_QFnYu6vDVQ\]\[twitter\]\[0\]/$body.$_path/]
+
 <1> Search results are returned, but were omitted here for brevity
 
 Even for a simple query, the response is relatively complicated.  Let's break it down piece-by-piece before moving
@@ -139,11 +162,11 @@ First, the overall structure of the profile response is as follows:
    "profile": {
         "shards": [
            {
-              "id": "[2aE02wS1R8q_QFnYu6vDVQ][test][1]",  <1>
+              "id": "[2aE02wS1R8q_QFnYu6vDVQ][twitter][0]",  <1>
               "searches": [
                  {
                     "query": [...],             <2>
-                    "rewrite_time": 870954,     <3>
+                    "rewrite_time": 51443,      <3>
                     "collector": [...]          <4>
                  }
               ],
@@ -153,6 +176,12 @@ First, the overall structure of the profile response is as follows:
      }
 }
 --------------------------------------------------
+// TESTRESPONSE[s/"profile": /"took": $body.took, "timed_out": $body.timed_out, "_shards": $body._shards, "hits": $body.hits, "profile": /]
+// TESTRESPONSE[s/(?<=[" ])\d+(\.\d+)?[ms]*/$body.$_path/]
+// TESTRESPONSE[s/\[2aE02wS1R8q_QFnYu6vDVQ\]\[twitter\]\[0\]/$body.$_path/]
+// TESTRESPONSE[s/"query": \[...\]/"query": $body.$_path/]
+// TESTRESPONSE[s/"collector": \[...\]/"collector": $body.$_path/]
+// TESTRESPONSE[s/"aggregations": \[...\]/"aggregations": []/]
 <1> A profile is returned for each shard that participated in the response, and is identified
 by a unique ID
 <2> Each profile contains a section which holds details about the query execution
@@ -195,73 +224,90 @@ the `advance` phase of that query is the cause, for example.
 The `query` section contains detailed timing of the query tree executed by Lucene on a particular shard.
 The overall structure of this query tree will resemble your original Elasticsearch query, but may be slightly
 (or sometimes very) different.  It will also use similar but not always identical naming.  Using our previous
-`term` query example, let's analyze the `query` section:
+`match` query example, let's analyze the `query` section:
 
 [source,js]
 --------------------------------------------------
 "query": [
     {
        "type": "BooleanQuery",
-       "description": "message:search message:test",
-       "time": "15.52889800ms",
+       "description": "message:message message:number",
+       "time": "1.873811000ms",
+       "time_in_nanos": "1873811",
        "breakdown": {...},               <1>
        "children": [
           {
              "type": "TermQuery",
-             "description": "message:search",
-             "time": "4.938855000ms",
+             "description": "message:message",
+             "time": "0.3919430000ms",
+             "time_in_nanos": "391943",
              "breakdown": {...}
           },
           {
              "type": "TermQuery",
-             "description": "message:test",
-             "time": "0.5016660000ms",
+             "description": "message:number",
+             "time": "0.2106820000ms",
+             "time_in_nanos": "210682",
              "breakdown": {...}
           }
        ]
     }
 ]
 --------------------------------------------------
+// TESTRESPONSE[s/^/{\n"took": $body.took,\n"timed_out": $body.timed_out,\n"_shards": $body._shards,\n"hits": $body.hits,\n"profile": {\n"shards": [ {\n"id": "$body.$_path",\n"searches": [{\n/]
+// TESTRESPONSE[s/]$/],"rewrite_time": $body.$_path, "collector": $body.$_path}], "aggregations": []}]}}/]
+// TESTRESPONSE[s/(?<=[" ])\d+(\.\d+)?[ms]*/$body.$_path/]
+// TESTRESPONSE[s/"breakdown": \{...\}/"breakdown": $body.$_path/]
 <1> The breakdown timings are omitted for simplicity
 
 Based on the profile structure, we can see that our `match` query was rewritten by Lucene into a BooleanQuery with two
-clauses (both holding a TermQuery).  The `"type"` field displays the Lucene class name, and often aligns with
-the equivalent name in Elasticsearch.  The `"description"` field displays the Lucene explanation text for the query, and
-is made available to help differentiating between parts of your query (e.g. both `"message:search"` and `"message:test"`
+clauses (both holding a TermQuery).  The `type` field displays the Lucene class name, and often aligns with
+the equivalent name in Elasticsearch.  The `description` field displays the Lucene explanation text for the query, and
+is made available to help differentiating between parts of your query (e.g. both `message:search` and `message:test`
 are TermQuery's and would appear identical otherwise.
 
-The `"time"` field shows that this query took ~15ms for the entire BooleanQuery to execute.  The recorded time is inclusive
+The `time` field shows that this query took ~1.8ms for the entire BooleanQuery to execute.  The recorded time is inclusive
 of all children.
 
-The `"breakdown"` field will give detailed stats about how the time was spent, we'll look at
-that in a moment.  Finally, the `"children"` array lists any sub-queries that may be present.  Because we searched for two
+The `time_in nanos` field shows the timing information in an exact, machine readable format in nanoseconds.
+
+The `breakdown` field will give detailed stats about how the time was spent, we'll look at
+that in a moment.  Finally, the `children` array lists any sub-queries that may be present.  Because we searched for two
 values ("search test"), our BooleanQuery holds two children TermQueries.  They have identical information (type, time,
 breakdown, etc).  Children are allowed to have their own children.
 
+NOTE: The `time` field is only intended for human consumption. If you need exact timing values use `time_in nanos`. The `time`
+field is currently printed by default, but this will change with the next major version (6.0.0) where `time_in_nanos` will be
+printed by default.
+
+
 ===== Timing Breakdown
 
-The `"breakdown"` component lists detailed timing statistics about low-level Lucene execution:
+The `breakdown` component lists detailed timing statistics about low-level Lucene execution:
 
 [source,js]
 --------------------------------------------------
 "breakdown": {
-    "score": 5014,
-    "score_count": 1,
-    "build_scorer": 1689333,
-    "build_scorer_count": 1,
-    "match": 0,
-    "match_count": 0,
-    "create_weight": 166587,
-    "create_weight_count": 1,
-    "next_doc": 5542,
-    "next_doc_count": 2,
-    "advance": 0,
-    "advance_count": 0
+   "score": 51306,
+   "score_count": 4,
+   "build_scorer": 2935582,
+   "build_scorer_count": 1,
+   "match": 0,
+   "match_count": 0,
+   "create_weight": 919297,
+   "create_weight_count": 1,
+   "next_doc": 53876,
+   "next_doc_count": 5,
+   "advance": 0,
+   "advance_count": 0
 }
 --------------------------------------------------
+// TESTRESPONSE[s/^/{\n"took": $body.took,\n"timed_out": $body.timed_out,\n"_shards": $body._shards,\n"hits": $body.hits,\n"profile": {\n"shards": [ {\n"id": "$body.$_path",\n"searches": [{\n"query": [{\n"type": "BooleanQuery",\n"description": "message:message message:number",\n"time_in_nanos": $body.$_path,\n"time": $body.$_path,/]
+// TESTRESPONSE[s/}$/},\n"children": $body.$_path}],\n"rewrite_time": $body.$_path, "collector": $body.$_path}], "aggregations": []}]}}/]
+// TESTRESPONSE[s/(?<=[" ])\d+(\.\d+)?[ms]*/$body.$_path/]
 
 Timings are listed in wall-clock nanoseconds and are not normalized at all.  All caveats about the overall
-`"time"` apply here.  The intention of the breakdown is to give you a feel for A) what machinery in Lucene is
+`time` apply here.  The intention of the breakdown is to give you a feel for A) what machinery in Lucene is
 actually eating time, and B) the magnitude of differences in times between the various components.  Like the overall time,
 the breakdown is inclusive of all children times.
 
@@ -345,18 +391,31 @@ Looking at the previous example:
 [source,js]
 --------------------------------------------------
 "collector": [
-    {
-       "name": "SimpleTopScoreDocCollector",
-       "reason": "search_top_hits",
-       "time": "2.206529000ms"
-    }
+   {
+      "name": "CancellableCollector",
+      "reason": "search_cancelled",
+      "time": "0.3043110000ms",
+      "time_in_nanos": "304311",
+      "children": [
+        {
+          "name": "SimpleTopScoreDocCollector",
+          "reason": "search_top_hits",
+          "time": "0.03227300000ms",
+          "time_in_nanos": "32273"
+        }
+      ]
+   }
 ]
 --------------------------------------------------
+// TESTRESPONSE[s/^/{\n"took": $body.took,\n"timed_out": $body.timed_out,\n"_shards": $body._shards,\n"hits": $body.hits,\n"profile": {\n"shards": [ {\n"id": "$body.$_path",\n"searches": [{\n"query": $body.$_path,\n"rewrite_time": $body.$_path,/]
+// TESTRESPONSE[s/]$/]}], "aggregations": []}]}}/]
+// TESTRESPONSE[s/(?<=[" ])\d+(\.\d+)?[ms]*/$body.$_path/]
 
-We see a single collector named `SimpleTopScoreDocCollector`.  This is the default "scoring and sorting" Collector
-used by Elasticsearch.  The `"reason"` field attempts to give a plain english description of the class name.  The
-`"time"` is similar to the time in the Query tree: a wall-clock time inclusive of all children.  Similarly, `children` lists
-all sub-collectors.
+We see a single collector named `SimpleTopScoreDocCollector` wrapped into `CancellableCollector`. `SimpleTopScoreDocCollector` is the default "scoring and sorting"
+`Collector` used by Elasticsearch.  The `reason` field attempts to give a plain english description of the class name.  The
+`time` is similar to the time in the Query tree: a wall-clock time inclusive of all children.  Similarly, `children` lists
+all sub-collectors. The `CancellableCollector` that wraps `SimpleTopScoreDocCollector` is used by elasticsearch to detect if the current
+search was cancelled and stop collecting documents as soon as it occurs.
 
 It should be noted that Collector times are **independent** from the Query times.  They are calculated, combined
 and normalized independently!  Due to the nature of Lucene's execution, it is impossible to "merge" the times
@@ -473,6 +532,8 @@ GET /test/_search
   }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[s/^/PUT test\n/]
 
 This example has:
 
@@ -498,6 +559,7 @@ And the response:
                            "type": "TermQuery",
                            "description": "my_field:foo",
                            "time": "0.4094560000ms",
+                           "time_in_nanos": "409456",
                            "breakdown": {
                               "score": 0,
                               "score_count": 1,
@@ -509,7 +571,7 @@ And the response:
                               "create_weight_count": 1,
                               "build_scorer": 377872,
                               "build_scorer_count": 1,
-                              "advance": 0
+                              "advance": 0,
                               "advance_count": 0
                            }
                         },
@@ -517,6 +579,7 @@ And the response:
                            "type": "TermQuery",
                            "description": "message:search",
                            "time": "0.3037020000ms",
+                           "time_in_nanos": "303702",
                            "breakdown": {
                               "score": 0,
                               "score_count": 1,
@@ -528,7 +591,7 @@ And the response:
                               "create_weight_count": 1,
                               "build_scorer": 112551,
                               "build_scorer_count": 1,
-                              "advance": 0
+                              "advance": 0,
                               "advance_count": 0
                            }
                         }
@@ -539,23 +602,27 @@ And the response:
                            "name": "MultiCollector",
                            "reason": "search_multi",
                            "time": "1.378943000ms",
+                           "time_in_nanos": "1378943",
                            "children": [
                               {
                                  "name": "FilteredCollector",
                                  "reason": "search_post_filter",
                                  "time": "0.4036590000ms",
+                                 "time_in_nanos": "403659",
                                  "children": [
                                     {
                                        "name": "SimpleTopScoreDocCollector",
                                        "reason": "search_top_hits",
-                                       "time": "0.006391000000ms"
+                                       "time": "0.006391000000ms",
+                                       "time_in_nanos": "6391"
                                     }
                                  ]
                               },
                               {
                                  "name": "BucketCollector: [[non_global_term, another_agg]]",
                                  "reason": "aggregation",
-                                 "time": "0.9546020000ms"
+                                 "time": "0.9546020000ms",
+                                 "time_in_nanos": "954602"
                               }
                            ]
                         }
@@ -567,6 +634,7 @@ And the response:
                            "type": "MatchAllDocsQuery",
                            "description": "*:*",
                            "time": "0.04829300000ms",
+                           "time_in_nanos": "48293",
                            "breakdown": {
                               "score": 0,
                               "score_count": 1,
@@ -578,7 +646,7 @@ And the response:
                               "create_weight_count": 1,
                               "build_scorer": 38310,
                               "build_scorer_count": 1,
-                              "advance": 0
+                              "advance": 0,
                               "advance_count": 0
                            }
                         }
@@ -588,7 +656,8 @@ And the response:
                         {
                            "name": "GlobalAggregator: [global_agg]",
                            "reason": "aggregation_global",
-                           "time": "0.1226310000ms"
+                           "time": "0.1226310000ms",
+                           "time_in_nanos": "122631"
                         }
                      ]
                   }
@@ -640,7 +709,7 @@ the following example aggregations request:
 
 [source,js]
 --------------------------------------------------
-curl -XGET "http://localhost:9200/house-prices/_search" -d'
+GET /house-prices/_search
 {
   "profile": true,
   "size": 0,
@@ -658,8 +727,10 @@ curl -XGET "http://localhost:9200/house-prices/_search" -d'
       }
     }
   }
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[s/^/PUT house-prices\n/]
 
 Which yields the following aggregation profile output
 
@@ -670,6 +741,7 @@ Which yields the following aggregation profile output
     "type": "org.elasticsearch.search.aggregations.bucket.terms.GlobalOrdinalsStringTermsAggregator",
     "description": "property_type",
     "time": "4280.456978ms",
+    "time_in_nanos": "4280456978",
     "breakdown": {
       "reduce": 0,
       "reduce_count": 0,
@@ -685,6 +757,7 @@ Which yields the following aggregation profile output
         "type": "org.elasticsearch.search.aggregations.metrics.avg.AvgAggregator",
         "description": "avg_price",
         "time": "1124.864392ms",
+        "time_in_nanos": "1124864392",
         "breakdown": {
           "reduce": 0,
           "reduce_count": 0,
@@ -704,16 +777,16 @@ Which yields the following aggregation profile output
 From the profile structure we can see our `property_type` terms aggregation which is internally represented by the
 `GlobalOrdinalsStringTermsAggregator` class and the sub aggregator `avg_price` which is internally represented by the `AvgAggregator` class. The `type` field displays the class used internally to represent the aggregation. The `description` field displays the name of the aggregation.
 
-The `"time"` field shows that it took ~4 seconds for the entire aggregation to execute.  The recorded time is inclusive
+The `time` field shows that it took ~4 seconds for the entire aggregation to execute.  The recorded time is inclusive
 of all children.
 
-The `"breakdown"` field will give detailed stats about how the time was spent, we'll look at
-that in a moment.  Finally, the `"children"` array lists any sub-aggregations that may be present.  Because we have an `avg_price` aggregation as a sub-aggregation to the `property_type` aggregation we see it listed as a child of the `property_type` aggregation.  the two aggregation outputs have identical information (type, time,
+The `breakdown` field will give detailed stats about how the time was spent, we'll look at
+that in a moment.  Finally, the `children` array lists any sub-aggregations that may be present.  Because we have an `avg_price` aggregation as a sub-aggregation to the `property_type` aggregation we see it listed as a child of the `property_type` aggregation.  the two aggregation outputs have identical information (type, time,
 breakdown, etc).  Children are allowed to have their own children.
 
 ===== Timing Breakdown
 
-The `"breakdown"` component lists detailed timing statistics about low-level Lucene execution:
+The `breakdown` component lists detailed timing statistics about low-level Lucene execution:
 
 [source,js]
 --------------------------------------------------
diff --git a/docs/reference/search/request-body.asciidoc b/docs/reference/search/request-body.asciidoc
index a9adc157bd395..151160a1dbc31 100644
--- a/docs/reference/search/request-body.asciidoc
+++ b/docs/reference/search/request-body.asciidoc
@@ -7,41 +7,50 @@ example:
 
 [source,js]
 --------------------------------------------------
-$ curl -XGET 'http://localhost:9200/twitter/tweet/_search' -d '{
+GET /twitter/tweet/_search
+{
     "query" : {
         "term" : { "user" : "kimchy" }
     }
 }
-'
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
 
 And here is a sample response:
 
 [source,js]
 --------------------------------------------------
 {
+    "took": 1,
+    "timed_out": false,
     "_shards":{
-        "total" : 5,
-        "successful" : 5,
+        "total" : 1,
+        "successful" : 1,
+        "skipped" : 0,
         "failed" : 0
     },
     "hits":{
         "total" : 1,
+        "max_score": 1.3862944,
         "hits" : [
             {
                 "_index" : "twitter",
                 "_type" : "tweet",
-                "_id" : "1",
+                "_id" : "0",
+                "_score": 1.3862944,
                 "_source" : {
                     "user" : "kimchy",
-                    "postDate" : "2009-11-15T14:12:12",
-                    "message" : "trying out Elasticsearch"
+                    "message": "trying out Elasticsearch",
+                    "date" : "2009-11-15T14:12:12",
+                    "likes" : 0
                 }
             }
         ]
     }
 }
 --------------------------------------------------
+// TESTRESPONSE[s/"took": 1/"took": $body.took/]
 
 [float]
 === Parameters
@@ -85,6 +94,14 @@ And here is a sample response:
     the query execution has actually terminated_early. Defaults to no
     terminate_after.
 
+`batched_reduce_size`::
+
+    The number of shard results that should be reduced  at once on the
+    coordinating node. This value should be used as a protection  mechanism to
+    reduce the memory overhead per search request if the potential  number of
+    shards in the request can be large.
+
+
 
 Out of the above, the `search_type` and the `request_cache` must be passed as
 query-string parameters. The rest of the search request should be passed
@@ -105,8 +122,10 @@ matching document was found (per shard).
 
 [source,js]
 --------------------------------------------------
-$ curl -XGET 'http://localhost:9200/_search?q=tag:wow&size=0&terminate_after=1'
+GET /_search?q=message:elasticsearch&size=0&terminate_after=1
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
 
 The response will not contain any hits as the `size` was set to `0`. The
 `hits.total` will be either equal to `0`, indicating that there were no
@@ -124,16 +143,17 @@ be set to `true` in the response.
   "_shards": {
     "total": 1,
     "successful": 1,
+    "skipped" : 0,
     "failed": 0
   },
   "hits": {
     "total": 1,
-    "max_score": 0,
+    "max_score": 0.0,
     "hits": []
   }
 }
 --------------------------------------------------
-
+// TESTRESPONSE[s/"took": 3/"took": $body.took/]
 
 include::request/query.asciidoc[]
 
@@ -173,4 +193,6 @@ include::request/named-queries-and-filters.asciidoc[]
 
 include::request/inner-hits.asciidoc[]
 
+include::request/collapse.asciidoc[]
+
 include::request/search-after.asciidoc[]
diff --git a/docs/reference/search/request/collapse.asciidoc b/docs/reference/search/request/collapse.asciidoc
new file mode 100644
index 0000000000000..b4322e36f9343
--- /dev/null
+++ b/docs/reference/search/request/collapse.asciidoc
@@ -0,0 +1,118 @@
+[[search-request-collapse]]
+=== Field Collapsing
+
+Allows to collapse search results based on field values.
+The collapsing is done by selecting only the top sorted document per collapse key.
+For instance the query below retrieves the best tweet for each user and sorts them by number of likes.
+
+[source,js]
+--------------------------------------------------
+GET /twitter/tweet/_search
+{
+    "query": {
+        "match": {
+            "message": "elasticsearch"
+        }
+    },
+    "collapse" : {
+        "field" : "user" <1>
+    },
+    "sort": ["likes"], <2>
+    "from": 10 <3>
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
+<1> collapse the result set using the "user" field
+<2> sort the top docs by number of likes
+<3> define the offset of the first collapsed result
+
+WARNING: The total number of hits in the response indicates the number of matching documents without collapsing.
+The total number of distinct group is unknown.
+
+The field used for collapsing must be a single valued <<keyword, `keyword`>> or <<number, `numeric`>> field with <<doc-values, `doc_values`>> activated
+
+NOTE: The collapsing is applied to the top hits only and does not affect aggregations.
+
+
+==== Expand collapse results
+
+It is also possible to expand each collapsed top hits with the `inner_hits` option.
+
+[source,js]
+--------------------------------------------------
+GET /twitter/tweet/_search
+{
+    "query": {
+        "match": {
+            "message": "elasticsearch"
+        }
+    },
+    "collapse" : {
+        "field" : "user", <1>
+        "inner_hits": {
+            "name": "last_tweets", <2>
+            "size": 5, <3>
+            "sort": [{ "date": "asc" }] <4>
+        },
+        "max_concurrent_group_searches": 4 <5>
+    },
+    "sort": ["likes"]
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
+<1> collapse the result set using the "user" field
+<2> the name used for the inner hit section in the response
+<3> the number of inner_hits to retrieve per collapse key
+<4> how to sort the document inside each group
+<5> the number of concurrent requests allowed to retrieve the inner_hits` per group
+
+See <<search-request-inner-hits, inner hits>> for the complete list of supported options and the format of the response.
+
+It is also possible to request multiple `inner_hits` for each collapsed hit.  This can be useful when you want to get
+multiple representations of the collapsed hits.
+
+[source,js]
+--------------------------------------------------
+GET /twitter/tweet/_search
+{
+    "query": {
+        "match": {
+            "message": "elasticsearch"
+        }
+    },
+    "collapse" : {
+        "field" : "user", <1>
+        "inner_hits": [
+            {
+                "name": "most_liked",  <2>
+                "size": 3,
+                "sort": ["likes"]
+            },
+            {
+                "name": "most_recent", <3>
+                "size": 3,
+                "sort": [{ "date": "asc" }]
+            }
+        ]
+    },
+    "sort": ["likes"]
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
+<1> collapse the result set using the "user" field
+<2> return the three most liked tweets for the user
+<3> return the three most recent tweets for the user
+
+The expansion of the group is done by sending an additional query for each
+`inner_hit` request for each collapsed hit returned in the response.  This can significantly slow things down
+if you have too many groups and/or `inner_hit` requests.
+
+The `max_concurrent_group_searches` request parameter can be used to control
+the maximum number of concurrent searches allowed in this phase.
+The default is based on the number of data nodes and the default search thread pool size.
+
+WARNING: `collapse` cannot be used in conjunction with <<search-request-scroll, scroll>>,
+<<search-request-rescore, rescore>> or <<search-request-search-after, search after>>.
diff --git a/docs/reference/search/request/highlighting.asciidoc b/docs/reference/search/request/highlighting.asciidoc
index 2e7bc9f180528..c8e9a90cee400 100644
--- a/docs/reference/search/request/highlighting.asciidoc
+++ b/docs/reference/search/request/highlighting.asciidoc
@@ -11,7 +11,7 @@ The following is an example of the search request body:
 GET /_search
 {
     "query" : {
-        "match": { "user": "kimchy" }
+        "match": { "content": "kimchy" }
     },
     "highlight" : {
         "fields" : {
@@ -21,6 +21,7 @@ GET /_search
 }
 --------------------------------------------------
 // CONSOLE
+// TEST[setup:twitter]
 
 In the above case, the `content` field will be highlighted for each
 search hit (there will be another element in each search hit, called
@@ -59,8 +60,13 @@ This is repeated for every field and every document that needs highlighting. If
 [[postings-highlighter]]
 ==== Postings highlighter
 
+deprecated[5.6.0,This highlighter type is deprecated and will be removed in 6.0. Please use the `unified` highlighter instead. ]
+
+
 If `index_options` is set to `offsets` in the mapping the postings highlighter
-will be used instead of the plain highlighter. The postings highlighter:
+will be used instead of the plain highlighter.
+
+The postings highlighter:
 
 * Is faster since it doesn't require to reanalyze the text to be highlighted:
 the larger the documents the better the performance gain should be
@@ -76,12 +82,21 @@ highlighting using the postings highlighter on it:
 
 [source,js]
 --------------------------------------------------
+PUT /example
 {
-    "type_name" : {
-        "content" : {"index_options" : "offsets"}
+  "mappings": {
+    "doc" : {
+      "properties": {
+        "comment" : {
+          "type": "text",
+          "index_options" : "offsets"
+        }
+      }
     }
+  }
 }
 --------------------------------------------------
+// CONSOLE
 
 [NOTE]
 Note that the postings highlighter is meant to perform simple query terms
@@ -103,8 +118,7 @@ If `term_vector` information is provided by setting `term_vector` to
 will be used instead of the plain highlighter.  The fast vector highlighter:
 
 * Is faster especially for large fields (> `1MB`)
-* Can be customized with `boundary_chars`, `boundary_max_scan`, and
- `fragment_offset` (see <<boundary-characters,below>>)
+* Can be customized with `boundary_scanner` (see <<boundary-scanners,below>>)
 * Requires setting `term_vector` to `with_positions_offsets` which
   increases the size of the index
 * Can combine matches from multiple fields into one result.  See
@@ -119,12 +133,40 @@ the index to be bigger):
 
 [source,js]
 --------------------------------------------------
+PUT /example
 {
-    "type_name" : {
-        "content" : {"term_vector" : "with_positions_offsets"}
+  "mappings": {
+    "doc" : {
+      "properties": {
+        "comment" : {
+          "type": "text",
+          "term_vector" : "with_positions_offsets"
+        }
+      }
     }
+  }
 }
 --------------------------------------------------
+// CONSOLE
+
+==== Unified Highlighter
+
+experimental[]
+
+The `unified` highlighter can extract offsets from either postings, term vectors, or via re-analyzing text.
+Under the hood it uses Lucene UnifiedHighlighter which picks its strategy depending on the field and the query to highlight.
+Independently of the strategy this highlighter breaks the text into sentences and scores individual sentences as
+if they were documents in this corpus, using the BM25 algorithm.
+It supports accurate phrase and multi-term (fuzzy, prefix, regex) highlighting and can be used with the following options:
+
+* `force_source`
+* `encoder`
+* `highlight_query`
+* `pre_tags and `post_tags`
+* `require_field_match`
+* `boundary_scanner` (`sentence` (**default**)  or `word`)
+* `max_fragment_length` (only for `sentence` scanner)
+* `no_match_size`
 
 ==== Force highlighter type
 
@@ -148,6 +190,7 @@ GET /_search
 }
 --------------------------------------------------
 // CONSOLE
+// TEST[setup:twitter]
 
 ==== Force highlighting on source
 
@@ -169,6 +212,7 @@ GET /_search
 }
 --------------------------------------------------
 // CONSOLE
+// TEST[setup:twitter]
 
 [[tags]]
 ==== Highlighting Tags
@@ -194,6 +238,7 @@ GET /_search
 }
 --------------------------------------------------
 // CONSOLE
+// TEST[setup:twitter]
 
 Using the fast vector highlighter there can be more tags, and the "importance"
 is ordered.
@@ -215,11 +260,12 @@ GET /_search
 }
 --------------------------------------------------
 // CONSOLE
+// TEST[setup:twitter]
 
 There are also built in "tag" schemas, with currently a single schema
 called `styled` with the following `pre_tags`:
 
-[source,js]
+[source,html]
 --------------------------------------------------
 <em class="hlt1">, <em class="hlt2">, <em class="hlt3">,
 <em class="hlt4">, <em class="hlt5">, <em class="hlt6">,
@@ -247,7 +293,7 @@ GET /_search
 }
 --------------------------------------------------
 // CONSOLE
-
+// TEST[setup:twitter]
 
 ==== Encoder
 
@@ -277,6 +323,7 @@ GET /_search
 }
 --------------------------------------------------
 // CONSOLE
+// TEST[setup:twitter]
 
 The `fragment_size` is ignored when using the postings highlighter, as it
 outputs sentences regardless of their length.
@@ -300,6 +347,7 @@ GET /_search
 }
 --------------------------------------------------
 // CONSOLE
+// TEST[setup:twitter]
 
 If the `number_of_fragments` value is set to `0` then no fragments are
 produced, instead the whole content of the field is returned, and of
@@ -323,6 +371,7 @@ GET /_search
 }
 --------------------------------------------------
 // CONSOLE
+// TEST[setup:twitter]
 
 When using `fvh` one can use `fragment_offset`
 parameter to control the margin to start highlighting from.
@@ -330,7 +379,7 @@ parameter to control the margin to start highlighting from.
 In the case where there is no matching fragment to highlight, the default is
 to not return anything. Instead, we can return a snippet of text from the
 beginning of the field by setting `no_match_size` (default `0`) to the length
-of the text that you want returned. The actual length may be shorter than
+of the text that you want returned. The actual length may be shorter or longer than
 specified as it tries to break on a word boundary. When using the postings
 highlighter it is not possible to control the actual size of the snippet,
 therefore the first sentence gets returned whenever `no_match_size` is
@@ -355,7 +404,130 @@ GET /_search
 }
 --------------------------------------------------
 // CONSOLE
+// TEST[setup:twitter]
+
+==== Fragmenter
+
+Fragmenter can control how text should be broken up in highlight snippets.
+However, this option is applicable only for the Plain Highlighter.
+There are two options:
+
+[horizontal]
+`simple`:: Breaks up text into same sized fragments.
+`span`:: Same as the simple fragmenter, but tries not to break up text between highlighted terms (this is applicable when using phrase like queries). This is the default.
+
+[source,js]
+--------------------------------------------------
+GET twitter/tweet/_search
+{
+    "query" : {
+        "match_phrase": { "message": "number 1" }
+    },
+    "highlight" : {
+        "fields" : {
+            "message" : {
+                "fragment_size" : 15,
+                "number_of_fragments" : 3,
+                "fragmenter": "simple"
+            }
+        }
+    }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
+
+Response:
+
+[source,js]
+--------------------------------------------------
+{
+    ...
+    "hits": {
+        "total": 1,
+        "max_score": 1.4818809,
+        "hits": [
+            {
+                "_index": "twitter",
+                "_type": "tweet",
+                "_id": "1",
+                "_score": 1.4818809,
+                "_source": {
+                    "user": "test",
+                    "message": "some message with the number 1",
+                    "date": "2009-11-15T14:12:12",
+                    "likes": 1
+                },
+                "highlight": {
+                    "message": [
+                        " with the <em>number</em>",
+                        " <em>1</em>"
+                    ]
+                }
+            }
+        ]
+    }
+}
+--------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,/]
+
+[source,js]
+--------------------------------------------------
+GET twitter/tweet/_search
+{
+    "query" : {
+        "match_phrase": { "message": "number 1" }
+    },
+    "highlight" : {
+        "fields" : {
+            "message" : {
+                "fragment_size" : 15,
+                "number_of_fragments" : 3,
+                "fragmenter": "span"
+            }
+        }
+    }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
 
+Response:
+
+[source,js]
+--------------------------------------------------
+{
+    ...
+    "hits": {
+        "total": 1,
+        "max_score": 1.4818809,
+        "hits": [
+            {
+                "_index": "twitter",
+                "_type": "tweet",
+                "_id": "1",
+                "_score": 1.4818809,
+                "_source": {
+                    "user": "test",
+                    "message": "some message with the number 1",
+                    "date": "2009-11-15T14:12:12",
+                    "likes": 1
+                },
+                "highlight": {
+                    "message": [
+                        " with the <em>number</em> <em>1</em>"
+                    ]
+                }
+            }
+        ]
+    }
+}
+--------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,/]
+
+If the `number_of_fragments` option is set to `0`,
+`NullFragmenter` is used which does not fragment the text at all.
+This is useful for highlighting the entire content of a document or field.
 
 ==== Highlight query
 
@@ -427,6 +599,7 @@ GET /_search
 }
 --------------------------------------------------
 // CONSOLE
+// TEST[setup:twitter]
 
 Note that the score of text fragment in this case is calculated by the Lucene
 highlighting framework. For implementation details you can check the
@@ -460,6 +633,7 @@ GET /_search
 }
 --------------------------------------------------
 // CONSOLE
+// TEST[setup:twitter]
 
 [[field-match]]
 ==== Require Field Match
@@ -485,18 +659,30 @@ GET /_search
 }
 --------------------------------------------------
 // CONSOLE
+// TEST[setup:twitter]
 
-[[boundary-characters]]
-==== Boundary Characters
+[[boundary-scanners]]
+==== Boundary Scanners
 
-When highlighting a field using the fast vector highlighter,
-`boundary_chars` can be configured to define what constitutes a boundary
-for highlighting. It's a single string with each boundary character
-defined in it. It defaults to `.,!? \t\n`.
+When highlighting a field using the unified highlighter or the fast vector highlighter,
+you can specify how to break the highlighted fragments using `boundary_scanner`, which accepts
+the following values:
 
-The `boundary_max_scan` allows to control how far to look for boundary
-characters, and defaults to `20`.
+* `chars` (default mode for the FVH): allows to configure which characters (`boundary_chars`)
+constitute a boundary for highlighting. It's a single string with each boundary
+character defined in it (defaults to `.,!? \t\n`). It also allows configuring
+the `boundary_max_scan` to control how far to look for boundary characters
+(defaults to `20`). Works only with the Fast Vector Highlighter.
 
+* `sentence` and `word`: use Java's https://docs.oracle.com/javase/8/docs/api/java/text/BreakIterator.html[BreakIterator]
+to break the highlighted fragments at the next  _sentence_ or _word_ boundary.
+You can further specify `boundary_scanner_locale` to control which Locale is used
+to search the text for these boundaries.
+
+[NOTE]
+When used with the `unified` highlighter, the `sentence` scanner splits sentence
+bigger than `fragment_size` at the first word boundary next to `fragment_size`.
+You can set `fragment_size` to 0 to never split any sentence.
 
 [[matched-fields]]
 ==== Matched Fields
@@ -533,6 +719,8 @@ GET /_search
 }
 --------------------------------------------------
 // CONSOLE
+// TEST[setup:twitter]
+
 The above matches both "run with scissors" and "running with scissors"
 and would highlight "running" and "scissors" but not "run". If both
 phrases appear in a large document then "running with scissors" is
@@ -561,6 +749,8 @@ GET /_search
 }
 --------------------------------------------------
 // CONSOLE
+// TEST[setup:twitter]
+
 The above highlights "run" as well as "running" and "scissors" but
 still sorts "running with scissors" above "run with scissors" because
 the plain match ("running") is boosted.
@@ -587,6 +777,7 @@ GET /_search
 }
 --------------------------------------------------
 // CONSOLE
+// TEST[setup:twitter]
 
 The above query wouldn't highlight "run" or "scissor" but shows that
 it is just fine not to list the field to which the matches are combined
@@ -610,6 +801,7 @@ There is a small amount of overhead involved with setting
         }
     }
 --------------------------------------------------
+// NOTCONSOLE
 to
 [source,js]
 --------------------------------------------------
@@ -622,6 +814,7 @@ to
         }
     }
 --------------------------------------------------
+// NOTCONSOLE
 ===================================================================
 
 [[phrase-limit]]
@@ -643,12 +836,18 @@ json spec objects are unordered but if you need to be explicit about the order
 that fields are highlighted then you can use an array for `fields` like this:
 [source,js]
 --------------------------------------------------
+GET /_search
+{
     "highlight": {
         "fields": [
-            {"title":{ /*params*/ }},
-            {"text":{ /*params*/ }}
+            { "title": {} },
+            { "text": {} }
         ]
     }
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
+
 None of the highlighters built into Elasticsearch care about the order that the
 fields are highlighted but a plugin may.
diff --git a/docs/reference/search/request/index-boost.asciidoc b/docs/reference/search/request/index-boost.asciidoc
index bf766ce8a8c05..683fe910f5e6e 100644
--- a/docs/reference/search/request/index-boost.asciidoc
+++ b/docs/reference/search/request/index-boost.asciidoc
@@ -6,6 +6,7 @@ across more than one indices. This is very handy when hits coming from
 one index matter more than hits coming from another index (think social
 graph where each user has an index).
 
+deprecated[5.2.0, This format is deprecated. Please use array format instead.]
 [source,js]
 --------------------------------------------------
 GET /_search
@@ -17,3 +18,23 @@ GET /_search
 }
 --------------------------------------------------
 // CONSOLE
+// TEST[setup:index_boost warning:Object format in indices_boost is deprecated, please use array format instead]
+
+You can also specify it as an array to control the order of boosts.
+
+[source,js]
+--------------------------------------------------
+GET /_search
+{
+    "indices_boost" : [
+        { "alias1" : 1.4 },
+        { "index*" : 1.3 }
+    ]
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+This is important when you use aliases or wildcard expression.
+If multiple matches are found, the first match will be used.
+For example, if an index is included in both `alias1` and `index*`, boost value of `1.4` is applied.
\ No newline at end of file
diff --git a/docs/reference/search/request/inner-hits.asciidoc b/docs/reference/search/request/inner-hits.asciidoc
index 28914cd9b5e02..ed4488f502be2 100644
--- a/docs/reference/search/request/inner-hits.asciidoc
+++ b/docs/reference/search/request/inner-hits.asciidoc
@@ -1,7 +1,7 @@
 [[search-request-inner-hits]]
 === Inner hits
 
-The <<mapping-parent-field, parent/child>> and <<nested, nested>> features allow the return of documents that
+The <<parent-join, parent-join> and <<nested, nested>> features allow the return of documents that
 have matches in a different scope. In the parent/child case, parent documents are returned based on matches in child
 documents or child documents are returned based on matches in parent documents. In the nested case, documents are returned
 based on matches in nested inner objects.
@@ -22,6 +22,7 @@ The structure looks like this:
     }
 }
 --------------------------------------------------
+// NOTCONSOLE
 
 If `inner_hits` is defined on a query that supports it then each search hit will contain an `inner_hits` json object with the following structure:
 
@@ -52,6 +53,7 @@ If `inner_hits` is defined on a query that supports it then each search hit will
      ...
 ]
 --------------------------------------------------
+// NOTCONSOLE
 
 ==== Options
 
@@ -80,22 +82,50 @@ Inner hits also supports the following per document features:
 
 The nested `inner_hits` can be used to include nested inner objects as inner hits to a search hit.
 
-The example below assumes that there is a nested object field defined with the name `comments`:
-
 [source,js]
 --------------------------------------------------
+PUT test
 {
-    "query" : {
-        "nested" : {
-            "path" : "comments",
-            "query" : {
-                "match" : {"comments.message" : "[actual query]"}
-            },
-            "inner_hits" : {} <1>
+  "mappings": {
+    "doc": {
+      "properties": {
+        "comments": {
+          "type": "nested"
         }
+      }
+    }
+  }
+}
+
+PUT test/doc/1?refresh
+{
+  "title": "Test title",
+  "comments": [
+    {
+      "author": "kimchy",
+      "number": 1
+    },
+    {
+      "author": "nik9000",
+      "number": 2
     }
+  ]
+}
+
+POST test/_search
+{
+  "query": {
+    "nested": {
+      "path": "comments",
+      "query": {
+        "match": {"comments.number" : 2}
+      },
+      "inner_hits": {} <1>
+    }
+  }
 }
 --------------------------------------------------
+// CONSOLE
 
 <1> The inner hit definition in the nested query. No other options need to be defined.
 
@@ -103,35 +133,46 @@ An example of a response snippet that could be generated from the above search r
 
 [source,js]
 --------------------------------------------------
-...
-"hits": {
-  ...
-  "hits": [
-     {
-        "_index": "my-index",
-        "_type": "question",
+{
+  ...,
+  "hits": {
+    "total": 1,
+    "max_score": 1.0,
+    "hits": [
+      {
+        "_index": "test",
+        "_type": "doc",
         "_id": "1",
+        "_score": 1.0,
         "_source": ...,
         "inner_hits": {
-           "comments": { <1>
-              "hits": {
-                 "total": ...,
-                 "hits": [
-                    {
-                       "_nested": {
-                          "field": "comments",
-                          "offset": 2
-                       },
-                       "_source": ...
-                    },
-                    ...
-                 ]
-              }
-           }
+          "comments": { <1>
+            "hits": {
+              "total": 1,
+              "max_score": 1.0,
+              "hits": [
+                {
+                  "_nested": {
+                    "field": "comments",
+                    "offset": 1
+                  },
+                  "_score": 1.0,
+                  "_source": {
+                    "author": "nik9000",
+                    "number": 2
+                  }
+                }
+              ]
+            }
+          }
         }
-     },
-     ...
+      }
+    ]
+  }
+}
 --------------------------------------------------
+// TESTRESPONSE[s/"_source": \.\.\./"_source": $body.hits.hits.0._source/]
+// TESTRESPONSE[s/\.\.\./"timed_out": false, "took": $body.took, "_shards": $body._shards/]
 
 <1> The name used in the inner hit definition in the search request. A custom key can be used via the `name` option.
 
@@ -148,6 +189,120 @@ An important default is that the `_source` returned in hits inside `inner_hits`
 So in the above example only the comment part is returned per nested hit and not the entire source of the top level
 document that contained the comment.
 
+[[nested-inner-hits-source]]
+==== Nested inner hits and +_source+
+
+Nested document don't have a `_source` field, because the entire source of document is stored with the root document under
+its `_source` field. To include the source of just the nested document, the source of the root document is parsed and just
+the relevant bit for the nested document is included as source in the inner hit. Doing this for each matching nested document
+has an impact on the time it takes to execute the entire search request, especially when `size` and the inner hits' `size`
+are set higher than the default. To avoid the relatively expensive source extraction for nested inner hits, one can disable
+including the source and solely rely on stored fields. Like this:
+
+[source,js]
+--------------------------------------------------
+PUT test
+{
+  "mappings": {
+    "doc": {
+      "properties": {
+        "comments": {
+          "type": "nested",
+          "properties": {
+            "text": {
+              "type": "text",
+              "store": true
+            }
+          }
+        }
+      }
+    }
+  }
+}
+
+PUT test/doc/1?refresh
+{
+  "title": "Test title",
+  "comments": [
+    {
+      "author": "kimchy",
+      "text": "comment text"
+    },
+    {
+      "author": "nik9000",
+      "text": "words words words"
+    }
+  ]
+}
+
+POST test/_search
+{
+  "query": {
+    "nested": {
+      "path": "comments",
+      "query": {
+        "match": {"comments.text" : "words"}
+      },
+      "inner_hits": {
+        "_source" : false,
+        "stored_fields" : ["comments.text"]
+      }
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+
+////
+
+Response not included in text but tested for completeness sake.
+
+[source,js]
+--------------------------------------------------
+{
+  ...,
+  "hits": {
+    "total": 1,
+    "max_score": 0.9651416,
+    "hits": [
+      {
+        "_index": "test",
+        "_type": "doc",
+        "_id": "1",
+        "_score": 0.9651416,
+        "_source": ...,
+        "inner_hits": {
+          "comments": { <1>
+            "hits": {
+              "total": 1,
+              "max_score": 0.9651416,
+              "hits": [
+                {
+                  "_nested": {
+                    "field": "comments",
+                    "offset": 1
+                  },
+                  "_score": 0.9651416,
+                  "fields": {
+                    "comments.text": [
+                      "words words words"
+                    ]
+                  }
+                }
+              ]
+            }
+          }
+        }
+      }
+    ]
+  }
+}
+--------------------------------------------------
+// TESTRESPONSE[s/"_source": \.\.\./"_source": $body.hits.hits.0._source/]
+// TESTRESPONSE[s/\.\.\./"timed_out": false, "took": $body.took, "_shards": $body._shards/]
+
+////
+
 [[hierarchical-nested-inner-hits]]
 ==== Hierarchical levels of nested object fields and inner hits.
 
@@ -157,40 +312,173 @@ with the root hits then the following path can be defined:
 
 [source,js]
 --------------------------------------------------
+PUT test
 {
-   "query" : {
-      "nested" : {
-         "path" : "comments.votes",
-         "query" : { ... },
-         "inner_hits" : {}
+  "mappings": {
+    "doc": {
+      "properties": {
+        "comments": {
+          "type": "nested",
+          "properties": {
+            "message": {
+              "type": "text",
+              "store": true
+            },
+            "votes": {
+              "type": "nested"
+            }
+          }
+        }
       }
     }
+  }
+}
+
+PUT test/doc/1?refresh
+{
+  "title": "Test title",
+  "comments": [
+    {
+      "author": "kimchy",
+      "text": "comment text",
+      "votes": []
+    },
+    {
+      "author": "nik9000",
+      "text": "words words words",
+      "votes": [
+        {"value": 1 , "voter": "kimchy"},
+        {"value": -1, "voter": "other"}
+      ]
+    }
+  ]
+}
+
+POST test/_search
+{
+  "query": {
+    "nested": {
+      "path": "comments.votes",
+        "query": {
+          "match": {
+            "comments.votes.voter": "kimchy"
+          }
+        },
+        "inner_hits" : {}
+    }
+  }
 }
 --------------------------------------------------
+// CONSOLE
+
+Which would look like:
+
+[source,js]
+--------------------------------------------------
+{
+  ...,
+  "hits": {
+    "total": 1,
+    "max_score": 0.6931472,
+    "hits": [
+      {
+        "_index": "test",
+        "_type": "doc",
+        "_id": "1",
+        "_score": 0.6931472,
+        "_source": ...,
+        "inner_hits": {
+          "comments.votes": { <1>
+            "hits": {
+              "total": 1,
+              "max_score": 0.6931472,
+              "hits": [
+                {
+                  "_nested": {
+                    "field": "comments",
+                    "offset": 1,
+                    "_nested": {
+                      "field": "votes",
+                      "offset": 0
+                    }
+                  },
+                  "_score": 0.6931472,
+                  "_source": {
+                    "value": 1,
+                    "voter": "kimchy"
+                  }
+                }
+              ]
+            }
+          }
+        }
+      }
+    ]
+  }
+}
+--------------------------------------------------
+// TESTRESPONSE[s/"_source": \.\.\./"_source": $body.hits.hits.0._source/]
+// TESTRESPONSE[s/\.\.\./"timed_out": false, "took": $body.took, "_shards": $body._shards/]
 
 This indirect referencing is only supported for nested inner hits.
 
 [[parent-child-inner-hits]]
 ==== Parent/child inner hits
 
-The parent/child `inner_hits` can be used to include parent or child
-
-The examples below assumes that there is a `_parent` field mapping in the `comment` type:
+The parent/child `inner_hits` can be used to include parent or child:
 
 [source,js]
 --------------------------------------------------
+PUT test
 {
-    "query" : {
-        "has_child" : {
-            "type" : "comment",
-            "query" : {
-                "match" : {"message" : "[actual query]"}
-            },
-            "inner_hits" : {} <1>
+  "settings": {
+    "mapping.single_type": true
+  },
+  "mappings": {
+    "doc": {
+      "properties": {
+        "my_join_field": {
+          "type": "join",
+          "relations": {
+            "my_parent": "my_child"
+          }
+        }
+      }
+    }
+  }
+}
+
+PUT test/doc/1?refresh
+{
+  "number": 1,
+  "my_join_field": "my_parent"
+}
+
+PUT test/doc/2?routing=1&refresh
+{
+  "number": 1,
+  "my_join_field": {
+    "name": "my_child",
+    "parent": "1"
+  }
+}
+
+POST test/_search
+{
+  "query": {
+    "has_child": {
+      "type": "my_child",
+      "query": {
+        "match": {
+          "number": 1
         }
+      },
+      "inner_hits": {}    <1>
     }
+  }
 }
 --------------------------------------------------
+// CONSOLE
 
 <1> The inner hit definition like in the nested example.
 
@@ -198,30 +486,48 @@ An example of a response snippet that could be generated from the above search r
 
 [source,js]
 --------------------------------------------------
-...
-"hits": {
-  ...
-  "hits": [
-     {
-        "_index": "my-index",
-        "_type": "question",
-        "_id": "1",
-        "_source": ...,
-        "inner_hits": {
-           "comment": {
-              "hits": {
-                 "total": ...,
-                 "hits": [
-                    {
-                       "_type": "comment",
-                       "_id": "5",
-                       "_source": ...
-                    },
-                    ...
-                 ]
-              }
-           }
-        }
-     },
-     ...
---------------------------------------------------
\ No newline at end of file
+{
+    ...,
+    "hits": {
+        "total": 1,
+        "max_score": 1.0,
+        "hits": [
+            {
+                "_index": "test",
+                "_type": "doc",
+                "_id": "1",
+                "_score": 1.0,
+                "_source": {
+                    "number": 1,
+                    "my_join_field": "my_parent"
+                },
+                "inner_hits": {
+                    "my_child": {
+                        "hits": {
+                            "total": 1,
+                            "max_score": 1.0,
+                            "hits": [
+                                {
+                                    "_type": "doc",
+                                    "_id": "2",
+                                    "_score": 1.0,
+                                    "_routing": "1",
+                                    "_source": {
+                                        "number": 1,
+                                        "my_join_field": {
+                                            "name": "my_child",
+                                            "parent": "1"
+                                        }
+                                    }
+                                }
+                            ]
+                        }
+                    }
+                }
+            }
+        ]
+    }
+}
+--------------------------------------------------
+// TESTRESPONSE[s/"_source": \.\.\./"_source": $body.hits.hits.0._source/]
+// TESTRESPONSE[s/\.\.\./"timed_out": false, "took": $body.took, "_shards": $body._shards/]
diff --git a/docs/reference/search/request/post-filter.asciidoc b/docs/reference/search/request/post-filter.asciidoc
index 493b4261c8274..249a8e4b1f6f0 100644
--- a/docs/reference/search/request/post-filter.asciidoc
+++ b/docs/reference/search/request/post-filter.asciidoc
@@ -8,7 +8,7 @@ best explained by example:
 Imagine that you are selling shirts that have the following properties:
 
 [source,js]
--------------------------------------------------
+--------------------------------------------------
 PUT /shirts
 {
     "mappings": {
@@ -28,7 +28,7 @@ PUT /shirts/item/1?refresh
     "color": "red",
     "model": "slim"
 }
-------------------------------------------------
+--------------------------------------------------
 // CONSOLE
 // TESTSETUP
 
diff --git a/docs/reference/search/request/preference.asciidoc b/docs/reference/search/request/preference.asciidoc
index d5237812b9890..d0f60d700a82c 100644
--- a/docs/reference/search/request/preference.asciidoc
+++ b/docs/reference/search/request/preference.asciidoc
@@ -1,9 +1,8 @@
 [[search-request-preference]]
 === Preference
 
-Controls a `preference` of which shard replicas to execute the search
-request on. By default, the operation is randomized between the shard
-replicas.
+Controls a `preference` of which shard copies on which to execute the
+search. By default, the operation is randomized among the available shard copies.
 
 The `preference` is a query string parameter which can be set to:
 
@@ -34,11 +33,10 @@ The `preference` is a query string parameter which can be set to:
 `_shards:2,3`:: 
 	Restricts the operation to the specified shards. (`2`
 	and `3` in this case). This preference can be combined with other
-	preferences but it has to appear first: `_shards:2,3;_primary`
+	preferences but it has to appear first: `_shards:2,3|_primary`
 
 `_only_nodes`::
-    Restricts the operation to nodes specified in node specification
-    https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster.html
+    Restricts the operation to nodes specified in <<cluster,node specification>>
 
 Custom (string) value:: 
 	A custom value will be used to guarantee that
diff --git a/docs/reference/search/request/rescore.asciidoc b/docs/reference/search/request/rescore.asciidoc
index 488884ef8da4a..204960544a688 100644
--- a/docs/reference/search/request/rescore.asciidoc
+++ b/docs/reference/search/request/rescore.asciidoc
@@ -12,9 +12,11 @@ A `rescore` request is executed on each shard before it returns its
 results to be sorted by the node handling the overall search request.
 
 Currently the rescore API has only one implementation: the query
-rescorer, which uses a query to tweak the scoring. In the future, 
+rescorer, which uses a query to tweak the scoring. In the future,
 alternative rescorers may be made available, for example, a pair-wise rescorer.
 
+NOTE: the `rescore` phase is not executed when <<search-request-sort,`sort`>> is used.
+
 NOTE: when exposing pagination to your users, you should not change
 `window_size` as you step through each page (by passing different
 `from` values) since that can alter the top hits causing results to
@@ -39,13 +41,13 @@ For example:
 
 [source,js]
 --------------------------------------------------
-curl -s -XPOST 'localhost:9200/_search' -d '{
+POST /_search
+{
    "query" : {
       "match" : {
-         "field1" : {
+         "message" : {
             "operator" : "or",
-            "query" : "the quick brown",
-            "type" : "boolean"
+            "query" : "the quick brown"
          }
       }
    },
@@ -53,10 +55,9 @@ curl -s -XPOST 'localhost:9200/_search' -d '{
       "window_size" : 50,
       "query" : {
          "rescore_query" : {
-            "match" : {
-               "field1" : {
+            "match_phrase" : {
+               "message" : {
                   "query" : "the quick brown",
-                  "type" : "phrase",
                   "slop" : 2
                }
             }
@@ -66,8 +67,9 @@ curl -s -XPOST 'localhost:9200/_search' -d '{
       }
    }
 }
-'
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
 
 The way the scores are combined can be controlled with the `score_mode`:
 [cols="<,<",options="header",]
@@ -84,15 +86,16 @@ for <<query-dsl-function-score-query,`function query`>> rescores.
 ==== Multiple Rescores
 
 It is also possible to execute multiple rescores in sequence:
+
 [source,js]
 --------------------------------------------------
-curl -s -XPOST 'localhost:9200/_search' -d '{
+POST /_search
+{
    "query" : {
       "match" : {
-         "field1" : {
+         "message" : {
             "operator" : "or",
-            "query" : "the quick brown",
-            "type" : "boolean"
+            "query" : "the quick brown"
          }
       }
    },
@@ -100,10 +103,9 @@ curl -s -XPOST 'localhost:9200/_search' -d '{
       "window_size" : 100,
       "query" : {
          "rescore_query" : {
-            "match" : {
-               "field1" : {
+            "match_phrase" : {
+               "message" : {
                   "query" : "the quick brown",
-                  "type" : "phrase",
                   "slop" : 2
                }
             }
@@ -119,8 +121,7 @@ curl -s -XPOST 'localhost:9200/_search' -d '{
             "function_score" : {
                "script_score": {
                   "script": {
-                    "lang": "painless",
-                    "inline": "Math.log10(doc['numeric'].value + 2)"
+                    "source": "Math.log10(doc.likes.value + 2)"
                   }
                }
             }
@@ -128,8 +129,9 @@ curl -s -XPOST 'localhost:9200/_search' -d '{
       }
    } ]
 }
-'
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
 
 The first one gets the results of the query then the second one gets the
 results of the first, etc.  The second rescore will "see" the sorting done
diff --git a/docs/reference/search/request/script-fields.asciidoc b/docs/reference/search/request/script-fields.asciidoc
index b544c79e4f202..403cad9c5dd01 100644
--- a/docs/reference/search/request/script-fields.asciidoc
+++ b/docs/reference/search/request/script-fields.asciidoc
@@ -15,13 +15,13 @@ GET /_search
         "test1" : {
             "script" : {
                 "lang": "painless",
-                "inline": "doc['my_field_name'].value * 2"
+                "source": "doc['my_field_name'].value * 2"
             }
         },
         "test2" : {
             "script" : {
                 "lang": "painless",
-                "inline": "doc['my_field_name'].value * factor",
+                "source": "doc['my_field_name'].value * factor",
                 "params" : {
                     "factor"  : 2.0
                 }
@@ -37,9 +37,9 @@ Script fields can work on fields that are not stored (`my_field_name` in
 the above case), and allow to return custom values to be returned (the
 evaluated value of the script).
 
-Script fields can also access the actual `_source` document indexed and
-extract specific elements to be returned from it (can be an "object"
-type). Here is an example:
+Script fields can also access the actual `_source` document and
+extract specific elements to be returned from it by using `params['_source']`.
+Here is an example:
 
 [source,js]
 --------------------------------------------------
@@ -50,22 +50,24 @@ GET /_search
         },
         "script_fields" : {
             "test1" : {
-                "script" : "_source.obj1.obj2"
+                "script" : "params['_source']['message']"
             }
         }
     }
 --------------------------------------------------
 // CONSOLE
+// TEST[setup:twitter]
 
 Note the `_source` keyword here to navigate the json-like model.
 
 It's important to understand the difference between
-`doc['my_field'].value` and `_source.my_field`. The first, using the doc
-keyword, will cause the terms for that field to be loaded to memory
-(cached), which will result in faster execution, but more memory
+`doc['my_field'].value` and `params['_source']['my_field']`. The first,
+using the doc keyword, will cause the terms for that field to be loaded to
+memory (cached), which will result in faster execution, but more memory
 consumption. Also, the `doc[...]` notation only allows for simple valued
 fields (can't return a json object from it) and make sense only on
-non-analyzed or single term based fields.
+non-analyzed or single term based fields.  However, using `doc` is
+still the recommended way to access values from the document, if at all
+possible, because `_source` must be loaded and parsed every time it's used.
+Using `_source` is very slow.
 
-The `_source` on the other hand causes the source to be loaded, parsed,
-and then only the relevant part of the json is returned.
diff --git a/docs/reference/search/request/scroll.asciidoc b/docs/reference/search/request/scroll.asciidoc
index a082bf3ba4c50..91f7b0ec85d6b 100644
--- a/docs/reference/search/request/scroll.asciidoc
+++ b/docs/reference/search/request/scroll.asciidoc
@@ -18,8 +18,8 @@ scrolled searches and reindexing of documents from one index to another:
 
 Perl::
 
-    See https://metacpan.org/pod/Search::Elasticsearch::Bulk[Search::Elasticsearch::Bulk]
-    and https://metacpan.org/pod/Search::Elasticsearch::Scroll[Search::Elasticsearch::Scroll]
+    See https://metacpan.org/pod/Search::Elasticsearch::Client::5_0::Bulk[Search::Elasticsearch::Client::5_0::Bulk]
+    and https://metacpan.org/pod/Search::Elasticsearch::Client::5_0::Scroll[Search::Elasticsearch::Client::5_0::Scroll]
 
 Python::
 
@@ -38,16 +38,18 @@ should keep the ``search context'' alive (see <<scroll-search-context>>), eg `?s
 
 [source,js]
 --------------------------------------------------
-curl -XGET 'localhost:9200/twitter/tweet/_search?scroll=1m' -d '
+POST /twitter/tweet/_search?scroll=1m
 {
+    "size": 100,
     "query": {
         "match" : {
             "title" : "elasticsearch"
         }
     }
 }
-'
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
 
 The result from the above request includes a `_scroll_id`, which should
 be passed to the `scroll` API in order to retrieve the next batch of
@@ -55,13 +57,14 @@ results.
 
 [source,js]
 --------------------------------------------------
-curl -XGET <1> 'localhost:9200/_search/scroll' <2> -d'
+POST <1> /_search/scroll <2>
 {
     "scroll" : "1m", <3>
-    "scroll_id" : "c2Nhbjs2OzM0NDg1ODpzRlBLc0FXNlNyNm5JWUc1" <4>
+    "scroll_id" : "DXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAD4WYm9laVYtZndUQlNsdDcwakFMNjU1QQ==" <4>
 }
-'
 --------------------------------------------------
+// CONSOLE
+// TEST[continued s/DXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAD4WYm9laVYtZndUQlNsdDcwakFMNjU1QQ==/$body._scroll_id/]
 
 <1> `GET` or `POST` can be used.
 <2> The URL should not include the `index` or `type` name -- these
@@ -70,16 +73,10 @@ curl -XGET <1> 'localhost:9200/_search/scroll' <2> -d'
     for another `1m`.
 <4> The `scroll_id` parameter
 
-Each call to the `scroll` API returns the next batch of results until there
-are no more results left to return, ie the `hits` array is empty.
-
-For backwards compatibility, `scroll_id` and `scroll` can be passed in the query string.
-And the `scroll_id` can be passed in the request body
-
-[source,js]
---------------------------------------------------
-curl -XGET 'localhost:9200/_search/scroll?scroll=1m' -d 'c2Nhbjs2OzM0NDg1ODpzRlBLc0FXNlNyNm5JWUc1'
---------------------------------------------------
+The `size` parameter allows you to configure the maximum number of hits to be 
+returned with each batch of results.  Each call to the `scroll` API returns the 
+next batch of results until there are no more results left to return, ie the 
+`hits` array is empty.
 
 IMPORTANT: The initial search request and each subsequent scroll request
 returns a new `_scroll_id` -- only the most recent `_scroll_id` should be
@@ -94,14 +91,15 @@ order, this is the most efficient option:
 
 [source,js]
 --------------------------------------------------
-curl -XGET 'localhost:9200/_search?scroll=1m' -d '
+GET /_search?scroll=1m
 {
   "sort": [
     "_doc"
   ]
 }
-'
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
 
 [[scroll-search-context]]
 ==== Keeping the search context alive
@@ -130,8 +128,9 @@ You can check how many search contexts are open with the
 
 [source,js]
 ---------------------------------------
-curl -XGET localhost:9200/_nodes/stats/indices/search?pretty
+GET /_nodes/stats/indices/search
 ---------------------------------------
+// CONSOLE
 
 ==== Clear scroll API
 
@@ -143,39 +142,48 @@ cleared as soon as the scroll is not being used anymore using the
 
 [source,js]
 ---------------------------------------
-curl -XDELETE localhost:9200/_search/scroll -d '
+DELETE /_search/scroll
 {
-    "scroll_id" : ["c2Nhbjs2OzM0NDg1ODpzRlBLc0FXNlNyNm5JWUc1"]
-}'
+    "scroll_id" : "DXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAD4WYm9laVYtZndUQlNsdDcwakFMNjU1QQ=="
+}
 ---------------------------------------
+// CONSOLE
+// TEST[catch:missing]
 
 Multiple scroll IDs can be passed as array:
 
 [source,js]
 ---------------------------------------
-curl -XDELETE localhost:9200/_search/scroll -d '
+DELETE /_search/scroll
 {
-    "scroll_id" : ["c2Nhbjs2OzM0NDg1ODpzRlBLc0FXNlNyNm5JWUc1", "aGVuRmV0Y2g7NTsxOnkxaDZ"]
-}'
+    "scroll_id" : [
+      "DXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAD4WYm9laVYtZndUQlNsdDcwakFMNjU1QQ==",
+      "DnF1ZXJ5VGhlbkZldGNoBQAAAAAAAAABFmtSWWRRWUJrU2o2ZExpSGJCVmQxYUEAAAAAAAAAAxZrUllkUVlCa1NqNmRMaUhiQlZkMWFBAAAAAAAAAAIWa1JZZFFZQmtTajZkTGlIYkJWZDFhQQAAAAAAAAAFFmtSWWRRWUJrU2o2ZExpSGJCVmQxYUEAAAAAAAAABBZrUllkUVlCa1NqNmRMaUhiQlZkMWFB"
+    ]
+}
 ---------------------------------------
+// CONSOLE
+// TEST[catch:missing]
 
 All search contexts can be cleared with the `_all` parameter:
 
 [source,js]
 ---------------------------------------
-curl -XDELETE localhost:9200/_search/scroll/_all
+DELETE /_search/scroll/_all
 ---------------------------------------
+// CONSOLE
 
 The `scroll_id` can also be passed as a query string parameter or in the request body.
 Multiple scroll IDs can be passed as comma separated values:
 
 [source,js]
 ---------------------------------------
-curl -XDELETE localhost:9200/_search/scroll \
-     -d 'c2Nhbjs2OzM0NDg1ODpzRlBLc0FXNlNyNm5JWUc1,aGVuRmV0Y2g7NTsxOnkxaDZ'
+DELETE /_search/scroll/DXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAD4WYm9laVYtZndUQlNsdDcwakFMNjU1QQ==,DnF1ZXJ5VGhlbkZldGNoBQAAAAAAAAABFmtSWWRRWUJrU2o2ZExpSGJCVmQxYUEAAAAAAAAAAxZrUllkUVlCa1NqNmRMaUhiQlZkMWFBAAAAAAAAAAIWa1JZZFFZQmtTajZkTGlIYkJWZDFhQQAAAAAAAAAFFmtSWWRRWUJrU2o2ZExpSGJCVmQxYUEAAAAAAAAABBZrUllkUVlCa1NqNmRMaUhiQlZkMWFB
 ---------------------------------------
+// CONSOLE
+// TEST[catch:missing]
 
-
+[[sliced-scroll]]
 ==== Sliced Scroll
 
 For scroll queries that return a lot of documents it is possible to split the scroll in multiple slices which
@@ -183,7 +191,7 @@ can be consumed independently:
 
 [source,js]
 --------------------------------------------------
-curl -XGET 'localhost:9200/twitter/tweet/_search?scroll=1m' -d '
+GET /twitter/tweet/_search?scroll=1m
 {
     "slice": {
         "id": 0, <1>
@@ -195,9 +203,7 @@ curl -XGET 'localhost:9200/twitter/tweet/_search?scroll=1m' -d '
         }
     }
 }
-'
-
-curl -XGET 'localhost:9200/twitter/tweet/_search?scroll=1m' -d '
+GET /twitter/tweet/_search?scroll=1m
 {
     "slice": {
         "id": 1,
@@ -209,8 +215,9 @@ curl -XGET 'localhost:9200/twitter/tweet/_search?scroll=1m' -d '
         }
     }
 }
-'
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:big_twitter]
 
 <1> The id of the slice
 <2> The maximum number of slices
@@ -247,10 +254,10 @@ slice gets deterministic results.
 
 [source,js]
 --------------------------------------------------
-curl -XGET 'localhost:9200/twitter/tweet/_search?scroll=1m' -d '
+GET /twitter/tweet/_search?scroll=1m
 {
     "slice": {
-        "field": "my_random_integer_field",
+        "field": "date",
         "id": 0,
         "max": 10
     },
@@ -260,10 +267,11 @@ curl -XGET 'localhost:9200/twitter/tweet/_search?scroll=1m' -d '
         }
     }
 }
-'
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:big_twitter]
 
 For append only time-based indices, the `timestamp` field can be used safely.
 
 NOTE: By default the maximum number of slices allowed per scroll is limited to 1024.
-You can update the `index.max_slices_per_scroll` index setting to bypass this limit.
\ No newline at end of file
+You can update the `index.max_slices_per_scroll` index setting to bypass this limit.
diff --git a/docs/reference/search/request/sort.asciidoc b/docs/reference/search/request/sort.asciidoc
index 85c5d1e675fe1..89ad7ea2c479e 100644
--- a/docs/reference/search/request/sort.asciidoc
+++ b/docs/reference/search/request/sort.asciidoc
@@ -95,7 +95,7 @@ the average price per document.
 PUT /my_index/my_type/1?refresh
 {
    "product": "chocolate",
-    "price": [20, 4]
+   "price": [20, 4]
 }
 
 POST /_search
@@ -165,7 +165,10 @@ scripts and sorting by geo distance.
 The `missing` parameter specifies how docs which are missing
 the field should be treated: The `missing` value can be
 set to `_last`, `_first`, or a custom value (that
-will be used for missing docs as the sort value). For example:
+will be used for missing docs as the sort value).
+The default is `_last`.
+
+For example:
 
 [source,js]
 --------------------------------------------------
@@ -213,7 +216,7 @@ then Elasticsearch will handle it as if there was a mapping of type
 [[geo-sorting]]
 ==== Geo Distance Sorting
 
-Allow to sort by `_geo_distance`. Here is an example:
+Allow to sort by `_geo_distance`. Here is an example, assuming `pin.location` is a field of type `geo_point`:
 
 [source,js]
 --------------------------------------------------
@@ -226,7 +229,7 @@ GET /_search
                 "order" : "asc",
                 "unit" : "km",
 		"mode" : "min",
-		"distance_type" : "sloppy_arc"
+		"distance_type" : "arc"
             }
         }
     ],
@@ -241,10 +244,22 @@ GET /_search
 
 `distance_type`::
 
-    How to compute the distance. Can either be `sloppy_arc` (default), `arc` (slightly more precise but significantly slower) or `plane` (faster, but inaccurate on long distances and close to the poles).
+    How to compute the distance. Can either be `arc` (default), or `plane` (faster, but inaccurate on long distances and close to the poles).
+
+`mode`::
+
+    What to do in case a field has several geo points. By default, the shortest
+    distance is taken into account when sorting in ascending order and the
+    longest distance when sorting in descending order. Supported values are
+    `min`, `max`, `median` and `avg`.
+
+`unit`::
+
+    The unit to use when computing sort values. The default is `m` (meters).
 
-Note: the geo distance sorting supports `sort_mode` options: `min`,
-`max` and `avg`.
+NOTE: geo distance sorting does not support configurable missing values: the
+distance will always be considered equal to +Infinity+ when a document does not
+have values for the field that is used for distance computation.
 
 The following formats are supported in providing the coordinates:
 
@@ -351,9 +366,23 @@ Multiple geo points can be passed as an array containing any `geo_point` format,
 
 [source,js]
 --------------------------------------------------
-"pin.location" : [[-70, 40], [-71, 42]]
-"pin.location" : [{"lat": 40, "lon": -70}, {"lat": 42, "lon": -71}]
+GET /_search
+{
+    "sort" : [
+        {
+            "_geo_distance" : {
+                "pin.location" : [[-70, 40], [-71, 42]],
+                "order" : "asc",
+                "unit" : "km"
+            }
+        }
+    ],
+    "query" : {
+        "term" : { "user" : "kimchy" }
+    }
+}
 --------------------------------------------------
+// CONSOLE
 
 and so forth.
 
@@ -377,7 +406,7 @@ GET /_search
             "type" : "number",
             "script" : {
                 "lang": "painless",
-                "inline": "doc['field_name'].value * params.factor",
+                "source": "doc['field_name'].value * params.factor",
                 "params" : {
                     "factor" : 1.1
                 }
diff --git a/docs/reference/search/request/source-filtering.asciidoc b/docs/reference/search/request/source-filtering.asciidoc
index 03c02538a0ea6..5f42b5422bca4 100644
--- a/docs/reference/search/request/source-filtering.asciidoc
+++ b/docs/reference/search/request/source-filtering.asciidoc
@@ -5,7 +5,7 @@
 Allows to control how the `_source` field is returned with every hit.
 
 By default operations return the contents of the `_source` field unless
-you have used the `fields` parameter or if the `_source` field is disabled.
+you have used the `stored_fields` parameter or if the `_source` field is disabled.
 
 You can turn off `_source` retrieval by using the `_source` parameter:
 
diff --git a/docs/reference/search/request/stored-fields.asciidoc b/docs/reference/search/request/stored-fields.asciidoc
index b9723b40a059c..b2a64c7e1412b 100644
--- a/docs/reference/search/request/stored-fields.asciidoc
+++ b/docs/reference/search/request/stored-fields.asciidoc
@@ -38,12 +38,9 @@ GET /_search
 --------------------------------------------------
 // CONSOLE
 
+If the requested fields are not stored (`store` mapping set to `false`), they will be ignored.
 
-For backwards compatibility, if the fields parameter specifies fields which are not stored (`store` mapping set to
-`false`), it will load the `_source` and extract it from it. This functionality has been replaced by the
-<<search-request-source-filtering,source filtering>> parameter.
-
-Field values fetched from the document it self are always returned as an array. Metadata fields like `_routing` and
+Stored field values fetched from the document itself are always returned as an array. On the contrary, metadata fields like `_routing` and
 `_parent` fields are never returned as an array.
 
 Also only leaf fields can be returned via the `field` option. So object fields can't be returned and such requests
@@ -55,7 +52,7 @@ things like `_source.obj1.field1` can be used, though not recommended, as
 
 ==== Disable stored fields entirely
 
-To disable the stored fields (and metadata fields) entirely use: `\_none_`:
+To disable the stored fields (and metadata fields) entirely use: `_none_`:
 
 [source,js]
 --------------------------------------------------
diff --git a/docs/reference/search/search-shards.asciidoc b/docs/reference/search/search-shards.asciidoc
index 718d47a986f20..6f3256e71e6a4 100644
--- a/docs/reference/search/search-shards.asciidoc
+++ b/docs/reference/search/search-shards.asciidoc
@@ -3,10 +3,13 @@
 
 The search shards api returns the indices and shards that a search request would
 be executed against. This can give useful feedback for working out issues or
-planning optimizations with routing and shard preferences.
+planning optimizations with routing and shard preferences. When filtered aliases
+are used, the filter is returned as part of the `indices` section added[5.1.0].
 
 The `index` and `type` parameters may be single values, or comma-separated.
 
+The `type` parameter is deprecated deprecated[5.1.0,was ignored in previous versions].
+
 [float]
 === Usage
 
@@ -14,19 +17,19 @@ Full example:
 
 [source,js]
 --------------------------------------------------
-curl -XGET 'localhost:9200/twitter/_search_shards'
+GET /twitter/_search_shards
 --------------------------------------------------
+// CONSOLE
+// TEST[s/^/PUT twitter\n/]
 
 This will yield the following result:
 
 [source,js]
 --------------------------------------------------
 {
-  "nodes": {
-    "JklnKbD7Tyqi9TP3_Q_tBg": {
-      "name": "Rl'nnd",
-      "transport_address": "inet[/192.168.1.113:9300]"
-    }
+  "nodes": ...,
+  "indices" : {
+    "twitter": { }
   },
   "shards": [
     [
@@ -34,9 +37,10 @@ This will yield the following result:
         "index": "twitter",
         "node": "JklnKbD7Tyqi9TP3_Q_tBg",
         "primary": true,
-        "relocating_node": null,
-        "shard": 3,
-        "state": "STARTED"
+        "shard": 0,
+        "state": "STARTED",
+        "allocation_id": {"id":"0TvkCyF7TAmM1wHP4a42-A"},
+        "relocating_node": null
       }
     ],
     [
@@ -44,9 +48,10 @@ This will yield the following result:
         "index": "twitter",
         "node": "JklnKbD7Tyqi9TP3_Q_tBg",
         "primary": true,
-        "relocating_node": null,
-        "shard": 4,
-        "state": "STARTED"
+        "shard": 1,
+        "state": "STARTED",
+        "allocation_id": {"id":"fMju3hd1QHWmWrIgFnI4Ww"},
+        "relocating_node": null
       }
     ],
     [
@@ -54,9 +59,10 @@ This will yield the following result:
         "index": "twitter",
         "node": "JklnKbD7Tyqi9TP3_Q_tBg",
         "primary": true,
-        "relocating_node": null,
-        "shard": 0,
-        "state": "STARTED"
+        "shard": 2,
+        "state": "STARTED",
+        "allocation_id": {"id":"Nwl0wbMBTHCWjEEbGYGapg"},
+        "relocating_node": null
       }
     ],
     [
@@ -64,9 +70,10 @@ This will yield the following result:
         "index": "twitter",
         "node": "JklnKbD7Tyqi9TP3_Q_tBg",
         "primary": true,
-        "relocating_node": null,
-        "shard": 2,
-        "state": "STARTED"
+        "shard": 3,
+        "state": "STARTED",
+        "allocation_id": {"id":"bU_KLGJISbW0RejwnwDPKw"},
+        "relocating_node": null
       }
     ],
     [
@@ -74,32 +81,40 @@ This will yield the following result:
         "index": "twitter",
         "node": "JklnKbD7Tyqi9TP3_Q_tBg",
         "primary": true,
-        "relocating_node": null,
-        "shard": 1,
-        "state": "STARTED"
+        "shard": 4,
+        "state": "STARTED",
+        "allocation_id": {"id":"DMs7_giNSwmdqVukF7UydA"},
+        "relocating_node": null
       }
     ]
   ]
 }
 --------------------------------------------------
+// TESTRESPONSE[s/"nodes": ...,/"nodes": $body.nodes,/]
+// TESTRESPONSE[s/JklnKbD7Tyqi9TP3_Q_tBg/$body.shards.0.0.node/]
+// TESTRESPONSE[s/0TvkCyF7TAmM1wHP4a42-A/$body.shards.0.0.allocation_id.id/]
+// TESTRESPONSE[s/fMju3hd1QHWmWrIgFnI4Ww/$body.shards.1.0.allocation_id.id/]
+// TESTRESPONSE[s/Nwl0wbMBTHCWjEEbGYGapg/$body.shards.2.0.allocation_id.id/]
+// TESTRESPONSE[s/bU_KLGJISbW0RejwnwDPKw/$body.shards.3.0.allocation_id.id/]
+// TESTRESPONSE[s/DMs7_giNSwmdqVukF7UydA/$body.shards.4.0.allocation_id.id/]
 
 And specifying the same request, this time with a routing value:
 
 [source,js]
 --------------------------------------------------
-curl -XGET 'localhost:9200/twitter/_search_shards?routing=foo,baz'
+GET /twitter/_search_shards?routing=foo,baz
 --------------------------------------------------
+// CONSOLE
+// TEST[s/^/PUT twitter\n/]
 
 This will yield the following result:
 
 [source,js]
 --------------------------------------------------
 {
-  "nodes": {
-    "JklnKbD7Tyqi9TP3_Q_tBg": {
-      "name": "Rl'nnd",
-      "transport_address": "inet[/192.168.1.113:9300]"
-    }
+  "nodes": ...,
+  "indices" : {
+      "twitter": { }
   },
   "shards": [
     [
@@ -107,9 +122,10 @@ This will yield the following result:
         "index": "twitter",
         "node": "JklnKbD7Tyqi9TP3_Q_tBg",
         "primary": true,
-        "relocating_node": null,
-        "shard": 2,
-        "state": "STARTED"
+        "shard": 0,
+        "state": "STARTED",
+        "allocation_id": {"id":"0TvkCyF7TAmM1wHP4a42-A"},
+        "relocating_node": null
       }
     ],
     [
@@ -117,14 +133,19 @@ This will yield the following result:
         "index": "twitter",
         "node": "JklnKbD7Tyqi9TP3_Q_tBg",
         "primary": true,
-        "relocating_node": null,
-        "shard": 4,
-        "state": "STARTED"
+        "shard": 1,
+        "state": "STARTED",
+        "allocation_id": {"id":"fMju3hd1QHWmWrIgFnI4Ww"},
+        "relocating_node": null
       }
     ]
   ]
 }
 --------------------------------------------------
+// TESTRESPONSE[s/"nodes": ...,/"nodes": $body.nodes,/]
+// TESTRESPONSE[s/JklnKbD7Tyqi9TP3_Q_tBg/$body.shards.0.0.node/]
+// TESTRESPONSE[s/0TvkCyF7TAmM1wHP4a42-A/$body.shards.0.0.allocation_id.id/]
+// TESTRESPONSE[s/fMju3hd1QHWmWrIgFnI4Ww/$body.shards.1.0.allocation_id.id/]
 
 This time the search will only be executed against two of the shards, because
 routing values have been specified.
@@ -146,4 +167,4 @@ routing values have been specified.
 `local`::
     A boolean value whether to read the cluster state locally in order to
     determine where shards are allocated instead of using the Master node's
-    cluster state.
+    cluster state.
\ No newline at end of file
diff --git a/docs/reference/search/search-template.asciidoc b/docs/reference/search/search-template.asciidoc
index 7ed6f2de8c864..2f1079816eecf 100644
--- a/docs/reference/search/search-template.asciidoc
+++ b/docs/reference/search/search-template.asciidoc
@@ -6,20 +6,21 @@ before they are executed and fill existing templates with template parameters.
 
 [source,js]
 ------------------------------------------
-GET /_search/template
+GET _search/template
 {
-    "inline" : {
+    "source" : {
       "query": { "match" : { "{{my_field}}" : "{{my_value}}" } },
       "size" : "{{my_size}}"
     },
     "params" : {
-        "my_field" : "foo",
-        "my_value" : "bar",
+        "my_field" : "message",
+        "my_value" : "some message",
         "my_size" : 5
     }
 }
 ------------------------------------------
-
+// CONSOLE
+// TEST[setup:twitter]
 
 For more information on how Mustache templating and what kind of templating you
 can do with it check out the http://mustache.github.io/mustache.5.html[online
@@ -27,8 +28,8 @@ documentation of the mustache project].
 
 NOTE: The mustache language is implemented in elasticsearch as a sandboxed
 scripting language, hence it obeys settings that may be used to enable or
-disable scripts per language, source and operation as described in
-<<enable-dynamic-scripting, scripting docs>>
+disable scripts per language, source and operation as described in the
+<<security-script-source, scripting docs>>
 
 [float]
 ==== More template examples
@@ -38,12 +39,12 @@ disable scripts per language, source and operation as described in
 
 [source,js]
 ------------------------------------------
-GET /_search/template
+GET _search/template
 {
-    "inline": {
+    "source": {
         "query": {
-            "match": {
-                "title": "{{query_string}}"
+            "term": {
+                "message": "{{query_string}}"
             }
         }
     },
@@ -52,23 +53,28 @@ GET /_search/template
     }
 }
 ------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
 
 [float]
 ===== Converting parameters to JSON
 
-The `{{toJson}}parameter{{/toJson}}` function can be used to convert parameters
+The `{{#toJson}}parameter{{/toJson}}` function can be used to convert parameters
 like maps and array to their JSON representation:
 
 [source,js]
 ------------------------------------------
-GET /_search/template
+GET _search/template
 {
-  "inline": "{ \"query\": { \"terms\": { \"status\": {{#toJson}}status{{/toJson}} }}}",
+  "source": "{ \"query\": { \"terms\": {{#toJson}}statuses{{/toJson}} }}",
   "params": {
-    "status": [ "pending", "published" ]
+    "statuses" : {
+        "status": [ "pending", "published" ]
+    }
   }
 }
 ------------------------------------------
+// CONSOLE
 
 which is rendered as:
 
@@ -85,21 +91,24 @@ which is rendered as:
   }
 }
 ------------------------------------------
+// NOTCONSOLE
 
 A more complex example substitutes an array of JSON objects:
 
 [source,js]
 ------------------------------------------
+GET _search/template
 {
-    "inline": "{\"query\":{\"bool\":{\"must\": {{#toJson}}clauses{{/toJson}} }}}",
+    "source": "{\"query\":{\"bool\":{\"must\": {{#toJson}}clauses{{/toJson}} }}}",
     "params": {
         "clauses": [
-            { "term": "foo" },
-            { "term": "bar" }
+            { "term": { "user" : "foo" } },
+            { "term": { "user" : "bar" } }
         ]
    }
 }
 ------------------------------------------
+// CONSOLE
 
 which is rendered as:
 
@@ -110,17 +119,21 @@ which is rendered as:
       "bool" : {
         "must" : [
           {
-            "term" : "foo"
+            "term" : {
+                "user" : "foo"
+            }
           },
           {
-            "term" : "bar"
+            "term" : {
+                "user" : "bar"
+            }
           }
         ]
       }
     }
 }
 ------------------------------------------
-
+// NOTCONSOLE
 
 [float]
 ===== Concatenating array of values
@@ -130,9 +143,9 @@ values of an array as a comma delimited string:
 
 [source,js]
 ------------------------------------------
-GET /_search/template
+GET _search/template
 {
-  "inline": {
+  "source": {
     "query": {
       "match": {
         "emails": "{{#join}}emails{{/join}}"
@@ -144,6 +157,7 @@ GET /_search/template
   }
 }
 ------------------------------------------
+// CONSOLE
 
 which is rendered as:
 
@@ -157,14 +171,15 @@ which is rendered as:
     }
 }
 ------------------------------------------
+// NOTCONSOLE
 
 The function also accepts a custom delimiter:
 
 [source,js]
 ------------------------------------------
-GET /_search/template
+GET _search/template
 {
-  "inline": {
+  "source": {
     "query": {
       "range": {
         "born": {
@@ -184,6 +199,7 @@ GET /_search/template
   }
 }
 ------------------------------------------
+// CONSOLE
 
 which is rendered as:
 
@@ -202,7 +218,7 @@ which is rendered as:
 }
 
 ------------------------------------------
-
+// NOTCONSOLE
 
 [float]
 ===== Default values
@@ -212,7 +228,7 @@ A default value is written as `{{var}}{{^var}}default{{/var}}` for instance:
 [source,js]
 ------------------------------------------
 {
-  "inline": {
+  "source": {
     "query": {
       "range": {
         "line_no": {
@@ -225,6 +241,7 @@ A default value is written as `{{var}}{{^var}}default{{/var}}` for instance:
   "params": { ... }
 }
 ------------------------------------------
+// NOTCONSOLE
 
 When `params` is `{ "start": 10, "end": 15 }` this query would be rendered as:
 
@@ -239,6 +256,7 @@ When `params` is `{ "start": 10, "end": 15 }` this query would be rendered as:
   }
 }
 ------------------------------------------
+// NOTCONSOLE
 
 But when `params` is `{ "start": 10 }` this query would use the default value
 for `end`:
@@ -254,7 +272,7 @@ for `end`:
     }
 }
 ------------------------------------------
-
+// NOTCONSOLE
 
 [float]
 ===== Conditional clauses
@@ -277,6 +295,7 @@ The `params` would look like:
     }
 }
 ------------------------------------------
+// NOTCONSOLE
 <1> All three of these elements are optional.
 
 We could write the query as:
@@ -310,6 +329,7 @@ We could write the query as:
   }
 }
 ------------------------------------------
+// NOTCONSOLE
 <1> Fill in the value of param `text`
 <2> Include the `range` filter only if `line_no` is specified
 <3> Include the `gte` clause only if `line_no.start` is specified
@@ -328,22 +348,67 @@ via the REST API, should be written as a string:
 
 [source,js]
 --------------------
-"inline": "{\"query\":{\"bool\":{\"must\":{\"match\":{\"line\":\"{{text}}\"}},\"filter\":{{{#line_no}}\"range\":{\"line_no\":{{{#start}}\"gte\":\"{{start}}\"{{#end}},{{/end}}{{/start}}{{#end}}\"lte\":\"{{end}}\"{{/end}}}}{{/line_no}}}}}}"
+"source": "{\"query\":{\"bool\":{\"must\":{\"match\":{\"line\":\"{{text}}\"}},\"filter\":{{{#line_no}}\"range\":{\"line_no\":{{{#start}}\"gte\":\"{{start}}\"{{#end}},{{/end}}{{/start}}{{#end}}\"lte\":\"{{end}}\"{{/end}}}}{{/line_no}}}}}}"
 --------------------
-
+// NOTCONSOLE
 ==================================
 
+
+[float]
+===== Encoding URLs
+
+The `{{#url}}value{{/url}}` function can be used to encode a string value
+in a HTML encoding form as defined in by the http://www.w3.org/TR/html4/[HTML specification].
+
+As an example, it is useful to encode a URL:
+
+[source,js]
+------------------------------------------
+GET _render/template
+{
+    "source" : {
+        "query" : {
+            "term": {
+                "http_access_log": "{{#url}}{{host}}/{{page}}{{/url}}"
+            }
+        }
+    },
+    "params": {
+        "host": "https://www.elastic.co/",
+        "page": "learn"
+    }
+}
+------------------------------------------
+// CONSOLE
+
+The previous query will be rendered as:
+
+[source,js]
+------------------------------------------
+{
+    "template_output" : {
+        "query" : {
+            "term" : {
+                "http_access_log" : "https%3A%2F%2Fwww.elastic.co%2F%2Flearn"
+            }
+        }
+    }
+}
+------------------------------------------
+// TESTRESPONSE
+
+
 [float]
 [[pre-registered-templates]]
 ===== Pre-registered template
 
 You can register search templates by storing it in the `config/scripts` directory, in a file using the `.mustache` extension.
-In order to execute the stored template, reference it by it's name under the `template` key:
+In order to execute the stored template, reference it by its name under the `template` key:
 
 
 [source,js]
 ------------------------------------------
-GET /_search/template
+GET _search/template
 {
     "file": "storedTemplate", <1>
     "params": {
@@ -351,68 +416,109 @@ GET /_search/template
     }
 }
 ------------------------------------------
+// CONSOLE
+// TEST[catch:request]
 
 <1> Name of the query template in `config/scripts/`, i.e., `storedTemplate.mustache`.
 
-You can also register search templates by storing it in the cluster state.
-There are REST APIs to manage these indexed templates.
+You can also register search templates by using the stored scripts api.
 
 [source,js]
 ------------------------------------------
-POST /_search/template/<templatename>
+POST _scripts/<templatename>
 {
-    "template": {
-        "query": {
-            "match": {
-                "title": "{{query_string}}"
+    "script": {
+        "lang": "mustache",
+        "source": {
+            "query": {
+                "match": {
+                    "title": "{{query_string}}"
+                }
             }
         }
     }
 }
 ------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+//////////////////////////
+
+We want to be sure that the template has been created,
+because we'll use it later.
+
+[source,js]
+--------------------------------------------------
+{
+    "acknowledged" : true
+}
+--------------------------------------------------
+// TESTRESPONSE
+
+//////////////////////////
 
 This template can be retrieved by
 
 [source,js]
 ------------------------------------------
-GET /_search/template/<templatename>
+GET _scripts/<templatename>
 ------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 which is rendered as:
 
 [source,js]
 ------------------------------------------
 {
-    "template": {
-        "query": {
-            "match": {
-                "title": "{{query_string}}"
-            }
+    "script" : {
+        "lang" : "mustache",
+        "source" : "{\"query\":{\"match\":{\"title\":\"{{query_string}}\"}}}",
+        "options": {
+          "content_type" : "application/json; charset=UTF-8"
         }
-    }
+    },
+    "_id": "<templatename>",
+    "found": true
 }
 ------------------------------------------
+// TESTRESPONSE
 
 This template can be deleted by
 
 [source,js]
 ------------------------------------------
-DELETE /_search/template/<templatename>
+DELETE _scripts/<templatename>
 ------------------------------------------
+// CONSOLE
+// TEST[continued]
 
-To use an indexed template at search time use:
+//////////////////////////
+
+[source,js]
+--------------------------------------------------
+{
+    "acknowledged" : true
+}
+--------------------------------------------------
+// TESTRESPONSE
+
+//////////////////////////
 
+To use an indexed template at search time use:
 
 [source,js]
 ------------------------------------------
-GET /_search/template
+GET _search/template
 {
-    "id": "templateName", <1>
+    "id": "<templateName>", <1>
     "params": {
         "query_string": "search for these words"
     }
 }
 ------------------------------------------
+// CONSOLE
+// TEST[catch:missing]
 <1> Name of the query template stored in the `.scripts` index.
 
 [float]
@@ -422,24 +528,17 @@ A template can be rendered in a response with given parameters using
 
 [source,js]
 ------------------------------------------
-GET /_render/template
+GET _render/template
 {
-  "inline": {
-    "query": {
-      "terms": {
-        "status": [
-          "{{#status}}",
-          "{{.}}",
-          "{{/status}}"
-        ]
-      }
-    }
-  },
+  "source": "{ \"query\": { \"terms\": {{#toJson}}statuses{{/toJson}} }}",
   "params": {
-    "status": [ "pending", "published" ]
+    "statuses" : {
+        "status": [ "pending", "published" ]
+    }
   }
 }
 ------------------------------------------
+// CONSOLE
 
 This call will return the rendered template:
 
@@ -458,6 +557,7 @@ This call will return the rendered template:
   }
 }
 ------------------------------------------
+// TESTRESPONSE
 <1> `status` array has been populated with values from the `params` object.
 
 File and indexed templates can also be rendered by replacing `inline` with
@@ -465,7 +565,7 @@ File and indexed templates can also be rendered by replacing `inline` with
 
 [source,js]
 ------------------------------------------
-GET /_render/template
+GET _render/template
 {
   "file": "my_template",
   "params": {
@@ -473,18 +573,59 @@ GET /_render/template
   }
 }
 ------------------------------------------
+// CONSOLE
+// TEST[catch:request]
 
 Pre-registered templates can also be rendered using
 
 [source,js]
 ------------------------------------------
-GET /_render/template/<template_name>
+GET _render/template/<template_name>
 {
   "params": {
     "..."
   }
 }
 ------------------------------------------
+// NOTCONSOLE
+
+[float]
+===== Explain
+
+You can use `explain` parameter when running a template:
+
+[source,js]
+------------------------------------------
+GET _search/template
+{
+  "file": "my_template",
+  "params": {
+    "status": [ "pending", "published" ]
+  },
+  "explain": true
+}
+------------------------------------------
+// CONSOLE
+// TEST[catch:request]
+
+[float]
+===== Profiling
+
+You can use `profile` parameter when running a template:
+
+[source,js]
+------------------------------------------
+GET _search/template
+{
+  "file": "my_template",
+  "params": {
+    "status": [ "pending", "published" ]
+  },
+  "profile": true
+}
+------------------------------------------
+// CONSOLE
+// TEST[catch:request]
 
 [[multi-search-template]]
 == Multi Search Template
@@ -502,6 +643,7 @@ body\n
 header\n
 body\n
 --------------------------------------------------
+// NOTCONSOLE
 
 The header part supports the same `index`, `types`, `search_type`,
 `preference`, and `routing` options as the usual Multi Search API.
@@ -513,16 +655,18 @@ stored and file templates. Here is an example:
 --------------------------------------------------
 $ cat requests
 {"index": "test"}
-{"inline": {"query": {"match":  {"user" : "{{username}}" }}}, "params": {"username": "john"}} <1>
+{"source": {"query": {"match":  {"user" : "{{username}}" }}}, "params": {"username": "john"}} <1>
 {"index": "_all", "types": "accounts"}
-{"inline": {"query": {"{{query_type}}": {"name": "{{name}}" }}}, "params": {"query_type": "match_phrase_prefix", "name": "Smith"}}
+{"source": {"query": {"{{query_type}}": {"name": "{{name}}" }}}, "params": {"query_type": "match_phrase_prefix", "name": "Smith"}}
 {"index": "_all"}
 {"id": "template_1", "params": {"query_string": "search for these words" }} <2>
 {"types": "users"}
 {"file": "template_2", "params": {"field_name": "fullname", "field_value": "john smith" }} <3>
 
-$ curl -XGET localhost:9200/_msearch/template --data-binary "@requests"; echo
+$ curl -H "Content-Type: application/x-ndjson" -XGET localhost:9200/_msearch/template --data-binary "@requests"; echo
 --------------------------------------------------
+// NOTCONSOLE
+// Not converting to console because this shows how curl works
 <1> Inline search template request
 
 <2> Search template request based on a stored template
diff --git a/docs/reference/search/search.asciidoc b/docs/reference/search/search.asciidoc
index 40118e7693138..816cf54c5290c 100644
--- a/docs/reference/search/search.asciidoc
+++ b/docs/reference/search/search.asciidoc
@@ -17,42 +17,60 @@ twitter index:
 
 [source,js]
 --------------------------------------------------
-$ curl -XGET 'http://localhost:9200/twitter/_search?q=user:kimchy'
+GET /twitter/_search?q=user:kimchy
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
 
 We can also search within specific types:
 
 [source,js]
 --------------------------------------------------
-$ curl -XGET 'http://localhost:9200/twitter/tweet,user/_search?q=user:kimchy'
+GET /twitter/tweet,user/_search?q=user:kimchy
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
 
 We can also search all tweets with a certain tag across several indices
 (for example, when each user has his own index):
 
 [source,js]
 --------------------------------------------------
-$ curl -XGET 'http://localhost:9200/kimchy,elasticsearch/tweet/_search?q=tag:wow'
+GET /kimchy,elasticsearch/tweet/_search?q=tag:wow
 --------------------------------------------------
+// CONSOLE
+// TEST[s/^/PUT kimchy\nPUT elasticsearch\n/]
 
 Or we can search all tweets across all available indices using `_all`
 placeholder:
 
 [source,js]
 --------------------------------------------------
-$ curl -XGET 'http://localhost:9200/_all/tweet/_search?q=tag:wow'
+GET /_all/tweet/_search?q=tag:wow
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
 
 Or even search across all indices and all types:
 
 [source,js]
 --------------------------------------------------
-$ curl -XGET 'http://localhost:9200/_search?q=tag:wow'
+GET /_search?q=tag:wow
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
 
-By default elasticsearch rejects search requests that would query more than
-1000 shards. The reason is that such large numbers of shards make the job of
-the coordinating node very CPU and memory intensive. It is usually a better
-idea to organize data in such a way that there are fewer larger shards. In
-case you would like to bypass this limit, which is discouraged, you can update
-the `action.search.shard_count.limit` cluster setting to a greater value.
+By default elasticsearch doesn't reject any search requests based on the number
+of shards the request hits. While elasticsearch will optimize the search execution
+on the coordinating node a large number of shards can have a significant impact
+CPU and memory wise. It is usually a better idea to organize data in such a way
+that there are fewer larger shards. In case you would like to configure a soft
+limit, you can update the `action.search.shard_count.limit` cluster setting in order
+to reject search requests that hit too many shards.
+
+The search's `max_concurrent_shard_requests` request parameter can be used to control
+the maximum number of concurrent shard requests the search API will execute for this request.
+This parameter should be used to protect a singe request from overloading a cluster ie. a default
+request will hit all indices in a cluster which could cause shard request rejections if the
+number of shards per node is high. This default is based on the number of data nodes in
+the cluster but at most `256`.
\ No newline at end of file
diff --git a/docs/reference/search/suggesters.asciidoc b/docs/reference/search/suggesters.asciidoc
index 2da4a6073074a..2cfd2a4f04ab0 100644
--- a/docs/reference/search/suggesters.asciidoc
+++ b/docs/reference/search/suggesters.asciidoc
@@ -5,8 +5,12 @@ The suggest feature suggests similar looking terms based on a provided
 text by using a suggester. Parts of the suggest feature are still under
 development.
 
-The suggest request part is either defined alongside the query part in a
-`_search` request or via the REST `_suggest` endpoint.
+The suggest request part is defined alongside the query part in a `_search`
+request.
+
+NOTE: `_suggest` endpoint has been deprecated in favour of using suggest via
+`_search` endpoint. In 5.0, the `_search` endpoint has been optimized for
+suggest only search requests.
 
 [source,js]
 --------------------------------------------------
@@ -30,25 +34,6 @@ POST twitter/_search
 // CONSOLE
 // TEST[setup:twitter]
 
-Suggest requests executed against the `_suggest` endpoint should omit
-the surrounding `suggest` element which is only used if the suggest
-request is part of a search.
-
-[source,js]
---------------------------------------------------
-POST _suggest
-{
-  "my-suggestion" : {
-    "text" : "tring out Elasticsearch",
-    "term" : {
-      "field" : "message"
-    }
-  }
-}
---------------------------------------------------
-// CONSOLE
-// TEST[setup:twitter]
-
 Several suggestions can be specified per request. Each suggestion is
 identified with an arbitrary name. In the example below two suggestions
 are requested. Both `my-suggest-1` and `my-suggest-2` suggestions use
@@ -56,18 +41,20 @@ the `term` suggester, but have a different `text`.
 
 [source,js]
 --------------------------------------------------
-POST _suggest
+POST _search
 {
-  "my-suggest-1" : {
-    "text" : "tring out Elasticsearch",
-    "term" : {
-      "field" : "message"
-    }
-  },
-  "my-suggest-2" : {
-    "text" : "kmichy",
-    "term" : {
-      "field" : "user"
+  "suggest": {
+    "my-suggest-1" : {
+      "text" : "tring out Elasticsearch",
+      "term" : {
+        "field" : "message"
+      }
+    },
+    "my-suggest-2" : {
+      "text" : "kmichy",
+      "term" : {
+        "field" : "user"
+      }
     }
   }
 }
@@ -85,27 +72,34 @@ in the suggest text and if found an arbitrary number of options.
 --------------------------------------------------
 {
   "_shards": ...
-  "my-suggest-1": [ {
-    "text": "tring",
-    "offset": 0,
-    "length": 5,
-    "options": [ {"text": "trying", "score": 0.8, "freq": 1 } ]
-  }, {
-    "text": "out",
-    "offset": 6,
-    "length": 3,
-    "options": []
-  }, {
-    "text": "elasticsearch",
-    "offset": 10,
-    "length": 13,
-    "options": []
-  } ],
-  "my-suggest-2": ...
+  "hits": ...
+  "took": 2,
+  "timed_out": false,
+  "suggest": {
+    "my-suggest-1": [ {
+      "text": "tring",
+      "offset": 0,
+      "length": 5,
+      "options": [ {"text": "trying", "score": 0.8, "freq": 1 } ]
+    }, {
+      "text": "out",
+      "offset": 6,
+      "length": 3,
+      "options": []
+    }, {
+      "text": "elasticsearch",
+      "offset": 10,
+      "length": 13,
+      "options": []
+    } ],
+    "my-suggest-2": ...
+  }
 }
 --------------------------------------------------
 // TESTRESPONSE[s/"_shards": \.\.\./"_shards": "$body._shards",/]
-// TESTRESPONSE[s/"my-suggest-2": \.\.\./"my-suggest-2": "$body.my-suggest-2"/]
+// TESTRESPONSE[s/"hits": .../"hits": "$body.hits",/]
+// TESTRESPONSE[s/"took": 2,/"took": "$body.took",/]
+// TESTRESPONSE[s/"my-suggest-2": \.\.\./"my-suggest-2": "$body.suggest.my-suggest-2"/]
 
 
 Each options array contains an option object that includes the
@@ -123,17 +117,19 @@ and applies to the `my-suggest-1` and `my-suggest-2` suggestions.
 
 [source,js]
 --------------------------------------------------
-POST _suggest
+POST _search
 {
-  "text" : "tring out Elasticsearch",
-  "my-suggest-1" : {
-    "term" : {
-      "field" : "message"
-    }
-  },
-  "my-suggest-2" : {
-    "term" : {
-      "field" : "user"
+  "suggest": {
+    "text" : "tring out Elasticsearch",
+    "my-suggest-1" : {
+      "term" : {
+        "field" : "message"
+      }
+    },
+    "my-suggest-2" : {
+       "term" : {
+        "field" : "user"
+       }
     }
   }
 }
@@ -151,3 +147,5 @@ include::suggesters/phrase-suggest.asciidoc[]
 include::suggesters/completion-suggest.asciidoc[]
 
 include::suggesters/context-suggest.asciidoc[]
+
+include::suggesters/misc.asciidoc[]
diff --git a/docs/reference/search/suggesters/completion-suggest.asciidoc b/docs/reference/search/suggesters/completion-suggest.asciidoc
index 60065ce96bb52..6d61042f42c55 100644
--- a/docs/reference/search/suggesters/completion-suggest.asciidoc
+++ b/docs/reference/search/suggesters/completion-suggest.asciidoc
@@ -130,7 +130,7 @@ PUT music/song/1?refresh
 // TEST[continued]
 
 You can use the following shorthand form. Note that you can not specify
-a weight with suggestion(s).
+a weight with suggestion(s) in the shorthand form.
 
 [source,js]
 --------------------------------------------------
@@ -152,12 +152,14 @@ documents once deleted are never shown. This request:
 
 [source,js]
 --------------------------------------------------
-POST music/_suggest?pretty
+POST music/_search?pretty
 {
-    "song-suggest" : {
-        "prefix" : "nir",
-        "completion" : {
-            "field" : "suggest"
+    "suggest": {
+        "song-suggest" : {
+            "prefix" : "nir", <1>
+            "completion" : { <2>
+                "field" : "suggest" <3>
+            }
         }
     }
 }
@@ -165,6 +167,10 @@ POST music/_suggest?pretty
 // CONSOLE
 // TEST[continued]
 
+<1> Prefix used to search for suggestions
+<2> Type of suggestions
+<3> Name of the field to search for suggestions in
+
 returns this response:
 
 [source,js]
@@ -173,26 +179,33 @@ returns this response:
   "_shards" : {
     "total" : 5,
     "successful" : 5,
+    "skipped" : 0,
     "failed" : 0
   },
-  "song-suggest" : [ {
-    "text" : "nir",
-    "offset" : 0,
-    "length" : 3,
-    "options" : [ {
-      "text" : "Nirvana",
-      "_index": "music",
-      "_type": "song",
-      "_id": "1",
-      "_score": 1.0,
-      "_source": {
-        "suggest": ["Nevermind", "Nirvana"]
-      }
+  "hits": ...
+  "took": 2,
+  "timed_out": false,
+  "suggest": {
+    "song-suggest" : [ {
+      "text" : "nir",
+      "offset" : 0,
+      "length" : 3,
+      "options" : [ {
+        "text" : "Nirvana",
+        "_index": "music",
+        "_type": "song",
+        "_id": "1",
+        "_score": 1.0,
+        "_source": {
+          "suggest": ["Nevermind", "Nirvana"]
+        }
+      } ]
     } ]
-  } ]
+  }
 }
 --------------------------------------------------
-// TESTRESPONSE
+// TESTRESPONSE[s/"hits": .../"hits": "$body.hits",/]
+// TESTRESPONSE[s/"took": 2,/"took": "$body.took",/]
 
 
 IMPORTANT: `_source` meta-field must be enabled, which is the default
@@ -202,24 +215,72 @@ The configured weight for a suggestion is returned as `_score`. The
 `text` field uses the `input` of your indexed suggestion. Suggestions
 return the full document `_source` by default. The size of the `_source`
 can impact performance due to disk fetch and network transport overhead.
-For best performance, filter out unnecessary fields from the `_source`
+To save some network overhead, filter out unnecessary fields from the `_source`
 using <<search-request-source-filtering, source filtering>> to minimize
-`_source` size. The following demonstrates an example completion query
-with source filtering:
+`_source` size. Note that the _suggest endpoint doesn't support source
+filtering but using suggest on the `_search` endpoint does:
 
 [source,js]
 --------------------------------------------------
-POST music/_suggest
+POST music/_search
 {
-    "_source": "completion.*",
-    "song-suggest" : {
-        "prefix" : "nir",
-        "completion" : {
-            "field" : "suggest"
+    "_source": "suggest", <1>
+    "suggest": {
+        "song-suggest" : {
+            "prefix" : "nir",
+            "completion" : {
+                "field" : "suggest", <2>
+                "size" : 5 <3>
+            }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+<1> Filter the source to return only the `suggest` field
+<2> Name of the field to search for suggestions in
+<3> Number of suggestions to return
+
+Which should look like:
+
+[source,js]
+--------------------------------------------------
+{
+    "took": 6,
+    "timed_out": false,
+    "_shards" : {
+        "total" : 5,
+        "successful" : 5,
+        "skipped" : 0,
+        "failed" : 0
+    },
+    "hits": {
+        "total" : 0,
+        "max_score" : 0.0,
+        "hits" : []
+    },
+    "suggest": {
+        "song-suggest" : [ {
+            "text" : "nir",
+            "offset" : 0,
+            "length" : 3,
+            "options" : [ {
+                "text" : "Nirvana",
+                "_index": "music",
+                "_type": "song",
+                "_id": "1",
+                "_score": 1.0,
+                "_source": {
+                    "suggest": ["Nevermind", "Nirvana"]
+                }
+            } ]
+        } ]
+    }
+}
+--------------------------------------------------
+// TESTRESPONSE[s/"took": 6,/"took": $body.took,/]
 
 The basic completion suggester query supports the following parameters:
 
@@ -247,14 +308,16 @@ you can have a typo in your search and still get results back.
 
 [source,js]
 --------------------------------------------------
-POST music/_suggest?pretty
+POST music/_search?pretty
 {
-    "song-suggest" : {
-        "prefix" : "nor",
-        "completion" : {
-            "field" : "suggest",
-            "fuzzy" : {
-                "fuzziness" : 2
+    "suggest": {
+        "song-suggest" : {
+            "prefix" : "nor",
+            "completion" : {
+                "field" : "suggest",
+                "fuzzy" : {
+                    "fuzziness" : 2
+                }
             }
         }
     }
@@ -304,12 +367,14 @@ you can express a prefix as a regular expression
 
 [source,js]
 --------------------------------------------------
-POST music/_suggest?pretty
+POST music/_search?pretty
 {
-    "song-suggest" : {
-        "regex" : "n[ever|i]r",
-        "completion" : {
-            "field" : "suggest"
+    "suggest": {
+        "song-suggest" : {
+            "regex" : "n[ever|i]r",
+            "completion" : {
+                "field" : "suggest"
+            }
         }
     }
 }
diff --git a/docs/reference/search/suggesters/context-suggest.asciidoc b/docs/reference/search/suggesters/context-suggest.asciidoc
index 293e8bb573cdf..064f919c25d36 100644
--- a/docs/reference/search/suggesters/context-suggest.asciidoc
+++ b/docs/reference/search/suggesters/context-suggest.asciidoc
@@ -28,8 +28,7 @@ PUT place
                     "contexts": [
                         { <1>
                             "name": "place_type",
-                            "type": "category",
-                            "path": "cat"
+                            "type": "category"
                         },
                         { <2>
                             "name": "location",
@@ -138,15 +137,17 @@ filters suggestions by multiple categories:
 
 [source,js]
 --------------------------------------------------
-POST place/_suggest?pretty
+POST place/_search?pretty
 {
-    "suggest" : {
-        "prefix" : "tim",
-        "completion" : {
-            "field" : "suggest",
-            "size": 10,
-            "contexts": {
-                "place_type": [ "cafe", "restaurants" ]
+    "suggest": {
+        "place_suggestion" : {
+            "prefix" : "tim",
+            "completion" : {
+                "field" : "suggest",
+                "size": 10,
+                "contexts": {
+                    "place_type": [ "cafe", "restaurants" ]
+                }
             }
         }
     }
@@ -165,18 +166,20 @@ suggestions associated with some categories:
 
 [source,js]
 --------------------------------------------------
-POST place/_suggest?pretty
+POST place/_search?pretty
 {
-    "suggest" : {
-        "prefix" : "tim",
-        "completion" : {
-            "field" : "suggest",
-            "size": 10,
-            "contexts": {
-                "place_type": [ <1>
-                    { "context" : "cafe" },
-                    { "context" : "restaurants", "boost": 2 }
-                 ]
+    "suggest": {
+        "place_suggestion" : {
+            "prefix" : "tim",
+            "completion" : {
+                "field" : "suggest",
+                "size": 10,
+                "contexts": {
+                    "place_type": [ <1>
+                        { "context" : "cafe" },
+                        { "context" : "restaurants", "boost": 2 }
+                     ]
+                }
             }
         }
     }
@@ -220,7 +223,6 @@ at index time. At query time, suggestions can be filtered and boosted if they ar
 a certain distance of a specified geo location.
 
 Internally, geo points are encoded as geohashes with the specified precision.
-See <<geohash>> for more background details.
 
 [float]
 ===== Geo Mapping
@@ -276,17 +278,19 @@ the encoded geohash of a geo point:
 
 [source,js]
 --------------------------------------------------
-POST place/_suggest
+POST place/_search
 {
-    "suggest" : {
-        "prefix" : "tim",
-        "completion" : {
-            "field" : "suggest",
-            "size": 10,
-            "contexts": {
-                "location": {
-                    "lat": 43.662,
-                    "lon": -79.380
+    "suggest": {
+        "place_suggestion" : {
+            "prefix" : "tim",
+            "completion" : {
+                "field" : "suggest",
+                "size": 10,
+                "contexts": {
+                    "location": {
+                        "lat": 43.662,
+                        "lon": -79.380
+                    }
                 }
             }
         }
@@ -304,28 +308,30 @@ than others, as shown by the following:
 
 [source,js]
 --------------------------------------------------
-POST place/_suggest?pretty
+POST place/_search?pretty
 {
-    "suggest" : {
-        "prefix" : "tim",
-        "completion" : {
-            "field" : "suggest",
-            "size": 10,
-            "contexts": {
-                "location": [ <1>
-                    {
-                        "lat": 43.6624803,
-                        "lon": -79.3863353,
-                        "precision": 2
-                    },
-                    {
-                        "context": {
+    "suggest": {
+        "place_suggestion" : {
+            "prefix" : "tim",
+            "completion" : {
+                "field" : "suggest",
+                "size": 10,
+                "contexts": {
+                    "location": [ <1>
+                        {
                             "lat": 43.6624803,
-                            "lon": -79.3863353
+                            "lon": -79.3863353,
+                            "precision": 2
                         },
-                        "boost": 2
-                    }
-                 ]
+                        {
+                            "context": {
+                                "lat": 43.6624803,
+                                "lon": -79.3863353
+                            },
+                            "boost": 2
+                        }
+                     ]
+                }
             }
         }
     }
diff --git a/docs/reference/search/suggesters/misc.asciidoc b/docs/reference/search/suggesters/misc.asciidoc
new file mode 100644
index 0000000000000..6866df24c0960
--- /dev/null
+++ b/docs/reference/search/suggesters/misc.asciidoc
@@ -0,0 +1,84 @@
+[[returning-suggesters-type]]
+=== Returning the type of the suggester
+
+Sometimes you need to know the exact type of a suggester in order to parse its results. The `typed_keys` parameter
+ can be used to change the suggester's name in the response so that it will be prefixed by its type.
+
+Considering the following example with two suggesters `term` and `phrase`:
+
+[source,js]
+--------------------------------------------------
+POST _search?typed_keys
+{
+  "suggest": {
+    "text" : "some test mssage",
+    "my-first-suggester" : {
+      "term" : {
+        "field" : "message"
+      }
+    },
+    "my-second-suggester" : {
+      "phrase" : {
+        "field" : "message"
+      }
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
+
+In the response, the suggester names will be changed to respectively `term#my-first-suggester` and
+`phrase#my-second-suggester`, reflecting the types of each suggestion:
+
+[source,js]
+--------------------------------------------------
+{
+  "suggest": {
+    "term#my-first-suggester": [ <1>
+      {
+        "text": "some",
+        "offset": 0,
+        "length": 4,
+        "options": []
+      },
+      {
+        "text": "test",
+        "offset": 5,
+        "length": 4,
+        "options": []
+      },
+      {
+        "text": "mssage",
+        "offset": 10,
+        "length": 6,
+        "options": [
+          {
+            "text": "message",
+            "score": 0.8333333,
+            "freq": 4
+          }
+        ]
+      }
+    ],
+    "phrase#my-second-suggester": [ <2>
+      {
+        "text": "some test mssage",
+        "offset": 0,
+        "length": 16,
+        "options": [
+          {
+            "text": "some test message",
+            "score": 0.030227963
+          }
+        ]
+      }
+    ]
+  },
+  ...
+}
+--------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": "$body.took", "timed_out": false, "_shards": "$body._shards", "hits": "$body.hits"/]
+
+<1> The name `my-first-suggester` now contains the `term` prefix.
+<2> The name `my-second-suggester` now contains the `phrase` prefix.
diff --git a/docs/reference/search/suggesters/phrase-suggest.asciidoc b/docs/reference/search/suggesters/phrase-suggest.asciidoc
index dace399d650af..d622f09e27dd2 100644
--- a/docs/reference/search/suggesters/phrase-suggest.asciidoc
+++ b/docs/reference/search/suggesters/phrase-suggest.asciidoc
@@ -71,11 +71,10 @@ PUT test
     }
   }
 }
-POST test/test
+POST test/test?refresh=true
 {"title": "noble warriors"}
-POST test/test
+POST test/test?refresh=true
 {"title": "nobel prize"}
-POST _refresh
 --------------------------------------------------
 // CONSOLE
 // TESTSETUP
@@ -85,21 +84,23 @@ suggester in the same spot you'd use the `term` suggester:
 
 [source,js]
 --------------------------------------------------
-POST _suggest
+POST test/_search
 {
-  "text": "noble prize",
-  "simple_phrase": {
-    "phrase": {
-      "field": "title.trigram",
-      "size": 1,
-      "gram_size": 3,
-      "direct_generator": [ {
+  "suggest": {
+    "text": "noble prize",
+    "simple_phrase": {
+      "phrase": {
         "field": "title.trigram",
-        "suggest_mode": "always"
-      } ],
-      "highlight": {
-        "pre_tag": "<em>",
-        "post_tag": "</em>"
+        "size": 1,
+        "gram_size": 3,
+        "direct_generator": [ {
+          "field": "title.trigram",
+          "suggest_mode": "always"
+        } ],
+        "highlight": {
+          "pre_tag": "<em>",
+          "post_tag": "</em>"
+        }
       }
     }
   }
@@ -107,32 +108,34 @@ POST _suggest
 --------------------------------------------------
 // CONSOLE
 
-The response contains suggestions scored by the most likely spell
-correction first. In this case we received the expected correction
-`xorr the god jewel` first while the second correction is less
-conservative where only one of the errors is corrected. Note, the
-request is executed with `max_errors` set to `0.5` so 50% of the terms
-can contain misspellings (See parameter descriptions below).
+The response contains suggestions scored by the most likely spell correction first. In this case we received the expected correction "nobel prize".
 
 [source,js]
 --------------------------------------------------
 {
   "_shards": ...
-  "simple_phrase" : [
-    {
-      "text" : "noble prize",
-      "offset" : 0,
-      "length" : 11,
-      "options" : [ {
-        "text" : "nobel prize",
-        "highlighted": "<em>nobel</em> prize",
-        "score" : 0.40765354
-      }]
-    }
-  ]
+  "hits": ...
+  "timed_out": false,
+  "took": 3,
+  "suggest": {
+    "simple_phrase" : [
+      {
+        "text" : "noble prize",
+        "offset" : 0,
+        "length" : 11,
+        "options" : [ {
+          "text" : "nobel prize",
+          "highlighted": "<em>nobel</em> prize",
+          "score" : 0.5962314
+        }]
+      }
+    ]
+  }
 }
 --------------------------------------------------
 // TESTRESPONSE[s/"_shards": .../"_shards": "$body._shards",/]
+// TESTRESPONSE[s/"hits": .../"hits": "$body.hits",/]
+// TESTRESPONSE[s/"took": 3,/"took": "$body.took",/]
 
 ==== Basic Phrase suggest API parameters
 
@@ -223,28 +226,30 @@ can contain misspellings (See parameter descriptions below).
 
 [source,js]
 --------------------------------------------------
-POST _suggest
+POST _search
 {
-  "text" : "noble prize",
-  "simple_phrase" : {
-    "phrase" : {
-      "field" :  "title.trigram",
-      "size" :   1,
-      "direct_generator" : [ {
-        "field" :            "title.trigram",
-        "suggest_mode" :     "always",
-        "min_word_length" :  1
-      } ],
-      "collate": {
-        "query": { <1>
-          "inline" : {
-            "match": {
-              "{{field_name}}" : "{{suggestion}}" <2>
+  "suggest": {
+    "text" : "noble prize",
+    "simple_phrase" : {
+      "phrase" : {
+        "field" :  "title.trigram",
+        "size" :   1,
+        "direct_generator" : [ {
+          "field" :            "title.trigram",
+          "suggest_mode" :     "always",
+          "min_word_length" :  1
+        } ],
+        "collate": {
+          "query": { <1>
+            "source" : {
+              "match": {
+                "{{field_name}}" : "{{suggestion}}" <2>
+              }
             }
-          }
-        },
-        "params": {"field_name" : "title"}, <3>
-        "prune": true <4>
+          },
+          "params": {"field_name" : "title"}, <3>
+          "prune": true <4>
+        }
       }
     }
   }
@@ -321,7 +326,7 @@ The direct generators support the following parameters:
     filtered out using `confidence`. Three possible values can be specified:
     ** `missing`: Only generate suggestions for terms that are not in the
                  shard. This is the default.
-    ** `popular`: Only suggest terms that occur in more docs on the shard then
+    ** `popular`: Only suggest terms that occur in more docs on the shard than
                  the original term.
     ** `always`: Suggest any matching suggestions based on terms in the
                  suggest text.
@@ -387,22 +392,24 @@ accept ordinary analyzer names.
 
 [source,js]
 --------------------------------------------------
-POST _suggest
+POST _search
 {
-  "text" : "obel prize",
-  "simple_phrase" : {
-    "phrase" : {
-      "field" : "title.trigram",
-      "size" : 1,
-      "direct_generator" : [ {
+  "suggest": {
+    "text" : "obel prize",
+    "simple_phrase" : {
+      "phrase" : {
         "field" : "title.trigram",
-        "suggest_mode" : "always"
-      }, {
-        "field" : "title.reverse",
-        "suggest_mode" : "always",
-        "pre_filter" : "reverse",
-        "post_filter" : "reverse"
-      } ]
+        "size" : 1,
+        "direct_generator" : [ {
+          "field" : "title.trigram",
+          "suggest_mode" : "always"
+        }, {
+          "field" : "title.reverse",
+          "suggest_mode" : "always",
+          "pre_filter" : "reverse",
+          "post_filter" : "reverse"
+        } ]
+      }
     }
   }
 }
diff --git a/docs/reference/search/suggesters/term-suggest.asciidoc b/docs/reference/search/suggesters/term-suggest.asciidoc
index ff5cd38cf2144..f76b17e0ed298 100644
--- a/docs/reference/search/suggesters/term-suggest.asciidoc
+++ b/docs/reference/search/suggesters/term-suggest.asciidoc
@@ -45,7 +45,7 @@ doesn't take the query into account that is part of request.
 +
      ** `missing`:  Only provide suggestions for suggest text terms that are
                     not in the index. This is the default.
-     ** `popular`:  Only suggest suggestions that occur in more docs then
+     ** `popular`:  Only suggest suggestions that occur in more docs than
                     the original suggest text term.
      ** `always`:   Suggest any matching suggestions based on terms in the
                     suggest text.
diff --git a/docs/reference/search/uri-request.asciidoc b/docs/reference/search/uri-request.asciidoc
index 95ce6a8ff6a29..9ae2d06835172 100644
--- a/docs/reference/search/uri-request.asciidoc
+++ b/docs/reference/search/uri-request.asciidoc
@@ -23,6 +23,7 @@ And here is a sample response:
     "_shards":{
         "total" : 1,
         "successful" : 1,
+        "skipped" : 0,
         "failed" : 0
     },
     "hits":{
@@ -64,12 +65,14 @@ query.
 
 |`analyzer` |The analyzer name to be used when analyzing the query string.
 
-|`lowercase_expanded_terms` |Should terms be automatically lowercased or
-not. Defaults to `true`.
-
 |`analyze_wildcard` |Should wildcard and prefix queries be analyzed or
 not. Defaults to `false`.
 
+|`batched_reduce_size` | The number of shard results that should be reduced
+at once on the coordinating node. This value should be used as a protection
+mechanism to reduce the memory overhead per search request if the potential
+number of shards in the request can be large.
+
 |`default_operator` |The default operator to be used, can be `AND` or
 `OR`. Defaults to `OR`.
 
diff --git a/docs/reference/search/validate.asciidoc b/docs/reference/search/validate.asciidoc
index 5fb4ad9b7ce15..1626c5b87ce3a 100644
--- a/docs/reference/search/validate.asciidoc
+++ b/docs/reference/search/validate.asciidoc
@@ -52,9 +52,6 @@ query.
 |`lenient` |If set to true will cause format based failures (like
 providing text to a numeric field) to be ignored. Defaults to false.
 
-|`lowercase_expanded_terms` |Should terms be automatically lowercased or
-not. Defaults to `true`.
-
 |`analyze_wildcard` |Should wildcard and prefix queries be analyzed or
 not. Defaults to `false`.
 |=======================================================================
@@ -133,24 +130,24 @@ When the query is valid, the explanation defaults to the string
 representation of that query. With `rewrite` set to `true`, the explanation
 is more detailed showing the actual Lucene query that will be executed.
 
-For Fuzzy Queries:
+For More Like This:
 
 [source,js]
 --------------------------------------------------
 GET twitter/tweet/_validate/query?rewrite=true
 {
   "query": {
-    "match": {
-      "user": {
-        "query": "kimchy",
-        "fuzziness": "auto"
-      }
+    "more_like_this": {
+      "like": {
+        "_id": "2"
+      },
+      "boost_terms": 1
     }
   }
 }
 --------------------------------------------------
 // CONSOLE
-// TEST[skip:https://github.com/elastic/elasticsearch/issues/18254]
+// TEST[skip:the output is randomized depending on which shard we hit]
 
 Response:
 
@@ -167,54 +164,80 @@ Response:
       {
          "index": "twitter",
          "valid": true,
-         "explanation": "+user:kimchy +user:kimchi^0.75 #(ConstantScore(_type:tweet))^0.0"
+         "explanation": "((user:terminator^3.71334 plot:future^2.763601 plot:human^2.8415773 plot:sarah^3.4193945 plot:kyle^3.8244398 plot:cyborg^3.9177752 plot:connor^4.040236 plot:reese^4.7133346 ... )~6) -ConstantScore(_uid:tweet#2)) #(ConstantScore(_type:tweet))^0.0"
       }
    ]
 }
 --------------------------------------------------
 // TESTRESPONSE
 
-For More Like This:
+By default, the request is executed on a single shard only, which is randomly
+selected. The detailed explanation of the query may depend on which shard is
+being hit, and therefore may vary from one request to another. So, in case of
+query rewrite the `all_shards` parameter should be used to get response from
+all available shards.
+
+For Fuzzy Queries:
 
 [source,js]
 --------------------------------------------------
-GET twitter/tweet/_validate/query?rewrite=true
+GET twitter/tweet/_validate/query?rewrite=true&all_shards=true
 {
   "query": {
-    "more_like_this": {
-      "like": {
-        "_id": "2"
-      },
-      "boost_terms": 1
+    "match": {
+      "user": {
+        "query": "kimchy",
+        "fuzziness": "auto"
+      }
     }
   }
 }
 --------------------------------------------------
 // CONSOLE
-// TEST[skip:https://github.com/elastic/elasticsearch/issues/18254]
 
 Response:
 
 [source,js]
 --------------------------------------------------
 {
-   "valid": true,
-   "_shards": {
-      "total": 1,
-      "successful": 1,
-      "failed": 0
-   },
-   "explanations": [
-      {
-         "index": "twitter",
-         "valid": true,
-         "explanation": "((user:terminator^3.71334 plot:future^2.763601 plot:human^2.8415773 plot:sarah^3.4193945 plot:kyle^3.8244398 plot:cyborg^3.9177752 plot:connor^4.040236 plot:reese^4.7133346 ... )~6) -ConstantScore(_uid:tweet#2)) #(ConstantScore(_type:tweet))^0.0"
-      }
-   ]
+  "valid": true,
+  "_shards": {
+    "total": 5,
+    "successful": 5,
+    "failed": 0
+  },
+  "explanations": [
+    {
+      "index": "twitter",
+      "shard": 0,
+      "valid": true,
+      "explanation": "+MatchNoDocsQuery(\"empty BooleanQuery\") #ConstantScore(MatchNoDocsQuery(\"empty BooleanQuery\"))"
+    },
+    {
+      "index": "twitter",
+      "shard": 1,
+      "valid": true,
+      "explanation": "+MatchNoDocsQuery(\"empty BooleanQuery\") #ConstantScore(MatchNoDocsQuery(\"empty BooleanQuery\"))"
+    },
+    {
+      "index": "twitter",
+      "shard": 2,
+      "valid": true,
+      "explanation": "(user:kimchi)^0.8333333"
+    },
+    {
+      "index": "twitter",
+      "shard": 3,
+      "valid": true,
+      "explanation": "user:kimchy"
+    },
+    {
+      "index": "twitter",
+      "shard": 4,
+      "valid": true,
+      "explanation": "+MatchNoDocsQuery(\"empty BooleanQuery\") #ConstantScore(MatchNoDocsQuery(\"empty BooleanQuery\"))"
+    }
+  ]
 }
 --------------------------------------------------
 // TESTRESPONSE
-
-CAUTION: The request is executed on a single shard only, which is randomly
-selected. The detailed explanation of the query may depend on which shard is
-being hit, and therefore may vary from one request to another.
diff --git a/docs/reference/setup.asciidoc b/docs/reference/setup.asciidoc
index ae3c5b3beb458..164307c640c4a 100644
--- a/docs/reference/setup.asciidoc
+++ b/docs/reference/setup.asciidoc
@@ -1,5 +1,5 @@
 [[setup]]
-= Setup Elasticsearch
+= Set up Elasticsearch
 
 [partintro]
 --
@@ -34,6 +34,10 @@ refuse to start if a known-bad version of Java is used.
 The version of Java that Elasticsearch will use can be configured by setting
 the `JAVA_HOME` environment variable.
 
+NOTE: Elasticsearch ships with default configuration for running Elasticsearch on 64-bit server JVMs. If you are using a 32-bit client JVM,
+you must remove `-server` from <<jvm-options,jvm.options>> and if you are using any 32-bit JVM you should reconfigure the thread stack size
+from `-Xss1m` to `-Xss320k`.
+
 --
 
 include::setup/install.asciidoc[]
@@ -42,6 +46,8 @@ include::setup/configuration.asciidoc[]
 
 include::setup/important-settings.asciidoc[]
 
+include::setup/secure-settings.asciidoc[]
+
 include::setup/bootstrap-checks.asciidoc[]
 
 include::setup/sysconfig.asciidoc[]
diff --git a/docs/reference/setup/bootstrap-checks.asciidoc b/docs/reference/setup/bootstrap-checks.asciidoc
index 47b51975ea550..2cca78388d124 100644
--- a/docs/reference/setup/bootstrap-checks.asciidoc
+++ b/docs/reference/setup/bootstrap-checks.asciidoc
@@ -23,14 +23,43 @@ documented individually.
 [float]
 === Development vs. production mode
 
-By default, Elasticsearch binds and publishes to `localhost`. This is
-fine for downloading and playing with Elasticsearch, and everyday
-development but it's useless for production systems. For a production
-installation to be reachable, it must either bind or publish to an
-external interface. Thus, we consider Elasticsearch to be in development
-mode if it does not bind nor publish to an external interface (the
-default), and is otherwise in production mode if it does bind or publish
-to an external interface.
+By default, Elasticsearch binds to `localhost` for <<modules-http,HTTP>> and
+<<modules-transport,transport (internal)>> communication. This is fine for
+downloading and playing with Elasticsearch, and everyday development but it's
+useless for production systems. To join a cluster, an Elasticsearch node must be
+reachable via transport communication. To join a cluster over an external
+network interface, a node must bind transport to an external interface and not
+be using <<single-node-discovery,single-node discovery>>. Thus, we consider an
+Elasticsearch node to be in development mode if it can not form a cluster with
+another machine over an external network interface, and is otherwise in
+production mode if it can join a cluster over an external interface.
+
+Note that HTTP and transport can be configured independently via
+<<modules-http,`http.host`>> and <<modules-transport,`transport.host`>>; this
+can be useful for configuring a single node to be reachable via HTTP for testing
+purposes without triggering production mode.
+
+[[single-node-discovery]]
+[float]
+=== Single-node discovery
+We recognize that some users need to bind transport to an external interface for
+testing their usage of the transport client. For this situation, we provide the
+discovery type `single-node` (configure it by setting `discovery.type` to
+`single-node`); in this situation, a node will elect itself master and will not
+join a cluster with any other node.
+
+
+[float]
+=== Forcing the bootstrap checks
+If you are running a single node in production, it is possible to evade the
+bootstrap checks (either by not binding transport to an external interface, or
+by binding transport to an external interface and setting the discovery type to
+`single-node`). For this situation, you can force execution of the bootstrap
+checks by setting the system property `es.enforce.bootstrap.checks` to `true`
+(set this in <<jvm-options>>, or by adding `-Des.enforce.bootstrap.checks=true`
+to the environment variable `ES_JAVA_OPTS`). We strongly encourage you to do
+this if you are in this specific situation. This system property can be used to
+force execution of the bootstrap checks independent of the node configuration.
 
 === Heap size check
 
@@ -38,11 +67,11 @@ If a JVM is started with unequal initial and max heap size, it can be
 prone to pauses as the JVM heap is resized during system usage. To avoid
 these resize pauses, it's best to start the JVM with the initial heap
 size equal to the maximum heap size. Additionally, if
-<<bootstrap.memory_lock,`bootstrap.memory_lock`>> is enabled, the JVM will
-lock the initial size of the heap on startup. If the initial heap size
-is not equal to the maximum heap size, after a resize it will not be the
-case that all of the JVM heap is locked in memory. To pass the heap size
-check, you must configure the <<heap-size,heap size>>.
+<<bootstrap.memory_lock,`bootstrap.memory_lock`>> is enabled, the JVM
+will lock the initial size of the heap on startup. If the initial heap
+size is not equal to the maximum heap size, after a resize it will not
+be the case that all of the JVM heap is locked in memory. To pass the
+heap size check, you must configure the <<heap-size,heap size>>.
 
 
 === File descriptor check
@@ -51,7 +80,7 @@ File descriptors are a Unix construct for tracking open "files". In Unix
 though, https://en.wikipedia.org/wiki/Everything_is_a_file[everything is
 a file]. For example, "files" could be a physical file, a virtual file
 (e.g., `/proc/loadavg`), or network sockets. Elasticsearch requires
-lots file descriptors (e.g., every shard is composed of multiple
+lots of file descriptors (e.g., every shard is composed of multiple
 segments and other files, plus connections to other nodes, etc.). This
 bootstrap check is enforced on OS X and Linux. To pass the file
 descriptor check, you might have to configure <<file-descriptors,file
@@ -66,14 +95,15 @@ Elasticsearch would much rather use to service requests. There are
 several ways to configure a system to disallow swapping. One way is by
 requesting the JVM to lock the heap in memory through `mlockall` (Unix)
 or virtual lock (Windows). This is done via the Elasticsearch setting
-<<bootstrap.memory_lock,`bootstrap.memory_lock`>>. However, there are cases
-where this setting can be passed to Elasticsearch but Elasticsearch is
-not able to lock the heap (e.g., if the `elasticsearch` user does not
-have `memlock unlimited`). The memory lock check verifies that *if* the
-`bootstrap.memory_lock` setting is enabled, that the JVM was successfully
-able to lock the heap. To pass the memory lock check, you might have to
-configure <<mlockall,`mlockall`>>.
-
+<<bootstrap.memory_lock,`bootstrap.memory_lock`>>. However, there are
+cases where this setting can be passed to Elasticsearch but
+Elasticsearch is not able to lock the heap (e.g., if the `elasticsearch`
+user does not have `memlock unlimited`). The memory lock check verifies
+that *if* the `bootstrap.memory_lock` setting is enabled, that the JVM
+was successfully able to lock the heap. To pass the memory lock check,
+you might have to configure <<mlockall,`mlockall`>>.
+
+[[max-number-threads-check]]
 === Maximum number of threads check
 
 Elasticsearch executes requests by breaking the request down into stages
@@ -101,9 +131,22 @@ process has unlimited address space and is enforced only on Linux. To
 pass the maximum size virtual memory check, you must configure your
 system to allow the Elasticsearch process the ability to have unlimited
 address space. This can be done via `/etc/security/limits.conf` using
-the `as` setting to `unlimited` (note that you might have to increaes
+the `as` setting to `unlimited` (note that you might have to increase
 the limits for the `root` user too).
 
+=== Max file size check
+
+The segment files that are the components of individual shards and the translog
+generations that are components of the translog can get large (exceeding
+multiple gigabytes). On systems where the max size of files that can be created
+by the Elasticsearch process is limited, this can lead to failed
+writes. Therefore, the safest option here is that the max file size is unlimited
+and that is what the max file size bootstrap check enforces. To pass the max
+file check, you must configure your system to allow the Elasticsearch process
+the ability to write files of unlimited size. This can be done via
+`/etc/security/limits.conf` using the `fsize` setting to `unlimited` (note that
+you might have to increase the limits for the `root` user too).
+
 === Maximum map count check
 
 Continuing from the previous <<max-size-virtual-memory-check,point>>, to
@@ -127,6 +170,34 @@ systems and operating systems, the server VM is the
 default. Additionally, Elasticsearch is configured by default to force
 the server VM.
 
+=== Use serial collector check
+
+There are various garbage collectors for the OpenJDK-derived JVMs
+targeting different workloads. The serial collector in particular is
+best suited for single logical CPU machines or extremely small heaps,
+neither of which are suitable for running Elasticsearch. Using the
+serial collector with Elasticsearch can be devastating for performance.
+The serial collector check ensures that Elasticsearch is not configured
+to run with the serial collector. To pass the serial collector check,
+you must not start Elasticsearch with the serial collector (whether it's
+from the defaults for the JVM that you're using, or you've explicitly
+specified it with `-XX:+UseSerialGC`). Note that the default JVM
+configuration that ships with Elasticsearch configures Elasticsearch to
+use the CMS collector.
+
+[[system-call-filter-check]]
+=== System call filter check
+Elasticsearch installs system call filters of various flavors depending
+on the operating system (e.g., seccomp on Linux). These system call
+filters are installed to prevent the ability to execute system calls
+related to forking as a defense mechanism against arbitrary code
+execution attacks on Elasticsearch The system call filter check ensures
+that if system call filters are enabled, then they were successfully
+installed. To pass the system call filter check you must either fix any
+configuration errors on your system that prevented system call filters
+from installing (check your logs), or *at your own risk* disable system
+call filters by setting `bootstrap.system_call_filter` to `false`.
+
 === OnError and OnOutOfMemoryError checks
 
 The JVM options `OnError` and `OnOutOfMemoryError` enable executing
@@ -142,3 +213,18 @@ enabled. This check is always enforced. To pass this check do not enable
 use the JVM flag `ExitOnOutOfMemoryError`. While this does not have the
 full capabilities of `OnError` nor `OnOutOfMemoryError`, arbitrary
 forking will not be supported with seccomp enabled.
+
+=== Early-access check
+
+The OpenJDK project provides early-access snapshots of upcoming releases. These
+releases are not suitable for production. The early-access check detects these
+early-access snapshots. To pass this check, you must start Elasticsearch on a
+release build of the JVM.
+
+=== G1GC check
+
+Early versions of the HotSpot JVM that shipped with JDK 8 are known to
+have issues that can lead to index corruption when the G1GC collector is
+enabled.  The versions impacted are those earlier than the version of
+HotSpot that shipped with JDK 8u40. The G1GC check detects these early
+versions of the HotSpot JVM.
diff --git a/docs/reference/setup/cluster_restart.asciidoc b/docs/reference/setup/cluster_restart.asciidoc
index 7f9f8f89afac8..68844cb522883 100644
--- a/docs/reference/setup/cluster_restart.asciidoc
+++ b/docs/reference/setup/cluster_restart.asciidoc
@@ -8,7 +8,9 @@ required.
 
 The process to perform an upgrade with a full cluster restart is as follows:
 
-==== Step 1: Disable shard allocation
+. *Disable shard allocation*
++
+--
 
 When you shut down a node, the allocation process will immediately try to
 replicate the shards that were on that node to other nodes in the cluster,
@@ -26,8 +28,11 @@ PUT _cluster/settings
 --------------------------------------------------
 // CONSOLE
 // TEST[skip:indexes don't assign]
+--
 
-==== Step 2: Perform a synced flush
+. *Perform a synced flush*
++
+--
 
 Shard recovery will be much faster if you stop indexing and issue a
 <<indices-synced-flush, synced-flush>> request:
@@ -41,19 +46,28 @@ POST _flush/synced
 A synced flush request is a ``best effort'' operation. It will fail if there
 are any pending indexing operations, but it is safe to reissue the request
 multiple times if necessary.
+--
 
-==== Step 3: Shutdown and upgrade all nodes
+. *Shutdown and upgrade all nodes*
++
+--
 
 Stop all Elasticsearch services on all nodes in the cluster. Each node can be
 upgraded following the same procedure described in <<upgrade-node>>.
+--
 
-==== Step 4: Upgrade any plugins
+. *Upgrade any plugins*
++
+--
 
 Elasticsearch plugins must be upgraded when upgrading a node.  Use the
 `elasticsearch-plugin` script to install the correct version of any plugins
 that you need.
+--
 
-==== Step 5: Start the cluster
+. *Start the cluster*
++
+--
 
 If you have dedicated master nodes -- nodes with `node.master` set to
 `true`(the default) and `node.data` set to `false` --  then it is a good idea
@@ -75,8 +89,11 @@ GET _cat/nodes
 // CONSOLE
 
 Use these APIs to check that all nodes have successfully joined the cluster.
+--
 
-==== Step 6: Wait for yellow
+. *Wait for yellow*
++
+--
 
 As soon as each node has joined the cluster, it will start to recover any
 primary shards that are stored locally.  Initially, the
@@ -87,8 +104,11 @@ Once each node has recovered its local shards, the `status` will become
 `yellow`, meaning all primary shards have been recovered, but not all replica
 shards are allocated.  This is to be expected because allocation is still
 disabled.
+--
 
-==== Step 7: Reenable allocation
+. *Reenable allocation*
++
+--
 
 Delaying the allocation of replicas until all nodes have joined the cluster
 allows the master to allocate replicas to nodes which already have local shard
@@ -105,6 +125,7 @@ PUT _cluster/settings
 }
 ------------------------------------------------------
 // CONSOLE
+// TEST[warning:[script.max_compilations_per_minute] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version.]
 
 The cluster will now start allocating replica shards to all data nodes. At this
 point it is safe to resume indexing and searching, but your cluster will
@@ -124,3 +145,4 @@ GET _cat/recovery
 
 Once the `status` column in the `_cat/health` output has reached `green`, all
 primary and replica shards have been successfully allocated.
+--
diff --git a/docs/reference/setup/configuration.asciidoc b/docs/reference/setup/configuration.asciidoc
index 518fb24a8bde0..c3a8b936d3cbd 100644
--- a/docs/reference/setup/configuration.asciidoc
+++ b/docs/reference/setup/configuration.asciidoc
@@ -51,7 +51,7 @@ path.logs: /var/log/elasticsearch
 --------------------------------------------------
 
 [float]
-=== Environment variable subsitution
+=== Environment variable substitution
 
 Environment variables referenced with the `${...}` notation within the
 configuration file will be replaced with the value of the environment
@@ -89,45 +89,36 @@ Enter value for [node.name]:
 NOTE: Elasticsearch will not start if `${prompt.text}` or `${prompt.secret}`
 is used in the settings and the process is run as a service or in the background.
 
-[float]
-=== Setting default settings
-
-New default settings may be specified on the command line using the
-`default.` prefix.  This will specify a value that will be used by
-default unless another value is specified in the config file.
-
-For instance, if Elasticsearch is started as follows:
-
-[source,sh]
----------------------------
-./bin/elasticsearch -Edefault.node.name=My_Node
----------------------------
-
-the value for `node.name` will be `My_Node`, unless it is overwritten on the
-command line with `es.node.name` or in the config file with `node.name`.
-
 [float]
 [[logging]]
 == Logging configuration
 
-Elasticsearch uses http://logging.apache.org/log4j/2.x/[Log4j 2] for
+Elasticsearch uses https://logging.apache.org/log4j/2.x/[Log4j 2] for
 logging. Log4j 2 can be configured using the log4j2.properties
-file. Elasticsearch exposes a single property `${sys:es.logs}` that can be
-referenced in the configuration file to determine the location of the log files;
-this will resolve to a prefix for the Elasticsearch log file at runtime.
+file. Elasticsearch exposes three properties, `${sys:es.logs.base_path}`,
+`${sys:es.logs.cluster_name}`, and `${sys:es.logs.node_name}` (if the node name
+is explicitly set via `node.name`) that can be referenced in the configuration
+file to determine the location of the log files. The property
+`${sys:es.logs.base_path}` will resolve to the log directory,
+`${sys:es.logs.cluster_name}` will resolve to the cluster name (used as the
+prefix of log filenames in the default configuration), and
+`${sys:es.logs.node_name}` will resolve to the node name (if the node name is
+explicitly set).
 
 For example, if your log directory (`path.logs`) is `/var/log/elasticsearch` and
-your cluster is named `production` then `${sys:es.logs}` will resolve to
-`/var/log/elasticsearch/production`.
+your cluster is named `production` then `${sys:es.logs.base_path}` will resolve
+to `/var/log/elasticsearch` and
+`${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log`
+will resolve to `/var/log/elasticsearch/production.log`.
 
 [source,properties]
 --------------------------------------------------
 appender.rolling.type = RollingFile <1>
 appender.rolling.name = rolling
-appender.rolling.fileName = ${sys:es.logs}.log <2>
+appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log <2>
 appender.rolling.layout.type = PatternLayout
 appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %.10000m%n
-appender.rolling.filePattern = ${sys:es.logs}-%d{yyyy-MM-dd}.log <3>
+appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}.log <3>
 appender.rolling.policies.type = Policies
 appender.rolling.policies.time.type = TimeBasedTriggeringPolicy <4>
 appender.rolling.policies.time.interval = 1 <5>
@@ -142,19 +133,117 @@ appender.rolling.policies.time.modulate = true <6>
 <6> Align rolls on the day boundary (as opposed to rolling every twenty-four
     hours)
 
+NOTE: Log4j's configuration parsing gets confused by any extraneous whitespace;
+if you copy and paste any Log4j settings on this page, or enter any Log4j
+configuration in general, be sure to trim any leading and trailing whitespace.
+
 If you append `.gz` or `.zip` to `appender.rolling.filePattern`, then the logs
 will be compressed as they are rolled.
 
+If you want to retain log files for a specified period of time, you can use a
+rollover strategy with a delete action.
+
+[source,properties]
+--------------------------------------------------
+appender.rolling.strategy.type = DefaultRolloverStrategy <1>
+appender.rolling.strategy.action.type = Delete <2>
+appender.rolling.strategy.action.basepath = ${sys:es.logs.base_path} <3>
+appender.rolling.strategy.action.condition.type = IfLastModified <4>
+appender.rolling.strategy.action.condition.age = 7D <5>
+appender.rolling.strategy.action.PathConditions.type = IfFileName <6>
+appender.rolling.strategy.action.PathConditions.glob = ${sys:es.logs.cluster_name}-* <7>
+--------------------------------------------------
+
+<1> Configure the `DefaultRolloverStrategy`
+<2> Configure the `Delete` action for handling rollovers
+<3> The base path to the Elasticsearch logs
+<4> The condition to apply when handling rollovers
+<5> Retain logs for seven days
+<6> Only delete files older than seven days if they match the specified glob
+<7> Delete files from the base path matching the glob
+    `${sys:es.logs.cluster_name}-*`; this is the glob that log files are rolled
+    to; this is needed to only delete the rolled Elasticsearch logs but not also
+    delete the deprecation and slow logs
+
 Multiple configuration files can be loaded (in which case they will get merged)
 as long as they are named `log4j2.properties` and have the Elasticsearch config
 directory as an ancestor; this is useful for plugins that expose additional
-loggers.  The logger section contains the java packages and their corresponding
-log level, where it is possible to omit the `org.elasticsearch` prefix. The
-appender section contains the destinations for the logs. Extensive information
-on how to customize logging and all the supported appenders can be found on the
+loggers. The logger section contains the java packages and their corresponding
+log level. The appender section contains the destinations for the logs.
+Extensive information on how to customize logging and all the supported
+appenders can be found on the
 http://logging.apache.org/log4j/2.x/manual/configuration.html[Log4j
 documentation].
 
+[float]
+[[configuring-logging-levels]]
+=== Configuring logging levels
+
+There are four ways to configuring logging levels, each having situations in which they are appropriate to use.
+
+1. Via the command-line: `-E <name of logging hierarchy>=<level>` (e.g.,
+   `-E logger.org.elasticsearch.transport=trace`). This is most appropriate when
+   you are temporarily debugging a problem on a single node (for example, a
+   problem with startup, or during development).
+2. Via `elasticsearch.yml`: `<name of logging hierarchy>: <level>` (e.g.,
+   `logger.org.elasticsearch.transport: trace`). This is most appropriate when
+   you are temporarily debugging a problem but are not starting Elasticsearch
+   via the command-line (e.g., via a service) or you want a logging level
+   adjusted on a more permanent basis.
+3. Via <<cluster-logger,cluster settings>>:
++
+--
+[source,js]
+-------------------------------
+PUT /_cluster/settings
+{
+  "transient": {
+    "<name of logging hierarchy>": "<level>"
+  }
+}
+-------------------------------
+// NOTCONSOLE
+
+For example:
+
+[source,js]
+-------------------------------
+PUT /_cluster/settings
+{
+  "transient": {
+    "logger.org.elasticsearch.transport": "trace"
+  }
+}
+-------------------------------
+// CONSOLE
+// TEST[warning:[script.max_compilations_per_minute] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version.]
+
+This is most appropriate when you need to dynamically need to adjust a logging
+level on an actively-running cluster.
+
+--
+4. Via the `log4j2.properties`:
++
+--
+[source,properties]
+--------------------------------------------------
+logger.<unique_identifier>.name = <name of logging hierarchy>
+logger.<unique_identifier>.level = <level>
+--------------------------------------------------
+
+For example:
+
+[source,properties]
+--------------------------------------------------
+logger.transport.name = org.elasticsearch.transport
+logger.transport.level = trace
+--------------------------------------------------
+
+This is most appropriate when you need fine-grained control over the logger (for
+example, you want to send the logger to another file, or manage the logger
+differently; this is a rare use-case).
+--
+
 [float]
 [[deprecation-logging]]
 === Deprecation logging
@@ -179,4 +268,4 @@ logs to roll and compress after 1 GB, and to preserve a maximum of five log
 files (four rolled logs, and the active log).
 
 You can disable it in the `config/log4j2.properties` file by setting the deprecation
-log level to `info`.
+log level to `error`.
diff --git a/docs/reference/setup/important-settings.asciidoc b/docs/reference/setup/important-settings.asciidoc
index aecd205b613af..fc43f5869a916 100644
--- a/docs/reference/setup/important-settings.asciidoc
+++ b/docs/reference/setup/important-settings.asciidoc
@@ -12,7 +12,6 @@ configured before going into production.
 * <<network.host,`network.host`>>
 * <<unicast.hosts,`discovery.zen.ping.unicast.hosts`>>
 * <<minimum_master_nodes,`discovery.zen.minimum_master_nodes`>>
-* <<node.max_local_storage_nodes,`node.max_local_storage_nodes`>>
 
 [float]
 [[path-settings]]
@@ -70,7 +69,7 @@ environments, otherwise you might end up with nodes joining the wrong cluster.
 [[node.name]]
 === `node.name`
 
-By default, Elasticsearch will take the 7 first charachter of the randomly generated uuid used as the node id.
+By default, Elasticsearch will take the 7 first character of the randomly generated uuid used as the node id.
 Note that the node id is persisted and does not change when a node restarts and therefore the default node name
 will also not change.
 
@@ -151,7 +150,7 @@ discovery.zen.ping.unicast.hosts:
    - 192.168.1.11 <1>
    - seeds.mydomain.com <2>
 --------------------------------------------------
-<1> The port will default to 9300 if not specified.
+<1> The port will default to `transport.profiles.default.port` and fallback to `transport.tcp.port` if not specified.
 <2> A hostname that resolves to multiple IP addresses will try all resolved addresses.
 
 [float]
@@ -159,16 +158,15 @@ discovery.zen.ping.unicast.hosts:
 === `discovery.zen.minimum_master_nodes`
 
 To prevent data loss, it is vital to configure the
-`discovery.zen.minimum_master_nodes setting` so that each master-eligible node
+`discovery.zen.minimum_master_nodes` setting so that each master-eligible node
 knows the _minimum number of master-eligible nodes_ that must be visible in
 order to form a cluster.
 
 Without this setting, a cluster that suffers a network failure is at risk of
 having the cluster split into two independent clusters -- a split brain --
-which will lead to data loss.  A more detailed explanation is provided
+which will lead to data loss. A more detailed explanation is provided
 in <<split-brain>>.
 
-
 To avoid a split brain, this setting should be set to a _quorum_ of master-
 eligible nodes:
 
@@ -182,35 +180,3 @@ nodes should be set to `(3 / 2) + 1` or `2`:
 discovery.zen.minimum_master_nodes: 2
 --------------------------------------------------
 
-IMPORTANT: If `discovery.zen.minimum_master_nodes` is not set when
-Elasticsearch is running in <<dev-vs-prod,production mode>>, an exception will
-be thrown which will prevent the node from starting.
-
-[float]
-[[node.max_local_storage_nodes]]
-=== `node.max_local_storage_nodes`
-
-It is possible to start more than one node on the same server from the same
-`$ES_HOME`, just by doing the following:
-
-[source,sh]
---------------------------------------------------
-./bin/elasticsearch -d
-./bin/elasticsearch -d
---------------------------------------------------
-
-This works just fine: the data directory structure is designed to let multiple
-nodes coexist.  However, a single instance of Elasticsearch is able to use all
-of the resources of a single server and it seldom makes sense to run multiple
-nodes on the same server in production.
-
-It is, however, possible to start more than one node on the same server by
-mistake and to be completely unaware that this problem exists. To prevent more
-than one node from sharing the same data directory, it is advisable to add the
-following setting:
-
-[source,yaml]
---------------------------------------------------
-node.max_local_storage_nodes: 1
---------------------------------------------------
-
diff --git a/docs/reference/setup/install.asciidoc b/docs/reference/setup/install.asciidoc
index 322bd0e0d2424..f145728f8e35c 100644
--- a/docs/reference/setup/install.asciidoc
+++ b/docs/reference/setup/install.asciidoc
@@ -7,9 +7,9 @@ Elasticsearch is provided in the following package formats:
 `zip`/`tar.gz`::
 
 The `zip` and `tar.gz` packages are suitable for installation on any system
-and are the easiest choice for getting started with Elasticsearch.
+and are the easiest choice for getting started with Elasticsearch on most systems.
 +
-<<zip-targz>> or <<windows>>
+<<zip-targz>> or <<zip-windows>>
 
 `deb`::
 
@@ -27,6 +27,20 @@ Elasticsearch website or from our RPM repository.
 +
 <<rpm>>
 
+`msi`::
+
+The `msi` package is suitable for installation on Windows 64-bit systems with at least 
+.NET 4.5 framework installed, and is the easiest choice for getting started with 
+Elasticsearch on Windows. MSIs may be downloaded from the Elasticsearch website.
++
+<<windows>>
+
+`docker`::
+
+An image is available for running Elasticsearch as a Docker container. It ships with {xpack-ref}/index.html[X-Pack] pre-installed and may be downloaded from the Elastic Docker Registry.
++
+<<docker>>
+
 [float]
 [[config-mgmt-tools]]
 === Configuration Management Tools
@@ -37,14 +51,17 @@ large deployments:
 [horizontal]
 Puppet:: https://github.com/elastic/puppet-elasticsearch[puppet-elasticsearch]
 Chef::   https://github.com/elastic/cookbook-elasticsearch[cookbook-elasticsearch]
+Ansible:: https://github.com/elastic/ansible-elasticsearch[ansible-elasticsearch]
 
 
 include::install/zip-targz.asciidoc[]
 
+include::install/zip-windows.asciidoc[]
+
 include::install/deb.asciidoc[]
 
 include::install/rpm.asciidoc[]
 
 include::install/windows.asciidoc[]
 
-
+include::install/docker.asciidoc[]
diff --git a/docs/reference/setup/install/check-running.asciidoc b/docs/reference/setup/install/check-running.asciidoc
index 2e255ec35e5d9..a3ba7fc9123be 100644
--- a/docs/reference/setup/install/check-running.asciidoc
+++ b/docs/reference/setup/install/check-running.asciidoc
@@ -16,6 +16,7 @@ which should give you a response something like this:
 {
   "name" : "Cp8oag6",
   "cluster_name" : "elasticsearch",
+  "cluster_uuid" : "AT69_T_DTp-1qgIJlatQqA",
   "version" : {
     "number" : "{version}",
     "build_hash" : "f27399d",
@@ -28,6 +29,7 @@ which should give you a response something like this:
 --------------------------------------------
 // TESTRESPONSE[s/"name" : "Cp8oag6",/"name" : "$body.name",/]
 // TESTRESPONSE[s/"cluster_name" : "elasticsearch",/"cluster_name" : "$body.cluster_name",/]
+// TESTRESPONSE[s/"cluster_uuid" : "AT69_T_DTp-1qgIJlatQqA",/"cluster_uuid" : "$body.cluster_uuid",/]
 // TESTRESPONSE[s/"build_hash" : "f27399d",/"build_hash" : "$body.version.build_hash",/]
 // TESTRESPONSE[s/"build_date" : "2016-03-30T09:51:41.449Z",/"build_date" : $body.version.build_date,/]
 // TESTRESPONSE[s/"build_snapshot" : false,/"build_snapshot" : $body.version.build_snapshot,/]
diff --git a/docs/reference/setup/install/deb.asciidoc b/docs/reference/setup/install/deb.asciidoc
index a9bf4170791a4..7fae4cce6fec0 100644
--- a/docs/reference/setup/install/deb.asciidoc
+++ b/docs/reference/setup/install/deb.asciidoc
@@ -1,7 +1,7 @@
 [[deb]]
 === Install Elasticsearch with Debian Package
 
-The Debian package for Elasticsearch can be <<install-rpm,downloaded from our website>>
+The Debian package for Elasticsearch can be <<install-deb,downloaded from our website>>
 or from our  <<deb-repo,APT repository>>. It can be used to install
 Elasticsearch on any Debian-based system such as Debian and Ubuntu.
 
@@ -9,6 +9,10 @@ The latest stable version of Elasticsearch can be found on the
 link:/downloads/elasticsearch[Download Elasticsearch] page. Other versions can
 be found on the link:/downloads/past-releases[Past Releases page].
 
+NOTE: Elasticsearch requires Java 8 or later. Use the
+http://www.oracle.com/technetwork/java/javase/downloads/index.html[official Oracle distribution]
+or an open-source distribution such as http://openjdk.java.net[OpenJDK].
+
 [[deb-key]]
 ==== Import the Elasticsearch PGP Key
 
@@ -16,12 +20,20 @@ include::key.asciidoc[]
 
 [source,sh]
 -------------------------
-wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
+wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
 -------------------------
 
 [[deb-repo]]
 ==== Installing from the APT repository
 
+ifeval::["{release-state}"=="unreleased"]
+
+Version {version} of Elasticsearch has not yet been released.
+
+endif::[]
+
+ifeval::["{release-state}"!="unreleased"]
+
 You may need to install the `apt-transport-https` package on Debian before proceeding:
 
 [source,sh]
@@ -29,25 +41,44 @@ You may need to install the `apt-transport-https` package on Debian before proce
 sudo apt-get install apt-transport-https
 --------------------------------------------------
 
-Save the repository definition to  +/etc/apt/sources.list.d/elasticsearch-{major-version}.list+:
+Save the repository definition to  +/etc/apt/sources.list.d/elastic-{major-version}.list+:
+
+ifeval::["{release-state}"=="released"]
 
 ["source","sh",subs="attributes,callouts"]
 --------------------------------------------------
-echo "deb https://packages.elastic.co/elasticsearch/{major-version}/debian stable main" | sudo tee -a /etc/apt/sources.list.d/elasticsearch-{major-version}.list
+echo "deb https://artifacts.elastic.co/packages/{major-version}/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-{major-version}.list
 --------------------------------------------------
 
-[WARNING]
+endif::[]
+
+ifeval::["{release-state}"=="prerelease"]
+
+["source","sh",subs="attributes,callouts"]
+--------------------------------------------------
+echo "deb https://artifacts.elastic.co/packages/{major-version}/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-{major-version}.list
+--------------------------------------------------
+
+endif::[]
+
+
+[NOTE]
 ==================================================
 
-Do not use `add-apt-repository` as it will add a `deb-src` entry as well, but
-we do not provide a source package. If you have added the `deb-src` entry, you
-will see an error like the following:
+These instructions do not use `add-apt-repository` for several reasons:
+
+. `add-apt-repository` adds entries to the system `/etc/apt/sources.list` file
+  rather than a clean per-repository file in `/etc/apt/sources.list.d`
+. `add-apt-repository` is not part of the default install on many distributions
+  and requires a number of non-default dependencies.
+. Older versions of `add-apt-repository` always add a `deb-src` entry which
+  will cause errors because we do not provide a source package. If you have added
+  the `deb-src` entry, you will see an error like the following until you delete
+  the `deb-src` line:
 
     Unable to find expected entry 'main/source/Sources' in Release file
     (Wrong sources.list entry or malformed file)
 
-Delete the `deb-src` entry from the `/etc/apt/sources.list` file and the
-installation should work as expected.
 ==================================================
 
 You can install the Elasticsearch Debian package with:
@@ -63,25 +94,38 @@ If two entries exist for the same Elasticsearch repository, you will see an erro
 
 ["literal",subs="attributes,callouts"]
 
-Duplicate sources.list entry https://packages.elastic.co/elasticsearch/{major-version}/debian/ ...`
+Duplicate sources.list entry https://artifacts.elastic.co/packages/{major-version}/apt/ ...`
 
 Examine +/etc/apt/sources.list.d/elasticsearch-{major-version}.list+ for the duplicate entry or locate the duplicate entry amongst the files in `/etc/apt/sources.list.d/` and the `/etc/apt/sources.list` file.
 ==================================================
 
+endif::[]
+
+include::skip-set-kernel-parameters.asciidoc[]
 
 [[install-deb]]
 ==== Download and install the Debian package manually
 
-The Debian package for Elastisearch v{version} can be downloaded from the website and installed as follows:
+ifeval::["{release-state}"=="unreleased"]
+
+Version {version} of Elasticsearch has not yet been released.
+
+endif::[]
+
+ifeval::["{release-state}"!="unreleased"]
+
+The Debian package for Elasticsearch v{version} can be downloaded from the website and installed as follows:
 
 ["source","sh",subs="attributes"]
 --------------------------------------------
-wget https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/deb/elasticsearch/{version}/elasticsearch-{version}.deb
+wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.deb
 sha1sum elasticsearch-{version}.deb <1>
 sudo dpkg -i elasticsearch-{version}.deb
 --------------------------------------------
 <1> Compare the SHA produced by `sha1sum` or `shasum` with the
-    https://download.elastics.co/elasticsearch/release/org/elasticsearch/distribution/deb/elasticsearch/{version}/elasticsearch-{version}.deb.sha1[published SHA].
+    https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.deb.sha1[published SHA].
+
+endif::[]
 
 include::init-systemd.asciidoc[]
 
@@ -116,9 +160,7 @@ include::check-running.asciidoc[]
 [[deb-configuring]]
 ==== Configuring Elasticsearch
 
-Elasticsearch loads its configuration from the `/etc/elasticsearch/elasticsearch.yml`
-file by default.  The format of this config file is explained in
-<<settings>>.
+include::etc-elasticsearch.asciidoc[]
 
 The Debian package also has a system configuration file (`/etc/default/elasticsearch`),
 which allows you to set the following parameters:
@@ -174,6 +216,7 @@ locations for a Debian-based system:
 | plugins
   | Plugin files location. Each plugin will be contained in a subdirectory.
   | /usr/share/elasticsearch/plugins
+  |
 
 | repo
   | Shared file system repository locations. Can hold multiple locations. A file system repository can be placed in to any subdirectory of any directory specified here.
diff --git a/docs/reference/setup/install/docker.asciidoc b/docs/reference/setup/install/docker.asciidoc
new file mode 100644
index 0000000000000..db5518694abd6
--- /dev/null
+++ b/docs/reference/setup/install/docker.asciidoc
@@ -0,0 +1,297 @@
+[[docker]]
+=== Install Elasticsearch with Docker
+
+Elasticsearch is also available as a Docker image.
+The image is built with {xpack-ref}/index.html[X-Pack] and uses https://hub.docker.com/_/centos/[centos:7] as the base image.
+The source code can be found on https://github.com/elastic/elasticsearch-docker/tree/{branch}[GitHub].
+
+==== Security note
+
+NOTE: {xpack-ref}/index.html[X-Pack] is preinstalled in this image.
+Please take a few minutes to familiarize yourself with {xpack-ref}/security-getting-started.html[X-Pack Security] and how to change default passwords. The default password for the `elastic` user is `changeme`.
+
+NOTE: X-Pack includes a trial license for 30 days. After that, you can obtain one of the https://www.elastic.co/subscriptions[available subscriptions] or {ref}/security-settings.html[disable Security]. The Basic license is free and includes the https://www.elastic.co/products/x-pack/monitoring[Monitoring] extension.
+
+Obtaining Elasticsearch for Docker is as simple as issuing a +docker pull+ command against the Elastic Docker registry.
+
+ifeval::["{release-state}"=="unreleased"]
+
+WARNING: Version {version} of Elasticsearch has not yet been released, so no Docker image is currently available for this version.
+
+endif::[]
+
+ifeval::["{release-state}"!="unreleased"]
+
+The Docker image can be retrieved with the following command:
+
+["source","sh",subs="attributes"]
+--------------------------------------------
+docker pull {docker-image}
+--------------------------------------------
+
+endif::[]
+
+[[docker-cli-run]]
+==== Running Elasticsearch from the command line
+
+[[docker-cli-run-dev-mode]]
+===== Development mode
+
+ifeval::["{release-state}"=="unreleased"]
+
+WARNING: Version {version} of the Elasticsearch Docker image has not yet been released.
+
+endif::[]
+
+ifeval::["{release-state}"!="unreleased"]
+
+Elasticsearch can be quickly started for development or testing use with the following command:
+
+["source","sh",subs="attributes"]
+--------------------------------------------
+docker run -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" {docker-image}
+--------------------------------------------
+
+endif::[]
+
+[[docker-cli-run-prod-mode]]
+===== Production mode
+
+[[docker-prod-prerequisites]]
+[IMPORTANT]
+=========================
+
+The `vm.max_map_count` kernel setting needs to be set to at least `262144` for production use.
+Depending on your platform:
+
+* Linux
++
+The `vm.max_map_count` setting should be set permanently in /etc/sysctl.conf:
++
+[source,sh]
+--------------------------------------------
+$ grep vm.max_map_count /etc/sysctl.conf
+vm.max_map_count=262144
+----------------------------------
++
+To apply the setting on a live system type: `sysctl -w vm.max_map_count=262144`
++
+* OSX with https://docs.docker.com/engine/installation/mac/#/docker-for-mac[Docker for Mac]
++
+The `vm.max_map_count` setting must be set within the xhyve virtual machine:
++
+["source","sh"]
+--------------------------------------------
+$ screen ~/Library/Containers/com.docker.docker/Data/com.docker.driver.amd64-linux/tty
+--------------------------------------------
++
+Log in with 'root' and no password.
+Then configure the `sysctl` setting as you would for Linux:
++
+["source","sh"]
+--------------------------------------------
+sysctl -w vm.max_map_count=262144
+--------------------------------------------
++
+* OSX with https://docs.docker.com/engine/installation/mac/#docker-toolbox[Docker Toolbox]
++
+The `vm.max_map_count` setting must be set via docker-machine:
++
+["source","sh"]
+--------------------------------------------
+docker-machine ssh
+sudo sysctl -w vm.max_map_count=262144
+--------------------------------------------
+=========================
+
+The following example brings up a cluster comprising two Elasticsearch nodes.
+To bring up the cluster, use the <<docker-prod-cluster-composefile,`docker-compose.yml`>> and just type:
+
+ifeval::["{release-state}"=="unreleased"]
+
+WARNING: Version {version} of the Elasticsearch Docker image has not yet been released, so a `docker-compose.yml` is not available for this version.
+
+endif::[]
+
+ifeval::["{release-state}"!="unreleased"]
+
+["source","sh"]
+--------------------------------------------
+docker-compose up
+--------------------------------------------
+
+endif::[]
+
+[NOTE]
+`docker-compose` is not pre-installed with Docker on Linux.
+Instructions for installing it can be found on the https://docs.docker.com/compose/install/#install-using-pip[docker-compose webpage].
+
+The node `elasticsearch1` listens on `localhost:9200` while `elasticsearch2` talks to `elasticsearch1` over a Docker network.
+
+This example also uses https://docs.docker.com/engine/tutorials/dockervolumes[Docker named volumes], called `esdata1` and `esdata2` which will be created if not already present.
+
+[[docker-prod-cluster-composefile]]
+`docker-compose.yml`:
+ifeval::["{release-state}"=="unreleased"]
+
+WARNING: Version {version} of the Elasticsearch Docker image has not yet been released, so a `docker-compose.yml` is not available for this version.
+
+endif::[]
+
+ifeval::["{release-state}"!="unreleased"]
+["source","yaml",subs="attributes"]
+--------------------------------------------
+version: '2'
+services:
+  elasticsearch1:
+    image: docker.elastic.co/elasticsearch/elasticsearch:{version}
+    container_name: elasticsearch1
+    environment:
+      - cluster.name=docker-cluster
+      - bootstrap.memory_lock=true
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    mem_limit: 1g
+    volumes:
+      - esdata1:/usr/share/elasticsearch/data
+    ports:
+      - 9200:9200
+    networks:
+      - esnet
+  elasticsearch2:
+    image: docker.elastic.co/elasticsearch/elasticsearch:{version}
+    environment:
+      - cluster.name=docker-cluster
+      - bootstrap.memory_lock=true
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+      - "discovery.zen.ping.unicast.hosts=elasticsearch1"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    mem_limit: 1g
+    volumes:
+      - esdata2:/usr/share/elasticsearch/data
+    networks:
+      - esnet
+
+volumes:
+  esdata1:
+    driver: local
+  esdata2:
+    driver: local
+
+networks:
+  esnet:
+--------------------------------------------
+endif::[]
+
+To stop the cluster, type `docker-compose down`. Data volumes will persist, so it's possible to start the cluster again with the same data using `docker-compose up`.
+To destroy the cluster **and the data volumes** just type `docker-compose down -v`.
+
+===== Inspect status of cluster:
+
+["source","sh"]
+--------------------------------------------
+curl -u elastic http://127.0.0.1:9200/_cat/health
+Enter host password for user 'elastic':
+1472225929 15:38:49 docker-cluster green 2 2 4 2 0 0 0 0 - 100.0%
+--------------------------------------------
+// NOTCONSOLE
+// This is demonstrating curl. Console will prompt you for a username and
+// password so no need to demonstrate that. Converting this would not show the
+// important `-u elastic` parameters for `curl`.
+
+Log messages go to the console and are handled by the configured Docker logging driver. By default you can access logs with `docker logs`.
+
+[[docker-configuration-methods]]
+==== Configuring Elasticsearch with Docker
+
+Elasticsearch loads its configuration from files under `/usr/share/elasticsearch/config/`. These configuration files are documented in <<settings>> and <<jvm-options>>.
+
+The image offers several methods for configuring Elasticsearch settings with the conventional approach being to provide customized files, i.e. `elasticsearch.yml`, but it's also possible to use environment variables to set options:
+
+===== A. Present the parameters via Docker environment variables
+For example, to define the cluster name with `docker run` you can pass `-e "cluster.name=mynewclustername"`. Double quotes are required.
+
+===== B. Bind-mounted configuration
+Create your custom config file and mount this over the image's corresponding file.
+For example, bind-mounting a `custom_elasticsearch.yml` with `docker run` can be accomplished with the parameter:
+
+["source","sh"]
+--------------------------------------------
+-v full_path_to/custom_elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
+--------------------------------------------
+IMPORTANT: The container **runs Elasticsearch as user `elasticsearch` using uid:gid `1000:1000`**. Bind mounted host directories and files, such as `custom_elasticsearch.yml` above, **need to be accessible by this user**. For the https://www.elastic.co/guide/en/elasticsearch/reference/current/important-settings.html#path-settings[data and log dirs], such as `/usr/share/elasticsearch/data`, write access is required as well.
+
+===== C. Customized image
+In some environments, it may make more sense to prepare a custom image containing your configuration. A `Dockerfile` to achieve this may be as simple as:
+
+["source","sh",subs="attributes"]
+--------------------------------------------
+FROM docker.elastic.co/elasticsearch/elasticsearch:{version}
+ADD elasticsearch.yml /usr/share/elasticsearch/config/
+USER root
+RUN chown elasticsearch:elasticsearch config/elasticsearch.yml
+USER elasticsearch
+--------------------------------------------
+
+You could then build and try the image with something like:
+
+["source","sh"]
+--------------------------------------------
+docker build --tag=elasticsearch-custom .
+docker run -ti -v /usr/share/elasticsearch/data elasticsearch-custom
+--------------------------------------------
+
+===== D. Override the image's default https://docs.docker.com/engine/reference/run/#cmd-default-command-or-options[CMD]
+
+Options can be passed as command-line options to the Elasticsearch process by
+overriding the default command for the image. For example:
+
+["source","sh"]
+--------------------------------------------
+docker run <various parameters> bin/elasticsearch -Ecluster.name=mynewclustername
+--------------------------------------------
+
+==== Notes for production use and defaults
+
+We have collected a number of best practices for production use.
+
+NOTE: Any Docker parameters mentioned below assume the use of `docker run`.
+
+. Elasticsearch runs inside the container as user `elasticsearch` using uid:gid `1000:1000`. If you are bind-mounting a local directory or file, ensure it is readable by this user, while the <<path-settings,data and log dirs>> additionally require write access.
++
+. It is important to ensure increased ulimits for <<setting-system-settings,nofile>> and <<max-number-threads-check,nproc>> are available for the Elasticsearch containers. Verify the https://github.com/moby/moby/tree/ea4d1243953e6b652082305a9c3cda8656edab26/contrib/init[init system] for the Docker daemon is already setting those to acceptable values and, if needed, adjust them in the Daemon, or override them per container, for example using `docker run`:
++
+  --ulimit nofile=65536:65536
++
+NOTE: One way of checking the Docker daemon defaults for the aforementioned ulimits is by running:
++
+  docker run --rm centos:7 /bin/bash -c 'ulimit -Hn && ulimit -Sn && ulimit -Hu && ulimit -Su'
++
+. Swapping needs to be disabled for performance and node stability. This can be achieved through any of the methods mentioned in the <<setup-configuration-memory,Elasticsearch docs>>. If you opt for the `bootstrap.memory_lock: true` approach, apart from defining it through any of the <<docker-configuration-methods,configuration methods>>, you will additionally need the `memlock: true` ulimit, either defined in the https://docs.docker.com/engine/reference/commandline/dockerd/#default-ulimits[Docker Daemon] or specifically set for the container. This has been demonstrated earlier in the <<docker-prod-cluster-composefile,docker-compose.yml>>, or using `docker run`:
++
+  -e "bootstrap.memory_lock=true" --ulimit memlock=-1:-1
++
+. The image https://docs.docker.com/engine/reference/builder/#/expose[exposes] TCP ports 9200 and 9300. For clusters it is recommended to randomize the published ports with `--publish-all`, unless you are pinning one container per host.
++
+. Use the `ES_JAVA_OPTS` environment variable to set heap size, e.g. to use 16GB use `-e ES_JAVA_OPTS="-Xms16g -Xmx16g"` with `docker run`. It is also recommended to set a https://docs.docker.com/engine/reference/run/#user-memory-constraints[memory limit] for the container.
++
+. Pin your deployments to a specific version of the Elasticsearch Docker image, e.g. +docker.elastic.co/elasticsearch/elasticsearch:{version}+.
++
+. Always use a volume bound on `/usr/share/elasticsearch/data`, as shown in the <<docker-cli-run-prod-mode,production example>>, for the following reasons:
++
+.. The data of your elasticsearch node won't be lost if the container is killed
+.. Elasticsearch is I/O sensitive and the Docker storage driver is not ideal for fast I/O
+.. It allows the use of advanced https://docs.docker.com/engine/extend/plugins/#volume-plugins[Docker volume plugins]
++
+. If you are using the devicemapper storage driver (default on at least RedHat (rpm) based distributions) make sure you are not using the default `loop-lvm` mode. Configure docker-engine to use https://docs.docker.com/engine/userguide/storagedriver/device-mapper-driver/#configure-docker-with-devicemapper[direct-lvm] instead.
++
+. Consider centralizing your logs by using a different https://docs.docker.com/engine/admin/logging/overview/[logging driver]. Also note that the default json-file logging driver is not ideally suited for production use.
+
+
+include::next-steps.asciidoc[]
diff --git a/docs/reference/setup/install/etc-elasticsearch.asciidoc b/docs/reference/setup/install/etc-elasticsearch.asciidoc
new file mode 100644
index 0000000000000..593d146bfd200
--- /dev/null
+++ b/docs/reference/setup/install/etc-elasticsearch.asciidoc
@@ -0,0 +1,12 @@
+Elasticsearch defaults to using `/etc/elasticsearch` for runtime configuration.
+The ownership of this directory and all files in this directory are set to
+`root:elasticsearch` on package installation and the directory has the `setgid`
+flag set so that any files and subdirectories created under `/etc/elasticsearch`
+are created with this ownership as well (e.g., if a keystore is created using
+the <<secure-settings,keystore tool>>). It is expected that this be maintained so
+that the Elasticsearch process can read the files under this directory via the
+group permissions.
+
+Elasticsearch loads its configuration from the
+`/etc/elasticsearch/elasticsearch.yml` file by default.  The format of this
+config file is explained in <<settings>>.
diff --git a/docs/reference/setup/install/init-systemd.asciidoc b/docs/reference/setup/install/init-systemd.asciidoc
index 3e252ca94cddf..1532c5313aefd 100644
--- a/docs/reference/setup/install/init-systemd.asciidoc
+++ b/docs/reference/setup/install/init-systemd.asciidoc
@@ -1,7 +1,7 @@
 ==== SysV `init` vs `systemd`
 
 Elasticsearch is not started automatically after installation. How to start
-and stop Elasticsearch depends on whether your sytem uses SysV `init` or
+and stop Elasticsearch depends on whether your system uses SysV `init` or
 `systemd` (used by newer distributions).  You can tell which is being used by
 running this command:
 
diff --git a/docs/reference/setup/install/rpm.asciidoc b/docs/reference/setup/install/rpm.asciidoc
index 08dfc253f0d2e..839d899eaea20 100644
--- a/docs/reference/setup/install/rpm.asciidoc
+++ b/docs/reference/setup/install/rpm.asciidoc
@@ -13,6 +13,10 @@ The latest stable version of Elasticsearch can be found on the
 link:/downloads/elasticsearch[Download Elasticsearch] page. Other versions can
 be found on the link:/downloads/past-releases[Past Releases page].
 
+NOTE: Elasticsearch requires Java 8 or later. Use the
+http://www.oracle.com/technetwork/java/javase/downloads/index.html[official Oracle distribution]
+or an open-source distribution such as http://openjdk.java.net[OpenJDK].
+
 [[rpm-key]]
 ==== Import the Elasticsearch PGP Key
 
@@ -20,28 +24,56 @@ include::key.asciidoc[]
 
 [source,sh]
 -------------------------
-rpm --import https://packages.elastic.co/GPG-KEY-elasticsearch
+rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch
 -------------------------
 
 [[rpm-repo]]
 ==== Installing from the RPM repository
 
+ifeval::["{release-state}"=="unreleased"]
+
+Version {version} of Elasticsearch has not yet been released.
+
+endif::[]
+
+ifeval::["{release-state}"!="unreleased"]
+
 Create a file called `elasticsearch.repo` in the `/etc/yum.repos.d/` directory
 for RedHat based distributions, or in the `/etc/zypp/repos.d/` directory for
 OpenSuSE based distributions, containing:
 
+ifeval::["{release-state}"=="released"]
+
+["source","sh",subs="attributes,callouts"]
+--------------------------------------------------
+[elasticsearch-{major-version}]
+name=Elasticsearch repository for {major-version} packages
+baseurl=https://artifacts.elastic.co/packages/{major-version}/yum
+gpgcheck=1
+gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
+enabled=1
+autorefresh=1
+type=rpm-md
+--------------------------------------------------
+
+endif::[]
+
+ifeval::["{release-state}"=="prerelease"]
+
 ["source","sh",subs="attributes,callouts"]
 --------------------------------------------------
 [elasticsearch-{major-version}]
 name=Elasticsearch repository for {major-version} packages
-baseurl=https://packages.elastic.co/elasticsearch/{major-version}/centos
+baseurl=https://artifacts.elastic.co/packages/{major-version}/yum
 gpgcheck=1
-gpgkey=https://packages.elastic.co/GPG-KEY-elasticsearch
+gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
 enabled=1
 autorefresh=1
 type=rpm-md
 --------------------------------------------------
 
+endif::[]
+
 And your repository is ready for use. You can now install Elasticsearch with one of the following commands:
 
 [source,sh]
@@ -54,19 +86,33 @@ sudo zypper install elasticsearch <3>
 <2> Use `dnf` on Fedora and other newer Red Hat distributions.
 <3> Use `zypper` on OpenSUSE based distributions
 
+endif::[]
+
 [[install-rpm]]
 ==== Download and install the RPM manually
 
-The RPM for Elastisearch v{version} can be downloaded from the website and installed as follows:
+ifeval::["{release-state}"=="unreleased"]
+
+Version {version} of Elasticsearch has not yet been released.
+
+endif::[]
+
+ifeval::["{release-state}"!="unreleased"]
+
+The RPM for Elasticsearch v{version} can be downloaded from the website and installed as follows:
 
 ["source","sh",subs="attributes"]
 --------------------------------------------
-wget https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/rpm/elasticsearch/{version}/elasticsearch-{version}.rpm
+wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.rpm
 sha1sum elasticsearch-{version}.rpm <1>
 sudo rpm --install elasticsearch-{version}.rpm
 --------------------------------------------
 <1> Compare the SHA produced by `sha1sum` or `shasum` with the
-    https://download.elastics.co/elasticsearch/release/org/elasticsearch/distribution/rpm/elasticsearch/{version}/elasticsearch-{version}.rpm.sha1[published SHA].
+    https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.rpm.sha1[published SHA].
+
+endif::[]
+
+include::skip-set-kernel-parameters.asciidoc[]
 
 include::init-systemd.asciidoc[]
 
@@ -102,9 +148,7 @@ include::check-running.asciidoc[]
 [[rpm-configuring]]
 ==== Configuring Elasticsearch
 
-Elasticsearch loads its configuration from the `/etc/elasticsearch/elasticsearch.yml`
-file by default.  The format of this config file is explained in
-<<settings>>.
+include::etc-elasticsearch.asciidoc[]
 
 The RPM also has a system configuration file (`/etc/sysconfig/elasticsearch`),
 which allows you to set the following parameters:
@@ -160,6 +204,7 @@ locations for an RPM-based system:
 | plugins
   | Plugin files location. Each plugin will be contained in a subdirectory.
   | /usr/share/elasticsearch/plugins
+  |
 
 | repo
   | Shared file system repository locations. Can hold multiple locations. A file system repository can be placed in to any subdirectory of any directory specified here.
diff --git a/docs/reference/setup/install/skip-set-kernel-parameters.asciidoc b/docs/reference/setup/install/skip-set-kernel-parameters.asciidoc
new file mode 100644
index 0000000000000..32faabc4b0894
--- /dev/null
+++ b/docs/reference/setup/install/skip-set-kernel-parameters.asciidoc
@@ -0,0 +1,2 @@
+NOTE: On systemd-based distributions, the installation scripts will attempt to set kernel parameters (e.g.,
+`vm.max_map_count`); you can skip this by masking the systemd-sysctl.service unit.
diff --git a/docs/reference/setup/install/sysconfig-file.asciidoc b/docs/reference/setup/install/sysconfig-file.asciidoc
index 8cf3482bf3336..1ab0057f01ea3 100644
--- a/docs/reference/setup/install/sysconfig-file.asciidoc
+++ b/docs/reference/setup/install/sysconfig-file.asciidoc
@@ -17,7 +17,7 @@
 
 `MAX_LOCKED_MEMORY`::
 
-    Maximum locked memory size. Set to `unlimited if you use the
+    Maximum locked memory size. Set to `unlimited` if you use the
     `bootstrap.memory_lock` option in elasticsearch.yml.
 
 `MAX_MAP_COUNT`::
diff --git a/docs/reference/setup/install/systemd.asciidoc b/docs/reference/setup/install/systemd.asciidoc
index 035932a83f870..bf94e95fb63df 100644
--- a/docs/reference/setup/install/systemd.asciidoc
+++ b/docs/reference/setup/install/systemd.asciidoc
@@ -18,13 +18,36 @@ sudo systemctl stop elasticsearch.service
 --------------------------------------------
 
 These commands provide no feedback as to whether Elasticsearch was started
-successfully or not.  Instead, this information will be written to the
-`systemd` journal, which can be tailed as follows:
+successfully or not. Instead, this information will be written in the log
+files located in `/var/log/elasticsearch/`.
+
+By default the Elasticsearch service doesn't log information in the `systemd`
+journal. To enable `journalctl` logging, the `--quiet` option must be removed
+ from the `ExecStart` command line in the `elasticsearch.service` file.
+
+When `systemd` logging is enabled, the logging information are available using
+the `journalctl` commands:
+
+To tail the journal:
 
 [source,sh]
 --------------------------------------------
 sudo journalctl -f
 --------------------------------------------
 
-Log files can be found in `/var/log/elasticsearch/`.
+To list journal entries for the elasticsearch service:
+
+[source,sh]
+--------------------------------------------
+sudo journalctl --unit elasticsearch
+--------------------------------------------
+
+To list journal entries for the elasticsearch service starting from a given time:
+
+[source,sh]
+--------------------------------------------
+sudo journalctl --unit elasticsearch --since  "2016-10-30 18:17:16"
+--------------------------------------------
 
+Check `man journalctl` or https://www.freedesktop.org/software/systemd/man/journalctl.html for
+more command line options.
diff --git a/docs/reference/setup/install/windows.asciidoc b/docs/reference/setup/install/windows.asciidoc
index ef0c5f2a71f69..2245fe07003c8 100644
--- a/docs/reference/setup/install/windows.asciidoc
+++ b/docs/reference/setup/install/windows.asciidoc
@@ -1,250 +1,434 @@
 [[windows]]
-=== Install Elasticsearch on Windows
+=== Install Elasticsearch with Windows MSI Installer
 
-Elasticsearch can be installed on Windows using the `.zip` package.  This
-comes with a `service.bat` command which will setup Elasticsearch to run as a
-service.
+beta[]
+
+Elasticsearch can be installed on Windows using the `.msi` package.  This can
+install Elasticsearch as a Windows service or allow it to be run manually using
+the included `elasticsearch.exe` executable.
+
+TIP: Elasticsearch has historically been installed on Windows using the <<zip-windows, .zip>> archive.
+You can continue using the `.zip` approach if you prefer.
 
 The latest stable version of Elasticsearch can be found on the
 link:/downloads/elasticsearch[Download Elasticsearch] page.
 Other versions can be found on the
 link:/downloads/past-releases[Past Releases page].
 
-[[install-windows]]
-==== Download and install the `.zip` package
+NOTE: Elasticsearch requires Java 8 or later. Use the
+http://www.oracle.com/technetwork/java/javase/downloads/index.html[official Oracle distribution]
+or an open-source distribution such as http://openjdk.java.net[OpenJDK].
 
-Download the `.zip` archive for Elastisearch v{version} from: https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/zip/elasticsearch/{version}/elasticsearch-{version}.zip
+[[download-msi]]
+==== Download the `.msi` package
 
-Unzip it with your favourite unzip tool.  This will create a folder called
-+elasticsearch-{version}+, which we will refer to as `%ES_HOME%`. In a terminal
-window, `CD` to the `%ES_HOME%` directory, for instance:
+ifeval::["{release-state}"=="unreleased"]
 
-["source","sh",subs="attributes"]
-----------------------------
-CD c:\elasticsearch-{version}
-----------------------------
+Version {version} of Elasticsearch has not yet been released.
 
-[[windows-running]]
-==== Running Elasticsearch from the command line
+endif::[]
 
-Elasticsearch can be started from the command line as follows:
+ifeval::["{release-state}"!="unreleased"]
 
-[source,sh]
---------------------------------------------
-.\bin\elasticsearch
---------------------------------------------
+Download the `.msi` package for Elasticsearch v{version} from https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.msi
 
-By default, Elasticsearch runs in the foreground, prints its logs to `STDOUT`,
-and can be stopped by pressing `Ctrl-C`.
+endif::[]
 
-[[windows-configuring]]
-==== Configuring Elasticsearch on the command line
+[[install-msi-gui]]
+==== Install using the graphical user interface (GUI)
 
-Elasticsearch loads its configuration from the `%ES_HOME%/config/elasticsearch.yml`
-file by default.  The format of this config file is explained in
-<<settings>>.
+Double-click the downloaded `.msi` package to launch a GUI wizard that will guide you through the 
+installation process. You can view help on any step by clicking the `?` button, which reveals an
+aside panel with additional information for each input:
 
-Any settings that can be specified in the config file can also be specified on
-the command line, using the `-E` syntax as follows:
+[[msi-installer-help]]
+image::images/msi_installer/msi_installer_help.png[]
 
-[source,sh]
+Within the first screen, select the directory for the installation. In addition, select directories for where
+data, logs and configuration will reside or <<msi-command-line-options,roll with the default locations>>:
+
+[[msi-installer-locations]]
+image::images/msi_installer/msi_installer_locations.png[]
+
+Then select whether to install as a service or start Elasticsearch manually as needed. When
+installing as a service, you can also decide which account to run the service under as well
+as whether the service should be started after installation and when Windows is started or 
+restarted:
+
+[[msi-installer-service]]
+image::images/msi_installer/msi_installer_service.png[]
+
+IMPORTANT: When selecting an account to run the service with, be sure that the chosen account
+has sufficient privileges to access the installation and other deployment directories chosen.
+
+Common configuration settings are exposed within the Configuration section, allowing the cluster
+name, node name and roles to be set, in addition to memory and network settings:
+
+[[msi-installer-configuration]]
+image::images/msi_installer/msi_installer_configuration.png[]
+
+Finally, the installer provides a list of common plugins that can be downloaded and installed as
+part of the installation:
+
+[[msi-installer-selected-plugins]]
+image::images/msi_installer/msi_installer_selected_plugins.png[]
+
+NOTE: X-Pack includes a trial license for 30 days. After that, you can obtain one of the https://www.elastic.co/subscriptions[available subscriptions] or {ref}/security-settings.html[disable Security]. The Basic license is free and includes the https://www.elastic.co/products/x-pack/monitoring[Monitoring] extension.
+
+After clicking the install button, Elasticsearch will be installed: 
+
+[[msi-installer-success]]
+image::images/msi_installer/msi_installer_success.png[]
+
+[[install-msi-command-line]]
+==== Install using the command line
+
+The `.msi` can also install Elasticsearch using the command line. The simplest installation
+using the same defaults as the GUI is achieved by first navigating to the download directory,
+then running:
+
+["source","sh",subs="attributes,callouts"]
 --------------------------------------------
-./bin/elasticsearch -Ecluster.name=my_cluster -Enode.name=node_1
+msiexec.exe /i elasticsearch-{version}.msi /qn
 --------------------------------------------
 
-NOTE: Values that contain spaces must be surrounded with quotes.  For instance `-Epath.logs="C:\My Logs\logs"`.
+By default, msiexec does not wait for the installation process to complete, since it runs in the 
+Windows subsystem. To wait on the process to finish and ensure that `%ERRORLEVEL%` is set
+accordingly, it is recommended to use `start /wait` to create a process and wait for it to exit
 
-TIP: Typically, any cluster-wide settings (like `cluster.name`) should be
-added to the `elasticsearch.yml` config file, while any node-specific settings
-such as `node.name` could be specified on the command line.
+["source","sh",subs="attributes,callouts"]
+--------------------------------------------
+start /wait msiexec.exe /i elasticsearch-{version}.msi /qn
+--------------------------------------------
 
-include::check-running.asciidoc[]
+As with any MSI installation package, a log file for the installation process can be found 
+within the `%TEMP%` directory, with a randomly generated name adhering to the format 
+`MSI<random>.LOG`. The path to a log file can be supplied using the `/l` command line argument
 
-[[windows-service]]
-==== Installing Elasticsearch as a Service on Windows
+["source","sh",subs="attributes,callouts"]
+--------------------------------------------
+start /wait msiexec.exe /i elasticsearch-{version}.msi /qn /l install.log
+--------------------------------------------
 
-Elasticsearch can be installed as a service to run in the background or start
-automatically at boot time without any user interaction. This can be achieved
-through the `service.bat` script in the `bin\` folder which allows one to
-install, remove, manage or configure the service and potentially start and
-stop the service, all from the command-line.
+Supported Windows Installer command line arguments can be viewed using
 
 ["source","sh",subs="attributes,callouts"]
---------------------------------------------------
-c:\elasticsearch-{version}{backslash}bin>service
+--------------------------------------------
+msiexec.exe /help
+--------------------------------------------
 
-Usage: service.bat install|remove|start|stop|manager [SERVICE_ID]
---------------------------------------------------
+or by consulting the https://msdn.microsoft.com/en-us/library/windows/desktop/aa367988(v=vs.85).aspx[Windows Installer SDK Command-Line Options].
 
-The script requires one parameter (the command to execute) followed by an
-optional one indicating the service id (useful when installing multiple
-Elasticsearch services).
+[[msi-command-line-options]]
+==== Command line options
 
-The commands available are:
+All settings exposed within the GUI are also available as command line arguments (referred to
+as _properties_ within Windows Installer documentation) that can be passed to msiexec:
 
 [horizontal]
-`install`:: Install Elasticsearch as a service
+`INSTALLDIR`:: 
 
-`remove`:: Remove the installed Elasticsearch service (and stop the service if started)
+  The installation directory. Defaults to `%ProgramW6432%\Elastic\Elasticsearch`
 
-`start`:: Start the Elasticsearch service (if installed)
+`DATADIRECTORY`:: 
 
-`stop`:: Stop the Elasticsearch service (if started)
+  The directory in which to store your data. 
+Defaults to `%ALLUSERSPROFILE%\Elastic\Elasticsearch\data`
 
-`manager`:: Start a GUI for managing the installed service
+`CONFIGDIRECTORY`:: 
 
-Based on the architecture of the available JDK/JRE (set through `JAVA_HOME`),
-the appropriate 64-bit(x64) or 32-bit(x86) service will be installed. This
-information is made available during install:
+  The directory in which to store your configuration. 
+  Defaults to `%ALLUSERSPROFILE%\Elastic\Elasticsearch\config`
 
-["source","sh",subs="attributes"]
---------------------------------------------------
-c:\elasticsearch-{version}{backslash}bin>service install
-Installing service      :  "elasticsearch-service-x64"
-Using JAVA_HOME (64-bit):  "c:\jvm\jdk1.8"
-The service 'elasticsearch-service-x64' has been installed.
---------------------------------------------------
+`LOGSDIRECTORY`:: 
 
-NOTE: The service installer requires that the thread stack size setting
-be configured in jvm.options *before* you install the service. On
-32-bit Windows, you should add `-Xss320k` to the jvm.options file, and
-on 64-bit Windows you should add `-Xss1m` to the jvm.options file.
+  The directory in which to store your logs. 
+  Defaults to `%ALLUSERSPROFILE%\Elastic\Elasticsearch\logs`
 
-NOTE: While a JRE can be used for the Elasticsearch service, due to its use of a client VM (as opposed to a server JVM which offers better performance for long-running applications) its usage is discouraged and a warning will be issued.
+`PLACEWRITABLELOCATIONSINSAMEPATH`:: 
 
-NOTE: Upgrading (or downgrading) JVM versions does not require the service to be reinstalled. However, upgrading across JVM types (e.g. JRE versus SE) is not supported, and does require the service to be reinstalled.
+  Whether the data, configuration and logs directories
+  should be created under the installation directory. Defaults to `false`
 
-[[windows-service-settings]]
-[float]
-=== Customizing service settings
+`INSTALLASSERVICE`:: 
 
-The Elasticsearch service can be configured prior to installation by setting the the following environment variables (either using the https://technet.microsoft.com/en-us/library/cc754250(v=ws.10).aspx[set command] from the command line, or through the `System Properties->Environment Variables` GUI).
+  Whether Elasticsearch is installed and configured as a Windows Service. 
+  Defaults to `true`
 
-[horizontal]
-`SERVICE_ID`::
+`STARTAFTERINSTALL`:: 
+
+  Whether the Windows Service is started after installation finishes. 
+  Defaults to `true`
+
+`STARTWHENWINDOWSSTARTS`:: 
+
+  Whether the Windows Service is started when Windows is started. 
+  Defaults to `true`
+
+`USELOCALSYSTEM`:: 
+
+  Whether the Windows service runs under the LocalSystem Account. 
+  Defaults to `true`
+
+`USENETWORKSERVICE`:: 
+
+  Whether the Windows service runs under the NetworkService Account. Defaults
+  to `false`
+
+`USEEXISTINGUSER`:: 
+
+  Whether the Windows service runs under a specified existing account. Defaults
+  to `false`
+
+`USER`:: 
+
+  The username for the account under which the Windows service runs. Defaults to `""`
+
+`PASSWORD`:: 
+
+  The password for the account under which the Windows service runs. Defaults to `""`
+
+`CLUSTERNAME`:: 
+
+  The name of the cluster. Defaults to `elasticsearch`
+
+`NODENAME`:: 
+
+  The name of the node. Defaults to `%COMPUTERNAME%`
+
+`MASTERNODE`:: 
+
+  Whether Elasticsearch is configured as a master node. Defaults to `true`
+
+`DATANODE`:: 
+
+  Whether Elasticsearch is configured as a data node. Defaults to `true`
+
+`INGESTNODE`:: 
+
+  Whether Elasticsearch is configured as an ingest node. Defaults to `true`
+
+`SELECTEDMEMORY`:: 
 
-  A unique identifier for the service.  Useful if installing multiple instances on the same machine.  Defaults to `elasticsearch-service-x86` (on 32-bit Windows) or `elasticsearch-service-x64` (on 64-bit Windows).
+  The amount of memory to allocate to the JVM heap for Elasticsearch. 
+  Defaults to `2048`. If the target machine has less than 4GB in total, defaults
+  to 50% of total memory.
 
-`SERVICE_USERNAME`::
+`LOCKMEMORY`:: 
 
-  The user to run as, defaults to the local system account.
+  Whether `bootstrap.memory_lock` should be used to try to lock the process
+  address space into RAM. Defaults to `false`
 
-`SERVICE_PASSWORD`::
+`UNICASTNODES`:: 
 
-  The password for the user specified in `%SERVICE_USERNAME%`.
+  A comma separated list of hosts in the form `host:port` or `host` to be used for
+  unicast discovery. Defaults to `""`
 
-`SERVICE_DISPLAY_NAME`::
+`MINIMUMMASTERNODES`:: 
 
-  The name of the service.  Defaults to `Elasticsearch <version> %SERVICE_ID%`.
+  The minimum number of master-eligible nodes that must be visible 
+  in order to form a cluster. Defaults to `""`
 
-`SERVICE_DESCRIPTION`::
+`NETWORKHOST`:: 
 
-  The description of the service.  Defaults to `Elasticsearch <version> Windows Service - https://elastic.co`.
+  The hostname or IP address to bind the node to and _publish_ (advertise) this 
+  host to other nodes in the cluster. Defaults to `""`
 
-`JAVA_HOME`::
+`HTTPPORT`:: 
 
-  The installation directory of the desired JVM to run the service under.
+  The port to use for exposing Elasticsearch APIs over HTTP. Defaults to `9200`
 
-`LOG_DIR`::
+`TRANSPORTPORT`:: 
 
-    Log directory, defaults to `%ES_HOME%\logs`.
+  The port to use for internal communication between nodes within the cluster. 
+  Defaults to `9300`
 
-`DATA_DIR`::
+`PLUGINS`:: 
 
-    Data directory, defaults to `%ES_HOME%\data`.
+  A comma separated list of the plugins to download and install as part of the installation. Defaults `""`
 
-`CONF_DIR`::
+To pass a value, simply append the property name and value using the format `<PROPERTYNAME>="<VALUE>"` to
+the installation command. For example, to use a different installation directory to the default one and to install https://www.elastic.co/products/x-pack[X-Pack]:
 
-    Configuration file directory (which needs to include `elasticsearch.yml`
-    and `log4j2.properties` files), defaults to `%ES_HOME%\conf`.
+["source","sh",subs="attributes,callouts"]
+--------------------------------------------
+start /wait msiexec.exe /i elasticsearch-{version}.msi /qn INSTALLDIR="C:\Custom Install Directory" PLUGINS="x-pack"
+--------------------------------------------
+
+Consult the https://msdn.microsoft.com/en-us/library/windows/desktop/aa367988(v=vs.85).aspx[Windows Installer SDK Command-Line Options]
+for additional rules related to values containing quotation marks. 
+
+[[msi-installer-command-line-running]]
+==== Running Elasticsearch from the command line
+
+Once installed, Elasticsearch can be started from the command line, if not installed as a service
+and configured to start when installation completes, as follows:
+
+["source","sh",subs="attributes,callouts"]
+--------------------------------------------
+.\bin\elasticsearch.exe
+--------------------------------------------
+
+The command line terminal will display output similar to the following:
+
+[[msi-installer-elasticsearch-exe]]
+image::images/msi_installer/elasticsearch_exe.png[]
+
+By default, Elasticsearch runs in the foreground, prints its logs to `STDOUT` in addition
+to the `<cluster name>.log` file within `LOGSDIRECTORY`, and can be stopped by pressing `Ctrl-C`.
+
+[[msi-installer-command-line-configuration]]
+==== Configuring Elasticsearch on the command line
+
+Elasticsearch loads its configuration from the `%ES_HOME%\config\elasticsearch.yml`
+file by default.  The format of this config file is explained in
+<<settings>>.
+
+Any settings that can be specified in the config file can also be specified on
+the command line, using the `-E` syntax as follows:
+
+["source","sh",subs="attributes,callouts"]
+--------------------------------------------
+.\bin\elasticsearch.exe -E cluster.name=my_cluster -E node.name=node_1
+--------------------------------------------
+
+NOTE: Values that contain spaces must be surrounded with quotes.  For instance `-E path.logs="C:\My Logs\logs"`.
+
+TIP: Typically, any cluster-wide settings (like `cluster.name`) should be
+added to the `elasticsearch.yml` config file, while any node-specific settings
+such as `node.name` could be specified on the command line.
+
+include::check-running.asciidoc[]
 
-`ES_JAVA_OPTS`::
+[[msi-installer-windows-service]]
+==== Installing Elasticsearch as a Service on Windows
 
-    Any additional JVM system properties you may want to apply.
+Elasticsearch can be installed as a service to run in the background or start
+automatically at boot time without any user interaction. This can be achieved upon installation
+using the following command line options
 
-`ES_START_TYPE`::
+* `INSTALLASSERVICE=true` 
+* `STARTAFTERINSTALL=true` 
+* `STARTWHENWINDOWSSTARTS=true` 
 
-    Startup mode for the service.  Can be either `auto` or `manual` (default).
+Once installed, Elasticsearch will appear within the Services control panel:
 
-`ES_STOP_TIMEOUT` ::
+[[msi-installer-installed-service]]
+image::images/msi_installer/msi_installer_installed_service.png[]
 
-  The timeout in seconds that procrun waits for service to exit gracefully.  Defaults to `0`.
+and can be stopped and restarted from within the control panel, or from the command line using:
 
-NOTE: At its core, `service.bat` relies on http://commons.apache.org/proper/commons-daemon/[Apache Commons Daemon] project
-to install the service. Environment variables set prior to the service installation are copied and will be used during the service lifecycle. This means any changes made to them after the installation will not be picked up unless the service is reinstalled.
+with Command Prompt:
 
-NOTE: On Windows, the <<heap-size,heap size>> can be configured as for
-any other Elasticsearch installation when running Elasticsearch from the
-command line, or when installing Elasticsearch as a service for the
-first time. To adjust the heap size for an already installed service,
-use the service manager: `bin\service.bat manager`.
+[source,sh]
+--------------------------------------------
+sc.exe stop Elasticsearch
+sc.exe start Elasticsearch
+--------------------------------------------
+
+with PowerShell:
+
+[source,powershell]
+--------------------------------------------
+Get-Service Elasticsearch | Stop-Service | Start-Service
+--------------------------------------------
 
-Using the Manager GUI::
+Changes can be made to jvm.options and elasticsearch.yml configuration files to configure the 
+service after installation. Most changes (like JVM settings) will require a restart of the 
+service in order to take affect.
 
-It is also possible to configure the service after it's been installed using the manager GUI (`elasticsearch-service-mgr.exe`), which offers insight into the installed service, including its status, startup type, JVM, start and stop settings amongst other things.  Simply invoking `service.bat manager` from the command-line will open up the manager window:
+[[upgrade-msi-gui]]
+==== Upgrade using the graphical user interface (GUI)
 
-image::images/service-manager-win.png["Windows Service Manager GUI",align="center"]
+The `.msi` package supports upgrading an installed version of Elasticsearch to a newer 
+version of Elasticsearch. The upgrade process through the GUI handles upgrading all 
+installed plugins as well as retaining both your data and configuration.
 
-Most changes (like JVM settings) made through the manager GUI will require a restart of the service in order to take affect.
+Downloading and clicking on a newer version of the `.msi` package will launch the GUI wizard. 
+The first step will list the read only properties from the previous installation:
 
-[[windows-layout]]
-==== Directory layout of `.zip` archive
+[[msi-installer-upgrade-notice]]
+image::images/msi_installer/msi_installer_upgrade_notice.png[]
 
-The `.zip` package is entirely self-contained. All files and directories are,
-by default, contained within `%ES_HOME%` -- the directory created when
-unpacking the archive.
+The following configuration step allows certain configuration options to be changed:
 
-This is very convenient because you don't have to create any directories to
-start using Elasticsearch, and uninstalling Elasticsearch is as easy as
-removing the `%ES_HOME%` directory.  However, it is advisable to change the
-default locations of the config directory, the data directory, and the logs
-directory so that you do not delete important data later on.
+[[msi-installer-upgrade-configuration]]
+image::images/msi_installer/msi_installer_upgrade_configuration.png[]
 
+Finally, the plugins step allows currently installed plugins to be upgraded or removed, and 
+for plugins not currently installed, to be downloaded and installed:
 
-[cols="<h,<,<m,<m",options="header",]
-|=======================================================================
-| Type | Description | Default Location | Setting
-| home
-  | Elasticsearch home directory or `%ES_HOME%`
- d| Directory created by unpacking the archive
-  |
+[[msi-installer-upgrade-plugins]]
+image::images/msi_installer/msi_installer_upgrade_plugins.png[]
 
-| bin
-  | Binary scripts including `elasticsearch` to start a node
-    and `elasticsearch-plugin` to install plugins
-  | %ES_HOME%\bin
- d|
+[[upgrade-msi-command-line]]
+==== Upgrade using the command line
 
-| conf
-  | Configuration files including `elasticsearch.yml`
-  | %ES_HOME%\config
-  | path.conf
+The `.msi` can also upgrade Elasticsearch using the command line. 
 
-| data
-  | The location of the data files of each index / shard allocated
-    on the node. Can hold multiple locations.
-  | %ES_HOME%\data
-  | path.data
+[IMPORTANT]
+===========================================
+A command line upgrade requires passing the **same** command line properties as
+used at first install time; the Windows Installer does not remember these properties.
 
-| logs
-  | Log files location.
-  | %ES_HOME%\logs
-  | path.logs
+For example, if you originally installed with the command line options `PLUGINS="x-pack"` and
+`LOCKMEMORY="true"`, then you must pass these same values when performing an
+upgrade from the command line.
+===========================================
 
-| plugins
-  | Plugin files location. Each plugin will be contained in a subdirectory.
-  | %ES_HOME%\plugins
+The simplest upgrade, assuming Elasticsearch was installed using all defaults, 
+is achieved by first navigating to the download directory, then running:
 
-| repo
-  | Shared file system repository locations. Can hold multiple locations. A file system repository can be placed in to any subdirectory of any directory specified here.
- d| Not configured
-  | path.repo
+["source","sh",subs="attributes,callouts"]
+--------------------------------------------
+start /wait msiexec.exe /i elasticsearch-{version}.msi /qn
+--------------------------------------------
+
+Similar to the install process, a path to a log file for the upgrade process can
+be passed using the `/l` command line argument
+
+["source","sh",subs="attributes,callouts"]
+--------------------------------------------
+start /wait msiexec.exe /i elasticsearch-{version}.msi /qn /l upgrade.log
+--------------------------------------------
 
-| script
-  | Location of script files.
-  | %ES_HOME%\scripts
-  | path.scripts
+[[uninstall-msi-gui]]
+==== Uninstall using Add/Remove Programs
 
-|=======================================================================
+The `.msi` package handles uninstallation of all directories and files added as part of installation.
+
+WARNING: Uninstallation will remove all directories and their contents created as part of 
+installation, including data within the data directory. If you wish to retain your data upon
+uninstallation, it is recommended that you make a copy of the data directory before uninstallation.
+
+MSI installer packages do not provide a GUI for uninstallation. An installed program can be uninstalled
+by pressing the Windows key and typing `add or remove programs` to open the system settings.
+
+Once opened, find the Elasticsearch installation within the list of installed applications, click
+and choose `Uninstall`:
+
+[[msi-installer-uninstall]]
+image::images/msi_installer/msi_installer_uninstall.png[]
+
+This will launch the uninstallation process.
+
+[[uninstall-msi-command-line]]
+==== Uninstall using the command line
+
+Uninstallation can also be performed from the command line by navigating to the directory
+containing the `.msi` package and running:
+
+["source","sh",subs="attributes,callouts"]
+--------------------------------------------
+start /wait msiexec.exe /x elasticsearch-{version}.msi /qn
+--------------------------------------------
+
+Similar to the install process, a path to a log file for the uninstallation process can
+be passed using the `/l` command line argument
+
+["source","sh",subs="attributes,callouts"]
+--------------------------------------------
+start /wait msiexec.exe /x elasticsearch-{version}.msi /qn /l uninstall.log
+--------------------------------------------
 
 include::next-steps.asciidoc[]
diff --git a/docs/reference/setup/install/zip-targz.asciidoc b/docs/reference/setup/install/zip-targz.asciidoc
index 5f85447bdcdc8..73ac476ed8755 100644
--- a/docs/reference/setup/install/zip-targz.asciidoc
+++ b/docs/reference/setup/install/zip-targz.asciidoc
@@ -10,38 +10,64 @@ link:/downloads/elasticsearch[Download Elasticsearch] page.
 Other versions can be found on the
 link:/downloads/past-releases[Past Releases page].
 
+NOTE: Elasticsearch requires Java 8 or later. Use the
+http://www.oracle.com/technetwork/java/javase/downloads/index.html[official Oracle distribution]
+or an open-source distribution such as http://openjdk.java.net[OpenJDK].
+
 [[install-zip]]
 ==== Download and install the `.zip` package
 
-The `.zip` archive for Elastisearch v{version} can be downloaded and installed as follows:
+ifeval::["{release-state}"=="unreleased"]
+
+Version {version} of Elasticsearch has not yet been released.
+
+endif::[]
+
+ifeval::["{release-state}"!="unreleased"]
+
+The `.zip` archive for Elasticsearch v{version} can be downloaded and installed as follows:
+
 
 ["source","sh",subs="attributes"]
 --------------------------------------------
-wget https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/zip/elasticsearch/{version}/elasticsearch-{version}.zip
+wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.zip
 sha1sum elasticsearch-{version}.zip <1>
 unzip elasticsearch-{version}.zip
 cd elasticsearch-{version}/ <2>
 --------------------------------------------
 <1> Compare the SHA produced by `sha1sum` or `shasum` with the
-    https://download.elastics.co/elasticsearch/release/org/elasticsearch/distribution/zip/elasticsearch/{version}/elasticsearch-{version}.zip.sha1[published SHA].
+    https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.zip.sha1[published SHA].
 <2> This directory is known as `$ES_HOME`.
 
+endif::[]
+
+
 [[install-targz]]
 ==== Download and install the `.tar.gz` package
 
-The `.tar.gz` archive for Elastisearch v{version} can be downloaded and installed as follows:
+ifeval::["{release-state}"=="unreleased"]
+
+Version {version} of Elasticsearch has not yet been released.
+
+endif::[]
+
+ifeval::["{release-state}"!="unreleased"]
+
+The `.tar.gz` archive for Elasticsearch v{version} can be downloaded and installed as follows:
 
 ["source","sh",subs="attributes"]
 --------------------------------------------
-wget https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/{version}/elasticsearch-{version}.tar.gz
+wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.tar.gz
 sha1sum elasticsearch-{version}.tar.gz <1>
 tar -xzf elasticsearch-{version}.tar.gz
 cd elasticsearch-{version}/ <2>
 --------------------------------------------
 <1> Compare the SHA produced by `sha1sum` or `shasum` with the
-    https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/{version}/elasticsearch-{version}.tar.gz.sha1[published SHA].
+    https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.tar.gz.sha1[published SHA].
 <2> This directory is known as `$ES_HOME`.
 
+endif::[]
+
 [[zip-targz-running]]
 ==== Running Elasticsearch from the command line
 
@@ -52,11 +78,19 @@ Elasticsearch can be started from the command line as follows:
 ./bin/elasticsearch
 --------------------------------------------
 
-By default, Elasticsearch runs in the foreground, prints its logs to `STDOUT`,
-and can be stopped by pressing `Ctrl-C`.
+By default, Elasticsearch runs in the foreground, prints its logs to the
+standard output (`stdout`), and can be stopped by pressing `Ctrl-C`.
+
+NOTE: All scripts packaged with Elasticsearch require a version of Bash
+that supports arrays and assume that Bash is available at `/bin/bash`.
+As such, Bash should be available at this path either directly or via a
+symbolic link.
 
 include::check-running.asciidoc[]
 
+Log printing to `stdout` can be disabled using the `-q` or `--quiet`
+option on the command line.
+
 [[setup-installation-daemon]]
 ==== Running as a daemon
 
@@ -146,6 +180,7 @@ directory so that you do not delete important data later on.
 | plugins
   | Plugin files location. Each plugin will be contained in a subdirectory.
   | $ES_HOME/plugins
+  |
 
 | repo
   | Shared file system repository locations. Can hold multiple locations. A file system repository can be placed in to any subdirectory of any directory specified here.
diff --git a/docs/reference/setup/install/zip-windows.asciidoc b/docs/reference/setup/install/zip-windows.asciidoc
new file mode 100644
index 0000000000000..cea777052b93f
--- /dev/null
+++ b/docs/reference/setup/install/zip-windows.asciidoc
@@ -0,0 +1,269 @@
+[[zip-windows]]
+=== Install Elasticsearch with `.zip` on Windows
+
+Elasticsearch can be installed on Windows using the `.zip` package.  This
+comes with a `elasticsearch-service.bat` command which will setup Elasticsearch to run as a
+service.
+
+TIP: Elasticsearch has historically been installed on Windows using the `.zip` archive.
+An <<windows, MSI installer package>> is available that provides the easiest getting started
+experience for Windows. You can continue using the `.zip` approach if you prefer.
+
+The latest stable version of Elasticsearch can be found on the
+link:/downloads/elasticsearch[Download Elasticsearch] page.
+Other versions can be found on the
+link:/downloads/past-releases[Past Releases page].
+
+NOTE: Elasticsearch requires Java 8 or later. Use the
+http://www.oracle.com/technetwork/java/javase/downloads/index.html[official Oracle distribution]
+or an open-source distribution such as http://openjdk.java.net[OpenJDK].
+
+[[install-windows]]
+==== Download and install the `.zip` package
+
+ifeval::["{release-state}"=="unreleased"]
+
+Version {version} of Elasticsearch has not yet been released.
+
+endif::[]
+
+ifeval::["{release-state}"!="unreleased"]
+
+Download the `.zip` archive for Elasticsearch v{version} from: https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.zip
+
+Unzip it with your favourite unzip tool.  This will create a folder called
++elasticsearch-{version}+, which we will refer to as `%ES_HOME%`. In a terminal
+window, `cd` to the `%ES_HOME%` directory, for instance:
+
+["source","sh",subs="attributes"]
+----------------------------
+cd c:\elasticsearch-{version}
+----------------------------
+
+endif::[]
+
+[[windows-running]]
+==== Running Elasticsearch from the command line
+
+Elasticsearch can be started from the command line as follows:
+
+[source,sh]
+--------------------------------------------
+.\bin\elasticsearch.bat
+--------------------------------------------
+
+By default, Elasticsearch runs in the foreground, prints its logs to `STDOUT`,
+and can be stopped by pressing `Ctrl-C`.
+
+[[windows-configuring]]
+==== Configuring Elasticsearch on the command line
+
+Elasticsearch loads its configuration from the `%ES_HOME%\config\elasticsearch.yml`
+file by default.  The format of this config file is explained in
+<<settings>>.
+
+Any settings that can be specified in the config file can also be specified on
+the command line, using the `-E` syntax as follows:
+
+[source,sh]
+--------------------------------------------
+.\bin\elasticsearch.bat -Ecluster.name=my_cluster -Enode.name=node_1
+--------------------------------------------
+
+NOTE: Values that contain spaces must be surrounded with quotes.  For instance `-Epath.logs="C:\My Logs\logs"`.
+
+TIP: Typically, any cluster-wide settings (like `cluster.name`) should be
+added to the `elasticsearch.yml` config file, while any node-specific settings
+such as `node.name` could be specified on the command line.
+
+include::check-running.asciidoc[]
+
+[[windows-service]]
+==== Installing Elasticsearch as a Service on Windows
+
+Elasticsearch can be installed as a service to run in the background or start
+automatically at boot time without any user interaction. This can be achieved
+through the `elasticsearch-service.bat` script in the `bin\` folder which allows one to
+install, remove, manage or configure the service and potentially start and
+stop the service, all from the command-line.
+
+["source","sh",subs="attributes,callouts"]
+--------------------------------------------------
+c:\elasticsearch-{version}{backslash}bin>elasticsearch-service.bat
+
+Usage: elasticsearch-service.bat install|remove|start|stop|manager [SERVICE_ID]
+--------------------------------------------------
+
+The script requires one parameter (the command to execute) followed by an
+optional one indicating the service id (useful when installing multiple
+Elasticsearch services).
+
+The commands available are:
+
+[horizontal]
+`install`:: Install Elasticsearch as a service
+
+`remove`:: Remove the installed Elasticsearch service (and stop the service if started)
+
+`start`:: Start the Elasticsearch service (if installed)
+
+`stop`:: Stop the Elasticsearch service (if started)
+
+`manager`:: Start a GUI for managing the installed service
+
+Based on the architecture of the available JDK/JRE (set through `JAVA_HOME`),
+the appropriate 64-bit(x64) or 32-bit(x86) service will be installed. This
+information is made available during install:
+
+["source","sh",subs="attributes"]
+--------------------------------------------------
+c:\elasticsearch-{version}{backslash}bin>elasticsearch-service.bat install
+Installing service      :  "elasticsearch-service-x64"
+Using JAVA_HOME (64-bit):  "c:\jvm\jdk1.8"
+The service 'elasticsearch-service-x64' has been installed.
+--------------------------------------------------
+
+NOTE: While a JRE can be used for the Elasticsearch service, due to its use of a client VM (as opposed to a server JVM which offers better performance for long-running applications) its usage is discouraged and a warning will be issued.
+
+NOTE: The system environment variable `JAVA_HOME` should be set to the path to
+the JDK installation that you want the service to use. If you upgrade the JDK,
+you are not required to the reinstall the service but you must set the value of
+the system environment variable `JAVA_HOME` to the path to the new JDK
+installation. However, upgrading across JVM types (e.g. JRE versus SE) is not
+supported, and does require the service to be reinstalled.
+
+[[windows-service-settings]]
+[float]
+=== Customizing service settings
+
+The Elasticsearch service can be configured prior to installation by setting the following environment variables (either using the https://technet.microsoft.com/en-us/library/cc754250(v=ws.10).aspx[set command] from the command line, or through the `System Properties->Environment Variables` GUI).
+
+[horizontal]
+`SERVICE_ID`::
+
+  A unique identifier for the service.  Useful if installing multiple instances on the same machine.  Defaults to `elasticsearch-service-x86` (on 32-bit Windows) or `elasticsearch-service-x64` (on 64-bit Windows).
+
+`SERVICE_USERNAME`::
+
+  The user to run as, defaults to the local system account.
+
+`SERVICE_PASSWORD`::
+
+  The password for the user specified in `%SERVICE_USERNAME%`.
+
+`SERVICE_DISPLAY_NAME`::
+
+  The name of the service.  Defaults to `Elasticsearch <version> %SERVICE_ID%`.
+
+`SERVICE_DESCRIPTION`::
+
+  The description of the service.  Defaults to `Elasticsearch <version> Windows Service - https://elastic.co`.
+
+`JAVA_HOME`::
+
+  The installation directory of the desired JVM to run the service under.
+
+`LOG_DIR`::
+
+    Log directory, defaults to `%ES_HOME%\logs`.
+
+`DATA_DIR`::
+
+    Data directory, defaults to `%ES_HOME%\data`.
+
+`CONF_DIR`::
+
+    Configuration file directory (which needs to include `elasticsearch.yml`
+    and `log4j2.properties` files), defaults to `%ES_HOME%\conf`.
+
+`ES_JAVA_OPTS`::
+
+    Any additional JVM system properties you may want to apply.
+
+`ES_START_TYPE`::
+
+    Startup mode for the service.  Can be either `auto` or `manual` (default).
+
+`ES_STOP_TIMEOUT` ::
+
+  The timeout in seconds that procrun waits for service to exit gracefully.  Defaults to `0`.
+
+NOTE: At its core, `elasticsearch-service.bat` relies on http://commons.apache.org/proper/commons-daemon/[Apache Commons Daemon] project
+to install the service. Environment variables set prior to the service installation are copied and will be used during the service lifecycle. This means any changes made to them after the installation will not be picked up unless the service is reinstalled.
+
+NOTE: On Windows, the <<heap-size,heap size>> can be configured as for
+any other Elasticsearch installation when running Elasticsearch from the
+command line, or when installing Elasticsearch as a service for the
+first time. To adjust the heap size for an already installed service,
+use the service manager: `bin\elasticsearch-service.bat manager`.
+
+Using the Manager GUI::
+
+It is also possible to configure the service after it's been installed using the manager GUI (`elasticsearch-service-mgr.exe`), which offers insight into the installed service, including its status, startup type, JVM, start and stop settings amongst other things.  Simply invoking `elasticsearch-service.bat manager` from the command-line will open up the manager window:
+
+image::images/service-manager-win.png["Windows Service Manager GUI",align="center"]
+
+Most changes (like JVM settings) made through the manager GUI will require a restart of the service in order to take affect.
+
+[[windows-layout]]
+==== Directory layout of `.zip` archive
+
+The `.zip` package is entirely self-contained. All files and directories are,
+by default, contained within `%ES_HOME%` -- the directory created when
+unpacking the archive.
+
+This is very convenient because you don't have to create any directories to
+start using Elasticsearch, and uninstalling Elasticsearch is as easy as
+removing the `%ES_HOME%` directory.  However, it is advisable to change the
+default locations of the config directory, the data directory, and the logs
+directory so that you do not delete important data later on.
+
+
+[cols="<h,<,<m,<m",options="header",]
+|=======================================================================
+| Type | Description | Default Location | Setting
+| home
+  | Elasticsearch home directory or `%ES_HOME%`
+ d| Directory created by unpacking the archive
+  |
+
+| bin
+  | Binary scripts including `elasticsearch` to start a node
+    and `elasticsearch-plugin` to install plugins
+  | %ES_HOME%\bin
+ d|
+
+| conf
+  | Configuration files including `elasticsearch.yml`
+  | %ES_HOME%\config
+  | path.conf
+
+| data
+  | The location of the data files of each index / shard allocated
+    on the node. Can hold multiple locations.
+  | %ES_HOME%\data
+  | path.data
+
+| logs
+  | Log files location.
+  | %ES_HOME%\logs
+  | path.logs
+
+| plugins
+  | Plugin files location. Each plugin will be contained in a subdirectory.
+  | %ES_HOME%\plugins
+  |
+
+| repo
+  | Shared file system repository locations. Can hold multiple locations. A file system repository can be placed in to any subdirectory of any directory specified here.
+ d| Not configured
+  | path.repo
+
+| script
+  | Location of script files.
+  | %ES_HOME%\scripts
+  | path.scripts
+
+|=======================================================================
+
+include::next-steps.asciidoc[]
\ No newline at end of file
diff --git a/docs/reference/setup/reindex_upgrade.asciidoc b/docs/reference/setup/reindex_upgrade.asciidoc
new file mode 100644
index 0000000000000..7f3c4bb7e5432
--- /dev/null
+++ b/docs/reference/setup/reindex_upgrade.asciidoc
@@ -0,0 +1,111 @@
+[[reindex-upgrade]]
+=== Reindex to upgrade
+
+Elasticsearch is able to use indices created in the previous major version
+only.  For instance, Elasticsearch 5.x can use indices created in
+Elasticsearch 2.x, but not those created in Elasticsearch 1.x or before.
+
+NOTE: Elasticsearch 5.x nodes will fail to start in the presence of too old indices.
+
+If you are running an Elasticsearch 2.x cluster which contains indices that
+were created before 2.x, you will either need to delete those old indices or
+to reindex them before upgrading to 5.x.  See <<reindex-upgrade-inplace>>.
+
+If you are running an Elasticsearch 1.x cluster, you have two options:
+
+* First upgrade to Elasticsearch 2.4.x, reindex the old indices, then upgrade
+  to 5.x. See <<reindex-upgrade-inplace>>.
+
+* Create a new 5.x cluster and use reindex-from-remote to import indices
+  directly from the 1.x cluster. See <<reindex-upgrade-remote>>.
+
+.Time-based indices and retention periods
+*******************************************
+
+For many use cases with time-based indices, you will not need to worry about
+carrying old 1.x indices with you to 5.x.  Data in time-based indices usually
+becomes less interesting as time passes. Old indices can be deleted once they
+fall outside of your retention period.
+
+Users in this position can continue to use 2.x until all old 1.x indices have
+been deleted, then upgrade to 5.x directly.
+
+*******************************************
+
+
+[[reindex-upgrade-inplace]]
+==== Reindex in place
+
+The easiest way to reindex old (1.x) indices in place is to use the
+https://github.com/elastic/elasticsearch-migration/tree/2.x[Elasticsearch
+Migration Plugin].  You will need to upgrade to Elasticsearch 2.3.x or 2.4.x
+first.
+
+The reindex utility provided in the migration plugin does the following:
+
+* Creates a new index with the Elasticsearch version appended to the old index
+  name (e.g. `my_index-2.4.1`), copying the mappings and settings from the old
+  index.  Refresh is disabled on the new index and the number of replicas is
+  set to `0` for efficient reindexing.
+
+* Sets the old index to read only to ensure that no data is written to the
+  old index.
+
+* Reindexes all documents from the old index to the new index.
+
+* Resets the `refresh_interval` and `number_of_replicas` to the values
+  used in the old index, and waits for the index to become green.
+
+* Adds any aliases that existed on the old index to the new index.
+
+* Deletes the old index.
+
+* Adds an alias to the new index with the old index name, e.g. alias
+  `my_index` points to index `my_index-2.4.1`.
+
+At the end of this process, you will have a new 2.x index which can be used
+by an Elasticsearch 5.x cluster.
+
+[[reindex-upgrade-remote]]
+==== Upgrading with reindex-from-remote
+
+If you are running a 1.x cluster and would like to migrate directly to 5.x
+without first migrating to 2.x, you can do so using
+<<reindex-from-remote,reindex-from-remote>>.
+
+[WARNING]
+=============================================
+
+Elasticsearch includes backwards compatibility code that allows indices from
+the previous major version to be upgraded to the current major version.  By
+moving directly from Elasticsearch 1.x to 5.x, you will have to solve any
+backwards compatibility issues yourself.
+
+=============================================
+
+You will need to set up a 5.x cluster alongside your existing 1.x cluster.
+The 5.x cluster needs to have access to the REST API of the 1.x cluster.
+
+For each 1.x index that you want to transfer to the 5.x cluster, you will need
+to:
+
+* Create a new index in 5.x with the appropriate mappings and settings.  Set
+  the `refresh_interval` to `-1` and set `number_of_replicas` to `0` for
+  faster reindexing.
+
+* Use <<reindex-from-remote,reindex-from-remote>> to pull documents from the
+  1.x index into the new 5.x index.
+
+* If you run the reindex job in the background (with `wait_for_completion` set
+  to `false`), the reindex request will return a `task_id` which can be used to
+  monitor progress of the reindex job in the <<tasks,task API>>:
+  `GET _tasks/TASK_ID`.
+
+* Once reindex has completed, set the `refresh_interval` and
+  `number_of_replicas` to the desired values (the defaults are `30s` and `1`
+  respectively).
+
+* Once the new index has finished replication, you can delete the old index.
+
+The 5.x cluster can start out small, and you can gradually move nodes from the
+1.x cluster to the 5.x cluster as you migrate indices across.
diff --git a/docs/reference/setup/rolling_upgrade.asciidoc b/docs/reference/setup/rolling_upgrade.asciidoc
index 5ba578ce768fa..838628880656c 100644
--- a/docs/reference/setup/rolling_upgrade.asciidoc
+++ b/docs/reference/setup/rolling_upgrade.asciidoc
@@ -12,7 +12,9 @@ supported for your version of Elasticsearch.
 
 To perform a rolling upgrade:
 
-==== Step 1: Disable shard allocation
+. *Disable shard allocation*
++
+--
 
 When you shut down a node, the allocation process will wait for one minute
 before starting to replicate the shards that were on that node to other nodes
@@ -30,8 +32,11 @@ PUT _cluster/settings
 --------------------------------------------------
 // CONSOLE
 // TEST[skip:indexes don't assign]
+--
 
-==== Step 2: Stop non-essential indexing and perform a synced flush (Optional)
+. *Stop non-essential indexing and perform a synced flush (Optional)*
++
+--
 
 You may happily continue indexing during the upgrade.  However, shard recovery
 will be much faster if you temporarily stop non-essential indexing and issue a
@@ -46,9 +51,11 @@ POST _flush/synced
 A synced flush request is a ``best effort'' operation. It will fail if there
 are any pending indexing operations, but it is safe to reissue the request
 multiple times if necessary.
+--
 
-[[upgrade-node]]
-==== Step 3: Stop and upgrade a single node
+. [[upgrade-node]] *Stop and upgrade a single node*
++
+--
 
 Shut down one of the nodes in the cluster *before* starting the upgrade.
 
@@ -62,7 +69,8 @@ default.
 It is a good idea to place these directories in a different location so that
 there is no chance of deleting them when upgrading Elasticsearch.  These
 custom paths can be <<path-settings,configured>> with the `path.conf`,
-`path.logs`, and `path.data` settings.
+`path.logs`, and `path.data` settings, and using `ES_JVM_OPTIONS` to specify
+the location of the `jvm.options` file.
 
 The <<deb,Debian>> and <<rpm,RPM>> packages place these directories in the
 appropriate place for each operating system.
@@ -81,20 +89,27 @@ To upgrade using a zip or compressed tarball:
     overwrite the `config` or `data` directories.
 
 *   Either copy the files in the `config` directory from your old installation
-    to your new installation, or use the `-E path.conf=` option on the command
-    line to point to an external config directory.
+    to your new installation, or set the environment variable `ES_JVM_OPTIONS`
+    to the location of the `jvm.options` file and use the `-E path.conf=`
+    option on the command line to point to an external config directory.
 
 *   Either copy the files in the `data` directory from your old installation
     to your new installation, or configure the location of the data directory
     in the `config/elasticsearch.yml` file, with the `path.data` setting.
+--
 
-==== Step 4: Upgrade any plugins
+. *Upgrade any plugins*
++
+--
 
 Elasticsearch plugins must be upgraded when upgrading a node.  Use the
 `elasticsearch-plugin` script to install the correct version of any plugins
 that you need.
+--
 
-==== Step 5: Start the upgraded node
+. *Start the upgraded node*
++
+--
 
 Start the now upgraded node and confirm that it joins the cluster by checking
 the log file or by checking the output of this request:
@@ -104,8 +119,11 @@ the log file or by checking the output of this request:
 GET _cat/nodes
 --------------------------------------------------
 // CONSOLE
+--
 
-==== Step 6: Reenable shard allocation
+. *Reenable shard allocation*
++
+--
 
 Once the node has joined the cluster, reenable shard allocation to start using
 the node:
@@ -120,8 +138,12 @@ PUT _cluster/settings
 }
 --------------------------------------------------
 // CONSOLE
+// TEST[warning:[script.max_compilations_per_minute] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version.]
+--
 
-==== Step 7: Wait for the node to recover
+. *Wait for the node to recover*
++
+--
 
 You should wait for the cluster to finish shard allocation before upgrading
 the next node.  You can check on progress with the <<cat-health,`_cat/health`>>
@@ -168,8 +190,30 @@ GET _cat/recovery
 
 If you stopped indexing, then it is safe to resume indexing as soon as
 recovery has completed.
+--
 
-==== Step 8: Repeat
+. *Repeat*
++
+--
 
 When the cluster is stable and the node has recovered, repeat the above steps
 for all remaining nodes.
+--
+
+[IMPORTANT]
+====================================================
+
+During a rolling upgrade the cluster will continue to operate as normal. Any
+new functionality will be disabled or work in a backward compatible manner
+until all nodes of the cluster have been upgraded. Once the upgrade is
+completed and all nodes are on the new version, the new functionality will
+become operational. Once that has happened, it is practically impossible to
+go back to operating in a backward compatible mode. To protect against such a
+scenario, nodes from the previous major version (e.g. 5.x) will not be allowed
+to join a cluster where all nodes are of a higher major version (e.g. 6.x).
+
+In the unlikely case of a network malfunction during upgrades, where all
+remaining old nodes are isolated from the cluster, you will have to take all
+old nodes offline and upgrade them before they can rejoin the cluster.
+
+====================================================
diff --git a/docs/reference/setup/secure-settings.asciidoc b/docs/reference/setup/secure-settings.asciidoc
new file mode 100644
index 0000000000000..e3337c86c3686
--- /dev/null
+++ b/docs/reference/setup/secure-settings.asciidoc
@@ -0,0 +1,68 @@
+[[secure-settings]]
+== Secure Settings
+
+Some settings are sensitive, and relying on filesystem permissions to protect
+their values is not sufficient. For this use case, elasticsearch provides a
+keystore, which may be password protected, and the `elasticsearch-keystore`
+tool to manage the settings in the keystore.
+
+NOTE: All commands here should be run as the user which will run elasticsearch.
+
+NOTE: Only some settings are designed to be read from the keystore. See
+documentation for each setting to see if it is supported as part of the keystore.
+
+[float]
+[[creating-keystore]]
+=== Creating the keystore
+
+To create the `elasticsearch.keystore`, use the `create` command:
+
+[source,sh]
+----------------------------------------------------------------
+bin/elasticsearch-keystore create
+----------------------------------------------------------------
+
+The file `elasticsearch.keystore` will be created alongside `elasticsearch.yml`.
+
+[float]
+[[list-settings]]
+=== Listing settings in the keystore
+
+A list of the settings in the keystore is available with the `list` command:
+
+[source,sh]
+----------------------------------------------------------------
+bin/elasticsearch-keystore list 
+----------------------------------------------------------------
+
+[float]
+[[add-string-to-keystore]]
+=== Adding string settings
+
+Sensitive string settings, like authentication credentials for cloud
+plugins, can be added using the `add` command:
+
+[source,sh]
+----------------------------------------------------------------
+bin/elasticsearch-keystore add the.setting.name.to.set
+----------------------------------------------------------------
+
+The tool will prompt for the value of the setting. To pass the value
+through stdin, use the `--stdin` flag:
+
+[source,sh]
+----------------------------------------------------------------
+cat /file/containing/setting/value | bin/elasticsearch-keystore add --stdin the.setting.name.to.set
+----------------------------------------------------------------
+
+[float]
+[[remove-settings]]
+=== Removing settings
+
+To remove a setting from the keystore, use the `remove` command:
+
+[source,sh]
+----------------------------------------------------------------
+bin/elasticsearch-keystore remove the.setting.name.to.remove
+----------------------------------------------------------------
+
diff --git a/docs/reference/setup/stopping.asciidoc b/docs/reference/setup/stopping.asciidoc
index 45a3b122a9018..4f632ec06f1dd 100644
--- a/docs/reference/setup/stopping.asciidoc
+++ b/docs/reference/setup/stopping.asciidoc
@@ -23,7 +23,7 @@ From the Elasticsearch startup logs:
 
 [source,sh]
 --------------------------------------------------
-[2016-07-07 12:26:18,908][INFO ][node                     ] [Reaper] version[5.0.0-alpha4], pid[15399], build[3f5b994/2016-06-27T16:23:46.861Z], OS[Mac OS X/10.11.5/x86_64], JVM[Oracle Corporation/Java HotSpot(TM) 64-Bit Server VM/1.8.0_92/25.92-b14]
+[2016-07-07 12:26:18,908][INFO ][node                     ] [I8hydUG] version[5.0.0-alpha4], pid[15399], build[3f5b994/2016-06-27T16:23:46.861Z], OS[Mac OS X/10.11.5/x86_64], JVM[Oracle Corporation/Java HotSpot(TM) 64-Bit Server VM/1.8.0_92/25.92-b14]
 --------------------------------------------------
 
 Or by specifying a location to write a PID file to on startup (`-p <path>`):
diff --git a/docs/reference/setup/sysconfig/configuring.asciidoc b/docs/reference/setup/sysconfig/configuring.asciidoc
index 84d0f832141c1..3df618fd77946 100644
--- a/docs/reference/setup/sysconfig/configuring.asciidoc
+++ b/docs/reference/setup/sysconfig/configuring.asciidoc
@@ -92,17 +92,27 @@ specified via systemd.
 The systemd service file (`/usr/lib/systemd/system/elasticsearch.service`)
 contains the limits that are applied by default.
 
-To override these, add a file called
-`/etc/systemd/system/elasticsearch.service.d/elasticsearch.conf`) and specify
-any changes in that file, such as:
+To override them, add a file called
+`/etc/systemd/system/elasticsearch.service.d/override.conf` (alternatively,
+you may run `sudo systemctl edit elasticsearch` which opens the file 
+automatically inside your default editor). Set any changes in this file,
+such as:
 
 [source,sh]
 ---------------------------------
+[Service]
 LimitMEMLOCK=infinity
 ---------------------------------
 
-[[es-java-opts]]
-==== Setting JVM system properties
+Once finished, run the following command to reload units:
+
+[source,sh]
+---------------------------------
+sudo systemctl daemon-reload
+---------------------------------
+
+[[jvm-options]]
+==== Setting JVM options
 
 The preferred method of setting Java Virtual Machine options (including
 system properties and JVM flags) is via the `jvm.options` configuration
diff --git a/docs/reference/setup/sysconfig/file-descriptors.asciidoc b/docs/reference/setup/sysconfig/file-descriptors.asciidoc
index 25a2214146ca0..17e7884be0d33 100644
--- a/docs/reference/setup/sysconfig/file-descriptors.asciidoc
+++ b/docs/reference/setup/sysconfig/file-descriptors.asciidoc
@@ -1,6 +1,12 @@
 [[file-descriptors]]
 === File Descriptors
 
+[NOTE]
+This is only relevant for Linux and macOS and can be safely ignored if running
+Elasticsearch on Windows. On Windows that JVM uses an
+https://msdn.microsoft.com/en-us/library/windows/desktop/aa363858(v=vs.85).aspx[API]
+limited only by available resources.
+
 Elasticsearch uses a lot of file descriptors or file handles.  Running out of
 file descriptors can be disastrous and will most probably lead to data loss.
 Make sure to increase the limit on the number of open files descriptors for
@@ -10,6 +16,9 @@ For the `.zip` and `.tar.gz` packages, set <<ulimit,`ulimit -n 65536`>> as
 root before starting Elasticsearch,   or set `nofile` to `65536` in
 <<limits.conf,`/etc/security/limits.conf`>>.
 
+On macOS, you must also pass the JVM option `-XX:-MaxFDLimit`
+to Elasticsearch in order for it to make use of the higher file descriptor limit.
+
 RPM and Debian packages already default the maximum number of file
 descriptors to 65536 and do not require further configuration.
 
diff --git a/docs/reference/setup/sysconfig/heap_size.asciidoc b/docs/reference/setup/sysconfig/heap_size.asciidoc
index 00c4553b97ffb..30023618f3fc9 100644
--- a/docs/reference/setup/sysconfig/heap_size.asciidoc
+++ b/docs/reference/setup/sysconfig/heap_size.asciidoc
@@ -1,12 +1,12 @@
 [[heap-size]]
 === Set JVM heap size via jvm.options
 
-In development mode, Elasticsearch tells the JVM to use a heap with a minimum
-size of 256 MB and a maximum size of 1 GB. When moving to production, it is
+By default, Elasticsearch tells the JVM to use a heap with a minimum
+and maximum size of 2 GB. When moving to production, it is
 important to configure heap size to ensure that Elasticsearch has enough
 heap available.
 
-Elasticsearch will assign the entire heap specified in <<sysconfig,es-java-opts>>
+Elasticsearch will assign the entire heap specified in <<jvm-options,jvm.options>>
 via the Xms (minimum heap size) and Xmx (maximum heap size) settings.
 
 The value for these setting depends on the amount of RAM available on
@@ -48,8 +48,8 @@ Here are examples of how to set the heap size via the jvm.options file:
 
 [source,txt]
 ------------------
-Xms2g <1>
-Xmx2g <2>
+-Xms2g <1>
+-Xmx2g <2>
 ------------------
 <1> Set the minimum heap size to 2g.
 <2> Set the maximum heap size to 2g.
@@ -67,7 +67,7 @@ ES_JAVA_OPTS="-Xms4000m -Xmx4000m" ./bin/elasticsearch <2>
 <2> Set the minimum and maximum heap size to 4000 MB.
 
 NOTE: Configuring the heap for the <<windows-service,Windows service>>
-is different than the above. The values initiallly populated for the
+is different than the above. The values initially populated for the
 Windows service can be configured as above but are different after the
 service has been installed. Consult the
 <<windows-service,Windows service documentation>> for additional
diff --git a/docs/reference/setup/sysconfig/swap.asciidoc b/docs/reference/setup/sysconfig/swap.asciidoc
index ce95e52fbebc7..df9ad7babaf10 100644
--- a/docs/reference/setup/sysconfig/swap.asciidoc
+++ b/docs/reference/setup/sysconfig/swap.asciidoc
@@ -2,49 +2,82 @@
 === Disable swapping
 
 Most operating systems try to use as much memory as possible for file system
-caches and eagerly swap out unused application memory. This can result in
-parts of the JVM heap being swapped out to disk.
+caches and eagerly swap out unused application memory. This can result in parts
+of the JVM heap or even its executable pages being swapped out to disk.
 
-Swapping is very bad for performance and for node stability and should be
-avoided at all costs. It can cause garbage collections to last for **minutes**
-instead of milliseconds and can cause nodes to respond slowly or even to
-disconnect from the cluster.
+Swapping is very bad for performance, for node stability, and should be avoided
+at all costs. It can cause garbage collections to last for **minutes** instead
+of milliseconds and can cause nodes to respond slowly or even to disconnect
+from the cluster. In a resilient distributed system, it's more effective to let
+the operating system kill the node.
 
-There are three approaches to disabling swapping:
+There are three approaches to disabling swapping. The preferred option is to
+completely disable swap. If this is not an option, whether or not to prefer
+minimizing swappiness versus memory locking is dependent on your environment.
+
+[[disable-swap-files]]
+==== Disable all swap files
+
+Usually Elasticsearch is the only service running on a box, and its memory usage
+is controlled by the JVM options. There should be no need to have swap enabled.
+
+On Linux systems, you can disable swap temporarily by running:
+
+[source,sh]
+--------------
+sudo swapoff -a
+--------------
+
+To disable it permanently, you will need to edit the `/etc/fstab` file and
+comment out any lines that contain the word `swap`.
+
+On Windows, the equivalent can be achieved by disabling the paging file entirely
+via `System Properties → Advanced → Performance → Advanced → Virtual memory`.
+
+[[swappiness]]
+==== Configure `swappiness`
+
+Another option available on Linux systems is to ensure that the sysctl value
+`vm.swappiness` is set to `1`. This reduces the kernel's tendency to swap and
+should not lead to swapping under normal circumstances, while still allowing the
+whole system to swap in emergency conditions.
 
 [[mlockall]]
 ==== Enable `bootstrap.memory_lock`
 
-The first option is to use
-http://opengroup.org/onlinepubs/007908799/xsh/mlockall.html[mlockall] on Linux/Unix systems, or https://msdn.microsoft.com/en-us/library/windows/desktop/aa366895%28v=vs.85%29.aspx[VirtualLock] on Windows, to
-try to lock the process address space into RAM, preventing any Elasticsearch
-memory from being swapped out.  This can be done, by adding this line
-to the `config/elasticsearch.yml` file:
+Another option is to use
+http://opengroup.org/onlinepubs/007908799/xsh/mlockall.html[mlockall] on
+Linux/Unix systems, or
+https://msdn.microsoft.com/en-us/library/windows/desktop/aa366895%28v=vs.85%29.aspx[VirtualLock]
+on Windows, to try to lock the process address space into RAM, preventing any
+Elasticsearch memory from being swapped out. This can be done, by adding this
+line to the `config/elasticsearch.yml` file:
 
 [source,yaml]
 --------------
 bootstrap.memory_lock: true
 --------------
 
-WARNING: `mlockall` might cause the JVM or shell session to exit if it tries
-to allocate more memory than is available!
+WARNING: `mlockall` might cause the JVM or shell session to exit if it tries to
+allocate more memory than is available!
 
 After starting Elasticsearch, you can see whether this setting was applied
 successfully by checking the value of `mlockall` in the output from this
 request:
 
-[source,sh]
+[source,js]
 --------------
 GET _nodes?filter_path=**.mlockall
 --------------
 // CONSOLE
 
 If you see that `mlockall` is `false`, then it means that the `mlockall`
-request has failed.  You will also see a line with more information in the
-logs with the words `Unable to lock JVM Memory`.
+request has failed.  You will also see a line with more information in the logs
+with the words `Unable to lock JVM Memory`.
 
 The most probable reason, on Linux/Unix systems, is that the user running
-Elasticsearch doesn't have permission to lock memory.  This can be granted as follows:
+Elasticsearch doesn't have permission to lock memory.  This can be granted as
+follows:
 
 `.zip` and `.tar.gz`::
 
@@ -55,23 +88,16 @@ Elasticsearch doesn't have permission to lock memory.  This can be granted as fo
 RPM and Debian::
 
   Set `MAX_LOCKED_MEMORY` to `unlimited` in the
-  <<sysconfig,system configuration file>> (or see below for systems using `systemd`).
+  <<sysconfig,system configuration file>> (or see below for systems using
+  `systemd`).
 
 Systems using `systemd`::
 
   Set `LimitMEMLOCK` to `infinity` in the <<systemd,systemd configuration>>.
 
-
 Another possible reason why `mlockall` can fail is that the temporary directory
 (usually `/tmp`) is mounted with the `noexec` option. This can be solved by
-specifying a new temp directory, by starting Elasticsearch with:
-
-[source,sh]
---------------
-./bin/elasticsearch -Djava.io.tmpdir=/path/to/temp/dir
---------------
-
-or using the `ES_JAVA_OPTS` environment variable:
+specifying a new temp directory using the `ES_JAVA_OPTS` environment variable:
 
 [source,sh]
 --------------
@@ -79,25 +105,4 @@ export ES_JAVA_OPTS="$ES_JAVA_OPTS -Djava.io.tmpdir=/path/to/temp/dir"
 ./bin/elasticsearch
 --------------
 
-[[disable-swap-files]]
-==== Disable all swap files
-
-The second option is to completely disable swap. Usually Elasticsearch
-is the only service running on a box, and its memory usage is controlled
-by the JVM options.  There should be no need to have swap enabled.
-
-On Linux systems, you can disable swap temporarily
-by running: `sudo swapoff -a`. To disable it permanently, you will need
-to edit the `/etc/fstab` file and comment out any lines that contain the
-word `swap`.
-
-On Windows, the equivalent can be achieved by disabling the paging file entirely
-via `System Properties → Advanced → Performance → Advanced → Virtual memory`.
-
-[[swappiness]]
-==== Configure `swappiness`
-
-The second option available on Linux systems is to ensure that the sysctl value
-`vm.swappiness` is set to `1`. This reduces the kernel's tendency to swap and
-should not lead to swapping under normal circumstances, while still allowing
-the whole system to swap in emergency conditions.
+or setting this JVM flag in the jvm.options configuration file.
diff --git a/docs/reference/setup/sysconfig/threads.asciidoc b/docs/reference/setup/sysconfig/threads.asciidoc
index b83e022146084..1cc65b7a03b5f 100644
--- a/docs/reference/setup/sysconfig/threads.asciidoc
+++ b/docs/reference/setup/sysconfig/threads.asciidoc
@@ -2,7 +2,7 @@
 === Number of threads
 
 Elasticsearch uses a number of thread pools for different types of operations.
-It is important that it is able to created new threads whenever needed.   Make
+It is important that it is able to create new threads whenever needed.   Make
 sure that the number of threads that the Elasticsearch user can create is at
 least 2048.
 
diff --git a/docs/reference/setup/sysconfig/virtual-memory.asciidoc b/docs/reference/setup/sysconfig/virtual-memory.asciidoc
index b9b97a6e2d6ce..ac654c9b020ca 100644
--- a/docs/reference/setup/sysconfig/virtual-memory.asciidoc
+++ b/docs/reference/setup/sysconfig/virtual-memory.asciidoc
@@ -1,8 +1,8 @@
 [[vm-max-map-count]]
 === Virtual memory
 
-Elasticsearch uses a <<default_fs,`hybrid mmapfs / niofs`>> directory by
-default to store its indices.  The default operating system limits on mmap
+Elasticsearch uses a <<default_fs,`mmapfs`>> directory by
+default for 64bit systems to store its indices.  The default operating system limits on mmap
 counts is likely to be too low, which may result in out of memory exceptions.
 
 On Linux, you can increase the limits by running the following command as
diff --git a/docs/reference/setup/upgrade.asciidoc b/docs/reference/setup/upgrade.asciidoc
index 03bce8a91fbd4..6adb9901c6de3 100644
--- a/docs/reference/setup/upgrade.asciidoc
+++ b/docs/reference/setup/upgrade.asciidoc
@@ -1,11 +1,13 @@
 [[setup-upgrade]]
-== Upgrading
+== Upgrading Elasticsearch
 
 [IMPORTANT]
 ===========================================
 Before upgrading Elasticsearch:
 
 * Consult the <<breaking-changes,breaking changes>> docs.
+* Use the https://github.com/elastic/elasticsearch-migration/[Elasticsearch Migration Plugin]
+  to detect potential issues before upgrading.
 * Test upgrades in a dev environment before upgrading your production cluster.
 * Always <<modules-snapshots,back up your data>> before upgrading.
   You **cannot roll back** to an earlier version unless you have a backup of your data.
@@ -22,14 +24,34 @@ consult this table:
 [cols="1<m,1<m,3",options="header",]
 |=======================================================================
 |Upgrade From   |Upgrade To     |Supported Upgrade Type
+|1.x            |5.x            |<<reindex-upgrade,Reindex to upgrade>>
 |2.x            |2.y            |<<rolling-upgrades,Rolling upgrade>> (where `y > x`)
-|5.x            |5.y            |<<rolling-upgrades,Rolling upgrade>> (where `y > x`)
 |2.x            |5.x            |<<restart-upgrade,Full cluster restart>>
-|5.0.0-alpha1   |5.y            |<<restart-upgrade,Full cluster restart>>
-|5.0.0-alpha2   |5.y            |<<restart-upgrade,Full cluster restart>>
-|5.0.0-beta1    |5.y            |<<restart-upgrade,Full cluster restart>>
+|5.0.0 pre GA   |5.x            |<<restart-upgrade,Full cluster restart>>
+|5.x            |5.y            |<<rolling-upgrades,Rolling upgrade>> (where `y > x`)
 |=======================================================================
 
+[IMPORTANT]
+.Indices created in Elasticsearch 1.x or before
+===============================================
+
+Elasticsearch is able to read indices created in the *previous major version
+only*.  For instance, Elasticsearch 5.x can use indices created in
+Elasticsearch 2.x, but not those created in Elasticsearch 1.x or before.
+
+This condition also applies to indices backed up with
+<<modules-snapshots,snapshot and restore>>.  If an index was originally
+created in 1.x, it cannot be restored into a 5.x cluster even if the
+snapshot was made by a 2.x cluster.
+
+Elasticsearch 5.x nodes will fail to start in the presence of too old indices.
+
+See <<reindex-upgrade>> for more information about how to upgrade old indices.
+===============================================
+
+
 include::rolling_upgrade.asciidoc[]
 
 include::cluster_restart.asciidoc[]
+
+include::reindex_upgrade.asciidoc[]
\ No newline at end of file
diff --git a/docs/reference/testing/testing-framework.asciidoc b/docs/reference/testing/testing-framework.asciidoc
index 0bf99b2fafac7..d1fe769f3c108 100644
--- a/docs/reference/testing/testing-framework.asciidoc
+++ b/docs/reference/testing/testing-framework.asciidoc
@@ -30,11 +30,10 @@ First, you need to include the testing dependency in your project, along with th
     <scope>test</scope>
   </dependency>
   <dependency>
-    <groupId>org.elasticsearch</groupId>
-    <artifactId>elasticsearch</artifactId>
+    <groupId>org.elasticsearch.test</groupId>
+    <artifactId>framework</artifactId>
     <version>${elasticsearch.version}</version>
     <scope>test</scope>
-    <type>test-jar</type>
   </dependency>
 </dependencies>
 --------------------------------------------------
@@ -172,7 +171,7 @@ As elasticsearch is using JUnit 4, using the `@Before` and `@After` annotations
 -----------------------------------------
 @Override
 protected Collection<Class<? extends Plugin>> nodePlugins() {
-  return pluginList(CustomSuggesterPlugin.class);
+  return Arrays.asList(CustomSuggesterPlugin.class);
 }
 -----------------------------------------
 
@@ -253,7 +252,7 @@ Usually, you would combine assertions and matchers in your test like this
 
 [source,java]
 ----------------------------
-SearchResponse seearchResponse = client().prepareSearch() ...;
+SearchResponse searchResponse = client().prepareSearch() ...;
 assertHitCount(searchResponse, 4);
 assertFirstHit(searchResponse, hasId("4"));
 assertSearchHits(searchResponse, "1", "2", "3", "4");
diff --git a/docs/resiliency/index.asciidoc b/docs/resiliency/index.asciidoc
deleted file mode 100644
index 258b880f88f99..0000000000000
--- a/docs/resiliency/index.asciidoc
+++ /dev/null
@@ -1,561 +0,0 @@
-= Elasticsearch Resiliency Status
-
-:JIRA: https://issues.apache.org/jira/browse/LUCENE-
-:GIT:  https://github.com/elastic/elasticsearch/issues/
-
-== Overview
-
-The team at Elasticsearch is committed to continuously improving both
-Elasticsearch and Apache Lucene to protect your data.  As with any distributed
-system, Elasticsearch is complex and has many moving parts, each of which can
-encounter edge cases that require proper handling.  Our resiliency project is
-an ongoing effort to find and fix these edge cases. If you want to keep up
-with all this project on GitHub, see our issues list under the tag
-https://github.com/elastic/elasticsearch/issues?q=label%3Aresiliency[resiliency].
-
-While GitHub is great for sharing our work, it can be difficult to get an
-overview of the current state of affairs and the previous work that has been
-done from an issues list. This page provides an overview of all the
-resiliency-related issues that we are aware of, improvements that have already
-been made and current in-progress work. We’ve also listed some historical
-improvements throughout this page to provide the full context.
-
-If you’re interested in more on how we approach ensuring resiliency in
-Elasticsearch, you may be interested in Igor Motov’s recent talk
-http://www.elastic.co/videos/improving-elasticsearch-resiliency[Improving Elasticsearch Resiliency].
-
-You may also be interested in our blog post
-http://www.elastic.co/blog/resiliency-elasticsearch[Resiliency in Elasticsearch],
-which details our thought processes when addressing resiliency in both
-Elasticsearch and the work our developers do upstream in Apache Lucene.
-
-== Data Store Recommendations
-
-Some customers use Elasticsearch as a primary datastore, some set-up
-comprehensive back-up solutions using features such as our Snapshot and
-Restore, while others use Elasticsearch in conjunction with a data storage
-system like Hadoop or even flat files. Elasticsearch can be used for so many
-different use cases which is why we have created this page to make sure you
-are fully informed when you are architecting your system.
-
-== Work in Progress
-
-[float]
-=== Known Unknowns (STATUS: ONGOING)
-
-We consider this topic to be the most important in our quest for
-resiliency. We put a tremendous amount of effort into testing
-Elasticsearch to simulate failures and randomize configuration to
-produce extreme conditions. In addition, our users are an important
-source of information on unexpected edge cases and your bug reports
-help us make fixes that ensure that our system continues to be
-resilient.
-
-If you encounter an issue, https://github.com/elastic/elasticsearch/issues[please report it]!
-
-We are committed to tracking down and fixing all the issues that are posted.
-
-[float]
-==== Jepsen Tests
-
-The Jepsen platform is specifically designed to test distributed systems. It is not a single test and is regularly adapted
-to create new scenarios. We have currently ported all published Jepsen scenarios that deal with loss of acknowledged writes to our testing
-framework. As the Jepsen tests evolve, we will continue porting new scenarios that are not covered yet. We are committed to investigating
-all new scenarios and will report issues that we find on this page and in our GitHub repository.
-
-[float]
-=== Better request retry mechanism when nodes are disconnected (STATUS: ONGOING)
-
-If the node holding a primary shard is disconnected for whatever reason, the
-coordinating node retries the request on the same or a new primary shard.  In
-certain rare conditions, where the node disconnects and immediately
-reconnects, it is possible that the original request has already been
-successfully applied but has not been reported, resulting in duplicate
-requests. This is particularly true when retrying bulk requests, where some
-actions may  have completed and some may not have.
-
-An optimization which disabled the existence check for documents indexed with
-auto-generated IDs could result in the creation of duplicate documents. This
-optimization has been removed. {GIT}9468[#9468] (STATUS: DONE, v1.5.0)
-
-Further issues remain with the retry mechanism:
-
-* Unversioned index requests could increment the `_version` twice,
-  obscuring a `created` status.
-* Versioned index requests could return a conflict exception, even
-  though they were applied correctly.
-* Update requests could be applied twice.
-
-See {GIT}9967[#9967]. (STATUS: ONGOING)
-
-[float]
-=== OOM resiliency (STATUS: ONGOING)
-
-The family of circuit breakers has greatly reduced the occurrence of OOM
-exceptions, but it is still possible to cause a node to run out of heap
-space.  The following issues have been identified:
-
-* Set a hard limit on `from`/`size` parameters {GIT}9311[#9311]. (STATUS: DONE, v2.1.0)
-* Prevent combinatorial explosion in aggregations from causing OOM {GIT}8081[#8081]. (STATUS: ONGOING)
-* Add the byte size of each hit to the request circuit breaker {GIT}9310[#9310]. (STATUS: ONGOING)
-* Limit the size of individual requests and also add a circuit breaker for the total memory used by in-flight request objects {GIT}16011[#16011]. (STATUS: DONE, v5.0.0)
-
-Other safeguards are tracked in the meta-issue {GIT}11511[#11511].
-
-[float]
-=== The _version field may not uniquely identify document content during a network partition (STATUS: ONGOING)
-
-When a primary has been partitioned away from the cluster there is a short period of time until it detects this. During that time it will continue
-indexing writes locally, thereby updating document versions. When it tries to replicate the operation, however, it will discover that it is
-partitioned away. It won't acknowledge the write and will wait until the partition is resolved to negotiate with the master on how to proceed.
-The master will decide to either fail any replicas which failed to index the operations on the primary or tell the primary that it has to
-step down because a new primary has been chosen in the meantime. Since the old primary has already written documents, clients may already have read from
-the old primary before it shuts itself down. The version numbers of these reads may not be unique if the new primary has already accepted
-writes for the same document (see {GIT}19269[#19269]).
-
-We are currently implementing Sequence numbers {GIT}10708[#10708] which better track primary changes. Sequence numbers thus provide a basis
-for uniquely identifying writes even in the presence of network partitions and will replace `_version` in operations that require this.
-
-[float]
-=== Relocating shards omitted by reporting infrastructure (STATUS: ONGOING)
-
-Indices stats and indices segments requests reach out to all nodes that have shards of that index. Shards that have relocated from a node
-while the stats request arrives will make that part of the request fail and are just ignored in the overall stats result. {GIT}13719[#13719]
-
-[float]
-=== Documentation of guarantees and handling of failures (STATUS: ONGOING)
-
-This status page is a start, but we can do a better job of explicitly documenting the processes at work in Elasticsearch and what happens
-in the case of each type of failure. The plan is to have a test case that validates each behavior under simulated conditions. Every test
- will document the expected results, the associated test code, and an explicit PASS or FAIL status for each simulated case.
-
-[float]
-=== Run Jepsen (STATUS: ONGOING)
-
-We have ported the known scenarios in the Jepsen blogs that check loss of acknowledged writes to our testing infrastructure.
-The new tests are run continuously in our testing farm and are passing. We are also working on running Jepsen independently to verify
-that no failures are found.
-
-
-== Unreleased
-
-[float]
-=== Port Jepsen tests dealing with loss of acknowledged writes to our testing framework (STATUS: UNRELEASED, V5.0.0)
-
-We have increased our test coverage to include scenarios tested by Jepsen that demonstrate loss of acknowledged writes, as described in
-the Elasticsearch related blogs. We make heavy use of randomization to expand on the scenarios that can be tested and to introduce
-new error conditions.
-You can follow the work on the master branch of the
-https://github.com/elastic/elasticsearch/blob/master/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java[`DiscoveryWithServiceDisruptionsIT` class],
-where the `testAckedIndexing` test was specifically added to check that we don't lose acknowledged writes in various failure scenarios.
-
-
-[float]
-=== Loss of documents during network partition (STATUS: UNRELEASED, v5.0.0)
-
-If a network partition separates a node from the master, there is some window of time before the node detects it. The length of the window is dependent on the type of the partition. This window is extremely small if a socket is broken. More adversarial partitions, for example, silently dropping requests without breaking the socket can take longer (up to 3x30s using current defaults).
-
-If the node hosts a primary shard at the moment of partition, and ends up being isolated from the cluster (which could have resulted in {GIT}2488[split-brain] before), some documents that are being indexed into the primary may be lost if they fail to reach one of the allocated replicas (due to the partition) and that replica is later promoted to primary by the master ({GIT}7572[#7572]).
-To prevent this situation, the primary needs to wait for the master to acknowledge replica shard failures before acknowledging the write to the client. {GIT}14252[#14252]
-
-[float]
-=== Safe primary relocations (STATUS: UNRELEASED, v5.0.0)
-
-When primary relocation completes, a cluster state is propagated that deactivates the old primary and marks the new primary as active. As
-cluster state changes are not applied synchronously on all nodes, there can be a time interval where the relocation target has processed the
-cluster state and believes to be the active primary and the relocation source has not yet processed the cluster state update and still
-believes itself to be the active primary. This means that an index request that gets routed to the new primary does not get replicated to
-the old primary (as it has been deactivated from point of view of the new primary). If a subsequent read request gets routed to the old
-primary, it cannot see the indexed document. {GIT}15900[#15900]
-
-In the reverse situation where a cluster state update that completes primary relocation is first applied on the relocation source and then
-on the relocation target, each of the nodes believes the other to be the active primary. This leads to the issue of indexing requests
-chasing the primary being quickly sent back and forth between the nodes, potentially making them both go OOM. {GIT}12573[#12573]
-
-[float]
-=== Do not allow stale shards to automatically be promoted to primary (STATUS: UNRELEASED, v5.0.0)
-
-In some scenarios, after the loss of all valid copies, a stale replica shard can be automatically assigned as a primary, preferring old data
-to no data at all ({GIT}14671[#14671]). This can lead to a loss of acknowledged writes if the valid copies are not lost but are rather
-temporarily unavailable. Allocation IDs ({GIT}14739[#14739]) solve this issue by tracking non-stale shard copies in the cluster and using
-this tracking information to allocate primary shards. When all shard copies are lost or only stale ones available, Elasticsearch will wait
-for one of the good shard copies to reappear. In case where all good copies are lost, a manual override command can be used to allocate a
-stale shard copy.
-
-[float]
-=== Make index creation resilient to index closing and full cluster crashes (STATUS: UNRELEASED, v5.0.0)
-
-Recovering an index requires a quorum (with an exception for 2) of shard copies to be available to allocate a primary. This means that
-a primary cannot be assigned if the cluster dies before enough shards have been allocated ({GIT}9126[#9126]). The same happens if an index
-is closed before enough shard copies were started, making it impossible to reopen the index ({GIT}15281[#15281]).
-Allocation IDs ({GIT}14739[#14739]) solve this issue by tracking allocated shard copies in the cluster. This makes it possible to safely
-recover an index in the presence of a single shard copy. Allocation IDs can also distinguish the situation where an index has been created
-but none of the shards have been started. If such an index was inadvertently closed before at least one shard could be started, a fresh
-shard will be allocated upon reopening the index.
-
-
-[float]
-=== Use two phase commit for Cluster State publishing (STATUS: UNRELEASED, v5.0.0)
-
-A master node in Elasticsearch continuously https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-discovery-zen.html#fault-detection[monitors the cluster nodes]
-and removes any node from the cluster that doesn't respond to its pings in a timely
-fashion. If the master is left with fewer nodes than the `discovery.zen.minimum_master_nodes`
-settings, it will step down and a new master election will start.
-
-When a network partition causes a master node to lose many followers, there is a short window
-in time until the node loss is detected and the master steps down. During that window, the
-master may erroneously accept and acknowledge cluster state changes. To avoid this, we introduce
-a new phase to cluster state publishing where the proposed cluster state is sent to all nodes
-but is not yet committed. Only once enough nodes (`discovery.zen.minimum_master_nodes`) actively acknowledge
-the change, it is committed and commit messages are sent to the nodes. See {GIT}13062[#13062].
-
-== Completed
-
-[float]
-=== Wait on incoming joins before electing local node as master (STATUS: DONE, v2.0.0)
-
-During master election each node pings in order to discover other nodes and validate the liveness of existing
-nodes. Based on this information the node either discovers an existing master or, if enough nodes are found
-(see https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-discovery-zen.html#master-election[`discovery.zen.minimum_master_nodes`]) a new master will be elected. Currently, the node that is
-elected as master will update the cluster state to indicate the result of the election. Other nodes will submit
-a join request to the newly elected master node. Instead of immediately processing the election result, the elected master
-node should wait for the incoming joins from other nodes, thus validating that the result of the election is properly applied. As soon as enough
-nodes have sent their joins request (based on the `minimum_master_nodes` settings) the cluster state is updated.
-{GIT}12161[#12161]
-
-[float]
-=== Mapping changes should be applied synchronously (STATUS: DONE, v2.0.0)
-
-When introducing new fields using dynamic mapping, it is possible that the same
-field can be added to different shards with different data types.  Each shard
-will operate with its local data type but, if the shard is relocated, the
-data type from the cluster state will be applied to the new shard, which
-can result in a corrupt shard.  To prevent this, new fields should not
-be added to a shard's mapping until confirmed by the master.
-{GIT}8688[#8688] (STATUS: DONE)
-
-[float]
-=== Add per-segment and per-commit ID to help replication (STATUS: DONE, v2.0.0)
-
-{JIRA}5895[LUCENE-5895] adds a unique ID for each segment and each commit point. File-based replication (as performed by snapshot/restore) can use this ID to know whether the segment/commit on the source and destination machines are the same.  Fixed in Lucene 5.0.
-
-[float]
-=== Write index metadata on data nodes where shards allocated (STATUS: DONE, v2.0.0)
-
-Today, index metadata is written only on nodes that are master-eligible, not on
-data-only nodes.  This is not a problem when running with multiple master nodes,
-as recommended, as the loss of all but one master node is still recoverable.
-However, users running with a single master node are at risk of losing
-their index metadata if the master fails.  Instead, this metadata should
-also be written on any node where a shard is allocated. {GIT}8823[#8823], {GIT}9952[#9952]
-
-[float]
-=== Better file distribution with multiple data paths (STATUS: DONE, v2.0.0)
-
-Today, a node configured with multiple data paths distributes writes across
-all paths by writing one file to each path in turn.  This can mean that the
-failure of a single disk corrupts many shards at once.  Instead, by allocating
-an entire shard to a single data path, the extent of the damage can be limited
-to just the shards on that disk. {GIT}9498[#9498]
-
-[float]
-=== Lucene checksums phase 3 (STATUS: DONE, v2.0.0)
-
-Almost all files in Elasticsearch now have checksums which are validated before use.  A few changes remain:
-
-* {GIT}7586[#7586] adds checksums for cluster and index state files. (STATUS: DONE, Fixed in v1.5.0)
-* {GIT}9183[#9183] supports validating the checksums on all files when starting a node. (STATUS: DONE, Fixed in v2.0.0)
-* {JIRA}5894[LUCENE-5894] lays the groundwork for extending more efficient checksum validation to all files during optimized bulk merges. (STATUS: DONE, Fixed in v2.0.0)
-* {GIT}8403[#8403] to add validation of checksums on Lucene `segments_N` files. (STATUS: DONE, v2.0.0)
-
-[float]
-=== Report shard-level statuses on write operations (STATUS: DONE, v2.0.0)
-
-Make write calls return the number of total/successful/missing shards in the same way that we do in search, which ensures transparency in the consistency of write operations. {GIT}7994[#7994]. (STATUS: DONE, v2.0.0)
-
-[float]
-=== Take filter cache key size into account (STATUS: DONE, v2.0.0)
-
-Commonly used filters are cached in Elasticsearch. That cache is limited in size
-(10% of node's memory by default) and is being evicted based on a least recently
-used policy. The amount of memory used by the cache depends on two primary
-components - the values it stores and the keys associated with them. Calculating
-the memory footprint of the values is easy enough but the keys accounting is
-trickier to achieve as they are, by default, raw Lucene objects. This is largely
-not a problem as the keys are dominated by the values. However, recent
-optimizations in Lucene have changed the balance causing the filter cache to
-grow beyond it's size.
-
-As a temporary solution, we introduced a minimum weight of 1k for each cache entry.
-This puts an effective limit on the number of entries in the cache. See {GIT}8304[#8304] (STATUS: DONE, fixed in v1.4.0)
-
-The issue has been completely solved by the move to Lucene's query cache. See {GIT}10897[#10897]
-
-[float]
-=== Ensure shard state ID is incremental (STATUS: DONE, v1.5.1)
-
-It is possible in very extreme cases during a complicated full cluster restart,
-that the current shard state ID can be reset or even go backwards.
-Elasticsearch now ensures that the state ID always moves
-forwards, and throws an exception when a legacy ID is higher than the
-current ID.  See {GIT}10316[#10316] (STATUS: DONE, v1.5.1)
-
-[float]
-=== Verification of index UUIDs (STATUS: DONE, v1.5.0)
-
-When deleting and recreating indices rapidly, it is possible that cluster state
-updates can arrive out of sync and old states can be merged incorrectly.  Instead,
-Elasticsearch now checks the index UUID to ensure that cluster state updates
-refer to the same index version that is present on the local node.
-See {GIT}9541[#9541] and {GIT}10200[#10200] (STATUS: DONE, Fixed in v1.5.0)
-
-[float]
-=== Disable recovery from known buggy versions (STATUS: DONE, v1.5.0)
-
-Corruptions have been known to occur when doing a rolling restart from older, buggy versions.
-Now, shards from versions before v1.4.0 are copied over in full and recovery from versions
-before v1.3.2 are disabled entirely. See {GIT}9925[#9925] (STATUS: DONE, Fixed in v1.5.0)
-
-
-[float]
-=== Upgrade 3.x segments metadata on engine startup (STATUS: DONE, v1.5.0)
-
-Upgrading the metadata of old 3.x segments on node upgrade can be error prone
-and can result in corruption when merges are being run concurrently. Instead,
-Elasticsearch will now upgrade the metadata of 3.x segments before the engine
-starts.  See {GIT}9899[#9899] (STATUS; DONE, fixed in v1.5.0)
-
-[float]
-=== Prevent setting minimum_master_nodes to more than the current node count (STATUS: DONE, v1.5.0)
-
-Setting `zen.discovery.minimum_master_nodes` to a value higher than the current node count
-effectively leaves the cluster without a master and unable to process requests.  The only
-way to fix this is to add more master-eligible nodes.  {GIT}8321[#8321] adds a mechanism
-to validate settings before applying them, and {GIT}9051[#9051] extends this validation
-support to settings applied during a cluster restore. (STATUS: DONE, Fixed in v1.5.0)
-
-[float]
-=== Simplify and harden shard recovery and allocation (STATUS: DONE, v1.5.0)
-
-Randomized testing combined with chaotic failures has revealed corner cases
-where the recovery and allocation of shards in a concurrent manner can result
-in shard corruption.  There is an ongoing effort to reduce the complexity of
-these operations in order to make them more deterministic.  These include:
-
-* Introduce shard level locks to prevent concurrent shard modifications {GIT}8436[#8436]. (STATUS: DONE, Fixed in v1.5.0)
-* Delete shard contents under a lock {GIT}9083[#9083]. (STATUS: DONE, Fixed in v1.5.0)
-* Delete shard under a lock {GIT}8579[#8579]. (STATUS: DONE, Fixed in v1.5.0)
-* Refactor RecoveryTarget state management {GIT}8092[#8092]. (STATUS: DONE, Fixed in v1.5.0)
-* Cancelling a recovery may leave temporary files behind {GIT}7893[#7893]. (STATUS: DONE, Fixed in v1.5.0)
-* Quick cluster state processing can result in both shard copies being deleted {GIT}9503[#9503]. (STATUS: DONE, Fixed in v1.5.0)
-* Rapid creation and deletion of an index can cause reuse of old index metadata {GIT}9489[#9489]. (STATUS: DONE, Fixed in v1.5.0)
-* Flush immediately after the last concurrent recovery finishes to clear out the translog before a new recovery starts {GIT}9439[#9439]. (STATUS: DONE, Fixed in v1.5.0)
-
-[float]
-=== Prevent use of known-bad Java versions (STATUS: DONE, v1.5.0)
-
-Certain versions of the JVM are known to have bugs which can cause index corruption.  {GIT}7580[#7580] prevents Elasticsearch startup if known bad versions are in use.
-
-[float]
-=== Make recovery be more resilient to partial network partitions (STATUS: DONE, v1.5.0)
-
-When a node is experience network issues, the master detects it and removes the node from the cluster. That causes all ongoing recoveries from and to that node to be stopped and a new location is found for the relevant shards. However, in the of case partial network partition, where there are connectivity issues between the source and target nodes of a recovery but not between those nodes and the current master things may go wrong. While the nodes successfully restore the connection, the on going recoveries may have encountered issues. In {GIT}8720[#8720], we added test simulations for these and solved several issues that were flagged by them.
-
-[float]
-=== Improving Zen Discovery (STATUS: DONE, v1.4.0.Beta1)
-
-Recovery from failure is a complicated process, especially in an asynchronous distributed system like Elasticsearch. With several processes happening in parallel, it is important to ensure that recovery proceeds swiftly and safely. While fixing the {GIT}2488[split-brain issue] we have been hunting down corner cases that were not handled optimally, adding tests to demonstrate the issues, and working on fixes:
-
-* Faster & better detection of master & node failures, including not trying to reconnect upon disconnect, fail on disconnect error on ping, verify cluster names in pings. Previously, Elasticsearch had to wait a bit for the node to complete the process required to join the cluster. Recent changes guarantee that a node has fully joined the cluster before we start the fault detection process. Therefore we can do an immediate check causing faster detection of errors and validation of cluster state after a minimum master node breach. {GIT}6706[#6706], {GIT}7399[#7399] (STATUS: DONE, v1.4.0.Beta1)
-* Broaden Unicast pinging when master fails: When a node loses it’s current master it will start pinging to find a new one. Previously, when using unicast based pinging, the node would ping a set of predefined nodes asking them whether the master had really disappeared or whether there was a network hiccup. Now, we ping all nodes in the cluster to increase coverage. In the case that all unicast hosts are disconnected from the current master during a network failure, this improvement is essential to allow the cluster to reform once the partition is healed. {GIT}7336[#7336] (STATUS: DONE, v1.4.0.Beta1)
-* After joining a cluster, validate that the join was successful and that the master has been set in the local cluster state. {GIT}6969[#6969]. (STATUS: DONE, v1.4.0.Beta1)
-* Write additional tests that use the test infrastructure to verify proper behavior during network disconnections and garbage collections. {GIT}7082[#7082] (STATUS: DONE, v1.4.0.Beta1)
-
-[float]
-=== Lucene checksums phase 2 (STATUS:DONE, v1.4.0.Beta1)
-
-When Lucene opens a segment for reading, it validates the checksum on the smaller segment files -- those which it reads entirely into memory -- but not the large files like term frequencies and positions, as this would be very expensive. During merges, term vectors and stored fields are validated, as long the segments being merged come from the same version of Lucene. Checksumming for term vectors and stored fields is important because merging consists of performing optimized byte copies. Term frequencies, term positions, payloads, doc values, and norms are currently not checked during merges, although Lucene provides the option to do so.  These files are less prone to silent corruption as they are actively decoded during merge, and so are more likely to throw exceptions if there is any corruption.
-
-The following changes have been made:
-
-* {GIT}7360[#7360] validates checksums on all segment files during merges. (STATUS: DONE, fixed in v1.4.0.Beta1)
-* {JIRA}5842[LUCENE-5842] validates the structure of the checksum footer of the postings lists, doc values, stored fields and term vectors when opening a new segment, to ensure that these files have not been truncated. (STATUS: DONE, Fixed in Lucene 4.10 and v1.4.0.Beta1)
-* {GIT}8407[#8407] validates Lucene checksums for legacy files. (STATUS: DONE; Fixed in v1.3.6)
-
-[float]
-=== Don't allow unsupported codecs (STATUS: DONE, v1.4.0.Beta1)
-
-Lucene 4 added a number of alternative codecs for experimentation purposes, and Elasticsearch exposed the ability to change codecs.  Since then, Lucene has settled on the best choice of codec and provides backwards compatibility only for the default codec.  {GIT}7566[#7566] removes the ability to set alternate codecs.
-
-[float]
-=== Use checksums to identify entire segments (STATUS: DONE, v1.4.0.Beta1)
-
-A hash collision makes it possible for two different files to have the same length and the same checksum. Instead, a segment's identity should rely on checksums from all of the files in a single segment, which greatly reduces the chance of a collision. This change has been merged ({GIT}7351[#7351]).
-
-[float]
-=== Fix ''Split Brain can occur even with minimum_master_nodes'' (STATUS: DONE, v1.4.0.Beta1)
-
-Even when minimum master nodes is set, split brain can still occur under certain conditions, e.g. disconnection between master eligible nodes, which can lead to data loss. The scenario is described in detail in {GIT}2488[issue 2488]:
-
-* Introduce a new testing infrastructure to simulate different types of node disconnections, including loss of network connection, lost messages, message delays, etc. See {GIT}5631[MockTransportService] support and {GIT}6505[service disruption] for more details. (STATUS: DONE, v1.4.0.Beta1).
-* Added tests that simulated the bug described in issue 2488. You can take a look at the https://github.com/elastic/elasticsearch/commit/7bf3ffe73c44f1208d1f7a78b0629eb48836e726[original commit] of a reproduction on master. (STATUS: DONE, v1.2.0)
-* The bug described in {GIT}2488[issue 2488] is caused by an issue in our zen discovery gossip protocol. This specific issue has been fixed, and work has been done to make the algorithm more resilient. (STATUS: DONE, v1.4.0.Beta1)
-
-[float]
-=== Translog Entry Checksum (STATUS: DONE, v1.4.0.Beta1)
-
-Each translog entry in Elasticsearch should have its own checksum, and potentially additional information, so that we can properly detect corrupted translog entries and act accordingly. You can find more detail in issue {GIT}6554[#6554].
-
-To start, we will begin by adding checksums to the translog to detect corrupt entries. Once this work has been completed, we will add translog entry markers so that corrupt entries can be skipped in the translog if/when desired.
-
-[float]
-=== Request-Level Memory Circuit Breaker (STATUS: DONE, v1.4.0.Beta1)
-
-We are in the process of introducing multiple circuit breakers in Elasticsearch, which can “borrow” space from each other in the event that one runs out of memory. This architecture will allow limits for certain parts of memory, but still allow flexibility in the event that another reserve like field data is not being used. This change includes adding a breaker for the BigArrays internal object used for some aggregations. See issue {GIT}6739[#6739] for more details.
-
-[float]
-=== Doc Values (STATUS: DONE, v1.4.0.Beta1)
-
-Fielddata is one of the largest consumers of heap memory, and thus one of the primary reasons for running out of memory and causing node instability. Elasticsearch has had the “doc values” option for a while, which allows you to build these structures at index time so that they live on disk instead of in memory. Up until recently, doc values were significantly slower than in-memory fielddata.
-
-By benchmarking and profiling both Lucene and Elasticsearch, we identified the bottlenecks and have made a series of improvements to improve the performance of doc values. They are now almost as fast as the in-memory option.
-
-See {GIT}6967[#6967], {GIT}6908[#6908], {GIT}4548[#4548], {GIT}3829[#3829], {GIT}4518[#4518], {GIT}5669[#5669], {JIRA}5748[LUCENE-5748], {JIRA}5703[LUCENE-5703], {JIRA}5750[LUCENE-5750], {JIRA}5721[LUCENE-5721], {JIRA}5799[LUCENE-5799].
-
-[float]
-=== Index corruption when upgrading Lucene 3.x indices (STATUS: DONE, v1.4.0.Beta1)
-
-Upgrading indices create with Lucene 3.x (Elasticsearch v0.20 and before) to Lucene 4.7 - 4.9 (Elasticsearch v1.1.0 to v1.3.x), could result in index corruption. {JIRA}5907[LUCENE-5907] fixes this issue in Lucene 4.10.
-
-[float]
-=== Improve error handling when deleting files (STATUS: DONE, v1.4.0.Beta1)
-
-Lucene uses reference counting to prevent files that are still in use from being deleted.  Lucene testing discovered a bug ({JIRA}5919[LUCENE-5919]) when decrementing the ref count on a batch of files. If deleting some of the files resulted in an exception (e.g. due to interference from a virus scanner), the files that had had their ref counts decremented successfully could later have their ref counts deleted again, incorrectly, resulting in files being physically deleted before their time. This is fixed in Lucene 4.10.
-
-[float]
-=== Using Lucene Checksums to verify shards during snapshot/restore (STATUS:DONE, v1.3.3)
-
-The snapshot process should verify checksums for each file that is being snapshotted to make sure that created snapshot doesn’t contain corrupted files. If a corrupted file is detected, the snapshot should fail with an error. In order to implement this feature we need to have correct and verifiable checksums stored with segment files, which is only possible for files that were written by the officially supported append-only codecs. See {GIT}7159[#7159].
-
-[float]
-=== Rare compression corruption during shard recovery (STATUS: DONE, v1.3.2)
-
-During recovery, the primary shard is copied over the network to become a new replica shard. In rare cases, it was possible for a hash collision to trigger a bug in the compression library that is used to produce corruption in the replica shard. This bug was exposed by the change to validate checksums during recovery. We tracked down the bug in the in compression library and submitted a patch, which was accepted and merged by the upstream project. See {GIT}7210[#7210].
-
-[float]
-=== Safer recovery of replica shards (STATUS: DONE, v1.3.0)
-
-If a primary shard fails or is closed while a replica is using it for recovery, we need to ensure that the replica is properly failed as well, and allow recovery to start from the new primary. Also check that an active copy of a shard is available on another node before physically removing an inactive shard from disk. {GIT}6825[#6825], {GIT}6645[#6645], {GIT}6995[#6995].
-
-[float]
-=== Using Lucene Checksums to verify shards during recovery (STATUS: DONE, v1.3.0)
-
-Elasticsearch can use Lucene checksums to validate files while {GIT}6776[recovering a replica shard from a primary].
-
-This issue exposed a bug in Elasticsearch’s handling of primary shard failure when having more than 2 replicas, causing the second replica to not be properly unassigned if it is in the middle of recovery. It was fixed with the merge of issue {GIT}6808[#6808].
-
-In order to verify the checksumming mechanism, we added functionality to our testing infrastructure that can corrupt an arbitrary index file and at any point, such as while it’s traveling over the wire or residing on disk. The tests utilizing this feature expect full or partial recovery from the failure while neither losing data nor spreading the corruption.
-
-[float]
-=== Detect File Corruption (STATUS: DONE, v1.3.0)
-
-When a corrupted index can be detected during merging or refresh, Elasticsearch will fail the shard if a checksum failure is detected. You can read the full details in pull request {GIT}6776[#6776].
-
-[float]
-=== Network disconnect events could be lost, causing a zombie node to stay in the cluster state (STATUS: DONE, v1.3.0)
-
-Previously, there was a very short window in which we could lose a node disconnect event. To prevent this from occurring, we added extra handling of connection errors to our nodes & master fault detection pinging to make sure the node disconnect event is detected. See issue {GIT}6686[#6686].
-
-[float]
-=== Other fixes to Lucene to address resiliency (STATUS: DONE, v1.3.0)
-
-* NativeLock is released if Lock is closed after failing on obtain {JIRA}5738[LUCENE-5738].
-* NRT Reader close can wipe an index it doesn’t own. {JIRA}5574[LUCENE-5574]
-* FSDirectory’s fsync() is lenient, now throws exceptions when errors occur {JIRA}5570[LUCENE-5570]
-* fsync() directory when committing {JIRA}5588[LUCENE-5588]
-
-[float]
-=== Backwards Compatibility Testings (STATUS: DONE, v1.3.0)
-
-Since founding Elasticsearch Inc, we grew our test base from ~1k tests to about 4k in just about over a year. We invested massively into our testing infrastructure, running our tests continuously on different operating systems, bare metal hardware and cloud environments, all while randomizing JVMs and their settings.
-
-Yet, backwards compatibility testing was a very manual thing until we released a pretty {GIT}6393[insane bug] with Elasticsearch 1.2. We tried to fix places where the absolute value of a number was negative (a documented behavior of Math.abs(int) in Java) and missed that the fix for this also changed the result of our routing function. No matter how much randomization we applied to the tests, we didn’t catch this particular failure. We always had backwards compatibility tests on our list of things to do, but didn’t have them in place back then.
-
-We recently tweaked our testing infrastructure to be able to run tests against a hybrid cluster composed of a released version of Elasticsearch and our current stable branch. This test pattern allowed us to mimic typical upgrade scenarios like rolling upgrades, index backwards compatibility and recovering from old to new nodes.
-
-Now, even the simplest test that relies on routing fails against 1.2.0, which is exactly we were aiming for. The test would not have caught the aforementioned {GIT}6393[routing bug] before releasing 1.2.0, but it immediately saved us from {GIT}6660[another problem] in the stable branch.
-
-The work on our testing infrastructure is more than just issue prevention, it allows us to develop and test upgrade paths, introduce new features and evolve indexing over time. It isn’t enough to introduce more resilient implementations, we also have to ensure that users take advantage of them when they upgrade.
-
-You can read more about backwards compatibility tests in issue {GIT}6497[#6497].
-
-[float]
-=== Full Translog Writes on all Platforms (STATUS: DONE, v1.2.2 and v1.3.0)
-
-We have recently received bug reports of transaction log corruption that can occur when indexing very large documents (in the area of 300 KB). Although some Linux users reported this behavior, it appears the problem occurs more frequently when running Windows. We traced the source of the problem to the fact that when serializing documents to the transaction log, the Operating System can actually write only part of the document before returning from the write call. We can now detect this situation and make sure that the entire document is properly written. You can read the full details in pull request {GIT}6576[#6576].
-
-[float]
-=== Lucene Checksums (STATUS: DONE, v1.2.0)
-
-Before Apache Lucene version 4.8, checksums were not computed on generated index files. The result was that it was difficult to identify when or if a Lucene index got corrupted, whether by hardware failure, JVM bug or for an entirely different reason.
-
-For an idea of the checksum efforts in progress in Apache Lucene, see issues {JIRA}2446[LUCENE-2446], {JIRA}5580[LUCENE-5580] and {JIRA}5602[LUCENE-5602]. The gist is that Lucene 4.8+ now computes full checksums on all index files and it verifies them when opening metadata or other smaller files as well as other files during merges.
-
-[float]
-=== Detect errors faster by locally failing a shard upon an indexing error (STATUS: DONE, v1.2.0)
-
-Previously, Elasticsearch notified the master of the shard failure and waited for the master to close the local copy of the shard, thus assigning it to other nodes. This architecture caused delays in failure detection, potentially causing unneeded failures of other incoming requests. In rare cases, such as concurrency racing conditions or certain network partitions configurations, we could lose these failure notifications. We solved this issue by locally failing shards upon indexing errors. See issue {GIT}5847[#5847].
-
-[float]
-=== Snapshot/Restore API (STATUS: DONE, v1.0.0)
-
-In Elasticsearch version 1.0, we significantly improved the backup process by introducing the Snapshot/Restore API. While it was always possible to make backups of Elasticsearch, the Snapshot/Restore API made the backup process much easier.
-
-The backup process is incremental, making it very efficient since only files changed since the last backup are copied. Even with this efficiency introduced, each snapshot contains a full picture of the cluster at the moment when backup started. The restore API allows speedy recovery of a full cluster as well as selected indices.
-
-Since that first release in version 1.0, the API has continued to evolve. In version 1.1.0, we added a new snapshot status API that allows users to monitor the snapshot process. In 1.3.0 we added the ability to {GIT}6457[restore indices without their aliases] and in 1.4 we are planning to add the ability to {GIT}6368[restore partial snapshots].
-
-The Snapshot/Restore API supports a number of different repository types for storing backups. Currently, it’s possible to make backups to a shared file system, Amazon S3, HDFS, and Azure storage. We are continuing to work on adding other types of storage systems, as well as improving the robustness of the snapshot/restore process.
-
-[float]
-=== Circuit Breaker: Fielddata (STATUS: DONE, v1.0.0)
-
-Currently, the circuit breaker protects against loading too much field data by estimating how much memory the field data will take to load, then aborting the request if the memory requirements are too high. This feature was added in Elasticsearch version 1.0.0.
-
-[float]
-=== Use of Paginated Data Structures to Ease Garbage Collection (STATUS: DONE, v1.0.0 & v1.2.0)
-
-Elasticsearch has moved from an object-based cache to a page-based cache recycler as described in issue {GIT}4557[#4557]. This change makes garbage collection easier by limiting fragmentation, since all pages have the same size and are recycled. It also allows managing the size of the cache not based on the number of objects it contains, but on the memory that it uses.
-
-These pages are used for two main purposes: implementing higher level data structures such as hash tables that are used internally by aggregations to eg. map terms to counts, as well as reusing memory in the translog/transport layer as detailed in issue {GIT}5691[#5691].
-
-[float]
-=== Dedicated Master Nodes Resiliency (STATUS: DONE, v1.0.0)
-
-In order to run a more resilient cluster, we recommend running dedicated master nodes to ensure master nodes are not affected by resources consumed by data nodes. We also have made master nodes more resilient to heavy resource usage, mainly associated with large clusters / cluster states.
-
-These changes include:
-
-* Improve the balancing algorithm to execute faster across large clusters / many indices. (See issue {GIT}4458[#4458] and {GIT}4459[#4459])
-* Improve cluster state publishing to not create an additional network buffer per node. (More in https://github.com/elastic/elasticsearch/commit/a9e259d438c3cb1d3bef757db2d2a91cf85be609[this commit].)
-* Improve master handling of large scale mapping updates from data nodes by batching them into a single cluster event. (See issue {GIT}4373[#4373].)
-* Add an ack mechanism where next phase cluster updates are processed only when nodes acknowledged they received the previous cluster state. (See issues {GIT}3736[#3736], {GIT}3786[#3786], {GIT}4114[#4114], {GIT}4169[#4169], {GIT}4228[#4228] and {GIT}4421[#4421], which also include enhancements to the ack mechanism implementation.)
-
-[float]
-=== Multi Data Paths May Falsely Report Corrupt Index (STATUS: DONE, v1.0.0)
-
-When using multiple data paths, an index could be falsely reported as corrupted. This has been fixed with pull request {GIT}4674[#4674].
-
-[float]
-=== Randomized Testing (STATUS: DONE, v1.0.0)
-
-In order to best validate for resiliency in Elasticsearch, we rewrote the Elasticsearch test infrastructure to introduce the concept of http://berlinbuzzwords.de/sites/berlinbuzzwords.de/files/media/documents/dawidweiss-randomizedtesting-pub.pdf[randomized testing]. Randomized testing allows us to easily enhance the Elasticsearch testing infrastructure with predictably irrational conditions, making the resulting code base more resilient.
-
-Each of our integration tests runs against a cluster with a random number of nodes, and indices have a random number of shards and replicas. Merge settings change for every run, indexing is done in serial or async fashion or even wrapped in a bulk operation and thread pool sizes vary to ensure that we don’t produce a deadlock no matter what happens. The list of places we use this randomization infrastructure is long, and growing every day, and has saved us headaches several times before we shipped a particular feature.
-
-At Elasticsearch, we live the philosophy that we can miss a bug once, but never a second time. We make our tests more evil as you go, introducing randomness in all the areas where we discovered bugs. We figure if our tests don’t fail, we are not trying hard enough! If you are interested in how we have evolved our test infrastructure over time check out https://github.com/elastic/elasticsearch/issues?q=label%3Atest[issues labeled with ``test'' on GitHub].
-
-[float]
-=== Lucene Loses Data On File Descriptors Failure (STATUS: DONE, v0.90.0)
-
-When a process runs out of file descriptors, Lucene can causes an index to be completely deleted. This issue was fixed in Lucene ({JIRA}4870[version 4.2.1]) and fixed in an early version of Elasticsearch. See issue {GIT}2812[#2812].
diff --git a/docs/ruby/client.asciidoc b/docs/ruby/client.asciidoc
deleted file mode 100644
index 0c824938901d2..0000000000000
--- a/docs/ruby/client.asciidoc
+++ /dev/null
@@ -1,109 +0,0 @@
-== The Ruby Client
-
-The `elasticsearch` http://rubygems.org/gems/elasticsearch[Rubygem] provides a low-level client
-for communicating with an Elasticsearch cluster, fully compatible with other official clients.
-
-Full documentation is hosted at https://github.com/elastic/elasticsearch-ruby[Github]
-and http://rubydoc.info/gems/elasticsearch[RubyDoc]
--- this documentation provides only an overview of features.
-
-=== Elasticsearch Version Compatibility
-
-The Ruby API is compatible with both Elasticsearch 0.90.x and 1.0.x versions, you have to install
-a matching http://rubygems.org/gems/elasticsearch/versions[gem version], though:
-
-[cols="<,<",options="header",]
-|=========================================
-| Elasticsearch version | Ruby gem version
-| 0.90.x                | 0.4.x
-| 1.x                   | 1.x
-|=========================================
-
-=== Installation
-
-Install the Ruby gem for Elasticsearch *1.x*:
-
-[source,sh]
-------------------------------------
-gem install elasticsearch
-------------------------------------
-
-...or add it do your Gemfile:
-
-[source,ruby]
-------------------------------------
-gem 'elasticsearch'
-------------------------------------
-
-Install the Ruby gem for Elasticsearch *0.90.x*:
-
-[source,sh]
-------------------------------------
-gem install elasticsearch -v 0.4.10
-------------------------------------
-
-...or add it do your Gemfile:
-
-[source,ruby]
-------------------------------------
-gem 'elasticsearch', '~> 0.4'
-------------------------------------
-
-=== Example Usage
-
-[source,ruby]
-------------------------------------
-require 'elasticsearch'
-
-client = Elasticsearch::Client.new log: true
-
-client.cluster.health
-
-client.index index: 'my-index', type: 'my-document', id: 1, body: { title: 'Test' }
-
-client.indices.refresh index: 'my-index'
-
-client.search index: 'my-index', body: { query: { match: { title: 'test' } } }
-------------------------------------
-
-
-=== Features at a Glance
-
-* Pluggable logging and tracing
-* Pluggable connection selection strategies (round-robin, random, custom)
-* Pluggable transport implementation, customizable and extendable
-* Pluggable serializer implementation
-* Request retries and dead connections handling
-* Node reloading (based on cluster state) on errors or on demand
-* Modular API implementation
-* 100% REST API coverage
-
-
-=== Transport and API
-
-The `elasticsearch` gem combines two separate Rubygems:
-
-* https://github.com/elastic/elasticsearch-ruby/tree/master/elasticsearch-transport[`elasticsearch-transport`]
-provides a HTTP Ruby client for connecting to the Elasticsearch cluster,
-
-* https://github.com/elastic/elasticsearch-ruby/tree/master/elasticsearch-api[`elasticsearch-api`]
-provides a Ruby API for the Elasticsearch RESTful API.
-
-Please see their respective documentation for configuration options and technical details.
-
-Notably, the documentation and comprehensive examples for all the API methods is contained in the source,
-and available online at http://rubydoc.info/gems/elasticsearch-api/Elasticsearch/API/Actions[Rubydoc].
-
-Keep in mind, that for optimal performance, you should use a HTTP library which supports
-persistent ("keep-alive") HTTP connections.
-
-
-=== Extensions
-
-The https://github.com/elastic/elasticsearch-ruby/tree/master/elasticsearch-extensions[`elasticsearch-extensions`]
-Rubygem provides a number of extensions to the core client, such as an API to programatically launch
-Elasticsearch clusters (eg. for testing purposes), and more.
-
-Please see its
-https://github.com/elastic/elasticsearch-ruby/tree/master/elasticsearch-extensions[documentation]
-for more information.
diff --git a/docs/ruby/copyright.asciidoc b/docs/ruby/copyright.asciidoc
deleted file mode 100644
index cf5c19c9fee67..0000000000000
--- a/docs/ruby/copyright.asciidoc
+++ /dev/null
@@ -1,5 +0,0 @@
-== Copyright and License
-
-This software is Copyright (c) 2013-2016 by Elasticsearch BV.
-
-This is free software, licensed under The Apache License Version 2.0.
diff --git a/docs/ruby/index.asciidoc b/docs/ruby/index.asciidoc
deleted file mode 100644
index cc1defcd304ce..0000000000000
--- a/docs/ruby/index.asciidoc
+++ /dev/null
@@ -1,11 +0,0 @@
-= Ruby And Rails Integrations
-
-include::client.asciidoc[]
-
-include::model.asciidoc[]
-
-include::rails.asciidoc[]
-
-include::persistence.asciidoc[]
-
-include::copyright.asciidoc[]
diff --git a/docs/ruby/model.asciidoc b/docs/ruby/model.asciidoc
deleted file mode 100644
index 0b0be45708fa8..0000000000000
--- a/docs/ruby/model.asciidoc
+++ /dev/null
@@ -1,66 +0,0 @@
-== ActiveModel / ActiveRecord
-
-The `elasticsearch-model` http://rubygems.org/gems/elasticsearch-model[Rubygem]
-provides integration with Ruby domain objects ("models"), commonly found e.g. in Ruby on Rails applications.
-
-It uses the `elasticsearch` Rubygem as the client communicating with the Elasticsearch cluster.
-
-
-=== Features at a Glance
-
-* ActiveModel integration with adapters for ActiveRecord and Mongoid
-* Enumerable-based wrapper for search results
-* ActiveRecord::Relation-based wrapper for returning search results as records
-* Convenience model methods such as `search`, `mapping`, `import`, etc
-* Support for Kaminari and WillPaginate pagination
-* Extension implemented via proxy object to shield model namespace from collisions
-* Convenience methods for (re)creating the index, setting up mappings, indexing documents, ...
-
-
-=== Usage
-
-Add the library to your Gemfile:
-
-[source,ruby]
-------------------------------------
-gem 'elasticsearch-rails'
-------------------------------------
-
-Include the extension module in your model class:
-
-[source,ruby]
-------------------------------------
-class Article < ActiveRecord::Base
-  include Elasticsearch::Model
-end
-------------------------------------
-
-Import some data and perform a search:
-
-[source,ruby]
-------------------------------------
-Article.import
-
-response = Article.search 'fox dog'
-response.took
-# => 3
-------------------------------------
-
-It is possible to either return results as model instances, or decorated documents from Elasticsearch,
-with the `records` and `results` methods, respectively:
-
-[source,ruby]
-------------------------------------
-response.records.first
-# Article Load (0.4ms)  SELECT "articles".* FROM "articles"  WHERE ...
-=> #<Article id: 3, title: "Foo " ...>
-
-response.results.first._score
-# => 0.02250402
-
-response.results.first._source.title
-# => "Quick brown fox"
-------------------------------------
-
-Please see the full https://github.com/elastic/elasticsearch-rails/tree/master/elasticsearch-model[documentation]
-for more information.
diff --git a/docs/ruby/persistence.asciidoc b/docs/ruby/persistence.asciidoc
deleted file mode 100644
index 7d361978ee703..0000000000000
--- a/docs/ruby/persistence.asciidoc
+++ /dev/null
@@ -1,226 +0,0 @@
-== Persistence
-
-The `elasticsearch-persistence` http://rubygems.org/gems/elasticsearch-persistence[Rubygem]
-provides persistence layer for Ruby domain objects.
-
-It supports two design patterns for integrating with your objects: _repository_ and _active record_.
-
-=== Repository
-
-The `Elasticsearch::Persistence::Repository` module provides an implementation of the repository pattern and allows to save, delete, find and search objects stored in Elasticsearch, as well as configure mappings and settings for the index.
-
-==== Features At a Glance
-
-* Access to the Elasticsearch client
-* Setting the index name, document type, and object class for deserialization
-* Composing mappings and settings for the index
-* Creating, deleting or refreshing the index
-* Finding or searching for documents
-* Providing access both to domain objects and hits for search results
-* Providing access to the Elasticsearch response for search results (aggregations, total, ...)
-* Defining the methods for serialization and deserialization
-
-==== Usage
-
-Let's have a simple plain old Ruby object (PORO):
-
-[source,ruby]
-------------------------------------
-class Note
-  attr_reader :attributes
-
-  def initialize(attributes={})
-    @attributes = attributes
-  end
-
-  def to_hash
-    @attributes
-  end
-end
-------------------------------------
-
-
-Let's create a default, "dumb" repository, as a first step:
-
-[source,ruby]
-------------------------------------
-require 'elasticsearch/persistence'
-repository = Elasticsearch::Persistence::Repository.new
-------------------------------------
-
-We can save a `Note` instance into the repository, find it, search it, delete it:
-
-[source,ruby]
-------------------------------------
-note = Note.new id: 1, text: 'Test'
-
-repository.save(note)
-# PUT http://localhost:9200/repository/note/1
-# > {"id":1,"text":"Test"}
-# < {"_index":"repository","_type":"note","_id":"1","_version":1,"created":true}
-
-n = repository.find(1)
-# GET http://localhost:9200/repository/_all/1
-# < {"_index":"repository","_type":"note","_id":"1","_version":2,"found":true, "_source" : {"id":1,"text":"Test"}}
-=> <Note:0x007fcbfc0c4980 @attributes={"id"=>1, "text"=>"Test"}>
-
-repository.search(query: { match: { text: 'test' } }).first
-# GET http://localhost:9200/repository/_search
-# > {"query":{"match":{"text":"test"}}}
-# < {"took":2, ... "hits":{"total":1, ... "hits":[{ ... "_source" : {"id":1,"text":"Test"}}]}}
-=> <Note:0x007fcbfc1c7b70 @attributes={"id"=>1, "text"=>"Test"}>
-
-repository.delete(note)
-# DELETE http://localhost:9200/repository/note/1
-# < {"found":true,"_index":"repository","_type":"note","_id":"1","_version":3}
-=> {"found"=>true, "_index"=>"repository", "_type"=>"note", "_id"=>"1", "_version"=>2}
-------------------------------------
-
-The repository module provides a number of features and facilities to configure and customize the behaviour,
-as well as support for extending your own, custom repository class.
-
-Please refer to the
-https://github.com/elastic/elasticsearch-rails/tree/master/elasticsearch-persistence#the-repository-pattern[documentation]
-for more information.
-
-Also, check out the
-https://github.com/elastic/elasticsearch-rails/tree/master/elasticsearch-persistence#example-application[example application] which demonstrates the usage patterns of the _repository_ approach to persistence.
-
-=== Active Record
-
-The `Elasticsearch::Persistence::Model` module provides an implementation of the active record pattern, with
-a familiar interface for using Elasticsearch as a persistence layer in Ruby on Rails applications. The model
-is fully compatible with Rails' conventions and helpers, such as `url_for`.
-
-All the methods are documented with comprehensive examples in the source code, available also
-http://rubydoc.info/gems/elasticsearch-persistence/Elasticsearch/Persistence/Model[online].
-
-==== Features At a Glance
-
-* Familiar interface for persistence methods from ActiveRecord
-* Common model features such as validations and callbacks
-* Methods for defining model attributes, including Elasticsearch mappings
-* Support for fetching data in bulk (`find_in_batches`, `find_each`)
-* Decorated search results for easy access to model instances and meta data (such as highlights or aggregations)
-* Easy access to the underlying gateway and client
-
-==== Usage
-
-To use the library in a Rails application, add it to your Gemfile with a require statement:
-
-[source,ruby]
-------------------------------------
-gem "elasticsearch-persistence", require: 'elasticsearch/persistence/model'
-------------------------------------
-
-Include the module in a plain Ruby class, and set up the properties, mappings, etc:
-
-[source,ruby]
-------------------------------------
-class Article
-  include Elasticsearch::Persistence::Model
-
-  # Define a plain `title` attribute
-  #
-  attribute :title,  String
-
-  # Define an `author` attribute, with multiple analyzers for this field
-  #
-  attribute :author, String, mapping: { fields: {
-                               author: { type: 'string'},
-                               raw:    { type: 'string', analyzer: 'keyword' }
-                             } }
-
-
-  # Define a `views` attribute, with default value
-  #
-  attribute :views,  Integer, default: 0, mapping: { type: 'integer' }
-
-  # Validate the presence of the `title` attribute
-  #
-  validates :title, presence: true
-
-  # Execute code after saving the model.
-  #
-  after_save { puts "Successfully saved: #{self}" }
-end
-------------------------------------
-
-The model attribute definition support is implemented with the https://github.com/solnic/virtus[_Virtus_] Rubygem,
-and the naming, validation, etc. features with the https://github.com/rails/rails/tree/master/activemodel[_ActiveModel_] Rubygem.
-
-Attribute validations work like for any other ActiveModel-compatible implementation:
-
-[source,ruby]
-------------------------------------
-article = Article.new                                                                                             # => #<Article { ... }>
-
-article.valid?
-# => false
-
-article.errors.to_a
-# => ["Title can't be blank"]
-------------------------------------
-
-We can create a new article in the database and find it:
-
-[source,ruby]
-------------------------------------
-Article.create id: 1, title: 'Test', author: 'John'
-# PUT http://localhost:9200/articles/article/1 [status:201, request:0.015s, query:n/a]
-
-article = Article.find(1)
-# => #<Article { ... }>
-
-article._index
-# => "articles"
-
-article.id
-# => "1"
-
-article.title
-# => "Test"
-------------------------------------
-
-To update the model, either update the attribute and save the model or use the `update_attributes` method:
-
-[source,ruby]
-------------------------------------
-article.title = 'Updated'
-
-article.save
-# => {"_index"=>"articles", "_type"=>"article", "_id"=>"1", "_version"=>2, "created"=>false}
-
-article.update_attributes title: 'Test', author: 'Mary'
-# => {"_index"=>"articles", "_type"=>"article", "_id"=>"1", "_version"=>3}
-------------------------------------
-
-The implementation supports the familiar interface for updating model timestamps and numeric attributes:
-
-[source,ruby]
-------------------------------------
-article.touch
-# => => { ... "_version"=>4}
-
-article.views
-# => 0
-
-article.increment :views
-article.views
-# => 1
-------------------------------------
-
-Any callbacks defined in the model will be triggered during the persistence operations:
-
-[source,ruby]
-------------------------------------
-article.save
-# Successfully saved: #<Article {...}>
-------------------------------------
-
-Please see the extensive documentation in the library
-https://github.com/elastic/elasticsearch-rails/tree/master/elasticsearch-persistence#the-activerecord-pattern[README]
-for detailed information.
-
-Also, check out the
-https://github.com/elastic/elasticsearch-rails/tree/master/elasticsearch-persistence#example-application-1[example application] which demonstrates the usage patterns of the _active record_ approach to persistence.
diff --git a/docs/ruby/rails.asciidoc b/docs/ruby/rails.asciidoc
deleted file mode 100644
index 1fef3f42381a6..0000000000000
--- a/docs/ruby/rails.asciidoc
+++ /dev/null
@@ -1,16 +0,0 @@
-== Ruby On Rails
-
-The `elasticsearch-rails` http://rubygems.org/gems/elasticsearch-rails[Rubygem]
-provides features suitable for Ruby on Rails applications.
-
-=== Features at a Glance
-
-* Rake tasks for importing data from application models
-* Integration with Rails' instrumentation framework
-* Templates for generating example Rails application
-
-=== Example applications
-
-You can generate a fully working example Ruby on Rails application with templates provides.
-
-Please refer to the https://github.com/elastic/elasticsearch-rails/tree/master/elasticsearch-rails[documentation] for more information.
diff --git a/docs/src/test/cluster/config/analysis/example_word_list.txt b/docs/src/test/cluster/config/analysis/example_word_list.txt
new file mode 100644
index 0000000000000..f79aea42af203
--- /dev/null
+++ b/docs/src/test/cluster/config/analysis/example_word_list.txt
@@ -0,0 +1,4 @@
+test
+list
+of
+words
diff --git a/docs/src/test/cluster/config/analysis/hyphenation_patterns.xml b/docs/src/test/cluster/config/analysis/hyphenation_patterns.xml
new file mode 100644
index 0000000000000..6241b3fc6cc80
--- /dev/null
+++ b/docs/src/test/cluster/config/analysis/hyphenation_patterns.xml
@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE hyphenation-info SYSTEM "hyphenation.dtd">
+
+<!-- Example hyphenation patterns file. -->
+
+<hyphenation-info>
+
+<hyphen-char value="-"/>
+<hyphen-min before="2" after="2"/>
+
+<classes>
+aA
+</classes>
+
+<exceptions>
+</exceptions>
+
+<patterns>
+.a2
+</patterns>
+</hyphenation-info>
diff --git a/docs/src/test/cluster/config/analysis/stemmer_override.txt b/docs/src/test/cluster/config/analysis/stemmer_override.txt
new file mode 100644
index 0000000000000..6f6cd771cf572
--- /dev/null
+++ b/docs/src/test/cluster/config/analysis/stemmer_override.txt
@@ -0,0 +1,3 @@
+running => run
+
+stemmer => stemmer
diff --git a/docs/src/test/cluster/config/analysis/synonym.txt b/docs/src/test/cluster/config/analysis/synonym.txt
new file mode 100644
index 0000000000000..b51d975d8f5f2
--- /dev/null
+++ b/docs/src/test/cluster/config/analysis/synonym.txt
@@ -0,0 +1,31 @@
+# Blank lines and lines starting with pound are comments.
+
+# Explicit mappings match any token sequence on the LHS of "=>"
+# and replace with all alternatives on the RHS.  These types of mappings
+# ignore the expand parameter in the schema.
+# Examples:
+i-pod, i pod => ipod,
+sea biscuit, sea biscit => seabiscuit
+
+# Equivalent synonyms may be separated with commas and give
+# no explicit mapping.  In this case the mapping behavior will
+# be taken from the expand parameter in the schema.  This allows
+# the same synonym file to be used in different synonym handling strategies.
+# Examples:
+ipod, i-pod, i pod
+foozball , foosball
+universe , cosmos
+lol, laughing out loud
+
+# If expand==true, "ipod, i-pod, i pod" is equivalent
+# to the explicit mapping:
+ipod, i-pod, i pod => ipod, i-pod, i pod
+# If expand==false, "ipod, i-pod, i pod" is equivalent
+# to the explicit mapping:
+ipod, i-pod, i pod => ipod
+
+# Multiple synonym mapping entries are merged.
+foo => foo bar
+foo => baz
+# is equivalent to
+foo => foo bar, baz
diff --git a/docs/src/test/cluster/config/scripts/calculate_score.painless b/docs/src/test/cluster/config/scripts/calculate_score.painless
new file mode 100644
index 0000000000000..0fad3fc59f950
--- /dev/null
+++ b/docs/src/test/cluster/config/scripts/calculate_score.painless
@@ -0,0 +1 @@
+Math.log(_score * 2) + params.my_modifier
diff --git a/docs/src/test/cluster/config/scripts/my_combine_script.painless b/docs/src/test/cluster/config/scripts/my_combine_script.painless
new file mode 100644
index 0000000000000..106ef08d91fd3
--- /dev/null
+++ b/docs/src/test/cluster/config/scripts/my_combine_script.painless
@@ -0,0 +1,5 @@
+double profit = 0;
+for (t in params._agg.transactions) {
+  profit += t
+}
+return profit
diff --git a/docs/src/test/cluster/config/scripts/my_init_script.painless b/docs/src/test/cluster/config/scripts/my_init_script.painless
new file mode 100644
index 0000000000000..fb6aa11723cf1
--- /dev/null
+++ b/docs/src/test/cluster/config/scripts/my_init_script.painless
@@ -0,0 +1 @@
+params._agg.transactions = []
diff --git a/docs/src/test/cluster/config/scripts/my_map_script.painless b/docs/src/test/cluster/config/scripts/my_map_script.painless
new file mode 100644
index 0000000000000..f4700482d5585
--- /dev/null
+++ b/docs/src/test/cluster/config/scripts/my_map_script.painless
@@ -0,0 +1 @@
+params._agg.transactions.add(doc.type.value == 'sale' ? doc.amount.value : -1 * doc.amount.value)
diff --git a/docs/src/test/cluster/config/scripts/my_reduce_script.painless b/docs/src/test/cluster/config/scripts/my_reduce_script.painless
new file mode 100644
index 0000000000000..ca4f67ca2db01
--- /dev/null
+++ b/docs/src/test/cluster/config/scripts/my_reduce_script.painless
@@ -0,0 +1,5 @@
+double profit = 0;
+for (a in params._aggs) {
+  profit += a
+}
+return profit
diff --git a/docs/src/test/cluster/config/scripts/my_script.painless b/docs/src/test/cluster/config/scripts/my_script.painless
new file mode 100644
index 0000000000000..55d0e99ed0f0c
--- /dev/null
+++ b/docs/src/test/cluster/config/scripts/my_script.painless
@@ -0,0 +1,2 @@
+// Simple script to load a field. Not really a good example, but a simple one.
+doc[params.field].value
diff --git a/docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java b/docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java
index 688eefc98d534..05d072acf810b 100644
--- a/docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java
+++ b/docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java
@@ -21,13 +21,18 @@
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
-
+import org.elasticsearch.Version;
+import org.elasticsearch.client.RestClient;
+import org.apache.http.HttpHost;
+import org.elasticsearch.test.rest.yaml.ClientYamlDocsTestClient;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
+import org.elasticsearch.test.rest.yaml.ClientYamlTestClient;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
+import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestSpec;
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Map;
 
 public class DocsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
@@ -36,8 +41,8 @@ public DocsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandi
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
-        return ESClientYamlSuiteTestCase.createParameters(0, 1);
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
     }
 
     @Override
@@ -49,5 +54,16 @@ protected void afterIfFailed(List<Throwable> errors) {
         logger.error("This failing test was generated by documentation starting at {}. It may include many snippets. "
                 + "See docs/README.asciidoc for an explanation of test generation.", name);
     }
+
+    @Override
+    protected boolean randomizeContentType() {
+        return false;
+    }
+
+    @Override
+    protected ClientYamlTestClient initClientYamlTestClient(ClientYamlSuiteRestSpec restSpec, RestClient restClient, List<HttpHost> hosts,
+                                                            Version esVersion, Map<HttpHost, Version> hostVersionMap) throws IOException {
+        return new ClientYamlDocsTestClient(restSpec, restClient, hosts, esVersion, hostVersionMap);
+    }
 }
 
diff --git a/docs/src/test/resources/accounts.json b/docs/src/test/resources/accounts.json
new file mode 100644
index 0000000000000..28b2b82c3f57f
--- /dev/null
+++ b/docs/src/test/resources/accounts.json
@@ -0,0 +1,2000 @@
+{"index":{"_id":"1"}}
+{"account_number":1,"balance":39225,"firstname":"Amber","lastname":"Duke","age":32,"gender":"M","address":"880 Holmes Lane","employer":"Pyrami","email":"amberduke@pyrami.com","city":"Brogan","state":"IL"}
+{"index":{"_id":"6"}}
+{"account_number":6,"balance":5686,"firstname":"Hattie","lastname":"Bond","age":36,"gender":"M","address":"671 Bristol Street","employer":"Netagy","email":"hattiebond@netagy.com","city":"Dante","state":"TN"}
+{"index":{"_id":"13"}}
+{"account_number":13,"balance":32838,"firstname":"Nanette","lastname":"Bates","age":28,"gender":"F","address":"789 Madison Street","employer":"Quility","email":"nanettebates@quility.com","city":"Nogal","state":"VA"}
+{"index":{"_id":"18"}}
+{"account_number":18,"balance":4180,"firstname":"Dale","lastname":"Adams","age":33,"gender":"M","address":"467 Hutchinson Court","employer":"Boink","email":"daleadams@boink.com","city":"Orick","state":"MD"}
+{"index":{"_id":"20"}}
+{"account_number":20,"balance":16418,"firstname":"Elinor","lastname":"Ratliff","age":36,"gender":"M","address":"282 Kings Place","employer":"Scentric","email":"elinorratliff@scentric.com","city":"Ribera","state":"WA"}
+{"index":{"_id":"25"}}
+{"account_number":25,"balance":40540,"firstname":"Virginia","lastname":"Ayala","age":39,"gender":"F","address":"171 Putnam Avenue","employer":"Filodyne","email":"virginiaayala@filodyne.com","city":"Nicholson","state":"PA"}
+{"index":{"_id":"32"}}
+{"account_number":32,"balance":48086,"firstname":"Dillard","lastname":"Mcpherson","age":34,"gender":"F","address":"702 Quentin Street","employer":"Quailcom","email":"dillardmcpherson@quailcom.com","city":"Veguita","state":"IN"}
+{"index":{"_id":"37"}}
+{"account_number":37,"balance":18612,"firstname":"Mcgee","lastname":"Mooney","age":39,"gender":"M","address":"826 Fillmore Place","employer":"Reversus","email":"mcgeemooney@reversus.com","city":"Tooleville","state":"OK"}
+{"index":{"_id":"44"}}
+{"account_number":44,"balance":34487,"firstname":"Aurelia","lastname":"Harding","age":37,"gender":"M","address":"502 Baycliff Terrace","employer":"Orbalix","email":"aureliaharding@orbalix.com","city":"Yardville","state":"DE"}
+{"index":{"_id":"49"}}
+{"account_number":49,"balance":29104,"firstname":"Fulton","lastname":"Holt","age":23,"gender":"F","address":"451 Humboldt Street","employer":"Anocha","email":"fultonholt@anocha.com","city":"Sunriver","state":"RI"}
+{"index":{"_id":"51"}}
+{"account_number":51,"balance":14097,"firstname":"Burton","lastname":"Meyers","age":31,"gender":"F","address":"334 River Street","employer":"Bezal","email":"burtonmeyers@bezal.com","city":"Jacksonburg","state":"MO"}
+{"index":{"_id":"56"}}
+{"account_number":56,"balance":14992,"firstname":"Josie","lastname":"Nelson","age":32,"gender":"M","address":"857 Tabor Court","employer":"Emtrac","email":"josienelson@emtrac.com","city":"Sunnyside","state":"UT"}
+{"index":{"_id":"63"}}
+{"account_number":63,"balance":6077,"firstname":"Hughes","lastname":"Owens","age":30,"gender":"F","address":"510 Sedgwick Street","employer":"Valpreal","email":"hughesowens@valpreal.com","city":"Guilford","state":"KS"}
+{"index":{"_id":"68"}}
+{"account_number":68,"balance":44214,"firstname":"Hall","lastname":"Key","age":25,"gender":"F","address":"927 Bay Parkway","employer":"Eventex","email":"hallkey@eventex.com","city":"Shawmut","state":"CA"}
+{"index":{"_id":"70"}}
+{"account_number":70,"balance":38172,"firstname":"Deidre","lastname":"Thompson","age":33,"gender":"F","address":"685 School Lane","employer":"Netplode","email":"deidrethompson@netplode.com","city":"Chestnut","state":"GA"}
+{"index":{"_id":"75"}}
+{"account_number":75,"balance":40500,"firstname":"Sandoval","lastname":"Kramer","age":22,"gender":"F","address":"166 Irvington Place","employer":"Overfork","email":"sandovalkramer@overfork.com","city":"Limestone","state":"NH"}
+{"index":{"_id":"82"}}
+{"account_number":82,"balance":41412,"firstname":"Concetta","lastname":"Barnes","age":39,"gender":"F","address":"195 Bayview Place","employer":"Fitcore","email":"concettabarnes@fitcore.com","city":"Summerfield","state":"NC"}
+{"index":{"_id":"87"}}
+{"account_number":87,"balance":1133,"firstname":"Hewitt","lastname":"Kidd","age":22,"gender":"M","address":"446 Halleck Street","employer":"Isologics","email":"hewittkidd@isologics.com","city":"Coalmont","state":"ME"}
+{"index":{"_id":"94"}}
+{"account_number":94,"balance":41060,"firstname":"Brittany","lastname":"Cabrera","age":30,"gender":"F","address":"183 Kathleen Court","employer":"Mixers","email":"brittanycabrera@mixers.com","city":"Cornucopia","state":"AZ"}
+{"index":{"_id":"99"}}
+{"account_number":99,"balance":47159,"firstname":"Ratliff","lastname":"Heath","age":39,"gender":"F","address":"806 Rockwell Place","employer":"Zappix","email":"ratliffheath@zappix.com","city":"Shaft","state":"ND"}
+{"index":{"_id":"102"}}
+{"account_number":102,"balance":29712,"firstname":"Dena","lastname":"Olson","age":27,"gender":"F","address":"759 Newkirk Avenue","employer":"Hinway","email":"denaolson@hinway.com","city":"Choctaw","state":"NJ"}
+{"index":{"_id":"107"}}
+{"account_number":107,"balance":48844,"firstname":"Randi","lastname":"Rich","age":28,"gender":"M","address":"694 Jefferson Street","employer":"Netplax","email":"randirich@netplax.com","city":"Bellfountain","state":"SC"}
+{"index":{"_id":"114"}}
+{"account_number":114,"balance":43045,"firstname":"Josephine","lastname":"Joseph","age":31,"gender":"F","address":"451 Oriental Court","employer":"Turnabout","email":"josephinejoseph@turnabout.com","city":"Sedley","state":"AL"}
+{"index":{"_id":"119"}}
+{"account_number":119,"balance":49222,"firstname":"Laverne","lastname":"Johnson","age":28,"gender":"F","address":"302 Howard Place","employer":"Senmei","email":"lavernejohnson@senmei.com","city":"Herlong","state":"DC"}
+{"index":{"_id":"121"}}
+{"account_number":121,"balance":19594,"firstname":"Acevedo","lastname":"Dorsey","age":32,"gender":"M","address":"479 Nova Court","employer":"Netropic","email":"acevedodorsey@netropic.com","city":"Islandia","state":"CT"}
+{"index":{"_id":"126"}}
+{"account_number":126,"balance":3607,"firstname":"Effie","lastname":"Gates","age":39,"gender":"F","address":"620 National Drive","employer":"Digitalus","email":"effiegates@digitalus.com","city":"Blodgett","state":"MD"}
+{"index":{"_id":"133"}}
+{"account_number":133,"balance":26135,"firstname":"Deena","lastname":"Richmond","age":36,"gender":"F","address":"646 Underhill Avenue","employer":"Sunclipse","email":"deenarichmond@sunclipse.com","city":"Austinburg","state":"SC"}
+{"index":{"_id":"138"}}
+{"account_number":138,"balance":9006,"firstname":"Daniel","lastname":"Arnold","age":39,"gender":"F","address":"422 Malbone Street","employer":"Ecstasia","email":"danielarnold@ecstasia.com","city":"Gardiner","state":"MO"}
+{"index":{"_id":"140"}}
+{"account_number":140,"balance":26696,"firstname":"Cotton","lastname":"Christensen","age":32,"gender":"M","address":"878 Schermerhorn Street","employer":"Prowaste","email":"cottonchristensen@prowaste.com","city":"Mayfair","state":"LA"}
+{"index":{"_id":"145"}}
+{"account_number":145,"balance":47406,"firstname":"Rowena","lastname":"Wilkinson","age":32,"gender":"M","address":"891 Elton Street","employer":"Asimiline","email":"rowenawilkinson@asimiline.com","city":"Ripley","state":"NH"}
+{"index":{"_id":"152"}}
+{"account_number":152,"balance":8088,"firstname":"Wolfe","lastname":"Rocha","age":21,"gender":"M","address":"457 Guernsey Street","employer":"Hivedom","email":"wolferocha@hivedom.com","city":"Adelino","state":"MS"}
+{"index":{"_id":"157"}}
+{"account_number":157,"balance":39868,"firstname":"Claudia","lastname":"Terry","age":20,"gender":"F","address":"132 Gunnison Court","employer":"Lumbrex","email":"claudiaterry@lumbrex.com","city":"Castleton","state":"MD"}
+{"index":{"_id":"164"}}
+{"account_number":164,"balance":9101,"firstname":"Cummings","lastname":"Little","age":26,"gender":"F","address":"308 Schaefer Street","employer":"Comtrak","email":"cummingslittle@comtrak.com","city":"Chaparrito","state":"WI"}
+{"index":{"_id":"169"}}
+{"account_number":169,"balance":45953,"firstname":"Hollie","lastname":"Osborn","age":34,"gender":"M","address":"671 Seaview Court","employer":"Musaphics","email":"hollieosborn@musaphics.com","city":"Hanover","state":"GA"}
+{"index":{"_id":"171"}}
+{"account_number":171,"balance":7091,"firstname":"Nelda","lastname":"Hopper","age":39,"gender":"M","address":"742 Prospect Place","employer":"Equicom","email":"neldahopper@equicom.com","city":"Finderne","state":"SC"}
+{"index":{"_id":"176"}}
+{"account_number":176,"balance":18607,"firstname":"Kemp","lastname":"Walters","age":28,"gender":"F","address":"906 Howard Avenue","employer":"Eyewax","email":"kempwalters@eyewax.com","city":"Why","state":"KY"}
+{"index":{"_id":"183"}}
+{"account_number":183,"balance":14223,"firstname":"Hudson","lastname":"English","age":26,"gender":"F","address":"823 Herkimer Place","employer":"Xinware","email":"hudsonenglish@xinware.com","city":"Robbins","state":"ND"}
+{"index":{"_id":"188"}}
+{"account_number":188,"balance":41504,"firstname":"Tia","lastname":"Miranda","age":24,"gender":"F","address":"583 Ainslie Street","employer":"Jasper","email":"tiamiranda@jasper.com","city":"Summerset","state":"UT"}
+{"index":{"_id":"190"}}
+{"account_number":190,"balance":3150,"firstname":"Blake","lastname":"Davidson","age":30,"gender":"F","address":"636 Diamond Street","employer":"Quantasis","email":"blakedavidson@quantasis.com","city":"Crumpler","state":"KY"}
+{"index":{"_id":"195"}}
+{"account_number":195,"balance":5025,"firstname":"Kaye","lastname":"Gibson","age":31,"gender":"M","address":"955 Hopkins Street","employer":"Zork","email":"kayegibson@zork.com","city":"Ola","state":"WY"}
+{"index":{"_id":"203"}}
+{"account_number":203,"balance":21890,"firstname":"Eve","lastname":"Wyatt","age":33,"gender":"M","address":"435 Furman Street","employer":"Assitia","email":"evewyatt@assitia.com","city":"Jamestown","state":"MN"}
+{"index":{"_id":"208"}}
+{"account_number":208,"balance":40760,"firstname":"Garcia","lastname":"Hess","age":26,"gender":"F","address":"810 Nostrand Avenue","employer":"Quiltigen","email":"garciahess@quiltigen.com","city":"Brooktrails","state":"GA"}
+{"index":{"_id":"210"}}
+{"account_number":210,"balance":33946,"firstname":"Cherry","lastname":"Carey","age":24,"gender":"M","address":"539 Tiffany Place","employer":"Martgo","email":"cherrycarey@martgo.com","city":"Fairacres","state":"AK"}
+{"index":{"_id":"215"}}
+{"account_number":215,"balance":37427,"firstname":"Copeland","lastname":"Solomon","age":20,"gender":"M","address":"741 McDonald Avenue","employer":"Recognia","email":"copelandsolomon@recognia.com","city":"Edmund","state":"ME"}
+{"index":{"_id":"222"}}
+{"account_number":222,"balance":14764,"firstname":"Rachelle","lastname":"Rice","age":36,"gender":"M","address":"333 Narrows Avenue","employer":"Enaut","email":"rachellerice@enaut.com","city":"Wright","state":"AZ"}
+{"index":{"_id":"227"}}
+{"account_number":227,"balance":19780,"firstname":"Coleman","lastname":"Berg","age":22,"gender":"M","address":"776 Little Street","employer":"Exoteric","email":"colemanberg@exoteric.com","city":"Eagleville","state":"WV"}
+{"index":{"_id":"234"}}
+{"account_number":234,"balance":44207,"firstname":"Betty","lastname":"Hall","age":37,"gender":"F","address":"709 Garfield Place","employer":"Miraclis","email":"bettyhall@miraclis.com","city":"Bendon","state":"NY"}
+{"index":{"_id":"239"}}
+{"account_number":239,"balance":25719,"firstname":"Chang","lastname":"Boyer","age":36,"gender":"M","address":"895 Brigham Street","employer":"Qaboos","email":"changboyer@qaboos.com","city":"Belgreen","state":"NH"}
+{"index":{"_id":"241"}}
+{"account_number":241,"balance":25379,"firstname":"Schroeder","lastname":"Harrington","age":26,"gender":"M","address":"610 Tapscott Avenue","employer":"Otherway","email":"schroederharrington@otherway.com","city":"Ebro","state":"TX"}
+{"index":{"_id":"246"}}
+{"account_number":246,"balance":28405,"firstname":"Katheryn","lastname":"Foster","age":21,"gender":"F","address":"259 Kane Street","employer":"Quantalia","email":"katherynfoster@quantalia.com","city":"Bath","state":"TX"}
+{"index":{"_id":"253"}}
+{"account_number":253,"balance":20240,"firstname":"Melissa","lastname":"Gould","age":31,"gender":"M","address":"440 Fuller Place","employer":"Buzzopia","email":"melissagould@buzzopia.com","city":"Lumberton","state":"MD"}
+{"index":{"_id":"258"}}
+{"account_number":258,"balance":5712,"firstname":"Lindsey","lastname":"Hawkins","age":37,"gender":"M","address":"706 Frost Street","employer":"Enormo","email":"lindseyhawkins@enormo.com","city":"Gardners","state":"AK"}
+{"index":{"_id":"260"}}
+{"account_number":260,"balance":2726,"firstname":"Kari","lastname":"Skinner","age":30,"gender":"F","address":"735 Losee Terrace","employer":"Singavera","email":"kariskinner@singavera.com","city":"Rushford","state":"WV"}
+{"index":{"_id":"265"}}
+{"account_number":265,"balance":46910,"firstname":"Marion","lastname":"Schneider","age":26,"gender":"F","address":"574 Everett Avenue","employer":"Evidends","email":"marionschneider@evidends.com","city":"Maplewood","state":"WY"}
+{"index":{"_id":"272"}}
+{"account_number":272,"balance":19253,"firstname":"Lilly","lastname":"Morgan","age":25,"gender":"F","address":"689 Fleet Street","employer":"Biolive","email":"lillymorgan@biolive.com","city":"Sunbury","state":"OH"}
+{"index":{"_id":"277"}}
+{"account_number":277,"balance":29564,"firstname":"Romero","lastname":"Lott","age":31,"gender":"M","address":"456 Danforth Street","employer":"Plasto","email":"romerolott@plasto.com","city":"Vincent","state":"VT"}
+{"index":{"_id":"284"}}
+{"account_number":284,"balance":22806,"firstname":"Randolph","lastname":"Banks","age":29,"gender":"M","address":"875 Hamilton Avenue","employer":"Caxt","email":"randolphbanks@caxt.com","city":"Crawfordsville","state":"WA"}
+{"index":{"_id":"289"}}
+{"account_number":289,"balance":7798,"firstname":"Blair","lastname":"Church","age":29,"gender":"M","address":"370 Sutton Street","employer":"Cubix","email":"blairchurch@cubix.com","city":"Nile","state":"NH"}
+{"index":{"_id":"291"}}
+{"account_number":291,"balance":19955,"firstname":"Lynn","lastname":"Pollard","age":40,"gender":"F","address":"685 Pierrepont Street","employer":"Slambda","email":"lynnpollard@slambda.com","city":"Mappsville","state":"ID"}
+{"index":{"_id":"296"}}
+{"account_number":296,"balance":24606,"firstname":"Rosa","lastname":"Oliver","age":34,"gender":"M","address":"168 Woodbine Street","employer":"Idetica","email":"rosaoliver@idetica.com","city":"Robinson","state":"WY"}
+{"index":{"_id":"304"}}
+{"account_number":304,"balance":28647,"firstname":"Palmer","lastname":"Clark","age":35,"gender":"M","address":"866 Boulevard Court","employer":"Maximind","email":"palmerclark@maximind.com","city":"Avalon","state":"NH"}
+{"index":{"_id":"309"}}
+{"account_number":309,"balance":3830,"firstname":"Rosemarie","lastname":"Nieves","age":30,"gender":"M","address":"206 Alice Court","employer":"Zounds","email":"rosemarienieves@zounds.com","city":"Ferney","state":"AR"}
+{"index":{"_id":"311"}}
+{"account_number":311,"balance":13388,"firstname":"Vinson","lastname":"Ballard","age":23,"gender":"F","address":"960 Glendale Court","employer":"Gynk","email":"vinsonballard@gynk.com","city":"Fairforest","state":"WY"}
+{"index":{"_id":"316"}}
+{"account_number":316,"balance":8214,"firstname":"Anita","lastname":"Ewing","age":32,"gender":"M","address":"396 Lombardy Street","employer":"Panzent","email":"anitaewing@panzent.com","city":"Neahkahnie","state":"WY"}
+{"index":{"_id":"323"}}
+{"account_number":323,"balance":42230,"firstname":"Chelsea","lastname":"Gamble","age":34,"gender":"F","address":"356 Dare Court","employer":"Isosphere","email":"chelseagamble@isosphere.com","city":"Dundee","state":"MD"}
+{"index":{"_id":"328"}}
+{"account_number":328,"balance":12523,"firstname":"Good","lastname":"Campbell","age":27,"gender":"F","address":"438 Hicks Street","employer":"Gracker","email":"goodcampbell@gracker.com","city":"Marion","state":"CA"}
+{"index":{"_id":"330"}}
+{"account_number":330,"balance":41620,"firstname":"Yvette","lastname":"Browning","age":34,"gender":"F","address":"431 Beekman Place","employer":"Marketoid","email":"yvettebrowning@marketoid.com","city":"Talpa","state":"CO"}
+{"index":{"_id":"335"}}
+{"account_number":335,"balance":35433,"firstname":"Vera","lastname":"Hansen","age":24,"gender":"M","address":"252 Bushwick Avenue","employer":"Zanilla","email":"verahansen@zanilla.com","city":"Manila","state":"TN"}
+{"index":{"_id":"342"}}
+{"account_number":342,"balance":33670,"firstname":"Vivian","lastname":"Wells","age":36,"gender":"M","address":"570 Cobek Court","employer":"Nutralab","email":"vivianwells@nutralab.com","city":"Fontanelle","state":"OK"}
+{"index":{"_id":"347"}}
+{"account_number":347,"balance":36038,"firstname":"Gould","lastname":"Carson","age":24,"gender":"F","address":"784 Pulaski Street","employer":"Mobildata","email":"gouldcarson@mobildata.com","city":"Goochland","state":"MI"}
+{"index":{"_id":"354"}}
+{"account_number":354,"balance":21294,"firstname":"Kidd","lastname":"Mclean","age":22,"gender":"M","address":"691 Saratoga Avenue","employer":"Ronbert","email":"kiddmclean@ronbert.com","city":"Tioga","state":"ME"}
+{"index":{"_id":"359"}}
+{"account_number":359,"balance":29927,"firstname":"Vanessa","lastname":"Harvey","age":28,"gender":"F","address":"679 Rutledge Street","employer":"Zentime","email":"vanessaharvey@zentime.com","city":"Williston","state":"IL"}
+{"index":{"_id":"361"}}
+{"account_number":361,"balance":23659,"firstname":"Noreen","lastname":"Shelton","age":36,"gender":"M","address":"702 Tillary Street","employer":"Medmex","email":"noreenshelton@medmex.com","city":"Derwood","state":"NH"}
+{"index":{"_id":"366"}}
+{"account_number":366,"balance":42368,"firstname":"Lydia","lastname":"Cooke","age":31,"gender":"M","address":"470 Coleman Street","employer":"Comstar","email":"lydiacooke@comstar.com","city":"Datil","state":"TN"}
+{"index":{"_id":"373"}}
+{"account_number":373,"balance":9671,"firstname":"Simpson","lastname":"Carpenter","age":21,"gender":"M","address":"837 Horace Court","employer":"Snips","email":"simpsoncarpenter@snips.com","city":"Tolu","state":"MA"}
+{"index":{"_id":"378"}}
+{"account_number":378,"balance":27100,"firstname":"Watson","lastname":"Simpson","age":36,"gender":"F","address":"644 Thomas Street","employer":"Wrapture","email":"watsonsimpson@wrapture.com","city":"Keller","state":"TX"}
+{"index":{"_id":"380"}}
+{"account_number":380,"balance":35628,"firstname":"Fernandez","lastname":"Reid","age":33,"gender":"F","address":"154 Melba Court","employer":"Cosmosis","email":"fernandezreid@cosmosis.com","city":"Boyd","state":"NE"}
+{"index":{"_id":"385"}}
+{"account_number":385,"balance":11022,"firstname":"Rosalinda","lastname":"Valencia","age":22,"gender":"M","address":"933 Lloyd Street","employer":"Zoarere","email":"rosalindavalencia@zoarere.com","city":"Waverly","state":"GA"}
+{"index":{"_id":"392"}}
+{"account_number":392,"balance":31613,"firstname":"Dotson","lastname":"Dean","age":35,"gender":"M","address":"136 Ford Street","employer":"Petigems","email":"dotsondean@petigems.com","city":"Chical","state":"SD"}
+{"index":{"_id":"397"}}
+{"account_number":397,"balance":37418,"firstname":"Leonard","lastname":"Gray","age":36,"gender":"F","address":"840 Morgan Avenue","employer":"Recritube","email":"leonardgray@recritube.com","city":"Edenburg","state":"AL"}
+{"index":{"_id":"400"}}
+{"account_number":400,"balance":20685,"firstname":"Kane","lastname":"King","age":21,"gender":"F","address":"405 Cornelia Street","employer":"Tri@Tribalog","email":"kaneking@tri@tribalog.com","city":"Gulf","state":"VT"}
+{"index":{"_id":"405"}}
+{"account_number":405,"balance":5679,"firstname":"Strickland","lastname":"Fuller","age":26,"gender":"M","address":"990 Concord Street","employer":"Digique","email":"stricklandfuller@digique.com","city":"Southmont","state":"NV"}
+{"index":{"_id":"412"}}
+{"account_number":412,"balance":27436,"firstname":"Ilene","lastname":"Abbott","age":26,"gender":"M","address":"846 Vine Street","employer":"Typhonica","email":"ileneabbott@typhonica.com","city":"Cedarville","state":"VT"}
+{"index":{"_id":"417"}}
+{"account_number":417,"balance":1788,"firstname":"Wheeler","lastname":"Ayers","age":35,"gender":"F","address":"677 Hope Street","employer":"Fortean","email":"wheelerayers@fortean.com","city":"Ironton","state":"PA"}
+{"index":{"_id":"424"}}
+{"account_number":424,"balance":36818,"firstname":"Tracie","lastname":"Gregory","age":34,"gender":"M","address":"112 Hunterfly Place","employer":"Comstruct","email":"traciegregory@comstruct.com","city":"Onton","state":"TN"}
+{"index":{"_id":"429"}}
+{"account_number":429,"balance":46970,"firstname":"Cantu","lastname":"Lindsey","age":31,"gender":"M","address":"404 Willoughby Avenue","employer":"Inquala","email":"cantulindsey@inquala.com","city":"Cowiche","state":"IA"}
+{"index":{"_id":"431"}}
+{"account_number":431,"balance":13136,"firstname":"Laurie","lastname":"Shaw","age":26,"gender":"F","address":"263 Aviation Road","employer":"Zillanet","email":"laurieshaw@zillanet.com","city":"Harmon","state":"WV"}
+{"index":{"_id":"436"}}
+{"account_number":436,"balance":27585,"firstname":"Alexander","lastname":"Sargent","age":23,"gender":"M","address":"363 Albemarle Road","employer":"Fangold","email":"alexandersargent@fangold.com","city":"Calpine","state":"OR"}
+{"index":{"_id":"443"}}
+{"account_number":443,"balance":7588,"firstname":"Huff","lastname":"Thomas","age":23,"gender":"M","address":"538 Erskine Loop","employer":"Accufarm","email":"huffthomas@accufarm.com","city":"Corinne","state":"AL"}
+{"index":{"_id":"448"}}
+{"account_number":448,"balance":22776,"firstname":"Adriana","lastname":"Mcfadden","age":35,"gender":"F","address":"984 Woodside Avenue","employer":"Telequiet","email":"adrianamcfadden@telequiet.com","city":"Darrtown","state":"WI"}
+{"index":{"_id":"450"}}
+{"account_number":450,"balance":2643,"firstname":"Bradford","lastname":"Nielsen","age":25,"gender":"M","address":"487 Keen Court","employer":"Exovent","email":"bradfordnielsen@exovent.com","city":"Hamilton","state":"DE"}
+{"index":{"_id":"455"}}
+{"account_number":455,"balance":39556,"firstname":"Lynn","lastname":"Tran","age":36,"gender":"M","address":"741 Richmond Street","employer":"Optyk","email":"lynntran@optyk.com","city":"Clinton","state":"WV"}
+{"index":{"_id":"462"}}
+{"account_number":462,"balance":10871,"firstname":"Calderon","lastname":"Day","age":27,"gender":"M","address":"810 Milford Street","employer":"Cofine","email":"calderonday@cofine.com","city":"Kula","state":"OK"}
+{"index":{"_id":"467"}}
+{"account_number":467,"balance":6312,"firstname":"Angelica","lastname":"May","age":32,"gender":"F","address":"384 Karweg Place","employer":"Keeg","email":"angelicamay@keeg.com","city":"Tetherow","state":"IA"}
+{"index":{"_id":"474"}}
+{"account_number":474,"balance":35896,"firstname":"Obrien","lastname":"Walton","age":40,"gender":"F","address":"192 Ide Court","employer":"Suremax","email":"obrienwalton@suremax.com","city":"Crucible","state":"UT"}
+{"index":{"_id":"479"}}
+{"account_number":479,"balance":31865,"firstname":"Cameron","lastname":"Ross","age":40,"gender":"M","address":"904 Bouck Court","employer":"Telpod","email":"cameronross@telpod.com","city":"Nord","state":"MO"}
+{"index":{"_id":"481"}}
+{"account_number":481,"balance":20024,"firstname":"Lina","lastname":"Stanley","age":33,"gender":"M","address":"361 Hanover Place","employer":"Strozen","email":"linastanley@strozen.com","city":"Wyoming","state":"NC"}
+{"index":{"_id":"486"}}
+{"account_number":486,"balance":35902,"firstname":"Dixie","lastname":"Fuentes","age":22,"gender":"F","address":"991 Applegate Court","employer":"Portico","email":"dixiefuentes@portico.com","city":"Salix","state":"VA"}
+{"index":{"_id":"493"}}
+{"account_number":493,"balance":5871,"firstname":"Campbell","lastname":"Best","age":24,"gender":"M","address":"297 Friel Place","employer":"Fanfare","email":"campbellbest@fanfare.com","city":"Kidder","state":"GA"}
+{"index":{"_id":"498"}}
+{"account_number":498,"balance":10516,"firstname":"Stella","lastname":"Hinton","age":39,"gender":"F","address":"649 Columbia Place","employer":"Flyboyz","email":"stellahinton@flyboyz.com","city":"Crenshaw","state":"SC"}
+{"index":{"_id":"501"}}
+{"account_number":501,"balance":16572,"firstname":"Kelley","lastname":"Ochoa","age":36,"gender":"M","address":"451 Clifton Place","employer":"Bluplanet","email":"kelleyochoa@bluplanet.com","city":"Gouglersville","state":"CT"}
+{"index":{"_id":"506"}}
+{"account_number":506,"balance":43440,"firstname":"Davidson","lastname":"Salas","age":28,"gender":"M","address":"731 Cleveland Street","employer":"Sequitur","email":"davidsonsalas@sequitur.com","city":"Lloyd","state":"ME"}
+{"index":{"_id":"513"}}
+{"account_number":513,"balance":30040,"firstname":"Maryellen","lastname":"Rose","age":37,"gender":"F","address":"428 Durland Place","employer":"Waterbaby","email":"maryellenrose@waterbaby.com","city":"Kiskimere","state":"RI"}
+{"index":{"_id":"518"}}
+{"account_number":518,"balance":48954,"firstname":"Finch","lastname":"Curtis","age":29,"gender":"F","address":"137 Ryder Street","employer":"Viagrand","email":"finchcurtis@viagrand.com","city":"Riverton","state":"MO"}
+{"index":{"_id":"520"}}
+{"account_number":520,"balance":27987,"firstname":"Brandy","lastname":"Calhoun","age":32,"gender":"M","address":"818 Harden Street","employer":"Maxemia","email":"brandycalhoun@maxemia.com","city":"Sidman","state":"OR"}
+{"index":{"_id":"525"}}
+{"account_number":525,"balance":23545,"firstname":"Holly","lastname":"Miles","age":25,"gender":"M","address":"746 Ludlam Place","employer":"Xurban","email":"hollymiles@xurban.com","city":"Harold","state":"AR"}
+{"index":{"_id":"532"}}
+{"account_number":532,"balance":17207,"firstname":"Hardin","lastname":"Kirk","age":26,"gender":"M","address":"268 Canarsie Road","employer":"Exposa","email":"hardinkirk@exposa.com","city":"Stouchsburg","state":"IL"}
+{"index":{"_id":"537"}}
+{"account_number":537,"balance":31069,"firstname":"Morin","lastname":"Frost","age":29,"gender":"M","address":"910 Lake Street","employer":"Primordia","email":"morinfrost@primordia.com","city":"Rivera","state":"DE"}
+{"index":{"_id":"544"}}
+{"account_number":544,"balance":41735,"firstname":"Short","lastname":"Dennis","age":21,"gender":"F","address":"908 Glen Street","employer":"Minga","email":"shortdennis@minga.com","city":"Dale","state":"KY"}
+{"index":{"_id":"549"}}
+{"account_number":549,"balance":1932,"firstname":"Jacqueline","lastname":"Maxwell","age":40,"gender":"M","address":"444 Schenck Place","employer":"Fuelworks","email":"jacquelinemaxwell@fuelworks.com","city":"Oretta","state":"OR"}
+{"index":{"_id":"551"}}
+{"account_number":551,"balance":21732,"firstname":"Milagros","lastname":"Travis","age":27,"gender":"F","address":"380 Murdock Court","employer":"Sloganaut","email":"milagrostravis@sloganaut.com","city":"Homeland","state":"AR"}
+{"index":{"_id":"556"}}
+{"account_number":556,"balance":36420,"firstname":"Collier","lastname":"Odonnell","age":35,"gender":"M","address":"591 Nolans Lane","employer":"Sultraxin","email":"collierodonnell@sultraxin.com","city":"Fulford","state":"MD"}
+{"index":{"_id":"563"}}
+{"account_number":563,"balance":43403,"firstname":"Morgan","lastname":"Torres","age":30,"gender":"F","address":"672 Belvidere Street","employer":"Quonata","email":"morgantorres@quonata.com","city":"Hollymead","state":"KY"}
+{"index":{"_id":"568"}}
+{"account_number":568,"balance":36628,"firstname":"Lesa","lastname":"Maynard","age":29,"gender":"F","address":"295 Whitty Lane","employer":"Coash","email":"lesamaynard@coash.com","city":"Broadlands","state":"VT"}
+{"index":{"_id":"570"}}
+{"account_number":570,"balance":26751,"firstname":"Church","lastname":"Mercado","age":24,"gender":"F","address":"892 Wyckoff Street","employer":"Xymonk","email":"churchmercado@xymonk.com","city":"Gloucester","state":"KY"}
+{"index":{"_id":"575"}}
+{"account_number":575,"balance":12588,"firstname":"Buchanan","lastname":"Pope","age":39,"gender":"M","address":"581 Sumner Place","employer":"Stucco","email":"buchananpope@stucco.com","city":"Ellerslie","state":"MD"}
+{"index":{"_id":"582"}}
+{"account_number":582,"balance":33371,"firstname":"Manning","lastname":"Guthrie","age":24,"gender":"F","address":"271 Jodie Court","employer":"Xerex","email":"manningguthrie@xerex.com","city":"Breinigsville","state":"NM"}
+{"index":{"_id":"587"}}
+{"account_number":587,"balance":3468,"firstname":"Carly","lastname":"Johns","age":33,"gender":"M","address":"390 Noll Street","employer":"Gallaxia","email":"carlyjohns@gallaxia.com","city":"Emison","state":"DC"}
+{"index":{"_id":"594"}}
+{"account_number":594,"balance":28194,"firstname":"Golden","lastname":"Donovan","age":26,"gender":"M","address":"199 Jewel Street","employer":"Organica","email":"goldendonovan@organica.com","city":"Macdona","state":"RI"}
+{"index":{"_id":"599"}}
+{"account_number":599,"balance":11944,"firstname":"Joanna","lastname":"Jennings","age":36,"gender":"F","address":"318 Irving Street","employer":"Extremo","email":"joannajennings@extremo.com","city":"Bartley","state":"MI"}
+{"index":{"_id":"602"}}
+{"account_number":602,"balance":38699,"firstname":"Mcgowan","lastname":"Mcclain","age":33,"gender":"M","address":"361 Stoddard Place","employer":"Oatfarm","email":"mcgowanmcclain@oatfarm.com","city":"Kapowsin","state":"MI"}
+{"index":{"_id":"607"}}
+{"account_number":607,"balance":38350,"firstname":"White","lastname":"Small","age":38,"gender":"F","address":"736 Judge Street","employer":"Immunics","email":"whitesmall@immunics.com","city":"Fairfield","state":"HI"}
+{"index":{"_id":"614"}}
+{"account_number":614,"balance":13157,"firstname":"Salazar","lastname":"Howard","age":35,"gender":"F","address":"847 Imlay Street","employer":"Retrack","email":"salazarhoward@retrack.com","city":"Grill","state":"FL"}
+{"index":{"_id":"619"}}
+{"account_number":619,"balance":48755,"firstname":"Grimes","lastname":"Reynolds","age":36,"gender":"M","address":"378 Denton Place","employer":"Frenex","email":"grimesreynolds@frenex.com","city":"Murillo","state":"LA"}
+{"index":{"_id":"621"}}
+{"account_number":621,"balance":35480,"firstname":"Leslie","lastname":"Sloan","age":26,"gender":"F","address":"336 Kansas Place","employer":"Dancity","email":"lesliesloan@dancity.com","city":"Corriganville","state":"AR"}
+{"index":{"_id":"626"}}
+{"account_number":626,"balance":19498,"firstname":"Ava","lastname":"Richardson","age":31,"gender":"F","address":"666 Nautilus Avenue","employer":"Cinaster","email":"avarichardson@cinaster.com","city":"Sutton","state":"AL"}
+{"index":{"_id":"633"}}
+{"account_number":633,"balance":35874,"firstname":"Conner","lastname":"Ramos","age":34,"gender":"M","address":"575 Agate Court","employer":"Insource","email":"connerramos@insource.com","city":"Madaket","state":"OK"}
+{"index":{"_id":"638"}}
+{"account_number":638,"balance":2658,"firstname":"Bridget","lastname":"Gallegos","age":31,"gender":"M","address":"383 Wogan Terrace","employer":"Songlines","email":"bridgetgallegos@songlines.com","city":"Linganore","state":"WA"}
+{"index":{"_id":"640"}}
+{"account_number":640,"balance":35596,"firstname":"Candace","lastname":"Hancock","age":25,"gender":"M","address":"574 Riverdale Avenue","employer":"Animalia","email":"candacehancock@animalia.com","city":"Blandburg","state":"KY"}
+{"index":{"_id":"645"}}
+{"account_number":645,"balance":29362,"firstname":"Edwina","lastname":"Hutchinson","age":26,"gender":"F","address":"892 Pacific Street","employer":"Essensia","email":"edwinahutchinson@essensia.com","city":"Dowling","state":"NE"}
+{"index":{"_id":"652"}}
+{"account_number":652,"balance":17363,"firstname":"Bonner","lastname":"Garner","age":26,"gender":"M","address":"219 Grafton Street","employer":"Utarian","email":"bonnergarner@utarian.com","city":"Vandiver","state":"PA"}
+{"index":{"_id":"657"}}
+{"account_number":657,"balance":40475,"firstname":"Kathleen","lastname":"Wilder","age":34,"gender":"F","address":"286 Sutter Avenue","employer":"Solgan","email":"kathleenwilder@solgan.com","city":"Graniteville","state":"MI"}
+{"index":{"_id":"664"}}
+{"account_number":664,"balance":16163,"firstname":"Hart","lastname":"Mccormick","age":40,"gender":"M","address":"144 Guider Avenue","employer":"Dyno","email":"hartmccormick@dyno.com","city":"Carbonville","state":"ID"}
+{"index":{"_id":"669"}}
+{"account_number":669,"balance":16934,"firstname":"Jewel","lastname":"Estrada","age":28,"gender":"M","address":"896 Meeker Avenue","employer":"Zilla","email":"jewelestrada@zilla.com","city":"Goodville","state":"PA"}
+{"index":{"_id":"671"}}
+{"account_number":671,"balance":29029,"firstname":"Antoinette","lastname":"Cook","age":34,"gender":"M","address":"375 Cumberland Street","employer":"Harmoney","email":"antoinettecook@harmoney.com","city":"Bergoo","state":"VT"}
+{"index":{"_id":"676"}}
+{"account_number":676,"balance":23842,"firstname":"Lisa","lastname":"Dudley","age":34,"gender":"M","address":"506 Vanderveer Street","employer":"Tropoli","email":"lisadudley@tropoli.com","city":"Konterra","state":"NY"}
+{"index":{"_id":"683"}}
+{"account_number":683,"balance":4381,"firstname":"Matilda","lastname":"Berger","age":39,"gender":"M","address":"884 Noble Street","employer":"Fibrodyne","email":"matildaberger@fibrodyne.com","city":"Shepardsville","state":"TN"}
+{"index":{"_id":"688"}}
+{"account_number":688,"balance":17931,"firstname":"Freeman","lastname":"Zamora","age":22,"gender":"F","address":"114 Herzl Street","employer":"Elemantra","email":"freemanzamora@elemantra.com","city":"Libertytown","state":"NM"}
+{"index":{"_id":"690"}}
+{"account_number":690,"balance":18127,"firstname":"Russo","lastname":"Swanson","age":35,"gender":"F","address":"256 Roebling Street","employer":"Zaj","email":"russoswanson@zaj.com","city":"Hoagland","state":"MI"}
+{"index":{"_id":"695"}}
+{"account_number":695,"balance":36800,"firstname":"Gonzales","lastname":"Mcfarland","age":26,"gender":"F","address":"647 Louisa Street","employer":"Songbird","email":"gonzalesmcfarland@songbird.com","city":"Crisman","state":"ID"}
+{"index":{"_id":"703"}}
+{"account_number":703,"balance":27443,"firstname":"Dona","lastname":"Burton","age":29,"gender":"M","address":"489 Flatlands Avenue","employer":"Cytrex","email":"donaburton@cytrex.com","city":"Reno","state":"VA"}
+{"index":{"_id":"708"}}
+{"account_number":708,"balance":34002,"firstname":"May","lastname":"Ortiz","age":28,"gender":"F","address":"244 Chauncey Street","employer":"Syntac","email":"mayortiz@syntac.com","city":"Munjor","state":"ID"}
+{"index":{"_id":"710"}}
+{"account_number":710,"balance":33650,"firstname":"Shelton","lastname":"Stark","age":37,"gender":"M","address":"404 Ovington Avenue","employer":"Kraggle","email":"sheltonstark@kraggle.com","city":"Ogema","state":"TN"}
+{"index":{"_id":"715"}}
+{"account_number":715,"balance":23734,"firstname":"Tammi","lastname":"Hodge","age":24,"gender":"M","address":"865 Church Lane","employer":"Netur","email":"tammihodge@netur.com","city":"Lacomb","state":"KS"}
+{"index":{"_id":"722"}}
+{"account_number":722,"balance":27256,"firstname":"Roberts","lastname":"Beasley","age":34,"gender":"F","address":"305 Kings Hwy","employer":"Quintity","email":"robertsbeasley@quintity.com","city":"Hayden","state":"PA"}
+{"index":{"_id":"727"}}
+{"account_number":727,"balance":27263,"firstname":"Natasha","lastname":"Knapp","age":36,"gender":"M","address":"723 Hubbard Street","employer":"Exostream","email":"natashaknapp@exostream.com","city":"Trexlertown","state":"LA"}
+{"index":{"_id":"734"}}
+{"account_number":734,"balance":20325,"firstname":"Keri","lastname":"Kinney","age":23,"gender":"M","address":"490 Balfour Place","employer":"Retrotex","email":"kerikinney@retrotex.com","city":"Salunga","state":"PA"}
+{"index":{"_id":"739"}}
+{"account_number":739,"balance":39063,"firstname":"Gwen","lastname":"Hardy","age":33,"gender":"F","address":"733 Stuart Street","employer":"Exozent","email":"gwenhardy@exozent.com","city":"Drytown","state":"NY"}
+{"index":{"_id":"741"}}
+{"account_number":741,"balance":33074,"firstname":"Nielsen","lastname":"Good","age":22,"gender":"M","address":"404 Norfolk Street","employer":"Kiggle","email":"nielsengood@kiggle.com","city":"Cumberland","state":"WA"}
+{"index":{"_id":"746"}}
+{"account_number":746,"balance":15970,"firstname":"Marguerite","lastname":"Wall","age":28,"gender":"F","address":"364 Crosby Avenue","employer":"Aquoavo","email":"margueritewall@aquoavo.com","city":"Jeff","state":"MI"}
+{"index":{"_id":"753"}}
+{"account_number":753,"balance":33340,"firstname":"Katina","lastname":"Alford","age":21,"gender":"F","address":"690 Ross Street","employer":"Intrawear","email":"katinaalford@intrawear.com","city":"Grimsley","state":"OK"}
+{"index":{"_id":"758"}}
+{"account_number":758,"balance":15739,"firstname":"Berta","lastname":"Short","age":28,"gender":"M","address":"149 Surf Avenue","employer":"Ozean","email":"bertashort@ozean.com","city":"Odessa","state":"UT"}
+{"index":{"_id":"760"}}
+{"account_number":760,"balance":40996,"firstname":"Rhea","lastname":"Blair","age":37,"gender":"F","address":"440 Hubbard Place","employer":"Bicol","email":"rheablair@bicol.com","city":"Stockwell","state":"LA"}
+{"index":{"_id":"765"}}
+{"account_number":765,"balance":31278,"firstname":"Knowles","lastname":"Cunningham","age":23,"gender":"M","address":"753 Macdougal Street","employer":"Thredz","email":"knowlescunningham@thredz.com","city":"Thomasville","state":"WA"}
+{"index":{"_id":"772"}}
+{"account_number":772,"balance":37849,"firstname":"Eloise","lastname":"Sparks","age":21,"gender":"M","address":"608 Willow Street","employer":"Satiance","email":"eloisesparks@satiance.com","city":"Richford","state":"NY"}
+{"index":{"_id":"777"}}
+{"account_number":777,"balance":48294,"firstname":"Adkins","lastname":"Mejia","age":32,"gender":"M","address":"186 Oxford Walk","employer":"Datagen","email":"adkinsmejia@datagen.com","city":"Faywood","state":"OK"}
+{"index":{"_id":"784"}}
+{"account_number":784,"balance":25291,"firstname":"Mabel","lastname":"Thornton","age":21,"gender":"M","address":"124 Louisiana Avenue","employer":"Zolavo","email":"mabelthornton@zolavo.com","city":"Lynn","state":"AL"}
+{"index":{"_id":"789"}}
+{"account_number":789,"balance":8760,"firstname":"Cunningham","lastname":"Kerr","age":27,"gender":"F","address":"154 Sharon Street","employer":"Polarium","email":"cunninghamkerr@polarium.com","city":"Tuskahoma","state":"MS"}
+{"index":{"_id":"791"}}
+{"account_number":791,"balance":48249,"firstname":"Janine","lastname":"Huber","age":38,"gender":"F","address":"348 Porter Avenue","employer":"Viocular","email":"janinehuber@viocular.com","city":"Fivepointville","state":"MA"}
+{"index":{"_id":"796"}}
+{"account_number":796,"balance":23503,"firstname":"Mona","lastname":"Craft","age":35,"gender":"F","address":"511 Henry Street","employer":"Opticom","email":"monacraft@opticom.com","city":"Websterville","state":"IN"}
+{"index":{"_id":"804"}}
+{"account_number":804,"balance":23610,"firstname":"Rojas","lastname":"Oneal","age":27,"gender":"M","address":"669 Sandford Street","employer":"Glukgluk","email":"rojasoneal@glukgluk.com","city":"Wheaton","state":"ME"}
+{"index":{"_id":"809"}}
+{"account_number":809,"balance":47812,"firstname":"Christie","lastname":"Strickland","age":30,"gender":"M","address":"346 Bancroft Place","employer":"Anarco","email":"christiestrickland@anarco.com","city":"Baden","state":"NV"}
+{"index":{"_id":"811"}}
+{"account_number":811,"balance":26007,"firstname":"Walls","lastname":"Rogers","age":28,"gender":"F","address":"352 Freeman Street","employer":"Geekmosis","email":"wallsrogers@geekmosis.com","city":"Caroleen","state":"NV"}
+{"index":{"_id":"816"}}
+{"account_number":816,"balance":9567,"firstname":"Cornelia","lastname":"Lane","age":20,"gender":"F","address":"384 Bainbridge Street","employer":"Sulfax","email":"cornelialane@sulfax.com","city":"Elizaville","state":"MS"}
+{"index":{"_id":"823"}}
+{"account_number":823,"balance":48726,"firstname":"Celia","lastname":"Bernard","age":33,"gender":"F","address":"466 Amboy Street","employer":"Mitroc","email":"celiabernard@mitroc.com","city":"Skyland","state":"GA"}
+{"index":{"_id":"828"}}
+{"account_number":828,"balance":44890,"firstname":"Blanche","lastname":"Holmes","age":33,"gender":"F","address":"605 Stryker Court","employer":"Motovate","email":"blancheholmes@motovate.com","city":"Loomis","state":"KS"}
+{"index":{"_id":"830"}}
+{"account_number":830,"balance":45210,"firstname":"Louella","lastname":"Chan","age":23,"gender":"M","address":"511 Heath Place","employer":"Conferia","email":"louellachan@conferia.com","city":"Brookfield","state":"OK"}
+{"index":{"_id":"835"}}
+{"account_number":835,"balance":46558,"firstname":"Glover","lastname":"Rutledge","age":25,"gender":"F","address":"641 Royce Street","employer":"Ginkogene","email":"gloverrutledge@ginkogene.com","city":"Dixonville","state":"VA"}
+{"index":{"_id":"842"}}
+{"account_number":842,"balance":49587,"firstname":"Meagan","lastname":"Buckner","age":23,"gender":"F","address":"833 Bushwick Court","employer":"Biospan","email":"meaganbuckner@biospan.com","city":"Craig","state":"TX"}
+{"index":{"_id":"847"}}
+{"account_number":847,"balance":8652,"firstname":"Antonia","lastname":"Duncan","age":23,"gender":"M","address":"644 Stryker Street","employer":"Talae","email":"antoniaduncan@talae.com","city":"Dawn","state":"MO"}
+{"index":{"_id":"854"}}
+{"account_number":854,"balance":49795,"firstname":"Jimenez","lastname":"Barry","age":25,"gender":"F","address":"603 Cooper Street","employer":"Verton","email":"jimenezbarry@verton.com","city":"Moscow","state":"AL"}
+{"index":{"_id":"859"}}
+{"account_number":859,"balance":20734,"firstname":"Beulah","lastname":"Stuart","age":24,"gender":"F","address":"651 Albemarle Terrace","employer":"Hatology","email":"beulahstuart@hatology.com","city":"Waiohinu","state":"RI"}
+{"index":{"_id":"861"}}
+{"account_number":861,"balance":44173,"firstname":"Jaime","lastname":"Wilson","age":35,"gender":"M","address":"680 Richardson Street","employer":"Temorak","email":"jaimewilson@temorak.com","city":"Fidelis","state":"FL"}
+{"index":{"_id":"866"}}
+{"account_number":866,"balance":45565,"firstname":"Araceli","lastname":"Woodward","age":28,"gender":"M","address":"326 Meadow Street","employer":"Olympix","email":"araceliwoodward@olympix.com","city":"Dana","state":"KS"}
+{"index":{"_id":"873"}}
+{"account_number":873,"balance":43931,"firstname":"Tisha","lastname":"Cotton","age":39,"gender":"F","address":"432 Lincoln Road","employer":"Buzzmaker","email":"tishacotton@buzzmaker.com","city":"Bluetown","state":"GA"}
+{"index":{"_id":"878"}}
+{"account_number":878,"balance":49159,"firstname":"Battle","lastname":"Blackburn","age":40,"gender":"F","address":"234 Hendrix Street","employer":"Zilphur","email":"battleblackburn@zilphur.com","city":"Wanamie","state":"PA"}
+{"index":{"_id":"880"}}
+{"account_number":880,"balance":22575,"firstname":"Christian","lastname":"Myers","age":35,"gender":"M","address":"737 Crown Street","employer":"Combogen","email":"christianmyers@combogen.com","city":"Abrams","state":"OK"}
+{"index":{"_id":"885"}}
+{"account_number":885,"balance":31661,"firstname":"Valdez","lastname":"Roberson","age":40,"gender":"F","address":"227 Scholes Street","employer":"Delphide","email":"valdezroberson@delphide.com","city":"Chilton","state":"MT"}
+{"index":{"_id":"892"}}
+{"account_number":892,"balance":44974,"firstname":"Hill","lastname":"Hayes","age":29,"gender":"M","address":"721 Dooley Street","employer":"Fuelton","email":"hillhayes@fuelton.com","city":"Orason","state":"MT"}
+{"index":{"_id":"897"}}
+{"account_number":897,"balance":45973,"firstname":"Alyson","lastname":"Irwin","age":25,"gender":"M","address":"731 Poplar Street","employer":"Quizka","email":"alysonirwin@quizka.com","city":"Singer","state":"VA"}
+{"index":{"_id":"900"}}
+{"account_number":900,"balance":6124,"firstname":"Gonzalez","lastname":"Watson","age":23,"gender":"M","address":"624 Sullivan Street","employer":"Marvane","email":"gonzalezwatson@marvane.com","city":"Wikieup","state":"IL"}
+{"index":{"_id":"905"}}
+{"account_number":905,"balance":29438,"firstname":"Schultz","lastname":"Moreno","age":20,"gender":"F","address":"761 Cedar Street","employer":"Paragonia","email":"schultzmoreno@paragonia.com","city":"Glenshaw","state":"SC"}
+{"index":{"_id":"912"}}
+{"account_number":912,"balance":13675,"firstname":"Flora","lastname":"Alvarado","age":26,"gender":"M","address":"771 Vandervoort Avenue","employer":"Boilicon","email":"floraalvarado@boilicon.com","city":"Vivian","state":"ID"}
+{"index":{"_id":"917"}}
+{"account_number":917,"balance":47782,"firstname":"Parks","lastname":"Hurst","age":24,"gender":"M","address":"933 Cozine Avenue","employer":"Pyramis","email":"parkshurst@pyramis.com","city":"Lindcove","state":"GA"}
+{"index":{"_id":"924"}}
+{"account_number":924,"balance":3811,"firstname":"Hilary","lastname":"Leonard","age":24,"gender":"M","address":"235 Hegeman Avenue","employer":"Metroz","email":"hilaryleonard@metroz.com","city":"Roosevelt","state":"ME"}
+{"index":{"_id":"929"}}
+{"account_number":929,"balance":34708,"firstname":"Willie","lastname":"Hickman","age":35,"gender":"M","address":"430 Devoe Street","employer":"Apextri","email":"williehickman@apextri.com","city":"Clay","state":"MS"}
+{"index":{"_id":"931"}}
+{"account_number":931,"balance":8244,"firstname":"Ingrid","lastname":"Garcia","age":23,"gender":"F","address":"674 Indiana Place","employer":"Balooba","email":"ingridgarcia@balooba.com","city":"Interlochen","state":"AZ"}
+{"index":{"_id":"936"}}
+{"account_number":936,"balance":22430,"firstname":"Beth","lastname":"Frye","age":36,"gender":"M","address":"462 Thatford Avenue","employer":"Puria","email":"bethfrye@puria.com","city":"Hiseville","state":"LA"}
+{"index":{"_id":"943"}}
+{"account_number":943,"balance":24187,"firstname":"Wagner","lastname":"Griffin","age":23,"gender":"M","address":"489 Ellery Street","employer":"Gazak","email":"wagnergriffin@gazak.com","city":"Lorraine","state":"HI"}
+{"index":{"_id":"948"}}
+{"account_number":948,"balance":37074,"firstname":"Sargent","lastname":"Powers","age":40,"gender":"M","address":"532 Fiske Place","employer":"Accuprint","email":"sargentpowers@accuprint.com","city":"Umapine","state":"AK"}
+{"index":{"_id":"950"}}
+{"account_number":950,"balance":30916,"firstname":"Sherrie","lastname":"Patel","age":32,"gender":"F","address":"658 Langham Street","employer":"Futurize","email":"sherriepatel@futurize.com","city":"Garfield","state":"OR"}
+{"index":{"_id":"955"}}
+{"account_number":955,"balance":41621,"firstname":"Klein","lastname":"Kemp","age":33,"gender":"M","address":"370 Vanderbilt Avenue","employer":"Synkgen","email":"kleinkemp@synkgen.com","city":"Bonanza","state":"FL"}
+{"index":{"_id":"962"}}
+{"account_number":962,"balance":32096,"firstname":"Trujillo","lastname":"Wilcox","age":21,"gender":"F","address":"914 Duffield Street","employer":"Extragene","email":"trujillowilcox@extragene.com","city":"Golconda","state":"MA"}
+{"index":{"_id":"967"}}
+{"account_number":967,"balance":19161,"firstname":"Carrie","lastname":"Huffman","age":36,"gender":"F","address":"240 Sands Street","employer":"Injoy","email":"carriehuffman@injoy.com","city":"Leroy","state":"CA"}
+{"index":{"_id":"974"}}
+{"account_number":974,"balance":38082,"firstname":"Deborah","lastname":"Yang","age":26,"gender":"F","address":"463 Goodwin Place","employer":"Entogrok","email":"deborahyang@entogrok.com","city":"Herald","state":"KY"}
+{"index":{"_id":"979"}}
+{"account_number":979,"balance":43130,"firstname":"Vaughn","lastname":"Pittman","age":29,"gender":"M","address":"446 Tompkins Place","employer":"Phormula","email":"vaughnpittman@phormula.com","city":"Fingerville","state":"WI"}
+{"index":{"_id":"981"}}
+{"account_number":981,"balance":20278,"firstname":"Nolan","lastname":"Warner","age":29,"gender":"F","address":"753 Channel Avenue","employer":"Interodeo","email":"nolanwarner@interodeo.com","city":"Layhill","state":"MT"}
+{"index":{"_id":"986"}}
+{"account_number":986,"balance":35086,"firstname":"Norris","lastname":"Hubbard","age":31,"gender":"M","address":"600 Celeste Court","employer":"Printspan","email":"norrishubbard@printspan.com","city":"Cassel","state":"MI"}
+{"index":{"_id":"993"}}
+{"account_number":993,"balance":26487,"firstname":"Campos","lastname":"Olsen","age":37,"gender":"M","address":"873 Covert Street","employer":"Isbol","email":"camposolsen@isbol.com","city":"Glendale","state":"AK"}
+{"index":{"_id":"998"}}
+{"account_number":998,"balance":16869,"firstname":"Letha","lastname":"Baker","age":40,"gender":"F","address":"206 Llama Court","employer":"Dognosis","email":"lethabaker@dognosis.com","city":"Dunlo","state":"WV"}
+{"index":{"_id":"2"}}
+{"account_number":2,"balance":28838,"firstname":"Roberta","lastname":"Bender","age":22,"gender":"F","address":"560 Kingsway Place","employer":"Chillium","email":"robertabender@chillium.com","city":"Bennett","state":"LA"}
+{"index":{"_id":"7"}}
+{"account_number":7,"balance":39121,"firstname":"Levy","lastname":"Richard","age":22,"gender":"M","address":"820 Logan Street","employer":"Teraprene","email":"levyrichard@teraprene.com","city":"Shrewsbury","state":"MO"}
+{"index":{"_id":"14"}}
+{"account_number":14,"balance":20480,"firstname":"Erma","lastname":"Kane","age":39,"gender":"F","address":"661 Vista Place","employer":"Stockpost","email":"ermakane@stockpost.com","city":"Chamizal","state":"NY"}
+{"index":{"_id":"19"}}
+{"account_number":19,"balance":27894,"firstname":"Schwartz","lastname":"Buchanan","age":28,"gender":"F","address":"449 Mersereau Court","employer":"Sybixtex","email":"schwartzbuchanan@sybixtex.com","city":"Greenwich","state":"KS"}
+{"index":{"_id":"21"}}
+{"account_number":21,"balance":7004,"firstname":"Estella","lastname":"Paul","age":38,"gender":"M","address":"859 Portal Street","employer":"Zillatide","email":"estellapaul@zillatide.com","city":"Churchill","state":"WV"}
+{"index":{"_id":"26"}}
+{"account_number":26,"balance":14127,"firstname":"Lorraine","lastname":"Mccullough","age":39,"gender":"F","address":"157 Dupont Street","employer":"Zosis","email":"lorrainemccullough@zosis.com","city":"Dennard","state":"NH"}
+{"index":{"_id":"33"}}
+{"account_number":33,"balance":35439,"firstname":"Savannah","lastname":"Kirby","age":30,"gender":"F","address":"372 Malta Street","employer":"Musanpoly","email":"savannahkirby@musanpoly.com","city":"Muse","state":"AK"}
+{"index":{"_id":"38"}}
+{"account_number":38,"balance":10511,"firstname":"Erna","lastname":"Fields","age":32,"gender":"M","address":"357 Maple Street","employer":"Eweville","email":"ernafields@eweville.com","city":"Twilight","state":"MS"}
+{"index":{"_id":"40"}}
+{"account_number":40,"balance":33882,"firstname":"Pace","lastname":"Molina","age":40,"gender":"M","address":"263 Ovington Court","employer":"Cytrak","email":"pacemolina@cytrak.com","city":"Silkworth","state":"OR"}
+{"index":{"_id":"45"}}
+{"account_number":45,"balance":44478,"firstname":"Geneva","lastname":"Morin","age":21,"gender":"F","address":"357 Herkimer Street","employer":"Ezent","email":"genevamorin@ezent.com","city":"Blanco","state":"AZ"}
+{"index":{"_id":"52"}}
+{"account_number":52,"balance":46425,"firstname":"Kayla","lastname":"Bradshaw","age":31,"gender":"M","address":"449 Barlow Drive","employer":"Magnemo","email":"kaylabradshaw@magnemo.com","city":"Wawona","state":"AZ"}
+{"index":{"_id":"57"}}
+{"account_number":57,"balance":8705,"firstname":"Powell","lastname":"Herring","age":21,"gender":"M","address":"263 Merit Court","employer":"Digiprint","email":"powellherring@digiprint.com","city":"Coral","state":"MT"}
+{"index":{"_id":"64"}}
+{"account_number":64,"balance":44036,"firstname":"Miles","lastname":"Battle","age":35,"gender":"F","address":"988 Homecrest Avenue","employer":"Koffee","email":"milesbattle@koffee.com","city":"Motley","state":"ID"}
+{"index":{"_id":"69"}}
+{"account_number":69,"balance":14253,"firstname":"Desiree","lastname":"Harrison","age":24,"gender":"M","address":"694 Garland Court","employer":"Barkarama","email":"desireeharrison@barkarama.com","city":"Hackneyville","state":"GA"}
+{"index":{"_id":"71"}}
+{"account_number":71,"balance":38201,"firstname":"Sharpe","lastname":"Hoffman","age":39,"gender":"F","address":"450 Conklin Avenue","employer":"Centree","email":"sharpehoffman@centree.com","city":"Urbana","state":"WY"}
+{"index":{"_id":"76"}}
+{"account_number":76,"balance":38345,"firstname":"Claudette","lastname":"Beard","age":24,"gender":"F","address":"748 Dorset Street","employer":"Repetwire","email":"claudettebeard@repetwire.com","city":"Caln","state":"TX"}
+{"index":{"_id":"83"}}
+{"account_number":83,"balance":35928,"firstname":"Mayo","lastname":"Cleveland","age":28,"gender":"M","address":"720 Brooklyn Road","employer":"Indexia","email":"mayocleveland@indexia.com","city":"Roberts","state":"ND"}
+{"index":{"_id":"88"}}
+{"account_number":88,"balance":26418,"firstname":"Adela","lastname":"Tyler","age":21,"gender":"F","address":"737 Clove Road","employer":"Surelogic","email":"adelatyler@surelogic.com","city":"Boling","state":"SD"}
+{"index":{"_id":"90"}}
+{"account_number":90,"balance":25332,"firstname":"Herman","lastname":"Snyder","age":22,"gender":"F","address":"737 College Place","employer":"Lunchpod","email":"hermansnyder@lunchpod.com","city":"Flintville","state":"IA"}
+{"index":{"_id":"95"}}
+{"account_number":95,"balance":1650,"firstname":"Dominguez","lastname":"Le","age":20,"gender":"M","address":"539 Grace Court","employer":"Portica","email":"dominguezle@portica.com","city":"Wollochet","state":"KS"}
+{"index":{"_id":"103"}}
+{"account_number":103,"balance":11253,"firstname":"Calhoun","lastname":"Bruce","age":33,"gender":"F","address":"731 Clarkson Avenue","employer":"Automon","email":"calhounbruce@automon.com","city":"Marienthal","state":"IL"}
+{"index":{"_id":"108"}}
+{"account_number":108,"balance":19015,"firstname":"Christensen","lastname":"Weaver","age":21,"gender":"M","address":"398 Dearborn Court","employer":"Quilk","email":"christensenweaver@quilk.com","city":"Belvoir","state":"TX"}
+{"index":{"_id":"110"}}
+{"account_number":110,"balance":4850,"firstname":"Daphne","lastname":"Byrd","age":23,"gender":"F","address":"239 Conover Street","employer":"Freakin","email":"daphnebyrd@freakin.com","city":"Taft","state":"MN"}
+{"index":{"_id":"115"}}
+{"account_number":115,"balance":18750,"firstname":"Nikki","lastname":"Doyle","age":31,"gender":"F","address":"537 Clara Street","employer":"Fossiel","email":"nikkidoyle@fossiel.com","city":"Caron","state":"MS"}
+{"index":{"_id":"122"}}
+{"account_number":122,"balance":17128,"firstname":"Aurora","lastname":"Fry","age":31,"gender":"F","address":"227 Knapp Street","employer":"Makingway","email":"aurorafry@makingway.com","city":"Maybell","state":"NE"}
+{"index":{"_id":"127"}}
+{"account_number":127,"balance":48734,"firstname":"Diann","lastname":"Mclaughlin","age":33,"gender":"F","address":"340 Clermont Avenue","employer":"Enomen","email":"diannmclaughlin@enomen.com","city":"Rutherford","state":"ND"}
+{"index":{"_id":"134"}}
+{"account_number":134,"balance":33829,"firstname":"Madelyn","lastname":"Norris","age":30,"gender":"F","address":"176 Noel Avenue","employer":"Endicil","email":"madelynnorris@endicil.com","city":"Walker","state":"NE"}
+{"index":{"_id":"139"}}
+{"account_number":139,"balance":18444,"firstname":"Rios","lastname":"Todd","age":35,"gender":"F","address":"281 Georgia Avenue","employer":"Uberlux","email":"riostodd@uberlux.com","city":"Hannasville","state":"PA"}
+{"index":{"_id":"141"}}
+{"account_number":141,"balance":20790,"firstname":"Liliana","lastname":"Caldwell","age":29,"gender":"M","address":"414 Huron Street","employer":"Rubadub","email":"lilianacaldwell@rubadub.com","city":"Hiwasse","state":"OK"}
+{"index":{"_id":"146"}}
+{"account_number":146,"balance":39078,"firstname":"Lang","lastname":"Kaufman","age":32,"gender":"F","address":"626 Beverley Road","employer":"Rodeomad","email":"langkaufman@rodeomad.com","city":"Mahtowa","state":"RI"}
+{"index":{"_id":"153"}}
+{"account_number":153,"balance":32074,"firstname":"Bird","lastname":"Cochran","age":31,"gender":"F","address":"691 Bokee Court","employer":"Supremia","email":"birdcochran@supremia.com","city":"Barrelville","state":"NE"}
+{"index":{"_id":"158"}}
+{"account_number":158,"balance":9380,"firstname":"Natalie","lastname":"Mcdowell","age":27,"gender":"M","address":"953 Roder Avenue","employer":"Myopium","email":"nataliemcdowell@myopium.com","city":"Savage","state":"ND"}
+{"index":{"_id":"160"}}
+{"account_number":160,"balance":48974,"firstname":"Hull","lastname":"Cherry","age":23,"gender":"F","address":"275 Beaumont Street","employer":"Noralex","email":"hullcherry@noralex.com","city":"Whipholt","state":"WA"}
+{"index":{"_id":"165"}}
+{"account_number":165,"balance":18956,"firstname":"Sims","lastname":"Mckay","age":40,"gender":"F","address":"205 Jackson Street","employer":"Comtour","email":"simsmckay@comtour.com","city":"Tilden","state":"DC"}
+{"index":{"_id":"172"}}
+{"account_number":172,"balance":18356,"firstname":"Marie","lastname":"Whitehead","age":20,"gender":"M","address":"704 Monaco Place","employer":"Sultrax","email":"mariewhitehead@sultrax.com","city":"Dragoon","state":"IL"}
+{"index":{"_id":"177"}}
+{"account_number":177,"balance":48972,"firstname":"Harris","lastname":"Gross","age":40,"gender":"F","address":"468 Suydam Street","employer":"Kidstock","email":"harrisgross@kidstock.com","city":"Yettem","state":"KY"}
+{"index":{"_id":"184"}}
+{"account_number":184,"balance":9157,"firstname":"Cathy","lastname":"Morrison","age":27,"gender":"M","address":"882 Pine Street","employer":"Zytrek","email":"cathymorrison@zytrek.com","city":"Fedora","state":"FL"}
+{"index":{"_id":"189"}}
+{"account_number":189,"balance":20167,"firstname":"Ada","lastname":"Cortez","age":38,"gender":"F","address":"700 Forest Place","employer":"Micronaut","email":"adacortez@micronaut.com","city":"Eagletown","state":"TX"}
+{"index":{"_id":"191"}}
+{"account_number":191,"balance":26172,"firstname":"Barr","lastname":"Sharpe","age":28,"gender":"M","address":"428 Auburn Place","employer":"Ziggles","email":"barrsharpe@ziggles.com","city":"Springdale","state":"KS"}
+{"index":{"_id":"196"}}
+{"account_number":196,"balance":29931,"firstname":"Caldwell","lastname":"Daniel","age":28,"gender":"F","address":"405 Oliver Street","employer":"Furnigeer","email":"caldwelldaniel@furnigeer.com","city":"Zortman","state":"NE"}
+{"index":{"_id":"204"}}
+{"account_number":204,"balance":27714,"firstname":"Mavis","lastname":"Deleon","age":39,"gender":"F","address":"400 Waldane Court","employer":"Lotron","email":"mavisdeleon@lotron.com","city":"Stollings","state":"LA"}
+{"index":{"_id":"209"}}
+{"account_number":209,"balance":31052,"firstname":"Myers","lastname":"Noel","age":30,"gender":"F","address":"691 Alton Place","employer":"Greeker","email":"myersnoel@greeker.com","city":"Hinsdale","state":"KY"}
+{"index":{"_id":"211"}}
+{"account_number":211,"balance":21539,"firstname":"Graciela","lastname":"Vaughan","age":22,"gender":"M","address":"558 Montauk Court","employer":"Fishland","email":"gracielavaughan@fishland.com","city":"Madrid","state":"PA"}
+{"index":{"_id":"216"}}
+{"account_number":216,"balance":11422,"firstname":"Price","lastname":"Haley","age":35,"gender":"M","address":"233 Portland Avenue","employer":"Zeam","email":"pricehaley@zeam.com","city":"Titanic","state":"UT"}
+{"index":{"_id":"223"}}
+{"account_number":223,"balance":9528,"firstname":"Newton","lastname":"Fletcher","age":26,"gender":"F","address":"654 Dewitt Avenue","employer":"Assistia","email":"newtonfletcher@assistia.com","city":"Nipinnawasee","state":"AK"}
+{"index":{"_id":"228"}}
+{"account_number":228,"balance":10543,"firstname":"Rosella","lastname":"Albert","age":20,"gender":"M","address":"185 Gotham Avenue","employer":"Isoplex","email":"rosellaalbert@isoplex.com","city":"Finzel","state":"NY"}
+{"index":{"_id":"230"}}
+{"account_number":230,"balance":10829,"firstname":"Chris","lastname":"Raymond","age":28,"gender":"F","address":"464 Remsen Street","employer":"Cogentry","email":"chrisraymond@cogentry.com","city":"Bowmansville","state":"SD"}
+{"index":{"_id":"235"}}
+{"account_number":235,"balance":17729,"firstname":"Mcpherson","lastname":"Mueller","age":31,"gender":"M","address":"541 Strong Place","employer":"Tingles","email":"mcphersonmueller@tingles.com","city":"Brantleyville","state":"AR"}
+{"index":{"_id":"242"}}
+{"account_number":242,"balance":42318,"firstname":"Berger","lastname":"Roach","age":21,"gender":"M","address":"125 Wakeman Place","employer":"Ovium","email":"bergerroach@ovium.com","city":"Hessville","state":"WI"}
+{"index":{"_id":"247"}}
+{"account_number":247,"balance":45123,"firstname":"Mccormick","lastname":"Moon","age":37,"gender":"M","address":"582 Brighton Avenue","employer":"Norsup","email":"mccormickmoon@norsup.com","city":"Forestburg","state":"DE"}
+{"index":{"_id":"254"}}
+{"account_number":254,"balance":35104,"firstname":"Yang","lastname":"Dodson","age":21,"gender":"M","address":"531 Lott Street","employer":"Mondicil","email":"yangdodson@mondicil.com","city":"Enoree","state":"UT"}
+{"index":{"_id":"259"}}
+{"account_number":259,"balance":41877,"firstname":"Eleanor","lastname":"Gonzalez","age":30,"gender":"M","address":"800 Sumpter Street","employer":"Futuris","email":"eleanorgonzalez@futuris.com","city":"Jenkinsville","state":"ID"}
+{"index":{"_id":"261"}}
+{"account_number":261,"balance":39998,"firstname":"Millicent","lastname":"Pickett","age":34,"gender":"F","address":"722 Montieth Street","employer":"Gushkool","email":"millicentpickett@gushkool.com","city":"Norwood","state":"MS"}
+{"index":{"_id":"266"}}
+{"account_number":266,"balance":2777,"firstname":"Monique","lastname":"Conner","age":35,"gender":"F","address":"489 Metrotech Courtr","employer":"Flotonic","email":"moniqueconner@flotonic.com","city":"Retsof","state":"MD"}
+{"index":{"_id":"273"}}
+{"account_number":273,"balance":11181,"firstname":"Murphy","lastname":"Chandler","age":20,"gender":"F","address":"569 Bradford Street","employer":"Zilch","email":"murphychandler@zilch.com","city":"Vicksburg","state":"FL"}
+{"index":{"_id":"278"}}
+{"account_number":278,"balance":22530,"firstname":"Tamra","lastname":"Navarro","age":27,"gender":"F","address":"175 Woodruff Avenue","employer":"Norsul","email":"tamranavarro@norsul.com","city":"Glasgow","state":"VT"}
+{"index":{"_id":"280"}}
+{"account_number":280,"balance":3380,"firstname":"Vilma","lastname":"Shields","age":26,"gender":"F","address":"133 Berriman Street","employer":"Applidec","email":"vilmashields@applidec.com","city":"Adamstown","state":"ME"}
+{"index":{"_id":"285"}}
+{"account_number":285,"balance":47369,"firstname":"Hilda","lastname":"Phillips","age":28,"gender":"F","address":"618 Nixon Court","employer":"Comcur","email":"hildaphillips@comcur.com","city":"Siglerville","state":"NC"}
+{"index":{"_id":"292"}}
+{"account_number":292,"balance":26679,"firstname":"Morrow","lastname":"Greene","age":20,"gender":"F","address":"691 Nassau Street","employer":"Columella","email":"morrowgreene@columella.com","city":"Sanborn","state":"FL"}
+{"index":{"_id":"297"}}
+{"account_number":297,"balance":20508,"firstname":"Tucker","lastname":"Patrick","age":35,"gender":"F","address":"978 Whitwell Place","employer":"Valreda","email":"tuckerpatrick@valreda.com","city":"Deseret","state":"CO"}
+{"index":{"_id":"300"}}
+{"account_number":300,"balance":25654,"firstname":"Lane","lastname":"Tate","age":26,"gender":"F","address":"632 Kay Court","employer":"Genesynk","email":"lanetate@genesynk.com","city":"Lowell","state":"MO"}
+{"index":{"_id":"305"}}
+{"account_number":305,"balance":11655,"firstname":"Augusta","lastname":"Winters","age":29,"gender":"F","address":"377 Paerdegat Avenue","employer":"Vendblend","email":"augustawinters@vendblend.com","city":"Gwynn","state":"MA"}
+{"index":{"_id":"312"}}
+{"account_number":312,"balance":8511,"firstname":"Burgess","lastname":"Gentry","age":25,"gender":"F","address":"382 Bergen Court","employer":"Orbixtar","email":"burgessgentry@orbixtar.com","city":"Conestoga","state":"WI"}
+{"index":{"_id":"317"}}
+{"account_number":317,"balance":31968,"firstname":"Ruiz","lastname":"Morris","age":31,"gender":"F","address":"972 Dean Street","employer":"Apex","email":"ruizmorris@apex.com","city":"Jacksonwald","state":"WV"}
+{"index":{"_id":"324"}}
+{"account_number":324,"balance":44976,"firstname":"Gladys","lastname":"Erickson","age":22,"gender":"M","address":"250 Battery Avenue","employer":"Eternis","email":"gladyserickson@eternis.com","city":"Marne","state":"IA"}
+{"index":{"_id":"329"}}
+{"account_number":329,"balance":31138,"firstname":"Nellie","lastname":"Mercer","age":25,"gender":"M","address":"967 Ebony Court","employer":"Scenty","email":"nelliemercer@scenty.com","city":"Jardine","state":"AK"}
+{"index":{"_id":"331"}}
+{"account_number":331,"balance":46004,"firstname":"Gibson","lastname":"Potts","age":34,"gender":"F","address":"994 Dahill Road","employer":"Zensus","email":"gibsonpotts@zensus.com","city":"Frizzleburg","state":"CO"}
+{"index":{"_id":"336"}}
+{"account_number":336,"balance":40891,"firstname":"Dudley","lastname":"Avery","age":25,"gender":"M","address":"405 Powers Street","employer":"Genmom","email":"dudleyavery@genmom.com","city":"Clarksburg","state":"CO"}
+{"index":{"_id":"343"}}
+{"account_number":343,"balance":37684,"firstname":"Robbie","lastname":"Logan","age":29,"gender":"M","address":"488 Linden Boulevard","employer":"Hydrocom","email":"robbielogan@hydrocom.com","city":"Stockdale","state":"TN"}
+{"index":{"_id":"348"}}
+{"account_number":348,"balance":1360,"firstname":"Karina","lastname":"Russell","age":37,"gender":"M","address":"797 Moffat Street","employer":"Limozen","email":"karinarussell@limozen.com","city":"Riegelwood","state":"RI"}
+{"index":{"_id":"350"}}
+{"account_number":350,"balance":4267,"firstname":"Wyatt","lastname":"Wise","age":22,"gender":"F","address":"896 Bleecker Street","employer":"Rockyard","email":"wyattwise@rockyard.com","city":"Joes","state":"MS"}
+{"index":{"_id":"355"}}
+{"account_number":355,"balance":40961,"firstname":"Gregory","lastname":"Delacruz","age":38,"gender":"M","address":"876 Cortelyou Road","employer":"Oulu","email":"gregorydelacruz@oulu.com","city":"Waterloo","state":"WV"}
+{"index":{"_id":"362"}}
+{"account_number":362,"balance":14938,"firstname":"Jimmie","lastname":"Dejesus","age":26,"gender":"M","address":"351 Navy Walk","employer":"Ecolight","email":"jimmiedejesus@ecolight.com","city":"Berlin","state":"ME"}
+{"index":{"_id":"367"}}
+{"account_number":367,"balance":40458,"firstname":"Elaine","lastname":"Workman","age":20,"gender":"M","address":"188 Ridge Boulevard","employer":"Colaire","email":"elaineworkman@colaire.com","city":"Herbster","state":"AK"}
+{"index":{"_id":"374"}}
+{"account_number":374,"balance":19521,"firstname":"Blanchard","lastname":"Stein","age":30,"gender":"M","address":"313 Bartlett Street","employer":"Cujo","email":"blanchardstein@cujo.com","city":"Cascades","state":"OR"}
+{"index":{"_id":"379"}}
+{"account_number":379,"balance":12962,"firstname":"Ruthie","lastname":"Lamb","age":21,"gender":"M","address":"796 Rockaway Avenue","employer":"Incubus","email":"ruthielamb@incubus.com","city":"Hickory","state":"TX"}
+{"index":{"_id":"381"}}
+{"account_number":381,"balance":40978,"firstname":"Sophie","lastname":"Mays","age":31,"gender":"M","address":"261 Varanda Place","employer":"Uneeq","email":"sophiemays@uneeq.com","city":"Cressey","state":"AR"}
+{"index":{"_id":"386"}}
+{"account_number":386,"balance":42588,"firstname":"Wallace","lastname":"Barr","age":39,"gender":"F","address":"246 Beverly Road","employer":"Concility","email":"wallacebarr@concility.com","city":"Durham","state":"IN"}
+{"index":{"_id":"393"}}
+{"account_number":393,"balance":43936,"firstname":"William","lastname":"Kelly","age":24,"gender":"M","address":"178 Lawrence Avenue","employer":"Techtrix","email":"williamkelly@techtrix.com","city":"Orin","state":"PA"}
+{"index":{"_id":"398"}}
+{"account_number":398,"balance":8543,"firstname":"Leticia","lastname":"Duran","age":35,"gender":"F","address":"305 Senator Street","employer":"Xleen","email":"leticiaduran@xleen.com","city":"Cavalero","state":"PA"}
+{"index":{"_id":"401"}}
+{"account_number":401,"balance":29408,"firstname":"Contreras","lastname":"Randolph","age":38,"gender":"M","address":"104 Lewis Avenue","employer":"Inrt","email":"contrerasrandolph@inrt.com","city":"Chesapeake","state":"CT"}
+{"index":{"_id":"406"}}
+{"account_number":406,"balance":28127,"firstname":"Mccarthy","lastname":"Dunlap","age":28,"gender":"F","address":"684 Seacoast Terrace","employer":"Canopoly","email":"mccarthydunlap@canopoly.com","city":"Elliott","state":"NC"}
+{"index":{"_id":"413"}}
+{"account_number":413,"balance":15631,"firstname":"Pugh","lastname":"Hamilton","age":39,"gender":"F","address":"124 Euclid Avenue","employer":"Techade","email":"pughhamilton@techade.com","city":"Beaulieu","state":"CA"}
+{"index":{"_id":"418"}}
+{"account_number":418,"balance":10207,"firstname":"Reed","lastname":"Goff","age":32,"gender":"M","address":"959 Everit Street","employer":"Zillan","email":"reedgoff@zillan.com","city":"Hiko","state":"WV"}
+{"index":{"_id":"420"}}
+{"account_number":420,"balance":44699,"firstname":"Brandie","lastname":"Hayden","age":22,"gender":"M","address":"291 Ash Street","employer":"Digifad","email":"brandiehayden@digifad.com","city":"Spelter","state":"NM"}
+{"index":{"_id":"425"}}
+{"account_number":425,"balance":41308,"firstname":"Queen","lastname":"Leach","age":30,"gender":"M","address":"105 Fair Street","employer":"Magneato","email":"queenleach@magneato.com","city":"Barronett","state":"NH"}
+{"index":{"_id":"432"}}
+{"account_number":432,"balance":28969,"firstname":"Preston","lastname":"Ferguson","age":40,"gender":"F","address":"239 Greenwood Avenue","employer":"Bitendrex","email":"prestonferguson@bitendrex.com","city":"Idledale","state":"ND"}
+{"index":{"_id":"437"}}
+{"account_number":437,"balance":41225,"firstname":"Rosales","lastname":"Marquez","age":29,"gender":"M","address":"873 Ryerson Street","employer":"Ronelon","email":"rosalesmarquez@ronelon.com","city":"Allendale","state":"CA"}
+{"index":{"_id":"444"}}
+{"account_number":444,"balance":44219,"firstname":"Dolly","lastname":"Finch","age":24,"gender":"F","address":"974 Interborough Parkway","employer":"Zytrac","email":"dollyfinch@zytrac.com","city":"Vowinckel","state":"WY"}
+{"index":{"_id":"449"}}
+{"account_number":449,"balance":41950,"firstname":"Barnett","lastname":"Cantrell","age":39,"gender":"F","address":"945 Bedell Lane","employer":"Zentility","email":"barnettcantrell@zentility.com","city":"Swartzville","state":"ND"}
+{"index":{"_id":"451"}}
+{"account_number":451,"balance":31950,"firstname":"Mason","lastname":"Mcleod","age":31,"gender":"F","address":"438 Havemeyer Street","employer":"Omatom","email":"masonmcleod@omatom.com","city":"Ryderwood","state":"NE"}
+{"index":{"_id":"456"}}
+{"account_number":456,"balance":21419,"firstname":"Solis","lastname":"Kline","age":33,"gender":"M","address":"818 Ashford Street","employer":"Vetron","email":"soliskline@vetron.com","city":"Ruffin","state":"NY"}
+{"index":{"_id":"463"}}
+{"account_number":463,"balance":36672,"firstname":"Heidi","lastname":"Acosta","age":20,"gender":"F","address":"692 Kenmore Terrace","employer":"Elpro","email":"heidiacosta@elpro.com","city":"Ezel","state":"SD"}
+{"index":{"_id":"468"}}
+{"account_number":468,"balance":18400,"firstname":"Foreman","lastname":"Fowler","age":40,"gender":"M","address":"443 Jackson Court","employer":"Zillactic","email":"foremanfowler@zillactic.com","city":"Wakarusa","state":"WA"}
+{"index":{"_id":"470"}}
+{"account_number":470,"balance":20455,"firstname":"Schneider","lastname":"Hull","age":35,"gender":"M","address":"724 Apollo Street","employer":"Exospeed","email":"schneiderhull@exospeed.com","city":"Watchtower","state":"ID"}
+{"index":{"_id":"475"}}
+{"account_number":475,"balance":24427,"firstname":"Morales","lastname":"Jacobs","age":22,"gender":"F","address":"225 Desmond Court","employer":"Oronoko","email":"moralesjacobs@oronoko.com","city":"Clayville","state":"CT"}
+{"index":{"_id":"482"}}
+{"account_number":482,"balance":14834,"firstname":"Janie","lastname":"Bass","age":39,"gender":"M","address":"781 Grattan Street","employer":"Manglo","email":"janiebass@manglo.com","city":"Kenwood","state":"IA"}
+{"index":{"_id":"487"}}
+{"account_number":487,"balance":30718,"firstname":"Sawyer","lastname":"Vincent","age":26,"gender":"F","address":"238 Lancaster Avenue","employer":"Brainquil","email":"sawyervincent@brainquil.com","city":"Galesville","state":"MS"}
+{"index":{"_id":"494"}}
+{"account_number":494,"balance":3592,"firstname":"Holden","lastname":"Bowen","age":30,"gender":"M","address":"374 Elmwood Avenue","employer":"Endipine","email":"holdenbowen@endipine.com","city":"Rosine","state":"ID"}
+{"index":{"_id":"499"}}
+{"account_number":499,"balance":26060,"firstname":"Lara","lastname":"Perkins","age":26,"gender":"M","address":"703 Monroe Street","employer":"Paprikut","email":"laraperkins@paprikut.com","city":"Barstow","state":"NY"}
+{"index":{"_id":"502"}}
+{"account_number":502,"balance":31898,"firstname":"Woodard","lastname":"Bailey","age":31,"gender":"F","address":"585 Albee Square","employer":"Imperium","email":"woodardbailey@imperium.com","city":"Matheny","state":"MT"}
+{"index":{"_id":"507"}}
+{"account_number":507,"balance":27675,"firstname":"Blankenship","lastname":"Ramirez","age":31,"gender":"M","address":"630 Graham Avenue","employer":"Bytrex","email":"blankenshipramirez@bytrex.com","city":"Bancroft","state":"CT"}
+{"index":{"_id":"514"}}
+{"account_number":514,"balance":30125,"firstname":"Solomon","lastname":"Bush","age":34,"gender":"M","address":"409 Harkness Avenue","employer":"Snacktion","email":"solomonbush@snacktion.com","city":"Grayhawk","state":"TX"}
+{"index":{"_id":"519"}}
+{"account_number":519,"balance":3282,"firstname":"Lorna","lastname":"Franco","age":31,"gender":"F","address":"722 Schenck Court","employer":"Zentia","email":"lornafranco@zentia.com","city":"National","state":"FL"}
+{"index":{"_id":"521"}}
+{"account_number":521,"balance":16348,"firstname":"Josefa","lastname":"Buckley","age":34,"gender":"F","address":"848 Taylor Street","employer":"Mazuda","email":"josefabuckley@mazuda.com","city":"Saranap","state":"NM"}
+{"index":{"_id":"526"}}
+{"account_number":526,"balance":35375,"firstname":"Sweeney","lastname":"Fulton","age":33,"gender":"F","address":"550 Martense Street","employer":"Cormoran","email":"sweeneyfulton@cormoran.com","city":"Chalfant","state":"IA"}
+{"index":{"_id":"533"}}
+{"account_number":533,"balance":13761,"firstname":"Margarita","lastname":"Diaz","age":23,"gender":"M","address":"295 Tapscott Street","employer":"Zilodyne","email":"margaritadiaz@zilodyne.com","city":"Hondah","state":"ID"}
+{"index":{"_id":"538"}}
+{"account_number":538,"balance":16416,"firstname":"Koch","lastname":"Barker","age":21,"gender":"M","address":"919 Gerry Street","employer":"Xplor","email":"kochbarker@xplor.com","city":"Dixie","state":"WY"}
+{"index":{"_id":"540"}}
+{"account_number":540,"balance":40235,"firstname":"Tammy","lastname":"Wiggins","age":32,"gender":"F","address":"186 Schenectady Avenue","employer":"Speedbolt","email":"tammywiggins@speedbolt.com","city":"Salvo","state":"LA"}
+{"index":{"_id":"545"}}
+{"account_number":545,"balance":27011,"firstname":"Lena","lastname":"Lucas","age":20,"gender":"M","address":"110 Lamont Court","employer":"Kindaloo","email":"lenalucas@kindaloo.com","city":"Harleigh","state":"KY"}
+{"index":{"_id":"552"}}
+{"account_number":552,"balance":14727,"firstname":"Kate","lastname":"Estes","age":39,"gender":"M","address":"785 Willmohr Street","employer":"Rodeocean","email":"kateestes@rodeocean.com","city":"Elfrida","state":"HI"}
+{"index":{"_id":"557"}}
+{"account_number":557,"balance":3119,"firstname":"Landry","lastname":"Buck","age":20,"gender":"M","address":"558 Schweikerts Walk","employer":"Protodyne","email":"landrybuck@protodyne.com","city":"Edneyville","state":"AL"}
+{"index":{"_id":"564"}}
+{"account_number":564,"balance":43631,"firstname":"Owens","lastname":"Bowers","age":22,"gender":"M","address":"842 Congress Street","employer":"Nspire","email":"owensbowers@nspire.com","city":"Machias","state":"VA"}
+{"index":{"_id":"569"}}
+{"account_number":569,"balance":40019,"firstname":"Sherri","lastname":"Rowe","age":39,"gender":"F","address":"591 Arlington Place","employer":"Netility","email":"sherrirowe@netility.com","city":"Bridgetown","state":"SC"}
+{"index":{"_id":"571"}}
+{"account_number":571,"balance":3014,"firstname":"Ayers","lastname":"Duffy","age":28,"gender":"F","address":"721 Wortman Avenue","employer":"Aquasseur","email":"ayersduffy@aquasseur.com","city":"Tilleda","state":"MS"}
+{"index":{"_id":"576"}}
+{"account_number":576,"balance":29682,"firstname":"Helena","lastname":"Robertson","age":33,"gender":"F","address":"774 Devon Avenue","employer":"Vicon","email":"helenarobertson@vicon.com","city":"Dyckesville","state":"NV"}
+{"index":{"_id":"583"}}
+{"account_number":583,"balance":26558,"firstname":"Castro","lastname":"West","age":34,"gender":"F","address":"814 Williams Avenue","employer":"Cipromox","email":"castrowest@cipromox.com","city":"Nescatunga","state":"IL"}
+{"index":{"_id":"588"}}
+{"account_number":588,"balance":43531,"firstname":"Martina","lastname":"Collins","age":31,"gender":"M","address":"301 Anna Court","employer":"Geekwagon","email":"martinacollins@geekwagon.com","city":"Oneida","state":"VA"}
+{"index":{"_id":"590"}}
+{"account_number":590,"balance":4652,"firstname":"Ladonna","lastname":"Tucker","age":31,"gender":"F","address":"162 Kane Place","employer":"Infotrips","email":"ladonnatucker@infotrips.com","city":"Utting","state":"IA"}
+{"index":{"_id":"595"}}
+{"account_number":595,"balance":12478,"firstname":"Mccall","lastname":"Britt","age":36,"gender":"F","address":"823 Hill Street","employer":"Cablam","email":"mccallbritt@cablam.com","city":"Vernon","state":"CA"}
+{"index":{"_id":"603"}}
+{"account_number":603,"balance":28145,"firstname":"Janette","lastname":"Guzman","age":31,"gender":"F","address":"976 Kingston Avenue","employer":"Splinx","email":"janetteguzman@splinx.com","city":"Boomer","state":"NC"}
+{"index":{"_id":"608"}}
+{"account_number":608,"balance":47091,"firstname":"Carey","lastname":"Whitley","age":32,"gender":"F","address":"976 Lawrence Street","employer":"Poshome","email":"careywhitley@poshome.com","city":"Weogufka","state":"NE"}
+{"index":{"_id":"610"}}
+{"account_number":610,"balance":40571,"firstname":"Foster","lastname":"Weber","age":24,"gender":"F","address":"323 Rochester Avenue","employer":"Firewax","email":"fosterweber@firewax.com","city":"Winston","state":"NY"}
+{"index":{"_id":"615"}}
+{"account_number":615,"balance":28726,"firstname":"Delgado","lastname":"Curry","age":28,"gender":"F","address":"706 Butler Street","employer":"Zoxy","email":"delgadocurry@zoxy.com","city":"Gracey","state":"SD"}
+{"index":{"_id":"622"}}
+{"account_number":622,"balance":9661,"firstname":"Paulette","lastname":"Hartman","age":38,"gender":"M","address":"375 Emerald Street","employer":"Locazone","email":"paulettehartman@locazone.com","city":"Canterwood","state":"OH"}
+{"index":{"_id":"627"}}
+{"account_number":627,"balance":47546,"firstname":"Crawford","lastname":"Sears","age":37,"gender":"F","address":"686 Eastern Parkway","employer":"Updat","email":"crawfordsears@updat.com","city":"Bison","state":"VT"}
+{"index":{"_id":"634"}}
+{"account_number":634,"balance":29805,"firstname":"Deloris","lastname":"Levy","age":38,"gender":"M","address":"838 Foster Avenue","employer":"Homelux","email":"delorislevy@homelux.com","city":"Kempton","state":"PA"}
+{"index":{"_id":"639"}}
+{"account_number":639,"balance":28875,"firstname":"Caitlin","lastname":"Clements","age":32,"gender":"F","address":"627 Aster Court","employer":"Bunga","email":"caitlinclements@bunga.com","city":"Cetronia","state":"SC"}
+{"index":{"_id":"641"}}
+{"account_number":641,"balance":18345,"firstname":"Sheppard","lastname":"Everett","age":39,"gender":"F","address":"791 Norwood Avenue","employer":"Roboid","email":"sheppardeverett@roboid.com","city":"Selma","state":"AK"}
+{"index":{"_id":"646"}}
+{"account_number":646,"balance":15559,"firstname":"Lavonne","lastname":"Reyes","age":31,"gender":"F","address":"983 Newport Street","employer":"Parcoe","email":"lavonnereyes@parcoe.com","city":"Monument","state":"LA"}
+{"index":{"_id":"653"}}
+{"account_number":653,"balance":7606,"firstname":"Marcia","lastname":"Bennett","age":33,"gender":"F","address":"455 Bragg Street","employer":"Opticall","email":"marciabennett@opticall.com","city":"Magnolia","state":"NC"}
+{"index":{"_id":"658"}}
+{"account_number":658,"balance":10210,"firstname":"Bass","lastname":"Mcconnell","age":32,"gender":"F","address":"274 Ocean Avenue","employer":"Combot","email":"bassmcconnell@combot.com","city":"Beyerville","state":"OH"}
+{"index":{"_id":"660"}}
+{"account_number":660,"balance":46427,"firstname":"Moon","lastname":"Wood","age":33,"gender":"F","address":"916 Amersfort Place","employer":"Olucore","email":"moonwood@olucore.com","city":"Como","state":"VA"}
+{"index":{"_id":"665"}}
+{"account_number":665,"balance":15215,"firstname":"Britney","lastname":"Young","age":36,"gender":"M","address":"766 Sackman Street","employer":"Geoforma","email":"britneyyoung@geoforma.com","city":"Tuttle","state":"WI"}
+{"index":{"_id":"672"}}
+{"account_number":672,"balance":12621,"firstname":"Camille","lastname":"Munoz","age":36,"gender":"F","address":"959 Lewis Place","employer":"Vantage","email":"camillemunoz@vantage.com","city":"Whitmer","state":"IN"}
+{"index":{"_id":"677"}}
+{"account_number":677,"balance":8491,"firstname":"Snider","lastname":"Benton","age":26,"gender":"M","address":"827 Evans Street","employer":"Medicroix","email":"sniderbenton@medicroix.com","city":"Kaka","state":"UT"}
+{"index":{"_id":"684"}}
+{"account_number":684,"balance":46091,"firstname":"Warren","lastname":"Snow","age":25,"gender":"M","address":"756 Oakland Place","employer":"Bizmatic","email":"warrensnow@bizmatic.com","city":"Hatteras","state":"NE"}
+{"index":{"_id":"689"}}
+{"account_number":689,"balance":14985,"firstname":"Ines","lastname":"Chaney","age":28,"gender":"M","address":"137 Dikeman Street","employer":"Zidant","email":"ineschaney@zidant.com","city":"Nettie","state":"DC"}
+{"index":{"_id":"691"}}
+{"account_number":691,"balance":10792,"firstname":"Mclean","lastname":"Colon","age":22,"gender":"M","address":"876 Classon Avenue","employer":"Elentrix","email":"mcleancolon@elentrix.com","city":"Unionville","state":"OK"}
+{"index":{"_id":"696"}}
+{"account_number":696,"balance":17568,"firstname":"Crane","lastname":"Matthews","age":32,"gender":"F","address":"721 Gerritsen Avenue","employer":"Intradisk","email":"cranematthews@intradisk.com","city":"Brewster","state":"WV"}
+{"index":{"_id":"704"}}
+{"account_number":704,"balance":45347,"firstname":"Peters","lastname":"Kent","age":22,"gender":"F","address":"871 Independence Avenue","employer":"Extragen","email":"peterskent@extragen.com","city":"Morriston","state":"CA"}
+{"index":{"_id":"709"}}
+{"account_number":709,"balance":11015,"firstname":"Abbott","lastname":"Odom","age":29,"gender":"M","address":"893 Union Street","employer":"Jimbies","email":"abbottodom@jimbies.com","city":"Leeper","state":"NJ"}
+{"index":{"_id":"711"}}
+{"account_number":711,"balance":26939,"firstname":"Villarreal","lastname":"Horton","age":35,"gender":"F","address":"861 Creamer Street","employer":"Lexicondo","email":"villarrealhorton@lexicondo.com","city":"Lydia","state":"MS"}
+{"index":{"_id":"716"}}
+{"account_number":716,"balance":19789,"firstname":"Paul","lastname":"Mason","age":34,"gender":"F","address":"618 Nichols Avenue","employer":"Slax","email":"paulmason@slax.com","city":"Snowville","state":"OK"}
+{"index":{"_id":"723"}}
+{"account_number":723,"balance":16421,"firstname":"Nixon","lastname":"Moran","age":27,"gender":"M","address":"569 Campus Place","employer":"Cuizine","email":"nixonmoran@cuizine.com","city":"Buxton","state":"DC"}
+{"index":{"_id":"728"}}
+{"account_number":728,"balance":44818,"firstname":"Conley","lastname":"Preston","age":28,"gender":"M","address":"450 Coventry Road","employer":"Obones","email":"conleypreston@obones.com","city":"Alden","state":"CO"}
+{"index":{"_id":"730"}}
+{"account_number":730,"balance":41299,"firstname":"Moore","lastname":"Lee","age":30,"gender":"M","address":"797 Turner Place","employer":"Orbean","email":"moorelee@orbean.com","city":"Highland","state":"DE"}
+{"index":{"_id":"735"}}
+{"account_number":735,"balance":3984,"firstname":"Loraine","lastname":"Willis","age":32,"gender":"F","address":"928 Grove Street","employer":"Gadtron","email":"lorainewillis@gadtron.com","city":"Lowgap","state":"NY"}
+{"index":{"_id":"742"}}
+{"account_number":742,"balance":24765,"firstname":"Merle","lastname":"Wooten","age":26,"gender":"M","address":"317 Pooles Lane","employer":"Tropolis","email":"merlewooten@tropolis.com","city":"Bentley","state":"ND"}
+{"index":{"_id":"747"}}
+{"account_number":747,"balance":16617,"firstname":"Diaz","lastname":"Austin","age":38,"gender":"M","address":"676 Harway Avenue","employer":"Irack","email":"diazaustin@irack.com","city":"Cliff","state":"HI"}
+{"index":{"_id":"754"}}
+{"account_number":754,"balance":10779,"firstname":"Jones","lastname":"Vega","age":25,"gender":"F","address":"795 India Street","employer":"Gluid","email":"jonesvega@gluid.com","city":"Tyhee","state":"FL"}
+{"index":{"_id":"759"}}
+{"account_number":759,"balance":38007,"firstname":"Rose","lastname":"Carlson","age":27,"gender":"M","address":"987 Navy Street","employer":"Aquasure","email":"rosecarlson@aquasure.com","city":"Carlton","state":"CT"}
+{"index":{"_id":"761"}}
+{"account_number":761,"balance":7663,"firstname":"Rae","lastname":"Juarez","age":34,"gender":"F","address":"560 Gilmore Court","employer":"Entropix","email":"raejuarez@entropix.com","city":"Northchase","state":"ID"}
+{"index":{"_id":"766"}}
+{"account_number":766,"balance":21957,"firstname":"Thomas","lastname":"Gillespie","age":38,"gender":"M","address":"993 Williams Place","employer":"Octocore","email":"thomasgillespie@octocore.com","city":"Defiance","state":"MS"}
+{"index":{"_id":"773"}}
+{"account_number":773,"balance":31126,"firstname":"Liza","lastname":"Coffey","age":36,"gender":"F","address":"540 Bulwer Place","employer":"Assurity","email":"lizacoffey@assurity.com","city":"Gilgo","state":"WV"}
+{"index":{"_id":"778"}}
+{"account_number":778,"balance":46007,"firstname":"Underwood","lastname":"Wheeler","age":28,"gender":"M","address":"477 Provost Street","employer":"Decratex","email":"underwoodwheeler@decratex.com","city":"Sardis","state":"ID"}
+{"index":{"_id":"780"}}
+{"account_number":780,"balance":4682,"firstname":"Maryanne","lastname":"Hendricks","age":26,"gender":"F","address":"709 Wolcott Street","employer":"Sarasonic","email":"maryannehendricks@sarasonic.com","city":"Santel","state":"NH"}
+{"index":{"_id":"785"}}
+{"account_number":785,"balance":25078,"firstname":"Fields","lastname":"Lester","age":29,"gender":"M","address":"808 Chestnut Avenue","employer":"Visualix","email":"fieldslester@visualix.com","city":"Rowe","state":"PA"}
+{"index":{"_id":"792"}}
+{"account_number":792,"balance":13109,"firstname":"Becky","lastname":"Jimenez","age":40,"gender":"F","address":"539 Front Street","employer":"Isologia","email":"beckyjimenez@isologia.com","city":"Summertown","state":"MI"}
+{"index":{"_id":"797"}}
+{"account_number":797,"balance":6854,"firstname":"Lindsay","lastname":"Mills","age":26,"gender":"F","address":"919 Quay Street","employer":"Zoinage","email":"lindsaymills@zoinage.com","city":"Elliston","state":"VA"}
+{"index":{"_id":"800"}}
+{"account_number":800,"balance":26217,"firstname":"Candy","lastname":"Oconnor","age":28,"gender":"M","address":"200 Newel Street","employer":"Radiantix","email":"candyoconnor@radiantix.com","city":"Sandston","state":"OH"}
+{"index":{"_id":"805"}}
+{"account_number":805,"balance":18426,"firstname":"Jackson","lastname":"Sampson","age":27,"gender":"F","address":"722 Kenmore Court","employer":"Daido","email":"jacksonsampson@daido.com","city":"Bellamy","state":"ME"}
+{"index":{"_id":"812"}}
+{"account_number":812,"balance":42593,"firstname":"Graves","lastname":"Newman","age":32,"gender":"F","address":"916 Joralemon Street","employer":"Ecrater","email":"gravesnewman@ecrater.com","city":"Crown","state":"PA"}
+{"index":{"_id":"817"}}
+{"account_number":817,"balance":36582,"firstname":"Padilla","lastname":"Bauer","age":36,"gender":"F","address":"310 Cadman Plaza","employer":"Exoblue","email":"padillabauer@exoblue.com","city":"Ahwahnee","state":"MN"}
+{"index":{"_id":"824"}}
+{"account_number":824,"balance":6053,"firstname":"Dyer","lastname":"Henson","age":33,"gender":"M","address":"650 Seaview Avenue","employer":"Nitracyr","email":"dyerhenson@nitracyr.com","city":"Gibsonia","state":"KS"}
+{"index":{"_id":"829"}}
+{"account_number":829,"balance":20263,"firstname":"Althea","lastname":"Bell","age":37,"gender":"M","address":"319 Cook Street","employer":"Hyplex","email":"altheabell@hyplex.com","city":"Wadsworth","state":"DC"}
+{"index":{"_id":"831"}}
+{"account_number":831,"balance":25375,"firstname":"Wendy","lastname":"Savage","age":37,"gender":"M","address":"421 Veranda Place","employer":"Neurocell","email":"wendysavage@neurocell.com","city":"Fresno","state":"MS"}
+{"index":{"_id":"836"}}
+{"account_number":836,"balance":20797,"firstname":"Lloyd","lastname":"Lindsay","age":25,"gender":"F","address":"953 Dinsmore Place","employer":"Suretech","email":"lloydlindsay@suretech.com","city":"Conway","state":"VA"}
+{"index":{"_id":"843"}}
+{"account_number":843,"balance":15555,"firstname":"Patricia","lastname":"Barton","age":34,"gender":"F","address":"406 Seabring Street","employer":"Providco","email":"patriciabarton@providco.com","city":"Avoca","state":"RI"}
+{"index":{"_id":"848"}}
+{"account_number":848,"balance":15443,"firstname":"Carmella","lastname":"Cash","age":38,"gender":"M","address":"988 Exeter Street","employer":"Bristo","email":"carmellacash@bristo.com","city":"Northridge","state":"ID"}
+{"index":{"_id":"850"}}
+{"account_number":850,"balance":6531,"firstname":"Carlene","lastname":"Gaines","age":37,"gender":"F","address":"753 Monroe Place","employer":"Naxdis","email":"carlenegaines@naxdis.com","city":"Genoa","state":"OR"}
+{"index":{"_id":"855"}}
+{"account_number":855,"balance":40170,"firstname":"Mia","lastname":"Stevens","age":31,"gender":"F","address":"326 Driggs Avenue","employer":"Aeora","email":"miastevens@aeora.com","city":"Delwood","state":"IL"}
+{"index":{"_id":"862"}}
+{"account_number":862,"balance":38792,"firstname":"Clayton","lastname":"Golden","age":38,"gender":"F","address":"620 Regent Place","employer":"Accusage","email":"claytongolden@accusage.com","city":"Ona","state":"NC"}
+{"index":{"_id":"867"}}
+{"account_number":867,"balance":45453,"firstname":"Blanca","lastname":"Ellison","age":23,"gender":"F","address":"593 McKibben Street","employer":"Koogle","email":"blancaellison@koogle.com","city":"Frystown","state":"WY"}
+{"index":{"_id":"874"}}
+{"account_number":874,"balance":23079,"firstname":"Lynette","lastname":"Higgins","age":22,"gender":"M","address":"377 McKinley Avenue","employer":"Menbrain","email":"lynettehiggins@menbrain.com","city":"Manitou","state":"TX"}
+{"index":{"_id":"879"}}
+{"account_number":879,"balance":48332,"firstname":"Sabrina","lastname":"Lancaster","age":31,"gender":"F","address":"382 Oak Street","employer":"Webiotic","email":"sabrinalancaster@webiotic.com","city":"Lindisfarne","state":"AZ"}
+{"index":{"_id":"881"}}
+{"account_number":881,"balance":26684,"firstname":"Barnes","lastname":"Ware","age":38,"gender":"F","address":"666 Hooper Street","employer":"Norali","email":"barnesware@norali.com","city":"Cazadero","state":"GA"}
+{"index":{"_id":"886"}}
+{"account_number":886,"balance":14867,"firstname":"Willa","lastname":"Leblanc","age":38,"gender":"F","address":"773 Bergen Street","employer":"Nurali","email":"willaleblanc@nurali.com","city":"Hilltop","state":"NC"}
+{"index":{"_id":"893"}}
+{"account_number":893,"balance":42584,"firstname":"Moses","lastname":"Campos","age":38,"gender":"F","address":"991 Bevy Court","employer":"Trollery","email":"mosescampos@trollery.com","city":"Freetown","state":"AK"}
+{"index":{"_id":"898"}}
+{"account_number":898,"balance":12019,"firstname":"Lori","lastname":"Stevenson","age":29,"gender":"M","address":"910 Coles Street","employer":"Honotron","email":"loristevenson@honotron.com","city":"Shindler","state":"VT"}
+{"index":{"_id":"901"}}
+{"account_number":901,"balance":35038,"firstname":"Irma","lastname":"Dotson","age":23,"gender":"F","address":"245 Mayfair Drive","employer":"Bleeko","email":"irmadotson@bleeko.com","city":"Lodoga","state":"UT"}
+{"index":{"_id":"906"}}
+{"account_number":906,"balance":24073,"firstname":"Vicki","lastname":"Suarez","age":36,"gender":"M","address":"829 Roosevelt Place","employer":"Utara","email":"vickisuarez@utara.com","city":"Albrightsville","state":"AR"}
+{"index":{"_id":"913"}}
+{"account_number":913,"balance":47657,"firstname":"Margery","lastname":"Monroe","age":25,"gender":"M","address":"941 Fanchon Place","employer":"Exerta","email":"margerymonroe@exerta.com","city":"Bannock","state":"MD"}
+{"index":{"_id":"918"}}
+{"account_number":918,"balance":36776,"firstname":"Dianna","lastname":"Hernandez","age":25,"gender":"M","address":"499 Moultrie Street","employer":"Isologica","email":"diannahernandez@isologica.com","city":"Falconaire","state":"ID"}
+{"index":{"_id":"920"}}
+{"account_number":920,"balance":41513,"firstname":"Jerri","lastname":"Mitchell","age":26,"gender":"M","address":"831 Kent Street","employer":"Tasmania","email":"jerrimitchell@tasmania.com","city":"Cotopaxi","state":"IA"}
+{"index":{"_id":"925"}}
+{"account_number":925,"balance":18295,"firstname":"Rosario","lastname":"Jackson","age":24,"gender":"M","address":"178 Leonora Court","employer":"Progenex","email":"rosariojackson@progenex.com","city":"Rivereno","state":"DE"}
+{"index":{"_id":"932"}}
+{"account_number":932,"balance":3111,"firstname":"Summer","lastname":"Porter","age":33,"gender":"F","address":"949 Grand Avenue","employer":"Multiflex","email":"summerporter@multiflex.com","city":"Spokane","state":"OK"}
+{"index":{"_id":"937"}}
+{"account_number":937,"balance":43491,"firstname":"Selma","lastname":"Anderson","age":24,"gender":"M","address":"205 Reed Street","employer":"Dadabase","email":"selmaanderson@dadabase.com","city":"Malo","state":"AL"}
+{"index":{"_id":"944"}}
+{"account_number":944,"balance":46478,"firstname":"Donaldson","lastname":"Woodard","age":38,"gender":"F","address":"498 Laurel Avenue","employer":"Zogak","email":"donaldsonwoodard@zogak.com","city":"Hasty","state":"ID"}
+{"index":{"_id":"949"}}
+{"account_number":949,"balance":48703,"firstname":"Latasha","lastname":"Mullins","age":29,"gender":"F","address":"272 Lefferts Place","employer":"Zenolux","email":"latashamullins@zenolux.com","city":"Kieler","state":"MN"}
+{"index":{"_id":"951"}}
+{"account_number":951,"balance":36337,"firstname":"Tran","lastname":"Burris","age":25,"gender":"F","address":"561 Rutland Road","employer":"Geoform","email":"tranburris@geoform.com","city":"Longbranch","state":"IL"}
+{"index":{"_id":"956"}}
+{"account_number":956,"balance":19477,"firstname":"Randall","lastname":"Lynch","age":22,"gender":"F","address":"490 Madison Place","employer":"Cosmetex","email":"randalllynch@cosmetex.com","city":"Wells","state":"SD"}
+{"index":{"_id":"963"}}
+{"account_number":963,"balance":30461,"firstname":"Griffin","lastname":"Sheppard","age":20,"gender":"M","address":"682 Linden Street","employer":"Zanymax","email":"griffinsheppard@zanymax.com","city":"Fannett","state":"NM"}
+{"index":{"_id":"968"}}
+{"account_number":968,"balance":32371,"firstname":"Luella","lastname":"Burch","age":39,"gender":"M","address":"684 Arkansas Drive","employer":"Krag","email":"luellaburch@krag.com","city":"Brambleton","state":"SD"}
+{"index":{"_id":"970"}}
+{"account_number":970,"balance":19648,"firstname":"Forbes","lastname":"Wallace","age":28,"gender":"M","address":"990 Mill Road","employer":"Pheast","email":"forbeswallace@pheast.com","city":"Lopezo","state":"AK"}
+{"index":{"_id":"975"}}
+{"account_number":975,"balance":5239,"firstname":"Delores","lastname":"Booker","age":27,"gender":"F","address":"328 Conselyea Street","employer":"Centice","email":"deloresbooker@centice.com","city":"Williams","state":"HI"}
+{"index":{"_id":"982"}}
+{"account_number":982,"balance":16511,"firstname":"Buck","lastname":"Robinson","age":24,"gender":"M","address":"301 Melrose Street","employer":"Calcu","email":"buckrobinson@calcu.com","city":"Welch","state":"PA"}
+{"index":{"_id":"987"}}
+{"account_number":987,"balance":4072,"firstname":"Brock","lastname":"Sandoval","age":20,"gender":"F","address":"977 Gem Street","employer":"Fiberox","email":"brocksandoval@fiberox.com","city":"Celeryville","state":"NY"}
+{"index":{"_id":"994"}}
+{"account_number":994,"balance":33298,"firstname":"Madge","lastname":"Holcomb","age":31,"gender":"M","address":"612 Hawthorne Street","employer":"Escenta","email":"madgeholcomb@escenta.com","city":"Alafaya","state":"OR"}
+{"index":{"_id":"999"}}
+{"account_number":999,"balance":6087,"firstname":"Dorothy","lastname":"Barron","age":22,"gender":"F","address":"499 Laurel Avenue","employer":"Xurban","email":"dorothybarron@xurban.com","city":"Belvoir","state":"CA"}
+{"index":{"_id":"4"}}
+{"account_number":4,"balance":27658,"firstname":"Rodriquez","lastname":"Flores","age":31,"gender":"F","address":"986 Wyckoff Avenue","employer":"Tourmania","email":"rodriquezflores@tourmania.com","city":"Eastvale","state":"HI"}
+{"index":{"_id":"9"}}
+{"account_number":9,"balance":24776,"firstname":"Opal","lastname":"Meadows","age":39,"gender":"M","address":"963 Neptune Avenue","employer":"Cedward","email":"opalmeadows@cedward.com","city":"Olney","state":"OH"}
+{"index":{"_id":"11"}}
+{"account_number":11,"balance":20203,"firstname":"Jenkins","lastname":"Haney","age":20,"gender":"M","address":"740 Ferry Place","employer":"Qimonk","email":"jenkinshaney@qimonk.com","city":"Steinhatchee","state":"GA"}
+{"index":{"_id":"16"}}
+{"account_number":16,"balance":35883,"firstname":"Adrian","lastname":"Pitts","age":34,"gender":"F","address":"963 Fay Court","employer":"Combogene","email":"adrianpitts@combogene.com","city":"Remington","state":"SD"}
+{"index":{"_id":"23"}}
+{"account_number":23,"balance":42374,"firstname":"Kirsten","lastname":"Fox","age":20,"gender":"M","address":"330 Dumont Avenue","employer":"Codax","email":"kirstenfox@codax.com","city":"Walton","state":"AK"}
+{"index":{"_id":"28"}}
+{"account_number":28,"balance":42112,"firstname":"Vega","lastname":"Flynn","age":20,"gender":"M","address":"647 Hyman Court","employer":"Accupharm","email":"vegaflynn@accupharm.com","city":"Masthope","state":"OH"}
+{"index":{"_id":"30"}}
+{"account_number":30,"balance":19087,"firstname":"Lamb","lastname":"Townsend","age":26,"gender":"M","address":"169 Lyme Avenue","employer":"Geeknet","email":"lambtownsend@geeknet.com","city":"Epworth","state":"AL"}
+{"index":{"_id":"35"}}
+{"account_number":35,"balance":42039,"firstname":"Darla","lastname":"Bridges","age":27,"gender":"F","address":"315 Central Avenue","employer":"Xeronk","email":"darlabridges@xeronk.com","city":"Woodlake","state":"RI"}
+{"index":{"_id":"42"}}
+{"account_number":42,"balance":21137,"firstname":"Harding","lastname":"Hobbs","age":26,"gender":"F","address":"474 Ridgewood Place","employer":"Xth","email":"hardinghobbs@xth.com","city":"Heil","state":"ND"}
+{"index":{"_id":"47"}}
+{"account_number":47,"balance":33044,"firstname":"Georgia","lastname":"Wilkerson","age":23,"gender":"M","address":"369 Herbert Street","employer":"Endipin","email":"georgiawilkerson@endipin.com","city":"Dellview","state":"WI"}
+{"index":{"_id":"54"}}
+{"account_number":54,"balance":23406,"firstname":"Angel","lastname":"Mann","age":22,"gender":"F","address":"229 Ferris Street","employer":"Amtas","email":"angelmann@amtas.com","city":"Calverton","state":"WA"}
+{"index":{"_id":"59"}}
+{"account_number":59,"balance":37728,"firstname":"Malone","lastname":"Justice","age":37,"gender":"F","address":"721 Russell Street","employer":"Emoltra","email":"malonejustice@emoltra.com","city":"Trucksville","state":"HI"}
+{"index":{"_id":"61"}}
+{"account_number":61,"balance":6856,"firstname":"Shawn","lastname":"Baird","age":20,"gender":"M","address":"605 Monument Walk","employer":"Moltonic","email":"shawnbaird@moltonic.com","city":"Darlington","state":"MN"}
+{"index":{"_id":"66"}}
+{"account_number":66,"balance":25939,"firstname":"Franks","lastname":"Salinas","age":28,"gender":"M","address":"437 Hamilton Walk","employer":"Cowtown","email":"frankssalinas@cowtown.com","city":"Chase","state":"VT"}
+{"index":{"_id":"73"}}
+{"account_number":73,"balance":33457,"firstname":"Irene","lastname":"Stephenson","age":32,"gender":"M","address":"684 Miller Avenue","employer":"Hawkster","email":"irenestephenson@hawkster.com","city":"Levant","state":"AR"}
+{"index":{"_id":"78"}}
+{"account_number":78,"balance":48656,"firstname":"Elvira","lastname":"Patterson","age":23,"gender":"F","address":"834 Amber Street","employer":"Assistix","email":"elvirapatterson@assistix.com","city":"Dunbar","state":"TN"}
+{"index":{"_id":"80"}}
+{"account_number":80,"balance":13445,"firstname":"Lacey","lastname":"Blanchard","age":30,"gender":"F","address":"823 Himrod Street","employer":"Comdom","email":"laceyblanchard@comdom.com","city":"Matthews","state":"MO"}
+{"index":{"_id":"85"}}
+{"account_number":85,"balance":48735,"firstname":"Wilcox","lastname":"Sellers","age":20,"gender":"M","address":"212 Irving Avenue","employer":"Confrenzy","email":"wilcoxsellers@confrenzy.com","city":"Kipp","state":"MT"}
+{"index":{"_id":"92"}}
+{"account_number":92,"balance":26753,"firstname":"Gay","lastname":"Brewer","age":34,"gender":"M","address":"369 Ditmars Street","employer":"Savvy","email":"gaybrewer@savvy.com","city":"Moquino","state":"HI"}
+{"index":{"_id":"97"}}
+{"account_number":97,"balance":49671,"firstname":"Karen","lastname":"Trujillo","age":40,"gender":"F","address":"512 Cumberland Walk","employer":"Tsunamia","email":"karentrujillo@tsunamia.com","city":"Fredericktown","state":"MO"}
+{"index":{"_id":"100"}}
+{"account_number":100,"balance":29869,"firstname":"Madden","lastname":"Woods","age":32,"gender":"F","address":"696 Ryder Avenue","employer":"Slumberia","email":"maddenwoods@slumberia.com","city":"Deercroft","state":"ME"}
+{"index":{"_id":"105"}}
+{"account_number":105,"balance":29654,"firstname":"Castillo","lastname":"Dickerson","age":33,"gender":"F","address":"673 Oxford Street","employer":"Tellifly","email":"castillodickerson@tellifly.com","city":"Succasunna","state":"NY"}
+{"index":{"_id":"112"}}
+{"account_number":112,"balance":38395,"firstname":"Frederick","lastname":"Case","age":30,"gender":"F","address":"580 Lexington Avenue","employer":"Talkalot","email":"frederickcase@talkalot.com","city":"Orovada","state":"MA"}
+{"index":{"_id":"117"}}
+{"account_number":117,"balance":48831,"firstname":"Robin","lastname":"Hays","age":38,"gender":"F","address":"347 Hornell Loop","employer":"Pasturia","email":"robinhays@pasturia.com","city":"Sims","state":"WY"}
+{"index":{"_id":"124"}}
+{"account_number":124,"balance":16425,"firstname":"Fern","lastname":"Lambert","age":20,"gender":"M","address":"511 Jay Street","employer":"Furnitech","email":"fernlambert@furnitech.com","city":"Cloverdale","state":"FL"}
+{"index":{"_id":"129"}}
+{"account_number":129,"balance":42409,"firstname":"Alexandria","lastname":"Sanford","age":33,"gender":"F","address":"934 Ridgecrest Terrace","employer":"Kyagoro","email":"alexandriasanford@kyagoro.com","city":"Concho","state":"UT"}
+{"index":{"_id":"131"}}
+{"account_number":131,"balance":28030,"firstname":"Dollie","lastname":"Koch","age":22,"gender":"F","address":"287 Manhattan Avenue","employer":"Skinserve","email":"dolliekoch@skinserve.com","city":"Shasta","state":"PA"}
+{"index":{"_id":"136"}}
+{"account_number":136,"balance":45801,"firstname":"Winnie","lastname":"Holland","age":38,"gender":"M","address":"198 Mill Lane","employer":"Neteria","email":"winnieholland@neteria.com","city":"Urie","state":"IL"}
+{"index":{"_id":"143"}}
+{"account_number":143,"balance":43093,"firstname":"Cohen","lastname":"Noble","age":39,"gender":"M","address":"454 Nelson Street","employer":"Buzzworks","email":"cohennoble@buzzworks.com","city":"Norvelt","state":"CO"}
+{"index":{"_id":"148"}}
+{"account_number":148,"balance":3662,"firstname":"Annmarie","lastname":"Snider","age":34,"gender":"F","address":"857 Lafayette Walk","employer":"Edecine","email":"annmariesnider@edecine.com","city":"Hollins","state":"OH"}
+{"index":{"_id":"150"}}
+{"account_number":150,"balance":15306,"firstname":"Ortega","lastname":"Dalton","age":20,"gender":"M","address":"237 Mermaid Avenue","employer":"Rameon","email":"ortegadalton@rameon.com","city":"Maxville","state":"NH"}
+{"index":{"_id":"155"}}
+{"account_number":155,"balance":27878,"firstname":"Atkinson","lastname":"Hudson","age":39,"gender":"F","address":"434 Colin Place","employer":"Qualitern","email":"atkinsonhudson@qualitern.com","city":"Hoehne","state":"OH"}
+{"index":{"_id":"162"}}
+{"account_number":162,"balance":6302,"firstname":"Griffith","lastname":"Calderon","age":35,"gender":"M","address":"871 Vandervoort Place","employer":"Quotezart","email":"griffithcalderon@quotezart.com","city":"Barclay","state":"FL"}
+{"index":{"_id":"167"}}
+{"account_number":167,"balance":42051,"firstname":"Hampton","lastname":"Ryan","age":20,"gender":"M","address":"618 Fleet Place","employer":"Zipak","email":"hamptonryan@zipak.com","city":"Irwin","state":"KS"}
+{"index":{"_id":"174"}}
+{"account_number":174,"balance":1464,"firstname":"Gamble","lastname":"Pierce","age":23,"gender":"F","address":"650 Eagle Street","employer":"Matrixity","email":"gamblepierce@matrixity.com","city":"Abiquiu","state":"OR"}
+{"index":{"_id":"179"}}
+{"account_number":179,"balance":13265,"firstname":"Elise","lastname":"Drake","age":25,"gender":"M","address":"305 Christopher Avenue","employer":"Turnling","email":"elisedrake@turnling.com","city":"Loretto","state":"LA"}
+{"index":{"_id":"181"}}
+{"account_number":181,"balance":27983,"firstname":"Bennett","lastname":"Hampton","age":22,"gender":"F","address":"435 Billings Place","employer":"Voipa","email":"bennetthampton@voipa.com","city":"Rodman","state":"WY"}
+{"index":{"_id":"186"}}
+{"account_number":186,"balance":18373,"firstname":"Kline","lastname":"Joyce","age":32,"gender":"M","address":"285 Falmouth Street","employer":"Tetratrex","email":"klinejoyce@tetratrex.com","city":"Klondike","state":"SD"}
+{"index":{"_id":"193"}}
+{"account_number":193,"balance":13412,"firstname":"Patty","lastname":"Petty","age":34,"gender":"F","address":"251 Vermont Street","employer":"Kinetica","email":"pattypetty@kinetica.com","city":"Grantville","state":"MS"}
+{"index":{"_id":"198"}}
+{"account_number":198,"balance":19686,"firstname":"Rachael","lastname":"Sharp","age":38,"gender":"F","address":"443 Vernon Avenue","employer":"Powernet","email":"rachaelsharp@powernet.com","city":"Canoochee","state":"UT"}
+{"index":{"_id":"201"}}
+{"account_number":201,"balance":14586,"firstname":"Ronda","lastname":"Perry","age":25,"gender":"F","address":"856 Downing Street","employer":"Artiq","email":"rondaperry@artiq.com","city":"Colton","state":"WV"}
+{"index":{"_id":"206"}}
+{"account_number":206,"balance":47423,"firstname":"Kelli","lastname":"Francis","age":20,"gender":"M","address":"671 George Street","employer":"Exoswitch","email":"kellifrancis@exoswitch.com","city":"Babb","state":"NJ"}
+{"index":{"_id":"213"}}
+{"account_number":213,"balance":34172,"firstname":"Bauer","lastname":"Summers","age":27,"gender":"M","address":"257 Boynton Place","employer":"Voratak","email":"bauersummers@voratak.com","city":"Oceola","state":"NC"}
+{"index":{"_id":"218"}}
+{"account_number":218,"balance":26702,"firstname":"Garrison","lastname":"Bryan","age":24,"gender":"F","address":"478 Greenpoint Avenue","employer":"Uniworld","email":"garrisonbryan@uniworld.com","city":"Comptche","state":"WI"}
+{"index":{"_id":"220"}}
+{"account_number":220,"balance":3086,"firstname":"Tania","lastname":"Middleton","age":22,"gender":"F","address":"541 Gunther Place","employer":"Zerology","email":"taniamiddleton@zerology.com","city":"Linwood","state":"IN"}
+{"index":{"_id":"225"}}
+{"account_number":225,"balance":21949,"firstname":"Maryann","lastname":"Murphy","age":24,"gender":"F","address":"894 Bridgewater Street","employer":"Cinesanct","email":"maryannmurphy@cinesanct.com","city":"Cartwright","state":"RI"}
+{"index":{"_id":"232"}}
+{"account_number":232,"balance":11984,"firstname":"Carr","lastname":"Jensen","age":34,"gender":"F","address":"995 Micieli Place","employer":"Biohab","email":"carrjensen@biohab.com","city":"Waikele","state":"OH"}
+{"index":{"_id":"237"}}
+{"account_number":237,"balance":5603,"firstname":"Kirby","lastname":"Watkins","age":27,"gender":"F","address":"348 Blake Court","employer":"Sonique","email":"kirbywatkins@sonique.com","city":"Freelandville","state":"PA"}
+{"index":{"_id":"244"}}
+{"account_number":244,"balance":8048,"firstname":"Judith","lastname":"Riggs","age":27,"gender":"F","address":"590 Kosciusko Street","employer":"Arctiq","email":"judithriggs@arctiq.com","city":"Gorham","state":"DC"}
+{"index":{"_id":"249"}}
+{"account_number":249,"balance":16822,"firstname":"Mckinney","lastname":"Gallagher","age":38,"gender":"F","address":"939 Seigel Court","employer":"Premiant","email":"mckinneygallagher@premiant.com","city":"Catharine","state":"NH"}
+{"index":{"_id":"251"}}
+{"account_number":251,"balance":13475,"firstname":"Marks","lastname":"Graves","age":39,"gender":"F","address":"427 Lawn Court","employer":"Dentrex","email":"marksgraves@dentrex.com","city":"Waukeenah","state":"IL"}
+{"index":{"_id":"256"}}
+{"account_number":256,"balance":48318,"firstname":"Simon","lastname":"Hogan","age":31,"gender":"M","address":"789 Suydam Place","employer":"Dancerity","email":"simonhogan@dancerity.com","city":"Dargan","state":"GA"}
+{"index":{"_id":"263"}}
+{"account_number":263,"balance":12837,"firstname":"Thornton","lastname":"Meyer","age":29,"gender":"M","address":"575 Elliott Place","employer":"Peticular","email":"thorntonmeyer@peticular.com","city":"Dotsero","state":"NH"}
+{"index":{"_id":"268"}}
+{"account_number":268,"balance":20925,"firstname":"Avis","lastname":"Blackwell","age":36,"gender":"M","address":"569 Jerome Avenue","employer":"Magnina","email":"avisblackwell@magnina.com","city":"Bethany","state":"MD"}
+{"index":{"_id":"270"}}
+{"account_number":270,"balance":43951,"firstname":"Moody","lastname":"Harmon","age":39,"gender":"F","address":"233 Vanderbilt Street","employer":"Otherside","email":"moodyharmon@otherside.com","city":"Elwood","state":"MT"}
+{"index":{"_id":"275"}}
+{"account_number":275,"balance":2384,"firstname":"Reynolds","lastname":"Barnett","age":31,"gender":"M","address":"394 Stockton Street","employer":"Austex","email":"reynoldsbarnett@austex.com","city":"Grandview","state":"MS"}
+{"index":{"_id":"282"}}
+{"account_number":282,"balance":38540,"firstname":"Gay","lastname":"Schultz","age":25,"gender":"F","address":"805 Claver Place","employer":"Handshake","email":"gayschultz@handshake.com","city":"Tampico","state":"MA"}
+{"index":{"_id":"287"}}
+{"account_number":287,"balance":10845,"firstname":"Valerie","lastname":"Lang","age":35,"gender":"F","address":"423 Midwood Street","employer":"Quarx","email":"valerielang@quarx.com","city":"Cannondale","state":"VT"}
+{"index":{"_id":"294"}}
+{"account_number":294,"balance":29582,"firstname":"Pitts","lastname":"Haynes","age":26,"gender":"M","address":"901 Broome Street","employer":"Aquazure","email":"pittshaynes@aquazure.com","city":"Turah","state":"SD"}
+{"index":{"_id":"299"}}
+{"account_number":299,"balance":40825,"firstname":"Angela","lastname":"Talley","age":36,"gender":"F","address":"822 Bills Place","employer":"Remold","email":"angelatalley@remold.com","city":"Bethpage","state":"DC"}
+{"index":{"_id":"302"}}
+{"account_number":302,"balance":11298,"firstname":"Isabella","lastname":"Hewitt","age":40,"gender":"M","address":"455 Bedford Avenue","employer":"Cincyr","email":"isabellahewitt@cincyr.com","city":"Blanford","state":"IN"}
+{"index":{"_id":"307"}}
+{"account_number":307,"balance":43355,"firstname":"Enid","lastname":"Ashley","age":23,"gender":"M","address":"412 Emerson Place","employer":"Avenetro","email":"enidashley@avenetro.com","city":"Catherine","state":"WI"}
+{"index":{"_id":"314"}}
+{"account_number":314,"balance":5848,"firstname":"Norton","lastname":"Norton","age":35,"gender":"M","address":"252 Ditmas Avenue","employer":"Talkola","email":"nortonnorton@talkola.com","city":"Veyo","state":"SC"}
+{"index":{"_id":"319"}}
+{"account_number":319,"balance":15430,"firstname":"Ferrell","lastname":"Mckinney","age":36,"gender":"M","address":"874 Cranberry Street","employer":"Portaline","email":"ferrellmckinney@portaline.com","city":"Rose","state":"WV"}
+{"index":{"_id":"321"}}
+{"account_number":321,"balance":43370,"firstname":"Marta","lastname":"Larsen","age":35,"gender":"M","address":"617 Williams Court","employer":"Manufact","email":"martalarsen@manufact.com","city":"Sisquoc","state":"MA"}
+{"index":{"_id":"326"}}
+{"account_number":326,"balance":9692,"firstname":"Pearl","lastname":"Reese","age":30,"gender":"F","address":"451 Colonial Court","employer":"Accruex","email":"pearlreese@accruex.com","city":"Westmoreland","state":"MD"}
+{"index":{"_id":"333"}}
+{"account_number":333,"balance":22778,"firstname":"Trudy","lastname":"Sweet","age":27,"gender":"F","address":"881 Kiely Place","employer":"Acumentor","email":"trudysweet@acumentor.com","city":"Kent","state":"IA"}
+{"index":{"_id":"338"}}
+{"account_number":338,"balance":6969,"firstname":"Pierce","lastname":"Lawrence","age":35,"gender":"M","address":"318 Gallatin Place","employer":"Lunchpad","email":"piercelawrence@lunchpad.com","city":"Iola","state":"MD"}
+{"index":{"_id":"340"}}
+{"account_number":340,"balance":42072,"firstname":"Juarez","lastname":"Gutierrez","age":40,"gender":"F","address":"802 Seba Avenue","employer":"Billmed","email":"juarezgutierrez@billmed.com","city":"Malott","state":"OH"}
+{"index":{"_id":"345"}}
+{"account_number":345,"balance":9812,"firstname":"Parker","lastname":"Hines","age":38,"gender":"M","address":"715 Mill Avenue","employer":"Baluba","email":"parkerhines@baluba.com","city":"Blackgum","state":"KY"}
+{"index":{"_id":"352"}}
+{"account_number":352,"balance":20290,"firstname":"Kendra","lastname":"Mcintosh","age":31,"gender":"F","address":"963 Wolf Place","employer":"Orboid","email":"kendramcintosh@orboid.com","city":"Bladensburg","state":"AK"}
+{"index":{"_id":"357"}}
+{"account_number":357,"balance":15102,"firstname":"Adele","lastname":"Carroll","age":39,"gender":"F","address":"381 Arion Place","employer":"Aquafire","email":"adelecarroll@aquafire.com","city":"Springville","state":"RI"}
+{"index":{"_id":"364"}}
+{"account_number":364,"balance":35247,"firstname":"Felicia","lastname":"Merrill","age":40,"gender":"F","address":"229 Branton Street","employer":"Prosely","email":"feliciamerrill@prosely.com","city":"Dola","state":"MA"}
+{"index":{"_id":"369"}}
+{"account_number":369,"balance":17047,"firstname":"Mcfadden","lastname":"Guy","age":28,"gender":"F","address":"445 Lott Avenue","employer":"Kangle","email":"mcfaddenguy@kangle.com","city":"Greenbackville","state":"DE"}
+{"index":{"_id":"371"}}
+{"account_number":371,"balance":19751,"firstname":"Barker","lastname":"Allen","age":32,"gender":"F","address":"295 Wallabout Street","employer":"Nexgene","email":"barkerallen@nexgene.com","city":"Nanafalia","state":"NE"}
+{"index":{"_id":"376"}}
+{"account_number":376,"balance":44407,"firstname":"Mcmillan","lastname":"Dunn","age":21,"gender":"F","address":"771 Dorchester Road","employer":"Eargo","email":"mcmillandunn@eargo.com","city":"Yogaville","state":"RI"}
+{"index":{"_id":"383"}}
+{"account_number":383,"balance":48889,"firstname":"Knox","lastname":"Larson","age":28,"gender":"F","address":"962 Bartlett Place","employer":"Bostonic","email":"knoxlarson@bostonic.com","city":"Smeltertown","state":"TX"}
+{"index":{"_id":"388"}}
+{"account_number":388,"balance":9606,"firstname":"Julianne","lastname":"Nicholson","age":26,"gender":"F","address":"338 Crescent Street","employer":"Viasia","email":"juliannenicholson@viasia.com","city":"Alleghenyville","state":"MO"}
+{"index":{"_id":"390"}}
+{"account_number":390,"balance":7464,"firstname":"Ramona","lastname":"Roy","age":32,"gender":"M","address":"135 Banner Avenue","employer":"Deminimum","email":"ramonaroy@deminimum.com","city":"Dodge","state":"ID"}
+{"index":{"_id":"395"}}
+{"account_number":395,"balance":18679,"firstname":"Juliet","lastname":"Whitaker","age":31,"gender":"M","address":"128 Remsen Avenue","employer":"Toyletry","email":"julietwhitaker@toyletry.com","city":"Yonah","state":"LA"}
+{"index":{"_id":"403"}}
+{"account_number":403,"balance":18833,"firstname":"Williamson","lastname":"Horn","age":32,"gender":"M","address":"223 Strickland Avenue","employer":"Nimon","email":"williamsonhorn@nimon.com","city":"Bawcomville","state":"NJ"}
+{"index":{"_id":"408"}}
+{"account_number":408,"balance":34666,"firstname":"Lidia","lastname":"Guerrero","age":30,"gender":"M","address":"254 Stratford Road","employer":"Snowpoke","email":"lidiaguerrero@snowpoke.com","city":"Fairlee","state":"LA"}
+{"index":{"_id":"410"}}
+{"account_number":410,"balance":31200,"firstname":"Fox","lastname":"Cardenas","age":39,"gender":"M","address":"987 Monitor Street","employer":"Corpulse","email":"foxcardenas@corpulse.com","city":"Southview","state":"NE"}
+{"index":{"_id":"415"}}
+{"account_number":415,"balance":19449,"firstname":"Martinez","lastname":"Benson","age":36,"gender":"M","address":"172 Berkeley Place","employer":"Enersol","email":"martinezbenson@enersol.com","city":"Chumuckla","state":"AL"}
+{"index":{"_id":"422"}}
+{"account_number":422,"balance":40162,"firstname":"Brigitte","lastname":"Scott","age":26,"gender":"M","address":"662 Vermont Court","employer":"Waretel","email":"brigittescott@waretel.com","city":"Elrama","state":"VA"}
+{"index":{"_id":"427"}}
+{"account_number":427,"balance":1463,"firstname":"Rebekah","lastname":"Garrison","age":36,"gender":"F","address":"837 Hampton Avenue","employer":"Niquent","email":"rebekahgarrison@niquent.com","city":"Zarephath","state":"NY"}
+{"index":{"_id":"434"}}
+{"account_number":434,"balance":11329,"firstname":"Christa","lastname":"Huff","age":25,"gender":"M","address":"454 Oriental Boulevard","employer":"Earthpure","email":"christahuff@earthpure.com","city":"Stevens","state":"DC"}
+{"index":{"_id":"439"}}
+{"account_number":439,"balance":22752,"firstname":"Lula","lastname":"Williams","age":35,"gender":"M","address":"630 Furman Avenue","employer":"Vinch","email":"lulawilliams@vinch.com","city":"Newcastle","state":"ME"}
+{"index":{"_id":"441"}}
+{"account_number":441,"balance":47947,"firstname":"Dickson","lastname":"Mcgee","age":29,"gender":"M","address":"478 Knight Court","employer":"Gogol","email":"dicksonmcgee@gogol.com","city":"Laurelton","state":"AR"}
+{"index":{"_id":"446"}}
+{"account_number":446,"balance":23071,"firstname":"Lolita","lastname":"Fleming","age":32,"gender":"F","address":"918 Bridge Street","employer":"Vidto","email":"lolitafleming@vidto.com","city":"Brownlee","state":"HI"}
+{"index":{"_id":"453"}}
+{"account_number":453,"balance":21520,"firstname":"Hood","lastname":"Powell","age":24,"gender":"F","address":"479 Brevoort Place","employer":"Vortexaco","email":"hoodpowell@vortexaco.com","city":"Alderpoint","state":"CT"}
+{"index":{"_id":"458"}}
+{"account_number":458,"balance":8865,"firstname":"Aida","lastname":"Wolf","age":21,"gender":"F","address":"403 Thames Street","employer":"Isis","email":"aidawolf@isis.com","city":"Bordelonville","state":"ME"}
+{"index":{"_id":"460"}}
+{"account_number":460,"balance":37734,"firstname":"Aguirre","lastname":"White","age":21,"gender":"F","address":"190 Crooke Avenue","employer":"Unq","email":"aguirrewhite@unq.com","city":"Albany","state":"NJ"}
+{"index":{"_id":"465"}}
+{"account_number":465,"balance":10681,"firstname":"Pearlie","lastname":"Holman","age":29,"gender":"M","address":"916 Evergreen Avenue","employer":"Hometown","email":"pearlieholman@hometown.com","city":"Needmore","state":"UT"}
+{"index":{"_id":"472"}}
+{"account_number":472,"balance":25571,"firstname":"Lee","lastname":"Long","age":32,"gender":"F","address":"288 Mill Street","employer":"Comverges","email":"leelong@comverges.com","city":"Movico","state":"MT"}
+{"index":{"_id":"477"}}
+{"account_number":477,"balance":25892,"firstname":"Holcomb","lastname":"Cobb","age":40,"gender":"M","address":"369 Marconi Place","employer":"Steeltab","email":"holcombcobb@steeltab.com","city":"Byrnedale","state":"CA"}
+{"index":{"_id":"484"}}
+{"account_number":484,"balance":3274,"firstname":"Staci","lastname":"Melendez","age":35,"gender":"F","address":"751 Otsego Street","employer":"Namebox","email":"stacimelendez@namebox.com","city":"Harborton","state":"NV"}
+{"index":{"_id":"489"}}
+{"account_number":489,"balance":7879,"firstname":"Garrett","lastname":"Langley","age":36,"gender":"M","address":"331 Bowne Street","employer":"Zillidium","email":"garrettlangley@zillidium.com","city":"Riviera","state":"LA"}
+{"index":{"_id":"491"}}
+{"account_number":491,"balance":42942,"firstname":"Teresa","lastname":"Owen","age":24,"gender":"F","address":"713 Canton Court","employer":"Plasmos","email":"teresaowen@plasmos.com","city":"Bartonsville","state":"NH"}
+{"index":{"_id":"496"}}
+{"account_number":496,"balance":14869,"firstname":"Alison","lastname":"Conrad","age":35,"gender":"F","address":"347 Varet Street","employer":"Perkle","email":"alisonconrad@perkle.com","city":"Cliffside","state":"OH"}
+{"index":{"_id":"504"}}
+{"account_number":504,"balance":49205,"firstname":"Shanna","lastname":"Chambers","age":23,"gender":"M","address":"220 Beard Street","employer":"Corporana","email":"shannachambers@corporana.com","city":"Cashtown","state":"AZ"}
+{"index":{"_id":"509"}}
+{"account_number":509,"balance":34754,"firstname":"Durham","lastname":"Pacheco","age":40,"gender":"M","address":"129 Plymouth Street","employer":"Datacator","email":"durhampacheco@datacator.com","city":"Loveland","state":"NC"}
+{"index":{"_id":"511"}}
+{"account_number":511,"balance":40908,"firstname":"Elba","lastname":"Grant","age":24,"gender":"F","address":"157 Bijou Avenue","employer":"Dognost","email":"elbagrant@dognost.com","city":"Coyote","state":"MT"}
+{"index":{"_id":"516"}}
+{"account_number":516,"balance":44940,"firstname":"Roy","lastname":"Smith","age":37,"gender":"M","address":"770 Cherry Street","employer":"Parleynet","email":"roysmith@parleynet.com","city":"Carrsville","state":"RI"}
+{"index":{"_id":"523"}}
+{"account_number":523,"balance":28729,"firstname":"Amalia","lastname":"Benjamin","age":40,"gender":"F","address":"173 Bushwick Place","employer":"Sentia","email":"amaliabenjamin@sentia.com","city":"Jacumba","state":"OK"}
+{"index":{"_id":"528"}}
+{"account_number":528,"balance":4071,"firstname":"Thompson","lastname":"Hoover","age":27,"gender":"F","address":"580 Garden Street","employer":"Portalis","email":"thompsonhoover@portalis.com","city":"Knowlton","state":"AL"}
+{"index":{"_id":"530"}}
+{"account_number":530,"balance":8840,"firstname":"Kathrine","lastname":"Evans","age":37,"gender":"M","address":"422 Division Place","employer":"Spherix","email":"kathrineevans@spherix.com","city":"Biddle","state":"CO"}
+{"index":{"_id":"535"}}
+{"account_number":535,"balance":8715,"firstname":"Fry","lastname":"George","age":34,"gender":"M","address":"722 Green Street","employer":"Ewaves","email":"frygeorge@ewaves.com","city":"Kenmar","state":"DE"}
+{"index":{"_id":"542"}}
+{"account_number":542,"balance":23285,"firstname":"Michelle","lastname":"Mayo","age":35,"gender":"M","address":"657 Caton Place","employer":"Biflex","email":"michellemayo@biflex.com","city":"Beaverdale","state":"WY"}
+{"index":{"_id":"547"}}
+{"account_number":547,"balance":12870,"firstname":"Eaton","lastname":"Rios","age":32,"gender":"M","address":"744 Withers Street","employer":"Podunk","email":"eatonrios@podunk.com","city":"Chelsea","state":"IA"}
+{"index":{"_id":"554"}}
+{"account_number":554,"balance":33163,"firstname":"Townsend","lastname":"Atkins","age":39,"gender":"M","address":"566 Ira Court","employer":"Acruex","email":"townsendatkins@acruex.com","city":"Valle","state":"IA"}
+{"index":{"_id":"559"}}
+{"account_number":559,"balance":11450,"firstname":"Tonia","lastname":"Schmidt","age":38,"gender":"F","address":"508 Sheffield Avenue","employer":"Extro","email":"toniaschmidt@extro.com","city":"Newry","state":"CT"}
+{"index":{"_id":"561"}}
+{"account_number":561,"balance":12370,"firstname":"Sellers","lastname":"Davis","age":30,"gender":"M","address":"860 Madoc Avenue","employer":"Isodrive","email":"sellersdavis@isodrive.com","city":"Trail","state":"KS"}
+{"index":{"_id":"566"}}
+{"account_number":566,"balance":6183,"firstname":"Cox","lastname":"Roman","age":37,"gender":"M","address":"349 Winthrop Street","employer":"Medcom","email":"coxroman@medcom.com","city":"Rosewood","state":"WY"}
+{"index":{"_id":"573"}}
+{"account_number":573,"balance":32171,"firstname":"Callie","lastname":"Castaneda","age":36,"gender":"M","address":"799 Scott Avenue","employer":"Earthwax","email":"calliecastaneda@earthwax.com","city":"Marshall","state":"NH"}
+{"index":{"_id":"578"}}
+{"account_number":578,"balance":34259,"firstname":"Holmes","lastname":"Mcknight","age":37,"gender":"M","address":"969 Metropolitan Avenue","employer":"Cubicide","email":"holmesmcknight@cubicide.com","city":"Aguila","state":"PA"}
+{"index":{"_id":"580"}}
+{"account_number":580,"balance":13716,"firstname":"Mcmahon","lastname":"York","age":34,"gender":"M","address":"475 Beacon Court","employer":"Zillar","email":"mcmahonyork@zillar.com","city":"Farmington","state":"MO"}
+{"index":{"_id":"585"}}
+{"account_number":585,"balance":26745,"firstname":"Nieves","lastname":"Nolan","age":32,"gender":"M","address":"115 Seagate Terrace","employer":"Jumpstack","email":"nievesnolan@jumpstack.com","city":"Eastmont","state":"UT"}
+{"index":{"_id":"592"}}
+{"account_number":592,"balance":32968,"firstname":"Head","lastname":"Webster","age":36,"gender":"F","address":"987 Lefferts Avenue","employer":"Empirica","email":"headwebster@empirica.com","city":"Rockingham","state":"TN"}
+{"index":{"_id":"597"}}
+{"account_number":597,"balance":11246,"firstname":"Penny","lastname":"Knowles","age":33,"gender":"M","address":"139 Forbell Street","employer":"Ersum","email":"pennyknowles@ersum.com","city":"Vallonia","state":"IA"}
+{"index":{"_id":"600"}}
+{"account_number":600,"balance":10336,"firstname":"Simmons","lastname":"Byers","age":37,"gender":"M","address":"250 Dictum Court","employer":"Qualitex","email":"simmonsbyers@qualitex.com","city":"Wanship","state":"OH"}
+{"index":{"_id":"605"}}
+{"account_number":605,"balance":38427,"firstname":"Mcclain","lastname":"Manning","age":24,"gender":"M","address":"832 Leonard Street","employer":"Qiao","email":"mcclainmanning@qiao.com","city":"Calvary","state":"TX"}
+{"index":{"_id":"612"}}
+{"account_number":612,"balance":11868,"firstname":"Dunn","lastname":"Cameron","age":32,"gender":"F","address":"156 Lorimer Street","employer":"Isonus","email":"dunncameron@isonus.com","city":"Virgie","state":"ND"}
+{"index":{"_id":"617"}}
+{"account_number":617,"balance":35445,"firstname":"Kitty","lastname":"Cooley","age":22,"gender":"M","address":"788 Seagate Avenue","employer":"Ultrimax","email":"kittycooley@ultrimax.com","city":"Clarktown","state":"MD"}
+{"index":{"_id":"624"}}
+{"account_number":624,"balance":27538,"firstname":"Roxanne","lastname":"Franklin","age":39,"gender":"F","address":"299 Woodrow Court","employer":"Silodyne","email":"roxannefranklin@silodyne.com","city":"Roulette","state":"VA"}
+{"index":{"_id":"629"}}
+{"account_number":629,"balance":32987,"firstname":"Mcclure","lastname":"Rodgers","age":26,"gender":"M","address":"806 Pierrepont Place","employer":"Elita","email":"mcclurerodgers@elita.com","city":"Brownsville","state":"MI"}
+{"index":{"_id":"631"}}
+{"account_number":631,"balance":21657,"firstname":"Corrine","lastname":"Barber","age":32,"gender":"F","address":"447 Hunts Lane","employer":"Quarmony","email":"corrinebarber@quarmony.com","city":"Wyano","state":"IL"}
+{"index":{"_id":"636"}}
+{"account_number":636,"balance":8036,"firstname":"Agnes","lastname":"Hooper","age":25,"gender":"M","address":"865 Hanson Place","employer":"Digial","email":"agneshooper@digial.com","city":"Sperryville","state":"OK"}
+{"index":{"_id":"643"}}
+{"account_number":643,"balance":8057,"firstname":"Hendricks","lastname":"Stokes","age":23,"gender":"F","address":"142 Barbey Street","employer":"Remotion","email":"hendricksstokes@remotion.com","city":"Lewis","state":"MA"}
+{"index":{"_id":"648"}}
+{"account_number":648,"balance":11506,"firstname":"Terry","lastname":"Montgomery","age":21,"gender":"F","address":"115 Franklin Avenue","employer":"Enervate","email":"terrymontgomery@enervate.com","city":"Bascom","state":"MA"}
+{"index":{"_id":"650"}}
+{"account_number":650,"balance":18091,"firstname":"Benton","lastname":"Knight","age":28,"gender":"F","address":"850 Aitken Place","employer":"Pholio","email":"bentonknight@pholio.com","city":"Cobbtown","state":"AL"}
+{"index":{"_id":"655"}}
+{"account_number":655,"balance":22912,"firstname":"Eula","lastname":"Taylor","age":30,"gender":"M","address":"520 Orient Avenue","employer":"Miracula","email":"eulataylor@miracula.com","city":"Wacissa","state":"IN"}
+{"index":{"_id":"662"}}
+{"account_number":662,"balance":10138,"firstname":"Daisy","lastname":"Burnett","age":33,"gender":"M","address":"114 Norman Avenue","employer":"Liquicom","email":"daisyburnett@liquicom.com","city":"Grahamtown","state":"MD"}
+{"index":{"_id":"667"}}
+{"account_number":667,"balance":22559,"firstname":"Juliana","lastname":"Chase","age":32,"gender":"M","address":"496 Coleridge Street","employer":"Comtract","email":"julianachase@comtract.com","city":"Wilsonia","state":"NJ"}
+{"index":{"_id":"674"}}
+{"account_number":674,"balance":36038,"firstname":"Watts","lastname":"Shannon","age":22,"gender":"F","address":"600 Story Street","employer":"Joviold","email":"wattsshannon@joviold.com","city":"Fairhaven","state":"ID"}
+{"index":{"_id":"679"}}
+{"account_number":679,"balance":20149,"firstname":"Henrietta","lastname":"Bonner","age":33,"gender":"M","address":"461 Bond Street","employer":"Geekol","email":"henriettabonner@geekol.com","city":"Richville","state":"WA"}
+{"index":{"_id":"681"}}
+{"account_number":681,"balance":34244,"firstname":"Velazquez","lastname":"Wolfe","age":33,"gender":"M","address":"773 Eckford Street","employer":"Zisis","email":"velazquezwolfe@zisis.com","city":"Smock","state":"ME"}
+{"index":{"_id":"686"}}
+{"account_number":686,"balance":10116,"firstname":"Decker","lastname":"Mcclure","age":30,"gender":"F","address":"236 Commerce Street","employer":"Everest","email":"deckermcclure@everest.com","city":"Gibbsville","state":"TN"}
+{"index":{"_id":"693"}}
+{"account_number":693,"balance":31233,"firstname":"Tabatha","lastname":"Zimmerman","age":30,"gender":"F","address":"284 Emmons Avenue","employer":"Pushcart","email":"tabathazimmerman@pushcart.com","city":"Esmont","state":"NC"}
+{"index":{"_id":"698"}}
+{"account_number":698,"balance":14965,"firstname":"Baker","lastname":"Armstrong","age":36,"gender":"F","address":"796 Tehama Street","employer":"Nurplex","email":"bakerarmstrong@nurplex.com","city":"Starks","state":"UT"}
+{"index":{"_id":"701"}}
+{"account_number":701,"balance":23772,"firstname":"Gardner","lastname":"Griffith","age":27,"gender":"M","address":"187 Moore Place","employer":"Vertide","email":"gardnergriffith@vertide.com","city":"Coventry","state":"NV"}
+{"index":{"_id":"706"}}
+{"account_number":706,"balance":5282,"firstname":"Eliza","lastname":"Potter","age":39,"gender":"M","address":"945 Dunham Place","employer":"Playce","email":"elizapotter@playce.com","city":"Woodruff","state":"AK"}
+{"index":{"_id":"713"}}
+{"account_number":713,"balance":20054,"firstname":"Iris","lastname":"Mcguire","age":21,"gender":"F","address":"508 Benson Avenue","employer":"Duflex","email":"irismcguire@duflex.com","city":"Hillsboro","state":"MO"}
+{"index":{"_id":"718"}}
+{"account_number":718,"balance":13876,"firstname":"Hickman","lastname":"Dillard","age":22,"gender":"F","address":"132 Etna Street","employer":"Genmy","email":"hickmandillard@genmy.com","city":"Curtice","state":"NV"}
+{"index":{"_id":"720"}}
+{"account_number":720,"balance":31356,"firstname":"Ruth","lastname":"Vance","age":32,"gender":"F","address":"229 Adams Street","employer":"Zilidium","email":"ruthvance@zilidium.com","city":"Allison","state":"IA"}
+{"index":{"_id":"725"}}
+{"account_number":725,"balance":14677,"firstname":"Reeves","lastname":"Tillman","age":26,"gender":"M","address":"674 Ivan Court","employer":"Cemention","email":"reevestillman@cemention.com","city":"Navarre","state":"MA"}
+{"index":{"_id":"732"}}
+{"account_number":732,"balance":38445,"firstname":"Delia","lastname":"Cruz","age":37,"gender":"F","address":"870 Cheever Place","employer":"Multron","email":"deliacruz@multron.com","city":"Cresaptown","state":"NH"}
+{"index":{"_id":"737"}}
+{"account_number":737,"balance":40431,"firstname":"Sampson","lastname":"Yates","age":23,"gender":"F","address":"214 Cox Place","employer":"Signidyne","email":"sampsonyates@signidyne.com","city":"Brazos","state":"GA"}
+{"index":{"_id":"744"}}
+{"account_number":744,"balance":8690,"firstname":"Bernard","lastname":"Martinez","age":21,"gender":"M","address":"148 Dunne Place","employer":"Dragbot","email":"bernardmartinez@dragbot.com","city":"Moraida","state":"MN"}
+{"index":{"_id":"749"}}
+{"account_number":749,"balance":1249,"firstname":"Rush","lastname":"Boyle","age":36,"gender":"M","address":"310 Argyle Road","employer":"Sportan","email":"rushboyle@sportan.com","city":"Brady","state":"WA"}
+{"index":{"_id":"751"}}
+{"account_number":751,"balance":49252,"firstname":"Patrick","lastname":"Osborne","age":23,"gender":"M","address":"915 Prospect Avenue","employer":"Gynko","email":"patrickosborne@gynko.com","city":"Takilma","state":"MO"}
+{"index":{"_id":"756"}}
+{"account_number":756,"balance":40006,"firstname":"Jasmine","lastname":"Howell","age":32,"gender":"M","address":"605 Elliott Walk","employer":"Ecratic","email":"jasminehowell@ecratic.com","city":"Harrodsburg","state":"OH"}
+{"index":{"_id":"763"}}
+{"account_number":763,"balance":12091,"firstname":"Liz","lastname":"Bentley","age":22,"gender":"F","address":"933 Debevoise Avenue","employer":"Nipaz","email":"lizbentley@nipaz.com","city":"Glenville","state":"NJ"}
+{"index":{"_id":"768"}}
+{"account_number":768,"balance":2213,"firstname":"Sondra","lastname":"Soto","age":21,"gender":"M","address":"625 Colonial Road","employer":"Navir","email":"sondrasoto@navir.com","city":"Benson","state":"VA"}
+{"index":{"_id":"770"}}
+{"account_number":770,"balance":39505,"firstname":"Joann","lastname":"Crane","age":26,"gender":"M","address":"798 Farragut Place","employer":"Lingoage","email":"joanncrane@lingoage.com","city":"Kirk","state":"MA"}
+{"index":{"_id":"775"}}
+{"account_number":775,"balance":27943,"firstname":"Wilson","lastname":"Merritt","age":33,"gender":"F","address":"288 Thornton Street","employer":"Geeky","email":"wilsonmerritt@geeky.com","city":"Holtville","state":"HI"}
+{"index":{"_id":"782"}}
+{"account_number":782,"balance":3960,"firstname":"Maldonado","lastname":"Craig","age":36,"gender":"F","address":"345 Myrtle Avenue","employer":"Zilencio","email":"maldonadocraig@zilencio.com","city":"Yukon","state":"ID"}
+{"index":{"_id":"787"}}
+{"account_number":787,"balance":11876,"firstname":"Harper","lastname":"Wynn","age":21,"gender":"F","address":"139 Oceanic Avenue","employer":"Interfind","email":"harperwynn@interfind.com","city":"Gerber","state":"ND"}
+{"index":{"_id":"794"}}
+{"account_number":794,"balance":16491,"firstname":"Walker","lastname":"Charles","age":32,"gender":"M","address":"215 Kenilworth Place","employer":"Orbin","email":"walkercharles@orbin.com","city":"Rivers","state":"WI"}
+{"index":{"_id":"799"}}
+{"account_number":799,"balance":2889,"firstname":"Myra","lastname":"Guerra","age":28,"gender":"F","address":"625 Dahlgreen Place","employer":"Digigene","email":"myraguerra@digigene.com","city":"Draper","state":"CA"}
+{"index":{"_id":"802"}}
+{"account_number":802,"balance":19630,"firstname":"Gracie","lastname":"Foreman","age":40,"gender":"F","address":"219 Kent Avenue","employer":"Supportal","email":"gracieforeman@supportal.com","city":"Westboro","state":"NH"}
+{"index":{"_id":"807"}}
+{"account_number":807,"balance":29206,"firstname":"Hatfield","lastname":"Lowe","age":23,"gender":"M","address":"499 Adler Place","employer":"Lovepad","email":"hatfieldlowe@lovepad.com","city":"Wiscon","state":"DC"}
+{"index":{"_id":"814"}}
+{"account_number":814,"balance":9838,"firstname":"Morse","lastname":"Mcbride","age":26,"gender":"F","address":"776 Calyer Street","employer":"Inear","email":"morsemcbride@inear.com","city":"Kingstowne","state":"ND"}
+{"index":{"_id":"819"}}
+{"account_number":819,"balance":3971,"firstname":"Karyn","lastname":"Medina","age":24,"gender":"F","address":"417 Utica Avenue","employer":"Qnekt","email":"karynmedina@qnekt.com","city":"Kerby","state":"WY"}
+{"index":{"_id":"821"}}
+{"account_number":821,"balance":33271,"firstname":"Trisha","lastname":"Blankenship","age":22,"gender":"M","address":"329 Jamaica Avenue","employer":"Chorizon","email":"trishablankenship@chorizon.com","city":"Sexton","state":"VT"}
+{"index":{"_id":"826"}}
+{"account_number":826,"balance":11548,"firstname":"Summers","lastname":"Vinson","age":22,"gender":"F","address":"742 Irwin Street","employer":"Globoil","email":"summersvinson@globoil.com","city":"Callaghan","state":"WY"}
+{"index":{"_id":"833"}}
+{"account_number":833,"balance":46154,"firstname":"Woodward","lastname":"Hood","age":22,"gender":"M","address":"398 Atkins Avenue","employer":"Zedalis","email":"woodwardhood@zedalis.com","city":"Stonybrook","state":"NE"}
+{"index":{"_id":"838"}}
+{"account_number":838,"balance":24629,"firstname":"Latonya","lastname":"Blake","age":37,"gender":"F","address":"531 Milton Street","employer":"Rugstars","email":"latonyablake@rugstars.com","city":"Tedrow","state":"WA"}
+{"index":{"_id":"840"}}
+{"account_number":840,"balance":39615,"firstname":"Boone","lastname":"Gomez","age":38,"gender":"M","address":"256 Hampton Place","employer":"Geekular","email":"boonegomez@geekular.com","city":"Westerville","state":"HI"}
+{"index":{"_id":"845"}}
+{"account_number":845,"balance":35422,"firstname":"Tracy","lastname":"Vaughn","age":39,"gender":"M","address":"645 Rockaway Parkway","employer":"Andryx","email":"tracyvaughn@andryx.com","city":"Wilmington","state":"ME"}
+{"index":{"_id":"852"}}
+{"account_number":852,"balance":6041,"firstname":"Allen","lastname":"Hammond","age":26,"gender":"M","address":"793 Essex Street","employer":"Tersanki","email":"allenhammond@tersanki.com","city":"Osmond","state":"NC"}
+{"index":{"_id":"857"}}
+{"account_number":857,"balance":39678,"firstname":"Alyce","lastname":"Douglas","age":23,"gender":"M","address":"326 Robert Street","employer":"Earbang","email":"alycedouglas@earbang.com","city":"Thornport","state":"GA"}
+{"index":{"_id":"864"}}
+{"account_number":864,"balance":21804,"firstname":"Duffy","lastname":"Anthony","age":23,"gender":"M","address":"582 Cooke Court","employer":"Schoolio","email":"duffyanthony@schoolio.com","city":"Brenton","state":"CO"}
+{"index":{"_id":"869"}}
+{"account_number":869,"balance":43544,"firstname":"Corinne","lastname":"Robbins","age":25,"gender":"F","address":"732 Quentin Road","employer":"Orbaxter","email":"corinnerobbins@orbaxter.com","city":"Roy","state":"TN"}
+{"index":{"_id":"871"}}
+{"account_number":871,"balance":35854,"firstname":"Norma","lastname":"Burt","age":32,"gender":"M","address":"934 Cyrus Avenue","employer":"Magnafone","email":"normaburt@magnafone.com","city":"Eden","state":"TN"}
+{"index":{"_id":"876"}}
+{"account_number":876,"balance":48568,"firstname":"Brady","lastname":"Glover","age":21,"gender":"F","address":"565 Oceanview Avenue","employer":"Comvex","email":"bradyglover@comvex.com","city":"Noblestown","state":"ID"}
+{"index":{"_id":"883"}}
+{"account_number":883,"balance":33679,"firstname":"Austin","lastname":"Jefferson","age":34,"gender":"M","address":"846 Lincoln Avenue","employer":"Polarax","email":"austinjefferson@polarax.com","city":"Savannah","state":"CT"}
+{"index":{"_id":"888"}}
+{"account_number":888,"balance":22277,"firstname":"Myrna","lastname":"Herman","age":39,"gender":"F","address":"649 Harwood Place","employer":"Enthaze","email":"myrnaherman@enthaze.com","city":"Idamay","state":"AR"}
+{"index":{"_id":"890"}}
+{"account_number":890,"balance":31198,"firstname":"Alvarado","lastname":"Pate","age":25,"gender":"M","address":"269 Ashland Place","employer":"Ovolo","email":"alvaradopate@ovolo.com","city":"Volta","state":"MI"}
+{"index":{"_id":"895"}}
+{"account_number":895,"balance":7327,"firstname":"Lara","lastname":"Mcdaniel","age":36,"gender":"M","address":"854 Willow Place","employer":"Acusage","email":"laramcdaniel@acusage.com","city":"Imperial","state":"NC"}
+{"index":{"_id":"903"}}
+{"account_number":903,"balance":10238,"firstname":"Wade","lastname":"Page","age":35,"gender":"F","address":"685 Waldorf Court","employer":"Eplosion","email":"wadepage@eplosion.com","city":"Welda","state":"AL"}
+{"index":{"_id":"908"}}
+{"account_number":908,"balance":45975,"firstname":"Mosley","lastname":"Holloway","age":31,"gender":"M","address":"929 Eldert Lane","employer":"Anivet","email":"mosleyholloway@anivet.com","city":"Biehle","state":"MS"}
+{"index":{"_id":"910"}}
+{"account_number":910,"balance":36831,"firstname":"Esmeralda","lastname":"James","age":23,"gender":"F","address":"535 High Street","employer":"Terrasys","email":"esmeraldajames@terrasys.com","city":"Dubois","state":"IN"}
+{"index":{"_id":"915"}}
+{"account_number":915,"balance":19816,"firstname":"Farrell","lastname":"French","age":35,"gender":"F","address":"126 McKibbin Street","employer":"Techmania","email":"farrellfrench@techmania.com","city":"Wescosville","state":"AL"}
+{"index":{"_id":"922"}}
+{"account_number":922,"balance":39347,"firstname":"Irwin","lastname":"Pugh","age":32,"gender":"M","address":"463 Shale Street","employer":"Idego","email":"irwinpugh@idego.com","city":"Ivanhoe","state":"ID"}
+{"index":{"_id":"927"}}
+{"account_number":927,"balance":19976,"firstname":"Jeanette","lastname":"Acevedo","age":26,"gender":"M","address":"694 Polhemus Place","employer":"Halap","email":"jeanetteacevedo@halap.com","city":"Harrison","state":"MO"}
+{"index":{"_id":"934"}}
+{"account_number":934,"balance":43987,"firstname":"Freida","lastname":"Daniels","age":34,"gender":"M","address":"448 Cove Lane","employer":"Vurbo","email":"freidadaniels@vurbo.com","city":"Snelling","state":"NJ"}
+{"index":{"_id":"939"}}
+{"account_number":939,"balance":31228,"firstname":"Hodges","lastname":"Massey","age":37,"gender":"F","address":"431 Dahl Court","employer":"Kegular","email":"hodgesmassey@kegular.com","city":"Katonah","state":"MD"}
+{"index":{"_id":"941"}}
+{"account_number":941,"balance":38796,"firstname":"Kim","lastname":"Moss","age":28,"gender":"F","address":"105 Onderdonk Avenue","employer":"Digirang","email":"kimmoss@digirang.com","city":"Centerville","state":"TX"}
+{"index":{"_id":"946"}}
+{"account_number":946,"balance":42794,"firstname":"Ina","lastname":"Obrien","age":36,"gender":"M","address":"339 Rewe Street","employer":"Eclipsent","email":"inaobrien@eclipsent.com","city":"Soham","state":"RI"}
+{"index":{"_id":"953"}}
+{"account_number":953,"balance":1110,"firstname":"Baxter","lastname":"Black","age":27,"gender":"M","address":"720 Stillwell Avenue","employer":"Uplinx","email":"baxterblack@uplinx.com","city":"Drummond","state":"MN"}
+{"index":{"_id":"958"}}
+{"account_number":958,"balance":32849,"firstname":"Brown","lastname":"Wilkins","age":40,"gender":"M","address":"686 Delmonico Place","employer":"Medesign","email":"brownwilkins@medesign.com","city":"Shelby","state":"WY"}
+{"index":{"_id":"960"}}
+{"account_number":960,"balance":2905,"firstname":"Curry","lastname":"Vargas","age":40,"gender":"M","address":"242 Blake Avenue","employer":"Pearlesex","email":"curryvargas@pearlesex.com","city":"Henrietta","state":"NH"}
+{"index":{"_id":"965"}}
+{"account_number":965,"balance":21882,"firstname":"Patrica","lastname":"Melton","age":28,"gender":"M","address":"141 Rodney Street","employer":"Flexigen","email":"patricamelton@flexigen.com","city":"Klagetoh","state":"MD"}
+{"index":{"_id":"972"}}
+{"account_number":972,"balance":24719,"firstname":"Leona","lastname":"Christian","age":26,"gender":"F","address":"900 Woodpoint Road","employer":"Extrawear","email":"leonachristian@extrawear.com","city":"Roderfield","state":"MA"}
+{"index":{"_id":"977"}}
+{"account_number":977,"balance":6744,"firstname":"Rodgers","lastname":"Mccray","age":21,"gender":"F","address":"612 Duryea Place","employer":"Papricut","email":"rodgersmccray@papricut.com","city":"Marenisco","state":"MD"}
+{"index":{"_id":"984"}}
+{"account_number":984,"balance":1904,"firstname":"Viola","lastname":"Crawford","age":35,"gender":"F","address":"354 Linwood Street","employer":"Ginkle","email":"violacrawford@ginkle.com","city":"Witmer","state":"AR"}
+{"index":{"_id":"989"}}
+{"account_number":989,"balance":48622,"firstname":"Franklin","lastname":"Frank","age":38,"gender":"M","address":"270 Carlton Avenue","employer":"Shopabout","email":"franklinfrank@shopabout.com","city":"Guthrie","state":"NC"}
+{"index":{"_id":"991"}}
+{"account_number":991,"balance":4239,"firstname":"Connie","lastname":"Berry","age":28,"gender":"F","address":"647 Gardner Avenue","employer":"Flumbo","email":"connieberry@flumbo.com","city":"Frierson","state":"MO"}
+{"index":{"_id":"996"}}
+{"account_number":996,"balance":17541,"firstname":"Andrews","lastname":"Herrera","age":30,"gender":"F","address":"570 Vandam Street","employer":"Klugger","email":"andrewsherrera@klugger.com","city":"Whitehaven","state":"MN"}
+{"index":{"_id":"0"}}
+{"account_number":0,"balance":16623,"firstname":"Bradshaw","lastname":"Mckenzie","age":29,"gender":"F","address":"244 Columbus Place","employer":"Euron","email":"bradshawmckenzie@euron.com","city":"Hobucken","state":"CO"}
+{"index":{"_id":"5"}}
+{"account_number":5,"balance":29342,"firstname":"Leola","lastname":"Stewart","age":30,"gender":"F","address":"311 Elm Place","employer":"Diginetic","email":"leolastewart@diginetic.com","city":"Fairview","state":"NJ"}
+{"index":{"_id":"12"}}
+{"account_number":12,"balance":37055,"firstname":"Stafford","lastname":"Brock","age":20,"gender":"F","address":"296 Wythe Avenue","employer":"Uncorp","email":"staffordbrock@uncorp.com","city":"Bend","state":"AL"}
+{"index":{"_id":"17"}}
+{"account_number":17,"balance":7831,"firstname":"Bessie","lastname":"Orr","age":31,"gender":"F","address":"239 Hinsdale Street","employer":"Skyplex","email":"bessieorr@skyplex.com","city":"Graball","state":"FL"}
+{"index":{"_id":"24"}}
+{"account_number":24,"balance":44182,"firstname":"Wood","lastname":"Dale","age":39,"gender":"M","address":"582 Gelston Avenue","employer":"Besto","email":"wooddale@besto.com","city":"Juntura","state":"MI"}
+{"index":{"_id":"29"}}
+{"account_number":29,"balance":27323,"firstname":"Leah","lastname":"Santiago","age":33,"gender":"M","address":"193 Schenck Avenue","employer":"Isologix","email":"leahsantiago@isologix.com","city":"Gerton","state":"ND"}
+{"index":{"_id":"31"}}
+{"account_number":31,"balance":30443,"firstname":"Kristen","lastname":"Santana","age":22,"gender":"F","address":"130 Middagh Street","employer":"Dogspa","email":"kristensantana@dogspa.com","city":"Vale","state":"MA"}
+{"index":{"_id":"36"}}
+{"account_number":36,"balance":15902,"firstname":"Alexandra","lastname":"Nguyen","age":39,"gender":"F","address":"389 Elizabeth Place","employer":"Bittor","email":"alexandranguyen@bittor.com","city":"Hemlock","state":"KY"}
+{"index":{"_id":"43"}}
+{"account_number":43,"balance":33474,"firstname":"Ryan","lastname":"Howe","age":25,"gender":"M","address":"660 Huntington Street","employer":"Microluxe","email":"ryanhowe@microluxe.com","city":"Clara","state":"CT"}
+{"index":{"_id":"48"}}
+{"account_number":48,"balance":40608,"firstname":"Peck","lastname":"Downs","age":39,"gender":"F","address":"594 Dwight Street","employer":"Ramjob","email":"peckdowns@ramjob.com","city":"Coloma","state":"WA"}
+{"index":{"_id":"50"}}
+{"account_number":50,"balance":43695,"firstname":"Sheena","lastname":"Kirkland","age":33,"gender":"M","address":"598 Bank Street","employer":"Zerbina","email":"sheenakirkland@zerbina.com","city":"Walland","state":"IN"}
+{"index":{"_id":"55"}}
+{"account_number":55,"balance":22020,"firstname":"Shelia","lastname":"Puckett","age":33,"gender":"M","address":"265 Royce Place","employer":"Izzby","email":"sheliapuckett@izzby.com","city":"Slovan","state":"HI"}
+{"index":{"_id":"62"}}
+{"account_number":62,"balance":43065,"firstname":"Lester","lastname":"Stanton","age":37,"gender":"M","address":"969 Doughty Street","employer":"Geekko","email":"lesterstanton@geekko.com","city":"Itmann","state":"DC"}
+{"index":{"_id":"67"}}
+{"account_number":67,"balance":39430,"firstname":"Isabelle","lastname":"Spence","age":39,"gender":"M","address":"718 Troy Avenue","employer":"Geeketron","email":"isabellespence@geeketron.com","city":"Camptown","state":"WA"}
+{"index":{"_id":"74"}}
+{"account_number":74,"balance":47167,"firstname":"Lauri","lastname":"Saunders","age":38,"gender":"F","address":"768 Lynch Street","employer":"Securia","email":"laurisaunders@securia.com","city":"Caroline","state":"TN"}
+{"index":{"_id":"79"}}
+{"account_number":79,"balance":28185,"firstname":"Booker","lastname":"Lowery","age":29,"gender":"M","address":"817 Campus Road","employer":"Sensate","email":"bookerlowery@sensate.com","city":"Carlos","state":"MT"}
+{"index":{"_id":"81"}}
+{"account_number":81,"balance":46568,"firstname":"Dennis","lastname":"Gilbert","age":40,"gender":"M","address":"619 Minna Street","employer":"Melbacor","email":"dennisgilbert@melbacor.com","city":"Kersey","state":"ND"}
+{"index":{"_id":"86"}}
+{"account_number":86,"balance":15428,"firstname":"Walton","lastname":"Butler","age":36,"gender":"M","address":"999 Schenck Street","employer":"Unisure","email":"waltonbutler@unisure.com","city":"Bentonville","state":"IL"}
+{"index":{"_id":"93"}}
+{"account_number":93,"balance":17728,"firstname":"Jeri","lastname":"Booth","age":31,"gender":"M","address":"322 Roosevelt Court","employer":"Geekology","email":"jeribooth@geekology.com","city":"Leming","state":"ND"}
+{"index":{"_id":"98"}}
+{"account_number":98,"balance":15085,"firstname":"Cora","lastname":"Barrett","age":24,"gender":"F","address":"555 Neptune Court","employer":"Kiosk","email":"corabarrett@kiosk.com","city":"Independence","state":"MN"}
+{"index":{"_id":"101"}}
+{"account_number":101,"balance":43400,"firstname":"Cecelia","lastname":"Grimes","age":31,"gender":"M","address":"972 Lincoln Place","employer":"Ecosys","email":"ceceliagrimes@ecosys.com","city":"Manchester","state":"AR"}
+{"index":{"_id":"106"}}
+{"account_number":106,"balance":8212,"firstname":"Josefina","lastname":"Wagner","age":36,"gender":"M","address":"418 Estate Road","employer":"Kyaguru","email":"josefinawagner@kyaguru.com","city":"Darbydale","state":"FL"}
+{"index":{"_id":"113"}}
+{"account_number":113,"balance":41652,"firstname":"Burt","lastname":"Moses","age":27,"gender":"M","address":"633 Berry Street","employer":"Uni","email":"burtmoses@uni.com","city":"Russellville","state":"CT"}
+{"index":{"_id":"118"}}
+{"account_number":118,"balance":2223,"firstname":"Ballard","lastname":"Vasquez","age":33,"gender":"F","address":"101 Bush Street","employer":"Intergeek","email":"ballardvasquez@intergeek.com","city":"Century","state":"MN"}
+{"index":{"_id":"120"}}
+{"account_number":120,"balance":38565,"firstname":"Browning","lastname":"Rodriquez","age":33,"gender":"M","address":"910 Moore Street","employer":"Opportech","email":"browningrodriquez@opportech.com","city":"Cutter","state":"ND"}
+{"index":{"_id":"125"}}
+{"account_number":125,"balance":5396,"firstname":"Tanisha","lastname":"Dixon","age":30,"gender":"M","address":"482 Hancock Street","employer":"Junipoor","email":"tanishadixon@junipoor.com","city":"Wauhillau","state":"IA"}
+{"index":{"_id":"132"}}
+{"account_number":132,"balance":37707,"firstname":"Horton","lastname":"Romero","age":35,"gender":"M","address":"427 Rutherford Place","employer":"Affluex","email":"hortonromero@affluex.com","city":"Hall","state":"AK"}
+{"index":{"_id":"137"}}
+{"account_number":137,"balance":3596,"firstname":"Frost","lastname":"Freeman","age":29,"gender":"F","address":"191 Dennett Place","employer":"Beadzza","email":"frostfreeman@beadzza.com","city":"Sabillasville","state":"HI"}
+{"index":{"_id":"144"}}
+{"account_number":144,"balance":43257,"firstname":"Evans","lastname":"Dyer","age":30,"gender":"F","address":"912 Post Court","employer":"Magmina","email":"evansdyer@magmina.com","city":"Gordon","state":"HI"}
+{"index":{"_id":"149"}}
+{"account_number":149,"balance":22994,"firstname":"Megan","lastname":"Gonzales","age":21,"gender":"M","address":"836 Tampa Court","employer":"Andershun","email":"megangonzales@andershun.com","city":"Rockhill","state":"AL"}
+{"index":{"_id":"151"}}
+{"account_number":151,"balance":34473,"firstname":"Kent","lastname":"Joyner","age":20,"gender":"F","address":"799 Truxton Street","employer":"Kozgene","email":"kentjoyner@kozgene.com","city":"Allamuchy","state":"DC"}
+{"index":{"_id":"156"}}
+{"account_number":156,"balance":40185,"firstname":"Sloan","lastname":"Pennington","age":24,"gender":"F","address":"573 Opal Court","employer":"Hopeli","email":"sloanpennington@hopeli.com","city":"Evergreen","state":"CT"}
+{"index":{"_id":"163"}}
+{"account_number":163,"balance":43075,"firstname":"Wilda","lastname":"Norman","age":33,"gender":"F","address":"173 Beadel Street","employer":"Kog","email":"wildanorman@kog.com","city":"Bodega","state":"ME"}
+{"index":{"_id":"168"}}
+{"account_number":168,"balance":49568,"firstname":"Carissa","lastname":"Simon","age":20,"gender":"M","address":"975 Flatbush Avenue","employer":"Zillacom","email":"carissasimon@zillacom.com","city":"Neibert","state":"IL"}
+{"index":{"_id":"170"}}
+{"account_number":170,"balance":6025,"firstname":"Mann","lastname":"Madden","age":36,"gender":"F","address":"161 Radde Place","employer":"Farmex","email":"mannmadden@farmex.com","city":"Thermal","state":"LA"}
+{"index":{"_id":"175"}}
+{"account_number":175,"balance":16213,"firstname":"Montoya","lastname":"Donaldson","age":28,"gender":"F","address":"481 Morton Street","employer":"Envire","email":"montoyadonaldson@envire.com","city":"Delco","state":"MA"}
+{"index":{"_id":"182"}}
+{"account_number":182,"balance":7803,"firstname":"Manuela","lastname":"Dillon","age":21,"gender":"M","address":"742 Garnet Street","employer":"Moreganic","email":"manueladillon@moreganic.com","city":"Ilchester","state":"TX"}
+{"index":{"_id":"187"}}
+{"account_number":187,"balance":26581,"firstname":"Autumn","lastname":"Hodges","age":35,"gender":"M","address":"757 Granite Street","employer":"Ezentia","email":"autumnhodges@ezentia.com","city":"Martinsville","state":"KY"}
+{"index":{"_id":"194"}}
+{"account_number":194,"balance":16311,"firstname":"Beck","lastname":"Rosario","age":39,"gender":"M","address":"721 Cambridge Place","employer":"Zoid","email":"beckrosario@zoid.com","city":"Efland","state":"ID"}
+{"index":{"_id":"199"}}
+{"account_number":199,"balance":18086,"firstname":"Branch","lastname":"Love","age":26,"gender":"M","address":"458 Commercial Street","employer":"Frolix","email":"branchlove@frolix.com","city":"Caspar","state":"NC"}
+{"index":{"_id":"202"}}
+{"account_number":202,"balance":26466,"firstname":"Medina","lastname":"Brown","age":31,"gender":"F","address":"519 Sunnyside Court","employer":"Bleendot","email":"medinabrown@bleendot.com","city":"Winfred","state":"MI"}
+{"index":{"_id":"207"}}
+{"account_number":207,"balance":45535,"firstname":"Evelyn","lastname":"Lara","age":35,"gender":"F","address":"636 Chestnut Street","employer":"Ultrasure","email":"evelynlara@ultrasure.com","city":"Logan","state":"MI"}
+{"index":{"_id":"214"}}
+{"account_number":214,"balance":24418,"firstname":"Luann","lastname":"Faulkner","age":37,"gender":"F","address":"697 Hazel Court","employer":"Zolar","email":"luannfaulkner@zolar.com","city":"Ticonderoga","state":"TX"}
+{"index":{"_id":"219"}}
+{"account_number":219,"balance":17127,"firstname":"Edwards","lastname":"Hurley","age":25,"gender":"M","address":"834 Stockholm Street","employer":"Austech","email":"edwardshurley@austech.com","city":"Bayview","state":"NV"}
+{"index":{"_id":"221"}}
+{"account_number":221,"balance":15803,"firstname":"Benjamin","lastname":"Barrera","age":34,"gender":"M","address":"568 Main Street","employer":"Zaphire","email":"benjaminbarrera@zaphire.com","city":"Germanton","state":"WY"}
+{"index":{"_id":"226"}}
+{"account_number":226,"balance":37720,"firstname":"Wilkins","lastname":"Brady","age":40,"gender":"F","address":"486 Baltic Street","employer":"Dogtown","email":"wilkinsbrady@dogtown.com","city":"Condon","state":"MT"}
+{"index":{"_id":"233"}}
+{"account_number":233,"balance":23020,"firstname":"Washington","lastname":"Walsh","age":27,"gender":"M","address":"366 Church Avenue","employer":"Candecor","email":"washingtonwalsh@candecor.com","city":"Westphalia","state":"MA"}
+{"index":{"_id":"238"}}
+{"account_number":238,"balance":21287,"firstname":"Constance","lastname":"Wong","age":28,"gender":"M","address":"496 Brown Street","employer":"Grainspot","email":"constancewong@grainspot.com","city":"Cecilia","state":"IN"}
+{"index":{"_id":"240"}}
+{"account_number":240,"balance":49741,"firstname":"Oconnor","lastname":"Clay","age":35,"gender":"F","address":"659 Highland Boulevard","employer":"Franscene","email":"oconnorclay@franscene.com","city":"Kilbourne","state":"NH"}
+{"index":{"_id":"245"}}
+{"account_number":245,"balance":22026,"firstname":"Fran","lastname":"Bolton","age":28,"gender":"F","address":"147 Jerome Street","employer":"Solaren","email":"franbolton@solaren.com","city":"Nash","state":"RI"}
+{"index":{"_id":"252"}}
+{"account_number":252,"balance":18831,"firstname":"Elvia","lastname":"Poole","age":22,"gender":"F","address":"836 Delevan Street","employer":"Velity","email":"elviapoole@velity.com","city":"Groveville","state":"MI"}
+{"index":{"_id":"257"}}
+{"account_number":257,"balance":5318,"firstname":"Olive","lastname":"Oneil","age":35,"gender":"F","address":"457 Decatur Street","employer":"Helixo","email":"oliveoneil@helixo.com","city":"Chicopee","state":"MI"}
+{"index":{"_id":"264"}}
+{"account_number":264,"balance":22084,"firstname":"Samantha","lastname":"Ferrell","age":35,"gender":"F","address":"488 Fulton Street","employer":"Flum","email":"samanthaferrell@flum.com","city":"Brandywine","state":"MT"}
+{"index":{"_id":"269"}}
+{"account_number":269,"balance":43317,"firstname":"Crosby","lastname":"Figueroa","age":34,"gender":"M","address":"910 Aurelia Court","employer":"Pyramia","email":"crosbyfigueroa@pyramia.com","city":"Leyner","state":"OH"}
+{"index":{"_id":"271"}}
+{"account_number":271,"balance":11864,"firstname":"Holt","lastname":"Walter","age":30,"gender":"F","address":"645 Poplar Avenue","employer":"Grupoli","email":"holtwalter@grupoli.com","city":"Mansfield","state":"OR"}
+{"index":{"_id":"276"}}
+{"account_number":276,"balance":11606,"firstname":"Pittman","lastname":"Mathis","age":23,"gender":"F","address":"567 Charles Place","employer":"Zuvy","email":"pittmanmathis@zuvy.com","city":"Roeville","state":"KY"}
+{"index":{"_id":"283"}}
+{"account_number":283,"balance":24070,"firstname":"Fuentes","lastname":"Foley","age":30,"gender":"M","address":"729 Walker Court","employer":"Knowlysis","email":"fuentesfoley@knowlysis.com","city":"Tryon","state":"TN"}
+{"index":{"_id":"288"}}
+{"account_number":288,"balance":27243,"firstname":"Wong","lastname":"Stone","age":39,"gender":"F","address":"440 Willoughby Street","employer":"Zentix","email":"wongstone@zentix.com","city":"Wheatfields","state":"DC"}
+{"index":{"_id":"290"}}
+{"account_number":290,"balance":26103,"firstname":"Neva","lastname":"Burgess","age":37,"gender":"F","address":"985 Wyona Street","employer":"Slofast","email":"nevaburgess@slofast.com","city":"Cawood","state":"DC"}
+{"index":{"_id":"295"}}
+{"account_number":295,"balance":37358,"firstname":"Howe","lastname":"Nash","age":20,"gender":"M","address":"833 Union Avenue","employer":"Aquacine","email":"howenash@aquacine.com","city":"Indio","state":"MN"}
+{"index":{"_id":"303"}}
+{"account_number":303,"balance":21976,"firstname":"Huffman","lastname":"Green","age":24,"gender":"F","address":"455 Colby Court","employer":"Comtest","email":"huffmangreen@comtest.com","city":"Weeksville","state":"UT"}
+{"index":{"_id":"308"}}
+{"account_number":308,"balance":33989,"firstname":"Glass","lastname":"Schroeder","age":25,"gender":"F","address":"670 Veterans Avenue","employer":"Realmo","email":"glassschroeder@realmo.com","city":"Gratton","state":"NY"}
+{"index":{"_id":"310"}}
+{"account_number":310,"balance":23049,"firstname":"Shannon","lastname":"Morton","age":39,"gender":"F","address":"412 Pleasant Place","employer":"Ovation","email":"shannonmorton@ovation.com","city":"Edgar","state":"AZ"}
+{"index":{"_id":"315"}}
+{"account_number":315,"balance":1314,"firstname":"Clare","lastname":"Morrow","age":33,"gender":"F","address":"728 Madeline Court","employer":"Gaptec","email":"claremorrow@gaptec.com","city":"Mapletown","state":"PA"}
+{"index":{"_id":"322"}}
+{"account_number":322,"balance":6303,"firstname":"Gilliam","lastname":"Horne","age":27,"gender":"M","address":"414 Florence Avenue","employer":"Shepard","email":"gilliamhorne@shepard.com","city":"Winesburg","state":"WY"}
+{"index":{"_id":"327"}}
+{"account_number":327,"balance":29294,"firstname":"Nell","lastname":"Contreras","age":27,"gender":"M","address":"694 Gold Street","employer":"Momentia","email":"nellcontreras@momentia.com","city":"Cumminsville","state":"AL"}
+{"index":{"_id":"334"}}
+{"account_number":334,"balance":9178,"firstname":"Cross","lastname":"Floyd","age":21,"gender":"F","address":"815 Herkimer Court","employer":"Maroptic","email":"crossfloyd@maroptic.com","city":"Kraemer","state":"AK"}
+{"index":{"_id":"339"}}
+{"account_number":339,"balance":3992,"firstname":"Franco","lastname":"Welch","age":38,"gender":"F","address":"776 Brightwater Court","employer":"Earthplex","email":"francowelch@earthplex.com","city":"Naomi","state":"ME"}
+{"index":{"_id":"341"}}
+{"account_number":341,"balance":44367,"firstname":"Alberta","lastname":"Bradford","age":30,"gender":"F","address":"670 Grant Avenue","employer":"Bugsall","email":"albertabradford@bugsall.com","city":"Romeville","state":"MT"}
+{"index":{"_id":"346"}}
+{"account_number":346,"balance":26594,"firstname":"Shelby","lastname":"Sanchez","age":36,"gender":"F","address":"257 Fillmore Avenue","employer":"Geekus","email":"shelbysanchez@geekus.com","city":"Seymour","state":"CO"}
+{"index":{"_id":"353"}}
+{"account_number":353,"balance":45182,"firstname":"Rivera","lastname":"Sherman","age":37,"gender":"M","address":"603 Garden Place","employer":"Bovis","email":"riverasherman@bovis.com","city":"Otranto","state":"CA"}
+{"index":{"_id":"358"}}
+{"account_number":358,"balance":44043,"firstname":"Hale","lastname":"Baldwin","age":40,"gender":"F","address":"845 Menahan Street","employer":"Kidgrease","email":"halebaldwin@kidgrease.com","city":"Day","state":"AK"}
+{"index":{"_id":"360"}}
+{"account_number":360,"balance":26651,"firstname":"Ward","lastname":"Hicks","age":34,"gender":"F","address":"592 Brighton Court","employer":"Biotica","email":"wardhicks@biotica.com","city":"Kanauga","state":"VT"}
+{"index":{"_id":"365"}}
+{"account_number":365,"balance":3176,"firstname":"Sanders","lastname":"Holder","age":31,"gender":"F","address":"453 Cypress Court","employer":"Geekola","email":"sandersholder@geekola.com","city":"Staples","state":"TN"}
+{"index":{"_id":"372"}}
+{"account_number":372,"balance":28566,"firstname":"Alba","lastname":"Forbes","age":24,"gender":"M","address":"814 Meserole Avenue","employer":"Isostream","email":"albaforbes@isostream.com","city":"Clarence","state":"OR"}
+{"index":{"_id":"377"}}
+{"account_number":377,"balance":5374,"firstname":"Margo","lastname":"Gay","age":34,"gender":"F","address":"613 Chase Court","employer":"Rotodyne","email":"margogay@rotodyne.com","city":"Waumandee","state":"KS"}
+{"index":{"_id":"384"}}
+{"account_number":384,"balance":48758,"firstname":"Sallie","lastname":"Houston","age":31,"gender":"F","address":"836 Polar Street","employer":"Squish","email":"salliehouston@squish.com","city":"Morningside","state":"NC"}
+{"index":{"_id":"389"}}
+{"account_number":389,"balance":8839,"firstname":"York","lastname":"Cummings","age":27,"gender":"M","address":"778 Centre Street","employer":"Insurity","email":"yorkcummings@insurity.com","city":"Freeburn","state":"RI"}
+{"index":{"_id":"391"}}
+{"account_number":391,"balance":14733,"firstname":"Holman","lastname":"Jordan","age":30,"gender":"M","address":"391 Forrest Street","employer":"Maineland","email":"holmanjordan@maineland.com","city":"Cade","state":"CT"}
+{"index":{"_id":"396"}}
+{"account_number":396,"balance":14613,"firstname":"Marsha","lastname":"Elliott","age":38,"gender":"F","address":"297 Liberty Avenue","employer":"Orbiflex","email":"marshaelliott@orbiflex.com","city":"Windsor","state":"TX"}
+{"index":{"_id":"404"}}
+{"account_number":404,"balance":34978,"firstname":"Massey","lastname":"Becker","age":26,"gender":"F","address":"930 Pitkin Avenue","employer":"Genekom","email":"masseybecker@genekom.com","city":"Blairstown","state":"OR"}
+{"index":{"_id":"409"}}
+{"account_number":409,"balance":36960,"firstname":"Maura","lastname":"Glenn","age":31,"gender":"M","address":"183 Poly Place","employer":"Viagreat","email":"mauraglenn@viagreat.com","city":"Foscoe","state":"DE"}
+{"index":{"_id":"411"}}
+{"account_number":411,"balance":1172,"firstname":"Guzman","lastname":"Whitfield","age":22,"gender":"M","address":"181 Perry Terrace","employer":"Springbee","email":"guzmanwhitfield@springbee.com","city":"Balm","state":"IN"}
+{"index":{"_id":"416"}}
+{"account_number":416,"balance":27169,"firstname":"Hunt","lastname":"Schwartz","age":28,"gender":"F","address":"461 Havens Place","employer":"Danja","email":"huntschwartz@danja.com","city":"Grenelefe","state":"NV"}
+{"index":{"_id":"423"}}
+{"account_number":423,"balance":38852,"firstname":"Hines","lastname":"Underwood","age":21,"gender":"F","address":"284 Louise Terrace","employer":"Namegen","email":"hinesunderwood@namegen.com","city":"Downsville","state":"CO"}
+{"index":{"_id":"428"}}
+{"account_number":428,"balance":13925,"firstname":"Stephens","lastname":"Cain","age":20,"gender":"F","address":"189 Summit Street","employer":"Rocklogic","email":"stephenscain@rocklogic.com","city":"Bourg","state":"HI"}
+{"index":{"_id":"430"}}
+{"account_number":430,"balance":15251,"firstname":"Alejandra","lastname":"Chavez","age":34,"gender":"M","address":"651 Butler Place","employer":"Gology","email":"alejandrachavez@gology.com","city":"Allensworth","state":"VT"}
+{"index":{"_id":"435"}}
+{"account_number":435,"balance":14654,"firstname":"Sue","lastname":"Lopez","age":22,"gender":"F","address":"632 Stone Avenue","employer":"Emergent","email":"suelopez@emergent.com","city":"Waterford","state":"TN"}
+{"index":{"_id":"442"}}
+{"account_number":442,"balance":36211,"firstname":"Lawanda","lastname":"Leon","age":27,"gender":"F","address":"126 Canal Avenue","employer":"Xixan","email":"lawandaleon@xixan.com","city":"Berwind","state":"TN"}
+{"index":{"_id":"447"}}
+{"account_number":447,"balance":11402,"firstname":"Lucia","lastname":"Livingston","age":35,"gender":"M","address":"773 Lake Avenue","employer":"Soprano","email":"lucialivingston@soprano.com","city":"Edgewater","state":"TN"}
+{"index":{"_id":"454"}}
+{"account_number":454,"balance":31687,"firstname":"Alicia","lastname":"Rollins","age":22,"gender":"F","address":"483 Verona Place","employer":"Boilcat","email":"aliciarollins@boilcat.com","city":"Lutsen","state":"MD"}
+{"index":{"_id":"459"}}
+{"account_number":459,"balance":18869,"firstname":"Pamela","lastname":"Henry","age":20,"gender":"F","address":"361 Locust Avenue","employer":"Imageflow","email":"pamelahenry@imageflow.com","city":"Greenfields","state":"OH"}
+{"index":{"_id":"461"}}
+{"account_number":461,"balance":38807,"firstname":"Mcbride","lastname":"Padilla","age":34,"gender":"F","address":"550 Borinquen Pl","employer":"Zepitope","email":"mcbridepadilla@zepitope.com","city":"Emory","state":"AZ"}
+{"index":{"_id":"466"}}
+{"account_number":466,"balance":25109,"firstname":"Marcie","lastname":"Mcmillan","age":30,"gender":"F","address":"947 Gain Court","employer":"Entroflex","email":"marciemcmillan@entroflex.com","city":"Ronco","state":"ND"}
+{"index":{"_id":"473"}}
+{"account_number":473,"balance":5391,"firstname":"Susan","lastname":"Luna","age":25,"gender":"F","address":"521 Bogart Street","employer":"Zaya","email":"susanluna@zaya.com","city":"Grazierville","state":"MI"}
+{"index":{"_id":"478"}}
+{"account_number":478,"balance":28044,"firstname":"Dana","lastname":"Decker","age":35,"gender":"M","address":"627 Dobbin Street","employer":"Acrodance","email":"danadecker@acrodance.com","city":"Sharon","state":"MN"}
+{"index":{"_id":"480"}}
+{"account_number":480,"balance":40807,"firstname":"Anastasia","lastname":"Parker","age":24,"gender":"M","address":"650 Folsom Place","employer":"Zilladyne","email":"anastasiaparker@zilladyne.com","city":"Oberlin","state":"WY"}
+{"index":{"_id":"485"}}
+{"account_number":485,"balance":44235,"firstname":"Albert","lastname":"Roberts","age":40,"gender":"M","address":"385 Harman Street","employer":"Stralum","email":"albertroberts@stralum.com","city":"Watrous","state":"NM"}
+{"index":{"_id":"492"}}
+{"account_number":492,"balance":31055,"firstname":"Burnett","lastname":"Briggs","age":35,"gender":"M","address":"987 Cass Place","employer":"Pharmex","email":"burnettbriggs@pharmex.com","city":"Cornfields","state":"TX"}
+{"index":{"_id":"497"}}
+{"account_number":497,"balance":13493,"firstname":"Doyle","lastname":"Jenkins","age":30,"gender":"M","address":"205 Nevins Street","employer":"Unia","email":"doylejenkins@unia.com","city":"Nicut","state":"DC"}
+{"index":{"_id":"500"}}
+{"account_number":500,"balance":39143,"firstname":"Pope","lastname":"Keith","age":28,"gender":"F","address":"537 Fane Court","employer":"Zboo","email":"popekeith@zboo.com","city":"Courtland","state":"AL"}
+{"index":{"_id":"505"}}
+{"account_number":505,"balance":45493,"firstname":"Shelley","lastname":"Webb","age":29,"gender":"M","address":"873 Crawford Avenue","employer":"Quadeebo","email":"shelleywebb@quadeebo.com","city":"Topanga","state":"IL"}
+{"index":{"_id":"512"}}
+{"account_number":512,"balance":47432,"firstname":"Alisha","lastname":"Morales","age":29,"gender":"M","address":"623 Batchelder Street","employer":"Terragen","email":"alishamorales@terragen.com","city":"Gilmore","state":"VA"}
+{"index":{"_id":"517"}}
+{"account_number":517,"balance":3022,"firstname":"Allyson","lastname":"Walls","age":38,"gender":"F","address":"334 Coffey Street","employer":"Gorganic","email":"allysonwalls@gorganic.com","city":"Dahlen","state":"GA"}
+{"index":{"_id":"524"}}
+{"account_number":524,"balance":49334,"firstname":"Salas","lastname":"Farley","age":30,"gender":"F","address":"499 Trucklemans Lane","employer":"Xumonk","email":"salasfarley@xumonk.com","city":"Noxen","state":"AL"}
+{"index":{"_id":"529"}}
+{"account_number":529,"balance":21788,"firstname":"Deann","lastname":"Fisher","age":23,"gender":"F","address":"511 Buffalo Avenue","employer":"Twiist","email":"deannfisher@twiist.com","city":"Templeton","state":"WA"}
+{"index":{"_id":"531"}}
+{"account_number":531,"balance":39770,"firstname":"Janet","lastname":"Pena","age":38,"gender":"M","address":"645 Livonia Avenue","employer":"Corecom","email":"janetpena@corecom.com","city":"Garberville","state":"OK"}
+{"index":{"_id":"536"}}
+{"account_number":536,"balance":6255,"firstname":"Emma","lastname":"Adkins","age":33,"gender":"F","address":"971 Calder Place","employer":"Ontagene","email":"emmaadkins@ontagene.com","city":"Ruckersville","state":"GA"}
+{"index":{"_id":"543"}}
+{"account_number":543,"balance":48022,"firstname":"Marina","lastname":"Rasmussen","age":31,"gender":"M","address":"446 Love Lane","employer":"Crustatia","email":"marinarasmussen@crustatia.com","city":"Statenville","state":"MD"}
+{"index":{"_id":"548"}}
+{"account_number":548,"balance":36930,"firstname":"Sandra","lastname":"Andrews","age":37,"gender":"M","address":"973 Prospect Street","employer":"Datagene","email":"sandraandrews@datagene.com","city":"Inkerman","state":"MO"}
+{"index":{"_id":"550"}}
+{"account_number":550,"balance":32238,"firstname":"Walsh","lastname":"Goodwin","age":22,"gender":"M","address":"953 Canda Avenue","employer":"Proflex","email":"walshgoodwin@proflex.com","city":"Ypsilanti","state":"MT"}
+{"index":{"_id":"555"}}
+{"account_number":555,"balance":10750,"firstname":"Fannie","lastname":"Slater","age":31,"gender":"M","address":"457 Tech Place","employer":"Kineticut","email":"fannieslater@kineticut.com","city":"Basye","state":"MO"}
+{"index":{"_id":"562"}}
+{"account_number":562,"balance":10737,"firstname":"Sarah","lastname":"Strong","age":39,"gender":"F","address":"177 Pioneer Street","employer":"Megall","email":"sarahstrong@megall.com","city":"Ladera","state":"WY"}
+{"index":{"_id":"567"}}
+{"account_number":567,"balance":6507,"firstname":"Diana","lastname":"Dominguez","age":40,"gender":"M","address":"419 Albany Avenue","employer":"Ohmnet","email":"dianadominguez@ohmnet.com","city":"Wildwood","state":"TX"}
+{"index":{"_id":"574"}}
+{"account_number":574,"balance":32954,"firstname":"Andrea","lastname":"Mosley","age":24,"gender":"M","address":"368 Throop Avenue","employer":"Musix","email":"andreamosley@musix.com","city":"Blende","state":"DC"}
+{"index":{"_id":"579"}}
+{"account_number":579,"balance":12044,"firstname":"Banks","lastname":"Sawyer","age":36,"gender":"M","address":"652 Doone Court","employer":"Rooforia","email":"bankssawyer@rooforia.com","city":"Foxworth","state":"ND"}
+{"index":{"_id":"581"}}
+{"account_number":581,"balance":16525,"firstname":"Fuller","lastname":"Mcintyre","age":32,"gender":"M","address":"169 Bergen Place","employer":"Applideck","email":"fullermcintyre@applideck.com","city":"Kenvil","state":"NY"}
+{"index":{"_id":"586"}}
+{"account_number":586,"balance":13644,"firstname":"Love","lastname":"Velasquez","age":26,"gender":"F","address":"290 Girard Street","employer":"Zomboid","email":"lovevelasquez@zomboid.com","city":"Villarreal","state":"SD"}
+{"index":{"_id":"593"}}
+{"account_number":593,"balance":41230,"firstname":"Muriel","lastname":"Vazquez","age":37,"gender":"M","address":"395 Montgomery Street","employer":"Sustenza","email":"murielvazquez@sustenza.com","city":"Strykersville","state":"OK"}
+{"index":{"_id":"598"}}
+{"account_number":598,"balance":33251,"firstname":"Morgan","lastname":"Coleman","age":33,"gender":"M","address":"324 McClancy Place","employer":"Aclima","email":"morgancoleman@aclima.com","city":"Bowden","state":"WA"}
+{"index":{"_id":"601"}}
+{"account_number":601,"balance":20796,"firstname":"Vickie","lastname":"Valentine","age":34,"gender":"F","address":"432 Bassett Avenue","employer":"Comvene","email":"vickievalentine@comvene.com","city":"Teasdale","state":"UT"}
+{"index":{"_id":"606"}}
+{"account_number":606,"balance":28770,"firstname":"Michael","lastname":"Bray","age":31,"gender":"M","address":"935 Lake Place","employer":"Telepark","email":"michaelbray@telepark.com","city":"Lemoyne","state":"CT"}
+{"index":{"_id":"613"}}
+{"account_number":613,"balance":39340,"firstname":"Eddie","lastname":"Mccarty","age":34,"gender":"F","address":"971 Richards Street","employer":"Bisba","email":"eddiemccarty@bisba.com","city":"Fruitdale","state":"NY"}
+{"index":{"_id":"618"}}
+{"account_number":618,"balance":8976,"firstname":"Cheri","lastname":"Ford","age":30,"gender":"F","address":"803 Ridgewood Avenue","employer":"Zorromop","email":"cheriford@zorromop.com","city":"Gambrills","state":"VT"}
+{"index":{"_id":"620"}}
+{"account_number":620,"balance":7224,"firstname":"Coleen","lastname":"Bartlett","age":38,"gender":"M","address":"761 Carroll Street","employer":"Idealis","email":"coleenbartlett@idealis.com","city":"Mathews","state":"DE"}
+{"index":{"_id":"625"}}
+{"account_number":625,"balance":46010,"firstname":"Cynthia","lastname":"Johnston","age":23,"gender":"M","address":"142 Box Street","employer":"Zentry","email":"cynthiajohnston@zentry.com","city":"Makena","state":"MA"}
+{"index":{"_id":"632"}}
+{"account_number":632,"balance":40470,"firstname":"Kay","lastname":"Warren","age":20,"gender":"F","address":"422 Alabama Avenue","employer":"Realysis","email":"kaywarren@realysis.com","city":"Homestead","state":"HI"}
+{"index":{"_id":"637"}}
+{"account_number":637,"balance":3169,"firstname":"Kathy","lastname":"Carter","age":27,"gender":"F","address":"410 Jamison Lane","employer":"Limage","email":"kathycarter@limage.com","city":"Ernstville","state":"WA"}
+{"index":{"_id":"644"}}
+{"account_number":644,"balance":44021,"firstname":"Etta","lastname":"Miller","age":21,"gender":"F","address":"376 Lawton Street","employer":"Bluegrain","email":"ettamiller@bluegrain.com","city":"Baker","state":"MD"}
+{"index":{"_id":"649"}}
+{"account_number":649,"balance":20275,"firstname":"Jeanine","lastname":"Malone","age":26,"gender":"F","address":"114 Dodworth Street","employer":"Nixelt","email":"jeaninemalone@nixelt.com","city":"Keyport","state":"AK"}
+{"index":{"_id":"651"}}
+{"account_number":651,"balance":18360,"firstname":"Young","lastname":"Reeves","age":34,"gender":"M","address":"581 Plaza Street","employer":"Krog","email":"youngreeves@krog.com","city":"Sussex","state":"WY"}
+{"index":{"_id":"656"}}
+{"account_number":656,"balance":21632,"firstname":"Olson","lastname":"Hunt","age":36,"gender":"M","address":"342 Jaffray Street","employer":"Volax","email":"olsonhunt@volax.com","city":"Bangor","state":"WA"}
+{"index":{"_id":"663"}}
+{"account_number":663,"balance":2456,"firstname":"Rollins","lastname":"Richards","age":37,"gender":"M","address":"129 Sullivan Place","employer":"Geostele","email":"rollinsrichards@geostele.com","city":"Morgandale","state":"FL"}
+{"index":{"_id":"668"}}
+{"account_number":668,"balance":45069,"firstname":"Potter","lastname":"Michael","age":27,"gender":"M","address":"803 Glenmore Avenue","employer":"Ontality","email":"pottermichael@ontality.com","city":"Newkirk","state":"KS"}
+{"index":{"_id":"670"}}
+{"account_number":670,"balance":10178,"firstname":"Ollie","lastname":"Riley","age":22,"gender":"M","address":"252 Jackson Place","employer":"Adornica","email":"ollieriley@adornica.com","city":"Brethren","state":"WI"}
+{"index":{"_id":"675"}}
+{"account_number":675,"balance":36102,"firstname":"Fisher","lastname":"Shepard","age":27,"gender":"F","address":"859 Varick Street","employer":"Qot","email":"fishershepard@qot.com","city":"Diaperville","state":"MD"}
+{"index":{"_id":"682"}}
+{"account_number":682,"balance":14168,"firstname":"Anne","lastname":"Hale","age":22,"gender":"F","address":"708 Anthony Street","employer":"Cytrek","email":"annehale@cytrek.com","city":"Beechmont","state":"WV"}
+{"index":{"_id":"687"}}
+{"account_number":687,"balance":48630,"firstname":"Caroline","lastname":"Cox","age":31,"gender":"M","address":"626 Hillel Place","employer":"Opticon","email":"carolinecox@opticon.com","city":"Loma","state":"ND"}
+{"index":{"_id":"694"}}
+{"account_number":694,"balance":33125,"firstname":"Craig","lastname":"Palmer","age":31,"gender":"F","address":"273 Montrose Avenue","employer":"Comvey","email":"craigpalmer@comvey.com","city":"Cleary","state":"OK"}
+{"index":{"_id":"699"}}
+{"account_number":699,"balance":4156,"firstname":"Gallagher","lastname":"Marshall","age":37,"gender":"F","address":"648 Clifford Place","employer":"Exiand","email":"gallaghermarshall@exiand.com","city":"Belfair","state":"KY"}
+{"index":{"_id":"702"}}
+{"account_number":702,"balance":46490,"firstname":"Meadows","lastname":"Delgado","age":26,"gender":"M","address":"612 Jardine Place","employer":"Daisu","email":"meadowsdelgado@daisu.com","city":"Venice","state":"AR"}
+{"index":{"_id":"707"}}
+{"account_number":707,"balance":30325,"firstname":"Sonya","lastname":"Trevino","age":30,"gender":"F","address":"181 Irving Place","employer":"Atgen","email":"sonyatrevino@atgen.com","city":"Enetai","state":"TN"}
+{"index":{"_id":"714"}}
+{"account_number":714,"balance":16602,"firstname":"Socorro","lastname":"Murray","age":34,"gender":"F","address":"810 Manhattan Court","employer":"Isoswitch","email":"socorromurray@isoswitch.com","city":"Jugtown","state":"AZ"}
+{"index":{"_id":"719"}}
+{"account_number":719,"balance":33107,"firstname":"Leanna","lastname":"Reed","age":25,"gender":"F","address":"528 Krier Place","employer":"Rodeology","email":"leannareed@rodeology.com","city":"Carrizo","state":"WI"}
+{"index":{"_id":"721"}}
+{"account_number":721,"balance":32958,"firstname":"Mara","lastname":"Dickson","age":26,"gender":"M","address":"810 Harrison Avenue","employer":"Comtours","email":"maradickson@comtours.com","city":"Thynedale","state":"DE"}
+{"index":{"_id":"726"}}
+{"account_number":726,"balance":44737,"firstname":"Rosemary","lastname":"Salazar","age":21,"gender":"M","address":"290 Croton Loop","employer":"Rockabye","email":"rosemarysalazar@rockabye.com","city":"Helen","state":"IA"}
+{"index":{"_id":"733"}}
+{"account_number":733,"balance":15722,"firstname":"Lakeisha","lastname":"Mccarthy","age":37,"gender":"M","address":"782 Turnbull Avenue","employer":"Exosis","email":"lakeishamccarthy@exosis.com","city":"Caberfae","state":"NM"}
+{"index":{"_id":"738"}}
+{"account_number":738,"balance":44936,"firstname":"Rosalind","lastname":"Hunter","age":32,"gender":"M","address":"644 Eaton Court","employer":"Zolarity","email":"rosalindhunter@zolarity.com","city":"Cataract","state":"SD"}
+{"index":{"_id":"740"}}
+{"account_number":740,"balance":6143,"firstname":"Chambers","lastname":"Hahn","age":22,"gender":"M","address":"937 Windsor Place","employer":"Medalert","email":"chambershahn@medalert.com","city":"Dorneyville","state":"DC"}
+{"index":{"_id":"745"}}
+{"account_number":745,"balance":4572,"firstname":"Jacobs","lastname":"Sweeney","age":32,"gender":"M","address":"189 Lott Place","employer":"Comtent","email":"jacobssweeney@comtent.com","city":"Advance","state":"NJ"}
+{"index":{"_id":"752"}}
+{"account_number":752,"balance":14039,"firstname":"Jerry","lastname":"Rush","age":31,"gender":"M","address":"632 Dank Court","employer":"Ebidco","email":"jerryrush@ebidco.com","city":"Geyserville","state":"AR"}
+{"index":{"_id":"757"}}
+{"account_number":757,"balance":34628,"firstname":"Mccullough","lastname":"Moore","age":30,"gender":"F","address":"304 Hastings Street","employer":"Nikuda","email":"mcculloughmoore@nikuda.com","city":"Charco","state":"DC"}
+{"index":{"_id":"764"}}
+{"account_number":764,"balance":3728,"firstname":"Noemi","lastname":"Gill","age":30,"gender":"M","address":"427 Chester Street","employer":"Avit","email":"noemigill@avit.com","city":"Chesterfield","state":"AL"}
+{"index":{"_id":"769"}}
+{"account_number":769,"balance":15362,"firstname":"Francis","lastname":"Beck","age":28,"gender":"M","address":"454 Livingston Street","employer":"Furnafix","email":"francisbeck@furnafix.com","city":"Dunnavant","state":"HI"}
+{"index":{"_id":"771"}}
+{"account_number":771,"balance":32784,"firstname":"Jocelyn","lastname":"Boone","age":23,"gender":"M","address":"513 Division Avenue","employer":"Collaire","email":"jocelynboone@collaire.com","city":"Lisco","state":"VT"}
+{"index":{"_id":"776"}}
+{"account_number":776,"balance":29177,"firstname":"Duke","lastname":"Atkinson","age":24,"gender":"M","address":"520 Doscher Street","employer":"Tripsch","email":"dukeatkinson@tripsch.com","city":"Lafferty","state":"NC"}
+{"index":{"_id":"783"}}
+{"account_number":783,"balance":11911,"firstname":"Faith","lastname":"Cooper","age":25,"gender":"F","address":"539 Rapelye Street","employer":"Insuron","email":"faithcooper@insuron.com","city":"Jennings","state":"MN"}
+{"index":{"_id":"788"}}
+{"account_number":788,"balance":12473,"firstname":"Marianne","lastname":"Aguilar","age":39,"gender":"F","address":"213 Holly Street","employer":"Marqet","email":"marianneaguilar@marqet.com","city":"Alfarata","state":"HI"}
+{"index":{"_id":"790"}}
+{"account_number":790,"balance":29912,"firstname":"Ellis","lastname":"Sullivan","age":39,"gender":"F","address":"877 Coyle Street","employer":"Enersave","email":"ellissullivan@enersave.com","city":"Canby","state":"MS"}
+{"index":{"_id":"795"}}
+{"account_number":795,"balance":31450,"firstname":"Bruce","lastname":"Avila","age":34,"gender":"M","address":"865 Newkirk Placez","employer":"Plasmosis","email":"bruceavila@plasmosis.com","city":"Ada","state":"ID"}
+{"index":{"_id":"803"}}
+{"account_number":803,"balance":49567,"firstname":"Marissa","lastname":"Spears","age":25,"gender":"M","address":"963 Highland Avenue","employer":"Centregy","email":"marissaspears@centregy.com","city":"Bloomington","state":"MS"}
+{"index":{"_id":"808"}}
+{"account_number":808,"balance":11251,"firstname":"Nola","lastname":"Quinn","age":20,"gender":"M","address":"863 Wythe Place","employer":"Iplax","email":"nolaquinn@iplax.com","city":"Cuylerville","state":"NH"}
+{"index":{"_id":"810"}}
+{"account_number":810,"balance":10563,"firstname":"Alyssa","lastname":"Ortega","age":40,"gender":"M","address":"977 Clymer Street","employer":"Eventage","email":"alyssaortega@eventage.com","city":"Convent","state":"SC"}
+{"index":{"_id":"815"}}
+{"account_number":815,"balance":19336,"firstname":"Guthrie","lastname":"Morse","age":30,"gender":"M","address":"685 Vandalia Avenue","employer":"Gronk","email":"guthriemorse@gronk.com","city":"Fowlerville","state":"OR"}
+{"index":{"_id":"822"}}
+{"account_number":822,"balance":13024,"firstname":"Hicks","lastname":"Farrell","age":25,"gender":"M","address":"468 Middleton Street","employer":"Zolarex","email":"hicksfarrell@zolarex.com","city":"Columbus","state":"OR"}
+{"index":{"_id":"827"}}
+{"account_number":827,"balance":37536,"firstname":"Naomi","lastname":"Ball","age":29,"gender":"F","address":"319 Stewart Street","employer":"Isotronic","email":"naomiball@isotronic.com","city":"Trona","state":"NM"}
+{"index":{"_id":"834"}}
+{"account_number":834,"balance":38049,"firstname":"Sybil","lastname":"Carrillo","age":25,"gender":"M","address":"359 Baughman Place","employer":"Phuel","email":"sybilcarrillo@phuel.com","city":"Kohatk","state":"CT"}
+{"index":{"_id":"839"}}
+{"account_number":839,"balance":38292,"firstname":"Langley","lastname":"Neal","age":39,"gender":"F","address":"565 Newton Street","employer":"Liquidoc","email":"langleyneal@liquidoc.com","city":"Osage","state":"AL"}
+{"index":{"_id":"841"}}
+{"account_number":841,"balance":28291,"firstname":"Dalton","lastname":"Waters","age":21,"gender":"M","address":"859 Grand Street","employer":"Malathion","email":"daltonwaters@malathion.com","city":"Tonopah","state":"AZ"}
+{"index":{"_id":"846"}}
+{"account_number":846,"balance":35099,"firstname":"Maureen","lastname":"Glass","age":22,"gender":"M","address":"140 Amherst Street","employer":"Stelaecor","email":"maureenglass@stelaecor.com","city":"Cucumber","state":"IL"}
+{"index":{"_id":"853"}}
+{"account_number":853,"balance":38353,"firstname":"Travis","lastname":"Parks","age":40,"gender":"M","address":"930 Bay Avenue","employer":"Pyramax","email":"travisparks@pyramax.com","city":"Gadsden","state":"ND"}
+{"index":{"_id":"858"}}
+{"account_number":858,"balance":23194,"firstname":"Small","lastname":"Hatfield","age":36,"gender":"M","address":"593 Tennis Court","employer":"Letpro","email":"smallhatfield@letpro.com","city":"Haena","state":"KS"}
+{"index":{"_id":"860"}}
+{"account_number":860,"balance":23613,"firstname":"Clark","lastname":"Boyd","age":37,"gender":"M","address":"501 Rock Street","employer":"Deepends","email":"clarkboyd@deepends.com","city":"Whitewater","state":"MA"}
+{"index":{"_id":"865"}}
+{"account_number":865,"balance":10574,"firstname":"Cook","lastname":"Kelley","age":28,"gender":"F","address":"865 Lincoln Terrace","employer":"Quizmo","email":"cookkelley@quizmo.com","city":"Kansas","state":"KY"}
+{"index":{"_id":"872"}}
+{"account_number":872,"balance":26314,"firstname":"Jane","lastname":"Greer","age":36,"gender":"F","address":"717 Hewes Street","employer":"Newcube","email":"janegreer@newcube.com","city":"Delshire","state":"DE"}
+{"index":{"_id":"877"}}
+{"account_number":877,"balance":42879,"firstname":"Tracey","lastname":"Ruiz","age":34,"gender":"F","address":"141 Tompkins Avenue","employer":"Waab","email":"traceyruiz@waab.com","city":"Zeba","state":"NM"}
+{"index":{"_id":"884"}}
+{"account_number":884,"balance":29316,"firstname":"Reva","lastname":"Rosa","age":40,"gender":"M","address":"784 Greene Avenue","employer":"Urbanshee","email":"revarosa@urbanshee.com","city":"Bakersville","state":"MS"}
+{"index":{"_id":"889"}}
+{"account_number":889,"balance":26464,"firstname":"Fischer","lastname":"Klein","age":38,"gender":"F","address":"948 Juliana Place","employer":"Comtext","email":"fischerklein@comtext.com","city":"Jackpot","state":"PA"}
+{"index":{"_id":"891"}}
+{"account_number":891,"balance":34829,"firstname":"Jacobson","lastname":"Clemons","age":24,"gender":"F","address":"507 Wilson Street","employer":"Quilm","email":"jacobsonclemons@quilm.com","city":"Muir","state":"TX"}
+{"index":{"_id":"896"}}
+{"account_number":896,"balance":31947,"firstname":"Buckley","lastname":"Peterson","age":26,"gender":"M","address":"217 Beayer Place","employer":"Earwax","email":"buckleypeterson@earwax.com","city":"Franklin","state":"DE"}
+{"index":{"_id":"904"}}
+{"account_number":904,"balance":27707,"firstname":"Mendez","lastname":"Mcneil","age":26,"gender":"M","address":"431 Halsey Street","employer":"Macronaut","email":"mendezmcneil@macronaut.com","city":"Troy","state":"OK"}
+{"index":{"_id":"909"}}
+{"account_number":909,"balance":18421,"firstname":"Stark","lastname":"Lewis","age":36,"gender":"M","address":"409 Tilden Avenue","employer":"Frosnex","email":"starklewis@frosnex.com","city":"Axis","state":"CA"}
+{"index":{"_id":"911"}}
+{"account_number":911,"balance":42655,"firstname":"Annie","lastname":"Lyons","age":21,"gender":"M","address":"518 Woods Place","employer":"Enerforce","email":"annielyons@enerforce.com","city":"Stagecoach","state":"MA"}
+{"index":{"_id":"916"}}
+{"account_number":916,"balance":47887,"firstname":"Jarvis","lastname":"Alexander","age":40,"gender":"M","address":"406 Bergen Avenue","employer":"Equitax","email":"jarvisalexander@equitax.com","city":"Haring","state":"KY"}
+{"index":{"_id":"923"}}
+{"account_number":923,"balance":48466,"firstname":"Mueller","lastname":"Mckee","age":26,"gender":"M","address":"298 Ruby Street","employer":"Luxuria","email":"muellermckee@luxuria.com","city":"Coleville","state":"TN"}
+{"index":{"_id":"928"}}
+{"account_number":928,"balance":19611,"firstname":"Hester","lastname":"Copeland","age":22,"gender":"F","address":"425 Cropsey Avenue","employer":"Dymi","email":"hestercopeland@dymi.com","city":"Wolcott","state":"NE"}
+{"index":{"_id":"930"}}
+{"account_number":930,"balance":47257,"firstname":"Kinney","lastname":"Lawson","age":39,"gender":"M","address":"501 Raleigh Place","employer":"Neptide","email":"kinneylawson@neptide.com","city":"Deltaville","state":"MD"}
+{"index":{"_id":"935"}}
+{"account_number":935,"balance":4959,"firstname":"Flowers","lastname":"Robles","age":30,"gender":"M","address":"201 Hull Street","employer":"Xelegyl","email":"flowersrobles@xelegyl.com","city":"Rehrersburg","state":"AL"}
+{"index":{"_id":"942"}}
+{"account_number":942,"balance":21299,"firstname":"Hamilton","lastname":"Clayton","age":26,"gender":"M","address":"413 Debevoise Street","employer":"Architax","email":"hamiltonclayton@architax.com","city":"Terlingua","state":"NM"}
+{"index":{"_id":"947"}}
+{"account_number":947,"balance":22039,"firstname":"Virgie","lastname":"Garza","age":30,"gender":"M","address":"903 Matthews Court","employer":"Plasmox","email":"virgiegarza@plasmox.com","city":"Somerset","state":"WY"}
+{"index":{"_id":"954"}}
+{"account_number":954,"balance":49404,"firstname":"Jenna","lastname":"Martin","age":22,"gender":"M","address":"688 Hart Street","employer":"Zinca","email":"jennamartin@zinca.com","city":"Oasis","state":"MD"}
+{"index":{"_id":"959"}}
+{"account_number":959,"balance":34743,"firstname":"Shaffer","lastname":"Cervantes","age":40,"gender":"M","address":"931 Varick Avenue","employer":"Oceanica","email":"shaffercervantes@oceanica.com","city":"Bowie","state":"AL"}
+{"index":{"_id":"961"}}
+{"account_number":961,"balance":43219,"firstname":"Betsy","lastname":"Hyde","age":27,"gender":"F","address":"183 Junius Street","employer":"Tubalum","email":"betsyhyde@tubalum.com","city":"Driftwood","state":"TX"}
+{"index":{"_id":"966"}}
+{"account_number":966,"balance":20619,"firstname":"Susanne","lastname":"Rodriguez","age":35,"gender":"F","address":"255 Knickerbocker Avenue","employer":"Comtrek","email":"susannerodriguez@comtrek.com","city":"Trinway","state":"TX"}
+{"index":{"_id":"973"}}
+{"account_number":973,"balance":45756,"firstname":"Rice","lastname":"Farmer","age":31,"gender":"M","address":"476 Nassau Avenue","employer":"Photobin","email":"ricefarmer@photobin.com","city":"Suitland","state":"ME"}
+{"index":{"_id":"978"}}
+{"account_number":978,"balance":21459,"firstname":"Melanie","lastname":"Rojas","age":33,"gender":"M","address":"991 Java Street","employer":"Kage","email":"melanierojas@kage.com","city":"Greenock","state":"VT"}
+{"index":{"_id":"980"}}
+{"account_number":980,"balance":42436,"firstname":"Cash","lastname":"Collier","age":33,"gender":"F","address":"999 Sapphire Street","employer":"Ceprene","email":"cashcollier@ceprene.com","city":"Glidden","state":"AK"}
+{"index":{"_id":"985"}}
+{"account_number":985,"balance":20083,"firstname":"Martin","lastname":"Gardner","age":28,"gender":"F","address":"644 Fairview Place","employer":"Golistic","email":"martingardner@golistic.com","city":"Connerton","state":"NJ"}
+{"index":{"_id":"992"}}
+{"account_number":992,"balance":11413,"firstname":"Kristie","lastname":"Kennedy","age":33,"gender":"F","address":"750 Hudson Avenue","employer":"Ludak","email":"kristiekennedy@ludak.com","city":"Warsaw","state":"WY"}
+{"index":{"_id":"997"}}
+{"account_number":997,"balance":25311,"firstname":"Combs","lastname":"Frederick","age":20,"gender":"M","address":"586 Lloyd Court","employer":"Pathways","email":"combsfrederick@pathways.com","city":"Williamson","state":"CA"}
+{"index":{"_id":"3"}}
+{"account_number":3,"balance":44947,"firstname":"Levine","lastname":"Burks","age":26,"gender":"F","address":"328 Wilson Avenue","employer":"Amtap","email":"levineburks@amtap.com","city":"Cochranville","state":"HI"}
+{"index":{"_id":"8"}}
+{"account_number":8,"balance":48868,"firstname":"Jan","lastname":"Burns","age":35,"gender":"M","address":"699 Visitation Place","employer":"Glasstep","email":"janburns@glasstep.com","city":"Wakulla","state":"AZ"}
+{"index":{"_id":"10"}}
+{"account_number":10,"balance":46170,"firstname":"Dominique","lastname":"Park","age":37,"gender":"F","address":"100 Gatling Place","employer":"Conjurica","email":"dominiquepark@conjurica.com","city":"Omar","state":"NJ"}
+{"index":{"_id":"15"}}
+{"account_number":15,"balance":43456,"firstname":"Bobbie","lastname":"Sexton","age":21,"gender":"M","address":"232 Sedgwick Place","employer":"Zytrex","email":"bobbiesexton@zytrex.com","city":"Hendersonville","state":"CA"}
+{"index":{"_id":"22"}}
+{"account_number":22,"balance":40283,"firstname":"Barrera","lastname":"Terrell","age":23,"gender":"F","address":"292 Orange Street","employer":"Steelfab","email":"barreraterrell@steelfab.com","city":"Bynum","state":"ME"}
+{"index":{"_id":"27"}}
+{"account_number":27,"balance":6176,"firstname":"Meyers","lastname":"Williamson","age":26,"gender":"F","address":"675 Henderson Walk","employer":"Plexia","email":"meyerswilliamson@plexia.com","city":"Richmond","state":"AZ"}
+{"index":{"_id":"34"}}
+{"account_number":34,"balance":35379,"firstname":"Ellison","lastname":"Kim","age":30,"gender":"F","address":"986 Revere Place","employer":"Signity","email":"ellisonkim@signity.com","city":"Sehili","state":"IL"}
+{"index":{"_id":"39"}}
+{"account_number":39,"balance":38688,"firstname":"Bowers","lastname":"Mendez","age":22,"gender":"F","address":"665 Bennet Court","employer":"Farmage","email":"bowersmendez@farmage.com","city":"Duryea","state":"PA"}
+{"index":{"_id":"41"}}
+{"account_number":41,"balance":36060,"firstname":"Hancock","lastname":"Holden","age":20,"gender":"M","address":"625 Gaylord Drive","employer":"Poochies","email":"hancockholden@poochies.com","city":"Alamo","state":"KS"}
+{"index":{"_id":"46"}}
+{"account_number":46,"balance":12351,"firstname":"Karla","lastname":"Bowman","age":23,"gender":"M","address":"554 Chapel Street","employer":"Undertap","email":"karlabowman@undertap.com","city":"Sylvanite","state":"DC"}
+{"index":{"_id":"53"}}
+{"account_number":53,"balance":28101,"firstname":"Kathryn","lastname":"Payne","age":29,"gender":"F","address":"467 Louis Place","employer":"Katakana","email":"kathrynpayne@katakana.com","city":"Harviell","state":"SD"}
+{"index":{"_id":"58"}}
+{"account_number":58,"balance":31697,"firstname":"Marva","lastname":"Cannon","age":40,"gender":"M","address":"993 Highland Place","employer":"Comcubine","email":"marvacannon@comcubine.com","city":"Orviston","state":"MO"}
+{"index":{"_id":"60"}}
+{"account_number":60,"balance":45955,"firstname":"Maude","lastname":"Casey","age":31,"gender":"F","address":"566 Strauss Street","employer":"Quilch","email":"maudecasey@quilch.com","city":"Enlow","state":"GA"}
+{"index":{"_id":"65"}}
+{"account_number":65,"balance":23282,"firstname":"Leonor","lastname":"Pruitt","age":24,"gender":"M","address":"974 Terrace Place","employer":"Velos","email":"leonorpruitt@velos.com","city":"Devon","state":"WI"}
+{"index":{"_id":"72"}}
+{"account_number":72,"balance":9732,"firstname":"Barlow","lastname":"Rhodes","age":25,"gender":"F","address":"891 Clinton Avenue","employer":"Zialactic","email":"barlowrhodes@zialactic.com","city":"Echo","state":"TN"}
+{"index":{"_id":"77"}}
+{"account_number":77,"balance":5724,"firstname":"Byrd","lastname":"Conley","age":24,"gender":"F","address":"698 Belmont Avenue","employer":"Zidox","email":"byrdconley@zidox.com","city":"Rockbridge","state":"SC"}
+{"index":{"_id":"84"}}
+{"account_number":84,"balance":3001,"firstname":"Hutchinson","lastname":"Newton","age":34,"gender":"F","address":"553 Locust Street","employer":"Zaggles","email":"hutchinsonnewton@zaggles.com","city":"Snyderville","state":"DC"}
+{"index":{"_id":"89"}}
+{"account_number":89,"balance":13263,"firstname":"Mcdowell","lastname":"Bradley","age":28,"gender":"M","address":"960 Howard Alley","employer":"Grok","email":"mcdowellbradley@grok.com","city":"Toftrees","state":"TX"}
+{"index":{"_id":"91"}}
+{"account_number":91,"balance":29799,"firstname":"Vonda","lastname":"Galloway","age":20,"gender":"M","address":"988 Voorhies Avenue","employer":"Illumity","email":"vondagalloway@illumity.com","city":"Holcombe","state":"HI"}
+{"index":{"_id":"96"}}
+{"account_number":96,"balance":15933,"firstname":"Shirley","lastname":"Edwards","age":38,"gender":"M","address":"817 Caton Avenue","employer":"Equitox","email":"shirleyedwards@equitox.com","city":"Nelson","state":"MA"}
+{"index":{"_id":"104"}}
+{"account_number":104,"balance":32619,"firstname":"Casey","lastname":"Roth","age":29,"gender":"M","address":"963 Railroad Avenue","employer":"Hotcakes","email":"caseyroth@hotcakes.com","city":"Davenport","state":"OH"}
+{"index":{"_id":"109"}}
+{"account_number":109,"balance":25812,"firstname":"Gretchen","lastname":"Dawson","age":31,"gender":"M","address":"610 Bethel Loop","employer":"Tetak","email":"gretchendawson@tetak.com","city":"Hailesboro","state":"CO"}
+{"index":{"_id":"111"}}
+{"account_number":111,"balance":1481,"firstname":"Traci","lastname":"Allison","age":35,"gender":"M","address":"922 Bryant Street","employer":"Enjola","email":"traciallison@enjola.com","city":"Robinette","state":"OR"}
+{"index":{"_id":"116"}}
+{"account_number":116,"balance":21335,"firstname":"Hobbs","lastname":"Wright","age":24,"gender":"M","address":"965 Temple Court","employer":"Netbook","email":"hobbswright@netbook.com","city":"Strong","state":"CA"}
+{"index":{"_id":"123"}}
+{"account_number":123,"balance":3079,"firstname":"Cleo","lastname":"Beach","age":27,"gender":"F","address":"653 Haring Street","employer":"Proxsoft","email":"cleobeach@proxsoft.com","city":"Greensburg","state":"ME"}
+{"index":{"_id":"128"}}
+{"account_number":128,"balance":3556,"firstname":"Mack","lastname":"Bullock","age":34,"gender":"F","address":"462 Ingraham Street","employer":"Terascape","email":"mackbullock@terascape.com","city":"Eureka","state":"PA"}
+{"index":{"_id":"130"}}
+{"account_number":130,"balance":24171,"firstname":"Roxie","lastname":"Cantu","age":33,"gender":"M","address":"841 Catherine Street","employer":"Skybold","email":"roxiecantu@skybold.com","city":"Deputy","state":"NE"}
+{"index":{"_id":"135"}}
+{"account_number":135,"balance":24885,"firstname":"Stevenson","lastname":"Crosby","age":40,"gender":"F","address":"473 Boardwalk ","employer":"Accel","email":"stevensoncrosby@accel.com","city":"Norris","state":"OK"}
+{"index":{"_id":"142"}}
+{"account_number":142,"balance":4544,"firstname":"Vang","lastname":"Hughes","age":27,"gender":"M","address":"357 Landis Court","employer":"Bolax","email":"vanghughes@bolax.com","city":"Emerald","state":"WY"}
+{"index":{"_id":"147"}}
+{"account_number":147,"balance":35921,"firstname":"Charmaine","lastname":"Whitney","age":28,"gender":"F","address":"484 Seton Place","employer":"Comveyer","email":"charmainewhitney@comveyer.com","city":"Dexter","state":"DC"}
+{"index":{"_id":"154"}}
+{"account_number":154,"balance":40945,"firstname":"Burns","lastname":"Solis","age":31,"gender":"M","address":"274 Lorraine Street","employer":"Rodemco","email":"burnssolis@rodemco.com","city":"Ballico","state":"WI"}
+{"index":{"_id":"159"}}
+{"account_number":159,"balance":1696,"firstname":"Alvarez","lastname":"Mack","age":22,"gender":"F","address":"897 Manor Court","employer":"Snorus","email":"alvarezmack@snorus.com","city":"Rosedale","state":"CA"}
+{"index":{"_id":"161"}}
+{"account_number":161,"balance":4659,"firstname":"Doreen","lastname":"Randall","age":37,"gender":"F","address":"178 Court Street","employer":"Calcula","email":"doreenrandall@calcula.com","city":"Belmont","state":"TX"}
+{"index":{"_id":"166"}}
+{"account_number":166,"balance":33847,"firstname":"Rutledge","lastname":"Rivas","age":23,"gender":"M","address":"352 Verona Street","employer":"Virxo","email":"rutledgerivas@virxo.com","city":"Brandermill","state":"NE"}
+{"index":{"_id":"173"}}
+{"account_number":173,"balance":5989,"firstname":"Whitley","lastname":"Blevins","age":32,"gender":"M","address":"127 Brooklyn Avenue","employer":"Pawnagra","email":"whitleyblevins@pawnagra.com","city":"Rodanthe","state":"ND"}
+{"index":{"_id":"178"}}
+{"account_number":178,"balance":36735,"firstname":"Clements","lastname":"Finley","age":39,"gender":"F","address":"270 Story Court","employer":"Imaginart","email":"clementsfinley@imaginart.com","city":"Lookingglass","state":"MN"}
+{"index":{"_id":"180"}}
+{"account_number":180,"balance":34236,"firstname":"Ursula","lastname":"Goodman","age":32,"gender":"F","address":"414 Clinton Street","employer":"Earthmark","email":"ursulagoodman@earthmark.com","city":"Rote","state":"AR"}
+{"index":{"_id":"185"}}
+{"account_number":185,"balance":43532,"firstname":"Laurel","lastname":"Cline","age":40,"gender":"M","address":"788 Fenimore Street","employer":"Prismatic","email":"laurelcline@prismatic.com","city":"Frank","state":"UT"}
+{"index":{"_id":"192"}}
+{"account_number":192,"balance":23508,"firstname":"Ramsey","lastname":"Carr","age":31,"gender":"F","address":"209 Williamsburg Street","employer":"Strezzo","email":"ramseycarr@strezzo.com","city":"Grapeview","state":"NM"}
+{"index":{"_id":"197"}}
+{"account_number":197,"balance":17246,"firstname":"Sweet","lastname":"Sanders","age":33,"gender":"F","address":"712 Homecrest Court","employer":"Isosure","email":"sweetsanders@isosure.com","city":"Sheatown","state":"VT"}
+{"index":{"_id":"200"}}
+{"account_number":200,"balance":26210,"firstname":"Teri","lastname":"Hester","age":39,"gender":"M","address":"653 Abbey Court","employer":"Electonic","email":"terihester@electonic.com","city":"Martell","state":"MD"}
+{"index":{"_id":"205"}}
+{"account_number":205,"balance":45493,"firstname":"Johnson","lastname":"Chang","age":28,"gender":"F","address":"331 John Street","employer":"Gleamink","email":"johnsonchang@gleamink.com","city":"Sultana","state":"KS"}
+{"index":{"_id":"212"}}
+{"account_number":212,"balance":10299,"firstname":"Marisol","lastname":"Fischer","age":39,"gender":"M","address":"362 Prince Street","employer":"Autograte","email":"marisolfischer@autograte.com","city":"Oley","state":"SC"}
+{"index":{"_id":"217"}}
+{"account_number":217,"balance":33730,"firstname":"Sally","lastname":"Mccoy","age":38,"gender":"F","address":"854 Corbin Place","employer":"Omnigog","email":"sallymccoy@omnigog.com","city":"Escondida","state":"FL"}
+{"index":{"_id":"224"}}
+{"account_number":224,"balance":42708,"firstname":"Billie","lastname":"Nixon","age":28,"gender":"F","address":"241 Kaufman Place","employer":"Xanide","email":"billienixon@xanide.com","city":"Chapin","state":"NY"}
+{"index":{"_id":"229"}}
+{"account_number":229,"balance":2740,"firstname":"Jana","lastname":"Hensley","age":30,"gender":"M","address":"176 Erasmus Street","employer":"Isotrack","email":"janahensley@isotrack.com","city":"Caledonia","state":"ME"}
+{"index":{"_id":"231"}}
+{"account_number":231,"balance":46180,"firstname":"Essie","lastname":"Clarke","age":34,"gender":"F","address":"308 Harbor Lane","employer":"Pharmacon","email":"essieclarke@pharmacon.com","city":"Fillmore","state":"MS"}
+{"index":{"_id":"236"}}
+{"account_number":236,"balance":41200,"firstname":"Suzanne","lastname":"Bird","age":39,"gender":"F","address":"219 Luquer Street","employer":"Imant","email":"suzannebird@imant.com","city":"Bainbridge","state":"NY"}
+{"index":{"_id":"243"}}
+{"account_number":243,"balance":29902,"firstname":"Evangelina","lastname":"Perez","age":20,"gender":"M","address":"787 Joval Court","employer":"Keengen","email":"evangelinaperez@keengen.com","city":"Mulberry","state":"SD"}
+{"index":{"_id":"248"}}
+{"account_number":248,"balance":49989,"firstname":"West","lastname":"England","age":36,"gender":"M","address":"717 Hendrickson Place","employer":"Obliq","email":"westengland@obliq.com","city":"Maury","state":"WA"}
+{"index":{"_id":"250"}}
+{"account_number":250,"balance":27893,"firstname":"Earlene","lastname":"Ellis","age":39,"gender":"F","address":"512 Bay Street","employer":"Codact","email":"earleneellis@codact.com","city":"Sunwest","state":"GA"}
+{"index":{"_id":"255"}}
+{"account_number":255,"balance":49339,"firstname":"Iva","lastname":"Rivers","age":38,"gender":"M","address":"470 Rost Place","employer":"Mantrix","email":"ivarivers@mantrix.com","city":"Disautel","state":"MD"}
+{"index":{"_id":"262"}}
+{"account_number":262,"balance":30289,"firstname":"Tameka","lastname":"Levine","age":36,"gender":"F","address":"815 Atlantic Avenue","employer":"Acium","email":"tamekalevine@acium.com","city":"Winchester","state":"SD"}
+{"index":{"_id":"267"}}
+{"account_number":267,"balance":42753,"firstname":"Weeks","lastname":"Castillo","age":21,"gender":"F","address":"526 Holt Court","employer":"Talendula","email":"weekscastillo@talendula.com","city":"Washington","state":"NV"}
+{"index":{"_id":"274"}}
+{"account_number":274,"balance":12104,"firstname":"Frieda","lastname":"House","age":33,"gender":"F","address":"171 Banker Street","employer":"Quonk","email":"friedahouse@quonk.com","city":"Aberdeen","state":"NJ"}
+{"index":{"_id":"279"}}
+{"account_number":279,"balance":15904,"firstname":"Chapman","lastname":"Hart","age":32,"gender":"F","address":"902 Bliss Terrace","employer":"Kongene","email":"chapmanhart@kongene.com","city":"Bradenville","state":"NJ"}
+{"index":{"_id":"281"}}
+{"account_number":281,"balance":39830,"firstname":"Bean","lastname":"Aguirre","age":20,"gender":"F","address":"133 Pilling Street","employer":"Amril","email":"beanaguirre@amril.com","city":"Waterview","state":"TX"}
+{"index":{"_id":"286"}}
+{"account_number":286,"balance":39063,"firstname":"Rosetta","lastname":"Turner","age":35,"gender":"M","address":"169 Jefferson Avenue","employer":"Spacewax","email":"rosettaturner@spacewax.com","city":"Stewart","state":"MO"}
+{"index":{"_id":"293"}}
+{"account_number":293,"balance":29867,"firstname":"Cruz","lastname":"Carver","age":28,"gender":"F","address":"465 Boerum Place","employer":"Vitricomp","email":"cruzcarver@vitricomp.com","city":"Crayne","state":"CO"}
+{"index":{"_id":"298"}}
+{"account_number":298,"balance":34334,"firstname":"Bullock","lastname":"Marsh","age":20,"gender":"M","address":"589 Virginia Place","employer":"Renovize","email":"bullockmarsh@renovize.com","city":"Coinjock","state":"UT"}
+{"index":{"_id":"301"}}
+{"account_number":301,"balance":16782,"firstname":"Minerva","lastname":"Graham","age":35,"gender":"M","address":"532 Harrison Place","employer":"Sureplex","email":"minervagraham@sureplex.com","city":"Belleview","state":"GA"}
+{"index":{"_id":"306"}}
+{"account_number":306,"balance":2171,"firstname":"Hensley","lastname":"Hardin","age":40,"gender":"M","address":"196 Maujer Street","employer":"Neocent","email":"hensleyhardin@neocent.com","city":"Reinerton","state":"HI"}
+{"index":{"_id":"313"}}
+{"account_number":313,"balance":34108,"firstname":"Alston","lastname":"Henderson","age":36,"gender":"F","address":"132 Prescott Place","employer":"Prosure","email":"alstonhenderson@prosure.com","city":"Worton","state":"IA"}
+{"index":{"_id":"318"}}
+{"account_number":318,"balance":8512,"firstname":"Nichole","lastname":"Pearson","age":34,"gender":"F","address":"656 Lacon Court","employer":"Yurture","email":"nicholepearson@yurture.com","city":"Juarez","state":"MO"}
+{"index":{"_id":"320"}}
+{"account_number":320,"balance":34521,"firstname":"Patti","lastname":"Brennan","age":37,"gender":"F","address":"870 Degraw Street","employer":"Cognicode","email":"pattibrennan@cognicode.com","city":"Torboy","state":"FL"}
+{"index":{"_id":"325"}}
+{"account_number":325,"balance":1956,"firstname":"Magdalena","lastname":"Simmons","age":25,"gender":"F","address":"681 Townsend Street","employer":"Geekosis","email":"magdalenasimmons@geekosis.com","city":"Sterling","state":"CA"}
+{"index":{"_id":"332"}}
+{"account_number":332,"balance":37770,"firstname":"Shepherd","lastname":"Davenport","age":28,"gender":"F","address":"586 Montague Terrace","employer":"Ecraze","email":"shepherddavenport@ecraze.com","city":"Accoville","state":"NM"}
+{"index":{"_id":"337"}}
+{"account_number":337,"balance":43432,"firstname":"Monroe","lastname":"Stafford","age":37,"gender":"F","address":"183 Seigel Street","employer":"Centuria","email":"monroestafford@centuria.com","city":"Camino","state":"DE"}
+{"index":{"_id":"344"}}
+{"account_number":344,"balance":42654,"firstname":"Sasha","lastname":"Baxter","age":35,"gender":"F","address":"700 Bedford Place","employer":"Callflex","email":"sashabaxter@callflex.com","city":"Campo","state":"MI"}
+{"index":{"_id":"349"}}
+{"account_number":349,"balance":24180,"firstname":"Allison","lastname":"Fitzpatrick","age":22,"gender":"F","address":"913 Arlington Avenue","employer":"Veraq","email":"allisonfitzpatrick@veraq.com","city":"Marbury","state":"TX"}
+{"index":{"_id":"351"}}
+{"account_number":351,"balance":47089,"firstname":"Hendrix","lastname":"Stephens","age":29,"gender":"M","address":"181 Beaver Street","employer":"Recrisys","email":"hendrixstephens@recrisys.com","city":"Denio","state":"OR"}
+{"index":{"_id":"356"}}
+{"account_number":356,"balance":34540,"firstname":"Lourdes","lastname":"Valdez","age":20,"gender":"F","address":"700 Anchorage Place","employer":"Interloo","email":"lourdesvaldez@interloo.com","city":"Goldfield","state":"OK"}
+{"index":{"_id":"363"}}
+{"account_number":363,"balance":34007,"firstname":"Peggy","lastname":"Bright","age":21,"gender":"M","address":"613 Engert Avenue","employer":"Inventure","email":"peggybright@inventure.com","city":"Chautauqua","state":"ME"}
+{"index":{"_id":"368"}}
+{"account_number":368,"balance":23535,"firstname":"Hooper","lastname":"Tyson","age":39,"gender":"M","address":"892 Taaffe Place","employer":"Zaggle","email":"hoopertyson@zaggle.com","city":"Nutrioso","state":"ME"}
+{"index":{"_id":"370"}}
+{"account_number":370,"balance":28499,"firstname":"Oneill","lastname":"Carney","age":25,"gender":"F","address":"773 Adelphi Street","employer":"Bedder","email":"oneillcarney@bedder.com","city":"Yorklyn","state":"FL"}
+{"index":{"_id":"375"}}
+{"account_number":375,"balance":23860,"firstname":"Phoebe","lastname":"Patton","age":25,"gender":"M","address":"564 Hale Avenue","employer":"Xoggle","email":"phoebepatton@xoggle.com","city":"Brule","state":"NM"}
+{"index":{"_id":"382"}}
+{"account_number":382,"balance":42061,"firstname":"Finley","lastname":"Singleton","age":37,"gender":"F","address":"407 Clay Street","employer":"Quarex","email":"finleysingleton@quarex.com","city":"Bedias","state":"LA"}
+{"index":{"_id":"387"}}
+{"account_number":387,"balance":35916,"firstname":"April","lastname":"Hill","age":29,"gender":"M","address":"818 Bayard Street","employer":"Kengen","email":"aprilhill@kengen.com","city":"Chloride","state":"NC"}
+{"index":{"_id":"394"}}
+{"account_number":394,"balance":6121,"firstname":"Lorrie","lastname":"Nunez","age":38,"gender":"M","address":"221 Ralph Avenue","employer":"Bullzone","email":"lorrienunez@bullzone.com","city":"Longoria","state":"ID"}
+{"index":{"_id":"399"}}
+{"account_number":399,"balance":32587,"firstname":"Carmela","lastname":"Franks","age":23,"gender":"M","address":"617 Dewey Place","employer":"Zensure","email":"carmelafranks@zensure.com","city":"Sanders","state":"DC"}
+{"index":{"_id":"402"}}
+{"account_number":402,"balance":1282,"firstname":"Pacheco","lastname":"Rosales","age":32,"gender":"M","address":"538 Pershing Loop","employer":"Circum","email":"pachecorosales@circum.com","city":"Elbert","state":"ID"}
+{"index":{"_id":"407"}}
+{"account_number":407,"balance":36417,"firstname":"Gilda","lastname":"Jacobson","age":29,"gender":"F","address":"883 Loring Avenue","employer":"Comveyor","email":"gildajacobson@comveyor.com","city":"Topaz","state":"NH"}
+{"index":{"_id":"414"}}
+{"account_number":414,"balance":17506,"firstname":"Conway","lastname":"Daugherty","age":37,"gender":"F","address":"643 Kermit Place","employer":"Lyria","email":"conwaydaugherty@lyria.com","city":"Vaughn","state":"NV"}
+{"index":{"_id":"419"}}
+{"account_number":419,"balance":34847,"firstname":"Helen","lastname":"Montoya","age":29,"gender":"F","address":"736 Kingsland Avenue","employer":"Hairport","email":"helenmontoya@hairport.com","city":"Edinburg","state":"NE"}
+{"index":{"_id":"421"}}
+{"account_number":421,"balance":46868,"firstname":"Tamika","lastname":"Mccall","age":27,"gender":"F","address":"764 Bragg Court","employer":"Eventix","email":"tamikamccall@eventix.com","city":"Tivoli","state":"RI"}
+{"index":{"_id":"426"}}
+{"account_number":426,"balance":4499,"firstname":"Julie","lastname":"Parsons","age":31,"gender":"M","address":"768 Keap Street","employer":"Goko","email":"julieparsons@goko.com","city":"Coldiron","state":"VA"}
+{"index":{"_id":"433"}}
+{"account_number":433,"balance":19266,"firstname":"Wilkinson","lastname":"Flowers","age":39,"gender":"M","address":"154 Douglass Street","employer":"Xsports","email":"wilkinsonflowers@xsports.com","city":"Coultervillle","state":"MN"}
+{"index":{"_id":"438"}}
+{"account_number":438,"balance":16367,"firstname":"Walter","lastname":"Velez","age":27,"gender":"F","address":"931 Farragut Road","employer":"Virva","email":"waltervelez@virva.com","city":"Tyro","state":"WV"}
+{"index":{"_id":"440"}}
+{"account_number":440,"balance":41590,"firstname":"Ray","lastname":"Wiley","age":31,"gender":"F","address":"102 Barwell Terrace","employer":"Polaria","email":"raywiley@polaria.com","city":"Hardyville","state":"IA"}
+{"index":{"_id":"445"}}
+{"account_number":445,"balance":41178,"firstname":"Rodriguez","lastname":"Macias","age":34,"gender":"M","address":"164 Boerum Street","employer":"Xylar","email":"rodriguezmacias@xylar.com","city":"Riner","state":"AL"}
+{"index":{"_id":"452"}}
+{"account_number":452,"balance":3589,"firstname":"Blackwell","lastname":"Delaney","age":39,"gender":"F","address":"443 Sackett Street","employer":"Imkan","email":"blackwelldelaney@imkan.com","city":"Gasquet","state":"DC"}
+{"index":{"_id":"457"}}
+{"account_number":457,"balance":14057,"firstname":"Bush","lastname":"Gordon","age":34,"gender":"M","address":"975 Dakota Place","employer":"Softmicro","email":"bushgordon@softmicro.com","city":"Chemung","state":"PA"}
+{"index":{"_id":"464"}}
+{"account_number":464,"balance":20504,"firstname":"Cobb","lastname":"Humphrey","age":21,"gender":"M","address":"823 Sunnyside Avenue","employer":"Apexia","email":"cobbhumphrey@apexia.com","city":"Wintersburg","state":"NY"}
+{"index":{"_id":"469"}}
+{"account_number":469,"balance":26509,"firstname":"Marci","lastname":"Shepherd","age":26,"gender":"M","address":"565 Hall Street","employer":"Shadease","email":"marcishepherd@shadease.com","city":"Springhill","state":"IL"}
+{"index":{"_id":"471"}}
+{"account_number":471,"balance":7629,"firstname":"Juana","lastname":"Silva","age":36,"gender":"M","address":"249 Amity Street","employer":"Artworlds","email":"juanasilva@artworlds.com","city":"Norfolk","state":"TX"}
+{"index":{"_id":"476"}}
+{"account_number":476,"balance":33386,"firstname":"Silva","lastname":"Marks","age":31,"gender":"F","address":"183 Eldert Street","employer":"Medifax","email":"silvamarks@medifax.com","city":"Hachita","state":"RI"}
+{"index":{"_id":"483"}}
+{"account_number":483,"balance":6344,"firstname":"Kelley","lastname":"Harper","age":29,"gender":"M","address":"758 Preston Court","employer":"Xyqag","email":"kelleyharper@xyqag.com","city":"Healy","state":"IA"}
+{"index":{"_id":"488"}}
+{"account_number":488,"balance":6289,"firstname":"Wilma","lastname":"Hopkins","age":38,"gender":"M","address":"428 Lee Avenue","employer":"Entality","email":"wilmahopkins@entality.com","city":"Englevale","state":"WI"}
+{"index":{"_id":"490"}}
+{"account_number":490,"balance":1447,"firstname":"Strong","lastname":"Hendrix","age":26,"gender":"F","address":"134 Beach Place","employer":"Duoflex","email":"stronghendrix@duoflex.com","city":"Allentown","state":"ND"}
+{"index":{"_id":"495"}}
+{"account_number":495,"balance":13478,"firstname":"Abigail","lastname":"Nichols","age":40,"gender":"F","address":"887 President Street","employer":"Enquility","email":"abigailnichols@enquility.com","city":"Bagtown","state":"NM"}
+{"index":{"_id":"503"}}
+{"account_number":503,"balance":42649,"firstname":"Leta","lastname":"Stout","age":39,"gender":"F","address":"518 Bowery Street","employer":"Pivitol","email":"letastout@pivitol.com","city":"Boonville","state":"ND"}
+{"index":{"_id":"508"}}
+{"account_number":508,"balance":41300,"firstname":"Lawrence","lastname":"Mathews","age":27,"gender":"F","address":"987 Rose Street","employer":"Deviltoe","email":"lawrencemathews@deviltoe.com","city":"Woodburn","state":"FL"}
+{"index":{"_id":"510"}}
+{"account_number":510,"balance":48504,"firstname":"Petty","lastname":"Sykes","age":28,"gender":"M","address":"566 Village Road","employer":"Nebulean","email":"pettysykes@nebulean.com","city":"Wedgewood","state":"MO"}
+{"index":{"_id":"515"}}
+{"account_number":515,"balance":18531,"firstname":"Lott","lastname":"Keller","age":27,"gender":"M","address":"827 Miami Court","employer":"Translink","email":"lottkeller@translink.com","city":"Gila","state":"TX"}
+{"index":{"_id":"522"}}
+{"account_number":522,"balance":19879,"firstname":"Faulkner","lastname":"Garrett","age":29,"gender":"F","address":"396 Grove Place","employer":"Pigzart","email":"faulknergarrett@pigzart.com","city":"Felt","state":"AR"}
+{"index":{"_id":"527"}}
+{"account_number":527,"balance":2028,"firstname":"Carver","lastname":"Peters","age":35,"gender":"M","address":"816 Victor Road","employer":"Housedown","email":"carverpeters@housedown.com","city":"Nadine","state":"MD"}
+{"index":{"_id":"534"}}
+{"account_number":534,"balance":20470,"firstname":"Cristina","lastname":"Russo","age":25,"gender":"F","address":"500 Highlawn Avenue","employer":"Cyclonica","email":"cristinarusso@cyclonica.com","city":"Gorst","state":"KS"}
+{"index":{"_id":"539"}}
+{"account_number":539,"balance":24560,"firstname":"Tami","lastname":"Maddox","age":23,"gender":"F","address":"741 Pineapple Street","employer":"Accidency","email":"tamimaddox@accidency.com","city":"Kennedyville","state":"OH"}
+{"index":{"_id":"541"}}
+{"account_number":541,"balance":42915,"firstname":"Logan","lastname":"Burke","age":32,"gender":"M","address":"904 Clarendon Road","employer":"Overplex","email":"loganburke@overplex.com","city":"Johnsonburg","state":"OH"}
+{"index":{"_id":"546"}}
+{"account_number":546,"balance":43242,"firstname":"Bernice","lastname":"Sims","age":33,"gender":"M","address":"382 Columbia Street","employer":"Verbus","email":"bernicesims@verbus.com","city":"Sena","state":"KY"}
+{"index":{"_id":"553"}}
+{"account_number":553,"balance":28390,"firstname":"Aimee","lastname":"Cohen","age":28,"gender":"M","address":"396 Lafayette Avenue","employer":"Eplode","email":"aimeecohen@eplode.com","city":"Thatcher","state":"NJ"}
+{"index":{"_id":"558"}}
+{"account_number":558,"balance":8922,"firstname":"Horne","lastname":"Valenzuela","age":20,"gender":"F","address":"979 Kensington Street","employer":"Isoternia","email":"hornevalenzuela@isoternia.com","city":"Greenbush","state":"NC"}
+{"index":{"_id":"560"}}
+{"account_number":560,"balance":24514,"firstname":"Felecia","lastname":"Oneill","age":26,"gender":"M","address":"995 Autumn Avenue","employer":"Mediot","email":"feleciaoneill@mediot.com","city":"Joppa","state":"IN"}
+{"index":{"_id":"565"}}
+{"account_number":565,"balance":15197,"firstname":"Taylor","lastname":"Ingram","age":37,"gender":"F","address":"113 Will Place","employer":"Lyrichord","email":"tayloringram@lyrichord.com","city":"Collins","state":"ME"}
+{"index":{"_id":"572"}}
+{"account_number":572,"balance":49355,"firstname":"Therese","lastname":"Espinoza","age":20,"gender":"M","address":"994 Chester Court","employer":"Gonkle","email":"thereseespinoza@gonkle.com","city":"Hayes","state":"UT"}
+{"index":{"_id":"577"}}
+{"account_number":577,"balance":21398,"firstname":"Gilbert","lastname":"Serrano","age":38,"gender":"F","address":"294 Troutman Street","employer":"Senmao","email":"gilbertserrano@senmao.com","city":"Greer","state":"MT"}
+{"index":{"_id":"584"}}
+{"account_number":584,"balance":5346,"firstname":"Pearson","lastname":"Bryant","age":40,"gender":"F","address":"971 Heyward Street","employer":"Anacho","email":"pearsonbryant@anacho.com","city":"Bluffview","state":"MN"}
+{"index":{"_id":"589"}}
+{"account_number":589,"balance":33260,"firstname":"Ericka","lastname":"Cote","age":39,"gender":"F","address":"425 Bath Avenue","employer":"Venoflex","email":"erickacote@venoflex.com","city":"Blue","state":"CT"}
+{"index":{"_id":"591"}}
+{"account_number":591,"balance":48997,"firstname":"Rivers","lastname":"Macdonald","age":34,"gender":"F","address":"919 Johnson Street","employer":"Ziore","email":"riversmacdonald@ziore.com","city":"Townsend","state":"IL"}
+{"index":{"_id":"596"}}
+{"account_number":596,"balance":4063,"firstname":"Letitia","lastname":"Walker","age":26,"gender":"F","address":"963 Vanderveer Place","employer":"Zizzle","email":"letitiawalker@zizzle.com","city":"Rossmore","state":"ID"}
+{"index":{"_id":"604"}}
+{"account_number":604,"balance":10675,"firstname":"Isabel","lastname":"Gilliam","age":23,"gender":"M","address":"854 Broadway ","employer":"Zenthall","email":"isabelgilliam@zenthall.com","city":"Ventress","state":"WI"}
+{"index":{"_id":"609"}}
+{"account_number":609,"balance":28586,"firstname":"Montgomery","lastname":"Washington","age":30,"gender":"M","address":"169 Schroeders Avenue","employer":"Kongle","email":"montgomerywashington@kongle.com","city":"Croom","state":"AZ"}
+{"index":{"_id":"611"}}
+{"account_number":611,"balance":17528,"firstname":"Katherine","lastname":"Prince","age":33,"gender":"F","address":"705 Elm Avenue","employer":"Zillacon","email":"katherineprince@zillacon.com","city":"Rew","state":"MI"}
+{"index":{"_id":"616"}}
+{"account_number":616,"balance":25276,"firstname":"Jessie","lastname":"Mayer","age":35,"gender":"F","address":"683 Chester Avenue","employer":"Emtrak","email":"jessiemayer@emtrak.com","city":"Marysville","state":"HI"}
+{"index":{"_id":"623"}}
+{"account_number":623,"balance":20514,"firstname":"Rose","lastname":"Combs","age":32,"gender":"F","address":"312 Grimes Road","employer":"Aquamate","email":"rosecombs@aquamate.com","city":"Fostoria","state":"OH"}
+{"index":{"_id":"628"}}
+{"account_number":628,"balance":42736,"firstname":"Buckner","lastname":"Chen","age":37,"gender":"M","address":"863 Rugby Road","employer":"Jamnation","email":"bucknerchen@jamnation.com","city":"Camas","state":"TX"}
+{"index":{"_id":"630"}}
+{"account_number":630,"balance":46060,"firstname":"Leanne","lastname":"Jones","age":31,"gender":"M","address":"451 Bayview Avenue","employer":"Wazzu","email":"leannejones@wazzu.com","city":"Kylertown","state":"OK"}
+{"index":{"_id":"635"}}
+{"account_number":635,"balance":44705,"firstname":"Norman","lastname":"Gilmore","age":33,"gender":"M","address":"330 Gates Avenue","employer":"Comfirm","email":"normangilmore@comfirm.com","city":"Riceville","state":"TN"}
+{"index":{"_id":"642"}}
+{"account_number":642,"balance":32852,"firstname":"Reyna","lastname":"Harris","age":35,"gender":"M","address":"305 Powell Street","employer":"Bedlam","email":"reynaharris@bedlam.com","city":"Florence","state":"KS"}
+{"index":{"_id":"647"}}
+{"account_number":647,"balance":10147,"firstname":"Annabelle","lastname":"Velazquez","age":30,"gender":"M","address":"299 Kensington Walk","employer":"Sealoud","email":"annabellevelazquez@sealoud.com","city":"Soudan","state":"ME"}
+{"index":{"_id":"654"}}
+{"account_number":654,"balance":38695,"firstname":"Armstrong","lastname":"Frazier","age":25,"gender":"M","address":"899 Seeley Street","employer":"Zensor","email":"armstrongfrazier@zensor.com","city":"Cherokee","state":"UT"}
+{"index":{"_id":"659"}}
+{"account_number":659,"balance":29648,"firstname":"Dorsey","lastname":"Sosa","age":40,"gender":"M","address":"270 Aberdeen Street","employer":"Daycore","email":"dorseysosa@daycore.com","city":"Chamberino","state":"SC"}
+{"index":{"_id":"661"}}
+{"account_number":661,"balance":3679,"firstname":"Joanne","lastname":"Spencer","age":39,"gender":"F","address":"910 Montauk Avenue","employer":"Visalia","email":"joannespencer@visalia.com","city":"Valmy","state":"NH"}
+{"index":{"_id":"666"}}
+{"account_number":666,"balance":13880,"firstname":"Mcguire","lastname":"Lloyd","age":40,"gender":"F","address":"658 Just Court","employer":"Centrexin","email":"mcguirelloyd@centrexin.com","city":"Warren","state":"MT"}
+{"index":{"_id":"673"}}
+{"account_number":673,"balance":11303,"firstname":"Mcdaniel","lastname":"Harrell","age":33,"gender":"M","address":"565 Montgomery Place","employer":"Eyeris","email":"mcdanielharrell@eyeris.com","city":"Garnet","state":"NV"}
+{"index":{"_id":"678"}}
+{"account_number":678,"balance":43663,"firstname":"Ruby","lastname":"Shaffer","age":28,"gender":"M","address":"350 Clark Street","employer":"Comtrail","email":"rubyshaffer@comtrail.com","city":"Aurora","state":"MA"}
+{"index":{"_id":"680"}}
+{"account_number":680,"balance":31561,"firstname":"Melton","lastname":"Camacho","age":32,"gender":"F","address":"771 Montana Place","employer":"Insuresys","email":"meltoncamacho@insuresys.com","city":"Sparkill","state":"IN"}
+{"index":{"_id":"685"}}
+{"account_number":685,"balance":22249,"firstname":"Yesenia","lastname":"Rowland","age":24,"gender":"F","address":"193 Dekalb Avenue","employer":"Coriander","email":"yeseniarowland@coriander.com","city":"Lupton","state":"NC"}
+{"index":{"_id":"692"}}
+{"account_number":692,"balance":10435,"firstname":"Haney","lastname":"Barlow","age":21,"gender":"F","address":"267 Lenox Road","employer":"Egypto","email":"haneybarlow@egypto.com","city":"Detroit","state":"IN"}
+{"index":{"_id":"697"}}
+{"account_number":697,"balance":48745,"firstname":"Mallory","lastname":"Emerson","age":24,"gender":"F","address":"318 Dunne Court","employer":"Exoplode","email":"malloryemerson@exoplode.com","city":"Montura","state":"LA"}
+{"index":{"_id":"700"}}
+{"account_number":700,"balance":19164,"firstname":"Patel","lastname":"Durham","age":21,"gender":"F","address":"440 King Street","employer":"Icology","email":"pateldurham@icology.com","city":"Mammoth","state":"IL"}
+{"index":{"_id":"705"}}
+{"account_number":705,"balance":28415,"firstname":"Krystal","lastname":"Cross","age":22,"gender":"M","address":"604 Drew Street","employer":"Tubesys","email":"krystalcross@tubesys.com","city":"Dalton","state":"MO"}
+{"index":{"_id":"712"}}
+{"account_number":712,"balance":12459,"firstname":"Butler","lastname":"Alston","age":37,"gender":"M","address":"486 Hemlock Street","employer":"Quordate","email":"butleralston@quordate.com","city":"Verdi","state":"MS"}
+{"index":{"_id":"717"}}
+{"account_number":717,"balance":29270,"firstname":"Erickson","lastname":"Mcdonald","age":31,"gender":"M","address":"873 Franklin Street","employer":"Exotechno","email":"ericksonmcdonald@exotechno.com","city":"Jessie","state":"MS"}
+{"index":{"_id":"724"}}
+{"account_number":724,"balance":12548,"firstname":"Hopper","lastname":"Peck","age":31,"gender":"M","address":"849 Hendrickson Street","employer":"Uxmox","email":"hopperpeck@uxmox.com","city":"Faxon","state":"UT"}
+{"index":{"_id":"729"}}
+{"account_number":729,"balance":41812,"firstname":"Katy","lastname":"Rivera","age":36,"gender":"F","address":"791 Olive Street","employer":"Blurrybus","email":"katyrivera@blurrybus.com","city":"Innsbrook","state":"MI"}
+{"index":{"_id":"731"}}
+{"account_number":731,"balance":4994,"firstname":"Lorene","lastname":"Weiss","age":35,"gender":"M","address":"990 Ocean Court","employer":"Comvoy","email":"loreneweiss@comvoy.com","city":"Lavalette","state":"WI"}
+{"index":{"_id":"736"}}
+{"account_number":736,"balance":28677,"firstname":"Rogers","lastname":"Mcmahon","age":21,"gender":"F","address":"423 Cameron Court","employer":"Brainclip","email":"rogersmcmahon@brainclip.com","city":"Saddlebrooke","state":"FL"}
+{"index":{"_id":"743"}}
+{"account_number":743,"balance":14077,"firstname":"Susana","lastname":"Moody","age":23,"gender":"M","address":"842 Fountain Avenue","employer":"Bitrex","email":"susanamoody@bitrex.com","city":"Temperanceville","state":"TN"}
+{"index":{"_id":"748"}}
+{"account_number":748,"balance":38060,"firstname":"Ford","lastname":"Branch","age":25,"gender":"M","address":"926 Cypress Avenue","employer":"Buzzness","email":"fordbranch@buzzness.com","city":"Beason","state":"DC"}
+{"index":{"_id":"750"}}
+{"account_number":750,"balance":40481,"firstname":"Cherie","lastname":"Brooks","age":20,"gender":"F","address":"601 Woodhull Street","employer":"Kaggle","email":"cheriebrooks@kaggle.com","city":"Groton","state":"MA"}
+{"index":{"_id":"755"}}
+{"account_number":755,"balance":43878,"firstname":"Bartlett","lastname":"Conway","age":22,"gender":"M","address":"453 Times Placez","employer":"Konnect","email":"bartlettconway@konnect.com","city":"Belva","state":"VT"}
+{"index":{"_id":"762"}}
+{"account_number":762,"balance":10291,"firstname":"Amanda","lastname":"Head","age":20,"gender":"F","address":"990 Ocean Parkway","employer":"Zentury","email":"amandahead@zentury.com","city":"Hegins","state":"AR"}
+{"index":{"_id":"767"}}
+{"account_number":767,"balance":26220,"firstname":"Anthony","lastname":"Sutton","age":27,"gender":"F","address":"179 Fayette Street","employer":"Xiix","email":"anthonysutton@xiix.com","city":"Iberia","state":"TN"}
+{"index":{"_id":"774"}}
+{"account_number":774,"balance":35287,"firstname":"Lynnette","lastname":"Alvarez","age":38,"gender":"F","address":"991 Brightwater Avenue","employer":"Gink","email":"lynnettealvarez@gink.com","city":"Leola","state":"NC"}
+{"index":{"_id":"779"}}
+{"account_number":779,"balance":40983,"firstname":"Maggie","lastname":"Pace","age":32,"gender":"F","address":"104 Harbor Court","employer":"Bulljuice","email":"maggiepace@bulljuice.com","city":"Floris","state":"MA"}
+{"index":{"_id":"781"}}
+{"account_number":781,"balance":29961,"firstname":"Sanford","lastname":"Mullen","age":26,"gender":"F","address":"879 Dover Street","employer":"Zanity","email":"sanfordmullen@zanity.com","city":"Martinez","state":"TX"}
+{"index":{"_id":"786"}}
+{"account_number":786,"balance":3024,"firstname":"Rene","lastname":"Vang","age":33,"gender":"M","address":"506 Randolph Street","employer":"Isopop","email":"renevang@isopop.com","city":"Vienna","state":"NJ"}
+{"index":{"_id":"793"}}
+{"account_number":793,"balance":16911,"firstname":"Alford","lastname":"Compton","age":36,"gender":"M","address":"186 Veronica Place","employer":"Zyple","email":"alfordcompton@zyple.com","city":"Sugartown","state":"AK"}
+{"index":{"_id":"798"}}
+{"account_number":798,"balance":3165,"firstname":"Catherine","lastname":"Ward","age":30,"gender":"F","address":"325 Burnett Street","employer":"Dreamia","email":"catherineward@dreamia.com","city":"Glenbrook","state":"SD"}
+{"index":{"_id":"801"}}
+{"account_number":801,"balance":14954,"firstname":"Molly","lastname":"Maldonado","age":37,"gender":"M","address":"518 Maple Avenue","employer":"Straloy","email":"mollymaldonado@straloy.com","city":"Hebron","state":"WI"}
+{"index":{"_id":"806"}}
+{"account_number":806,"balance":36492,"firstname":"Carson","lastname":"Riddle","age":31,"gender":"M","address":"984 Lois Avenue","employer":"Terrago","email":"carsonriddle@terrago.com","city":"Leland","state":"MN"}
+{"index":{"_id":"813"}}
+{"account_number":813,"balance":30833,"firstname":"Ebony","lastname":"Bishop","age":20,"gender":"M","address":"487 Ridge Court","employer":"Optique","email":"ebonybishop@optique.com","city":"Fairmount","state":"WA"}
+{"index":{"_id":"818"}}
+{"account_number":818,"balance":24433,"firstname":"Espinoza","lastname":"Petersen","age":26,"gender":"M","address":"641 Glenwood Road","employer":"Futurity","email":"espinozapetersen@futurity.com","city":"Floriston","state":"MD"}
+{"index":{"_id":"820"}}
+{"account_number":820,"balance":1011,"firstname":"Shepard","lastname":"Ramsey","age":24,"gender":"F","address":"806 Village Court","employer":"Mantro","email":"shepardramsey@mantro.com","city":"Tibbie","state":"NV"}
+{"index":{"_id":"825"}}
+{"account_number":825,"balance":49000,"firstname":"Terra","lastname":"Witt","age":21,"gender":"F","address":"590 Conway Street","employer":"Insectus","email":"terrawitt@insectus.com","city":"Forbestown","state":"AR"}
+{"index":{"_id":"832"}}
+{"account_number":832,"balance":8582,"firstname":"Laura","lastname":"Gibbs","age":39,"gender":"F","address":"511 Osborn Street","employer":"Corepan","email":"lauragibbs@corepan.com","city":"Worcester","state":"KS"}
+{"index":{"_id":"837"}}
+{"account_number":837,"balance":14485,"firstname":"Amy","lastname":"Villarreal","age":35,"gender":"M","address":"381 Stillwell Place","employer":"Fleetmix","email":"amyvillarreal@fleetmix.com","city":"Sanford","state":"IA"}
+{"index":{"_id":"844"}}
+{"account_number":844,"balance":26840,"firstname":"Jill","lastname":"David","age":31,"gender":"M","address":"346 Legion Street","employer":"Zytrax","email":"jilldavid@zytrax.com","city":"Saticoy","state":"SC"}
+{"index":{"_id":"849"}}
+{"account_number":849,"balance":16200,"firstname":"Barry","lastname":"Chapman","age":26,"gender":"M","address":"931 Dekoven Court","employer":"Darwinium","email":"barrychapman@darwinium.com","city":"Whitestone","state":"WY"}
+{"index":{"_id":"851"}}
+{"account_number":851,"balance":22026,"firstname":"Henderson","lastname":"Price","age":33,"gender":"F","address":"530 Hausman Street","employer":"Plutorque","email":"hendersonprice@plutorque.com","city":"Brutus","state":"RI"}
+{"index":{"_id":"856"}}
+{"account_number":856,"balance":27583,"firstname":"Alissa","lastname":"Knox","age":25,"gender":"M","address":"258 Empire Boulevard","employer":"Geologix","email":"alissaknox@geologix.com","city":"Hartsville/Hartley","state":"MN"}
+{"index":{"_id":"863"}}
+{"account_number":863,"balance":23165,"firstname":"Melendez","lastname":"Fernandez","age":40,"gender":"M","address":"661 Johnson Avenue","employer":"Vixo","email":"melendezfernandez@vixo.com","city":"Farmers","state":"IL"}
+{"index":{"_id":"868"}}
+{"account_number":868,"balance":27624,"firstname":"Polly","lastname":"Barron","age":22,"gender":"M","address":"129 Frank Court","employer":"Geofarm","email":"pollybarron@geofarm.com","city":"Loyalhanna","state":"ND"}
+{"index":{"_id":"870"}}
+{"account_number":870,"balance":43882,"firstname":"Goff","lastname":"Phelps","age":21,"gender":"M","address":"164 Montague Street","employer":"Digigen","email":"goffphelps@digigen.com","city":"Weedville","state":"IL"}
+{"index":{"_id":"875"}}
+{"account_number":875,"balance":19655,"firstname":"Mercer","lastname":"Pratt","age":24,"gender":"M","address":"608 Perry Place","employer":"Twiggery","email":"mercerpratt@twiggery.com","city":"Eggertsville","state":"MO"}
+{"index":{"_id":"882"}}
+{"account_number":882,"balance":10895,"firstname":"Mari","lastname":"Landry","age":39,"gender":"M","address":"963 Gerald Court","employer":"Kenegy","email":"marilandry@kenegy.com","city":"Lithium","state":"NC"}
+{"index":{"_id":"887"}}
+{"account_number":887,"balance":31772,"firstname":"Eunice","lastname":"Watts","age":36,"gender":"F","address":"707 Stuyvesant Avenue","employer":"Memora","email":"eunicewatts@memora.com","city":"Westwood","state":"TN"}
+{"index":{"_id":"894"}}
+{"account_number":894,"balance":1031,"firstname":"Tyler","lastname":"Fitzgerald","age":32,"gender":"M","address":"787 Meserole Street","employer":"Jetsilk","email":"tylerfitzgerald@jetsilk.com","city":"Woodlands","state":"WV"}
+{"index":{"_id":"899"}}
+{"account_number":899,"balance":32953,"firstname":"Carney","lastname":"Callahan","age":23,"gender":"M","address":"724 Kimball Street","employer":"Mangelica","email":"carneycallahan@mangelica.com","city":"Tecolotito","state":"MT"}
+{"index":{"_id":"902"}}
+{"account_number":902,"balance":13345,"firstname":"Hallie","lastname":"Jarvis","age":23,"gender":"F","address":"237 Duryea Court","employer":"Anixang","email":"halliejarvis@anixang.com","city":"Boykin","state":"IN"}
+{"index":{"_id":"907"}}
+{"account_number":907,"balance":12961,"firstname":"Ingram","lastname":"William","age":36,"gender":"M","address":"826 Overbaugh Place","employer":"Genmex","email":"ingramwilliam@genmex.com","city":"Kimmell","state":"AK"}
+{"index":{"_id":"914"}}
+{"account_number":914,"balance":7120,"firstname":"Esther","lastname":"Bean","age":32,"gender":"F","address":"583 Macon Street","employer":"Applica","email":"estherbean@applica.com","city":"Homeworth","state":"MN"}
+{"index":{"_id":"919"}}
+{"account_number":919,"balance":39655,"firstname":"Shauna","lastname":"Hanson","age":27,"gender":"M","address":"557 Hart Place","employer":"Exospace","email":"shaunahanson@exospace.com","city":"Outlook","state":"LA"}
+{"index":{"_id":"921"}}
+{"account_number":921,"balance":49119,"firstname":"Barbara","lastname":"Wade","age":29,"gender":"M","address":"687 Hoyts Lane","employer":"Roughies","email":"barbarawade@roughies.com","city":"Sattley","state":"CO"}
+{"index":{"_id":"926"}}
+{"account_number":926,"balance":49433,"firstname":"Welch","lastname":"Mcgowan","age":21,"gender":"M","address":"833 Quincy Street","employer":"Atomica","email":"welchmcgowan@atomica.com","city":"Hampstead","state":"VT"}
+{"index":{"_id":"933"}}
+{"account_number":933,"balance":18071,"firstname":"Tabitha","lastname":"Cole","age":21,"gender":"F","address":"916 Rogers Avenue","employer":"Eclipto","email":"tabithacole@eclipto.com","city":"Lawrence","state":"TX"}
+{"index":{"_id":"938"}}
+{"account_number":938,"balance":9597,"firstname":"Sharron","lastname":"Santos","age":40,"gender":"F","address":"215 Matthews Place","employer":"Zenco","email":"sharronsantos@zenco.com","city":"Wattsville","state":"VT"}
+{"index":{"_id":"940"}}
+{"account_number":940,"balance":23285,"firstname":"Melinda","lastname":"Mendoza","age":38,"gender":"M","address":"806 Kossuth Place","employer":"Kneedles","email":"melindamendoza@kneedles.com","city":"Coaldale","state":"OK"}
+{"index":{"_id":"945"}}
+{"account_number":945,"balance":23085,"firstname":"Hansen","lastname":"Hebert","age":33,"gender":"F","address":"287 Conduit Boulevard","employer":"Capscreen","email":"hansenhebert@capscreen.com","city":"Taycheedah","state":"AK"}
+{"index":{"_id":"952"}}
+{"account_number":952,"balance":21430,"firstname":"Angelique","lastname":"Weeks","age":33,"gender":"M","address":"659 Reeve Place","employer":"Exodoc","email":"angeliqueweeks@exodoc.com","city":"Turpin","state":"MD"}
+{"index":{"_id":"957"}}
+{"account_number":957,"balance":11373,"firstname":"Michael","lastname":"Giles","age":31,"gender":"M","address":"668 Court Square","employer":"Yogasm","email":"michaelgiles@yogasm.com","city":"Rosburg","state":"WV"}
+{"index":{"_id":"964"}}
+{"account_number":964,"balance":26154,"firstname":"Elena","lastname":"Waller","age":34,"gender":"F","address":"618 Crystal Street","employer":"Insurety","email":"elenawaller@insurety.com","city":"Gallina","state":"NY"}
+{"index":{"_id":"969"}}
+{"account_number":969,"balance":22214,"firstname":"Briggs","lastname":"Lynn","age":30,"gender":"M","address":"952 Lester Court","employer":"Quinex","email":"briggslynn@quinex.com","city":"Roland","state":"ID"}
+{"index":{"_id":"971"}}
+{"account_number":971,"balance":22772,"firstname":"Gabrielle","lastname":"Reilly","age":32,"gender":"F","address":"964 Tudor Terrace","employer":"Blanet","email":"gabriellereilly@blanet.com","city":"Falmouth","state":"AL"}
+{"index":{"_id":"976"}}
+{"account_number":976,"balance":31707,"firstname":"Mullen","lastname":"Tanner","age":26,"gender":"M","address":"711 Whitney Avenue","employer":"Pulze","email":"mullentanner@pulze.com","city":"Mooresburg","state":"MA"}
+{"index":{"_id":"983"}}
+{"account_number":983,"balance":47205,"firstname":"Mattie","lastname":"Eaton","age":24,"gender":"F","address":"418 Allen Avenue","employer":"Trasola","email":"mattieeaton@trasola.com","city":"Dupuyer","state":"NJ"}
+{"index":{"_id":"988"}}
+{"account_number":988,"balance":17803,"firstname":"Lucy","lastname":"Castro","age":34,"gender":"F","address":"425 Fleet Walk","employer":"Geekfarm","email":"lucycastro@geekfarm.com","city":"Mulino","state":"VA"}
+{"index":{"_id":"990"}}
+{"account_number":990,"balance":44456,"firstname":"Kelly","lastname":"Steele","age":35,"gender":"M","address":"809 Hoyt Street","employer":"Eschoir","email":"kellysteele@eschoir.com","city":"Stewartville","state":"ID"}
+{"index":{"_id":"995"}}
+{"account_number":995,"balance":21153,"firstname":"Phelps","lastname":"Parrish","age":25,"gender":"M","address":"666 Miller Place","employer":"Pearlessa","email":"phelpsparrish@pearlessa.com","city":"Brecon","state":"ME"}
diff --git a/modules/aggs-matrix-stats/build.gradle b/modules/aggs-matrix-stats/build.gradle
index b3060fa178621..d3af124b8af71 100644
--- a/modules/aggs-matrix-stats/build.gradle
+++ b/modules/aggs-matrix-stats/build.gradle
@@ -20,4 +20,5 @@
 esplugin {
     description 'Adds aggregations whose input are a list of numeric fields and output includes a matrix.'
     classname 'org.elasticsearch.search.aggregations.matrix.MatrixAggregationPlugin'
+    hasClientJar = true
 }
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/spi/MatrixStatsNamedXContentProvider.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/spi/MatrixStatsNamedXContentProvider.java
new file mode 100644
index 0000000000000..bb71e3085de62
--- /dev/null
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/spi/MatrixStatsNamedXContentProvider.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.matrix.spi;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ContextParser;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.plugins.spi.NamedXContentProvider;
+import org.elasticsearch.search.aggregations.Aggregation;
+import org.elasticsearch.search.aggregations.matrix.stats.MatrixStatsAggregationBuilder;
+import org.elasticsearch.search.aggregations.matrix.stats.ParsedMatrixStats;
+
+import java.util.List;
+
+import static java.util.Collections.singletonList;
+
+public class MatrixStatsNamedXContentProvider implements NamedXContentProvider {
+
+    @Override
+    public List<NamedXContentRegistry.Entry> getNamedXContentParsers() {
+        ParseField parseField = new ParseField(MatrixStatsAggregationBuilder.NAME);
+        ContextParser<Object, Aggregation> contextParser = (p, name) -> ParsedMatrixStats.fromXContent(p, (String) name);
+        return singletonList(new NamedXContentRegistry.Entry(Aggregation.class, parseField, contextParser));
+    }
+}
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStats.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStats.java
index d97fb64a16d70..6ce0a850515f9 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStats.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStats.java
@@ -22,7 +22,6 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.aggregations.InternalAggregation;
-import org.elasticsearch.search.aggregations.metrics.InternalMetricsAggregation;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 
 import java.io.IOException;
@@ -35,7 +34,7 @@
 /**
  * Computes distribution statistics over multiple fields
  */
-public class InternalMatrixStats extends InternalMetricsAggregation implements MatrixStats {
+public class InternalMatrixStats extends InternalAggregation implements MatrixStats {
     /** per shard stats needed to compute stats */
     private final RunningStats stats;
     /** final result */
@@ -73,6 +72,9 @@ public String getWriteableName() {
     /** get the number of documents */
     @Override
     public long getDocCount() {
+        if (stats == null) {
+            return 0;
+        }
         return stats.docCount;
     }
 
@@ -153,6 +155,7 @@ static class Fields {
 
     @Override
     public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+        builder.field(CommonFields.DOC_COUNT.getPreferredName(), getDocCount());
         if (results != null && results.getFieldCounts().keySet().isEmpty() == false) {
             builder.startArray(Fields.FIELDS);
             for (String fieldName : results.getFieldCounts().keySet()) {
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregationBuilder.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregationBuilder.java
index 867bcdbdfa533..40e227681c405 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregationBuilder.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregationBuilder.java
@@ -25,14 +25,13 @@
 import org.elasticsearch.search.MultiValueMode;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.MultiValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.Map;
@@ -40,19 +39,18 @@
 public class MatrixStatsAggregationBuilder
     extends MultiValuesSourceAggregationBuilder.LeafOnly<ValuesSource.Numeric, MatrixStatsAggregationBuilder> {
     public static final String NAME = "matrix_stats";
-    public static final Type TYPE = new Type(NAME);
 
     private MultiValueMode multiValueMode = MultiValueMode.AVG;
 
     public MatrixStatsAggregationBuilder(String name) {
-        super(name, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(name, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     /**
      * Read from a stream.
      */
     public MatrixStatsAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(in, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     @Override
@@ -70,9 +68,9 @@ public MultiValueMode multiValueMode() {
     }
 
     @Override
-    protected MatrixStatsAggregatorFactory innerBuild(AggregationContext context, Map<String, ValuesSourceConfig<Numeric>> configs,
+    protected MatrixStatsAggregatorFactory innerBuild(SearchContext context, Map<String, ValuesSourceConfig<Numeric>> configs,
             AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder) throws IOException {
-        return new MatrixStatsAggregatorFactory(name, type, configs, multiValueMode, context, parent, subFactoriesBuilder, metaData);
+        return new MatrixStatsAggregatorFactory(name, configs, multiValueMode, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
@@ -92,7 +90,7 @@ protected boolean innerEquals(Object obj) {
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregator.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregator.java
index 75f0c37d89f0d..4665e74daa72a 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregator.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregator.java
@@ -30,9 +30,9 @@
 import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
 import org.elasticsearch.search.aggregations.metrics.MetricsAggregator;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.MultiValuesSource.NumericMultiValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -48,7 +48,7 @@ public class MatrixStatsAggregator extends MetricsAggregator {
     /** array of descriptive stats, per shard, needed to compute the correlation */
     ObjectArray<RunningStats> stats;
 
-    public MatrixStatsAggregator(String name, Map<String, ValuesSource.Numeric> valuesSources, AggregationContext context,
+    public MatrixStatsAggregator(String name, Map<String, ValuesSource.Numeric> valuesSources, SearchContext context,
                                  Aggregator parent, MultiValueMode multiValueMode, List<PipelineAggregator> pipelineAggregators,
                                  Map<String,Object> metaData) throws IOException {
         super(name, context, parent, pipelineAggregators, metaData);
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorFactory.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorFactory.java
index 26e7910dcac1f..c0aae08e5d836 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorFactory.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorFactory.java
@@ -22,12 +22,11 @@
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.MultiValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -40,11 +39,11 @@ public class MatrixStatsAggregatorFactory
 
     private final MultiValueMode multiValueMode;
 
-    public MatrixStatsAggregatorFactory(String name, InternalAggregation.Type type,
+    public MatrixStatsAggregatorFactory(String name,
             Map<String, ValuesSourceConfig<ValuesSource.Numeric>> configs, MultiValueMode multiValueMode,
-            AggregationContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
+            SearchContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
             Map<String, Object> metaData) throws IOException {
-        super(name, type, configs, context, parent, subFactoriesBuilder, metaData);
+        super(name, configs, context, parent, subFactoriesBuilder, metaData);
         this.multiValueMode = multiValueMode;
     }
 
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsParser.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsParser.java
index ea383b642c2b8..33da7560ff3a4 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsParser.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsParser.java
@@ -19,7 +19,6 @@
 package org.elasticsearch.search.aggregations.matrix.stats;
 
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.search.MultiValueMode;
 import org.elasticsearch.search.aggregations.support.MultiValuesSourceParser.NumericValuesSourceParser;
@@ -41,8 +40,8 @@ public MatrixStatsParser() {
 
     @Override
     protected boolean token(String aggregationName, String currentFieldName, XContentParser.Token token, XContentParser parser,
-                            ParseFieldMatcher parseFieldMatcher, Map<ParseField, Object> otherOptions) throws IOException {
-        if (parseFieldMatcher.match(currentFieldName, MULTIVALUE_MODE_FIELD)) {
+                            Map<ParseField, Object> otherOptions) throws IOException {
+        if (MULTIVALUE_MODE_FIELD.match(currentFieldName)) {
             if (token == XContentParser.Token.VALUE_STRING) {
                 otherOptions.put(MULTIVALUE_MODE_FIELD, parser.text());
                 return true;
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsResults.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsResults.java
index 1ae29e6576168..7bc26087e104d 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsResults.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsResults.java
@@ -39,13 +39,13 @@ class MatrixStatsResults implements Writeable {
     protected final Map<String, HashMap<String, Double>> correlation;
 
     /** Base ctor */
-    public MatrixStatsResults() {
+    MatrixStatsResults() {
         results = new RunningStats();
         this.correlation = new HashMap<>();
     }
 
     /** creates and computes result from provided stats */
-    public MatrixStatsResults(RunningStats stats) {
+    MatrixStatsResults(RunningStats stats) {
         this.results = stats.clone();
         this.correlation = new HashMap<>();
         this.compute();
@@ -161,7 +161,7 @@ public Double getCorrelation(String fieldX, String fieldY) {
     }
 
     /** return the value for two fields in an upper triangular matrix, regardless of row col location. */
-    private double getValFromUpperTriangularMatrix(Map<String, HashMap<String, Double>> map, String fieldX, String fieldY) {
+    static <M extends Map<String, Double>> double getValFromUpperTriangularMatrix(Map<String, M> map, String fieldX, String fieldY) {
         // for the co-value to exist, one of the two (or both) fields has to be a row key
         if (map.containsKey(fieldX) == false && map.containsKey(fieldY) == false) {
             throw new IllegalArgumentException("neither field " + fieldX + " nor " + fieldY + " exist");
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/ParsedMatrixStats.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/ParsedMatrixStats.java
new file mode 100644
index 0000000000000..a0b9224ad0d25
--- /dev/null
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/ParsedMatrixStats.java
@@ -0,0 +1,232 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.matrix.stats;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.ParsedAggregation;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Objects;
+
+public class ParsedMatrixStats extends ParsedAggregation implements MatrixStats {
+
+    private final Map<String, Long> counts = new LinkedHashMap<>();
+    private final Map<String, Double> means = new HashMap<>();
+    private final Map<String, Double> variances = new HashMap<>();
+    private final Map<String, Double> skewness = new HashMap<>();
+    private final Map<String, Double> kurtosis = new HashMap<>();
+    private final Map<String, Map<String, Double>> covariances = new HashMap<>();
+    private final Map<String, Map<String, Double>> correlations = new HashMap<>();
+
+    private long docCount;
+
+    @Override
+    public String getType() {
+        return MatrixStatsAggregationBuilder.NAME;
+    }
+
+    private void setDocCount(long docCount) {
+        this.docCount = docCount;
+    }
+
+    @Override
+    public long getDocCount() {
+        return docCount;
+    }
+
+    @Override
+    public long getFieldCount(String field) {
+        if (counts.containsKey(field) == false) {
+            return 0;
+        }
+        return counts.get(field);
+    }
+
+    @Override
+    public double getMean(String field) {
+        return checkedGet(means, field);
+    }
+
+    @Override
+    public double getVariance(String field) {
+        return checkedGet(variances, field);
+    }
+
+    @Override
+    public double getSkewness(String field) {
+        return checkedGet(skewness, field);
+    }
+
+    @Override
+    public double getKurtosis(String field) {
+        return checkedGet(kurtosis, field);
+    }
+
+    @Override
+    public double getCovariance(String fieldX, String fieldY) {
+        if (fieldX.equals(fieldY)) {
+            return checkedGet(variances, fieldX);
+        }
+        return MatrixStatsResults.getValFromUpperTriangularMatrix(covariances, fieldX, fieldY);
+    }
+
+    @Override
+    public double getCorrelation(String fieldX, String fieldY) {
+        if (fieldX.equals(fieldY)) {
+            return 1.0;
+        }
+        return MatrixStatsResults.getValFromUpperTriangularMatrix(correlations, fieldX, fieldY);
+    }
+
+    @Override
+    protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+        builder.field(CommonFields.DOC_COUNT.getPreferredName(), getDocCount());
+        if (counts != null && counts.isEmpty() == false) {
+            builder.startArray(InternalMatrixStats.Fields.FIELDS);
+            for (String fieldName : counts.keySet()) {
+                builder.startObject();
+                builder.field(InternalMatrixStats.Fields.NAME, fieldName);
+                builder.field(InternalMatrixStats.Fields.COUNT, getFieldCount(fieldName));
+                builder.field(InternalMatrixStats.Fields.MEAN, getMean(fieldName));
+                builder.field(InternalMatrixStats.Fields.VARIANCE, getVariance(fieldName));
+                builder.field(InternalMatrixStats.Fields.SKEWNESS, getSkewness(fieldName));
+                builder.field(InternalMatrixStats.Fields.KURTOSIS, getKurtosis(fieldName));
+                {
+                    builder.startObject(InternalMatrixStats.Fields.COVARIANCE);
+                    Map<String, Double> covars = covariances.get(fieldName);
+                    if (covars != null) {
+                        for (Map.Entry<String, Double> covar : covars.entrySet()) {
+                            builder.field(covar.getKey(), covar.getValue());
+                        }
+                    }
+                    builder.endObject();
+                }
+                {
+                    builder.startObject(InternalMatrixStats.Fields.CORRELATION);
+                    Map<String, Double> correls = correlations.get(fieldName);
+                    if (correls != null) {
+                        for (Map.Entry<String, Double> correl : correls.entrySet()) {
+                            builder.field(correl.getKey(), correl.getValue());
+                        }
+                    }
+                    builder.endObject();
+                }
+                builder.endObject();
+            }
+            builder.endArray();
+        }
+        return builder;
+    }
+
+    private static <T> T checkedGet(final Map<String, T> values, final String fieldName) {
+        if (fieldName == null) {
+            throw new IllegalArgumentException("field name cannot be null");
+        }
+        if (values.containsKey(fieldName) == false) {
+            throw new IllegalArgumentException("field " + fieldName + " does not exist");
+        }
+        return values.get(fieldName);
+    }
+
+    private static ObjectParser<ParsedMatrixStats, Void> PARSER =
+            new ObjectParser<>(ParsedMatrixStats.class.getSimpleName(), true, ParsedMatrixStats::new);
+    static {
+        declareAggregationFields(PARSER);
+        PARSER.declareLong(ParsedMatrixStats::setDocCount, CommonFields.DOC_COUNT);
+        PARSER.declareObjectArray((matrixStats, results) -> {
+            for (ParsedMatrixStatsResult result : results) {
+                final String fieldName = result.name;
+                matrixStats.counts.put(fieldName, result.count);
+                matrixStats.means.put(fieldName, result.mean);
+                matrixStats.variances.put(fieldName, result.variance);
+                matrixStats.skewness.put(fieldName, result.skewness);
+                matrixStats.kurtosis.put(fieldName, result.kurtosis);
+                matrixStats.covariances.put(fieldName, result.covariances);
+                matrixStats.correlations.put(fieldName, result.correlations);
+            }
+        }, (p, c) -> ParsedMatrixStatsResult.fromXContent(p), new ParseField(InternalMatrixStats.Fields.FIELDS));
+    }
+
+    public static ParsedMatrixStats fromXContent(XContentParser parser, String name) throws IOException {
+        ParsedMatrixStats aggregation = PARSER.parse(parser, null);
+        aggregation.setName(name);
+        return aggregation;
+    }
+
+    static class ParsedMatrixStatsResult {
+
+        String name;
+        Long count;
+        Double mean;
+        Double variance;
+        Double skewness;
+        Double kurtosis;
+        Map<String, Double> covariances;
+        Map<String, Double> correlations;
+
+        private static ObjectParser<ParsedMatrixStatsResult, Void> RESULT_PARSER =
+                new ObjectParser<>(ParsedMatrixStatsResult.class.getSimpleName(), true, ParsedMatrixStatsResult::new);
+        static {
+            RESULT_PARSER.declareString((result, name) -> result.name = name,
+                    new ParseField(InternalMatrixStats.Fields.NAME));
+            RESULT_PARSER.declareLong((result, count) -> result.count = count,
+                    new ParseField(InternalMatrixStats.Fields.COUNT));
+            RESULT_PARSER.declareDouble((result, mean) -> result.mean = mean,
+                    new ParseField(InternalMatrixStats.Fields.MEAN));
+            RESULT_PARSER.declareDouble((result, variance) -> result.variance = variance,
+                    new ParseField(InternalMatrixStats.Fields.VARIANCE));
+            RESULT_PARSER.declareDouble((result, skewness) -> result.skewness = skewness,
+                    new ParseField(InternalMatrixStats.Fields.SKEWNESS));
+            RESULT_PARSER.declareDouble((result, kurtosis) -> result.kurtosis = kurtosis,
+                    new ParseField(InternalMatrixStats.Fields.KURTOSIS));
+
+            RESULT_PARSER.declareObject((ParsedMatrixStatsResult result, Map<String, Object> covars) -> {
+                result.covariances = new LinkedHashMap<>(covars.size());
+                for (Map.Entry<String, Object> covar : covars.entrySet()) {
+                    result.covariances.put(covar.getKey(), mapValueAsDouble(covar.getValue()));
+                }
+            }, (p, c) -> p.mapOrdered(), new ParseField(InternalMatrixStats.Fields.COVARIANCE));
+
+            RESULT_PARSER.declareObject((ParsedMatrixStatsResult result, Map<String, Object> correls) -> {
+                result.correlations = new LinkedHashMap<>(correls.size());
+                for (Map.Entry<String, Object> correl : correls.entrySet()) {
+                    result.correlations.put(correl.getKey(), mapValueAsDouble(correl.getValue()));
+                }
+            }, (p, c) -> p.mapOrdered(), new ParseField(InternalMatrixStats.Fields.CORRELATION));
+        }
+
+        private static Double mapValueAsDouble(Object value) {
+            if (value instanceof Double) {
+                return (Double) value;
+            }
+            return Double.valueOf(Objects.toString(value));
+        }
+
+        static ParsedMatrixStatsResult fromXContent(XContentParser parser) throws IOException {
+            return RESULT_PARSER.parse(parser, null);
+        }
+    }
+}
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java
index e3aa171fe3dcd..1d32c9d985175 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java
@@ -29,12 +29,12 @@
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.AggregationInitializationException;
 import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
+import org.elasticsearch.search.aggregations.AggregationInitializationException;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -55,29 +55,29 @@ public abstract class MultiValuesSourceAggregationBuilder<VS extends ValuesSourc
     public abstract static class LeafOnly<VS extends ValuesSource, AB extends MultiValuesSourceAggregationBuilder<VS, AB>>
             extends MultiValuesSourceAggregationBuilder<VS, AB> {
 
-        protected LeafOnly(String name, Type type, ValuesSourceType valuesSourceType, ValueType targetValueType) {
-            super(name, type, valuesSourceType, targetValueType);
+        protected LeafOnly(String name, ValuesSourceType valuesSourceType, ValueType targetValueType) {
+            super(name, valuesSourceType, targetValueType);
         }
 
         /**
          * Read from a stream that does not serialize its targetValueType. This should be used by most subclasses.
          */
-        protected LeafOnly(StreamInput in, Type type, ValuesSourceType valuesSourceType, ValueType targetValueType) throws IOException {
-            super(in, type, valuesSourceType, targetValueType);
+        protected LeafOnly(StreamInput in, ValuesSourceType valuesSourceType, ValueType targetValueType) throws IOException {
+            super(in, valuesSourceType, targetValueType);
         }
 
         /**
          * Read an aggregation from a stream that serializes its targetValueType. This should only be used by subclasses that override
          * {@link #serializeTargetValueType()} to return true.
          */
-        protected LeafOnly(StreamInput in, Type type, ValuesSourceType valuesSourceType) throws IOException {
-            super(in, type, valuesSourceType);
+        protected LeafOnly(StreamInput in, ValuesSourceType valuesSourceType) throws IOException {
+            super(in, valuesSourceType);
         }
 
         @Override
         public AB subAggregations(Builder subFactories) {
             throw new AggregationInitializationException("Aggregator [" + name + "] of type [" +
-                type + "] cannot accept sub-aggregations");
+                getType() + "] cannot accept sub-aggregations");
         }
     }
 
@@ -89,8 +89,8 @@ public AB subAggregations(Builder subFactories) {
     private Object missing = null;
     private Map<String, Object> missingMap = Collections.emptyMap();
 
-    protected MultiValuesSourceAggregationBuilder(String name, Type type, ValuesSourceType valuesSourceType, ValueType targetValueType) {
-        super(name, type);
+    protected MultiValuesSourceAggregationBuilder(String name, ValuesSourceType valuesSourceType, ValueType targetValueType) {
+        super(name);
         if (valuesSourceType == null) {
             throw new IllegalArgumentException("[valuesSourceType] must not be null: [" + name + "]");
         }
@@ -98,17 +98,17 @@ protected MultiValuesSourceAggregationBuilder(String name, Type type, ValuesSour
         this.targetValueType = targetValueType;
     }
 
-    protected MultiValuesSourceAggregationBuilder(StreamInput in, Type type, ValuesSourceType valuesSourceType, ValueType targetValueType)
+    protected MultiValuesSourceAggregationBuilder(StreamInput in, ValuesSourceType valuesSourceType, ValueType targetValueType)
         throws IOException {
-        super(in, type);
+        super(in);
         assert false == serializeTargetValueType() : "Wrong read constructor called for subclass that provides its targetValueType";
         this.valuesSourceType = valuesSourceType;
         this.targetValueType = targetValueType;
         read(in);
     }
 
-    protected MultiValuesSourceAggregationBuilder(StreamInput in, Type type, ValuesSourceType valuesSourceType) throws IOException {
-        super(in, type);
+    protected MultiValuesSourceAggregationBuilder(StreamInput in, ValuesSourceType valuesSourceType) throws IOException {
+        super(in);
         assert serializeTargetValueType() : "Wrong read constructor called for subclass that serializes its targetValueType";
         this.valuesSourceType = valuesSourceType;
         this.targetValueType = in.readOptionalWriteable(ValueType::readFromStream);
@@ -222,14 +222,14 @@ public Map<String, Object> missingMap() {
     }
 
     @Override
-    protected final MultiValuesSourceAggregatorFactory<VS, ?> doBuild(AggregationContext context, AggregatorFactory<?> parent,
+    protected final MultiValuesSourceAggregatorFactory<VS, ?> doBuild(SearchContext context, AggregatorFactory<?> parent,
             AggregatorFactories.Builder subFactoriesBuilder) throws IOException {
         Map<String, ValuesSourceConfig<VS>> configs = resolveConfig(context);
         MultiValuesSourceAggregatorFactory<VS, ?> factory = innerBuild(context, configs, parent, subFactoriesBuilder);
         return factory;
     }
 
-    protected Map<String, ValuesSourceConfig<VS>> resolveConfig(AggregationContext context) {
+    protected Map<String, ValuesSourceConfig<VS>> resolveConfig(SearchContext context) {
         HashMap<String, ValuesSourceConfig<VS>> configs = new HashMap<>();
         for (String field : fields) {
             ValuesSourceConfig<VS> config = config(context, field, null);
@@ -238,17 +238,16 @@ protected Map<String, ValuesSourceConfig<VS>> resolveConfig(AggregationContext c
         return configs;
     }
 
-    protected abstract MultiValuesSourceAggregatorFactory<VS, ?> innerBuild(AggregationContext context,
+    protected abstract MultiValuesSourceAggregatorFactory<VS, ?> innerBuild(SearchContext context,
             Map<String, ValuesSourceConfig<VS>> configs, AggregatorFactory<?> parent,
             AggregatorFactories.Builder subFactoriesBuilder) throws IOException;
 
-    public ValuesSourceConfig<VS> config(AggregationContext context, String field, Script script) {
+    public ValuesSourceConfig<VS> config(SearchContext context, String field, Script script) {
 
         ValueType valueType = this.valueType != null ? this.valueType : targetValueType;
 
         if (field == null) {
             if (script == null) {
-                @SuppressWarnings("unchecked")
                 ValuesSourceConfig<VS> config = new ValuesSourceConfig<>(ValuesSourceType.ANY);
                 return config.format(resolveFormat(null, valueType));
             }
@@ -265,7 +264,7 @@ public ValuesSourceConfig<VS> config(AggregationContext context, String field, S
             return config.format(resolveFormat(format, valueType));
         }
 
-        MappedFieldType fieldType = context.searchContext().smartNameFieldType(field);
+        MappedFieldType fieldType = context.smartNameFieldType(field);
         if (fieldType == null) {
             ValuesSourceType valuesSourceType = valueType != null ? valueType.getValuesSourceType() : this.valuesSourceType;
             ValuesSourceConfig<VS> config = new ValuesSourceConfig<>(valuesSourceType);
@@ -274,7 +273,7 @@ public ValuesSourceConfig<VS> config(AggregationContext context, String field, S
             return config.unmapped(true);
         }
 
-        IndexFieldData<?> indexFieldData = context.searchContext().fieldData().getForField(fieldType);
+        IndexFieldData<?> indexFieldData = context.fieldData().getForField(fieldType);
 
         ValuesSourceConfig<VS> config;
         if (valuesSourceType == ValuesSourceType.ANY) {
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregatorFactory.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregatorFactory.java
index 956f953da108b..7d5c56a571bbe 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregatorFactory.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregatorFactory.java
@@ -22,8 +22,8 @@
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -35,10 +35,10 @@ public abstract class MultiValuesSourceAggregatorFactory<VS extends ValuesSource
 
     protected Map<String, ValuesSourceConfig<VS>> configs;
 
-    public MultiValuesSourceAggregatorFactory(String name, Type type, Map<String, ValuesSourceConfig<VS>> configs,
-        AggregationContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
-        Map<String, Object> metaData) throws IOException {
-        super(name, type, context, parent, subFactoriesBuilder, metaData);
+    public MultiValuesSourceAggregatorFactory(String name, Map<String, ValuesSourceConfig<VS>> configs,
+            SearchContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
+            Map<String, Object> metaData) throws IOException {
+        super(name, context, parent, subFactoriesBuilder, metaData);
         this.configs = configs;
     }
 
@@ -48,7 +48,7 @@ public Aggregator createInternal(Aggregator parent, boolean collectsFromSingleBu
         HashMap<String, VS> valuesSources = new HashMap<>();
 
         for (Map.Entry<String, ValuesSourceConfig<VS>> config : configs.entrySet()) {
-            VS vs = context.valuesSource(config.getValue(), context.searchContext());
+            VS vs = config.getValue().toValuesSource(context.getQueryShardContext());
             if (vs != null) {
                 valuesSources.put(config.getKey(), vs);
             }
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceParser.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceParser.java
index 0de5e13c05880..10c28d3c814f3 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceParser.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceParser.java
@@ -20,13 +20,12 @@
 package org.elasticsearch.search.aggregations.support;
 
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.script.Script.ScriptField;
-import org.elasticsearch.search.aggregations.Aggregator;
+import org.elasticsearch.script.Script;
 import org.elasticsearch.search.aggregations.AggregationBuilder.CommonFields;
+import org.elasticsearch.search.aggregations.Aggregator;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -88,47 +87,45 @@ private MultiValuesSourceParser(boolean formattable, ValuesSourceType valuesSour
         String format = null;
         Map<String, Object> missingMap = null;
         Map<ParseField, Object> otherOptions = new HashMap<>();
-        final ParseFieldMatcher parseFieldMatcher = context.getParseFieldMatcher();
-
         XContentParser.Token token;
         String currentFieldName = null;
         while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.VALUE_STRING) {
-                if (parseFieldMatcher.match(currentFieldName, CommonFields.FIELDS)) {
+                if (CommonFields.FIELDS.match(currentFieldName)) {
                     fields = Collections.singletonList(parser.text());
-                } else if (formattable && parseFieldMatcher.match(currentFieldName, CommonFields.FORMAT)) {
+                } else if (formattable && CommonFields.FORMAT.match(currentFieldName)) {
                     format = parser.text();
-                } else if (parseFieldMatcher.match(currentFieldName, CommonFields.VALUE_TYPE)) {
+                } else if (CommonFields.VALUE_TYPE.match(currentFieldName)) {
                     throw new ParsingException(parser.getTokenLocation(),
                         "Unexpected token " + token + " [" + currentFieldName + "] in [" + aggregationName + "]. " +
                             "Multi-field aggregations do not support scripts.");
-                } else if (!token(aggregationName, currentFieldName, token, parser, context.getParseFieldMatcher(), otherOptions)) {
+                } else if (!token(aggregationName, currentFieldName, token, parser, otherOptions)) {
                     throw new ParsingException(parser.getTokenLocation(),
                             "Unexpected token " + token + " [" + currentFieldName + "] in [" + aggregationName + "].");
                 }
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (parseFieldMatcher.match(currentFieldName, CommonFields.MISSING)) {
+                if (CommonFields.MISSING.match(currentFieldName)) {
                     missingMap = new HashMap<>();
                     while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
                         parseMissingAndAdd(aggregationName, currentFieldName, parser, missingMap);
                     }
-                } else if (context.getParseFieldMatcher().match(currentFieldName, ScriptField.SCRIPT)) {
+                } else if (Script.SCRIPT_PARSE_FIELD.match(currentFieldName)) {
                     throw new ParsingException(parser.getTokenLocation(),
                         "Unexpected token " + token + " [" + currentFieldName + "] in [" + aggregationName + "]. " +
                             "Multi-field aggregations do not support scripts.");
 
-                } else if (!token(aggregationName, currentFieldName, token, parser, context.getParseFieldMatcher(), otherOptions)) {
+                } else if (!token(aggregationName, currentFieldName, token, parser, otherOptions)) {
                     throw new ParsingException(parser.getTokenLocation(),
                             "Unexpected token " + token + " [" + currentFieldName + "] in [" + aggregationName + "].");
                 }
             } else if (token == XContentParser.Token.START_ARRAY) {
-                if (context.getParseFieldMatcher().match(currentFieldName, ScriptField.SCRIPT)) {
+                if (Script.SCRIPT_PARSE_FIELD.match(currentFieldName)) {
                     throw new ParsingException(parser.getTokenLocation(),
                         "Unexpected token " + token + " [" + currentFieldName + "] in [" + aggregationName + "]. " +
                             "Multi-field aggregations do not support scripts.");
-                } else if (parseFieldMatcher.match(currentFieldName, CommonFields.FIELDS)) {
+                } else if (CommonFields.FIELDS.match(currentFieldName)) {
                     fields = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         if (token == XContentParser.Token.VALUE_STRING) {
@@ -138,11 +135,11 @@ private MultiValuesSourceParser(boolean formattable, ValuesSourceType valuesSour
                                     "Unexpected token " + token + " [" + currentFieldName + "] in [" + aggregationName + "].");
                         }
                     }
-                } else if (!token(aggregationName, currentFieldName, token, parser, context.getParseFieldMatcher(), otherOptions)) {
+                } else if (!token(aggregationName, currentFieldName, token, parser, otherOptions)) {
                     throw new ParsingException(parser.getTokenLocation(),
                             "Unexpected token " + token + " [" + currentFieldName + "] in [" + aggregationName + "].");
                 }
-            } else if (!token(aggregationName, currentFieldName, token, parser, context.getParseFieldMatcher(), otherOptions)) {
+            } else if (!token(aggregationName, currentFieldName, token, parser, otherOptions)) {
                 throw new ParsingException(parser.getTokenLocation(),
                         "Unexpected token " + token + " [" + currentFieldName + "] in [" + aggregationName + "].");
             }
@@ -201,8 +198,7 @@ private void parseMissingAndAdd(final String aggregationName, final String curre
      *            the target type of the final value output by the aggregation
      * @param otherOptions
      *            a {@link Map} containing the extra options parsed by the
-     *            {@link #token(String, String, org.elasticsearch.common.xcontent.XContentParser.Token,
-     *             XContentParser, ParseFieldMatcher, Map)}
+     *            {@link #token(String, String, XContentParser.Token, XContentParser, Map)}
      *            method
      * @return the created factory
      */
@@ -222,8 +218,6 @@ private void parseMissingAndAdd(final String aggregationName, final String curre
      *            the current token for the parser
      * @param parser
      *            the parser
-     * @param parseFieldMatcher
-     *            the {@link ParseFieldMatcher} to use to match field names
      * @param otherOptions
      *            a {@link Map} of options to be populated by successive calls
      *            to this method which will then be passed to the
@@ -235,5 +229,5 @@ private void parseMissingAndAdd(final String aggregationName, final String curre
      *             if an error occurs whilst parsing
      */
     protected abstract boolean token(String aggregationName, String currentFieldName, XContentParser.Token token, XContentParser parser,
-        ParseFieldMatcher parseFieldMatcher, Map<ParseField, Object> otherOptions) throws IOException;
+                                     Map<ParseField, Object> otherOptions) throws IOException;
 }
diff --git a/modules/aggs-matrix-stats/src/main/resources/META-INF/services/org.elasticsearch.plugins.spi.NamedXContentProvider b/modules/aggs-matrix-stats/src/main/resources/META-INF/services/org.elasticsearch.plugins.spi.NamedXContentProvider
new file mode 100644
index 0000000000000..a2d706a39a60c
--- /dev/null
+++ b/modules/aggs-matrix-stats/src/main/resources/META-INF/services/org.elasticsearch.plugins.spi.NamedXContentProvider
@@ -0,0 +1 @@
+org.elasticsearch.search.aggregations.matrix.spi.MatrixStatsNamedXContentProvider
\ No newline at end of file
diff --git a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/MatrixStatsClientYamlTestSuiteIT.java b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/MatrixStatsClientYamlTestSuiteIT.java
index 6fb70ad3d63b9..be27e9050df98 100644
--- a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/MatrixStatsClientYamlTestSuiteIT.java
+++ b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/MatrixStatsClientYamlTestSuiteIT.java
@@ -23,9 +23,6 @@
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-
-import java.io.IOException;
 
 public class MatrixStatsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     public MatrixStatsClientYamlTestSuiteIT(@Name("yaml")ClientYamlTestCandidate testCandidate) {
@@ -33,7 +30,7 @@ public MatrixStatsClientYamlTestSuiteIT(@Name("yaml")ClientYamlTestCandidate tes
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
-        return ESClientYamlSuiteTestCase.createParameters(0, 1);
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStatsTests.java b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStatsTests.java
new file mode 100644
index 0000000000000..538993eda0d55
--- /dev/null
+++ b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStatsTests.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.matrix.stats;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.MockBigArrays;
+import org.elasticsearch.common.xcontent.ContextParser;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.search.aggregations.Aggregation;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.ParsedAggregation;
+import org.elasticsearch.search.aggregations.matrix.stats.InternalMatrixStats.Fields;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.test.InternalAggregationTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Predicate;
+
+public class InternalMatrixStatsTests extends InternalAggregationTestCase<InternalMatrixStats> {
+
+    private String[] fields;
+    private boolean hasMatrixStatsResults;
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        hasMatrixStatsResults = frequently();
+        int numFields = hasMatrixStatsResults ? randomInt(128) : 0;
+        fields = new String[numFields];
+        for (int i = 0; i < numFields; i++) {
+            fields[i] = "field_" + i;
+        }
+    }
+
+    @Override
+    protected List<NamedXContentRegistry.Entry> getNamedXContents() {
+        List<NamedXContentRegistry.Entry> namedXContents = new ArrayList<>(getDefaultNamedXContents());
+        ContextParser<Object, Aggregation> parser = (p, c) -> ParsedMatrixStats.fromXContent(p, (String) c);
+        namedXContents.add(new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(MatrixStatsAggregationBuilder.NAME), parser));
+        return namedXContents;
+    }
+
+    @Override
+    protected InternalMatrixStats createTestInstance(String name, List<PipelineAggregator> pipelineAggregators,
+                                                     Map<String, Object> metaData) {
+        double[] values = new double[fields.length];
+        for (int i = 0; i < fields.length; i++) {
+            values[i] = randomDouble();
+        }
+
+        RunningStats runningStats = new RunningStats();
+        runningStats.add(fields, values);
+        MatrixStatsResults matrixStatsResults = hasMatrixStatsResults ? new MatrixStatsResults(runningStats) : null;
+        return new InternalMatrixStats(name, 1L, runningStats, matrixStatsResults, Collections.emptyList(), Collections.emptyMap());
+    }
+
+    @Override
+    protected void assertFromXContent(InternalMatrixStats expected, ParsedAggregation parsedAggregation) throws IOException {
+        assertTrue(parsedAggregation instanceof ParsedMatrixStats);
+        ParsedMatrixStats actual = (ParsedMatrixStats) parsedAggregation;
+
+        assertEquals(expected.getDocCount(), actual.getDocCount());
+
+        for (String field : fields) {
+            assertEquals(expected.getFieldCount(field), actual.getFieldCount(field));
+            assertEquals(expected.getMean(field), actual.getMean(field), 0.0);
+            assertEquals(expected.getVariance(field), actual.getVariance(field), 0.0);
+            assertEquals(expected.getSkewness(field), actual.getSkewness(field), 0.0);
+            assertEquals(expected.getKurtosis(field), actual.getKurtosis(field), 0.0);
+
+            for (String other : fields) {
+                assertEquals(expected.getCovariance(field, other), actual.getCovariance(field, other), 0.0);
+                assertEquals(expected.getCorrelation(field, other), actual.getCorrelation(field, other), 0.0);
+            }
+        }
+
+        final String unknownField = randomAlphaOfLength(3);
+        final String other = randomAlphaOfLength(3);
+
+        for (MatrixStats matrix : Arrays.asList(actual)) {
+
+            // getFieldCount returns 0 for unknown fields
+            assertEquals(0.0, matrix.getFieldCount(unknownField), 0.0);
+
+            expectThrows(IllegalArgumentException.class, () -> matrix.getMean(unknownField));
+            expectThrows(IllegalArgumentException.class, () -> matrix.getVariance(unknownField));
+            expectThrows(IllegalArgumentException.class, () -> matrix.getSkewness(unknownField));
+            expectThrows(IllegalArgumentException.class, () -> matrix.getKurtosis(unknownField));
+
+            expectThrows(IllegalArgumentException.class, () -> matrix.getCovariance(unknownField, unknownField));
+            expectThrows(IllegalArgumentException.class, () -> matrix.getCovariance(unknownField, other));
+            expectThrows(IllegalArgumentException.class, () -> matrix.getCovariance(other, unknownField));
+
+            assertEquals(1.0, matrix.getCorrelation(unknownField, unknownField), 0.0);
+            expectThrows(IllegalArgumentException.class, () -> matrix.getCorrelation(unknownField, other));
+            expectThrows(IllegalArgumentException.class, () -> matrix.getCorrelation(other, unknownField));
+        }
+    }
+
+    @Override
+    protected Predicate<String> excludePathsFromXContentInsertion() {
+        return path -> path.endsWith(Fields.CORRELATION) || path.endsWith(Fields.COVARIANCE);
+    }
+}
diff --git a/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/20_empty_bucket.yaml b/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/20_empty_bucket.yaml
index b6fade2cf1880..5ed2fe542ee24 100644
--- a/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/20_empty_bucket.yaml
+++ b/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/20_empty_bucket.yaml
@@ -1,41 +1,41 @@
 ---
 "Empty Bucket Aggregation":
   - do:
-    indices.create:
-      index: empty_bucket_idx
-      body:
-        settings:
-          number_of_shards: "3"
-        mappings:
-          test:
-            "properties":
-              "value":
-                "type": "integer"
-              "val1":
-                "type": "double"
+      indices.create:
+        index: empty_bucket_idx
+        body:
+          settings:
+            number_of_shards: "3"
+          mappings:
+            test:
+              "properties":
+                "value":
+                  "type": "integer"
+                "val1":
+                  "type": "double"
 
   - do:
-    index:
-      index:  empty_bucket_idx
-      type:   test
-      id:     1
-      body:   { "value": 0, "val1": 3.1 }
+      index:
+        index:  empty_bucket_idx
+        type:   test
+        id:     1
+        body:   { "value": 0, "val1": 3.1 }
 
   - do:
-    index:
-      index:  empty_bucket_idx
-      type:   test
-      id:     2
-      body:   { "value": 2, "val1": -3.1 }
+      index:
+        index:  empty_bucket_idx
+        type:   test
+        id:     2
+        body:   { "value": 2, "val1": -3.1 }
 
   - do:
-    indices.refresh:
-      index: [empty_bucket_idx]
+      indices.refresh:
+        index: [empty_bucket_idx]
 
   - do:
-    search:
-      index: empty_bucket_idx
-      type: test
+      search:
+        index: empty_bucket_idx
+        type: test
 
   - match: {hits.total: 2}
 
diff --git a/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/30_single_value_field.yaml b/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/30_single_value_field.yaml
index 6c2098f181e05..782e5d5ebbc4a 100644
--- a/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/30_single_value_field.yaml
+++ b/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/30_single_value_field.yaml
@@ -2,122 +2,122 @@
 setup:
 
   - do:
-    indices.create:
-      index: test
-      body:
-        settings:
-          number_of_shards: 3
-        mappings:
-          test:
-            "properties":
-              "val1":
-                "type": "double"
-              "val2":
-                "type": "double"
-              "val3":
-                "type": "double"
-
-  - do:
-    indices.create:
-      index: unmapped
-      body:
-        settings:
-          number_of_shards: 3
-
-  - do:
-    index:
-      index:  test
-      type:   test
-      id:     1
-      body:   { "val1": 1.9, "val2": 3.1, "val3": 2.3 }
-  - do:
-    index:
-      index:  test
-      type:   test
-      id:     2
-      body:   { "val1": -5.2, "val2": -3.4, "val3": 2.3}
-  - do:
-    index:
-      index:  test
-      type:   test
-      id:     3
-      body:   { "val1": -5.2, "val3": 2.3}
-  - do:
-    index:
-      index:  test
-      type:   test
-      id:     4
-      body:   { "val1": 18.3, "val2": 104.4, "val3": 2.3}
-  - do:
-    index:
-      index:  test
-      type:   test
-      id:     5
-      body:   { "val1": -53.2, "val2": -322.4, "val3": 2.3}
-  - do:
-    index:
-      index:  test
-      type:   test
-      id:     6
-      body:   { "val1": -578.9, "val2": 69.9, "val3": 2.3}
-  - do:
-    index:
-      index:  test
-      type:   test
-      id:     7
-      body:   { "val1": 16.2, "val2": 17.2, "val3": 2.3}
-  - do:
-    index:
-      index:  test
-      type:   test
-      id:     8
-      body:   { "val1": -4222.63, "val2": 316.44, "val3": 2.3}
-  - do:
-    index:
-      index:  test
-      type:   test
-      id:     9
-      body:   { "val1": -59999.55, "val2": -3163.4, "val3": 2.3}
-  - do:
-    index:
-      index:  test
-      type:   test
-      id:     10
-      body:   { "val1": 782.7, "val2": 789.7, "val3": 2.3}
-  - do:
-    index:
-      index:  test
-      type:   test
-      id:     11
-      body:   { "val1": -1.2, "val2": 6.3, "val3": 2.3}
-  - do:
-    index:
-      index:  test
-      type:   test
-      id:     12
-      body:   { "val1": 0, "val2": 1.11, "val3": 2.3}
-  - do:
-    index:
-      index:  test
-      type:   test
-      id:     13
-      body:   { "val1": 0.1, "val2": 0.92, "val3": 2.3}
-  - do:
-    index:
-      index:  test
-      type:   test
-      id:     14
-      body:   { "val1": 0.12, "val2": -82.4, "val3": 2.3}
-  - do:
-    index:
-      index:  test
-      type:   test
-      id:     15
-      body:   { "val1": 98.2, "val2": 32.4, "val3": 2.3}
-
-  - do:
-    indices.refresh:
-      index: [test, unmapped]
+      indices.create:
+        index: test
+        body:
+          settings:
+            number_of_shards: 3
+          mappings:
+            test:
+              "properties":
+                "val1":
+                  "type": "double"
+                "val2":
+                  "type": "double"
+                "val3":
+                  "type": "double"
+
+  - do:
+      indices.create:
+        index: unmapped
+        body:
+          settings:
+            number_of_shards: 3
+
+  - do:
+      index:
+        index:  test
+        type:   test
+        id:     1
+        body:   { "val1": 1.9, "val2": 3.1, "val3": 2.3 }
+  - do:
+      index:
+        index:  test
+        type:   test
+        id:     2
+        body:   { "val1": -5.2, "val2": -3.4, "val3": 2.3}
+  - do:
+      index:
+        index:  test
+        type:   test
+        id:     3
+        body:   { "val1": -5.2, "val3": 2.3}
+  - do:
+      index:
+        index:  test
+        type:   test
+        id:     4
+        body:   { "val1": 18.3, "val2": 104.4, "val3": 2.3}
+  - do:
+      index:
+        index:  test
+        type:   test
+        id:     5
+        body:   { "val1": -53.2, "val2": -322.4, "val3": 2.3}
+  - do:
+      index:
+        index:  test
+        type:   test
+        id:     6
+        body:   { "val1": -578.9, "val2": 69.9, "val3": 2.3}
+  - do:
+      index:
+        index:  test
+        type:   test
+        id:     7
+        body:   { "val1": 16.2, "val2": 17.2, "val3": 2.3}
+  - do:
+      index:
+        index:  test
+        type:   test
+        id:     8
+        body:   { "val1": -4222.63, "val2": 316.44, "val3": 2.3}
+  - do:
+      index:
+        index:  test
+        type:   test
+        id:     9
+        body:   { "val1": -59999.55, "val2": -3163.4, "val3": 2.3}
+  - do:
+      index:
+        index:  test
+        type:   test
+        id:     10
+        body:   { "val1": 782.7, "val2": 789.7, "val3": 2.3}
+  - do:
+      index:
+        index:  test
+        type:   test
+        id:     11
+        body:   { "val1": -1.2, "val2": 6.3, "val3": 2.3}
+  - do:
+      index:
+        index:  test
+        type:   test
+        id:     12
+        body:   { "val1": 0, "val2": 1.11, "val3": 2.3}
+  - do:
+      index:
+        index:  test
+        type:   test
+        id:     13
+        body:   { "val1": 0.1, "val2": 0.92, "val3": 2.3}
+  - do:
+      index:
+        index:  test
+        type:   test
+        id:     14
+        body:   { "val1": 0.12, "val2": -82.4, "val3": 2.3}
+  - do:
+      index:
+        index:  test
+        type:   test
+        id:     15
+        body:   { "val1": 98.2, "val2": 32.4, "val3": 2.3}
+
+  - do:
+      indices.refresh:
+        index: [test, unmapped]
 
   - do:
       cluster.health:
@@ -133,6 +133,7 @@ setup:
         body: {"aggs": { "mfs" : { "matrix_stats": {"fields": ["val1", "val2", "val3"]} } } }
 
   - match: {hits.total: 0}
+  - match: {aggregations.mfs.doc_count: 0}
 
 ---
 "Single value field":
@@ -144,6 +145,7 @@ setup:
         body: {"aggs": { "mfs" : { "matrix_stats": {"fields": ["val1", "val3"]} } } }
 
   - match: {hits.total: 15}
+  - match: {aggregations.mfs.doc_count: 15}
   - match: {aggregations.mfs.fields.0.count: 15}
 
 ---
@@ -156,6 +158,7 @@ setup:
         body: {"aggs": { "mfs" : { "matrix_stats": {"fields": ["val1", "val2", "val3"]} } } }
 
   - match: {hits.total: 15}
+  - match: {aggregations.mfs.doc_count: 14}
   - match: {aggregations.mfs.fields.0.count: 14}
   - match: {aggregations.mfs.fields.2.correlation.val2: 0.9569513137793205}
 
@@ -169,6 +172,7 @@ setup:
         body: {"aggs": { "mfs" : { "matrix_stats": {"fields": ["val1", "val2", "val3"], "missing" : {"val2" : 10} } } } }
 
   - match: {hits.total: 15}
+  - match: {aggregations.mfs.doc_count: 15}
   - match: {aggregations.mfs.fields.0.count: 15}
   - match: {aggregations.mfs.fields.2.correlation.val2: 0.9567970467908384}
 
@@ -180,7 +184,7 @@ setup:
       search:
         index: test
         type:  test
-        body: {"aggs": { "mfs" : { "matrix_stats": {"fields": ["val1", "val2"], "script" : { "my_script" : {"inline" : "1 + doc['val1'].value", "lang" : "js"} } } } } }
+        body: {"aggs": { "mfs" : { "matrix_stats": {"fields": ["val1", "val2"], "script" : { "my_script" : {"source" : "1 + doc['val1'].value", "lang" : "js"} } } } } }
 
 ---
 "With script params":
@@ -190,4 +194,4 @@ setup:
       search:
         index: test
         type:  test
-        body: {"aggs": { "mfs" : { "matrix_stats": {"fields": ["val1", "val2", "val3"], "script" : { "my_script" : {"inline" : "my_var + doc['val1'].value", "params" : { "my_var" : 1 }, "lang" : "js" } } } } } }
+        body: {"aggs": { "mfs" : { "matrix_stats": {"fields": ["val1", "val2", "val3"], "script" : { "my_script" : {"source" : "my_var + doc['val1'].value", "params" : { "my_var" : 1 }, "lang" : "js" } } } } } }
diff --git a/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/40_multi_value_field.yaml b/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/40_multi_value_field.yaml
index 10b722555d887..b4abf41a515e9 100644
--- a/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/40_multi_value_field.yaml
+++ b/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/40_multi_value_field.yaml
@@ -2,122 +2,122 @@
 setup:
 
   - do:
-    indices.create:
-      index: test
-      body:
-        settings:
-          number_of_shards: 3
-        mappings:
-          test:
-            "properties":
-              "val1":
-                "type": "double"
-              "val2":
-                "type": "double"
-              "val3":
-                "type": "double"
-
-  - do:
-    indices.create:
-      index: unmapped
-      body:
-        settings:
-          number_of_shards: 3
-
-  - do:
-    index:
-      index:  test
-      type:   test
-      id:     1
-      body:   { "val1": 1.9, "val2": 3.1, "val3": 2.3, "vals" : [1.9, 16.143] }
-  - do:
-    index:
-      index:  test
-      type:   test
-      id:     2
-      body:   { "val1": -5.2, "val2": -3.4, "val3": 2.3, "vals" : [155, 16.23]}
-  - do:
-    index:
-      index:  test
-      type:   test
-      id:     3
-      body:   { "val1": -5.2, "val3": 2.3, "vals" : [-455, -32.32]}
-  - do:
-    index:
-      index:  test
-      type:   test
-      id:     4
-      body:   { "val1": 18.3, "val2": 104.4, "val3": 2.3, "vals" : [0.14, 92.1]}
-  - do:
-    index:
-      index:  test
-      type:   test
-      id:     5
-      body:   { "val1": -53.2, "val2": -322.4, "val3": 2.3, "vals" : [16, 16]}
-  - do:
-    index:
-      index:  test
-      type:   test
-      id:     6
-      body:   { "val1": -578.9, "val2": 69.9, "val3": 2.3}
-  - do:
-    index:
-      index:  test
-      type:   test
-      id:     7
-      body:   { "val1": 16.2, "val2": 17.2, "val3": 2.3, "vals" : [1234.3, -3433]}
-  - do:
-    index:
-      index:  test
-      type:   test
-      id:     8
-      body:   { "val1": -4222.63, "val2": 316.44, "val3": 2.3, "vals" : [177.2, -93.333]}
-  - do:
-    index:
-      index:  test
-      type:   test
-      id:     9
-      body:   { "val1": -59999.55, "val2": -3163.4, "val3": 2.3, "vals" : [-29.9, 163.0]}
-  - do:
-    index:
-      index:  test
-      type:   test
-      id:     10
-      body:   { "val1": 782.7, "val2": 789.7, "val3": 2.3, "vals" : [-0.2, 1343.3]}
-  - do:
-    index:
-      index:  test
-      type:   test
-      id:     11
-      body:   { "val1": -1.2, "val2": 6.3, "val3": 2.3, "vals" : [15.3, 16.9]}
-  - do:
-    index:
-      index:  test
-      type:   test
-      id:     12
-      body:   { "val1": 0, "val2": 1.11, "val3": 2.3, "vals" : [-644.4, -644.4]}
-  - do:
-    index:
-      index:  test
-      type:   test
-      id:     13
-      body:   { "val1": 0.1, "val2": 0.92, "val3": 2.3, "vals" : [73.2, 0.12]}
-  - do:
-    index:
-      index:  test
-      type:   test
-      id:     14
-      body:   { "val1": 0.12, "val2": -82.4, "val3": 2.3, "vals" : [-0.001, 1295.3]}
-  - do:
-    index:
-      index:  test
-      type:   test
-      id:     15
-      body:   { "val1": 98.2, "val2": 32.4, "val3": 2.3, "vals" : [15.5, 16.5]}
-
-  - do:
-    indices.refresh:
-      index: [test, unmapped]
+      indices.create:
+        index: test
+        body:
+          settings:
+            number_of_shards: 3
+          mappings:
+            test:
+              "properties":
+                "val1":
+                  "type": "double"
+                "val2":
+                  "type": "double"
+                "val3":
+                  "type": "double"
+
+  - do:
+      indices.create:
+        index: unmapped
+        body:
+          settings:
+            number_of_shards: 3
+
+  - do:
+      index:
+        index:  test
+        type:   test
+        id:     1
+        body:   { "val1": 1.9, "val2": 3.1, "val3": 2.3, "vals" : [1.9, 16.143] }
+  - do:
+      index:
+        index:  test
+        type:   test
+        id:     2
+        body:   { "val1": -5.2, "val2": -3.4, "val3": 2.3, "vals" : [155, 16.23]}
+  - do:
+      index:
+        index:  test
+        type:   test
+        id:     3
+        body:   { "val1": -5.2, "val3": 2.3, "vals" : [-455, -32.32]}
+  - do:
+      index:
+        index:  test
+        type:   test
+        id:     4
+        body:   { "val1": 18.3, "val2": 104.4, "val3": 2.3, "vals" : [0.14, 92.1]}
+  - do:
+      index:
+        index:  test
+        type:   test
+        id:     5
+        body:   { "val1": -53.2, "val2": -322.4, "val3": 2.3, "vals" : [16, 16]}
+  - do:
+      index:
+        index:  test
+        type:   test
+        id:     6
+        body:   { "val1": -578.9, "val2": 69.9, "val3": 2.3}
+  - do:
+      index:
+        index:  test
+        type:   test
+        id:     7
+        body:   { "val1": 16.2, "val2": 17.2, "val3": 2.3, "vals" : [1234.3, -3433]}
+  - do:
+      index:
+        index:  test
+        type:   test
+        id:     8
+        body:   { "val1": -4222.63, "val2": 316.44, "val3": 2.3, "vals" : [177.2, -93.333]}
+  - do:
+      index:
+        index:  test
+        type:   test
+        id:     9
+        body:   { "val1": -59999.55, "val2": -3163.4, "val3": 2.3, "vals" : [-29.9, 163.0]}
+  - do:
+      index:
+        index:  test
+        type:   test
+        id:     10
+        body:   { "val1": 782.7, "val2": 789.7, "val3": 2.3, "vals" : [-0.2, 1343.3]}
+  - do:
+      index:
+        index:  test
+        type:   test
+        id:     11
+        body:   { "val1": -1.2, "val2": 6.3, "val3": 2.3, "vals" : [15.3, 16.9]}
+  - do:
+      index:
+        index:  test
+        type:   test
+        id:     12
+        body:   { "val1": 0, "val2": 1.11, "val3": 2.3, "vals" : [-644.4, -644.4]}
+  - do:
+      index:
+        index:  test
+        type:   test
+        id:     13
+        body:   { "val1": 0.1, "val2": 0.92, "val3": 2.3, "vals" : [73.2, 0.12]}
+  - do:
+      index:
+        index:  test
+        type:   test
+        id:     14
+        body:   { "val1": 0.12, "val2": -82.4, "val3": 2.3, "vals" : [-0.001, 1295.3]}
+  - do:
+      index:
+        index:  test
+        type:   test
+        id:     15
+        body:   { "val1": 98.2, "val2": 32.4, "val3": 2.3, "vals" : [15.5, 16.5]}
+
+  - do:
+      indices.refresh:
+        index: [test, unmapped]
 
   - do:
       cluster.health:
@@ -133,6 +133,7 @@ setup:
         body: {"aggs": { "mfs" : { "matrix_stats": {"fields": ["val1", "val2", "vals"]} } } }
 
   - match: {hits.total: 0}
+  - match: {aggregations.mfs.doc_count: 0}
 
 ---
 "Multi value field Max":
@@ -144,6 +145,7 @@ setup:
         body: {"aggs": { "mfs" : { "matrix_stats": {"fields": ["val1", "vals"], "mode" : "max"} } } }
 
   - match: {hits.total: 15}
+  - match: {aggregations.mfs.doc_count: 14}
   - match: {aggregations.mfs.fields.0.count: 14}
   - match: {aggregations.mfs.fields.0.correlation.val1: 0.06838646533369998}
 
@@ -157,6 +159,7 @@ setup:
         body: {"aggs": { "mfs" : { "matrix_stats": {"fields": ["val1", "vals"], "mode" : "min"} } } }
 
   - match: {hits.total: 15}
+  - match: {aggregations.mfs.doc_count: 14}
   - match: {aggregations.mfs.fields.0.count: 14}
   - match: {aggregations.mfs.fields.0.correlation.val1: -0.09777682707831963}
 
@@ -170,6 +173,7 @@ setup:
         body: {"aggs": { "mfs" : { "matrix_stats": {"fields": ["val1", "val2", "vals"]} } } }
 
   - match: {hits.total: 15}
+  - match: {aggregations.mfs.doc_count: 13}
   - match: {aggregations.mfs.fields.0.count: 13}
   - match: {aggregations.mfs.fields.0.correlation.val1: -0.044997535185684244}
 
@@ -183,6 +187,7 @@ setup:
         body: {"aggs": { "mfs" : { "matrix_stats": {"fields": ["val1", "val2", "vals"], "missing" : {"val2" : 10, "vals" : 5 } } } } }
 
   - match: {hits.total: 15}
+  - match: {aggregations.mfs.doc_count: 15}
   - match: {aggregations.mfs.fields.0.count: 15}
   - match: {aggregations.mfs.fields.0.correlation.val2: 0.04028024709708195}
 
@@ -194,7 +199,7 @@ setup:
       search:
         index: test
         type:  test
-        body: {"aggs": { "mfs" : { "matrix_stats": {"fields": ["vals", "val3"], "script" : { "my_script" : {"inline" : "1 + doc['val1'].value", "lang" : "js"} } } } } }
+        body: {"aggs": { "mfs" : { "matrix_stats": {"fields": ["vals", "val3"], "script" : { "my_script" : {"source" : "1 + doc['val1'].value", "lang" : "js"} } } } } }
 
 ---
 "With script params":
@@ -204,4 +209,4 @@ setup:
       search:
         index: test
         type:  test
-        body: {"aggs": { "mfs" : { "matrix_stats": {"fields": ["val1", "val3", "vals"], "script" : { "my_script" : {"inline" : "my_var + doc['val1'].value", "params" : { "my_var" : 1 }, "lang" : "js" } } } } } }
+        body: {"aggs": { "mfs" : { "matrix_stats": {"fields": ["val1", "val3", "vals"], "script" : { "my_script" : {"source" : "my_var + doc['val1'].value", "params" : { "my_var" : 1 }, "lang" : "js" } } } } } }
diff --git a/modules/build.gradle b/modules/build.gradle
index 11131c28e23c4..b3dbde24936e9 100644
--- a/modules/build.gradle
+++ b/modules/build.gradle
@@ -25,6 +25,11 @@ subprojects {
     // for local ES plugins, the name of the plugin is the same as the directory
     name project.name
   }
+  
+  run {
+    // these cannot be run with the normal distribution, since they are included in it!
+    distribution = 'integ-test-zip'
+  }
 
   if (project.file('src/main/packaging').exists()) {
     throw new InvalidModelException("Modules cannot contain packaging files") 
@@ -35,8 +40,4 @@ subprojects {
   if (project.file('src/main/config').exists()) {
     throw new InvalidModelException("Modules cannot contain config files") 
   }
-
-  // these are implementation details of our build, no need to publish them!
-  install.enabled = false
-  uploadArchives.enabled = false
 }
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java
index 079ff73846a2f..9c163290757bd 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java
@@ -32,29 +32,45 @@
  */
 abstract class AbstractStringProcessor extends AbstractProcessor {
     private final String field;
+    private final boolean ignoreMissing;
+    private final String targetField;
 
-    protected AbstractStringProcessor(String tag, String field) {
+    AbstractStringProcessor(String tag, String field, boolean ignoreMissing, String targetField) {
         super(tag);
         this.field = field;
+        this.ignoreMissing = ignoreMissing;
+        this.targetField = targetField;
     }
 
     public String getField() {
         return field;
     }
 
+    boolean isIgnoreMissing() {
+        return ignoreMissing;
+    }
+
+    String getTargetField() {
+        return targetField;
+    }
+
     @Override
     public final void execute(IngestDocument document) {
-        String val = document.getFieldValue(field, String.class);
-        if (val == null) {
+        String val = document.getFieldValue(field, String.class, ignoreMissing);
+
+        if (val == null && ignoreMissing) {
+            return;
+        } else if (val == null) {
             throw new IllegalArgumentException("field [" + field + "] is null, cannot process it.");
         }
-        document.setFieldValue(field, process(val));
+
+        document.setFieldValue(targetField, process(val));
     }
 
     protected abstract String process(String value);
 
     abstract static class Factory implements Processor.Factory {
-        protected final String processorType;
+        final String processorType;
 
         protected Factory(String processorType) {
             this.processorType = processorType;
@@ -64,9 +80,13 @@ protected Factory(String processorType) {
         public AbstractStringProcessor create(Map<String, Processor.Factory> registry, String tag,
                                               Map<String, Object> config) throws Exception {
             String field = ConfigurationUtils.readStringProperty(processorType, tag, config, "field");
-            return newProcessor(tag, field);
+            boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(processorType, tag, config, "ignore_missing", false);
+            String targetField = ConfigurationUtils.readStringProperty(processorType, tag, config, "target_field", field);
+
+            return newProcessor(tag, config, field, ignoreMissing, targetField);
         }
 
-        protected abstract AbstractStringProcessor newProcessor(String processorTag, String field);
+        protected abstract AbstractStringProcessor newProcessor(String processorTag, Map<String, Object> config, String field,
+                                                                boolean ignoreMissing, String targetField);
     }
 }
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AppendProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AppendProcessor.java
index 7bc6a17152a32..ddcb4c4bb95f0 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AppendProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AppendProcessor.java
@@ -21,7 +21,6 @@
 
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.ConfigurationUtils;
-import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.ingest.Processor;
 import org.elasticsearch.ingest.TemplateService;
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java
index c85e8d17a0060..8ecd4d99eb2bf 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java
@@ -114,12 +114,14 @@ public static Type fromString(String processorTag, String propertyName, String t
     private final String field;
     private final String targetField;
     private final Type convertType;
+    private final boolean ignoreMissing;
 
-    ConvertProcessor(String tag, String field, String targetField, Type convertType) {
+    ConvertProcessor(String tag, String field, String targetField, Type convertType, boolean ignoreMissing) {
         super(tag);
         this.field = field;
         this.targetField = targetField;
         this.convertType = convertType;
+        this.ignoreMissing = ignoreMissing;
     }
 
     String getField() {
@@ -134,17 +136,24 @@ Type getConvertType() {
         return convertType;
     }
 
+    boolean isIgnoreMissing() {
+        return ignoreMissing;
+    }
+
     @Override
     public void execute(IngestDocument document) {
-        Object oldValue = document.getFieldValue(field, Object.class);
+        Object oldValue = document.getFieldValue(field, Object.class, ignoreMissing);
         Object newValue;
-        if (oldValue == null) {
+
+        if (oldValue == null && ignoreMissing) {
+            return;
+        } else if (oldValue == null) {
             throw new IllegalArgumentException("Field [" + field + "] is null, cannot be converted to type [" + convertType + "]");
         }
 
         if (oldValue instanceof List) {
             List<?> list = (List<?>) oldValue;
-            List<Object> newList = new ArrayList<>();
+            List<Object> newList = new ArrayList<>(list.size());
             for (Object value : list) {
                 newList.add(convertType.convert(value));
             }
@@ -168,7 +177,8 @@ public ConvertProcessor create(Map<String, Processor.Factory> registry, String p
             String typeProperty = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "type");
             String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", field);
             Type convertType = Type.fromString(processorTag, "type", typeProperty);
-            return new ConvertProcessor(processorTag, field, targetField, convertType);
+            boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
+            return new ConvertProcessor(processorTag, field, targetField, convertType, ignoreMissing);
         }
     }
 }
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java
index 4a97a7e7ab2f5..bf664afb40777 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java
@@ -22,6 +22,7 @@
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
 import org.joda.time.format.ISODateTimeFormat;
 
 import java.util.Locale;
@@ -37,7 +38,7 @@ Function<String, DateTime> getFunction(String format, DateTimeZone timezone, Loc
     Unix {
         @Override
         Function<String, DateTime> getFunction(String format, DateTimeZone timezone, Locale locale) {
-            return (date) -> new DateTime((long)(Float.parseFloat(date) * 1000), timezone);
+            return (date) -> new DateTime((long)(Double.parseDouble(date) * 1000), timezone);
         }
     },
     UnixMs {
@@ -65,9 +66,8 @@ private long parseMillis(String date) {
     Joda {
         @Override
         Function<String, DateTime> getFunction(String format, DateTimeZone timezone, Locale locale) {
-            return DateTimeFormat.forPattern(format)
-                .withDefaultYear((new DateTime(DateTimeZone.UTC)).getYear())
-                .withZone(timezone).withLocale(locale)::parseDateTime;
+            DateTimeFormatter parser = DateTimeFormat.forPattern(format).withZone(timezone).withLocale(locale);
+            return text -> parser.withDefaultYear((new DateTime(DateTimeZone.UTC)).getYear()).parseDateTime(text);
         }
     };
 
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java
index 6ed5f0b66cf9b..e25891aca4e35 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java
@@ -25,6 +25,7 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Objects;
 import java.util.function.Function;
 
 import org.elasticsearch.ExceptionsHelper;
@@ -61,7 +62,8 @@ public final class DateIndexNameProcessor extends AbstractProcessor {
 
     @Override
     public void execute(IngestDocument ingestDocument) throws Exception {
-        String date = ingestDocument.getFieldValue(field, String.class);
+        // Date can be specified as a string or long:
+        String date = Objects.toString(ingestDocument.getFieldValue(field, Object.class));
 
         DateTime dateTime = null;
         Exception lastException = null;
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java
index 676cf2e7419c2..e6034138873aa 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java
@@ -54,7 +54,7 @@ public final class DateProcessor extends AbstractProcessor {
         this.field = field;
         this.targetField = targetField;
         this.formats = formats;
-        this.dateParsers = new ArrayList<>();
+        this.dateParsers = new ArrayList<>(this.formats.size());
         for (String format : formats) {
             DateFormat dateFormat = DateFormat.fromString(format);
             dateParsers.add(dateFormat.getFunction(format, timezone, locale));
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ForEachProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ForEachProcessor.java
index e5a720011a5ae..2a1046acb9cdb 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ForEachProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ForEachProcessor.java
@@ -38,6 +38,8 @@
  *
  * This can be useful in cases to do string operations on json array of strings,
  * or remove a field from objects inside a json array.
+ *
+ * Note that this processor is experimental.
  */
 public final class ForEachProcessor extends AbstractProcessor {
 
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/Grok.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/Grok.java
index b6e1877d3e0ac..576a3b85eb31c 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/Grok.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/Grok.java
@@ -60,12 +60,12 @@ final class Grok {
     private final String expression;
 
 
-    public Grok(Map<String, String> patternBank, String grokPattern) {
+    Grok(Map<String, String> patternBank, String grokPattern) {
         this(patternBank, grokPattern, true);
     }
 
     @SuppressWarnings("unchecked")
-    public Grok(Map<String, String> patternBank, String grokPattern, boolean namedCaptures) {
+    Grok(Map<String, String> patternBank, String grokPattern, boolean namedCaptures) {
         this.patternBank = patternBank;
         this.namedCaptures = namedCaptures;
 
@@ -106,8 +106,13 @@ public String toRegex(String grokPattern) {
             // TODO(tal): Support definitions
             String definition = groupMatch(DEFINITION_GROUP, region, grokPattern);
             String patternName = groupMatch(PATTERN_GROUP, region, grokPattern);
+
             String pattern = patternBank.get(patternName);
 
+            if (pattern == null) {
+                throw new IllegalArgumentException("Unable to find pattern [" + patternName + "] in Grok's pattern dictionary");
+            }
+
             String grokPart;
             if (namedCaptures && subName != null) {
                 grokPart = String.format(Locale.US, "(?<%s>%s)", namedPatternRef, pattern);
@@ -125,12 +130,25 @@ public String toRegex(String grokPattern) {
         return grokPattern;
     }
 
+    /**
+     * Checks whether a specific text matches the defined grok expression.
+     *
+     * @param text the string to match
+     * @return true if grok expression matches text, false otherwise.
+     */
     public boolean match(String text) {
         Matcher matcher = compiledExpression.matcher(text.getBytes(StandardCharsets.UTF_8));
         int result = matcher.search(0, text.length(), Option.DEFAULT);
         return (result != -1);
     }
 
+    /**
+     * Matches and returns any named captures within a compiled grok expression that matched
+     * within the provided text.
+     *
+     * @param text the text to match and extract values from.
+     * @return a map containing field names and their respective coerced values that matched.
+     */
     public Map<String, Object> captures(String text) {
         byte[] textAsBytes = text.getBytes(StandardCharsets.UTF_8);
         Map<String, Object> fields = new HashMap<>();
@@ -140,16 +158,17 @@ public Map<String, Object> captures(String text) {
             Region region = matcher.getEagerRegion();
             for (Iterator<NameEntry> entry = compiledExpression.namedBackrefIterator(); entry.hasNext();) {
                 NameEntry e = entry.next();
-                int number = e.getBackRefs()[0];
-
                 String groupName = new String(e.name, e.nameP, e.nameEnd - e.nameP, StandardCharsets.UTF_8);
-                String matchValue = null;
-                if (region.beg[number] >= 0) {
-                    matchValue = new String(textAsBytes, region.beg[number], region.end[number] - region.beg[number],
+                for (int number : e.getBackRefs()) {
+                    if (region.beg[number] >= 0) {
+                        String matchValue = new String(textAsBytes, region.beg[number], region.end[number] - region.beg[number],
                             StandardCharsets.UTF_8);
+                        GrokMatchGroup match = new GrokMatchGroup(groupName, matchValue);
+                        fields.put(match.getName(), match.getValue());
+                        break;
+                    }
                 }
-                GrokMatchGroup match = new GrokMatchGroup(groupName, matchValue);
-                fields.put(match.getName(), match.getValue());
+
             }
             return fields;
         } else if (result != -1) {
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokMatchGroup.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokMatchGroup.java
index 35f8fd312ff31..924e298e462c2 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokMatchGroup.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokMatchGroup.java
@@ -26,7 +26,7 @@ final class GrokMatchGroup {
     private final String type;
     private final String groupValue;
 
-    public GrokMatchGroup(String groupName, String groupValue) {
+    GrokMatchGroup(String groupName, String groupValue) {
         String[] parts = groupName.split(":");
         patternName = parts[0];
         if (parts.length >= 2) {
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java
index 44528bdac8224..f8eb49b923989 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java
@@ -37,38 +37,49 @@ public final class GrokProcessor extends AbstractProcessor {
     private static final String PATTERN_MATCH_KEY = "_ingest._grok_match_index";
 
     private final String matchField;
+    private final List<String> matchPatterns;
     private final Grok grok;
     private final boolean traceMatch;
+    private final boolean ignoreMissing;
 
-    public GrokProcessor(String tag, Map<String, String> patternBank, List<String> matchPatterns, String matchField) {
-        this(tag, patternBank, matchPatterns, matchField, false);
-    }
-
-    public GrokProcessor(String tag, Map<String, String> patternBank, List<String> matchPatterns, String matchField, boolean traceMatch) {
+    public GrokProcessor(String tag, Map<String, String> patternBank, List<String> matchPatterns, String matchField,
+                         boolean traceMatch, boolean ignoreMissing) {
         super(tag);
         this.matchField = matchField;
+        this.matchPatterns = matchPatterns;
         this.grok = new Grok(patternBank, combinePatterns(matchPatterns, traceMatch));
         this.traceMatch = traceMatch;
+        this.ignoreMissing = ignoreMissing;
     }
 
     @Override
     public void execute(IngestDocument ingestDocument) throws Exception {
-        String fieldValue = ingestDocument.getFieldValue(matchField, String.class);
+        String fieldValue = ingestDocument.getFieldValue(matchField, String.class, ignoreMissing);
+
+        if (fieldValue == null && ignoreMissing) {
+            return;
+        } else if (fieldValue == null) {
+            throw new IllegalArgumentException("field [" + matchField + "] is null, cannot process it.");
+        }
+
         Map<String, Object> matches = grok.captures(fieldValue);
         if (matches == null) {
             throw new IllegalArgumentException("Provided Grok expressions do not match field value: [" + fieldValue + "]");
         }
 
         matches.entrySet().stream()
-            .filter((e) -> Objects.nonNull(e.getValue()))
             .forEach((e) -> ingestDocument.setFieldValue(e.getKey(), e.getValue()));
 
         if (traceMatch) {
-            @SuppressWarnings("unchecked")
-            HashMap<String, String> matchMap = (HashMap<String, String>) ingestDocument.getFieldValue(PATTERN_MATCH_KEY, Object.class);
-            matchMap.keySet().stream().findFirst().ifPresent((index) -> {
-                ingestDocument.setFieldValue(PATTERN_MATCH_KEY, index);
-            });
+            if (matchPatterns.size() > 1) {
+                @SuppressWarnings("unchecked")
+                HashMap<String, String> matchMap = (HashMap<String, String>) ingestDocument.getFieldValue(PATTERN_MATCH_KEY, Object.class);
+                matchMap.keySet().stream().findFirst().ifPresent((index) -> {
+                    ingestDocument.setFieldValue(PATTERN_MATCH_KEY, index);
+                });
+            } else {
+                ingestDocument.setFieldValue(PATTERN_MATCH_KEY, "0");
+            }
         }
     }
 
@@ -77,35 +88,39 @@ public String getType() {
         return TYPE;
     }
 
-    public Grok getGrok() {
+    Grok getGrok() {
         return grok;
     }
 
+    boolean isIgnoreMissing() {
+        return ignoreMissing;
+    }
+
     String getMatchField() {
         return matchField;
     }
 
+    List<String> getMatchPatterns() {
+        return matchPatterns;
+    }
+
     static String combinePatterns(List<String> patterns, boolean traceMatch) {
         String combinedPattern;
         if (patterns.size() > 1) {
-            if (traceMatch) {
-                combinedPattern = "";
-                for (int i = 0; i < patterns.size(); i++) {
-                    String valueWrap = "(?<" + PATTERN_MATCH_KEY + "." + i + ">" + patterns.get(i) + ")";
-                    if (combinedPattern.equals("")) {
-                        combinedPattern = valueWrap;
-                    } else {
-                        combinedPattern = combinedPattern + "|" + valueWrap;
-                    }
+            combinedPattern = "";
+            for (int i = 0; i < patterns.size(); i++) {
+                String pattern = patterns.get(i);
+                String valueWrap;
+                if (traceMatch) {
+                    valueWrap = "(?<" + PATTERN_MATCH_KEY + "." + i + ">" + pattern + ")";
+                } else {
+                    valueWrap = "(?:" + patterns.get(i) + ")";
+                }
+                if (combinedPattern.equals("")) {
+                    combinedPattern = valueWrap;
+                } else {
+                    combinedPattern = combinedPattern + "|" + valueWrap;
                 }
-            } else {
-                combinedPattern = patterns.stream().reduce("", (prefix, value) -> {
-                    if (prefix.equals("")) {
-                        return "(?:" + value + ")";
-                    } else {
-                        return prefix + "|" + "(?:" + value + ")";
-                    }
-                });
             }
         }  else {
             combinedPattern = patterns.get(0);
@@ -128,6 +143,7 @@ public GrokProcessor create(Map<String, Processor.Factory> registry, String proc
             String matchField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
             List<String> matchPatterns = ConfigurationUtils.readList(TYPE, processorTag, config, "patterns");
             boolean traceMatch = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "trace_match", false);
+            boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
 
             if (matchPatterns.isEmpty()) {
                 throw newConfigurationException(TYPE, processorTag, "patterns", "List of patterns must not be empty");
@@ -139,7 +155,7 @@ public GrokProcessor create(Map<String, Processor.Factory> registry, String proc
             }
 
             try {
-                return new GrokProcessor(processorTag, patternBank, matchPatterns, matchField, traceMatch);
+                return new GrokProcessor(processorTag, patternBank, matchPatterns, matchField, traceMatch, ignoreMissing);
             } catch (Exception e) {
                 throw newConfigurationException(TYPE, processorTag, "patterns",
                     "Invalid regex pattern found in: " + matchPatterns + ". " + e.getMessage());
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessorGetAction.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessorGetAction.java
new file mode 100644
index 0000000000000..e70038e0300d9
--- /dev/null
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessorGetAction.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.ingest.common;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.BytesRestResponse;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestResponse;
+import org.elasticsearch.rest.action.RestBuilderListener;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+
+import java.io.IOException;
+import java.util.Map;
+
+import static org.elasticsearch.ingest.common.IngestCommonPlugin.GROK_PATTERNS;
+import static org.elasticsearch.rest.RestRequest.Method.GET;
+import static org.elasticsearch.rest.RestStatus.OK;
+
+public class GrokProcessorGetAction extends Action<GrokProcessorGetAction.Request,
+    GrokProcessorGetAction.Response, GrokProcessorGetAction.RequestBuilder> {
+
+    public static final GrokProcessorGetAction INSTANCE = new GrokProcessorGetAction();
+    public static final String NAME = "cluster:admin/ingest/processor/grok/get";
+
+    private GrokProcessorGetAction() {
+        super(NAME);
+    }
+
+    @Override
+    public RequestBuilder newRequestBuilder(ElasticsearchClient client) {
+        return new RequestBuilder(client);
+    }
+
+    @Override
+    public Response newResponse() {
+        return new Response(null);
+    }
+
+    public static class Request extends ActionRequest {
+        @Override
+        public ActionRequestValidationException validate() {
+            return null;
+        }
+    }
+
+    public static class RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder> {
+        public RequestBuilder(ElasticsearchClient client) {
+            super(client, GrokProcessorGetAction.INSTANCE, new Request());
+        }
+    }
+
+    public static class Response extends AcknowledgedResponse implements ToXContentObject {
+        private Map<String, String> grokPatterns;
+
+        public Response(Map<String, String> grokPatterns) {
+            this.grokPatterns = grokPatterns;
+        }
+
+        public Map<String, String> getGrokPatterns() {
+            return grokPatterns;
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            builder.field("patterns");
+            builder.map(grokPatterns);
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            super.readFrom(in);
+            grokPatterns = in.readMap(StreamInput::readString, StreamInput::readString);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeMap(grokPatterns, StreamOutput::writeString, StreamOutput::writeString);
+        }
+    }
+
+    public static class TransportAction extends HandledTransportAction<Request, Response> {
+
+        @Inject
+        public TransportAction(Settings settings, ThreadPool threadPool, TransportService transportService,
+                               ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
+            super(settings, NAME, threadPool, transportService, actionFilters,
+                indexNameExpressionResolver, Request::new);
+        }
+
+        @Override
+        protected void doExecute(Request request, ActionListener<Response> listener) {
+            try {
+                listener.onResponse(new Response(GROK_PATTERNS));
+            } catch (Exception e) {
+                listener.onFailure(e);
+            }
+        }
+    }
+
+    public static class RestAction extends BaseRestHandler {
+        public RestAction(Settings settings, RestController controller) {
+            super(settings);
+            controller.registerHandler(GET, "/_ingest/processor/grok", this);
+        }
+
+        public String getName() {
+            return "ingest_processor_grok_get";
+        }
+
+        @Override
+        protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+            return channel -> client.executeLocally(INSTANCE, new Request(), new RestBuilderListener<Response>(channel) {
+                @Override
+                public RestResponse buildResponse(Response response, XContentBuilder builder) throws Exception {
+                    response.toXContent(builder, ToXContent.EMPTY_PARAMS);
+                    return new BytesRestResponse(OK, builder);
+                }
+            });
+        }
+    }
+}
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GsubProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GsubProcessor.java
index 36bbfaf7142b5..39553910692fc 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GsubProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GsubProcessor.java
@@ -19,12 +19,7 @@
 
 package org.elasticsearch.ingest.common;
 
-import org.elasticsearch.ingest.AbstractProcessor;
-import org.elasticsearch.ingest.IngestDocument;
-import org.elasticsearch.ingest.Processor;
-
 import java.util.Map;
-import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException;
@@ -34,25 +29,19 @@
  * Processor that allows to search for patterns in field content and replace them with corresponding string replacement.
  * Support fields of string type only, throws exception if a field is of a different type.
  */
-public final class GsubProcessor extends AbstractProcessor {
+public final class GsubProcessor extends AbstractStringProcessor {
 
     public static final String TYPE = "gsub";
 
-    private final String field;
     private final Pattern pattern;
     private final String replacement;
 
-    GsubProcessor(String tag, String field, Pattern pattern, String replacement) {
-        super(tag);
-        this.field = field;
+    GsubProcessor(String tag, String field, Pattern pattern, String replacement, boolean ignoreMissing, String targetField) {
+        super(tag, field, ignoreMissing, targetField);
         this.pattern = pattern;
         this.replacement = replacement;
     }
 
-    String getField() {
-        return field;
-    }
-
     Pattern getPattern() {
         return pattern;
     }
@@ -61,16 +50,9 @@ String getReplacement() {
         return replacement;
     }
 
-
     @Override
-    public void execute(IngestDocument document) {
-        String oldVal = document.getFieldValue(field, String.class);
-        if (oldVal == null) {
-            throw new IllegalArgumentException("field [" + field + "] is null, cannot match pattern.");
-        }
-        Matcher matcher = pattern.matcher(oldVal);
-        String newVal = matcher.replaceAll(replacement);
-        document.setFieldValue(field, newVal);
+    protected String process(String value) {
+        return pattern.matcher(value).replaceAll(replacement);
     }
 
     @Override
@@ -78,11 +60,15 @@ public String getType() {
         return TYPE;
     }
 
-    public static final class Factory implements Processor.Factory {
+    public static final class Factory extends AbstractStringProcessor.Factory {
+
+        public Factory() {
+            super(TYPE);
+        }
+
         @Override
-        public GsubProcessor create(Map<String, Processor.Factory> registry, String processorTag,
-                                    Map<String, Object> config) throws Exception {
-            String field = readStringProperty(TYPE, processorTag, config, "field");
+        protected AbstractStringProcessor newProcessor(String processorTag, Map<String, Object> config, String field,
+                                                       boolean ignoreMissing, String targetField) {
             String pattern = readStringProperty(TYPE, processorTag, config, "pattern");
             String replacement = readStringProperty(TYPE, processorTag, config, "replacement");
             Pattern searchPattern;
@@ -91,7 +77,7 @@ public GsubProcessor create(Map<String, Processor.Factory> registry, String proc
             } catch (Exception e) {
                 throw newConfigurationException(TYPE, processorTag, "pattern", "Invalid regex pattern. " + e.getMessage());
             }
-            return new GsubProcessor(processorTag, field, searchPattern, replacement);
+            return new GsubProcessor(processorTag, field, searchPattern, replacement, ignoreMissing, targetField);
         }
     }
 }
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java
index e6948771d8d68..15f59fd20fe3f 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java
@@ -23,21 +23,48 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.io.UncheckedIOException;
 import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
+import java.util.function.Supplier;
 
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.IndexScopedSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.ingest.Processor;
+import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.IngestPlugin;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestHandler;
 
-public class IngestCommonPlugin extends Plugin implements IngestPlugin {
+public class IngestCommonPlugin extends Plugin implements ActionPlugin, IngestPlugin {
 
-    private final Map<String, String> builtinPatterns;
+    // Code for loading built-in grok patterns packaged with the jar file:
+    private static final String[] PATTERN_NAMES = new String[] {
+        "aws", "bacula", "bro", "exim", "firewalls", "grok-patterns", "haproxy",
+        "java", "junos", "linux-syslog", "mcollective-patterns", "mongodb", "nagios",
+        "postgresql", "rails", "redis", "ruby"
+    };
+    static final Map<String, String> GROK_PATTERNS;
+    static {
+        try {
+            GROK_PATTERNS = loadBuiltinPatterns();
+        } catch (IOException e) {
+            throw new UncheckedIOException("unable to load built-in grok patterns", e);
+        }
+    }
 
     public IngestCommonPlugin() throws IOException {
-        this.builtinPatterns = loadBuiltinPatterns();
     }
 
     @Override
@@ -59,19 +86,27 @@ public Map<String, Processor.Factory> getProcessors(Processor.Parameters paramet
         processors.put(ForEachProcessor.TYPE, new ForEachProcessor.Factory());
         processors.put(DateIndexNameProcessor.TYPE, new DateIndexNameProcessor.Factory());
         processors.put(SortProcessor.TYPE, new SortProcessor.Factory());
-        processors.put(GrokProcessor.TYPE, new GrokProcessor.Factory(builtinPatterns));
+        processors.put(GrokProcessor.TYPE, new GrokProcessor.Factory(GROK_PATTERNS));
         processors.put(ScriptProcessor.TYPE, new ScriptProcessor.Factory(parameters.scriptService));
         processors.put(DotExpanderProcessor.TYPE, new DotExpanderProcessor.Factory());
+        processors.put(JsonProcessor.TYPE, new JsonProcessor.Factory());
+        processors.put(KeyValueProcessor.TYPE, new KeyValueProcessor.Factory());
         return Collections.unmodifiableMap(processors);
     }
 
-    // Code for loading built-in grok patterns packaged with the jar file:
+    @Override
+    public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
+        return Arrays.asList(new ActionHandler<>(GrokProcessorGetAction.INSTANCE, GrokProcessorGetAction.TransportAction.class));
+    }
+
+    @Override
+    public List<RestHandler> getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings,
+                                             IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter,
+                                             IndexNameExpressionResolver indexNameExpressionResolver,
+                                             Supplier<DiscoveryNodes> nodesInCluster) {
+        return Arrays.asList(new GrokProcessorGetAction.RestAction(settings, restController));
+    }
 
-    private static final String[] PATTERN_NAMES = new String[] {
-            "aws", "bacula", "bro", "exim", "firewalls", "grok-patterns", "haproxy",
-            "java", "junos", "linux-syslog", "mcollective-patterns", "mongodb", "nagios",
-            "postgresql", "rails", "redis", "ruby"
-    };
 
     public static Map<String, String> loadBuiltinPatterns() throws IOException {
         Map<String, String> builtinPatterns = new HashMap<>();
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java
index 6b4327f726d19..57216a71e022c 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java
@@ -38,11 +38,13 @@ public final class JoinProcessor extends AbstractProcessor {
 
     private final String field;
     private final String separator;
+    private final String targetField;
 
-    JoinProcessor(String tag, String field, String separator) {
+    JoinProcessor(String tag, String field, String separator, String targetField) {
         super(tag);
         this.field = field;
         this.separator = separator;
+        this.targetField = targetField;
     }
 
     String getField() {
@@ -53,6 +55,10 @@ String getSeparator() {
         return separator;
     }
 
+    String getTargetField() {
+        return targetField;
+    }
+
     @Override
     public void execute(IngestDocument document) {
         List<?> list = document.getFieldValue(field, List.class);
@@ -62,7 +68,7 @@ public void execute(IngestDocument document) {
         String joined = list.stream()
                 .map(Object::toString)
                 .collect(Collectors.joining(separator));
-        document.setFieldValue(field, joined);
+        document.setFieldValue(targetField, joined);
     }
 
     @Override
@@ -76,7 +82,8 @@ public JoinProcessor create(Map<String, Processor.Factory> registry, String proc
                                     Map<String, Object> config) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
             String separator = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "separator");
-            return new JoinProcessor(processorTag, field, separator);
+            String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", field);
+            return new JoinProcessor(processorTag, field, separator, targetField);
         }
     }
 }
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java
new file mode 100644
index 0000000000000..d4c7efe0d5092
--- /dev/null
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest.common;
+
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.ingest.AbstractProcessor;
+import org.elasticsearch.ingest.ConfigurationUtils;
+import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
+
+import java.util.Map;
+
+import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException;
+
+/**
+ * Processor that serializes a string-valued field into a
+ * map of maps.
+ */
+public final class JsonProcessor extends AbstractProcessor {
+
+    public static final String TYPE = "json";
+
+    private final String field;
+    private final String targetField;
+    private final boolean addToRoot;
+
+    JsonProcessor(String tag, String field, String targetField, boolean addToRoot) {
+        super(tag);
+        this.field = field;
+        this.targetField = targetField;
+        this.addToRoot = addToRoot;
+    }
+
+    public String getField() {
+        return field;
+    }
+
+    public String getTargetField() {
+        return targetField;
+    }
+
+    boolean isAddToRoot() {
+        return addToRoot;
+    }
+
+    @Override
+    public void execute(IngestDocument document) throws Exception {
+        String stringValue = document.getFieldValue(field, String.class);
+        try {
+            Map<String, Object> mapValue = XContentHelper.convertToMap(JsonXContent.jsonXContent, stringValue, false);
+            if (addToRoot) {
+                for (Map.Entry<String, Object> entry : mapValue.entrySet()) {
+                    document.setFieldValue(entry.getKey(), entry.getValue());
+                }
+            } else {
+                document.setFieldValue(targetField, mapValue);
+            }
+        } catch (ElasticsearchParseException e) {
+            throw new IllegalArgumentException(e);
+        }
+    }
+
+    @Override
+    public String getType() {
+        return TYPE;
+    }
+
+    public static final class Factory implements Processor.Factory {
+        @Override
+        public JsonProcessor create(Map<String, Processor.Factory> registry, String processorTag,
+                                    Map<String, Object> config) throws Exception {
+            String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
+            String targetField = ConfigurationUtils.readOptionalStringProperty(TYPE, processorTag, config, "target_field");
+            boolean addToRoot = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "add_to_root", false);
+
+            if (addToRoot && targetField != null) {
+                throw newConfigurationException(TYPE, processorTag, "target_field",
+                    "Cannot set a target field while also setting `add_to_root` to true");
+            }
+
+            if (targetField == null) {
+                targetField = field;
+            }
+
+            return new JsonProcessor(processorTag, field, targetField, addToRoot);
+        }
+    }
+}
+
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/KeyValueProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/KeyValueProcessor.java
new file mode 100644
index 0000000000000..fb856e28c31a0
--- /dev/null
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/KeyValueProcessor.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest.common;
+
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.ingest.AbstractProcessor;
+import org.elasticsearch.ingest.ConfigurationUtils;
+import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * The KeyValueProcessor parses and extracts messages of the `key=value` variety into fields with values of the keys.
+ */
+public final class KeyValueProcessor extends AbstractProcessor {
+
+    public static final String TYPE = "kv";
+
+    private final String field;
+    private final String fieldSplit;
+    private final String valueSplit;
+    private final Set<String> includeKeys;
+    private final Set<String> excludeKeys;
+    private final String targetField;
+    private final boolean ignoreMissing;
+
+    KeyValueProcessor(String tag, String field, String fieldSplit, String valueSplit, Set<String> includeKeys,
+                      Set<String> excludeKeys, String targetField, boolean ignoreMissing) {
+        super(tag);
+        this.field = field;
+        this.targetField = targetField;
+        this.fieldSplit = fieldSplit;
+        this.valueSplit = valueSplit;
+        this.includeKeys = includeKeys;
+        this.excludeKeys = excludeKeys;
+        this.ignoreMissing = ignoreMissing;
+    }
+
+    String getField() {
+        return field;
+    }
+
+    String getFieldSplit() {
+        return fieldSplit;
+    }
+
+    String getValueSplit() {
+        return valueSplit;
+    }
+
+    Set<String> getIncludeKeys() {
+        return includeKeys;
+    }
+
+    Set<String> getExcludeKeys() {
+        return excludeKeys;
+    }
+
+    String getTargetField() {
+        return targetField;
+    }
+
+    boolean isIgnoreMissing() {
+        return ignoreMissing;
+    }
+
+    public void append(IngestDocument document, String targetField, String value) {
+        if (document.hasField(targetField)) {
+            document.appendFieldValue(targetField, value);
+        } else {
+            document.setFieldValue(targetField, value);
+        }
+    }
+
+    @Override
+    public void execute(IngestDocument document) {
+        String oldVal = document.getFieldValue(field, String.class, ignoreMissing);
+
+        if (oldVal == null && ignoreMissing) {
+            return;
+        } else if (oldVal == null) {
+            throw new IllegalArgumentException("field [" + field + "] is null, cannot extract key-value pairs.");
+        }
+
+        String fieldPathPrefix = (targetField == null) ? "" : targetField + ".";
+        Arrays.stream(oldVal.split(fieldSplit))
+            .map((f) -> {
+                String[] kv = f.split(valueSplit, 2);
+                if (kv.length != 2) {
+                    throw new IllegalArgumentException("field [" + field + "] does not contain value_split [" + valueSplit + "]");
+                }
+                return kv;
+            })
+            .filter((p) ->
+                (includeKeys == null || includeKeys.contains(p[0])) &&
+                    (excludeKeys == null || excludeKeys.contains(p[0]) == false))
+            .forEach((p) -> append(document, fieldPathPrefix + p[0], p[1]));
+    }
+
+    @Override
+    public String getType() {
+        return TYPE;
+    }
+
+    public static class Factory implements Processor.Factory {
+        @Override
+        public KeyValueProcessor create(Map<String, Processor.Factory> registry, String processorTag,
+                                        Map<String, Object> config) throws Exception {
+            String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
+            String targetField = ConfigurationUtils.readOptionalStringProperty(TYPE, processorTag, config, "target_field");
+            String fieldSplit = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field_split");
+            String valueSplit = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "value_split");
+            Set<String> includeKeys = null;
+            Set<String> excludeKeys = null;
+            List<String> includeKeysList = ConfigurationUtils.readOptionalList(TYPE, processorTag, config, "include_keys");
+            if (includeKeysList != null) {
+                includeKeys = Collections.unmodifiableSet(Sets.newHashSet(includeKeysList));
+            }
+            List<String> excludeKeysList = ConfigurationUtils.readOptionalList(TYPE, processorTag, config, "exclude_keys");
+            if (excludeKeysList != null) {
+                excludeKeys = Collections.unmodifiableSet(Sets.newHashSet(excludeKeysList));
+            }
+            boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
+            return new KeyValueProcessor(processorTag, field, fieldSplit, valueSplit, includeKeys, excludeKeys, targetField, ignoreMissing);
+        }
+    }
+}
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java
index a0ae8e13158c3..aef8b0cce24f6 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.ingest.common;
 
 import java.util.Locale;
+import java.util.Map;
 
 /**
  * Processor that converts the content of string fields to lowercase.
@@ -30,8 +31,8 @@ public final class LowercaseProcessor extends AbstractStringProcessor {
 
     public static final String TYPE = "lowercase";
 
-    LowercaseProcessor(String processorTag, String field) {
-        super(processorTag, field);
+    LowercaseProcessor(String processorTag, String field, boolean ignoreMissing, String targetField) {
+        super(processorTag, field, ignoreMissing, targetField);
     }
 
     @Override
@@ -51,8 +52,9 @@ public Factory() {
         }
 
         @Override
-        protected LowercaseProcessor newProcessor(String tag, String field) {
-            return new LowercaseProcessor(tag, field);
+        protected LowercaseProcessor newProcessor(String tag, Map<String, Object> config, String field,
+                                                  boolean ignoreMissing, String targetField) {
+            return new LowercaseProcessor(tag, field, ignoreMissing, targetField);
         }
     }
 }
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java
index b381eed723d02..061412e975ba0 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java
@@ -25,7 +25,10 @@
 import org.elasticsearch.ingest.Processor;
 import org.elasticsearch.ingest.TemplateService;
 
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 /**
  * Processor that removes existing fields. Nothing happens if the field is not present.
@@ -34,20 +37,20 @@ public final class RemoveProcessor extends AbstractProcessor {
 
     public static final String TYPE = "remove";
 
-    private final TemplateService.Template field;
+    private final List<TemplateService.Template> fields;
 
-    RemoveProcessor(String tag, TemplateService.Template field) {
+    RemoveProcessor(String tag, List<TemplateService.Template> fields) {
         super(tag);
-        this.field = field;
+        this.fields = new ArrayList<>(fields);
     }
 
-    public TemplateService.Template getField() {
-        return field;
+    public List<TemplateService.Template> getFields() {
+        return fields;
     }
 
     @Override
     public void execute(IngestDocument document) {
-        document.removeField(field);
+       fields.forEach(document::removeField);
     }
 
     @Override
@@ -66,10 +69,18 @@ public Factory(TemplateService templateService) {
         @Override
         public RemoveProcessor create(Map<String, Processor.Factory> registry, String processorTag,
                                       Map<String, Object> config) throws Exception {
-            String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
-            TemplateService.Template compiledTemplate = ConfigurationUtils.compileTemplate(TYPE, processorTag,
-                "field", field, templateService);
-            return new RemoveProcessor(processorTag, compiledTemplate);
+            final List<String> fields = new ArrayList<>();
+            final Object field = ConfigurationUtils.readObject(TYPE, processorTag, config, "field");
+            if (field instanceof List) {
+                fields.addAll((List) field);
+            } else {
+                fields.add((String) field);
+            }
+
+            final List<TemplateService.Template> compiledTemplates = fields.stream()
+                .map(f -> ConfigurationUtils.compileTemplate(TYPE, processorTag, "field", f, templateService))
+                .collect(Collectors.toList());
+            return new RemoveProcessor(processorTag, compiledTemplates);
         }
     }
 }
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RenameProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RenameProcessor.java
index 362453a26b394..90a6389d82e46 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RenameProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RenameProcessor.java
@@ -35,11 +35,13 @@ public final class RenameProcessor extends AbstractProcessor {
 
     private final String field;
     private final String targetField;
+    private final boolean ignoreMissing;
 
-    RenameProcessor(String tag, String field, String targetField) {
+    RenameProcessor(String tag, String field, String targetField, boolean ignoreMissing) {
         super(tag);
         this.field = field;
         this.targetField = targetField;
+        this.ignoreMissing = ignoreMissing;
     }
 
     String getField() {
@@ -50,10 +52,18 @@ String getTargetField() {
         return targetField;
     }
 
+    boolean isIgnoreMissing() {
+        return ignoreMissing;
+    }
+
     @Override
     public void execute(IngestDocument document) {
         if (document.hasField(field, true) == false) {
-            throw new IllegalArgumentException("field [" + field + "] doesn't exist");
+            if (ignoreMissing) {
+                return;
+            } else {
+                throw new IllegalArgumentException("field [" + field + "] doesn't exist");
+            }
         }
         // We fail here if the target field point to an array slot that is out of range.
         // If we didn't do this then we would fail if we set the value in the target_field
@@ -85,7 +95,8 @@ public RenameProcessor create(Map<String, Processor.Factory> registry, String pr
                                       Map<String, Object> config) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
             String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field");
-            return new RenameProcessor(processorTag, field, targetField);
+            boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
+            return new RenameProcessor(processorTag, field, targetField, ignoreMissing);
         }
     }
 }
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java
index 94f335cc12f80..b644bbcb939a4 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java
@@ -19,30 +19,33 @@
 
 package org.elasticsearch.ingest.common;
 
-import java.util.Map;
-
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.ingest.Processor;
+import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptContext;
+import org.elasticsearch.script.ScriptException;
 import org.elasticsearch.script.ScriptService;
 
+import java.util.Map;
+
 import static java.util.Collections.emptyMap;
 import static org.elasticsearch.common.Strings.hasLength;
 import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException;
 import static org.elasticsearch.ingest.ConfigurationUtils.readOptionalMap;
 import static org.elasticsearch.ingest.ConfigurationUtils.readOptionalStringProperty;
-import static org.elasticsearch.ingest.ConfigurationUtils.readStringProperty;
-import static org.elasticsearch.script.ScriptService.ScriptType.FILE;
-import static org.elasticsearch.script.ScriptService.ScriptType.INLINE;
-import static org.elasticsearch.script.ScriptService.ScriptType.STORED;
+import static org.elasticsearch.script.ScriptType.FILE;
+import static org.elasticsearch.script.ScriptType.INLINE;
+import static org.elasticsearch.script.ScriptType.STORED;
 
 /**
- * Processor that adds new fields with their corresponding values. If the field is already present, its value
- * will be replaced with the provided one.
+ * Processor that evaluates a script with an ingest document in its context.
  */
 public final class ScriptProcessor extends AbstractProcessor {
 
@@ -51,15 +54,28 @@ public final class ScriptProcessor extends AbstractProcessor {
     private final Script script;
     private final ScriptService scriptService;
 
+    /**
+     * Processor that evaluates a script with an ingest document in its context
+     *
+     * @param tag The processor's tag.
+     * @param script The {@link Script} to execute.
+     * @param scriptService The {@link ScriptService} used to execute the script.
+     */
     ScriptProcessor(String tag, Script script, ScriptService scriptService)  {
         super(tag);
         this.script = script;
         this.scriptService = scriptService;
     }
 
+    /**
+     * Executes the script with the Ingest document in context.
+     *
+     * @param document The Ingest document passed into the script context under the "ctx" object.
+     */
     @Override
     public void execute(IngestDocument document) {
-        ExecutableScript executableScript = scriptService.executable(script, ScriptContext.Standard.INGEST, emptyMap());
+        CompiledScript compiledScript = scriptService.compile(script, ScriptContext.Standard.INGEST);
+        ExecutableScript executableScript = scriptService.executable(compiledScript, script.getParams());
         executableScript.setNextVar("ctx",  document.getSourceAndMetadata());
         executableScript.run();
     }
@@ -69,7 +85,13 @@ public String getType() {
         return TYPE;
     }
 
+    Script getScript() {
+        return script;
+    }
+
     public static final class Factory implements Processor.Factory {
+        private final Logger logger = ESLoggerFactory.getLogger(Factory.class);
+        private final DeprecationLogger deprecationLogger = new DeprecationLogger(logger);
 
         private final ScriptService scriptService;
 
@@ -78,40 +100,63 @@ public Factory(ScriptService scriptService) {
         }
 
         @Override
+        @SuppressWarnings("unchecked")
         public ScriptProcessor create(Map<String, Processor.Factory> registry, String processorTag,
                                       Map<String, Object> config) throws Exception {
-            String lang = readStringProperty(TYPE, processorTag, config, "lang");
-            String inline = readOptionalStringProperty(TYPE, processorTag, config, "inline");
+            String lang = readOptionalStringProperty(TYPE, processorTag, config, "lang");
+            String source = readOptionalStringProperty(TYPE, processorTag, config, "source");
             String file = readOptionalStringProperty(TYPE, processorTag, config, "file");
             String id = readOptionalStringProperty(TYPE, processorTag, config, "id");
             Map<String, ?> params = readOptionalMap(TYPE, processorTag, config, "params");
 
-            boolean containsNoScript = !hasLength(file) && !hasLength(id) && !hasLength(inline);
+            if (source == null) {
+                source = readOptionalStringProperty(TYPE, processorTag, config, "inline");
+                if (source != null) {
+                    deprecationLogger.deprecated("Specifying script source with [inline] is deprecated, use [source] instead.");
+                }
+            }
+
+            boolean containsNoScript = !hasLength(file) && !hasLength(id) && !hasLength(source);
             if (containsNoScript) {
-                throw newConfigurationException(TYPE, processorTag, null, "Need [file], [id], or [inline] parameter to refer to scripts");
+                throw newConfigurationException(TYPE, processorTag, null, "Need [file], [id], or [source] parameter to refer to scripts");
             }
 
             boolean moreThanOneConfigured = (Strings.hasLength(file) && Strings.hasLength(id)) ||
-                (Strings.hasLength(file) && Strings.hasLength(inline)) || (Strings.hasLength(id) && Strings.hasLength(inline));
+                (Strings.hasLength(file) && Strings.hasLength(source)) || (Strings.hasLength(id) && Strings.hasLength(source));
             if (moreThanOneConfigured) {
-                throw newConfigurationException(TYPE, processorTag, null, "Only one of [file], [id], or [inline] may be configured");
+                throw newConfigurationException(TYPE, processorTag, null, "Only one of [file], [id], or [source] may be configured");
+            }
+
+            if (lang == null) {
+                lang = Script.DEFAULT_SCRIPT_LANG;
             }
 
-            if(params == null) {
+            if (params == null) {
                 params = emptyMap();
             }
 
             final Script script;
+            String scriptPropertyUsed;
             if (Strings.hasLength(file)) {
-                script = new Script(file, FILE, lang, params);
-            } else if (Strings.hasLength(inline)) {
-                script = new Script(inline, INLINE, lang, params);
+                script = new Script(FILE, lang, file, (Map<String, Object>)params);
+                scriptPropertyUsed = "file";
+            } else if (Strings.hasLength(source)) {
+                script = new Script(INLINE, lang, source, (Map<String, Object>)params);
+                scriptPropertyUsed = "source";
             } else if (Strings.hasLength(id)) {
-                script = new Script(id, STORED, lang, params);
+                script = new Script(STORED, lang, id, (Map<String, Object>)params);
+                scriptPropertyUsed = "id";
             } else {
                 throw newConfigurationException(TYPE, processorTag, null, "Could not initialize script");
             }
 
+            // verify script is able to be compiled before successfully creating processor.
+            try {
+                scriptService.compile(script, ScriptContext.Standard.INGEST);
+            } catch (ScriptException e) {
+                throw newConfigurationException(TYPE, processorTag, scriptPropertyUsed, e);
+            }
+
             return new ScriptProcessor(processorTag, script, scriptService);
         }
     }
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java
index 411b22adef063..28e568233ebf5 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.ingest.Processor;
 
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
@@ -69,11 +70,13 @@ public static SortOrder fromString(String value) {
 
     private final String field;
     private final SortOrder order;
+    private final String targetField;
 
-    SortProcessor(String tag, String field, SortOrder order) {
+    SortProcessor(String tag, String field, SortOrder order, String targetField) {
         super(tag);
         this.field = field;
         this.order = order;
+        this.targetField = targetField;
     }
 
     String getField() {
@@ -84,6 +87,10 @@ SortOrder getOrder() {
         return order;
     }
 
+    String getTargetField() {
+        return targetField;
+    }
+
     @Override
     @SuppressWarnings("unchecked")
     public void execute(IngestDocument document) {
@@ -93,17 +100,15 @@ public void execute(IngestDocument document) {
             throw new IllegalArgumentException("field [" + field + "] is null, cannot sort.");
         }
 
-        if (list.size() <= 1) {
-            return;
-        }
+        List<? extends Comparable> copy = new ArrayList<>(list);
 
         if (order.equals(SortOrder.ASCENDING)) {
-            Collections.sort(list);
+            Collections.sort(copy);
         } else {
-            Collections.sort(list, Collections.reverseOrder());
+            Collections.sort(copy, Collections.reverseOrder());
         }
 
-        document.setFieldValue(field, list);
+        document.setFieldValue(targetField, copy);
     }
 
     @Override
@@ -117,6 +122,7 @@ public static final class Factory implements Processor.Factory {
         public SortProcessor create(Map<String, Processor.Factory> registry, String processorTag,
                                     Map<String, Object> config) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, FIELD);
+            String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", field);
             try {
                 SortOrder direction = SortOrder.fromString(
                     ConfigurationUtils.readStringProperty(
@@ -125,7 +131,7 @@ public SortProcessor create(Map<String, Processor.Factory> registry, String proc
                         config,
                         ORDER,
                         DEFAULT_ORDER));
-                return new SortProcessor(processorTag, field, direction);
+                return new SortProcessor(processorTag, field, direction, targetField);
             } catch (IllegalArgumentException e) {
                 throw ConfigurationUtils.newConfigurationException(TYPE, processorTag, ORDER, e.getMessage());
             }
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SplitProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SplitProcessor.java
index 8ff841ef0e4c0..cdd90f937fd09 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SplitProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SplitProcessor.java
@@ -32,7 +32,7 @@
 /**
  * Processor that splits fields content into different items based on the occurrence of a specified separator.
  * New field value will be an array containing all of the different extracted items.
- * Throws exception if the field is null or a type other than string.
+ * Support fields of string type only, throws exception if a field is of a different type.
  */
 public final class SplitProcessor extends AbstractProcessor {
 
@@ -40,11 +40,15 @@ public final class SplitProcessor extends AbstractProcessor {
 
     private final String field;
     private final String separator;
+    private final boolean ignoreMissing;
+    private final String targetField;
 
-    SplitProcessor(String tag, String field, String separator) {
+    SplitProcessor(String tag, String field, String separator, boolean ignoreMissing, String targetField) {
         super(tag);
         this.field = field;
         this.separator = separator;
+        this.ignoreMissing = ignoreMissing;
+        this.targetField = targetField;
     }
 
     String getField() {
@@ -55,16 +59,28 @@ String getSeparator() {
         return separator;
     }
 
+    boolean isIgnoreMissing() {
+        return ignoreMissing;
+    }
+
+    String getTargetField() {
+        return targetField;
+    }
+
     @Override
     public void execute(IngestDocument document) {
-        String oldVal = document.getFieldValue(field, String.class);
-        if (oldVal == null) {
+        String oldVal = document.getFieldValue(field, String.class, ignoreMissing);
+
+        if (oldVal == null && ignoreMissing) {
+            return;
+        } else if (oldVal == null) {
             throw new IllegalArgumentException("field [" + field + "] is null, cannot split.");
         }
+
         String[] strings = oldVal.split(separator);
         List<String> splitList = new ArrayList<>(strings.length);
         Collections.addAll(splitList, strings);
-        document.setFieldValue(field, splitList);
+        document.setFieldValue(targetField, splitList);
     }
 
     @Override
@@ -77,7 +93,10 @@ public static class Factory implements Processor.Factory {
         public SplitProcessor create(Map<String, Processor.Factory> registry, String processorTag,
                                      Map<String, Object> config) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
-            return new SplitProcessor(processorTag, field, ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "separator"));
+            boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
+            String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", field);
+            return new SplitProcessor(processorTag, field,
+                ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "separator"), ignoreMissing, targetField);
         }
     }
 }
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TrimProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TrimProcessor.java
index e852f887da026..98fe1223e5391 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TrimProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TrimProcessor.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.ingest.common;
 
+import java.util.Map;
+
 /**
  * Processor that trims the content of string fields.
  * Throws exception is the field is not of type string.
@@ -27,8 +29,8 @@ public final class TrimProcessor extends AbstractStringProcessor {
 
     public static final String TYPE = "trim";
 
-    TrimProcessor(String processorTag, String field) {
-        super(processorTag, field);
+    TrimProcessor(String processorTag, String field, boolean ignoreMissing, String targetField) {
+        super(processorTag, field, ignoreMissing, targetField);
     }
 
     @Override
@@ -48,8 +50,9 @@ public Factory() {
         }
 
         @Override
-        protected TrimProcessor newProcessor(String tag, String field) {
-            return new TrimProcessor(tag, field);
+        protected TrimProcessor newProcessor(String tag, Map<String, Object> config, String field,
+                                             boolean ignoreMissing, String targetField) {
+            return new TrimProcessor(tag, field, ignoreMissing, targetField);
         }
     }
 }
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UppercaseProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UppercaseProcessor.java
index 5585a130eaf6c..af93f06a8f271 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UppercaseProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UppercaseProcessor.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.ingest.common;
 
 import java.util.Locale;
+import java.util.Map;
 
 /**
  * Processor that converts the content of string fields to uppercase.
@@ -29,8 +30,8 @@ public final class UppercaseProcessor extends AbstractStringProcessor {
 
     public static final String TYPE = "uppercase";
 
-    UppercaseProcessor(String processorTag, String field) {
-        super(processorTag, field);
+    UppercaseProcessor(String processorTag, String field, boolean ignoreMissing, String targetField) {
+        super(processorTag, field, ignoreMissing, targetField);
     }
 
     @Override
@@ -50,9 +51,9 @@ public Factory() {
         }
 
         @Override
-        protected UppercaseProcessor newProcessor(String tag, String field) {
-            return new UppercaseProcessor(tag, field);
+        protected UppercaseProcessor newProcessor(String tag, Map<String, Object> config, String field,
+                                                  boolean ignoreMissing, String targetField) {
+            return new UppercaseProcessor(tag, field, ignoreMissing, targetField);
         }
     }
 }
-
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AbstractStringProcessorFactoryTestCase.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AbstractStringProcessorFactoryTestCase.java
new file mode 100644
index 0000000000000..0465e24902842
--- /dev/null
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AbstractStringProcessorFactoryTestCase.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest.common;
+
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.ingest.RandomDocumentPicks;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.is;
+
+public abstract class AbstractStringProcessorFactoryTestCase extends ESTestCase {
+
+    protected abstract AbstractStringProcessor.Factory newFactory();
+
+    protected Map<String, Object> modifyConfig(Map<String, Object> config) {
+        return config;
+    }
+
+    protected void assertProcessor(AbstractStringProcessor processor) {}
+
+    public void testCreate() throws Exception {
+        AbstractStringProcessor.Factory factory = newFactory();
+        String fieldName = RandomDocumentPicks.randomFieldName(random());
+        String processorTag = randomAlphaOfLength(10);
+
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", fieldName);
+
+        AbstractStringProcessor processor = factory.create(null, processorTag, modifyConfig(config));
+        assertThat(processor.getTag(), equalTo(processorTag));
+        assertThat(processor.getField(), equalTo(fieldName));
+        assertThat(processor.isIgnoreMissing(), is(false));
+        assertThat(processor.getTargetField(), equalTo(fieldName));
+        assertProcessor(processor);
+    }
+
+    public void testCreateWithIgnoreMissing() throws Exception {
+        AbstractStringProcessor.Factory factory = newFactory();
+        String fieldName = RandomDocumentPicks.randomFieldName(random());
+        String processorTag = randomAlphaOfLength(10);
+
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", fieldName);
+        config.put("ignore_missing", true);
+
+        AbstractStringProcessor processor = factory.create(null, processorTag, modifyConfig(config));
+        assertThat(processor.getTag(), equalTo(processorTag));
+        assertThat(processor.getField(), equalTo(fieldName));
+        assertThat(processor.isIgnoreMissing(), is(true));
+        assertThat(processor.getTargetField(), equalTo(fieldName));
+        assertProcessor(processor);
+    }
+
+    public void testCreateWithTargetField() throws Exception {
+        AbstractStringProcessor.Factory factory = newFactory();
+        String fieldName = RandomDocumentPicks.randomFieldName(random());
+        String targetFieldName = RandomDocumentPicks.randomFieldName(random());
+        String processorTag = randomAlphaOfLength(10);
+
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", fieldName);
+        config.put("target_field", targetFieldName);
+
+        AbstractStringProcessor processor = factory.create(null, processorTag, modifyConfig(config));
+        assertThat(processor.getTag(), equalTo(processorTag));
+        assertThat(processor.getField(), equalTo(fieldName));
+        assertThat(processor.isIgnoreMissing(), is(false));
+        assertThat(processor.getTargetField(), equalTo(targetFieldName));
+        assertProcessor(processor);
+    }
+
+    public void testCreateMissingField() throws Exception {
+        AbstractStringProcessor.Factory factory = newFactory();
+        Map<String, Object> config = new HashMap<>();
+        try {
+            factory.create(null, null, config);
+            fail("factory create should have failed");
+        } catch(ElasticsearchParseException e) {
+            assertThat(e.getMessage(), equalTo("[field] required property is missing"));
+        }
+    }
+}
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AbstractStringProcessorTestCase.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AbstractStringProcessorTestCase.java
index e987510f25a0f..9d37f27bb33e5 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AbstractStringProcessorTestCase.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AbstractStringProcessorTestCase.java
@@ -27,12 +27,13 @@
 import java.util.Collections;
 import java.util.HashMap;
 
+import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 
 public abstract class AbstractStringProcessorTestCase extends ESTestCase {
 
-    protected abstract AbstractStringProcessor newProcessor(String field);
+    protected abstract AbstractStringProcessor newProcessor(String field, boolean ignoreMissing, String targetField);
 
     protected String modifyInput(String input) {
         return input;
@@ -44,45 +45,70 @@ public void testProcessor() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
         String fieldValue = RandomDocumentPicks.randomString(random());
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, modifyInput(fieldValue));
-        Processor processor = newProcessor(fieldName);
+        Processor processor = newProcessor(fieldName, randomBoolean(), fieldName);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue(fieldName, String.class), equalTo(expectedResult(fieldValue)));
     }
 
     public void testFieldNotFound() throws Exception {
         String fieldName = RandomDocumentPicks.randomFieldName(random());
-        Processor processor = newProcessor(fieldName);
+        Processor processor = newProcessor(fieldName, false, fieldName);
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
-        try {
-            processor.execute(ingestDocument);
-            fail("processor should have failed");
-        } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), containsString("not present as part of path [" + fieldName + "]"));
-        }
+        Exception e = expectThrows(Exception.class, () -> processor.execute(ingestDocument));
+        assertThat(e.getMessage(), containsString("not present as part of path [" + fieldName + "]"));
+    }
+
+    public void testFieldNotFoundWithIgnoreMissing() throws Exception {
+        String fieldName = RandomDocumentPicks.randomFieldName(random());
+        Processor processor = newProcessor(fieldName, true, fieldName);
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
+        IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
+        processor.execute(ingestDocument);
+        assertIngestDocument(originalIngestDocument, ingestDocument);
     }
 
     public void testNullValue() throws Exception {
-        Processor processor = newProcessor("field");
+        Processor processor = newProcessor("field", false, "field");
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", null));
-        try {
-            processor.execute(ingestDocument);
-            fail("processor should have failed");
-        } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), equalTo("field [field] is null, cannot process it."));
-        }
+        Exception e = expectThrows(Exception.class, () -> processor.execute(ingestDocument));
+        assertThat(e.getMessage(), equalTo("field [field] is null, cannot process it."));
+    }
+
+    public void testNullValueWithIgnoreMissing() throws Exception {
+        Processor processor = newProcessor("field", true, "field");
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", null));
+        IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
+        processor.execute(ingestDocument);
+        assertIngestDocument(originalIngestDocument, ingestDocument);
     }
 
     public void testNonStringValue() throws Exception {
         String fieldName = RandomDocumentPicks.randomFieldName(random());
-        Processor processor = newProcessor(fieldName);
+        Processor processor = newProcessor(fieldName, false, fieldName);
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
         ingestDocument.setFieldValue(fieldName, randomInt());
-        try {
-            processor.execute(ingestDocument);
-            fail("processor should have failed");
-        } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), equalTo("field [" + fieldName +
-                    "] of type [java.lang.Integer] cannot be cast to [java.lang.String]"));
-        }
+        Exception e = expectThrows(Exception.class, () -> processor.execute(ingestDocument));
+        assertThat(e.getMessage(), equalTo("field [" + fieldName +
+            "] of type [java.lang.Integer] cannot be cast to [java.lang.String]"));
+    }
+
+    public void testNonStringValueWithIgnoreMissing() throws Exception {
+        String fieldName = RandomDocumentPicks.randomFieldName(random());
+        Processor processor = newProcessor(fieldName, true, fieldName);
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
+        ingestDocument.setFieldValue(fieldName, randomInt());
+        Exception e = expectThrows(Exception.class, () -> processor.execute(ingestDocument));
+        assertThat(e.getMessage(), equalTo("field [" + fieldName +
+            "] of type [java.lang.Integer] cannot be cast to [java.lang.String]"));
+    }
+
+    public void testTargetField() throws Exception {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
+        String fieldValue = RandomDocumentPicks.randomString(random());
+        String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, modifyInput(fieldValue));
+        String targetFieldName = fieldName + "foo";
+        Processor processor = newProcessor(fieldName, randomBoolean(), targetFieldName);
+        processor.execute(ingestDocument);
+        assertThat(ingestDocument.getFieldValue(targetFieldName, String.class), equalTo(expectedResult(fieldValue)));
     }
 }
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorFactoryTests.java
index e70bc3434eeac..072aa6faf84a4 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorFactoryTests.java
@@ -51,7 +51,7 @@ public void testCreate() throws Exception {
             value = Arrays.asList("value1", "value2", "value3");
         }
         config.put("value", value);
-        String processorTag = randomAsciiOfLength(10);
+        String processorTag = randomAlphaOfLength(10);
         AppendProcessor appendProcessor = factory.create(null, processorTag, config);
         assertThat(appendProcessor.getTag(), equalTo(processorTag));
         assertThat(appendProcessor.getField().execute(Collections.emptyMap()), equalTo("field1"));
@@ -97,7 +97,7 @@ public void testInvalidMustacheTemplate() throws Exception {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "field1");
         config.put("value", "value1");
-        String processorTag = randomAsciiOfLength(10);
+        String processorTag = randomAlphaOfLength(10);
         ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, config));
         assertThat(exception.getMessage(), equalTo("java.lang.RuntimeException: could not compile script"));
         assertThat(exception.getHeader("processor_tag").get(0), equalTo(processorTag));
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorTests.java
index 16df08721d3d8..fbbfb8a8440b0 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorTests.java
@@ -130,13 +130,13 @@ public void testAppendMetadata() throws Exception {
         List<String> values = new ArrayList<>();
         Processor appendProcessor;
         if (randomBoolean()) {
-            String value = randomAsciiOfLengthBetween(1, 10);
+            String value = randomAlphaOfLengthBetween(1, 10);
             values.add(value);
             appendProcessor = createAppendProcessor(randomMetaData.getFieldName(), value);
         } else {
             int valuesSize = randomIntBetween(0, 10);
             for (int i = 0; i < valuesSize; i++) {
-                values.add(randomAsciiOfLengthBetween(1, 10));
+                values.add(randomAlphaOfLengthBetween(1, 10));
             }
             appendProcessor = createAppendProcessor(randomMetaData.getFieldName(), values);
         }
@@ -158,7 +158,7 @@ public void testAppendMetadata() throws Exception {
 
     private static Processor createAppendProcessor(String fieldName, Object fieldValue) {
         TemplateService templateService = TestTemplateService.instance();
-        return new AppendProcessor(randomAsciiOfLength(10), templateService.compile(fieldName), ValueSource.wrap(fieldValue,
+        return new AppendProcessor(randomAlphaOfLength(10), templateService.compile(fieldName), ValueSource.wrap(fieldValue,
                 templateService));
     }
 
@@ -186,7 +186,7 @@ Object randomValue() {
         }, STRING {
             @Override
             Object randomValue() {
-                return randomAsciiOfLengthBetween(1, 10);
+                return randomAlphaOfLengthBetween(1, 10);
             }
         }, MAP {
             @Override
@@ -194,7 +194,7 @@ Object randomValue() {
                 int numItems = randomIntBetween(1, 10);
                 Map<String, Object> map = new HashMap<>(numItems);
                 for (int i = 0; i < numItems; i++) {
-                    map.put(randomAsciiOfLengthBetween(1, 10), randomFrom(Scalar.values()).randomValue());
+                    map.put(randomAlphaOfLengthBetween(1, 10), randomFrom(Scalar.values()).randomValue());
                 }
                 return map;
             }
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorFactoryTests.java
index bc04053f907a1..9e4acd7b17f83 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorFactoryTests.java
@@ -27,6 +27,7 @@
 import java.util.Map;
 
 import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
 
 public class ConvertProcessorFactoryTests extends ESTestCase {
@@ -37,18 +38,19 @@ public void testCreate() throws Exception {
         ConvertProcessor.Type type = randomFrom(ConvertProcessor.Type.values());
         config.put("field", "field1");
         config.put("type", type.toString());
-        String processorTag = randomAsciiOfLength(10);
+        String processorTag = randomAlphaOfLength(10);
         ConvertProcessor convertProcessor = factory.create(null, processorTag, config);
         assertThat(convertProcessor.getTag(), equalTo(processorTag));
         assertThat(convertProcessor.getField(), equalTo("field1"));
         assertThat(convertProcessor.getTargetField(), equalTo("field1"));
         assertThat(convertProcessor.getConvertType(), equalTo(type));
+        assertThat(convertProcessor.isIgnoreMissing(), is(false));
     }
 
     public void testCreateUnsupportedType() throws Exception {
         ConvertProcessor.Factory factory = new ConvertProcessor.Factory();
         Map<String, Object> config = new HashMap<>();
-        String type = "type-" + randomAsciiOfLengthBetween(1, 10);
+        String type = "type-" + randomAlphaOfLengthBetween(1, 10);
         config.put("field", "field1");
         config.put("type", type);
         try {
@@ -65,7 +67,7 @@ public void testCreateUnsupportedType() throws Exception {
     public void testCreateNoFieldPresent() throws Exception {
         ConvertProcessor.Factory factory = new ConvertProcessor.Factory();
         Map<String, Object> config = new HashMap<>();
-        String type = "type-" + randomAsciiOfLengthBetween(1, 10);
+        String type = "type-" + randomAlphaOfLengthBetween(1, 10);
         config.put("type", type);
         try {
             factory.create(null, null, config);
@@ -94,11 +96,28 @@ public void testCreateWithExplicitTargetField() throws Exception {
         config.put("field", "field1");
         config.put("target_field", "field2");
         config.put("type", type.toString());
-        String processorTag = randomAsciiOfLength(10);
+        String processorTag = randomAlphaOfLength(10);
         ConvertProcessor convertProcessor = factory.create(null, processorTag, config);
         assertThat(convertProcessor.getTag(), equalTo(processorTag));
         assertThat(convertProcessor.getField(), equalTo("field1"));
         assertThat(convertProcessor.getTargetField(), equalTo("field2"));
         assertThat(convertProcessor.getConvertType(), equalTo(type));
+        assertThat(convertProcessor.isIgnoreMissing(), is(false));
+    }
+
+    public void testCreateWithIgnoreMissing() throws Exception {
+        ConvertProcessor.Factory factory = new ConvertProcessor.Factory();
+        Map<String, Object> config = new HashMap<>();
+        ConvertProcessor.Type type = randomFrom(ConvertProcessor.Type.values());
+        config.put("field", "field1");
+        config.put("type", type.toString());
+        config.put("ignore_missing", true);
+        String processorTag = randomAlphaOfLength(10);
+        ConvertProcessor convertProcessor = factory.create(null, processorTag, config);
+        assertThat(convertProcessor.getTag(), equalTo(processorTag));
+        assertThat(convertProcessor.getField(), equalTo("field1"));
+        assertThat(convertProcessor.getTargetField(), equalTo("field1"));
+        assertThat(convertProcessor.getConvertType(), equalTo(type));
+        assertThat(convertProcessor.isIgnoreMissing(), is(true));
     }
 }
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java
index 02f859ecfa523..7cab5fff45c6f 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java
@@ -31,6 +31,7 @@
 import java.util.Locale;
 import java.util.Map;
 
+import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
 import static org.elasticsearch.ingest.common.ConvertProcessor.Type;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -42,7 +43,7 @@ public void testConvertInt() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
         int randomInt = randomInt();
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, randomInt);
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.INTEGER);
+        Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.INTEGER, false);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue(fieldName, Integer.class), equalTo(randomInt));
     }
@@ -58,7 +59,7 @@ public void testConvertIntList() throws Exception {
             expectedList.add(randomInt);
         }
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.INTEGER);
+        Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.INTEGER, false);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(expectedList));
     }
@@ -66,10 +67,10 @@ public void testConvertIntList() throws Exception {
     public void testConvertIntError() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
         String fieldName = RandomDocumentPicks.randomFieldName(random());
-        String value = "string-" + randomAsciiOfLengthBetween(1, 10);
+        String value = "string-" + randomAlphaOfLengthBetween(1, 10);
         ingestDocument.setFieldValue(fieldName, value);
 
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.INTEGER);
+        Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.INTEGER, false);
         try {
             processor.execute(ingestDocument);
             fail("processor execute should have failed");
@@ -85,7 +86,7 @@ public void testConvertFloat() throws Exception {
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, randomFloat);
         expectedResult.put(fieldName, randomFloat);
 
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.FLOAT);
+        Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.FLOAT, false);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue(fieldName, Float.class), equalTo(randomFloat));
     }
@@ -101,7 +102,7 @@ public void testConvertFloatList() throws Exception {
             expectedList.add(randomFloat);
         }
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.FLOAT);
+        Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.FLOAT, false);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(expectedList));
     }
@@ -109,10 +110,10 @@ public void testConvertFloatList() throws Exception {
     public void testConvertFloatError() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
         String fieldName = RandomDocumentPicks.randomFieldName(random());
-        String value = "string-" + randomAsciiOfLengthBetween(1, 10);
+        String value = "string-" + randomAlphaOfLengthBetween(1, 10);
         ingestDocument.setFieldValue(fieldName, value);
 
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.FLOAT);
+        Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.FLOAT, false);
         try {
             processor.execute(ingestDocument);
             fail("processor execute should have failed");
@@ -130,7 +131,7 @@ public void testConvertBoolean() throws Exception {
         }
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, booleanString);
 
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.BOOLEAN);
+        Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.BOOLEAN, false);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue(fieldName, Boolean.class), equalTo(randomBoolean));
     }
@@ -150,7 +151,7 @@ public void testConvertBooleanList() throws Exception {
             expectedList.add(randomBoolean);
         }
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.BOOLEAN);
+        Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.BOOLEAN, false);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(expectedList));
     }
@@ -160,14 +161,14 @@ public void testConvertBooleanError() throws Exception {
         String fieldName = RandomDocumentPicks.randomFieldName(random());
         String fieldValue;
         if (randomBoolean()) {
-            fieldValue = "string-" + randomAsciiOfLengthBetween(1, 10);
+            fieldValue = "string-" + randomAlphaOfLengthBetween(1, 10);
         } else {
             //verify that only proper boolean values are supported and we are strict about it
             fieldValue = randomFrom("on", "off", "yes", "no", "0", "1");
         }
         ingestDocument.setFieldValue(fieldName, fieldValue);
 
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.BOOLEAN);
+        Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.BOOLEAN, false);
         try {
             processor.execute(ingestDocument);
             fail("processor execute should have failed");
@@ -201,7 +202,7 @@ public void testConvertString() throws Exception {
         }
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
 
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.STRING);
+        Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.STRING, false);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue(fieldName, String.class), equalTo(expectedFieldValue));
     }
@@ -237,7 +238,7 @@ public void testConvertStringList() throws Exception {
             expectedList.add(randomValueString);
         }
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.STRING);
+        Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.STRING, false);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(expectedList));
     }
@@ -246,7 +247,7 @@ public void testConvertNonExistingField() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
         String fieldName = RandomDocumentPicks.randomFieldName(random());
         Type type = randomFrom(Type.values());
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, type);
+        Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, type, false);
         try {
             processor.execute(ingestDocument);
             fail("processor execute should have failed");
@@ -258,7 +259,7 @@ public void testConvertNonExistingField() throws Exception {
     public void testConvertNullField() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", null));
         Type type = randomFrom(Type.values());
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", type);
+        Processor processor = new ConvertProcessor(randomAlphaOfLength(10), "field", "field", type, false);
         try {
             processor.execute(ingestDocument);
             fail("processor execute should have failed");
@@ -267,6 +268,25 @@ public void testConvertNullField() throws Exception {
         }
     }
 
+    public void testConvertNonExistingFieldWithIgnoreMissing() throws Exception {
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
+        IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
+        String fieldName = RandomDocumentPicks.randomFieldName(random());
+        Type type = randomFrom(Type.values());
+        Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, type, true);
+        processor.execute(ingestDocument);
+        assertIngestDocument(originalIngestDocument, ingestDocument);
+    }
+
+    public void testConvertNullFieldWithIgnoreMissing() throws Exception {
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", null));
+        IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
+        Type type = randomFrom(Type.values());
+        Processor processor = new ConvertProcessor(randomAlphaOfLength(10), "field", "field", type, true);
+        processor.execute(ingestDocument);
+        assertIngestDocument(originalIngestDocument, ingestDocument);
+    }
+
     public void testAutoConvertNotString() throws Exception {
         Object randomValue;
         switch(randomIntBetween(0, 2)) {
@@ -286,7 +306,7 @@ public void testAutoConvertNotString() throws Exception {
                 throw new UnsupportedOperationException();
         }
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", randomValue));
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", Type.AUTO);
+        Processor processor = new ConvertProcessor(randomAlphaOfLength(10), "field", "field", Type.AUTO, false);
         processor.execute(ingestDocument);
         Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
         assertThat(convertedValue, sameInstance(randomValue));
@@ -295,7 +315,7 @@ public void testAutoConvertNotString() throws Exception {
     public void testAutoConvertStringNotMatched() throws Exception {
         String value = "notAnIntFloatOrBool";
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", value));
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", Type.AUTO);
+        Processor processor = new ConvertProcessor(randomAlphaOfLength(10), "field", "field", Type.AUTO, false);
         processor.execute(ingestDocument);
         Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
         assertThat(convertedValue, sameInstance(value));
@@ -306,7 +326,7 @@ public void testAutoConvertMatchBoolean() throws Exception {
         String booleanString = Boolean.toString(randomBoolean);
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(),
             Collections.singletonMap("field", booleanString));
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", Type.AUTO);
+        Processor processor = new ConvertProcessor(randomAlphaOfLength(10), "field", "field", Type.AUTO, false);
         processor.execute(ingestDocument);
         Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
         assertThat(convertedValue, equalTo(randomBoolean));
@@ -316,7 +336,7 @@ public void testAutoConvertMatchInteger() throws Exception {
         int randomInt = randomInt();
         String randomString = Integer.toString(randomInt);
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", randomString));
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", Type.AUTO);
+        Processor processor = new ConvertProcessor(randomAlphaOfLength(10), "field", "field", Type.AUTO, false);
         processor.execute(ingestDocument);
         Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
         assertThat(convertedValue, equalTo(randomInt));
@@ -326,7 +346,7 @@ public void testAutoConvertMatchFloat() throws Exception {
         float randomFloat = randomFloat();
         String randomString = Float.toString(randomFloat);
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", randomString));
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", Type.AUTO);
+        Processor processor = new ConvertProcessor(randomAlphaOfLength(10), "field", "field", Type.AUTO, false);
         processor.execute(ingestDocument);
         Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
         assertThat(convertedValue, equalTo(randomFloat));
@@ -336,11 +356,10 @@ public void testTargetField() throws Exception {
         IngestDocument ingestDocument = new IngestDocument(new HashMap<>(), new HashMap<>());
         int randomInt = randomInt();
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, String.valueOf(randomInt));
-        String targetField = fieldName + randomAsciiOfLength(5);
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, targetField, Type.INTEGER);
+        String targetField = fieldName + randomAlphaOfLength(5);
+        Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, targetField, Type.INTEGER, false);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue(fieldName, String.class), equalTo(String.valueOf(randomInt)));
         assertThat(ingestDocument.getFieldValue(targetField, Integer.class), equalTo(randomInt));
-
     }
 }
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java
index 0885371f867bf..415ee8720930b 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java
@@ -50,6 +50,10 @@ public void testParseUnix() {
         assertThat(DateFormat.Unix.getFunction(null, DateTimeZone.UTC, null).apply("1000.5").getMillis(), equalTo(1000500L));
     }
 
+    public void testParseUnixWithMsPrecision() {
+        assertThat(DateFormat.Unix.getFunction(null, DateTimeZone.UTC, null).apply("1495718015").getMillis(), equalTo(1495718015000L));
+    }
+
     public void testParseISO8601() {
         assertThat(DateFormat.Iso8601.getFunction(null, DateTimeZone.UTC, null).apply("2001-01-01T00:00:00-0800").getMillis(),
                 equalTo(978336000000L));
@@ -81,6 +85,6 @@ public void testFromString() {
         assertThat(DateFormat.fromString("iso8601"), equalTo(DateFormat.Joda));
         assertThat(DateFormat.fromString("TAI64N"), equalTo(DateFormat.Tai64n));
         assertThat(DateFormat.fromString("tai64n"), equalTo(DateFormat.Joda));
-        assertThat(DateFormat.fromString("prefix-" + randomAsciiOfLengthBetween(1, 10)), equalTo(DateFormat.Joda));
+        assertThat(DateFormat.fromString("prefix-" + randomAlphaOfLengthBetween(1, 10)), equalTo(DateFormat.Joda));
     }
 }
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java
index c546f021823c1..d052ce0cd44c3 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java
@@ -62,6 +62,11 @@ public void testUnixMs()throws Exception {
                 Collections.singletonMap("_field", "1000500"));
         dateProcessor.execute(document);
         assertThat(document.getSourceAndMetadata().get("_index"), equalTo("<events-{19700101||/m{yyyyMMdd|UTC}}>"));
+
+        document = new IngestDocument("_index", "_type", "_id", null, null, null, null,
+                Collections.singletonMap("_field", 1000500L));
+        dateProcessor.execute(document);
+        assertThat(document.getSourceAndMetadata().get("_index"), equalTo("<events-{19700101||/m{yyyyMMdd|UTC}}>"));
     }
 
     public void testUnix()throws Exception {
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorFactoryTests.java
index 95ad68bb110a4..2dc16ad7bd7fd 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorFactoryTests.java
@@ -37,10 +37,10 @@ public class DateProcessorFactoryTests extends ESTestCase {
     public void testBuildDefaults() throws Exception {
         DateProcessor.Factory factory = new DateProcessor.Factory();
         Map<String, Object> config = new HashMap<>();
-        String sourceField = randomAsciiOfLengthBetween(1, 10);
+        String sourceField = randomAlphaOfLengthBetween(1, 10);
         config.put("field", sourceField);
         config.put("formats", Collections.singletonList("dd/MM/yyyyy"));
-        String processorTag = randomAsciiOfLength(10);
+        String processorTag = randomAlphaOfLength(10);
         DateProcessor processor = factory.create(null, processorTag, config);
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.getField(), equalTo(sourceField));
@@ -53,7 +53,7 @@ public void testBuildDefaults() throws Exception {
     public void testMatchFieldIsMandatory() throws Exception {
         DateProcessor.Factory factory = new DateProcessor.Factory();
         Map<String, Object> config = new HashMap<>();
-        String targetField = randomAsciiOfLengthBetween(1, 10);
+        String targetField = randomAlphaOfLengthBetween(1, 10);
         config.put("target_field", targetField);
         config.put("formats", Collections.singletonList("dd/MM/yyyyy"));
 
@@ -68,8 +68,8 @@ public void testMatchFieldIsMandatory() throws Exception {
     public void testMatchFormatsIsMandatory() throws Exception {
         DateProcessor.Factory factory = new DateProcessor.Factory();
         Map<String, Object> config = new HashMap<>();
-        String sourceField = randomAsciiOfLengthBetween(1, 10);
-        String targetField = randomAsciiOfLengthBetween(1, 10);
+        String sourceField = randomAlphaOfLengthBetween(1, 10);
+        String targetField = randomAlphaOfLengthBetween(1, 10);
         config.put("field", sourceField);
         config.put("target_field", targetField);
 
@@ -84,7 +84,7 @@ public void testMatchFormatsIsMandatory() throws Exception {
     public void testParseLocale() throws Exception {
         DateProcessor.Factory factory = new DateProcessor.Factory();
         Map<String, Object> config = new HashMap<>();
-        String sourceField = randomAsciiOfLengthBetween(1, 10);
+        String sourceField = randomAlphaOfLengthBetween(1, 10);
         config.put("field", sourceField);
         config.put("formats", Collections.singletonList("dd/MM/yyyyy"));
         Locale locale = randomLocale(random());
@@ -97,7 +97,7 @@ public void testParseLocale() throws Exception {
     public void testParseInvalidLocale() throws Exception {
         DateProcessor.Factory factory = new DateProcessor.Factory();
         Map<String, Object> config = new HashMap<>();
-        String sourceField = randomAsciiOfLengthBetween(1, 10);
+        String sourceField = randomAlphaOfLengthBetween(1, 10);
         config.put("field", sourceField);
         config.put("formats", Collections.singletonList("dd/MM/yyyyy"));
         config.put("locale", "invalid_locale");
@@ -112,7 +112,7 @@ public void testParseInvalidLocale() throws Exception {
     public void testParseTimezone() throws Exception {
         DateProcessor.Factory factory = new DateProcessor.Factory();
         Map<String, Object> config = new HashMap<>();
-        String sourceField = randomAsciiOfLengthBetween(1, 10);
+        String sourceField = randomAlphaOfLengthBetween(1, 10);
         config.put("field", sourceField);
         config.put("formats", Collections.singletonList("dd/MM/yyyyy"));
 
@@ -125,7 +125,7 @@ public void testParseTimezone() throws Exception {
     public void testParseInvalidTimezone() throws Exception {
         DateProcessor.Factory factory = new DateProcessor.Factory();
         Map<String, Object> config = new HashMap<>();
-        String sourceField = randomAsciiOfLengthBetween(1, 10);
+        String sourceField = randomAlphaOfLengthBetween(1, 10);
         config.put("field", sourceField);
         config.put("match_formats", Collections.singletonList("dd/MM/yyyyy"));
         config.put("timezone", "invalid_timezone");
@@ -140,7 +140,7 @@ public void testParseInvalidTimezone() throws Exception {
     public void testParseMatchFormats() throws Exception {
         DateProcessor.Factory factory = new DateProcessor.Factory();
         Map<String, Object> config = new HashMap<>();
-        String sourceField = randomAsciiOfLengthBetween(1, 10);
+        String sourceField = randomAlphaOfLengthBetween(1, 10);
         config.put("field", sourceField);
         config.put("formats", Arrays.asList("dd/MM/yyyy", "dd-MM-yyyy"));
 
@@ -151,7 +151,7 @@ public void testParseMatchFormats() throws Exception {
     public void testParseMatchFormatsFailure() throws Exception {
         DateProcessor.Factory factory = new DateProcessor.Factory();
         Map<String, Object> config = new HashMap<>();
-        String sourceField = randomAsciiOfLengthBetween(1, 10);
+        String sourceField = randomAlphaOfLengthBetween(1, 10);
         config.put("field", sourceField);
         config.put("formats", "dd/MM/yyyy");
 
@@ -166,8 +166,8 @@ public void testParseMatchFormatsFailure() throws Exception {
     public void testParseTargetField() throws Exception {
         DateProcessor.Factory factory = new DateProcessor.Factory();
         Map<String, Object> config = new HashMap<>();
-        String sourceField = randomAsciiOfLengthBetween(1, 10);
-        String targetField = randomAsciiOfLengthBetween(1, 10);
+        String sourceField = randomAlphaOfLengthBetween(1, 10);
+        String targetField = randomAlphaOfLengthBetween(1, 10);
         config.put("field", sourceField);
         config.put("target_field", targetField);
         config.put("formats", Arrays.asList("dd/MM/yyyy", "dd-MM-yyyy"));
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java
index 768f402ee4b96..d92f2e84be135 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java
@@ -38,7 +38,7 @@
 public class DateProcessorTests extends ESTestCase {
 
     public void testJodaPattern() {
-        DateProcessor dateProcessor = new DateProcessor(randomAsciiOfLength(10), DateTimeZone.forID("Europe/Amsterdam"), Locale.ENGLISH,
+        DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10), DateTimeZone.forID("Europe/Amsterdam"), Locale.ENGLISH,
                 "date_as_string", Collections.singletonList("yyyy dd MM hh:mm:ss"), "date_as_date");
         Map<String, Object> document = new HashMap<>();
         document.put("date_as_string", "2010 12 06 11:05:15");
@@ -52,7 +52,7 @@ public void testJodaPatternMultipleFormats() {
         matchFormats.add("yyyy dd MM");
         matchFormats.add("dd/MM/yyyy");
         matchFormats.add("dd-MM-yyyy");
-        DateProcessor dateProcessor = new DateProcessor(randomAsciiOfLength(10), DateTimeZone.forID("Europe/Amsterdam"), Locale.ENGLISH,
+        DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10), DateTimeZone.forID("Europe/Amsterdam"), Locale.ENGLISH,
                 "date_as_string", matchFormats, "date_as_date");
 
         Map<String, Object> document = new HashMap<>();
@@ -86,7 +86,7 @@ public void testJodaPatternMultipleFormats() {
 
     public void testInvalidJodaPattern() {
         try {
-            new DateProcessor(randomAsciiOfLength(10), DateTimeZone.UTC, randomLocale(random()),
+            new DateProcessor(randomAlphaOfLength(10), DateTimeZone.UTC, randomLocale(random()),
                 "date_as_string", Collections.singletonList("invalid pattern"), "date_as_date");
             fail("date processor initialization should have failed");
         } catch(IllegalArgumentException e) {
@@ -95,7 +95,7 @@ public void testInvalidJodaPattern() {
     }
 
     public void testJodaPatternLocale() {
-        DateProcessor dateProcessor = new DateProcessor(randomAsciiOfLength(10), DateTimeZone.forID("Europe/Amsterdam"), Locale.ITALIAN,
+        DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10), DateTimeZone.forID("Europe/Amsterdam"), Locale.ITALIAN,
                 "date_as_string", Collections.singletonList("yyyy dd MMM"), "date_as_date");
         Map<String, Object> document = new HashMap<>();
         document.put("date_as_string", "2010 12 giugno");
@@ -105,18 +105,18 @@ public void testJodaPatternLocale() {
     }
 
     public void testJodaPatternDefaultYear() {
-        DateProcessor dateProcessor = new DateProcessor(randomAsciiOfLength(10), DateTimeZone.forID("Europe/Amsterdam"), Locale.ENGLISH,
-                "date_as_string", Collections.singletonList("dd/MM"), "date_as_date");
+        DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10), DateTimeZone.forID("Europe/Amsterdam"), Locale.ENGLISH,
+            "date_as_string", Collections.singletonList("dd/MM"), "date_as_date");
         Map<String, Object> document = new HashMap<>();
         document.put("date_as_string", "12/06");
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
         dateProcessor.execute(ingestDocument);
-        assertThat(ingestDocument.getFieldValue("date_as_date", String.class), equalTo(DateTime.now().getYear() +
-                "-06-12T00:00:00.000+02:00"));
+        assertThat(ingestDocument.getFieldValue("date_as_date", String.class),
+            equalTo(DateTime.now().getYear() + "-06-12T00:00:00.000+02:00"));
     }
 
     public void testTAI64N() {
-        DateProcessor dateProcessor = new DateProcessor(randomAsciiOfLength(10), DateTimeZone.forOffsetHours(2), randomLocale(random()),
+        DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10), DateTimeZone.forOffsetHours(2), randomLocale(random()),
                 "date_as_string", Collections.singletonList("TAI64N"), "date_as_date");
         Map<String, Object> document = new HashMap<>();
         String dateAsString = (randomBoolean() ? "@" : "") + "4000000050d506482dbdf024";
@@ -127,7 +127,7 @@ public void testTAI64N() {
     }
 
     public void testUnixMs() {
-        DateProcessor dateProcessor = new DateProcessor(randomAsciiOfLength(10), DateTimeZone.UTC, randomLocale(random()),
+        DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10), DateTimeZone.UTC, randomLocale(random()),
                 "date_as_string", Collections.singletonList("UNIX_MS"), "date_as_date");
         Map<String, Object> document = new HashMap<>();
         document.put("date_as_string", "1000500");
@@ -137,7 +137,7 @@ public void testUnixMs() {
     }
 
     public void testUnix() {
-        DateProcessor dateProcessor = new DateProcessor(randomAsciiOfLength(10), DateTimeZone.UTC, randomLocale(random()),
+        DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10), DateTimeZone.UTC, randomLocale(random()),
                 "date_as_string", Collections.singletonList("UNIX"), "date_as_date");
         Map<String, Object> document = new HashMap<>();
         document.put("date_as_string", "1000.5");
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java
index 1802090e0e5c7..199c475a8f0bf 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java
@@ -86,7 +86,7 @@ public void testEscapeFields_valueField() throws Exception {
         // so because foo is no branch field but a value field the `foo.bar` field can't be expanded
         // into [foo].[bar], so foo should be renamed first into `[foo].[bar]:
         IngestDocument document = new IngestDocument(source, Collections.emptyMap());
-        Processor processor = new RenameProcessor("_tag", "foo", "foo.bar");
+        Processor processor = new RenameProcessor("_tag", "foo", "foo.bar", false);
         processor.execute(document);
         processor = new DotExpanderProcessor("_tag", null, "foo.bar");
         processor.execute(document);
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorFactoryTests.java
index 217a15cf5b395..ab0acad4aea1f 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorFactoryTests.java
@@ -43,7 +43,7 @@ public void init() {
     public void testCreate() throws Exception {
         Map<String, Object> config = new HashMap<>();
         config.put("message", "error");
-        String processorTag = randomAsciiOfLength(10);
+        String processorTag = randomAlphaOfLength(10);
         FailProcessor failProcessor = factory.create(null, processorTag, config);
         assertThat(failProcessor.getTag(), equalTo(processorTag));
         assertThat(failProcessor.getMessage().execute(Collections.emptyMap()), equalTo("error"));
@@ -63,7 +63,7 @@ public void testInvalidMustacheTemplate() throws Exception {
         FailProcessor.Factory factory = new FailProcessor.Factory(TestTemplateService.instance(true));
         Map<String, Object> config = new HashMap<>();
         config.put("message", "error");
-        String processorTag = randomAsciiOfLength(10);
+        String processorTag = randomAlphaOfLength(10);
         ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, config));
         assertThat(exception.getMessage(), equalTo("java.lang.RuntimeException: could not compile script"));
         assertThat(exception.getHeader("processor_tag").get(0), equalTo(processorTag));
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorTests.java
index 76685854d21eb..c05323c12e3a6 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorTests.java
@@ -31,8 +31,8 @@ public class FailProcessorTests extends ESTestCase {
 
     public void test() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
-        String message = randomAsciiOfLength(10);
-        Processor processor = new FailProcessor(randomAsciiOfLength(10), new TestTemplateService.MockTemplate(message));
+        String message = randomAlphaOfLength(10);
+        Processor processor = new FailProcessor(randomAlphaOfLength(10), new TestTemplateService.MockTemplate(message));
         try {
             processor.execute(ingestDocument);
             fail("fail processor should throw an exception");
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorTests.java
index 7a7b59bad269a..5d5a3d6a3890a 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorTests.java
@@ -49,7 +49,7 @@ public void testExecute() throws Exception {
         );
 
         ForEachProcessor processor = new ForEachProcessor(
-            "_tag", "values", new UppercaseProcessor("_tag", "_ingest._value")
+            "_tag", "values", new UppercaseProcessor("_tag", "_ingest._value", false, "_ingest._value")
         );
         processor.execute(ingestDocument);
 
@@ -197,9 +197,9 @@ public void testModifyFieldsOutsideArray() throws Exception {
 
         ForEachProcessor processor = new ForEachProcessor(
                 "_tag", "values", new CompoundProcessor(false,
-                Collections.singletonList(new UppercaseProcessor("_tag_upper", "_ingest._value")),
+                Collections.singletonList(new UppercaseProcessor("_tag_upper", "_ingest._value", false, "_ingest._value")),
                 Collections.singletonList(new AppendProcessor("_tag",
-                        ts.compile("errors"), (model) -> (Collections.singletonList("added"))))
+                    ts.compile("errors"), (model) -> (Collections.singletonList("added"))))
         ));
         processor.execute(ingestDocument);
 
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorFactoryTests.java
index 20537d2ced63a..4cac94cd5b571 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorFactoryTests.java
@@ -27,6 +27,7 @@
 import java.util.Map;
 
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
 
 public class GrokProcessorFactoryTests extends ESTestCase {
@@ -37,11 +38,27 @@ public void testBuild() throws Exception {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
         config.put("patterns", Collections.singletonList("(?<foo>\\w+)"));
-        String processorTag = randomAsciiOfLength(10);
+        String processorTag = randomAlphaOfLength(10);
         GrokProcessor processor = factory.create(null, processorTag, config);
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.getMatchField(), equalTo("_field"));
         assertThat(processor.getGrok(), notNullValue());
+        assertThat(processor.isIgnoreMissing(), is(false));
+    }
+
+    public void testBuildWithIgnoreMissing() throws Exception {
+        GrokProcessor.Factory factory = new GrokProcessor.Factory(Collections.emptyMap());
+
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", "_field");
+        config.put("patterns", Collections.singletonList("(?<foo>\\w+)"));
+        config.put("ignore_missing", true);
+        String processorTag = randomAlphaOfLength(10);
+        GrokProcessor processor = factory.create(null, processorTag, config);
+        assertThat(processor.getTag(), equalTo(processorTag));
+        assertThat(processor.getMatchField(), equalTo("_field"));
+        assertThat(processor.getGrok(), notNullValue());
+        assertThat(processor.isIgnoreMissing(), is(true));
     }
 
     public void testBuildMissingField() throws Exception {
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorGetActionTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorGetActionTests.java
new file mode 100644
index 0000000000000..aa54044f8454c
--- /dev/null
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorGetActionTests.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest.common;
+
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Collections;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.core.IsNull.nullValue;
+
+
+public class GrokProcessorGetActionTests extends ESTestCase {
+    private static final Map<String, String> TEST_PATTERNS = Collections.singletonMap("PATTERN", "foo");
+
+    public void testRequest() throws Exception {
+        GrokProcessorGetAction.Request request = new GrokProcessorGetAction.Request();
+        BytesStreamOutput out = new BytesStreamOutput();
+        request.writeTo(out);
+        StreamInput streamInput = out.bytes().streamInput();
+        GrokProcessorGetAction.Request otherRequest = new GrokProcessorGetAction.Request();
+        otherRequest.readFrom(streamInput);
+        assertThat(otherRequest.validate(), nullValue());
+    }
+
+    public void testResponseSerialization() throws Exception {
+        GrokProcessorGetAction.Response response = new GrokProcessorGetAction.Response(TEST_PATTERNS);
+        BytesStreamOutput out = new BytesStreamOutput();
+        response.writeTo(out);
+        StreamInput streamInput = out.bytes().streamInput();
+        GrokProcessorGetAction.Response otherResponse = new GrokProcessorGetAction.Response(null);
+        otherResponse.readFrom(streamInput);
+        assertThat(response.getGrokPatterns(), equalTo(TEST_PATTERNS));
+        assertThat(response.getGrokPatterns(), equalTo(otherResponse.getGrokPatterns()));
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testResponseToXContent() throws Exception {
+        GrokProcessorGetAction.Response response = new GrokProcessorGetAction.Response(TEST_PATTERNS);
+        try (XContentBuilder builder = JsonXContent.contentBuilder()) {
+            response.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            Map<String, Object> converted = XContentHelper.convertToMap(builder.bytes(), false, builder.contentType()).v2();
+            Map<String, String> patterns = (Map<String, String>) converted.get("patterns");
+            assertThat(patterns.size(), equalTo(1));
+            assertThat(patterns.get("PATTERN"), equalTo("foo"));
+        }
+    }
+}
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorTests.java
index f108e80e790a2..37c26db4b74f3 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorTests.java
@@ -28,6 +28,7 @@
 import java.util.HashMap;
 import java.util.Map;
 
+import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
 import static org.hamcrest.Matchers.equalTo;
 
 
@@ -37,8 +38,8 @@ public void testMatch() throws Exception {
         String fieldName = RandomDocumentPicks.randomFieldName(random());
         IngestDocument doc = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
         doc.setFieldValue(fieldName, "1");
-        GrokProcessor processor = new GrokProcessor(randomAsciiOfLength(10), Collections.singletonMap("ONE", "1"),
-            Collections.singletonList("%{ONE:one}"), fieldName);
+        GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), Collections.singletonMap("ONE", "1"),
+            Collections.singletonList("%{ONE:one}"), fieldName, false, false);
         processor.execute(doc);
         assertThat(doc.getFieldValue("one", String.class), equalTo("1"));
     }
@@ -47,29 +48,70 @@ public void testNoMatch() {
         String fieldName = RandomDocumentPicks.randomFieldName(random());
         IngestDocument doc = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
         doc.setFieldValue(fieldName, "23");
-        GrokProcessor processor = new GrokProcessor(randomAsciiOfLength(10), Collections.singletonMap("ONE", "1"),
-            Collections.singletonList("%{ONE:one}"), fieldName);
+        GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), Collections.singletonMap("ONE", "1"),
+            Collections.singletonList("%{ONE:one}"), fieldName, false, false);
         Exception e = expectThrows(Exception.class, () -> processor.execute(doc));
         assertThat(e.getMessage(), equalTo("Provided Grok expressions do not match field value: [23]"));
     }
 
+    public void testNoMatchingPatternName() {
+        String fieldName = RandomDocumentPicks.randomFieldName(random());
+        IngestDocument doc = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
+        doc.setFieldValue(fieldName, "23");
+        Exception e = expectThrows(IllegalArgumentException.class, () -> new GrokProcessor(randomAlphaOfLength(10),
+            Collections.singletonMap("ONE", "1"), Collections.singletonList("%{NOTONE:not_one}"), fieldName,
+            false, false));
+        assertThat(e.getMessage(), equalTo("Unable to find pattern [NOTONE] in Grok's pattern dictionary"));
+    }
+
     public void testMatchWithoutCaptures() throws Exception {
         String fieldName = "value";
         IngestDocument originalDoc = new IngestDocument(new HashMap<>(), new HashMap<>());
         originalDoc.setFieldValue(fieldName, fieldName);
         IngestDocument doc = new IngestDocument(originalDoc);
-        GrokProcessor processor = new GrokProcessor(randomAsciiOfLength(10), Collections.emptyMap(),
-            Collections.singletonList(fieldName), fieldName);
+        GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), Collections.emptyMap(),
+            Collections.singletonList(fieldName), fieldName, false, false);
         processor.execute(doc);
         assertThat(doc, equalTo(originalDoc));
     }
 
+    public void testNullField() {
+        String fieldName = RandomDocumentPicks.randomFieldName(random());
+        IngestDocument doc = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
+        doc.setFieldValue(fieldName, null);
+        GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), Collections.singletonMap("ONE", "1"),
+            Collections.singletonList("%{ONE:one}"), fieldName, false, false);
+        Exception e = expectThrows(Exception.class, () -> processor.execute(doc));
+        assertThat(e.getMessage(), equalTo("field [" + fieldName + "] is null, cannot process it."));
+    }
+
+    public void testNullFieldWithIgnoreMissing() throws Exception {
+        String fieldName = RandomDocumentPicks.randomFieldName(random());
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
+        originalIngestDocument.setFieldValue(fieldName, null);
+        IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
+        GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), Collections.singletonMap("ONE", "1"),
+            Collections.singletonList("%{ONE:one}"), fieldName, false, true);
+        processor.execute(ingestDocument);
+        assertIngestDocument(originalIngestDocument, ingestDocument);
+    }
+
     public void testNotStringField() {
         String fieldName = RandomDocumentPicks.randomFieldName(random());
         IngestDocument doc = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
         doc.setFieldValue(fieldName, 1);
-        GrokProcessor processor = new GrokProcessor(randomAsciiOfLength(10), Collections.singletonMap("ONE", "1"),
-            Collections.singletonList("%{ONE:one}"), fieldName);
+        GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), Collections.singletonMap("ONE", "1"),
+            Collections.singletonList("%{ONE:one}"), fieldName, false, false);
+        Exception e = expectThrows(Exception.class, () -> processor.execute(doc));
+        assertThat(e.getMessage(), equalTo("field [" + fieldName + "] of type [java.lang.Integer] cannot be cast to [java.lang.String]"));
+    }
+
+    public void testNotStringFieldWithIgnoreMissing() {
+        String fieldName = RandomDocumentPicks.randomFieldName(random());
+        IngestDocument doc = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
+        doc.setFieldValue(fieldName, 1);
+        GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), Collections.singletonMap("ONE", "1"),
+            Collections.singletonList("%{ONE:one}"), fieldName, false, true);
         Exception e = expectThrows(Exception.class, () -> processor.execute(doc));
         assertThat(e.getMessage(), equalTo("field [" + fieldName + "] of type [java.lang.Integer] cannot be cast to [java.lang.String]"));
     }
@@ -77,12 +119,22 @@ public void testNotStringField() {
     public void testMissingField() {
         String fieldName = "foo.bar";
         IngestDocument doc = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
-        GrokProcessor processor = new GrokProcessor(randomAsciiOfLength(10), Collections.singletonMap("ONE", "1"),
-            Collections.singletonList("%{ONE:one}"), fieldName);
+        GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), Collections.singletonMap("ONE", "1"),
+            Collections.singletonList("%{ONE:one}"), fieldName, false, false);
         Exception e = expectThrows(Exception.class, () -> processor.execute(doc));
         assertThat(e.getMessage(), equalTo("field [foo] not present as part of path [foo.bar]"));
     }
 
+    public void testMissingFieldWithIgnoreMissing() throws Exception {
+        String fieldName = "foo.bar";
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
+        GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), Collections.singletonMap("ONE", "1"),
+            Collections.singletonList("%{ONE:one}"), fieldName, false, true);
+        processor.execute(ingestDocument);
+        assertIngestDocument(originalIngestDocument, ingestDocument);
+    }
+
     public void testMultiplePatternsWithMatchReturn() throws Exception {
         String fieldName = RandomDocumentPicks.randomFieldName(random());
         IngestDocument doc = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
@@ -91,8 +143,8 @@ public void testMultiplePatternsWithMatchReturn() throws Exception {
         patternBank.put("ONE", "1");
         patternBank.put("TWO", "2");
         patternBank.put("THREE", "3");
-        GrokProcessor processor = new GrokProcessor(randomAsciiOfLength(10), patternBank,
-            Arrays.asList("%{ONE:one}", "%{TWO:two}", "%{THREE:three}"), fieldName);
+        GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), patternBank,
+            Arrays.asList("%{ONE:one}", "%{TWO:two}", "%{THREE:three}"), fieldName, false, false);
         processor.execute(doc);
         assertThat(doc.hasField("one"), equalTo(false));
         assertThat(doc.getFieldValue("two", String.class), equalTo("2"));
@@ -107,8 +159,8 @@ public void testSetMetadata() throws Exception {
         patternBank.put("ONE", "1");
         patternBank.put("TWO", "2");
         patternBank.put("THREE", "3");
-        GrokProcessor processor = new GrokProcessor(randomAsciiOfLength(10), patternBank,
-            Arrays.asList("%{ONE:one}", "%{TWO:two}", "%{THREE:three}"), fieldName, true);
+        GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), patternBank,
+            Arrays.asList("%{ONE:one}", "%{TWO:two}", "%{THREE:three}"), fieldName, true, false);
         processor.execute(doc);
         assertThat(doc.hasField("one"), equalTo(false));
         assertThat(doc.getFieldValue("two", String.class), equalTo("2"));
@@ -116,6 +168,19 @@ public void testSetMetadata() throws Exception {
         assertThat(doc.getFieldValue("_ingest._grok_match_index", String.class), equalTo("1"));
     }
 
+    public void testTraceWithOnePattern() throws Exception {
+        String fieldName = RandomDocumentPicks.randomFieldName(random());
+        IngestDocument doc = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
+        doc.setFieldValue(fieldName, "first1");
+        Map<String, String> patternBank = new HashMap<>();
+        patternBank.put("ONE", "1");
+        GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), patternBank,
+            Arrays.asList("%{ONE:one}"), fieldName, true, false);
+        processor.execute(doc);
+        assertThat(doc.hasField("one"), equalTo(true));
+        assertThat(doc.getFieldValue("_ingest._grok_match_index", String.class), equalTo("0"));
+    }
+
     public void testCombinedPatterns() {
         String combined;
         combined = GrokProcessor.combinePatterns(Arrays.asList(""), false);
@@ -131,4 +196,45 @@ public void testCombinedPatterns() {
         combined = GrokProcessor.combinePatterns(Arrays.asList("foo", "bar"), true);
         assertThat(combined, equalTo("(?<_ingest._grok_match_index.0>foo)|(?<_ingest._grok_match_index.1>bar)"));
     }
+
+    public void testCombineSamePatternNameAcrossPatterns() throws Exception {
+        String fieldName = RandomDocumentPicks.randomFieldName(random());
+        IngestDocument doc = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
+        doc.setFieldValue(fieldName, "1-3");
+        Map<String, String> patternBank = new HashMap<>();
+        patternBank.put("ONE", "1");
+        patternBank.put("TWO", "2");
+        patternBank.put("THREE", "3");
+        GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), patternBank,
+            Arrays.asList("%{ONE:first}-%{TWO:second}", "%{ONE:first}-%{THREE:second}"), fieldName, randomBoolean(), randomBoolean());
+        processor.execute(doc);
+        assertThat(doc.getFieldValue("first", String.class), equalTo("1"));
+        assertThat(doc.getFieldValue("second", String.class), equalTo("3"));
+    }
+
+    public void testFirstWinNamedCapture() throws Exception {
+        String fieldName = RandomDocumentPicks.randomFieldName(random());
+        IngestDocument doc = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
+        doc.setFieldValue(fieldName, "12");
+        Map<String, String> patternBank = new HashMap<>();
+        patternBank.put("ONETWO", "1|2");
+        GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), patternBank,
+            Collections.singletonList("%{ONETWO:first}%{ONETWO:first}"), fieldName, randomBoolean(), randomBoolean());
+        processor.execute(doc);
+        assertThat(doc.getFieldValue("first", String.class), equalTo("1"));
+    }
+
+    public void testUnmatchedNamesNotIncludedInDocument()  throws Exception {
+        String fieldName = RandomDocumentPicks.randomFieldName(random());
+        IngestDocument doc = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
+        doc.setFieldValue(fieldName, "3");
+        Map<String, String> patternBank = new HashMap<>();
+        patternBank.put("ONETWO", "1|2");
+        patternBank.put("THREE", "3");
+        GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), patternBank,
+            Collections.singletonList("%{ONETWO:first}|%{THREE:second}"), fieldName, randomBoolean(), randomBoolean());
+        processor.execute(doc);
+        assertFalse(doc.hasField("first"));
+        assertThat(doc.getFieldValue("second", String.class), equalTo("3"));
+    }
 }
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokTests.java
index e8b88ea0d1241..7649727ca0f43 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokTests.java
@@ -24,6 +24,7 @@
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -48,6 +49,11 @@ public void testMatchWithoutCaptures() {
         assertEquals(0, matches.size());
     }
 
+    public void testNoMatchingPatternInDictionary() {
+        Exception e = expectThrows(IllegalArgumentException.class, () -> new Grok(Collections.emptyMap(), "%{NOTFOUND}"));
+        assertThat(e.getMessage(), equalTo("Unable to find pattern [NOTFOUND] in Grok's pattern dictionary"));
+    }
+
     public void testSimpleSyslogLine() {
         String line = "Mar 16 00:01:25 evita postfix/smtpd[1713]: connect from camomile.cloud9.net[168.100.1.3]";
         Grok grok = new Grok(basePatterns, "%{SYSLOGLINE}");
@@ -307,4 +313,14 @@ public void testNoMatch() {
         Grok grok = new Grok(bank, "%{MONTHDAY:greatday}");
         assertThat(grok.captures("nomatch"), nullValue());
     }
+
+    public void testMultipleNamedCapturesWithSameName() {
+        Map<String, String> bank = new HashMap<>();
+        bank.put("SINGLEDIGIT", "[0-9]");
+        Grok grok = new Grok(bank, "%{SINGLEDIGIT:num}%{SINGLEDIGIT:num}");
+
+        Map<String, Object> expected = new HashMap<>();
+        expected.put("num", "1");
+        assertThat(grok.captures("12"), equalTo(expected));
+    }
 }
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorFactoryTests.java
index 51e246c67e913..4a70b4686e0a6 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorFactoryTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.test.ESTestCase;
 
 import java.util.HashMap;
 import java.util.Map;
@@ -28,33 +27,25 @@
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.containsString;
 
-public class GsubProcessorFactoryTests extends ESTestCase {
+public class GsubProcessorFactoryTests extends AbstractStringProcessorFactoryTestCase {
 
-    public void testCreate() throws Exception {
-        GsubProcessor.Factory factory = new GsubProcessor.Factory();
-        Map<String, Object> config = new HashMap<>();
-        config.put("field", "field1");
-        config.put("pattern", "\\.");
-        config.put("replacement", "-");
-        String processorTag = randomAsciiOfLength(10);
-        GsubProcessor gsubProcessor = factory.create(null, processorTag, config);
-        assertThat(gsubProcessor.getTag(), equalTo(processorTag));
-        assertThat(gsubProcessor.getField(), equalTo("field1"));
-        assertThat(gsubProcessor.getPattern().toString(), equalTo("\\."));
-        assertThat(gsubProcessor.getReplacement(), equalTo("-"));
+    @Override
+    protected AbstractStringProcessor.Factory newFactory() {
+        return new GsubProcessor.Factory();
     }
 
-    public void testCreateNoFieldPresent() throws Exception {
-        GsubProcessor.Factory factory = new GsubProcessor.Factory();
-        Map<String, Object> config = new HashMap<>();
+    @Override
+    protected Map<String, Object> modifyConfig(Map<String, Object> config) {
         config.put("pattern", "\\.");
         config.put("replacement", "-");
-        try {
-            factory.create(null, null, config);
-            fail("factory create should have failed");
-        } catch(ElasticsearchParseException e) {
-            assertThat(e.getMessage(), equalTo("[field] required property is missing"));
-        }
+        return config;
+    }
+
+    @Override
+    protected void assertProcessor(AbstractStringProcessor processor) {
+        GsubProcessor gsubProcessor = (GsubProcessor) processor;
+        assertThat(gsubProcessor.getPattern().toString(), equalTo("\\."));
+        assertThat(gsubProcessor.getReplacement(), equalTo("-"));
     }
 
     public void testCreateNoPatternPresent() throws Exception {
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorTests.java
index 20dc5b8c8dec9..38d0202d3a1e2 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorTests.java
@@ -19,62 +19,22 @@
 
 package org.elasticsearch.ingest.common;
 
-import org.elasticsearch.ingest.IngestDocument;
-import org.elasticsearch.ingest.Processor;
-import org.elasticsearch.ingest.RandomDocumentPicks;
-import org.elasticsearch.test.ESTestCase;
-
-import java.util.Collections;
-import java.util.HashMap;
 import java.util.regex.Pattern;
 
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.equalTo;
-
-public class GsubProcessorTests extends ESTestCase {
-
-    public void testGsub() throws Exception {
-        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
-        String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, "127.0.0.1");
-        Processor processor = new GsubProcessor(randomAsciiOfLength(10), fieldName, Pattern.compile("\\."), "-");
-        processor.execute(ingestDocument);
-        assertThat(ingestDocument.getFieldValue(fieldName, String.class), equalTo("127-0-0-1"));
-    }
+public class GsubProcessorTests extends AbstractStringProcessorTestCase {
 
-    public void testGsubNotAStringValue() throws Exception {
-        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
-        String fieldName = RandomDocumentPicks.randomFieldName(random());
-        ingestDocument.setFieldValue(fieldName, 123);
-        Processor processor = new GsubProcessor(randomAsciiOfLength(10), fieldName, Pattern.compile("\\."), "-");
-        try {
-            processor.execute(ingestDocument);
-            fail("processor execution should have failed");
-        } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), equalTo("field [" + fieldName +
-                    "] of type [java.lang.Integer] cannot be cast to [java.lang.String]"));
-        }
+    @Override
+    protected AbstractStringProcessor newProcessor(String field, boolean ignoreMissing, String targetField) {
+        return new GsubProcessor(randomAlphaOfLength(10), field, Pattern.compile("\\."), "-", ignoreMissing, targetField);
     }
 
-    public void testGsubFieldNotFound() throws Exception {
-        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
-        String fieldName = RandomDocumentPicks.randomFieldName(random());
-        Processor processor = new GsubProcessor(randomAsciiOfLength(10), fieldName, Pattern.compile("\\."), "-");
-        try {
-            processor.execute(ingestDocument);
-            fail("processor execution should have failed");
-        } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), containsString("not present as part of path [" + fieldName + "]"));
-        }
+    @Override
+    protected String modifyInput(String input) {
+        return "127.0.0.1";
     }
 
-    public void testGsubNullValue() throws Exception {
-        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", null));
-        Processor processor = new GsubProcessor(randomAsciiOfLength(10), "field", Pattern.compile("\\."), "-");
-        try {
-            processor.execute(ingestDocument);
-            fail("processor execution should have failed");
-        } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), equalTo("field [field] is null, cannot match pattern."));
-        }
+    @Override
+    protected String expectedResult(String input) {
+        return "127-0-0-1";
     }
 }
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/IngestCommonClientYamlTestSuiteIT.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/IngestCommonClientYamlTestSuiteIT.java
index 3c71f5710fd1f..6c10c16494b43 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/IngestCommonClientYamlTestSuiteIT.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/IngestCommonClientYamlTestSuiteIT.java
@@ -24,9 +24,6 @@
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-
-import java.io.IOException;
 
 public class IngestCommonClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
@@ -35,8 +32,8 @@ public IngestCommonClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate t
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
-        return ESClientYamlSuiteTestCase.createParameters(0, 1);
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
     }
 }
 
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorFactoryTests.java
index 68b2daecb4a48..6506b5f450a5b 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorFactoryTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.HashMap;
@@ -34,11 +35,12 @@ public void testCreate() throws Exception {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "field1");
         config.put("separator", "-");
-        String processorTag = randomAsciiOfLength(10);
+        String processorTag = randomAlphaOfLength(10);
         JoinProcessor joinProcessor = factory.create(null, processorTag, config);
         assertThat(joinProcessor.getTag(), equalTo(processorTag));
         assertThat(joinProcessor.getField(), equalTo("field1"));
         assertThat(joinProcessor.getSeparator(), equalTo("-"));
+        assertThat(joinProcessor.getTargetField(), equalTo("field1"));
     }
 
     public void testCreateNoFieldPresent() throws Exception {
@@ -64,4 +66,18 @@ public void testCreateNoSeparatorPresent() throws Exception {
             assertThat(e.getMessage(), equalTo("[separator] required property is missing"));
         }
     }
+
+    public void testCreateWithTargetField() throws Exception {
+        JoinProcessor.Factory factory = new JoinProcessor.Factory();
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", "field1");
+        config.put("separator", "-");
+        config.put("target_field", "target");
+        String processorTag = randomAlphaOfLength(10);
+        JoinProcessor joinProcessor = factory.create(null, processorTag, config);
+        assertThat(joinProcessor.getTag(), equalTo(processorTag));
+        assertThat(joinProcessor.getField(), equalTo("field1"));
+        assertThat(joinProcessor.getSeparator(), equalTo("-"));
+        assertThat(joinProcessor.getTargetField(), equalTo("target"));
+    }
 }
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorTests.java
index 078d63657097b..acb84f7d6b5b7 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorTests.java
@@ -43,7 +43,7 @@ public void testJoinStrings() throws Exception {
         List<String> fieldValue = new ArrayList<>(numItems);
         String expectedResult = "";
         for (int j = 0; j < numItems; j++) {
-            String value = randomAsciiOfLengthBetween(1, 10);
+            String value = randomAlphaOfLengthBetween(1, 10);
             fieldValue.add(value);
             expectedResult += value;
             if (j < numItems - 1) {
@@ -51,7 +51,7 @@ public void testJoinStrings() throws Exception {
             }
         }
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
-        Processor processor = new JoinProcessor(randomAsciiOfLength(10), fieldName, separator);
+        Processor processor = new JoinProcessor(randomAlphaOfLength(10), fieldName, separator, fieldName);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue(fieldName, String.class), equalTo(expectedResult));
     }
@@ -71,7 +71,7 @@ public void testJoinIntegers() throws Exception {
             }
         }
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
-        Processor processor = new JoinProcessor(randomAsciiOfLength(10), fieldName, separator);
+        Processor processor = new JoinProcessor(randomAlphaOfLength(10), fieldName, separator, fieldName);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue(fieldName, String.class), equalTo(expectedResult));
     }
@@ -79,8 +79,8 @@ public void testJoinIntegers() throws Exception {
     public void testJoinNonListField() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
         String fieldName = RandomDocumentPicks.randomFieldName(random());
-        ingestDocument.setFieldValue(fieldName, randomAsciiOfLengthBetween(1, 10));
-        Processor processor = new JoinProcessor(randomAsciiOfLength(10), fieldName, "-");
+        ingestDocument.setFieldValue(fieldName, randomAlphaOfLengthBetween(1, 10));
+        Processor processor = new JoinProcessor(randomAlphaOfLength(10), fieldName, "-", fieldName);
         try {
             processor.execute(ingestDocument);
         } catch(IllegalArgumentException e) {
@@ -91,7 +91,7 @@ public void testJoinNonListField() throws Exception {
     public void testJoinNonExistingField() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
         String fieldName = RandomDocumentPicks.randomFieldName(random());
-        Processor processor = new JoinProcessor(randomAsciiOfLength(10), fieldName, "-");
+        Processor processor = new JoinProcessor(randomAlphaOfLength(10), fieldName, "-", fieldName);
         try {
             processor.execute(ingestDocument);
         } catch(IllegalArgumentException e) {
@@ -101,11 +101,32 @@ public void testJoinNonExistingField() throws Exception {
 
     public void testJoinNullValue() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", null));
-        Processor processor = new JoinProcessor(randomAsciiOfLength(10), "field", "-");
+        Processor processor = new JoinProcessor(randomAlphaOfLength(10), "field", "-", "field");
         try {
             processor.execute(ingestDocument);
         } catch(IllegalArgumentException e) {
             assertThat(e.getMessage(), equalTo("field [field] is null, cannot join."));
         }
     }
+
+    public void testJoinWithTargetField() throws Exception {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
+        int numItems = randomIntBetween(1, 10);
+        String separator = randomFrom(SEPARATORS);
+        List<String> fieldValue = new ArrayList<>(numItems);
+        String expectedResult = "";
+        for (int j = 0; j < numItems; j++) {
+            String value = randomAlphaOfLengthBetween(1, 10);
+            fieldValue.add(value);
+            expectedResult += value;
+            if (j < numItems - 1) {
+                expectedResult += separator;
+            }
+        }
+        String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
+        String targetFieldName = RandomDocumentPicks.randomFieldName(random());
+        Processor processor = new JoinProcessor(randomAlphaOfLength(10), fieldName, separator, targetFieldName);
+        processor.execute(ingestDocument);
+        assertThat(ingestDocument.getFieldValue(targetFieldName, String.class), equalTo(expectedResult));
+    }
 }
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorFactoryTests.java
new file mode 100644
index 0000000000000..66f163a31eb8f
--- /dev/null
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorFactoryTests.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest.common;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+
+public class JsonProcessorFactoryTests extends ESTestCase {
+
+    private static final JsonProcessor.Factory FACTORY = new JsonProcessor.Factory();
+
+    public void testCreate() throws Exception {
+        String processorTag = randomAlphaOfLength(10);
+        String randomField = randomAlphaOfLength(10);
+        String randomTargetField = randomAlphaOfLength(5);
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", randomField);
+        config.put("target_field", randomTargetField);
+        JsonProcessor jsonProcessor = FACTORY.create(null, processorTag, config);
+        assertThat(jsonProcessor.getTag(), equalTo(processorTag));
+        assertThat(jsonProcessor.getField(), equalTo(randomField));
+        assertThat(jsonProcessor.getTargetField(), equalTo(randomTargetField));
+    }
+
+    public void testCreateWithAddToRoot() throws Exception {
+        String processorTag = randomAlphaOfLength(10);
+        String randomField = randomAlphaOfLength(10);
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", randomField);
+        config.put("add_to_root", true);
+        JsonProcessor jsonProcessor = FACTORY.create(null, processorTag, config);
+        assertThat(jsonProcessor.getTag(), equalTo(processorTag));
+        assertThat(jsonProcessor.getField(), equalTo(randomField));
+        assertThat(jsonProcessor.getTargetField(), equalTo(randomField));
+        assertTrue(jsonProcessor.isAddToRoot());
+    }
+
+    public void testCreateWithDefaultTarget() throws Exception {
+        String processorTag = randomAlphaOfLength(10);
+        String randomField = randomAlphaOfLength(10);
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", randomField);
+        JsonProcessor jsonProcessor = FACTORY.create(null, processorTag, config);
+        assertThat(jsonProcessor.getTag(), equalTo(processorTag));
+        assertThat(jsonProcessor.getField(), equalTo(randomField));
+        assertThat(jsonProcessor.getTargetField(), equalTo(randomField));
+    }
+
+    public void testCreateWithMissingField() throws Exception {
+        Map<String, Object> config = new HashMap<>();
+        String processorTag = randomAlphaOfLength(10);
+        ElasticsearchException exception = expectThrows(ElasticsearchParseException.class,
+            () -> FACTORY.create(null, processorTag, config));
+        assertThat(exception.getMessage(), equalTo("[field] required property is missing"));
+    }
+
+    public void testCreateWithBothTargetFieldAndAddToRoot() throws Exception {
+        String randomField = randomAlphaOfLength(10);
+        String randomTargetField = randomAlphaOfLength(5);
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", randomField);
+        config.put("target_field", randomTargetField);
+        config.put("add_to_root", true);
+        ElasticsearchException exception = expectThrows(ElasticsearchParseException.class,
+            () -> FACTORY.create(null, randomAlphaOfLength(10), config));
+        assertThat(exception.getMessage(), equalTo("[target_field] Cannot set a target field while also setting `add_to_root` to true"));
+    }
+}
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java
new file mode 100644
index 0000000000000..66ffb87e8da14
--- /dev/null
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest.common;
+
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.RandomDocumentPicks;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
+import static org.hamcrest.Matchers.equalTo;
+
+public class JsonProcessorTests extends ESTestCase {
+
+    @SuppressWarnings("unchecked")
+    public void testExecute() throws Exception {
+        String processorTag = randomAlphaOfLength(3);
+        String randomField = randomAlphaOfLength(3);
+        String randomTargetField = randomAlphaOfLength(2);
+        JsonProcessor jsonProcessor = new JsonProcessor(processorTag, randomField, randomTargetField, false);
+        Map<String, Object> document = new HashMap<>();
+
+        Map<String, Object> randomJsonMap = RandomDocumentPicks.randomSource(random());
+        XContentBuilder builder = JsonXContent.contentBuilder().map(randomJsonMap);
+        String randomJson = XContentHelper.convertToJson(builder.bytes(), false);
+        document.put(randomField, randomJson);
+
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
+        jsonProcessor.execute(ingestDocument);
+        Map<String, Object> jsonified = ingestDocument.getFieldValue(randomTargetField, Map.class);
+        assertIngestDocument(ingestDocument.getFieldValue(randomTargetField, Object.class), jsonified);
+    }
+
+    public void testInvalidJson() {
+        JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", false);
+        Map<String, Object> document = new HashMap<>();
+        document.put("field", "invalid json");
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
+
+        Exception exception = expectThrows(IllegalArgumentException.class, () -> jsonProcessor.execute(ingestDocument));
+        assertThat(exception.getCause().getCause().getMessage(), equalTo("Unrecognized token"
+                + " 'invalid': was expecting ('true', 'false' or 'null')\n"
+                + " at [Source: invalid json; line: 1, column: 8]"));
+    }
+
+    public void testFieldMissing() {
+        JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", false);
+        Map<String, Object> document = new HashMap<>();
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
+
+        Exception exception = expectThrows(IllegalArgumentException.class, () -> jsonProcessor.execute(ingestDocument));
+        assertThat(exception.getMessage(), equalTo("field [field] not present as part of path [field]"));
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testAddToRoot() throws Exception {
+        String processorTag = randomAlphaOfLength(3);
+        String randomTargetField = randomAlphaOfLength(2);
+        JsonProcessor jsonProcessor = new JsonProcessor(processorTag, "a", randomTargetField, true);
+        Map<String, Object> document = new HashMap<>();
+
+        String json = "{\"a\": 1, \"b\": 2}";
+        document.put("a", json);
+        document.put("c", "see");
+
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
+        jsonProcessor.execute(ingestDocument);
+
+        Map<String, Object> expected = new HashMap<>();
+        expected.put("a", 1);
+        expected.put("b", 2);
+        expected.put("c", "see");
+        IngestDocument expectedIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), expected);
+
+        assertIngestDocument(ingestDocument, expectedIngestDocument);
+    }
+}
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorFactoryTests.java
new file mode 100644
index 0000000000000..cce8cb5b24b1b
--- /dev/null
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorFactoryTests.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest.common;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.nullValue;
+
+public class KeyValueProcessorFactoryTests extends ESTestCase {
+
+    public void testCreateWithDefaults() throws Exception {
+        KeyValueProcessor.Factory factory = new KeyValueProcessor.Factory();
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", "field1");
+        config.put("field_split", "&");
+        config.put("value_split", "=");
+        String processorTag = randomAlphaOfLength(10);
+        KeyValueProcessor processor = factory.create(null, processorTag, config);
+        assertThat(processor.getTag(), equalTo(processorTag));
+        assertThat(processor.getField(), equalTo("field1"));
+        assertThat(processor.getFieldSplit(), equalTo("&"));
+        assertThat(processor.getValueSplit(), equalTo("="));
+        assertThat(processor.getIncludeKeys(), is(nullValue()));
+        assertThat(processor.getTargetField(), is(nullValue()));
+        assertFalse(processor.isIgnoreMissing());
+    }
+
+    public void testCreateWithAllFieldsSet() throws Exception {
+        KeyValueProcessor.Factory factory = new KeyValueProcessor.Factory();
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", "field1");
+        config.put("field_split", "&");
+        config.put("value_split", "=");
+        config.put("target_field", "target");
+        config.put("include_keys", Arrays.asList("a", "b"));
+        config.put("exclude_keys", Collections.emptyList());
+        config.put("ignore_missing", true);
+        String processorTag = randomAlphaOfLength(10);
+        KeyValueProcessor processor = factory.create(null, processorTag, config);
+        assertThat(processor.getTag(), equalTo(processorTag));
+        assertThat(processor.getField(), equalTo("field1"));
+        assertThat(processor.getFieldSplit(), equalTo("&"));
+        assertThat(processor.getValueSplit(), equalTo("="));
+        assertThat(processor.getIncludeKeys(), equalTo(Sets.newHashSet("a", "b")));
+        assertThat(processor.getExcludeKeys(), equalTo(Collections.emptySet()));
+        assertThat(processor.getTargetField(), equalTo("target"));
+        assertTrue(processor.isIgnoreMissing());
+    }
+
+    public void testCreateWithMissingField() {
+        KeyValueProcessor.Factory factory = new KeyValueProcessor.Factory();
+        Map<String, Object> config = new HashMap<>();
+        String processorTag = randomAlphaOfLength(10);
+        ElasticsearchException exception = expectThrows(ElasticsearchParseException.class,
+            () -> factory.create(null, processorTag, config));
+        assertThat(exception.getMessage(), equalTo("[field] required property is missing"));
+    }
+
+    public void testCreateWithMissingFieldSplit() {
+        KeyValueProcessor.Factory factory = new KeyValueProcessor.Factory();
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", "field1");
+        String processorTag = randomAlphaOfLength(10);
+        ElasticsearchException exception = expectThrows(ElasticsearchParseException.class,
+            () -> factory.create(null, processorTag, config));
+        assertThat(exception.getMessage(), equalTo("[field_split] required property is missing"));
+    }
+
+    public void testCreateWithMissingValueSplit() {
+        KeyValueProcessor.Factory factory = new KeyValueProcessor.Factory();
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", "field1");
+        config.put("field_split", "&");
+        String processorTag = randomAlphaOfLength(10);
+        ElasticsearchException exception = expectThrows(ElasticsearchParseException.class,
+            () -> factory.create(null, processorTag, config));
+        assertThat(exception.getMessage(), equalTo("[value_split] required property is missing"));
+    }
+}
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorTests.java
new file mode 100644
index 0000000000000..380af44c2515f
--- /dev/null
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorTests.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest.common;
+
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
+import org.elasticsearch.ingest.RandomDocumentPicks;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
+import static org.hamcrest.Matchers.equalTo;
+
+public class KeyValueProcessorTests extends ESTestCase {
+
+    public void test() throws Exception {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
+        String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, "first=hello&second=world&second=universe");
+        Processor processor = new KeyValueProcessor(randomAlphaOfLength(10), fieldName, "&", "=", null, null, "target", false);
+        processor.execute(ingestDocument);
+        assertThat(ingestDocument.getFieldValue("target.first", String.class), equalTo("hello"));
+        assertThat(ingestDocument.getFieldValue("target.second", List.class), equalTo(Arrays.asList("world", "universe")));
+    }
+
+    public void testRootTarget() throws Exception {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
+        ingestDocument.setFieldValue("myField", "first=hello&second=world&second=universe");
+        Processor processor = new KeyValueProcessor(randomAlphaOfLength(10), "myField", "&", "=", null, null,null, false);
+        processor.execute(ingestDocument);
+        assertThat(ingestDocument.getFieldValue("first", String.class), equalTo("hello"));
+        assertThat(ingestDocument.getFieldValue("second", List.class), equalTo(Arrays.asList("world", "universe")));
+    }
+
+    public void testKeySameAsSourceField() throws Exception {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
+        ingestDocument.setFieldValue("first", "first=hello");
+        Processor processor = new KeyValueProcessor(randomAlphaOfLength(10), "first", "&", "=", null, null,null, false);
+        processor.execute(ingestDocument);
+        assertThat(ingestDocument.getFieldValue("first", List.class), equalTo(Arrays.asList("first=hello", "hello")));
+    }
+
+    public void testIncludeKeys() throws Exception {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
+        String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, "first=hello&second=world&second=universe");
+        Processor processor = new KeyValueProcessor(randomAlphaOfLength(10), fieldName, "&", "=",
+            Sets.newHashSet("first"), null, "target", false);
+        processor.execute(ingestDocument);
+        assertThat(ingestDocument.getFieldValue("target.first", String.class), equalTo("hello"));
+        assertFalse(ingestDocument.hasField("target.second"));
+    }
+
+    public void testExcludeKeys() throws Exception {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
+        String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, "first=hello&second=world&second=universe");
+        Processor processor = new KeyValueProcessor(randomAlphaOfLength(10), fieldName, "&", "=",
+            null, Sets.newHashSet("second"), "target", false);
+        processor.execute(ingestDocument);
+        assertThat(ingestDocument.getFieldValue("target.first", String.class), equalTo("hello"));
+        assertFalse(ingestDocument.hasField("target.second"));
+    }
+
+    public void testIncludeAndExcludeKeys() throws Exception {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
+        String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument,
+            "first=hello&second=world&second=universe&third=bar");
+        Processor processor = new KeyValueProcessor(randomAlphaOfLength(10), fieldName, "&", "=",
+            Sets.newHashSet("first", "second"), Sets.newHashSet("first", "second"), "target", false);
+        processor.execute(ingestDocument);
+        assertFalse(ingestDocument.hasField("target.first"));
+        assertFalse(ingestDocument.hasField("target.second"));
+        assertFalse(ingestDocument.hasField("target.third"));
+    }
+
+    public void testMissingField() {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
+        Processor processor = new KeyValueProcessor(randomAlphaOfLength(10), "unknown", "&",
+            "=", null, null, "target", false);
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> processor.execute(ingestDocument));
+        assertThat(exception.getMessage(), equalTo("field [unknown] not present as part of path [unknown]"));
+    }
+
+    public void testNullValueWithIgnoreMissing() throws Exception {
+        String fieldName = RandomDocumentPicks.randomFieldName(random());
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(),
+            Collections.singletonMap(fieldName, null));
+        IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
+        Processor processor = new KeyValueProcessor(randomAlphaOfLength(10), fieldName, "", "", null, null, "target", true);
+        processor.execute(ingestDocument);
+        assertIngestDocument(originalIngestDocument, ingestDocument);
+    }
+
+    public void testNonExistentWithIgnoreMissing() throws Exception {
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
+        IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
+        Processor processor = new KeyValueProcessor(randomAlphaOfLength(10), "unknown", "", "", null, null, "target", true);
+        processor.execute(ingestDocument);
+        assertIngestDocument(originalIngestDocument, ingestDocument);
+    }
+
+    public void testFailFieldSplitMatch() throws Exception {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
+        String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, "first=hello|second=world|second=universe");
+        Processor processor = new KeyValueProcessor(randomAlphaOfLength(10), fieldName, "&", "=", null, null, "target", false);
+        processor.execute(ingestDocument);
+        assertThat(ingestDocument.getFieldValue("target.first", String.class), equalTo("hello|second=world|second=universe"));
+        assertFalse(ingestDocument.hasField("target.second"));
+    }
+
+    public void testFailValueSplitMatch() throws Exception {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("foo", "bar"));
+        Processor processor = new KeyValueProcessor(randomAlphaOfLength(10), "foo", "&", "=", null, null, "target", false);
+        Exception exception = expectThrows(IllegalArgumentException.class, () -> processor.execute(ingestDocument));
+        assertThat(exception.getMessage(), equalTo("field [foo] does not contain value_split [=]"));
+    }
+}
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/LowercaseProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/LowercaseProcessorFactoryTests.java
index 4d98efc4bb02e..c4bf4a84790b8 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/LowercaseProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/LowercaseProcessorFactoryTests.java
@@ -19,34 +19,9 @@
 
 package org.elasticsearch.ingest.common;
 
-import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.test.ESTestCase;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.hamcrest.CoreMatchers.equalTo;
-
-public class LowercaseProcessorFactoryTests extends ESTestCase {
-
-    public void testCreate() throws Exception {
-        LowercaseProcessor.Factory factory = new LowercaseProcessor.Factory();
-        Map<String, Object> config = new HashMap<>();
-        config.put("field", "field1");
-        String processorTag = randomAsciiOfLength(10);
-        LowercaseProcessor uppercaseProcessor = (LowercaseProcessor)factory.create(null, processorTag, config);
-        assertThat(uppercaseProcessor.getTag(), equalTo(processorTag));
-        assertThat(uppercaseProcessor.getField(), equalTo("field1"));
-    }
-
-    public void testCreateMissingField() throws Exception {
-        LowercaseProcessor.Factory factory = new LowercaseProcessor.Factory();
-        Map<String, Object> config = new HashMap<>();
-        try {
-            factory.create(null, null, config);
-            fail("factory create should have failed");
-        } catch(ElasticsearchParseException e) {
-            assertThat(e.getMessage(), equalTo("[field] required property is missing"));
-        }
+public class LowercaseProcessorFactoryTests extends AbstractStringProcessorFactoryTestCase {
+    @Override
+    protected AbstractStringProcessor.Factory newFactory() {
+        return new LowercaseProcessor.Factory();
     }
 }
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/LowercaseProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/LowercaseProcessorTests.java
index 413b3f6465034..67a73669c0387 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/LowercaseProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/LowercaseProcessorTests.java
@@ -23,8 +23,8 @@
 
 public class LowercaseProcessorTests extends AbstractStringProcessorTestCase {
     @Override
-    protected AbstractStringProcessor newProcessor(String field) {
-        return new LowercaseProcessor(randomAsciiOfLength(10), field);
+    protected AbstractStringProcessor newProcessor(String field, boolean ignoreMissing, String targetField) {
+        return new LowercaseProcessor(randomAlphaOfLength(10), field, ignoreMissing, targetField);
     }
 
     @Override
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorFactoryTests.java
index 71e878744d5f0..dd8ad480ac05b 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorFactoryTests.java
@@ -25,9 +25,11 @@
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 import static org.hamcrest.CoreMatchers.equalTo;
 
@@ -43,10 +45,21 @@ public void init() {
     public void testCreate() throws Exception {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "field1");
-        String processorTag = randomAsciiOfLength(10);
+        String processorTag = randomAlphaOfLength(10);
         RemoveProcessor removeProcessor = factory.create(null, processorTag, config);
         assertThat(removeProcessor.getTag(), equalTo(processorTag));
-        assertThat(removeProcessor.getField().execute(Collections.emptyMap()), equalTo("field1"));
+        assertThat(removeProcessor.getFields().get(0).execute(Collections.emptyMap()), equalTo("field1"));
+    }
+
+    public void testCreateMultipleFields() throws Exception {
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", Arrays.asList("field1", "field2"));
+        String processorTag = randomAlphaOfLength(10);
+        RemoveProcessor removeProcessor = factory.create(null, processorTag, config);
+        assertThat(removeProcessor.getTag(), equalTo(processorTag));
+        assertThat(removeProcessor.getFields().stream()
+            .map(template -> template.execute(Collections.emptyMap()))
+            .collect(Collectors.toList()), equalTo(Arrays.asList("field1", "field2")));
     }
 
     public void testCreateMissingField() throws Exception {
@@ -63,7 +76,7 @@ public void testInvalidMustacheTemplate() throws Exception {
         RemoveProcessor.Factory factory = new RemoveProcessor.Factory(TestTemplateService.instance(true));
         Map<String, Object> config = new HashMap<>();
         config.put("field", "field1");
-        String processorTag = randomAsciiOfLength(10);
+        String processorTag = randomAlphaOfLength(10);
         ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, config));
         assertThat(exception.getMessage(), equalTo("java.lang.RuntimeException: could not compile script"));
         assertThat(exception.getHeader("processor_tag").get(0), equalTo(processorTag));
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorTests.java
index fd569567d1af9..0b140b62e5870 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorTests.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.ingest.TestTemplateService;
 import org.elasticsearch.test.ESTestCase;
 
+import java.util.Collections;
 import java.util.HashMap;
 
 import static org.hamcrest.Matchers.containsString;
@@ -35,7 +36,8 @@ public class RemoveProcessorTests extends ESTestCase {
     public void testRemoveFields() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
         String field = RandomDocumentPicks.randomExistingFieldName(random(), ingestDocument);
-        Processor processor = new RemoveProcessor(randomAsciiOfLength(10), new TestTemplateService.MockTemplate(field));
+        Processor processor = new RemoveProcessor(randomAlphaOfLength(10),
+            Collections.singletonList(new TestTemplateService.MockTemplate(field)));
         processor.execute(ingestDocument);
         assertThat(ingestDocument.hasField(field), equalTo(false));
     }
@@ -43,7 +45,8 @@ public void testRemoveFields() throws Exception {
     public void testRemoveNonExistingField() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
         String fieldName = RandomDocumentPicks.randomFieldName(random());
-        Processor processor = new RemoveProcessor(randomAsciiOfLength(10), new TestTemplateService.MockTemplate(fieldName));
+        Processor processor = new RemoveProcessor(randomAlphaOfLength(10),
+            Collections.singletonList(new TestTemplateService.MockTemplate(fieldName)));
         try {
             processor.execute(ingestDocument);
             fail("remove field should have failed");
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorFactoryTests.java
index 68f28cb30d73e..1eaac36a64cca 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorFactoryTests.java
@@ -34,11 +34,26 @@ public void testCreate() throws Exception {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "old_field");
         config.put("target_field", "new_field");
-        String processorTag = randomAsciiOfLength(10);
+        String processorTag = randomAlphaOfLength(10);
         RenameProcessor renameProcessor = factory.create(null, processorTag, config);
         assertThat(renameProcessor.getTag(), equalTo(processorTag));
         assertThat(renameProcessor.getField(), equalTo("old_field"));
         assertThat(renameProcessor.getTargetField(), equalTo("new_field"));
+        assertThat(renameProcessor.isIgnoreMissing(), equalTo(false));
+    }
+
+    public void testCreateWithIgnoreMissing() throws Exception {
+        RenameProcessor.Factory factory = new RenameProcessor.Factory();
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", "old_field");
+        config.put("target_field", "new_field");
+        config.put("ignore_missing", true);
+        String processorTag = randomAlphaOfLength(10);
+        RenameProcessor renameProcessor = factory.create(null, processorTag, config);
+        assertThat(renameProcessor.getTag(), equalTo(processorTag));
+        assertThat(renameProcessor.getField(), equalTo("old_field"));
+        assertThat(renameProcessor.getTargetField(), equalTo("new_field"));
+        assertThat(renameProcessor.isIgnoreMissing(), equalTo(true));
     }
 
     public void testCreateNoFieldPresent() throws Exception {
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java
index 09edd4b825adb..758e5eb997297 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java
@@ -30,6 +30,7 @@
 import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.nullValue;
@@ -44,7 +45,7 @@ public void testRename() throws Exception {
         do {
             newFieldName = RandomDocumentPicks.randomFieldName(random());
         } while (RandomDocumentPicks.canAddField(newFieldName, ingestDocument) == false || newFieldName.equals(fieldName));
-        Processor processor = new RenameProcessor(randomAsciiOfLength(10), fieldName, newFieldName);
+        Processor processor = new RenameProcessor(randomAlphaOfLength(10), fieldName, newFieldName, false);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue(newFieldName, Object.class), equalTo(fieldValue));
     }
@@ -62,7 +63,7 @@ public void testRenameArrayElement() throws Exception {
         document.put("one", one);
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
 
-        Processor processor = new RenameProcessor(randomAsciiOfLength(10), "list.0", "item");
+        Processor processor = new RenameProcessor(randomAlphaOfLength(10), "list.0", "item", false);
         processor.execute(ingestDocument);
         Object actualObject = ingestDocument.getSourceAndMetadata().get("list");
         assertThat(actualObject, instanceOf(List.class));
@@ -75,7 +76,7 @@ public void testRenameArrayElement() throws Exception {
         assertThat(actualObject, instanceOf(String.class));
         assertThat(actualObject, equalTo("item1"));
 
-        processor = new RenameProcessor(randomAsciiOfLength(10), "list.0", "list.3");
+        processor = new RenameProcessor(randomAlphaOfLength(10), "list.0", "list.3", false);
         try {
             processor.execute(ingestDocument);
             fail("processor execute should have failed");
@@ -90,7 +91,8 @@ public void testRenameArrayElement() throws Exception {
     public void testRenameNonExistingField() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
         String fieldName = RandomDocumentPicks.randomFieldName(random());
-        Processor processor = new RenameProcessor(randomAsciiOfLength(10), fieldName, RandomDocumentPicks.randomFieldName(random()));
+        Processor processor = new RenameProcessor(randomAlphaOfLength(10), fieldName,
+            RandomDocumentPicks.randomFieldName(random()), false);
         try {
             processor.execute(ingestDocument);
             fail("processor execute should have failed");
@@ -99,11 +101,21 @@ public void testRenameNonExistingField() throws Exception {
         }
     }
 
+    public void testRenameNonExistingFieldWithIgnoreMissing() throws Exception {
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
+        IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
+        String fieldName = RandomDocumentPicks.randomFieldName(random());
+        Processor processor = new RenameProcessor(randomAlphaOfLength(10), fieldName,
+            RandomDocumentPicks.randomFieldName(random()), true);
+        processor.execute(ingestDocument);
+        assertIngestDocument(originalIngestDocument, ingestDocument);
+    }
+
     public void testRenameNewFieldAlreadyExists() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
         String fieldName = RandomDocumentPicks.randomExistingFieldName(random(), ingestDocument);
-        Processor processor = new RenameProcessor(randomAsciiOfLength(10), RandomDocumentPicks.randomExistingFieldName(
-                random(), ingestDocument), fieldName);
+        Processor processor = new RenameProcessor(randomAlphaOfLength(10), RandomDocumentPicks.randomExistingFieldName(
+                random(), ingestDocument), fieldName, false);
         try {
             processor.execute(ingestDocument);
             fail("processor execute should have failed");
@@ -117,7 +129,7 @@ public void testRenameExistingFieldNullValue() throws Exception {
         String fieldName = RandomDocumentPicks.randomFieldName(random());
         ingestDocument.setFieldValue(fieldName, null);
         String newFieldName = RandomDocumentPicks.randomFieldName(random());
-        Processor processor = new RenameProcessor(randomAsciiOfLength(10), fieldName, newFieldName);
+        Processor processor = new RenameProcessor(randomAlphaOfLength(10), fieldName, newFieldName, false);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.hasField(fieldName), equalTo(false));
         assertThat(ingestDocument.hasField(newFieldName), equalTo(true));
@@ -137,7 +149,7 @@ public Object put(String key, Object value) {
         source.put("list", Collections.singletonList("item"));
 
         IngestDocument ingestDocument = new IngestDocument(source, Collections.emptyMap());
-        Processor processor = new RenameProcessor(randomAsciiOfLength(10), "list", "new_field");
+        Processor processor = new RenameProcessor(randomAlphaOfLength(10), "list", "new_field", false);
         try {
             processor.execute(ingestDocument);
             fail("processor execute should have failed");
@@ -161,7 +173,7 @@ public Object remove(Object key) {
         source.put("list", Collections.singletonList("item"));
 
         IngestDocument ingestDocument = new IngestDocument(source, Collections.emptyMap());
-        Processor processor = new RenameProcessor(randomAsciiOfLength(10), "list", "new_field");
+        Processor processor = new RenameProcessor(randomAlphaOfLength(10), "list", "new_field", false);
         try {
             processor.execute(ingestDocument);
             fail("processor execute should have failed");
@@ -176,12 +188,12 @@ public void testRenameLeafIntoBranch() throws Exception {
         Map<String, Object> source = new HashMap<>();
         source.put("foo", "bar");
         IngestDocument ingestDocument = new IngestDocument(source, Collections.emptyMap());
-        Processor processor1 = new RenameProcessor(randomAsciiOfLength(10), "foo", "foo.bar");
+        Processor processor1 = new RenameProcessor(randomAlphaOfLength(10), "foo", "foo.bar", false);
         processor1.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue("foo", Map.class), equalTo(Collections.singletonMap("bar", "bar")));
         assertThat(ingestDocument.getFieldValue("foo.bar", String.class), equalTo("bar"));
 
-        Processor processor2 = new RenameProcessor(randomAsciiOfLength(10), "foo.bar", "foo.bar.baz");
+        Processor processor2 = new RenameProcessor(randomAlphaOfLength(10), "foo.bar", "foo.bar.baz", false);
         processor2.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue("foo", Map.class), equalTo(Collections.singletonMap("bar",
                 Collections.singletonMap("baz", "bar"))));
@@ -189,7 +201,7 @@ public void testRenameLeafIntoBranch() throws Exception {
         assertThat(ingestDocument.getFieldValue("foo.bar.baz", String.class), equalTo("bar"));
 
         // for fun lets try to restore it (which don't allow today)
-        Processor processor3 = new RenameProcessor(randomAsciiOfLength(10), "foo.bar.baz", "foo");
+        Processor processor3 = new RenameProcessor(randomAlphaOfLength(10), "foo.bar.baz", "foo", false);
         Exception e = expectThrows(IllegalArgumentException.class, () -> processor3.execute(ingestDocument));
         assertThat(e.getMessage(), equalTo("field [foo] already exists"));
     }
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java
index ef517d986cb3b..5ce68bbe339db 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java
@@ -20,7 +20,8 @@
 package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptException;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
@@ -29,25 +30,57 @@
 import java.util.HashMap;
 import java.util.Map;
 
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
+import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 public class ScriptProcessorFactoryTests extends ESTestCase {
 
     private ScriptProcessor.Factory factory;
+    private static final Map<String, String> ingestScriptParamToType;
+    static {
+        Map<String, String> map = new HashMap<>();
+        map.put("id", "stored");
+        map.put("source", "inline");
+        map.put("file", "file");
+        ingestScriptParamToType = Collections.unmodifiableMap(map);
+    }
 
     @Before
     public void init() {
         factory = new ScriptProcessor.Factory(mock(ScriptService.class));
     }
 
+    public void testFactoryValidationWithDefaultLang() throws Exception {
+        Map<String, Object> configMap = new HashMap<>();
+        String randomType = randomFrom("id", "source", "file");
+        configMap.put(randomType, "foo");
+        ScriptProcessor processor = factory.create(null, randomAlphaOfLength(10), configMap);
+        assertThat(processor.getScript().getLang(), equalTo(Script.DEFAULT_SCRIPT_LANG));
+        assertThat(processor.getScript().getType().toString(), equalTo(ingestScriptParamToType.get(randomType)));
+        assertThat(processor.getScript().getParams(), equalTo(Collections.emptyMap()));
+    }
+
+    public void testFactoryValidationWithParams() throws Exception {
+        Map<String, Object> configMap = new HashMap<>();
+        String randomType = randomFrom("id", "source", "file");
+        Map<String, Object> randomParams = Collections.singletonMap(randomAlphaOfLength(10), randomAlphaOfLength(10));
+        configMap.put(randomType, "foo");
+        configMap.put("params", randomParams);
+        ScriptProcessor processor = factory.create(null, randomAlphaOfLength(10), configMap);
+        assertThat(processor.getScript().getLang(), equalTo(Script.DEFAULT_SCRIPT_LANG));
+        assertThat(processor.getScript().getType().toString(), equalTo(ingestScriptParamToType.get(randomType)));
+        assertThat(processor.getScript().getParams(), equalTo(randomParams));
+    }
 
     public void testFactoryValidationForMultipleScriptingTypes() throws Exception {
         Map<String, Object> configMap = new HashMap<>();
-        String randomType = randomFrom("id", "inline", "file");
-        String otherRandomType = randomFrom("id", "inline", "file");
+        String randomType = randomFrom("id", "source", "file");
+        String otherRandomType = randomFrom("id", "source", "file");
         while (randomType.equals(otherRandomType)) {
-            otherRandomType = randomFrom("id", "inline", "file");
+            otherRandomType = randomFrom("id", "source", "file");
         }
 
         configMap.put(randomType, "foo");
@@ -55,8 +88,8 @@ public void testFactoryValidationForMultipleScriptingTypes() throws Exception {
         configMap.put("lang", "mockscript");
 
         ElasticsearchException exception = expectThrows(ElasticsearchException.class,
-            () -> factory.create(null, randomAsciiOfLength(10), configMap));
-        assertThat(exception.getMessage(), is("[null] Only one of [file], [id], or [inline] may be configured"));
+            () -> factory.create(null, randomAlphaOfLength(10), configMap));
+        assertThat(exception.getMessage(), is("Only one of [file], [id], or [source] may be configured"));
     }
 
     public void testFactoryValidationAtLeastOneScriptingType() throws Exception {
@@ -64,8 +97,34 @@ public void testFactoryValidationAtLeastOneScriptingType() throws Exception {
         configMap.put("lang", "mockscript");
 
         ElasticsearchException exception = expectThrows(ElasticsearchException.class,
-            () -> factory.create(null, randomAsciiOfLength(10), configMap));
+            () -> factory.create(null, randomAlphaOfLength(10), configMap));
+
+        assertThat(exception.getMessage(), is("Need [file], [id], or [source] parameter to refer to scripts"));
+    }
+
+    public void testInlineBackcompat() throws Exception {
+        Map<String, Object> configMap = new HashMap<>();
+        configMap.put("inline", "code");
+
+        factory.create(null, randomAlphaOfLength(10), configMap);
+        assertWarnings("Specifying script source with [inline] is deprecated, use [source] instead.");
+    }
+
+    public void testFactoryInvalidateWithInvalidCompiledScript() throws Exception {
+        String randomType = randomFrom("file", "source", "id");
+        ScriptService mockedScriptService = mock(ScriptService.class);
+        ScriptException thrownException = new ScriptException("compile-time exception", new RuntimeException(),
+            Collections.emptyList(), "script", "mockscript");
+        when(mockedScriptService.compile(any(), any())).thenThrow(thrownException);
+        factory = new ScriptProcessor.Factory(mockedScriptService);
+
+        Map<String, Object> configMap = new HashMap<>();
+        configMap.put("lang", "mockscript");
+        configMap.put(randomType, "my_script");
+
+        ElasticsearchException exception = expectThrows(ElasticsearchException.class,
+            () -> factory.create(null, randomAlphaOfLength(10), configMap));
 
-        assertThat(exception.getMessage(), is("[null] Need [file], [id], or [inline] parameter to refer to scripts"));
+        assertThat(exception.getMessage(), is("compile-time exception"));
     }
 }
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java
index c32b0f101a05f..5356d9c9e0b72 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java
@@ -29,7 +29,6 @@
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.test.ESTestCase;
-import org.mockito.stubbing.Answer;
 
 import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.core.Is.is;
@@ -46,9 +45,9 @@ public void testScripting() throws Exception {
         int randomBytesTotal = randomBytesIn + randomBytesOut;
 
         ScriptService scriptService = mock(ScriptService.class);
-        Script script = new Script("_script");
+        Script script = mockScript("_script");
         ExecutableScript executableScript = mock(ExecutableScript.class);
-        when(scriptService.executable(any(), any(), any())).thenReturn(executableScript);
+        when(scriptService.executable(any(CompiledScript.class), any())).thenReturn(executableScript);
 
         Map<String, Object> document = new HashMap<>();
         document.put("bytes_in", randomInt());
@@ -60,7 +59,7 @@ public void testScripting() throws Exception {
             return null;
         }).when(executableScript).run();
 
-        ScriptProcessor processor = new ScriptProcessor(randomAsciiOfLength(10), script, scriptService);
+        ScriptProcessor processor = new ScriptProcessor(randomAlphaOfLength(10), script, scriptService);
 
         processor.execute(ingestDocument);
 
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorFactoryTests.java
index 45f144e330502..57d0b7f7b9371 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorFactoryTests.java
@@ -44,7 +44,7 @@ public void testCreate() throws Exception {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "field1");
         config.put("value", "value1");
-        String processorTag = randomAsciiOfLength(10);
+        String processorTag = randomAlphaOfLength(10);
         SetProcessor setProcessor = factory.create(null, processorTag, config);
         assertThat(setProcessor.getTag(), equalTo(processorTag));
         assertThat(setProcessor.getField().execute(Collections.emptyMap()), equalTo("field1"));
@@ -58,7 +58,7 @@ public void testCreateWithOverride() throws Exception {
         config.put("field", "field1");
         config.put("value", "value1");
         config.put("override", overrideEnabled);
-        String processorTag = randomAsciiOfLength(10);
+        String processorTag = randomAlphaOfLength(10);
         SetProcessor setProcessor = factory.create(null, processorTag, config);
         assertThat(setProcessor.getTag(), equalTo(processorTag));
         assertThat(setProcessor.getField().execute(Collections.emptyMap()), equalTo("field1"));
@@ -105,7 +105,7 @@ public void testInvalidMustacheTemplate() throws Exception {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "field1");
         config.put("value", "value1");
-        String processorTag = randomAsciiOfLength(10);
+        String processorTag = randomAlphaOfLength(10);
         ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, config));
         assertThat(exception.getMessage(), equalTo("java.lang.RuntimeException: could not compile script"));
         assertThat(exception.getHeader("processor_tag").get(0), equalTo(processorTag));
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorTests.java
index 9ef8fa44ac97f..d5a693b2bc142 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorTests.java
@@ -110,7 +110,7 @@ public void testSetMetadata() throws Exception {
 
     private static Processor createSetProcessor(String fieldName, Object fieldValue, boolean overrideEnabled) {
         TemplateService templateService = TestTemplateService.instance();
-        return new SetProcessor(randomAsciiOfLength(10), templateService.compile(fieldName),
+        return new SetProcessor(randomAlphaOfLength(10), templateService.compile(fieldName),
                 ValueSource.wrap(fieldValue, templateService), overrideEnabled);
     }
 }
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SortProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SortProcessorFactoryTests.java
new file mode 100644
index 0000000000000..452f02e96880a
--- /dev/null
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SortProcessorFactoryTests.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest.common;
+
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.ingest.RandomDocumentPicks;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+
+public class SortProcessorFactoryTests extends ESTestCase {
+
+    public void testCreate() throws Exception {
+        String processorTag = randomAlphaOfLength(10);
+        String fieldName = RandomDocumentPicks.randomFieldName(random());
+
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", fieldName);
+
+        SortProcessor.Factory factory = new SortProcessor.Factory();
+        SortProcessor processor = factory.create(null, processorTag, config);
+        assertThat(processor.getTag(), equalTo(processorTag));
+        assertThat(processor.getField(), equalTo(fieldName));
+        assertThat(processor.getOrder(), equalTo(SortProcessor.SortOrder.ASCENDING));
+        assertThat(processor.getTargetField(), equalTo(fieldName));
+    }
+
+    public void testCreateWithOrder() throws Exception {
+        String processorTag = randomAlphaOfLength(10);
+        String fieldName = RandomDocumentPicks.randomFieldName(random());
+
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", fieldName);
+        config.put("order", "desc");
+
+        SortProcessor.Factory factory = new SortProcessor.Factory();
+        SortProcessor processor = factory.create(null, processorTag, config);
+        assertThat(processor.getTag(), equalTo(processorTag));
+        assertThat(processor.getField(), equalTo(fieldName));
+        assertThat(processor.getOrder(), equalTo(SortProcessor.SortOrder.DESCENDING));
+        assertThat(processor.getTargetField(), equalTo(fieldName));
+    }
+
+    public void testCreateWithTargetField() throws Exception {
+        String processorTag = randomAlphaOfLength(10);
+        String fieldName = RandomDocumentPicks.randomFieldName(random());
+        String targetFieldName = RandomDocumentPicks.randomFieldName(random());
+
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", fieldName);
+        config.put("target_field", targetFieldName);
+
+        SortProcessor.Factory factory = new SortProcessor.Factory();
+        SortProcessor processor = factory.create(null, processorTag, config);
+        assertThat(processor.getTag(), equalTo(processorTag));
+        assertThat(processor.getField(), equalTo(fieldName));
+        assertThat(processor.getOrder(), equalTo(SortProcessor.SortOrder.ASCENDING));
+        assertThat(processor.getTargetField(), equalTo(targetFieldName));
+    }
+
+    public void testCreateWithInvalidOrder() throws Exception {
+        String processorTag = randomAlphaOfLength(10);
+        String fieldName = RandomDocumentPicks.randomFieldName(random());
+
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", fieldName);
+        config.put("order", "invalid");
+
+        SortProcessor.Factory factory = new SortProcessor.Factory();
+        try {
+            factory.create(null, processorTag, config);
+            fail("factory create should have failed");
+        } catch (ElasticsearchParseException e) {
+            assertThat(e.getMessage(), equalTo("[order] Sort direction [invalid] not recognized. Valid values are: [asc, desc]"));
+        }
+    }
+
+    public void testCreateMissingField() throws Exception {
+        SortProcessor.Factory factory = new SortProcessor.Factory();
+        Map<String, Object> config = new HashMap<>();
+        try {
+            factory.create(null, null, config);
+            fail("factory create should have failed");
+        } catch(ElasticsearchParseException e) {
+            assertThat(e.getMessage(), equalTo("[field] required property is missing"));
+        }
+    }
+}
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SortProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SortProcessorTests.java
index 97352109e2cb0..5eca68f35de21 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SortProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SortProcessorTests.java
@@ -42,7 +42,7 @@ public void testSortStrings() throws Exception {
         List<String> fieldValue = new ArrayList<>(numItems);
         List<String> expectedResult = new ArrayList<>(numItems);
         for (int j = 0; j < numItems; j++) {
-            String value = randomAsciiOfLengthBetween(1, 10);
+            String value = randomAlphaOfLengthBetween(1, 10);
             fieldValue.add(value);
             expectedResult.add(value);
         }
@@ -54,7 +54,7 @@ public void testSortStrings() throws Exception {
         }
 
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
-        Processor processor = new SortProcessor(randomAsciiOfLength(10), fieldName, order);
+        Processor processor = new SortProcessor(randomAlphaOfLength(10), fieldName, order, fieldName);
         processor.execute(ingestDocument);
         assertEquals(ingestDocument.getFieldValue(fieldName, List.class), expectedResult);
     }
@@ -68,7 +68,7 @@ public void testSortIntegersNonRandom() throws Exception {
         Collections.shuffle(fieldValue, random());
 
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
-        Processor processor = new SortProcessor(randomAsciiOfLength(10), fieldName, SortOrder.ASCENDING);
+        Processor processor = new SortProcessor(randomAlphaOfLength(10), fieldName, SortOrder.ASCENDING, fieldName);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue(fieldName, List.class).toArray(), equalTo(expectedResult));
     }
@@ -91,7 +91,7 @@ public void testSortIntegers() throws Exception {
         }
 
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
-        Processor processor = new SortProcessor(randomAsciiOfLength(10), fieldName, order);
+        Processor processor = new SortProcessor(randomAlphaOfLength(10), fieldName, order, fieldName);
         processor.execute(ingestDocument);
         assertEquals(ingestDocument.getFieldValue(fieldName, List.class), expectedResult);
     }
@@ -114,7 +114,7 @@ public void testSortShorts() throws Exception {
         }
 
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
-        Processor processor = new SortProcessor(randomAsciiOfLength(10), fieldName, order);
+        Processor processor = new SortProcessor(randomAlphaOfLength(10), fieldName, order, fieldName);
         processor.execute(ingestDocument);
         assertEquals(ingestDocument.getFieldValue(fieldName, List.class), expectedResult);
     }
@@ -137,7 +137,7 @@ public void testSortDoubles() throws Exception {
         }
 
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
-        Processor processor = new SortProcessor(randomAsciiOfLength(10), fieldName, order);
+        Processor processor = new SortProcessor(randomAlphaOfLength(10), fieldName, order, fieldName);
         processor.execute(ingestDocument);
         assertEquals(ingestDocument.getFieldValue(fieldName, List.class), expectedResult);
     }
@@ -160,7 +160,7 @@ public void testSortFloats() throws Exception {
         }
 
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
-        Processor processor = new SortProcessor(randomAsciiOfLength(10), fieldName, order);
+        Processor processor = new SortProcessor(randomAlphaOfLength(10), fieldName, order, fieldName);
         processor.execute(ingestDocument);
         assertEquals(ingestDocument.getFieldValue(fieldName, List.class), expectedResult);
     }
@@ -183,7 +183,7 @@ public void testSortBytes() throws Exception {
         }
 
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
-        Processor processor = new SortProcessor(randomAsciiOfLength(10), fieldName, order);
+        Processor processor = new SortProcessor(randomAlphaOfLength(10), fieldName, order, fieldName);
         processor.execute(ingestDocument);
         assertEquals(ingestDocument.getFieldValue(fieldName, List.class), expectedResult);
     }
@@ -206,7 +206,7 @@ public void testSortBooleans() throws Exception {
         }
 
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
-        Processor processor = new SortProcessor(randomAsciiOfLength(10), fieldName, order);
+        Processor processor = new SortProcessor(randomAlphaOfLength(10), fieldName, order, fieldName);
         processor.execute(ingestDocument);
         assertEquals(ingestDocument.getFieldValue(fieldName, List.class), expectedResult);
     }
@@ -221,7 +221,7 @@ public void testSortMixedStrings() throws Exception {
             if (randomBoolean()) {
                 value = String.valueOf(randomIntBetween(0, 100));
             } else {
-                value = randomAsciiOfLengthBetween(1, 10);
+                value = randomAlphaOfLengthBetween(1, 10);
             }
             fieldValue.add(value);
             expectedResult.add(value);
@@ -234,7 +234,7 @@ public void testSortMixedStrings() throws Exception {
         }
 
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
-        Processor processor = new SortProcessor(randomAsciiOfLength(10), fieldName, order);
+        Processor processor = new SortProcessor(randomAlphaOfLength(10), fieldName, order, fieldName);
         processor.execute(ingestDocument);
         assertEquals(ingestDocument.getFieldValue(fieldName, List.class), expectedResult);
     }
@@ -242,9 +242,9 @@ public void testSortMixedStrings() throws Exception {
     public void testSortNonListField() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
         String fieldName = RandomDocumentPicks.randomFieldName(random());
-        ingestDocument.setFieldValue(fieldName, randomAsciiOfLengthBetween(1, 10));
+        ingestDocument.setFieldValue(fieldName, randomAlphaOfLengthBetween(1, 10));
         SortOrder order = randomBoolean() ? SortOrder.ASCENDING : SortOrder.DESCENDING;
-        Processor processor = new SortProcessor(randomAsciiOfLength(10), fieldName, order);
+        Processor processor = new SortProcessor(randomAlphaOfLength(10), fieldName, order, fieldName);
         try {
             processor.execute(ingestDocument);
         } catch(IllegalArgumentException e) {
@@ -256,7 +256,7 @@ public void testSortNonExistingField() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
         String fieldName = RandomDocumentPicks.randomFieldName(random());
         SortOrder order = randomBoolean() ? SortOrder.ASCENDING : SortOrder.DESCENDING;
-        Processor processor = new SortProcessor(randomAsciiOfLength(10), fieldName, order);
+        Processor processor = new SortProcessor(randomAlphaOfLength(10), fieldName, order, fieldName);
         try {
             processor.execute(ingestDocument);
         } catch(IllegalArgumentException e) {
@@ -267,7 +267,7 @@ public void testSortNonExistingField() throws Exception {
     public void testSortNullValue() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", null));
         SortOrder order = randomBoolean() ? SortOrder.ASCENDING : SortOrder.DESCENDING;
-        Processor processor = new SortProcessor(randomAsciiOfLength(10), "field", order);
+        Processor processor = new SortProcessor(randomAlphaOfLength(10), "field", order, "field");
         try {
             processor.execute(ingestDocument);
         } catch(IllegalArgumentException e) {
@@ -275,6 +275,64 @@ public void testSortNullValue() throws Exception {
         }
     }
 
+    public void testDescendingSortWithTargetField() throws Exception {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
+        int numItems = randomIntBetween(1, 10);
+        List<String> fieldValue = new ArrayList<>(numItems);
+        List<String> expectedResult = new ArrayList<>(numItems);
+        for (int j = 0; j < numItems; j++) {
+            String value = randomAlphaOfLengthBetween(1, 10);
+            fieldValue.add(value);
+            expectedResult.add(value);
+        }
+
+        Collections.sort(expectedResult, Collections.reverseOrder());
+
+        String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
+        String targetFieldName = fieldName + "foo";
+        Processor processor = new SortProcessor(randomAlphaOfLength(10), fieldName,
+            SortOrder.DESCENDING, targetFieldName);
+        processor.execute(ingestDocument);
+        assertEquals(ingestDocument.getFieldValue(targetFieldName, List.class), expectedResult);
+    }
+
+    public void testAscendingSortWithTargetField() throws Exception {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
+        int numItems = randomIntBetween(1, 10);
+        List<String> fieldValue = new ArrayList<>(numItems);
+        List<String> expectedResult = new ArrayList<>(numItems);
+        for (int j = 0; j < numItems; j++) {
+            String value = randomAlphaOfLengthBetween(1, 10);
+            fieldValue.add(value);
+            expectedResult.add(value);
+        }
 
+        Collections.sort(expectedResult);
 
+        String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
+        String targetFieldName = fieldName + "foo";
+        Processor processor = new SortProcessor(randomAlphaOfLength(10), fieldName,
+            SortOrder.ASCENDING, targetFieldName);
+        processor.execute(ingestDocument);
+        assertEquals(ingestDocument.getFieldValue(targetFieldName, List.class), expectedResult);
+    }
+
+    public void testSortWithTargetFieldLeavesOriginalUntouched() throws Exception {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
+        List<Integer> fieldValue = Arrays.asList(1, 5, 4);
+        List<Integer> expectedResult = new ArrayList<>(fieldValue);
+        Collections.sort(expectedResult);
+
+        SortOrder order = randomBoolean() ? SortOrder.ASCENDING : SortOrder.DESCENDING;
+        if (order.equals(SortOrder.DESCENDING)) {
+            Collections.reverse(expectedResult);
+        }
+
+        String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, new ArrayList<>(fieldValue));
+        String targetFieldName = fieldName + "foo";
+        Processor processor = new SortProcessor(randomAlphaOfLength(10), fieldName, order, targetFieldName);
+        processor.execute(ingestDocument);
+        assertEquals(ingestDocument.getFieldValue(targetFieldName, List.class), expectedResult);
+        assertEquals(ingestDocument.getFieldValue(fieldName, List.class), fieldValue);
+    }
 }
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorFactoryTests.java
index bef9a84a1e2cf..cb7857f6b459f 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorFactoryTests.java
@@ -34,11 +34,13 @@ public void testCreate() throws Exception {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "field1");
         config.put("separator", "\\.");
-        String processorTag = randomAsciiOfLength(10);
+        String processorTag = randomAlphaOfLength(10);
         SplitProcessor splitProcessor = factory.create(null, processorTag, config);
         assertThat(splitProcessor.getTag(), equalTo(processorTag));
         assertThat(splitProcessor.getField(), equalTo("field1"));
         assertThat(splitProcessor.getSeparator(), equalTo("\\."));
+        assertFalse(splitProcessor.isIgnoreMissing());
+        assertThat(splitProcessor.getTargetField(), equalTo("field1"));
     }
 
     public void testCreateNoFieldPresent() throws Exception {
@@ -64,4 +66,19 @@ public void testCreateNoSeparatorPresent() throws Exception {
             assertThat(e.getMessage(), equalTo("[separator] required property is missing"));
         }
     }
+
+    public void testCreateWithTargetField() throws Exception {
+        SplitProcessor.Factory factory = new SplitProcessor.Factory();
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", "field1");
+        config.put("separator", "\\.");
+        config.put("target_field", "target");
+        String processorTag = randomAlphaOfLength(10);
+        SplitProcessor splitProcessor = factory.create(null, processorTag, config);
+        assertThat(splitProcessor.getTag(), equalTo(processorTag));
+        assertThat(splitProcessor.getField(), equalTo("field1"));
+        assertThat(splitProcessor.getSeparator(), equalTo("\\."));
+        assertFalse(splitProcessor.isIgnoreMissing());
+        assertThat(splitProcessor.getTargetField(), equalTo("target"));
+    }
 }
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorTests.java
index edb3639679c28..b4800cd387766 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorTests.java
@@ -30,6 +30,7 @@
 import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 
@@ -38,7 +39,7 @@ public class SplitProcessorTests extends ESTestCase {
     public void testSplit() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, "127.0.0.1");
-        Processor processor = new SplitProcessor(randomAsciiOfLength(10), fieldName, "\\.");
+        Processor processor = new SplitProcessor(randomAlphaOfLength(10), fieldName, "\\.", false, fieldName);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(Arrays.asList("127", "0", "0", "1")));
     }
@@ -46,7 +47,7 @@ public void testSplit() throws Exception {
     public void testSplitFieldNotFound() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
         String fieldName = RandomDocumentPicks.randomFieldName(random());
-        Processor processor = new SplitProcessor(randomAsciiOfLength(10), fieldName, "\\.");
+        Processor processor = new SplitProcessor(randomAlphaOfLength(10), fieldName, "\\.", false, fieldName);
         try {
             processor.execute(ingestDocument);
             fail("split processor should have failed");
@@ -56,8 +57,9 @@ public void testSplitFieldNotFound() throws Exception {
     }
 
     public void testSplitNullValue() throws Exception {
-        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", null));
-        Processor processor = new SplitProcessor(randomAsciiOfLength(10), "field", "\\.");
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(),
+            Collections.singletonMap("field", null));
+        Processor processor = new SplitProcessor(randomAlphaOfLength(10), "field", "\\.", false, "field");
         try {
             processor.execute(ingestDocument);
             fail("split processor should have failed");
@@ -66,11 +68,29 @@ public void testSplitNullValue() throws Exception {
         }
     }
 
+    public void testSplitNullValueWithIgnoreMissing() throws Exception {
+        String fieldName = RandomDocumentPicks.randomFieldName(random());
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(),
+            Collections.singletonMap(fieldName, null));
+        IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
+        Processor processor = new SplitProcessor(randomAlphaOfLength(10), fieldName, "\\.", true, fieldName);
+        processor.execute(ingestDocument);
+        assertIngestDocument(originalIngestDocument, ingestDocument);
+    }
+
+    public void testSplitNonExistentWithIgnoreMissing() throws Exception {
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
+        IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
+        Processor processor = new SplitProcessor(randomAlphaOfLength(10), "field", "\\.", true, "field");
+        processor.execute(ingestDocument);
+        assertIngestDocument(originalIngestDocument, ingestDocument);
+    }
+
     public void testSplitNonStringValue() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
         String fieldName = RandomDocumentPicks.randomFieldName(random());
         ingestDocument.setFieldValue(fieldName, randomInt());
-        Processor processor = new SplitProcessor(randomAsciiOfLength(10), fieldName, "\\.");
+        Processor processor = new SplitProcessor(randomAlphaOfLength(10), fieldName, "\\.", false, fieldName);
         try {
             processor.execute(ingestDocument);
             fail("split processor should have failed");
@@ -96,4 +116,13 @@ public void testSplitAppendable() throws Exception {
         assertThat(ingestDocument.getFieldValue("flags", List.class), equalTo(Arrays.asList("new", "hot", "super",
                 "fun", "interesting", "additional_flag")));
     }
+
+    public void testSplitWithTargetField() throws Exception {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
+        String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, "127.0.0.1");
+        String targetFieldName = RandomDocumentPicks.randomFieldName(random());
+        Processor processor = new SplitProcessor(randomAlphaOfLength(10), fieldName, "\\.", false, targetFieldName);
+        processor.execute(ingestDocument);
+        assertThat(ingestDocument.getFieldValue(targetFieldName, List.class), equalTo(Arrays.asList("127", "0", "0", "1")));
+    }
 }
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/TrimProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/TrimProcessorFactoryTests.java
index 1bd57c79aca9c..98b203466a784 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/TrimProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/TrimProcessorFactoryTests.java
@@ -19,34 +19,9 @@
 
 package org.elasticsearch.ingest.common;
 
-import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.test.ESTestCase;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.hamcrest.CoreMatchers.equalTo;
-
-public class TrimProcessorFactoryTests extends ESTestCase {
-
-    public void testCreate() throws Exception {
-        TrimProcessor.Factory factory = new TrimProcessor.Factory();
-        Map<String, Object> config = new HashMap<>();
-        config.put("field", "field1");
-        String processorTag = randomAsciiOfLength(10);
-        TrimProcessor uppercaseProcessor = (TrimProcessor)factory.create(null, processorTag, config);
-        assertThat(uppercaseProcessor.getTag(), equalTo(processorTag));
-        assertThat(uppercaseProcessor.getField(), equalTo("field1"));
-    }
-
-    public void testCreateMissingField() throws Exception {
-        TrimProcessor.Factory factory = new TrimProcessor.Factory();
-        Map<String, Object> config = new HashMap<>();
-        try {
-            factory.create(null, null, config);
-            fail("factory create should have failed");
-        } catch(ElasticsearchParseException e) {
-            assertThat(e.getMessage(), equalTo("[field] required property is missing"));
-        }
+public class TrimProcessorFactoryTests extends AbstractStringProcessorFactoryTestCase {
+    @Override
+    protected AbstractStringProcessor.Factory newFactory() {
+        return new TrimProcessor.Factory();
     }
 }
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/TrimProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/TrimProcessorTests.java
index 4b776a2ee840a..f0ae554f5cad1 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/TrimProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/TrimProcessorTests.java
@@ -22,8 +22,8 @@
 public class TrimProcessorTests extends AbstractStringProcessorTestCase {
 
     @Override
-    protected AbstractStringProcessor newProcessor(String field) {
-        return new TrimProcessor(randomAsciiOfLength(10), field);
+    protected AbstractStringProcessor newProcessor(String field, boolean ignoreMissing, String targetField) {
+        return new TrimProcessor(randomAlphaOfLength(10), field, ignoreMissing, targetField);
     }
 
     @Override
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UppercaseProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UppercaseProcessorFactoryTests.java
index 0f8341195102f..e7b403e178815 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UppercaseProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UppercaseProcessorFactoryTests.java
@@ -19,34 +19,9 @@
 
 package org.elasticsearch.ingest.common;
 
-import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.test.ESTestCase;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.hamcrest.CoreMatchers.equalTo;
-
-public class UppercaseProcessorFactoryTests extends ESTestCase {
-
-    public void testCreate() throws Exception {
-        UppercaseProcessor.Factory factory = new UppercaseProcessor.Factory();
-        Map<String, Object> config = new HashMap<>();
-        config.put("field", "field1");
-        String processorTag = randomAsciiOfLength(10);
-        UppercaseProcessor uppercaseProcessor = (UppercaseProcessor)factory.create(null, processorTag, config);
-        assertThat(uppercaseProcessor.getTag(), equalTo(processorTag));
-        assertThat(uppercaseProcessor.getField(), equalTo("field1"));
-    }
-
-    public void testCreateMissingField() throws Exception {
-        UppercaseProcessor.Factory factory = new UppercaseProcessor.Factory();
-        Map<String, Object> config = new HashMap<>();
-        try {
-            factory.create(null, null, config);
-            fail("factory create should have failed");
-        } catch(ElasticsearchParseException e) {
-            assertThat(e.getMessage(), equalTo("[field] required property is missing"));
-        }
+public class UppercaseProcessorFactoryTests extends AbstractStringProcessorFactoryTestCase {
+    @Override
+    protected AbstractStringProcessor.Factory newFactory() {
+        return new UppercaseProcessor.Factory();
     }
 }
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UppercaseProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UppercaseProcessorTests.java
index ff8dc16b6e252..76459f8116890 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UppercaseProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UppercaseProcessorTests.java
@@ -24,8 +24,8 @@
 public class UppercaseProcessorTests extends AbstractStringProcessorTestCase {
 
     @Override
-    protected AbstractStringProcessor newProcessor(String field) {
-        return new UppercaseProcessor(randomAsciiOfLength(10), field);
+    protected AbstractStringProcessor newProcessor(String field, boolean ignoreMissing, String targetField) {
+        return new UppercaseProcessor(randomAlphaOfLength(10), field, ignoreMissing, targetField);
     }
 
     @Override
diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/10_basic.yaml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/10_basic.yaml
index e37b2d83183b8..a58c329a7c525 100644
--- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/10_basic.yaml
+++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/10_basic.yaml
@@ -19,12 +19,14 @@
     - match:  { nodes.$master.ingest.processors.7.type: grok }
     - match:  { nodes.$master.ingest.processors.8.type: gsub }
     - match:  { nodes.$master.ingest.processors.9.type: join }
-    - match:  { nodes.$master.ingest.processors.10.type: lowercase }
-    - match:  { nodes.$master.ingest.processors.11.type: remove }
-    - match:  { nodes.$master.ingest.processors.12.type: rename }
-    - match:  { nodes.$master.ingest.processors.13.type: script }
-    - match:  { nodes.$master.ingest.processors.14.type: set }
-    - match:  { nodes.$master.ingest.processors.15.type: sort }
-    - match:  { nodes.$master.ingest.processors.16.type: split }
-    - match:  { nodes.$master.ingest.processors.17.type: trim }
-    - match:  { nodes.$master.ingest.processors.18.type: uppercase }
+    - match:  { nodes.$master.ingest.processors.10.type: json }
+    - match:  { nodes.$master.ingest.processors.11.type: kv }
+    - match:  { nodes.$master.ingest.processors.12.type: lowercase }
+    - match:  { nodes.$master.ingest.processors.13.type: remove }
+    - match:  { nodes.$master.ingest.processors.14.type: rename }
+    - match:  { nodes.$master.ingest.processors.15.type: script }
+    - match:  { nodes.$master.ingest.processors.16.type: set }
+    - match:  { nodes.$master.ingest.processors.17.type: sort }
+    - match:  { nodes.$master.ingest.processors.18.type: split }
+    - match:  { nodes.$master.ingest.processors.19.type: trim }
+    - match:  { nodes.$master.ingest.processors.20.type: uppercase }
diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/120_grok.yaml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/120_grok.yaml
index 0d77ccce61c37..353365df95a78 100644
--- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/120_grok.yaml
+++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/120_grok.yaml
@@ -154,3 +154,10 @@ teardown:
   - length: { docs.0.doc._ingest: 2 }
   - match: { docs.0.doc._ingest._grok_match_index: "1" }
   - is_true: docs.0.doc._ingest.timestamp
+
+---
+"Test Grok Patterns Retrieval":
+  - do:
+      ingest.processor.grok: {}
+  - length: { patterns: 303 }
+  - match: { patterns.PATH: "(?:%{UNIXPATH}|%{WINPATH})" }
diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/140_json.yaml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/140_json.yaml
new file mode 100644
index 0000000000000..3d9f6a97c081a
--- /dev/null
+++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/140_json.yaml
@@ -0,0 +1,40 @@
+---
+teardown:
+  - do:
+      ingest.delete_pipeline:
+        id: "1"
+        ignore: 404
+
+---
+"Test JSON Processor":
+  - do:
+      ingest.put_pipeline:
+        id: "1"
+        body:  >
+          {
+            "processors": [
+              {
+                "json" : {
+                  "field" : "foo"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: test
+        type: test
+        id: 1
+        pipeline: "1"
+        body: {
+          foo: "{\"hello\": \"world\"}"
+        }
+
+  - do:
+      get:
+        index: test
+        type: test
+        id: 1
+  - match: { _source.foo.hello: "world" }
diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/150_kv.yaml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/150_kv.yaml
new file mode 100644
index 0000000000000..a1ecf10278c61
--- /dev/null
+++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/150_kv.yaml
@@ -0,0 +1,43 @@
+---
+teardown:
+  - do:
+      ingest.delete_pipeline:
+        id: "1"
+        ignore: 404
+
+---
+"Test KV Processor":
+  - do:
+      ingest.put_pipeline:
+        id: "1"
+        body:  >
+          {
+            "processors": [
+              {
+                "kv" : {
+                  "field" : "foo",
+                  "field_split": " ",
+                  "value_split": "="
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: test
+        type: test
+        id: 1
+        pipeline: "1"
+        body: {
+          foo: "goodbye=everybody hello=world"
+        }
+
+  - do:
+      get:
+        index: test
+        type: test
+        id: 1
+  - match: { _source.goodbye: "everybody" }
+  - match: { _source.hello: "world" }
diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/20_crud.yaml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/20_crud.yaml
index b12a65f8fb656..b041e0664bb6c 100644
--- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/20_crud.yaml
+++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/20_crud.yaml
@@ -50,6 +50,46 @@ teardown:
       catch: missing
       ingest.get_pipeline:
         id: "my_pipeline"
+---
+"Test wildcard pipeline delete":
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body:  >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "set" : {
+                  "field" : "field2",
+                  "value": "_value"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      ingest.get_pipeline:
+        id: "my_pipeline"
+  - match: { my_pipeline.description: "_description" }
+
+  - do:
+      ingest.delete_pipeline:
+        id: "my_*"
+  - match: { acknowledged: true }
+
+  - do:
+      catch: missing
+      ingest.get_pipeline:
+        id: "my_pipeline"
+
+  - do:
+      catch: missing
+      ingest.delete_pipeline:
+        id: "my_*"
+  - match: { "error.type": "resource_not_found_exception" }
+  - match: { "error.reason": "pipeline [my_*] is missing" }
 
 ---
 "Test Get All Pipelines":
diff --git a/modules/lang-expression/build.gradle b/modules/lang-expression/build.gradle
index ee8c330193461..23286edd28aea 100644
--- a/modules/lang-expression/build.gradle
+++ b/modules/lang-expression/build.gradle
@@ -35,8 +35,6 @@ dependencyLicenses {
   mapping from: /asm-.*/, to: 'asm'
 }
 
-integTest {
-  cluster {
-    setting 'script.max_compilations_per_minute', '1000'
-  }
+integTestCluster {
+  setting 'script.max_compilations_per_minute', '1000'
 }
diff --git a/modules/lang-expression/licenses/lucene-expressions-6.2.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-6.2.0.jar.sha1
deleted file mode 100644
index 205aaae6e6616..0000000000000
--- a/modules/lang-expression/licenses/lucene-expressions-6.2.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-99764b20aba5443f8a181f7015a806443c589844
\ No newline at end of file
diff --git a/modules/lang-expression/licenses/lucene-expressions-6.6.1.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-6.6.1.jar.sha1
new file mode 100644
index 0000000000000..8c640ad305941
--- /dev/null
+++ b/modules/lang-expression/licenses/lucene-expressions-6.6.1.jar.sha1
@@ -0,0 +1 @@
+c2fd30a3a86c9b9ccb87c24d8a8afc841c2507cc
\ No newline at end of file
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionExecutableScript.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionExecutableScript.java
index 85a82ac79da49..f5d95f6c88a2b 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionExecutableScript.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionExecutableScript.java
@@ -19,25 +19,22 @@
 
 package org.elasticsearch.script.expression;
 
-import java.util.HashMap;
-import java.util.Map;
-
 import org.apache.lucene.expressions.Expression;
 import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.GeneralScriptException;
 
+import java.util.HashMap;
+import java.util.Map;
+
 /**
  * A bridge to evaluate an {@link Expression} against a map of variables in the context
  * of an {@link ExecutableScript}.
  */
 public class ExpressionExecutableScript implements ExecutableScript {
-
-    private final int NO_DOCUMENT = -1;
-
     public final CompiledScript compiledScript;
-    public final Map<String, ReplaceableConstFunctionValues> functionValuesMap;
-    public final ReplaceableConstFunctionValues[] functionValuesArray;
+    public final Map<String, ReplaceableConstDoubleValues> functionValuesMap;
+    public final ReplaceableConstDoubleValues[] functionValuesArray;
 
     public ExpressionExecutableScript(CompiledScript compiledScript, Map<String, Object> vars) {
         this.compiledScript = compiledScript;
@@ -51,12 +48,12 @@ public ExpressionExecutableScript(CompiledScript compiledScript, Map<String, Obj
                     " [" + vars.size() + "].");
         }
 
-        functionValuesArray = new ReplaceableConstFunctionValues[functionValuesLength];
+        functionValuesArray = new ReplaceableConstDoubleValues[functionValuesLength];
         functionValuesMap = new HashMap<>();
 
         for (int functionValuesIndex = 0; functionValuesIndex < functionValuesLength; ++functionValuesIndex) {
             String variableName = expression.variables[functionValuesIndex];
-            functionValuesArray[functionValuesIndex] = new ReplaceableConstFunctionValues();
+            functionValuesArray[functionValuesIndex] = new ReplaceableConstDoubleValues();
             functionValuesMap.put(variableName, functionValuesArray[functionValuesIndex]);
         }
 
@@ -85,7 +82,7 @@ public void setNextVar(String name, Object value) {
     @Override
     public Object run() {
         try {
-            return ((Expression) compiledScript.compiled()).evaluate(NO_DOCUMENT, functionValuesArray);
+            return ((Expression) compiledScript.compiled()).evaluate(functionValuesArray);
         } catch (Exception exception) {
             throw new GeneralScriptException("Error evaluating " + compiledScript, exception);
         }
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java
index 36ce55067f475..96decffeea619 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java
@@ -118,14 +118,15 @@ public SearchScript search(CompiledScript compiledScript, SearchLookup lookup, @
         // NOTE: if we need to do anything complicated with bindings in the future, we can just extend Bindings,
         // instead of complicating SimpleBindings (which should stay simple)
         SimpleBindings bindings = new SimpleBindings();
-        ReplaceableConstValueSource specialValue = null;
-
+        ReplaceableConstDoubleValueSource specialValue = null;
+        boolean needsScores = false;
         for (String variable : expr.variables) {
             try {
                 if (variable.equals("_score")) {
                     bindings.add(new SortField("_score", SortField.Type.SCORE));
+                    needsScores = true;
                 } else if (variable.equals("_value")) {
-                    specialValue = new ReplaceableConstValueSource();
+                    specialValue = new ReplaceableConstDoubleValueSource();
                     bindings.add("_value", specialValue);
                     // noop: _value is special for aggregations, and is handled in ExpressionScriptBindings
                     // TODO: if some uses it in a scoring expression, they will get a nasty failure when evaluating...need a
@@ -137,7 +138,7 @@ public SearchScript search(CompiledScript compiledScript, SearchLookup lookup, @
                     // but if we were to reverse it, we could provide a way to supply dynamic defaults for documents missing the field?
                     Object value = vars.get(variable);
                     if (value instanceof Number) {
-                        bindings.add(variable, new DoubleConstValueSource(((Number) value).doubleValue()));
+                        bindings.add(variable, new DoubleConstValueSource(((Number) value).doubleValue()).asDoubleValuesSource());
                     } else {
                         throw new ParseException("Parameter [" + variable + "] must be a numeric type", 0);
                     }
@@ -227,16 +228,14 @@ public SearchScript search(CompiledScript compiledScript, SearchLookup lookup, @
                     } else {
                         throw new ParseException("Field [" + fieldname + "] must be numeric, date, or geopoint", 5);
                     }
-
-                    bindings.add(variable, valueSource);
+                    needsScores |= valueSource.getSortField(false).needsScores();
+                    bindings.add(variable, valueSource.asDoubleValuesSource());
                 }
             } catch (Exception e) {
                 // we defer "binding" of variables until here: give context for that variable
                 throw convertToScriptException("link error", expr.sourceText, variable, e);
             }
         }
-
-        final boolean needsScores = expr.getSortField(bindings, false).needsScores();
         return new ExpressionSearchScript(compiledScript, bindings, specialValue, needsScores);
     }
 
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java
index af89450613c78..11a17d38c5993 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java
@@ -19,23 +19,21 @@
 
 package org.elasticsearch.script.expression;
 
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Map;
-
 import org.apache.lucene.expressions.Bindings;
 import org.apache.lucene.expressions.Expression;
 import org.apache.lucene.expressions.SimpleBindings;
 import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.queries.function.FunctionValues;
-import org.apache.lucene.queries.function.ValueSource;
+import org.apache.lucene.search.DoubleValues;
+import org.apache.lucene.search.DoubleValuesSource;
 import org.apache.lucene.search.Scorer;
-import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.script.CompiledScript;
-import org.elasticsearch.script.LeafSearchScript;
 import org.elasticsearch.script.GeneralScriptException;
+import org.elasticsearch.script.LeafSearchScript;
 import org.elasticsearch.script.SearchScript;
 
+import java.io.IOException;
+import java.util.Map;
+
 /**
  * A bridge to evaluate an {@link Expression} against {@link Bindings} in the context
  * of a {@link SearchScript}.
@@ -44,16 +42,16 @@ class ExpressionSearchScript implements SearchScript {
 
     final CompiledScript compiledScript;
     final SimpleBindings bindings;
-    final ValueSource source;
-    final ReplaceableConstValueSource specialValue; // _value
+    final DoubleValuesSource source;
+    final ReplaceableConstDoubleValueSource specialValue; // _value
     final boolean needsScores;
     Scorer scorer;
     int docid;
 
-    ExpressionSearchScript(CompiledScript c, SimpleBindings b, ReplaceableConstValueSource v, boolean needsScores) {
+    ExpressionSearchScript(CompiledScript c, SimpleBindings b, ReplaceableConstDoubleValueSource v, boolean needsScores) {
         compiledScript = c;
         bindings = b;
-        source = ((Expression)compiledScript.compiled()).getValueSource(bindings);
+        source = ((Expression)compiledScript.compiled()).getDoubleValuesSource(bindings);
         specialValue = v;
         this.needsScores = needsScores;
     }
@@ -63,15 +61,25 @@ public boolean needsScores() {
         return needsScores;
     }
 
+
     @Override
     public LeafSearchScript getLeafSearchScript(final LeafReaderContext leaf) throws IOException {
         return new LeafSearchScript() {
+            // Fake the scorer until setScorer is called.
+            DoubleValues values = source.getValues(leaf, new DoubleValues() {
+                @Override
+                public double doubleValue() throws IOException {
+                    return Double.NaN;
+                }
 
-            FunctionValues values = source.getValues(Collections.singletonMap("scorer", Lucene.illegalScorer("Scores are not available in the current context")), leaf);
-
+                @Override
+                public boolean advanceExact(int doc) throws IOException {
+                    return true;
+                }
+            });
             double evaluate() {
                 try {
-                    return values.doubleVal(docid);
+                    return values.doubleValue();
                 } catch (Exception exception) {
                     throw new GeneralScriptException("Error evaluating " + compiledScript, exception);
                 }
@@ -89,6 +97,11 @@ public LeafSearchScript getLeafSearchScript(final LeafReaderContext leaf) throws
             @Override
             public void setDocument(int d) {
                 docid = d;
+                try {
+                    values.advanceExact(d);
+                } catch (IOException e) {
+                    throw new IllegalStateException("Can't advance to doc using " + compiledScript, e);
+                }
             }
 
             @Override
@@ -96,7 +109,17 @@ public void setScorer(Scorer s) {
                 scorer = s;
                 try {
                     // We have a new binding for the scorer so we need to reset the values
-                    values = source.getValues(Collections.singletonMap("scorer", scorer), leaf);
+                    values = source.getValues(leaf, new DoubleValues() {
+                        @Override
+                        public double doubleValue() throws IOException {
+                            return scorer.score();
+                        }
+
+                        @Override
+                        public boolean advanceExact(int doc) throws IOException {
+                            return true;
+                        }
+                    });
                 } catch (IOException e) {
                     throw new IllegalStateException("Can't get values using " + compiledScript, e);
                 }
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstDoubleValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstDoubleValueSource.java
new file mode 100644
index 0000000000000..f8daa6ddf8825
--- /dev/null
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstDoubleValueSource.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.script.expression;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.DoubleValues;
+import org.apache.lucene.search.DoubleValuesSource;
+
+import java.io.IOException;
+
+/**
+ * A {@link DoubleValuesSource} which has a stub {@link DoubleValues} that holds a dynamically replaceable constant double.
+ */
+final class ReplaceableConstDoubleValueSource extends DoubleValuesSource {
+    final ReplaceableConstDoubleValues fv;
+
+    ReplaceableConstDoubleValueSource() {
+        fv = new ReplaceableConstDoubleValues();
+    }
+
+    @Override
+    public DoubleValues getValues(LeafReaderContext ctx, DoubleValues scores) throws IOException {
+        return fv;
+    }
+
+    @Override
+    public boolean needsScores() {
+        return false;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        return o == this;
+    }
+
+    @Override
+    public int hashCode() {
+        return System.identityHashCode(this);
+    }
+
+    public void setValue(double v) {
+        fv.setValue(v);
+    }
+}
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstDoubleValues.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstDoubleValues.java
new file mode 100644
index 0000000000000..557f9ad812e0c
--- /dev/null
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstDoubleValues.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.script.expression;
+
+import org.apache.lucene.search.DoubleValues;
+
+import java.io.IOException;
+
+/**
+ * A support class for an executable expression script that allows the double returned
+ * by a {@link DoubleValues} to be modified.
+ */
+final class ReplaceableConstDoubleValues extends DoubleValues {
+    private double value = 0;
+
+    void setValue(double value) {
+        this.value = value;
+    }
+
+    @Override
+    public double doubleValue() throws IOException {
+        return value;
+    }
+
+    @Override
+    public boolean advanceExact(int doc) throws IOException {
+        return true;
+    }
+}
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstFunctionValues.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstFunctionValues.java
deleted file mode 100644
index eab9131bf10f5..0000000000000
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstFunctionValues.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.script.expression;
-
-import org.apache.lucene.queries.function.FunctionValues;
-
-/**
- * A support class for an executable expression script that allows the double returned
- * by a {@link FunctionValues} to be modified.
- */
-final class ReplaceableConstFunctionValues extends FunctionValues {
-    private double value = 0;
-
-    void setValue(double value) {
-        this.value = value;
-    }
-
-    @Override
-    public double doubleVal(int doc) {
-        return value;
-    }
-
-    @Override
-    public String toString(int i) {
-        return "ReplaceableConstFunctionValues: " + value;
-    }
-}
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstValueSource.java
deleted file mode 100644
index ae0f862ca70f1..0000000000000
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstValueSource.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.script.expression;
-
-import java.io.IOException;
-import java.util.Map;
-
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.queries.function.FunctionValues;
-import org.apache.lucene.queries.function.ValueSource;
-
-/**
- * A {@link ValueSource} which has a stub {@link FunctionValues} that holds a dynamically replaceable constant double.
- */
-final class ReplaceableConstValueSource extends ValueSource {
-    final ReplaceableConstFunctionValues fv;
-
-    ReplaceableConstValueSource() {
-        fv = new ReplaceableConstFunctionValues();
-    }
-
-    @Override
-    @SuppressWarnings("rawtypes") // ValueSource uses a rawtype
-    public FunctionValues getValues(Map map, LeafReaderContext atomicReaderContext) throws IOException {
-        return fv;
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        return o == this;
-    }
-
-    @Override
-    public int hashCode() {
-        return System.identityHashCode(this);
-    }
-
-    @Override
-    public String description() {
-        return "replaceableConstDouble";
-    }
-
-    public void setValue(double v) {
-        fv.setValue(v);
-    }
-}
diff --git a/modules/lang-expression/src/main/plugin-metadata/plugin-security.policy b/modules/lang-expression/src/main/plugin-metadata/plugin-security.policy
index c11af51e46467..4835ab0530efc 100644
--- a/modules/lang-expression/src/main/plugin-metadata/plugin-security.policy
+++ b/modules/lang-expression/src/main/plugin-metadata/plugin-security.policy
@@ -24,7 +24,7 @@ grant {
   // expression runtime
   permission org.elasticsearch.script.ClassPermission "java.lang.String";
   permission org.elasticsearch.script.ClassPermission "org.apache.lucene.expressions.Expression";
-  permission org.elasticsearch.script.ClassPermission "org.apache.lucene.queries.function.FunctionValues";
+  permission org.elasticsearch.script.ClassPermission "org.apache.lucene.search.DoubleValues";
   // available functions
   permission org.elasticsearch.script.ClassPermission "java.lang.Math";
   permission org.elasticsearch.script.ClassPermission "org.apache.lucene.util.MathUtil";
diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTests.java
index 2b74aaa9f1ea3..b9c628926e3ac 100644
--- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTests.java
+++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTests.java
@@ -23,7 +23,7 @@
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.ScriptException;
-import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.lookup.SearchLookup;
 import org.elasticsearch.test.ESSingleNodeTestCase;
@@ -34,7 +34,7 @@
 public class ExpressionTests extends ESSingleNodeTestCase {
     ExpressionScriptEngineService service;
     SearchLookup lookup;
-    
+
     @Override
     public void setUp() throws Exception {
         super.setUp();
@@ -42,7 +42,7 @@ public void setUp() throws Exception {
         service = new ExpressionScriptEngineService(Settings.EMPTY);
         lookup = new SearchLookup(index.mapperService(), index.fieldData(), null);
     }
-    
+
     private SearchScript compile(String expression) {
         Object compiled = service.compile(null, expression, Collections.emptyMap());
         return service.search(new CompiledScript(ScriptType.INLINE, "randomName", "expression", compiled), lookup, Collections.<String, Object>emptyMap());
@@ -54,14 +54,14 @@ public void testNeedsScores() {
         assertTrue(compile("1/_score").needsScores());
         assertTrue(compile("doc['d'].value * _score").needsScores());
     }
-    
+
     public void testCompileError() {
         ScriptException e = expectThrows(ScriptException.class, () -> {
             compile("doc['d'].value * *@#)(@$*@#$ + 4");
         });
         assertTrue(e.getCause() instanceof ParseException);
     }
-    
+
     public void testLinkError() {
         ScriptException e = expectThrows(ScriptException.class, () -> {
             compile("doc['e'].value * 5");
diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/IndexedExpressionTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/IndexedExpressionTests.java
deleted file mode 100644
index 731e8760e800f..0000000000000
--- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/IndexedExpressionTests.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.script.expression;
-
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.script.ScriptService.ScriptType;
-import org.elasticsearch.search.aggregations.AggregationBuilders;
-import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.elasticsearch.test.ESIntegTestCase;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Collections;
-
-import static org.hamcrest.Matchers.containsString;
-
-//TODO: please convert to unit tests!
-public class IndexedExpressionTests extends ESIntegTestCase {
-    @Override
-    protected Settings nodeSettings(int nodeOrdinal) {
-        Settings.Builder builder = Settings.builder().put(super.nodeSettings(nodeOrdinal));
-        builder.put("script.engine.expression.stored.update", "false");
-        builder.put("script.engine.expression.stored.search", "false");
-        return builder.build();
-    }
-
-    @Override
-    protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Collections.singleton(ExpressionPlugin.class);
-    }
-
-    public void testAllOpsDisabledIndexedScripts() throws IOException {
-        client().admin().cluster().preparePutStoredScript()
-                .setScriptLang(ExpressionScriptEngineService.NAME)
-                .setId("script1")
-                .setSource(new BytesArray("{\"script\":\"2\"}"))
-                .get();
-        client().prepareIndex("test", "scriptTest", "1").setSource("{\"theField\":\"foo\"}").get();
-        try {
-            client().prepareUpdate("test", "scriptTest", "1")
-                    .setScript(new Script("script1", ScriptService.ScriptType.STORED, ExpressionScriptEngineService.NAME, null)).get();
-            fail("update script should have been rejected");
-        } catch(Exception e) {
-            assertThat(e.getMessage(), containsString("failed to execute script"));
-            assertThat(e.getCause().getMessage(), containsString("scripts of type [stored], operation [update] and lang [expression] are disabled"));
-        }
-        try {
-            client().prepareSearch()
-                    .setSource(
-                            new SearchSourceBuilder().scriptField("test1", new Script("script1", ScriptType.STORED, "expression", null)))
-                    .setIndices("test").setTypes("scriptTest").get();
-            fail("search script should have been rejected");
-        } catch(Exception e) {
-            assertThat(e.toString(), containsString("scripts of type [stored], operation [search] and lang [expression] are disabled"));
-        }
-        try {
-            client().prepareSearch("test")
-                    .setSource(
-                            new SearchSourceBuilder().aggregation(AggregationBuilders.terms("test").script(
-                                    new Script("script1", ScriptType.STORED, "expression", null)))).get();
-        } catch (Exception e) {
-            assertThat(e.toString(), containsString("scripts of type [stored], operation [aggs] and lang [expression] are disabled"));
-        }
-    }
-}
diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/LangExpressionClientYamlTestSuiteIT.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/LangExpressionClientYamlTestSuiteIT.java
index 3d1071ee17c85..295c7a7f86f25 100644
--- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/LangExpressionClientYamlTestSuiteIT.java
+++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/LangExpressionClientYamlTestSuiteIT.java
@@ -24,9 +24,6 @@
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-
-import java.io.IOException;
 
 public class LangExpressionClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
@@ -35,8 +32,8 @@ public LangExpressionClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
-        return ESClientYamlSuiteTestCase.createParameters(0, 1);
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
     }
 }
 
diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java
index d71d09f2f3775..79a936d20b99c 100644
--- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java
+++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java
@@ -36,7 +36,7 @@
 import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.GeneralScriptException;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
@@ -85,7 +85,7 @@ private SearchRequestBuilder buildRequest(String script, Object... params) {
         req.setQuery(QueryBuilders.matchAllQuery())
                 .addSort(SortBuilders.fieldSort("_uid")
                         .order(SortOrder.ASC))
-                .addScriptField("foo", new Script(script, ScriptType.INLINE, "expression", paramsMap));
+                .addScriptField("foo", new Script(ScriptType.INLINE, "expression", script, paramsMap));
         return req;
     }
 
@@ -124,7 +124,7 @@ public void testScore() throws Exception {
                 client().prepareIndex("test", "doc", "1").setSource("text", "hello goodbye"),
                 client().prepareIndex("test", "doc", "2").setSource("text", "hello hello hello goodbye"),
                 client().prepareIndex("test", "doc", "3").setSource("text", "hello hello goodebye"));
-        ScoreFunctionBuilder<?> score = ScoreFunctionBuilders.scriptFunction(new Script("1 / _score", ScriptType.INLINE, "expression", null));
+        ScoreFunctionBuilder<?> score = ScoreFunctionBuilders.scriptFunction(new Script(ScriptType.INLINE, "expression", "1 / _score", Collections.emptyMap()));
         SearchRequestBuilder req = client().prepareSearch().setIndices("test");
         req.setQuery(QueryBuilders.functionScoreQuery(QueryBuilders.termQuery("text", "hello"), score).boostMode(CombineFunction.REPLACE));
         req.setSearchType(SearchType.DFS_QUERY_THEN_FETCH); // make sure DF is consistent
@@ -164,7 +164,7 @@ public void testDateMethods() throws Exception {
         assertEquals(1985.0, hits.getAt(0).field("foo").getValue(), 0.0D);
         assertEquals(1983.0, hits.getAt(1).field("foo").getValue(), 0.0D);
     }
-    
+
     public void testDateObjectMethods() throws Exception {
         ElasticsearchAssertions.assertAcked(prepareCreate("test").addMapping("doc", "date0", "type=date", "date1", "type=date"));
         ensureGreen("test");
@@ -257,7 +257,7 @@ public void testMultiValueMethods() throws Exception {
         assertEquals(2.5, hits.getAt(0).field("foo").getValue(), 0.0D);
         assertEquals(5.0, hits.getAt(1).field("foo").getValue(), 0.0D);
         assertEquals(1.5, hits.getAt(2).field("foo").getValue(), 0.0D);
-        
+
         // make sure count() works for missing
         rsp = buildRequest("doc['double2'].count()").get();
         assertSearchResponse(rsp);
@@ -266,7 +266,7 @@ public void testMultiValueMethods() throws Exception {
         assertEquals(1.0, hits.getAt(0).field("foo").getValue(), 0.0D);
         assertEquals(0.0, hits.getAt(1).field("foo").getValue(), 0.0D);
         assertEquals(0.0, hits.getAt(2).field("foo").getValue(), 0.0D);
-        
+
         // make sure .empty works in the same way
         rsp = buildRequest("doc['double2'].empty ? 5.0 : 2.0").get();
         assertSearchResponse(rsp);
@@ -429,13 +429,16 @@ public void testSpecialValueVariable() throws Exception {
         req.setQuery(QueryBuilders.matchAllQuery())
                 .addAggregation(
                         AggregationBuilders.stats("int_agg").field("x")
-                                .script(new Script("_value * 3", ScriptType.INLINE, ExpressionScriptEngineService.NAME, null)))
+                                .script(new Script(ScriptType.INLINE,
+                                    ExpressionScriptEngineService.NAME, "_value * 3", Collections.emptyMap())))
                 .addAggregation(
                         AggregationBuilders.stats("double_agg").field("y")
-                                .script(new Script("_value - 1.1", ScriptType.INLINE, ExpressionScriptEngineService.NAME, null)))
+                                .script(new Script(ScriptType.INLINE,
+                                    ExpressionScriptEngineService.NAME, "_value - 1.1", Collections.emptyMap())))
                 .addAggregation(
                         AggregationBuilders.stats("const_agg").field("x") // specifically to test a script w/o _value
-                                .script(new Script("3.0", ScriptType.INLINE, ExpressionScriptEngineService.NAME, null))
+                                .script(new Script(ScriptType.INLINE,
+                                    ExpressionScriptEngineService.NAME, "3.0", Collections.emptyMap()))
                 );
 
         SearchResponse rsp = req.get();
@@ -469,7 +472,8 @@ public void testStringSpecialValueVariable() throws Exception {
         req.setQuery(QueryBuilders.matchAllQuery())
                 .addAggregation(
                         AggregationBuilders.terms("term_agg").field("text")
-                                .script(new Script("_value", ScriptType.INLINE, ExpressionScriptEngineService.NAME, null)));
+                                .script(
+                                    new Script(ScriptType.INLINE, ExpressionScriptEngineService.NAME, "_value", Collections.emptyMap())));
 
         String message;
         try {
@@ -559,7 +563,7 @@ public void testInvalidUpdateScript() throws Exception {
             UpdateRequestBuilder urb = client().prepareUpdate().setIndex("test_index");
             urb.setType("doc");
             urb.setId("1");
-            urb.setScript(new Script("0", ScriptType.INLINE, ExpressionScriptEngineService.NAME, null));
+            urb.setScript(new Script(ScriptType.INLINE, ExpressionScriptEngineService.NAME, "0", Collections.emptyMap()));
             urb.get();
             fail("Expression scripts should not be allowed to run as update scripts.");
         } catch (Exception e) {
@@ -590,14 +594,15 @@ public void testPipelineAggregationScript() throws Exception {
                                 .subAggregation(sum("threeSum").field("three"))
                                 .subAggregation(sum("fourSum").field("four"))
                                 .subAggregation(bucketScript("totalSum",
-                                    new Script("_value0 + _value1 + _value2", ScriptType.INLINE, ExpressionScriptEngineService.NAME, null),
+                                    new Script(ScriptType.INLINE,
+                                        ExpressionScriptEngineService.NAME, "_value0 + _value1 + _value2", Collections.emptyMap()),
                                     "twoSum", "threeSum", "fourSum")))
                 .execute().actionGet();
 
         Histogram histogram = response.getAggregations().get("histogram");
         assertThat(histogram, notNullValue());
         assertThat(histogram.getName(), equalTo("histogram"));
-        List<Histogram.Bucket> buckets = histogram.getBuckets();
+        List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
 
         for (int bucketCount = 0; bucketCount < buckets.size(); ++bucketCount) {
             Histogram.Bucket bucket = buckets.get(bucketCount);
@@ -616,7 +621,7 @@ public void testPipelineAggregationScript() throws Exception {
             }
         }
     }
-    
+
     public void testGeo() throws Exception {
         XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type1")
                 .startObject("properties").startObject("location").field("type", "geo_point");
@@ -649,7 +654,7 @@ public void testGeo() throws Exception {
         assertEquals(1, rsp.getHits().getTotalHits());
         assertEquals(3170D, rsp.getHits().getAt(0).field("foo").getValue(), 50D);
     }
-    
+
     public void testBoolean() throws Exception {
         XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type1")
                 .startObject("properties").startObject("vip").field("type", "boolean");
diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/StoredExpressionTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/StoredExpressionTests.java
new file mode 100644
index 0000000000000..697dbf625752b
--- /dev/null
+++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/StoredExpressionTests.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.script.expression;
+
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptType;
+import org.elasticsearch.search.aggregations.AggregationBuilders;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.test.ESIntegTestCase;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+
+import static org.hamcrest.Matchers.containsString;
+
+//TODO: please convert to unit tests!
+public class StoredExpressionTests extends ESIntegTestCase {
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        Settings.Builder builder = Settings.builder().put(super.nodeSettings(nodeOrdinal));
+        builder.put("script.engine.expression.stored.update", "false");
+        builder.put("script.engine.expression.stored.search", "false");
+        return builder.build();
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Collections.singleton(ExpressionPlugin.class);
+    }
+
+    public void testAllOpsDisabledIndexedScripts() throws IOException {
+        client().admin().cluster().preparePutStoredScript()
+                .setLang(ExpressionScriptEngineService.NAME)
+                .setId("script1")
+                .setContent(new BytesArray("{\"script\":\"2\"}"), XContentType.JSON)
+                .get();
+        client().prepareIndex("test", "scriptTest", "1").setSource("{\"theField\":\"foo\"}", XContentType.JSON).get();
+        try {
+            client().prepareUpdate("test", "scriptTest", "1")
+                    .setScript(new Script(ScriptType.STORED, ExpressionScriptEngineService.NAME, "script1", Collections.emptyMap())).get();
+            fail("update script should have been rejected");
+        } catch(Exception e) {
+            assertThat(e.getMessage(), containsString("failed to execute script"));
+            assertThat(e.getCause().getMessage(), containsString("scripts of type [stored], operation [update] and lang [expression] are not supported"));
+        }
+        try {
+            client().prepareSearch()
+                    .setSource(
+                            new SearchSourceBuilder().scriptField("test1", new Script(ScriptType.STORED, "expression", "script1", Collections.emptyMap())))
+                    .setIndices("test").setTypes("scriptTest").get();
+            fail("search script should have been rejected");
+        } catch(Exception e) {
+            assertThat(e.toString(), containsString("scripts of type [stored], operation [search] and lang [expression] are disabled"));
+        }
+        try {
+            client().prepareSearch("test")
+                    .setSource(
+                            new SearchSourceBuilder().aggregation(AggregationBuilders.terms("test").script(
+                                    new Script(ScriptType.STORED, "expression", "script1", Collections.emptyMap())))).get();
+        } catch (Exception e) {
+            assertThat(e.toString(), containsString("scripts of type [stored], operation [aggs] and lang [expression] are disabled"));
+        }
+    }
+}
diff --git a/modules/lang-expression/src/test/resources/rest-api-spec/test/lang_expression/20_search.yaml b/modules/lang-expression/src/test/resources/rest-api-spec/test/lang_expression/20_search.yaml
index 36ff7f58ea69b..3f26d078bd0ba 100644
--- a/modules/lang-expression/src/test/resources/rest-api-spec/test/lang_expression/20_search.yaml
+++ b/modules/lang-expression/src/test/resources/rest-api-spec/test/lang_expression/20_search.yaml
@@ -22,6 +22,6 @@ setup:
 ---
 "Expressions scripting test":
 
-  - do: { search: { body: { script_fields : { my_field : { script: { lang: expression, inline: 'doc["age"].value + 19' } } } } } }
+  - do: { search: { body: { script_fields : { my_field : { script: { lang: expression, source: 'doc["age"].value + 19' } } } } } }
   - match:  { hits.hits.0.fields.my_field.0: 42.0 }
 
diff --git a/modules/lang-groovy/build.gradle b/modules/lang-groovy/build.gradle
index 77e28da7f84d1..26e7d14bddca5 100644
--- a/modules/lang-groovy/build.gradle
+++ b/modules/lang-groovy/build.gradle
@@ -26,12 +26,10 @@ dependencies {
   compile 'org.codehaus.groovy:groovy:2.4.6:indy'
 }
 
-integTest {
-  cluster {
-    setting 'script.inline', 'true'
-    setting 'script.stored', 'true'
-    setting 'script.max_compilations_per_minute', '1000'
-  }
+integTestCluster {
+  setting 'script.inline', 'true'
+  setting 'script.stored', 'true'
+  setting 'script.max_compilations_per_minute', '1000'
 }
 
 thirdPartyAudit.excludes = [
diff --git a/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java b/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java
index 7f390ade0881a..52d9d6052b5f9 100644
--- a/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java
+++ b/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java
@@ -28,9 +28,9 @@
 import org.apache.logging.log4j.util.Supplier;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.Scorer;
+import org.codehaus.groovy.GroovyBugError;
 import org.codehaus.groovy.ast.ClassCodeExpressionTransformer;
 import org.codehaus.groovy.ast.ClassNode;
-import org.codehaus.groovy.ast.Parameter;
 import org.codehaus.groovy.ast.expr.ConstantExpression;
 import org.codehaus.groovy.ast.expr.Expression;
 import org.codehaus.groovy.classgen.GeneratorContext;
@@ -68,6 +68,7 @@
 import java.security.AccessController;
 import java.security.PrivilegedAction;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -97,8 +98,6 @@ public class GroovyScriptEngineService extends AbstractComponent implements Scri
     public GroovyScriptEngineService(Settings settings) {
         super(settings);
 
-        deprecationLogger.deprecated("[groovy] scripts are deprecated, use [painless] scripts instead");
-
         // Creates the classloader here in order to isolate Groovy-land code
         final SecurityManager sm = System.getSecurityManager();
         if (sm != null) {
@@ -303,20 +302,24 @@ public Object run() {
                 // NOTE: we truncate the stack because IndyInterface has security issue (needs getClassLoader)
                 // we don't do a security check just as a tradeoff, it cannot really escalate to anything.
                 return AccessController.doPrivileged((PrivilegedAction<Object>) script::run);
-            } catch (AssertionError ae) {
+            } catch (final AssertionError ae) {
+                if (ae instanceof GroovyBugError) {
+                    // we encountered a bug in Groovy; we wrap this so it does not go to the uncaught exception handler and tear us down
+                    final String message =  "encountered bug in Groovy while executing script [" + compiledScript.name() + "]";
+                    throw new ScriptException(message, ae, Collections.emptyList(), compiledScript.toString(), compiledScript.lang());
+                }
                 // Groovy asserts are not java asserts, and cannot be disabled, so we do a best-effort trying to determine if this is a
                 // Groovy assert (in which case we wrap it and throw), or a real Java assert, in which case we rethrow it as-is, likely
                 // resulting in the uncaughtExceptionHandler handling it.
                 final StackTraceElement[] elements = ae.getStackTrace();
                 if (elements.length > 0 && "org.codehaus.groovy.runtime.InvokerHelper".equals(elements[0].getClassName())) {
-                    logger.trace((Supplier<?>) () -> new ParameterizedMessage("failed to run {}", compiledScript), ae);
-                    throw new ScriptException("Error evaluating " + compiledScript.name(),
-                            ae, emptyList(), "", compiledScript.lang());
+                    logger.debug((Supplier<?>) () -> new ParameterizedMessage("failed to run {}", compiledScript), ae);
+                    throw new ScriptException("error evaluating " + compiledScript.name(), ae, emptyList(), "", compiledScript.lang());
                 }
                 throw ae;
             } catch (Exception | NoClassDefFoundError e) {
-                logger.trace("failed to run {}", e, compiledScript);
-                throw new ScriptException("Error evaluating " + compiledScript.name(), e, emptyList(), "", compiledScript.lang());
+                logger.trace((Supplier<?>) () -> new ParameterizedMessage("failed to run {}", compiledScript), e);
+                throw new ScriptException("error evaluating " + compiledScript.name(), e, emptyList(), "", compiledScript.lang());
             }
         }
 
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyIndexedScriptTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyIndexedScriptTests.java
index b0d5fd3366baa..cbf0fceb94d28 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyIndexedScriptTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyIndexedScriptTests.java
@@ -28,10 +28,7 @@
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.script.ScriptService.ScriptType;
-import org.elasticsearch.script.groovy.GroovyPlugin;
-import org.elasticsearch.script.groovy.GroovyScriptEngineService;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
@@ -70,13 +67,13 @@ protected Settings nodeSettings(int nodeOrdinal) {
     public void testFieldIndexedScript()  throws ExecutionException, InterruptedException {
         client().admin().cluster().preparePutStoredScript()
                 .setId("script1")
-                .setScriptLang(GroovyScriptEngineService.NAME)
-                .setSource(new BytesArray("{ \"script\" : \"2\"}"))
+                .setLang(GroovyScriptEngineService.NAME)
+                .setContent(new BytesArray("{ \"script\" : \"2\"}"))
                 .get();
         client().admin().cluster().preparePutStoredScript()
                 .setId("script2")
-                .setScriptLang(GroovyScriptEngineService.NAME)
-                .setSource(new BytesArray("{ \"script\" : \"factor * 2\"}"))
+                .setLang(GroovyScriptEngineService.NAME)
+                .setContent(new BytesArray("{ \"script\" : \"factor * 2\"}"))
                 .get();
 
         List<IndexRequestBuilder> builders = new ArrayList<>();
@@ -93,9 +90,10 @@ public void testFieldIndexedScript()  throws ExecutionException, InterruptedExce
                 .prepareSearch()
                 .setSource(
                         new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()).size(1)
-                                .scriptField("test1", new Script("script1", ScriptType.STORED, GroovyScriptEngineService.NAME, null))
+                                .scriptField("test1",
+                                    new Script(ScriptType.STORED, GroovyScriptEngineService.NAME, "script1", Collections.emptyMap()))
                                 .scriptField("test2",
-                                    new Script("script2", ScriptType.STORED, GroovyScriptEngineService.NAME, script2Params)))
+                                    new Script(ScriptType.STORED, GroovyScriptEngineService.NAME, "script2", script2Params)))
                 .setIndices("test").setTypes("scriptTest").get();
         assertHitCount(searchResponse, 5);
         assertTrue(searchResponse.getHits().hits().length == 1);
@@ -114,14 +112,14 @@ public void testUpdateScripts() {
         int iterations = randomIntBetween(2, 11);
         for (int i = 1; i < iterations; i++) {
             assertAcked(client().admin().cluster().preparePutStoredScript()
-                    .setScriptLang(GroovyScriptEngineService.NAME)
+                    .setLang(GroovyScriptEngineService.NAME)
                     .setId("script1")
-                    .setSource(new BytesArray("{\"script\":\"" + i + "\"}")));
+                    .setContent(new BytesArray("{\"script\":\"" + i + "\"}")));
             SearchResponse searchResponse = client()
                     .prepareSearch()
                     .setSource(
                             new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()).scriptField("test_field",
-                                    new Script("script1", ScriptType.STORED, GroovyScriptEngineService.NAME, null)))
+                                    new Script(ScriptType.STORED, GroovyScriptEngineService.NAME, "script1", Collections.emptyMap())))
                                             .setIndices("test_index")
                     .setTypes("test_type").get();
             assertHitCount(searchResponse, 1);
@@ -132,13 +130,13 @@ public void testUpdateScripts() {
 
     public void testDisabledUpdateIndexedScriptsOnly() {
         assertAcked(client().admin().cluster().preparePutStoredScript()
-                .setScriptLang(GroovyScriptEngineService.NAME)
+                .setLang(GroovyScriptEngineService.NAME)
                 .setId("script1")
-                .setSource(new BytesArray("{\"script\":\"2\"}")));
+                .setContent(new BytesArray("{\"script\":\"2\"}")));
         client().prepareIndex("test", "scriptTest", "1").setSource("{\"theField\":\"foo\"}").get();
         try {
             client().prepareUpdate("test", "scriptTest", "1")
-                    .setScript(new Script("script1", ScriptService.ScriptType.STORED, GroovyScriptEngineService.NAME, null)).get();
+                    .setScript(new Script(ScriptType.STORED, GroovyScriptEngineService.NAME, "script1", Collections.emptyMap())).get();
             fail("update script should have been rejected");
         } catch (Exception e) {
             assertThat(e.getMessage(), containsString("failed to execute script"));
@@ -150,16 +148,16 @@ public void testDisabledUpdateIndexedScriptsOnly() {
     public void testDisabledAggsDynamicScripts() {
         //dynamic scripts don't need to be enabled for an indexed script to be indexed and later on executed
         assertAcked(client().admin().cluster().preparePutStoredScript()
-                .setScriptLang(GroovyScriptEngineService.NAME)
+                .setLang(GroovyScriptEngineService.NAME)
                 .setId("script1")
-                .setSource(new BytesArray("{\"script\":\"2\"}")));
+                .setContent(new BytesArray("{\"script\":\"2\"}")));
         client().prepareIndex("test", "scriptTest", "1").setSource("{\"theField\":\"foo\"}").get();
         refresh();
         SearchResponse searchResponse = client()
                 .prepareSearch("test")
                 .setSource(
                         new SearchSourceBuilder().aggregation(AggregationBuilders.terms("test").script(
-                                new Script("script1", ScriptType.STORED, GroovyScriptEngineService.NAME, null)))).get();
+                                new Script(ScriptType.STORED, GroovyScriptEngineService.NAME, "script1", Collections.emptyMap())))).get();
         assertHitCount(searchResponse, 1);
         assertThat(searchResponse.getAggregations().get("test"), notNullValue());
     }
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyScriptTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyScriptTests.java
index 88d9b7be1de7f..196d878ae6636 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyScriptTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyScriptTests.java
@@ -26,7 +26,7 @@
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.sort.ScriptSortBuilder.ScriptSortType;
 import org.elasticsearch.search.sort.SortBuilders;
@@ -68,7 +68,7 @@ public void testGroovyBigDecimalTransformation() {
     }
 
     public void assertScript(String scriptString) {
-        Script script = new Script(scriptString, ScriptType.INLINE, GroovyScriptEngineService.NAME, null);
+        Script script = new Script(ScriptType.INLINE, GroovyScriptEngineService.NAME, scriptString, Collections.emptyMap());
         SearchResponse resp = client().prepareSearch("test")
                 .setSource(new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()).sort(SortBuilders.
                         scriptSort(script, ScriptSortType.NUMBER)))
@@ -85,8 +85,8 @@ public void testGroovyExceptionSerialization() throws Exception {
         try {
             client().prepareSearch("test")
                     .setQuery(
-                            constantScoreQuery(scriptQuery(new Script("1 == not_found", ScriptType.INLINE, GroovyScriptEngineService.NAME,
-                                    null)))).get();
+                            constantScoreQuery(scriptQuery(new Script(ScriptType.INLINE, GroovyScriptEngineService.NAME, "1 == not_found",
+                                Collections.emptyMap())))).get();
             fail("should have thrown an exception");
         } catch (SearchPhaseExecutionException e) {
             assertThat(e.toString()+ "should not contained NotSerializableTransportException",
@@ -100,7 +100,7 @@ public void testGroovyExceptionSerialization() throws Exception {
         try {
             client().prepareSearch("test")
                     .setQuery(constantScoreQuery(scriptQuery(
-                            new Script("null.foo", ScriptType.INLINE, GroovyScriptEngineService.NAME, null)))).get();
+                            new Script(ScriptType.INLINE, GroovyScriptEngineService.NAME, "null.foo", Collections.emptyMap())))).get();
             fail("should have thrown an exception");
         } catch (SearchPhaseExecutionException e) {
             assertThat(e.toString() + "should not contained NotSerializableTransportException",
@@ -120,7 +120,7 @@ public void testGroovyScriptAccess() {
 
         // doc[] access
         SearchResponse resp = client().prepareSearch("test").setQuery(functionScoreQuery(scriptFunction(
-                new Script("doc['bar'].value", ScriptType.INLINE, GroovyScriptEngineService.NAME, null)))
+                new Script(ScriptType.INLINE, GroovyScriptEngineService.NAME, "doc['bar'].value", Collections.emptyMap())))
                         .boostMode(CombineFunction.REPLACE)).get();
 
         assertNoFailures(resp);
@@ -135,7 +135,7 @@ public void testScoreAccess() {
 
         // _score can be accessed
         SearchResponse resp = client().prepareSearch("test").setQuery(functionScoreQuery(matchQuery("foo", "dog"),
-                scriptFunction(new Script("_score", ScriptType.INLINE, GroovyScriptEngineService.NAME, null)))
+                scriptFunction(new Script(ScriptType.INLINE, GroovyScriptEngineService.NAME, "_score", Collections.emptyMap())))
             .boostMode(CombineFunction.REPLACE)).get();
         assertNoFailures(resp);
         assertSearchHits(resp, "3", "1");
@@ -147,7 +147,8 @@ public void testScoreAccess() {
                 .prepareSearch("test")
                 .setQuery(
                         functionScoreQuery(matchQuery("foo", "dog"), scriptFunction(
-                                new Script("_score > 0.0 ? _score : 0", ScriptType.INLINE, GroovyScriptEngineService.NAME, null)))
+                                new Script(ScriptType.INLINE,
+                                    GroovyScriptEngineService.NAME, "_score > 0.0 ? _score : 0", Collections.emptyMap())))
                                         .boostMode(CombineFunction.REPLACE)).get();
         assertNoFailures(resp);
         assertSearchHits(resp, "3", "1");
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovySecurityTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovySecurityTests.java
index 31dc154a9e23a..6fa1b786d2a86 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovySecurityTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovySecurityTests.java
@@ -25,9 +25,10 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.ScriptException;
-import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.test.ESTestCase;
 
+import java.io.IOException;
 import java.nio.file.Path;
 import java.security.PrivilegedActionException;
 import java.util.AbstractMap;
@@ -121,6 +122,7 @@ public void testEvilGroovyScripts() throws Exception {
         if (!Constants.WINDOWS) {
             assertFailure("new File(\"" + dir + "\").exists()", MultipleCompilationErrorsException.class);
         }
+        assertWarnings("[groovy] scripts are deprecated, use [painless] scripts instead");
     }
 
     public void testGroovyScriptsThatThrowErrors() throws Exception {
@@ -128,6 +130,16 @@ public void testGroovyScriptsThatThrowErrors() throws Exception {
         assertFailure("def foo=false; assert foo;", AssertionError.class);
         // Groovy's asserts require org.codehaus.groovy.runtime.InvokerHelper, so they are denied
         assertFailure("def foo=false; assert foo, \"msg2\";", NoClassDefFoundError.class);
+        assertWarnings("[groovy] scripts are deprecated, use [painless] scripts instead");
+    }
+
+    public void testGroovyBugError() throws IOException {
+        // this script throws a GroovyBugError because our security manager permissions prevent Groovy from accessing this private field
+        // and Groovy does not handle it gracefully; this test will likely start failing if the bug is fixed upstream so that a
+        // GroovyBugError no longer surfaces here in which case the script should be replaced with another script that intentionally
+        // surfaces a GroovyBugError
+        assertFailure("[1, 2].size", AssertionError.class);
+        assertWarnings("[groovy] scripts are deprecated, use [painless] scripts instead");
     }
 
     /** runs a script */
@@ -139,7 +151,7 @@ private void doTest(String script) {
         vars.put("myarray", Arrays.asList("foo"));
         vars.put("myobject", new MyObject());
 
-        se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "test", "js", se.compile(null, script, Collections.emptyMap())), vars).run();
+        se.executable(new CompiledScript(ScriptType.INLINE, "test", "js", se.compile(null, script, Collections.emptyMap())), vars).run();
     }
 
     public static class MyObject {
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/LangGroovyClientYamlTestSuiteIT.java b/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/LangGroovyClientYamlTestSuiteIT.java
index c8e9c74827a58..04862602be198 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/LangGroovyClientYamlTestSuiteIT.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/LangGroovyClientYamlTestSuiteIT.java
@@ -24,9 +24,6 @@
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-
-import java.io.IOException;
 
 public class LangGroovyClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
@@ -35,8 +32,8 @@ public LangGroovyClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate tes
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
-        return ESClientYamlSuiteTestCase.createParameters(0, 1);
+    public static Iterable<Object[]> parameters() throws Exception {
+        return createParameters();
     }
 }
 
diff --git a/modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/15_update.yaml b/modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/15_update.yaml
index 683b2c6a2df7e..a4834baaafe91 100644
--- a/modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/15_update.yaml
+++ b/modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/15_update.yaml
@@ -2,7 +2,9 @@
 "Update Script":
 
   - skip:
-      features: groovy_scripting
+      features:
+        - groovy_scripting
+        - warnings
 
   - do:
       index:
@@ -14,6 +16,8 @@
               count:  1
 
   - do:
+      warnings:
+        - '[groovy] scripts are deprecated, use [painless] scripts instead'
       update:
           index:  test_1
           type:   test
@@ -21,7 +25,7 @@
           body:
             script:
               lang:   groovy
-              inline: "ctx._source.foo = bar"
+              source: "ctx._source.foo = bar"
               params: { bar: 'xxx' }
 
   - match: { _index:   test_1 }
@@ -39,6 +43,8 @@
   - match: { _source.count:      1   }
 
   - do:
+      warnings:
+        - '[groovy] scripts are deprecated, use [painless] scripts instead'
       update:
           index:  test_1
           type:   test
@@ -46,7 +52,7 @@
           body:
             script:
               lang:   groovy
-              inline: "ctx._source.foo = 'yyy'"
+              source: "ctx._source.foo = 'yyy'"
 
   - match: { _index:   test_1 }
   - match: { _type:    test   }
@@ -70,7 +76,7 @@
           id:     1
           body:
             script:
-              inline: "1"
+              source: "1"
               lang:   "doesnotexist"
               params: { bar: 'xxx' }
 
@@ -83,5 +89,4 @@
           body:
             script:
                 lang:   doesnotexist
-                inline: "1"
-
+                source: "1"
diff --git a/modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/16_update2.yaml b/modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/16_update2.yaml
index 5849874c02e43..a96ab7f7dbb50 100644
--- a/modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/16_update2.yaml
+++ b/modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/16_update2.yaml
@@ -1,73 +1,58 @@
 ---
-"Indexed script":
+"Stored script":
 
   - skip:
       features: groovy_scripting
 
   - do:
       put_script:
-        id: "1"
-        lang: "groovy"
-        body: { "script":  "_score * doc[\"myParent.weight\"].value" }
+        lang: "1"
+        body: { "script": {"lang": "groovy", "source": "_score * doc[\"myParent.weight\"].value" } }
   - match: { acknowledged: true }
 
   - do:
      get_script:
-       id: "1"
-       lang: "groovy"
+       lang: "1"
   - match: { found: true }
-  - match: { lang: groovy }
   - match: { _id: "1" }
-  - match: { "script":  "_score * doc[\"myParent.weight\"].value" }
+  - match: { "script.lang":  "groovy" }
+  - match: { "script.source":  "_score * doc[\"myParent.weight\"].value" }
 
   - do:
      catch: missing
      get_script:
-       id: "2"
-       lang: "groovy"
+       lang: "2"
   - match: { found: false }
-  - match: { lang: groovy }
   - match: { _id: "2" }
   - is_false: script
 
   - do:
      delete_script:
-       id: "1"
-       lang: "groovy"
+       lang: "1"
   - match: { acknowledged: true }
 
   - do:
      catch: missing
      delete_script:
-       id: "non_existing"
-       lang: "groovy"
+       lang: "non_existing"
 
   - do:
       catch: request
       put_script:
-        id: "1"
-        lang: "groovy"
-        body: { "script":  "_score * foo bar + doc[\"myParent.weight\"].value" }
+        lang: "1"
+        body: { "script": {"lang": "groovy", "source": "_score * foo bar + doc[\"myParent.weight\"].value" } }
 
 
   - do:
       catch: /script_exception,.+Error.compiling.script.*/
       put_script:
-        id: "1"
-        lang: "groovy"
-        body: { "script":  "_score * foo bar + doc[\"myParent.weight\"].value" }
+        lang: "1"
+        body: { "script":  {"lang": "groovy", "source": "_score * foo bar + doc[\"myParent.weight\"].value" } }
 
   - do:
       catch: request
       put_script:
-        id: "1"
-        lang: "foobar"
-        body: { "script" :  "_score * doc[\"myParent.weight\"].value" }
+        lang: "1"
+        body: { "script" : {"lang": "foobar", "source": "_score * doc[\"myParent.weight\"].value" } }
 
-  - do:
-      catch: /script_lang.not.supported/
-      put_script:
-        id: "1"
-        lang: "foobar"
-        body: { "script" :  "_score * doc[\"myParent.weight\"].value" }
 
diff --git a/modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/25_script_upsert.yaml b/modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/25_script_upsert.yaml
index 6ba4d39e71b2d..ef5ef74141f38 100644
--- a/modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/25_script_upsert.yaml
+++ b/modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/25_script_upsert.yaml
@@ -2,7 +2,9 @@
 "Script upsert":
 
   - skip:
-      features: groovy_scripting
+      features:
+        - groovy_scripting
+        - warnings
 
   - do:
       update:
@@ -11,7 +13,7 @@
           id:       1
           body:
             script:
-              inline: "ctx._source.foo = bar"
+              source: "ctx._source.foo = bar"
               params: { bar: 'xxx' }
               lang: "groovy"
             upsert: { foo: baz }
@@ -26,13 +28,15 @@
 
 
   - do:
+      warnings:
+        - '[groovy] scripts are deprecated, use [painless] scripts instead'
       update:
           index:    test_1
           type:     test
           id:       1
           body:
             script:
-              inline: "ctx._source.foo = bar"
+              source: "ctx._source.foo = bar"
               params: { bar: 'xxx' }
               lang: "groovy"
             upsert: { foo: baz }
@@ -46,13 +50,15 @@
   - match:  { _source.foo: xxx }
 
   - do:
+      warnings:
+        - '[groovy] scripts are deprecated, use [painless] scripts instead'
       update:
           index:    test_1
           type:     test
           id:       2
           body:
             script:
-              inline: "ctx._source.foo = bar"
+              source: "ctx._source.foo = bar"
               params: { bar: 'xxx' }
               lang: "groovy"
             upsert: { foo: baz }
@@ -65,5 +71,3 @@
           id:     2
 
   - match:  { _source.foo: xxx }
-
-
diff --git a/modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/30_compile_limit.yaml b/modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/30_compile_limit.yaml
index f23cbe22f3bb3..950069b629132 100644
--- a/modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/30_compile_limit.yaml
+++ b/modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/30_compile_limit.yaml
@@ -10,24 +10,25 @@ setup:
           body:   { foo: bar }
           refresh: wait_for
 
----
-teardown:
-  - do:
-      cluster.put_settings:
-        body:
-          transient:
-            script.max_compilations_per_minute: null
-
 ---
 "circuit breaking with too many scripts":
 
+  - skip:
+      features:
+        - groovy_scripting
+        - warnings
+
   - do:
+      warnings:
+        - "[script.max_compilations_per_minute] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version."
       cluster.put_settings:
         body:
           transient:
             script.max_compilations_per_minute: 1
 
   - do:
+      warnings:
+        - '[groovy] scripts are deprecated, use [painless] scripts instead'
       search:
         index: test
         type:  test
@@ -36,7 +37,7 @@ teardown:
           script_fields:
             myfield:
               script:
-                inline: "\"aoeu\""
+                source: "\"aoeu\""
                 lang: groovy
 
   - match: { hits.total: 1 }
@@ -51,9 +52,18 @@ teardown:
           script_fields:
             myfield:
               script:
-                inline: "\"aoeuaoeu\""
+                source: "\"aoeuaoeu\""
                 lang: groovy
 
+  - do:
+      warnings:
+        - "[script.max_compilations_per_minute] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version."
+      cluster.put_settings:
+        body:
+          transient:
+            script.max_compilations_per_minute: null
+
+
 ---
 "no bad settings":
 
diff --git a/modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/90_missing.yaml b/modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/90_missing.yaml
index 999d9f610ffb0..4cadee311a705 100644
--- a/modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/90_missing.yaml
+++ b/modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/90_missing.yaml
@@ -32,7 +32,7 @@
           id:     1
           body:
             script:
-              inline: "ctx._source.foo = bar"
+              source: "ctx._source.foo = bar"
               params: { bar: 'xxx' }
               lang: "groovy"
 
@@ -44,5 +44,5 @@
           ignore: 404
           body:
             script:
-              inline:       "ctx._source.foo = bar"
+              source:       "ctx._source.foo = bar"
               params:       { bar: 'xxx' }
diff --git a/modules/lang-mustache/build.gradle b/modules/lang-mustache/build.gradle
index 8fed78aca3249..8b695fbf574bc 100644
--- a/modules/lang-mustache/build.gradle
+++ b/modules/lang-mustache/build.gradle
@@ -27,11 +27,9 @@ dependencies {
   compile "com.github.spullara.mustache.java:compiler:0.9.3"
 }
 
-integTest {
-  cluster {
-    setting 'script.inline', 'true'
-    setting 'script.stored', 'true'
-    setting 'script.max_compilations_per_minute', '1000'
-    setting 'path.scripts', "${project.buildDir}/resources/test/templates"
-  }
+integTestCluster {
+  setting 'script.inline', 'true'
+  setting 'script.stored', 'true'
+  setting 'script.max_compilations_per_minute', '1000'
+  setting 'path.scripts', "${project.buildDir}/resources/test/templates"
 }
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomMustacheFactory.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomMustacheFactory.java
index 8419730dc1c45..799d378e05fc7 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomMustacheFactory.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomMustacheFactory.java
@@ -27,6 +27,7 @@
 import com.github.mustachejava.MustacheException;
 import com.github.mustachejava.MustacheVisitor;
 import com.github.mustachejava.TemplateContext;
+import com.github.mustachejava.codes.DefaultMustache;
 import com.github.mustachejava.codes.IterableCode;
 import com.github.mustachejava.codes.WriteCode;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -35,33 +36,65 @@
 import java.io.IOException;
 import java.io.StringWriter;
 import java.io.Writer;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.StringJoiner;
-import java.util.function.BiConsumer;
 import java.util.function.Function;
+import java.util.function.Supplier;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 public class CustomMustacheFactory extends DefaultMustacheFactory {
 
-    private final BiConsumer<String, Writer> encoder;
+    static final String JSON_MIME_TYPE_WITH_CHARSET = "application/json; charset=UTF-8";
+    static final String JSON_MIME_TYPE = "application/json";
+    static final String PLAIN_TEXT_MIME_TYPE = "text/plain";
+    static final String X_WWW_FORM_URLENCODED_MIME_TYPE = "application/x-www-form-urlencoded";
+
+    private static final String DEFAULT_MIME_TYPE = JSON_MIME_TYPE;
+
+    private static final Map<String, Supplier<Encoder>> ENCODERS;
+    static {
+        Map<String, Supplier<Encoder>> encoders = new HashMap<>();
+        encoders.put(JSON_MIME_TYPE_WITH_CHARSET, JsonEscapeEncoder::new);
+        encoders.put(JSON_MIME_TYPE, JsonEscapeEncoder::new);
+        encoders.put(PLAIN_TEXT_MIME_TYPE, DefaultEncoder::new);
+        encoders.put(X_WWW_FORM_URLENCODED_MIME_TYPE, UrlEncoder::new);
+        ENCODERS = Collections.unmodifiableMap(encoders);
+    }
+
+    private final Encoder encoder;
 
-    public CustomMustacheFactory(boolean escaping) {
+    public CustomMustacheFactory(String mimeType) {
         super();
         setObjectHandler(new CustomReflectionObjectHandler());
-        if (escaping) {
-            this.encoder = new JsonEscapeEncoder();
-        } else {
-            this.encoder = new NoEscapeEncoder();
-        }
+        this.encoder = createEncoder(mimeType);
+    }
+
+    public CustomMustacheFactory() {
+        this(DEFAULT_MIME_TYPE);
     }
 
     @Override
     public void encode(String value, Writer writer) {
-        encoder.accept(value, writer);
+        try {
+            encoder.encode(value, writer);
+        } catch (IOException e) {
+            throw new MustacheException("Unable to encode value", e);
+        }
+    }
+
+    static Encoder createEncoder(String mimeType) {
+        Supplier<Encoder> supplier = ENCODERS.get(mimeType);
+        if (supplier == null) {
+            throw new IllegalArgumentException("No encoder found for MIME type [" + mimeType + "]");
+        }
+        return supplier.get();
     }
 
     @Override
@@ -71,7 +104,7 @@ public MustacheVisitor createMustacheVisitor() {
 
     class CustomMustacheVisitor extends DefaultMustacheVisitor {
 
-        public CustomMustacheVisitor(DefaultMustacheFactory df) {
+        CustomMustacheVisitor(DefaultMustacheFactory df) {
             super(df);
         }
 
@@ -83,6 +116,8 @@ public void iterable(TemplateContext templateContext, String variable, Mustache
                 list.add(new JoinerCode(templateContext, df, mustache));
             } else if (CustomJoinerCode.match(variable)) {
                 list.add(new CustomJoinerCode(templateContext, df, mustache, variable));
+            } else if (UrlEncoderCode.match(variable)) {
+                list.add(new UrlEncoderCode(templateContext, df, mustache, variable));
             } else {
                 list.add(new IterableCode(templateContext, df, mustache, variable));
             }
@@ -96,7 +131,7 @@ abstract static class CustomCode extends IterableCode {
 
         private final String code;
 
-        public CustomCode(TemplateContext tc, DefaultMustacheFactory df, Mustache mustache, String code) {
+        CustomCode(TemplateContext tc, DefaultMustacheFactory df, Mustache mustache, String code) {
             super(tc, df, mustache, extractVariableName(code, mustache, tc));
             this.code = Objects.requireNonNull(code);
         }
@@ -151,7 +186,7 @@ static class ToJsonCode extends CustomCode {
 
         private static final String CODE = "toJson";
 
-        public ToJsonCode(TemplateContext tc, DefaultMustacheFactory df, Mustache mustache, String variable) {
+        ToJsonCode(TemplateContext tc, DefaultMustacheFactory df, Mustache mustache, String variable) {
             super(tc, df, mustache, CODE);
             if (CODE.equalsIgnoreCase(variable) == false) {
                 throw new MustacheException("Mismatch function code [" + CODE + "] cannot be applied to [" + variable + "]");
@@ -202,12 +237,12 @@ static class JoinerCode extends CustomCode {
 
         private final String delimiter;
 
-        public JoinerCode(TemplateContext tc, DefaultMustacheFactory df, Mustache mustache, String delimiter) {
+        JoinerCode(TemplateContext tc, DefaultMustacheFactory df, Mustache mustache, String delimiter) {
             super(tc, df, mustache, CODE);
             this.delimiter = delimiter;
         }
 
-        public JoinerCode(TemplateContext tc, DefaultMustacheFactory df, Mustache mustache) {
+        JoinerCode(TemplateContext tc, DefaultMustacheFactory df, Mustache mustache) {
             this(tc, df, mustache, DEFAULT_DELIMITER);
         }
 
@@ -236,7 +271,7 @@ static class CustomJoinerCode extends JoinerCode {
 
         private static final Pattern PATTERN = Pattern.compile("^(?:" + CODE + " delimiter='(.*)')$");
 
-        public CustomJoinerCode(TemplateContext tc, DefaultMustacheFactory df, Mustache mustache, String variable) {
+        CustomJoinerCode(TemplateContext tc, DefaultMustacheFactory df, Mustache mustache, String variable) {
             super(tc, df, mustache, extractDelimiter(variable));
         }
 
@@ -253,27 +288,85 @@ static boolean match(String variable) {
         }
     }
 
-    class NoEscapeEncoder implements BiConsumer<String, Writer> {
+    /**
+     * This function encodes a string using the {@link URLEncoder#encode(String, String)} method
+     * with the UTF-8 charset.
+     */
+    static class UrlEncoderCode extends DefaultMustache {
+
+        private static final String CODE = "url";
+        private final Encoder encoder;
+
+        UrlEncoderCode(TemplateContext tc, DefaultMustacheFactory df, Mustache mustache, String variable) {
+            super(tc, df, mustache.getCodes(), variable);
+            this.encoder = new UrlEncoder();
+        }
 
         @Override
-        public void accept(String s, Writer writer) {
-            try {
-                writer.write(s);
-            } catch (IOException e) {
-                throw new MustacheException("Failed to encode value: " + s);
+        public Writer run(Writer writer, List<Object> scopes) {
+            if (getCodes() != null) {
+                for (Code code : getCodes()) {
+                    try (StringWriter capture = new StringWriter()) {
+                        code.execute(capture, scopes);
+
+                        String s = capture.toString();
+                        if (s != null) {
+                            encoder.encode(s, writer);
+                        }
+                    } catch (IOException e) {
+                        throw new MustacheException("Exception while parsing mustache function at line " + tc.line(), e);
+                    }
+                }
             }
+            return writer;
+        }
+
+        static boolean match(String variable) {
+            return CODE.equalsIgnoreCase(variable);
         }
     }
 
-    class JsonEscapeEncoder implements BiConsumer<String, Writer> {
+    @FunctionalInterface
+    interface Encoder {
+        /**
+         * Encodes the {@code s} string and writes it to the {@code writer} {@link Writer}.
+         *
+         * @param s      The string to encode
+         * @param writer The {@link Writer} to which the encoded string will be written to
+         */
+        void encode(String s, Writer writer) throws IOException;
+    }
+
+    /**
+     * Encoder that simply writes the string to the writer without encoding.
+     */
+    static class DefaultEncoder implements Encoder {
 
         @Override
-        public void accept(String s, Writer writer) {
-            try {
-                writer.write(JsonStringEncoder.getInstance().quoteAsString(s));
-            } catch (IOException e) {
-                throw new MustacheException("Failed to escape and encode value: " + s);
-            }
+        public void encode(String s, Writer writer) throws IOException {
+            writer.write(s);
+        }
+    }
+
+    /**
+     * Encoder that escapes JSON string values/fields.
+     */
+    static class JsonEscapeEncoder implements Encoder {
+
+        @Override
+        public void encode(String s, Writer writer) throws IOException {
+            writer.write(JsonStringEncoder.getInstance().quoteAsString(s));
+        }
+    }
+
+    /**
+     * Encoder that escapes strings using HTML form encoding
+     */
+    static class UrlEncoder implements Encoder {
+
+        @Override
+        public void encode(String s, Writer writer) throws IOException {
+            writer.write(URLEncoder.encode(s, StandardCharsets.UTF_8.name()));
         }
     }
 }
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomReflectionObjectHandler.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomReflectionObjectHandler.java
index dd3055ba8e88c..eef9d7af8dd01 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomReflectionObjectHandler.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomReflectionObjectHandler.java
@@ -54,7 +54,7 @@ static final class ArrayMap extends AbstractMap<Object, Object> implements Itera
         private final Object array;
         private final int length;
 
-        public ArrayMap(Object array) {
+        ArrayMap(Object array) {
             this.array = array;
             this.length = Array.getLength(array);
         }
@@ -113,7 +113,7 @@ static final class CollectionMap extends AbstractMap<Object, Object> implements
 
         private final Collection<Object> col;
 
-        public CollectionMap(Collection<Object> col) {
+        CollectionMap(Collection<Object> col) {
             this.col = col;
         }
 
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequest.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequest.java
index b1db44defa042..bd294db6d2957 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequest.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequest.java
@@ -19,10 +19,10 @@
 
 package org.elasticsearch.script.mustache;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.CompositeIndicesRequest;
-import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -33,8 +33,9 @@
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 
-public class MultiSearchTemplateRequest extends ActionRequest<MultiSearchTemplateRequest> implements CompositeIndicesRequest {
+public class MultiSearchTemplateRequest extends ActionRequest implements CompositeIndicesRequest {
 
+    private int maxConcurrentSearchRequests = 0;
     private List<SearchTemplateRequest> requests = new ArrayList<>();
 
     private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpenAndForbidClosed();
@@ -57,13 +58,28 @@ public MultiSearchTemplateRequest add(SearchTemplateRequest request) {
         return this;
     }
 
-    public List<SearchTemplateRequest> requests() {
-        return this.requests;
+
+    /**
+     * Returns the amount of search requests specified in this multi search requests are allowed to be ran concurrently.
+     */
+    public int maxConcurrentSearchRequests() {
+        return maxConcurrentSearchRequests;
     }
 
-    @Override
-    public List<? extends IndicesRequest> subRequests() {
-        return requests;
+    /**
+     * Sets how many search requests specified in this multi search requests are allowed to be ran concurrently.
+     */
+    public MultiSearchTemplateRequest maxConcurrentSearchRequests(int maxConcurrentSearchRequests) {
+        if (maxConcurrentSearchRequests < 1) {
+            throw new IllegalArgumentException("maxConcurrentSearchRequests must be positive");
+        }
+
+        this.maxConcurrentSearchRequests = maxConcurrentSearchRequests;
+        return this;
+    }
+
+    public List<SearchTemplateRequest> requests() {
+        return this.requests;
     }
 
     @Override
@@ -96,12 +112,18 @@ public MultiSearchTemplateRequest indicesOptions(IndicesOptions indicesOptions)
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
+        if (in.getVersion().onOrAfter(Version.V_5_5_0)) {
+            maxConcurrentSearchRequests = in.readVInt();
+        }
         requests = in.readStreamableList(SearchTemplateRequest::new);
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
+        if (out.getVersion().onOrAfter(Version.V_5_5_0)) {
+            out.writeVInt(maxConcurrentSearchRequests);
+        }
         out.writeStreamableList(requests);
     }
 }
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequestBuilder.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequestBuilder.java
index 4624e8caa2433..4ef6c593d9a41 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequestBuilder.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequestBuilder.java
@@ -30,10 +30,6 @@ protected MultiSearchTemplateRequestBuilder(ElasticsearchClient client, MultiSea
         super(client, action, new MultiSearchTemplateRequest());
     }
 
-    public MultiSearchTemplateRequestBuilder(ElasticsearchClient client) {
-        this(client, MultiSearchTemplateAction.INSTANCE);
-    }
-
     public MultiSearchTemplateRequestBuilder add(SearchTemplateRequest request) {
         if (request.getRequest().indicesOptions() == IndicesOptions.strictExpandOpenAndForbidClosed()
                 && request().indicesOptions() != IndicesOptions.strictExpandOpenAndForbidClosed()) {
@@ -58,4 +54,12 @@ public MultiSearchTemplateRequestBuilder setIndicesOptions(IndicesOptions indice
         request().indicesOptions(indicesOptions);
         return this;
     }
+
+    /**
+     * Sets how many search requests specified in this multi search requests are allowed to be ran concurrently.
+     */
+    public MultiSearchTemplateRequestBuilder setMaxConcurrentSearchRequests(int maxConcurrentSearchRequests) {
+        request().maxConcurrentSearchRequests(maxConcurrentSearchRequests);
+        return this;
+    }
 }
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateResponse.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateResponse.java
index 6201e9305ef2b..37d72ac4c0d04 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateResponse.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateResponse.java
@@ -22,18 +22,19 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
 import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
 
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Iterator;
 
-public class MultiSearchTemplateResponse extends ActionResponse implements Iterable<MultiSearchTemplateResponse.Item>, ToXContent {
+public class MultiSearchTemplateResponse extends ActionResponse implements Iterable<MultiSearchTemplateResponse.Item>, ToXContentObject {
 
     /**
      * A search template response item, holding the actual search template response, or an error message if it failed.
@@ -146,36 +147,28 @@ public void writeTo(StreamOutput out) throws IOException {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
+        builder.startObject();
         builder.startArray(Fields.RESPONSES);
         for (Item item : items) {
-            builder.startObject();
             if (item.isFailure()) {
-                ElasticsearchException.renderException(builder, params, item.getFailure());
+                builder.startObject();
+                ElasticsearchException.generateFailureXContent(builder, params, item.getFailure(), true);
+                builder.endObject();
             } else {
                 item.getResponse().toXContent(builder, params);
             }
-            builder.endObject();
         }
         builder.endArray();
+        builder.endObject();
         return builder;
     }
 
     static final class Fields {
         static final String RESPONSES = "responses";
-        static final String ERROR = "error";
-        static final String ROOT_CAUSE = "root_cause";
     }
 
     @Override
     public String toString() {
-        try {
-            XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint();
-            builder.startObject();
-            toXContent(builder, EMPTY_PARAMS);
-            builder.endObject();
-            return builder.string();
-        } catch (IOException e) {
-            return "{ \"error\" : \"" + e.getMessage() + "\"}";
-        }
+        return Strings.toString(this);
     }
 }
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java
index 170070564f9d9..9315a0fbd4b1f 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java
@@ -21,16 +21,23 @@
 
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.ScriptPlugin;
 import org.elasticsearch.plugins.SearchPlugin;
+import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestHandler;
 import org.elasticsearch.script.ScriptEngineService;
 
 import java.util.Arrays;
 import java.util.List;
+import java.util.function.Supplier;
 
 import static java.util.Collections.singletonList;
 
@@ -38,11 +45,11 @@ public class MustachePlugin extends Plugin implements ScriptPlugin, ActionPlugin
 
     @Override
     public ScriptEngineService getScriptEngineService(Settings settings) {
-        return new MustacheScriptEngineService(settings);
+        return new MustacheScriptEngineService();
     }
 
     @Override
-    public List<ActionHandler<? extends ActionRequest<?>, ? extends ActionResponse>> getActions() {
+    public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
         return Arrays.asList(new ActionHandler<>(SearchTemplateAction.INSTANCE, TransportSearchTemplateAction.class),
                 new ActionHandler<>(MultiSearchTemplateAction.INSTANCE, TransportMultiSearchTemplateAction.class));
     }
@@ -53,8 +60,15 @@ public List<QuerySpec<?>> getQueries() {
     }
 
     @Override
-    public List<Class<? extends RestHandler>> getRestHandlers() {
-        return Arrays.asList(RestSearchTemplateAction.class, RestMultiSearchTemplateAction.class, RestGetSearchTemplateAction.class,
-                RestPutSearchTemplateAction.class, RestDeleteSearchTemplateAction.class, RestRenderSearchTemplateAction.class);
+    public List<RestHandler> getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings,
+            IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver,
+            Supplier<DiscoveryNodes> nodesInCluster) {
+        return Arrays.asList(
+                new RestSearchTemplateAction(settings, restController),
+                new RestMultiSearchTemplateAction(settings, restController),
+                new RestGetSearchTemplateAction(settings, restController),
+                new RestPutSearchTemplateAction(settings, restController),
+                new RestDeleteSearchTemplateAction(settings, restController),
+                new RestRenderSearchTemplateAction(settings, restController));
     }
 }
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java
index b7d7087373cf7..f18c229ceb6b3 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java
@@ -20,18 +20,20 @@
 
 import com.github.mustachejava.Mustache;
 import com.github.mustachejava.MustacheFactory;
+
+import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
 import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.io.FastStringReader;
 import org.elasticsearch.common.io.UTF8StreamWriter;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.GeneralScriptException;
+import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptEngineService;
 import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.lookup.SearchLookup;
@@ -51,14 +53,11 @@
  * process: First compile the string representing the template, the resulting
  * {@link Mustache} object can then be re-used for subsequent executions.
  */
-public final class MustacheScriptEngineService extends AbstractComponent implements ScriptEngineService {
+public final class MustacheScriptEngineService implements ScriptEngineService {
+    private static final Logger logger = ESLoggerFactory.getLogger(MustacheScriptEngineService.class);
 
     public static final String NAME = "mustache";
 
-    static final String CONTENT_TYPE_PARAM = "content_type";
-    static final String JSON_CONTENT_TYPE = "application/json";
-    static final String PLAIN_TEXT_CONTENT_TYPE = "text/plain";
-
     /** Thread local UTF8StreamWriter to store template execution results in, thread local to save object creation.*/
     private static ThreadLocal<SoftReference<UTF8StreamWriter>> utf8StreamWriter = new ThreadLocal<>();
 
@@ -74,30 +73,25 @@ private static UTF8StreamWriter utf8StreamWriter() {
         return writer;
     }
 
-    /**
-     * @param settings automatically wired by Guice.
-     * */
-    public MustacheScriptEngineService(Settings settings) {
-        super(settings);
-    }
-
     /**
      * Compile a template string to (in this case) a Mustache object than can
      * later be re-used for execution to fill in missing parameter values.
      *
-     * @param templateSource
-     *            a string representing the template to compile.
+     * @param templateSource a string representing the template to compile.
      * @return a compiled template object for later execution.
      * */
     @Override
     public Object compile(String templateName, String templateSource, Map<String, String> params) {
-        final MustacheFactory factory = new CustomMustacheFactory(isJsonEscapingEnabled(params));
+        final MustacheFactory factory = createMustacheFactory(params);
         Reader reader = new FastStringReader(templateSource);
         return factory.compile(reader, "query-template");
     }
 
-    private boolean isJsonEscapingEnabled(Map<String, String> params) {
-        return JSON_CONTENT_TYPE.equals(params.getOrDefault(CONTENT_TYPE_PARAM, JSON_CONTENT_TYPE));
+    private CustomMustacheFactory createMustacheFactory(Map<String, String> params) {
+        if (params == null || params.isEmpty() || params.containsKey(Script.CONTENT_TYPE_OPTION) == false) {
+            return new CustomMustacheFactory();
+        }
+        return new CustomMustacheFactory(params.get(Script.CONTENT_TYPE_OPTION));
     }
 
     @Override
@@ -143,9 +137,9 @@ private class MustacheExecutableScript implements ExecutableScript {
          * @param template the compiled template object wrapper
          * @param vars the parameters to fill above object with
          **/
-        public MustacheExecutableScript(CompiledScript template, Map<String, Object> vars) {
+        MustacheExecutableScript(CompiledScript template, Map<String, Object> vars) {
             this.template = template;
-            this.vars = vars == null ? Collections.<String, Object>emptyMap() : vars;
+            this.vars = vars == null ? Collections.emptyMap() : vars;
         }
 
         @Override
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestDeleteSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestDeleteSearchTemplateAction.java
index a6ea92e2818c0..94e2014339ff7 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestDeleteSearchTemplateAction.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestDeleteSearchTemplateAction.java
@@ -18,24 +18,37 @@
  */
 package org.elasticsearch.script.mustache;
 
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.action.admin.cluster.RestDeleteStoredScriptAction;
+import org.elasticsearch.rest.action.AcknowledgedRestListener;
+import org.elasticsearch.script.Script;
+
+import java.io.IOException;
 
 import static org.elasticsearch.rest.RestRequest.Method.DELETE;
 
-public class RestDeleteSearchTemplateAction extends RestDeleteStoredScriptAction {
+public class RestDeleteSearchTemplateAction extends BaseRestHandler {
+    private static final DeprecationLogger DEPRECATION_LOGGER =
+        new DeprecationLogger(Loggers.getLogger(RestDeleteSearchTemplateAction.class));
 
-    @Inject
     public RestDeleteSearchTemplateAction(Settings settings, RestController controller) {
-        super(settings, controller, false);
-        controller.registerHandler(DELETE, "/_search/template/{id}", this);
+        super(settings);
+
+        controller.registerAsDeprecatedHandler(DELETE, "/_search/template/{id}", this,
+            "The stored search template API is deprecated. Use stored scripts instead.", DEPRECATION_LOGGER);
     }
 
     @Override
-    protected String getScriptLang(RestRequest request) {
-        return "mustache";
+    public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+        String id = request.param("id");
+
+        DeleteStoredScriptRequest deleteStoredScriptRequest = new DeleteStoredScriptRequest(id, Script.DEFAULT_TEMPLATE_LANG);
+        return channel -> client.admin().cluster().deleteStoredScript(deleteStoredScriptRequest, new AcknowledgedRestListener<>(channel));
     }
 }
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestGetSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestGetSearchTemplateAction.java
index 16ab14bc6c8e6..45cae8da7b234 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestGetSearchTemplateAction.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestGetSearchTemplateAction.java
@@ -18,31 +18,69 @@
  */
 package org.elasticsearch.script.mustache;
 
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest;
+import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptResponse;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.action.admin.cluster.RestGetStoredScriptAction;
+import org.elasticsearch.rest.RestResponse;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.rest.action.RestBuilderListener;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.StoredScriptSource;
+
+import java.io.IOException;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
-public class RestGetSearchTemplateAction extends RestGetStoredScriptAction {
+public class RestGetSearchTemplateAction extends BaseRestHandler {
+    private static final DeprecationLogger DEPRECATION_LOGGER =
+        new DeprecationLogger(Loggers.getLogger(RestGetSearchTemplateAction.class));
+
+    public static final ParseField _ID_PARSE_FIELD = new ParseField("_id");
 
-    private static final String TEMPLATE = "template";
+    public static final ParseField FOUND_PARSE_FIELD = new ParseField("found");
 
-    @Inject
     public RestGetSearchTemplateAction(Settings settings, RestController controller) {
-        super(settings, controller, false);
-        controller.registerHandler(GET, "/_search/template/{id}", this);
-    }
+        super(settings);
 
-    @Override
-    protected String getScriptLang(RestRequest request) {
-        return "mustache";
+        controller.registerAsDeprecatedHandler(GET, "/_search/template/{id}", this,
+            "The stored search template API is deprecated. Use stored scripts instead.", DEPRECATION_LOGGER);
     }
 
     @Override
-    protected String getScriptFieldName() {
-        return TEMPLATE;
+    public RestChannelConsumer prepareRequest(final RestRequest request, NodeClient client) throws IOException {
+        String id = request.param("id");
+
+        GetStoredScriptRequest getRequest = new GetStoredScriptRequest(id, Script.DEFAULT_TEMPLATE_LANG);
+
+        return channel -> client.admin().cluster().getStoredScript(getRequest, new RestBuilderListener<GetStoredScriptResponse>(channel) {
+            @Override
+            public RestResponse buildResponse(GetStoredScriptResponse response, XContentBuilder builder) throws Exception {
+                builder.startObject();
+                builder.field(_ID_PARSE_FIELD.getPreferredName(), id);
+
+                builder.field(StoredScriptSource.LANG_PARSE_FIELD.getPreferredName(), Script.DEFAULT_TEMPLATE_LANG);
+
+                StoredScriptSource source = response.getSource();
+                boolean found = source != null;
+                builder.field(FOUND_PARSE_FIELD.getPreferredName(), found);
+
+                if (found) {
+                    builder.field(StoredScriptSource.TEMPLATE_PARSE_FIELD.getPreferredName(), source.getSource());
+                }
+
+                builder.endObject();
+
+                return new BytesRestResponse(found ? RestStatus.OK : RestStatus.NOT_FOUND, builder);
+            }
+        });
     }
 }
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java
index a7b8615372fff..f995d0c06f7de 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java
@@ -22,26 +22,27 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.action.RestActions;
 import org.elasticsearch.rest.action.RestToXContentListener;
 import org.elasticsearch.rest.action.search.RestMultiSearchAction;
+import org.elasticsearch.rest.action.search.RestSearchAction;
 
 import java.io.IOException;
+import java.util.Collections;
+import java.util.Set;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
 public class RestMultiSearchTemplateAction extends BaseRestHandler {
 
+    private static final Set<String> RESPONSE_PARAMS = Collections.singleton(RestSearchAction.TYPED_KEYS_PARAM);
+
     private final boolean allowExplicitIndex;
 
-    @Inject
     public RestMultiSearchTemplateAction(Settings settings, RestController controller) {
         super(settings);
         this.allowExplicitIndex = MULTI_ALLOW_EXPLICIT_INDEX.get(settings);
@@ -55,21 +56,20 @@ public RestMultiSearchTemplateAction(Settings settings, RestController controlle
     }
 
     @Override
-    public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
-        if (RestActions.hasBodyContent(request) == false) {
-            throw new ElasticsearchException("request body is required");
-        }
-
+    public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
         MultiSearchTemplateRequest multiRequest = parseRequest(request, allowExplicitIndex);
-        client.execute(MultiSearchTemplateAction.INSTANCE, multiRequest, new RestToXContentListener<>(channel));
+        return channel -> client.execute(MultiSearchTemplateAction.INSTANCE, multiRequest, new RestToXContentListener<>(channel));
     }
 
     /**
      * Parses a {@link RestRequest} body and returns a {@link MultiSearchTemplateRequest}
      */
     public static MultiSearchTemplateRequest parseRequest(RestRequest restRequest, boolean allowExplicitIndex) throws IOException {
-
         MultiSearchTemplateRequest multiRequest = new MultiSearchTemplateRequest();
+        if (restRequest.hasParam("max_concurrent_searches")) {
+            multiRequest.maxConcurrentSearchRequests(restRequest.paramAsInt("max_concurrent_searches", 0));
+        }
+
         RestMultiSearchAction.parseMultiLineRequest(restRequest, multiRequest.indicesOptions(), allowExplicitIndex,
                 (searchRequest, bytes) -> {
                     try {
@@ -86,4 +86,14 @@ public static MultiSearchTemplateRequest parseRequest(RestRequest restRequest, b
                 });
         return multiRequest;
     }
+
+    @Override
+    public boolean supportsContentStream() {
+        return true;
+    }
+
+    @Override
+    protected Set<String> responseParams() {
+        return RESPONSE_PARAMS;
+    }
 }
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestPutSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestPutSearchTemplateAction.java
index 718afb608e699..6b88539ef3b7e 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestPutSearchTemplateAction.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestPutSearchTemplateAction.java
@@ -18,26 +18,42 @@
  */
 package org.elasticsearch.script.mustache;
 
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequest;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.action.admin.cluster.RestPutStoredScriptAction;
+import org.elasticsearch.rest.action.AcknowledgedRestListener;
+import org.elasticsearch.script.Script;
+
+import java.io.IOException;
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.rest.RestRequest.Method.PUT;
 
-public class RestPutSearchTemplateAction extends RestPutStoredScriptAction {
+public class RestPutSearchTemplateAction extends BaseRestHandler {
+
+    private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(RestPutSearchTemplateAction.class));
 
-    @Inject
     public RestPutSearchTemplateAction(Settings settings, RestController controller) {
-        super(settings, controller, false);
-        controller.registerHandler(POST, "/_search/template/{id}", this);
-        controller.registerHandler(PUT, "/_search/template/{id}", this);
+        super(settings);
+
+        controller.registerAsDeprecatedHandler(POST, "/_search/template/{id}", this,
+            "The stored search template API is deprecated. Use stored scripts instead.", DEPRECATION_LOGGER);
+        controller.registerAsDeprecatedHandler(PUT, "/_search/template/{id}", this,
+            "The stored search template API is deprecated. Use stored scripts instead.", DEPRECATION_LOGGER);
     }
 
     @Override
-    protected String getScriptLang(RestRequest request) {
-        return "mustache";
+    public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+        String id = request.param("id");
+        BytesReference content = request.requiredContent();
+
+        PutStoredScriptRequest put = new PutStoredScriptRequest(id, Script.DEFAULT_TEMPLATE_LANG, content, request.getXContentType());
+        return channel -> client.admin().cluster().putStoredScript(put, new AcknowledgedRestListener<>(channel));
     }
 }
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestRenderSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestRenderSearchTemplateAction.java
index 74dc6363f3bb4..6647bd4bd4522 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestRenderSearchTemplateAction.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestRenderSearchTemplateAction.java
@@ -20,22 +20,20 @@
 package org.elasticsearch.script.mustache;
 
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.action.RestActions;
 import org.elasticsearch.rest.action.RestToXContentListener;
-import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptType;
+
+import java.io.IOException;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
 public class RestRenderSearchTemplateAction extends BaseRestHandler {
-
-    @Inject
     public RestRenderSearchTemplateAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_render/template", this);
@@ -45,17 +43,20 @@ public RestRenderSearchTemplateAction(Settings settings, RestController controll
     }
 
     @Override
-    public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
+    public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
         // Creates the render template request
-        SearchTemplateRequest renderRequest = RestSearchTemplateAction.parse(RestActions.getRestContent(request));
+        SearchTemplateRequest renderRequest;
+        try (XContentParser parser = request.contentOrSourceParamParser()) {
+            renderRequest = RestSearchTemplateAction.parse(parser);
+        }
         renderRequest.setSimulate(true);
 
         String id = request.param("id");
         if (id != null) {
-            renderRequest.setScriptType(ScriptService.ScriptType.STORED);
+            renderRequest.setScriptType(ScriptType.STORED);
             renderRequest.setScript(id);
         }
 
-        client.execute(SearchTemplateAction.INSTANCE, renderRequest, new RestToXContentListener<>(channel));
+        return channel -> client.execute(SearchTemplateAction.INSTANCE, renderRequest, new RestToXContentListener<>(channel));
     }
 }
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java
index 18637b15e840b..a44dbfc4419ad 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java
@@ -23,71 +23,63 @@
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.action.RestActions;
 import org.elasticsearch.rest.action.RestStatusToXContentListener;
 import org.elasticsearch.rest.action.search.RestSearchAction;
-import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.search.SearchRequestParsers;
-import org.elasticsearch.search.aggregations.AggregatorParsers;
-import org.elasticsearch.search.suggest.Suggesters;
+import org.elasticsearch.script.ScriptType;
 
 import java.io.IOException;
+import java.util.Collections;
+import java.util.Set;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
 public class RestSearchTemplateAction extends BaseRestHandler {
 
-    private static ObjectParser<SearchTemplateRequest, ParseFieldMatcherSupplier> PARSER;
+    private static final Set<String> RESPONSE_PARAMS = Collections.singleton(RestSearchAction.TYPED_KEYS_PARAM);
+
+    private static final ObjectParser<SearchTemplateRequest, Void> PARSER;
     static {
         PARSER = new ObjectParser<>("search_template");
         PARSER.declareField((parser, request, s) ->
                         request.setScriptParams(parser.map())
                 , new ParseField("params"), ObjectParser.ValueType.OBJECT);
         PARSER.declareString((request, s) -> {
-            request.setScriptType(ScriptService.ScriptType.FILE);
+            request.setScriptType(ScriptType.FILE);
             request.setScript(s);
         }, new ParseField("file"));
         PARSER.declareString((request, s) -> {
-            request.setScriptType(ScriptService.ScriptType.STORED);
+            request.setScriptType(ScriptType.STORED);
             request.setScript(s);
         }, new ParseField("id"));
+        PARSER.declareBoolean(SearchTemplateRequest::setExplain, new ParseField("explain"));
+        PARSER.declareBoolean(SearchTemplateRequest::setProfile, new ParseField("profile"));
         PARSER.declareField((parser, request, value) -> {
-            request.setScriptType(ScriptService.ScriptType.INLINE);
+            request.setScriptType(ScriptType.INLINE);
             if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
-                try (XContentBuilder builder = XContentFactory.contentBuilder(parser.contentType())) {
-                    request.setScript(builder.copyCurrentStructure(parser).bytes().utf8ToString());
+                //convert the template to json which is the only supported XContentType (see CustomMustacheFactory#createEncoder)
+                try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
+                    request.setScript(builder.copyCurrentStructure(parser).string());
                 } catch (IOException e) {
                     throw new ParsingException(parser.getTokenLocation(), "Could not parse inline template", e);
                 }
             } else {
                 request.setScript(parser.text());
             }
-        }, new ParseField("inline", "template"), ObjectParser.ValueType.OBJECT_OR_STRING);
+        }, new ParseField("source", "inline", "template"), ObjectParser.ValueType.OBJECT_OR_STRING);
     }
 
-    private final SearchRequestParsers searchRequestParsers;
-
-    @Inject
-    public RestSearchTemplateAction(Settings settings, RestController controller, SearchRequestParsers searchRequestParsers) {
+    public RestSearchTemplateAction(Settings settings, RestController controller) {
         super(settings);
-        this.searchRequestParsers = searchRequestParsers;
 
         controller.registerHandler(GET, "/_search/template", this);
         controller.registerHandler(POST, "/_search/template", this);
@@ -98,25 +90,27 @@ public RestSearchTemplateAction(Settings settings, RestController controller, Se
     }
 
     @Override
-    public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
-        if (RestActions.hasBodyContent(request) == false) {
-            throw new ElasticsearchException("request body is required");
-        }
-
+    public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
         // Creates the search request with all required params
         SearchRequest searchRequest = new SearchRequest();
-        RestSearchAction.parseSearchRequest(searchRequest, request, searchRequestParsers, parseFieldMatcher, null);
+        RestSearchAction.parseSearchRequest(searchRequest, request, null);
 
         // Creates the search template request
-        SearchTemplateRequest searchTemplateRequest = parse(RestActions.getRestContent(request));
+        SearchTemplateRequest searchTemplateRequest;
+        try (XContentParser parser = request.contentOrSourceParamParser()) {
+            searchTemplateRequest = PARSER.parse(parser, new SearchTemplateRequest(), null);
+        }
         searchTemplateRequest.setRequest(searchRequest);
 
-        client.execute(SearchTemplateAction.INSTANCE, searchTemplateRequest, new RestStatusToXContentListener<>(channel));
+        return channel -> client.execute(SearchTemplateAction.INSTANCE, searchTemplateRequest, new RestStatusToXContentListener<>(channel));
     }
 
-    public static SearchTemplateRequest parse(BytesReference bytes) throws IOException {
-        try (XContentParser parser = XContentHelper.createParser(bytes)) {
-            return PARSER.parse(parser, new SearchTemplateRequest(), () -> ParseFieldMatcher.STRICT);
-        }
+    public static SearchTemplateRequest parse(XContentParser parser) throws IOException {
+        return PARSER.parse(parser, new SearchTemplateRequest(), null);
+    }
+
+    @Override
+    protected Set<String> responseParams() {
+        return RESPONSE_PARAMS;
     }
 }
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateRequest.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateRequest.java
index a4a6533604613..8ff30fb0e5b4e 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateRequest.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateRequest.java
@@ -21,13 +21,11 @@
 
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
-import org.elasticsearch.action.IndicesRequest;
+import org.elasticsearch.action.CompositeIndicesRequest;
 import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptType;
 
 import java.io.IOException;
 import java.util.Map;
@@ -37,11 +35,13 @@
 /**
  * A request to execute a search based on a search template.
  */
-public class SearchTemplateRequest extends ActionRequest<SearchTemplateRequest> implements IndicesRequest {
+public class SearchTemplateRequest extends ActionRequest implements CompositeIndicesRequest {
 
     private SearchRequest request;
     private boolean simulate = false;
-    private ScriptService.ScriptType scriptType;
+    private boolean explain = false;
+    private boolean profile = false;
+    private ScriptType scriptType;
     private String script;
     private Map<String, Object> scriptParams;
 
@@ -69,11 +69,27 @@ public void setSimulate(boolean simulate) {
         this.simulate = simulate;
     }
 
-    public ScriptService.ScriptType getScriptType() {
+    public boolean isExplain() {
+        return explain;
+    }
+
+    public void setExplain(boolean explain) {
+        this.explain = explain;
+    }
+
+    public boolean isProfile() {
+        return profile;
+    }
+
+    public void setProfile(boolean profile) {
+        this.profile = profile;
+    }
+
+    public ScriptType getScriptType() {
         return scriptType;
     }
 
-    public void setScriptType(ScriptService.ScriptType scriptType) {
+    public void setScriptType(ScriptType scriptType) {
         this.scriptType = scriptType;
     }
 
@@ -123,7 +139,9 @@ public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
         request = in.readOptionalStreamable(SearchRequest::new);
         simulate = in.readBoolean();
-        scriptType = ScriptService.ScriptType.readFrom(in);
+        explain = in.readBoolean();
+        profile = in.readBoolean();
+        scriptType = ScriptType.readFrom(in);
         script = in.readOptionalString();
         if (in.readBoolean()) {
             scriptParams = in.readMap();
@@ -135,7 +153,9 @@ public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         out.writeOptionalStreamable(request);
         out.writeBoolean(simulate);
-        ScriptService.ScriptType.writeTo(scriptType, out);
+        out.writeBoolean(explain);
+        out.writeBoolean(profile);
+        scriptType.writeTo(out);
         out.writeOptionalString(script);
         boolean hasParams = scriptParams != null;
         out.writeBoolean(hasParams);
@@ -143,14 +163,4 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeMap(scriptParams);
         }
     }
-
-    @Override
-    public String[] indices() {
-        return request != null ? request.indices() : Strings.EMPTY_ARRAY;
-    }
-
-    @Override
-    public IndicesOptions indicesOptions() {
-        return request != null ? request.indicesOptions() : SearchRequest.DEFAULT_INDICES_OPTIONS;
-    }
 }
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateRequestBuilder.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateRequestBuilder.java
index 811c2523e0651..02d27ac79fc3d 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateRequestBuilder.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateRequestBuilder.java
@@ -22,7 +22,7 @@
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.client.ElasticsearchClient;
-import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptType;
 
 import java.util.Map;
 
@@ -47,7 +47,23 @@ public SearchTemplateRequestBuilder setSimulate(boolean simulate) {
         return this;
     }
 
-    public SearchTemplateRequestBuilder setScriptType(ScriptService.ScriptType scriptType) {
+    /**
+     * Enables explanation for each hit on how its score was computed. Disabled by default
+     */
+    public SearchTemplateRequestBuilder setExplain(boolean explain) {
+        request.setExplain(explain);
+        return this;
+    }
+
+    /**
+     * Enables profiling of the query. Disabled by default
+     */
+    public SearchTemplateRequestBuilder setProfile(boolean profile) {
+        request.setProfile(profile);
+        return this;
+    }
+
+    public SearchTemplateRequestBuilder setScriptType(ScriptType scriptType) {
         request.setScriptType(scriptType);
         return this;
     }
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateResponse.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateResponse.java
index f4647b713de9a..22d7da774eb7c 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateResponse.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateResponse.java
@@ -24,13 +24,14 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.StatusToXContent;
+import org.elasticsearch.common.xcontent.StatusToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
 
-public class SearchTemplateResponse  extends ActionResponse implements StatusToXContent {
+public class SearchTemplateResponse  extends ActionResponse implements StatusToXContentObject {
 
     /** Contains the source of the rendered template **/
     private BytesReference source;
@@ -80,7 +81,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (hasResponse()) {
             response.toXContent(builder, params);
         } else {
-            builder.rawField("template_output", source);
+            builder.startObject();
+            //we can assume the template is always json as we convert it before compiling it
+            builder.rawField("template_output", source, XContentType.JSON);
+            builder.endObject();
         }
         return builder;
     }
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TemplateQueryBuilder.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TemplateQueryBuilder.java
index 4a19bcb4b93e0..1eaf64957faea 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TemplateQueryBuilder.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TemplateQueryBuilder.java
@@ -35,10 +35,8 @@
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryRewriteContext;
 import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptContext;
-import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptType;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -59,12 +57,14 @@ public class TemplateQueryBuilder extends AbstractQueryBuilder<TemplateQueryBuil
     /** Template to fill. */
     private final Script template;
 
-    public TemplateQueryBuilder(String template, ScriptService.ScriptType scriptType, Map<String, Object> params) {
-        this(new Script(template, scriptType, "mustache", params));
+    public TemplateQueryBuilder(String template, ScriptType scriptType, Map<String, Object> params) {
+        this(new Script(scriptType, "mustache", template, params));
     }
 
-    public TemplateQueryBuilder(String template, ScriptService.ScriptType scriptType, Map<String, Object> params, XContentType ct) {
-        this(new Script(template, scriptType, "mustache", params, ct));
+    public TemplateQueryBuilder(String template, ScriptType scriptType, Map<String, Object> params, XContentType ct) {
+        this(new Script(scriptType, "mustache", template, scriptType == ScriptType.INLINE ?
+            (ct == null ? Collections.emptyMap() : Collections.singletonMap(Script.CONTENT_TYPE_OPTION, ct.mediaType()))
+            : null, params));
     }
 
     TemplateQueryBuilder(Script template) {
@@ -120,10 +120,9 @@ protected boolean doEquals(TemplateQueryBuilder other) {
 
     @Override
     protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException {
-        ExecutableScript executable = queryRewriteContext.getScriptService().executable(template,
-            ScriptContext.Standard.SEARCH, Collections.emptyMap());
-        BytesReference querySource = (BytesReference) executable.run();
-        try (XContentParser qSourceParser = XContentFactory.xContent(querySource).createParser(querySource)) {
+        BytesReference querySource = queryRewriteContext.getTemplateBytes(template);
+        try (XContentParser qSourceParser = XContentFactory.xContent(querySource).createParser(queryRewriteContext.getXContentRegistry(),
+                querySource)) {
             final QueryParseContext queryParseContext = queryRewriteContext.newParseContext(qSourceParser);
             final QueryBuilder queryBuilder = queryParseContext.parseInnerQueryBuilder().orElseThrow(
                     () -> new ParsingException(qSourceParser.getTokenLocation(), "inner query in [" + NAME + "] cannot be empty"));
@@ -142,7 +141,14 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws
      */
     public static Optional<TemplateQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
-        Script template = Script.parse(parser, parseContext.getParseFieldMatcher(), "mustache");
+        Script template = Script.parse(parser, Script.DEFAULT_TEMPLATE_LANG);
+
+        // for deprecation of stored script namespaces the default lang is ignored,
+        // so the template lang must be set for a stored script
+        if (template.getType() == ScriptType.STORED) {
+            template = new Script(ScriptType.STORED, Script.DEFAULT_TEMPLATE_LANG, template.getIdOrCode(), template.getParams());
+        }
+
         return Optional.of(new TemplateQueryBuilder(template));
     }
 }
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportMultiSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportMultiSearchTemplateAction.java
index a613f15972b73..167190d8f5d04 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportMultiSearchTemplateAction.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportMultiSearchTemplateAction.java
@@ -20,59 +20,81 @@
 package org.elasticsearch.script.mustache;
 
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.search.MultiSearchRequest;
+import org.elasticsearch.action.search.MultiSearchResponse;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.TransportMultiSearchAction;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.AtomicArray;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
-import java.util.concurrent.atomic.AtomicInteger;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.elasticsearch.script.mustache.TransportSearchTemplateAction.convert;
 
 public class TransportMultiSearchTemplateAction extends HandledTransportAction<MultiSearchTemplateRequest, MultiSearchTemplateResponse> {
 
-    private final TransportSearchTemplateAction searchTemplateAction;
+    private final ScriptService scriptService;
+    private final NamedXContentRegistry xContentRegistry;
+    private final TransportMultiSearchAction multiSearchAction;
 
     @Inject
     public TransportMultiSearchTemplateAction(Settings settings, ThreadPool threadPool, TransportService transportService,
                                               ActionFilters actionFilters, IndexNameExpressionResolver resolver,
-                                              TransportSearchTemplateAction searchTemplateAction) {
+                                              ScriptService scriptService, NamedXContentRegistry xContentRegistry,
+                                              TransportMultiSearchAction multiSearchAction) {
         super(settings, MultiSearchTemplateAction.NAME, threadPool, transportService, actionFilters, resolver,
                 MultiSearchTemplateRequest::new);
-        this.searchTemplateAction = searchTemplateAction;
+        this.scriptService = scriptService;
+        this.xContentRegistry = xContentRegistry;
+        this.multiSearchAction = multiSearchAction;
     }
 
     @Override
     protected void doExecute(MultiSearchTemplateRequest request, ActionListener<MultiSearchTemplateResponse> listener) {
-        final AtomicArray<MultiSearchTemplateResponse.Item> responses = new AtomicArray<>(request.requests().size());
-        final AtomicInteger counter = new AtomicInteger(responses.length());
-
-        for (int i = 0; i < responses.length(); i++) {
-            final int index = i;
-            searchTemplateAction.execute(request.requests().get(i), new ActionListener<SearchTemplateResponse>() {
-                @Override
-                public void onResponse(SearchTemplateResponse searchTemplateResponse) {
-                    responses.set(index, new MultiSearchTemplateResponse.Item(searchTemplateResponse, null));
-                    if (counter.decrementAndGet() == 0) {
-                        finishHim();
-                    }
-                }
+        List<Integer> originalSlots = new ArrayList<>();
+        MultiSearchRequest multiSearchRequest = new MultiSearchRequest();
+        multiSearchRequest.indicesOptions(request.indicesOptions());
+        if (request.maxConcurrentSearchRequests() != 0) {
+            multiSearchRequest.maxConcurrentSearchRequests(request.maxConcurrentSearchRequests());
+        }
 
-                @Override
-                public void onFailure(Exception e) {
-                    responses.set(index, new MultiSearchTemplateResponse.Item(null, e));
-                    if (counter.decrementAndGet() == 0) {
-                        finishHim();
-                    }
-                }
+        MultiSearchTemplateResponse.Item[] items = new MultiSearchTemplateResponse.Item[request.requests().size()];
+        for (int i = 0; i < items.length; i++) {
+            SearchTemplateRequest searchTemplateRequest = request.requests().get(i);
+            SearchTemplateResponse searchTemplateResponse = new SearchTemplateResponse();
+            SearchRequest searchRequest;
+            try {
+                searchRequest = convert(searchTemplateRequest, searchTemplateResponse, scriptService, xContentRegistry);
+            } catch (Exception e) {
+                items[i] = new MultiSearchTemplateResponse.Item(null, e);
+                continue;
+            }
+            items[i] = new MultiSearchTemplateResponse.Item(searchTemplateResponse, null);
+            if (searchRequest != null) {
+                multiSearchRequest.add(searchRequest);
+                originalSlots.add(i);
+            }
+        }
 
-                private void finishHim() {
-                    MultiSearchTemplateResponse.Item[] items = responses.toArray(new MultiSearchTemplateResponse.Item[responses.length()]);
-                    listener.onResponse(new MultiSearchTemplateResponse(items));
+        multiSearchAction.execute(multiSearchRequest, ActionListener.wrap(r -> {
+            for (int i = 0; i < r.getResponses().length; i++) {
+                MultiSearchResponse.Item item = r.getResponses()[i];
+                int originalSlot = originalSlots.get(i);
+                if (item.isFailure()) {
+                    items[originalSlot] = new MultiSearchTemplateResponse.Item(null, item.getFailure());
+                } else {
+                    items[originalSlot].getResponse().setResponse(item.getResponse());
                 }
-            });
-        }
+            }
+            listener.onResponse(new MultiSearchTemplateResponse(items));
+        }, listener::onFailure));
     }
 }
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java
index 473e287c9a250..b88a7b8e59a99 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.script.mustache;
 
+import org.apache.logging.log4j.util.Supplier;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
@@ -29,21 +30,24 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
+import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.search.SearchRequestParsers;
-import org.elasticsearch.search.aggregations.AggregatorParsers;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.elasticsearch.search.suggest.Suggesters;
+import org.elasticsearch.template.CompiledTemplate;
+import org.elasticsearch.template.CompiledTemplate;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
-import static java.util.Collections.emptyMap;
+import java.io.IOException;
+import java.util.Collections;
+
 import static org.elasticsearch.script.ScriptContext.Standard.SEARCH;
 
 public class TransportSearchTemplateAction extends HandledTransportAction<SearchTemplateRequest, SearchTemplateResponse> {
@@ -52,43 +56,26 @@ public class TransportSearchTemplateAction extends HandledTransportAction<Search
 
     private final ScriptService scriptService;
     private final TransportSearchAction searchAction;
-    private final SearchRequestParsers searchRequestParsers;
+    private final NamedXContentRegistry xContentRegistry;
 
     @Inject
     public TransportSearchTemplateAction(Settings settings, ThreadPool threadPool, TransportService transportService,
                                          ActionFilters actionFilters, IndexNameExpressionResolver resolver,
                                          ScriptService scriptService,
-                                         TransportSearchAction searchAction, SearchRequestParsers searchRequestParsers) {
+                                         TransportSearchAction searchAction,
+                                         NamedXContentRegistry xContentRegistry) {
         super(settings, SearchTemplateAction.NAME, threadPool, transportService, actionFilters, resolver, SearchTemplateRequest::new);
         this.scriptService = scriptService;
         this.searchAction = searchAction;
-        this.searchRequestParsers = searchRequestParsers;
+        this.xContentRegistry = xContentRegistry;
     }
 
     @Override
     protected void doExecute(SearchTemplateRequest request, ActionListener<SearchTemplateResponse> listener) {
         final SearchTemplateResponse response = new SearchTemplateResponse();
         try {
-            Script script = new Script(request.getScript(), request.getScriptType(), TEMPLATE_LANG, request.getScriptParams());
-            ExecutableScript executable = scriptService.executable(script, SEARCH, emptyMap());
-
-            BytesReference source = (BytesReference) executable.run();
-            response.setSource(source);
-
-            if (request.isSimulate()) {
-                listener.onResponse(response);
-                return;
-            }
-
-            // Executes the search
-            SearchRequest searchRequest = request.getRequest();
-
-            try (XContentParser parser = XContentFactory.xContent(source).createParser(source)) {
-                SearchSourceBuilder builder = SearchSourceBuilder.searchSource();
-                builder.parseXContent(new QueryParseContext(searchRequestParsers.queryParsers, parser, parseFieldMatcher),
-                    searchRequestParsers.aggParsers, searchRequestParsers.suggesters);
-                searchRequest.source(builder);
-
+            SearchRequest searchRequest = convert(request, response, scriptService, xContentRegistry);
+            if (searchRequest != null) {
                 searchAction.execute(searchRequest, new ActionListener<SearchResponse>() {
                     @Override
                     public void onResponse(SearchResponse searchResponse) {
@@ -105,9 +92,35 @@ public void onFailure(Exception t) {
                         listener.onFailure(t);
                     }
                 });
+            } else {
+                listener.onResponse(response);
             }
-        } catch (Exception t) {
-            listener.onFailure(t);
+        } catch (IOException e) {
+            listener.onFailure(e);
+        }
+    }
+
+    static SearchRequest convert(SearchTemplateRequest searchTemplateRequest, SearchTemplateResponse response, ScriptService scriptService,
+                                 NamedXContentRegistry xContentRegistry) throws IOException {
+        Script script = new Script(searchTemplateRequest.getScriptType(), TEMPLATE_LANG, searchTemplateRequest.getScript(),
+                searchTemplateRequest.getScriptParams() == null ? Collections.emptyMap() : searchTemplateRequest.getScriptParams());
+        CompiledTemplate compiledScript = scriptService.compileTemplate(script, SEARCH);
+        BytesReference source = compiledScript.run(script.getParams());
+        response.setSource(source);
+
+        SearchRequest searchRequest = searchTemplateRequest.getRequest();
+        response.setSource(source);
+        if (searchTemplateRequest.isSimulate()) {
+            return null;
+        }
+
+        try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(xContentRegistry, source)) {
+            SearchSourceBuilder builder = SearchSourceBuilder.searchSource();
+            builder.parseXContent(new QueryParseContext(parser));
+            builder.explain(searchTemplateRequest.isExplain());
+            builder.profile(searchTemplateRequest.isProfile());
+            searchRequest.source(builder);
         }
+        return searchRequest;
     }
 }
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/CustomMustacheFactoryTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/CustomMustacheFactoryTests.java
new file mode 100644
index 0000000000000..265edb7d53bd4
--- /dev/null
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/CustomMustacheFactoryTests.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.script.mustache;
+
+import com.github.mustachejava.Mustache;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.script.CompiledScript;
+import org.elasticsearch.script.ExecutableScript;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptEngineService;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Map;
+
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.singletonMap;
+import static org.elasticsearch.script.ScriptType.INLINE;
+import static org.elasticsearch.script.mustache.CustomMustacheFactory.JSON_MIME_TYPE;
+import static org.elasticsearch.script.mustache.CustomMustacheFactory.PLAIN_TEXT_MIME_TYPE;
+import static org.elasticsearch.script.mustache.CustomMustacheFactory.X_WWW_FORM_URLENCODED_MIME_TYPE;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+
+public class CustomMustacheFactoryTests extends ESTestCase {
+
+    public void testCreateEncoder() {
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> CustomMustacheFactory.createEncoder(null));
+        assertThat(e.getMessage(), equalTo("No encoder found for MIME type [null]"));
+
+        e = expectThrows(IllegalArgumentException.class, () -> CustomMustacheFactory.createEncoder(""));
+        assertThat(e.getMessage(), equalTo("No encoder found for MIME type []"));
+
+        e = expectThrows(IllegalArgumentException.class, () -> CustomMustacheFactory.createEncoder("test"));
+        assertThat(e.getMessage(), equalTo("No encoder found for MIME type [test]"));
+
+        assertThat(CustomMustacheFactory.createEncoder(CustomMustacheFactory.JSON_MIME_TYPE_WITH_CHARSET),
+            instanceOf(CustomMustacheFactory.JsonEscapeEncoder.class));
+        assertThat(CustomMustacheFactory.createEncoder(CustomMustacheFactory.JSON_MIME_TYPE),
+                instanceOf(CustomMustacheFactory.JsonEscapeEncoder.class));
+        assertThat(CustomMustacheFactory.createEncoder(CustomMustacheFactory.PLAIN_TEXT_MIME_TYPE),
+                instanceOf(CustomMustacheFactory.DefaultEncoder.class));
+        assertThat(CustomMustacheFactory.createEncoder(CustomMustacheFactory.X_WWW_FORM_URLENCODED_MIME_TYPE),
+                instanceOf(CustomMustacheFactory.UrlEncoder.class));
+    }
+
+    public void testJsonEscapeEncoder() {
+        final ScriptEngineService engine = new MustacheScriptEngineService();
+        final Map<String, String> params = randomBoolean() ? singletonMap(Script.CONTENT_TYPE_OPTION, JSON_MIME_TYPE) : emptyMap();
+
+        Mustache script = (Mustache) engine.compile(null, "{\"field\": \"{{value}}\"}", params);
+        CompiledScript compiled = new CompiledScript(INLINE, null, MustacheScriptEngineService.NAME, script);
+
+        ExecutableScript executable = engine.executable(compiled, singletonMap("value", "a \"value\""));
+        BytesReference result = (BytesReference) executable.run();
+        assertThat(result.utf8ToString(), equalTo("{\"field\": \"a \\\"value\\\"\"}"));
+    }
+
+    public void testDefaultEncoder() {
+        final ScriptEngineService engine = new MustacheScriptEngineService();
+        final Map<String, String> params = singletonMap(Script.CONTENT_TYPE_OPTION, PLAIN_TEXT_MIME_TYPE);
+
+        Mustache script = (Mustache) engine.compile(null, "{\"field\": \"{{value}}\"}", params);
+        CompiledScript compiled = new CompiledScript(INLINE, null, MustacheScriptEngineService.NAME, script);
+
+        ExecutableScript executable = engine.executable(compiled, singletonMap("value", "a \"value\""));
+        BytesReference result = (BytesReference) executable.run();
+        assertThat(result.utf8ToString(), equalTo("{\"field\": \"a \"value\"\"}"));
+    }
+
+    public void testUrlEncoder() {
+        final ScriptEngineService engine = new MustacheScriptEngineService();
+        final Map<String, String> params = singletonMap(Script.CONTENT_TYPE_OPTION, X_WWW_FORM_URLENCODED_MIME_TYPE);
+
+        Mustache script = (Mustache) engine.compile(null, "{\"field\": \"{{value}}\"}", params);
+        CompiledScript compiled = new CompiledScript(INLINE, null, MustacheScriptEngineService.NAME, script);
+
+        ExecutableScript executable = engine.executable(compiled, singletonMap("value", "tilde~ AND date:[2016 FROM*]"));
+        BytesReference result = (BytesReference) executable.run();
+        assertThat(result.utf8ToString(), equalTo("{\"field\": \"tilde%7E+AND+date%3A%5B2016+FROM*%5D\"}"));
+    }
+}
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/LangMustacheClientYamlTestSuiteIT.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/LangMustacheClientYamlTestSuiteIT.java
index 160327dbab6da..4e9c8c1b90c61 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/LangMustacheClientYamlTestSuiteIT.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/LangMustacheClientYamlTestSuiteIT.java
@@ -24,9 +24,6 @@
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-
-import java.io.IOException;
 
 public class LangMustacheClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
@@ -35,8 +32,8 @@ public LangMustacheClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate t
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
-        return ESClientYamlSuiteTestCase.createParameters(0, 1);
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
     }
 }
 
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateIT.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateIT.java
index f5d1a9dd79121..91fc4db43dddd 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateIT.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateIT.java
@@ -23,7 +23,7 @@
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.test.ESIntegTestCase;
 
 import java.util.Arrays;
@@ -74,7 +74,7 @@ public void testBasic() throws Exception {
         // Search #1
         SearchTemplateRequest search1 = new SearchTemplateRequest();
         search1.setRequest(new SearchRequest("msearch"));
-        search1.setScriptType(ScriptService.ScriptType.INLINE);
+        search1.setScriptType(ScriptType.INLINE);
         search1.setScript(template);
 
         Map<String, Object> params1 = new HashMap<>();
@@ -87,7 +87,7 @@ public void testBasic() throws Exception {
         // Search #2 (Simulate is true)
         SearchTemplateRequest search2 = new SearchTemplateRequest();
         search2.setRequest(new SearchRequest("msearch"));
-        search2.setScriptType(ScriptService.ScriptType.INLINE);
+        search2.setScriptType(ScriptType.INLINE);
         search2.setScript(template);
         search2.setSimulate(true);
 
@@ -101,7 +101,7 @@ public void testBasic() throws Exception {
         // Search #3
         SearchTemplateRequest search3 = new SearchTemplateRequest();
         search3.setRequest(new SearchRequest("msearch"));
-        search3.setScriptType(ScriptService.ScriptType.INLINE);
+        search3.setScriptType(ScriptType.INLINE);
         search3.setScript(template);
         search3.setSimulate(false);
 
@@ -115,7 +115,7 @@ public void testBasic() throws Exception {
         // Search #4 (Fail because of unknown index)
         SearchTemplateRequest search4 = new SearchTemplateRequest();
         search4.setRequest(new SearchRequest("unknown"));
-        search4.setScriptType(ScriptService.ScriptType.INLINE);
+        search4.setScriptType(ScriptType.INLINE);
         search4.setScript(template);
 
         Map<String, Object> params4 = new HashMap<>();
@@ -128,7 +128,7 @@ public void testBasic() throws Exception {
         // Search #5 (Simulate is true)
         SearchTemplateRequest search5 = new SearchTemplateRequest();
         search5.setRequest(new SearchRequest("msearch"));
-        search5.setScriptType(ScriptService.ScriptType.INLINE);
+        search5.setScriptType(ScriptType.INLINE);
         search5.setScript("{{! ignore me }}{\"query\":{\"terms\":{\"group\":[{{#groups}}{{.}},{{/groups}}]}}}");
         search5.setSimulate(true);
 
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequestTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequestTests.java
index d7807c1a268c1..1ff5479623765 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequestTests.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequestTests.java
@@ -20,8 +20,9 @@
 package org.elasticsearch.script.mustache;
 
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.StreamsUtils;
 import org.elasticsearch.test.rest.FakeRestRequest;
@@ -34,23 +35,24 @@ public class MultiSearchTemplateRequestTests extends ESTestCase {
 
     public void testParseRequest() throws Exception {
         byte[] data = StreamsUtils.copyToBytesFromClasspath("/org/elasticsearch/script/mustache/simple-msearch-template.json");
-        RestRequest restRequest = new FakeRestRequest.Builder().withContent(new BytesArray(data)).build();
+        RestRequest restRequest = new FakeRestRequest.Builder(xContentRegistry())
+            .withContent(new BytesArray(data), XContentType.JSON).build();
 
         MultiSearchTemplateRequest request = RestMultiSearchTemplateAction.parseRequest(restRequest, true);
 
         assertThat(request.requests().size(), equalTo(3));
         assertThat(request.requests().get(0).getRequest().indices()[0], equalTo("test0"));
         assertThat(request.requests().get(0).getRequest().indices()[1], equalTo("test1"));
-        assertThat(request.requests().get(0).indices(), arrayContaining("test0", "test1"));
+        assertThat(request.requests().get(0).getRequest().indices(), arrayContaining("test0", "test1"));
         assertThat(request.requests().get(0).getRequest().requestCache(), equalTo(true));
         assertThat(request.requests().get(0).getRequest().preference(), nullValue());
-        assertThat(request.requests().get(1).indices()[0], equalTo("test2"));
-        assertThat(request.requests().get(1).indices()[1], equalTo("test3"));
+        assertThat(request.requests().get(1).getRequest().indices()[0], equalTo("test2"));
+        assertThat(request.requests().get(1).getRequest().indices()[1], equalTo("test3"));
         assertThat(request.requests().get(1).getRequest().types()[0], equalTo("type1"));
         assertThat(request.requests().get(1).getRequest().requestCache(), nullValue());
         assertThat(request.requests().get(1).getRequest().preference(), equalTo("_local"));
-        assertThat(request.requests().get(2).indices()[0], equalTo("test4"));
-        assertThat(request.requests().get(2).indices()[1], equalTo("test1"));
+        assertThat(request.requests().get(2).getRequest().indices()[0], equalTo("test4"));
+        assertThat(request.requests().get(2).getRequest().indices()[1], equalTo("test1"));
         assertThat(request.requests().get(2).getRequest().types()[0], equalTo("type2"));
         assertThat(request.requests().get(2).getRequest().types()[1], equalTo("type1"));
         assertThat(request.requests().get(2).getRequest().routing(), equalTo("123"));
@@ -58,9 +60,9 @@ public void testParseRequest() throws Exception {
         assertNotNull(request.requests().get(1).getScript());
         assertNotNull(request.requests().get(2).getScript());
 
-        assertEquals(ScriptService.ScriptType.INLINE, request.requests().get(0).getScriptType());
-        assertEquals(ScriptService.ScriptType.INLINE, request.requests().get(1).getScriptType());
-        assertEquals(ScriptService.ScriptType.INLINE, request.requests().get(2).getScriptType());
+        assertEquals(ScriptType.INLINE, request.requests().get(0).getScriptType());
+        assertEquals(ScriptType.INLINE, request.requests().get(1).getScriptType());
+        assertEquals(ScriptType.INLINE, request.requests().get(2).getScriptType());
         assertEquals("{\"query\":{\"match_{{template}}\":{}}}", request.requests().get(0).getScript());
         assertEquals("{\"query\":{\"match_{{template}}\":{}}}", request.requests().get(1).getScript());
         assertEquals("{\"query\":{\"match_{{template}}\":{}}}", request.requests().get(2).getScript());
@@ -68,4 +70,32 @@ public void testParseRequest() throws Exception {
         assertEquals(1, request.requests().get(1).getScriptParams().size());
         assertEquals(1, request.requests().get(2).getScriptParams().size());
     }
+
+    public void testParseWithCarriageReturn() throws Exception {
+        final String content = "{\"index\":[\"test0\", \"test1\"], \"request_cache\": true}\r\n" +
+            "{\"source\": {\"query\" : {\"match_{{template}}\" :{}}}, \"params\": {\"template\": \"all\" } }\r\n";
+        RestRequest restRequest = new FakeRestRequest.Builder(xContentRegistry())
+            .withContent(new BytesArray(content), XContentType.JSON).build();
+
+        MultiSearchTemplateRequest request = RestMultiSearchTemplateAction.parseRequest(restRequest, true);
+
+        assertThat(request.requests().size(), equalTo(1));
+        assertThat(request.requests().get(0).getRequest().indices()[0], equalTo("test0"));
+        assertThat(request.requests().get(0).getRequest().indices()[1], equalTo("test1"));
+        assertThat(request.requests().get(0).getRequest().indices(), arrayContaining("test0", "test1"));
+        assertThat(request.requests().get(0).getRequest().requestCache(), equalTo(true));
+        assertThat(request.requests().get(0).getRequest().preference(), nullValue());
+        assertNotNull(request.requests().get(0).getScript());
+        assertEquals(ScriptType.INLINE, request.requests().get(0).getScriptType());
+        assertEquals("{\"query\":{\"match_{{template}}\":{}}}", request.requests().get(0).getScript());
+        assertEquals(1, request.requests().get(0).getScriptParams().size());
+    }
+
+    public void testMaxConcurrentSearchRequests() {
+        MultiSearchTemplateRequest request = new MultiSearchTemplateRequest();
+        request.maxConcurrentSearchRequests(randomIntBetween(1, Integer.MAX_VALUE));
+        expectThrows(IllegalArgumentException.class, () ->
+                request.maxConcurrentSearchRequests(randomIntBetween(Integer.MIN_VALUE, 0)));
+    }
+
 }
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java
index 693ada174b924..8a59ca494feee 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java
@@ -19,15 +19,14 @@
 package org.elasticsearch.script.mustache;
 
 import com.github.mustachejava.MustacheFactory;
-import org.elasticsearch.common.ParseFieldMatcher;
+
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
@@ -48,8 +47,8 @@ public class MustacheScriptEngineTests extends ESTestCase {
 
     @Before
     public void setup() {
-        qe = new MustacheScriptEngineService(Settings.Builder.EMPTY_SETTINGS);
-        factory = new CustomMustacheFactory(true);
+        qe = new MustacheScriptEngineService();
+        factory = new CustomMustacheFactory();
     }
 
     public void testSimpleParameterReplace() {
@@ -59,7 +58,7 @@ public void testSimpleParameterReplace() {
                     + "\"negative\": {\"term\": {\"body\": {\"value\": \"solr\"}" + "}}, \"negative_boost\": {{boost_val}} } }}";
             Map<String, Object> vars = new HashMap<>();
             vars.put("boost_val", "0.3");
-            BytesReference o = (BytesReference) qe.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "", "mustache",
+            BytesReference o = (BytesReference) qe.executable(new CompiledScript(ScriptType.INLINE, "", "mustache",
                     qe.compile(null, template, compileParams)), vars).run();
             assertEquals("GET _search {\"query\": {\"boosting\": {\"positive\": {\"match\": {\"body\": \"gift\"}},"
                     + "\"negative\": {\"term\": {\"body\": {\"value\": \"solr\"}}}, \"negative_boost\": 0.3 } }}",
@@ -71,7 +70,7 @@ public void testSimpleParameterReplace() {
             Map<String, Object> vars = new HashMap<>();
             vars.put("boost_val", "0.3");
             vars.put("body_val", "\"quick brown\"");
-            BytesReference o = (BytesReference) qe.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "", "mustache",
+            BytesReference o = (BytesReference) qe.executable(new CompiledScript(ScriptType.INLINE, "", "mustache",
                     qe.compile(null, template, compileParams)), vars).run();
             assertEquals("GET _search {\"query\": {\"boosting\": {\"positive\": {\"match\": {\"body\": \"gift\"}},"
                     + "\"negative\": {\"term\": {\"body\": {\"value\": \"\\\"quick brown\\\"\"}}}, \"negative_boost\": 0.3 } }}",
@@ -80,22 +79,31 @@ public void testSimpleParameterReplace() {
     }
 
     public void testSimple() throws IOException {
-        String templateString = "{" + "\"inline\":{\"match_{{template}}\": {}}," + "\"params\":{\"template\":\"all\"}" + "}";
-        XContentParser parser = XContentFactory.xContent(templateString).createParser(templateString);
-        Script script = Script.parse(parser, new ParseFieldMatcher(false));
-        CompiledScript compiledScript = new CompiledScript(ScriptService.ScriptType.INLINE, null, "mustache",
-                qe.compile(null, script.getScript(), Collections.emptyMap()));
+        String templateString =
+                  "{" 
+                + "\"source\":{\"match_{{template}}\": {}},"
+                + "\"params\":{\"template\":\"all\"}"
+                + "}";
+        XContentParser parser = createParser(JsonXContent.jsonXContent, templateString);
+        Script script = Script.parse(parser);
+        CompiledScript compiledScript = new CompiledScript(ScriptType.INLINE, null, "mustache",
+                qe.compile(null, script.getIdOrCode(), Collections.emptyMap()));
         ExecutableScript executableScript = qe.executable(compiledScript, script.getParams());
         assertThat(((BytesReference) executableScript.run()).utf8ToString(), equalTo("{\"match_all\":{}}"));
     }
 
     public void testParseTemplateAsSingleStringWithConditionalClause() throws IOException {
-        String templateString = "{" + "  \"inline\" : \"{ \\\"match_{{#use_it}}{{template}}{{/use_it}}\\\":{} }\"," + "  \"params\":{"
-                + "    \"template\":\"all\"," + "    \"use_it\": true" + "  }" + "}";
-        XContentParser parser = XContentFactory.xContent(templateString).createParser(templateString);
-        Script script = Script.parse(parser, new ParseFieldMatcher(false));
-        CompiledScript compiledScript = new CompiledScript(ScriptService.ScriptType.INLINE, null, "mustache",
-                qe.compile(null, script.getScript(), Collections.emptyMap()));
+        String templateString =
+                  "{"
+                + "  \"source\" : \"{ \\\"match_{{#use_it}}{{template}}{{/use_it}}\\\":{} }\"," + "  \"params\":{"
+                + "    \"template\":\"all\","
+                + "    \"use_it\": true"
+                + "  }"
+                + "}";
+        XContentParser parser = createParser(JsonXContent.jsonXContent, templateString);
+        Script script = Script.parse(parser);
+        CompiledScript compiledScript = new CompiledScript(ScriptType.INLINE, null, "mustache",
+                qe.compile(null, script.getIdOrCode(), Collections.emptyMap()));
         ExecutableScript executableScript = qe.executable(compiledScript, script.getParams());
         assertThat(((BytesReference) executableScript.run()).utf8ToString(), equalTo("{ \"match_all\":{} }"));
     }
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java
index 9b48afe834a82..0d527590e478c 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java
@@ -20,8 +20,8 @@
 
 import com.github.mustachejava.Mustache;
 import com.github.mustachejava.MustacheException;
+
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.script.CompiledScript;
@@ -30,6 +30,8 @@
 import org.elasticsearch.test.ESTestCase;
 import org.hamcrest.Matcher;
 
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
@@ -42,9 +44,7 @@
 import static java.util.Collections.singleton;
 import static java.util.Collections.singletonMap;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.elasticsearch.script.ScriptService.ScriptType.INLINE;
-import static org.elasticsearch.script.mustache.MustacheScriptEngineService.CONTENT_TYPE_PARAM;
-import static org.elasticsearch.script.mustache.MustacheScriptEngineService.PLAIN_TEXT_CONTENT_TYPE;
+import static org.elasticsearch.script.ScriptType.INLINE;
 import static org.hamcrest.Matchers.both;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -55,7 +55,7 @@
 
 public class MustacheTests extends ESTestCase {
 
-    private ScriptEngineService engine = new MustacheScriptEngineService(Settings.EMPTY);
+    private ScriptEngineService engine = new MustacheScriptEngineService();
 
     public void testBasics() {
         String template = "GET _search {\"query\": " + "{\"boosting\": {"
@@ -144,24 +144,6 @@ public void testMapInArrayAccess() throws Exception {
         assertThat(bytes.utf8ToString(), both(containsString("foo")).and(containsString("bar")));
     }
 
-    public void testEscaping() {
-        // json string escaping enabled:
-        Mustache mustache = (Mustache) engine.compile(null, "{ \"field1\": \"{{value}}\"}", Collections.emptyMap());
-        CompiledScript compiledScript = new CompiledScript(INLINE, "name", "mustache", mustache);
-        ExecutableScript executableScript = engine.executable(compiledScript, Collections.singletonMap("value", "a \"value\""));
-        BytesReference rawResult = (BytesReference) executableScript.run();
-        String result = rawResult.utf8ToString();
-        assertThat(result, equalTo("{ \"field1\": \"a \\\"value\\\"\"}"));
-
-        // json string escaping disabled:
-        mustache = (Mustache) engine.compile(null, "{ \"field1\": \"{{value}}\"}",
-                Collections.singletonMap(CONTENT_TYPE_PARAM, PLAIN_TEXT_CONTENT_TYPE));
-        compiledScript = new CompiledScript(INLINE, "name", "mustache", mustache);
-        executableScript = engine.executable(compiledScript, Collections.singletonMap("value", "a \"value\""));
-        rawResult = (BytesReference) executableScript.run();
-        result = rawResult.utf8ToString();
-        assertThat(result, equalTo("{ \"field1\": \"a \"value\"\"}"));
-    }
 
     public void testSizeAccessForCollectionsAndArrays() throws Exception {
         String[] randomArrayValues = generateRandomStringArray(10, 20, false);
@@ -297,7 +279,8 @@ public void testEmbeddedToJSON() throws Exception {
                         .endArray()
                     .endObject();
 
-        Map<String, Object> ctx = Collections.singletonMap("ctx", XContentHelper.convertToMap(builder.bytes(), false).v2());
+        Map<String, Object> ctx =
+            Collections.singletonMap("ctx", XContentHelper.convertToMap(builder.bytes(), false, builder.contentType()).v2());
 
         assertScript("{{#ctx.bulks}}{{#toJson}}.{{/toJson}}{{/ctx.bulks}}", ctx,
                 equalTo("{\"index\":\"index-1\",\"id\":1,\"type\":\"type-1\"}{\"index\":\"index-2\",\"id\":2,\"type\":\"type-2\"}"));
@@ -338,7 +321,8 @@ public void testEmbeddedArrayJoin() throws Exception {
                                                     .endArray()
                                                 .endObject();
 
-        Map<String, Object> ctx = Collections.singletonMap("ctx", XContentHelper.convertToMap(builder.bytes(), false).v2());
+        Map<String, Object> ctx =
+            Collections.singletonMap("ctx", XContentHelper.convertToMap(builder.bytes(), false, builder.contentType()).v2());
 
         assertScript("{{#join}}ctx.people.0.emails{{/join}}", ctx,
                 equalTo("john@smith.com,john.smith@email.com,jsmith@email.com"));
@@ -375,6 +359,44 @@ public void testJoinWithCustomDelimiter() {
         assertScript("{{#join delimiter=' and '}}params{{/join delimiter=' and '}}", params, equalTo("1 and 2 and 3 and 4"));
     }
 
+    public void testUrlEncoder() {
+        Map<String, String> urls = new HashMap<>();
+        urls.put("https://www.elastic.co",
+                "https%3A%2F%2Fwww.elastic.co");
+        urls.put("<logstash-{now/d}>",
+                "%3Clogstash-%7Bnow%2Fd%7D%3E");
+        urls.put("?query=(foo:A OR baz:B) AND title:/joh?n(ath[oa]n)/ AND date:{* TO 2012-01}",
+                "%3Fquery%3D%28foo%3AA+OR+baz%3AB%29+AND+title%3A%2Fjoh%3Fn%28ath%5Boa%5Dn%29%2F+AND+date%3A%7B*+TO+2012-01%7D");
+
+        for (Map.Entry<String, String> url : urls.entrySet()) {
+            assertScript("{{#url}}{{params}}{{/url}}", singletonMap("params", url.getKey()), equalTo(url.getValue()));
+        }
+    }
+
+    public void testUrlEncoderWithParam() throws Exception {
+        assertScript("{{#url}}{{index}}{{/url}}", singletonMap("index", "<logstash-{now/d{YYYY.MM.dd|+12:00}}>"),
+                equalTo("%3Clogstash-%7Bnow%2Fd%7BYYYY.MM.dd%7C%2B12%3A00%7D%7D%3E"));
+
+        final String random = randomAlphaOfLength(10);
+        assertScript("{{#url}}prefix_{{s}}{{/url}}", singletonMap("s", random),
+                equalTo("prefix_" + URLEncoder.encode(random, StandardCharsets.UTF_8.name())));
+    }
+
+    public void testUrlEncoderWithJoin() {
+        Map<String, Object> params = singletonMap("emails", Arrays.asList("john@smith.com", "john.smith@email.com", "jsmith@email.com"));
+        assertScript("?query={{#url}}{{#join}}emails{{/join}}{{/url}}", params,
+                equalTo("?query=john%40smith.com%2Cjohn.smith%40email.com%2Cjsmith%40email.com"));
+
+        params = singletonMap("indices", new String[]{"<logstash-{now/d-2d}>", "<logstash-{now/d-1d}>", "<logstash-{now/d}>"});
+        assertScript("{{#url}}https://localhost:9200/{{#join}}indices{{/join}}/_stats{{/url}}", params,
+                equalTo("https%3A%2F%2Flocalhost%3A9200%2F%3Clogstash-%7Bnow%2Fd-2d%7D" +
+                        "%3E%2C%3Clogstash-%7Bnow%2Fd-1d%7D%3E%2C%3Clogstash-%7Bnow%2Fd%7D%3E%2F_stats"));
+
+        params = singletonMap("fibonacci", new int[]{1, 1, 2, 3, 5, 8, 13, 21, 34, 55});
+        assertScript("{{#url}}{{#join delimiter='+'}}fibonacci{{/join delimiter='+'}}{{/url}}", params,
+                equalTo("1%2B1%2B2%2B3%2B5%2B8%2B13%2B21%2B34%2B55"));
+    }
+
     private void assertScript(String script, Map<String, Object> vars, Matcher<Object> matcher) {
         Object result = engine.executable(new CompiledScript(INLINE, "inline", "mustache", compile(script)), vars).run();
         assertThat(result, notNullValue());
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateIT.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateIT.java
index c3656029bc4c5..d37d508fee279 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateIT.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateIT.java
@@ -24,9 +24,10 @@
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.junit.Before;
 
@@ -94,9 +95,13 @@ public void testSearchRequestFail() throws Exception {
     public void testTemplateQueryAsEscapedString() throws Exception {
         SearchRequest searchRequest = new SearchRequest();
         searchRequest.indices("_all");
-        String query = "{" + "  \"inline\" : \"{ \\\"size\\\": \\\"{{size}}\\\", \\\"query\\\":{\\\"match_all\\\":{}}}\","
-                + "  \"params\":{" + "    \"size\": 1" + "  }" + "}";
-        SearchTemplateRequest request = RestSearchTemplateAction.parse(new BytesArray(query));
+        String query =
+                  "{" + "  \"source\" : \"{ \\\"size\\\": \\\"{{size}}\\\", \\\"query\\\":{\\\"match_all\\\":{}}}\","
+                + "  \"params\":{"
+                + "    \"size\": 1"
+                + "  }"
+                + "}";
+        SearchTemplateRequest request = RestSearchTemplateAction.parse(createParser(JsonXContent.jsonXContent, query));
         request.setRequest(searchRequest);
         SearchTemplateResponse searchResponse = client().execute(SearchTemplateAction.INSTANCE, request).get();
         assertThat(searchResponse.getResponse().getHits().hits().length, equalTo(1));
@@ -109,10 +114,15 @@ public void testTemplateQueryAsEscapedString() throws Exception {
     public void testTemplateQueryAsEscapedStringStartingWithConditionalClause() throws Exception {
         SearchRequest searchRequest = new SearchRequest();
         searchRequest.indices("_all");
-        String templateString = "{"
-                + "  \"inline\" : \"{ {{#use_size}} \\\"size\\\": \\\"{{size}}\\\", {{/use_size}} \\\"query\\\":{\\\"match_all\\\":{}}}\","
-                + "  \"params\":{" + "    \"size\": 1," + "    \"use_size\": true" + "  }" + "}";
-        SearchTemplateRequest request = RestSearchTemplateAction.parse(new BytesArray(templateString));
+        String templateString =
+                  "{"
+                + "  \"source\" : \"{ {{#use_size}} \\\"size\\\": \\\"{{size}}\\\", {{/use_size}} \\\"query\\\":{\\\"match_all\\\":{}}}\","
+                + "  \"params\":{"
+                + "    \"size\": 1,"
+                + "    \"use_size\": true"
+                + "  }"
+                + "}";
+        SearchTemplateRequest request = RestSearchTemplateAction.parse(createParser(JsonXContent.jsonXContent, templateString));
         request.setRequest(searchRequest);
         SearchTemplateResponse searchResponse = client().execute(SearchTemplateAction.INSTANCE, request).get();
         assertThat(searchResponse.getResponse().getHits().hits().length, equalTo(1));
@@ -125,10 +135,15 @@ public void testTemplateQueryAsEscapedStringStartingWithConditionalClause() thro
     public void testTemplateQueryAsEscapedStringWithConditionalClauseAtEnd() throws Exception {
         SearchRequest searchRequest = new SearchRequest();
         searchRequest.indices("_all");
-        String templateString = "{"
-                + "  \"inline\" : \"{ \\\"query\\\":{\\\"match_all\\\":{}} {{#use_size}}, \\\"size\\\": \\\"{{size}}\\\" {{/use_size}} }\","
-                + "  \"params\":{" + "    \"size\": 1," + "    \"use_size\": true" + "  }" + "}";
-        SearchTemplateRequest request = RestSearchTemplateAction.parse(new BytesArray(templateString));
+        String templateString =
+                  "{"
+                + "  \"source\" : \"{ \\\"query\\\":{\\\"match_all\\\":{}} {{#use_size}}, \\\"size\\\": \\\"{{size}}\\\" {{/use_size}} }\","
+                + "  \"params\":{"
+                + "    \"size\": 1,"
+                + "    \"use_size\": true"
+                + "  }"
+                + "}";
+        SearchTemplateRequest request = RestSearchTemplateAction.parse(createParser(JsonXContent.jsonXContent, templateString));
         request.setRequest(searchRequest);
         SearchTemplateResponse searchResponse = client().execute(SearchTemplateAction.INSTANCE, request).get();
         assertThat(searchResponse.getResponse().getHits().hits().length, equalTo(1));
@@ -136,39 +151,39 @@ public void testTemplateQueryAsEscapedStringWithConditionalClauseAtEnd() throws
 
     public void testIndexedTemplateClient() throws Exception {
         assertAcked(client().admin().cluster().preparePutStoredScript()
-                .setScriptLang(MustacheScriptEngineService.NAME)
+                .setLang(MustacheScriptEngineService.NAME)
                 .setId("testTemplate")
-                .setSource(new BytesArray("{" +
+                .setContent(new BytesArray("{" +
                         "\"template\":{" +
                         "                \"query\":{" +
                         "                   \"match\":{" +
                         "                    \"theField\" : \"{{fieldParam}}\"}" +
                         "       }" +
                         "}" +
-                        "}")));
+                        "}"), XContentType.JSON));
 
 
         assertAcked(client().admin().cluster().preparePutStoredScript()
-                .setScriptLang(MustacheScriptEngineService.NAME)
-                .setId("testTemplate").setSource(new BytesArray("{" +
+                .setLang(MustacheScriptEngineService.NAME)
+                .setId("testTemplate").setContent(new BytesArray("{" +
                         "\"template\":{" +
                         "                \"query\":{" +
                         "                   \"match\":{" +
                         "                    \"theField\" : \"{{fieldParam}}\"}" +
                         "       }" +
                         "}" +
-                        "}")));
+                        "}"), XContentType.JSON));
 
         GetStoredScriptResponse getResponse = client().admin().cluster()
                 .prepareGetStoredScript(MustacheScriptEngineService.NAME, "testTemplate").get();
-        assertNotNull(getResponse.getStoredScript());
+        assertNotNull(getResponse.getSource());
 
         BulkRequestBuilder bulkRequestBuilder = client().prepareBulk();
-        bulkRequestBuilder.add(client().prepareIndex("test", "type", "1").setSource("{\"theField\":\"foo\"}"));
-        bulkRequestBuilder.add(client().prepareIndex("test", "type", "2").setSource("{\"theField\":\"foo 2\"}"));
-        bulkRequestBuilder.add(client().prepareIndex("test", "type", "3").setSource("{\"theField\":\"foo 3\"}"));
-        bulkRequestBuilder.add(client().prepareIndex("test", "type", "4").setSource("{\"theField\":\"foo 4\"}"));
-        bulkRequestBuilder.add(client().prepareIndex("test", "type", "5").setSource("{\"theField\":\"bar\"}"));
+        bulkRequestBuilder.add(client().prepareIndex("test", "type", "1").setSource("{\"theField\":\"foo\"}", XContentType.JSON));
+        bulkRequestBuilder.add(client().prepareIndex("test", "type", "2").setSource("{\"theField\":\"foo 2\"}", XContentType.JSON));
+        bulkRequestBuilder.add(client().prepareIndex("test", "type", "3").setSource("{\"theField\":\"foo 3\"}", XContentType.JSON));
+        bulkRequestBuilder.add(client().prepareIndex("test", "type", "4").setSource("{\"theField\":\"foo 4\"}", XContentType.JSON));
+        bulkRequestBuilder.add(client().prepareIndex("test", "type", "5").setSource("{\"theField\":\"bar\"}", XContentType.JSON));
         bulkRequestBuilder.get();
         client().admin().indices().prepareRefresh().get();
 
@@ -186,20 +201,20 @@ public void testIndexedTemplateClient() throws Exception {
 
         getResponse = client().admin().cluster()
                 .prepareGetStoredScript(MustacheScriptEngineService.NAME, "testTemplate").get();
-        assertNull(getResponse.getStoredScript());
+        assertNull(getResponse.getSource());
 
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new SearchTemplateRequestBuilder(client())
                 .setRequest(new SearchRequest("test").types("type"))
                 .setScript("/template_index/mustache/1000").setScriptType(ScriptType.STORED).setScriptParams(templateParams)
                 .get());
-        assertThat(e.getMessage(), containsString("Illegal index script format [/template_index/mustache/1000] should be /lang/id"));
+        assertThat(e.getMessage(), containsString("illegal stored script format [/template_index/mustache/1000] use only <id>"));
     }
 
     public void testIndexedTemplate() throws Exception {
         assertAcked(client().admin().cluster().preparePutStoredScript()
-                .setScriptLang(MustacheScriptEngineService.NAME)
+                .setLang(MustacheScriptEngineService.NAME)
                 .setId("1a")
-                .setSource(new BytesArray("{" +
+                .setContent(new BytesArray("{" +
                         "\"template\":{" +
                         "                \"query\":{" +
                         "                   \"match\":{" +
@@ -207,37 +222,37 @@ public void testIndexedTemplate() throws Exception {
                         "       }" +
                         "}" +
                         "}"
-                ))
+                ), XContentType.JSON)
         );
         assertAcked(client().admin().cluster().preparePutStoredScript()
-                .setScriptLang(MustacheScriptEngineService.NAME)
+                .setLang(MustacheScriptEngineService.NAME)
                 .setId("2")
-                .setSource(new BytesArray("{" +
+                .setContent(new BytesArray("{" +
                         "\"template\":{" +
                         "                \"query\":{" +
                         "                   \"match\":{" +
                         "                    \"theField\" : \"{{fieldParam}}\"}" +
                         "       }" +
                         "}" +
-                        "}"))
+                        "}"), XContentType.JSON)
         );
         assertAcked(client().admin().cluster().preparePutStoredScript()
-                .setScriptLang(MustacheScriptEngineService.NAME)
+                .setLang(MustacheScriptEngineService.NAME)
                 .setId("3")
-                .setSource(new BytesArray("{" +
+                .setContent(new BytesArray("{" +
                         "\"template\":{" +
                         "             \"match\":{" +
                         "                    \"theField\" : \"{{fieldParam}}\"}" +
                         "       }" +
-                        "}"))
+                        "}"), XContentType.JSON)
         );
 
         BulkRequestBuilder bulkRequestBuilder = client().prepareBulk();
-        bulkRequestBuilder.add(client().prepareIndex("test", "type", "1").setSource("{\"theField\":\"foo\"}"));
-        bulkRequestBuilder.add(client().prepareIndex("test", "type", "2").setSource("{\"theField\":\"foo 2\"}"));
-        bulkRequestBuilder.add(client().prepareIndex("test", "type", "3").setSource("{\"theField\":\"foo 3\"}"));
-        bulkRequestBuilder.add(client().prepareIndex("test", "type", "4").setSource("{\"theField\":\"foo 4\"}"));
-        bulkRequestBuilder.add(client().prepareIndex("test", "type", "5").setSource("{\"theField\":\"bar\"}"));
+        bulkRequestBuilder.add(client().prepareIndex("test", "type", "1").setSource("{\"theField\":\"foo\"}", XContentType.JSON));
+        bulkRequestBuilder.add(client().prepareIndex("test", "type", "2").setSource("{\"theField\":\"foo 2\"}", XContentType.JSON));
+        bulkRequestBuilder.add(client().prepareIndex("test", "type", "3").setSource("{\"theField\":\"foo 3\"}", XContentType.JSON));
+        bulkRequestBuilder.add(client().prepareIndex("test", "type", "4").setSource("{\"theField\":\"foo 4\"}", XContentType.JSON));
+        bulkRequestBuilder.add(client().prepareIndex("test", "type", "5").setSource("{\"theField\":\"bar\"}", XContentType.JSON));
         bulkRequestBuilder.get();
         client().admin().indices().prepareRefresh().get();
 
@@ -246,7 +261,7 @@ public void testIndexedTemplate() throws Exception {
 
         SearchTemplateResponse searchResponse = new SearchTemplateRequestBuilder(client())
                 .setRequest(new SearchRequest().indices("test").types("type"))
-                .setScript("/mustache/1a")
+                .setScript("1a")
                 .setScriptType(ScriptType.STORED)
                 .setScriptParams(templateParams)
                 .get();
@@ -269,17 +284,20 @@ public void testIndexedTemplate() throws Exception {
         templateParams.put("fieldParam", "bar");
         searchResponse = new SearchTemplateRequestBuilder(client())
                 .setRequest(new SearchRequest("test").types("type"))
-                .setScript("/mustache/2").setScriptType(ScriptService.ScriptType.STORED).setScriptParams(templateParams)
+                .setScript("/mustache/2").setScriptType(ScriptType.STORED).setScriptParams(templateParams)
                 .get();
         assertHitCount(searchResponse.getResponse(), 1);
+        assertWarnings("use of </lang/id> [/mustache/2] for looking up" +
+            " stored scripts/templates has been deprecated, use only <id> [2] instead");
 
         Map<String, Object> vars = new HashMap<>();
         vars.put("fieldParam", "bar");
 
-        TemplateQueryBuilder builder = new TemplateQueryBuilder("3", ScriptService.ScriptType.STORED, vars);
+        TemplateQueryBuilder builder = new TemplateQueryBuilder("3", ScriptType.STORED, vars);
         SearchResponse sr = client().prepareSearch().setQuery(builder)
                 .execute().actionGet();
         assertHitCount(sr, 1);
+        assertWarnings("[template] query is deprecated, use search template api instead");
     }
 
     // Relates to #10397
@@ -295,35 +313,37 @@ public void testIndexedTemplateOverwrite() throws Exception {
         int iterations = randomIntBetween(2, 11);
         for (int i = 1; i < iterations; i++) {
             assertAcked(client().admin().cluster().preparePutStoredScript()
-                    .setScriptLang(MustacheScriptEngineService.NAME)
+                    .setLang(MustacheScriptEngineService.NAME)
                     .setId("git01")
-                    .setSource(new BytesArray("{\"template\":{\"query\": {\"match\": {\"searchtext\": {\"query\": \"{{P_Keyword1}}\"," +
-                            "\"type\": \"ooophrase_prefix\"}}}}}")));
+                    .setContent(new BytesArray("{\"template\":{\"query\": {\"match\": {\"searchtext\": {\"query\": \"{{P_Keyword1}}\"," +
+                            "\"type\": \"ooophrase_prefix\"}}}}}"), XContentType.JSON));
 
             GetStoredScriptResponse getResponse = client().admin().cluster()
                     .prepareGetStoredScript(MustacheScriptEngineService.NAME, "git01").get();
-            assertNotNull(getResponse.getStoredScript());
+            assertNotNull(getResponse.getSource());
 
             Map<String, Object> templateParams = new HashMap<>();
             templateParams.put("P_Keyword1", "dev");
 
             ParsingException e = expectThrows(ParsingException.class, () -> new SearchTemplateRequestBuilder(client())
                     .setRequest(new SearchRequest("testindex").types("test"))
-                    .setScript("git01").setScriptType(ScriptService.ScriptType.STORED).setScriptParams(templateParams)
+                    .setScript("git01").setScriptType(ScriptType.STORED).setScriptParams(templateParams)
                     .get());
             assertThat(e.getMessage(), containsString("[match] query does not support type ooophrase_prefix"));
+            assertWarnings("Deprecated field [type] used, replaced by [match_phrase and match_phrase_prefix query]");
 
             assertAcked(client().admin().cluster().preparePutStoredScript()
-                    .setScriptLang(MustacheScriptEngineService.NAME)
+                    .setLang(MustacheScriptEngineService.NAME)
                     .setId("git01")
-                    .setSource(new BytesArray("{\"query\": {\"match\": {\"searchtext\": {\"query\": \"{{P_Keyword1}}\"," +
-                            "\"type\": \"phrase_prefix\"}}}}")));
+                    .setContent(new BytesArray("{\"query\": {\"match\": {\"searchtext\": {\"query\": \"{{P_Keyword1}}\"," +
+                            "\"type\": \"phrase_prefix\"}}}}"), XContentType.JSON));
 
             SearchTemplateResponse searchResponse = new SearchTemplateRequestBuilder(client())
                     .setRequest(new SearchRequest("testindex").types("test"))
-                    .setScript("git01").setScriptType(ScriptService.ScriptType.STORED).setScriptParams(templateParams)
+                    .setScript("git01").setScriptType(ScriptType.STORED).setScriptParams(templateParams)
                     .get();
             assertHitCount(searchResponse.getResponse(), 1);
+            assertWarnings("Deprecated field [type] used, replaced by [match_phrase and match_phrase_prefix query]");
         }
     }
 
@@ -331,16 +351,16 @@ public void testIndexedTemplateWithArray() throws Exception {
         String multiQuery = "{\"query\":{\"terms\":{\"theField\":[\"{{#fieldParam}}\",\"{{.}}\",\"{{/fieldParam}}\"]}}}";
         assertAcked(
                 client().admin().cluster().preparePutStoredScript()
-                        .setScriptLang(MustacheScriptEngineService.NAME)
+                        .setLang(MustacheScriptEngineService.NAME)
                         .setId("4")
-                        .setSource(jsonBuilder().startObject().field("template", multiQuery).endObject().bytes())
+                        .setContent(jsonBuilder().startObject().field("template", multiQuery).endObject().bytes(), XContentType.JSON)
         );
         BulkRequestBuilder bulkRequestBuilder = client().prepareBulk();
-        bulkRequestBuilder.add(client().prepareIndex("test", "type", "1").setSource("{\"theField\":\"foo\"}"));
-        bulkRequestBuilder.add(client().prepareIndex("test", "type", "2").setSource("{\"theField\":\"foo 2\"}"));
-        bulkRequestBuilder.add(client().prepareIndex("test", "type", "3").setSource("{\"theField\":\"foo 3\"}"));
-        bulkRequestBuilder.add(client().prepareIndex("test", "type", "4").setSource("{\"theField\":\"foo 4\"}"));
-        bulkRequestBuilder.add(client().prepareIndex("test", "type", "5").setSource("{\"theField\":\"bar\"}"));
+        bulkRequestBuilder.add(client().prepareIndex("test", "type", "1").setSource("{\"theField\":\"foo\"}", XContentType.JSON));
+        bulkRequestBuilder.add(client().prepareIndex("test", "type", "2").setSource("{\"theField\":\"foo 2\"}", XContentType.JSON));
+        bulkRequestBuilder.add(client().prepareIndex("test", "type", "3").setSource("{\"theField\":\"foo 3\"}", XContentType.JSON));
+        bulkRequestBuilder.add(client().prepareIndex("test", "type", "4").setSource("{\"theField\":\"foo 4\"}", XContentType.JSON));
+        bulkRequestBuilder.add(client().prepareIndex("test", "type", "5").setSource("{\"theField\":\"bar\"}", XContentType.JSON));
         bulkRequestBuilder.get();
         client().admin().indices().prepareRefresh().get();
 
@@ -350,7 +370,7 @@ public void testIndexedTemplateWithArray() throws Exception {
 
         SearchTemplateResponse searchResponse = new SearchTemplateRequestBuilder(client())
                 .setRequest(new SearchRequest("test").types("type"))
-                .setScript("/mustache/4").setScriptType(ScriptService.ScriptType.STORED).setScriptParams(arrayTemplateParams)
+                .setScript("4").setScriptType(ScriptType.STORED).setScriptParams(arrayTemplateParams)
                 .get();
         assertHitCount(searchResponse.getResponse(), 5);
     }
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateRequestTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateRequestTests.java
index b5831a3c5a8d7..b958a4156edbe 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateRequestTests.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateRequestTests.java
@@ -20,11 +20,12 @@
 package org.elasticsearch.script.mustache;
 
 import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.test.ESTestCase;
 
+import java.io.IOException;
 import java.util.List;
 
 import static org.hamcrest.Matchers.equalTo;
@@ -37,7 +38,7 @@ public class SearchTemplateRequestTests extends ESTestCase {
 
     public void testParseInlineTemplate() throws Exception {
         String source = "{" +
-                "    'inline' : {\n" +
+                "    'source' : {\n" +
                 "    'query': {\n" +
                 "      'terms': {\n" +
                 "        'status': [\n" +
@@ -50,15 +51,15 @@ public void testParseInlineTemplate() throws Exception {
                 "  }" +
                 "}";
 
-        SearchTemplateRequest request = RestSearchTemplateAction.parse(newBytesReference(source));
+        SearchTemplateRequest request = RestSearchTemplateAction.parse(newParser(source));
         assertThat(request.getScript(), equalTo("{\"query\":{\"terms\":{\"status\":[\"{{#status}}\",\"{{.}}\",\"{{/status}}\"]}}}"));
-        assertThat(request.getScriptType(), equalTo(ScriptService.ScriptType.INLINE));
+        assertThat(request.getScriptType(), equalTo(ScriptType.INLINE));
         assertThat(request.getScriptParams(), nullValue());
     }
 
     public void testParseInlineTemplateWithParams() throws Exception {
         String source = "{" +
-                "    'inline' : {" +
+                "    'source' : {" +
                 "      'query': { 'match' : { '{{my_field}}' : '{{my_value}}' } }," +
                 "      'size' : '{{my_size}}'" +
                 "    }," +
@@ -69,9 +70,9 @@ public void testParseInlineTemplateWithParams() throws Exception {
                 "    }" +
                 "}";
 
-        SearchTemplateRequest request = RestSearchTemplateAction.parse(newBytesReference(source));
+        SearchTemplateRequest request = RestSearchTemplateAction.parse(newParser(source));
         assertThat(request.getScript(), equalTo("{\"query\":{\"match\":{\"{{my_field}}\":\"{{my_value}}\"}},\"size\":\"{{my_size}}\"}"));
-        assertThat(request.getScriptType(), equalTo(ScriptService.ScriptType.INLINE));
+        assertThat(request.getScriptType(), equalTo(ScriptType.INLINE));
         assertThat(request.getScriptParams().size(), equalTo(3));
         assertThat(request.getScriptParams(), hasEntry("my_field", "foo"));
         assertThat(request.getScriptParams(), hasEntry("my_value", "bar"));
@@ -79,22 +80,22 @@ public void testParseInlineTemplateWithParams() throws Exception {
     }
 
     public void testParseInlineTemplateAsString() throws Exception {
-        String source = "{'inline' : '{\\\"query\\\":{\\\"bool\\\":{\\\"must\\\":{\\\"match\\\":{\\\"foo\\\":\\\"{{text}}\\\"}}}}}'}";
+        String source = "{'source' : '{\\\"query\\\":{\\\"bool\\\":{\\\"must\\\":{\\\"match\\\":{\\\"foo\\\":\\\"{{text}}\\\"}}}}}'}";
 
-        SearchTemplateRequest request = RestSearchTemplateAction.parse(newBytesReference(source));
+        SearchTemplateRequest request = RestSearchTemplateAction.parse(newParser(source));
         assertThat(request.getScript(), equalTo("{\"query\":{\"bool\":{\"must\":{\"match\":{\"foo\":\"{{text}}\"}}}}}"));
-        assertThat(request.getScriptType(), equalTo(ScriptService.ScriptType.INLINE));
+        assertThat(request.getScriptType(), equalTo(ScriptType.INLINE));
         assertThat(request.getScriptParams(), nullValue());
     }
 
     @SuppressWarnings("unchecked")
     public void testParseInlineTemplateAsStringWithParams() throws Exception {
-        String source = "{'inline' : '{\\\"query\\\":{\\\"match\\\":{\\\"{{field}}\\\":\\\"{{value}}\\\"}}}', " +
+        String source = "{'source' : '{\\\"query\\\":{\\\"match\\\":{\\\"{{field}}\\\":\\\"{{value}}\\\"}}}', " +
                 "'params': {'status': ['pending', 'published']}}";
 
-        SearchTemplateRequest request = RestSearchTemplateAction.parse(newBytesReference(source));
+        SearchTemplateRequest request = RestSearchTemplateAction.parse(newParser(source));
         assertThat(request.getScript(), equalTo("{\"query\":{\"match\":{\"{{field}}\":\"{{value}}\"}}}"));
-        assertThat(request.getScriptType(), equalTo(ScriptService.ScriptType.INLINE));
+        assertThat(request.getScriptType(), equalTo(ScriptType.INLINE));
         assertThat(request.getScriptParams().size(), equalTo(1));
         assertThat(request.getScriptParams(), hasKey("status"));
         assertThat((List<String>) request.getScriptParams().get("status"), hasItems("pending", "published"));
@@ -103,18 +104,18 @@ public void testParseInlineTemplateAsStringWithParams() throws Exception {
     public void testParseFileTemplate() throws Exception {
         String source = "{'file' : 'fileTemplate'}";
 
-        SearchTemplateRequest request = RestSearchTemplateAction.parse(newBytesReference(source));
+        SearchTemplateRequest request = RestSearchTemplateAction.parse(newParser(source));
         assertThat(request.getScript(), equalTo("fileTemplate"));
-        assertThat(request.getScriptType(), equalTo(ScriptService.ScriptType.FILE));
+        assertThat(request.getScriptType(), equalTo(ScriptType.FILE));
         assertThat(request.getScriptParams(), nullValue());
     }
 
     public void testParseFileTemplateWithParams() throws Exception {
         String source = "{'file' : 'template_foo', 'params' : {'foo': 'bar', 'size': 500}}";
 
-        SearchTemplateRequest request = RestSearchTemplateAction.parse(newBytesReference(source));
+        SearchTemplateRequest request = RestSearchTemplateAction.parse(newParser(source));
         assertThat(request.getScript(), equalTo("template_foo"));
-        assertThat(request.getScriptType(), equalTo(ScriptService.ScriptType.FILE));
+        assertThat(request.getScriptType(), equalTo(ScriptType.FILE));
         assertThat(request.getScriptParams().size(), equalTo(2));
         assertThat(request.getScriptParams(), hasEntry("foo", "bar"));
         assertThat(request.getScriptParams(), hasEntry("size", 500));
@@ -123,32 +124,32 @@ public void testParseFileTemplateWithParams() throws Exception {
     public void testParseStoredTemplate() throws Exception {
         String source = "{'id' : 'storedTemplate'}";
 
-        SearchTemplateRequest request = RestSearchTemplateAction.parse(newBytesReference(source));
+        SearchTemplateRequest request = RestSearchTemplateAction.parse(newParser(source));
         assertThat(request.getScript(), equalTo("storedTemplate"));
-        assertThat(request.getScriptType(), equalTo(ScriptService.ScriptType.STORED));
+        assertThat(request.getScriptType(), equalTo(ScriptType.STORED));
         assertThat(request.getScriptParams(), nullValue());
     }
 
     public void testParseStoredTemplateWithParams() throws Exception {
         String source = "{'id' : 'another_template', 'params' : {'bar': 'foo'}}";
 
-        SearchTemplateRequest request = RestSearchTemplateAction.parse(newBytesReference(source));
+        SearchTemplateRequest request = RestSearchTemplateAction.parse(newParser(source));
         assertThat(request.getScript(), equalTo("another_template"));
-        assertThat(request.getScriptType(), equalTo(ScriptService.ScriptType.STORED));
+        assertThat(request.getScriptType(), equalTo(ScriptType.STORED));
         assertThat(request.getScriptParams().size(), equalTo(1));
         assertThat(request.getScriptParams(), hasEntry("bar", "foo"));
     }
 
     public void testParseWrongTemplate() {
         // Unclosed template id
-        expectThrows(ParsingException.class, () -> RestSearchTemplateAction.parse(newBytesReference("{'id' : 'another_temp }")));
+        expectThrows(ParsingException.class, () -> RestSearchTemplateAction.parse(newParser("{'id' : 'another_temp }")));
     }
 
     /**
-     * Creates a {@link BytesReference} with the given string while replacing single quote to double quotes.
+     * Creates a {@link XContentParser} with the given String while replacing single quote to double quotes.
      */
-    private static BytesReference newBytesReference(String s) {
+    private XContentParser newParser(String s) throws IOException {
         assertNotNull(s);
-        return new BytesArray(s.replace("'", "\""));
+        return createParser(JsonXContent.jsonXContent, s.replace("'", "\""));
     }
 }
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateWithoutContentIT.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateWithoutContentIT.java
new file mode 100644
index 0000000000000..cbc6adf6be227
--- /dev/null
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateWithoutContentIT.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.script.mustache;
+
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.test.rest.ESRestTestCase;
+
+import java.io.IOException;
+
+import static org.hamcrest.CoreMatchers.containsString;
+
+public class SearchTemplateWithoutContentIT extends ESRestTestCase {
+
+    public void testSearchTemplateMissingBody() throws IOException {
+        ResponseException responseException = expectThrows(ResponseException.class, () -> client().performRequest(
+                randomBoolean() ? "POST" : "GET", "/_search/template"));
+        assertEquals(400, responseException.getResponse().getStatusLine().getStatusCode());
+        assertThat(responseException.getMessage(), containsString("request body or source parameter is required"));
+    }
+
+    public void testMultiSearchTemplateMissingBody() throws IOException {
+        ResponseException responseException = expectThrows(ResponseException.class, () -> client().performRequest(
+                randomBoolean() ? "POST" : "GET", "/_msearch/template"));
+        assertEquals(400, responseException.getResponse().getStatusLine().getStatusCode());
+        assertThat(responseException.getMessage(), containsString("request body or source parameter is required"));
+    }
+}
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/TemplateQueryBuilderTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/TemplateQueryBuilderTests.java
index dd5c0a1832847..a0f7a57030d5a 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/TemplateQueryBuilderTests.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/TemplateQueryBuilderTests.java
@@ -22,8 +22,8 @@
 import org.apache.lucene.index.memory.MemoryIndex;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
@@ -36,8 +36,10 @@
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.MockScriptPlugin;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.script.ScriptType;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
+import org.junit.After;
 import org.junit.Before;
 
 import java.io.IOException;
@@ -48,6 +50,8 @@
 import java.util.Map;
 import java.util.function.Function;
 
+import static org.hamcrest.Matchers.containsString;
+
 public class TemplateQueryBuilderTests extends AbstractQueryTestCase<TemplateQueryBuilder> {
 
     /**
@@ -55,6 +59,14 @@ public class TemplateQueryBuilderTests extends AbstractQueryTestCase<TemplateQue
      */
     private QueryBuilder templateBase;
 
+    /**
+     * All tests in this class cause deprecation warnings when a new {@link TemplateQueryBuilder} is created.
+     * Instead of having to check them in every single test, we do it after each test is run
+     */
+    @After void assertWarnings() {
+        assertWarnings("[template] query is deprecated, use search template api instead");
+    }
+
     @Override
     protected Collection<Class<? extends Plugin>> getPlugins() {
         return Arrays.asList(MustachePlugin.class, CustomScriptPlugin.class);
@@ -93,7 +105,7 @@ protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
     }
 
     @Before
-    public void before() {
+    public void setup() {
         templateBase = new MatchQueryBuilder("field", "some values");
     }
 
@@ -104,12 +116,13 @@ protected boolean supportsBoostAndQueryName() {
 
     @Override
     protected TemplateQueryBuilder doCreateTestQueryBuilder() {
-        return new TemplateQueryBuilder(new Script(templateBase.toString(), ScriptType.INLINE, "mustache", null, null));
+        return new TemplateQueryBuilder(new Script(ScriptType.INLINE, "mustache", templateBase.toString(), Collections.emptyMap()));
     }
 
     @Override
-    protected void doAssertLuceneQuery(TemplateQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
-        assertEquals(rewrite(QueryBuilder.rewriteQuery(templateBase, context).toQuery(context)), rewrite(query));
+    protected void doAssertLuceneQuery(TemplateQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
+        QueryShardContext queryShardContext = context.getQueryShardContext();
+        assertEquals(rewrite(QueryBuilder.rewriteQuery(templateBase, queryShardContext).toQuery(queryShardContext)), rewrite(query));
     }
 
     public void testIllegalArgument() {
@@ -123,14 +136,12 @@ public void testIllegalArgument() {
     @Override
     public void testUnknownField() throws IOException {
         TemplateQueryBuilder testQuery = createTestQueryBuilder();
-        String testQueryAsString = toXContent(testQuery, randomFrom(XContentType.JSON, XContentType.YAML)).string();
-        String queryAsString = testQueryAsString.replace("inline", "bogusField");
-        try {
-            parseQuery(queryAsString);
-            fail("ScriptParseException expected.");
-        } catch (ElasticsearchParseException e) {
-            assertTrue(e.getMessage().contains("bogusField"));
-        }
+        XContentType xContentType = randomFrom(XContentType.JSON, XContentType.YAML);
+        String testQueryAsString = toShuffledXContent(testQuery, xContentType, ToXContent.EMPTY_PARAMS, randomBoolean()).utf8ToString();
+        String queryAsString = testQueryAsString.replace("source", "bogusField");
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> parseQuery(createParser(xContentType.xContent(), queryAsString)));
+        assertThat(e.getMessage(), containsString("[script] unknown field [bogusField], parser not found"));
     }
 
     public void testJSONGeneration() throws IOException {
@@ -142,13 +153,13 @@ public void testJSONGeneration() throws IOException {
         builder.doXContent(content, null);
         content.endObject();
         content.close();
-        assertEquals("{\"template\":{\"inline\":\"I am a $template string\",\"lang\":\"mustache\",\"params\":{\"template\":\"filled\"}}}",
+        assertEquals("{\"template\":{\"source\":\"I am a $template string\",\"lang\":\"mustache\",\"params\":{\"template\":\"filled\"}}}",
                 content.string());
     }
 
     public void testRawEscapedTemplate() throws IOException {
         String expectedTemplateString = "{\"match_{{template}}\": {}}\"";
-        String query = "{\"template\": {\"inline\": \"{\\\"match_{{template}}\\\": {}}\\\"\",\"params\" : {\"template\" : \"all\"}}}";
+        String query = "{\"template\": {\"source\": \"{\\\"match_{{template}}\\\": {}}\\\"\",\"params\" : {\"template\" : \"all\"}}}";
         Map<String, Object> params = new HashMap<>();
         params.put("template", "all");
         QueryBuilder expectedBuilder = new TemplateQueryBuilder(expectedTemplateString, ScriptType.INLINE, params);
@@ -157,7 +168,7 @@ public void testRawEscapedTemplate() throws IOException {
 
     public void testRawTemplate() throws IOException {
         String expectedTemplateString = "{\"match_{{template}}\":{}}";
-        String query = "{\"template\": {\"inline\": {\"match_{{template}}\": {}},\"params\" : {\"template\" : \"all\"}}}";
+        String query = "{\"template\": {\"source\": {\"match_{{template}}\": {}},\"params\" : {\"template\" : \"all\"}}}";
         Map<String, Object> params = new HashMap<>();
         params.put("template", "all");
         QueryBuilder expectedBuilder = new TemplateQueryBuilder(expectedTemplateString, ScriptType.INLINE, params, XContentType.JSON);
@@ -167,8 +178,8 @@ public void testRawTemplate() throws IOException {
     @Override
     public void testMustRewrite() throws IOException {
         String query = "{ \"match_all\" : {}}";
-        QueryBuilder builder = new TemplateQueryBuilder(new Script(query, ScriptType.INLINE, "mockscript",
-        Collections.emptyMap(), XContentType.JSON));
+        QueryBuilder builder = new TemplateQueryBuilder(new Script(ScriptType.INLINE, "mockscript", query,
+            Collections.singletonMap(Script.CONTENT_TYPE_OPTION, XContentType.JSON.mediaType()), Collections.emptyMap()));
         try {
             builder.toQuery(createShardContext());
             fail();
@@ -180,24 +191,24 @@ public void testMustRewrite() throws IOException {
 
     public void testRewriteWithInnerName() throws IOException {
         final String query = "{ \"match_all\" : {\"_name\" : \"foobar\"}}";
-        QueryBuilder builder = new TemplateQueryBuilder(new Script(query, ScriptType.INLINE, "mockscript",
-             Collections.emptyMap(), XContentType.JSON));
+        QueryBuilder builder = new TemplateQueryBuilder(new Script(ScriptType.INLINE, "mockscript", query,
+            Collections.singletonMap(Script.CONTENT_TYPE_OPTION, XContentType.JSON.mediaType()), Collections.emptyMap()));
         assertEquals(new MatchAllQueryBuilder().queryName("foobar"), builder.rewrite(createShardContext()));
 
-        builder = new TemplateQueryBuilder(new Script(query, ScriptType.INLINE, "mockscript",
-            Collections.emptyMap(), XContentType.JSON)).queryName("outer");
+        builder = new TemplateQueryBuilder(new Script(ScriptType.INLINE, "mockscript", query, Collections.singletonMap(
+            Script.CONTENT_TYPE_OPTION, XContentType.JSON.mediaType()), Collections.emptyMap())).queryName("outer");
         assertEquals(new BoolQueryBuilder().must(new MatchAllQueryBuilder().queryName("foobar")).queryName("outer"),
             builder.rewrite(createShardContext()));
     }
 
     public void testRewriteWithInnerBoost() throws IOException {
         final TermQueryBuilder query = new TermQueryBuilder("foo", "bar").boost(2);
-        QueryBuilder builder = new TemplateQueryBuilder(new Script(query.toString(), ScriptType.INLINE, "mockscript",
-            Collections.emptyMap(), XContentType.JSON));
+        QueryBuilder builder = new TemplateQueryBuilder(new Script(ScriptType.INLINE, "mockscript", query.toString(),
+            Collections.singletonMap(Script.CONTENT_TYPE_OPTION, XContentType.JSON.mediaType()), Collections.emptyMap()));
         assertEquals(query, builder.rewrite(createShardContext()));
 
-        builder = new TemplateQueryBuilder(new Script(query.toString(), ScriptType.INLINE, "mockscript",
-            Collections.emptyMap(), XContentType.JSON)).boost(3);
+        builder = new TemplateQueryBuilder(new Script(ScriptType.INLINE, "mockscript", query.toString(),
+            Collections.singletonMap(Script.CONTENT_TYPE_OPTION, XContentType.JSON.mediaType()), Collections.emptyMap())).boost(3);
         assertEquals(new BoolQueryBuilder().must(query).boost(3), builder.rewrite(createShardContext()));
     }
 
diff --git a/modules/lang-mustache/src/test/resources/org/elasticsearch/script/mustache/simple-msearch-template.json b/modules/lang-mustache/src/test/resources/org/elasticsearch/script/mustache/simple-msearch-template.json
index ac99bc299b335..11a0091492c4d 100644
--- a/modules/lang-mustache/src/test/resources/org/elasticsearch/script/mustache/simple-msearch-template.json
+++ b/modules/lang-mustache/src/test/resources/org/elasticsearch/script/mustache/simple-msearch-template.json
@@ -1,6 +1,6 @@
 {"index":["test0", "test1"], "request_cache": true}
-{"inline": {"query" : {"match_{{template}}" :{}}}, "params": {"template": "all" } }
+{"source": {"query" : {"match_{{template}}" :{}}}, "params": {"template": "all" } }
 {"index" : "test2,test3", "type" : "type1", "preference": "_local"}
-{"inline": {"query" : {"match_{{template}}" :{}}}, "params": {"template": "all" } }
+{"source": {"query" : {"match_{{template}}" :{}}}, "params": {"template": "all" } }
 {"index" : ["test4", "test1"], "type" :  [ "type2", "type1" ], "routing": "123"}
-{"inline": {"query" : {"match_{{template}}" :{}}}, "params": {"template": "all" } }
+{"source": {"query" : {"match_{{template}}" :{}}}, "params": {"template": "all" } }
diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/10_basic.yaml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/10_basic.yaml
index 9533cde81acf3..0c61625a78c79 100644
--- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/10_basic.yaml
+++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/10_basic.yaml
@@ -13,15 +13,21 @@
     - match:  { nodes.$master.modules.0.name: lang-mustache  }
 
 ---
-"Indexed template":
+"Stored template":
+  - skip:
+      features: "warnings"
 
   - do:
+      warnings:
+        - "The stored search template API is deprecated. Use stored scripts instead."
       put_template:
         id: "1"
         body: { "template": { "query": { "match_all": {}}, "size": "{{my_size}}" } }
   - match: { acknowledged: true }
 
   - do:
+      warnings:
+        - "The stored search template API is deprecated. Use stored scripts instead."
       get_template:
         id: 1
   - match: { found: true }
@@ -30,6 +36,8 @@
   - match: { template: /.*query\S\S\S\Smatch_all.*/ }
 
   - do:
+      warnings:
+        - "The stored search template API is deprecated. Use stored scripts instead."
       catch: missing
       get_template:
         id: 2
@@ -38,23 +46,31 @@
   - match: { _id: "2" }
 
   - do:
+      warnings:
+        - "The stored search template API is deprecated. Use stored scripts instead."
       delete_template:
         id: "1"
   - match: { acknowledged: true }
 
   - do:
+      warnings:
+        - "The stored search template API is deprecated. Use stored scripts instead."
       catch: missing
       delete_template:
         id: "non_existing"
 
   - do:
+      warnings:
+        - "The stored search template API is deprecated. Use stored scripts instead."
       catch: request
       put_template:
         id: "1"
         body: { "template": { "query": { "match{{}}_all": {}}, "size": "{{my_size}}" } }
 
   - do:
-      catch: /Unable\sto\sparse.*/
+      warnings:
+        - "The stored search template API is deprecated. Use stored scripts instead."
+      catch: /failed\sto\sparse.*/
       put_template:
         id: "1"
         body: { "template": { "query": { "match{{}}_all": {}}, "size": "{{my_size}}" } }
diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/20_render_search_template.yaml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/20_render_search_template.yaml
index 9a9a1b0c01764..bce720e553c19 100644
--- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/20_render_search_template.yaml
+++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/20_render_search_template.yaml
@@ -2,9 +2,9 @@
 "Stored Template validate tests":
 
   - do:
-      put_template:
-        id: "1"
-        body: { "template": { "query": { "match": { "text": "{{my_value}}" } }, "aggs": { "my_terms": { "terms": { "field": "{{my_field}}" } } } } }
+      put_script:
+        lang: "1"
+        body: { "script": { "lang": "mustache", "source": { "query": { "match": { "text": "{{my_value}}" } }, "aggs": { "my_terms": { "terms": { "field": "{{my_field}}" } } } } } }
   - match: { acknowledged: true }
 
   - do:
@@ -51,14 +51,14 @@
 
   - do:
       render_search_template:
-        body: { "inline": { "query": { "match": { "text": "{{my_value}}" } }, "aggs": { "my_terms": { "terms": { "field": "{{my_field}}" } } } }, "params": { "my_value": "foo", "my_field": "field1" } }
+        body: { "source": { "query": { "match": { "text": "{{my_value}}" } }, "aggs": { "my_terms": { "terms": { "field": "{{my_field}}" } } } }, "params": { "my_value": "foo", "my_field": "field1" } }
 
   - match: { template_output.query.match.text: "foo" }
   - match: { template_output.aggs.my_terms.terms.field: "field1" }
 
   - do:
       render_search_template:
-        body: { "inline": { "query": { "match": { "text": "{{my_value}}" } }, "aggs": { "my_terms": { "terms": { "field": "{{my_field}}" } } } }, "params": { "my_value": "bar", "my_field": "my_other_field" } }
+        body: { "source": { "query": { "match": { "text": "{{my_value}}" } }, "aggs": { "my_terms": { "terms": { "field": "{{my_field}}" } } } }, "params": { "my_value": "bar", "my_field": "my_other_field" } }
 
   - match: { template_output.query.match.text: "bar" }
   - match: { template_output.aggs.my_terms.terms.field: "my_other_field" }
@@ -66,14 +66,14 @@
   - do:
       catch: /Improperly.closed.variable.in.query-template/
       render_search_template:
-        body: { "inline": { "query": { "match": { "text": "{{{my_value}}" } }, "aggs": { "my_terms": { "terms": { "field": "{{my_field}}" } } } }, "params": { "my_value": "bar", "my_field": "field1" } }
+        body: { "source": { "query": { "match": { "text": "{{{my_value}}" } }, "aggs": { "my_terms": { "terms": { "field": "{{my_field}}" } } } }, "params": { "my_value": "bar", "my_field": "field1" } }
 ---
 "Escaped Stored Template validate tests":
 
   - do:
-      put_template:
-        id: "1"
-        body: { "template": "{ \"query\": { \"match\": { \"text\": \"{{my_value}}\" } }, \"size\": {{my_size}} }" }
+      put_script:
+        lang: "1"
+        body: { "script": { "lang" : "mustache", "source" : "{ \"query\": { \"match\": { \"text\": \"{{my_value}}\" } }, \"size\": {{my_size}} }" } }
   - match: { acknowledged: true }
 
   - do:
@@ -103,14 +103,14 @@
 
   - do:
       render_search_template:
-        body: { "inline": "{ \"query\": { \"match\": { \"text\": \"{{my_value}}\" } }, \"size\": {{my_size}} }", "params": { "my_value": "foo", "my_size": 20 } }
+        body: { "source": "{ \"query\": { \"match\": { \"text\": \"{{my_value}}\" } }, \"size\": {{my_size}} }", "params": { "my_value": "foo", "my_size": 20 } }
 
   - match: { template_output.query.match.text: "foo" }
   - match: { template_output.size: 20 }
 
   - do:
       render_search_template:
-        body: { "inline": "{ \"query\": { \"match\": { \"text\": \"{{my_value}}\" } }, \"size\": {{my_size}} }", "params": { "my_value": "bar", "my_size": 100 } }
+        body: { "source": "{ \"query\": { \"match\": { \"text\": \"{{my_value}}\" } }, \"size\": {{my_size}} }", "params": { "my_value": "bar", "my_size": 100 } }
 
   - match: { template_output.query.match.text: "bar" }
   - match: { template_output.size: 100 }
@@ -118,7 +118,7 @@
   - do:
       catch: /Improperly.closed.variable.in.query-template/
       render_search_template:
-        body: { "inline": "{ \"query\": { \"match\": { \"text\": \"{{{my_value}}\" } }, \"size\": {{my_size}} }", "params": { "my_value": "bar", "my_size": 100 } }
+        body: { "source": "{ \"query\": { \"match\": { \"text\": \"{{{my_value}}\" } }, \"size\": {{my_size}} }", "params": { "my_value": "bar", "my_size": 100 } }
 
 ---
 "Indexed Template query tests":
@@ -139,9 +139,9 @@
       indices.refresh: {}
 
   - do:
-      put_template:
-        id: "1"
-        body: { "template": { "query": { "match" : { "text": "{{my_value}}" } }, "size": "{{my_size}}" } }
+      put_script:
+        lang: "1"
+        body: { "script": { "lang": "mustache", "source": { "query": { "match" : { "text": "{{my_value}}" } }, "size": "{{my_size}}" } } }
   - match: { acknowledged: true }
 
   - do:
@@ -150,7 +150,7 @@
   - match: { hits.total: 1 }
 
   - do:
-      catch: /Unable.to.find.on.disk.file.script.\[simple1\].using.lang.\[mustache\]/
+      catch: /unable.to.find.file.script.\[simple1\].using.lang.\[mustache\]/
       search_template:
         body: { "file" : "simple1"}
 
diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/25_custom_functions.yaml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/25_custom_functions.yaml
new file mode 100644
index 0000000000000..f58d40402b147
--- /dev/null
+++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/25_custom_functions.yaml
@@ -0,0 +1,43 @@
+---
+"Rendering using {{url}} function":
+
+  - do:
+      render_search_template:
+        body:  >
+          {
+            "source": {
+              "query": {
+                "match": {
+                  "url": "https://localhost:9200/{{#url}}{{index}}{{/url}}/{{#url}}{{type}}{{/url}}/_search"
+                }
+              }
+            },
+            "params": {
+              "index": "<logstash-{now/d-2d}>",
+              "type" : "métriques"
+            }
+          }
+
+  - match: { template_output.query.match.url: "https://localhost:9200/%3Clogstash-%7Bnow%2Fd-2d%7D%3E/m%C3%A9triques/_search" }
+
+---
+"Rendering using {{url}} and {{join}} functions":
+
+  - do:
+      render_search_template:
+        body:  >
+          {
+            "source": {
+              "query": {
+                "match": {
+                  "url": "{{#url}}https://localhost:9200/{{#join}}indices{{/join}}/_stats{{/url}}"
+                }
+              }
+            },
+            "params": {
+              "indices": ["<logstash-{now/d-2d}>", "<logstash-{now/d-1d}>", "<logstash-{now/d}>"]
+            }
+          }
+
+    # Decoded URL is https://localhost:9200/<logstash-{now/d-2d}>,<logstash-{now/d-1d}>,<logstash-{now/d}>/_stats
+  - match: { template_output.query.match.url: "https%3A%2F%2Flocalhost%3A9200%2F%3Clogstash-%7Bnow%2Fd-2d%7D%3E%2C%3Clogstash-%7Bnow%2Fd-1d%7D%3E%2C%3Clogstash-%7Bnow%2Fd%7D%3E%2F_stats" }
diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yaml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yaml
index c7e4c91b4cc1f..712e7e53ca2f4 100644
--- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yaml
+++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yaml
@@ -18,7 +18,7 @@
 
   - do:
       search_template:
-        body: { "inline" : { "query": { "term": { "text": { "value": "{{template}}" } } } }, "params": { "template": "value1" } }
+        body: { "source" : { "query": { "term": { "text": { "value": "{{template}}" } } } }, "params": { "template": "value1" } }
 
   - match: { hits.total: 1 }
 
@@ -29,9 +29,9 @@
   - match: { hits.total: 1 }
 
   - do:
-      put_template:
-        id: "1"
-        body: { "template": { "query": { "term": { "text": "{{template}}" } } } }
+      put_script:
+        lang: "1"
+        body: { "script": { "lang": "mustache", "source": { "query": { "term": { "text": "{{template}}" } } } } }
   - match: { acknowledged: true }
 
   - do:
@@ -41,7 +41,7 @@
 
   - do:
       search_template:
-        body: { "inline" : { "query": { "match_{{template}}": {} } }, "params" : { "template" : "all" } }
+        body: { "source" : { "query": { "match_{{template}}": {} } }, "params" : { "template" : "all" } }
 
   - match: { hits.total: 2 }
 
@@ -106,3 +106,19 @@
 
   - match: { hits.total: 1 }
   - length: { hits.hits: 1 }
+
+  - do:
+      search_template:
+        body: { "file" : "template_1", "params": { "size": "2", "field": "otherField", "value": "foo" }, "explain" : true }
+
+  - match: { hits.total: 1 }
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0._explanation.description: "weight(otherField:foo in 0) [PerFieldSimilarity], result of:" }
+
+  - do:
+      search_template:
+        body: { "file" : "template_1", "params": { "size": "2", "field": "otherField", "value": "foo" }, "profile" : true }
+
+  - match: { hits.total: 1 }
+  - length: { hits.hits: 1 }
+  - length: { profile: 1 }
diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/40_template_query.yaml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/40_template_query.yaml
index 2360dfc37f0bf..c0ac0a6020886 100644
--- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/40_template_query.yaml
+++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/40_template_query.yaml
@@ -19,16 +19,16 @@
       indices.refresh: {}
 
   - do:
-      put_template:
-        id: "1"
-        body: { "template": { "match": { "text": "{{my_value}}" } } }
+      put_script:
+        lang: "1"
+        body: { "script": { "lang": "mustache", "source": { "match": { "text": "{{my_value}}" } } } }
   - match: { acknowledged: true }
 
   - do:
       warnings:
           - '[template] query is deprecated, use search template api instead'
       search:
-        body: { "query": { "template": { "inline": { "term": { "text": { "value": "{{template}}" } } }, "params": { "template": "value1" } } } }
+        body: { "query": { "template": { "source": { "term": { "text": { "value": "{{template}}" } } }, "params": { "template": "value1" } } } }
 
   - match: { hits.total: 1 }
 
@@ -52,7 +52,7 @@
       warnings:
           - '[template] query is deprecated, use search template api instead'
       search:
-        body: { "query": { "template": { "id": "/mustache/1", "params": { "my_value": "value1" } } } }
+        body: { "query": { "template": { "id": "1", "params": { "my_value": "value1" } } } }
 
   - match: { hits.total: 1 }
 
@@ -60,7 +60,7 @@
       warnings:
           - '[template] query is deprecated, use search template api instead'
       search:
-        body: { "query": { "template": { "inline": {"match_{{template}}": {}}, "params" : { "template" : "all" } } } }
+        body: { "query": { "template": { "source": {"match_{{template}}": {}}, "params" : { "template" : "all" } } } }
 
   - match: { hits.total: 2 }
 
@@ -68,7 +68,7 @@
       warnings:
           - '[template] query is deprecated, use search template api instead'
       search:
-        body: { "query": { "template": { "inline": "{ \"term\": { \"text\": { \"value\": \"{{template}}\" } } }", "params": { "template": "value1" } } } }
+        body: { "query": { "template": { "source": "{ \"term\": { \"text\": { \"value\": \"{{template}}\" } } }", "params": { "template": "value1" } } } }
 
   - match: { hits.total: 1 }
 
@@ -76,7 +76,7 @@
       warnings:
           - '[template] query is deprecated, use search template api instead'
       search:
-        body: { "query": { "template": { "inline": "{\"match_{{template}}\": {}}", "params" : { "template" : "all" } } } }
+        body: { "query": { "template": { "source": "{\"match_{{template}}\": {}}", "params" : { "template" : "all" } } } }
 
   - match: { hits.total: 2 }
 
@@ -84,7 +84,7 @@
       warnings:
           - '[template] query is deprecated, use search template api instead'
       search:
-        body: { "query": { "template": { "inline": "{\"match_all\": {}}", "params" : {} } } }
+        body: { "query": { "template": { "source": "{\"match_all\": {}}", "params" : {} } } }
 
   - match: { hits.total: 2 }
 
@@ -92,5 +92,5 @@
       warnings:
           - '[template] query is deprecated, use search template api instead'
       search:
-        body: { "query": { "template": { "inline": "{\"query_string\": { \"query\" : \"{{query}}\" }}", "params" : { "query" : "text:\"value2 value3\"" } } } }
+        body: { "query": { "template": { "source": "{\"query_string\": { \"query\" : \"{{query}}\" }}", "params" : { "query" : "text:\"value2 value3\"" } } } }
   - match: { hits.total: 1 }
diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yaml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yaml
index 8b8ffcf8ae97a..0c801c48f74a7 100644
--- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yaml
+++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yaml
@@ -39,21 +39,21 @@ setup:
       msearch_template:
         body:
           - index: index_*
-          - inline: '{"query": {"match": {"foo": "{{value}}"} } }'
+          - source: '{"query": {"match": {"foo": "{{value}}"} } }'
             params:
               value: "foo"
           - index: index_*
-          - inline: '{"query": {"match": {"{{field}}": "{{value}}"} } }'
+          - source: '{"query": {"match": {"{{field}}": "{{value}}"} } }'
             params:
               field: "foo"
               value: "bar"
           - index: _all
-          - inline: '{"query": {"{{query_type}}": {{query_content}} } }'
+          - source: '{"query": {"{{query_type}}": {{query_content}} } }'
             params:
               query_type: "match_all"
               query_content: "{}"
           - index: _all
-          - inline:
+          - source:
               query:
                 match: {foo: "{{text}}"}
               size: 0
@@ -75,25 +75,25 @@ setup:
         body:
           # Search 0 is OK
           - index: index_*
-          - inline: '{"query": {"match": {"foo": "{{value}}"} } }'
+          - source: '{"query": {"match": {"foo": "{{value}}"} } }'
             params:
               value: "foo"
           # Search 1 has an unclosed JSON template
           - index: index_*
-          - inline: '{"query": {"match": {'
+          - source: '{"query": {"match": {'
             params:
               field: "foo"
               value: "bar"
           # Search 2 is OK
           - index: _all
-          - inline:
+          - source:
               query:
                 match: {foo: "{{text}}"}
             params:
               text: "baz"
           # Search 3 has an unknown query type
           - index: index_*
-          - inline: '{"query": {"{{query_type}}": {} }' # Unknown query type
+          - source: '{"query": {"{{query_type}}": {} }' # Unknown query type
             params:
               query_type: "unknown"
 
@@ -113,15 +113,15 @@ setup:
         body:
           # Search 0 is OK
           - index: index_*
-          - inline: '{"query": {"match": {"foo": "{{value}}"} } }'
+          - source: '{"query": {"match": {"foo": "{{value}}"} } }'
             params:
               value: "foo"
           # Search 1 has not template
           - index: _all
-          - inline: ""
+          - source: ""
           # Search 2 is OK
           - index: index_*
-          - inline: '{"query": {"match": {"foo": "{{value}}"} } }'
+          - source: '{"query": {"match": {"foo": "{{value}}"} } }'
             params:
               value: "bar"
 
@@ -129,9 +129,9 @@ setup:
 "Basic multi-search using stored template":
 
   - do:
-      put_template:
-        id: stored_template_1
-        body: {"template": {"query": {"match": {"{{field}}": "{{value}}" }}}}
+      put_script:
+        lang: stored_template_1
+        body: { "script": { "lang" : "mustache", "source": { "query": {"match": {"{{field}}": "{{value}}" }}}}}
   - match: { acknowledged: true }
 
   - do:
@@ -194,4 +194,3 @@ setup:
   - match:  { responses.0.hits.total:     2  }
   - match:  { responses.1.hits.total:     1  }
   - match:  { responses.2.hits.total:     1  }
-
diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/60_typed_keys.yaml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/60_typed_keys.yaml
new file mode 100644
index 0000000000000..5f44399c99100
--- /dev/null
+++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/60_typed_keys.yaml
@@ -0,0 +1,139 @@
+setup:
+  - do:
+      indices.put_template:
+        name: index_template
+        body:
+          template: test-*
+          settings:
+            number_of_replicas: 0
+          mappings:
+            user:
+              properties:
+                ip:
+                   type: ip
+                integer:
+                   type: integer
+                float:
+                   type: float
+                name:
+                   type: keyword
+
+  - do:
+     bulk:
+        refresh: true
+        body:
+          - '{"index": {"_index": "test-0", "_type": "user"}}'
+          - '{"ip": "10.0.0.1", "integer": 38, "float": 12.5713, "name": "Ruth", "bool": true}'
+          - '{"index": {"_index": "test-0", "_type": "user"}}'
+          - '{"ip": "10.0.0.2", "integer": 42, "float": 15.3393, "name": "Jackie", "surname": "Bowling", "bool": false}'
+          - '{"index": {"_index": "test-1", "_type": "user"}}'
+          - '{"ip": "10.0.0.3", "integer": 29, "float": 19.0517, "name": "Stephanie", "bool": true}'
+          - '{"index": {"_index": "test-1", "_type": "user"}}'
+          - '{"ip": "10.0.0.4", "integer": 19, "float": 19.3717, "surname": "Hamilton", "bool": true}'
+          - '{"index": {"_index": "test-2", "_type": "user"}}'
+          - '{"ip": "10.0.0.5", "integer": 0, "float": 17.3349, "name": "Natalie", "bool": false}'
+
+---
+"Search template with typed_keys parameter":
+
+  - do:
+      put_script:
+        lang: template_1
+        body:
+          script:
+            lang: mustache
+            source:
+              query:
+                match:
+                  bool: "{{bool_value}}"
+              aggs:
+                test_missing:
+                  missing:
+                    field: "{{missing_field}}"
+              suggest:
+                term_suggest:
+                  text: "{{suggest_text}}"
+                  term:
+                    field: "{{suggest_field}}"
+
+  - match: { acknowledged: true }
+
+  - do:
+      search_template:
+        index: test-*
+        typed_keys: true
+        body:
+          id: template_1
+          params:
+            bool_value: true
+            missing_field: name
+            suggest_field: name
+            suggest_text: Hamilt
+
+  - match:    { hits.total: 3 }
+  - match:    { aggregations.missing#test_missing.doc_count: 1 }
+  - is_true:  suggest.term#term_suggest
+
+---
+"Multisearch template with typed_keys parameter":
+
+  - do:
+      put_script:
+        lang: registered_template
+        body:
+          script:
+            lang: mustache
+            source:
+              query:
+                range:
+                  integer:
+                   gte: "{{starting_value}}"
+              aggs:
+                test_histogram:
+                  histogram:
+                    field:    "{{histo.field}}"
+                    interval: "{{histo.interval}}"
+              suggest:
+                phrase_suggester:
+                  text: "{{keywords}}"
+                  phrase:
+                    field: name
+
+  - match: { acknowledged: true }
+
+  - do:
+      msearch_template:
+        typed_keys: true
+        body:
+          - index: test-*
+          - source:
+              query:
+                match:
+                 name: "{{name_value}}"
+              aggs:
+                test_global:
+                 global: {}
+                 aggs:
+                  test_ip_range:
+                    ip_range:
+                      field: "{{ip_field}}"
+                      ranges:
+                        to: "10.0.0.6"
+            params:
+              name_value: "Stephanie"
+              ip_field: "ip"
+          - index: test-0,test-1
+          - id: registered_template
+            params:
+              starting_value: 30
+              histo:
+                field: float
+                interval: 5
+              keywords: Ruht
+
+  - match:  { responses.0.hits.total: 1 }
+  - match:  { responses.0.aggregations.global#test_global.doc_count: 5 }
+  - match:  { responses.0.aggregations.global#test_global.ip_range#test_ip_range.buckets.0.doc_count: 5 }
+  - match:  { responses.1.hits.total: 2 }
+  - match:  { responses.1.aggregations.histogram#test_histogram.buckets.0.doc_count: 1 }
+  - is_true:  responses.1.suggest.phrase#phrase_suggester
diff --git a/modules/lang-painless/ant.xml b/modules/lang-painless/ant.xml
deleted file mode 100644
index 9d3cf9ece91cd..0000000000000
--- a/modules/lang-painless/ant.xml
+++ /dev/null
@@ -1,153 +0,0 @@
-<?xml version="1.0"?>
-<project name="ant-stuff">
-
-<!-- 
- grammar regeneration logic
- we do this with ant for several reasons:
- * remove generated tabs for forbidden-apis
- * remove generated timestamps/filenames for reproducible build
- * fix CRLF line endings for windows consistency
- * ability to make classes package-private
- * keeping in source code control is easier on IDEs
- * regeneration should be rare, no reason to be religious about generated files 
- * all logic already written and battle tested in lucene build
--->
-  <target name="regenerate" description="Regenerate antlr lexer and parser" depends="run-antlr"/>
-
-  <target name="run-antlr">
-    <regen-delete grammar="Painless"/>
-    <regen-lexer grammar="Painless"/>
-    <regen-parser grammar="Painless"/>
-    <regen-fix grammar="Painless"/>
-  </target>
-
-  <macrodef name="replace-value">
-    <attribute name="value" />
-    <attribute name="property" />
-    <attribute name="from" />
-    <attribute name="to" />
-    <sequential>
-      <loadresource property="@{property}">
-        <string value="@{value}"/>
-        <filterchain>
-          <tokenfilter>
-            <filetokenizer/>
-            <replacestring from="@{from}" to="@{to}"/>
-          </tokenfilter>
-        </filterchain>
-      </loadresource>
-    </sequential>
-  </macrodef>
-
-  <macrodef name="regen-delete">
-    <attribute name="grammar" />
-    <sequential>
-      <local name="grammar.path"/>
-      <property name="grammar.path" location="src/main/antlr"/>
-      <!-- delete token files so files will be generated -->
-      <delete dir="${grammar.path}" includes="@{grammar}*.tokens"/>
-      <local name="output.path"/>
-      <patternset id="grammar.@{grammar}.patternset">
-        <include name="@{grammar}Lexer.java" />
-        <include name="@{grammar}Parser.java" />
-        <include name="@{grammar}Lexer.tokens" />
-        <include name="@{grammar}Parser.tokens" />
-        <include name="@{grammar}ParserVisitor.java" />
-        <include name="@{grammar}ParserBaseVisitor.java" />
-      </patternset>
-      <property name="output.path" location="src/main/java/org/elasticsearch/painless/antlr"/>
-      <!-- delete parser and lexer so files will be generated -->
-      <delete dir="${output.path}">
-        <patternset refid="grammar.@{grammar}.patternset"/>
-      </delete>
-    </sequential>
-  </macrodef>
-
-  <macrodef name="regen-lexer">
-    <attribute name="grammar" />
-    <sequential>
-      <local name="grammar.path"/>
-      <local name="output.path"/>
-      <property name="grammar.path" location="src/main/antlr"/>
-      <property name="output.path" location="src/main/java/org/elasticsearch/painless/antlr"/>
-      <!-- invoke ANTLR4 -->
-      <java classname="org.antlr.v4.Tool" fork="true" failonerror="true" classpathref="regenerate.classpath" taskname="antlr">
-        <sysproperty key="file.encoding" value="UTF-8"/>
-        <sysproperty key="user.language" value="en"/>
-        <sysproperty key="user.country" value="US"/>
-        <sysproperty key="user.variant" value=""/>
-        <arg value="-Werror"/>
-        <arg value="-package"/>
-        <arg value="org.elasticsearch.painless.antlr"/>
-        <arg value="-o"/>
-        <arg path="${output.path}"/>
-        <arg path="${grammar.path}/@{grammar}Lexer.g4"/>
-      </java>
-    </sequential>
-  </macrodef>
-
-  <macrodef name="regen-parser">
-    <attribute name="grammar" />
-    <sequential>
-      <local name="grammar.path"/>
-      <local name="output.path"/>
-      <property name="grammar.path" location="src/main/antlr"/>
-      <property name="output.path" location="src/main/java/org/elasticsearch/painless/antlr"/>
-      <!-- invoke ANTLR4 -->
-      <java classname="org.antlr.v4.Tool" fork="true" failonerror="true" classpathref="regenerate.classpath" taskname="antlr">
-        <sysproperty key="file.encoding" value="UTF-8"/>
-        <sysproperty key="user.language" value="en"/>
-        <sysproperty key="user.country" value="US"/>
-        <sysproperty key="user.variant" value=""/>
-        <arg value="-Werror"/>
-        <arg value="-package"/>
-        <arg value="org.elasticsearch.painless.antlr"/>
-        <arg value="-no-listener"/>
-        <arg value="-visitor"/>
-        <!-- <arg value="-Xlog"/> -->
-        <arg value="-o"/>
-        <arg path="${output.path}"/>
-        <arg path="${grammar.path}/@{grammar}Parser.g4"/>
-      </java>
-    </sequential>
-  </macrodef>
-
-  <macrodef name="regen-fix">
-    <attribute name="grammar" />
-    <sequential>
-      <local name="grammar.path"/>
-      <local name="output.path"/>
-      <property name="grammar.path" location="src/main/antlr"/>
-      <property name="output.path" location="src/main/java/org/elasticsearch/painless/antlr"/>
-      <patternset id="grammar.@{grammar}.patternset">
-        <include name="@{grammar}Lexer.java" />
-        <include name="@{grammar}Parser.java" />
-        <include name="@{grammar}ParserVisitor.java" />
-        <include name="@{grammar}ParserBaseVisitor.java" />
-      </patternset>
-      <!-- fileset with files to edit -->
-      <fileset id="grammar.fileset" dir="${output.path}">
-        <patternset refid="grammar.@{grammar}.patternset"/>
-      </fileset>
-      <!-- moves token files to grammar directory for use with IDE's -->
-      <move file="${output.path}/@{grammar}Lexer.tokens" todir="${grammar.path}"/>
-      <move file="${output.path}/@{grammar}Parser.tokens" todir="${grammar.path}"/>
-      <!-- make the generated classes package private -->
-      <replaceregexp match="public ((interface|class) \Q@{grammar}\E\w+)" replace="\1" encoding="UTF-8">
-        <fileset refid="grammar.fileset"/>
-      </replaceregexp>
-      <!-- nuke timestamps/filenames in generated files -->
-      <replaceregexp match="\Q// Generated from \E.*" replace="\/\/ ANTLR GENERATED CODE: DO NOT EDIT" encoding="UTF-8">
-        <fileset refid="grammar.fileset"/>
-      </replaceregexp>
-      <!-- remove tabs in antlr generated files -->
-      <replaceregexp match="\t" flags="g" replace="  " encoding="UTF-8">
-        <fileset refid="grammar.fileset"/>
-      </replaceregexp>
-      <!-- fix line endings -->
-      <fixcrlf srcdir="${output.path}">
-        <patternset refid="grammar.@{grammar}.patternset"/>
-      </fixcrlf>
-    </sequential>
-  </macrodef>
-</project>
diff --git a/modules/lang-painless/build.gradle b/modules/lang-painless/build.gradle
index 531e06d4eba78..6eaf3053dee1b 100644
--- a/modules/lang-painless/build.gradle
+++ b/modules/lang-painless/build.gradle
@@ -33,8 +33,34 @@ dependencyLicenses {
   mapping from: /asm-.*/, to: 'asm'
 }
 
-// regeneration logic, comes in via ant right now
-// don't port it to gradle, it works fine.
+integTestCluster {
+  setting 'script.max_compilations_per_minute', '1000'
+}
+
+/* Build Javadoc for the Java classes in Painless's public API that are in the
+ * Painless plugin */
+task apiJavadoc(type: Javadoc) {
+  source = sourceSets.main.allJava
+  classpath = sourceSets.main.runtimeClasspath
+  include '**/org/elasticsearch/painless/api/'
+  destinationDir = new File(docsDir, 'apiJavadoc')
+}
+task apiJavadocJar(type: Jar) {
+  classifier = 'apiJavadoc'
+  from apiJavadoc
+}
+assemble.dependsOn apiJavadocJar
+
+// Reference documentation for Painless's public API.
+task generatePainlessApi(type: JavaExec) {
+  main = 'org.elasticsearch.painless.PainlessDocGenerator'
+  classpath = sourceSets.test.runtimeClasspath
+  args file('../../docs/painless/painless-api-reference')
+}
+
+/**********************************************
+ *            Parser regeneration             *
+ **********************************************/
 
 configurations {
   regenerate
@@ -44,11 +70,80 @@ dependencies {
   regenerate 'org.antlr:antlr4:4.5.1-1'
 }
 
-ant.references['regenerate.classpath'] = new Path(ant.project, configurations.regenerate.asPath)
-ant.importBuild 'ant.xml'
+String grammarPath = 'src/main/antlr'
+String outputPath = 'src/main/java/org/elasticsearch/painless/antlr'
+
+task cleanGenerated(type: Delete) {
+  delete fileTree(grammarPath) {
+    include '*.tokens'
+  }
+  delete fileTree(outputPath) {
+    include 'Painless*.java'
+  }
+}
+
+task regenLexer(type: JavaExec) {
+  dependsOn cleanGenerated
+  main = 'org.antlr.v4.Tool'
+  classpath = configurations.regenerate
+  systemProperty 'file.encoding', 'UTF-8'
+  systemProperty 'user.language', 'en'
+  systemProperty 'user.country', 'US'
+  systemProperty 'user.variant', ''
+  args '-Werror',
+       '-package', 'org.elasticsearch.painless.antlr',
+       '-o', outputPath,
+       "${file(grammarPath)}/PainlessLexer.g4"
+}
+
+task regenParser(type: JavaExec) {
+  dependsOn regenLexer
+  main = 'org.antlr.v4.Tool'
+  classpath = configurations.regenerate
+  systemProperty 'file.encoding', 'UTF-8'
+  systemProperty 'user.language', 'en'
+  systemProperty 'user.country', 'US'
+  systemProperty 'user.variant', ''
+  args '-Werror',
+       '-package', 'org.elasticsearch.painless.antlr',
+       '-no-listener',
+       '-visitor',
+       // '-Xlog',
+       '-o', outputPath,
+       "${file(grammarPath)}/PainlessParser.g4"
+}
 
-integTest {
-  cluster {
-    setting 'script.max_compilations_per_minute', '1000'
+task regen {
+  dependsOn regenParser
+  doLast {
+    // moves token files to grammar directory for use with IDE's
+    ant.move(file: "${outputPath}/PainlessLexer.tokens", toDir: grammarPath)
+    ant.move(file: "${outputPath}/PainlessParser.tokens", toDir: grammarPath)
+    // make the generated classes package private
+    ant.replaceregexp(match: 'public ((interface|class) \\QPainless\\E\\w+)',
+                      replace: '\\1',
+                      encoding: 'UTF-8') {
+      fileset(dir: outputPath, includes: 'Painless*.java')
+    }
+    // make the lexer abstract
+    ant.replaceregexp(match: '(class \\QPainless\\ELexer)',
+                      replace: 'abstract \\1',
+                      encoding: 'UTF-8') {
+      fileset(dir: outputPath, includes: 'PainlessLexer.java')
+    }
+    // nuke timestamps/filenames in generated files
+    ant.replaceregexp(match: '\\Q// Generated from \\E.*',
+                      replace: '\\/\\/ ANTLR GENERATED CODE: DO NOT EDIT',
+                      encoding: 'UTF-8') {
+      fileset(dir: outputPath, includes: 'Painless*.java')
+    }
+    // remove tabs in antlr generated files
+    ant.replaceregexp(match: '\t', flags: 'g', replace: '  ', encoding: 'UTF-8') {
+      fileset(dir: outputPath, includes: 'Painless*.java')
+    }
+    // fix line endings
+    ant.fixcrlf(srcdir: outputPath) {
+      patternset(includes: 'Painless*.java')
+    }
   }
 }
diff --git a/modules/lang-painless/src/main/antlr/PainlessLexer.g4 b/modules/lang-painless/src/main/antlr/PainlessLexer.g4
index a2e7b921300f3..6ab6a86113595 100644
--- a/modules/lang-painless/src/main/antlr/PainlessLexer.g4
+++ b/modules/lang-painless/src/main/antlr/PainlessLexer.g4
@@ -19,8 +19,20 @@
 
 lexer grammar PainlessLexer;
 
-@header {
-import org.elasticsearch.painless.Definition;
+@members{
+/**
+ * Check against the current whitelist to determine whether a token is a type
+ * or not. Called by the {@code TYPE} token defined in {@code PainlessLexer.g4}.
+ * See also
+ * <a href="https://en.wikipedia.org/wiki/The_lexer_hack">The lexer hack</a>.
+ */
+protected abstract boolean isSimpleType(String name);
+
+/**
+ * Is the preceding {@code /} a the beginning of a regex (true) or a division
+ * (false).
+ */
+protected abstract boolean slashIsRegex();
 }
 
 WS: [ \t\n\r]+ -> skip;
@@ -36,7 +48,8 @@ RP:        ')';
 // between shortcuts and decimal values.  Without the mode switch
 // shortcuts such as id.0.0 will fail because 0.0 will be interpreted
 // as a decimal value instead of two individual list-style shortcuts.
-DOT:       '.' -> mode(AFTER_DOT);
+DOT:       '.'  -> mode(AFTER_DOT);
+NSDOT:     '?.' -> mode(AFTER_DOT);
 COMMA:     ',';
 SEMICOLON: ';';
 IF:        'if';
@@ -58,7 +71,7 @@ INSTANCEOF: 'instanceof';
 BOOLNOT: '!';
 BWNOT:   '~';
 MUL:     '*';
-DIV:     '/' { false == SlashStrategy.slashIsRegex(this) }?;
+DIV:     '/' { false == slashIsRegex() }?;
 REM:     '%';
 ADD:     '+';
 SUB:     '-';
@@ -80,6 +93,7 @@ BOOLAND: '&&';
 BOOLOR:  '||';
 COND:    '?';
 COLON:   ':';
+ELVIS:   '?:';
 REF:     '::';
 ARROW:   '->';
 FIND:    '=~';
@@ -103,10 +117,10 @@ AUSH:   '>>>=';
 OCTAL: '0' [0-7]+ [lL]?;
 HEX: '0' [xX] [0-9a-fA-F]+ [lL]?;
 INTEGER: ( '0' | [1-9] [0-9]* ) [lLfFdD]?;
-DECIMAL: ( '0' | [1-9] [0-9]* ) (DOT [0-9]+)? ( [eE] [+\-]? [0-9]+ )? [fF]?;
+DECIMAL: ( '0' | [1-9] [0-9]* ) (DOT [0-9]+)? ( [eE] [+\-]? [0-9]+ )? [fFdD]?;
 
-STRING: ( '"' ( '\\"' | '\\\\' | ~[\\"] )*? '"' ) | ( '\'' ( '\\\'' | '\\\\' | ~[\\"] )*? '\'' );
-REGEX: '/' ( ~('/' | '\n') | '\\' ~'\n' )+ '/' [cilmsUux]* { SlashStrategy.slashIsRegex(this) }?;
+STRING: ( '"' ( '\\"' | '\\\\' | ~[\\"] )*? '"' ) | ( '\'' ( '\\\'' | '\\\\' | ~[\\'] )*? '\'' );
+REGEX: '/' ( '\\' ~'\n' | ~('/' | '\n') )+? '/' [cilmsUux]* { slashIsRegex() }?;
 
 TRUE:  'true';
 FALSE: 'false';
@@ -119,7 +133,7 @@ NULL: 'null';
 // or not.  Note this works by processing one character at a time
 // and the rule is added or removed as this happens.  This is also known
 // as "the lexer hack."  See (https://en.wikipedia.org/wiki/The_lexer_hack).
-TYPE: ID ( DOT ID )* { Definition.isSimpleType(getText()) }?;
+TYPE: ID ( DOT ID )* { isSimpleType(getText()) }?;
 ID: [_a-zA-Z] [_a-zA-Z0-9]*;
 
 mode AFTER_DOT;
diff --git a/modules/lang-painless/src/main/antlr/PainlessLexer.tokens b/modules/lang-painless/src/main/antlr/PainlessLexer.tokens
index dc62fe36b28c2..90422e091603f 100644
--- a/modules/lang-painless/src/main/antlr/PainlessLexer.tokens
+++ b/modules/lang-painless/src/main/antlr/PainlessLexer.tokens
@@ -7,79 +7,81 @@ RBRACE=6
 LP=7
 RP=8
 DOT=9
-COMMA=10
-SEMICOLON=11
-IF=12
-IN=13
-ELSE=14
-WHILE=15
-DO=16
-FOR=17
-CONTINUE=18
-BREAK=19
-RETURN=20
-NEW=21
-TRY=22
-CATCH=23
-THROW=24
-THIS=25
-INSTANCEOF=26
-BOOLNOT=27
-BWNOT=28
-MUL=29
-DIV=30
-REM=31
-ADD=32
-SUB=33
-LSH=34
-RSH=35
-USH=36
-LT=37
-LTE=38
-GT=39
-GTE=40
-EQ=41
-EQR=42
-NE=43
-NER=44
-BWAND=45
-XOR=46
-BWOR=47
-BOOLAND=48
-BOOLOR=49
-COND=50
-COLON=51
-REF=52
-ARROW=53
-FIND=54
-MATCH=55
-INCR=56
-DECR=57
-ASSIGN=58
-AADD=59
-ASUB=60
-AMUL=61
-ADIV=62
-AREM=63
-AAND=64
-AXOR=65
-AOR=66
-ALSH=67
-ARSH=68
-AUSH=69
-OCTAL=70
-HEX=71
-INTEGER=72
-DECIMAL=73
-STRING=74
-REGEX=75
-TRUE=76
-FALSE=77
-NULL=78
-TYPE=79
-ID=80
-DOTINTEGER=81
-DOTID=82
+NSDOT=10
+COMMA=11
+SEMICOLON=12
+IF=13
+IN=14
+ELSE=15
+WHILE=16
+DO=17
+FOR=18
+CONTINUE=19
+BREAK=20
+RETURN=21
+NEW=22
+TRY=23
+CATCH=24
+THROW=25
+THIS=26
+INSTANCEOF=27
+BOOLNOT=28
+BWNOT=29
+MUL=30
+DIV=31
+REM=32
+ADD=33
+SUB=34
+LSH=35
+RSH=36
+USH=37
+LT=38
+LTE=39
+GT=40
+GTE=41
+EQ=42
+EQR=43
+NE=44
+NER=45
+BWAND=46
+XOR=47
+BWOR=48
+BOOLAND=49
+BOOLOR=50
+COND=51
+COLON=52
+ELVIS=53
+REF=54
+ARROW=55
+FIND=56
+MATCH=57
+INCR=58
+DECR=59
+ASSIGN=60
+AADD=61
+ASUB=62
+AMUL=63
+ADIV=64
+AREM=65
+AAND=66
+AXOR=67
+AOR=68
+ALSH=69
+ARSH=70
+AUSH=71
+OCTAL=72
+HEX=73
+INTEGER=74
+DECIMAL=75
+STRING=76
+REGEX=77
+TRUE=78
+FALSE=79
+NULL=80
+TYPE=81
+ID=82
+DOTINTEGER=83
+DOTID=84
 '{'=3
 '}'=4
 '['=5
@@ -87,66 +89,68 @@ DOTID=82
 '('=7
 ')'=8
 '.'=9
-','=10
-';'=11
-'if'=12
-'in'=13
-'else'=14
-'while'=15
-'do'=16
-'for'=17
-'continue'=18
-'break'=19
-'return'=20
-'new'=21
-'try'=22
-'catch'=23
-'throw'=24
-'this'=25
-'instanceof'=26
-'!'=27
-'~'=28
-'*'=29
-'/'=30
-'%'=31
-'+'=32
-'-'=33
-'<<'=34
-'>>'=35
-'>>>'=36
-'<'=37
-'<='=38
-'>'=39
-'>='=40
-'=='=41
-'==='=42
-'!='=43
-'!=='=44
-'&'=45
-'^'=46
-'|'=47
-'&&'=48
-'||'=49
-'?'=50
-':'=51
-'::'=52
-'->'=53
-'=~'=54
-'==~'=55
-'++'=56
-'--'=57
-'='=58
-'+='=59
-'-='=60
-'*='=61
-'/='=62
-'%='=63
-'&='=64
-'^='=65
-'|='=66
-'<<='=67
-'>>='=68
-'>>>='=69
-'true'=76
-'false'=77
-'null'=78
+'?.'=10
+','=11
+';'=12
+'if'=13
+'in'=14
+'else'=15
+'while'=16
+'do'=17
+'for'=18
+'continue'=19
+'break'=20
+'return'=21
+'new'=22
+'try'=23
+'catch'=24
+'throw'=25
+'this'=26
+'instanceof'=27
+'!'=28
+'~'=29
+'*'=30
+'/'=31
+'%'=32
+'+'=33
+'-'=34
+'<<'=35
+'>>'=36
+'>>>'=37
+'<'=38
+'<='=39
+'>'=40
+'>='=41
+'=='=42
+'==='=43
+'!='=44
+'!=='=45
+'&'=46
+'^'=47
+'|'=48
+'&&'=49
+'||'=50
+'?'=51
+':'=52
+'?:'=53
+'::'=54
+'->'=55
+'=~'=56
+'==~'=57
+'++'=58
+'--'=59
+'='=60
+'+='=61
+'-='=62
+'*='=63
+'/='=64
+'%='=65
+'&='=66
+'^='=67
+'|='=68
+'<<='=69
+'>>='=70
+'>>>='=71
+'true'=78
+'false'=79
+'null'=80
diff --git a/modules/lang-painless/src/main/antlr/PainlessParser.g4 b/modules/lang-painless/src/main/antlr/PainlessParser.g4
index 42876b18f0d3f..bfa4ee28dcc88 100644
--- a/modules/lang-painless/src/main/antlr/PainlessParser.g4
+++ b/modules/lang-painless/src/main/antlr/PainlessParser.g4
@@ -110,6 +110,7 @@ expression
     |               expression BOOLAND expression                         # bool
     |               expression BOOLOR expression                          # bool
     | <assoc=right> expression COND expression COLON expression           # conditional
+    | <assoc=right> expression ELVIS expression                           # elvis
     | <assoc=right> expression ( ASSIGN | AADD | ASUB | AMUL |
                                  ADIV   | AREM | AAND | AXOR |
                                  AOR    | ALSH | ARSH | AUSH ) expression # assignment
@@ -156,11 +157,11 @@ postdot
     ;
 
 callinvoke
-    : DOT DOTID arguments
+    : ( DOT | NSDOT ) DOTID arguments
     ;
 
 fieldaccess
-    : DOT ( DOTID | DOTINTEGER )
+    : ( DOT | NSDOT ) ( DOTID | DOTINTEGER )
     ;
 
 braceaccess
diff --git a/modules/lang-painless/src/main/antlr/PainlessParser.tokens b/modules/lang-painless/src/main/antlr/PainlessParser.tokens
index dc62fe36b28c2..90422e091603f 100644
--- a/modules/lang-painless/src/main/antlr/PainlessParser.tokens
+++ b/modules/lang-painless/src/main/antlr/PainlessParser.tokens
@@ -7,79 +7,81 @@ RBRACE=6
 LP=7
 RP=8
 DOT=9
-COMMA=10
-SEMICOLON=11
-IF=12
-IN=13
-ELSE=14
-WHILE=15
-DO=16
-FOR=17
-CONTINUE=18
-BREAK=19
-RETURN=20
-NEW=21
-TRY=22
-CATCH=23
-THROW=24
-THIS=25
-INSTANCEOF=26
-BOOLNOT=27
-BWNOT=28
-MUL=29
-DIV=30
-REM=31
-ADD=32
-SUB=33
-LSH=34
-RSH=35
-USH=36
-LT=37
-LTE=38
-GT=39
-GTE=40
-EQ=41
-EQR=42
-NE=43
-NER=44
-BWAND=45
-XOR=46
-BWOR=47
-BOOLAND=48
-BOOLOR=49
-COND=50
-COLON=51
-REF=52
-ARROW=53
-FIND=54
-MATCH=55
-INCR=56
-DECR=57
-ASSIGN=58
-AADD=59
-ASUB=60
-AMUL=61
-ADIV=62
-AREM=63
-AAND=64
-AXOR=65
-AOR=66
-ALSH=67
-ARSH=68
-AUSH=69
-OCTAL=70
-HEX=71
-INTEGER=72
-DECIMAL=73
-STRING=74
-REGEX=75
-TRUE=76
-FALSE=77
-NULL=78
-TYPE=79
-ID=80
-DOTINTEGER=81
-DOTID=82
+NSDOT=10
+COMMA=11
+SEMICOLON=12
+IF=13
+IN=14
+ELSE=15
+WHILE=16
+DO=17
+FOR=18
+CONTINUE=19
+BREAK=20
+RETURN=21
+NEW=22
+TRY=23
+CATCH=24
+THROW=25
+THIS=26
+INSTANCEOF=27
+BOOLNOT=28
+BWNOT=29
+MUL=30
+DIV=31
+REM=32
+ADD=33
+SUB=34
+LSH=35
+RSH=36
+USH=37
+LT=38
+LTE=39
+GT=40
+GTE=41
+EQ=42
+EQR=43
+NE=44
+NER=45
+BWAND=46
+XOR=47
+BWOR=48
+BOOLAND=49
+BOOLOR=50
+COND=51
+COLON=52
+ELVIS=53
+REF=54
+ARROW=55
+FIND=56
+MATCH=57
+INCR=58
+DECR=59
+ASSIGN=60
+AADD=61
+ASUB=62
+AMUL=63
+ADIV=64
+AREM=65
+AAND=66
+AXOR=67
+AOR=68
+ALSH=69
+ARSH=70
+AUSH=71
+OCTAL=72
+HEX=73
+INTEGER=74
+DECIMAL=75
+STRING=76
+REGEX=77
+TRUE=78
+FALSE=79
+NULL=80
+TYPE=81
+ID=82
+DOTINTEGER=83
+DOTID=84
 '{'=3
 '}'=4
 '['=5
@@ -87,66 +89,68 @@ DOTID=82
 '('=7
 ')'=8
 '.'=9
-','=10
-';'=11
-'if'=12
-'in'=13
-'else'=14
-'while'=15
-'do'=16
-'for'=17
-'continue'=18
-'break'=19
-'return'=20
-'new'=21
-'try'=22
-'catch'=23
-'throw'=24
-'this'=25
-'instanceof'=26
-'!'=27
-'~'=28
-'*'=29
-'/'=30
-'%'=31
-'+'=32
-'-'=33
-'<<'=34
-'>>'=35
-'>>>'=36
-'<'=37
-'<='=38
-'>'=39
-'>='=40
-'=='=41
-'==='=42
-'!='=43
-'!=='=44
-'&'=45
-'^'=46
-'|'=47
-'&&'=48
-'||'=49
-'?'=50
-':'=51
-'::'=52
-'->'=53
-'=~'=54
-'==~'=55
-'++'=56
-'--'=57
-'='=58
-'+='=59
-'-='=60
-'*='=61
-'/='=62
-'%='=63
-'&='=64
-'^='=65
-'|='=66
-'<<='=67
-'>>='=68
-'>>>='=69
-'true'=76
-'false'=77
-'null'=78
+'?.'=10
+','=11
+';'=12
+'if'=13
+'in'=14
+'else'=15
+'while'=16
+'do'=17
+'for'=18
+'continue'=19
+'break'=20
+'return'=21
+'new'=22
+'try'=23
+'catch'=24
+'throw'=25
+'this'=26
+'instanceof'=27
+'!'=28
+'~'=29
+'*'=30
+'/'=31
+'%'=32
+'+'=33
+'-'=34
+'<<'=35
+'>>'=36
+'>>>'=37
+'<'=38
+'<='=39
+'>'=40
+'>='=41
+'=='=42
+'==='=43
+'!='=44
+'!=='=45
+'&'=46
+'^'=47
+'|'=48
+'&&'=49
+'||'=50
+'?'=51
+':'=52
+'?:'=53
+'::'=54
+'->'=55
+'=~'=56
+'==~'=57
+'++'=58
+'--'=59
+'='=60
+'+='=61
+'-='=62
+'*='=63
+'/='=64
+'%='=65
+'&='=66
+'^='=67
+'|='=68
+'<<='=69
+'>>='=70
+'>>>='=71
+'true'=78
+'false'=79
+'null'=80
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
index 864b44fb766a8..54ec8b5ac39ba 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
@@ -23,6 +23,29 @@
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Definition.Type;
 
+import java.util.Objects;
+
+import static org.elasticsearch.painless.Definition.BOOLEAN_OBJ_TYPE;
+import static org.elasticsearch.painless.Definition.BOOLEAN_TYPE;
+import static org.elasticsearch.painless.Definition.BYTE_OBJ_TYPE;
+import static org.elasticsearch.painless.Definition.BYTE_TYPE;
+import static org.elasticsearch.painless.Definition.CHAR_OBJ_TYPE;
+import static org.elasticsearch.painless.Definition.CHAR_TYPE;
+import static org.elasticsearch.painless.Definition.DEF_TYPE;
+import static org.elasticsearch.painless.Definition.DOUBLE_OBJ_TYPE;
+import static org.elasticsearch.painless.Definition.DOUBLE_TYPE;
+import static org.elasticsearch.painless.Definition.FLOAT_OBJ_TYPE;
+import static org.elasticsearch.painless.Definition.FLOAT_TYPE;
+import static org.elasticsearch.painless.Definition.INT_OBJ_TYPE;
+import static org.elasticsearch.painless.Definition.INT_TYPE;
+import static org.elasticsearch.painless.Definition.LONG_OBJ_TYPE;
+import static org.elasticsearch.painless.Definition.LONG_TYPE;
+import static org.elasticsearch.painless.Definition.NUMBER_TYPE;
+import static org.elasticsearch.painless.Definition.OBJECT_TYPE;
+import static org.elasticsearch.painless.Definition.SHORT_OBJ_TYPE;
+import static org.elasticsearch.painless.Definition.SHORT_TYPE;
+import static org.elasticsearch.painless.Definition.STRING_TYPE;
+
 /**
  * Used during the analysis phase to collect legal type casts and promotions
  * for type-checking and later to write necessary casts in the bytecode.
@@ -30,6 +53,9 @@
 public final class AnalyzerCaster {
 
     public static Cast getLegalCast(Location location, Type actual, Type expected, boolean explicit, boolean internal) {
+        Objects.requireNonNull(actual);
+        Objects.requireNonNull(expected);
+
         if (actual.equals(expected)) {
             return null;
         }
@@ -38,15 +64,15 @@ public static Cast getLegalCast(Location location, Type actual, Type expected, b
             case BOOL:
                 switch (expected.sort) {
                     case DEF:
-                        return new Cast(actual, Definition.DEF_TYPE, explicit, false, false, true, false);
+                        return new Cast(BOOLEAN_OBJ_TYPE, DEF_TYPE, explicit, null, null, BOOLEAN_TYPE, null);
                     case OBJECT:
-                        if (Definition.OBJECT_TYPE.equals(expected) && internal)
-                            return new Cast(actual, actual, explicit, false, false, false, true);
+                        if (OBJECT_TYPE.equals(expected) && internal)
+                            return new Cast(BOOLEAN_OBJ_TYPE, OBJECT_TYPE, explicit, null, null, BOOLEAN_TYPE, null);
 
                         break;
                     case BOOL_OBJ:
                         if (internal)
-                            return new Cast(actual, actual, explicit, false, false, false, true);
+                            return new Cast(BOOLEAN_TYPE, BOOLEAN_TYPE, explicit, null, null, null, BOOLEAN_TYPE);
                 }
 
                 break;
@@ -57,53 +83,57 @@ public static Cast getLegalCast(Location location, Type actual, Type expected, b
                     case LONG:
                     case FLOAT:
                     case DOUBLE:
-                        return new Cast(actual, expected, explicit);
+                        return new Cast(BYTE_TYPE, expected, explicit);
                     case CHAR:
                         if (explicit)
-                            return new Cast(actual, expected, true);
+                            return new Cast(BYTE_TYPE, CHAR_TYPE, true);
 
                         break;
                     case DEF:
-                        return new Cast(actual, Definition.DEF_TYPE, explicit, false, false, true, false);
+                        return new Cast(BYTE_OBJ_TYPE, DEF_TYPE, explicit, null, null, BYTE_TYPE, null);
                     case OBJECT:
-                        if (Definition.OBJECT_TYPE.equals(expected) && internal)
-                            return new Cast(actual, actual, explicit, false, false, false, true);
+                        if (OBJECT_TYPE.equals(expected) && internal)
+                            return new Cast(BYTE_OBJ_TYPE, OBJECT_TYPE, explicit, null, null, BYTE_TYPE, null);
 
                         break;
                     case NUMBER:
+                        if (internal)
+                            return new Cast(BYTE_OBJ_TYPE, NUMBER_TYPE, explicit, null, null, BYTE_TYPE, null);
+
+                        break;
                     case BYTE_OBJ:
                         if (internal)
-                            return new Cast(actual, actual, explicit, false, false, false, true);
+                            return new Cast(BYTE_TYPE, BYTE_TYPE, explicit, null, null, null, BYTE_TYPE);
 
                         break;
                     case SHORT_OBJ:
                         if (internal)
-                            return new Cast(actual,Definition.SHORT_TYPE, explicit, false, false, false, true);
+                            return new Cast(BYTE_TYPE, SHORT_TYPE, explicit, null, null, null, SHORT_TYPE);
 
                         break;
                     case INT_OBJ:
                         if (internal)
-                            return new Cast(actual, Definition.INT_TYPE, explicit, false, false, false, true);
+                            return new Cast(BYTE_TYPE, INT_TYPE, explicit, null, null, null, INT_TYPE);
 
                         break;
                     case LONG_OBJ:
                         if (internal)
-                            return new Cast(actual, Definition.LONG_TYPE, explicit, false, false, false, true);
+                            return new Cast(BYTE_TYPE, LONG_TYPE, explicit, null, null, null, LONG_TYPE);
 
                         break;
                     case FLOAT_OBJ:
                         if (internal)
-                            return new Cast(actual, Definition.FLOAT_TYPE, explicit, false, false, false, true);
+                            return new Cast(BYTE_TYPE, FLOAT_TYPE, explicit, null, null, null, FLOAT_TYPE);
 
                         break;
                     case DOUBLE_OBJ:
                         if (internal)
-                            return new Cast(actual, Definition.DOUBLE_TYPE, explicit, false, false, false, true);
+                            return new Cast(BYTE_TYPE, DOUBLE_TYPE, explicit, null, null, null, DOUBLE_TYPE);
 
                         break;
                     case CHAR_OBJ:
                         if (explicit && internal)
-                            return new Cast(actual, Definition.CHAR_TYPE, explicit, false, false, false, true);
+                            return new Cast(BYTE_TYPE, CHAR_TYPE, true, null, null, null, CHAR_TYPE);
 
                         break;
                 }
@@ -115,54 +145,58 @@ public static Cast getLegalCast(Location location, Type actual, Type expected, b
                     case LONG:
                     case FLOAT:
                     case DOUBLE:
-                        return new Cast(actual, expected, explicit);
+                        return new Cast(SHORT_TYPE, expected, explicit);
                     case BYTE:
                     case CHAR:
                         if (explicit)
-                            return new Cast(actual, expected, true);
+                            return new Cast(SHORT_TYPE, expected, true);
 
                         break;
                     case DEF:
-                        return new Cast(actual, Definition.DEF_TYPE, explicit, false, false, true, false);
+                        return new Cast(SHORT_OBJ_TYPE, DEF_TYPE, explicit, null, null, SHORT_TYPE, null);
                     case OBJECT:
-                        if (Definition.OBJECT_TYPE.equals(expected) && internal)
-                            return new Cast(actual, actual, explicit, false, false, false, true);
+                        if (OBJECT_TYPE.equals(expected) && internal)
+                            return new Cast(SHORT_OBJ_TYPE, OBJECT_TYPE, explicit, null, null, SHORT_TYPE, null);
 
                         break;
                     case NUMBER:
+                        if (internal)
+                            return new Cast(SHORT_OBJ_TYPE, NUMBER_TYPE, explicit, null, null, SHORT_TYPE, null);
+
+                        break;
                     case SHORT_OBJ:
                         if (internal)
-                            return new Cast(actual, actual, explicit, false, false, false, true);
+                            return new Cast(SHORT_TYPE, SHORT_TYPE, explicit, null, null, null, SHORT_TYPE);
 
                         break;
                     case INT_OBJ:
                         if (internal)
-                            return new Cast(actual, Definition.INT_TYPE, explicit, false, false, false, true);
+                            return new Cast(SHORT_TYPE, INT_TYPE, explicit, null, null, null, INT_TYPE);
 
                         break;
                     case LONG_OBJ:
                         if (internal)
-                            return new Cast(actual, Definition.LONG_TYPE, explicit, false, false, false, true);
+                            return new Cast(SHORT_TYPE, LONG_TYPE, explicit, null, null, null, LONG_TYPE);
 
                         break;
                     case FLOAT_OBJ:
                         if (internal)
-                            return new Cast(actual, Definition.FLOAT_TYPE, explicit, false, false, false, true);
+                            return new Cast(SHORT_TYPE, FLOAT_TYPE, explicit, null, null, null, FLOAT_TYPE);
 
                         break;
                     case DOUBLE_OBJ:
                         if (internal)
-                            return new Cast(actual, Definition.DOUBLE_TYPE, explicit, false, false, false, true);
+                            return new Cast(SHORT_TYPE, DOUBLE_TYPE, explicit, null, null, null, DOUBLE_TYPE);
 
                         break;
                     case BYTE_OBJ:
                         if (explicit && internal)
-                            return new Cast(actual, Definition.BYTE_TYPE, true, false, false, false, true);
+                            return new Cast(SHORT_TYPE, BYTE_TYPE, true, null, null, null, BYTE_TYPE);
 
                         break;
                     case CHAR_OBJ:
                         if (explicit && internal)
-                            return new Cast(actual, Definition.CHAR_TYPE, true, false, false, false, true);
+                            return new Cast(SHORT_TYPE, CHAR_TYPE, true, null, null, null, CHAR_TYPE);
 
                         break;
                 }
@@ -174,7 +208,7 @@ public static Cast getLegalCast(Location location, Type actual, Type expected, b
                     case LONG:
                     case FLOAT:
                     case DOUBLE:
-                        return new Cast(actual, expected, explicit);
+                        return new Cast(CHAR_TYPE, expected, explicit);
                     case BYTE:
                     case SHORT:
                         if (explicit)
@@ -182,48 +216,52 @@ public static Cast getLegalCast(Location location, Type actual, Type expected, b
 
                         break;
                     case DEF:
-                        return new Cast(actual, Definition.DEF_TYPE, explicit, false, false, true, false);
+                        return new Cast(CHAR_OBJ_TYPE, DEF_TYPE, explicit, null, null, CHAR_TYPE, null);
                     case OBJECT:
-                        if (Definition.OBJECT_TYPE.equals(expected) && internal)
-                            return new Cast(actual, actual, explicit, false, false, false, true);
+                        if (OBJECT_TYPE.equals(expected) && internal)
+                            return new Cast(CHAR_OBJ_TYPE, OBJECT_TYPE, explicit, null, null, CHAR_TYPE, null);
 
                         break;
                     case NUMBER:
+                        if (internal)
+                            return new Cast(CHAR_OBJ_TYPE, NUMBER_TYPE, explicit, null, null, CHAR_TYPE, null);
+
+                        break;
                     case CHAR_OBJ:
                         if (internal)
-                            return new Cast(actual, actual, explicit, false, false, false, true);
+                            return new Cast(CHAR_TYPE, CHAR_TYPE, explicit, null, null, null, CHAR_TYPE);
 
                         break;
                     case STRING:
-                        return new Cast(actual, Definition.STRING_TYPE, explicit, false, false, false, false);
+                        return new Cast(CHAR_TYPE, STRING_TYPE, explicit);
                     case INT_OBJ:
                         if (internal)
-                            return new Cast(actual, Definition.INT_TYPE, explicit, false, false, false, true);
+                            return new Cast(CHAR_TYPE, INT_TYPE, explicit, null, null, null, INT_TYPE);
 
                         break;
                     case LONG_OBJ:
                         if (internal)
-                            return new Cast(actual, Definition.LONG_TYPE, explicit, false, false, false, true);
+                            return new Cast(CHAR_TYPE, LONG_TYPE, explicit, null, null, null, LONG_TYPE);
 
                         break;
                     case FLOAT_OBJ:
                         if (internal)
-                            return new Cast(actual, Definition.FLOAT_TYPE, explicit, false, false, false, true);
+                            return new Cast(CHAR_TYPE, FLOAT_TYPE, explicit, null, null, null, FLOAT_TYPE);
 
                         break;
                     case DOUBLE_OBJ:
                         if (internal)
-                            return new Cast(actual, Definition.DOUBLE_TYPE, explicit, false, false, false, true);
+                            return new Cast(CHAR_TYPE, DOUBLE_TYPE, explicit, null, null, null, DOUBLE_TYPE);
 
                         break;
                     case BYTE_OBJ:
                         if (explicit && internal)
-                            return new Cast(actual, Definition.BYTE_TYPE, true, false, false, false, true);
+                            return new Cast(CHAR_TYPE, BYTE_TYPE, true, null, null, null, BYTE_TYPE);
 
                         break;
                     case SHORT_OBJ:
                         if (explicit && internal)
-                            return new Cast(actual, Definition.SHORT_TYPE, true, false, false, false, true);
+                            return new Cast(CHAR_TYPE, SHORT_TYPE, true, null, null, null, SHORT_TYPE);
 
                         break;
                 }
@@ -234,55 +272,59 @@ public static Cast getLegalCast(Location location, Type actual, Type expected, b
                     case LONG:
                     case FLOAT:
                     case DOUBLE:
-                        return new Cast(actual, expected, explicit);
+                        return new Cast(INT_TYPE, expected, explicit);
                     case BYTE:
                     case SHORT:
                     case CHAR:
                         if (explicit)
-                            return new Cast(actual, expected, true);
+                            return new Cast(INT_TYPE, expected, true);
 
                         break;
                     case DEF:
-                        return new Cast(actual, Definition.DEF_TYPE, explicit, false, false, true, false);
+                        return new Cast(INT_OBJ_TYPE, DEF_TYPE, explicit, null, null, INT_TYPE, null);
                     case OBJECT:
-                        if (Definition.OBJECT_TYPE.equals(expected) && internal)
-                            return new Cast(actual, actual, explicit, false, false, false, true);
+                        if (OBJECT_TYPE.equals(expected) && internal)
+                            return new Cast(INT_OBJ_TYPE, OBJECT_TYPE, explicit, null, null, INT_TYPE, null);
 
                         break;
                     case NUMBER:
+                        if (internal)
+                            return new Cast(INT_OBJ_TYPE, NUMBER_TYPE, explicit, null, null, INT_TYPE, null);
+
+                        break;
                     case INT_OBJ:
                         if (internal)
-                            return new Cast(actual, actual, explicit, false, false, false, true);
+                            return new Cast(INT_TYPE, INT_TYPE, explicit, null, null, null, INT_TYPE);
 
                         break;
                     case LONG_OBJ:
                         if (internal)
-                            return new Cast(actual, Definition.LONG_TYPE, explicit, false, false, false, true);
+                            return new Cast(INT_TYPE, LONG_TYPE, explicit, null, null, null, LONG_TYPE);
 
                         break;
                     case FLOAT_OBJ:
                         if (internal)
-                            return new Cast(actual, Definition.FLOAT_TYPE, explicit, false, false, false, true);
+                            return new Cast(INT_TYPE, FLOAT_TYPE, explicit, null, null, null, FLOAT_TYPE);
 
                         break;
                     case DOUBLE_OBJ:
                         if (internal)
-                            return new Cast(actual, Definition.DOUBLE_TYPE, explicit, false, false, false, true);
+                            return new Cast(INT_TYPE, DOUBLE_TYPE, explicit, null, null, null, DOUBLE_TYPE);
 
                         break;
                     case BYTE_OBJ:
                         if (explicit && internal)
-                            return new Cast(actual, Definition.BYTE_TYPE, true, false, false, false, true);
+                            return new Cast(INT_TYPE, BYTE_TYPE, true, null, null, null, BYTE_TYPE);
 
                         break;
                     case SHORT_OBJ:
                         if (explicit && internal)
-                            return new Cast(actual, Definition.SHORT_TYPE, true, false, false, false, true);
+                            return new Cast(INT_TYPE, SHORT_TYPE, true, null, null, null, SHORT_TYPE);
 
                         break;
                     case CHAR_OBJ:
                         if (explicit && internal)
-                            return new Cast(actual, Definition.CHAR_TYPE, true, false, false, false, true);
+                            return new Cast(INT_TYPE, CHAR_TYPE, true, null, null, null, CHAR_TYPE);
 
                         break;
                 }
@@ -292,7 +334,7 @@ public static Cast getLegalCast(Location location, Type actual, Type expected, b
                 switch (expected.sort) {
                     case FLOAT:
                     case DOUBLE:
-                        return new Cast(actual, expected, explicit);
+                        return new Cast(LONG_TYPE, expected, explicit);
                     case BYTE:
                     case SHORT:
                     case CHAR:
@@ -302,46 +344,50 @@ public static Cast getLegalCast(Location location, Type actual, Type expected, b
 
                         break;
                     case DEF:
-                        return new Cast(actual, Definition.DEF_TYPE, explicit, false, false, true, false);
+                        return new Cast(LONG_TYPE, DEF_TYPE, explicit, null, null, LONG_TYPE, null);
                     case OBJECT:
-                        if (Definition.OBJECT_TYPE.equals(expected) && internal)
-                            return new Cast(actual, actual, explicit, false, false, false, true);
+                        if (OBJECT_TYPE.equals(expected) && internal)
+                            return new Cast(LONG_TYPE, actual, explicit, null, null, LONG_TYPE, null);
 
                         break;
                     case NUMBER:
+                        if (internal)
+                            return new Cast(LONG_OBJ_TYPE, NUMBER_TYPE, explicit, null, null, LONG_TYPE, null);
+
+                        break;
                     case LONG_OBJ:
                         if (internal)
-                            return new Cast(actual, actual, explicit, false, false, false, true);
+                            return new Cast(LONG_TYPE, LONG_TYPE, explicit, null, null, null, LONG_TYPE);
 
                         break;
                     case FLOAT_OBJ:
                         if (internal)
-                            return new Cast(actual, Definition.FLOAT_TYPE, explicit, false, false, false, true);
+                            return new Cast(LONG_TYPE, FLOAT_TYPE, explicit, null, null, null, FLOAT_TYPE);
 
                         break;
                     case DOUBLE_OBJ:
                         if (internal)
-                            return new Cast(actual, Definition.DOUBLE_TYPE, explicit, false, false, false, true);
+                            return new Cast(LONG_TYPE, DOUBLE_TYPE, explicit, null, null, null, DOUBLE_TYPE);
 
                         break;
                     case BYTE_OBJ:
                         if (explicit && internal)
-                            return new Cast(actual, Definition.BYTE_TYPE, true, false, false, false, true);
+                            return new Cast(LONG_TYPE, BYTE_TYPE, true, null, null, null, BYTE_TYPE);
 
                         break;
                     case SHORT_OBJ:
                         if (explicit && internal)
-                            return new Cast(actual, Definition.SHORT_TYPE, true, false, false, false, true);
+                            return new Cast(LONG_TYPE, SHORT_TYPE, true, null, null, null, SHORT_TYPE);
 
                         break;
                     case CHAR_OBJ:
                         if (explicit && internal)
-                            return new Cast(actual, Definition.CHAR_TYPE, true, false, false, false, true);
+                            return new Cast(LONG_TYPE, CHAR_TYPE, true, null, null, null, CHAR_TYPE);
 
                         break;
                     case INT_OBJ:
                         if (explicit && internal)
-                            return new Cast(actual, Definition.INT_TYPE, true, false, false, false, true);
+                            return new Cast(LONG_TYPE, INT_TYPE, true, null, null, null, INT_TYPE);
 
                         break;
                 }
@@ -355,52 +401,56 @@ public static Cast getLegalCast(Location location, Type actual, Type expected, b
                     case SHORT:
                     case CHAR:
                     case INT:
-                    case FLOAT:
+                    case LONG:
                         if (explicit)
                             return new Cast(actual, expected, true);
 
                         break;
                     case DEF:
-                        return new Cast(actual, Definition.DEF_TYPE, explicit, false, false, true, false);
+                        return new Cast(FLOAT_OBJ_TYPE, DEF_TYPE, explicit, null, null, FLOAT_TYPE, null);
                     case OBJECT:
-                        if (Definition.OBJECT_TYPE.equals(expected) && internal)
-                            return new Cast(actual, actual, explicit, false, false, false, true);
+                        if (OBJECT_TYPE.equals(expected) && internal)
+                            return new Cast(FLOAT_OBJ_TYPE, OBJECT_TYPE, explicit, null, null, FLOAT_TYPE, null);
 
                         break;
                     case NUMBER:
+                        if (internal)
+                            return new Cast(FLOAT_OBJ_TYPE, NUMBER_TYPE, explicit, null, null, FLOAT_TYPE, null);
+
+                        break;
                     case FLOAT_OBJ:
                         if (internal)
-                            return new Cast(actual, actual, explicit, false, false, false, true);
+                            return new Cast(FLOAT_TYPE, FLOAT_TYPE, explicit, null, null, null, FLOAT_TYPE);
 
                         break;
                     case DOUBLE_OBJ:
                         if (internal)
-                            return new Cast(actual, Definition.DOUBLE_TYPE, explicit, false, false, false, true);
+                            return new Cast(FLOAT_TYPE, DOUBLE_TYPE, explicit, null, null, null, DOUBLE_TYPE);
 
                         break;
                     case BYTE_OBJ:
                         if (explicit && internal)
-                            return new Cast(actual, Definition.BYTE_TYPE, true, false, false, false, true);
+                            return new Cast(FLOAT_TYPE, BYTE_TYPE, true, null, null, null, BYTE_TYPE);
 
                         break;
                     case SHORT_OBJ:
                         if (explicit && internal)
-                            return new Cast(actual, Definition.SHORT_TYPE, true, false, false, false, true);
+                            return new Cast(FLOAT_TYPE, SHORT_TYPE, true, null, null, null, SHORT_TYPE);
 
                         break;
                     case CHAR_OBJ:
                         if (explicit && internal)
-                            return new Cast(actual, Definition.CHAR_TYPE, true, false, false, false, true);
+                            return new Cast(FLOAT_TYPE, CHAR_TYPE, true, null, null, null, CHAR_TYPE);
 
                         break;
                     case INT_OBJ:
                         if (explicit && internal)
-                            return new Cast(actual, Definition.INT_TYPE, true, false, false, false, true);
+                            return new Cast(FLOAT_TYPE, INT_TYPE, true, null, null, null, INT_TYPE);
 
                         break;
                     case LONG_OBJ:
                         if (explicit && internal)
-                            return new Cast(actual, Definition.LONG_TYPE, true, false, false, false, true);
+                            return new Cast(FLOAT_TYPE, LONG_TYPE, true, null, null, null, LONG_TYPE);
 
                         break;
                 }
@@ -412,93 +462,98 @@ public static Cast getLegalCast(Location location, Type actual, Type expected, b
                     case SHORT:
                     case CHAR:
                     case INT:
+                    case LONG:
                     case FLOAT:
                         if (explicit)
-                            return new Cast(actual, expected, true);
+                            return new Cast(DOUBLE_TYPE, expected, true);
 
                         break;
                     case DEF:
-                        return new Cast(actual, Definition.DEF_TYPE, explicit, false, false, true, false);
+                        return new Cast(DOUBLE_OBJ_TYPE, DEF_TYPE, explicit, null, null, DOUBLE_TYPE, null);
                     case OBJECT:
-                        if (Definition.OBJECT_TYPE.equals(expected) && internal)
-                            return new Cast(actual, actual, explicit, false, false, false, true);
+                        if (OBJECT_TYPE.equals(expected) && internal)
+                            return new Cast(DOUBLE_OBJ_TYPE, OBJECT_TYPE, explicit, null, null, DOUBLE_TYPE, null);
 
                         break;
                     case NUMBER:
+                        if (internal)
+                            return new Cast(DOUBLE_OBJ_TYPE, NUMBER_TYPE, explicit, null, null, DOUBLE_TYPE, null);
+
+                        break;
                     case DOUBLE_OBJ:
                         if (internal)
-                            return new Cast(actual, actual, explicit, false, false, false, true);
+                            return new Cast(DOUBLE_TYPE, DOUBLE_TYPE, explicit, null, null, null, DOUBLE_TYPE);
 
                         break;
                     case BYTE_OBJ:
                         if (explicit && internal)
-                            return new Cast(actual, Definition.BYTE_TYPE, true, false, false, false, true);
+                            return new Cast(DOUBLE_TYPE, BYTE_TYPE, true, null, null, null, BYTE_TYPE);
 
                         break;
                     case SHORT_OBJ:
                         if (explicit && internal)
-                            return new Cast(actual, Definition.SHORT_TYPE, true, false, false, false, true);
+                            return new Cast(DOUBLE_TYPE, SHORT_TYPE, true, null, null, null, SHORT_TYPE);
 
                         break;
                     case CHAR_OBJ:
                         if (explicit && internal)
-                            return new Cast(actual, Definition.CHAR_TYPE, true, false, false, false, true);
+                            return new Cast(DOUBLE_TYPE, CHAR_TYPE, true, null, null, null, CHAR_TYPE);
 
                         break;
                     case INT_OBJ:
                         if (explicit && internal)
-                            return new Cast(actual, Definition.INT_TYPE, true, false, false, false, true);
+                            return new Cast(DOUBLE_TYPE, INT_TYPE, true, null, null, null, INT_TYPE);
 
                         break;
                     case LONG_OBJ:
                         if (explicit && internal)
-                            return new Cast(actual, Definition.LONG_TYPE, true, false, false, false, true);
+                            return new Cast(DOUBLE_TYPE, LONG_TYPE, true, null, null, null, LONG_TYPE);
 
                         break;
                     case FLOAT_OBJ:
                         if (explicit && internal)
-                            return new Cast(actual, Definition.FLOAT_TYPE, true, false, false, false, true);
+                            return new Cast(DOUBLE_TYPE, FLOAT_TYPE, true, null, null, null, FLOAT_TYPE);
 
                         break;
                 }
 
                 break;
             case OBJECT:
-                if (Definition.OBJECT_TYPE.equals(actual))
+                if (OBJECT_TYPE.equals(actual))
                     switch (expected.sort) {
                         case BYTE:
                             if (internal && explicit)
-                                return new Cast(actual, Definition.BYTE_OBJ_TYPE, true, false, true, false, false);
+                                return new Cast(OBJECT_TYPE, BYTE_OBJ_TYPE, true, null, BYTE_TYPE, null, null);
 
                             break;
                         case SHORT:
                             if (internal && explicit)
-                                return new Cast(actual, Definition.SHORT_OBJ_TYPE, true, false, true, false, false);
+                                return new Cast(OBJECT_TYPE, SHORT_OBJ_TYPE, true, null, SHORT_TYPE, null, null);
 
                             break;
                         case CHAR:
                             if (internal && explicit)
-                                return new Cast(actual, Definition.CHAR_OBJ_TYPE, true, false, true, false, false);
+                                return new Cast(OBJECT_TYPE, CHAR_OBJ_TYPE, true, null, CHAR_TYPE, null, null);
 
                             break;
                         case INT:
                             if (internal && explicit)
-                                return new Cast(actual, Definition.INT_OBJ_TYPE, true, false, true, false, false);
+                                return new Cast(OBJECT_TYPE, INT_OBJ_TYPE, true, null, INT_TYPE, null, null);
 
                             break;
                         case LONG:
                             if (internal && explicit)
-                                return new Cast(actual, Definition.LONG_OBJ_TYPE, true, false, true, false, false);
+                                return new Cast(OBJECT_TYPE, LONG_OBJ_TYPE, true, null, LONG_TYPE, null, null);
 
                             break;
                         case FLOAT:
                             if (internal && explicit)
-                                return new Cast(actual, Definition.FLOAT_OBJ_TYPE, true, false, true, false, false);
+                                return new Cast(OBJECT_TYPE, FLOAT_OBJ_TYPE, true, null, FLOAT_TYPE, null, null);
 
                             break;
                         case DOUBLE:
                             if (internal && explicit)
-                                return new Cast(actual, Definition.DOUBLE_OBJ_TYPE, true, false, true, false, false);
+                                return new Cast(OBJECT_TYPE, DOUBLE_OBJ_TYPE, true, null, DOUBLE_TYPE, null, null);
 
                             break;
                     }
@@ -507,37 +562,37 @@ public static Cast getLegalCast(Location location, Type actual, Type expected, b
                 switch (expected.sort) {
                     case BYTE:
                         if (internal && explicit)
-                            return new Cast(actual, Definition.BYTE_OBJ_TYPE, true, false, true, false, false);
+                            return new Cast(NUMBER_TYPE, BYTE_OBJ_TYPE, true, null, BYTE_TYPE, null, null);
 
                         break;
                     case SHORT:
                         if (internal && explicit)
-                            return new Cast(actual, Definition.SHORT_OBJ_TYPE, true, false, true, false, false);
+                            return new Cast(NUMBER_TYPE, SHORT_OBJ_TYPE, true, null, SHORT_TYPE, null, null);
 
                         break;
                     case CHAR:
                         if (internal && explicit)
-                            return new Cast(actual, Definition.CHAR_OBJ_TYPE, true, false, true, false, false);
+                            return new Cast(NUMBER_TYPE, CHAR_OBJ_TYPE, true, null, CHAR_TYPE, null, null);
 
                         break;
                     case INT:
                         if (internal && explicit)
-                            return new Cast(actual, Definition.INT_OBJ_TYPE, true, false, true, false, false);
+                            return new Cast(NUMBER_TYPE, INT_OBJ_TYPE, true, null, INT_TYPE, null, null);
 
                         break;
                     case LONG:
                         if (internal && explicit)
-                            return new Cast(actual, Definition.LONG_OBJ_TYPE, true, false, true, false, false);
+                            return new Cast(NUMBER_TYPE, LONG_OBJ_TYPE, true, null, LONG_TYPE, null, null);
 
                         break;
                     case FLOAT:
                         if (internal && explicit)
-                            return new Cast(actual, Definition.FLOAT_OBJ_TYPE, true, false, true, false, false);
+                            return new Cast(NUMBER_TYPE, FLOAT_OBJ_TYPE, true, null, FLOAT_TYPE, null, null);
 
                         break;
                     case DOUBLE:
                         if (internal && explicit)
-                            return new Cast(actual, Definition.DOUBLE_OBJ_TYPE, true, false, true, false, false);
+                            return new Cast(NUMBER_TYPE, DOUBLE_OBJ_TYPE, true, null, DOUBLE_TYPE, null, null);
 
                         break;
                 }
@@ -547,7 +602,7 @@ public static Cast getLegalCast(Location location, Type actual, Type expected, b
                 switch (expected.sort) {
                     case BOOL:
                         if (internal)
-                            return new Cast(actual, expected, explicit, true, false, false, false);
+                            return new Cast(BOOLEAN_TYPE, BOOLEAN_TYPE, explicit, BOOLEAN_TYPE, null, null, null);
 
                         break;
                 }
@@ -562,12 +617,12 @@ public static Cast getLegalCast(Location location, Type actual, Type expected, b
                     case FLOAT:
                     case DOUBLE:
                         if (internal)
-                            return new Cast(actual, expected, explicit, true, false, false, false);
+                            return new Cast(BYTE_TYPE, expected, explicit, BYTE_TYPE, null, null, null);
 
                         break;
                     case CHAR:
                         if (internal && explicit)
-                            return new Cast(actual, expected, true, true, false, false, false);
+                            return new Cast(BYTE_TYPE, expected, true, BYTE_TYPE, null, null, null);
 
                         break;
                 }
@@ -581,13 +636,13 @@ public static Cast getLegalCast(Location location, Type actual, Type expected, b
                     case FLOAT:
                     case DOUBLE:
                         if (internal)
-                            return new Cast(actual, expected, explicit, true, false, false, false);
+                            return new Cast(SHORT_TYPE, expected, explicit, SHORT_TYPE, null, null, null);
 
                         break;
                     case BYTE:
                     case CHAR:
                         if (internal && explicit)
-                            return new Cast(actual, expected, true, true, false, false, false);
+                            return new Cast(SHORT_TYPE, expected, true, SHORT_TYPE, null, null, null);
 
                         break;
                 }
@@ -601,13 +656,13 @@ public static Cast getLegalCast(Location location, Type actual, Type expected, b
                     case FLOAT:
                     case DOUBLE:
                         if (internal)
-                            return new Cast(actual, expected, explicit, true, false, false, false);
+                            return new Cast(CHAR_TYPE, expected, explicit, CHAR_TYPE, null, null, null);
 
                         break;
                     case BYTE:
                     case SHORT:
                         if (internal && explicit)
-                            return new Cast(actual, expected, true, true, false, false, false);
+                            return new Cast(CHAR_TYPE, expected, true, CHAR_TYPE, null, null, null);
 
                         break;
                 }
@@ -620,14 +675,14 @@ public static Cast getLegalCast(Location location, Type actual, Type expected, b
                     case FLOAT:
                     case DOUBLE:
                         if (internal)
-                            return new Cast(actual, expected, explicit, true, false, false, false);
+                            return new Cast(INT_TYPE, expected, explicit, INT_TYPE, null, null, null);
 
                         break;
                     case BYTE:
                     case SHORT:
                     case CHAR:
                         if (internal && explicit)
-                            return new Cast(actual, expected, true, true, false, false, false);
+                            return new Cast(INT_TYPE, expected, true, INT_TYPE, null, null, null);
 
                         break;
                 }
@@ -639,7 +694,7 @@ public static Cast getLegalCast(Location location, Type actual, Type expected, b
                     case FLOAT:
                     case DOUBLE:
                         if (internal)
-                            return new Cast(actual, expected, explicit, true, false, false, false);
+                            return new Cast(LONG_TYPE, expected, explicit, LONG_TYPE, null, null, null);
 
                         break;
                     case BYTE:
@@ -647,7 +702,7 @@ public static Cast getLegalCast(Location location, Type actual, Type expected, b
                     case CHAR:
                     case INT:
                         if (internal && explicit)
-                            return new Cast(actual, expected, true, true, false, false, false);
+                            return new Cast(LONG_TYPE, expected, true, LONG_TYPE, null, null, null);
 
                         break;
                 }
@@ -658,7 +713,7 @@ public static Cast getLegalCast(Location location, Type actual, Type expected, b
                     case FLOAT:
                     case DOUBLE:
                         if (internal)
-                            return new Cast(actual, expected, explicit, true, false, false, false);
+                            return new Cast(FLOAT_TYPE, expected, explicit, FLOAT_TYPE, null, null, null);
 
                         break;
                     case BYTE:
@@ -667,7 +722,7 @@ public static Cast getLegalCast(Location location, Type actual, Type expected, b
                     case INT:
                     case LONG:
                         if (internal && explicit)
-                            return new Cast(actual, expected, true, true, false, false, false);
+                            return new Cast(FLOAT_TYPE, expected, true, FLOAT_TYPE, null, null, null);
 
                         break;
                 }
@@ -675,10 +730,9 @@ public static Cast getLegalCast(Location location, Type actual, Type expected, b
                 break;
             case DOUBLE_OBJ:
                 switch (expected.sort) {
-                    case FLOAT:
                     case DOUBLE:
                         if (internal)
-                            return new Cast(actual, expected, explicit, true, false, false, false);
+                            return new Cast(DOUBLE_TYPE, expected, explicit, DOUBLE_TYPE, null, null, null);
 
                         break;
                     case BYTE:
@@ -686,8 +740,9 @@ public static Cast getLegalCast(Location location, Type actual, Type expected, b
                     case CHAR:
                     case INT:
                     case LONG:
+                    case FLOAT:
                         if (internal && explicit)
-                            return new Cast(actual, expected, true, true, false, false, false);
+                            return new Cast(DOUBLE_TYPE, expected, true, DOUBLE_TYPE, null, null, null);
 
                         break;
                 }
@@ -696,14 +751,21 @@ public static Cast getLegalCast(Location location, Type actual, Type expected, b
             case DEF:
                 switch (expected.sort) {
                     case BOOL:
+                        return new Cast(DEF_TYPE, BOOLEAN_OBJ_TYPE, explicit, null, BOOLEAN_TYPE, null, null);
                     case BYTE:
+                        return new Cast(DEF_TYPE, BYTE_OBJ_TYPE, explicit, null, BYTE_TYPE, null, null);
                     case SHORT:
+                        return new Cast(DEF_TYPE, SHORT_OBJ_TYPE, explicit, null, SHORT_TYPE, null, null);
                     case CHAR:
+                        return new Cast(DEF_TYPE, CHAR_OBJ_TYPE, explicit, null, CHAR_TYPE, null, null);
                     case INT:
+                        return new Cast(DEF_TYPE, INT_OBJ_TYPE, explicit, null, INT_TYPE, null, null);
                     case LONG:
+                        return new Cast(DEF_TYPE, LONG_OBJ_TYPE, explicit, null, LONG_TYPE, null, null);
                     case FLOAT:
+                        return new Cast(DEF_TYPE, FLOAT_OBJ_TYPE, explicit, null, FLOAT_TYPE, null, null);
                     case DOUBLE:
-                            return new Cast(actual, expected, explicit, true, false, false, false);
+                            return new Cast(DEF_TYPE, DOUBLE_OBJ_TYPE, explicit, null, DOUBLE_TYPE, null, null);
                 }
 
                 break;
@@ -711,7 +773,7 @@ public static Cast getLegalCast(Location location, Type actual, Type expected, b
                 switch (expected.sort) {
                     case CHAR:
                         if (explicit)
-                            return new Cast(actual, expected, true, false, false, false, false);
+                            return new Cast(STRING_TYPE, CHAR_TYPE, true);
 
                         break;
                 }
@@ -719,9 +781,10 @@ public static Cast getLegalCast(Location location, Type actual, Type expected, b
                 break;
         }
 
-        if (actual.sort == Sort.DEF || expected.sort == Sort.DEF ||
-            expected.clazz.isAssignableFrom(actual.clazz) ||
-            explicit && actual.clazz.isAssignableFrom(expected.clazz)) {
+        if (       actual.sort == Sort.DEF
+                || (actual.sort != Sort.VOID && expected.sort == Sort.DEF)
+                || expected.clazz.isAssignableFrom(actual.clazz)
+                || (explicit && actual.clazz.isAssignableFrom(expected.clazz))) {
             return new Cast(actual, expected, explicit);
         } else {
             throw location.createError(new ClassCastException("Cannot cast from [" + actual.name + "] to [" + expected.name + "]."));
@@ -769,15 +832,15 @@ public static Type promoteNumeric(Type from, boolean decimal) {
         final Sort sort = from.sort;
 
         if (sort == Sort.DEF) {
-            return Definition.DEF_TYPE;
+            return DEF_TYPE;
         } else if ((sort == Sort.DOUBLE) && decimal) {
-            return Definition.DOUBLE_TYPE;
+            return DOUBLE_TYPE;
         } else if ((sort == Sort.FLOAT) && decimal) {
-            return  Definition.FLOAT_TYPE;
+            return  FLOAT_TYPE;
         } else if (sort == Sort.LONG) {
-            return Definition.LONG_TYPE;
+            return LONG_TYPE;
         } else if (sort == Sort.INT || sort == Sort.CHAR || sort == Sort.SHORT || sort == Sort.BYTE) {
-            return Definition.INT_TYPE;
+            return INT_TYPE;
         }
 
         return null;
@@ -788,24 +851,24 @@ public static Type promoteNumeric(Type from0, Type from1, boolean decimal) {
         final Sort sort1 = from1.sort;
 
         if (sort0 == Sort.DEF || sort1 == Sort.DEF) {
-            return Definition.DEF_TYPE;
+            return DEF_TYPE;
         }
 
         if (decimal) {
             if (sort0 == Sort.DOUBLE || sort1 == Sort.DOUBLE) {
-                return Definition.DOUBLE_TYPE;
+                return DOUBLE_TYPE;
             } else if (sort0 == Sort.FLOAT || sort1 == Sort.FLOAT) {
-                return Definition.FLOAT_TYPE;
+                return FLOAT_TYPE;
             }
         }
 
         if (sort0 == Sort.LONG || sort1 == Sort.LONG) {
-            return Definition.LONG_TYPE;
+            return LONG_TYPE;
         } else if (sort0 == Sort.INT   || sort1 == Sort.INT   ||
                    sort0 == Sort.CHAR  || sort1 == Sort.CHAR  ||
                    sort0 == Sort.SHORT || sort1 == Sort.SHORT ||
                    sort0 == Sort.BYTE  || sort1 == Sort.BYTE) {
-            return Definition.INT_TYPE;
+            return INT_TYPE;
         }
 
         return null;
@@ -816,7 +879,7 @@ public static Type promoteAdd(final Type from0, final Type from1) {
         final Sort sort1 = from1.sort;
 
         if (sort0 == Sort.STRING || sort1 == Sort.STRING) {
-            return Definition.STRING_TYPE;
+            return STRING_TYPE;
         }
 
         return promoteNumeric(from0, from1, true);
@@ -827,11 +890,11 @@ public static Type promoteXor(final Type from0, final Type from1) {
         final Sort sort1 = from1.sort;
 
         if (sort0 == Sort.DEF || sort1 == Sort.DEF) {
-            return Definition.DEF_TYPE;
+            return DEF_TYPE;
         }
 
         if (sort0.bool || sort1.bool) {
-            return Definition.BOOLEAN_TYPE;
+            return BOOLEAN_TYPE;
         }
 
         return promoteNumeric(from0, from1, false);
@@ -842,12 +905,12 @@ public static Type promoteEquality(final Type from0, final Type from1) {
         final Sort sort1 = from1.sort;
 
         if (sort0 == Sort.DEF || sort1 == Sort.DEF) {
-            return Definition.DEF_TYPE;
+            return DEF_TYPE;
         }
 
         if (sort0.primitive && sort1.primitive) {
             if (sort0.bool && sort1.bool) {
-                return Definition.BOOLEAN_TYPE;
+                return BOOLEAN_TYPE;
             }
 
             if (sort0.numeric && sort1.numeric) {
@@ -855,7 +918,7 @@ public static Type promoteEquality(final Type from0, final Type from1) {
             }
         }
 
-        return Definition.OBJECT_TYPE;
+        return OBJECT_TYPE;
     }
 
     public static Type promoteConditional(final Type from0, final Type from1, final Object const0, final Object const1) {
@@ -867,46 +930,46 @@ public static Type promoteConditional(final Type from0, final Type from1, final
         final Sort sort1 = from1.sort;
 
         if (sort0 == Sort.DEF || sort1 == Sort.DEF) {
-            return Definition.DEF_TYPE;
+            return DEF_TYPE;
         }
 
         if (sort0.primitive && sort1.primitive) {
             if (sort0.bool && sort1.bool) {
-                return Definition.BOOLEAN_TYPE;
+                return BOOLEAN_TYPE;
             }
 
             if (sort0 == Sort.DOUBLE || sort1 == Sort.DOUBLE) {
-                return Definition.DOUBLE_TYPE;
+                return DOUBLE_TYPE;
             } else if (sort0 == Sort.FLOAT || sort1 == Sort.FLOAT) {
-                return Definition.FLOAT_TYPE;
+                return FLOAT_TYPE;
             } else if (sort0 == Sort.LONG || sort1 == Sort.LONG) {
-                return Definition.LONG_TYPE;
+                return LONG_TYPE;
             } else {
                 if (sort0 == Sort.BYTE) {
                     if (sort1 == Sort.BYTE) {
-                        return Definition.BYTE_TYPE;
+                        return BYTE_TYPE;
                     } else if (sort1 == Sort.SHORT) {
                         if (const1 != null) {
                             final short constant = (short)const1;
 
                             if (constant <= Byte.MAX_VALUE && constant >= Byte.MIN_VALUE) {
-                                return Definition.BYTE_TYPE;
+                                return BYTE_TYPE;
                             }
                         }
 
-                        return Definition.SHORT_TYPE;
+                        return SHORT_TYPE;
                     } else if (sort1 == Sort.CHAR) {
-                        return Definition.INT_TYPE;
+                        return INT_TYPE;
                     } else if (sort1 == Sort.INT) {
                         if (const1 != null) {
                             final int constant = (int)const1;
 
                             if (constant <= Byte.MAX_VALUE && constant >= Byte.MIN_VALUE) {
-                                return Definition.BYTE_TYPE;
+                                return BYTE_TYPE;
                             }
                         }
 
-                        return Definition.INT_TYPE;
+                        return INT_TYPE;
                     }
                 } else if (sort0 == Sort.SHORT) {
                     if (sort1 == Sort.BYTE) {
@@ -914,43 +977,43 @@ public static Type promoteConditional(final Type from0, final Type from1, final
                             final short constant = (short)const0;
 
                             if (constant <= Byte.MAX_VALUE && constant >= Byte.MIN_VALUE) {
-                                return Definition.BYTE_TYPE;
+                                return BYTE_TYPE;
                             }
                         }
 
-                        return Definition.SHORT_TYPE;
+                        return SHORT_TYPE;
                     } else if (sort1 == Sort.SHORT) {
-                        return Definition.SHORT_TYPE;
+                        return SHORT_TYPE;
                     } else if (sort1 == Sort.CHAR) {
-                        return Definition.INT_TYPE;
+                        return INT_TYPE;
                     } else if (sort1 == Sort.INT) {
                         if (const1 != null) {
                             final int constant = (int)const1;
 
                             if (constant <= Short.MAX_VALUE && constant >= Short.MIN_VALUE) {
-                                return Definition.SHORT_TYPE;
+                                return SHORT_TYPE;
                             }
                         }
 
-                        return Definition.INT_TYPE;
+                        return INT_TYPE;
                     }
                 } else if (sort0 == Sort.CHAR) {
                     if (sort1 == Sort.BYTE) {
-                        return Definition.INT_TYPE;
+                        return INT_TYPE;
                     } else if (sort1 == Sort.SHORT) {
-                        return Definition.INT_TYPE;
+                        return INT_TYPE;
                     } else if (sort1 == Sort.CHAR) {
-                        return Definition.CHAR_TYPE;
+                        return CHAR_TYPE;
                     } else if (sort1 == Sort.INT) {
                         if (const1 != null) {
                             final int constant = (int)const1;
 
                             if (constant <= Character.MAX_VALUE && constant >= Character.MIN_VALUE) {
-                                return Definition.BYTE_TYPE;
+                                return BYTE_TYPE;
                             }
                         }
 
-                        return Definition.INT_TYPE;
+                        return INT_TYPE;
                     }
                 } else if (sort0 == Sort.INT) {
                     if (sort1 == Sort.BYTE) {
@@ -958,33 +1021,33 @@ public static Type promoteConditional(final Type from0, final Type from1, final
                             final int constant = (int)const0;
 
                             if (constant <= Byte.MAX_VALUE && constant >= Byte.MIN_VALUE) {
-                                return Definition.BYTE_TYPE;
+                                return BYTE_TYPE;
                             }
                         }
 
-                        return Definition.INT_TYPE;
+                        return INT_TYPE;
                     } else if (sort1 == Sort.SHORT) {
                         if (const0 != null) {
                             final int constant = (int)const0;
 
                             if (constant <= Short.MAX_VALUE && constant >= Short.MIN_VALUE) {
-                                return Definition.BYTE_TYPE;
+                                return BYTE_TYPE;
                             }
                         }
 
-                        return Definition.INT_TYPE;
+                        return INT_TYPE;
                     } else if (sort1 == Sort.CHAR) {
                         if (const0 != null) {
                             final int constant = (int)const0;
 
                             if (constant <= Character.MAX_VALUE && constant >= Character.MIN_VALUE) {
-                                return Definition.BYTE_TYPE;
+                                return BYTE_TYPE;
                             }
                         }
 
-                        return Definition.INT_TYPE;
+                        return INT_TYPE;
                     } else if (sort1 == Sort.INT) {
-                        return Definition.INT_TYPE;
+                        return INT_TYPE;
                     }
                 }
             }
@@ -994,7 +1057,7 @@ public static Type promoteConditional(final Type from0, final Type from1, final
         //       to calculate the highest upper bound for the two types and return that.
         //       However, for now we just return objectType that may require an extra cast.
 
-        return Definition.OBJECT_TYPE;
+        return OBJECT_TYPE;
     }
 
     private AnalyzerCaster() {}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
index b81a9dee24cb1..0d92f109eea44 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
@@ -30,6 +30,7 @@
 import java.security.SecureClassLoader;
 import java.security.cert.Certificate;
 import java.util.BitSet;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import static org.elasticsearch.painless.WriterConstants.CLASS_NAME;
 
@@ -43,7 +44,7 @@ final class Compiler {
     /**
      * The maximum number of characters allowed in the script source.
      */
-    static int MAXIMUM_SOURCE_LENGTH = 16384;
+    static final int MAXIMUM_SOURCE_LENGTH = 16384;
 
     /**
      * Define the class with lowest privileges.
@@ -66,6 +67,8 @@ final class Compiler {
      * A secure class loader used to define Painless scripts.
      */
     static final class Loader extends SecureClassLoader {
+        private final AtomicInteger lambdaCounter = new AtomicInteger(0);
+        
         /**
          * @param parent The parent ClassLoader.
          */
@@ -77,39 +80,63 @@ static final class Loader extends SecureClassLoader {
          * Generates a Class object from the generated byte code.
          * @param name The name of the class.
          * @param bytes The generated byte code.
-         * @return A Class object extending {@link Executable}.
+         * @return A Class object extending {@link PainlessScript}.
          */
-        Class<? extends Executable> define(String name, byte[] bytes) {
-            return defineClass(name, bytes, 0, bytes.length, CODESOURCE).asSubclass(Executable.class);
+        Class<? extends PainlessScript> defineScript(String name, byte[] bytes) {
+            return defineClass(name, bytes, 0, bytes.length, CODESOURCE).asSubclass(PainlessScript.class);
+        }
+
+        /**
+         * Generates a Class object for a lambda method.
+         * @param name The name of the class.
+         * @param bytes The generated byte code.
+         * @return A Class object.
+         */
+        Class<?> defineLambda(String name, byte[] bytes) {
+            return defineClass(name, bytes, 0, bytes.length, CODESOURCE);
+        }
+        
+        /**
+         * A counter used to generate a unique name for each lambda
+         * function/reference class in this classloader.
+         */
+        int newLambdaIdentifier() {
+            return lambdaCounter.getAndIncrement();
         }
     }
 
     /**
      * Runs the two-pass compiler to generate a Painless script.
+     * @param <T> the type of the script
      * @param loader The ClassLoader used to define the script.
+     * @param iface Interface the compiled script should implement
      * @param name The name of the script.
      * @param source The source code for the script.
      * @param settings The CompilerSettings to be used during the compilation.
-     * @return An {@link Executable} Painless script.
+     * @return An executable script that implements both {@code <T>} and is a subclass of {@link PainlessScript}
      */
-    static Executable compile(Loader loader, String name, String source, CompilerSettings settings) {
+    static <T> T compile(Loader loader, Class<T> iface, String name, String source, CompilerSettings settings) {
         if (source.length() > MAXIMUM_SOURCE_LENGTH) {
             throw new IllegalArgumentException("Scripts may be no longer than " + MAXIMUM_SOURCE_LENGTH +
                 " characters.  The passed in script is " + source.length() + " characters.  Consider using a" +
                 " plugin if a script longer than this length is a requirement.");
         }
+        Definition definition = Definition.BUILTINS;
+        ScriptInterface scriptInterface = new ScriptInterface(definition, iface);
 
-        SSource root = Walker.buildPainlessTree(name, source, settings, null);
+        SSource root = Walker.buildPainlessTree(scriptInterface, name, source, settings, definition,
+                null);
 
-        root.analyze();
+        root.analyze(definition);
         root.write();
 
         try {
-            Class<? extends Executable> clazz = loader.define(CLASS_NAME, root.getBytes());
-            java.lang.reflect.Constructor<? extends Executable> constructor =
+            Class<? extends PainlessScript> clazz = loader.defineScript(CLASS_NAME, root.getBytes());
+            clazz.getField("$DEFINITION").set(null, definition);
+            java.lang.reflect.Constructor<? extends PainlessScript> constructor =
                     clazz.getConstructor(String.class, String.class, BitSet.class);
 
-            return constructor.newInstance(name, source, root.getStatements());
+            return iface.cast(constructor.newInstance(name, source, root.getStatements()));
         } catch (Exception exception) { // Catch everything to let the user know this is something caused internally.
             throw new IllegalStateException("An internal error occurred attempting to define the script [" + name + "].", exception);
         }
@@ -117,20 +144,24 @@ static Executable compile(Loader loader, String name, String source, CompilerSet
 
     /**
      * Runs the two-pass compiler to generate a Painless script.  (Used by the debugger.)
+     * @param iface Interface the compiled script should implement
      * @param source The source code for the script.
      * @param settings The CompilerSettings to be used during the compilation.
      * @return The bytes for compilation.
      */
-    static byte[] compile(String name, String source, CompilerSettings settings, Printer debugStream) {
+    static byte[] compile(Class<?> iface, String name, String source, CompilerSettings settings, Printer debugStream) {
         if (source.length() > MAXIMUM_SOURCE_LENGTH) {
             throw new IllegalArgumentException("Scripts may be no longer than " + MAXIMUM_SOURCE_LENGTH +
                 " characters.  The passed in script is " + source.length() + " characters.  Consider using a" +
                 " plugin if a script longer than this length is a requirement.");
         }
+        Definition definition = Definition.BUILTINS;
+        ScriptInterface scriptInterface = new ScriptInterface(definition, iface);
 
-        SSource root = Walker.buildPainlessTree(name, source, settings, debugStream);
+        SSource root = Walker.buildPainlessTree(scriptInterface, name, source, settings, definition,
+                debugStream);
 
-        root.analyze();
+        root.analyze(definition);
         root.write();
 
         return root.getBytes();
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java
index f0e1bde74d097..e723081e36c0c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java
@@ -19,10 +19,18 @@
 
 package org.elasticsearch.painless;
 
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
+
 /**
  * Settings to use when compiling a script.
  */
 public final class CompilerSettings {
+    /**
+     * Are regexes enabled? This is a node level setting because regexes break out of painless's lovely sandbox and can cause stack
+     * overflows and we can't analyze the regex to be sure it won't.
+     */
+    public static final Setting<Boolean> REGEX_ENABLED = Setting.boolSetting("script.painless.regex.enabled", false, Property.NodeScope);
 
     /**
      * Constant to be used when specifying the maximum loop counter when compiling a script.
@@ -33,28 +41,37 @@ public final class CompilerSettings {
      * Constant to be used for enabling additional internal compilation checks (slower).
      */
     public static final String PICKY = "picky";
-    
+
     /**
-     * For testing: do not use.
+     * Hack to set the initial "depth" for the {@link DefBootstrap.PIC} and {@link DefBootstrap.MIC}. Only used for testing: do not
+     * overwrite.
      */
     public static final String INITIAL_CALL_SITE_DEPTH = "initialCallSiteDepth";
 
     /**
      * The maximum number of statements allowed to be run in a loop.
+     * For now the number is set fairly high to accommodate users
+     * doing large update queries.
      */
-    private int maxLoopCounter = 10000;
+    private int maxLoopCounter = 1000000;
 
     /**
      * Whether to throw exception on ambiguity or other internal parsing issues. This option
      * makes things slower too, it is only for debugging.
      */
     private boolean picky = false;
-    
+
     /**
      * For testing. Do not use.
      */
     private int initialCallSiteDepth = 0;
 
+    /**
+     * Are regexes enabled? They are currently disabled by default because they break out of the loop counter and even fairly simple
+     * <strong>looking</strong> regexes can cause stack overflows.
+     */
+    private boolean regexesEnabled = false;
+
     /**
      * Returns the value for the cumulative total number of statements that can be made in all loops
      * in a script before an exception is thrown.  This attempts to prevent infinite loops.  Note if
@@ -88,7 +105,7 @@ public boolean isPicky() {
     public void setPicky(boolean picky) {
       this.picky = picky;
     }
-    
+
     /**
      * Returns initial call site depth. This means we pretend we've already seen N different types,
      * to better exercise fallback code in tests.
@@ -96,7 +113,7 @@ public void setPicky(boolean picky) {
     public int getInitialCallSiteDepth() {
         return initialCallSiteDepth;
     }
-    
+
     /**
      * For testing megamorphic fallbacks. Do not use.
      * @see #getInitialCallSiteDepth()
@@ -104,4 +121,20 @@ public int getInitialCallSiteDepth() {
     public void setInitialCallSiteDepth(int depth) {
         this.initialCallSiteDepth = depth;
     }
+
+    /**
+     * Are regexes enabled? They are currently disabled by default because they break out of the loop counter and even fairly simple
+     * <strong>looking</strong> regexes can cause stack overflows.
+     */
+    public boolean areRegexesEnabled() {
+        return regexesEnabled;
+    }
+
+    /**
+     * Are regexes enabled? They are currently disabled by default because they break out of the loop counter and even fairly simple
+     * <strong>looking</strong> regexes can cause stack overflows.
+     */
+    public void setRegexesEnabled(boolean regexesEnabled) {
+        this.regexesEnabled = regexesEnabled;
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index cd761d0ad449d..ebd25032220fe 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.painless.Definition.RuntimeClass;
 
 import java.lang.invoke.CallSite;
-import java.lang.invoke.LambdaMetafactory;
 import java.lang.invoke.MethodHandle;
 import java.lang.invoke.MethodHandles;
 import java.lang.invoke.MethodHandles.Lookup;
@@ -109,8 +108,12 @@ private ArrayLengthHelper() {}
     private static final MethodHandle LIST_SET;
     /** pointer to Iterable.iterator() */
     private static final MethodHandle ITERATOR;
-    /** factory for arraylength MethodHandle (intrinsic) from Java 9 */
-    private static final MethodHandle JAVA9_ARRAY_LENGTH_MH_FACTORY;
+    /** pointer to {@link Def#mapIndexNormalize}. */
+    private static final MethodHandle MAP_INDEX_NORMALIZE;
+    /** pointer to {@link Def#listIndexNormalize}. */
+    private static final MethodHandle LIST_INDEX_NORMALIZE;
+    /** factory for arraylength MethodHandle (intrinsic) from Java 9 (pkg-private for tests) */
+    static final MethodHandle JAVA9_ARRAY_LENGTH_MH_FACTORY;
 
     static {
         final Lookup lookup = MethodHandles.publicLookup();
@@ -121,10 +124,14 @@ private ArrayLengthHelper() {}
             LIST_GET = lookup.findVirtual(List.class, "get", MethodType.methodType(Object.class, int.class));
             LIST_SET = lookup.findVirtual(List.class, "set", MethodType.methodType(Object.class, int.class, Object.class));
             ITERATOR = lookup.findVirtual(Iterable.class, "iterator", MethodType.methodType(Iterator.class));
+            MAP_INDEX_NORMALIZE = lookup.findStatic(Def.class, "mapIndexNormalize",
+                    MethodType.methodType(Object.class, Map.class, Object.class));
+            LIST_INDEX_NORMALIZE = lookup.findStatic(Def.class, "listIndexNormalize",
+                    MethodType.methodType(int.class, List.class, int.class));
         } catch (final ReflectiveOperationException roe) {
             throw new AssertionError(roe);
         }
-        
+
         // lookup up the factory for arraylength MethodHandle (intrinsic) from Java 9:
         // https://bugs.openjdk.java.net/browse/JDK-8156915
         MethodHandle arrayLengthMHFactory;
@@ -142,7 +149,7 @@ private ArrayLengthHelper() {}
     static <T extends Throwable> void rethrow(Throwable t) throws T {
         throw (T) t;
     }
-    
+
     /** Returns an array length getter MethodHandle for the given array type */
     static MethodHandle arrayLengthGetter(Class<?> arrayType) {
         if (JAVA9_ARRAY_LENGTH_MH_FACTORY != null) {
@@ -167,17 +174,18 @@ static MethodHandle arrayLengthGetter(Class<?> arrayType) {
      * until it finds a matching whitelisted method. If one is not found, it throws an exception.
      * Otherwise it returns the matching method.
      * <p>
+     * @params definition the whitelist
      * @param receiverClass Class of the object to invoke the method on.
      * @param name Name of the method.
      * @param arity arity of method
      * @return matching method to invoke. never returns null.
      * @throws IllegalArgumentException if no matching whitelisted method was found.
      */
-    static Method lookupMethodInternal(Class<?> receiverClass, String name, int arity) {
+    static Method lookupMethodInternal(Definition definition, Class<?> receiverClass, String name, int arity) {
         Definition.MethodKey key = new Definition.MethodKey(name, arity);
         // check whitelist for matching method
         for (Class<?> clazz = receiverClass; clazz != null; clazz = clazz.getSuperclass()) {
-            RuntimeClass struct = Definition.getRuntimeClass(clazz);
+            RuntimeClass struct = definition.getRuntimeClass(clazz);
 
             if (struct != null) {
                 Method method = struct.methods.get(key);
@@ -187,7 +195,7 @@ static Method lookupMethodInternal(Class<?> receiverClass, String name, int arit
             }
 
             for (Class<?> iface : clazz.getInterfaces()) {
-                struct = Definition.getRuntimeClass(iface);
+                struct = definition.getRuntimeClass(iface);
 
                 if (struct != null) {
                     Method method = struct.methods.get(key);
@@ -197,7 +205,7 @@ static Method lookupMethodInternal(Class<?> receiverClass, String name, int arit
                 }
             }
         }
-        
+
         throw new IllegalArgumentException("Unable to find dynamic method [" + name + "] with [" + arity + "] arguments " +
                                            "for class [" + receiverClass.getCanonicalName() + "].");
     }
@@ -212,6 +220,7 @@ static Method lookupMethodInternal(Class<?> receiverClass, String name, int arit
      * until it finds a matching whitelisted method. If one is not found, it throws an exception.
      * Otherwise it returns a handle to the matching method.
      * <p>
+     * @param definition the whitelist
      * @param lookup caller's lookup
      * @param callSiteType callsite's type
      * @param receiverClass Class of the object to invoke the method on.
@@ -221,23 +230,23 @@ static Method lookupMethodInternal(Class<?> receiverClass, String name, int arit
      * @throws IllegalArgumentException if no matching whitelisted method was found.
      * @throws Throwable if a method reference cannot be converted to an functional interface
      */
-     static MethodHandle lookupMethod(Lookup lookup, MethodType callSiteType, 
+    static MethodHandle lookupMethod(Definition definition, Lookup lookup, MethodType callSiteType,
              Class<?> receiverClass, String name, Object args[]) throws Throwable {
          String recipeString = (String) args[0];
          int numArguments = callSiteType.parameterCount();
          // simple case: no lambdas
          if (recipeString.isEmpty()) {
-             return lookupMethodInternal(receiverClass, name, numArguments - 1).handle;
+             return lookupMethodInternal(definition, receiverClass, name, numArguments - 1).handle;
          }
-         
+
          // convert recipe string to a bitset for convenience (the code below should be refactored...)
-         BitSet lambdaArgs = new BitSet();
+         BitSet lambdaArgs = new BitSet(recipeString.length());
          for (int i = 0; i < recipeString.length(); i++) {
              lambdaArgs.set(recipeString.charAt(i));
          }
 
          // otherwise: first we have to compute the "real" arity. This is because we have extra arguments:
-         // e.g. f(a, g(x), b, h(y), i()) looks like f(a, g, x, b, h, y, i). 
+         // e.g. f(a, g(x), b, h(y), i()) looks like f(a, g, x, b, h, y, i).
          int arity = callSiteType.parameterCount() - 1;
          int upTo = 1;
          for (int i = 1; i < numArguments; i++) {
@@ -247,10 +256,10 @@ static MethodHandle lookupMethod(Lookup lookup, MethodType callSiteType,
                  arity -= numCaptures;
              }
          }
-         
+
          // lookup the method with the proper arity, then we know everything (e.g. interface types of parameters).
          // based on these we can finally link any remaining lambdas that were deferred.
-         Method method = lookupMethodInternal(receiverClass, name, arity);
+         Method method = lookupMethodInternal(definition, receiverClass, name, arity);
          MethodHandle handle = method.handle;
 
          int replaced = 0;
@@ -258,9 +267,9 @@ static MethodHandle lookupMethod(Lookup lookup, MethodType callSiteType,
          for (int i = 1; i < numArguments; i++) {
              // its a functional reference, replace the argument with an impl
              if (lambdaArgs.get(i - 1)) {
-                 // decode signature of form 'type.call,2' 
+                 // decode signature of form 'type.call,2'
                  String signature = (String) args[upTo++];
-                 int separator = signature.indexOf('.');
+                 int separator = signature.lastIndexOf('.');
                  int separator2 = signature.indexOf(',');
                  String type = signature.substring(1, separator);
                  String call = signature.substring(separator+1, separator2);
@@ -274,7 +283,8 @@ static MethodHandle lookupMethod(Lookup lookup, MethodType callSiteType,
                  if (signature.charAt(0) == 'S') {
                      // the implementation is strongly typed, now that we know the interface type,
                      // we have everything.
-                     filter = lookupReferenceInternal(lookup,
+                     filter = lookupReferenceInternal(definition,
+                                                      lookup,
                                                       interfaceType,
                                                       type,
                                                       call,
@@ -284,7 +294,8 @@ static MethodHandle lookupMethod(Lookup lookup, MethodType callSiteType,
                      // this is dynamically based on the receiver type (and cached separately, underneath
                      // this cache). It won't blow up since we never nest here (just references)
                      MethodType nestedType = MethodType.methodType(interfaceType.clazz, captures);
-                     CallSite nested = DefBootstrap.bootstrap(lookup, 
+                     CallSite nested = DefBootstrap.bootstrap(definition,
+                                                              lookup,
                                                               call,
                                                               nestedType,
                                                               0,
@@ -301,31 +312,33 @@ static MethodHandle lookupMethod(Lookup lookup, MethodType callSiteType,
                  replaced += numCaptures;
              }
          }
-         
+
          return handle;
      }
-     
+
      /**
       * Returns an implementation of interfaceClass that calls receiverClass.name
       * <p>
       * This is just like LambdaMetaFactory, only with a dynamic type. The interface type is known,
       * so we simply need to lookup the matching implementation method based on receiver type.
       */
-     static MethodHandle lookupReference(Lookup lookup, String interfaceClass, 
-                                         Class<?> receiverClass, String name) throws Throwable {
-         Definition.Type interfaceType = Definition.getType(interfaceClass);
+    static MethodHandle lookupReference(Definition definition, Lookup lookup, String interfaceClass,
+            Class<?> receiverClass, String name) throws Throwable {
+         Definition.Type interfaceType = definition.getType(interfaceClass);
          Method interfaceMethod = interfaceType.struct.getFunctionalMethod();
          if (interfaceMethod == null) {
              throw new IllegalArgumentException("Class [" + interfaceClass + "] is not a functional interface");
          }
          int arity = interfaceMethod.arguments.size();
-         Method implMethod = lookupMethodInternal(receiverClass, name, arity);
-         return lookupReferenceInternal(lookup, interfaceType, implMethod.owner.name, implMethod.name, receiverClass);
+         Method implMethod = lookupMethodInternal(definition, receiverClass, name, arity);
+        return lookupReferenceInternal(definition, lookup, interfaceType, implMethod.owner.name,
+                implMethod.name, receiverClass);
      }
-     
+
      /** Returns a method handle to an implementation of clazz, given method reference signature. */
-     private static MethodHandle lookupReferenceInternal(Lookup lookup, Definition.Type clazz, String type,
-                                                         String call, Class<?>... captures) throws Throwable {
+    private static MethodHandle lookupReferenceInternal(Definition definition, Lookup lookup,
+            Definition.Type clazz, String type, String call, Class<?>... captures)
+            throws Throwable {
          final FunctionRef ref;
          if ("this".equals(type)) {
              // user written method
@@ -337,47 +350,37 @@ private static MethodHandle lookupReferenceInternal(Lookup lookup, Definition.Ty
              int arity = interfaceMethod.arguments.size() + captures.length;
              final MethodHandle handle;
              try {
-                 MethodHandle accessor = lookup.findStaticGetter(lookup.lookupClass(), 
-                                                                 getUserFunctionHandleFieldName(call, arity), 
+                 MethodHandle accessor = lookup.findStaticGetter(lookup.lookupClass(),
+                                                                 getUserFunctionHandleFieldName(call, arity),
                                                                  MethodHandle.class);
-                 handle = (MethodHandle) accessor.invokeExact();
+                 handle = (MethodHandle)accessor.invokeExact();
              } catch (NoSuchFieldException | IllegalAccessException e) {
                  // is it a synthetic method? If we generated the method ourselves, be more helpful. It can only fail
                  // because the arity does not match the expected interface type.
                  if (call.contains("$")) {
-                     throw new IllegalArgumentException("Incorrect number of parameters for [" + interfaceMethod.name + 
+                     throw new IllegalArgumentException("Incorrect number of parameters for [" + interfaceMethod.name +
                                                         "] in [" + clazz.clazz + "]");
                  }
                  throw new IllegalArgumentException("Unknown call [" + call + "] with [" + arity + "] arguments.");
              }
-             ref = new FunctionRef(clazz, interfaceMethod, handle, captures.length);
+             ref = new FunctionRef(clazz, interfaceMethod, call, handle.type(), captures.length);
          } else {
              // whitelist lookup
-             ref = new FunctionRef(clazz, type, call, captures.length);
-         }
-         final CallSite callSite;
-         if (ref.needsBridges()) {
-             callSite = LambdaMetafactory.altMetafactory(lookup, 
-                     ref.invokedName, 
-                     ref.invokedType,
-                     ref.samMethodType,
-                     ref.implMethod,
-                     ref.samMethodType,
-                     LambdaMetafactory.FLAG_BRIDGES,
-                     1,
-                     ref.interfaceMethodType);
-         } else {
-             callSite = LambdaMetafactory.altMetafactory(lookup, 
-                     ref.invokedName, 
-                     ref.invokedType,
-                     ref.samMethodType,
-                     ref.implMethod,
-                     ref.samMethodType,
-                     0);
+             ref = new FunctionRef(definition, clazz, type, call, captures.length);
          }
+         final CallSite callSite = LambdaBootstrap.lambdaBootstrap(
+             lookup,
+             ref.interfaceMethodName,
+             ref.factoryMethodType,
+             ref.interfaceMethodType,
+             ref.delegateClassName,
+             ref.delegateInvokeType,
+             ref.delegateMethodName,
+             ref.delegateMethodType
+         );
          return callSite.dynamicInvoker().asType(MethodType.methodType(clazz.clazz, captures));
      }
-     
+
      /** gets the field name used to lookup up the MethodHandle for a function. */
      public static String getUserFunctionHandleFieldName(String name, int arity) {
          return "handle$" + name + "$" + arity;
@@ -403,15 +406,16 @@ public static String getUserFunctionHandleFieldName(String name, int arity) {
      * until it finds a matching whitelisted getter. If one is not found, it throws an exception.
      * Otherwise it returns a handle to the matching getter.
      * <p>
+     * @param definition the whitelist
      * @param receiverClass Class of the object to retrieve the field from.
      * @param name Name of the field.
      * @return pointer to matching field. never returns null.
      * @throws IllegalArgumentException if no matching whitelisted field was found.
      */
-    static MethodHandle lookupGetter(Class<?> receiverClass, String name) {
+    static MethodHandle lookupGetter(Definition definition, Class<?> receiverClass, String name) {
         // first try whitelist
         for (Class<?> clazz = receiverClass; clazz != null; clazz = clazz.getSuperclass()) {
-            RuntimeClass struct = Definition.getRuntimeClass(clazz);
+            RuntimeClass struct = definition.getRuntimeClass(clazz);
 
             if (struct != null) {
                 MethodHandle handle = struct.getters.get(name);
@@ -421,7 +425,7 @@ static MethodHandle lookupGetter(Class<?> receiverClass, String name) {
             }
 
             for (final Class<?> iface : clazz.getInterfaces()) {
-                struct = Definition.getRuntimeClass(iface);
+                struct = definition.getRuntimeClass(iface);
 
                 if (struct != null) {
                     MethodHandle handle = struct.getters.get(name);
@@ -473,15 +477,16 @@ static MethodHandle lookupGetter(Class<?> receiverClass, String name) {
      * until it finds a matching whitelisted setter. If one is not found, it throws an exception.
      * Otherwise it returns a handle to the matching setter.
      * <p>
+     * @param definition the whitelist
      * @param receiverClass Class of the object to retrieve the field from.
      * @param name Name of the field.
      * @return pointer to matching field. never returns null.
      * @throws IllegalArgumentException if no matching whitelisted field was found.
      */
-    static MethodHandle lookupSetter(Class<?> receiverClass, String name) {
+    static MethodHandle lookupSetter(Definition definition, Class<?> receiverClass, String name) {
         // first try whitelist
         for (Class<?> clazz = receiverClass; clazz != null; clazz = clazz.getSuperclass()) {
-            RuntimeClass struct = Definition.getRuntimeClass(clazz);
+            RuntimeClass struct = definition.getRuntimeClass(clazz);
 
             if (struct != null) {
                 MethodHandle handle = struct.setters.get(name);
@@ -491,7 +496,7 @@ static MethodHandle lookupSetter(Class<?> receiverClass, String name) {
             }
 
             for (final Class<?> iface : clazz.getInterfaces()) {
-                struct = Definition.getRuntimeClass(iface);
+                struct = definition.getRuntimeClass(iface);
 
                 if (struct != null) {
                     MethodHandle handle = struct.setters.get(name);
@@ -522,6 +527,26 @@ static MethodHandle lookupSetter(Class<?> receiverClass, String name) {
                                            "for class [" + receiverClass.getCanonicalName() + "].");
     }
 
+    /**
+     * Returns a method handle to normalize the index into an array. This is what makes lists and arrays stored in {@code def} support
+     * negative offsets.
+     * @param receiverClass Class of the array to store the value in
+     * @return a MethodHandle that accepts the receiver as first argument, the index as second argument, and returns the normalized index
+     *   to use with array loads and array stores
+     */
+    static MethodHandle lookupIndexNormalize(Class<?> receiverClass) {
+        if (receiverClass.isArray()) {
+            return ArrayIndexNormalizeHelper.arrayIndexNormalizer(receiverClass);
+        } else if (Map.class.isAssignableFrom(receiverClass)) {
+            // noop so that mymap[key] doesn't do funny things with negative keys
+            return MAP_INDEX_NORMALIZE;
+        } else if (List.class.isAssignableFrom(receiverClass)) {
+            return LIST_INDEX_NORMALIZE;
+        }
+        throw new IllegalArgumentException("Attempting to address a non-array-like type " +
+                                           "[" + receiverClass.getCanonicalName() + "] as an array.");
+    }
+
     /**
      * Returns a method handle to do an array store.
      * @param receiverClass Class of the array to store the value in
@@ -559,7 +584,7 @@ static MethodHandle lookupArrayLoad(Class<?> receiverClass) {
         throw new IllegalArgumentException("Attempting to address a non-array type " +
                                            "[" + receiverClass.getCanonicalName() + "] as an array.");
     }
-    
+
     /** Helper class for isolating MethodHandles and methods to get iterators over arrays
      * (to emulate "enhanced for loop" using MethodHandles). These cause boxing, and are not as efficient
      * as they could be, but works.
@@ -814,4 +839,62 @@ public static double DefTodoubleExplicit(final Object value) {
             return ((Number)value).doubleValue();
         }
     }
+
+    /**
+     * "Normalizes" the index into a {@code Map} by making no change to the index.
+     */
+    public static Object mapIndexNormalize(final Map<?, ?> value, Object index) {
+        return index;
+    }
+
+    /**
+     * "Normalizes" the idnex into a {@code List} by flipping negative indexes around so they are "from the end" of the list.
+     */
+    public static int listIndexNormalize(final List<?> value, int index) {
+        return index >= 0 ? index : value.size() + index;
+    }
+
+    /**
+     * Methods to normalize array indices to support negative indices into arrays stored in {@code def}s.
+     */
+    @SuppressWarnings("unused") // normalizeIndex() methods are are actually used, javac just does not know :)
+    private static final class ArrayIndexNormalizeHelper {
+        private static final Lookup PRIV_LOOKUP = MethodHandles.lookup();
+
+        private static final Map<Class<?>,MethodHandle> ARRAY_TYPE_MH_MAPPING = Collections.unmodifiableMap(
+            Stream.of(boolean[].class, byte[].class, short[].class, int[].class, long[].class,
+                char[].class, float[].class, double[].class, Object[].class)
+                .collect(Collectors.toMap(Function.identity(), type -> {
+                    try {
+                        return PRIV_LOOKUP.findStatic(PRIV_LOOKUP.lookupClass(), "normalizeIndex",
+                                MethodType.methodType(int.class, type, int.class));
+                    } catch (ReflectiveOperationException e) {
+                        throw new AssertionError(e);
+                    }
+                }))
+        );
+
+        private static final MethodHandle OBJECT_ARRAY_MH = ARRAY_TYPE_MH_MAPPING.get(Object[].class);
+
+        static int normalizeIndex(final boolean[] array, final int index) { return index >= 0 ? index : index + array.length; }
+        static int normalizeIndex(final byte[] array, final int index) { return index >= 0 ? index : index + array.length; }
+        static int normalizeIndex(final short[] array, final int index) { return index >= 0 ? index : index + array.length; }
+        static int normalizeIndex(final int[] array, final int index) { return index >= 0 ? index : index + array.length; }
+        static int normalizeIndex(final long[] array, final int index) { return index >= 0 ? index : index + array.length; }
+        static int normalizeIndex(final char[] array, final int index) { return index >= 0 ? index : index + array.length; }
+        static int normalizeIndex(final float[] array, final int index) { return index >= 0 ? index : index + array.length; }
+        static int normalizeIndex(final double[] array, final int index) { return index >= 0 ? index : index + array.length; }
+        static int normalizeIndex(final Object[] array, final int index) { return index >= 0 ? index : index + array.length; }
+
+        static MethodHandle arrayIndexNormalizer(Class<?> arrayType) {
+            if (!arrayType.isArray()) {
+                throw new IllegalArgumentException("type must be an array");
+            }
+            return (ARRAY_TYPE_MH_MAPPING.containsKey(arrayType)) ?
+                ARRAY_TYPE_MH_MAPPING.get(arrayType) :
+                OBJECT_ARRAY_MH.asType(OBJECT_ARRAY_MH.type().changeParameterType(0, arrayType));
+        }
+
+        private ArrayIndexNormalizeHelper() {}
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
index 9640629cb87e8..31fba8f757954 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
@@ -32,9 +32,10 @@
 /**
  * Painless invokedynamic bootstrap for the call site.
  * <p>
- * Has 7 flavors (passed as static bootstrap parameters): dynamic method call,
+ * Has 11 flavors (passed as static bootstrap parameters): dynamic method call,
  * dynamic field load (getter), and dynamic field store (setter), dynamic array load,
- * dynamic array store, iterator, and method reference.
+ * dynamic array store, iterator, method reference, unary operator, binary operator,
+ * shift operator, and dynamic array index normalize.
  * <p>
  * When a new type is encountered at the call site, we lookup from the appropriate
  * whitelist, and cache with a guard. If we encounter too many types, we stop caching.
@@ -69,6 +70,8 @@ private DefBootstrap() {} // no instance!
     public static final int BINARY_OPERATOR = 8;
     /** static bootstrap parameter indicating a shift operator, e.g. foo &gt;&gt; bar */
     public static final int SHIFT_OPERATOR = 9;
+    /** static bootstrap parameter indicating a request to normalize an index for array-like-access */
+    public static final int INDEX_NORMALIZE = 10;
     
     // constants for the flags parameter of operators
     /** 
@@ -101,17 +104,19 @@ static final class PIC extends MutableCallSite {
         /** maximum number of types before we go megamorphic */
         static final int MAX_DEPTH = 5;
 
+        private final Definition definition;
         private final Lookup lookup;
         private final String name;
         private final int flavor;
         private final Object[] args;
         int depth; // pkg-protected for testing
 
-        PIC(Lookup lookup, String name, MethodType type, int initialDepth, int flavor, Object[] args) {
+        PIC(Definition definition, Lookup lookup, String name, MethodType type, int initialDepth, int flavor, Object[] args) {
             super(type);
             if (type.parameterType(0) != Object.class) {
                 throw new BootstrapMethodError("The receiver type (1st arg) of invokedynamic descriptor must be Object.");
             }
+            this.definition = definition;
             this.lookup = lookup;
             this.name = name;
             this.flavor = flavor;
@@ -139,11 +144,11 @@ static boolean checkClass(Class<?> clazz, Object receiver) {
         private MethodHandle lookup(int flavor, String name, Class<?> receiver) throws Throwable {
             switch(flavor) {
                 case METHOD_CALL:
-                    return Def.lookupMethod(lookup, type(), receiver, name, args);
+                    return Def.lookupMethod(definition, lookup, type(), receiver, name, args);
                 case LOAD:
-                    return Def.lookupGetter(receiver, name);
+                    return Def.lookupGetter(definition, receiver, name);
                 case STORE:
-                    return Def.lookupSetter(receiver, name);
+                    return Def.lookupSetter(definition, receiver, name);
                 case ARRAY_LOAD:
                     return Def.lookupArrayLoad(receiver);
                 case ARRAY_STORE:
@@ -151,7 +156,9 @@ private MethodHandle lookup(int flavor, String name, Class<?> receiver) throws T
                 case ITERATOR:
                     return Def.lookupIterator(receiver);
                 case REFERENCE:
-                    return Def.lookupReference(lookup, (String) args[0], receiver, name);
+                    return Def.lookupReference(definition, lookup, (String) args[0], receiver, name);
+                case INDEX_NORMALIZE:
+                    return Def.lookupIndexNormalize(receiver);
                 default: throw new AssertionError();
             }
         }
@@ -232,7 +239,7 @@ Object fallback(final Object[] callArgs) throws Throwable {
      */
     static final class MIC extends MutableCallSite {
         private boolean initialized;
-        
+
         private final String name;
         private final int flavor;
         private final int flags;
@@ -414,16 +421,18 @@ static boolean checkBoth(Class<?> left, Class<?> right, Object leftObject, Objec
     /**
      * invokeDynamic bootstrap method
      * <p>
-     * In addition to ordinary parameters, we also take some static parameters:
+     * In addition to ordinary parameters, we also take some parameters defined at the call site:
      * <ul>
      *   <li>{@code initialDepth}: initial call site depth. this is used to exercise megamorphic fallback.
      *   <li>{@code flavor}: type of dynamic call it is (and which part of whitelist to look at).
      *   <li>{@code args}: flavor-specific args.
      * </ul>
+     * And we take the {@link Definition} used to compile the script for whitelist checking.
      * <p>
      * see https://docs.oracle.com/javase/specs/jvms/se7/html/jvms-6.html#jvms-6.5.invokedynamic
      */
-    public static CallSite bootstrap(Lookup lookup, String name, MethodType type, int initialDepth, int flavor, Object... args) {
+    public static CallSite bootstrap(Definition definition, Lookup lookup, String name, MethodType type, int initialDepth, int flavor,
+            Object... args) {
         // validate arguments
         switch(flavor) {
             // "function-call" like things get a polymorphic cache
@@ -442,16 +451,17 @@ public static CallSite bootstrap(Lookup lookup, String name, MethodType type, in
                 if (args.length != numLambdas + 1) {
                     throw new BootstrapMethodError("Illegal number of parameters: expected " + numLambdas + " references");
                 }
-                return new PIC(lookup, name, type, initialDepth, flavor, args);
+                return new PIC(definition, lookup, name, type, initialDepth, flavor, args);
             case LOAD:
             case STORE:
             case ARRAY_LOAD:
             case ARRAY_STORE:
             case ITERATOR:
+            case INDEX_NORMALIZE:
                 if (args.length > 0) {
                     throw new BootstrapMethodError("Illegal static bootstrap parameters for flavor: " + flavor);
                 }
-                return new PIC(lookup, name, type, initialDepth, flavor, args);
+                return new PIC(definition, lookup, name, type, initialDepth, flavor, args);
             case REFERENCE:
                 if (args.length != 1) {
                     throw new BootstrapMethodError("Invalid number of parameters for reference call");
@@ -459,7 +469,7 @@ public static CallSite bootstrap(Lookup lookup, String name, MethodType type, in
                 if (args[0] instanceof String == false) {
                     throw new BootstrapMethodError("Illegal parameter for reference call: " + args[0]);
                 }
-                return new PIC(lookup, name, type, initialDepth, flavor, args);
+                return new PIC(definition, lookup, name, type, initialDepth, flavor, args);
 
             // operators get monomorphic cache, with a generic impl for a fallback
             case UNARY_OPERATOR:
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefMath.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefMath.java
index f89b1429f386b..6628484660699 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefMath.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefMath.java
@@ -379,8 +379,10 @@ private static boolean add(boolean a, boolean b) {
     }
     
     private static Object add(Object left, Object right) {
-        if (left instanceof String || right instanceof String) {
-            return "" + left + right;
+        if (left instanceof String) {
+            return (String) left + right;
+        } else if (right instanceof String) {
+            return left + (String) right;
         } else if (left instanceof Number) {
             if (right instanceof Number) {
                 if (left instanceof Double || right instanceof Double) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
index e0bdfbf17c28d..f8bee4e5cfc66 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
@@ -21,6 +21,7 @@
 
 import org.apache.lucene.util.Constants;
 import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.painless.api.Augmentation;
 
 import java.io.InputStream;
 import java.io.InputStreamReader;
@@ -33,6 +34,7 @@
 import java.time.LocalDate;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -63,32 +65,39 @@ public final class Definition {
                       "java.util.stream.txt",
                       "joda.time.txt"));
 
-    private static final Definition INSTANCE = new Definition();
+    /**
+     * Whitelist that is "built in" to Painless and required by all scripts.
+     */
+    public static final Definition BUILTINS = new Definition();
 
     /** Some native types as constants: */
-    public static final Type VOID_TYPE = getType("void");
-    public static final Type BOOLEAN_TYPE = getType("boolean");
-    public static final Type BOOLEAN_OBJ_TYPE = getType("Boolean");
-    public static final Type BYTE_TYPE = getType("byte");
-    public static final Type BYTE_OBJ_TYPE = getType("Byte");
-    public static final Type SHORT_TYPE = getType("short");
-    public static final Type SHORT_OBJ_TYPE = getType("Short");
-    public static final Type INT_TYPE = getType("int");
-    public static final Type INT_OBJ_TYPE = getType("Integer");
-    public static final Type LONG_TYPE = getType("long");
-    public static final Type LONG_OBJ_TYPE = getType("Long");
-    public static final Type FLOAT_TYPE = getType("float");
-    public static final Type FLOAT_OBJ_TYPE = getType("Float");
-    public static final Type DOUBLE_TYPE = getType("double");
-    public static final Type DOUBLE_OBJ_TYPE = getType("Double");
-    public static final Type CHAR_TYPE = getType("char");
-    public static final Type CHAR_OBJ_TYPE = getType("Character");
-    public static final Type OBJECT_TYPE = getType("Object");
-    public static final Type DEF_TYPE = getType("def");
-    public static final Type STRING_TYPE = getType("String");
-    public static final Type EXCEPTION_TYPE = getType("Exception");
-    public static final Type PATTERN_TYPE = getType("Pattern");
-    public static final Type MATCHER_TYPE = getType("Matcher");
+    public static final Type VOID_TYPE = BUILTINS.getType("void");
+    public static final Type BOOLEAN_TYPE = BUILTINS.getType("boolean");
+    public static final Type BOOLEAN_OBJ_TYPE = BUILTINS.getType("Boolean");
+    public static final Type BYTE_TYPE = BUILTINS.getType("byte");
+    public static final Type BYTE_OBJ_TYPE = BUILTINS.getType("Byte");
+    public static final Type SHORT_TYPE = BUILTINS.getType("short");
+    public static final Type SHORT_OBJ_TYPE = BUILTINS.getType("Short");
+    public static final Type INT_TYPE = BUILTINS.getType("int");
+    public static final Type INT_OBJ_TYPE = BUILTINS.getType("Integer");
+    public static final Type LONG_TYPE = BUILTINS.getType("long");
+    public static final Type LONG_OBJ_TYPE = BUILTINS.getType("Long");
+    public static final Type FLOAT_TYPE = BUILTINS.getType("float");
+    public static final Type FLOAT_OBJ_TYPE = BUILTINS.getType("Float");
+    public static final Type DOUBLE_TYPE = BUILTINS.getType("double");
+    public static final Type DOUBLE_OBJ_TYPE = BUILTINS.getType("Double");
+    public static final Type CHAR_TYPE = BUILTINS.getType("char");
+    public static final Type CHAR_OBJ_TYPE = BUILTINS.getType("Character");
+    public static final Type OBJECT_TYPE = BUILTINS.getType("Object");
+    public static final Type DEF_TYPE = BUILTINS.getType("def");
+    public static final Type NUMBER_TYPE = BUILTINS.getType("Number");
+    public static final Type STRING_TYPE = BUILTINS.getType("String");
+    public static final Type EXCEPTION_TYPE = BUILTINS.getType("Exception");
+    public static final Type PATTERN_TYPE = BUILTINS.getType("Pattern");
+    public static final Type MATCHER_TYPE = BUILTINS.getType("Matcher");
+    public static final Type ITERATOR_TYPE = BUILTINS.getType("Iterator");
+    public static final Type ARRAY_LIST_TYPE = BUILTINS.getType("ArrayList");
+    public static final Type HASH_MAP_TYPE = BUILTINS.getType("HashMap");
 
     public enum Sort {
         VOID(       void.class      , Void.class      , null          , 0 , true  , false , false , false ),
@@ -432,23 +441,23 @@ public static class Cast {
         public final Type from;
         public final Type to;
         public final boolean explicit;
-        public final boolean unboxFrom;
-        public final boolean unboxTo;
-        public final boolean boxFrom;
-        public final boolean boxTo;
+        public final Type unboxFrom;
+        public final Type unboxTo;
+        public final Type boxFrom;
+        public final Type boxTo;
 
         public Cast(final Type from, final Type to, final boolean explicit) {
             this.from = from;
             this.to = to;
             this.explicit = explicit;
-            this.unboxFrom = false;
-            this.unboxTo = false;
-            this.boxFrom = false;
-            this.boxTo = false;
+            this.unboxFrom = null;
+            this.unboxTo = null;
+            this.boxFrom = null;
+            this.boxTo = null;
         }
 
         public Cast(final Type from, final Type to, final boolean explicit,
-                    final boolean unboxFrom, final boolean unboxTo, final boolean boxFrom, final boolean boxTo) {
+                    final Type unboxFrom, final Type unboxTo, final Type boxFrom, final Type boxTo) {
             this.from = from;
             this.to = to;
             this.explicit = explicit;
@@ -461,46 +470,46 @@ public Cast(final Type from, final Type to, final boolean explicit,
     }
 
     public static final class RuntimeClass {
+        private final Struct struct;
         public final Map<MethodKey, Method> methods;
         public final Map<String, MethodHandle> getters;
         public final Map<String, MethodHandle> setters;
 
-        private RuntimeClass(final Map<MethodKey, Method> methods,
+        private RuntimeClass(final Struct struct, final Map<MethodKey, Method> methods,
                              final Map<String, MethodHandle> getters, final Map<String, MethodHandle> setters) {
+            this.struct = struct;
             this.methods = Collections.unmodifiableMap(methods);
             this.getters = Collections.unmodifiableMap(getters);
             this.setters = Collections.unmodifiableMap(setters);
         }
-    }
 
-    /** Returns whether or not a non-array type exists. */
-    public static boolean isSimpleType(final String name) {
-        return INSTANCE.structsMap.containsKey(name);
+        public Struct getStruct() {
+            return struct;
+        }
     }
 
-    /** Returns whether or not a type exists without an exception. */
-    public static boolean isType(final String name) {
-        try {
-            INSTANCE.getTypeInternal(name);
-
-            return true;
-        } catch (IllegalArgumentException exception) {
-            return false;
-        }
+    /** Returns whether or not a non-array type exists. */
+    public boolean isSimpleType(final String name) {
+        return BUILTINS.structsMap.containsKey(name);
     }
 
     /** Gets the type given by its name */
-    public static Type getType(final String name) {
-        return INSTANCE.getTypeInternal(name);
+    public Type getType(final String name) {
+        return BUILTINS.getTypeInternal(name);
     }
 
     /** Creates an array type from the given Struct. */
-    public static Type getType(final Struct struct, final int dimensions) {
-        return INSTANCE.getTypeInternal(struct, dimensions);
+    public Type getType(final Struct struct, final int dimensions) {
+        return BUILTINS.getTypeInternal(struct, dimensions);
     }
 
-    public static RuntimeClass getRuntimeClass(Class<?> clazz) {
-        return INSTANCE.runtimeMap.get(clazz);
+    public RuntimeClass getRuntimeClass(Class<?> clazz) {
+        return BUILTINS.runtimeMap.get(clazz);
+    }
+
+    /** Collection of all simple types. Used by {@code PainlessDocGenerator} to generate an API reference. */
+    static Collection<Type> allSimpleTypes() {
+        return BUILTINS.simpleTypesMap.values();
     }
 
     // INTERNAL IMPLEMENTATION:
@@ -566,11 +575,11 @@ private Map<String,List<String>> addStructs() {
                         }
                         if (line.startsWith("class ")) {
                             String elements[] = line.split("\u0020");
-                            assert elements[2].equals("->");
+                            assert elements[2].equals("->") : "Invalid struct definition [" + String.join(" ", elements) +"]";
                             if (elements.length == 7) {
                                 hierarchy.put(elements[1], Arrays.asList(elements[5].split(",")));
                             } else {
-                                assert elements.length == 5;
+                                assert elements.length == 5 : "Invalid struct definition [" + String.join(" ", elements) + "]";
                             }
                             String className = elements[1];
                             String javaPeer = elements[3];
@@ -612,7 +621,7 @@ private Map<String,List<String>> addStructs() {
                     }
                 }
             } catch (Exception e) {
-                throw new RuntimeException("syntax error in " + file + ", line: " + currentLine, e);
+                throw new RuntimeException("error in " + file + ", line: " + currentLine, e);
             }
         }
         return hierarchy;
@@ -1047,7 +1056,7 @@ private void addRuntimeClass(final Struct struct) {
             }
         }
 
-        runtimeMap.put(struct.clazz, new RuntimeClass(methods, getters, setters));
+        runtimeMap.put(struct.clazz, new RuntimeClass(struct, methods, getters, setters));
     }
 
     /** computes the functional interface method for a class, or returns null */
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Executable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Executable.java
deleted file mode 100644
index c6e38374aa7b6..0000000000000
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Executable.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.painless;
-
-import org.apache.lucene.search.Scorer;
-import org.elasticsearch.search.lookup.LeafDocLookup;
-
-import java.util.BitSet;
-import java.util.Map;
-
-/**
- * The superclass used to build all Painless scripts on top of.
- */
-public abstract class Executable {
-
-    private final String name;
-    private final String source;
-    private final BitSet statements;
-
-    public Executable(String name, String source, BitSet statements) {
-        this.name = name;
-        this.source = source;
-        this.statements = statements;
-    }
-
-    public String getName() {
-        return name;
-    }
-
-    public String getSource() {
-        return source;
-    }
-
-    /** 
-     * Finds the start of the first statement boundary that is
-     * on or before {@code offset}. If one is not found, {@code -1}
-     * is returned.
-     */
-    public int getPreviousStatement(int offset) {
-        return statements.previousSetBit(offset);
-    }
-    
-    /** 
-     * Finds the start of the first statement boundary that is
-     * after {@code offset}. If one is not found, {@code -1}
-     * is returned.
-     */
-    public int getNextStatement(int offset) {
-        return statements.nextSetBit(offset+1);
-    }
-
-    public abstract Object execute(
-        final Map<String, Object> params, final Scorer scorer, final LeafDocLookup doc, final Object value);
-}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FeatureTest.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FeatureTest.java
index 603023e61fe1e..1561aeddf83e3 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FeatureTest.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FeatureTest.java
@@ -1,5 +1,6 @@
 package org.elasticsearch.painless;
 
+import java.util.List;
 import java.util.function.Function;
 
 /*
@@ -25,11 +26,11 @@
 public class FeatureTest {
     private int x;
     private int y;
-    
+
     /** empty ctor */
     public FeatureTest() {
     }
-    
+
     /** ctor with params */
     public FeatureTest(int x, int y) {
         this.x = x;
@@ -60,14 +61,18 @@ public void setY(int y) {
     public static boolean overloadedStatic() {
         return true;
     }
-    
+
     /** static method that returns what you ask it */
     public static boolean overloadedStatic(boolean whatToReturn) {
         return whatToReturn;
     }
-    
+
     /** method taking two functions! */
     public Object twoFunctionsOfX(Function<Object,Object> f, Function<Object,Object> g) {
         return f.apply(g.apply(x));
     }
+
+    public void listInput(List<Object> list) {
+
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
index d5e02e12058bf..6bfe911d974c7 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
@@ -20,121 +20,138 @@
 package org.elasticsearch.painless;
 
 import org.elasticsearch.painless.Definition.Method;
-import org.objectweb.asm.Handle;
-import org.objectweb.asm.Opcodes;
+import org.elasticsearch.painless.Definition.Type;
+import org.elasticsearch.painless.api.Augmentation;
 
-import java.lang.invoke.MethodHandle;
 import java.lang.invoke.MethodType;
 import java.lang.reflect.Modifier;
 
-/** 
- * Reference to a function or lambda. 
+import static org.elasticsearch.painless.WriterConstants.CLASS_NAME;
+import static org.objectweb.asm.Opcodes.H_INVOKEINTERFACE;
+import static org.objectweb.asm.Opcodes.H_INVOKESTATIC;
+import static org.objectweb.asm.Opcodes.H_INVOKEVIRTUAL;
+import static org.objectweb.asm.Opcodes.H_NEWINVOKESPECIAL;
+
+/**
+ * Reference to a function or lambda.
  * <p>
- * Once you have created one of these, you have "everything you need" to call LambdaMetaFactory
- * either statically from bytecode with invokedynamic, or at runtime from Java.  
+ * Once you have created one of these, you have "everything you need" to call {@link LambdaBootstrap}
+ * either statically from bytecode with invokedynamic, or at runtime from Java.
  */
 public class FunctionRef {
-    /** Function Object's method name */
-    public final String invokedName;
-    /** CallSite signature */
-    public final MethodType invokedType;
-    /** Implementation method */
-    public final MethodHandle implMethod;
-    /** Function Object's method signature */
-    public final MethodType samMethodType;
-    /** When bridging is required, request this bridge interface */
+
+    /** functional interface method name */
+    public final String interfaceMethodName;
+    /** factory (CallSite) method signature */
+    public final MethodType factoryMethodType;
+    /** functional interface method signature */
     public final MethodType interfaceMethodType;
-    
-    /** ASM "Handle" to the method, for the constant pool */
-    public final Handle implMethodASM;
-    
+    /** class of the delegate method to be called */
+    public final String delegateClassName;
+    /** the invocation type of the delegate method */
+    public final int delegateInvokeType;
+    /** the name of the delegate method */
+    public final String delegateMethodName;
+    /** delegate method signature */
+    public final MethodType delegateMethodType;
+
+    /** interface method */
+    public final Method interfaceMethod;
+    /** delegate method */
+    public final Method delegateMethod;
+
+    /** factory method type descriptor */
+    public final String factoryDescriptor;
+    /** functional interface method as type */
+    public final org.objectweb.asm.Type interfaceType;
+    /** delegate method type method as type */
+    public final org.objectweb.asm.Type delegateType;
+
     /**
      * Creates a new FunctionRef, which will resolve {@code type::call} from the whitelist.
-     * @param expected interface type to implement.
+     * @param definition the whitelist against which this script is being compiled
+     * @param expected functional interface type to implement.
      * @param type the left hand side of a method reference expression
      * @param call the right hand side of a method reference expression
      * @param numCaptures number of captured arguments
-     */    
-    public FunctionRef(Definition.Type expected, String type, String call, int numCaptures) {
-        this(expected, expected.struct.getFunctionalMethod(), lookup(expected, type, call, numCaptures > 0), numCaptures);
+     */
+    public FunctionRef(Definition definition, Type expected, String type, String call, int numCaptures) {
+        this(expected, expected.struct.getFunctionalMethod(), lookup(definition, expected, type, call, numCaptures > 0), numCaptures);
     }
 
     /**
      * Creates a new FunctionRef (already resolved)
-     * @param expected interface type to implement
-     * @param method functional interface method
-     * @param impl implementation method
+     * @param expected functional interface type to implement
+     * @param interfaceMethod functional interface method
+     * @param delegateMethod implementation method
      * @param numCaptures number of captured arguments
-     */   
-    public FunctionRef(Definition.Type expected, Definition.Method method, Definition.Method impl, int numCaptures) {
-        // e.g. compareTo
-        invokedName = method.name;
-        // e.g. (Object)Comparator
-        MethodType implType = impl.getMethodType();
-        // only include captured parameters as arguments
-        invokedType = MethodType.methodType(expected.clazz, 
-                implType.dropParameterTypes(numCaptures, implType.parameterCount()));
-        // e.g. (Object,Object)int
-        interfaceMethodType = method.getMethodType().dropParameterTypes(0, 1);
-
-        final int tag;
-        if ("<init>".equals(impl.name)) {
-            tag = Opcodes.H_NEWINVOKESPECIAL;
-        } else if (Modifier.isStatic(impl.modifiers)) {
-            tag = Opcodes.H_INVOKESTATIC;
-        } else if (impl.owner.clazz.isInterface()) {
-            tag = Opcodes.H_INVOKEINTERFACE;
+     */
+    public FunctionRef(Type expected, Method interfaceMethod, Method delegateMethod, int numCaptures) {
+        MethodType delegateMethodType = delegateMethod.getMethodType();
+
+        interfaceMethodName = interfaceMethod.name;
+        factoryMethodType = MethodType.methodType(expected.clazz,
+                delegateMethodType.dropParameterTypes(numCaptures, delegateMethodType.parameterCount()));
+        interfaceMethodType = interfaceMethod.getMethodType().dropParameterTypes(0, 1);
+
+        // the Painless$Script class can be inferred if owner is null
+        if (delegateMethod.owner == null) {
+            delegateClassName = CLASS_NAME;
+        } else if (delegateMethod.augmentation) {
+            delegateClassName = Augmentation.class.getName();
         } else {
-            tag = Opcodes.H_INVOKEVIRTUAL;
+            delegateClassName = delegateMethod.owner.clazz.getName();
         }
-        final String owner;
-        final boolean ownerIsInterface;
-        if (impl.owner == null) {
-            // owner == null: script class itself
-            ownerIsInterface = false;
-            owner = WriterConstants.CLASS_TYPE.getInternalName();
-        } else if (impl.augmentation) {
-            ownerIsInterface = false;
-            owner = WriterConstants.AUGMENTATION_TYPE.getInternalName();
+
+        if ("<init>".equals(delegateMethod.name)) {
+            delegateInvokeType = H_NEWINVOKESPECIAL;
+        } else if (Modifier.isStatic(delegateMethod.modifiers)) {
+            delegateInvokeType = H_INVOKESTATIC;
+        } else if (delegateMethod.owner.clazz.isInterface()) {
+            delegateInvokeType = H_INVOKEINTERFACE;
         } else {
-            ownerIsInterface = impl.owner.clazz.isInterface();
-            owner = impl.owner.type.getInternalName();
+            delegateInvokeType = H_INVOKEVIRTUAL;
         }
-        implMethodASM = new Handle(tag, owner, impl.name, impl.method.getDescriptor(), ownerIsInterface);
-        implMethod = impl.handle;
-        
-        // remove any prepended captured arguments for the 'natural' signature.
-        samMethodType = adapt(interfaceMethodType, impl.getMethodType().dropParameterTypes(0, numCaptures));
+
+        delegateMethodName = delegateMethod.name;
+        this.delegateMethodType = delegateMethodType.dropParameterTypes(0, numCaptures);
+
+        this.interfaceMethod = interfaceMethod;
+        this.delegateMethod = delegateMethod;
+
+        factoryDescriptor = factoryMethodType.toMethodDescriptorString();
+        interfaceType = org.objectweb.asm.Type.getMethodType(interfaceMethodType.toMethodDescriptorString());
+        delegateType = org.objectweb.asm.Type.getMethodType(this.delegateMethodType.toMethodDescriptorString());
     }
 
     /**
-     * Creates a new FunctionRef (low level). 
-     * <p>
-     * This will <b>not</b> set implMethodASM. It is for runtime use only.
+     * Creates a new FunctionRef (low level).
+     * It is for runtime use only.
      */
-    public FunctionRef(Definition.Type expected, Definition.Method method, MethodHandle impl, int numCaptures) {
-        // e.g. compareTo
-        invokedName = method.name;
-        // e.g. (Object)Comparator
-        MethodType implType = impl.type();
-        // only include captured parameters as arguments
-        invokedType = MethodType.methodType(expected.clazz, 
-                implType.dropParameterTypes(numCaptures, implType.parameterCount()));
-        // e.g. (Object,Object)int
-        interfaceMethodType = method.getMethodType().dropParameterTypes(0, 1);
-
-        implMethod = impl;
-        
-        implMethodASM = null;
-        
-        // remove any prepended captured arguments for the 'natural' signature.
-        samMethodType = adapt(interfaceMethodType, impl.type().dropParameterTypes(0, numCaptures));
+    public FunctionRef(Type expected, Method interfaceMethod, String delegateMethodName, MethodType delegateMethodType, int numCaptures) {
+        interfaceMethodName = interfaceMethod.name;
+        factoryMethodType = MethodType.methodType(expected.clazz,
+            delegateMethodType.dropParameterTypes(numCaptures, delegateMethodType.parameterCount()));
+        interfaceMethodType = interfaceMethod.getMethodType().dropParameterTypes(0, 1);
+
+        delegateClassName = CLASS_NAME;
+        delegateInvokeType = H_INVOKESTATIC;
+        this.delegateMethodName = delegateMethodName;
+        this.delegateMethodType = delegateMethodType.dropParameterTypes(0, numCaptures);
+
+        this.interfaceMethod = null;
+        delegateMethod = null;
+
+        factoryDescriptor = null;
+        interfaceType = null;
+        delegateType = null;
     }
 
-    /** 
+    /**
      * Looks up {@code type::call} from the whitelist, and returns a matching method.
      */
-    private static Definition.Method lookup(Definition.Type expected, String type, String call, boolean receiverCaptured) {
+    private static Definition.Method lookup(Definition definition, Definition.Type expected,
+                                            String type, String call, boolean receiverCaptured) {
         // check its really a functional interface
         // for e.g. Comparable
         Method method = expected.struct.getFunctionalMethod();
@@ -144,7 +161,7 @@ private static Definition.Method lookup(Definition.Type expected, String type, S
         }
 
         // lookup requested method
-        Definition.Struct struct = Definition.getType(type).struct;
+        Definition.Struct struct = definition.getType(type).struct;
         final Definition.Method impl;
         // ctor ref
         if ("new".equals(call)) {
@@ -173,27 +190,4 @@ private static Definition.Method lookup(Definition.Type expected, String type, S
         }
         return impl;
     }
-
-    /** Returns true if you should ask LambdaMetaFactory to construct a bridge for the interface signature */
-    public boolean needsBridges() {
-        // currently if the interface differs, we ask for a bridge, but maybe we should do smarter checking?
-        // either way, stuff will fail if its wrong :)
-        return interfaceMethodType.equals(samMethodType) == false;
-    }
-    
-    /** 
-     * If the interface expects a primitive type to be returned, we can't return Object,
-     * But we can set SAM to the wrapper version, and a cast will take place 
-     */
-    private MethodType adapt(MethodType expected, MethodType actual) {
-        // add some checks, now that we've set everything up, to deliver exceptions as early as possible.
-        if (expected.parameterCount() != actual.parameterCount()) {
-            throw new IllegalArgumentException("Incorrect number of parameters for [" + invokedName + 
-                                               "] in [" + invokedType.returnType() + "]");
-        }
-        if (expected.returnType().isPrimitive() && actual.returnType() == Object.class) {
-            actual = actual.changeReturnType(MethodType.methodType(expected.returnType()).wrap().returnType());
-        }
-        return actual;
-    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/GenericElasticsearchScript.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/GenericElasticsearchScript.java
new file mode 100644
index 0000000000000..69f5462c22813
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/GenericElasticsearchScript.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+import org.elasticsearch.index.fielddata.ScriptDocValues;
+
+import java.util.Map;
+
+/**
+ * Generic script interface that Painless implements for all Elasticsearch scripts.
+ */
+public interface GenericElasticsearchScript {
+    String[] ARGUMENTS = new String[] {"params", "_score", "doc", "_value", "ctx"};
+    Object execute(Map<String, Object> params, double _score, Map<String, ScriptDocValues<?>> doc, Object _value, Map<?, ?> ctx);
+
+    boolean uses$_score();
+    boolean uses$ctx();
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/LambdaBootstrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/LambdaBootstrap.java
new file mode 100644
index 0000000000000..3a0b322d5eaf7
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/LambdaBootstrap.java
@@ -0,0 +1,515 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+import org.objectweb.asm.ClassWriter;
+import org.objectweb.asm.FieldVisitor;
+import org.objectweb.asm.Handle;
+import org.objectweb.asm.Type;
+import org.objectweb.asm.commons.GeneratorAdapter;
+import org.objectweb.asm.commons.Method;
+
+import java.lang.invoke.CallSite;
+import java.lang.invoke.ConstantCallSite;
+import java.lang.invoke.LambdaConversionException;
+import java.lang.invoke.MethodHandle;
+import java.lang.invoke.MethodHandles;
+import java.lang.invoke.MethodType;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+
+import static java.lang.invoke.MethodHandles.Lookup;
+import static org.elasticsearch.painless.Compiler.Loader;
+import static org.elasticsearch.painless.WriterConstants.CLASS_VERSION;
+import static org.elasticsearch.painless.WriterConstants.CTOR_METHOD_NAME;
+import static org.elasticsearch.painless.WriterConstants.DELEGATE_BOOTSTRAP_HANDLE;
+import static org.objectweb.asm.Opcodes.ACC_FINAL;
+import static org.objectweb.asm.Opcodes.ACC_PRIVATE;
+import static org.objectweb.asm.Opcodes.ACC_PUBLIC;
+import static org.objectweb.asm.Opcodes.ACC_STATIC;
+import static org.objectweb.asm.Opcodes.ACC_SUPER;
+import static org.objectweb.asm.Opcodes.ACC_SYNTHETIC;
+import static org.objectweb.asm.Opcodes.H_INVOKEINTERFACE;
+import static org.objectweb.asm.Opcodes.H_INVOKESTATIC;
+import static org.objectweb.asm.Opcodes.H_INVOKEVIRTUAL;
+import static org.objectweb.asm.Opcodes.H_NEWINVOKESPECIAL;
+
+/**
+ * LambdaBootstrap is used to generate all the code necessary to execute
+ * lambda functions and method references within Painless.  The code generation
+ * used here is based upon the following article:
+ * http://cr.openjdk.java.net/~briangoetz/lambda/lambda-translation.html
+ * However, it is a simplified version as Painless has no concept of generics
+ * or serialization.  LambdaBootstrap is being used as a replacement for
+ * {@link java.lang.invoke.LambdaMetafactory} since the Painless casting model
+ * cannot be fully supported through this class.
+ *
+ * For each lambda function/method reference used within a Painless script
+ * a class will be generated at link-time using the
+ * {@link LambdaBootstrap#lambdaBootstrap} method that contains the following:
+ * 1. member fields for any captured variables
+ * 2. a constructor that will take in captured variables and assign them to
+ * their respective member fields
+ * 3. a static ctor delegation method, if the lambda function is a ctor.
+ * 4. a method that will load the member fields representing captured variables
+ * and take in any other necessary values based on the arguments passed into the
+ * lambda function/reference method; it will then make a delegated call to the
+ * actual lambda function/reference method
+ *
+ * Take for example the following Painless script:
+ *
+ * {@code
+ * List list1 = new ArrayList(); "
+ * list1.add(2); "
+ * List list2 = new ArrayList(); "
+ * list1.forEach(x -> list2.add(x));"
+ * return list[0]"
+ * }
+ *
+ * The script contains a lambda function with a captured variable.
+ * The following Lambda class would be generated:
+ *
+ * {@code
+ *     public static final class $$Lambda0 implements Consumer {
+ *         private List arg$0;
+ *
+ *         private $$Lambda0(List arg$0) {
+ *             this.arg$0 = arg$0;
+ *         }
+ *         
+ *         public static Consumer create$lambda(List arg$0) {
+ *             return new $$Lambda0(arg$0);
+ *         }
+ *
+ *         public void accept(Object val$0) {
+ *             Painless$Script.lambda$0(this.arg$0, val$0);
+ *         }
+ *     }
+ *
+ *     public class Painless$Script implements ... {
+ *         ...
+ *         public static lambda$0(List list2, Object x) {
+ *             list2.add(x);
+ *         }
+ *         ...
+ *     }
+ * }
+ *
+ * Also the accept method actually uses an invokedynamic
+ * instruction to call the lambda$0 method so that
+ * {@link MethodHandle#asType} can be used to do the necessary
+ * conversions between argument types without having to hard
+ * code them. For method references to a constructor, a static
+ * wrapper method is created, that creates a class instance and
+ * calls the constructor. This method is used by the
+ * invokedynamic call to initialize the instance.
+ *
+ * When the {@link CallSite} is linked the linked method depends
+ * on whether or not there are captures.  If there are no captures
+ * the same instance of the generated lambda class will be
+ * returned each time by the factory method as there are no
+ * changing values other than the arguments, the lambda is a singleton.
+ * If there are captures, a new instance of the generated lambda class
+ * will be returned each time with the captures passed into the
+ * factory method to be stored in the member fields.
+ * Instead of calling the ctor, a static factory method is created
+ * in the lambda class, because a method handle to the ctor directly
+ * is (currently) preventing Hotspot optimizer from correctly doing
+ * escape analysis. Escape analysis is important to optimize the
+ * code in a way, that a new instance is not created on each lambda
+ * invocation with captures, stressing garbage collector (thanks
+ * to Rémi Forax for the explanation about this on Jaxcon 2017!).
+ */
+public final class LambdaBootstrap {
+
+    /**
+     * Metadata for a captured variable used during code generation.
+     */
+    private static final class Capture {
+        private final String name;
+        private final Type type;
+        private final String desc;
+
+        /**
+         * Converts incoming parameters into the name, type, and
+         * descriptor for the captured argument.
+         * @param count The captured argument count
+         * @param type The class type of the captured argument
+         */
+        private Capture(int count, Class<?> type) {
+            this.name = "arg$" + count;
+            this.type = Type.getType(type);
+            this.desc = this.type.getDescriptor();
+        }
+    }
+
+    /**
+     * This method name is used to generate a static wrapper method to handle delegation of ctors.
+     */
+    private static final String DELEGATED_CTOR_WRAPPER_NAME = "delegate$ctor";
+
+    /**
+     * This method name is used to generate the static factory for capturing lambdas.
+     */
+    private static final String LAMBDA_FACTORY_METHOD_NAME = "create$lambda";
+
+    /**
+     * Generates a lambda class for a lambda function/method reference
+     * within a Painless script.  Variables with the prefix interface are considered
+     * to represent values for code generated for the lambda class. Variables with
+     * the prefix delegate are considered to represent values for code generated
+     * within the Painless script.  The interface method delegates (calls) to the
+     * delegate method.
+     * @param lookup Standard {@link MethodHandles#lookup}
+     * @param interfaceMethodName Name of functional interface method that is called
+     * @param factoryMethodType The type of method to be linked to this CallSite; note that
+     *                          captured types are based on the parameters for this method
+     * @param interfaceMethodType The type of method representing the functional interface method
+     * @param delegateClassName The name of the class to delegate method call to
+     * @param delegateInvokeType The type of method call to be made
+     *                           (static, virtual, interface, or constructor)
+     * @param delegateMethodName The name of the method to be called in the Painless script class
+     * @param delegateMethodType The type of method call in the Painless script class without
+     *                           the captured types
+     * @return A {@link CallSite} linked to a factory method for creating a lambda class
+     * that implements the expected functional interface
+     * @throws LambdaConversionException Thrown when an illegal type conversion occurs at link time
+     */
+    public static CallSite lambdaBootstrap(
+            Lookup lookup,
+            String interfaceMethodName,
+            MethodType factoryMethodType,
+            MethodType interfaceMethodType,
+            String delegateClassName,
+            int delegateInvokeType,
+            String delegateMethodName,
+            MethodType delegateMethodType)
+            throws LambdaConversionException {
+        Loader loader = (Loader)lookup.lookupClass().getClassLoader();
+        String lambdaClassName = Type.getInternalName(lookup.lookupClass()) + "$$Lambda" + loader.newLambdaIdentifier();
+        Type lambdaClassType = Type.getObjectType(lambdaClassName);
+        Type delegateClassType = Type.getObjectType(delegateClassName.replace('.', '/'));
+
+        validateTypes(interfaceMethodType, delegateMethodType);
+
+        ClassWriter cw = beginLambdaClass(lambdaClassName, factoryMethodType.returnType());
+        Capture[] captures = generateCaptureFields(cw, factoryMethodType);
+        generateLambdaConstructor(cw, lambdaClassType, factoryMethodType, captures);
+        
+        // Handles the special case where a method reference refers to a ctor (we need a static wrapper method):
+        if (delegateInvokeType == H_NEWINVOKESPECIAL) {
+            assert CTOR_METHOD_NAME.equals(delegateMethodName);
+            generateStaticCtorDelegator(cw, ACC_PRIVATE, DELEGATED_CTOR_WRAPPER_NAME, delegateClassType, delegateMethodType);
+            // replace the delegate with our static wrapper:
+            delegateMethodName = DELEGATED_CTOR_WRAPPER_NAME;
+            delegateClassType = lambdaClassType;
+            delegateInvokeType = H_INVOKESTATIC;
+        }
+
+        generateInterfaceMethod(cw, factoryMethodType, lambdaClassType, interfaceMethodName,
+            interfaceMethodType, delegateClassType, delegateInvokeType,
+            delegateMethodName, delegateMethodType, captures);
+        
+        endLambdaClass(cw);
+
+        Class<?> lambdaClass = createLambdaClass(loader, cw, lambdaClassType);
+        if (captures.length > 0) {
+            return createCaptureCallSite(lookup, factoryMethodType, lambdaClass);
+        } else {
+            return createNoCaptureCallSite(factoryMethodType, lambdaClass);
+        }
+    }
+
+    /**
+     * Validates some conversions at link time.  Currently, only ensures that the lambda method
+     * with a return value cannot delegate to a delegate method with no return type.
+     */
+    private static void validateTypes(MethodType interfaceMethodType, MethodType delegateMethodType)
+            throws LambdaConversionException {
+
+        if (interfaceMethodType.returnType() != void.class &&
+            delegateMethodType.returnType() == void.class) {
+            throw new LambdaConversionException("lambda expects return type ["
+                + interfaceMethodType.returnType() + "], but found return type [void]");
+        }
+    }
+
+    /**
+     * Creates the {@link ClassWriter} to be used for the lambda class generation.
+     */
+    private static ClassWriter beginLambdaClass(String lambdaClassName, Class<?> lambdaInterface) {
+        String baseClass = Type.getInternalName(Object.class);
+        int modifiers = ACC_PUBLIC | ACC_SUPER | ACC_FINAL | ACC_SYNTHETIC;
+
+        ClassWriter cw = new ClassWriter(ClassWriter.COMPUTE_MAXS);
+        cw.visit(CLASS_VERSION,
+            modifiers, lambdaClassName, null, baseClass, new String[] { Type.getInternalName(lambdaInterface) });
+
+        return cw;
+    }
+
+    /**
+     * Generates member fields for captured variables
+     * based on the parameters for the factory method.
+     * @return An array of captured variable metadata
+     * for generating method arguments later on
+     */
+    private static Capture[] generateCaptureFields(ClassWriter cw, MethodType factoryMethodType) {
+        int captureTotal = factoryMethodType.parameterCount();
+        Capture[] captures = new Capture[captureTotal];
+
+        for (int captureCount = 0; captureCount < captureTotal; ++captureCount) {
+            captures[captureCount] =
+                new Capture(captureCount, factoryMethodType.parameterType(captureCount));
+            int modifiers = ACC_PRIVATE | ACC_FINAL;
+
+            FieldVisitor fv = cw.visitField(
+                modifiers, captures[captureCount].name, captures[captureCount].desc, null, null);
+            fv.visitEnd();
+        }
+
+        return captures;
+    }
+
+    /**
+     * Generates a constructor that will take in captured
+     * arguments if any and store them in their respective
+     * member fields.
+     */
+    private static void generateLambdaConstructor(
+            ClassWriter cw,
+            Type lambdaClassType,
+            MethodType factoryMethodType,
+            Capture[] captures) {
+
+        String conDesc = factoryMethodType.changeReturnType(void.class).toMethodDescriptorString();
+        Method conMeth = new Method(CTOR_METHOD_NAME, conDesc);
+        Type baseConType = Type.getType(Object.class);
+        Method baseConMeth = new Method(CTOR_METHOD_NAME,
+            MethodType.methodType(void.class).toMethodDescriptorString());
+        int modifiers = (captures.length > 0) ? ACC_PRIVATE : ACC_PUBLIC;
+
+        GeneratorAdapter constructor = new GeneratorAdapter(modifiers, conMeth,
+            cw.visitMethod(modifiers, CTOR_METHOD_NAME, conDesc, null, null));
+        constructor.visitCode();
+        constructor.loadThis();
+        constructor.invokeConstructor(baseConType, baseConMeth);
+
+        for (int captureCount = 0; captureCount < captures.length; ++captureCount) {
+            constructor.loadThis();
+            constructor.loadArg(captureCount);
+            constructor.putField(
+                lambdaClassType, captures[captureCount].name, captures[captureCount].type);
+        }
+
+        constructor.returnValue();
+        constructor.endMethod();
+        
+        // Add a factory method, if lambda takes captures.
+        // @uschindler says: I talked with Rémi Forax about this. Technically, a plain ctor
+        // and a MethodHandle to the ctor would be enough - BUT: Hotspot is unable to
+        // do escape analysis through a MethodHandles.findConstructor generated handle.
+        // Because of this we create a factory method. With this factory method, the
+        // escape analysis can figure out that everything is final and we don't need
+        // an instance, so it can omit object creation on heap!
+        if (captures.length > 0) {
+            generateStaticCtorDelegator(cw, ACC_PUBLIC, LAMBDA_FACTORY_METHOD_NAME, lambdaClassType, factoryMethodType);
+        }
+    }
+
+    /**
+     * Generates a factory method to delegate to constructors.
+     */
+    private static void generateStaticCtorDelegator(ClassWriter cw, int access, String delegatorMethodName, 
+            Type delegateClassType, MethodType delegateMethodType) {
+        Method wrapperMethod = new Method(delegatorMethodName, delegateMethodType.toMethodDescriptorString());
+        Method constructorMethod = 
+            new Method(CTOR_METHOD_NAME, delegateMethodType.changeReturnType(void.class).toMethodDescriptorString());
+        int modifiers = access | ACC_STATIC;
+
+        GeneratorAdapter factory = new GeneratorAdapter(modifiers, wrapperMethod,
+            cw.visitMethod(modifiers, delegatorMethodName, delegateMethodType.toMethodDescriptorString(), null, null));
+        factory.visitCode();
+        factory.newInstance(delegateClassType);
+        factory.dup();
+        factory.loadArgs();
+        factory.invokeConstructor(delegateClassType, constructorMethod);
+        factory.returnValue();
+        factory.endMethod();
+    }
+
+    /**
+     * Generates the interface method that will delegate (call) to the delegate method
+     * with {@code INVOKEDYNAMIC} using the {@link #delegateBootstrap} type converter.
+     */
+    private static void generateInterfaceMethod(
+            ClassWriter cw,
+            MethodType factoryMethodType,
+            Type lambdaClassType,
+            String interfaceMethodName,
+            MethodType interfaceMethodType,
+            Type delegateClassType,
+            int delegateInvokeType,
+            String delegateMethodName,
+            MethodType delegateMethodType,
+            Capture[] captures)
+            throws LambdaConversionException {
+
+        String lamDesc = interfaceMethodType.toMethodDescriptorString();
+        Method lamMeth = new Method(lambdaClassType.getInternalName(), lamDesc);
+        int modifiers = ACC_PUBLIC;
+
+        GeneratorAdapter iface = new GeneratorAdapter(modifiers, lamMeth,
+            cw.visitMethod(modifiers, interfaceMethodName, lamDesc, null, null));
+        iface.visitCode();
+        
+        // Loads any captured variables onto the stack.
+        for (int captureCount = 0; captureCount < captures.length; ++captureCount) {
+            iface.loadThis();
+            iface.getField(
+                lambdaClassType, captures[captureCount].name, captures[captureCount].type);
+        }
+
+        // Loads any passed in arguments onto the stack.
+        iface.loadArgs();
+
+        // Handles the case for a lambda function or a static reference method.
+        // interfaceMethodType and delegateMethodType both have the captured types
+        // inserted into their type signatures.  This later allows the delegate
+        // method to be invoked dynamically and have the interface method types
+        // appropriately converted to the delegate method types.
+        // Example: Integer::parseInt
+        // Example: something.each(x -> x + 1)
+        if (delegateInvokeType == H_INVOKESTATIC) {
+            interfaceMethodType =
+                interfaceMethodType.insertParameterTypes(0, factoryMethodType.parameterArray());
+            delegateMethodType =
+                delegateMethodType.insertParameterTypes(0, factoryMethodType.parameterArray());
+        } else if (delegateInvokeType == H_INVOKEVIRTUAL ||
+            delegateInvokeType == H_INVOKEINTERFACE) {
+            // Handles the case for a virtual or interface reference method with no captures.
+            // delegateMethodType drops the 'this' parameter because it will be re-inserted
+            // when the method handle for the dynamically invoked delegate method is created.
+            // Example: Object::toString
+            if (captures.length == 0) {
+                Class<?> clazz = delegateMethodType.parameterType(0);
+                delegateClassType = Type.getType(clazz);
+                delegateMethodType = delegateMethodType.dropParameterTypes(0, 1);
+            // Handles the case for a virtual or interface reference method with 'this'
+            // captured. interfaceMethodType inserts the 'this' type into its
+            // method signature. This later allows the delegate
+            // method to be invoked dynamically and have the interface method types
+            // appropriately converted to the delegate method types.
+            // Example: something::toString
+            } else if (captures.length == 1) {
+                Class<?> clazz = factoryMethodType.parameterType(0);
+                delegateClassType = Type.getType(clazz);
+                interfaceMethodType = interfaceMethodType.insertParameterTypes(0, clazz);
+            } else {
+                throw new LambdaConversionException(
+                    "unexpected number of captures [ " + captures.length + "]");
+            }
+        } else {
+            throw new IllegalStateException(
+                "unexpected invocation type [" + delegateInvokeType + "]");
+        }
+
+        Handle delegateHandle =
+            new Handle(delegateInvokeType, delegateClassType.getInternalName(),
+                delegateMethodName, delegateMethodType.toMethodDescriptorString(),
+                delegateInvokeType == H_INVOKEINTERFACE);
+        iface.invokeDynamic(delegateMethodName, Type.getMethodType(interfaceMethodType
+                .toMethodDescriptorString()).getDescriptor(), DELEGATE_BOOTSTRAP_HANDLE,
+                delegateHandle);
+
+        iface.returnValue();
+        iface.endMethod();
+    }
+
+    /**
+     * Closes the {@link ClassWriter}.
+     */
+    private static void endLambdaClass(ClassWriter cw) {
+        cw.visitEnd();
+    }
+
+    /**
+     * Defines the {@link Class} for the lambda class using the same {@link Loader}
+     * that originally defined the class for the Painless script.
+     */
+    private static Class<?> createLambdaClass(
+            Loader loader,
+            ClassWriter cw,
+            Type lambdaClassType) {
+
+        byte[] classBytes = cw.toByteArray();
+        // DEBUG:
+        // new ClassReader(classBytes).accept(new TraceClassVisitor(new PrintWriter(System.out)), ClassReader.SKIP_DEBUG);
+        return AccessController.doPrivileged((PrivilegedAction<Class<?>>)() ->
+            loader.defineLambda(lambdaClassType.getClassName(), classBytes));
+    }
+
+    /**
+     * Creates an {@link ConstantCallSite} that will return the same instance
+     * of the generated lambda class every time this linked factory method is called.
+     */
+    private static CallSite createNoCaptureCallSite(
+            MethodType factoryMethodType,
+            Class<?> lambdaClass) {
+        
+        try {
+            return new ConstantCallSite(MethodHandles.constant(
+                factoryMethodType.returnType(), lambdaClass.getConstructor().newInstance()));
+        } catch (ReflectiveOperationException exception) {
+            throw new IllegalStateException("unable to instantiate lambda class", exception);
+        }
+    }
+
+    /**
+     * Creates an {@link ConstantCallSite}
+     */
+    private static CallSite createCaptureCallSite(
+            Lookup lookup,
+            MethodType factoryMethodType,
+            Class<?> lambdaClass) {
+
+        try {
+            return new ConstantCallSite(
+                lookup.findStatic(lambdaClass, LAMBDA_FACTORY_METHOD_NAME, factoryMethodType));
+        } catch (ReflectiveOperationException exception) {
+            throw new IllegalStateException("unable to create lambda class", exception);
+        }
+    }
+
+    /**
+     * Links the delegate method to the returned {@link CallSite}.  The linked
+     * delegate method will use converted types from the interface method.  Using
+     * invokedynamic to make the delegate method call allows
+     * {@link MethodHandle#asType} to be used to do the type conversion instead
+     * of either a lot more code or requiring many {@link Definition.Type}s to be looked
+     * up at link-time.
+     */
+    public static CallSite delegateBootstrap(Lookup lookup,
+                                             String delegateMethodName,
+                                             MethodType interfaceMethodType,
+                                             MethodHandle delegateMethodHandle) {
+        return new ConstantCallSite(delegateMethodHandle.asType(interfaceMethodType));
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java
index b02ea085904ac..6eff9e3228bb4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.Definition.MethodKey;
 import org.elasticsearch.painless.Definition.Type;
+import org.elasticsearch.painless.ScriptInterface.MethodArgument;
 
 import java.util.Arrays;
 import java.util.Collection;
@@ -36,42 +37,30 @@
  * Tracks user defined methods and variables across compilation phases.
  */
 public final class Locals {
-    
-    /** Reserved word: params map parameter */
-    public static final String PARAMS = "params";
-    /** Reserved word: Lucene scorer parameter */
-    public static final String SCORER = "#scorer";
-    /** Reserved word: _value variable for aggregations */
-    public static final String VALUE  = "_value";
-    /** Reserved word: _score variable for search scripts */
-    public static final String SCORE  = "_score";
-    /** Reserved word: ctx map for executable scripts */
-    public static final String CTX    = "ctx";
+
     /** Reserved word: loop counter */
     public static final String LOOP   = "#loop";
     /** Reserved word: unused */
     public static final String THIS   = "#this";
-    /** Reserved word: unused */
-    public static final String DOC    = "doc";
-    
-    /** Map of always reserved keywords */
+
+    /** Set of reserved keywords. */
     public static final Set<String> KEYWORDS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(
-            THIS,PARAMS,SCORER,DOC,VALUE,SCORE,CTX,LOOP
+        THIS, LOOP
     )));
-    
+
     /** Creates a new local variable scope (e.g. loop) inside the current scope */
     public static Locals newLocalScope(Locals currentScope) {
         return new Locals(currentScope);
     }
-    
-    /** 
+
+    /**
      * Creates a new lambda scope inside the current scope
      * <p>
      * This is just like {@link #newFunctionScope}, except the captured parameters are made read-only.
      */
-    public static Locals newLambdaScope(Locals programScope, Type returnType, List<Parameter> parameters, 
+    public static Locals newLambdaScope(Locals programScope, Type returnType, List<Parameter> parameters,
                                         int captureCount, int maxLoopCounter) {
-        Locals locals = new Locals(programScope, returnType);
+        Locals locals = new Locals(programScope, programScope.definition, returnType, KEYWORDS);
         for (int i = 0; i < parameters.size(); i++) {
             Parameter parameter = parameters.get(i);
             // TODO: allow non-captures to be r/w:
@@ -87,10 +76,10 @@ public static Locals newLambdaScope(Locals programScope, Type returnType, List<P
         }
         return locals;
     }
-    
+
     /** Creates a new function scope inside the current scope */
     public static Locals newFunctionScope(Locals programScope, Type returnType, List<Parameter> parameters, int maxLoopCounter) {
-        Locals locals = new Locals(programScope, returnType);
+        Locals locals = new Locals(programScope, programScope.definition, returnType, KEYWORDS);
         for (Parameter parameter : parameters) {
             locals.addVariable(parameter.location, parameter.type, parameter.name, false);
         }
@@ -100,35 +89,17 @@ public static Locals newFunctionScope(Locals programScope, Type returnType, List
         }
         return locals;
     }
-    
-    /** Creates a new main method scope */
-    public static Locals newMainMethodScope(Locals programScope, boolean usesScore, boolean usesCtx, int maxLoopCounter) {
-        Locals locals = new Locals(programScope, Definition.OBJECT_TYPE);
-        // This reference.  Internal use only.
-        locals.defineVariable(null, Definition.getType("Object"), THIS, true);
-
-        // Input map of variables passed to the script.
-        locals.defineVariable(null, Definition.getType("Map"), PARAMS, true);
 
-        // Scorer parameter passed to the script.  Internal use only.
-        locals.defineVariable(null, Definition.DEF_TYPE, SCORER, true);
-
-        // Doc parameter passed to the script. TODO: Currently working as a Map, we can do better?
-        locals.defineVariable(null, Definition.getType("Map"), DOC, true);
-
-        // Aggregation _value parameter passed to the script.
-        locals.defineVariable(null, Definition.DEF_TYPE, VALUE, true);
-
-        // Shortcut variables.
-
-        // Document's score as a read-only double.
-        if (usesScore) {
-            locals.defineVariable(null, Definition.DOUBLE_TYPE, SCORE, true);
-        }
+    /** Creates a new main method scope */
+    public static Locals newMainMethodScope(ScriptInterface scriptInterface, Locals programScope, int maxLoopCounter) {
+        Locals locals = new Locals(programScope, programScope.definition,
+                scriptInterface.getExecuteMethodReturnType(), KEYWORDS);
+        // This reference. Internal use only.
+        locals.defineVariable(null, programScope.definition.getType("Object"), THIS, true);
 
-        // The ctx map set by executable scripts as a read-only map.
-        if (usesCtx) {
-            locals.defineVariable(null, Definition.getType("Map"), CTX, true);
+        // Method arguments
+        for (MethodArgument arg : scriptInterface.getExecuteArguments()) {
+            locals.defineVariable(null, arg.getType(), arg.getName(), true);
         }
 
         // Loop counter to catch infinite loops.  Internal use only.
@@ -137,16 +108,16 @@ public static Locals newMainMethodScope(Locals programScope, boolean usesScore,
         }
         return locals;
     }
-    
+
     /** Creates a new program scope: the list of methods. It is the parent for all methods */
-    public static Locals newProgramScope(Collection<Method> methods) {
-        Locals locals = new Locals(null, null);
+    public static Locals newProgramScope(Definition definition, Collection<Method> methods) {
+        Locals locals = new Locals(null, definition, null, null);
         for (Method method : methods) {
             locals.addMethod(method);
         }
         return locals;
     }
-    
+
     /** Checks if a variable exists or not, in this scope or any parents. */
     public boolean hasVariable(String name) {
         Variable variable = lookupVariable(null, name);
@@ -158,7 +129,7 @@ public boolean hasVariable(String name) {
         }
         return false;
     }
-    
+
     /** Accesses a variable. This will throw IAE if the variable does not exist */
     public Variable getVariable(Location location, String name) {
         Variable variable = lookupVariable(location, name);
@@ -170,7 +141,7 @@ public Variable getVariable(Location location, String name) {
         }
         throw location.createError(new IllegalArgumentException("Variable [" + name + "] is not defined."));
     }
-    
+
     /** Looks up a method. Returns null if the method does not exist. */
     public Method getMethod(MethodKey key) {
         Method method = lookupMethod(key);
@@ -182,23 +153,23 @@ public Method getMethod(MethodKey key) {
         }
         return null;
     }
-    
+
     /** Creates a new variable. Throws IAE if the variable has already been defined (even in a parent) or reserved. */
     public Variable addVariable(Location location, Type type, String name, boolean readonly) {
         if (hasVariable(name)) {
             throw location.createError(new IllegalArgumentException("Variable [" + name + "] is already defined."));
         }
-        if (KEYWORDS.contains(name)) {
+        if (keywords.contains(name)) {
             throw location.createError(new IllegalArgumentException("Variable [" + name + "] is reserved."));
         }
         return defineVariable(location, type, name, readonly);
     }
-    
+
     /** Return type of this scope (e.g. int, if inside a function that returns int) */
     public Type getReturnType() {
         return returnType;
     }
-    
+
     /** Returns the top-level program scope. */
     public Locals getProgramScope() {
         Locals locals = this;
@@ -207,13 +178,22 @@ public Locals getProgramScope() {
         }
         return locals;
     }
-    
+
+    /** Whitelist against which this script is being compiled. */
+    public Definition getDefinition() {
+        return definition;
+    }
+
     ///// private impl
 
+    /** Whitelist against which thhis script is being compiled. */
+    private final Definition definition;
     // parent scope
     private final Locals parent;
     // return type of this scope
     private final Type returnType;
+    // keywords for this scope
+    private final Set<String> keywords;
     // next slot number to assign
     private int nextSlotNumber;
     // variable name -> variable
@@ -225,15 +205,17 @@ public Locals getProgramScope() {
      * Create a new Locals
      */
     private Locals(Locals parent) {
-        this(parent, parent.getReturnType());
+        this(parent, parent.definition, parent.returnType, parent.keywords);
     }
-    
+
     /**
      * Create a new Locals with specified return type
      */
-    private Locals(Locals parent, Type returnType) {
+    private Locals(Locals parent, Definition definition, Type returnType, Set<String> keywords) {
         this.parent = parent;
+        this.definition = definition;
         this.returnType = returnType;
+        this.keywords = keywords;
         if (parent == null) {
             this.nextSlotNumber = 0;
         } else {
@@ -262,7 +244,7 @@ private Method lookupMethod(MethodKey key) {
         return methods.get(key);
     }
 
-    
+
     /** Defines a variable at this scope internally. */
     private Variable defineVariable(Location location, Type type, String name, boolean readonly) {
         if (variables == null) {
@@ -273,7 +255,7 @@ private Variable defineVariable(Location location, Type type, String name, boole
         nextSlotNumber += type.type.getSize();
         return variable;
     }
-    
+
     private void addMethod(Method method) {
         if (methods == null) {
             methods = new HashMap<>();
@@ -293,7 +275,7 @@ public static final class Variable {
         public final Type type;
         public final boolean readonly;
         private final int slot;
-        
+
         public Variable(Location location, String name, Type type, int slot, boolean readonly) {
             this.location = location;
             this.name = name;
@@ -301,12 +283,25 @@ public Variable(Location location, String name, Type type, int slot, boolean rea
             this.slot = slot;
             this.readonly = readonly;
         }
-        
+
         public int getSlot() {
             return slot;
         }
+
+        @Override
+        public String toString() {
+            StringBuilder b = new StringBuilder();
+            b.append("Variable[type=").append(type);
+            b.append(",name=").append(name);
+            b.append(",slot=").append(slot);
+            if (readonly) {
+                b.append(",readonly");
+            }
+            b.append(']');
+            return b.toString();
+        }
     }
-    
+
     public static final class Parameter {
         public final Location location;
         public final String name;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Location.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Location.java
index eb1df00edd2a8..4a4803844a4c4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Location.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Location.java
@@ -66,7 +66,7 @@ public RuntimeException createError(RuntimeException exception) {
         return exception;
     }
 
-    // This maximum length is theoretically 65535 bytes, but as it's CESU-8 encoded we dont know how large it is in bytes, so be safe
+    // This maximum length is theoretically 65535 bytes, but as it's CESU-8 encoded we don't know how large it is in bytes, so be safe
     private static final int MAX_NAME_LENGTH = 256;
     
     /** Computes the file name (mostly important for stacktraces) */
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
index 43fd54c51a479..ac902ee134e13 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
@@ -30,7 +30,6 @@
 
 import java.util.ArrayDeque;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.BitSet;
 import java.util.Deque;
 import java.util.List;
@@ -132,51 +131,48 @@ public void writeLoopCounter(int slot, int count, Location location) {
 
     public void writeCast(final Cast cast) {
         if (cast != null) {
-            final Type from = cast.from;
-            final Type to = cast.to;
-
-            if (from.sort == Sort.CHAR && to.sort == Sort.STRING) {
+            if (cast.from.sort == Sort.CHAR && cast.to.sort == Sort.STRING) {
                 invokeStatic(UTILITY_TYPE, CHAR_TO_STRING);
-            } else if (from.sort == Sort.STRING && to.sort == Sort.CHAR) {
+            } else if (cast.from.sort == Sort.STRING && cast.to.sort == Sort.CHAR) {
                 invokeStatic(UTILITY_TYPE, STRING_TO_CHAR);
-            } else if (cast.unboxFrom) {
-                if (from.sort == Sort.DEF) {
+            } else if (cast.unboxFrom != null) {
+                unbox(cast.unboxFrom.type);
+                writeCast(cast.from, cast.to);
+            } else if (cast.unboxTo != null) {
+                if (cast.from.sort == Sort.DEF) {
                     if (cast.explicit) {
-                        if      (to.sort == Sort.BOOL)   invokeStatic(DEF_UTIL_TYPE, DEF_TO_BOOLEAN);
-                        else if (to.sort == Sort.BYTE)   invokeStatic(DEF_UTIL_TYPE, DEF_TO_BYTE_EXPLICIT);
-                        else if (to.sort == Sort.SHORT)  invokeStatic(DEF_UTIL_TYPE, DEF_TO_SHORT_EXPLICIT);
-                        else if (to.sort == Sort.CHAR)   invokeStatic(DEF_UTIL_TYPE, DEF_TO_CHAR_EXPLICIT);
-                        else if (to.sort == Sort.INT)    invokeStatic(DEF_UTIL_TYPE, DEF_TO_INT_EXPLICIT);
-                        else if (to.sort == Sort.LONG)   invokeStatic(DEF_UTIL_TYPE, DEF_TO_LONG_EXPLICIT);
-                        else if (to.sort == Sort.FLOAT)  invokeStatic(DEF_UTIL_TYPE, DEF_TO_FLOAT_EXPLICIT);
-                        else if (to.sort == Sort.DOUBLE) invokeStatic(DEF_UTIL_TYPE, DEF_TO_DOUBLE_EXPLICIT);
+                        if (cast.to.sort == Sort.BOOL_OBJ) invokeStatic(DEF_UTIL_TYPE, DEF_TO_BOOLEAN);
+                        else if (cast.to.sort == Sort.BYTE_OBJ) invokeStatic(DEF_UTIL_TYPE, DEF_TO_BYTE_EXPLICIT);
+                        else if (cast.to.sort == Sort.SHORT_OBJ) invokeStatic(DEF_UTIL_TYPE, DEF_TO_SHORT_EXPLICIT);
+                        else if (cast.to.sort == Sort.CHAR_OBJ) invokeStatic(DEF_UTIL_TYPE, DEF_TO_CHAR_EXPLICIT);
+                        else if (cast.to.sort == Sort.INT_OBJ) invokeStatic(DEF_UTIL_TYPE, DEF_TO_INT_EXPLICIT);
+                        else if (cast.to.sort == Sort.LONG_OBJ) invokeStatic(DEF_UTIL_TYPE, DEF_TO_LONG_EXPLICIT);
+                        else if (cast.to.sort == Sort.FLOAT_OBJ) invokeStatic(DEF_UTIL_TYPE, DEF_TO_FLOAT_EXPLICIT);
+                        else if (cast.to.sort == Sort.DOUBLE_OBJ) invokeStatic(DEF_UTIL_TYPE, DEF_TO_DOUBLE_EXPLICIT);
                         else throw new IllegalStateException("Illegal tree structure.");
                     } else {
-                        if      (to.sort == Sort.BOOL)   invokeStatic(DEF_UTIL_TYPE, DEF_TO_BOOLEAN);
-                        else if (to.sort == Sort.BYTE)   invokeStatic(DEF_UTIL_TYPE, DEF_TO_BYTE_IMPLICIT);
-                        else if (to.sort == Sort.SHORT)  invokeStatic(DEF_UTIL_TYPE, DEF_TO_SHORT_IMPLICIT);
-                        else if (to.sort == Sort.CHAR)   invokeStatic(DEF_UTIL_TYPE, DEF_TO_CHAR_IMPLICIT);
-                        else if (to.sort == Sort.INT)    invokeStatic(DEF_UTIL_TYPE, DEF_TO_INT_IMPLICIT);
-                        else if (to.sort == Sort.LONG)   invokeStatic(DEF_UTIL_TYPE, DEF_TO_LONG_IMPLICIT);
-                        else if (to.sort == Sort.FLOAT)  invokeStatic(DEF_UTIL_TYPE, DEF_TO_FLOAT_IMPLICIT);
-                        else if (to.sort == Sort.DOUBLE) invokeStatic(DEF_UTIL_TYPE, DEF_TO_DOUBLE_IMPLICIT);
+                        if (cast.to.sort == Sort.BOOL_OBJ) invokeStatic(DEF_UTIL_TYPE, DEF_TO_BOOLEAN);
+                        else if (cast.to.sort == Sort.BYTE_OBJ) invokeStatic(DEF_UTIL_TYPE, DEF_TO_BYTE_IMPLICIT);
+                        else if (cast.to.sort == Sort.SHORT_OBJ) invokeStatic(DEF_UTIL_TYPE, DEF_TO_SHORT_IMPLICIT);
+                        else if (cast.to.sort == Sort.CHAR_OBJ) invokeStatic(DEF_UTIL_TYPE, DEF_TO_CHAR_IMPLICIT);
+                        else if (cast.to.sort == Sort.INT_OBJ) invokeStatic(DEF_UTIL_TYPE, DEF_TO_INT_IMPLICIT);
+                        else if (cast.to.sort == Sort.LONG_OBJ) invokeStatic(DEF_UTIL_TYPE, DEF_TO_LONG_IMPLICIT);
+                        else if (cast.to.sort == Sort.FLOAT_OBJ) invokeStatic(DEF_UTIL_TYPE, DEF_TO_FLOAT_IMPLICIT);
+                        else if (cast.to.sort == Sort.DOUBLE_OBJ) invokeStatic(DEF_UTIL_TYPE, DEF_TO_DOUBLE_IMPLICIT);
                         else throw new IllegalStateException("Illegal tree structure.");
                     }
                 } else {
-                    unbox(from.type);
-                    writeCast(from, to);
+                    writeCast(cast.from, cast.to);
+                    unbox(cast.unboxTo.type);
                 }
-            } else if (cast.unboxTo) {
-                writeCast(from, to);
-                unbox(to.type);
-            } else if (cast.boxFrom) {
-                box(from.type);
-                writeCast(from, to);
-            } else if (cast.boxTo) {
-                writeCast(from, to);
-                box(to.type);
+            } else if (cast.boxFrom != null) {
+                box(cast.boxFrom.type);
+                writeCast(cast.from, cast.to);
+            } else if (cast.boxTo != null) {
+                writeCast(cast.from, cast.to);
+                box(cast.boxTo.type);
             } else {
-                writeCast(from, to);
+                writeCast(cast.from, cast.to);
             }
         }
     }
@@ -270,19 +266,19 @@ public void writeToStrings() {
     }
 
     /** Writes a dynamic binary instruction: returnType, lhs, and rhs can be different */
-    public void writeDynamicBinaryInstruction(Location location, Type returnType, Type lhs, Type rhs, 
+    public void writeDynamicBinaryInstruction(Location location, Type returnType, Type lhs, Type rhs,
                                               Operation operation, int flags) {
         org.objectweb.asm.Type methodType = org.objectweb.asm.Type.getMethodType(returnType.type, lhs.type, rhs.type);
-        
+
         switch (operation) {
             case MUL:
-                invokeDefCall("mul", methodType, DefBootstrap.BINARY_OPERATOR, flags); 
+                invokeDefCall("mul", methodType, DefBootstrap.BINARY_OPERATOR, flags);
                 break;
             case DIV:
-                invokeDefCall("div", methodType, DefBootstrap.BINARY_OPERATOR, flags); 
+                invokeDefCall("div", methodType, DefBootstrap.BINARY_OPERATOR, flags);
                 break;
             case REM:
-                invokeDefCall("rem", methodType, DefBootstrap.BINARY_OPERATOR, flags); 
+                invokeDefCall("rem", methodType, DefBootstrap.BINARY_OPERATOR, flags);
                 break;
             case ADD:
                 // if either side is primitive, then the + operator should always throw NPE on null,
@@ -295,31 +291,31 @@ public void writeDynamicBinaryInstruction(Location location, Type returnType, Ty
                 invokeDefCall("add", methodType, DefBootstrap.BINARY_OPERATOR, flags);
                 break;
             case SUB:
-                invokeDefCall("sub", methodType, DefBootstrap.BINARY_OPERATOR, flags); 
+                invokeDefCall("sub", methodType, DefBootstrap.BINARY_OPERATOR, flags);
                 break;
             case LSH:
                 invokeDefCall("lsh", methodType, DefBootstrap.SHIFT_OPERATOR, flags);
                 break;
             case USH:
-                invokeDefCall("ush", methodType, DefBootstrap.SHIFT_OPERATOR, flags); 
+                invokeDefCall("ush", methodType, DefBootstrap.SHIFT_OPERATOR, flags);
                 break;
             case RSH:
-                invokeDefCall("rsh", methodType, DefBootstrap.SHIFT_OPERATOR, flags); 
+                invokeDefCall("rsh", methodType, DefBootstrap.SHIFT_OPERATOR, flags);
                 break;
-            case BWAND: 
+            case BWAND:
                 invokeDefCall("and", methodType, DefBootstrap.BINARY_OPERATOR, flags);
                 break;
-            case XOR:   
+            case XOR:
                 invokeDefCall("xor", methodType, DefBootstrap.BINARY_OPERATOR, flags);
                 break;
-            case BWOR:  
+            case BWOR:
                 invokeDefCall("or", methodType, DefBootstrap.BINARY_OPERATOR, flags);
                 break;
             default:
                 throw location.createError(new IllegalStateException("Illegal tree structure."));
         }
     }
-    
+
     /** Writes a static binary instruction */
     public void writeBinaryInstruction(Location location, Type type, Operation operation) {
         final Sort sort = type.sort;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/NeedsScore.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/NeedsScore.java
deleted file mode 100644
index 6812e1019b22d..0000000000000
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/NeedsScore.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.painless;
-
-/**
- * Marker interface that a generated {@link Executable} uses the {@code _score} value
- */
-public interface NeedsScore {
-}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Operation.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Operation.java
index e32a77022f6a5..828e26e56b49d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Operation.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Operation.java
@@ -27,7 +27,7 @@
  */
 public enum Operation {
 
-    MUL     ( "+"   ),
+    MUL     ( "*"   ),
     DIV     ( "/"   ),
     REM     ( "%"   ),
     ADD     ( "+"   ),
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessExplainError.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessExplainError.java
new file mode 100644
index 0000000000000..291d852bdde22
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessExplainError.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+import org.elasticsearch.painless.api.Debug;
+import org.elasticsearch.script.ScriptException;
+
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+import static java.util.Collections.singletonList;
+
+/**
+ * Thrown by {@link Debug#explain(Object)} to explain an object. Subclass of {@linkplain Error} so it cannot be caught by painless
+ * scripts.
+ */
+public class PainlessExplainError extends Error {
+    private final Object objectToExplain;
+
+    public PainlessExplainError(Object objectToExplain) {
+        this.objectToExplain = objectToExplain;
+    }
+
+    Object getObjectToExplain() {
+        return objectToExplain;
+    }
+
+    /**
+     * Headers to be added to the {@link ScriptException} for structured rendering.
+     */
+    public Map<String, List<String>> getHeaders(Definition definition) {
+        Map<String, List<String>> headers = new TreeMap<>();
+        String toString = "null";
+        String javaClassName = null;
+        String painlessClassName = null;
+        if (objectToExplain != null) {
+            toString = objectToExplain.toString();
+            javaClassName = objectToExplain.getClass().getName();
+            Definition.RuntimeClass runtimeClass = definition.getRuntimeClass(objectToExplain.getClass());
+            if (runtimeClass != null) {
+                painlessClassName = runtimeClass.getStruct().name;
+            }
+        }
+
+        headers.put("es.to_string", singletonList(toString));
+        if (painlessClassName != null) {
+            headers.put("es.painless_class", singletonList(painlessClassName));
+        }
+        if (javaClassName != null) {
+            headers.put("es.java_class", singletonList(javaClassName));
+        }
+        return headers;
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java
index 954314286bcf7..c00dc64310237 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java
@@ -20,19 +20,21 @@
 package org.elasticsearch.painless;
 
 
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.ScriptPlugin;
-import org.elasticsearch.script.ScriptEngineRegistry;
 import org.elasticsearch.script.ScriptEngineService;
-import org.elasticsearch.script.ScriptModule;
+
+import java.util.Arrays;
+import java.util.List;
 
 /**
  * Registers Painless as a plugin.
  */
 public final class PainlessPlugin extends Plugin implements ScriptPlugin {
 
-    // force to pare our definition at startup (not on the user's first script)
+    // force to parse our definition at startup (not on the user's first script)
     static {
         Definition.VOID_TYPE.hashCode();
     }
@@ -41,4 +43,9 @@ public final class PainlessPlugin extends Plugin implements ScriptPlugin {
     public ScriptEngineService getScriptEngineService(Settings settings) {
         return new PainlessScriptEngineService(settings);
     }
+
+    @Override
+    public List<Setting<?>> getSettings() {
+        return Arrays.asList(CompilerSettings.REGEX_ENABLED);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScript.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScript.java
new file mode 100644
index 0000000000000..b4e7c157f1fe2
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScript.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+import org.elasticsearch.script.ScriptException;
+
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Abstract superclass on top of which all Painless scripts are built.
+ */
+public abstract class PainlessScript {
+    /**
+     * Name of the script set at compile time.
+     */
+    private final String name;
+    /**
+     * Source of the script.
+     */
+    private final String source;
+    /**
+     * Character number of the start of each statement.
+     */
+    private final BitSet statements;
+
+    protected PainlessScript(String name, String source, BitSet statements) {
+        this.name = name;
+        this.source = source;
+        this.statements = statements;
+    }
+
+    /**
+     * Adds stack trace and other useful information to exceptions thrown
+     * from a Painless script.
+     * @param t The throwable to build an exception around.
+     * @return The generated ScriptException.
+     */
+    protected final ScriptException convertToScriptException(Throwable t, Map<String, List<String>> extraMetadata) {
+        // create a script stack: this is just the script portion
+        List<String> scriptStack = new ArrayList<>();
+        for (StackTraceElement element : t.getStackTrace()) {
+            if (WriterConstants.CLASS_NAME.equals(element.getClassName())) {
+                // found the script portion
+                int offset = element.getLineNumber();
+                if (offset == -1) {
+                    scriptStack.add("<<< unknown portion of script >>>");
+                } else {
+                    offset--; // offset is 1 based, line numbers must be!
+                    int startOffset = getPreviousStatement(offset);
+                    if (startOffset == -1) {
+                        assert false; // should never happen unless we hit exc in ctor prologue...
+                        startOffset = 0;
+                    }
+                    int endOffset = getNextStatement(startOffset);
+                    if (endOffset == -1) {
+                        endOffset = source.length();
+                    }
+                    // TODO: if this is still too long, truncate and use ellipses
+                    String snippet = source.substring(startOffset, endOffset);
+                    scriptStack.add(snippet);
+                    StringBuilder pointer = new StringBuilder();
+                    for (int i = startOffset; i < offset; i++) {
+                        pointer.append(' ');
+                    }
+                    pointer.append("^---- HERE");
+                    scriptStack.add(pointer.toString());
+                }
+                break;
+            // but filter our own internal stacks (e.g. indy bootstrap)
+            } else if (!shouldFilter(element)) {
+                scriptStack.add(element.toString());
+            }
+        }
+        // build a name for the script:
+        final String name;
+        if (PainlessScriptEngineService.INLINE_NAME.equals(this.name)) {
+            name = source;
+        } else {
+            name = this.name;
+        }
+        ScriptException scriptException = new ScriptException("runtime error", t, scriptStack, name, PainlessScriptEngineService.NAME);
+        for (Map.Entry<String, List<String>> entry : extraMetadata.entrySet()) {
+            scriptException.addMetadata(entry.getKey(), entry.getValue());
+        }
+        return scriptException;
+    }
+
+    /** returns true for methods that are part of the runtime */
+    private static boolean shouldFilter(StackTraceElement element) {
+        return element.getClassName().startsWith("org.elasticsearch.painless.") ||
+               element.getClassName().startsWith("java.lang.invoke.") ||
+               element.getClassName().startsWith("sun.invoke.");
+    }
+
+    /**
+     * Finds the start of the first statement boundary that is on or before {@code offset}. If one is not found, {@code -1} is returned.
+     */
+    private int getPreviousStatement(int offset) {
+        return statements.previousSetBit(offset);
+    }
+
+    /**
+     * Finds the start of the first statement boundary that is after {@code offset}. If one is not found, {@code -1} is returned.
+     */
+    private int getNextStatement(int offset) {
+        return statements.nextSetBit(offset + 1);
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
index 834593aeb9913..f498be733044b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
@@ -53,11 +53,6 @@ public final class PainlessScriptEngineService extends AbstractComponent impleme
      */
     public static final String NAME = "painless";
 
-    /**
-     * Default compiler settings to be used.
-     */
-    private static final CompilerSettings DEFAULT_COMPILER_SETTINGS = new CompilerSettings();
-
     /**
      * Permissions context used during compilation.
      */
@@ -74,12 +69,19 @@ public final class PainlessScriptEngineService extends AbstractComponent impleme
         });
     }
 
+    /**
+     * Default compiler settings to be used. Note that {@link CompilerSettings} is mutable but this instance shouldn't be mutated outside
+     * of {@link PainlessScriptEngineService#PainlessScriptEngineService(Settings)}.
+     */
+    private final CompilerSettings defaultCompilerSettings = new CompilerSettings();
+
     /**
      * Constructor.
      * @param settings The settings to initialize the engine with.
      */
     public PainlessScriptEngineService(final Settings settings) {
         super(settings);
+        defaultCompilerSettings.setRegexesEnabled(CompilerSettings.REGEX_ENABLED.get(settings));
     }
 
     /**
@@ -107,33 +109,44 @@ public String getExtension() {
 
     @Override
     public Object compile(String scriptName, final String scriptSource, final Map<String, String> params) {
+        return compile(GenericElasticsearchScript.class, scriptName, scriptSource, params);
+    }
+
+    <T> T compile(Class<T> iface, String scriptName, final String scriptSource, final Map<String, String> params) {
         final CompilerSettings compilerSettings;
 
         if (params.isEmpty()) {
             // Use the default settings.
-            compilerSettings = DEFAULT_COMPILER_SETTINGS;
+            compilerSettings = defaultCompilerSettings;
         } else {
             // Use custom settings specified by params.
             compilerSettings = new CompilerSettings();
+
+            // Except regexes enabled - this is a node level setting and can't be changed in the request.
+            compilerSettings.setRegexesEnabled(defaultCompilerSettings.areRegexesEnabled());
+
             Map<String, String> copy = new HashMap<>(params);
-            String value = copy.remove(CompilerSettings.MAX_LOOP_COUNTER);
 
+            String value = copy.remove(CompilerSettings.MAX_LOOP_COUNTER);
             if (value != null) {
                 compilerSettings.setMaxLoopCounter(Integer.parseInt(value));
             }
 
             value = copy.remove(CompilerSettings.PICKY);
-
             if (value != null) {
                 compilerSettings.setPicky(Boolean.parseBoolean(value));
             }
 
             value = copy.remove(CompilerSettings.INITIAL_CALL_SITE_DEPTH);
-
             if (value != null) {
                 compilerSettings.setInitialCallSiteDepth(Integer.parseInt(value));
             }
 
+            value = copy.remove(CompilerSettings.REGEX_ENABLED.getKey());
+            if (value != null) {
+                throw new IllegalArgumentException("[painless.regex.enabled] can only be set on node startup.");
+            }
+
             if (!copy.isEmpty()) {
                 throw new IllegalArgumentException("Unrecognized compile-time parameter(s): " + copy);
             }
@@ -156,14 +169,15 @@ public Loader run() {
 
         try {
             // Drop all permissions to actually compile the code itself.
-            return AccessController.doPrivileged(new PrivilegedAction<Executable>() {
+            return AccessController.doPrivileged(new PrivilegedAction<T>() {
                 @Override
-                public Executable run() {
-                    return Compiler.compile(loader, scriptName == null ? INLINE_NAME : scriptName, scriptSource, compilerSettings);
+                public T run() {
+                    String name = scriptName == null ? INLINE_NAME : scriptName;
+                    return Compiler.compile(loader, iface, name, scriptSource, compilerSettings);
                 }
             }, COMPILATION_CONTEXT);
         // Note that it is safe to catch any of the following errors since Painless is stateless.
-        } catch (OutOfMemoryError | StackOverflowError | Exception e) {
+        } catch (OutOfMemoryError | StackOverflowError | VerifyError | Exception e) {
             throw convertToScriptException(scriptName == null ? scriptSource : scriptName, scriptSource, e);
         }
     }
@@ -176,7 +190,7 @@ public Executable run() {
      */
     @Override
     public ExecutableScript executable(final CompiledScript compiledScript, final Map<String, Object> vars) {
-        return new ScriptImpl((Executable)compiledScript.compiled(), vars, null);
+        return new ScriptImpl((GenericElasticsearchScript) compiledScript.compiled(), vars, null);
     }
 
     /**
@@ -196,7 +210,7 @@ public SearchScript search(final CompiledScript compiledScript, final SearchLook
              */
             @Override
             public LeafSearchScript getLeafSearchScript(final LeafReaderContext context) throws IOException {
-                return new ScriptImpl((Executable)compiledScript.compiled(), vars, lookup.getLeafSearchLookup(context));
+                return new ScriptImpl((GenericElasticsearchScript) compiledScript.compiled(), vars, lookup.getLeafSearchLookup(context));
             }
 
             /**
@@ -204,7 +218,7 @@ public LeafSearchScript getLeafSearchScript(final LeafReaderContext context) thr
              */
             @Override
             public boolean needsScores() {
-                return compiledScript.compiled() instanceof NeedsScore;
+                return ((GenericElasticsearchScript) compiledScript.compiled()).uses$_score();
             }
         };
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java
index 0d42f8d23365d..c07a340838660 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java
@@ -20,16 +20,16 @@
 package org.elasticsearch.painless;
 
 import org.apache.lucene.search.Scorer;
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.LeafSearchScript;
-import org.elasticsearch.script.ScriptException;
 import org.elasticsearch.search.lookup.LeafDocLookup;
 import org.elasticsearch.search.lookup.LeafSearchLookup;
 
-import java.util.ArrayList;
+import java.io.IOException;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
+import java.util.function.Function;
 
 /**
  * ScriptImpl can be used as either an {@link ExecutableScript} or a {@link LeafSearchScript}
@@ -38,9 +38,9 @@
 final class ScriptImpl implements ExecutableScript, LeafSearchScript {
 
     /**
-     * The Painless Executable script that can be run.
+     * The Painless script that can be run.
      */
-    private final Executable executable;
+    private final GenericElasticsearchScript script;
 
     /**
      * A map that can be used to access input parameters at run-time.
@@ -57,6 +57,16 @@ final class ScriptImpl implements ExecutableScript, LeafSearchScript {
      */
     private final LeafDocLookup doc;
 
+    /**
+     * Looks up the {@code _score} from {@link #scorer} if {@code _score} is used, otherwise returns {@code 0.0}.
+     */
+    private final ScoreLookup scoreLookup;
+
+    /**
+     * Looks up the {@code ctx} from the {@link #variables} if {@code ctx} is used, otherwise return {@code null}.
+     */
+    private final Function<Map<String, Object>, Map<?, ?>> ctxLookup;
+
     /**
      * Current scorer being used
      * @see #setScorer(Scorer)
@@ -71,12 +81,12 @@ final class ScriptImpl implements ExecutableScript, LeafSearchScript {
 
     /**
      * Creates a ScriptImpl for the a previously compiled Painless script.
-     * @param executable The previously compiled Painless script.
+     * @param script The previously compiled Painless script.
      * @param vars The initial variables to run the script with.
      * @param lookup The lookup to allow search fields to be available if this is run as a search script.
      */
-    ScriptImpl(final Executable executable, final Map<String, Object> vars, final LeafSearchLookup lookup) {
-        this.executable = executable;
+    ScriptImpl(final GenericElasticsearchScript script, final Map<String, Object> vars, final LeafSearchLookup lookup) {
+        this.script = script;
         this.lookup = lookup;
         this.variables = new HashMap<>();
 
@@ -90,6 +100,9 @@ final class ScriptImpl implements ExecutableScript, LeafSearchScript {
         } else {
             doc = null;
         }
+
+        scoreLookup = script.uses$_score() ? ScriptImpl::getScore : scorer -> 0.0;
+        ctxLookup = script.uses$ctx() ? variables -> (Map<?, ?>) variables.get("ctx") : variables -> null;
     }
 
     /**
@@ -117,71 +130,7 @@ public void setNextAggregationValue(Object value) {
      */
     @Override
     public Object run() {
-        try {
-            return executable.execute(variables, scorer, doc, aggregationValue);
-        // Note that it is safe to catch any of the following errors since Painless is stateless.
-        } catch (PainlessError | BootstrapMethodError | OutOfMemoryError | StackOverflowError | Exception e) {
-            throw convertToScriptException(e);
-        }
-    }
-
-    /**
-     * Adds stack trace and other useful information to exceptiosn thrown
-     * from a Painless script.
-     * @param t The throwable to build an exception around.
-     * @return The generated ScriptException.
-     */
-    private ScriptException convertToScriptException(Throwable t) {
-        // create a script stack: this is just the script portion
-        List<String> scriptStack = new ArrayList<>();
-        for (StackTraceElement element : t.getStackTrace()) {
-            if (WriterConstants.CLASS_NAME.equals(element.getClassName())) {
-                // found the script portion
-                int offset = element.getLineNumber();
-                if (offset == -1) {
-                    scriptStack.add("<<< unknown portion of script >>>");
-                } else {
-                    offset--; // offset is 1 based, line numbers must be!
-                    int startOffset = executable.getPreviousStatement(offset);
-                    if (startOffset == -1) {
-                        assert false; // should never happen unless we hit exc in ctor prologue...
-                        startOffset = 0;
-                    }
-                    int endOffset = executable.getNextStatement(startOffset);
-                    if (endOffset == -1) {
-                        endOffset = executable.getSource().length();
-                    }
-                    // TODO: if this is still too long, truncate and use ellipses
-                    String snippet = executable.getSource().substring(startOffset, endOffset);
-                    scriptStack.add(snippet);
-                    StringBuilder pointer = new StringBuilder();
-                    for (int i = startOffset; i < offset; i++) {
-                        pointer.append(' ');
-                    }
-                    pointer.append("^---- HERE");
-                    scriptStack.add(pointer.toString());
-                }
-                break;
-            // but filter our own internal stacks (e.g. indy bootstrap)
-            } else if (!shouldFilter(element)) {
-                scriptStack.add(element.toString());
-            }
-        }
-        // build a name for the script:
-        final String name;
-        if (PainlessScriptEngineService.INLINE_NAME.equals(executable.getName())) {
-            name = executable.getSource();
-        } else {
-            name = executable.getName();
-        }
-        throw new ScriptException("runtime error", t, scriptStack, name, PainlessScriptEngineService.NAME);
-    }
-
-    /** returns true for methods that are part of the runtime */
-    private static boolean shouldFilter(StackTraceElement element) {
-        return element.getClassName().startsWith("org.elasticsearch.painless.") ||
-               element.getClassName().startsWith("java.lang.invoke.") ||
-               element.getClassName().startsWith("sun.invoke.");
+        return script.execute(variables, scoreLookup.apply(scorer), doc, aggregationValue, ctxLookup.apply(variables));
     }
 
     /**
@@ -232,4 +181,16 @@ public void setSource(final Map<String, Object> source) {
             lookup.source().setSource(source);
         }
     }
+
+    private static double getScore(Scorer scorer) {
+        try {
+            return scorer.score();
+        } catch (IOException e) {
+            throw new ElasticsearchException("couldn't lookup score", e);
+        }
+    }
+
+    interface ScoreLookup {
+        double apply(Scorer scorer);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptInterface.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptInterface.java
new file mode 100644
index 0000000000000..28fa6fd42806c
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptInterface.java
@@ -0,0 +1,212 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+import java.lang.invoke.MethodType;
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.function.Function;
+
+import static java.util.Collections.unmodifiableList;
+import static org.elasticsearch.painless.WriterConstants.USES_PARAMETER_METHOD_TYPE;
+
+/**
+ * Information about the interface being implemented by the painless script.
+ */
+public class ScriptInterface {
+    private final Class<?> iface;
+    private final org.objectweb.asm.commons.Method executeMethod;
+    private final Definition.Type executeMethodReturnType;
+    private final List<MethodArgument> executeArguments;
+    private final List<org.objectweb.asm.commons.Method> usesMethods;
+
+    public ScriptInterface(Definition definition, Class<?> iface) {
+        this.iface = iface;
+
+        // Find the main method and the uses$argName methods
+        java.lang.reflect.Method executeMethod = null;
+        List<org.objectweb.asm.commons.Method> usesMethods = new ArrayList<>();
+        for (java.lang.reflect.Method m : iface.getMethods()) {
+            if (m.isDefault()) {
+                continue;
+            }
+            if (m.getName().equals("execute")) {
+                if (executeMethod == null) {
+                    executeMethod = m;
+                } else {
+                    throw new IllegalArgumentException(
+                            "Painless can only implement interfaces that have a single method named [execute] but [" + iface.getName()
+                                    + "] has more than one.");
+                }
+                continue;
+            }
+            if (m.getName().startsWith("uses$")) {
+                if (false == m.getReturnType().equals(boolean.class)) {
+                    throw new IllegalArgumentException("Painless can only implement uses$ methods that return boolean but ["
+                            + iface.getName() + "#" + m.getName() + "] returns [" + m.getReturnType().getName() + "].");
+                }
+                if (m.getParameterTypes().length > 0) {
+                    throw new IllegalArgumentException("Painless can only implement uses$ methods that do not take parameters but ["
+                            + iface.getName() + "#" + m.getName() + "] does.");
+                }
+                usesMethods.add(new org.objectweb.asm.commons.Method(m.getName(), USES_PARAMETER_METHOD_TYPE.toMethodDescriptorString()));
+                continue;
+            }
+            throw new IllegalArgumentException("Painless can only implement methods named [execute] and [uses$argName] but ["
+                    + iface.getName() + "] contains a method named [" + m.getName() + "]");
+        }
+        MethodType methodType = MethodType.methodType(executeMethod.getReturnType(), executeMethod.getParameterTypes());
+        this.executeMethod = new org.objectweb.asm.commons.Method(executeMethod.getName(), methodType.toMethodDescriptorString());
+        executeMethodReturnType = definitionTypeForClass(definition, executeMethod.getReturnType(),
+                componentType -> "Painless can only implement execute methods returning a whitelisted type but [" + iface.getName()
+                        + "#execute] returns [" + componentType.getName() + "] which isn't whitelisted.");
+
+        // Look up the argument names
+        Set<String> argumentNames = new LinkedHashSet<>();
+        List<MethodArgument> arguments = new ArrayList<>();
+        String[] argumentNamesConstant = readArgumentNamesConstant(iface);
+        Class<?>[] types = executeMethod.getParameterTypes();
+        if (argumentNamesConstant.length != types.length) {
+            throw new IllegalArgumentException("[" + iface.getName() + "#ARGUMENTS] has length [2] but ["
+                    + iface.getName() + "#execute] takes [1] argument.");
+        }
+        for (int arg = 0; arg < types.length; arg++) {
+            arguments.add(methodArgument(definition, types[arg], argumentNamesConstant[arg]));
+            argumentNames.add(argumentNamesConstant[arg]);
+        }
+        this.executeArguments = unmodifiableList(arguments);
+
+        // Validate that the uses$argName methods reference argument names
+        for (org.objectweb.asm.commons.Method usesMethod : usesMethods) {
+            if (false == argumentNames.contains(usesMethod.getName().substring("uses$".length()))) {
+                throw new IllegalArgumentException("Painless can only implement uses$ methods that match a parameter name but ["
+                        + iface.getName() + "#" + usesMethod.getName() + "] doesn't match any of " + argumentNames + ".");
+            }
+        }
+        this.usesMethods = unmodifiableList(usesMethods);
+    }
+
+    /**
+     * The interface that the Painless script should implement.
+     */
+    public Class<?> getInterface() {
+        return iface;
+    }
+
+    /**
+     * An asm method descriptor for the {@code execute} method.
+     */
+    public org.objectweb.asm.commons.Method getExecuteMethod() {
+        return executeMethod;
+    }
+
+    /**
+     * The Painless {@link Definition.Type} or the return type of the {@code execute} method. This is used to generate the appropriate
+     * return bytecode.
+     */
+    public Definition.Type getExecuteMethodReturnType() {
+        return executeMethodReturnType;
+    }
+
+    /**
+     * Painless {@link Definition.Type}s and names of the arguments to the {@code execute} method. The names are exposed to the Painless
+     * script.
+     */
+    public List<MethodArgument> getExecuteArguments() {
+        return executeArguments;
+    }
+
+    /**
+     * The {@code uses$varName} methods that must be implemented by Painless to complete implementing the interface.
+     */
+    public List<org.objectweb.asm.commons.Method> getUsesMethods() {
+        return usesMethods;
+    }
+
+    /**
+     * Painless {@link Definition.Type}s and name of the argument to the {@code execute} method.
+     */
+    public static class MethodArgument {
+        private final Definition.Type type;
+        private final String name;
+
+        public MethodArgument(Definition.Type type, String name) {
+            this.type = type;
+            this.name = name;
+        }
+
+        public Definition.Type getType() {
+            return type;
+        }
+
+        public String getName() {
+            return name;
+        }
+    }
+
+    private MethodArgument methodArgument(Definition definition, Class<?> type, String argName) {
+        Definition.Type defType = definitionTypeForClass(definition, type, componentType -> "[" + argName + "] is of unknown type ["
+                + componentType.getName() + ". Painless interfaces can only accept arguments that are of whitelisted types.");
+        return new MethodArgument(defType, argName);
+    }
+
+    private static Definition.Type definitionTypeForClass(Definition definition, Class<?> type,
+            Function<Class<?>, String> unknownErrorMessageSource) {
+        int dimensions = 0;
+        Class<?> componentType = type;
+        while (componentType.isArray()) {
+            dimensions++;
+            componentType = componentType.getComponentType();
+        }
+        Definition.Struct struct;
+        if (componentType.equals(Object.class)) {
+            struct = Definition.DEF_TYPE.struct;
+        } else {
+            Definition.RuntimeClass runtimeClass = definition.getRuntimeClass(componentType);
+            if (runtimeClass == null) {
+                throw new IllegalArgumentException(unknownErrorMessageSource.apply(componentType));
+            }
+            struct = runtimeClass.getStruct();
+        }
+        return definition.getType(struct, dimensions);
+    }
+
+    private static String[] readArgumentNamesConstant(Class<?> iface) {
+        Field argumentNamesField;
+        try {
+            argumentNamesField = iface.getField("ARGUMENTS");
+        } catch (NoSuchFieldException e) {
+            throw new IllegalArgumentException("Painless needs a constant [String[] ARGUMENTS] on all interfaces it implements with the "
+                    + "names of the method arguments but [" + iface.getName() + "] doesn't have one.", e);
+        }
+        if (false == argumentNamesField.getType().equals(String[].class)) {
+            throw new IllegalArgumentException("Painless needs a constant [String[] ARGUMENTS] on all interfaces it implements with the "
+                    + "names of the method arguments but [" + iface.getName() + "] doesn't have one.");
+        }
+        try {
+            return (String[]) argumentNamesField.get(null);
+        } catch (IllegalArgumentException | IllegalAccessException e) {
+            throw new IllegalArgumentException("Error trying to read [" + iface.getName() + "#ARGUMENTS]", e);
+        }
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
index c546207b1ee07..af9f84424b4af 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
@@ -19,19 +19,20 @@
 
 package org.elasticsearch.painless;
 
-import org.apache.lucene.search.Scorer;
-import org.elasticsearch.search.lookup.LeafDocLookup;
+import org.elasticsearch.painless.api.Augmentation;
+import org.elasticsearch.script.ScriptException;
 import org.objectweb.asm.Handle;
 import org.objectweb.asm.Opcodes;
 import org.objectweb.asm.Type;
 import org.objectweb.asm.commons.Method;
 
 import java.lang.invoke.CallSite;
-import java.lang.invoke.LambdaMetafactory;
 import java.lang.invoke.MethodHandle;
 import java.lang.invoke.MethodHandles;
 import java.lang.invoke.MethodType;
 import java.util.BitSet;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.Objects;
@@ -45,22 +46,34 @@ public final class WriterConstants {
 
     public static final int CLASS_VERSION = Opcodes.V1_8;
     public static final int ASM_VERSION = Opcodes.ASM5;
-    public static final String BASE_CLASS_NAME = Executable.class.getName();
-    public static final Type BASE_CLASS_TYPE   = Type.getType(Executable.class);
+    public static final String BASE_CLASS_NAME = PainlessScript.class.getName();
+    public static final Type BASE_CLASS_TYPE   = Type.getType(PainlessScript.class);
+    public static final Method CONVERT_TO_SCRIPT_EXCEPTION_METHOD = getAsmMethod(ScriptException.class, "convertToScriptException",
+            Throwable.class, Map.class);
 
     public static final String CLASS_NAME      = BASE_CLASS_NAME + "$Script";
     public static final Type CLASS_TYPE        = Type.getObjectType(CLASS_NAME.replace('.', '/'));
+    
+    public static final String CTOR_METHOD_NAME = "<init>";
 
-    public static final Method CONSTRUCTOR = getAsmMethod(void.class, "<init>", String.class, String.class, BitSet.class);
+    public static final Method CONSTRUCTOR = getAsmMethod(void.class, CTOR_METHOD_NAME, String.class, String.class, BitSet.class);
     public static final Method CLINIT      = getAsmMethod(void.class, "<clinit>");
-    public static final Method EXECUTE     =
-        getAsmMethod(Object.class, "execute", Map.class, Scorer.class, LeafDocLookup.class, Object.class);
 
-    public static final Type PAINLESS_ERROR_TYPE = Type.getType(PainlessError.class);
+    // All of these types are caught by the main method and rethrown as ScriptException
+    public static final Type PAINLESS_ERROR_TYPE         = Type.getType(PainlessError.class);
+    public static final Type BOOTSTRAP_METHOD_ERROR_TYPE = Type.getType(BootstrapMethodError.class);
+    public static final Type OUT_OF_MEMORY_ERROR_TYPE    = Type.getType(OutOfMemoryError.class);
+    public static final Type STACK_OVERFLOW_ERROR_TYPE   = Type.getType(StackOverflowError.class);
+    public static final Type EXCEPTION_TYPE              = Type.getType(Exception.class);
+    public static final Type PAINLESS_EXPLAIN_ERROR_TYPE = Type.getType(PainlessExplainError.class);
+    public static final Method PAINLESS_EXPLAIN_ERROR_GET_HEADERS_METHOD = getAsmMethod(Map.class, "getHeaders", Definition.class);
+
+    public static final Type DEFINITION_TYPE = Type.getType(Definition.class);
+
+    public static final Type COLLECTIONS_TYPE = Type.getType(Collections.class);
+    public static final Method EMPTY_MAP_METHOD = getAsmMethod(Map.class, "emptyMap");
 
-    public static final Type NEEDS_SCORE_TYPE = Type.getType(NeedsScore.class);
-    public static final Type SCORER_TYPE = Type.getType(Scorer.class);
-    public static final Method SCORER_SCORE = getAsmMethod(float.class, "score");
+    public static final MethodType USES_PARAMETER_METHOD_TYPE = MethodType.methodType(boolean.class);
 
     public static final Type MAP_TYPE  = Type.getType(Map.class);
     public static final Method MAP_GET = getAsmMethod(Object.class, "get", Object.class);
@@ -72,9 +85,10 @@ public final class WriterConstants {
     public static final Type UTILITY_TYPE = Type.getType(Utility.class);
     public static final Method STRING_TO_CHAR = getAsmMethod(char.class, "StringTochar", String.class);
     public static final Method CHAR_TO_STRING = getAsmMethod(String.class, "charToString", char.class);
-    
+
+
     public static final Type METHOD_HANDLE_TYPE = Type.getType(MethodHandle.class);
-    
+
     public static final Type AUGMENTATION_TYPE = Type.getType(Augmentation.class);
 
     /**
@@ -88,13 +102,13 @@ public final class WriterConstants {
     public static final Method MATCHER_MATCHES = getAsmMethod(boolean.class, "matches");
     public static final Method MATCHER_FIND = getAsmMethod(boolean.class, "find");
 
-    /** dynamic callsite bootstrap signature */
-    static final MethodType DEF_BOOTSTRAP_TYPE =
-        MethodType.methodType(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class, 
-                              int.class, int.class, Object[].class);
-    static final Handle DEF_BOOTSTRAP_HANDLE =
-        new Handle(Opcodes.H_INVOKESTATIC, Type.getInternalName(DefBootstrap.class),
-            "bootstrap", DEF_BOOTSTRAP_TYPE.toMethodDescriptorString(), false);
+    public static final Method DEF_BOOTSTRAP_METHOD = getAsmMethod(CallSite.class, "$bootstrapDef", MethodHandles.Lookup.class,
+            String.class, MethodType.class, int.class, int.class, Object[].class);
+    static final Handle DEF_BOOTSTRAP_HANDLE = new Handle(Opcodes.H_INVOKESTATIC, CLASS_TYPE.getInternalName(), "$bootstrapDef",
+            DEF_BOOTSTRAP_METHOD.getDescriptor(), false);
+    public static final Type DEF_BOOTSTRAP_DELEGATE_TYPE = Type.getType(DefBootstrap.class);
+    public static final Method DEF_BOOTSTRAP_DELEGATE_METHOD = getAsmMethod(CallSite.class, "bootstrap", Definition.class,
+            MethodHandles.Lookup.class, String.class, MethodType.class, int.class, int.class, Object[].class);
 
     public static final Type DEF_UTIL_TYPE = Type.getType(Def.class);
     public static final Method DEF_TO_BOOLEAN         = getAsmMethod(boolean.class, "DefToboolean"       , Object.class);
@@ -112,14 +126,20 @@ public final class WriterConstants {
     public static final Method DEF_TO_LONG_EXPLICIT   = getAsmMethod(long.class   , "DefTolongExplicit"  , Object.class);
     public static final Method DEF_TO_FLOAT_EXPLICIT  = getAsmMethod(float.class  , "DefTofloatExplicit" , Object.class);
     public static final Method DEF_TO_DOUBLE_EXPLICIT = getAsmMethod(double.class , "DefTodoubleExplicit", Object.class);
+    public static final Type DEF_ARRAY_LENGTH_METHOD_TYPE = Type.getMethodType(Type.INT_TYPE, Definition.DEF_TYPE.type);
 
     /** invokedynamic bootstrap for lambda expression/method references */
     public static final MethodType LAMBDA_BOOTSTRAP_TYPE =
             MethodType.methodType(CallSite.class, MethodHandles.Lookup.class, String.class,
-                                  MethodType.class, Object[].class);
+                                  MethodType.class, MethodType.class, String.class, int.class, String.class, MethodType.class);
     public static final Handle LAMBDA_BOOTSTRAP_HANDLE =
-            new Handle(Opcodes.H_INVOKESTATIC, Type.getInternalName(LambdaMetafactory.class),
-                "altMetafactory", LAMBDA_BOOTSTRAP_TYPE.toMethodDescriptorString(), false);
+            new Handle(Opcodes.H_INVOKESTATIC, Type.getInternalName(LambdaBootstrap.class),
+                "lambdaBootstrap", LAMBDA_BOOTSTRAP_TYPE.toMethodDescriptorString(), false);
+    public static final MethodType DELEGATE_BOOTSTRAP_TYPE =
+        MethodType.methodType(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class, MethodHandle.class);
+    public static final Handle DELEGATE_BOOTSTRAP_HANDLE =
+        new Handle(Opcodes.H_INVOKESTATIC, Type.getInternalName(LambdaBootstrap.class),
+            "delegateBootstrap", DELEGATE_BOOTSTRAP_TYPE.toMethodDescriptorString(), false);
 
     /** dynamic invokedynamic bootstrap for indy string concats (Java 9+) */
     public static final Handle INDY_STRING_CONCAT_BOOTSTRAP_HANDLE;
@@ -144,7 +164,7 @@ public final class WriterConstants {
     public static final Type STRING_TYPE = Type.getType(String.class);
     public static final Type STRINGBUILDER_TYPE = Type.getType(StringBuilder.class);
 
-    public static final Method STRINGBUILDER_CONSTRUCTOR    = getAsmMethod(void.class, "<init>");
+    public static final Method STRINGBUILDER_CONSTRUCTOR    = getAsmMethod(void.class, CTOR_METHOD_NAME);
     public static final Method STRINGBUILDER_APPEND_BOOLEAN = getAsmMethod(StringBuilder.class, "append", boolean.class);
     public static final Method STRINGBUILDER_APPEND_CHAR    = getAsmMethod(StringBuilder.class, "append", char.class);
     public static final Method STRINGBUILDER_APPEND_INT     = getAsmMethod(StringBuilder.class, "append", int.class);
@@ -158,6 +178,9 @@ public final class WriterConstants {
     public static final Type OBJECTS_TYPE = Type.getType(Objects.class);
     public static final Method EQUALS = getAsmMethod(boolean.class, "equals", Object.class, Object.class);
 
+    public static final Type COLLECTION_TYPE = Type.getType(Collection.class);
+    public static final Method COLLECTION_SIZE = getAsmMethod(int.class, "size");
+
     private static Method getAsmMethod(final Class<?> rtype, final String name, final Class<?>... ptypes) {
         return new Method(name, MethodType.methodType(rtype, ptypes).toMethodDescriptorString());
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/EnhancedPainlessLexer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/EnhancedPainlessLexer.java
index 94a2c25897474..506ac8fcdecdb 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/EnhancedPainlessLexer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/EnhancedPainlessLexer.java
@@ -26,25 +26,31 @@
 import org.antlr.v4.runtime.TokenSource;
 import org.antlr.v4.runtime.misc.Interval;
 import org.antlr.v4.runtime.misc.Pair;
+import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Location;
 
 /**
- * A lexer that is customized for painless. It will:
+ * A lexer that is customized for painless. It:
  * <ul>
- * <li>will override the default error behavior to fail on the first error
- * <li>store the last token in case we need to do lookbehind for semicolon insertion and regex vs division detection
- * <li>insert semicolons where they'd improve the language's readability. Rather than hack this into the parser and create a ton of
+ * <li>Overrides the default error behavior to fail on the first error.
+ * <li>Stores the last token in case we need to do lookbehind for semicolon insertion and regex vs division detection.
+ * <li>Implements the regex vs division detection.
+ * <li>Insert semicolons where they'd improve the language's readability. Rather than hack this into the parser and create a ton of
  * ambiguity we hack them here where we can use heuristics to do it quickly.
+ * <li>Enhances the error message when a string contains invalid escape sequences to include a list of valid escape sequences.
  * </ul>
  */
 final class EnhancedPainlessLexer extends PainlessLexer {
-    final String sourceName;
+    private final String sourceName;
+    private final Definition definition;
+
     private Token stashedNext = null;
     private Token previous = null;
 
-    EnhancedPainlessLexer(CharStream charStream, String sourceName) {
+    EnhancedPainlessLexer(CharStream charStream, String sourceName, Definition definition) {
         super(charStream);
         this.sourceName = sourceName;
+        this.definition = definition;
     }
 
     public Token getPreviousToken() {
@@ -77,7 +83,42 @@ public void recover(final LexerNoViableAltException lnvae) {
         final String text = charStream.getText(Interval.of(startIndex, charStream.index()));
 
         Location location = new Location(sourceName, _tokenStartCharIndex);
-        throw location.createError(new IllegalArgumentException("unexpected character [" + getErrorDisplay(text) + "].", lnvae));
+        String message = "unexpected character [" + getErrorDisplay(text) + "].";
+        char firstChar = text.charAt(0);
+        if ((firstChar == '\'' || firstChar == '"') && text.length() - 2 > 0 && text.charAt(text.length() - 2) == '\\') {
+            /* Use a simple heuristic to guess if the unrecognized characters were trying to be a string but has a broken escape sequence.
+             * If it was add an extra message about valid string escape sequences. */
+            message += " The only valid escape sequences in strings starting with [" + firstChar + "] are [\\\\] and [\\"
+                    + firstChar + "].";
+        }
+        throw location.createError(new IllegalArgumentException(message, lnvae));
+    }
+
+    @Override
+    protected boolean isSimpleType(String name) {
+        return definition.isSimpleType(name);
+    }
+
+    @Override
+    protected boolean slashIsRegex() {
+        Token lastToken = getPreviousToken();
+        if (lastToken == null) {
+            return true;
+        }
+        switch (lastToken.getType()) {
+        case PainlessLexer.RBRACE:
+        case PainlessLexer.RP:
+        case PainlessLexer.OCTAL:
+        case PainlessLexer.HEX:
+        case PainlessLexer.INTEGER:
+        case PainlessLexer.DECIMAL:
+        case PainlessLexer.ID:
+        case PainlessLexer.DOTINTEGER:
+        case PainlessLexer.DOTID:
+            return false;
+        default:
+            return true;
+        }
     }
 
     private static boolean insertSemicolon(Token previous, Token next) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java
index eae7150b9be19..fd32c59b4ff03 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java
@@ -1,8 +1,5 @@
 // ANTLR GENERATED CODE: DO NOT EDIT
 package org.elasticsearch.painless.antlr;
-
-import org.elasticsearch.painless.Definition;
-
 import org.antlr.v4.runtime.Lexer;
 import org.antlr.v4.runtime.CharStream;
 import org.antlr.v4.runtime.Token;
@@ -13,7 +10,7 @@
 import org.antlr.v4.runtime.misc.*;
 
 @SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast"})
-class PainlessLexer extends Lexer {
+abstract class PainlessLexer extends Lexer {
   static { RuntimeMetaData.checkVersion("4.5.1", RuntimeMetaData.VERSION); }
 
   protected static final DFA[] _decisionToDFA;
@@ -21,16 +18,16 @@ class PainlessLexer extends Lexer {
     new PredictionContextCache();
   public static final int
     WS=1, COMMENT=2, LBRACK=3, RBRACK=4, LBRACE=5, RBRACE=6, LP=7, RP=8, DOT=9, 
-    COMMA=10, SEMICOLON=11, IF=12, IN=13, ELSE=14, WHILE=15, DO=16, FOR=17, 
-    CONTINUE=18, BREAK=19, RETURN=20, NEW=21, TRY=22, CATCH=23, THROW=24, 
-    THIS=25, INSTANCEOF=26, BOOLNOT=27, BWNOT=28, MUL=29, DIV=30, REM=31, 
-    ADD=32, SUB=33, LSH=34, RSH=35, USH=36, LT=37, LTE=38, GT=39, GTE=40, 
-    EQ=41, EQR=42, NE=43, NER=44, BWAND=45, XOR=46, BWOR=47, BOOLAND=48, BOOLOR=49, 
-    COND=50, COLON=51, REF=52, ARROW=53, FIND=54, MATCH=55, INCR=56, DECR=57, 
-    ASSIGN=58, AADD=59, ASUB=60, AMUL=61, ADIV=62, AREM=63, AAND=64, AXOR=65, 
-    AOR=66, ALSH=67, ARSH=68, AUSH=69, OCTAL=70, HEX=71, INTEGER=72, DECIMAL=73, 
-    STRING=74, REGEX=75, TRUE=76, FALSE=77, NULL=78, TYPE=79, ID=80, DOTINTEGER=81, 
-    DOTID=82;
+    NSDOT=10, COMMA=11, SEMICOLON=12, IF=13, IN=14, ELSE=15, WHILE=16, DO=17, 
+    FOR=18, CONTINUE=19, BREAK=20, RETURN=21, NEW=22, TRY=23, CATCH=24, THROW=25, 
+    THIS=26, INSTANCEOF=27, BOOLNOT=28, BWNOT=29, MUL=30, DIV=31, REM=32, 
+    ADD=33, SUB=34, LSH=35, RSH=36, USH=37, LT=38, LTE=39, GT=40, GTE=41, 
+    EQ=42, EQR=43, NE=44, NER=45, BWAND=46, XOR=47, BWOR=48, BOOLAND=49, BOOLOR=50, 
+    COND=51, COLON=52, ELVIS=53, REF=54, ARROW=55, FIND=56, MATCH=57, INCR=58, 
+    DECR=59, ASSIGN=60, AADD=61, ASUB=62, AMUL=63, ADIV=64, AREM=65, AAND=66, 
+    AXOR=67, AOR=68, ALSH=69, ARSH=70, AUSH=71, OCTAL=72, HEX=73, INTEGER=74, 
+    DECIMAL=75, STRING=76, REGEX=77, TRUE=78, FALSE=79, NULL=80, TYPE=81, 
+    ID=82, DOTINTEGER=83, DOTID=84;
   public static final int AFTER_DOT = 1;
   public static String[] modeNames = {
     "DEFAULT_MODE", "AFTER_DOT"
@@ -38,39 +35,39 @@ class PainlessLexer extends Lexer {
 
   public static final String[] ruleNames = {
     "WS", "COMMENT", "LBRACK", "RBRACK", "LBRACE", "RBRACE", "LP", "RP", "DOT", 
-    "COMMA", "SEMICOLON", "IF", "IN", "ELSE", "WHILE", "DO", "FOR", "CONTINUE", 
-    "BREAK", "RETURN", "NEW", "TRY", "CATCH", "THROW", "THIS", "INSTANCEOF", 
-    "BOOLNOT", "BWNOT", "MUL", "DIV", "REM", "ADD", "SUB", "LSH", "RSH", "USH", 
-    "LT", "LTE", "GT", "GTE", "EQ", "EQR", "NE", "NER", "BWAND", "XOR", "BWOR", 
-    "BOOLAND", "BOOLOR", "COND", "COLON", "REF", "ARROW", "FIND", "MATCH", 
-    "INCR", "DECR", "ASSIGN", "AADD", "ASUB", "AMUL", "ADIV", "AREM", "AAND", 
-    "AXOR", "AOR", "ALSH", "ARSH", "AUSH", "OCTAL", "HEX", "INTEGER", "DECIMAL", 
-    "STRING", "REGEX", "TRUE", "FALSE", "NULL", "TYPE", "ID", "DOTINTEGER", 
-    "DOTID"
+    "NSDOT", "COMMA", "SEMICOLON", "IF", "IN", "ELSE", "WHILE", "DO", "FOR", 
+    "CONTINUE", "BREAK", "RETURN", "NEW", "TRY", "CATCH", "THROW", "THIS", 
+    "INSTANCEOF", "BOOLNOT", "BWNOT", "MUL", "DIV", "REM", "ADD", "SUB", "LSH", 
+    "RSH", "USH", "LT", "LTE", "GT", "GTE", "EQ", "EQR", "NE", "NER", "BWAND", 
+    "XOR", "BWOR", "BOOLAND", "BOOLOR", "COND", "COLON", "ELVIS", "REF", "ARROW", 
+    "FIND", "MATCH", "INCR", "DECR", "ASSIGN", "AADD", "ASUB", "AMUL", "ADIV", 
+    "AREM", "AAND", "AXOR", "AOR", "ALSH", "ARSH", "AUSH", "OCTAL", "HEX", 
+    "INTEGER", "DECIMAL", "STRING", "REGEX", "TRUE", "FALSE", "NULL", "TYPE", 
+    "ID", "DOTINTEGER", "DOTID"
   };
 
   private static final String[] _LITERAL_NAMES = {
-    null, null, null, "'{'", "'}'", "'['", "']'", "'('", "')'", "'.'", "','", 
-    "';'", "'if'", "'in'", "'else'", "'while'", "'do'", "'for'", "'continue'", 
+    null, null, null, "'{'", "'}'", "'['", "']'", "'('", "')'", "'.'", "'?.'", 
+    "','", "';'", "'if'", "'in'", "'else'", "'while'", "'do'", "'for'", "'continue'", 
     "'break'", "'return'", "'new'", "'try'", "'catch'", "'throw'", "'this'", 
     "'instanceof'", "'!'", "'~'", "'*'", "'/'", "'%'", "'+'", "'-'", "'<<'", 
     "'>>'", "'>>>'", "'<'", "'<='", "'>'", "'>='", "'=='", "'==='", "'!='", 
-    "'!=='", "'&'", "'^'", "'|'", "'&&'", "'||'", "'?'", "':'", "'::'", "'->'", 
-    "'=~'", "'==~'", "'++'", "'--'", "'='", "'+='", "'-='", "'*='", "'/='", 
-    "'%='", "'&='", "'^='", "'|='", "'<<='", "'>>='", "'>>>='", null, null, 
-    null, null, null, null, "'true'", "'false'", "'null'"
+    "'!=='", "'&'", "'^'", "'|'", "'&&'", "'||'", "'?'", "':'", "'?:'", "'::'", 
+    "'->'", "'=~'", "'==~'", "'++'", "'--'", "'='", "'+='", "'-='", "'*='", 
+    "'/='", "'%='", "'&='", "'^='", "'|='", "'<<='", "'>>='", "'>>>='", null, 
+    null, null, null, null, null, "'true'", "'false'", "'null'"
   };
   private static final String[] _SYMBOLIC_NAMES = {
     null, "WS", "COMMENT", "LBRACK", "RBRACK", "LBRACE", "RBRACE", "LP", "RP", 
-    "DOT", "COMMA", "SEMICOLON", "IF", "IN", "ELSE", "WHILE", "DO", "FOR", 
-    "CONTINUE", "BREAK", "RETURN", "NEW", "TRY", "CATCH", "THROW", "THIS", 
-    "INSTANCEOF", "BOOLNOT", "BWNOT", "MUL", "DIV", "REM", "ADD", "SUB", "LSH", 
-    "RSH", "USH", "LT", "LTE", "GT", "GTE", "EQ", "EQR", "NE", "NER", "BWAND", 
-    "XOR", "BWOR", "BOOLAND", "BOOLOR", "COND", "COLON", "REF", "ARROW", "FIND", 
-    "MATCH", "INCR", "DECR", "ASSIGN", "AADD", "ASUB", "AMUL", "ADIV", "AREM", 
-    "AAND", "AXOR", "AOR", "ALSH", "ARSH", "AUSH", "OCTAL", "HEX", "INTEGER", 
-    "DECIMAL", "STRING", "REGEX", "TRUE", "FALSE", "NULL", "TYPE", "ID", "DOTINTEGER", 
-    "DOTID"
+    "DOT", "NSDOT", "COMMA", "SEMICOLON", "IF", "IN", "ELSE", "WHILE", "DO", 
+    "FOR", "CONTINUE", "BREAK", "RETURN", "NEW", "TRY", "CATCH", "THROW", 
+    "THIS", "INSTANCEOF", "BOOLNOT", "BWNOT", "MUL", "DIV", "REM", "ADD", 
+    "SUB", "LSH", "RSH", "USH", "LT", "LTE", "GT", "GTE", "EQ", "EQR", "NE", 
+    "NER", "BWAND", "XOR", "BWOR", "BOOLAND", "BOOLOR", "COND", "COLON", "ELVIS", 
+    "REF", "ARROW", "FIND", "MATCH", "INCR", "DECR", "ASSIGN", "AADD", "ASUB", 
+    "AMUL", "ADIV", "AREM", "AAND", "AXOR", "AOR", "ALSH", "ARSH", "AUSH", 
+    "OCTAL", "HEX", "INTEGER", "DECIMAL", "STRING", "REGEX", "TRUE", "FALSE", 
+    "NULL", "TYPE", "ID", "DOTINTEGER", "DOTID"
   };
   public static final Vocabulary VOCABULARY = new VocabularyImpl(_LITERAL_NAMES, _SYMBOLIC_NAMES);
 
@@ -106,6 +103,21 @@ public Vocabulary getVocabulary() {
   }
 
 
+  /**
+   * Check against the current whitelist to determine whether a token is a type
+   * or not. Called by the {@code TYPE} token defined in {@code PainlessLexer.g4}.
+   * See also
+   * <a href="https://en.wikipedia.org/wiki/The_lexer_hack">The lexer hack</a>.
+   */
+  protected abstract boolean isSimpleType(String name);
+
+  /**
+   * Is the preceding {@code /} a the beginning of a regex (true) or a division
+   * (false).
+   */
+  protected abstract boolean slashIsRegex();
+
+
   public PainlessLexer(CharStream input) {
     super(input);
     _interp = new LexerATNSimulator(this,_ATN,_decisionToDFA,_sharedContextCache);
@@ -129,11 +141,11 @@ public PainlessLexer(CharStream input) {
   @Override
   public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) {
     switch (ruleIndex) {
-    case 29:
+    case 30:
       return DIV_sempred((RuleContext)_localctx, predIndex);
-    case 74:
+    case 76:
       return REGEX_sempred((RuleContext)_localctx, predIndex);
-    case 78:
+    case 80:
       return TYPE_sempred((RuleContext)_localctx, predIndex);
     }
     return true;
@@ -141,27 +153,27 @@ public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) {
   private boolean DIV_sempred(RuleContext _localctx, int predIndex) {
     switch (predIndex) {
     case 0:
-      return  false == SlashStrategy.slashIsRegex(this) ;
+      return  false == slashIsRegex() ;
     }
     return true;
   }
   private boolean REGEX_sempred(RuleContext _localctx, int predIndex) {
     switch (predIndex) {
     case 1:
-      return  SlashStrategy.slashIsRegex(this) ;
+      return  slashIsRegex() ;
     }
     return true;
   }
   private boolean TYPE_sempred(RuleContext _localctx, int predIndex) {
     switch (predIndex) {
     case 2:
-      return  Definition.isSimpleType(getText()) ;
+      return  isSimpleType(getText()) ;
     }
     return true;
   }
 
   public static final String _serializedATN =
-    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\2T\u024b\b\1\b\1\4"+
+    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\2V\u0257\b\1\b\1\4"+
     "\2\t\2\4\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n"+
     "\4\13\t\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22"+
     "\t\22\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\4\31"+
@@ -170,202 +182,206 @@ private boolean TYPE_sempred(RuleContext _localctx, int predIndex) {
     "+\4,\t,\4-\t-\4.\t.\4/\t/\4\60\t\60\4\61\t\61\4\62\t\62\4\63\t\63\4\64"+
     "\t\64\4\65\t\65\4\66\t\66\4\67\t\67\48\t8\49\t9\4:\t:\4;\t;\4<\t<\4=\t"+
     "=\4>\t>\4?\t?\4@\t@\4A\tA\4B\tB\4C\tC\4D\tD\4E\tE\4F\tF\4G\tG\4H\tH\4"+
-    "I\tI\4J\tJ\4K\tK\4L\tL\4M\tM\4N\tN\4O\tO\4P\tP\4Q\tQ\4R\tR\4S\tS\3\2\6"+
-    "\2\u00aa\n\2\r\2\16\2\u00ab\3\2\3\2\3\3\3\3\3\3\3\3\7\3\u00b4\n\3\f\3"+
-    "\16\3\u00b7\13\3\3\3\3\3\3\3\3\3\3\3\7\3\u00be\n\3\f\3\16\3\u00c1\13\3"+
-    "\3\3\3\3\5\3\u00c5\n\3\3\3\3\3\3\4\3\4\3\5\3\5\3\6\3\6\3\7\3\7\3\b\3\b"+
-    "\3\t\3\t\3\n\3\n\3\n\3\n\3\13\3\13\3\f\3\f\3\r\3\r\3\r\3\16\3\16\3\16"+
-    "\3\17\3\17\3\17\3\17\3\17\3\20\3\20\3\20\3\20\3\20\3\20\3\21\3\21\3\21"+
-    "\3\22\3\22\3\22\3\22\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\24"+
-    "\3\24\3\24\3\24\3\24\3\24\3\25\3\25\3\25\3\25\3\25\3\25\3\25\3\26\3\26"+
-    "\3\26\3\26\3\27\3\27\3\27\3\27\3\30\3\30\3\30\3\30\3\30\3\30\3\31\3\31"+
-    "\3\31\3\31\3\31\3\31\3\32\3\32\3\32\3\32\3\32\3\33\3\33\3\33\3\33\3\33"+
-    "\3\33\3\33\3\33\3\33\3\33\3\33\3\34\3\34\3\35\3\35\3\36\3\36\3\37\3\37"+
-    "\3\37\3 \3 \3!\3!\3\"\3\"\3#\3#\3#\3$\3$\3$\3%\3%\3%\3%\3&\3&\3\'\3\'"+
-    "\3\'\3(\3(\3)\3)\3)\3*\3*\3*\3+\3+\3+\3+\3,\3,\3,\3-\3-\3-\3-\3.\3.\3"+
-    "/\3/\3\60\3\60\3\61\3\61\3\61\3\62\3\62\3\62\3\63\3\63\3\64\3\64\3\65"+
-    "\3\65\3\65\3\66\3\66\3\66\3\67\3\67\3\67\38\38\38\38\39\39\39\3:\3:\3"+
-    ":\3;\3;\3<\3<\3<\3=\3=\3=\3>\3>\3>\3?\3?\3?\3@\3@\3@\3A\3A\3A\3B\3B\3"+
-    "B\3C\3C\3C\3D\3D\3D\3D\3E\3E\3E\3E\3F\3F\3F\3F\3F\3G\3G\6G\u01ac\nG\r"+
-    "G\16G\u01ad\3G\5G\u01b1\nG\3H\3H\3H\6H\u01b6\nH\rH\16H\u01b7\3H\5H\u01bb"+
-    "\nH\3I\3I\3I\7I\u01c0\nI\fI\16I\u01c3\13I\5I\u01c5\nI\3I\5I\u01c8\nI\3"+
-    "J\3J\3J\7J\u01cd\nJ\fJ\16J\u01d0\13J\5J\u01d2\nJ\3J\3J\6J\u01d6\nJ\rJ"+
-    "\16J\u01d7\5J\u01da\nJ\3J\3J\5J\u01de\nJ\3J\6J\u01e1\nJ\rJ\16J\u01e2\5"+
-    "J\u01e5\nJ\3J\5J\u01e8\nJ\3K\3K\3K\3K\3K\3K\7K\u01f0\nK\fK\16K\u01f3\13"+
-    "K\3K\3K\3K\3K\3K\3K\3K\7K\u01fc\nK\fK\16K\u01ff\13K\3K\5K\u0202\nK\3L"+
-    "\3L\3L\3L\6L\u0208\nL\rL\16L\u0209\3L\3L\7L\u020e\nL\fL\16L\u0211\13L"+
-    "\3L\3L\3M\3M\3M\3M\3M\3N\3N\3N\3N\3N\3N\3O\3O\3O\3O\3O\3P\3P\3P\3P\7P"+
-    "\u0229\nP\fP\16P\u022c\13P\3P\3P\3Q\3Q\7Q\u0232\nQ\fQ\16Q\u0235\13Q\3"+
-    "R\3R\3R\7R\u023a\nR\fR\16R\u023d\13R\5R\u023f\nR\3R\3R\3S\3S\7S\u0245"+
-    "\nS\fS\16S\u0248\13S\3S\3S\6\u00b5\u00bf\u01f1\u01fd\2T\4\3\6\4\b\5\n"+
-    "\6\f\7\16\b\20\t\22\n\24\13\26\f\30\r\32\16\34\17\36\20 \21\"\22$\23&"+
-    "\24(\25*\26,\27.\30\60\31\62\32\64\33\66\348\35:\36<\37> @!B\"D#F$H%J"+
-    "&L\'N(P)R*T+V,X-Z.\\/^\60`\61b\62d\63f\64h\65j\66l\67n8p9r:t;v<x=z>|?"+
-    "~@\u0080A\u0082B\u0084C\u0086D\u0088E\u008aF\u008cG\u008eH\u0090I\u0092"+
-    "J\u0094K\u0096L\u0098M\u009aN\u009cO\u009eP\u00a0Q\u00a2R\u00a4S\u00a6"+
-    "T\4\2\3\24\5\2\13\f\17\17\"\"\4\2\f\f\17\17\3\2\629\4\2NNnn\4\2ZZzz\5"+
-    "\2\62;CHch\3\2\63;\3\2\62;\b\2FFHHNNffhhnn\4\2GGgg\4\2--//\4\2HHhh\4\2"+
-    "$$^^\4\2\f\f\61\61\3\2\f\f\t\2WWeekknouuwwzz\5\2C\\aac|\6\2\62;C\\aac"+
-    "|\u026b\2\4\3\2\2\2\2\6\3\2\2\2\2\b\3\2\2\2\2\n\3\2\2\2\2\f\3\2\2\2\2"+
-    "\16\3\2\2\2\2\20\3\2\2\2\2\22\3\2\2\2\2\24\3\2\2\2\2\26\3\2\2\2\2\30\3"+
-    "\2\2\2\2\32\3\2\2\2\2\34\3\2\2\2\2\36\3\2\2\2\2 \3\2\2\2\2\"\3\2\2\2\2"+
-    "$\3\2\2\2\2&\3\2\2\2\2(\3\2\2\2\2*\3\2\2\2\2,\3\2\2\2\2.\3\2\2\2\2\60"+
-    "\3\2\2\2\2\62\3\2\2\2\2\64\3\2\2\2\2\66\3\2\2\2\28\3\2\2\2\2:\3\2\2\2"+
-    "\2<\3\2\2\2\2>\3\2\2\2\2@\3\2\2\2\2B\3\2\2\2\2D\3\2\2\2\2F\3\2\2\2\2H"+
-    "\3\2\2\2\2J\3\2\2\2\2L\3\2\2\2\2N\3\2\2\2\2P\3\2\2\2\2R\3\2\2\2\2T\3\2"+
-    "\2\2\2V\3\2\2\2\2X\3\2\2\2\2Z\3\2\2\2\2\\\3\2\2\2\2^\3\2\2\2\2`\3\2\2"+
-    "\2\2b\3\2\2\2\2d\3\2\2\2\2f\3\2\2\2\2h\3\2\2\2\2j\3\2\2\2\2l\3\2\2\2\2"+
-    "n\3\2\2\2\2p\3\2\2\2\2r\3\2\2\2\2t\3\2\2\2\2v\3\2\2\2\2x\3\2\2\2\2z\3"+
-    "\2\2\2\2|\3\2\2\2\2~\3\2\2\2\2\u0080\3\2\2\2\2\u0082\3\2\2\2\2\u0084\3"+
-    "\2\2\2\2\u0086\3\2\2\2\2\u0088\3\2\2\2\2\u008a\3\2\2\2\2\u008c\3\2\2\2"+
-    "\2\u008e\3\2\2\2\2\u0090\3\2\2\2\2\u0092\3\2\2\2\2\u0094\3\2\2\2\2\u0096"+
-    "\3\2\2\2\2\u0098\3\2\2\2\2\u009a\3\2\2\2\2\u009c\3\2\2\2\2\u009e\3\2\2"+
-    "\2\2\u00a0\3\2\2\2\2\u00a2\3\2\2\2\3\u00a4\3\2\2\2\3\u00a6\3\2\2\2\4\u00a9"+
-    "\3\2\2\2\6\u00c4\3\2\2\2\b\u00c8\3\2\2\2\n\u00ca\3\2\2\2\f\u00cc\3\2\2"+
-    "\2\16\u00ce\3\2\2\2\20\u00d0\3\2\2\2\22\u00d2\3\2\2\2\24\u00d4\3\2\2\2"+
-    "\26\u00d8\3\2\2\2\30\u00da\3\2\2\2\32\u00dc\3\2\2\2\34\u00df\3\2\2\2\36"+
-    "\u00e2\3\2\2\2 \u00e7\3\2\2\2\"\u00ed\3\2\2\2$\u00f0\3\2\2\2&\u00f4\3"+
-    "\2\2\2(\u00fd\3\2\2\2*\u0103\3\2\2\2,\u010a\3\2\2\2.\u010e\3\2\2\2\60"+
-    "\u0112\3\2\2\2\62\u0118\3\2\2\2\64\u011e\3\2\2\2\66\u0123\3\2\2\28\u012e"+
-    "\3\2\2\2:\u0130\3\2\2\2<\u0132\3\2\2\2>\u0134\3\2\2\2@\u0137\3\2\2\2B"+
-    "\u0139\3\2\2\2D\u013b\3\2\2\2F\u013d\3\2\2\2H\u0140\3\2\2\2J\u0143\3\2"+
-    "\2\2L\u0147\3\2\2\2N\u0149\3\2\2\2P\u014c\3\2\2\2R\u014e\3\2\2\2T\u0151"+
-    "\3\2\2\2V\u0154\3\2\2\2X\u0158\3\2\2\2Z\u015b\3\2\2\2\\\u015f\3\2\2\2"+
-    "^\u0161\3\2\2\2`\u0163\3\2\2\2b\u0165\3\2\2\2d\u0168\3\2\2\2f\u016b\3"+
-    "\2\2\2h\u016d\3\2\2\2j\u016f\3\2\2\2l\u0172\3\2\2\2n\u0175\3\2\2\2p\u0178"+
-    "\3\2\2\2r\u017c\3\2\2\2t\u017f\3\2\2\2v\u0182\3\2\2\2x\u0184\3\2\2\2z"+
-    "\u0187\3\2\2\2|\u018a\3\2\2\2~\u018d\3\2\2\2\u0080\u0190\3\2\2\2\u0082"+
-    "\u0193\3\2\2\2\u0084\u0196\3\2\2\2\u0086\u0199\3\2\2\2\u0088\u019c\3\2"+
-    "\2\2\u008a\u01a0\3\2\2\2\u008c\u01a4\3\2\2\2\u008e\u01a9\3\2\2\2\u0090"+
-    "\u01b2\3\2\2\2\u0092\u01c4\3\2\2\2\u0094\u01d1\3\2\2\2\u0096\u0201\3\2"+
-    "\2\2\u0098\u0203\3\2\2\2\u009a\u0214\3\2\2\2\u009c\u0219\3\2\2\2\u009e"+
-    "\u021f\3\2\2\2\u00a0\u0224\3\2\2\2\u00a2\u022f\3\2\2\2\u00a4\u023e\3\2"+
-    "\2\2\u00a6\u0242\3\2\2\2\u00a8\u00aa\t\2\2\2\u00a9\u00a8\3\2\2\2\u00aa"+
-    "\u00ab\3\2\2\2\u00ab\u00a9\3\2\2\2\u00ab\u00ac\3\2\2\2\u00ac\u00ad\3\2"+
-    "\2\2\u00ad\u00ae\b\2\2\2\u00ae\5\3\2\2\2\u00af\u00b0\7\61\2\2\u00b0\u00b1"+
-    "\7\61\2\2\u00b1\u00b5\3\2\2\2\u00b2\u00b4\13\2\2\2\u00b3\u00b2\3\2\2\2"+
-    "\u00b4\u00b7\3\2\2\2\u00b5\u00b6\3\2\2\2\u00b5\u00b3\3\2\2\2\u00b6\u00b8"+
-    "\3\2\2\2\u00b7\u00b5\3\2\2\2\u00b8\u00c5\t\3\2\2\u00b9\u00ba\7\61\2\2"+
-    "\u00ba\u00bb\7,\2\2\u00bb\u00bf\3\2\2\2\u00bc\u00be\13\2\2\2\u00bd\u00bc"+
-    "\3\2\2\2\u00be\u00c1\3\2\2\2\u00bf\u00c0\3\2\2\2\u00bf\u00bd\3\2\2\2\u00c0"+
-    "\u00c2\3\2\2\2\u00c1\u00bf\3\2\2\2\u00c2\u00c3\7,\2\2\u00c3\u00c5\7\61"+
-    "\2\2\u00c4\u00af\3\2\2\2\u00c4\u00b9\3\2\2\2\u00c5\u00c6\3\2\2\2\u00c6"+
-    "\u00c7\b\3\2\2\u00c7\7\3\2\2\2\u00c8\u00c9\7}\2\2\u00c9\t\3\2\2\2\u00ca"+
-    "\u00cb\7\177\2\2\u00cb\13\3\2\2\2\u00cc\u00cd\7]\2\2\u00cd\r\3\2\2\2\u00ce"+
-    "\u00cf\7_\2\2\u00cf\17\3\2\2\2\u00d0\u00d1\7*\2\2\u00d1\21\3\2\2\2\u00d2"+
-    "\u00d3\7+\2\2\u00d3\23\3\2\2\2\u00d4\u00d5\7\60\2\2\u00d5\u00d6\3\2\2"+
-    "\2\u00d6\u00d7\b\n\3\2\u00d7\25\3\2\2\2\u00d8\u00d9\7.\2\2\u00d9\27\3"+
-    "\2\2\2\u00da\u00db\7=\2\2\u00db\31\3\2\2\2\u00dc\u00dd\7k\2\2\u00dd\u00de"+
-    "\7h\2\2\u00de\33\3\2\2\2\u00df\u00e0\7k\2\2\u00e0\u00e1\7p\2\2\u00e1\35"+
-    "\3\2\2\2\u00e2\u00e3\7g\2\2\u00e3\u00e4\7n\2\2\u00e4\u00e5\7u\2\2\u00e5"+
-    "\u00e6\7g\2\2\u00e6\37\3\2\2\2\u00e7\u00e8\7y\2\2\u00e8\u00e9\7j\2\2\u00e9"+
-    "\u00ea\7k\2\2\u00ea\u00eb\7n\2\2\u00eb\u00ec\7g\2\2\u00ec!\3\2\2\2\u00ed"+
-    "\u00ee\7f\2\2\u00ee\u00ef\7q\2\2\u00ef#\3\2\2\2\u00f0\u00f1\7h\2\2\u00f1"+
-    "\u00f2\7q\2\2\u00f2\u00f3\7t\2\2\u00f3%\3\2\2\2\u00f4\u00f5\7e\2\2\u00f5"+
-    "\u00f6\7q\2\2\u00f6\u00f7\7p\2\2\u00f7\u00f8\7v\2\2\u00f8\u00f9\7k\2\2"+
-    "\u00f9\u00fa\7p\2\2\u00fa\u00fb\7w\2\2\u00fb\u00fc\7g\2\2\u00fc\'\3\2"+
-    "\2\2\u00fd\u00fe\7d\2\2\u00fe\u00ff\7t\2\2\u00ff\u0100\7g\2\2\u0100\u0101"+
-    "\7c\2\2\u0101\u0102\7m\2\2\u0102)\3\2\2\2\u0103\u0104\7t\2\2\u0104\u0105"+
-    "\7g\2\2\u0105\u0106\7v\2\2\u0106\u0107\7w\2\2\u0107\u0108\7t\2\2\u0108"+
-    "\u0109\7p\2\2\u0109+\3\2\2\2\u010a\u010b\7p\2\2\u010b\u010c\7g\2\2\u010c"+
-    "\u010d\7y\2\2\u010d-\3\2\2\2\u010e\u010f\7v\2\2\u010f\u0110\7t\2\2\u0110"+
-    "\u0111\7{\2\2\u0111/\3\2\2\2\u0112\u0113\7e\2\2\u0113\u0114\7c\2\2\u0114"+
-    "\u0115\7v\2\2\u0115\u0116\7e\2\2\u0116\u0117\7j\2\2\u0117\61\3\2\2\2\u0118"+
-    "\u0119\7v\2\2\u0119\u011a\7j\2\2\u011a\u011b\7t\2\2\u011b\u011c\7q\2\2"+
-    "\u011c\u011d\7y\2\2\u011d\63\3\2\2\2\u011e\u011f\7v\2\2\u011f\u0120\7"+
-    "j\2\2\u0120\u0121\7k\2\2\u0121\u0122\7u\2\2\u0122\65\3\2\2\2\u0123\u0124"+
-    "\7k\2\2\u0124\u0125\7p\2\2\u0125\u0126\7u\2\2\u0126\u0127\7v\2\2\u0127"+
-    "\u0128\7c\2\2\u0128\u0129\7p\2\2\u0129\u012a\7e\2\2\u012a\u012b\7g\2\2"+
-    "\u012b\u012c\7q\2\2\u012c\u012d\7h\2\2\u012d\67\3\2\2\2\u012e\u012f\7"+
-    "#\2\2\u012f9\3\2\2\2\u0130\u0131\7\u0080\2\2\u0131;\3\2\2\2\u0132\u0133"+
-    "\7,\2\2\u0133=\3\2\2\2\u0134\u0135\7\61\2\2\u0135\u0136\6\37\2\2\u0136"+
-    "?\3\2\2\2\u0137\u0138\7\'\2\2\u0138A\3\2\2\2\u0139\u013a\7-\2\2\u013a"+
-    "C\3\2\2\2\u013b\u013c\7/\2\2\u013cE\3\2\2\2\u013d\u013e\7>\2\2\u013e\u013f"+
-    "\7>\2\2\u013fG\3\2\2\2\u0140\u0141\7@\2\2\u0141\u0142\7@\2\2\u0142I\3"+
-    "\2\2\2\u0143\u0144\7@\2\2\u0144\u0145\7@\2\2\u0145\u0146\7@\2\2\u0146"+
-    "K\3\2\2\2\u0147\u0148\7>\2\2\u0148M\3\2\2\2\u0149\u014a\7>\2\2\u014a\u014b"+
-    "\7?\2\2\u014bO\3\2\2\2\u014c\u014d\7@\2\2\u014dQ\3\2\2\2\u014e\u014f\7"+
-    "@\2\2\u014f\u0150\7?\2\2\u0150S\3\2\2\2\u0151\u0152\7?\2\2\u0152\u0153"+
-    "\7?\2\2\u0153U\3\2\2\2\u0154\u0155\7?\2\2\u0155\u0156\7?\2\2\u0156\u0157"+
-    "\7?\2\2\u0157W\3\2\2\2\u0158\u0159\7#\2\2\u0159\u015a\7?\2\2\u015aY\3"+
-    "\2\2\2\u015b\u015c\7#\2\2\u015c\u015d\7?\2\2\u015d\u015e\7?\2\2\u015e"+
-    "[\3\2\2\2\u015f\u0160\7(\2\2\u0160]\3\2\2\2\u0161\u0162\7`\2\2\u0162_"+
-    "\3\2\2\2\u0163\u0164\7~\2\2\u0164a\3\2\2\2\u0165\u0166\7(\2\2\u0166\u0167"+
-    "\7(\2\2\u0167c\3\2\2\2\u0168\u0169\7~\2\2\u0169\u016a\7~\2\2\u016ae\3"+
-    "\2\2\2\u016b\u016c\7A\2\2\u016cg\3\2\2\2\u016d\u016e\7<\2\2\u016ei\3\2"+
-    "\2\2\u016f\u0170\7<\2\2\u0170\u0171\7<\2\2\u0171k\3\2\2\2\u0172\u0173"+
-    "\7/\2\2\u0173\u0174\7@\2\2\u0174m\3\2\2\2\u0175\u0176\7?\2\2\u0176\u0177"+
-    "\7\u0080\2\2\u0177o\3\2\2\2\u0178\u0179\7?\2\2\u0179\u017a\7?\2\2\u017a"+
-    "\u017b\7\u0080\2\2\u017bq\3\2\2\2\u017c\u017d\7-\2\2\u017d\u017e\7-\2"+
-    "\2\u017es\3\2\2\2\u017f\u0180\7/\2\2\u0180\u0181\7/\2\2\u0181u\3\2\2\2"+
-    "\u0182\u0183\7?\2\2\u0183w\3\2\2\2\u0184\u0185\7-\2\2\u0185\u0186\7?\2"+
-    "\2\u0186y\3\2\2\2\u0187\u0188\7/\2\2\u0188\u0189\7?\2\2\u0189{\3\2\2\2"+
-    "\u018a\u018b\7,\2\2\u018b\u018c\7?\2\2\u018c}\3\2\2\2\u018d\u018e\7\61"+
-    "\2\2\u018e\u018f\7?\2\2\u018f\177\3\2\2\2\u0190\u0191\7\'\2\2\u0191\u0192"+
-    "\7?\2\2\u0192\u0081\3\2\2\2\u0193\u0194\7(\2\2\u0194\u0195\7?\2\2\u0195"+
-    "\u0083\3\2\2\2\u0196\u0197\7`\2\2\u0197\u0198\7?\2\2\u0198\u0085\3\2\2"+
-    "\2\u0199\u019a\7~\2\2\u019a\u019b\7?\2\2\u019b\u0087\3\2\2\2\u019c\u019d"+
-    "\7>\2\2\u019d\u019e\7>\2\2\u019e\u019f\7?\2\2\u019f\u0089\3\2\2\2\u01a0"+
-    "\u01a1\7@\2\2\u01a1\u01a2\7@\2\2\u01a2\u01a3\7?\2\2\u01a3\u008b\3\2\2"+
-    "\2\u01a4\u01a5\7@\2\2\u01a5\u01a6\7@\2\2\u01a6\u01a7\7@\2\2\u01a7\u01a8"+
-    "\7?\2\2\u01a8\u008d\3\2\2\2\u01a9\u01ab\7\62\2\2\u01aa\u01ac\t\4\2\2\u01ab"+
-    "\u01aa\3\2\2\2\u01ac\u01ad\3\2\2\2\u01ad\u01ab\3\2\2\2\u01ad\u01ae\3\2"+
-    "\2\2\u01ae\u01b0\3\2\2\2\u01af\u01b1\t\5\2\2\u01b0\u01af\3\2\2\2\u01b0"+
-    "\u01b1\3\2\2\2\u01b1\u008f\3\2\2\2\u01b2\u01b3\7\62\2\2\u01b3\u01b5\t"+
-    "\6\2\2\u01b4\u01b6\t\7\2\2\u01b5\u01b4\3\2\2\2\u01b6\u01b7\3\2\2\2\u01b7"+
-    "\u01b5\3\2\2\2\u01b7\u01b8\3\2\2\2\u01b8\u01ba\3\2\2\2\u01b9\u01bb\t\5"+
-    "\2\2\u01ba\u01b9\3\2\2\2\u01ba\u01bb\3\2\2\2\u01bb\u0091\3\2\2\2\u01bc"+
-    "\u01c5\7\62\2\2\u01bd\u01c1\t\b\2\2\u01be\u01c0\t\t\2\2\u01bf\u01be\3"+
-    "\2\2\2\u01c0\u01c3\3\2\2\2\u01c1\u01bf\3\2\2\2\u01c1\u01c2\3\2\2\2\u01c2"+
-    "\u01c5\3\2\2\2\u01c3\u01c1\3\2\2\2\u01c4\u01bc\3\2\2\2\u01c4\u01bd\3\2"+
-    "\2\2\u01c5\u01c7\3\2\2\2\u01c6\u01c8\t\n\2\2\u01c7\u01c6\3\2\2\2\u01c7"+
-    "\u01c8\3\2\2\2\u01c8\u0093\3\2\2\2\u01c9\u01d2\7\62\2\2\u01ca\u01ce\t"+
-    "\b\2\2\u01cb\u01cd\t\t\2\2\u01cc\u01cb\3\2\2\2\u01cd\u01d0\3\2\2\2\u01ce"+
-    "\u01cc\3\2\2\2\u01ce\u01cf\3\2\2\2\u01cf\u01d2\3\2\2\2\u01d0\u01ce\3\2"+
-    "\2\2\u01d1\u01c9\3\2\2\2\u01d1\u01ca\3\2\2\2\u01d2\u01d9\3\2\2\2\u01d3"+
-    "\u01d5\5\24\n\2\u01d4\u01d6\t\t\2\2\u01d5\u01d4\3\2\2\2\u01d6\u01d7\3"+
-    "\2\2\2\u01d7\u01d5\3\2\2\2\u01d7\u01d8\3\2\2\2\u01d8\u01da\3\2\2\2\u01d9"+
-    "\u01d3\3\2\2\2\u01d9\u01da\3\2\2\2\u01da\u01e4\3\2\2\2\u01db\u01dd\t\13"+
-    "\2\2\u01dc\u01de\t\f\2\2\u01dd\u01dc\3\2\2\2\u01dd\u01de\3\2\2\2\u01de"+
-    "\u01e0\3\2\2\2\u01df\u01e1\t\t\2\2\u01e0\u01df\3\2\2\2\u01e1\u01e2\3\2"+
-    "\2\2\u01e2\u01e0\3\2\2\2\u01e2\u01e3\3\2\2\2\u01e3\u01e5\3\2\2\2\u01e4"+
-    "\u01db\3\2\2\2\u01e4\u01e5\3\2\2\2\u01e5\u01e7\3\2\2\2\u01e6\u01e8\t\r"+
-    "\2\2\u01e7\u01e6\3\2\2\2\u01e7\u01e8\3\2\2\2\u01e8\u0095\3\2\2\2\u01e9"+
-    "\u01f1\7$\2\2\u01ea\u01eb\7^\2\2\u01eb\u01f0\7$\2\2\u01ec\u01ed\7^\2\2"+
-    "\u01ed\u01f0\7^\2\2\u01ee\u01f0\n\16\2\2\u01ef\u01ea\3\2\2\2\u01ef\u01ec"+
-    "\3\2\2\2\u01ef\u01ee\3\2\2\2\u01f0\u01f3\3\2\2\2\u01f1\u01f2\3\2\2\2\u01f1"+
-    "\u01ef\3\2\2\2\u01f2\u01f4\3\2\2\2\u01f3\u01f1\3\2\2\2\u01f4\u0202\7$"+
-    "\2\2\u01f5\u01fd\7)\2\2\u01f6\u01f7\7^\2\2\u01f7\u01fc\7)\2\2\u01f8\u01f9"+
-    "\7^\2\2\u01f9\u01fc\7^\2\2\u01fa\u01fc\n\16\2\2\u01fb\u01f6\3\2\2\2\u01fb"+
-    "\u01f8\3\2\2\2\u01fb\u01fa\3\2\2\2\u01fc\u01ff\3\2\2\2\u01fd\u01fe\3\2"+
-    "\2\2\u01fd\u01fb\3\2\2\2\u01fe\u0200\3\2\2\2\u01ff\u01fd\3\2\2\2\u0200"+
-    "\u0202\7)\2\2\u0201\u01e9\3\2\2\2\u0201\u01f5\3\2\2\2\u0202\u0097\3\2"+
-    "\2\2\u0203\u0207\7\61\2\2\u0204\u0208\n\17\2\2\u0205\u0206\7^\2\2\u0206"+
-    "\u0208\n\20\2\2\u0207\u0204\3\2\2\2\u0207\u0205\3\2\2\2\u0208\u0209\3"+
-    "\2\2\2\u0209\u0207\3\2\2\2\u0209\u020a\3\2\2\2\u020a\u020b\3\2\2\2\u020b"+
-    "\u020f\7\61\2\2\u020c\u020e\t\21\2\2\u020d\u020c\3\2\2\2\u020e\u0211\3"+
-    "\2\2\2\u020f\u020d\3\2\2\2\u020f\u0210\3\2\2\2\u0210\u0212\3\2\2\2\u0211"+
-    "\u020f\3\2\2\2\u0212\u0213\6L\3\2\u0213\u0099\3\2\2\2\u0214\u0215\7v\2"+
-    "\2\u0215\u0216\7t\2\2\u0216\u0217\7w\2\2\u0217\u0218\7g\2\2\u0218\u009b"+
-    "\3\2\2\2\u0219\u021a\7h\2\2\u021a\u021b\7c\2\2\u021b\u021c\7n\2\2\u021c"+
-    "\u021d\7u\2\2\u021d\u021e\7g\2\2\u021e\u009d\3\2\2\2\u021f\u0220\7p\2"+
-    "\2\u0220\u0221\7w\2\2\u0221\u0222\7n\2\2\u0222\u0223\7n\2\2\u0223\u009f"+
-    "\3\2\2\2\u0224\u022a\5\u00a2Q\2\u0225\u0226\5\24\n\2\u0226\u0227\5\u00a2"+
-    "Q\2\u0227\u0229\3\2\2\2\u0228\u0225\3\2\2\2\u0229\u022c\3\2\2\2\u022a"+
-    "\u0228\3\2\2\2\u022a\u022b\3\2\2\2\u022b\u022d\3\2\2\2\u022c\u022a\3\2"+
-    "\2\2\u022d\u022e\6P\4\2\u022e\u00a1\3\2\2\2\u022f\u0233\t\22\2\2\u0230"+
-    "\u0232\t\23\2\2\u0231\u0230\3\2\2\2\u0232\u0235\3\2\2\2\u0233\u0231\3"+
-    "\2\2\2\u0233\u0234\3\2\2\2\u0234\u00a3\3\2\2\2\u0235\u0233\3\2\2\2\u0236"+
-    "\u023f\7\62\2\2\u0237\u023b\t\b\2\2\u0238\u023a\t\t\2\2\u0239\u0238\3"+
-    "\2\2\2\u023a\u023d\3\2\2\2\u023b\u0239\3\2\2\2\u023b\u023c\3\2\2\2\u023c"+
-    "\u023f\3\2\2\2\u023d\u023b\3\2\2\2\u023e\u0236\3\2\2\2\u023e\u0237\3\2"+
-    "\2\2\u023f\u0240\3\2\2\2\u0240\u0241\bR\4\2\u0241\u00a5\3\2\2\2\u0242"+
-    "\u0246\t\22\2\2\u0243\u0245\t\23\2\2\u0244\u0243\3\2\2\2\u0245\u0248\3"+
-    "\2\2\2\u0246\u0244\3\2\2\2\u0246\u0247\3\2\2\2\u0247\u0249\3\2\2\2\u0248"+
-    "\u0246\3\2\2\2\u0249\u024a\bS\4\2\u024a\u00a7\3\2\2\2$\2\3\u00ab\u00b5"+
-    "\u00bf\u00c4\u01ad\u01b0\u01b7\u01ba\u01c1\u01c4\u01c7\u01ce\u01d1\u01d7"+
-    "\u01d9\u01dd\u01e2\u01e4\u01e7\u01ef\u01f1\u01fb\u01fd\u0201\u0207\u0209"+
-    "\u020f\u022a\u0233\u023b\u023e\u0246\5\b\2\2\4\3\2\4\2\2";
+    "I\tI\4J\tJ\4K\tK\4L\tL\4M\tM\4N\tN\4O\tO\4P\tP\4Q\tQ\4R\tR\4S\tS\4T\t"+
+    "T\4U\tU\3\2\6\2\u00ae\n\2\r\2\16\2\u00af\3\2\3\2\3\3\3\3\3\3\3\3\7\3\u00b8"+
+    "\n\3\f\3\16\3\u00bb\13\3\3\3\3\3\3\3\3\3\3\3\7\3\u00c2\n\3\f\3\16\3\u00c5"+
+    "\13\3\3\3\3\3\5\3\u00c9\n\3\3\3\3\3\3\4\3\4\3\5\3\5\3\6\3\6\3\7\3\7\3"+
+    "\b\3\b\3\t\3\t\3\n\3\n\3\n\3\n\3\13\3\13\3\13\3\13\3\13\3\f\3\f\3\r\3"+
+    "\r\3\16\3\16\3\16\3\17\3\17\3\17\3\20\3\20\3\20\3\20\3\20\3\21\3\21\3"+
+    "\21\3\21\3\21\3\21\3\22\3\22\3\22\3\23\3\23\3\23\3\23\3\24\3\24\3\24\3"+
+    "\24\3\24\3\24\3\24\3\24\3\24\3\25\3\25\3\25\3\25\3\25\3\25\3\26\3\26\3"+
+    "\26\3\26\3\26\3\26\3\26\3\27\3\27\3\27\3\27\3\30\3\30\3\30\3\30\3\31\3"+
+    "\31\3\31\3\31\3\31\3\31\3\32\3\32\3\32\3\32\3\32\3\32\3\33\3\33\3\33\3"+
+    "\33\3\33\3\34\3\34\3\34\3\34\3\34\3\34\3\34\3\34\3\34\3\34\3\34\3\35\3"+
+    "\35\3\36\3\36\3\37\3\37\3 \3 \3 \3!\3!\3\"\3\"\3#\3#\3$\3$\3$\3%\3%\3"+
+    "%\3&\3&\3&\3&\3\'\3\'\3(\3(\3(\3)\3)\3*\3*\3*\3+\3+\3+\3,\3,\3,\3,\3-"+
+    "\3-\3-\3.\3.\3.\3.\3/\3/\3\60\3\60\3\61\3\61\3\62\3\62\3\62\3\63\3\63"+
+    "\3\63\3\64\3\64\3\65\3\65\3\66\3\66\3\66\3\67\3\67\3\67\38\38\38\39\3"+
+    "9\39\3:\3:\3:\3:\3;\3;\3;\3<\3<\3<\3=\3=\3>\3>\3>\3?\3?\3?\3@\3@\3@\3"+
+    "A\3A\3A\3B\3B\3B\3C\3C\3C\3D\3D\3D\3E\3E\3E\3F\3F\3F\3F\3G\3G\3G\3G\3"+
+    "H\3H\3H\3H\3H\3I\3I\6I\u01b8\nI\rI\16I\u01b9\3I\5I\u01bd\nI\3J\3J\3J\6"+
+    "J\u01c2\nJ\rJ\16J\u01c3\3J\5J\u01c7\nJ\3K\3K\3K\7K\u01cc\nK\fK\16K\u01cf"+
+    "\13K\5K\u01d1\nK\3K\5K\u01d4\nK\3L\3L\3L\7L\u01d9\nL\fL\16L\u01dc\13L"+
+    "\5L\u01de\nL\3L\3L\6L\u01e2\nL\rL\16L\u01e3\5L\u01e6\nL\3L\3L\5L\u01ea"+
+    "\nL\3L\6L\u01ed\nL\rL\16L\u01ee\5L\u01f1\nL\3L\5L\u01f4\nL\3M\3M\3M\3"+
+    "M\3M\3M\7M\u01fc\nM\fM\16M\u01ff\13M\3M\3M\3M\3M\3M\3M\3M\7M\u0208\nM"+
+    "\fM\16M\u020b\13M\3M\5M\u020e\nM\3N\3N\3N\3N\6N\u0214\nN\rN\16N\u0215"+
+    "\3N\3N\7N\u021a\nN\fN\16N\u021d\13N\3N\3N\3O\3O\3O\3O\3O\3P\3P\3P\3P\3"+
+    "P\3P\3Q\3Q\3Q\3Q\3Q\3R\3R\3R\3R\7R\u0235\nR\fR\16R\u0238\13R\3R\3R\3S"+
+    "\3S\7S\u023e\nS\fS\16S\u0241\13S\3T\3T\3T\7T\u0246\nT\fT\16T\u0249\13"+
+    "T\5T\u024b\nT\3T\3T\3U\3U\7U\u0251\nU\fU\16U\u0254\13U\3U\3U\7\u00b9\u00c3"+
+    "\u01fd\u0209\u0215\2V\4\3\6\4\b\5\n\6\f\7\16\b\20\t\22\n\24\13\26\f\30"+
+    "\r\32\16\34\17\36\20 \21\"\22$\23&\24(\25*\26,\27.\30\60\31\62\32\64\33"+
+    "\66\348\35:\36<\37> @!B\"D#F$H%J&L\'N(P)R*T+V,X-Z.\\/^\60`\61b\62d\63"+
+    "f\64h\65j\66l\67n8p9r:t;v<x=z>|?~@\u0080A\u0082B\u0084C\u0086D\u0088E"+
+    "\u008aF\u008cG\u008eH\u0090I\u0092J\u0094K\u0096L\u0098M\u009aN\u009c"+
+    "O\u009eP\u00a0Q\u00a2R\u00a4S\u00a6T\u00a8U\u00aaV\4\2\3\25\5\2\13\f\17"+
+    "\17\"\"\4\2\f\f\17\17\3\2\629\4\2NNnn\4\2ZZzz\5\2\62;CHch\3\2\63;\3\2"+
+    "\62;\b\2FFHHNNffhhnn\4\2GGgg\4\2--//\6\2FFHHffhh\4\2$$^^\4\2))^^\3\2\f"+
+    "\f\4\2\f\f\61\61\t\2WWeekknouuwwzz\5\2C\\aac|\6\2\62;C\\aac|\u0277\2\4"+
+    "\3\2\2\2\2\6\3\2\2\2\2\b\3\2\2\2\2\n\3\2\2\2\2\f\3\2\2\2\2\16\3\2\2\2"+
+    "\2\20\3\2\2\2\2\22\3\2\2\2\2\24\3\2\2\2\2\26\3\2\2\2\2\30\3\2\2\2\2\32"+
+    "\3\2\2\2\2\34\3\2\2\2\2\36\3\2\2\2\2 \3\2\2\2\2\"\3\2\2\2\2$\3\2\2\2\2"+
+    "&\3\2\2\2\2(\3\2\2\2\2*\3\2\2\2\2,\3\2\2\2\2.\3\2\2\2\2\60\3\2\2\2\2\62"+
+    "\3\2\2\2\2\64\3\2\2\2\2\66\3\2\2\2\28\3\2\2\2\2:\3\2\2\2\2<\3\2\2\2\2"+
+    ">\3\2\2\2\2@\3\2\2\2\2B\3\2\2\2\2D\3\2\2\2\2F\3\2\2\2\2H\3\2\2\2\2J\3"+
+    "\2\2\2\2L\3\2\2\2\2N\3\2\2\2\2P\3\2\2\2\2R\3\2\2\2\2T\3\2\2\2\2V\3\2\2"+
+    "\2\2X\3\2\2\2\2Z\3\2\2\2\2\\\3\2\2\2\2^\3\2\2\2\2`\3\2\2\2\2b\3\2\2\2"+
+    "\2d\3\2\2\2\2f\3\2\2\2\2h\3\2\2\2\2j\3\2\2\2\2l\3\2\2\2\2n\3\2\2\2\2p"+
+    "\3\2\2\2\2r\3\2\2\2\2t\3\2\2\2\2v\3\2\2\2\2x\3\2\2\2\2z\3\2\2\2\2|\3\2"+
+    "\2\2\2~\3\2\2\2\2\u0080\3\2\2\2\2\u0082\3\2\2\2\2\u0084\3\2\2\2\2\u0086"+
+    "\3\2\2\2\2\u0088\3\2\2\2\2\u008a\3\2\2\2\2\u008c\3\2\2\2\2\u008e\3\2\2"+
+    "\2\2\u0090\3\2\2\2\2\u0092\3\2\2\2\2\u0094\3\2\2\2\2\u0096\3\2\2\2\2\u0098"+
+    "\3\2\2\2\2\u009a\3\2\2\2\2\u009c\3\2\2\2\2\u009e\3\2\2\2\2\u00a0\3\2\2"+
+    "\2\2\u00a2\3\2\2\2\2\u00a4\3\2\2\2\2\u00a6\3\2\2\2\3\u00a8\3\2\2\2\3\u00aa"+
+    "\3\2\2\2\4\u00ad\3\2\2\2\6\u00c8\3\2\2\2\b\u00cc\3\2\2\2\n\u00ce\3\2\2"+
+    "\2\f\u00d0\3\2\2\2\16\u00d2\3\2\2\2\20\u00d4\3\2\2\2\22\u00d6\3\2\2\2"+
+    "\24\u00d8\3\2\2\2\26\u00dc\3\2\2\2\30\u00e1\3\2\2\2\32\u00e3\3\2\2\2\34"+
+    "\u00e5\3\2\2\2\36\u00e8\3\2\2\2 \u00eb\3\2\2\2\"\u00f0\3\2\2\2$\u00f6"+
+    "\3\2\2\2&\u00f9\3\2\2\2(\u00fd\3\2\2\2*\u0106\3\2\2\2,\u010c\3\2\2\2."+
+    "\u0113\3\2\2\2\60\u0117\3\2\2\2\62\u011b\3\2\2\2\64\u0121\3\2\2\2\66\u0127"+
+    "\3\2\2\28\u012c\3\2\2\2:\u0137\3\2\2\2<\u0139\3\2\2\2>\u013b\3\2\2\2@"+
+    "\u013d\3\2\2\2B\u0140\3\2\2\2D\u0142\3\2\2\2F\u0144\3\2\2\2H\u0146\3\2"+
+    "\2\2J\u0149\3\2\2\2L\u014c\3\2\2\2N\u0150\3\2\2\2P\u0152\3\2\2\2R\u0155"+
+    "\3\2\2\2T\u0157\3\2\2\2V\u015a\3\2\2\2X\u015d\3\2\2\2Z\u0161\3\2\2\2\\"+
+    "\u0164\3\2\2\2^\u0168\3\2\2\2`\u016a\3\2\2\2b\u016c\3\2\2\2d\u016e\3\2"+
+    "\2\2f\u0171\3\2\2\2h\u0174\3\2\2\2j\u0176\3\2\2\2l\u0178\3\2\2\2n\u017b"+
+    "\3\2\2\2p\u017e\3\2\2\2r\u0181\3\2\2\2t\u0184\3\2\2\2v\u0188\3\2\2\2x"+
+    "\u018b\3\2\2\2z\u018e\3\2\2\2|\u0190\3\2\2\2~\u0193\3\2\2\2\u0080\u0196"+
+    "\3\2\2\2\u0082\u0199\3\2\2\2\u0084\u019c\3\2\2\2\u0086\u019f\3\2\2\2\u0088"+
+    "\u01a2\3\2\2\2\u008a\u01a5\3\2\2\2\u008c\u01a8\3\2\2\2\u008e\u01ac\3\2"+
+    "\2\2\u0090\u01b0\3\2\2\2\u0092\u01b5\3\2\2\2\u0094\u01be\3\2\2\2\u0096"+
+    "\u01d0\3\2\2\2\u0098\u01dd\3\2\2\2\u009a\u020d\3\2\2\2\u009c\u020f\3\2"+
+    "\2\2\u009e\u0220\3\2\2\2\u00a0\u0225\3\2\2\2\u00a2\u022b\3\2\2\2\u00a4"+
+    "\u0230\3\2\2\2\u00a6\u023b\3\2\2\2\u00a8\u024a\3\2\2\2\u00aa\u024e\3\2"+
+    "\2\2\u00ac\u00ae\t\2\2\2\u00ad\u00ac\3\2\2\2\u00ae\u00af\3\2\2\2\u00af"+
+    "\u00ad\3\2\2\2\u00af\u00b0\3\2\2\2\u00b0\u00b1\3\2\2\2\u00b1\u00b2\b\2"+
+    "\2\2\u00b2\5\3\2\2\2\u00b3\u00b4\7\61\2\2\u00b4\u00b5\7\61\2\2\u00b5\u00b9"+
+    "\3\2\2\2\u00b6\u00b8\13\2\2\2\u00b7\u00b6\3\2\2\2\u00b8\u00bb\3\2\2\2"+
+    "\u00b9\u00ba\3\2\2\2\u00b9\u00b7\3\2\2\2\u00ba\u00bc\3\2\2\2\u00bb\u00b9"+
+    "\3\2\2\2\u00bc\u00c9\t\3\2\2\u00bd\u00be\7\61\2\2\u00be\u00bf\7,\2\2\u00bf"+
+    "\u00c3\3\2\2\2\u00c0\u00c2\13\2\2\2\u00c1\u00c0\3\2\2\2\u00c2\u00c5\3"+
+    "\2\2\2\u00c3\u00c4\3\2\2\2\u00c3\u00c1\3\2\2\2\u00c4\u00c6\3\2\2\2\u00c5"+
+    "\u00c3\3\2\2\2\u00c6\u00c7\7,\2\2\u00c7\u00c9\7\61\2\2\u00c8\u00b3\3\2"+
+    "\2\2\u00c8\u00bd\3\2\2\2\u00c9\u00ca\3\2\2\2\u00ca\u00cb\b\3\2\2\u00cb"+
+    "\7\3\2\2\2\u00cc\u00cd\7}\2\2\u00cd\t\3\2\2\2\u00ce\u00cf\7\177\2\2\u00cf"+
+    "\13\3\2\2\2\u00d0\u00d1\7]\2\2\u00d1\r\3\2\2\2\u00d2\u00d3\7_\2\2\u00d3"+
+    "\17\3\2\2\2\u00d4\u00d5\7*\2\2\u00d5\21\3\2\2\2\u00d6\u00d7\7+\2\2\u00d7"+
+    "\23\3\2\2\2\u00d8\u00d9\7\60\2\2\u00d9\u00da\3\2\2\2\u00da\u00db\b\n\3"+
+    "\2\u00db\25\3\2\2\2\u00dc\u00dd\7A\2\2\u00dd\u00de\7\60\2\2\u00de\u00df"+
+    "\3\2\2\2\u00df\u00e0\b\13\3\2\u00e0\27\3\2\2\2\u00e1\u00e2\7.\2\2\u00e2"+
+    "\31\3\2\2\2\u00e3\u00e4\7=\2\2\u00e4\33\3\2\2\2\u00e5\u00e6\7k\2\2\u00e6"+
+    "\u00e7\7h\2\2\u00e7\35\3\2\2\2\u00e8\u00e9\7k\2\2\u00e9\u00ea\7p\2\2\u00ea"+
+    "\37\3\2\2\2\u00eb\u00ec\7g\2\2\u00ec\u00ed\7n\2\2\u00ed\u00ee\7u\2\2\u00ee"+
+    "\u00ef\7g\2\2\u00ef!\3\2\2\2\u00f0\u00f1\7y\2\2\u00f1\u00f2\7j\2\2\u00f2"+
+    "\u00f3\7k\2\2\u00f3\u00f4\7n\2\2\u00f4\u00f5\7g\2\2\u00f5#\3\2\2\2\u00f6"+
+    "\u00f7\7f\2\2\u00f7\u00f8\7q\2\2\u00f8%\3\2\2\2\u00f9\u00fa\7h\2\2\u00fa"+
+    "\u00fb\7q\2\2\u00fb\u00fc\7t\2\2\u00fc\'\3\2\2\2\u00fd\u00fe\7e\2\2\u00fe"+
+    "\u00ff\7q\2\2\u00ff\u0100\7p\2\2\u0100\u0101\7v\2\2\u0101\u0102\7k\2\2"+
+    "\u0102\u0103\7p\2\2\u0103\u0104\7w\2\2\u0104\u0105\7g\2\2\u0105)\3\2\2"+
+    "\2\u0106\u0107\7d\2\2\u0107\u0108\7t\2\2\u0108\u0109\7g\2\2\u0109\u010a"+
+    "\7c\2\2\u010a\u010b\7m\2\2\u010b+\3\2\2\2\u010c\u010d\7t\2\2\u010d\u010e"+
+    "\7g\2\2\u010e\u010f\7v\2\2\u010f\u0110\7w\2\2\u0110\u0111\7t\2\2\u0111"+
+    "\u0112\7p\2\2\u0112-\3\2\2\2\u0113\u0114\7p\2\2\u0114\u0115\7g\2\2\u0115"+
+    "\u0116\7y\2\2\u0116/\3\2\2\2\u0117\u0118\7v\2\2\u0118\u0119\7t\2\2\u0119"+
+    "\u011a\7{\2\2\u011a\61\3\2\2\2\u011b\u011c\7e\2\2\u011c\u011d\7c\2\2\u011d"+
+    "\u011e\7v\2\2\u011e\u011f\7e\2\2\u011f\u0120\7j\2\2\u0120\63\3\2\2\2\u0121"+
+    "\u0122\7v\2\2\u0122\u0123\7j\2\2\u0123\u0124\7t\2\2\u0124\u0125\7q\2\2"+
+    "\u0125\u0126\7y\2\2\u0126\65\3\2\2\2\u0127\u0128\7v\2\2\u0128\u0129\7"+
+    "j\2\2\u0129\u012a\7k\2\2\u012a\u012b\7u\2\2\u012b\67\3\2\2\2\u012c\u012d"+
+    "\7k\2\2\u012d\u012e\7p\2\2\u012e\u012f\7u\2\2\u012f\u0130\7v\2\2\u0130"+
+    "\u0131\7c\2\2\u0131\u0132\7p\2\2\u0132\u0133\7e\2\2\u0133\u0134\7g\2\2"+
+    "\u0134\u0135\7q\2\2\u0135\u0136\7h\2\2\u01369\3\2\2\2\u0137\u0138\7#\2"+
+    "\2\u0138;\3\2\2\2\u0139\u013a\7\u0080\2\2\u013a=\3\2\2\2\u013b\u013c\7"+
+    ",\2\2\u013c?\3\2\2\2\u013d\u013e\7\61\2\2\u013e\u013f\6 \2\2\u013fA\3"+
+    "\2\2\2\u0140\u0141\7\'\2\2\u0141C\3\2\2\2\u0142\u0143\7-\2\2\u0143E\3"+
+    "\2\2\2\u0144\u0145\7/\2\2\u0145G\3\2\2\2\u0146\u0147\7>\2\2\u0147\u0148"+
+    "\7>\2\2\u0148I\3\2\2\2\u0149\u014a\7@\2\2\u014a\u014b\7@\2\2\u014bK\3"+
+    "\2\2\2\u014c\u014d\7@\2\2\u014d\u014e\7@\2\2\u014e\u014f\7@\2\2\u014f"+
+    "M\3\2\2\2\u0150\u0151\7>\2\2\u0151O\3\2\2\2\u0152\u0153\7>\2\2\u0153\u0154"+
+    "\7?\2\2\u0154Q\3\2\2\2\u0155\u0156\7@\2\2\u0156S\3\2\2\2\u0157\u0158\7"+
+    "@\2\2\u0158\u0159\7?\2\2\u0159U\3\2\2\2\u015a\u015b\7?\2\2\u015b\u015c"+
+    "\7?\2\2\u015cW\3\2\2\2\u015d\u015e\7?\2\2\u015e\u015f\7?\2\2\u015f\u0160"+
+    "\7?\2\2\u0160Y\3\2\2\2\u0161\u0162\7#\2\2\u0162\u0163\7?\2\2\u0163[\3"+
+    "\2\2\2\u0164\u0165\7#\2\2\u0165\u0166\7?\2\2\u0166\u0167\7?\2\2\u0167"+
+    "]\3\2\2\2\u0168\u0169\7(\2\2\u0169_\3\2\2\2\u016a\u016b\7`\2\2\u016ba"+
+    "\3\2\2\2\u016c\u016d\7~\2\2\u016dc\3\2\2\2\u016e\u016f\7(\2\2\u016f\u0170"+
+    "\7(\2\2\u0170e\3\2\2\2\u0171\u0172\7~\2\2\u0172\u0173\7~\2\2\u0173g\3"+
+    "\2\2\2\u0174\u0175\7A\2\2\u0175i\3\2\2\2\u0176\u0177\7<\2\2\u0177k\3\2"+
+    "\2\2\u0178\u0179\7A\2\2\u0179\u017a\7<\2\2\u017am\3\2\2\2\u017b\u017c"+
+    "\7<\2\2\u017c\u017d\7<\2\2\u017do\3\2\2\2\u017e\u017f\7/\2\2\u017f\u0180"+
+    "\7@\2\2\u0180q\3\2\2\2\u0181\u0182\7?\2\2\u0182\u0183\7\u0080\2\2\u0183"+
+    "s\3\2\2\2\u0184\u0185\7?\2\2\u0185\u0186\7?\2\2\u0186\u0187\7\u0080\2"+
+    "\2\u0187u\3\2\2\2\u0188\u0189\7-\2\2\u0189\u018a\7-\2\2\u018aw\3\2\2\2"+
+    "\u018b\u018c\7/\2\2\u018c\u018d\7/\2\2\u018dy\3\2\2\2\u018e\u018f\7?\2"+
+    "\2\u018f{\3\2\2\2\u0190\u0191\7-\2\2\u0191\u0192\7?\2\2\u0192}\3\2\2\2"+
+    "\u0193\u0194\7/\2\2\u0194\u0195\7?\2\2\u0195\177\3\2\2\2\u0196\u0197\7"+
+    ",\2\2\u0197\u0198\7?\2\2\u0198\u0081\3\2\2\2\u0199\u019a\7\61\2\2\u019a"+
+    "\u019b\7?\2\2\u019b\u0083\3\2\2\2\u019c\u019d\7\'\2\2\u019d\u019e\7?\2"+
+    "\2\u019e\u0085\3\2\2\2\u019f\u01a0\7(\2\2\u01a0\u01a1\7?\2\2\u01a1\u0087"+
+    "\3\2\2\2\u01a2\u01a3\7`\2\2\u01a3\u01a4\7?\2\2\u01a4\u0089\3\2\2\2\u01a5"+
+    "\u01a6\7~\2\2\u01a6\u01a7\7?\2\2\u01a7\u008b\3\2\2\2\u01a8\u01a9\7>\2"+
+    "\2\u01a9\u01aa\7>\2\2\u01aa\u01ab\7?\2\2\u01ab\u008d\3\2\2\2\u01ac\u01ad"+
+    "\7@\2\2\u01ad\u01ae\7@\2\2\u01ae\u01af\7?\2\2\u01af\u008f\3\2\2\2\u01b0"+
+    "\u01b1\7@\2\2\u01b1\u01b2\7@\2\2\u01b2\u01b3\7@\2\2\u01b3\u01b4\7?\2\2"+
+    "\u01b4\u0091\3\2\2\2\u01b5\u01b7\7\62\2\2\u01b6\u01b8\t\4\2\2\u01b7\u01b6"+
+    "\3\2\2\2\u01b8\u01b9\3\2\2\2\u01b9\u01b7\3\2\2\2\u01b9\u01ba\3\2\2\2\u01ba"+
+    "\u01bc\3\2\2\2\u01bb\u01bd\t\5\2\2\u01bc\u01bb\3\2\2\2\u01bc\u01bd\3\2"+
+    "\2\2\u01bd\u0093\3\2\2\2\u01be\u01bf\7\62\2\2\u01bf\u01c1\t\6\2\2\u01c0"+
+    "\u01c2\t\7\2\2\u01c1\u01c0\3\2\2\2\u01c2\u01c3\3\2\2\2\u01c3\u01c1\3\2"+
+    "\2\2\u01c3\u01c4\3\2\2\2\u01c4\u01c6\3\2\2\2\u01c5\u01c7\t\5\2\2\u01c6"+
+    "\u01c5\3\2\2\2\u01c6\u01c7\3\2\2\2\u01c7\u0095\3\2\2\2\u01c8\u01d1\7\62"+
+    "\2\2\u01c9\u01cd\t\b\2\2\u01ca\u01cc\t\t\2\2\u01cb\u01ca\3\2\2\2\u01cc"+
+    "\u01cf\3\2\2\2\u01cd\u01cb\3\2\2\2\u01cd\u01ce\3\2\2\2\u01ce\u01d1\3\2"+
+    "\2\2\u01cf\u01cd\3\2\2\2\u01d0\u01c8\3\2\2\2\u01d0\u01c9\3\2\2\2\u01d1"+
+    "\u01d3\3\2\2\2\u01d2\u01d4\t\n\2\2\u01d3\u01d2\3\2\2\2\u01d3\u01d4\3\2"+
+    "\2\2\u01d4\u0097\3\2\2\2\u01d5\u01de\7\62\2\2\u01d6\u01da\t\b\2\2\u01d7"+
+    "\u01d9\t\t\2\2\u01d8\u01d7\3\2\2\2\u01d9\u01dc\3\2\2\2\u01da\u01d8\3\2"+
+    "\2\2\u01da\u01db\3\2\2\2\u01db\u01de\3\2\2\2\u01dc\u01da\3\2\2\2\u01dd"+
+    "\u01d5\3\2\2\2\u01dd\u01d6\3\2\2\2\u01de\u01e5\3\2\2\2\u01df\u01e1\5\24"+
+    "\n\2\u01e0\u01e2\t\t\2\2\u01e1\u01e0\3\2\2\2\u01e2\u01e3\3\2\2\2\u01e3"+
+    "\u01e1\3\2\2\2\u01e3\u01e4\3\2\2\2\u01e4\u01e6\3\2\2\2\u01e5\u01df\3\2"+
+    "\2\2\u01e5\u01e6\3\2\2\2\u01e6\u01f0\3\2\2\2\u01e7\u01e9\t\13\2\2\u01e8"+
+    "\u01ea\t\f\2\2\u01e9\u01e8\3\2\2\2\u01e9\u01ea\3\2\2\2\u01ea\u01ec\3\2"+
+    "\2\2\u01eb\u01ed\t\t\2\2\u01ec\u01eb\3\2\2\2\u01ed\u01ee\3\2\2\2\u01ee"+
+    "\u01ec\3\2\2\2\u01ee\u01ef\3\2\2\2\u01ef\u01f1\3\2\2\2\u01f0\u01e7\3\2"+
+    "\2\2\u01f0\u01f1\3\2\2\2\u01f1\u01f3\3\2\2\2\u01f2\u01f4\t\r\2\2\u01f3"+
+    "\u01f2\3\2\2\2\u01f3\u01f4\3\2\2\2\u01f4\u0099\3\2\2\2\u01f5\u01fd\7$"+
+    "\2\2\u01f6\u01f7\7^\2\2\u01f7\u01fc\7$\2\2\u01f8\u01f9\7^\2\2\u01f9\u01fc"+
+    "\7^\2\2\u01fa\u01fc\n\16\2\2\u01fb\u01f6\3\2\2\2\u01fb\u01f8\3\2\2\2\u01fb"+
+    "\u01fa\3\2\2\2\u01fc\u01ff\3\2\2\2\u01fd\u01fe\3\2\2\2\u01fd\u01fb\3\2"+
+    "\2\2\u01fe\u0200\3\2\2\2\u01ff\u01fd\3\2\2\2\u0200\u020e\7$\2\2\u0201"+
+    "\u0209\7)\2\2\u0202\u0203\7^\2\2\u0203\u0208\7)\2\2\u0204\u0205\7^\2\2"+
+    "\u0205\u0208\7^\2\2\u0206\u0208\n\17\2\2\u0207\u0202\3\2\2\2\u0207\u0204"+
+    "\3\2\2\2\u0207\u0206\3\2\2\2\u0208\u020b\3\2\2\2\u0209\u020a\3\2\2\2\u0209"+
+    "\u0207\3\2\2\2\u020a\u020c\3\2\2\2\u020b\u0209\3\2\2\2\u020c\u020e\7)"+
+    "\2\2\u020d\u01f5\3\2\2\2\u020d\u0201\3\2\2\2\u020e\u009b\3\2\2\2\u020f"+
+    "\u0213\7\61\2\2\u0210\u0211\7^\2\2\u0211\u0214\n\20\2\2\u0212\u0214\n"+
+    "\21\2\2\u0213\u0210\3\2\2\2\u0213\u0212\3\2\2\2\u0214\u0215\3\2\2\2\u0215"+
+    "\u0216\3\2\2\2\u0215\u0213\3\2\2\2\u0216\u0217\3\2\2\2\u0217\u021b\7\61"+
+    "\2\2\u0218\u021a\t\22\2\2\u0219\u0218\3\2\2\2\u021a\u021d\3\2\2\2\u021b"+
+    "\u0219\3\2\2\2\u021b\u021c\3\2\2\2\u021c\u021e\3\2\2\2\u021d\u021b\3\2"+
+    "\2\2\u021e\u021f\6N\3\2\u021f\u009d\3\2\2\2\u0220\u0221\7v\2\2\u0221\u0222"+
+    "\7t\2\2\u0222\u0223\7w\2\2\u0223\u0224\7g\2\2\u0224\u009f\3\2\2\2\u0225"+
+    "\u0226\7h\2\2\u0226\u0227\7c\2\2\u0227\u0228\7n\2\2\u0228\u0229\7u\2\2"+
+    "\u0229\u022a\7g\2\2\u022a\u00a1\3\2\2\2\u022b\u022c\7p\2\2\u022c\u022d"+
+    "\7w\2\2\u022d\u022e\7n\2\2\u022e\u022f\7n\2\2\u022f\u00a3\3\2\2\2\u0230"+
+    "\u0236\5\u00a6S\2\u0231\u0232\5\24\n\2\u0232\u0233\5\u00a6S\2\u0233\u0235"+
+    "\3\2\2\2\u0234\u0231\3\2\2\2\u0235\u0238\3\2\2\2\u0236\u0234\3\2\2\2\u0236"+
+    "\u0237\3\2\2\2\u0237\u0239\3\2\2\2\u0238\u0236\3\2\2\2\u0239\u023a\6R"+
+    "\4\2\u023a\u00a5\3\2\2\2\u023b\u023f\t\23\2\2\u023c\u023e\t\24\2\2\u023d"+
+    "\u023c\3\2\2\2\u023e\u0241\3\2\2\2\u023f\u023d\3\2\2\2\u023f\u0240\3\2"+
+    "\2\2\u0240\u00a7\3\2\2\2\u0241\u023f\3\2\2\2\u0242\u024b\7\62\2\2\u0243"+
+    "\u0247\t\b\2\2\u0244\u0246\t\t\2\2\u0245\u0244\3\2\2\2\u0246\u0249\3\2"+
+    "\2\2\u0247\u0245\3\2\2\2\u0247\u0248\3\2\2\2\u0248\u024b\3\2\2\2\u0249"+
+    "\u0247\3\2\2\2\u024a\u0242\3\2\2\2\u024a\u0243\3\2\2\2\u024b\u024c\3\2"+
+    "\2\2\u024c\u024d\bT\4\2\u024d\u00a9\3\2\2\2\u024e\u0252\t\23\2\2\u024f"+
+    "\u0251\t\24\2\2\u0250\u024f\3\2\2\2\u0251\u0254\3\2\2\2\u0252\u0250\3"+
+    "\2\2\2\u0252\u0253\3\2\2\2\u0253\u0255\3\2\2\2\u0254\u0252\3\2\2\2\u0255"+
+    "\u0256\bU\4\2\u0256\u00ab\3\2\2\2$\2\3\u00af\u00b9\u00c3\u00c8\u01b9\u01bc"+
+    "\u01c3\u01c6\u01cd\u01d0\u01d3\u01da\u01dd\u01e3\u01e5\u01e9\u01ee\u01f0"+
+    "\u01f3\u01fb\u01fd\u0207\u0209\u020d\u0213\u0215\u021b\u0236\u023f\u0247"+
+    "\u024a\u0252\5\b\2\2\4\3\2\4\2\2";
   public static final ATN _ATN =
     new ATNDeserializer().deserialize(_serializedATN.toCharArray());
   static {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
index 8766dc9f89c0e..619c582d04a2f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
@@ -18,16 +18,16 @@ class PainlessParser extends Parser {
     new PredictionContextCache();
   public static final int
     WS=1, COMMENT=2, LBRACK=3, RBRACK=4, LBRACE=5, RBRACE=6, LP=7, RP=8, DOT=9, 
-    COMMA=10, SEMICOLON=11, IF=12, IN=13, ELSE=14, WHILE=15, DO=16, FOR=17, 
-    CONTINUE=18, BREAK=19, RETURN=20, NEW=21, TRY=22, CATCH=23, THROW=24, 
-    THIS=25, INSTANCEOF=26, BOOLNOT=27, BWNOT=28, MUL=29, DIV=30, REM=31, 
-    ADD=32, SUB=33, LSH=34, RSH=35, USH=36, LT=37, LTE=38, GT=39, GTE=40, 
-    EQ=41, EQR=42, NE=43, NER=44, BWAND=45, XOR=46, BWOR=47, BOOLAND=48, BOOLOR=49, 
-    COND=50, COLON=51, REF=52, ARROW=53, FIND=54, MATCH=55, INCR=56, DECR=57, 
-    ASSIGN=58, AADD=59, ASUB=60, AMUL=61, ADIV=62, AREM=63, AAND=64, AXOR=65, 
-    AOR=66, ALSH=67, ARSH=68, AUSH=69, OCTAL=70, HEX=71, INTEGER=72, DECIMAL=73, 
-    STRING=74, REGEX=75, TRUE=76, FALSE=77, NULL=78, TYPE=79, ID=80, DOTINTEGER=81, 
-    DOTID=82;
+    NSDOT=10, COMMA=11, SEMICOLON=12, IF=13, IN=14, ELSE=15, WHILE=16, DO=17, 
+    FOR=18, CONTINUE=19, BREAK=20, RETURN=21, NEW=22, TRY=23, CATCH=24, THROW=25, 
+    THIS=26, INSTANCEOF=27, BOOLNOT=28, BWNOT=29, MUL=30, DIV=31, REM=32, 
+    ADD=33, SUB=34, LSH=35, RSH=36, USH=37, LT=38, LTE=39, GT=40, GTE=41, 
+    EQ=42, EQR=43, NE=44, NER=45, BWAND=46, XOR=47, BWOR=48, BOOLAND=49, BOOLOR=50, 
+    COND=51, COLON=52, ELVIS=53, REF=54, ARROW=55, FIND=56, MATCH=57, INCR=58, 
+    DECR=59, ASSIGN=60, AADD=61, ASUB=62, AMUL=63, ADIV=64, AREM=65, AAND=66, 
+    AXOR=67, AOR=68, ALSH=69, ARSH=70, AUSH=71, OCTAL=72, HEX=73, INTEGER=74, 
+    DECIMAL=75, STRING=76, REGEX=77, TRUE=78, FALSE=79, NULL=80, TYPE=81, 
+    ID=82, DOTINTEGER=83, DOTID=84;
   public static final int
     RULE_source = 0, RULE_function = 1, RULE_parameters = 2, RULE_statement = 3, 
     RULE_trailer = 4, RULE_block = 5, RULE_empty = 6, RULE_initializer = 7, 
@@ -48,27 +48,27 @@ class PainlessParser extends Parser {
   };
 
   private static final String[] _LITERAL_NAMES = {
-    null, null, null, "'{'", "'}'", "'['", "']'", "'('", "')'", "'.'", "','", 
-    "';'", "'if'", "'in'", "'else'", "'while'", "'do'", "'for'", "'continue'", 
+    null, null, null, "'{'", "'}'", "'['", "']'", "'('", "')'", "'.'", "'?.'", 
+    "','", "';'", "'if'", "'in'", "'else'", "'while'", "'do'", "'for'", "'continue'", 
     "'break'", "'return'", "'new'", "'try'", "'catch'", "'throw'", "'this'", 
     "'instanceof'", "'!'", "'~'", "'*'", "'/'", "'%'", "'+'", "'-'", "'<<'", 
     "'>>'", "'>>>'", "'<'", "'<='", "'>'", "'>='", "'=='", "'==='", "'!='", 
-    "'!=='", "'&'", "'^'", "'|'", "'&&'", "'||'", "'?'", "':'", "'::'", "'->'", 
-    "'=~'", "'==~'", "'++'", "'--'", "'='", "'+='", "'-='", "'*='", "'/='", 
-    "'%='", "'&='", "'^='", "'|='", "'<<='", "'>>='", "'>>>='", null, null, 
-    null, null, null, null, "'true'", "'false'", "'null'"
+    "'!=='", "'&'", "'^'", "'|'", "'&&'", "'||'", "'?'", "':'", "'?:'", "'::'", 
+    "'->'", "'=~'", "'==~'", "'++'", "'--'", "'='", "'+='", "'-='", "'*='", 
+    "'/='", "'%='", "'&='", "'^='", "'|='", "'<<='", "'>>='", "'>>>='", null, 
+    null, null, null, null, null, "'true'", "'false'", "'null'"
   };
   private static final String[] _SYMBOLIC_NAMES = {
     null, "WS", "COMMENT", "LBRACK", "RBRACK", "LBRACE", "RBRACE", "LP", "RP", 
-    "DOT", "COMMA", "SEMICOLON", "IF", "IN", "ELSE", "WHILE", "DO", "FOR", 
-    "CONTINUE", "BREAK", "RETURN", "NEW", "TRY", "CATCH", "THROW", "THIS", 
-    "INSTANCEOF", "BOOLNOT", "BWNOT", "MUL", "DIV", "REM", "ADD", "SUB", "LSH", 
-    "RSH", "USH", "LT", "LTE", "GT", "GTE", "EQ", "EQR", "NE", "NER", "BWAND", 
-    "XOR", "BWOR", "BOOLAND", "BOOLOR", "COND", "COLON", "REF", "ARROW", "FIND", 
-    "MATCH", "INCR", "DECR", "ASSIGN", "AADD", "ASUB", "AMUL", "ADIV", "AREM", 
-    "AAND", "AXOR", "AOR", "ALSH", "ARSH", "AUSH", "OCTAL", "HEX", "INTEGER", 
-    "DECIMAL", "STRING", "REGEX", "TRUE", "FALSE", "NULL", "TYPE", "ID", "DOTINTEGER", 
-    "DOTID"
+    "DOT", "NSDOT", "COMMA", "SEMICOLON", "IF", "IN", "ELSE", "WHILE", "DO", 
+    "FOR", "CONTINUE", "BREAK", "RETURN", "NEW", "TRY", "CATCH", "THROW", 
+    "THIS", "INSTANCEOF", "BOOLNOT", "BWNOT", "MUL", "DIV", "REM", "ADD", 
+    "SUB", "LSH", "RSH", "USH", "LT", "LTE", "GT", "GTE", "EQ", "EQR", "NE", 
+    "NER", "BWAND", "XOR", "BWOR", "BOOLAND", "BOOLOR", "COND", "COLON", "ELVIS", 
+    "REF", "ARROW", "FIND", "MATCH", "INCR", "DECR", "ASSIGN", "AADD", "ASUB", 
+    "AMUL", "ADIV", "AREM", "AAND", "AXOR", "AOR", "ALSH", "ARSH", "AUSH", 
+    "OCTAL", "HEX", "INTEGER", "DECIMAL", "STRING", "REGEX", "TRUE", "FALSE", 
+    "NULL", "TYPE", "ID", "DOTINTEGER", "DOTID"
   };
   public static final Vocabulary VOCABULARY = new VocabularyImpl(_LITERAL_NAMES, _SYMBOLIC_NAMES);
 
@@ -171,7 +171,7 @@ public final SourceContext source() throws RecognitionException {
       setState(73);
       _errHandler.sync(this);
       _la = _input.LA(1);
-      while ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LBRACE) | (1L << LP) | (1L << IF) | (1L << WHILE) | (1L << DO) | (1L << FOR) | (1L << CONTINUE) | (1L << BREAK) | (1L << RETURN) | (1L << NEW) | (1L << TRY) | (1L << THROW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR))) != 0) || ((((_la - 70)) & ~0x3f) == 0 && ((1L << (_la - 70)) & ((1L << (OCTAL - 70)) | (1L << (HEX - 70)) | (1L << (INTEGER - 70)) | (1L << (DECIMAL - 70)) | (1L << (STRING - 70)) | (1L << (REGEX - 70)) | (1L << (TRUE - 70)) | (1L << (FALSE - 70)) | (1L << (NULL - 70)) | (1L << (TYPE - 70)) | (1L << (ID - 70)))) != 0)) {
+      while ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LBRACE) | (1L << LP) | (1L << IF) | (1L << WHILE) | (1L << DO) | (1L << FOR) | (1L << CONTINUE) | (1L << BREAK) | (1L << RETURN) | (1L << NEW) | (1L << TRY) | (1L << THROW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR))) != 0) || ((((_la - 72)) & ~0x3f) == 0 && ((1L << (_la - 72)) & ((1L << (OCTAL - 72)) | (1L << (HEX - 72)) | (1L << (INTEGER - 72)) | (1L << (DECIMAL - 72)) | (1L << (STRING - 72)) | (1L << (REGEX - 72)) | (1L << (TRUE - 72)) | (1L << (FALSE - 72)) | (1L << (NULL - 72)) | (1L << (TYPE - 72)) | (1L << (ID - 72)))) != 0)) {
         {
         {
         setState(70);
@@ -703,7 +703,7 @@ public final StatementContext statement() throws RecognitionException {
         match(LP);
         setState(128);
         _la = _input.LA(1);
-        if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LBRACE) | (1L << LP) | (1L << NEW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR))) != 0) || ((((_la - 70)) & ~0x3f) == 0 && ((1L << (_la - 70)) & ((1L << (OCTAL - 70)) | (1L << (HEX - 70)) | (1L << (INTEGER - 70)) | (1L << (DECIMAL - 70)) | (1L << (STRING - 70)) | (1L << (REGEX - 70)) | (1L << (TRUE - 70)) | (1L << (FALSE - 70)) | (1L << (NULL - 70)) | (1L << (TYPE - 70)) | (1L << (ID - 70)))) != 0)) {
+        if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LBRACE) | (1L << LP) | (1L << NEW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR))) != 0) || ((((_la - 72)) & ~0x3f) == 0 && ((1L << (_la - 72)) & ((1L << (OCTAL - 72)) | (1L << (HEX - 72)) | (1L << (INTEGER - 72)) | (1L << (DECIMAL - 72)) | (1L << (STRING - 72)) | (1L << (REGEX - 72)) | (1L << (TRUE - 72)) | (1L << (FALSE - 72)) | (1L << (NULL - 72)) | (1L << (TYPE - 72)) | (1L << (ID - 72)))) != 0)) {
           {
           setState(127);
           initializer();
@@ -714,7 +714,7 @@ public final StatementContext statement() throws RecognitionException {
         match(SEMICOLON);
         setState(132);
         _la = _input.LA(1);
-        if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LBRACE) | (1L << LP) | (1L << NEW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR))) != 0) || ((((_la - 70)) & ~0x3f) == 0 && ((1L << (_la - 70)) & ((1L << (OCTAL - 70)) | (1L << (HEX - 70)) | (1L << (INTEGER - 70)) | (1L << (DECIMAL - 70)) | (1L << (STRING - 70)) | (1L << (REGEX - 70)) | (1L << (TRUE - 70)) | (1L << (FALSE - 70)) | (1L << (NULL - 70)) | (1L << (TYPE - 70)) | (1L << (ID - 70)))) != 0)) {
+        if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LBRACE) | (1L << LP) | (1L << NEW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR))) != 0) || ((((_la - 72)) & ~0x3f) == 0 && ((1L << (_la - 72)) & ((1L << (OCTAL - 72)) | (1L << (HEX - 72)) | (1L << (INTEGER - 72)) | (1L << (DECIMAL - 72)) | (1L << (STRING - 72)) | (1L << (REGEX - 72)) | (1L << (TRUE - 72)) | (1L << (FALSE - 72)) | (1L << (NULL - 72)) | (1L << (TYPE - 72)) | (1L << (ID - 72)))) != 0)) {
           {
           setState(131);
           expression(0);
@@ -725,7 +725,7 @@ public final StatementContext statement() throws RecognitionException {
         match(SEMICOLON);
         setState(136);
         _la = _input.LA(1);
-        if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LBRACE) | (1L << LP) | (1L << NEW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR))) != 0) || ((((_la - 70)) & ~0x3f) == 0 && ((1L << (_la - 70)) & ((1L << (OCTAL - 70)) | (1L << (HEX - 70)) | (1L << (INTEGER - 70)) | (1L << (DECIMAL - 70)) | (1L << (STRING - 70)) | (1L << (REGEX - 70)) | (1L << (TRUE - 70)) | (1L << (FALSE - 70)) | (1L << (NULL - 70)) | (1L << (TYPE - 70)) | (1L << (ID - 70)))) != 0)) {
+        if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LBRACE) | (1L << LP) | (1L << NEW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR))) != 0) || ((((_la - 72)) & ~0x3f) == 0 && ((1L << (_la - 72)) & ((1L << (OCTAL - 72)) | (1L << (HEX - 72)) | (1L << (INTEGER - 72)) | (1L << (DECIMAL - 72)) | (1L << (STRING - 72)) | (1L << (REGEX - 72)) | (1L << (TRUE - 72)) | (1L << (FALSE - 72)) | (1L << (NULL - 72)) | (1L << (TYPE - 72)) | (1L << (ID - 72)))) != 0)) {
           {
           setState(135);
           afterthought();
@@ -1044,7 +1044,7 @@ public final BlockContext block() throws RecognitionException {
       setState(195);
       _errHandler.sync(this);
       _la = _input.LA(1);
-      while ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LBRACE) | (1L << LP) | (1L << IF) | (1L << WHILE) | (1L << DO) | (1L << FOR) | (1L << CONTINUE) | (1L << BREAK) | (1L << RETURN) | (1L << NEW) | (1L << TRY) | (1L << THROW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR))) != 0) || ((((_la - 70)) & ~0x3f) == 0 && ((1L << (_la - 70)) & ((1L << (OCTAL - 70)) | (1L << (HEX - 70)) | (1L << (INTEGER - 70)) | (1L << (DECIMAL - 70)) | (1L << (STRING - 70)) | (1L << (REGEX - 70)) | (1L << (TRUE - 70)) | (1L << (FALSE - 70)) | (1L << (NULL - 70)) | (1L << (TYPE - 70)) | (1L << (ID - 70)))) != 0)) {
+      while ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LBRACE) | (1L << LP) | (1L << IF) | (1L << WHILE) | (1L << DO) | (1L << FOR) | (1L << CONTINUE) | (1L << BREAK) | (1L << RETURN) | (1L << NEW) | (1L << TRY) | (1L << THROW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR))) != 0) || ((((_la - 72)) & ~0x3f) == 0 && ((1L << (_la - 72)) & ((1L << (OCTAL - 72)) | (1L << (HEX - 72)) | (1L << (INTEGER - 72)) | (1L << (DECIMAL - 72)) | (1L << (STRING - 72)) | (1L << (REGEX - 72)) | (1L << (TRUE - 72)) | (1L << (FALSE - 72)) | (1L << (NULL - 72)) | (1L << (TYPE - 72)) | (1L << (ID - 72)))) != 0)) {
         {
         {
         setState(192);
@@ -1588,6 +1588,21 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
       else return visitor.visitChildren(this);
     }
   }
+  public static class ElvisContext extends ExpressionContext {
+    public List<ExpressionContext> expression() {
+      return getRuleContexts(ExpressionContext.class);
+    }
+    public ExpressionContext expression(int i) {
+      return getRuleContext(ExpressionContext.class,i);
+    }
+    public TerminalNode ELVIS() { return getToken(PainlessParser.ELVIS, 0); }
+    public ElvisContext(ExpressionContext ctx) { copyFrom(ctx); }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitElvis(this);
+      else return visitor.visitChildren(this);
+    }
+  }
   public static class InstanceofContext extends ExpressionContext {
     public ExpressionContext expression() {
       return getRuleContext(ExpressionContext.class,0);
@@ -1629,7 +1644,7 @@ private ExpressionContext expression(int _p) throws RecognitionException {
       unary();
       }
       _ctx.stop = _input.LT(-1);
-      setState(289);
+      setState(292);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,19,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
@@ -1637,14 +1652,14 @@ private ExpressionContext expression(int _p) throws RecognitionException {
           if ( _parseListeners!=null ) triggerExitRuleEvent();
           _prevctx = _localctx;
           {
-          setState(287);
+          setState(290);
           switch ( getInterpreter().adaptivePredict(_input,18,_ctx) ) {
           case 1:
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
             setState(242);
-            if (!(precpred(_ctx, 14))) throw new FailedPredicateException(this, "precpred(_ctx, 14)");
+            if (!(precpred(_ctx, 15))) throw new FailedPredicateException(this, "precpred(_ctx, 15)");
             setState(243);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << MUL) | (1L << DIV) | (1L << REM))) != 0)) ) {
@@ -1653,7 +1668,7 @@ private ExpressionContext expression(int _p) throws RecognitionException {
               consume();
             }
             setState(244);
-            expression(15);
+            expression(16);
             }
             break;
           case 2:
@@ -1661,7 +1676,7 @@ private ExpressionContext expression(int _p) throws RecognitionException {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
             setState(245);
-            if (!(precpred(_ctx, 13))) throw new FailedPredicateException(this, "precpred(_ctx, 13)");
+            if (!(precpred(_ctx, 14))) throw new FailedPredicateException(this, "precpred(_ctx, 14)");
             setState(246);
             _la = _input.LA(1);
             if ( !(_la==ADD || _la==SUB) ) {
@@ -1670,7 +1685,7 @@ private ExpressionContext expression(int _p) throws RecognitionException {
               consume();
             }
             setState(247);
-            expression(14);
+            expression(15);
             }
             break;
           case 3:
@@ -1678,7 +1693,7 @@ private ExpressionContext expression(int _p) throws RecognitionException {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
             setState(248);
-            if (!(precpred(_ctx, 12))) throw new FailedPredicateException(this, "precpred(_ctx, 12)");
+            if (!(precpred(_ctx, 13))) throw new FailedPredicateException(this, "precpred(_ctx, 13)");
             setState(249);
             _la = _input.LA(1);
             if ( !(_la==FIND || _la==MATCH) ) {
@@ -1687,7 +1702,7 @@ private ExpressionContext expression(int _p) throws RecognitionException {
               consume();
             }
             setState(250);
-            expression(13);
+            expression(14);
             }
             break;
           case 4:
@@ -1695,7 +1710,7 @@ private ExpressionContext expression(int _p) throws RecognitionException {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
             setState(251);
-            if (!(precpred(_ctx, 11))) throw new FailedPredicateException(this, "precpred(_ctx, 11)");
+            if (!(precpred(_ctx, 12))) throw new FailedPredicateException(this, "precpred(_ctx, 12)");
             setState(252);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LSH) | (1L << RSH) | (1L << USH))) != 0)) ) {
@@ -1704,7 +1719,7 @@ private ExpressionContext expression(int _p) throws RecognitionException {
               consume();
             }
             setState(253);
-            expression(12);
+            expression(13);
             }
             break;
           case 5:
@@ -1712,7 +1727,7 @@ private ExpressionContext expression(int _p) throws RecognitionException {
             _localctx = new CompContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
             setState(254);
-            if (!(precpred(_ctx, 10))) throw new FailedPredicateException(this, "precpred(_ctx, 10)");
+            if (!(precpred(_ctx, 11))) throw new FailedPredicateException(this, "precpred(_ctx, 11)");
             setState(255);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LT) | (1L << LTE) | (1L << GT) | (1L << GTE))) != 0)) ) {
@@ -1721,7 +1736,7 @@ private ExpressionContext expression(int _p) throws RecognitionException {
               consume();
             }
             setState(256);
-            expression(11);
+            expression(12);
             }
             break;
           case 6:
@@ -1729,7 +1744,7 @@ private ExpressionContext expression(int _p) throws RecognitionException {
             _localctx = new CompContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
             setState(257);
-            if (!(precpred(_ctx, 8))) throw new FailedPredicateException(this, "precpred(_ctx, 8)");
+            if (!(precpred(_ctx, 9))) throw new FailedPredicateException(this, "precpred(_ctx, 9)");
             setState(258);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << EQ) | (1L << EQR) | (1L << NE) | (1L << NER))) != 0)) ) {
@@ -1738,7 +1753,7 @@ private ExpressionContext expression(int _p) throws RecognitionException {
               consume();
             }
             setState(259);
-            expression(9);
+            expression(10);
             }
             break;
           case 7:
@@ -1746,11 +1761,11 @@ private ExpressionContext expression(int _p) throws RecognitionException {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
             setState(260);
-            if (!(precpred(_ctx, 7))) throw new FailedPredicateException(this, "precpred(_ctx, 7)");
+            if (!(precpred(_ctx, 8))) throw new FailedPredicateException(this, "precpred(_ctx, 8)");
             setState(261);
             match(BWAND);
             setState(262);
-            expression(8);
+            expression(9);
             }
             break;
           case 8:
@@ -1758,11 +1773,11 @@ private ExpressionContext expression(int _p) throws RecognitionException {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
             setState(263);
-            if (!(precpred(_ctx, 6))) throw new FailedPredicateException(this, "precpred(_ctx, 6)");
+            if (!(precpred(_ctx, 7))) throw new FailedPredicateException(this, "precpred(_ctx, 7)");
             setState(264);
             match(XOR);
             setState(265);
-            expression(7);
+            expression(8);
             }
             break;
           case 9:
@@ -1770,11 +1785,11 @@ private ExpressionContext expression(int _p) throws RecognitionException {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
             setState(266);
-            if (!(precpred(_ctx, 5))) throw new FailedPredicateException(this, "precpred(_ctx, 5)");
+            if (!(precpred(_ctx, 6))) throw new FailedPredicateException(this, "precpred(_ctx, 6)");
             setState(267);
             match(BWOR);
             setState(268);
-            expression(6);
+            expression(7);
             }
             break;
           case 10:
@@ -1782,11 +1797,11 @@ private ExpressionContext expression(int _p) throws RecognitionException {
             _localctx = new BoolContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
             setState(269);
-            if (!(precpred(_ctx, 4))) throw new FailedPredicateException(this, "precpred(_ctx, 4)");
+            if (!(precpred(_ctx, 5))) throw new FailedPredicateException(this, "precpred(_ctx, 5)");
             setState(270);
             match(BOOLAND);
             setState(271);
-            expression(5);
+            expression(6);
             }
             break;
           case 11:
@@ -1794,11 +1809,11 @@ private ExpressionContext expression(int _p) throws RecognitionException {
             _localctx = new BoolContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
             setState(272);
-            if (!(precpred(_ctx, 3))) throw new FailedPredicateException(this, "precpred(_ctx, 3)");
+            if (!(precpred(_ctx, 4))) throw new FailedPredicateException(this, "precpred(_ctx, 4)");
             setState(273);
             match(BOOLOR);
             setState(274);
-            expression(4);
+            expression(5);
             }
             break;
           case 12:
@@ -1806,7 +1821,7 @@ private ExpressionContext expression(int _p) throws RecognitionException {
             _localctx = new ConditionalContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
             setState(275);
-            if (!(precpred(_ctx, 2))) throw new FailedPredicateException(this, "precpred(_ctx, 2)");
+            if (!(precpred(_ctx, 3))) throw new FailedPredicateException(this, "precpred(_ctx, 3)");
             setState(276);
             match(COND);
             setState(277);
@@ -1814,42 +1829,54 @@ private ExpressionContext expression(int _p) throws RecognitionException {
             setState(278);
             match(COLON);
             setState(279);
-            expression(2);
+            expression(3);
             }
             break;
           case 13:
             {
-            _localctx = new AssignmentContext(new ExpressionContext(_parentctx, _parentState));
+            _localctx = new ElvisContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
             setState(281);
-            if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)");
+            if (!(precpred(_ctx, 2))) throw new FailedPredicateException(this, "precpred(_ctx, 2)");
             setState(282);
+            match(ELVIS);
+            setState(283);
+            expression(2);
+            }
+            break;
+          case 14:
+            {
+            _localctx = new AssignmentContext(new ExpressionContext(_parentctx, _parentState));
+            pushNewRecursionContext(_localctx, _startState, RULE_expression);
+            setState(284);
+            if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)");
+            setState(285);
             _la = _input.LA(1);
-            if ( !(((((_la - 58)) & ~0x3f) == 0 && ((1L << (_la - 58)) & ((1L << (ASSIGN - 58)) | (1L << (AADD - 58)) | (1L << (ASUB - 58)) | (1L << (AMUL - 58)) | (1L << (ADIV - 58)) | (1L << (AREM - 58)) | (1L << (AAND - 58)) | (1L << (AXOR - 58)) | (1L << (AOR - 58)) | (1L << (ALSH - 58)) | (1L << (ARSH - 58)) | (1L << (AUSH - 58)))) != 0)) ) {
+            if ( !(((((_la - 60)) & ~0x3f) == 0 && ((1L << (_la - 60)) & ((1L << (ASSIGN - 60)) | (1L << (AADD - 60)) | (1L << (ASUB - 60)) | (1L << (AMUL - 60)) | (1L << (ADIV - 60)) | (1L << (AREM - 60)) | (1L << (AAND - 60)) | (1L << (AXOR - 60)) | (1L << (AOR - 60)) | (1L << (ALSH - 60)) | (1L << (ARSH - 60)) | (1L << (AUSH - 60)))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(283);
+            setState(286);
             expression(1);
             }
             break;
-          case 14:
+          case 15:
             {
             _localctx = new InstanceofContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(284);
-            if (!(precpred(_ctx, 9))) throw new FailedPredicateException(this, "precpred(_ctx, 9)");
-            setState(285);
+            setState(287);
+            if (!(precpred(_ctx, 10))) throw new FailedPredicateException(this, "precpred(_ctx, 10)");
+            setState(288);
             match(INSTANCEOF);
-            setState(286);
+            setState(289);
             decltype();
             }
             break;
           }
           } 
         }
-        setState(291);
+        setState(294);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,19,_ctx);
       }
@@ -1951,20 +1978,20 @@ public final UnaryContext unary() throws RecognitionException {
     enterRule(_localctx, 30, RULE_unary);
     int _la;
     try {
-      setState(305);
+      setState(308);
       switch ( getInterpreter().adaptivePredict(_input,20,_ctx) ) {
       case 1:
         _localctx = new PreContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(292);
+        setState(295);
         _la = _input.LA(1);
         if ( !(_la==INCR || _la==DECR) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
         }
-        setState(293);
+        setState(296);
         chain();
         }
         break;
@@ -1972,9 +1999,9 @@ public final UnaryContext unary() throws RecognitionException {
         _localctx = new PostContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(294);
+        setState(297);
         chain();
-        setState(295);
+        setState(298);
         _la = _input.LA(1);
         if ( !(_la==INCR || _la==DECR) ) {
         _errHandler.recoverInline(this);
@@ -1987,7 +2014,7 @@ public final UnaryContext unary() throws RecognitionException {
         _localctx = new ReadContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(297);
+        setState(300);
         chain();
         }
         break;
@@ -1995,14 +2022,14 @@ public final UnaryContext unary() throws RecognitionException {
         _localctx = new OperatorContext(_localctx);
         enterOuterAlt(_localctx, 4);
         {
-        setState(298);
+        setState(301);
         _la = _input.LA(1);
         if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB))) != 0)) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
         }
-        setState(299);
+        setState(302);
         unary();
         }
         break;
@@ -2010,13 +2037,13 @@ public final UnaryContext unary() throws RecognitionException {
         _localctx = new CastContext(_localctx);
         enterOuterAlt(_localctx, 5);
         {
-        setState(300);
+        setState(303);
         match(LP);
-        setState(301);
+        setState(304);
         decltype();
-        setState(302);
+        setState(305);
         match(RP);
-        setState(303);
+        setState(306);
         unary();
         }
         break;
@@ -2098,27 +2125,27 @@ public final ChainContext chain() throws RecognitionException {
     enterRule(_localctx, 32, RULE_chain);
     try {
       int _alt;
-      setState(323);
+      setState(326);
       switch ( getInterpreter().adaptivePredict(_input,23,_ctx) ) {
       case 1:
         _localctx = new DynamicContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(307);
+        setState(310);
         primary();
-        setState(311);
+        setState(314);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,21,_ctx);
         while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
           if ( _alt==1 ) {
             {
             {
-            setState(308);
+            setState(311);
             postfix();
             }
             } 
           }
-          setState(313);
+          setState(316);
           _errHandler.sync(this);
           _alt = getInterpreter().adaptivePredict(_input,21,_ctx);
         }
@@ -2128,23 +2155,23 @@ public final ChainContext chain() throws RecognitionException {
         _localctx = new StaticContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(314);
+        setState(317);
         decltype();
-        setState(315);
+        setState(318);
         postdot();
-        setState(319);
+        setState(322);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,22,_ctx);
         while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
           if ( _alt==1 ) {
             {
             {
-            setState(316);
+            setState(319);
             postfix();
             }
             } 
           }
-          setState(321);
+          setState(324);
           _errHandler.sync(this);
           _alt = getInterpreter().adaptivePredict(_input,22,_ctx);
         }
@@ -2154,7 +2181,7 @@ public final ChainContext chain() throws RecognitionException {
         _localctx = new NewarrayContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(322);
+        setState(325);
         arrayinitializer();
         }
         break;
@@ -2314,17 +2341,17 @@ public final PrimaryContext primary() throws RecognitionException {
     enterRule(_localctx, 34, RULE_primary);
     int _la;
     try {
-      setState(343);
+      setState(346);
       switch ( getInterpreter().adaptivePredict(_input,24,_ctx) ) {
       case 1:
         _localctx = new PrecedenceContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(325);
+        setState(328);
         match(LP);
-        setState(326);
+        setState(329);
         expression(0);
-        setState(327);
+        setState(330);
         match(RP);
         }
         break;
@@ -2332,9 +2359,9 @@ public final PrimaryContext primary() throws RecognitionException {
         _localctx = new NumericContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(329);
+        setState(332);
         _la = _input.LA(1);
-        if ( !(((((_la - 70)) & ~0x3f) == 0 && ((1L << (_la - 70)) & ((1L << (OCTAL - 70)) | (1L << (HEX - 70)) | (1L << (INTEGER - 70)) | (1L << (DECIMAL - 70)))) != 0)) ) {
+        if ( !(((((_la - 72)) & ~0x3f) == 0 && ((1L << (_la - 72)) & ((1L << (OCTAL - 72)) | (1L << (HEX - 72)) | (1L << (INTEGER - 72)) | (1L << (DECIMAL - 72)))) != 0)) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
@@ -2345,7 +2372,7 @@ public final PrimaryContext primary() throws RecognitionException {
         _localctx = new TrueContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(330);
+        setState(333);
         match(TRUE);
         }
         break;
@@ -2353,7 +2380,7 @@ public final PrimaryContext primary() throws RecognitionException {
         _localctx = new FalseContext(_localctx);
         enterOuterAlt(_localctx, 4);
         {
-        setState(331);
+        setState(334);
         match(FALSE);
         }
         break;
@@ -2361,7 +2388,7 @@ public final PrimaryContext primary() throws RecognitionException {
         _localctx = new NullContext(_localctx);
         enterOuterAlt(_localctx, 5);
         {
-        setState(332);
+        setState(335);
         match(NULL);
         }
         break;
@@ -2369,7 +2396,7 @@ public final PrimaryContext primary() throws RecognitionException {
         _localctx = new StringContext(_localctx);
         enterOuterAlt(_localctx, 6);
         {
-        setState(333);
+        setState(336);
         match(STRING);
         }
         break;
@@ -2377,7 +2404,7 @@ public final PrimaryContext primary() throws RecognitionException {
         _localctx = new RegexContext(_localctx);
         enterOuterAlt(_localctx, 7);
         {
-        setState(334);
+        setState(337);
         match(REGEX);
         }
         break;
@@ -2385,7 +2412,7 @@ public final PrimaryContext primary() throws RecognitionException {
         _localctx = new ListinitContext(_localctx);
         enterOuterAlt(_localctx, 8);
         {
-        setState(335);
+        setState(338);
         listinitializer();
         }
         break;
@@ -2393,7 +2420,7 @@ public final PrimaryContext primary() throws RecognitionException {
         _localctx = new MapinitContext(_localctx);
         enterOuterAlt(_localctx, 9);
         {
-        setState(336);
+        setState(339);
         mapinitializer();
         }
         break;
@@ -2401,7 +2428,7 @@ public final PrimaryContext primary() throws RecognitionException {
         _localctx = new VariableContext(_localctx);
         enterOuterAlt(_localctx, 10);
         {
-        setState(337);
+        setState(340);
         match(ID);
         }
         break;
@@ -2409,9 +2436,9 @@ public final PrimaryContext primary() throws RecognitionException {
         _localctx = new CalllocalContext(_localctx);
         enterOuterAlt(_localctx, 11);
         {
-        setState(338);
+        setState(341);
         match(ID);
-        setState(339);
+        setState(342);
         arguments();
         }
         break;
@@ -2419,11 +2446,11 @@ public final PrimaryContext primary() throws RecognitionException {
         _localctx = new NewobjectContext(_localctx);
         enterOuterAlt(_localctx, 12);
         {
-        setState(340);
+        setState(343);
         match(NEW);
-        setState(341);
+        setState(344);
         match(TYPE);
-        setState(342);
+        setState(345);
         arguments();
         }
         break;
@@ -2465,26 +2492,26 @@ public final PostfixContext postfix() throws RecognitionException {
     PostfixContext _localctx = new PostfixContext(_ctx, getState());
     enterRule(_localctx, 36, RULE_postfix);
     try {
-      setState(348);
+      setState(351);
       switch ( getInterpreter().adaptivePredict(_input,25,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(345);
+        setState(348);
         callinvoke();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(346);
+        setState(349);
         fieldaccess();
         }
         break;
       case 3:
         enterOuterAlt(_localctx, 3);
         {
-        setState(347);
+        setState(350);
         braceaccess();
         }
         break;
@@ -2523,19 +2550,19 @@ public final PostdotContext postdot() throws RecognitionException {
     PostdotContext _localctx = new PostdotContext(_ctx, getState());
     enterRule(_localctx, 38, RULE_postdot);
     try {
-      setState(352);
+      setState(355);
       switch ( getInterpreter().adaptivePredict(_input,26,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(350);
+        setState(353);
         callinvoke();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(351);
+        setState(354);
         fieldaccess();
         }
         break;
@@ -2553,11 +2580,12 @@ public final PostdotContext postdot() throws RecognitionException {
   }
 
   public static class CallinvokeContext extends ParserRuleContext {
-    public TerminalNode DOT() { return getToken(PainlessParser.DOT, 0); }
     public TerminalNode DOTID() { return getToken(PainlessParser.DOTID, 0); }
     public ArgumentsContext arguments() {
       return getRuleContext(ArgumentsContext.class,0);
     }
+    public TerminalNode DOT() { return getToken(PainlessParser.DOT, 0); }
+    public TerminalNode NSDOT() { return getToken(PainlessParser.NSDOT, 0); }
     public CallinvokeContext(ParserRuleContext parent, int invokingState) {
       super(parent, invokingState);
     }
@@ -2572,14 +2600,20 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
   public final CallinvokeContext callinvoke() throws RecognitionException {
     CallinvokeContext _localctx = new CallinvokeContext(_ctx, getState());
     enterRule(_localctx, 40, RULE_callinvoke);
+    int _la;
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(354);
-      match(DOT);
-      setState(355);
+      setState(357);
+      _la = _input.LA(1);
+      if ( !(_la==DOT || _la==NSDOT) ) {
+      _errHandler.recoverInline(this);
+      } else {
+        consume();
+      }
+      setState(358);
       match(DOTID);
-      setState(356);
+      setState(359);
       arguments();
       }
     }
@@ -2596,6 +2630,7 @@ public final CallinvokeContext callinvoke() throws RecognitionException {
 
   public static class FieldaccessContext extends ParserRuleContext {
     public TerminalNode DOT() { return getToken(PainlessParser.DOT, 0); }
+    public TerminalNode NSDOT() { return getToken(PainlessParser.NSDOT, 0); }
     public TerminalNode DOTID() { return getToken(PainlessParser.DOTID, 0); }
     public TerminalNode DOTINTEGER() { return getToken(PainlessParser.DOTINTEGER, 0); }
     public FieldaccessContext(ParserRuleContext parent, int invokingState) {
@@ -2616,9 +2651,14 @@ public final FieldaccessContext fieldaccess() throws RecognitionException {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(358);
-      match(DOT);
-      setState(359);
+      setState(361);
+      _la = _input.LA(1);
+      if ( !(_la==DOT || _la==NSDOT) ) {
+      _errHandler.recoverInline(this);
+      } else {
+        consume();
+      }
+      setState(362);
       _la = _input.LA(1);
       if ( !(_la==DOTINTEGER || _la==DOTID) ) {
       _errHandler.recoverInline(this);
@@ -2661,11 +2701,11 @@ public final BraceaccessContext braceaccess() throws RecognitionException {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(361);
+      setState(364);
       match(LBRACE);
-      setState(362);
+      setState(365);
       expression(0);
-      setState(363);
+      setState(366);
       match(RBRACE);
       }
     }
@@ -2762,17 +2802,17 @@ public final ArrayinitializerContext arrayinitializer() throws RecognitionExcept
     int _la;
     try {
       int _alt;
-      setState(409);
+      setState(412);
       switch ( getInterpreter().adaptivePredict(_input,34,_ctx) ) {
       case 1:
         _localctx = new NewstandardarrayContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(365);
+        setState(368);
         match(NEW);
-        setState(366);
+        setState(369);
         match(TYPE);
-        setState(371); 
+        setState(374); 
         _errHandler.sync(this);
         _alt = 1;
         do {
@@ -2780,11 +2820,11 @@ public final ArrayinitializerContext arrayinitializer() throws RecognitionExcept
           case 1:
             {
             {
-            setState(367);
+            setState(370);
             match(LBRACE);
-            setState(368);
+            setState(371);
             expression(0);
-            setState(369);
+            setState(372);
             match(RBRACE);
             }
             }
@@ -2792,29 +2832,29 @@ public final ArrayinitializerContext arrayinitializer() throws RecognitionExcept
           default:
             throw new NoViableAltException(this);
           }
-          setState(373); 
+          setState(376); 
           _errHandler.sync(this);
           _alt = getInterpreter().adaptivePredict(_input,27,_ctx);
         } while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER );
-        setState(382);
+        setState(385);
         switch ( getInterpreter().adaptivePredict(_input,29,_ctx) ) {
         case 1:
           {
-          setState(375);
+          setState(378);
           postdot();
-          setState(379);
+          setState(382);
           _errHandler.sync(this);
           _alt = getInterpreter().adaptivePredict(_input,28,_ctx);
           while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
             if ( _alt==1 ) {
               {
               {
-              setState(376);
+              setState(379);
               postfix();
               }
               } 
             }
-            setState(381);
+            setState(384);
             _errHandler.sync(this);
             _alt = getInterpreter().adaptivePredict(_input,28,_ctx);
           }
@@ -2827,65 +2867,65 @@ public final ArrayinitializerContext arrayinitializer() throws RecognitionExcept
         _localctx = new NewinitializedarrayContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(384);
+        setState(387);
         match(NEW);
-        setState(385);
+        setState(388);
         match(TYPE);
-        setState(386);
+        setState(389);
         match(LBRACE);
-        setState(387);
+        setState(390);
         match(RBRACE);
-        setState(388);
+        setState(391);
         match(LBRACK);
-        setState(397);
+        setState(400);
         _la = _input.LA(1);
-        if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LBRACE) | (1L << LP) | (1L << NEW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR))) != 0) || ((((_la - 70)) & ~0x3f) == 0 && ((1L << (_la - 70)) & ((1L << (OCTAL - 70)) | (1L << (HEX - 70)) | (1L << (INTEGER - 70)) | (1L << (DECIMAL - 70)) | (1L << (STRING - 70)) | (1L << (REGEX - 70)) | (1L << (TRUE - 70)) | (1L << (FALSE - 70)) | (1L << (NULL - 70)) | (1L << (TYPE - 70)) | (1L << (ID - 70)))) != 0)) {
+        if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LBRACE) | (1L << LP) | (1L << NEW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR))) != 0) || ((((_la - 72)) & ~0x3f) == 0 && ((1L << (_la - 72)) & ((1L << (OCTAL - 72)) | (1L << (HEX - 72)) | (1L << (INTEGER - 72)) | (1L << (DECIMAL - 72)) | (1L << (STRING - 72)) | (1L << (REGEX - 72)) | (1L << (TRUE - 72)) | (1L << (FALSE - 72)) | (1L << (NULL - 72)) | (1L << (TYPE - 72)) | (1L << (ID - 72)))) != 0)) {
           {
-          setState(389);
+          setState(392);
           expression(0);
-          setState(394);
+          setState(397);
           _errHandler.sync(this);
           _la = _input.LA(1);
           while (_la==COMMA) {
             {
             {
-            setState(390);
+            setState(393);
             match(COMMA);
-            setState(391);
+            setState(394);
             expression(0);
             }
             }
-            setState(396);
+            setState(399);
             _errHandler.sync(this);
             _la = _input.LA(1);
           }
           }
         }
 
-        setState(400);
+        setState(403);
         _la = _input.LA(1);
         if (_la==SEMICOLON) {
           {
-          setState(399);
+          setState(402);
           match(SEMICOLON);
           }
         }
 
-        setState(402);
+        setState(405);
         match(RBRACK);
-        setState(406);
+        setState(409);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,33,_ctx);
         while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
           if ( _alt==1 ) {
             {
             {
-            setState(403);
+            setState(406);
             postfix();
             }
             } 
           }
-          setState(408);
+          setState(411);
           _errHandler.sync(this);
           _alt = getInterpreter().adaptivePredict(_input,33,_ctx);
         }
@@ -2933,41 +2973,41 @@ public final ListinitializerContext listinitializer() throws RecognitionExceptio
     enterRule(_localctx, 48, RULE_listinitializer);
     int _la;
     try {
-      setState(424);
+      setState(427);
       switch ( getInterpreter().adaptivePredict(_input,36,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(411);
+        setState(414);
         match(LBRACE);
-        setState(412);
+        setState(415);
         expression(0);
-        setState(417);
+        setState(420);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==COMMA) {
           {
           {
-          setState(413);
+          setState(416);
           match(COMMA);
-          setState(414);
+          setState(417);
           expression(0);
           }
           }
-          setState(419);
+          setState(422);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
-        setState(420);
+        setState(423);
         match(RBRACE);
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(422);
+        setState(425);
         match(LBRACE);
-        setState(423);
+        setState(426);
         match(RBRACE);
         }
         break;
@@ -3014,43 +3054,43 @@ public final MapinitializerContext mapinitializer() throws RecognitionException
     enterRule(_localctx, 50, RULE_mapinitializer);
     int _la;
     try {
-      setState(440);
+      setState(443);
       switch ( getInterpreter().adaptivePredict(_input,38,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(426);
+        setState(429);
         match(LBRACE);
-        setState(427);
+        setState(430);
         maptoken();
-        setState(432);
+        setState(435);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==COMMA) {
           {
           {
-          setState(428);
+          setState(431);
           match(COMMA);
-          setState(429);
+          setState(432);
           maptoken();
           }
           }
-          setState(434);
+          setState(437);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
-        setState(435);
+        setState(438);
         match(RBRACE);
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(437);
+        setState(440);
         match(LBRACE);
-        setState(438);
+        setState(441);
         match(COLON);
-        setState(439);
+        setState(442);
         match(RBRACE);
         }
         break;
@@ -3092,11 +3132,11 @@ public final MaptokenContext maptoken() throws RecognitionException {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(442);
+      setState(445);
       expression(0);
-      setState(443);
+      setState(446);
       match(COLON);
-      setState(444);
+      setState(447);
       expression(0);
       }
     }
@@ -3143,34 +3183,34 @@ public final ArgumentsContext arguments() throws RecognitionException {
       enterOuterAlt(_localctx, 1);
       {
       {
-      setState(446);
+      setState(449);
       match(LP);
-      setState(455);
+      setState(458);
       _la = _input.LA(1);
-      if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LBRACE) | (1L << LP) | (1L << NEW) | (1L << THIS) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR))) != 0) || ((((_la - 70)) & ~0x3f) == 0 && ((1L << (_la - 70)) & ((1L << (OCTAL - 70)) | (1L << (HEX - 70)) | (1L << (INTEGER - 70)) | (1L << (DECIMAL - 70)) | (1L << (STRING - 70)) | (1L << (REGEX - 70)) | (1L << (TRUE - 70)) | (1L << (FALSE - 70)) | (1L << (NULL - 70)) | (1L << (TYPE - 70)) | (1L << (ID - 70)))) != 0)) {
+      if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LBRACE) | (1L << LP) | (1L << NEW) | (1L << THIS) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR))) != 0) || ((((_la - 72)) & ~0x3f) == 0 && ((1L << (_la - 72)) & ((1L << (OCTAL - 72)) | (1L << (HEX - 72)) | (1L << (INTEGER - 72)) | (1L << (DECIMAL - 72)) | (1L << (STRING - 72)) | (1L << (REGEX - 72)) | (1L << (TRUE - 72)) | (1L << (FALSE - 72)) | (1L << (NULL - 72)) | (1L << (TYPE - 72)) | (1L << (ID - 72)))) != 0)) {
         {
-        setState(447);
+        setState(450);
         argument();
-        setState(452);
+        setState(455);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==COMMA) {
           {
           {
-          setState(448);
+          setState(451);
           match(COMMA);
-          setState(449);
+          setState(452);
           argument();
           }
           }
-          setState(454);
+          setState(457);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
         }
       }
 
-      setState(457);
+      setState(460);
       match(RP);
       }
       }
@@ -3211,26 +3251,26 @@ public final ArgumentContext argument() throws RecognitionException {
     ArgumentContext _localctx = new ArgumentContext(_ctx, getState());
     enterRule(_localctx, 56, RULE_argument);
     try {
-      setState(462);
+      setState(465);
       switch ( getInterpreter().adaptivePredict(_input,41,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(459);
+        setState(462);
         expression(0);
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(460);
+        setState(463);
         lambda();
         }
         break;
       case 3:
         enterOuterAlt(_localctx, 3);
         {
-        setState(461);
+        setState(464);
         funcref();
         }
         break;
@@ -3285,58 +3325,58 @@ public final LambdaContext lambda() throws RecognitionException {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(477);
+      setState(480);
       switch (_input.LA(1)) {
       case TYPE:
       case ID:
         {
-        setState(464);
+        setState(467);
         lamtype();
         }
         break;
       case LP:
         {
-        setState(465);
+        setState(468);
         match(LP);
-        setState(474);
+        setState(477);
         _la = _input.LA(1);
         if (_la==TYPE || _la==ID) {
           {
-          setState(466);
+          setState(469);
           lamtype();
-          setState(471);
+          setState(474);
           _errHandler.sync(this);
           _la = _input.LA(1);
           while (_la==COMMA) {
             {
             {
-            setState(467);
+            setState(470);
             match(COMMA);
-            setState(468);
+            setState(471);
             lamtype();
             }
             }
-            setState(473);
+            setState(476);
             _errHandler.sync(this);
             _la = _input.LA(1);
           }
           }
         }
 
-        setState(476);
+        setState(479);
         match(RP);
         }
         break;
       default:
         throw new NoViableAltException(this);
       }
-      setState(479);
-      match(ARROW);
       setState(482);
+      match(ARROW);
+      setState(485);
       switch (_input.LA(1)) {
       case LBRACK:
         {
-        setState(480);
+        setState(483);
         block();
         }
         break;
@@ -3361,7 +3401,7 @@ public final LambdaContext lambda() throws RecognitionException {
       case TYPE:
       case ID:
         {
-        setState(481);
+        setState(484);
         expression(0);
         }
         break;
@@ -3404,16 +3444,16 @@ public final LamtypeContext lamtype() throws RecognitionException {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(485);
+      setState(488);
       _la = _input.LA(1);
       if (_la==TYPE) {
         {
-        setState(484);
+        setState(487);
         decltype();
         }
       }
 
-      setState(487);
+      setState(490);
       match(ID);
       }
     }
@@ -3492,17 +3532,17 @@ public final FuncrefContext funcref() throws RecognitionException {
     FuncrefContext _localctx = new FuncrefContext(_ctx, getState());
     enterRule(_localctx, 62, RULE_funcref);
     try {
-      setState(502);
+      setState(505);
       switch ( getInterpreter().adaptivePredict(_input,47,_ctx) ) {
       case 1:
         _localctx = new ClassfuncrefContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(489);
+        setState(492);
         match(TYPE);
-        setState(490);
+        setState(493);
         match(REF);
-        setState(491);
+        setState(494);
         match(ID);
         }
         break;
@@ -3510,11 +3550,11 @@ public final FuncrefContext funcref() throws RecognitionException {
         _localctx = new ConstructorfuncrefContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(492);
+        setState(495);
         decltype();
-        setState(493);
+        setState(496);
         match(REF);
-        setState(494);
+        setState(497);
         match(NEW);
         }
         break;
@@ -3522,11 +3562,11 @@ public final FuncrefContext funcref() throws RecognitionException {
         _localctx = new CapturingfuncrefContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(496);
+        setState(499);
         match(ID);
-        setState(497);
+        setState(500);
         match(REF);
-        setState(498);
+        setState(501);
         match(ID);
         }
         break;
@@ -3534,11 +3574,11 @@ public final FuncrefContext funcref() throws RecognitionException {
         _localctx = new LocalfuncrefContext(_localctx);
         enterOuterAlt(_localctx, 4);
         {
-        setState(499);
+        setState(502);
         match(THIS);
-        setState(500);
+        setState(503);
         match(REF);
-        setState(501);
+        setState(504);
         match(ID);
         }
         break;
@@ -3574,39 +3614,41 @@ private boolean statement_sempred(StatementContext _localctx, int predIndex) {
   private boolean expression_sempred(ExpressionContext _localctx, int predIndex) {
     switch (predIndex) {
     case 1:
-      return precpred(_ctx, 14);
+      return precpred(_ctx, 15);
     case 2:
-      return precpred(_ctx, 13);
+      return precpred(_ctx, 14);
     case 3:
-      return precpred(_ctx, 12);
+      return precpred(_ctx, 13);
     case 4:
-      return precpred(_ctx, 11);
+      return precpred(_ctx, 12);
     case 5:
-      return precpred(_ctx, 10);
+      return precpred(_ctx, 11);
     case 6:
-      return precpred(_ctx, 8);
+      return precpred(_ctx, 9);
     case 7:
-      return precpred(_ctx, 7);
+      return precpred(_ctx, 8);
     case 8:
-      return precpred(_ctx, 6);
+      return precpred(_ctx, 7);
     case 9:
-      return precpred(_ctx, 5);
+      return precpred(_ctx, 6);
     case 10:
-      return precpred(_ctx, 4);
+      return precpred(_ctx, 5);
     case 11:
-      return precpred(_ctx, 3);
+      return precpred(_ctx, 4);
     case 12:
-      return precpred(_ctx, 2);
+      return precpred(_ctx, 3);
     case 13:
-      return precpred(_ctx, 1);
+      return precpred(_ctx, 2);
     case 14:
-      return precpred(_ctx, 9);
+      return precpred(_ctx, 1);
+    case 15:
+      return precpred(_ctx, 10);
     }
     return true;
   }
 
   public static final String _serializedATN =
-    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3T\u01fb\4\2\t\2\4"+
+    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3V\u01fe\4\2\t\2\4"+
     "\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n\4\13\t"+
     "\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22\t\22"+
     "\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\4\31\t\31"+
@@ -3626,179 +3668,181 @@ private boolean expression_sempred(ExpressionContext _localctx, int predIndex) {
     "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+
     "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+
     "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+
-    "\3\20\3\20\3\20\3\20\3\20\3\20\7\20\u0122\n\20\f\20\16\20\u0125\13\20"+
-    "\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\5\21"+
-    "\u0134\n\21\3\22\3\22\7\22\u0138\n\22\f\22\16\22\u013b\13\22\3\22\3\22"+
-    "\3\22\7\22\u0140\n\22\f\22\16\22\u0143\13\22\3\22\5\22\u0146\n\22\3\23"+
-    "\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23"+
-    "\3\23\3\23\3\23\5\23\u015a\n\23\3\24\3\24\3\24\5\24\u015f\n\24\3\25\3"+
-    "\25\5\25\u0163\n\25\3\26\3\26\3\26\3\26\3\27\3\27\3\27\3\30\3\30\3\30"+
-    "\3\30\3\31\3\31\3\31\3\31\3\31\3\31\6\31\u0176\n\31\r\31\16\31\u0177\3"+
-    "\31\3\31\7\31\u017c\n\31\f\31\16\31\u017f\13\31\5\31\u0181\n\31\3\31\3"+
-    "\31\3\31\3\31\3\31\3\31\3\31\3\31\7\31\u018b\n\31\f\31\16\31\u018e\13"+
-    "\31\5\31\u0190\n\31\3\31\5\31\u0193\n\31\3\31\3\31\7\31\u0197\n\31\f\31"+
-    "\16\31\u019a\13\31\5\31\u019c\n\31\3\32\3\32\3\32\3\32\7\32\u01a2\n\32"+
-    "\f\32\16\32\u01a5\13\32\3\32\3\32\3\32\3\32\5\32\u01ab\n\32\3\33\3\33"+
-    "\3\33\3\33\7\33\u01b1\n\33\f\33\16\33\u01b4\13\33\3\33\3\33\3\33\3\33"+
-    "\3\33\5\33\u01bb\n\33\3\34\3\34\3\34\3\34\3\35\3\35\3\35\3\35\7\35\u01c5"+
-    "\n\35\f\35\16\35\u01c8\13\35\5\35\u01ca\n\35\3\35\3\35\3\36\3\36\3\36"+
-    "\5\36\u01d1\n\36\3\37\3\37\3\37\3\37\3\37\7\37\u01d8\n\37\f\37\16\37\u01db"+
-    "\13\37\5\37\u01dd\n\37\3\37\5\37\u01e0\n\37\3\37\3\37\3\37\5\37\u01e5"+
-    "\n\37\3 \5 \u01e8\n \3 \3 \3!\3!\3!\3!\3!\3!\3!\3!\3!\3!\3!\3!\3!\5!\u01f9"+
-    "\n!\3!\2\3\36\"\2\4\6\b\n\f\16\20\22\24\26\30\32\34\36 \"$&(*,.\60\62"+
-    "\64\668:<>@\2\16\3\3\r\r\3\2\37!\3\2\"#\3\289\3\2$&\3\2\'*\3\2+.\3\2<"+
-    "G\3\2:;\4\2\35\36\"#\3\2HK\3\2ST\u0233\2E\3\2\2\2\4P\3\2\2\2\6U\3\2\2"+
-    "\2\b\u00bb\3\2\2\2\n\u00bf\3\2\2\2\f\u00c1\3\2\2\2\16\u00ca\3\2\2\2\20"+
-    "\u00ce\3\2\2\2\22\u00d0\3\2\2\2\24\u00d2\3\2\2\2\26\u00db\3\2\2\2\30\u00e3"+
-    "\3\2\2\2\32\u00e8\3\2\2\2\34\u00ef\3\2\2\2\36\u00f1\3\2\2\2 \u0133\3\2"+
-    "\2\2\"\u0145\3\2\2\2$\u0159\3\2\2\2&\u015e\3\2\2\2(\u0162\3\2\2\2*\u0164"+
-    "\3\2\2\2,\u0168\3\2\2\2.\u016b\3\2\2\2\60\u019b\3\2\2\2\62\u01aa\3\2\2"+
-    "\2\64\u01ba\3\2\2\2\66\u01bc\3\2\2\28\u01c0\3\2\2\2:\u01d0\3\2\2\2<\u01df"+
-    "\3\2\2\2>\u01e7\3\2\2\2@\u01f8\3\2\2\2BD\5\4\3\2CB\3\2\2\2DG\3\2\2\2E"+
-    "C\3\2\2\2EF\3\2\2\2FK\3\2\2\2GE\3\2\2\2HJ\5\b\5\2IH\3\2\2\2JM\3\2\2\2"+
-    "KI\3\2\2\2KL\3\2\2\2LN\3\2\2\2MK\3\2\2\2NO\7\2\2\3O\3\3\2\2\2PQ\5\26\f"+
-    "\2QR\7R\2\2RS\5\6\4\2ST\5\f\7\2T\5\3\2\2\2Ua\7\t\2\2VW\5\26\f\2W^\7R\2"+
-    "\2XY\7\f\2\2YZ\5\26\f\2Z[\7R\2\2[]\3\2\2\2\\X\3\2\2\2]`\3\2\2\2^\\\3\2"+
-    "\2\2^_\3\2\2\2_b\3\2\2\2`^\3\2\2\2aV\3\2\2\2ab\3\2\2\2bc\3\2\2\2cd\7\n"+
-    "\2\2d\7\3\2\2\2ef\7\16\2\2fg\7\t\2\2gh\5\36\20\2hi\7\n\2\2im\5\n\6\2j"+
-    "k\7\20\2\2kn\5\n\6\2ln\6\5\2\2mj\3\2\2\2ml\3\2\2\2n\u00bc\3\2\2\2op\7"+
-    "\21\2\2pq\7\t\2\2qr\5\36\20\2ru\7\n\2\2sv\5\n\6\2tv\5\16\b\2us\3\2\2\2"+
-    "ut\3\2\2\2v\u00bc\3\2\2\2wx\7\22\2\2xy\5\f\7\2yz\7\21\2\2z{\7\t\2\2{|"+
-    "\5\36\20\2|}\7\n\2\2}~\5\34\17\2~\u00bc\3\2\2\2\177\u0080\7\23\2\2\u0080"+
-    "\u0082\7\t\2\2\u0081\u0083\5\20\t\2\u0082\u0081\3\2\2\2\u0082\u0083\3"+
-    "\2\2\2\u0083\u0084\3\2\2\2\u0084\u0086\7\r\2\2\u0085\u0087\5\36\20\2\u0086"+
-    "\u0085\3\2\2\2\u0086\u0087\3\2\2\2\u0087\u0088\3\2\2\2\u0088\u008a\7\r"+
-    "\2\2\u0089\u008b\5\22\n\2\u008a\u0089\3\2\2\2\u008a\u008b\3\2\2\2\u008b"+
-    "\u008c\3\2\2\2\u008c\u008f\7\n\2\2\u008d\u0090\5\n\6\2\u008e\u0090\5\16"+
-    "\b\2\u008f\u008d\3\2\2\2\u008f\u008e\3\2\2\2\u0090\u00bc\3\2\2\2\u0091"+
-    "\u0092\7\23\2\2\u0092\u0093\7\t\2\2\u0093\u0094\5\26\f\2\u0094\u0095\7"+
-    "R\2\2\u0095\u0096\7\65\2\2\u0096\u0097\5\36\20\2\u0097\u0098\7\n\2\2\u0098"+
-    "\u0099\5\n\6\2\u0099\u00bc\3\2\2\2\u009a\u009b\7\23\2\2\u009b\u009c\7"+
-    "\t\2\2\u009c\u009d\7R\2\2\u009d\u009e\7\17\2\2\u009e\u009f\5\36\20\2\u009f"+
-    "\u00a0\7\n\2\2\u00a0\u00a1\5\n\6\2\u00a1\u00bc\3\2\2\2\u00a2\u00a3\5\24"+
-    "\13\2\u00a3\u00a4\5\34\17\2\u00a4\u00bc\3\2\2\2\u00a5\u00a6\7\24\2\2\u00a6"+
-    "\u00bc\5\34\17\2\u00a7\u00a8\7\25\2\2\u00a8\u00bc\5\34\17\2\u00a9\u00aa"+
-    "\7\26\2\2\u00aa\u00ab\5\36\20\2\u00ab\u00ac\5\34\17\2\u00ac\u00bc\3\2"+
-    "\2\2\u00ad\u00ae\7\30\2\2\u00ae\u00b0\5\f\7\2\u00af\u00b1\5\32\16\2\u00b0"+
-    "\u00af\3\2\2\2\u00b1\u00b2\3\2\2\2\u00b2\u00b0\3\2\2\2\u00b2\u00b3\3\2"+
-    "\2\2\u00b3\u00bc\3\2\2\2\u00b4\u00b5\7\32\2\2\u00b5\u00b6\5\36\20\2\u00b6"+
-    "\u00b7\5\34\17\2\u00b7\u00bc\3\2\2\2\u00b8\u00b9\5\36\20\2\u00b9\u00ba"+
-    "\5\34\17\2\u00ba\u00bc\3\2\2\2\u00bbe\3\2\2\2\u00bbo\3\2\2\2\u00bbw\3"+
-    "\2\2\2\u00bb\177\3\2\2\2\u00bb\u0091\3\2\2\2\u00bb\u009a\3\2\2\2\u00bb"+
-    "\u00a2\3\2\2\2\u00bb\u00a5\3\2\2\2\u00bb\u00a7\3\2\2\2\u00bb\u00a9\3\2"+
-    "\2\2\u00bb\u00ad\3\2\2\2\u00bb\u00b4\3\2\2\2\u00bb\u00b8\3\2\2\2\u00bc"+
-    "\t\3\2\2\2\u00bd\u00c0\5\f\7\2\u00be\u00c0\5\b\5\2\u00bf\u00bd\3\2\2\2"+
-    "\u00bf\u00be\3\2\2\2\u00c0\13\3\2\2\2\u00c1\u00c5\7\5\2\2\u00c2\u00c4"+
-    "\5\b\5\2\u00c3\u00c2\3\2\2\2\u00c4\u00c7\3\2\2\2\u00c5\u00c3\3\2\2\2\u00c5"+
-    "\u00c6\3\2\2\2\u00c6\u00c8\3\2\2\2\u00c7\u00c5\3\2\2\2\u00c8\u00c9\7\6"+
-    "\2\2\u00c9\r\3\2\2\2\u00ca\u00cb\7\r\2\2\u00cb\17\3\2\2\2\u00cc\u00cf"+
-    "\5\24\13\2\u00cd\u00cf\5\36\20\2\u00ce\u00cc\3\2\2\2\u00ce\u00cd\3\2\2"+
-    "\2\u00cf\21\3\2\2\2\u00d0\u00d1\5\36\20\2\u00d1\23\3\2\2\2\u00d2\u00d3"+
-    "\5\26\f\2\u00d3\u00d8\5\30\r\2\u00d4\u00d5\7\f\2\2\u00d5\u00d7\5\30\r"+
-    "\2\u00d6\u00d4\3\2\2\2\u00d7\u00da\3\2\2\2\u00d8\u00d6\3\2\2\2\u00d8\u00d9"+
-    "\3\2\2\2\u00d9\25\3\2\2\2\u00da\u00d8\3\2\2\2\u00db\u00e0\7Q\2\2\u00dc"+
-    "\u00dd\7\7\2\2\u00dd\u00df\7\b\2\2\u00de\u00dc\3\2\2\2\u00df\u00e2\3\2"+
-    "\2\2\u00e0\u00de\3\2\2\2\u00e0\u00e1\3\2\2\2\u00e1\27\3\2\2\2\u00e2\u00e0"+
-    "\3\2\2\2\u00e3\u00e6\7R\2\2\u00e4\u00e5\7<\2\2\u00e5\u00e7\5\36\20\2\u00e6"+
-    "\u00e4\3\2\2\2\u00e6\u00e7\3\2\2\2\u00e7\31\3\2\2\2\u00e8\u00e9\7\31\2"+
-    "\2\u00e9\u00ea\7\t\2\2\u00ea\u00eb\7Q\2\2\u00eb\u00ec\7R\2\2\u00ec\u00ed"+
-    "\7\n\2\2\u00ed\u00ee\5\f\7\2\u00ee\33\3\2\2\2\u00ef\u00f0\t\2\2\2\u00f0"+
-    "\35\3\2\2\2\u00f1\u00f2\b\20\1\2\u00f2\u00f3\5 \21\2\u00f3\u0123\3\2\2"+
-    "\2\u00f4\u00f5\f\20\2\2\u00f5\u00f6\t\3\2\2\u00f6\u0122\5\36\20\21\u00f7"+
-    "\u00f8\f\17\2\2\u00f8\u00f9\t\4\2\2\u00f9\u0122\5\36\20\20\u00fa\u00fb"+
-    "\f\16\2\2\u00fb\u00fc\t\5\2\2\u00fc\u0122\5\36\20\17\u00fd\u00fe\f\r\2"+
-    "\2\u00fe\u00ff\t\6\2\2\u00ff\u0122\5\36\20\16\u0100\u0101\f\f\2\2\u0101"+
-    "\u0102\t\7\2\2\u0102\u0122\5\36\20\r\u0103\u0104\f\n\2\2\u0104\u0105\t"+
-    "\b\2\2\u0105\u0122\5\36\20\13\u0106\u0107\f\t\2\2\u0107\u0108\7/\2\2\u0108"+
-    "\u0122\5\36\20\n\u0109\u010a\f\b\2\2\u010a\u010b\7\60\2\2\u010b\u0122"+
-    "\5\36\20\t\u010c\u010d\f\7\2\2\u010d\u010e\7\61\2\2\u010e\u0122\5\36\20"+
-    "\b\u010f\u0110\f\6\2\2\u0110\u0111\7\62\2\2\u0111\u0122\5\36\20\7\u0112"+
-    "\u0113\f\5\2\2\u0113\u0114\7\63\2\2\u0114\u0122\5\36\20\6\u0115\u0116"+
-    "\f\4\2\2\u0116\u0117\7\64\2\2\u0117\u0118\5\36\20\2\u0118\u0119\7\65\2"+
-    "\2\u0119\u011a\5\36\20\4\u011a\u0122\3\2\2\2\u011b\u011c\f\3\2\2\u011c"+
-    "\u011d\t\t\2\2\u011d\u0122\5\36\20\3\u011e\u011f\f\13\2\2\u011f\u0120"+
-    "\7\34\2\2\u0120\u0122\5\26\f\2\u0121\u00f4\3\2\2\2\u0121\u00f7\3\2\2\2"+
-    "\u0121\u00fa\3\2\2\2\u0121\u00fd\3\2\2\2\u0121\u0100\3\2\2\2\u0121\u0103"+
-    "\3\2\2\2\u0121\u0106\3\2\2\2\u0121\u0109\3\2\2\2\u0121\u010c\3\2\2\2\u0121"+
-    "\u010f\3\2\2\2\u0121\u0112\3\2\2\2\u0121\u0115\3\2\2\2\u0121\u011b\3\2"+
-    "\2\2\u0121\u011e\3\2\2\2\u0122\u0125\3\2\2\2\u0123\u0121\3\2\2\2\u0123"+
-    "\u0124\3\2\2\2\u0124\37\3\2\2\2\u0125\u0123\3\2\2\2\u0126\u0127\t\n\2"+
-    "\2\u0127\u0134\5\"\22\2\u0128\u0129\5\"\22\2\u0129\u012a\t\n\2\2\u012a"+
-    "\u0134\3\2\2\2\u012b\u0134\5\"\22\2\u012c\u012d\t\13\2\2\u012d\u0134\5"+
-    " \21\2\u012e\u012f\7\t\2\2\u012f\u0130\5\26\f\2\u0130\u0131\7\n\2\2\u0131"+
-    "\u0132\5 \21\2\u0132\u0134\3\2\2\2\u0133\u0126\3\2\2\2\u0133\u0128\3\2"+
-    "\2\2\u0133\u012b\3\2\2\2\u0133\u012c\3\2\2\2\u0133\u012e\3\2\2\2\u0134"+
-    "!\3\2\2\2\u0135\u0139\5$\23\2\u0136\u0138\5&\24\2\u0137\u0136\3\2\2\2"+
-    "\u0138\u013b\3\2\2\2\u0139\u0137\3\2\2\2\u0139\u013a\3\2\2\2\u013a\u0146"+
-    "\3\2\2\2\u013b\u0139\3\2\2\2\u013c\u013d\5\26\f\2\u013d\u0141\5(\25\2"+
-    "\u013e\u0140\5&\24\2\u013f\u013e\3\2\2\2\u0140\u0143\3\2\2\2\u0141\u013f"+
-    "\3\2\2\2\u0141\u0142\3\2\2\2\u0142\u0146\3\2\2\2\u0143\u0141\3\2\2\2\u0144"+
-    "\u0146\5\60\31\2\u0145\u0135\3\2\2\2\u0145\u013c\3\2\2\2\u0145\u0144\3"+
-    "\2\2\2\u0146#\3\2\2\2\u0147\u0148\7\t\2\2\u0148\u0149\5\36\20\2\u0149"+
-    "\u014a\7\n\2\2\u014a\u015a\3\2\2\2\u014b\u015a\t\f\2\2\u014c\u015a\7N"+
-    "\2\2\u014d\u015a\7O\2\2\u014e\u015a\7P\2\2\u014f\u015a\7L\2\2\u0150\u015a"+
-    "\7M\2\2\u0151\u015a\5\62\32\2\u0152\u015a\5\64\33\2\u0153\u015a\7R\2\2"+
-    "\u0154\u0155\7R\2\2\u0155\u015a\58\35\2\u0156\u0157\7\27\2\2\u0157\u0158"+
-    "\7Q\2\2\u0158\u015a\58\35\2\u0159\u0147\3\2\2\2\u0159\u014b\3\2\2\2\u0159"+
-    "\u014c\3\2\2\2\u0159\u014d\3\2\2\2\u0159\u014e\3\2\2\2\u0159\u014f\3\2"+
-    "\2\2\u0159\u0150\3\2\2\2\u0159\u0151\3\2\2\2\u0159\u0152\3\2\2\2\u0159"+
-    "\u0153\3\2\2\2\u0159\u0154\3\2\2\2\u0159\u0156\3\2\2\2\u015a%\3\2\2\2"+
-    "\u015b\u015f\5*\26\2\u015c\u015f\5,\27\2\u015d\u015f\5.\30\2\u015e\u015b"+
-    "\3\2\2\2\u015e\u015c\3\2\2\2\u015e\u015d\3\2\2\2\u015f\'\3\2\2\2\u0160"+
-    "\u0163\5*\26\2\u0161\u0163\5,\27\2\u0162\u0160\3\2\2\2\u0162\u0161\3\2"+
-    "\2\2\u0163)\3\2\2\2\u0164\u0165\7\13\2\2\u0165\u0166\7T\2\2\u0166\u0167"+
-    "\58\35\2\u0167+\3\2\2\2\u0168\u0169\7\13\2\2\u0169\u016a\t\r\2\2\u016a"+
-    "-\3\2\2\2\u016b\u016c\7\7\2\2\u016c\u016d\5\36\20\2\u016d\u016e\7\b\2"+
-    "\2\u016e/\3\2\2\2\u016f\u0170\7\27\2\2\u0170\u0175\7Q\2\2\u0171\u0172"+
-    "\7\7\2\2\u0172\u0173\5\36\20\2\u0173\u0174\7\b\2\2\u0174\u0176\3\2\2\2"+
-    "\u0175\u0171\3\2\2\2\u0176\u0177\3\2\2\2\u0177\u0175\3\2\2\2\u0177\u0178"+
-    "\3\2\2\2\u0178\u0180\3\2\2\2\u0179\u017d\5(\25\2\u017a\u017c\5&\24\2\u017b"+
-    "\u017a\3\2\2\2\u017c\u017f\3\2\2\2\u017d\u017b\3\2\2\2\u017d\u017e\3\2"+
-    "\2\2\u017e\u0181\3\2\2\2\u017f\u017d\3\2\2\2\u0180\u0179\3\2\2\2\u0180"+
-    "\u0181\3\2\2\2\u0181\u019c\3\2\2\2\u0182\u0183\7\27\2\2\u0183\u0184\7"+
-    "Q\2\2\u0184\u0185\7\7\2\2\u0185\u0186\7\b\2\2\u0186\u018f\7\5\2\2\u0187"+
-    "\u018c\5\36\20\2\u0188\u0189\7\f\2\2\u0189\u018b\5\36\20\2\u018a\u0188"+
-    "\3\2\2\2\u018b\u018e\3\2\2\2\u018c\u018a\3\2\2\2\u018c\u018d\3\2\2\2\u018d"+
-    "\u0190\3\2\2\2\u018e\u018c\3\2\2\2\u018f\u0187\3\2\2\2\u018f\u0190\3\2"+
-    "\2\2\u0190\u0192\3\2\2\2\u0191\u0193\7\r\2\2\u0192\u0191\3\2\2\2\u0192"+
-    "\u0193\3\2\2\2\u0193\u0194\3\2\2\2\u0194\u0198\7\6\2\2\u0195\u0197\5&"+
-    "\24\2\u0196\u0195\3\2\2\2\u0197\u019a\3\2\2\2\u0198\u0196\3\2\2\2\u0198"+
-    "\u0199\3\2\2\2\u0199\u019c\3\2\2\2\u019a\u0198\3\2\2\2\u019b\u016f\3\2"+
-    "\2\2\u019b\u0182\3\2\2\2\u019c\61\3\2\2\2\u019d\u019e\7\7\2\2\u019e\u01a3"+
-    "\5\36\20\2\u019f\u01a0\7\f\2\2\u01a0\u01a2\5\36\20\2\u01a1\u019f\3\2\2"+
-    "\2\u01a2\u01a5\3\2\2\2\u01a3\u01a1\3\2\2\2\u01a3\u01a4\3\2\2\2\u01a4\u01a6"+
-    "\3\2\2\2\u01a5\u01a3\3\2\2\2\u01a6\u01a7\7\b\2\2\u01a7\u01ab\3\2\2\2\u01a8"+
-    "\u01a9\7\7\2\2\u01a9\u01ab\7\b\2\2\u01aa\u019d\3\2\2\2\u01aa\u01a8\3\2"+
-    "\2\2\u01ab\63\3\2\2\2\u01ac\u01ad\7\7\2\2\u01ad\u01b2\5\66\34\2\u01ae"+
-    "\u01af\7\f\2\2\u01af\u01b1\5\66\34\2\u01b0\u01ae\3\2\2\2\u01b1\u01b4\3"+
-    "\2\2\2\u01b2\u01b0\3\2\2\2\u01b2\u01b3\3\2\2\2\u01b3\u01b5\3\2\2\2\u01b4"+
-    "\u01b2\3\2\2\2\u01b5\u01b6\7\b\2\2\u01b6\u01bb\3\2\2\2\u01b7\u01b8\7\7"+
-    "\2\2\u01b8\u01b9\7\65\2\2\u01b9\u01bb\7\b\2\2\u01ba\u01ac\3\2\2\2\u01ba"+
-    "\u01b7\3\2\2\2\u01bb\65\3\2\2\2\u01bc\u01bd\5\36\20\2\u01bd\u01be\7\65"+
-    "\2\2\u01be\u01bf\5\36\20\2\u01bf\67\3\2\2\2\u01c0\u01c9\7\t\2\2\u01c1"+
-    "\u01c6\5:\36\2\u01c2\u01c3\7\f\2\2\u01c3\u01c5\5:\36\2\u01c4\u01c2\3\2"+
-    "\2\2\u01c5\u01c8\3\2\2\2\u01c6\u01c4\3\2\2\2\u01c6\u01c7\3\2\2\2\u01c7"+
-    "\u01ca\3\2\2\2\u01c8\u01c6\3\2\2\2\u01c9\u01c1\3\2\2\2\u01c9\u01ca\3\2"+
-    "\2\2\u01ca\u01cb\3\2\2\2\u01cb\u01cc\7\n\2\2\u01cc9\3\2\2\2\u01cd\u01d1"+
-    "\5\36\20\2\u01ce\u01d1\5<\37\2\u01cf\u01d1\5@!\2\u01d0\u01cd\3\2\2\2\u01d0"+
-    "\u01ce\3\2\2\2\u01d0\u01cf\3\2\2\2\u01d1;\3\2\2\2\u01d2\u01e0\5> \2\u01d3"+
-    "\u01dc\7\t\2\2\u01d4\u01d9\5> \2\u01d5\u01d6\7\f\2\2\u01d6\u01d8\5> \2"+
-    "\u01d7\u01d5\3\2\2\2\u01d8\u01db\3\2\2\2\u01d9\u01d7\3\2\2\2\u01d9\u01da"+
-    "\3\2\2\2\u01da\u01dd\3\2\2\2\u01db\u01d9\3\2\2\2\u01dc\u01d4\3\2\2\2\u01dc"+
-    "\u01dd\3\2\2\2\u01dd\u01de\3\2\2\2\u01de\u01e0\7\n\2\2\u01df\u01d2\3\2"+
-    "\2\2\u01df\u01d3\3\2\2\2\u01e0\u01e1\3\2\2\2\u01e1\u01e4\7\67\2\2\u01e2"+
-    "\u01e5\5\f\7\2\u01e3\u01e5\5\36\20\2\u01e4\u01e2\3\2\2\2\u01e4\u01e3\3"+
-    "\2\2\2\u01e5=\3\2\2\2\u01e6\u01e8\5\26\f\2\u01e7\u01e6\3\2\2\2\u01e7\u01e8"+
-    "\3\2\2\2\u01e8\u01e9\3\2\2\2\u01e9\u01ea\7R\2\2\u01ea?\3\2\2\2\u01eb\u01ec"+
-    "\7Q\2\2\u01ec\u01ed\7\66\2\2\u01ed\u01f9\7R\2\2\u01ee\u01ef\5\26\f\2\u01ef"+
-    "\u01f0\7\66\2\2\u01f0\u01f1\7\27\2\2\u01f1\u01f9\3\2\2\2\u01f2\u01f3\7"+
-    "R\2\2\u01f3\u01f4\7\66\2\2\u01f4\u01f9\7R\2\2\u01f5\u01f6\7\33\2\2\u01f6"+
-    "\u01f7\7\66\2\2\u01f7\u01f9\7R\2\2\u01f8\u01eb\3\2\2\2\u01f8\u01ee\3\2"+
-    "\2\2\u01f8\u01f2\3\2\2\2\u01f8\u01f5\3\2\2\2\u01f9A\3\2\2\2\62EK^amu\u0082"+
-    "\u0086\u008a\u008f\u00b2\u00bb\u00bf\u00c5\u00ce\u00d8\u00e0\u00e6\u0121"+
-    "\u0123\u0133\u0139\u0141\u0145\u0159\u015e\u0162\u0177\u017d\u0180\u018c"+
-    "\u018f\u0192\u0198\u019b\u01a3\u01aa\u01b2\u01ba\u01c6\u01c9\u01d0\u01d9"+
-    "\u01dc\u01df\u01e4\u01e7\u01f8";
+    "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\7\20\u0125\n\20\f\20\16"+
+    "\20\u0128\13\20\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21"+
+    "\3\21\3\21\5\21\u0137\n\21\3\22\3\22\7\22\u013b\n\22\f\22\16\22\u013e"+
+    "\13\22\3\22\3\22\3\22\7\22\u0143\n\22\f\22\16\22\u0146\13\22\3\22\5\22"+
+    "\u0149\n\22\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23"+
+    "\3\23\3\23\3\23\3\23\3\23\3\23\5\23\u015d\n\23\3\24\3\24\3\24\5\24\u0162"+
+    "\n\24\3\25\3\25\5\25\u0166\n\25\3\26\3\26\3\26\3\26\3\27\3\27\3\27\3\30"+
+    "\3\30\3\30\3\30\3\31\3\31\3\31\3\31\3\31\3\31\6\31\u0179\n\31\r\31\16"+
+    "\31\u017a\3\31\3\31\7\31\u017f\n\31\f\31\16\31\u0182\13\31\5\31\u0184"+
+    "\n\31\3\31\3\31\3\31\3\31\3\31\3\31\3\31\3\31\7\31\u018e\n\31\f\31\16"+
+    "\31\u0191\13\31\5\31\u0193\n\31\3\31\5\31\u0196\n\31\3\31\3\31\7\31\u019a"+
+    "\n\31\f\31\16\31\u019d\13\31\5\31\u019f\n\31\3\32\3\32\3\32\3\32\7\32"+
+    "\u01a5\n\32\f\32\16\32\u01a8\13\32\3\32\3\32\3\32\3\32\5\32\u01ae\n\32"+
+    "\3\33\3\33\3\33\3\33\7\33\u01b4\n\33\f\33\16\33\u01b7\13\33\3\33\3\33"+
+    "\3\33\3\33\3\33\5\33\u01be\n\33\3\34\3\34\3\34\3\34\3\35\3\35\3\35\3\35"+
+    "\7\35\u01c8\n\35\f\35\16\35\u01cb\13\35\5\35\u01cd\n\35\3\35\3\35\3\36"+
+    "\3\36\3\36\5\36\u01d4\n\36\3\37\3\37\3\37\3\37\3\37\7\37\u01db\n\37\f"+
+    "\37\16\37\u01de\13\37\5\37\u01e0\n\37\3\37\5\37\u01e3\n\37\3\37\3\37\3"+
+    "\37\5\37\u01e8\n\37\3 \5 \u01eb\n \3 \3 \3!\3!\3!\3!\3!\3!\3!\3!\3!\3"+
+    "!\3!\3!\3!\5!\u01fc\n!\3!\2\3\36\"\2\4\6\b\n\f\16\20\22\24\26\30\32\34"+
+    "\36 \"$&(*,.\60\62\64\668:<>@\2\17\3\3\16\16\3\2 \"\3\2#$\3\2:;\3\2%\'"+
+    "\3\2(+\3\2,/\3\2>I\3\2<=\4\2\36\37#$\3\2JM\3\2\13\f\3\2UV\u0237\2E\3\2"+
+    "\2\2\4P\3\2\2\2\6U\3\2\2\2\b\u00bb\3\2\2\2\n\u00bf\3\2\2\2\f\u00c1\3\2"+
+    "\2\2\16\u00ca\3\2\2\2\20\u00ce\3\2\2\2\22\u00d0\3\2\2\2\24\u00d2\3\2\2"+
+    "\2\26\u00db\3\2\2\2\30\u00e3\3\2\2\2\32\u00e8\3\2\2\2\34\u00ef\3\2\2\2"+
+    "\36\u00f1\3\2\2\2 \u0136\3\2\2\2\"\u0148\3\2\2\2$\u015c\3\2\2\2&\u0161"+
+    "\3\2\2\2(\u0165\3\2\2\2*\u0167\3\2\2\2,\u016b\3\2\2\2.\u016e\3\2\2\2\60"+
+    "\u019e\3\2\2\2\62\u01ad\3\2\2\2\64\u01bd\3\2\2\2\66\u01bf\3\2\2\28\u01c3"+
+    "\3\2\2\2:\u01d3\3\2\2\2<\u01e2\3\2\2\2>\u01ea\3\2\2\2@\u01fb\3\2\2\2B"+
+    "D\5\4\3\2CB\3\2\2\2DG\3\2\2\2EC\3\2\2\2EF\3\2\2\2FK\3\2\2\2GE\3\2\2\2"+
+    "HJ\5\b\5\2IH\3\2\2\2JM\3\2\2\2KI\3\2\2\2KL\3\2\2\2LN\3\2\2\2MK\3\2\2\2"+
+    "NO\7\2\2\3O\3\3\2\2\2PQ\5\26\f\2QR\7T\2\2RS\5\6\4\2ST\5\f\7\2T\5\3\2\2"+
+    "\2Ua\7\t\2\2VW\5\26\f\2W^\7T\2\2XY\7\r\2\2YZ\5\26\f\2Z[\7T\2\2[]\3\2\2"+
+    "\2\\X\3\2\2\2]`\3\2\2\2^\\\3\2\2\2^_\3\2\2\2_b\3\2\2\2`^\3\2\2\2aV\3\2"+
+    "\2\2ab\3\2\2\2bc\3\2\2\2cd\7\n\2\2d\7\3\2\2\2ef\7\17\2\2fg\7\t\2\2gh\5"+
+    "\36\20\2hi\7\n\2\2im\5\n\6\2jk\7\21\2\2kn\5\n\6\2ln\6\5\2\2mj\3\2\2\2"+
+    "ml\3\2\2\2n\u00bc\3\2\2\2op\7\22\2\2pq\7\t\2\2qr\5\36\20\2ru\7\n\2\2s"+
+    "v\5\n\6\2tv\5\16\b\2us\3\2\2\2ut\3\2\2\2v\u00bc\3\2\2\2wx\7\23\2\2xy\5"+
+    "\f\7\2yz\7\22\2\2z{\7\t\2\2{|\5\36\20\2|}\7\n\2\2}~\5\34\17\2~\u00bc\3"+
+    "\2\2\2\177\u0080\7\24\2\2\u0080\u0082\7\t\2\2\u0081\u0083\5\20\t\2\u0082"+
+    "\u0081\3\2\2\2\u0082\u0083\3\2\2\2\u0083\u0084\3\2\2\2\u0084\u0086\7\16"+
+    "\2\2\u0085\u0087\5\36\20\2\u0086\u0085\3\2\2\2\u0086\u0087\3\2\2\2\u0087"+
+    "\u0088\3\2\2\2\u0088\u008a\7\16\2\2\u0089\u008b\5\22\n\2\u008a\u0089\3"+
+    "\2\2\2\u008a\u008b\3\2\2\2\u008b\u008c\3\2\2\2\u008c\u008f\7\n\2\2\u008d"+
+    "\u0090\5\n\6\2\u008e\u0090\5\16\b\2\u008f\u008d\3\2\2\2\u008f\u008e\3"+
+    "\2\2\2\u0090\u00bc\3\2\2\2\u0091\u0092\7\24\2\2\u0092\u0093\7\t\2\2\u0093"+
+    "\u0094\5\26\f\2\u0094\u0095\7T\2\2\u0095\u0096\7\66\2\2\u0096\u0097\5"+
+    "\36\20\2\u0097\u0098\7\n\2\2\u0098\u0099\5\n\6\2\u0099\u00bc\3\2\2\2\u009a"+
+    "\u009b\7\24\2\2\u009b\u009c\7\t\2\2\u009c\u009d\7T\2\2\u009d\u009e\7\20"+
+    "\2\2\u009e\u009f\5\36\20\2\u009f\u00a0\7\n\2\2\u00a0\u00a1\5\n\6\2\u00a1"+
+    "\u00bc\3\2\2\2\u00a2\u00a3\5\24\13\2\u00a3\u00a4\5\34\17\2\u00a4\u00bc"+
+    "\3\2\2\2\u00a5\u00a6\7\25\2\2\u00a6\u00bc\5\34\17\2\u00a7\u00a8\7\26\2"+
+    "\2\u00a8\u00bc\5\34\17\2\u00a9\u00aa\7\27\2\2\u00aa\u00ab\5\36\20\2\u00ab"+
+    "\u00ac\5\34\17\2\u00ac\u00bc\3\2\2\2\u00ad\u00ae\7\31\2\2\u00ae\u00b0"+
+    "\5\f\7\2\u00af\u00b1\5\32\16\2\u00b0\u00af\3\2\2\2\u00b1\u00b2\3\2\2\2"+
+    "\u00b2\u00b0\3\2\2\2\u00b2\u00b3\3\2\2\2\u00b3\u00bc\3\2\2\2\u00b4\u00b5"+
+    "\7\33\2\2\u00b5\u00b6\5\36\20\2\u00b6\u00b7\5\34\17\2\u00b7\u00bc\3\2"+
+    "\2\2\u00b8\u00b9\5\36\20\2\u00b9\u00ba\5\34\17\2\u00ba\u00bc\3\2\2\2\u00bb"+
+    "e\3\2\2\2\u00bbo\3\2\2\2\u00bbw\3\2\2\2\u00bb\177\3\2\2\2\u00bb\u0091"+
+    "\3\2\2\2\u00bb\u009a\3\2\2\2\u00bb\u00a2\3\2\2\2\u00bb\u00a5\3\2\2\2\u00bb"+
+    "\u00a7\3\2\2\2\u00bb\u00a9\3\2\2\2\u00bb\u00ad\3\2\2\2\u00bb\u00b4\3\2"+
+    "\2\2\u00bb\u00b8\3\2\2\2\u00bc\t\3\2\2\2\u00bd\u00c0\5\f\7\2\u00be\u00c0"+
+    "\5\b\5\2\u00bf\u00bd\3\2\2\2\u00bf\u00be\3\2\2\2\u00c0\13\3\2\2\2\u00c1"+
+    "\u00c5\7\5\2\2\u00c2\u00c4\5\b\5\2\u00c3\u00c2\3\2\2\2\u00c4\u00c7\3\2"+
+    "\2\2\u00c5\u00c3\3\2\2\2\u00c5\u00c6\3\2\2\2\u00c6\u00c8\3\2\2\2\u00c7"+
+    "\u00c5\3\2\2\2\u00c8\u00c9\7\6\2\2\u00c9\r\3\2\2\2\u00ca\u00cb\7\16\2"+
+    "\2\u00cb\17\3\2\2\2\u00cc\u00cf\5\24\13\2\u00cd\u00cf\5\36\20\2\u00ce"+
+    "\u00cc\3\2\2\2\u00ce\u00cd\3\2\2\2\u00cf\21\3\2\2\2\u00d0\u00d1\5\36\20"+
+    "\2\u00d1\23\3\2\2\2\u00d2\u00d3\5\26\f\2\u00d3\u00d8\5\30\r\2\u00d4\u00d5"+
+    "\7\r\2\2\u00d5\u00d7\5\30\r\2\u00d6\u00d4\3\2\2\2\u00d7\u00da\3\2\2\2"+
+    "\u00d8\u00d6\3\2\2\2\u00d8\u00d9\3\2\2\2\u00d9\25\3\2\2\2\u00da\u00d8"+
+    "\3\2\2\2\u00db\u00e0\7S\2\2\u00dc\u00dd\7\7\2\2\u00dd\u00df\7\b\2\2\u00de"+
+    "\u00dc\3\2\2\2\u00df\u00e2\3\2\2\2\u00e0\u00de\3\2\2\2\u00e0\u00e1\3\2"+
+    "\2\2\u00e1\27\3\2\2\2\u00e2\u00e0\3\2\2\2\u00e3\u00e6\7T\2\2\u00e4\u00e5"+
+    "\7>\2\2\u00e5\u00e7\5\36\20\2\u00e6\u00e4\3\2\2\2\u00e6\u00e7\3\2\2\2"+
+    "\u00e7\31\3\2\2\2\u00e8\u00e9\7\32\2\2\u00e9\u00ea\7\t\2\2\u00ea\u00eb"+
+    "\7S\2\2\u00eb\u00ec\7T\2\2\u00ec\u00ed\7\n\2\2\u00ed\u00ee\5\f\7\2\u00ee"+
+    "\33\3\2\2\2\u00ef\u00f0\t\2\2\2\u00f0\35\3\2\2\2\u00f1\u00f2\b\20\1\2"+
+    "\u00f2\u00f3\5 \21\2\u00f3\u0126\3\2\2\2\u00f4\u00f5\f\21\2\2\u00f5\u00f6"+
+    "\t\3\2\2\u00f6\u0125\5\36\20\22\u00f7\u00f8\f\20\2\2\u00f8\u00f9\t\4\2"+
+    "\2\u00f9\u0125\5\36\20\21\u00fa\u00fb\f\17\2\2\u00fb\u00fc\t\5\2\2\u00fc"+
+    "\u0125\5\36\20\20\u00fd\u00fe\f\16\2\2\u00fe\u00ff\t\6\2\2\u00ff\u0125"+
+    "\5\36\20\17\u0100\u0101\f\r\2\2\u0101\u0102\t\7\2\2\u0102\u0125\5\36\20"+
+    "\16\u0103\u0104\f\13\2\2\u0104\u0105\t\b\2\2\u0105\u0125\5\36\20\f\u0106"+
+    "\u0107\f\n\2\2\u0107\u0108\7\60\2\2\u0108\u0125\5\36\20\13\u0109\u010a"+
+    "\f\t\2\2\u010a\u010b\7\61\2\2\u010b\u0125\5\36\20\n\u010c\u010d\f\b\2"+
+    "\2\u010d\u010e\7\62\2\2\u010e\u0125\5\36\20\t\u010f\u0110\f\7\2\2\u0110"+
+    "\u0111\7\63\2\2\u0111\u0125\5\36\20\b\u0112\u0113\f\6\2\2\u0113\u0114"+
+    "\7\64\2\2\u0114\u0125\5\36\20\7\u0115\u0116\f\5\2\2\u0116\u0117\7\65\2"+
+    "\2\u0117\u0118\5\36\20\2\u0118\u0119\7\66\2\2\u0119\u011a\5\36\20\5\u011a"+
+    "\u0125\3\2\2\2\u011b\u011c\f\4\2\2\u011c\u011d\7\67\2\2\u011d\u0125\5"+
+    "\36\20\4\u011e\u011f\f\3\2\2\u011f\u0120\t\t\2\2\u0120\u0125\5\36\20\3"+
+    "\u0121\u0122\f\f\2\2\u0122\u0123\7\35\2\2\u0123\u0125\5\26\f\2\u0124\u00f4"+
+    "\3\2\2\2\u0124\u00f7\3\2\2\2\u0124\u00fa\3\2\2\2\u0124\u00fd\3\2\2\2\u0124"+
+    "\u0100\3\2\2\2\u0124\u0103\3\2\2\2\u0124\u0106\3\2\2\2\u0124\u0109\3\2"+
+    "\2\2\u0124\u010c\3\2\2\2\u0124\u010f\3\2\2\2\u0124\u0112\3\2\2\2\u0124"+
+    "\u0115\3\2\2\2\u0124\u011b\3\2\2\2\u0124\u011e\3\2\2\2\u0124\u0121\3\2"+
+    "\2\2\u0125\u0128\3\2\2\2\u0126\u0124\3\2\2\2\u0126\u0127\3\2\2\2\u0127"+
+    "\37\3\2\2\2\u0128\u0126\3\2\2\2\u0129\u012a\t\n\2\2\u012a\u0137\5\"\22"+
+    "\2\u012b\u012c\5\"\22\2\u012c\u012d\t\n\2\2\u012d\u0137\3\2\2\2\u012e"+
+    "\u0137\5\"\22\2\u012f\u0130\t\13\2\2\u0130\u0137\5 \21\2\u0131\u0132\7"+
+    "\t\2\2\u0132\u0133\5\26\f\2\u0133\u0134\7\n\2\2\u0134\u0135\5 \21\2\u0135"+
+    "\u0137\3\2\2\2\u0136\u0129\3\2\2\2\u0136\u012b\3\2\2\2\u0136\u012e\3\2"+
+    "\2\2\u0136\u012f\3\2\2\2\u0136\u0131\3\2\2\2\u0137!\3\2\2\2\u0138\u013c"+
+    "\5$\23\2\u0139\u013b\5&\24\2\u013a\u0139\3\2\2\2\u013b\u013e\3\2\2\2\u013c"+
+    "\u013a\3\2\2\2\u013c\u013d\3\2\2\2\u013d\u0149\3\2\2\2\u013e\u013c\3\2"+
+    "\2\2\u013f\u0140\5\26\f\2\u0140\u0144\5(\25\2\u0141\u0143\5&\24\2\u0142"+
+    "\u0141\3\2\2\2\u0143\u0146\3\2\2\2\u0144\u0142\3\2\2\2\u0144\u0145\3\2"+
+    "\2\2\u0145\u0149\3\2\2\2\u0146\u0144\3\2\2\2\u0147\u0149\5\60\31\2\u0148"+
+    "\u0138\3\2\2\2\u0148\u013f\3\2\2\2\u0148\u0147\3\2\2\2\u0149#\3\2\2\2"+
+    "\u014a\u014b\7\t\2\2\u014b\u014c\5\36\20\2\u014c\u014d\7\n\2\2\u014d\u015d"+
+    "\3\2\2\2\u014e\u015d\t\f\2\2\u014f\u015d\7P\2\2\u0150\u015d\7Q\2\2\u0151"+
+    "\u015d\7R\2\2\u0152\u015d\7N\2\2\u0153\u015d\7O\2\2\u0154\u015d\5\62\32"+
+    "\2\u0155\u015d\5\64\33\2\u0156\u015d\7T\2\2\u0157\u0158\7T\2\2\u0158\u015d"+
+    "\58\35\2\u0159\u015a\7\30\2\2\u015a\u015b\7S\2\2\u015b\u015d\58\35\2\u015c"+
+    "\u014a\3\2\2\2\u015c\u014e\3\2\2\2\u015c\u014f\3\2\2\2\u015c\u0150\3\2"+
+    "\2\2\u015c\u0151\3\2\2\2\u015c\u0152\3\2\2\2\u015c\u0153\3\2\2\2\u015c"+
+    "\u0154\3\2\2\2\u015c\u0155\3\2\2\2\u015c\u0156\3\2\2\2\u015c\u0157\3\2"+
+    "\2\2\u015c\u0159\3\2\2\2\u015d%\3\2\2\2\u015e\u0162\5*\26\2\u015f\u0162"+
+    "\5,\27\2\u0160\u0162\5.\30\2\u0161\u015e\3\2\2\2\u0161\u015f\3\2\2\2\u0161"+
+    "\u0160\3\2\2\2\u0162\'\3\2\2\2\u0163\u0166\5*\26\2\u0164\u0166\5,\27\2"+
+    "\u0165\u0163\3\2\2\2\u0165\u0164\3\2\2\2\u0166)\3\2\2\2\u0167\u0168\t"+
+    "\r\2\2\u0168\u0169\7V\2\2\u0169\u016a\58\35\2\u016a+\3\2\2\2\u016b\u016c"+
+    "\t\r\2\2\u016c\u016d\t\16\2\2\u016d-\3\2\2\2\u016e\u016f\7\7\2\2\u016f"+
+    "\u0170\5\36\20\2\u0170\u0171\7\b\2\2\u0171/\3\2\2\2\u0172\u0173\7\30\2"+
+    "\2\u0173\u0178\7S\2\2\u0174\u0175\7\7\2\2\u0175\u0176\5\36\20\2\u0176"+
+    "\u0177\7\b\2\2\u0177\u0179\3\2\2\2\u0178\u0174\3\2\2\2\u0179\u017a\3\2"+
+    "\2\2\u017a\u0178\3\2\2\2\u017a\u017b\3\2\2\2\u017b\u0183\3\2\2\2\u017c"+
+    "\u0180\5(\25\2\u017d\u017f\5&\24\2\u017e\u017d\3\2\2\2\u017f\u0182\3\2"+
+    "\2\2\u0180\u017e\3\2\2\2\u0180\u0181\3\2\2\2\u0181\u0184\3\2\2\2\u0182"+
+    "\u0180\3\2\2\2\u0183\u017c\3\2\2\2\u0183\u0184\3\2\2\2\u0184\u019f\3\2"+
+    "\2\2\u0185\u0186\7\30\2\2\u0186\u0187\7S\2\2\u0187\u0188\7\7\2\2\u0188"+
+    "\u0189\7\b\2\2\u0189\u0192\7\5\2\2\u018a\u018f\5\36\20\2\u018b\u018c\7"+
+    "\r\2\2\u018c\u018e\5\36\20\2\u018d\u018b\3\2\2\2\u018e\u0191\3\2\2\2\u018f"+
+    "\u018d\3\2\2\2\u018f\u0190\3\2\2\2\u0190\u0193\3\2\2\2\u0191\u018f\3\2"+
+    "\2\2\u0192\u018a\3\2\2\2\u0192\u0193\3\2\2\2\u0193\u0195\3\2\2\2\u0194"+
+    "\u0196\7\16\2\2\u0195\u0194\3\2\2\2\u0195\u0196\3\2\2\2\u0196\u0197\3"+
+    "\2\2\2\u0197\u019b\7\6\2\2\u0198\u019a\5&\24\2\u0199\u0198\3\2\2\2\u019a"+
+    "\u019d\3\2\2\2\u019b\u0199\3\2\2\2\u019b\u019c\3\2\2\2\u019c\u019f\3\2"+
+    "\2\2\u019d\u019b\3\2\2\2\u019e\u0172\3\2\2\2\u019e\u0185\3\2\2\2\u019f"+
+    "\61\3\2\2\2\u01a0\u01a1\7\7\2\2\u01a1\u01a6\5\36\20\2\u01a2\u01a3\7\r"+
+    "\2\2\u01a3\u01a5\5\36\20\2\u01a4\u01a2\3\2\2\2\u01a5\u01a8\3\2\2\2\u01a6"+
+    "\u01a4\3\2\2\2\u01a6\u01a7\3\2\2\2\u01a7\u01a9\3\2\2\2\u01a8\u01a6\3\2"+
+    "\2\2\u01a9\u01aa\7\b\2\2\u01aa\u01ae\3\2\2\2\u01ab\u01ac\7\7\2\2\u01ac"+
+    "\u01ae\7\b\2\2\u01ad\u01a0\3\2\2\2\u01ad\u01ab\3\2\2\2\u01ae\63\3\2\2"+
+    "\2\u01af\u01b0\7\7\2\2\u01b0\u01b5\5\66\34\2\u01b1\u01b2\7\r\2\2\u01b2"+
+    "\u01b4\5\66\34\2\u01b3\u01b1\3\2\2\2\u01b4\u01b7\3\2\2\2\u01b5\u01b3\3"+
+    "\2\2\2\u01b5\u01b6\3\2\2\2\u01b6\u01b8\3\2\2\2\u01b7\u01b5\3\2\2\2\u01b8"+
+    "\u01b9\7\b\2\2\u01b9\u01be\3\2\2\2\u01ba\u01bb\7\7\2\2\u01bb\u01bc\7\66"+
+    "\2\2\u01bc\u01be\7\b\2\2\u01bd\u01af\3\2\2\2\u01bd\u01ba\3\2\2\2\u01be"+
+    "\65\3\2\2\2\u01bf\u01c0\5\36\20\2\u01c0\u01c1\7\66\2\2\u01c1\u01c2\5\36"+
+    "\20\2\u01c2\67\3\2\2\2\u01c3\u01cc\7\t\2\2\u01c4\u01c9\5:\36\2\u01c5\u01c6"+
+    "\7\r\2\2\u01c6\u01c8\5:\36\2\u01c7\u01c5\3\2\2\2\u01c8\u01cb\3\2\2\2\u01c9"+
+    "\u01c7\3\2\2\2\u01c9\u01ca\3\2\2\2\u01ca\u01cd\3\2\2\2\u01cb\u01c9\3\2"+
+    "\2\2\u01cc\u01c4\3\2\2\2\u01cc\u01cd\3\2\2\2\u01cd\u01ce\3\2\2\2\u01ce"+
+    "\u01cf\7\n\2\2\u01cf9\3\2\2\2\u01d0\u01d4\5\36\20\2\u01d1\u01d4\5<\37"+
+    "\2\u01d2\u01d4\5@!\2\u01d3\u01d0\3\2\2\2\u01d3\u01d1\3\2\2\2\u01d3\u01d2"+
+    "\3\2\2\2\u01d4;\3\2\2\2\u01d5\u01e3\5> \2\u01d6\u01df\7\t\2\2\u01d7\u01dc"+
+    "\5> \2\u01d8\u01d9\7\r\2\2\u01d9\u01db\5> \2\u01da\u01d8\3\2\2\2\u01db"+
+    "\u01de\3\2\2\2\u01dc\u01da\3\2\2\2\u01dc\u01dd\3\2\2\2\u01dd\u01e0\3\2"+
+    "\2\2\u01de\u01dc\3\2\2\2\u01df\u01d7\3\2\2\2\u01df\u01e0\3\2\2\2\u01e0"+
+    "\u01e1\3\2\2\2\u01e1\u01e3\7\n\2\2\u01e2\u01d5\3\2\2\2\u01e2\u01d6\3\2"+
+    "\2\2\u01e3\u01e4\3\2\2\2\u01e4\u01e7\79\2\2\u01e5\u01e8\5\f\7\2\u01e6"+
+    "\u01e8\5\36\20\2\u01e7\u01e5\3\2\2\2\u01e7\u01e6\3\2\2\2\u01e8=\3\2\2"+
+    "\2\u01e9\u01eb\5\26\f\2\u01ea\u01e9\3\2\2\2\u01ea\u01eb\3\2\2\2\u01eb"+
+    "\u01ec\3\2\2\2\u01ec\u01ed\7T\2\2\u01ed?\3\2\2\2\u01ee\u01ef\7S\2\2\u01ef"+
+    "\u01f0\78\2\2\u01f0\u01fc\7T\2\2\u01f1\u01f2\5\26\f\2\u01f2\u01f3\78\2"+
+    "\2\u01f3\u01f4\7\30\2\2\u01f4\u01fc\3\2\2\2\u01f5\u01f6\7T\2\2\u01f6\u01f7"+
+    "\78\2\2\u01f7\u01fc\7T\2\2\u01f8\u01f9\7\34\2\2\u01f9\u01fa\78\2\2\u01fa"+
+    "\u01fc\7T\2\2\u01fb\u01ee\3\2\2\2\u01fb\u01f1\3\2\2\2\u01fb\u01f5\3\2"+
+    "\2\2\u01fb\u01f8\3\2\2\2\u01fcA\3\2\2\2\62EK^amu\u0082\u0086\u008a\u008f"+
+    "\u00b2\u00bb\u00bf\u00c5\u00ce\u00d8\u00e0\u00e6\u0124\u0126\u0136\u013c"+
+    "\u0144\u0148\u015c\u0161\u0165\u017a\u0180\u0183\u018f\u0192\u0195\u019b"+
+    "\u019e\u01a6\u01ad\u01b5\u01bd\u01c9\u01cc\u01d3\u01dc\u01df\u01e2\u01e7"+
+    "\u01ea\u01fb";
   public static final ATN _ATN =
     new ATNDeserializer().deserialize(_serializedATN.toCharArray());
   static {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java
index d3a87f1a099b5..8c4741e672533 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java
@@ -235,6 +235,13 @@ class PainlessParserBaseVisitor<T> extends AbstractParseTreeVisitor<T> implement
    * {@link #visitChildren} on {@code ctx}.</p>
    */
   @Override public T visitBinary(PainlessParser.BinaryContext ctx) { return visitChildren(ctx); }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation returns the result of calling
+   * {@link #visitChildren} on {@code ctx}.</p>
+   */
+  @Override public T visitElvis(PainlessParser.ElvisContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java
index 0f7fef6185a14..47bfd4a1d05b9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java
@@ -221,6 +221,13 @@ interface PainlessParserVisitor<T> extends ParseTreeVisitor<T> {
    * @return the visitor result
    */
   T visitBinary(PainlessParser.BinaryContext ctx);
+  /**
+   * Visit a parse tree produced by the {@code elvis}
+   * labeled alternative in {@link PainlessParser#expression}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitElvis(PainlessParser.ElvisContext ctx);
   /**
    * Visit a parse tree produced by the {@code instanceof}
    * labeled alternative in {@link PainlessParser#expression}.
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/SlashStrategy.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/SlashStrategy.java
deleted file mode 100644
index 698a9dfc364b5..0000000000000
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/SlashStrategy.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.painless.antlr;
-
-import org.antlr.v4.runtime.Token;
-
-/**
- * Utility to figure out if a {@code /} is division or the start of a regex literal.
- */
-public class SlashStrategy {
-    public static boolean slashIsRegex(PainlessLexer lexer) {
-        EnhancedPainlessLexer realLexer = (EnhancedPainlessLexer) lexer;
-        Token lastToken = realLexer.getPreviousToken();
-        if (lastToken == null) {
-            return true;
-        }
-        switch (lastToken.getType()) {
-        case PainlessLexer.RBRACE:
-        case PainlessLexer.RP:
-        case PainlessLexer.OCTAL:
-        case PainlessLexer.HEX:
-        case PainlessLexer.INTEGER:
-        case PainlessLexer.DECIMAL:
-        case PainlessLexer.ID:
-        case PainlessLexer.DOTINTEGER:
-        case PainlessLexer.DOTID:
-            return false;
-        default:
-            return true;
-        }
-    }
-}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/StashingTokenFactory.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/StashingTokenFactory.java
index 4f22137e5fb1f..3ac45705d5549 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/StashingTokenFactory.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/StashingTokenFactory.java
@@ -27,7 +27,7 @@
 import org.antlr.v4.runtime.misc.Pair;
 
 /**
- * Token factory that preseres that last non-whitespace token so you can do token level lookbehind in the lexer.
+ * Token factory that preserves that last non-whitespace token so you can do token level lookbehind in the lexer.
  */
 public class StashingTokenFactory<T extends Token> implements TokenFactory<T> {
     private final TokenFactory<T> delegate;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
index 61269419fdf94..0210b7ce151d5 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
@@ -29,9 +29,11 @@
 import org.antlr.v4.runtime.atn.PredictionMode;
 import org.antlr.v4.runtime.tree.TerminalNode;
 import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Operation;
+import org.elasticsearch.painless.ScriptInterface;
 import org.elasticsearch.painless.antlr.PainlessParser.AfterthoughtContext;
 import org.elasticsearch.painless.antlr.PainlessParser.ArgumentContext;
 import org.elasticsearch.painless.antlr.PainlessParser.ArgumentsContext;
@@ -58,6 +60,7 @@
 import org.elasticsearch.painless.antlr.PainlessParser.DoContext;
 import org.elasticsearch.painless.antlr.PainlessParser.DynamicContext;
 import org.elasticsearch.painless.antlr.PainlessParser.EachContext;
+import org.elasticsearch.painless.antlr.PainlessParser.ElvisContext;
 import org.elasticsearch.painless.antlr.PainlessParser.EmptyContext;
 import org.elasticsearch.painless.antlr.PainlessParser.ExprContext;
 import org.elasticsearch.painless.antlr.PainlessParser.ExpressionContext;
@@ -117,6 +120,7 @@
 import org.elasticsearch.painless.node.EComp;
 import org.elasticsearch.painless.node.EConditional;
 import org.elasticsearch.painless.node.EDecimal;
+import org.elasticsearch.painless.node.EElvis;
 import org.elasticsearch.painless.node.EExplicit;
 import org.elasticsearch.painless.node.EFunctionRef;
 import org.elasticsearch.painless.node.EInstanceof;
@@ -170,32 +174,40 @@
  */
 public final class Walker extends PainlessParserBaseVisitor<ANode> {
 
-    public static SSource buildPainlessTree(String sourceName, String sourceText, CompilerSettings settings, Printer debugStream) {
-        return new Walker(sourceName, sourceText, settings, debugStream).source;
+    public static SSource buildPainlessTree(ScriptInterface mainMethod, String sourceName,
+            String sourceText, CompilerSettings settings, Definition definition,
+            Printer debugStream) {
+        return new Walker(mainMethod, sourceName, sourceText, settings, definition,
+                debugStream).source;
     }
 
+    private final ScriptInterface scriptInterface;
     private final SSource source;
     private final CompilerSettings settings;
     private final Printer debugStream;
     private final String sourceName;
     private final String sourceText;
+    private final Definition definition;
 
     private final Deque<Reserved> reserved = new ArrayDeque<>();
     private final Globals globals;
     private int syntheticCounter = 0;
 
-    private Walker(String sourceName, String sourceText, CompilerSettings settings, Printer debugStream) {
+    private Walker(ScriptInterface scriptInterface, String sourceName, String sourceText,
+            CompilerSettings settings, Definition definition, Printer debugStream) {
+        this.scriptInterface = scriptInterface;
         this.debugStream = debugStream;
         this.settings = settings;
         this.sourceName = Location.computeSourceName(sourceName, sourceText);
         this.sourceText = sourceText;
         this.globals = new Globals(new BitSet(sourceText.length()));
+        this.definition = definition;
         this.source = (SSource)visit(buildAntlrTree(sourceText));
     }
 
     private SourceContext buildAntlrTree(String source) {
         ANTLRInputStream stream = new ANTLRInputStream(source);
-        PainlessLexer lexer = new EnhancedPainlessLexer(stream, sourceName);
+        PainlessLexer lexer = new EnhancedPainlessLexer(stream, sourceName, definition);
         PainlessParser parser = new PainlessParser(new CommonTokenStream(lexer));
         ParserErrorStrategy strategy = new ParserErrorStrategy(sourceName);
 
@@ -254,7 +266,7 @@ public ANode visitSource(SourceContext ctx) {
             statements.add((AStatement)visit(statement));
         }
 
-        return new SSource(settings, sourceName, sourceText, debugStream, (MainMethodReserved)reserved.pop(),
+        return new SSource(scriptInterface, settings, sourceName, sourceText, debugStream, (MainMethodReserved)reserved.pop(),
                            location(ctx), functions, globals, statements);
     }
 
@@ -616,6 +628,14 @@ public ANode visitConditional(ConditionalContext ctx) {
         return new EConditional(location(ctx), condition, left, right);
     }
 
+    @Override
+    public ANode visitElvis(ElvisContext ctx) {
+        AExpression left = (AExpression)visit(ctx.expression(0));
+        AExpression right = (AExpression)visit(ctx.expression(1));
+
+        return new EElvis(location(ctx), left, right);
+    }
+
     @Override
     public ANode visitAssignment(AssignmentContext ctx) {
         AExpression lhs = (AExpression)visit(ctx.expression(0));
@@ -789,13 +809,36 @@ public ANode visitNull(NullContext ctx) {
 
     @Override
     public ANode visitString(StringContext ctx) {
-        String string = ctx.STRING().getText().substring(1, ctx.STRING().getText().length() - 1);
+        StringBuilder string = new StringBuilder(ctx.STRING().getText());
+
+        // Strip the leading and trailing quotes and replace the escape sequences with their literal equivalents
+        int src = 1;
+        int dest = 0;
+        int end = string.length() - 1;
+        assert string.charAt(0) == '"' || string.charAt(0) == '\'' : "expected string to start with a quote but was [" + string + "]";
+        assert string.charAt(end) == '"' || string.charAt(end) == '\'' : "expected string to end with a quote was [" + string + "]";
+        while (src < end) {
+            char current = string.charAt(src);
+            if (current == '\\') {
+                src++;
+                current = string.charAt(src);
+            }
+            string.setCharAt(dest, current);
+            src++;
+            dest++;
+        }
+        string.setLength(dest);
 
-        return new EString(location(ctx), string);
+        return new EString(location(ctx), string.toString());
     }
 
     @Override
     public ANode visitRegex(RegexContext ctx) {
+        if (false == settings.areRegexesEnabled()) {
+            throw location(ctx).createError(new IllegalStateException("Regexes are disabled. Set [script.painless.regex.enabled] to [true] "
+                    + "in elasticsearch.yaml to allow them. Be careful though, regexes break out of Painless's protection against deep "
+                    + "recursion and long loops."));
+        }
         String text = ctx.REGEX().getText();
         int lastSlash = text.lastIndexOf('/');
         String pattern = text.substring(1, lastSlash);
@@ -817,7 +860,7 @@ public ANode visitMapinit(MapinitContext ctx) {
     @Override
     public ANode visitVariable(VariableContext ctx) {
         String name = ctx.ID().getText();
-        reserved.peek().markReserved(name);
+        reserved.peek().markUsedVariable(name);
 
         return new EVariable(location(ctx), name);
     }
@@ -893,7 +936,7 @@ public AExpression visitCallinvoke(CallinvokeContext ctx, AExpression prefix) {
         String name = ctx.DOTID().getText();
         List<AExpression> arguments = collectArguments(ctx.arguments());
 
-        return new PCallInvoke(location(ctx), prefix, name, arguments);
+        return new PCallInvoke(location(ctx), prefix, name, ctx.NSDOT() != null, arguments);
     }
 
     @Override
@@ -912,7 +955,7 @@ public AExpression visitFieldaccess(FieldaccessContext ctx, AExpression prefix)
             throw location(ctx).createError(new IllegalStateException("Illegal tree structure."));
         }
 
-        return new PField(location(ctx), prefix, value);
+        return new PField(location(ctx), prefix, ctx.NSDOT() != null, value);
     }
 
     @Override
@@ -1017,7 +1060,7 @@ public ANode visitLambda(LambdaContext ctx) {
 
         for (LamtypeContext lamtype : ctx.lamtype()) {
             if (lamtype.decltype() == null) {
-                paramTypes.add("def");
+                paramTypes.add(null);
             } else {
                 paramTypes.add(lamtype.decltype().getText());
             }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/package-info.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/package-info.java
new file mode 100644
index 0000000000000..041118ba029f4
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/package-info.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Lexer, parser, and tree {@link Walker} responsible for turning the code
+ * generating nodes in {@link org.elasticsearch.painless.node}.
+ */
+package org.elasticsearch.painless.antlr;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Augmentation.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/Augmentation.java
similarity index 95%
rename from modules/lang-painless/src/main/java/org/elasticsearch/painless/Augmentation.java
rename to modules/lang-painless/src/main/java/org/elasticsearch/painless/api/Augmentation.java
index 9302f3c899cc2..0b751b7d2f78f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Augmentation.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/Augmentation.java
@@ -17,9 +17,11 @@
  * under the License.
  */
 
-package org.elasticsearch.painless;
+package org.elasticsearch.painless.api;
 
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
+import java.util.Base64;
 import java.util.Collection;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -486,4 +488,19 @@ public static String replaceFirst(CharSequence receiver, Pattern pattern, Functi
     private static int initialBufferForReplaceWith(CharSequence seq) {
         return seq.length() + 16;
     }
+
+    /**
+     * Encode a String in Base64. Use {@link Base64.Encoder#encodeToString(byte[])} if you have to encode bytes rather than a string.
+     */
+    public static String encodeBase64(String receiver) {
+        return Base64.getEncoder().encodeToString(receiver.getBytes(StandardCharsets.UTF_8));
+    }
+
+    /**
+     * Decode some Base64 bytes and build a UTF-8 encoded string. Use {@link Base64.Decoder#decode(String)} if you'd prefer bytes to work
+     * with bytes.
+     */
+    public static String decodeBase64(String receiver) {
+        return new String(Base64.getDecoder().decode(receiver.getBytes(StandardCharsets.UTF_8)), StandardCharsets.UTF_8);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/Debug.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/Debug.java
new file mode 100644
index 0000000000000..de0c9b3adbde3
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/Debug.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.api;
+
+import org.elasticsearch.painless.PainlessExplainError;
+
+/**
+ * Utility methods for debugging painless scripts that are accessible to painless scripts.
+ */
+public class Debug {
+    private Debug() {}
+
+    /**
+     * Throw an {@link Error} that "explains" an object.
+     */
+    public static void explain(Object objectToExplain) throws PainlessExplainError {
+        throw new PainlessExplainError(objectToExplain);
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/package-info.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/package-info.java
new file mode 100644
index 0000000000000..b1c78e28ec125
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Java code publicly exposed to Painless code.
+ */
+package org.elasticsearch.painless.api;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java
index da4e00f018386..567ffcef8a38f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java
@@ -24,9 +24,18 @@
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
 import java.util.Objects;
 import java.util.Set;
 
+import static java.lang.Math.max;
+import static java.util.Collections.emptyList;
+
 /**
  * The superclass for all nodes.
  */
@@ -62,7 +71,87 @@ public abstract class ANode {
      */
     abstract void write(MethodWriter writer, Globals globals);
 
+    /**
+     * Create an error with location information pointing to this node.
+     */
     RuntimeException createError(RuntimeException exception) {
         return location.createError(exception);
     }
+
+    /**
+     * Subclasses should implement this with a method like {@link #singleLineToString(Object...)} or
+     * {@link #multilineToString(Collection, Collection)}.
+     */
+    public abstract String toString();
+
+    // Below are utilities for building the toString
+
+    /**
+     * Build {@link #toString()} for a node without inserting line breaks between the sub-nodes.
+     */
+    protected String singleLineToString(Object... subs) {
+        return singleLineToString(Arrays.asList(subs));
+    }
+
+    /**
+     * Build {@link #toString()} for a node without inserting line breaks between the sub-nodes.
+     */
+    protected String singleLineToString(Collection<? extends Object> subs) {
+        return joinWithName(getClass().getSimpleName(), subs, emptyList());
+    }
+
+    /**
+     * Build {@link #toString()} for a node that optionally ends in {@code (Args some arguments here)}. Usually function calls.
+     */
+    protected String singleLineToStringWithOptionalArgs(Collection<? extends ANode> arguments, Object... restOfSubs) {
+        List<Object> subs = new ArrayList<>();
+        Collections.addAll(subs, restOfSubs);
+        if (false == arguments.isEmpty()) {
+            subs.add(joinWithName("Args", arguments, emptyList()));
+        }
+        return singleLineToString(subs);
+    }
+
+    /**
+     * Build {@link #toString()} for a node that should have new lines after some of its sub-nodes.
+     */
+    protected String multilineToString(Collection<? extends Object> sameLine, Collection<? extends Object> ownLine) {
+        return joinWithName(getClass().getSimpleName(), sameLine, ownLine);
+    }
+
+    /**
+     * Zip two (potentially uneven) lists together into for {@link #toString()}.
+     */
+    protected List<String> pairwiseToString(Collection<? extends Object> lefts, Collection<? extends Object> rights) {
+        List<String> pairs = new ArrayList<>(max(lefts.size(), rights.size()));
+        Iterator<? extends Object> left = lefts.iterator();
+        Iterator<? extends Object> right = rights.iterator();
+        while (left.hasNext() || right.hasNext()) {
+            pairs.add(joinWithName("Pair",
+                    Arrays.asList(left.hasNext() ? left.next() : "<uneven>", right.hasNext() ? right.next() : "<uneven>"),
+                    emptyList()));
+        }
+        return pairs;
+    }
+
+    /**
+     * Build a {@link #toString()} for some expressions. Usually best to use {@link #singleLineToString(Object...)} or
+     * {@link #multilineToString(Collection, Collection)} instead because they include the name of the node by default.
+     */
+    protected String joinWithName(String name, Collection<? extends Object> sameLine,
+            Collection<? extends Object> ownLine) {
+        StringBuilder b = new StringBuilder();
+        b.append('(').append(name);
+        for (Object sub : sameLine) {
+            b.append(' ').append(sub);
+        }
+        if (ownLine.size() == 1 && sameLine.isEmpty()) {
+            b.append(' ').append(ownLine.iterator().next());
+        } else {
+            for (Object sub : ownLine) {
+                b.append("\n  ").append(Objects.toString(sub).replace("\n", "\n  "));
+            }
+        }
+        return b.append(')').toString();
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStoreable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStoreable.java
index 71b8ccd4da1a1..3cff6bab08ef5 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStoreable.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStoreable.java
@@ -23,8 +23,11 @@
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
+import org.objectweb.asm.Label;
+import org.objectweb.asm.Opcodes;
 
 import java.util.Objects;
+import java.util.function.Consumer;
 
 /**
  * The super class for an expression that can store a value in local memory.
@@ -100,4 +103,21 @@ abstract class AStoreable extends AExpression {
      * Called to store a storabable to local memory.
      */
     abstract void store(MethodWriter writer, Globals globals);
+
+    /**
+     * Writes the opcodes to flip a negative array index (meaning slots from the end of the array) into a 0-based one (meaning slots from
+     * the start of the array).
+     */
+    static void writeIndexFlip(MethodWriter writer, Consumer<MethodWriter> writeGetLength) {
+        Label noFlip = new Label();
+        // Everywhere when it says 'array' below that could also be a list
+        // The stack after each instruction:       array, unnormalized_index
+        writer.dup();                           // array, unnormalized_index, unnormalized_index
+        writer.ifZCmp(Opcodes.IFGE, noFlip);    // array, unnormalized_index
+        writer.swap();                          // negative_index, array
+        writer.dupX1();                         // array, negative_index, array
+        writeGetLength.accept(writer);          // array, negative_index, length
+        writer.visitInsn(Opcodes.IADD);         // array, noralized_index
+        writer.mark(noFlip);                    // array, noralized_index
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java
index 50b56505eb8b6..de2e9fc0535bd 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java
@@ -31,6 +31,8 @@
 import org.elasticsearch.painless.MethodWriter;
 import org.elasticsearch.painless.Operation;
 
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Objects;
 import java.util.Set;
 
@@ -262,8 +264,8 @@ void write(MethodWriter writer, Globals globals) {
 
             rhs.write(writer, globals); // write the bytecode for the rhs
 
-            if (!(rhs instanceof EBinary) || ((EBinary)rhs).cat) {
-                writer.writeAppendStrings(rhs.actual); // append the rhs's value unless it's also a concatenation
+            if (!(rhs instanceof EBinary) || !((EBinary)rhs).cat) { // check to see if the rhs has already done a concatenation
+                writer.writeAppendStrings(rhs.actual); // append the rhs's value since it's hasn't already
             }
 
             writer.writeToStrings(); // put the value for string concat onto the stack
@@ -322,4 +324,24 @@ void write(MethodWriter writer, Globals globals) {
             lhs.store(writer, globals); // store the lhs's value from the stack in its respective variable/field/array
         }
     }
+
+    @Override
+    public String toString() {
+        List<Object> subs = new ArrayList<>();
+        subs.add(lhs);
+        if (rhs != null) {
+            // Make sure "=" is in the symbol so this is easy to read at a glance
+            subs.add(operation == null ? "=" : operation.symbol + "=");
+            subs.add(rhs);
+            return singleLineToString(subs);
+        }
+        subs.add(operation.symbol);
+        if (pre) {
+            subs.add("pre");
+        }
+        if (post) {
+            subs.add("post");
+        }
+        return singleLineToString(subs);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
index ca2858feb8d33..72748d7069e28 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
@@ -291,7 +291,7 @@ private void analyzeAdd(Locals variables) {
             } else if (sort == Sort.DOUBLE) {
                 constant = (double)left.constant + (double)right.constant;
             } else if (sort == Sort.STRING) {
-                constant = "" + left.constant + right.constant;
+                constant = left.constant.toString() + right.constant.toString();
             } else {
                 throw createError(new IllegalStateException("Illegal tree structure."));
             }
@@ -681,4 +681,9 @@ void write(MethodWriter writer, Globals globals) {
             }
         }
     }
+
+    @Override
+    public String toString() {
+        return singleLineToString(left, operation.symbol, right);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
index 706a00c49255f..e1ca0b122922d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
@@ -114,4 +114,9 @@ void write(MethodWriter writer, Globals globals) {
             throw createError(new IllegalStateException("Illegal tree structure."));
         }
     }
+
+    @Override
+    public String toString() {
+        return singleLineToString(left, operation.symbol, right);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
index 17c72b43a4f78..c9e9ba921d7b6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
@@ -56,4 +56,9 @@ void analyze(Locals locals) {
     void write(MethodWriter adapter, Globals globals) {
         throw createError(new IllegalStateException("Illegal tree structure."));
     }
+
+    @Override
+    public String toString() {
+        return singleLineToString(constant);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java
index c50cb3d6eed6d..4c977fa66e89a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java
@@ -88,4 +88,9 @@ void write(MethodWriter writer, Globals globals) {
 
         writer.invokeStatic(CLASS_TYPE, method.method);
     }
+
+    @Override
+    public String toString() {
+        return singleLineToStringWithOptionalArgs(arguments, name);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
index bf85665e4dfb5..f40883186dd7a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.painless.node;
 
+import org.elasticsearch.painless.AnalyzerCaster;
 import org.elasticsearch.painless.DefBootstrap;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.FunctionRef;
@@ -30,10 +31,10 @@
 import org.objectweb.asm.Opcodes;
 import org.objectweb.asm.Type;
 
-import java.lang.invoke.LambdaMetafactory;
 import java.util.Objects;
 import java.util.Set;
 
+import static org.elasticsearch.painless.Definition.VOID_TYPE;
 import static org.elasticsearch.painless.WriterConstants.LAMBDA_BOOTSTRAP_HANDLE;
 
 /**
@@ -60,8 +61,8 @@ void extractVariables(Set<String> variables) {
     }
 
     @Override
-    void analyze(Locals variables) {
-        captured = variables.getVariable(location, variable);
+    void analyze(Locals locals) {
+        captured = locals.getVariable(location, variable);
         if (expected == null) {
             if (captured.type.sort == Definition.Sort.DEF) {
                 // dynamic implementation
@@ -70,13 +71,24 @@ void analyze(Locals variables) {
                 // typed implementation
                 defPointer = "S" + captured.type.name + "." + call + ",1";
             }
-            actual = Definition.getType("String");
+            actual = locals.getDefinition().getType("String");
         } else {
             defPointer = null;
             // static case
             if (captured.type.sort != Definition.Sort.DEF) {
                 try {
-                    ref = new FunctionRef(expected, captured.type.name, call, 1);
+                    ref = new FunctionRef(locals.getDefinition(), expected, captured.type.name, call, 1);
+
+                    // check casts between the interface method and the delegate method are legal
+                    for (int i = 0; i < ref.interfaceMethod.arguments.size(); ++i) {
+                        Definition.Type from = ref.interfaceMethod.arguments.get(i);
+                        Definition.Type to = ref.delegateMethod.arguments.get(i);
+                        AnalyzerCaster.getLegalCast(location, from, to, false, true);
+                    }
+
+                    if (ref.interfaceMethod.rtn != VOID_TYPE) {
+                        AnalyzerCaster.getLegalCast(location, ref.delegateMethod.rtn, ref.interfaceMethod.rtn, false, true);
+                    }
                 } catch (IllegalArgumentException e) {
                     throw createError(e);
                 }
@@ -101,29 +113,16 @@ void write(MethodWriter writer, Globals globals) {
         } else {
             // typed interface, typed implementation
             writer.visitVarInsn(captured.type.type.getOpcode(Opcodes.ILOAD), captured.getSlot());
-            // convert MethodTypes to asm Type for the constant pool.
-            String invokedType = ref.invokedType.toMethodDescriptorString();
-            Type samMethodType = Type.getMethodType(ref.samMethodType.toMethodDescriptorString());
-            Type interfaceType = Type.getMethodType(ref.interfaceMethodType.toMethodDescriptorString());
-            if (ref.needsBridges()) {
-                writer.invokeDynamic(ref.invokedName,
-                                     invokedType,
-                                     LAMBDA_BOOTSTRAP_HANDLE,
-                                     samMethodType,
-                                     ref.implMethodASM,
-                                     samMethodType,
-                                     LambdaMetafactory.FLAG_BRIDGES,
-                                     1,
-                                     interfaceType);
-            } else {
-                writer.invokeDynamic(ref.invokedName,
-                                     invokedType,
-                                     LAMBDA_BOOTSTRAP_HANDLE,
-                                     samMethodType,
-                                     ref.implMethodASM,
-                                     samMethodType,
-                                     0);
-            }
+            writer.invokeDynamic(
+                ref.interfaceMethodName,
+                ref.factoryDescriptor,
+                LAMBDA_BOOTSTRAP_HANDLE,
+                ref.interfaceType,
+                ref.delegateClassName,
+                ref.delegateInvokeType,
+                ref.delegateMethodName,
+                ref.delegateType
+            );
         }
     }
 
@@ -136,4 +135,9 @@ public String getPointer() {
     public Type[] getCaptures() {
         return new Type[] { captured.type.type };
     }
+
+    @Override
+    public String toString() {
+        return singleLineToString(variable, call);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java
index e575af241273c..5c2a149876139 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java
@@ -60,4 +60,9 @@ void write(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
         writer.writeCast(cast);
     }
+
+    @Override
+    public String toString() {
+        return singleLineToString(cast.to, child);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
index 4b4e902774ad4..d66f73ec685bd 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
@@ -148,13 +148,8 @@ private void analyzeEqR(Locals variables) {
                 "[" + left.actual.name + "] and [" + right.actual.name + "]."));
         }
 
-        if (promotedType.sort == Sort.DEF) {
-            left.expected = left.actual;
-            right.expected = right.actual;
-        } else {
-            left.expected = promotedType;
-            right.expected = promotedType;
-        }
+        left.expected = promotedType;
+        right.expected = promotedType;
 
         left = left.cast(variables);
         right = right.cast(variables);
@@ -246,13 +241,8 @@ private void analyzeNER(Locals variables) {
                 "[" + left.actual.name + "] and [" + right.actual.name + "]."));
         }
 
-        if (promotedType.sort == Sort.DEF) {
-            left.expected = left.actual;
-            right.expected = right.actual;
-        } else {
-            left.expected = promotedType;
-            right.expected = promotedType;
-        }
+        left.expected = promotedType;
+        right.expected = promotedType;
 
         left = left.cast(variables);
         right = right.cast(variables);
@@ -568,4 +558,9 @@ void write(MethodWriter writer, Globals globals) {
             writer.mark(end);
         }
     }
+
+    @Override
+    public String toString() {
+        return singleLineToString(left, operation.symbol, right);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
index 1f9fe8bdfcbc9..40f4b27ed156f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
@@ -34,7 +34,7 @@
 import org.objectweb.asm.Opcodes;
 
 /**
- * Respresents a conditional expression.
+ * Represents a conditional expression.
  */
 public final class EConditional extends AExpression {
 
@@ -106,4 +106,9 @@ void write(MethodWriter writer, Globals globals) {
         right.write(writer, globals);
         writer.mark(end);
     }
+
+    @Override
+    public String toString() {
+        return singleLineToString(condition, left, right);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
index 722a339c8222b..4242496c51ac5 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
@@ -89,4 +89,13 @@ void write(MethodWriter writer, Globals globals) {
                 throw createError(new IllegalStateException("Illegal tree structure."));
         }
     }
+
+    @Override
+    public String toString() {
+        String c = constant.toString();
+        if (constant instanceof String) {
+            c = "'" + c + "'";
+        }
+        return singleLineToString(constant.getClass().getSimpleName(), c);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
index 643861477e73c..85675a02892c9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
@@ -58,8 +58,12 @@ void analyze(Locals locals) {
                 throw createError(new IllegalArgumentException("Invalid float constant [" + value + "]."));
             }
         } else {
+            String toParse = value;
+            if (toParse.endsWith("d") || value.endsWith("D")) {
+                toParse = toParse.substring(0, value.length() - 1);
+            }
             try {
-                constant = Double.parseDouble(value);
+                constant = Double.parseDouble(toParse);
                 actual = Definition.DOUBLE_TYPE;
             } catch (NumberFormatException exception) {
                 throw createError(new IllegalArgumentException("Invalid double constant [" + value + "]."));
@@ -71,4 +75,9 @@ void analyze(Locals locals) {
     void write(MethodWriter writer, Globals globals) {
         throw createError(new IllegalStateException("Illegal tree structure."));
     }
+
+    @Override
+    public String toString() {
+        return singleLineToString(value);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EElvis.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EElvis.java
new file mode 100644
index 0000000000000..a711f17c5b0b4
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EElvis.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.AnalyzerCaster;
+import org.elasticsearch.painless.Definition.Type;
+import org.elasticsearch.painless.Globals;
+import org.elasticsearch.painless.Locals;
+import org.elasticsearch.painless.Location;
+import org.elasticsearch.painless.MethodWriter;
+import org.objectweb.asm.Label;
+
+import java.util.Set;
+
+import static java.util.Objects.requireNonNull;
+
+/**
+ * The Elvis operator ({@code ?:}), a null coalescing operator. Binary operator that evaluates the first expression and return it if it is
+ * non null. If the first expression is null then it evaluates the second expression and returns it.
+ */
+public class EElvis extends AExpression {
+    private AExpression lhs;
+    private AExpression rhs;
+
+    public EElvis(Location location, AExpression lhs, AExpression rhs) {
+        super(location);
+
+        this.lhs = requireNonNull(lhs);
+        this.rhs = requireNonNull(rhs);
+    }
+
+    @Override
+    void extractVariables(Set<String> variables) {
+        lhs.extractVariables(variables);
+        rhs.extractVariables(variables);
+    }
+
+    @Override
+    void analyze(Locals locals) {
+        if (expected != null && expected.sort.primitive) {
+            throw createError(new IllegalArgumentException("Evlis operator cannot return primitives"));
+        }
+        lhs.expected = expected;
+        lhs.explicit = explicit;
+        lhs.internal = internal;
+        rhs.expected = expected;
+        rhs.explicit = explicit;
+        rhs.internal = internal;
+        actual = expected;
+        lhs.analyze(locals);
+        rhs.analyze(locals);
+
+        if (lhs.isNull) {
+            throw createError(new IllegalArgumentException("Extraneous elvis operator. LHS is null."));
+        }
+        if (lhs.constant != null) {
+            throw createError(new IllegalArgumentException("Extraneous elvis operator. LHS is a constant."));
+        }
+        if (lhs.actual.sort.primitive) {
+            throw createError(new IllegalArgumentException("Extraneous elvis operator. LHS is a primitive."));
+        }
+        if (rhs.isNull) {
+            throw createError(new IllegalArgumentException("Extraneous elvis operator. RHS is null."));
+        }
+
+        if (expected == null) {
+            final Type promote = AnalyzerCaster.promoteConditional(lhs.actual, rhs.actual, lhs.constant, rhs.constant);
+
+            lhs.expected = promote;
+            rhs.expected = promote;
+            actual = promote;
+        }
+
+        lhs = lhs.cast(locals);
+        rhs = rhs.cast(locals);
+    }
+
+    @Override
+    void write(MethodWriter writer, Globals globals) {
+        writer.writeDebugInfo(location);
+
+        Label end = new Label();
+
+        lhs.write(writer, globals);
+        writer.dup();
+        writer.ifNonNull(end);
+        writer.pop();
+        rhs.write(writer, globals);
+        writer.mark(end);
+    }
+
+    @Override
+    public String toString() {
+        return singleLineToString(lhs, rhs);
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
index 527bcc9a0bbdd..2624735aa07ee 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
@@ -19,10 +19,9 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Globals;
-import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Locals;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 
 import java.util.Objects;
@@ -51,7 +50,7 @@ void extractVariables(Set<String> variables) {
     @Override
     void analyze(Locals locals) {
         try {
-            actual = Definition.getType(this.type);
+            actual = locals.getDefinition().getType(type);
         } catch (IllegalArgumentException exception) {
             throw createError(new IllegalArgumentException("Not a type [" + this.type + "]."));
         }
@@ -74,4 +73,9 @@ AExpression cast(Locals locals) {
 
         return child.cast(locals);
     }
+
+    @Override
+    public String toString() {
+        return singleLineToString(type, child);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
index 17ab7c014faf4..c9b2a5d91d232 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.painless.node;
 
+import org.elasticsearch.painless.AnalyzerCaster;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.Definition.MethodKey;
@@ -29,10 +30,10 @@
 import org.elasticsearch.painless.MethodWriter;
 import org.objectweb.asm.Type;
 
-import java.lang.invoke.LambdaMetafactory;
 import java.util.Objects;
 import java.util.Set;
 
+import static org.elasticsearch.painless.Definition.VOID_TYPE;
 import static org.elasticsearch.painless.WriterConstants.LAMBDA_BOOTSTRAP_HANDLE;
 
 /**
@@ -59,7 +60,7 @@ void extractVariables(Set<String> variables) {}
     void analyze(Locals locals) {
         if (expected == null) {
             ref = null;
-            actual = Definition.getType("String");
+            actual = locals.getDefinition().getType("String");
             defPointer = "S" + type + "." + call + ",0";
         } else {
             defPointer = null;
@@ -71,16 +72,28 @@ void analyze(Locals locals) {
                         throw new IllegalArgumentException("Cannot convert function reference [" + type + "::" + call + "] " +
                                                            "to [" + expected.name + "], not a functional interface");
                     }
-                    Method implMethod = locals.getMethod(new MethodKey(call, interfaceMethod.arguments.size()));
-                    if (implMethod == null) {
+                    Method delegateMethod = locals.getMethod(new MethodKey(call, interfaceMethod.arguments.size()));
+                    if (delegateMethod == null) {
                         throw new IllegalArgumentException("Cannot convert function reference [" + type + "::" + call + "] " +
                                                            "to [" + expected.name + "], function not found");
                     }
-                    ref = new FunctionRef(expected, interfaceMethod, implMethod, 0);
+                    ref = new FunctionRef(expected, interfaceMethod, delegateMethod, 0);
+
+                    // check casts between the interface method and the delegate method are legal
+                    for (int i = 0; i < interfaceMethod.arguments.size(); ++i) {
+                        Definition.Type from = interfaceMethod.arguments.get(i);
+                        Definition.Type to = delegateMethod.arguments.get(i);
+                        AnalyzerCaster.getLegalCast(location, from, to, false, true);
+                    }
+
+                    if (interfaceMethod.rtn != VOID_TYPE) {
+                        AnalyzerCaster.getLegalCast(location, delegateMethod.rtn, interfaceMethod.rtn, false, true);
+                    }
                 } else {
                     // whitelist lookup
-                    ref = new FunctionRef(expected, type, call, 0);
+                    ref = new FunctionRef(locals.getDefinition(), expected, type, call, 0);
                 }
+
             } catch (IllegalArgumentException e) {
                 throw createError(e);
             }
@@ -92,29 +105,16 @@ void analyze(Locals locals) {
     void write(MethodWriter writer, Globals globals) {
         if (ref != null) {
             writer.writeDebugInfo(location);
-            // convert MethodTypes to asm Type for the constant pool.
-            String invokedType = ref.invokedType.toMethodDescriptorString();
-            Type samMethodType = Type.getMethodType(ref.samMethodType.toMethodDescriptorString());
-            Type interfaceType = Type.getMethodType(ref.interfaceMethodType.toMethodDescriptorString());
-            if (ref.needsBridges()) {
-                writer.invokeDynamic(ref.invokedName,
-                                     invokedType,
-                                     LAMBDA_BOOTSTRAP_HANDLE,
-                                     samMethodType,
-                                     ref.implMethodASM,
-                                     samMethodType,
-                                     LambdaMetafactory.FLAG_BRIDGES,
-                                     1,
-                                     interfaceType);
-            } else {
-                writer.invokeDynamic(ref.invokedName,
-                                     invokedType,
-                                     LAMBDA_BOOTSTRAP_HANDLE,
-                                     samMethodType,
-                                     ref.implMethodASM,
-                                     samMethodType,
-                                     0);
-            }
+            writer.invokeDynamic(
+                ref.interfaceMethodName,
+                ref.factoryDescriptor,
+                LAMBDA_BOOTSTRAP_HANDLE,
+                ref.interfaceType,
+                ref.delegateClassName,
+                ref.delegateInvokeType,
+                ref.delegateMethodName,
+                ref.delegateType
+            );
         } else {
             // TODO: don't do this: its just to cutover :)
             writer.push((String)null);
@@ -130,4 +130,9 @@ public String getPointer() {
     public Type[] getCaptures() {
         return new Type[0]; // no captures
     }
+
+    @Override
+    public String toString() {
+        return singleLineToString(type, call);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java
index ba8a3454ee295..c9b2c95bb4c2d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java
@@ -30,7 +30,7 @@
 import java.util.Set;
 
 /**
- * Represents instanceof operator.
+ * Represents {@code instanceof} operator.
  * <p>
  * Unlike java's, this works for primitive types too.
  */
@@ -59,7 +59,7 @@ void analyze(Locals locals) {
 
         // ensure the specified type is part of the definition
         try {
-            type = Definition.getType(this.type);
+            type = locals.getDefinition().getType(this.type);
         } catch (IllegalArgumentException exception) {
             throw createError(new IllegalArgumentException("Not a type [" + this.type + "]."));
         }
@@ -96,4 +96,9 @@ void write(MethodWriter writer, Globals globals) {
             writer.instanceOf(org.objectweb.asm.Type.getType(resolvedType));
         }
     }
+
+    @Override
+    public String toString() {
+        return singleLineToString(expression, type);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
index 802da80860d16..748ff67ddd4b8 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
@@ -19,19 +19,19 @@
 
 package org.elasticsearch.painless.node;
 
+import org.elasticsearch.painless.AnalyzerCaster;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.Definition.Type;
+import org.elasticsearch.painless.FunctionRef;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Locals.Variable;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
-import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.node.SFunction.FunctionReserved;
-import org.elasticsearch.painless.Definition;
-import org.elasticsearch.painless.FunctionRef;
-import org.elasticsearch.painless.Globals;
 import org.objectweb.asm.Opcodes;
 
-import java.lang.invoke.LambdaMetafactory;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
@@ -39,7 +39,10 @@
 import java.util.Objects;
 import java.util.Set;
 
+import static org.elasticsearch.painless.Definition.VOID_TYPE;
+import static org.elasticsearch.painless.WriterConstants.CLASS_NAME;
 import static org.elasticsearch.painless.WriterConstants.LAMBDA_BOOTSTRAP_HANDLE;
+import static org.objectweb.asm.Opcodes.H_INVOKESTATIC;
 
 /**
  * Lambda expression node.
@@ -109,8 +112,15 @@ void analyze(Locals locals) {
             interfaceMethod = null;
             // we don't know anything: treat as def
             returnType = Definition.DEF_TYPE;
-            // don't infer any types
-            actualParamTypeStrs = paramTypeStrs;
+            // don't infer any types, replace any null types with def
+            actualParamTypeStrs = new ArrayList<>(paramTypeStrs.size());
+            for (String type : paramTypeStrs) {
+                if (type == null) {
+                    actualParamTypeStrs.add("def");
+                } else {
+                    actualParamTypeStrs.add(type);
+                }
+            }
         } else {
             // we know the method statically, infer return type and any unknown/def types
             interfaceMethod = expected.struct.getFunctionalMethod();
@@ -128,11 +138,11 @@ void analyze(Locals locals) {
             } else {
                 returnType = interfaceMethod.rtn;
             }
-            // replace any def types with the actual type (which could still be def)
-            actualParamTypeStrs = new ArrayList<String>();
+            // replace any null types with the actual type
+            actualParamTypeStrs = new ArrayList<>(paramTypeStrs.size());
             for (int i = 0; i < paramTypeStrs.size(); i++) {
                 String paramType = paramTypeStrs.get(i);
-                if (paramType.equals(Definition.DEF_TYPE.name)) {
+                if (paramType == null) {
                     actualParamTypeStrs.add(interfaceMethod.arguments.get(i).name);
                 } else {
                     actualParamTypeStrs.add(paramType);
@@ -152,8 +162,8 @@ void analyze(Locals locals) {
             }
         }
         // prepend capture list to lambda's arguments
-        List<String> paramTypes = new ArrayList<>();
-        List<String> paramNames = new ArrayList<>();
+        List<String> paramTypes = new ArrayList<>(captures.size() + actualParamTypeStrs.size());
+        List<String> paramNames = new ArrayList<>(captures.size() + paramNameStrs.size());
         for (Variable var : captures) {
             paramTypes.add(var.type.name);
             paramNames.add(var.name);
@@ -164,14 +174,14 @@ void analyze(Locals locals) {
         // desugar lambda body into a synthetic method
         desugared = new SFunction(reserved, location, returnType.name, name,
                                             paramTypes, paramNames, statements, true);
-        desugared.generateSignature();
+        desugared.generateSignature(locals.getDefinition());
         desugared.analyze(Locals.newLambdaScope(locals.getProgramScope(), returnType, desugared.parameters,
                                                 captures.size(), reserved.getMaxLoopCounter()));
 
         // setup method reference to synthetic method
         if (expected == null) {
             ref = null;
-            actual = Definition.getType("String");
+            actual = locals.getDefinition().getType("String");
             defPointer = "Sthis." + name + "," + captures.size();
         } else {
             defPointer = null;
@@ -180,6 +190,18 @@ void analyze(Locals locals) {
             } catch (IllegalArgumentException e) {
                 throw createError(e);
             }
+
+            // check casts between the interface method and the delegate method are legal
+            for (int i = 0; i < interfaceMethod.arguments.size(); ++i) {
+                Type from = interfaceMethod.arguments.get(i);
+                Type to = desugared.parameters.get(i + captures.size()).type;
+                AnalyzerCaster.getLegalCast(location, from, to, false, true);
+            }
+
+            if (interfaceMethod.rtn != VOID_TYPE) {
+                AnalyzerCaster.getLegalCast(location, desugared.rtnType, interfaceMethod.rtn, false, true);
+            }
+
             actual = expected;
         }
     }
@@ -194,31 +216,17 @@ void write(MethodWriter writer, Globals globals) {
             for (Variable capture : captures) {
                 writer.visitVarInsn(capture.type.type.getOpcode(Opcodes.ILOAD), capture.getSlot());
             }
-            // convert MethodTypes to asm Type for the constant pool.
-            String invokedType = ref.invokedType.toMethodDescriptorString();
-            org.objectweb.asm.Type samMethodType =
-                org.objectweb.asm.Type.getMethodType(ref.samMethodType.toMethodDescriptorString());
-            org.objectweb.asm.Type interfaceType =
-                org.objectweb.asm.Type.getMethodType(ref.interfaceMethodType.toMethodDescriptorString());
-            if (ref.needsBridges()) {
-                writer.invokeDynamic(ref.invokedName,
-                                     invokedType,
-                                     LAMBDA_BOOTSTRAP_HANDLE,
-                                     samMethodType,
-                                     ref.implMethodASM,
-                                     samMethodType,
-                                     LambdaMetafactory.FLAG_BRIDGES,
-                                     1,
-                                     interfaceType);
-            } else {
-                writer.invokeDynamic(ref.invokedName,
-                                     invokedType,
-                                     LAMBDA_BOOTSTRAP_HANDLE,
-                                     samMethodType,
-                                     ref.implMethodASM,
-                                     samMethodType,
-                                     0);
-            }
+
+            writer.invokeDynamic(
+                ref.interfaceMethodName,
+                ref.factoryDescriptor,
+                LAMBDA_BOOTSTRAP_HANDLE,
+                ref.interfaceType,
+                ref.delegateClassName,
+                ref.delegateInvokeType,
+                ref.delegateMethodName,
+                ref.delegateType
+            );
         } else {
             // placeholder
             writer.push((String)null);
@@ -245,4 +253,9 @@ public org.objectweb.asm.Type[] getCaptures() {
         }
         return types;
     }
+
+    @Override
+    public String toString() {
+        return multilineToString(pairwiseToString(paramTypeStrs, paramNameStrs), statements);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java
index dd93ec0ee72c7..999d35551cef2 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java
@@ -58,11 +58,7 @@ void analyze(Locals locals) {
             throw createError(new IllegalArgumentException("Must read from list initializer."));
         }
 
-        try {
-            actual = Definition.getType("ArrayList");
-        } catch (IllegalArgumentException exception) {
-            throw createError(new IllegalStateException("Illegal tree structure."));
-        }
+        actual = Definition.ARRAY_LIST_TYPE;
 
         constructor = actual.struct.constructors.get(new MethodKey("<init>", 0));
 
@@ -101,4 +97,9 @@ void write(MethodWriter writer, Globals globals) {
             writer.pop();
         }
     }
+
+    @Override
+    public String toString() {
+        return singleLineToString(values);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java
index 0dcba39d457c2..0647b5716e047 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java
@@ -64,11 +64,7 @@ void analyze(Locals locals) {
             throw createError(new IllegalArgumentException("Must read from map initializer."));
         }
 
-        try {
-            actual = Definition.getType("HashMap");
-        } catch (IllegalArgumentException exception) {
-            throw createError(new IllegalStateException("Illegal tree structure."));
-        }
+        actual = Definition.HASH_MAP_TYPE;
 
         constructor = actual.struct.constructors.get(new MethodKey("<init>", 0));
 
@@ -124,4 +120,9 @@ void write(MethodWriter writer, Globals globals) {
             writer.pop();
         }
     }
+
+    @Override
+    public String toString() {
+        return singleLineToString(pairwiseToString(keys, values));
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArray.java
index 3334b05e74eed..d32a153b79730 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArray.java
@@ -20,10 +20,10 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
-import org.elasticsearch.painless.Globals;
-import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Type;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 
 import java.util.List;
@@ -63,7 +63,7 @@ void analyze(Locals locals) {
         final Type type;
 
         try {
-            type = Definition.getType(this.type);
+            type = locals.getDefinition().getType(this.type);
         } catch (IllegalArgumentException exception) {
             throw createError(new IllegalArgumentException("Not a type [" + this.type + "]."));
         }
@@ -71,13 +71,14 @@ void analyze(Locals locals) {
         for (int argument = 0; argument < arguments.size(); ++argument) {
             AExpression expression = arguments.get(argument);
 
-            expression.expected = initialize ? Definition.getType(type.struct, 0) : Definition.INT_TYPE;
+            expression.expected = initialize ? locals.getDefinition().getType(type.struct, 0)
+                    : Definition.INT_TYPE;
             expression.internal = true;
             expression.analyze(locals);
             arguments.set(argument, expression.cast(locals));
         }
 
-        actual = Definition.getType(type.struct, initialize ? 1 : arguments.size());
+        actual = locals.getDefinition().getType(type.struct, initialize ? 1 : arguments.size());
     }
 
     @Override
@@ -86,7 +87,7 @@ void write(MethodWriter writer, Globals globals) {
 
         if (initialize) {
             writer.push(arguments.size());
-            writer.newArray(Definition.getType(actual.struct, 0).type);
+            writer.newArray(actual.struct.type);
 
             for (int index = 0; index < arguments.size(); ++index) {
                 AExpression argument = arguments.get(index);
@@ -94,7 +95,7 @@ void write(MethodWriter writer, Globals globals) {
                 writer.dup();
                 writer.push(index);
                 argument.write(writer, globals);
-                writer.arrayStore(Definition.getType(actual.struct, 0).type);
+                writer.arrayStore(actual.struct.type);
             }
         } else {
             for (AExpression argument : arguments) {
@@ -104,8 +105,13 @@ void write(MethodWriter writer, Globals globals) {
             if (arguments.size() > 1) {
                 writer.visitMultiANewArrayInsn(actual.type.getDescriptor(), actual.type.getDimensions());
             } else {
-                writer.newArray(Definition.getType(actual.struct, 0).type);
+                writer.newArray(actual.struct.type);
             }
         }
     }
+
+    @Override
+    public String toString() {
+        return singleLineToStringWithOptionalArgs(arguments, type, initialize ? "init" : "dims");
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java
index 7a19491fd0974..08cd76917c5fd 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java
@@ -20,12 +20,12 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
-import org.elasticsearch.painless.Globals;
-import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.Definition.Struct;
 import org.elasticsearch.painless.Definition.Type;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 
 import java.util.List;
@@ -61,7 +61,7 @@ void analyze(Locals locals) {
         final Type type;
 
         try {
-            type = Definition.getType(this.type);
+            type = locals.getDefinition().getType(this.type);
         } catch (IllegalArgumentException exception) {
             throw createError(new IllegalArgumentException("Not a type [" + this.type + "]."));
         }
@@ -110,4 +110,9 @@ void write(MethodWriter writer, Globals globals) {
 
         writer.invokeConstructor(constructor.owner.type, constructor.method);
     }
+
+    @Override
+    public String toString() {
+        return singleLineToStringWithOptionalArgs(arguments, type);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
index b07344e86b128..0ab574b2e0712 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
@@ -66,4 +66,9 @@ void analyze(Locals locals) {
     void write(MethodWriter writer, Globals globals) {
         writer.visitInsn(Opcodes.ACONST_NULL);
     }
+
+    @Override
+    public String toString() {
+        return singleLineToString();
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
index 102a849122635..8e14f7f1ea9df 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
@@ -31,7 +31,7 @@
 import org.elasticsearch.painless.MethodWriter;
 
 /**
- * Respresents a non-decimal numeric constant.
+ * Represents a non-decimal numeric constant.
  */
 public final class ENumeric extends AExpression {
 
@@ -104,6 +104,14 @@ void analyze(Locals locals) {
                     actual = Definition.INT_TYPE;
                 }
             } catch (NumberFormatException exception) {
+                try {
+                    // Check if we can parse as a long. If so then hint that the user might prefer that.
+                    Long.parseLong(value, radix);
+                    throw createError(new IllegalArgumentException("Invalid int constant [" + value + "]. If you want a long constant "
+                            + "then change it to [" + value + "L]."));
+                } catch (NumberFormatException longNoGood) {
+                    // Ignored
+                }
                 throw createError(new IllegalArgumentException("Invalid int constant [" + value + "]."));
             }
         }
@@ -113,4 +121,12 @@ void analyze(Locals locals) {
     void write(MethodWriter writer, Globals globals) {
         throw createError(new IllegalStateException("Illegal tree structure."));
     }
+
+    @Override
+    public String toString() {
+        if (radix != 10) {
+            return singleLineToString(value, radix);
+        }
+        return singleLineToString(value);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ERegex.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ERegex.java
index 8b9e1bcdc9cf1..4b38868b1b1fc 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ERegex.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ERegex.java
@@ -68,8 +68,9 @@ void analyze(Locals locals) {
 
         try {
             Pattern.compile(pattern, flags);
-        } catch (PatternSyntaxException exception) {
-            throw createError(exception);
+        } catch (PatternSyntaxException e) {
+            throw new Location(location.getSourceName(), location.getOffset() + 1 + e.getIndex()).createError(
+                    new IllegalArgumentException("Error compiling regex: " + e.getDescription()));
         }
 
         constant = new Constant(location, Definition.PATTERN_TYPE.type, "regexAt$" + location.getOffset(), this::initializeConstant);
@@ -104,4 +105,23 @@ private int flagForChar(char c) {
                 throw new IllegalArgumentException("Unknown flag [" + c + "]");
         }
     }
+
+    @Override
+    public String toString() {
+        StringBuilder f = new StringBuilder();
+        if ((flags & Pattern.CANON_EQ) != 0)                f.append('c');
+        if ((flags & Pattern.CASE_INSENSITIVE) != 0)        f.append('i');
+        if ((flags & Pattern.LITERAL) != 0)                 f.append('l');
+        if ((flags & Pattern.MULTILINE) != 0)               f.append('m');
+        if ((flags & Pattern.DOTALL) != 0)                  f.append('s');
+        if ((flags & Pattern.UNICODE_CHARACTER_CLASS) != 0) f.append('U');
+        if ((flags & Pattern.UNICODE_CASE) != 0)            f.append('u');
+        if ((flags & Pattern.COMMENTS) != 0)                f.append('x');
+
+        String p = "/" + pattern + "/";
+        if (f.length() == 0) {
+            return singleLineToString(p);
+        }
+        return singleLineToString(p, f);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EStatic.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EStatic.java
index 8e6846c4ef976..e675fb1510894 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EStatic.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EStatic.java
@@ -19,16 +19,14 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Globals;
+import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 
 import java.util.Objects;
 import java.util.Set;
 
-import org.elasticsearch.painless.Locals;
-
 /**
  * Represents a static type target.
  */
@@ -50,7 +48,7 @@ void extractVariables(Set<String> variables) {
     @Override
     void analyze(Locals locals) {
         try {
-            actual = Definition.getType(type);
+            actual = locals.getDefinition().getType(type);
         } catch (IllegalArgumentException exception) {
             throw createError(new IllegalArgumentException("Not a type [" + type + "]."));
         }
@@ -60,4 +58,9 @@ void analyze(Locals locals) {
     void write(MethodWriter writer, Globals globals) {
         // Do nothing.
     }
+
+    @Override
+    public String toString() {
+        return singleLineToString(type);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EString.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EString.java
index 84062ffccac58..0c2cec13e2431 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EString.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EString.java
@@ -57,4 +57,9 @@ void analyze(Locals locals) {
     void write(MethodWriter writer, Globals globals) {
         throw new IllegalStateException("Illegal tree structure.");
     }
+
+    @Override
+    public String toString() {
+        return singleLineToString("'" + constant.toString() + "'");
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
index 129e6c67f6eda..cd483d7d71a24 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
@@ -249,4 +249,9 @@ void write(MethodWriter writer, Globals globals) {
             }
         }
     }
+
+    @Override
+    public String toString() {
+        return singleLineToString(operation.symbol, child);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EVariable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EVariable.java
index 11c29dc7514af..40cf194df2370 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EVariable.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EVariable.java
@@ -95,4 +95,9 @@ void load(MethodWriter writer, Globals globals) {
     void store(MethodWriter writer, Globals globals) {
         writer.visitVarInsn(actual.type.getOpcode(Opcodes.ISTORE), variable.getSlot());
     }
+
+    @Override
+    public String toString() {
+        return singleLineToString(name);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PBrace.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PBrace.java
index 9732ab7288615..4f04db092d58a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PBrace.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PBrace.java
@@ -117,4 +117,9 @@ void load(MethodWriter writer, Globals globals) {
     void store(MethodWriter writer, Globals globals) {
         sub.store(writer, globals);
     }
+
+    @Override
+    public String toString() {
+        return singleLineToString(prefix, index);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java
index e8cfb1eba2e7d..80bdc3d597c8a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.Definition.MethodKey;
 import org.elasticsearch.painless.Definition.Sort;
@@ -39,14 +38,16 @@
 public final class PCallInvoke extends AExpression {
 
     private final String name;
+    private final boolean nullSafe;
     private final List<AExpression> arguments;
 
     private AExpression sub = null;
 
-    public PCallInvoke(Location location, AExpression prefix, String name, List<AExpression> arguments) {
+    public PCallInvoke(Location location, AExpression prefix, String name, boolean nullSafe, List<AExpression> arguments) {
         super(location, prefix);
 
         this.name = Objects.requireNonNull(name);
+        this.nullSafe = nullSafe;
         this.arguments = Objects.requireNonNull(arguments);
     }
 
@@ -72,7 +73,7 @@ void analyze(Locals locals) {
         Struct struct = prefix.actual.struct;
 
         if (prefix.actual.sort.primitive) {
-            struct = Definition.getType(prefix.actual.sort.boxed.getSimpleName()).struct;
+            struct = locals.getDefinition().getType(prefix.actual.sort.boxed.getSimpleName()).struct;
         }
 
         MethodKey methodKey = new MethodKey(name, arguments.size());
@@ -87,6 +88,10 @@ void analyze(Locals locals) {
                 "Unknown call [" + name + "] with [" + arguments.size() + "] arguments on type [" + struct.name + "]."));
         }
 
+        if (nullSafe) {
+            sub = new PSubNullSafeCallInvoke(location, sub);
+        }
+
         sub.expected = expected;
         sub.explicit = explicit;
         sub.analyze(locals);
@@ -100,4 +105,9 @@ void write(MethodWriter writer, Globals globals) {
         prefix.write(writer, globals);
         sub.write(writer, globals);
     }
+
+    @Override
+    public String toString() {
+        return singleLineToStringWithOptionalArgs(arguments, prefix, name);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java
index 21a3def318936..f22a5d49203eb 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java
@@ -40,13 +40,15 @@
  */
 public final class PField extends AStoreable {
 
+    private final boolean nullSafe;
     private final String value;
 
     private AStoreable sub = null;
 
-    public PField(Location location, AExpression prefix, String value) {
+    public PField(Location location, AExpression prefix, boolean nullSafe, String value) {
         super(location, prefix);
 
+        this.nullSafe = nullSafe;
         this.value = Objects.requireNonNull(value);
     }
 
@@ -64,7 +66,7 @@ void analyze(Locals locals) {
         Sort sort = prefix.actual.sort;
 
         if (sort == Sort.ARRAY) {
-            sub = new PSubArrayLength(location,prefix.actual.name, value);
+            sub = new PSubArrayLength(location, prefix.actual.name, value);
         } else if (sort == Sort.DEF) {
             sub = new PSubDefField(location, value);
         } else {
@@ -106,6 +108,10 @@ void analyze(Locals locals) {
             throw createError(new IllegalArgumentException("Unknown field [" + value + "] for type [" + prefix.actual.name + "]."));
         }
 
+        if (nullSafe) {
+            sub = new PSubNullSafeField(location, sub);
+        }
+
         sub.write = write;
         sub.read = read;
         sub.expected = expected;
@@ -151,4 +157,12 @@ void load(MethodWriter writer, Globals globals) {
     void store(MethodWriter writer, Globals globals) {
         sub.store(writer, globals);
     }
+
+    @Override
+    public String toString() {
+        if (nullSafe) {
+            return singleLineToString("nullSafe", prefix, value);
+        }
+        return singleLineToString(prefix, value);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubArrayLength.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubArrayLength.java
index 610bd2df77d9e..806f971850f01 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubArrayLength.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubArrayLength.java
@@ -97,4 +97,9 @@ void load(MethodWriter writer, Globals globals) {
     void store(MethodWriter writer, Globals globals) {
         throw new IllegalStateException("Illegal tree structure.");
     }
+
+    @Override
+    public String toString() {
+        return singleLineToString(prefix);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubBrace.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubBrace.java
index 45b3ef88cd117..1f3d4109bca8a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubBrace.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubBrace.java
@@ -55,15 +55,13 @@ void analyze(Locals locals) {
         index.analyze(locals);
         index = index.cast(locals);
 
-        actual = Definition.getType(type.struct, type.dimensions - 1);
+        actual = locals.getDefinition().getType(type.struct, type.dimensions - 1);
     }
 
     @Override
     void write(MethodWriter writer, Globals globals) {
-        if (!write) {
-            setup(writer, globals);
-            load(writer, globals);
-        }
+        setup(writer, globals);
+        load(writer, globals);
     }
 
     @Override
@@ -84,6 +82,7 @@ void updateActual(Type actual) {
     @Override
     void setup(MethodWriter writer, Globals globals) {
         index.write(writer, globals);
+        writeIndexFlip(writer, MethodWriter::arrayLength);
     }
 
     @Override
@@ -97,4 +96,9 @@ void store(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
         writer.arrayStore(actual.type);
     }
+
+    @Override
+    public String toString() {
+        return singleLineToString(prefix, index);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubCallInvoke.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubCallInvoke.java
index 02dd89bb6025c..5665f7163af47 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubCallInvoke.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubCallInvoke.java
@@ -39,7 +39,7 @@ final class PSubCallInvoke extends AExpression {
     private final Type box;
     private final List<AExpression> arguments;
 
-    public PSubCallInvoke(Location location, Method method, Type box, List<AExpression> arguments) {
+    PSubCallInvoke(Location location, Method method, Type box, List<AExpression> arguments) {
         super(location);
 
         this.method = Objects.requireNonNull(method);
@@ -81,4 +81,9 @@ void write(MethodWriter writer, Globals globals) {
 
         method.write(writer);
     }
+
+    @Override
+    public String toString() {
+        return singleLineToStringWithOptionalArgs(arguments, prefix, method.name);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefArray.java
index 2153897a00055..c13c1bcb99238 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefArray.java
@@ -34,7 +34,6 @@
  * Represents an array load/store or shortcut on a def type.  (Internal only.)
  */
 final class PSubDefArray extends AStoreable {
-
     private AExpression index;
 
     PSubDefArray(Location location, AExpression index) {
@@ -59,13 +58,8 @@ void analyze(Locals locals) {
 
     @Override
     void write(MethodWriter writer, Globals globals) {
-        index.write(writer, globals);
-
-        writer.writeDebugInfo(location);
-
-        org.objectweb.asm.Type methodType =
-            org.objectweb.asm.Type.getMethodType(actual.type, Definition.DEF_TYPE.type, index.actual.type);
-        writer.invokeDefCall("arrayLoad", methodType, DefBootstrap.ARRAY_LOAD);
+        setup(writer, globals);
+        load(writer, globals);
     }
 
     @Override
@@ -85,7 +79,12 @@ void updateActual(Type actual) {
 
     @Override
     void setup(MethodWriter writer, Globals globals) {
-        index.write(writer, globals);
+        // Current stack:                                                                    def
+        writer.dup();                                                                     // def, def
+        index.write(writer, globals);                                                     // def, def, unnormalized_index
+        org.objectweb.asm.Type methodType = org.objectweb.asm.Type.getMethodType(
+                index.actual.type, Definition.DEF_TYPE.type, index.actual.type);
+        writer.invokeDefCall("normalizeIndex", methodType, DefBootstrap.INDEX_NORMALIZE); // def, normalized_index
     }
 
     @Override
@@ -105,4 +104,9 @@ void store(MethodWriter writer, Globals globals) {
             org.objectweb.asm.Type.getMethodType(Definition.VOID_TYPE.type, Definition.DEF_TYPE.type, index.actual.type, actual.type);
         writer.invokeDefCall("arrayStore", methodType, DefBootstrap.ARRAY_STORE);
     }
+
+    @Override
+    public String toString() {
+        return singleLineToString(prefix, index);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefCall.java
index 7f1d0dd1e9585..2d0734915af8b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefCall.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefCall.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.painless.node;
 
+import java.util.Collections;
 import org.elasticsearch.painless.DefBootstrap;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Globals;
@@ -97,10 +98,7 @@ void write(MethodWriter writer, Globals globals) {
 
             if (argument instanceof ILambda) {
                 ILambda lambda = (ILambda) argument;
-
-                for (Type capture : lambda.getCaptures()) {
-                    parameterTypes.add(capture);
-                }
+                Collections.addAll(parameterTypes, lambda.getCaptures());
             }
 
             argument.write(writer, globals);
@@ -114,4 +112,9 @@ void write(MethodWriter writer, Globals globals) {
         args.addAll(pointers);
         writer.invokeDefCall(name, methodType, DefBootstrap.METHOD_CALL, args.toArray());
     }
+
+    @Override
+    public String toString() {
+        return singleLineToStringWithOptionalArgs(arguments, prefix, name);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefField.java
index 8b098fa99f6c8..24c6c91422d08 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefField.java
@@ -99,4 +99,9 @@ void store(MethodWriter writer, Globals globals) {
             org.objectweb.asm.Type.getMethodType(Definition.VOID_TYPE.type, Definition.DEF_TYPE.type, actual.type);
         writer.invokeDefCall(value, methodType, DefBootstrap.STORE);
     }
+
+    @Override
+    public String toString() {
+        return singleLineToString(prefix, value);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubField.java
index 8b85a260d51db..efed81c4c81da 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubField.java
@@ -37,7 +37,7 @@ final class PSubField extends AStoreable {
 
     private final Field field;
 
-    public PSubField(Location location, Field field) {
+    PSubField(Location location, Field field) {
         super(location);
 
         this.field = Objects.requireNonNull(field);
@@ -110,4 +110,9 @@ void store(MethodWriter writer, Globals globals) {
             writer.putField(field.owner.type, field.javaName, field.type.type);
         }
     }
+
+    @Override
+    public String toString() {
+        return singleLineToString(prefix, field.name);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java
index c13f823582110..fc5cecc1b5e4a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
+import org.elasticsearch.painless.WriterConstants;
 
 import java.util.Objects;
 import java.util.Set;
@@ -87,15 +88,8 @@ void analyze(Locals locals) {
 
     @Override
     void write(MethodWriter writer, Globals globals) {
-        index.write(writer, globals);
-
-        writer.writeDebugInfo(location);
-
-        getter.write(writer);
-
-        if (!getter.rtn.clazz.equals(getter.handle.type().returnType())) {
-            writer.checkCast(getter.rtn.type);
-        }
+        setup(writer, globals);
+        load(writer, globals);
     }
 
     @Override
@@ -116,6 +110,9 @@ void updateActual(Type actual) {
     @Override
     void setup(MethodWriter writer, Globals globals) {
         index.write(writer, globals);
+        writeIndexFlip(writer, w -> {
+            w.invokeInterface(WriterConstants.COLLECTION_TYPE, WriterConstants.COLLECTION_SIZE);
+        });
     }
 
     @Override
@@ -137,4 +134,9 @@ void store(MethodWriter writer, Globals globals) {
 
         writer.writePop(setter.rtn.sort.size);
     }
+
+    @Override
+    public String toString() {
+        return singleLineToString(prefix, index);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java
index 183ee8237522c..c98aba5e1f09b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java
@@ -137,4 +137,9 @@ void store(MethodWriter writer, Globals globals) {
 
         writer.writePop(setter.rtn.sort.size);
     }
+
+    @Override
+    public String toString() {
+        return singleLineToString(prefix, index);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubNullSafeCallInvoke.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubNullSafeCallInvoke.java
new file mode 100644
index 0000000000000..6291e94252d43
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubNullSafeCallInvoke.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.Globals;
+import org.elasticsearch.painless.Locals;
+import org.elasticsearch.painless.Location;
+import org.elasticsearch.painless.MethodWriter;
+import org.objectweb.asm.Label;
+
+import java.util.Set;
+
+import static java.util.Objects.requireNonNull;
+
+/**
+ * Implements a call who's value is null if the prefix is null rather than throwing an NPE.
+ */
+public class PSubNullSafeCallInvoke extends AExpression {
+    /**
+     * The expression gaurded by the null check. Required at construction time and replaced at analysis time.
+     */
+    private AExpression guarded;
+
+    public PSubNullSafeCallInvoke(Location location, AExpression guarded) {
+        super(location);
+        this.guarded = requireNonNull(guarded);
+    }
+
+    @Override
+    void extractVariables(Set<String> variables) {
+        guarded.extractVariables(variables);
+    }
+
+    @Override
+    void analyze(Locals locals) {
+        guarded.analyze(locals);
+        actual = guarded.actual;
+        if (actual.sort.primitive) {
+            throw new IllegalArgumentException("Result of null safe operator must be nullable");
+        }
+    }
+
+    @Override
+    void write(MethodWriter writer, Globals globals) {
+        writer.writeDebugInfo(location);
+
+        Label end = new Label();
+        writer.dup();
+        writer.ifNull(end);
+        guarded.write(writer, globals);
+        writer.mark(end);
+    }
+
+    @Override
+    public String toString() {
+        return singleLineToString(guarded);
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubNullSafeField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubNullSafeField.java
new file mode 100644
index 0000000000000..de01ea710d90e
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubNullSafeField.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.Definition.Type;
+import org.elasticsearch.painless.Globals;
+import org.elasticsearch.painless.Locals;
+import org.elasticsearch.painless.Location;
+import org.elasticsearch.painless.MethodWriter;
+import org.objectweb.asm.Label;
+
+import java.util.Set;
+
+/**
+ * Implements a field who's value is null if the prefix is null rather than throwing an NPE.
+ */
+public class PSubNullSafeField extends AStoreable {
+    private AStoreable guarded;
+
+    public PSubNullSafeField(Location location, AStoreable guarded) {
+        super(location);
+        this.guarded = guarded;
+    }
+
+    @Override
+    void extractVariables(Set<String> variables) {
+        guarded.extractVariables(variables);
+    }
+
+    @Override
+    void analyze(Locals locals) {
+        if (write) {
+            throw createError(new IllegalArgumentException("Can't write to null safe reference"));
+        }
+        guarded.read = read;
+        guarded.analyze(locals);
+        actual = guarded.actual;
+        if (actual.sort.primitive) {
+            throw new IllegalArgumentException("Result of null safe operator must be nullable");
+        }
+    }
+
+
+    @Override
+    int accessElementCount() {
+        return guarded.accessElementCount();
+    }
+
+    @Override
+    boolean isDefOptimized() {
+        return guarded.isDefOptimized();
+    }
+
+    @Override
+    void updateActual(Type actual) {
+        guarded.updateActual(actual);
+    }
+
+    @Override
+    void write(MethodWriter writer, Globals globals) {
+        Label end = new Label();
+        writer.dup();
+        writer.ifNull(end);
+        guarded.write(writer, globals);
+        writer.mark(end);
+    }
+
+    @Override
+    void setup(MethodWriter writer, Globals globals) {
+        throw createError(new IllegalArgumentException("Can't write to null safe field"));
+    }
+
+    @Override
+    void load(MethodWriter writer, Globals globals) {
+        throw createError(new IllegalArgumentException("Can't write to null safe field"));
+    }
+
+    @Override
+    void store(MethodWriter writer, Globals globals) {
+        throw createError(new IllegalArgumentException("Can't write to null safe field"));
+    }
+
+    @Override
+    public String toString() {
+        return singleLineToString(guarded);
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubShortcut.java
index 34720a275db48..5cd62f9fb1ed3 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubShortcut.java
@@ -126,4 +126,9 @@ void store(MethodWriter writer, Globals globals) {
 
         writer.writePop(setter.rtn.sort.size);
     }
+
+    @Override
+    public String toString() {
+        return singleLineToString(prefix, value);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
index d81437242a87f..43e0a62fb00fd 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
@@ -28,6 +28,8 @@
 import java.util.List;
 import java.util.Set;
 
+import static java.util.Collections.emptyList;
+
 /**
  * Represents a set of statements as a branch of control-flow.
  */
@@ -86,4 +88,9 @@ void write(MethodWriter writer, Globals globals) {
             statement.write(writer, globals);
         }
     }
+
+    @Override
+    public String toString() {
+        return multilineToString(emptyList(), statements);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
index 71d737fcd66ae..bd446901e3ac6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
@@ -56,4 +56,9 @@ void analyze(Locals locals) {
     void write(MethodWriter writer, Globals globals) {
         writer.goTo(brake);
     }
+
+    @Override
+    public String toString() {
+        return singleLineToString();
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java
index d73aead49c8ce..6940e48342a26 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
@@ -69,7 +68,7 @@ void analyze(Locals locals) {
         final Type type;
 
         try {
-            type = Definition.getType(this.type);
+            type = locals.getDefinition().getType(this.type);
         } catch (IllegalArgumentException exception) {
             throw createError(new IllegalArgumentException("Not a type [" + this.type + "]."));
         }
@@ -117,4 +116,9 @@ void write(MethodWriter writer, Globals globals) {
             writer.goTo(exception);
         }
     }
+
+    @Override
+    public String toString() {
+        return singleLineToString(type, name, block);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
index 5504ae8b268a6..2cd1ed15cd245 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
@@ -59,4 +59,9 @@ void analyze(Locals locals) {
     void write(MethodWriter writer, Globals globals) {
         writer.goTo(continu);
     }
+
+    @Override
+    public String toString() {
+        return singleLineToString();
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
index 4bade4625eed1..95af41ba2b88e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
@@ -28,6 +28,8 @@
 import java.util.List;
 import java.util.Set;
 
+import static java.util.Collections.emptyList;
+
 /**
  * Represents a series of declarations.
  */
@@ -63,4 +65,9 @@ void write(MethodWriter writer, Globals globals) {
             declaration.write(writer, globals);
         }
     }
+
+    @Override
+    public String toString() {
+        return multilineToString(emptyList(), declarations);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
index c0911232d2169..ab9e58db23ebb 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
@@ -64,7 +63,7 @@ void analyze(Locals locals) {
         final Type type;
 
         try {
-            type = Definition.getType(this.type);
+            type = locals.getDefinition().getType(this.type);
         } catch (IllegalArgumentException exception) {
             throw createError(new IllegalArgumentException("Not a type [" + this.type + "]."));
         }
@@ -101,4 +100,12 @@ void write(MethodWriter writer, Globals globals) {
 
         writer.visitVarInsn(variable.type.type.getOpcode(Opcodes.ISTORE), variable.getSlot());
     }
+
+    @Override
+    public String toString() {
+        if (expression == null) {
+            return singleLineToString(type, name);
+        }
+        return singleLineToString(type, name, expression);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
index d63910683bee6..979bd4eec9fab 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
@@ -125,4 +125,9 @@ void write(MethodWriter writer, Globals globals) {
         writer.goTo(start);
         writer.mark(end);
     }
+
+    @Override
+    public String toString() {
+        return singleLineToString(condition, block);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
index 6d0d0ee1bcc68..003f303b5e023 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.Globals;
@@ -72,7 +71,7 @@ void analyze(Locals locals) {
         final Type type;
 
         try {
-            type = Definition.getType(this.type);
+            type = locals.getDefinition().getType(this.type);
         } catch (IllegalArgumentException exception) {
             throw createError(new IllegalArgumentException("Not a type [" + this.type + "]."));
         }
@@ -114,4 +113,9 @@ void analyze(Locals locals) {
     void write(MethodWriter writer, Globals globals) {
         sub.write(writer, globals);
     }
+
+    @Override
+    public String toString() {
+        return singleLineToString(type, name, expression, block);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
index 4493de0953354..ab7a645c39621 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
@@ -82,4 +82,9 @@ void write(MethodWriter writer, Globals globals) {
             writer.writePop(expression.expected.sort.size);
         }
     }
+
+    @Override
+    public String toString() {
+        return singleLineToString(expression);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
index eb24ebb5e3f9f..18e7e11a7d1d0 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
@@ -27,8 +27,11 @@
 import org.objectweb.asm.Label;
 import org.objectweb.asm.Opcodes;
 
+import java.util.Arrays;
 import java.util.Set;
 
+import static java.util.Collections.emptyList;
+
 /**
  * Represents a for loop.
  */
@@ -203,4 +206,9 @@ void write(MethodWriter writer, Globals globals) {
 
         writer.mark(end);
     }
+
+    @Override
+    public String toString() {
+        return multilineToString(emptyList(), Arrays.asList(initializer, condition, afterthought, block));
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
index 44afe828ef24c..257f2975c939a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
@@ -46,6 +46,7 @@
 import java.util.Objects;
 import java.util.Set;
 
+import static java.util.Collections.emptyList;
 import static org.elasticsearch.painless.WriterConstants.CLASS_TYPE;
 
 /**
@@ -53,19 +54,13 @@
  */
 public final class SFunction extends AStatement {
     public static final class FunctionReserved implements Reserved {
-        public static final String THIS = "#this";
-        public static final String LOOP = "#loop";
-
         private int maxLoopCounter = 0;
 
-        public void markReserved(String name) {
+        @Override
+        public void markUsedVariable(String name) {
             // Do nothing.
         }
 
-        public boolean isReserved(String name) {
-            return name.equals(THIS) || name.equals(LOOP);
-        }
-
         @Override
         public void setMaxLoopCounter(int max) {
             maxLoopCounter = max;
@@ -111,9 +106,9 @@ void extractVariables(Set<String> variables) {
         throw new IllegalStateException("Illegal tree structure");
     }
 
-    void generateSignature() {
+    void generateSignature(Definition definition) {
         try {
-            rtnType = Definition.getType(rtnTypeStr);
+            rtnType = definition.getType(rtnTypeStr);
         } catch (IllegalArgumentException exception) {
             throw createError(new IllegalArgumentException("Illegal return type [" + rtnTypeStr + "] for function [" + name + "]."));
         }
@@ -127,7 +122,7 @@ void generateSignature() {
 
         for (int param = 0; param < this.paramTypeStrs.size(); ++param) {
             try {
-                Type paramType = Definition.getType(this.paramTypeStrs.get(param));
+                Type paramType = definition.getType(this.paramTypeStrs.get(param));
 
                 paramClasses[param] = paramType.clazz;
                 paramTypes.add(paramType);
@@ -173,13 +168,13 @@ void analyze(Locals locals) {
         }
 
         if (reserved.getMaxLoopCounter() > 0) {
-            loop = locals.getVariable(null, FunctionReserved.LOOP);
+            loop = locals.getVariable(null, Locals.LOOP);
         }
     }
 
     /** Writes the function to given ClassVisitor. */
     void write (ClassVisitor writer, CompilerSettings settings, Globals globals) {
-        int access = Opcodes.ACC_PRIVATE | Opcodes.ACC_STATIC;
+        int access = Opcodes.ACC_PUBLIC | Opcodes.ACC_STATIC;
         if (synthetic) {
             access |= Opcodes.ACC_SYNTHETIC;
         }
@@ -223,4 +218,15 @@ private void initializeConstant(MethodWriter writer) {
                 false);
         writer.push(handle);
     }
+
+    @Override
+    public String toString() {
+        List<Object> description = new ArrayList<>();
+        description.add(rtnTypeStr);
+        description.add(name);
+        if (false == (paramTypeStrs.isEmpty() && paramNameStrs.isEmpty())) {
+            description.add(joinWithName("Args", pairwiseToString(paramTypeStrs, paramNameStrs), emptyList()));
+        }
+        return multilineToString(description, statements);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java
index a266df464ea44..08495cd8d7f1a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java
@@ -94,4 +94,9 @@ void write(MethodWriter writer, Globals globals) {
 
         writer.mark(fals);
     }
+
+    @Override
+    public String toString() {
+        return singleLineToString(condition, ifblock);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
index d36fb6cd37082..4feb6b3fe62b1 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
@@ -27,9 +27,12 @@
 import org.objectweb.asm.Label;
 import org.objectweb.asm.Opcodes;
 
+import java.util.Arrays;
 import java.util.Objects;
 import java.util.Set;
 
+import static java.util.Collections.singleton;
+
 /**
  * Represents an if/else block.
  */
@@ -128,4 +131,9 @@ void write(MethodWriter writer, Globals globals) {
 
         writer.mark(end);
     }
+
+    @Override
+    public String toString() {
+        return multilineToString(singleton(condition), Arrays.asList(ifblock, elseblock));
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
index e6986f5bdf66c..f57d6339442b9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
@@ -65,4 +65,9 @@ void write(MethodWriter writer, Globals globals) {
         expression.write(writer, globals);
         writer.returnValue();
     }
+
+    @Override
+    public String toString() {
+        return singleLineToString(expression); 
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
index a4cf1cc8eee5f..eb323e3293fdd 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
@@ -21,19 +21,22 @@
 
 import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Constant;
+import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.Definition.MethodKey;
-import org.elasticsearch.painless.Executable;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Locals.Variable;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
+import org.elasticsearch.painless.ScriptInterface;
 import org.elasticsearch.painless.SimpleChecksAdapter;
 import org.elasticsearch.painless.WriterConstants;
 import org.objectweb.asm.ClassVisitor;
 import org.objectweb.asm.ClassWriter;
+import org.objectweb.asm.Label;
 import org.objectweb.asm.Opcodes;
+import org.objectweb.asm.Type;
 import org.objectweb.asm.util.Printer;
 import org.objectweb.asm.util.TraceClassVisitor;
 
@@ -42,17 +45,31 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 
+import static java.util.Collections.emptyList;
+import static java.util.Collections.unmodifiableSet;
 import static org.elasticsearch.painless.WriterConstants.BASE_CLASS_TYPE;
+import static org.elasticsearch.painless.WriterConstants.BOOTSTRAP_METHOD_ERROR_TYPE;
 import static org.elasticsearch.painless.WriterConstants.CLASS_TYPE;
+import static org.elasticsearch.painless.WriterConstants.COLLECTIONS_TYPE;
 import static org.elasticsearch.painless.WriterConstants.CONSTRUCTOR;
-import static org.elasticsearch.painless.WriterConstants.EXECUTE;
-import static org.elasticsearch.painless.WriterConstants.MAP_GET;
-import static org.elasticsearch.painless.WriterConstants.MAP_TYPE;
+import static org.elasticsearch.painless.WriterConstants.CONVERT_TO_SCRIPT_EXCEPTION_METHOD;
+import static org.elasticsearch.painless.WriterConstants.DEFINITION_TYPE;
+import static org.elasticsearch.painless.WriterConstants.DEF_BOOTSTRAP_DELEGATE_METHOD;
+import static org.elasticsearch.painless.WriterConstants.DEF_BOOTSTRAP_DELEGATE_TYPE;
+import static org.elasticsearch.painless.WriterConstants.DEF_BOOTSTRAP_METHOD;
+import static org.elasticsearch.painless.WriterConstants.EMPTY_MAP_METHOD;
+import static org.elasticsearch.painless.WriterConstants.EXCEPTION_TYPE;
+import static org.elasticsearch.painless.WriterConstants.OUT_OF_MEMORY_ERROR_TYPE;
+import static org.elasticsearch.painless.WriterConstants.PAINLESS_ERROR_TYPE;
+import static org.elasticsearch.painless.WriterConstants.PAINLESS_EXPLAIN_ERROR_GET_HEADERS_METHOD;
+import static org.elasticsearch.painless.WriterConstants.PAINLESS_EXPLAIN_ERROR_TYPE;
+import static org.elasticsearch.painless.WriterConstants.STACK_OVERFLOW_ERROR_TYPE;
 
 /**
  * The root of all Painless trees.  Contains a series of statements.
@@ -60,44 +77,25 @@
 public final class SSource extends AStatement {
 
     /**
-     * Tracks reserved variables.  Must be given to any source of input
+     * Tracks derived arguments and the loop counter.  Must be given to any source of input
      * prior to beginning the analysis phase so that reserved variables
      * are known ahead of time to assign appropriate slots without
      * being wasteful.
      */
     public interface Reserved {
-        void markReserved(String name);
-        boolean isReserved(String name);
+        void markUsedVariable(String name);
 
         void setMaxLoopCounter(int max);
         int getMaxLoopCounter();
     }
 
     public static final class MainMethodReserved implements Reserved {
-        private boolean score = false;
-        private boolean ctx = false;
+        private final Set<String> usedVariables = new HashSet<>();
         private int maxLoopCounter = 0;
 
         @Override
-        public void markReserved(String name) {
-            if (Locals.SCORE.equals(name)) {
-                score = true;
-            } else if (Locals.CTX.equals(name)) {
-                ctx = true;
-            }
-        }
-
-        @Override
-        public boolean isReserved(String name) {
-            return Locals.KEYWORDS.contains(name);
-        }
-
-        public boolean usesScore() {
-            return score;
-        }
-
-        public boolean usesCtx() {
-            return ctx;
+        public void markUsedVariable(String name) {
+            usedVariables.add(name);
         }
 
         @Override
@@ -109,8 +107,13 @@ public void setMaxLoopCounter(int max) {
         public int getMaxLoopCounter() {
             return maxLoopCounter;
         }
+
+        public Set<String> getUsedVariables() {
+            return unmodifiableSet(usedVariables);
+        }
     }
 
+    private final ScriptInterface scriptInterface;
     private final CompilerSettings settings;
     private final String name;
     private final String source;
@@ -123,10 +126,11 @@ public int getMaxLoopCounter() {
     private Locals mainMethod;
     private byte[] bytes;
 
-    public SSource(CompilerSettings settings, String name, String source, Printer debugStream,
+    public SSource(ScriptInterface scriptInterface, CompilerSettings settings, String name, String source, Printer debugStream,
                    MainMethodReserved reserved, Location location,
                    List<SFunction> functions, Globals globals, List<AStatement> statements) {
         super(location);
+        this.scriptInterface = Objects.requireNonNull(scriptInterface);
         this.settings = Objects.requireNonNull(settings);
         this.name = Objects.requireNonNull(name);
         this.source = Objects.requireNonNull(source);
@@ -146,11 +150,11 @@ void extractVariables(Set<String> variables) {
         throw new IllegalStateException("Illegal tree structure.");
     }
 
-    public void analyze() {
+    public void analyze(Definition definition) {
         Map<MethodKey, Method> methods = new HashMap<>();
 
         for (SFunction function : functions) {
-            function.generateSignature();
+            function.generateSignature(definition);
 
             MethodKey key = new MethodKey(function.name, function.parameters.size());
 
@@ -159,7 +163,7 @@ public void analyze() {
             }
         }
 
-        analyze(Locals.newProgramScope(methods.values()));
+        analyze(Locals.newProgramScope(definition, methods.values()));
     }
 
     @Override
@@ -174,7 +178,7 @@ void analyze(Locals program) {
             throw createError(new IllegalArgumentException("Cannot generate an empty script."));
         }
 
-        mainMethod = Locals.newMainMethodScope(program, reserved.usesScore(), reserved.usesCtx(), reserved.getMaxLoopCounter());
+        mainMethod = Locals.newMainMethodScope(scriptInterface, program, reserved.getMaxLoopCounter());
 
         AStatement last = statements.get(statements.size() - 1);
 
@@ -201,7 +205,7 @@ public void write() {
         int classAccess = Opcodes.ACC_PUBLIC | Opcodes.ACC_SUPER | Opcodes.ACC_FINAL;
         String classBase = BASE_CLASS_TYPE.getInternalName();
         String className = CLASS_TYPE.getInternalName();
-        String classInterfaces[] = reserved.usesScore() ? new String[] { WriterConstants.NEEDS_SCORE_TYPE.getInternalName() } : null;
+        String classInterfaces[] = new String[] { Type.getType(scriptInterface.getInterface()).getInternalName() };
 
         ClassWriter writer = new ClassWriter(classFrames);
         ClassVisitor visitor = writer;
@@ -217,20 +221,34 @@ public void write() {
         visitor.visit(WriterConstants.CLASS_VERSION, classAccess, className, null, classBase, classInterfaces);
         visitor.visitSource(Location.computeSourceName(name, source), null);
 
+        // Write the a method to bootstrap def calls
+        MethodWriter bootstrapDef = new MethodWriter(Opcodes.ACC_STATIC | Opcodes.ACC_VARARGS, DEF_BOOTSTRAP_METHOD, visitor,
+                globals.getStatements(), settings);
+        bootstrapDef.visitCode();
+        bootstrapDef.getStatic(CLASS_TYPE, "$DEFINITION", DEFINITION_TYPE);
+        bootstrapDef.loadArgs();
+        bootstrapDef.invokeStatic(DEF_BOOTSTRAP_DELEGATE_TYPE, DEF_BOOTSTRAP_DELEGATE_METHOD);
+        bootstrapDef.returnValue();
+        bootstrapDef.endMethod();
+
+        // Write the static variable used by the method to bootstrap def calls
+        visitor.visitField(Opcodes.ACC_PUBLIC | Opcodes.ACC_STATIC, "$DEFINITION", DEFINITION_TYPE.getDescriptor(), null, null).visitEnd();
+
         // Write the constructor:
         MethodWriter constructor = new MethodWriter(Opcodes.ACC_PUBLIC, CONSTRUCTOR, visitor, globals.getStatements(), settings);
         constructor.visitCode();
         constructor.loadThis();
         constructor.loadArgs();
-        constructor.invokeConstructor(org.objectweb.asm.Type.getType(Executable.class), CONSTRUCTOR);
+        constructor.invokeConstructor(BASE_CLASS_TYPE, CONSTRUCTOR);
         constructor.returnValue();
         constructor.endMethod();
 
-        // Write the execute method:
-        MethodWriter execute = new MethodWriter(Opcodes.ACC_PUBLIC, EXECUTE, visitor, globals.getStatements(), settings);
-        execute.visitCode();
-        write(execute, globals);
-        execute.endMethod();
+        // Write the method defined in the interface:
+        MethodWriter executeMethod = new MethodWriter(Opcodes.ACC_PUBLIC, scriptInterface.getExecuteMethod(), visitor,
+                globals.getStatements(), settings);
+        executeMethod.visitCode();
+        write(executeMethod, globals);
+        executeMethod.endMethod();
 
         // Write all functions:
         for (SFunction function : functions) {
@@ -272,6 +290,15 @@ public void write() {
             clinit.endMethod();
         }
 
+        // Write any uses$varName methods for used variables
+        for (org.objectweb.asm.commons.Method usesMethod : scriptInterface.getUsesMethods()) {
+            MethodWriter ifaceMethod = new MethodWriter(Opcodes.ACC_PUBLIC, usesMethod, visitor, globals.getStatements(), settings);
+            ifaceMethod.visitCode();
+            ifaceMethod.push(reserved.getUsedVariables().contains(usesMethod.getName().substring("uses$".length())));
+            ifaceMethod.returnValue();
+            ifaceMethod.endMethod();
+        }
+
         // End writing the class and store the generated bytes.
 
         visitor.visitEnd();
@@ -280,30 +307,13 @@ public void write() {
 
     @Override
     void write(MethodWriter writer, Globals globals) {
-        if (reserved.usesScore()) {
-            // if the _score value is used, we do this once:
-            // final double _score = scorer.score();
-            Variable scorer = mainMethod.getVariable(null, Locals.SCORER);
-            Variable score = mainMethod.getVariable(null, Locals.SCORE);
-
-            writer.visitVarInsn(Opcodes.ALOAD, scorer.getSlot());
-            writer.invokeVirtual(WriterConstants.SCORER_TYPE, WriterConstants.SCORER_SCORE);
-            writer.visitInsn(Opcodes.F2D);
-            writer.visitVarInsn(Opcodes.DSTORE, score.getSlot());
-        }
-
-        if (reserved.usesCtx()) {
-            // if the _ctx value is used, we do this once:
-            // final Map<String,Object> ctx = input.get("ctx");
-
-            Variable input = mainMethod.getVariable(null, Locals.PARAMS);
-            Variable ctx = mainMethod.getVariable(null, Locals.CTX);
-
-            writer.visitVarInsn(Opcodes.ALOAD, input.getSlot());
-            writer.push(Locals.CTX);
-            writer.invokeInterface(MAP_TYPE, MAP_GET);
-            writer.visitVarInsn(Opcodes.ASTORE, ctx.getSlot());
-        }
+        // We wrap the whole method in a few try/catches to handle and/or convert other exceptions to ScriptException
+        Label startTry = new Label();
+        Label endTry = new Label();
+        Label startExplainCatch = new Label();
+        Label startOtherCatch = new Label();
+        Label endCatch = new Label();
+        writer.mark(startTry);
 
         if (reserved.getMaxLoopCounter() > 0) {
             // if there is infinite loop protection, we do this once:
@@ -320,9 +330,52 @@ void write(MethodWriter writer, Globals globals) {
         }
 
         if (!methodEscape) {
-            writer.visitInsn(Opcodes.ACONST_NULL);
+            switch (scriptInterface.getExecuteMethod().getReturnType().getSort()) {
+            case org.objectweb.asm.Type.VOID:    break;
+            case org.objectweb.asm.Type.BOOLEAN: writer.push(false); break;
+            case org.objectweb.asm.Type.BYTE:    writer.push(0); break;
+            case org.objectweb.asm.Type.SHORT:   writer.push(0); break;
+            case org.objectweb.asm.Type.INT:     writer.push(0); break;
+            case org.objectweb.asm.Type.LONG:    writer.push(0L); break;
+            case org.objectweb.asm.Type.FLOAT:   writer.push(0f); break;
+            case org.objectweb.asm.Type.DOUBLE:  writer.push(0d); break;
+            default:                             writer.visitInsn(Opcodes.ACONST_NULL);
+            }
             writer.returnValue();
         }
+
+        writer.mark(endTry);
+        writer.goTo(endCatch);
+        // This looks like:
+        // } catch (PainlessExplainError e) {
+        //   throw this.convertToScriptException(e, e.getHeaders($DEFINITION))
+        // }
+        writer.visitTryCatchBlock(startTry, endTry, startExplainCatch, PAINLESS_EXPLAIN_ERROR_TYPE.getInternalName());
+        writer.mark(startExplainCatch);
+        writer.loadThis();
+        writer.swap();
+        writer.dup();
+        writer.getStatic(CLASS_TYPE, "$DEFINITION", DEFINITION_TYPE);
+        writer.invokeVirtual(PAINLESS_EXPLAIN_ERROR_TYPE, PAINLESS_EXPLAIN_ERROR_GET_HEADERS_METHOD);
+        writer.invokeVirtual(BASE_CLASS_TYPE, CONVERT_TO_SCRIPT_EXCEPTION_METHOD);
+        writer.throwException();
+        // This looks like:
+        // } catch (PainlessError | BootstrapMethodError | OutOfMemoryError | StackOverflowError | Exception e) {
+        //   throw this.convertToScriptException(e, e.getHeaders())
+        // }
+        // We *think* it is ok to catch OutOfMemoryError and StackOverflowError because Painless is stateless
+        writer.visitTryCatchBlock(startTry, endTry, startOtherCatch, PAINLESS_ERROR_TYPE.getInternalName());
+        writer.visitTryCatchBlock(startTry, endTry, startOtherCatch, BOOTSTRAP_METHOD_ERROR_TYPE.getInternalName());
+        writer.visitTryCatchBlock(startTry, endTry, startOtherCatch, OUT_OF_MEMORY_ERROR_TYPE.getInternalName());
+        writer.visitTryCatchBlock(startTry, endTry, startOtherCatch, STACK_OVERFLOW_ERROR_TYPE.getInternalName());
+        writer.visitTryCatchBlock(startTry, endTry, startOtherCatch, EXCEPTION_TYPE.getInternalName());
+        writer.mark(startOtherCatch);
+        writer.loadThis();
+        writer.swap();
+        writer.invokeStatic(COLLECTIONS_TYPE, EMPTY_MAP_METHOD);
+        writer.invokeVirtual(BASE_CLASS_TYPE, CONVERT_TO_SCRIPT_EXCEPTION_METHOD);
+        writer.throwException();
+        writer.mark(endCatch);
     }
 
     public BitSet getStatements() {
@@ -332,4 +385,12 @@ public BitSet getStatements() {
     public byte[] getBytes() {
         return bytes;
     }
+
+    @Override
+    public String toString() {
+        List<Object> subs = new ArrayList<>(functions.size() + statements.size());
+        subs.addAll(functions);
+        subs.addAll(statements);
+        return multilineToString(emptyList(), subs);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachArray.java
index 2841aff46bb39..2083d3ddbe5e5 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachArray.java
@@ -47,7 +47,7 @@ final class SSubEachArray extends AStatement {
     private Variable index = null;
     private Type indexed = null;
 
-    public SSubEachArray(Location location, Variable variable, AExpression expression, SBlock block) {
+    SSubEachArray(Location location, Variable variable, AExpression expression, SBlock block) {
         super(location);
 
         this.variable = Objects.requireNonNull(variable);
@@ -64,9 +64,12 @@ void extractVariables(Set<String> variables) {
     void analyze(Locals locals) {
         // We must store the array and index as variables for securing slots on the stack, and
         // also add the location offset to make the names unique in case of nested for each loops.
-        array = locals.addVariable(location, expression.actual, "#array" + location.getOffset(), true);
-        index = locals.addVariable(location, Definition.INT_TYPE, "#index" + location.getOffset(), true);
-        indexed = Definition.getType(expression.actual.struct, expression.actual.dimensions - 1);
+        array = locals.addVariable(location, expression.actual, "#array" + location.getOffset(),
+                true);
+        index = locals.addVariable(location, Definition.INT_TYPE, "#index" + location.getOffset(),
+                true);
+        indexed = locals.getDefinition().getType(expression.actual.struct,
+                expression.actual.dimensions - 1);
         cast = AnalyzerCaster.getLegalCast(location, indexed, variable.type, true, true);
     }
 
@@ -107,4 +110,9 @@ void write(MethodWriter writer, Globals globals) {
         writer.goTo(begin);
         writer.mark(end);
     }
+
+    @Override
+    public String toString() {
+        return singleLineToString(variable.type.name, variable.name, expression, block);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java
index 57acb07aa6abe..b014e952e32a2 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java
@@ -26,7 +26,6 @@
 import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.Definition.MethodKey;
 import org.elasticsearch.painless.Definition.Sort;
-import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Locals.Variable;
@@ -55,7 +54,7 @@ final class SSubEachIterable extends AStatement {
     private Variable iterator = null;
     private Method method = null;
 
-    public SSubEachIterable(Location location, Variable variable, AExpression expression, SBlock block) {
+    SSubEachIterable(Location location, Variable variable, AExpression expression, SBlock block) {
         super(location);
 
         this.variable = Objects.requireNonNull(variable);
@@ -72,7 +71,8 @@ void extractVariables(Set<String> variables) {
     void analyze(Locals locals) {
         // We must store the iterator as a variable for securing a slot on the stack, and
         // also add the location offset to make the name unique in case of nested for each loops.
-        iterator = locals.addVariable(location, Definition.getType("Iterator"), "#itr" + location.getOffset(), true);
+        iterator = locals.addVariable(location, locals.getDefinition().getType("Iterator"),
+                "#itr" + location.getOffset(), true);
 
         if (expression.actual.sort == Sort.DEF) {
             method = null;
@@ -95,8 +95,8 @@ void write(MethodWriter writer, Globals globals) {
         expression.write(writer, globals);
 
         if (method == null) {
-            Type itr = Definition.getType("Iterator");
-            org.objectweb.asm.Type methodType = org.objectweb.asm.Type.getMethodType(itr.type, Definition.DEF_TYPE.type);
+            org.objectweb.asm.Type methodType = org.objectweb.asm.Type
+                    .getMethodType(Definition.ITERATOR_TYPE.type, Definition.DEF_TYPE.type);
             writer.invokeDefCall("iterator", methodType, DefBootstrap.ITERATOR);
         } else {
             method.write(writer);
@@ -129,4 +129,9 @@ void write(MethodWriter writer, Globals globals) {
         writer.goTo(begin);
         writer.mark(end);
     }
+
+    @Override
+    public String toString() {
+        return singleLineToString(variable.type.name, variable.name, expression, block);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
index b3442b2f2eed2..e87d862b2acc7 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
@@ -64,4 +64,9 @@ void write(MethodWriter writer, Globals globals) {
         expression.write(writer, globals);
         writer.throwException();
     }
+
+    @Override
+    public String toString() {
+        return singleLineToString(expression);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
index 47dc955a914db..d9cd2da0f9272 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
@@ -29,6 +29,8 @@
 import java.util.List;
 import java.util.Set;
 
+import static java.util.Collections.singleton;
+
 /**
  * Represents the try block as part of a try-catch block.
  */
@@ -124,4 +126,9 @@ void write(MethodWriter writer, Globals globals) {
             writer.mark(exception);
         }
     }
+
+    @Override
+    public String toString() {
+        return multilineToString(singleton(block), catches);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
index 514ba801bbf55..7663c853a0c94 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
@@ -134,4 +134,9 @@ void write(MethodWriter writer, Globals globals) {
 
         writer.mark(end);
     }
+
+    @Override
+    public String toString() {
+        return singleLineToString(condition, block);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/package-info.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/package-info.java
new file mode 100644
index 0000000000000..d45d4fb6d49eb
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Implementation of the Painless language.
+ */
+package org.elasticsearch.painless;
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.lang.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.lang.txt
index 13f28d3ebeb61..a1cde1711bc51 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.lang.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.lang.txt
@@ -756,6 +756,8 @@ class String -> java.lang.String extends CharSequence,Comparable,Object {
   boolean contentEquals(CharSequence)
   String copyValueOf(char[])
   String copyValueOf(char[],int,int)
+  String decodeBase64*()
+  String encodeBase64*()
   boolean endsWith(String)
   boolean equalsIgnoreCase(String)
   String format(Locale,String,def[])
@@ -1101,4 +1103,3 @@ class UnsupportedOperationException -> java.lang.UnsupportedOperationException e
   UnsupportedOperationException <init>()
   UnsupportedOperationException <init>(String)
 }
-
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.chrono.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.chrono.txt
index 7374efea04bed..286141e29215b 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.chrono.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.chrono.txt
@@ -82,8 +82,6 @@ class Chronology -> java.time.chrono.Chronology extends Comparable {
   ChronoLocalDate date(Era,int,int,int)
   ChronoLocalDate date(int,int,int)
   ChronoLocalDate dateEpochDay(long)
-  ChronoLocalDate dateNow()
-  ChronoLocalDate dateNow(ZoneId)
   ChronoLocalDate dateYearDay(Era,int,int)
   ChronoLocalDate dateYearDay(int,int)
   boolean equals(Object)
@@ -171,8 +169,6 @@ class HijrahChronology -> java.time.chrono.HijrahChronology extends AbstractChro
   HijrahDate date(int,int,int)
   HijrahDate date(Era,int,int,int)
   HijrahDate dateEpochDay(long)
-  HijrahDate dateNow()
-  HijrahDate dateNow(ZoneId)
   HijrahDate dateYearDay(int,int)
   HijrahDate dateYearDay(Era,int,int)
   HijrahEra eraOf(int)
@@ -185,8 +181,6 @@ class HijrahDate -> java.time.chrono.HijrahDate extends ChronoLocalDate,Temporal
   HijrahEra getEra()
   HijrahDate minus(TemporalAmount)
   HijrahDate minus(long,TemporalUnit)
-  HijrahDate now()
-  HijrahDate now(ZoneId)
   HijrahDate of(int,int,int)
   HijrahDate plus(TemporalAmount)
   HijrahDate plus(long,TemporalUnit)
@@ -201,8 +195,6 @@ class IsoChronology -> java.time.chrono.IsoChronology extends AbstractChronology
   LocalDate date(int,int,int)
   LocalDate date(Era,int,int,int)
   LocalDate dateEpochDay(long)
-  LocalDate dateNow()
-  LocalDate dateNow(ZoneId)
   LocalDate dateYearDay(int,int)
   LocalDate dateYearDay(Era,int,int)
   IsoEra eraOf(int)
@@ -219,8 +211,6 @@ class JapaneseChronology -> java.time.chrono.JapaneseChronology extends Abstract
   JapaneseDate date(int,int,int)
   JapaneseDate date(Era,int,int,int)
   JapaneseDate dateEpochDay(long)
-  JapaneseDate dateNow()
-  JapaneseDate dateNow(ZoneId)
   JapaneseDate dateYearDay(int,int)
   JapaneseDate dateYearDay(Era,int,int)
   JapaneseEra eraOf(int)
@@ -228,8 +218,6 @@ class JapaneseChronology -> java.time.chrono.JapaneseChronology extends Abstract
 }
 
 class JapaneseDate -> java.time.chrono.JapaneseDate extends ChronoLocalDate,Temporal,TemporalAccessor,TemporalAdjuster,Comparable,Object {
-  JapaneseDate now()
-  JapaneseDate now(ZoneId)
   JapaneseDate of(int,int,int)
   JapaneseDate from(TemporalAccessor)
   JapaneseChronology getChronology()
@@ -259,8 +247,6 @@ class MinguoChronology -> java.time.chrono.MinguoChronology extends AbstractChro
   MinguoDate date(int,int,int)
   MinguoDate date(Era,int,int,int)
   MinguoDate dateEpochDay(long)
-  MinguoDate dateNow()
-  MinguoDate dateNow(ZoneId)
   MinguoDate dateYearDay(int,int)
   MinguoDate dateYearDay(Era,int,int)
   MinguoEra eraOf(int)
@@ -268,8 +254,6 @@ class MinguoChronology -> java.time.chrono.MinguoChronology extends AbstractChro
 }
 
 class MinguoDate -> java.time.chrono.MinguoDate extends ChronoLocalDate,Temporal,TemporalAccessor,TemporalAdjuster,Comparable,Object {
-  MinguoDate now()
-  MinguoDate now(ZoneId)
   MinguoDate of(int,int,int)
   MinguoDate from(TemporalAccessor)
   MinguoChronology getChronology()
@@ -288,8 +272,6 @@ class ThaiBuddhistChronology -> java.time.chrono.ThaiBuddhistChronology extends
   ThaiBuddhistDate date(int,int,int)
   ThaiBuddhistDate date(Era,int,int,int)
   ThaiBuddhistDate dateEpochDay(long)
-  ThaiBuddhistDate dateNow()
-  ThaiBuddhistDate dateNow(ZoneId)
   ThaiBuddhistDate dateYearDay(int,int)
   ThaiBuddhistDate dateYearDay(Era,int,int)
   ThaiBuddhistEra eraOf(int)
@@ -297,8 +279,6 @@ class ThaiBuddhistChronology -> java.time.chrono.ThaiBuddhistChronology extends
 }
 
 class ThaiBuddhistDate -> java.time.chrono.ThaiBuddhistDate extends ChronoLocalDate,Temporal,TemporalAccessor,TemporalAdjuster,Comparable,Object {
-  ThaiBuddhistDate now()
-  ThaiBuddhistDate now(ZoneId)
   ThaiBuddhistDate of(int,int,int)
   ThaiBuddhistDate from(TemporalAccessor)
   ThaiBuddhistChronology getChronology()
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.txt
index 4481004fdf6c5..35f19b0abddea 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.txt
@@ -30,13 +30,7 @@ class Clock -> java.time.Clock extends Object {
   Instant instant()
   long millis()
   Clock offset(Clock,Duration)
-  Clock system(ZoneId)
-  Clock systemDefaultZone()
-  Clock systemUTC()
   Clock tick(Clock,Duration)
-  Clock tickMinutes(ZoneId)
-  Clock tickSeconds(ZoneId)
-  Clock withZone(ZoneId)
 }
 
 class Duration -> java.time.Duration extends Comparable,TemporalAmount,Object {
@@ -103,8 +97,6 @@ class Instant -> java.time.Instant extends Comparable,Temporal,TemporalAccessor,
   Instant minusMillis(long)
   Instant minusNanos(long)
   Instant minusSeconds(long)
-  Instant now()
-  Instant now(Clock)
   Instant ofEpochSecond(long)
   Instant ofEpochSecond(long,long)
   Instant ofEpochMilli(long)
@@ -143,8 +135,6 @@ class LocalDate -> java.time.LocalDate extends ChronoLocalDate,Temporal,Temporal
   LocalDate minusMonths(long)
   LocalDate minusWeeks(long)
   LocalDate minusDays(long)
-  LocalDate now()
-  LocalDate now(ZoneId)
   LocalDate of(int,int,int)
   LocalDate ofYearDay(int,int)
   LocalDate ofEpochDay(long)
@@ -191,8 +181,6 @@ class LocalDateTime -> java.time.LocalDateTime extends ChronoLocalDateTime,Tempo
   LocalDateTime minusSeconds(long)
   LocalDateTime minusWeeks(long)
   LocalDateTime minusYears(long)
-  LocalDateTime now()
-  LocalDateTime now(ZoneId)
   LocalDateTime of(LocalDate,LocalTime)
   LocalDateTime of(int,int,int,int,int)
   LocalDateTime of(int,int,int,int,int,int)
@@ -246,8 +234,6 @@ class LocalTime -> java.time.LocalTime extends Temporal,TemporalAccessor,Tempora
   LocalTime minusMinutes(long)
   LocalTime minusNanos(long)
   LocalTime minusSeconds(long)
-  LocalTime now()
-  LocalTime now(ZoneId)
   LocalTime of(int,int)
   LocalTime of(int,int,int)
   LocalTime of(int,int,int,int)
@@ -283,8 +269,6 @@ class MonthDay -> java.time.MonthDay extends TemporalAccessor,TemporalAdjuster,C
   boolean isAfter(MonthDay)
   boolean isBefore(MonthDay)
   boolean isValidYear(int)
-  MonthDay now()
-  MonthDay now(ZoneId)
   MonthDay of(int,int)
   MonthDay parse(CharSequence)   
   MonthDay parse(CharSequence,DateTimeFormatter)
@@ -325,8 +309,6 @@ class OffsetDateTime -> java.time.OffsetDateTime extends Temporal,TemporalAccess
   OffsetDateTime minusMinutes(long)
   OffsetDateTime minusSeconds(long)
   OffsetDateTime minusNanos(long)
-  OffsetDateTime now()
-  OffsetDateTime now(ZoneId)
   OffsetDateTime of(LocalDate,LocalTime,ZoneOffset)
   OffsetDateTime of(LocalDateTime,ZoneOffset)
   OffsetDateTime of(int,int,int,int,int,int,int,ZoneOffset)
@@ -380,8 +362,6 @@ class OffsetTime -> java.time.OffsetTime extends Temporal,TemporalAccessor,Tempo
   boolean isAfter(OffsetTime)
   boolean isBefore(OffsetTime)
   boolean isEqual(OffsetTime)
-  OffsetTime now()
-  OffsetTime now(ZoneId)
   OffsetTime of(LocalTime,ZoneOffset)
   OffsetTime of(int,int,int,int,ZoneOffset)
   OffsetTime ofInstant(Instant,ZoneId)
@@ -460,8 +440,6 @@ class Year -> java.time.Year extends Temporal,TemporalAccessor,TemporalAdjuster,
   Year minus(TemporalAmount)
   Year minus(long,TemporalUnit)
   Year minusYears(long)
-  Year now()
-  Year now(ZoneId)
   Year of(int)
   Year parse(CharSequence)
   Year parse(CharSequence,DateTimeFormatter)
@@ -491,8 +469,6 @@ class YearMonth -> java.time.YearMonth extends Temporal,TemporalAccessor,Tempora
   YearMonth minus(long,TemporalUnit)
   YearMonth minusYears(long)
   YearMonth minusMonths(long)
-  YearMonth now()
-  YearMonth now(ZoneId)
   YearMonth of(int,int)
   YearMonth parse(CharSequence)
   YearMonth parse(CharSequence,DateTimeFormatter)
@@ -530,8 +506,6 @@ class ZonedDateTime -> java.time.ZonedDateTime extends ChronoZonedDateTime,Tempo
   ZonedDateTime minusMinutes(long)
   ZonedDateTime minusSeconds(long)
   ZonedDateTime minusNanos(long)
-  ZonedDateTime now()
-  ZonedDateTime now(ZoneId)
   ZonedDateTime of(LocalDate,LocalTime,ZoneId)
   ZonedDateTime of(LocalDateTime,ZoneId)
   ZonedDateTime of(int,int,int,int,int,int,int,ZoneId)
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt
index 3757e4fb76c6d..5ebda73e3132d 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt
@@ -57,6 +57,13 @@ class def -> java.lang.Object {
   String toString()
 }
 
+
+#### Painless debugging API
+
+class Debug -> org.elasticsearch.painless.api.Debug extends Object {
+  void explain(Object)
+}
+
 #### ES Scripting API
 
 class org.elasticsearch.common.geo.GeoPoint -> org.elasticsearch.common.geo.GeoPoint extends Object {
@@ -75,6 +82,7 @@ class org.elasticsearch.index.fielddata.ScriptDocValues.Longs -> org.elasticsear
   long getValue()
   List getValues()
   org.joda.time.ReadableDateTime getDate()
+  List getDates()
 }
 
 class org.elasticsearch.index.fielddata.ScriptDocValues.Doubles -> org.elasticsearch.index.fielddata.ScriptDocValues$Doubles extends List,Collection,Iterable,Object {
@@ -101,6 +109,32 @@ class org.elasticsearch.index.fielddata.ScriptDocValues.GeoPoints -> org.elastic
   double geohashDistanceWithDefault(String,double)
 }
 
+class org.elasticsearch.index.fielddata.ScriptDocValues.Booleans -> org.elasticsearch.index.fielddata.ScriptDocValues$Booleans extends List,Collection,Iterable,Object {
+  Boolean get(int)
+  boolean getValue()
+  List getValues()
+}
+
+class org.elasticsearch.index.fielddata.ScriptDocValues.BytesRefs -> org.elasticsearch.index.fielddata.ScriptDocValues$BytesRefs extends List,Collection,Iterable,Object {
+  BytesRef get(int)
+  BytesRef getValue()
+  List getValues()
+}
+
+class BytesRef -> org.apache.lucene.util.BytesRef extends Object {
+  byte[] bytes
+  int offset
+  int length
+  boolean bytesEquals(BytesRef)
+  String utf8ToString()
+}
+
+class org.elasticsearch.index.mapper.IpFieldMapper.IpFieldType.IpScriptDocValues -> org.elasticsearch.index.mapper.IpFieldMapper$IpFieldType$IpScriptDocValues extends List,Collection,Iterable,Object {
+  String get(int)
+  String getValue()
+  List getValues()
+}
+
 # for testing.
 # currently FeatureTest exposes overloaded constructor, field load store, and overloaded static methods
 class org.elasticsearch.painless.FeatureTest -> org.elasticsearch.painless.FeatureTest extends Object {
@@ -113,4 +147,11 @@ class org.elasticsearch.painless.FeatureTest -> org.elasticsearch.painless.Featu
   boolean overloadedStatic()
   boolean overloadedStatic(boolean)
   Object twoFunctionsOfX(Function,Function)
+  void listInput(List)
+}
+
+class org.elasticsearch.search.lookup.FieldLookup -> org.elasticsearch.search.lookup.FieldLookup extends Object {
+  def getValue()
+  List getValues()
+  boolean isEmpty()
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AnalyzerCasterTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AnalyzerCasterTests.java
new file mode 100644
index 0000000000000..e8d2f1332e094
--- /dev/null
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AnalyzerCasterTests.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+import org.elasticsearch.painless.Definition.Cast;
+import org.elasticsearch.painless.Definition.Type;
+import org.elasticsearch.test.ESTestCase;
+
+import static org.elasticsearch.painless.Definition.BYTE_TYPE;
+import static org.elasticsearch.painless.Definition.DOUBLE_TYPE;
+import static org.elasticsearch.painless.Definition.FLOAT_TYPE;
+import static org.elasticsearch.painless.Definition.INT_TYPE;
+import static org.elasticsearch.painless.Definition.LONG_TYPE;
+import static org.elasticsearch.painless.Definition.SHORT_TYPE;
+
+public class AnalyzerCasterTests extends ESTestCase {
+
+    private static void assertCast(Type actual, Type expected, boolean mustBeExplicit) {
+        Location location = new Location("dummy", 0);
+
+        if (actual.equals(expected)) {
+            assertFalse(mustBeExplicit);
+            assertNull(AnalyzerCaster.getLegalCast(location, actual, expected, false, false));
+            assertNull(AnalyzerCaster.getLegalCast(location, actual, expected, true, false));
+            return;
+        }
+
+        Cast cast = AnalyzerCaster.getLegalCast(location, actual, expected, true, false);
+        assertEquals(actual, cast.from);
+        assertEquals(expected, cast.to);
+
+        if (mustBeExplicit) {
+            ClassCastException error = expectThrows(ClassCastException.class,
+                    () -> AnalyzerCaster.getLegalCast(location, actual, expected, false, false));
+            assertTrue(error.getMessage().startsWith("Cannot cast"));
+        } else {
+            cast = AnalyzerCaster.getLegalCast(location, actual, expected, false, false);
+            assertEquals(actual, cast.from);
+            assertEquals(expected, cast.to);
+        }
+    }
+
+    public void testNumericCasts() {
+        assertCast(BYTE_TYPE, BYTE_TYPE, false);
+        assertCast(BYTE_TYPE, SHORT_TYPE, false);
+        assertCast(BYTE_TYPE, INT_TYPE, false);
+        assertCast(BYTE_TYPE, LONG_TYPE, false);
+        assertCast(BYTE_TYPE, FLOAT_TYPE, false);
+        assertCast(BYTE_TYPE, DOUBLE_TYPE, false);
+
+        assertCast(SHORT_TYPE, BYTE_TYPE, true);
+        assertCast(SHORT_TYPE, SHORT_TYPE, false);
+        assertCast(SHORT_TYPE, INT_TYPE, false);
+        assertCast(SHORT_TYPE, LONG_TYPE, false);
+        assertCast(SHORT_TYPE, FLOAT_TYPE, false);
+        assertCast(SHORT_TYPE, DOUBLE_TYPE, false);
+
+        assertCast(INT_TYPE, BYTE_TYPE, true);
+        assertCast(INT_TYPE, SHORT_TYPE, true);
+        assertCast(INT_TYPE, INT_TYPE, false);
+        assertCast(INT_TYPE, LONG_TYPE, false);
+        assertCast(INT_TYPE, FLOAT_TYPE, false);
+        assertCast(INT_TYPE, DOUBLE_TYPE, false);
+
+        assertCast(LONG_TYPE, BYTE_TYPE, true);
+        assertCast(LONG_TYPE, SHORT_TYPE, true);
+        assertCast(LONG_TYPE, INT_TYPE, true);
+        assertCast(LONG_TYPE, LONG_TYPE, false);
+        assertCast(LONG_TYPE, FLOAT_TYPE, false);
+        assertCast(LONG_TYPE, DOUBLE_TYPE, false);
+
+        assertCast(FLOAT_TYPE, BYTE_TYPE, true);
+        assertCast(FLOAT_TYPE, SHORT_TYPE, true);
+        assertCast(FLOAT_TYPE, INT_TYPE, true);
+        assertCast(FLOAT_TYPE, LONG_TYPE, true);
+        assertCast(FLOAT_TYPE, FLOAT_TYPE, false);
+        assertCast(FLOAT_TYPE, DOUBLE_TYPE, false);
+
+        assertCast(DOUBLE_TYPE, BYTE_TYPE, true);
+        assertCast(DOUBLE_TYPE, SHORT_TYPE, true);
+        assertCast(DOUBLE_TYPE, INT_TYPE, true);
+        assertCast(DOUBLE_TYPE, LONG_TYPE, true);
+        assertCast(DOUBLE_TYPE, FLOAT_TYPE, true);
+        assertCast(DOUBLE_TYPE, DOUBLE_TYPE, false);
+    }
+
+}
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayLikeObjectTestCase.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayLikeObjectTestCase.java
new file mode 100644
index 0000000000000..5fc41c8c63038
--- /dev/null
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayLikeObjectTestCase.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+import org.elasticsearch.common.Nullable;
+import org.hamcrest.Matcher;
+
+import static java.util.Collections.singletonMap;
+
+/**
+ * Superclass for testing array-like objects (arrays and lists).
+ */
+public abstract class ArrayLikeObjectTestCase extends ScriptTestCase {
+    /**
+     * Build the string for declaring the variable holding the array-like-object to test. So {@code int[]} for arrays and {@code List} for
+     * lists.
+     */
+    protected abstract String declType(String valueType);
+    /**
+     * Build the string for calling the constructor for the array-like-object to test. So {@code new int[5]} for arrays and
+     * {@code [0, 0, 0, 0, 0]} or {@code [null, null, null, null, null]} for lists.
+     */
+    protected abstract String valueCtorCall(String valueType, int size);
+    /**
+     * Matcher for the message of the out of bounds exceptions thrown for too negative or too positive offsets.
+     */
+    protected abstract Matcher<String> outOfBoundsExceptionMessageMatcher(int index, int size);
+
+    private void arrayLoadStoreTestCase(boolean declareAsDef, String valueType, Object val, @Nullable Number valPlusOne) {
+        String declType = declareAsDef ? "def" : declType(valueType);
+        String valueCtorCall = valueCtorCall(valueType, 5);
+        String decl = declType + " x = " + valueCtorCall;
+        assertEquals(5, exec(decl + "; return x.length", true));
+        assertEquals(val, exec(decl + "; x[ 0] = params.val; return x[ 0];", singletonMap("val", val), true));
+        assertEquals(val, exec(decl + "; x[ 0] = params.val; return x[-5];", singletonMap("val", val), true));
+        assertEquals(val, exec(decl + "; x[-5] = params.val; return x[-5];", singletonMap("val", val), true));
+
+        expectOutOfBounds( 6, decl + "; return x[ 6]", val);
+        expectOutOfBounds(-1, decl + "; return x[-6]", val);
+        expectOutOfBounds( 6, decl + "; x[ 6] = params.val; return 0", val);
+        expectOutOfBounds(-1, decl + "; x[-6] = params.val; return 0", val);
+
+        if (valPlusOne != null) {
+            assertEquals(val,        exec(decl + "; x[0] = params.val; x[ 0] = x[ 0]++; return x[0];", singletonMap("val", val), true));
+            assertEquals(val,        exec(decl + "; x[0] = params.val; x[ 0] = x[-5]++; return x[0];", singletonMap("val", val), true));
+            assertEquals(valPlusOne, exec(decl + "; x[0] = params.val; x[ 0] = ++x[ 0]; return x[0];", singletonMap("val", val), true));
+            assertEquals(valPlusOne, exec(decl + "; x[0] = params.val; x[ 0] = ++x[-5]; return x[0];", singletonMap("val", val), true));
+            assertEquals(valPlusOne, exec(decl + "; x[0] = params.val; x[ 0]++        ; return x[0];", singletonMap("val", val), true));
+            assertEquals(valPlusOne, exec(decl + "; x[0] = params.val; x[-5]++        ; return x[0];", singletonMap("val", val), true));
+            assertEquals(valPlusOne, exec(decl + "; x[0] = params.val; x[ 0] += 1     ; return x[0];", singletonMap("val", val), true));
+            assertEquals(valPlusOne, exec(decl + "; x[0] = params.val; x[-5] += 1     ; return x[0];", singletonMap("val", val), true));
+
+            expectOutOfBounds( 6, decl + "; return x[ 6]++", val);
+            expectOutOfBounds(-1, decl + "; return x[-6]++", val);
+            expectOutOfBounds( 6, decl + "; return ++x[ 6]", val);
+            expectOutOfBounds(-1, decl + "; return ++x[-6]", val);
+            expectOutOfBounds( 6, decl + "; x[ 6] += 1; return 0", val);
+            expectOutOfBounds(-1, decl + "; x[-6] += 1; return 0", val);
+        }
+    }
+
+    private void expectOutOfBounds(int index, String script, Object val) {
+        IndexOutOfBoundsException e = expectScriptThrows(IndexOutOfBoundsException.class, () ->
+            exec(script, singletonMap("val", val), true));
+        try {
+            assertThat(e.getMessage(), outOfBoundsExceptionMessageMatcher(index, 5));
+        } catch (AssertionError ae) {
+            ae.addSuppressed(e);   // Mark the exception we are testing as suppressed so we get its stack trace. If it has one :(
+            throw ae;
+        }
+    }
+
+    public void testInts() {         arrayLoadStoreTestCase(false, "int",    5,         6); }
+    public void testIntsInDef() {    arrayLoadStoreTestCase(true,  "int",    5,         6); }
+    public void testLongs() {        arrayLoadStoreTestCase(false, "long",   5L,        6L); }
+    public void testLongsInDef() {   arrayLoadStoreTestCase(true,  "long",   5L,        6L); }
+    public void testShorts() {       arrayLoadStoreTestCase(false, "short",  (short) 5, (short) 6); }
+    public void testShortsInDef() {  arrayLoadStoreTestCase(true,  "short",  (short) 5, (short) 6); }
+    public void testBytes() {        arrayLoadStoreTestCase(false, "byte",   (byte) 5,  (byte) 6); }
+    public void testBytesInDef() {   arrayLoadStoreTestCase(true,  "byte",   (byte) 5,  (byte) 6); }
+    public void testFloats() {       arrayLoadStoreTestCase(false, "float",  5.0f,      6.0f); }
+    public void testFloatsInDef() {  arrayLoadStoreTestCase(true,  "float",  5.0f,      6.0f); }
+    public void testDoubles() {      arrayLoadStoreTestCase(false, "double", 5.0d,      6.0d); }
+    public void testDoublesInDef() { arrayLoadStoreTestCase(true,  "double", 5.0d,      6.0d); }
+    public void testStrings() {      arrayLoadStoreTestCase(false, "String", "cat",     null); }
+    public void testStringsInDef() { arrayLoadStoreTestCase(true,  "String", "cat",     null); }
+    public void testDef() {          arrayLoadStoreTestCase(true,  "def",    5,         null); }
+}
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java
index acacc613ab353..5c1141a49e557 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java
@@ -19,13 +19,33 @@
 
 package org.elasticsearch.painless;
 
+import org.apache.lucene.util.Constants;
+import org.hamcrest.Matcher;
+
 import java.lang.invoke.MethodHandle;
 import java.lang.invoke.MethodType;
 
-/** Tests for or operator across all types */
-public class ArrayTests extends ScriptTestCase {
+import static org.hamcrest.Matchers.equalTo;
+
+/** Tests for working with arrays. */
+public class ArrayTests extends ArrayLikeObjectTestCase {
+    @Override
+    protected String declType(String valueType) {
+        return valueType + "[]";
+    }
+
+    @Override
+    protected String valueCtorCall(String valueType, int size) {
+        return "new " + valueType + "[" + size + "]";
+    }
+
+    @Override
+    protected Matcher<String> outOfBoundsExceptionMessageMatcher(int index, int size) {
+        return equalTo(Integer.toString(index));
+    }
 
     public void testArrayLengthHelper() throws Throwable {
+        assertEquals(Constants.JRE_IS_MINIMUM_JAVA9, Def.JAVA9_ARRAY_LENGTH_MH_FACTORY != null);
         assertArrayLength(2, new int[2]);
         assertArrayLength(3, new long[3]);
         assertArrayLength(4, new byte[4]);
@@ -45,29 +65,6 @@ private void assertArrayLength(int length, Object array) throws Throwable {
                 .invokeExact(array));
     }
 
-    public void testArrayLoadStoreInt() {
-        assertEquals(5, exec("def x = new int[5]; return x.length"));
-        assertEquals(5, exec("def x = new int[4]; x[0] = 5; return x[0];"));
-    }
-
-    public void testArrayLoadStoreString() {
-        assertEquals(5, exec("def x = new String[5]; return x.length"));
-        assertEquals("foobar", exec("def x = new String[4]; x[0] = 'foobar'; return x[0];"));
-    }
-
-    public void testArrayLoadStoreDef() {
-        assertEquals(5, exec("def x = new def[5]; return x.length"));
-        assertEquals(5, exec("def x = new def[4]; x[0] = 5; return x[0];"));
-    }
-
-    public void testArrayCompoundInt() {
-        assertEquals(6, exec("int[] x = new int[5]; x[0] = 5; x[0]++; return x[0];"));
-    }
-
-    public void testArrayCompoundDef() {
-        assertEquals(6, exec("def x = new int[5]; x[0] = 5; x[0]++; return x[0];"));
-    }
-
     public void testJacksCrazyExpression1() {
         assertEquals(1, exec("int x; def[] y = new def[1]; x = y[0] = 1; return x;"));
     }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AugmentationTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AugmentationTests.java
index 241de3c5db82e..acf698e2fc7be 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AugmentationTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AugmentationTests.java
@@ -25,22 +25,22 @@
 import java.util.Map;
 
 public class AugmentationTests extends ScriptTestCase {
-    
+
     public void testStatic() {
         assertEquals(1, exec("ArrayList l = new ArrayList(); l.add(1); return l.getLength();"));
         assertEquals(1, exec("ArrayList l = new ArrayList(); l.add(1); return l.length;"));
     }
-    
+
     public void testSubclass() {
         assertEquals(1, exec("List l = new ArrayList(); l.add(1); return l.getLength();"));
         assertEquals(1, exec("List l = new ArrayList(); l.add(1); return l.length;"));
     }
-    
+
     public void testDef() {
         assertEquals(1, exec("def l = new ArrayList(); l.add(1); return l.getLength();"));
         assertEquals(1, exec("def l = new ArrayList(); l.add(1); return l.length;"));
     }
-    
+
     public void testCapturingReference() {
         assertEquals(1, exec("int foo(Supplier t) { return t.get() }" +
                              "ArrayList l = new ArrayList(); l.add(1);" +
@@ -52,140 +52,140 @@ public void testCapturingReference() {
                              "def l = new ArrayList(); l.add(1);" +
                              "return foo(l::getLength);"));
     }
-    
+
     public void testIterable_Any() {
-        assertEquals(true, 
+        assertEquals(true,
                 exec("List l = new ArrayList(); l.add(1); l.any(x -> x == 1)"));
     }
-    
+
     public void testIterable_AsCollection() {
-        assertEquals(true, 
+        assertEquals(true,
                 exec("List l = new ArrayList(); return l.asCollection() === l"));
     }
-    
+
     public void testIterable_AsList() {
-        assertEquals(true, 
+        assertEquals(true,
                 exec("List l = new ArrayList(); return l.asList() === l"));
-        assertEquals(5, 
+        assertEquals(5,
                 exec("Set l = new HashSet(); l.add(5); return l.asList()[0]"));
     }
-    
+
     public void testIterable_Each() {
-        assertEquals(1, 
+        assertEquals(1,
                 exec("List l = new ArrayList(); l.add(1); List l2 = new ArrayList(); l.each(l2::add); return l2.size()"));
     }
-    
+
     public void testIterable_EachWithIndex() {
-        assertEquals(0, 
+        assertEquals(0,
                 exec("List l = new ArrayList(); l.add(2); Map m = new HashMap(); l.eachWithIndex(m::put); return m.get(2)"));
     }
-    
+
     public void testIterable_Every() {
         assertEquals(false, exec("List l = new ArrayList(); l.add(1); l.add(2); l.every(x -> x == 1)"));
     }
-    
+
     public void testIterable_FindResults() {
-        assertEquals(1, 
+        assertEquals(1,
                 exec("List l = new ArrayList(); l.add(1); l.add(2); l.findResults(x -> x == 1 ? x : null).size()"));
     }
-    
+
     public void testIterable_GroupBy() {
-        assertEquals(2, 
+        assertEquals(2,
                 exec("List l = new ArrayList(); l.add(1); l.add(-1); l.groupBy(x -> x < 0 ? 'negative' : 'positive').size()"));
     }
-    
+
     public void testIterable_Join() {
-        assertEquals("test,ing", 
+        assertEquals("test,ing",
                 exec("List l = new ArrayList(); l.add('test'); l.add('ing'); l.join(',')"));
     }
-    
+
     public void testIterable_Sum() {
         assertEquals(3.0D, exec("def l = [1,2]; return l.sum()"));
-        assertEquals(5.0D, 
+        assertEquals(5.0D,
                 exec("List l = new ArrayList(); l.add(1); l.add(2); l.sum(x -> x + 1)"));
     }
-    
+
     public void testCollection_Collect() {
-        assertEquals(Arrays.asList(2, 3), 
+        assertEquals(Arrays.asList(2, 3),
                 exec("List l = new ArrayList(); l.add(1); l.add(2); l.collect(x -> x + 1)"));
-        assertEquals(asSet(2, 3), 
+        assertEquals(asSet(2, 3),
                 exec("List l = new ArrayList(); l.add(1); l.add(2); l.collect(new HashSet(), x -> x + 1)"));
     }
-    
+
     public void testCollection_Find() {
-        assertEquals(2, 
+        assertEquals(2,
                 exec("List l = new ArrayList(); l.add(1); l.add(2); return l.find(x -> x == 2)"));
     }
-    
+
     public void testCollection_FindAll() {
-        assertEquals(Arrays.asList(2), 
+        assertEquals(Arrays.asList(2),
                 exec("List l = new ArrayList(); l.add(1); l.add(2); return l.findAll(x -> x == 2)"));
     }
-    
+
     public void testCollection_FindResult() {
-        assertEquals("found", 
+        assertEquals("found",
                 exec("List l = new ArrayList(); l.add(1); l.add(2); return l.findResult(x -> x > 1 ? 'found' : null)"));
-        assertEquals("notfound", 
+        assertEquals("notfound",
                 exec("List l = new ArrayList(); l.add(1); l.add(2); return l.findResult('notfound', x -> x > 10 ? 'found' : null)"));
     }
-    
+
     public void testCollection_Split() {
-        assertEquals(Arrays.asList(Arrays.asList(2), Arrays.asList(1)), 
+        assertEquals(Arrays.asList(Arrays.asList(2), Arrays.asList(1)),
                 exec("List l = new ArrayList(); l.add(1); l.add(2); return l.split(x -> x == 2)"));
     }
-    
+
     public void testMap_Collect() {
-        assertEquals(Arrays.asList("one1", "two2"), 
+        assertEquals(Arrays.asList("one1", "two2"),
                 exec("Map m = new TreeMap(); m.one = 1; m.two = 2; m.collect((key,value) -> key + value)"));
-        assertEquals(asSet("one1", "two2"), 
+        assertEquals(asSet("one1", "two2"),
                 exec("Map m = new TreeMap(); m.one = 1; m.two = 2; m.collect(new HashSet(), (key,value) -> key + value)"));
     }
-    
+
     public void testMap_Count() {
-        assertEquals(1, 
+        assertEquals(1,
                 exec("Map m = new TreeMap(); m.one = 1; m.two = 2; m.count((key,value) -> value == 2)"));
     }
-    
+
     public void testMap_Each() {
-        assertEquals(2, 
+        assertEquals(2,
                 exec("Map m = new TreeMap(); m.one = 1; m.two = 2; Map m2 = new TreeMap(); m.each(m2::put); return m2.size()"));
     }
-    
+
     public void testMap_Every() {
-        assertEquals(false, 
+        assertEquals(false,
                 exec("Map m = new TreeMap(); m.one = 1; m.two = 2; m.every((key,value) -> value == 2)"));
     }
-    
+
     public void testMap_Find() {
-        assertEquals("two", 
+        assertEquals("two",
                 exec("Map m = new TreeMap(); m.one = 1; m.two = 2; return m.find((key,value) -> value == 2).key"));
     }
-    
+
     public void testMap_FindAll() {
-        assertEquals(Collections.singletonMap("two", 2), 
+        assertEquals(Collections.singletonMap("two", 2),
                 exec("Map m = new TreeMap(); m.one = 1; m.two = 2; return m.findAll((key,value) -> value == 2)"));
     }
-    
+
     public void testMap_FindResult() {
-        assertEquals("found", 
+        assertEquals("found",
                 exec("Map m = new TreeMap(); m.one = 1; m.two = 2; return m.findResult((key,value) -> value == 2 ? 'found' : null)"));
-        assertEquals("notfound", 
-                exec("Map m = new TreeMap(); m.one = 1; m.two = 2; " + 
+        assertEquals("notfound",
+                exec("Map m = new TreeMap(); m.one = 1; m.two = 2; " +
                      "return m.findResult('notfound', (key,value) -> value == 10 ? 'found' : null)"));
     }
-    
+
     public void testMap_FindResults() {
         assertEquals(Arrays.asList("negative", "positive"),
-                exec("Map m = new TreeMap(); m.a = -1; m.b = 1; " + 
+                exec("Map m = new TreeMap(); m.a = -1; m.b = 1; " +
                      "return m.findResults((key,value) -> value < 0 ? 'negative' : 'positive')"));
     }
-    
+
     public void testMap_GroupBy() {
         Map<String,Map<String,Integer>> expected = new HashMap<>();
         expected.put("negative", Collections.singletonMap("a", -1));
         expected.put("positive", Collections.singletonMap("b", 1));
         assertEquals(expected,
-                exec("Map m = new TreeMap(); m.a = -1; m.b = 1; " + 
+                exec("Map m = new TreeMap(); m.a = -1; m.b = 1; " +
                      "return m.groupBy((key,value) -> value < 0 ? 'negative' : 'positive')"));
     }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicExpressionTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicExpressionTests.java
index cbfdd31b143f7..97e1f01fdfc94 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicExpressionTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicExpressionTests.java
@@ -2,6 +2,8 @@
 
 import java.util.Collections;
 
+import static java.util.Collections.singletonMap;
+
 /*
  * Licensed to Elasticsearch under one or more contributor
  * license agreements. See the NOTICE file distributed with
@@ -25,16 +27,25 @@ public class BasicExpressionTests extends ScriptTestCase {
 
     /** simple tests returning a constant value */
     public void testReturnConstant() {
-        assertEquals(5, exec("return 5;"));
-        assertEquals(7L, exec("return 7L;"));
-        assertEquals(7.0, exec("return 7.0;"));
-        assertEquals(32.0F, exec("return 32.0F;"));
-        assertEquals((byte)255, exec("return (byte)255;"));
-        assertEquals((short)5, exec("return (short)5;"));
-        assertEquals("string", exec("return \"string\";"));
-        assertEquals(true, exec("return true;"));
-        assertEquals(false, exec("return false;"));
-        assertNull(exec("return null;"));
+        assertEquals(5, exec("return 5"));
+        assertEquals(6L, exec("return 6l"));
+        assertEquals(7L, exec("return 7L"));
+        assertEquals(7.0d, exec("return 7.0"));
+        assertEquals(18.0d, exec("return 18d"));
+        assertEquals(19.0d, exec("return 19.0d"));
+        assertEquals(20.0d, exec("return 20D"));
+        assertEquals(21.0d, exec("return 21.0D"));
+        assertEquals(32.0F, exec("return 32.0f"));
+        assertEquals(33.0F, exec("return 33f"));
+        assertEquals(34.0F, exec("return 34.0F"));
+        assertEquals(35.0F, exec("return 35F"));
+        assertEquals((byte)255, exec("return (byte)255"));
+        assertEquals((short)5, exec("return (short)5"));
+        assertEquals("string", exec("return \"string\""));
+        assertEquals("string", exec("return 'string'"));
+        assertEquals(true, exec("return true"));
+        assertEquals(false, exec("return false"));
+        assertNull(exec("return null"));
     }
 
     public void testReturnConstantChar() {
@@ -45,6 +56,37 @@ public void testConstantCharTruncation() {
         assertEquals('蚠', exec("return (char)100000;"));
     }
 
+    public void testStringEscapes() {
+        // The readability of this test suffers from having to escape `\` and `"` in java strings. Please be careful. Sorry!
+        // `\\` is a `\`
+        assertEquals("\\string", exec("\"\\\\string\""));
+        assertEquals("\\string", exec("'\\\\string'"));
+        // `\"` is a `"` if surrounded by `"`s
+        assertEquals("\"string", exec("\"\\\"string\""));
+        Exception e = expectScriptThrows(IllegalArgumentException.class, () -> exec("'\\\"string'", false));
+        assertEquals("unexpected character ['\\\"]. The only valid escape sequences in strings starting with ['] are [\\\\] and [\\'].",
+                e.getMessage());
+        // `\'` is a `'` if surrounded by `'`s
+        e = expectScriptThrows(IllegalArgumentException.class, () -> exec("\"\\'string\"", false));
+        assertEquals("unexpected character [\"\\']. The only valid escape sequences in strings starting with [\"] are [\\\\] and [\\\"].",
+                e.getMessage());
+        assertEquals("'string", exec("'\\'string'"));
+        // We don't break native escapes like new line
+        assertEquals("\nstring", exec("\"\nstring\""));
+        assertEquals("\nstring", exec("'\nstring'"));
+
+        // And we're ok with strings with multiple escape sequences
+        assertEquals("\\str\"in\\g", exec("\"\\\\str\\\"in\\\\g\""));
+        assertEquals("st\\r'i\\ng", exec("'st\\\\r\\'i\\\\ng'"));
+    }
+
+    public void testStringTermination() {
+        // `'` inside of a string delimited with `"` should be ok
+        assertEquals("test'", exec("\"test'\""));
+        // `"` inside of a string delimited with `'` should be ok
+        assertEquals("test\"", exec("'test\"'"));
+    }
+
     /** declaring variables for primitive types */
     public void testDeclareVariable() {
         assertEquals(5, exec("int i = 5; return i;"));
@@ -135,4 +177,95 @@ public void testPrecedence() {
         assertEquals(2, exec("int x = 5; return (x+x)/x;"));
         assertEquals(true, exec("boolean t = true, f = false; return t && (f || t);"));
     }
+
+    public void testNullSafeDeref() {
+        // Objects in general
+        //   Call
+        assertNull(         exec("String a = null;  return a?.toString()"));
+        assertEquals("foo", exec("String a = 'foo'; return a?.toString()"));
+        assertNull(         exec("def    a = null;  return a?.toString()"));
+        assertEquals("foo", exec("def    a = 'foo'; return a?.toString()"));
+        //   Call with primitive result
+        assertMustBeNullable(    "String a = null;  return a?.length()");
+        assertMustBeNullable(    "String a = 'foo'; return a?.length()");
+        assertNull(         exec("def    a = null;  return a?.length()"));
+        assertEquals(3,     exec("def    a = 'foo'; return a?.length()"));
+        //   Read shortcut
+        assertMustBeNullable(    "org.elasticsearch.painless.FeatureTest a = null; return a?.x");
+        assertMustBeNullable(
+                "org.elasticsearch.painless.FeatureTest a = new org.elasticsearch.painless.FeatureTest(); return a?.x");
+        assertNull(         exec("def    a = null;  return a?.x"));
+        assertEquals(0,     exec("def    a = new org.elasticsearch.painless.FeatureTest(); return a?.x"));
+
+        // Maps
+        //   Call
+        assertNull(        exec("Map a = null;        return a?.toString()"));
+        assertEquals("{}", exec("Map a = [:];         return a?.toString()"));
+        assertNull(        exec("def a = null;        return a?.toString()"));
+        assertEquals("{}", exec("def a = [:];         return a?.toString()"));
+        //   Call with primitive result
+        assertMustBeNullable(   "Map a = [:];  return a?.size()");
+        assertMustBeNullable(   "Map a = null; return a?.size()");
+        assertNull(        exec("def a = null;        return a?.size()"));
+        assertEquals(0,    exec("def a = [:];         return a?.size()"));
+        //   Read shortcut
+        assertNull(        exec("Map a = null;        return a?.other"));       // Read shortcut
+        assertEquals(1,    exec("Map a = ['other':1]; return a?.other"));       // Read shortcut
+        assertNull(        exec("def a = null;        return a?.other"));       // Read shortcut
+        assertEquals(1,    exec("def a = ['other':1]; return a?.other"));       // Read shortcut
+
+        // Array
+        // Since you can't invoke methods on arrays we skip the toString and hashCode tests
+        assertMustBeNullable("int[] a = null;             return a?.length");
+        assertMustBeNullable("int[] a = new int[] {2, 3}; return a?.length");
+        assertNull(     exec("def a = null;               return a?.length"));
+        assertEquals(2, exec("def a = new int[] {2, 3};   return a?.length"));
+
+        // Results from maps (should just work but let's test anyway)
+        FeatureTest t = new FeatureTest();
+        assertNull(     exec("Map a = ['thing': params.t]; return a.other?.getX()", singletonMap("t", t), true));
+        assertNull(     exec("Map a = ['thing': params.t]; return a.other?.x",      singletonMap("t", t), true));
+        assertNull(     exec("def a = ['thing': params.t]; return a.other?.getX()", singletonMap("t", t), true));
+        assertNull(     exec("def a = ['thing': params.t]; return a.other?.x",      singletonMap("t", t), true));
+        assertEquals(0, exec("Map a = ['other': params.t]; return a.other?.getX()", singletonMap("t", t), true));
+        assertEquals(0, exec("Map a = ['other': params.t]; return a.other?.x",      singletonMap("t", t), true));
+        assertEquals(0, exec("def a = ['other': params.t]; return a.other?.getX()", singletonMap("t", t), true));
+        assertEquals(0, exec("def a = ['other': params.t]; return a.other?.x",      singletonMap("t", t), true));
+
+        // Chains
+        assertNull(     exec("Map a = ['thing': ['cat': params.t]]; return a.other?.cat?.getX()", singletonMap("t", t), true));
+        assertNull(     exec("Map a = ['thing': ['cat': params.t]]; return a.other?.cat?.x",      singletonMap("t", t), true));
+        assertNull(     exec("def a = ['thing': ['cat': params.t]]; return a.other?.cat?.getX()", singletonMap("t", t), true));
+        assertNull(     exec("def a = ['thing': ['cat': params.t]]; return a.other?.cat?.x",      singletonMap("t", t), true));
+        assertEquals(0, exec("Map a = ['other': ['cat': params.t]]; return a.other?.cat?.getX()", singletonMap("t", t), true));
+        assertEquals(0, exec("Map a = ['other': ['cat': params.t]]; return a.other?.cat?.x",      singletonMap("t", t), true));
+        assertEquals(0, exec("def a = ['other': ['cat': params.t]]; return a.other?.cat?.getX()", singletonMap("t", t), true));
+        assertEquals(0, exec("def a = ['other': ['cat': params.t]]; return a.other?.cat?.x",      singletonMap("t", t), true));
+
+        // Assignments
+        assertNull(exec(
+                    "def a = [:];\n"
+                  + "a.missing_length = a.missing?.length();\n"
+                  + "return a.missing_length", true));
+        assertEquals(3, exec(
+                    "def a = [:];\n"
+                  + "a.missing = 'foo';\n"
+                  + "a.missing_length = a.missing?.length();\n"
+                  + "return a.missing_length", true));
+
+        // Writes, all unsupported at this point
+//        assertEquals(null, exec("org.elasticsearch.painless.FeatureTest a = null; return a?.x"));            // Read field
+//        assertEquals(null, exec("org.elasticsearch.painless.FeatureTest a = null; a?.x = 7; return a?.x"));  // Write field
+//        assertEquals(null, exec("Map a = null; a?.other = 'wow'; return a?.other")); // Write shortcut
+//        assertEquals(null, exec("def a = null; a?.other = 'cat'; return a?.other")); // Write shortcut
+//        assertEquals(null, exec("Map a = ['thing': 'bar']; a.other?.cat = 'no'; return a.other?.cat"));
+//        assertEquals(null, exec("def a = ['thing': 'bar']; a.other?.cat = 'no'; return a.other?.cat"));
+//        assertEquals(null, exec("Map a = ['thing': 'bar']; a.other?.cat?.dog = 'wombat'; return a.other?.cat?.dog"));
+//        assertEquals(null, exec("def a = ['thing': 'bar']; a.other?.cat?.dog = 'wombat'; return a.other?.cat?.dog"));
+    }
+
+    private void assertMustBeNullable(String script) {
+        Exception e = expectScriptThrows(IllegalArgumentException.class, false, () -> exec(script));
+        assertEquals("Result of null safe operator must be nullable", e.getMessage());
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/CastTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/CastTests.java
index 0551965f067a5..0ca72f993e520 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/CastTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/CastTests.java
@@ -21,7 +21,7 @@
 
 /** Tests for explicit casts */
 public class CastTests extends ScriptTestCase {
-    
+
     /**
      * Unary operator with explicit cast
      */
@@ -34,7 +34,7 @@ public void testUnaryOperator() {
         assertEquals(5L, exec("long x = 5L; return (long) (+x);"));
         assertEquals(5D, exec("long x = 5L; return (double) (+x);"));
     }
-    
+
     /**
      * Binary operators with explicit cast
      */
@@ -73,7 +73,7 @@ public void testBinaryPrefix() {
         assertEquals(6L, exec("long x = 5L; return (long) (++x);"));
         assertEquals(6D, exec("long x = 5L; return (double) (++x);"));
     }
-    
+
     /**
      * Binary compound postifx with explicit cast
      */
@@ -86,7 +86,7 @@ public void testBinaryPostfix() {
         assertEquals(5L, exec("long x = 5L; return (long) (x++);"));
         assertEquals(5D, exec("long x = 5L; return (double) (x++);"));
     }
-    
+
     /**
      * Shift operators with explicit cast
      */
@@ -99,7 +99,7 @@ public void testShiftOperator() {
         assertEquals(10L, exec("long x = 5L; return (long) (x << 1);"));
         assertEquals(10D, exec("long x = 5L; return (double) (x << 1);"));
     }
-    
+
     /**
      * Shift compound assignment with explicit cast
      */
@@ -112,7 +112,7 @@ public void testShiftCompoundAssignment() {
         assertEquals(10L, exec("long x = 5L; return (long) (x <<= 1);"));
         assertEquals(10D, exec("long x = 5L; return (double) (x <<= 1);"));
     }
-    
+
     /**
      * Test that without a cast, we fail when conversions would narrow.
      */
@@ -136,7 +136,7 @@ public void testIllegalConversions() {
             exec("long x = 5L; boolean y = (x + x); return y");
         });
     }
-    
+
     /**
      * Test that even with a cast, some things aren't allowed.
      */
@@ -161,7 +161,7 @@ public void testIllegalExplicitConversions() {
     public void testMethodCallDef() {
         assertEquals(5, exec("def x = 5; return (int)x.longValue();"));
     }
-    
+
     /**
      * Currently these do not adopt the argument value, we issue a separate cast!
      */
@@ -170,7 +170,7 @@ public void testArgumentsDef() {
         assertEquals(6, exec("def x = 5; def y = 1L; return x + (int)y"));
         assertEquals('b', exec("def x = 'abcdeg'; def y = 1L; x.charAt((int)y)"));
     }
-    
+
     /**
      * Unary operators adopt the return value
      */
@@ -183,7 +183,7 @@ public void testUnaryOperatorDef() {
         assertEquals(5L, exec("def x = 5L; return (long) (+x);"));
         assertEquals(5D, exec("def x = 5L; return (double) (+x);"));
     }
-    
+
     /**
      * Binary operators adopt the return value
      */
@@ -196,7 +196,7 @@ public void testBinaryOperatorDef() {
         assertEquals(6L, exec("def x = 5L; return (long) (x + 1);"));
         assertEquals(6D, exec("def x = 5L; return (double) (x + 1);"));
     }
-    
+
     /**
      * Binary operators don't yet adopt the return value with compound assignment
      */
@@ -209,7 +209,7 @@ public void testBinaryCompoundAssignmentDef() {
         assertEquals(6L, exec("def x = 5L; return (long) (x += 1);"));
         assertEquals(6D, exec("def x = 5L; return (double) (x += 1);"));
     }
-    
+
     /**
      * Binary operators don't yet adopt the return value with compound assignment
      */
@@ -222,7 +222,7 @@ public void testBinaryCompoundAssignmentPrefix() {
         assertEquals(6L, exec("def x = 5L; return (long) (++x);"));
         assertEquals(6D, exec("def x = 5L; return (double) (++x);"));
     }
-    
+
     /**
      * Binary operators don't yet adopt the return value with compound assignment
      */
@@ -235,7 +235,7 @@ public void testBinaryCompoundAssignmentPostfix() {
         assertEquals(5L, exec("def x = 5L; return (long) (x++);"));
         assertEquals(5D, exec("def x = 5L; return (double) (x++);"));
     }
-    
+
     /**
      * Shift operators adopt the return value
      */
@@ -248,7 +248,7 @@ public void testShiftOperatorDef() {
         assertEquals(10L, exec("def x = 5L; return (long) (x << 1);"));
         assertEquals(10D, exec("def x = 5L; return (double) (x << 1);"));
     }
-    
+
     /**
      * Shift operators don't yet adopt the return value with compound assignment
      */
@@ -261,7 +261,7 @@ public void testShiftCompoundAssignmentDef() {
         assertEquals(10L, exec("def x = 5L; return (long) (x <<= 1);"));
         assertEquals(10D, exec("def x = 5L; return (double) (x <<= 1);"));
     }
-    
+
     /**
      * Test that without a cast, we fail when conversions would narrow.
      */
@@ -285,7 +285,21 @@ public void testIllegalConversionsDef() {
             exec("def x = 5L; boolean y = (x + x); return y");
         });
     }
-    
+
+    public void testUnboxMethodParameters() {
+        assertEquals('a', exec("'a'.charAt(Integer.valueOf(0))"));
+    }
+
+    public void testIllegalCastInMethodArgument() {
+        assertEquals('a', exec("'a'.charAt(0)"));
+        Exception e = expectScriptThrows(ClassCastException.class, () -> exec("'a'.charAt(0L)"));
+        assertEquals("Cannot cast from [long] to [int].", e.getMessage());
+        e = expectScriptThrows(ClassCastException.class, () -> exec("'a'.charAt(0.0f)"));
+        assertEquals("Cannot cast from [float] to [int].", e.getMessage());
+        e = expectScriptThrows(ClassCastException.class, () -> exec("'a'.charAt(0.0d)"));
+        assertEquals("Cannot cast from [double] to [int].", e.getMessage());
+    }
+
     /**
      * Test that even with a cast, some things aren't allowed.
      * (stuff that methodhandles explicitCastArguments would otherwise allow)
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DebugTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DebugTests.java
new file mode 100644
index 0000000000000..c1098a1e7afa4
--- /dev/null
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DebugTests.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.script.ScriptException;
+
+import java.io.IOException;
+import java.util.Map;
+
+import static java.util.Collections.singletonList;
+import static java.util.Collections.singletonMap;
+import static org.hamcrest.Matchers.hasEntry;
+import static org.hamcrest.Matchers.hasKey;
+import static org.hamcrest.Matchers.not;
+
+public class DebugTests extends ScriptTestCase {
+    private final Definition definition = Definition.BUILTINS;
+
+    public void testExplain() {
+        // Debug.explain can explain an object
+        Object dummy = new Object();
+        PainlessExplainError e = expectScriptThrows(PainlessExplainError.class, () -> exec(
+                "Debug.explain(params.a)", singletonMap("a", dummy), true));
+        assertSame(dummy, e.getObjectToExplain());
+        assertThat(e.getHeaders(definition), hasEntry("es.to_string", singletonList(dummy.toString())));
+        assertThat(e.getHeaders(definition), hasEntry("es.java_class", singletonList("java.lang.Object")));
+        assertThat(e.getHeaders(definition), hasEntry("es.painless_class", singletonList("Object")));
+
+        // Null should be ok
+        e = expectScriptThrows(PainlessExplainError.class, () -> exec("Debug.explain(null)"));
+        assertNull(e.getObjectToExplain());
+        assertThat(e.getHeaders(definition), hasEntry("es.to_string", singletonList("null")));
+        assertThat(e.getHeaders(definition), not(hasKey("es.java_class")));
+        assertThat(e.getHeaders(definition), not(hasKey("es.painless_class")));
+
+        // You can't catch the explain exception
+        e = expectScriptThrows(PainlessExplainError.class, () -> exec(
+                "try {\n"
+              + "  Debug.explain(params.a)\n"
+              + "} catch (Exception e) {\n"
+              + "  return 1\n"
+              + "}", singletonMap("a", dummy), true));
+        assertSame(dummy, e.getObjectToExplain());
+    }
+
+    /**
+     * {@link PainlessExplainError} doesn't serialize but the headers still make it.
+     */
+    public void testPainlessExplainErrorSerialization() throws IOException {
+        Map<String, Object> params = singletonMap("a", "jumped over the moon");
+        ScriptException e = expectThrows(ScriptException.class, () -> exec("Debug.explain(params.a)", params, true));
+        assertEquals(singletonList("jumped over the moon"), e.getMetadata("es.to_string"));
+        assertEquals(singletonList("java.lang.String"), e.getMetadata("es.java_class"));
+        assertEquals(singletonList("String"), e.getMetadata("es.painless_class"));
+
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            out.writeException(e);
+            try (StreamInput in = out.bytes().streamInput()) {
+                ElasticsearchException read = (ScriptException) in.readException();
+                assertEquals(singletonList("jumped over the moon"), read.getMetadata("es.to_string"));
+                assertEquals(singletonList("java.lang.String"), read.getMetadata("es.java_class"));
+                assertEquals(singletonList("String"), read.getMetadata("es.painless_class"));
+            }
+        }
+    }
+}
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java
index d8e6896f083c2..9d06281b4267a 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java
@@ -30,16 +30,16 @@ final class Debugger {
 
     /** compiles source to bytecode, and returns debugging output */
     static String toString(final String source) {
-        return toString(source, new CompilerSettings());
+        return toString(GenericElasticsearchScript.class, source, new CompilerSettings());
     }
 
     /** compiles to bytecode, and returns debugging output */
-    static String toString(String source, CompilerSettings settings) {
+    static String toString(Class<?> iface, String source, CompilerSettings settings) {
         StringWriter output = new StringWriter();
         PrintWriter outputWriter = new PrintWriter(output);
         Textifier textifier = new Textifier();
         try {
-            Compiler.compile("<debugging>", source, settings, textifier);
+            Compiler.compile(iface, "<debugging>", source, settings, textifier);
         } catch (Exception e) {
             textifier.print(outputWriter);
             e.addSuppressed(new Exception("current bytecode: \n" + output));
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java
index 68d184cb1732b..bc7e4b5ebf774 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java
@@ -30,14 +30,17 @@
 import org.elasticsearch.test.ESTestCase;
 
 public class DefBootstrapTests extends ESTestCase {
+    private final Definition definition = Definition.BUILTINS;
     
     /** calls toString() on integers, twice */
     public void testOneType() throws Throwable {
-        CallSite site = DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
-                                                  "toString", 
-                                                  MethodType.methodType(String.class, Object.class), 
-                                                  0,
-                                                  DefBootstrap.METHOD_CALL, "");
+        CallSite site = DefBootstrap.bootstrap(definition,
+                                               MethodHandles.publicLookup(), 
+                                               "toString", 
+                                               MethodType.methodType(String.class, Object.class), 
+                                               0,
+                                               DefBootstrap.METHOD_CALL,
+                                               "");
         MethodHandle handle = site.dynamicInvoker();
         assertDepthEquals(site, 0);
 
@@ -51,11 +54,13 @@ public void testOneType() throws Throwable {
     }
     
     public void testTwoTypes() throws Throwable {
-        CallSite site = DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
-                                                  "toString", 
-                                                  MethodType.methodType(String.class, Object.class), 
-                                                  0,
-                                                  DefBootstrap.METHOD_CALL, "");
+        CallSite site = DefBootstrap.bootstrap(definition,
+                                               MethodHandles.publicLookup(), 
+                                               "toString", 
+                                               MethodType.methodType(String.class, Object.class), 
+                                               0,
+                                               DefBootstrap.METHOD_CALL,
+                                               "");
         MethodHandle handle = site.dynamicInvoker();
         assertDepthEquals(site, 0);
 
@@ -74,11 +79,13 @@ public void testTwoTypes() throws Throwable {
     public void testTooManyTypes() throws Throwable {
         // if this changes, test must be rewritten
         assertEquals(5, DefBootstrap.PIC.MAX_DEPTH);
-        CallSite site = DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
-                                                  "toString", 
-                                                  MethodType.methodType(String.class, Object.class), 
-                                                  0,
-                                                  DefBootstrap.METHOD_CALL, "");
+        CallSite site = DefBootstrap.bootstrap(definition,
+                                               MethodHandles.publicLookup(), 
+                                               "toString", 
+                                               MethodType.methodType(String.class, Object.class), 
+                                               0,
+                                               DefBootstrap.METHOD_CALL,
+                                               "");
         MethodHandle handle = site.dynamicInvoker();
         assertDepthEquals(site, 0);
 
@@ -98,11 +105,13 @@ public void testTooManyTypes() throws Throwable {
     
     /** test that we revert to the megamorphic classvalue cache and that it works as expected */
     public void testMegamorphic() throws Throwable {
-        DefBootstrap.PIC site = (DefBootstrap.PIC) DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
+        DefBootstrap.PIC site = (DefBootstrap.PIC) DefBootstrap.bootstrap(definition, 
+                                                                          MethodHandles.publicLookup(), 
                                                                           "size", 
                                                                           MethodType.methodType(int.class, Object.class), 
                                                                           0,
-                                                                          DefBootstrap.METHOD_CALL, "");
+                                                                          DefBootstrap.METHOD_CALL,
+                                                                          "");
         site.depth = DefBootstrap.PIC.MAX_DEPTH; // mark megamorphic
         MethodHandle handle = site.dynamicInvoker();
         assertEquals(2, (int)handle.invokeExact((Object) Arrays.asList("1", "2")));
@@ -128,43 +137,51 @@ public void testMegamorphic() throws Throwable {
     // test operators with null guards
 
     public void testNullGuardAdd() throws Throwable {
-        DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
-                                                               "add", 
-                                                               MethodType.methodType(Object.class, Object.class, Object.class),
-                                                               0,
-                                                               DefBootstrap.BINARY_OPERATOR, DefBootstrap.OPERATOR_ALLOWS_NULL);
+        DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(definition,
+                                                                          MethodHandles.publicLookup(), 
+                                                                          "add",
+                                                                          MethodType.methodType(Object.class, Object.class, Object.class),
+                                                                          0,
+                                                                          DefBootstrap.BINARY_OPERATOR,
+                                                                          DefBootstrap.OPERATOR_ALLOWS_NULL);
         MethodHandle handle = site.dynamicInvoker();
         assertEquals("nulltest", (Object)handle.invokeExact((Object)null, (Object)"test"));
     }
     
     public void testNullGuardAddWhenCached() throws Throwable {
-        DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
-                                                               "add", 
-                                                               MethodType.methodType(Object.class, Object.class, Object.class),
-                                                               0,
-                                                               DefBootstrap.BINARY_OPERATOR, DefBootstrap.OPERATOR_ALLOWS_NULL);
+        DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(definition,
+                                                                          MethodHandles.publicLookup(),
+                                                                          "add",
+                                                                          MethodType.methodType(Object.class, Object.class, Object.class),
+                                                                          0,
+                                                                          DefBootstrap.BINARY_OPERATOR,
+                                                                          DefBootstrap.OPERATOR_ALLOWS_NULL);
         MethodHandle handle = site.dynamicInvoker();
         assertEquals(2, (Object)handle.invokeExact((Object)1, (Object)1));
         assertEquals("nulltest", (Object)handle.invokeExact((Object)null, (Object)"test"));
     }
     
     public void testNullGuardEq() throws Throwable {
-        DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
-                                                               "eq", 
-                                                               MethodType.methodType(boolean.class, Object.class, Object.class),
-                                                               0,
-                                                               DefBootstrap.BINARY_OPERATOR, DefBootstrap.OPERATOR_ALLOWS_NULL);
+        DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(definition,
+                                                                          MethodHandles.publicLookup(), 
+                                                                          "eq", 
+                                                                          MethodType.methodType(boolean.class, Object.class, Object.class),
+                                                                          0,
+                                                                          DefBootstrap.BINARY_OPERATOR,
+                                                                          DefBootstrap.OPERATOR_ALLOWS_NULL);
         MethodHandle handle = site.dynamicInvoker();
         assertFalse((boolean) handle.invokeExact((Object)null, (Object)"test"));
         assertTrue((boolean) handle.invokeExact((Object)null, (Object)null));
     }
     
     public void testNullGuardEqWhenCached() throws Throwable {
-        DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
-                                                               "eq", 
-                                                               MethodType.methodType(boolean.class, Object.class, Object.class),
-                                                               0,
-                                                               DefBootstrap.BINARY_OPERATOR, DefBootstrap.OPERATOR_ALLOWS_NULL);
+        DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(definition,
+                                                                          MethodHandles.publicLookup(),
+                                                                          "eq",
+                                                                          MethodType.methodType(boolean.class, Object.class, Object.class),
+                                                                          0,
+                                                                          DefBootstrap.BINARY_OPERATOR,
+                                                                          DefBootstrap.OPERATOR_ALLOWS_NULL);
         MethodHandle handle = site.dynamicInvoker();
         assertTrue((boolean) handle.invokeExact((Object)1, (Object)1));
         assertFalse((boolean) handle.invokeExact((Object)null, (Object)"test"));
@@ -176,11 +193,13 @@ public void testNullGuardEqWhenCached() throws Throwable {
     // and can be disabled in some circumstances.
     
     public void testNoNullGuardAdd() throws Throwable {
-        DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
-                                                               "add", 
-                                                               MethodType.methodType(Object.class, int.class, Object.class),
-                                                               0,
-                                                               DefBootstrap.BINARY_OPERATOR, 0);
+        DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(definition,
+                                                                          MethodHandles.publicLookup(), 
+                                                                          "add",
+                                                                          MethodType.methodType(Object.class, int.class, Object.class),
+                                                                          0,
+                                                                          DefBootstrap.BINARY_OPERATOR,
+                                                                          0);
         MethodHandle handle = site.dynamicInvoker();
         expectThrows(NullPointerException.class, () -> {
             assertNotNull((Object)handle.invokeExact(5, (Object)null));
@@ -188,11 +207,13 @@ public void testNoNullGuardAdd() throws Throwable {
     }
     
     public void testNoNullGuardAddWhenCached() throws Throwable {
-        DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
-                                                               "add", 
-                                                               MethodType.methodType(Object.class, int.class, Object.class),
-                                                               0,
-                                                               DefBootstrap.BINARY_OPERATOR, 0);
+        DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(definition,
+                                                                          MethodHandles.publicLookup(),
+                                                                          "add",
+                                                                          MethodType.methodType(Object.class, int.class, Object.class),
+                                                                          0,
+                                                                          DefBootstrap.BINARY_OPERATOR,
+                                                                          0);
         MethodHandle handle = site.dynamicInvoker();
         assertEquals(2, (Object)handle.invokeExact(1, (Object)1));
         expectThrows(NullPointerException.class, () -> {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ElvisTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ElvisTests.java
new file mode 100644
index 0000000000000..fc760c479a139
--- /dev/null
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ElvisTests.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+import static java.util.Collections.singletonMap;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.lessThan;
+
+/**
+ * Tests for the Elvis operator ({@code ?:}).
+ */
+public class ElvisTests extends ScriptTestCase {
+    public void testBasics() {
+        // Basics
+        assertEquals("str", exec("return params.a ?: 'str'"));
+        assertEquals("str", exec("return params.a ?: 'str2'", singletonMap("a", "str"), true));
+        assertEquals("str", exec("return params.a ?: 'asdf'", singletonMap("a", "str"), true));
+
+        // Assigning to a primitive
+        assertCannotReturnPrimitive("int i = params.a ?: 1; return i");
+        assertCannotReturnPrimitive("Integer a = Integer.valueOf(1); int b = a ?: 2; return b");
+        assertCannotReturnPrimitive("Integer a = Integer.valueOf(1); int b = a ?: Integer.valueOf(2); return b");
+        assertEquals(2, exec("int i = (params.a ?: Integer.valueOf(2)).intValue(); return i"));
+        assertEquals(1, exec("int i = (params.a ?: Integer.valueOf(2)).intValue(); return i", singletonMap("a", 1), true));
+        assertEquals(1, exec("Integer a = Integer.valueOf(1); int b = (a ?: Integer.valueOf(2)).intValue(); return b"));
+        assertEquals(2, exec("Integer a = null; int b = (a ?: Integer.valueOf(2)).intValue(); return b"));
+
+        // Assigning to an object
+        assertEquals(1, exec("Integer i = params.a ?: Integer.valueOf(1); return i"));
+        assertEquals(1, exec("Integer i = params.a ?: Integer.valueOf(2); return i", singletonMap("a", 1), true));
+        assertEquals(1, exec("Integer a = Integer.valueOf(1); Integer b = a ?: Integer.valueOf(2); return b"));
+        assertEquals(2, exec("Integer a = null; Integer b = a ?: Integer.valueOf(2); return b"));
+
+        // Explicit casting
+        assertEquals(1, exec("return (Integer)(params.a ?: Integer.valueOf(1))"));
+        assertEquals(1, exec("return (Integer)(params.a ?: Integer.valueOf(2))", singletonMap("a", 1), true));
+        assertCannotReturnPrimitive("return (int)(params.a ?: 1)");
+
+        // Now some chains
+        assertEquals(1, exec("return params.a ?: params.a ?: 1"));
+        assertEquals(1, exec("return params.a ?: params.b ?: 'j'", singletonMap("b", 1), true));
+        assertEquals(1, exec("return params.a ?: params.b ?: 'j'", singletonMap("a", 1), true));
+
+        // Precedence
+        assertEquals(1, exec("return params.a ?: 2 + 2", singletonMap("a", 1), true));
+        assertEquals(4, exec("return params.a ?: 2 + 2"));
+        assertEquals(2, exec("return params.a + 1 ?: 2 + 2", singletonMap("a", 1), true)); // Yes, this is silly, but it should be valid
+
+        // Weird casts
+        assertEquals(1,     exec("int i = params.i;     String s = params.s; return s ?: i", singletonMap("i", 1), true));
+        assertEquals("str", exec("Integer i = params.i; String s = params.s; return s ?: i", singletonMap("s", "str"), true));
+
+        // Combining
+        assertEquals(2, exec("return (params.a ?: 0) + 1", singletonMap("a", 1), true));
+        assertEquals(1, exec("return (params.a ?: 0) + 1"));
+        assertEquals(2, exec("return (params.a ?: ['b': 10]).b + 1", singletonMap("a", singletonMap("b", 1)), true));
+        assertEquals(11, exec("return (params.a ?: ['b': 10]).b + 1"));
+    }
+
+    public void testWithNullSafeDereferences() {
+        assertEquals(1, exec("return params.a?.b ?: 1"));
+        assertEquals(1, exec("return params.a?.b ?: 2", singletonMap("a", singletonMap("b", 1)), true));
+
+        // TODO This could be expanded to allow primitives where neither of the two operations allow them alone
+    }
+
+    public void testLazy() {
+        assertEquals(1, exec("def fail() {throw new RuntimeException('test')} return params.a ?: fail()", singletonMap("a", 1), true));
+        Exception e = expectScriptThrows(RuntimeException.class, () ->
+            exec("def fail() {throw new RuntimeException('test')} return params.a ?: fail()"));
+        assertEquals(e.getMessage(), "test");
+    }
+
+    /**
+     * Checks that {@code a ?: b ?: c} is be parsed as {@code a ?: (b ?: c)} instead of {@code (a ?: b) ?: c} which is nice because the
+     * first one only needs one comparison if the {@code a} is non-null while the second one needs two.
+     */
+    public void testRightAssociative() {
+        checkOneBranch("params.a ?: (params.b ?: params.c)", true);
+        checkOneBranch("(params.a ?: params.b) ?: params.c", false);
+        checkOneBranch("params.a ?: params.b ?: params.c", true);
+    }
+
+    private void checkOneBranch(String code, boolean expectOneBranch) {
+        /* Sadly this is a super finicky about the output of the disassembly but I think it is worth having because it makes sure that
+         * the code generated for the elvis operator is as efficient as possible. */
+        String disassembled = Debugger.toString(code);
+        int firstLookup = disassembled.indexOf("INVOKEINTERFACE java/util/Map.get (Ljava/lang/Object;)Ljava/lang/Object;");
+        assertThat(disassembled, firstLookup, greaterThan(-1));
+        int firstElvisDestinationLabelIndex = disassembled.indexOf("IFNONNULL L", firstLookup);
+        assertThat(disassembled, firstElvisDestinationLabelIndex, greaterThan(-1));
+        String firstElvisDestinationLabel = disassembled.substring(firstElvisDestinationLabelIndex + "IFNONNULL ".length(),
+                disassembled.indexOf('\n', firstElvisDestinationLabelIndex));
+        int firstElvisDestionation = disassembled.indexOf("   " + firstElvisDestinationLabel);
+        assertThat(disassembled, firstElvisDestionation, greaterThan(-1));
+        int ifAfterFirstElvisDestination = disassembled.indexOf("IF", firstElvisDestionation);
+        if (expectOneBranch) {
+            assertThat(disassembled, ifAfterFirstElvisDestination, lessThan(0));
+        } else {
+            assertThat(disassembled, ifAfterFirstElvisDestination, greaterThan(-1));
+        }
+        int returnAfterFirstElvisDestination = disassembled.indexOf("RETURN", firstElvisDestionation);
+        assertThat(disassembled, returnAfterFirstElvisDestination, greaterThan(-1));
+    }
+
+    public void testExtraneous() {
+        Exception e = expectScriptThrows(IllegalArgumentException.class, () -> exec("int i = params.a; return i ?: 1"));
+        assertEquals("Extraneous elvis operator. LHS is a primitive.", e.getMessage());
+        expectScriptThrows(IllegalArgumentException.class, () -> exec("int i = params.a; return i + 10 ?: 'ignored'"));
+        assertEquals("Extraneous elvis operator. LHS is a primitive.", e.getMessage());
+        e = expectScriptThrows(IllegalArgumentException.class, () -> exec("return 'cat' ?: 1"));
+        assertEquals("Extraneous elvis operator. LHS is a constant.", e.getMessage());
+        e = expectScriptThrows(IllegalArgumentException.class, () -> exec("return null ?: 'j'"));
+        assertEquals("Extraneous elvis operator. LHS is null.", e.getMessage());
+        e = expectScriptThrows(IllegalArgumentException.class, () -> exec("return params.a ?: null ?: 'j'"));
+        assertEquals("Extraneous elvis operator. LHS is null.", e.getMessage());
+        e = expectScriptThrows(IllegalArgumentException.class, () -> exec("return params.a ?: null"));
+        assertEquals("Extraneous elvis operator. RHS is null.", e.getMessage());
+    }
+
+    public void testQuestionSpaceColonIsNotElvis() {
+        Exception e = expectScriptThrows(IllegalArgumentException.class, () -> exec("return params.a ? : 1", false));
+        assertEquals("invalid sequence of tokens near [':'].", e.getMessage());
+    }
+
+    private void assertCannotReturnPrimitive(String script) {
+        Exception e = expectScriptThrows(IllegalArgumentException.class, () -> exec(script));
+        assertEquals("Evlis operator cannot return primitives", e.getMessage());
+    }
+}
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/EqualsTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/EqualsTests.java
index 1df7eb349fb69..74374c98ff84a 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/EqualsTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/EqualsTests.java
@@ -1,5 +1,3 @@
-package org.elasticsearch.painless;
-
 /*
  * Licensed to Elasticsearch under one or more contributor
  * license agreements. See the NOTICE file distributed with
@@ -19,6 +17,12 @@
  * under the License.
  */
 
+package org.elasticsearch.painless;
+
+import org.elasticsearch.test.ESTestCase;
+
+import static java.util.Collections.singletonMap;
+
 // TODO: Figure out a way to test autobox caching properly from methods such as Integer.valueOf(int);
 public class EqualsTests extends ScriptTestCase {
     public void testTypesEquals() {
@@ -129,6 +133,16 @@ public void testBranchEquals() {
         assertEquals(0, exec("def a = 1; Object b = new HashMap(); if (a === (Object)b) return 1; else return 0;"));
     }
 
+    public void testBranchEqualsDefAndPrimitive() {
+        /* This test needs an Integer that isn't cached by Integer.valueOf so we draw one randomly. We can't use any fixed integer because
+         * we can never be sure that the JVM hasn't configured itself to cache that Integer. It is sneaky like that. */
+        int uncachedAutoboxedInt = randomValueOtherThanMany(i -> Integer.valueOf(i) == Integer.valueOf(i), ESTestCase::randomInt);
+        assertEquals(true, exec("def x = params.i; int y = params.i; return x == y;", singletonMap("i", uncachedAutoboxedInt), true));
+        assertEquals(false, exec("def x = params.i; int y = params.i; return x === y;", singletonMap("i", uncachedAutoboxedInt), true));
+        assertEquals(true, exec("def x = params.i; int y = params.i; return y == x;", singletonMap("i", uncachedAutoboxedInt), true));
+        assertEquals(false, exec("def x = params.i; int y = params.i; return y === x;", singletonMap("i", uncachedAutoboxedInt), true));
+    }
+
     public void testBranchNotEquals() {
         assertEquals(1, exec("def a = (char)'a'; def b = (char)'b'; if (a != b) return 1; else return 0;"));
         assertEquals(0, exec("def a = (char)'a'; def b = (char)'a'; if (a != b) return 1; else return 0;"));
@@ -139,6 +153,16 @@ public void testBranchNotEquals() {
         assertEquals(1, exec("def a = 1; Object b = new HashMap(); if (a !== (Object)b) return 1; else return 0;"));
     }
 
+    public void testBranchNotEqualsDefAndPrimitive() {
+        /* This test needs an Integer that isn't cached by Integer.valueOf so we draw one randomly. We can't use any fixed integer because
+         * we can never be sure that the JVM hasn't configured itself to cache that Integer. It is sneaky like that. */
+        int uncachedAutoboxedInt = randomValueOtherThanMany(i -> Integer.valueOf(i) == Integer.valueOf(i), ESTestCase::randomInt);
+        assertEquals(false, exec("def x = params.i; int y = params.i; return x != y;", singletonMap("i", uncachedAutoboxedInt), true));
+        assertEquals(true,  exec("def x = params.i; int y = params.i; return x !== y;", singletonMap("i", uncachedAutoboxedInt), true));
+        assertEquals(false, exec("def x = params.i; int y = params.i; return y != x;", singletonMap("i", uncachedAutoboxedInt), true));
+        assertEquals(true,  exec("def x = params.i; int y = params.i; return y !== x;", singletonMap("i", uncachedAutoboxedInt), true));
+    }
+
     public void testRightHandNull() {
         assertEquals(false, exec("HashMap a = new HashMap(); return a == null;"));
         assertEquals(false, exec("HashMap a = new HashMap(); return a === null;"));
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
index 7136807bd7e0c..7c49d042108ef 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
@@ -19,12 +19,22 @@
 
 package org.elasticsearch.painless;
 
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+
+import java.lang.invoke.LambdaConversionException;
+
+import static java.util.Collections.singletonMap;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.endsWith;
+import static org.hamcrest.Matchers.startsWith;
+
 public class FunctionRefTests extends ScriptTestCase {
 
     public void testStaticMethodReference() {
         assertEquals(1, exec("List l = new ArrayList(); l.add(2); l.add(1); l.sort(Integer::compare); return l.get(0);"));
     }
-    
+
     public void testStaticMethodReferenceDef() {
         assertEquals(1, exec("def l = new ArrayList(); l.add(2); l.add(1); l.sort(Integer::compare); return l.get(0);"));
     }
@@ -32,42 +42,73 @@ public void testStaticMethodReferenceDef() {
     public void testVirtualMethodReference() {
         assertEquals(2, exec("List l = new ArrayList(); l.add(1); l.add(1); return l.stream().mapToInt(Integer::intValue).sum();"));
     }
-    
+
     public void testVirtualMethodReferenceDef() {
         assertEquals(2, exec("def l = new ArrayList(); l.add(1); l.add(1); return l.stream().mapToInt(Integer::intValue).sum();"));
     }
 
+    public void testQualifiedStaticMethodReference() {
+        assertEquals(true,
+                exec("List l = [true]; l.stream().map(org.elasticsearch.painless.FeatureTest::overloadedStatic).findFirst().get()"));
+    }
+
+    public void testQualifiedStaticMethodReferenceDef() {
+        assertEquals(true,
+                exec("def l = [true]; l.stream().map(org.elasticsearch.painless.FeatureTest::overloadedStatic).findFirst().get()"));
+    }
+
+    public void testQualifiedVirtualMethodReference() {
+        long instant = randomLong();
+        assertEquals(instant, exec(
+                "List l = [params.d]; return l.stream().mapToLong(org.joda.time.ReadableDateTime::getMillis).sum()",
+                singletonMap("d", new DateTime(instant, DateTimeZone.UTC)), true));
+    }
+
+    public void testQualifiedVirtualMethodReferenceDef() {
+        long instant = randomLong();
+        assertEquals(instant, exec(
+                "def l = [params.d]; return l.stream().mapToLong(org.joda.time.ReadableDateTime::getMillis).sum()",
+                singletonMap("d", new DateTime(instant, DateTimeZone.UTC)), true));
+    }
+
     public void testCtorMethodReference() {
-        assertEquals(3.0D, 
-                exec("List l = new ArrayList(); l.add(1.0); l.add(2.0); " + 
-                        "DoubleStream doubleStream = l.stream().mapToDouble(Double::doubleValue);" + 
+        assertEquals(3.0D,
+                exec("List l = new ArrayList(); l.add(1.0); l.add(2.0); " +
+                        "DoubleStream doubleStream = l.stream().mapToDouble(Double::doubleValue);" +
                         "DoubleSummaryStatistics stats = doubleStream.collect(DoubleSummaryStatistics::new, " +
                         "DoubleSummaryStatistics::accept, " +
-                        "DoubleSummaryStatistics::combine); " + 
+                        "DoubleSummaryStatistics::combine); " +
                         "return stats.getSum()"));
     }
-    
+
     public void testCtorMethodReferenceDef() {
-        assertEquals(3.0D, 
-            exec("def l = new ArrayList(); l.add(1.0); l.add(2.0); " + 
-                 "def doubleStream = l.stream().mapToDouble(Double::doubleValue);" + 
+        assertEquals(3.0D,
+            exec("def l = new ArrayList(); l.add(1.0); l.add(2.0); " +
+                 "def doubleStream = l.stream().mapToDouble(Double::doubleValue);" +
                  "def stats = doubleStream.collect(DoubleSummaryStatistics::new, " +
                                                   "DoubleSummaryStatistics::accept, " +
-                                                  "DoubleSummaryStatistics::combine); " + 
+                                                  "DoubleSummaryStatistics::combine); " +
                  "return stats.getSum()"));
     }
 
+    public void testCtorWithParams() {
+        assertArrayEquals(new Object[] { "foo", "bar" },
+                (Object[]) exec("List l = new ArrayList(); l.add('foo'); l.add('bar'); " +
+                        "Stream stream = l.stream().map(StringBuilder::new);" +
+                        "return stream.map(Object::toString).toArray()"));
+    }
+
     public void testArrayCtorMethodRef() {
-        assertEquals(1.0D, 
-                exec("List l = new ArrayList(); l.add(1.0); l.add(2.0); " + 
-                     "def[] array = l.stream().toArray(Double[]::new);" + 
+        assertEquals(1.0D,
+                exec("List l = new ArrayList(); l.add(1.0); l.add(2.0); " +
+                     "def[] array = l.stream().toArray(Double[]::new);" +
                      "return array[0];"));
     }
 
     public void testArrayCtorMethodRefDef() {
-        assertEquals(1.0D, 
-                exec("def l = new ArrayList(); l.add(1.0); l.add(2.0); " + 
-                     "def[] array = l.stream().toArray(Double[]::new);" + 
+        assertEquals(1.0D,
+                exec("def l = new ArrayList(); l.add(1.0); l.add(2.0); " +
+                     "def[] array = l.stream().toArray(Double[]::new);" +
                      "return array[0];"));
     }
 
@@ -75,86 +116,109 @@ public void testCapturingMethodReference() {
         assertEquals("5", exec("Integer x = Integer.valueOf(5); return Optional.empty().orElseGet(x::toString);"));
         assertEquals("[]", exec("List l = new ArrayList(); return Optional.empty().orElseGet(l::toString);"));
     }
-    
+
     public void testCapturingMethodReferenceDefImpl() {
         assertEquals("5", exec("def x = Integer.valueOf(5); return Optional.empty().orElseGet(x::toString);"));
         assertEquals("[]", exec("def l = new ArrayList(); return Optional.empty().orElseGet(l::toString);"));
     }
-    
+
     public void testCapturingMethodReferenceDefInterface() {
         assertEquals("5", exec("Integer x = Integer.valueOf(5); def opt = Optional.empty(); return opt.orElseGet(x::toString);"));
         assertEquals("[]", exec("List l = new ArrayList(); def opt = Optional.empty(); return opt.orElseGet(l::toString);"));
     }
-    
+
     public void testCapturingMethodReferenceDefEverywhere() {
         assertEquals("5", exec("def x = Integer.valueOf(5); def opt = Optional.empty(); return opt.orElseGet(x::toString);"));
         assertEquals("[]", exec("def l = new ArrayList(); def opt = Optional.empty(); return opt.orElseGet(l::toString);"));
     }
-    
+
     public void testCapturingMethodReferenceMultipleLambdas() {
         assertEquals("testingcdefg", exec(
                 "String x = 'testing';" +
-                "String y = 'abcdefg';" + 
-                "org.elasticsearch.painless.FeatureTest test = new org.elasticsearch.painless.FeatureTest(2,3);" + 
+                "String y = 'abcdefg';" +
+                "org.elasticsearch.painless.FeatureTest test = new org.elasticsearch.painless.FeatureTest(2,3);" +
                 "return test.twoFunctionsOfX(x::concat, y::substring);"));
     }
-    
+
     public void testCapturingMethodReferenceMultipleLambdasDefImpls() {
         assertEquals("testingcdefg", exec(
                 "def x = 'testing';" +
-                "def y = 'abcdefg';" + 
-                "org.elasticsearch.painless.FeatureTest test = new org.elasticsearch.painless.FeatureTest(2,3);" + 
+                "def y = 'abcdefg';" +
+                "org.elasticsearch.painless.FeatureTest test = new org.elasticsearch.painless.FeatureTest(2,3);" +
                 "return test.twoFunctionsOfX(x::concat, y::substring);"));
     }
-    
+
     public void testCapturingMethodReferenceMultipleLambdasDefInterface() {
         assertEquals("testingcdefg", exec(
                 "String x = 'testing';" +
-                "String y = 'abcdefg';" + 
-                "def test = new org.elasticsearch.painless.FeatureTest(2,3);" + 
+                "String y = 'abcdefg';" +
+                "def test = new org.elasticsearch.painless.FeatureTest(2,3);" +
                 "return test.twoFunctionsOfX(x::concat, y::substring);"));
     }
-    
+
     public void testCapturingMethodReferenceMultipleLambdasDefEverywhere() {
         assertEquals("testingcdefg", exec(
                 "def x = 'testing';" +
-                "def y = 'abcdefg';" + 
-                "def test = new org.elasticsearch.painless.FeatureTest(2,3);" + 
+                "def y = 'abcdefg';" +
+                "def test = new org.elasticsearch.painless.FeatureTest(2,3);" +
                 "return test.twoFunctionsOfX(x::concat, y::substring);"));
     }
-    
+
     public void testOwnStaticMethodReference() {
         assertEquals(2, exec("int mycompare(int i, int j) { j - i } " +
                              "List l = new ArrayList(); l.add(2); l.add(1); l.sort(this::mycompare); return l.get(0);"));
     }
-    
+
     public void testOwnStaticMethodReferenceDef() {
         assertEquals(2, exec("int mycompare(int i, int j) { j - i } " +
                              "def l = new ArrayList(); l.add(2); l.add(1); l.sort(this::mycompare); return l.get(0);"));
     }
 
     public void testInterfaceDefaultMethod() {
-        assertEquals("bar", exec("String f(BiFunction function) { function.apply('foo', 'bar') }" + 
+        assertEquals("bar", exec("String f(BiFunction function) { function.apply('foo', 'bar') }" +
                                  "Map map = new HashMap(); f(map::getOrDefault)"));
     }
-    
+
     public void testInterfaceDefaultMethodDef() {
-        assertEquals("bar", exec("String f(BiFunction function) { function.apply('foo', 'bar') }" + 
+        assertEquals("bar", exec("String f(BiFunction function) { function.apply('foo', 'bar') }" +
                                  "def map = new HashMap(); f(map::getOrDefault)"));
     }
 
     public void testMethodMissing() {
-        IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
-            exec("List l = new ArrayList(); l.add(2); l.add(1); l.sort(Integer::bogus); return l.get(0);");
+        Exception e = expectScriptThrows(IllegalArgumentException.class, () -> {
+            exec("List l = [2, 1]; l.sort(Integer::bogus); return l.get(0);");
         });
-        assertTrue(expected.getMessage().contains("Unknown reference"));
+        assertThat(e.getMessage(), startsWith("Unknown reference"));
+    }
+
+    public void testQualifiedMethodMissing() {
+        Exception e = expectScriptThrows(IllegalArgumentException.class, () -> {
+            exec("List l = [2, 1]; l.sort(org.joda.time.ReadableDateTime::bogus); return l.get(0);", false);
+        });
+        assertThat(e.getMessage(), startsWith("Unknown reference"));
+    }
+
+    public void testClassMissing() {
+        Exception e = expectScriptThrows(IllegalArgumentException.class, () -> {
+            exec("List l = [2, 1]; l.sort(Bogus::bogus); return l.get(0);", false);
+        });
+        assertThat(e.getMessage(), endsWith("Variable [Bogus] is not defined."));
+    }
+
+    public void testQualifiedClassMissing() {
+        Exception e = expectScriptThrows(IllegalArgumentException.class, () -> {
+            exec("List l = [2, 1]; l.sort(org.joda.time.BogusDateTime::bogus); return l.get(0);", false);
+        });
+        /* Because the type isn't known and we use the lexer hack this fails to parse. I find this error message confusing but it is the one
+         * we have... */
+        assertEquals("invalid sequence of tokens near ['::'].", e.getMessage());
     }
 
     public void testNotFunctionalInterface() {
         IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("List l = new ArrayList(); l.add(2); l.add(1); l.add(Integer::bogus); return l.get(0);");
         });
-        assertTrue(expected.getMessage().contains("Cannot convert function reference"));
+        assertThat(expected.getMessage(), containsString("Cannot convert function reference"));
     }
 
     public void testIncompatible() {
@@ -162,32 +226,56 @@ public void testIncompatible() {
             exec("List l = new ArrayList(); l.add(2); l.add(1); l.sort(String::startsWith); return l.get(0);");
         });
     }
-    
+
     public void testWrongArity() {
         IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("Optional.empty().orElseGet(String::startsWith);");
         });
-        assertTrue(expected.getMessage().contains("Unknown reference"));
+        assertThat(expected.getMessage(), containsString("Unknown reference"));
     }
-    
+
     public void testWrongArityNotEnough() {
         IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("List l = new ArrayList(); l.add(2); l.add(1); l.sort(String::isEmpty);");
         });
         assertTrue(expected.getMessage().contains("Unknown reference"));
     }
-    
+
     public void testWrongArityDef() {
         IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("def y = Optional.empty(); return y.orElseGet(String::startsWith);");
         });
-        assertTrue(expected.getMessage().contains("Unknown reference"));
+        assertThat(expected.getMessage(), containsString("Unknown reference"));
     }
-    
+
     public void testWrongArityNotEnoughDef() {
         IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("def l = new ArrayList(); l.add(2); l.add(1); l.sort(String::isEmpty);");
         });
-        assertTrue(expected.getMessage().contains("Unknown reference"));
+        assertThat(expected.getMessage(), containsString("Unknown reference"));
+    }
+
+    public void testReturnVoid() {
+        Throwable expected = expectScriptThrows(ClassCastException.class, () -> {
+            exec("StringBuilder b = new StringBuilder(); List l = [1, 2]; l.stream().mapToLong(b::setLength).sum();");
+        });
+        assertThat(expected.getMessage(), containsString("Cannot cast from [void] to [long]."));
+    }
+
+    public void testReturnVoidDef() {
+        Exception expected = expectScriptThrows(LambdaConversionException.class, () -> {
+            exec("StringBuilder b = new StringBuilder(); def l = [1, 2]; l.stream().mapToLong(b::setLength);");
+        });
+        assertThat(expected.getMessage(), containsString("lambda expects return type [long], but found return type [void]"));
+
+        expected = expectScriptThrows(LambdaConversionException.class, () -> {
+            exec("def b = new StringBuilder(); def l = [1, 2]; l.stream().mapToLong(b::setLength);");
+        });
+        assertThat(expected.getMessage(), containsString("lambda expects return type [long], but found return type [void]"));
+
+        expected = expectScriptThrows(LambdaConversionException.class, () -> {
+            exec("def b = new StringBuilder(); List l = [1, 2]; l.stream().mapToLong(b::setLength);");
+        });
+        assertThat(expected.getMessage(), containsString("lambda expects return type [long], but found return type [void]"));
     }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionTests.java
index 48c09cd4025a8..8f9505d09c96c 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionTests.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.painless;
 
+import static org.hamcrest.Matchers.containsString;
+
 public class FunctionTests extends ScriptTestCase {
     public void testBasic() {
         assertEquals(5, exec("int get() {5;} get()"));
@@ -49,21 +51,58 @@ public void testEmpty() {
         Exception expected = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("void test(int x) {} test()");
         });
-        assertTrue(expected.getMessage().contains("Cannot generate an empty function"));
+        assertThat(expected.getMessage(), containsString("Cannot generate an empty function"));
+    }
+
+    public void testReturnsAreUnboxedIfNeeded() {
+        assertEquals((byte) 5, exec("byte get() {Byte.valueOf(5)} get()"));
+        assertEquals((short) 5, exec("short get() {Byte.valueOf(5)} get()"));
+        assertEquals(5, exec("int get() {Byte.valueOf(5)} get()"));
+        assertEquals((short) 5, exec("short get() {Short.valueOf(5)} get()"));
+        assertEquals(5, exec("int get() {Integer.valueOf(5)} get()"));
+        assertEquals(5.0f, exec("float get() {Float.valueOf(5)} get()"));
+        assertEquals(5.0d, exec("double get() {Float.valueOf(5)} get()"));
+        assertEquals(5.0d, exec("double get() {Double.valueOf(5)} get()"));
+        assertEquals(true, exec("boolean get() {Boolean.TRUE} get()"));
     }
 
     public void testDuplicates() {
         Exception expected = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("void test(int x) {x = 2;} void test(def y) {y = 3;} test()");
         });
-        assertTrue(expected.getMessage().contains("Duplicate functions"));
+        assertThat(expected.getMessage(), containsString("Duplicate functions"));
+    }
+
+    public void testBadCastFromMethod() {
+        Exception e = expectScriptThrows(ClassCastException.class, () -> exec("int get() {5L} get()"));
+        assertEquals("Cannot cast from [long] to [int].", e.getMessage());
+        e = expectScriptThrows(ClassCastException.class, () -> exec("int get() {5.1f} get()"));
+        assertEquals("Cannot cast from [float] to [int].", e.getMessage());
+        e = expectScriptThrows(ClassCastException.class, () -> exec("int get() {5.1d} get()"));
+        assertEquals("Cannot cast from [double] to [int].", e.getMessage());
     }
 
     public void testInfiniteLoop() {
         Error expected = expectScriptThrows(PainlessError.class, () -> {
             exec("void test() {boolean x = true; while (x) {}} test()");
         });
-        assertTrue(expected.getMessage().contains(
-            "The maximum number of statements that can be executed in a loop has been reached."));
+        assertThat(expected.getMessage(),
+                containsString("The maximum number of statements that can be executed in a loop has been reached."));
+    }
+
+    public void testReturnVoid() {
+        assertEquals(null, exec("void test(StringBuilder b, int i) {b.setLength(i)} test(new StringBuilder(), 1)"));
+        Exception expected = expectScriptThrows(IllegalArgumentException.class, () -> {
+            exec("int test(StringBuilder b, int i) {b.setLength(i)} test(new StringBuilder(), 1)");
+        });
+        assertEquals("Not all paths provide a return value for method [test].", expected.getMessage());
+        expected = expectScriptThrows(ClassCastException.class, () -> {
+            exec("int test(StringBuilder b, int i) {return b.setLength(i)} test(new StringBuilder(), 1)");
+        });
+        assertEquals("Cannot cast from [void] to [int].", expected.getMessage());
+        expected = expectScriptThrows(ClassCastException.class, () -> {
+            exec("def test(StringBuilder b, int i) {return b.setLength(i)} test(new StringBuilder(), 1)");
+        });
+        assertEquals("Cannot cast from [void] to [def].", expected.getMessage());
     }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ImplementInterfacesTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ImplementInterfacesTests.java
new file mode 100644
index 0000000000000..c3861add319dd
--- /dev/null
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ImplementInterfacesTests.java
@@ -0,0 +1,445 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.singletonMap;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.startsWith;
+
+/**
+ * Tests for Painless implementing different interfaces.
+ */
+public class ImplementInterfacesTests extends ScriptTestCase {
+    public interface NoArgs {
+        String[] ARGUMENTS = new String[] {};
+        Object execute();
+    }
+    public void testNoArgs() {
+        assertEquals(1, scriptEngine.compile(NoArgs.class, null, "1", emptyMap()).execute());
+        assertEquals("foo", scriptEngine.compile(NoArgs.class, null, "'foo'", emptyMap()).execute());
+
+        Exception e = expectScriptThrows(IllegalArgumentException.class, () ->
+            scriptEngine.compile(NoArgs.class, null, "doc", emptyMap()));
+        assertEquals("Variable [doc] is not defined.", e.getMessage());
+        // _score was once embedded into painless by deep magic
+        e = expectScriptThrows(IllegalArgumentException.class, () ->
+            scriptEngine.compile(NoArgs.class, null, "_score", emptyMap()));
+        assertEquals("Variable [_score] is not defined.", e.getMessage());
+
+        String debug = Debugger.toString(NoArgs.class, "int i = 0", new CompilerSettings());
+        /* Elasticsearch requires that scripts that return nothing return null. We hack that together by returning null from scripts that
+         * return Object if they don't return anything. */
+        assertThat(debug, containsString("ACONST_NULL"));
+        assertThat(debug, containsString("ARETURN"));
+    }
+
+    public interface OneArg {
+        String[] ARGUMENTS = new String[] {"arg"};
+        Object execute(Object arg);
+    }
+    public void testOneArg() {
+        Object rando = randomInt();
+        assertEquals(rando, scriptEngine.compile(OneArg.class, null, "arg", emptyMap()).execute(rando));
+        rando = randomAlphaOfLength(5);
+        assertEquals(rando, scriptEngine.compile(OneArg.class, null, "arg", emptyMap()).execute(rando));
+
+        Exception e = expectScriptThrows(IllegalArgumentException.class, () ->
+            scriptEngine.compile(NoArgs.class, null, "doc", emptyMap()));
+        assertEquals("Variable [doc] is not defined.", e.getMessage());
+        // _score was once embedded into painless by deep magic
+        e = expectScriptThrows(IllegalArgumentException.class, () ->
+            scriptEngine.compile(NoArgs.class, null, "_score", emptyMap()));
+        assertEquals("Variable [_score] is not defined.", e.getMessage());
+    }
+
+    public interface ArrayArg {
+        String[] ARGUMENTS = new String[] {"arg"};
+        Object execute(String[] arg);
+    }
+    public void testArrayArg() {
+        String rando = randomAlphaOfLength(5);
+        assertEquals(rando, scriptEngine.compile(ArrayArg.class, null, "arg[0]", emptyMap()).execute(new String[] {rando, "foo"}));
+    }
+
+    public interface PrimitiveArrayArg {
+        String[] ARGUMENTS = new String[] {"arg"};
+        Object execute(int[] arg);
+    }
+    public void testPrimitiveArrayArg() {
+        int rando = randomInt();
+        assertEquals(rando, scriptEngine.compile(PrimitiveArrayArg.class, null, "arg[0]", emptyMap()).execute(new int[] {rando, 10}));
+    }
+
+    public interface DefArrayArg {
+        String[] ARGUMENTS = new String[] {"arg"};
+        Object execute(Object[] arg);
+    }
+    public void testDefArrayArg() {
+        Object rando = randomInt();
+        assertEquals(rando, scriptEngine.compile(DefArrayArg.class, null, "arg[0]", emptyMap()).execute(new Object[] {rando, 10}));
+        rando = randomAlphaOfLength(5);
+        assertEquals(rando, scriptEngine.compile(DefArrayArg.class, null, "arg[0]", emptyMap()).execute(new Object[] {rando, 10}));
+        assertEquals(5, scriptEngine.compile(DefArrayArg.class, null, "arg[0].length()", emptyMap()).execute(new Object[] {rando, 10}));
+    }
+
+    public interface ManyArgs {
+        String[] ARGUMENTS = new String[] {"a", "b", "c", "d"};
+        Object execute(int a, int b, int c, int d);
+        boolean uses$a();
+        boolean uses$b();
+        boolean uses$c();
+        boolean uses$d();
+    }
+    public void testManyArgs() {
+        int rando = randomInt();
+        assertEquals(rando, scriptEngine.compile(ManyArgs.class, null, "a", emptyMap()).execute(rando, 0, 0, 0));
+        assertEquals(10, scriptEngine.compile(ManyArgs.class, null, "a + b + c + d", emptyMap()).execute(1, 2, 3, 4));
+
+        // While we're here we can verify that painless correctly finds used variables
+        ManyArgs script = scriptEngine.compile(ManyArgs.class, null, "a", emptyMap());
+        assertTrue(script.uses$a());
+        assertFalse(script.uses$b());
+        assertFalse(script.uses$c());
+        assertFalse(script.uses$d());
+        script = scriptEngine.compile(ManyArgs.class, null, "a + b + c", emptyMap());
+        assertTrue(script.uses$a());
+        assertTrue(script.uses$b());
+        assertTrue(script.uses$c());
+        assertFalse(script.uses$d());
+        script = scriptEngine.compile(ManyArgs.class, null, "a + b + c + d", emptyMap());
+        assertTrue(script.uses$a());
+        assertTrue(script.uses$b());
+        assertTrue(script.uses$c());
+        assertTrue(script.uses$d());
+    }
+
+    public interface VarargTest {
+        String[] ARGUMENTS = new String[] {"arg"};
+        Object execute(String... arg);
+    }
+    public void testVararg() {
+        assertEquals("foo bar baz", scriptEngine.compile(VarargTest.class, null, "String.join(' ', Arrays.asList(arg))", emptyMap())
+                    .execute("foo", "bar", "baz"));
+    }
+
+    public interface DefaultMethods {
+        String[] ARGUMENTS = new String[] {"a", "b", "c", "d"};
+        Object execute(int a, int b, int c, int d);
+        default Object executeWithOne() {
+            return execute(1, 1, 1, 1);
+        }
+        default Object executeWithASingleOne(int a, int b, int c) {
+            return execute(a, b, c, 1);
+        }
+    }
+    public void testDefaultMethods() {
+        int rando = randomInt();
+        assertEquals(rando, scriptEngine.compile(DefaultMethods.class, null, "a", emptyMap()).execute(rando, 0, 0, 0));
+        assertEquals(rando, scriptEngine.compile(DefaultMethods.class, null, "a", emptyMap()).executeWithASingleOne(rando, 0, 0));
+        assertEquals(10, scriptEngine.compile(DefaultMethods.class, null, "a + b + c + d", emptyMap()).execute(1, 2, 3, 4));
+        assertEquals(4, scriptEngine.compile(DefaultMethods.class, null, "a + b + c + d", emptyMap()).executeWithOne());
+        assertEquals(7, scriptEngine.compile(DefaultMethods.class, null, "a + b + c + d", emptyMap()).executeWithASingleOne(1, 2, 3));
+    }
+
+    public interface ReturnsVoid {
+        String[] ARGUMENTS = new String[] {"map"};
+        void execute(Map<String, Object> map);
+    }
+    public void testReturnsVoid() {
+        Map<String, Object> map = new HashMap<>();
+        scriptEngine.compile(ReturnsVoid.class, null, "map.a = 'foo'", emptyMap()).execute(map);
+        assertEquals(singletonMap("a", "foo"), map);
+        scriptEngine.compile(ReturnsVoid.class, null, "map.remove('a')", emptyMap()).execute(map);
+        assertEquals(emptyMap(), map);
+
+        String debug = Debugger.toString(ReturnsVoid.class, "int i = 0", new CompilerSettings());
+        // The important thing is that this contains the opcode for returning void
+        assertThat(debug, containsString(" RETURN"));
+        // We shouldn't contain any weird "default to null" logic
+        assertThat(debug, not(containsString("ACONST_NULL")));
+    }
+
+    public interface ReturnsPrimitiveBoolean {
+        String[] ARGUMENTS = new String[] {};
+        boolean execute();
+    }
+    public void testReturnsPrimitiveBoolean() {
+        assertEquals(true, scriptEngine.compile(ReturnsPrimitiveBoolean.class, null, "true", emptyMap()).execute());
+        assertEquals(false, scriptEngine.compile(ReturnsPrimitiveBoolean.class, null, "false", emptyMap()).execute());
+        assertEquals(true, scriptEngine.compile(ReturnsPrimitiveBoolean.class, null, "Boolean.TRUE", emptyMap()).execute());
+        assertEquals(false, scriptEngine.compile(ReturnsPrimitiveBoolean.class, null, "Boolean.FALSE", emptyMap()).execute());
+
+        assertEquals(true, scriptEngine.compile(ReturnsPrimitiveBoolean.class, null, "def i = true; i", emptyMap()).execute());
+        assertEquals(true, scriptEngine.compile(ReturnsPrimitiveBoolean.class, null, "def i = Boolean.TRUE; i", emptyMap()).execute());
+
+        assertEquals(true, scriptEngine.compile(ReturnsPrimitiveBoolean.class, null, "true || false", emptyMap()).execute());
+
+        String debug = Debugger.toString(ReturnsPrimitiveBoolean.class, "false", new CompilerSettings());
+        assertThat(debug, containsString("ICONST_0"));
+        // The important thing here is that we have the bytecode for returning an integer instead of an object. booleans are integers.
+        assertThat(debug, containsString("IRETURN"));
+
+        Exception e = expectScriptThrows(ClassCastException.class, () ->
+                scriptEngine.compile(ReturnsPrimitiveBoolean.class, null, "1L", emptyMap()).execute());
+        assertEquals("Cannot cast from [long] to [boolean].", e.getMessage());
+        e = expectScriptThrows(ClassCastException.class, () ->
+                scriptEngine.compile(ReturnsPrimitiveBoolean.class, null, "1.1f", emptyMap()).execute());
+        assertEquals("Cannot cast from [float] to [boolean].", e.getMessage());
+        e = expectScriptThrows(ClassCastException.class, () ->
+                scriptEngine.compile(ReturnsPrimitiveBoolean.class, null, "1.1d", emptyMap()).execute());
+        assertEquals("Cannot cast from [double] to [boolean].", e.getMessage());
+        expectScriptThrows(ClassCastException.class, () ->
+                scriptEngine.compile(ReturnsPrimitiveBoolean.class, null, "def i = 1L; i", emptyMap()).execute());
+        expectScriptThrows(ClassCastException.class, () ->
+                scriptEngine.compile(ReturnsPrimitiveBoolean.class, null, "def i = 1.1f; i", emptyMap()).execute());
+        expectScriptThrows(ClassCastException.class, () ->
+                scriptEngine.compile(ReturnsPrimitiveBoolean.class, null, "def i = 1.1d; i", emptyMap()).execute());
+
+        assertEquals(false, scriptEngine.compile(ReturnsPrimitiveBoolean.class, null, "int i = 0", emptyMap()).execute());
+    }
+
+    public interface ReturnsPrimitiveInt {
+        String[] ARGUMENTS = new String[] {};
+        int execute();
+    }
+    public void testReturnsPrimitiveInt() {
+        assertEquals(1, scriptEngine.compile(ReturnsPrimitiveInt.class, null, "1", emptyMap()).execute());
+        assertEquals(1, scriptEngine.compile(ReturnsPrimitiveInt.class, null, "(int) 1L", emptyMap()).execute());
+        assertEquals(1, scriptEngine.compile(ReturnsPrimitiveInt.class, null, "(int) 1.1d", emptyMap()).execute());
+        assertEquals(1, scriptEngine.compile(ReturnsPrimitiveInt.class, null, "(int) 1.1f", emptyMap()).execute());
+        assertEquals(1, scriptEngine.compile(ReturnsPrimitiveInt.class, null, "Integer.valueOf(1)", emptyMap()).execute());
+
+        assertEquals(1, scriptEngine.compile(ReturnsPrimitiveInt.class, null, "def i = 1; i", emptyMap()).execute());
+        assertEquals(1, scriptEngine.compile(ReturnsPrimitiveInt.class, null, "def i = Integer.valueOf(1); i", emptyMap()).execute());
+
+        assertEquals(2, scriptEngine.compile(ReturnsPrimitiveInt.class, null, "1 + 1", emptyMap()).execute());
+
+        String debug = Debugger.toString(ReturnsPrimitiveInt.class, "1", new CompilerSettings());
+        assertThat(debug, containsString("ICONST_1"));
+        // The important thing here is that we have the bytecode for returning an integer instead of an object
+        assertThat(debug, containsString("IRETURN"));
+
+        Exception e = expectScriptThrows(ClassCastException.class, () ->
+                scriptEngine.compile(ReturnsPrimitiveInt.class, null, "1L", emptyMap()).execute());
+        assertEquals("Cannot cast from [long] to [int].", e.getMessage());
+        e = expectScriptThrows(ClassCastException.class, () ->
+                scriptEngine.compile(ReturnsPrimitiveInt.class, null, "1.1f", emptyMap()).execute());
+        assertEquals("Cannot cast from [float] to [int].", e.getMessage());
+        e = expectScriptThrows(ClassCastException.class, () ->
+                scriptEngine.compile(ReturnsPrimitiveInt.class, null, "1.1d", emptyMap()).execute());
+        assertEquals("Cannot cast from [double] to [int].", e.getMessage());
+        expectScriptThrows(ClassCastException.class, () ->
+                scriptEngine.compile(ReturnsPrimitiveInt.class, null, "def i = 1L; i", emptyMap()).execute());
+        expectScriptThrows(ClassCastException.class, () ->
+                scriptEngine.compile(ReturnsPrimitiveInt.class, null, "def i = 1.1f; i", emptyMap()).execute());
+        expectScriptThrows(ClassCastException.class, () ->
+                scriptEngine.compile(ReturnsPrimitiveInt.class, null, "def i = 1.1d; i", emptyMap()).execute());
+
+        assertEquals(0, scriptEngine.compile(ReturnsPrimitiveInt.class, null, "int i = 0", emptyMap()).execute());
+    }
+
+    public interface ReturnsPrimitiveFloat {
+        String[] ARGUMENTS = new String[] {};
+        float execute();
+    }
+    public void testReturnsPrimitiveFloat() {
+        assertEquals(1.1f, scriptEngine.compile(ReturnsPrimitiveFloat.class, null, "1.1f", emptyMap()).execute(), 0);
+        assertEquals(1.1f, scriptEngine.compile(ReturnsPrimitiveFloat.class, null, "(float) 1.1d", emptyMap()).execute(), 0);
+        assertEquals(1.1f, scriptEngine.compile(ReturnsPrimitiveFloat.class, null, "def d = 1.1f; d", emptyMap()).execute(), 0);
+        assertEquals(1.1f,
+                scriptEngine.compile(ReturnsPrimitiveFloat.class, null, "def d = Float.valueOf(1.1f); d", emptyMap()).execute(), 0);
+
+        assertEquals(1.1f + 6.7f, scriptEngine.compile(ReturnsPrimitiveFloat.class, null, "1.1f + 6.7f", emptyMap()).execute(), 0);
+
+        Exception e = expectScriptThrows(ClassCastException.class, () ->
+                scriptEngine.compile(ReturnsPrimitiveFloat.class, null, "1.1d", emptyMap()).execute());
+        assertEquals("Cannot cast from [double] to [float].", e.getMessage());
+        e = expectScriptThrows(ClassCastException.class, () ->
+                scriptEngine.compile(ReturnsPrimitiveFloat.class, null, "def d = 1.1d; d", emptyMap()).execute());
+        e = expectScriptThrows(ClassCastException.class, () ->
+                scriptEngine.compile(ReturnsPrimitiveFloat.class, null, "def d = Double.valueOf(1.1); d", emptyMap()).execute());
+
+        String debug = Debugger.toString(ReturnsPrimitiveFloat.class, "1f", new CompilerSettings());
+        assertThat(debug, containsString("FCONST_1"));
+        // The important thing here is that we have the bytecode for returning a float instead of an object
+        assertThat(debug, containsString("FRETURN"));
+
+        assertEquals(0.0f, scriptEngine.compile(ReturnsPrimitiveFloat.class, null, "int i = 0", emptyMap()).execute(), 0);
+    }
+
+    public interface ReturnsPrimitiveDouble {
+        String[] ARGUMENTS = new String[] {};
+        double execute();
+    }
+    public void testReturnsPrimitiveDouble() {
+        assertEquals(1.0, scriptEngine.compile(ReturnsPrimitiveDouble.class, null, "1", emptyMap()).execute(), 0);
+        assertEquals(1.0, scriptEngine.compile(ReturnsPrimitiveDouble.class, null, "1L", emptyMap()).execute(), 0);
+        assertEquals(1.1, scriptEngine.compile(ReturnsPrimitiveDouble.class, null, "1.1d", emptyMap()).execute(), 0);
+        assertEquals((double) 1.1f, scriptEngine.compile(ReturnsPrimitiveDouble.class, null, "1.1f", emptyMap()).execute(), 0);
+        assertEquals(1.1, scriptEngine.compile(ReturnsPrimitiveDouble.class, null, "Double.valueOf(1.1)", emptyMap()).execute(), 0);
+        assertEquals((double) 1.1f,
+                    scriptEngine.compile(ReturnsPrimitiveDouble.class, null, "Float.valueOf(1.1f)", emptyMap()).execute(), 0);
+
+        assertEquals(1.0, scriptEngine.compile(ReturnsPrimitiveDouble.class, null, "def d = 1; d", emptyMap()).execute(), 0);
+        assertEquals(1.0, scriptEngine.compile(ReturnsPrimitiveDouble.class, null, "def d = 1L; d", emptyMap()).execute(), 0);
+        assertEquals(1.1, scriptEngine.compile(ReturnsPrimitiveDouble.class, null, "def d = 1.1d; d", emptyMap()).execute(), 0);
+        assertEquals((double) 1.1f, scriptEngine.compile(ReturnsPrimitiveDouble.class, null, "def d = 1.1f; d", emptyMap()).execute(), 0);
+        assertEquals(1.1,
+                scriptEngine.compile(ReturnsPrimitiveDouble.class, null, "def d = Double.valueOf(1.1); d", emptyMap()).execute(), 0);
+        assertEquals((double) 1.1f,
+                scriptEngine.compile(ReturnsPrimitiveDouble.class, null, "def d = Float.valueOf(1.1f); d", emptyMap()).execute(), 0);
+
+        assertEquals(1.1 + 6.7, scriptEngine.compile(ReturnsPrimitiveDouble.class, null, "1.1 + 6.7", emptyMap()).execute(), 0);
+
+        String debug = Debugger.toString(ReturnsPrimitiveDouble.class, "1", new CompilerSettings());
+        assertThat(debug, containsString("DCONST_1"));
+        // The important thing here is that we have the bytecode for returning a double instead of an object
+        assertThat(debug, containsString("DRETURN"));
+
+        assertEquals(0.0, scriptEngine.compile(ReturnsPrimitiveDouble.class, null, "int i = 0", emptyMap()).execute(), 0);
+    }
+
+    public interface NoArgumentsConstant {
+        Object execute(String foo);
+    }
+    public void testNoArgumentsConstant() {
+        Exception e = expectScriptThrows(IllegalArgumentException.class, false, () ->
+            scriptEngine.compile(NoArgumentsConstant.class, null, "1", emptyMap()));
+        assertThat(e.getMessage(), startsWith("Painless needs a constant [String[] ARGUMENTS] on all interfaces it implements with the "
+                + "names of the method arguments but [" + NoArgumentsConstant.class.getName() + "] doesn't have one."));
+    }
+
+    public interface WrongArgumentsConstant {
+        boolean[] ARGUMENTS = new boolean[] {false};
+        Object execute(String foo);
+    }
+    public void testWrongArgumentsConstant() {
+        Exception e = expectScriptThrows(IllegalArgumentException.class, false, () ->
+            scriptEngine.compile(WrongArgumentsConstant.class, null, "1", emptyMap()));
+        assertThat(e.getMessage(), startsWith("Painless needs a constant [String[] ARGUMENTS] on all interfaces it implements with the "
+                + "names of the method arguments but [" + WrongArgumentsConstant.class.getName() + "] doesn't have one."));
+    }
+
+    public interface WrongLengthOfArgumentConstant {
+        String[] ARGUMENTS = new String[] {"foo", "bar"};
+        Object execute(String foo);
+    }
+    public void testWrongLengthOfArgumentConstant() {
+        Exception e = expectScriptThrows(IllegalArgumentException.class, false, () ->
+            scriptEngine.compile(WrongLengthOfArgumentConstant.class, null, "1", emptyMap()));
+        assertThat(e.getMessage(), startsWith("[" + WrongLengthOfArgumentConstant.class.getName() + "#ARGUMENTS] has length [2] but ["
+                + WrongLengthOfArgumentConstant.class.getName() + "#execute] takes [1] argument."));
+    }
+
+    public interface UnknownArgType {
+        String[] ARGUMENTS = new String[] {"foo"};
+        Object execute(UnknownArgType foo);
+    }
+    public void testUnknownArgType() {
+        Exception e = expectScriptThrows(IllegalArgumentException.class, false, () ->
+            scriptEngine.compile(UnknownArgType.class, null, "1", emptyMap()));
+        assertEquals("[foo] is of unknown type [" + UnknownArgType.class.getName() + ". Painless interfaces can only accept arguments "
+                + "that are of whitelisted types.", e.getMessage());
+    }
+
+    public interface UnknownReturnType {
+        String[] ARGUMENTS = new String[] {"foo"};
+        UnknownReturnType execute(String foo);
+    }
+    public void testUnknownReturnType() {
+        Exception e = expectScriptThrows(IllegalArgumentException.class, false, () ->
+            scriptEngine.compile(UnknownReturnType.class, null, "1", emptyMap()));
+        assertEquals("Painless can only implement execute methods returning a whitelisted type but [" + UnknownReturnType.class.getName()
+                + "#execute] returns [" + UnknownReturnType.class.getName() + "] which isn't whitelisted.", e.getMessage());
+    }
+
+    public interface UnknownArgTypeInArray {
+        String[] ARGUMENTS = new String[] {"foo"};
+        Object execute(UnknownArgTypeInArray[] foo);
+    }
+    public void testUnknownArgTypeInArray() {
+        Exception e = expectScriptThrows(IllegalArgumentException.class, false, () ->
+            scriptEngine.compile(UnknownArgTypeInArray.class, null, "1", emptyMap()));
+        assertEquals("[foo] is of unknown type [" + UnknownArgTypeInArray.class.getName() + ". Painless interfaces can only accept "
+                + "arguments that are of whitelisted types.", e.getMessage());
+    }
+
+    public interface TwoExecuteMethods {
+        Object execute();
+        Object execute(boolean foo);
+    }
+    public void testTwoExecuteMethods() {
+        Exception e = expectScriptThrows(IllegalArgumentException.class, false, () ->
+            scriptEngine.compile(TwoExecuteMethods.class, null, "null", emptyMap()));
+        assertEquals("Painless can only implement interfaces that have a single method named [execute] but ["
+                + TwoExecuteMethods.class.getName() + "] has more than one.", e.getMessage());
+    }
+
+    public interface BadMethod {
+        Object something();
+    }
+    public void testBadMethod() {
+        Exception e = expectScriptThrows(IllegalArgumentException.class, false, () ->
+            scriptEngine.compile(BadMethod.class, null, "null", emptyMap()));
+        assertEquals("Painless can only implement methods named [execute] and [uses$argName] but [" + BadMethod.class.getName()
+                + "] contains a method named [something]", e.getMessage());
+    }
+
+    public interface BadUsesReturn {
+        String[] ARGUMENTS = new String[] {"foo"};
+        Object execute(String foo);
+        Object uses$foo();
+    }
+    public void testBadUsesReturn() {
+        Exception e = expectScriptThrows(IllegalArgumentException.class, false, () ->
+            scriptEngine.compile(BadUsesReturn.class, null, "null", emptyMap()));
+        assertEquals("Painless can only implement uses$ methods that return boolean but [" + BadUsesReturn.class.getName()
+                + "#uses$foo] returns [java.lang.Object].", e.getMessage());
+    }
+
+    public interface BadUsesParameter {
+        String[] ARGUMENTS = new String[] {"foo", "bar"};
+        Object execute(String foo, String bar);
+        boolean uses$bar(boolean foo);
+    }
+    public void testBadUsesParameter() {
+        Exception e = expectScriptThrows(IllegalArgumentException.class, false, () ->
+            scriptEngine.compile(BadUsesParameter.class, null, "null", emptyMap()));
+        assertEquals("Painless can only implement uses$ methods that do not take parameters but [" + BadUsesParameter.class.getName()
+                + "#uses$bar] does.", e.getMessage());
+    }
+
+    public interface BadUsesName {
+        String[] ARGUMENTS = new String[] {"foo", "bar"};
+        Object execute(String foo, String bar);
+        boolean uses$baz();
+    }
+    public void testBadUsesName() {
+        Exception e = expectScriptThrows(IllegalArgumentException.class, false, () ->
+            scriptEngine.compile(BadUsesName.class, null, "null", emptyMap()));
+        assertEquals("Painless can only implement uses$ methods that match a parameter name but [" + BadUsesName.class.getName()
+                + "#uses$baz] doesn't match any of [foo, bar].", e.getMessage());
+    }
+}
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java
index 6bb800eb92c96..20e257e574709 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java
@@ -19,61 +19,67 @@
 
 package org.elasticsearch.painless;
 
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.containsString;
+
 public class LambdaTests extends ScriptTestCase {
 
     public void testNoArgLambda() {
         assertEquals(1, exec("Optional.empty().orElseGet(() -> 1);"));
     }
-    
+
     public void testNoArgLambdaDef() {
         assertEquals(1, exec("def x = Optional.empty(); x.orElseGet(() -> 1);"));
     }
-    
+
     public void testLambdaWithArgs() {
-        assertEquals("short", exec("List l = new ArrayList(); l.add('looooong'); l.add('short'); " 
+        assertEquals("short", exec("List l = new ArrayList(); l.add('looooong'); l.add('short'); "
                                  + "l.sort((a, b) -> a.length() - b.length()); return l.get(0)"));
 
     }
-    
+
     public void testLambdaWithTypedArgs() {
-        assertEquals("short", exec("List l = new ArrayList(); l.add('looooong'); l.add('short'); " 
+        assertEquals("short", exec("List l = new ArrayList(); l.add('looooong'); l.add('short'); "
                                  + "l.sort((String a, String b) -> a.length() - b.length()); return l.get(0)"));
 
     }
-    
+
     public void testPrimitiveLambdas() {
         assertEquals(4, exec("List l = new ArrayList(); l.add(1); l.add(1); "
                            + "return l.stream().mapToInt(x -> x + 1).sum();"));
     }
-    
+
     public void testPrimitiveLambdasWithTypedArgs() {
         assertEquals(4, exec("List l = new ArrayList(); l.add(1); l.add(1); "
                            + "return l.stream().mapToInt(int x -> x + 1).sum();"));
     }
-    
+
     public void testPrimitiveLambdasDef() {
         assertEquals(4, exec("def l = new ArrayList(); l.add(1); l.add(1); "
                            + "return l.stream().mapToInt(x -> x + 1).sum();"));
     }
-    
+
     public void testPrimitiveLambdasWithTypedArgsDef() {
         assertEquals(4, exec("def l = new ArrayList(); l.add(1); l.add(1); "
                            + "return l.stream().mapToInt(int x -> x + 1).sum();"));
     }
-    
+
     public void testPrimitiveLambdasConvertible() {
-        assertEquals(2, exec("List l = new ArrayList(); l.add(1); l.add(1); "
-                           + "return l.stream().mapToInt(byte x -> x).sum();"));
+        assertEquals(2, exec("List l = new ArrayList(); l.add((short)1); l.add(1); "
+                           + "return l.stream().mapToInt(long x -> (int)1).sum();"));
     }
-    
+
     public void testPrimitiveArgs() {
         assertEquals(2, exec("int applyOne(IntFunction arg) { arg.apply(1) } applyOne(x -> x + 1)"));
     }
-    
+
     public void testPrimitiveArgsTyped() {
         assertEquals(2, exec("int applyOne(IntFunction arg) { arg.apply(1) } applyOne(int x -> x + 1)"));
     }
-    
+
     public void testPrimitiveArgsTypedOddly() {
         assertEquals(2L, exec("long applyOne(IntFunction arg) { arg.apply(1) } applyOne(long x -> x + 1)"));
     }
@@ -85,7 +91,7 @@ public void testMultipleStatements() {
     public void testUnneededCurlyStatements() {
         assertEquals(2, exec("int applyOne(IntFunction arg) { arg.apply(1) } applyOne(x -> { x + 1 })"));
     }
-    
+
     /** interface ignores return value */
     public void testVoidReturn() {
         assertEquals(2, exec("List list = new ArrayList(); "
@@ -94,7 +100,7 @@ public void testVoidReturn() {
                            + "list.forEach(x -> list2.add(x));"
                            + "return list[0]"));
     }
-    
+
     /** interface ignores return value */
     public void testVoidReturnDef() {
         assertEquals(2, exec("def list = new ArrayList(); "
@@ -121,15 +127,15 @@ public void testLambdaInLoop() {
                                "}" +
                                "return sum;"));
     }
-    
+
     public void testCapture() {
         assertEquals(5, exec("int x = 5; return Optional.empty().orElseGet(() -> x);"));
     }
-    
+
     public void testTwoCaptures() {
         assertEquals("1test", exec("int x = 1; String y = 'test'; return Optional.empty().orElseGet(() -> x + y);"));
     }
-    
+
     public void testCapturesAreReadOnly() {
         IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("List l = new ArrayList(); l.add(1); l.add(1); "
@@ -137,13 +143,13 @@ public void testCapturesAreReadOnly() {
         });
         assertTrue(expected.getMessage().contains("is read-only"));
     }
-    
+
     @AwaitsFix(bugUrl = "def type tracking")
     public void testOnlyCapturesAreReadOnly() {
         assertEquals(4, exec("List l = new ArrayList(); l.add(1); l.add(1); "
                            + "return l.stream().mapToInt(x -> { x += 1; return x }).sum();"));
     }
-    
+
     /** Lambda parameters shouldn't be able to mask a variable already in scope */
     public void testNoParamMasking() {
         IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
@@ -156,39 +162,39 @@ public void testNoParamMasking() {
     public void testCaptureDef() {
         assertEquals(5, exec("int x = 5; def y = Optional.empty(); y.orElseGet(() -> x);"));
     }
-    
+
     public void testNestedCapture() {
         assertEquals(1, exec("boolean x = false; int y = 1;" +
                              "return Optional.empty().orElseGet(() -> x ? 5 : Optional.empty().orElseGet(() -> y));"));
     }
-    
+
     public void testNestedCaptureParams() {
         assertEquals(2, exec("int foo(Function f) { return f.apply(1) }" +
                              "return foo(x -> foo(y -> x + 1))"));
     }
-    
+
     public void testWrongArity() {
-        IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
+        IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, false, () -> {
             exec("Optional.empty().orElseGet(x -> x);");
         });
         assertTrue(expected.getMessage().contains("Incorrect number of parameters"));
     }
-    
+
     public void testWrongArityDef() {
         IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("def y = Optional.empty(); return y.orElseGet(x -> x);");
         });
         assertTrue(expected.getMessage(), expected.getMessage().contains("Incorrect number of parameters"));
     }
-    
+
     public void testWrongArityNotEnough() {
-        IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
+        IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, false, () -> {
             exec("List l = new ArrayList(); l.add(1); l.add(1); "
                + "return l.stream().mapToInt(() -> 5).sum();");
         });
         assertTrue(expected.getMessage().contains("Incorrect number of parameters"));
     }
-    
+
     public void testWrongArityNotEnoughDef() {
         IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("def l = new ArrayList(); l.add(1); l.add(1); "
@@ -196,12 +202,57 @@ public void testWrongArityNotEnoughDef() {
         });
         assertTrue(expected.getMessage().contains("Incorrect number of parameters"));
     }
-    
+
     public void testLambdaInFunction() {
         assertEquals(5, exec("def foo() { Optional.empty().orElseGet(() -> 5) } return foo();"));
     }
-    
+
     public void testLambdaCaptureFunctionParam() {
         assertEquals(5, exec("def foo(int x) { Optional.empty().orElseGet(() -> x) } return foo(5);"));
     }
+
+    public void testReservedCapture() {
+        String compare = "boolean compare(Supplier s, def v) {s.get() == v}";
+        assertEquals(true, exec(compare + "compare(() -> new ArrayList(), new ArrayList())"));
+        assertEquals(true, exec(compare + "compare(() -> { new ArrayList() }, new ArrayList())"));
+
+        Map<String, Object> params = new HashMap<>();
+        params.put("key", "value");
+        params.put("number", 2);
+
+        assertEquals(true, exec(compare + "compare(() -> { return params['key'] }, 'value')", params, true));
+        assertEquals(false, exec(compare + "compare(() -> { return params['nokey'] }, 'value')", params, true));
+        assertEquals(true, exec(compare + "compare(() -> { return params['nokey'] }, null)", params, true));
+        assertEquals(true, exec(compare + "compare(() -> { return params['number'] }, 2)", params, true));
+        assertEquals(false, exec(compare + "compare(() -> { return params['number'] }, 'value')", params, true));
+        assertEquals(false, exec(compare + "compare(() -> { if (params['number'] == 2) { return params['number'] }" +
+            "else { return params['key'] } }, 'value')", params, true));
+        assertEquals(true, exec(compare + "compare(() -> { if (params['number'] == 2) { return params['number'] }" +
+            "else { return params['key'] } }, 2)", params, true));
+        assertEquals(true, exec(compare + "compare(() -> { if (params['number'] == 1) { return params['number'] }" +
+            "else { return params['key'] } }, 'value')", params, true));
+        assertEquals(false, exec(compare + "compare(() -> { if (params['number'] == 1) { return params['number'] }" +
+            "else { return params['key'] } }, 2)", params, true));
+    }
+
+    public void testReturnVoid() {
+        Throwable expected = expectScriptThrows(ClassCastException.class, () -> {
+            exec("StringBuilder b = new StringBuilder(); List l = [1, 2]; l.stream().mapToLong(i -> b.setLength(i))");
+        });
+        assertThat(expected.getMessage(), containsString("Cannot cast from [void] to [long]."));
+    }
+
+    public void testReturnVoidDef() {
+        // If we can catch the error at compile time we do
+        Exception expected = expectScriptThrows(ClassCastException.class, () -> {
+            exec("StringBuilder b = new StringBuilder(); def l = [1, 2]; l.stream().mapToLong(i -> b.setLength(i))");
+        });
+        assertThat(expected.getMessage(), containsString("Cannot cast from [void] to [def]."));
+
+        // Otherwise we convert the void into a null
+        assertEquals(Arrays.asList(null, null),
+                exec("def b = new StringBuilder(); def l = [1, 2]; l.stream().map(i -> b.setLength(i)).collect(Collectors.toList())"));
+        assertEquals(Arrays.asList(null, null),
+                exec("def b = new StringBuilder(); List l = [1, 2]; l.stream().map(i -> b.setLength(i)).collect(Collectors.toList())"));
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/LangPainlessClientYamlTestSuiteIT.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/LangPainlessClientYamlTestSuiteIT.java
index ca95dafd0b8ee..2d3f299bbcb2c 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/LangPainlessClientYamlTestSuiteIT.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/LangPainlessClientYamlTestSuiteIT.java
@@ -24,9 +24,6 @@
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-
-import java.io.IOException;
 
 /** Runs yaml rest tests */
 public class LangPainlessClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
@@ -36,8 +33,8 @@ public LangPainlessClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate t
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
-        return ESClientYamlSuiteTestCase.createParameters(0, 1);
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
     }
 }
 
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ListTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ListTests.java
new file mode 100644
index 0000000000000..1ae7ca0bc4fb3
--- /dev/null
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ListTests.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+import org.hamcrest.Matcher;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.hamcrest.Matchers.anyOf;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+
+/** Tests for working with lists. */
+public class ListTests extends ArrayLikeObjectTestCase {
+    @Override
+    protected String declType(String valueType) {
+        return "List";
+    }
+
+    @Override
+    protected String valueCtorCall(String valueType, int size) {
+        String[] fill = new String[size];
+        Arrays.fill(fill, fillValue(valueType));
+        return "[" + String.join(",", fill) + "]";
+    }
+
+    private String fillValue(String valueType) {
+        switch (valueType) {
+        case "int":    return "0";
+        case "long":   return "0L";
+        case "short":  return "(short) 0";
+        case "byte":   return "(byte) 0";
+        case "float":  return "0.0f";
+        case "double": return "0.0"; // Double is implicit for decimal constants
+        default:       return null;
+        }
+    }
+
+    @Override
+    protected Matcher<String> outOfBoundsExceptionMessageMatcher(int index, int size) {
+        if ("1.8".equals(Runtime.class.getPackage().getSpecificationVersion())) {
+            if (index > size) {
+                return equalTo("Index: " + index + ", Size: " + size);
+            }
+            Matcher<String> matcher = equalTo(Integer.toString(index));
+            // If we set -XX:-OmitStackTraceInFastThrow we wouldn't need this
+            matcher = anyOf(matcher, nullValue());
+            return matcher;
+        } else {
+            // This exception is locale dependent so we attempt to reproduce it
+            List<Object> list = new ArrayList<>();
+            for (int i = 0; i < size; i++) {
+                list.add(new Object());
+            }
+            Exception e = expectThrows(IndexOutOfBoundsException.class, () -> list.get(index));
+            return equalTo(e.getMessage());
+        }
+    }
+
+}
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/MapTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/MapTests.java
new file mode 100644
index 0000000000000..034213e74be9c
--- /dev/null
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/MapTests.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+import static java.util.Collections.singletonMap;
+
+/** Tests for working with maps. */
+public class MapTests extends ScriptTestCase {
+    private void mapAccessesTestCase(String listType) {
+        Object val = randomFrom("test", 1, 1.3, new Object());
+        String decl = listType + " x = ['a': 1, 'b': 2, 0: 2, -5: 'slot', 123.1: 12]";
+        assertEquals(5, exec(decl + "; return x.size()"));
+        assertEquals(2, exec(decl + "; return x[0];", true));
+        assertEquals(1, exec(decl + "; return x['a'];", true));
+        assertEquals(12, exec(decl + "; return x[123.1];", true));
+        assertEquals(val,    exec(decl + "; x[ 0] = params.val; return x[ 0];", singletonMap("val", val), true));
+        assertEquals("slot", exec(decl + "; x[ 0] = params.val; return x[-5];", singletonMap("val", val), true));
+        assertEquals(val,    exec(decl + "; x[-5] = params.val; return x[-5];", singletonMap("val", val), true));
+    }
+
+    public void testMapInDefAccesses() {
+        mapAccessesTestCase("def");
+    }
+
+    public void testMapAccesses() {
+        mapAccessesTestCase("Map");
+    }
+}
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java
index 3fe071c52210f..2de25ba54b08e 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java
@@ -22,7 +22,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.script.CompiledScript;
-import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.lookup.SearchLookup;
 import org.elasticsearch.test.ESSingleNodeTestCase;
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java
new file mode 100644
index 0000000000000..4c5a8ebd7dd0d
--- /dev/null
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java
@@ -0,0 +1,359 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.common.io.PathUtils;
+import org.elasticsearch.common.logging.ESLoggerFactory;
+import org.elasticsearch.painless.Definition.Field;
+import org.elasticsearch.painless.Definition.Method;
+import org.elasticsearch.painless.Definition.Struct;
+import org.elasticsearch.painless.Definition.Type;
+import org.elasticsearch.painless.api.Augmentation;
+
+import java.io.IOException;
+import java.io.PrintStream;
+import java.lang.reflect.Modifier;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.function.Consumer;
+
+import static java.util.Comparator.comparing;
+import static java.util.stream.Collectors.toList;
+
+/**
+ * Generates an API reference from the method and type whitelists in {@link Definition}.
+ */
+public class PainlessDocGenerator {
+    private static final Logger logger = ESLoggerFactory.getLogger(PainlessDocGenerator.class);
+    private static final Comparator<Field> FIELD_NAME = comparing(f -> f.name);
+    private static final Comparator<Method> METHOD_NAME = comparing(m -> m.name);
+    private static final Comparator<Method> NUMBER_OF_ARGS = comparing(m -> m.arguments.size());
+
+    public static void main(String[] args) throws IOException {
+        Path apiRootPath = PathUtils.get(args[0]);
+
+        // Blow away the last execution and recreate it from scratch
+        IOUtils.rm(apiRootPath);
+        Files.createDirectories(apiRootPath);
+
+        Path indexPath = apiRootPath.resolve("index.asciidoc");
+        logger.info("Starting to write [index.asciidoc]");
+        try (PrintStream indexStream = new PrintStream(
+                Files.newOutputStream(indexPath, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE),
+                false, StandardCharsets.UTF_8.name())) {
+            emitGeneratedWarning(indexStream);
+            List<Type> types = Definition.allSimpleTypes().stream().sorted(comparing(t -> t.name)).collect(toList());
+            for (Type type : types) {
+                if (type.sort.primitive) {
+                    // Primitives don't have methods to reference
+                    continue;
+                }
+                if ("def".equals(type.name)) {
+                    // def is special but doesn't have any methods all of its own.
+                    continue;
+                }
+                indexStream.print("include::");
+                indexStream.print(type.struct.name);
+                indexStream.println(".asciidoc[]");
+
+                Path typePath = apiRootPath.resolve(type.struct.name + ".asciidoc");
+                logger.info("Writing [{}.asciidoc]", type.name);
+                try (PrintStream typeStream = new PrintStream(
+                        Files.newOutputStream(typePath, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE),
+                        false, StandardCharsets.UTF_8.name())) {
+                    emitGeneratedWarning(typeStream);
+                    typeStream.print("[[");
+                    emitAnchor(typeStream, type.struct);
+                    typeStream.print("]]++");
+                    typeStream.print(type.name);
+                    typeStream.println("++::");
+
+                    Consumer<Field> documentField = field -> PainlessDocGenerator.documentField(typeStream, field);
+                    Consumer<Method> documentMethod = method -> PainlessDocGenerator.documentMethod(typeStream, method);
+                    type.struct.staticMembers.values().stream().sorted(FIELD_NAME).forEach(documentField);
+                    type.struct.members.values().stream().sorted(FIELD_NAME).forEach(documentField);
+                    type.struct.staticMethods.values().stream().sorted(METHOD_NAME.thenComparing(NUMBER_OF_ARGS)).forEach(documentMethod);
+                    type.struct.constructors.values().stream().sorted(NUMBER_OF_ARGS).forEach(documentMethod);
+                    Map<String, Struct> inherited = new TreeMap<>();
+                    type.struct.methods.values().stream().sorted(METHOD_NAME.thenComparing(NUMBER_OF_ARGS)).forEach(method -> {
+                        if (method.owner == type.struct) {
+                            documentMethod(typeStream, method);
+                        } else {
+                            inherited.put(method.owner.name, method.owner);
+                        }
+                    });
+
+                    if (false == inherited.isEmpty()) {
+                        typeStream.print("* Inherits methods from ");
+                        boolean first = true;
+                        for (Struct inheritsFrom : inherited.values()) {
+                            if (first) {
+                                first = false;
+                            } else {
+                                typeStream.print(", ");
+                            }
+                            typeStream.print("++");
+                            emitStruct(typeStream, inheritsFrom);
+                            typeStream.print("++");
+                        }
+                        typeStream.println();
+                    }
+                }
+            }
+        }
+        logger.info("Done writing [index.asciidoc]");
+    }
+
+    private static void documentField(PrintStream stream, Field field) {
+        stream.print("** [[");
+        emitAnchor(stream, field);
+        stream.print("]]");
+
+        if (Modifier.isStatic(field.modifiers)) {
+            stream.print("static ");
+        }
+
+        emitType(stream, field.type);
+        stream.print(' ');
+
+        String javadocRoot = javadocRoot(field);
+        emitJavadocLink(stream, javadocRoot, field);
+        stream.print('[');
+        stream.print(field.name);
+        stream.print(']');
+
+        if (javadocRoot.equals("java8")) {
+            stream.print(" (");
+            emitJavadocLink(stream, "java9", field);
+            stream.print("[java 9])");
+        }
+
+        stream.println();
+    }
+
+    /**
+     * Document a method.
+     */
+    private static void documentMethod(PrintStream stream, Method method) {
+        stream.print("* ++[[");
+        emitAnchor(stream, method);
+        stream.print("]]");
+
+        if (false == method.augmentation && Modifier.isStatic(method.modifiers)) {
+            stream.print("static ");
+        }
+
+        if (false == method.name.equals("<init>")) {
+            emitType(stream, method.rtn);
+            stream.print(' ');
+        }
+
+        String javadocRoot = javadocRoot(method);
+        emitJavadocLink(stream, javadocRoot, method);
+        stream.print('[');
+
+        stream.print(methodName(method));
+
+        stream.print("](");
+        boolean first = true;
+        for (Type arg : method.arguments) {
+            if (first) {
+                first = false;
+            } else {
+                stream.print(", ");
+            }
+            emitType(stream, arg);
+        }
+        stream.print(")++");
+
+        if (javadocRoot.equals("java8")) {
+            stream.print(" (");
+            emitJavadocLink(stream, "java9", method);
+            stream.print("[java 9])");
+        }
+
+        stream.println();
+    }
+
+    /**
+     * Anchor text for a {@link Struct}.
+     */
+    private static void emitAnchor(PrintStream stream, Struct struct) {
+        stream.print("painless-api-reference-");
+        stream.print(struct.name.replace('.', '-'));
+    }
+
+    /**
+     * Anchor text for a {@link Method}.
+     */
+    private static void emitAnchor(PrintStream stream, Method method) {
+        emitAnchor(stream, method.owner);
+        stream.print('-');
+        stream.print(methodName(method));
+        stream.print('-');
+        stream.print(method.arguments.size());
+    }
+
+    /**
+     * Anchor text for a {@link Field}.
+     */
+    private static void emitAnchor(PrintStream stream, Field field) {
+        emitAnchor(stream, field.owner);
+        stream.print('-');
+        stream.print(field.name);
+    }
+
+    private static String methodName(Method method) {
+        return method.name.equals("<init>") ? method.owner.name : method.name;
+    }
+
+    /**
+     * Emit a {@link Type}. If the type is primitive or an array of primitives this just emits the name of the type. Otherwise this emits an
+     * internal link with the text.
+     */
+    private static void emitType(PrintStream stream, Type type) {
+        emitStruct(stream, type.struct);
+        for (int i = 0; i < type.dimensions; i++) {
+            stream.print("[]");
+        }
+    }
+
+    /**
+     * Emit a {@link Struct}. If the {@linkplain Struct} is primitive or def this just emits the name of the struct. Otherwise this emits an
+     * internal link with the name.
+     */
+    private static void emitStruct(PrintStream stream, Struct struct) {
+        if (false == struct.clazz.isPrimitive() && false == struct.name.equals("def")) {
+            stream.print("<<");
+            emitAnchor(stream, struct);
+            stream.print(',');
+            stream.print(struct.name);
+            stream.print(">>");
+        } else {
+            stream.print(struct.name);
+        }
+    }
+
+    /**
+     * Emit an external link to Javadoc for a {@link Method}.
+     *
+     * @param root name of the root uri variable
+     */
+    private static void emitJavadocLink(PrintStream stream, String root, Method method) {
+        stream.print("link:{");
+        stream.print(root);
+        stream.print("-javadoc}/");
+        stream.print(classUrlPath(method.augmentation ? Augmentation.class : method.owner.clazz));
+        stream.print(".html#");
+        stream.print(methodName(method));
+        stream.print("%2D");
+        boolean first = true;
+        if (method.augmentation) {
+            first = false;
+            stream.print(method.owner.clazz.getName());
+        }
+        for (Type arg: method.arguments) {
+            if (first) {
+                first = false;
+            } else {
+                stream.print("%2D");
+            }
+            stream.print(arg.struct.clazz.getName());
+            if (arg.dimensions > 0) {
+                stream.print(":A");
+            }
+        }
+        stream.print("%2D");
+    }
+
+    /**
+     * Emit an external link to Javadoc for a {@link Field}.
+     *
+     * @param root name of the root uri variable
+     */
+    private static void emitJavadocLink(PrintStream stream, String root, Field field) {
+        stream.print("link:{");
+        stream.print(root);
+        stream.print("-javadoc}/");
+        stream.print(classUrlPath(field.owner.clazz));
+        stream.print(".html#");
+        stream.print(field.javaName);
+    }
+
+    /**
+     * Pick the javadoc root for a {@link Method}.
+     */
+    private static String javadocRoot(Method method) {
+        if (method.augmentation) {
+            return "painless";
+        }
+        return javadocRoot(method.owner);
+    }
+
+    /**
+     * Pick the javadoc root for a {@link Field}.
+     */
+    private static String javadocRoot(Field field) {
+        return javadocRoot(field.owner);
+    }
+
+    /**
+     * Pick the javadoc root for a {@link Struct}.
+     */
+    private static String javadocRoot(Struct struct) {
+        String classPackage = struct.clazz.getPackage().getName();
+        if (classPackage.startsWith("java")) {
+            return "java8";
+        }
+        if (classPackage.startsWith("org.elasticsearch.painless")) {
+            return "painless";
+        }
+        if (classPackage.startsWith("org.elasticsearch")) {
+            return "elasticsearch";
+        }
+        if (classPackage.startsWith("org.joda.time")) {
+            return "joda-time";
+        }
+        if (classPackage.startsWith("org.apache.lucene")) {
+            return "lucene-core";
+        }
+        throw new IllegalArgumentException("Unrecognized packge: " + classPackage);
+    }
+
+    private static void emitGeneratedWarning(PrintStream stream) {
+        stream.println("////");
+        stream.println("Automatically generated by PainlessDocGenerator. Do not edit.");
+        stream.println("Rebuild by running `gradle generatePainlessApi`.");
+        stream.println("////");
+        stream.println();
+    }
+
+    private static String classUrlPath(Class<?> clazz) {
+        return clazz.getName().replace('.', '/').replace('$', '.');
+    }
+}
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexTests.java
index dbbb9958d7120..92ff9ef3c9334 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexTests.java
@@ -19,24 +19,41 @@
 
 package org.elasticsearch.painless;
 
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.script.ScriptException;
+
 import java.nio.CharBuffer;
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.regex.Pattern;
-import java.util.regex.PatternSyntaxException;
 
-import static java.util.Collections.emptyMap;
 import static java.util.Collections.singletonMap;
-import static org.hamcrest.Matchers.containsString;
 
 public class RegexTests extends ScriptTestCase {
+    @Override
+    protected Settings scriptEngineSettings() {
+        // Enable regexes just for this test. They are disabled by default.
+        return Settings.builder()
+                .put(CompilerSettings.REGEX_ENABLED.getKey(), true)
+                .build();
+    }
+
     public void testPatternAfterReturn() {
         assertEquals(true, exec("return 'foo' ==~ /foo/"));
         assertEquals(false, exec("return 'bar' ==~ /foo/"));
     }
 
-    public void testSlashesEscapePattern() {
-        assertEquals(true, exec("return '//' ==~ /\\/\\//"));
+    public void testBackslashEscapesForwardSlash() {
+        assertEquals(true, exec("'//' ==~ /\\/\\//"));
+    }
+
+    public void testBackslashEscapeBackslash() {
+        // Both of these are single backslashes but java escaping + Painless escaping....
+        assertEquals(true, exec("'\\\\' ==~ /\\\\/"));
+    }
+
+    public void testRegexIsNonGreedy() {
+        assertEquals(true, exec("def s = /\\\\/.split('.\\\\.'); return s[1] ==~ /\\./"));
     }
 
     public void testPatternAfterAssignment() {
@@ -239,11 +256,15 @@ public void testCantUsePatternCompile() {
     }
 
     public void testBadRegexPattern() {
-        PatternSyntaxException e = expectScriptThrows(PatternSyntaxException.class, () -> {
+        ScriptException e = expectThrows(ScriptException.class, () -> {
             exec("/\\ujjjj/"); // Invalid unicode
         });
-        assertThat(e.getMessage(), containsString("Illegal Unicode escape sequence near index 2"));
-        assertThat(e.getMessage(), containsString("\\ujjjj"));
+        assertEquals("Error compiling regex: Illegal Unicode escape sequence", e.getCause().getMessage());
+
+        // And make sure the location of the error points to the offset inside the pattern
+        assertScriptStack(e,
+                "/\\ujjjj/",
+                "   ^---- HERE");
     }
 
     public void testRegexAgainstNumber() {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptEngineTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptEngineTests.java
index 2cd21c0596f07..89385c8862cfc 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptEngineTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptEngineTests.java
@@ -21,7 +21,7 @@
 
 import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.ExecutableScript;
-import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptType;
 
 import java.util.Arrays;
 import java.util.Collections;
@@ -83,7 +83,7 @@ public void testChangingVarsCrossExecution1() {
 
         Object compiledScript = scriptEngine.compile(null,
                 "return ctx.value;", Collections.emptyMap());
-        ExecutableScript script = scriptEngine.executable(new CompiledScript(ScriptService.ScriptType.INLINE,
+        ExecutableScript script = scriptEngine.executable(new CompiledScript(ScriptType.INLINE,
                 "testChangingVarsCrossExecution1", "painless", compiledScript), vars);
 
         ctx.put("value", 1);
@@ -99,7 +99,7 @@ public void testChangingVarsCrossExecution2() {
         Map<String, Object> vars = new HashMap<>();
         Object compiledScript = scriptEngine.compile(null, "return params['value'];", Collections.emptyMap());
 
-        ExecutableScript script = scriptEngine.executable(new CompiledScript(ScriptService.ScriptType.INLINE,
+        ExecutableScript script = scriptEngine.executable(new CompiledScript(ScriptType.INLINE,
                 "testChangingVarsCrossExecution2", "painless", compiledScript), vars);
 
         script.setNextVar("value", 1);
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java
index 63c929a69a75f..1ab5aa14508ce 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.painless;
 
+import junit.framework.AssertionFailedError;
+
 import org.apache.lucene.search.Scorer;
 import org.elasticsearch.common.lucene.ScorerAware;
 import org.elasticsearch.common.settings.Settings;
@@ -26,15 +28,15 @@
 import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.ScriptException;
-import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
-import junit.framework.AssertionFailedError;
-
 import java.util.HashMap;
 import java.util.Map;
 
+import static org.hamcrest.Matchers.hasSize;
+
 /**
  * Base test case for scripting unit tests.
  * <p>
@@ -45,7 +47,14 @@ public abstract class ScriptTestCase extends ESTestCase {
 
     @Before
     public void setup() {
-        scriptEngine = new PainlessScriptEngineService(Settings.EMPTY);
+        scriptEngine = new PainlessScriptEngineService(scriptEngineSettings());
+    }
+
+    /**
+     * Settings used to build the script engine. Override to customize settings like {@link RegexTests} does to enable regexes.
+     */
+    protected Settings scriptEngineSettings() {
+        return Settings.EMPTY;
     }
 
     /** Compiles and returns the result of {@code script} */
@@ -69,13 +78,17 @@ public Object exec(String script, Map<String, Object> vars, boolean picky) {
     public Object exec(String script, Map<String, Object> vars, Map<String,String> compileParams, Scorer scorer, boolean picky) {
         // test for ambiguity errors before running the actual script if picky is true
         if (picky) {
+            Definition definition = Definition.BUILTINS;
+            ScriptInterface scriptInterface = new ScriptInterface(definition, GenericElasticsearchScript.class);
             CompilerSettings pickySettings = new CompilerSettings();
             pickySettings.setPicky(true);
-            Walker.buildPainlessTree(getTestName(), script, pickySettings, null);
+            pickySettings.setRegexesEnabled(CompilerSettings.REGEX_ENABLED.get(scriptEngineSettings()));
+            Walker.buildPainlessTree(scriptInterface, getTestName(), script, pickySettings,
+                    definition, null);
         }
         // test actual script execution
         Object object = scriptEngine.compile(null, script, compileParams);
-        CompiledScript compiled = new CompiledScript(ScriptService.ScriptType.INLINE, getTestName(), "painless", object);
+        CompiledScript compiled = new CompiledScript(ScriptType.INLINE, getTestName(), "painless", object);
         ExecutableScript executableScript = scriptEngine.executable(compiled, vars);
         if (scorer != null) {
             ((ScorerAware)executableScript).setScorer(scorer);
@@ -103,10 +116,29 @@ public void assertBytecodeHasPattern(String script, String pattern) {
 
     /** Checks a specific exception class is thrown (boxed inside ScriptException) and returns it. */
     public static <T extends Throwable> T expectScriptThrows(Class<T> expectedType, ThrowingRunnable runnable) {
+        return expectScriptThrows(expectedType, true, runnable);
+    }
+
+    /** Checks a specific exception class is thrown (boxed inside ScriptException) and returns it. */
+    public static <T extends Throwable> T expectScriptThrows(Class<T> expectedType, boolean shouldHaveScriptStack,
+            ThrowingRunnable runnable) {
         try {
             runnable.run();
         } catch (Throwable e) {
             if (e instanceof ScriptException) {
+                boolean hasEmptyScriptStack = ((ScriptException) e).getScriptStack().isEmpty();
+                if (shouldHaveScriptStack && hasEmptyScriptStack) {
+                    if (0 != e.getCause().getStackTrace().length) {
+                        // Without -XX:-OmitStackTraceInFastThrow the jvm can eat the stack trace which causes us to ignore script_stack
+                        AssertionFailedError assertion = new AssertionFailedError("ScriptException should have a scriptStack");
+                        assertion.initCause(e);
+                        throw assertion;
+                    }
+                } else if (false == shouldHaveScriptStack && false == hasEmptyScriptStack) {
+                    AssertionFailedError assertion = new AssertionFailedError("ScriptException shouldn't have a scriptStack");
+                    assertion.initCause(e);
+                    throw assertion;
+                }
                 e = e.getCause();
                 if (expectedType.isInstance(e)) {
                     return expectedType.cast(e);
@@ -123,4 +155,21 @@ public static <T extends Throwable> T expectScriptThrows(Class<T> expectedType,
         }
         throw new AssertionFailedError("Expected exception " + expectedType.getSimpleName());
     }
+
+    /**
+     * Asserts that the script_stack looks right.
+     */
+    public static void assertScriptStack(ScriptException e, String... stack) {
+        // This particular incantation of assertions makes the error messages more useful
+        try {
+            assertThat(e.getScriptStack(), hasSize(stack.length));
+            for (int i = 0; i < stack.length; i++) {
+                assertEquals(stack[i], e.getScriptStack().get(i));
+            }
+        } catch (AssertionError assertion) {
+            assertion.initCause(e);
+            throw assertion;
+        }
+    }
+
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java
index 873e773b9a348..55cce62b8193a 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java
@@ -19,9 +19,14 @@
 
 package org.elasticsearch.painless;
 
-import static org.elasticsearch.painless.WriterConstants.MAX_INDY_STRING_CONCAT_ARGS;
+import org.apache.lucene.util.Constants;
 
+import java.util.HashMap;
 import java.util.Locale;
+import java.util.Map;
+
+import static java.util.Collections.singletonMap;
+import static org.elasticsearch.painless.WriterConstants.MAX_INDY_STRING_CONCAT_ARGS;
 
 public class StringTests extends ScriptTestCase {
 
@@ -162,12 +167,12 @@ public void testStringAndCharacter() {
         assertEquals('c', exec("String s = \"c\"; (char)s"));
         assertEquals('c', exec("String s = 'c'; (char)s"));
 
-        ClassCastException expected = expectScriptThrows(ClassCastException.class, () -> {
+        ClassCastException expected = expectScriptThrows(ClassCastException.class, false, () -> {
             assertEquals("cc", exec("return (String)(char)\"cc\""));
         });
         assertTrue(expected.getMessage().contains("Cannot cast [String] with length greater than one to [char]."));
 
-        expected = expectScriptThrows(ClassCastException.class, () -> {
+        expected = expectScriptThrows(ClassCastException.class, false, () -> {
             assertEquals("cc", exec("return (String)(char)'cc'"));
         });
         assertTrue(expected.getMessage().contains("Cannot cast [String] with length greater than one to [char]."));
@@ -182,7 +187,7 @@ public void testStringAndCharacter() {
         });
         assertTrue(expected.getMessage().contains("Cannot cast [String] with length greater than one to [char]."));
     }
-    
+
     public void testDefConcat() {
         assertEquals("a" + (byte)2, exec("def x = 'a'; def y = (byte)2; return x + y"));
         assertEquals("a" + (short)2, exec("def x = 'a'; def y = (short)2; return x + y"));
@@ -205,7 +210,7 @@ public void testDefConcat() {
             exec("def x = null; def y = null; return x + y");
         });
     }
-    
+
     public void testDefCompoundAssignment() {
         assertEquals("a" + (byte)2, exec("def x = 'a'; x += (byte)2; return x"));
         assertEquals("a" + (short)2, exec("def x = 'a'; x  += (short)2; return x"));
@@ -222,7 +227,35 @@ public void testDefCompoundAssignment() {
         });
     }
 
+    public void testComplexCompoundAssignment() {
+        Map<String, Object> params = new HashMap<>();
+        Map<String, Object> ctx = new HashMap<>();
+        ctx.put("_id", "somerandomid");
+        params.put("ctx", ctx);
+
+        assertEquals("somerandomid.somerandomid", exec("ctx._id += '.' + ctx._id", params, false));
+        assertEquals("somerandomid.somerandomid", exec("String x = 'somerandomid'; x += '.' + x"));
+        assertEquals("somerandomid.somerandomid", exec("def x = 'somerandomid'; x += '.' + x"));
+    }
+
     public void testAppendStringIntoMap() {
         assertEquals("nullcat", exec("def a = new HashMap(); a.cat += 'cat'"));
     }
+
+    public void testBase64Augmentations() {
+        assertEquals("Y2F0", exec("'cat'.encodeBase64()"));
+        assertEquals("cat", exec("'Y2F0'.decodeBase64()"));
+        assertEquals("6KiA6Kqe", exec("'\u8A00\u8A9E'.encodeBase64()"));
+        assertEquals("\u8A00\u8A9E", exec("'6KiA6Kqe'.decodeBase64()"));
+
+        String rando = randomRealisticUnicodeOfLength(between(5, 1000));
+        assertEquals(rando, exec("params.rando.encodeBase64().decodeBase64()", singletonMap("rando", rando), true));
+    }
+    
+    public void testJava9StringConcatBytecode() {
+        assumeTrue("Needs Java 9 to test indified String concat", Constants.JRE_IS_MINIMUM_JAVA9);
+        assertNotNull(WriterConstants.INDY_STRING_CONCAT_BOOTSTRAP_HANDLE);
+        assertBytecodeExists("String s = \"cat\"; return s + true + 'abc' + null;", 
+                "INVOKEDYNAMIC concat(Ljava/lang/String;ZLjava/lang/String;Ljava/lang/Object;)Ljava/lang/String;");
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
index 1d60eb9c29a59..d60da7b795fbc 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.painless;
 
+import junit.framework.AssertionFailedError;
+
 import org.apache.lucene.util.Constants;
 import org.elasticsearch.script.ScriptException;
 
@@ -27,53 +29,94 @@
 import java.util.Collections;
 
 import static java.util.Collections.emptyMap;
-import static org.hamcrest.Matchers.containsString;
+import static java.util.Collections.singletonMap;
+import static org.hamcrest.Matchers.instanceOf;
 
 public class WhenThingsGoWrongTests extends ScriptTestCase {
     public void testNullPointer() {
         expectScriptThrows(NullPointerException.class, () -> {
             exec("int x = params['missing']; return x;");
         });
-    }
-
-    /** test "line numbers" in the bytecode, which are really 1-based offsets */
-    public void testLineNumbers() {
-        // trigger NPE at line 1 of the script
-        NullPointerException exception = expectScriptThrows(NullPointerException.class, () -> {
-            exec("String x = null; boolean y = x.isEmpty();\n" +
-                 "return y;");
-        });
-        // null deref at x.isEmpty(), the '.' is offset 30 (+1)
-        assertEquals(30 + 1, exception.getStackTrace()[0].getLineNumber());
-
-        // trigger NPE at line 2 of the script
-        exception = expectScriptThrows(NullPointerException.class, () -> {
-            exec("String x = null;\n" +
-                 "return x.isEmpty();");
+        expectScriptThrows(NullPointerException.class, () -> {
+            exec("Double.parseDouble(params['missing'])");
         });
-        // null deref at x.isEmpty(), the '.' is offset 25 (+1)
-        assertEquals(25 + 1, exception.getStackTrace()[0].getLineNumber());
+    }
 
-        // trigger NPE at line 3 of the script
-        exception = expectScriptThrows(NullPointerException.class, () -> {
-            exec("String x = null;\n" +
-                 "String y = x;\n" +
-                 "return y.isEmpty();");
-        });
-        // null deref at y.isEmpty(), the '.' is offset 39 (+1)
-        assertEquals(39 + 1, exception.getStackTrace()[0].getLineNumber());
+    /**
+     * Test that the scriptStack looks good. By implication this tests that we build proper "line numbers" in stack trace. These line
+     * numbers are really 1 based character numbers.
+     */
+    public void testScriptStack() {
+        for (String type : new String[] {"String", "def   "}) {
+            // trigger NPE at line 1 of the script
+            ScriptException exception = expectThrows(ScriptException.class, () -> {
+                exec(type + " x = null; boolean y = x.isEmpty();\n" +
+                     "return y;");
+            });
+            // null deref at x.isEmpty(), the '.' is offset 30
+            assertScriptElementColumn(30, exception);
+            assertScriptStack(exception,
+                    "y = x.isEmpty();\n",
+                    "     ^---- HERE");
+            assertThat(exception.getCause(), instanceOf(NullPointerException.class));
+
+            // trigger NPE at line 2 of the script
+            exception = expectThrows(ScriptException.class, () -> {
+                exec(type + " x = null;\n" +
+                     "return x.isEmpty();");
+            });
+            // null deref at x.isEmpty(), the '.' is offset 25
+            assertScriptElementColumn(25, exception);
+            assertScriptStack(exception,
+                    "return x.isEmpty();",
+                    "        ^---- HERE");
+            assertThat(exception.getCause(), instanceOf(NullPointerException.class));
+
+            // trigger NPE at line 3 of the script
+            exception = expectThrows(ScriptException.class, () -> {
+                exec(type + " x = null;\n" +
+                     type + " y = x;\n" +
+                     "return y.isEmpty();");
+            });
+            // null deref at y.isEmpty(), the '.' is offset 39
+            assertScriptElementColumn(39, exception);
+            assertScriptStack(exception,
+                    "return y.isEmpty();",
+                    "        ^---- HERE");
+            assertThat(exception.getCause(), instanceOf(NullPointerException.class));
+
+            // trigger NPE at line 4 in script (inside conditional)
+            exception = expectThrows(ScriptException.class, () -> {
+                exec(type + " x = null;\n" +
+                     "boolean y = false;\n" +
+                     "if (!y) {\n" +
+                     "  y = x.isEmpty();\n" +
+                     "}\n" +
+                     "return y;");
+            });
+            // null deref at x.isEmpty(), the '.' is offset 53
+            assertScriptElementColumn(53, exception);
+            assertScriptStack(exception,
+                    "y = x.isEmpty();\n}\n",
+                    "     ^---- HERE");
+            assertThat(exception.getCause(), instanceOf(NullPointerException.class));
+        }
+    }
 
-        // trigger NPE at line 4 in script (inside conditional)
-        exception = expectScriptThrows(NullPointerException.class, () -> {
-            exec("String x = null;\n" +
-                 "boolean y = false;\n" +
-                 "if (!y) {\n" +
-                 "  y = x.isEmpty();\n" +
-                 "}\n" +
-                 "return y;");
-        });
-        // null deref at x.isEmpty(), the '.' is offset 53 (+1)
-        assertEquals(53 + 1, exception.getStackTrace()[0].getLineNumber());
+    private void assertScriptElementColumn(int expectedColumn, ScriptException exception) {
+        StackTraceElement[] stackTrace = exception.getCause().getStackTrace();
+        for (int i = 0; i < stackTrace.length; i++) {
+            if (WriterConstants.CLASS_NAME.equals(stackTrace[i].getClassName())) {
+                if (expectedColumn + 1 != stackTrace[i].getLineNumber()) {
+                    AssertionFailedError assertion = new AssertionFailedError("Expected column to be [" + expectedColumn + "] but was ["
+                            + stackTrace[i].getLineNumber() + "]");
+                    assertion.initCause(exception);
+                    throw assertion;
+                }
+                return;
+            }
+        }
+        fail("didn't find script stack element");
     }
 
     public void testInvalidShift() {
@@ -149,10 +192,10 @@ public void testInfiniteLoops() {
 
     public void testLoopLimits() {
         // right below limit: ok
-        exec("for (int x = 0; x < 9999; ++x) {}");
+        exec("for (int x = 0; x < 999999; ++x) {}");
 
         PainlessError expected = expectScriptThrows(PainlessError.class, () -> {
-            exec("for (int x = 0; x < 10000; ++x) {}");
+            exec("for (int x = 0; x < 1000000; ++x) {}");
         });
         assertTrue(expected.getMessage().contains(
                    "The maximum number of statements that can be executed in a loop has been reached."));
@@ -162,7 +205,7 @@ public void testSourceLimits() {
         final char[] tooManyChars = new char[Compiler.MAXIMUM_SOURCE_LENGTH + 1];
         Arrays.fill(tooManyChars, '0');
 
-        IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
+        IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, false, () -> {
             exec(new String(tooManyChars));
         });
         assertTrue(expected.getMessage().contains("Scripts may be no longer than"));
@@ -234,4 +277,53 @@ public void testStackOverflowError() {
             exec("void recurse(int x, int y) {recurse(x, y)} recurse(1, 2);");
         });
     }
+
+    public void testRegexDisabledByDefault() {
+        IllegalStateException e = expectThrows(IllegalStateException.class, () -> exec("return 'foo' ==~ /foo/"));
+        assertEquals("Regexes are disabled. Set [script.painless.regex.enabled] to [true] in elasticsearch.yaml to allow them. "
+                + "Be careful though, regexes break out of Painless's protection against deep recursion and long loops.", e.getMessage());
+    }
+
+    public void testCanNotOverrideRegexEnabled() {
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> exec("", null, singletonMap(CompilerSettings.REGEX_ENABLED.getKey(), "true"), null, false));
+        assertEquals("[painless.regex.enabled] can only be set on node startup.", e.getMessage());
+    }
+
+    public void testInvalidIntConstantSuggestsLong() {
+        IllegalArgumentException e = expectScriptThrows(IllegalArgumentException.class, () -> exec("return 864000000000"));
+        assertEquals("Invalid int constant [864000000000]. If you want a long constant then change it to [864000000000L].", e.getMessage());
+        assertEquals(864000000000L, exec("return 864000000000L"));
+        e = expectScriptThrows(IllegalArgumentException.class, () -> exec("return -864000000000"));
+        assertEquals("Invalid int constant [-864000000000]. If you want a long constant then change it to [-864000000000L].",
+                e.getMessage());
+        assertEquals(-864000000000L, exec("return -864000000000L"));
+
+        // If it isn't a valid long we don't give any suggestions
+        e = expectScriptThrows(IllegalArgumentException.class, () -> exec("return 92233720368547758070"));
+        assertEquals("Invalid int constant [92233720368547758070].", e.getMessage());
+        e = expectScriptThrows(IllegalArgumentException.class, () -> exec("return -92233720368547758070"));
+        assertEquals("Invalid int constant [-92233720368547758070].", e.getMessage());
+    }
+
+    public void testQuestionSpaceDotIsNotNullSafeDereference() {
+        Exception e = expectScriptThrows(IllegalArgumentException.class, () -> exec("return params.a? .b", false));
+        assertEquals("invalid sequence of tokens near ['.'].", e.getMessage());
+    }
+
+    public void testBadStringEscape() {
+        Exception e = expectScriptThrows(IllegalArgumentException.class, () -> exec("'\\a'", false));
+        assertEquals("unexpected character ['\\a]. The only valid escape sequences in strings starting with ['] are [\\\\] and [\\'].",
+                e.getMessage());
+        e = expectScriptThrows(IllegalArgumentException.class, () -> exec("\"\\a\"", false));
+        assertEquals("unexpected character [\"\\a]. The only valid escape sequences in strings starting with [\"] are [\\\\] and [\\\"].",
+                e.getMessage());
+    }
+
+    public void testRegularUnexpectedCharacter() {
+        Exception e = expectScriptThrows(IllegalArgumentException.class, () -> exec("'", false));
+        assertEquals("unexpected character ['].", e.getMessage());
+        e = expectScriptThrows(IllegalArgumentException.class, () -> exec("'cat", false));
+        assertEquals("unexpected character ['cat].", e.getMessage());
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java
new file mode 100644
index 0000000000000..13d48bd2a0119
--- /dev/null
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java
@@ -0,0 +1,911 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Definition.Cast;
+import org.elasticsearch.painless.Definition.Field;
+import org.elasticsearch.painless.Definition.Method;
+import org.elasticsearch.painless.Definition.MethodKey;
+import org.elasticsearch.painless.Definition.RuntimeClass;
+import org.elasticsearch.painless.Definition.Struct;
+import org.elasticsearch.painless.FeatureTest;
+import org.elasticsearch.painless.GenericElasticsearchScript;
+import org.elasticsearch.painless.Locals.Variable;
+import org.elasticsearch.painless.Location;
+import org.elasticsearch.painless.ScriptInterface;
+import org.elasticsearch.painless.Operation;
+import org.elasticsearch.painless.antlr.Walker;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import static java.util.Collections.emptyList;
+import static java.util.Collections.singletonList;
+
+/**
+ * Tests {@link Object#toString} implementations on all extensions of {@link ANode}.
+ */
+public class NodeToStringTests extends ESTestCase {
+    private final Definition definition = Definition.BUILTINS;
+
+    public void testEAssignment() {
+        assertToString(
+                "(SSource\n"
+              + "  (SDeclBlock (SDeclaration def i))\n"
+              + "  (SExpression (EAssignment (EVariable i) = (ENumeric 2)))\n"
+              + "  (SReturn (EVariable i)))",
+                "def i;\n"
+              + "i = 2;\n"
+              + "return i");
+        for (String operator : new String[] {"+", "-", "*", "/", "%", "&", "^", "|", "<<", ">>", ">>>"}) {
+            assertToString(
+                    "(SSource\n"
+                  + "  (SDeclBlock (SDeclaration def i (ENumeric 1)))\n"
+                  + "  (SExpression (EAssignment (EVariable i) " + operator + "= (ENumeric 2)))\n"
+                  + "  (SReturn (EVariable i)))",
+                    "def i = 1;\n"
+                  + "i " + operator + "= 2;\n"
+                  + "return i");
+        }
+        // Compound
+        assertToString(
+                "(SSource\n"
+              + "  (SDeclBlock (SDeclaration def i))\n"
+              + "  (SReturn (EAssignment (EVariable i) = (ENumeric 2))))",
+                "def i;\n"
+              + "return i = 2");
+        assertToString(
+                "(SSource\n"
+              + "  (SDeclBlock (SDeclaration def i))\n"
+              + "  (SReturn (EAssignment (EVariable i) ++ post)))",
+                "def i;\n"
+              + "return i++");
+        assertToString(
+                "(SSource\n"
+              + "  (SDeclBlock (SDeclaration def i))\n"
+              + "  (SReturn (EAssignment (EVariable i) ++ pre)))",
+                "def i;\n"
+              + "return ++i");
+        assertToString(
+                "(SSource\n"
+              + "  (SDeclBlock (SDeclaration def i))\n"
+              + "  (SReturn (EAssignment (EVariable i) -- post)))",
+                "def i;\n"
+              + "return i--");
+        assertToString(
+                "(SSource\n"
+              + "  (SDeclBlock (SDeclaration def i))\n"
+              + "  (SReturn (EAssignment (EVariable i) -- pre)))",
+                "def i;\n"
+              + "return --i");
+    }
+
+    public void testEBinary() {
+        assertToString(     "(SSource (SReturn (EBinary (ENumeric 1) * (ENumeric 1))))", "return 1 * 1");
+        assertToString(     "(SSource (SReturn (EBinary (ENumeric 1) / (ENumeric 1))))", "return 1 / 1");
+        assertToString(     "(SSource (SReturn (EBinary (ENumeric 1) % (ENumeric 1))))", "return 1 % 1");
+        assertToString(     "(SSource (SReturn (EBinary (ENumeric 1) + (ENumeric 1))))", "return 1 + 1");
+        assertToString(     "(SSource (SReturn (EBinary (ENumeric 1) - (ENumeric 1))))", "return 1 - 1");
+        assertToString( "(SSource (SReturn (EBinary (EString 'asb') =~ (ERegex /cat/))))", "return 'asb' =~ /cat/");
+        assertToString("(SSource (SReturn (EBinary (EString 'asb') ==~ (ERegex /cat/))))", "return 'asb' ==~ /cat/");
+        assertToString(    "(SSource (SReturn (EBinary (ENumeric 1) << (ENumeric 1))))", "return 1 << 1");
+        assertToString(    "(SSource (SReturn (EBinary (ENumeric 1) >> (ENumeric 1))))", "return 1 >> 1");
+        assertToString(   "(SSource (SReturn (EBinary (ENumeric 1) >>> (ENumeric 1))))", "return 1 >>> 1");
+        assertToString(     "(SSource (SReturn (EBinary (ENumeric 1) & (ENumeric 1))))", "return 1 & 1");
+        assertToString(     "(SSource (SReturn (EBinary (ENumeric 1) ^ (ENumeric 1))))", "return 1 ^ 1");
+        assertToString(     "(SSource (SReturn (EBinary (ENumeric 1) | (ENumeric 1))))", "return 1 | 1");
+    }
+
+    public void testEBool() {
+        assertToString("(SSource (SReturn (EBool (EBoolean true) && (EBoolean false))))", "return true && false");
+        assertToString("(SSource (SReturn (EBool (EBoolean true) || (EBoolean false))))", "return true || false");
+    }
+
+    public void testEBoolean() {
+        assertToString("(SSource (SReturn (EBoolean true)))",  "return true");
+        assertToString("(SSource (SReturn (EBoolean false)))", "return false");
+    }
+
+    public void testECallLocal() {
+        assertToString(
+                "(SSource\n"
+              + "  (SFunction def a\n"
+              + "    (SReturn (EBoolean true)))\n"
+              + "  (SReturn (ECallLocal a)))",
+                "def a() {\n"
+              + "  return true\n"
+              + "}\n"
+              + "return a()");
+        assertToString(
+                "(SSource\n"
+              + "  (SFunction def a (Args (Pair int i) (Pair int j))\n"
+              + "    (SReturn (EBoolean true)))\n"
+              + "  (SReturn (ECallLocal a (Args (ENumeric 1) (ENumeric 2)))))",
+                "def a(int i, int j) {\n"
+              + "  return true\n"
+              + "}\n"
+              + "return a(1, 2)");
+    }
+
+    public void testECapturingFunctionRef() {
+        assertToString(
+                  "(SSource\n"
+                + "  (SDeclBlock (SDeclaration Integer x (PCallInvoke (EStatic Integer) valueOf (Args (ENumeric 5)))))\n"
+                + "  (SReturn (PCallInvoke (PCallInvoke (EStatic Optional) empty) orElseGet (Args (ECapturingFunctionRef x toString)))))",
+                  "Integer x = Integer.valueOf(5);\n"
+                + "return Optional.empty().orElseGet(x::toString)");
+    }
+
+    public void testECast() {
+        Location l = new Location(getTestName(), 0);
+        AExpression child = new EConstant(l, "test");
+        Cast cast = new Cast(Definition.STRING_TYPE, Definition.INT_OBJ_TYPE, true);
+        assertEquals("(ECast Integer (EConstant String 'test'))", new ECast(l, child, cast).toString());
+
+        l = new Location(getTestName(), 1);
+        child = new EBinary(l, Operation.ADD, new EConstant(l, "test"), new EConstant(l, 12));
+        cast = new Cast(Definition.INT_OBJ_TYPE, Definition.BOOLEAN_OBJ_TYPE, true);
+        assertEquals("(ECast Boolean (EBinary (EConstant String 'test') + (EConstant Integer 12)))", new ECast(l, child, cast).toString());
+    }
+
+    public void testEComp() {
+        assertToString(  "(SSource (SReturn (EComp (PField (EVariable params) a) < (ENumeric 10))))", "return params.a < 10");
+        assertToString( "(SSource (SReturn (EComp (PField (EVariable params) a) <= (ENumeric 10))))", "return params.a <= 10");
+        assertToString(  "(SSource (SReturn (EComp (PField (EVariable params) a) > (ENumeric 10))))", "return params.a > 10");
+        assertToString( "(SSource (SReturn (EComp (PField (EVariable params) a) >= (ENumeric 10))))", "return params.a >= 10");
+        assertToString( "(SSource (SReturn (EComp (PField (EVariable params) a) == (ENumeric 10))))", "return params.a == 10");
+        assertToString("(SSource (SReturn (EComp (PField (EVariable params) a) === (ENumeric 10))))", "return params.a === 10");
+        assertToString( "(SSource (SReturn (EComp (PField (EVariable params) a) != (ENumeric 10))))", "return params.a != 10");
+        assertToString("(SSource (SReturn (EComp (PField (EVariable params) a) !== (ENumeric 10))))", "return params.a !== 10");
+    }
+
+    public void testEConditional() {
+        assertToString("(SSource (SReturn (EConditional (PField (EVariable params) a) (ENumeric 1) (ENumeric 6))))",
+                "return params.a ? 1 : 6");
+    }
+
+    public void testEConstant() {
+        assertEquals("(EConstant String '121')", new EConstant(new Location(getTestName(), 0), "121").toString());
+        assertEquals("(EConstant String '92 ')", new EConstant(new Location(getTestName(), 0), "92 ").toString());
+        assertEquals("(EConstant Integer 1237)", new EConstant(new Location(getTestName(), 1), 1237).toString());
+        assertEquals("(EConstant Boolean true)", new EConstant(new Location(getTestName(), 2), true).toString());
+    }
+
+    public void testEDecimal() {
+        assertToString("(SSource (SReturn (EDecimal 1.0)))", "return 1.0");
+        assertToString("(SSource (SReturn (EDecimal 14.121d)))", "return 14.121d");
+        assertToString("(SSource (SReturn (EDecimal 2234.1f)))", "return 2234.1f");
+        assertToString("(SSource (SReturn (EDecimal 14.121D)))", "return 14.121D");
+        assertToString("(SSource (SReturn (EDecimal 1234.1F)))", "return 1234.1F");
+    }
+
+    public void testEElvis() {
+        assertToString("(SSource (SReturn (EElvis (PField (EVariable params) a) (ENumeric 1))))", "return params.a ?: 1");
+    }
+
+    public void testEExplicit() {
+        assertToString("(SSource (SReturn (EExplicit byte (PField (EVariable params) a))))", "return (byte)(params.a)");
+    }
+
+    public void testEFunctionRef() {
+        assertToString(
+                "(SSource (SReturn "
+                        + "(PCallInvoke (PCallInvoke (EStatic Optional) empty) orElseGet (Args (EFunctionRef Optional empty)))))",
+                "return Optional.empty().orElseGet(Optional::empty)");
+    }
+
+    public void testEInstanceOf() {
+        assertToString("(SSource (SReturn (EInstanceof (ENewObj Object) Object)))", "return new Object() instanceof Object");
+        assertToString("(SSource (SReturn (EInstanceof (ENumeric 12) double)))", "return 12 instanceof double");
+    }
+
+    public void testELambda() {
+        assertToString(
+                  "(SSource (SReturn (PCallInvoke (PCallInvoke (EStatic Optional) empty) orElseGet (Args "
+                + "(ELambda (SReturn (ENumeric 1)))))))",
+                  "return Optional.empty().orElseGet(() -> {\n"
+                + "  return 1\n"
+                + "})");
+        assertToString(
+                  "(SSource (SReturn (PCallInvoke (PCallInvoke (EStatic Optional) empty) orElseGet (Args "
+                + "(ELambda (SReturn (ENumeric 1)))))))",
+                  "return Optional.empty().orElseGet(() -> 1)");
+        assertToString(
+                  "(SSource (SReturn (PCallInvoke (PCallInvoke (PCallInvoke (EListInit (ENumeric 1) (ENumeric 2) (ENumeric 3)) stream) "
+                + "mapToInt (Args (ELambda (Pair def x)\n"
+                + "  (SReturn (EBinary (EVariable x) + (ENumeric 1)))))) sum)))",
+                  "return [1, 2, 3].stream().mapToInt((def x) -> {\n"
+                + "  return x + 1\n"
+                + "}).sum()");
+        assertToString(
+                  "(SSource (SReturn (PCallInvoke (PCallInvoke (PCallInvoke (EListInit (ENumeric 1) (ENumeric 2) (ENumeric 3)) stream) "
+                + "mapToInt (Args (ELambda (Pair null x)\n"
+                + "  (SReturn (EBinary (EVariable x) + (ENumeric 1)))))) sum)))",
+                  "return [1, 2, 3].stream().mapToInt(x -> x + 1).sum()");
+        assertToString(
+                  "(SSource (SReturn (PCallInvoke (EListInit (EString 'a') (EString 'b')) sort (Args (ELambda (Pair def a) (Pair def b)\n"
+                + "  (SReturn (EBinary (PCallInvoke (EVariable a) length) - (PCallInvoke (EVariable b) length))))))))",
+                  "return ['a', 'b'].sort((def a, def b) -> {\n"
+                + "  return a.length() - b.length()\n"
+                + "})");
+        assertToString(
+                  "(SSource (SReturn (PCallInvoke (EListInit (EString 'a') (EString 'b')) sort (Args (ELambda (Pair null a) (Pair null b)\n"
+                + "  (SReturn (EBinary (PCallInvoke (EVariable a) length) - (PCallInvoke (EVariable b) length))))))))",
+                  "return ['a', 'b'].sort((a, b) -> a.length() - b.length())");
+        assertToString(
+                "(SSource (SReturn (PCallInvoke (EListInit (EString 'a') (EString 'b')) sort (Args (ELambda (Pair def a) (Pair def b)\n"
+              + "  (SIf (EComp (EVariable a) < (EVariable b)) (SBlock "
+                  + "(SReturn (EBinary (PCallInvoke (EVariable a) length) - (PCallInvoke (EVariable b) length)))))\n"
+              + "  (SReturn (ENumeric 1)))))))",
+                "return ['a', 'b'].sort((def a, def b) -> {\n"
+              + "  if (a < b) {\n"
+              + "    return a.length() - b.length()\n"
+              + "  }\n"
+              + "  return 1\n"
+              + "})");
+    }
+
+    public void testEListInit() {
+        assertToString("(SSource (SReturn (EListInit (ENumeric 1) (ENumeric 2) (EString 'cat') (EString 'dog') (ENewObj Object))))",
+                "return [1, 2, 'cat', 'dog', new Object()]");
+        assertToString("(SSource (SReturn (EListInit)))", "return []");
+    }
+
+    public void testEMapInit() {
+        assertToString("(SSource (SReturn (EMapInit "
+                    + "(Pair (EString 'a') (ENumeric 1)) "
+                    + "(Pair (EString 'b') (ENumeric 3)) "
+                    + "(Pair (ENumeric 12) (ENewObj Object)))))",
+                "return ['a': 1, 'b': 3, 12: new Object()]");
+        assertToString("(SSource (SReturn (EMapInit)))", "return [:]");
+    }
+
+    public void testENewArray() {
+        assertToString("(SSource (SReturn (ENewArray int dims (Args (ENumeric 10)))))", "return new int[10]");
+        assertToString("(SSource (SReturn (ENewArray int dims (Args (ENumeric 10) (ENumeric 4) (ENumeric 5)))))",
+                "return new int[10][4][5]");
+        assertToString("(SSource (SReturn (ENewArray int init (Args (ENumeric 1) (ENumeric 2) (ENumeric 3)))))",
+                "return new int[] {1, 2, 3}");
+        assertToString("(SSource (SReturn (ENewArray def init (Args (ENumeric 1) (ENumeric 2) (EString 'bird')))))",
+                "return new def[] {1, 2, 'bird'}");
+    }
+
+    public void testENewObj() {
+        assertToString("(SSource (SReturn (ENewObj Object)))", "return new Object()");
+        assertToString("(SSource (SReturn (ENewObj DateTimeException (Args (EString 'test')))))", "return new DateTimeException('test')");
+    }
+
+    public void testENull() {
+        assertToString("(SSource (SReturn (ENull)))", "return null");
+    }
+
+    public void testENumeric() {
+        assertToString("(SSource (SReturn (ENumeric 1)))", "return 1");
+        assertToString("(SSource (SReturn (ENumeric 114121d)))", "return 114121d");
+        assertToString("(SSource (SReturn (ENumeric 114134f)))", "return 114134f");
+        assertToString("(SSource (SReturn (ENumeric 114121D)))", "return 114121D");
+        assertToString("(SSource (SReturn (ENumeric 111234F)))", "return 111234F");
+        assertToString("(SSource (SReturn (ENumeric 774121l)))", "return 774121l");
+        assertToString("(SSource (SReturn (ENumeric 881234L)))", "return 881234L");
+
+        assertToString("(SSource (SReturn (ENumeric 1 16)))", "return 0x1");
+        assertToString("(SSource (SReturn (ENumeric 774121l 16)))", "return 0x774121l");
+        assertToString("(SSource (SReturn (ENumeric 881234L 16)))", "return 0x881234L");
+
+        assertToString("(SSource (SReturn (ENumeric 1 8)))", "return 01");
+        assertToString("(SSource (SReturn (ENumeric 774121l 8)))", "return 0774121l");
+        assertToString("(SSource (SReturn (ENumeric 441234L 8)))", "return 0441234L");
+    }
+
+    public void testERegex() {
+        assertToString("(SSource (SReturn (ERegex /foo/)))", "return /foo/");
+        assertToString("(SSource (SReturn (ERegex /foo/ cix)))", "return /foo/cix");
+        assertToString("(SSource (SReturn (ERegex /foo/ cix)))", "return /foo/xci");
+    }
+
+    public void testEStatic() {
+        assertToString("(SSource (SReturn (PCallInvoke (EStatic Optional) empty)))", "return Optional.empty()");
+    }
+
+    public void testEString() {
+        assertToString("(SSource (SReturn (EString 'foo')))", "return 'foo'");
+        assertToString("(SSource (SReturn (EString ' oo')))", "return ' oo'");
+        assertToString("(SSource (SReturn (EString 'fo ')))", "return 'fo '");
+        assertToString("(SSource (SReturn (EString ' o ')))", "return ' o '");
+    }
+
+    public void testEUnary() {
+        assertToString("(SSource (SReturn (EUnary ! (EBoolean true))))", "return !true");
+        assertToString("(SSource (SReturn (EUnary ~ (ENumeric 1))))", "return ~1");
+        assertToString("(SSource (SReturn (EUnary + (ENumeric 1))))", "return +1");
+        assertToString("(SSource (SReturn (EUnary - (ENumeric 1))))", "return -(1)");
+    }
+
+    public void testEVariable() {
+        assertToString("(SSource (SReturn (EVariable params)))", "return params");
+        assertToString(
+                  "(SSource\n"
+                + "  (SDeclBlock (SDeclaration def a (ENumeric 1)))\n"
+                + "  (SReturn (EVariable a)))",
+                  "def a = 1;\n"
+                + "return a");
+    }
+
+    public void testPBrace() {
+        assertToString("(SSource (SReturn (PBrace (PField (EVariable params) a) (ENumeric 10))))", "return params.a[10]");
+        assertToString("(SSource (SReturn (PBrace (EVariable params) (EString 'a'))))", "return params['a']");
+    }
+
+    public void testPCallInvoke() {
+        assertToString("(SSource (SReturn (PCallInvoke (EStatic Optional) empty)))", "return Optional.empty()");
+        assertToString("(SSource (SReturn (PCallInvoke (EStatic Optional) of (Args (ENumeric 1)))))", "return Optional.of(1)");
+        assertToString("(SSource (SReturn (PCallInvoke (EStatic Objects) equals (Args (ENumeric 1) (ENumeric 2)))))",
+                "return Objects.equals(1, 2)");
+        assertToString("(SSource (SReturn (PCallInvoke (EVariable params) equals (Args (ENumeric 1)))))", "return params.equals(1)");
+    }
+
+    public void testPField() {
+        assertToString("(SSource (SReturn (PField (EVariable params) a)))", "return params.a");
+        assertToString("(SSource (SReturn (PField nullSafe (EVariable params) a)))", "return params?.a");
+        assertToString(
+                  "(SSource\n"
+                + "  (SDeclBlock (SDeclaration int[] a (ENewArray int dims (Args (ENumeric 10)))))\n"
+                + "  (SReturn (PField (EVariable a) length)))",
+                  "int[] a = new int[10];\n"
+                + "return a.length");
+        assertToString(
+                "(SSource\n"
+              + "  (SDeclBlock (SDeclaration org.elasticsearch.painless.FeatureTest a (ENewObj org.elasticsearch.painless.FeatureTest)))\n"
+              + "  (SExpression (EAssignment (PField (EVariable a) x) = (ENumeric 10)))\n"
+              + "  (SReturn (PField (EVariable a) x)))",
+                "org.elasticsearch.painless.FeatureTest a = new org.elasticsearch.painless.FeatureTest();\n"
+              + "a.x = 10;\n"
+              + "return a.x");
+    }
+
+    public void testPSubArrayLength() {
+        Location l = new Location(getTestName(), 0);
+        PSubArrayLength node = new PSubArrayLength(l, "int", "a");
+        node.prefix = new EVariable(l, "a");
+        assertEquals("(PSubArrayLength (EVariable a))", node.toString());
+        assertEquals("(PSubNullSafeField (PSubArrayLength (EVariable a)))", new PSubNullSafeField(l, node).toString());
+    }
+
+    public void testPSubBrace() {
+        Location l = new Location(getTestName(), 0);
+        PSubBrace node = new PSubBrace(l, Definition.INT_TYPE, new ENumeric(l, "1", 10));
+        node.prefix = new EVariable(l, "a");
+        assertEquals("(PSubBrace (EVariable a) (ENumeric 1))", node.toString());
+    }
+
+    public void testPSubCallInvoke() {
+        Location l = new Location(getTestName(), 0);
+        RuntimeClass c = definition.getRuntimeClass(Integer.class);
+        Method m = c.methods.get(new MethodKey("toString", 0));
+        PSubCallInvoke node = new PSubCallInvoke(l, m, null, emptyList());
+        node.prefix = new EVariable(l, "a");
+        assertEquals("(PSubCallInvoke (EVariable a) toString)", node.toString());
+        assertEquals("(PSubNullSafeCallInvoke (PSubCallInvoke (EVariable a) toString))", new PSubNullSafeCallInvoke(l, node).toString());
+
+        l = new Location(getTestName(), 1);
+        m = c.methods.get(new MethodKey("equals", 1));
+        node = new PSubCallInvoke(l, m, null, singletonList(new EVariable(l, "b")));
+        node.prefix = new EVariable(l, "a");
+        assertEquals("(PSubCallInvoke (EVariable a) equals (Args (EVariable b)))", node.toString());
+        assertEquals("(PSubNullSafeCallInvoke (PSubCallInvoke (EVariable a) equals (Args (EVariable b))))",
+                new PSubNullSafeCallInvoke(l, node).toString());
+    }
+
+    public void testPSubDefArray() {
+        Location l = new Location(getTestName(), 0);
+        PSubDefArray node = new PSubDefArray(l, new EConstant(l, 1));
+        node.prefix = new EVariable(l, "a");
+        assertEquals("(PSubDefArray (EVariable a) (EConstant Integer 1))", node.toString());
+    }
+
+    public void testPSubDefCall() {
+        Location l = new Location(getTestName(), 0);
+        PSubDefCall node = new PSubDefCall(l, "toString", emptyList());
+        node.prefix = new EVariable(l, "a");
+        assertEquals("(PSubDefCall (EVariable a) toString)", node.toString());
+        assertEquals("(PSubNullSafeCallInvoke (PSubDefCall (EVariable a) toString))", new PSubNullSafeCallInvoke(l, node).toString());
+
+        l = new Location(getTestName(), 0);
+        node = new PSubDefCall(l, "equals", singletonList(new EVariable(l, "b")));
+        node.prefix = new EVariable(l, "a");
+        assertEquals("(PSubDefCall (EVariable a) equals (Args (EVariable b)))", node.toString());
+        assertEquals("(PSubNullSafeCallInvoke (PSubDefCall (EVariable a) equals (Args (EVariable b))))",
+                new PSubNullSafeCallInvoke(l, node).toString());
+
+        l = new Location(getTestName(), 0);
+        node = new PSubDefCall(l, "superWeird", Arrays.asList(new EVariable(l, "b"), new EVariable(l, "c"), new EVariable(l, "d")));
+        node.prefix = new EVariable(l, "a");
+        assertEquals("(PSubDefCall (EVariable a) superWeird (Args (EVariable b) (EVariable c) (EVariable d)))", node.toString());
+        assertEquals("(PSubNullSafeCallInvoke (PSubDefCall (EVariable a) superWeird (Args (EVariable b) (EVariable c) (EVariable d))))",
+                new PSubNullSafeCallInvoke(l, node).toString());
+    }
+
+    public void testPSubDefField() {
+        Location l = new Location(getTestName(), 0);
+        PSubDefField node = new PSubDefField(l, "ok");
+        node.prefix = new EVariable(l, "a");
+        assertEquals("(PSubDefField (EVariable a) ok)", node.toString());
+        assertEquals("(PSubNullSafeCallInvoke (PSubDefField (EVariable a) ok))", new PSubNullSafeCallInvoke(l, node).toString());
+    }
+
+    public void testPSubField() {
+        Location l = new Location(getTestName(), 0);
+        Struct s = definition.getType(Boolean.class.getSimpleName()).struct;
+        Field f = s.staticMembers.get("TRUE");
+        PSubField node = new PSubField(l, f);
+        node.prefix = new EStatic(l, "Boolean");
+        assertEquals("(PSubField (EStatic Boolean) TRUE)", node.toString());
+        assertEquals("(PSubNullSafeCallInvoke (PSubField (EStatic Boolean) TRUE))", new PSubNullSafeCallInvoke(l, node).toString());
+    }
+
+    public void testPSubListShortcut() {
+        Location l = new Location(getTestName(), 0);
+        Struct s = definition.getType(List.class.getSimpleName()).struct;
+        PSubListShortcut node = new PSubListShortcut(l, s, new EConstant(l, 1));
+        node.prefix = new EVariable(l, "a");
+        assertEquals("(PSubListShortcut (EVariable a) (EConstant Integer 1))", node.toString());
+        assertEquals("(PSubNullSafeCallInvoke (PSubListShortcut (EVariable a) (EConstant Integer 1)))",
+                new PSubNullSafeCallInvoke(l, node).toString());
+
+        l = new Location(getTestName(), 0);
+        s = definition.getType(List.class.getSimpleName()).struct;
+        node = new PSubListShortcut(l, s, new EBinary(l, Operation.ADD, new EConstant(l, 1), new EConstant(l, 4)));
+        node.prefix = new EVariable(l, "a");
+        assertEquals("(PSubListShortcut (EVariable a) (EBinary (EConstant Integer 1) + (EConstant Integer 4)))", node.toString());
+    }
+
+    public void testPSubMapShortcut() {
+        Location l = new Location(getTestName(), 0);
+        Struct s = definition.getType(Map.class.getSimpleName()).struct;
+        PSubMapShortcut node = new PSubMapShortcut(l, s, new EConstant(l, "cat"));
+        node.prefix = new EVariable(l, "a");
+        assertEquals("(PSubMapShortcut (EVariable a) (EConstant String 'cat'))", node.toString());
+        assertEquals("(PSubNullSafeCallInvoke (PSubMapShortcut (EVariable a) (EConstant String 'cat')))",
+                new PSubNullSafeCallInvoke(l, node).toString());
+
+        l = new Location(getTestName(), 1);
+        s = definition.getType(Map.class.getSimpleName()).struct;
+        node = new PSubMapShortcut(l, s, new EBinary(l, Operation.ADD, new EConstant(l, 1), new EConstant(l, 4)));
+        node.prefix = new EVariable(l, "a");
+        assertEquals("(PSubMapShortcut (EVariable a) (EBinary (EConstant Integer 1) + (EConstant Integer 4)))", node.toString());
+    }
+
+    public void testPSubShortcut() {
+        Location l = new Location(getTestName(), 0);
+        Struct s = definition.getType(FeatureTest.class.getName()).struct;
+        Method getter = s.methods.get(new MethodKey("getX", 0));
+        Method setter = s.methods.get(new MethodKey("setX", 1));
+        PSubShortcut node = new PSubShortcut(l, "x", FeatureTest.class.getName(), getter, setter);
+        node.prefix = new EVariable(l, "a");
+        assertEquals("(PSubShortcut (EVariable a) x)", node.toString());
+        assertEquals("(PSubNullSafeCallInvoke (PSubShortcut (EVariable a) x))",
+                new PSubNullSafeCallInvoke(l, node).toString());
+    }
+
+    public void testSBreak() {
+        assertToString(
+                "(SSource\n"
+              + "  (SDeclBlock (SDeclaration int itr (ENumeric 2)))\n"
+              + "  (SDeclBlock (SDeclaration int a (ENumeric 1)))\n"
+              + "  (SDeclBlock (SDeclaration int b (ENumeric 1)))\n"
+              + "  (SDo (EComp (EVariable b) < (ENumeric 1000)) (SBlock\n"
+              + "    (SExpression (EAssignment (EVariable itr) ++ post))\n"
+              + "    (SIf (EComp (EVariable itr) > (ENumeric 10000)) (SBlock (SBreak)))\n"
+              + "    (SDeclBlock (SDeclaration int tmp (EVariable a)))\n"
+              + "    (SExpression (EAssignment (EVariable a) = (EVariable b)))\n"
+              + "    (SExpression (EAssignment (EVariable b) = (EBinary (EVariable tmp) + (EVariable b))))))\n"
+              + "  (SReturn (EVariable b)))",
+                "int itr = 2;\n"
+              + "int a = 1;\n"
+              + "int b = 1;\n"
+              + "do {\n"
+              + "  itr++;\n"
+              + "  if (itr > 10000) {\n"
+              + "    break\n"
+              + "  }\n"
+              + "  int tmp = a;\n"
+              + "  a = b;\n"
+              + "  b = tmp + b\n"
+              + "} while (b < 1000);\n"
+              + "return b");
+    }
+
+    public void testSContinue() {
+        assertToString(
+                "(SSource\n"
+              + "  (SDeclBlock (SDeclaration int itr (ENumeric 2)))\n"
+              + "  (SDeclBlock (SDeclaration int a (ENumeric 1)))\n"
+              + "  (SDeclBlock (SDeclaration int b (ENumeric 1)))\n"
+              + "  (SDo (EComp (EVariable b) < (ENumeric 1000)) (SBlock\n"
+              + "    (SExpression (EAssignment (EVariable itr) ++ post))\n"
+              + "    (SIf (EComp (EVariable itr) < (ENumeric 10000)) (SBlock (SContinue)))\n"
+              + "    (SDeclBlock (SDeclaration int tmp (EVariable a)))\n"
+              + "    (SExpression (EAssignment (EVariable a) = (EVariable b)))\n"
+              + "    (SExpression (EAssignment (EVariable b) = (EBinary (EVariable tmp) + (EVariable b))))))\n"
+              + "  (SReturn (EVariable b)))",
+                "int itr = 2;\n"
+              + "int a = 1;\n"
+              + "int b = 1;\n"
+              + "do {\n"
+              + "  itr++;\n"
+              + "  if (itr < 10000) {\n"
+              + "    continue\n"
+              + "  }\n"
+              + "  int tmp = a;\n"
+              + "  a = b;\n"
+              + "  b = tmp + b\n"
+              + "} while (b < 1000);\n"
+              + "return b");
+    }
+
+    public void testSDeclBlock() {
+        assertToString(
+                  "(SSource\n"
+                + "  (SDeclBlock (SDeclaration def a))\n"
+                + "  (SExpression (EAssignment (EVariable a) = (ENumeric 10)))\n"
+                + "  (SReturn (EVariable a)))",
+                  "def a;\n"
+                + "a = 10;\n"
+                + "return a");
+        assertToString(
+                "(SSource\n"
+              + "  (SDeclBlock (SDeclaration def a (ENumeric 10)))\n"
+              + "  (SReturn (EVariable a)))",
+                "def a = 10;\n"
+              + "return a");
+        assertToString(
+                "(SSource\n"
+              + "  (SDeclBlock\n"
+              + "    (SDeclaration def a)\n"
+              + "    (SDeclaration def b)\n"
+              + "    (SDeclaration def c))\n"
+              + "  (SReturn (EVariable a)))",
+                "def a, b, c;\n"
+              + "return a");
+        assertToString(
+                "(SSource\n"
+              + "  (SDeclBlock\n"
+              + "    (SDeclaration def a (ENumeric 10))\n"
+              + "    (SDeclaration def b (ENumeric 20))\n"
+              + "    (SDeclaration def c (ENumeric 100)))\n"
+              + "  (SReturn (EVariable a)))",
+                "def a = 10, b = 20, c = 100;\n"
+              + "return a");
+        assertToString(
+                "(SSource\n"
+              + "  (SDeclBlock\n"
+              + "    (SDeclaration def a (ENumeric 10))\n"
+              + "    (SDeclaration def b)\n"
+              + "    (SDeclaration def c (ENumeric 100)))\n"
+              + "  (SReturn (EVariable a)))",
+                "def a = 10, b, c = 100;\n"
+              + "return a");
+        assertToString(
+                "(SSource\n"
+              + "  (SIf (PField (EVariable params) a) (SBlock\n"
+              + "    (SDeclBlock\n"
+              + "      (SDeclaration def a (ENumeric 10))\n"
+              + "      (SDeclaration def b)\n"
+              + "      (SDeclaration def c (ENumeric 100)))\n"
+              + "    (SReturn (EVariable a))))\n"
+              + "  (SReturn (EBoolean false)))",
+                "if (params.a) {"
+              + "  def a = 10, b, c = 100;\n"
+              + "  return a\n"
+              + "}\n"
+              + "return false");
+    }
+
+    public void testSDo() {
+        assertToString(
+                  "(SSource\n"
+                + "  (SDeclBlock (SDeclaration int itr (ENumeric 2)))\n"
+                + "  (SDeclBlock (SDeclaration int a (ENumeric 1)))\n"
+                + "  (SDeclBlock (SDeclaration int b (ENumeric 1)))\n"
+                + "  (SDo (EComp (EVariable b) < (ENumeric 1000)) (SBlock\n"
+                + "    (SExpression (EAssignment (EVariable itr) ++ post))\n"
+                + "    (SDeclBlock (SDeclaration int tmp (EVariable a)))\n"
+                + "    (SExpression (EAssignment (EVariable a) = (EVariable b)))\n"
+                + "    (SExpression (EAssignment (EVariable b) = (EBinary (EVariable tmp) + (EVariable b))))))\n"
+                + "  (SReturn (EVariable b)))",
+                  "int itr = 2;\n"
+                + "int a = 1;\n"
+                + "int b = 1;\n"
+                + "do {\n"
+                + "  itr++;\n"
+                + "  int tmp = a;\n"
+                + "  a = b;\n"
+                + "  b = tmp + b\n"
+                + "} while (b < 1000);\n"
+                + "return b");
+    }
+
+    public void testSEach() {
+        assertToString(
+                "(SSource\n"
+              + "  (SDeclBlock (SDeclaration int l (ENumeric 0)))\n"
+              + "  (SEach String s (EListInit (EString 'cat') (EString 'dog') (EString 'chicken')) (SBlock "
+                  + "(SExpression (EAssignment (EVariable l) += (PCallInvoke (EVariable s) length)))))\n"
+              + "  (SReturn (EVariable l)))",
+                  "int l = 0;\n"
+                + "for (String s : ['cat', 'dog', 'chicken']) {\n"
+                + "  l += s.length()\n"
+                + "}\n"
+                + "return l");
+        assertToString(
+                "(SSource\n"
+              + "  (SDeclBlock (SDeclaration int l (ENumeric 0)))\n"
+              + "  (SEach String s (EListInit (EString 'cat') (EString 'dog') (EString 'chicken')) (SBlock\n"
+              + "    (SDeclBlock (SDeclaration String s2 (EBinary (EString 'dire ') + (EVariable s))))\n"
+              + "    (SExpression (EAssignment (EVariable l) += (PCallInvoke (EVariable s2) length)))))\n"
+              + "  (SReturn (EVariable l)))",
+                "int l = 0;\n"
+              + "for (String s : ['cat', 'dog', 'chicken']) {\n"
+              + "  String s2 = 'dire ' + s;\n"
+              + "  l += s2.length()\n"
+              + "}\n"
+              + "return l");
+    }
+
+    public void testSFor() {
+        assertToString(
+                "(SSource\n"
+              + "  (SDeclBlock (SDeclaration int sum (ENumeric 0)))\n"
+              + "  (SFor\n"
+              + "    (SDeclBlock (SDeclaration int i (ENumeric 0)))\n"
+              + "    (EComp (EVariable i) < (ENumeric 1000))\n"
+              + "    (EAssignment (EVariable i) ++ post)\n"
+              + "    (SBlock (SExpression (EAssignment (EVariable sum) += (EVariable i)))))\n"
+              + "  (SReturn (EVariable sum)))",
+                  "int sum = 0;\n"
+                + "for (int i = 0; i < 1000; i++) {\n"
+                + "  sum += i\n"
+                + "}\n"
+                + "return sum");
+        assertToString(
+                "(SSource\n"
+              + "  (SDeclBlock (SDeclaration int sum (ENumeric 0)))\n"
+              + "  (SFor\n"
+              + "    (SDeclBlock (SDeclaration int i (ENumeric 0)))\n"
+              + "    (EComp (EVariable i) < (ENumeric 1000))\n"
+              + "    (EAssignment (EVariable i) ++ post)\n"
+              + "    (SBlock (SFor\n"
+              + "      (SDeclBlock (SDeclaration int j (ENumeric 0)))\n"
+              + "      (EComp (EVariable j) < (ENumeric 1000))\n"
+              + "      (EAssignment (EVariable j) ++ post)\n"
+              + "      (SBlock (SExpression (EAssignment (EVariable sum) += (EBinary (EVariable i) * (EVariable j))))))))\n"
+              + "  (SReturn (EVariable sum)))",
+                "int sum = 0;\n"
+              + "for (int i = 0; i < 1000; i++) {\n"
+              + "  for (int j = 0; j < 1000; j++) {\n"
+              + "    sum += i * j\n"
+              + "  }\n"
+              + "}\n"
+              + "return sum");
+    }
+
+    public void testSIf() {
+        assertToString(
+                "(SSource (SIf (PField (EVariable param) a) (SBlock (SReturn (EBoolean true)))))",
+                  "if (param.a) {\n"
+                + "  return true\n"
+                +"}");
+        assertToString(
+                "(SSource (SIf (PField (EVariable param) a) (SBlock\n"
+              + "  (SIf (PField (EVariable param) b) (SBlock (SReturn (EBoolean true))))\n"
+              + "  (SReturn (EBoolean false)))))",
+                "if (param.a) {\n"
+              + "  if (param.b) {\n"
+              + "    return true\n"
+              + "  }\n"
+              + "  return false\n"
+              +"}");
+    }
+
+    public void testSIfElse() {
+        assertToString(
+                  "(SSource (SIfElse (PField (EVariable param) a)\n"
+                + "  (SBlock (SReturn (EBoolean true)))\n"
+                + "  (SBlock (SReturn (EBoolean false)))))",
+                  "if (param.a) {\n"
+                + "  return true\n"
+                + "} else {\n"
+                + "  return false\n"
+                + "}");
+        assertToString(
+                "(SSource\n"
+              + "  (SDeclBlock (SDeclaration int i (ENumeric 0)))\n"
+              + "  (SIfElse (PField (EVariable param) a)\n"
+              + "    (SBlock (SIfElse (PField (EVariable param) b)\n"
+              + "      (SBlock (SReturn (EBoolean true)))\n"
+              + "      (SBlock (SReturn (EString 'cat')))))\n"
+              + "    (SBlock (SReturn (EBoolean false)))))",
+                "int i = 0;\n"
+              + "if (param.a) {\n"
+              + "  if (param.b) {\n"
+              + "    return true\n"
+              + "  } else {\n"
+              + "    return 'cat'\n"
+              + "  }\n"
+              + "} else {"
+              + "  return false\n"
+              + "}");
+    }
+
+    public void testSSubEachArray() {
+        Location l = new Location(getTestName(), 0);
+        Variable v = new Variable(l, "test", Definition.INT_TYPE, 5, false);
+        AExpression e = new ENewArray(l, "int", Arrays.asList(new EConstant(l, 1), new EConstant(l, 2), new EConstant(l, 3)), true);
+        SBlock b = new SBlock(l, singletonList(new SReturn(l, new EConstant(l, 5))));
+        SSubEachArray node = new SSubEachArray(l, v, e, b);
+        assertEquals(
+                "(SSubEachArray int test (ENewArray int init (Args (EConstant Integer 1) (EConstant Integer 2) (EConstant Integer 3))) "
+              + "(SBlock (SReturn (EConstant Integer 5))))",
+                node.toString());
+    }
+
+    public void testSSubEachIterable() {
+        Location l = new Location(getTestName(), 0);
+        Variable v = new Variable(l, "test", Definition.INT_TYPE, 5, false);
+        AExpression e = new EListInit(l, Arrays.asList(new EConstant(l, 1), new EConstant(l, 2), new EConstant(l, 3)));
+        SBlock b = new SBlock(l, singletonList(new SReturn(l, new EConstant(l, 5))));
+        SSubEachIterable node = new SSubEachIterable(l, v, e, b);
+        assertEquals(
+                  "(SSubEachIterable int test (EListInit (EConstant Integer 1) (EConstant Integer 2) (EConstant Integer 3)) (SBlock "
+                + "(SReturn (EConstant Integer 5))))",
+                  node.toString());
+    }
+
+    public void testSThrow() {
+        assertToString("(SSource (SThrow (ENewObj RuntimeException)))", "throw new RuntimeException()");
+    }
+
+    public void testSWhile() {
+        assertToString(
+                  "(SSource\n"
+                + "  (SDeclBlock (SDeclaration int i (ENumeric 0)))\n"
+                + "  (SWhile (EComp (EVariable i) < (ENumeric 10)) (SBlock (SExpression (EAssignment (EVariable i) ++ post))))\n"
+                + "  (SReturn (EVariable i)))",
+                  "int i = 0;\n"
+                + "while (i < 10) {\n"
+                + "  i++\n"
+                + "}\n"
+                + "return i");
+    }
+
+    public void testSFunction() {
+        assertToString(
+                  "(SSource\n"
+                + "  (SFunction def a\n"
+                + "    (SReturn (EBoolean true)))\n"
+                + "  (SReturn (EBoolean true)))",
+                  "def a() {\n"
+                + "  return true\n"
+                + "}\n"
+                + "return true");
+        assertToString(
+                "(SSource\n"
+              + "  (SFunction def a (Args (Pair int i) (Pair int j))\n"
+              + "    (SReturn (EBoolean true)))\n"
+              + "  (SReturn (EBoolean true)))",
+                "def a(int i, int j) {\n"
+              + "  return true\n"
+              + "}\n"
+              + "return true");
+        assertToString(
+                "(SSource\n"
+              + "  (SFunction def a (Args (Pair int i) (Pair int j))\n"
+              + "    (SIf (EComp (EVariable i) < (EVariable j)) (SBlock (SReturn (EBoolean true))))\n"
+              + "    (SDeclBlock (SDeclaration int k (EBinary (EVariable i) + (EVariable j))))\n"
+              + "    (SReturn (EVariable k)))\n"
+              + "  (SReturn (EBoolean true)))",
+                "def a(int i, int j) {\n"
+              + "  if (i < j) {\n"
+              + "    return true\n"
+              + "  }\n"
+              + "  int k = i + j;\n"
+              + "  return k\n"
+              + "}\n"
+              + "return true");
+        assertToString(
+                "(SSource\n"
+              + "  (SFunction def a\n"
+              + "    (SReturn (EBoolean true)))\n"
+              + "  (SFunction def b\n"
+              + "    (SReturn (EBoolean false)))\n"
+              + "  (SReturn (EBoolean true)))",
+                "def a() {\n"
+              + "  return true\n"
+              + "}\n"
+              + "def b() {\n"
+              + "  return false\n"
+              + "}\n"
+              + "return true");
+    }
+
+    public void testSTryAndSCatch() {
+        assertToString(
+                  "(SSource (STry (SBlock (SReturn (ENumeric 1)))\n"
+                + "  (SCatch Exception e (SBlock (SReturn (ENumeric 2))))))",
+                  "try {\n"
+                + "  return 1\n"
+                + "} catch (Exception e) {\n"
+                + "  return 2\n"
+                + "}");
+        assertToString(
+                "(SSource (STry (SBlock\n"
+              + "  (SDeclBlock (SDeclaration int i (ENumeric 1)))\n"
+              + "  (SReturn (ENumeric 1)))\n"
+              + "  (SCatch Exception e (SBlock (SReturn (ENumeric 2))))))",
+                "try {\n"
+              + "  int i = 1;"
+              + "  return 1\n"
+              + "} catch (Exception e) {\n"
+              + "  return 2\n"
+              + "}");
+        assertToString(
+                "(SSource (STry (SBlock (SReturn (ENumeric 1)))\n"
+              + "  (SCatch Exception e (SBlock\n"
+              + "    (SDeclBlock (SDeclaration int i (ENumeric 1)))\n"
+              + "    (SReturn (ENumeric 2))))))",
+                "try {\n"
+              + "  return 1\n"
+              + "} catch (Exception e) {"
+              + "  int i = 1;\n"
+              + "  return 2\n"
+              + "}");
+        assertToString(
+                "(SSource (STry (SBlock (SReturn (ENumeric 1)))\n"
+              + "  (SCatch NullPointerException e (SBlock (SReturn (ENumeric 2))))\n"
+              + "  (SCatch Exception e (SBlock (SReturn (ENumeric 3))))))",
+                "try {\n"
+              + "  return 1\n"
+              + "} catch (NullPointerException e) {\n"
+              + "  return 2\n"
+              + "} catch (Exception e) {\n"
+              + "  return 3\n"
+              + "}");
+    }
+
+    private void assertToString(String expected, String code) {
+        assertEquals(expected, walk(code).toString());
+    }
+
+    private SSource walk(String code) {
+        ScriptInterface scriptInterface = new ScriptInterface(definition, GenericElasticsearchScript.class);
+        CompilerSettings compilerSettings = new CompilerSettings();
+        compilerSettings.setRegexesEnabled(true);
+        try {
+            return Walker.buildPainlessTree(scriptInterface, getTestName(), code, compilerSettings,
+                    definition, null);
+        } catch (Exception e) {
+            throw new AssertionError("Failed to compile: " + code, e);
+        }
+    }
+}
diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/10_basic.yaml b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/10_basic.yaml
similarity index 100%
rename from modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/10_basic.yaml
rename to modules/lang-painless/src/test/resources/rest-api-spec/test/painless/10_basic.yaml
diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/15_update.yaml b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/15_update.yaml
new file mode 100644
index 0000000000000..a64ad904c4963
--- /dev/null
+++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/15_update.yaml
@@ -0,0 +1,140 @@
+---
+"Update Script":
+
+  - do:
+      index:
+          index:  test_1
+          type:   test
+          id:     1
+          body:
+              foo:    bar
+              count:  1
+
+  - do:
+      update:
+          index:  test_1
+          type:   test
+          id:     1
+          body:
+            script:
+              lang:   painless
+              source: "ctx._source.foo = params.bar"
+              params: { bar: 'xxx' }
+
+  - match: { _index:   test_1 }
+  - match: { _type:    test   }
+  - match: { _id:      "1"    }
+  - match: { _version: 2      }
+
+  - do:
+      get:
+          index:  test_1
+          type:   test
+          id:     1
+
+  - match: { _source.foo:        xxx }
+  - match: { _source.count:      1   }
+
+  - do:
+      update:
+          index:  test_1
+          type:   test
+          id:     1
+          body:
+            script:
+              lang:   painless
+              source: "ctx._source.foo = 'yyy'"
+
+  - match: { _index:   test_1 }
+  - match: { _type:    test   }
+  - match: { _id:      "1"    }
+  - match: { _version: 3      }
+
+  - do:
+      get:
+          index:  test_1
+          type:   test
+          id:     1
+
+  - match: { _source.foo:        yyy }
+  - match: { _source.count:      1   }
+
+  - do:
+      update:
+          index:  test_1
+          type:   test
+          id:     1
+          body:
+            script:
+              lang:   painless
+              source: "ctx._source.missing_length = ctx._source.missing?.length()"
+
+  - match: { _index:   test_1 }
+  - match: { _type:    test   }
+  - match: { _id:      "1"    }
+  - match: { _version: 4      }
+
+  - do:
+      get:
+          index:  test_1
+          type:   test
+          id:     1
+
+  - match: { _source.foo:        yyy }
+  - match: { _source.count:      1   }
+  - is_false: _source.missing
+  - is_false: _source.missing_length
+
+  - do:
+      update:
+          index:  test_1
+          type:   test
+          id:     1
+          body:
+            script:
+              lang:   painless
+              source: "ctx._source.foo_length = ctx._source.foo?.length()"
+
+  - match: { _index:   test_1 }
+  - match: { _type:    test   }
+  - match: { _id:      "1"    }
+  - match: { _version: 5      }
+
+  - do:
+      get:
+          index:  test_1
+          type:   test
+          id:     1
+
+  - match: { _source.foo:        yyy }
+  - match: { _source.foo_length: 3 }
+  - match: { _source.count:      1   }
+  - is_false: _source.missing
+  - is_false: _source.missing_length
+
+---
+"Update Script with script error":
+  - do:
+      index:
+          index:  test_1
+          type:   test
+          id:     2
+          body:
+              foo:    bar
+              count:  1
+
+  - do:
+      catch: request
+      update:
+          index:  test_1
+          type:   test
+          id:     2
+          body:
+            script:
+              lang:   painless
+              source: "for (def key : params.keySet()) { ctx._source[key] = params[key]}"
+              params: { bar: 'xxx' }
+
+  - match: { error.root_cause.0.type: "remote_transport_exception" }
+  - match: { error.type: "illegal_argument_exception" }
+  - match: { error.reason: "Object has already been built and is self-referencing itself" }
diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/16_update2.yaml b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/16_update2.yaml
new file mode 100644
index 0000000000000..7041fed2c417c
--- /dev/null
+++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/16_update2.yaml
@@ -0,0 +1,47 @@
+---
+"Stored script":
+  - skip:
+      features: warnings
+
+  - do:
+      put_script:
+        lang: "1"
+        body: { "script": {"lang": "painless", "source": "_score * doc['myParent.weight'].value" } }
+  - match: { acknowledged: true }
+
+  - do:
+     get_script:
+       lang: "1"
+  - match: { found: true }
+  - match: { _id: "1" }
+  - match: { "script": {"lang": "painless", "source": "_score * doc['myParent.weight'].value"} }
+
+  - do:
+     catch: missing
+     get_script:
+       lang: "2"
+  - match: { found: false }
+  - match: { _id: "2" }
+  - is_false: script
+
+  - do:
+     delete_script:
+       lang: "1"
+  - match: { acknowledged: true }
+
+  - do:
+     catch: missing
+     delete_script:
+       lang: "non_existing"
+
+  - do:
+      catch: request
+      put_script:
+        lang: "1"
+        body: { "script": {"lang": "painless", "source": "_score * foo bar + doc['myParent.weight'].value"} }
+
+  - do:
+      catch: /compile error/
+      put_script:
+        lang: "1"
+        body: { "script": {"lang": "painless", "source": "_score * foo bar + doc['myParent.weight'].value"} }
diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/20_scriptfield.yaml b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/20_scriptfield.yaml
new file mode 100644
index 0000000000000..b6296ac4e1b03
--- /dev/null
+++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/20_scriptfield.yaml
@@ -0,0 +1,152 @@
+# Integration tests for using a scripted field
+#
+setup:
+    - do:
+        indices.create:
+          index: test
+          body:
+            mappings:
+              test:
+                properties:
+                  foo:
+                    type: keyword
+                  missing:
+                    type: keyword
+                  date:
+                    type: date
+                    format: yyyy/MM/dd
+                  dates:
+                    type: date
+                    format: yyyy/MM/dd
+    - do:
+        index:
+            index: test
+            type: test
+            id: 1
+            body: {
+              "foo": "aaa",
+              "date": "2017/01/01",
+              "dates": ["2017/01/01", "2017/02/01", "2017/03/01"]
+            }
+    - do:
+        indices.refresh: {}
+
+---
+"Scripted Field":
+    - do:
+        search:
+            body:
+                script_fields:
+                    bar:
+                        script:
+                            source: "doc['foo'].value + params.x;"
+                            params:
+                                x: "bbb"
+
+    - match: { hits.hits.0.fields.bar.0: "aaabbb"}
+
+---
+"Scripted Field Doing Compare":
+    - do:
+        search:
+            body:
+                script_fields:
+                    bar:
+                        script:
+                            source: "boolean compare(Supplier s, def v) {return s.get() == v;}
+                                     compare(() -> { return doc['foo'].value }, params.x);"
+                            params:
+                                x: "aaa"
+
+    - match: { hits.hits.0.fields.bar.0: true}
+    - do:
+        search:
+            body:
+                script_fields:
+                    bar:
+                        script:
+                            source: "boolean compare(Supplier s, def v) {return s.get() == v;}
+                                     compare(() -> { return doc['foo'].value }, params.x);"
+                            params:
+                                x: "bbb"
+
+    - match: { hits.hits.0.fields.bar.0: false}
+
+---
+"Scripted Field with a null safe dereference (non-null)":
+    - do:
+        search:
+            body:
+                script_fields:
+                    bar:
+                        script:
+                            source: "(doc['foo'].value?.length() ?: 0) + params.x;"
+                            params:
+                                x: 5
+
+    - match: { hits.hits.0.fields.bar.0: 8}
+
+---
+"Scripted Field with a null safe dereference (null)":
+    # Change this to ?: once we have it implemented
+    - do:
+        search:
+            body:
+                script_fields:
+                    bar:
+                        script:
+                            source: "(doc['missing'].value?.length() ?: 0) + params.x;"
+                            params:
+                                x: 5
+
+    - match: { hits.hits.0.fields.bar.0: 5}
+
+---
+"Access a date":
+    - do:
+        search:
+            body:
+                script_fields:
+                    bar:
+                        script:
+                            source: "doc.date.date.dayOfWeek"
+
+    - match: { hits.hits.0.fields.bar.0: 7}
+
+---
+"Access many dates":
+    - do:
+        search:
+            body:
+                script_fields:
+                    bar:
+                        script:
+                            source: >
+                              StringBuilder b = new StringBuilder();
+                              for (def date : doc.dates.dates) {
+                                b.append(" ").append(date.getDayOfWeek());
+                              }
+                              return b.toString().trim()
+
+    - match: { hits.hits.0.fields.bar.0: "7 3 3"}
+
+---
+"Scripted Field with script error":
+    - do:
+        catch: request
+        search:
+          body:
+            script_fields:
+              bar:
+                script:
+                  source: "while (true) {}"
+
+    - match: { error.root_cause.0.type: "script_exception" }
+    - match: { error.root_cause.0.reason: "compile error" }
+    - match: { error.type: "search_phase_execution_exception" }
+    - match: { error.reason: "all shards failed" }
+    - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" }
+    - match: { error.failed_shards.0.reason.caused_by.reason: "While loop has no escape." }
+    - match: { error.failed_shards.0.reason.type: "script_exception" }
+    - match: { error.failed_shards.0.reason.reason: "compile error" }
+
diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/25_script_upsert.yaml b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/25_script_upsert.yaml
new file mode 100644
index 0000000000000..9622e6f4d3168
--- /dev/null
+++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/25_script_upsert.yaml
@@ -0,0 +1,86 @@
+---
+"Script upsert":
+
+  - do:
+      update:
+          index:    test_1
+          type:     test
+          id:       1
+          body:
+            script:
+              source: "ctx._source.foo = params.bar"
+              lang: "painless"
+              params: { bar: 'xxx' }
+            upsert: { foo: baz }
+
+  - do:
+      get:
+          index:  test_1
+          type:   test
+          id:     1
+
+  - match:  { _source.foo: baz }
+
+
+  - do:
+      update:
+          index:    test_1
+          type:     test
+          id:       1
+          body:
+            script:
+              source: "ctx._source.foo = params.bar"
+              lang: "painless"
+              params: { bar: 'xxx' }
+            upsert: { foo: baz }
+
+  - do:
+      get:
+          index:  test_1
+          type:   test
+          id:     1
+
+  - match:  { _source.foo: xxx }
+
+  - do:
+      update:
+          index:    test_1
+          type:     test
+          id:       2
+          body:
+            script:
+              source: "ctx._source.foo = params.bar"
+              lang: "painless"
+              params: { bar: 'xxx' }
+            upsert: { foo: baz }
+            scripted_upsert: true
+
+  - do:
+      get:
+          index:  test_1
+          type:   test
+          id:     2
+
+  - match:  { _source.foo: xxx }
+
+  - do:
+      update:
+          index:    test_1
+          type:     test
+          id:       3
+          body:
+            script:
+              source: "ctx._source.has_now = ctx._now > 0"
+              lang: "painless"
+            upsert: { has_now: false }
+            scripted_upsert: true
+
+  - do:
+      get:
+          index:  test_1
+          type:   test
+          id:     3
+
+  - match:  { _source.has_now: true }
+
+
diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/30_search.yaml b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/30_search.yaml
similarity index 85%
rename from modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/30_search.yaml
rename to modules/lang-painless/src/test/resources/rest-api-spec/test/painless/30_search.yaml
index f1051ba71066f..28679cb223fd1 100644
--- a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/30_search.yaml
+++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/30_search.yaml
@@ -6,19 +6,19 @@
             index: test
             type: test
             id: 1
-            body: { "test": "value beck", "num1": 1.0 }
+            body: { "test": "value beck", "num1": 1.0, "bool": true }
     - do:
         index:
             index: test
             type: test
             id: 2
-            body: { "test": "value beck", "num1": 2.0 }
+            body: { "test": "value beck", "num1": 2.0, "bool": false }
     - do:
         index:
             index: test
             type: test
             id: 3
-            body: { "test": "value beck", "num1": 3.0 }
+            body: { "test": "value beck", "num1": 3.0, "bool": true }
     - do:
         indices.refresh: {}
 
@@ -29,12 +29,12 @@
                 query:
                     script:
                         script:
-                            inline: "doc['num1'].value > 1;"
+                            source: "doc['num1'].value > 1;"
                             lang: painless
                 script_fields:
                     sNum1:
                         script: 
-                            inline: "doc['num1'].value;"
+                            source: "doc['num1'].value;"
                             lang: painless
                 sort:
                     num1:
@@ -51,7 +51,7 @@
                 query:
                     script:
                         script:
-                            inline: "doc['num1'].value > params.param1;"
+                            source: "doc['num1'].value > params.param1;"
                             lang: painless
                             params:
                                 param1: 1
@@ -59,7 +59,7 @@
                 script_fields:
                     sNum1:
                         script:
-                            inline: "return doc['num1'].value;"
+                            source: "return doc['num1'].value;"
                             lang: painless
                 sort:
                     num1:
@@ -76,7 +76,7 @@
                 query:
                     script:
                         script:
-                            inline: "doc['num1'].value > params.param1;"
+                            source: "doc['num1'].value > params.param1;"
                             lang: painless
                             params:
                                 param1: -1
@@ -84,7 +84,7 @@
                 script_fields:
                     sNum1:
                         script: 
-                            inline: "doc['num1'].value;"
+                            source: "doc['num1'].value;"
                             lang: painless
                 sort:
                     num1:
@@ -95,6 +95,19 @@
     - match: { hits.hits.1.fields.sNum1.0: 2.0 }
     - match: { hits.hits.2.fields.sNum1.0: 3.0 }
 
+    - do:
+        index: test
+        search:
+            body:
+                query:
+                    script:
+                        script:
+                            source: "doc['bool'].value == false"
+                            lang: painless
+
+    - match: { hits.total: 1 }
+    - match: { hits.hits.0._id: "2" }
+
 ---
 
 "Custom Script Boost":
@@ -126,7 +139,7 @@
                             "script_score": {
                                 "script": {
                                     "lang": "painless",
-                                    "inline": "doc['num1'].value"
+                                    "source": "doc['num1'].value"
                                 }
                             }
                         }]
@@ -148,7 +161,7 @@
                             "script_score": {
                                 "script": {
                                     "lang": "painless",
-                                    "inline": "-doc['num1'].value"
+                                    "source": "-doc['num1'].value"
                                 }
                             }
                         }]
@@ -170,7 +183,7 @@
                             "script_score": {
                                 "script": {
                                     "lang": "painless",
-                                    "inline": "Math.pow(doc['num1'].value, 2)"
+                                    "source": "Math.pow(doc['num1'].value, 2)"
                                 }
                             }
                         }]
@@ -192,7 +205,7 @@
                             "script_score": {
                                 "script": {
                                     "lang": "painless",
-                                    "inline": "Math.max(doc['num1'].value, 1)"
+                                    "source": "Math.max(doc['num1'].value, 1)"
                                 }
                             }
                         }]
@@ -214,7 +227,7 @@
                             "script_score": {
                                 "script": {
                                     "lang": "painless",
-                                    "inline": "doc['num1'].value * _score"
+                                    "source": "doc['num1'].value * _score"
                                 }
                             }
                         }]
@@ -236,7 +249,7 @@
                             "script_score": {
                                 "script": {
                                     "lang": "painless",
-                                    "inline": "params.param1 * params.param2 * _score",
+                                    "source": "params.param1 * params.param2 * _score",
                                     "params": {
                                         "param1": 2,
                                         "param2": 2
@@ -273,14 +286,14 @@
                                         "script_score": {
                                             "script": {
                                                 "lang": "painless",
-                                                "inline": "1"
+                                                "source": "1"
                                             }
                                         }
                                     }, {
                                         "script_score": {
                                             "script": {
                                                 "lang": "painless",
-                                                "inline": "_score"
+                                                "source": "_score"
                                             }
                                         }
                                     }
@@ -289,7 +302,7 @@
                             "script_score": {
                                 "script": {
                                     "lang": "painless",
-                                    "inline": "_score"
+                                    "source": "_score"
                                 }
                             }
                         }]
@@ -321,7 +334,7 @@
                             "script_score": {
                                 "script": {
                                     "lang": "painless",
-                                    "inline": "_score"
+                                    "source": "_score"
                                 }
                             }
                         }]
@@ -330,7 +343,7 @@
                         terms:
                             script:
                                 lang: painless
-                                inline: "_score"
+                                source: "_score"
 
     - match: { hits.total: 1 }
     - match: { hits.hits.0._score: 1.0 }
@@ -357,7 +370,7 @@
                 script_fields:
                     foobar:
                         script: 
-                            inline: "doc['f'].values.size()"
+                            source: "doc['f'].values.size()"
                             lang: painless
 
 
@@ -387,7 +400,7 @@
                             field: dummy_field
                             script:
                                 lang: painless
-                                inline: "_value + 1"
+                                source: "_value + 1"
 
     - match: { hits.total: 1 }
     - match: { hits.hits.0._score: 1.0 }
diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/40_disabled.yaml b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/40_disabled.yaml
new file mode 100644
index 0000000000000..bcf02f657b0aa
--- /dev/null
+++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/40_disabled.yaml
@@ -0,0 +1,33 @@
+---
+"Regex in update fails":
+
+  - do:
+      index:
+          index:  test_1
+          type:   test
+          id:     1
+          body:
+              foo:    bar
+              count:  1
+
+  - do:
+      catch: /Regexes are disabled. Set \[script.painless.regex.enabled\] to \[true\] in elasticsearch.yaml to allow them. Be careful though, regexes break out of Painless's protection against deep recursion and long loops./
+      update:
+          index:  test_1
+          type:   test
+          id:     1
+          body:
+            script:
+              lang:   painless
+              inline: "ctx._source.foo = params.bar ==~ /cat/"
+              params: { bar: 'xxx' }
+
+---
+"Regex enabled is not a dynamic setting":
+
+  - do:
+      catch: /setting \[script.painless.regex.enabled\], not dynamically updateable/
+      cluster.put_settings:
+        body:
+          transient:
+            script.painless.regex.enabled: true
diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/50_script_doc_values.yaml b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/50_script_doc_values.yaml
new file mode 100644
index 0000000000000..8c0dfeb184469
--- /dev/null
+++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/50_script_doc_values.yaml
@@ -0,0 +1,344 @@
+setup:
+    - do:
+        indices.create:
+            index: test
+            body:
+                mappings:
+                    test:
+                        properties:
+                            boolean:
+                                type: boolean
+                            date:
+                                type: date
+                            geo_point:
+                                type: geo_point
+                            ip:
+                                type: ip
+                            keyword:
+                                type: keyword
+                            long:
+                                type: long
+                            integer:
+                                type: integer
+                            short:
+                                type: short
+                            byte:
+                                type: byte
+                            double:
+                                type: double
+                            float:
+                                type: float
+                            half_float:
+                                type: half_float
+                            scaled_float:
+                                type: scaled_float
+                                scaling_factor: 100
+                            token_count:
+                                type: token_count
+                                analyzer: standard
+
+    - do:
+        index:
+            index: test
+            type: test
+            id: 1
+            body:
+                boolean: true
+                date: 2017-01-01T12:11:12
+                geo_point: 41.12,-71.34
+                ip: 192.168.0.1
+                keyword: not split at all
+                long: 12348732141234
+                integer: 134134566
+                short: 1324
+                byte: 12
+                double: 3.14159265358979
+                float: 3.141592654
+                half_float: 3.140625
+                scaled_float: 3.14
+                token_count: count all these words please
+
+    - do:
+        indices.refresh: {}
+
+---
+"boolean":
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            source: "doc['boolean'].get(0)"
+    - match: { hits.hits.0.fields.field.0: true }
+
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            source: "doc['boolean'].value"
+    - match: { hits.hits.0.fields.field.0: true }
+
+---
+"date":
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            source: "doc['date'].get(0)"
+    - match: { hits.hits.0.fields.field.0: 1483272672000 }
+
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            source: "doc['date'].value"
+    - match: { hits.hits.0.fields.field.0: 1483272672000 }
+
+---
+"geo_point":
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            source: "doc['geo_point'].get(0)"
+    - match: { hits.hits.0.fields.field.0.lat: 41.1199999647215 }
+    - match: { hits.hits.0.fields.field.0.lon: -71.34000004269183 }
+
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            source: "doc['geo_point'].value"
+    - match: { hits.hits.0.fields.field.0.lat: 41.1199999647215 }
+    - match: { hits.hits.0.fields.field.0.lon: -71.34000004269183 }
+
+---
+"ip":
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            source: "doc['ip'].get(0)"
+    - match: { hits.hits.0.fields.field.0: "192.168.0.1" }
+
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            source: "doc['ip'].value"
+    - match: { hits.hits.0.fields.field.0: "192.168.0.1" }
+
+---
+"keyword":
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            source: "doc['keyword'].get(0)"
+    - match: { hits.hits.0.fields.field.0: "not split at all" }
+
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            source: "doc['keyword'].value"
+    - match: { hits.hits.0.fields.field.0: "not split at all" }
+
+---
+"long":
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            source: "doc['long'].get(0)"
+    - match: { hits.hits.0.fields.field.0: 12348732141234 }
+
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            source: "doc['long'].value"
+    - match: { hits.hits.0.fields.field.0: 12348732141234 }
+
+---
+"integer":
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            source: "doc['integer'].get(0)"
+    - match: { hits.hits.0.fields.field.0: 134134566 }
+
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            source: "doc['integer'].value"
+    - match: { hits.hits.0.fields.field.0: 134134566 }
+
+---
+"short":
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            source: "doc['short'].get(0)"
+    - match: { hits.hits.0.fields.field.0: 1324 }
+
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            source: "doc['short'].value"
+    - match: { hits.hits.0.fields.field.0: 1324 }
+
+---
+"byte":
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            source: "doc['byte'].get(0)"
+    - match: { hits.hits.0.fields.field.0: 12 }
+
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            source: "doc['byte'].value"
+    - match: { hits.hits.0.fields.field.0: 12 }
+
+---
+"double":
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            source: "doc['double'].get(0)"
+    - match: { hits.hits.0.fields.field.0: 3.14159265358979 }
+
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            source: "doc['double'].value"
+    - match: { hits.hits.0.fields.field.0: 3.14159265358979 }
+
+---
+"float":
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            source: "doc['float'].get(0)"
+    - match: { hits.hits.0.fields.field.0: 3.1415927410125732 } # this ends up as a double
+
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            source: "doc['float'].value"
+    - match: { hits.hits.0.fields.field.0: 3.1415927410125732 } # this ends up as a double
+
+---
+"half_float":
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            source: "doc['half_float'].get(0)"
+    - match: { hits.hits.0.fields.field.0: 3.140625 }
+
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            source: "doc['half_float'].value"
+    - match: { hits.hits.0.fields.field.0: 3.140625 }
+
+---
+"scaled_float":
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            source: "doc['scaled_float'].get(0)"
+    - match: { hits.hits.0.fields.field.0: 3.14 }
+
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            source: "doc['scaled_float'].value"
+    - match: { hits.hits.0.fields.field.0: 3.14 }
+
+---
+"token_count":
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            source: "doc['token_count'].get(0)"
+    - match: { hits.hits.0.fields.field.0: 5 }
+
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            source: "doc['token_count'].value"
+    - match: { hits.hits.0.fields.field.0: 5 }
diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/60_script_doc_values_binary.yaml b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/60_script_doc_values_binary.yaml
new file mode 100644
index 0000000000000..413c84f56c46b
--- /dev/null
+++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/60_script_doc_values_binary.yaml
@@ -0,0 +1,47 @@
+---
+"binary":
+    - skip:
+        features: ["headers"]
+    - do:
+        indices.create:
+            index: test
+            body:
+                mappings:
+                    test:
+                        properties:
+                            binary:
+                                type: binary
+                                doc_values: true
+
+    - do:
+        #set the header so we won't randomize it
+        headers:
+            Content-Type: application/json
+        index:
+            index: test
+            type: test
+            id: 1
+            body:
+                binary: U29tZSBiaW5hcnkgYmxvYg==
+
+    - do:
+        indices.refresh: {}
+
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            source: "doc['binary'].get(0).utf8ToString()"
+    - match: { hits.hits.0.fields.field.0: "Some binary blob" }
+
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            source: "doc['binary'].value.utf8ToString()"
+    - match: { hits.hits.0.fields.field.0: "Some binary blob" }
+
diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/15_update.yaml b/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/15_update.yaml
deleted file mode 100644
index 8e7e3d787e278..0000000000000
--- a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/15_update.yaml
+++ /dev/null
@@ -1,61 +0,0 @@
----
-"Update Script":
-
-  - do:
-      index:
-          index:  test_1
-          type:   test
-          id:     1
-          body:
-              foo:    bar
-              count:  1
-
-  - do:
-      update:
-          index:  test_1
-          type:   test
-          id:     1
-          body:
-            script:
-              lang:   painless
-              inline: "ctx._source.foo = params.bar"
-              params: { bar: 'xxx' }
-
-  - match: { _index:   test_1 }
-  - match: { _type:    test   }
-  - match: { _id:      "1"    }
-  - match: { _version: 2      }
-
-  - do:
-      get:
-          index:  test_1
-          type:   test
-          id:     1
-
-  - match: { _source.foo:        xxx }
-  - match: { _source.count:      1   }
-
-  - do:
-      update:
-          index:  test_1
-          type:   test
-          id:     1
-          body:
-            script:
-              lang:   painless
-              inline: "ctx._source.foo = 'yyy'"
-
-  - match: { _index:   test_1 }
-  - match: { _type:    test   }
-  - match: { _id:      "1"    }
-  - match: { _version: 3      }
-
-  - do:
-      get:
-          index:  test_1
-          type:   test
-          id:     1
-
-  - match: { _source.foo:        yyy }
-  - match: { _source.count:      1   }
-  
diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/16_update2.yaml b/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/16_update2.yaml
deleted file mode 100644
index 524f203b2fc3e..0000000000000
--- a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/16_update2.yaml
+++ /dev/null
@@ -1,54 +0,0 @@
----
-"Indexed script":
-
-  - do:
-      put_script:
-        id: "1"
-        lang: "painless"
-        body: { "script":  "_score * doc['myParent.weight'].value" }
-  - match: { acknowledged: true }
-
-  - do:
-     get_script:
-       id: "1"
-       lang: "painless"
-  - match: { found: true }
-  - match: { lang: painless }
-  - match: { _id: "1" }
-  - match: { "script":  "_score * doc['myParent.weight'].value" }
-
-  - do:
-     catch: missing
-     get_script:
-       id: "2"
-       lang: "painless"
-  - match: { found: false }
-  - match: { lang: painless }
-  - match: { _id: "2" }
-  - is_false: script
-
-  - do:
-     delete_script:
-       id: "1"
-       lang: "painless"
-  - match: { acknowledged: true }
-
-  - do:
-     catch: missing
-     delete_script:
-       id: "non_existing"
-       lang: "painless"
-
-  - do:
-      catch: request
-      put_script:
-        id: "1"
-        lang: "painless"
-        body: { "script":  "_score * foo bar + doc['myParent.weight'].value" }
-
-  - do:
-      catch: /compile error/
-      put_script:
-        id: "1"
-        lang: "painless"
-        body: { "script":  "_score * foo bar + doc['myParent.weight'].value" }
diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/20_scriptfield.yaml b/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/20_scriptfield.yaml
deleted file mode 100644
index b92012959d18f..0000000000000
--- a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/20_scriptfield.yaml
+++ /dev/null
@@ -1,36 +0,0 @@
-# Integration tests for using a scripted field
-#
-setup:
-    - do:
-      indices.create:
-        index: test
-        body:
-          mappings:
-            test:
-              properties:
-                foo:
-                  type: keyword
-    - do:
-        index:
-            index: test
-            type: test
-            id: 1
-            body: { "foo": "aaa" }
-    - do:
-        indices.refresh: {}
-
----
-
-"Scripted Field":
-    - do:
-        search:
-            body:
-                script_fields:
-                    bar:
-                        script: 
-                            inline: "doc['foo'].value + params.x;"
-                            lang: painless
-                            params:
-                                x: "bbb"
-
-    - match: { hits.hits.0.fields.bar.0: "aaabbb"}
diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/25_script_upsert.yaml b/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/25_script_upsert.yaml
deleted file mode 100644
index 2adf0de747ffc..0000000000000
--- a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/25_script_upsert.yaml
+++ /dev/null
@@ -1,66 +0,0 @@
----
-"Script upsert":
-
-  - do:
-      update:
-          index:    test_1
-          type:     test
-          id:       1
-          body:
-            script:
-              inline: "ctx._source.foo = params.bar"
-              lang: "painless"
-              params: { bar: 'xxx' }
-            upsert: { foo: baz }
-
-  - do:
-      get:
-          index:  test_1
-          type:   test
-          id:     1
-
-  - match:  { _source.foo: baz }
-
-
-  - do:
-      update:
-          index:    test_1
-          type:     test
-          id:       1
-          body:
-            script:
-              inline: "ctx._source.foo = params.bar"
-              lang: "painless"
-              params: { bar: 'xxx' }
-            upsert: { foo: baz }
-
-  - do:
-      get:
-          index:  test_1
-          type:   test
-          id:     1
-
-  - match:  { _source.foo: xxx }
-
-  - do:
-      update:
-          index:    test_1
-          type:     test
-          id:       2
-          body:
-            script:
-              inline: "ctx._source.foo = params.bar"
-              lang: "painless"
-              params: { bar: 'xxx' }
-            upsert: { foo: baz }
-            scripted_upsert: true
-
-  - do:
-      get:
-          index:  test_1
-          type:   test
-          id:     2
-
-  - match:  { _source.foo: xxx }
-
-
diff --git a/modules/parent-join/build.gradle b/modules/parent-join/build.gradle
new file mode 100644
index 0000000000000..67bcc9d54e8e7
--- /dev/null
+++ b/modules/parent-join/build.gradle
@@ -0,0 +1,24 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+esplugin {
+  description 'This module adds the support parent-child queries and aggregations'
+  classname 'org.elasticsearch.join.ParentJoinPlugin'
+  hasClientJar = true
+}
diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/ParentJoinPlugin.java b/modules/parent-join/src/main/java/org/elasticsearch/join/ParentJoinPlugin.java
new file mode 100644
index 0000000000000..2236662b5d54b
--- /dev/null
+++ b/modules/parent-join/src/main/java/org/elasticsearch/join/ParentJoinPlugin.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.join;
+
+import org.elasticsearch.index.mapper.Mapper;
+import org.elasticsearch.join.aggregations.ChildrenAggregationBuilder;
+import org.elasticsearch.join.aggregations.InternalChildren;
+import org.elasticsearch.join.mapper.ParentJoinFieldMapper;
+import org.elasticsearch.join.query.HasChildQueryBuilder;
+import org.elasticsearch.join.query.HasParentQueryBuilder;
+import org.elasticsearch.join.query.ParentIdQueryBuilder;
+import org.elasticsearch.plugins.MapperPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.SearchPlugin;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+public class ParentJoinPlugin extends Plugin implements SearchPlugin, MapperPlugin {
+
+    public ParentJoinPlugin() {
+    }
+
+    @Override
+    public List<QuerySpec<?>> getQueries() {
+        return Arrays.asList(
+            new QuerySpec<>(HasChildQueryBuilder.NAME, HasChildQueryBuilder::new, HasChildQueryBuilder::fromXContent),
+            new QuerySpec<>(HasParentQueryBuilder.NAME, HasParentQueryBuilder::new, HasParentQueryBuilder::fromXContent),
+            new QuerySpec<>(ParentIdQueryBuilder.NAME, ParentIdQueryBuilder::new, ParentIdQueryBuilder::fromXContent)
+        );
+    }
+
+    @Override
+    public List<AggregationSpec> getAggregations() {
+        return Collections.singletonList(
+          new AggregationSpec(ChildrenAggregationBuilder.NAME, ChildrenAggregationBuilder::new, ChildrenAggregationBuilder::parse)
+              .addResultReader(InternalChildren::new)
+        );
+    }
+
+    @Override
+    public Map<String, Mapper.TypeParser> getMappers() {
+        return Collections.singletonMap(ParentJoinFieldMapper.CONTENT_TYPE, new ParentJoinFieldMapper.TypeParser());
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/Children.java b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/Children.java
similarity index 94%
rename from core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/Children.java
rename to modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/Children.java
index b1e4b2877a3a0..394c690709d8a 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/Children.java
+++ b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/Children.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.search.aggregations.bucket.children;
+package org.elasticsearch.join.aggregations;
 
 import org.elasticsearch.search.aggregations.bucket.SingleBucketAggregation;
 
diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregationBuilder.java b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregationBuilder.java
new file mode 100644
index 0000000000000..3560736e448a4
--- /dev/null
+++ b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregationBuilder.java
@@ -0,0 +1,193 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.join.aggregations;
+
+import org.apache.lucene.search.Query;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData;
+import org.elasticsearch.index.mapper.DocumentMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.ParentFieldMapper;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.join.mapper.ParentIdFieldMapper;
+import org.elasticsearch.join.mapper.ParentJoinFieldMapper;
+import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
+import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.support.FieldContext;
+import org.elasticsearch.search.aggregations.support.ValueType;
+import org.elasticsearch.search.aggregations.support.ValuesSource.Bytes.WithOrdinals;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
+import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.aggregations.support.ValuesSourceType;
+import org.elasticsearch.search.internal.SearchContext;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class ChildrenAggregationBuilder
+        extends ValuesSourceAggregationBuilder<WithOrdinals, ChildrenAggregationBuilder> {
+
+    public static final String NAME = "children";
+
+    private final String childType;
+    private Query parentFilter;
+    private Query childFilter;
+
+    /**
+     * @param name
+     *            the name of this aggregation
+     * @param childType
+     *            the type of children documents
+     */
+    public ChildrenAggregationBuilder(String name, String childType) {
+        super(name, ValuesSourceType.BYTES, ValueType.STRING);
+        if (childType == null) {
+            throw new IllegalArgumentException("[childType] must not be null: [" + name + "]");
+        }
+        this.childType = childType;
+    }
+
+    /**
+     * Read from a stream.
+     */
+    public ChildrenAggregationBuilder(StreamInput in) throws IOException {
+        super(in, ValuesSourceType.BYTES, ValueType.STRING);
+        childType = in.readString();
+    }
+
+    @Override
+    protected void innerWriteTo(StreamOutput out) throws IOException {
+        out.writeString(childType);
+    }
+
+    @Override
+    protected ValuesSourceAggregatorFactory<WithOrdinals, ?> innerBuild(SearchContext context,
+                                                                        ValuesSourceConfig<WithOrdinals> config,
+                                                                        AggregatorFactory<?> parent,
+                                                                        Builder subFactoriesBuilder) throws IOException {
+        return new ChildrenAggregatorFactory(name, config, childFilter, parentFilter, context, parent,
+                subFactoriesBuilder, metaData);
+    }
+
+    @Override
+    protected ValuesSourceConfig<WithOrdinals> resolveConfig(SearchContext context) {
+        ValuesSourceConfig<WithOrdinals> config = new ValuesSourceConfig<>(ValuesSourceType.BYTES);
+        if (context.mapperService().getIndexSettings().isSingleType()) {
+            joinFieldResolveConfig(context, config);
+        } else {
+            parentFieldResolveConfig(context, config);
+        }
+        return config;
+    }
+
+    private void joinFieldResolveConfig(SearchContext context, ValuesSourceConfig<WithOrdinals> config) {
+        ParentJoinFieldMapper parentJoinFieldMapper = ParentJoinFieldMapper.getMapper(context.mapperService());
+        ParentIdFieldMapper parentIdFieldMapper = parentJoinFieldMapper.getParentIdFieldMapper(childType, false);
+        if (parentIdFieldMapper != null) {
+            parentFilter = parentIdFieldMapper.getParentFilter();
+            childFilter = parentIdFieldMapper.getChildFilter(childType);
+            MappedFieldType fieldType = parentIdFieldMapper.fieldType();
+            final SortedSetDVOrdinalsIndexFieldData fieldData = context.fieldData().getForField(fieldType);
+            config.fieldContext(new FieldContext(fieldType.name(), fieldData, fieldType));
+        } else {
+            config.unmapped(true);
+        }
+    }
+
+    private void parentFieldResolveConfig(SearchContext context, ValuesSourceConfig<WithOrdinals> config) {
+        DocumentMapper childDocMapper = context.mapperService().documentMapper(childType);
+        if (childDocMapper != null) {
+            ParentFieldMapper parentFieldMapper = childDocMapper.parentFieldMapper();
+            if (!parentFieldMapper.active()) {
+                throw new IllegalArgumentException("[children] no [_parent] field not configured that points to a parent type");
+            }
+            String parentType = parentFieldMapper.type();
+            DocumentMapper parentDocMapper = context.mapperService().documentMapper(parentType);
+            if (parentDocMapper != null) {
+                parentFilter = parentDocMapper.typeFilter(context.getQueryShardContext());
+                childFilter = childDocMapper.typeFilter(context.getQueryShardContext());
+                MappedFieldType parentFieldType = parentDocMapper.parentFieldMapper().getParentJoinFieldType();
+                final SortedSetDVOrdinalsIndexFieldData fieldData = context.fieldData().getForField(parentFieldType);
+                config.fieldContext(new FieldContext(parentFieldType.name(), fieldData,
+                    parentFieldType));
+            } else {
+                config.unmapped(true);
+            }
+        } else {
+            config.unmapped(true);
+        }
+    }
+
+    @Override
+    protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+        builder.field(ParentToChildrenAggregator.TYPE_FIELD.getPreferredName(), childType);
+        return builder;
+    }
+
+    public static ChildrenAggregationBuilder parse(String aggregationName, QueryParseContext context) throws IOException {
+        String childType = null;
+
+        XContentParser.Token token;
+        String currentFieldName = null;
+        XContentParser parser = context.parser();
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token == XContentParser.Token.VALUE_STRING) {
+                if ("type".equals(currentFieldName)) {
+                    childType = parser.text();
+                } else {
+                    throw new ParsingException(parser.getTokenLocation(),
+                            "Unknown key for a " + token + " in [" + aggregationName + "]: [" + currentFieldName + "].");
+                }
+            } else {
+                throw new ParsingException(parser.getTokenLocation(), "Unexpected token " + token + " in [" + aggregationName + "].");
+            }
+        }
+
+        if (childType == null) {
+            throw new ParsingException(parser.getTokenLocation(),
+                    "Missing [child_type] field for children aggregation [" + aggregationName + "]");
+        }
+
+        return new ChildrenAggregationBuilder(aggregationName, childType);
+    }
+
+    @Override
+    protected int innerHashCode() {
+        return Objects.hash(childType);
+    }
+
+    @Override
+    protected boolean innerEquals(Object obj) {
+        ChildrenAggregationBuilder other = (ChildrenAggregationBuilder) obj;
+        return Objects.equals(childType, other.childType);
+    }
+
+    @Override
+    public String getType() {
+        return NAME;
+    }
+}
diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregatorFactory.java b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregatorFactory.java
new file mode 100644
index 0000000000000..9c38fa2eae6b9
--- /dev/null
+++ b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregatorFactory.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.join.aggregations;
+
+import org.apache.lucene.search.Query;
+import org.elasticsearch.search.aggregations.Aggregator;
+import org.elasticsearch.search.aggregations.AggregatorFactories;
+import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.NonCollectingAggregator;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.search.aggregations.support.ValuesSource.Bytes.WithOrdinals;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
+import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.internal.SearchContext;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+public class ChildrenAggregatorFactory
+        extends ValuesSourceAggregatorFactory<WithOrdinals, ChildrenAggregatorFactory> {
+
+    private final Query parentFilter;
+    private final Query childFilter;
+
+    public ChildrenAggregatorFactory(String name, ValuesSourceConfig<WithOrdinals> config,
+                                     Query childFilter, Query parentFilter, SearchContext context, AggregatorFactory<?> parent,
+                                     AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
+        this.childFilter = childFilter;
+        this.parentFilter = parentFilter;
+    }
+
+    @Override
+    protected Aggregator createUnmapped(Aggregator parent, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData)
+            throws IOException {
+        return new NonCollectingAggregator(name, context, parent, pipelineAggregators, metaData) {
+
+            @Override
+            public InternalAggregation buildEmptyAggregation() {
+                return new InternalChildren(name, 0, buildEmptySubAggregations(), pipelineAggregators(), metaData());
+            }
+
+        };
+    }
+
+    @Override
+    protected Aggregator doCreateInternal(WithOrdinals valuesSource, Aggregator parent,
+            boolean collectsFromSingleBucket, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData)
+                    throws IOException {
+        long maxOrd = valuesSource.globalMaxOrd(context.searcher());
+        return new ParentToChildrenAggregator(name, factories, context, parent, childFilter,
+            parentFilter, valuesSource, maxOrd, pipelineAggregators, metaData);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/InternalChildren.java b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/InternalChildren.java
similarity index 97%
rename from core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/InternalChildren.java
rename to modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/InternalChildren.java
index 05a38c8cd59b5..05cd40e3d3366 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/InternalChildren.java
+++ b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/InternalChildren.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.search.aggregations.bucket.children;
+package org.elasticsearch.join.aggregations;
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.search.aggregations.InternalAggregations;
diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/JoinAggregationBuilders.java b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/JoinAggregationBuilders.java
new file mode 100644
index 0000000000000..73522a68b4595
--- /dev/null
+++ b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/JoinAggregationBuilders.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.join.aggregations;
+
+public abstract class JoinAggregationBuilders {
+    /**
+     * Create a new {@link Children} aggregation with the given name.
+     */
+    public static ChildrenAggregationBuilder children(String name, String childType) {
+        return new ChildrenAggregationBuilder(name, childType);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/ParentToChildrenAggregator.java b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentToChildrenAggregator.java
similarity index 79%
rename from core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/ParentToChildrenAggregator.java
rename to modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentToChildrenAggregator.java
index c844f968b9ccf..a0655bb44d06a 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/ParentToChildrenAggregator.java
+++ b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentToChildrenAggregator.java
@@ -16,11 +16,11 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.search.aggregations.bucket.children;
+package org.elasticsearch.join.aggregations;
 
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.search.ConstantScoreScorer;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.Query;
@@ -38,8 +38,8 @@
 import org.elasticsearch.search.aggregations.LeafBucketCollector;
 import org.elasticsearch.search.aggregations.bucket.SingleBucketAggregator;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -52,10 +52,9 @@ public class ParentToChildrenAggregator extends SingleBucketAggregator {
 
     static final ParseField TYPE_FIELD = new ParseField("type");
 
-    private final String parentType;
     private final Weight childFilter;
     private final Weight parentFilter;
-    private final ValuesSource.Bytes.WithOrdinals.ParentChild valuesSource;
+    private final ValuesSource.Bytes.WithOrdinals valuesSource;
 
     // Maybe use PagedGrowableWriter? This will be less wasteful than LongArray, but then we don't have the reuse feature of BigArrays.
     // Also if we know the highest possible value that a parent agg will create then we store multiple values into one slot
@@ -67,25 +66,25 @@ public class ParentToChildrenAggregator extends SingleBucketAggregator {
     private final LongObjectPagedHashMap<long[]> parentOrdToOtherBuckets;
     private boolean multipleBucketsPerParentOrd = false;
 
-    public ParentToChildrenAggregator(String name, AggregatorFactories factories, AggregationContext aggregationContext,
-                                      Aggregator parent, String parentType, Query childFilter, Query parentFilter,
-                                      ValuesSource.Bytes.WithOrdinals.ParentChild valuesSource,
-            long maxOrd, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException {
-        super(name, factories, aggregationContext, parent, pipelineAggregators, metaData);
-        this.parentType = parentType;
+    public ParentToChildrenAggregator(String name, AggregatorFactories factories,
+            SearchContext context, Aggregator parent, Query childFilter,
+            Query parentFilter, ValuesSource.Bytes.WithOrdinals valuesSource,
+            long maxOrd, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData)
+            throws IOException {
+        super(name, factories, context, parent, pipelineAggregators, metaData);
         // these two filters are cached in the parser
-        this.childFilter = aggregationContext.searchContext().searcher().createNormalizedWeight(childFilter, false);
-        this.parentFilter = aggregationContext.searchContext().searcher().createNormalizedWeight(parentFilter, false);
-        this.parentOrdToBuckets = aggregationContext.bigArrays().newLongArray(maxOrd, false);
+        this.childFilter = context.searcher().createNormalizedWeight(childFilter, false);
+        this.parentFilter = context.searcher().createNormalizedWeight(parentFilter, false);
+        this.parentOrdToBuckets = context.bigArrays().newLongArray(maxOrd, false);
         this.parentOrdToBuckets.fill(0, maxOrd, -1);
-        this.parentOrdToOtherBuckets = new LongObjectPagedHashMap<>(aggregationContext.bigArrays());
+        this.parentOrdToOtherBuckets = new LongObjectPagedHashMap<>(context.bigArrays());
         this.valuesSource = valuesSource;
     }
 
     @Override
     public InternalAggregation buildAggregation(long owningBucketOrdinal) throws IOException {
-        return new InternalChildren(name, bucketDocCount(owningBucketOrdinal), bucketAggregations(owningBucketOrdinal), pipelineAggregators(),
-                metaData());
+        return new InternalChildren(name, bucketDocCount(owningBucketOrdinal),
+            bucketAggregations(owningBucketOrdinal), pipelineAggregators(), metaData());
     }
 
     @Override
@@ -99,17 +98,15 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx,
         if (valuesSource == null) {
             return LeafBucketCollector.NO_OP_COLLECTOR;
         }
-
-        final SortedDocValues globalOrdinals = valuesSource.globalOrdinalsValues(parentType, ctx);
-        assert globalOrdinals != null;
-        Scorer parentScorer = parentFilter.scorer(ctx);
-        final Bits parentDocs = Lucene.asSequentialAccessBits(ctx.reader().maxDoc(), parentScorer);
+        final SortedSetDocValues globalOrdinals = valuesSource.globalOrdinalsValues(ctx);
+        final Bits parentDocs = Lucene.asSequentialAccessBits(ctx.reader().maxDoc(), parentFilter.scorerSupplier(ctx));
         return new LeafBucketCollector() {
 
             @Override
             public void collect(int docId, long bucket) throws IOException {
                 if (parentDocs.get(docId)) {
-                    long globalOrdinal = globalOrdinals.getOrd(docId);
+                    globalOrdinals.setDocument(docId);
+                    long globalOrdinal = globalOrdinals.nextOrd();
                     if (globalOrdinal != -1) {
                         if (parentOrdToBuckets.get(globalOrdinal) == -1) {
                             parentOrdToBuckets.set(globalOrdinal, bucket);
@@ -132,7 +129,7 @@ public void collect(int docId, long bucket) throws IOException {
 
     @Override
     protected void doPostCollection() throws IOException {
-        IndexReader indexReader = context().searchContext().searcher().getIndexReader();
+        IndexReader indexReader = context().searcher().getIndexReader();
         for (LeafReaderContext ctx : indexReader.leaves()) {
             Scorer childDocsScorer = childFilter.scorer(ctx);
             if (childDocsScorer == null) {
@@ -141,8 +138,7 @@ protected void doPostCollection() throws IOException {
             DocIdSetIterator childDocsIter = childDocsScorer.iterator();
 
             final LeafBucketCollector sub = collectableSubAggregators.getLeafCollector(ctx);
-            final SortedDocValues globalOrdinals = valuesSource.globalOrdinalsValues(parentType, ctx);
-
+            final SortedSetDocValues globalOrdinals = valuesSource.globalOrdinalsValues(ctx);
             // Set the scorer, since we now replay only the child docIds
             sub.setScorer(new ConstantScoreScorer(null, 1f,childDocsIter));
 
@@ -151,7 +147,8 @@ protected void doPostCollection() throws IOException {
                 if (liveDocs != null && liveDocs.get(docId) == false) {
                     continue;
                 }
-                long globalOrdinal = globalOrdinals.getOrd(docId);
+                globalOrdinals.setDocument(docId);
+                long globalOrdinal = globalOrdinals.nextOrd();
                 if (globalOrdinal != -1) {
                     long bucketOrd = parentOrdToBuckets.get(globalOrdinal);
                     if (bucketOrd != -1) {
diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParsedChildren.java b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParsedChildren.java
new file mode 100644
index 0000000000000..fa370325f74f3
--- /dev/null
+++ b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParsedChildren.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.join.aggregations;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.bucket.ParsedSingleBucketAggregation;
+
+import java.io.IOException;
+
+public class ParsedChildren extends ParsedSingleBucketAggregation implements Children {
+
+    @Override
+    public String getType() {
+        return ChildrenAggregationBuilder.NAME;
+    }
+
+    public static ParsedChildren fromXContent(XContentParser parser, final String name) throws IOException {
+        return parseXContent(parser, new ParsedChildren(), name);
+    }
+}
diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/fetch/ParentJoinFieldSubFetchPhase.java b/modules/parent-join/src/main/java/org/elasticsearch/join/fetch/ParentJoinFieldSubFetchPhase.java
new file mode 100644
index 0000000000000..92a248c83af2e
--- /dev/null
+++ b/modules/parent-join/src/main/java/org/elasticsearch/join/fetch/ParentJoinFieldSubFetchPhase.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.join.fetch;
+
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.index.mapper.DocumentMapper;
+import org.elasticsearch.join.mapper.ParentIdFieldMapper;
+import org.elasticsearch.join.mapper.ParentJoinFieldMapper;
+import org.elasticsearch.search.SearchHitField;
+import org.elasticsearch.search.fetch.FetchSubPhase;
+import org.elasticsearch.search.internal.SearchContext;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * A sub fetch phase that retrieves the join name and the parent id for each document containing
+ * a {@link ParentJoinFieldMapper} field.
+ */
+public final class ParentJoinFieldSubFetchPhase implements FetchSubPhase {
+    @Override
+    public void hitExecute(SearchContext context, HitContext hitContext) {
+        if (context.storedFieldsContext() != null && context.storedFieldsContext().fetchFields() == false) {
+            return;
+        }
+        ParentJoinFieldMapper mapper = ParentJoinFieldMapper.getMapper(context.mapperService());
+        if (mapper == null) {
+            // hit has no join field.
+            return;
+        }
+        String joinName = getSortedDocValue(mapper.name(), hitContext.reader(), hitContext.docId());
+        if (joinName == null) {
+            return;
+        }
+
+        // if the hit is a children we extract the parentId (if it's a parent we can use the _id field directly)
+        ParentIdFieldMapper parentMapper = mapper.getParentIdFieldMapper(joinName, false);
+        String parentId = null;
+        if (parentMapper != null) {
+            parentId = getSortedDocValue(parentMapper.name(), hitContext.reader(), hitContext.docId());
+        }
+
+        Map<String, SearchHitField> fields = hitContext.hit().fieldsOrNull();
+        if (fields == null) {
+            fields = new HashMap<>();
+            hitContext.hit().fields(fields);
+        }
+        fields.put(mapper.name(), new SearchHitField(mapper.name(), Collections.singletonList(joinName)));
+        if (parentId != null) {
+            fields.put(parentMapper.name(), new SearchHitField(parentMapper.name(), Collections.singletonList(parentId)));
+        }
+    }
+
+    private String getSortedDocValue(String field, LeafReader reader, int docId) {
+        try {
+            SortedDocValues docValues = reader.getSortedDocValues(field);
+            if (docValues == null) {
+                return null;
+            }
+            int ord = docValues.getOrd(docId);
+            if (ord == -1) {
+                return null;
+            }
+            BytesRef joinName = docValues.lookupOrd(ord);
+            return joinName.utf8ToString();
+        } catch (IOException e) {
+            throw ExceptionsHelper.convertToElastic(e);
+        }
+    }
+}
diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java
new file mode 100644
index 0000000000000..db9fae9b47d2a
--- /dev/null
+++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.join.mapper;
+
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
+import org.elasticsearch.index.mapper.FieldMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.ParseContext;
+import org.elasticsearch.index.mapper.StringFieldType;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Simple field mapper hack to ensure that there is a one and only {@link ParentJoinFieldMapper} per mapping.
+ * This field mapper is not used to index or query any data, it is used as a marker in the mapping that
+ * denotes the presence of a parent-join field and forbids the addition of any additional ones.
+ * This class is also used to quickly retrieve the parent-join field defined in a mapping without
+ * specifying the name of the field.
+ */
+public class MetaJoinFieldMapper extends FieldMapper {
+    static final String NAME = "_parent_join";
+    static final String CONTENT_TYPE = "parent_join";
+
+    static class Defaults {
+        public static final MappedFieldType FIELD_TYPE = new MetaJoinFieldType();
+
+        static {
+            FIELD_TYPE.setStored(false);
+            FIELD_TYPE.setHasDocValues(false);
+            FIELD_TYPE.setIndexOptions(IndexOptions.NONE);
+            FIELD_TYPE.freeze();
+        }
+    }
+
+    static class Builder extends FieldMapper.Builder<Builder, MetaJoinFieldMapper> {
+        Builder() {
+            super(NAME, Defaults.FIELD_TYPE, Defaults.FIELD_TYPE);
+            builder = this;
+        }
+
+        @Override
+        public MetaJoinFieldMapper build(BuilderContext context) {
+            fieldType.setName(NAME);
+            return new MetaJoinFieldMapper(name, fieldType, context.indexSettings());
+        }
+    }
+
+    public static class MetaJoinFieldType extends StringFieldType {
+
+        private ParentJoinFieldMapper mapper;
+
+        MetaJoinFieldType() {}
+
+        protected MetaJoinFieldType(MetaJoinFieldType ref) {
+            super(ref);
+        }
+
+        public MetaJoinFieldType clone() {
+            return new MetaJoinFieldType(this);
+        }
+
+        @Override
+        public String typeName() {
+            return CONTENT_TYPE;
+        }
+
+        @Override
+        public IndexFieldData.Builder fielddataBuilder() {
+            failIfNoDocValues();
+            return new DocValuesIndexFieldData.Builder();
+        }
+
+        @Override
+        public Object valueForDisplay(Object value) {
+            if (value == null) {
+                return null;
+            }
+            BytesRef binaryValue = (BytesRef) value;
+            return binaryValue.utf8ToString();
+        }
+
+        public ParentJoinFieldMapper getMapper() {
+            return mapper;
+        }
+    }
+
+    MetaJoinFieldMapper(String name, MappedFieldType fieldType, Settings indexSettings) {
+        super(name, fieldType, ParentIdFieldMapper.Defaults.FIELD_TYPE, indexSettings, MultiFields.empty(), null);
+    }
+
+    void setFieldMapper(ParentJoinFieldMapper mapper) {
+        fieldType().mapper = mapper;
+    }
+
+    @Override
+    public MetaJoinFieldType fieldType() {
+        return (MetaJoinFieldType) super.fieldType();
+    }
+
+    @Override
+    protected MetaJoinFieldMapper clone() {
+        return (MetaJoinFieldMapper) super.clone();
+    }
+
+    @Override
+    protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
+        throw new IllegalStateException("Should never be called");
+    }
+
+    @Override
+    protected String contentType() {
+        return CONTENT_TYPE;
+    }
+}
diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java
new file mode 100644
index 0000000000000..cc2815c373aa8
--- /dev/null
+++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java
@@ -0,0 +1,200 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.join.mapper;
+
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.ConstantScoreQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.lucene.Lucene;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
+import org.elasticsearch.index.mapper.FieldMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.Mapper;
+import org.elasticsearch.index.mapper.ParseContext;
+import org.elasticsearch.index.mapper.StringFieldType;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * A field mapper used internally by the {@link ParentJoinFieldMapper} to index
+ * the value that link documents in the index (parent _id or _id if the document is a parent).
+ */
+public final class ParentIdFieldMapper extends FieldMapper {
+    static final String CONTENT_TYPE = "parent";
+
+    static class Defaults {
+        static final MappedFieldType FIELD_TYPE = new ParentIdFieldType();
+
+        static {
+            FIELD_TYPE.setTokenized(false);
+            FIELD_TYPE.setOmitNorms(true);
+            FIELD_TYPE.setHasDocValues(true);
+            FIELD_TYPE.setIndexOptions(IndexOptions.DOCS);
+            FIELD_TYPE.freeze();
+        }
+    }
+
+    static class Builder extends FieldMapper.Builder<Builder, ParentIdFieldMapper> {
+        private final String parent;
+        private final Set<String> children;
+
+        Builder(String name, String parent, Set<String> children) {
+            super(name, Defaults.FIELD_TYPE, Defaults.FIELD_TYPE);
+            builder = this;
+            this.parent = parent;
+            this.children = children;
+        }
+
+        public Set<String> getChildren() {
+            return children;
+        }
+
+        public Builder eagerGlobalOrdinals(boolean eagerGlobalOrdinals) {
+            fieldType().setEagerGlobalOrdinals(eagerGlobalOrdinals);
+            return builder;
+        }
+
+        @Override
+        public ParentIdFieldMapper build(BuilderContext context) {
+            fieldType.setName(name);
+            return new ParentIdFieldMapper(name, parent, children, fieldType, context.indexSettings());
+        }
+    }
+
+    public static final class ParentIdFieldType extends StringFieldType {
+        ParentIdFieldType() {
+            setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
+            setSearchAnalyzer(Lucene.KEYWORD_ANALYZER);
+        }
+
+        protected ParentIdFieldType(ParentIdFieldType ref) {
+            super(ref);
+        }
+
+        public ParentIdFieldType clone() {
+            return new ParentIdFieldType(this);
+        }
+
+        @Override
+        public String typeName() {
+            return CONTENT_TYPE;
+        }
+
+        @Override
+        public IndexFieldData.Builder fielddataBuilder() {
+            failIfNoDocValues();
+            return new DocValuesIndexFieldData.Builder();
+        }
+
+        @Override
+        public Object valueForDisplay(Object value) {
+            if (value == null) {
+                return null;
+            }
+            BytesRef binaryValue = (BytesRef) value;
+            return binaryValue.utf8ToString();
+        }
+    }
+
+    private final String parentName;
+    private Set<String> children;
+
+    protected ParentIdFieldMapper(String simpleName,
+                                  String parentName,
+                                  Set<String> children,
+                                  MappedFieldType fieldType,
+                                  Settings indexSettings) {
+        super(simpleName, fieldType, Defaults.FIELD_TYPE, indexSettings, MultiFields.empty(), null);
+        this.parentName = parentName;
+        this.children = children;
+    }
+
+    @Override
+    protected ParentIdFieldMapper clone() {
+        return (ParentIdFieldMapper) super.clone();
+    }
+
+    /**
+     * Returns the parent name associated with this mapper.
+     */
+    public String getParentName() {
+        return parentName;
+    }
+
+    public Query getParentFilter() {
+        return new TermQuery(new Term(name().substring(0, name().indexOf('#')), parentName));
+    }
+    /**
+     * Returns the children names associated with this mapper.
+     */
+    public Collection<String> getChildren() {
+        return children;
+    }
+
+    public Query getChildFilter(String type) {
+        return new TermQuery(new Term(name().substring(0, name().indexOf('#')), type));
+    }
+
+    public Query getChildrenFilter() {
+        BooleanQuery.Builder builder = new BooleanQuery.Builder();
+        for (String child : children) {
+            builder.add(getChildFilter(child), BooleanClause.Occur.SHOULD);
+        }
+        return new ConstantScoreQuery(builder.build());
+    }
+
+    @Override
+    protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
+        if (context.externalValueSet() == false) {
+            throw new IllegalStateException("external value not set");
+        }
+        String refId = (String) context.externalValue();
+        BytesRef binaryValue = new BytesRef(refId);
+        Field field = new Field(fieldType().name(), binaryValue, fieldType());
+        fields.add(field);
+        fields.add(new SortedDocValuesField(fieldType().name(), binaryValue));
+    }
+
+
+    @Override
+    protected void doMerge(Mapper mergeWith, boolean updateAllTypes) {
+        super.doMerge(mergeWith, updateAllTypes);
+        ParentIdFieldMapper parentMergeWith = (ParentIdFieldMapper) mergeWith;
+        this.children = parentMergeWith.children;
+    }
+
+    @Override
+    protected String contentType() {
+        return CONTENT_TYPE;
+    }
+}
diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java
new file mode 100644
index 0000000000000..83fd158c5df25
--- /dev/null
+++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java
@@ -0,0 +1,455 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.join.mapper;
+
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.lucene.Lucene;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
+import org.elasticsearch.index.mapper.ContentPath;
+import org.elasticsearch.index.mapper.FieldMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.Mapper;
+import org.elasticsearch.index.mapper.MapperParsingException;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.ParentFieldMapper;
+import org.elasticsearch.index.mapper.ParseContext;
+import org.elasticsearch.index.mapper.StringFieldType;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A {@link FieldMapper} that creates hierarchical joins (parent-join) between documents in the same index.
+ * Only one parent-join field can be defined per index. The verification of this assumption is done
+ * through the {@link MetaJoinFieldMapper} which declares a meta field called "_parent_join".
+ * This field is only used to ensure that there is a single parent-join field defined in the mapping and
+ * cannot be used to index or query any data.
+ */
+public final class ParentJoinFieldMapper extends FieldMapper {
+    public static final String NAME = "join";
+    public static final String CONTENT_TYPE = "join";
+
+    public static class Defaults {
+        public static final MappedFieldType FIELD_TYPE = new JoinFieldType();
+
+        static {
+            FIELD_TYPE.setTokenized(false);
+            FIELD_TYPE.setOmitNorms(true);
+            FIELD_TYPE.setHasDocValues(true);
+            FIELD_TYPE.setIndexOptions(IndexOptions.DOCS);
+            FIELD_TYPE.freeze();
+        }
+    }
+
+    /**
+     * Returns the {@link ParentJoinFieldMapper} associated with the <code>service</code> or null
+     * if there is no parent-join field in this mapping.
+     */
+    public static ParentJoinFieldMapper getMapper(MapperService service) {
+        MetaJoinFieldMapper.MetaJoinFieldType fieldType =
+            (MetaJoinFieldMapper.MetaJoinFieldType) service.fullName(MetaJoinFieldMapper.NAME);
+        return fieldType == null ? null : fieldType.getMapper();
+    }
+
+    private static String getParentIdFieldName(String joinFieldName, String parentName) {
+        return joinFieldName + "#" + parentName;
+    }
+
+    private static void checkIndexCompatibility(IndexSettings settings, String name) {
+        if (settings.getIndexMetaData().isRoutingPartitionedIndex()) {
+            throw new IllegalStateException("cannot create join field [" + name + "] " +
+                "for the partitioned index " + "[" + settings.getIndex().getName() + "]");
+        }
+        if (settings.isSingleType() == false) {
+            throw new IllegalStateException("cannot create join field [" + name + "] " +
+                "on multi-types index [" + settings.getIndex().getName() + "]");
+        }
+    }
+
+    private static void checkObjectOrNested(ContentPath path, String name) {
+        if (path.pathAsText(name).contains(".")) {
+            throw new IllegalArgumentException("join field [" + path.pathAsText(name) + "] " +
+                "cannot be added inside an object or in a multi-field");
+        }
+    }
+
+    private static void checkParentFields(String name, List<ParentIdFieldMapper> mappers) {
+        Set<String> children = new HashSet<>();
+        List<String> conflicts = new ArrayList<>();
+        for (ParentIdFieldMapper mapper : mappers) {
+            for (String child : mapper.getChildren()) {
+                if (children.add(child) == false) {
+                    conflicts.add("[" + child + "] cannot have multiple parents");
+                }
+            }
+        }
+        if (conflicts.isEmpty() == false) {
+            throw new IllegalArgumentException("invalid definition for join field [" + name + "]:\n" + conflicts.toString());
+        }
+    }
+
+    public static class Builder extends FieldMapper.Builder<Builder, ParentJoinFieldMapper> {
+        final List<ParentIdFieldMapper.Builder> parentIdFieldBuilders = new ArrayList<>();
+        boolean eagerGlobalOrdinals = true;
+
+        public Builder(String name) {
+            super(name, Defaults.FIELD_TYPE, Defaults.FIELD_TYPE);
+            builder = this;
+        }
+
+        @Override
+        public JoinFieldType fieldType() {
+            return (JoinFieldType) super.fieldType();
+        }
+
+        public Builder addParent(String parent, Set<String> children) {
+            String parentIdFieldName = getParentIdFieldName(name, parent);
+            parentIdFieldBuilders.add(new ParentIdFieldMapper.Builder(parentIdFieldName, parent, children));
+            return builder;
+        }
+
+        public Builder eagerGlobalOrdinals(boolean eagerGlobalOrdinals) {
+            this.eagerGlobalOrdinals = eagerGlobalOrdinals;
+            return builder;
+        }
+
+        @Override
+        public ParentJoinFieldMapper build(BuilderContext context) {
+            checkObjectOrNested(context.path(), name);
+            fieldType.setName(name);
+            final List<ParentIdFieldMapper> parentIdFields = new ArrayList<>();
+            parentIdFieldBuilders.stream()
+                .map((parentBuilder) -> {
+                    if (eagerGlobalOrdinals) {
+                        parentBuilder.eagerGlobalOrdinals(true);
+                    }
+                    return parentBuilder.build(context);
+                })
+                .forEach(parentIdFields::add);
+            checkParentFields(name(), parentIdFields);
+            MetaJoinFieldMapper unique = new MetaJoinFieldMapper.Builder().build(context);
+            return new ParentJoinFieldMapper(name, fieldType, context.indexSettings(),
+                unique, Collections.unmodifiableList(parentIdFields), eagerGlobalOrdinals);
+        }
+    }
+
+    public static class TypeParser implements Mapper.TypeParser {
+        @Override
+        public Mapper.Builder<?,?> parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
+            final IndexSettings indexSettings = parserContext.mapperService().getIndexSettings();
+            checkIndexCompatibility(indexSettings, name);
+
+            Builder builder = new Builder(name);
+            for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
+                Map.Entry<String, Object> entry = iterator.next();
+                if ("type".equals(entry.getKey())) {
+                    continue;
+                }
+                if ("eager_global_ordinals".equals(entry.getKey())) {
+                    builder.eagerGlobalOrdinals(XContentMapValues.nodeBooleanValue(entry.getValue()));
+                    iterator.remove();
+                    continue;
+                }
+                if ("relations".equals(entry.getKey())) {
+                    Map<String, Object> relations = XContentMapValues.nodeMapValue(entry.getValue(), "relations");
+                    for (Iterator<Map.Entry<String, Object>> relIt = relations.entrySet().iterator(); relIt.hasNext(); ) {
+                        Map.Entry<String, Object> relation = relIt.next();
+                        final String parent = relation.getKey();
+                        Set<String> children;
+                        if (XContentMapValues.isArray(relation.getValue())) {
+                            children = new HashSet<>(Arrays.asList(XContentMapValues.nodeStringArrayValue(relation.getValue())));
+                        } else {
+                            children = Collections.singleton(relation.getValue().toString());
+                        }
+                        builder.addParent(parent, children);
+                    }
+                    iterator.remove();
+                }
+            }
+            return builder;
+        }
+    }
+
+    public static final class JoinFieldType extends StringFieldType {
+        public JoinFieldType() {
+            setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
+            setSearchAnalyzer(Lucene.KEYWORD_ANALYZER);
+        }
+
+        protected JoinFieldType(JoinFieldType ref) {
+            super(ref);
+        }
+
+        public JoinFieldType clone() {
+            return new JoinFieldType(this);
+        }
+
+        @Override
+        public String typeName() {
+            return CONTENT_TYPE;
+        }
+
+        @Override
+        public IndexFieldData.Builder fielddataBuilder() {
+            failIfNoDocValues();
+            return new DocValuesIndexFieldData.Builder();
+        }
+
+        @Override
+        public Object valueForDisplay(Object value) {
+            if (value == null) {
+                return null;
+            }
+            BytesRef binaryValue = (BytesRef) value;
+            return binaryValue.utf8ToString();
+        }
+    }
+
+    // The meta field that ensures that there is no other parent-join in the mapping
+    private MetaJoinFieldMapper uniqueFieldMapper;
+    private List<ParentIdFieldMapper> parentIdFields;
+    private boolean eagerGlobalOrdinals;
+
+    protected ParentJoinFieldMapper(String simpleName,
+                                    MappedFieldType fieldType,
+                                    Settings indexSettings,
+                                    MetaJoinFieldMapper uniqueFieldMapper,
+                                    List<ParentIdFieldMapper> parentIdFields,
+                                    boolean eagerGlobalOrdinals) {
+        super(simpleName, fieldType, Defaults.FIELD_TYPE, indexSettings, MultiFields.empty(), null);
+        this.parentIdFields = parentIdFields;
+        this.uniqueFieldMapper = uniqueFieldMapper;
+        this.uniqueFieldMapper.setFieldMapper(this);
+        this.eagerGlobalOrdinals = eagerGlobalOrdinals;
+    }
+
+    @Override
+    protected String contentType() {
+        return CONTENT_TYPE;
+    }
+
+    @Override
+    protected ParentJoinFieldMapper clone() {
+        return (ParentJoinFieldMapper) super.clone();
+    }
+
+    @Override
+    public JoinFieldType fieldType() {
+        return (JoinFieldType) super.fieldType();
+    }
+
+    @Override
+    public Iterator<Mapper> iterator() {
+        List<Mapper> mappers = new ArrayList<> (parentIdFields);
+        mappers.add(uniqueFieldMapper);
+        return mappers.iterator();
+    }
+
+    /**
+     * Returns true if <code>name</code> is a parent name in the field.
+     */
+    public boolean hasParent(String name) {
+        return parentIdFields.stream().anyMatch((mapper) -> name.equals(mapper.getParentName()));
+    }
+
+    /**
+     * Returns true if <code>name</code> is a child name in the field.
+     */
+    public boolean hasChild(String name) {
+        return parentIdFields.stream().anyMatch((mapper) -> mapper.getChildren().contains(name));
+    }
+
+    /**
+     * Returns the parent Id field mapper associated with a parent <code>name</code>
+     * if <code>isParent</code> is true and a child <code>name</code> otherwise.
+     */
+    public ParentIdFieldMapper getParentIdFieldMapper(String name, boolean isParent) {
+        for (ParentIdFieldMapper mapper : parentIdFields) {
+            if (isParent && name.equals(mapper.getParentName())) {
+                return mapper;
+            } else if (isParent == false && mapper.getChildren().contains(name)) {
+                return mapper;
+            }
+        }
+        return null;
+    }
+
+    @Override
+    protected void doMerge(Mapper mergeWith, boolean updateAllTypes) {
+        super.doMerge(mergeWith, updateAllTypes);
+        ParentJoinFieldMapper joinMergeWith = (ParentJoinFieldMapper) mergeWith;
+        List<String> conflicts = new ArrayList<>();
+        for (ParentIdFieldMapper mapper : parentIdFields) {
+            if (joinMergeWith.getParentIdFieldMapper(mapper.getParentName(), true) == null) {
+                conflicts.add("cannot remove parent [" + mapper.getParentName() + "] in join field [" + name() + "]");
+            }
+        }
+
+        final List<ParentIdFieldMapper> newParentIdFields = new ArrayList<>();
+        for (ParentIdFieldMapper mergeWithMapper : joinMergeWith.parentIdFields) {
+            ParentIdFieldMapper self = getParentIdFieldMapper(mergeWithMapper.getParentName(), true);
+            if (self == null) {
+                if (getParentIdFieldMapper(mergeWithMapper.getParentName(), false) != null) {
+                    // it is forbidden to add a parent to an existing child
+                    conflicts.add("cannot create parent [" + mergeWithMapper.getParentName()  + "] from an existing child");
+                }
+                for (String child : mergeWithMapper.getChildren()) {
+                    if (getParentIdFieldMapper(child, true) != null) {
+                        // it is forbidden to add a parent to an existing child
+                        conflicts.add("cannot create child [" + child  + "] from an existing parent");
+                    }
+                }
+                newParentIdFields.add(mergeWithMapper);
+            } else {
+                for (String child : self.getChildren()) {
+                    if (mergeWithMapper.getChildren().contains(child) == false) {
+                        conflicts.add("cannot remove child [" + child + "] in join field [" + name() + "]");
+                    }
+                }
+                ParentIdFieldMapper merged = (ParentIdFieldMapper) self.merge(mergeWithMapper, updateAllTypes);
+                newParentIdFields.add(merged);
+            }
+        }
+        if (conflicts.isEmpty() == false) {
+            throw new IllegalStateException("invalid update for join field [" + name() + "]:\n" + conflicts.toString());
+        }
+        this.eagerGlobalOrdinals = joinMergeWith.eagerGlobalOrdinals;
+        this.parentIdFields = Collections.unmodifiableList(newParentIdFields);
+        this.uniqueFieldMapper = (MetaJoinFieldMapper) uniqueFieldMapper.merge(joinMergeWith.uniqueFieldMapper, updateAllTypes);
+        uniqueFieldMapper.setFieldMapper(this);
+    }
+
+    @Override
+    public FieldMapper updateFieldType(Map<String, MappedFieldType> fullNameToFieldType) {
+        ParentJoinFieldMapper fieldMapper = (ParentJoinFieldMapper) super.updateFieldType(fullNameToFieldType);
+        final List<ParentIdFieldMapper> newMappers = new ArrayList<> ();
+        for (ParentIdFieldMapper mapper : fieldMapper.parentIdFields) {
+            newMappers.add((ParentIdFieldMapper) mapper.updateFieldType(fullNameToFieldType));
+        }
+        fieldMapper.parentIdFields = Collections.unmodifiableList(newMappers);
+        this.uniqueFieldMapper = (MetaJoinFieldMapper) uniqueFieldMapper.updateFieldType(fullNameToFieldType);
+        uniqueFieldMapper.setFieldMapper(this);
+        return fieldMapper;
+    }
+
+    @Override
+    protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
+        throw new UnsupportedOperationException("parsing is implemented in parse(), this method should NEVER be called");
+    }
+
+    @Override
+    public Mapper parse(ParseContext context) throws IOException {
+        context.path().add(simpleName());
+        XContentParser.Token token = context.parser().currentToken();
+        String name = null;
+        String parent = null;
+        if (token == XContentParser.Token.START_OBJECT) {
+            String currentFieldName = null;
+            while ((token = context.parser().nextToken()) != XContentParser.Token.END_OBJECT) {
+                if (token == XContentParser.Token.FIELD_NAME) {
+                    currentFieldName = context.parser().currentName();
+                } else if (token == XContentParser.Token.VALUE_STRING) {
+                    if ("name".equals(currentFieldName)) {
+                        name = context.parser().text();
+                    } else if ("parent".equals(currentFieldName)) {
+                        parent = context.parser().text();
+                    } else {
+                        throw new IllegalArgumentException("unknown field name [" + currentFieldName + "] in join field [" + name() + "]");
+                    }
+                } else if (token == XContentParser.Token.VALUE_NUMBER) {
+                    if ("parent".equals(currentFieldName)) {
+                        parent = context.parser().numberValue().toString();
+                    } else {
+                        throw new IllegalArgumentException("unknown field name [" + currentFieldName + "] in join field [" + name() + "]");
+                    }
+                }
+            }
+        } else if (token == XContentParser.Token.VALUE_STRING) {
+            name = context.parser().text();
+            parent = null;
+        } else {
+            throw new IllegalStateException("[" + name  + "] expected START_OBJECT or VALUE_STRING but was: " + token);
+        }
+
+        ParentIdFieldMapper parentIdField = getParentIdFieldMapper(name, true);
+        ParentIdFieldMapper childParentIdField = getParentIdFieldMapper(name, false);
+        if (parentIdField == null && childParentIdField == null) {
+            throw new IllegalArgumentException("unknown join name [" + name + "] for field [" + name() + "]");
+        }
+        if (childParentIdField != null) {
+            // Index the document as a child
+            if (parent == null) {
+                throw new IllegalArgumentException("[parent] is missing for join field [" + name() + "]");
+            }
+            if (context.sourceToParse().routing() == null) {
+                throw new IllegalArgumentException("[routing] is missing for join field [" + name() + "]");
+            }
+            assert childParentIdField.getChildren().contains(name);
+            ParseContext externalContext = context.createExternalValueContext(parent);
+            childParentIdField.parse(externalContext);
+        }
+        if (parentIdField != null) {
+            // Index the document as a parent
+            assert parentIdField.getParentName().equals(name);
+            ParseContext externalContext = context.createExternalValueContext(context.sourceToParse().id());
+            parentIdField.parse(externalContext);
+        }
+
+        BytesRef binaryValue = new BytesRef(name);
+        Field field = new Field(fieldType().name(), binaryValue, fieldType());
+        context.doc().add(field);
+        context.doc().add(new SortedDocValuesField(fieldType().name(), binaryValue));
+        context.path().remove();
+        return null;
+    }
+
+    @Override
+    protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException {
+        builder.field("type", contentType());
+        builder.field("eager_global_ordinals", eagerGlobalOrdinals);
+        builder.startObject("relations");
+        for (ParentIdFieldMapper field : parentIdFields) {
+            if (field.getChildren().size() == 1) {
+                builder.field(field.getParentName(), field.getChildren().iterator().next());
+            } else {
+                builder.field(field.getParentName(), field.getChildren());
+            }
+        }
+        builder.endObject();
+    }
+
+}
diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasChildQueryBuilder.java b/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasChildQueryBuilder.java
new file mode 100644
index 0000000000000..817ba6ba9e1f7
--- /dev/null
+++ b/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasChildQueryBuilder.java
@@ -0,0 +1,536 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.join.query;
+
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.MultiDocValues;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchNoDocsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.join.JoinUtil;
+import org.apache.lucene.search.join.ScoreMode;
+import org.apache.lucene.search.similarities.Similarity;
+import org.elasticsearch.Version;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.lucene.search.Queries;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
+import org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData;
+import org.elasticsearch.index.mapper.DocumentMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.ParentFieldMapper;
+import org.elasticsearch.index.query.AbstractQueryBuilder;
+import org.elasticsearch.index.query.InnerHitBuilder;
+import org.elasticsearch.index.query.InnerHitContextBuilder;
+import org.elasticsearch.index.query.NestedQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.index.query.QueryRewriteContext;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.query.QueryShardException;
+import org.elasticsearch.join.mapper.ParentIdFieldMapper;
+import org.elasticsearch.join.mapper.ParentJoinFieldMapper;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+
+/**
+ * A query builder for <tt>has_child</tt> query.
+ */
+public class HasChildQueryBuilder extends AbstractQueryBuilder<HasChildQueryBuilder> {
+    public static final String NAME = "has_child";
+
+    /**
+     * The default maximum number of children that are required to match for the parent to be considered a match.
+     */
+    public static final int DEFAULT_MAX_CHILDREN = Integer.MAX_VALUE;
+    /**
+     * The default minimum number of children that are required to match for the parent to be considered a match.
+     */
+    public static final int DEFAULT_MIN_CHILDREN = 0;
+
+    /**
+     * The default value for ignore_unmapped.
+     */
+    public static final boolean DEFAULT_IGNORE_UNMAPPED = false;
+
+    private static final ParseField QUERY_FIELD = new ParseField("query", "filter");
+    private static final ParseField TYPE_FIELD = new ParseField("type", "child_type");
+    private static final ParseField MAX_CHILDREN_FIELD = new ParseField("max_children");
+    private static final ParseField MIN_CHILDREN_FIELD = new ParseField("min_children");
+    private static final ParseField SCORE_MODE_FIELD = new ParseField("score_mode");
+    private static final ParseField INNER_HITS_FIELD = new ParseField("inner_hits");
+    private static final ParseField IGNORE_UNMAPPED_FIELD = new ParseField("ignore_unmapped");
+
+    private final QueryBuilder query;
+    private final String type;
+    private final ScoreMode scoreMode;
+    private InnerHitBuilder innerHitBuilder;
+    private int minChildren = DEFAULT_MIN_CHILDREN;
+    private int maxChildren = DEFAULT_MAX_CHILDREN;
+    private boolean ignoreUnmapped = false;
+
+    public HasChildQueryBuilder(String type, QueryBuilder query, ScoreMode scoreMode) {
+        this(type, query, DEFAULT_MIN_CHILDREN, DEFAULT_MAX_CHILDREN, scoreMode, null);
+    }
+
+    private HasChildQueryBuilder(String type, QueryBuilder query, int minChildren, int maxChildren, ScoreMode scoreMode,
+                                InnerHitBuilder innerHitBuilder) {
+        this.type = requireValue(type, "[" + NAME + "] requires 'type' field");
+        this.query = requireValue(query, "[" + NAME + "] requires 'query' field");
+        this.scoreMode = requireValue(scoreMode, "[" + NAME + "] requires 'score_mode' field");
+        this.innerHitBuilder = innerHitBuilder;
+        this.minChildren = minChildren;
+        this.maxChildren = maxChildren;
+    }
+
+    /**
+     * Read from a stream.
+     */
+    public HasChildQueryBuilder(StreamInput in) throws IOException {
+        super(in);
+        type = in.readString();
+        minChildren = in.readInt();
+        maxChildren = in.readInt();
+        scoreMode = ScoreMode.values()[in.readVInt()];
+        query = in.readNamedWriteable(QueryBuilder.class);
+        innerHitBuilder = in.readOptionalWriteable(InnerHitBuilder::new);
+        ignoreUnmapped = in.readBoolean();
+    }
+
+    @Override
+    protected void doWriteTo(StreamOutput out) throws IOException {
+        out.writeString(type);
+        out.writeInt(minChildren);
+        out.writeInt(maxChildren);
+        out.writeVInt(scoreMode.ordinal());
+        out.writeNamedWriteable(query);
+        if (out.getVersion().before(Version.V_5_5_0)) {
+            final boolean hasInnerHit = innerHitBuilder != null;
+            out.writeBoolean(hasInnerHit);
+            if (hasInnerHit) {
+                innerHitBuilder.writeToParentChildBWC(out, query, type);
+            }
+        } else {
+            out.writeOptionalWriteable(innerHitBuilder);
+        }
+        out.writeBoolean(ignoreUnmapped);
+    }
+
+    /**
+     * Defines the minimum number of children that are required to match for the parent to be considered a match and
+     * the maximum number of children that are required to match for the parent to be considered a match.
+     */
+    public HasChildQueryBuilder minMaxChildren(int minChildren, int maxChildren) {
+        if (minChildren < 0) {
+            throw new IllegalArgumentException("[" + NAME + "] requires non-negative 'min_children' field");
+        }
+        if (maxChildren < 0) {
+            throw new IllegalArgumentException("[" + NAME + "] requires non-negative 'max_children' field");
+        }
+        if (maxChildren < minChildren) {
+            throw new IllegalArgumentException("[" + NAME + "] 'max_children' is less than 'min_children'");
+        }
+        this.minChildren = minChildren;
+        this.maxChildren = maxChildren;
+        return this;
+    }
+
+    /**
+     * Returns inner hit definition in the scope of this query and reusing the defined type and query.
+     */
+    public InnerHitBuilder innerHit() {
+        return innerHitBuilder;
+    }
+
+    public HasChildQueryBuilder innerHit(InnerHitBuilder innerHit) {
+        this.innerHitBuilder = innerHit;
+        return this;
+    }
+
+    /**
+     * Returns the children query to execute.
+     */
+    public QueryBuilder query() {
+        return query;
+    }
+
+    /**
+     * Returns the child type
+     */
+    public String childType() {
+        return type;
+    }
+
+    /**
+     * Returns how the scores from the matching child documents are mapped into the parent document.
+     */
+    public ScoreMode scoreMode() {
+        return scoreMode;
+    }
+
+    /**
+     * Returns the minimum number of children that are required to match for the parent to be considered a match.
+     * The default is {@value #DEFAULT_MAX_CHILDREN}
+     */
+    public int minChildren() {
+        return minChildren;
+    }
+
+    /**
+     * Returns the maximum number of children that are required to match for the parent to be considered a match.
+     * The default is {@value #DEFAULT_MIN_CHILDREN}
+     */
+    public int maxChildren() { return maxChildren; }
+
+    /**
+     * Sets whether the query builder should ignore unmapped types (and run a
+     * {@link MatchNoDocsQuery} in place of this query) or throw an exception if
+     * the type is unmapped.
+     */
+    public HasChildQueryBuilder ignoreUnmapped(boolean ignoreUnmapped) {
+        this.ignoreUnmapped = ignoreUnmapped;
+        return this;
+    }
+
+    /**
+     * Gets whether the query builder will ignore unmapped types (and run a
+     * {@link MatchNoDocsQuery} in place of this query) or throw an exception if
+     * the type is unmapped.
+     */
+    public boolean ignoreUnmapped() {
+        return ignoreUnmapped;
+    }
+
+    @Override
+    protected void doXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject(NAME);
+        builder.field(QUERY_FIELD.getPreferredName());
+        query.toXContent(builder, params);
+        builder.field(TYPE_FIELD.getPreferredName(), type);
+        builder.field(SCORE_MODE_FIELD.getPreferredName(), NestedQueryBuilder.scoreModeAsString(scoreMode));
+        builder.field(MIN_CHILDREN_FIELD.getPreferredName(), minChildren);
+        builder.field(MAX_CHILDREN_FIELD.getPreferredName(), maxChildren);
+        builder.field(IGNORE_UNMAPPED_FIELD.getPreferredName(), ignoreUnmapped);
+        printBoostAndQueryName(builder);
+        if (innerHitBuilder != null) {
+            builder.field(INNER_HITS_FIELD.getPreferredName(), innerHitBuilder, params);
+        }
+        builder.endObject();
+    }
+
+    public static Optional<HasChildQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
+        XContentParser parser = parseContext.parser();
+        float boost = AbstractQueryBuilder.DEFAULT_BOOST;
+        String childType = null;
+        ScoreMode scoreMode = ScoreMode.None;
+        int minChildren = HasChildQueryBuilder.DEFAULT_MIN_CHILDREN;
+        int maxChildren = HasChildQueryBuilder.DEFAULT_MAX_CHILDREN;
+        boolean ignoreUnmapped = DEFAULT_IGNORE_UNMAPPED;
+        String queryName = null;
+        InnerHitBuilder innerHitBuilder = null;
+        String currentFieldName = null;
+        XContentParser.Token token;
+        Optional<QueryBuilder> iqb = Optional.empty();
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (parseContext.isDeprecatedSetting(currentFieldName)) {
+                // skip
+            } else if (token == XContentParser.Token.START_OBJECT) {
+                if (QUERY_FIELD.match(currentFieldName)) {
+                    iqb = parseContext.parseInnerQueryBuilder();
+                } else if (INNER_HITS_FIELD.match(currentFieldName)) {
+                    innerHitBuilder = InnerHitBuilder.fromXContent(parseContext);
+                } else {
+                    throw new ParsingException(parser.getTokenLocation(), "[has_child] query does not support [" + currentFieldName + "]");
+                }
+            } else if (token.isValue()) {
+                if (TYPE_FIELD.match(currentFieldName)) {
+                    childType = parser.text();
+                } else if (SCORE_MODE_FIELD.match(currentFieldName)) {
+                    scoreMode = NestedQueryBuilder.parseScoreMode(parser.text());
+                } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
+                    boost = parser.floatValue();
+                } else if (MIN_CHILDREN_FIELD.match(currentFieldName)) {
+                    minChildren = parser.intValue(true);
+                } else if (MAX_CHILDREN_FIELD.match(currentFieldName)) {
+                    maxChildren = parser.intValue(true);
+                } else if (IGNORE_UNMAPPED_FIELD.match(currentFieldName)) {
+                    ignoreUnmapped = parser.booleanValue();
+                } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
+                    queryName = parser.text();
+                } else {
+                    throw new ParsingException(parser.getTokenLocation(), "[has_child] query does not support [" + currentFieldName + "]");
+                }
+            }
+        }
+
+        if (iqb.isPresent() == false) {
+            // if inner query is empty, bubble this up to caller so they can decide how to deal with it
+            return Optional.empty();
+        }
+
+        HasChildQueryBuilder hasChildQueryBuilder = new HasChildQueryBuilder(childType, iqb.get(), scoreMode);
+        hasChildQueryBuilder.minMaxChildren(minChildren, maxChildren);
+        hasChildQueryBuilder.queryName(queryName);
+        hasChildQueryBuilder.boost(boost);
+        hasChildQueryBuilder.ignoreUnmapped(ignoreUnmapped);
+        if (innerHitBuilder != null) {
+            hasChildQueryBuilder.innerHit(innerHitBuilder);
+            hasChildQueryBuilder.ignoreUnmapped(ignoreUnmapped);
+        }
+        return Optional.of(hasChildQueryBuilder);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    protected Query doToQuery(QueryShardContext context) throws IOException {
+        if (context.getIndexSettings().isSingleType()) {
+            return joinFieldDoToQuery(context);
+        } else {
+            return parentFieldDoToQuery(context);
+        }
+    }
+
+    private Query joinFieldDoToQuery(QueryShardContext context) throws IOException {
+        ParentJoinFieldMapper joinFieldMapper = ParentJoinFieldMapper.getMapper(context.getMapperService());
+        if (joinFieldMapper == null) {
+            if (ignoreUnmapped) {
+                return new MatchNoDocsQuery();
+            } else {
+                throw new QueryShardException(context, "[" + NAME + "] no join field has been configured");
+            }
+        }
+
+        ParentIdFieldMapper parentIdFieldMapper = joinFieldMapper.getParentIdFieldMapper(type, false);
+        if (parentIdFieldMapper != null) {
+            Query parentFilter = parentIdFieldMapper.getParentFilter();
+            Query childFilter = parentIdFieldMapper.getChildFilter(type);
+            Query innerQuery = Queries.filtered(query.toQuery(context), childFilter);
+            MappedFieldType fieldType = parentIdFieldMapper.fieldType();
+            final SortedSetDVOrdinalsIndexFieldData fieldData = context.getForField(fieldType);
+            return new LateParsingQuery(parentFilter, innerQuery, minChildren(), maxChildren(),
+                fieldType.name(), scoreMode, fieldData, context.getSearchSimilarity());
+        } else {
+            if (ignoreUnmapped) {
+                return new MatchNoDocsQuery();
+            } else {
+                throw new QueryShardException(context, "[" + NAME + "] join field [" + joinFieldMapper.name() +
+                    "] doesn't hold [" + type + "] as a child");
+            }
+        }
+    }
+
+    private Query parentFieldDoToQuery(QueryShardContext context) throws IOException {
+        Query innerQuery;
+        final String[] previousTypes = context.getTypes();
+        context.setTypes(type);
+        try {
+            innerQuery = query.toQuery(context);
+        } finally {
+            context.setTypes(previousTypes);
+        }
+        DocumentMapper childDocMapper = context.getMapperService().documentMapper(type);
+        if (childDocMapper == null) {
+            if (ignoreUnmapped) {
+                return new MatchNoDocsQuery();
+            } else {
+                throw new QueryShardException(context, "[" + NAME + "] no mapping found for type [" + type + "]");
+            }
+        }
+        ParentFieldMapper parentFieldMapper = childDocMapper.parentFieldMapper();
+        if (parentFieldMapper.active() == false) {
+            throw new QueryShardException(context, "[" + NAME + "] _parent field has no parent type configured");
+        }
+        String parentType = parentFieldMapper.type();
+        DocumentMapper parentDocMapper = context.getMapperService().documentMapper(parentType);
+        if (parentDocMapper == null) {
+            throw new QueryShardException(context,
+                "[" + NAME + "] Type [" + type + "] points to a non existent parent type [" + parentType + "]");
+        }
+
+        // wrap the query with type query
+        innerQuery = Queries.filtered(innerQuery, childDocMapper.typeFilter(context));
+
+        String joinField = ParentFieldMapper.joinField(parentType);
+        final MappedFieldType parentFieldType = parentDocMapper.parentFieldMapper().getParentJoinFieldType();
+        final SortedSetDVOrdinalsIndexFieldData fieldData = context.getForField(parentFieldType);
+        return new LateParsingQuery(parentDocMapper.typeFilter(context), innerQuery, minChildren(), maxChildren(),
+            joinField, scoreMode, fieldData, context.getSearchSimilarity());
+    }
+
+    /**
+     * A query that rewrites into another query using
+     * {@link JoinUtil#createJoinQuery(String, Query, Query, IndexSearcher, ScoreMode, MultiDocValues.OrdinalMap, int, int)}
+     * that executes the actual join.
+     *
+     * This query is exclusively used by the {@link HasChildQueryBuilder} and {@link HasParentQueryBuilder} to get access
+     * to the {@link DirectoryReader} used by the current search in order to retrieve the {@link MultiDocValues.OrdinalMap}.
+     * The {@link MultiDocValues.OrdinalMap} is required by {@link JoinUtil} to execute the join.
+     */
+    // TODO: Find a way to remove this query and let doToQuery(...) just return the query from JoinUtil.createJoinQuery(...)
+    public static final class LateParsingQuery extends Query {
+
+        private final Query toQuery;
+        private final Query innerQuery;
+        private final int minChildren;
+        private final int maxChildren;
+        private final String joinField;
+        private final ScoreMode scoreMode;
+        private final SortedSetDVOrdinalsIndexFieldData fieldDataJoin;
+        private final Similarity similarity;
+
+        LateParsingQuery(Query toQuery, Query innerQuery, int minChildren, int maxChildren,
+                         String joinField, ScoreMode scoreMode,
+                         SortedSetDVOrdinalsIndexFieldData fieldData, Similarity similarity) {
+            this.toQuery = toQuery;
+            this.innerQuery = innerQuery;
+            this.minChildren = minChildren;
+            this.maxChildren = maxChildren;
+            this.joinField = joinField;
+            this.scoreMode = scoreMode;
+            this.fieldDataJoin = fieldData;
+            this.similarity = similarity;
+        }
+
+        @Override
+        public Query rewrite(IndexReader reader) throws IOException {
+            Query rewritten = super.rewrite(reader);
+            if (rewritten != this) {
+                return rewritten;
+            }
+            if (reader instanceof DirectoryReader) {
+                IndexSearcher indexSearcher = new IndexSearcher(reader);
+                indexSearcher.setQueryCache(null);
+                indexSearcher.setSimilarity(similarity);
+                IndexOrdinalsFieldData indexParentChildFieldData = fieldDataJoin.loadGlobal((DirectoryReader) reader);
+                MultiDocValues.OrdinalMap ordinalMap = indexParentChildFieldData.getOrdinalMap();
+                return JoinUtil.createJoinQuery(joinField, innerQuery, toQuery, indexSearcher, scoreMode,
+                    ordinalMap, minChildren, maxChildren);
+            } else {
+                if (reader.leaves().isEmpty() && reader.numDocs() == 0) {
+                    // asserting reader passes down a MultiReader during rewrite which makes this
+                    // blow up since for this query to work we have to have a DirectoryReader otherwise
+                    // we can't load global ordinals - for this to work we simply check if the reader has no leaves
+                    // and rewrite to match nothing
+                    return new MatchNoDocsQuery();
+                }
+                throw new IllegalStateException("can't load global ordinals for reader of type: " +
+                    reader.getClass() + " must be a DirectoryReader");
+            }
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (sameClassAs(o) == false) return false;
+
+            LateParsingQuery that = (LateParsingQuery) o;
+
+            if (minChildren != that.minChildren) return false;
+            if (maxChildren != that.maxChildren) return false;
+            if (!toQuery.equals(that.toQuery)) return false;
+            if (!innerQuery.equals(that.innerQuery)) return false;
+            if (!joinField.equals(that.joinField)) return false;
+            return scoreMode == that.scoreMode;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(getClass(), toQuery, innerQuery, minChildren, maxChildren, joinField, scoreMode);
+        }
+
+        @Override
+        public String toString(String s) {
+            return "LateParsingQuery {joinField=" + joinField + "}";
+        }
+
+        public int getMinChildren() {
+            return minChildren;
+        }
+
+        public int getMaxChildren() {
+            return maxChildren;
+        }
+
+        public ScoreMode getScoreMode() {
+            return scoreMode;
+        }
+
+        public Query getInnerQuery() {
+            return innerQuery;
+        }
+
+        public Similarity getSimilarity() {
+            return similarity;
+        }
+    }
+
+    @Override
+    protected boolean doEquals(HasChildQueryBuilder that) {
+        return Objects.equals(query, that.query)
+                && Objects.equals(type, that.type)
+                && Objects.equals(scoreMode, that.scoreMode)
+                && Objects.equals(minChildren, that.minChildren)
+                && Objects.equals(maxChildren, that.maxChildren)
+                && Objects.equals(innerHitBuilder, that.innerHitBuilder)
+                && Objects.equals(ignoreUnmapped, that.ignoreUnmapped);
+    }
+
+    @Override
+    protected int doHashCode() {
+        return Objects.hash(query, type, scoreMode, minChildren, maxChildren, innerHitBuilder, ignoreUnmapped);
+    }
+
+    @Override
+    protected QueryBuilder doRewrite(QueryRewriteContext queryShardContext) throws IOException {
+        QueryBuilder rewrittenQuery = query.rewrite(queryShardContext);
+        if (rewrittenQuery != query) {
+            HasChildQueryBuilder hasChildQueryBuilder =
+                new HasChildQueryBuilder(type, rewrittenQuery, minChildren, maxChildren, scoreMode, innerHitBuilder);
+            hasChildQueryBuilder.ignoreUnmapped(ignoreUnmapped);
+            return hasChildQueryBuilder;
+        }
+        return this;
+    }
+
+    @Override
+    protected void extractInnerHitBuilders(Map<String, InnerHitContextBuilder> innerHits) {
+        if (innerHitBuilder != null) {
+            Map<String, InnerHitContextBuilder> children = new HashMap<>();
+            InnerHitContextBuilder.extractInnerHits(query, children);
+            String name = innerHitBuilder.getName() != null ? innerHitBuilder.getName() : type;
+            InnerHitContextBuilder innerHitContextBuilder =
+                new ParentChildInnerHitContextBuilder(type, true, query, innerHitBuilder, children);
+            innerHits.put(name, innerHitContextBuilder);
+        }
+    }
+}
diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasParentQueryBuilder.java b/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasParentQueryBuilder.java
new file mode 100644
index 0000000000000..d09152869ac50
--- /dev/null
+++ b/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasParentQueryBuilder.java
@@ -0,0 +1,393 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.join.query;
+
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.MatchNoDocsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.join.ScoreMode;
+import org.elasticsearch.Version;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.lucene.search.Queries;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData;
+import org.elasticsearch.index.mapper.DocumentMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.ParentFieldMapper;
+import org.elasticsearch.index.query.AbstractQueryBuilder;
+import org.elasticsearch.index.query.InnerHitBuilder;
+import org.elasticsearch.index.query.InnerHitContextBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.index.query.QueryRewriteContext;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.query.QueryShardException;
+import org.elasticsearch.join.mapper.ParentIdFieldMapper;
+import org.elasticsearch.join.mapper.ParentJoinFieldMapper;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * Builder for the 'has_parent' query.
+ */
+public class HasParentQueryBuilder extends AbstractQueryBuilder<HasParentQueryBuilder> {
+    public static final String NAME = "has_parent";
+
+    /**
+     * The default value for ignore_unmapped.
+     */
+    public static final boolean DEFAULT_IGNORE_UNMAPPED = false;
+
+    private static final ParseField QUERY_FIELD = new ParseField("query", "filter");
+    private static final ParseField SCORE_MODE_FIELD = new ParseField("score_mode").withAllDeprecated("score");
+    private static final ParseField TYPE_FIELD = new ParseField("parent_type", "type");
+    private static final ParseField SCORE_FIELD = new ParseField("score");
+    private static final ParseField INNER_HITS_FIELD = new ParseField("inner_hits");
+    private static final ParseField IGNORE_UNMAPPED_FIELD = new ParseField("ignore_unmapped");
+
+    private final QueryBuilder query;
+    private final String type;
+    private final boolean score;
+    private InnerHitBuilder innerHitBuilder;
+    private boolean ignoreUnmapped = false;
+
+    public HasParentQueryBuilder(String type, QueryBuilder query, boolean score) {
+        this(type, query, score, null);
+    }
+
+    private HasParentQueryBuilder(String type, QueryBuilder query, boolean score, InnerHitBuilder innerHitBuilder) {
+        this.type = requireValue(type, "[" + NAME + "] requires 'type' field");
+        this.query = requireValue(query, "[" + NAME + "] requires 'query' field");
+        this.score = score;
+        this.innerHitBuilder = innerHitBuilder;
+    }
+
+    /**
+     * Read from a stream.
+     */
+    public HasParentQueryBuilder(StreamInput in) throws IOException {
+        super(in);
+        type = in.readString();
+        score = in.readBoolean();
+        query = in.readNamedWriteable(QueryBuilder.class);
+        innerHitBuilder = in.readOptionalWriteable(InnerHitBuilder::new);
+        ignoreUnmapped = in.readBoolean();
+    }
+
+    @Override
+    protected void doWriteTo(StreamOutput out) throws IOException {
+        out.writeString(type);
+        out.writeBoolean(score);
+        out.writeNamedWriteable(query);
+        if (out.getVersion().before(Version.V_5_5_0)) {
+            final boolean hasInnerHit = innerHitBuilder != null;
+            out.writeBoolean(hasInnerHit);
+            if (hasInnerHit) {
+                innerHitBuilder.writeToParentChildBWC(out, query, type);
+            }
+        } else {
+            out.writeOptionalWriteable(innerHitBuilder);
+        }
+        out.writeBoolean(ignoreUnmapped);
+    }
+
+    /**
+     * Returns the query to execute.
+     */
+    public QueryBuilder query() {
+        return query;
+    }
+
+    /**
+     * Returns <code>true</code> if the parent score is mapped into the child documents
+     */
+    public boolean score() {
+        return score;
+    }
+
+    /**
+     * Returns the parents type name
+     */
+    public String type() {
+        return type;
+    }
+
+    /**
+     *  Returns inner hit definition in the scope of this query and reusing the defined type and query.
+     */
+    public InnerHitBuilder innerHit() {
+        return innerHitBuilder;
+    }
+
+    public HasParentQueryBuilder innerHit(InnerHitBuilder innerHit) {
+        this.innerHitBuilder = innerHit;
+        return this;
+    }
+
+    /**
+     * Sets whether the query builder should ignore unmapped types (and run a
+     * {@link MatchNoDocsQuery} in place of this query) or throw an exception if
+     * the type is unmapped.
+     */
+    public HasParentQueryBuilder ignoreUnmapped(boolean ignoreUnmapped) {
+        this.ignoreUnmapped = ignoreUnmapped;
+        return this;
+    }
+
+    /**
+     * Gets whether the query builder will ignore unmapped types (and run a
+     * {@link MatchNoDocsQuery} in place of this query) or throw an exception if
+     * the type is unmapped.
+     */
+    public boolean ignoreUnmapped() {
+        return ignoreUnmapped;
+    }
+
+    @Override
+    protected Query doToQuery(QueryShardContext context) throws IOException {
+        if (context.getIndexSettings().isSingleType()) {
+            return joinFieldDoToQuery(context);
+        } else  {
+            return parentFieldDoToQuery(context);
+        }
+    }
+
+    private Query joinFieldDoToQuery(QueryShardContext context) throws IOException {
+        ParentJoinFieldMapper joinFieldMapper = ParentJoinFieldMapper.getMapper(context.getMapperService());
+        if (joinFieldMapper == null) {
+            if (ignoreUnmapped) {
+                return new MatchNoDocsQuery();
+            } else {
+                throw new QueryShardException(context, "[" + NAME + "] no join field has been configured");
+            }
+        }
+
+        ParentIdFieldMapper parentIdFieldMapper = joinFieldMapper.getParentIdFieldMapper(type, true);
+        if (parentIdFieldMapper != null) {
+            Query parentFilter = parentIdFieldMapper.getParentFilter();
+            Query innerQuery = Queries.filtered(query.toQuery(context), parentFilter);
+            Query childFilter = parentIdFieldMapper.getChildrenFilter();
+            MappedFieldType fieldType = parentIdFieldMapper.fieldType();
+            final SortedSetDVOrdinalsIndexFieldData fieldData = context.getForField(fieldType);
+            return new HasChildQueryBuilder.LateParsingQuery(childFilter, innerQuery,
+                HasChildQueryBuilder.DEFAULT_MIN_CHILDREN, HasChildQueryBuilder.DEFAULT_MAX_CHILDREN,
+                fieldType.name(), score ? ScoreMode.Max : ScoreMode.None, fieldData, context.getSearchSimilarity());
+        } else {
+            if (ignoreUnmapped) {
+                return new MatchNoDocsQuery();
+            } else {
+                throw new QueryShardException(context, "[" + NAME + "] join field [" + joinFieldMapper.name() +
+                    "] doesn't hold [" + type + "] as a parent");
+            }
+        }
+    }
+
+    private Query parentFieldDoToQuery(QueryShardContext context) throws IOException {
+        Query innerQuery;
+        String[] previousTypes = context.getTypes();
+        context.setTypes(type);
+        try {
+            innerQuery = query.toQuery(context);
+        } finally {
+            context.setTypes(previousTypes);
+        }
+
+        DocumentMapper parentDocMapper = context.documentMapper(type);
+        if (parentDocMapper == null) {
+            if (ignoreUnmapped) {
+                return new MatchNoDocsQuery();
+            } else {
+                throw new QueryShardException(context,
+                    "[" + NAME + "] query configured 'parent_type' [" + type + "] is not a valid type");
+            }
+        }
+
+        Set<String> childTypes = new HashSet<>();
+        for (DocumentMapper documentMapper : context.getMapperService().docMappers(false)) {
+            ParentFieldMapper parentFieldMapper = documentMapper.parentFieldMapper();
+            if (parentFieldMapper.active() && type.equals(parentFieldMapper.type())) {
+                childTypes.add(documentMapper.type());
+            }
+        }
+        if (childTypes.isEmpty()) {
+            throw new QueryShardException(context, "[" + NAME + "] no child types found for type [" + type + "]");
+        }
+
+        Query childrenQuery;
+        if (childTypes.size() == 1) {
+            DocumentMapper documentMapper = context.getMapperService().documentMapper(childTypes.iterator().next());
+            childrenQuery = documentMapper.typeFilter(context);
+        } else {
+            BooleanQuery.Builder childrenFilter = new BooleanQuery.Builder();
+            for (String childrenTypeStr : childTypes) {
+                DocumentMapper documentMapper = context.getMapperService().documentMapper(childrenTypeStr);
+                childrenFilter.add(documentMapper.typeFilter(context), BooleanClause.Occur.SHOULD);
+            }
+            childrenQuery = childrenFilter.build();
+        }
+
+        // wrap the query with type query
+        innerQuery = Queries.filtered(innerQuery, parentDocMapper.typeFilter(context));
+
+        final MappedFieldType parentType = parentDocMapper.parentFieldMapper().getParentJoinFieldType();
+        final SortedSetDVOrdinalsIndexFieldData fieldData = context.getForField(parentType);
+        return new HasChildQueryBuilder.LateParsingQuery(childrenQuery,
+            innerQuery,
+            HasChildQueryBuilder.DEFAULT_MIN_CHILDREN,
+            HasChildQueryBuilder.DEFAULT_MAX_CHILDREN,
+            ParentFieldMapper.joinField(type),
+            score ? ScoreMode.Max : ScoreMode.None,
+            fieldData,
+            context.getSearchSimilarity());
+    }
+
+    @Override
+    protected void doXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject(NAME);
+        builder.field(QUERY_FIELD.getPreferredName());
+        query.toXContent(builder, params);
+        builder.field(TYPE_FIELD.getPreferredName(), type);
+        builder.field(SCORE_FIELD.getPreferredName(), score);
+        builder.field(IGNORE_UNMAPPED_FIELD.getPreferredName(), ignoreUnmapped);
+        printBoostAndQueryName(builder);
+        if (innerHitBuilder != null) {
+            builder.field(INNER_HITS_FIELD.getPreferredName(), innerHitBuilder, params);
+        }
+        builder.endObject();
+    }
+
+    public static Optional<HasParentQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
+        XContentParser parser = parseContext.parser();
+        float boost = AbstractQueryBuilder.DEFAULT_BOOST;
+        String parentType = null;
+        boolean score = false;
+        String queryName = null;
+        InnerHitBuilder innerHits = null;
+        boolean ignoreUnmapped = DEFAULT_IGNORE_UNMAPPED;
+
+        String currentFieldName = null;
+        XContentParser.Token token;
+        Optional<QueryBuilder> iqb = Optional.empty();
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token == XContentParser.Token.START_OBJECT) {
+                if (QUERY_FIELD.match(currentFieldName)) {
+                    iqb = parseContext.parseInnerQueryBuilder();
+                } else if (INNER_HITS_FIELD.match(currentFieldName)) {
+                    innerHits = InnerHitBuilder.fromXContent(parseContext);
+                } else {
+                    throw new ParsingException(parser.getTokenLocation(),
+                        "[has_parent] query does not support [" + currentFieldName + "]");
+                }
+            } else if (token.isValue()) {
+                if (TYPE_FIELD.match(currentFieldName)) {
+                    parentType = parser.text();
+                } else if (SCORE_MODE_FIELD.match(currentFieldName)) {
+                    String scoreModeValue = parser.text();
+                    if ("score".equals(scoreModeValue)) {
+                        score = true;
+                    } else if ("none".equals(scoreModeValue)) {
+                        score = false;
+                    } else {
+                        throw new ParsingException(parser.getTokenLocation(), "[has_parent] query does not support [" +
+                                scoreModeValue + "] as an option for score_mode");
+                    }
+                } else if (SCORE_FIELD.match(currentFieldName)) {
+                    score = parser.booleanValue();
+                } else if (IGNORE_UNMAPPED_FIELD.match(currentFieldName)) {
+                    ignoreUnmapped = parser.booleanValue();
+                } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
+                    boost = parser.floatValue();
+                } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
+                    queryName = parser.text();
+                } else {
+                    throw new ParsingException(parser.getTokenLocation(),
+                        "[has_parent] query does not support [" + currentFieldName + "]");
+                }
+            }
+        }
+        if (iqb.isPresent() == false) {
+            // if inner query is empty, bubble this up to caller so they can decide how to deal with it
+            return Optional.empty();
+        }
+        HasParentQueryBuilder queryBuilder =  new HasParentQueryBuilder(parentType, iqb.get(), score)
+                .ignoreUnmapped(ignoreUnmapped)
+                .queryName(queryName)
+                .boost(boost);
+        if (innerHits != null) {
+            queryBuilder.innerHit(innerHits);
+        }
+        return Optional.of(queryBuilder);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    protected boolean doEquals(HasParentQueryBuilder that) {
+        return Objects.equals(query, that.query)
+                && Objects.equals(type, that.type)
+                && Objects.equals(score, that.score)
+                && Objects.equals(innerHitBuilder, that.innerHitBuilder)
+                && Objects.equals(ignoreUnmapped, that.ignoreUnmapped);
+    }
+
+    @Override
+    protected int doHashCode() {
+        return Objects.hash(query, type, score, innerHitBuilder, ignoreUnmapped);
+    }
+
+    @Override
+    protected QueryBuilder doRewrite(QueryRewriteContext queryShardContext) throws IOException {
+        QueryBuilder rewrittenQuery = query.rewrite(queryShardContext);
+        if (rewrittenQuery != query) {
+            HasParentQueryBuilder hasParentQueryBuilder = new HasParentQueryBuilder(type, rewrittenQuery, score, innerHitBuilder);
+            hasParentQueryBuilder.ignoreUnmapped(ignoreUnmapped);
+            return hasParentQueryBuilder;
+        }
+        return this;
+    }
+
+    @Override
+    protected void extractInnerHitBuilders(Map<String, InnerHitContextBuilder> innerHits) {
+        if (innerHitBuilder != null) {
+            Map<String, InnerHitContextBuilder> children = new HashMap<>();
+            InnerHitContextBuilder.extractInnerHits(query, children);
+            String name = innerHitBuilder.getName() != null ? innerHitBuilder.getName() : type;
+            InnerHitContextBuilder innerHitContextBuilder =
+                new ParentChildInnerHitContextBuilder(type, false, query, innerHitBuilder, children);
+            innerHits.put(name, innerHitContextBuilder);
+        }
+    }
+
+}
diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/query/JoinQueryBuilders.java b/modules/parent-join/src/main/java/org/elasticsearch/join/query/JoinQueryBuilders.java
new file mode 100644
index 0000000000000..d3ef43b7f2f1a
--- /dev/null
+++ b/modules/parent-join/src/main/java/org/elasticsearch/join/query/JoinQueryBuilders.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.join.query;
+
+import org.apache.lucene.search.join.ScoreMode;
+import org.elasticsearch.index.query.QueryBuilder;
+
+public abstract class JoinQueryBuilders {
+    /**
+     * Constructs a new has_child query, with the child type and the query to run on the child documents. The
+     * results of this query are the parent docs that those child docs matched.
+     *
+     * @param type      The child type.
+     * @param query     The query.
+     * @param scoreMode How the scores from the children hits should be aggregated into the parent hit.
+     */
+    public static HasChildQueryBuilder hasChildQuery(String type, QueryBuilder query, ScoreMode scoreMode) {
+        return new HasChildQueryBuilder(type, query, scoreMode);
+    }
+
+    /**
+     * Constructs a new parent query, with the parent type and the query to run on the parent documents. The
+     * results of this query are the children docs that those parent docs matched.
+     *
+     * @param type      The parent type.
+     * @param query     The query.
+     * @param score     Whether the score from the parent hit should propagate to the child hit
+     */
+    public static HasParentQueryBuilder hasParentQuery(String type, QueryBuilder query, boolean score) {
+        return new HasParentQueryBuilder(type, query, score);
+    }
+
+    /**
+     * Constructs a new parent id query that returns all child documents of the specified type that
+     * point to the specified id.
+     */
+    public static ParentIdQueryBuilder parentId(String type, String id) {
+        return new ParentIdQueryBuilder(type, id);
+    }
+}
diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/query/ParentChildInnerHitContextBuilder.java b/modules/parent-join/src/main/java/org/elasticsearch/join/query/ParentChildInnerHitContextBuilder.java
new file mode 100644
index 0000000000000..10af4ec19c33a
--- /dev/null
+++ b/modules/parent-join/src/main/java/org/elasticsearch/join/query/ParentChildInnerHitContextBuilder.java
@@ -0,0 +1,293 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.join.query;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.ReaderUtil;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.DocValuesTermsQuery;
+import org.apache.lucene.search.MatchNoDocsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.TopDocsCollector;
+import org.apache.lucene.search.TopFieldCollector;
+import org.apache.lucene.search.TopScoreDocCollector;
+import org.apache.lucene.search.TotalHitCountCollector;
+import org.apache.lucene.search.Weight;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.common.lucene.Lucene;
+import org.elasticsearch.index.mapper.DocumentMapper;
+import org.elasticsearch.index.mapper.IdFieldMapper;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.ParentFieldMapper;
+import org.elasticsearch.index.query.InnerHitBuilder;
+import org.elasticsearch.index.query.InnerHitContextBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.join.mapper.ParentIdFieldMapper;
+import org.elasticsearch.join.mapper.ParentJoinFieldMapper;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchHitField;
+import org.elasticsearch.search.fetch.subphase.InnerHitsContext;
+import org.elasticsearch.search.internal.SearchContext;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.search.fetch.subphase.InnerHitsContext.intersect;
+
+class ParentChildInnerHitContextBuilder extends InnerHitContextBuilder {
+    private final String typeName;
+    private final boolean fetchChildInnerHits;
+
+    ParentChildInnerHitContextBuilder(String typeName, boolean fetchChildInnerHits, QueryBuilder query,
+                                      InnerHitBuilder innerHitBuilder, Map<String, InnerHitContextBuilder> children) {
+        super(query, innerHitBuilder, children);
+        this.typeName = typeName;
+        this.fetchChildInnerHits = fetchChildInnerHits;
+    }
+
+    @Override
+    public void build(SearchContext parentSearchContext, InnerHitsContext innerHitsContext) throws IOException {
+        if (parentSearchContext.mapperService().getIndexSettings().isSingleType()) {
+            handleJoinFieldInnerHits(parentSearchContext, innerHitsContext);
+        } else {
+            handleParentFieldInnerHits(parentSearchContext, innerHitsContext);
+        }
+    }
+
+    private void handleJoinFieldInnerHits(SearchContext context, InnerHitsContext innerHitsContext) throws IOException {
+        QueryShardContext queryShardContext = context.getQueryShardContext();
+        ParentJoinFieldMapper joinFieldMapper = ParentJoinFieldMapper.getMapper(context.mapperService());
+        if (joinFieldMapper != null) {
+            String name = innerHitBuilder.getName() != null ? innerHitBuilder.getName() : typeName;
+            JoinFieldInnerHitSubContext joinFieldInnerHits = new JoinFieldInnerHitSubContext(name, context, typeName,
+                fetchChildInnerHits, joinFieldMapper);
+            setupInnerHitsContext(queryShardContext, joinFieldInnerHits);
+            innerHitsContext.addInnerHitDefinition(joinFieldInnerHits);
+        } else {
+            if (innerHitBuilder.isIgnoreUnmapped() == false) {
+                throw new IllegalStateException("no join field has been configured");
+            }
+        }
+    }
+
+    private void handleParentFieldInnerHits(SearchContext context, InnerHitsContext innerHitsContext) throws IOException {
+        QueryShardContext queryShardContext = context.getQueryShardContext();
+        DocumentMapper documentMapper = queryShardContext.documentMapper(typeName);
+        if (documentMapper == null) {
+            if (innerHitBuilder.isIgnoreUnmapped() == false) {
+                throw new IllegalStateException("[" + query.getName() + "] no mapping found for type [" + typeName + "]");
+            } else {
+                return;
+            }
+        }
+        String name = innerHitBuilder.getName() != null ? innerHitBuilder.getName() : documentMapper.type();
+        ParentChildInnerHitSubContext parentChildInnerHits = new ParentChildInnerHitSubContext(
+            name, context, queryShardContext.getMapperService(), documentMapper
+        );
+        setupInnerHitsContext(queryShardContext, parentChildInnerHits);
+        innerHitsContext.addInnerHitDefinition(parentChildInnerHits);
+    }
+
+    static final class JoinFieldInnerHitSubContext extends InnerHitsContext.InnerHitSubContext {
+        private final String typeName;
+        private final boolean fetchChildInnerHits;
+        private final ParentJoinFieldMapper joinFieldMapper;
+
+        JoinFieldInnerHitSubContext(String name, SearchContext context, String typeName, boolean fetchChildInnerHits,
+                                    ParentJoinFieldMapper joinFieldMapper) {
+            super(name, context);
+            this.typeName = typeName;
+            this.fetchChildInnerHits = fetchChildInnerHits;
+            this.joinFieldMapper = joinFieldMapper;
+        }
+
+        @Override
+        public TopDocs[] topDocs(SearchHit[] hits) throws IOException {
+            Weight innerHitQueryWeight = createInnerHitQueryWeight();
+            TopDocs[] result = new TopDocs[hits.length];
+            for (int i = 0; i < hits.length; i++) {
+                SearchHit hit = hits[i];
+                String joinName = getSortedDocValue(joinFieldMapper.name(), context, hit.docId());
+                if (joinName == null) {
+                    result[i] = Lucene.EMPTY_TOP_DOCS;
+                    continue;
+                }
+
+                QueryShardContext qsc = context.getQueryShardContext();
+                ParentIdFieldMapper parentIdFieldMapper =
+                    joinFieldMapper.getParentIdFieldMapper(typeName, fetchChildInnerHits == false);
+                if (parentIdFieldMapper == null) {
+                    result[i] = Lucene.EMPTY_TOP_DOCS;
+                    continue;
+                }
+
+                Query q;
+                if (fetchChildInnerHits) {
+                    Query hitQuery = parentIdFieldMapper.fieldType().termQuery(hit.getId(), qsc);
+                    q = new BooleanQuery.Builder()
+                        // Only include child documents that have the current hit as parent:
+                        .add(hitQuery, BooleanClause.Occur.FILTER)
+                        // and only include child documents of a single relation:
+                        .add(joinFieldMapper.fieldType().termQuery(typeName, qsc), BooleanClause.Occur.FILTER)
+                        .build();
+                } else {
+                    String parentId = getSortedDocValue(parentIdFieldMapper.name(), context, hit.docId());
+                    q = context.mapperService().fullName(IdFieldMapper.NAME).termQuery(parentId, qsc);
+                }
+
+                Weight weight = context.searcher().createNormalizedWeight(q, false);
+                if (size() == 0) {
+                    TotalHitCountCollector totalHitCountCollector = new TotalHitCountCollector();
+                    for (LeafReaderContext ctx : context.searcher().getIndexReader().leaves()) {
+                        intersect(weight, innerHitQueryWeight, totalHitCountCollector, ctx);
+                    }
+                    result[i] = new TopDocs(totalHitCountCollector.getTotalHits(), Lucene.EMPTY_SCORE_DOCS, 0);
+                } else {
+                    int topN = Math.min(from() + size(), context.searcher().getIndexReader().maxDoc());
+                    TopDocsCollector<?> topDocsCollector;
+                    if (sort() != null) {
+                        topDocsCollector = TopFieldCollector.create(sort().sort, topN, true, trackScores(), trackScores());
+                    } else {
+                        topDocsCollector = TopScoreDocCollector.create(topN);
+                    }
+                    try {
+                        for (LeafReaderContext ctx : context.searcher().getIndexReader().leaves()) {
+                            intersect(weight, innerHitQueryWeight, topDocsCollector, ctx);
+                        }
+                    } finally {
+                        clearReleasables(Lifetime.COLLECTION);
+                    }
+                    result[i] = topDocsCollector.topDocs(from(), size());
+                }
+            }
+            return result;
+        }
+
+        private String getSortedDocValue(String field, SearchContext context, int docId) {
+            try {
+                List<LeafReaderContext> ctxs = context.searcher().getIndexReader().leaves();
+                LeafReaderContext ctx = ctxs.get(ReaderUtil.subIndex(docId, ctxs));
+                SortedDocValues docValues = ctx.reader().getSortedDocValues(field);
+                int segmentDocId = docId - ctx.docBase;
+                if (docValues == null) {
+                    return null;
+                }
+                int ord = docValues.getOrd(segmentDocId);
+                if (ord == -1) {
+                    return null;
+                }
+                BytesRef joinName = docValues.lookupOrd(ord);
+                return joinName.utf8ToString();
+            } catch (IOException e) {
+                throw ExceptionsHelper.convertToElastic(e);
+            }
+        }
+
+    }
+
+    static final class ParentChildInnerHitSubContext extends InnerHitsContext.InnerHitSubContext {
+        private final MapperService mapperService;
+        private final DocumentMapper documentMapper;
+
+        ParentChildInnerHitSubContext(String name, SearchContext context, MapperService mapperService, DocumentMapper documentMapper) {
+            super(name, context);
+            this.mapperService = mapperService;
+            this.documentMapper = documentMapper;
+        }
+
+        @Override
+        public TopDocs[] topDocs(SearchHit[] hits) throws IOException {
+            Weight innerHitQueryWeight = createInnerHitQueryWeight();
+            TopDocs[] result = new TopDocs[hits.length];
+            for (int i = 0; i < hits.length; i++) {
+                SearchHit hit = hits[i];
+                final Query hitQuery;
+                if (isParentHit(hit)) {
+                    String field = ParentFieldMapper.joinField(hit.getType());
+                    hitQuery = new DocValuesTermsQuery(field, hit.getId());
+                } else if (isChildHit(hit)) {
+                    DocumentMapper hitDocumentMapper = mapperService.documentMapper(hit.getType());
+                    final String parentType = hitDocumentMapper.parentFieldMapper().type();
+                    SearchHitField parentField = hit.field(ParentFieldMapper.NAME);
+                    if (parentField == null) {
+                        throw new IllegalStateException("All children must have a _parent");
+                    }
+                    Term uidTerm = context.mapperService().createUidTerm(parentType, parentField.getValue());
+                    if (uidTerm == null) {
+                        hitQuery = new MatchNoDocsQuery("Missing type: " + parentType);
+                    } else {
+                        hitQuery = new TermQuery(uidTerm);
+                    }
+                } else {
+                    result[i] = Lucene.EMPTY_TOP_DOCS;
+                    continue;
+                }
+
+                BooleanQuery q = new BooleanQuery.Builder()
+                    // Only include docs that have the current hit as parent
+                    .add(hitQuery, BooleanClause.Occur.FILTER)
+                    // Only include docs that have this inner hits type
+                    .add(documentMapper.typeFilter(context.getQueryShardContext()), BooleanClause.Occur.FILTER)
+                    .build();
+                Weight weight = context.searcher().createNormalizedWeight(q, false);
+                if (size() == 0) {
+                    TotalHitCountCollector totalHitCountCollector = new TotalHitCountCollector();
+                    for (LeafReaderContext ctx : context.searcher().getIndexReader().leaves()) {
+                        intersect(weight, innerHitQueryWeight, totalHitCountCollector, ctx);
+                    }
+                    result[i] = new TopDocs(totalHitCountCollector.getTotalHits(), Lucene.EMPTY_SCORE_DOCS, 0);
+                } else {
+                    int topN = Math.min(from() + size(), context.searcher().getIndexReader().maxDoc());
+                    TopDocsCollector<?> topDocsCollector;
+                    if (sort() != null) {
+                        topDocsCollector = TopFieldCollector.create(sort().sort, topN, true, trackScores(), trackScores());
+                    } else {
+                        topDocsCollector = TopScoreDocCollector.create(topN);
+                    }
+                    try {
+                        for (LeafReaderContext ctx : context.searcher().getIndexReader().leaves()) {
+                            intersect(weight, innerHitQueryWeight, topDocsCollector, ctx);
+                        }
+                    } finally {
+                        clearReleasables(Lifetime.COLLECTION);
+                    }
+                    result[i] = topDocsCollector.topDocs(from(), size());
+                }
+            }
+            return result;
+        }
+
+        private boolean isParentHit(SearchHit hit) {
+            return hit.getType().equals(documentMapper.parentFieldMapper().type());
+        }
+
+        private boolean isChildHit(SearchHit hit) {
+            DocumentMapper hitDocumentMapper = mapperService.documentMapper(hit.getType());
+            return documentMapper.type().equals(hitDocumentMapper.parentFieldMapper().type());
+        }
+    }
+}
diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/query/ParentIdQueryBuilder.java b/modules/parent-join/src/main/java/org/elasticsearch/join/query/ParentIdQueryBuilder.java
new file mode 100644
index 0000000000000..29ff3d38cbedc
--- /dev/null
+++ b/modules/parent-join/src/main/java/org/elasticsearch/join/query/ParentIdQueryBuilder.java
@@ -0,0 +1,236 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.join.query;
+
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.DocValuesTermsQuery;
+import org.apache.lucene.search.MatchNoDocsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.mapper.DocumentMapper;
+import org.elasticsearch.index.mapper.ParentFieldMapper;
+import org.elasticsearch.index.mapper.TypeFieldMapper;
+import org.elasticsearch.index.query.AbstractQueryBuilder;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.query.QueryShardException;
+import org.elasticsearch.join.mapper.ParentIdFieldMapper;
+import org.elasticsearch.join.mapper.ParentJoinFieldMapper;
+
+import java.io.IOException;
+import java.util.Objects;
+import java.util.Optional;
+
+public final class ParentIdQueryBuilder extends AbstractQueryBuilder<ParentIdQueryBuilder> {
+    public static final String NAME = "parent_id";
+
+    /**
+     * The default value for ignore_unmapped.
+     */
+    public static final boolean DEFAULT_IGNORE_UNMAPPED = false;
+
+    private static final ParseField ID_FIELD = new ParseField("id");
+    private static final ParseField TYPE_FIELD = new ParseField("type", "child_type");
+    private static final ParseField IGNORE_UNMAPPED_FIELD = new ParseField("ignore_unmapped");
+
+    private final String type;
+    private final String id;
+
+    private boolean ignoreUnmapped = false;
+
+    public ParentIdQueryBuilder(String type, String id) {
+        this.type = type;
+        this.id = id;
+    }
+
+    /**
+     * Read from a stream.
+     */
+    public ParentIdQueryBuilder(StreamInput in) throws IOException {
+        super(in);
+        type = in.readString();
+        id = in.readString();
+        ignoreUnmapped = in.readBoolean();
+    }
+
+    @Override
+    protected void doWriteTo(StreamOutput out) throws IOException {
+        out.writeString(type);
+        out.writeString(id);
+        out.writeBoolean(ignoreUnmapped);
+    }
+
+    public String getType() {
+        return type;
+    }
+
+    public String getId() {
+        return id;
+    }
+
+    /**
+     * Sets whether the query builder should ignore unmapped types (and run a
+     * {@link MatchNoDocsQuery} in place of this query) or throw an exception if
+     * the type is unmapped.
+     */
+    public ParentIdQueryBuilder ignoreUnmapped(boolean ignoreUnmapped) {
+        this.ignoreUnmapped = ignoreUnmapped;
+        return this;
+    }
+
+    /**
+     * Gets whether the query builder will ignore unmapped types (and run a
+     * {@link MatchNoDocsQuery} in place of this query) or throw an exception if
+     * the type is unmapped.
+     */
+    public boolean ignoreUnmapped() {
+        return ignoreUnmapped;
+    }
+
+    @Override
+    protected void doXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject(NAME);
+        builder.field(TYPE_FIELD.getPreferredName(), type);
+        builder.field(ID_FIELD.getPreferredName(), id);
+        builder.field(IGNORE_UNMAPPED_FIELD.getPreferredName(), ignoreUnmapped);
+        printBoostAndQueryName(builder);
+        builder.endObject();
+    }
+
+    public static Optional<ParentIdQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
+        XContentParser parser = parseContext.parser();
+        float boost = AbstractQueryBuilder.DEFAULT_BOOST;
+        String type = null;
+        String id = null;
+        String queryName = null;
+        String currentFieldName = null;
+        boolean ignoreUnmapped = DEFAULT_IGNORE_UNMAPPED;
+        XContentParser.Token token;
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token.isValue()) {
+                if (TYPE_FIELD.match(currentFieldName)) {
+                    type = parser.text();
+                } else if (ID_FIELD.match(currentFieldName)) {
+                    id = parser.text();
+                } else if (IGNORE_UNMAPPED_FIELD.match(currentFieldName)) {
+                    ignoreUnmapped = parser.booleanValue();
+                } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
+                    boost = parser.floatValue();
+                } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
+                    queryName = parser.text();
+                } else {
+                    throw new ParsingException(parser.getTokenLocation(), "[parent_id] query does not support [" + currentFieldName + "]");
+                }
+            } else {
+                throw new ParsingException(parser.getTokenLocation(), "[parent_id] query does not support [" + currentFieldName + "]");
+            }
+        }
+        ParentIdQueryBuilder queryBuilder = new ParentIdQueryBuilder(type, id);
+        queryBuilder.queryName(queryName);
+        queryBuilder.boost(boost);
+        queryBuilder.ignoreUnmapped(ignoreUnmapped);
+        return Optional.of(queryBuilder);
+    }
+
+
+    @Override
+    protected Query doToQuery(QueryShardContext context) throws IOException {
+        if (context.getIndexSettings().isSingleType() == false) {
+            // BWC for indices with multiple types
+            return doToQueryBWC(context);
+        }
+
+        ParentJoinFieldMapper joinFieldMapper = ParentJoinFieldMapper.getMapper(context.getMapperService());
+        if (joinFieldMapper == null) {
+            if (ignoreUnmapped) {
+                return new MatchNoDocsQuery();
+            } else {
+                final String indexName = context.getIndexSettings().getIndex().getName();
+                throw new QueryShardException(context, "[" + NAME + "] no join field found for index [" + indexName  + "]");
+            }
+        }
+        final ParentIdFieldMapper childMapper = joinFieldMapper.getParentIdFieldMapper(type, false);
+        if (childMapper == null) {
+            if (ignoreUnmapped) {
+                return new MatchNoDocsQuery();
+            } else {
+                throw new QueryShardException(context, "[" + NAME + "] no relation found for child [" + type + "]");
+            }
+        }
+        return new BooleanQuery.Builder()
+            .add(childMapper.fieldType().termQuery(id, context), BooleanClause.Occur.MUST)
+            // Need to take child type into account, otherwise a child doc of different type with the same id could match
+            .add(joinFieldMapper.fieldType().termQuery(type, context), BooleanClause.Occur.FILTER)
+            .build();
+    }
+
+    /**
+     * Creates parent_id query from a {@link ParentFieldMapper}
+     * Only used for BWC with multi-types indices
+     */
+    private Query doToQueryBWC(QueryShardContext context) throws IOException {
+        DocumentMapper childDocMapper = context.getMapperService().documentMapper(type);
+        if (childDocMapper == null) {
+            if (ignoreUnmapped) {
+                return new MatchNoDocsQuery();
+            } else {
+                throw new QueryShardException(context, "[" + NAME + "] no mapping found for type [" + type + "]");
+            }
+        }
+        ParentFieldMapper parentFieldMapper = childDocMapper.parentFieldMapper();
+        if (parentFieldMapper.active() == false) {
+            throw new QueryShardException(context, "[" + NAME + "] _parent field has no parent type configured");
+        }
+        String fieldName = ParentFieldMapper.joinField(parentFieldMapper.type());
+
+        return new BooleanQuery.Builder()
+            .add(new DocValuesTermsQuery(fieldName, id), BooleanClause.Occur.MUST)
+            // Need to take child type into account, otherwise a child doc of different type with the same id could match
+            .add(new TermQuery(new Term(TypeFieldMapper.NAME, type)), BooleanClause.Occur.FILTER)
+            .build();
+    }
+
+    @Override
+    protected boolean doEquals(ParentIdQueryBuilder that) {
+        return Objects.equals(type, that.type)
+                && Objects.equals(id, that.id)
+                && Objects.equals(ignoreUnmapped, that.ignoreUnmapped);
+    }
+
+    @Override
+    protected int doHashCode() {
+        return Objects.hash(type, id, ignoreUnmapped);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+}
diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/spi/ParentJoinNamedXContentProvider.java b/modules/parent-join/src/main/java/org/elasticsearch/join/spi/ParentJoinNamedXContentProvider.java
new file mode 100644
index 0000000000000..250241014613c
--- /dev/null
+++ b/modules/parent-join/src/main/java/org/elasticsearch/join/spi/ParentJoinNamedXContentProvider.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.join.spi;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ContextParser;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.join.aggregations.ChildrenAggregationBuilder;
+import org.elasticsearch.join.aggregations.ParsedChildren;
+import org.elasticsearch.plugins.spi.NamedXContentProvider;
+import org.elasticsearch.search.aggregations.Aggregation;
+
+import java.util.List;
+
+import static java.util.Collections.singletonList;
+
+public class ParentJoinNamedXContentProvider implements NamedXContentProvider {
+
+    @Override
+    public List<NamedXContentRegistry.Entry> getNamedXContentParsers() {
+        ParseField parseField = new ParseField(ChildrenAggregationBuilder.NAME);
+        ContextParser<Object, Aggregation> contextParser = (p, name) -> ParsedChildren.fromXContent(p, (String) name);
+        return singletonList(new NamedXContentRegistry.Entry(Aggregation.class, parseField, contextParser));
+    }
+}
diff --git a/modules/parent-join/src/main/resources/META-INF/services/org.elasticsearch.plugins.spi.NamedXContentProvider b/modules/parent-join/src/main/resources/META-INF/services/org.elasticsearch.plugins.spi.NamedXContentProvider
new file mode 100644
index 0000000000000..48687c21c3250
--- /dev/null
+++ b/modules/parent-join/src/main/resources/META-INF/services/org.elasticsearch.plugins.spi.NamedXContentProvider
@@ -0,0 +1 @@
+org.elasticsearch.join.spi.ParentJoinNamedXContentProvider
\ No newline at end of file
diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/ParentChildClientYamlTestSuiteIT.java b/modules/parent-join/src/test/java/org/elasticsearch/join/ParentChildClientYamlTestSuiteIT.java
new file mode 100644
index 0000000000000..666fa736d45de
--- /dev/null
+++ b/modules/parent-join/src/test/java/org/elasticsearch/join/ParentChildClientYamlTestSuiteIT.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.join;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
+import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+
+public class ParentChildClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
+    public ParentChildClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return createParameters();
+    }
+}
diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenIT.java b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenIT.java
new file mode 100644
index 0000000000000..4f6d32ee91b11
--- /dev/null
+++ b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenIT.java
@@ -0,0 +1,522 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.join.aggregations;
+
+import org.apache.lucene.search.join.ScoreMode;
+import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.update.UpdateResponse;
+import org.elasticsearch.client.Requests;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.join.query.ParentChildTestCase;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.aggregations.AggregationBuilders;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.bucket.terms.Terms;
+import org.elasticsearch.search.aggregations.metrics.sum.Sum;
+import org.elasticsearch.search.aggregations.metrics.tophits.TopHits;
+import org.elasticsearch.search.sort.SortOrder;
+import org.junit.Before;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
+import static org.elasticsearch.index.query.QueryBuilders.termQuery;
+import static org.elasticsearch.join.aggregations.JoinAggregationBuilders.children;
+import static org.elasticsearch.join.query.JoinQueryBuilders.hasChildQuery;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.sum;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.terms;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.topHits;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.sameInstance;
+
+public class ChildrenIT extends ParentChildTestCase {
+    private static final Map<String, Control> categoryToControl = new HashMap<>();
+
+    @Before
+    public void setupCluster() throws Exception {
+        categoryToControl.clear();
+        if (legacy()) {
+            assertAcked(
+                prepareCreate("test")
+                    .addMapping("article", "category", "type=keyword")
+                    .addMapping("comment", "_parent", "type=article", "commenter", "type=keyword")
+            );
+        } else {
+            assertAcked(
+                prepareCreate("test")
+                    .addMapping("doc",
+                        addFieldMappings(buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "article", "comment"),
+                            "commenter", "keyword", "category", "keyword"))
+            );
+        }
+
+        List<IndexRequestBuilder> requests = new ArrayList<>();
+        String[] uniqueCategories = new String[randomIntBetween(1, 25)];
+        for (int i = 0; i < uniqueCategories.length; i++) {
+            uniqueCategories[i] = Integer.toString(i);
+        }
+        int catIndex = 0;
+
+        int numParentDocs = randomIntBetween(uniqueCategories.length, uniqueCategories.length * 5);
+        for (int i = 0; i < numParentDocs; i++) {
+            String id = "article-" + i;
+
+            // TODO: this array is always of length 1, and testChildrenAggs fails if this is changed
+            String[] categories = new String[randomIntBetween(1,1)];
+            for (int j = 0; j < categories.length; j++) {
+                String category = categories[j] = uniqueCategories[catIndex++ % uniqueCategories.length];
+                Control control = categoryToControl.get(category);
+                if (control == null) {
+                    categoryToControl.put(category, control = new Control(category));
+                }
+                control.articleIds.add(id);
+            }
+
+            requests.add(createIndexRequest("test", "article", id, null, "category", categories, "randomized", true));
+        }
+
+        String[] commenters = new String[randomIntBetween(5, 50)];
+        for (int i = 0; i < commenters.length; i++) {
+            commenters[i] = Integer.toString(i);
+        }
+
+        int id = 0;
+        for (Control control : categoryToControl.values()) {
+            for (String articleId : control.articleIds) {
+                int numChildDocsPerParent = randomIntBetween(0, 5);
+                for (int i = 0; i < numChildDocsPerParent; i++) {
+                    String commenter = commenters[id % commenters.length];
+                    String idValue = "comment-" + id++;
+                    control.commentIds.add(idValue);
+                    Set<String> ids = control.commenterToCommentId.get(commenter);
+                    if (ids == null) {
+                        control.commenterToCommentId.put(commenter, ids = new HashSet<>());
+                    }
+                    ids.add(idValue);
+                    requests.add(createIndexRequest("test", "comment", idValue, articleId, "commenter", commenter));
+                }
+            }
+        }
+
+        requests.add(createIndexRequest("test", "article", "a", null, "category", new String[]{"a"}, "randomized", false));
+        requests.add(createIndexRequest("test", "article", "b", null, "category", new String[]{"a", "b"}, "randomized", false));
+        requests.add(createIndexRequest("test", "article", "c", null, "category", new String[]{"a", "b", "c"}, "randomized", false));
+        requests.add(createIndexRequest("test", "article", "d", null, "category", new String[]{"c"}, "randomized", false));
+        requests.add(createIndexRequest("test", "comment", "e", "a"));
+        requests.add(createIndexRequest("test", "comment", "f", "c"));
+
+        indexRandom(true, requests);
+        ensureSearchable("test");
+    }
+
+    public void testChildrenAggs() throws Exception {
+        SearchResponse searchResponse = client().prepareSearch("test")
+                .setQuery(matchQuery("randomized", true))
+                .addAggregation(
+                        terms("category").field("category").size(10000).subAggregation(children("to_comment", "comment")
+                                .subAggregation(
+                                        terms("commenters").field("commenter").size(10000).subAggregation(
+                                                topHits("top_comments")
+                                        ))
+                        )
+                ).get();
+        assertSearchResponse(searchResponse);
+
+        Terms categoryTerms = searchResponse.getAggregations().get("category");
+        assertThat(categoryTerms.getBuckets().size(), equalTo(categoryToControl.size()));
+        for (Map.Entry<String, Control> entry1 : categoryToControl.entrySet()) {
+            Terms.Bucket categoryBucket = categoryTerms.getBucketByKey(entry1.getKey());
+            assertThat(categoryBucket.getKeyAsString(), equalTo(entry1.getKey()));
+            assertThat(categoryBucket.getDocCount(), equalTo((long) entry1.getValue().articleIds.size()));
+
+            Children childrenBucket = categoryBucket.getAggregations().get("to_comment");
+            assertThat(childrenBucket.getName(), equalTo("to_comment"));
+            assertThat(childrenBucket.getDocCount(), equalTo((long) entry1.getValue().commentIds.size()));
+            assertThat(((InternalAggregation)childrenBucket).getProperty("_count"),
+                equalTo((long) entry1.getValue().commentIds.size()));
+
+            Terms commentersTerms = childrenBucket.getAggregations().get("commenters");
+            assertThat(((InternalAggregation)childrenBucket).getProperty("commenters"), sameInstance(commentersTerms));
+            assertThat(commentersTerms.getBuckets().size(), equalTo(entry1.getValue().commenterToCommentId.size()));
+            for (Map.Entry<String, Set<String>> entry2 : entry1.getValue().commenterToCommentId.entrySet()) {
+                Terms.Bucket commentBucket = commentersTerms.getBucketByKey(entry2.getKey());
+                assertThat(commentBucket.getKeyAsString(), equalTo(entry2.getKey()));
+                assertThat(commentBucket.getDocCount(), equalTo((long) entry2.getValue().size()));
+
+                TopHits topHits = commentBucket.getAggregations().get("top_comments");
+                for (SearchHit searchHit : topHits.getHits().getHits()) {
+                    assertThat(entry2.getValue().contains(searchHit.getId()), is(true));
+                }
+            }
+        }
+    }
+
+    public void testParentWithMultipleBuckets() throws Exception {
+        SearchResponse searchResponse = client().prepareSearch("test")
+                .setQuery(matchQuery("randomized", false))
+                .addAggregation(
+                        terms("category").field("category").size(10000).subAggregation(
+                        children("to_comment", "comment").subAggregation(topHits("top_comments").sort("_uid", SortOrder.ASC))
+                        )
+                ).get();
+        assertSearchResponse(searchResponse);
+
+        Terms categoryTerms = searchResponse.getAggregations().get("category");
+        assertThat(categoryTerms.getBuckets().size(), equalTo(3));
+
+        for (Terms.Bucket bucket : categoryTerms.getBuckets()) {
+            logger.info("bucket={}", bucket.getKey());
+            Children childrenBucket = bucket.getAggregations().get("to_comment");
+            TopHits topHits = childrenBucket.getAggregations().get("top_comments");
+            logger.info("total_hits={}", topHits.getHits().getTotalHits());
+            for (SearchHit searchHit : topHits.getHits()) {
+                logger.info("hit= {} {} {}", searchHit.sortValues()[0], searchHit.getType(), searchHit.getId());
+            }
+        }
+
+        Terms.Bucket categoryBucket = categoryTerms.getBucketByKey("a");
+        assertThat(categoryBucket.getKeyAsString(), equalTo("a"));
+        assertThat(categoryBucket.getDocCount(), equalTo(3L));
+
+        Children childrenBucket = categoryBucket.getAggregations().get("to_comment");
+        assertThat(childrenBucket.getName(), equalTo("to_comment"));
+        assertThat(childrenBucket.getDocCount(), equalTo(2L));
+        TopHits topHits = childrenBucket.getAggregations().get("top_comments");
+        assertThat(topHits.getHits().getTotalHits(), equalTo(2L));
+        assertThat(topHits.getHits().getAt(0).getId(), equalTo("e"));
+        assertThat(topHits.getHits().getAt(1).getId(), equalTo("f"));
+
+        categoryBucket = categoryTerms.getBucketByKey("b");
+        assertThat(categoryBucket.getKeyAsString(), equalTo("b"));
+        assertThat(categoryBucket.getDocCount(), equalTo(2L));
+
+        childrenBucket = categoryBucket.getAggregations().get("to_comment");
+        assertThat(childrenBucket.getName(), equalTo("to_comment"));
+        assertThat(childrenBucket.getDocCount(), equalTo(1L));
+        topHits = childrenBucket.getAggregations().get("top_comments");
+        assertThat(topHits.getHits().getTotalHits(), equalTo(1L));
+        assertThat(topHits.getHits().getAt(0).getId(), equalTo("f"));
+
+        categoryBucket = categoryTerms.getBucketByKey("c");
+        assertThat(categoryBucket.getKeyAsString(), equalTo("c"));
+        assertThat(categoryBucket.getDocCount(), equalTo(2L));
+
+        childrenBucket = categoryBucket.getAggregations().get("to_comment");
+        assertThat(childrenBucket.getName(), equalTo("to_comment"));
+        assertThat(childrenBucket.getDocCount(), equalTo(1L));
+        topHits = childrenBucket.getAggregations().get("top_comments");
+        assertThat(topHits.getHits().getTotalHits(), equalTo(1L));
+        assertThat(topHits.getHits().getAt(0).getId(), equalTo("f"));
+    }
+
+    public void testWithDeletes() throws Exception {
+        String indexName = "xyz";
+        if (legacy()) {
+            assertAcked(
+                    prepareCreate(indexName)
+                            .addMapping("parent")
+                            .addMapping("child", "_parent", "type=parent", "count", "type=long")
+            );
+        } else {
+            assertAcked(
+                    prepareCreate(indexName)
+                        .addMapping("doc",
+                            addFieldMappings(buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "parent", "child"),
+                                "name", "keyword"))
+            );
+        }
+
+        List<IndexRequestBuilder> requests = new ArrayList<>();
+        requests.add(createIndexRequest(indexName, "parent", "1", null));
+        requests.add(createIndexRequest(indexName, "child", "2", "1", "count", 1));
+        requests.add(createIndexRequest(indexName, "child", "3", "1", "count", 1));
+        requests.add(createIndexRequest(indexName, "child", "4", "1", "count", 1));
+        requests.add(createIndexRequest(indexName, "child", "5", "1", "count", 1));
+        indexRandom(true, requests);
+
+        for (int i = 0; i < 10; i++) {
+            SearchResponse searchResponse = client().prepareSearch(indexName)
+                    .addAggregation(children("children", "child").subAggregation(sum("counts").field("count")))
+                    .get();
+
+            assertNoFailures(searchResponse);
+            Children children = searchResponse.getAggregations().get("children");
+            assertThat(children.getDocCount(), equalTo(4L));
+
+            Sum count = children.getAggregations().get("counts");
+            assertThat(count.getValue(), equalTo(4.));
+
+            String idToUpdate = Integer.toString(2 + randomInt(3));
+            /*
+             * The whole point of this test is to test these things with deleted
+             * docs in the index so we turn off detect_noop to make sure that
+             * the updates cause that.
+             */
+            UpdateResponse updateResponse;
+            if (legacy()) {
+                updateResponse = client().prepareUpdate(indexName, "child", idToUpdate)
+                        .setParent("1")
+                        .setDoc(Requests.INDEX_CONTENT_TYPE, "count", 1)
+                        .setDetectNoop(false)
+                        .get();
+            } else {
+                updateResponse = client().prepareUpdate(indexName, "doc", idToUpdate)
+                        .setRouting("1")
+                        .setDoc(Requests.INDEX_CONTENT_TYPE, "count", 1)
+                        .setDetectNoop(false)
+                        .get();
+            }
+            assertThat(updateResponse.getVersion(), greaterThan(1L));
+            refresh();
+        }
+    }
+
+    public void testNonExistingChildType() throws Exception {
+        SearchResponse searchResponse = client().prepareSearch("test")
+                .addAggregation(
+                    children("non-existing", "xyz")
+                ).get();
+        assertSearchResponse(searchResponse);
+
+        Children children = searchResponse.getAggregations().get("non-existing");
+        assertThat(children.getName(), equalTo("non-existing"));
+        assertThat(children.getDocCount(), equalTo(0L));
+    }
+
+    public void testPostCollection() throws Exception {
+        String indexName = "prodcatalog";
+        String masterType = "masterprod";
+        String childType = "variantsku";
+        if (legacy()) {
+            assertAcked(
+                    prepareCreate(indexName)
+                            .setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+                                    .put(indexSettings())
+                                    .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+                                    .put("index.mapping.single_type", false))
+                            .addMapping(masterType, "brand", "type=text", "name", "type=keyword", "material", "type=text")
+                            .addMapping(childType, "_parent", "type=masterprod", "color", "type=keyword", "size", "type=keyword")
+            );
+        } else {
+            assertAcked(
+                    prepareCreate(indexName)
+                            .setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+                                    .put(indexSettings())
+                                    .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0))
+                        .addMapping("doc",
+                            addFieldMappings(buildParentJoinFieldMappingFromSimplifiedDef("join_field", true,
+                                masterType, childType),
+                                "brand", "text", "name", "keyword", "material", "text", "color", "keyword", "size", "keyword"))
+            );
+        }
+
+        List<IndexRequestBuilder> requests = new ArrayList<>();
+        requests.add(createIndexRequest(indexName, masterType, "1", null, "brand", "Levis", "name",
+                "Style 501", "material", "Denim"));
+        requests.add(createIndexRequest(indexName, childType, "3", "1", "color", "blue", "size", "32"));
+        requests.add(createIndexRequest(indexName, childType, "4", "1", "color", "blue", "size", "34"));
+        requests.add(createIndexRequest(indexName, childType, "5", "1", "color", "blue", "size", "36"));
+        requests.add(createIndexRequest(indexName, childType, "6", "1", "color", "black", "size", "38"));
+        requests.add(createIndexRequest(indexName, childType, "7", "1", "color", "black", "size", "40"));
+        requests.add(createIndexRequest(indexName, childType, "8", "1", "color", "gray", "size", "36"));
+
+        requests.add(createIndexRequest(indexName, masterType, "2", null, "brand", "Wrangler", "name",
+                "Regular Cut", "material", "Leather"));
+        requests.add(createIndexRequest(indexName, childType, "9", "2", "color", "blue", "size", "32"));
+        requests.add(createIndexRequest(indexName, childType, "10", "2", "color", "blue", "size", "34"));
+        requests.add(createIndexRequest(indexName, childType, "12", "2", "color", "black", "size", "36"));
+        requests.add(createIndexRequest(indexName, childType, "13", "2", "color", "black", "size", "38"));
+        requests.add(createIndexRequest(indexName, childType, "14", "2", "color", "black", "size", "40"));
+        requests.add(createIndexRequest(indexName, childType, "15", "2", "color", "orange", "size", "36"));
+        requests.add(createIndexRequest(indexName, childType, "16", "2", "color", "green", "size", "44"));
+        indexRandom(true, requests);
+
+        SearchResponse response = client().prepareSearch(indexName)
+                .setQuery(hasChildQuery(childType, termQuery("color", "orange"), ScoreMode.None))
+                        .addAggregation(children("my-refinements", childType)
+                                .subAggregation(terms("my-colors").field("color"))
+                                .subAggregation(terms("my-sizes").field("size"))
+                ).get();
+        assertNoFailures(response);
+        assertHitCount(response, 1);
+
+        Children childrenAgg = response.getAggregations().get("my-refinements");
+        assertThat(childrenAgg.getDocCount(), equalTo(7L));
+
+        Terms termsAgg = childrenAgg.getAggregations().get("my-colors");
+        assertThat(termsAgg.getBuckets().size(), equalTo(4));
+        assertThat(termsAgg.getBucketByKey("black").getDocCount(), equalTo(3L));
+        assertThat(termsAgg.getBucketByKey("blue").getDocCount(), equalTo(2L));
+        assertThat(termsAgg.getBucketByKey("green").getDocCount(), equalTo(1L));
+        assertThat(termsAgg.getBucketByKey("orange").getDocCount(), equalTo(1L));
+
+        termsAgg = childrenAgg.getAggregations().get("my-sizes");
+        assertThat(termsAgg.getBuckets().size(), equalTo(6));
+        assertThat(termsAgg.getBucketByKey("36").getDocCount(), equalTo(2L));
+        assertThat(termsAgg.getBucketByKey("32").getDocCount(), equalTo(1L));
+        assertThat(termsAgg.getBucketByKey("34").getDocCount(), equalTo(1L));
+        assertThat(termsAgg.getBucketByKey("38").getDocCount(), equalTo(1L));
+        assertThat(termsAgg.getBucketByKey("40").getDocCount(), equalTo(1L));
+        assertThat(termsAgg.getBucketByKey("44").getDocCount(), equalTo(1L));
+    }
+
+    public void testHierarchicalChildrenAggs() {
+        String indexName = "geo";
+        String grandParentType = "continent";
+        String parentType = "country";
+        String childType = "city";
+        if (legacy()) {
+            assertAcked(
+                    prepareCreate(indexName)
+                            .addMapping(grandParentType, "name", "type=keyword")
+                            .addMapping(parentType, "_parent", "type=" + grandParentType)
+                            .addMapping(childType, "_parent", "type=" + parentType)
+
+            );
+        } else {
+            assertAcked(
+                    prepareCreate(indexName)
+                        .addMapping("doc",
+                            addFieldMappings(buildParentJoinFieldMappingFromSimplifiedDef("join_field", true,
+                                grandParentType, parentType, parentType, childType),
+                                "name", "keyword"))
+            );
+        }
+
+        createIndexRequest(indexName, grandParentType, "1", null, "name", "europe").get();
+        createIndexRequest(indexName, parentType, "2", "1", "name", "belgium").get();
+        createIndexRequest(indexName, childType, "3", "2", "name", "brussels").setRouting("1").get();
+        refresh();
+
+        SearchResponse response = client().prepareSearch(indexName)
+                .setQuery(matchQuery("name", "europe"))
+                .addAggregation(
+                children(parentType, parentType).subAggregation(children(childType, childType).subAggregation(
+                                        terms("name").field("name")
+                                )
+                        )
+                )
+                .get();
+        assertNoFailures(response);
+        assertHitCount(response, 1);
+
+        Children children = response.getAggregations().get(parentType);
+        assertThat(children.getName(), equalTo(parentType));
+        assertThat(children.getDocCount(), equalTo(1L));
+        children = children.getAggregations().get(childType);
+        assertThat(children.getName(), equalTo(childType));
+        assertThat(children.getDocCount(), equalTo(1L));
+        Terms terms = children.getAggregations().get("name");
+        assertThat(terms.getBuckets().size(), equalTo(1));
+        assertThat(terms.getBuckets().get(0).getKey().toString(), equalTo("brussels"));
+        assertThat(terms.getBuckets().get(0).getDocCount(), equalTo(1L));
+    }
+
+    public void testPostCollectAllLeafReaders() throws Exception {
+        // The 'towns' and 'parent_names' aggs operate on parent docs and if child docs are in different segments we need
+        // to ensure those segments which child docs are also evaluated to in the post collect phase.
+
+        // Before we only evaluated segments that yielded matches in 'towns' and 'parent_names' aggs, which caused
+        // us to miss to evaluate child docs in segments we didn't have parent matches for.
+        if (legacy()) {
+            assertAcked(
+                    prepareCreate("index")
+                            .addMapping("parentType", "name", "type=keyword", "town", "type=keyword")
+                            .addMapping("childType", "_parent", "type=parentType", "name", "type=keyword", "age", "type=integer")
+            );
+        } else {
+            assertAcked(
+                    prepareCreate("index")
+                        .addMapping("doc",
+                            addFieldMappings(buildParentJoinFieldMappingFromSimplifiedDef("join_field", true,
+                                "parentType", "childType"),
+                                "name", "keyword", "town", "keyword", "age", "integer"))
+            );
+        }
+        List<IndexRequestBuilder> requests = new ArrayList<>();
+        requests.add(createIndexRequest("index", "parentType", "1", null, "name", "Bob", "town", "Memphis"));
+        requests.add(createIndexRequest("index", "parentType", "2", null, "name", "Alice", "town", "Chicago"));
+        requests.add(createIndexRequest("index", "parentType", "3", null, "name", "Bill", "town", "Chicago"));
+        requests.add(createIndexRequest("index", "childType", "4", "1", "name", "Jill", "age", 5));
+        requests.add(createIndexRequest("index", "childType", "5", "1", "name", "Joey", "age", 3));
+        requests.add(createIndexRequest("index", "childType", "6", "2", "name", "John", "age", 2));
+        requests.add(createIndexRequest("index", "childType", "7", "3", "name", "Betty", "age", 6));
+        requests.add(createIndexRequest("index", "childType", "8", "3", "name", "Dan", "age", 1));
+        indexRandom(true, requests);
+
+        SearchResponse response = client().prepareSearch("index")
+            .setSize(0)
+            .addAggregation(AggregationBuilders.terms("towns").field("town")
+                .subAggregation(AggregationBuilders.terms("parent_names").field("name")
+                    .subAggregation(children("child_docs", "childType"))
+                )
+            ).get();
+
+        Terms towns = response.getAggregations().get("towns");
+        assertThat(towns.getBuckets().size(), equalTo(2));
+        assertThat(towns.getBuckets().get(0).getKeyAsString(), equalTo("Chicago"));
+        assertThat(towns.getBuckets().get(0).getDocCount(), equalTo(2L));
+
+        Terms parents = towns.getBuckets().get(0).getAggregations().get("parent_names");
+        assertThat(parents.getBuckets().size(), equalTo(2));
+        assertThat(parents.getBuckets().get(0).getKeyAsString(), equalTo("Alice"));
+        assertThat(parents.getBuckets().get(0).getDocCount(), equalTo(1L));
+        Children children = parents.getBuckets().get(0).getAggregations().get("child_docs");
+        assertThat(children.getDocCount(), equalTo(1L));
+
+        assertThat(parents.getBuckets().get(1).getKeyAsString(), equalTo("Bill"));
+        assertThat(parents.getBuckets().get(1).getDocCount(), equalTo(1L));
+        children = parents.getBuckets().get(1).getAggregations().get("child_docs");
+        assertThat(children.getDocCount(), equalTo(2L));
+
+        assertThat(towns.getBuckets().get(1).getKeyAsString(), equalTo("Memphis"));
+        assertThat(towns.getBuckets().get(1).getDocCount(), equalTo(1L));
+        parents = towns.getBuckets().get(1).getAggregations().get("parent_names");
+        assertThat(parents.getBuckets().size(), equalTo(1));
+        assertThat(parents.getBuckets().get(0).getKeyAsString(), equalTo("Bob"));
+        assertThat(parents.getBuckets().get(0).getDocCount(), equalTo(1L));
+        children = parents.getBuckets().get(0).getAggregations().get("child_docs");
+        assertThat(children.getDocCount(), equalTo(2L));
+    }
+
+    private static final class Control {
+
+        final String category;
+        final Set<String> articleIds = new HashSet<>();
+        final Set<String> commentIds = new HashSet<>();
+        final Map<String, Set<String>> commenterToCommentId = new HashMap<>();
+
+        private Control(String category) {
+            this.category = category;
+        }
+    }
+}
diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenTests.java
new file mode 100644
index 0000000000000..85a97c4b9b413
--- /dev/null
+++ b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenTests.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.join.aggregations;
+
+import org.elasticsearch.join.ParentJoinPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
+
+import java.util.Collection;
+import java.util.Collections;
+
+public class ChildrenTests extends BaseAggregationTestCase<ChildrenAggregationBuilder> {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return Collections.singleton(ParentJoinPlugin.class);
+    }
+
+    @Override
+    protected ChildrenAggregationBuilder createTestAggregatorBuilder() {
+        String name = randomAlphaOfLengthBetween(3, 20);
+        String childType = randomAlphaOfLengthBetween(5, 40);
+        ChildrenAggregationBuilder factory = new ChildrenAggregationBuilder(name, childType);
+        return factory;
+    }
+
+}
diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/InternalChildrenTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/InternalChildrenTests.java
new file mode 100644
index 0000000000000..86e397f0da8ed
--- /dev/null
+++ b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/InternalChildrenTests.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.join.aggregations;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry.Entry;
+import org.elasticsearch.search.aggregations.Aggregation;
+import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.aggregations.InternalSingleBucketAggregationTestCase;
+import org.elasticsearch.search.aggregations.bucket.ParsedSingleBucketAggregation;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class InternalChildrenTests extends InternalSingleBucketAggregationTestCase<InternalChildren> {
+
+    @Override
+    protected List<NamedXContentRegistry.Entry> getNamedXContents() {
+        List<Entry> extendedNamedXContents = new ArrayList<>(super.getNamedXContents());
+        extendedNamedXContents.add(new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(ChildrenAggregationBuilder.NAME),
+                (p, c) -> ParsedChildren.fromXContent(p, (String) c)));
+        return extendedNamedXContents ;
+    }
+
+    @Override
+    protected InternalChildren createTestInstance(String name, long docCount, InternalAggregations aggregations,
+            List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
+        return new InternalChildren(name, docCount, aggregations, pipelineAggregators, metaData);
+    }
+
+    @Override
+    protected Class<? extends ParsedSingleBucketAggregation> implementationClass() {
+        return ParsedChildren.class;
+    }
+}
diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/LegacyChildrenIT.java b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/LegacyChildrenIT.java
new file mode 100644
index 0000000000000..ecf4d06f325e8
--- /dev/null
+++ b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/LegacyChildrenIT.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.join.aggregations;
+
+public class LegacyChildrenIT extends ChildrenIT {
+
+    @Override
+    protected boolean legacy() {
+        return true;
+    }
+}
diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ParentToChildrenAggregatorTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ParentToChildrenAggregatorTests.java
new file mode 100644
index 0000000000000..fd25fcaf932d3
--- /dev/null
+++ b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ParentToChildrenAggregatorTests.java
@@ -0,0 +1,218 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.join.aggregations;
+
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.SortedNumericDocValuesField;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermInSetQuery;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
+import org.elasticsearch.common.regex.Regex;
+import org.elasticsearch.common.settings.IndexScopedSettings;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.mapper.ContentPath;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.Mapper;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.NumberFieldMapper;
+import org.elasticsearch.index.mapper.Uid;
+import org.elasticsearch.index.mapper.UidFieldMapper;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.join.mapper.MetaJoinFieldMapper;
+import org.elasticsearch.join.mapper.ParentJoinFieldMapper;
+import org.elasticsearch.search.aggregations.AggregatorTestCase;
+import org.elasticsearch.search.aggregations.metrics.min.InternalMin;
+import org.elasticsearch.search.aggregations.metrics.min.MinAggregationBuilder;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Consumer;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class ParentToChildrenAggregatorTests extends AggregatorTestCase {
+
+    private static final String CHILD_TYPE = "child_type";
+    private static final String PARENT_TYPE = "parent_type";
+
+    public void testNoDocs() throws IOException {
+        Directory directory = newDirectory();
+
+        RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
+        // intentionally not writing any docs
+        indexWriter.close();
+        IndexReader indexReader = DirectoryReader.open(directory);
+
+        testCase(new MatchAllDocsQuery(), newSearcher(indexReader, false, true), parentToChild -> {
+            assertEquals(0, parentToChild.getDocCount());
+            assertEquals(Double.POSITIVE_INFINITY, ((InternalMin) parentToChild.getAggregations().get("in_child")).getValue(),
+                    Double.MIN_VALUE);
+        });
+        indexReader.close();
+        directory.close();
+    }
+
+    public void testParentChild() throws IOException {
+        Directory directory = newDirectory();
+        RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
+
+        final Map<String, Tuple<Integer, Integer>> expectedParentChildRelations = setupIndex(indexWriter);
+        indexWriter.close();
+
+        IndexReader indexReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(directory),
+                new ShardId(new Index("foo", "_na_"), 1));
+        // TODO set "maybeWrap" to true for IndexSearcher once #23338 is resolved
+        IndexSearcher indexSearcher = newSearcher(indexReader, false, true);
+
+        testCase(new MatchAllDocsQuery(), indexSearcher, child -> {
+            int expectedTotalChildren = 0;
+            int expectedMinValue = Integer.MAX_VALUE;
+            for (Tuple<Integer, Integer> expectedValues : expectedParentChildRelations.values()) {
+                expectedTotalChildren += expectedValues.v1();
+                expectedMinValue = Math.min(expectedMinValue, expectedValues.v2());
+            }
+            assertEquals(expectedTotalChildren, child.getDocCount());
+            assertEquals(expectedMinValue, ((InternalMin) child.getAggregations().get("in_child")).getValue(), Double.MIN_VALUE);
+        });
+
+        for (String parent : expectedParentChildRelations.keySet()) {
+            testCase(new TermInSetQuery(UidFieldMapper.NAME, new BytesRef(Uid.createUid(PARENT_TYPE, parent))), indexSearcher, child -> {
+                assertEquals((long) expectedParentChildRelations.get(parent).v1(), child.getDocCount());
+                assertEquals(expectedParentChildRelations.get(parent).v2(),
+                        ((InternalMin) child.getAggregations().get("in_child")).getValue(), Double.MIN_VALUE);
+            });
+        }
+        indexReader.close();
+        directory.close();
+    }
+
+    private static Map<String, Tuple<Integer, Integer>> setupIndex(RandomIndexWriter iw) throws IOException {
+        Map<String, Tuple<Integer, Integer>> expectedValues = new HashMap<>();
+        int numParents = randomIntBetween(1, 10);
+        for (int i = 0; i < numParents; i++) {
+            String parent = "parent" + i;
+            iw.addDocument(createParentDocument(parent));
+            int numChildren = randomIntBetween(1, 10);
+            int minValue = Integer.MAX_VALUE;
+            for (int c = 0; c < numChildren; c++) {
+                int randomValue = randomIntBetween(0, 100);
+                minValue = Math.min(minValue, randomValue);
+                iw.addDocument(createChildDocument("child" + c + "_" + parent, parent, randomValue));
+            }
+            expectedValues.put(parent, new Tuple<>(numChildren, minValue));
+        }
+        return expectedValues;
+    }
+
+    private static List<Field> createParentDocument(String id) {
+        return Arrays.asList(
+                new StringField(UidFieldMapper.NAME, Uid.createUid(PARENT_TYPE, id), Field.Store.NO),
+                new StringField("join_field", PARENT_TYPE, Field.Store.NO),
+                createJoinField(PARENT_TYPE, id)
+        );
+    }
+
+    private static List<Field> createChildDocument(String childId, String parentId, int value) {
+        return Arrays.asList(
+                new StringField(UidFieldMapper.NAME, Uid.createUid(CHILD_TYPE, childId), Field.Store.NO),
+                new StringField("join_field", CHILD_TYPE, Field.Store.NO),
+                createJoinField(PARENT_TYPE, parentId),
+                new SortedNumericDocValuesField("number", value)
+        );
+    }
+
+    private static SortedDocValuesField createJoinField(String parentType, String id) {
+        return new SortedDocValuesField("join_field#" + parentType, new BytesRef(id));
+    }
+
+    private static IndexMetaData newIndexMeta(String name, Settings indexSettings) {
+        Settings build = Settings.builder()
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(indexSettings)
+            .put("index.mapping.single_type", true)
+            .build();
+        IndexMetaData metaData = IndexMetaData.builder(name).settings(build).build();
+        return metaData;
+    }
+
+    private static IndexSettings newIndexSettings(IndexMetaData metaData) {
+        Set<Setting<?>> settingSet = new HashSet<>(IndexScopedSettings.BUILT_IN_INDEX_SETTINGS);
+        return new IndexSettings(metaData, Settings.EMPTY, (idx) -> Regex.simpleMatch(idx, metaData.getIndex().getName()),
+            new IndexScopedSettings(Settings.EMPTY, settingSet));
+    }
+
+    @Override
+    protected IndexSettings indexSettings() {
+        return newIndexSettings(newIndexMeta("_index", Settings.EMPTY));
+    }
+
+    @Override
+    protected MapperService mapperServiceMock() {
+        IndexSettings indexSettings = indexSettings();
+        ParentJoinFieldMapper joinFieldMapper = createJoinFieldMapper(indexSettings.getSettings());
+        MapperService mapperService = mock(MapperService.class);
+        MetaJoinFieldMapper.MetaJoinFieldType metaJoinFieldType = mock(MetaJoinFieldMapper.MetaJoinFieldType.class);
+        when(metaJoinFieldType.getMapper()).thenReturn(joinFieldMapper);
+        when(mapperService.fullName("_parent_join")).thenReturn(metaJoinFieldType);
+        return mapperService;
+    }
+
+    private static ParentJoinFieldMapper createJoinFieldMapper(Settings settings) {
+        return new ParentJoinFieldMapper.Builder("join_field")
+                .addParent(PARENT_TYPE, Collections.singleton(CHILD_TYPE))
+                .build(new Mapper.BuilderContext(settings, new ContentPath(0)));
+    }
+
+    private void testCase(Query query, IndexSearcher indexSearcher, Consumer<InternalChildren> verify)
+            throws IOException {
+
+        ChildrenAggregationBuilder aggregationBuilder = new ChildrenAggregationBuilder("_name", CHILD_TYPE);
+        aggregationBuilder.subAggregation(new MinAggregationBuilder("in_child").field("number"));
+
+        MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
+        fieldType.setName("number");
+        InternalChildren result = search(indexSearcher, query, aggregationBuilder, fieldType);
+        verify.accept(result);
+    }
+}
diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/mapper/ParentJoinFieldMapperTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/mapper/ParentJoinFieldMapperTests.java
new file mode 100644
index 0000000000000..f0c69fc497bd8
--- /dev/null
+++ b/modules/parent-join/src/test/java/org/elasticsearch/join/mapper/ParentJoinFieldMapperTests.java
@@ -0,0 +1,482 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.join.mapper;
+
+import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.mapper.DocumentMapper;
+import org.elasticsearch.index.mapper.MapperException;
+import org.elasticsearch.index.mapper.MapperParsingException;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.ParsedDocument;
+import org.elasticsearch.index.mapper.SourceToParse;
+import org.elasticsearch.join.ParentJoinPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+
+import java.util.Collection;
+import java.util.Collections;
+
+import static org.hamcrest.Matchers.containsString;
+
+public class ParentJoinFieldMapperTests extends ESSingleNodeTestCase {
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return Collections.singletonList(ParentJoinPlugin.class);
+    }
+
+    @Override
+    protected IndexService createIndex(String index) {
+        Settings settings = Settings.builder()
+            .put("index.mapping.single_type", true)
+            .build();
+        return super.createIndex(index, settings);
+    }
+
+    public void testSingleLevel() throws Exception {
+        String mapping = XContentFactory.jsonBuilder().startObject()
+            .startObject("properties")
+                .startObject("join_field")
+                    .field("type", "join")
+                    .startObject("relations")
+                        .field("parent", "child")
+                    .endObject()
+                .endObject()
+            .endObject()
+            .endObject().string();
+        IndexService service = createIndex("test");
+        DocumentMapper docMapper = service.mapperService().merge("type", new CompressedXContent(mapping),
+            MapperService.MergeReason.MAPPING_UPDATE, false);
+        assertTrue(docMapper.mappers().getMapper("join_field") == ParentJoinFieldMapper.getMapper(service.mapperService()));
+
+        // Doc without join
+        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "0",
+            XContentFactory.jsonBuilder().startObject().endObject().bytes(), XContentType.JSON));
+        assertNull(doc.rootDoc().getBinaryValue("join_field"));
+
+        // Doc parent
+        doc = docMapper.parse(SourceToParse.source("test", "type", "1",
+            XContentFactory.jsonBuilder().startObject()
+                .field("join_field", "parent")
+                .endObject().bytes(), XContentType.JSON));
+        assertEquals("1", doc.rootDoc().getBinaryValue("join_field#parent").utf8ToString());
+        assertEquals("parent", doc.rootDoc().getBinaryValue("join_field").utf8ToString());
+
+        // Doc child
+        doc = docMapper.parse(SourceToParse.source("test", "type", "2",
+            XContentFactory.jsonBuilder().startObject()
+                .startObject("join_field")
+                    .field("name", "child")
+                    .field("parent", "1")
+                .endObject()
+                .endObject().bytes(), XContentType.JSON).routing("1"));
+        assertEquals("1", doc.rootDoc().getBinaryValue("join_field#parent").utf8ToString());
+        assertEquals("child", doc.rootDoc().getBinaryValue("join_field").utf8ToString());
+
+        // Unkwnown join name
+        MapperException exc = expectThrows(MapperParsingException.class,
+            () -> docMapper.parse(SourceToParse.source("test", "type", "1",
+                XContentFactory.jsonBuilder().startObject()
+                    .field("join_field", "unknown")
+                    .endObject().bytes(), XContentType.JSON)));
+        assertThat(exc.getRootCause().getMessage(), containsString("unknown join name [unknown] for field [join_field]"));
+    }
+
+    public void testParentIdSpecifiedAsNumber() throws Exception {
+        String mapping = XContentFactory.jsonBuilder().startObject()
+            .startObject("properties")
+                .startObject("join_field")
+                    .field("type", "join")
+                        .startObject("relations")
+                            .field("parent", "child")
+                        .endObject()
+                    .endObject()
+                .endObject()
+            .endObject().string();
+        IndexService service = createIndex("test");
+        DocumentMapper docMapper = service.mapperService().merge("type", new CompressedXContent(mapping),
+            MapperService.MergeReason.MAPPING_UPDATE, false);
+        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "2",
+            XContentFactory.jsonBuilder().startObject()
+                .startObject("join_field")
+                .field("name", "child")
+                .field("parent", 1)
+                .endObject()
+                .endObject().bytes(), XContentType.JSON).routing("1"));
+        assertEquals("1", doc.rootDoc().getBinaryValue("join_field#parent").utf8ToString());
+        assertEquals("child", doc.rootDoc().getBinaryValue("join_field").utf8ToString());
+        doc = docMapper.parse(SourceToParse.source("test", "type", "2",
+            XContentFactory.jsonBuilder().startObject()
+                .startObject("join_field")
+                .field("name", "child")
+                .field("parent", 1.0)
+                .endObject()
+                .endObject().bytes(), XContentType.JSON).routing("1"));
+        assertEquals("1.0", doc.rootDoc().getBinaryValue("join_field#parent").utf8ToString());
+        assertEquals("child", doc.rootDoc().getBinaryValue("join_field").utf8ToString());
+    }
+
+    public void testMultipleLevels() throws Exception {
+        String mapping = XContentFactory.jsonBuilder().startObject()
+            .startObject("properties")
+                .startObject("join_field")
+                    .field("type", "join")
+                    .startObject("relations")
+                        .field("parent", "child")
+                        .field("child", "grand_child")
+                    .endObject()
+                .endObject()
+            .endObject()
+            .endObject().string();
+        IndexService service = createIndex("test");
+        DocumentMapper docMapper = service.mapperService().merge("type", new CompressedXContent(mapping),
+            MapperService.MergeReason.MAPPING_UPDATE, false);
+        assertTrue(docMapper.mappers().getMapper("join_field") == ParentJoinFieldMapper.getMapper(service.mapperService()));
+
+        // Doc without join
+        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "0",
+            XContentFactory.jsonBuilder().startObject().endObject().bytes(), XContentType.JSON));
+        assertNull(doc.rootDoc().getBinaryValue("join_field"));
+
+        // Doc parent
+        doc = docMapper.parse(SourceToParse.source("test", "type", "1",
+            XContentFactory.jsonBuilder()
+                .startObject()
+                    .field("join_field", "parent")
+                .endObject().bytes(), XContentType.JSON));
+        assertEquals("1", doc.rootDoc().getBinaryValue("join_field#parent").utf8ToString());
+        assertEquals("parent", doc.rootDoc().getBinaryValue("join_field").utf8ToString());
+
+        // Doc child
+        doc = docMapper.parse(SourceToParse.source("test", "type", "2",
+            XContentFactory.jsonBuilder().startObject()
+                .startObject("join_field")
+                    .field("name", "child")
+                    .field("parent", "1")
+                .endObject()
+                .endObject().bytes(), XContentType.JSON).routing("1"));
+        assertEquals("1", doc.rootDoc().getBinaryValue("join_field#parent").utf8ToString());
+        assertEquals("2", doc.rootDoc().getBinaryValue("join_field#child").utf8ToString());
+        assertEquals("child", doc.rootDoc().getBinaryValue("join_field").utf8ToString());
+
+        // Doc child missing parent
+        MapperException exc = expectThrows(MapperParsingException.class,
+            () -> docMapper.parse(SourceToParse.source("test", "type", "2",
+                XContentFactory.jsonBuilder().startObject()
+                    .field("join_field", "child")
+                    .endObject().bytes(), XContentType.JSON).routing("1")));
+        assertThat(exc.getRootCause().getMessage(), containsString("[parent] is missing for join field [join_field]"));
+
+        // Doc child missing routing
+        exc = expectThrows(MapperParsingException.class,
+            () -> docMapper.parse(SourceToParse.source("test", "type", "2",
+                XContentFactory.jsonBuilder().startObject()
+                    .startObject("join_field")
+                    .field("name", "child")
+                    .field("parent", "1")
+                    .endObject()
+                    .endObject().bytes(), XContentType.JSON)));
+        assertThat(exc.getRootCause().getMessage(), containsString("[routing] is missing for join field [join_field]"));
+
+        // Doc grand_child
+        doc = docMapper.parse(SourceToParse.source("test", "type", "3",
+            XContentFactory.jsonBuilder().startObject()
+                .startObject("join_field")
+                    .field("name", "grand_child")
+                    .field("parent", "2")
+                .endObject()
+                .endObject().bytes(), XContentType.JSON).routing("1"));
+        assertEquals("2", doc.rootDoc().getBinaryValue("join_field#child").utf8ToString());
+        assertEquals("grand_child", doc.rootDoc().getBinaryValue("join_field").utf8ToString());
+
+        // Unkwnown join name
+        exc = expectThrows(MapperParsingException.class,
+            () -> docMapper.parse(SourceToParse.source("test", "type", "1",
+                XContentFactory.jsonBuilder().startObject()
+                    .field("join_field", "unknown")
+                    .endObject().bytes(), XContentType.JSON)));
+        assertThat(exc.getRootCause().getMessage(), containsString("unknown join name [unknown] for field [join_field]"));
+    }
+
+    public void testUpdateRelations() throws Exception {
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("properties")
+            .startObject("join_field")
+                .field("type", "join")
+                .startObject("relations")
+                    .field("parent", "child")
+                    .array("child", "grand_child1", "grand_child2")
+                .endObject()
+            .endObject()
+            .endObject().endObject().string();
+        IndexService indexService = createIndex("test");
+        DocumentMapper docMapper = indexService.mapperService().merge("type", new CompressedXContent(mapping),
+            MapperService.MergeReason.MAPPING_UPDATE, false);
+        assertTrue(docMapper.mappers().getMapper("join_field") == ParentJoinFieldMapper.getMapper(indexService.mapperService()));
+
+        {
+            final String updateMapping = XContentFactory.jsonBuilder().startObject().startObject("properties")
+                .startObject("join_field")
+                    .field("type", "join")
+                    .startObject("relations")
+                        .array("child", "grand_child1", "grand_child2")
+                    .endObject()
+                .endObject()
+                .endObject().endObject().string();
+            IllegalStateException exc = expectThrows(IllegalStateException.class,
+                () -> indexService.mapperService().merge("type", new CompressedXContent(updateMapping),
+                    MapperService.MergeReason.MAPPING_UPDATE, false));
+            assertThat(exc.getMessage(), containsString("cannot remove parent [parent] in join field [join_field]"));
+        }
+
+        {
+            final String updateMapping = XContentFactory.jsonBuilder().startObject().startObject("properties")
+                .startObject("join_field")
+                    .field("type", "join")
+                    .startObject("relations")
+                        .field("parent", "child")
+                        .field("child", "grand_child1")
+                    .endObject()
+                .endObject()
+                .endObject().endObject().string();
+            IllegalStateException exc = expectThrows(IllegalStateException.class,
+                () -> indexService.mapperService().merge("type", new CompressedXContent(updateMapping),
+                    MapperService.MergeReason.MAPPING_UPDATE, false));
+            assertThat(exc.getMessage(), containsString("cannot remove child [grand_child2] in join field [join_field]"));
+        }
+
+        {
+            final String updateMapping = XContentFactory.jsonBuilder().startObject().startObject("properties")
+                .startObject("join_field")
+                    .field("type", "join")
+                    .startObject("relations")
+                        .field("uber_parent", "parent")
+                        .field("parent", "child")
+                        .array("child", "grand_child1", "grand_child2")
+                    .endObject()
+                .endObject()
+                .endObject().endObject().string();
+            IllegalStateException exc = expectThrows(IllegalStateException.class,
+                () -> indexService.mapperService().merge("type", new CompressedXContent(updateMapping),
+                    MapperService.MergeReason.MAPPING_UPDATE, false));
+            assertThat(exc.getMessage(), containsString("cannot create child [parent] from an existing parent"));
+        }
+
+        {
+            final String updateMapping = XContentFactory.jsonBuilder().startObject().startObject("properties")
+                .startObject("join_field")
+                    .field("type", "join")
+                    .startObject("relations")
+                        .field("parent", "child")
+                        .array("child", "grand_child1", "grand_child2")
+                        .field("grand_child2", "grand_grand_child")
+                    .endObject()
+                .endObject()
+                .endObject().endObject().string();
+            IllegalStateException exc = expectThrows(IllegalStateException.class,
+                () -> indexService.mapperService().merge("type", new CompressedXContent(updateMapping),
+                    MapperService.MergeReason.MAPPING_UPDATE, false));
+            assertThat(exc.getMessage(), containsString("cannot create parent [grand_child2] from an existing child]"));
+        }
+
+        {
+            final String updateMapping = XContentFactory.jsonBuilder().startObject().startObject("properties")
+                .startObject("join_field")
+                    .field("type", "join")
+                    .startObject("relations")
+                        .array("parent", "child", "child2")
+                        .array("child", "grand_child1", "grand_child2")
+                    .endObject()
+                .endObject()
+                .endObject().endObject().string();
+            docMapper = indexService.mapperService().merge("type", new CompressedXContent(updateMapping),
+                MapperService.MergeReason.MAPPING_UPDATE, true);
+            assertTrue(docMapper.mappers().getMapper("join_field") == ParentJoinFieldMapper.getMapper(indexService.mapperService()));
+            ParentJoinFieldMapper mapper = ParentJoinFieldMapper.getMapper(indexService.mapperService());
+            assertTrue(mapper.hasChild("child2"));
+            assertFalse(mapper.hasParent("child2"));
+            assertTrue(mapper.hasChild("grand_child2"));
+            assertFalse(mapper.hasParent("grand_child2"));
+        }
+
+        {
+            final String updateMapping = XContentFactory.jsonBuilder().startObject().startObject("properties")
+                .startObject("join_field")
+                    .field("type", "join")
+                    .startObject("relations")
+                        .array("parent", "child", "child2")
+                        .array("child", "grand_child1", "grand_child2")
+                        .array("other", "child_other1", "child_other2")
+                    .endObject()
+                .endObject()
+                .endObject().endObject().string();
+            docMapper = indexService.mapperService().merge("type", new CompressedXContent(updateMapping),
+                MapperService.MergeReason.MAPPING_UPDATE, true);
+            assertTrue(docMapper.mappers().getMapper("join_field") == ParentJoinFieldMapper.getMapper(indexService.mapperService()));
+            ParentJoinFieldMapper mapper = ParentJoinFieldMapper.getMapper(indexService.mapperService());
+            assertTrue(mapper.hasParent("other"));
+            assertFalse(mapper.hasChild("other"));
+            assertTrue(mapper.hasChild("child_other1"));
+            assertFalse(mapper.hasParent("child_other1"));
+            assertTrue(mapper.hasChild("child_other2"));
+            assertFalse(mapper.hasParent("child_other2"));
+        }
+    }
+
+    public void testInvalidJoinFieldInsideObject() throws Exception {
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("properties")
+            .startObject("object")
+                .startObject("properties")
+                    .startObject("join_field")
+                        .field("type", "join")
+                        .startObject("relations")
+                            .field("parent", "child")
+                        .endObject()
+                    .endObject()
+                .endObject()
+            .endObject()
+            .endObject().endObject().string();
+        IndexService indexService = createIndex("test");
+        MapperParsingException exc = expectThrows(MapperParsingException.class,
+            () -> indexService.mapperService().merge("type", new CompressedXContent(mapping),
+                MapperService.MergeReason.MAPPING_UPDATE, false));
+        assertThat(exc.getRootCause().getMessage(),
+            containsString("join field [object.join_field] cannot be added inside an object or in a multi-field"));
+    }
+
+    public void testInvalidJoinFieldInsideMultiFields() throws Exception {
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("properties")
+            .startObject("number")
+                .field("type", "integer")
+                .startObject("fields")
+                    .startObject("join_field")
+                        .field("type", "join")
+                        .startObject("relations")
+                            .field("parent", "child")
+                        .endObject()
+                    .endObject()
+                .endObject()
+            .endObject()
+            .endObject().endObject().string();
+        IndexService indexService = createIndex("test");
+        MapperParsingException exc = expectThrows(MapperParsingException.class,
+            () -> indexService.mapperService().merge("type", new CompressedXContent(mapping),
+                MapperService.MergeReason.MAPPING_UPDATE, false));
+        assertThat(exc.getRootCause().getMessage(),
+            containsString("join field [number.join_field] cannot be added inside an object or in a multi-field"));
+    }
+
+    public void testMultipleJoinFields() throws Exception {
+        IndexService indexService = createIndex("test");
+        {
+            String mapping = XContentFactory.jsonBuilder().startObject()
+                .startObject("properties")
+                    .startObject("join_field")
+                        .field("type", "join")
+                        .startObject("relations")
+                            .field("parent", "child")
+                            .field("child", "grand_child")
+                        .endObject()
+                    .endObject()
+                    .startObject("another_join_field")
+                        .field("type", "join")
+                        .startObject("relations")
+                            .field("product", "item")
+                        .endObject()
+                    .endObject()
+                .endObject()
+                .endObject().string();
+            IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> indexService.mapperService().merge("type",
+                new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE, false));
+            assertThat(exc.getMessage(), containsString("Field [_parent_join] is defined twice in [type]"));
+        }
+
+        {
+            String mapping = XContentFactory.jsonBuilder().startObject()
+                .startObject("properties")
+                    .startObject("join_field")
+                        .field("type", "join")
+                        .startObject("relations")
+                            .field("parent", "child")
+                            .field("child", "grand_child")
+                        .endObject()
+                    .endObject()
+                .endObject()
+                .endObject().string();
+            indexService.mapperService().merge("type",
+                new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE, false);
+            String updateMapping = XContentFactory.jsonBuilder().startObject()
+                .startObject("properties")
+                    .startObject("another_join_field")
+                        .field("type", "join")
+                    .endObject()
+                .endObject()
+                .endObject().string();
+            IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> indexService.mapperService().merge("type",
+                new CompressedXContent(updateMapping), MapperService.MergeReason.MAPPING_UPDATE, false));
+            assertThat(exc.getMessage(), containsString("Field [_parent_join] is defined twice in [type]"));
+        }
+    }
+
+    public void testEagerGlobalOrdinals() throws Exception {
+        String mapping = XContentFactory.jsonBuilder().startObject()
+            .startObject("properties")
+                .startObject("join_field")
+                    .field("type", "join")
+                    .startObject("relations")
+                        .field("parent", "child")
+                        .field("child", "grand_child")
+                    .endObject()
+                .endObject()
+            .endObject()
+            .endObject().string();
+        IndexService service = createIndex("test");
+        DocumentMapper docMapper = service.mapperService().merge("type", new CompressedXContent(mapping),
+            MapperService.MergeReason.MAPPING_UPDATE, false);
+        assertTrue(docMapper.mappers().getMapper("join_field") == ParentJoinFieldMapper.getMapper(service.mapperService()));
+        assertFalse(service.mapperService().fullName("join_field").eagerGlobalOrdinals());
+        assertNotNull(service.mapperService().fullName("join_field#parent"));
+        assertTrue(service.mapperService().fullName("join_field#parent").eagerGlobalOrdinals());
+        assertNotNull(service.mapperService().fullName("join_field#child"));
+        assertTrue(service.mapperService().fullName("join_field#child").eagerGlobalOrdinals());
+
+        mapping = XContentFactory.jsonBuilder().startObject()
+            .startObject("properties")
+                .startObject("join_field")
+                    .field("type", "join")
+                    .field("eager_global_ordinals", false)
+                    .startObject("relations")
+                        .field("parent", "child")
+                        .field("child", "grand_child")
+                    .endObject()
+                .endObject()
+            .endObject()
+            .endObject().string();
+        service.mapperService().merge("type", new CompressedXContent(mapping),
+            MapperService.MergeReason.MAPPING_UPDATE, false);
+        assertFalse(service.mapperService().fullName("join_field").eagerGlobalOrdinals());
+        assertNotNull(service.mapperService().fullName("join_field#parent"));
+        assertFalse(service.mapperService().fullName("join_field#parent").eagerGlobalOrdinals());
+        assertNotNull(service.mapperService().fullName("join_field#child"));
+        assertFalse(service.mapperService().fullName("join_field#child").eagerGlobalOrdinals());
+    }
+}
diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/query/ChildQuerySearchIT.java b/modules/parent-join/src/test/java/org/elasticsearch/join/query/ChildQuerySearchIT.java
new file mode 100644
index 0000000000000..978733145ff41
--- /dev/null
+++ b/modules/parent-join/src/test/java/org/elasticsearch/join/query/ChildQuerySearchIT.java
@@ -0,0 +1,2117 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.join.query;
+
+import org.apache.lucene.search.join.ScoreMode;
+import org.elasticsearch.action.explain.ExplainResponse;
+import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.action.search.SearchPhaseExecutionException;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.SearchType;
+import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
+import org.elasticsearch.common.lucene.search.function.CombineFunction;
+import org.elasticsearch.common.lucene.search.function.FiltersFunctionScoreQuery;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.IdsQueryBuilder;
+import org.elasticsearch.index.query.InnerHitBuilder;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
+import org.elasticsearch.index.query.MatchQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.aggregations.AggregationBuilders;
+import org.elasticsearch.search.aggregations.bucket.filter.Filter;
+import org.elasticsearch.search.aggregations.bucket.global.Global;
+import org.elasticsearch.search.aggregations.bucket.terms.Terms;
+import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
+import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder.Field;
+import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
+import org.elasticsearch.search.sort.SortBuilders;
+import org.elasticsearch.search.sort.SortOrder;
+import org.hamcrest.Matchers;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.common.xcontent.support.XContentMapValues.extractValue;
+import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
+import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery;
+import static org.elasticsearch.index.query.QueryBuilders.idsQuery;
+import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
+import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
+import static org.elasticsearch.index.query.QueryBuilders.prefixQuery;
+import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery;
+import static org.elasticsearch.index.query.QueryBuilders.termQuery;
+import static org.elasticsearch.index.query.QueryBuilders.termsQuery;
+import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.fieldValueFactorFunction;
+import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.weightFactorFunction;
+import static org.elasticsearch.join.query.JoinQueryBuilders.hasChildQuery;
+import static org.elasticsearch.join.query.JoinQueryBuilders.hasParentQuery;
+import static org.elasticsearch.join.query.JoinQueryBuilders.parentId;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHit;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasId;
+import static org.hamcrest.Matchers.anyOf;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+
+public class ChildQuerySearchIT extends ParentChildTestCase {
+
+    public void testSelfReferentialIsForbidden() {
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () ->
+            prepareCreate("test").addMapping("type", "_parent", "type=type").get());
+        assertThat(e.getMessage(), equalTo("The [_parent.type] option can't point to the same type"));
+    }
+
+    public void testMultiLevelChild() throws Exception {
+        if (legacy()) {
+            assertAcked(prepareCreate("test")
+                    .addMapping("parent")
+                    .addMapping("child", "_parent", "type=parent")
+                    .addMapping("grandchild", "_parent", "type=child"));
+        } else {
+            assertAcked(prepareCreate("test")
+                .addMapping("doc", buildParentJoinFieldMappingFromSimplifiedDef("join_field", true,
+                    "parent", "child", "child", "grandchild")));
+        }
+
+        createIndexRequest("test", "parent", "p1", null, "p_field", "p_value1").get();
+        createIndexRequest("test", "child", "c1", "p1", "c_field", "c_value1").get();
+        createIndexRequest("test", "grandchild", "gc1", "c1", "gc_field", "gc_value1").setRouting("p1").get();
+        refresh();
+
+        SearchResponse searchResponse = client()
+                .prepareSearch("test")
+                .setQuery(
+                        boolQuery()
+                                .must(matchAllQuery())
+                                .filter(hasChildQuery(
+                                        "child",
+                                        boolQuery().must(termQuery("c_field", "c_value1"))
+                                                .filter(hasChildQuery("grandchild", termQuery("gc_field", "gc_value1"), ScoreMode.None))
+                                        , ScoreMode.None))).get();
+        assertNoFailures(searchResponse);
+        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
+        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("p1"));
+
+        searchResponse = client().prepareSearch("test")
+                .setQuery(boolQuery().must(matchAllQuery())
+                    .filter(hasParentQuery("parent", termQuery("p_field", "p_value1"), false))).execute()
+                .actionGet();
+        assertNoFailures(searchResponse);
+        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
+        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("c1"));
+
+        searchResponse = client().prepareSearch("test")
+                .setQuery(boolQuery().must(matchAllQuery())
+                    .filter(hasParentQuery("child", termQuery("c_field", "c_value1"), false))).execute()
+                .actionGet();
+        assertNoFailures(searchResponse);
+        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
+        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("gc1"));
+
+        searchResponse = client().prepareSearch("test")
+            .setQuery(hasParentQuery("parent", termQuery("p_field", "p_value1"), false)).execute()
+                .actionGet();
+        assertNoFailures(searchResponse);
+        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
+        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("c1"));
+
+        searchResponse = client().prepareSearch("test")
+            .setQuery(hasParentQuery("child", termQuery("c_field", "c_value1"), false)).execute()
+                .actionGet();
+        assertNoFailures(searchResponse);
+        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
+        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("gc1"));
+    }
+
+    // see #2744
+    public void test2744() throws IOException {
+        if (legacy()) {
+            assertAcked(prepareCreate("test")
+                    .addMapping("foo")
+                    .addMapping("test", "_parent", "type=foo"));
+        } else {
+            assertAcked(prepareCreate("test")
+                .addMapping("doc", buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "foo", "test")));
+        }
+        ensureGreen();
+
+        // index simple data
+        createIndexRequest("test", "foo", "1", null, "foo", 1).get();
+        createIndexRequest("test", "test", "2", "1", "foo", 1).get();
+        refresh();
+        SearchResponse searchResponse = client().prepareSearch("test").
+                setQuery(hasChildQuery("test", matchQuery("foo", 1), ScoreMode.None))
+                .get();
+        assertNoFailures(searchResponse);
+        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
+        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("1"));
+
+    }
+
+    public void testSimpleChildQuery() throws Exception {
+        if (legacy()) {
+            assertAcked(prepareCreate("test")
+                    .addMapping("parent")
+                    .addMapping("child", "_parent", "type=parent"));
+        } else {
+            assertAcked(prepareCreate("test")
+                    .addMapping("doc", buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "parent", "child")));
+        }
+        ensureGreen();
+
+        // index simple data
+        createIndexRequest("test", "parent", "p1", null, "p_field", "p_value1").get();
+        createIndexRequest("test", "child", "c1", "p1", "c_field", "red").get();
+        createIndexRequest("test", "child", "c2", "p1", "c_field", "yellow").get();
+        createIndexRequest("test", "parent", "p2", null, "p_field", "p_value2").get();
+        createIndexRequest("test", "child", "c3", "p2", "c_field", "blue").get();
+        createIndexRequest("test", "child", "c4", "p2", "c_field", "red").get();
+        refresh();
+
+        // TEST FETCHING _parent from child
+        SearchResponse searchResponse;
+        if (legacy()) {
+            searchResponse = client().prepareSearch("test")
+                    .setQuery(idsQuery("child").addIds("c1")).storedFields("_parent").get();
+            assertNoFailures(searchResponse);
+            assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
+            assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("c1"));
+            assertThat(searchResponse.getHits().getAt(0).field("_parent").getValue(), equalTo("p1"));
+        } else {
+            searchResponse = client().prepareSearch("test")
+                    .setQuery(idsQuery("doc").addIds("c1")).get();
+            assertNoFailures(searchResponse);
+            assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
+            assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("c1"));
+            assertThat(extractValue("join_field.name", searchResponse.getHits().getAt(0).getSourceAsMap()), equalTo("child"));
+            assertThat(extractValue("join_field.parent", searchResponse.getHits().getAt(0).getSourceAsMap()), equalTo("p1"));
+        }
+
+        // TEST matching on parent
+        if (legacy()) {
+            searchResponse = client().prepareSearch("test").setQuery(termQuery("_parent#parent", "p1")).storedFields("_parent").get();
+            assertNoFailures(searchResponse);
+            assertThat(searchResponse.getHits().getTotalHits(), equalTo(2L));
+            assertThat(searchResponse.getHits().getAt(0).getId(), anyOf(equalTo("c1"), equalTo("c2")));
+            assertThat(searchResponse.getHits().getAt(0).field("_parent").getValue(), equalTo("p1"));
+            assertThat(searchResponse.getHits().getAt(1).getId(), anyOf(equalTo("c1"), equalTo("c2")));
+            assertThat(searchResponse.getHits().getAt(1).field("_parent").getValue(), equalTo("p1"));
+        } else {
+            searchResponse = client().prepareSearch("test")
+                    .setQuery(boolQuery().filter(termQuery("join_field#parent", "p1")).filter(termQuery("join_field", "child")))
+                    .get();
+            assertNoFailures(searchResponse);
+            assertThat(searchResponse.getHits().getTotalHits(), equalTo(2L));
+            assertThat(searchResponse.getHits().getAt(0).getId(), anyOf(equalTo("c1"), equalTo("c2")));
+            assertThat(extractValue("join_field.name", searchResponse.getHits().getAt(0).getSourceAsMap()), equalTo("child"));
+            assertThat(extractValue("join_field.parent", searchResponse.getHits().getAt(0).getSourceAsMap()), equalTo("p1"));
+            assertThat(searchResponse.getHits().getAt(1).getId(), anyOf(equalTo("c1"), equalTo("c2")));
+            assertThat(extractValue("join_field.name", searchResponse.getHits().getAt(1).getSourceAsMap()), equalTo("child"));
+            assertThat(extractValue("join_field.parent", searchResponse.getHits().getAt(1).getSourceAsMap()), equalTo("p1"));
+        }
+
+        if (legacy()) {
+            searchResponse = client().prepareSearch("test").setQuery(queryStringQuery("_parent#parent:p1")).storedFields("_parent").get();
+            assertNoFailures(searchResponse);
+            assertThat(searchResponse.getHits().getTotalHits(), equalTo(2L));
+            assertThat(searchResponse.getHits().getAt(0).getId(), anyOf(equalTo("c1"), equalTo("c2")));
+            assertThat(searchResponse.getHits().getAt(0).field("_parent").getValue(), equalTo("p1"));
+            assertThat(searchResponse.getHits().getAt(1).getId(), anyOf(equalTo("c1"), equalTo("c2")));
+            assertThat(searchResponse.getHits().getAt(1).field("_parent").getValue(), equalTo("p1"));
+        } else {
+            // doesn't make sense for join field, because query string & term query om this field have no special logic.
+        }
+
+        // HAS CHILD
+        searchResponse = client().prepareSearch("test").setQuery(randomHasChild("child", "c_field", "yellow"))
+                .get();
+        assertHitCount(searchResponse, 1L);
+        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
+        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("p1"));
+
+        searchResponse = client().prepareSearch("test").setQuery(randomHasChild("child", "c_field", "blue")).execute()
+                .actionGet();
+        assertHitCount(searchResponse, 1L);
+        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("p2"));
+
+        searchResponse = client().prepareSearch("test").setQuery(randomHasChild("child", "c_field", "red")).get();
+        assertHitCount(searchResponse, 2L);
+        assertThat(searchResponse.getHits().getAt(0).id(), anyOf(equalTo("p2"), equalTo("p1")));
+        assertThat(searchResponse.getHits().getAt(1).id(), anyOf(equalTo("p2"), equalTo("p1")));
+
+        // HAS PARENT
+        searchResponse = client().prepareSearch("test")
+                .setQuery(randomHasParent("parent", "p_field", "p_value2")).get();
+        assertNoFailures(searchResponse);
+        assertHitCount(searchResponse, 2L);
+        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("c3"));
+        assertThat(searchResponse.getHits().getAt(1).id(), equalTo("c4"));
+
+        searchResponse = client().prepareSearch("test")
+                .setQuery(randomHasParent("parent", "p_field", "p_value1")).get();
+        assertHitCount(searchResponse, 2L);
+        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("c1"));
+        assertThat(searchResponse.getHits().getAt(1).id(), equalTo("c2"));
+    }
+
+    // Issue #3290
+    public void testCachingBugWithFqueryFilter() throws Exception {
+        if (legacy()) {
+            assertAcked(prepareCreate("test")
+                .addMapping("parent")
+                .addMapping("child", "_parent", "type=parent"));
+        } else {
+            assertAcked(prepareCreate("test")
+                .addMapping("doc", buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "parent", "child")));
+        }
+        ensureGreen();
+        List<IndexRequestBuilder> builders = new ArrayList<>();
+        // index simple data
+        for (int i = 0; i < 10; i++) {
+            builders.add(createIndexRequest("test", "parent", Integer.toString(i), null, "p_field", i));
+        }
+        indexRandom(randomBoolean(), builders);
+        builders.clear();
+        for (int j = 0; j < 2; j++) {
+            for (int i = 0; i < 10; i++) {
+                builders.add(createIndexRequest("test", "child", j + "-" + i, "0", "c_field", i));
+            }
+            for (int i = 10; i < 20; i++) {
+                builders.add(createIndexRequest("test", "child", j + "-" + i, Integer.toString(i), "c_field", i));
+            }
+
+            if (randomBoolean()) {
+                break; // randomly break out and dont' have deletes / updates
+            }
+        }
+        indexRandom(true, builders);
+
+        for (int i = 1; i <= 10; i++) {
+            logger.info("Round {}", i);
+            SearchResponse searchResponse = client().prepareSearch("test")
+                    .setQuery(constantScoreQuery(hasChildQuery("child", matchAllQuery(), ScoreMode.Max)))
+                    .get();
+            assertNoFailures(searchResponse);
+            searchResponse = client().prepareSearch("test")
+                    .setQuery(constantScoreQuery(hasParentQuery("parent", matchAllQuery(), true)))
+                    .get();
+            assertNoFailures(searchResponse);
+        }
+    }
+
+    public void testHasParentFilter() throws Exception {
+        if (legacy()) {
+            assertAcked(prepareCreate("test")
+                .addMapping("parent")
+                .addMapping("child", "_parent", "type=parent"));
+        } else {
+            assertAcked(prepareCreate("test")
+                .addMapping("doc", buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "parent", "child")));
+        }
+        ensureGreen();
+        Map<String, Set<String>> parentToChildren = new HashMap<>();
+        // Childless parent
+        createIndexRequest("test", "parent", "p0", null, "p_field", "p0").get();
+        parentToChildren.put("p0", new HashSet<>());
+
+        String previousParentId = null;
+        int numChildDocs = 32;
+        int numChildDocsPerParent = 0;
+        List<IndexRequestBuilder> builders = new ArrayList<>();
+        for (int i = 1; i <= numChildDocs; i++) {
+
+            if (previousParentId == null || i % numChildDocsPerParent == 0) {
+                previousParentId = "p" + i;
+                builders.add(createIndexRequest("test", "parent", previousParentId, null, "p_field", previousParentId));
+                numChildDocsPerParent++;
+            }
+
+            String childId = "c" + i;
+            builders.add(createIndexRequest("test", "child", childId, previousParentId, "c_field", childId));
+
+            if (!parentToChildren.containsKey(previousParentId)) {
+                parentToChildren.put(previousParentId, new HashSet<>());
+            }
+            assertThat(parentToChildren.get(previousParentId).add(childId), is(true));
+        }
+        indexRandom(true, builders.toArray(new IndexRequestBuilder[builders.size()]));
+
+        assertThat(parentToChildren.isEmpty(), equalTo(false));
+        for (Map.Entry<String, Set<String>> parentToChildrenEntry : parentToChildren.entrySet()) {
+            SearchResponse searchResponse = client().prepareSearch("test")
+                    .setQuery(constantScoreQuery(hasParentQuery("parent", termQuery("p_field", parentToChildrenEntry.getKey()), false)))
+                    .setSize(numChildDocsPerParent).get();
+
+            assertNoFailures(searchResponse);
+            Set<String> childIds = parentToChildrenEntry.getValue();
+            assertThat(searchResponse.getHits().totalHits(), equalTo((long) childIds.size()));
+            for (int i = 0; i < searchResponse.getHits().totalHits(); i++) {
+                assertThat(childIds.remove(searchResponse.getHits().getAt(i).id()), is(true));
+                assertThat(searchResponse.getHits().getAt(i).score(), is(1.0f));
+            }
+            assertThat(childIds.size(), is(0));
+        }
+    }
+
+    public void testSimpleChildQueryWithFlush() throws Exception {
+        if (legacy()) {
+            assertAcked(prepareCreate("test")
+                .addMapping("parent")
+                .addMapping("child", "_parent", "type=parent"));
+        } else {
+            assertAcked(prepareCreate("test")
+                .addMapping("doc", buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "parent", "child")));
+        }
+        ensureGreen();
+
+        // index simple data with flushes, so we have many segments
+        createIndexRequest("test", "parent", "p1", null, "p_field", "p_value1").get();
+        client().admin().indices().prepareFlush().get();
+        createIndexRequest("test", "child", "c1", "p1", "c_field", "red").get();
+        client().admin().indices().prepareFlush().get();
+        createIndexRequest("test", "child", "c2", "p1", "c_field", "yellow").get();
+        client().admin().indices().prepareFlush().get();
+        createIndexRequest("test", "parent", "p2", null, "p_field", "p_value2").get();
+        client().admin().indices().prepareFlush().get();
+        createIndexRequest("test", "child", "c3", "p2", "c_field", "blue").get();
+        client().admin().indices().prepareFlush().get();
+        createIndexRequest("test", "child", "c4", "p2", "c_field", "red").get();
+        client().admin().indices().prepareFlush().get();
+        refresh();
+
+        // HAS CHILD QUERY
+
+        SearchResponse searchResponse = client().prepareSearch("test")
+                .setQuery(hasChildQuery("child", termQuery("c_field", "yellow"), ScoreMode.None))
+                .get();
+        assertNoFailures(searchResponse);
+        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
+        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("p1"));
+
+        searchResponse = client().prepareSearch("test")
+                .setQuery(hasChildQuery("child", termQuery("c_field", "blue"), ScoreMode.None))
+                .get();
+        assertNoFailures(searchResponse);
+        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
+        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("p2"));
+
+        searchResponse = client().prepareSearch("test")
+                .setQuery(hasChildQuery("child", termQuery("c_field", "red"), ScoreMode.None))
+                .get();
+        assertNoFailures(searchResponse);
+        assertThat(searchResponse.getHits().totalHits(), equalTo(2L));
+        assertThat(searchResponse.getHits().getAt(0).id(), anyOf(equalTo("p2"), equalTo("p1")));
+        assertThat(searchResponse.getHits().getAt(1).id(), anyOf(equalTo("p2"), equalTo("p1")));
+
+        // HAS CHILD FILTER
+        searchResponse = client().prepareSearch("test")
+                .setQuery(constantScoreQuery(hasChildQuery("child", termQuery("c_field", "yellow"), ScoreMode.None)))
+                .get();
+        assertNoFailures(searchResponse);
+        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
+        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("p1"));
+
+        searchResponse = client().prepareSearch("test")
+                .setQuery(constantScoreQuery(hasChildQuery("child", termQuery("c_field", "blue"), ScoreMode.None)))
+                .get();
+        assertNoFailures(searchResponse);
+        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
+        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("p2"));
+
+        searchResponse = client().prepareSearch("test")
+                .setQuery(constantScoreQuery(hasChildQuery("child", termQuery("c_field", "red"), ScoreMode.None)))
+                .get();
+        assertNoFailures(searchResponse);
+        assertThat(searchResponse.getHits().totalHits(), equalTo(2L));
+        assertThat(searchResponse.getHits().getAt(0).id(), anyOf(equalTo("p2"), equalTo("p1")));
+        assertThat(searchResponse.getHits().getAt(1).id(), anyOf(equalTo("p2"), equalTo("p1")));
+    }
+
+    public void testScopedFacet() throws Exception {
+        if (legacy()) {
+            assertAcked(prepareCreate("test")
+                .addMapping("parent")
+                .addMapping("child", "_parent", "type=parent", "c_field", "type=keyword"));
+        } else {
+            assertAcked(prepareCreate("test")
+                .addMapping("doc",
+                    addFieldMappings(buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "parent", "child"),
+                        "c_field", "keyword")));
+        }
+        ensureGreen();
+
+        // index simple data
+        createIndexRequest("test", "parent", "p1", null, "p_field", "p_value1").get();
+        createIndexRequest("test", "child", "c1", "p1", "c_field", "red").get();
+        createIndexRequest("test", "child", "c2", "p1", "c_field", "yellow").get();
+        createIndexRequest("test", "parent", "p2", null, "p_field", "p_value2").get();
+        createIndexRequest("test", "child", "c3", "p2", "c_field", "blue").get();
+        createIndexRequest("test", "child", "c4", "p2", "c_field", "red").get();
+
+        refresh();
+
+        SearchResponse searchResponse = client()
+                .prepareSearch("test")
+                .setQuery(hasChildQuery("child",
+                    boolQuery().should(termQuery("c_field", "red")).should(termQuery("c_field", "yellow")), ScoreMode.None))
+                .addAggregation(AggregationBuilders.global("global").subAggregation(
+                        AggregationBuilders.filter("filter",
+                            boolQuery().should(termQuery("c_field", "red")).should(termQuery("c_field", "yellow"))).subAggregation(
+                                AggregationBuilders.terms("facet1").field("c_field")))).get();
+        assertNoFailures(searchResponse);
+        assertThat(searchResponse.getHits().totalHits(), equalTo(2L));
+        assertThat(searchResponse.getHits().getAt(0).id(), anyOf(equalTo("p2"), equalTo("p1")));
+        assertThat(searchResponse.getHits().getAt(1).id(), anyOf(equalTo("p2"), equalTo("p1")));
+
+        Global global = searchResponse.getAggregations().get("global");
+        Filter filter = global.getAggregations().get("filter");
+        Terms termsFacet = filter.getAggregations().get("facet1");
+        assertThat(termsFacet.getBuckets().size(), equalTo(2));
+        assertThat(termsFacet.getBuckets().get(0).getKeyAsString(), equalTo("red"));
+        assertThat(termsFacet.getBuckets().get(0).getDocCount(), equalTo(2L));
+        assertThat(termsFacet.getBuckets().get(1).getKeyAsString(), equalTo("yellow"));
+        assertThat(termsFacet.getBuckets().get(1).getDocCount(), equalTo(1L));
+    }
+
+    public void testDeletedParent() throws Exception {
+        if (legacy()) {
+            assertAcked(prepareCreate("test")
+                .addMapping("parent")
+                .addMapping("child", "_parent", "type=parent"));
+        } else {
+            assertAcked(prepareCreate("test")
+                .addMapping("doc", buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "parent", "child")));
+        }
+        ensureGreen();
+        // index simple data
+        createIndexRequest("test", "parent", "p1", null, "p_field", "p_value1").get();
+        createIndexRequest("test", "child", "c1", "p1", "c_field", "red").get();
+        createIndexRequest("test", "child", "c2", "p1", "c_field", "yellow").get();
+        createIndexRequest("test", "parent", "p2", null, "p_field", "p_value2").get();
+        createIndexRequest("test", "child", "c3", "p2", "c_field", "blue").get();
+        createIndexRequest("test", "child", "c4", "p2", "c_field", "red").get();
+
+        refresh();
+
+        SearchResponse searchResponse = client().prepareSearch("test")
+                .setQuery(constantScoreQuery(hasChildQuery("child", termQuery("c_field", "yellow"), ScoreMode.None))).get();
+        assertNoFailures(searchResponse);
+        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
+        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("p1"));
+        assertThat(searchResponse.getHits().getAt(0).sourceAsString(), containsString("\"p_value1\""));
+
+        // update p1 and see what that we get updated values...
+
+        createIndexRequest("test", "parent", "p1", null, "p_field", "p_value1_updated").get();
+        client().admin().indices().prepareRefresh().get();
+
+        searchResponse = client().prepareSearch("test")
+                .setQuery(constantScoreQuery(hasChildQuery("child", termQuery("c_field", "yellow"), ScoreMode.None))).get();
+        assertNoFailures(searchResponse);
+        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
+        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("p1"));
+        assertThat(searchResponse.getHits().getAt(0).sourceAsString(), containsString("\"p_value1_updated\""));
+    }
+
+    public void testDfsSearchType() throws Exception {
+        if (legacy()) {
+            assertAcked(prepareCreate("test")
+                .addMapping("parent")
+                .addMapping("child", "_parent", "type=parent"));
+        } else {
+            assertAcked(prepareCreate("test")
+                .addMapping("doc", buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "parent", "child")));
+        }
+        ensureGreen();
+
+        // index simple data
+        createIndexRequest("test", "parent", "p1", null, "p_field", "p_value1").get();
+        createIndexRequest("test", "child", "c1", "p1", "c_field", "red").get();
+        createIndexRequest("test", "child", "c2", "p1", "c_field", "yellow").get();
+        createIndexRequest("test", "parent", "p2", null, "p_field", "p_value2").get();
+        createIndexRequest("test", "child", "c3", "p3", "c_field", "blue").get();
+        createIndexRequest("test", "child", "c4", "p2", "c_field", "red").get();
+
+        refresh();
+
+        SearchResponse searchResponse = client().prepareSearch("test").setSearchType(SearchType.DFS_QUERY_THEN_FETCH)
+                .setQuery(boolQuery().mustNot(hasChildQuery("child", boolQuery().should(queryStringQuery("c_field:*")), ScoreMode.None)))
+                .get();
+        assertNoFailures(searchResponse);
+
+        searchResponse = client().prepareSearch("test").setSearchType(SearchType.DFS_QUERY_THEN_FETCH)
+                .setQuery(boolQuery().mustNot(hasParentQuery("parent",
+                    boolQuery().should(queryStringQuery("p_field:*")), false))).execute()
+                .actionGet();
+        assertNoFailures(searchResponse);
+    }
+
+    public void testHasChildAndHasParentFailWhenSomeSegmentsDontContainAnyParentOrChildDocs() throws Exception {
+        if (legacy()) {
+            assertAcked(prepareCreate("test")
+                .addMapping("parent")
+                .addMapping("child", "_parent", "type=parent"));
+        } else {
+            assertAcked(prepareCreate("test")
+                .addMapping("doc", buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "parent", "child")));
+        }
+        ensureGreen();
+
+        createIndexRequest("test", "parent", "1", null, "p_field", 1).get();
+        createIndexRequest("test", "child", "2", "1", "c_field", 1).get();
+        client().admin().indices().prepareFlush("test").get();
+
+        client().prepareIndex("test", legacy() ? "type1" : "doc", "3").setSource("p_field", 1).get();
+        client().admin().indices().prepareFlush("test").get();
+
+        SearchResponse searchResponse = client().prepareSearch("test")
+                .setQuery(boolQuery().must(matchAllQuery()).filter(hasChildQuery("child", matchAllQuery(), ScoreMode.None))).get();
+        assertNoFailures(searchResponse);
+        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
+
+        searchResponse = client().prepareSearch("test")
+                .setQuery(boolQuery().must(matchAllQuery()).filter(hasParentQuery("parent", matchAllQuery(), false))).get();
+        assertNoFailures(searchResponse);
+        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
+    }
+
+    public void testCountApiUsage() throws Exception {
+        if (legacy()) {
+            assertAcked(prepareCreate("test")
+                .addMapping("parent")
+                .addMapping("child", "_parent", "type=parent"));
+        } else {
+            assertAcked(prepareCreate("test")
+                .addMapping("doc", buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "parent", "child")));
+        }
+        ensureGreen();
+
+        String parentId = "p1";
+        createIndexRequest("test", "parent", parentId, null, "p_field", "1").get();
+        createIndexRequest("test", "child", "c1", parentId, "c_field", "1").get();
+        refresh();
+
+        SearchResponse countResponse = client().prepareSearch("test").setSize(0)
+                .setQuery(hasChildQuery("child", termQuery("c_field", "1"), ScoreMode.Max))
+                .get();
+        assertHitCount(countResponse, 1L);
+
+        countResponse = client().prepareSearch("test").setSize(0)
+            .setQuery(hasParentQuery("parent", termQuery("p_field", "1"), true))
+                .get();
+        assertHitCount(countResponse, 1L);
+
+        countResponse = client().prepareSearch("test").setSize(0)
+                .setQuery(constantScoreQuery(hasChildQuery("child", termQuery("c_field", "1"), ScoreMode.None)))
+                .get();
+        assertHitCount(countResponse, 1L);
+
+        countResponse = client().prepareSearch("test").setSize(0)
+            .setQuery(constantScoreQuery(hasParentQuery("parent", termQuery("p_field", "1"), false)))
+                .get();
+        assertHitCount(countResponse, 1L);
+    }
+
+    public void testExplainUsage() throws Exception {
+        if (legacy()) {
+            assertAcked(prepareCreate("test")
+                .addMapping("parent")
+                .addMapping("child", "_parent", "type=parent"));
+        } else {
+            assertAcked(prepareCreate("test")
+                .addMapping("doc", buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "parent", "child")));
+        }
+        ensureGreen();
+
+        String parentId = "p1";
+        createIndexRequest("test", "parent", parentId, null, "p_field", "1").get();
+        createIndexRequest("test", "child", "c1", parentId, "c_field", "1").get();
+        refresh();
+
+        SearchResponse searchResponse = client().prepareSearch("test")
+                .setExplain(true)
+                .setQuery(hasChildQuery("child", termQuery("c_field", "1"), ScoreMode.Max))
+                .get();
+        assertHitCount(searchResponse, 1L);
+        assertThat(searchResponse.getHits().getAt(0).explanation().getDescription(), containsString("join value p1"));
+
+        searchResponse = client().prepareSearch("test")
+                .setExplain(true)
+                .setQuery(hasParentQuery("parent", termQuery("p_field", "1"), true))
+                .get();
+        assertHitCount(searchResponse, 1L);
+        assertThat(searchResponse.getHits().getAt(0).explanation().getDescription(), containsString("join value p1"));
+
+        ExplainResponse explainResponse = client().prepareExplain("test", legacy() ? "parent" : "doc", parentId)
+                .setQuery(hasChildQuery("child", termQuery("c_field", "1"), ScoreMode.Max))
+                .get();
+        assertThat(explainResponse.isExists(), equalTo(true));
+        assertThat(explainResponse.getExplanation().toString(), containsString("join value p1"));
+    }
+
+    List<IndexRequestBuilder> createDocBuilders() {
+        List<IndexRequestBuilder> indexBuilders = new ArrayList<>();
+        // Parent 1 and its children
+        indexBuilders.add(createIndexRequest("test", "parent", "1", null, "p_field", "p_value1"));
+        indexBuilders.add(createIndexRequest("test", "child", "4", "1", "c_field1", 1, "c_field2", 0));
+        indexBuilders.add(createIndexRequest("test", "child", "5", "1", "c_field1", 1, "c_field2", 0));
+        indexBuilders.add(createIndexRequest("test", "child", "6", "1", "c_field1", 2, "c_field2", 0));
+        indexBuilders.add(createIndexRequest("test", "child", "7", "1", "c_field1", 2, "c_field2", 0));
+        indexBuilders.add(createIndexRequest("test", "child", "8", "1", "c_field1", 1, "c_field2", 1));
+        indexBuilders.add(createIndexRequest("test", "child", "9", "1", "c_field1", 1, "c_field2", 2));
+
+        // Parent 2 and its children
+        indexBuilders.add(createIndexRequest("test", "parent", "2", null, "p_field", "p_value2"));
+        indexBuilders.add(createIndexRequest("test", "child", "10", "2", "c_field1", 3, "c_field2", 0));
+        indexBuilders.add(createIndexRequest("test", "child", "11", "2", "c_field1", 1, "c_field2", 1));
+        indexBuilders.add(createIndexRequest("test", "child", "12", "p", "c_field1", 1, "c_field2", 1)); // why "p"????
+        indexBuilders.add(createIndexRequest("test", "child", "13", "2", "c_field1", 1, "c_field2", 1));
+        indexBuilders.add(createIndexRequest("test", "child", "14", "2", "c_field1", 1, "c_field2", 1));
+        indexBuilders.add(createIndexRequest("test", "child", "15", "2", "c_field1", 1, "c_field2", 2));
+
+        // Parent 3 and its children
+        indexBuilders.add(createIndexRequest("test", "parent", "3", null, "p_field1", "p_value3", "p_field2", 5));
+        indexBuilders.add(createIndexRequest("test", "child", "16", "3", "c_field1", 4, "c_field2", 0, "c_field3", 0));
+        indexBuilders.add(createIndexRequest("test", "child", "17", "3", "c_field1", 1, "c_field2", 1, "c_field3", 1));
+        indexBuilders.add(createIndexRequest("test", "child", "18", "3", "c_field1", 1, "c_field2", 2, "c_field3", 2));
+        indexBuilders.add(createIndexRequest("test", "child", "19", "3", "c_field1", 1, "c_field2", 2, "c_field3", 3));
+        indexBuilders.add(createIndexRequest("test", "child", "20", "3", "c_field1", 1, "c_field2", 2, "c_field3", 4));
+        indexBuilders.add(createIndexRequest("test", "child", "21", "3", "c_field1", 1, "c_field2", 2, "c_field3", 5));
+        indexBuilders.add(createIndexRequest("test", "child1", "22", "3", "c_field1", 1, "c_field2", 2, "c_field3", 6));
+
+        return indexBuilders;
+    }
+
+    public void testScoreForParentChildQueriesWithFunctionScore() throws Exception {
+        if (legacy()) {
+            assertAcked(prepareCreate("test")
+                .addMapping("parent")
+                .addMapping("child", "_parent", "type=parent")
+                .addMapping("child1", "_parent", "type=parent"));
+        } else {
+            assertAcked(prepareCreate("test")
+                .addMapping("doc", jsonBuilder().startObject().startObject("doc").startObject("properties")
+                    .startObject("join_field")
+                        .field("type", "join")
+                        .startObject("relations")
+                            .field("parent", new String[] {"child", "child1"})
+                        .endObject()
+                    .endObject()
+                    .endObject().endObject().endObject()
+                ));
+        }
+        ensureGreen();
+
+        indexRandom(true, createDocBuilders().toArray(new IndexRequestBuilder[0]));
+        SearchResponse response = client()
+                .prepareSearch("test")
+                .setQuery(
+                        hasChildQuery(
+                            "child",
+                            QueryBuilders.functionScoreQuery(matchQuery("c_field2", 0),
+                                fieldValueFactorFunction("c_field1"))
+                                .boostMode(CombineFunction.REPLACE), ScoreMode.Total)).get();
+
+        assertThat(response.getHits().totalHits(), equalTo(3L));
+        assertThat(response.getHits().hits()[0].id(), equalTo("1"));
+        assertThat(response.getHits().hits()[0].score(), equalTo(6f));
+        assertThat(response.getHits().hits()[1].id(), equalTo("3"));
+        assertThat(response.getHits().hits()[1].score(), equalTo(4f));
+        assertThat(response.getHits().hits()[2].id(), equalTo("2"));
+        assertThat(response.getHits().hits()[2].score(), equalTo(3f));
+
+        response = client()
+                .prepareSearch("test")
+                .setQuery(
+                        hasChildQuery(
+                            "child",
+                            QueryBuilders.functionScoreQuery(matchQuery("c_field2", 0),
+                                fieldValueFactorFunction("c_field1"))
+                                .boostMode(CombineFunction.REPLACE), ScoreMode.Max)).get();
+
+        assertThat(response.getHits().totalHits(), equalTo(3L));
+        assertThat(response.getHits().hits()[0].id(), equalTo("3"));
+        assertThat(response.getHits().hits()[0].score(), equalTo(4f));
+        assertThat(response.getHits().hits()[1].id(), equalTo("2"));
+        assertThat(response.getHits().hits()[1].score(), equalTo(3f));
+        assertThat(response.getHits().hits()[2].id(), equalTo("1"));
+        assertThat(response.getHits().hits()[2].score(), equalTo(2f));
+
+        response = client()
+                .prepareSearch("test")
+                .setQuery(
+                        hasChildQuery(
+                            "child",
+                            QueryBuilders.functionScoreQuery(matchQuery("c_field2", 0),
+                                fieldValueFactorFunction("c_field1"))
+                                .boostMode(CombineFunction.REPLACE), ScoreMode.Avg)).get();
+
+        assertThat(response.getHits().totalHits(), equalTo(3L));
+        assertThat(response.getHits().hits()[0].id(), equalTo("3"));
+        assertThat(response.getHits().hits()[0].score(), equalTo(4f));
+        assertThat(response.getHits().hits()[1].id(), equalTo("2"));
+        assertThat(response.getHits().hits()[1].score(), equalTo(3f));
+        assertThat(response.getHits().hits()[2].id(), equalTo("1"));
+        assertThat(response.getHits().hits()[2].score(), equalTo(1.5f));
+
+        response = client()
+                .prepareSearch("test")
+                .setQuery(
+                        hasParentQuery(
+                            "parent",
+                            QueryBuilders.functionScoreQuery(matchQuery("p_field1", "p_value3"),
+                                fieldValueFactorFunction("p_field2"))
+                                .boostMode(CombineFunction.REPLACE), true))
+                .addSort(SortBuilders.fieldSort("c_field3")).addSort(SortBuilders.scoreSort()).get();
+
+        assertThat(response.getHits().getTotalHits(), equalTo(7L));
+        assertThat(response.getHits().getHits()[0].getId(), equalTo("16"));
+        assertThat(response.getHits().getHits()[0].getScore(), equalTo(5f));
+        assertThat(response.getHits().getHits()[1].getId(), equalTo("17"));
+        assertThat(response.getHits().getHits()[1].getScore(), equalTo(5f));
+        assertThat(response.getHits().getHits()[2].getId(), equalTo("18"));
+        assertThat(response.getHits().getHits()[2].getScore(), equalTo(5f));
+        assertThat(response.getHits().getHits()[3].getId(), equalTo("19"));
+        assertThat(response.getHits().getHits()[3].getScore(), equalTo(5f));
+        assertThat(response.getHits().getHits()[4].getId(), equalTo("20"));
+        assertThat(response.getHits().getHits()[4].getScore(), equalTo(5f));
+        assertThat(response.getHits().getHits()[5].getId(), equalTo("21"));
+        assertThat(response.getHits().getHits()[5].getScore(), equalTo(5f));
+        assertThat(response.getHits().getHits()[6].getId(), equalTo("22"));
+        assertThat(response.getHits().getHits()[6].getScore(), equalTo(5f));
+    }
+
+    // Issue #2536
+    public void testParentChildQueriesCanHandleNoRelevantTypesInIndex() throws Exception {
+        if (legacy()) {
+            assertAcked(prepareCreate("test")
+                .addMapping("parent")
+                .addMapping("child", "_parent", "type=parent"));
+        } else {
+            assertAcked(prepareCreate("test")
+                .addMapping("doc", buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "parent", "child")));
+        }
+        ensureGreen();
+
+        SearchResponse response = client().prepareSearch("test")
+                .setQuery(hasChildQuery("child", matchQuery("text", "value"), ScoreMode.None)).get();
+        assertNoFailures(response);
+        assertThat(response.getHits().totalHits(), equalTo(0L));
+
+        if (legacy()) {
+            client().prepareIndex("test", "child1").setSource(jsonBuilder().startObject().field("text", "value").endObject())
+                .setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
+        } else {
+            client().prepareIndex("test", "doc").setSource(jsonBuilder().startObject().field("text", "value").endObject())
+                .setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
+        }
+
+        response = client().prepareSearch("test")
+                .setQuery(hasChildQuery("child", matchQuery("text", "value"), ScoreMode.None)).get();
+        assertNoFailures(response);
+        assertThat(response.getHits().totalHits(), equalTo(0L));
+
+        response = client().prepareSearch("test").setQuery(hasChildQuery("child", matchQuery("text", "value"), ScoreMode.Max))
+                .get();
+        assertNoFailures(response);
+        assertThat(response.getHits().totalHits(), equalTo(0L));
+
+        response = client().prepareSearch("test")
+            .setQuery(hasParentQuery("parent", matchQuery("text", "value"), false)).get();
+        assertNoFailures(response);
+        assertThat(response.getHits().totalHits(), equalTo(0L));
+
+        response = client().prepareSearch("test").setQuery(hasParentQuery("parent", matchQuery("text", "value"), true))
+                .get();
+        assertNoFailures(response);
+        assertThat(response.getHits().totalHits(), equalTo(0L));
+    }
+
+    public void testHasChildAndHasParentFilter_withFilter() throws Exception {
+        if (legacy()) {
+            assertAcked(prepareCreate("test")
+                .addMapping("parent")
+                .addMapping("child", "_parent", "type=parent"));
+        } else {
+            assertAcked(prepareCreate("test")
+                .addMapping("doc", buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "parent", "child")));
+        }
+        ensureGreen();
+
+        createIndexRequest("test", "parent", "1", null, "p_field", 1).get();
+        createIndexRequest("test", "child", "2", "1", "c_field", 1).get();
+        client().admin().indices().prepareFlush("test").get();
+
+        if (legacy()) {
+            client().prepareIndex("test", "type1", "3").setSource("p_field", 2).get();
+        } else {
+            client().prepareIndex("test", "doc", "3").setSource("p_field", 2).get();
+        }
+        client().admin().indices().prepareFlush("test").get();
+
+        SearchResponse searchResponse = client().prepareSearch("test")
+                .setQuery(boolQuery().must(matchAllQuery()).filter(hasChildQuery("child", termQuery("c_field", 1), ScoreMode.None)))
+                .get();
+        assertNoFailures(searchResponse);
+        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
+        assertThat(searchResponse.getHits().hits()[0].id(), equalTo("1"));
+
+        searchResponse = client().prepareSearch("test")
+                .setQuery(boolQuery().must(matchAllQuery())
+                    .filter(hasParentQuery("parent", termQuery("p_field", 1), false))).get();
+        assertNoFailures(searchResponse);
+        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
+        assertThat(searchResponse.getHits().hits()[0].id(), equalTo("2"));
+    }
+
+    public void testHasChildInnerHitsHighlighting() throws Exception {
+        if (legacy()) {
+            assertAcked(prepareCreate("test")
+                .addMapping("parent")
+                .addMapping("child", "_parent", "type=parent"));
+        } else {
+            assertAcked(prepareCreate("test")
+                .addMapping("doc", buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "parent", "child")));
+        }
+        ensureGreen();
+
+        createIndexRequest("test", "parent", "1", null, "p_field", 1).get();
+        createIndexRequest("test", "child", "2", "1", "c_field", "foo bar").get();
+        client().admin().indices().prepareFlush("test").get();
+
+        SearchResponse searchResponse = client().prepareSearch("test").setQuery(
+                hasChildQuery("child", matchQuery("c_field", "foo"), ScoreMode.None)
+                    .innerHit(new InnerHitBuilder().setHighlightBuilder(
+                        new HighlightBuilder().field(new Field("c_field")
+                                .highlightQuery(QueryBuilders.matchQuery("c_field", "bar"))))))
+                .get();
+        assertNoFailures(searchResponse);
+        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
+        assertThat(searchResponse.getHits().hits()[0].id(), equalTo("1"));
+        SearchHit[] searchHits = searchResponse.getHits().hits()[0].getInnerHits().get("child").hits();
+        assertThat(searchHits.length, equalTo(1));
+        assertThat(searchHits[0].getHighlightFields().get("c_field").getFragments().length, equalTo(1));
+        assertThat(searchHits[0].getHighlightFields().get("c_field").getFragments()[0].string(), equalTo("foo <em>bar</em>"));
+    }
+
+    public void testHasChildAndHasParentWrappedInAQueryFilter() throws Exception {
+        if (legacy()) {
+            assertAcked(prepareCreate("test")
+                .addMapping("parent")
+                .addMapping("child", "_parent", "type=parent"));
+        } else {
+            assertAcked(prepareCreate("test")
+                .addMapping("doc", buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "parent", "child")));
+        }
+        ensureGreen();
+
+        // query filter in case for p/c shouldn't execute per segment, but rather
+        createIndexRequest("test", "parent", "1", null, "p_field", 1).get();
+        client().admin().indices().prepareFlush("test").setForce(true).get();
+        createIndexRequest("test", "child", "2", "1", "c_field", 1).get();
+        refresh();
+
+        SearchResponse searchResponse = client().prepareSearch("test")
+                .setQuery(boolQuery().must(matchAllQuery()).filter(hasChildQuery("child", matchQuery("c_field", 1), ScoreMode.None)))
+                .get();
+        assertSearchHit(searchResponse, 1, hasId("1"));
+
+        searchResponse = client().prepareSearch("test")
+                .setQuery(boolQuery().must(matchAllQuery()).filter(hasParentQuery("parent", matchQuery("p_field", 1), false))).get();
+        assertSearchHit(searchResponse, 1, hasId("2"));
+
+        searchResponse = client().prepareSearch("test")
+                .setQuery(boolQuery().must(matchAllQuery())
+                    .filter(boolQuery().must(hasChildQuery("child", matchQuery("c_field", 1), ScoreMode.None))))
+                .get();
+        assertSearchHit(searchResponse, 1, hasId("1"));
+
+        searchResponse = client().prepareSearch("test")
+                .setQuery(boolQuery().must(matchAllQuery())
+                    .filter(boolQuery().must(hasParentQuery("parent", matchQuery("p_field", 1), false)))).get();
+        assertSearchHit(searchResponse, 1, hasId("2"));
+    }
+
+    public void testSimpleQueryRewrite() throws Exception {
+        if (legacy()) {
+            assertAcked(prepareCreate("test")
+                .addMapping("parent", "p_field", "type=keyword")
+                .addMapping("child", "_parent", "type=parent", "c_field", "type=keyword"));
+        } else {
+            assertAcked(prepareCreate("test")
+                .addMapping("doc", addFieldMappings(buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "parent", "child"),
+                    "c_field", "keyword", "p_field", "keyword")));
+        }
+        ensureGreen();
+
+        // index simple data
+        int childId = 0;
+        for (int i = 0; i < 10; i++) {
+            String parentId = String.format(Locale.ROOT, "p%03d", i);
+            createIndexRequest("test", "parent", parentId, null, "p_field", parentId).get();
+            int j = childId;
+            for (; j < childId + 50; j++) {
+                String childUid = String.format(Locale.ROOT, "c%03d", j);
+                createIndexRequest("test", "child", childUid, parentId, "c_field", childUid).get();
+            }
+            childId = j;
+        }
+        refresh();
+
+        SearchType[] searchTypes = new SearchType[]{SearchType.QUERY_THEN_FETCH, SearchType.DFS_QUERY_THEN_FETCH};
+        for (SearchType searchType : searchTypes) {
+            SearchResponse searchResponse = client().prepareSearch("test").setSearchType(searchType)
+                    .setQuery(hasChildQuery("child", prefixQuery("c_field", "c"), ScoreMode.Max))
+                    .addSort("p_field", SortOrder.ASC)
+                    .setSize(5).get();
+            assertNoFailures(searchResponse);
+            assertThat(searchResponse.getHits().totalHits(), equalTo(10L));
+            assertThat(searchResponse.getHits().hits()[0].id(), equalTo("p000"));
+            assertThat(searchResponse.getHits().hits()[1].id(), equalTo("p001"));
+            assertThat(searchResponse.getHits().hits()[2].id(), equalTo("p002"));
+            assertThat(searchResponse.getHits().hits()[3].id(), equalTo("p003"));
+            assertThat(searchResponse.getHits().hits()[4].id(), equalTo("p004"));
+
+            searchResponse = client().prepareSearch("test").setSearchType(searchType)
+                    .setQuery(hasParentQuery("parent", prefixQuery("p_field", "p"), true)).addSort("c_field", SortOrder.ASC)
+                    .setSize(5).get();
+            assertNoFailures(searchResponse);
+            assertThat(searchResponse.getHits().totalHits(), equalTo(500L));
+            assertThat(searchResponse.getHits().hits()[0].id(), equalTo("c000"));
+            assertThat(searchResponse.getHits().hits()[1].id(), equalTo("c001"));
+            assertThat(searchResponse.getHits().hits()[2].id(), equalTo("c002"));
+            assertThat(searchResponse.getHits().hits()[3].id(), equalTo("c003"));
+            assertThat(searchResponse.getHits().hits()[4].id(), equalTo("c004"));
+        }
+    }
+
+    // Issue #3144
+    public void testReIndexingParentAndChildDocuments() throws Exception {
+        if (legacy()) {
+            assertAcked(prepareCreate("test")
+                .addMapping("parent")
+                .addMapping("child", "_parent", "type=parent"));
+        } else {
+            assertAcked(prepareCreate("test")
+                .addMapping("doc", buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "parent", "child")));
+        }
+        ensureGreen();
+
+        // index simple data
+        createIndexRequest("test", "parent", "p1", null, "p_field", "p_value1").get();
+        createIndexRequest("test", "child", "c1", "p1", "c_field", "red").get();
+        createIndexRequest("test", "child", "c2", "p1", "c_field", "yellow").get();
+        createIndexRequest("test", "parent", "p2", null, "p_field", "p_value2").get();
+        createIndexRequest("test", "child", "c3", "p2", "c_field", "x").get();
+        createIndexRequest("test", "child", "c4", "p2", "c_field", "x").get();
+
+        refresh();
+
+        SearchResponse searchResponse = client().prepareSearch("test")
+                .setQuery(hasChildQuery("child", termQuery("c_field", "yellow"), ScoreMode.Total)).get();
+        assertNoFailures(searchResponse);
+        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
+        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("p1"));
+        assertThat(searchResponse.getHits().getAt(0).sourceAsString(), containsString("\"p_value1\""));
+
+        searchResponse = client()
+                .prepareSearch("test")
+                .setQuery(
+                        boolQuery().must(matchQuery("c_field", "x")).must(
+                                hasParentQuery("parent", termQuery("p_field", "p_value2"), true))).get();
+        assertNoFailures(searchResponse);
+        assertThat(searchResponse.getHits().totalHits(), equalTo(2L));
+        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("c3"));
+        assertThat(searchResponse.getHits().getAt(1).id(), equalTo("c4"));
+
+        // re-index
+        for (int i = 0; i < 10; i++) {
+            createIndexRequest("test", "parent", "p1", null, "p_field", "p_value1").get();
+            createIndexRequest("test", "child", "d" + i, "p1", "c_field", "red").get();
+            createIndexRequest("test", "parent", "p2", null, "p_field", "p_value2").get();
+            createIndexRequest("test", "child", "c3", "p2", "c_field", "x").get();
+            client().admin().indices().prepareRefresh("test").get();
+        }
+
+        searchResponse = client().prepareSearch("test")
+            .setQuery(hasChildQuery("child", termQuery("c_field", "yellow"), ScoreMode.Total))
+                .get();
+        assertNoFailures(searchResponse);
+        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
+        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("p1"));
+        assertThat(searchResponse.getHits().getAt(0).sourceAsString(), containsString("\"p_value1\""));
+
+        searchResponse = client()
+                .prepareSearch("test")
+                .setQuery(
+                        boolQuery().must(matchQuery("c_field", "x")).must(
+                                hasParentQuery("parent", termQuery("p_field", "p_value2"), true))).get();
+        assertNoFailures(searchResponse);
+        assertThat(searchResponse.getHits().totalHits(), equalTo(2L));
+        assertThat(searchResponse.getHits().getAt(0).id(), Matchers.anyOf(equalTo("c3"), equalTo("c4")));
+        assertThat(searchResponse.getHits().getAt(1).id(), Matchers.anyOf(equalTo("c3"), equalTo("c4")));
+    }
+
+    // Issue #3203
+    public void testHasChildQueryWithMinimumScore() throws Exception {
+        if (legacy()) {
+            assertAcked(prepareCreate("test")
+                .addMapping("parent")
+                .addMapping("child", "_parent", "type=parent"));
+        } else {
+            assertAcked(prepareCreate("test")
+                .addMapping("doc", buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "parent", "child")));
+        }
+        ensureGreen();
+
+        // index simple data
+        createIndexRequest("test", "parent", "p1", null, "p_field", "p_value1").get();
+        createIndexRequest("test", "child", "c1", "p1", "c_field", "x").get();
+        createIndexRequest("test", "parent", "p2", null, "p_field", "p_value2").get();
+        createIndexRequest("test", "child", "c3", "p2", "c_field", "x").get();
+        createIndexRequest("test", "child", "c4", "p2", "c_field", "x").get();
+        createIndexRequest("test", "child", "c5", "p2", "c_field", "x").get();
+        refresh();
+
+        SearchResponse searchResponse = client()
+            .prepareSearch("test").setQuery(hasChildQuery("child", matchAllQuery(), ScoreMode.Total))
+                .setMinScore(3) // Score needs to be 3 or above!
+                .get();
+        assertNoFailures(searchResponse);
+        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
+        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("p2"));
+        assertThat(searchResponse.getHits().getAt(0).score(), equalTo(3.0f));
+    }
+
+    public void testParentFieldQuery() throws Exception {
+        if (legacy()) {
+            assertAcked(prepareCreate("test")
+                .setSettings(Settings.builder()
+                    .put(indexSettings())
+                    .put("index.refresh_interval", -1)
+                )
+                .addMapping("parent")
+                .addMapping("child", "_parent", "type=parent"));
+        } else {
+            assertAcked(prepareCreate("test")
+                .setSettings(Settings.builder()
+                    .put(indexSettings())
+                    .put("index.refresh_interval", -1)
+                )
+                .addMapping("doc", buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "parent", "child")));
+        }
+        ensureGreen();
+
+        SearchResponse response;
+        if (legacy()){
+            response = client().prepareSearch("test").setQuery(termQuery("_parent#parent:p1", "p1"))
+                .get();
+        } else {
+            response = client().prepareSearch("test")
+                .setQuery(boolQuery().filter(termQuery("join_field#parent", "p1")).filter(termQuery("join_field", "child")))
+                .get();
+        }
+        assertHitCount(response, 0L);
+
+        createIndexRequest("test", "child", "c1", "p1").get();
+        refresh();
+
+        if (legacy()){
+            response = client().prepareSearch("test").setQuery(termQuery("_parent#parent", "p1"))
+                .get();
+        } else {
+            response = client().prepareSearch("test")
+                .setQuery(boolQuery().filter(termQuery("join_field#parent", "p1")).filter(termQuery("join_field", "child")))
+                .get();
+        }
+        assertHitCount(response, 1L);
+
+        if (legacy()) {
+            response = client().prepareSearch("test").setQuery(queryStringQuery("_parent#parent:p1")).get();
+            assertHitCount(response, 1L);
+        }
+
+        createIndexRequest("test", "child", "c2", "p2").get();
+        refresh();
+        if (legacy()) {
+            response = client().prepareSearch("test").setQuery(termsQuery("_parent#parent", "p1", "p2")).get();
+            assertHitCount(response, 2L);
+        }
+
+        if (legacy()) {
+            response = client().prepareSearch("test")
+                .setQuery(boolQuery()
+                    .should(termQuery("_parent#parent", "p1"))
+                    .should(termQuery("_parent#parent", "p2"))
+                ).get();
+        } else {
+            response = client().prepareSearch("test")
+                .setQuery(boolQuery()
+                    .should(boolQuery().filter(termQuery("join_field#parent", "p1")).filter(termQuery("join_field", "child")))
+                    .should(boolQuery().filter(termQuery("join_field#parent", "p2")).filter(termQuery("join_field", "child")))
+                ).get();
+        }
+        assertHitCount(response, 2L);
+    }
+
+    public void testParentIdQuery() throws Exception {
+        if (legacy()) {
+            assertAcked(prepareCreate("test")
+                .setSettings(Settings.builder()
+                    .put(indexSettings())
+                    .put("index.refresh_interval", -1)
+                )
+                .addMapping("parent")
+                .addMapping("child", "_parent", "type=parent"));
+        } else {
+            assertAcked(prepareCreate("test")
+                .setSettings(Settings.builder()
+                    .put(indexSettings())
+                    .put("index.refresh_interval", -1)
+                )
+                .addMapping("doc", buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "parent", "child")));
+        }
+        ensureGreen();
+
+        createIndexRequest("test", "child", "c1", "p1").get();
+        refresh();
+
+        SearchResponse response = client().prepareSearch("test").setQuery(parentId("child", "p1")).get();
+        assertHitCount(response, 1L);
+
+        createIndexRequest("test", "child", "c2", "p2").get();
+        refresh();
+
+        response = client().prepareSearch("test")
+            .setQuery(boolQuery()
+                .should(parentId("child", "p1"))
+                .should(parentId("child", "p2"))
+            ).get();
+        assertHitCount(response, 2L);
+    }
+
+    public void testHasChildNotBeingCached() throws IOException {
+        if (legacy()) {
+            assertAcked(prepareCreate("test")
+                .addMapping("parent")
+                .addMapping("child", "_parent", "type=parent"));
+        } else {
+            assertAcked(prepareCreate("test")
+                .addMapping("doc", buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "parent", "child")));
+        }
+        ensureGreen();
+
+        // index simple data
+        createIndexRequest("test", "parent", "p1", null, "p_field", "p_value1").get();
+        createIndexRequest("test", "parent", "p2", null, "p_field", "p_value2").get();
+        createIndexRequest("test", "parent", "p3", null, "p_field", "p_value3").get();
+        createIndexRequest("test", "parent", "p4", null, "p_field", "p_value4").get();
+        createIndexRequest("test", "parent", "p5", null, "p_field", "p_value5").get();
+        createIndexRequest("test", "parent", "p6", null, "p_field", "p_value6").get();
+        createIndexRequest("test", "parent", "p7", null, "p_field", "p_value7").get();
+        createIndexRequest("test", "parent", "p8", null, "p_field", "p_value8").get();
+        createIndexRequest("test", "parent", "p9", null, "p_field", "p_value9").get();
+        createIndexRequest("test", "parent", "p10", null, "p_field", "p_value10").get();
+        createIndexRequest("test", "child", "c1", "p1", "c_field", "blue").get();
+        client().admin().indices().prepareFlush("test").get();
+        client().admin().indices().prepareRefresh("test").get();
+
+        SearchResponse searchResponse = client().prepareSearch("test")
+                .setQuery(constantScoreQuery(hasChildQuery("child", termQuery("c_field", "blue"), ScoreMode.None)))
+                .get();
+        assertNoFailures(searchResponse);
+        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
+
+        createIndexRequest("test", "child", "c2", "p2", "c_field", "blue").get();
+        client().admin().indices().prepareRefresh("test").get();
+
+        searchResponse = client().prepareSearch("test")
+                .setQuery(constantScoreQuery(hasChildQuery("child", termQuery("c_field", "blue"), ScoreMode.None)))
+                .get();
+        assertNoFailures(searchResponse);
+        assertThat(searchResponse.getHits().totalHits(), equalTo(2L));
+    }
+
+    private QueryBuilder randomHasChild(String type, String field, String value) {
+        if (randomBoolean()) {
+            if (randomBoolean()) {
+                return constantScoreQuery(hasChildQuery(type, termQuery(field, value), ScoreMode.None));
+            } else {
+                return boolQuery().must(matchAllQuery()).filter(hasChildQuery(type, termQuery(field, value), ScoreMode.None));
+            }
+        } else {
+            return hasChildQuery(type, termQuery(field, value), ScoreMode.None);
+        }
+    }
+
+    private QueryBuilder randomHasParent(String type, String field, String value) {
+        if (randomBoolean()) {
+            if (randomBoolean()) {
+                return constantScoreQuery(hasParentQuery(type, termQuery(field, value), false));
+            } else {
+                return boolQuery().must(matchAllQuery()).filter(hasParentQuery(type, termQuery(field, value), false));
+            }
+        } else {
+            return hasParentQuery(type, termQuery(field, value), false);
+        }
+    }
+
+    // Issue #3818
+    public void testHasChildQueryOnlyReturnsSingleChildType() throws Exception {
+        if (legacy()) {
+            assertAcked(prepareCreate("grandissue")
+                .addMapping("grandparent", "name", "type=text")
+                .addMapping("parent", "_parent", "type=grandparent")
+                .addMapping("child_type_one", "_parent", "type=parent")
+                .addMapping("child_type_two", "_parent", "type=parent"));
+        } else {
+            assertAcked(prepareCreate("grandissue")
+                .addMapping("doc", jsonBuilder().startObject().startObject("doc").startObject("properties")
+                    .startObject("join_field")
+                        .field("type", "join")
+                        .startObject("relations")
+                            .field("grandparent", "parent")
+                            .field("parent", new String[] {"child_type_one", "child_type_two"})
+                        .endObject()
+                    .endObject()
+                    .endObject().endObject().endObject()
+                ));
+        }
+
+        createIndexRequest("grandissue", "grandparent", "1", null, "name", "Grandpa").get();
+        createIndexRequest("grandissue", "parent", "2", "1", "name", "Dana").get();
+        createIndexRequest("grandissue", "child_type_one", "3", "2", "name", "William").setRouting("1").get();
+        createIndexRequest("grandissue", "child_type_two", "4", "2", "name", "Kate").setRouting("1").get();
+        refresh();
+
+        SearchResponse searchResponse = client().prepareSearch("grandissue").setQuery(
+                boolQuery().must(
+                        hasChildQuery(
+                                "parent",
+                                boolQuery().must(
+                                        hasChildQuery(
+                                                "child_type_one",
+                                                boolQuery().must(
+                                                        queryStringQuery("name:William*")
+                                                ),
+                                                ScoreMode.None)
+                                ),
+                                ScoreMode.None)
+                )
+        ).get();
+        assertHitCount(searchResponse, 1L);
+
+        searchResponse = client().prepareSearch("grandissue").setQuery(
+                boolQuery().must(
+                        hasChildQuery(
+                                "parent",
+                                boolQuery().must(
+                                        hasChildQuery(
+                                                "child_type_two",
+                                                boolQuery().must(
+                                                        queryStringQuery("name:William*")
+                                                ),
+                                                ScoreMode.None)
+                                ),
+                                ScoreMode.None)
+                )
+        ).get();
+        assertHitCount(searchResponse, 0L);
+    }
+
+    public void testHasChildQueryWithNestedInnerObjects() throws Exception {
+        if (legacy()) {
+            assertAcked(prepareCreate("test")
+                .addMapping("parent", "objects", "type=nested")
+                .addMapping("child", "_parent", "type=parent"));
+        } else {
+            assertAcked(prepareCreate("test")
+                .addMapping("doc",
+                    addFieldMappings(buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "parent", "child"),
+                        "objects", "nested")));
+        }
+        ensureGreen();
+
+        createIndexRequest("test", "parent", "p1", null, jsonBuilder().startObject().field("p_field", "1").startArray("objects")
+                        .startObject().field("i_field", "1").endObject()
+                        .startObject().field("i_field", "2").endObject()
+                        .startObject().field("i_field", "3").endObject()
+                        .startObject().field("i_field", "4").endObject()
+                        .startObject().field("i_field", "5").endObject()
+                        .startObject().field("i_field", "6").endObject()
+                        .endArray().endObject())
+                .get();
+        createIndexRequest("test", "parent", "p2", null, jsonBuilder().startObject().field("p_field", "2").startArray("objects")
+                        .startObject().field("i_field", "1").endObject()
+                        .startObject().field("i_field", "2").endObject()
+                        .endArray().endObject())
+                .get();
+        createIndexRequest("test", "child", "c1", "p1", "c_field", "blue").get();
+        createIndexRequest("test", "child", "c2", "p1", "c_field", "red").get();
+        createIndexRequest("test", "child", "c3", "p2", "c_field", "red").get();
+        refresh();
+
+        ScoreMode scoreMode = randomFrom(ScoreMode.values());
+        SearchResponse searchResponse = client().prepareSearch("test")
+                .setQuery(boolQuery().must(hasChildQuery("child", termQuery("c_field", "blue"), scoreMode))
+                    .filter(boolQuery().mustNot(termQuery("p_field", "3"))))
+                .get();
+        assertNoFailures(searchResponse);
+        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
+
+        searchResponse = client().prepareSearch("test")
+                .setQuery(boolQuery().must(hasChildQuery("child", termQuery("c_field", "red"), scoreMode))
+                    .filter(boolQuery().mustNot(termQuery("p_field", "3"))))
+                .get();
+        assertNoFailures(searchResponse);
+        assertThat(searchResponse.getHits().totalHits(), equalTo(2L));
+    }
+
+    public void testNamedFilters() throws Exception {
+        if (legacy()) {
+            assertAcked(prepareCreate("test")
+                .addMapping("parent")
+                .addMapping("child", "_parent", "type=parent"));
+        } else {
+            assertAcked(prepareCreate("test")
+                .addMapping("doc", buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "parent", "child")));
+        }
+        ensureGreen();
+
+        String parentId = "p1";
+        createIndexRequest("test", "parent", parentId, null, "p_field", "1").get();
+        createIndexRequest("test", "child", "c1", parentId, "c_field", "1").get();
+        refresh();
+
+        SearchResponse searchResponse = client().prepareSearch("test").setQuery(hasChildQuery("child",
+            termQuery("c_field", "1"), ScoreMode.Max).queryName("test"))
+                .get();
+        assertHitCount(searchResponse, 1L);
+        assertThat(searchResponse.getHits().getAt(0).getMatchedQueries().length, equalTo(1));
+        assertThat(searchResponse.getHits().getAt(0).getMatchedQueries()[0], equalTo("test"));
+
+        searchResponse = client().prepareSearch("test").setQuery(hasParentQuery("parent",
+            termQuery("p_field", "1"), true).queryName("test"))
+                .get();
+        assertHitCount(searchResponse, 1L);
+        assertThat(searchResponse.getHits().getAt(0).getMatchedQueries().length, equalTo(1));
+        assertThat(searchResponse.getHits().getAt(0).getMatchedQueries()[0], equalTo("test"));
+
+        searchResponse = client().prepareSearch("test").setQuery(constantScoreQuery(hasChildQuery("child",
+            termQuery("c_field", "1"), ScoreMode.None).queryName("test")))
+                .get();
+        assertHitCount(searchResponse, 1L);
+        assertThat(searchResponse.getHits().getAt(0).getMatchedQueries().length, equalTo(1));
+        assertThat(searchResponse.getHits().getAt(0).getMatchedQueries()[0], equalTo("test"));
+
+        searchResponse = client().prepareSearch("test").setQuery(constantScoreQuery(hasParentQuery("parent",
+            termQuery("p_field", "1"), false).queryName("test")))
+                .get();
+        assertHitCount(searchResponse, 1L);
+        assertThat(searchResponse.getHits().getAt(0).getMatchedQueries().length, equalTo(1));
+        assertThat(searchResponse.getHits().getAt(0).getMatchedQueries()[0], equalTo("test"));
+    }
+
+    public void testParentChildQueriesNoParentType() throws Exception {
+        assertAcked(prepareCreate("test")
+                .setSettings(Settings.builder()
+                        .put(indexSettings())
+                        .put("index.refresh_interval", -1)));
+        ensureGreen();
+
+        String parentId = "p1";
+        client().prepareIndex("test", legacy() ? "parent" : "doc", parentId).setSource("p_field", "1").get();
+        refresh();
+
+        try {
+            client().prepareSearch("test")
+                    .setQuery(hasChildQuery("child", termQuery("c_field", "1"), ScoreMode.None))
+                    .get();
+            fail();
+        } catch (SearchPhaseExecutionException e) {
+            assertThat(e.status(), equalTo(RestStatus.BAD_REQUEST));
+        }
+
+        try {
+            client().prepareSearch("test")
+                    .setQuery(hasChildQuery("child", termQuery("c_field", "1"), ScoreMode.Max))
+                    .get();
+            fail();
+        } catch (SearchPhaseExecutionException e) {
+            assertThat(e.status(), equalTo(RestStatus.BAD_REQUEST));
+        }
+
+        try {
+            client().prepareSearch("test")
+                    .setPostFilter(hasChildQuery("child", termQuery("c_field", "1"), ScoreMode.None))
+                    .get();
+            fail();
+        } catch (SearchPhaseExecutionException e) {
+            assertThat(e.status(), equalTo(RestStatus.BAD_REQUEST));
+        }
+
+        try {
+            client().prepareSearch("test")
+                    .setQuery(hasParentQuery("parent", termQuery("p_field", "1"), true))
+                    .get();
+            fail();
+        } catch (SearchPhaseExecutionException e) {
+            assertThat(e.status(), equalTo(RestStatus.BAD_REQUEST));
+        }
+
+        try {
+            client().prepareSearch("test")
+                    .setPostFilter(hasParentQuery("parent", termQuery("p_field", "1"), false))
+                    .get();
+            fail();
+        } catch (SearchPhaseExecutionException e) {
+            assertThat(e.status(), equalTo(RestStatus.BAD_REQUEST));
+        }
+    }
+
+    public void testParentChildCaching() throws Exception {
+        if (legacy()) {
+            assertAcked(prepareCreate("test")
+                .setSettings(Settings.builder()
+                    .put(indexSettings())
+                    .put("index.refresh_interval", -1)
+                )
+                .addMapping("parent")
+                .addMapping("child", "_parent", "type=parent"));
+        } else {
+            assertAcked(prepareCreate("test")
+                .setSettings(Settings.builder()
+                    .put(indexSettings())
+                    .put("index.refresh_interval", -1)
+                )
+                .addMapping("doc", buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "parent", "child")));
+        }
+        ensureGreen();
+
+        // index simple data
+        createIndexRequest("test", "parent", "p1", null, "p_field", "p_value1").get();
+        createIndexRequest("test", "parent", "p2", null, "p_field", "p_value2").get();
+        createIndexRequest("test", "child", "c1", "p1", "c_field", "blue").get();
+        createIndexRequest("test", "child", "c2", "p1", "c_field", "red").get();
+        createIndexRequest("test", "child", "c3", "p2", "c_field", "red").get();
+        client().admin().indices().prepareForceMerge("test").setMaxNumSegments(1).setFlush(true).get();
+        createIndexRequest("test", "parent", "p3", null, "p_field", "p_value3").get();
+        createIndexRequest("test", "parent", "p4", null, "p_field", "p_value4").get();
+        createIndexRequest("test", "child", "c4", "p3", "c_field", "green").get();
+        createIndexRequest("test", "child", "c5", "p3", "c_field", "blue").get();
+        createIndexRequest("test", "child", "c6", "p4", "c_field", "blue").get();
+        client().admin().indices().prepareFlush("test").get();
+        client().admin().indices().prepareRefresh("test").get();
+
+        for (int i = 0; i < 2; i++) {
+            SearchResponse searchResponse = client().prepareSearch()
+                    .setQuery(boolQuery().must(matchAllQuery()).filter(boolQuery()
+                            .must(hasChildQuery("child", matchQuery("c_field", "red"), ScoreMode.None))
+                            .must(matchAllQuery())))
+                    .get();
+            assertThat(searchResponse.getHits().totalHits(), equalTo(2L));
+        }
+
+
+       createIndexRequest("test", "child", "c3", "p2", "c_field", "blue").get();
+        client().admin().indices().prepareRefresh("test").get();
+
+        SearchResponse searchResponse = client().prepareSearch()
+                .setQuery(boolQuery().must(matchAllQuery()).filter(boolQuery()
+                        .must(hasChildQuery("child", matchQuery("c_field", "red"), ScoreMode.None))
+                        .must(matchAllQuery())))
+                .get();
+
+        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
+    }
+
+    public void testParentChildQueriesViaScrollApi() throws Exception {
+        if (legacy()) {
+            assertAcked(prepareCreate("test")
+                .addMapping("parent")
+                .addMapping("child", "_parent", "type=parent"));
+        } else {
+            assertAcked(prepareCreate("test")
+                .addMapping("doc", buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "parent", "child")));
+        }
+        ensureGreen();
+        for (int i = 0; i < 10; i++) {
+            createIndexRequest("test", "parent", "p" + i, null).get();
+            createIndexRequest("test", "child", "c" + i, "p" + i).get();
+        }
+
+        refresh();
+
+        QueryBuilder[] queries = new QueryBuilder[]{
+                hasChildQuery("child", matchAllQuery(), ScoreMode.None),
+                boolQuery().must(matchAllQuery()).filter(hasChildQuery("child", matchAllQuery(), ScoreMode.None)),
+                hasParentQuery("parent", matchAllQuery(), false),
+                boolQuery().must(matchAllQuery()).filter(hasParentQuery("parent", matchAllQuery(), false))
+        };
+
+        for (QueryBuilder query : queries) {
+            SearchResponse scrollResponse = client().prepareSearch("test")
+                    .setScroll(TimeValue.timeValueSeconds(30))
+                    .setSize(1)
+                    .addStoredField("_id")
+                    .setQuery(query)
+                    .execute()
+                    .actionGet();
+
+            assertNoFailures(scrollResponse);
+            assertThat(scrollResponse.getHits().totalHits(), equalTo(10L));
+            int scannedDocs = 0;
+            do {
+                assertThat(scrollResponse.getHits().totalHits(), equalTo(10L));
+                scannedDocs += scrollResponse.getHits().getHits().length;
+                scrollResponse = client()
+                        .prepareSearchScroll(scrollResponse.getScrollId())
+                        .setScroll(TimeValue.timeValueSeconds(30)).get();
+            } while (scrollResponse.getHits().getHits().length > 0);
+            clearScroll(scrollResponse.getScrollId());
+            assertThat(scannedDocs, equalTo(10));
+        }
+    }
+
+    public void testTypeIsAppliedInHasParentInnerQuery() throws Exception {
+        if (legacy()) {
+            assertAcked(prepareCreate("test")
+                .addMapping("parent")
+                .addMapping("child", "_parent", "type=parent"));
+        } else {
+            assertAcked(prepareCreate("test")
+                .addMapping("doc", buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "parent", "child")));
+        }
+        ensureGreen();
+
+        List<IndexRequestBuilder> indexRequests = new ArrayList<>();
+        indexRequests.add(createIndexRequest("test", "parent", "p1", null, "field1", "a"));
+        indexRequests.add(createIndexRequest("test", "child", "c1", "p1"));
+        indexRequests.add(createIndexRequest("test", "child", "c2", "p1"));
+        indexRandom(true, indexRequests);
+
+        SearchResponse searchResponse = client().prepareSearch("test")
+                .setQuery(constantScoreQuery(hasParentQuery("parent", boolQuery().mustNot(termQuery("field1", "a")), false)))
+                .get();
+        assertHitCount(searchResponse, 0L);
+
+        searchResponse = client().prepareSearch("test")
+                .setQuery(hasParentQuery("parent", constantScoreQuery(boolQuery().mustNot(termQuery("field1", "a"))), false))
+                .get();
+        assertHitCount(searchResponse, 0L);
+
+        searchResponse = client().prepareSearch("test")
+                .setQuery(constantScoreQuery(hasParentQuery("parent", termQuery("field1", "a"), false)))
+                .get();
+        assertHitCount(searchResponse, 2L);
+
+        searchResponse = client().prepareSearch("test")
+                .setQuery(hasParentQuery("parent", constantScoreQuery(termQuery("field1", "a")), false))
+                .get();
+        assertHitCount(searchResponse, 2L);
+    }
+
+    private List<IndexRequestBuilder> createMinMaxDocBuilders() {
+        List<IndexRequestBuilder> indexBuilders = new ArrayList<>();
+        // Parent 1 and its children
+        indexBuilders.add(createIndexRequest("test", "parent", "1", null, "id",1));
+        indexBuilders.add(createIndexRequest("test", "child", "10", "1", "foo", "one"));
+
+        // Parent 2 and its children
+        indexBuilders.add(createIndexRequest("test", "parent", "2", null, "id",2));
+        indexBuilders.add(createIndexRequest("test", "child", "11", "2", "foo", "one"));
+        indexBuilders.add(createIndexRequest("test", "child", "12", "2", "foo", "one two"));
+
+        // Parent 3 and its children
+        indexBuilders.add(createIndexRequest("test", "parent", "3", null, "id",3));
+        indexBuilders.add(createIndexRequest("test", "child", "13", "3", "foo", "one"));
+        indexBuilders.add(createIndexRequest("test", "child", "14", "3", "foo", "one two"));
+        indexBuilders.add(createIndexRequest("test", "child", "15", "3", "foo", "one two three"));
+
+        // Parent 4 and its children
+        indexBuilders.add(createIndexRequest("test", "parent", "4", null, "id", 4));
+        indexBuilders.add(createIndexRequest("test", "child", "16", "4", "foo", "one"));
+        indexBuilders.add(createIndexRequest("test", "child", "17", "4", "foo", "one two"));
+        indexBuilders.add(createIndexRequest("test", "child", "18", "4", "foo", "one two three"));
+        indexBuilders.add(createIndexRequest("test", "child", "19", "4", "foo", "one two three four"));
+
+        return indexBuilders;
+    }
+
+    private SearchResponse minMaxQuery(ScoreMode scoreMode, int minChildren, Integer maxChildren) throws SearchPhaseExecutionException {
+        HasChildQueryBuilder hasChildQuery = hasChildQuery(
+                "child",
+                QueryBuilders.functionScoreQuery(constantScoreQuery(QueryBuilders.termQuery("foo", "two")),
+                        new FunctionScoreQueryBuilder.FilterFunctionBuilder[]{
+                                new FunctionScoreQueryBuilder.FilterFunctionBuilder(weightFactorFunction(1)),
+                                new FunctionScoreQueryBuilder.FilterFunctionBuilder(QueryBuilders.termQuery("foo", "three"),
+                                    weightFactorFunction(1)),
+                                new FunctionScoreQueryBuilder.FilterFunctionBuilder(QueryBuilders.termQuery("foo", "four"),
+                                    weightFactorFunction(1))
+                        }).boostMode(CombineFunction.REPLACE).scoreMode(FiltersFunctionScoreQuery.ScoreMode.SUM), scoreMode)
+                .minMaxChildren(minChildren, maxChildren != null ? maxChildren : HasChildQueryBuilder.DEFAULT_MAX_CHILDREN);
+
+        return client()
+                .prepareSearch("test")
+                .setQuery(hasChildQuery)
+                .addSort("_score", SortOrder.DESC).addSort("id", SortOrder.ASC).get();
+    }
+
+    public void testMinMaxChildren() throws Exception {
+        if (legacy()) {
+            assertAcked(prepareCreate("test")
+                .addMapping("parent", "id", "type=long")
+                .addMapping("child", "_parent", "type=parent"));
+        } else {
+            assertAcked(prepareCreate("test")
+                .addMapping("doc", buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "parent", "child")));
+        }
+        ensureGreen();
+
+        indexRandom(true, createMinMaxDocBuilders().toArray(new IndexRequestBuilder[0]));
+        SearchResponse response;
+
+        // Score mode = NONE
+        response = minMaxQuery(ScoreMode.None, 0, null);
+
+        assertThat(response.getHits().totalHits(), equalTo(3L));
+        assertThat(response.getHits().hits()[0].id(), equalTo("2"));
+        assertThat(response.getHits().hits()[0].score(), equalTo(1f));
+        assertThat(response.getHits().hits()[1].id(), equalTo("3"));
+        assertThat(response.getHits().hits()[1].score(), equalTo(1f));
+        assertThat(response.getHits().hits()[2].id(), equalTo("4"));
+        assertThat(response.getHits().hits()[2].score(), equalTo(1f));
+
+        response = minMaxQuery(ScoreMode.None, 1, null);
+
+        assertThat(response.getHits().totalHits(), equalTo(3L));
+        assertThat(response.getHits().hits()[0].id(), equalTo("2"));
+        assertThat(response.getHits().hits()[0].score(), equalTo(1f));
+        assertThat(response.getHits().hits()[1].id(), equalTo("3"));
+        assertThat(response.getHits().hits()[1].score(), equalTo(1f));
+        assertThat(response.getHits().hits()[2].id(), equalTo("4"));
+        assertThat(response.getHits().hits()[2].score(), equalTo(1f));
+
+        response = minMaxQuery(ScoreMode.None, 2, null);
+
+        assertThat(response.getHits().totalHits(), equalTo(2L));
+        assertThat(response.getHits().hits()[0].id(), equalTo("3"));
+        assertThat(response.getHits().hits()[0].score(), equalTo(1f));
+        assertThat(response.getHits().hits()[1].id(), equalTo("4"));
+        assertThat(response.getHits().hits()[1].score(), equalTo(1f));
+
+        response = minMaxQuery(ScoreMode.None, 3, null);
+
+        assertThat(response.getHits().totalHits(), equalTo(1L));
+        assertThat(response.getHits().hits()[0].id(), equalTo("4"));
+        assertThat(response.getHits().hits()[0].score(), equalTo(1f));
+
+        response = minMaxQuery(ScoreMode.None, 4, null);
+
+        assertThat(response.getHits().totalHits(), equalTo(0L));
+
+        response = minMaxQuery(ScoreMode.None, 0, 4);
+
+        assertThat(response.getHits().totalHits(), equalTo(3L));
+        assertThat(response.getHits().hits()[0].id(), equalTo("2"));
+        assertThat(response.getHits().hits()[0].score(), equalTo(1f));
+        assertThat(response.getHits().hits()[1].id(), equalTo("3"));
+        assertThat(response.getHits().hits()[1].score(), equalTo(1f));
+        assertThat(response.getHits().hits()[2].id(), equalTo("4"));
+        assertThat(response.getHits().hits()[2].score(), equalTo(1f));
+
+        response = minMaxQuery(ScoreMode.None, 0, 3);
+
+        assertThat(response.getHits().totalHits(), equalTo(3L));
+        assertThat(response.getHits().hits()[0].id(), equalTo("2"));
+        assertThat(response.getHits().hits()[0].score(), equalTo(1f));
+        assertThat(response.getHits().hits()[1].id(), equalTo("3"));
+        assertThat(response.getHits().hits()[1].score(), equalTo(1f));
+        assertThat(response.getHits().hits()[2].id(), equalTo("4"));
+        assertThat(response.getHits().hits()[2].score(), equalTo(1f));
+
+        response = minMaxQuery(ScoreMode.None, 0, 2);
+
+        assertThat(response.getHits().totalHits(), equalTo(2L));
+        assertThat(response.getHits().hits()[0].id(), equalTo("2"));
+        assertThat(response.getHits().hits()[0].score(), equalTo(1f));
+        assertThat(response.getHits().hits()[1].id(), equalTo("3"));
+        assertThat(response.getHits().hits()[1].score(), equalTo(1f));
+
+        response = minMaxQuery(ScoreMode.None, 2, 2);
+
+        assertThat(response.getHits().totalHits(), equalTo(1L));
+        assertThat(response.getHits().hits()[0].id(), equalTo("3"));
+        assertThat(response.getHits().hits()[0].score(), equalTo(1f));
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> minMaxQuery(ScoreMode.None, 3, 2));
+        assertThat(e.getMessage(), equalTo("[has_child] 'max_children' is less than 'min_children'"));
+
+        // Score mode = SUM
+        response = minMaxQuery(ScoreMode.Total, 0, null);
+
+        assertThat(response.getHits().totalHits(), equalTo(3L));
+        assertThat(response.getHits().hits()[0].id(), equalTo("4"));
+        assertThat(response.getHits().hits()[0].score(), equalTo(6f));
+        assertThat(response.getHits().hits()[1].id(), equalTo("3"));
+        assertThat(response.getHits().hits()[1].score(), equalTo(3f));
+        assertThat(response.getHits().hits()[2].id(), equalTo("2"));
+        assertThat(response.getHits().hits()[2].score(), equalTo(1f));
+
+        response = minMaxQuery(ScoreMode.Total, 1, null);
+
+        assertThat(response.getHits().totalHits(), equalTo(3L));
+        assertThat(response.getHits().hits()[0].id(), equalTo("4"));
+        assertThat(response.getHits().hits()[0].score(), equalTo(6f));
+        assertThat(response.getHits().hits()[1].id(), equalTo("3"));
+        assertThat(response.getHits().hits()[1].score(), equalTo(3f));
+        assertThat(response.getHits().hits()[2].id(), equalTo("2"));
+        assertThat(response.getHits().hits()[2].score(), equalTo(1f));
+
+        response = minMaxQuery(ScoreMode.Total, 2, null);
+
+        assertThat(response.getHits().totalHits(), equalTo(2L));
+        assertThat(response.getHits().hits()[0].id(), equalTo("4"));
+        assertThat(response.getHits().hits()[0].score(), equalTo(6f));
+        assertThat(response.getHits().hits()[1].id(), equalTo("3"));
+        assertThat(response.getHits().hits()[1].score(), equalTo(3f));
+
+        response = minMaxQuery(ScoreMode.Total, 3, null);
+
+        assertThat(response.getHits().totalHits(), equalTo(1L));
+        assertThat(response.getHits().hits()[0].id(), equalTo("4"));
+        assertThat(response.getHits().hits()[0].score(), equalTo(6f));
+
+        response = minMaxQuery(ScoreMode.Total, 4, null);
+
+        assertThat(response.getHits().totalHits(), equalTo(0L));
+
+        response = minMaxQuery(ScoreMode.Total, 0, 4);
+
+        assertThat(response.getHits().totalHits(), equalTo(3L));
+        assertThat(response.getHits().hits()[0].id(), equalTo("4"));
+        assertThat(response.getHits().hits()[0].score(), equalTo(6f));
+        assertThat(response.getHits().hits()[1].id(), equalTo("3"));
+        assertThat(response.getHits().hits()[1].score(), equalTo(3f));
+        assertThat(response.getHits().hits()[2].id(), equalTo("2"));
+        assertThat(response.getHits().hits()[2].score(), equalTo(1f));
+
+        response = minMaxQuery(ScoreMode.Total, 0, 3);
+
+        assertThat(response.getHits().totalHits(), equalTo(3L));
+        assertThat(response.getHits().hits()[0].id(), equalTo("4"));
+        assertThat(response.getHits().hits()[0].score(), equalTo(6f));
+        assertThat(response.getHits().hits()[1].id(), equalTo("3"));
+        assertThat(response.getHits().hits()[1].score(), equalTo(3f));
+        assertThat(response.getHits().hits()[2].id(), equalTo("2"));
+        assertThat(response.getHits().hits()[2].score(), equalTo(1f));
+
+        response = minMaxQuery(ScoreMode.Total, 0, 2);
+
+        assertThat(response.getHits().totalHits(), equalTo(2L));
+        assertThat(response.getHits().hits()[0].id(), equalTo("3"));
+        assertThat(response.getHits().hits()[0].score(), equalTo(3f));
+        assertThat(response.getHits().hits()[1].id(), equalTo("2"));
+        assertThat(response.getHits().hits()[1].score(), equalTo(1f));
+
+        response = minMaxQuery(ScoreMode.Total, 2, 2);
+
+        assertThat(response.getHits().totalHits(), equalTo(1L));
+        assertThat(response.getHits().hits()[0].id(), equalTo("3"));
+        assertThat(response.getHits().hits()[0].score(), equalTo(3f));
+
+        e = expectThrows(IllegalArgumentException.class, () -> minMaxQuery(ScoreMode.Total, 3, 2));
+        assertThat(e.getMessage(), equalTo("[has_child] 'max_children' is less than 'min_children'"));
+
+        // Score mode = MAX
+        response = minMaxQuery(ScoreMode.Max, 0, null);
+
+        assertThat(response.getHits().totalHits(), equalTo(3L));
+        assertThat(response.getHits().hits()[0].id(), equalTo("4"));
+        assertThat(response.getHits().hits()[0].score(), equalTo(3f));
+        assertThat(response.getHits().hits()[1].id(), equalTo("3"));
+        assertThat(response.getHits().hits()[1].score(), equalTo(2f));
+        assertThat(response.getHits().hits()[2].id(), equalTo("2"));
+        assertThat(response.getHits().hits()[2].score(), equalTo(1f));
+
+        response = minMaxQuery(ScoreMode.Max, 1, null);
+
+        assertThat(response.getHits().totalHits(), equalTo(3L));
+        assertThat(response.getHits().hits()[0].id(), equalTo("4"));
+        assertThat(response.getHits().hits()[0].score(), equalTo(3f));
+        assertThat(response.getHits().hits()[1].id(), equalTo("3"));
+        assertThat(response.getHits().hits()[1].score(), equalTo(2f));
+        assertThat(response.getHits().hits()[2].id(), equalTo("2"));
+        assertThat(response.getHits().hits()[2].score(), equalTo(1f));
+
+        response = minMaxQuery(ScoreMode.Max, 2, null);
+
+        assertThat(response.getHits().totalHits(), equalTo(2L));
+        assertThat(response.getHits().hits()[0].id(), equalTo("4"));
+        assertThat(response.getHits().hits()[0].score(), equalTo(3f));
+        assertThat(response.getHits().hits()[1].id(), equalTo("3"));
+        assertThat(response.getHits().hits()[1].score(), equalTo(2f));
+
+        response = minMaxQuery(ScoreMode.Max, 3, null);
+
+        assertThat(response.getHits().totalHits(), equalTo(1L));
+        assertThat(response.getHits().hits()[0].id(), equalTo("4"));
+        assertThat(response.getHits().hits()[0].score(), equalTo(3f));
+
+        response = minMaxQuery(ScoreMode.Max, 4, null);
+
+        assertThat(response.getHits().totalHits(), equalTo(0L));
+
+        response = minMaxQuery(ScoreMode.Max, 0, 4);
+
+        assertThat(response.getHits().totalHits(), equalTo(3L));
+        assertThat(response.getHits().hits()[0].id(), equalTo("4"));
+        assertThat(response.getHits().hits()[0].score(), equalTo(3f));
+        assertThat(response.getHits().hits()[1].id(), equalTo("3"));
+        assertThat(response.getHits().hits()[1].score(), equalTo(2f));
+        assertThat(response.getHits().hits()[2].id(), equalTo("2"));
+        assertThat(response.getHits().hits()[2].score(), equalTo(1f));
+
+        response = minMaxQuery(ScoreMode.Max, 0, 3);
+
+        assertThat(response.getHits().totalHits(), equalTo(3L));
+        assertThat(response.getHits().hits()[0].id(), equalTo("4"));
+        assertThat(response.getHits().hits()[0].score(), equalTo(3f));
+        assertThat(response.getHits().hits()[1].id(), equalTo("3"));
+        assertThat(response.getHits().hits()[1].score(), equalTo(2f));
+        assertThat(response.getHits().hits()[2].id(), equalTo("2"));
+        assertThat(response.getHits().hits()[2].score(), equalTo(1f));
+
+        response = minMaxQuery(ScoreMode.Max, 0, 2);
+
+        assertThat(response.getHits().totalHits(), equalTo(2L));
+        assertThat(response.getHits().hits()[0].id(), equalTo("3"));
+        assertThat(response.getHits().hits()[0].score(), equalTo(2f));
+        assertThat(response.getHits().hits()[1].id(), equalTo("2"));
+        assertThat(response.getHits().hits()[1].score(), equalTo(1f));
+
+        response = minMaxQuery(ScoreMode.Max, 2, 2);
+
+        assertThat(response.getHits().totalHits(), equalTo(1L));
+        assertThat(response.getHits().hits()[0].id(), equalTo("3"));
+        assertThat(response.getHits().hits()[0].score(), equalTo(2f));
+
+        e = expectThrows(IllegalArgumentException.class, () -> minMaxQuery(ScoreMode.Max, 3, 2));
+        assertThat(e.getMessage(), equalTo("[has_child] 'max_children' is less than 'min_children'"));
+
+        // Score mode = AVG
+        response = minMaxQuery(ScoreMode.Avg, 0, null);
+
+        assertThat(response.getHits().totalHits(), equalTo(3L));
+        assertThat(response.getHits().hits()[0].id(), equalTo("4"));
+        assertThat(response.getHits().hits()[0].score(), equalTo(2f));
+        assertThat(response.getHits().hits()[1].id(), equalTo("3"));
+        assertThat(response.getHits().hits()[1].score(), equalTo(1.5f));
+        assertThat(response.getHits().hits()[2].id(), equalTo("2"));
+        assertThat(response.getHits().hits()[2].score(), equalTo(1f));
+
+        response = minMaxQuery(ScoreMode.Avg, 1, null);
+
+        assertThat(response.getHits().totalHits(), equalTo(3L));
+        assertThat(response.getHits().hits()[0].id(), equalTo("4"));
+        assertThat(response.getHits().hits()[0].score(), equalTo(2f));
+        assertThat(response.getHits().hits()[1].id(), equalTo("3"));
+        assertThat(response.getHits().hits()[1].score(), equalTo(1.5f));
+        assertThat(response.getHits().hits()[2].id(), equalTo("2"));
+        assertThat(response.getHits().hits()[2].score(), equalTo(1f));
+
+        response = minMaxQuery(ScoreMode.Avg, 2, null);
+
+        assertThat(response.getHits().totalHits(), equalTo(2L));
+        assertThat(response.getHits().hits()[0].id(), equalTo("4"));
+        assertThat(response.getHits().hits()[0].score(), equalTo(2f));
+        assertThat(response.getHits().hits()[1].id(), equalTo("3"));
+        assertThat(response.getHits().hits()[1].score(), equalTo(1.5f));
+
+        response = minMaxQuery(ScoreMode.Avg, 3, null);
+
+        assertThat(response.getHits().totalHits(), equalTo(1L));
+        assertThat(response.getHits().hits()[0].id(), equalTo("4"));
+        assertThat(response.getHits().hits()[0].score(), equalTo(2f));
+
+        response = minMaxQuery(ScoreMode.Avg, 4, null);
+
+        assertThat(response.getHits().totalHits(), equalTo(0L));
+
+        response = minMaxQuery(ScoreMode.Avg, 0, 4);
+
+        assertThat(response.getHits().totalHits(), equalTo(3L));
+        assertThat(response.getHits().hits()[0].id(), equalTo("4"));
+        assertThat(response.getHits().hits()[0].score(), equalTo(2f));
+        assertThat(response.getHits().hits()[1].id(), equalTo("3"));
+        assertThat(response.getHits().hits()[1].score(), equalTo(1.5f));
+        assertThat(response.getHits().hits()[2].id(), equalTo("2"));
+        assertThat(response.getHits().hits()[2].score(), equalTo(1f));
+
+        response = minMaxQuery(ScoreMode.Avg, 0, 3);
+
+        assertThat(response.getHits().totalHits(), equalTo(3L));
+        assertThat(response.getHits().hits()[0].id(), equalTo("4"));
+        assertThat(response.getHits().hits()[0].score(), equalTo(2f));
+        assertThat(response.getHits().hits()[1].id(), equalTo("3"));
+        assertThat(response.getHits().hits()[1].score(), equalTo(1.5f));
+        assertThat(response.getHits().hits()[2].id(), equalTo("2"));
+        assertThat(response.getHits().hits()[2].score(), equalTo(1f));
+
+        response = minMaxQuery(ScoreMode.Avg, 0, 2);
+
+        assertThat(response.getHits().totalHits(), equalTo(2L));
+        assertThat(response.getHits().hits()[0].id(), equalTo("3"));
+        assertThat(response.getHits().hits()[0].score(), equalTo(1.5f));
+        assertThat(response.getHits().hits()[1].id(), equalTo("2"));
+        assertThat(response.getHits().hits()[1].score(), equalTo(1f));
+
+        response = minMaxQuery(ScoreMode.Avg, 2, 2);
+
+        assertThat(response.getHits().totalHits(), equalTo(1L));
+        assertThat(response.getHits().hits()[0].id(), equalTo("3"));
+        assertThat(response.getHits().hits()[0].score(), equalTo(1.5f));
+
+        e = expectThrows(IllegalArgumentException.class, () -> minMaxQuery(ScoreMode.Avg, 3, 2));
+        assertThat(e.getMessage(), equalTo("[has_child] 'max_children' is less than 'min_children'"));
+    }
+
+    public void testHasParentInnerQueryType() {
+        if (legacy()) {
+            assertAcked(prepareCreate("test")
+                .addMapping("parent-type").addMapping("child-type", "_parent", "type=parent-type"));
+        } else {
+            assertAcked(prepareCreate("test")
+                .addMapping("doc", buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "parent-type", "child-type")));
+        }
+        createIndexRequest("test", "child-type", "child-id", "parent-id").get();
+        createIndexRequest("test", "parent-type", "parent-id", null).get();
+        refresh();
+
+        //make sure that when we explicitly set a type, the inner query is executed in the context of the child type instead
+        SearchResponse searchResponse = client().prepareSearch("test").setQuery(
+            hasChildQuery("child-type", new IdsQueryBuilder().addIds("child-id"), ScoreMode.None)).get();
+        assertSearchHits(searchResponse, "parent-id");
+        //make sure that when we explicitly set a type, the inner query is executed in the context of the parent type instead
+        searchResponse = client().prepareSearch("test").setQuery(
+                hasParentQuery("parent-type", new IdsQueryBuilder().addIds("parent-id"), false)).get();
+        assertSearchHits(searchResponse, "child-id");
+    }
+
+    public void testHighlightersIgnoreParentChild() throws IOException {
+        if (legacy()) {
+            assertAcked(prepareCreate("test")
+                .addMapping("parent-type", "searchText", "type=text,term_vector=with_positions_offsets,index_options=offsets")
+                .addMapping("child-type", "_parent", "type=parent-type", "searchText",
+                    "type=text,term_vector=with_positions_offsets,index_options=offsets"));
+        } else {
+            assertAcked(prepareCreate("test")
+                .addMapping("doc", jsonBuilder().startObject().startObject("properties")
+                    .startObject("join_field")
+                        .field("type", "join")
+                        .startObject("relations")
+                            .field("parent-type", "child-type")
+                        .endObject()
+                    .endObject()
+                    .startObject("searchText")
+                        .field("type", "text")
+                        .field("term_vector", "with_positions_offsets")
+                        .field("index_options", "offsets")
+                    .endObject()
+                    .endObject().endObject()
+                ));
+        }
+        createIndexRequest("test", "parent-type", "parent-id", null, "searchText", "quick brown fox").get();
+        createIndexRequest("test", "child-type", "child-id", "parent-id", "searchText", "quick brown fox").get();
+        refresh();
+
+        String[] highlightTypes = new String[] {"plain", "fvh", "unified"};
+        for (String highlightType : highlightTypes) {
+            logger.info("Testing with highlight type [{}]", highlightType);
+            SearchResponse searchResponse = client().prepareSearch("test")
+                    .setQuery(new BoolQueryBuilder()
+                            .must(new MatchQueryBuilder("searchText", "fox"))
+                            .must(new HasChildQueryBuilder("child-type", new MatchAllQueryBuilder(), ScoreMode.None))
+                    )
+                    .highlighter(new HighlightBuilder().field(new HighlightBuilder.Field("searchText").highlighterType(highlightType)))
+                    .get();
+            assertHitCount(searchResponse, 1);
+            assertThat(searchResponse.getHits().getAt(0).id(), equalTo("parent-id"));
+            HighlightField highlightField = searchResponse.getHits().getAt(0).getHighlightFields().get("searchText");
+            assertThat(highlightField.getFragments()[0].string(), equalTo("quick brown <em>fox</em>"));
+
+            searchResponse = client().prepareSearch("test")
+                    .setQuery(new BoolQueryBuilder()
+                            .must(new MatchQueryBuilder("searchText", "fox"))
+                            .must(new HasParentQueryBuilder("parent-type", new MatchAllQueryBuilder(), false))
+                    )
+                    .highlighter(new HighlightBuilder().field(new HighlightBuilder.Field("searchText").highlighterType(highlightType)))
+                    .get();
+            assertHitCount(searchResponse, 1);
+            assertThat(searchResponse.getHits().getAt(0).id(), equalTo("child-id"));
+            highlightField = searchResponse.getHits().getAt(0).getHighlightFields().get("searchText");
+            assertThat(highlightField.getFragments()[0].string(), equalTo("quick brown <em>fox</em>"));
+        }
+    }
+
+    public void testAliasesFilterWithHasChildQuery() throws Exception {
+        if (legacy()) {
+            assertAcked(prepareCreate("my-index")
+                .addMapping("parent")
+                .addMapping("child", "_parent", "type=parent")
+            );
+        } else {
+            assertAcked(prepareCreate("my-index")
+                .addMapping("doc", buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "parent", "child")));
+        }
+        createIndexRequest("my-index", "parent", "1", null).get();
+        createIndexRequest("my-index", "child", "2", "1").get();
+        refresh();
+
+        assertAcked(admin().indices().prepareAliases().addAlias("my-index", "filter1",
+            hasChildQuery("child", matchAllQuery(), ScoreMode.None)));
+        assertAcked(admin().indices().prepareAliases().addAlias("my-index", "filter2",
+            hasParentQuery("parent", matchAllQuery(), false)));
+
+        SearchResponse response = client().prepareSearch("filter1").get();
+        assertHitCount(response, 1);
+        assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
+        response = client().prepareSearch("filter2").get();
+        assertHitCount(response, 1);
+        assertThat(response.getHits().getAt(0).getId(), equalTo("2"));
+    }
+
+}
diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasChildQueryBuilderTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasChildQueryBuilderTests.java
new file mode 100644
index 0000000000000..a66c5e18aa94a
--- /dev/null
+++ b/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasChildQueryBuilderTests.java
@@ -0,0 +1,387 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.join.query;
+
+import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.ConstantScoreQuery;
+import org.apache.lucene.search.MatchNoDocsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermInSetQuery;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.join.ScoreMode;
+import org.apache.lucene.search.similarities.PerFieldSimilarityWrapper;
+import org.apache.lucene.search.similarities.Similarity;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.index.mapper.IdFieldMapper;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.query.IdsQueryBuilder;
+import org.elasticsearch.index.query.InnerHitBuilder;
+import org.elasticsearch.index.query.InnerHitContextBuilder;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.query.QueryShardException;
+import org.elasticsearch.index.query.TermQueryBuilder;
+import org.elasticsearch.index.query.WrapperQueryBuilder;
+import org.elasticsearch.index.similarity.SimilarityService;
+import org.elasticsearch.join.ParentJoinPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.search.fetch.subphase.InnerHitsContext;
+import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.search.sort.FieldSortBuilder;
+import org.elasticsearch.search.sort.SortOrder;
+import org.elasticsearch.test.AbstractQueryTestCase;
+import org.elasticsearch.test.VersionUtils;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.join.query.JoinQueryBuilders.hasChildQuery;
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.hamcrest.CoreMatchers.notNullValue;
+
+public class HasChildQueryBuilderTests extends AbstractQueryTestCase<HasChildQueryBuilder> {
+
+    private static final String TYPE = "doc";
+    private static final String PARENT_DOC = "parent";
+    private static final String CHILD_DOC = "child";
+
+    private static String similarity;
+
+    boolean requiresRewrite = false;
+
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return Collections.singletonList(ParentJoinPlugin.class);
+    }
+
+    @Override
+    protected Settings indexSettings() {
+        return Settings.builder()
+            .put(super.indexSettings())
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+            .put("index.mapping.single_type", true)
+            .build();
+    }
+
+    @Override
+    protected void initializeAdditionalMappings(MapperService mapperService) throws IOException {
+        similarity = randomFrom("classic", "BM25");
+        XContentBuilder mapping = jsonBuilder().startObject().startObject("doc").startObject("properties")
+            .startObject("join_field")
+                .field("type", "join")
+                .startObject("relations")
+                    .field(PARENT_DOC, CHILD_DOC)
+                .endObject()
+            .endObject()
+            .startObject(STRING_FIELD_NAME)
+                .field("type", "text")
+            .endObject()
+            .startObject(STRING_FIELD_NAME_2)
+                .field("type", "keyword")
+            .endObject()
+            .startObject(INT_FIELD_NAME)
+                .field("type", "integer")
+            .endObject()
+            .startObject(DOUBLE_FIELD_NAME)
+                .field("type", "double")
+            .endObject()
+            .startObject(BOOLEAN_FIELD_NAME)
+            .field("type", "boolean")
+            .endObject()
+            .startObject(DATE_FIELD_NAME)
+                .field("type", "date")
+            .endObject()
+            .startObject(OBJECT_FIELD_NAME)
+                .field("type", "object")
+            .endObject()
+            .startObject("custom_string")
+                .field("type", "text")
+                .field("similarity", similarity)
+            .endObject()
+            .endObject().endObject().endObject();
+
+        mapperService.merge(TYPE,
+            new CompressedXContent(mapping.string()), MapperService.MergeReason.MAPPING_UPDATE, false);
+    }
+
+    /**
+     * @return a {@link HasChildQueryBuilder} with random values all over the place
+     */
+    @Override
+    protected HasChildQueryBuilder doCreateTestQueryBuilder() {
+        int min = randomIntBetween(0, Integer.MAX_VALUE / 2);
+        int max = randomIntBetween(min, Integer.MAX_VALUE);
+
+        QueryBuilder innerQueryBuilder = new MatchAllQueryBuilder();
+        if (randomBoolean()) {
+            requiresRewrite = true;
+            innerQueryBuilder = new WrapperQueryBuilder(innerQueryBuilder.toString());
+        }
+
+        HasChildQueryBuilder hqb = new HasChildQueryBuilder(CHILD_DOC, innerQueryBuilder,
+                RandomPicks.randomFrom(random(), ScoreMode.values()));
+        hqb.minMaxChildren(min, max);
+        hqb.ignoreUnmapped(randomBoolean());
+        if (randomBoolean()) {
+            hqb.innerHit(new InnerHitBuilder()
+                    .setName(randomAlphaOfLengthBetween(1, 10))
+                    .setSize(randomIntBetween(0, 100))
+                    .addSort(new FieldSortBuilder(STRING_FIELD_NAME_2).order(SortOrder.ASC))
+                    .setIgnoreUnmapped(hqb.ignoreUnmapped()));
+        }
+        return hqb;
+    }
+
+    @Override
+    protected void doAssertLuceneQuery(HasChildQueryBuilder queryBuilder, Query query, SearchContext searchContext) throws IOException {
+        assertThat(query, instanceOf(HasChildQueryBuilder.LateParsingQuery.class));
+        HasChildQueryBuilder.LateParsingQuery lpq = (HasChildQueryBuilder.LateParsingQuery) query;
+        assertEquals(queryBuilder.minChildren(), lpq.getMinChildren());
+        assertEquals(queryBuilder.maxChildren(), lpq.getMaxChildren());
+        assertEquals(queryBuilder.scoreMode(), lpq.getScoreMode()); // WTF is this why do we have two?
+        if (queryBuilder.innerHit() != null) {
+            // have to rewrite again because the provided queryBuilder hasn't been rewritten (directly returned from
+            // doCreateTestQueryBuilder)
+            queryBuilder = (HasChildQueryBuilder) queryBuilder.rewrite(searchContext.getQueryShardContext());
+            Map<String, InnerHitContextBuilder> innerHitBuilders = new HashMap<>();
+            InnerHitContextBuilder.extractInnerHits(queryBuilder, innerHitBuilders);
+            for (InnerHitContextBuilder builder : innerHitBuilders.values()) {
+                builder.build(searchContext, searchContext.innerHits());
+            }
+            assertNotNull(searchContext.innerHits());
+            assertEquals(1, searchContext.innerHits().getInnerHits().size());
+            assertTrue(searchContext.innerHits().getInnerHits().containsKey(queryBuilder.innerHit().getName()));
+            InnerHitsContext.InnerHitSubContext innerHits =
+                    searchContext.innerHits().getInnerHits().get(queryBuilder.innerHit().getName());
+            assertEquals(innerHits.size(), queryBuilder.innerHit().getSize());
+            assertEquals(innerHits.sort().sort.getSort().length, 1);
+            assertEquals(innerHits.sort().sort.getSort()[0].getField(), STRING_FIELD_NAME_2);
+        }
+    }
+
+    /**
+     * Test (de)serialization on all previous released versions
+     */
+    public void testSerializationBWC() throws IOException {
+        for (Version version : VersionUtils.allReleasedVersions()) {
+            HasChildQueryBuilder testQuery = createTestQueryBuilder();
+            if (version.before(Version.V_5_2_0) && testQuery.innerHit() != null) {
+                // ignore unmapped for inner_hits has been added on 5.2
+                testQuery.innerHit().setIgnoreUnmapped(false);
+            }
+            assertSerialization(testQuery, version);
+        }
+    }
+
+    public void testIllegalValues() {
+        QueryBuilder query = new MatchAllQueryBuilder();
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> hasChildQuery(null, query, ScoreMode.None));
+        assertEquals("[has_child] requires 'type' field", e.getMessage());
+
+        e = expectThrows(IllegalArgumentException.class, () -> hasChildQuery("foo", null, ScoreMode.None));
+        assertEquals("[has_child] requires 'query' field", e.getMessage());
+
+        e = expectThrows(IllegalArgumentException.class, () -> hasChildQuery("foo", query, null));
+        assertEquals("[has_child] requires 'score_mode' field", e.getMessage());
+
+        int positiveValue = randomIntBetween(0, Integer.MAX_VALUE);
+        HasChildQueryBuilder foo = hasChildQuery("foo", query, ScoreMode.None); // all good
+        e = expectThrows(IllegalArgumentException.class, () -> foo.minMaxChildren(randomIntBetween(Integer.MIN_VALUE, -1), positiveValue));
+        assertEquals("[has_child] requires non-negative 'min_children' field", e.getMessage());
+
+        e = expectThrows(IllegalArgumentException.class, () -> foo.minMaxChildren(positiveValue, randomIntBetween(Integer.MIN_VALUE, -1)));
+        assertEquals("[has_child] requires non-negative 'max_children' field", e.getMessage());
+
+        e = expectThrows(IllegalArgumentException.class, () -> foo.minMaxChildren(positiveValue, positiveValue - 10));
+        assertEquals("[has_child] 'max_children' is less than 'min_children'", e.getMessage());
+    }
+
+    public void testFromJson() throws IOException {
+        String query =
+                "{\n" +
+                "  \"has_child\" : {\n" +
+                "    \"query\" : {\n" +
+                "      \"range\" : {\n" +
+                "        \"mapped_string\" : {\n" +
+                "          \"from\" : \"agJhRET\",\n" +
+                "          \"to\" : \"zvqIq\",\n" +
+                "          \"include_lower\" : true,\n" +
+                "          \"include_upper\" : true,\n" +
+                "          \"boost\" : 1.0\n" +
+                "        }\n" +
+                "      }\n" +
+                "    },\n" +
+                "    \"type\" : \"child\",\n" +
+                "    \"score_mode\" : \"avg\",\n" +
+                "    \"min_children\" : 883170873,\n" +
+                "    \"max_children\" : 1217235442,\n" +
+                "    \"ignore_unmapped\" : false,\n" +
+                "    \"boost\" : 2.0,\n" +
+                "    \"_name\" : \"WNzYMJKRwePuRBh\",\n" +
+                "    \"inner_hits\" : {\n" +
+                "      \"name\" : \"inner_hits_name\",\n" +
+                "      \"ignore_unmapped\" : false,\n" +
+                "      \"from\" : 0,\n" +
+                "      \"size\" : 100,\n" +
+                "      \"version\" : false,\n" +
+                "      \"explain\" : false,\n" +
+                "      \"track_scores\" : false,\n" +
+                "      \"sort\" : [ {\n" +
+                "        \"mapped_string\" : {\n" +
+                "          \"order\" : \"asc\"\n" +
+                "        }\n" +
+                "      } ]\n" +
+                "    }\n" +
+                "  }\n" +
+                "}";
+        HasChildQueryBuilder queryBuilder = (HasChildQueryBuilder) parseQuery(query);
+        checkGeneratedJson(query, queryBuilder);
+        assertEquals(query, queryBuilder.maxChildren(), 1217235442);
+        assertEquals(query, queryBuilder.minChildren(), 883170873);
+        assertEquals(query, queryBuilder.boost(), 2.0f, 0.0f);
+        assertEquals(query, queryBuilder.queryName(), "WNzYMJKRwePuRBh");
+        assertEquals(query, queryBuilder.childType(), "child");
+        assertEquals(query, queryBuilder.scoreMode(), ScoreMode.Avg);
+        assertNotNull(query, queryBuilder.innerHit());
+        InnerHitBuilder expected = new InnerHitBuilder("child")
+                .setName("inner_hits_name")
+                .setSize(100)
+                .addSort(new FieldSortBuilder("mapped_string").order(SortOrder.ASC));
+        assertEquals(query, queryBuilder.innerHit(), expected);
+    }
+
+    /**
+     * we resolve empty inner clauses by representing this whole query as empty optional upstream
+     */
+    public void testFromJsonEmptyQueryBody() throws IOException {
+        String query =  "{\n" +
+                "  \"has_child\" : {\n" +
+                "    \"query\" : { },\n" +
+                "    \"type\" : \"child\"" +
+                "   }" +
+                "}";
+        XContentParser parser = createParser(JsonXContent.jsonXContent, query);
+        QueryParseContext context = createParseContext(parser);
+        Optional<QueryBuilder> innerQueryBuilder = context.parseInnerQueryBuilder();
+        assertFalse(innerQueryBuilder.isPresent());
+        assertWarnings("query malformed, empty clause found at [3:17]");
+    }
+
+    public void testToQueryInnerQueryType() throws IOException {
+        String[] searchTypes = new String[]{TYPE};
+        QueryShardContext shardContext = createShardContext();
+        shardContext.setTypes(searchTypes);
+        HasChildQueryBuilder hasChildQueryBuilder = hasChildQuery(CHILD_DOC, new IdsQueryBuilder().addIds("id"), ScoreMode.None);
+        Query query = hasChildQueryBuilder.toQuery(shardContext);
+        //verify that the context types are still the same as the ones we previously set
+        assertThat(shardContext.getTypes(), equalTo(searchTypes));
+        assertLateParsingQuery(query, CHILD_DOC, "id");
+    }
+
+    static void assertLateParsingQuery(Query query, String type, String id) throws IOException {
+        assertThat(query, instanceOf(HasChildQueryBuilder.LateParsingQuery.class));
+        HasChildQueryBuilder.LateParsingQuery lateParsingQuery = (HasChildQueryBuilder.LateParsingQuery) query;
+        assertThat(lateParsingQuery.getInnerQuery(), instanceOf(BooleanQuery.class));
+        BooleanQuery booleanQuery = (BooleanQuery) lateParsingQuery.getInnerQuery();
+        assertThat(booleanQuery.clauses().size(), equalTo(2));
+        //check the inner ids query, we have to call rewrite to get to check the type it's executed against
+        assertThat(booleanQuery.clauses().get(0).getOccur(), equalTo(BooleanClause.Occur.MUST));
+        assertThat(booleanQuery.clauses().get(0).getQuery(), instanceOf(TermInSetQuery.class));
+        TermInSetQuery termsQuery = (TermInSetQuery) booleanQuery.clauses().get(0).getQuery();
+        Query rewrittenTermsQuery = termsQuery.rewrite(null);
+        assertThat(rewrittenTermsQuery, instanceOf(ConstantScoreQuery.class));
+        ConstantScoreQuery constantScoreQuery = (ConstantScoreQuery) rewrittenTermsQuery;
+        assertThat(constantScoreQuery.getQuery(), instanceOf(BooleanQuery.class));
+        BooleanQuery booleanTermsQuery = (BooleanQuery) constantScoreQuery.getQuery();
+        assertThat(booleanTermsQuery.clauses().toString(), booleanTermsQuery.clauses().size(), equalTo(1));
+        assertThat(booleanTermsQuery.clauses().get(0).getOccur(), equalTo(BooleanClause.Occur.SHOULD));
+        assertThat(booleanTermsQuery.clauses().get(0).getQuery(), instanceOf(TermQuery.class));
+        TermQuery termQuery = (TermQuery) booleanTermsQuery.clauses().get(0).getQuery();
+        assertThat(termQuery.getTerm().field(), equalTo(IdFieldMapper.NAME));
+        assertThat(termQuery.getTerm().bytes().utf8ToString(), equalTo(id));
+        //check the type filter
+        assertThat(booleanQuery.clauses().get(1).getOccur(), equalTo(BooleanClause.Occur.FILTER));
+        assertEquals(new TermQuery(new Term("join_field", type)), booleanQuery.clauses().get(1).getQuery());
+    }
+
+    @Override
+    public void testMustRewrite() throws IOException {
+        try {
+            super.testMustRewrite();
+        } catch (UnsupportedOperationException e) {
+            if (requiresRewrite == false) {
+                throw e;
+            }
+        }
+    }
+
+    public void testNonDefaultSimilarity() throws Exception {
+        QueryShardContext shardContext = createShardContext();
+        HasChildQueryBuilder hasChildQueryBuilder =
+            hasChildQuery(CHILD_DOC, new TermQueryBuilder("custom_string", "value"), ScoreMode.None);
+        HasChildQueryBuilder.LateParsingQuery query = (HasChildQueryBuilder.LateParsingQuery) hasChildQueryBuilder.toQuery(shardContext);
+        Similarity expected = SimilarityService.BUILT_IN.get(similarity).apply(similarity, Settings.EMPTY).get();
+        assertThat(((PerFieldSimilarityWrapper) query.getSimilarity()).get("custom_string"), instanceOf(expected.getClass()));
+    }
+
+    public void testIgnoreUnmapped() throws IOException {
+        final HasChildQueryBuilder queryBuilder = new HasChildQueryBuilder("unmapped", new MatchAllQueryBuilder(), ScoreMode.None);
+        queryBuilder.ignoreUnmapped(true);
+        Query query = queryBuilder.toQuery(createShardContext());
+        assertThat(query, notNullValue());
+        assertThat(query, instanceOf(MatchNoDocsQuery.class));
+
+        final HasChildQueryBuilder failingQueryBuilder = new HasChildQueryBuilder("unmapped", new MatchAllQueryBuilder(), ScoreMode.None);
+        failingQueryBuilder.ignoreUnmapped(false);
+        QueryShardException e = expectThrows(QueryShardException.class, () -> failingQueryBuilder.toQuery(createShardContext()));
+        assertThat(e.getMessage(), containsString("[" + HasChildQueryBuilder.NAME +
+            "] join field [join_field] doesn't hold [unmapped] as a child"));
+    }
+
+    public void testIgnoreUnmappedWithRewrite() throws IOException {
+        // WrapperQueryBuilder makes sure we always rewrite
+        final HasChildQueryBuilder queryBuilder
+            = new HasChildQueryBuilder("unmapped", new WrapperQueryBuilder(new MatchAllQueryBuilder().toString()), ScoreMode.None);
+        queryBuilder.ignoreUnmapped(true);
+        QueryShardContext queryShardContext = createShardContext();
+        Query query = queryBuilder.rewrite(queryShardContext).toQuery(queryShardContext);
+        assertThat(query, notNullValue());
+        assertThat(query, instanceOf(MatchNoDocsQuery.class));
+    }
+}
diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasParentQueryBuilderTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasParentQueryBuilderTests.java
new file mode 100644
index 0000000000000..386d8f97e959d
--- /dev/null
+++ b/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasParentQueryBuilderTests.java
@@ -0,0 +1,308 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.join.query;
+
+import org.apache.lucene.search.MatchNoDocsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.join.ScoreMode;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.query.IdsQueryBuilder;
+import org.elasticsearch.index.query.InnerHitBuilder;
+import org.elasticsearch.index.query.InnerHitContextBuilder;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.query.QueryShardException;
+import org.elasticsearch.index.query.TermQueryBuilder;
+import org.elasticsearch.index.query.WrapperQueryBuilder;
+import org.elasticsearch.join.ParentJoinPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.search.fetch.subphase.InnerHitsContext;
+import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.search.sort.FieldSortBuilder;
+import org.elasticsearch.search.sort.SortOrder;
+import org.elasticsearch.test.AbstractQueryTestCase;
+import org.elasticsearch.test.VersionUtils;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.join.query.JoinQueryBuilders.hasParentQuery;
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.hamcrest.CoreMatchers.notNullValue;
+
+public class HasParentQueryBuilderTests extends AbstractQueryTestCase<HasParentQueryBuilder> {
+    private static final String TYPE = "doc";
+    private static final String PARENT_DOC = "parent";
+    private static final String CHILD_DOC = "child";
+
+    boolean requiresRewrite = false;
+
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return Collections.singletonList(ParentJoinPlugin.class);
+    }
+
+    @Override
+    protected Settings indexSettings() {
+        return Settings.builder()
+            .put(super.indexSettings())
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+            .put("index.mapping.single_type", true)
+            .build();
+    }
+
+    @Override
+    protected void initializeAdditionalMappings(MapperService mapperService) throws IOException {
+        XContentBuilder mapping = jsonBuilder().startObject().startObject("doc").startObject("properties")
+            .startObject("join_field")
+                .field("type", "join")
+                .startObject("relations")
+                    .field(PARENT_DOC, CHILD_DOC)
+                .endObject()
+            .endObject()
+            .startObject(STRING_FIELD_NAME)
+                .field("type", "text")
+            .endObject()
+                .startObject(STRING_FIELD_NAME_2)
+            .field("type", "keyword")
+            .endObject()
+            .startObject(INT_FIELD_NAME)
+                .field("type", "integer")
+            .endObject()
+            .startObject(DOUBLE_FIELD_NAME)
+                .field("type", "double")
+            .endObject()
+            .startObject(BOOLEAN_FIELD_NAME)
+                .field("type", "boolean")
+            .endObject()
+            .startObject(DATE_FIELD_NAME)
+                .field("type", "date")
+            .endObject()
+            .startObject(OBJECT_FIELD_NAME)
+                .field("type", "object")
+            .endObject()
+            .endObject().endObject().endObject();
+
+        mapperService.merge(TYPE,
+            new CompressedXContent(mapping.string()), MapperService.MergeReason.MAPPING_UPDATE, false);
+    }
+
+    /**
+     * @return a {@link HasChildQueryBuilder} with random values all over the place
+     */
+    @Override
+    protected HasParentQueryBuilder doCreateTestQueryBuilder() {
+        QueryBuilder innerQueryBuilder = new MatchAllQueryBuilder();
+        if (randomBoolean()) {
+            requiresRewrite = true;
+            innerQueryBuilder = new WrapperQueryBuilder(innerQueryBuilder.toString());
+        }
+        HasParentQueryBuilder hqb = new HasParentQueryBuilder(PARENT_DOC, innerQueryBuilder, randomBoolean());
+        hqb.ignoreUnmapped(randomBoolean());
+        if (randomBoolean()) {
+            hqb.innerHit(new InnerHitBuilder()
+                    .setName(randomAlphaOfLengthBetween(1, 10))
+                    .setSize(randomIntBetween(0, 100))
+                    .addSort(new FieldSortBuilder(STRING_FIELD_NAME_2).order(SortOrder.ASC))
+                    .setIgnoreUnmapped(hqb.ignoreUnmapped()));
+        }
+        return hqb;
+    }
+
+    @Override
+    protected void doAssertLuceneQuery(HasParentQueryBuilder queryBuilder, Query query, SearchContext searchContext) throws IOException {
+        assertThat(query, instanceOf(HasChildQueryBuilder.LateParsingQuery.class));
+        HasChildQueryBuilder.LateParsingQuery lpq = (HasChildQueryBuilder.LateParsingQuery) query;
+        assertEquals(queryBuilder.score() ? ScoreMode.Max : ScoreMode.None, lpq.getScoreMode());
+
+        if (queryBuilder.innerHit() != null) {
+            // have to rewrite again because the provided queryBuilder hasn't been rewritten (directly returned from
+            // doCreateTestQueryBuilder)
+            queryBuilder = (HasParentQueryBuilder) queryBuilder.rewrite(searchContext.getQueryShardContext());
+
+            assertNotNull(searchContext);
+            Map<String, InnerHitContextBuilder> innerHitBuilders = new HashMap<>();
+            InnerHitContextBuilder.extractInnerHits(queryBuilder, innerHitBuilders);
+            for (InnerHitContextBuilder builder : innerHitBuilders.values()) {
+                builder.build(searchContext, searchContext.innerHits());
+            }
+            assertNotNull(searchContext.innerHits());
+            assertEquals(1, searchContext.innerHits().getInnerHits().size());
+            assertTrue(searchContext.innerHits().getInnerHits().containsKey(queryBuilder.innerHit().getName()));
+            InnerHitsContext.InnerHitSubContext innerHits = searchContext.innerHits()
+                    .getInnerHits().get(queryBuilder.innerHit().getName());
+            assertEquals(innerHits.size(), queryBuilder.innerHit().getSize());
+            assertEquals(innerHits.sort().sort.getSort().length, 1);
+            assertEquals(innerHits.sort().sort.getSort()[0].getField(), STRING_FIELD_NAME_2);
+        }
+    }
+
+    /**
+     * Test (de)serialization on all previous released versions
+     */
+    public void testSerializationBWC() throws IOException {
+        for (Version version : VersionUtils.allReleasedVersions()) {
+            HasParentQueryBuilder testQuery = createTestQueryBuilder();
+            if (version.before(Version.V_5_2_0) && testQuery.innerHit() != null) {
+                // ignore unmapped for inner_hits has been added on 5.2
+                testQuery.innerHit().setIgnoreUnmapped(false);
+            }
+            assertSerialization(testQuery, version);
+        }
+    }
+
+    public void testIllegalValues() throws IOException {
+        QueryBuilder query = new MatchAllQueryBuilder();
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> hasParentQuery(null, query, false));
+        assertThat(e.getMessage(), equalTo("[has_parent] requires 'type' field"));
+
+        e = expectThrows(IllegalArgumentException.class,
+                () -> hasParentQuery("foo", null, false));
+        assertThat(e.getMessage(), equalTo("[has_parent] requires 'query' field"));
+
+        QueryShardContext context = createShardContext();
+        HasParentQueryBuilder qb = hasParentQuery("just_a_type", new MatchAllQueryBuilder(), false);
+        QueryShardException qse = expectThrows(QueryShardException.class, () -> qb.doToQuery(context));
+        assertThat(qse.getMessage(), equalTo("[has_parent] join field [join_field] doesn't hold [just_a_type] as a parent"));
+    }
+
+    public void testDeprecatedXContent() throws IOException {
+        XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint();
+        builder.startObject();
+        builder.startObject("has_parent");
+        builder.field("query");
+        new TermQueryBuilder("a", "a").toXContent(builder, ToXContent.EMPTY_PARAMS);
+        builder.field("type", "foo"); // deprecated
+        builder.endObject();
+        builder.endObject();
+        HasParentQueryBuilder queryBuilder = (HasParentQueryBuilder) parseQuery(builder.string());
+        assertEquals("foo", queryBuilder.type());
+        assertWarnings("Deprecated field [type] used, expected [parent_type] instead");
+    }
+
+    public void testToQueryInnerQueryType() throws IOException {
+        String[] searchTypes = new String[]{TYPE};
+        QueryShardContext shardContext = createShardContext();
+        shardContext.setTypes(searchTypes);
+        HasParentQueryBuilder hasParentQueryBuilder = new HasParentQueryBuilder(PARENT_DOC, new IdsQueryBuilder().addIds("id"),
+                false);
+        Query query = hasParentQueryBuilder.toQuery(shardContext);
+        //verify that the context types are still the same as the ones we previously set
+        assertThat(shardContext.getTypes(), equalTo(searchTypes));
+        HasChildQueryBuilderTests.assertLateParsingQuery(query, PARENT_DOC, "id");
+    }
+
+    @Override
+    public void testMustRewrite() throws IOException {
+        try {
+            super.testMustRewrite();
+        } catch (UnsupportedOperationException e) {
+            if (requiresRewrite == false) {
+                throw e;
+            }
+        }
+    }
+
+    public void testFromJson() throws IOException {
+        String json =
+                "{\n" +
+                "  \"has_parent\" : {\n" +
+                "    \"query\" : {\n" +
+                "      \"term\" : {\n" +
+                "        \"tag\" : {\n" +
+                "          \"value\" : \"something\",\n" +
+                "          \"boost\" : 1.0\n" +
+                "        }\n" +
+                "      }\n" +
+                "    },\n" +
+                "    \"parent_type\" : \"blog\",\n" +
+                "    \"score\" : true,\n" +
+                "    \"ignore_unmapped\" : false,\n" +
+                "    \"boost\" : 1.0\n" +
+                "  }\n" +
+                "}";
+        HasParentQueryBuilder parsed = (HasParentQueryBuilder) parseQuery(json);
+        checkGeneratedJson(json, parsed);
+        assertEquals(json, "blog", parsed.type());
+        assertEquals(json, "something", ((TermQueryBuilder) parsed.query()).value());
+    }
+
+    /**
+     * we resolve empty inner clauses by representing this whole query as empty optional upstream
+     */
+    public void testFromJsonEmptyQueryBody() throws IOException {
+        String query =  "{\n" +
+                "  \"has_parent\" : {\n" +
+                "    \"query\" : { },\n" +
+                "    \"parent_type\" : \"blog\"" +
+                "   }" +
+                "}";
+        XContentParser parser = createParser(JsonXContent.jsonXContent, query);
+        QueryParseContext context = createParseContext(parser);
+        Optional<QueryBuilder> innerQueryBuilder = context.parseInnerQueryBuilder();
+        assertFalse(innerQueryBuilder.isPresent());
+        assertWarnings("query malformed, empty clause found at [3:17]");
+    }
+
+    public void testIgnoreUnmapped() throws IOException {
+        final HasParentQueryBuilder queryBuilder = new HasParentQueryBuilder("unmapped", new MatchAllQueryBuilder(), false);
+        queryBuilder.ignoreUnmapped(true);
+        Query query = queryBuilder.toQuery(createShardContext());
+        assertThat(query, notNullValue());
+        assertThat(query, instanceOf(MatchNoDocsQuery.class));
+
+        final HasParentQueryBuilder failingQueryBuilder = new HasParentQueryBuilder("unmapped", new MatchAllQueryBuilder(), false);
+        failingQueryBuilder.ignoreUnmapped(false);
+        QueryShardException e = expectThrows(QueryShardException.class, () -> failingQueryBuilder.toQuery(createShardContext()));
+        assertThat(e.getMessage(),
+                    containsString("[has_parent] join field [join_field] doesn't hold [unmapped] as a parent"));
+    }
+
+    public void testIgnoreUnmappedWithRewrite() throws IOException {
+        // WrapperQueryBuilder makes sure we always rewrite
+        final HasParentQueryBuilder queryBuilder =
+            new HasParentQueryBuilder("unmapped", new WrapperQueryBuilder(new MatchAllQueryBuilder().toString()), false);
+        queryBuilder.ignoreUnmapped(true);
+        QueryShardContext queryShardContext = createShardContext();
+        Query query = queryBuilder.rewrite(queryShardContext).toQuery(queryShardContext);
+        assertThat(query, notNullValue());
+        assertThat(query, instanceOf(MatchNoDocsQuery.class));
+    }
+}
diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/query/InnerHitsIT.java b/modules/parent-join/src/test/java/org/elasticsearch/join/query/InnerHitsIT.java
new file mode 100644
index 0000000000000..9787dcfdb8139
--- /dev/null
+++ b/modules/parent-join/src/test/java/org/elasticsearch/join/query/InnerHitsIT.java
@@ -0,0 +1,588 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.join.query;
+
+import org.apache.lucene.search.join.ScoreMode;
+import org.apache.lucene.util.ArrayUtil;
+import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.InnerHitBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.join.ParentJoinPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.script.MockScriptEngine;
+import org.elasticsearch.script.MockScriptPlugin;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptType;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
+import org.elasticsearch.search.sort.FieldSortBuilder;
+import org.elasticsearch.search.sort.SortBuilders;
+import org.elasticsearch.search.sort.SortOrder;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.function.Function;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.common.xcontent.support.XContentMapValues.extractValue;
+import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
+import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery;
+import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
+import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
+import static org.elasticsearch.index.query.QueryBuilders.nestedQuery;
+import static org.elasticsearch.index.query.QueryBuilders.termQuery;
+import static org.elasticsearch.join.query.JoinQueryBuilders.hasChildQuery;
+import static org.elasticsearch.join.query.JoinQueryBuilders.hasParentQuery;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHit;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasId;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+
+public class InnerHitsIT extends ParentChildTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Arrays.asList(ParentJoinPlugin.class, CustomScriptPlugin.class);
+    }
+
+    public static class CustomScriptPlugin extends MockScriptPlugin {
+        @Override
+        protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
+            return Collections.singletonMap("5", script -> "5");
+        }
+    }
+
+    public void testSimpleParentChild() throws Exception {
+        if (legacy()) {
+            assertAcked(prepareCreate("articles")
+                .addMapping("article", "title", "type=text")
+                .addMapping("comment", "_parent", "type=article", "message", "type=text,fielddata=true")
+            );
+        } else {
+            assertAcked(prepareCreate("articles")
+                .addMapping("doc", jsonBuilder().startObject().startObject("doc").startObject("properties")
+                    .startObject("join_field")
+                        .field("type", "join")
+                        .startObject("relations")
+                            .field("article", "comment")
+                        .endObject()
+                    .endObject()
+                    .startObject("title")
+                        .field("type", "text")
+                    .endObject()
+                    .startObject("message")
+                        .field("type", "text")
+                        .field("fielddata", true)
+                    .endObject()
+                    .endObject().endObject().endObject()
+                ));
+        }
+
+        List<IndexRequestBuilder> requests = new ArrayList<>();
+        requests.add(createIndexRequest("articles", "article", "p1", null, "title", "quick brown fox"));
+        requests.add(createIndexRequest("articles", "comment", "c1", "p1", "message", "fox eat quick"));
+        requests.add(createIndexRequest("articles", "comment", "c2", "p1", "message", "fox ate rabbit x y z"));
+        requests.add(createIndexRequest("articles", "comment", "c3", "p1", "message", "rabbit got away"));
+        requests.add(createIndexRequest("articles", "article", "p2", null, "title", "big gray elephant"));
+        requests.add(createIndexRequest("articles", "comment", "c4", "p2", "message", "elephant captured"));
+        requests.add(createIndexRequest("articles", "comment", "c5", "p2", "message", "mice squashed by elephant x"));
+        requests.add(createIndexRequest("articles", "comment", "c6", "p2", "message", "elephant scared by mice x y"));
+        indexRandom(true, requests);
+
+        SearchResponse response = client().prepareSearch("articles")
+            .setQuery(hasChildQuery("comment", matchQuery("message", "fox"), ScoreMode.None)
+                .innerHit(new InnerHitBuilder()))
+            .get();
+        assertNoFailures(response);
+        assertHitCount(response, 1);
+        assertSearchHit(response, 1, hasId("p1"));
+        assertThat(response.getHits().getAt(0).getShard(), notNullValue());
+
+        assertThat(response.getHits().getAt(0).getInnerHits().size(), equalTo(1));
+        SearchHits innerHits = response.getHits().getAt(0).getInnerHits().get("comment");
+        assertThat(innerHits.getTotalHits(), equalTo(2L));
+
+        assertThat(innerHits.getAt(0).getId(), equalTo("c1"));
+        assertThat(innerHits.getAt(0).getType(), equalTo(legacy() ? "comment" : "doc"));
+        assertThat(innerHits.getAt(1).getId(), equalTo("c2"));
+        assertThat(innerHits.getAt(1).getType(), equalTo(legacy() ? "comment" : "doc"));
+
+        response = client().prepareSearch("articles")
+            .setQuery(hasChildQuery("comment", matchQuery("message", "elephant"), ScoreMode.None)
+                .innerHit(new InnerHitBuilder()))
+            .get();
+        assertNoFailures(response);
+        assertHitCount(response, 1);
+        assertSearchHit(response, 1, hasId("p2"));
+
+        assertThat(response.getHits().getAt(0).getInnerHits().size(), equalTo(1));
+        innerHits = response.getHits().getAt(0).getInnerHits().get("comment");
+        assertThat(innerHits.getTotalHits(), equalTo(3L));
+
+        assertThat(innerHits.getAt(0).getId(), equalTo("c4"));
+        assertThat(innerHits.getAt(0).getType(), equalTo(legacy() ? "comment" : "doc"));
+        assertThat(innerHits.getAt(1).getId(), equalTo("c5"));
+        assertThat(innerHits.getAt(1).getType(), equalTo(legacy() ? "comment" : "doc"));
+        assertThat(innerHits.getAt(2).getId(), equalTo("c6"));
+        assertThat(innerHits.getAt(2).getType(), equalTo(legacy() ? "comment" : "doc"));
+
+        response = client().prepareSearch("articles")
+            .setQuery(
+                hasChildQuery("comment", matchQuery("message", "fox"), ScoreMode.None).innerHit(
+                    new InnerHitBuilder()
+                        .addDocValueField("message")
+                        .setHighlightBuilder(new HighlightBuilder().field("message"))
+                        .setExplain(true).setSize(1)
+                        .addScriptField("script", new Script(ScriptType.INLINE, MockScriptEngine.NAME, "5",
+                            Collections.emptyMap())))
+            ).get();
+        assertNoFailures(response);
+        innerHits = response.getHits().getAt(0).getInnerHits().get("comment");
+        assertThat(innerHits.getHits().length, equalTo(1));
+        assertThat(innerHits.getAt(0).getHighlightFields().get("message").getFragments()[0].string(), equalTo("<em>fox</em> eat quick"));
+        assertThat(innerHits.getAt(0).getExplanation().toString(), containsString("weight(message:fox"));
+        assertThat(innerHits.getAt(0).getFields().get("message").getValue().toString(), equalTo("eat"));
+        assertThat(innerHits.getAt(0).getFields().get("script").getValue().toString(), equalTo("5"));
+    }
+
+    public void testRandomParentChild() throws Exception {
+        if (legacy()) {
+            assertAcked(prepareCreate("idx")
+                .addMapping("parent")
+                .addMapping("child1", "_parent", "type=parent")
+                .addMapping("child2", "_parent", "type=parent")
+            );
+        } else {
+            assertAcked(prepareCreate("idx")
+                .addMapping("doc", jsonBuilder().startObject().startObject("doc").startObject("properties")
+                    .startObject("join_field")
+                        .field("type", "join")
+                        .startObject("relations")
+                            .field("parent", new String[] {"child1", "child2"})
+                        .endObject()
+                    .endObject()
+                    .endObject().endObject().endObject()
+                ));
+        }
+        int numDocs = scaledRandomIntBetween(5, 50);
+        List<IndexRequestBuilder> requestBuilders = new ArrayList<>();
+
+        int child1 = 0;
+        int child2 = 0;
+        int[] child1InnerObjects = new int[numDocs];
+        int[] child2InnerObjects = new int[numDocs];
+        for (int parent = 0; parent < numDocs; parent++) {
+            String parentId = String.format(Locale.ENGLISH, "p_%03d", parent);
+            requestBuilders.add(createIndexRequest("idx", "parent", parentId, null));
+
+            int numChildDocs = child1InnerObjects[parent] = scaledRandomIntBetween(1, numDocs);
+            int limit = child1 + numChildDocs;
+            for (; child1 < limit; child1++) {
+                requestBuilders.add(createIndexRequest("idx", "child1", String.format(Locale.ENGLISH, "c1_%04d", child1), parentId));
+            }
+            numChildDocs = child2InnerObjects[parent] = scaledRandomIntBetween(1, numDocs);
+            limit = child2 + numChildDocs;
+            for (; child2 < limit; child2++) {
+                requestBuilders.add(createIndexRequest("idx", "child2", String.format(Locale.ENGLISH, "c2_%04d", child2), parentId));
+            }
+        }
+        indexRandom(true, requestBuilders);
+
+        int size = randomIntBetween(0, numDocs);
+        BoolQueryBuilder boolQuery = new BoolQueryBuilder();
+        boolQuery.should(constantScoreQuery(hasChildQuery("child1", matchAllQuery(), ScoreMode.None)
+            .innerHit(new InnerHitBuilder().setName("a")
+                .addSort(new FieldSortBuilder("_uid").order(SortOrder.ASC)).setSize(size))));
+        boolQuery.should(constantScoreQuery(hasChildQuery("child2", matchAllQuery(), ScoreMode.None)
+            .innerHit(new InnerHitBuilder().setName("b")
+                .addSort(new FieldSortBuilder("_uid").order(SortOrder.ASC)).setSize(size))));
+        SearchResponse searchResponse = client().prepareSearch("idx")
+            .setSize(numDocs)
+            .addSort("_uid", SortOrder.ASC)
+            .setQuery(boolQuery)
+            .get();
+
+        assertNoFailures(searchResponse);
+        assertHitCount(searchResponse, numDocs);
+        assertThat(searchResponse.getHits().getHits().length, equalTo(numDocs));
+
+        int offset1 = 0;
+        int offset2 = 0;
+        for (int parent = 0; parent < numDocs; parent++) {
+            SearchHit searchHit = searchResponse.getHits().getAt(parent);
+            assertThat(searchHit.getType(), equalTo(legacy() ? "parent" : "doc"));
+            assertThat(searchHit.getId(), equalTo(String.format(Locale.ENGLISH, "p_%03d", parent)));
+            assertThat(searchHit.getShard(), notNullValue());
+
+            SearchHits inner = searchHit.getInnerHits().get("a");
+            assertThat(inner.getTotalHits(), equalTo((long) child1InnerObjects[parent]));
+            for (int child = 0; child < child1InnerObjects[parent] && child < size; child++) {
+                SearchHit innerHit =  inner.getAt(child);
+                assertThat(innerHit.getType(), equalTo(legacy() ? "child1" : "doc"));
+                String childId = String.format(Locale.ENGLISH, "c1_%04d", offset1 + child);
+                assertThat(innerHit.getId(), equalTo(childId));
+                assertThat(innerHit.getNestedIdentity(), nullValue());
+            }
+            offset1 += child1InnerObjects[parent];
+
+            inner = searchHit.getInnerHits().get("b");
+            assertThat(inner.getTotalHits(), equalTo((long) child2InnerObjects[parent]));
+            for (int child = 0; child < child2InnerObjects[parent] && child < size; child++) {
+                SearchHit innerHit = inner.getAt(child);
+                assertThat(innerHit.getType(), equalTo(legacy() ? "child2" : "doc"));
+                String childId = String.format(Locale.ENGLISH, "c2_%04d", offset2 + child);
+                assertThat(innerHit.getId(), equalTo(childId));
+                assertThat(innerHit.getNestedIdentity(), nullValue());
+            }
+            offset2 += child2InnerObjects[parent];
+        }
+    }
+
+    public void testInnerHitsOnHasParent() throws Exception {
+        if (legacy()) {
+            assertAcked(prepareCreate("stack")
+                .addMapping("question", "body", "type=text")
+                .addMapping("answer", "_parent", "type=question", "body", "type=text")
+            );
+        } else {
+            assertAcked(prepareCreate("stack")
+                .addMapping("doc", addFieldMappings(buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "question", "answer"),
+                    "body", "text")));
+        }
+        List<IndexRequestBuilder> requests = new ArrayList<>();
+        requests.add(createIndexRequest("stack", "question", "1", null, "body", "I'm using HTTPS + Basic authentication "
+            + "to protect a resource. How can I throttle authentication attempts to protect against brute force attacks?"));
+        requests.add(createIndexRequest("stack", "answer", "3", "1", "body", "install fail2ban and enable rules for apache"));
+        requests.add(createIndexRequest("stack", "question", "2", null, "body",
+            "I have firewall rules set up and also denyhosts installed.\\ndo I also need to install fail2ban?"));
+        requests.add(createIndexRequest("stack", "answer", "4", "2", "body",
+            "Denyhosts protects only ssh; Fail2Ban protects all daemons."));
+        indexRandom(true, requests);
+
+        SearchResponse response = client().prepareSearch("stack")
+            .addSort("_uid", SortOrder.ASC)
+            .setQuery(
+                boolQuery()
+                    .must(matchQuery("body", "fail2ban"))
+                    .must(hasParentQuery("question", matchAllQuery(), false).innerHit(new InnerHitBuilder()))
+            ).get();
+        assertNoFailures(response);
+        assertHitCount(response, 2);
+
+        SearchHit searchHit = response.getHits().getAt(0);
+        assertThat(searchHit.getId(), equalTo("3"));
+        assertThat(searchHit.getType(), equalTo(legacy() ? "answer" : "doc"));
+        assertThat(searchHit.getInnerHits().get("question").getTotalHits(), equalTo(1L));
+        assertThat(searchHit.getInnerHits().get("question").getAt(0).getType(), equalTo(legacy() ? "question" : "doc"));
+        assertThat(searchHit.getInnerHits().get("question").getAt(0).getId(), equalTo("1"));
+
+        searchHit = response.getHits().getAt(1);
+        assertThat(searchHit.getId(), equalTo("4"));
+        assertThat(searchHit.getType(), equalTo(legacy() ? "answer" : "doc"));
+        assertThat(searchHit.getInnerHits().get("question").getTotalHits(), equalTo(1L));
+        assertThat(searchHit.getInnerHits().get("question").getAt(0).getType(), equalTo(legacy() ? "question" : "doc"));
+        assertThat(searchHit.getInnerHits().get("question").getAt(0).getId(), equalTo("2"));
+    }
+
+    public void testParentChildMultipleLayers() throws Exception {
+        if (legacy()) {
+            assertAcked(prepareCreate("articles")
+                .addMapping("article", "title", "type=text")
+                .addMapping("comment", "_parent", "type=article", "message", "type=text")
+                .addMapping("remark", "_parent", "type=comment", "message", "type=text")
+            );
+        } else {
+            assertAcked(prepareCreate("articles")
+                .addMapping("doc",
+                    addFieldMappings(buildParentJoinFieldMappingFromSimplifiedDef("join_field", true,
+                        "article", "comment", "comment", "remark"), "title", "text", "message", "text")));
+        }
+
+        List<IndexRequestBuilder> requests = new ArrayList<>();
+        requests.add(createIndexRequest("articles", "article", "1", null, "title", "quick brown fox"));
+        requests.add(createIndexRequest("articles", "comment", "3", "1", "message", "fox eat quick"));
+        requests.add(createIndexRequest("articles", "remark", "5", "3", "message", "good").setRouting("1"));
+        requests.add(createIndexRequest("articles", "article", "2", null, "title", "big gray elephant"));
+        requests.add(createIndexRequest("articles", "comment", "4", "2", "message", "elephant captured"));
+        requests.add(createIndexRequest("articles", "remark", "6", "4", "message", "bad").setRouting("2"));
+        indexRandom(true, requests);
+
+        SearchResponse response = client().prepareSearch("articles")
+            .setQuery(hasChildQuery("comment",
+                hasChildQuery("remark", matchQuery("message", "good"), ScoreMode.None).innerHit(new InnerHitBuilder()),
+                ScoreMode.None).innerHit(new InnerHitBuilder()))
+            .get();
+
+        assertNoFailures(response);
+        assertHitCount(response, 1);
+        assertSearchHit(response, 1, hasId("1"));
+
+        assertThat(response.getHits().getAt(0).getInnerHits().size(), equalTo(1));
+        SearchHits innerHits = response.getHits().getAt(0).getInnerHits().get("comment");
+        assertThat(innerHits.getTotalHits(), equalTo(1L));
+        assertThat(innerHits.getAt(0).getId(), equalTo("3"));
+        assertThat(innerHits.getAt(0).getType(), equalTo(legacy() ? "comment" : "doc"));
+
+        innerHits = innerHits.getAt(0).getInnerHits().get("remark");
+        assertThat(innerHits.getTotalHits(), equalTo(1L));
+        assertThat(innerHits.getAt(0).getId(), equalTo("5"));
+        assertThat(innerHits.getAt(0).getType(), equalTo(legacy() ? "remark" : "doc"));
+
+        response = client().prepareSearch("articles")
+            .setQuery(hasChildQuery("comment",
+                hasChildQuery("remark", matchQuery("message", "bad"), ScoreMode.None).innerHit(new InnerHitBuilder()),
+                ScoreMode.None).innerHit(new InnerHitBuilder()))
+            .get();
+
+        assertNoFailures(response);
+        assertHitCount(response, 1);
+        assertSearchHit(response, 1, hasId("2"));
+
+        assertThat(response.getHits().getAt(0).getInnerHits().size(), equalTo(1));
+        innerHits = response.getHits().getAt(0).getInnerHits().get("comment");
+        assertThat(innerHits.getTotalHits(), equalTo(1L));
+        assertThat(innerHits.getAt(0).getId(), equalTo("4"));
+        assertThat(innerHits.getAt(0).getType(), equalTo(legacy() ? "comment" : "doc"));
+
+        innerHits = innerHits.getAt(0).getInnerHits().get("remark");
+        assertThat(innerHits.getTotalHits(), equalTo(1L));
+        assertThat(innerHits.getAt(0).getId(), equalTo("6"));
+        assertThat(innerHits.getAt(0).getType(), equalTo(legacy() ? "remark" : "doc"));
+    }
+
+    public void testRoyals() throws Exception {
+        if (legacy()) {
+            assertAcked(
+                prepareCreate("royals")
+                    .addMapping("king")
+                    .addMapping("prince", "_parent", "type=king")
+                    .addMapping("duke", "_parent", "type=prince")
+                    .addMapping("earl", "_parent", "type=duke")
+                    .addMapping("baron", "_parent", "type=earl")
+            );
+        } else {
+            assertAcked(prepareCreate("royals")
+                .addMapping("doc", buildParentJoinFieldMappingFromSimplifiedDef("join_field", true,
+                    "king", "prince", "prince", "duke", "duke", "earl", "earl", "baron")));
+        }
+
+        List<IndexRequestBuilder> requests = new ArrayList<>();
+        requests.add(createIndexRequest("royals", "king", "king", null));
+        requests.add(createIndexRequest("royals", "prince", "prince", "king"));
+        requests.add(createIndexRequest("royals", "duke", "duke", "prince").setRouting("king"));
+        requests.add(createIndexRequest("royals", "earl", "earl1", "duke").setRouting("king"));
+        requests.add(createIndexRequest("royals", "earl", "earl2", "duke").setRouting("king"));
+        requests.add(createIndexRequest("royals", "earl", "earl3", "duke").setRouting("king"));
+        requests.add(createIndexRequest("royals", "earl", "earl4", "duke").setRouting("king"));
+        requests.add(createIndexRequest("royals", "baron", "baron1", "earl1").setRouting("king"));
+        requests.add(createIndexRequest("royals", "baron", "baron2", "earl2").setRouting("king"));
+        requests.add(createIndexRequest("royals", "baron", "baron3", "earl3").setRouting("king"));
+        requests.add(createIndexRequest("royals", "baron", "baron4", "earl4").setRouting("king"));
+        indexRandom(true, requests);
+
+        SearchResponse response = client().prepareSearch("royals")
+            .setQuery(boolQuery()
+                .filter(hasParentQuery("prince",
+                    hasParentQuery("king", matchAllQuery(), false).innerHit(new InnerHitBuilder().setName("kings")),
+                    false).innerHit(new InnerHitBuilder().setName("princes"))
+                )
+                .filter(hasChildQuery("earl",
+                    hasChildQuery("baron", matchAllQuery(), ScoreMode.None)
+                        .innerHit(new InnerHitBuilder().setName("barons")),
+                    ScoreMode.None).innerHit(new InnerHitBuilder()
+                    .addSort(SortBuilders.fieldSort("_uid").order(SortOrder.ASC))
+                    .setName("earls")
+                    .setSize(4))
+                )
+            ).get();
+        assertHitCount(response, 1);
+        assertThat(response.getHits().getAt(0).getId(), equalTo("duke"));
+
+        SearchHits innerHits = response.getHits().getAt(0).getInnerHits().get("earls");
+        assertThat(innerHits.getTotalHits(), equalTo(4L));
+        assertThat(innerHits.getAt(0).getId(), equalTo("earl1"));
+        assertThat(innerHits.getAt(1).getId(), equalTo("earl2"));
+        assertThat(innerHits.getAt(2).getId(), equalTo("earl3"));
+        assertThat(innerHits.getAt(3).getId(), equalTo("earl4"));
+
+        SearchHits innerInnerHits = innerHits.getAt(0).getInnerHits().get("barons");
+        assertThat(innerInnerHits.getTotalHits(), equalTo(1L));
+        assertThat(innerInnerHits.getAt(0).getId(), equalTo("baron1"));
+
+        innerInnerHits = innerHits.getAt(1).getInnerHits().get("barons");
+        assertThat(innerInnerHits.getTotalHits(), equalTo(1L));
+        assertThat(innerInnerHits.getAt(0).getId(), equalTo("baron2"));
+
+        innerInnerHits = innerHits.getAt(2).getInnerHits().get("barons");
+        assertThat(innerInnerHits.getTotalHits(), equalTo(1L));
+        assertThat(innerInnerHits.getAt(0).getId(), equalTo("baron3"));
+
+        innerInnerHits = innerHits.getAt(3).getInnerHits().get("barons");
+        assertThat(innerInnerHits.getTotalHits(), equalTo(1L));
+        assertThat(innerInnerHits.getAt(0).getId(), equalTo("baron4"));
+
+        innerHits = response.getHits().getAt(0).getInnerHits().get("princes");
+        assertThat(innerHits.getTotalHits(), equalTo(1L));
+        assertThat(innerHits.getAt(0).getId(), equalTo("prince"));
+
+        innerInnerHits = innerHits.getAt(0).getInnerHits().get("kings");
+        assertThat(innerInnerHits.getTotalHits(), equalTo(1L));
+        assertThat(innerInnerHits.getAt(0).getId(), equalTo("king"));
+    }
+
+    public void testMatchesQueriesParentChildInnerHits() throws Exception {
+        if (legacy()) {
+            assertAcked(prepareCreate("index")
+                .addMapping("child", "_parent", "type=parent"));
+        } else {
+            assertAcked(prepareCreate("index")
+                .addMapping("doc", buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "parent", "child")));
+        }
+        List<IndexRequestBuilder> requests = new ArrayList<>();
+        requests.add(createIndexRequest("index", "parent", "1", null));
+        requests.add(createIndexRequest("index", "child", "3", "1", "field", "value1"));
+        requests.add(createIndexRequest("index", "child", "4", "1", "field", "value2"));
+        requests.add(createIndexRequest("index", "parent", "2", null));
+        requests.add(createIndexRequest("index", "child", "5", "2", "field", "value1"));
+        indexRandom(true, requests);
+
+        SearchResponse response = client().prepareSearch("index")
+            .setQuery(hasChildQuery("child", matchQuery("field", "value1").queryName("_name1"), ScoreMode.None)
+                .innerHit(new InnerHitBuilder()))
+            .addSort("_uid", SortOrder.ASC)
+            .get();
+        assertHitCount(response, 2);
+        assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
+        assertThat(response.getHits().getAt(0).getInnerHits().get("child").getTotalHits(), equalTo(1L));
+        assertThat(response.getHits().getAt(0).getInnerHits().get("child").getAt(0).getMatchedQueries().length, equalTo(1));
+        assertThat(response.getHits().getAt(0).getInnerHits().get("child").getAt(0).getMatchedQueries()[0], equalTo("_name1"));
+
+        assertThat(response.getHits().getAt(1).getId(), equalTo("2"));
+        assertThat(response.getHits().getAt(1).getInnerHits().get("child").getTotalHits(), equalTo(1L));
+        assertThat(response.getHits().getAt(1).getInnerHits().get("child").getAt(0).getMatchedQueries().length, equalTo(1));
+        assertThat(response.getHits().getAt(1).getInnerHits().get("child").getAt(0).getMatchedQueries()[0], equalTo("_name1"));
+
+        QueryBuilder query = hasChildQuery("child", matchQuery("field", "value2").queryName("_name2"), ScoreMode.None)
+            .innerHit(new InnerHitBuilder());
+        response = client().prepareSearch("index")
+            .setQuery(query)
+            .addSort("_uid", SortOrder.ASC)
+            .get();
+        assertHitCount(response, 1);
+        assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
+        assertThat(response.getHits().getAt(0).getInnerHits().get("child").getTotalHits(), equalTo(1L));
+        assertThat(response.getHits().getAt(0).getInnerHits().get("child").getAt(0).getMatchedQueries().length, equalTo(1));
+        assertThat(response.getHits().getAt(0).getInnerHits().get("child").getAt(0).getMatchedQueries()[0], equalTo("_name2"));
+    }
+
+    public void testDontExplode() throws Exception {
+        if (legacy()) {
+            assertAcked(prepareCreate("index1").addMapping("child", "_parent", "type=parent"));
+        } else {
+            assertAcked(prepareCreate("index1")
+                .addMapping("doc", buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "parent", "child")));
+        }
+        List<IndexRequestBuilder> requests = new ArrayList<>();
+        requests.add(createIndexRequest("index1", "parent", "1", null));
+        requests.add(createIndexRequest("index1", "child", "2", "1", "field", "value1"));
+        indexRandom(true, requests);
+
+        QueryBuilder query = hasChildQuery("child", matchQuery("field", "value1"), ScoreMode.None)
+            .innerHit(new InnerHitBuilder().setSize(ArrayUtil.MAX_ARRAY_LENGTH - 1));
+        SearchResponse response = client().prepareSearch("index1")
+            .setQuery(query)
+            .get();
+        assertNoFailures(response);
+        assertHitCount(response, 1);
+    }
+
+    public void testNestedInnerHitWrappedInParentChildInnerhit() throws Exception {
+        if (legacy()) {
+            assertAcked(prepareCreate("test")
+                .addMapping("child_type", "_parent", "type=parent_type", "nested_type", "type=nested"));
+        } else {
+            assertAcked(prepareCreate("test")
+                .addMapping("doc", addFieldMappings(buildParentJoinFieldMappingFromSimplifiedDef("join_field", true,
+                    "parent_type", "child_type"), "nested_type", "nested")));
+        }
+        createIndexRequest("test", "parent_type", "1", null, "key", "value").get();
+        createIndexRequest("test", "child_type", "2", "1", "nested_type", Collections.singletonMap("key", "value")).get();
+        refresh();
+        SearchResponse response = client().prepareSearch("test")
+            .setQuery(boolQuery().must(matchQuery("key", "value"))
+                .should(hasChildQuery("child_type", nestedQuery("nested_type", matchAllQuery(), ScoreMode.None)
+                    .innerHit(new InnerHitBuilder()), ScoreMode.None).innerHit(new InnerHitBuilder())))
+            .get();
+        assertHitCount(response, 1);
+        SearchHit hit = response.getHits().getAt(0);
+        if (legacy()) {
+            assertThat(hit.getInnerHits().get("child_type").getAt(0).field("_parent").getValue(), equalTo("1"));
+        } else {
+            String parentId = (String) extractValue("join_field.parent", hit.getInnerHits().get("child_type").getAt(0).getSourceAsMap());
+            assertThat(parentId, equalTo("1"));
+        }
+        assertThat(hit.getInnerHits().get("child_type").getAt(0).getInnerHits().get("nested_type").getAt(0).field("_parent"), nullValue());
+    }
+
+    public void testInnerHitsWithIgnoreUnmapped() throws Exception {
+        if (legacy()) {
+            assertAcked(prepareCreate("index1")
+                .addMapping("parent_type", "nested_type", "type=nested")
+                .addMapping("child_type", "_parent", "type=parent_type")
+            );
+        } else {
+            assertAcked(prepareCreate("index1")
+                .addMapping("doc", addFieldMappings(
+                    buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "parent_type", "child_type"),
+                    "nested_type", "nested"))
+            );
+        }
+        assertAcked(prepareCreate("index2"));
+        createIndexRequest("index1", "parent_type", "1", null, "nested_type", Collections.singletonMap("key", "value")).get();
+        createIndexRequest("index1", "child_type", "2", "1").get();
+        client().prepareIndex("index2", "type", "3").setSource("key", "value").get();
+        refresh();
+
+        SearchResponse response = client().prepareSearch("index1", "index2")
+            .setQuery(boolQuery()
+                .should(hasChildQuery("child_type", matchAllQuery(), ScoreMode.None).ignoreUnmapped(true)
+                    .innerHit(new InnerHitBuilder().setIgnoreUnmapped(true)))
+                .should(termQuery("key", "value"))
+            )
+            .get();
+        assertNoFailures(response);
+        assertHitCount(response, 2);
+        assertSearchHits(response, "1", "3");
+    }
+}
diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/query/LegacyChildQuerySearchIT.java b/modules/parent-join/src/test/java/org/elasticsearch/join/query/LegacyChildQuerySearchIT.java
new file mode 100644
index 0000000000000..a227d9e44be73
--- /dev/null
+++ b/modules/parent-join/src/test/java/org/elasticsearch/join/query/LegacyChildQuerySearchIT.java
@@ -0,0 +1,308 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.join.query;
+
+import org.apache.lucene.search.join.ScoreMode;
+import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
+import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
+import org.elasticsearch.action.bulk.BulkRequestBuilder;
+import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.search.SearchPhaseExecutionException;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+
+import java.io.IOException;
+import java.util.Map;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
+import static org.elasticsearch.index.query.QueryBuilders.multiMatchQuery;
+import static org.elasticsearch.join.query.JoinQueryBuilders.hasChildQuery;
+import static org.elasticsearch.join.query.JoinQueryBuilders.hasParentQuery;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.notNullValue;
+
+public class LegacyChildQuerySearchIT extends ChildQuerySearchIT {
+
+    @Override
+    protected boolean legacy() {
+        return true;
+    }
+
+    public void testIndexChildDocWithNoParentMapping() throws IOException {
+        assertAcked(prepareCreate("test")
+            .addMapping("parent")
+            .addMapping("child1"));
+        ensureGreen();
+
+        client().prepareIndex("test", "parent", "p1").setSource("p_field", "p_value1").get();
+        try {
+            client().prepareIndex("test", "child1", "c1").setParent("p1").setSource("c_field", "blue").get();
+            fail();
+        } catch (IllegalArgumentException e) {
+            assertThat(e.toString(), containsString("can't specify parent if no parent field has been configured"));
+        }
+        try {
+            client().prepareIndex("test", "child2", "c2").setParent("p1").setSource("c_field", "blue").get();
+            fail();
+        } catch (IllegalArgumentException e) {
+            assertThat(e.toString(), containsString("can't specify parent if no parent field has been configured"));
+        }
+
+        refresh();
+    }
+
+    public void testAddingParentToExistingMapping() throws IOException {
+        createIndex("test");
+        ensureGreen();
+
+        PutMappingResponse putMappingResponse = client().admin().indices()
+            .preparePutMapping("test").setType("child").setSource("number", "type=integer")
+            .get();
+        assertThat(putMappingResponse.isAcknowledged(), equalTo(true));
+
+        GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings("test").get();
+        Map<String, Object> mapping = getMappingsResponse.getMappings().get("test").get("child").getSourceAsMap();
+        assertThat(mapping.size(), greaterThanOrEqualTo(1)); // there are potentially some meta fields configured randomly
+        assertThat(mapping.get("properties"), notNullValue());
+
+        try {
+            // Adding _parent metadata field to existing mapping is prohibited:
+            client().admin().indices().preparePutMapping("test").setType("child").setSource(jsonBuilder().startObject().startObject("child")
+                .startObject("_parent").field("type", "parent").endObject()
+                .endObject().endObject()).get();
+            fail();
+        } catch (IllegalArgumentException e) {
+            assertThat(e.toString(), containsString("The _parent field's type option can't be changed: [null]->[parent]"));
+        }
+    }
+
+    // Issue #5783
+    public void testQueryBeforeChildType() throws Exception {
+        assertAcked(prepareCreate("test")
+            .addMapping("features")
+            .addMapping("posts", "_parent", "type=features")
+            .addMapping("specials"));
+        ensureGreen();
+
+        client().prepareIndex("test", "features", "1").setSource("field", "foo").get();
+        client().prepareIndex("test", "posts", "1").setParent("1").setSource("field", "bar").get();
+        refresh();
+
+        SearchResponse resp;
+        resp = client().prepareSearch("test")
+            .setSource(new SearchSourceBuilder().query(hasChildQuery("posts",
+                QueryBuilders.matchQuery("field", "bar"), ScoreMode.None)))
+            .get();
+        assertHitCount(resp, 1L);
+    }
+
+    // Issue #6256
+    public void testParentFieldInMultiMatchField() throws Exception {
+        assertAcked(prepareCreate("test")
+            .addMapping("type1")
+            .addMapping("type2", "_parent", "type=type1")
+        );
+        ensureGreen();
+
+        client().prepareIndex("test", "type2", "1").setParent("1").setSource("field", "value").get();
+        refresh();
+
+        SearchResponse response = client().prepareSearch("test")
+            .setQuery(multiMatchQuery("1", "_parent#type1"))
+            .get();
+
+        assertThat(response.getHits().getTotalHits(), equalTo(1L));
+        assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
+    }
+
+    public void testParentFieldToNonExistingType() {
+        assertAcked(prepareCreate("test")
+            .addMapping("parent").addMapping("child", "_parent", "type=parent2"));
+        client().prepareIndex("test", "parent", "1").setSource("{}", XContentType.JSON).get();
+        client().prepareIndex("test", "child", "1").setParent("1").setSource("{}", XContentType.JSON).get();
+        refresh();
+
+        try {
+            client().prepareSearch("test")
+                .setQuery(hasChildQuery("child", matchAllQuery(), ScoreMode.None))
+                .get();
+            fail();
+        } catch (SearchPhaseExecutionException e) {
+        }
+    }
+
+    /*
+   Test for https://github.com/elastic/elasticsearch/issues/3444
+    */
+    public void testBulkUpdateDocAsUpsertWithParent() throws Exception {
+        assertAcked(prepareCreate("test")
+            .addMapping("parent", "{\"parent\":{}}", XContentType.JSON)
+            .addMapping("child", "{\"child\": {\"_parent\": {\"type\": \"parent\"}}}", XContentType.JSON));
+        ensureGreen();
+
+        BulkRequestBuilder builder = client().prepareBulk();
+
+        // It's important to use JSON parsing here and request objects: issue 3444 is related to incomplete option parsing
+        byte[] addParent = new BytesArray(
+            "{" +
+                "  \"index\" : {" +
+                "    \"_index\" : \"test\"," +
+                "    \"_type\"  : \"parent\"," +
+                "    \"_id\"    : \"parent1\"" +
+                "  }" +
+                "}" +
+                "\n" +
+                "{" +
+                "  \"field1\" : \"value1\"" +
+                "}" +
+                "\n").array();
+
+        byte[] addChild = new BytesArray(
+            "{" +
+                "  \"update\" : {" +
+                "    \"_index\" : \"test\"," +
+                "    \"_type\"  : \"child\"," +
+                "    \"_id\"    : \"child1\"," +
+                "    \"parent\" : \"parent1\"" +
+                "  }" +
+                "}" +
+                "\n" +
+                "{" +
+                "  \"doc\" : {" +
+                "    \"field1\" : \"value1\"" +
+                "  }," +
+                "  \"doc_as_upsert\" : \"true\"" +
+                "}" +
+                "\n").array();
+
+        builder.add(addParent, 0, addParent.length, XContentType.JSON);
+        builder.add(addChild, 0, addChild.length, XContentType.JSON);
+
+        BulkResponse bulkResponse = builder.get();
+        assertThat(bulkResponse.getItems().length, equalTo(2));
+        assertThat(bulkResponse.getItems()[0].isFailed(), equalTo(false));
+        assertThat(bulkResponse.getItems()[1].isFailed(), equalTo(false));
+
+        client().admin().indices().prepareRefresh("test").get();
+
+        //we check that the _parent field was set on the child document by using the has parent query
+        SearchResponse searchResponse = client().prepareSearch("test")
+            .setQuery(hasParentQuery("parent", QueryBuilders.matchAllQuery(), false))
+            .get();
+
+        assertNoFailures(searchResponse);
+        assertSearchHits(searchResponse, "child1");
+    }
+
+    /*
+    Test for https://github.com/elastic/elasticsearch/issues/3444
+     */
+    public void testBulkUpdateUpsertWithParent() throws Exception {
+        assertAcked(prepareCreate("test")
+            .addMapping("parent", "{\"parent\":{}}", XContentType.JSON)
+            .addMapping("child", "{\"child\": {\"_parent\": {\"type\": \"parent\"}}}", XContentType.JSON));
+        ensureGreen();
+
+        BulkRequestBuilder builder = client().prepareBulk();
+
+        byte[] addParent = new BytesArray(
+            "{" +
+                "  \"index\" : {" +
+                "    \"_index\" : \"test\"," +
+                "    \"_type\"  : \"parent\"," +
+                "    \"_id\"    : \"parent1\"" +
+                "  }" +
+                "}" +
+                "\n" +
+                "{" +
+                "  \"field1\" : \"value1\"" +
+                "}" +
+                "\n").array();
+
+        byte[] addChild1 = new BytesArray(
+            "{" +
+                "  \"update\" : {" +
+                "    \"_index\" : \"test\"," +
+                "    \"_type\"  : \"child\"," +
+                "    \"_id\"    : \"child1\"," +
+                "    \"parent\" : \"parent1\"" +
+                "  }" +
+                "}" +
+                "\n" +
+                "{" +
+                "  \"script\" : {" +
+                "    \"inline\" : \"ctx._source.field2 = 'value2'\"" +
+                "  }," +
+                "  \"lang\" : \"" + InnerHitsIT.CustomScriptPlugin.NAME + "\"," +
+                "  \"upsert\" : {" +
+                "    \"field1\" : \"value1'\"" +
+                "  }" +
+                "}" +
+                "\n").array();
+
+        byte[] addChild2 = new BytesArray(
+            "{" +
+                "  \"update\" : {" +
+                "    \"_index\" : \"test\"," +
+                "    \"_type\"  : \"child\"," +
+                "    \"_id\"    : \"child1\"," +
+                "    \"parent\" : \"parent1\"" +
+                "  }" +
+                "}" +
+                "\n" +
+                "{" +
+                "  \"script\" : \"ctx._source.field2 = 'value2'\"," +
+                "  \"upsert\" : {" +
+                "    \"field1\" : \"value1'\"" +
+                "  }" +
+                "}" +
+                "\n").array();
+
+        builder.add(addParent, 0, addParent.length, XContentType.JSON);
+        builder.add(addChild1, 0, addChild1.length, XContentType.JSON);
+        builder.add(addChild2, 0, addChild2.length, XContentType.JSON);
+
+        BulkResponse bulkResponse = builder.get();
+        assertThat(bulkResponse.getItems().length, equalTo(3));
+        assertThat(bulkResponse.getItems()[0].isFailed(), equalTo(false));
+        assertThat(bulkResponse.getItems()[1].isFailed(), equalTo(false));
+        assertThat(bulkResponse.getItems()[2].isFailed(), equalTo(true));
+        assertThat(bulkResponse.getItems()[2].getFailure().getCause().getCause().getMessage(),
+            equalTo("script_lang not supported [painless]"));
+
+        client().admin().indices().prepareRefresh("test").get();
+
+        SearchResponse searchResponse = client().prepareSearch("test")
+            .setQuery(hasParentQuery("parent", QueryBuilders.matchAllQuery(), false))
+            .get();
+
+        assertSearchHits(searchResponse, "child1");
+    }
+
+}
diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/query/LegacyHasChildQueryBuilderTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/query/LegacyHasChildQueryBuilderTests.java
new file mode 100644
index 0000000000000..1fec930cb7997
--- /dev/null
+++ b/modules/parent-join/src/test/java/org/elasticsearch/join/query/LegacyHasChildQueryBuilderTests.java
@@ -0,0 +1,349 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.join.query;
+
+import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.ConstantScoreQuery;
+import org.apache.lucene.search.MatchNoDocsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermInSetQuery;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.join.ScoreMode;
+import org.apache.lucene.search.similarities.PerFieldSimilarityWrapper;
+import org.apache.lucene.search.similarities.Similarity;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
+import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.TypeFieldMapper;
+import org.elasticsearch.index.mapper.Uid;
+import org.elasticsearch.index.mapper.UidFieldMapper;
+import org.elasticsearch.index.query.IdsQueryBuilder;
+import org.elasticsearch.index.query.InnerHitBuilder;
+import org.elasticsearch.index.query.InnerHitContextBuilder;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.query.QueryShardException;
+import org.elasticsearch.index.query.TermQueryBuilder;
+import org.elasticsearch.index.query.WrapperQueryBuilder;
+import org.elasticsearch.index.similarity.SimilarityService;
+import org.elasticsearch.join.ParentJoinPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.search.fetch.subphase.InnerHitsContext;
+import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.search.sort.FieldSortBuilder;
+import org.elasticsearch.search.sort.SortOrder;
+import org.elasticsearch.test.AbstractQueryTestCase;
+import org.elasticsearch.test.VersionUtils;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.elasticsearch.join.query.JoinQueryBuilders.hasChildQuery;
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.hamcrest.CoreMatchers.notNullValue;
+
+public class LegacyHasChildQueryBuilderTests extends AbstractQueryTestCase<HasChildQueryBuilder> {
+    protected static final String PARENT_TYPE = "parent";
+    protected static final String CHILD_TYPE = "child";
+
+    private static String similarity;
+
+    boolean requiresRewrite = false;
+
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return Collections.singletonList(ParentJoinPlugin.class);
+    }
+
+    @Override
+    protected void initializeAdditionalMappings(MapperService mapperService) throws IOException {
+        similarity = randomFrom("classic", "BM25");
+        // TODO: use a single type when inner hits have been changed to work with join field,
+        // this test randomly generates queries with inner hits
+        mapperService.merge(PARENT_TYPE, new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef(PARENT_TYPE,
+                STRING_FIELD_NAME, "type=text",
+                STRING_FIELD_NAME_2, "type=keyword",
+                INT_FIELD_NAME, "type=integer",
+                DOUBLE_FIELD_NAME, "type=double",
+                BOOLEAN_FIELD_NAME, "type=boolean",
+                DATE_FIELD_NAME, "type=date",
+                OBJECT_FIELD_NAME, "type=object"
+        ).string()), MapperService.MergeReason.MAPPING_UPDATE, false);
+        mapperService.merge(CHILD_TYPE, new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef(CHILD_TYPE,
+                "_parent", "type=" + PARENT_TYPE,
+                STRING_FIELD_NAME, "type=text",
+                "custom_string", "type=text,similarity=" + similarity,
+                INT_FIELD_NAME, "type=integer",
+                DOUBLE_FIELD_NAME, "type=double",
+                BOOLEAN_FIELD_NAME, "type=boolean",
+                DATE_FIELD_NAME, "type=date",
+                OBJECT_FIELD_NAME, "type=object"
+        ).string()), MapperService.MergeReason.MAPPING_UPDATE, false);
+    }
+
+    @Override
+    protected Settings indexSettings() {
+        return Settings.builder()
+            .put(super.indexSettings())
+            .build();
+    }
+
+    /**
+     * @return a {@link HasChildQueryBuilder} with random values all over the place
+     */
+    @Override
+    protected HasChildQueryBuilder doCreateTestQueryBuilder() {
+        int min = randomIntBetween(0, Integer.MAX_VALUE / 2);
+        int max = randomIntBetween(min, Integer.MAX_VALUE);
+
+        QueryBuilder innerQueryBuilder = new MatchAllQueryBuilder();
+        if (randomBoolean()) {
+            requiresRewrite = true;
+            innerQueryBuilder = new WrapperQueryBuilder(innerQueryBuilder.toString());
+        }
+
+        HasChildQueryBuilder hqb = new HasChildQueryBuilder(CHILD_TYPE, innerQueryBuilder,
+                RandomPicks.randomFrom(random(), ScoreMode.values()));
+        hqb.minMaxChildren(min, max);
+        hqb.ignoreUnmapped(randomBoolean());
+        if (randomBoolean()) {
+            hqb.innerHit(new InnerHitBuilder()
+                    .setName(randomAlphaOfLengthBetween(1, 10))
+                    .setSize(randomIntBetween(0, 100))
+                    .addSort(new FieldSortBuilder(STRING_FIELD_NAME_2).order(SortOrder.ASC))
+                    .setIgnoreUnmapped(hqb.ignoreUnmapped()));
+        }
+        return hqb;
+    }
+
+    @Override
+    protected void doAssertLuceneQuery(HasChildQueryBuilder queryBuilder, Query query, SearchContext searchContext) throws IOException {
+        assertThat(query, instanceOf(HasChildQueryBuilder.LateParsingQuery.class));
+        HasChildQueryBuilder.LateParsingQuery lpq = (HasChildQueryBuilder.LateParsingQuery) query;
+        assertEquals(queryBuilder.minChildren(), lpq.getMinChildren());
+        assertEquals(queryBuilder.maxChildren(), lpq.getMaxChildren());
+        assertEquals(queryBuilder.scoreMode(), lpq.getScoreMode()); // WTF is this why do we have two?
+        if (queryBuilder.innerHit() != null) {
+            // have to rewrite again because the provided queryBuilder hasn't been rewritten (directly returned from
+            // doCreateTestQueryBuilder)
+            queryBuilder = (HasChildQueryBuilder) queryBuilder.rewrite(searchContext.getQueryShardContext());
+            Map<String, InnerHitContextBuilder> innerHitBuilders = new HashMap<>();
+            InnerHitContextBuilder.extractInnerHits(queryBuilder, innerHitBuilders);
+            for (InnerHitContextBuilder builder : innerHitBuilders.values()) {
+                builder.build(searchContext, searchContext.innerHits());
+            }
+            assertNotNull(searchContext.innerHits());
+            assertEquals(1, searchContext.innerHits().getInnerHits().size());
+            assertTrue(searchContext.innerHits().getInnerHits().containsKey(queryBuilder.innerHit().getName()));
+            InnerHitsContext.InnerHitSubContext innerHits =
+                    searchContext.innerHits().getInnerHits().get(queryBuilder.innerHit().getName());
+            assertEquals(innerHits.size(), queryBuilder.innerHit().getSize());
+            assertEquals(innerHits.sort().sort.getSort().length, 1);
+            assertEquals(innerHits.sort().sort.getSort()[0].getField(), STRING_FIELD_NAME_2);
+        }
+    }
+
+    /**
+     * Test (de)serialization on all previous released versions
+     */
+    public void testSerializationBWC() throws IOException {
+        for (Version version : VersionUtils.allReleasedVersions()) {
+            HasChildQueryBuilder testQuery = createTestQueryBuilder();
+            if (version.before(Version.V_5_2_0) && testQuery.innerHit() != null) {
+                // ignore unmapped for inner_hits has been added on 5.2
+                testQuery.innerHit().setIgnoreUnmapped(false);
+            }
+            assertSerialization(testQuery, version);
+        }
+    }
+
+    public void testIllegalValues() {
+        QueryBuilder query = new MatchAllQueryBuilder();
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> hasChildQuery(null, query, ScoreMode.None));
+        assertEquals("[has_child] requires 'type' field", e.getMessage());
+
+        e = expectThrows(IllegalArgumentException.class, () -> hasChildQuery("foo", null, ScoreMode.None));
+        assertEquals("[has_child] requires 'query' field", e.getMessage());
+
+        e = expectThrows(IllegalArgumentException.class, () -> hasChildQuery("foo", query, null));
+        assertEquals("[has_child] requires 'score_mode' field", e.getMessage());
+
+        int positiveValue = randomIntBetween(0, Integer.MAX_VALUE);
+        HasChildQueryBuilder foo = hasChildQuery("foo", query, ScoreMode.None); // all good
+        e = expectThrows(IllegalArgumentException.class, () -> foo.minMaxChildren(randomIntBetween(Integer.MIN_VALUE, -1), positiveValue));
+        assertEquals("[has_child] requires non-negative 'min_children' field", e.getMessage());
+
+        e = expectThrows(IllegalArgumentException.class, () -> foo.minMaxChildren(positiveValue, randomIntBetween(Integer.MIN_VALUE, -1)));
+        assertEquals("[has_child] requires non-negative 'max_children' field", e.getMessage());
+
+        e = expectThrows(IllegalArgumentException.class, () -> foo.minMaxChildren(positiveValue, positiveValue - 10));
+        assertEquals("[has_child] 'max_children' is less than 'min_children'", e.getMessage());
+    }
+
+    public void testFromJson() throws IOException {
+        String query =
+                "{\n" +
+                "  \"has_child\" : {\n" +
+                "    \"query\" : {\n" +
+                "      \"range\" : {\n" +
+                "        \"mapped_string\" : {\n" +
+                "          \"from\" : \"agJhRET\",\n" +
+                "          \"to\" : \"zvqIq\",\n" +
+                "          \"include_lower\" : true,\n" +
+                "          \"include_upper\" : true,\n" +
+                "          \"boost\" : 1.0\n" +
+                "        }\n" +
+                "      }\n" +
+                "    },\n" +
+                "    \"type\" : \"child\",\n" +
+                "    \"score_mode\" : \"avg\",\n" +
+                "    \"min_children\" : 883170873,\n" +
+                "    \"max_children\" : 1217235442,\n" +
+                "    \"ignore_unmapped\" : false,\n" +
+                "    \"boost\" : 2.0,\n" +
+                "    \"_name\" : \"WNzYMJKRwePuRBh\",\n" +
+                "    \"inner_hits\" : {\n" +
+                "      \"name\" : \"inner_hits_name\",\n" +
+                "      \"ignore_unmapped\" : false,\n" +
+                "      \"from\" : 0,\n" +
+                "      \"size\" : 100,\n" +
+                "      \"version\" : false,\n" +
+                "      \"explain\" : false,\n" +
+                "      \"track_scores\" : false,\n" +
+                "      \"sort\" : [ {\n" +
+                "        \"mapped_string\" : {\n" +
+                "          \"order\" : \"asc\"\n" +
+                "        }\n" +
+                "      } ]\n" +
+                "    }\n" +
+                "  }\n" +
+                "}";
+        HasChildQueryBuilder queryBuilder = (HasChildQueryBuilder) parseQuery(query);
+        checkGeneratedJson(query, queryBuilder);
+        assertEquals(query, queryBuilder.maxChildren(), 1217235442);
+        assertEquals(query, queryBuilder.minChildren(), 883170873);
+        assertEquals(query, queryBuilder.boost(), 2.0f, 0.0f);
+        assertEquals(query, queryBuilder.queryName(), "WNzYMJKRwePuRBh");
+        assertEquals(query, queryBuilder.childType(), "child");
+        assertEquals(query, queryBuilder.scoreMode(), ScoreMode.Avg);
+        assertNotNull(query, queryBuilder.innerHit());
+        InnerHitBuilder expected = new InnerHitBuilder("child")
+                .setName("inner_hits_name")
+                .setSize(100)
+                .addSort(new FieldSortBuilder("mapped_string").order(SortOrder.ASC));
+        assertEquals(query, queryBuilder.innerHit(), expected);
+    }
+
+    public void testToQueryInnerQueryType() throws IOException {
+        String[] searchTypes = new String[]{PARENT_TYPE};
+        QueryShardContext shardContext = createShardContext();
+        shardContext.setTypes(searchTypes);
+        HasChildQueryBuilder hasChildQueryBuilder = hasChildQuery(CHILD_TYPE, new IdsQueryBuilder().addIds("id"), ScoreMode.None);
+        Query query = hasChildQueryBuilder.toQuery(shardContext);
+        //verify that the context types are still the same as the ones we previously set
+        assertThat(shardContext.getTypes(), equalTo(searchTypes));
+        assertLateParsingQuery(query, CHILD_TYPE, "id");
+    }
+
+    static void assertLateParsingQuery(Query query, String type, String id) throws IOException {
+        assertThat(query, instanceOf(HasChildQueryBuilder.LateParsingQuery.class));
+        HasChildQueryBuilder.LateParsingQuery lateParsingQuery = (HasChildQueryBuilder.LateParsingQuery) query;
+        assertThat(lateParsingQuery.getInnerQuery(), instanceOf(BooleanQuery.class));
+        BooleanQuery booleanQuery = (BooleanQuery) lateParsingQuery.getInnerQuery();
+        assertThat(booleanQuery.clauses().size(), equalTo(2));
+        //check the inner ids query, we have to call rewrite to get to check the type it's executed against
+        assertThat(booleanQuery.clauses().get(0).getOccur(), equalTo(BooleanClause.Occur.MUST));
+        assertThat(booleanQuery.clauses().get(0).getQuery(), instanceOf(TermInSetQuery.class));
+        TermInSetQuery termsQuery = (TermInSetQuery) booleanQuery.clauses().get(0).getQuery();
+        Query rewrittenTermsQuery = termsQuery.rewrite(null);
+        assertThat(rewrittenTermsQuery, instanceOf(ConstantScoreQuery.class));
+        ConstantScoreQuery constantScoreQuery = (ConstantScoreQuery) rewrittenTermsQuery;
+        assertThat(constantScoreQuery.getQuery(), instanceOf(BooleanQuery.class));
+        BooleanQuery booleanTermsQuery = (BooleanQuery) constantScoreQuery.getQuery();
+        assertThat(booleanTermsQuery.clauses().toString(), booleanTermsQuery.clauses().size(), equalTo(1));
+        assertThat(booleanTermsQuery.clauses().get(0).getOccur(), equalTo(BooleanClause.Occur.SHOULD));
+        assertThat(booleanTermsQuery.clauses().get(0).getQuery(), instanceOf(TermQuery.class));
+        TermQuery termQuery = (TermQuery) booleanTermsQuery.clauses().get(0).getQuery();
+        assertThat(termQuery.getTerm().field(), equalTo(UidFieldMapper.NAME));
+        //we want to make sure that the inner ids query gets executed against the child type rather
+        // than the main type we initially set to the context
+        BytesRef[] ids = Uid.createUidsForTypesAndIds(Collections.singletonList(type), Collections.singletonList(id));
+        assertThat(termQuery.getTerm().bytes(), equalTo(ids[0]));
+        //check the type filter
+        assertThat(booleanQuery.clauses().get(1).getOccur(), equalTo(BooleanClause.Occur.FILTER));
+        assertEquals(new TypeFieldMapper.TypesQuery(new BytesRef(type)), booleanQuery.clauses().get(1).getQuery());
+    }
+
+    @Override
+    public void testMustRewrite() throws IOException {
+        try {
+            super.testMustRewrite();
+        } catch (UnsupportedOperationException e) {
+            if (requiresRewrite == false) {
+                throw e;
+            }
+        }
+    }
+
+    public void testNonDefaultSimilarity() throws Exception {
+        QueryShardContext shardContext = createShardContext();
+        HasChildQueryBuilder hasChildQueryBuilder =
+            hasChildQuery(CHILD_TYPE, new TermQueryBuilder("custom_string", "value"), ScoreMode.None);
+        HasChildQueryBuilder.LateParsingQuery query = (HasChildQueryBuilder.LateParsingQuery) hasChildQueryBuilder.toQuery(shardContext);
+        Similarity expected = SimilarityService.BUILT_IN.get(similarity).apply(similarity, Settings.EMPTY).get();
+        assertThat(((PerFieldSimilarityWrapper) query.getSimilarity()).get("custom_string"), instanceOf(expected.getClass()));
+    }
+
+    public void testIgnoreUnmapped() throws IOException {
+        final HasChildQueryBuilder queryBuilder = new HasChildQueryBuilder("unmapped", new MatchAllQueryBuilder(), ScoreMode.None);
+        queryBuilder.ignoreUnmapped(true);
+        Query query = queryBuilder.toQuery(createShardContext());
+        assertThat(query, notNullValue());
+        assertThat(query, instanceOf(MatchNoDocsQuery.class));
+
+        final HasChildQueryBuilder failingQueryBuilder = new HasChildQueryBuilder("unmapped", new MatchAllQueryBuilder(), ScoreMode.None);
+        failingQueryBuilder.ignoreUnmapped(false);
+        QueryShardException e = expectThrows(QueryShardException.class, () -> failingQueryBuilder.toQuery(createShardContext()));
+        assertThat(e.getMessage(), containsString("[" + HasChildQueryBuilder.NAME + "] no mapping found for type [unmapped]"));
+    }
+
+    public void testIgnoreUnmappedWithRewrite() throws IOException {
+        // WrapperQueryBuilder makes sure we always rewrite
+        final HasChildQueryBuilder queryBuilder
+            = new HasChildQueryBuilder("unmapped", new WrapperQueryBuilder(new MatchAllQueryBuilder().toString()), ScoreMode.None);
+        queryBuilder.ignoreUnmapped(true);
+        QueryShardContext queryShardContext = createShardContext();
+        Query query = queryBuilder.rewrite(queryShardContext).toQuery(queryShardContext);
+        assertThat(query, notNullValue());
+        assertThat(query, instanceOf(MatchNoDocsQuery.class));
+    }
+}
diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/query/LegacyHasParentQueryBuilderTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/query/LegacyHasParentQueryBuilderTests.java
new file mode 100644
index 0000000000000..a87073bcf55f2
--- /dev/null
+++ b/modules/parent-join/src/test/java/org/elasticsearch/join/query/LegacyHasParentQueryBuilderTests.java
@@ -0,0 +1,273 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.join.query;
+
+import org.apache.lucene.search.MatchNoDocsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.join.ScoreMode;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
+import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.query.IdsQueryBuilder;
+import org.elasticsearch.index.query.InnerHitBuilder;
+import org.elasticsearch.index.query.InnerHitContextBuilder;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.query.QueryShardException;
+import org.elasticsearch.index.query.TermQueryBuilder;
+import org.elasticsearch.index.query.WrapperQueryBuilder;
+import org.elasticsearch.join.ParentJoinPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.search.fetch.subphase.InnerHitsContext;
+import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.search.sort.FieldSortBuilder;
+import org.elasticsearch.search.sort.SortOrder;
+import org.elasticsearch.test.AbstractQueryTestCase;
+import org.elasticsearch.test.VersionUtils;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.elasticsearch.join.query.JoinQueryBuilders.hasParentQuery;
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.hamcrest.CoreMatchers.notNullValue;
+
+public class LegacyHasParentQueryBuilderTests extends AbstractQueryTestCase<HasParentQueryBuilder> {
+    protected static final String PARENT_TYPE = "parent";
+    protected static final String CHILD_TYPE = "child";
+
+    boolean requiresRewrite = false;
+
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return Collections.singletonList(ParentJoinPlugin.class);
+    }
+
+    @Override
+    protected Settings indexSettings() {
+        return Settings.builder()
+            .put(super.indexSettings())
+            .build();
+    }
+
+    @Override
+    protected void initializeAdditionalMappings(MapperService mapperService) throws IOException {
+        // TODO: use a single type when inner hits have been changed to work with join field,
+        // this test randomly generates queries with inner hits
+        mapperService.merge(PARENT_TYPE, new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef(PARENT_TYPE,
+                STRING_FIELD_NAME, "type=text",
+                STRING_FIELD_NAME_2, "type=keyword",
+                INT_FIELD_NAME, "type=integer",
+                DOUBLE_FIELD_NAME, "type=double",
+                BOOLEAN_FIELD_NAME, "type=boolean",
+                DATE_FIELD_NAME, "type=date",
+                OBJECT_FIELD_NAME, "type=object"
+        ).string()), MapperService.MergeReason.MAPPING_UPDATE, false);
+        mapperService.merge(CHILD_TYPE, new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef(CHILD_TYPE,
+                "_parent", "type=" + PARENT_TYPE,
+                STRING_FIELD_NAME, "type=text",
+                STRING_FIELD_NAME_2, "type=keyword",
+                INT_FIELD_NAME, "type=integer",
+                DOUBLE_FIELD_NAME, "type=double",
+                BOOLEAN_FIELD_NAME, "type=boolean",
+                DATE_FIELD_NAME, "type=date",
+                OBJECT_FIELD_NAME, "type=object"
+        ).string()), MapperService.MergeReason.MAPPING_UPDATE, false);
+        mapperService.merge("just_a_type", new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef("just_a_type"
+        ).string()), MapperService.MergeReason.MAPPING_UPDATE, false);
+    }
+
+    /**
+     * @return a {@link HasChildQueryBuilder} with random values all over the place
+     */
+    @Override
+    protected HasParentQueryBuilder doCreateTestQueryBuilder() {
+        QueryBuilder innerQueryBuilder = new MatchAllQueryBuilder();
+        if (randomBoolean()) {
+            requiresRewrite = true;
+            innerQueryBuilder = new WrapperQueryBuilder(innerQueryBuilder.toString());
+        }
+        HasParentQueryBuilder hqb = new HasParentQueryBuilder(PARENT_TYPE, innerQueryBuilder, randomBoolean());
+        hqb.ignoreUnmapped(randomBoolean());
+        if (randomBoolean()) {
+            hqb.innerHit(new InnerHitBuilder()
+                    .setName(randomAlphaOfLengthBetween(1, 10))
+                    .setSize(randomIntBetween(0, 100))
+                    .addSort(new FieldSortBuilder(STRING_FIELD_NAME_2).order(SortOrder.ASC))
+                    .setIgnoreUnmapped(hqb.ignoreUnmapped()));
+        }
+        return hqb;
+    }
+
+    @Override
+    protected void doAssertLuceneQuery(HasParentQueryBuilder queryBuilder, Query query, SearchContext searchContext) throws IOException {
+        assertThat(query, instanceOf(HasChildQueryBuilder.LateParsingQuery.class));
+        HasChildQueryBuilder.LateParsingQuery lpq = (HasChildQueryBuilder.LateParsingQuery) query;
+        assertEquals(queryBuilder.score() ? ScoreMode.Max : ScoreMode.None, lpq.getScoreMode());
+
+        if (queryBuilder.innerHit() != null) {
+            // have to rewrite again because the provided queryBuilder hasn't been rewritten (directly returned from
+            // doCreateTestQueryBuilder)
+            queryBuilder = (HasParentQueryBuilder) queryBuilder.rewrite(searchContext.getQueryShardContext());
+
+            assertNotNull(searchContext);
+            Map<String, InnerHitContextBuilder> innerHitBuilders = new HashMap<>();
+            InnerHitContextBuilder.extractInnerHits(queryBuilder, innerHitBuilders);
+            for (InnerHitContextBuilder builder : innerHitBuilders.values()) {
+                builder.build(searchContext, searchContext.innerHits());
+            }
+            assertNotNull(searchContext.innerHits());
+            assertEquals(1, searchContext.innerHits().getInnerHits().size());
+            assertTrue(searchContext.innerHits().getInnerHits().containsKey(queryBuilder.innerHit().getName()));
+            InnerHitsContext.InnerHitSubContext innerHits = searchContext.innerHits()
+                    .getInnerHits().get(queryBuilder.innerHit().getName());
+            assertEquals(innerHits.size(), queryBuilder.innerHit().getSize());
+            assertEquals(innerHits.sort().sort.getSort().length, 1);
+            assertEquals(innerHits.sort().sort.getSort()[0].getField(), STRING_FIELD_NAME_2);
+        }
+    }
+
+    /**
+     * Test (de)serialization on all previous released versions
+     */
+    public void testSerializationBWC() throws IOException {
+        for (Version version : VersionUtils.allReleasedVersions()) {
+            HasParentQueryBuilder testQuery = createTestQueryBuilder();
+            if (version.before(Version.V_5_2_0) && testQuery.innerHit() != null) {
+                // ignore unmapped for inner_hits has been added on 5.2
+                testQuery.innerHit().setIgnoreUnmapped(false);
+            }
+            assertSerialization(testQuery, version);
+        }
+    }
+
+    public void testIllegalValues() throws IOException {
+        QueryBuilder query = new MatchAllQueryBuilder();
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> hasParentQuery(null, query, false));
+        assertThat(e.getMessage(), equalTo("[has_parent] requires 'type' field"));
+
+        e = expectThrows(IllegalArgumentException.class,
+                () -> hasParentQuery("foo", null, false));
+        assertThat(e.getMessage(), equalTo("[has_parent] requires 'query' field"));
+
+        QueryShardContext context = createShardContext();
+        HasParentQueryBuilder qb = hasParentQuery("just_a_type", new MatchAllQueryBuilder(), false);
+        QueryShardException qse = expectThrows(QueryShardException.class, () -> qb.doToQuery(context));
+        assertThat(qse.getMessage(), equalTo("[has_parent] no child types found for type [just_a_type]"));
+    }
+
+    public void testDeprecatedXContent() throws IOException {
+        XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint();
+        builder.startObject();
+        builder.startObject("has_parent");
+        builder.field("query");
+        new TermQueryBuilder("a", "a").toXContent(builder, ToXContent.EMPTY_PARAMS);
+        builder.field("type", "foo"); // deprecated
+        builder.endObject();
+        builder.endObject();
+        HasParentQueryBuilder queryBuilder = (HasParentQueryBuilder) parseQuery(builder.string());
+        assertEquals("foo", queryBuilder.type());
+        assertWarnings("Deprecated field [type] used, expected [parent_type] instead");
+    }
+
+    public void testToQueryInnerQueryType() throws IOException {
+        String[] searchTypes = new String[]{CHILD_TYPE};
+        QueryShardContext shardContext = createShardContext();
+        shardContext.setTypes(searchTypes);
+        HasParentQueryBuilder hasParentQueryBuilder = new HasParentQueryBuilder(PARENT_TYPE, new IdsQueryBuilder().addIds("id"),
+                false);
+        Query query = hasParentQueryBuilder.toQuery(shardContext);
+        //verify that the context types are still the same as the ones we previously set
+        assertThat(shardContext.getTypes(), equalTo(searchTypes));
+        LegacyHasChildQueryBuilderTests.assertLateParsingQuery(query, PARENT_TYPE, "id");
+    }
+
+    @Override
+    public void testMustRewrite() throws IOException {
+        try {
+            super.testMustRewrite();
+        } catch (UnsupportedOperationException e) {
+            if (requiresRewrite == false) {
+                throw e;
+            }
+        }
+    }
+
+    public void testFromJson() throws IOException {
+        String json =
+                "{\n" +
+                "  \"has_parent\" : {\n" +
+                "    \"query\" : {\n" +
+                "      \"term\" : {\n" +
+                "        \"tag\" : {\n" +
+                "          \"value\" : \"something\",\n" +
+                "          \"boost\" : 1.0\n" +
+                "        }\n" +
+                "      }\n" +
+                "    },\n" +
+                "    \"parent_type\" : \"blog\",\n" +
+                "    \"score\" : true,\n" +
+                "    \"ignore_unmapped\" : false,\n" +
+                "    \"boost\" : 1.0\n" +
+                "  }\n" +
+                "}";
+        HasParentQueryBuilder parsed = (HasParentQueryBuilder) parseQuery(json);
+        checkGeneratedJson(json, parsed);
+        assertEquals(json, "blog", parsed.type());
+        assertEquals(json, "something", ((TermQueryBuilder) parsed.query()).value());
+    }
+
+    public void testIgnoreUnmapped() throws IOException {
+        final HasParentQueryBuilder queryBuilder = new HasParentQueryBuilder("unmapped", new MatchAllQueryBuilder(), false);
+        queryBuilder.ignoreUnmapped(true);
+        Query query = queryBuilder.toQuery(createShardContext());
+        assertThat(query, notNullValue());
+        assertThat(query, instanceOf(MatchNoDocsQuery.class));
+
+        final HasParentQueryBuilder failingQueryBuilder = new HasParentQueryBuilder("unmapped", new MatchAllQueryBuilder(), false);
+        failingQueryBuilder.ignoreUnmapped(false);
+        QueryShardException e = expectThrows(QueryShardException.class, () -> failingQueryBuilder.toQuery(createShardContext()));
+        assertThat(e.getMessage(),
+                    containsString("[" + HasParentQueryBuilder.NAME + "] query configured 'parent_type' [unmapped] is not a valid type"));
+    }
+
+    public void testIgnoreUnmappedWithRewrite() throws IOException {
+        // WrapperQueryBuilder makes sure we always rewrite
+        final HasParentQueryBuilder queryBuilder =
+            new HasParentQueryBuilder("unmapped", new WrapperQueryBuilder(new MatchAllQueryBuilder().toString()), false);
+        queryBuilder.ignoreUnmapped(true);
+        QueryShardContext queryShardContext = createShardContext();
+        Query query = queryBuilder.rewrite(queryShardContext).toQuery(queryShardContext);
+        assertThat(query, notNullValue());
+        assertThat(query, instanceOf(MatchNoDocsQuery.class));
+    }
+}
diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/query/LegacyInnerHitsIT.java b/modules/parent-join/src/test/java/org/elasticsearch/join/query/LegacyInnerHitsIT.java
new file mode 100644
index 0000000000000..7dff631837393
--- /dev/null
+++ b/modules/parent-join/src/test/java/org/elasticsearch/join/query/LegacyInnerHitsIT.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.join.query;
+
+public class LegacyInnerHitsIT extends InnerHitsIT {
+
+    @Override
+    protected boolean legacy() {
+        return true;
+    }
+
+}
diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/query/LegacyParentIdQueryBuilderTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/query/LegacyParentIdQueryBuilderTests.java
new file mode 100644
index 0000000000000..b58d541f2542d
--- /dev/null
+++ b/modules/parent-join/src/test/java/org/elasticsearch/join/query/LegacyParentIdQueryBuilderTests.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.join.query;
+
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.DocValuesTermsQuery;
+import org.apache.lucene.search.MatchNoDocsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
+import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
+import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.TypeFieldMapper;
+import org.elasticsearch.index.query.QueryShardException;
+import org.elasticsearch.join.ParentJoinPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.test.AbstractQueryTestCase;
+import org.hamcrest.Matchers;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.hamcrest.CoreMatchers.notNullValue;
+
+public class LegacyParentIdQueryBuilderTests extends AbstractQueryTestCase<ParentIdQueryBuilder> {
+
+    protected static final String PARENT_TYPE = "parent";
+    protected static final String CHILD_TYPE = "child";
+
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return Collections.singletonList(ParentJoinPlugin.class);
+    }
+
+    @Override
+    protected Settings indexSettings() {
+        return Settings.builder()
+            .put(super.indexSettings())
+            .build();
+    }
+
+    @Override
+    protected void initializeAdditionalMappings(MapperService mapperService) throws IOException {
+        mapperService.merge(PARENT_TYPE, new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef(PARENT_TYPE,
+                STRING_FIELD_NAME, "type=text",
+                INT_FIELD_NAME, "type=integer",
+                DOUBLE_FIELD_NAME, "type=double",
+                BOOLEAN_FIELD_NAME, "type=boolean",
+                DATE_FIELD_NAME, "type=date",
+                OBJECT_FIELD_NAME, "type=object"
+        ).string()), MapperService.MergeReason.MAPPING_UPDATE, false);
+        mapperService.merge(CHILD_TYPE, new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef(CHILD_TYPE,
+                "_parent", "type=" + PARENT_TYPE,
+                STRING_FIELD_NAME, "type=text",
+                INT_FIELD_NAME, "type=integer",
+                DOUBLE_FIELD_NAME, "type=double",
+                BOOLEAN_FIELD_NAME, "type=boolean",
+                DATE_FIELD_NAME, "type=date",
+                OBJECT_FIELD_NAME, "type=object"
+        ).string()), MapperService.MergeReason.MAPPING_UPDATE, false);
+    }
+
+    @Override
+    protected ParentIdQueryBuilder doCreateTestQueryBuilder() {
+        return new ParentIdQueryBuilder(CHILD_TYPE, randomAlphaOfLength(4)).ignoreUnmapped(randomBoolean());
+    }
+
+    @Override
+    protected void doAssertLuceneQuery(ParentIdQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
+        assertThat(query, Matchers.instanceOf(BooleanQuery.class));
+        BooleanQuery booleanQuery = (BooleanQuery) query;
+        assertThat(booleanQuery.clauses().size(), Matchers.equalTo(2));
+        DocValuesTermsQuery idQuery = (DocValuesTermsQuery) booleanQuery.clauses().get(0).getQuery();
+        // there are no getters to get the field and terms on DocValuesTermsQuery, so lets validate by creating a
+        // new query based on the builder:
+        assertThat(idQuery, Matchers.equalTo(new DocValuesTermsQuery("_parent#" + PARENT_TYPE, queryBuilder.getId())));
+
+        TermQuery typeQuery = (TermQuery) booleanQuery.clauses().get(1).getQuery();
+        assertThat(typeQuery.getTerm().field(), Matchers.equalTo(TypeFieldMapper.NAME));
+        assertThat(typeQuery.getTerm().text(), Matchers.equalTo(queryBuilder.getType()));
+    }
+
+    public void testIgnoreUnmapped() throws IOException {
+        final ParentIdQueryBuilder queryBuilder = new ParentIdQueryBuilder("unmapped", "foo");
+        queryBuilder.ignoreUnmapped(true);
+        Query query = queryBuilder.toQuery(createShardContext());
+        assertThat(query, notNullValue());
+        assertThat(query, instanceOf(MatchNoDocsQuery.class));
+
+        final ParentIdQueryBuilder failingQueryBuilder = new ParentIdQueryBuilder("unmapped", "foo");
+        failingQueryBuilder.ignoreUnmapped(false);
+        QueryShardException e = expectThrows(QueryShardException.class, () -> failingQueryBuilder.toQuery(createShardContext()));
+        assertThat(e.getMessage(), containsString("[" + ParentIdQueryBuilder.NAME + "] no mapping found for type [unmapped]"));
+    }
+
+}
diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/query/ParentChildTestCase.java b/modules/parent-join/src/test/java/org/elasticsearch/join/query/ParentChildTestCase.java
new file mode 100644
index 0000000000000..709612004c369
--- /dev/null
+++ b/modules/parent-join/src/test/java/org/elasticsearch/join/query/ParentChildTestCase.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.join.query;
+
+import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.index.IndexModule;
+import org.elasticsearch.join.ParentJoinPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESIntegTestCase;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE)
+public abstract class ParentChildTestCase extends ESIntegTestCase {
+
+    @Override
+    protected boolean ignoreExternalCluster() {
+        return true;
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Collections.singleton(ParentJoinPlugin.class);
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> transportClientPlugins() {
+        return nodePlugins();
+    }
+
+    @Override
+    public Settings indexSettings() {
+        Settings.Builder builder =  Settings.builder().put(super.indexSettings())
+            // aggressive filter caching so that we can assert on the filter cache size
+            .put(IndexModule.INDEX_QUERY_CACHE_ENABLED_SETTING.getKey(), true)
+            .put(IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING.getKey(), true);
+
+        if (legacy() == false) {
+            builder.put("index.mapping.single_type", true);
+        }
+
+        return builder.build();
+    }
+
+    protected boolean legacy() {
+        return false;
+    }
+
+    protected IndexRequestBuilder createIndexRequest(String index, String type, String id, String parentId, Object... fields) {
+        Map<String, Object> source = new HashMap<>();
+        for (int i = 0; i < fields.length; i += 2) {
+            source.put((String) fields[i], fields[i + 1]);
+        }
+        return createIndexRequest(index, type, id, parentId, source);
+    }
+
+    protected IndexRequestBuilder createIndexRequest(String index, String type, String id, String parentId,
+                                                   XContentBuilder builder) throws IOException {
+        Map<String, Object> source = XContentHelper.convertToMap(JsonXContent.jsonXContent, builder.string(), false);
+        return createIndexRequest(index, type, id, parentId, source);
+    }
+
+    public static Map<String, Object> buildParentJoinFieldMappingFromSimplifiedDef(String joinFieldName,
+                                                                                   boolean eagerGlobalOrdinals,
+                                                                                   String... relations) {
+        Map<String, Object> fields = new HashMap<>();
+
+        Map<String, Object> joinField = new HashMap<>();
+        joinField.put("type", "join");
+        joinField.put("eager_global_ordinals", eagerGlobalOrdinals);
+        Map<String, Object> relationMap = new HashMap<>();
+        for (int i = 0; i < relations.length; i+=2) {
+            String[] children = relations[i+1].split(",");
+            if (children.length > 1) {
+                relationMap.put(relations[i], children);
+            } else {
+                relationMap.put(relations[i], children[0]);
+            }
+        }
+        joinField.put("relations", relationMap);
+        fields.put(joinFieldName, joinField);
+        return Collections.singletonMap("properties", fields);
+    }
+
+    @SuppressWarnings("unchecked")
+    public static Map<String, Object> addFieldMappings(Map<String, Object> map, String... fields) {
+        Map<String, Object> propsMap = (Map<String, Object>) map.get("properties");
+        for (int i = 0; i < fields.length; i+=2) {
+            String field = fields[i];
+            String type = fields[i + 1];
+            propsMap.put(field, Collections.singletonMap("type", type));
+        }
+        return map;
+    }
+
+    private IndexRequestBuilder createIndexRequest(String index, String type, String id, String parentId, Map<String, Object> source) {
+        String name = type;
+        if (legacy() == false) {
+            type = "doc";
+        }
+
+        IndexRequestBuilder indexRequestBuilder = client().prepareIndex(index, type, id);
+        if (legacy()) {
+            if (parentId != null) {
+                indexRequestBuilder.setParent(parentId);
+            }
+            indexRequestBuilder.setSource(source);
+        } else {
+            Map<String, Object> joinField = new HashMap<>();
+            if (parentId != null) {
+                joinField.put("name", name);
+                joinField.put("parent", parentId);
+                indexRequestBuilder.setRouting(parentId);
+            } else {
+                joinField.put("name", name);
+            }
+            source.put("join_field", joinField);
+            indexRequestBuilder.setSource(source);
+        }
+        return indexRequestBuilder;
+    }
+
+}
diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/query/ParentIdQueryBuilderTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/query/ParentIdQueryBuilderTests.java
new file mode 100644
index 0000000000000..75f6f8a6bdebc
--- /dev/null
+++ b/modules/parent-join/src/test/java/org/elasticsearch/join/query/ParentIdQueryBuilderTests.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.join.query;
+
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.DocValuesTermsQuery;
+import org.apache.lucene.search.MatchNoDocsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.TypeFieldMapper;
+import org.elasticsearch.index.query.QueryShardException;
+import org.elasticsearch.join.ParentJoinPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.test.AbstractQueryTestCase;
+import org.hamcrest.Matchers;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.hamcrest.CoreMatchers.notNullValue;
+
+public class ParentIdQueryBuilderTests extends AbstractQueryTestCase<ParentIdQueryBuilder> {
+
+    private static final String TYPE = "doc";
+    private static final String JOIN_FIELD_NAME = "join_field";
+    private static final String PARENT_NAME = "parent";
+    private static final String CHILD_NAME = "child";
+
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return Collections.singletonList(ParentJoinPlugin.class);
+    }
+
+    @Override
+    protected Settings indexSettings() {
+        return Settings.builder()
+            .put(super.indexSettings())
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+            .put("index.mapping.single_type", true)
+            .build();
+    }
+
+    @Override
+    protected void initializeAdditionalMappings(MapperService mapperService) throws IOException {
+        XContentBuilder mapping = jsonBuilder().startObject().startObject("doc").startObject("properties")
+            .startObject("join_field")
+                .field("type", "join")
+                .startObject("relations")
+                    .field("parent", "child")
+                .endObject()
+            .endObject()
+            .startObject(STRING_FIELD_NAME)
+                .field("type", "text")
+            .endObject()
+            .startObject(STRING_FIELD_NAME_2)
+                .field("type", "keyword")
+            .endObject()
+            .startObject(INT_FIELD_NAME)
+                .field("type", "integer")
+            .endObject()
+            .startObject(DOUBLE_FIELD_NAME)
+                .field("type", "double")
+            .endObject()
+            .startObject(BOOLEAN_FIELD_NAME)
+                .field("type", "boolean")
+            .endObject()
+            .startObject(DATE_FIELD_NAME)
+                .field("type", "date")
+            .endObject()
+            .startObject(OBJECT_FIELD_NAME)
+                .field("type", "object")
+            .endObject()
+            .endObject().endObject().endObject();
+
+        mapperService.merge(TYPE,
+            new CompressedXContent(mapping.string()), MapperService.MergeReason.MAPPING_UPDATE, false);
+    }
+
+    @Override
+    protected ParentIdQueryBuilder doCreateTestQueryBuilder() {
+        return new ParentIdQueryBuilder(CHILD_NAME, randomAlphaOfLength(4)).ignoreUnmapped(randomBoolean());
+    }
+
+    @Override
+    protected void doAssertLuceneQuery(ParentIdQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
+        assertThat(query, Matchers.instanceOf(BooleanQuery.class));
+        BooleanQuery booleanQuery = (BooleanQuery) query;
+        assertThat(booleanQuery.clauses().size(), Matchers.equalTo(2));
+        BooleanQuery expected = new BooleanQuery.Builder()
+            .add(new TermQuery(new Term(JOIN_FIELD_NAME + "#" + PARENT_NAME, queryBuilder.getId())), BooleanClause.Occur.MUST)
+            .add(new TermQuery(new Term(JOIN_FIELD_NAME, queryBuilder.getType())), BooleanClause.Occur.FILTER)
+            .build();
+        assertThat(expected, equalTo(query));
+    }
+
+    public void testFromJson() throws IOException {
+        String query =
+            "{\n" +
+                "  \"parent_id\" : {\n" +
+                "    \"type\" : \"child\",\n" +
+                "    \"id\" : \"123\",\n" +
+                "    \"ignore_unmapped\" : false,\n" +
+                "    \"boost\" : 3.0,\n" +
+                "    \"_name\" : \"name\"" +
+                "  }\n" +
+                "}";
+        ParentIdQueryBuilder queryBuilder = (ParentIdQueryBuilder) parseQuery(query);
+        checkGeneratedJson(query, queryBuilder);
+        assertThat(queryBuilder.getType(), Matchers.equalTo("child"));
+        assertThat(queryBuilder.getId(), Matchers.equalTo("123"));
+        assertThat(queryBuilder.boost(), Matchers.equalTo(3f));
+        assertThat(queryBuilder.queryName(), Matchers.equalTo("name"));
+    }
+
+    public void testIgnoreUnmapped() throws IOException {
+        final ParentIdQueryBuilder queryBuilder = new ParentIdQueryBuilder("unmapped", "foo");
+        queryBuilder.ignoreUnmapped(true);
+        Query query = queryBuilder.toQuery(createShardContext());
+        assertThat(query, notNullValue());
+        assertThat(query, instanceOf(MatchNoDocsQuery.class));
+
+        final ParentIdQueryBuilder failingQueryBuilder = new ParentIdQueryBuilder("unmapped", "foo");
+        failingQueryBuilder.ignoreUnmapped(false);
+        QueryShardException e = expectThrows(QueryShardException.class, () -> failingQueryBuilder.toQuery(createShardContext()));
+        assertThat(e.getMessage(), containsString("[" + ParentIdQueryBuilder.NAME + "] no relation found for child [unmapped]"));
+    }
+
+}
diff --git a/modules/parent-join/src/test/resources/rest-api-spec/test/10_parent_child.yml b/modules/parent-join/src/test/resources/rest-api-spec/test/10_parent_child.yml
new file mode 100644
index 0000000000000..19e278cd82b44
--- /dev/null
+++ b/modules/parent-join/src/test/resources/rest-api-spec/test/10_parent_child.yml
@@ -0,0 +1,42 @@
+setup:
+  - do:
+      indices.create:
+        index: test
+        body:
+            mappings:
+              type_2: {}
+              type_3:
+                _parent:
+                  type: type_2
+
+---
+"Parent/child inner hits":
+    - do:
+        index:
+          index: test
+          type:  type_2
+          id:    1
+          body:  {"foo": "bar"}
+
+    - do:
+        index:
+          index: test
+          type:  type_3
+          id:    1
+          parent: 1
+          body:  {"bar": "baz"}
+
+    - do:
+        indices.refresh: {}
+
+    - do:
+        search:
+          body: { "query" : { "has_child" : { "type" : "type_3", "query" : { "match_all" : {} }, "inner_hits" : {} } } }
+    - match: { hits.total: 1 }
+    - match: { hits.hits.0._index: "test" }
+    - match: { hits.hits.0._type: "type_2" }
+    - match: { hits.hits.0._id: "1" }
+    - is_false: hits.hits.0.inner_hits.type_3.hits.hits.0._index
+    - match: { hits.hits.0.inner_hits.type_3.hits.hits.0._type: "type_3" }
+    - match: { hits.hits.0.inner_hits.type_3.hits.hits.0._id: "1" }
+    - is_false: hits.hits.0.inner_hits.type_3.hits.hits.0._nested
diff --git a/modules/parent-join/src/test/resources/rest-api-spec/test/11_parent_child.yml b/modules/parent-join/src/test/resources/rest-api-spec/test/11_parent_child.yml
new file mode 100644
index 0000000000000..bb5e4a4bdce06
--- /dev/null
+++ b/modules/parent-join/src/test/resources/rest-api-spec/test/11_parent_child.yml
@@ -0,0 +1,48 @@
+setup:
+  - do:
+      indices.create:
+        index: test
+        body:
+            settings:
+              mapping.single_type: true
+            mappings:
+              doc:
+                properties:
+                  join_field:
+                    type: join
+                    relations:
+                      parent: child
+
+---
+"Parent/child inner hits":
+    - skip:
+        version: " - 5.5.99"
+        reason:  parent-join was added in 5.6
+
+    - do:
+        index:
+          index: test
+          type:  doc
+          id:    1
+          body:  {"foo": "bar", "join_field": {"name" : "parent"} }
+
+    - do:
+        index:
+          index: test
+          type:  doc
+          id:    2
+          routing: 1
+          body:  {"bar": "baz", "join_field": { "name" : "child", "parent": "1"} }
+
+    - do:
+        indices.refresh: {}
+
+    - do:
+        search:
+          body: { "query" : { "has_child" : { "type" : "child", "query" : { "match_all" : {} }, "inner_hits" : {} } } }
+    - match: { hits.total: 1 }
+    - match: { hits.hits.0._index: "test" }
+    - match: { hits.hits.0._id: "1" }
+    - is_false: hits.hits.0.inner_hits.child.hits.hits.0._index
+    - match: { hits.hits.0.inner_hits.child.hits.hits.0._id: "2" }
+    - is_false: hits.hits.0.inner_hits.child.hits.hits.0._nested
diff --git a/modules/parent-join/src/test/resources/rest-api-spec/test/20_parent_join.yml b/modules/parent-join/src/test/resources/rest-api-spec/test/20_parent_join.yml
new file mode 100644
index 0000000000000..cb660fe7b794b
--- /dev/null
+++ b/modules/parent-join/src/test/resources/rest-api-spec/test/20_parent_join.yml
@@ -0,0 +1,134 @@
+setup:
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            mapping.single_type: true
+          mappings:
+            doc:
+              properties:
+                join_field: { "type": "join", "relations": { "parent": "child", "child": "grand_child" } }
+
+  - do:
+      index:
+        index: test
+        type:  doc
+        id:    1
+        body:  { "join_field": { "name": "parent" } }
+
+  - do:
+      index:
+        index: test
+        type:  doc
+        id:    2
+        body:  { "join_field": { "name": "parent" } }
+
+  - do:
+      index:
+        index: test
+        type:  doc
+        id:    3
+        routing: 1
+        body:  { "join_field": { "name": "child", "parent": "1" } }
+
+  - do:
+      index:
+        index: test
+        type:  doc
+        id:    4
+        routing: 1
+        body:  { "join_field": { "name": "child", "parent": "1" } }
+
+  - do:
+      index:
+        index: test
+        type:  doc
+        id:    5
+        routing: 1
+        body:  { "join_field": { "name": "child", "parent": "2" } }
+
+  - do:
+      index:
+        index: test
+        type:  doc
+        id:    6
+        routing: 1
+        body:  { "join_field": { "name": "grand_child", "parent": "5" } }
+
+  - do:
+      indices.refresh: {}
+
+---
+"Test basic":
+    - skip:
+        version: " - 5.5.99"
+        reason:  parent-join was added in 5.6
+
+    - do:
+        search:
+          body: { sort: ["join_field", "_id"] }
+
+    - match: { hits.total: 6 }
+    - match: { hits.hits.0._index: "test" }
+    - match: { hits.hits.0._type: "doc" }
+    - match: { hits.hits.0._id: "3" }
+    - match: { hits.hits.0._source.join_field.name: "child" }
+    - match: { hits.hits.0._source.join_field.parent: "1" }
+    - is_false: hits.hits.0.fields.join_field#child }
+    - match: { hits.hits.1._index: "test" }
+    - match: { hits.hits.1._type: "doc" }
+    - match: { hits.hits.1._id: "4" }
+    - match: { hits.hits.1._source.join_field.name: "child" }
+    - match: { hits.hits.1._source.join_field.parent: "1" }
+    - is_false: hits.hits.1.fields.join_field#child }
+    - match: { hits.hits.2._index: "test" }
+    - match: { hits.hits.2._type: "doc" }
+    - match: { hits.hits.2._id: "5" }
+    - match: { hits.hits.2._source.join_field.name: "child" }
+    - match: { hits.hits.2._source.join_field.parent: "2" }
+    - is_false: hits.hits.2.fields.join_field#child }
+    - match: { hits.hits.3._index: "test" }
+    - match: { hits.hits.3._type: "doc" }
+    - match: { hits.hits.3._id: "6" }
+    - match: { hits.hits.3._source.join_field.name: "grand_child" }
+    - match: { hits.hits.3._source.join_field.parent: "5" }
+    - match: { hits.hits.4._index: "test" }
+    - match: { hits.hits.4._type: "doc" }
+    - match: { hits.hits.4._id: "1" }
+    - match: { hits.hits.4._source.join_field.name: "parent" }
+    - is_false: hits.hits.4._source.join_field.parent
+    - match: { hits.hits.5._index: "test" }
+    - match: { hits.hits.5._type: "doc" }
+    - match: { hits.hits.5._id: "2" }
+    - match: { hits.hits.5._source.join_field.name: "parent" }
+    - is_false: hits.hits.5._source.join_field.parent
+
+---
+"Test parent_id query":
+    - skip:
+        version: " - 5.5.99"
+        reason:  parent-join was added in 5.6.
+
+    - do:
+        search:
+          body:
+            sort: [ "_id" ]
+            query:
+              parent_id:
+                type: child
+                id: 1
+
+    - match: { hits.total: 2 }
+    - match: { hits.hits.0._index: "test" }
+    - match: { hits.hits.0._type: "doc" }
+    - match: { hits.hits.0._id: "3" }
+    - match: { hits.hits.0._source.join_field.name: "child" }
+    - match: { hits.hits.0._source.join_field.parent: "1" }
+    - match: { hits.hits.1._index: "test" }
+    - match: { hits.hits.1._type: "doc" }
+    - match: { hits.hits.1._id: "4" }
+    - match: { hits.hits.1._source.join_field.name: "child" }
+    - match: { hits.hits.1._source.join_field.parent: "1" }
+
+
diff --git a/modules/percolator/build.gradle b/modules/percolator/build.gradle
index 60fb82bdf4e32..cf55368861aef 100644
--- a/modules/percolator/build.gradle
+++ b/modules/percolator/build.gradle
@@ -23,5 +23,9 @@ esplugin {
     hasClientJar = true
 }
 
+dependencies {
+    // for testing hasChild and hasParent rejections
+    testCompile project(path: ':modules:parent-join', configuration: 'runtime')
+}
 compileJava.options.compilerArgs << "-Xlint:-deprecation,-rawtypes"
 compileTestJava.options.compilerArgs << "-Xlint:-deprecation,-rawtypes"
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateRequest.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateRequest.java
index a18c12bb76978..20fa14fb6784e 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateRequest.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateRequest.java
@@ -22,16 +22,17 @@
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.CompositeIndicesRequest;
-import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -48,7 +49,7 @@
  * @deprecated Instead use multi search API with {@link PercolateQueryBuilder}
  */
 @Deprecated
-public class MultiPercolateRequest extends ActionRequest<MultiPercolateRequest> implements CompositeIndicesRequest {
+public class MultiPercolateRequest extends ActionRequest implements CompositeIndicesRequest {
 
     private String[] indices;
     private String documentType;
@@ -89,7 +90,7 @@ public MultiPercolateRequest add(byte[] data, int from, int length) throws Excep
     /**
      * Embeds a percolate request which request body is defined as raw bytes to this multi percolate request
      */
-    public MultiPercolateRequest add(BytesReference data, boolean allowExplicitIndex) throws Exception {
+    public MultiPercolateRequest add(BytesReference data, boolean allowExplicitIndex) throws IOException {
         XContent xContent = XContentFactory.xContent(data);
         int from = 0;
         int length = data.length();
@@ -118,7 +119,8 @@ public MultiPercolateRequest add(BytesReference data, boolean allowExplicitIndex
 
             // now parse the action
             if (nextMarker - from > 0) {
-                try (XContentParser parser = xContent.createParser(data.slice(from, nextMarker - from))) {
+                // EMPTY is safe here because we don't call namedObject
+                try (XContentParser parser = xContent.createParser(NamedXContentRegistry.EMPTY, data.slice(from, nextMarker - from))) {
                     // Move to START_OBJECT, if token is null, its an empty data
                     XContentParser.Token token = parser.nextToken();
                     if (token != null) {
@@ -154,7 +156,13 @@ public MultiPercolateRequest add(BytesReference data, boolean allowExplicitIndex
                 break;
             }
 
-            percolateRequest.source(data.slice(from, nextMarker - from));
+            final BytesReference sliced;
+            if (XContentType.JSON == xContent.type() && data.get(nextMarker - 1) == (byte) '\r') {
+                sliced = data.slice(from, nextMarker - from - 1);
+            } else {
+                sliced = data.slice(from, nextMarker - from);
+            }
+            percolateRequest.source(sliced);
             // move pointers
             from = nextMarker + 1;
 
@@ -164,11 +172,6 @@ public MultiPercolateRequest add(BytesReference data, boolean allowExplicitIndex
         return this;
     }
 
-    @Override
-    public List<? extends IndicesRequest> subRequests() {
-        return requests;
-    }
-
     private void parsePercolateAction(XContentParser parser, PercolateRequest percolateRequest, boolean allowExplicitIndex) throws IOException {
         String globalIndex = indices != null && indices.length > 0 ? indices[0] : null;
 
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateResponse.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateResponse.java
index afc5b7ab6c745..0992d8574d5db 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateResponse.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateResponse.java
@@ -24,7 +24,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
 import java.io.IOException;
@@ -40,7 +40,7 @@
  * @deprecated Instead use multi search API with {@link PercolateQueryBuilder}
  */
 @Deprecated
-public class MultiPercolateResponse extends ActionResponse implements Iterable<MultiPercolateResponse.Item>, ToXContent {
+public class MultiPercolateResponse extends ActionResponse implements Iterable<MultiPercolateResponse.Item>, ToXContentObject {
 
     private Item[] items;
 
@@ -73,17 +73,19 @@ public Item[] getItems() {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
         builder.startArray(Fields.RESPONSES);
         for (MultiPercolateResponse.Item item : items) {
-            builder.startObject();
             if (item.isFailure()) {
-                ElasticsearchException.renderException(builder, params, item.getFailure());
+                builder.startObject();
+                ElasticsearchException.generateFailureXContent(builder, params, item.getFailure(), true);
+                builder.endObject();
             } else {
                 item.getResponse().toXContent(builder, params);
             }
-            builder.endObject();
         }
         builder.endArray();
+        builder.endObject();
         return builder;
     }
 
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java
index 40218e50a4f57..408de68e691b4 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java
@@ -27,11 +27,13 @@
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.ScorerSupplier;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.search.TwoPhaseIterator;
 import org.apache.lucene.search.Weight;
 import org.apache.lucene.util.Accountable;
 import org.apache.lucene.util.Bits;
+import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.lucene.Lucene;
 
@@ -90,8 +92,8 @@ public Explanation explain(LeafReaderContext leafReaderContext, int docId) throw
                     if (result == docId) {
                         if (twoPhaseIterator.matches()) {
                             if (needsScores) {
-                                QueryStore.Leaf percolatorQueries = queryStore.getQueries(leafReaderContext);
-                                Query query = percolatorQueries.getQuery(docId);
+                                CheckedFunction<Integer, Query, IOException> percolatorQueries = queryStore.getQueries(leafReaderContext);
+                                Query query = percolatorQueries.apply(docId);
                                 Explanation detail = percolatorIndexSearcher.explain(query, 0);
                                 return Explanation.match(scorer.score(), "PercolateQuery", detail);
                             } else {
@@ -120,7 +122,7 @@ public Scorer scorer(LeafReaderContext leafReaderContext) throws IOException {
                     return null;
                 }
 
-                final QueryStore.Leaf queries = queryStore.getQueries(leafReaderContext);
+                final CheckedFunction<Integer, Query, IOException> queries = queryStore.getQueries(leafReaderContext);
                 if (needsScores) {
                     return new BaseScorer(this, approximation, queries, percolatorIndexSearcher) {
 
@@ -128,7 +130,7 @@ public Scorer scorer(LeafReaderContext leafReaderContext) throws IOException {
 
                         @Override
                         boolean matchDocId(int docId) throws IOException {
-                            Query query = percolatorQueries.getQuery(docId);
+                            Query query = percolatorQueries.apply(docId);
                             if (query != null) {
                                 TopDocs topDocs = percolatorIndexSearcher.search(query, 1);
                                 if (topDocs.totalHits > 0) {
@@ -148,7 +150,7 @@ public float score() throws IOException {
                         }
                     };
                 } else {
-                    Scorer verifiedDocsScorer = verifiedMatchesWeight.scorer(leafReaderContext);
+                    ScorerSupplier verifiedDocsScorer = verifiedMatchesWeight.scorerSupplier(leafReaderContext);
                     Bits verifiedDocsBits = Lucene.asSequentialAccessBits(leafReaderContext.reader().maxDoc(), verifiedDocsScorer);
                     return new BaseScorer(this, approximation, queries, percolatorIndexSearcher) {
 
@@ -166,7 +168,7 @@ boolean matchDocId(int docId) throws IOException {
                             if (verifiedDocsBits.get(docId)) {
                                 return true;
                             }
-                            Query query = percolatorQueries.getQuery(docId);
+                            Query query = percolatorQueries.apply(docId);
                             return query != null && Lucene.exists(percolatorIndexSearcher, query);
                         }
                     };
@@ -191,25 +193,19 @@ public QueryStore getQueryStore() {
         return queryStore;
     }
 
+    // Comparing identity here to avoid being cached
+    // Note that in theory if the same instance gets used multiple times it could still get cached,
+    // however since we create a new query instance each time we this query this shouldn't happen and thus
+    // this risk neglectable.
     @Override
     public boolean equals(Object o) {
-        if (this == o) return true;
-        if (o == null || getClass() != o.getClass()) return false;
-        if (sameClassAs(o) == false) return false;
-
-        PercolateQuery that = (PercolateQuery) o;
-
-        if (!documentType.equals(that.documentType)) return false;
-        return documentSource.equals(that.documentSource);
-
+        return this == o;
     }
 
+    // Computing hashcode based on identity to avoid caching.
     @Override
     public int hashCode() {
-        int result = classHash();
-        result = 31 * result + documentType.hashCode();
-        result = 31 * result + documentSource.hashCode();
-        return result;
+        return System.identityHashCode(this);
     }
 
     @Override
@@ -224,26 +220,18 @@ public long ramBytesUsed() {
     }
 
     @FunctionalInterface
-    public interface QueryStore {
-
-        Leaf getQueries(LeafReaderContext ctx) throws IOException;
-
-        @FunctionalInterface
-        interface Leaf {
-
-            Query getQuery(int docId) throws IOException;
-
-        }
-
+    interface QueryStore {
+        CheckedFunction<Integer, Query, IOException> getQueries(LeafReaderContext ctx) throws IOException;
     }
 
     abstract static class BaseScorer extends Scorer {
 
         final Scorer approximation;
-        final QueryStore.Leaf percolatorQueries;
+        final CheckedFunction<Integer, Query, IOException> percolatorQueries;
         final IndexSearcher percolatorIndexSearcher;
 
-        BaseScorer(Weight weight, Scorer approximation, QueryStore.Leaf percolatorQueries, IndexSearcher percolatorIndexSearcher) {
+        BaseScorer(Weight weight, Scorer approximation, CheckedFunction<Integer, Query, IOException> percolatorQueries,
+                   IndexSearcher percolatorIndexSearcher) {
             super(weight);
             this.approximation = approximation;
             this.percolatorQueries = percolatorQueries;
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
index 09e8afbca0bfe..27a1a57c40a62 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
@@ -24,19 +24,26 @@
 import org.apache.lucene.index.BinaryDocValues;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.IndexReaderContext;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.ReaderUtil;
 import org.apache.lucene.index.StoredFieldVisitor;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.memory.MemoryIndex;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.Weight;
+import org.apache.lucene.search.join.BitSetProducer;
 import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.util.BitDocIdSet;
+import org.apache.lucene.util.BitSet;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.ElasticsearchException;
@@ -50,7 +57,6 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
 import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -59,6 +65,8 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.analysis.FieldNameAnalyzer;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.DocumentMapperForType;
 import org.elasticsearch.index.mapper.MappedFieldType;
@@ -72,6 +80,8 @@
 import org.elasticsearch.index.query.QueryRewriteContext;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.query.QueryShardException;
+import org.elasticsearch.indices.breaker.CircuitBreakerService;
+import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -96,6 +106,7 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder<PercolateQueryBu
     private final String field;
     private final String documentType;
     private final BytesReference document;
+    private final XContentType documentXContentType;
 
     private final String indexedDocumentIndex;
     private final String indexedDocumentType;
@@ -104,7 +115,16 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder<PercolateQueryBu
     private final String indexedDocumentPreference;
     private final Long indexedDocumentVersion;
 
+    /**
+     * @deprecated use {@link #PercolateQueryBuilder(String, String, BytesReference, XContentType)} with the document content type to avoid
+     * autodetection
+     */
+    @Deprecated
     public PercolateQueryBuilder(String field, String documentType, BytesReference document) {
+        this(field, documentType, document, XContentFactory.xContentType(document));
+    }
+
+    public PercolateQueryBuilder(String field, String documentType, BytesReference document, XContentType documentXContentType) {
         if (field == null) {
             throw new IllegalArgumentException("[field] is a required argument");
         }
@@ -117,6 +137,7 @@ public PercolateQueryBuilder(String field, String documentType, BytesReference d
         this.field = field;
         this.documentType = documentType;
         this.document = document;
+        this.documentXContentType = Objects.requireNonNull(documentXContentType);
         indexedDocumentIndex = null;
         indexedDocumentType = null;
         indexedDocumentId = null;
@@ -152,6 +173,7 @@ public PercolateQueryBuilder(String field, String documentType, String indexedDo
         this.indexedDocumentPreference = indexedDocumentPreference;
         this.indexedDocumentVersion = indexedDocumentVersion;
         this.document = null;
+        this.documentXContentType = null;
     }
 
     /**
@@ -172,6 +194,15 @@ public PercolateQueryBuilder(String field, String documentType, String indexedDo
             indexedDocumentVersion = null;
         }
         document = in.readOptionalBytesReference();
+        if (document != null) {
+            if (in.getVersion().onOrAfter(Version.V_5_3_0)) {
+                documentXContentType = XContentType.readFrom(in);
+            } else {
+                documentXContentType = XContentFactory.xContentType(document);
+            }
+        } else {
+            documentXContentType = null;
+        }
     }
 
     @Override
@@ -190,6 +221,9 @@ protected void doWriteTo(StreamOutput out) throws IOException {
             out.writeBoolean(false);
         }
         out.writeOptionalBytesReference(document);
+        if (document != null && out.getVersion().onOrAfter(Version.V_5_3_0)) {
+            documentXContentType.writeTo(out);
+        }
     }
 
     @Override
@@ -198,16 +232,7 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep
         builder.field(DOCUMENT_TYPE_FIELD.getPreferredName(), documentType);
         builder.field(QUERY_FIELD.getPreferredName(), field);
         if (document != null) {
-            XContentType contentType = XContentFactory.xContentType(document);
-            if (contentType == builder.contentType()) {
-                builder.rawField(DOCUMENT_FIELD.getPreferredName(), document);
-            } else {
-                try (XContentParser parser = XContentFactory.xContent(contentType).createParser(document)) {
-                    parser.nextToken();
-                    builder.field(DOCUMENT_FIELD.getPreferredName());
-                    builder.copyCurrentStructure(parser);
-                }
-            }
+            builder.rawField(DOCUMENT_FIELD.getPreferredName(), document);
         }
         if (indexedDocumentIndex != null || indexedDocumentType != null || indexedDocumentId != null) {
             if (indexedDocumentIndex != null) {
@@ -257,7 +282,7 @@ public static Optional<PercolateQueryBuilder> fromXContent(QueryParseContext par
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, DOCUMENT_FIELD)) {
+                if (DOCUMENT_FIELD.match(currentFieldName)) {
                     try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
                         builder.copyCurrentStructure(parser);
                         builder.flush();
@@ -268,25 +293,25 @@ public static Optional<PercolateQueryBuilder> fromXContent(QueryParseContext par
                             "] query does not support [" + token + "]");
                 }
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, QUERY_FIELD)) {
+                if (QUERY_FIELD.match(currentFieldName)) {
                     field = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, DOCUMENT_TYPE_FIELD)) {
+                } else if (DOCUMENT_TYPE_FIELD.match(currentFieldName)) {
                     documentType = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, INDEXED_DOCUMENT_FIELD_INDEX)) {
+                } else if (INDEXED_DOCUMENT_FIELD_INDEX.match(currentFieldName)) {
                     indexedDocumentIndex = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, INDEXED_DOCUMENT_FIELD_TYPE)) {
+                } else if (INDEXED_DOCUMENT_FIELD_TYPE.match(currentFieldName)) {
                     indexedDocumentType = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, INDEXED_DOCUMENT_FIELD_ID)) {
+                } else if (INDEXED_DOCUMENT_FIELD_ID.match(currentFieldName)) {
                     indexedDocumentId = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, INDEXED_DOCUMENT_FIELD_ROUTING)) {
+                } else if (INDEXED_DOCUMENT_FIELD_ROUTING.match(currentFieldName)) {
                     indexedDocumentRouting = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, INDEXED_DOCUMENT_FIELD_PREFERENCE)) {
+                } else if (INDEXED_DOCUMENT_FIELD_PREFERENCE.match(currentFieldName)) {
                     indexedDocumentPreference = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, INDEXED_DOCUMENT_FIELD_VERSION)) {
+                } else if (INDEXED_DOCUMENT_FIELD_VERSION.match(currentFieldName)) {
                     indexedDocumentVersion = parser.longValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[" + PercolateQueryBuilder.NAME +
@@ -305,7 +330,7 @@ public static Optional<PercolateQueryBuilder> fromXContent(QueryParseContext par
 
         PercolateQueryBuilder queryBuilder;
         if (source != null) {
-            queryBuilder = new PercolateQueryBuilder(field, documentType, source);
+            queryBuilder = new PercolateQueryBuilder(field, documentType, source, XContentType.JSON);
         } else if (indexedDocumentId != null) {
             queryBuilder = new PercolateQueryBuilder(field, documentType, indexedDocumentIndex, indexedDocumentType,
                     indexedDocumentId, indexedDocumentRouting, indexedDocumentPreference, indexedDocumentVersion);
@@ -361,11 +386,15 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryShardContext) throws I
                 "indexed document [" + indexedDocumentIndex + "/" + indexedDocumentType + "/" + indexedDocumentId + "] source disabled"
             );
         }
-        return new PercolateQueryBuilder(field, documentType, getResponse.getSourceAsBytesRef());
+        final BytesReference source = getResponse.getSourceAsBytesRef();
+        return new PercolateQueryBuilder(field, documentType, source, XContentFactory.xContentType(source));
     }
 
     @Override
     protected Query doToQuery(QueryShardContext context) throws IOException {
+        // Call nowInMillis() so that this query becomes un-cacheable since we
+        // can't be sure that it doesn't use now or scripts
+        context.nowInMillis();
         if (indexedDocumentIndex != null || indexedDocumentType != null || indexedDocumentId != null) {
             throw new IllegalStateException("query builder must be rewritten first");
         }
@@ -378,7 +407,7 @@ protected Query doToQuery(QueryShardContext context) throws IOException {
         DocumentMapperForType docMapperForType = mapperService.documentMapperWithAutoCreate(documentType);
         DocumentMapper docMapper = docMapperForType.getDocumentMapper();
 
-        ParsedDocument doc = docMapper.parse(source(context.index().getName(), documentType, "_temp_id", document));
+        ParsedDocument doc = docMapper.parse(source(context.index().getName(), documentType, "_temp_id", document, documentXContentType));
 
         FieldNameAnalyzer fieldNameAnalyzer = (FieldNameAnalyzer) docMapper.mappers().indexAnalyzer();
         // Need to this custom impl because FieldNameAnalyzer is strict and the percolator sometimes isn't when
@@ -390,7 +419,7 @@ protected Analyzer getWrappedAnalyzer(String fieldName) {
                 if (analyzer != null) {
                     return analyzer;
                 } else {
-                    return context.getAnalysisService().defaultIndexAnalyzer();
+                    return context.getIndexAnalyzers().getDefaultIndexAnalyzer();
                 }
             }
         };
@@ -407,6 +436,7 @@ protected Analyzer getWrappedAnalyzer(String fieldName) {
         Version indexVersionCreated = context.getIndexSettings().getIndexVersionCreated();
         boolean mapUnmappedFieldsAsString = context.getIndexSettings()
                 .getValue(PercolatorFieldMapper.INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING);
+        QueryShardContext percolateShardContext = wrap(context);
         if (indexVersionCreated.onOrAfter(Version.V_5_0_0_alpha1)) {
             MappedFieldType fieldType = context.fieldMapper(field);
             if (fieldType == null) {
@@ -418,11 +448,11 @@ protected Analyzer getWrappedAnalyzer(String fieldName) {
                         "] to be of type [percolator], but is of type [" + fieldType.typeName() + "]");
             }
             PercolatorFieldMapper.FieldType pft = (PercolatorFieldMapper.FieldType) fieldType;
-            PercolateQuery.QueryStore queryStore = createStore(pft, context, mapUnmappedFieldsAsString);
+            PercolateQuery.QueryStore queryStore = createStore(pft, percolateShardContext, mapUnmappedFieldsAsString);
             return pft.percolateQuery(documentType, queryStore, document, docSearcher);
         } else {
             Query percolateTypeQuery = new TermQuery(new Term(TypeFieldMapper.NAME, MapperService.PERCOLATOR_LEGACY_TYPE_NAME));
-            PercolateQuery.QueryStore queryStore = createLegacyStore(context, mapUnmappedFieldsAsString);
+            PercolateQuery.QueryStore queryStore = createLegacyStore(percolateShardContext, mapUnmappedFieldsAsString);
             return new PercolateQuery(documentType, queryStore, document, percolateTypeQuery, docSearcher,
                     new MatchNoDocsQuery("pre 5.0.0-alpha1 index, no verified matches"));
         }
@@ -440,6 +470,11 @@ public BytesReference getDocument() {
         return document;
     }
 
+    //pkg-private for testing
+    XContentType getXContentType() {
+        return documentXContentType;
+    }
+
     static IndexSearcher createMultiDocumentSearcher(Analyzer analyzer, ParsedDocument doc) {
         RAMDirectory ramDirectory = new RAMDirectory();
         try (IndexWriter indexWriter = new IndexWriter(ramDirectory, new IndexWriterConfig(analyzer))) {
@@ -481,7 +516,8 @@ private static PercolateQuery.QueryStore createStore(PercolatorFieldMapper.Field
                     BytesRef qbSource = binaryDocValues.get(docId);
                     if (qbSource.length > 0) {
                         XContent xContent = PercolatorFieldMapper.QUERY_BUILDER_CONTENT_TYPE.xContent();
-                        try (XContentParser sourceParser = xContent.createParser(qbSource.bytes, qbSource.offset, qbSource.length)) {
+                        try (XContentParser sourceParser = xContent.createParser(context.getXContentRegistry(), qbSource.bytes,
+                                qbSource.offset, qbSource.length)) {
                             return parseQuery(context, mapUnmappedFieldsAsString, sourceParser);
                         }
                     } else {
@@ -504,7 +540,7 @@ private static PercolateQuery.QueryStore createLegacyStore(QueryShardContext con
                     throw new IllegalStateException("No source found for document with docid [" + docId + "]");
                 }
 
-                try (XContentParser sourceParser = XContentHelper.createParser(visitor.source)) {
+                try (XContentParser sourceParser = XContentHelper.createParser(context.getXContentRegistry(), visitor.source)) {
                     String currentFieldName = null;
                     XContentParser.Token token = sourceParser.nextToken(); // move the START_OBJECT
                     if (token != XContentParser.Token.START_OBJECT) {
@@ -553,4 +589,36 @@ public Status needsField(FieldInfo fieldInfo) throws IOException {
 
     }
 
+    static QueryShardContext wrap(QueryShardContext shardContext) {
+        return new QueryShardContext(shardContext) {
+
+            @Override
+            public BitSetProducer bitsetFilter(Query query) {
+                return context -> {
+                    final IndexReaderContext topLevelContext = ReaderUtil.getTopLevelContext(context);
+                    final IndexSearcher searcher = new IndexSearcher(topLevelContext);
+                    searcher.setQueryCache(null);
+                    final Weight weight = searcher.createNormalizedWeight(query, false);
+                    final Scorer s = weight.scorer(context);
+
+                    if (s != null) {
+                        return new BitDocIdSet(BitSet.of(s.iterator(), context.reader().maxDoc())).bits();
+                    } else {
+                        return null;
+                    }
+                };
+            }
+
+            @Override
+            @SuppressWarnings("unchecked")
+            public <IFD extends IndexFieldData<?>> IFD getForField(MappedFieldType fieldType) {
+                IndexFieldData.Builder builder = fieldType.fielddataBuilder();
+                IndexFieldDataCache cache = new IndexFieldDataCache.None();
+                CircuitBreakerService circuitBreaker = new NoneCircuitBreakerService();
+                return (IFD) builder.build(shardContext.getIndexSettings(), fieldType, cache, circuitBreaker,
+                        shardContext.getMapperService());
+            }
+        };
+    }
+
 }
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateRequest.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateRequest.java
index 4509ab3cb509a..bc449ea932d0f 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateRequest.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateRequest.java
@@ -44,7 +44,7 @@
  * @deprecated Instead use search API with {@link PercolateQueryBuilder}
  */
 @Deprecated
-public class PercolateRequest extends ActionRequest<PercolateRequest> implements IndicesRequest.Replaceable {
+public class PercolateRequest extends ActionRequest implements IndicesRequest.Replaceable {
 
     protected String[] indices;
     private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpenAndForbidClosed();
@@ -242,8 +242,8 @@ public ActionRequestValidationException validate() {
         if (source == null && getRequest == null) {
             validationException = addValidationError("source or get is missing", validationException);
         }
-        if (getRequest != null && getRequest.fields() != null) {
-            validationException = addValidationError("get fields option isn't supported via percolate request", validationException);
+        if (getRequest != null && getRequest.storedFields() != null) {
+            validationException = addValidationError("get stored fields option isn't supported via percolate request", validationException);
         }
         return validationException;
     }
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateResponse.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateResponse.java
index 97003dd273700..7e25af7ea6e72 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateResponse.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateResponse.java
@@ -26,7 +26,7 @@
 import org.elasticsearch.common.io.stream.Streamable;
 import org.elasticsearch.common.text.Text;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.action.RestActions;
 import org.elasticsearch.search.aggregations.InternalAggregations;
@@ -45,7 +45,7 @@
  * @deprecated Instead use search API with {@link PercolateQueryBuilder}
  */
 @Deprecated
-public class PercolateResponse extends BroadcastResponse implements Iterable<PercolateResponse.Match>, ToXContent {
+public class PercolateResponse extends BroadcastResponse implements Iterable<PercolateResponse.Match>, ToXContentObject {
 
     public static final Match[] EMPTY = new Match[0];
     // PercolateQuery emits this score if no 'query' is defined in the percolate request
@@ -113,6 +113,13 @@ public Iterator<Match> iterator() {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        innerToXContent(builder, params);
+        builder.endObject();
+        return builder;
+    }
+
+    public XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
         builder.field(Fields.TOOK, tookInMillis);
         RestActions.buildBroadcastShardsHeader(builder, params, this);
 
@@ -136,16 +143,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                     if (match.getHighlightFields().isEmpty() == false) {
                         builder.startObject(Fields.HIGHLIGHT);
                         for (HighlightField field : match.getHighlightFields().values()) {
-                            builder.field(field.name());
-                            if (field.fragments() == null) {
-                                builder.nullValue();
-                            } else {
-                                builder.startArray();
-                                for (Text fragment : field.fragments()) {
-                                    builder.value(fragment);
-                                }
-                                builder.endArray();
-                            }
+                            field.toXContent(builder, params);
                         }
                         builder.endObject();
                     }
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java
index a502e92402b2f..8361d1e5b0a27 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java
@@ -23,19 +23,23 @@
 import org.apache.lucene.index.Fields;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.queries.TermsQuery;
+import org.apache.lucene.search.BooleanClause.Occur;
+import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermInSetQuery;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefBuilder;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -53,13 +57,11 @@
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.BoostingQueryBuilder;
 import org.elasticsearch.index.query.ConstantScoreQueryBuilder;
-import org.elasticsearch.index.query.HasChildQueryBuilder;
-import org.elasticsearch.index.query.HasParentQueryBuilder;
+import org.elasticsearch.index.query.DisMaxQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.query.QueryShardException;
-import org.elasticsearch.index.query.RangeQueryBuilder;
 import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
 
 import java.io.IOException;
@@ -69,6 +71,7 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Supplier;
 
 public class PercolatorFieldMapper extends FieldMapper {
 
@@ -89,9 +92,9 @@ public class PercolatorFieldMapper extends FieldMapper {
 
     public static class Builder extends FieldMapper.Builder<Builder, PercolatorFieldMapper> {
 
-        private final QueryShardContext queryShardContext;
+        private final Supplier<QueryShardContext> queryShardContext;
 
-        public Builder(String fieldName, QueryShardContext queryShardContext) {
+        public Builder(String fieldName, Supplier<QueryShardContext> queryShardContext) {
             super(fieldName, FIELD_TYPE, FIELD_TYPE);
             this.queryShardContext = queryShardContext;
         }
@@ -136,7 +139,7 @@ public static class TypeParser implements FieldMapper.TypeParser {
 
         @Override
         public Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
-            return new Builder(name, parserContext.queryShardContext());
+            return new Builder(name, parserContext.queryShardContextSupplier());
         }
     }
 
@@ -192,12 +195,7 @@ public Query percolateQuery(String documentType, PercolateQuery.QueryStore query
         }
 
         Query createCandidateQuery(IndexReader indexReader) throws IOException {
-            List<Term> extractedTerms = new ArrayList<>();
-            // include extractionResultField:failed, because docs with this term have no extractedTermsField
-            // and otherwise we would fail to return these docs. Docs that failed query term extraction
-            // always need to be verified by MemoryIndex:
-            extractedTerms.add(new Term(extractionResultField.name(), EXTRACTION_FAILED));
-
+            List<BytesRef> extractedTerms = new ArrayList<>();
             LeafReader reader = indexReader.leaves().get(0).reader();
             Fields fields = reader.fields();
             for (String field : fields) {
@@ -213,22 +211,32 @@ Query createCandidateQuery(IndexReader indexReader) throws IOException {
                     builder.append(fieldBr);
                     builder.append(FIELD_VALUE_SEPARATOR);
                     builder.append(term);
-                    extractedTerms.add(new Term(queryTermsField.name(), builder.toBytesRef()));
+                    extractedTerms.add(builder.toBytesRef());
                 }
             }
-            return new TermsQuery(extractedTerms);
+            Query extractionSuccess = new TermInSetQuery(queryTermsField.name(), extractedTerms);
+            // include extractionResultField:failed, because docs with this term have no extractedTermsField
+            // and otherwise we would fail to return these docs. Docs that failed query term extraction
+            // always need to be verified by MemoryIndex:
+            Query extractionFailure = new TermQuery(new Term(extractionResultField.name(), EXTRACTION_FAILED));
+
+            return new BooleanQuery.Builder()
+                    .add(extractionSuccess, Occur.SHOULD)
+                    .add(extractionFailure, Occur.SHOULD)
+                    .build();
         }
 
     }
 
     private final boolean mapUnmappedFieldAsString;
-    private final QueryShardContext queryShardContext;
+    private final Supplier<QueryShardContext> queryShardContext;
     private KeywordFieldMapper queryTermsField;
     private KeywordFieldMapper extractionResultField;
     private BinaryFieldMapper queryBuilderField;
 
     public PercolatorFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
-                                 Settings indexSettings, MultiFields multiFields, CopyTo copyTo, QueryShardContext queryShardContext,
+                                 Settings indexSettings, MultiFields multiFields, CopyTo copyTo,
+                                 Supplier<QueryShardContext> queryShardContext,
                                  KeywordFieldMapper queryTermsField, KeywordFieldMapper extractionResultField,
                                  BinaryFieldMapper queryBuilderField) {
         super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo);
@@ -261,7 +269,7 @@ public FieldMapper updateFieldType(Map<String, MappedFieldType> fullNameToFieldT
 
     @Override
     public Mapper parse(ParseContext context) throws IOException {
-        QueryShardContext queryShardContext = new QueryShardContext(this.queryShardContext);
+        QueryShardContext queryShardContext = this.queryShardContext.get();
         if (context.doc().getField(queryBuilderField.name()) != null) {
             // If a percolator query has been defined in an array object then multiple percolator queries
             // could be provided. In order to prevent this we fail if we try to parse more than one query
@@ -275,7 +283,7 @@ public Mapper parse(ParseContext context) throws IOException {
         );
         verifyQuery(queryBuilder);
         // Fetching of terms, shapes and indexed scripts happen during this rewrite:
-        queryBuilder = queryBuilder.rewrite(queryShardContext);
+        queryBuilder = QueryBuilder.rewriteQuery(queryBuilder, queryShardContext);
 
         try (XContentBuilder builder = XContentFactory.contentBuilder(QUERY_BUILDER_CONTENT_TYPE)) {
             queryBuilder.toXContent(builder, new MapParams(Collections.emptyMap()));
@@ -337,6 +345,14 @@ static Query toQuery(QueryShardContext context, boolean mapUnmappedFieldsAsStrin
         // as an analyzed string.
         context.setAllowUnmappedFields(false);
         context.setMapUnmappedFieldAsString(mapUnmappedFieldsAsString);
+
+        // Rewriting the query at percolate time, because this is sometimes necessary:
+        // * From 5.0 and onwards the percolator rewrites the query at index time,
+        //   this is not the case for percolator queries in indices created before 5.0
+        if (context.getIndexSettings().getIndexVersionCreated().before(Version.V_5_0_0_alpha1)) {
+            queryBuilder = QueryBuilder.rewriteQuery(queryBuilder, context);
+        }
+
         return queryBuilder.toQuery(context);
     }
 
@@ -355,7 +371,7 @@ public Iterator<Mapper> iterator() {
     }
 
     @Override
-    protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
+    protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
         throw new UnsupportedOperationException("should not be invoked");
     }
 
@@ -364,26 +380,16 @@ protected String contentType() {
         return CONTENT_TYPE;
     }
 
+
     /**
      * Fails if a percolator contains an unsupported query. The following queries are not supported:
-     * 1) a range query with a date range based on current time
-     * 2) a has_child query
-     * 3) a has_parent query
+     * 1) a has_child query
+     * 2) a has_parent query
      */
     static void verifyQuery(QueryBuilder queryBuilder) {
-        if (queryBuilder instanceof RangeQueryBuilder) {
-            RangeQueryBuilder rangeQueryBuilder = (RangeQueryBuilder) queryBuilder;
-            if (rangeQueryBuilder.from() instanceof String) {
-                String from = (String) rangeQueryBuilder.from();
-                String to = (String) rangeQueryBuilder.to();
-                if (from.contains("now") || to.contains("now")) {
-                    throw new IllegalArgumentException("percolator queries containing time range queries based on the " +
-                            "current time is unsupported");
-                }
-            }
-        } else if (queryBuilder instanceof HasChildQueryBuilder) {
+        if (queryBuilder.getName().equals("has_child")) {
             throw new IllegalArgumentException("the [has_child] query is unsupported inside a percolator query");
-        } else if (queryBuilder instanceof HasParentQueryBuilder) {
+        } else if (queryBuilder.getName().equals("has_parent")) {
             throw new IllegalArgumentException("the [has_parent] query is unsupported inside a percolator query");
         } else if (queryBuilder instanceof BoolQueryBuilder) {
             BoolQueryBuilder boolQueryBuilder = (BoolQueryBuilder) queryBuilder;
@@ -402,6 +408,11 @@ static void verifyQuery(QueryBuilder queryBuilder) {
         } else if (queryBuilder instanceof BoostingQueryBuilder) {
             verifyQuery(((BoostingQueryBuilder) queryBuilder).negativeQuery());
             verifyQuery(((BoostingQueryBuilder) queryBuilder).positiveQuery());
+        } else if (queryBuilder instanceof DisMaxQueryBuilder) {
+            DisMaxQueryBuilder disMaxQueryBuilder = (DisMaxQueryBuilder) queryBuilder;
+            for (QueryBuilder innerQueryBuilder : disMaxQueryBuilder.innerQueries()) {
+                verifyQuery(innerQueryBuilder);
+            }
         }
     }
 
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java
index 2afa2c92ed1c7..64404cbead0b5 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java
@@ -37,7 +37,7 @@
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightPhase;
 import org.elasticsearch.search.fetch.subphase.highlight.Highlighter;
 import org.elasticsearch.search.fetch.subphase.highlight.SearchContextHighlight;
-import org.elasticsearch.search.internal.InternalSearchHit;
+import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.internal.SubSearchContext;
 
@@ -50,10 +50,11 @@
  * Highlighting in the case of the percolate query is a bit different, because the PercolateQuery itself doesn't get highlighted,
  * but the source of the PercolateQuery gets highlighted by each hit containing a query.
  */
-public final class PercolatorHighlightSubFetchPhase extends HighlightPhase {
+final class PercolatorHighlightSubFetchPhase implements FetchSubPhase {
+    private final HighlightPhase highlightPhase;
 
-    public PercolatorHighlightSubFetchPhase(Settings settings, Map<String, Highlighter> highlighters) {
-        super(settings, highlighters);
+    PercolatorHighlightSubFetchPhase(Settings settings, Map<String, Highlighter> highlighters) {
+        this.highlightPhase = new HighlightPhase(settings, highlighters);
     }
 
 
@@ -62,7 +63,7 @@ boolean hitsExecutionNeeded(SearchContext context) { // for testing
     }
 
     @Override
-    public void hitsExecute(SearchContext context, InternalSearchHit[] hits) {
+    public void hitsExecute(SearchContext context, SearchHit[] hits) {
         if (hitsExecutionNeeded(context) == false) {
             return;
         }
@@ -81,24 +82,24 @@ public void hitsExecute(SearchContext context, InternalSearchHit[] hits) {
         SubSearchContext subSearchContext =
                 createSubSearchContext(context, percolatorLeafReaderContext, percolateQuery.getDocumentSource());
 
-        for (InternalSearchHit hit : hits) {
+        for (SearchHit hit : hits) {
             final Query query;
             try {
                 LeafReaderContext ctx = ctxs.get(ReaderUtil.subIndex(hit.docId(), ctxs));
                 int segmentDocId = hit.docId() - ctx.docBase;
-                query = queryStore.getQueries(ctx).getQuery(segmentDocId);
+                query = queryStore.getQueries(ctx).apply(segmentDocId);
             } catch (IOException e) {
                 throw new RuntimeException(e);
             }
             if (query != null) {
                 subSearchContext.parsedQuery(new ParsedQuery(query));
                 hitContext.reset(
-                        new InternalSearchHit(0, "unknown", new Text(percolateQuery.getDocumentType()), Collections.emptyMap()),
+                        new SearchHit(0, "unknown", new Text(percolateQuery.getDocumentType()), Collections.emptyMap()),
                         percolatorLeafReaderContext, 0, percolatorIndexSearcher
                 );
                 hitContext.cache().clear();
-                super.hitExecute(subSearchContext, hitContext);
-                hit.highlightFields().putAll(hitContext.hit().getHighlightFields());
+                highlightPhase.hitExecute(subSearchContext, hitContext);
+                hit.getHighlightFields().putAll(hitContext.hit().getHighlightFields());
             }
         }
     }
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java
index 8b602e3c478ef..ba2c9a8acdf72 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java
@@ -21,13 +21,19 @@
 
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.SearchPlugin;
+import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestHandler;
 import org.elasticsearch.search.fetch.FetchSubPhase;
 
@@ -35,6 +41,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Supplier;
 
 import static java.util.Collections.singletonList;
 
@@ -47,14 +54,18 @@ public PercolatorPlugin(Settings settings) {
     }
 
     @Override
-    public List<ActionHandler<? extends ActionRequest<?>, ? extends ActionResponse>> getActions() {
+    public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
         return Arrays.asList(new ActionHandler<>(PercolateAction.INSTANCE, TransportPercolateAction.class),
                 new ActionHandler<>(MultiPercolateAction.INSTANCE, TransportMultiPercolateAction.class));
     }
 
     @Override
-    public List<Class<? extends RestHandler>> getRestHandlers() {
-        return Arrays.asList(RestPercolateAction.class, RestMultiPercolateAction.class);
+    public List<RestHandler> getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings,
+            IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver,
+            Supplier<DiscoveryNodes> nodesInCluster) {
+        return Arrays.asList(
+                new RestPercolateAction(settings, restController),
+                new RestMultiPercolateAction(settings, restController));
     }
 
     @Override
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/QueryAnalyzer.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/QueryAnalyzer.java
index c30a988a4f2c0..d8a684658a30c 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/QueryAnalyzer.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/QueryAnalyzer.java
@@ -22,15 +22,17 @@
 import org.apache.lucene.index.Term;
 import org.apache.lucene.queries.BlendedTermQuery;
 import org.apache.lucene.queries.CommonTermsQuery;
-import org.apache.lucene.queries.TermsQuery;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.DisjunctionMaxQuery;
+import org.apache.lucene.search.MatchNoDocsQuery;
+import org.apache.lucene.search.MultiPhraseQuery;
 import org.apache.lucene.search.PhraseQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.SynonymQuery;
+import org.apache.lucene.search.TermInSetQuery;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.spans.SpanFirstQuery;
 import org.apache.lucene.search.spans.SpanNearQuery;
@@ -40,10 +42,10 @@
 import org.apache.lucene.search.spans.SpanTermQuery;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.logging.LoggerMessageFormat;
-import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
 import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -63,10 +65,11 @@ public final class QueryAnalyzer {
         map.put(ConstantScoreQuery.class, constantScoreQuery());
         map.put(BoostQuery.class, boostQuery());
         map.put(TermQuery.class, termQuery());
-        map.put(TermsQuery.class, termsQuery());
+        map.put(TermInSetQuery.class, termInSetQuery());
         map.put(CommonTermsQuery.class, commonTermsQuery());
         map.put(BlendedTermQuery.class, blendedTermQuery());
         map.put(PhraseQuery.class, phraseQuery());
+        map.put(MultiPhraseQuery.class, multiPhraseQuery());
         map.put(SpanTermQuery.class, spanTermQuery());
         map.put(SpanNearQuery.class, spanNearQuery());
         map.put(SpanOrQuery.class, spanOrQuery());
@@ -146,11 +149,11 @@ static Function<Query, Result> termQuery() {
         });
     }
 
-    static Function<Query, Result> termsQuery() {
+    static Function<Query, Result> termInSetQuery() {
         return query -> {
-            TermsQuery termsQuery = (TermsQuery) query;
+            TermInSetQuery termInSetQuery = (TermInSetQuery) query;
             Set<Term> terms = new HashSet<>();
-            PrefixCodedTerms.TermIterator iterator = termsQuery.getTermData().iterator();
+            PrefixCodedTerms.TermIterator iterator = termInSetQuery.getTermData().iterator();
             for (BytesRef term = iterator.next(); term != null; term = iterator.next()) {
                 terms.add(new Term(iterator.field(), term));
             }
@@ -198,6 +201,21 @@ static Function<Query, Result> phraseQuery() {
         };
     }
 
+    static Function<Query, Result> multiPhraseQuery() {
+        return query -> {
+            Term[][] terms = ((MultiPhraseQuery) query).getTermArrays();
+            if (terms.length == 0) {
+                return new Result(true, Collections.emptySet());
+            }
+
+            Set<Term> bestTermArr = null;
+            for (Term[] termArr : terms) {
+                bestTermArr = selectTermListWithTheLongestShortestTerm(bestTermArr, new HashSet<>(Arrays.asList(termArr)));
+            }
+            return new Result(false, bestTermArr);
+        };
+    }
+
     static Function<Query, Result> spanTermQuery() {
         return query -> {
             Term term = ((SpanTermQuery) query).getTerm();
@@ -381,7 +399,7 @@ static class UnsupportedQueryException extends RuntimeException {
 
         private final Query unsupportedQuery;
 
-        public UnsupportedQueryException(Query unsupportedQuery) {
+        UnsupportedQueryException(Query unsupportedQuery) {
             super(LoggerMessageFormat.format("no query terms can be extracted from query [{}]", unsupportedQuery));
             this.unsupportedQuery = unsupportedQuery;
         }
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/RestMultiPercolateAction.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/RestMultiPercolateAction.java
index 3d8631624eaf1..0bfd99395cd75 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/RestMultiPercolateAction.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/RestMultiPercolateAction.java
@@ -16,6 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+
 package org.elasticsearch.percolator;
 
 import org.elasticsearch.action.support.IndicesOptions;
@@ -24,12 +25,12 @@
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.action.RestActions;
 import org.elasticsearch.rest.action.RestToXContentListener;
 
+import java.io.IOException;
+
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
@@ -53,14 +54,17 @@ public RestMultiPercolateAction(Settings settings, RestController controller) {
     }
 
     @Override
-    public void handleRequest(final RestRequest restRequest, final RestChannel restChannel, final NodeClient client) throws Exception {
+    public RestChannelConsumer prepareRequest(final RestRequest restRequest, final NodeClient client) throws IOException {
         MultiPercolateRequest multiPercolateRequest = new MultiPercolateRequest();
         multiPercolateRequest.indicesOptions(IndicesOptions.fromRequest(restRequest, multiPercolateRequest.indicesOptions()));
         multiPercolateRequest.indices(Strings.splitStringByCommaToArray(restRequest.param("index")));
         multiPercolateRequest.documentType(restRequest.param("type"));
-        multiPercolateRequest.add(RestActions.getRestContent(restRequest), allowExplicitIndex);
-        client.execute(MultiPercolateAction.INSTANCE, multiPercolateRequest,
-                new RestToXContentListener<MultiPercolateResponse>(restChannel));
+        multiPercolateRequest.add(restRequest.contentOrSourceParam().v2(), allowExplicitIndex);
+        return channel -> client.execute(MultiPercolateAction.INSTANCE, multiPercolateRequest, new RestToXContentListener<>(channel));
     }
 
+    @Override
+    public boolean supportsContentStream() {
+        return true;
+    }
 }
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/RestPercolateAction.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/RestPercolateAction.java
index 35d9f2c604ad7..e490283939587 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/RestPercolateAction.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/RestPercolateAction.java
@@ -16,6 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+
 package org.elasticsearch.percolator;
 
 import org.elasticsearch.action.get.GetRequest;
@@ -32,11 +33,14 @@
 import org.elasticsearch.rest.action.RestActions;
 import org.elasticsearch.rest.action.RestToXContentListener;
 
+import java.io.IOException;
+
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
 @Deprecated
 public class RestPercolateAction extends BaseRestHandler {
+
     @Inject
     public RestPercolateAction(Settings settings, RestController controller) {
         super(settings);
@@ -56,19 +60,18 @@ public RestPercolateAction(Settings settings, RestController controller) {
         controller.registerHandler(POST, "/{index}/{type}/{id}/_percolate/count", countExistingDocHandler);
     }
 
-    void parseDocPercolate(PercolateRequest percolateRequest, RestRequest restRequest, RestChannel restChannel, NodeClient client) {
+    private RestChannelConsumer parseDocPercolate(PercolateRequest percolateRequest, RestRequest restRequest, NodeClient client) {
         percolateRequest.indices(Strings.splitStringByCommaToArray(restRequest.param("index")));
         percolateRequest.documentType(restRequest.param("type"));
         percolateRequest.routing(restRequest.param("routing"));
         percolateRequest.preference(restRequest.param("preference"));
-        percolateRequest.source(RestActions.getRestContent(restRequest));
+        percolateRequest.source(restRequest.contentOrSourceParam().v2());
 
         percolateRequest.indicesOptions(IndicesOptions.fromRequest(restRequest, percolateRequest.indicesOptions()));
-        executePercolate(client, percolateRequest, restChannel);
+        return channel -> executePercolate(client, percolateRequest, channel);
     }
 
-    void parseExistingDocPercolate(PercolateRequest percolateRequest, RestRequest restRequest, RestChannel restChannel,
-            NodeClient client) {
+    private RestChannelConsumer parseExistingDocPercolate(PercolateRequest percolateRequest, RestRequest restRequest, NodeClient client) {
         String index = restRequest.param("index");
         String type = restRequest.param("type");
         percolateRequest.indices(Strings.splitStringByCommaToArray(restRequest.param("percolate_index", index)));
@@ -86,60 +89,66 @@ void parseExistingDocPercolate(PercolateRequest percolateRequest, RestRequest re
         percolateRequest.getRequest(getRequest);
         percolateRequest.routing(restRequest.param("percolate_routing"));
         percolateRequest.preference(restRequest.param("percolate_preference"));
-        percolateRequest.source(RestActions.getRestContent(restRequest));
+        if (restRequest.hasContentOrSourceParam()) {
+            percolateRequest.source(restRequest.contentOrSourceParam().v2());
+        }
 
         percolateRequest.indicesOptions(IndicesOptions.fromRequest(restRequest, percolateRequest.indicesOptions()));
-        executePercolate(client, percolateRequest, restChannel);
+        return channel -> executePercolate(client, percolateRequest, channel);
     }
 
-    void executePercolate(final NodeClient client, final PercolateRequest percolateRequest, final RestChannel restChannel) {
+    private void executePercolate(final NodeClient client, final PercolateRequest percolateRequest, final RestChannel restChannel) {
         client.execute(PercolateAction.INSTANCE, percolateRequest, new RestToXContentListener<>(restChannel));
     }
 
     @Override
-    public void handleRequest(RestRequest restRequest, RestChannel restChannel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(RestRequest restRequest, final NodeClient client) throws IOException {
         PercolateRequest percolateRequest = new PercolateRequest();
-        parseDocPercolate(percolateRequest, restRequest, restChannel, client);
+        return parseDocPercolate(percolateRequest, restRequest, client);
     }
 
-    final class RestCountPercolateDocHandler extends BaseRestHandler {
+    private final class RestCountPercolateDocHandler extends BaseRestHandler {
 
         private RestCountPercolateDocHandler(Settings settings) {
             super(settings);
         }
 
         @Override
-        public void handleRequest(RestRequest restRequest, RestChannel restChannel, final NodeClient client) {
+        public RestChannelConsumer prepareRequest(RestRequest restRequest, final NodeClient client) throws IOException {
             PercolateRequest percolateRequest = new PercolateRequest();
             percolateRequest.onlyCount(true);
-            parseDocPercolate(percolateRequest, restRequest, restChannel, client);
+            return parseDocPercolate(percolateRequest, restRequest, client);
         }
+
     }
 
-    final class RestPercolateExistingDocHandler extends BaseRestHandler {
+    private final class RestPercolateExistingDocHandler extends BaseRestHandler {
 
-        protected RestPercolateExistingDocHandler(Settings settings) {
+        RestPercolateExistingDocHandler(Settings settings) {
             super(settings);
         }
 
         @Override
-        public void handleRequest(RestRequest restRequest, RestChannel restChannel, final NodeClient client) {
+        public RestChannelConsumer prepareRequest(RestRequest restRequest, final NodeClient client) throws IOException {
             PercolateRequest percolateRequest = new PercolateRequest();
-            parseExistingDocPercolate(percolateRequest, restRequest, restChannel, client);
+            return parseExistingDocPercolate(percolateRequest, restRequest, client);
         }
+
     }
 
-    final class RestCountPercolateExistingDocHandler extends BaseRestHandler {
+    private final class RestCountPercolateExistingDocHandler extends BaseRestHandler {
 
-        protected RestCountPercolateExistingDocHandler(Settings settings) {
+        RestCountPercolateExistingDocHandler(Settings settings) {
             super(settings);
         }
 
         @Override
-        public void handleRequest(RestRequest restRequest, RestChannel restChannel, final NodeClient client) {
+        public RestChannelConsumer prepareRequest(RestRequest restRequest, final NodeClient client) throws IOException {
             PercolateRequest percolateRequest = new PercolateRequest();
             percolateRequest.onlyCount(true);
-            parseExistingDocPercolate(percolateRequest, restRequest, restChannel, client);
+            return parseExistingDocPercolate(percolateRequest, restRequest, client);
         }
+
     }
+
 }
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportMultiPercolateAction.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportMultiPercolateAction.java
index 44914e140b128..10145e73f760c 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportMultiPercolateAction.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportMultiPercolateAction.java
@@ -32,14 +32,11 @@
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
-import org.elasticsearch.search.SearchRequestParsers;
-import org.elasticsearch.search.aggregations.AggregatorParsers;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -54,18 +51,16 @@
 public class TransportMultiPercolateAction extends HandledTransportAction<MultiPercolateRequest, MultiPercolateResponse> {
 
     private final Client client;
-    private final ParseFieldMatcher parseFieldMatcher;
-    private final SearchRequestParsers searchRequestParsers;
+    private final NamedXContentRegistry xContentRegistry;
 
     @Inject
     public TransportMultiPercolateAction(Settings settings, ThreadPool threadPool, TransportService transportService,
                                          ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
-                                         Client client, SearchRequestParsers searchRequestParsers) {
+                                         Client client, NamedXContentRegistry xContentRegistry) {
         super(settings, MultiPercolateAction.NAME, threadPool, transportService, actionFilters,
               indexNameExpressionResolver, MultiPercolateRequest::new);
         this.client = client;
-        this.searchRequestParsers = searchRequestParsers;
-        this.parseFieldMatcher = new ParseFieldMatcher(settings);
+        this.xContentRegistry = xContentRegistry;
     }
 
     @Override
@@ -158,10 +153,7 @@ private MultiSearchRequest createMultiSearchRequest(MultiPercolateRequest multiP
             PercolateRequest percolateRequest = multiPercolateRequest.requests().get(i);
             BytesReference docSource = getResponseSources.get(i);
             try {
-                SearchRequest searchRequest = TransportPercolateAction.createSearchRequest(
-                    percolateRequest, docSource, searchRequestParsers.queryParsers,
-                    searchRequestParsers.aggParsers, parseFieldMatcher
-                );
+                SearchRequest searchRequest = TransportPercolateAction.createSearchRequest(percolateRequest, docSource, xContentRegistry);
                 multiSearchRequest.add(searchRequest);
             } catch (Exception e) {
                 preFailures.put(i, new MultiPercolateResponse.Item(e));
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportPercolateAction.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportPercolateAction.java
index 55e7d922c34cb..3e2399da4f22b 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportPercolateAction.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportPercolateAction.java
@@ -30,11 +30,11 @@
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.text.Text;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentHelper;
@@ -44,11 +44,8 @@
 import org.elasticsearch.index.query.ConstantScoreQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHits;
-import org.elasticsearch.search.SearchRequestParsers;
-import org.elasticsearch.search.aggregations.AggregatorParsers;
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -62,17 +59,16 @@
 public class TransportPercolateAction extends HandledTransportAction<PercolateRequest, PercolateResponse> {
 
     private final Client client;
-    private final ParseFieldMatcher parseFieldMatcher;
-    private final SearchRequestParsers searchRequestParsers;
+    private final NamedXContentRegistry xContentRegistry;
 
     @Inject
     public TransportPercolateAction(Settings settings, ThreadPool threadPool, TransportService transportService,
                                     ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
-                                    Client client, SearchRequestParsers searchRequestParsers) {
-        super(settings, PercolateAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, PercolateRequest::new);
+                                    Client client, NamedXContentRegistry xContentRegistry) {
+        super(settings, PercolateAction.NAME, threadPool, transportService, actionFilters,
+                indexNameExpressionResolver, PercolateRequest::new);
         this.client = client;
-        this.searchRequestParsers = searchRequestParsers;
-        this.parseFieldMatcher = new ParseFieldMatcher(settings);
+        this.xContentRegistry = xContentRegistry;
     }
 
     @Override
@@ -84,7 +80,8 @@ public void onResponse(GetResponse getResponse) {
                     if (getResponse.isExists()) {
                         innerDoExecute(request, getResponse.getSourceAsBytesRef(), listener);
                     } else {
-                        onFailure(new ResourceNotFoundException("percolate document [{}/{}/{}] doesn't exist", request.getRequest().index(), request.getRequest().type(), request.getRequest().id()));
+                        onFailure(new ResourceNotFoundException("percolate document [{}/{}/{}] doesn't exist",
+                                request.getRequest().index(), request.getRequest().type(), request.getRequest().id()));
                     }
                 }
 
@@ -101,8 +98,7 @@ public void onFailure(Exception e) {
     private void innerDoExecute(PercolateRequest request, BytesReference docSource, ActionListener<PercolateResponse> listener) {
         SearchRequest searchRequest;
         try {
-            searchRequest = createSearchRequest(request, docSource, searchRequestParsers.queryParsers,
-                searchRequestParsers.aggParsers, parseFieldMatcher);
+            searchRequest = createSearchRequest(request, docSource, xContentRegistry);
         } catch (IOException e) {
             listener.onFailure(e);
             return;
@@ -124,7 +120,9 @@ public void onFailure(Exception e) {
         });
     }
 
-    public static SearchRequest createSearchRequest(PercolateRequest percolateRequest, BytesReference documentSource, IndicesQueriesRegistry queryRegistry, AggregatorParsers aggParsers, ParseFieldMatcher parseFieldMatcher) throws IOException {
+    public static SearchRequest createSearchRequest(PercolateRequest percolateRequest, BytesReference documentSource,
+                                                    NamedXContentRegistry xContentRegistry)
+            throws IOException {
         SearchRequest searchRequest = new SearchRequest();
         if (percolateRequest.indices() != null) {
             searchRequest.indices(percolateRequest.indices());
@@ -136,7 +134,7 @@ public static SearchRequest createSearchRequest(PercolateRequest percolateReques
         BytesReference querySource = null;
         XContentBuilder searchSource = XContentFactory.jsonBuilder().startObject();
         if (percolateRequest.source() != null && percolateRequest.source().length() > 0) {
-            try (XContentParser parser = XContentHelper.createParser(percolateRequest.source())) {
+            try (XContentParser parser = XContentHelper.createParser(xContentRegistry, percolateRequest.source())) {
                 String currentFieldName = null;
                 XContentParser.Token token = parser.nextToken();
                 if (token != XContentParser.Token.START_OBJECT) {
@@ -200,8 +198,8 @@ public static SearchRequest createSearchRequest(PercolateRequest percolateReques
         PercolateQueryBuilder percolateQueryBuilder =
                 new PercolateQueryBuilder("query", percolateRequest.documentType(), documentSource);
         if (querySource != null) {
-            try (XContentParser parser = XContentHelper.createParser(querySource)) {
-                QueryParseContext queryParseContext = new QueryParseContext(queryRegistry, parser, parseFieldMatcher);
+            try (XContentParser parser = XContentHelper.createParser(xContentRegistry, querySource)) {
+                QueryParseContext queryParseContext = new QueryParseContext(parser);
                 BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
                 queryParseContext.parseInnerQueryBuilder().ifPresent(boolQueryBuilder::must);
                 boolQueryBuilder.filter(percolateQueryBuilder);
@@ -218,9 +216,9 @@ public static SearchRequest createSearchRequest(PercolateRequest percolateReques
         searchSource.flush();
         BytesReference source = searchSource.bytes();
         SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
-        try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(source)) {
-            QueryParseContext context = new QueryParseContext(queryRegistry, parser, parseFieldMatcher);
-            searchSourceBuilder.parseXContent(context, aggParsers, null);
+        try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(xContentRegistry, source)) {
+            QueryParseContext context = new QueryParseContext(parser);
+            searchSourceBuilder.parseXContent(context);
             searchRequest.source(searchSourceBuilder);
             return searchRequest;
         }
@@ -235,7 +233,8 @@ public static PercolateResponse createPercolateResponse(SearchResponse searchRes
             matches = new PercolateResponse.Match[hits.getHits().length];
             for (int i = 0; i < hits.getHits().length; i++) {
                 SearchHit hit = hits.getHits()[i];
-                matches[i] = new PercolateResponse.Match(new Text(hit.getIndex()), new Text(hit.getId()), hit.getScore(), hit.getHighlightFields());
+                matches[i] = new PercolateResponse.Match(new Text(hit.getIndex()),
+                        new Text(hit.getId()), hit.getScore(), hit.getHighlightFields());
             }
         }
 
@@ -246,8 +245,8 @@ public static PercolateResponse createPercolateResponse(SearchResponse searchRes
         }
 
         return new PercolateResponse(
-            searchResponse.getTotalShards(), searchResponse.getSuccessfulShards(), searchResponse.getFailedShards(),
-            shardFailures, matches, hits.getTotalHits(), searchResponse.getTookInMillis(), (InternalAggregations) searchResponse.getAggregations()
+            searchResponse.getTotalShards(), searchResponse.getSuccessfulShards(), searchResponse.getFailedShards(), shardFailures,
+                matches, hits.getTotalHits(), searchResponse.getTookInMillis(), (InternalAggregations) searchResponse.getAggregations()
         );
     }
 
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java
index 8ac9890afa839..65005e957ac5c 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java
@@ -47,6 +47,7 @@
 import org.apache.lucene.search.FilteredDocIdSetIterator;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.PrefixQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.Scorer;
@@ -59,9 +60,9 @@
 import org.apache.lucene.search.spans.SpanOrQuery;
 import org.apache.lucene.search.spans.SpanTermQuery;
 import org.apache.lucene.store.Directory;
+import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.compress.CompressedXContent;
-import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.IndexService;
@@ -384,13 +385,13 @@ public String toString() {
                 @Override
                 public Scorer scorer(LeafReaderContext context) throws IOException {
                     DocIdSetIterator allDocs = DocIdSetIterator.all(context.reader().maxDoc());
-                    PercolateQuery.QueryStore.Leaf leaf = queryStore.getQueries(context);
+                    CheckedFunction<Integer, Query, IOException> leaf = queryStore.getQueries(context);
                     FilteredDocIdSetIterator memoryIndexIterator = new FilteredDocIdSetIterator(allDocs) {
 
                         @Override
                         protected boolean match(int doc) {
                             try {
-                                Query query = leaf.getQuery(doc);
+                                Query query = leaf.apply(doc);
                                 float score = memoryIndex.search(query);
                                 if (score != 0f) {
                                     if (needsScores) {
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/MultiPercolatorRequestTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/MultiPercolatorRequestTests.java
index 6f64dc6318992..8e3eeebc6229a 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/MultiPercolatorRequestTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/MultiPercolatorRequestTests.java
@@ -21,6 +21,7 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.StreamsUtils;
 
@@ -45,7 +46,7 @@ public void testParseBulkRequests() throws Exception {
         assertThat(percolateRequest.onlyCount(), equalTo(false));
         assertThat(percolateRequest.getRequest(), nullValue());
         assertThat(percolateRequest.source(), notNullValue());
-        Map sourceMap = XContentFactory.xContent(percolateRequest.source()).createParser(percolateRequest.source()).map();
+        Map sourceMap = createParser(JsonXContent.jsonXContent, percolateRequest.source()).map();
         assertThat(sourceMap.get("doc"), equalTo((Object) MapBuilder.newMapBuilder().put("field1", "value1").map()));
 
         percolateRequest = request.requests().get(1);
@@ -58,7 +59,7 @@ public void testParseBulkRequests() throws Exception {
         assertThat(percolateRequest.onlyCount(), equalTo(false));
         assertThat(percolateRequest.getRequest(), nullValue());
         assertThat(percolateRequest.source(), notNullValue());
-        sourceMap = XContentFactory.xContent(percolateRequest.source()).createParser(percolateRequest.source()).map();
+        sourceMap = createParser(JsonXContent.jsonXContent, percolateRequest.source()).map();
         assertThat(sourceMap.get("doc"), equalTo((Object) MapBuilder.newMapBuilder().put("field1", "value2").map()));
 
         percolateRequest = request.requests().get(2);
@@ -71,7 +72,7 @@ public void testParseBulkRequests() throws Exception {
         assertThat(percolateRequest.onlyCount(), equalTo(true));
         assertThat(percolateRequest.getRequest(), nullValue());
         assertThat(percolateRequest.source(), notNullValue());
-        sourceMap = XContentFactory.xContent(percolateRequest.source()).createParser(percolateRequest.source()).map();
+        sourceMap = createParser(JsonXContent.jsonXContent, percolateRequest.source()).map();
         assertThat(sourceMap.get("doc"), equalTo((Object) MapBuilder.newMapBuilder().put("field1", "value3").map()));
 
         percolateRequest = request.requests().get(3);
@@ -111,7 +112,7 @@ public void testParseBulkRequests() throws Exception {
         assertThat(percolateRequest.onlyCount(), equalTo(false));
         assertThat(percolateRequest.getRequest(), nullValue());
         assertThat(percolateRequest.source(), notNullValue());
-        sourceMap = XContentFactory.xContent(percolateRequest.source()).createParser(percolateRequest.source()).map();
+        sourceMap = createParser(JsonXContent.jsonXContent, percolateRequest.source()).map();
         assertThat(sourceMap.get("doc"), equalTo((Object) MapBuilder.newMapBuilder().put("field1", "value4").map()));
 
         percolateRequest = request.requests().get(6);
@@ -127,7 +128,7 @@ public void testParseBulkRequests() throws Exception {
         assertThat(percolateRequest.indicesOptions(), equalTo(IndicesOptions.strictExpandOpenAndForbidClosed()));
         assertThat(percolateRequest.onlyCount(), equalTo(false));
         assertThat(percolateRequest.source(), notNullValue());
-        sourceMap = XContentFactory.xContent(percolateRequest.source()).createParser(percolateRequest.source()).map();
+        sourceMap = createParser(JsonXContent.jsonXContent, percolateRequest.source()).map();
         assertThat(sourceMap.get("doc"), nullValue());
 
         percolateRequest = request.requests().get(7);
@@ -143,7 +144,7 @@ public void testParseBulkRequests() throws Exception {
         assertThat(percolateRequest.indicesOptions(), equalTo(IndicesOptions.fromOptions(false, false, true, false, IndicesOptions.strictExpandOpenAndForbidClosed())));
         assertThat(percolateRequest.onlyCount(), equalTo(false));
         assertThat(percolateRequest.source(), notNullValue());
-        sourceMap = XContentFactory.xContent(percolateRequest.source()).createParser(percolateRequest.source()).map();
+        sourceMap = createParser(JsonXContent.jsonXContent, percolateRequest.source()).map();
         assertThat(sourceMap.get("doc"), nullValue());
     }
 
@@ -163,7 +164,7 @@ public void testParseBulkRequestsDefaults() throws Exception {
         assertThat(percolateRequest.onlyCount(), equalTo(false));
         assertThat(percolateRequest.getRequest(), nullValue());
         assertThat(percolateRequest.source(), notNullValue());
-        Map sourceMap = XContentFactory.xContent(percolateRequest.source()).createParser(percolateRequest.source()).map();
+        Map sourceMap = createParser(JsonXContent.jsonXContent, percolateRequest.source()).map();
         assertThat(sourceMap.get("doc"), equalTo((Object) MapBuilder.newMapBuilder().put("field1", "value1").map()));
 
         percolateRequest = request.requests().get(1);
@@ -175,7 +176,7 @@ public void testParseBulkRequestsDefaults() throws Exception {
         assertThat(percolateRequest.onlyCount(), equalTo(false));
         assertThat(percolateRequest.getRequest(), nullValue());
         assertThat(percolateRequest.source(), notNullValue());
-        sourceMap = XContentFactory.xContent(percolateRequest.source()).createParser(percolateRequest.source()).map();
+        sourceMap = createParser(JsonXContent.jsonXContent, percolateRequest.source()).map();
         assertThat(sourceMap.get("doc"), equalTo((Object) MapBuilder.newMapBuilder().put("field1", "value2").map()));
 
         percolateRequest = request.requests().get(2);
@@ -185,7 +186,7 @@ public void testParseBulkRequestsDefaults() throws Exception {
         assertThat(percolateRequest.onlyCount(), equalTo(false));
         assertThat(percolateRequest.getRequest(), nullValue());
         assertThat(percolateRequest.source(), notNullValue());
-        sourceMap = XContentFactory.xContent(percolateRequest.source()).createParser(percolateRequest.source()).map();
+        sourceMap = createParser(JsonXContent.jsonXContent, percolateRequest.source()).map();
         assertThat(sourceMap.get("doc"), equalTo((Object) MapBuilder.newMapBuilder().put("field1", "value3").map()));
     }
 
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java
index abb75b4dbd642..cc5d28c4095b1 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java
@@ -27,27 +27,32 @@
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.ResourceNotFoundException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.get.GetResult;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
 import org.hamcrest.Matchers;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Base64;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
@@ -80,9 +85,9 @@ protected Collection<Class<? extends Plugin>> getPlugins() {
 
     @Override
     protected void initializeAdditionalMappings(MapperService mapperService) throws IOException {
-        queryField = randomAsciiOfLength(4);
-        docType = randomAsciiOfLength(4);
-        mapperService.merge("query_type", new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef("query_type",
+        queryField = randomAlphaOfLength(4);
+        docType = "doc";
+        mapperService.merge("doc", new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef("doc",
                 queryField, "type=percolator"
         ).string()), MapperService.MergeReason.MAPPING_UPDATE, false);
         mapperService.merge(docType, new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef(docType,
@@ -98,16 +103,16 @@ protected PercolateQueryBuilder doCreateTestQueryBuilder() {
     private PercolateQueryBuilder doCreateTestQueryBuilder(boolean indexedDocument) {
         documentSource = randomSource();
         if (indexedDocument) {
-            indexedDocumentIndex = randomAsciiOfLength(4);
-            indexedDocumentType = randomAsciiOfLength(4);
-            indexedDocumentId = randomAsciiOfLength(4);
-            indexedDocumentRouting = randomAsciiOfLength(4);
-            indexedDocumentPreference = randomAsciiOfLength(4);
+            indexedDocumentIndex = randomAlphaOfLength(4);
+            indexedDocumentType = "doc";
+            indexedDocumentId = randomAlphaOfLength(4);
+            indexedDocumentRouting = randomAlphaOfLength(4);
+            indexedDocumentPreference = randomAlphaOfLength(4);
             indexedDocumentVersion = (long) randomIntBetween(0, Integer.MAX_VALUE);
             return new PercolateQueryBuilder(queryField, docType, indexedDocumentIndex, indexedDocumentType, indexedDocumentId,
                     indexedDocumentRouting, indexedDocumentPreference, indexedDocumentVersion);
         } else {
-            return new PercolateQueryBuilder(queryField, docType, documentSource);
+            return new PercolateQueryBuilder(queryField, docType, documentSource, XContentType.JSON);
         }
     }
 
@@ -142,7 +147,7 @@ protected GetResponse executeGet(GetRequest getRequest) {
     }
 
     @Override
-    protected void doAssertLuceneQuery(PercolateQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
+    protected void doAssertLuceneQuery(PercolateQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
         assertThat(query, Matchers.instanceOf(PercolateQuery.class));
         PercolateQuery percolateQuery = (PercolateQuery) query;
         assertThat(percolateQuery.getDocumentType(), Matchers.equalTo(queryBuilder.getDocumentType()));
@@ -155,7 +160,8 @@ public void testMustRewrite() throws IOException {
         IllegalStateException e = expectThrows(IllegalStateException.class, () -> pqb.toQuery(createShardContext()));
         assertThat(e.getMessage(), equalTo("query builder must be rewritten first"));
         QueryBuilder rewrite = pqb.rewrite(createShardContext());
-        PercolateQueryBuilder geoShapeQueryBuilder = new PercolateQueryBuilder(pqb.getField(), pqb.getDocumentType(), documentSource);
+        PercolateQueryBuilder geoShapeQueryBuilder =
+            new PercolateQueryBuilder(pqb.getField(), pqb.getDocumentType(), documentSource, XContentType.JSON);
         assertEquals(geoShapeQueryBuilder, rewrite);
     }
 
@@ -176,14 +182,16 @@ protected Set<String> getObjectsHoldingArbitraryContent() {
 
     public void testRequiredParameters() {
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {
-            new PercolateQueryBuilder(null, null, new BytesArray("{}"));
+            new PercolateQueryBuilder(null, null, new BytesArray("{}"), XContentType.JSON);
         });
         assertThat(e.getMessage(), equalTo("[field] is a required argument"));
 
-        e = expectThrows(IllegalArgumentException.class, () -> new PercolateQueryBuilder("_field", null, new BytesArray("{}")));
+        e = expectThrows(IllegalArgumentException.class,
+            () -> new PercolateQueryBuilder("_field", null, new BytesArray("{}"), XContentType.JSON));
         assertThat(e.getMessage(), equalTo("[document_type] is a required argument"));
 
-        e = expectThrows(IllegalArgumentException.class, () -> new PercolateQueryBuilder("_field", "_document_type", null));
+        e = expectThrows(IllegalArgumentException.class,
+            () -> new PercolateQueryBuilder("_field", "_document_type", null, null));
         assertThat(e.getMessage(), equalTo("[document] is a required argument"));
 
         e = expectThrows(IllegalArgumentException.class, () -> {
@@ -226,7 +234,7 @@ public void testCreateMultiDocumentSearcher() throws Exception {
         }
 
         Analyzer analyzer = new WhitespaceAnalyzer();
-        ParsedDocument parsedDocument = new ParsedDocument(null, "_id", "_type", null, -1L, -1L, docs, null, null);
+        ParsedDocument parsedDocument = new ParsedDocument(null, "_id", "_type", null, -1L, -1L, docs, null, null, null);
         IndexSearcher indexSearcher = PercolateQueryBuilder.createMultiDocumentSearcher(analyzer, parsedDocument);
         assertThat(indexSearcher.getIndexReader().numDocs(), equalTo(numDocs));
 
@@ -239,6 +247,26 @@ public void testCreateMultiDocumentSearcher() throws Exception {
         assertThat(result.clauses().get(1).getOccur(), equalTo(BooleanClause.Occur.MUST_NOT));
     }
 
+    public void testSerializationBwc() throws IOException {
+        final byte[] data = Base64.getDecoder().decode("P4AAAAAFZmllbGQEdHlwZQAAAAAAAA57ImZvbyI6ImJhciJ9AAAAAA==");
+        final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2,
+            Version.V_5_1_1, Version.V_5_1_2, Version.V_5_2_0);
+        try (StreamInput in = StreamInput.wrap(data)) {
+            in.setVersion(version);
+            PercolateQueryBuilder queryBuilder = new PercolateQueryBuilder(in);
+            assertEquals("type", queryBuilder.getDocumentType());
+            assertEquals("field", queryBuilder.getField());
+            assertEquals("{\"foo\":\"bar\"}", queryBuilder.getDocument().utf8ToString());
+            assertEquals(XContentType.JSON, queryBuilder.getXContentType());
+
+            try (BytesStreamOutput out = new BytesStreamOutput()) {
+                out.setVersion(version);
+                queryBuilder.writeTo(out);
+                assertArrayEquals(data, out.bytes().toBytesRef().bytes);
+            }
+        }
+    }
+
     private static BytesReference randomSource() {
         try {
             XContentBuilder xContent = XContentFactory.jsonBuilder();
@@ -248,4 +276,14 @@ private static BytesReference randomSource() {
             throw new RuntimeException(e);
         }
     }
+
+    @Override
+    protected boolean isCachable(PercolateQueryBuilder queryBuilder) {
+        return false;
+    }
+
+    @Override
+    protected boolean builderGeneratesCacheableQueries() {
+        return false;
+    }
 }
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java
index eda9ce15c73a6..c5588ed312f2f 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java
@@ -35,6 +35,7 @@
 import org.apache.lucene.search.Explanation;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.PhraseQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
@@ -43,7 +44,6 @@
 import org.apache.lucene.search.spans.SpanTermQuery;
 import org.apache.lucene.store.Directory;
 import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.After;
 import org.junit.Before;
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorBackwardsCompatibilityTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorBackwardsCompatibilityTests.java
index 48459858ba527..80f8cb1f4db77 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorBackwardsCompatibilityTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorBackwardsCompatibilityTests.java
@@ -21,7 +21,6 @@
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.TestUtil;
 import org.elasticsearch.Version;
-import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
@@ -44,7 +43,6 @@
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
-import static org.elasticsearch.index.query.QueryBuilders.scriptQuery;
 import static org.elasticsearch.percolator.PercolatorTestUtil.preparePercolate;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
@@ -72,7 +70,7 @@ public void testOldPercolatorIndex() throws Exception {
 
         // verify cluster state:
         ClusterState state = client().admin().cluster().prepareState().get().getState();
-        assertThat(state.metaData().indices().size(), equalTo(1));
+        assertThat(state.metaData().indices().size(), equalTo(2));
         assertThat(state.metaData().indices().get(INDEX_NAME), notNullValue());
         assertThat(state.metaData().indices().get(INDEX_NAME).getCreationVersion(), equalTo(Version.V_2_0_0));
         assertThat(state.metaData().indices().get(INDEX_NAME).getUpgradedVersion(), equalTo(Version.CURRENT));
@@ -88,17 +86,18 @@ public void testOldPercolatorIndex() throws Exception {
             .setTypes(".percolator")
             .addSort("_uid", SortOrder.ASC)
             .get();
-        assertThat(searchResponse.getHits().getTotalHits(), equalTo(4L));
-        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("1"));
-        assertThat(searchResponse.getHits().getAt(1).id(), equalTo("2"));
-        assertThat(searchResponse.getHits().getAt(2).id(), equalTo("3"));
-        assertThat(searchResponse.getHits().getAt(3).id(), equalTo("4"));
+        assertThat(searchResponse.getHits().getTotalHits(), equalTo(5L));
+        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("0"));
+        assertThat(searchResponse.getHits().getAt(1).id(), equalTo("1"));
+        assertThat(searchResponse.getHits().getAt(2).id(), equalTo("2"));
+        assertThat(searchResponse.getHits().getAt(3).id(), equalTo("3"));
+        assertThat(searchResponse.getHits().getAt(4).id(), equalTo("4"));
         assertThat(XContentMapValues.extractValue("query.script.script.inline",
-                searchResponse.getHits().getAt(3).sourceAsMap()), equalTo("return true"));
+                searchResponse.getHits().getAt(4).sourceAsMap()), equalTo("return true"));
         // we don't upgrade the script definitions so that they include explicitly the lang,
         // because we read / parse the query at search time.
         assertThat(XContentMapValues.extractValue("query.script.script.lang",
-                searchResponse.getHits().getAt(3).sourceAsMap()), nullValue());
+                searchResponse.getHits().getAt(4).sourceAsMap()), nullValue());
 
         // verify percolate response
         PercolateResponse percolateResponse = preparePercolate(client())
@@ -107,9 +106,10 @@ public void testOldPercolatorIndex() throws Exception {
                 .setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc("{}"))
                 .get();
 
-        assertThat(percolateResponse.getCount(), equalTo(1L));
-        assertThat(percolateResponse.getMatches().length, equalTo(1));
+        assertThat(percolateResponse.getCount(), equalTo(2L));
+        assertThat(percolateResponse.getMatches().length, equalTo(2));
         assertThat(percolateResponse.getMatches()[0].getId().string(), equalTo("4"));
+        assertThat(percolateResponse.getMatches()[1].getId().string(), equalTo("0"));
 
         percolateResponse = preparePercolate(client())
             .setIndices(INDEX_NAME)
@@ -117,11 +117,12 @@ public void testOldPercolatorIndex() throws Exception {
             .setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc("message", "the quick brown fox jumps over the lazy dog"))
             .get();
 
-        assertThat(percolateResponse.getCount(), equalTo(3L));
-        assertThat(percolateResponse.getMatches().length, equalTo(3));
+        assertThat(percolateResponse.getCount(), equalTo(4L));
+        assertThat(percolateResponse.getMatches().length, equalTo(4));
         assertThat(percolateResponse.getMatches()[0].getId().string(), equalTo("1"));
         assertThat(percolateResponse.getMatches()[1].getId().string(), equalTo("2"));
         assertThat(percolateResponse.getMatches()[2].getId().string(), equalTo("4"));
+        assertThat(percolateResponse.getMatches()[3].getId().string(), equalTo("0"));
 
         // add an extra query and verify the results
         client().prepareIndex(INDEX_NAME, ".percolator", "5")
@@ -135,11 +136,13 @@ public void testOldPercolatorIndex() throws Exception {
             .setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc("message", "the quick brown fox jumps over the lazy dog"))
             .get();
 
-        assertThat(percolateResponse.getCount(), equalTo(4L));
-        assertThat(percolateResponse.getMatches().length, equalTo(4));
+        assertThat(percolateResponse.getCount(), equalTo(5L));
+        assertThat(percolateResponse.getMatches().length, equalTo(5));
         assertThat(percolateResponse.getMatches()[0].getId().string(), equalTo("1"));
         assertThat(percolateResponse.getMatches()[1].getId().string(), equalTo("2"));
         assertThat(percolateResponse.getMatches()[2].getId().string(), equalTo("4"));
+        assertThat(percolateResponse.getMatches()[3].getId().string(), equalTo("0"));
+        assertThat(percolateResponse.getMatches()[4].getId().string(), equalTo("5"));
     }
 
     private void setupNode() throws Exception {
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorClientYamlTestSuiteIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorClientYamlTestSuiteIT.java
index 24e92715ede80..7012f7bd436a7 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorClientYamlTestSuiteIT.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorClientYamlTestSuiteIT.java
@@ -24,9 +24,6 @@
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-
-import java.io.IOException;
 
 public class PercolatorClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     public PercolatorClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
@@ -34,7 +31,7 @@ public PercolatorClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate tes
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
-        return ESClientYamlSuiteTestCase.createParameters(0, 1);
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java
index 621cb07d3cdf6..af2d336fb6254 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java
@@ -26,10 +26,11 @@
 import org.apache.lucene.index.PrefixCodedTerms;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.memory.MemoryIndex;
-import org.apache.lucene.queries.TermsQuery;
 import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.PhraseQuery;
+import org.apache.lucene.search.TermInSetQuery;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TermRangeQuery;
 import org.apache.lucene.search.join.ScoreMode;
@@ -54,8 +55,7 @@
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.BoostingQueryBuilder;
 import org.elasticsearch.index.query.ConstantScoreQueryBuilder;
-import org.elasticsearch.index.query.HasChildQueryBuilder;
-import org.elasticsearch.index.query.HasParentQueryBuilder;
+import org.elasticsearch.index.query.DisMaxQueryBuilder;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
@@ -64,6 +64,9 @@
 import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
 import org.elasticsearch.index.query.functionscore.RandomScoreFunctionBuilder;
 import org.elasticsearch.indices.TermsLookup;
+import org.elasticsearch.join.ParentJoinPlugin;
+import org.elasticsearch.join.query.HasChildQueryBuilder;
+import org.elasticsearch.join.query.HasParentQueryBuilder;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.MockScriptPlugin;
 import org.elasticsearch.script.Script;
@@ -108,7 +111,7 @@ public class PercolatorFieldMapperTests extends ESSingleNodeTestCase {
 
     @Override
     protected Collection<Class<? extends Plugin>> getPlugins() {
-        return pluginList(InternalSettingsPlugin.class, PercolatorPlugin.class, FoolMeScriptPlugin.class);
+        return pluginList(InternalSettingsPlugin.class, PercolatorPlugin.class, FoolMeScriptPlugin.class, ParentJoinPlugin.class);
     }
 
     @Before
@@ -131,8 +134,8 @@ public void init() throws Exception {
     }
 
     private void addQueryMapping() throws Exception {
-        typeName = randomAsciiOfLength(4);
-        fieldName = randomAsciiOfLength(4);
+        typeName = randomAlphaOfLength(4);
+        fieldName = randomAlphaOfLength(4);
         String percolatorMapper = XContentFactory.jsonBuilder().startObject().startObject(typeName)
                 .startObject("properties").startObject(fieldName).field("type", "percolator").endObject().endObject()
                 .endObject().endObject().string();
@@ -207,10 +210,13 @@ public void testCreateCandidateQuery() throws Exception {
 
         IndexReader indexReader = memoryIndex.createSearcher().getIndexReader();
 
-        TermsQuery termsQuery = (TermsQuery) fieldType.createCandidateQuery(indexReader);
+        BooleanQuery candidateQuery = (BooleanQuery) fieldType.createCandidateQuery(indexReader);
+        assertEquals(2, candidateQuery.clauses().size());
+        assertEquals(Occur.SHOULD, candidateQuery.clauses().get(0).getOccur());
+        TermInSetQuery termsQuery = (TermInSetQuery) candidateQuery.clauses().get(0).getQuery();
 
         PrefixCodedTerms terms = termsQuery.getTermData();
-        assertThat(terms.size(), equalTo(15L));
+        assertThat(terms.size(), equalTo(14L));
         PrefixCodedTerms.TermIterator termIterator = terms.iterator();
         assertTermIterator(termIterator, "_field3\u0000me", fieldType.queryTermsField.name());
         assertTermIterator(termIterator, "_field3\u0000unhide", fieldType.queryTermsField.name());
@@ -226,7 +232,10 @@ public void testCreateCandidateQuery() throws Exception {
         assertTermIterator(termIterator, "field2\u0000some", fieldType.queryTermsField.name());
         assertTermIterator(termIterator, "field2\u0000text", fieldType.queryTermsField.name());
         assertTermIterator(termIterator, "field4\u0000123", fieldType.queryTermsField.name());
-        assertTermIterator(termIterator, EXTRACTION_FAILED, fieldType.extractionResultField.name());
+
+        assertEquals(Occur.SHOULD, candidateQuery.clauses().get(1).getOccur());
+        assertEquals(new TermQuery(new Term(fieldType.extractionResultField.name(), EXTRACTION_FAILED)),
+                candidateQuery.clauses().get(1).getQuery());
     }
 
     private void assertTermIterator(PrefixCodedTerms.TermIterator termIterator, String expectedValue, String expectedField) {
@@ -292,7 +301,8 @@ public void testQueryWithRewrite() throws Exception {
                 .field(fieldName, queryBuilder)
                 .endObject().bytes());
         BytesRef qbSource = doc.rootDoc().getFields(fieldType.queryBuilderField.name())[0].binaryValue();
-        assertQueryBuilder(qbSource, queryBuilder.rewrite(indexService.newQueryShardContext()));
+        assertQueryBuilder(qbSource, queryBuilder.rewrite(indexService.newQueryShardContext(
+                randomInt(20), null, () -> { throw new UnsupportedOperationException(); })));
     }
 
 
@@ -331,12 +341,9 @@ public void testAllowNoAdditionalSettings() throws Exception {
         String percolatorMapper = XContentFactory.jsonBuilder().startObject().startObject(typeName)
             .startObject("properties").startObject(fieldName).field("type", "percolator").field("index", "no").endObject().endObject()
             .endObject().endObject().string();
-        try {
-            mapperService.merge(typeName, new CompressedXContent(percolatorMapper), MapperService.MergeReason.MAPPING_UPDATE, true);
-            fail("MapperParsingException expected");
-        } catch (MapperParsingException e) {
-            assertThat(e.getMessage(), equalTo("Mapping definition for [" + fieldName + "] has unsupported parameters:  [index : no]"));
-        }
+        MapperParsingException e = expectThrows(MapperParsingException.class, () ->
+            mapperService.merge(typeName, new CompressedXContent(percolatorMapper), MapperService.MergeReason.MAPPING_UPDATE, true));
+        assertThat(e.getMessage(), containsString("Mapping definition for [" + fieldName + "] has unsupported parameters:  [index : no]"));
     }
 
     // multiple percolator fields are allowed in the mapping, but only one field can be used at index time.
@@ -418,53 +425,27 @@ public void testNestedPercolatorField() throws Exception {
         assertThat(e.getCause().getMessage(), equalTo("a document can only contain one percolator query"));
     }
 
-    public void testRangeQueryWithNowRangeIsForbidden() throws Exception {
-        addQueryMapping();
-        MapperParsingException e = expectThrows(MapperParsingException.class, () -> {
-            mapperService.documentMapper(typeName).parse("test", typeName, "1",
-                        jsonBuilder().startObject()
-                                .field(fieldName, rangeQuery("date_field").from("2016-01-01||/D").to("now"))
-                                .endObject().bytes());
-            }
-        );
-        assertThat(e.getCause(), instanceOf(IllegalArgumentException.class));
-        e = expectThrows(MapperParsingException.class, () -> {
-            mapperService.documentMapper(typeName).parse("test", typeName, "1",
-                        jsonBuilder().startObject()
-                                .field(fieldName, rangeQuery("date_field").from("2016-01-01||/D").to("now/D"))
-                                .endObject().bytes());
-                }
-        );
-        assertThat(e.getCause(), instanceOf(IllegalArgumentException.class));
-        e = expectThrows(MapperParsingException.class, () -> {
-            mapperService.documentMapper(typeName).parse("test", typeName, "1",
-                        jsonBuilder().startObject()
-                                .field(fieldName, rangeQuery("date_field").from("now-1d").to("now"))
-                                .endObject().bytes());
-                }
-        );
-        assertThat(e.getCause(), instanceOf(IllegalArgumentException.class));
-    }
-
     public void testUnsupportedQueries() {
         RangeQueryBuilder rangeQuery1 = new RangeQueryBuilder("field").from("2016-01-01||/D").to("2017-01-01||/D");
         RangeQueryBuilder rangeQuery2 = new RangeQueryBuilder("field").from("2016-01-01||/D").to("now");
         PercolatorFieldMapper.verifyQuery(rangeQuery1);
-        expectThrows(IllegalArgumentException.class, () -> PercolatorFieldMapper.verifyQuery(rangeQuery2));
-        PercolatorFieldMapper.verifyQuery(new BoolQueryBuilder().must(rangeQuery1));
+        PercolatorFieldMapper.verifyQuery(rangeQuery2);
+
+        HasChildQueryBuilder hasChildQuery = new HasChildQueryBuilder("_type", new MatchAllQueryBuilder(), ScoreMode.None);
         expectThrows(IllegalArgumentException.class, () ->
-                PercolatorFieldMapper.verifyQuery(new BoolQueryBuilder().must(rangeQuery2)));
+                PercolatorFieldMapper.verifyQuery(new BoolQueryBuilder().must(hasChildQuery)));
+        expectThrows(IllegalArgumentException.class, () ->
+                PercolatorFieldMapper.verifyQuery(new DisMaxQueryBuilder().add(hasChildQuery)));
         PercolatorFieldMapper.verifyQuery(new ConstantScoreQueryBuilder((rangeQuery1)));
         expectThrows(IllegalArgumentException.class, () ->
-                PercolatorFieldMapper.verifyQuery(new ConstantScoreQueryBuilder(rangeQuery2)));
+                PercolatorFieldMapper.verifyQuery(new ConstantScoreQueryBuilder(hasChildQuery)));
         PercolatorFieldMapper.verifyQuery(new BoostingQueryBuilder(rangeQuery1, new MatchAllQueryBuilder()));
         expectThrows(IllegalArgumentException.class, () ->
-                PercolatorFieldMapper.verifyQuery(new BoostingQueryBuilder(rangeQuery2, new MatchAllQueryBuilder())));
+                PercolatorFieldMapper.verifyQuery(new BoostingQueryBuilder(hasChildQuery, new MatchAllQueryBuilder())));
         PercolatorFieldMapper.verifyQuery(new FunctionScoreQueryBuilder(rangeQuery1, new RandomScoreFunctionBuilder()));
         expectThrows(IllegalArgumentException.class, () ->
-                PercolatorFieldMapper.verifyQuery(new FunctionScoreQueryBuilder(rangeQuery2, new RandomScoreFunctionBuilder())));
+                PercolatorFieldMapper.verifyQuery(new FunctionScoreQueryBuilder(hasChildQuery, new RandomScoreFunctionBuilder())));
 
-        HasChildQueryBuilder hasChildQuery = new HasChildQueryBuilder("_type", new MatchAllQueryBuilder(), ScoreMode.None);
         expectThrows(IllegalArgumentException.class, () -> PercolatorFieldMapper.verifyQuery(hasChildQuery));
         expectThrows(IllegalArgumentException.class, () -> PercolatorFieldMapper.verifyQuery(new BoolQueryBuilder().must(hasChildQuery)));
 
@@ -474,9 +455,11 @@ public void testUnsupportedQueries() {
     }
 
     private void assertQueryBuilder(BytesRef actual, QueryBuilder expected) throws IOException {
-        XContentParser sourceParser = PercolatorFieldMapper.QUERY_BUILDER_CONTENT_TYPE.xContent()
-                .createParser(actual.bytes, actual.offset, actual.length);
-        QueryParseContext qsc = indexService.newQueryShardContext().newParseContext(sourceParser);
+        XContentParser sourceParser = createParser(PercolatorFieldMapper.QUERY_BUILDER_CONTENT_TYPE.xContent(),
+                new BytesArray(actual));
+        QueryParseContext qsc = indexService.newQueryShardContext(
+                randomInt(20), null, () -> { throw new UnsupportedOperationException(); })
+                .newParseContext(sourceParser);
         assertThat(qsc.parseInnerQueryBuilder().get(), equalTo(expected));
     }
 
@@ -511,7 +494,7 @@ public void testImplicitlySetDefaultScriptLang() throws Exception {
             query.field("script", "return true");
         } else {
             query.startObject("script");
-            query.field("inline", "return true");
+            query.field("source", "return true");
             query.endObject();
         }
         query.endObject();
@@ -519,7 +502,7 @@ public void testImplicitlySetDefaultScriptLang() throws Exception {
 
         ParsedDocument doc = mapperService.documentMapper(typeName).parse("test", typeName, "1",
                 XContentFactory.jsonBuilder().startObject()
-                        .rawField(fieldName, new BytesArray(query.string()))
+                        .rawField(fieldName, new BytesArray(query.string()), query.contentType())
                         .endObject().bytes());
         BytesRef querySource = doc.rootDoc().getFields(fieldType.queryBuilderField.name())[0].binaryValue();
         Map<String, Object> parsedQuery = XContentHelper.convertToMap(new BytesArray(querySource), true).v2();
@@ -535,7 +518,7 @@ public void testImplicitlySetDefaultScriptLang() throws Exception {
             query.field("script", "return true");
         } else {
             query.startObject("script");
-            query.field("inline", "return true");
+            query.field("source", "return true");
             query.endObject();
         }
         query.endObject();
@@ -546,7 +529,7 @@ public void testImplicitlySetDefaultScriptLang() throws Exception {
 
         doc = mapperService.documentMapper(typeName).parse("test", typeName, "1",
                 XContentFactory.jsonBuilder().startObject()
-                        .rawField(fieldName, new BytesArray(query.string()))
+                        .rawField(fieldName, new BytesArray(query.string()), query.contentType())
                         .endObject().bytes());
         querySource = doc.rootDoc().getFields(fieldType.queryBuilderField.name())[0].binaryValue();
         parsedQuery = XContentHelper.convertToMap(new BytesArray(querySource), true).v2();
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorIT.java
index 7d10b831bc86c..c1e048c31bd61 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorIT.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorIT.java
@@ -41,6 +41,7 @@
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.QueryShardException;
 import org.elasticsearch.index.query.functionscore.WeightBuilder;
+import org.elasticsearch.join.ParentJoinPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -49,7 +50,6 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -65,10 +65,9 @@
 import static org.elasticsearch.common.xcontent.XContentFactory.smileBuilder;
 import static org.elasticsearch.common.xcontent.XContentFactory.yamlBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
-import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery;
 import static org.elasticsearch.index.query.QueryBuilders.functionScoreQuery;
 import static org.elasticsearch.index.query.QueryBuilders.geoShapeQuery;
-import static org.elasticsearch.index.query.QueryBuilders.hasChildQuery;
+import static org.elasticsearch.join.query.JoinQueryBuilders.hasChildQuery;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
 import static org.elasticsearch.index.query.QueryBuilders.rangeQuery;
@@ -94,14 +93,19 @@ public class PercolatorIT extends ESIntegTestCase {
     private static final String INDEX_NAME = "queries";
     private static final String TYPE_NAME = "query";
 
+    @Override
+    protected boolean ignoreExternalCluster() {
+        return true;
+    }
+
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Collections.singleton(PercolatorPlugin.class);
+        return Arrays.asList(ParentJoinPlugin.class, PercolatorPlugin.class);
     }
 
     @Override
     protected Collection<Class<? extends Plugin>> transportClientPlugins() {
-        return Collections.singleton(PercolatorPlugin.class);
+        return nodePlugins();
     }
 
     public void testSimple1() throws Exception {
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java
index b21c131a62577..10212deb80409 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java
@@ -19,26 +19,44 @@
 package org.elasticsearch.percolator;
 
 import org.apache.lucene.search.join.ScoreMode;
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.action.search.MultiSearchResponse;
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
+import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.query.MatchPhraseQueryBuilder;
 import org.elasticsearch.index.query.MultiMatchQueryBuilder;
 import org.elasticsearch.index.query.Operator;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.script.MockScriptPlugin;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
+import org.elasticsearch.search.lookup.LeafDocLookup;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
+import java.io.IOException;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.Function;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.common.xcontent.XContentFactory.smileBuilder;
+import static org.elasticsearch.common.xcontent.XContentFactory.yamlBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
 import static org.elasticsearch.index.query.QueryBuilders.commonTermsQuery;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
@@ -49,16 +67,56 @@
 import static org.elasticsearch.index.query.QueryBuilders.spanNotQuery;
 import static org.elasticsearch.index.query.QueryBuilders.spanTermQuery;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
+import static org.elasticsearch.percolator.PercolateSourceBuilder.docBuilder;
+import static org.elasticsearch.percolator.PercolatorTestUtil.assertMatchCount;
+import static org.elasticsearch.percolator.PercolatorTestUtil.convertFromTextArray;
+import static org.elasticsearch.percolator.PercolatorTestUtil.preparePercolate;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits;
+import static org.hamcrest.Matchers.arrayContainingInAnyOrder;
+import static org.hamcrest.Matchers.arrayWithSize;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
 
 public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
 
     @Override
     protected Collection<Class<? extends Plugin>> getPlugins() {
-        return Collections.singleton(PercolatorPlugin.class);
+        return Arrays.asList(PercolatorPlugin.class, CustomScriptPlugin.class);
+    }
+
+    public static class CustomScriptPlugin extends MockScriptPlugin {
+        @Override
+        protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
+            Map<String, Function<Map<String, Object>, Object>> scripts = new HashMap<>();
+            scripts.put("1==1", vars -> Boolean.TRUE);
+            scripts.put("use_fielddata_please", vars -> {
+                LeafDocLookup leafDocLookup = (LeafDocLookup) vars.get("_doc");
+                ScriptDocValues scriptDocValues = leafDocLookup.get("employees.name");
+                return "virginia_potts".equals(scriptDocValues.get(0));
+            });
+            return scripts;
+        }
+    }
+
+    public void testPercolateScriptQuery() throws IOException {
+        client().admin().indices().prepareCreate("index").addMapping("type", "query", "type=percolator").get();
+        ensureGreen();
+        client().prepareIndex("index", "type", "1")
+            .setSource(jsonBuilder().startObject().field("query", QueryBuilders.scriptQuery(
+                new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "1==1", Collections.emptyMap()))).endObject())
+            .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+            .execute().actionGet();
+        PercolateResponse response = preparePercolate(client())
+            .setIndices("index").setDocumentType("type")
+            .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", "b").endObject()))
+            .execute().actionGet();
+        assertMatchCount(response, 1L);
+        assertThat(response.getMatches(), arrayWithSize(1));
+        assertThat(convertFromTextArray(response.getMatches(), "index"), arrayContainingInAnyOrder("1"));
     }
 
     public void testPercolatorQuery() throws Exception {
@@ -83,7 +141,7 @@ public void testPercolatorQuery() throws Exception {
         BytesReference source = jsonBuilder().startObject().endObject().bytes();
         logger.info("percolating empty doc");
         SearchResponse response = client().prepareSearch()
-                .setQuery(new PercolateQueryBuilder("query", "type", source))
+                .setQuery(new PercolateQueryBuilder("query", "type", source, XContentType.JSON))
                 .get();
         assertHitCount(response, 1);
         assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
@@ -91,7 +149,7 @@ public void testPercolatorQuery() throws Exception {
         source = jsonBuilder().startObject().field("field1", "value").endObject().bytes();
         logger.info("percolating doc with 1 field");
         response = client().prepareSearch()
-                .setQuery(new PercolateQueryBuilder("query", "type", source))
+                .setQuery(new PercolateQueryBuilder("query", "type", source, XContentType.JSON))
                 .addSort("_uid", SortOrder.ASC)
                 .get();
         assertHitCount(response, 2);
@@ -101,7 +159,7 @@ public void testPercolatorQuery() throws Exception {
         source = jsonBuilder().startObject().field("field1", "value").field("field2", "value").endObject().bytes();
         logger.info("percolating doc with 2 fields");
         response = client().prepareSearch()
-                .setQuery(new PercolateQueryBuilder("query", "type", source))
+                .setQuery(new PercolateQueryBuilder("query", "type", source, XContentType.JSON))
                 .addSort("_uid", SortOrder.ASC)
                 .get();
         assertHitCount(response, 3);
@@ -112,7 +170,7 @@ public void testPercolatorQuery() throws Exception {
 
     public void testPercolatorRangeQueries() throws Exception {
         createIndex("test", client().admin().indices().prepareCreate("test")
-                .addMapping("type", "field1", "type=long", "field2", "type=double", "field3", "type=ip")
+                .addMapping("type", "field1", "type=long", "field2", "type=double", "field3", "type=ip", "field4", "type=date")
                 .addMapping("queries", "query", "type=percolator")
         );
 
@@ -155,12 +213,17 @@ public void testPercolatorRangeQueries() throws Exception {
                         .must(rangeQuery("field3").from("192.168.1.0").to("192.168.1.5"))
                         .must(rangeQuery("field3").from("192.168.1.5").to("192.168.1.10"))
                 ).endObject()).get();
+        client().prepareIndex("test", "queries", "10")
+                .setSource(jsonBuilder().startObject().field("query", boolQuery()
+                        .must(rangeQuery("field4").from("2010-01-01").to("2018-01-01"))
+                        .must(rangeQuery("field4").from("2010-01-01").to("now"))
+                ).endObject()).get();
         client().admin().indices().prepareRefresh().get();
 
         // Test long range:
         BytesReference source = jsonBuilder().startObject().field("field1", 12).endObject().bytes();
         SearchResponse response = client().prepareSearch()
-                .setQuery(new PercolateQueryBuilder("query", "type", source))
+                .setQuery(new PercolateQueryBuilder("query", "type", source, XContentType.JSON))
                 .get();
         assertHitCount(response, 2);
         assertThat(response.getHits().getAt(0).getId(), equalTo("3"));
@@ -168,7 +231,7 @@ public void testPercolatorRangeQueries() throws Exception {
 
         source = jsonBuilder().startObject().field("field1", 11).endObject().bytes();
         response = client().prepareSearch()
-                .setQuery(new PercolateQueryBuilder("query", "type", source))
+                .setQuery(new PercolateQueryBuilder("query", "type", source, XContentType.JSON))
                 .get();
         assertHitCount(response, 1);
         assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
@@ -176,7 +239,7 @@ public void testPercolatorRangeQueries() throws Exception {
         // Test double range:
         source = jsonBuilder().startObject().field("field2", 12).endObject().bytes();
         response = client().prepareSearch()
-                .setQuery(new PercolateQueryBuilder("query", "type", source))
+                .setQuery(new PercolateQueryBuilder("query", "type", source, XContentType.JSON))
                 .get();
         assertHitCount(response, 2);
         assertThat(response.getHits().getAt(0).getId(), equalTo("6"));
@@ -184,7 +247,7 @@ public void testPercolatorRangeQueries() throws Exception {
 
         source = jsonBuilder().startObject().field("field2", 11).endObject().bytes();
         response = client().prepareSearch()
-                .setQuery(new PercolateQueryBuilder("query", "type", source))
+                .setQuery(new PercolateQueryBuilder("query", "type", source, XContentType.JSON))
                 .get();
         assertHitCount(response, 1);
         assertThat(response.getHits().getAt(0).getId(), equalTo("4"));
@@ -192,7 +255,7 @@ public void testPercolatorRangeQueries() throws Exception {
         // Test IP range:
         source = jsonBuilder().startObject().field("field3", "192.168.1.5").endObject().bytes();
         response = client().prepareSearch()
-                .setQuery(new PercolateQueryBuilder("query", "type", source))
+                .setQuery(new PercolateQueryBuilder("query", "type", source, XContentType.JSON))
                 .get();
         assertHitCount(response, 2);
         assertThat(response.getHits().getAt(0).getId(), equalTo("9"));
@@ -200,10 +263,18 @@ public void testPercolatorRangeQueries() throws Exception {
 
         source = jsonBuilder().startObject().field("field3", "192.168.1.4").endObject().bytes();
         response = client().prepareSearch()
-                .setQuery(new PercolateQueryBuilder("query", "type", source))
+                .setQuery(new PercolateQueryBuilder("query", "type", source, XContentType.JSON))
                 .get();
         assertHitCount(response, 1);
         assertThat(response.getHits().getAt(0).getId(), equalTo("7"));
+
+        // Test date range:
+        source = jsonBuilder().startObject().field("field4", "2016-05-15").endObject().bytes();
+        response = client().prepareSearch()
+                .setQuery(new PercolateQueryBuilder("query", "type", source, XContentType.JSON))
+                .get();
+        assertHitCount(response, 1);
+        assertThat(response.getHits().getAt(0).getId(), equalTo("10"));
     }
 
     public void testPercolatorQueryExistingDocument() throws Exception {
@@ -224,7 +295,7 @@ public void testPercolatorQueryExistingDocument() throws Exception {
                         .must(matchQuery("field2", "value"))
                 ).endObject()).get();
 
-        client().prepareIndex("test", "type", "1").setSource("{}").get();
+        client().prepareIndex("test", "type", "1").setSource("{}", XContentType.JSON).get();
         client().prepareIndex("test", "type", "2").setSource("field1", "value").get();
         client().prepareIndex("test", "type", "3").setSource("field1", "value", "field2", "value").get();
         client().admin().indices().prepareRefresh().get();
@@ -266,7 +337,7 @@ public void testPercolatorQueryExistingDocumentSourceDisabled() throws Exception
             .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
             .get();
 
-        client().prepareIndex("test", "type", "1").setSource("{}").get();
+        client().prepareIndex("test", "type", "1").setSource("{}", XContentType.JSON).get();
         client().admin().indices().prepareRefresh().get();
 
         logger.info("percolating empty doc with source disabled");
@@ -337,7 +408,7 @@ public void testPercolatorSpecificQueries()  throws Exception {
                 .field("field2", "the quick brown fox falls down into the well")
                 .endObject().bytes();
         SearchResponse response = client().prepareSearch()
-                .setQuery(new PercolateQueryBuilder("query", "type", source))
+                .setQuery(new PercolateQueryBuilder("query", "type", source, XContentType.JSON))
                 .addSort("_uid", SortOrder.ASC)
                 .get();
         assertHitCount(response, 4);
@@ -384,7 +455,7 @@ public void testPercolatorQueryWithHighlighting() throws Exception {
                 .field("field1", "The quick brown fox jumps over the lazy dog")
                 .endObject().bytes();
         SearchResponse searchResponse = client().prepareSearch()
-                .setQuery(new PercolateQueryBuilder("query", "type", document))
+                .setQuery(new PercolateQueryBuilder("query", "type", document, XContentType.JSON))
                 .highlighter(new HighlightBuilder().field("field1"))
                 .addSort("_uid", SortOrder.ASC)
                 .get();
@@ -418,7 +489,7 @@ public void testTakePositionOffsetGapIntoAccount() throws Exception {
         client().admin().indices().prepareRefresh().get();
 
         SearchResponse response = client().prepareSearch().setQuery(
-                new PercolateQueryBuilder("query", "type", new BytesArray("{\"field\" : [\"brown\", \"fox\"]}"))
+                new PercolateQueryBuilder("query", "type", new BytesArray("{\"field\" : [\"brown\", \"fox\"]}"), XContentType.JSON)
         ).get();
         assertHitCount(response, 1);
         assertThat(response.getHits().getAt(0).getId(), equalTo("2"));
@@ -426,7 +497,7 @@ public void testTakePositionOffsetGapIntoAccount() throws Exception {
 
 
     public void testManyPercolatorFields() throws Exception {
-        String queryFieldName = randomAsciiOfLength(8);
+        String queryFieldName = randomAlphaOfLength(8);
         createIndex("test", client().admin().indices().prepareCreate("test")
                 .addMapping("doc_type", "field", "type=keyword")
                 .addMapping("query_type1", queryFieldName, "type=percolator")
@@ -446,7 +517,7 @@ public void testManyPercolatorFields() throws Exception {
     }
 
     public void testWithMultiplePercolatorFields() throws Exception {
-        String queryFieldName = randomAsciiOfLength(8);
+        String queryFieldName = randomAlphaOfLength(8);
         createIndex("test1", client().admin().indices().prepareCreate("test1")
                 .addMapping("doc_type", "field", "type=keyword")
                 .addMapping("query_type", queryFieldName, "type=percolator"));
@@ -478,7 +549,7 @@ public void testWithMultiplePercolatorFields() throws Exception {
 
         BytesReference source = jsonBuilder().startObject().field("field", "value").endObject().bytes();
         SearchResponse response = client().prepareSearch()
-                .setQuery(new PercolateQueryBuilder(queryFieldName, "doc_type", source))
+                .setQuery(new PercolateQueryBuilder(queryFieldName, "doc_type", source, XContentType.JSON))
                 .setIndices("test1")
                 .get();
         assertHitCount(response, 1);
@@ -487,7 +558,7 @@ public void testWithMultiplePercolatorFields() throws Exception {
         assertThat(response.getHits().getAt(0).index(), equalTo("test1"));
 
         response = client().prepareSearch()
-                .setQuery(new PercolateQueryBuilder("object_field." + queryFieldName, "doc_type", source))
+                .setQuery(new PercolateQueryBuilder("object_field." + queryFieldName, "doc_type", source, XContentType.JSON))
                 .setIndices("test2")
                 .get();
         assertHitCount(response, 1);
@@ -537,7 +608,7 @@ public void testPercolateQueryWithNestedDocuments() throws Exception {
                                     .startObject().field("name", "virginia potts").endObject()
                                     .startObject().field("name", "tony stark").endObject()
                                 .endArray()
-                            .endObject().bytes()))
+                            .endObject().bytes(), XContentType.JSON))
                 .addSort("_doc", SortOrder.ASC)
                 .get();
         assertHitCount(response, 1);
@@ -551,17 +622,213 @@ public void testPercolateQueryWithNestedDocuments() throws Exception {
                                     .startObject().field("name", "virginia stark").endObject()
                                     .startObject().field("name", "tony stark").endObject()
                                 .endArray()
-                            .endObject().bytes()))
+                            .endObject().bytes(), XContentType.JSON))
                 .addSort("_doc", SortOrder.ASC)
                 .get();
         assertHitCount(response, 0);
 
         response = client().prepareSearch()
                 .setQuery(new PercolateQueryBuilder("query", "employee",
-                        XContentFactory.jsonBuilder().startObject().field("companyname", "notstark").endObject().bytes()))
+                        XContentFactory.jsonBuilder().startObject().field("companyname", "notstark").endObject().bytes(),
+                    XContentType.JSON))
                 .addSort("_doc", SortOrder.ASC)
                 .get();
         assertHitCount(response, 0);
     }
 
+    public void testPercolateQueryWithNestedDocuments_doNotLeakBitsetCacheEntries() throws Exception {
+        XContentBuilder mapping = XContentFactory.jsonBuilder();
+        mapping.startObject().startObject("properties").startObject("companyname").field("type", "text").endObject()
+            .startObject("query").field("type", "percolator").endObject()
+            .startObject("employee").field("type", "nested").startObject("properties")
+            .startObject("name").field("type", "text").endObject().endObject().endObject().endObject()
+            .endObject();
+        createIndex("test", client().admin().indices().prepareCreate("test")
+            // to avoid normal document from being cached by BitsetFilterCache
+            .setSettings(Settings.builder().put(BitsetFilterCache.INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING.getKey(), false))
+            .addMapping("employee", mapping)
+        );
+        client().prepareIndex("test", "employee", "q1").setSource(jsonBuilder().startObject()
+            .field("query", QueryBuilders.nestedQuery("employee",
+                QueryBuilders.matchQuery("employee.name", "virginia potts").operator(Operator.AND), ScoreMode.Avg)
+            ).endObject())
+            .get();
+        client().admin().indices().prepareRefresh().get();
+
+        for (int i = 0; i < 32; i++) {
+            SearchResponse response = client().prepareSearch()
+                .setQuery(new PercolateQueryBuilder("query", "employee",
+                    XContentFactory.jsonBuilder()
+                        .startObject().field("companyname", "stark")
+                        .startArray("employee")
+                        .startObject().field("name", "virginia potts").endObject()
+                        .startObject().field("name", "tony stark").endObject()
+                        .endArray()
+                        .endObject().bytes(), XContentType.JSON))
+                .addSort("_doc", SortOrder.ASC)
+                // size 0, because other wise load bitsets for normal document in FetchPhase#findRootDocumentIfNested(...)
+                .setSize(0)
+                .get();
+            assertHitCount(response, 1);
+        }
+
+        // We can't check via api... because BitsetCacheListener requires that it can extract shardId from index reader
+        // and for percolator it can't do that, but that means we don't keep track of
+        // memory for BitsetCache in case of percolator
+        long bitsetSize = client().admin().cluster().prepareClusterStats().get()
+            .getIndicesStats().getSegments().getBitsetMemoryInBytes();
+        assertEquals("The percolator works with in-memory index and therefor shouldn't use bitset cache", 0L, bitsetSize);
+    }
+
+    public void testPercolateQueryWithNestedDocuments_doLeakFieldDataCacheEntries() throws Exception {
+        XContentBuilder mapping = XContentFactory.jsonBuilder();
+        mapping.startObject();
+        {
+            mapping.startObject("properties");
+            {
+                mapping.startObject("query");
+                mapping.field("type", "percolator");
+                mapping.endObject();
+            }
+            {
+                mapping.startObject("companyname");
+                mapping.field("type", "text");
+                mapping.endObject();
+            }
+            {
+                mapping.startObject("employees");
+                mapping.field("type", "nested");
+                {
+                    mapping.startObject("properties");
+                    {
+                        mapping.startObject("name");
+                        mapping.field("type", "text");
+                        mapping.field("fielddata", true);
+                        mapping.endObject();
+                    }
+                    mapping.endObject();
+                }
+                mapping.endObject();
+            }
+            mapping.endObject();
+        }
+        mapping.endObject();
+        createIndex("test", client().admin().indices().prepareCreate("test")
+            .addMapping("employee", mapping)
+        );
+        Script script = new Script(ScriptType.INLINE, MockScriptPlugin.NAME, "use_fielddata_please", Collections.emptyMap());
+        client().prepareIndex("test", "employee", "q1").setSource(jsonBuilder().startObject()
+            .field("query", QueryBuilders.nestedQuery("employees",
+                QueryBuilders.scriptQuery(script), ScoreMode.Avg)
+            ).endObject()).get();
+        client().admin().indices().prepareRefresh().get();
+        XContentBuilder doc = jsonBuilder();
+        doc.startObject();
+        {
+            doc.field("companyname", "stark");
+            doc.startArray("employees");
+            {
+                doc.startObject();
+                doc.field("name", "virginia_potts");
+                doc.endObject();
+            }
+            {
+                doc.startObject();
+                doc.field("name", "tony_stark");
+                doc.endObject();
+            }
+            doc.endArray();
+        }
+        doc.endObject();
+        for (int i = 0; i < 32; i++) {
+            SearchResponse response = client().prepareSearch()
+                .setQuery(new PercolateQueryBuilder("query", "employee", doc.bytes(), XContentType.JSON))
+                .addSort("_doc", SortOrder.ASC)
+                .get();
+            assertHitCount(response, 1);
+        }
+
+        long fieldDataSize = client().admin().cluster().prepareClusterStats().get()
+            .getIndicesStats().getFieldData().getMemorySizeInBytes();
+        assertEquals("The percolator works with in-memory index and therefor shouldn't use field-data cache", 0L, fieldDataSize);
+    }
+
+    public void testPercolatorQueryViaMultiSearch() throws Exception {
+        createIndex("test", client().admin().indices().prepareCreate("test")
+            .addMapping("type", "field1", "type=text")
+            .addMapping("queries", "query", "type=percolator")
+        );
+
+        client().prepareIndex("test", "queries", "1")
+            .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "b")).field("a", "b").endObject())
+            .execute().actionGet();
+        client().prepareIndex("test", "queries", "2")
+            .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "c")).endObject())
+            .execute().actionGet();
+        client().prepareIndex("test", "queries", "3")
+            .setSource(jsonBuilder().startObject().field("query", boolQuery()
+                .must(matchQuery("field1", "b"))
+                .must(matchQuery("field1", "c"))
+            ).endObject())
+            .execute().actionGet();
+        client().prepareIndex("test", "queries", "4")
+            .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
+            .execute().actionGet();
+        client().prepareIndex("test", "type", "1")
+            .setSource(jsonBuilder().startObject().field("field1", "c").endObject())
+            .execute().actionGet();
+        client().admin().indices().prepareRefresh().get();
+
+        MultiSearchResponse response = client().prepareMultiSearch()
+            .add(client().prepareSearch("test")
+                .setQuery(new PercolateQueryBuilder("query", "type",
+                    jsonBuilder().startObject().field("field1", "b").endObject().bytes(), XContentType.JSON)))
+            .add(client().prepareSearch("test")
+                .setQuery(new PercolateQueryBuilder("query", "type",
+                    yamlBuilder().startObject().field("field1", "c").endObject().bytes(), XContentType.JSON)))
+            .add(client().prepareSearch("test")
+                .setQuery(new PercolateQueryBuilder("query", "type",
+                    smileBuilder().startObject().field("field1", "b c").endObject().bytes(), XContentType.JSON)))
+            .add(client().prepareSearch("test")
+                .setQuery(new PercolateQueryBuilder("query", "type",
+                    jsonBuilder().startObject().field("field1", "d").endObject().bytes(), XContentType.JSON)))
+            .add(client().prepareSearch("test")
+                .setQuery(new PercolateQueryBuilder("query", "type", "test", "type", "1", null, null, null)))
+            .add(client().prepareSearch("test") // non existing doc, so error element
+                .setQuery(new PercolateQueryBuilder("query", "type", "test", "type", "2", null, null, null)))
+            .get();
+
+        MultiSearchResponse.Item item = response.getResponses()[0];
+        assertHitCount(item.getResponse(), 2L);
+        assertSearchHits(item.getResponse(), "1", "4");
+        assertThat(item.getFailureMessage(), nullValue());
+
+        item = response.getResponses()[1];
+        assertHitCount(item.getResponse(), 2L);
+        assertSearchHits(item.getResponse(), "2", "4");
+        assertThat(item.getFailureMessage(), nullValue());
+
+        item = response.getResponses()[2];
+        assertHitCount(item.getResponse(), 4L);
+        assertSearchHits(item.getResponse(), "1", "2", "3", "4");
+        assertThat(item.getFailureMessage(), nullValue());
+
+        item = response.getResponses()[3];
+        assertHitCount(item.getResponse(), 1L);
+        assertSearchHits(item.getResponse(), "4");
+        assertThat(item.getFailureMessage(), nullValue());
+
+        item = response.getResponses()[4];
+        assertHitCount(item.getResponse(), 2L);
+        assertSearchHits(item.getResponse(), "2", "4");
+        assertThat(item.getFailureMessage(), nullValue());
+
+        item = response.getResponses()[5];
+        assertThat(item.getResponse(), nullValue());
+        assertThat(item.getFailureMessage(), notNullValue());
+        assertThat(item.getFailureMessage(), equalTo("all shards failed"));
+        assertThat(ExceptionsHelper.unwrapCause(item.getFailure().getCause()).getMessage(),
+            containsString("[test/type/2] couldn't be found"));
+    }
+
 }
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java
index 1b8b123aa13bf..558d237f1259c 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java
@@ -21,15 +21,17 @@
 import org.apache.lucene.index.Term;
 import org.apache.lucene.queries.BlendedTermQuery;
 import org.apache.lucene.queries.CommonTermsQuery;
-import org.apache.lucene.queries.TermsQuery;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.DisjunctionMaxQuery;
 import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.MatchNoDocsQuery;
+import org.apache.lucene.search.MultiPhraseQuery;
 import org.apache.lucene.search.PhraseQuery;
 import org.apache.lucene.search.SynonymQuery;
+import org.apache.lucene.search.TermInSetQuery;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TermRangeQuery;
 import org.apache.lucene.search.spans.SpanFirstQuery;
@@ -38,7 +40,6 @@
 import org.apache.lucene.search.spans.SpanOrQuery;
 import org.apache.lucene.search.spans.SpanTermQuery;
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
 import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery;
 import org.elasticsearch.common.lucene.search.function.RandomScoreFunction;
 import org.elasticsearch.percolator.QueryAnalyzer.Result;
@@ -71,7 +72,7 @@ public void testExtractQueryMetadata_termQuery() {
     }
 
     public void testExtractQueryMetadata_termsQuery() {
-        TermsQuery termsQuery = new TermsQuery("_field", new BytesRef("_term1"), new BytesRef("_term2"));
+        TermInSetQuery termsQuery = new TermInSetQuery("_field", new BytesRef("_term1"), new BytesRef("_term2"));
         Result result = analyze(termsQuery);
         assertThat(result.verified, is(true));
         List<Term> terms = new ArrayList<>(result.terms);
@@ -81,18 +82,6 @@ public void testExtractQueryMetadata_termsQuery() {
         assertThat(terms.get(0).text(), equalTo("_term1"));
         assertThat(terms.get(1).field(), equalTo("_field"));
         assertThat(terms.get(1).text(), equalTo("_term2"));
-
-        // test with different fields
-        termsQuery = new TermsQuery(new Term("_field1", "_term1"), new Term("_field2", "_term2"));
-        result = analyze(termsQuery);
-        assertThat(result.verified, is(true));
-        terms = new ArrayList<>(result.terms);
-        Collections.sort(terms);
-        assertThat(terms.size(), equalTo(2));
-        assertThat(terms.get(0).field(), equalTo("_field1"));
-        assertThat(terms.get(0).text(), equalTo("_term1"));
-        assertThat(terms.get(1).field(), equalTo("_field2"));
-        assertThat(terms.get(1).text(), equalTo("_term2"));
     }
 
     public void testExtractQueryMetadata_phraseQuery() {
@@ -105,6 +94,21 @@ public void testExtractQueryMetadata_phraseQuery() {
         assertThat(terms.get(0).bytes(), equalTo(phraseQuery.getTerms()[0].bytes()));
     }
 
+    public void testExtractQueryMetadata_multiPhraseQuery() {
+        MultiPhraseQuery multiPhraseQuery = new MultiPhraseQuery.Builder()
+                .add(new Term("_field", "_long_term"))
+                .add(new Term[] {new Term("_field", "_long_term"), new Term("_field", "_term")})
+                .add(new Term[] {new Term("_field", "_long_term"), new Term("_field", "_very_long_term")})
+                .add(new Term[] {new Term("_field", "_very_long_term")})
+                .build();
+        Result result = analyze(multiPhraseQuery);
+        assertThat(result.verified, is(false));
+        List<Term> terms = new ArrayList<>(result.terms);
+        assertThat(terms.size(), equalTo(1));
+        assertThat(terms.get(0).field(), equalTo("_field"));
+        assertThat(terms.get(0).bytes().utf8ToString(), equalTo("_very_long_term"));
+    }
+
     public void testExtractQueryMetadata_booleanQuery() {
         BooleanQuery.Builder builder = new BooleanQuery.Builder();
         TermQuery termQuery1 = new TermQuery(new Term("_field", "_term"));
@@ -532,7 +536,7 @@ public void testSelectTermsListWithHighestSumOfTermLength() {
         while (sumTermLength > 0) {
             int length = randomInt(sumTermLength);
             shortestTerms1Length = Math.min(shortestTerms1Length, length);
-            terms1.add(new Term("field", randomAsciiOfLength(length)));
+            terms1.add(new Term("field", randomAlphaOfLength(length)));
             sumTermLength -= length;
         }
 
@@ -542,7 +546,7 @@ public void testSelectTermsListWithHighestSumOfTermLength() {
         while (sumTermLength > 0) {
             int length = randomInt(sumTermLength);
             shortestTerms2Length = Math.min(shortestTerms2Length, length);
-            terms2.add(new Term("field", randomAsciiOfLength(length)));
+            terms2.add(new Term("field", randomAlphaOfLength(length)));
             sumTermLength -= length;
         }
 
diff --git a/modules/percolator/src/test/resources/indices/percolator/bwc_index_2.0.0.zip b/modules/percolator/src/test/resources/indices/percolator/bwc_index_2.0.0.zip
index 43a8cceb1937d..777597abc45c0 100644
Binary files a/modules/percolator/src/test/resources/indices/percolator/bwc_index_2.0.0.zip and b/modules/percolator/src/test/resources/indices/percolator/bwc_index_2.0.0.zip differ
diff --git a/modules/percolator/src/test/resources/rest-api-spec/test/percolate/16_existing_doc.yaml b/modules/percolator/src/test/resources/rest-api-spec/test/percolate/16_existing_doc.yaml
index 2d15e2cd22224..4bb66ee7310cb 100644
--- a/modules/percolator/src/test/resources/rest-api-spec/test/percolate/16_existing_doc.yaml
+++ b/modules/percolator/src/test/resources/rest-api-spec/test/percolate/16_existing_doc.yaml
@@ -114,5 +114,8 @@
             term:
               tag: tag1
 
-  - match: {'matches': [{_index: percolator_index, _id: test_percolator, _score: 0.2876821}]}
+  - length: {matches: 1}
+  - match: {matches.0._index: percolator_index}
+  - match: {matches.0._id: test_percolator}
+  - is_true: matches.0._score
 
diff --git a/modules/reindex/build.gradle b/modules/reindex/build.gradle
index fa4c44d922474..ea04bcacb4577 100644
--- a/modules/reindex/build.gradle
+++ b/modules/reindex/build.gradle
@@ -23,23 +23,23 @@ esplugin {
   hasClientJar = true
 }
 
-integTest {
-  cluster {
-    // Whitelist reindexing from the local node so we can test it.
-    setting 'reindex.remote.whitelist', 'myself'
-  }
+integTestCluster {
+  // Whitelist reindexing from the local node so we can test it.
+  setting 'reindex.remote.whitelist', '127.0.0.1:*'
 }
 
 run {
   // Whitelist reindexing from the local node so we can test it.
-  setting 'reindex.remote.whitelist', 'myself'
+  setting 'reindex.remote.whitelist', '127.0.0.1:*'
 }
 
 dependencies {
-  compile "org.elasticsearch.client:rest:${version}"
+  compile "org.elasticsearch.client:elasticsearch-rest-client:${version}"
   // for http - testing reindex from remote
   testCompile project(path: ':modules:transport-netty3', configuration: 'runtime')
   testCompile project(path: ':modules:transport-netty4', configuration: 'runtime')
+  // for parent/child testing
+  testCompile project(path: ':modules:parent-join', configuration: 'runtime')
 }
 
 dependencyLicenses {
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java
index 32824e969d9fb..50e23bc6ce74c 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java
@@ -20,7 +20,9 @@
 package org.elasticsearch.index.reindex;
 
 import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
 import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
@@ -30,32 +32,56 @@
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.bulk.Retry;
-import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.index.reindex.ScrollableHitSource.SearchFailure;
+import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.client.ParentTaskAssigningClient;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
-import org.elasticsearch.index.reindex.ScrollableHitSource.SearchFailure;
+import org.elasticsearch.index.VersionType;
+import org.elasticsearch.index.mapper.IdFieldMapper;
+import org.elasticsearch.index.mapper.IndexFieldMapper;
+import org.elasticsearch.index.mapper.ParentFieldMapper;
+import org.elasticsearch.index.mapper.RoutingFieldMapper;
+import org.elasticsearch.index.mapper.SourceFieldMapper;
+import org.elasticsearch.index.mapper.TTLFieldMapper;
+import org.elasticsearch.index.mapper.TimestampFieldMapper;
+import org.elasticsearch.index.mapper.TypeFieldMapper;
+import org.elasticsearch.index.mapper.VersionFieldMapper;
+import org.elasticsearch.script.CompiledScript;
+import org.elasticsearch.script.ExecutableScript;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptContext;
+import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.sort.SortBuilder;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.BiFunction;
 
 import static java.lang.Math.max;
 import static java.lang.Math.min;
 import static java.util.Collections.emptyList;
 import static java.util.Collections.unmodifiableList;
 import static org.elasticsearch.action.bulk.BackoffPolicy.exponentialBackoff;
-import static org.elasticsearch.common.unit.TimeValue.timeValueNanos;
 import static org.elasticsearch.index.reindex.AbstractBulkByScrollRequest.SIZE_ALL_MATCHES;
+import static org.elasticsearch.common.unit.TimeValue.timeValueNanos;
 import static org.elasticsearch.rest.RestStatus.CONFLICT;
 import static org.elasticsearch.search.sort.SortBuilders.fieldSort;
 
@@ -65,8 +91,11 @@
  */
 public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBulkByScrollRequest<Request>> {
     protected final Logger logger;
-    protected final BulkByScrollTask task;
+    protected final WorkingBulkByScrollTask task;
     protected final ThreadPool threadPool;
+    protected final ScriptService scriptService;
+    protected final ClusterState clusterState;
+
     /**
      * The request for this action. Named mainRequest because we create lots of <code>request</code> variables all representing child
      * requests of this mainRequest.
@@ -77,21 +106,41 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
     private final Set<String> destinationIndices = Collections.newSetFromMap(new ConcurrentHashMap<>());
 
     private final ParentTaskAssigningClient client;
-    private final ActionListener<BulkIndexByScrollResponse> listener;
+    private final ActionListener<BulkByScrollResponse> listener;
     private final Retry bulkRetry;
     private final ScrollableHitSource scrollSource;
+    private final Settings settings;
+
+    /**
+     * This BiFunction is used to apply various changes depending of the Reindex action and  the search hit,
+     * from copying search hit metadata (parent, routing, etc) to potentially transforming the
+     * {@link RequestWrapper} completely.
+     */
+    private final BiFunction<RequestWrapper<?>, ScrollableHitSource.Hit, RequestWrapper<?>> scriptApplier;
+
+    public AbstractAsyncBulkByScrollAction(WorkingBulkByScrollTask task, Logger logger, ParentTaskAssigningClient client,
+                                           ThreadPool threadPool, Request mainRequest, ScriptService scriptService,
+                                           ClusterState clusterState, ActionListener<BulkByScrollResponse> listener) {
+        this(task, logger, client, threadPool, mainRequest, scriptService, clusterState, listener, client.settings());
+    }
 
-    public AbstractAsyncBulkByScrollAction(BulkByScrollTask task, Logger logger, ParentTaskAssigningClient client,
-                                           ThreadPool threadPool, Request mainRequest, ActionListener<BulkIndexByScrollResponse> listener) {
+    public AbstractAsyncBulkByScrollAction(WorkingBulkByScrollTask task, Logger logger, ParentTaskAssigningClient client,
+            ThreadPool threadPool, Request mainRequest, ScriptService scriptService, ClusterState clusterState,
+            ActionListener<BulkByScrollResponse> listener, Settings settings) {
         this.task = task;
         this.logger = logger;
         this.client = client;
+        this.settings = settings;
         this.threadPool = threadPool;
+        this.scriptService = scriptService;
+        this.clusterState = clusterState;
         this.mainRequest = mainRequest;
         this.listener = listener;
         BackoffPolicy backoffPolicy = buildBackoffPolicy();
-        bulkRetry = Retry.on(EsRejectedExecutionException.class).policy(BackoffPolicy.wrap(backoffPolicy, task::countBulkRetry));
+        bulkRetry = Retry.on(EsRejectedExecutionException.class)
+                .policy(BackoffPolicy.wrap(backoffPolicy, task::countBulkRetry)).using(threadPool);
         scrollSource = buildScrollableResultSource(backoffPolicy);
+        scriptApplier = Objects.requireNonNull(buildScriptApplier(), "script applier must not be null");
         /*
          * Default to sorting by doc. We can't do this in the request itself because it is normal to *add* to the sorts rather than replace
          * them and if we add _doc as the first sort by default then sorts will never work.... So we add it here, only if there isn't
@@ -104,12 +153,83 @@ public AbstractAsyncBulkByScrollAction(BulkByScrollTask task, Logger logger, Par
         mainRequest.getSearchRequest().source().version(needsSourceDocumentVersions());
     }
 
+    /**
+     * Build the {@link BiFunction} to apply to all {@link RequestWrapper}.
+     *
+     * Public for testings....
+     */
+    public BiFunction<RequestWrapper<?>, ScrollableHitSource.Hit, RequestWrapper<?>> buildScriptApplier() {
+        // The default script applier executes a no-op
+        return (request, searchHit) -> request;
+    }
+
     /**
      * Does this operation need the versions of the source documents?
      */
     protected abstract boolean needsSourceDocumentVersions();
 
-    protected abstract BulkRequest buildBulk(Iterable<? extends ScrollableHitSource.Hit> docs);
+    /**
+     * Build the {@link RequestWrapper} for a single search hit. This shouldn't handle
+     * metadata or scripting. That will be handled by copyMetadata and
+     * apply functions that can be overridden.
+     */
+    protected abstract RequestWrapper<?> buildRequest(ScrollableHitSource.Hit doc);
+
+    /**
+     * Copies the metadata from a hit to the request.
+     */
+    protected RequestWrapper<?> copyMetadata(RequestWrapper<?> request, ScrollableHitSource.Hit doc) {
+        request.setParent(doc.getParent());
+        copyRouting(request, doc.getRouting());
+
+        // Comes back as a Long but needs to be a string
+        Long timestamp = doc.getTimestamp();
+        if (timestamp != null) {
+            request.setTimestamp(timestamp.toString());
+        }
+        Long ttl = doc.getTTL();
+        if (ttl != null) {
+            request.setTtl(ttl);
+        }
+        return request;
+    }
+
+    /**
+     * Copy the routing from a search hit to the request.
+     */
+    protected void copyRouting(RequestWrapper<?> request, String routing) {
+        request.setRouting(routing);
+    }
+
+    /**
+     * Used to accept or ignore a search hit. Ignored search hits will be excluded
+     * from the bulk request. It is also where we fail on invalid search hits, like
+     * when the document has no source but it's required.
+     */
+    protected boolean accept(ScrollableHitSource.Hit doc) {
+        if (doc.getSource() == null) {
+            /*
+             * Either the document didn't store _source or we didn't fetch it for some reason. Since we don't allow the user to
+             * change the "fields" part of the search request it is unlikely that we got here because we didn't fetch _source.
+             * Thus the error message assumes that it wasn't stored.
+             */
+            throw new IllegalArgumentException("[" + doc.getIndex() + "][" + doc.getType() + "][" + doc.getId() + "] didn't store _source");
+        }
+        return true;
+    }
+
+    private BulkRequest buildBulk(Iterable<? extends ScrollableHitSource.Hit> docs) {
+        BulkRequest bulkRequest = new BulkRequest();
+        for (ScrollableHitSource.Hit doc : docs) {
+            if (accept(doc)) {
+                RequestWrapper<?> request = scriptApplier.apply(copyMetadata(buildRequest(doc), doc), doc);
+                if (request != null) {
+                    bulkRequest.add(request.self());
+                }
+            }
+        }
+        return bulkRequest;
+    }
 
     protected ScrollableHitSource buildScrollableResultSource(BackoffPolicy backoffPolicy) {
         return new ClientScrollableHitSource(logger, backoffPolicy, threadPool, task::countSearchRetry, this::finishHim, client,
@@ -119,16 +239,18 @@ protected ScrollableHitSource buildScrollableResultSource(BackoffPolicy backoffP
     /**
      * Build the response for reindex actions.
      */
-    protected BulkIndexByScrollResponse buildResponse(TimeValue took, List<BulkItemResponse.Failure> indexingFailures,
+    protected BulkByScrollResponse buildResponse(TimeValue took, List<BulkItemResponse.Failure> indexingFailures,
                                                       List<SearchFailure> searchFailures, boolean timedOut) {
-        return new BulkIndexByScrollResponse(took, task.getStatus(), indexingFailures, searchFailures, timedOut);
+        return new BulkByScrollResponse(took, task.getStatus(), indexingFailures, searchFailures, timedOut);
     }
 
     /**
      * Start the action by firing the initial search request.
      */
     public void start() {
+        logger.debug("[{}]: starting", task.getId());
         if (task.isCancelled()) {
+            logger.debug("[{}]: finishing early because the task was cancelled", task.getId());
             finishHim(null);
             return;
         }
@@ -147,7 +269,9 @@ public void start() {
      * @param response the scroll response to process
      */
     void onScrollResponse(TimeValue lastBatchStartTime, int lastBatchSize, ScrollableHitSource.Response response) {
+        logger.debug("[{}]: got scroll response with [{}] hits", task.getId(), response.getHits().size());
         if (task.isCancelled()) {
+            logger.debug("[{}]: finishing early because the task was cancelled", task.getId());
             finishHim(null);
             return;
         }
@@ -189,7 +313,9 @@ public void onFailure(Exception e) {
      * thread may be blocked by the user script.
      */
     void prepareBulkRequest(TimeValue thisBatchStartTime, ScrollableHitSource.Response response) {
+        logger.debug("[{}]: preparing bulk request", task.getId());
         if (task.isCancelled()) {
+            logger.debug("[{}]: finishing early because the task was cancelled", task.getId());
             finishHim(null);
             return;
         }
@@ -211,15 +337,11 @@ void prepareBulkRequest(TimeValue thisBatchStartTime, ScrollableHitSource.Respon
             /*
              * If we noop-ed the entire batch then just skip to the next batch or the BulkRequest would fail validation.
              */
-            startNextScroll(thisBatchStartTime, 0);
+            startNextScroll(thisBatchStartTime, timeValueNanos(System.nanoTime()), 0);
             return;
         }
         request.timeout(mainRequest.getTimeout());
         request.waitForActiveShards(mainRequest.getWaitForActiveShards());
-        if (logger.isDebugEnabled()) {
-            logger.debug("sending [{}] entry, [{}] bulk request", request.requests().size(),
-                    new ByteSizeValue(request.estimatedSizeInBytes()));
-        }
         sendBulkRequest(thisBatchStartTime, request);
     }
 
@@ -227,11 +349,16 @@ void prepareBulkRequest(TimeValue thisBatchStartTime, ScrollableHitSource.Respon
      * Send a bulk request, handling retries.
      */
     void sendBulkRequest(TimeValue thisBatchStartTime, BulkRequest request) {
+        if (logger.isDebugEnabled()) {
+            logger.debug("[{}]: sending [{}] entry, [{}] bulk request", task.getId(), request.requests().size(),
+                    new ByteSizeValue(request.estimatedSizeInBytes()));
+        }
         if (task.isCancelled()) {
+            logger.debug("[{}]: finishing early because the task was cancelled", task.getId());
             finishHim(null);
             return;
         }
-        bulkRetry.withAsyncBackoff(client, request, new ActionListener<BulkResponse>() {
+        bulkRetry.withAsyncBackoff(client::bulk, request, new ActionListener<BulkResponse>() {
             @Override
             public void onResponse(BulkResponse response) {
                 onBulkResponse(thisBatchStartTime, response);
@@ -241,7 +368,7 @@ public void onResponse(BulkResponse response) {
             public void onFailure(Exception e) {
                 finishHim(e);
             }
-        });
+        }, settings);
     }
 
     /**
@@ -258,26 +385,27 @@ void onBulkResponse(TimeValue thisBatchStartTime, BulkResponse response) {
                 }
 
                 switch (item.getOpType()) {
-                case "index":
-                case "create":
-                    IndexResponse ir = item.getResponse();
-                    if (ir.getResult() == DocWriteResponse.Result.CREATED) {
-                        task.countCreated();
-                    } else {
-                        task.countUpdated();
-                    }
-                    break;
-                case "delete":
-                    task.countDeleted();
-                    break;
-                default:
-                    throw new IllegalArgumentException("Unknown op type:  " + item.getOpType());
+                    case CREATE:
+                    case INDEX:
+                        if (item.getResponse().getResult() == DocWriteResponse.Result.CREATED) {
+                            task.countCreated();
+                        } else {
+                            task.countUpdated();
+                        }
+                        break;
+                    case UPDATE:
+                         task.countUpdated();
+                         break;
+                    case DELETE:
+                        task.countDeleted();
+                        break;
                 }
                 // Track the indexes we've seen so we can refresh them if requested
                 destinationIndicesThisBatch.add(item.getIndex());
             }
 
             if (task.isCancelled()) {
+                logger.debug("[{}]: Finishing early because the task was cancelled", task.getId());
                 finishHim(null);
                 return;
             }
@@ -295,7 +423,7 @@ void onBulkResponse(TimeValue thisBatchStartTime, BulkResponse response) {
                 return;
             }
 
-            startNextScroll(thisBatchStartTime, response.getItems().length);
+            startNextScroll(thisBatchStartTime, timeValueNanos(System.nanoTime()), response.getItems().length);
         } catch (Exception t) {
             finishHim(t);
         }
@@ -307,12 +435,13 @@ void onBulkResponse(TimeValue thisBatchStartTime, BulkResponse response) {
      * @param lastBatchSize the number of requests sent in the last batch. This is used to calculate the throttling values which are applied
      *        when the scroll returns
      */
-    void startNextScroll(TimeValue lastBatchStartTime, int lastBatchSize) {
+    void startNextScroll(TimeValue lastBatchStartTime, TimeValue now, int lastBatchSize) {
         if (task.isCancelled()) {
+            logger.debug("[{}]: finishing early because the task was cancelled", task.getId());
             finishHim(null);
             return;
         }
-        TimeValue extraKeepAlive = task.throttleWaitTime(lastBatchStartTime, lastBatchSize);
+        TimeValue extraKeepAlive = task.throttleWaitTime(lastBatchStartTime, now, lastBatchSize);
         scrollSource.startNextScroll(extraKeepAlive, response -> {
             onScrollResponse(lastBatchStartTime, lastBatchSize, response);
         });
@@ -339,6 +468,7 @@ void refreshAndFinish(List<Failure> indexingFailures, List<SearchFailure> search
         }
         RefreshRequest refresh = new RefreshRequest();
         refresh.indices(destinationIndices.toArray(new String[destinationIndices.size()]));
+        logger.debug("[{}]: refreshing", task.getId());
         client.admin().indices().refresh(refresh, new ActionListener<RefreshResponse>() {
             @Override
             public void onResponse(RefreshResponse response) {
@@ -357,7 +487,8 @@ public void onFailure(Exception e) {
      *
      * @param failure if non null then the request failed catastrophically with this exception
      */
-    void finishHim(Exception failure) {
+    protected void finishHim(Exception failure) {
+        logger.debug(() -> new ParameterizedMessage("[{}]: finishing with a catastrophic failure", task.getId()), failure);
         finishHim(failure, emptyList(), emptyList(), false);
     }
 
@@ -368,14 +499,19 @@ void finishHim(Exception failure) {
      * @param searchFailures any search failures accumulated during the request
      * @param timedOut have any of the sub-requests timed out?
      */
-    void finishHim(Exception failure, List<Failure> indexingFailures, List<SearchFailure> searchFailures, boolean timedOut) {
-        scrollSource.close();
-        if (failure == null) {
-            listener.onResponse(
-                    buildResponse(timeValueNanos(System.nanoTime() - startTime.get()), indexingFailures, searchFailures, timedOut));
-        } else {
-            listener.onFailure(failure);
-        }
+    protected void finishHim(Exception failure, List<Failure> indexingFailures,
+            List<SearchFailure> searchFailures, boolean timedOut) {
+        logger.debug("[{}]: finishing without any catastrophic failures", task.getId());
+        scrollSource.close(() -> {
+            if (failure == null) {
+                BulkByScrollResponse response = buildResponse(
+                        timeValueNanos(System.nanoTime() - startTime.get()),
+                        indexingFailures, searchFailures, timedOut);
+                listener.onResponse(response);
+            } else {
+                listener.onFailure(failure);
+            }
+        });
     }
 
     /**
@@ -399,4 +535,453 @@ void addDestinationIndices(Collection<String> indices) {
     void setScroll(String scroll) {
         scrollSource.setScroll(scroll);
     }
+
+    /**
+     * Wrapper for the {@link IndexRequest} or {@link DeleteRequest} that are used in this action class.
+     */
+    public interface RequestWrapper<Self extends DocWriteRequest<?>> {
+
+        void setIndex(String index);
+
+        String getIndex();
+
+        void setType(String type);
+
+        String getType();
+
+        void setId(String id);
+
+        String getId();
+
+        void setVersion(long version);
+
+        long getVersion();
+
+        void setVersionType(VersionType versionType);
+
+        void setParent(String parent);
+
+        String getParent();
+
+        void setRouting(String routing);
+
+        String getRouting();
+
+        void setTimestamp(String timestamp);
+
+        void setTtl(Long ttl);
+
+        void setSource(Map<String, Object> source);
+
+        Map<String, Object> getSource();
+
+        Self self();
+    }
+
+    /**
+     * {@link RequestWrapper} for {@link IndexRequest}
+     */
+    public static class IndexRequestWrapper implements RequestWrapper<IndexRequest> {
+
+        private final IndexRequest request;
+
+        IndexRequestWrapper(IndexRequest request) {
+            this.request = Objects.requireNonNull(request, "Wrapped IndexRequest can not be null");
+        }
+
+        @Override
+        public void setIndex(String index) {
+            request.index(index);
+        }
+
+        @Override
+        public String getIndex() {
+            return request.index();
+        }
+
+        @Override
+        public void setType(String type) {
+            request.type(type);
+        }
+
+        @Override
+        public String getType() {
+            return request.type();
+        }
+
+        @Override
+        public void setId(String id) {
+            request.id(id);
+        }
+
+        @Override
+        public String getId() {
+            return request.id();
+        }
+
+        @Override
+        public void setVersion(long version) {
+            request.version(version);
+        }
+
+        @Override
+        public long getVersion() {
+            return request.version();
+        }
+
+        @Override
+        public void setVersionType(VersionType versionType) {
+            request.versionType(versionType);
+        }
+
+        @Override
+        public void setParent(String parent) {
+            request.parent(parent);
+        }
+
+        @Override
+        public String getParent() {
+            return request.parent();
+        }
+
+        @Override
+        public void setRouting(String routing) {
+            request.routing(routing);
+        }
+
+        @Override
+        public String getRouting() {
+            return request.routing();
+        }
+
+        @Override
+        public Map<String, Object> getSource() {
+            return request.sourceAsMap();
+        }
+
+        @Override
+        public void setTimestamp(String timestamp) {
+            request.timestamp(timestamp);
+        }
+
+        @Override
+        public void setTtl(Long ttl) {
+            if (ttl == null) {
+                request.ttl((TimeValue) null);
+            } else {
+                request.ttl(ttl);
+            }
+        }
+
+        @Override
+        public void setSource(Map<String, Object> source) {
+            request.source(source);
+        }
+
+        @Override
+        public IndexRequest self() {
+            return request;
+        }
+    }
+
+    /**
+     * Wraps a {@link IndexRequest} in a {@link RequestWrapper}
+     */
+    public static RequestWrapper<IndexRequest> wrap(IndexRequest request) {
+        return new IndexRequestWrapper(request);
+    }
+
+    /**
+     * {@link RequestWrapper} for {@link DeleteRequest}
+     */
+    public static class DeleteRequestWrapper implements RequestWrapper<DeleteRequest> {
+
+        private final DeleteRequest request;
+
+        DeleteRequestWrapper(DeleteRequest request) {
+            this.request = Objects.requireNonNull(request, "Wrapped DeleteRequest can not be null");
+        }
+
+        @Override
+        public void setIndex(String index) {
+            request.index(index);
+        }
+
+        @Override
+        public String getIndex() {
+            return request.index();
+        }
+
+        @Override
+        public void setType(String type) {
+            request.type(type);
+        }
+
+        @Override
+        public String getType() {
+            return request.type();
+        }
+
+        @Override
+        public void setId(String id) {
+            request.id(id);
+        }
+
+        @Override
+        public String getId() {
+            return request.id();
+        }
+
+        @Override
+        public void setVersion(long version) {
+            request.version(version);
+        }
+
+        @Override
+        public long getVersion() {
+            return request.version();
+        }
+
+        @Override
+        public void setVersionType(VersionType versionType) {
+            request.versionType(versionType);
+        }
+
+        @Override
+        public void setParent(String parent) {
+            request.parent(parent);
+        }
+
+        @Override
+        public String getParent() {
+            return request.parent();
+        }
+
+        @Override
+        public void setRouting(String routing) {
+            request.routing(routing);
+        }
+
+        @Override
+        public String getRouting() {
+            return request.routing();
+        }
+
+        @Override
+        public void setTimestamp(String timestamp) {
+            throw new UnsupportedOperationException("unable to set [timestamp] on action request [" + request.getClass() + "]");
+        }
+
+        @Override
+        public void setTtl(Long ttl) {
+            throw new UnsupportedOperationException("unable to set [ttl] on action request [" + request.getClass() + "]");
+        }
+
+        @Override
+        public Map<String, Object> getSource() {
+            throw new UnsupportedOperationException("unable to get source from action request [" + request.getClass() + "]");
+        }
+
+        @Override
+        public void setSource(Map<String, Object> source) {
+            throw new UnsupportedOperationException("unable to set [source] on action request [" + request.getClass() + "]");
+        }
+
+        @Override
+        public DeleteRequest self() {
+            return request;
+        }
+    }
+
+    /**
+     * Wraps a {@link DeleteRequest} in a {@link RequestWrapper}
+     */
+    public static RequestWrapper<DeleteRequest> wrap(DeleteRequest request) {
+        return new DeleteRequestWrapper(request);
+    }
+
+    /**
+     * Apply a {@link Script} to a {@link RequestWrapper}
+     */
+    public abstract class ScriptApplier implements BiFunction<RequestWrapper<?>, ScrollableHitSource.Hit, RequestWrapper<?>> {
+
+        private final WorkingBulkByScrollTask task;
+        private final ScriptService scriptService;
+        private final Script script;
+        private final Map<String, Object> params;
+
+        private ExecutableScript executable;
+        private Map<String, Object> context;
+
+        public ScriptApplier(WorkingBulkByScrollTask task, ScriptService scriptService, Script script,
+                             Map<String, Object> params) {
+            this.task = task;
+            this.scriptService = scriptService;
+            this.script = script;
+            this.params = params;
+        }
+
+        @Override
+        @SuppressWarnings("unchecked")
+        public RequestWrapper<?> apply(RequestWrapper<?> request, ScrollableHitSource.Hit doc) {
+            if (script == null) {
+                return request;
+            }
+            if (executable == null) {
+                CompiledScript compiled = scriptService.compile(script, ScriptContext.Standard.UPDATE);
+                executable = scriptService.executable(compiled, params);
+            }
+            if (context == null) {
+                context = new HashMap<>();
+            } else {
+                context.clear();
+            }
+
+            context.put(IndexFieldMapper.NAME, doc.getIndex());
+            context.put(TypeFieldMapper.NAME, doc.getType());
+            context.put(IdFieldMapper.NAME, doc.getId());
+            Long oldVersion = doc.getVersion();
+            context.put(VersionFieldMapper.NAME, oldVersion);
+            String oldParent = doc.getParent();
+            context.put(ParentFieldMapper.NAME, oldParent);
+            String oldRouting = doc.getRouting();
+            context.put(RoutingFieldMapper.NAME, oldRouting);
+            Long oldTimestamp = doc.getTimestamp();
+            context.put(TimestampFieldMapper.NAME, oldTimestamp);
+            Long oldTTL = doc.getTTL();
+            context.put(TTLFieldMapper.NAME, oldTTL);
+            context.put(SourceFieldMapper.NAME, request.getSource());
+
+            OpType oldOpType = OpType.INDEX;
+            context.put("op", oldOpType.toString());
+
+            executable.setNextVar("ctx", context);
+            executable.run();
+
+            Map<String, Object> resultCtx = (Map<String, Object>) executable.unwrap(context);
+            String newOp = (String) resultCtx.remove("op");
+            if (newOp == null) {
+                throw new IllegalArgumentException("Script cleared operation type");
+            }
+
+            /*
+             * It'd be lovely to only set the source if we know its been modified
+             * but it isn't worth keeping two copies of it around just to check!
+             */
+            request.setSource((Map<String, Object>) resultCtx.remove(SourceFieldMapper.NAME));
+
+            Object newValue = resultCtx.remove(IndexFieldMapper.NAME);
+            if (false == doc.getIndex().equals(newValue)) {
+                scriptChangedIndex(request, newValue);
+            }
+            newValue = resultCtx.remove(TypeFieldMapper.NAME);
+            if (false == doc.getType().equals(newValue)) {
+                scriptChangedType(request, newValue);
+            }
+            newValue = resultCtx.remove(IdFieldMapper.NAME);
+            if (false == doc.getId().equals(newValue)) {
+                scriptChangedId(request, newValue);
+            }
+            newValue = resultCtx.remove(VersionFieldMapper.NAME);
+            if (false == Objects.equals(oldVersion, newValue)) {
+                scriptChangedVersion(request, newValue);
+            }
+            newValue = resultCtx.remove(ParentFieldMapper.NAME);
+            if (false == Objects.equals(oldParent, newValue)) {
+                scriptChangedParent(request, newValue);
+            }
+            /*
+             * Its important that routing comes after parent in case you want to
+             * change them both.
+             */
+            newValue = resultCtx.remove(RoutingFieldMapper.NAME);
+            if (false == Objects.equals(oldRouting, newValue)) {
+                scriptChangedRouting(request, newValue);
+            }
+            newValue = resultCtx.remove(TimestampFieldMapper.NAME);
+            if (false == Objects.equals(oldTimestamp, newValue)) {
+                scriptChangedTimestamp(request, newValue);
+            }
+            newValue = resultCtx.remove(TTLFieldMapper.NAME);
+            if (false == Objects.equals(oldTTL, newValue)) {
+                scriptChangedTTL(request, newValue);
+            }
+
+            OpType newOpType = OpType.fromString(newOp);
+            if (newOpType != oldOpType) {
+                return scriptChangedOpType(request, oldOpType, newOpType);
+            }
+
+            if (false == resultCtx.isEmpty()) {
+                throw new IllegalArgumentException("Invalid fields added to context [" + String.join(",", resultCtx.keySet()) + ']');
+            }
+            return request;
+        }
+
+        protected RequestWrapper<?> scriptChangedOpType(RequestWrapper<?> request, OpType oldOpType, OpType newOpType) {
+            switch (newOpType) {
+            case NOOP:
+                task.countNoop();
+                return null;
+            case DELETE:
+                RequestWrapper<DeleteRequest> delete = wrap(new DeleteRequest(request.getIndex(), request.getType(), request.getId()));
+                delete.setVersion(request.getVersion());
+                delete.setVersionType(VersionType.INTERNAL);
+                delete.setParent(request.getParent());
+                delete.setRouting(request.getRouting());
+                return delete;
+            default:
+                throw new IllegalArgumentException("Unsupported operation type change from [" + oldOpType + "] to [" + newOpType + "]");
+            }
+        }
+
+        protected abstract void scriptChangedIndex(RequestWrapper<?> request, Object to);
+
+        protected abstract void scriptChangedType(RequestWrapper<?> request, Object to);
+
+        protected abstract void scriptChangedId(RequestWrapper<?> request, Object to);
+
+        protected abstract void scriptChangedVersion(RequestWrapper<?> request, Object to);
+
+        protected abstract void scriptChangedRouting(RequestWrapper<?> request, Object to);
+
+        protected abstract void scriptChangedParent(RequestWrapper<?> request, Object to);
+
+        protected abstract void scriptChangedTimestamp(RequestWrapper<?> request, Object to);
+
+        protected abstract void scriptChangedTTL(RequestWrapper<?> request, Object to);
+    }
+
+    public enum OpType {
+
+        NOOP("noop"),
+        INDEX("index"),
+        DELETE("delete");
+
+        private final String id;
+
+        OpType(String id) {
+            this.id = id;
+        }
+
+        public static OpType fromString(String opType) {
+            String lowerOpType = opType.toLowerCase(Locale.ROOT);
+            switch (lowerOpType) {
+                case "noop":
+                    return OpType.NOOP;
+                case "index":
+                    return OpType.INDEX;
+                case "delete":
+                    return OpType.DELETE;
+                default:
+                    throw new IllegalArgumentException("Operation type [" + lowerOpType + "] not allowed, only " +
+                            Arrays.toString(values()) + " are allowed");
+            }
+        }
+
+        @Override
+        public String toString() {
+            return id.toLowerCase(Locale.ROOT);
+        }
+    }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollAction.java
deleted file mode 100644
index f441e527222ab..0000000000000
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollAction.java
+++ /dev/null
@@ -1,605 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.reindex;
-
-import org.apache.logging.log4j.Logger;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ActionRequest;
-import org.elasticsearch.action.bulk.BulkRequest;
-import org.elasticsearch.action.delete.DeleteRequest;
-import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.client.ParentTaskAssigningClient;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.index.VersionType;
-import org.elasticsearch.index.mapper.IdFieldMapper;
-import org.elasticsearch.index.mapper.IndexFieldMapper;
-import org.elasticsearch.index.mapper.ParentFieldMapper;
-import org.elasticsearch.index.mapper.RoutingFieldMapper;
-import org.elasticsearch.index.mapper.SourceFieldMapper;
-import org.elasticsearch.index.mapper.TTLFieldMapper;
-import org.elasticsearch.index.mapper.TimestampFieldMapper;
-import org.elasticsearch.index.mapper.TypeFieldMapper;
-import org.elasticsearch.index.mapper.VersionFieldMapper;
-import org.elasticsearch.script.CompiledScript;
-import org.elasticsearch.script.ExecutableScript;
-import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptContext;
-import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.threadpool.ThreadPool;
-
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Objects;
-import java.util.function.BiFunction;
-
-import static java.util.Collections.emptyMap;
-
-/**
- * Abstract base for scrolling across a search and executing bulk indexes on all
- * results.
- */
-public abstract class AbstractAsyncBulkIndexByScrollAction<Request extends AbstractBulkByScrollRequest<Request>>
-        extends AbstractAsyncBulkByScrollAction<Request> {
-
-    protected final ScriptService scriptService;
-    protected final ClusterState clusterState;
-
-    /**
-     * This BiFunction is used to apply various changes depending of the Reindex action and  the search hit,
-     * from copying search hit metadata (parent, routing, etc) to potentially transforming the
-     * {@link RequestWrapper} completely.
-     */
-    private final BiFunction<RequestWrapper<?>, ScrollableHitSource.Hit, RequestWrapper<?>> scriptApplier;
-
-    public AbstractAsyncBulkIndexByScrollAction(BulkByScrollTask task, Logger logger, ParentTaskAssigningClient client,
-                                                ThreadPool threadPool, Request mainRequest,
-                                                ActionListener<BulkIndexByScrollResponse> listener,
-                                                ScriptService scriptService, ClusterState clusterState) {
-        super(task, logger, client, threadPool, mainRequest, listener);
-        this.scriptService = scriptService;
-        this.clusterState = clusterState;
-        this.scriptApplier = Objects.requireNonNull(buildScriptApplier(), "script applier must not be null");
-    }
-
-    /**
-     * Build the {@link BiFunction} to apply to all {@link RequestWrapper}.
-     */
-    protected BiFunction<RequestWrapper<?>, ScrollableHitSource.Hit, RequestWrapper<?>> buildScriptApplier() {
-        // The default script applier executes a no-op
-        return (request, searchHit) -> request;
-    }
-
-    @Override
-    protected BulkRequest buildBulk(Iterable<? extends ScrollableHitSource.Hit> docs) {
-        BulkRequest bulkRequest = new BulkRequest();
-        for (ScrollableHitSource.Hit doc : docs) {
-            if (accept(doc)) {
-                RequestWrapper<?> request = scriptApplier.apply(copyMetadata(buildRequest(doc), doc), doc);
-                if (request != null) {
-                    bulkRequest.add(request.self());
-                }
-            }
-        }
-        return bulkRequest;
-    }
-
-    /**
-     * Used to accept or ignore a search hit. Ignored search hits will be excluded
-     * from the bulk request. It is also where we fail on invalid search hits, like
-     * when the document has no source but it's required.
-     */
-    protected boolean accept(ScrollableHitSource.Hit doc) {
-        if (doc.getSource() == null) {
-            /*
-             * Either the document didn't store _source or we didn't fetch it for some reason. Since we don't allow the user to
-             * change the "fields" part of the search request it is unlikely that we got here because we didn't fetch _source.
-             * Thus the error message assumes that it wasn't stored.
-             */
-            throw new IllegalArgumentException("[" + doc.getIndex() + "][" + doc.getType() + "][" + doc.getId() + "] didn't store _source");
-        }
-        return true;
-    }
-
-    /**
-     * Build the {@link RequestWrapper} for a single search hit. This shouldn't handle
-     * metadata or scripting. That will be handled by copyMetadata and
-     * apply functions that can be overridden.
-     */
-    protected abstract RequestWrapper<?> buildRequest(ScrollableHitSource.Hit doc);
-
-    /**
-     * Copies the metadata from a hit to the request.
-     */
-    protected RequestWrapper<?> copyMetadata(RequestWrapper<?> request, ScrollableHitSource.Hit doc) {
-        request.setParent(doc.getParent());
-        copyRouting(request, doc.getRouting());
-
-        // Comes back as a Long but needs to be a string
-        Long timestamp = doc.getTimestamp();
-        if (timestamp != null) {
-            request.setTimestamp(timestamp.toString());
-        }
-        Long ttl = doc.getTTL();
-        if (ttl != null) {
-            request.setTtl(ttl);
-        }
-        return request;
-    }
-
-    /**
-     * Copy the routing from a search hit to the request.
-     */
-    protected void copyRouting(RequestWrapper<?> request, String routing) {
-        request.setRouting(routing);
-    }
-
-    /**
-     * Wrapper for the {@link ActionRequest} that are used in this action class.
-     */
-    interface RequestWrapper<Self extends ActionRequest<Self>> {
-
-        void setIndex(String index);
-
-        String getIndex();
-
-        void setType(String type);
-
-        String getType();
-
-        void setId(String id);
-
-        String getId();
-
-        void setVersion(long version);
-
-        long getVersion();
-
-        void setVersionType(VersionType versionType);
-
-        void setParent(String parent);
-
-        String getParent();
-
-        void setRouting(String routing);
-
-        String getRouting();
-
-        void setTimestamp(String timestamp);
-
-        void setTtl(Long ttl);
-
-        void setSource(Map<String, Object> source);
-
-        Map<String, Object> getSource();
-
-        Self self();
-    }
-
-    /**
-     * {@link RequestWrapper} for {@link IndexRequest}
-     */
-    public static class IndexRequestWrapper implements RequestWrapper<IndexRequest> {
-
-        private final IndexRequest request;
-
-        IndexRequestWrapper(IndexRequest request) {
-            this.request = Objects.requireNonNull(request, "Wrapped IndexRequest can not be null");
-        }
-
-        @Override
-        public void setIndex(String index) {
-            request.index(index);
-        }
-
-        @Override
-        public String getIndex() {
-            return request.index();
-        }
-
-        @Override
-        public void setType(String type) {
-            request.type(type);
-        }
-
-        @Override
-        public String getType() {
-            return request.type();
-        }
-
-        @Override
-        public void setId(String id) {
-            request.id(id);
-        }
-
-        @Override
-        public String getId() {
-            return request.id();
-        }
-
-        @Override
-        public void setVersion(long version) {
-            request.version(version);
-        }
-
-        @Override
-        public long getVersion() {
-            return request.version();
-        }
-
-        @Override
-        public void setVersionType(VersionType versionType) {
-            request.versionType(versionType);
-        }
-
-        @Override
-        public void setParent(String parent) {
-            request.parent(parent);
-        }
-
-        @Override
-        public String getParent() {
-            return request.parent();
-        }
-
-        @Override
-        public void setRouting(String routing) {
-            request.routing(routing);
-        }
-
-        @Override
-        public String getRouting() {
-            return request.routing();
-        }
-
-        @Override
-        public void setTimestamp(String timestamp) {
-            request.timestamp(timestamp);
-        }
-
-        @Override
-        public void setTtl(Long ttl) {
-            if (ttl == null) {
-                request.ttl((TimeValue) null);
-            } else {
-                request.ttl(ttl);
-            }
-        }
-
-        @Override
-        public Map<String, Object> getSource() {
-            return request.sourceAsMap();
-        }
-
-        @Override
-        public void setSource(Map<String, Object> source) {
-            request.source(source);
-        }
-
-        @Override
-        public IndexRequest self() {
-            return request;
-        }
-    }
-
-    /**
-     * Wraps a {@link IndexRequest} in a {@link RequestWrapper}
-     */
-    static RequestWrapper<IndexRequest> wrap(IndexRequest request) {
-        return new IndexRequestWrapper(request);
-    }
-
-    /**
-     * {@link RequestWrapper} for {@link DeleteRequest}
-     */
-    public static class DeleteRequestWrapper implements RequestWrapper<DeleteRequest> {
-
-        private final DeleteRequest request;
-
-        DeleteRequestWrapper(DeleteRequest request) {
-            this.request = Objects.requireNonNull(request, "Wrapped DeleteRequest can not be null");
-        }
-
-        @Override
-        public void setIndex(String index) {
-            request.index(index);
-        }
-
-        @Override
-        public String getIndex() {
-            return request.index();
-        }
-
-        @Override
-        public void setType(String type) {
-            request.type(type);
-        }
-
-        @Override
-        public String getType() {
-            return request.type();
-        }
-
-        @Override
-        public void setId(String id) {
-            request.id(id);
-        }
-
-        @Override
-        public String getId() {
-            return request.id();
-        }
-
-        @Override
-        public void setVersion(long version) {
-            request.version(version);
-        }
-
-        @Override
-        public long getVersion() {
-            return request.version();
-        }
-
-        @Override
-        public void setVersionType(VersionType versionType) {
-            request.versionType(versionType);
-        }
-
-        @Override
-        public void setParent(String parent) {
-            request.parent(parent);
-        }
-
-        @Override
-        public String getParent() {
-            return request.parent();
-        }
-
-        @Override
-        public void setRouting(String routing) {
-            request.routing(routing);
-        }
-
-        @Override
-        public String getRouting() {
-            return request.routing();
-        }
-
-        @Override
-        public void setTimestamp(String timestamp) {
-            throw new UnsupportedOperationException("unable to set [timestamp] on action request [" + request.getClass() + "]");
-        }
-
-        @Override
-        public void setTtl(Long ttl) {
-            throw new UnsupportedOperationException("unable to set [ttl] on action request [" + request.getClass() + "]");
-        }
-
-        @Override
-        public Map<String, Object> getSource() {
-            throw new UnsupportedOperationException("unable to get source from action request [" + request.getClass() + "]");
-        }
-
-        @Override
-        public void setSource(Map<String, Object> source) {
-            throw new UnsupportedOperationException("unable to set [source] on action request [" + request.getClass() + "]");
-        }
-
-        @Override
-        public DeleteRequest self() {
-            return request;
-        }
-    }
-
-    /**
-     * Wraps a {@link DeleteRequest} in a {@link RequestWrapper}
-     */
-    static RequestWrapper<DeleteRequest> wrap(DeleteRequest request) {
-        return new DeleteRequestWrapper(request);
-    }
-
-    /**
-     * Apply a {@link Script} to a {@link RequestWrapper}
-     */
-    public abstract class ScriptApplier implements BiFunction<RequestWrapper<?>, ScrollableHitSource.Hit, RequestWrapper<?>> {
-
-        private final BulkByScrollTask task;
-        private final ScriptService scriptService;
-        private final Script script;
-        private final Map<String, Object> params;
-
-        private ExecutableScript executable;
-        private Map<String, Object> context;
-
-        public ScriptApplier(BulkByScrollTask task, ScriptService scriptService, Script script,
-                             Map<String, Object> params) {
-            this.task = task;
-            this.scriptService = scriptService;
-            this.script = script;
-            this.params = params;
-        }
-
-        @Override
-        @SuppressWarnings("unchecked")
-        public RequestWrapper<?> apply(RequestWrapper<?> request, ScrollableHitSource.Hit doc) {
-            if (script == null) {
-                return request;
-            }
-            if (executable == null) {
-                CompiledScript compiled = scriptService.compile(script, ScriptContext.Standard.UPDATE, emptyMap());
-                executable = scriptService.executable(compiled, params);
-            }
-            if (context == null) {
-                context = new HashMap<>();
-            } else {
-                context.clear();
-            }
-
-            context.put(IndexFieldMapper.NAME, doc.getIndex());
-            context.put(TypeFieldMapper.NAME, doc.getType());
-            context.put(IdFieldMapper.NAME, doc.getId());
-            Long oldVersion = doc.getVersion();
-            context.put(VersionFieldMapper.NAME, oldVersion);
-            String oldParent = doc.getParent();
-            context.put(ParentFieldMapper.NAME, oldParent);
-            String oldRouting = doc.getRouting();
-            context.put(RoutingFieldMapper.NAME, oldRouting);
-            Long oldTimestamp = doc.getTimestamp();
-            context.put(TimestampFieldMapper.NAME, oldTimestamp);
-            Long oldTTL = doc.getTTL();
-            context.put(TTLFieldMapper.NAME, oldTTL);
-            context.put(SourceFieldMapper.NAME, request.getSource());
-
-            OpType oldOpType = OpType.INDEX;
-            context.put("op", oldOpType.toString());
-
-            executable.setNextVar("ctx", context);
-            executable.run();
-
-            Map<String, Object> resultCtx = (Map<String, Object>) executable.unwrap(context);
-            String newOp = (String) resultCtx.remove("op");
-            if (newOp == null) {
-                throw new IllegalArgumentException("Script cleared operation type");
-            }
-
-            /*
-             * It'd be lovely to only set the source if we know its been modified
-             * but it isn't worth keeping two copies of it around just to check!
-             */
-            request.setSource((Map<String, Object>) resultCtx.remove(SourceFieldMapper.NAME));
-
-            Object newValue = resultCtx.remove(IndexFieldMapper.NAME);
-            if (false == doc.getIndex().equals(newValue)) {
-                scriptChangedIndex(request, newValue);
-            }
-            newValue = resultCtx.remove(TypeFieldMapper.NAME);
-            if (false == doc.getType().equals(newValue)) {
-                scriptChangedType(request, newValue);
-            }
-            newValue = resultCtx.remove(IdFieldMapper.NAME);
-            if (false == doc.getId().equals(newValue)) {
-                scriptChangedId(request, newValue);
-            }
-            newValue = resultCtx.remove(VersionFieldMapper.NAME);
-            if (false == Objects.equals(oldVersion, newValue)) {
-                scriptChangedVersion(request, newValue);
-            }
-            newValue = resultCtx.remove(ParentFieldMapper.NAME);
-            if (false == Objects.equals(oldParent, newValue)) {
-                scriptChangedParent(request, newValue);
-            }
-            /*
-             * Its important that routing comes after parent in case you want to
-             * change them both.
-             */
-            newValue = resultCtx.remove(RoutingFieldMapper.NAME);
-            if (false == Objects.equals(oldRouting, newValue)) {
-                scriptChangedRouting(request, newValue);
-            }
-            newValue = resultCtx.remove(TimestampFieldMapper.NAME);
-            if (false == Objects.equals(oldTimestamp, newValue)) {
-                scriptChangedTimestamp(request, newValue);
-            }
-            newValue = resultCtx.remove(TTLFieldMapper.NAME);
-            if (false == Objects.equals(oldTTL, newValue)) {
-                scriptChangedTTL(request, newValue);
-            }
-
-            OpType newOpType = OpType.fromString(newOp);
-            if (newOpType != oldOpType) {
-                return scriptChangedOpType(request, oldOpType, newOpType);
-            }
-
-            if (false == resultCtx.isEmpty()) {
-                throw new IllegalArgumentException("Invalid fields added to context [" + String.join(",", resultCtx.keySet()) + ']');
-            }
-            return request;
-        }
-
-        protected RequestWrapper<?> scriptChangedOpType(RequestWrapper<?> request, OpType oldOpType, OpType newOpType) {
-            switch (newOpType) {
-            case NOOP:
-                task.countNoop();
-                return null;
-            case DELETE:
-                RequestWrapper<DeleteRequest> delete = wrap(new DeleteRequest(request.getIndex(), request.getType(), request.getId()));
-                delete.setVersion(request.getVersion());
-                delete.setVersionType(VersionType.INTERNAL);
-                delete.setParent(request.getParent());
-                delete.setRouting(request.getRouting());
-                return delete;
-            default:
-                throw new IllegalArgumentException("Unsupported operation type change from [" + oldOpType + "] to [" + newOpType + "]");
-            }
-        }
-
-        protected abstract void scriptChangedIndex(RequestWrapper<?> request, Object to);
-
-        protected abstract void scriptChangedType(RequestWrapper<?> request, Object to);
-
-        protected abstract void scriptChangedId(RequestWrapper<?> request, Object to);
-
-        protected abstract void scriptChangedVersion(RequestWrapper<?> request, Object to);
-
-        protected abstract void scriptChangedRouting(RequestWrapper<?> request, Object to);
-
-        protected abstract void scriptChangedParent(RequestWrapper<?> request, Object to);
-
-        protected abstract void scriptChangedTimestamp(RequestWrapper<?> request, Object to);
-
-        protected abstract void scriptChangedTTL(RequestWrapper<?> request, Object to);
-
-    }
-
-    public enum OpType {
-
-        NOOP("noop"),
-        INDEX("index"),
-        DELETE("delete");
-
-        private final String id;
-
-        OpType(String id) {
-            this.id = id;
-        }
-
-        public static OpType fromString(String opType) {
-            String lowerOpType = opType.toLowerCase(Locale.ROOT);
-            switch (lowerOpType) {
-                case "noop":
-                    return OpType.NOOP;
-                case "index":
-                    return OpType.INDEX;
-                case "delete":
-                    return OpType.DELETE;
-                default:
-                    throw new IllegalArgumentException("Operation type [" + lowerOpType + "] not allowed, only " +
-                            Arrays.toString(values()) + " are allowed");
-            }
-        }
-
-        @Override
-        public String toString() {
-            return id.toLowerCase(Locale.ROOT);
-        }
-    }
-}
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java
index faa2607ce206a..64b02c4be8150 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java
@@ -23,18 +23,12 @@
 import org.elasticsearch.action.GenericAction;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.search.SearchRequestParsers;
-import org.elasticsearch.search.aggregations.AggregatorParsers;
-import org.elasticsearch.search.suggest.Suggesters;
 import org.elasticsearch.tasks.LoggingTaskListener;
 import org.elasticsearch.tasks.Task;
 
@@ -44,23 +38,18 @@
 
 public abstract class AbstractBaseReindexRestHandler<
                 Request extends AbstractBulkByScrollRequest<Request>,
-                A extends GenericAction<Request, BulkIndexByScrollResponse>
+                A extends GenericAction<Request, BulkByScrollResponse>
             > extends BaseRestHandler {
 
-    protected final SearchRequestParsers searchRequestParsers;
-    private final ClusterService clusterService;
     private final A action;
 
-    protected AbstractBaseReindexRestHandler(Settings settings, SearchRequestParsers searchRequestParsers,
-                                             ClusterService clusterService, A action) {
+    protected AbstractBaseReindexRestHandler(Settings settings, A action) {
         super(settings);
-        this.searchRequestParsers = searchRequestParsers;
-        this.clusterService = clusterService;
         this.action = action;
     }
 
-    protected void handleRequest(RestRequest request, RestChannel channel, NodeClient client,
-                                 boolean includeCreated, boolean includeUpdated) throws IOException {
+    protected RestChannelConsumer doPrepareRequest(RestRequest request, NodeClient client,
+                                                   boolean includeCreated, boolean includeUpdated) throws IOException {
         // Build the internal request
         Request internal = setCommonOptions(request, buildRequest(request));
 
@@ -70,8 +59,7 @@ protected void handleRequest(RestRequest request, RestChannel channel, NodeClien
             params.put(BulkByScrollTask.Status.INCLUDE_CREATED, Boolean.toString(includeCreated));
             params.put(BulkByScrollTask.Status.INCLUDE_UPDATED, Boolean.toString(includeUpdated));
 
-            client.executeLocally(action, internal, new BulkIndexByScrollResponseContentListener(channel, params));
-            return;
+            return channel -> client.executeLocally(action, internal, new BulkIndexByScrollResponseContentListener(channel, params));
         } else {
             internal.setShouldStoreResult(true);
         }
@@ -83,10 +71,9 @@ protected void handleRequest(RestRequest request, RestChannel channel, NodeClien
          */
         ActionRequestValidationException validationException = internal.validate();
         if (validationException != null) {
-            channel.sendResponse(new BytesRestResponse(channel, validationException));
-            return;
+            throw validationException;
         }
-        sendTask(channel, client.executeLocally(action, internal, LoggingTaskListener.instance()));
+        return sendTask(client.getLocalNodeId(), client.executeLocally(action, internal, LoggingTaskListener.instance()));
     }
 
     /**
@@ -103,6 +90,7 @@ protected Request setCommonOptions(RestRequest restRequest, Request request) {
 
         request.setRefresh(restRequest.paramAsBoolean("refresh", request.isRefresh()));
         request.setTimeout(restRequest.paramAsTime("timeout", request.getTimeout()));
+        request.setSlices(restRequest.paramAsInt("slices", request.getSlices()));
 
         String waitForActiveShards = restRequest.param("wait_for_active_shards");
         if (waitForActiveShards != null) {
@@ -116,13 +104,15 @@ protected Request setCommonOptions(RestRequest restRequest, Request request) {
         return request;
     }
 
-    private void sendTask(RestChannel channel, Task task) throws IOException {
-        try (XContentBuilder builder = channel.newBuilder()) {
-            builder.startObject();
-            builder.field("task", clusterService.localNode().getId() + ":" + task.getId());
-            builder.endObject();
-            channel.sendResponse(new BytesRestResponse(RestStatus.OK, builder));
-        }
+    private RestChannelConsumer sendTask(String localNodeId, Task task) throws IOException {
+        return channel -> {
+            try (XContentBuilder builder = channel.newBuilder()) {
+                builder.startObject();
+                builder.field("task", localNodeId + ":" + task.getId());
+                builder.endObject();
+                channel.sendResponse(new BytesRestResponse(RestStatus.OK, builder));
+            }
+        };
     }
 
     /**
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByQueryRestHandler.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByQueryRestHandler.java
index 2db5f4123cf17..64209653c6f4f 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByQueryRestHandler.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByQueryRestHandler.java
@@ -19,24 +19,18 @@
 
 package org.elasticsearch.index.reindex;
 
-import java.io.IOException;
-import java.util.Map;
-import java.util.function.Consumer;
-
 import org.elasticsearch.action.GenericAction;
 import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.action.RestActions;
 import org.elasticsearch.rest.action.search.RestSearchAction;
-import org.elasticsearch.search.SearchRequestParsers;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.function.Consumer;
 
 import static org.elasticsearch.index.reindex.AbstractBulkByScrollRequest.SIZE_ALL_MATCHES;
 
@@ -45,15 +39,14 @@
  */
 public abstract class AbstractBulkByQueryRestHandler<
         Request extends AbstractBulkByScrollRequest<Request>,
-        A extends GenericAction<Request, BulkIndexByScrollResponse>> extends AbstractBaseReindexRestHandler<Request, A> {
+        A extends GenericAction<Request, BulkByScrollResponse>> extends AbstractBaseReindexRestHandler<Request, A> {
 
-    protected AbstractBulkByQueryRestHandler(Settings settings, SearchRequestParsers searchRequestParsers,
-                                             ClusterService clusterService, A action) {
-        super(settings, searchRequestParsers, clusterService, action);
+    protected AbstractBulkByQueryRestHandler(Settings settings, A action) {
+        super(settings, action);
     }
 
     protected void parseInternalRequest(Request internal, RestRequest restRequest,
-                                        Map<String, Consumer<Object>> consumers) throws IOException {
+                                        Map<String, Consumer<Object>> bodyConsumers) throws IOException {
         assert internal != null : "Request should not be null";
         assert restRequest != null : "RestRequest should not be null";
 
@@ -61,7 +54,9 @@ protected void parseInternalRequest(Request internal, RestRequest restRequest,
         int scrollSize = searchRequest.source().size();
         searchRequest.source().size(SIZE_ALL_MATCHES);
 
-        parseSearchRequest(searchRequest, restRequest, consumers);
+        try (XContentParser parser = extractRequestSpecificFields(restRequest, bodyConsumers)) {
+            RestSearchAction.parseSearchRequest(searchRequest, restRequest, parser);
+        }
 
         internal.setSize(searchRequest.source().size());
         searchRequest.source().size(restRequest.paramAsInt("scroll_size", scrollSize));
@@ -77,37 +72,29 @@ protected void parseInternalRequest(Request internal, RestRequest restRequest,
         }
     }
 
-    protected void parseSearchRequest(SearchRequest searchRequest, RestRequest restRequest,
-                                      Map<String, Consumer<Object>> consumers) throws IOException {
-        assert searchRequest != null : "SearchRequest should not be null";
-        assert restRequest != null : "RestRequest should not be null";
+    /**
+     * We can't send parseSearchRequest REST content that it doesn't support
+     * so we will have to remove the content that is valid in addition to
+     * what it supports from the content first. This is a temporary hack and
+     * should get better when SearchRequest has full ObjectParser support
+     * then we can delegate and stuff.
+     */
+    private XContentParser extractRequestSpecificFields(RestRequest restRequest,
+                                                        Map<String, Consumer<Object>> bodyConsumers) throws IOException {
+        if (restRequest.hasContentOrSourceParam() == false) {
+            return null; // body is optional
+        }
+        try (XContentParser parser = restRequest.contentOrSourceParamParser();
+             XContentBuilder builder = XContentFactory.contentBuilder(parser.contentType())) {
+            Map<String, Object> body = parser.map();
 
-        /*
-         * We can't send parseSearchRequest REST content that it doesn't support
-         * so we will have to remove the content that is valid in addition to
-         * what it supports from the content first. This is a temporary hack and
-         * should get better when SearchRequest has full ObjectParser support
-         * then we can delegate and stuff.
-         */
-        BytesReference content = RestActions.hasBodyContent(restRequest) ? RestActions.getRestContent(restRequest) : null;
-        if ((content != null) && (consumers != null && consumers.size() > 0)) {
-            Tuple<XContentType, Map<String, Object>> body = XContentHelper.convertToMap(content, false);
-            boolean modified = false;
-            for (Map.Entry<String, Consumer<Object>> consumer : consumers.entrySet()) {
-                Object value = body.v2().remove(consumer.getKey());
+            for (Map.Entry<String, Consumer<Object>> consumer : bodyConsumers.entrySet()) {
+                Object value = body.remove(consumer.getKey());
                 if (value != null) {
                     consumer.getValue().accept(value);
-                    modified = true;
-                }
-            }
-
-            if (modified) {
-                try (XContentBuilder builder = XContentFactory.contentBuilder(body.v1())) {
-                    content = builder.map(body.v2()).bytes();
                 }
             }
+            return parser.contentType().xContent().createParser(parser.getXContentRegistry(), builder.map(body).bytes());
         }
-
-        RestSearchAction.parseSearchRequest(searchRequest, restRequest, searchRequestParsers, parseFieldMatcher, content);
     }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java
deleted file mode 100644
index 1329aeb6d8cdb..0000000000000
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java
+++ /dev/null
@@ -1,364 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.reindex;
-
-import org.elasticsearch.action.ActionRequest;
-import org.elasticsearch.action.ActionRequestValidationException;
-import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.support.ActiveShardCount;
-import org.elasticsearch.action.support.replication.ReplicationRequest;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.elasticsearch.tasks.Task;
-import org.elasticsearch.tasks.TaskId;
-
-import java.io.IOException;
-import java.util.Arrays;
-
-import static org.elasticsearch.action.ValidateActions.addValidationError;
-import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
-import static org.elasticsearch.common.unit.TimeValue.timeValueMinutes;
-
-public abstract class AbstractBulkByScrollRequest<Self extends AbstractBulkByScrollRequest<Self>>
-        extends ActionRequest<Self> {
-    public static final int SIZE_ALL_MATCHES = -1;
-    private static final TimeValue DEFAULT_SCROLL_TIMEOUT = timeValueMinutes(5);
-    private static final int DEFAULT_SCROLL_SIZE = 1000;
-
-    /**
-     * The search to be executed.
-     */
-    private SearchRequest searchRequest;
-
-    /**
-     * Maximum number of processed documents. Defaults to -1 meaning process all
-     * documents.
-     */
-    private int size = SIZE_ALL_MATCHES;
-
-    /**
-     * Should version conflicts cause aborts? Defaults to true.
-     */
-    private boolean abortOnVersionConflict = true;
-
-    /**
-     * Call refresh on the indexes we've written to after the request ends?
-     */
-    private boolean refresh = false;
-
-    /**
-     * Timeout to wait for the shards on to be available for each bulk request?
-     */
-    private TimeValue timeout = ReplicationRequest.DEFAULT_TIMEOUT;
-
-    /**
-     * The number of shard copies that must be active before proceeding with the write.
-     */
-    private ActiveShardCount activeShardCount = ActiveShardCount.DEFAULT;
-
-    /**
-     * Initial delay after a rejection before retrying a bulk request. With the default maxRetries the total backoff for retrying rejections
-     * is about one minute per bulk request. Once the entire bulk request is successful the retry counter resets.
-     */
-    private TimeValue retryBackoffInitialTime = timeValueMillis(500);
-
-    /**
-     * Total number of retries attempted for rejections. There is no way to ask for unlimited retries.
-     */
-    private int maxRetries = 11;
-
-    /**
-     * The throttle for this request in sub-requests per second. {@link Float#POSITIVE_INFINITY} means set no throttle and that is the
-     * default. Throttling is done between batches, as we start the next scroll requests. That way we can increase the scroll's timeout to
-     * make sure that it contains any time that we might wait.
-     */
-    private float requestsPerSecond = Float.POSITIVE_INFINITY;
-
-    /**
-     * Should this task store its result?
-     */
-    private boolean shouldStoreResult;
-
-    public AbstractBulkByScrollRequest() {
-    }
-
-    public AbstractBulkByScrollRequest(SearchRequest source) {
-        this.searchRequest = source;
-
-        // Set the defaults which differ from SearchRequest's defaults.
-        source.scroll(DEFAULT_SCROLL_TIMEOUT);
-        source.source(new SearchSourceBuilder());
-        source.source().size(DEFAULT_SCROLL_SIZE);
-    }
-
-    /**
-     * `this` cast to Self. Used for building fluent methods without cast
-     * warnings.
-     */
-    protected abstract Self self();
-
-    @Override
-    public ActionRequestValidationException validate() {
-        ActionRequestValidationException e = searchRequest.validate();
-        if (searchRequest.source().from() != -1) {
-            e = addValidationError("from is not supported in this context", e);
-        }
-        if (searchRequest.source().storedFields() != null) {
-            e = addValidationError("stored_fields is not supported in this context", e);
-        }
-        if (maxRetries < 0) {
-            e = addValidationError("retries cannnot be negative", e);
-        }
-        if (false == (size == -1 || size > 0)) {
-            e = addValidationError(
-                    "size should be greater than 0 if the request is limited to some number of documents or -1 if it isn't but it was ["
-                            + size + "]",
-                    e);
-        }
-        return e;
-    }
-
-    /**
-     * Maximum number of processed documents. Defaults to -1 meaning process all
-     * documents.
-     */
-    public int getSize() {
-        return size;
-    }
-
-    /**
-     * Maximum number of processed documents. Defaults to -1 meaning process all
-     * documents.
-     */
-    public Self setSize(int size) {
-        this.size = size;
-        return self();
-    }
-
-    /**
-     * Should version conflicts cause aborts? Defaults to false.
-     */
-    public boolean isAbortOnVersionConflict() {
-        return abortOnVersionConflict;
-    }
-
-    /**
-     * Should version conflicts cause aborts? Defaults to false.
-     */
-    public Self setAbortOnVersionConflict(boolean abortOnVersionConflict) {
-        this.abortOnVersionConflict = abortOnVersionConflict;
-        return self();
-    }
-
-    /**
-     * Sets abortOnVersionConflict based on REST-friendly names.
-     */
-    public void setConflicts(String conflicts) {
-        switch (conflicts) {
-        case "proceed":
-            setAbortOnVersionConflict(false);
-            return;
-        case "abort":
-            setAbortOnVersionConflict(true);
-            return;
-        default:
-            throw new IllegalArgumentException("conflicts may only be \"proceed\" or \"abort\" but was [" + conflicts + "]");
-        }
-    }
-
-    /**
-     * The search request that matches the documents to process.
-     */
-    public SearchRequest getSearchRequest() {
-        return searchRequest;
-    }
-
-    /**
-     * Call refresh on the indexes we've written to after the request ends?
-     */
-    public boolean isRefresh() {
-        return refresh;
-    }
-
-    /**
-     * Call refresh on the indexes we've written to after the request ends?
-     */
-    public Self setRefresh(boolean refresh) {
-        this.refresh = refresh;
-        return self();
-    }
-
-    /**
-     * Timeout to wait for the shards on to be available for each bulk request?
-     */
-    public TimeValue getTimeout() {
-        return timeout;
-    }
-
-    /**
-     * Timeout to wait for the shards on to be available for each bulk request?
-     */
-    public Self setTimeout(TimeValue timeout) {
-        this.timeout = timeout;
-        return self();
-    }
-
-    /**
-     * The number of shard copies that must be active before proceeding with the write.
-     */
-    public ActiveShardCount getWaitForActiveShards() {
-        return activeShardCount;
-    }
-
-    /**
-     * Sets the number of shard copies that must be active before proceeding with the write.
-     * See {@link ReplicationRequest#waitForActiveShards(ActiveShardCount)} for details.
-     */
-    public Self setWaitForActiveShards(ActiveShardCount activeShardCount) {
-        this.activeShardCount = activeShardCount;
-        return self();
-    }
-
-    /**
-     * A shortcut for {@link #setWaitForActiveShards(ActiveShardCount)} where the numerical
-     * shard count is passed in, instead of having to first call {@link ActiveShardCount#from(int)}
-     * to get the ActiveShardCount.
-     */
-    public Self setWaitForActiveShards(final int waitForActiveShards) {
-        return setWaitForActiveShards(ActiveShardCount.from(waitForActiveShards));
-    }
-
-    /**
-     * Initial delay after a rejection before retrying request.
-     */
-    public TimeValue getRetryBackoffInitialTime() {
-        return retryBackoffInitialTime;
-    }
-
-    /**
-     * Set the initial delay after a rejection before retrying request.
-     */
-    public Self setRetryBackoffInitialTime(TimeValue retryBackoffInitialTime) {
-        this.retryBackoffInitialTime = retryBackoffInitialTime;
-        return self();
-    }
-
-    /**
-     * Total number of retries attempted for rejections.
-     */
-    public int getMaxRetries() {
-        return maxRetries;
-    }
-
-    /**
-     * Set the total number of retries attempted for rejections. There is no way to ask for unlimited retries.
-     */
-    public Self setMaxRetries(int maxRetries) {
-        this.maxRetries = maxRetries;
-        return self();
-    }
-
-    /**
-     * The throttle for this request in sub-requests per second. {@link Float#POSITIVE_INFINITY} means set no throttle and that is the
-     * default. Throttling is done between batches, as we start the next scroll requests. That way we can increase the scroll's timeout to
-     * make sure that it contains any time that we might wait.
-     */
-    public float getRequestsPerSecond() {
-        return requestsPerSecond;
-    }
-
-    /**
-     * Set the throttle for this request in sub-requests per second. {@link Float#POSITIVE_INFINITY} means set no throttle and that is the
-     * default. Throttling is done between batches, as we start the next scroll requests. That way we can increase the scroll's timeout to
-     * make sure that it contains any time that we might wait.
-     */
-    public Self setRequestsPerSecond(float requestsPerSecond) {
-        if (requestsPerSecond <= 0) {
-            throw new IllegalArgumentException(
-                    "[requests_per_second] must be greater than 0. Use Float.POSITIVE_INFINITY to disable throttling.");
-        }
-        this.requestsPerSecond = requestsPerSecond;
-        return self();
-    }
-
-    /**
-     * Should this task store its result after it has finished?
-     */
-    public Self setShouldStoreResult(boolean shouldStoreResult) {
-        this.shouldStoreResult = shouldStoreResult;
-        return self();
-    }
-
-    @Override
-    public boolean getShouldStoreResult() {
-        return shouldStoreResult;
-    }
-
-    @Override
-    public Task createTask(long id, String type, String action, TaskId parentTaskId) {
-        return new BulkByScrollTask(id, type, action, getDescription(), parentTaskId, requestsPerSecond);
-    }
-
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        searchRequest = new SearchRequest();
-        searchRequest.readFrom(in);
-        abortOnVersionConflict = in.readBoolean();
-        size = in.readVInt();
-        refresh = in.readBoolean();
-        timeout = new TimeValue(in);
-        activeShardCount = ActiveShardCount.readFrom(in);
-        retryBackoffInitialTime = new TimeValue(in);
-        maxRetries = in.readVInt();
-        requestsPerSecond = in.readFloat();
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        super.writeTo(out);
-        searchRequest.writeTo(out);
-        out.writeBoolean(abortOnVersionConflict);
-        out.writeVInt(size);
-        out.writeBoolean(refresh);
-        timeout.writeTo(out);
-        activeShardCount.writeTo(out);
-        retryBackoffInitialTime.writeTo(out);
-        out.writeVInt(maxRetries);
-        out.writeFloat(requestsPerSecond);
-    }
-
-    /**
-     * Append a short description of the search request to a StringBuilder. Used
-     * to make toString.
-     */
-    protected void searchToString(StringBuilder b) {
-        if (searchRequest.indices() != null && searchRequest.indices().length != 0) {
-            b.append(Arrays.toString(searchRequest.indices()));
-        } else {
-            b.append("[all indices]");
-        }
-        if (searchRequest.types() != null && searchRequest.types().length != 0) {
-            b.append(Arrays.toString(searchRequest.types()));
-        }
-    }
-}
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AsyncDeleteByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AsyncDeleteByQueryAction.java
new file mode 100644
index 0000000000000..2608f5715ba3e
--- /dev/null
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AsyncDeleteByQueryAction.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex;
+
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.client.ParentTaskAssigningClient;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.threadpool.ThreadPool;
+
+/**
+ * Implementation of delete-by-query using scrolling and bulk.
+ */
+public class AsyncDeleteByQueryAction extends AbstractAsyncBulkByScrollAction<DeleteByQueryRequest> {
+    public AsyncDeleteByQueryAction(WorkingBulkByScrollTask task, Logger logger, ParentTaskAssigningClient client,
+                                    ThreadPool threadPool, DeleteByQueryRequest request, ScriptService scriptService,
+                                    ClusterState clusterState, ActionListener<BulkByScrollResponse> listener) {
+        super(task, logger, client, threadPool, request, scriptService, clusterState, listener);
+    }
+
+    @Override
+    protected boolean needsSourceDocumentVersions() {
+        /*
+         * We always need the version of the source document so we can report a version conflict if we try to delete it and it has been
+         * changed.
+         */
+        return true;
+    }
+
+    @Override
+    protected boolean accept(ScrollableHitSource.Hit doc) {
+        // Delete-by-query does not require the source to delete a document
+        // and the default implementation checks for it
+        return true;
+    }
+
+    @Override
+    protected RequestWrapper<DeleteRequest> buildRequest(ScrollableHitSource.Hit doc) {
+        DeleteRequest delete = new DeleteRequest();
+        delete.index(doc.getIndex());
+        delete.type(doc.getType());
+        delete.id(doc.getId());
+        delete.version(doc.getVersion());
+        return wrap(delete);
+    }
+
+    /**
+     * Overrides the parent's implementation is much more Update/Reindex oriented and so also copies things like timestamp/ttl which we
+     * don't care for a deletion.
+     */
+    @Override
+    protected RequestWrapper<?> copyMetadata(RequestWrapper<?> request, ScrollableHitSource.Hit doc) {
+        request.setParent(doc.getParent());
+        request.setRouting(doc.getRouting());
+        return request;
+    }
+
+}
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java
new file mode 100644
index 0000000000000..48f1030645450
--- /dev/null
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.index.mapper.UidFieldMapper;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.slice.SliceBuilder;
+import org.elasticsearch.tasks.TaskId;
+import org.elasticsearch.tasks.TaskManager;
+
+/**
+ * Helps parallelize reindex requests using sliced scrolls.
+ */
+class BulkByScrollParallelizationHelper {
+    private BulkByScrollParallelizationHelper() {}
+
+    public static <Request extends AbstractBulkByScrollRequest<Request>> void startSlices(Client client, TaskManager taskManager,
+                               Action<Request, BulkByScrollResponse, ?> action,
+                               String localNodeId, ParentBulkByScrollTask task, Request request,
+                               ActionListener<BulkByScrollResponse> listener) {
+        TaskId parentTaskId = new TaskId(localNodeId, task.getId());
+        for (final SearchRequest slice : sliceIntoSubRequests(request.getSearchRequest(), UidFieldMapper.NAME, request.getSlices())) {
+            // TODO move the request to the correct node. maybe here or somehow do it as part of startup for reindex in general....
+            Request requestForSlice = request.forSlice(parentTaskId, slice);
+            ActionListener<BulkByScrollResponse> sliceListener = ActionListener.wrap(
+                    r -> task.onSliceResponse(listener, slice.source().slice().getId(), r),
+                    e -> task.onSliceFailure(listener, slice.source().slice().getId(), e));
+            client.execute(action, requestForSlice, sliceListener);
+        }
+    }
+
+    /**
+     * Slice a search request into {@code times} separate search requests slicing on {@code field}. Note that the slices are *shallow*
+     * copies of this request so don't change them.
+     */
+    static SearchRequest[] sliceIntoSubRequests(SearchRequest request, String field, int times) {
+        SearchRequest[] slices = new SearchRequest[times];
+        for (int slice = 0; slice < times; slice++) {
+            SliceBuilder sliceBuilder = new SliceBuilder(field, slice, times);
+            SearchSourceBuilder slicedSource;
+            if (request.source() == null) {
+                slicedSource = new SearchSourceBuilder().slice(sliceBuilder);
+            } else {
+                if (request.source().slice() != null) {
+                    throw new IllegalStateException("Can't slice a request that already has a slice configuration");
+                }
+                slicedSource = request.source().copyWithNewSlice(sliceBuilder);
+            }
+            slices[slice] = new SearchRequest()
+                    .source(slicedSource)
+                    .searchType(request.searchType())
+                    .indices(request.indices())
+                    .types(request.types())
+                    .routing(request.routing())
+                    .preference(request.preference())
+                    .requestCache(request.requestCache())
+                    .scroll(request.scroll())
+                    .indicesOptions(request.indicesOptions());
+        }
+        return slices;
+    }
+
+}
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java
deleted file mode 100644
index ade1f8c2f84b5..0000000000000
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java
+++ /dev/null
@@ -1,553 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.reindex;
-
-import org.apache.logging.log4j.Logger;
-import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.logging.ESLoggerFactory;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.util.concurrent.AbstractRunnable;
-import org.elasticsearch.common.util.concurrent.FutureUtils;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.tasks.CancellableTask;
-import org.elasticsearch.tasks.Task;
-import org.elasticsearch.tasks.TaskId;
-import org.elasticsearch.threadpool.ThreadPool;
-
-import java.io.IOException;
-import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.atomic.AtomicReference;
-
-import static java.lang.Math.max;
-import static java.lang.Math.round;
-import static org.elasticsearch.common.unit.TimeValue.timeValueNanos;
-
-/**
- * Task storing information about a currently running BulkByScroll request.
- */
-public class BulkByScrollTask extends CancellableTask {
-    private static final Logger logger = ESLoggerFactory.getLogger(BulkByScrollTask.class.getPackage().getName());
-
-    /**
-     * The total number of documents this request will process. 0 means we don't yet know or, possibly, there are actually 0 documents
-     * to process. Its ok that these have the same meaning because any request with 0 actual documents should be quite short lived.
-     */
-    private final AtomicLong total = new AtomicLong(0);
-    private final AtomicLong updated = new AtomicLong(0);
-    private final AtomicLong created = new AtomicLong(0);
-    private final AtomicLong deleted = new AtomicLong(0);
-    private final AtomicLong noops = new AtomicLong(0);
-    private final AtomicInteger batch = new AtomicInteger(0);
-    private final AtomicLong versionConflicts = new AtomicLong(0);
-    private final AtomicLong bulkRetries = new AtomicLong(0);
-    private final AtomicLong searchRetries = new AtomicLong(0);
-    private final AtomicLong throttledNanos = new AtomicLong();
-    /**
-     * The number of requests per second to which to throttle the request that this task represents. The other variables are all AtomicXXX
-     * style variables but there isn't an AtomicFloat so we just use a volatile.
-     */
-    private volatile float requestsPerSecond;
-    /**
-     * Reference to any the last delayed prepareBulkRequest call. Used during rethrottling and canceling to reschedule the request.
-     */
-    private final AtomicReference<DelayedPrepareBulkRequest> delayedPrepareBulkRequestReference = new AtomicReference<>();
-
-    public BulkByScrollTask(long id, String type, String action, String description, TaskId parentTask, float requestsPerSecond) {
-        super(id, type, action, description, parentTask);
-        setRequestsPerSecond(requestsPerSecond);
-    }
-
-    @Override
-    protected void onCancelled() {
-        // Drop the throttle to 0, immediately rescheduling all outstanding tasks so the task will wake up and cancel itself.
-        rethrottle(0);
-    }
-
-    @Override
-    public Status getStatus() {
-        return new Status(total.get(), updated.get(), created.get(), deleted.get(), batch.get(), versionConflicts.get(), noops.get(),
-                bulkRetries.get(), searchRetries.get(), timeValueNanos(throttledNanos.get()), getRequestsPerSecond(), getReasonCancelled(),
-                throttledUntil());
-    }
-
-    private TimeValue throttledUntil() {
-        DelayedPrepareBulkRequest delayed = delayedPrepareBulkRequestReference.get();
-        if (delayed == null) {
-            return timeValueNanos(0);
-        }
-        if (delayed.future == null) {
-            return timeValueNanos(0);
-        }
-        return timeValueNanos(max(0, delayed.future.getDelay(TimeUnit.NANOSECONDS)));
-    }
-
-    /**
-     * Total number of successfully processed documents.
-     */
-    public long getSuccessfullyProcessed() {
-        return updated.get() + created.get() + deleted.get();
-    }
-
-    public static class Status implements Task.Status {
-        public static final String NAME = "bulk-by-scroll";
-
-        /**
-         * XContent param name to indicate if "created" count must be included
-         * in the response.
-         */
-        public static final String INCLUDE_CREATED = "include_created";
-
-        /**
-         * XContent param name to indicate if "updated" count must be included
-         * in the response.
-         */
-        public static final String INCLUDE_UPDATED = "include_updated";
-
-        private final long total;
-        private final long updated;
-        private final long created;
-        private final long deleted;
-        private final int batches;
-        private final long versionConflicts;
-        private final long noops;
-        private final long bulkRetries;
-        private final long searchRetries;
-        private final TimeValue throttled;
-        private final float requestsPerSecond;
-        private final String reasonCancelled;
-        private final TimeValue throttledUntil;
-
-        public Status(long total, long updated, long created, long deleted, int batches, long versionConflicts, long noops,
-                long bulkRetries, long searchRetries, TimeValue throttled, float requestsPerSecond, @Nullable String reasonCancelled,
-                TimeValue throttledUntil) {
-            this.total = checkPositive(total, "total");
-            this.updated = checkPositive(updated, "updated");
-            this.created = checkPositive(created, "created");
-            this.deleted = checkPositive(deleted, "deleted");
-            this.batches = checkPositive(batches, "batches");
-            this.versionConflicts = checkPositive(versionConflicts, "versionConflicts");
-            this.noops = checkPositive(noops, "noops");
-            this.bulkRetries = checkPositive(bulkRetries, "bulkRetries");
-            this.searchRetries = checkPositive(searchRetries, "searchRetries");
-            this.throttled = throttled;
-            this.requestsPerSecond = requestsPerSecond;
-            this.reasonCancelled = reasonCancelled;
-            this.throttledUntil = throttledUntil;
-        }
-
-        public Status(StreamInput in) throws IOException {
-            total = in.readVLong();
-            updated = in.readVLong();
-            created = in.readVLong();
-            deleted = in.readVLong();
-            batches = in.readVInt();
-            versionConflicts = in.readVLong();
-            noops = in.readVLong();
-            bulkRetries = in.readVLong();
-            searchRetries = in.readVLong();
-            throttled = new TimeValue(in);
-            requestsPerSecond = in.readFloat();
-            reasonCancelled = in.readOptionalString();
-            throttledUntil = new TimeValue(in);
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            out.writeVLong(total);
-            out.writeVLong(updated);
-            out.writeVLong(created);
-            out.writeVLong(deleted);
-            out.writeVInt(batches);
-            out.writeVLong(versionConflicts);
-            out.writeVLong(noops);
-            out.writeVLong(bulkRetries);
-            out.writeVLong(searchRetries);
-            throttled.writeTo(out);
-            out.writeFloat(requestsPerSecond);
-            out.writeOptionalString(reasonCancelled);
-            throttledUntil.writeTo(out);
-        }
-
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startObject();
-            innerXContent(builder, params);
-            return builder.endObject();
-        }
-
-        public XContentBuilder innerXContent(XContentBuilder builder, Params params)
-                throws IOException {
-            builder.field("total", total);
-            if (params.paramAsBoolean(INCLUDE_UPDATED, true)) {
-                builder.field("updated", updated);
-            }
-            if (params.paramAsBoolean(INCLUDE_CREATED, true)) {
-                builder.field("created", created);
-            }
-            builder.field("deleted", deleted);
-            builder.field("batches", batches);
-            builder.field("version_conflicts", versionConflicts);
-            builder.field("noops", noops);
-            builder.startObject("retries"); {
-                builder.field("bulk", bulkRetries);
-                builder.field("search", searchRetries);
-            }
-            builder.endObject();
-            builder.timeValueField("throttled_millis", "throttled", throttled);
-            builder.field("requests_per_second", requestsPerSecond == Float.POSITIVE_INFINITY ? -1 : requestsPerSecond);
-            if (reasonCancelled != null) {
-                builder.field("canceled", reasonCancelled);
-            }
-            builder.timeValueField("throttled_until_millis", "throttled_until", throttledUntil);
-            return builder;
-        }
-
-        @Override
-        public String toString() {
-            StringBuilder builder = new StringBuilder();
-            builder.append("BulkIndexByScrollResponse[");
-            innerToString(builder);
-            return builder.append(']').toString();
-        }
-
-        public void innerToString(StringBuilder builder) {
-            builder.append("updated=").append(updated);
-            builder.append(",created=").append(created);
-            builder.append(",deleted=").append(deleted);
-            builder.append(",batches=").append(batches);
-            builder.append(",versionConflicts=").append(versionConflicts);
-            builder.append(",noops=").append(noops);
-            builder.append(",retries=").append(bulkRetries);
-            if (reasonCancelled != null) {
-                builder.append(",canceled=").append(reasonCancelled);
-            }
-            builder.append(",throttledUntil=").append(throttledUntil);
-        }
-
-        @Override
-        public String getWriteableName() {
-            return NAME;
-        }
-
-        /**
-         * The total number of documents this request will process. 0 means we don't yet know or, possibly, there are actually 0 documents
-         * to process. Its ok that these have the same meaning because any request with 0 actual documents should be quite short lived.
-         */
-        public long getTotal() {
-            return total;
-        }
-
-        /**
-         * Count of documents updated.
-         */
-        public long getUpdated() {
-            return updated;
-        }
-
-        /**
-         * Count of documents created.
-         */
-        public long getCreated() {
-            return created;
-        }
-
-        /**
-         * Count of successful delete operations.
-         */
-        public long getDeleted() {
-            return deleted;
-        }
-
-        /**
-         * Number of scan responses this request has processed.
-         */
-        public int getBatches() {
-            return batches;
-        }
-
-        /**
-         * Number of version conflicts this request has hit.
-         */
-        public long getVersionConflicts() {
-            return versionConflicts;
-        }
-
-        /**
-         * Number of noops (skipped bulk items) as part of this request.
-         */
-        public long getNoops() {
-            return noops;
-        }
-
-        /**
-         * Number of retries that had to be attempted due to bulk actions being rejected.
-         */
-        public long getBulkRetries() {
-            return bulkRetries;
-        }
-
-        /**
-         * Number of retries that had to be attempted due to search actions being rejected.
-         */
-        public long getSearchRetries() {
-            return searchRetries;
-        }
-
-        /**
-         * The total time this request has throttled itself not including the current throttle time if it is currently sleeping.
-         */
-        public TimeValue getThrottled() {
-            return throttled;
-        }
-
-        /**
-         * The number of requests per second to which to throttle the request. Float.POSITIVE_INFINITY means unlimited.
-         */
-        public float getRequestsPerSecond() {
-            return requestsPerSecond;
-        }
-
-        /**
-         * The reason that the request was canceled or null if it hasn't been.
-         */
-        public String getReasonCancelled() {
-            return reasonCancelled;
-        }
-
-        /**
-         * Remaining delay of any current throttle sleep or 0 if not sleeping.
-         */
-        public TimeValue getThrottledUntil() {
-            return throttledUntil;
-        }
-
-        private int checkPositive(int value, String name) {
-            if (value < 0) {
-                throw new IllegalArgumentException(name + " must be greater than 0 but was [" + value + "]");
-            }
-            return value;
-        }
-
-        private long checkPositive(long value, String name) {
-            if (value < 0) {
-                throw new IllegalArgumentException(name + " must be greater than 0 but was [" + value + "]");
-            }
-            return value;
-        }
-    }
-
-    void setTotal(long totalHits) {
-        total.set(totalHits);
-    }
-
-    void countBatch() {
-        batch.incrementAndGet();
-    }
-
-    void countNoop() {
-        noops.incrementAndGet();
-    }
-
-    void countCreated() {
-        created.incrementAndGet();
-    }
-
-    void countUpdated() {
-        updated.incrementAndGet();
-    }
-
-    void countDeleted() {
-        deleted.incrementAndGet();
-    }
-
-    void countVersionConflict() {
-        versionConflicts.incrementAndGet();
-    }
-
-    void countBulkRetry() {
-        bulkRetries.incrementAndGet();
-    }
-
-    void countSearchRetry() {
-        searchRetries.incrementAndGet();
-    }
-
-    float getRequestsPerSecond() {
-        return requestsPerSecond;
-    }
-
-    /**
-     * Schedule prepareBulkRequestRunnable to run after some delay. This is where throttling plugs into reindexing so the request can be
-     * rescheduled over and over again.
-     */
-    void delayPrepareBulkRequest(ThreadPool threadPool, TimeValue lastBatchStartTime, int lastBatchSize,
-            AbstractRunnable prepareBulkRequestRunnable) {
-        // Synchronize so we are less likely to schedule the same request twice.
-        synchronized (delayedPrepareBulkRequestReference) {
-            TimeValue delay = throttleWaitTime(lastBatchStartTime, lastBatchSize);
-            delayedPrepareBulkRequestReference.set(new DelayedPrepareBulkRequest(threadPool, getRequestsPerSecond(),
-                    delay, new RunOnce(prepareBulkRequestRunnable)));
-        }
-    }
-
-    TimeValue throttleWaitTime(TimeValue lastBatchStartTime, int lastBatchSize) {
-        long earliestNextBatchStartTime = lastBatchStartTime.nanos() + (long) perfectlyThrottledBatchTime(lastBatchSize);
-        return timeValueNanos(max(0, earliestNextBatchStartTime - System.nanoTime()));
-    }
-
-    /**
-     * How many nanoseconds should a batch of lastBatchSize have taken if it were perfectly throttled? Package private for testing.
-     */
-    float perfectlyThrottledBatchTime(int lastBatchSize) {
-        if (requestsPerSecond == Float.POSITIVE_INFINITY) {
-            return 0;
-        }
-        //       requests
-        // ------------------- == seconds
-        // request per seconds
-        float targetBatchTimeInSeconds = lastBatchSize / requestsPerSecond;
-        // nanoseconds per seconds * seconds == nanoseconds
-        return TimeUnit.SECONDS.toNanos(1) * targetBatchTimeInSeconds;
-    }
-
-    private void setRequestsPerSecond(float requestsPerSecond) {
-        this.requestsPerSecond = requestsPerSecond;
-    }
-
-    void rethrottle(float newRequestsPerSecond) {
-        synchronized (delayedPrepareBulkRequestReference) {
-            if (logger.isDebugEnabled()) {
-                logger.debug("[{}]: Rethrottling to [{}] requests per second", getId(), newRequestsPerSecond);
-            }
-            setRequestsPerSecond(newRequestsPerSecond);
-
-            DelayedPrepareBulkRequest delayedPrepareBulkRequest = this.delayedPrepareBulkRequestReference.get();
-            if (delayedPrepareBulkRequest == null) {
-                if (logger.isDebugEnabled()) {
-                    logger.debug("[{}]: Skipping rescheduling because there is no scheduled task", getId());
-                }
-                // No request has been queued yet so nothing to reschedule.
-                return;
-            }
-
-            this.delayedPrepareBulkRequestReference.set(delayedPrepareBulkRequest.rethrottle(newRequestsPerSecond));
-        }
-    }
-
-    class DelayedPrepareBulkRequest {
-        private final ThreadPool threadPool;
-        private final AbstractRunnable command;
-        private final float requestsPerSecond;
-        private final ScheduledFuture<?> future;
-
-        DelayedPrepareBulkRequest(ThreadPool threadPool, float requestsPerSecond, TimeValue delay, AbstractRunnable command) {
-            this.threadPool = threadPool;
-            this.requestsPerSecond = requestsPerSecond;
-            this.command = command;
-            this.future = threadPool.schedule(delay, ThreadPool.Names.GENERIC, new AbstractRunnable() {
-                @Override
-                protected void doRun() throws Exception {
-                    throttledNanos.addAndGet(delay.nanos());
-                    command.run();
-                }
-
-                @Override
-                public void onFailure(Exception e) {
-                    command.onFailure(e);
-                }
-            });
-        }
-
-        DelayedPrepareBulkRequest rethrottle(float newRequestsPerSecond) {
-            if (newRequestsPerSecond != 0 && newRequestsPerSecond < requestsPerSecond) {
-                /*
-                 * The user is attempting to slow the request down. We'll let the change in throttle take effect the next time we delay
-                 * prepareBulkRequest. We can't just reschedule the request further out in the future the bulk context might time out.
-                 */
-                if (logger.isDebugEnabled()) {
-                    logger.debug("[{}]: Skipping rescheduling because the new throttle [{}] is slower than the old one [{}].", getId(),
-                            newRequestsPerSecond, requestsPerSecond);
-                }
-                return this;
-            }
-
-            long remainingDelay = future.getDelay(TimeUnit.NANOSECONDS);
-            // Actually reschedule the task
-            if (false == FutureUtils.cancel(future)) {
-                // Couldn't cancel, probably because the task has finished or been scheduled. Either way we have nothing to do here.
-                if (logger.isDebugEnabled()) {
-                    logger.debug("[{}]: Skipping rescheduling we couldn't cancel the task.", getId());
-                }
-                return this;
-            }
-
-            /*
-             * Strangely enough getting here doesn't mean that you actually cancelled the request, just that you probably did. If you stress
-             * test it you'll find that requests sneak through. So each request is given a runOnce boolean to prevent that.
-             */
-            TimeValue newDelay = newDelay(remainingDelay, newRequestsPerSecond);
-            if (logger.isDebugEnabled()) {
-                logger.debug("[{}]: Rescheduling for [{}] in the future.", getId(), newDelay);
-            }
-            return new DelayedPrepareBulkRequest(threadPool, requestsPerSecond, newDelay, command);
-        }
-
-        /**
-         * Scale back remaining delay to fit the new delay.
-         */
-        TimeValue newDelay(long remainingDelay, float newRequestsPerSecond) {
-            if (remainingDelay < 0 || newRequestsPerSecond == 0) {
-                return timeValueNanos(0);
-            }
-            return timeValueNanos(round(remainingDelay * requestsPerSecond / newRequestsPerSecond));
-        }
-    }
-
-    /**
-     * Runnable that can only be run one time. This is paranoia to prevent furiously rethrottling from running the command multiple times.
-     * Without it the command would be run multiple times.
-     */
-    private static class RunOnce extends AbstractRunnable {
-        private final AtomicBoolean hasRun = new AtomicBoolean(false);
-        private final AbstractRunnable delegate;
-
-        public RunOnce(AbstractRunnable delegate) {
-            this.delegate = delegate;
-        }
-
-        @Override
-        protected void doRun() throws Exception {
-            if (hasRun.compareAndSet(false, true)) {
-                delegate.run();
-            }
-        }
-
-        @Override
-        public void onFailure(Exception e) {
-            delegate.onFailure(e);
-        }
-    }
-}
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponse.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponse.java
deleted file mode 100644
index d717c1ad4def6..0000000000000
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponse.java
+++ /dev/null
@@ -1,182 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.reindex;
-
-import org.elasticsearch.action.ActionResponse;
-import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.index.reindex.ScrollableHitSource.SearchFailure;
-
-import java.io.IOException;
-import java.util.List;
-
-import static java.lang.Math.min;
-import static java.util.Objects.requireNonNull;
-
-/**
- * Response used for actions that index many documents using a scroll request.
- */
-public class BulkIndexByScrollResponse extends ActionResponse implements ToXContent {
-    private TimeValue took;
-    private BulkByScrollTask.Status status;
-    private List<Failure> bulkFailures;
-    private List<SearchFailure> searchFailures;
-    private boolean timedOut;
-
-    public BulkIndexByScrollResponse() {
-    }
-
-    public BulkIndexByScrollResponse(TimeValue took, BulkByScrollTask.Status status, List<Failure> bulkFailures,
-                                     List<SearchFailure> searchFailures, boolean timedOut) {
-        this.took = took;
-        this.status = requireNonNull(status, "Null status not supported");
-        this.bulkFailures = bulkFailures;
-        this.searchFailures = searchFailures;
-        this.timedOut = timedOut;
-    }
-
-    public TimeValue getTook() {
-        return took;
-    }
-
-    protected BulkByScrollTask.Status getStatus() {
-        return status;
-    }
-
-    public long getCreated() {
-        return status.getCreated();
-    }
-
-    public long getDeleted() {
-        return status.getDeleted();
-    }
-
-    public long getUpdated() {
-        return status.getUpdated();
-    }
-
-    public int getBatches() {
-        return status.getBatches();
-    }
-
-    public long getVersionConflicts() {
-        return status.getVersionConflicts();
-    }
-
-    public long getNoops() {
-        return status.getNoops();
-    }
-
-    /**
-     * The reason that the request was canceled or null if it hasn't been.
-     */
-    public String getReasonCancelled() {
-        return status.getReasonCancelled();
-    }
-
-    /**
-     * The number of times that the request had retry bulk actions.
-     */
-    public long getBulkRetries() {
-        return status.getBulkRetries();
-    }
-
-    /**
-     * The number of times that the request had retry search actions.
-     */
-    public long getSearchRetries() {
-        return status.getSearchRetries();
-    }
-
-    /**
-     * All of the bulk failures. Version conflicts are only included if the request sets abortOnVersionConflict to true (the default).
-     */
-    public List<Failure> getBulkFailures() {
-        return bulkFailures;
-    }
-
-    /**
-     * All search failures.
-     */
-    public List<SearchFailure> getSearchFailures() {
-        return searchFailures;
-    }
-
-    /**
-     * Did any of the sub-requests that were part of this request timeout?
-     */
-    public boolean isTimedOut() {
-        return timedOut;
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        super.writeTo(out);
-        took.writeTo(out);
-        status.writeTo(out);
-        out.writeList(bulkFailures);
-        out.writeList(searchFailures);
-        out.writeBoolean(timedOut);
-    }
-
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        took = new TimeValue(in);
-        status = new BulkByScrollTask.Status(in);
-        bulkFailures = in.readList(Failure::new);
-        searchFailures = in.readList(SearchFailure::new);
-        timedOut = in.readBoolean();
-    }
-
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.field("took", took.millis());
-        builder.field("timed_out", timedOut);
-        status.innerXContent(builder, params);
-        builder.startArray("failures");
-        for (Failure failure: bulkFailures) {
-            builder.startObject();
-            failure.toXContent(builder, params);
-            builder.endObject();
-        }
-        for (SearchFailure failure: searchFailures) {
-            failure.toXContent(builder, params);
-        }
-        builder.endArray();
-        return builder;
-    }
-
-    @Override
-    public String toString() {
-        StringBuilder builder = new StringBuilder();
-        builder.append("BulkIndexByScrollResponse[");
-        builder.append("took=").append(took).append(',');
-        builder.append("timed_out=").append(timedOut).append(',');
-        status.innerToString(builder);
-        builder.append(",bulk_failures=").append(getBulkFailures().subList(0, min(3, getBulkFailures().size())));
-        builder.append(",search_failures=").append(getSearchFailures().subList(0, min(3, getSearchFailures().size())));
-        return builder.append(']').toString();
-    }
-}
\ No newline at end of file
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseContentListener.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseContentListener.java
index f531d7ce6d262..8e5dff170d481 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseContentListener.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseContentListener.java
@@ -21,9 +21,9 @@
 
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
+import org.elasticsearch.index.reindex.ScrollableHitSource.SearchFailure;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.index.reindex.ScrollableHitSource.SearchFailure;
 import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestResponse;
@@ -35,7 +35,7 @@
 /**
  * RestBuilderListener that returns higher than 200 status if there are any failures and allows to set XContent.Params.
  */
-public class BulkIndexByScrollResponseContentListener extends RestBuilderListener<BulkIndexByScrollResponse> {
+public class BulkIndexByScrollResponseContentListener extends RestBuilderListener<BulkByScrollResponse> {
 
     private final Map<String, String> params;
 
@@ -45,14 +45,14 @@ public BulkIndexByScrollResponseContentListener(RestChannel channel, Map<String,
     }
 
     @Override
-    public RestResponse buildResponse(BulkIndexByScrollResponse response, XContentBuilder builder) throws Exception {
+    public RestResponse buildResponse(BulkByScrollResponse response, XContentBuilder builder) throws Exception {
         builder.startObject();
         response.toXContent(builder, new ToXContent.DelegatingMapParams(params, channel.request()));
         builder.endObject();
         return new BytesRestResponse(getStatus(response), builder);
     }
 
-    private RestStatus getStatus(BulkIndexByScrollResponse response) {
+    private RestStatus getStatus(BulkByScrollResponse response) {
         /*
          * Return the highest numbered rest status under the assumption that higher numbered statuses are "more error" and thus more
          * interesting to the user.
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexPlugin.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexPlugin.java
index fe7bcb1f85bcc..d601f5c06e728 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexPlugin.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexPlugin.java
@@ -21,15 +21,23 @@
 
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestHandler;
 import org.elasticsearch.tasks.Task;
 
 import java.util.Arrays;
 import java.util.List;
+import java.util.function.Supplier;
 
 import static java.util.Collections.singletonList;
 
@@ -37,7 +45,7 @@ public class ReindexPlugin extends Plugin implements ActionPlugin {
     public static final String NAME = "reindex";
 
     @Override
-    public List<ActionHandler<? extends ActionRequest<?>, ? extends ActionResponse>> getActions() {
+    public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
         return Arrays.asList(new ActionHandler<>(ReindexAction.INSTANCE, TransportReindexAction.class),
                 new ActionHandler<>(UpdateByQueryAction.INSTANCE, TransportUpdateByQueryAction.class),
                 new ActionHandler<>(DeleteByQueryAction.INSTANCE, TransportDeleteByQueryAction.class),
@@ -51,9 +59,14 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
     }
 
     @Override
-    public List<Class<? extends RestHandler>> getRestHandlers() {
-        return Arrays.asList(RestReindexAction.class, RestUpdateByQueryAction.class, RestDeleteByQueryAction.class,
-                RestRethrottleAction.class);
+    public List<RestHandler> getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings,
+            IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver,
+            Supplier<DiscoveryNodes> nodesInCluster) {
+        return Arrays.asList(
+                new RestReindexAction(settings, restController),
+                new RestUpdateByQueryAction(settings, restController),
+                new RestDeleteByQueryAction(settings, restController),
+                new RestRethrottleAction(settings, restController, nodesInCluster));
     }
 
     @Override
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestDeleteByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestDeleteByQueryAction.java
index 92a96a880eb11..a32bea5ed3c7b 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestDeleteByQueryAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestDeleteByQueryAction.java
@@ -22,16 +22,9 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.search.SearchRequestParsers;
-import org.elasticsearch.search.aggregations.AggregatorParsers;
-import org.elasticsearch.search.suggest.Suggesters;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -41,26 +34,20 @@
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
 public class RestDeleteByQueryAction extends AbstractBulkByQueryRestHandler<DeleteByQueryRequest, DeleteByQueryAction> {
-
-    @Inject
-    public RestDeleteByQueryAction(Settings settings, RestController controller,
-                                   SearchRequestParsers searchRequestParsers, ClusterService clusterService) {
-        super(settings, searchRequestParsers, clusterService, DeleteByQueryAction.INSTANCE);
+    public RestDeleteByQueryAction(Settings settings, RestController controller) {
+        super(settings, DeleteByQueryAction.INSTANCE);
         controller.registerHandler(POST, "/{index}/_delete_by_query", this);
         controller.registerHandler(POST, "/{index}/{type}/_delete_by_query", this);
     }
 
     @Override
-    public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
-        if (false == request.hasContent()) {
-            throw new ElasticsearchException("_delete_by_query requires a request body");
-        }
-        handleRequest(request, channel, client, false, false);
+    public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+        return doPrepareRequest(request, client, false, false);
     }
 
     @Override
     protected DeleteByQueryRequest buildRequest(RestRequest request) throws IOException {
-         /*
+        /*
          * Passing the search request through DeleteByQueryRequest first allows
          * it to set its own defaults which differ from SearchRequest's
          * defaults. Then the parseInternalRequest can override them.
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java
index a8fe4f0311382..0714a9867ddb3 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java
@@ -19,17 +19,12 @@
 
 package org.elasticsearch.index.reindex;
 
-import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ObjectParser;
@@ -41,15 +36,9 @@
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.index.reindex.remote.RemoteInfo;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.search.SearchRequestParsers;
-import org.elasticsearch.search.aggregations.AggregatorParsers;
-import org.elasticsearch.search.suggest.Suggesters;
 
 import java.io.IOException;
 import java.util.List;
@@ -67,11 +56,11 @@
  * Expose reindex over rest.
  */
 public class RestReindexAction extends AbstractBaseReindexRestHandler<ReindexRequest, ReindexAction> {
-    static final ObjectParser<ReindexRequest, ReindexParseContext> PARSER = new ObjectParser<>("reindex");
+    static final ObjectParser<ReindexRequest, Void> PARSER = new ObjectParser<>("reindex");
     private static final Pattern HOST_PATTERN = Pattern.compile("(?<scheme>[^:]+)://(?<host>[^:]+):(?<port>\\d+)");
 
     static {
-        ObjectParser.Parser<ReindexRequest, ReindexParseContext> sourceParser = (parser, request, context) -> {
+        ObjectParser.Parser<ReindexRequest, Void> sourceParser = (parser, request, context) -> {
             // Funky hack to work around Search not having a proper ObjectParser and us wanting to extract query if using remote.
             Map<String, Object> source = parser.map();
             String[] indices = extractStringArray(source, "index");
@@ -85,13 +74,12 @@ public class RestReindexAction extends AbstractBaseReindexRestHandler<ReindexReq
             request.setRemoteInfo(buildRemoteInfo(source));
             XContentBuilder builder = XContentFactory.contentBuilder(parser.contentType());
             builder.map(source);
-            try (XContentParser innerParser = parser.contentType().xContent().createParser(builder.bytes())) {
-                request.getSearchRequest().source().parseXContent(context.queryParseContext(innerParser),
-                    context.searchRequestParsers.aggParsers, context.searchRequestParsers.suggesters);
+            try (XContentParser innerParser = parser.contentType().xContent().createParser(parser.getXContentRegistry(), builder.bytes())) {
+                request.getSearchRequest().source().parseXContent(new QueryParseContext(innerParser));
             }
         };
 
-        ObjectParser<IndexRequest, ParseFieldMatcherSupplier> destParser = new ObjectParser<>("dest");
+        ObjectParser<IndexRequest, Void> destParser = new ObjectParser<>("dest");
         destParser.declareString(IndexRequest::index, new ParseField("index"));
         destParser.declareString(IndexRequest::type, new ParseField("type"));
         destParser.declareString(IndexRequest::routing, new ParseField("routing"));
@@ -104,34 +92,33 @@ public class RestReindexAction extends AbstractBaseReindexRestHandler<ReindexReq
         destParser.declareString((i, ttl) -> i.ttl(parseTimeValue(ttl, TimeValue.timeValueMillis(-1), "ttl").millis()),
                 new ParseField("ttl"));
 
-        PARSER.declareField((p, v, c) -> sourceParser.parse(p, v, c), new ParseField("source"), ValueType.OBJECT);
+        PARSER.declareField(sourceParser, new ParseField("source"), ValueType.OBJECT);
         PARSER.declareField((p, v, c) -> destParser.parse(p, v.getDestination(), c), new ParseField("dest"), ValueType.OBJECT);
         PARSER.declareInt(ReindexRequest::setSize, new ParseField("size"));
-        PARSER.declareField((p, v, c) -> v.setScript(Script.parse(p, c.getParseFieldMatcher())), new ParseField("script"),
+        PARSER.declareField((p, v, c) -> v.setScript(Script.parse(p)), new ParseField("script"),
                 ValueType.OBJECT);
         PARSER.declareString(ReindexRequest::setConflicts, new ParseField("conflicts"));
     }
 
-    @Inject
-    public RestReindexAction(Settings settings, RestController controller,
-            SearchRequestParsers searchRequestParsers, ClusterService clusterService) {
-        super(settings, searchRequestParsers, clusterService, ReindexAction.INSTANCE);
+    public RestReindexAction(Settings settings, RestController controller) {
+        super(settings, ReindexAction.INSTANCE);
         controller.registerHandler(POST, "/_reindex", this);
     }
 
     @Override
-    public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws IOException {
-        if (false == request.hasContent()) {
-            throw new ElasticsearchException("_reindex requires a request body");
-        }
-        handleRequest(request, channel, client, true, true);
+    public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+        return doPrepareRequest(request, client, true, true);
     }
 
     @Override
     protected ReindexRequest buildRequest(RestRequest request) throws IOException {
+        if (request.hasParam("pipeline")) {
+            throw new IllegalArgumentException("_reindex doesn't support [pipeline] as a query parmaeter. "
+                    + "Specify it in the [dest] object instead.");
+        }
         ReindexRequest internal = new ReindexRequest(new SearchRequest(), new IndexRequest());
-        try (XContentParser xcontent = XContentFactory.xContent(request.content()).createParser(request.content())) {
-            PARSER.parse(xcontent, internal, new ReindexParseContext(searchRequestParsers, parseFieldMatcher));
+        try (XContentParser parser = request.contentParser()) {
+            PARSER.parse(parser, internal, null);
         }
         return internal;
     }
@@ -153,11 +140,13 @@ static RemoteInfo buildRemoteInfo(Map<String, Object> source) throws IOException
         String host = hostMatcher.group("host");
         int port = Integer.parseInt(hostMatcher.group("port"));
         Map<String, String> headers = extractStringStringMap(remote, "headers");
+        TimeValue socketTimeout = extractTimeValue(remote, "socket_timeout", RemoteInfo.DEFAULT_SOCKET_TIMEOUT);
+        TimeValue connectTimeout = extractTimeValue(remote, "connect_timeout", RemoteInfo.DEFAULT_CONNECT_TIMEOUT);
         if (false == remote.isEmpty()) {
             throw new IllegalArgumentException(
                     "Unsupported fields in [remote]: [" + Strings.collectionToCommaDelimitedString(remote.keySet()) + "]");
         }
-        return new RemoteInfo(scheme, host, port, queryForRemote(source), username, password, headers);
+        return new RemoteInfo(scheme, host, port, queryForRemote(source), username, password, headers, socketTimeout, connectTimeout);
     }
 
     /**
@@ -210,6 +199,11 @@ private static Map<String, String> extractStringStringMap(Map<String, Object> so
         return safe;
     }
 
+    private static TimeValue extractTimeValue(Map<String, Object> source, String name, TimeValue defaultValue) {
+        String string = extractString(source, name);
+        return string == null ? defaultValue : parseTimeValue(string, name);
+    }
+
     private static BytesReference queryForRemote(Map<String, Object> source) throws IOException {
         XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint();
         Object query = source.remove("query");
@@ -223,23 +217,4 @@ private static BytesReference queryForRemote(Map<String, Object> source) throws
         Map<String, Object> map = (Map<String, Object>) query;
         return builder.map(map).bytes();
     }
-
-    static class ReindexParseContext implements ParseFieldMatcherSupplier {
-        private final SearchRequestParsers searchRequestParsers;
-        private final ParseFieldMatcher parseFieldMatcher;
-
-        ReindexParseContext(SearchRequestParsers searchRequestParsers, ParseFieldMatcher parseFieldMatcher) {
-            this.searchRequestParsers = searchRequestParsers;
-            this.parseFieldMatcher = parseFieldMatcher;
-        }
-
-        QueryParseContext queryParseContext(XContentParser parser) {
-            return new QueryParseContext(searchRequestParsers.queryParsers, parser, parseFieldMatcher);
-        }
-
-        @Override
-        public ParseFieldMatcher getParseFieldMatcher() {
-            return this.parseFieldMatcher;
-        }
-    }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestRethrottleAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestRethrottleAction.java
index 26c2aad8ce95a..4aa46f1a07780 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestRethrottleAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestRethrottleAction.java
@@ -20,32 +20,32 @@
 package org.elasticsearch.index.reindex;
 
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.tasks.TaskId;
 
+import java.io.IOException;
+import java.util.function.Supplier;
+
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.rest.action.admin.cluster.RestListTasksAction.listTasksResponseListener;
 
 public class RestRethrottleAction extends BaseRestHandler {
-    private final ClusterService clusterService;
+    private final Supplier<DiscoveryNodes> nodesInCluster;
 
-    @Inject
-    public RestRethrottleAction(Settings settings, RestController controller, ClusterService clusterService) {
+    public RestRethrottleAction(Settings settings, RestController controller, Supplier<DiscoveryNodes> nodesInCluster) {
         super(settings);
-        this.clusterService = clusterService;
+        this.nodesInCluster = nodesInCluster;
         controller.registerHandler(POST, "/_update_by_query/{taskId}/_rethrottle", this);
         controller.registerHandler(POST, "/_delete_by_query/{taskId}/_rethrottle", this);
         controller.registerHandler(POST, "/_reindex/{taskId}/_rethrottle", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         RethrottleRequest internalRequest = new RethrottleRequest();
         internalRequest.setTaskId(new TaskId(request.param("taskId")));
         Float requestsPerSecond = AbstractBaseReindexRestHandler.parseRequestsPerSecond(request);
@@ -53,6 +53,8 @@ public void handleRequest(final RestRequest request, final RestChannel channel,
             throw new IllegalArgumentException("requests_per_second is a required parameter");
         }
         internalRequest.setRequestsPerSecond(requestsPerSecond);
-        client.execute(RethrottleAction.INSTANCE, internalRequest, listTasksResponseListener(clusterService, channel));
+        final String groupBy = request.param("group_by", "nodes");
+        return channel ->
+            client.execute(RethrottleAction.INSTANCE, internalRequest, listTasksResponseListener(nodesInCluster, groupBy, channel));
     }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestUpdateByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestUpdateByQueryAction.java
index 06adb76df78d2..30161503c113c 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestUpdateByQueryAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestUpdateByQueryAction.java
@@ -22,42 +22,32 @@
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.search.SearchRequestParsers;
-import org.elasticsearch.search.aggregations.AggregatorParsers;
-import org.elasticsearch.search.suggest.Suggesters;
+import org.elasticsearch.script.ScriptType;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.function.Consumer;
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
-import static org.elasticsearch.script.Script.ScriptField;
+import static org.elasticsearch.script.Script.DEFAULT_SCRIPT_LANG;
 
 public class RestUpdateByQueryAction extends AbstractBulkByQueryRestHandler<UpdateByQueryRequest, UpdateByQueryAction> {
-
-    @Inject
-    public RestUpdateByQueryAction(Settings settings, RestController controller,
-            SearchRequestParsers searchRequestParsers, ClusterService clusterService) {
-        super(settings, searchRequestParsers, clusterService, UpdateByQueryAction.INSTANCE);
+    public RestUpdateByQueryAction(Settings settings, RestController controller) {
+        super(settings, UpdateByQueryAction.INSTANCE);
         controller.registerHandler(POST, "/{index}/_update_by_query", this);
         controller.registerHandler(POST, "/{index}/{type}/_update_by_query", this);
     }
 
     @Override
-    public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
-        handleRequest(request, channel, client, false, true);
+    public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+        return doPrepareRequest(request, client, false, true);
     }
 
     @Override
@@ -72,7 +62,7 @@ protected UpdateByQueryRequest buildRequest(RestRequest request) throws IOExcept
 
         Map<String, Consumer<Object>> consumers = new HashMap<>();
         consumers.put("conflicts", o -> internal.setConflicts((String) o));
-        consumers.put("script", o -> internal.setScript(parseScript((Map<String, Object>)o, parseFieldMatcher)));
+        consumers.put("script", o -> internal.setScript(parseScript((Map<String, Object>)o)));
 
         parseInternalRequest(internal, request, consumers);
 
@@ -81,45 +71,45 @@ protected UpdateByQueryRequest buildRequest(RestRequest request) throws IOExcept
     }
 
     @SuppressWarnings("unchecked")
-    static Script parseScript(Map<String, Object> config, ParseFieldMatcher parseFieldMatcher) {
+    private static Script parseScript(Map<String, Object> config) {
         String script = null;
-        ScriptService.ScriptType type = null;
-        String lang = null;
-        Map<String, Object> params = null;
+        ScriptType type = null;
+        String lang = DEFAULT_SCRIPT_LANG;
+        Map<String, Object> params = Collections.emptyMap();
         for (Iterator<Map.Entry<String, Object>> itr = config.entrySet().iterator(); itr.hasNext();) {
             Map.Entry<String, Object> entry = itr.next();
             String parameterName = entry.getKey();
             Object parameterValue = entry.getValue();
-            if (parseFieldMatcher.match(parameterName, ScriptField.LANG)) {
+            if (Script.LANG_PARSE_FIELD.match(parameterName)) {
                 if (parameterValue instanceof String || parameterValue == null) {
                     lang = (String) parameterValue;
                 } else {
                     throw new ElasticsearchParseException("Value must be of type String: [" + parameterName + "]");
                 }
-            } else if (parseFieldMatcher.match(parameterName, ScriptField.PARAMS)) {
+            } else if (Script.PARAMS_PARSE_FIELD.match(parameterName)) {
                 if (parameterValue instanceof Map || parameterValue == null) {
                     params = (Map<String, Object>) parameterValue;
                 } else {
                     throw new ElasticsearchParseException("Value must be of type String: [" + parameterName + "]");
                 }
-            } else if (parseFieldMatcher.match(parameterName, ScriptService.ScriptType.INLINE.getParseField())) {
+            } else if (ScriptType.INLINE.getParseField().match(parameterName)) {
                 if (parameterValue instanceof String || parameterValue == null) {
                     script = (String) parameterValue;
-                    type = ScriptService.ScriptType.INLINE;
+                    type = ScriptType.INLINE;
                 } else {
                     throw new ElasticsearchParseException("Value must be of type String: [" + parameterName + "]");
                 }
-            } else if (parseFieldMatcher.match(parameterName, ScriptService.ScriptType.FILE.getParseField())) {
+            } else if (ScriptType.FILE.getParseField().match(parameterName)) {
                 if (parameterValue instanceof String || parameterValue == null) {
                     script = (String) parameterValue;
-                    type = ScriptService.ScriptType.FILE;
+                    type = ScriptType.FILE;
                 } else {
                     throw new ElasticsearchParseException("Value must be of type String: [" + parameterName + "]");
                 }
-            } else if (parseFieldMatcher.match(parameterName, ScriptService.ScriptType.STORED.getParseField())) {
+            } else if (ScriptType.STORED.getParseField().match(parameterName)) {
                 if (parameterValue instanceof String || parameterValue == null) {
                     script = (String) parameterValue;
-                    type = ScriptService.ScriptType.STORED;
+                    type = ScriptType.STORED;
                 } else {
                     throw new ElasticsearchParseException("Value must be of type String: [" + parameterName + "]");
                 }
@@ -127,10 +117,11 @@ static Script parseScript(Map<String, Object> config, ParseFieldMatcher parseFie
         }
         if (script == null) {
             throw new ElasticsearchParseException("expected one of [{}], [{}] or [{}] fields, but found none",
-                    ScriptService.ScriptType.INLINE.getParseField().getPreferredName(), ScriptService.ScriptType.FILE.getParseField()
-                    .getPreferredName(), ScriptService.ScriptType.STORED.getParseField().getPreferredName());
+                    ScriptType.INLINE.getParseField().getPreferredName(), ScriptType.FILE.getParseField()
+                    .getPreferredName(), ScriptType.STORED.getParseField().getPreferredName());
         }
         assert type != null : "if script is not null, type should definitely not be null";
-        return new Script(script, type, lang, params);
+
+        return new Script(type, lang, script, params);
     }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java
index 99362e75f9b21..99e1a9f166dd8 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java
@@ -19,9 +19,7 @@
 
 package org.elasticsearch.index.reindex;
 
-import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.Client;
@@ -36,7 +34,7 @@
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
-public class TransportDeleteByQueryAction extends HandledTransportAction<DeleteByQueryRequest, BulkIndexByScrollResponse> {
+public class TransportDeleteByQueryAction extends HandledTransportAction<DeleteByQueryRequest, BulkByScrollResponse> {
     private final Client client;
     private final ScriptService scriptService;
     private final ClusterService clusterService;
@@ -52,64 +50,20 @@ public TransportDeleteByQueryAction(Settings settings, ThreadPool threadPool, Ac
     }
 
     @Override
-    protected void doExecute(Task task, DeleteByQueryRequest request, ActionListener<BulkIndexByScrollResponse> listener) {
-        ClusterState state = clusterService.state();
-        ParentTaskAssigningClient client = new ParentTaskAssigningClient(this.client, clusterService.localNode(), task);
-        new AsyncDeleteBySearchAction((BulkByScrollTask) task, logger, client, threadPool, request, listener, scriptService, state).start();
+    public void doExecute(Task task, DeleteByQueryRequest request, ActionListener<BulkByScrollResponse> listener) {
+        if (request.getSlices() > 1) {
+            BulkByScrollParallelizationHelper.startSlices(client, taskManager, DeleteByQueryAction.INSTANCE,
+                    clusterService.localNode().getId(), (ParentBulkByScrollTask) task, request, listener);
+        } else {
+            ClusterState state = clusterService.state();
+            ParentTaskAssigningClient client = new ParentTaskAssigningClient(this.client, clusterService.localNode(), task);
+            new AsyncDeleteByQueryAction((WorkingBulkByScrollTask) task, logger, client, threadPool, request, scriptService, state,
+                    listener).start();
+        }
     }
 
     @Override
-    protected void doExecute(DeleteByQueryRequest request, ActionListener<BulkIndexByScrollResponse> listener) {
+    protected void doExecute(DeleteByQueryRequest request, ActionListener<BulkByScrollResponse> listener) {
         throw new UnsupportedOperationException("task required");
     }
-
-    /**
-     * Implementation of delete-by-query using scrolling and bulk.
-     */
-    static class AsyncDeleteBySearchAction extends AbstractAsyncBulkIndexByScrollAction<DeleteByQueryRequest> {
-
-        public AsyncDeleteBySearchAction(BulkByScrollTask task, Logger logger, ParentTaskAssigningClient client, ThreadPool threadPool,
-                                         DeleteByQueryRequest request, ActionListener<BulkIndexByScrollResponse> listener,
-                                         ScriptService scriptService, ClusterState clusterState) {
-            super(task, logger, client, threadPool, request, listener, scriptService, clusterState);
-        }
-
-        @Override
-        protected boolean needsSourceDocumentVersions() {
-            /*
-             * We always need the version of the source document so we can report a version conflict if we try to delete it and it has been
-             * changed.
-             */
-            return true;
-        }
-
-        @Override
-        protected boolean accept(ScrollableHitSource.Hit doc) {
-            // Delete-by-query does not require the source to delete a document
-            // and the default implementation checks for it
-            return true;
-        }
-
-        @Override
-        protected RequestWrapper<DeleteRequest> buildRequest(ScrollableHitSource.Hit doc) {
-            DeleteRequest delete = new DeleteRequest();
-            delete.index(doc.getIndex());
-            delete.type(doc.getType());
-            delete.id(doc.getId());
-            delete.version(doc.getVersion());
-            return wrap(delete);
-        }
-
-        /**
-         * Overrides the parent {@link AbstractAsyncBulkIndexByScrollAction#copyMetadata(RequestWrapper, ScrollableHitSource.Hit)}
-         * method that is much more Update/Reindex oriented and so also copies things like timestamp/ttl which we
-         * don't care for a deletion.
-         */
-        @Override
-        protected RequestWrapper<?> copyMetadata(RequestWrapper<?> request, ScrollableHitSource.Hit doc) {
-            request.setParent(doc.getParent());
-            request.setRouting(doc.getRouting());
-            return request;
-        }
-    }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java
index 33aca0283515b..13caabda8b6ab 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java
@@ -28,10 +28,16 @@
 import org.apache.http.impl.nio.reactor.IOReactorConfig;
 import org.apache.http.message.BasicHeader;
 import org.apache.logging.log4j.Logger;
+import org.apache.lucene.util.automaton.Automata;
+import org.apache.lucene.util.automaton.Automaton;
+import org.apache.lucene.util.automaton.CharacterRunAutomaton;
+import org.apache.lucene.util.automaton.MinimizationOperations;
+import org.apache.lucene.util.automaton.Operations;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.bulk.BackoffPolicy;
 import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
+import org.elasticsearch.index.reindex.ScrollableHitSource.SearchFailure;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.support.ActionFilters;
@@ -43,20 +49,20 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.lucene.uid.Versions;
+import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.TransportAddress;
-import org.elasticsearch.http.HttpInfo;
-import org.elasticsearch.http.HttpServer;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.mapper.TTLFieldMapper;
 import org.elasticsearch.index.mapper.VersionFieldMapper;
-import org.elasticsearch.index.reindex.ScrollableHitSource.SearchFailure;
-import org.elasticsearch.index.reindex.remote.RemoteInfo;
 import org.elasticsearch.index.reindex.remote.RemoteScrollableHitSource;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService;
@@ -64,12 +70,12 @@
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.util.ArrayList;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.BiFunction;
 import java.util.function.Function;
@@ -79,7 +85,7 @@
 import static java.util.Objects.requireNonNull;
 import static org.elasticsearch.index.VersionType.INTERNAL;
 
-public class TransportReindexAction extends HandledTransportAction<ReindexRequest, BulkIndexByScrollResponse> {
+public class TransportReindexAction extends HandledTransportAction<ReindexRequest, BulkByScrollResponse> {
     public static final Setting<List<String>> REMOTE_CLUSTER_WHITELIST =
             Setting.listSetting("reindex.remote.whitelist", emptyList(), Function.identity(), Property.NodeScope);
 
@@ -87,63 +93,71 @@ public class TransportReindexAction extends HandledTransportAction<ReindexReques
     private final ScriptService scriptService;
     private final AutoCreateIndex autoCreateIndex;
     private final Client client;
-    private final Set<String> remoteWhitelist;
-    private final HttpServer httpServer;
+    private final CharacterRunAutomaton remoteWhitelist;
 
     @Inject
     public TransportReindexAction(Settings settings, ThreadPool threadPool, ActionFilters actionFilters,
             IndexNameExpressionResolver indexNameExpressionResolver, ClusterService clusterService, ScriptService scriptService,
-            AutoCreateIndex autoCreateIndex, Client client, TransportService transportService, @Nullable HttpServer httpServer) {
+            AutoCreateIndex autoCreateIndex, Client client, TransportService transportService) {
         super(settings, ReindexAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver,
                 ReindexRequest::new);
         this.clusterService = clusterService;
         this.scriptService = scriptService;
         this.autoCreateIndex = autoCreateIndex;
         this.client = client;
-        remoteWhitelist = new HashSet<>(REMOTE_CLUSTER_WHITELIST.get(settings));
-        this.httpServer = httpServer;
+        remoteWhitelist = buildRemoteWhitelist(REMOTE_CLUSTER_WHITELIST.get(settings));
     }
 
     @Override
-    protected void doExecute(Task task, ReindexRequest request, ActionListener<BulkIndexByScrollResponse> listener) {
-        checkRemoteWhitelist(request.getRemoteInfo());
-        ClusterState state = clusterService.state();
-        validateAgainstAliases(request.getSearchRequest(), request.getDestination(), request.getRemoteInfo(), indexNameExpressionResolver,
-                autoCreateIndex, state);
-        ParentTaskAssigningClient client = new ParentTaskAssigningClient(this.client, clusterService.localNode(), task);
-        new AsyncIndexBySearchAction((BulkByScrollTask) task, logger, client, threadPool, request, listener, scriptService, state).start();
+    protected void doExecute(Task task, ReindexRequest request, ActionListener<BulkByScrollResponse> listener) {
+        if (request.getSlices() > 1) {
+            BulkByScrollParallelizationHelper.startSlices(client, taskManager, ReindexAction.INSTANCE, clusterService.localNode().getId(),
+                    (ParentBulkByScrollTask) task, request, listener);
+        } else {
+            checkRemoteWhitelist(remoteWhitelist, request.getRemoteInfo());
+            ClusterState state = clusterService.state();
+            validateAgainstAliases(request.getSearchRequest(), request.getDestination(), request.getRemoteInfo(),
+                    indexNameExpressionResolver, autoCreateIndex, state);
+            ParentTaskAssigningClient client = new ParentTaskAssigningClient(this.client, clusterService.localNode(), task);
+            new AsyncIndexBySearchAction((WorkingBulkByScrollTask) task, logger, client, threadPool, request, scriptService, state,
+                    listener).start();
+        }
     }
 
     @Override
-    protected void doExecute(ReindexRequest request, ActionListener<BulkIndexByScrollResponse> listener) {
+    protected void doExecute(ReindexRequest request, ActionListener<BulkByScrollResponse> listener) {
         throw new UnsupportedOperationException("task required");
     }
 
-    private void checkRemoteWhitelist(RemoteInfo remoteInfo) {
-        TransportAddress publishAddress = null;
-        HttpInfo httpInfo = httpServer == null ? null : httpServer.info();
-        if (httpInfo != null && httpInfo.getAddress() != null) {
-            publishAddress = httpInfo.getAddress().publishAddress();
+    static void checkRemoteWhitelist(CharacterRunAutomaton whitelist, RemoteInfo remoteInfo) {
+        if (remoteInfo == null) {
+            return;
         }
-        checkRemoteWhitelist(remoteWhitelist, remoteInfo, publishAddress);
-    }
-
-    static void checkRemoteWhitelist(Set<String> whitelist, RemoteInfo remoteInfo, TransportAddress publishAddress) {
-        if (remoteInfo == null) return;
         String check = remoteInfo.getHost() + ':' + remoteInfo.getPort();
-        if (whitelist.contains(check)) return;
-        /*
-         * For testing we support the key "myself" to allow connecting to the local node. We can't just change the setting to include the
-         * local node because it is intentionally not a dynamic setting for security purposes. We can't use something like "localhost:9200"
-         * because we don't know up front which port we'll get because the tests bind to port 0. Instead we try to resolve it here, taking
-         * "myself" to mean "my published http address".
-         */
-        if (whitelist.contains("myself") && publishAddress != null && publishAddress.toString().equals(check)) {
+        if (whitelist.run(check)) {
             return;
         }
         throw new IllegalArgumentException('[' + check + "] not whitelisted in " + REMOTE_CLUSTER_WHITELIST.getKey());
     }
 
+    /**
+     * Build the {@link CharacterRunAutomaton} that represents the reindex-from-remote whitelist and make sure that it doesn't whitelist
+     * the world.
+     */
+    static CharacterRunAutomaton buildRemoteWhitelist(List<String> whitelist) {
+        if (whitelist.isEmpty()) {
+            return new CharacterRunAutomaton(Automata.makeEmpty());
+        }
+        Automaton automaton = Regex.simpleMatchToAutomaton(whitelist.toArray(Strings.EMPTY_ARRAY));
+        automaton = MinimizationOperations.minimize(automaton, Operations.DEFAULT_MAX_DETERMINIZED_STATES);
+        if (Operations.isTotal(automaton)) {
+            throw new IllegalArgumentException("Refusing to start because whitelist " + whitelist + " accepts all addresses. "
+                    + "This would allow users to reindex-from-remote any URL they like effectively having Elasticsearch make HTTP GETs "
+                    + "for them.");
+        }
+        return new CharacterRunAutomaton(automaton);
+    }
+
     /**
      * Throws an ActionRequestValidationException if the request tries to index
      * back into the same index or into an index that points to two indexes.
@@ -188,6 +202,11 @@ static RestClient buildRestClient(RemoteInfo remoteInfo, long taskId, List<Threa
         }
         return RestClient.builder(new HttpHost(remoteInfo.getHost(), remoteInfo.getPort(), remoteInfo.getScheme()))
                 .setDefaultHeaders(clientHeaders)
+                .setRequestConfigCallback(c -> {
+                    c.setConnectTimeout(Math.toIntExact(remoteInfo.getConnectTimeout().millis()));
+                    c.setSocketTimeout(Math.toIntExact(remoteInfo.getSocketTimeout().millis()));
+                    return c;
+                })
                 .setHttpClientConfigCallback(c -> {
                     // Enable basic auth if it is configured
                     if (remoteInfo.getUsername() != null) {
@@ -217,7 +236,7 @@ static RestClient buildRestClient(RemoteInfo remoteInfo, long taskId, List<Threa
      * but this makes no attempt to do any of them so it can be as simple
      * possible.
      */
-    static class AsyncIndexBySearchAction extends AbstractAsyncBulkIndexByScrollAction<ReindexRequest> {
+    static class AsyncIndexBySearchAction extends AbstractAsyncBulkByScrollAction<ReindexRequest> {
         /**
          * List of threads created by this process. Usually actions don't create threads in Elasticsearch. Instead they use the builtin
          * {@link ThreadPool}s. But reindex-from-remote uses Elasticsearch's {@link RestClient} which doesn't use the
@@ -226,10 +245,16 @@ static class AsyncIndexBySearchAction extends AbstractAsyncBulkIndexByScrollActi
          */
         private List<Thread> createdThreads = emptyList();
 
-        public AsyncIndexBySearchAction(BulkByScrollTask task, Logger logger, ParentTaskAssigningClient client, ThreadPool threadPool,
-                                        ReindexRequest request, ActionListener<BulkIndexByScrollResponse> listener,
-                                        ScriptService scriptService, ClusterState clusterState) {
-            super(task, logger, client, threadPool, request, listener, scriptService, clusterState);
+        AsyncIndexBySearchAction(WorkingBulkByScrollTask task, Logger logger, ParentTaskAssigningClient client,
+                                 ThreadPool threadPool, ReindexRequest request, ScriptService scriptService, ClusterState clusterState,
+                                 ActionListener<BulkByScrollResponse> listener) {
+            this(task, logger, client, threadPool, request, scriptService, clusterState, listener, client.settings());
+        }
+
+        AsyncIndexBySearchAction(WorkingBulkByScrollTask task, Logger logger, ParentTaskAssigningClient client,
+                ThreadPool threadPool, ReindexRequest request, ScriptService scriptService, ClusterState clusterState,
+                ActionListener<BulkByScrollResponse> listener, Settings settings) {
+            super(task, logger, client, threadPool, request, scriptService, clusterState, listener, settings);
         }
 
         @Override
@@ -254,7 +279,7 @@ protected ScrollableHitSource buildScrollableResultSource(BackoffPolicy backoffP
         }
 
         @Override
-        void finishHim(Exception failure, List<Failure> indexingFailures, List<SearchFailure> searchFailures, boolean timedOut) {
+        protected void finishHim(Exception failure, List<Failure> indexingFailures, List<SearchFailure> searchFailures, boolean timedOut) {
             super.finishHim(failure, indexingFailures, searchFailures, timedOut);
             // A little extra paranoia so we log something if we leave any threads running
             for (Thread thread : createdThreads) {
@@ -266,7 +291,7 @@ void finishHim(Exception failure, List<Failure> indexingFailures, List<SearchFai
         }
 
         @Override
-        protected BiFunction<RequestWrapper<?>, ScrollableHitSource.Hit, RequestWrapper<?>> buildScriptApplier() {
+        public BiFunction<RequestWrapper<?>, ScrollableHitSource.Hit, RequestWrapper<?>> buildScriptApplier() {
             Script script = mainRequest.getScript();
             if (script != null) {
                 return new ReindexScriptApplier(task, scriptService, script, script.getParams());
@@ -302,7 +327,24 @@ protected RequestWrapper<IndexRequest> buildRequest(ScrollableHitSource.Hit doc)
 
             // id and source always come from the found doc. Scripts can change them but they operate on the index request.
             index.id(doc.getId());
-            index.source(doc.getSource());
+
+            // the source xcontent type and destination could be different
+            final XContentType sourceXContentType = doc.getXContentType();
+            final XContentType mainRequestXContentType = mainRequest.getDestination().getContentType();
+            if (mainRequestXContentType != null && doc.getXContentType() != mainRequestXContentType) {
+                // we need to convert
+                try (XContentParser parser = sourceXContentType.xContent().createParser(NamedXContentRegistry.EMPTY, doc.getSource());
+                     XContentBuilder builder = XContentBuilder.builder(mainRequestXContentType.xContent())) {
+                    parser.nextToken();
+                    builder.copyCurrentStructure(parser);
+                    index.source(builder.bytes(), builder.contentType());
+                } catch (IOException e) {
+                    throw new UncheckedIOException("failed to convert hit from " + sourceXContentType + " to "
+                        + mainRequestXContentType, e);
+                }
+            } else {
+                index.source(doc.getSource(), doc.getXContentType());
+            }
 
             /*
              * The rest of the index request just has to be copied from the template. It may be changed later from scripts or the superclass
@@ -312,7 +354,6 @@ protected RequestWrapper<IndexRequest> buildRequest(ScrollableHitSource.Hit doc)
             index.parent(mainRequest.getDestination().parent());
             index.timestamp(mainRequest.getDestination().timestamp());
             index.ttl(mainRequest.getDestination().ttl());
-            index.contentType(mainRequest.getDestination().getContentType());
             index.setPipeline(mainRequest.getDestination().getPipeline());
             // OpType is synthesized from version so it is handled when we copy version above.
 
@@ -347,7 +388,7 @@ protected void copyRouting(RequestWrapper<?> request, String routing) {
 
         class ReindexScriptApplier extends ScriptApplier {
 
-            ReindexScriptApplier(BulkByScrollTask task, ScriptService scriptService, Script script,
+            ReindexScriptApplier(WorkingBulkByScrollTask task, ScriptService scriptService, Script script,
                                  Map<String, Object> params) {
                 super(task, scriptService, script, params);
             }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java
index 89af3bd39bd29..bcfb281347496 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java
@@ -19,16 +19,20 @@
 
 package org.elasticsearch.index.reindex;
 
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.TaskOperationFailure;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.tasks.TransportTasksAction;
+import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.tasks.TaskInfo;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -37,18 +41,39 @@
 import java.util.List;
 
 public class TransportRethrottleAction extends TransportTasksAction<BulkByScrollTask, RethrottleRequest, ListTasksResponse, TaskInfo> {
+    private final Client client;
+
     @Inject
     public TransportRethrottleAction(Settings settings, ThreadPool threadPool, ClusterService clusterService,
-            TransportService transportService, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
+            TransportService transportService, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
+            Client client) {
         super(settings, RethrottleAction.NAME, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver,
                 RethrottleRequest::new, ListTasksResponse::new, ThreadPool.Names.MANAGEMENT);
+        this.client = client;
     }
 
     @Override
-    protected TaskInfo taskOperation(RethrottleRequest request, BulkByScrollTask task) {
-        // Apply the new throttle and fetch status of the task. The user might not want that status but they likely do and it is cheap.
-        task.rethrottle(request.getRequestsPerSecond());
-        return task.taskInfo(clusterService.localNode(), true);
+    protected void taskOperation(RethrottleRequest request, BulkByScrollTask task, ActionListener<TaskInfo> listener) {
+        rethrottle(logger, clusterService.localNode().getId(), client, task, request.getRequestsPerSecond(), listener);
+    }
+
+    static void rethrottle(Logger logger, String localNodeId, Client client, BulkByScrollTask task, float newRequestsPerSecond,
+            ActionListener<TaskInfo> listener) {
+        int runningSubTasks = task.runningSliceSubTasks();
+        if (runningSubTasks == 0) {
+            logger.debug("rethrottling local task [{}] to [{}] requests per second", task.getId(), newRequestsPerSecond);
+            task.rethrottle(newRequestsPerSecond);
+            listener.onResponse(task.taskInfo(localNodeId, true));
+            return;
+        }
+        RethrottleRequest subRequest = new RethrottleRequest();
+        subRequest.setRequestsPerSecond(newRequestsPerSecond / runningSubTasks);
+        subRequest.setParentTaskId(new TaskId(localNodeId, task.getId()));
+        logger.debug("rethrottling children of task [{}] to [{}] requests per second", task.getId(), subRequest.getRequestsPerSecond());
+        client.execute(RethrottleAction.INSTANCE, subRequest, ActionListener.wrap(r -> {
+            r.rethrowFailures("Rethrottle");
+            listener.onResponse(task.getInfoGivenSliceInfo(localNodeId, r.getTasks()));
+        }, listener::onFailure));
     }
 
     @Override
@@ -62,8 +87,4 @@ protected ListTasksResponse newResponse(RethrottleRequest request, List<TaskInfo
         return new ListTasksResponse(tasks, taskOperationFailures, failedNodeExceptions);
     }
 
-    @Override
-    protected boolean accumulateExceptions() {
-        return true;
-    }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java
index 0f4bf5695d165..a8f8528d35626 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java
@@ -48,7 +48,7 @@
 import java.util.Map;
 import java.util.function.BiFunction;
 
-public class TransportUpdateByQueryAction extends HandledTransportAction<UpdateByQueryRequest, BulkIndexByScrollResponse> {
+public class TransportUpdateByQueryAction extends HandledTransportAction<UpdateByQueryRequest, BulkByScrollResponse> {
     private final Client client;
     private final ScriptService scriptService;
     private final ClusterService clusterService;
@@ -65,26 +65,37 @@ public TransportUpdateByQueryAction(Settings settings, ThreadPool threadPool, Ac
     }
 
     @Override
-    protected void doExecute(Task task, UpdateByQueryRequest request, ActionListener<BulkIndexByScrollResponse> listener) {
-        ClusterState state = clusterService.state();
-        ParentTaskAssigningClient client = new ParentTaskAssigningClient(this.client, clusterService.localNode(), task);
-        new AsyncIndexBySearchAction((BulkByScrollTask) task, logger, client, threadPool, request, listener, scriptService, state).start();
+    protected void doExecute(Task task, UpdateByQueryRequest request, ActionListener<BulkByScrollResponse> listener) {
+        if (request.getSlices() > 1) {
+            BulkByScrollParallelizationHelper.startSlices(client, taskManager, UpdateByQueryAction.INSTANCE,
+                    clusterService.localNode().getId(), (ParentBulkByScrollTask) task, request, listener);
+        } else {
+            ClusterState state = clusterService.state();
+            ParentTaskAssigningClient client = new ParentTaskAssigningClient(this.client, clusterService.localNode(), task);
+            new AsyncIndexBySearchAction((WorkingBulkByScrollTask) task, logger, client, threadPool, request, scriptService, state,
+                    listener).start();
+        }
     }
 
     @Override
-    protected void doExecute(UpdateByQueryRequest request, ActionListener<BulkIndexByScrollResponse> listener) {
+    protected void doExecute(UpdateByQueryRequest request, ActionListener<BulkByScrollResponse> listener) {
         throw new UnsupportedOperationException("task required");
     }
 
     /**
      * Simple implementation of update-by-query using scrolling and bulk.
      */
-    static class AsyncIndexBySearchAction extends AbstractAsyncBulkIndexByScrollAction<UpdateByQueryRequest> {
+    static class AsyncIndexBySearchAction extends AbstractAsyncBulkByScrollAction<UpdateByQueryRequest> {
+        AsyncIndexBySearchAction(WorkingBulkByScrollTask task, Logger logger, ParentTaskAssigningClient client,
+                                 ThreadPool threadPool, UpdateByQueryRequest request, ScriptService scriptService,
+                                 ClusterState clusterState, ActionListener<BulkByScrollResponse> listener) {
+            this(task, logger, client, threadPool, request, scriptService, clusterState, listener, client.settings());
+        }
 
-        public AsyncIndexBySearchAction(BulkByScrollTask task, Logger logger, ParentTaskAssigningClient client, ThreadPool threadPool,
-                                        UpdateByQueryRequest request, ActionListener<BulkIndexByScrollResponse> listener,
-                                        ScriptService scriptService, ClusterState clusterState) {
-            super(task, logger, client, threadPool, request, listener, scriptService, clusterState);
+        AsyncIndexBySearchAction(WorkingBulkByScrollTask task, Logger logger, ParentTaskAssigningClient client,
+                ThreadPool threadPool, UpdateByQueryRequest request, ScriptService scriptService, ClusterState clusterState,
+                ActionListener<BulkByScrollResponse> listener, Settings settings) {
+            super(task, logger, client, threadPool, request, scriptService, clusterState, listener, settings);
         }
 
         @Override
@@ -97,7 +108,7 @@ protected boolean needsSourceDocumentVersions() {
         }
 
         @Override
-        protected BiFunction<RequestWrapper<?>, ScrollableHitSource.Hit, RequestWrapper<?>> buildScriptApplier() {
+        public BiFunction<RequestWrapper<?>, ScrollableHitSource.Hit, RequestWrapper<?>> buildScriptApplier() {
             Script script = mainRequest.getScript();
             if (script != null) {
                 return new UpdateByQueryScriptApplier(task, scriptService, script, script.getParams());
@@ -120,7 +131,7 @@ protected RequestWrapper<IndexRequest> buildRequest(ScrollableHitSource.Hit doc)
 
         class UpdateByQueryScriptApplier extends ScriptApplier {
 
-            UpdateByQueryScriptApplier(BulkByScrollTask task, ScriptService scriptService, Script script,
+            UpdateByQueryScriptApplier(WorkingBulkByScrollTask task, ScriptService scriptService, Script script,
                                  Map<String, Object> params) {
                 super(task, scriptService, script, params);
             }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteInfo.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteInfo.java
deleted file mode 100644
index 1405d656d9923..0000000000000
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteInfo.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.reindex.remote;
-
-import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Writeable;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-import static java.util.Collections.unmodifiableMap;
-import static java.util.Objects.requireNonNull;
-
-public class RemoteInfo implements Writeable {
-    private final String scheme;
-    private final String host;
-    private final int port;
-    private final BytesReference query;
-    private final String username;
-    private final String password;
-    private final Map<String, String> headers;
-
-    public RemoteInfo(String scheme, String host, int port, BytesReference query, String username, String password,
-            Map<String, String> headers) {
-        this.scheme = requireNonNull(scheme, "[scheme] must be specified to reindex from a remote cluster");
-        this.host = requireNonNull(host, "[host] must be specified to reindex from a remote cluster");
-        this.port = port;
-        this.query = requireNonNull(query, "[query] must be specified to reindex from a remote cluster");
-        this.username = username;
-        this.password = password;
-        this.headers = unmodifiableMap(requireNonNull(headers, "[headers] is required"));
-    }
-
-    /**
-     * Read from a stream.
-     */
-    public RemoteInfo(StreamInput in) throws IOException {
-        scheme = in.readString();
-        host = in.readString();
-        port = in.readVInt();
-        query = in.readBytesReference();
-        username = in.readOptionalString();
-        password = in.readOptionalString();
-        int headersLength = in.readVInt();
-        Map<String, String> headers = new HashMap<>(headersLength);
-        for (int i = 0; i < headersLength; i++) {
-            headers.put(in.readString(), in.readString());
-        }
-        this.headers = unmodifiableMap(headers);
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeString(scheme);
-        out.writeString(host);
-        out.writeVInt(port);
-        out.writeBytesReference(query);
-        out.writeOptionalString(username);
-        out.writeOptionalString(password);
-        out.writeVInt(headers.size());
-        for (Map.Entry<String, String> header : headers.entrySet()) {
-            out.writeString(header.getKey());
-            out.writeString(header.getValue());
-        }
-    }
-
-    public String getScheme() {
-        return scheme;
-    }
-
-    public String getHost() {
-        return host;
-    }
-
-    public int getPort() {
-        return port;
-    }
-
-    public BytesReference getQuery() {
-        return query;
-    }
-
-    @Nullable
-    public String getUsername() {
-        return username;
-    }
-
-    @Nullable
-    public String getPassword() {
-        return password;
-    }
-
-    public Map<String, String> getHeaders() {
-        return headers;
-    }
-
-    @Override
-    public String toString() {
-        StringBuilder b = new StringBuilder();
-        if (false == "http".equals(scheme)) {
-            // http is the default so it isn't worth taking up space if it is the scheme
-            b.append("scheme=").append(scheme).append(' ');
-        }
-        b.append("host=").append(host).append(" port=").append(port).append(" query=").append(query.utf8ToString());
-        if (username != null) {
-            b.append(" username=").append(username);
-        }
-        if (password != null) {
-            b.append(" password=<<>>");
-        }
-        return b.toString();
-    }
-}
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuilders.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuilders.java
index b2700618f020c..892ac7d1eace6 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuilders.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuilders.java
@@ -30,6 +30,7 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -42,6 +43,7 @@
 import java.util.Map;
 
 import static java.util.Collections.singletonMap;
+import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
 
 final class RemoteRequestBuilders {
     private RemoteRequestBuilders() {}
@@ -58,7 +60,14 @@ static String initialSearchPath(SearchRequest searchRequest) {
     static Map<String, String> initialSearchParams(SearchRequest searchRequest, Version remoteVersion) {
         Map<String, String> params = new HashMap<>();
         if (searchRequest.scroll() != null) {
-            params.put("scroll", searchRequest.scroll().keepAlive().toString());
+            TimeValue keepAlive = searchRequest.scroll().keepAlive();
+            if (remoteVersion.before(Version.V_5_0_0)) {
+                /* Versions of Elasticsearch before 5.0 couldn't parse nanos or micros
+                 * so we toss out that resolution, rounding up because more scroll
+                 * timeout seems safer than less. */
+                keepAlive = timeValueMillis((long) Math.ceil(keepAlive.millisFrac()));
+            }
+            params.put("scroll", keepAlive.getStringRep());
         }
         params.put("size", Integer.toString(searchRequest.source().size()));
         if (searchRequest.source().version() == null || searchRequest.source().version() == true) {
@@ -86,7 +95,17 @@ static Map<String, String> initialSearchParams(SearchRequest searchRequest, Vers
                 for (int i = 1; i < searchRequest.source().sorts().size(); i++) {
                     sorts.append(',').append(sortToUri(searchRequest.source().sorts().get(i)));
                 }
-                params.put("sorts", sorts.toString());
+                params.put("sort", sorts.toString());
+            }
+        }
+        if (remoteVersion.before(Version.V_2_0_0)) {
+            // Versions before 2.0.0 need prompting to return interesting fields. Note that timestamp isn't available at all....
+            searchRequest.source().storedField("_parent").storedField("_routing").storedField("_ttl");
+            if (remoteVersion.before(Version.fromId(1000099))) {
+                // Versions before 1.0.0 don't support `"_source": true` so we have to ask for the _source in a funny way.
+                if (false == searchRequest.source().storedFields().fieldNames().contains("_source")) {
+                    searchRequest.source().storedField("_source");
+                }
             }
         }
         if (searchRequest.source().storedFields() != null && false == searchRequest.source().storedFields().fieldNames().isEmpty()) {
@@ -100,20 +119,32 @@ static Map<String, String> initialSearchParams(SearchRequest searchRequest, Vers
         return params;
     }
 
-    static HttpEntity initialSearchEntity(BytesReference query) {
-        try (XContentBuilder entity = JsonXContent.contentBuilder(); XContentParser queryParser = XContentHelper.createParser(query)) {
+    static HttpEntity initialSearchEntity(SearchRequest searchRequest, BytesReference query, Version remoteVersion) {
+        // EMPTY is safe here because we're not calling namedObject
+        try (XContentBuilder entity = JsonXContent.contentBuilder();
+                XContentParser queryParser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, query)) {
             entity.startObject();
-            entity.field("query");
-            /*
-             * We're intentionally a bit paranoid here - copying the query as xcontent rather than writing a raw field. We don't want poorly
-             * written queries to escape. Ever.
-             */
-            entity.copyCurrentStructure(queryParser);
-            XContentParser.Token shouldBeEof = queryParser.nextToken();
-            if (shouldBeEof != null) {
-                throw new ElasticsearchException(
-                        "query was more than a single object. This first token after the object is [" + shouldBeEof + "]");
+
+            entity.field("query"); {
+                /* We're intentionally a bit paranoid here - copying the query as xcontent rather than writing a raw field. We don't want
+                 * poorly written queries to escape. Ever. */
+                entity.copyCurrentStructure(queryParser);
+                XContentParser.Token shouldBeEof = queryParser.nextToken();
+                if (shouldBeEof != null) {
+                    throw new ElasticsearchException(
+                            "query was more than a single object. This first token after the object is [" + shouldBeEof + "]");
+                }
+            }
+
+            if (searchRequest.source().fetchSource() != null) {
+                entity.field("_source", searchRequest.source().fetchSource());
+            } else {
+                if (remoteVersion.onOrAfter(Version.fromId(1000099))) {
+                    // Versions before 1.0 don't support `"_source": true` so we have to ask for the source as a stored field.
+                    entity.field("_source", true);
+                }
             }
+
             entity.endObject();
             BytesRef bytes = entity.bytes().toBytesRef();
             return new ByteArrayEntity(bytes.bytes, bytes.offset, bytes.length, ContentType.APPLICATION_JSON);
@@ -153,11 +184,41 @@ static String scrollPath() {
         return "/_search/scroll";
     }
 
-    static Map<String, String> scrollParams(TimeValue keepAlive) {
-        return singletonMap("scroll", keepAlive.toString());
+    static Map<String, String> scrollParams(TimeValue keepAlive, Version remoteVersion) {
+        if (remoteVersion.before(Version.V_5_0_0)) {
+            /* Versions of Elasticsearch before 5.0 couldn't parse nanos or micros
+             * so we toss out that resolution, rounding up so we shouldn't end up
+             * with 0s. */
+            keepAlive = timeValueMillis((long) Math.ceil(keepAlive.millisFrac()));
+        }
+        return singletonMap("scroll", keepAlive.getStringRep());
+    }
+
+    static HttpEntity scrollEntity(String scroll, Version remoteVersion) {
+        if (remoteVersion.before(Version.V_2_0_0)) {
+            // Versions before 2.0.0 extract the plain scroll_id from the body
+            return new StringEntity(scroll, ContentType.TEXT_PLAIN);
+        }
+        try (XContentBuilder entity = JsonXContent.contentBuilder()) {
+            return new StringEntity(entity.startObject()
+                .field("scroll_id", scroll)
+                .endObject().string(), ContentType.APPLICATION_JSON);
+        } catch (IOException e) {
+            throw new ElasticsearchException("failed to build scroll entity", e);
+        }
     }
 
-    static HttpEntity scrollEntity(String scroll) {
-        return new StringEntity(scroll, ContentType.TEXT_PLAIN);
+    static HttpEntity clearScrollEntity(String scroll, Version remoteVersion) {
+        if (remoteVersion.before(Version.V_2_0_0)) {
+            // Versions before 2.0.0 extract the plain scroll_id from the body
+            return new StringEntity(scroll, ContentType.TEXT_PLAIN);
+        }
+        try (XContentBuilder entity = JsonXContent.contentBuilder()) {
+            return new StringEntity(entity.startObject()
+                .array("scroll_id", scroll)
+                .endObject().string(), ContentType.APPLICATION_JSON);
+        } catch (IOException e) {
+            throw new ElasticsearchException("failed to build clear scroll entity", e);
+        }
     }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteResponseParsers.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteResponseParsers.java
index 3ae1f33df540a..06dcd92d2f28f 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteResponseParsers.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteResponseParsers.java
@@ -20,9 +20,13 @@
 package org.elasticsearch.index.reindex.remote;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.index.reindex.ScrollableHitSource.BasicHit;
+import org.elasticsearch.index.reindex.ScrollableHitSource.Hit;
+import org.elasticsearch.index.reindex.ScrollableHitSource.Response;
+import org.elasticsearch.index.reindex.ScrollableHitSource.SearchFailure;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.ObjectParser;
@@ -30,11 +34,7 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentLocation;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.json.JsonXContent;
-import org.elasticsearch.index.reindex.ScrollableHitSource.BasicHit;
-import org.elasticsearch.index.reindex.ScrollableHitSource.Hit;
-import org.elasticsearch.index.reindex.ScrollableHitSource.Response;
-import org.elasticsearch.index.reindex.ScrollableHitSource.SearchFailure;
+import org.elasticsearch.common.xcontent.XContentType;
 
 import java.io.IOException;
 import java.util.List;
@@ -47,10 +47,7 @@
 import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
 
 /**
- * Parsers to convert the response from the remote host into objects useful for {@link RemoteScrollableHitSource}. Lots of data is
- * intentionally thrown on the floor because we don't need it but ObjectParser and friends are strict about blowing up when they see
- * elements they don't understand. So you'll see a lot of BiConsumers that look like "(b, v) -&gt; {}". That means "I don't care about the
- * value here, just throw it away and don't blow up.
+ * Parsers to convert the response from the remote host into objects useful for {@link RemoteScrollableHitSource}.
  */
 final class RemoteResponseParsers {
     private RemoteResponseParsers() {}
@@ -58,8 +55,8 @@ private RemoteResponseParsers() {}
     /**
      * Parser for an individual {@code hit} element.
      */
-    public static final ConstructingObjectParser<BasicHit, ParseFieldMatcherSupplier> HIT_PARSER = new ConstructingObjectParser<>("hit",
-            a -> {
+    public static final ConstructingObjectParser<BasicHit, XContentType> HIT_PARSER =
+            new ConstructingObjectParser<>("hit", true, a -> {
                 int i = 0;
                 String index = (String) a[i++];
                 String type = (String) a[i++];
@@ -72,44 +69,60 @@ private RemoteResponseParsers() {}
         HIT_PARSER.declareString(constructorArg(), new ParseField("_type"));
         HIT_PARSER.declareString(constructorArg(), new ParseField("_id"));
         HIT_PARSER.declareLong(optionalConstructorArg(), new ParseField("_version"));
-        HIT_PARSER.declareObject(BasicHit::setSource, (p, s) -> {
+        HIT_PARSER.declareObject(((basicHit, tuple) -> basicHit.setSource(tuple.v1(), tuple.v2())), (p, s) -> {
             try {
                 /*
                  * We spool the data from the remote back into xcontent so we can get bytes to send. There ought to be a better way but for
                  * now this should do.
                  */
-                try (XContentBuilder b = JsonXContent.contentBuilder()) {
+                try (XContentBuilder b = XContentBuilder.builder(s.xContent())) {
                     b.copyCurrentStructure(p);
-                    return b.bytes();
+                    // a hack but this lets us get the right xcontent type to go with the source
+                    return new Tuple<>(b.bytes(), s);
                 }
             } catch (IOException e) {
                 throw new ParsingException(p.getTokenLocation(), "[hit] failed to parse [_source]", e);
             }
         }, new ParseField("_source"));
-        HIT_PARSER.declareString(BasicHit::setRouting, new ParseField("_routing"));
-        HIT_PARSER.declareString(BasicHit::setParent, new ParseField("_parent"));
-        HIT_PARSER.declareLong(BasicHit::setTTL, new ParseField("_ttl"));
+        ParseField routingField = new ParseField("_routing");
+        ParseField parentField = new ParseField("_parent");
+        ParseField ttlField = new ParseField("_ttl");
+        HIT_PARSER.declareString(BasicHit::setRouting, routingField);
+        HIT_PARSER.declareString(BasicHit::setParent, parentField);
+        HIT_PARSER.declareLong(BasicHit::setTTL, ttlField);
         HIT_PARSER.declareLong(BasicHit::setTimestamp, new ParseField("_timestamp"));
-        HIT_PARSER.declareField((b, v) -> {}, p -> null, new ParseField("_score"), ValueType.FLOAT_OR_NULL);
-        HIT_PARSER.declareStringArray((b, v) -> {}, new ParseField("sort"));
+        // Pre-2.0.0 parent and routing come back in "fields"
+        class Fields {
+            String routing;
+            String parent;
+            long ttl;
+        }
+        ObjectParser<Fields, XContentType> fieldsParser = new ObjectParser<>("fields", Fields::new);
+        HIT_PARSER.declareObject((hit, fields) -> {
+            hit.setRouting(fields.routing);
+            hit.setParent(fields.parent);
+            hit.setTTL(fields.ttl);
+        }, fieldsParser, new ParseField("fields"));
+        fieldsParser.declareString((fields, routing) -> fields.routing = routing, routingField);
+        fieldsParser.declareString((fields, parent) -> fields.parent = parent, parentField);
+        fieldsParser.declareLong((fields, ttl) -> fields.ttl = ttl, ttlField);
     }
 
     /**
      * Parser for the {@code hits} element. Parsed to an array of {@code [total (Long), hits (List<Hit>)]}.
      */
-    public static final ConstructingObjectParser<Object[], ParseFieldMatcherSupplier> HITS_PARSER = new ConstructingObjectParser<>("hits",
-            a -> a);
+    public static final ConstructingObjectParser<Object[], XContentType> HITS_PARSER =
+            new ConstructingObjectParser<>("hits", true, a -> a);
     static {
         HITS_PARSER.declareLong(constructorArg(), new ParseField("total"));
         HITS_PARSER.declareObjectArray(constructorArg(), HIT_PARSER, new ParseField("hits"));
-        HITS_PARSER.declareField((b, v) -> {}, p -> null, new ParseField("max_score"), ValueType.FLOAT_OR_NULL);
     }
 
     /**
      * Parser for {@code failed} shards in the {@code _shards} elements.
      */
-    public static final ConstructingObjectParser<SearchFailure, ParseFieldMatcherSupplier> SEARCH_FAILURE_PARSER =
-            new ConstructingObjectParser<>("failure", a -> {
+    public static final ConstructingObjectParser<SearchFailure, XContentType> SEARCH_FAILURE_PARSER =
+            new ConstructingObjectParser<>("failure", true, a -> {
                 int i = 0;
                 String index = (String) a[i++];
                 Integer shardId = (Integer) a[i++];
@@ -135,15 +148,14 @@ private RemoteResponseParsers() {}
                 return p.text();
             }
         }, new ParseField("reason"), ValueType.OBJECT_OR_STRING);
-        SEARCH_FAILURE_PARSER.declareInt((b, v) -> {}, new ParseField("status"));
     }
 
     /**
      * Parser for the {@code _shards} element. Throws everything out except the errors array if there is one. If there isn't one then it
      * parses to an empty list.
      */
-    public static final ConstructingObjectParser<List<Throwable>, ParseFieldMatcherSupplier> SHARDS_PARSER =
-            new ConstructingObjectParser<>("_shards", a -> {
+    public static final ConstructingObjectParser<List<Throwable>, XContentType> SHARDS_PARSER =
+            new ConstructingObjectParser<>("_shards", true, a -> {
                 @SuppressWarnings("unchecked")
                 List<Throwable> failures = (List<Throwable>) a[0];
                 failures = failures == null ? emptyList() : failures;
@@ -151,13 +163,10 @@ private RemoteResponseParsers() {}
             });
     static {
         SHARDS_PARSER.declareObjectArray(optionalConstructorArg(), SEARCH_FAILURE_PARSER, new ParseField("failures"));
-        SHARDS_PARSER.declareInt((b, v) -> {}, new ParseField("total"));
-        SHARDS_PARSER.declareInt((b, v) -> {}, new ParseField("successful"));
-        SHARDS_PARSER.declareInt((b, v) -> {}, new ParseField("failed"));
     }
 
-    public static final ConstructingObjectParser<Response, ParseFieldMatcherSupplier> RESPONSE_PARSER =
-            new ConstructingObjectParser<>("search_response", a -> {
+    public static final ConstructingObjectParser<Response, XContentType> RESPONSE_PARSER =
+            new ConstructingObjectParser<>("search_response", true, a -> {
                 int i = 0;
                 Throwable catastrophicFailure = (Throwable) a[i++];
                 if (catastrophicFailure != null) {
@@ -184,39 +193,28 @@ private RemoteResponseParsers() {}
                 return new Response(timedOut, failures, totalHits, hits, scroll);
             });
     static {
-        RESPONSE_PARSER.declareObject(optionalConstructorArg(), ThrowableBuilder.PARSER, new ParseField("error"));
+        RESPONSE_PARSER.declareObject(optionalConstructorArg(), ThrowableBuilder.PARSER::apply, new ParseField("error"));
         RESPONSE_PARSER.declareBoolean(optionalConstructorArg(), new ParseField("timed_out"));
         RESPONSE_PARSER.declareString(optionalConstructorArg(), new ParseField("_scroll_id"));
         RESPONSE_PARSER.declareObject(optionalConstructorArg(), HITS_PARSER, new ParseField("hits"));
         RESPONSE_PARSER.declareObject(optionalConstructorArg(), SHARDS_PARSER, new ParseField("_shards"));
-        RESPONSE_PARSER.declareInt((b, v) -> {}, new ParseField("took"));
-        RESPONSE_PARSER.declareBoolean((b, v) -> {}, new ParseField("terminated_early"));
-        RESPONSE_PARSER.declareInt((b, v) -> {}, new ParseField("status"));
     }
 
     /**
      * Collects stuff about Throwables and attempts to rebuild them.
      */
     public static class ThrowableBuilder {
-        public static final BiFunction<XContentParser, ParseFieldMatcherSupplier, Throwable> PARSER;
+        public static final BiFunction<XContentParser, XContentType, Throwable> PARSER;
         static {
-            ObjectParser<ThrowableBuilder, ParseFieldMatcherSupplier> parser = new ObjectParser<>("reason", ThrowableBuilder::new);
+            ObjectParser<ThrowableBuilder, XContentType> parser = new ObjectParser<>("reason", true, ThrowableBuilder::new);
             PARSER = parser.andThen(ThrowableBuilder::build);
             parser.declareString(ThrowableBuilder::setType, new ParseField("type"));
             parser.declareString(ThrowableBuilder::setReason, new ParseField("reason"));
-            parser.declareObject(ThrowableBuilder::setCausedBy, PARSER, new ParseField("caused_by"));
+            parser.declareObject(ThrowableBuilder::setCausedBy, PARSER::apply, new ParseField("caused_by"));
 
             // So we can give a nice error for parsing exceptions
             parser.declareInt(ThrowableBuilder::setLine, new ParseField("line"));
             parser.declareInt(ThrowableBuilder::setColumn, new ParseField("col"));
-
-            // So we don't blow up on search exceptions
-            parser.declareString((b, v) -> {}, new ParseField("phase"));
-            parser.declareBoolean((b, v) -> {}, new ParseField("grouped"));
-            parser.declareField((p, v, c) -> p.skipChildren(), new ParseField("failed_shards"), ValueType.OBJECT_ARRAY);
-
-            // Just throw away the root_cause
-            parser.declareField((p, v, c) -> p.skipChildren(), new ParseField("root_cause"), ValueType.OBJECT_ARRAY);
         }
 
         private String type;
@@ -227,7 +225,7 @@ public static class ThrowableBuilder {
 
         public Throwable build() {
             Throwable t = buildWithoutCause();
-            if (causedBy != null) { 
+            if (causedBy != null) {
                 t.initCause(causedBy);
             }
             return t;
@@ -269,33 +267,15 @@ public void setCausedBy(Throwable causedBy) {
         }
     }
 
-    /**
-     * Parses the {@code version} field of the main action. There are a surprising number of fields in this that we don't need!
-     */
-    public static final ConstructingObjectParser<Version, ParseFieldMatcherSupplier> VERSION_PARSER = new ConstructingObjectParser<>(
-            "version", a -> Version.fromString((String) a[0]));
-    static {
-        VERSION_PARSER.declareString(constructorArg(), new ParseField("number"));
-        VERSION_PARSER.declareBoolean((p, v) -> {}, new ParseField("snapshot_build"));
-        VERSION_PARSER.declareBoolean((p, v) -> {}, new ParseField("build_snapshot"));
-        VERSION_PARSER.declareString((p, v) -> {}, new ParseField("build_hash"));
-        VERSION_PARSER.declareString((p, v) -> {}, new ParseField("build_date"));
-        VERSION_PARSER.declareString((p, v) -> {}, new ParseField("build_timestamp"));
-        VERSION_PARSER.declareString((p, v) -> {}, new ParseField("lucene_version"));
-    }
-
     /**
      * Parses the main action to return just the {@linkplain Version} that it returns. We throw everything else out.
      */
-    public static final ConstructingObjectParser<Version, ParseFieldMatcherSupplier> MAIN_ACTION_PARSER = new ConstructingObjectParser<>(
-            "/", a -> (Version) a[0]);
+    public static final ConstructingObjectParser<Version, XContentType> MAIN_ACTION_PARSER = new ConstructingObjectParser<>(
+            "/", true, a -> (Version) a[0]);
     static {
-        MAIN_ACTION_PARSER.declareBoolean((p, v) -> {}, new ParseField("ok"));
-        MAIN_ACTION_PARSER.declareInt((p, v) -> {}, new ParseField("status"));
-        MAIN_ACTION_PARSER.declareString((p, v) -> {}, new ParseField("name"));
-        MAIN_ACTION_PARSER.declareString((p, v) -> {}, new ParseField("cluster_name"));
-        MAIN_ACTION_PARSER.declareString((p, v) -> {}, new ParseField("name"));
-        MAIN_ACTION_PARSER.declareString((p, v) -> {}, new ParseField("tagline"));
-        MAIN_ACTION_PARSER.declareObject(constructorArg(), VERSION_PARSER, new ParseField("version"));
+        ConstructingObjectParser<Version, XContentType> versionParser = new ConstructingObjectParser<>(
+                "version", true, a -> Version.fromString((String) a[0]));
+        versionParser.declareString(constructorArg(), new ParseField("number"));
+        MAIN_ACTION_PARSER.declareObject(constructorArg(), versionParser, new ParseField("version"));
     }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSource.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSource.java
index 207948c921552..b9592c2c4f941 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSource.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSource.java
@@ -19,7 +19,9 @@
 
 package org.elasticsearch.index.reindex.remote;
 
+import org.apache.http.ContentTooLongException;
 import org.apache.http.HttpEntity;
+import org.apache.http.entity.ContentType;
 import org.apache.http.util.EntityUtils;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
@@ -28,22 +30,21 @@
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.bulk.BackoffPolicy;
+import org.elasticsearch.index.reindex.ScrollableHitSource;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.ResponseListener;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
+import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.index.reindex.ScrollableHitSource;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.threadpool.ThreadPool;
 
@@ -57,6 +58,7 @@
 import static java.util.Collections.emptyMap;
 import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
 import static org.elasticsearch.common.unit.TimeValue.timeValueNanos;
+import static org.elasticsearch.index.reindex.remote.RemoteRequestBuilders.clearScrollEntity;
 import static org.elasticsearch.index.reindex.remote.RemoteRequestBuilders.initialSearchEntity;
 import static org.elasticsearch.index.reindex.remote.RemoteRequestBuilders.initialSearchParams;
 import static org.elasticsearch.index.reindex.remote.RemoteRequestBuilders.initialSearchPath;
@@ -80,21 +82,12 @@ public RemoteScrollableHitSource(Logger logger, BackoffPolicy backoffPolicy, Thr
         this.client = client;
     }
 
-    @Override
-    public void close() {
-        try {
-            client.close();
-        } catch (IOException e) {
-            fail.accept(new IOException("couldn't close the remote connection", e));
-        }
-    }
-
     @Override
     protected void doStart(Consumer<? super Response> onResponse) {
         lookupRemoteVersion(version -> {
             remoteVersion = version;
             execute("POST", initialSearchPath(searchRequest), initialSearchParams(searchRequest, version),
-                    initialSearchEntity(query), RESPONSE_PARSER, r -> onStartResponse(onResponse, r));
+                    initialSearchEntity(searchRequest, query, remoteVersion), RESPONSE_PARSER, r -> onStartResponse(onResponse, r));
         });
     }
 
@@ -113,30 +106,63 @@ private void onStartResponse(Consumer<? super Response> onResponse, Response res
 
     @Override
     protected void doStartNextScroll(String scrollId, TimeValue extraKeepAlive, Consumer<? super Response> onResponse) {
-        execute("POST", scrollPath(), scrollParams(timeValueNanos(searchRequest.scroll().keepAlive().nanos() + extraKeepAlive.nanos())),
-                scrollEntity(scrollId), RESPONSE_PARSER, onResponse);
+        Map<String, String> scrollParams = scrollParams(
+                timeValueNanos(searchRequest.scroll().keepAlive().nanos() + extraKeepAlive.nanos()),
+                remoteVersion);
+        execute("POST", scrollPath(), scrollParams, scrollEntity(scrollId, remoteVersion), RESPONSE_PARSER, onResponse);
     }
 
     @Override
-    protected void clearScroll(String scrollId) {
-        // Need to throw out response....
-        client.performRequestAsync("DELETE", scrollPath(), emptyMap(), scrollEntity(scrollId), new ResponseListener() {
+    protected void clearScroll(String scrollId, Runnable onCompletion) {
+        client.performRequestAsync("DELETE", scrollPath(), emptyMap(), clearScrollEntity(scrollId, remoteVersion), new ResponseListener() {
             @Override
             public void onSuccess(org.elasticsearch.client.Response response) {
                 logger.debug("Successfully cleared [{}]", scrollId);
+                onCompletion.run();
             }
 
             @Override
-            public void onFailure(Exception t) {
-                logger.warn((Supplier<?>) () -> new ParameterizedMessage("Failed to clear scroll [{}]", scrollId), t);
+            public void onFailure(Exception e) {
+                logFailure(e);
+                onCompletion.run();
+            }
+
+            private void logFailure(Exception e) {
+                if (e instanceof ResponseException) {
+                    ResponseException re = (ResponseException) e;
+                    if (remoteVersion.before(Version.V_2_0_0) && re.getResponse().getStatusLine().getStatusCode() == 404) {
+                        logger.debug((Supplier<?>) () -> new ParameterizedMessage(
+                                "Failed to clear scroll [{}] from pre-2.0 Elasticsearch. This is normal if the request terminated "
+                                        + "normally as the scroll has already been cleared automatically.", scrollId), e);
+                        return;
+                    }
+                }
+                logger.warn((Supplier<?>) () -> new ParameterizedMessage("Failed to clear scroll [{}]", scrollId), e);
+            }
+        });
+    }
+
+    @Override
+    protected void cleanup(Runnable onCompletion) {
+        /* This is called on the RestClient's thread pool and attempting to close the client on its
+         * own threadpool causes it to fail to close. So we always shutdown the RestClient
+         * asynchronously on a thread in Elasticsearch's generic thread pool. */
+        threadPool.generic().submit(() -> {
+            try {
+                client.close();
+                logger.debug("Shut down remote connection");
+            } catch (IOException e) {
+                logger.error("Failed to shutdown the remote connection", e);
+            } finally {
+                onCompletion.run();
             }
         });
     }
 
     private <T> void execute(String method, String uri, Map<String, String> params, HttpEntity entity,
-            BiFunction<XContentParser, ParseFieldMatcherSupplier, T> parser, Consumer<? super T> listener) {
+                             BiFunction<XContentParser, XContentType, T> parser, Consumer<? super T> listener) {
         // Preserve the thread context so headers survive after the call
-        ThreadContext.StoredContext ctx = threadPool.getThreadContext().newStoredContext();
+        java.util.function.Supplier<ThreadContext.StoredContext> contextSupplier = threadPool.getThreadContext().newRestorableContext(true);
         class RetryHelper extends AbstractRunnable {
             private final Iterator<TimeValue> retries = backoffPolicy.iterator();
 
@@ -146,46 +172,69 @@ protected void doRun() throws Exception {
                     @Override
                     public void onSuccess(org.elasticsearch.client.Response response) {
                         // Restore the thread context to get the precious headers
-                        ctx.restore();
-                        T parsedResponse;
-                        try {
-                            HttpEntity responseEntity = response.getEntity();
-                            InputStream content = responseEntity.getContent();
-                            XContentType xContentType = null;
-                            if (responseEntity.getContentType() != null) {
-                                 xContentType = XContentType.fromMediaTypeOrFormat(responseEntity.getContentType().getValue());
-                            }
-                            if (xContentType == null) {
-                                //auto-detect as a fallback
-                                xContentType = XContentFactory.xContentType(content);
-                            }
-                            try(XContentParser xContentParser = xContentType.xContent().createParser(content)) {
-                                parsedResponse = parser.apply(xContentParser, () -> ParseFieldMatcher.STRICT);
+                        try (ThreadContext.StoredContext ctx = contextSupplier.get()) {
+                            assert ctx != null; // eliminates compiler warning
+                            T parsedResponse;
+                            try {
+                                HttpEntity responseEntity = response.getEntity();
+                                InputStream content = responseEntity.getContent();
+                                XContentType xContentType = null;
+                                if (responseEntity.getContentType() != null) {
+                                    final String mimeType = ContentType.parse(responseEntity.getContentType().getValue()).getMimeType();
+                                    xContentType = XContentType.fromMediaType(mimeType);
+                                }
+                                if (xContentType == null) {
+                                    try {
+                                        throw new ElasticsearchException(
+                                            "Response didn't include Content-Type: " + bodyMessage(response.getEntity()));
+                                    } catch (IOException e) {
+                                        ElasticsearchException ee = new ElasticsearchException("Error extracting body from response");
+                                        ee.addSuppressed(e);
+                                        throw ee;
+                                    }
+                                }
+                                // EMPTY is safe here because we don't call namedObject
+                                try (XContentParser xContentParser = xContentType.xContent().createParser(NamedXContentRegistry.EMPTY,
+                                    content)) {
+                                    parsedResponse = parser.apply(xContentParser, xContentType);
+                                } catch (ParsingException e) {
+                                /* Because we're streaming the response we can't get a copy of it here. The best we can do is hint that it
+                                 * is totally wrong and we're probably not talking to Elasticsearch. */
+                                    throw new ElasticsearchException(
+                                        "Error parsing the response, remote is likely not an Elasticsearch instance", e);
+                                }
+                            } catch (IOException e) {
+                                throw new ElasticsearchException(
+                                    "Error deserializing response, remote is likely not an Elasticsearch instance", e);
                             }
-                        } catch (IOException e) {
-                            throw new ElasticsearchException("Error deserializing response", e);
+                            listener.accept(parsedResponse);
                         }
-                        listener.accept(parsedResponse);
                     }
 
                     @Override
                     public void onFailure(Exception e) {
-                        if (e instanceof ResponseException) {
-                            ResponseException re = (ResponseException) e;
-                            if (RestStatus.TOO_MANY_REQUESTS.getStatus() == re.getResponse().getStatusLine().getStatusCode()) {
-                                if (retries.hasNext()) {
-                                    TimeValue delay = retries.next();
-                                    logger.trace(
-                                        (Supplier<?>) () -> new ParameterizedMessage("retrying rejected search after [{}]", delay), e);
-                                    countSearchRetry.run();
-                                    threadPool.schedule(delay, ThreadPool.Names.SAME, RetryHelper.this);
-                                    return;
+                        try (ThreadContext.StoredContext ctx = contextSupplier.get()) {
+                            assert ctx != null; // eliminates compiler warning
+                            if (e instanceof ResponseException) {
+                                ResponseException re = (ResponseException) e;
+                                if (RestStatus.TOO_MANY_REQUESTS.getStatus() == re.getResponse().getStatusLine().getStatusCode()) {
+                                    if (retries.hasNext()) {
+                                        TimeValue delay = retries.next();
+                                        logger.trace(
+                                            (Supplier<?>) () -> new ParameterizedMessage("retrying rejected search after [{}]", delay), e);
+                                        countSearchRetry.run();
+                                        threadPool.schedule(delay, ThreadPool.Names.SAME, RetryHelper.this);
+                                        return;
+                                    }
                                 }
-                            }
-                            e = wrapExceptionToPreserveStatus(re.getResponse().getStatusLine().getStatusCode(),
+                                e = wrapExceptionToPreserveStatus(re.getResponse().getStatusLine().getStatusCode(),
                                     re.getResponse().getEntity(), re);
+                            } else if (e instanceof ContentTooLongException) {
+                                e = new IllegalArgumentException(
+                                    "Remote responded with a chunk that was too large. Use a smaller batch size.", e);
+                            }
+                            fail.accept(e);
                         }
-                        fail.accept(e);
                     }
                 });
             }
@@ -210,18 +259,20 @@ static ElasticsearchStatusException wrapExceptionToPreserveStatus(int statusCode
             messagePrefix = "Couldn't extract status [" + statusCode + "]. ";
             status = RestStatus.INTERNAL_SERVER_ERROR;
         }
-        String message;
+        try {
+            return new ElasticsearchStatusException(messagePrefix + bodyMessage(entity), status, cause);
+        } catch (IOException ioe) {
+            ElasticsearchStatusException e = new ElasticsearchStatusException(messagePrefix + "Failed to extract body.", status, cause);
+            e.addSuppressed(ioe);
+            return e;
+        }
+    }
+
+    static String bodyMessage(@Nullable HttpEntity entity) throws IOException {
         if (entity == null) {
-            message = messagePrefix + "No error body.";
+            return "No error body.";
         } else {
-            try {
-                message = messagePrefix + "body=" + EntityUtils.toString(entity);
-            } catch (IOException ioe) {
-                ElasticsearchStatusException e = new ElasticsearchStatusException(messagePrefix + "Failed to extract body.", status, cause);
-                e.addSuppressed(ioe);
-                return e;
-            }
+            return "body=" + EntityUtils.toString(entity);
         }
-        return new ElasticsearchStatusException(message, status, cause);
     }
 }
diff --git a/modules/reindex/src/main/plugin-metadata/plugin-security.policy b/modules/reindex/src/main/plugin-metadata/plugin-security.policy
new file mode 100644
index 0000000000000..70ac816613a60
--- /dev/null
+++ b/modules/reindex/src/main/plugin-metadata/plugin-security.policy
@@ -0,0 +1,28 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+grant codeBase "${codebase.elasticsearch-rest-client}" {
+  // rest client uses system properties which gets the default proxy
+  permission java.net.NetPermission "getProxySelector";
+};
+
+grant codeBase "${codebase.httpasyncclient-4.1.2.jar}" {
+  // rest client uses system properties which gets the default proxy
+  permission java.net.NetPermission "getProxySelector";
+};
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionMetadataTestCase.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionMetadataTestCase.java
new file mode 100644
index 0000000000000..4177ab5955d79
--- /dev/null
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionMetadataTestCase.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex;
+
+import org.elasticsearch.action.index.IndexRequest;
+
+import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
+
+public abstract class AbstractAsyncBulkByScrollActionMetadataTestCase<
+                Request extends AbstractBulkByScrollRequest<Request>,
+                Response extends BulkByScrollResponse>
+        extends AbstractAsyncBulkByScrollActionTestCase<Request, Response> {
+
+    protected ScrollableHitSource.BasicHit doc() {
+        return new ScrollableHitSource.BasicHit("index", "type", "id", 0);
+    }
+
+    public void testTimestampIsCopied() {
+        IndexRequest index = new IndexRequest();
+        action().copyMetadata(AbstractAsyncBulkByScrollAction.wrap(index), doc().setTimestamp(10L));
+        assertEquals("10", index.timestamp());
+    }
+
+    public void testTTL() throws Exception {
+        IndexRequest index = new IndexRequest();
+        action().copyMetadata(AbstractAsyncBulkByScrollAction.wrap(index), doc().setTTL(10L));
+        assertEquals(timeValueMillis(10), index.ttl());
+    }
+
+    protected abstract AbstractAsyncBulkByScrollAction<Request> action();
+}
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionScriptTestCase.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionScriptTestCase.java
new file mode 100644
index 0000000000000..6ddf6daa8806c
--- /dev/null
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionScriptTestCase.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.index.reindex.AbstractAsyncBulkByScrollAction.OpType;
+import org.elasticsearch.index.reindex.AbstractAsyncBulkByScrollAction.RequestWrapper;
+import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.script.CompiledScript;
+import org.elasticsearch.script.ExecutableScript;
+import org.elasticsearch.script.ScriptService;
+import org.junit.Before;
+import org.mockito.Matchers;
+
+import java.util.Map;
+import java.util.function.Consumer;
+
+import static java.util.Collections.singletonMap;
+import static org.hamcrest.Matchers.equalTo;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public abstract class AbstractAsyncBulkByScrollActionScriptTestCase<
+                Request extends AbstractBulkIndexByScrollRequest<Request>,
+                Response extends BulkByScrollResponse>
+        extends AbstractAsyncBulkByScrollActionTestCase<Request, Response> {
+
+    protected ScriptService scriptService;
+
+    @Before
+    public void setupScriptService() {
+        scriptService = mock(ScriptService.class);
+    }
+
+    @SuppressWarnings("unchecked")
+    protected <T extends ActionRequest> T applyScript(Consumer<Map<String, Object>> scriptBody) {
+        IndexRequest index = new IndexRequest("index", "type", "1").source(singletonMap("foo", "bar"));
+        ScrollableHitSource.Hit doc = new ScrollableHitSource.BasicHit("test", "type", "id", 0);
+        ExecutableScript executableScript = new SimpleExecutableScript(scriptBody);
+
+        when(scriptService.executable(any(CompiledScript.class), Matchers.<Map<String, Object>>any()))
+                .thenReturn(executableScript);
+        AbstractAsyncBulkByScrollAction<Request> action = action(scriptService, request().setScript(mockScript("")));
+        RequestWrapper<?> result = action.buildScriptApplier().apply(AbstractAsyncBulkByScrollAction.wrap(index), doc);
+        return (result != null) ? (T) result.self() : null;
+    }
+
+    public void testScriptAddingJunkToCtxIsError() {
+        try {
+            applyScript((Map<String, Object> ctx) -> ctx.put("junk", "junk"));
+            fail("Expected error");
+        } catch (IllegalArgumentException e) {
+            assertThat(e.getMessage(), equalTo("Invalid fields added to context [junk]"));
+        }
+    }
+
+    public void testChangeSource() {
+        IndexRequest index = applyScript((Map<String, Object> ctx) -> {
+            @SuppressWarnings("unchecked")
+            Map<String, Object> source = (Map<String, Object>) ctx.get("_source");
+            source.put("bar", "cat");
+        });
+        assertEquals("cat", index.sourceAsMap().get("bar"));
+    }
+
+    public void testSetOpTypeNoop() throws Exception {
+        assertThat(task.getStatus().getNoops(), equalTo(0L));
+        assertNull(applyScript((Map<String, Object> ctx) -> ctx.put("op", OpType.NOOP.toString())));
+        assertThat(task.getStatus().getNoops(), equalTo(1L));
+    }
+
+    public void testSetOpTypeDelete() throws Exception {
+        DeleteRequest delete = applyScript((Map<String, Object> ctx) -> ctx.put("op", OpType.DELETE.toString()));
+        assertThat(delete.index(), equalTo("index"));
+        assertThat(delete.type(), equalTo("type"));
+        assertThat(delete.id(), equalTo("1"));
+    }
+
+    public void testSetOpTypeUnknown() throws Exception {
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> applyScript((Map<String, Object> ctx) -> ctx.put("op", "unknown")));
+        assertThat(e.getMessage(), equalTo("Operation type [unknown] not allowed, only [noop, index, delete] are allowed"));
+    }
+
+    protected abstract AbstractAsyncBulkByScrollAction<Request> action(ScriptService scriptService, Request request);
+}
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollActionScriptTestCase.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollActionScriptTestCase.java
deleted file mode 100644
index f8351b262fc05..0000000000000
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollActionScriptTestCase.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.reindex;
-
-import org.elasticsearch.action.ActionRequest;
-import org.elasticsearch.action.delete.DeleteRequest;
-import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.index.reindex.AbstractAsyncBulkIndexByScrollAction.OpType;
-import org.elasticsearch.index.reindex.AbstractAsyncBulkIndexByScrollAction.RequestWrapper;
-import org.elasticsearch.script.CompiledScript;
-import org.elasticsearch.script.ExecutableScript;
-import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService;
-import org.junit.Before;
-import org.mockito.Matchers;
-
-import java.util.Map;
-import java.util.function.Consumer;
-
-import static java.util.Collections.singletonMap;
-import static org.hamcrest.Matchers.equalTo;
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-public abstract class AbstractAsyncBulkIndexByScrollActionScriptTestCase<
-                Request extends AbstractBulkIndexByScrollRequest<Request>,
-                Response extends BulkIndexByScrollResponse>
-        extends AbstractAsyncBulkIndexByScrollActionTestCase<Request, Response> {
-
-    private static final Script EMPTY_SCRIPT = new Script("");
-
-    protected ScriptService scriptService;
-
-    @Before
-    public void setupScriptService() {
-        scriptService = mock(ScriptService.class);
-    }
-
-    @SuppressWarnings("unchecked")
-    protected <T extends ActionRequest<?>> T applyScript(Consumer<Map<String, Object>> scriptBody) {
-        IndexRequest index = new IndexRequest("index", "type", "1").source(singletonMap("foo", "bar"));
-        ScrollableHitSource.Hit doc = new ScrollableHitSource.BasicHit("test", "type", "id", 0);
-        ExecutableScript executableScript = new SimpleExecutableScript(scriptBody);
-
-        when(scriptService.executable(any(CompiledScript.class), Matchers.<Map<String, Object>>any()))
-                .thenReturn(executableScript);
-        AbstractAsyncBulkIndexByScrollAction<Request> action = action(scriptService, request().setScript(EMPTY_SCRIPT));
-        RequestWrapper<?> result = action.buildScriptApplier().apply(AbstractAsyncBulkIndexByScrollAction.wrap(index), doc);
-        return (result != null) ? (T) result.self() : null;
-    }
-
-    public void testScriptAddingJunkToCtxIsError() {
-        try {
-            applyScript((Map<String, Object> ctx) -> ctx.put("junk", "junk"));
-            fail("Expected error");
-        } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), equalTo("Invalid fields added to context [junk]"));
-        }
-    }
-
-    public void testChangeSource() {
-        IndexRequest index = applyScript((Map<String, Object> ctx) -> {
-            @SuppressWarnings("unchecked")
-            Map<String, Object> source = (Map<String, Object>) ctx.get("_source");
-            source.put("bar", "cat");
-        });
-        assertEquals("cat", index.sourceAsMap().get("bar"));
-    }
-
-    public void testSetOpTypeNoop() throws Exception {
-        assertThat(task.getStatus().getNoops(), equalTo(0L));
-        assertNull(applyScript((Map<String, Object> ctx) -> ctx.put("op", OpType.NOOP.toString())));
-        assertThat(task.getStatus().getNoops(), equalTo(1L));
-    }
-
-    public void testSetOpTypeDelete() throws Exception {
-        DeleteRequest delete = applyScript((Map<String, Object> ctx) -> ctx.put("op", OpType.DELETE.toString()));
-        assertThat(delete.index(), equalTo("index"));
-        assertThat(delete.type(), equalTo("type"));
-        assertThat(delete.id(), equalTo("1"));
-    }
-
-    public void testSetOpTypeUnknown() throws Exception {
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-                () -> applyScript((Map<String, Object> ctx) -> ctx.put("op", "unknown")));
-        assertThat(e.getMessage(), equalTo("Operation type [unknown] not allowed, only [noop, index, delete] are allowed"));
-    }
-
-    protected abstract AbstractAsyncBulkIndexByScrollAction<Request> action(ScriptService scriptService, Request request);
-}
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollActionTestCase.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollActionTestCase.java
deleted file mode 100644
index 66c636f4f1d12..0000000000000
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollActionTestCase.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.reindex;
-
-import org.elasticsearch.action.support.PlainActionFuture;
-import org.elasticsearch.tasks.TaskId;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.threadpool.TestThreadPool;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.junit.After;
-import org.junit.Before;
-
-public abstract class AbstractAsyncBulkIndexByScrollActionTestCase<
-                Request extends AbstractBulkIndexByScrollRequest<Request>,
-                Response extends BulkIndexByScrollResponse>
-        extends ESTestCase {
-    protected ThreadPool threadPool;
-    protected BulkByScrollTask task;
-
-    @Before
-    public void setupForTest() {
-        threadPool = new TestThreadPool(getTestName());
-        task = new BulkByScrollTask(1, "test", "test", "test", TaskId.EMPTY_TASK_ID, 0);
-    }
-
-    @After
-    @Override
-    public void tearDown() throws Exception {
-        super.tearDown();
-        threadPool.shutdown();
-    }
-
-    protected abstract Request request();
-
-    protected PlainActionFuture<Response> listener() {
-        return new PlainActionFuture<>();
-    }
-}
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexbyScrollActionMetadataTestCase.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexbyScrollActionMetadataTestCase.java
deleted file mode 100644
index 4cc10334223c6..0000000000000
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexbyScrollActionMetadataTestCase.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.reindex;
-
-import org.elasticsearch.action.index.IndexRequest;
-
-import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
-
-public abstract class AbstractAsyncBulkIndexbyScrollActionMetadataTestCase<
-                Request extends AbstractBulkIndexByScrollRequest<Request>,
-                Response extends BulkIndexByScrollResponse>
-        extends AbstractAsyncBulkIndexByScrollActionTestCase<Request, Response> {
-
-    protected ScrollableHitSource.BasicHit doc() {
-        return new ScrollableHitSource.BasicHit("index", "type", "id", 0);
-    }
-
-    public void testTimestampIsCopied() {
-        IndexRequest index = new IndexRequest();
-        action().copyMetadata(AbstractAsyncBulkIndexByScrollAction.wrap(index), doc().setTimestamp(10L));
-        assertEquals("10", index.timestamp());
-    }
-
-    public void testTTL() throws Exception {
-        IndexRequest index = new IndexRequest();
-        action().copyMetadata(AbstractAsyncBulkIndexByScrollAction.wrap(index), doc().setTTL(10L));
-        assertEquals(timeValueMillis(10), index.ttl());
-    }
-
-    protected abstract AbstractAsyncBulkIndexByScrollAction<Request> action();
-}
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java
index f5fb927cda9ef..8d668fcb6bb58 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java
@@ -28,12 +28,16 @@
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.action.DocWriteResponse.Result;
+import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
 import org.elasticsearch.action.bulk.BackoffPolicy;
 import org.elasticsearch.action.bulk.BulkItemResponse;
 import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.index.reindex.ScrollableHitSource.Hit;
+import org.elasticsearch.index.reindex.ScrollableHitSource.SearchFailure;
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.delete.DeleteResponse;
 import org.elasticsearch.action.index.IndexRequest;
@@ -47,27 +51,26 @@
 import org.elasticsearch.action.search.SearchScrollRequest;
 import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.action.support.PlainActionFuture;
-import org.elasticsearch.action.support.replication.ReplicationRequest;
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.action.update.UpdateResponse;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.FilterClient;
 import org.elasticsearch.client.ParentTaskAssigningClient;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.text.Text;
 import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.engine.VersionConflictEngineException;
-import org.elasticsearch.index.reindex.ScrollableHitSource.Hit;
-import org.elasticsearch.index.reindex.ScrollableHitSource.SearchFailure;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.search.internal.InternalSearchHit;
-import org.elasticsearch.search.internal.InternalSearchHits;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.internal.InternalSearchResponse;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.tasks.TaskManager;
@@ -80,6 +83,7 @@
 
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.IdentityHashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -95,8 +99,10 @@
 import static java.util.Collections.emptyList;
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
+import static java.util.Collections.newSetFromMap;
 import static java.util.Collections.singleton;
 import static java.util.Collections.singletonList;
+import static java.util.Collections.synchronizedSet;
 import static org.apache.lucene.util.TestUtil.randomSimpleString;
 import static org.elasticsearch.action.bulk.BackoffPolicy.constantBackoff;
 import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
@@ -114,41 +120,45 @@
 
 public class AsyncBulkByScrollActionTests extends ESTestCase {
     private MyMockClient client;
-    private ThreadPool threadPool;
     private DummyAbstractBulkByScrollRequest testRequest;
     private SearchRequest firstSearchRequest;
-    private PlainActionFuture<BulkIndexByScrollResponse> listener;
+    private PlainActionFuture<BulkByScrollResponse> listener;
     private String scrollId;
     private TaskManager taskManager;
-    private BulkByScrollTask testTask;
+    private WorkingBulkByScrollTask testTask;
     private Map<String, String> expectedHeaders = new HashMap<>();
     private DiscoveryNode localNode;
     private TaskId taskId;
 
     @Before
     public void setupForTest() {
-        client = new MyMockClient(new NoOpClient(getTestName()));
-        threadPool = new TestThreadPool(getTestName());
+        // Fill the context with something random so we can make sure we inherited it appropriately.
+        expectedHeaders.clear();
+        expectedHeaders.put(randomSimpleString(random()), randomSimpleString(random()));
+
+        setupClient(new TestThreadPool(getTestName()));
         firstSearchRequest = new SearchRequest();
         testRequest = new DummyAbstractBulkByScrollRequest(firstSearchRequest);
         listener = new PlainActionFuture<>();
         scrollId = null;
         taskManager = new TaskManager(Settings.EMPTY);
-        testTask = (BulkByScrollTask) taskManager.register("don'tcare", "hereeither", testRequest);
+        testTask = (WorkingBulkByScrollTask) taskManager.register("don'tcare", "hereeither", testRequest);
 
-        // Fill the context with something random so we can make sure we inherited it appropriately.
-        expectedHeaders.clear();
-        expectedHeaders.put(randomSimpleString(random()), randomSimpleString(random()));
-        threadPool.getThreadContext().newStoredContext();
-        threadPool.getThreadContext().putHeader(expectedHeaders);
         localNode = new DiscoveryNode("thenode", new LocalTransportAddress("dead.end:666"), emptyMap(), emptySet(), Version.CURRENT);
         taskId = new TaskId(localNode.getId(), testTask.getId());
     }
 
+    private void setupClient(ThreadPool threadPool) {
+        if (client != null) {
+            client.close();
+        }
+        client = new MyMockClient(new NoOpClient(threadPool));
+        client.threadPool().getThreadContext().putHeader(expectedHeaders);
+    }
+
     @After
     public void tearDownAndVerifyCommonStuff() {
         client.close();
-        threadPool.shutdown();
     }
 
     /**
@@ -163,7 +173,7 @@ private String scrollId() {
 
     public void testStartRetriesOnRejectionAndSucceeds() throws Exception {
         client.searchesToReject = randomIntBetween(0, testRequest.getMaxRetries() - 1);
-        DummyAbstractAsyncBulkByScrollAction action = new DummyActionWithoutBackoff();
+        DummyAsyncBulkByScrollAction action = new DummyActionWithoutBackoff();
         action.start();
         assertBusy(() -> assertEquals(client.searchesToReject + 1, client.searchAttempts.get()));
         if (listener.isDone()) {
@@ -176,7 +186,7 @@ public void testStartRetriesOnRejectionAndSucceeds() throws Exception {
 
     public void testStartRetriesOnRejectionButFailsOnTooManyRejections() throws Exception {
         client.searchesToReject = testRequest.getMaxRetries() + randomIntBetween(1, 100);
-        DummyAbstractAsyncBulkByScrollAction action = new DummyActionWithoutBackoff();
+        DummyAsyncBulkByScrollAction action = new DummyActionWithoutBackoff();
         action.start();
         assertBusy(() -> assertEquals(testRequest.getMaxRetries() + 1, client.searchAttempts.get()));
         assertBusy(() -> assertTrue(listener.isDone()));
@@ -188,9 +198,10 @@ public void testStartRetriesOnRejectionButFailsOnTooManyRejections() throws Exce
 
     public void testStartNextScrollRetriesOnRejectionAndSucceeds() throws Exception {
         client.scrollsToReject = randomIntBetween(0, testRequest.getMaxRetries() - 1);
-        DummyAbstractAsyncBulkByScrollAction action = new DummyActionWithoutBackoff();
+        DummyAsyncBulkByScrollAction action = new DummyActionWithoutBackoff();
         action.setScroll(scrollId());
-        action.startNextScroll(timeValueNanos(System.nanoTime()), 0);
+        TimeValue now = timeValueNanos(System.nanoTime());
+        action.startNextScroll(now, now, 0);
         assertBusy(() -> assertEquals(client.scrollsToReject + 1, client.scrollAttempts.get()));
         if (listener.isDone()) {
             Object result = listener.get();
@@ -202,9 +213,10 @@ public void testStartNextScrollRetriesOnRejectionAndSucceeds() throws Exception
 
     public void testStartNextScrollRetriesOnRejectionButFailsOnTooManyRejections() throws Exception {
         client.scrollsToReject = testRequest.getMaxRetries() + randomIntBetween(1, 100);
-        DummyAbstractAsyncBulkByScrollAction action = new DummyActionWithoutBackoff();
+        DummyAsyncBulkByScrollAction action = new DummyActionWithoutBackoff();
         action.setScroll(scrollId());
-        action.startNextScroll(timeValueNanos(System.nanoTime()), 0);
+        TimeValue now = timeValueNanos(System.nanoTime());
+        action.startNextScroll(now, now, 0);
         assertBusy(() -> assertEquals(testRequest.getMaxRetries() + 1, client.scrollAttempts.get()));
         assertBusy(() -> assertTrue(listener.isDone()));
         ExecutionException e = expectThrows(ExecutionException.class, () -> listener.get());
@@ -219,7 +231,7 @@ public void testScrollResponseSetsTotal() {
 
         long total = randomIntBetween(0, Integer.MAX_VALUE);
         ScrollableHitSource.Response response = new ScrollableHitSource.Response(false, emptyList(), total, emptyList(), null);
-        simulateScrollResponse(new DummyAbstractAsyncBulkByScrollAction(), timeValueSeconds(0), 0, response);
+        simulateScrollResponse(new DummyAsyncBulkByScrollAction(), timeValueSeconds(0), 0, response);
         assertEquals(total, testTask.getStatus().getTotal());
     }
 
@@ -231,18 +243,12 @@ public void testScrollResponseBatchingBehavior() throws Exception {
         for (int batches = 1; batches < maxBatches; batches++) {
             Hit hit = new ScrollableHitSource.BasicHit("index", "type", "id", 0);
             ScrollableHitSource.Response response = new ScrollableHitSource.Response(false, emptyList(), 1, singletonList(hit), null);
-            DummyAbstractAsyncBulkByScrollAction action = new DummyAbstractAsyncBulkByScrollAction();
+            DummyAsyncBulkByScrollAction action = new DummyAsyncBulkByScrollAction();
             simulateScrollResponse(action, timeValueNanos(System.nanoTime()), 0, response);
 
             // Use assert busy because the update happens on another thread
             final int expectedBatches = batches;
             assertBusy(() -> assertEquals(expectedBatches, testTask.getStatus().getBatches()));
-
-            /*
-             * Also while we're here check that we preserved the headers from the last request. assertBusy because no requests might have
-             * come in yet.
-             */
-            assertBusy(() -> assertEquals(expectedHeaders, client.lastHeaders.get()));
         }
     }
 
@@ -257,37 +263,38 @@ public void testBulkResponseSetsLotsOfStatus() {
             BulkItemResponse[] responses = new BulkItemResponse[randomIntBetween(0, 100)];
             for (int i = 0; i < responses.length; i++) {
                 ShardId shardId = new ShardId(new Index("name", "uid"), 0);
-                String opType;
                 if (rarely()) {
-                    opType = randomSimpleString(random());
                     versionConflicts++;
-                    responses[i] = new BulkItemResponse(i, opType, new Failure(shardId.getIndexName(), "type", "id" + i,
-                            new VersionConflictEngineException(shardId, "type", "id", "test")));
+                    responses[i] = new BulkItemResponse(i, randomFrom(DocWriteRequest.OpType.values()),
+                            new Failure(shardId.getIndexName(), "type", "id" + i,
+                                        new VersionConflictEngineException(shardId, "type", "id", "test")));
                     continue;
                 }
                 boolean createdResponse;
+                DocWriteRequest.OpType opType;
                 switch (randomIntBetween(0, 2)) {
                 case 0:
-                    opType = randomFrom("index", "create");
                     createdResponse = true;
+                    opType = DocWriteRequest.OpType.CREATE;
                     created++;
                     break;
                 case 1:
-                    opType = randomFrom("index", "create");
                     createdResponse = false;
+                    opType = randomFrom(DocWriteRequest.OpType.INDEX, DocWriteRequest.OpType.UPDATE);
                     updated++;
                     break;
                 case 2:
-                    opType = "delete";
                     createdResponse = false;
+                    opType = DocWriteRequest.OpType.DELETE;
                     deleted++;
                     break;
                 default:
                     throw new RuntimeException("Bad scenario");
                 }
-                responses[i] = new BulkItemResponse(i, opType, new IndexResponse(shardId, "type", "id" + i, randomInt(), createdResponse));
+                responses[i] = new BulkItemResponse(i, opType,
+                        new IndexResponse(shardId, "type", "id" + i, randomInt(), createdResponse));
             }
-            new DummyAbstractAsyncBulkByScrollAction().onBulkResponse(timeValueNanos(System.nanoTime()), new BulkResponse(responses, 0));
+            new DummyAsyncBulkByScrollAction().onBulkResponse(timeValueNanos(System.nanoTime()), new BulkResponse(responses, 0));
             assertEquals(versionConflicts, testTask.getStatus().getVersionConflicts());
             assertEquals(updated, testTask.getStatus().getUpdated());
             assertEquals(created, testTask.getStatus().getCreated());
@@ -301,8 +308,7 @@ public void testBulkResponseSetsLotsOfStatus() {
      */
     public void testThreadPoolRejectionsAbortRequest() throws Exception {
         testTask.rethrottle(1);
-        threadPool.shutdown();
-        threadPool = new TestThreadPool(getTestName()) {
+        setupClient(new TestThreadPool(getTestName()) {
             @Override
             public ScheduledFuture<?> schedule(TimeValue delay, String name, Runnable command) {
                 // While we're here we can check that the sleep made it through
@@ -311,9 +317,9 @@ public ScheduledFuture<?> schedule(TimeValue delay, String name, Runnable comman
                 ((AbstractRunnable) command).onRejection(new EsRejectedExecutionException("test"));
                 return null;
             }
-        };
+        });
         ScrollableHitSource.Response response = new ScrollableHitSource.Response(false, emptyList(), 0, emptyList(), null);
-        simulateScrollResponse(new DummyAbstractAsyncBulkByScrollAction(), timeValueNanos(System.nanoTime()), 10, response);
+        simulateScrollResponse(new DummyAsyncBulkByScrollAction(), timeValueNanos(System.nanoTime()), 10, response);
         ExecutionException e = expectThrows(ExecutionException.class, () -> listener.get());
         assertThat(e.getMessage(), equalTo("EsRejectedExecutionException[test]"));
         assertThat(client.scrollsCleared, contains(scrollId));
@@ -330,8 +336,8 @@ public void testShardFailuresAbortRequest() throws Exception {
         SearchFailure shardFailure = new SearchFailure(new RuntimeException("test"));
         ScrollableHitSource.Response scrollResponse = new ScrollableHitSource.Response(false, singletonList(shardFailure), 0,
                 emptyList(), null);
-        simulateScrollResponse(new DummyAbstractAsyncBulkByScrollAction(), timeValueNanos(System.nanoTime()), 0, scrollResponse);
-        BulkIndexByScrollResponse response = listener.get();
+        simulateScrollResponse(new DummyAsyncBulkByScrollAction(), timeValueNanos(System.nanoTime()), 0, scrollResponse);
+        BulkByScrollResponse response = listener.get();
         assertThat(response.getBulkFailures(), empty());
         assertThat(response.getSearchFailures(), contains(shardFailure));
         assertFalse(response.isTimedOut());
@@ -344,8 +350,8 @@ public void testShardFailuresAbortRequest() throws Exception {
      */
     public void testSearchTimeoutsAbortRequest() throws Exception {
         ScrollableHitSource.Response scrollResponse = new ScrollableHitSource.Response(true, emptyList(), 0, emptyList(), null);
-        simulateScrollResponse(new DummyAbstractAsyncBulkByScrollAction(), timeValueNanos(System.nanoTime()), 0, scrollResponse);
-        BulkIndexByScrollResponse response = listener.get();
+        simulateScrollResponse(new DummyAsyncBulkByScrollAction(), timeValueNanos(System.nanoTime()), 0, scrollResponse);
+        BulkByScrollResponse response = listener.get();
         assertThat(response.getBulkFailures(), empty());
         assertThat(response.getSearchFailures(), empty());
         assertTrue(response.isTimedOut());
@@ -358,10 +364,11 @@ public void testSearchTimeoutsAbortRequest() throws Exception {
      */
     public void testBulkFailuresAbortRequest() throws Exception {
         Failure failure = new Failure("index", "type", "id", new RuntimeException("test"));
-        DummyAbstractAsyncBulkByScrollAction action = new DummyAbstractAsyncBulkByScrollAction();
-        BulkResponse bulkResponse = new BulkResponse(new BulkItemResponse[] {new BulkItemResponse(0, "index", failure)}, randomLong());
+        DummyAsyncBulkByScrollAction action = new DummyAsyncBulkByScrollAction();
+        BulkResponse bulkResponse = new BulkResponse(new BulkItemResponse[]
+            {new BulkItemResponse(0, DocWriteRequest.OpType.CREATE, failure)}, randomLong());
         action.onBulkResponse(timeValueNanos(System.nanoTime()), bulkResponse);
-        BulkIndexByScrollResponse response = listener.get();
+        BulkByScrollResponse response = listener.get();
         assertThat(response.getBulkFailures(), contains(failure));
         assertThat(response.getSearchFailures(), empty());
         assertNull(response.getReasonCancelled());
@@ -370,14 +377,15 @@ public void testBulkFailuresAbortRequest() throws Exception {
     /**
      * Mimicks script failures or general wrongness by implementers.
      */
-    public void testListenerReceiveBuildBulkExceptions() throws Exception {
-        DummyAbstractAsyncBulkByScrollAction action = new DummyAbstractAsyncBulkByScrollAction() {
+    public void testBuildRequestThrowsException() throws Exception {
+        DummyAsyncBulkByScrollAction action = new DummyAsyncBulkByScrollAction() {
             @Override
-            protected BulkRequest buildBulk(Iterable<? extends ScrollableHitSource.Hit> docs) {
+            protected AbstractAsyncBulkByScrollAction.RequestWrapper<?> buildRequest(Hit doc) {
                 throw new RuntimeException("surprise");
             }
         };
-        Hit hit = new ScrollableHitSource.BasicHit("index", "type", "id", 0);
+        ScrollableHitSource.BasicHit hit = new ScrollableHitSource.BasicHit("index", "type", "id", 0);
+        hit.setSource(new BytesArray("{}"), XContentType.JSON);
         ScrollableHitSource.Response response = new ScrollableHitSource.Response(false, emptyList(), 1, singletonList(hit), null);
         simulateScrollResponse(action, timeValueNanos(System.nanoTime()), 0, response);
         ExecutionException e = expectThrows(ExecutionException.class, () -> listener.get());
@@ -413,17 +421,16 @@ public void testScrollDelay() throws Exception {
          */
         AtomicReference<TimeValue> capturedDelay = new AtomicReference<>();
         AtomicReference<Runnable> capturedCommand = new AtomicReference<>();
-        threadPool.shutdown();
-        threadPool = new TestThreadPool(getTestName()) {
+        setupClient(new TestThreadPool(getTestName()) {
             @Override
             public ScheduledFuture<?> schedule(TimeValue delay, String name, Runnable command) {
                 capturedDelay.set(delay);
                 capturedCommand.set(command);
                 return null;
             }
-        };
+        });
 
-        DummyAbstractAsyncBulkByScrollAction action = new DummyAbstractAsyncBulkByScrollAction();
+        DummyAsyncBulkByScrollAction action = new DummyAsyncBulkByScrollAction();
         action.setScroll(scrollId());
 
         // Set the base for the scroll to wait - this is added to the figure we calculate below
@@ -432,27 +439,28 @@ public ScheduledFuture<?> schedule(TimeValue delay, String name, Runnable comman
         // Set throttle to 1 request per second to make the math simpler
         testTask.rethrottle(1f);
         // Make the last batch look nearly instant but have 100 documents
-        action.startNextScroll(timeValueNanos(System.nanoTime()), 100);
+        TimeValue lastBatchStartTime = timeValueNanos(System.nanoTime());
+        TimeValue now = timeValueNanos(lastBatchStartTime.nanos() + 1);
+        action.startNextScroll(lastBatchStartTime, now, 100);
 
         // So the next request is going to have to wait an extra 100 seconds or so (base was 10 seconds, so 110ish)
         assertThat(client.lastScroll.get().request.scroll().keepAlive().seconds(), either(equalTo(110L)).or(equalTo(109L)));
 
         // Now we can simulate a response and check the delay that we used for the task
-        InternalSearchHit hit = new InternalSearchHit(0, "id", new Text("type"), emptyMap());
-        InternalSearchHits hits = new InternalSearchHits(new InternalSearchHit[] { hit }, 0, 0);
-        InternalSearchResponse internalResponse = new InternalSearchResponse(hits, null, null, null, false, false);
-        SearchResponse searchResponse = new SearchResponse(internalResponse, scrollId(), 5, 4, randomLong(), null);
+        SearchHit hit = new SearchHit(0, "id", new Text("type"), emptyMap());
+        SearchHits hits = new SearchHits(new SearchHit[] { hit }, 0, 0);
+        InternalSearchResponse internalResponse = new InternalSearchResponse(hits, null, null, null, false, false, 1);
+        SearchResponse searchResponse = new SearchResponse(internalResponse, scrollId(), 5, 4, 0, randomLong(), null);
 
         if (randomBoolean()) {
             client.lastScroll.get().listener.onResponse(searchResponse);
-            // The delay is still 100ish seconds because there hasn't been much time between when we requested the bulk and when we got it.
-            assertThat(capturedDelay.get().seconds(), either(equalTo(100L)).or(equalTo(99L)));
+            assertEquals(99, capturedDelay.get().seconds());
         } else {
             // Let's rethrottle between the starting the scroll and getting the response
             testTask.rethrottle(10f);
             client.lastScroll.get().listener.onResponse(searchResponse);
             // The delay uses the new throttle
-            assertThat(capturedDelay.get().seconds(), either(equalTo(10L)).or(equalTo(9L)));
+            assertEquals(9, capturedDelay.get().seconds());
         }
 
         // Running the command ought to increment the delay counter on the task.
@@ -475,9 +483,9 @@ private void bulkRetryTestCase(boolean failWithRejection) throws Exception {
          * deal with it. We just wait for it to happen.
          */
         CountDownLatch successLatch = new CountDownLatch(1);
-        DummyAbstractAsyncBulkByScrollAction action = new DummyActionWithoutBackoff() {
+        DummyAsyncBulkByScrollAction action = new DummyActionWithoutBackoff() {
             @Override
-            void startNextScroll(TimeValue lastBatchStartTime, int lastBatchSize) {
+            void startNextScroll(TimeValue lastBatchStartTime, TimeValue now, int lastBatchSize) {
                 successLatch.countDown();
             }
         };
@@ -487,13 +495,13 @@ void startNextScroll(TimeValue lastBatchStartTime, int lastBatchSize) {
         }
         action.sendBulkRequest(timeValueNanos(System.nanoTime()), request);
         if (failWithRejection) {
-            BulkIndexByScrollResponse response = listener.get();
+            BulkByScrollResponse response = listener.get();
             assertThat(response.getBulkFailures(), hasSize(1));
             assertEquals(response.getBulkFailures().get(0).getStatus(), RestStatus.TOO_MANY_REQUESTS);
             assertThat(response.getSearchFailures(), empty());
             assertNull(response.getReasonCancelled());
         } else {
-            successLatch.await(10, TimeUnit.SECONDS);
+            assertTrue(successLatch.await(10, TimeUnit.SECONDS));
         }
     }
 
@@ -501,7 +509,7 @@ void startNextScroll(TimeValue lastBatchStartTime, int lastBatchSize) {
      * The default retry time matches what we say it is in the javadoc for the request.
      */
     public void testDefaultRetryTimes() {
-        Iterator<TimeValue> policy = new DummyAbstractAsyncBulkByScrollAction().buildBackoffPolicy().iterator();
+        Iterator<TimeValue> policy = new DummyAsyncBulkByScrollAction().buildBackoffPolicy().iterator();
         long millis = 0;
         while (policy.hasNext()) {
             millis += policy.next().millis();
@@ -534,7 +542,7 @@ private void refreshTestCase(Boolean refresh, boolean addDestinationIndexes, boo
         if (refresh != null) {
             testRequest.setRefresh(refresh);
         }
-        DummyAbstractAsyncBulkByScrollAction action = new DummyAbstractAsyncBulkByScrollAction();
+        DummyAsyncBulkByScrollAction action = new DummyAsyncBulkByScrollAction();
         if (addDestinationIndexes) {
             action.addDestinationIndices(singleton("foo"));
         }
@@ -547,34 +555,33 @@ private void refreshTestCase(Boolean refresh, boolean addDestinationIndexes, boo
     }
 
     public void testCancelBeforeInitialSearch() throws Exception {
-        cancelTaskCase((DummyAbstractAsyncBulkByScrollAction action) -> action.start());
+        cancelTaskCase((DummyAsyncBulkByScrollAction action) -> action.start());
     }
 
     public void testCancelBeforeScrollResponse() throws Exception {
-        // We bail so early we don't need to pass in a half way valid response.
-        cancelTaskCase((DummyAbstractAsyncBulkByScrollAction action) -> simulateScrollResponse(action, timeValueNanos(System.nanoTime()), 1,
-                null));
+        cancelTaskCase((DummyAsyncBulkByScrollAction action) -> simulateScrollResponse(action, timeValueNanos(System.nanoTime()), 1,
+                new ScrollableHitSource.Response(false, emptyList(), between(1, 100000), emptyList(), null)));
     }
 
     public void testCancelBeforeSendBulkRequest() throws Exception {
-        // We bail so early we don't need to pass in a half way valid request.
-        cancelTaskCase((DummyAbstractAsyncBulkByScrollAction action) -> action.sendBulkRequest(timeValueNanos(System.nanoTime()), null));
+        cancelTaskCase((DummyAsyncBulkByScrollAction action) ->
+            action.sendBulkRequest(timeValueNanos(System.nanoTime()), new BulkRequest()));
     }
 
     public void testCancelBeforeOnBulkResponse() throws Exception {
-        // We bail so early we don't need to pass in a half way valid response.
-        cancelTaskCase((DummyAbstractAsyncBulkByScrollAction action) ->
+        cancelTaskCase((DummyAsyncBulkByScrollAction action) ->
                 action.onBulkResponse(timeValueNanos(System.nanoTime()), new BulkResponse(new BulkItemResponse[0], 0)));
     }
 
     public void testCancelBeforeStartNextScroll() throws Exception {
-        cancelTaskCase((DummyAbstractAsyncBulkByScrollAction action) -> action.startNextScroll(timeValueNanos(System.nanoTime()), 0));
+        TimeValue now = timeValueNanos(System.nanoTime());
+        cancelTaskCase((DummyAsyncBulkByScrollAction action) -> action.startNextScroll(now, now, 0));
     }
 
     public void testCancelBeforeRefreshAndFinish() throws Exception {
         // Refresh or not doesn't matter - we don't try to refresh.
         testRequest.setRefresh(usually());
-        cancelTaskCase((DummyAbstractAsyncBulkByScrollAction action) -> action.refreshAndFinish(emptyList(), emptyList(), false));
+        cancelTaskCase((DummyAsyncBulkByScrollAction action) -> action.refreshAndFinish(emptyList(), emptyList(), false));
         assertNull("No refresh was attempted", client.lastRefreshRequest.get());
     }
 
@@ -589,8 +596,7 @@ public void testCancelWhileDelayedAfterScrollResponse() throws Exception {
          * Replace the thread pool with one that will cancel the task as soon as anything is scheduled, which reindex tries to do when there
          * is a delay.
          */
-        threadPool.shutdown();
-        threadPool = new TestThreadPool(getTestName()) {
+        setupClient(new TestThreadPool(getTestName()) {
             @Override
             public ScheduledFuture<?> schedule(TimeValue delay, String name, Runnable command) {
                 /*
@@ -601,14 +607,14 @@ public ScheduledFuture<?> schedule(TimeValue delay, String name, Runnable comman
                  * that good stuff.
                  */
                 if (delay.nanos() > 0) {
-                    generic().execute(() -> taskManager.cancel(testTask, reason, (Set<String> s) -> {}));
+                    generic().execute(() -> taskManager.cancel(testTask, reason, () -> {}));
                 }
                 return super.schedule(delay, name, command);
             }
-        };
+        });
 
         // Send the scroll response which will trigger the custom thread pool above, canceling the request before running the response
-        DummyAbstractAsyncBulkByScrollAction action = new DummyAbstractAsyncBulkByScrollAction();
+        DummyAsyncBulkByScrollAction action = new DummyAsyncBulkByScrollAction();
         boolean previousScrollSet = usually();
         if (previousScrollSet) {
             action.setScroll(scrollId());
@@ -627,14 +633,14 @@ public ScheduledFuture<?> schedule(TimeValue delay, String name, Runnable comman
         }
     }
 
-    private void cancelTaskCase(Consumer<DummyAbstractAsyncBulkByScrollAction> testMe) throws Exception {
-        DummyAbstractAsyncBulkByScrollAction action = new DummyAbstractAsyncBulkByScrollAction();
+    private void cancelTaskCase(Consumer<DummyAsyncBulkByScrollAction> testMe) throws Exception {
+        DummyAsyncBulkByScrollAction action = new DummyAsyncBulkByScrollAction();
         boolean previousScrollSet = usually();
         if (previousScrollSet) {
             action.setScroll(scrollId());
         }
         String reason = randomSimpleString(random());
-        taskManager.cancel(testTask, reason, (Set<String> s) -> {});
+        taskManager.cancel(testTask, reason, () -> {});
         testMe.accept(action);
         assertEquals(reason, listener.get().getReasonCancelled());
         if (previousScrollSet) {
@@ -646,17 +652,16 @@ private void cancelTaskCase(Consumer<DummyAbstractAsyncBulkByScrollAction> testM
     /**
      * Simulate a scroll response by setting the scroll id and firing the onScrollResponse method.
      */
-    private void simulateScrollResponse(DummyAbstractAsyncBulkByScrollAction action, TimeValue lastBatchTime, int lastBatchSize,
+    private void simulateScrollResponse(DummyAsyncBulkByScrollAction action, TimeValue lastBatchTime, int lastBatchSize,
             ScrollableHitSource.Response response) {
         action.setScroll(scrollId());
         action.onScrollResponse(lastBatchTime, lastBatchSize, response);
     }
 
-    private class DummyAbstractAsyncBulkByScrollAction
-            extends AbstractAsyncBulkByScrollAction<DummyAbstractBulkByScrollRequest> {
-        public DummyAbstractAsyncBulkByScrollAction() {
+    private class DummyAsyncBulkByScrollAction extends AbstractAsyncBulkByScrollAction<DummyAbstractBulkByScrollRequest> {
+        DummyAsyncBulkByScrollAction() {
             super(testTask, AsyncBulkByScrollActionTests.this.logger, new ParentTaskAssigningClient(client, localNode, testTask),
-                    AsyncBulkByScrollActionTests.this.threadPool, testRequest, listener);
+                    client.threadPool(), testRequest, null, null, listener, Settings.EMPTY);
         }
 
         @Override
@@ -665,15 +670,15 @@ protected boolean needsSourceDocumentVersions() {
         }
 
         @Override
-        protected BulkRequest buildBulk(Iterable<? extends ScrollableHitSource.Hit> docs) {
-            return new BulkRequest();
+        protected AbstractAsyncBulkByScrollAction.RequestWrapper<?> buildRequest(Hit doc) {
+            throw new UnsupportedOperationException("Use another override to test this.");
         }
     }
 
     /**
-     * An extension to {@linkplain DummyAbstractAsyncBulkByScrollAction} that uses a 0 delaying backoff policy.
+     * An extension to {@linkplain DummyAsyncBulkByScrollAction} that uses a 0 delaying backoff policy.
      */
-    private class DummyActionWithoutBackoff extends DummyAbstractAsyncBulkByScrollAction {
+    private class DummyActionWithoutBackoff extends DummyAsyncBulkByScrollAction {
         @Override
         BackoffPolicy buildBackoffPolicy() {
             // Force a backoff time of 0 to prevent sleeping
@@ -682,8 +687,13 @@ BackoffPolicy buildBackoffPolicy() {
     }
 
     private static class DummyAbstractBulkByScrollRequest extends AbstractBulkByScrollRequest<DummyAbstractBulkByScrollRequest> {
-        public DummyAbstractBulkByScrollRequest(SearchRequest searchRequest) {
-            super(searchRequest);
+        DummyAbstractBulkByScrollRequest(SearchRequest searchRequest) {
+            super(searchRequest, true);
+        }
+
+        @Override
+        public DummyAbstractBulkByScrollRequest forSlice(TaskId slicingTask, SearchRequest slice) {
+            throw new UnsupportedOperationException();
         }
 
         @Override
@@ -697,7 +707,6 @@ private class MyMockClient extends FilterClient {
         private final AtomicInteger bulksAttempts = new AtomicInteger();
         private final AtomicInteger searchAttempts = new AtomicInteger();
         private final AtomicInteger scrollAttempts = new AtomicInteger();
-        private final AtomicReference<Map<String, String>> lastHeaders = new AtomicReference<>();
         private final AtomicReference<RefreshRequest> lastRefreshRequest = new AtomicReference<>();
         /**
          * Last search attempt that wasn't rejected outright.
@@ -707,22 +716,30 @@ private class MyMockClient extends FilterClient {
          * Last scroll attempt that wasn't rejected outright.
          */
         private final AtomicReference<RequestAndListener<SearchScrollRequest, SearchResponse>> lastScroll = new AtomicReference<>();
-
+        /**
+         * Set of all scrolls we've already used. Used to check that we don't reuse the same request twice.
+         */
+        private final Set<SearchScrollRequest> usedScolls = synchronizedSet(newSetFromMap(new IdentityHashMap<>()));
 
         private int bulksToReject = 0;
         private int searchesToReject = 0;
         private int scrollsToReject = 0;
 
-        public MyMockClient(Client in) {
+        MyMockClient(Client in) {
             super(in);
         }
 
         @Override
         @SuppressWarnings("unchecked")
-        protected <Request extends ActionRequest<Request>, Response extends ActionResponse,
+        protected <Request extends ActionRequest, Response extends ActionResponse,
                 RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> void doExecute(
                 Action<Request, Response, RequestBuilder> action, Request request, ActionListener<Response> listener) {
-            lastHeaders.set(threadPool.getThreadContext().getHeaders());
+            if (false == expectedHeaders.equals(threadPool().getThreadContext().getHeaders())) {
+                listener.onFailure(
+                        new RuntimeException("Expected " + expectedHeaders + " but got " + threadPool().getThreadContext().getHeaders()));
+                return;
+            }
+
             if (request instanceof ClearScrollRequest) {
                 assertEquals(TaskId.EMPTY_TASK_ID, request.getParentTask());
             } else {
@@ -742,11 +759,14 @@ RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>>
                 return;
             }
             if (request instanceof SearchScrollRequest) {
+                SearchScrollRequest scroll = (SearchScrollRequest) request;
+                boolean newRequest = usedScolls.add(scroll);
+                assertTrue("We can't reuse scroll requests", newRequest);
                 if (scrollAttempts.incrementAndGet() <= scrollsToReject) {
                     listener.onFailure(wrappedRejectedException());
                     return;
                 }
-                lastScroll.set(new RequestAndListener<>((SearchScrollRequest) request, (ActionListener<SearchResponse>) listener));
+                lastScroll.set(new RequestAndListener<>(scroll, (ActionListener<SearchResponse>) listener));
                 return;
             }
             if (request instanceof ClearScrollRequest) {
@@ -765,33 +785,29 @@ RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>>
                 }
                 BulkItemResponse[] responses = new BulkItemResponse[bulk.requests().size()];
                 for (int i = 0; i < bulk.requests().size(); i++) {
-                    ActionRequest<?> item = bulk.requests().get(i);
-                    String opType;
+                    DocWriteRequest<?> item = bulk.requests().get(i);
                     DocWriteResponse response;
-                    ShardId shardId = new ShardId(new Index(((ReplicationRequest<?>) item).index(), "uuid"), 0);
+                    ShardId shardId = new ShardId(new Index(item.index(), "uuid"), 0);
                     if (item instanceof IndexRequest) {
                         IndexRequest index = (IndexRequest) item;
-                        opType = index.opType().lowercase();
                         response = new IndexResponse(shardId, index.type(), index.id(), randomIntBetween(0, Integer.MAX_VALUE),
                                 true);
                     } else if (item instanceof UpdateRequest) {
                         UpdateRequest update = (UpdateRequest) item;
-                        opType = "update";
                         response = new UpdateResponse(shardId, update.type(), update.id(),
-                                randomIntBetween(0, Integer.MAX_VALUE), DocWriteResponse.Result.CREATED);
+                                randomIntBetween(0, Integer.MAX_VALUE), Result.CREATED);
                     } else if (item instanceof DeleteRequest) {
                         DeleteRequest delete = (DeleteRequest) item;
-                        opType = "delete";
                         response = new DeleteResponse(shardId, delete.type(), delete.id(), randomIntBetween(0, Integer.MAX_VALUE),
                                 true);
                     } else {
                         throw new RuntimeException("Unknown request:  " + item);
                     }
                     if (i == toReject) {
-                        responses[i] = new BulkItemResponse(i, opType,
+                        responses[i] = new BulkItemResponse(i, item.opType(),
                                 new Failure(response.getIndex(), response.getType(), response.getId(), new EsRejectedExecutionException()));
                     } else {
-                        responses[i] = new BulkItemResponse(i, opType, response);
+                        responses[i] = new BulkItemResponse(i, item.opType(), response);
                     }
                 }
                 listener.onResponse((Response) new BulkResponse(responses, 1));
@@ -820,11 +836,11 @@ private Exception wrappedRejectedException() {
         }
     }
 
-    private static class RequestAndListener<Request extends ActionRequest<Request>, Response> {
+    private static class RequestAndListener<Request extends ActionRequest, Response> {
         private final Request request;
         private final ActionListener<Response> listener;
 
-        public RequestAndListener(Request request, ActionListener<Response> listener) {
+        RequestAndListener(Request request, ActionListener<Response> listener) {
             this.request = request;
             this.listener = listener;
         }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelperTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelperTests.java
new file mode 100644
index 0000000000000..a64415d08b1b1
--- /dev/null
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelperTests.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex;
+
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.index.mapper.IdFieldMapper;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+import static java.util.Collections.emptyList;
+import static org.elasticsearch.index.reindex.BulkByScrollParallelizationHelper.sliceIntoSubRequests;
+import static org.elasticsearch.search.RandomSearchRequestGenerator.randomSearchRequest;
+import static org.elasticsearch.search.RandomSearchRequestGenerator.randomSearchSourceBuilder;
+
+public class BulkByScrollParallelizationHelperTests extends ESTestCase {
+    public void testSliceIntoSubRequests() throws IOException {
+        SearchRequest searchRequest = randomSearchRequest(() -> randomSearchSourceBuilder(
+                () -> null,
+                () -> null,
+                () -> null,
+                () -> emptyList(),
+                () -> null));
+        if (searchRequest.source() != null) {
+            // Clear the slice builder if there is one set. We can't call sliceIntoSubRequests if it is.
+            searchRequest.source().slice(null);
+        }
+        int times = between(2, 100);
+        String field = randomBoolean() ? IdFieldMapper.NAME : randomAlphaOfLength(5);
+        int currentSliceId = 0;
+        for (SearchRequest slice : sliceIntoSubRequests(searchRequest, field, times)) {
+            assertEquals(field, slice.source().slice().getField());
+            assertEquals(currentSliceId, slice.source().slice().getId());
+            assertEquals(times, slice.source().slice().getMax());
+
+            // If you clear the slice then the slice should be the same request as the parent request
+            slice.source().slice(null);
+            if (searchRequest.source() == null) {
+                // Except that adding the slice might have added an empty builder
+                searchRequest.source(new SearchSourceBuilder());
+            }
+            assertEquals(searchRequest, slice);
+            currentSliceId++;
+        }
+    }
+}
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskTests.java
deleted file mode 100644
index 72c650805ce49..0000000000000
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskTests.java
+++ /dev/null
@@ -1,300 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.reindex;
-
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.util.concurrent.AbstractRunnable;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.json.JsonXContent;
-import org.elasticsearch.tasks.TaskId;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.threadpool.TestThreadPool;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.junit.Before;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.CyclicBarrier;
-import java.util.concurrent.Delayed;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import static org.elasticsearch.common.unit.TimeValue.parseTimeValue;
-import static org.elasticsearch.common.unit.TimeValue.timeValueNanos;
-import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
-import static org.hamcrest.Matchers.both;
-import static org.hamcrest.Matchers.closeTo;
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.empty;
-import static org.hamcrest.Matchers.greaterThanOrEqualTo;
-import static org.hamcrest.Matchers.lessThanOrEqualTo;
-
-public class BulkByScrollTaskTests extends ESTestCase {
-    private BulkByScrollTask task;
-
-    @Before
-    public void createTask() {
-        task = new BulkByScrollTask(1, "test_type", "test_action", "test", TaskId.EMPTY_TASK_ID, Float.POSITIVE_INFINITY);
-    }
-
-    public void testBasicData() {
-        assertEquals(1, task.getId());
-        assertEquals("test_type", task.getType());
-        assertEquals("test_action", task.getAction());
-    }
-
-    public void testProgress() {
-        long created = 0;
-        long updated = 0;
-        long deleted = 0;
-        long versionConflicts = 0;
-        long noops = 0;
-        int batch = 0;
-        BulkByScrollTask.Status status = task.getStatus();
-        assertEquals(0, status.getTotal());
-        assertEquals(created, status.getCreated());
-        assertEquals(updated, status.getUpdated());
-        assertEquals(deleted, status.getDeleted());
-        assertEquals(versionConflicts, status.getVersionConflicts());
-        assertEquals(batch, status.getBatches());
-        assertEquals(noops, status.getNoops());
-
-        long totalHits = randomIntBetween(10, 1000);
-        task.setTotal(totalHits);
-        for (long p = 0; p < totalHits; p++) {
-            status = task.getStatus();
-            assertEquals(totalHits, status.getTotal());
-            assertEquals(created, status.getCreated());
-            assertEquals(updated, status.getUpdated());
-            assertEquals(deleted, status.getDeleted());
-            assertEquals(versionConflicts, status.getVersionConflicts());
-            assertEquals(batch, status.getBatches());
-            assertEquals(noops, status.getNoops());
-
-            if (randomBoolean()) {
-                created++;
-                task.countCreated();
-            } else if (randomBoolean()) {
-                updated++;
-                task.countUpdated();
-            } else {
-                deleted++;
-                task.countDeleted();
-            }
-
-            if (rarely()) {
-                versionConflicts++;
-                task.countVersionConflict();
-            }
-
-            if (rarely()) {
-                batch++;
-                task.countBatch();
-            }
-
-            if (rarely()) {
-                noops++;
-                task.countNoop();
-            }
-        }
-        status = task.getStatus();
-        assertEquals(totalHits, status.getTotal());
-        assertEquals(created, status.getCreated());
-        assertEquals(updated, status.getUpdated());
-        assertEquals(deleted, status.getDeleted());
-        assertEquals(versionConflicts, status.getVersionConflicts());
-        assertEquals(batch, status.getBatches());
-        assertEquals(noops, status.getNoops());
-    }
-
-    public void testStatusHatesNegatives() {
-        checkStatusNegatives(-1, 0, 0, 0, 0, 0, 0, 0, 0, "total");
-        checkStatusNegatives(0, -1, 0, 0, 0, 0, 0, 0, 0, "updated");
-        checkStatusNegatives(0, 0, -1, 0, 0, 0, 0, 0, 0, "created");
-        checkStatusNegatives(0, 0, 0, -1, 0, 0, 0, 0, 0, "deleted");
-        checkStatusNegatives(0, 0, 0, 0, -1, 0, 0, 0, 0, "batches");
-        checkStatusNegatives(0, 0, 0, 0, 0, -1, 0, 0, 0, "versionConflicts");
-        checkStatusNegatives(0, 0, 0, 0, 0, 0, -1, 0, 0, "noops");
-        checkStatusNegatives(0, 0, 0, 0, 0, 0, 0, -1, 0, "bulkRetries");
-        checkStatusNegatives(0, 0, 0, 0, 0, 0, 0, 0, -1, "searchRetries");
-    }
-
-    /**
-     * Build a task status with only some values. Used for testing negative values.
-     */
-    private void checkStatusNegatives(long total, long updated, long created, long deleted, int batches, long versionConflicts,
-            long noops, long bulkRetries, long searchRetries, String fieldName) {
-        TimeValue throttle = parseTimeValue(randomPositiveTimeValue(), "test");
-        TimeValue throttledUntil = parseTimeValue(randomPositiveTimeValue(), "test");
-
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new BulkByScrollTask.Status(total, updated, created,
-                deleted, batches, versionConflicts, noops, bulkRetries, searchRetries, throttle, 0f, null, throttledUntil));
-        assertEquals(e.getMessage(), fieldName + " must be greater than 0 but was [-1]");
-    }
-
-    /**
-     * Furiously rethrottles a delayed request to make sure that we never run it twice.
-     */
-    public void testDelayAndRethrottle() throws IOException, InterruptedException {
-        List<Throwable> errors = new CopyOnWriteArrayList<>();
-        AtomicBoolean done = new AtomicBoolean();
-        int threads = between(1, 10);
-        CyclicBarrier waitForShutdown = new CyclicBarrier(threads);
-
-        /*
-         * We never end up waiting this long because the test rethrottles over and over again, ratcheting down the delay a random amount
-         * each time.
-         */
-        float originalRequestsPerSecond = (float) randomDoubleBetween(1, 10000, true);
-        task.rethrottle(originalRequestsPerSecond);
-        TimeValue maxDelay = timeValueSeconds(between(1, 5));
-        assertThat(maxDelay.nanos(), greaterThanOrEqualTo(0L));
-        int batchSizeForMaxDelay = (int) (maxDelay.seconds() * originalRequestsPerSecond);
-        ThreadPool threadPool = new TestThreadPool(getTestName()) {
-            @Override
-            public ScheduledFuture<?> schedule(TimeValue delay, String name, Runnable command) {
-                assertThat(delay.nanos(), both(greaterThanOrEqualTo(0L)).and(lessThanOrEqualTo(maxDelay.nanos())));
-                return super.schedule(delay, name, command);
-            }
-        };
-        try {
-            task.delayPrepareBulkRequest(threadPool, timeValueNanos(System.nanoTime()), batchSizeForMaxDelay, new AbstractRunnable() {
-                @Override
-                protected void doRun() throws Exception {
-                    boolean oldValue = done.getAndSet(true);
-                    if (oldValue) {
-                        throw new RuntimeException("Ran twice oh no!");
-                    }
-                }
-
-                @Override
-                public void onFailure(Exception e) {
-                    errors.add(e);
-                }
-            });
-
-            // Rethrottle on a random number of threads, on of which is this thread.
-            Runnable test = () -> {
-                try {
-                    int rethrottles = 0;
-                    while (false == done.get()) {
-                        float requestsPerSecond = (float) randomDoubleBetween(0, originalRequestsPerSecond * 2, true);
-                        task.rethrottle(requestsPerSecond);
-                        rethrottles += 1;
-                    }
-                    logger.info("Rethrottled [{}] times", rethrottles);
-                    waitForShutdown.await();
-                } catch (Exception e) {
-                    errors.add(e);
-                }
-            };
-            for (int i = 1; i < threads; i++) {
-                threadPool.generic().execute(test);
-            }
-            test.run();
-        } finally {
-            // Other threads should finish up quickly as they are checking the same AtomicBoolean.
-            threadPool.shutdown();
-            threadPool.awaitTermination(10, TimeUnit.SECONDS);
-        }
-        assertThat(errors, empty());
-    }
-
-    public void testDelayNeverNegative() throws IOException {
-        // Thread pool that returns a ScheduledFuture that claims to have a negative delay
-        ThreadPool threadPool = new TestThreadPool("test") {
-            public ScheduledFuture<?> schedule(TimeValue delay, String name, Runnable command) {
-                return new ScheduledFuture<Void>() {
-                    @Override
-                    public long getDelay(TimeUnit unit) {
-                        return -1;
-                    }
-
-                    @Override
-                    public int compareTo(Delayed o) {
-                        throw new UnsupportedOperationException();
-                    }
-
-                    @Override
-                    public boolean cancel(boolean mayInterruptIfRunning) {
-                        throw new UnsupportedOperationException();
-                    }
-
-                    @Override
-                    public boolean isCancelled() {
-                        throw new UnsupportedOperationException();
-                    }
-
-                    @Override
-                    public boolean isDone() {
-                        throw new UnsupportedOperationException();
-                    }
-
-                    @Override
-                    public Void get() throws InterruptedException, ExecutionException {
-                        throw new UnsupportedOperationException();
-                    }
-
-                    @Override
-                    public Void get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
-                        throw new UnsupportedOperationException();
-                    }
-                };
-            }
-        };
-        try {
-            // Have the task use the thread pool to delay a task that does nothing
-            task.delayPrepareBulkRequest(threadPool, timeValueSeconds(0), 1, new AbstractRunnable() {
-                @Override
-                protected void doRun() throws Exception {
-                }
-                @Override
-                public void onFailure(Exception e) {
-                    throw new UnsupportedOperationException();
-                }
-            });
-            // Even though the future returns a negative delay we just return 0 because the time is up.
-            assertEquals(timeValueSeconds(0), task.getStatus().getThrottledUntil());
-        } finally {
-            threadPool.shutdown();
-        }
-    }
-
-    public void testXContentRepresentationOfUnlimitedRequestsPerSecon() throws IOException {
-        XContentBuilder builder = JsonXContent.contentBuilder();
-        task.getStatus().toXContent(builder, ToXContent.EMPTY_PARAMS);
-        assertThat(builder.string(), containsString("\"requests_per_second\":-1"));
-    }
-
-    public void testPerfectlyThrottledBatchTime() {
-        task.rethrottle(Float.POSITIVE_INFINITY);
-        assertThat((double) task.perfectlyThrottledBatchTime(randomInt()), closeTo(0f, 0f));
-
-        int total = between(0, 1000000);
-        task.rethrottle(1);
-        assertThat((double) task.perfectlyThrottledBatchTime(total),
-                closeTo(TimeUnit.SECONDS.toNanos(total), TimeUnit.SECONDS.toNanos(1)));
-    }
-}
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseMatcher.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseMatcher.java
index c0c06b14d5534..cb2ff1a7ae24a 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseMatcher.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseMatcher.java
@@ -23,10 +23,13 @@
 import org.hamcrest.Matcher;
 import org.hamcrest.TypeSafeMatcher;
 
+import java.util.Collection;
+
+import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.nullValue;
 
-public class BulkIndexByScrollResponseMatcher extends TypeSafeMatcher<BulkIndexByScrollResponse> {
+public class BulkIndexByScrollResponseMatcher extends TypeSafeMatcher<BulkByScrollResponse> {
 
     private Matcher<Long> createdMatcher = equalTo(0L);
     private Matcher<Long> updatedMatcher = equalTo(0L);
@@ -39,6 +42,7 @@ public class BulkIndexByScrollResponseMatcher extends TypeSafeMatcher<BulkIndexB
     private Matcher<Long> versionConflictsMatcher = equalTo(0L);
     private Matcher<Integer> failuresMatcher = equalTo(0);
     private Matcher<String> reasonCancelledMatcher = nullValue(String.class);
+    private Matcher<Collection<? extends BulkIndexByScrollResponseMatcher>> slicesMatcher = empty();
 
     public BulkIndexByScrollResponseMatcher created(Matcher<Long> createdMatcher) {
         this.createdMatcher = createdMatcher;
@@ -117,15 +121,24 @@ public BulkIndexByScrollResponseMatcher reasonCancelled(Matcher<String> reasonCa
         return this;
     }
 
+    /**
+     * Set the matcher for the workers portion of the response.
+     */
+    public BulkIndexByScrollResponseMatcher slices(Matcher<Collection<? extends BulkIndexByScrollResponseMatcher>> slicesMatcher) {
+        this.slicesMatcher = slicesMatcher;
+        return this;
+    }
+
     @Override
-    protected boolean matchesSafely(BulkIndexByScrollResponse item) {
+    protected boolean matchesSafely(BulkByScrollResponse item) {
         return updatedMatcher.matches(item.getUpdated()) &&
                 createdMatcher.matches(item.getCreated()) &&
                 deletedMatcher.matches(item.getDeleted()) &&
                 (batchesMatcher == null || batchesMatcher.matches(item.getBatches())) &&
                 versionConflictsMatcher.matches(item.getVersionConflicts()) &&
                 failuresMatcher.matches(item.getBulkFailures().size()) &&
-                reasonCancelledMatcher.matches(item.getReasonCancelled());
+                reasonCancelledMatcher.matches(item.getReasonCancelled()) &&
+                slicesMatcher.matches(item.getStatus().getSliceStatuses());
     }
 
     @Override
@@ -139,5 +152,6 @@ public void describeTo(Description description) {
         description.appendText(" and versionConflicts matches ").appendDescriptionOf(versionConflictsMatcher);
         description.appendText(" and failures size matches ").appendDescriptionOf(failuresMatcher);
         description.appendText(" and reason cancelled matches ").appendDescriptionOf(reasonCancelledMatcher);
+        description.appendText(" and slices matches ").appendDescriptionOf(slicesMatcher);
     }
-}
\ No newline at end of file
+}
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseTests.java
new file mode 100644
index 0000000000000..6809a02585e1b
--- /dev/null
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseTests.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex;
+
+import org.elasticsearch.action.bulk.BulkItemResponse;
+import org.elasticsearch.index.reindex.ScrollableHitSource.SearchFailure;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import static java.util.Collections.emptyList;
+import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
+
+public class BulkIndexByScrollResponseTests extends ESTestCase {
+    public void testMergeConstructor() {
+        int mergeCount = between(2, 10);
+        List<BulkByScrollResponse> responses = new ArrayList<>(mergeCount);
+        int took = between(1000, 10000);
+        int tookIndex = between(0, mergeCount - 1);
+        List<BulkItemResponse.Failure> allBulkFailures = new ArrayList<>();
+        List<SearchFailure> allSearchFailures = new ArrayList<>();
+        boolean timedOut = false;
+        String reasonCancelled = rarely() ? randomAlphaOfLength(5) : null;
+
+        for (int i = 0; i < mergeCount; i++) {
+            // One of the merged responses gets the expected value for took, the others get a smaller value
+            TimeValue thisTook = timeValueMillis(i == tookIndex ? took : between(0, took));
+            // The actual status doesn't matter too much - we test merging those elsewhere
+            String thisReasonCancelled = rarely() ? randomAlphaOfLength(5) : null;
+            BulkByScrollTask.Status status = new BulkByScrollTask.Status(i, 0, 0, 0, 0, 0, 0, 0, 0, 0, timeValueMillis(0), 0f,
+                    thisReasonCancelled, timeValueMillis(0));
+            List<BulkItemResponse.Failure> bulkFailures = frequently() ? emptyList()
+                    : IntStream.range(0, between(1, 3)).mapToObj(j -> new BulkItemResponse.Failure("idx", "type", "id", new Exception()))
+                            .collect(Collectors.toList());
+            allBulkFailures.addAll(bulkFailures);
+            List<SearchFailure> searchFailures = frequently() ? emptyList()
+                    : IntStream.range(0, between(1, 3)).mapToObj(j -> new SearchFailure(new Exception())).collect(Collectors.toList());
+            allSearchFailures.addAll(searchFailures);
+            boolean thisTimedOut = rarely();
+            timedOut |= thisTimedOut;
+            responses.add(new BulkByScrollResponse(thisTook, status, bulkFailures, searchFailures, thisTimedOut));
+        }
+
+        BulkByScrollResponse merged = new BulkByScrollResponse(responses, reasonCancelled);
+
+        assertEquals(timeValueMillis(took), merged.getTook());
+        assertEquals(allBulkFailures, merged.getBulkFailures());
+        assertEquals(allSearchFailures, merged.getSearchFailures());
+        assertEquals(timedOut, merged.isTimedOut());
+        assertEquals(reasonCancelled, merged.getReasonCancelled());
+    }
+}
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java
index 8da47f1eeafe2..cd7de9790bca3 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java
@@ -25,18 +25,22 @@
 import org.elasticsearch.action.ingest.DeletePipelineRequest;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.engine.Engine.Operation.Origin;
+import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.shard.IndexingOperationListener;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.ingest.IngestTestPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.tasks.TaskInfo;
-import org.junit.BeforeClass;
+import org.elasticsearch.test.junit.annotations.TestLogging;
+import org.hamcrest.Matcher;
+import org.junit.Before;
 
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.List;
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
@@ -45,24 +49,21 @@
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
-import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.emptyIterable;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
 
 /**
  * Test that you can actually cancel a reindex/update-by-query/delete-by-query request and all the plumbing works. Doesn't test all of the
- * different cancellation places - that is the responsibility of {@link AsyncBulkByScrollActionTests} which have more precise control to
- * simulate failures but do not exercise important portion of the stack like transport and task management.
+ * different cancellation places - that is the responsibility of AsyncBulkByScrollActionTests which have more precise control to
+ * simulate failures but does not exercise important portion of the stack like transport and task management.
  */
+@TestLogging("org.elasticsearch.index.reindex:DEBUG,org.elasticsearch.action.bulk:DEBUG")
 public class CancelTests extends ReindexTestCase {
 
     protected static final String INDEX = "reindex-cancel-index";
     protected static final String TYPE = "reindex-cancel-type";
 
-    private static final int MIN_OPERATIONS = 2;
-    private static final int BLOCKING_OPERATIONS = 1;
-
     // Semaphore used to allow & block indexing operations during the test
     private static final Semaphore ALLOWED_OPERATIONS = new Semaphore(0);
 
@@ -74,19 +75,20 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
         return plugins;
     }
 
-    @BeforeClass
-    public static void clearAllowedOperations() {
+    @Before
+    public void clearAllowedOperations() {
         ALLOWED_OPERATIONS.drainPermits();
     }
 
     /**
      * Executes the cancellation test
      */
-    private void testCancel(String action, AbstractBulkByScrollRequestBuilder<?, ?> builder, CancelAssertion assertion) throws Exception {
+    private void testCancel(String action, AbstractBulkByScrollRequestBuilder<?, ?> builder, CancelAssertion assertion,
+                            Matcher<String> taskDescriptionMatcher) throws Exception {
         createIndex(INDEX);
 
-        // Total number of documents created for this test (~10 per primary shard)
-        int numDocs = getNumShards(INDEX).numPrimaries * 10;
+        // Total number of documents created for this test (~10 per primary shard per slice)
+        int numDocs = getNumShards(INDEX).numPrimaries * 10 * builder.request().getSlices();
         ALLOWED_OPERATIONS.release(numDocs);
 
         indexRandom(true, false, true, IntStream.range(0, numDocs)
@@ -100,61 +102,107 @@ private void testCancel(String action, AbstractBulkByScrollRequestBuilder<?, ?>
         // Scroll by 1 so that cancellation is easier to control
         builder.source().setSize(1);
 
-        // Allow a random number of the documents minus 1
-        // to be modified by the reindex action
-        int numModifiedDocs = randomIntBetween(MIN_OPERATIONS, numDocs);
-        ALLOWED_OPERATIONS.release(numModifiedDocs - BLOCKING_OPERATIONS);
+        /* Allow a random number of the documents less the number of workers to be modified by the reindex action. That way at least one
+         * worker is blocked. */
+        int numModifiedDocs = randomIntBetween(builder.request().getSlices() * 2, numDocs);
+        ALLOWED_OPERATIONS.release(numModifiedDocs - builder.request().getSlices());
 
         // Now execute the reindex action...
-        ListenableActionFuture<? extends BulkIndexByScrollResponse> future = builder.execute();
+        ListenableActionFuture<? extends BulkByScrollResponse> future = builder.execute();
 
-        // ... and waits for the indexing operation listeners to block
+        /* ... and waits for the indexing operation listeners to block. It is important to realize that some of the workers might have
+         * exhausted their slice while others might have quite a bit left to work on. We can't control that. */
         awaitBusy(() -> ALLOWED_OPERATIONS.hasQueuedThreads() && ALLOWED_OPERATIONS.availablePermits() == 0);
 
         // Status should show the task running
-        ListTasksResponse tasksList = client().admin().cluster().prepareListTasks().setActions(action).setDetailed(true).get();
-        assertThat(tasksList.getNodeFailures(), empty());
-        assertThat(tasksList.getTaskFailures(), empty());
-        assertThat(tasksList.getTasks(), hasSize(1));
-        BulkByScrollTask.Status status = (BulkByScrollTask.Status) tasksList.getTasks().get(0).getStatus();
+        TaskInfo mainTask = findTaskToCancel(action, builder.request().getSlices());
+        BulkByScrollTask.Status status = (BulkByScrollTask.Status) mainTask.getStatus();
         assertNull(status.getReasonCancelled());
 
+        // Description shouldn't be empty
+        assertThat(mainTask.getDescription(), taskDescriptionMatcher);
+
         // Cancel the request while the reindex action is blocked by the indexing operation listeners.
         // This will prevent further requests from being sent.
-        List<TaskInfo> cancelledTasks = client().admin().cluster().prepareCancelTasks().setActions(action).get().getTasks();
-        assertThat(cancelledTasks, hasSize(1));
-
-        // The status should now show canceled. The request will still be in the list because it is still blocked.
-        tasksList = client().admin().cluster().prepareListTasks().setActions(action).setDetailed(true).get();
-        assertThat(tasksList.getNodeFailures(), empty());
-        assertThat(tasksList.getTaskFailures(), empty());
-        assertThat(tasksList.getTasks(), hasSize(1));
-        status = (BulkByScrollTask.Status) tasksList.getTasks().get(0).getStatus();
+        ListTasksResponse cancelTasksResponse = client().admin().cluster().prepareCancelTasks().setTaskId(mainTask.getTaskId()).get();
+        cancelTasksResponse.rethrowFailures("Cancel");
+        assertThat(cancelTasksResponse.getTasks(), hasSize(1));
+
+        // The status should now show canceled. The request will still be in the list because it is (or its children are) still blocked.
+        mainTask = client().admin().cluster().prepareGetTask(mainTask.getTaskId()).get().getTask().getTask();
+        status = (BulkByScrollTask.Status) mainTask.getStatus();
         assertEquals(CancelTasksRequest.DEFAULT_REASON, status.getReasonCancelled());
+        if (builder.request().getSlices() > 1) {
+            boolean foundCancelled = false;
+            ListTasksResponse sliceList = client().admin().cluster().prepareListTasks().setParentTaskId(mainTask.getTaskId())
+                    .setDetailed(true).get();
+            sliceList.rethrowFailures("Fetch slice tasks");
+            for (TaskInfo slice: sliceList.getTasks()) {
+                BulkByScrollTask.Status sliceStatus = (BulkByScrollTask.Status) slice.getStatus();
+                if (sliceStatus.getReasonCancelled() == null) continue;
+                assertEquals(CancelTasksRequest.DEFAULT_REASON, sliceStatus.getReasonCancelled());
+                foundCancelled = true;
+            }
+            assertTrue("Didn't find at least one sub task that was cancelled", foundCancelled);
+        }
 
-        // Unblock the last operation
-        ALLOWED_OPERATIONS.release(BLOCKING_OPERATIONS);
+        // Unblock the last operations
+        ALLOWED_OPERATIONS.release(builder.request().getSlices());
 
         // Checks that no more operations are executed
-        assertBusy(() -> assertTrue(ALLOWED_OPERATIONS.availablePermits() == 0 && ALLOWED_OPERATIONS.getQueueLength() == 0));
+        assertBusy(() -> {
+            if (builder.request().getSlices() == 1) {
+                /* We can only be sure that we've drained all the permits if we only use a single worker. Otherwise some worker may have
+                 * exhausted all of its documents before we blocked. */
+                assertEquals(0, ALLOWED_OPERATIONS.availablePermits());
+            }
+            assertEquals(0, ALLOWED_OPERATIONS.getQueueLength());
+        });
 
         // And check the status of the response
-        BulkIndexByScrollResponse response = future.get();
+        BulkByScrollResponse response;
+        try {
+            response = future.get(30, TimeUnit.SECONDS);
+        } catch (Exception e) {
+            String tasks = client().admin().cluster().prepareListTasks().setParentTaskId(mainTask.getTaskId())
+                        .setDetailed(true).get().toString();
+            throw new RuntimeException("Exception while waiting for the response. Running tasks: " + tasks, e);
+        }
         assertThat(response.getReasonCancelled(), equalTo("by user request"));
         assertThat(response.getBulkFailures(), emptyIterable());
         assertThat(response.getSearchFailures(), emptyIterable());
 
+        if (builder.request().getSlices() >= 1) {
+            // If we have more than one worker we might not have made all the modifications
+            numModifiedDocs -= ALLOWED_OPERATIONS.availablePermits();
+        }
         flushAndRefresh(INDEX);
         assertion.assertThat(response, numDocs, numModifiedDocs);
     }
 
+    private TaskInfo findTaskToCancel(String actionName, int workerCount) {
+        ListTasksResponse tasks;
+        long start = System.nanoTime();
+        do {
+            tasks = client().admin().cluster().prepareListTasks().setActions(actionName).setDetailed(true).get();
+            tasks.rethrowFailures("Find tasks to cancel");
+            for (TaskInfo taskInfo : tasks.getTasks()) {
+                // Skip tasks with a parent because those are children of the task we want to cancel
+                if (false == taskInfo.getParentTaskId().isSet()) {
+                    return taskInfo;
+                }
+            }
+        } while (System.nanoTime() - start < TimeUnit.SECONDS.toNanos(10));
+        throw new AssertionError("Couldn't find task to rethrottle after waiting tasks=" + tasks.getTasks());
+    }
+
     public void testReindexCancel() throws Exception {
         testCancel(ReindexAction.NAME, reindex().source(INDEX).destination("dest", TYPE), (response, total, modified) -> {
             assertThat(response, matcher().created(modified).reasonCancelled(equalTo("by user request")));
 
             refresh("dest");
             assertHitCount(client().prepareSearch("dest").setTypes(TYPE).setSize(0).get(), modified);
-        });
+        }, equalTo("reindex from [" + INDEX + "] to [dest][" + TYPE + "]"));
     }
 
     public void testUpdateByQueryCancel() throws Exception {
@@ -164,28 +212,64 @@ public void testUpdateByQueryCancel() throws Exception {
                 "      \"test\" : {}\n" +
                 "  } ]\n" +
                 "}");
-        assertAcked(client().admin().cluster().preparePutPipeline("set-foo", pipeline).get());
+        assertAcked(client().admin().cluster().preparePutPipeline("set-processed", pipeline, XContentType.JSON).get());
 
-        testCancel(UpdateByQueryAction.NAME, updateByQuery().setPipeline("set-foo").source(INDEX), (response, total, modified) -> {
+        testCancel(UpdateByQueryAction.NAME, updateByQuery().setPipeline("set-processed").source(INDEX), (response, total, modified) -> {
             assertThat(response, matcher().updated(modified).reasonCancelled(equalTo("by user request")));
             assertHitCount(client().prepareSearch(INDEX).setSize(0).setQuery(termQuery("processed", true)).get(), modified);
-        });
+        }, equalTo("update-by-query [" + INDEX + "]"));
 
-        assertAcked(client().admin().cluster().deletePipeline(new DeletePipelineRequest("set-foo")).get());
+        assertAcked(client().admin().cluster().deletePipeline(new DeletePipelineRequest("set-processed")).get());
     }
 
     public void testDeleteByQueryCancel() throws Exception {
         testCancel(DeleteByQueryAction.NAME, deleteByQuery().source(INDEX), (response, total, modified) -> {
             assertThat(response, matcher().deleted(modified).reasonCancelled(equalTo("by user request")));
             assertHitCount(client().prepareSearch(INDEX).setSize(0).get(), total - modified);
-        });
+        }, equalTo("delete-by-query [" + INDEX + "]"));
+    }
+
+    public void testReindexCancelWithWorkers() throws Exception {
+        testCancel(ReindexAction.NAME,
+                reindex().source(INDEX).filter(QueryBuilders.matchAllQuery()).destination("dest", TYPE).setSlices(5),
+                (response, total, modified) -> {
+                    assertThat(response, matcher().created(modified).reasonCancelled(equalTo("by user request")).slices(hasSize(5)));
+                    refresh("dest");
+                    assertHitCount(client().prepareSearch("dest").setTypes(TYPE).setSize(0).get(), modified);
+                },
+                equalTo("reindex from [" + INDEX + "] to [dest][" + TYPE + "]"));
+    }
+
+    public void testUpdateByQueryCancelWithWorkers() throws Exception {
+        BytesReference pipeline = new BytesArray("{\n" +
+                "  \"description\" : \"sets processed to true\",\n" +
+                "  \"processors\" : [ {\n" +
+                "      \"test\" : {}\n" +
+                "  } ]\n" +
+                "}");
+        assertAcked(client().admin().cluster().preparePutPipeline("set-processed", pipeline, XContentType.JSON).get());
+
+        testCancel(UpdateByQueryAction.NAME, updateByQuery().setPipeline("set-processed").source(INDEX).setSlices(5),
+                (response, total, modified) -> {
+                    assertThat(response, matcher().updated(modified).reasonCancelled(equalTo("by user request")).slices(hasSize(5)));
+                    assertHitCount(client().prepareSearch(INDEX).setSize(0).setQuery(termQuery("processed", true)).get(), modified);
+                }, equalTo("update-by-query [" + INDEX + "]"));
+
+        assertAcked(client().admin().cluster().deletePipeline(new DeletePipelineRequest("set-processed")).get());
+    }
+
+    public void testDeleteByQueryCancelWithWorkers() throws Exception {
+        testCancel(DeleteByQueryAction.NAME, deleteByQuery().source(INDEX).setSlices(5), (response, total, modified) -> {
+            assertThat(response, matcher().deleted(modified).reasonCancelled(equalTo("by user request")).slices(hasSize(5)));
+            assertHitCount(client().prepareSearch(INDEX).setSize(0).get(), total - modified);
+        }, equalTo("delete-by-query [" + INDEX + "]"));
     }
 
     /**
-     * {@link CancelAssertion} is used to check the result of the cancel test.
+     * Used to check the result of the cancel test.
      */
     private interface CancelAssertion {
-        void assertThat(BulkIndexByScrollResponse response, int total, int modified);
+        void assertThat(BulkByScrollResponse response, int total, int modified);
     }
 
     public static class ReindexCancellationPlugin extends Plugin {
@@ -199,12 +283,12 @@ public void onIndexModule(IndexModule indexModule) {
     public static class BlockingOperationListener implements IndexingOperationListener {
 
         @Override
-        public Engine.Index preIndex(Engine.Index index) {
+        public Engine.Index preIndex(ShardId shardId, Engine.Index index) {
             return preCheck(index, index.type());
         }
 
         @Override
-        public Engine.Delete preDelete(Engine.Delete delete) {
+        public Engine.Delete preDelete(ShardId shardId, Engine.Delete delete) {
             return preCheck(delete, delete.type());
         }
 
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java
index a52b46233bc50..964831580b353 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java
@@ -21,7 +21,6 @@
 
 import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.action.index.IndexRequestBuilder;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.sort.SortOrder;
@@ -29,11 +28,14 @@
 import java.util.ArrayList;
 import java.util.List;
 
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY_ALLOW_DELETE;
 import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
 import static org.elasticsearch.index.query.QueryBuilders.rangeQuery;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.hamcrest.Matchers.hasSize;
 
 public class DeleteByQueryBasicTests extends ReindexTestCase {
 
@@ -165,7 +167,7 @@ public void testDeleteByMatchQuery() throws Exception {
         List<IndexRequestBuilder> builders = new ArrayList<>();
         for (int i = 0; i < docs; i++) {
             builders.add(client().prepareIndex("test", "test", Integer.toString(i))
-                    .setRouting(randomAsciiOfLengthBetween(1, 5))
+                    .setRouting(randomAlphaOfLengthBetween(1, 5))
                     .setSource("foo", "bar"));
         }
         indexRandom(true, true, true, builders);
@@ -199,13 +201,37 @@ public void testDeleteByQueryOnReadOnlyIndex() throws Exception {
         }
         indexRandom(true, true, true, builders);
 
+        String block = randomFrom(SETTING_READ_ONLY, SETTING_READ_ONLY_ALLOW_DELETE);
         try {
-            enableIndexBlock("test", IndexMetaData.SETTING_READ_ONLY);
+            enableIndexBlock("test", block);
             assertThat(deleteByQuery().source("test").refresh(true).get(), matcher().deleted(0).failures(docs));
         } finally {
-            disableIndexBlock("test", IndexMetaData.SETTING_READ_ONLY);
+            disableIndexBlock("test", block);
         }
 
         assertHitCount(client().prepareSearch("test").setSize(0).get(), docs);
     }
+
+    public void testWorkers() throws Exception {
+        indexRandom(true,
+                client().prepareIndex("test", "test", "1").setSource("foo", "a"),
+                client().prepareIndex("test", "test", "2").setSource("foo", "a"),
+                client().prepareIndex("test", "test", "3").setSource("foo", "b"),
+                client().prepareIndex("test", "test", "4").setSource("foo", "c"),
+                client().prepareIndex("test", "test", "5").setSource("foo", "d"),
+                client().prepareIndex("test", "test", "6").setSource("foo", "e"),
+                client().prepareIndex("test", "test", "7").setSource("foo", "f")
+        );
+        assertHitCount(client().prepareSearch("test").setTypes("test").setSize(0).get(), 7);
+
+        // Deletes the two docs that matches "foo:a"
+        assertThat(deleteByQuery().source("test").filter(termQuery("foo", "a")).refresh(true).setSlices(5).get(),
+                matcher().deleted(2).slices(hasSize(5)));
+        assertHitCount(client().prepareSearch("test").setTypes("test").setSize(0).get(), 5);
+
+        // Delete remaining docs
+        DeleteByQueryRequestBuilder request = deleteByQuery().source("test").refresh(true).setSlices(5);
+        assertThat(request.get(), matcher().deleted(5).slices(hasSize(5)));
+        assertHitCount(client().prepareSearch("test").setTypes("test").setSize(0).get(), 0);
+    }
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryConcurrentTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryConcurrentTests.java
index 40a776626a337..8954f1c35c179 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryConcurrentTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryConcurrentTests.java
@@ -96,7 +96,7 @@ public void testConcurrentDeleteByQueriesOnSameDocs() throws Throwable {
                 try {
                     start.await();
 
-                    BulkIndexByScrollResponse response = deleteByQuery().source("test").filter(query).refresh(true).get();
+                    BulkByScrollResponse response = deleteByQuery().source("test").filter(query).refresh(true).get();
                     // Some deletions might fail due to version conflict, but
                     // what matters here is the total of successful deletions
                     deleted.addAndGet(response.getDeleted());
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryRequestTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryRequestTests.java
deleted file mode 100644
index 6b615f45521c4..0000000000000
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryRequestTests.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.reindex;
-
-import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.test.ESTestCase;
-
-import static org.apache.lucene.util.TestUtil.randomSimpleString;
-
-public class DeleteByQueryRequestTests extends ESTestCase {
-    public void testDeleteteByQueryRequestImplementsIndicesRequestReplaceable() {
-        int numIndices = between(1, 100);
-        String[] indices = new String[numIndices];
-        for (int i = 0; i < numIndices; i++) {
-            indices[i] = randomSimpleString(random(), 1, 30);
-        }
-
-        SearchRequest searchRequest = new SearchRequest(indices);
-        IndicesOptions indicesOptions = IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean());
-        searchRequest.indicesOptions(indicesOptions);
-
-        DeleteByQueryRequest request = new DeleteByQueryRequest(searchRequest);
-        for (int i = 0; i < numIndices; i++) {
-            assertEquals(indices[i], request.indices()[i]);
-        }
-
-        assertSame(indicesOptions, request.indicesOptions());
-        assertSame(request.indicesOptions(), request.getSearchRequest().indicesOptions());
-
-        int numNewIndices = between(1, 100);
-        String[] newIndices = new String[numNewIndices];
-        for (int i = 0; i < numNewIndices; i++) {
-            newIndices[i] = randomSimpleString(random(), 1, 30);
-        }
-        request.indices(newIndices);
-        for (int i = 0; i < numNewIndices; i++) {;
-            assertEquals(newIndices[i], request.indices()[i]);
-        }
-        for (int i = 0; i < numNewIndices; i++) {;
-            assertEquals(newIndices[i], request.getSearchRequest().indices()[i]);
-        }
-    }
-}
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java
index 5742a37f11e21..00724eed1de4b 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java
@@ -26,6 +26,9 @@
 
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
 
 public class ReindexBasicTests extends ReindexTestCase {
     public void testFiltering() throws Exception {
@@ -82,4 +85,34 @@ public void testCopyMany() throws Exception {
         assertThat(copy.get(), matcher().created(half).batches(half, 5));
         assertHitCount(client().prepareSearch("dest").setTypes("half").setSize(0).get(), half);
     }
+
+    public void testCopyManyWithSlices() throws Exception {
+        int workers = between(2, 10);
+
+        List<IndexRequestBuilder> docs = new ArrayList<>();
+        int max = between(150, 500);
+        for (int i = 0; i < max; i++) {
+            docs.add(client().prepareIndex("source", "test", Integer.toString(i)).setSource("foo", "a"));
+        }
+
+        indexRandom(true, docs);
+        assertHitCount(client().prepareSearch("source").setSize(0).get(), max);
+
+        // Copy all the docs
+        ReindexRequestBuilder copy = reindex().source("source").destination("dest", "all").refresh(true).setSlices(workers);
+        // Use a small batch size so we have to use more than one batch
+        copy.source().setSize(5);
+        assertThat(copy.get(), matcher().created(max).batches(greaterThanOrEqualTo(max / 5)).slices(hasSize(workers)));
+        assertHitCount(client().prepareSearch("dest").setTypes("all").setSize(0).get(), max);
+
+        // Copy some of the docs
+        int half = max / 2;
+        copy = reindex().source("source").destination("dest", "half").refresh(true).setSlices(workers);
+        // Use a small batch size so we have to use more than one batch
+        copy.source().setSize(5);
+        copy.size(half); // The real "size" of the request.
+        BulkByScrollResponse response = copy.get();
+        assertThat(response, matcher().created(lessThanOrEqualTo((long) half)).slices(hasSize(workers)));
+        assertHitCount(client().prepareSearch("dest").setTypes("half").setSize(0).get(), response.getCreated());
+    }
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexClientYamlTestSuiteIT.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexClientYamlTestSuiteIT.java
index 186bb2f0a5ed6..3ec84e548a682 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexClientYamlTestSuiteIT.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexClientYamlTestSuiteIT.java
@@ -24,9 +24,6 @@
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-
-import java.io.IOException;
 
 public class ReindexClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     public ReindexClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
@@ -34,7 +31,7 @@ public ReindexClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCa
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
-        return ESClientYamlSuiteTestCase.createParameters(0, 1);
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFailureTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFailureTests.java
index b81be4a1bb24f..6ef9d43317bc6 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFailureTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFailureTests.java
@@ -22,13 +22,14 @@
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
 import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.test.junit.annotations.TestLogging;
 
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
 
-import static org.elasticsearch.action.index.IndexRequest.OpType.CREATE;
+import static org.elasticsearch.action.DocWriteRequest.OpType.CREATE;
 import static org.hamcrest.Matchers.both;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.either;
@@ -38,6 +39,7 @@
 /**
  * Tests failure capturing and abort-on-failure behavior of reindex.
  */
+@TestLogging("_root:DEBUG")
 public class ReindexFailureTests extends ReindexTestCase {
     public void testFailuresCauseAbortDefault() throws Exception {
         /*
@@ -57,7 +59,7 @@ public void testFailuresCauseAbortDefault() throws Exception {
          */
         copy.source().setSize(1);
 
-        BulkIndexByScrollResponse response = copy.get();
+        BulkByScrollResponse response = copy.get();
         assertThat(response, matcher()
                 .batches(1)
                 .failures(both(greaterThan(0)).and(lessThanOrEqualTo(maximumNumberOfShards()))));
@@ -77,7 +79,7 @@ public void testAbortOnVersionConflict() throws Exception {
         // CREATE will cause the conflict to prevent the write.
         copy.destination().setOpType(CREATE);
 
-        BulkIndexByScrollResponse response = copy.get();
+        BulkByScrollResponse response = copy.get();
         assertThat(response, matcher().batches(1).versionConflicts(1).failures(1).created(99));
         for (Failure failure: response.getBulkFailures()) {
             assertThat(failure.getMessage(), containsString("VersionConflictEngineException[[test]["));
@@ -99,7 +101,7 @@ public void testResponseOnSearchFailure() throws Exception {
             indexDocs(100);
             ReindexRequestBuilder copy = reindex().source("source").destination("dest");
             copy.source().setSize(10);
-            Future<BulkIndexByScrollResponse> response = copy.execute();
+            Future<BulkByScrollResponse> response = copy.execute();
             client().admin().indices().prepareDelete("source").get();
 
             try {
@@ -107,7 +109,7 @@ public void testResponseOnSearchFailure() throws Exception {
                 logger.info("Didn't trigger a reindex failure on the {} attempt", attempt);
                 attempt++;
             } catch (ExecutionException e) {
-                logger.info("Triggered a reindex failure on the {} attempt", attempt);
+                logger.info("Triggered a reindex failure on the {} attempt: {}", attempt, e.getMessage());
                 assertThat(e.getMessage(),
                         either(containsString("all shards failed"))
                         .or(containsString("No search context found"))
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteBuildRestClientTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteBuildRestClientTests.java
index 53c24851d5e1f..d7d9cfe051b8c 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteBuildRestClientTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteBuildRestClientTests.java
@@ -21,7 +21,6 @@
 
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.index.reindex.remote.RemoteInfo;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.ArrayList;
@@ -33,7 +32,8 @@
 
 public class ReindexFromRemoteBuildRestClientTests extends ESTestCase {
     public void testBuildRestClient() throws Exception {
-        RemoteInfo remoteInfo = new RemoteInfo("https", "localhost", 9200, new BytesArray("ignored"), null, null, emptyMap());
+        RemoteInfo remoteInfo = new RemoteInfo("https", "localhost", 9200, new BytesArray("ignored"), null, null, emptyMap(),
+                RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT);
         long taskId = randomLong();
         List<Thread> threads = synchronizedList(new ArrayList<>());
         RestClient client = TransportReindexAction.buildRestClient(remoteInfo, taskId, threads);
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteWhitelistTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteWhitelistTests.java
index 2e41550134e82..128cd4043e283 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteWhitelistTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteWhitelistTests.java
@@ -20,75 +20,101 @@
 package org.elasticsearch.index.reindex;
 
 import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.transport.InetSocketTransportAddress;
-import org.elasticsearch.common.transport.TransportAddress;
-import org.elasticsearch.index.reindex.remote.RemoteInfo;
 import org.elasticsearch.test.ESTestCase;
-import org.junit.Before;
 
-import java.net.InetAddress;
 import java.net.UnknownHostException;
-import java.util.HashSet;
-import java.util.Set;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
 
+import static java.util.Collections.emptyList;
 import static java.util.Collections.emptyMap;
-import static java.util.Collections.emptySet;
+import static java.util.Collections.singletonList;
+import static org.elasticsearch.index.reindex.TransportReindexAction.buildRemoteWhitelist;
 import static org.elasticsearch.index.reindex.TransportReindexAction.checkRemoteWhitelist;
 
 /**
  * Tests the reindex-from-remote whitelist of remotes.
  */
 public class ReindexFromRemoteWhitelistTests extends ESTestCase {
-    private TransportAddress localhost;
-
-    @Before
-    public void setupLocalhost() throws UnknownHostException {
-        localhost = new InetSocketTransportAddress(InetAddress.getByAddress(new byte[] { 0x7f, 0x00, 0x00, 0x01 }), 9200);
-    }
-
     public void testLocalRequestWithoutWhitelist() {
-        checkRemoteWhitelist(emptySet(), null, localhostOrNone());
+        checkRemoteWhitelist(buildRemoteWhitelist(emptyList()), null);
     }
 
     public void testLocalRequestWithWhitelist() {
-        checkRemoteWhitelist(randomWhitelist(), null, localhostOrNone());
+        checkRemoteWhitelist(buildRemoteWhitelist(randomWhitelist()), null);
+    }
+
+    /**
+     * Build a {@link RemoteInfo}, defaulting values that we don't care about in this test to values that don't hurt anything.
+     */
+    private RemoteInfo newRemoteInfo(String host, int port) {
+        return new RemoteInfo(randomAlphaOfLength(5), host, port, new BytesArray("test"), null, null, emptyMap(),
+                RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT);
     }
 
     public void testWhitelistedRemote() {
-        Set<String> whitelist = randomWhitelist();
+        List<String> whitelist = randomWhitelist();
         String[] inList = whitelist.iterator().next().split(":");
         String host = inList[0];
         int port = Integer.valueOf(inList[1]);
-        checkRemoteWhitelist(whitelist, new RemoteInfo(randomAsciiOfLength(5), host, port, new BytesArray("test"), null, null, emptyMap()),
-                localhostOrNone());
+        checkRemoteWhitelist(buildRemoteWhitelist(whitelist), newRemoteInfo(host, port));
+    }
+
+    public void testWhitelistedByPrefix() {
+        checkRemoteWhitelist(buildRemoteWhitelist(singletonList("*.example.com:9200")),
+                new RemoteInfo(randomAlphaOfLength(5), "es.example.com", 9200, new BytesArray("test"), null, null, emptyMap(),
+                        RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT));
+        checkRemoteWhitelist(buildRemoteWhitelist(singletonList("*.example.com:9200")),
+                newRemoteInfo("6e134134a1.us-east-1.aws.example.com", 9200));
+    }
+
+    public void testWhitelistedBySuffix() {
+        checkRemoteWhitelist(buildRemoteWhitelist(singletonList("es.example.com:*")), newRemoteInfo("es.example.com", 9200));
     }
 
-    public void testMyselfInWhitelistRemote() throws UnknownHostException {
-        Set<String> whitelist = randomWhitelist();
-        whitelist.add("myself");
-        TransportAddress publishAddress = new InetSocketTransportAddress(InetAddress.getByAddress(new byte[] {0x7f,0x00,0x00,0x01}), 9200);
-        checkRemoteWhitelist(whitelist,
-                new RemoteInfo(randomAsciiOfLength(5), "127.0.0.1", 9200, new BytesArray("test"), null, null, emptyMap()), publishAddress);
+    public void testWhitelistedByInfix() {
+        checkRemoteWhitelist(buildRemoteWhitelist(singletonList("es*.example.com:9200")), newRemoteInfo("es1.example.com", 9200));
+    }
+
+    public void testLoopbackInWhitelistRemote() throws UnknownHostException {
+        List<String> whitelist = randomWhitelist();
+        whitelist.add("127.0.0.1:*");
+        checkRemoteWhitelist(buildRemoteWhitelist(whitelist), newRemoteInfo("127.0.0.1", 9200));
     }
 
     public void testUnwhitelistedRemote() {
         int port = between(1, Integer.MAX_VALUE);
-        RemoteInfo remoteInfo = new RemoteInfo(randomAsciiOfLength(5), "not in list", port, new BytesArray("test"), null, null, emptyMap());
+        List<String> whitelist = randomBoolean() ? randomWhitelist() : emptyList();
         Exception e = expectThrows(IllegalArgumentException.class,
-                () -> checkRemoteWhitelist(randomWhitelist(), remoteInfo, localhostOrNone()));
+                () -> checkRemoteWhitelist(buildRemoteWhitelist(whitelist), newRemoteInfo("not in list", port)));
         assertEquals("[not in list:" + port + "] not whitelisted in reindex.remote.whitelist", e.getMessage());
     }
 
-    private Set<String> randomWhitelist() {
-        int size = between(1, 100);
-        Set<String> set = new HashSet<>(size);
-        while (set.size() < size) {
-            set.add(randomAsciiOfLength(5) + ':' + between(1, Integer.MAX_VALUE));
-        }
-        return set;
+    public void testRejectMatchAll() {
+        assertMatchesTooMuch(singletonList("*"));
+        assertMatchesTooMuch(singletonList("**"));
+        assertMatchesTooMuch(singletonList("***"));
+        assertMatchesTooMuch(Arrays.asList("realstuff", "*"));
+        assertMatchesTooMuch(Arrays.asList("*", "realstuff"));
+        List<String> random = randomWhitelist();
+        random.add("*");
+        assertMatchesTooMuch(random);
     }
 
-    private TransportAddress localhostOrNone() {
-        return randomFrom(random(), null, localhost);
+    private void assertMatchesTooMuch(List<String> whitelist) {
+        Exception e = expectThrows(IllegalArgumentException.class, () -> buildRemoteWhitelist(whitelist));
+        assertEquals("Refusing to start because whitelist " + whitelist + " accepts all addresses. "
+                + "This would allow users to reindex-from-remote any URL they like effectively having Elasticsearch make HTTP GETs "
+                + "for them.", e.getMessage());
+    }
+
+    private List<String> randomWhitelist() {
+        int size = between(1, 100);
+        List<String> whitelist = new ArrayList<>(size);
+        for (int i = 0; i < size; i++) {
+            whitelist.add(randomAlphaOfLength(5) + ':' + between(1, Integer.MAX_VALUE));
+        }
+        return whitelist;
     }
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteWithAuthTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteWithAuthTests.java
index 5c7a90157e7a7..c4b5c26e5c4ef 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteWithAuthTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteWithAuthTests.java
@@ -19,8 +19,8 @@
 
 package org.elasticsearch.index.reindex;
 
-import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
@@ -35,7 +35,6 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.index.reindex.remote.RemoteInfo;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.rest.RestStatus;
@@ -48,6 +47,7 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
+import java.util.Map;
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.singletonList;
@@ -69,10 +69,9 @@ protected Collection<Class<? extends Plugin>> getPlugins() {
     @Override
     protected Settings nodeSettings() {
         Settings.Builder settings = Settings.builder().put(super.nodeSettings());
-        // Weird incantation required to test with netty
         settings.put(NetworkModule.HTTP_ENABLED.getKey(), true);
         // Whitelist reindexing from the http host we're going to use
-        settings.put(TransportReindexAction.REMOTE_CLUSTER_WHITELIST.getKey(), "myself");
+        settings.put(TransportReindexAction.REMOTE_CLUSTER_WHITELIST.getKey(), "127.0.0.1:*");
         settings.put(NetworkModule.HTTP_TYPE_KEY, Netty4Plugin.NETTY_HTTP_TRANSPORT_NAME);
         return settings.build();
     }
@@ -88,29 +87,31 @@ public void fetchTransportAddress() {
         address = nodeInfo.getHttp().getAddress().publishAddress();
     }
 
+    /**
+     * Build a {@link RemoteInfo}, defaulting values that we don't care about in this test to values that don't hurt anything.
+     */
+    private RemoteInfo newRemoteInfo(String username, String password, Map<String, String> headers) {
+        return new RemoteInfo("http", address.getAddress(), address.getPort(), new BytesArray("{\"match_all\":{}}"), username, password,
+                headers, RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT);
+    }
+
     public void testReindexFromRemoteWithAuthentication() throws Exception {
-        RemoteInfo remote = new RemoteInfo("http", address.getHost(), address.getPort(), new BytesArray("{\"match_all\":{}}"), "Aladdin",
-                "open sesame", emptyMap());
         ReindexRequestBuilder request = ReindexAction.INSTANCE.newRequestBuilder(client()).source("source").destination("dest")
-                .setRemoteInfo(remote);
+                .setRemoteInfo(newRemoteInfo("Aladdin", "open sesame", emptyMap()));
         assertThat(request.get(), matcher().created(1));
     }
 
     public void testReindexSendsHeaders() throws Exception {
-        RemoteInfo remote = new RemoteInfo("http", address.getHost(), address.getPort(), new BytesArray("{\"match_all\":{}}"), null, null,
-                singletonMap(TestFilter.EXAMPLE_HEADER, "doesn't matter"));
         ReindexRequestBuilder request = ReindexAction.INSTANCE.newRequestBuilder(client()).source("source").destination("dest")
-                .setRemoteInfo(remote);
+                .setRemoteInfo(newRemoteInfo(null, null, singletonMap(TestFilter.EXAMPLE_HEADER, "doesn't matter")));
         ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, () -> request.get());
         assertEquals(RestStatus.BAD_REQUEST, e.status());
         assertThat(e.getMessage(), containsString("Hurray! Sent the header!"));
     }
 
     public void testReindexWithoutAuthenticationWhenRequired() throws Exception {
-        RemoteInfo remote = new RemoteInfo("http", address.getHost(), address.getPort(), new BytesArray("{\"match_all\":{}}"), null, null,
-                emptyMap());
         ReindexRequestBuilder request = ReindexAction.INSTANCE.newRequestBuilder(client()).source("source").destination("dest")
-                .setRemoteInfo(remote);
+                .setRemoteInfo(newRemoteInfo(null, null, emptyMap()));
         ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, () -> request.get());
         assertEquals(RestStatus.UNAUTHORIZED, e.status());
         assertThat(e.getMessage(), containsString("\"reason\":\"Authentication required\""));
@@ -118,10 +119,8 @@ public void testReindexWithoutAuthenticationWhenRequired() throws Exception {
     }
 
     public void testReindexWithBadAuthentication() throws Exception {
-        RemoteInfo remote = new RemoteInfo("http", address.getHost(), address.getPort(), new BytesArray("{\"match_all\":{}}"), "junk",
-                "auth", emptyMap());
         ReindexRequestBuilder request = ReindexAction.INSTANCE.newRequestBuilder(client()).source("source").destination("dest")
-                .setRemoteInfo(remote);
+                .setRemoteInfo(newRemoteInfo("junk", "auth", emptyMap()));
         ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, () -> request.get());
         assertThat(e.getMessage(), containsString("\"reason\":\"Bad Authorization\""));
     }
@@ -165,7 +164,7 @@ public int order() {
         }
 
         @Override
-        public <Request extends ActionRequest<Request>, Response extends ActionResponse> void apply(Task task, String action,
+        public <Request extends ActionRequest, Response extends ActionResponse> void apply(Task task, String action,
                 Request request, ActionListener<Response> listener, ActionFilterChain<Request, Response> chain) {
             if (false == action.equals(SearchAction.NAME)) {
                 chain.proceed(task, action, request, listener);
@@ -186,11 +185,5 @@ public <Request extends ActionRequest<Request>, Response extends ActionResponse>
             }
             chain.proceed(task, action, request, listener);
         }
-
-        @Override
-        public <Response extends ActionResponse> void apply(String action, Response response, ActionListener<Response> listener,
-                ActionFilterChain<?, Response> chain) {
-            chain.proceed(action, response, listener);
-        }
     }
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexMetadataTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexMetadataTests.java
index dab0cab8d8a3b..4611f9dcbcddb 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexMetadataTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexMetadataTests.java
@@ -19,58 +19,77 @@
 
 package org.elasticsearch.index.reindex;
 
+import org.elasticsearch.index.reindex.ScrollableHitSource.Hit;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.common.settings.Settings;
 
 /**
  * Index-by-search test for ttl, timestamp, and routing.
  */
-public class ReindexMetadataTests extends AbstractAsyncBulkIndexbyScrollActionMetadataTestCase<ReindexRequest, BulkIndexByScrollResponse> {
+public class ReindexMetadataTests extends AbstractAsyncBulkByScrollActionMetadataTestCase<ReindexRequest, BulkByScrollResponse> {
     public void testRoutingCopiedByDefault() throws Exception {
         IndexRequest index = new IndexRequest();
-        action().copyMetadata(AbstractAsyncBulkIndexByScrollAction.wrap(index), doc().setRouting("foo"));
+        action().copyMetadata(AbstractAsyncBulkByScrollAction.wrap(index), doc().setRouting("foo"));
         assertEquals("foo", index.routing());
     }
 
     public void testRoutingCopiedIfRequested() throws Exception {
-        TransportReindexAction.AsyncIndexBySearchAction action = action();
-        action.mainRequest.getDestination().routing("keep");
+        TestAction action = action();
+        action.mainRequest().getDestination().routing("keep");
         IndexRequest index = new IndexRequest();
-        action.copyMetadata(AbstractAsyncBulkIndexByScrollAction.wrap(index), doc().setRouting("foo"));
+        action.copyMetadata(AbstractAsyncBulkByScrollAction.wrap(index), doc().setRouting("foo"));
         assertEquals("foo", index.routing());
     }
 
     public void testRoutingDiscardedIfRequested() throws Exception {
-        TransportReindexAction.AsyncIndexBySearchAction action = action();
-        action.mainRequest.getDestination().routing("discard");
+        TestAction action = action();
+        action.mainRequest().getDestination().routing("discard");
         IndexRequest index = new IndexRequest();
-        action.copyMetadata(AbstractAsyncBulkIndexByScrollAction.wrap(index), doc().setRouting("foo"));
+        action.copyMetadata(AbstractAsyncBulkByScrollAction.wrap(index), doc().setRouting("foo"));
         assertEquals(null, index.routing());
     }
 
     public void testRoutingSetIfRequested() throws Exception {
-        TransportReindexAction.AsyncIndexBySearchAction action = action();
-        action.mainRequest.getDestination().routing("=cat");
+        TestAction action = action();
+        action.mainRequest().getDestination().routing("=cat");
         IndexRequest index = new IndexRequest();
-        action.copyMetadata(AbstractAsyncBulkIndexByScrollAction.wrap(index), doc().setRouting("foo"));
+        action.copyMetadata(AbstractAsyncBulkByScrollAction.wrap(index), doc().setRouting("foo"));
         assertEquals("cat", index.routing());
     }
 
     public void testRoutingSetIfWithDegenerateValue() throws Exception {
-        TransportReindexAction.AsyncIndexBySearchAction action = action();
-        action.mainRequest.getDestination().routing("==]");
+        TestAction action = action();
+        action.mainRequest().getDestination().routing("==]");
         IndexRequest index = new IndexRequest();
-        action.copyMetadata(AbstractAsyncBulkIndexByScrollAction.wrap(index), doc().setRouting("foo"));
+        action.copyMetadata(AbstractAsyncBulkByScrollAction.wrap(index), doc().setRouting("foo"));
         assertEquals("=]", index.routing());
     }
 
     @Override
-    protected TransportReindexAction.AsyncIndexBySearchAction action() {
-        return new TransportReindexAction.AsyncIndexBySearchAction(task, logger, null, threadPool, request(), listener(), null, null);
+    protected TestAction action() {
+        return new TestAction();
     }
 
     @Override
     protected ReindexRequest request() {
         return new ReindexRequest(new SearchRequest(), new IndexRequest());
     }
+
+    private class TestAction extends TransportReindexAction.AsyncIndexBySearchAction {
+        TestAction() {
+            super(ReindexMetadataTests.this.task, ReindexMetadataTests.this.logger, null, ReindexMetadataTests.this.threadPool, request(),
+                    null, null, listener(), Settings.EMPTY);
+        }
+
+        public ReindexRequest mainRequest() {
+            return this.mainRequest;
+        }
+
+        @Override
+        public AbstractAsyncBulkByScrollAction.RequestWrapper<?> copyMetadata(AbstractAsyncBulkByScrollAction.RequestWrapper<?> request,
+                Hit doc) {
+            return super.copyMetadata(request, doc);
+        }
+    }
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexParentChildTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexParentChildTests.java
index 58770e10a1caa..fc3e02ef91e01 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexParentChildTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexParentChildTests.java
@@ -21,12 +21,22 @@
 
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.join.ParentJoinPlugin;
+import org.elasticsearch.plugins.Plugin;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
 
-import static org.elasticsearch.index.query.QueryBuilders.hasParentQuery;
 import static org.elasticsearch.index.query.QueryBuilders.idsQuery;
+import static org.elasticsearch.join.query.JoinQueryBuilders.hasParentQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasToString;
+import static org.hamcrest.Matchers.instanceOf;
 
 /**
  * Index-by-search tests for parent/child.
@@ -36,6 +46,23 @@ public class ReindexParentChildTests extends ReindexTestCase {
     QueryBuilder findsCity;
     QueryBuilder findsNeighborhood;
 
+    @Override
+    protected boolean ignoreExternalCluster() {
+        return true;
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        final List<Class<? extends Plugin>> plugins = new ArrayList<>(super.nodePlugins());
+        plugins.add(ParentJoinPlugin.class);
+        return Collections.unmodifiableList(plugins);
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> transportClientPlugins() {
+        return nodePlugins();
+    }
+
     public void testParentChild() throws Exception {
         createParentChildIndex("source");
         createParentChildIndex("dest");
@@ -75,12 +102,11 @@ public void testErrorMessageWhenBadParentChild() throws Exception {
         createParentChildDocs("source");
 
         ReindexRequestBuilder copy = reindex().source("source").destination("dest").filter(findsCity);
-        try {
-            copy.get();
-            fail("Expected exception");
-        } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), equalTo("Can't specify parent if no parent field has been configured"));
-        }
+        final BulkByScrollResponse response = copy.get();
+        assertThat(response.getBulkFailures().size(), equalTo(1));
+        final Exception cause = response.getBulkFailures().get(0).getCause();
+        assertThat(cause, instanceOf(IllegalArgumentException.class));
+        assertThat(cause, hasToString(containsString("can't specify parent if no parent field has been configured")));
     }
 
     /**
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexRequestTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexRequestTests.java
deleted file mode 100644
index 0455e43ec090a..0000000000000
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexRequestTests.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.reindex;
-
-import org.elasticsearch.action.ActionRequestValidationException;
-import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.index.reindex.remote.RemoteInfo;
-import org.elasticsearch.test.ESTestCase;
-
-import static java.util.Collections.emptyMap;
-import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
-
-/**
- * Tests some of the validation of {@linkplain ReindexRequest}. See reindex's rest tests for much more.
- */
-public class ReindexRequestTests extends ESTestCase {
-    public void testTimestampAndTtlNotAllowed() {
-        ReindexRequest reindex = request();
-        reindex.getDestination().ttl("1s").timestamp("now");
-        ActionRequestValidationException e = reindex.validate();
-        assertEquals("Validation Failed: 1: setting ttl on destination isn't supported. use scripts instead.;"
-                + "2: setting timestamp on destination isn't supported. use scripts instead.;",
-                e.getMessage());
-    }
-
-    public void testReindexFromRemoteDoesNotSupportSearchQuery() {
-        ReindexRequest reindex = request();
-        reindex.setRemoteInfo(new RemoteInfo(randomAsciiOfLength(5), randomAsciiOfLength(5), between(1, Integer.MAX_VALUE),
-                new BytesArray("real_query"), null, null, emptyMap()));
-        reindex.getSearchRequest().source().query(matchAllQuery()); // Unsupported place to put query
-        ActionRequestValidationException e = reindex.validate();
-        assertEquals("Validation Failed: 1: reindex from remote sources should use RemoteInfo's query instead of source's query;",
-                e.getMessage());
-    }
-
-    private ReindexRequest request() {
-        ReindexRequest reindex = new ReindexRequest(new SearchRequest(), new IndexRequest());
-        reindex.getSearchRequest().indices("source");
-        reindex.getDestination().index("dest");
-        return reindex;
-    }
-}
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexScriptTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexScriptTests.java
index c70b80b8e374c..36d4a1d0c0ce8 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexScriptTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexScriptTests.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.common.lucene.uid.Versions;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.script.ScriptService;
 
 import java.util.Map;
@@ -32,7 +33,7 @@
 /**
  * Tests index-by-search with a script modifying the documents.
  */
-public class ReindexScriptTests extends AbstractAsyncBulkIndexByScrollActionScriptTestCase<ReindexRequest, BulkIndexByScrollResponse> {
+public class ReindexScriptTests extends AbstractAsyncBulkByScrollActionScriptTestCase<ReindexRequest, BulkByScrollResponse> {
 
     public void testSetIndex() throws Exception {
         Object dest = randomFrom(new Object[] {234, 234L, "pancake"});
@@ -136,8 +137,8 @@ protected ReindexRequest request() {
     }
 
     @Override
-    protected AbstractAsyncBulkIndexByScrollAction<ReindexRequest> action(ScriptService scriptService, ReindexRequest request) {
-        return new TransportReindexAction.AsyncIndexBySearchAction(task, logger, null, threadPool, request, listener(), scriptService,
-                null);
+    protected TransportReindexAction.AsyncIndexBySearchAction action(ScriptService scriptService, ReindexRequest request) {
+        return new TransportReindexAction.AsyncIndexBySearchAction(task, logger, null, threadPool, request, scriptService, null,
+                listener(), Settings.EMPTY);
     }
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexSourceTargetValidationTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexSourceTargetValidationTests.java
index 12ed0ed090fa6..28b9febe1c289 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexSourceTargetValidationTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexSourceTargetValidationTests.java
@@ -33,11 +33,8 @@
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.reindex.remote.RemoteInfo;
 import org.elasticsearch.test.ESTestCase;
 
-import java.util.Collections;
-
 import static java.util.Collections.emptyMap;
 import static org.hamcrest.Matchers.containsString;
 
@@ -91,10 +88,11 @@ public void testTargetIsAlias() {
 
     public void testRemoteInfoSkipsValidation() {
         // The index doesn't have to exist
-        succeeds(new RemoteInfo(randomAsciiOfLength(5), "test", 9200, new BytesArray("test"), null, null, emptyMap()), "does_not_exist",
-                "target");
+        succeeds(new RemoteInfo(randomAlphaOfLength(5), "test", 9200, new BytesArray("test"), null, null, emptyMap(),
+                RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT), "does_not_exist", "target");
         // And it doesn't matter if they are the same index. They are considered to be different because the remote one is, well, remote.
-        succeeds(new RemoteInfo(randomAsciiOfLength(5), "test", 9200, new BytesArray("test"), null, null, emptyMap()), "target", "target");
+        succeeds(new RemoteInfo(randomAlphaOfLength(5), "test", 9200, new BytesArray("test"), null, null, emptyMap(),
+                RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT), "target", "target");
     }
 
     private void fails(String target, String... sources) {
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexVersioningTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexVersioningTests.java
index 2988fcb5ca6b1..472dec4675f99 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexVersioningTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexVersioningTests.java
@@ -21,7 +21,7 @@
 
 import org.elasticsearch.action.get.GetResponse;
 
-import static org.elasticsearch.action.index.IndexRequest.OpType.CREATE;
+import static org.elasticsearch.action.DocWriteRequest.OpType.CREATE;
 import static org.elasticsearch.index.VersionType.EXTERNAL;
 import static org.elasticsearch.index.VersionType.INTERNAL;
 
@@ -88,7 +88,7 @@ public void testCreateVersionConflictsOnNewer() throws Exception {
     /**
      * Perform a reindex with EXTERNAL versioning which has "refresh" semantics.
      */
-    private BulkIndexByScrollResponse reindexExternal() {
+    private BulkByScrollResponse reindexExternal() {
         ReindexRequestBuilder reindex =  reindex().source("source").destination("dest").abortOnVersionConflict(false);
         reindex.destination().setVersionType(EXTERNAL);
         return reindex.get();
@@ -97,7 +97,7 @@ private BulkIndexByScrollResponse reindexExternal() {
     /**
      * Perform a reindex with INTERNAL versioning which has "overwrite" semantics.
      */
-    private BulkIndexByScrollResponse reindexInternal() {
+    private BulkByScrollResponse reindexInternal() {
         ReindexRequestBuilder reindex =  reindex().source("source").destination("dest").abortOnVersionConflict(false);
         reindex.destination().setVersionType(INTERNAL);
         return reindex.get();
@@ -106,7 +106,7 @@ private BulkIndexByScrollResponse reindexInternal() {
     /**
      * Perform a reindex with CREATE OpType which has "create" semantics.
      */
-    private BulkIndexByScrollResponse reindexCreate() {
+    private BulkByScrollResponse reindexCreate() {
         ReindexRequestBuilder reindex =  reindex().source("source").destination("dest").abortOnVersionConflict(false);
         reindex.destination().setOpType(CREATE);
         return reindex.get();
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexWithoutContentIT.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexWithoutContentIT.java
new file mode 100644
index 0000000000000..f580b1400c3bd
--- /dev/null
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexWithoutContentIT.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex;
+
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.test.rest.ESRestTestCase;
+
+import java.io.IOException;
+
+import static org.hamcrest.CoreMatchers.containsString;
+
+public class ReindexWithoutContentIT extends ESRestTestCase {
+
+    public void testReindexMissingBody() throws IOException {
+        ResponseException responseException = expectThrows(ResponseException.class, () -> client().performRequest(
+                "POST", "/_reindex"));
+        assertEquals(400, responseException.getResponse().getStatusLine().getStatusCode());
+        assertThat(responseException.getMessage(), containsString("request body is required"));
+    }
+}
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RestReindexActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RestReindexActionTests.java
index f8cbee17329fc..ffb609843f3b4 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RestReindexActionTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RestReindexActionTests.java
@@ -21,21 +21,23 @@
 
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
-import org.elasticsearch.index.reindex.RestReindexAction.ReindexParseContext;
-import org.elasticsearch.index.reindex.remote.RemoteInfo;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
-import org.elasticsearch.search.SearchRequestParsers;
+import org.elasticsearch.rest.RestController;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.rest.FakeRestRequest;
 
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 
+import static java.util.Collections.singletonMap;
+import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
+import static org.mockito.Mockito.mock;
+
 public class RestReindexActionTests extends ESTestCase {
     public void testBuildRemoteInfoNoRemote() throws IOException {
         assertNull(RestReindexAction.buildRemoteInfo(new HashMap<>()));
@@ -52,6 +54,8 @@ public void testBuildRemoteInfoFullyLoaded() throws IOException {
         remote.put("username", "testuser");
         remote.put("password", "testpass");
         remote.put("headers", headers);
+        remote.put("socket_timeout", "90s");
+        remote.put("connect_timeout", "10s");
 
         Map<String, Object> query = new HashMap<>();
         query.put("a", "b");
@@ -68,6 +72,8 @@ public void testBuildRemoteInfoFullyLoaded() throws IOException {
         assertEquals("testuser", remoteInfo.getUsername());
         assertEquals("testpass", remoteInfo.getPassword());
         assertEquals(headers, remoteInfo.getHeaders());
+        assertEquals(timeValueSeconds(90), remoteInfo.getSocketTimeout());
+        assertEquals(timeValueSeconds(10), remoteInfo.getConnectTimeout());
     }
 
     public void testBuildRemoteInfoWithoutAllParts() throws IOException {
@@ -76,16 +82,20 @@ public void testBuildRemoteInfoWithoutAllParts() throws IOException {
         expectThrows(IllegalArgumentException.class, () -> buildRemoteInfoHostTestCase("http://example.com"));
     }
 
-    public void testBuildRemoteInfoWithAllParts() throws IOException {
+    public void testBuildRemoteInfoWithAllHostParts() throws IOException {
         RemoteInfo info = buildRemoteInfoHostTestCase("http://example.com:9200");
         assertEquals("http", info.getScheme());
         assertEquals("example.com", info.getHost());
         assertEquals(9200, info.getPort());
+        assertEquals(RemoteInfo.DEFAULT_SOCKET_TIMEOUT, info.getSocketTimeout()); // Didn't set the timeout so we should get the default
+        assertEquals(RemoteInfo.DEFAULT_CONNECT_TIMEOUT, info.getConnectTimeout()); // Didn't set the timeout so we should get the default
 
         info = buildRemoteInfoHostTestCase("https://other.example.com:9201");
         assertEquals("https", info.getScheme());
         assertEquals("other.example.com", info.getHost());
         assertEquals(9201, info.getPort());
+        assertEquals(RemoteInfo.DEFAULT_SOCKET_TIMEOUT, info.getSocketTimeout());
+        assertEquals(RemoteInfo.DEFAULT_CONNECT_TIMEOUT, info.getConnectTimeout());
     }
 
     public void testReindexFromRemoteRequestParsing() throws IOException {
@@ -108,15 +118,38 @@ public void testReindexFromRemoteRequestParsing() throws IOException {
             b.endObject();
             request = b.bytes();
         }
-        try (XContentParser p = JsonXContent.jsonXContent.createParser(request)) {
+        try (XContentParser p = createParser(JsonXContent.jsonXContent, request)) {
             ReindexRequest r = new ReindexRequest(new SearchRequest(), new IndexRequest());
-            SearchRequestParsers searchParsers = new SearchRequestParsers(new IndicesQueriesRegistry(), null, null);
-            RestReindexAction.PARSER.parse(p, r, new ReindexParseContext(searchParsers, ParseFieldMatcher.STRICT));
+            RestReindexAction.PARSER.parse(p, r, null);
             assertEquals("localhost", r.getRemoteInfo().getHost());
             assertArrayEquals(new String[] {"source"}, r.getSearchRequest().indices());
         }
     }
 
+    public void testPipelineQueryParameterIsError() throws IOException {
+        RestReindexAction action = new RestReindexAction(Settings.EMPTY, mock(RestController.class));
+
+        FakeRestRequest.Builder request = new FakeRestRequest.Builder(xContentRegistry());
+        try (XContentBuilder body = JsonXContent.contentBuilder().prettyPrint()) {
+            body.startObject(); {
+                body.startObject("source"); {
+                    body.field("index", "source");
+                }
+                body.endObject();
+                body.startObject("dest"); {
+                    body.field("index", "dest");
+                }
+                body.endObject();
+            }
+            body.endObject();
+            request.withContent(body.bytes(), body.contentType());
+        }
+        request.withParams(singletonMap("pipeline", "doesn't matter"));
+        Exception e = expectThrows(IllegalArgumentException.class, () -> action.buildRequest(request.build()));
+
+        assertEquals("_reindex doesn't support [pipeline] as a query parmaeter. Specify it in the [dest] object instead.", e.getMessage());
+    }
+
     private RemoteInfo buildRemoteInfoHostTestCase(String hostInRest) throws IOException {
         Map<String, Object> remote = new HashMap<>();
         remote.put("host", hostInRest);
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java
index 7abd1212f6c23..deee488a35e2a 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java
@@ -21,8 +21,22 @@
 
 import org.elasticsearch.action.ListenableActionFuture;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
+import org.elasticsearch.action.admin.cluster.node.tasks.list.TaskGroup;
+import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.tasks.TaskId;
 
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import static org.hamcrest.Matchers.allOf;
+import static org.hamcrest.Matchers.both;
+import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.lessThan;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
 
 /**
  * Tests that you can set requests_per_second over the Java API and that you can rethrottle running requests. There are REST tests for this
@@ -43,31 +57,131 @@ public void testDeleteByQuery() throws Exception {
         testCase(deleteByQuery().source("test"), DeleteByQueryAction.NAME);
     }
 
-    private void testCase(AbstractBulkByScrollRequestBuilder<?, ?> request, String actionName)
-            throws Exception {
-        // Use a single shard so the reindex has to happen in multiple batches
-        client().admin().indices().prepareCreate("test").setSettings("index.number_of_shards", 1).get();
-        indexRandom(true,
-                client().prepareIndex("test", "test", "1").setSource("foo", "bar"),
-                client().prepareIndex("test", "test", "2").setSource("foo", "bar"),
-                client().prepareIndex("test", "test", "3").setSource("foo", "bar"));
+    public void testReindexWithWorkers() throws Exception {
+        testCase(reindex().source("test").destination("dest").setSlices(between(2, 10)), ReindexAction.NAME);
+    }
+
+    public void testUpdateByQueryWithWorkers() throws Exception {
+        testCase(updateByQuery().source("test").setSlices(between(2, 10)), UpdateByQueryAction.NAME);
+    }
+
+    public void testDeleteByQueryWithWorkers() throws Exception {
+        testCase(deleteByQuery().source("test").setSlices(between(2, 10)), DeleteByQueryAction.NAME);
+    }
+
+    private void testCase(AbstractBulkByScrollRequestBuilder<?, ?> request, String actionName) throws Exception {
+        logger.info("Starting test for [{}] with [{}] slices", actionName, request.request().getSlices());
+        /* Add ten documents per slice so most slices will have many documents to process, having to go to multiple batches.
+         * we can't rely on all of them doing so, but 
+         */
+        List<IndexRequestBuilder> docs = new ArrayList<>();
+        for (int i = 0; i < request.request().getSlices() * 10; i++) {
+            docs.add(client().prepareIndex("test", "test", Integer.toString(i)).setSource("foo", "bar"));
+        }
+        indexRandom(true, docs);
 
         // Start a request that will never finish unless we rethrottle it
         request.setRequestsPerSecond(.000001f);  // Throttle "forever"
         request.source().setSize(1);             // Make sure we use multiple batches
-        ListenableActionFuture<? extends BulkIndexByScrollResponse> responseListener = request.execute();
+        ListenableActionFuture<? extends BulkByScrollResponse> responseListener = request.execute();
+
+        TaskGroup taskGroupToRethrottle = findTaskToRethrottle(actionName, request.request().getSlices());
+        TaskId taskToRethrottle = taskGroupToRethrottle.getTaskInfo().getTaskId();
 
-        // Wait for the task to start
-        assertBusy(() -> assertEquals(1, client().admin().cluster().prepareListTasks().setActions(actionName).get().getTasks().size()));
+        if (request.request().getSlices() == 1) {
+            assertThat(taskGroupToRethrottle.getChildTasks(), empty());
+        } else {
+            // There should be a sane number of child tasks running
+            assertThat(taskGroupToRethrottle.getChildTasks(),
+                    hasSize(allOf(greaterThanOrEqualTo(1), lessThanOrEqualTo(request.request().getSlices()))));
+            // Wait for all of the sub tasks to start (or finish, some might finish early, all that matters is that not all do)
+            assertBusy(() -> {
+                BulkByScrollTask.Status parent = (BulkByScrollTask.Status) client().admin().cluster().prepareGetTask(taskToRethrottle).get()
+                        .getTask().getTask().getStatus();
+                long finishedSubTasks = parent.getSliceStatuses().stream().filter(s -> s != null).count();
+                ListTasksResponse list = client().admin().cluster().prepareListTasks().setParentTaskId(taskToRethrottle).get();
+                list.rethrowFailures("subtasks");
+                assertThat(finishedSubTasks + list.getTasks().size(), greaterThanOrEqualTo((long) request.request().getSlices()));
+                assertThat(list.getTasks().size(), greaterThan(0));
+            });
+        }
 
         // Now rethrottle it so it'll finish
-        ListTasksResponse rethrottleResponse = rethrottle().setActions(actionName).setRequestsPerSecond(Float.POSITIVE_INFINITY).get();
+        float newRequestsPerSecond = randomBoolean() ? Float.POSITIVE_INFINITY : between(1, 1000) * 100000; // No throttle or "very fast"
+        ListTasksResponse rethrottleResponse = rethrottle().setTaskId(taskToRethrottle).setRequestsPerSecond(newRequestsPerSecond).get();
+        rethrottleResponse.rethrowFailures("Rethrottle");
         assertThat(rethrottleResponse.getTasks(), hasSize(1));
         BulkByScrollTask.Status status = (BulkByScrollTask.Status) rethrottleResponse.getTasks().get(0).getStatus();
-        assertEquals(Float.POSITIVE_INFINITY, status.getRequestsPerSecond(), Float.MIN_NORMAL);
+        // Now check the resulting requests per second.
+        if (request.request().getSlices() == 1) {
+            // If there is a single slice it should match perfectly
+            assertEquals(newRequestsPerSecond, status.getRequestsPerSecond(), Float.MIN_NORMAL);
+        } else {
+            /* Check that at least one slice was rethrottled. We won't always rethrottle all of them because they might have completed.
+             * With multiple slices these numbers might not add up perfectly, thus the 1.01F. */
+            long unfinished = status.getSliceStatuses().stream()
+                    .filter(slice -> slice != null)
+                    .filter(slice -> slice.getStatus().getTotal() > slice.getStatus().getSuccessfullyProcessed())
+                    .count();
+            float maxExpectedSliceRequestsPerSecond = newRequestsPerSecond == Float.POSITIVE_INFINITY ?
+                    Float.POSITIVE_INFINITY : (newRequestsPerSecond / unfinished) * 1.01F;
+            float minExpectedSliceRequestsPerSecond = newRequestsPerSecond == Float.POSITIVE_INFINITY ?
+                    Float.POSITIVE_INFINITY : (newRequestsPerSecond / request.request().getSlices()) * 0.99F;
+            boolean oneSliceRethrottled = false;
+            float totalRequestsPerSecond = 0;
+            for (BulkByScrollTask.StatusOrException statusOrException : status.getSliceStatuses()) {
+                if (statusOrException == null) {
+                    /* The slice can be null here because it was completed but hadn't reported its success back to the task when the
+                     * rethrottle request came through. */
+                    continue;
+                }
+                assertNull(statusOrException.getException());
+                BulkByScrollTask.Status slice = statusOrException.getStatus();
+                if (slice.getTotal() > slice.getSuccessfullyProcessed()) {
+                    // This slice reports as not having completed so it should have been processed.
+                    assertThat(slice.getRequestsPerSecond(), both(greaterThanOrEqualTo(minExpectedSliceRequestsPerSecond))
+                            .and(lessThanOrEqualTo(maxExpectedSliceRequestsPerSecond)));
+                }
+                if (minExpectedSliceRequestsPerSecond <= slice.getRequestsPerSecond()
+                        && slice.getRequestsPerSecond() <= maxExpectedSliceRequestsPerSecond) {
+                    oneSliceRethrottled = true;
+                }
+                totalRequestsPerSecond += slice.getRequestsPerSecond();
+            }
+            assertTrue("At least one slice must be rethrottled", oneSliceRethrottled);
+
+            /* Now assert that the parent request has the total requests per second. This is a much weaker assertion than that the parent
+             * actually has the newRequestsPerSecond. For the most part it will. Sometimes it'll be greater because only unfinished requests
+             * are rethrottled, the finished ones just keep whatever requests per second they had while they were running. But it might
+             * also be less than newRequestsPerSecond because the newRequestsPerSecond is divided among running sub-requests and then the
+             * requests are rethrottled. If one request finishes in between the division and the application of the new throttle then it
+             * won't be rethrottled, thus only contributing its lower total. */ 
+            assertEquals(totalRequestsPerSecond, status.getRequestsPerSecond(), totalRequestsPerSecond * 0.0001f);
+        }
 
         // Now the response should come back quickly because we've rethrottled the request
-        BulkIndexByScrollResponse response = responseListener.get();
-        assertEquals("Batches didn't match, this may invalidate the test as throttling is done between batches", 3, response.getBatches());
+        BulkByScrollResponse response = responseListener.get();
+        assertThat("Entire request completed in a single batch. This may invalidate the test as throttling is done between batches.",
+                response.getBatches(), greaterThanOrEqualTo(request.request().getSlices()));
+    }
+
+    private TaskGroup findTaskToRethrottle(String actionName, int sliceCount) {
+        long start = System.nanoTime();
+        do {
+            ListTasksResponse tasks = client().admin().cluster().prepareListTasks().setActions(actionName).setDetailed(true).get();
+            tasks.rethrowFailures("Finding tasks to rethrottle");
+            assertThat(tasks.getTaskGroups(), hasSize(lessThan(2)));
+            if (0 == tasks.getTaskGroups().size()) {
+                continue;
+            }
+            TaskGroup taskGroup = tasks.getTaskGroups().get(0);
+            if (sliceCount != 1 && taskGroup.getChildTasks().size() == 0) {
+                // If there are child tasks wait for at least one to start
+                continue;
+            }
+            return taskGroup;
+        } while (System.nanoTime() - start < TimeUnit.SECONDS.toNanos(10));
+        throw new AssertionError("Couldn't find tasks to rethrottle. Here are the running tasks " +
+                client().admin().cluster().prepareListTasks().get());
     }
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java
index 92e2598a5aaf3..9538cb8366116 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java
@@ -31,7 +31,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
-import org.elasticsearch.index.reindex.remote.RemoteInfo;
+import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -73,8 +73,8 @@ public void setUp() throws Exception {
         for (int i = 0; i < DOC_COUNT; i++) {
             bulk.add(client().prepareIndex("source", "test").setSource("foo", "bar " + i));
         }
-        Retry retry = Retry.on(EsRejectedExecutionException.class).policy(BackoffPolicy.exponentialBackoff());
-        BulkResponse response = retry.withSyncBackoff(client(), bulk.request());
+        Retry retry = Retry.on(EsRejectedExecutionException.class).policy(BackoffPolicy.exponentialBackoff()).using(client().threadPool());
+        BulkResponse response = retry.withSyncBackoff(client()::bulk, bulk.request(), client().settings());
         assertFalse(response.buildFailureMessage(), response.hasFailures());
         client().admin().indices().prepareRefresh("source").get();
     }
@@ -86,6 +86,12 @@ public void forceUnblockAllExecutors() {
         }
     }
 
+    // disabling, this one conflicts with the executor blocking
+    @Override
+    protected boolean enableWarningsCheck() {
+        return false;
+    }
+
     @Override
     protected Collection<Class<? extends Plugin>> getPlugins() {
         return pluginList(
@@ -109,7 +115,7 @@ protected Settings nodeSettings() {
         // Enable http so we can test retries on reindex from remote. In this case the "remote" cluster is just this cluster.
         settings.put(NetworkModule.HTTP_ENABLED.getKey(), true);
         // Whitelist reindexing from the http host we're going to use
-        settings.put(TransportReindexAction.REMOTE_CLUSTER_WHITELIST.getKey(), "myself");
+        settings.put(TransportReindexAction.REMOTE_CLUSTER_WHITELIST.getKey(), "127.0.0.1:*");
         if (useNetty3) {
             settings.put(NetworkModule.HTTP_TYPE_KEY, Netty3Plugin.NETTY_HTTP_TRANSPORT_NAME);
             settings.put(NetworkModule.TRANSPORT_TYPE_KEY, Netty3Plugin.NETTY_TRANSPORT_NAME);
@@ -125,8 +131,8 @@ public void testReindex() throws Exception {
     public void testReindexFromRemote() throws Exception {
         NodeInfo nodeInfo = client().admin().cluster().prepareNodesInfo().get().getNodes().get(0);
         TransportAddress address = nodeInfo.getHttp().getAddress().publishAddress();
-        RemoteInfo remote = new RemoteInfo("http", address.getHost(), address.getPort(), new BytesArray("{\"match_all\":{}}"), null, null,
-                emptyMap());
+        RemoteInfo remote = new RemoteInfo("http", address.getAddress(), address.getPort(), new BytesArray("{\"match_all\":{}}"), null,
+            null, emptyMap(), RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT);
         ReindexRequestBuilder request = ReindexAction.INSTANCE.newRequestBuilder(client()).source("source").destination("dest")
                 .setRemoteInfo(remote);
         testCase(ReindexAction.NAME, request, matcher().created(DOC_COUNT));
@@ -138,8 +144,8 @@ public void testUpdateByQuery() throws Exception {
     }
 
     public void testDeleteByQuery() throws Exception {
-        testCase(DeleteByQueryAction.NAME, DeleteByQueryAction.INSTANCE.newRequestBuilder(client()).source("source"),
-                matcher().deleted(DOC_COUNT));
+        testCase(DeleteByQueryAction.NAME, DeleteByQueryAction.INSTANCE.newRequestBuilder(client()).source("source")
+                .filter(QueryBuilders.matchAllQuery()),matcher().deleted(DOC_COUNT));
     }
 
     private void testCase(String action, AbstractBulkByScrollRequestBuilder<?, ?> request, BulkIndexByScrollResponseMatcher matcher)
@@ -151,7 +157,7 @@ private void testCase(String action, AbstractBulkByScrollRequestBuilder<?, ?> re
         request.source().setSize(DOC_COUNT / randomIntBetween(2, 10));
 
         logger.info("Starting request");
-        ListenableActionFuture<BulkIndexByScrollResponse> responseListener = request.execute();
+        ListenableActionFuture<BulkByScrollResponse> responseListener = request.execute();
 
         try {
             logger.info("Waiting for search rejections on the initial search");
@@ -178,13 +184,13 @@ private void testCase(String action, AbstractBulkByScrollRequestBuilder<?, ?> re
             scrollBlock.await();
 
             logger.info("Waiting for the request to finish");
-            BulkIndexByScrollResponse response = responseListener.get();
+            BulkByScrollResponse response = responseListener.get();
             assertThat(response, matcher);
             assertThat(response.getBulkRetries(), greaterThan(0L));
             assertThat(response.getSearchRetries(), greaterThan(initialSearchRejections));
         } finally {
             // Fetch the response just in case we blew up half way through. This will make sure the failure is thrown up to the top level.
-            BulkIndexByScrollResponse response = responseListener.get();
+            BulkByScrollResponse response = responseListener.get();
             assertThat(response.getSearchFailures(), empty());
             assertThat(response.getBulkFailures(), empty());
         }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java
index 1a262a32d3dc8..1eff5b4098964 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java
@@ -19,40 +19,34 @@
 
 package org.elasticsearch.index.reindex;
 
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.Streamable;
 import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.index.reindex.ScrollableHitSource.SearchFailure;
-import org.elasticsearch.index.reindex.remote.RemoteInfo;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 
-import static java.lang.Math.abs;
-import static java.util.Collections.emptyList;
-import static java.util.Collections.emptyMap;
-import static java.util.Collections.singletonList;
 import static org.apache.lucene.util.TestUtil.randomSimpleString;
 import static org.elasticsearch.common.unit.TimeValue.parseTimeValue;
-import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
 
 /**
  * Round trip tests for all Streamable things declared in this plugin.
  */
 public class RoundTripTests extends ESTestCase {
+
     public void testReindexRequest() throws IOException {
         ReindexRequest reindex = new ReindexRequest(new SearchRequest(), new IndexRequest());
         randomRequest(reindex);
@@ -60,47 +54,82 @@ public void testReindexRequest() throws IOException {
         reindex.getDestination().index("test");
         if (randomBoolean()) {
             int port = between(1, Integer.MAX_VALUE);
-            BytesReference query = new BytesArray(randomAsciiOfLength(5));
-            String username = randomBoolean() ? randomAsciiOfLength(5) : null;
-            String password = username != null && randomBoolean() ? randomAsciiOfLength(5) : null;
+            BytesReference query = new BytesArray(randomAlphaOfLength(5));
+            String username = randomBoolean() ? randomAlphaOfLength(5) : null;
+            String password = username != null && randomBoolean() ? randomAlphaOfLength(5) : null;
             int headersCount = randomBoolean() ? 0 : between(1, 10);
             Map<String, String> headers = new HashMap<>(headersCount);
             while (headers.size() < headersCount) {
-                headers.put(randomAsciiOfLength(5), randomAsciiOfLength(5));
+                headers.put(randomAlphaOfLength(5), randomAlphaOfLength(5));
             }
-            reindex.setRemoteInfo(new RemoteInfo(randomAsciiOfLength(5), randomAsciiOfLength(5), port, query, username, password, headers));
+            TimeValue socketTimeout = parseTimeValue(randomPositiveTimeValue(), "socketTimeout");
+            TimeValue connectTimeout = parseTimeValue(randomPositiveTimeValue(), "connectTimeout");
+            reindex.setRemoteInfo(new RemoteInfo(randomAlphaOfLength(5), randomAlphaOfLength(5), port, query, username, password, headers,
+                    socketTimeout, connectTimeout));
         }
         ReindexRequest tripped = new ReindexRequest();
         roundTrip(reindex, tripped);
         assertRequestEquals(reindex, tripped);
-        assertEquals(reindex.getDestination().version(), tripped.getDestination().version());
-        assertEquals(reindex.getDestination().index(), tripped.getDestination().index());
-        if (reindex.getRemoteInfo() == null) {
-            assertNull(tripped.getRemoteInfo());
-        } else {
-            assertNotNull(tripped.getRemoteInfo());
-            assertEquals(reindex.getRemoteInfo().getScheme(), tripped.getRemoteInfo().getScheme());
-            assertEquals(reindex.getRemoteInfo().getHost(), tripped.getRemoteInfo().getHost());
-            assertEquals(reindex.getRemoteInfo().getQuery(), tripped.getRemoteInfo().getQuery());
-            assertEquals(reindex.getRemoteInfo().getUsername(), tripped.getRemoteInfo().getUsername());
-            assertEquals(reindex.getRemoteInfo().getPassword(), tripped.getRemoteInfo().getPassword());
-            assertEquals(reindex.getRemoteInfo().getHeaders(), tripped.getRemoteInfo().getHeaders());
-        }
+
+        // Try slices with a version that doesn't support slices. That should fail.
+        reindex.setSlices(between(2, 1000));
+        Exception e = expectThrows(IllegalArgumentException.class, () -> roundTrip(Version.V_5_0_0_rc1, reindex, null));
+        assertEquals("Attempting to send sliced reindex-style request to a node that doesn't support it. "
+                + "Version is [5.0.0-rc1] but must be [5.1.1]", e.getMessage());
+
+        // Try without slices with a version that doesn't support slices. That should work.
+        tripped = new ReindexRequest();
+        reindex.setSlices(1);
+        roundTrip(Version.V_5_0_0_rc1, reindex, tripped);
+        assertRequestEquals(Version.V_5_0_0_rc1, reindex, tripped);
     }
 
     public void testUpdateByQueryRequest() throws IOException {
         UpdateByQueryRequest update = new UpdateByQueryRequest(new SearchRequest());
         randomRequest(update);
         if (randomBoolean()) {
-            update.setPipeline(randomAsciiOfLength(5));
+            update.setPipeline(randomAlphaOfLength(5));
         }
         UpdateByQueryRequest tripped = new UpdateByQueryRequest();
         roundTrip(update, tripped);
         assertRequestEquals(update, tripped);
         assertEquals(update.getPipeline(), tripped.getPipeline());
+
+        // Try slices with a version that doesn't support slices. That should fail.
+        update.setSlices(between(2, 1000));
+        Exception e = expectThrows(IllegalArgumentException.class, () -> roundTrip(Version.V_5_0_0_rc1, update, null));
+        assertEquals("Attempting to send sliced reindex-style request to a node that doesn't support it. "
+                + "Version is [5.0.0-rc1] but must be [5.1.1]", e.getMessage());
+
+        // Try without slices with a version that doesn't support slices. That should work.
+        tripped = new UpdateByQueryRequest();
+        update.setSlices(1);
+        roundTrip(Version.V_5_0_0_rc1, update, tripped);
+        assertRequestEquals(update, tripped);
+        assertEquals(update.getPipeline(), tripped.getPipeline());
     }
 
-    private void randomRequest(AbstractBulkIndexByScrollRequest<?> request) {
+    public void testDeleteByQueryRequest() throws IOException {
+        DeleteByQueryRequest delete = new DeleteByQueryRequest(new SearchRequest());
+        randomRequest(delete);
+        DeleteByQueryRequest tripped = new DeleteByQueryRequest();
+        roundTrip(delete, tripped);
+        assertRequestEquals(delete, tripped);
+
+        // Try slices with a version that doesn't support slices. That should fail.
+        delete.setSlices(between(2, 1000));
+        Exception e = expectThrows(IllegalArgumentException.class, () -> roundTrip(Version.V_5_0_0_rc1, delete, null));
+        assertEquals("Attempting to send sliced reindex-style request to a node that doesn't support it. "
+                + "Version is [5.0.0-rc1] but must be [5.1.1]", e.getMessage());
+
+        // Try without slices with a version that doesn't support slices. That should work.
+        tripped = new DeleteByQueryRequest();
+        delete.setSlices(1);
+        roundTrip(Version.V_5_0_0_rc1, delete, tripped);
+        assertRequestEquals(delete, tripped);
+    }
+
+    private void randomRequest(AbstractBulkByScrollRequest<?> request) {
         request.getSearchRequest().indices("test");
         request.getSearchRequest().source().size(between(1, 1000));
         request.setSize(random().nextBoolean() ? between(1, Integer.MAX_VALUE) : -1);
@@ -108,55 +137,64 @@ private void randomRequest(AbstractBulkIndexByScrollRequest<?> request) {
         request.setRefresh(rarely());
         request.setTimeout(TimeValue.parseTimeValue(randomTimeValue(), null, "test"));
         request.setWaitForActiveShards(randomIntBetween(0, 10));
-        request.setScript(random().nextBoolean() ? null : randomScript());
         request.setRequestsPerSecond(between(0, Integer.MAX_VALUE));
+        request.setSlices(between(1, Integer.MAX_VALUE));
+    }
+
+    private void randomRequest(AbstractBulkIndexByScrollRequest<?> request) {
+        randomRequest((AbstractBulkByScrollRequest<?>) request);
+        request.setScript(random().nextBoolean() ? null : randomScript());
+    }
+
+    private void assertRequestEquals(Version version, ReindexRequest request, ReindexRequest tripped) {
+        assertRequestEquals((AbstractBulkIndexByScrollRequest<?>) request, (AbstractBulkIndexByScrollRequest<?>) tripped);
+        assertEquals(request.getDestination().version(), tripped.getDestination().version());
+        assertEquals(request.getDestination().index(), tripped.getDestination().index());
+        if (request.getRemoteInfo() == null) {
+            assertNull(tripped.getRemoteInfo());
+        } else {
+            assertNotNull(tripped.getRemoteInfo());
+            assertEquals(request.getRemoteInfo().getScheme(), tripped.getRemoteInfo().getScheme());
+            assertEquals(request.getRemoteInfo().getHost(), tripped.getRemoteInfo().getHost());
+            assertEquals(request.getRemoteInfo().getQuery(), tripped.getRemoteInfo().getQuery());
+            assertEquals(request.getRemoteInfo().getUsername(), tripped.getRemoteInfo().getUsername());
+            assertEquals(request.getRemoteInfo().getPassword(), tripped.getRemoteInfo().getPassword());
+            assertEquals(request.getRemoteInfo().getHeaders(), tripped.getRemoteInfo().getHeaders());
+            if (version.onOrAfter(Version.V_5_2_0)) {
+                assertEquals(request.getRemoteInfo().getSocketTimeout(), tripped.getRemoteInfo().getSocketTimeout());
+                assertEquals(request.getRemoteInfo().getConnectTimeout(), tripped.getRemoteInfo().getConnectTimeout());
+            } else {
+                assertEquals(RemoteInfo.DEFAULT_SOCKET_TIMEOUT, tripped.getRemoteInfo().getSocketTimeout());
+                assertEquals(RemoteInfo.DEFAULT_CONNECT_TIMEOUT, tripped.getRemoteInfo().getConnectTimeout());
+            }
+        }
     }
 
     private void assertRequestEquals(AbstractBulkIndexByScrollRequest<?> request,
             AbstractBulkIndexByScrollRequest<?> tripped) {
+        assertRequestEquals((AbstractBulkByScrollRequest<?>) request, (AbstractBulkByScrollRequest<?>) tripped);
+        assertEquals(request.getScript(), tripped.getScript());
+    }
+
+    private void assertRequestEquals(AbstractBulkByScrollRequest<?> request, AbstractBulkByScrollRequest<?> tripped) {
         assertArrayEquals(request.getSearchRequest().indices(), tripped.getSearchRequest().indices());
         assertEquals(request.getSearchRequest().source().size(), tripped.getSearchRequest().source().size());
         assertEquals(request.isAbortOnVersionConflict(), tripped.isAbortOnVersionConflict());
         assertEquals(request.isRefresh(), tripped.isRefresh());
         assertEquals(request.getTimeout(), tripped.getTimeout());
         assertEquals(request.getWaitForActiveShards(), tripped.getWaitForActiveShards());
-        assertEquals(request.getScript(), tripped.getScript());
         assertEquals(request.getRetryBackoffInitialTime(), tripped.getRetryBackoffInitialTime());
         assertEquals(request.getMaxRetries(), tripped.getMaxRetries());
         assertEquals(request.getRequestsPerSecond(), tripped.getRequestsPerSecond(), 0d);
     }
 
-    public void testBulkByTaskStatus() throws IOException {
-        BulkByScrollTask.Status status = randomStatus();
-        BytesStreamOutput out = new BytesStreamOutput();
-        status.writeTo(out);
-        BulkByScrollTask.Status tripped = new BulkByScrollTask.Status(out.bytes().streamInput());
-        assertTaskStatusEquals(status, tripped);
-    }
-
-    public void testReindexResponse() throws IOException {
-        BulkIndexByScrollResponse response = new BulkIndexByScrollResponse(timeValueMillis(randomPositiveLong()), randomStatus(),
-                randomIndexingFailures(), randomSearchFailures(), randomBoolean());
-        BulkIndexByScrollResponse tripped = new BulkIndexByScrollResponse();
-        roundTrip(response, tripped);
-        assertResponseEquals(response, tripped);
-    }
-
-    public void testBulkIndexByScrollResponse() throws IOException {
-        BulkIndexByScrollResponse response = new BulkIndexByScrollResponse(timeValueMillis(randomPositiveLong()), randomStatus(),
-                randomIndexingFailures(), randomSearchFailures(), randomBoolean());
-        BulkIndexByScrollResponse tripped = new BulkIndexByScrollResponse();
-        roundTrip(response, tripped);
-        assertResponseEquals(response, tripped);
-    }
-
     public void testRethrottleRequest() throws IOException {
         RethrottleRequest request = new RethrottleRequest();
         request.setRequestsPerSecond((float) randomDoubleBetween(0, Float.POSITIVE_INFINITY, false));
         if (randomBoolean()) {
             request.setActions(randomFrom(UpdateByQueryAction.NAME, ReindexAction.NAME));
         } else {
-            request.setTaskId(new TaskId(randomAsciiOfLength(5), randomLong()));
+            request.setTaskId(new TaskId(randomAlphaOfLength(5), randomLong()));
         }
         RethrottleRequest tripped = new RethrottleRequest();
         roundTrip(request, tripped);
@@ -165,85 +203,25 @@ public void testRethrottleRequest() throws IOException {
         assertEquals(request.getTaskId(), tripped.getTaskId());
     }
 
-    private BulkByScrollTask.Status randomStatus() {
-        return new BulkByScrollTask.Status(randomPositiveLong(), randomPositiveLong(), randomPositiveLong(), randomPositiveLong(),
-                randomInt(Integer.MAX_VALUE), randomPositiveLong(), randomPositiveLong(), randomPositiveLong(), randomPositiveLong(),
-                parseTimeValue(randomPositiveTimeValue(), "test"), abs(random().nextFloat()),
-                random().nextBoolean() ? null : randomSimpleString(random()), parseTimeValue(randomPositiveTimeValue(), "test"));
-    }
-
-    private List<Failure> randomIndexingFailures() {
-        return usually() ? emptyList()
-                : singletonList(new Failure(randomSimpleString(random()), randomSimpleString(random()),
-                        randomSimpleString(random()), new IllegalArgumentException("test")));
-    }
-
-    private List<SearchFailure> randomSearchFailures() {
-        if (randomBoolean()) {
-            return emptyList();
-        }
-        String index = null;
-        Integer shardId = null;
-        String nodeId = null;
-        if (randomBoolean()) {
-            index = randomAsciiOfLength(5);
-            shardId = randomInt();
-            nodeId = usually() ? randomAsciiOfLength(5) : null;
-        }
-        return singletonList(new SearchFailure(new ElasticsearchException("foo"), index, shardId, nodeId));
+    private void roundTrip(Streamable example, Streamable empty) throws IOException {
+        roundTrip(Version.CURRENT, example, empty);
     }
 
-    private void roundTrip(Streamable example, Streamable empty) throws IOException {
+    private void roundTrip(Version version, Streamable example, Streamable empty) throws IOException {
         BytesStreamOutput out = new BytesStreamOutput();
+        out.setVersion(version);
         example.writeTo(out);
-        empty.readFrom(out.bytes().streamInput());
+        StreamInput in = out.bytes().streamInput();
+        in.setVersion(version);
+        empty.readFrom(in);
     }
 
     private Script randomScript() {
-        return new Script(randomSimpleString(random()), // Name
-                randomFrom(ScriptType.values()), // Type
-                random().nextBoolean() ? null : randomSimpleString(random()), // Language
-                emptyMap()); // Params
-    }
-
-    private void assertResponseEquals(BulkIndexByScrollResponse expected, BulkIndexByScrollResponse actual) {
-        assertEquals(expected.getTook(), actual.getTook());
-        assertTaskStatusEquals(expected.getStatus(), actual.getStatus());
-        assertEquals(expected.getBulkFailures().size(), actual.getBulkFailures().size());
-        for (int i = 0; i < expected.getBulkFailures().size(); i++) {
-            Failure expectedFailure = expected.getBulkFailures().get(i);
-            Failure actualFailure = actual.getBulkFailures().get(i);
-            assertEquals(expectedFailure.getIndex(), actualFailure.getIndex());
-            assertEquals(expectedFailure.getType(), actualFailure.getType());
-            assertEquals(expectedFailure.getId(), actualFailure.getId());
-            assertEquals(expectedFailure.getMessage(), actualFailure.getMessage());
-            assertEquals(expectedFailure.getStatus(), actualFailure.getStatus());
-        }
-        assertEquals(expected.getSearchFailures().size(), actual.getSearchFailures().size());
-        for (int i = 0; i < expected.getSearchFailures().size(); i++) {
-            SearchFailure expectedFailure = expected.getSearchFailures().get(i);
-            SearchFailure actualFailure = actual.getSearchFailures().get(i);
-            assertEquals(expectedFailure.getIndex(), actualFailure.getIndex());
-            assertEquals(expectedFailure.getShardId(), actualFailure.getShardId());
-            assertEquals(expectedFailure.getNodeId(), actualFailure.getNodeId());
-            assertEquals(expectedFailure.getReason().getClass(), actualFailure.getReason().getClass());
-            assertEquals(expectedFailure.getReason().getMessage(), actualFailure.getReason().getMessage());
-        }
-
-    }
+        ScriptType type = randomFrom(ScriptType.values());
+        String lang = random().nextBoolean() ? Script.DEFAULT_SCRIPT_LANG : randomSimpleString(random());
+        String idOrCode = randomSimpleString(random());
+        Map<String, Object> params = Collections.emptyMap();
 
-    private void assertTaskStatusEquals(BulkByScrollTask.Status expected, BulkByScrollTask.Status actual) {
-        assertEquals(expected.getUpdated(), actual.getUpdated());
-        assertEquals(expected.getCreated(), actual.getCreated());
-        assertEquals(expected.getDeleted(), actual.getDeleted());
-        assertEquals(expected.getBatches(), actual.getBatches());
-        assertEquals(expected.getVersionConflicts(), actual.getVersionConflicts());
-        assertEquals(expected.getNoops(), actual.getNoops());
-        assertEquals(expected.getBulkRetries(), actual.getBulkRetries());
-        assertEquals(expected.getSearchRetries(), actual.getSearchRetries());
-        assertEquals(expected.getThrottled(), actual.getThrottled());
-        assertEquals(expected.getRequestsPerSecond(), actual.getRequestsPerSecond(), 0f);
-        assertEquals(expected.getReasonCancelled(), actual.getReasonCancelled());
-        assertEquals(expected.getThrottledUntil(), actual.getThrottledUntil());
+        return new Script(type, lang, idOrCode, params);
     }
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/TransportRethrottleActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/TransportRethrottleActionTests.java
new file mode 100644
index 0000000000000..222aedd2e9eeb
--- /dev/null
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/TransportRethrottleActionTests.java
@@ -0,0 +1,210 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.FailedNodeException;
+import org.elasticsearch.action.TaskOperationFailure;
+import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.tasks.TaskId;
+import org.elasticsearch.tasks.TaskInfo;
+import org.elasticsearch.test.ESTestCase;
+import org.hamcrest.Matcher;
+import org.junit.Before;
+import org.mockito.ArgumentCaptor;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Consumer;
+
+import static java.util.Collections.emptyList;
+import static java.util.Collections.singletonList;
+import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.hasToString;
+import static org.hamcrest.Matchers.theInstance;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.atMost;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+
+public class TransportRethrottleActionTests extends ESTestCase {
+    private int slices;
+    private ParentBulkByScrollTask task;
+
+    @Before
+    public void createTask() {
+        slices = between(2, 50);
+        task = new ParentBulkByScrollTask(1, "test_type", "test_action", "test", null, slices);
+    }
+
+    /**
+     * Test rethrottling.
+     * @param runningSlices the number of slices still running
+     * @param simulator simulate a response from the sub-request to rethrottle the child requests
+     * @param verifier verify the resulting response
+     */
+    private void rethrottleTestCase(int runningSlices, Consumer<ActionListener<ListTasksResponse>> simulator,
+            Consumer<ActionListener<TaskInfo>> verifier) {
+        Client client = mock(Client.class);
+        String localNodeId = randomAlphaOfLength(5);
+        float newRequestsPerSecond = randomValueOtherThanMany(f -> f <= 0, () -> randomFloat());
+        @SuppressWarnings("unchecked")
+        ActionListener<TaskInfo> listener = mock(ActionListener.class);
+
+        TransportRethrottleAction.rethrottle(logger, localNodeId, client, task, newRequestsPerSecond, listener);
+
+        // Capture the sub request and the listener so we can verify they are sane
+        ArgumentCaptor<RethrottleRequest> subRequest = ArgumentCaptor.forClass(RethrottleRequest.class);
+        @SuppressWarnings({ "unchecked", "rawtypes" }) // Magical generics incantation.....
+        ArgumentCaptor<ActionListener<ListTasksResponse>> subListener = ArgumentCaptor.forClass((Class) ActionListener.class);
+        if (runningSlices > 0) {
+            verify(client).execute(eq(RethrottleAction.INSTANCE), subRequest.capture(), subListener.capture());
+
+            assertEquals(new TaskId(localNodeId, task.getId()), subRequest.getValue().getParentTaskId());
+            assertEquals(newRequestsPerSecond / runningSlices, subRequest.getValue().getRequestsPerSecond(), 0.00001f);
+
+            simulator.accept(subListener.getValue());
+        }
+        verifier.accept(listener);
+    }
+
+    private Consumer<ActionListener<TaskInfo>> expectSuccessfulRethrottleWithStatuses(
+            List<BulkByScrollTask.StatusOrException> sliceStatuses) {
+        return listener -> {
+            TaskInfo taskInfo = captureResponse(TaskInfo.class, listener);
+            assertEquals(sliceStatuses, ((BulkByScrollTask.Status) taskInfo.getStatus()).getSliceStatuses());
+        };
+    }
+
+    public void testRethrottleSuccessfulResponse() {
+        List<TaskInfo> tasks = new ArrayList<>();
+        List<BulkByScrollTask.StatusOrException> sliceStatuses = new ArrayList<>(slices);
+        for (int i = 0; i < slices; i++) {
+            BulkByScrollTask.Status status = believeableInProgressStatus(i);
+            tasks.add(new TaskInfo(new TaskId("test", 123), "test", "test", "test", status, 0, 0, true, new TaskId("test", task.getId())));
+            sliceStatuses.add(new BulkByScrollTask.StatusOrException(status));
+        }
+        rethrottleTestCase(slices,
+                listener -> listener.onResponse(new ListTasksResponse(tasks, emptyList(), emptyList())),
+                expectSuccessfulRethrottleWithStatuses(sliceStatuses));
+    }
+
+    public void testRethrottleWithSomeSucceeded() {
+        int succeeded = between(1, slices - 1);
+        List<BulkByScrollTask.StatusOrException> sliceStatuses = new ArrayList<>(slices);
+        for (int i = 0; i < succeeded; i++) {
+            BulkByScrollTask.Status status = believeableCompletedStatus(i);
+            task.onSliceResponse(neverCalled(), i,
+                    new BulkByScrollResponse(timeValueMillis(10), status, emptyList(), emptyList(), false));
+            sliceStatuses.add(new BulkByScrollTask.StatusOrException(status));
+        }
+        List<TaskInfo> tasks = new ArrayList<>();
+        for (int i = succeeded; i < slices; i++) {
+            BulkByScrollTask.Status status = believeableInProgressStatus(i);
+            tasks.add(new TaskInfo(new TaskId("test", 123), "test", "test", "test", status, 0, 0, true, new TaskId("test", task.getId())));
+            sliceStatuses.add(new BulkByScrollTask.StatusOrException(status));
+        }
+        rethrottleTestCase(slices - succeeded,
+                listener -> listener.onResponse(new ListTasksResponse(tasks, emptyList(), emptyList())),
+                expectSuccessfulRethrottleWithStatuses(sliceStatuses));
+    }
+
+    public void testRethrottleWithAllSucceeded() {
+        List<BulkByScrollTask.StatusOrException> sliceStatuses = new ArrayList<>(slices);
+        for (int i = 0; i < slices; i++) {
+            @SuppressWarnings("unchecked")
+            ActionListener<BulkByScrollResponse> listener = i < slices - 1 ? neverCalled() : mock(ActionListener.class);
+            BulkByScrollTask.Status status = believeableCompletedStatus(i);
+            task.onSliceResponse(listener, i, new BulkByScrollResponse(timeValueMillis(10), status, emptyList(), emptyList(), false));
+            if (i == slices - 1) {
+                // The whole thing succeeded so we should have got the success
+                captureResponse(BulkByScrollResponse.class, listener).getStatus();
+            }
+            sliceStatuses.add(new BulkByScrollTask.StatusOrException(status));
+        }
+        rethrottleTestCase(0,
+                listener -> { /* There are no async tasks to simulate because the listener is called for us. */},
+                expectSuccessfulRethrottleWithStatuses(sliceStatuses));
+    }
+
+    private Consumer<ActionListener<TaskInfo>> expectException(Matcher<Exception> exceptionMatcher) {
+        return listener -> {
+            ArgumentCaptor<Exception> failure = ArgumentCaptor.forClass(Exception.class);
+            verify(listener).onFailure(failure.capture());
+            assertThat(failure.getValue(), exceptionMatcher);
+        };
+    }
+
+    public void testRethrottleCatastrophicFailures() {
+        Exception e = new Exception();
+        rethrottleTestCase(slices, listener -> listener.onFailure(e), expectException(theInstance(e)));
+    }
+
+    public void testRethrottleTaskOperationFailure() {
+        Exception e = new Exception();
+        TaskOperationFailure failure = new TaskOperationFailure("test", 123, e);
+        rethrottleTestCase(slices,
+                listener -> listener.onResponse(new ListTasksResponse(emptyList(), singletonList(failure), emptyList())),
+                expectException(hasToString(containsString("Rethrottle of [test:123] failed"))));
+    }
+
+    public void testRethrottleNodeFailure() {
+        FailedNodeException e = new FailedNodeException("test", "test", new Exception());
+        rethrottleTestCase(slices,
+                listener -> listener.onResponse(new ListTasksResponse(emptyList(), emptyList(), singletonList(e))),
+                expectException(theInstance(e)));
+    }
+
+    private BulkByScrollTask.Status believeableInProgressStatus(Integer sliceId) {
+        return new BulkByScrollTask.Status(sliceId, 10, 0, 0, 0, 0, 0, 0, 0, 0, timeValueMillis(0), 0, null, timeValueMillis(0));
+    }
+
+    private BulkByScrollTask.Status believeableCompletedStatus(Integer sliceId) {
+        return new BulkByScrollTask.Status(sliceId, 10, 10, 0, 0, 0, 0, 0, 0, 0, timeValueMillis(0), 0, null, timeValueMillis(0));
+    }
+
+    private <T> ActionListener<T> neverCalled() {
+        return new ActionListener<T>() {
+            @Override
+            public void onResponse(T response) {
+                throw new RuntimeException("Expected no interactions but got [" + response + "]");
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                throw new RuntimeException("Expected no interations but was received a failure", e);
+            }
+        };
+    }
+
+    private <T> T captureResponse(Class<T> responseClass, ActionListener<T> listener) {
+        ArgumentCaptor<Exception> failure = ArgumentCaptor.forClass(Exception.class);
+        // Rethrow any failures just so we get a nice exception if there were any. We don't expect any though.
+        verify(listener, atMost(1)).onFailure(failure.capture());
+        if (false == failure.getAllValues().isEmpty()) {
+            throw new AssertionError(failure.getValue());
+        }
+        ArgumentCaptor<T> response = ArgumentCaptor.forClass(responseClass);
+        verify(listener).onResponse(response.capture());
+        return response.getValue();
+    }
+}
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java
index 773e693a9bf4c..663575a293391 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java
@@ -23,6 +23,7 @@
 
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.hamcrest.Matchers.hasSize;
 
 public class UpdateByQueryBasicTests extends ReindexTestCase {
     public void testBasics() throws Exception {
@@ -61,4 +62,33 @@ public void testBasics() throws Exception {
         assertEquals(3, client().prepareGet("test", "test", "3").get().getVersion());
         assertEquals(2, client().prepareGet("test", "test", "4").get().getVersion());
     }
+
+    public void testWorkers() throws Exception {
+        indexRandom(true, client().prepareIndex("test", "test", "1").setSource("foo", "a"),
+                client().prepareIndex("test", "test", "2").setSource("foo", "a"),
+                client().prepareIndex("test", "test", "3").setSource("foo", "b"),
+                client().prepareIndex("test", "test", "4").setSource("foo", "c"));
+        assertHitCount(client().prepareSearch("test").setTypes("test").setSize(0).get(), 4);
+        assertEquals(1, client().prepareGet("test", "test", "1").get().getVersion());
+        assertEquals(1, client().prepareGet("test", "test", "4").get().getVersion());
+
+        // Reindex all the docs
+        assertThat(updateByQuery().source("test").refresh(true).setSlices(5).get(), matcher().updated(4).slices(hasSize(5)));
+        assertEquals(2, client().prepareGet("test", "test", "1").get().getVersion());
+        assertEquals(2, client().prepareGet("test", "test", "4").get().getVersion());
+
+        // Now none of them
+        assertThat(updateByQuery().source("test").filter(termQuery("foo", "no_match")).setSlices(5).refresh(true).get(),
+                matcher().updated(0).slices(hasSize(5)));
+        assertEquals(2, client().prepareGet("test", "test", "1").get().getVersion());
+        assertEquals(2, client().prepareGet("test", "test", "4").get().getVersion());
+
+        // Now half of them
+        assertThat(updateByQuery().source("test").filter(termQuery("foo", "a")).refresh(true).setSlices(5).get(),
+                matcher().updated(2).slices(hasSize(5)));
+        assertEquals(3, client().prepareGet("test", "test", "1").get().getVersion());
+        assertEquals(3, client().prepareGet("test", "test", "2").get().getVersion());
+        assertEquals(2, client().prepareGet("test", "test", "3").get().getVersion());
+        assertEquals(2, client().prepareGet("test", "test", "4").get().getVersion());
+    }
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryMetadataTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryMetadataTests.java
index 6ebb0749792da..b688ce019e3df 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryMetadataTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryMetadataTests.java
@@ -19,24 +19,40 @@
 
 package org.elasticsearch.index.reindex;
 
+import org.elasticsearch.index.reindex.ScrollableHitSource.Hit;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.common.settings.Settings;
 
 public class UpdateByQueryMetadataTests
-        extends AbstractAsyncBulkIndexbyScrollActionMetadataTestCase<UpdateByQueryRequest, BulkIndexByScrollResponse> {
+        extends AbstractAsyncBulkByScrollActionMetadataTestCase<UpdateByQueryRequest, BulkByScrollResponse> {
     public void testRoutingIsCopied() throws Exception {
         IndexRequest index = new IndexRequest();
-        action().copyMetadata(AbstractAsyncBulkIndexByScrollAction.wrap(index), doc().setRouting("foo"));
+        action().copyMetadata(AbstractAsyncBulkByScrollAction.wrap(index), doc().setRouting("foo"));
         assertEquals("foo", index.routing());
     }
 
     @Override
-    protected TransportUpdateByQueryAction.AsyncIndexBySearchAction action() {
-        return new TransportUpdateByQueryAction.AsyncIndexBySearchAction(task, logger, null, threadPool, request(), listener(), null, null);
+    protected TestAction action() {
+        return new TestAction();
     }
 
     @Override
     protected UpdateByQueryRequest request() {
         return new UpdateByQueryRequest(new SearchRequest());
     }
+
+    private class TestAction extends TransportUpdateByQueryAction.AsyncIndexBySearchAction {
+        TestAction() {
+            super(UpdateByQueryMetadataTests.this.task, UpdateByQueryMetadataTests.this.logger, null,
+                    UpdateByQueryMetadataTests.this.threadPool, request(), null, null, listener(),
+                Settings.EMPTY);
+        }
+
+        @Override
+        public AbstractAsyncBulkByScrollAction.RequestWrapper<?> copyMetadata(AbstractAsyncBulkByScrollAction.RequestWrapper<?> request,
+                Hit doc) {
+            return super.copyMetadata(request, doc);
+        }
+    }
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryRequestTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryRequestTests.java
deleted file mode 100644
index a4224a70c57a8..0000000000000
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryRequestTests.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.reindex;
-
-import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.test.ESTestCase;
-
-import static org.apache.lucene.util.TestUtil.randomSimpleString;
-
-public class UpdateByQueryRequestTests extends ESTestCase {
-    public void testUpdateByQueryRequestImplementsIndicesRequestReplaceable() {
-        int numIndices = between(1, 100);
-        String[] indices = new String[numIndices];
-        for (int i = 0; i < numIndices; i++) {
-            indices[i] = randomSimpleString(random(), 1, 30);
-        }
-
-        SearchRequest searchRequest = new SearchRequest(indices);
-        IndicesOptions indicesOptions = IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean());
-        searchRequest.indicesOptions(indicesOptions);
-
-        UpdateByQueryRequest request = new UpdateByQueryRequest(searchRequest);
-        for (int i = 0; i < numIndices; i++) {
-            assertEquals(indices[i], request.indices()[i]);
-        }
-
-        assertSame(indicesOptions, request.indicesOptions());
-        assertSame(request.indicesOptions(), request.getSearchRequest().indicesOptions());
-
-        int numNewIndices = between(1, 100);
-        String[] newIndices = new String[numNewIndices];
-        for (int i = 0; i < numNewIndices; i++) {
-            newIndices[i] = randomSimpleString(random(), 1, 30);
-        }
-        request.indices(newIndices);
-        for (int i = 0; i < numNewIndices; i++) {;
-            assertEquals(newIndices[i], request.indices()[i]);
-        }
-        for (int i = 0; i < numNewIndices; i++) {;
-            assertEquals(newIndices[i], request.getSearchRequest().indices()[i]);
-        }
-    }
-}
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWhileModifyingTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWhileModifyingTests.java
index 6bbcbd6e643b6..987830ddd3bc9 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWhileModifyingTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWhileModifyingTests.java
@@ -38,7 +38,7 @@
  */
 public class UpdateByQueryWhileModifyingTests extends ReindexTestCase {
     private static final int MAX_MUTATIONS = 50;
-    private static final int MAX_ATTEMPTS = 10;
+    private static final int MAX_ATTEMPTS = 50;
 
     public void testUpdateWhileReindexing() throws Exception {
         AtomicReference<String> value = new AtomicReference<>(randomSimpleString(random()));
@@ -49,7 +49,7 @@ public void testUpdateWhileReindexing() throws Exception {
         Thread updater = new Thread(() -> {
             while (keepUpdating.get()) {
                 try {
-                    BulkIndexByScrollResponse response = updateByQuery().source("test").refresh(true).abortOnVersionConflict(false).get();
+                    BulkByScrollResponse response = updateByQuery().source("test").refresh(true).abortOnVersionConflict(false).get();
                     assertThat(response, matcher().updated(either(equalTo(0L)).or(equalTo(1L)))
                             .versionConflicts(either(equalTo(0L)).or(equalTo(1L))));
                 } catch (Exception e) {
@@ -82,9 +82,8 @@ public void testUpdateWhileReindexing() throws Exception {
                             throw new RuntimeException(
                                     "Failed to index after [" + MAX_ATTEMPTS + "] attempts. Too many version conflicts!");
                         }
-                        logger.info(
-                                "Caught expected version conflict trying to perform mutation number {} with version {}. Retrying.",
-                                i, get.getVersion());
+                        logger.info("Caught expected version conflict trying to perform mutation number [{}] with version [{}] "
+                                + "on attempt [{}]. Retrying.", i, get.getVersion(), attempts);
                         get = client().prepareGet("test", "test", "test").get();
                     }
                 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWithScriptTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWithScriptTests.java
index c5b9d4da64fe6..608dad41c03e3 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWithScriptTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWithScriptTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.index.reindex;
 
 import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.script.ScriptService;
 
 import java.util.Date;
@@ -28,7 +29,7 @@
 import static org.hamcrest.Matchers.containsString;
 
 public class UpdateByQueryWithScriptTests
-        extends AbstractAsyncBulkIndexByScrollActionScriptTestCase<UpdateByQueryRequest, BulkIndexByScrollResponse> {
+        extends AbstractAsyncBulkByScrollActionScriptTestCase<UpdateByQueryRequest, BulkByScrollResponse> {
 
     public void testModifyingCtxNotAllowed() {
         /*
@@ -53,8 +54,8 @@ protected UpdateByQueryRequest request() {
     }
 
     @Override
-    protected AbstractAsyncBulkIndexByScrollAction<UpdateByQueryRequest> action(ScriptService scriptService, UpdateByQueryRequest request) {
-        return new TransportUpdateByQueryAction.AsyncIndexBySearchAction(task, logger, null, threadPool, request, listener(),
-                scriptService, null);
+    protected TransportUpdateByQueryAction.AsyncIndexBySearchAction action(ScriptService scriptService, UpdateByQueryRequest request) {
+        return new TransportUpdateByQueryAction.AsyncIndexBySearchAction(task, logger, null, threadPool, request, scriptService, null,
+                listener(), Settings.EMPTY);
     }
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteInfoTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteInfoTests.java
index 3ee647aa55bb2..d6ab599b43c2d 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteInfoTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteInfoTests.java
@@ -19,20 +19,24 @@
 
 package org.elasticsearch.index.reindex.remote;
 
+import org.elasticsearch.index.reindex.RemoteInfo;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.test.ESTestCase;
 
 import static java.util.Collections.emptyMap;
 
 public class RemoteInfoTests extends ESTestCase {
+    private RemoteInfo newRemoteInfo(String scheme, String username, String password) {
+        return new RemoteInfo(scheme, "testhost", 12344, new BytesArray("testquery"), username, password, emptyMap(),
+                RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT);
+    }
+
     public void testToString() {
-        RemoteInfo info = new RemoteInfo("http", "testhost", 12344, new BytesArray("testquery"), null, null, emptyMap());
-        assertEquals("host=testhost port=12344 query=testquery", info.toString());
-        info = new RemoteInfo("http", "testhost", 12344, new BytesArray("testquery"), "testuser", null, emptyMap());
-        assertEquals("host=testhost port=12344 query=testquery username=testuser", info.toString());
-        info = new RemoteInfo("http", "testhost", 12344, new BytesArray("testquery"), "testuser", "testpass", emptyMap());
-        assertEquals("host=testhost port=12344 query=testquery username=testuser password=<<>>", info.toString());
-        info = new RemoteInfo("https", "testhost", 12344, new BytesArray("testquery"), "testuser", "testpass", emptyMap());
-        assertEquals("scheme=https host=testhost port=12344 query=testquery username=testuser password=<<>>", info.toString());
+        assertEquals("host=testhost port=12344 query=testquery", newRemoteInfo("http", null, null).toString());
+        assertEquals("host=testhost port=12344 query=testquery username=testuser", newRemoteInfo("http", "testuser", null).toString());
+        assertEquals("host=testhost port=12344 query=testquery username=testuser password=<<>>",
+                newRemoteInfo("http", "testuser", "testpass").toString());
+        assertEquals("scheme=https host=testhost port=12344 query=testquery username=testuser password=<<>>",
+                newRemoteInfo("https", "testuser", "testpass").toString());
     }
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuildersTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuildersTests.java
index 9bbfd175a79e6..e1ad398169ff5 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuildersTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuildersTests.java
@@ -34,6 +34,8 @@
 import java.nio.charset.StandardCharsets;
 import java.util.Map;
 
+import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
+import static org.elasticsearch.index.reindex.remote.RemoteRequestBuilders.clearScrollEntity;
 import static org.elasticsearch.index.reindex.remote.RemoteRequestBuilders.initialSearchEntity;
 import static org.elasticsearch.index.reindex.remote.RemoteRequestBuilders.initialSearchParams;
 import static org.elasticsearch.index.reindex.remote.RemoteRequestBuilders.initialSearchPath;
@@ -41,6 +43,7 @@
 import static org.elasticsearch.index.reindex.remote.RemoteRequestBuilders.scrollParams;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.either;
+import static org.hamcrest.Matchers.endsWith;
 import static org.hamcrest.Matchers.hasEntry;
 import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.not;
@@ -96,7 +99,7 @@ public void testInitialSearchParamsSort() {
         // Test sort:_doc for versions that support it.
         Version remoteVersion = Version.fromId(between(Version.V_2_1_0_ID, Version.CURRENT.id));
         searchRequest.source().sort("_doc");
-        assertThat(initialSearchParams(searchRequest, remoteVersion), hasEntry("sorts", "_doc:asc"));
+        assertThat(initialSearchParams(searchRequest, remoteVersion), hasEntry("sort", "_doc:asc"));
 
         // Test search_type scan for versions that don't support sort:_doc.
         remoteVersion = Version.fromId(between(0, Version.V_2_1_0_ID - 1));
@@ -106,27 +109,41 @@ public void testInitialSearchParamsSort() {
         remoteVersion = Version.fromId(between(0, Version.CURRENT.id));
         searchRequest.source().sorts().clear();
         searchRequest.source().sort("foo");
-        assertThat(initialSearchParams(searchRequest, remoteVersion), hasEntry("sorts", "foo:asc"));
+        assertThat(initialSearchParams(searchRequest, remoteVersion), hasEntry("sort", "foo:asc"));
     }
 
     public void testInitialSearchParamsFields() {
         SearchRequest searchRequest = new SearchRequest().source(new SearchSourceBuilder());
 
         // Test request without any fields
-        Version remoteVersion = Version.fromId(between(0, Version.CURRENT.id));
+        Version remoteVersion = Version.fromId(between(Version.V_2_0_0_beta1_ID, Version.CURRENT.id));
         assertThat(initialSearchParams(searchRequest, remoteVersion),
                 not(either(hasKey("stored_fields")).or(hasKey("fields"))));
 
-        // Setup some fields for the next two tests
-        searchRequest.source().storedField("_source").storedField("_id");
-
         // Test stored_fields for versions that support it
+        searchRequest = new SearchRequest().source(new SearchSourceBuilder());
+        searchRequest.source().storedField("_source").storedField("_id");
         remoteVersion = Version.fromId(between(Version.V_5_0_0_alpha4_ID, Version.CURRENT.id));
         assertThat(initialSearchParams(searchRequest, remoteVersion), hasEntry("stored_fields", "_source,_id"));
 
         // Test fields for versions that support it
-        remoteVersion = Version.fromId(between(0, Version.V_5_0_0_alpha4_ID - 1));
+        searchRequest = new SearchRequest().source(new SearchSourceBuilder());
+        searchRequest.source().storedField("_source").storedField("_id");
+        remoteVersion = Version.fromId(between(Version.V_2_0_0_beta1_ID, Version.V_5_0_0_alpha4_ID - 1));
         assertThat(initialSearchParams(searchRequest, remoteVersion), hasEntry("fields", "_source,_id"));
+
+        // Test extra fields for versions that need it
+        searchRequest = new SearchRequest().source(new SearchSourceBuilder());
+        searchRequest.source().storedField("_source").storedField("_id");
+        remoteVersion = Version.fromId(between(0, Version.V_2_0_0_beta1_ID - 1));
+        assertThat(initialSearchParams(searchRequest, remoteVersion), hasEntry("fields", "_source,_id,_parent,_routing,_ttl"));
+
+        // But only versions before 1.0 force _source to be in the list
+        searchRequest = new SearchRequest().source(new SearchSourceBuilder());
+        searchRequest.source().storedField("_id");
+        remoteVersion = Version.fromId(between(1000099, Version.V_2_0_0_beta1_ID - 1));
+        assertThat(initialSearchParams(searchRequest, remoteVersion), hasEntry("fields", "_id,_parent,_routing,_ttl"));
+
     }
 
     public void testInitialSearchParamsMisc() {
@@ -148,33 +165,87 @@ public void testInitialSearchParamsMisc() {
 
         Map<String, String> params = initialSearchParams(searchRequest, remoteVersion);
 
-        assertThat(params, scroll == null ? not(hasKey("scroll")) : hasEntry("scroll", scroll.toString()));
+        if (scroll == null) {
+            assertThat(params, not(hasKey("scroll")));
+        } else {
+            assertScroll(remoteVersion, params, scroll);
+        }
         assertThat(params, hasEntry("size", Integer.toString(size)));
         assertThat(params, fetchVersion == null || fetchVersion == true ? hasEntry("version", null) : not(hasEntry("version", null)));
     }
 
+    private void assertScroll(Version remoteVersion, Map<String, String> params, TimeValue requested) {
+        if (remoteVersion.before(Version.V_5_0_0)) {
+            // Versions of Elasticsearch prior to 5.0 can't parse nanos or micros in TimeValue.
+            assertThat(params.get("scroll"), not(either(endsWith("nanos")).or(endsWith("micros"))));
+            if (requested.getStringRep().endsWith("nanos") || requested.getStringRep().endsWith("micros")) {
+                long millis = (long) Math.ceil(requested.millisFrac());
+                assertEquals(TimeValue.parseTimeValue(params.get("scroll"), "scroll"), timeValueMillis(millis));
+                return;
+            }
+        }
+        assertEquals(requested, TimeValue.parseTimeValue(params.get("scroll"), "scroll"));
+    }
+
     public void testInitialSearchEntity() throws IOException {
+        Version remoteVersion = Version.fromId(between(0, Version.CURRENT.id));
+
+        SearchRequest searchRequest = new SearchRequest();
+        searchRequest.source(new SearchSourceBuilder());
         String query = "{\"match_all\":{}}";
-        HttpEntity entity = initialSearchEntity(new BytesArray(query));
+        HttpEntity entity = initialSearchEntity(searchRequest, new BytesArray(query), remoteVersion);
+        assertEquals(ContentType.APPLICATION_JSON.toString(), entity.getContentType().getValue());
+        if (remoteVersion.onOrAfter(Version.fromId(1000099))) {
+            assertEquals("{\"query\":" + query + ",\"_source\":true}",
+                    Streams.copyToString(new InputStreamReader(entity.getContent(), StandardCharsets.UTF_8)));
+        } else {
+            assertEquals("{\"query\":" + query + "}",
+                    Streams.copyToString(new InputStreamReader(entity.getContent(), StandardCharsets.UTF_8)));
+        }
+
+        // Source filtering is included if set up
+        searchRequest.source().fetchSource(new String[] {"in1", "in2"}, new String[] {"out"});
+        entity = initialSearchEntity(searchRequest, new BytesArray(query), remoteVersion);
         assertEquals(ContentType.APPLICATION_JSON.toString(), entity.getContentType().getValue());
-        assertEquals("{\"query\":" + query + "}",
+        assertEquals("{\"query\":" + query + ",\"_source\":{\"includes\":[\"in1\",\"in2\"],\"excludes\":[\"out\"]}}",
                 Streams.copyToString(new InputStreamReader(entity.getContent(), StandardCharsets.UTF_8)));
 
         // Invalid XContent fails
-        RuntimeException e = expectThrows(RuntimeException.class, () -> initialSearchEntity(new BytesArray("{}, \"trailing\": {}")));
+        RuntimeException e = expectThrows(RuntimeException.class,
+                () -> initialSearchEntity(searchRequest, new BytesArray("{}, \"trailing\": {}"), remoteVersion));
         assertThat(e.getCause().getMessage(), containsString("Unexpected character (',' (code 44))"));
-        e = expectThrows(RuntimeException.class, () -> initialSearchEntity(new BytesArray("{")));
+        e = expectThrows(RuntimeException.class, () -> initialSearchEntity(searchRequest, new BytesArray("{"), remoteVersion));
         assertThat(e.getCause().getMessage(), containsString("Unexpected end-of-input"));
     }
 
     public void testScrollParams() {
+        Version remoteVersion = Version.fromId(between(0, Version.CURRENT.id));
         TimeValue scroll = TimeValue.parseTimeValue(randomPositiveTimeValue(), "test");
-        assertThat(scrollParams(scroll), hasEntry("scroll", scroll.toString()));
+        assertScroll(remoteVersion, scrollParams(scroll, remoteVersion), scroll);
     }
 
     public void testScrollEntity() throws IOException {
-        String scroll = randomAsciiOfLength(30);
-        HttpEntity entity = scrollEntity(scroll);
+        String scroll = randomAlphaOfLength(30);
+        HttpEntity entity = scrollEntity(scroll, Version.V_5_0_0);
+        assertEquals(ContentType.APPLICATION_JSON.toString(), entity.getContentType().getValue());
+        assertThat(Streams.copyToString(new InputStreamReader(entity.getContent(), StandardCharsets.UTF_8)),
+            containsString("\"" + scroll + "\""));
+
+        // Test with version < 2.0.0
+        entity = scrollEntity(scroll, Version.fromId(1070499));
+        assertEquals(ContentType.TEXT_PLAIN.toString(), entity.getContentType().getValue());
+        assertEquals(scroll, Streams.copyToString(new InputStreamReader(entity.getContent(), StandardCharsets.UTF_8)));
+    }
+
+    public void testClearScrollEntity() throws IOException {
+        String scroll = randomAlphaOfLength(30);
+        HttpEntity entity = clearScrollEntity(scroll, Version.V_5_0_0);
+        assertEquals(ContentType.APPLICATION_JSON.toString(), entity.getContentType().getValue());
+        assertThat(Streams.copyToString(new InputStreamReader(entity.getContent(), StandardCharsets.UTF_8)),
+            containsString("\"" + scroll + "\""));
+
+        // Test with version < 2.0.0
+        entity = clearScrollEntity(scroll, Version.fromId(1070499));
         assertEquals(ContentType.TEXT_PLAIN.toString(), entity.getContentType().getValue());
         assertEquals(scroll, Streams.copyToString(new InputStreamReader(entity.getContent(), StandardCharsets.UTF_8)));
     }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSourceTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSourceTests.java
index 6407bc0195b45..b94b01a6c887d 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSourceTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSourceTests.java
@@ -19,12 +19,14 @@
 
 package org.elasticsearch.index.reindex.remote;
 
+import org.apache.http.ContentTooLongException;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpEntityEnclosingRequest;
 import org.apache.http.HttpHost;
 import org.apache.http.HttpResponse;
 import org.apache.http.ProtocolVersion;
 import org.apache.http.StatusLine;
+import org.apache.http.client.protocol.HttpClientContext;
 import org.apache.http.concurrent.FutureCallback;
 import org.apache.http.entity.ContentType;
 import org.apache.http.entity.InputStreamEntity;
@@ -38,14 +40,18 @@
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.bulk.BackoffPolicy;
+import org.elasticsearch.index.reindex.ScrollableHitSource.Response;
 import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.client.HeapBufferedAsyncResponseConsumer;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.io.Streams;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
-import org.elasticsearch.index.reindex.ScrollableHitSource.Response;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.test.ESTestCase;
@@ -60,7 +66,7 @@
 import java.io.InputStreamReader;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
-import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -68,15 +74,18 @@
 
 import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
 import static org.elasticsearch.common.unit.TimeValue.timeValueMinutes;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 public class RemoteScrollableHitSourceTests extends ESTestCase {
-    private final String FAKE_SCROLL_ID = "DnF1ZXJ5VGhlbkZldGNoBQAAAfakescroll";
+    private static final String FAKE_SCROLL_ID = "DnF1ZXJ5VGhlbkZldGNoBQAAAfakescroll";
     private int retries;
     private ThreadPool threadPool;
     private SearchRequest searchRequest;
@@ -86,10 +95,11 @@ public class RemoteScrollableHitSourceTests extends ESTestCase {
     @Override
     public void setUp() throws Exception {
         super.setUp();
+        final ExecutorService directExecutor = EsExecutors.newDirectExecutorService();
         threadPool = new TestThreadPool(getTestName()) {
             @Override
-            public Executor executor(String name) {
-                return Runnable::run;
+            public ExecutorService executor(String name) {
+                return directExecutor;
             }
 
             @Override
@@ -113,11 +123,42 @@ public void tearDown() throws Exception {
     }
 
     public void testLookupRemoteVersion() throws Exception {
-        sourceWithMockedRemoteCall(false, "main/0_20_5.json").lookupRemoteVersion(v -> assertEquals(Version.fromString("0.20.5"), v));
-        sourceWithMockedRemoteCall(false, "main/0_90_13.json").lookupRemoteVersion(v -> assertEquals(Version.fromString("0.90.13"), v));
-        sourceWithMockedRemoteCall(false, "main/1_7_5.json").lookupRemoteVersion(v -> assertEquals(Version.fromString("1.7.5"), v));
-        sourceWithMockedRemoteCall(false, "main/2_3_3.json").lookupRemoteVersion(v -> assertEquals(Version.V_2_3_3, v));
-        sourceWithMockedRemoteCall(false, "main/5_0_0_alpha_3.json").lookupRemoteVersion(v -> assertEquals(Version.V_5_0_0_alpha3, v));
+        AtomicBoolean called = new AtomicBoolean();
+        sourceWithMockedRemoteCall(false, ContentType.APPLICATION_JSON, "main/0_20_5.json").lookupRemoteVersion(v -> {
+            assertEquals(Version.fromString("0.20.5"), v);
+            called.set(true);
+        });
+        assertTrue(called.get());
+        called.set(false);
+        sourceWithMockedRemoteCall(false, ContentType.APPLICATION_JSON, "main/0_90_13.json").lookupRemoteVersion(v -> {
+            assertEquals(Version.fromString("0.90.13"), v);
+            called.set(true);
+        });
+        assertTrue(called.get());
+        called.set(false);
+        sourceWithMockedRemoteCall(false, ContentType.APPLICATION_JSON, "main/1_7_5.json").lookupRemoteVersion(v -> {
+            assertEquals(Version.fromString("1.7.5"), v);
+            called.set(true);
+        });
+        assertTrue(called.get());
+        called.set(false);
+        sourceWithMockedRemoteCall(false, ContentType.APPLICATION_JSON, "main/2_3_3.json").lookupRemoteVersion(v -> {
+            assertEquals(Version.V_2_3_3, v);
+            called.set(true);
+        });
+        assertTrue(called.get());
+        called.set(false);
+        sourceWithMockedRemoteCall(false, ContentType.APPLICATION_JSON, "main/5_0_0_alpha_3.json").lookupRemoteVersion(v -> {
+            assertEquals(Version.V_5_0_0_alpha3, v);
+            called.set(true);
+        });
+        assertTrue(called.get());
+        called.set(false);
+        sourceWithMockedRemoteCall(false, ContentType.APPLICATION_JSON, "main/with_unknown_fields.json").lookupRemoteVersion(v -> {
+            assertEquals(Version.V_5_0_0_alpha3, v);
+            called.set(true);
+        });
+        assertTrue(called.get());
     }
 
     public void testParseStartOk() throws Exception {
@@ -161,7 +202,7 @@ public void testParseScrollOk() throws Exception {
     }
 
     /**
-     * Test for parsing _ttl, _timestamp, and _routing.
+     * Test for parsing _ttl, _timestamp, _routing, and _parent.
      */
     public void testParseScrollFullyLoaded() throws Exception {
         AtomicBoolean called = new AtomicBoolean();
@@ -177,6 +218,23 @@ public void testParseScrollFullyLoaded() throws Exception {
         assertTrue(called.get());
     }
 
+    /**
+     * Test for parsing _ttl, _routing, and _parent. _timestamp isn't available.
+     */
+    public void testParseScrollFullyLoadedFrom1_7() throws Exception {
+        AtomicBoolean called = new AtomicBoolean();
+        sourceWithMockedRemoteCall("scroll_fully_loaded_1_7.json").doStartNextScroll("", timeValueMillis(0), r -> {
+            assertEquals("AVToMiDL50DjIiBO3yKA", r.getHits().get(0).getId());
+            assertEquals("{\"test\":\"test3\"}", r.getHits().get(0).getSource().utf8ToString());
+            assertEquals((Long) 1234L, r.getHits().get(0).getTTL());
+            assertNull(r.getHits().get(0).getTimestamp()); // Not available from 1.7
+            assertEquals("testrouting", r.getHits().get(0).getRouting());
+            assertEquals("testparent", r.getHits().get(0).getParent());
+            called.set(true);
+        });
+        assertTrue(called.get());
+    }
+
     /**
      * Versions of Elasticsearch before 2.1.0 don't support sort:_doc and instead need to use search_type=scan. Scan doesn't return
      * documents the first iteration but reindex doesn't like that. So we jump start strait to the next iteration.
@@ -321,7 +379,7 @@ public void testRetryUntilYouRunOutOfTries() throws Exception {
     }
 
     public void testThreadContextRestored() throws Exception {
-        String header = randomAsciiOfLength(5);
+        String header = randomAlphaOfLength(5);
         threadPool.getThreadContext().putHeader("test", header);
         AtomicBoolean called = new AtomicBoolean();
         sourceWithMockedRemoteCall("start_ok.json").doStart(r -> {
@@ -380,8 +438,78 @@ public void testWrapExceptionToPreserveStatus() throws IOException {
         assertEquals(badEntityException, wrapped.getSuppressed()[0]);
     }
 
+    @SuppressWarnings({ "unchecked", "rawtypes" })
+    public void testTooLargeResponse() throws Exception {
+        ContentTooLongException tooLong = new ContentTooLongException("too long!");
+        CloseableHttpAsyncClient httpClient = mock(CloseableHttpAsyncClient.class);
+        when(httpClient.<HttpResponse>execute(any(HttpAsyncRequestProducer.class), any(HttpAsyncResponseConsumer.class),
+                any(HttpClientContext.class), any(FutureCallback.class))).then(new Answer<Future<HttpResponse>>() {
+            @Override
+            public Future<HttpResponse> answer(InvocationOnMock invocationOnMock) throws Throwable {
+                HeapBufferedAsyncResponseConsumer consumer = (HeapBufferedAsyncResponseConsumer) invocationOnMock.getArguments()[1];
+                FutureCallback callback = (FutureCallback) invocationOnMock.getArguments()[3];
+                assertEquals(new ByteSizeValue(100, ByteSizeUnit.MB).bytesAsInt(), consumer.getBufferLimit());
+                callback.failed(tooLong);
+                return null;
+            }
+        });
+        RemoteScrollableHitSource source = sourceWithMockedClient(true, httpClient);
+
+        AtomicBoolean called = new AtomicBoolean();
+        Consumer<Response> checkResponse = r -> called.set(true);
+        Throwable e = expectThrows(RuntimeException.class,
+                () -> source.doStartNextScroll(FAKE_SCROLL_ID, timeValueMillis(0), checkResponse));
+        // Unwrap the some artifacts from the test
+        while (e.getMessage().equals("failed")) {
+            e = e.getCause();
+        }
+        // This next exception is what the user sees
+        assertEquals("Remote responded with a chunk that was too large. Use a smaller batch size.", e.getMessage());
+        // And that exception is reported as being caused by the underlying exception returned by the client
+        assertSame(tooLong, e.getCause());
+        assertFalse(called.get());
+    }
+
+    public void testNoContentTypeIsError() throws Exception {
+        Exception e = expectThrows(RuntimeException.class, () ->
+                sourceWithMockedRemoteCall(false, null, "main/0_20_5.json").lookupRemoteVersion(null));
+        assertThat(e.getCause().getCause().getCause().getMessage(), containsString("Response didn't include Content-Type: body={"));
+    }
+
+    public void testInvalidJsonThinksRemoveIsNotES() throws IOException {
+        Exception e = expectThrows(RuntimeException.class, () -> sourceWithMockedRemoteCall("some_text.txt").doStart(null));
+        assertEquals("Error parsing the response, remote is likely not an Elasticsearch instance",
+                e.getCause().getCause().getCause().getMessage());
+    }
+
+    public void testUnexpectedJsonThinksRemoveIsNotES() throws IOException {
+        // Use the response from a main action instead of a proper start response to generate a parse error
+        Exception e = expectThrows(RuntimeException.class, () -> sourceWithMockedRemoteCall("main/2_3_3.json").doStart(null));
+        assertEquals("Error parsing the response, remote is likely not an Elasticsearch instance",
+                e.getCause().getCause().getCause().getMessage());
+    }
+
+    public void testCleanupSuccessful() throws Exception {
+        AtomicBoolean cleanupCallbackCalled = new AtomicBoolean();
+        RestClient client = mock(RestClient.class);
+        TestRemoteScrollableHitSource hitSource = new TestRemoteScrollableHitSource(client);
+        hitSource.cleanup(() -> cleanupCallbackCalled.set(true));
+        verify(client).close();
+        assertTrue(cleanupCallbackCalled.get());
+    }
+
+    public void testCleanupFailure() throws Exception {
+        AtomicBoolean cleanupCallbackCalled = new AtomicBoolean();
+        RestClient client = mock(RestClient.class);
+        doThrow(new RuntimeException("test")).when(client).close();
+        TestRemoteScrollableHitSource hitSource = new TestRemoteScrollableHitSource(client);
+        hitSource.cleanup(() -> cleanupCallbackCalled.set(true));
+        verify(client).close();
+        assertTrue(cleanupCallbackCalled.get());
+    }
+
     private RemoteScrollableHitSource sourceWithMockedRemoteCall(String... paths) throws Exception {
-        return sourceWithMockedRemoteCall(true, paths);
+        return sourceWithMockedRemoteCall(true, ContentType.APPLICATION_JSON, paths);
     }
 
     /**
@@ -389,7 +517,8 @@ private RemoteScrollableHitSource sourceWithMockedRemoteCall(String... paths) th
      * synchronously rather than asynchronously.
      */
     @SuppressWarnings("unchecked")
-    private RemoteScrollableHitSource sourceWithMockedRemoteCall(boolean mockRemoteVersion, String... paths) throws Exception {
+    private RemoteScrollableHitSource sourceWithMockedRemoteCall(boolean mockRemoteVersion, ContentType contentType, String... paths)
+            throws Exception {
         URL[] resources = new URL[paths.length];
         for (int i = 0; i < paths.length; i++) {
             resources[i] = Thread.currentThread().getContextClassLoader().getResource("responses/" + paths[i].replace("fail:", ""));
@@ -400,7 +529,7 @@ private RemoteScrollableHitSource sourceWithMockedRemoteCall(boolean mockRemoteV
 
         CloseableHttpAsyncClient httpClient = mock(CloseableHttpAsyncClient.class);
         when(httpClient.<HttpResponse>execute(any(HttpAsyncRequestProducer.class), any(HttpAsyncResponseConsumer.class),
-                any(FutureCallback.class))).thenAnswer(new Answer<Future<HttpResponse>>() {
+                any(HttpClientContext.class), any(FutureCallback.class))).thenAnswer(new Answer<Future<HttpResponse>>() {
 
             int responseCount = 0;
 
@@ -409,7 +538,7 @@ public Future<HttpResponse> answer(InvocationOnMock invocationOnMock) throws Thr
                 // Throw away the current thread context to simulate running async httpclient's thread pool
                 threadPool.getThreadContext().stashContext();
                 HttpAsyncRequestProducer requestProducer = (HttpAsyncRequestProducer) invocationOnMock.getArguments()[0];
-                FutureCallback<HttpResponse> futureCallback = (FutureCallback<HttpResponse>) invocationOnMock.getArguments()[2];
+                FutureCallback<HttpResponse> futureCallback = (FutureCallback<HttpResponse>) invocationOnMock.getArguments()[3];
                 HttpEntityEnclosingRequest request = (HttpEntityEnclosingRequest)requestProducer.generateRequest();
                 URL resource = resources[responseCount];
                 String path = paths[responseCount++];
@@ -426,14 +555,17 @@ public Future<HttpResponse> answer(InvocationOnMock invocationOnMock) throws Thr
                 } else {
                     StatusLine statusLine = new BasicStatusLine(protocolVersion, 200, "");
                     HttpResponse httpResponse = new BasicHttpResponse(statusLine);
-                    httpResponse.setEntity(new InputStreamEntity(resource.openStream(),
-                            randomBoolean() ? ContentType.APPLICATION_JSON : null));
+                    httpResponse.setEntity(new InputStreamEntity(resource.openStream(), contentType));
                     futureCallback.completed(httpResponse);
                 }
                 return null;
             }
         });
+        return sourceWithMockedClient(mockRemoteVersion, httpClient);
+    }
 
+    private RemoteScrollableHitSource sourceWithMockedClient(boolean mockRemoteVersion, CloseableHttpAsyncClient httpClient)
+            throws Exception {
         HttpAsyncClientBuilder clientBuilder = mock(HttpAsyncClientBuilder.class);
         when(clientBuilder.build()).thenReturn(httpClient);
 
diff --git a/modules/reindex/src/test/resources/responses/main/with_unknown_fields.json b/modules/reindex/src/test/resources/responses/main/with_unknown_fields.json
new file mode 100644
index 0000000000000..6aec2496314f2
--- /dev/null
+++ b/modules/reindex/src/test/resources/responses/main/with_unknown_fields.json
@@ -0,0 +1,22 @@
+{
+  "name" : "Crazy Node With Weird Stuff In The Response",
+  "cluster_name" : "distribution_run",
+  "cats": "knock things over",
+  "cake": "is tasty",
+  "version" : {
+    "number" : "5.0.0-alpha3",
+    "build_hash" : "42e092f",
+    "build_date" : "2016-05-26T16:55:45.405Z",
+    "build_snapshot" : true,
+    "lucene_version" : "6.0.0",
+    "blort_version" : "not even a valid version number, what are you going to do about it?"
+  },
+  "tagline" : "You Know, for Search",
+  "extra_object" : {
+    "stuff": "stuff"
+  },
+  "extra_array" : [
+    "stuff",
+    "more stuff"
+  ]
+}
diff --git a/modules/reindex/src/test/resources/responses/scroll_fully_loaded_1_7.json b/modules/reindex/src/test/resources/responses/scroll_fully_loaded_1_7.json
new file mode 100644
index 0000000000000..f8bebddecf3cf
--- /dev/null
+++ b/modules/reindex/src/test/resources/responses/scroll_fully_loaded_1_7.json
@@ -0,0 +1,31 @@
+{
+  "_scroll_id" : "DnF1ZXJ5VGhlbkZldGNoBQAAAfakescroll",
+  "took" : 3,
+  "timed_out" : false,
+  "terminated_early" : true,
+  "_shards" : {
+    "total" : 5,
+    "successful" : 5,
+    "failed" : 0
+  },
+  "hits" : {
+    "total" : 4,
+    "max_score" : null,
+    "hits" : [ {
+      "_index" : "test",
+      "_type" : "test",
+      "_id" : "AVToMiDL50DjIiBO3yKA",
+      "_version" : 1,
+      "_score" : null,
+      "_source" : {
+        "test" : "test3"
+      },
+      "sort" : [ 0 ],
+      "fields" : {
+        "_routing" : "testrouting",
+        "_ttl" : 1234,
+        "_parent" : "testparent"
+      }
+    } ]
+  }
+}
diff --git a/modules/reindex/src/test/resources/responses/some_text.txt b/modules/reindex/src/test/resources/responses/some_text.txt
new file mode 100644
index 0000000000000..39fe6c971a80e
--- /dev/null
+++ b/modules/reindex/src/test/resources/responses/some_text.txt
@@ -0,0 +1 @@
+I'm just text!
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/20_validation.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/20_validation.yaml
index 8833c844b220d..49bd88b5e6192 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/20_validation.yaml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/20_validation.yaml
@@ -5,6 +5,21 @@
       delete_by_query:
         index: _all
 
+---
+"no query logs deprecation":
+
+  - skip:
+      features: "warnings"
+      version: " - 5.3.99"
+      reason: deprecation logging for DBQ without explicit query was introduced in 5.4.0
+
+  - do:
+      warnings:
+          - A request to _delete_by_query without an explicit query is deprecated. Specify a query explicitly instead.
+      delete_by_query:
+        index: _all
+        body: {}
+
 ---
 "invalid conflicts fails":
   - do:
@@ -109,3 +124,24 @@
         body:
           query:
             match_all: {}
+
+---
+"junk in slices fails":
+  - do:
+      catch: /Failed to parse int parameter \[slices\] with value \[junk\]/
+      delete_by_query:
+        slices: junk
+        index: test
+        body:
+          query:
+            match_all: {}
+---
+"zero slices fails":
+  - do:
+      catch: /\[slices\] must be at least 1/
+      delete_by_query:
+        slices: 0
+        index: test
+        body:
+          query:
+            match_all: {}
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/50_wait_for_active_shards.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/50_wait_for_active_shards.yaml
index 522cd351cd087..ab21e6802f91d 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/50_wait_for_active_shards.yaml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/50_wait_for_active_shards.yaml
@@ -26,7 +26,7 @@
             match_all: {}
 
   - match:
-      failures.0.cause.reason: /Not.enough.active.copies.to.meet.shard.count.of.\[4\].\(have.1,.needed.4\)..Timeout\:.\[1s\],.request:.\[BulkShardRequest.to.\[test\].containing.\[1\].requests\]/
+      failures.0.cause.reason: /Not.enough.active.copies.to.meet.shard.count.of.\[4\].\(have.1,.needed.4\)..Timeout\:.\[1s\],.request:.+/
 
   - do:
       indices.refresh: {}
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/80_slices.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/80_slices.yaml
new file mode 100644
index 0000000000000..fe0c816ee14c0
--- /dev/null
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/80_slices.yaml
@@ -0,0 +1,279 @@
+---
+"Multiple slices":
+  - do:
+      index:
+        index:   test
+        type:    foo
+        id:      1
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   test
+        type:    foo
+        id:      2
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   test
+        type:    foo
+        id:      3
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   test
+        type:    foo
+        id:      4
+        body:    { "text": "test" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      delete_by_query:
+        index: test
+        slices: 5
+        body:
+          query:
+            match_all: {}
+
+  - is_false: timed_out
+  - match: {deleted: 4}
+  - is_false: created
+  - is_false: updated
+  - match: {version_conflicts: 0}
+  - match: {failures: []}
+  - match: {noops: 0}
+  - match: {throttled_millis: 0}
+  - gte: { took: 0 }
+  - is_false: task
+  - match: {slices.0.version_conflicts: 0}
+  - match: {slices.0.throttled_millis: 0}
+  - match: {slices.1.version_conflicts: 0}
+  - match: {slices.1.throttled_millis: 0}
+  - match: {slices.2.version_conflicts: 0}
+  - match: {slices.2.throttled_millis: 0}
+  - match: {slices.3.version_conflicts: 0}
+  - match: {slices.3.throttled_millis: 0}
+  - match: {slices.4.version_conflicts: 0}
+  - match: {slices.4.throttled_millis: 0}
+
+  - do:
+      indices.refresh: {}
+  - do:
+      count:
+        index: test
+  - match: {count: 0}
+
+---
+"Multiple slices with wait_for_completion=false":
+  - do:
+      index:
+        index:   test
+        type:    foo
+        id:      1
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   test
+        type:    foo
+        id:      2
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   test
+        type:    foo
+        id:      3
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   test
+        type:    foo
+        id:      4
+        body:    { "text": "test" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      delete_by_query:
+        index: test
+        wait_for_completion: false
+        slices: 5
+        body:
+          query:
+            match_all: {}
+  - is_false: timed_out
+  - match: {task: '/.+:\d+/'}
+  - set: {task: task}
+  - is_false: updated
+  - is_false: version_conflicts
+  - is_false: batches
+  - is_false: failures
+  - is_false: noops
+  - is_false: took
+  - is_false: throttled_millis
+  - is_false: created
+  - is_false: deleted
+
+  - do:
+      tasks.get:
+        wait_for_completion: true
+        task_id: $task
+  - is_false: node_failures
+  # The task will be in the response even if it finished before we got here
+  # because of task persistence.
+  - is_true: task
+  - match: {response.total: 4}
+  - match: {response.deleted: 4}
+  - match: {response.version_conflicts: 0}
+  - match: {response.failures: []}
+  - match: {response.throttled_millis: 0}
+  - gte: { response.took: 0 }
+  - is_false: response.task
+  - match: {response.slices.0.version_conflicts: 0}
+  - match: {response.slices.0.throttled_millis: 0}
+  - match: {response.slices.1.version_conflicts: 0}
+  - match: {response.slices.1.throttled_millis: 0}
+  - match: {response.slices.2.version_conflicts: 0}
+  - match: {response.slices.2.throttled_millis: 0}
+  - match: {response.slices.3.version_conflicts: 0}
+  - match: {response.slices.3.throttled_millis: 0}
+  - match: {response.slices.4.version_conflicts: 0}
+  - match: {response.slices.4.throttled_millis: 0}
+  - match: {task.status.total: 4}
+  - match: {task.status.deleted: 4}
+  - match: {task.status.version_conflicts: 0}
+  - match: {task.status.throttled_millis: 0}
+  - match: {task.status.slices.0.version_conflicts: 0}
+  - match: {task.status.slices.0.throttled_millis: 0}
+  - match: {task.status.slices.1.version_conflicts: 0}
+  - match: {task.status.slices.1.throttled_millis: 0}
+  - match: {task.status.slices.2.version_conflicts: 0}
+  - match: {task.status.slices.2.throttled_millis: 0}
+  - match: {task.status.slices.3.version_conflicts: 0}
+  - match: {task.status.slices.3.throttled_millis: 0}
+  - match: {task.status.slices.4.version_conflicts: 0}
+  - match: {task.status.slices.4.throttled_millis: 0}
+
+
+  # Only the "parent" reindex task wrote its status to the tasks index though
+  - do:
+      indices.refresh: {}
+  - do:
+      search:
+        index: .tasks
+  - match: { hits.total: 1 }
+
+  - do:
+      count:
+        index: test
+  - match: {count: 0}
+
+---
+"Multiple slices with rethrottle":
+  - do:
+      index:
+        index:   test
+        type:    foo
+        id:      1
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   test
+        type:    foo
+        id:      2
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   test
+        type:    foo
+        id:      3
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   test
+        type:    foo
+        id:      4
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   test
+        type:    foo
+        id:      5
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   test
+        type:    foo
+        id:      6
+        body:    { "text": "test" }
+  - do:
+      indices.refresh: {}
+
+  # Start the task with a requests_per_second that should make it take a very long time
+  - do:
+      delete_by_query:
+        index: test
+        wait_for_completion: false
+        slices: 2
+        requests_per_second: 0.0001
+        body:
+          query:
+            match_all: {}
+  - is_false: timed_out
+  - match: {task: '/.+:\d+/'}
+  - set: {task: task}
+  - is_false: updated
+  - is_false: version_conflicts
+  - is_false: batches
+  - is_false: failures
+  - is_false: noops
+  - is_false: took
+  - is_false: throttled_millis
+  - is_false: created
+  - is_false: deleted
+
+  # Allow the task to complete
+  - do:
+      reindex_rethrottle:
+        requests_per_second: -1
+        task_id: $task
+
+  - do:
+      tasks.get:
+        wait_for_completion: true
+        task_id: $task
+  - is_false: node_failures
+  # The task will be in the response even if it finished before we got here
+  # because of task persistence.
+  - is_true: task
+  - match: {response.total: 6}
+  - match: {response.deleted: 6}
+  - match: {response.version_conflicts: 0}
+  - match: {response.failures: []}
+  - match: {response.throttled_millis: 0}
+  - gte: { response.took: 0 }
+  - is_false: response.task
+  - match: {response.slices.0.version_conflicts: 0}
+  - match: {response.slices.0.throttled_millis: 0}
+  - match: {response.slices.1.version_conflicts: 0}
+  - match: {response.slices.1.throttled_millis: 0}
+  - match: {task.status.total: 6}
+  - match: {task.status.deleted: 6}
+  - match: {task.status.version_conflicts: 0}
+  - match: {task.status.throttled_millis: 0}
+  - match: {task.status.slices.0.version_conflicts: 0}
+  - match: {task.status.slices.0.throttled_millis: 0}
+  - match: {task.status.slices.1.version_conflicts: 0}
+  - match: {task.status.slices.1.throttled_millis: 0}
+
+  # Only the "parent" reindex task wrote its status to the tasks index though
+  - do:
+      indices.refresh: {}
+  - do:
+      search:
+        index: .tasks
+  - match: { hits.total: 1 }
+
+  - do:
+      count:
+        index: test
+  - match: {count: 0}
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/10_basic.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/10_basic.yaml
index a567ca67bfaab..3557cf9bad7f3 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/10_basic.yaml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/10_basic.yaml
@@ -26,6 +26,13 @@
   - is_false: task
   - is_false: deleted
 
+  # Make sure reindex closed all the scroll contexts
+  - do:
+      indices.stats:
+        index: source
+        metric: search
+  - match: {indices.source.total.search.open_contexts: 0}
+
 ---
 "Response format for updated":
   - do:
@@ -60,6 +67,13 @@
   - is_false: task
   - is_false: deleted
 
+  # Make sure reindex closed all the scroll contexts
+  - do:
+      indices.stats:
+        index: source
+        metric: search
+  - match: {indices.source.total.search.open_contexts: 0}
+
 ---
 "wait_for_completion=false":
   - do:
@@ -110,6 +124,13 @@
   - is_false: response.task
   - is_false: response.deleted
 
+  # Make sure reindex closed all the scroll contexts
+  - do:
+      indices.stats:
+        index: source
+        metric: search
+  - match: {indices.source.total.search.open_contexts: 0}
+
 ---
 "Response format for version conflict":
   - do:
@@ -151,6 +172,13 @@
   - match: {failures.0.cause.index:  dest}
   - gte: { took: 0 }
 
+  # Make sure reindex closed all the scroll contexts
+  - do:
+      indices.stats:
+        index: source
+        metric: search
+  - match: {indices.source.total.search.open_contexts: 0}
+
 ---
 "Response format for version conflict with conflicts=proceed":
   - do:
@@ -185,234 +213,89 @@
   - match: {throttled_millis: 0}
   - gte: { took: 0 }
 
----
-"Simplest example in docs":
-  - do:
-      index:
-        index:   twitter
-        type:    tweet
-        id:      1
-        body:    { "user": "kimchy" }
-  - do:
-      indices.refresh: {}
-
-  - do:
-      reindex:
-        refresh: true
-        body:
-          source:
-            index: twitter
-          dest:
-            index: new_twitter
-
-  - do:
-      search:
-        index: new_twitter
-  - match: { hits.total: 1 }
-
----
-"Limit by type example in docs":
-  - do:
-      index:
-        index:   twitter
-        type:    tweet
-        id:      1
-        body:    { "user": "kimchy" }
-  - do:
-      index:
-        index:   twitter
-        type:    junk
-        id:      1
-        body:    { "user": "kimchy" }
-  - do:
-      indices.refresh: {}
-
-  - do:
-      reindex:
-        refresh: true
-        body:
-          source:
-            index: twitter
-            type:  tweet
-          dest:
-            index: new_twitter
-
+  # Make sure reindex closed all the scroll contexts
   - do:
-      search:
-        index: new_twitter
-  - match: { hits.total: 1 }
+      indices.stats:
+        index: source
+        metric: search
+  - match: {indices.source.total.search.open_contexts: 0}
 
 ---
-"Limit by query example in docs":
+"Source document without any fields works":
   - do:
       index:
-        index:   twitter
-        type:    tweet
+        index:   source
+        type:    foo
         id:      1
-        body:    { "user": "kimchy" }
-  - do:
-      index:
-        index:   twitter
-        type:    tweet
-        id:      2
-        body:    { "user": "junk" }
+        body:    {}
   - do:
       indices.refresh: {}
 
   - do:
       reindex:
-        refresh: true
         body:
           source:
-            index: twitter
-            query:
-              match:
-                user: kimchy
+            index: source
           dest:
-            index: new_twitter
-
-  - do:
-      search:
-        index: new_twitter
-  - match: { hits.total: 1 }
+            index: dest
+  - match: {created: 1}
 
----
-"Override type example in docs":
-  - do:
-      index:
-        index:   twitter
-        type:    tweet
-        id:      1
-        body:    { "user": "kimchy" }
   - do:
-      index:
-        index:   twitter
-        type:    junk
+      get:
+        index:   dest
+        type:    foo
         id:      1
-        body:    { "user": "kimchy" }
-  - do:
-      indices.refresh: {}
-
-  - do:
-      reindex:
-        refresh: true
-        body:
-          source:
-            index: twitter
-            type:  tweet
-          dest:
-            index: new_twitter
-            type:  chirp
-
-  - do:
-      search:
-        index: new_twitter
-        type:  chirp
-  - match: { hits.total: 1 }
-
----
-"Multi index, multi type example from docs":
-  - do:
-      index:
-        index:  twitter
-        type:   tweet
-        id:     1
-        body:   { "user": "kimchy" }
-  - do:
-      index:
-        index:  blog
-        type:   post
-        id:     1
-        body:   { "user": "kimchy" }
-  - do:
-      indices.refresh: {}
-
-  - do:
-      reindex:
-        refresh: true
-        body:
-          source:
-            index: [twitter, blog]
-            type:  [tweet, post]
-          dest:
-            index: all_together
-
-  - do:
-      search:
-        index: all_together
-        type: tweet
-        body:
-          query:
-            match:
-              user: kimchy
-  - match: { hits.total: 1 }
-
-  - do:
-      search:
-        index: all_together
-        type: post
-        body:
-          query:
-            match:
-              user: kimchy
-  - match: { hits.total: 1 }
-
----
-"Limit by size example from docs":
-  - do:
-      index:
-        index:  twitter
-        type:   tweet
-        id:     1
-        body:   { "user": "kimchy" }
-  - do:
-      index:
-        index:  twitter
-        type:   tweet
-        id:     2
-        body:   { "user": "kimchy" }
-  - do:
-      indices.refresh: {}
-
-  - do:
-      reindex:
-        refresh: true
-        body:
-          size: 1
-          source:
-            index: twitter
-          dest:
-            index: new_twitter
+  - match: { _source: {} }
 
+  # Make sure reindex closed all the scroll contexts
   - do:
-      search:
-        index: new_twitter
-        type: tweet
-  - match: { hits.total: 1 }
+      indices.stats:
+        index: source
+        metric: search
+  - match: {indices.source.total.search.open_contexts: 0}
 
 ---
-"Source document without any fields works":
+"Reindex with source filtering":
   - do:
       index:
         index:   source
         type:    foo
         id:      1
-        body:    {}
-  - do:
-      indices.refresh: {}
+        body:    { "text": "test", "filtered": "removed" }
+        refresh: true
 
   - do:
       reindex:
+        refresh: true
         body:
           source:
             index: source
+            _source:
+              excludes:
+                - filtered
           dest:
             index: dest
   - match: {created: 1}
+  - match: {updated: 0}
+  - match: {version_conflicts: 0}
+  - match: {batches: 1}
+  - match: {failures: []}
+  - match: {throttled_millis: 0}
+  - gte: { took: 0 }
+  - is_false: task
+  - is_false: deleted
 
   - do:
       get:
-        index:   dest
-        type:    foo
-        id:      1
-  - match: { _source: {} }
+        index: dest
+        type: foo
+        id: 1
+  - match: { _source.text: "test" }
+  - is_false: _source.filtered
+
+  # Make sure reindex closed all the scroll contexts
+  - do:
+      indices.stats:
+        index: source
+        metric: search
+  - match: {indices.source.total.search.open_contexts: 0}
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yaml
index cab92310dbd17..6017843741d18 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yaml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yaml
@@ -279,3 +279,73 @@
             index: test
           dest:
             index: dest
+
+---
+"broken socket timeout in remote fails":
+  - do:
+      catch: /number_format_exception/
+      reindex:
+        body:
+          source:
+            remote:
+              host: http://okremote:9200
+              socket_timeout: borked
+            index: test
+          dest:
+            index: dest
+
+---
+"broken connect timeout in remote fails":
+  - do:
+      catch: /number_format_exception/
+      reindex:
+        body:
+          source:
+            remote:
+              host: http://okremote:9200
+              connect_timeout: borked
+            index: test
+          dest:
+            index: dest
+
+---
+"junk in slices fails":
+  - do:
+      catch: /Failed to parse int parameter \[slices\] with value \[junk\]/
+      reindex:
+        slices: junk
+        body:
+          source:
+            index: test
+          dest:
+            index: dest
+
+---
+"zero slices fails":
+  - do:
+      catch: /\[slices\] must be at least 1/
+      reindex:
+        slices: 0
+        body:
+          source:
+            index: test
+          dest:
+            index: dest
+
+---
+"_source:false is rejected":
+  - do:
+      index:
+        index:   source
+        type:    foo
+        id:      1
+        body:    { "text": "test" }
+  - do:
+      catch: /_source:false is not supported in this context/
+      reindex:
+        body:
+          source:
+            index: source
+            _source: false
+          dest:
+            index: dest
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/25_no_auto_create.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/25_no_auto_create.yaml
new file mode 100644
index 0000000000000..874174fda4c93
--- /dev/null
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/25_no_auto_create.yaml
@@ -0,0 +1,72 @@
+teardown:
+  - do:
+      cluster.put_settings:
+        body:
+          transient:
+            action.auto_create_index: null
+
+---
+"auto_create_index is false":
+  - do:
+      index:
+        index:   test
+        type:    test
+        id:      1
+        body:    { "text": "test" }
+  - do:
+      cluster.put_settings:
+        body:
+          transient:
+            action.auto_create_index: false
+  - do:
+      catch: /no such index and \[action.auto_create_index\] is \[false\]/
+      reindex:
+        body:
+          source:
+            index: test
+          dest:
+            index: dest
+
+---
+"auto_create_index doesn't include new index":
+  - do:
+      cluster.put_settings:
+        body:
+          transient:
+            action.auto_create_index: test
+  - do:
+      index:
+        index:   test
+        type:    test
+        id:      1
+        body:    { "text": "test" }
+  - do:
+      catch: /no such index and \[action.auto_create_index\] \(\[test\]\) doesn't match/
+      reindex:
+        body:
+          source:
+            index: test
+          dest:
+            index: dest
+
+---
+"auto_create_index forbids new index":
+  - do:
+      cluster.put_settings:
+        body:
+          transient:
+            action.auto_create_index: test,-dest
+  - do:
+      index:
+        index:   test
+        type:    test
+        id:      1
+        body:    { "text": "test" }
+  - do:
+      catch: /no such index and \[action.auto_create_index\] contains \[-dest\] which forbids automatic creation of the index/
+      reindex:
+        body:
+          source:
+            index: test
+          dest:
+            index: dest
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/60_wait_for_active_shards.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/60_wait_for_active_shards.yaml
index d3446243746b7..71601c35ac3fd 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/60_wait_for_active_shards.yaml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/60_wait_for_active_shards.yaml
@@ -26,7 +26,7 @@
           dest:
             index: dest
   - match:
-      failures.0.cause.reason: /Not.enough.active.copies.to.meet.shard.count.of.\[4\].\(have.1,.needed.4\)\..Timeout\:.\[1s\],.request:.\[BulkShardRequest.to.\[dest\].containing.\[1\].requests\]/
+      failures.0.cause.reason: /Not.enough.active.copies.to.meet.shard.count.of.\[4\].\(have.1,.needed.4\)\..Timeout\:.\[1s\],.request:.+/
 
   - do:
       reindex:
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/80_slices.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/80_slices.yaml
new file mode 100644
index 0000000000000..5c54adb5c088c
--- /dev/null
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/80_slices.yaml
@@ -0,0 +1,287 @@
+---
+"Multiple slices":
+  - do:
+      index:
+        index:   source
+        type:    foo
+        id:      1
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   source
+        type:    foo
+        id:      2
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   source
+        type:    foo
+        id:      3
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   source
+        type:    foo
+        id:      4
+        body:    { "text": "test" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      reindex:
+        slices: 5
+        body:
+          source:
+            index: source
+          dest:
+            index: dest
+  - match: {created: 4}
+  - match: {updated: 0}
+  - match: {version_conflicts: 0}
+  - match: {failures: []}
+  - match: {throttled_millis: 0}
+  - gte: { took: 0 }
+  - is_false: task
+  - is_false: deleted
+  - match: {slices.0.updated: 0}
+  - match: {slices.0.version_conflicts: 0}
+  - match: {slices.0.throttled_millis: 0}
+  - match: {slices.1.updated: 0}
+  - match: {slices.1.version_conflicts: 0}
+  - match: {slices.1.throttled_millis: 0}
+  - match: {slices.2.updated: 0}
+  - match: {slices.2.version_conflicts: 0}
+  - match: {slices.2.throttled_millis: 0}
+  - match: {slices.3.updated: 0}
+  - match: {slices.3.version_conflicts: 0}
+  - match: {slices.3.throttled_millis: 0}
+  - match: {slices.4.updated: 0}
+  - match: {slices.4.version_conflicts: 0}
+  - match: {slices.4.throttled_millis: 0}
+
+---
+"Multiple slices with wait_for_completion=false":
+  - do:
+      index:
+        index:   source
+        type:    foo
+        id:      1
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   source
+        type:    foo
+        id:      2
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   source
+        type:    foo
+        id:      3
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   source
+        type:    foo
+        id:      4
+        body:    { "text": "test" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      reindex:
+        slices: 5
+        wait_for_completion: false
+        body:
+          source:
+            index: source
+          dest:
+            index: dest
+  - is_false: timed_out
+  - match: {task: '/.+:\d+/'}
+  - set: {task: task}
+  - is_false: updated
+  - is_false: version_conflicts
+  - is_false: batches
+  - is_false: failures
+  - is_false: noops
+  - is_false: took
+  - is_false: throttled_millis
+  - is_false: created
+  - is_false: deleted
+
+  - do:
+      tasks.get:
+        wait_for_completion: true
+        task_id: $task
+  - is_false: node_failures
+  # The task will be in the response even if it finished before we got here
+  # because of task persistence.
+  - is_true: task
+  - match: {response.total: 4}
+  - match: {response.created: 4}
+  - match: {response.updated: 0}
+  - match: {response.version_conflicts: 0}
+  - match: {response.failures: []}
+  - match: {response.throttled_millis: 0}
+  - gte: { response.took: 0 }
+  - is_false: response.task
+  - is_false: response.deleted
+  - match: {response.slices.0.updated: 0}
+  - match: {response.slices.0.version_conflicts: 0}
+  - match: {response.slices.0.throttled_millis: 0}
+  - match: {response.slices.1.updated: 0}
+  - match: {response.slices.1.version_conflicts: 0}
+  - match: {response.slices.1.throttled_millis: 0}
+  - match: {response.slices.2.updated: 0}
+  - match: {response.slices.2.version_conflicts: 0}
+  - match: {response.slices.2.throttled_millis: 0}
+  - match: {response.slices.3.updated: 0}
+  - match: {response.slices.3.version_conflicts: 0}
+  - match: {response.slices.3.throttled_millis: 0}
+  - match: {response.slices.4.updated: 0}
+  - match: {response.slices.4.version_conflicts: 0}
+  - match: {response.slices.4.throttled_millis: 0}
+  - match: {task.status.total: 4}
+  - match: {task.status.created: 4}
+  - match: {task.status.updated: 0}
+  - match: {task.status.version_conflicts: 0}
+  - match: {task.status.throttled_millis: 0}
+  - match: {task.status.slices.0.updated: 0}
+  - match: {task.status.slices.0.version_conflicts: 0}
+  - match: {task.status.slices.0.throttled_millis: 0}
+  - match: {task.status.slices.1.updated: 0}
+  - match: {task.status.slices.1.version_conflicts: 0}
+  - match: {task.status.slices.1.throttled_millis: 0}
+  - match: {task.status.slices.2.updated: 0}
+  - match: {task.status.slices.2.version_conflicts: 0}
+  - match: {task.status.slices.2.throttled_millis: 0}
+  - match: {task.status.slices.3.updated: 0}
+  - match: {task.status.slices.3.version_conflicts: 0}
+  - match: {task.status.slices.3.throttled_millis: 0}
+  - match: {task.status.slices.4.updated: 0}
+  - match: {task.status.slices.4.version_conflicts: 0}
+  - match: {task.status.slices.4.throttled_millis: 0}
+
+  # Only the "parent" reindex task wrote its status to the tasks index though
+  - do:
+      indices.refresh: {}
+  - do:
+      search:
+        index: .tasks
+  - match: { hits.total: 1 }
+
+
+---
+"Multiple slices with rethrottle":
+  - do:
+      index:
+        index:   source
+        type:    foo
+        id:      1
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   source
+        type:    foo
+        id:      2
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   source
+        type:    foo
+        id:      3
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   source
+        type:    foo
+        id:      4
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   source
+        type:    foo
+        id:      5
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   source
+        type:    foo
+        id:      6
+        body:    { "text": "test" }
+  - do:
+      indices.refresh: {}
+
+  # Start the task with a requests_per_second that should make it take a very long time
+  - do:
+      reindex:
+        slices: 2
+        wait_for_completion: false
+        requests_per_second: 0.0001
+        body:
+          source:
+            index: source
+          dest:
+            index: dest
+  - is_false: timed_out
+  - match: {task: '/.+:\d+/'}
+  - set: {task: task}
+  - is_false: updated
+  - is_false: version_conflicts
+  - is_false: batches
+  - is_false: failures
+  - is_false: noops
+  - is_false: took
+  - is_false: throttled_millis
+  - is_false: created
+  - is_false: deleted
+
+  # Allow the task to complete
+  - do:
+      reindex_rethrottle:
+        requests_per_second: -1
+        task_id: $task
+
+  - do:
+      tasks.get:
+        wait_for_completion: true
+        task_id: $task
+  - is_false: node_failures
+  # The task will be in the response even if it finished before we got here
+  # because of task persistence.
+  - is_true: task
+  - match: {response.total: 6}
+  - match: {response.created: 6}
+  - match: {response.updated: 0}
+  - match: {response.version_conflicts: 0}
+  - match: {response.failures: []}
+  - match: {response.throttled_millis: 0}
+  - gte: { response.took: 0 }
+  - is_false: response.task
+  - is_false: response.deleted
+  - match: {response.slices.0.updated: 0}
+  - match: {response.slices.0.version_conflicts: 0}
+  - match: {response.slices.0.throttled_millis: 0}
+  - match: {response.slices.1.updated: 0}
+  - match: {response.slices.1.version_conflicts: 0}
+  - match: {response.slices.1.throttled_millis: 0}
+  - match: {task.status.total: 6}
+  - match: {task.status.created: 6}
+  - match: {task.status.updated: 0}
+  - match: {task.status.version_conflicts: 0}
+  - match: {task.status.throttled_millis: 0}
+  - match: {task.status.slices.0.updated: 0}
+  - match: {task.status.slices.0.version_conflicts: 0}
+  - match: {task.status.slices.0.throttled_millis: 0}
+  - match: {task.status.slices.1.updated: 0}
+  - match: {task.status.slices.1.version_conflicts: 0}
+  - match: {task.status.slices.1.throttled_millis: 0}
+
+  # Only the "parent" reindex task wrote its status to the tasks index though
+  - do:
+      indices.refresh: {}
+  - do:
+      search:
+        index: .tasks
+  - match: { hits.total: 1 }
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/90_remote.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/90_remote.yaml
index 6adac98ad771b..88a5327ebf142 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/90_remote.yaml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/90_remote.yaml
@@ -46,6 +46,13 @@
               text: test
   - match: {hits.total: 1}
 
+  # Make sure reindex closed all the scroll contexts
+  - do:
+      indices.stats:
+        index: source
+        metric: search
+  - match: {indices.source.total.search.open_contexts: 0}
+
 ---
 "Reindex from remote with query":
   - do:
@@ -95,6 +102,13 @@
             match_all: {}
   - match: {hits.total: 1}
 
+  # Make sure reindex closed all the scroll contexts
+  - do:
+      indices.stats:
+        index: source
+        metric: search
+  - match: {indices.source.total.search.open_contexts: 0}
+
 ---
 "Reindex from remote with routing":
   - do:
@@ -137,42 +151,272 @@
               text: test
   - match: {hits.total: 1}
 
+  # Make sure reindex closed all the scroll contexts
+  - do:
+      indices.stats:
+        index: source
+        metric: search
+  - match: {indices.source.total.search.open_contexts: 0}
+
 ---
-"Reindex from remote with parent/child":
+"Reindex from remote with timeouts":
+  # Validates that you can configure the socket_timeout and connect_timeout,
+  # not that they do anything.
   - do:
-      indices.create:
-          index: source
-          body:
-              mappings:
-                foo: {}
-                bar:
-                  _parent:
-                    type:  foo
+      index:
+        index:   source
+        type:    foo
+        id:      1
+        body:    { "text": "test" }
+        refresh: true
+
+  # Fetch the http host. We use the host of the master because we know there will always be a master.
   - do:
-      indices.create:
-          index: dest
-          body:
-              mappings:
-                foo: {}
-                bar:
-                  _parent:
-                    type:  foo
+      cluster.state: {}
+  - set: { master_node: master }
+  - do:
+      nodes.info:
+        metric: [ http ]
+  - is_true: nodes.$master.http.publish_address
+  - set: {nodes.$master.http.publish_address: host}
+  - do:
+      reindex:
+        refresh: true
+        body:
+          source:
+            remote:
+              host: http://${host}
+              socket_timeout: 1m
+              connect_timeout: 1m
+            index: source
+          dest:
+            index: dest
+  - match: {created: 1}
+  - match: {updated: 0}
+  - match: {version_conflicts: 0}
+  - match: {batches: 1}
+  - match: {failures: []}
+  - match: {throttled_millis: 0}
+  - gte: { took: 0 }
+  - is_false: task
+  - is_false: deleted
+
+  - do:
+      search:
+        index: dest
+        body:
+          query:
+            match:
+              text: test
+  - match: {hits.total: 1}
+
+  # Make sure reindex closed all the scroll contexts
+  - do:
+      indices.stats:
+        index: source
+        metric: search
+  - match: {indices.source.total.search.open_contexts: 0}
+
+---
+"Reindex from remote with size":
   - do:
       index:
         index:   source
         type:    foo
         id:      1
         body:    { "text": "test" }
+        refresh: true
   - do:
       index:
         index:   source
-        type:    bar
+        type:    foo
+        id:      2
+        body:    { "text": "test" }
+        refresh: true
+
+  # Fetch the http host. We use the host of the master because we know there will always be a master.
+  - do:
+      cluster.state: {}
+  - set: { master_node: master }
+  - do:
+      nodes.info:
+        metric: [ http ]
+  - is_true: nodes.$master.http.publish_address
+  - set: {nodes.$master.http.publish_address: host}
+  - do:
+      reindex:
+        refresh: true
+        body:
+          size: 1
+          source:
+            remote:
+              host: http://${host}
+            index: source
+          dest:
+            index: dest
+  - match: {created: 1}
+  - match: {updated: 0}
+  - match: {version_conflicts: 0}
+  - match: {batches: 1}
+  - match: {failures: []}
+  - match: {throttled_millis: 0}
+  - gte: { took: 0 }
+  - is_false: task
+  - is_false: deleted
+
+  - do:
+      search:
+        index: dest
+        body:
+          query:
+            match:
+              text: test
+  - match: {hits.total: 1}
+
+  # Make sure reindex closed all the scroll contexts
+  - do:
+      indices.stats:
+        index: source
+        metric: search
+  - match: {indices.source.total.search.open_contexts: 0}
+
+---
+"Reindex from remote with broken query":
+  - do:
+      index:
+        index:   source
+        type:    foo
         id:      1
-        parent:  1
-        body:    { "text": "test2" }
+        body:    { "text": "test" }
+        refresh: true
+
+  # Fetch the http host. We use the host of the master because we know there will always be a master.
+  - do:
+      cluster.state: {}
+  - set: { master_node: master }
+  - do:
+      nodes.info:
+        metric: [ http ]
+  - is_true: nodes.$master.http.publish_address
+  - set: {nodes.$master.http.publish_address: host}
+  - do:
+      catch: /query malformed, no start_object after query name/
+      reindex:
+        body:
+          source:
+            remote:
+              host: http://${host}
+            index: source
+            query:
+              garbage: not a query
+          dest:
+            index: dest
+
+---
+"Reindex from remote that you can't connect to":
+  - do:
+      index:
+        index:   source
+        type:    foo
+        id:      1
+        body:    { "text": "test" }
+        refresh: true
+
+  - do:
+      # sometimes IIS is listening on port 0. In that case we fail in other ways and this test isn't useful.
+      catch: /connect_exception|IIS Windows Server/
+      reindex:
+        body:
+          source:
+            remote:
+              host: http://127.0.0.1:0
+            index: source
+          dest:
+            index: dest
+
+---
+"Reindex from remote with source filtering":
+  - do:
+      index:
+        index:   source
+        type:    foo
+        id:      1
+        body:    { "text": "test", "filtered": "removed" }
+        refresh: true
+
+  # Fetch the http host. We use the host of the master because we know there will always be a master.
+  - do:
+      cluster.state: {}
+  - set: { master_node: master }
+  - do:
+      nodes.info:
+        metric: [ http ]
+  - is_true: nodes.$master.http.publish_address
+  - set: {nodes.$master.http.publish_address: host}
+  - do:
+      reindex:
+        refresh: true
+        body:
+          source:
+            remote:
+              host: http://${host}
+            index: source
+            _source:
+              excludes:
+                - filtered
+          dest:
+            index: dest
+  - match: {created: 1}
+  - match: {updated: 0}
+  - match: {version_conflicts: 0}
+  - match: {batches: 1}
+  - match: {failures: []}
+  - match: {throttled_millis: 0}
+  - gte: { took: 0 }
+  - is_false: task
+  - is_false: deleted
+
+  - do:
+      get:
+        index: dest
+        type: foo
+        id: 1
+  - match: { _source.text: "test" }
+  - is_false: _source.filtered
+
+---
+"Reindex from remote with rethrottle":
+  # Throttling happens between each scroll batch so we need to control the size of the batch by using a single shard
+  # and a small batch size on the request
+  - do:
+      indices.create:
+        index: source
+        body:
+          settings:
+            number_of_shards: "1"
+            number_of_replicas: "0"
+  - do:
+      index:
+        index:   source
+        type:    foo
+        id:      1
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   source
+        type:    foo
+        id:      2
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   source
+        type:    foo
+        id:      3
+        body:    { "text": "test" }
   - do:
       indices.refresh: {}
 
+
   # Fetch the http host. We use the host of the master because we know there will always be a master.
   - do:
       cluster.state: {}
@@ -184,24 +428,42 @@
   - set: {nodes.$master.http.publish_address: host}
   - do:
       reindex:
+        requests_per_second: .00000001 # About 9.5 years to complete the request
+        wait_for_completion: false
         refresh: true
         body:
           source:
             remote:
               host: http://${host}
             index: source
+            size: 1
           dest:
             index: dest
-  - match: {created: 2}
+  - match: {task: '/.+:\d+/'}
+  - set: {task: task}
+
+  - do:
+      reindex_rethrottle:
+        requests_per_second: -1
+        task_id: $task
+
+  - do:
+      tasks.get:
+        wait_for_completion: true
+        task_id: $task
 
   - do:
       search:
         index: dest
         body:
           query:
-            has_parent:
-              parent_type: foo
-              query:
-                match:
-                  text: test
-  - match: {hits.total: 1}
+            match:
+              text: test
+  - match: {hits.total: 3}
+
+  # Make sure reindex closed all the scroll contexts
+  - do:
+      indices.stats:
+        index: source
+        metric: search
+  - match: {indices.source.total.search.open_contexts: 0}
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/20_validation.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/20_validation.yaml
index 54eb262e9ba45..be83c0cb9f24f 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/20_validation.yaml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/20_validation.yaml
@@ -104,3 +104,19 @@
       update_by_query:
         requests_per_second: 0
         index: test
+
+---
+"junk in slices fails":
+  - do:
+      catch: /Failed to parse int parameter \[slices\] with value \[junk\]/
+      update_by_query:
+        slices: junk
+        index: test
+
+---
+"zero slices fails":
+  - do:
+      catch: /\[slices\] must be at least 1/
+      update_by_query:
+        slices: 0
+        index: test
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/50_consistency.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/50_consistency.yaml
index bb373866416f6..6e9c40146c56f 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/50_consistency.yaml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/50_consistency.yaml
@@ -22,7 +22,7 @@
         wait_for_active_shards: 4
         timeout: 1s
   - match:
-      failures.0.cause.reason: /Not.enough.active.copies.to.meet.shard.count.of.\[4\].\(have.1,.needed.4\)..Timeout\:.\[1s\],.request:.\[BulkShardRequest.to.\[test\].containing.\[1\].requests\]/
+      failures.0.cause.reason: /Not.enough.active.copies.to.meet.shard.count.of.\[4\].\(have.1,.needed.4\)..Timeout\:.\[1s\],.request:.+/
 
   - do:
       update_by_query:
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/70_slices.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/70_slices.yaml
new file mode 100644
index 0000000000000..f390d43f9f362
--- /dev/null
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/70_slices.yaml
@@ -0,0 +1,261 @@
+---
+"Multiple slices":
+  - do:
+      index:
+        index:   test
+        type:    foo
+        id:      1
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   test
+        type:    foo
+        id:      2
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   test
+        type:    foo
+        id:      3
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   test
+        type:    foo
+        id:      4
+        body:    { "text": "test" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      update_by_query:
+        index: test
+        slices: 5
+        body:
+          query:
+            match_all: {}
+
+  - is_false: timed_out
+  - match: {updated: 4}
+  - is_false: created
+  - match: {version_conflicts: 0}
+  - match: {failures: []}
+  - match: {noops: 0}
+  - match: {throttled_millis: 0}
+  - gte: { took: 0 }
+  - is_false: task
+  - match: {slices.0.version_conflicts: 0}
+  - match: {slices.0.throttled_millis: 0}
+  - match: {slices.1.version_conflicts: 0}
+  - match: {slices.1.throttled_millis: 0}
+  - match: {slices.2.version_conflicts: 0}
+  - match: {slices.2.throttled_millis: 0}
+  - match: {slices.3.version_conflicts: 0}
+  - match: {slices.3.throttled_millis: 0}
+  - match: {slices.4.version_conflicts: 0}
+  - match: {slices.4.throttled_millis: 0}
+
+---
+"Multiple slices with wait_for_completion=false":
+  - do:
+      index:
+        index:   test
+        type:    foo
+        id:      1
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   test
+        type:    foo
+        id:      2
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   test
+        type:    foo
+        id:      3
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   test
+        type:    foo
+        id:      4
+        body:    { "text": "test" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      update_by_query:
+        index: test
+        wait_for_completion: false
+        slices: 5
+        body:
+          query:
+            match_all: {}
+  - is_false: timed_out
+  - match: {task: '/.+:\d+/'}
+  - set: {task: task}
+  - is_false: updated
+  - is_false: version_conflicts
+  - is_false: batches
+  - is_false: failures
+  - is_false: noops
+  - is_false: took
+  - is_false: throttled_millis
+  - is_false: created
+  - is_false: deleted
+
+  - do:
+      tasks.get:
+        wait_for_completion: true
+        task_id: $task
+  - is_false: node_failures
+  # The task will be in the response even if it finished before we got here
+  # because of task persistence.
+  - is_true: task
+  - match: {response.total: 4}
+  - match: {response.updated: 4}
+  - match: {response.version_conflicts: 0}
+  - match: {response.failures: []}
+  - match: {response.throttled_millis: 0}
+  - gte: { response.took: 0 }
+  - is_false: response.task
+  - match: {response.slices.0.version_conflicts: 0}
+  - match: {response.slices.0.throttled_millis: 0}
+  - match: {response.slices.1.version_conflicts: 0}
+  - match: {response.slices.1.throttled_millis: 0}
+  - match: {response.slices.2.version_conflicts: 0}
+  - match: {response.slices.2.throttled_millis: 0}
+  - match: {response.slices.3.version_conflicts: 0}
+  - match: {response.slices.3.throttled_millis: 0}
+  - match: {response.slices.4.version_conflicts: 0}
+  - match: {response.slices.4.throttled_millis: 0}
+  - match: {task.status.total: 4}
+  - match: {task.status.updated: 4}
+  - match: {task.status.version_conflicts: 0}
+  - match: {task.status.throttled_millis: 0}
+  - match: {task.status.slices.0.version_conflicts: 0}
+  - match: {task.status.slices.0.throttled_millis: 0}
+  - match: {task.status.slices.1.version_conflicts: 0}
+  - match: {task.status.slices.1.throttled_millis: 0}
+  - match: {task.status.slices.2.version_conflicts: 0}
+  - match: {task.status.slices.2.throttled_millis: 0}
+  - match: {task.status.slices.3.version_conflicts: 0}
+  - match: {task.status.slices.3.throttled_millis: 0}
+  - match: {task.status.slices.4.version_conflicts: 0}
+  - match: {task.status.slices.4.throttled_millis: 0}
+
+
+  # Only the "parent" reindex task wrote its status to the tasks index though
+  - do:
+      indices.refresh: {}
+  - do:
+      search:
+        index: .tasks
+  - match: { hits.total: 1 }
+
+---
+"Multiple slices with rethrottle":
+  - do:
+      index:
+        index:   test
+        type:    foo
+        id:      1
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   test
+        type:    foo
+        id:      2
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   test
+        type:    foo
+        id:      3
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   test
+        type:    foo
+        id:      4
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   test
+        type:    foo
+        id:      5
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   test
+        type:    foo
+        id:      6
+        body:    { "text": "test" }
+  - do:
+      indices.refresh: {}
+
+  # Start the task with a requests_per_second that should make it take a very long time
+  - do:
+      update_by_query:
+        index: test
+        wait_for_completion: false
+        requests_per_second: 0.0001
+        slices: 2
+        body:
+          query:
+            match_all: {}
+  - is_false: timed_out
+  - match: {task: '/.+:\d+/'}
+  - set: {task: task}
+  - is_false: updated
+  - is_false: version_conflicts
+  - is_false: batches
+  - is_false: failures
+  - is_false: noops
+  - is_false: took
+  - is_false: throttled_millis
+  - is_false: created
+  - is_false: deleted
+
+  # Allow the task to complete
+  - do:
+      reindex_rethrottle:
+        requests_per_second: -1
+        task_id: $task
+
+  - do:
+      tasks.get:
+        wait_for_completion: true
+        task_id: $task
+  - is_false: node_failures
+  # The task will be in the response even if it finished before we got here
+  # because of task persistence.
+  - is_true: task
+  - match: {response.total: 6}
+  - match: {response.updated: 6}
+  - match: {response.version_conflicts: 0}
+  - match: {response.failures: []}
+  - match: {response.throttled_millis: 0}
+  - gte: { response.took: 0 }
+  - is_false: response.task
+  - match: {response.slices.0.version_conflicts: 0}
+  - match: {response.slices.0.throttled_millis: 0}
+  - match: {response.slices.1.version_conflicts: 0}
+  - match: {response.slices.1.throttled_millis: 0}
+  - match: {task.status.total: 6}
+  - match: {task.status.updated: 6}
+  - match: {task.status.version_conflicts: 0}
+  - match: {task.status.throttled_millis: 0}
+  - match: {task.status.slices.0.version_conflicts: 0}
+  - match: {task.status.slices.0.throttled_millis: 0}
+  - match: {task.status.slices.1.version_conflicts: 0}
+  - match: {task.status.slices.1.throttled_millis: 0}
+
+  # Only the "parent" reindex task wrote its status to the tasks index though
+  - do:
+      indices.refresh: {}
+  - do:
+      search:
+        index: .tasks
+  - match: { hits.total: 1 }
diff --git a/modules/transport-netty3/build.gradle b/modules/transport-netty3/build.gradle
index eae0608f92f07..358a64cd88d27 100644
--- a/modules/transport-netty3/build.gradle
+++ b/modules/transport-netty3/build.gradle
@@ -29,6 +29,7 @@ esplugin {
     hasClientJar = true
 }
 
+compileJava.options.compilerArgs << "-Xlint:-try"
 compileTestJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-unchecked"
 
 dependencies {
@@ -38,11 +39,12 @@ dependencies {
 
 integTest {
     includePackaged = true
-    cluster {
-        setting 'http.type', 'netty3'
-        setting 'transport.type', 'netty3'
-        numNodes = 2
-    }
+}
+
+integTestCluster {
+    setting 'http.type', 'netty3'
+    setting 'transport.type', 'netty3'
+    numNodes = 2
 }
 
 thirdPartyAudit.excludes = [
diff --git a/modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/Netty3HttpChannel.java b/modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/Netty3HttpChannel.java
index a715abfd87791..d79fae21b8c3a 100644
--- a/modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/Netty3HttpChannel.java
+++ b/modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/Netty3HttpChannel.java
@@ -33,6 +33,7 @@
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.RestStatus;
 import org.jboss.netty.buffer.ChannelBuffer;
+import org.jboss.netty.buffer.ChannelBuffers;
 import org.jboss.netty.channel.Channel;
 import org.jboss.netty.channel.ChannelFuture;
 import org.jboss.netty.channel.ChannelFutureListener;
@@ -41,6 +42,7 @@
 import org.jboss.netty.handler.codec.http.CookieEncoder;
 import org.jboss.netty.handler.codec.http.DefaultHttpResponse;
 import org.jboss.netty.handler.codec.http.HttpHeaders;
+import org.jboss.netty.handler.codec.http.HttpMethod;
 import org.jboss.netty.handler.codec.http.HttpResponse;
 import org.jboss.netty.handler.codec.http.HttpResponseStatus;
 import org.jboss.netty.handler.codec.http.HttpVersion;
@@ -109,7 +111,11 @@ public void sendResponse(RestResponse response) {
         boolean addedReleaseListener = false;
         try {
             buffer = Netty3Utils.toChannelBuffer(content);
-            resp.setContent(buffer);
+            if (HttpMethod.HEAD.equals(nettyRequest.getMethod())) {
+                resp.setContent(ChannelBuffers.EMPTY_BUFFER);
+            } else {
+                resp.setContent(buffer);
+            }
 
             // If our response doesn't specify a content-type header, set one
             setHeaderField(resp, HttpHeaders.Names.CONTENT_TYPE, response.contentType(), false);
diff --git a/modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/Netty3HttpRequest.java b/modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/Netty3HttpRequest.java
index 2d17a12090d06..2813340fc5a33 100644
--- a/modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/Netty3HttpRequest.java
+++ b/modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/Netty3HttpRequest.java
@@ -21,16 +21,22 @@
 
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.transport.netty3.Netty3Utils;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.RestUtils;
+import org.elasticsearch.transport.netty3.Netty3Utils;
 import org.jboss.netty.channel.Channel;
+import org.jboss.netty.handler.codec.http.HttpHeaders;
 import org.jboss.netty.handler.codec.http.HttpMethod;
 import org.jboss.netty.handler.codec.http.HttpRequest;
 
 import java.net.SocketAddress;
-import java.util.HashMap;
+import java.util.AbstractMap;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
 
 public class Netty3HttpRequest extends RestRequest {
 
@@ -38,8 +44,8 @@ public class Netty3HttpRequest extends RestRequest {
     private final Channel channel;
     private final BytesReference content;
 
-    public Netty3HttpRequest(HttpRequest request, Channel channel) {
-        super(request.getUri());
+    public Netty3HttpRequest(NamedXContentRegistry xContentRegistry, HttpRequest request, Channel channel) {
+        super(xContentRegistry, request.getUri(), new HttpHeadersMap(request.headers()));
         this.request = request;
         this.channel = channel;
         if (request.getContent().readable()) {
@@ -120,14 +126,82 @@ public Channel getChannel() {
         return channel;
     }
 
-    @Override
-    public String header(String name) {
-        return request.headers().get(name);
-    }
+    /**
+     * A wrapper of {@link HttpHeaders} that implements a map to prevent copying unnecessarily. This class does not support modifications
+     * and due to the underlying implementation, it performs case insensitive lookups of key to values.
+     *
+     * It is important to note that this implementation does have some downsides in that each invocation of the
+     * {@link #values()} and {@link #entrySet()} methods will perform a copy of the values in the HttpHeaders rather than returning a
+     * view of the underlying values.
+     */
+    private static class HttpHeadersMap implements Map<String, List<String>> {
 
-    @Override
-    public Iterable<Map.Entry<String, String>> headers() {
-        return request.headers().entries();
-    }
+        private final HttpHeaders httpHeaders;
+
+        private HttpHeadersMap(HttpHeaders httpHeaders) {
+            this.httpHeaders = httpHeaders;
+        }
+
+        @Override
+        public int size() {
+            return httpHeaders.names().size();
+        }
+
+        @Override
+        public boolean isEmpty() {
+            return httpHeaders.isEmpty();
+        }
+
+        @Override
+        public boolean containsKey(Object key) {
+            return key instanceof String && httpHeaders.contains((String) key);
+        }
+
+        @Override
+        public boolean containsValue(Object value) {
+            return value instanceof List && httpHeaders.names().stream().map(httpHeaders::getAll).anyMatch(value::equals);
+        }
 
+        @Override
+        public List<String> get(Object key) {
+            return key instanceof String ? httpHeaders.getAll((String) key) : null;
+        }
+
+        @Override
+        public List<String> put(String key, List<String> value) {
+            throw new UnsupportedOperationException("modifications are not supported");
+        }
+
+        @Override
+        public List<String> remove(Object key) {
+            throw new UnsupportedOperationException("modifications are not supported");
+        }
+
+        @Override
+        public void putAll(Map<? extends String, ? extends List<String>> m) {
+            throw new UnsupportedOperationException("modifications are not supported");
+        }
+
+        @Override
+        public void clear() {
+            throw new UnsupportedOperationException("modifications are not supported");
+        }
+
+        @Override
+        public Set<String> keySet() {
+            return httpHeaders.names();
+        }
+
+        @Override
+        public Collection<List<String>> values() {
+            return httpHeaders.names().stream().map(k -> Collections.unmodifiableList(httpHeaders.getAll(k))).collect(Collectors.toList());
+        }
+
+        @Override
+        public Set<Entry<String, List<String>>> entrySet() {
+            return httpHeaders.names().stream()
+                .map(k -> new AbstractMap.SimpleImmutableEntry<>(k, httpHeaders.getAll(k))).collect(Collectors.toSet());
+
+        }
+    }
 }
diff --git a/modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/Netty3HttpRequestHandler.java b/modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/Netty3HttpRequestHandler.java
index 829a2d9aa06b6..50c8b1a6d9ba5 100644
--- a/modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/Netty3HttpRequestHandler.java
+++ b/modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/Netty3HttpRequestHandler.java
@@ -21,6 +21,7 @@
 
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.http.netty3.pipelining.OrderedUpstreamMessageEvent;
+import org.elasticsearch.transport.netty3.Netty3Utils;
 import org.jboss.netty.channel.ChannelHandler;
 import org.jboss.netty.channel.ChannelHandlerContext;
 import org.jboss.netty.channel.ExceptionEvent;
@@ -55,8 +56,8 @@ public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Ex
         }
 
         // the netty HTTP handling always copy over the buffer to its own buffer, either in NioWorker internally
-        // when reading, or using a cumalation buffer
-        Netty3HttpRequest httpRequest = new Netty3HttpRequest(request, e.getChannel());
+        // when reading, or using a cumulation buffer
+        Netty3HttpRequest httpRequest = new Netty3HttpRequest(serverTransport.xContentRegistry, request, e.getChannel());
         Netty3HttpChannel channel = new Netty3HttpChannel(serverTransport, httpRequest, oue, detailedErrorsEnabled, threadContext);
         serverTransport.dispatchRequest(httpRequest, channel);
         super.messageReceived(ctx, e);
@@ -64,6 +65,7 @@ public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Ex
 
     @Override
     public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) throws Exception {
+        Netty3Utils.maybeDie(e.getCause());
         serverTransport.exceptionCaught(ctx, e);
     }
 }
diff --git a/modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/Netty3HttpServerTransport.java b/modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/Netty3HttpServerTransport.java
index df5c02c2d323c..b8c81f0a2bf91 100644
--- a/modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/Netty3HttpServerTransport.java
+++ b/modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/Netty3HttpServerTransport.java
@@ -25,7 +25,6 @@
 import org.apache.logging.log4j.util.Supplier;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Setting;
@@ -41,9 +40,10 @@
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.http.BindHttpException;
 import org.elasticsearch.http.HttpInfo;
-import org.elasticsearch.http.HttpServerAdapter;
 import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.http.HttpStats;
 import org.elasticsearch.http.netty3.cors.Netty3CorsConfig;
@@ -149,9 +149,9 @@ public class Netty3HttpServerTransport extends AbstractLifecycleComponent implem
         Setting.byteSizeSetting("transport.netty.receive_predictor_size",
             settings -> {
                 long defaultReceiverPredictor = 512 * 1024;
-                if (JvmInfo.jvmInfo().getMem().getDirectMemoryMax().bytes() > 0) {
+                if (JvmInfo.jvmInfo().getMem().getDirectMemoryMax().getBytes() > 0) {
                     // we can guess a better default...
-                    long l = (long) ((0.3 * JvmInfo.jvmInfo().getMem().getDirectMemoryMax().bytes()) / SETTING_HTTP_WORKER_COUNT.get
+                    long l = (long) ((0.3 * JvmInfo.jvmInfo().getMem().getDirectMemoryMax().getBytes()) / SETTING_HTTP_WORKER_COUNT.get
                             (settings));
                     defaultReceiverPredictor = Math.min(defaultReceiverPredictor, Math.max(l, 64 * 1024));
                 }
@@ -193,6 +193,10 @@ public class Netty3HttpServerTransport extends AbstractLifecycleComponent implem
 
     protected final boolean detailedErrorsEnabled;
     protected final ThreadPool threadPool;
+    /**
+     * The registry used to construct parsers so they support {@link XContentParser#namedObject(Class, String, Object)}.
+     */
+    protected final NamedXContentRegistry xContentRegistry;
 
     protected final boolean tcpNoDelay;
     protected final boolean tcpKeepAlive;
@@ -204,6 +208,7 @@ public class Netty3HttpServerTransport extends AbstractLifecycleComponent implem
 
     protected final ByteSizeValue maxCumulationBufferCapacity;
     protected final int maxCompositeBufferComponents;
+    private final Dispatcher dispatcher;
 
     protected volatile ServerBootstrap serverBootstrap;
 
@@ -214,16 +219,17 @@ public class Netty3HttpServerTransport extends AbstractLifecycleComponent implem
     // package private for testing
     Netty3OpenChannelsHandler serverOpenChannels;
 
-    protected volatile HttpServerAdapter httpServerAdapter;
-
     private final Netty3CorsConfig corsConfig;
 
-    @Inject
-    public Netty3HttpServerTransport(Settings settings, NetworkService networkService, BigArrays bigArrays, ThreadPool threadPool) {
+    public Netty3HttpServerTransport(Settings settings, NetworkService networkService, BigArrays bigArrays, ThreadPool threadPool,
+            NamedXContentRegistry xContentRegistry, HttpServerTransport.Dispatcher dispatcher) {
         super(settings);
+        deprecationLogger.deprecated(deprecationMessage());
         this.networkService = networkService;
         this.bigArrays = bigArrays;
         this.threadPool = threadPool;
+        this.xContentRegistry = xContentRegistry;
+        this.dispatcher = dispatcher;
 
         ByteSizeValue maxContentLength = SETTING_HTTP_MAX_CONTENT_LENGTH.get(settings);
         this.maxChunkSize = SETTING_HTTP_MAX_CHUNK_SIZE.get(settings);
@@ -248,11 +254,11 @@ public Netty3HttpServerTransport(Settings settings, NetworkService networkServic
         // See AdaptiveReceiveBufferSizePredictor#DEFAULT_XXX for default values in netty..., we can use higher ones for us, even fixed one
         ByteSizeValue receivePredictorMin = SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_MIN.get(settings);
         ByteSizeValue receivePredictorMax = SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_MAX.get(settings);
-        if (receivePredictorMax.bytes() == receivePredictorMin.bytes()) {
-            receiveBufferSizePredictorFactory = new FixedReceiveBufferSizePredictorFactory((int) receivePredictorMax.bytes());
+        if (receivePredictorMax.getBytes() == receivePredictorMin.getBytes()) {
+            receiveBufferSizePredictorFactory = new FixedReceiveBufferSizePredictorFactory((int) receivePredictorMax.getBytes());
         } else {
             receiveBufferSizePredictorFactory = new AdaptiveReceiveBufferSizePredictorFactory(
-                (int) receivePredictorMin.bytes(), (int) receivePredictorMin.bytes(), (int) receivePredictorMax.bytes());
+                (int) receivePredictorMin.getBytes(), (int) receivePredictorMin.getBytes(), (int) receivePredictorMax.getBytes());
         }
 
         this.compression = SETTING_HTTP_COMPRESSION.get(settings);
@@ -262,7 +268,7 @@ public Netty3HttpServerTransport(Settings settings, NetworkService networkServic
         this.corsConfig = buildCorsConfig(settings);
 
         // validate max content length
-        if (maxContentLength.bytes() > Integer.MAX_VALUE) {
+        if (maxContentLength.getBytes() > Integer.MAX_VALUE) {
             logger.warn("maxContentLength[{}] set to high value, resetting it to [100mb]", maxContentLength);
             maxContentLength = new ByteSizeValue(100, ByteSizeUnit.MB);
         }
@@ -273,45 +279,52 @@ public Netty3HttpServerTransport(Settings settings, NetworkService networkServic
             this.maxContentLength, receivePredictorMin, receivePredictorMax, pipelining, pipeliningMaxEvents);
     }
 
-    public Settings settings() {
-        return this.settings;
+    protected String deprecationMessage() {
+        return "http type [netty3] is deprecated";
     }
 
-    @Override
-    public void httpServerAdapter(HttpServerAdapter httpServerAdapter) {
-        this.httpServerAdapter = httpServerAdapter;
+    public Settings settings() {
+        return this.settings;
     }
 
     @Override
     protected void doStart() {
-        this.serverOpenChannels = new Netty3OpenChannelsHandler(logger);
-        if (blockingServer) {
-            serverBootstrap = new ServerBootstrap(new OioServerSocketChannelFactory(
-                Executors.newCachedThreadPool(daemonThreadFactory(settings, HTTP_SERVER_BOSS_THREAD_NAME_PREFIX)),
-                Executors.newCachedThreadPool(daemonThreadFactory(settings, HTTP_SERVER_WORKER_THREAD_NAME_PREFIX))
-            ));
-        } else {
-            serverBootstrap = new ServerBootstrap(new NioServerSocketChannelFactory(
-                Executors.newCachedThreadPool(daemonThreadFactory(settings, HTTP_SERVER_BOSS_THREAD_NAME_PREFIX)),
-                Executors.newCachedThreadPool(daemonThreadFactory(settings, HTTP_SERVER_WORKER_THREAD_NAME_PREFIX)),
-                workerCount));
-        }
-        serverBootstrap.setPipelineFactory(configureServerChannelPipelineFactory());
+        boolean success = false;
+        try {
+            this.serverOpenChannels = new Netty3OpenChannelsHandler(logger);
+            if (blockingServer) {
+                serverBootstrap = new ServerBootstrap(new OioServerSocketChannelFactory(
+                    Executors.newCachedThreadPool(daemonThreadFactory(settings, HTTP_SERVER_BOSS_THREAD_NAME_PREFIX)),
+                    Executors.newCachedThreadPool(daemonThreadFactory(settings, HTTP_SERVER_WORKER_THREAD_NAME_PREFIX))
+                ));
+            } else {
+                serverBootstrap = new ServerBootstrap(new NioServerSocketChannelFactory(
+                    Executors.newCachedThreadPool(daemonThreadFactory(settings, HTTP_SERVER_BOSS_THREAD_NAME_PREFIX)),
+                    Executors.newCachedThreadPool(daemonThreadFactory(settings, HTTP_SERVER_WORKER_THREAD_NAME_PREFIX)),
+                    workerCount));
+            }
+            serverBootstrap.setPipelineFactory(configureServerChannelPipelineFactory());
 
-        serverBootstrap.setOption("child.tcpNoDelay", tcpNoDelay);
-        serverBootstrap.setOption("child.keepAlive", tcpKeepAlive);
-        if (tcpSendBufferSize.bytes() > 0) {
+            serverBootstrap.setOption("child.tcpNoDelay", tcpNoDelay);
+            serverBootstrap.setOption("child.keepAlive", tcpKeepAlive);
+            if (tcpSendBufferSize.getBytes() > 0) {
 
-            serverBootstrap.setOption("child.sendBufferSize", tcpSendBufferSize.bytes());
-        }
-        if (tcpReceiveBufferSize.bytes() > 0) {
-            serverBootstrap.setOption("child.receiveBufferSize", tcpReceiveBufferSize.bytes());
+                serverBootstrap.setOption("child.sendBufferSize", tcpSendBufferSize.getBytes());
+            }
+            if (tcpReceiveBufferSize.getBytes() > 0) {
+                serverBootstrap.setOption("child.receiveBufferSize", tcpReceiveBufferSize.getBytes());
+            }
+            serverBootstrap.setOption("receiveBufferSizePredictorFactory", receiveBufferSizePredictorFactory);
+            serverBootstrap.setOption("child.receiveBufferSizePredictorFactory", receiveBufferSizePredictorFactory);
+            serverBootstrap.setOption("reuseAddress", reuseAddress);
+            serverBootstrap.setOption("child.reuseAddress", reuseAddress);
+            this.boundAddress = createBoundHttpAddress();
+            success = true;
+        } finally {
+            if (success == false) {
+                doStop();  // otherwise we leak threads since we never moved to started
+            }
         }
-        serverBootstrap.setOption("receiveBufferSizePredictorFactory", receiveBufferSizePredictorFactory);
-        serverBootstrap.setOption("child.receiveBufferSizePredictorFactory", receiveBufferSizePredictorFactory);
-        serverBootstrap.setOption("reuseAddress", reuseAddress);
-        serverBootstrap.setOption("child.reuseAddress", reuseAddress);
-        this.boundAddress = createBoundHttpAddress();
     }
 
     private BoundTransportAddress createBoundHttpAddress() {
@@ -405,24 +418,21 @@ private Netty3CorsConfig buildCorsConfig(Settings settings) {
     private InetSocketTransportAddress bindAddress(final InetAddress hostAddress) {
         final AtomicReference<Exception> lastException = new AtomicReference<>();
         final AtomicReference<InetSocketAddress> boundSocket = new AtomicReference<>();
-        boolean success = port.iterate(new PortsRange.PortCallback() {
-            @Override
-            public boolean onPortNumber(int portNumber) {
-                try {
-                    synchronized (serverChannels) {
-                        Channel channel = serverBootstrap.bind(new InetSocketAddress(hostAddress, portNumber));
-                        serverChannels.add(channel);
-                        boundSocket.set((InetSocketAddress) channel.getLocalAddress());
-                    }
-                } catch (Exception e) {
-                    lastException.set(e);
-                    return false;
+        boolean success = port.iterate(portNumber -> {
+            try {
+                synchronized (serverChannels) {
+                    Channel channel = serverBootstrap.bind(new InetSocketAddress(hostAddress, portNumber));
+                    serverChannels.add(channel);
+                    boundSocket.set((InetSocketAddress) channel.getLocalAddress());
                 }
-                return true;
+            } catch (Exception e) {
+                lastException.set(e);
+                return false;
             }
+            return true;
         });
         if (!success) {
-            throw new BindHttpException("Failed to bind to [" + port + "]", lastException.get());
+            throw new BindHttpException("Failed to bind to [" + port.getPortRangeString() + "]", lastException.get());
         }
 
         if (logger.isDebugEnabled()) {
@@ -468,7 +478,7 @@ public HttpInfo info() {
         if (boundTransportAddress == null) {
             return null;
         }
-        return new HttpInfo(boundTransportAddress, maxContentLength.bytes());
+        return new HttpInfo(boundTransportAddress, maxContentLength.getBytes());
     }
 
     @Override
@@ -482,10 +492,14 @@ public Netty3CorsConfig getCorsConfig() {
     }
 
     protected void dispatchRequest(RestRequest request, RestChannel channel) {
-        httpServerAdapter.dispatchRequest(request, channel, threadPool.getThreadContext());
+        final ThreadContext threadContext = threadPool.getThreadContext();
+        try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
+            dispatcher.dispatchRequest(request, channel, threadContext);
+        }
     }
 
     protected void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) throws Exception {
+        Netty3Utils.maybeDie(e.getCause());
         if (e.getCause() instanceof ReadTimeoutException) {
             if (logger.isTraceEnabled()) {
                 logger.trace("Connection timeout [{}]", ctx.getChannel().getRemoteAddress());
@@ -533,15 +547,15 @@ public ChannelPipeline getPipeline() throws Exception {
             ChannelPipeline pipeline = Channels.pipeline();
             pipeline.addLast("openChannels", transport.serverOpenChannels);
             HttpRequestDecoder requestDecoder = new HttpRequestDecoder(
-                    (int) transport.maxInitialLineLength.bytes(),
-                    (int) transport.maxHeaderSize.bytes(),
-                    (int) transport.maxChunkSize.bytes()
+                (int) transport.maxInitialLineLength.getBytes(),
+                (int) transport.maxHeaderSize.getBytes(),
+                (int) transport.maxChunkSize.getBytes()
             );
-            if (transport.maxCumulationBufferCapacity.bytes() >= 0) {
-                if (transport.maxCumulationBufferCapacity.bytes() > Integer.MAX_VALUE) {
+            if (transport.maxCumulationBufferCapacity.getBytes() >= 0) {
+                if (transport.maxCumulationBufferCapacity.getBytes() > Integer.MAX_VALUE) {
                     requestDecoder.setMaxCumulationBufferCapacity(Integer.MAX_VALUE);
                 } else {
-                    requestDecoder.setMaxCumulationBufferCapacity((int) transport.maxCumulationBufferCapacity.bytes());
+                    requestDecoder.setMaxCumulationBufferCapacity((int) transport.maxCumulationBufferCapacity.getBytes());
                 }
             }
             if (transport.maxCompositeBufferComponents != -1) {
@@ -549,7 +563,7 @@ public ChannelPipeline getPipeline() throws Exception {
             }
             pipeline.addLast("decoder", requestDecoder);
             pipeline.addLast("decoder_compress", new HttpContentDecompressor());
-            HttpChunkAggregator httpChunkAggregator = new HttpChunkAggregator((int) transport.maxContentLength.bytes());
+            HttpChunkAggregator httpChunkAggregator = new HttpChunkAggregator((int) transport.maxContentLength.getBytes());
             if (transport.maxCompositeBufferComponents != -1) {
                 httpChunkAggregator.setMaxCumulationBufferComponents(transport.maxCompositeBufferComponents);
             }
@@ -568,4 +582,5 @@ public ChannelPipeline getPipeline() throws Exception {
             return pipeline;
         }
     }
+
 }
diff --git a/modules/transport-netty3/src/main/java/org/elasticsearch/transport/Netty3Plugin.java b/modules/transport-netty3/src/main/java/org/elasticsearch/transport/Netty3Plugin.java
index a1f9985c6a04a..adec1b158b582 100644
--- a/modules/transport-netty3/src/main/java/org/elasticsearch/transport/Netty3Plugin.java
+++ b/modules/transport-netty3/src/main/java/org/elasticsearch/transport/Netty3Plugin.java
@@ -19,19 +19,33 @@
 
 package org.elasticsearch.transport;
 
-import org.elasticsearch.common.network.NetworkModule;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.http.netty3.Netty3HttpServerTransport;
+import org.elasticsearch.indices.breaker.CircuitBreakerService;
+import org.elasticsearch.plugins.NetworkPlugin;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.netty3.Netty3Transport;
+import org.elasticsearch.transport.netty3.Netty3Utils;
 
-import java.security.AccessController;
-import java.security.PrivilegedAction;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
+import java.util.Map;
+import java.util.function.Supplier;
+
+public class Netty3Plugin extends Plugin implements NetworkPlugin {
+
+    static {
+        Netty3Utils.setup();
+    }
 
-public class Netty3Plugin extends Plugin {
     public static final String NETTY_TRANSPORT_NAME = "netty3";
     public static final String NETTY_HTTP_TRANSPORT_NAME = "netty3";
 
@@ -57,11 +71,22 @@ public List<Setting<?>> getSettings() {
         );
     }
 
-    public void onModule(NetworkModule networkModule) {
-        if (networkModule.canRegisterHttpExtensions()) {
-            networkModule.registerHttpTransport(NETTY_HTTP_TRANSPORT_NAME, Netty3HttpServerTransport.class);
-        }
-        networkModule.registerTransport(NETTY_TRANSPORT_NAME, Netty3Transport.class);
+    @Override
+    public Map<String, Supplier<Transport>> getTransports(Settings settings, ThreadPool threadPool, BigArrays bigArrays,
+                                                          CircuitBreakerService circuitBreakerService,
+                                                          NamedWriteableRegistry namedWriteableRegistry, NetworkService networkService) {
+        return Collections.singletonMap(NETTY_TRANSPORT_NAME, () -> new Netty3Transport(settings, threadPool, networkService, bigArrays,
+            namedWriteableRegistry, circuitBreakerService));
     }
 
+    @Override
+    public Map<String, Supplier<HttpServerTransport>> getHttpTransports(Settings settings, ThreadPool threadPool, BigArrays bigArrays,
+                                                                        CircuitBreakerService circuitBreakerService,
+                                                                        NamedWriteableRegistry namedWriteableRegistry,
+                                                                        NamedXContentRegistry xContentRegistry,
+                                                                        NetworkService networkService,
+                                                                        HttpServerTransport.Dispatcher dispatcher) {
+        return Collections.singletonMap(NETTY_HTTP_TRANSPORT_NAME, () -> new Netty3HttpServerTransport(settings, networkService,
+            bigArrays, threadPool, xContentRegistry, dispatcher));
+    }
 }
diff --git a/modules/transport-netty3/src/main/java/org/elasticsearch/transport/netty3/ChannelBufferStreamInput.java b/modules/transport-netty3/src/main/java/org/elasticsearch/transport/netty3/ChannelBufferStreamInput.java
index 299e4216b4d31..a4b21a8d22a89 100644
--- a/modules/transport-netty3/src/main/java/org/elasticsearch/transport/netty3/ChannelBufferStreamInput.java
+++ b/modules/transport-netty3/src/main/java/org/elasticsearch/transport/netty3/ChannelBufferStreamInput.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.jboss.netty.buffer.ChannelBuffer;
 
+import java.io.EOFException;
 import java.io.IOException;
 
 /**
@@ -35,7 +36,7 @@ class ChannelBufferStreamInput extends StreamInput {
     private final int startIndex;
     private final int endIndex;
 
-    public ChannelBufferStreamInput(ChannelBuffer buffer, int length) {
+    ChannelBufferStreamInput(ChannelBuffer buffer, int length) {
         if (length > buffer.readableBytes()) {
             throw new IndexOutOfBoundsException();
         }
@@ -67,6 +68,14 @@ public int available() throws IOException {
         return endIndex - buffer.readerIndex();
     }
 
+    @Override
+    protected void ensureCanReadBytes(int length) throws EOFException {
+        int bytesAvailable = endIndex - buffer.readerIndex();
+        if (bytesAvailable < length) {
+            throw new EOFException("tried to read: " + length + " bytes but only " + bytesAvailable + " remaining");
+        }
+    }
+
     @Override
     public void mark(int readlimit) {
         buffer.markReaderIndex();
diff --git a/modules/transport-netty3/src/main/java/org/elasticsearch/transport/netty3/Netty3MessageChannelHandler.java b/modules/transport-netty3/src/main/java/org/elasticsearch/transport/netty3/Netty3MessageChannelHandler.java
index e2ac10873cdb6..1c24812e90874 100644
--- a/modules/transport-netty3/src/main/java/org/elasticsearch/transport/netty3/Netty3MessageChannelHandler.java
+++ b/modules/transport-netty3/src/main/java/org/elasticsearch/transport/netty3/Netty3MessageChannelHandler.java
@@ -79,6 +79,7 @@ public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Ex
 
     @Override
     public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) throws Exception {
+        Netty3Utils.maybeDie(e.getCause());
         transport.exceptionCaught(ctx, e);
     }
 }
diff --git a/modules/transport-netty3/src/main/java/org/elasticsearch/transport/netty3/Netty3Transport.java b/modules/transport-netty3/src/main/java/org/elasticsearch/transport/netty3/Netty3Transport.java
index 6f1f70698601f..1bc31fcb4642e 100644
--- a/modules/transport-netty3/src/main/java/org/elasticsearch/transport/netty3/Netty3Transport.java
+++ b/modules/transport-netty3/src/main/java/org/elasticsearch/transport/netty3/Netty3Transport.java
@@ -22,10 +22,10 @@
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.network.NetworkService;
@@ -36,13 +36,16 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.InetSocketTransportAddress;
 import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.monitor.jvm.JvmInfo;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.ConnectTransportException;
+import org.elasticsearch.transport.ConnectionProfile;
 import org.elasticsearch.transport.TcpTransport;
+import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportServiceAdapter;
 import org.elasticsearch.transport.TransportSettings;
 import org.jboss.netty.bootstrap.ClientBootstrap;
@@ -74,6 +77,7 @@
 import java.util.Map;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ThreadFactory;
+import java.util.function.Consumer;
 
 import static org.elasticsearch.common.settings.Setting.byteSizeSetting;
 import static org.elasticsearch.common.settings.Setting.intSetting;
@@ -111,9 +115,9 @@ public class Netty3Transport extends TcpTransport<Channel> {
             "transport.netty.receive_predictor_size",
             settings -> {
                 long defaultReceiverPredictor = 512 * 1024;
-                if (JvmInfo.jvmInfo().getMem().getDirectMemoryMax().bytes() > 0) {
+                if (JvmInfo.jvmInfo().getMem().getDirectMemoryMax().getBytes() > 0) {
                     // we can guess a better default...
-                    long l = (long) ((0.3 * JvmInfo.jvmInfo().getMem().getDirectMemoryMax().bytes()) / WORKER_COUNT.get(settings));
+                    long l = (long) ((0.3 * JvmInfo.jvmInfo().getMem().getDirectMemoryMax().getBytes()) / WORKER_COUNT.get(settings));
                     defaultReceiverPredictor = Math.min(defaultReceiverPredictor, Math.max(l, 64 * 1024));
                 }
                 return new ByteSizeValue(defaultReceiverPredictor).toString();
@@ -138,10 +142,10 @@ public class Netty3Transport extends TcpTransport<Channel> {
     protected volatile ClientBootstrap clientBootstrap;
     protected final Map<String, ServerBootstrap> serverBootstraps = newConcurrentMap();
 
-    @Inject
     public Netty3Transport(Settings settings, ThreadPool threadPool, NetworkService networkService, BigArrays bigArrays,
                            NamedWriteableRegistry namedWriteableRegistry, CircuitBreakerService circuitBreakerService) {
         super("netty3", settings, threadPool, bigArrays, circuitBreakerService, namedWriteableRegistry, networkService);
+        deprecationLogger.deprecated(deprecationMessage());
         this.workerCount = WORKER_COUNT.get(settings);
         this.maxCumulationBufferCapacity = NETTY_MAX_CUMULATION_BUFFER_CAPACITY.get(settings);
         this.maxCompositeBufferComponents = NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS.get(settings);
@@ -149,14 +153,18 @@ public Netty3Transport(Settings settings, ThreadPool threadPool, NetworkService
         // See AdaptiveReceiveBufferSizePredictor#DEFAULT_XXX for default values in netty..., we can use higher ones for us, even fixed one
         this.receivePredictorMin = NETTY_RECEIVE_PREDICTOR_MIN.get(settings);
         this.receivePredictorMax = NETTY_RECEIVE_PREDICTOR_MAX.get(settings);
-        if (receivePredictorMax.bytes() == receivePredictorMin.bytes()) {
-            receiveBufferSizePredictorFactory = new FixedReceiveBufferSizePredictorFactory((int) receivePredictorMax.bytes());
+        if (receivePredictorMax.getBytes() == receivePredictorMin.getBytes()) {
+            receiveBufferSizePredictorFactory = new FixedReceiveBufferSizePredictorFactory((int) receivePredictorMax.getBytes());
         } else {
-            receiveBufferSizePredictorFactory = new AdaptiveReceiveBufferSizePredictorFactory((int) receivePredictorMin.bytes(),
-                (int) receivePredictorMin.bytes(), (int) receivePredictorMax.bytes());
+            receiveBufferSizePredictorFactory = new AdaptiveReceiveBufferSizePredictorFactory((int) receivePredictorMin.getBytes(),
+                (int) receivePredictorMin.getBytes(), (int) receivePredictorMax.getBytes());
         }
     }
 
+    protected String deprecationMessage() {
+        return "transport type [netty3] is deprecated";
+    }
+
     TransportServiceAdapter transportServiceAdapter() {
         return transportServiceAdapter;
     }
@@ -204,7 +212,7 @@ private ClientBootstrap createClientBootstrap() {
                     new HashedWheelTimer(daemonThreadFactory(settings, "transport_client_timer"))));
         }
         clientBootstrap.setPipelineFactory(configureClientChannelPipelineFactory());
-        clientBootstrap.setOption("connectTimeoutMillis", connectTimeout.millis());
+        clientBootstrap.setOption("connectTimeoutMillis", defaultConnectionProfile.getConnectTimeout().millis());
 
         boolean tcpNoDelay = TCP_NO_DELAY.get(settings);
         clientBootstrap.setOption("tcpNoDelay", tcpNoDelay);
@@ -213,13 +221,13 @@ private ClientBootstrap createClientBootstrap() {
         clientBootstrap.setOption("keepAlive", tcpKeepAlive);
 
         ByteSizeValue tcpSendBufferSize = TCP_SEND_BUFFER_SIZE.get(settings);
-        if (tcpSendBufferSize.bytes() > 0) {
-            clientBootstrap.setOption("sendBufferSize", tcpSendBufferSize.bytes());
+        if (tcpSendBufferSize.getBytes() > 0) {
+            clientBootstrap.setOption("sendBufferSize", tcpSendBufferSize.getBytes());
         }
 
         ByteSizeValue tcpReceiveBufferSize = TCP_RECEIVE_BUFFER_SIZE.get(settings);
-        if (tcpReceiveBufferSize.bytes() > 0) {
-            clientBootstrap.setOption("receiveBufferSize", tcpReceiveBufferSize.bytes());
+        if (tcpReceiveBufferSize.getBytes() > 0) {
+            clientBootstrap.setOption("receiveBufferSize", tcpReceiveBufferSize.getBytes());
         }
 
         clientBootstrap.setOption("receiveBufferSizePredictorFactory", receiveBufferSizePredictorFactory);
@@ -254,13 +262,13 @@ private Settings createFallbackSettings() {
 
         ByteSizeValue fallbackTcpSendBufferSize = settings.getAsBytesSize("transport.netty.tcp_send_buffer_size",
             TCP_SEND_BUFFER_SIZE.get(settings));
-        if (fallbackTcpSendBufferSize.bytes() >= 0) {
+        if (fallbackTcpSendBufferSize.getBytes() >= 0) {
             fallbackSettingsBuilder.put("tcp_send_buffer_size", fallbackTcpSendBufferSize);
         }
 
         ByteSizeValue fallbackTcpBufferSize = settings.getAsBytesSize("transport.netty.tcp_receive_buffer_size",
             TCP_RECEIVE_BUFFER_SIZE.get(settings));
-        if (fallbackTcpBufferSize.bytes() >= 0) {
+        if (fallbackTcpBufferSize.getBytes() >= 0) {
             fallbackSettingsBuilder.put("tcp_receive_buffer_size", fallbackTcpBufferSize);
         }
 
@@ -281,8 +289,13 @@ private void createServerBootstrap(String name, Settings settings) {
         if (logger.isDebugEnabled()) {
             logger.debug("using profile[{}], worker_count[{}], port[{}], bind_host[{}], publish_host[{}], compress[{}], "
                     + "connect_timeout[{}], connections_per_node[{}/{}/{}/{}/{}], receive_predictor[{}->{}]",
-                name, workerCount, port, bindHost, publishHost, compress, connectTimeout, connectionsPerNodeRecovery,
-                connectionsPerNodeBulk, connectionsPerNodeReg, connectionsPerNodeState, connectionsPerNodePing, receivePredictorMin,
+                name, workerCount, port, bindHost, publishHost, compress,
+                defaultConnectionProfile.getConnectTimeout(),
+                defaultConnectionProfile.getNumConnectionsPerType(TransportRequestOptions.Type.RECOVERY),
+                defaultConnectionProfile.getNumConnectionsPerType(TransportRequestOptions.Type.BULK),
+                defaultConnectionProfile.getNumConnectionsPerType(TransportRequestOptions.Type.REG),
+                defaultConnectionProfile.getNumConnectionsPerType(TransportRequestOptions.Type.STATE),
+                defaultConnectionProfile.getNumConnectionsPerType(TransportRequestOptions.Type.PING), receivePredictorMin,
                 receivePredictorMax);
         }
 
@@ -307,11 +320,11 @@ private void createServerBootstrap(String name, Settings settings) {
         if (!"default".equals(tcpKeepAlive)) {
             serverBootstrap.setOption("child.keepAlive", Booleans.parseBoolean(tcpKeepAlive, null));
         }
-        if (tcpSendBufferSize != null && tcpSendBufferSize.bytes() > 0) {
-            serverBootstrap.setOption("child.sendBufferSize", tcpSendBufferSize.bytes());
+        if (tcpSendBufferSize != null && tcpSendBufferSize.getBytes() > 0) {
+            serverBootstrap.setOption("child.sendBufferSize", tcpSendBufferSize.getBytes());
         }
-        if (tcpReceiveBufferSize != null && tcpReceiveBufferSize.bytes() > 0) {
-            serverBootstrap.setOption("child.receiveBufferSize", tcpReceiveBufferSize.bytes());
+        if (tcpReceiveBufferSize != null && tcpReceiveBufferSize.getBytes() > 0) {
+            serverBootstrap.setOption("child.receiveBufferSize", tcpReceiveBufferSize.getBytes());
         }
         serverBootstrap.setOption("receiveBufferSizePredictorFactory", receiveBufferSizePredictorFactory);
         serverBootstrap.setOption("child.receiveBufferSizePredictorFactory", receiveBufferSizePredictorFactory);
@@ -333,55 +346,44 @@ public long serverOpen() {
         return channels == null ? 0 : channels.numberOfOpenChannels();
     }
 
-    protected NodeChannels connectToChannelsLight(DiscoveryNode node) {
-        InetSocketAddress address = ((InetSocketTransportAddress) node.getAddress()).address();
-        ChannelFuture connect = clientBootstrap.connect(address);
-        connect.awaitUninterruptibly((long) (connectTimeout.millis() * 1.5));
-        if (!connect.isSuccess()) {
-            throw new ConnectTransportException(node, "connect_timeout[" + connectTimeout + "]", connect.getCause());
-        }
-        Channel[] channels = new Channel[1];
-        channels[0] = connect.getChannel();
-        channels[0].getCloseFuture().addListener(new ChannelCloseListener(node));
-        return new NodeChannels(channels, channels, channels, channels, channels);
-    }
-    protected NodeChannels connectToChannels(DiscoveryNode node) throws IOException {
-        final NodeChannels nodeChannels = new NodeChannels(new Channel[connectionsPerNodeRecovery], new Channel[connectionsPerNodeBulk],
-            new Channel[connectionsPerNodeReg], new Channel[connectionsPerNodeState],
-            new Channel[connectionsPerNodePing]);
+    @Override
+    protected NodeChannels connectToChannels(DiscoveryNode node, ConnectionProfile profile, Consumer<Channel> onChannelClose) {
+        final Channel[] channels = new Channel[profile.getNumConnections()];
+        final NodeChannels nodeChannels = new NodeChannels(node, channels, profile);
         boolean success = false;
         try {
-            int numConnections = connectionsPerNodeBulk + connectionsPerNodePing + connectionsPerNodeRecovery + connectionsPerNodeReg
-                + connectionsPerNodeState;
-            ArrayList<ChannelFuture> connections = new ArrayList<>();
-            InetSocketAddress address = ((InetSocketTransportAddress) node.getAddress()).address();
-            for (int i = 0; i < numConnections; i++) {
+            final TimeValue connectTimeout;
+            final ClientBootstrap clientBootstrap;
+            final TimeValue defaultConnectTimeout = defaultConnectionProfile.getConnectTimeout();
+            if (profile.getConnectTimeout() != null && profile.getConnectTimeout().equals(defaultConnectTimeout) == false) {
+                clientBootstrap = new ClientBootstrap(this.clientBootstrap.getFactory());
+                clientBootstrap.setPipelineFactory(this.clientBootstrap.getPipelineFactory());
+                clientBootstrap.setOptions(this.clientBootstrap.getOptions());
+                clientBootstrap.setOption("connectTimeoutMillis", Math.toIntExact(profile.getConnectTimeout().millis()));
+                connectTimeout = profile.getConnectTimeout();
+            } else {
+                connectTimeout = defaultConnectTimeout;
+                clientBootstrap = this.clientBootstrap;
+            }
+            final ArrayList<ChannelFuture> connections = new ArrayList<>(channels.length);
+            final InetSocketAddress address = ((InetSocketTransportAddress) node.getAddress()).address();
+            for (int i = 0; i < channels.length; i++) {
                 connections.add(clientBootstrap.connect(address));
             }
+            final ChannelFutureListener listener = future -> onChannelClose.accept(future.getChannel());
             final Iterator<ChannelFuture> iterator = connections.iterator();
             try {
-                for (Channel[] channels : nodeChannels.getChannelArrays()) {
-                    for (int i = 0; i < channels.length; i++) {
-                        assert iterator.hasNext();
-                        ChannelFuture future = iterator.next();
-                        future.awaitUninterruptibly((long) (connectTimeout.millis() * 1.5));
-                        if (!future.isSuccess()) {
-                            throw new ConnectTransportException(node, "connect_timeout[" + connectTimeout + "]", future.getCause());
-                        }
-                        channels[i] = future.getChannel();
-                        channels[i].getCloseFuture().addListener(new ChannelCloseListener(node));
-                    }
-                }
-                if (nodeChannels.recovery.length == 0) {
-                    if (nodeChannels.bulk.length > 0) {
-                        nodeChannels.recovery = nodeChannels.bulk;
-                    } else {
-                        nodeChannels.recovery = nodeChannels.reg;
+                for (int i = 0; i < channels.length; i++) {
+                    assert iterator.hasNext();
+                    ChannelFuture future = iterator.next();
+                    future.awaitUninterruptibly((long) (connectTimeout.millis() * 1.5));
+                    if (!future.isSuccess()) {
+                        throw new ConnectTransportException(node, "connect_timeout[" + connectTimeout + "]", future.getCause());
                     }
+                    channels[i] = future.getChannel();
+                    channels[i].getCloseFuture().addListener(listener);
                 }
-                if (nodeChannels.bulk.length == 0) {
-                    nodeChannels.bulk = nodeChannels.reg;
-                }
+                assert iterator.hasNext() == false : "not all created connection have been consumed";
             } catch (RuntimeException e) {
                 for (ChannelFuture future : Collections.unmodifiableList(connections)) {
                     future.cancel();
@@ -398,7 +400,11 @@ protected NodeChannels connectToChannels(DiscoveryNode node) throws IOException
             success = true;
         } finally {
             if (success == false) {
-                nodeChannels.close();
+                try {
+                    nodeChannels.close();
+                } catch (IOException e) {
+                    logger.trace("exception while closing channels", e);
+                }
             }
         }
         return nodeChannels;
@@ -419,11 +425,11 @@ public ClientChannelPipelineFactory(Netty3Transport nettyTransport) {
         public ChannelPipeline getPipeline() throws Exception {
             ChannelPipeline channelPipeline = Channels.pipeline();
             Netty3SizeHeaderFrameDecoder sizeHeader = new Netty3SizeHeaderFrameDecoder();
-            if (nettyTransport.maxCumulationBufferCapacity.bytes() >= 0) {
-                if (nettyTransport.maxCumulationBufferCapacity.bytes() > Integer.MAX_VALUE) {
+            if (nettyTransport.maxCumulationBufferCapacity.getBytes() >= 0) {
+                if (nettyTransport.maxCumulationBufferCapacity.getBytes() > Integer.MAX_VALUE) {
                     sizeHeader.setMaxCumulationBufferCapacity(Integer.MAX_VALUE);
                 } else {
-                    sizeHeader.setMaxCumulationBufferCapacity((int) nettyTransport.maxCumulationBufferCapacity.bytes());
+                    sizeHeader.setMaxCumulationBufferCapacity((int) nettyTransport.maxCumulationBufferCapacity.getBytes());
                 }
             }
             if (nettyTransport.maxCompositeBufferComponents != -1) {
@@ -457,11 +463,11 @@ public ChannelPipeline getPipeline() throws Exception {
             ChannelPipeline channelPipeline = Channels.pipeline();
             channelPipeline.addLast("openChannels", nettyTransport.serverOpenChannels);
             Netty3SizeHeaderFrameDecoder sizeHeader = new Netty3SizeHeaderFrameDecoder();
-            if (nettyTransport.maxCumulationBufferCapacity.bytes() > 0) {
-                if (nettyTransport.maxCumulationBufferCapacity.bytes() > Integer.MAX_VALUE) {
+            if (nettyTransport.maxCumulationBufferCapacity.getBytes() > 0) {
+                if (nettyTransport.maxCumulationBufferCapacity.getBytes() > Integer.MAX_VALUE) {
                     sizeHeader.setMaxCumulationBufferCapacity(Integer.MAX_VALUE);
                 } else {
-                    sizeHeader.setMaxCumulationBufferCapacity((int) nettyTransport.maxCumulationBufferCapacity.bytes());
+                    sizeHeader.setMaxCumulationBufferCapacity((int) nettyTransport.maxCumulationBufferCapacity.getBytes());
                 }
             }
             if (nettyTransport.maxCompositeBufferComponents != -1) {
@@ -473,57 +479,39 @@ public ChannelPipeline getPipeline() throws Exception {
         }
     }
 
-    protected class ChannelCloseListener implements ChannelFutureListener {
-
-        private final DiscoveryNode node;
-
-        private ChannelCloseListener(DiscoveryNode node) {
-            this.node = node;
-        }
-
-        @Override
-        public void operationComplete(final ChannelFuture future) throws Exception {
-            NodeChannels nodeChannels = connectedNodes.get(node);
-            if (nodeChannels != null && nodeChannels.hasChannel(future.getChannel())) {
-                threadPool.generic().execute(() -> {
-                    disconnectFromNode(node, future.getChannel(), "channel closed event");
-                });
-            }
-        }
-    }
-
     @Override
-    protected void sendMessage(Channel channel, BytesReference reference, Runnable sendListener, boolean close) {
+    protected void sendMessage(Channel channel, BytesReference reference, ActionListener<Channel> listener) {
         final ChannelFuture future = channel.write(Netty3Utils.toChannelBuffer(reference));
-        if (close) {
-            future.addListener(f -> {
-                try {
-                    sendListener.run();
-                } finally {
-                    f.getChannel().close();
-                }
-            });
-        } else {
-            future.addListener(future1 -> sendListener.run());
-        }
+        future.addListener(f -> {
+            if (f.isSuccess()) {
+                listener.onResponse(channel);
+            } else {
+                final Throwable cause = f.getCause();
+                Netty3Utils.maybeDie(cause);
+                logger.warn((Supplier<?>) () ->
+                    new ParameterizedMessage("write and flush on the network layer failed (channel: {})", channel), cause);
+                assert cause instanceof Exception;
+                listener.onFailure((Exception) cause);
+            }
+        });
     }
 
     @Override
-    protected void closeChannels(List<Channel> channels) {
-        List<ChannelFuture> futures = new ArrayList<>();
-
-        for (Channel channel : channels) {
-            try {
+    protected void closeChannels(List<Channel> channels, boolean blocking) throws IOException {
+        if (blocking) {
+            Netty3Utils.closeChannels(channels);
+        } else {
+            for (final Channel channel : channels) {
                 if (channel != null && channel.isOpen()) {
-                    futures.add(channel.close());
+                    final ChannelFuture closeFuture = channel.close();
+                    closeFuture.addListener(f -> {
+                        if (f.isSuccess() == false) {
+                            logger.warn("failed to close channel", f.getCause());
+                        }
+                    });
                 }
-            } catch (Exception e) {
-                logger.trace("failed to close channel", e);
             }
         }
-        for (ChannelFuture future : futures) {
-            future.awaitUninterruptibly();
-        }
     }
 
     @Override
diff --git a/modules/transport-netty3/src/main/java/org/elasticsearch/transport/netty3/Netty3Utils.java b/modules/transport-netty3/src/main/java/org/elasticsearch/transport/netty3/Netty3Utils.java
index 2cbf92997b472..3150644fe005b 100644
--- a/modules/transport-netty3/src/main/java/org/elasticsearch/transport/netty3/Netty3Utils.java
+++ b/modules/transport-netty3/src/main/java/org/elasticsearch/transport/netty3/Netty3Utils.java
@@ -18,24 +18,38 @@
  */
 package org.elasticsearch.transport.netty3;
 
+import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefIterator;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.common.logging.Loggers;
 import org.jboss.netty.buffer.ChannelBuffer;
 import org.jboss.netty.buffer.ChannelBuffers;
+import org.jboss.netty.channel.Channel;
+import org.jboss.netty.channel.ChannelFuture;
 import org.jboss.netty.logging.InternalLogger;
 import org.jboss.netty.logging.InternalLoggerFactory;
 import org.jboss.netty.util.ThreadNameDeterminer;
 import org.jboss.netty.util.ThreadRenamingRunnable;
 
 import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
 import java.security.AccessController;
 import java.security.PrivilegedAction;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Optional;
+import java.util.Queue;
+import java.util.stream.Collectors;
 
 /**
  */
@@ -101,7 +115,6 @@ public String determineThreadName(String currentThreadName, String proposedThrea
         InternalLoggerFactory.setDefaultFactory(new InternalLoggerFactory() {
             @Override
             public InternalLogger newInstance(String name) {
-                name = name.replace("org.jboss.netty.", "netty3.").replace("org.jboss.netty.", "netty3.");
                 return new Netty3InternalESLogger(Loggers.getLogger(name));
             }
         });
@@ -171,4 +184,96 @@ public static BytesReference toBytesReference(ChannelBuffer channelBuffer) {
     public static BytesReference toBytesReference(ChannelBuffer channelBuffer, int size) {
         return new ChannelBufferBytesReference(channelBuffer, size);
     }
+
+    public static void closeChannels(Collection<Channel> channels) throws IOException {
+        IOException closingExceptions = null;
+        final List<ChannelFuture> futures = new ArrayList<>();
+        for (final Channel channel : channels) {
+            try {
+                if (channel != null && channel.isOpen()) {
+                    futures.add(channel.close());
+                }
+            } catch (final Exception e) {
+                if (closingExceptions == null) {
+                    closingExceptions = new IOException("failed to close channels");
+                }
+                closingExceptions.addSuppressed(e);
+            }
+        }
+        for (ChannelFuture future : futures) {
+            future.awaitUninterruptibly();
+        }
+
+        if (closingExceptions != null) {
+            throw closingExceptions;
+        }
+    }
+
+    /**
+     * If the specified cause is an unrecoverable error, this method will rethrow the cause on a separate thread so that it can not be
+     * caught and bubbles up to the uncaught exception handler.
+     *
+     * @param cause the throwable to test
+     */
+    public static void maybeDie(final Throwable cause) {
+        final Optional<Error> maybeError = maybeError(cause);
+        if (maybeError.isPresent()) {
+            /*
+             * Here be dragons. We want to rethrow this so that it bubbles up to the uncaught exception handler. Yet, Netty wraps too many
+             * invocations of user-code in try/catch blocks that swallow all throwables. This means that a rethrow here will not bubble up
+             * to where we want it to. So, we fork a thread and throw the exception from there where Netty can not get to it. We do not wrap
+             * the exception so as to not lose the original cause during exit.
+             */
+            try {
+                // try to log the current stack trace
+                final StackTraceElement[] stackTrace = Thread.currentThread().getStackTrace();
+                final String formatted = Arrays.stream(stackTrace).skip(1).map(e -> "\tat " + e).collect(Collectors.joining("\n"));
+                final Logger logger = ESLoggerFactory.getLogger(Netty3Utils.class);
+                logger.error("fatal error on the network layer\n{}", formatted);
+            } finally {
+                new Thread(
+                        () -> {
+                            throw maybeError.get();
+                        })
+                        .start();
+            }
+        }
+    }
+
+    static final int MAX_ITERATIONS = 1024;
+
+    /**
+     * Unwrap the specified throwable looking for any suppressed errors or errors as a root cause of the specified throwable.
+     *
+     * @param cause the root throwable
+     *
+     * @return an optional error if one is found suppressed or a root cause in the tree rooted at the specified throwable
+     */
+    static Optional<Error> maybeError(final Throwable cause) {
+        // early terminate if the cause is already an error
+        if (cause instanceof Error) {
+            return Optional.of((Error) cause);
+        }
+
+        final Queue<Throwable> queue = new LinkedList<>();
+        queue.add(cause);
+        int iterations = 0;
+        while (!queue.isEmpty()) {
+            iterations++;
+            if (iterations > MAX_ITERATIONS) {
+                ESLoggerFactory.getLogger(Netty3Utils.class).warn("giving up looking for fatal errors on the network layer", cause);
+                break;
+            }
+            final Throwable current = queue.remove();
+            if (current instanceof Error) {
+                return Optional.of((Error) current);
+            }
+            Collections.addAll(queue, current.getSuppressed());
+            if (current.getCause() != null) {
+                queue.add(current.getCause());
+            }
+        }
+        return Optional.empty();
+    }
+
 }
diff --git a/modules/transport-netty3/src/test/java/org/elasticsearch/http/netty3/Netty3ClientYamlTestSuiteIT.java b/modules/transport-netty3/src/test/java/org/elasticsearch/http/netty3/Netty3ClientYamlTestSuiteIT.java
index 9ed1df1cfed94..13c755436bcc3 100644
--- a/modules/transport-netty3/src/test/java/org/elasticsearch/http/netty3/Netty3ClientYamlTestSuiteIT.java
+++ b/modules/transport-netty3/src/test/java/org/elasticsearch/http/netty3/Netty3ClientYamlTestSuiteIT.java
@@ -22,15 +22,13 @@
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
+
 import org.apache.lucene.util.TimeUnits;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-
-import java.io.IOException;
 
 // TODO: remove timeout after address slow test issue
-@TimeoutSuite(millis = 30 * TimeUnits.MINUTE)
+@TimeoutSuite(millis = 40 * TimeUnits.MINUTE) // some of the windows test VMs are slow as hell
 public class Netty3ClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
     public Netty3ClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
@@ -38,8 +36,8 @@ public Netty3ClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCan
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
-        return ESClientYamlSuiteTestCase.createParameters(0, 1);
+    public static Iterable<Object[]> parameters() throws Exception {
+        return createParameters();
     }
 
 }
diff --git a/modules/transport-netty3/src/test/java/org/elasticsearch/http/netty3/Netty3HttpChannelTests.java b/modules/transport-netty3/src/test/java/org/elasticsearch/http/netty3/Netty3HttpChannelTests.java
index ccb8aedd94cf6..e71530a82cf6c 100644
--- a/modules/transport-netty3/src/test/java/org/elasticsearch/http/netty3/Netty3HttpChannelTests.java
+++ b/modules/transport-netty3/src/test/java/org/elasticsearch/http/netty3/Netty3HttpChannelTests.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.MockBigArrays;
 import org.elasticsearch.http.HttpTransportSettings;
+import org.elasticsearch.http.NullDispatcher;
 import org.elasticsearch.http.netty3.cors.Netty3CorsHandler;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.rest.RestResponse;
@@ -78,6 +79,7 @@ public void setup() throws Exception {
 
     @After
     public void shutdown() throws Exception {
+        assertWarnings("http type [netty3] is deprecated");
         if (threadPool != null) {
             threadPool.shutdownNow();
         }
@@ -175,11 +177,12 @@ public void testThatAnyOriginWorks() {
 
     public void testHeadersSet() {
         Settings settings = Settings.builder().build();
-        httpServerTransport = new Netty3HttpServerTransport(settings, networkService, bigArrays, threadPool);
+        httpServerTransport = new Netty3HttpServerTransport(settings, networkService, bigArrays, threadPool, xContentRegistry(),
+            new NullDispatcher());
         HttpRequest httpRequest = new TestHttpRequest();
         httpRequest.headers().add(HttpHeaders.Names.ORIGIN, "remote");
         WriteCapturingChannel writeCapturingChannel = new WriteCapturingChannel();
-        Netty3HttpRequest request = new Netty3HttpRequest(httpRequest, writeCapturingChannel);
+        Netty3HttpRequest request = new Netty3HttpRequest(xContentRegistry(), httpRequest, writeCapturingChannel);
 
         // send a response
         Netty3HttpChannel channel =
@@ -202,12 +205,13 @@ public void testHeadersSet() {
 
     private HttpResponse execRequestWithCors(final Settings settings, final String originValue, final String host) {
         // construct request and send it over the transport layer
-        httpServerTransport = new Netty3HttpServerTransport(settings, networkService, bigArrays, threadPool);
+        httpServerTransport = new Netty3HttpServerTransport(settings, networkService, bigArrays, threadPool, xContentRegistry(),
+            new NullDispatcher());
         HttpRequest httpRequest = new TestHttpRequest();
         httpRequest.headers().add(HttpHeaders.Names.ORIGIN, originValue);
         httpRequest.headers().add(HttpHeaders.Names.HOST, host);
         WriteCapturingChannel writeCapturingChannel = new WriteCapturingChannel();
-        Netty3HttpRequest request = new Netty3HttpRequest(httpRequest, writeCapturingChannel);
+        Netty3HttpRequest request = new Netty3HttpRequest(xContentRegistry(), httpRequest, writeCapturingChannel);
 
         Netty3HttpChannel channel =
                 new Netty3HttpChannel(httpServerTransport, request, null, randomBoolean(), threadPool.getThreadContext());
diff --git a/modules/transport-netty3/src/test/java/org/elasticsearch/http/netty3/Netty3HttpServerPipeliningTests.java b/modules/transport-netty3/src/test/java/org/elasticsearch/http/netty3/Netty3HttpServerPipeliningTests.java
index b432708de1595..2bc6eb5175db0 100644
--- a/modules/transport-netty3/src/test/java/org/elasticsearch/http/netty3/Netty3HttpServerPipeliningTests.java
+++ b/modules/transport-netty3/src/test/java/org/elasticsearch/http/netty3/Netty3HttpServerPipeliningTests.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.common.transport.InetSocketTransportAddress;
 import org.elasticsearch.common.util.MockBigArrays;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.http.NullDispatcher;
 import org.elasticsearch.http.netty3.Netty3HttpServerTransport.HttpChannelPipelineFactory;
 import org.elasticsearch.http.netty3.pipelining.OrderedDownstreamChannelEvent;
 import org.elasticsearch.http.netty3.pipelining.OrderedUpstreamMessageEvent;
@@ -41,23 +42,24 @@
 import org.jboss.netty.handler.codec.http.DefaultHttpResponse;
 import org.jboss.netty.handler.codec.http.HttpRequest;
 import org.jboss.netty.handler.codec.http.HttpResponse;
-import org.jboss.netty.handler.codec.http.QueryStringDecoder;
 import org.junit.After;
 import org.junit.Before;
 
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 
 import static org.elasticsearch.http.netty3.Netty3HttpClient.returnHttpResponseBodies;
+import static org.elasticsearch.test.hamcrest.RegexMatcher.matches;
+import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.hasSize;
-import static org.hamcrest.Matchers.is;
 import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.CONTENT_LENGTH;
 import static org.jboss.netty.handler.codec.http.HttpResponseStatus.OK;
 import static org.jboss.netty.handler.codec.http.HttpVersion.HTTP_1_1;
@@ -80,6 +82,7 @@ public void setup() throws Exception {
 
     @After
     public void shutdown() throws Exception {
+        assertWarnings("http type [netty3] is deprecated");
         if (threadPool != null) {
             threadPool.shutdownNow();
         }
@@ -95,14 +98,22 @@ public void testThatHttpPipeliningWorksWhenEnabled() throws Exception {
                                .build();
         httpServerTransport = new CustomNetty3HttpServerTransport(settings);
         httpServerTransport.start();
-        InetSocketTransportAddress transportAddress = (InetSocketTransportAddress) randomFrom(httpServerTransport.boundAddress()
-            .boundAddresses());
-
-        List<String> requests = Arrays.asList("/firstfast", "/slow?sleep=500", "/secondfast", "/slow?sleep=1000", "/thirdfast");
+        InetSocketTransportAddress transportAddress =
+            (InetSocketTransportAddress) randomFrom(httpServerTransport.boundAddress().boundAddresses());
+
+        final int numberOfRequests = randomIntBetween(4, 16);
+        final List<String> requests = new ArrayList<>(numberOfRequests);
+        for (int i = 0; i < numberOfRequests; i++) {
+            if (rarely()) {
+                requests.add("/slow/" + i);
+            } else {
+                requests.add("/" + i);
+            }
+        }
         try (Netty3HttpClient nettyHttpClient = new Netty3HttpClient()) {
             Collection<HttpResponse> responses = nettyHttpClient.get(transportAddress.address(), requests.toArray(new String[]{}));
             Collection<String> responseBodies = returnHttpResponseBodies(responses);
-            assertThat(responseBodies, contains("/firstfast", "/slow?sleep=500", "/secondfast", "/slow?sleep=1000", "/thirdfast"));
+            assertThat(responseBodies, contains(requests.toArray()));
         }
     }
 
@@ -113,17 +124,38 @@ public void testThatHttpPipeliningCanBeDisabled() throws Exception {
                                 .build();
         httpServerTransport = new CustomNetty3HttpServerTransport(settings);
         httpServerTransport.start();
-        InetSocketTransportAddress transportAddress = (InetSocketTransportAddress) randomFrom(httpServerTransport.boundAddress()
-            .boundAddresses());
+        InetSocketTransportAddress transportAddress =
+            (InetSocketTransportAddress) randomFrom(httpServerTransport.boundAddress().boundAddresses());
+
+        final int numberOfRequests = randomIntBetween(4, 16);
+        final Set<Integer> slowIds = new HashSet<>();
+        final List<String> requests = new ArrayList<>(numberOfRequests);
+        for (int i = 0; i < numberOfRequests; i++) {
+            if (rarely()) {
+                requests.add("/slow/" + i);
+                slowIds.add(i);
+            } else {
+                requests.add("/" + i);
+            }
+        }
 
-        List<String> requests = Arrays.asList("/slow?sleep=1000", "/firstfast", "/secondfast", "/thirdfast", "/slow?sleep=500");
         try (Netty3HttpClient nettyHttpClient = new Netty3HttpClient()) {
             Collection<HttpResponse> responses = nettyHttpClient.get(transportAddress.address(), requests.toArray(new String[]{}));
             List<String> responseBodies = new ArrayList<>(returnHttpResponseBodies(responses));
-            // we cannot be sure about the order of the fast requests, but the slow ones should have to be last
-            assertThat(responseBodies, hasSize(5));
-            assertThat(responseBodies.get(3), is("/slow?sleep=500"));
-            assertThat(responseBodies.get(4), is("/slow?sleep=1000"));
+            // we cannot be sure about the order of the responses, but the slow ones should come last
+            assertThat(responseBodies, hasSize(numberOfRequests));
+            for (int i = 0; i < numberOfRequests - slowIds.size(); i++) {
+                assertThat(responseBodies.get(i), matches("/\\d+"));
+            }
+
+            final Set<Integer> ids = new HashSet<>();
+            for (int i = 0; i < slowIds.size(); i++) {
+                final String response = responseBodies.get(numberOfRequests - slowIds.size() + i);
+                assertThat(response, matches("/slow/\\d+"));
+                assertTrue(ids.add(Integer.parseInt(response.split("/")[2])));
+            }
+
+            assertThat(ids, equalTo(slowIds));
         }
     }
 
@@ -131,10 +163,10 @@ class CustomNetty3HttpServerTransport extends Netty3HttpServerTransport {
 
         private final ExecutorService executorService;
 
-        public CustomNetty3HttpServerTransport(Settings settings) {
+        CustomNetty3HttpServerTransport(Settings settings) {
             super(settings, Netty3HttpServerPipeliningTests.this.networkService,
-                Netty3HttpServerPipeliningTests.this.bigArrays, Netty3HttpServerPipeliningTests.this.threadPool
-            );
+                Netty3HttpServerPipeliningTests.this.bigArrays, Netty3HttpServerPipeliningTests.this.threadPool, xContentRegistry(),
+                    new NullDispatcher());
             this.executorService = Executors.newFixedThreadPool(5);
         }
 
@@ -155,7 +187,7 @@ private class CustomHttpChannelPipelineFactory extends HttpChannelPipelineFactor
 
         private final ExecutorService executorService;
 
-        public CustomHttpChannelPipelineFactory(Netty3HttpServerTransport transport, ExecutorService executorService,
+        CustomHttpChannelPipelineFactory(Netty3HttpServerTransport transport, ExecutorService executorService,
                                                 ThreadContext threadContext) {
             super(transport, randomBoolean(), threadContext);
             this.executorService = executorService;
@@ -173,7 +205,7 @@ class PossiblySlowUpstreamHandler extends SimpleChannelUpstreamHandler {
 
         private final ExecutorService executorService;
 
-        public PossiblySlowUpstreamHandler(ExecutorService executorService) {
+        PossiblySlowUpstreamHandler(ExecutorService executorService) {
             this.executorService = executorService;
         }
 
@@ -194,7 +226,7 @@ class PossiblySlowRunnable implements Runnable {
         private ChannelHandlerContext ctx;
         private MessageEvent e;
 
-        public PossiblySlowRunnable(ChannelHandlerContext ctx, MessageEvent e) {
+        PossiblySlowRunnable(ChannelHandlerContext ctx, MessageEvent e) {
             this.ctx = ctx;
             this.e = e;
         }
@@ -216,17 +248,15 @@ public void run() {
             httpResponse.headers().add(CONTENT_LENGTH, buffer.readableBytes());
             httpResponse.setContent(buffer);
 
-            QueryStringDecoder decoder = new QueryStringDecoder(request.getUri());
-
-            final int timeout = request.getUri().startsWith("/slow") && decoder.getParameters().containsKey("sleep")
-                ? Integer.valueOf(decoder.getParameters().get("sleep").get(0)) : 0;
-            if (timeout > 0) {
+            final boolean slow = request.getUri().matches("/slow/\\d+");
+            if (slow) {
                 try {
-                    Thread.sleep(timeout);
+                    Thread.sleep(scaledRandomIntBetween(500, 1000));
                 } catch (InterruptedException e1) {
-                    Thread.currentThread().interrupt();
                     throw new RuntimeException(e1);
                 }
+            } else {
+                assert request.getUri().matches("/\\d+");
             }
 
             if (oue != null) {
diff --git a/modules/transport-netty3/src/test/java/org/elasticsearch/http/netty3/Netty3HttpServerTransportTests.java b/modules/transport-netty3/src/test/java/org/elasticsearch/http/netty3/Netty3HttpServerTransportTests.java
index d291f76ff38b4..09dfb1a186b7d 100644
--- a/modules/transport-netty3/src/test/java/org/elasticsearch/http/netty3/Netty3HttpServerTransportTests.java
+++ b/modules/transport-netty3/src/test/java/org/elasticsearch/http/netty3/Netty3HttpServerTransportTests.java
@@ -22,10 +22,17 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.util.MockBigArrays;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.http.BindHttpException;
+import org.elasticsearch.http.HttpServerTransport;
+import org.elasticsearch.http.NullDispatcher;
 import org.elasticsearch.http.netty3.cors.Netty3CorsConfig;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
+import org.elasticsearch.rest.RestChannel;
+import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -50,6 +57,7 @@
  * Tests for the {@link Netty3HttpServerTransport} class.
  */
 public class Netty3HttpServerTransportTests extends ESTestCase {
+
     private NetworkService networkService;
     private ThreadPool threadPool;
     private MockBigArrays bigArrays;
@@ -63,6 +71,7 @@ public void setup() throws Exception {
 
     @After
     public void shutdown() throws Exception {
+        assertWarnings("http type [netty3] is deprecated");
         if (threadPool != null) {
             threadPool.shutdownNow();
         }
@@ -81,7 +90,8 @@ public void testCorsConfig() {
                                       .put(SETTING_CORS_ALLOW_HEADERS.getKey(), Strings.collectionToCommaDelimitedString(headers))
                                       .put(SETTING_CORS_ALLOW_CREDENTIALS.getKey(), true)
                                       .build();
-        final Netty3HttpServerTransport transport = new Netty3HttpServerTransport(settings, networkService, bigArrays, threadPool);
+        final Netty3HttpServerTransport transport = new Netty3HttpServerTransport(settings, networkService, bigArrays, threadPool,
+                xContentRegistry(), new NullDispatcher());
         final Netty3CorsConfig corsConfig = transport.getCorsConfig();
         assertThat(corsConfig.isAnyOriginSupported(), equalTo(true));
         assertThat(corsConfig.allowedRequestHeaders(), equalTo(headers));
@@ -97,10 +107,55 @@ public void testCorsConfigDefaults() {
                                       .put(SETTING_CORS_ALLOW_ORIGIN.getKey(), "*")
                                       .put(SETTING_CORS_ALLOW_CREDENTIALS.getKey(), true)
                                       .build();
-        final Netty3HttpServerTransport transport = new Netty3HttpServerTransport(settings, networkService, bigArrays, threadPool);
+        final Netty3HttpServerTransport transport = new Netty3HttpServerTransport(settings, networkService, bigArrays, threadPool,
+                xContentRegistry(), new NullDispatcher());
         final Netty3CorsConfig corsConfig = transport.getCorsConfig();
         assertThat(corsConfig.allowedRequestHeaders(), equalTo(headers));
         assertThat(corsConfig.allowedRequestMethods().stream().map(HttpMethod::getName).collect(Collectors.toSet()), equalTo(methods));
         transport.close();
     }
+
+    public void testBindUnavailableAddress() {
+        try (Netty3HttpServerTransport transport = new Netty3HttpServerTransport(Settings.EMPTY, networkService, bigArrays, threadPool,
+                xContentRegistry(), new NullDispatcher())){
+            transport.start();
+            TransportAddress remoteAddress = randomFrom(transport.boundAddress().boundAddresses());
+            Settings settings = Settings.builder().put("http.port", remoteAddress.getPort()).build();
+            try (Netty3HttpServerTransport otherTransport = new Netty3HttpServerTransport(settings, networkService, bigArrays, threadPool,
+                    xContentRegistry(), new NullDispatcher())) {
+                BindHttpException bindHttpException = expectThrows(BindHttpException.class, () -> otherTransport.start());
+                assertEquals("Failed to bind to [" + remoteAddress.getPort() + "]", bindHttpException.getMessage());
+            }
+        }
+    }
+
+    public void testDispatchDoesNotModifyThreadContext() throws InterruptedException {
+        final HttpServerTransport.Dispatcher dispatcher = new HttpServerTransport.Dispatcher() {
+
+            @Override
+            public void dispatchRequest(final RestRequest request, final RestChannel channel, final ThreadContext threadContext) {
+                threadContext.putHeader("foo", "bar");
+                threadContext.putTransient("bar", "baz");
+            }
+
+            @Override
+            public void dispatchBadRequest(final RestRequest request,
+                                           final RestChannel channel,
+                                           final ThreadContext threadContext,
+                                           final Throwable cause) {
+                threadContext.putHeader("foo_bad", "bar");
+                threadContext.putTransient("bar_bad", "baz");
+            }
+
+        };
+
+        try (Netty3HttpServerTransport transport =
+                 new Netty3HttpServerTransport(Settings.EMPTY, networkService, bigArrays, threadPool, xContentRegistry(), dispatcher)) {
+            transport.start();
+
+            transport.dispatchRequest(null, null);
+            assertNull(threadPool.getThreadContext().getHeader("foo"));
+            assertNull(threadPool.getThreadContext().getTransient("bar"));
+        }
+    }
 }
diff --git a/modules/transport-netty3/src/test/java/org/elasticsearch/http/netty3/pipelining/HttpPipeliningHandlerTests.java b/modules/transport-netty3/src/test/java/org/elasticsearch/http/netty3/pipelining/HttpPipeliningHandlerTests.java
index 7db2368d34464..9900d9cd96c26 100644
--- a/modules/transport-netty3/src/test/java/org/elasticsearch/http/netty3/pipelining/HttpPipeliningHandlerTests.java
+++ b/modules/transport-netty3/src/test/java/org/elasticsearch/http/netty3/pipelining/HttpPipeliningHandlerTests.java
@@ -205,7 +205,7 @@ private class ChunkWriter implements TimerTask {
             private final OrderedUpstreamMessageEvent oue;
             private final int subSequence;
 
-            public ChunkWriter(final ChannelHandlerContext ctx, final MessageEvent e, final String uri,
+            ChunkWriter(final ChannelHandlerContext ctx, final MessageEvent e, final String uri,
                                final OrderedUpstreamMessageEvent oue, final int subSequence) {
                 this.ctx = ctx;
                 this.e = e;
diff --git a/modules/transport-netty3/src/test/java/org/elasticsearch/rest/Netty3HeadBodyIsEmptyIT.java b/modules/transport-netty3/src/test/java/org/elasticsearch/rest/Netty3HeadBodyIsEmptyIT.java
new file mode 100644
index 0000000000000..82e5ff9b666b7
--- /dev/null
+++ b/modules/transport-netty3/src/test/java/org/elasticsearch/rest/Netty3HeadBodyIsEmptyIT.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.rest;
+
+public class Netty3HeadBodyIsEmptyIT extends HeadBodyIsEmptyIntegTestCase {
+}
diff --git a/modules/transport-netty3/src/test/java/org/elasticsearch/transport/Netty3SizeHeaderFrameDecoderTests.java b/modules/transport-netty3/src/test/java/org/elasticsearch/transport/Netty3SizeHeaderFrameDecoderTests.java
index a7d9805cf3cd9..1d7daef880f49 100644
--- a/modules/transport-netty3/src/test/java/org/elasticsearch/transport/Netty3SizeHeaderFrameDecoderTests.java
+++ b/modules/transport-netty3/src/test/java/org/elasticsearch/transport/Netty3SizeHeaderFrameDecoderTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.transport;
 
+import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
@@ -59,15 +60,19 @@ public class Netty3SizeHeaderFrameDecoderTests extends ESTestCase {
     private int port;
     private InetAddress host;
 
+    @Override
     @Before
-    public void startThreadPool() {
+    public void setUp() throws Exception {
+        super.setUp();
         threadPool = new ThreadPool(settings);
         NetworkService networkService = new NetworkService(settings, Collections.emptyList());
         BigArrays bigArrays = new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService());
         nettyTransport = new Netty3Transport(settings, threadPool, networkService, bigArrays,
             new NamedWriteableRegistry(Collections.emptyList()), new NoneCircuitBreakerService());
         nettyTransport.start();
-        TransportService transportService = new TransportService(settings, nettyTransport, threadPool);
+        TransportService transportService = new TransportService(settings, nettyTransport, threadPool,
+            TransportService.NOOP_TRANSPORT_INTERCEPTOR,
+            (boundAddress) -> DiscoveryNode.createLocal(settings, boundAddress.publishAddress(), "someID"), null);
         nettyTransport.transportServiceAdapter(transportService.createAdapter());
 
         TransportAddress[] boundAddresses = nettyTransport.boundAddress().boundAddresses();
@@ -77,11 +82,14 @@ public void startThreadPool() {
 
     }
 
+    @Override
     @After
-    public void terminateThreadPool() throws InterruptedException {
+    public void tearDown() throws Exception {
+        assertWarnings("transport type [netty3] is deprecated");
         nettyTransport.stop();
         terminate(threadPool);
         threadPool = null;
+        super.tearDown();
     }
 
     public void testThatTextMessageIsReturnedOnHTTPLikeRequest() throws Exception {
diff --git a/modules/transport-netty3/src/test/java/org/elasticsearch/transport/netty3/ChannelBufferBytesReferenceTests.java b/modules/transport-netty3/src/test/java/org/elasticsearch/transport/netty3/ChannelBufferBytesReferenceTests.java
index ac40e16afe98b..9fc9c30a13c16 100644
--- a/modules/transport-netty3/src/test/java/org/elasticsearch/transport/netty3/ChannelBufferBytesReferenceTests.java
+++ b/modules/transport-netty3/src/test/java/org/elasticsearch/transport/netty3/ChannelBufferBytesReferenceTests.java
@@ -28,8 +28,14 @@
 import java.io.IOException;
 
 public class ChannelBufferBytesReferenceTests extends AbstractBytesReferenceTestCase {
+
     @Override
     protected BytesReference newBytesReference(int length) throws IOException {
+        return newBytesReferenceWithOffsetOfZero(length);
+    }
+
+    @Override
+    protected BytesReference newBytesReferenceWithOffsetOfZero(int length) throws IOException {
         ReleasableBytesStreamOutput out = new ReleasableBytesStreamOutput(length, bigarrays);
         for (int i = 0; i < length; i++) {
             out.writeByte((byte) random().nextInt(1 << 8));
diff --git a/modules/transport-netty3/src/test/java/org/elasticsearch/transport/netty3/Netty3ScheduledPingTests.java b/modules/transport-netty3/src/test/java/org/elasticsearch/transport/netty3/Netty3ScheduledPingTests.java
index bb169aa8d70d0..62bd42f457db6 100644
--- a/modules/transport-netty3/src/test/java/org/elasticsearch/transport/netty3/Netty3ScheduledPingTests.java
+++ b/modules/transport-netty3/src/test/java/org/elasticsearch/transport/netty3/Netty3ScheduledPingTests.java
@@ -18,12 +18,12 @@
  */
 package org.elasticsearch.transport.netty3;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
@@ -31,7 +31,7 @@
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.TransportResponseHandler;
+import org.elasticsearch.transport.ConnectionProfile;
 import org.elasticsearch.transport.TcpTransport;
 import org.elasticsearch.transport.TransportChannel;
 import org.elasticsearch.transport.TransportException;
@@ -39,18 +39,27 @@
 import org.elasticsearch.transport.TransportRequestHandler;
 import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportResponse;
+import org.elasticsearch.transport.TransportResponseHandler;
 import org.elasticsearch.transport.TransportResponseOptions;
+import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.transport.TransportSettings;
+import org.junit.After;
 
 import java.io.IOException;
 import java.util.Collections;
 
-import static java.util.Collections.emptyMap;
-import static java.util.Collections.emptySet;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 
 public class Netty3ScheduledPingTests extends ESTestCase {
+
+    @Override
+    @After
+    public void tearDown() throws Exception {
+        assertWarnings("transport type [netty3] is deprecated");
+        super.tearDown();
+    }
+
     public void testScheduledPing() throws Exception {
         ThreadPool threadPool = new TestThreadPool(getClass().getName());
 
@@ -65,32 +74,44 @@ public void testScheduledPing() throws Exception {
         NamedWriteableRegistry registry = new NamedWriteableRegistry(Collections.emptyList());
         final Netty3Transport nettyA = new Netty3Transport(settings, threadPool, new NetworkService(settings, Collections.emptyList()),
             BigArrays.NON_RECYCLING_INSTANCE, registry, circuitBreakerService);
-        MockTransportService serviceA = new MockTransportService(settings, nettyA, threadPool);
+        MockTransportService serviceA = new MockTransportService(settings, nettyA, threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR,
+                null);
         serviceA.start();
         serviceA.acceptIncomingRequests();
 
         final Netty3Transport nettyB = new Netty3Transport(settings, threadPool, new NetworkService(settings, Collections.emptyList()),
             BigArrays.NON_RECYCLING_INSTANCE, registry, circuitBreakerService);
-        MockTransportService serviceB = new MockTransportService(settings, nettyB, threadPool);
+        MockTransportService serviceB = new MockTransportService(settings, nettyB, threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR,
+                null);
 
         serviceB.start();
         serviceB.acceptIncomingRequests();
 
-        DiscoveryNode nodeA =
-            new DiscoveryNode("TS_A", "TS_A", serviceA.boundAddress().publishAddress(), emptyMap(), emptySet(), Version.CURRENT);
-        DiscoveryNode nodeB =
-            new DiscoveryNode("TS_B", "TS_B", serviceB.boundAddress().publishAddress(), emptyMap(), emptySet(), Version.CURRENT);
-
-        serviceA.connectToNode(nodeB);
-        serviceB.connectToNode(nodeA);
+        DiscoveryNode nodeA = serviceA.getLocalDiscoNode();
+        DiscoveryNode nodeB = serviceB.getLocalDiscoNode();
+
+        if (randomBoolean()) {
+            // use connection profile with different connect timeout
+            final ConnectionProfile connectionProfile = new ConnectionProfile.Builder()
+                .addConnections(1,
+                    TransportRequestOptions.Type.BULK,
+                    TransportRequestOptions.Type.PING,
+                    TransportRequestOptions.Type.RECOVERY,
+                    TransportRequestOptions.Type.REG,
+                    TransportRequestOptions.Type.STATE)
+                .setConnectTimeout(TimeValue.timeValueSeconds(42))
+                .build();
+            serviceA.connectToNode(nodeB, connectionProfile);
+            serviceB.connectToNode(nodeA, connectionProfile);
+        } else {
+            serviceA.connectToNode(nodeB);
+            serviceB.connectToNode(nodeA);
+        }
 
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
+        assertBusy(() -> {
                 assertThat(nettyA.getPing().getSuccessfulPings(), greaterThan(100L));
                 assertThat(nettyB.getPing().getSuccessfulPings(), greaterThan(100L));
-            }
-        });
+            });
         assertThat(nettyA.getPing().getFailedPings(), equalTo(0L));
         assertThat(nettyB.getPing().getFailedPings(), equalTo(0L));
 
diff --git a/modules/transport-netty3/src/test/java/org/elasticsearch/transport/netty3/Netty3TransportIT.java b/modules/transport-netty3/src/test/java/org/elasticsearch/transport/netty3/Netty3TransportIT.java
index f2863c5b5d2ec..ddfecdd8a4302 100644
--- a/modules/transport-netty3/src/test/java/org/elasticsearch/transport/netty3/Netty3TransportIT.java
+++ b/modules/transport-netty3/src/test/java/org/elasticsearch/transport/netty3/Netty3TransportIT.java
@@ -24,7 +24,6 @@
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.network.NetworkModule;
@@ -32,10 +31,12 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
+import org.elasticsearch.plugins.NetworkPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportSettings;
 import org.jboss.netty.channel.Channel;
 
@@ -45,6 +46,8 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
+import java.util.function.Supplier;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.is;
@@ -83,13 +86,19 @@ public void testThatConnectionFailsAsIntended() throws Exception {
 
     public static final class ExceptionThrowingNetty3Transport extends Netty3Transport {
 
-        public static class TestPlugin extends Plugin {
-            public void onModule(NetworkModule module) {
-                module.registerTransport("exception-throwing", ExceptionThrowingNetty3Transport.class);
+        public static class TestPlugin extends Plugin implements NetworkPlugin {
+
+            @Override
+            public Map<String, Supplier<Transport>> getTransports(Settings settings, ThreadPool threadPool, BigArrays bigArrays,
+                                                                  CircuitBreakerService circuitBreakerService,
+                                                                  NamedWriteableRegistry namedWriteableRegistry,
+                                                                  NetworkService networkService) {
+                return Collections.singletonMap("exception-throwing", () ->
+                    new ExceptionThrowingNetty3Transport(settings, threadPool, networkService, bigArrays,
+                        namedWriteableRegistry, circuitBreakerService));
             }
         }
 
-        @Inject
         public ExceptionThrowingNetty3Transport(
                 Settings settings,
                 ThreadPool threadPool,
@@ -102,9 +111,9 @@ public ExceptionThrowingNetty3Transport(
 
         protected String handleRequest(Channel channel, String profileName,
                                        StreamInput stream, long requestId, int messageLengthBytes, Version version,
-                                       InetSocketAddress remoteAddress) throws IOException {
+                                       InetSocketAddress remoteAddress, byte status) throws IOException {
             String action = super.handleRequest(channel, profileName, stream, requestId, messageLengthBytes, version,
-                remoteAddress);
+                remoteAddress, status);
             channelProfileName = TransportSettings.DEFAULT_PROFILE;
             return action;
         }
diff --git a/modules/transport-netty3/src/test/java/org/elasticsearch/transport/netty3/Netty3TransportMultiPortIntegrationIT.java b/modules/transport-netty3/src/test/java/org/elasticsearch/transport/netty3/Netty3TransportMultiPortIntegrationIT.java
index d25951e254c72..61843c67d728b 100644
--- a/modules/transport-netty3/src/test/java/org/elasticsearch/transport/netty3/Netty3TransportMultiPortIntegrationIT.java
+++ b/modules/transport-netty3/src/test/java/org/elasticsearch/transport/netty3/Netty3TransportMultiPortIntegrationIT.java
@@ -58,7 +58,7 @@ public class Netty3TransportMultiPortIntegrationIT extends ESNetty3IntegTestCase
     protected Settings nodeSettings(int nodeOrdinal) {
         if (randomPort == -1) {
             randomPort = randomIntBetween(49152, 65525);
-            randomPortRange = String.format(Locale.ROOT, "%s-%s", randomPort, randomPort+10);
+            randomPortRange = String.format(Locale.ROOT, "%s-%s", randomPort, randomPort + 10);
         }
         Settings.Builder builder = Settings.builder()
                 .put(super.nodeSettings(nodeOrdinal))
@@ -76,8 +76,12 @@ public void testThatTransportClientCanConnect() throws Exception {
                 .put(NetworkModule.TRANSPORT_TYPE_KEY, Netty3Plugin.NETTY_TRANSPORT_NAME)
                 .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
                 .build();
+        // we have to test all the ports that the data node might be bound to
         try (TransportClient transportClient = new MockTransportClient(settings, Netty3Plugin.class)) {
             transportClient.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName("127.0.0.1"), randomPort));
+            for (int i = 0; i <= 10; i++) {
+                transportClient.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName("127.0.0.1"), randomPort + i));
+            }
             ClusterHealthResponse response = transportClient.admin().cluster().prepareHealth().get();
             assertThat(response.getStatus(), is(ClusterHealthStatus.GREEN));
         }
diff --git a/modules/transport-netty3/src/test/java/org/elasticsearch/transport/netty3/Netty3TransportMultiPortTests.java b/modules/transport-netty3/src/test/java/org/elasticsearch/transport/netty3/Netty3TransportMultiPortTests.java
index ae2b24511e2ca..7b0ed25ba90c1 100644
--- a/modules/transport-netty3/src/test/java/org/elasticsearch/transport/netty3/Netty3TransportMultiPortTests.java
+++ b/modules/transport-netty3/src/test/java/org/elasticsearch/transport/netty3/Netty3TransportMultiPortTests.java
@@ -32,6 +32,7 @@
 import org.elasticsearch.transport.TcpTransport;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.transport.TransportSettings;
+import org.junit.After;
 import org.junit.Before;
 
 import java.util.Collections;
@@ -51,6 +52,13 @@ public void setup() {
         }
     }
 
+    @Override
+    @After
+    public void tearDown() throws Exception {
+        assertWarnings("transport type [netty3] is deprecated");
+        super.tearDown();
+    }
+
     public void testThatNettyCanBindToMultiplePorts() throws Exception {
         Settings settings = Settings.builder()
                 .put("network.host", host)
diff --git a/modules/transport-netty3/src/test/java/org/elasticsearch/transport/netty3/Netty3UtilsTests.java b/modules/transport-netty3/src/test/java/org/elasticsearch/transport/netty3/Netty3UtilsTests.java
index 8188569e3f45f..16cd0689f8c6d 100644
--- a/modules/transport-netty3/src/test/java/org/elasticsearch/transport/netty3/Netty3UtilsTests.java
+++ b/modules/transport-netty3/src/test/java/org/elasticsearch/transport/netty3/Netty3UtilsTests.java
@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.transport.netty3;
 
+import org.apache.commons.codec.DecoderException;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.bytes.AbstractBytesReferenceTestCase;
 import org.elasticsearch.common.bytes.BytesArray;
@@ -31,6 +32,9 @@
 import org.jboss.netty.buffer.CompositeChannelBuffer;
 
 import java.io.IOException;
+import java.util.Optional;
+
+import static org.hamcrest.CoreMatchers.equalTo;
 
 public class Netty3UtilsTests extends ESTestCase {
 
@@ -74,6 +78,60 @@ public void testToChannelBuffer() throws IOException {
         assertArrayEquals(BytesReference.toBytes(ref), BytesReference.toBytes(bytesReference));
     }
 
+    public void testMaybeError() {
+        final Error outOfMemoryError = new OutOfMemoryError();
+        assertError(outOfMemoryError, outOfMemoryError);
+
+        final DecoderException decoderException = new DecoderException(outOfMemoryError);
+        assertError(decoderException, outOfMemoryError);
+
+        final Exception e = new Exception();
+        e.addSuppressed(decoderException);
+        assertError(e, outOfMemoryError);
+
+        final int depth = randomIntBetween(1, 16);
+        Throwable cause = new Exception();
+        boolean fatal = false;
+        Error error = null;
+        for (int i = 0; i < depth; i++) {
+            final int length = randomIntBetween(1, 4);
+            for (int j = 0; j < length; j++) {
+                if (!fatal && rarely()) {
+                    error = new Error();
+                    cause.addSuppressed(error);
+                    fatal = true;
+                } else {
+                    cause.addSuppressed(new Exception());
+                }
+            }
+            if (!fatal && rarely()) {
+                cause = error = new Error(cause);
+                fatal = true;
+            } else {
+                cause = new Exception(cause);
+            }
+        }
+        if (fatal) {
+            assertError(cause, error);
+        } else {
+            assertFalse(Netty3Utils.maybeError(cause).isPresent());
+        }
+
+        assertFalse(Netty3Utils.maybeError(new Exception(new DecoderException())).isPresent());
+
+        Throwable chain = outOfMemoryError;
+        for (int i = 0; i < Netty3Utils.MAX_ITERATIONS; i++) {
+            chain = new Exception(chain);
+        }
+        assertFalse(Netty3Utils.maybeError(chain).isPresent());
+    }
+
+    private void assertError(final Throwable cause, final Error error) {
+        final Optional<Error> maybeError = Netty3Utils.maybeError(cause);
+        assertTrue(maybeError.isPresent());
+        assertThat(maybeError.get(), equalTo(error));
+    }
+
     private BytesReference getRandomizedBytesReference(int length) throws IOException {
         // we know bytes stream output always creates a paged bytes reference, we use it to create randomized content
         ReleasableBytesStreamOutput out = new ReleasableBytesStreamOutput(length, bigarrays);
diff --git a/modules/transport-netty3/src/test/java/org/elasticsearch/transport/netty3/SimpleNetty3TransportTests.java b/modules/transport-netty3/src/test/java/org/elasticsearch/transport/netty3/SimpleNetty3TransportTests.java
index 5b862908ea867..58a4bf5706f50 100644
--- a/modules/transport-netty3/src/test/java/org/elasticsearch/transport/netty3/SimpleNetty3TransportTests.java
+++ b/modules/transport-netty3/src/test/java/org/elasticsearch/transport/netty3/SimpleNetty3TransportTests.java
@@ -23,17 +23,26 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.network.NetworkService;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
+import org.elasticsearch.node.Node;
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.AbstractSimpleTransportTestCase;
+import org.elasticsearch.transport.BindTransportException;
 import org.elasticsearch.transport.ConnectTransportException;
+import org.elasticsearch.transport.TcpTransport;
 import org.elasticsearch.transport.Transport;
+import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.transport.TransportSettings;
+import org.jboss.netty.channel.Channel;
+import org.junit.After;
 
+import java.io.IOException;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
 import java.util.Collections;
@@ -44,28 +53,55 @@
 
 public class SimpleNetty3TransportTests extends AbstractSimpleTransportTestCase {
 
-    public static MockTransportService nettyFromThreadPool(
-        Settings settings,
-        ThreadPool threadPool, final Version version) {
+    @Override
+    @After
+    public void tearDown() throws Exception {
+        assertWarnings("transport type [netty3] is deprecated");
+        super.tearDown();
+    }
+
+    public static MockTransportService nettyFromThreadPool(Settings settings, ThreadPool threadPool, final Version version,
+                                                           ClusterSettings clusterSettings, boolean doHandshake) {
         NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(Collections.emptyList());
         Transport transport = new Netty3Transport(settings, threadPool, new NetworkService(settings, Collections.emptyList()),
             BigArrays.NON_RECYCLING_INSTANCE, namedWriteableRegistry, new NoneCircuitBreakerService()) {
+
+            @Override
+            protected Version executeHandshake(DiscoveryNode node, Channel channel, TimeValue timeout) throws IOException
+                , InterruptedException {
+                if (doHandshake) {
+                    return super.executeHandshake(node, channel, timeout);
+                } else {
+                    return version.minimumCompatibilityVersion();
+                }
+            }
+
             @Override
             protected Version getCurrentVersion() {
                 return version;
             }
         };
-        return new MockTransportService(Settings.EMPTY, transport, threadPool);
+        MockTransportService mockTransportService =
+            MockTransportService.createNewService(Settings.EMPTY, transport, version, threadPool, clusterSettings);
+        mockTransportService.start();
+        return mockTransportService;
     }
 
     @Override
-    protected MockTransportService build(Settings settings, Version version) {
+    protected MockTransportService build(Settings settings, Version version, ClusterSettings clusterSettings, boolean doHandshake) {
         settings = Settings.builder().put(settings).put(TransportSettings.PORT.getKey(), "0").build();
-        MockTransportService transportService = nettyFromThreadPool(settings, threadPool, version);
+        MockTransportService transportService = nettyFromThreadPool(settings, threadPool, version, clusterSettings, doHandshake);
         transportService.start();
         return transportService;
     }
 
+    @Override
+    protected void closeConnectionChannel(Transport transport, Transport.Connection connection) throws IOException {
+        final Netty3Transport t = (Netty3Transport) transport;
+        @SuppressWarnings("unchecked") final TcpTransport<Channel>.NodeChannels channels = (TcpTransport<Channel>.NodeChannels) connection;
+        t.closeChannels(channels.getChannels().subList(0, randomIntBetween(1, channels.getChannels().size())), true);
+    }
+
     public void testConnectException() throws UnknownHostException {
         try {
             serviceA.connectToNode(new DiscoveryNode("C", new InetSocketTransportAddress(InetAddress.getByName("localhost"), 9876),
@@ -76,4 +112,26 @@ public void testConnectException() throws UnknownHostException {
             assertThat(e.getMessage(), containsString("[127.0.0.1:9876]"));
         }
     }
+
+    public void testBindUnavailableAddress() {
+        // this is on a lower level since it needs access to the TransportService before it's started
+        int port = serviceA.boundAddress().publishAddress().getPort();
+        Settings settings = Settings.builder()
+            .put(Node.NODE_NAME_SETTING.getKey(), "foobar")
+            .put(TransportService.TRACE_LOG_INCLUDE_SETTING.getKey(), "")
+            .put(TransportService.TRACE_LOG_EXCLUDE_SETTING.getKey(), "NOTHING")
+            .put("transport.tcp.port", port)
+            .build();
+        ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        BindTransportException bindTransportException = expectThrows(BindTransportException.class, () -> {
+            MockTransportService transportService = nettyFromThreadPool(settings, threadPool, Version.CURRENT, clusterSettings, true);
+            try {
+                transportService.start();
+            } finally {
+                transportService.stop();
+                transportService.close();
+            }
+        });
+        assertEquals("Failed to bind to ["+ port + "]", bindTransportException.getMessage());
+    }
 }
diff --git a/modules/transport-netty4/build.gradle b/modules/transport-netty4/build.gradle
index 4fc38bc394763..109a87213e042 100644
--- a/modules/transport-netty4/build.gradle
+++ b/modules/transport-netty4/build.gradle
@@ -29,24 +29,42 @@ esplugin {
     hasClientJar = true
 }
 
+compileJava.options.compilerArgs << "-Xlint:-try"
 compileTestJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-unchecked"
 
 dependencies {
     // network stack
-    compile "io.netty:netty-buffer:4.1.5.Final"
-    compile "io.netty:netty-codec:4.1.5.Final"
-    compile "io.netty:netty-codec-http:4.1.5.Final"
-    compile "io.netty:netty-common:4.1.5.Final"
-    compile "io.netty:netty-handler:4.1.5.Final"
-    compile "io.netty:netty-resolver:4.1.5.Final"
-    compile "io.netty:netty-transport:4.1.5.Final"
+    compile "io.netty:netty-buffer:4.1.13.Final"
+    compile "io.netty:netty-codec:4.1.13.Final"
+    compile "io.netty:netty-codec-http:4.1.13.Final"
+    compile "io.netty:netty-common:4.1.13.Final"
+    compile "io.netty:netty-handler:4.1.13.Final"
+    compile "io.netty:netty-resolver:4.1.13.Final"
+    compile "io.netty:netty-transport:4.1.13.Final"
 }
 
-integTest {
-    includePackaged = true
-    cluster {
-        numNodes = 2
-    }
+integTestCluster {
+    numNodes = 2 // this tests needs at least 2 nodes
+}
+
+dependencyLicenses {
+  mapping from: /netty-.*/, to: 'netty'
+}
+
+test {
+    /*
+     * We have to disable setting the number of available processors as tests in the same JVM randomize processors and will step on each
+     * other if we allow them to set the number of available processors as it's set-once in Netty.
+     */
+    systemProperty 'es.set.netty.runtime.available.processors', 'false'
+}
+
+integTestRunner {
+    /*
+     * We have to disable setting the number of available processors as tests in the same JVM randomize processors and will step on each
+     * other if we allow them to set the number of available processors as it's set-once in Netty.
+     */
+    systemProperty 'es.set.netty.runtime.available.processors', 'false'
 }
 
 thirdPartyAudit.excludes = [
@@ -63,11 +81,10 @@ thirdPartyAudit.excludes = [
         'org.apache.commons.logging.LogFactory',
 
         // from io.netty.handler.ssl.OpenSslEngine (netty)
-        'org.apache.tomcat.jni.Buffer',
-        'org.apache.tomcat.jni.Library',
-        'org.apache.tomcat.jni.Pool',
-        'org.apache.tomcat.jni.SSL',
-        'org.apache.tomcat.jni.SSLContext',
+        'io.netty.internal.tcnative.Buffer',
+        'io.netty.internal.tcnative.Library',
+        'io.netty.internal.tcnative.SSL',
+        'io.netty.internal.tcnative.SSLContext',
 
         // from io.netty.handler.ssl.util.BouncyCastleSelfSignedCertGenerator (netty)
         'org.bouncycastle.asn1.x500.X500Name',
@@ -114,48 +131,29 @@ thirdPartyAudit.excludes = [
         'com.ning.compress.lzf.LZFEncoder',
         'com.ning.compress.lzf.util.ChunkDecoderFactory',
         'com.ning.compress.lzf.util.ChunkEncoderFactory',
-        'javassist.ClassClassPath',
-        'javassist.ClassPath',
-        'javassist.ClassPool',
-        'javassist.CtClass',
-        'javassist.CtMethod',
         'lzma.sdk.lzma.Encoder',
         'net.jpountz.lz4.LZ4Compressor',
         'net.jpountz.lz4.LZ4Factory',
         'net.jpountz.lz4.LZ4FastDecompressor',
         'net.jpountz.xxhash.StreamingXXHash32',
         'net.jpountz.xxhash.XXHashFactory',
-        'org.apache.tomcat.Apr',
-        'org.apache.tomcat.jni.CertificateRequestedCallback',
-        'org.apache.tomcat.jni.CertificateRequestedCallback$KeyMaterial',
-        'org.apache.tomcat.jni.CertificateVerifier',
-        'org.apache.tomcat.jni.SessionTicketKey',
+        'io.netty.internal.tcnative.CertificateRequestedCallback',
+        'io.netty.internal.tcnative.CertificateRequestedCallback$KeyMaterial',
+        'io.netty.internal.tcnative.CertificateVerifier',
+        'io.netty.internal.tcnative.SessionTicketKey',
+        'io.netty.internal.tcnative.SniHostNameMatcher',
         'org.eclipse.jetty.alpn.ALPN$ClientProvider',
         'org.eclipse.jetty.alpn.ALPN$ServerProvider',
         'org.eclipse.jetty.alpn.ALPN',
 
         'io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator',
         'io.netty.util.internal.PlatformDependent0',
+        'io.netty.util.internal.PlatformDependent0$1',
         'io.netty.util.internal.PlatformDependent0$2',
         'io.netty.util.internal.PlatformDependent0$3',
-        'io.netty.util.internal.UnsafeAtomicIntegerFieldUpdater',
-        'io.netty.util.internal.UnsafeAtomicLongFieldUpdater',
-        'io.netty.util.internal.UnsafeAtomicReferenceFieldUpdater',
-        'io.netty.util.internal.chmv8.ConcurrentHashMapV8',
-        'io.netty.util.internal.chmv8.ConcurrentHashMapV8$1',
-        'io.netty.util.internal.chmv8.ConcurrentHashMapV8$TreeBin',
-        'io.netty.util.internal.chmv8.CountedCompleter',
-        'io.netty.util.internal.chmv8.CountedCompleter$1',
-        'io.netty.util.internal.chmv8.ForkJoinPool',
-        'io.netty.util.internal.chmv8.ForkJoinPool$2',
-        'io.netty.util.internal.chmv8.ForkJoinPool$WorkQueue',
-        'io.netty.util.internal.chmv8.ForkJoinTask',
-        'io.netty.util.internal.chmv8.ForkJoinTask$1',
-        'io.netty.util.internal.chmv8.Striped64',
-        'io.netty.util.internal.chmv8.Striped64$1',
-        'io.netty.util.internal.chmv8.Striped64$Cell',
         'io.netty.util.internal.shaded.org.jctools.queues.BaseLinkedQueueConsumerNodeRef',
         'io.netty.util.internal.shaded.org.jctools.queues.BaseLinkedQueueProducerNodeRef',
+        'io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueue',
         'io.netty.util.internal.shaded.org.jctools.queues.ConcurrentSequencedCircularArrayQueue',
         'io.netty.util.internal.shaded.org.jctools.queues.LinkedQueueNode',
         'io.netty.util.internal.shaded.org.jctools.queues.MpmcArrayQueueConsumerField',
@@ -163,8 +161,10 @@ thirdPartyAudit.excludes = [
         'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueConsumerField',
         'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueHeadLimitField',
         'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueTailField',
-        'io.netty.util.internal.shaded.org.jctools.queues.MpscChunkedArrayQueue',
         'io.netty.util.internal.shaded.org.jctools.util.JvmInfo',
         'io.netty.util.internal.shaded.org.jctools.util.UnsafeAccess',
         'io.netty.util.internal.shaded.org.jctools.util.UnsafeRefArrayAccess',
+
+        'org.conscrypt.Conscrypt$Engines',
+        'org.conscrypt.HandshakeListener'
 ]
diff --git a/modules/transport-netty4/licenses/netty-codec-http-LICENSE.txt b/modules/transport-netty4/licenses/netty-LICENSE.txt
similarity index 100%
rename from modules/transport-netty4/licenses/netty-codec-http-LICENSE.txt
rename to modules/transport-netty4/licenses/netty-LICENSE.txt
diff --git a/modules/transport-netty4/licenses/netty-buffer-NOTICE.txt b/modules/transport-netty4/licenses/netty-NOTICE.txt
similarity index 100%
rename from modules/transport-netty4/licenses/netty-buffer-NOTICE.txt
rename to modules/transport-netty4/licenses/netty-NOTICE.txt
diff --git a/modules/transport-netty4/licenses/netty-buffer-4.1.13.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-buffer-4.1.13.Final.jar.sha1
new file mode 100644
index 0000000000000..31f015e158af4
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-buffer-4.1.13.Final.jar.sha1
@@ -0,0 +1 @@
+0e3f583ea8a2618a7563b1ee2aa696c23edcc3d8
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-buffer-4.1.5.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-buffer-4.1.5.Final.jar.sha1
deleted file mode 100644
index e64426f033a9a..0000000000000
--- a/modules/transport-netty4/licenses/netty-buffer-4.1.5.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b5fb6bccda4d63d4a74c9faccdf32f77ab66abc1
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-codec-4.1.13.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-4.1.13.Final.jar.sha1
new file mode 100644
index 0000000000000..6e7f1bdc14ced
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-codec-4.1.13.Final.jar.sha1
@@ -0,0 +1 @@
+370eeb6e9d92495a2a3be096ab6102755af76730
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-codec-4.1.5.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-4.1.5.Final.jar.sha1
deleted file mode 100644
index 66484a93047f1..0000000000000
--- a/modules/transport-netty4/licenses/netty-codec-4.1.5.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-66bbf9324fa36467d041083f89328e2a24ec4f67
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-codec-NOTICE.txt b/modules/transport-netty4/licenses/netty-codec-NOTICE.txt
deleted file mode 100644
index 5bbf91a14de23..0000000000000
--- a/modules/transport-netty4/licenses/netty-codec-NOTICE.txt
+++ /dev/null
@@ -1,116 +0,0 @@
-
-                            The Netty Project
-                            =================
-
-Please visit the Netty web site for more information:
-
-  * http://netty.io/
-
-Copyright 2011 The Netty Project
-
-The Netty Project licenses this file to you under the Apache License,
-version 2.0 (the "License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at:
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-License for the specific language governing permissions and limitations
-under the License.
-
-Also, please refer to each LICENSE.<component>.txt file, which is located in
-the 'license' directory of the distribution file, for the license terms of the
-components that this product depends on.
-
--------------------------------------------------------------------------------
-This product contains the extensions to Java Collections Framework which has
-been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene:
-
-  * LICENSE:
-    * license/LICENSE.jsr166y.txt (Public Domain)
-  * HOMEPAGE:
-    * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/
-    * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/
-
-This product contains a modified version of Robert Harder's Public Domain
-Base64 Encoder and Decoder, which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.base64.txt (Public Domain)
-  * HOMEPAGE:
-    * http://iharder.sourceforge.net/current/java/base64/
-
-This product contains a modified version of 'JZlib', a re-implementation of
-zlib in pure Java, which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.jzlib.txt (BSD Style License)
-  * HOMEPAGE:
-    * http://www.jcraft.com/jzlib/
-
-This product contains a modified version of 'Webbit', a Java event based
-WebSocket and HTTP server:
-
-  * LICENSE:
-    * license/LICENSE.webbit.txt (BSD License)
-  * HOMEPAGE:
-    * https://github.com/joewalnes/webbit
-
-This product optionally depends on 'Protocol Buffers', Google's data
-interchange format, which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.protobuf.txt (New BSD License)
-  * HOMEPAGE:
-    * http://code.google.com/p/protobuf/
-
-This product optionally depends on 'Bouncy Castle Crypto APIs' to generate
-a temporary self-signed X.509 certificate when the JVM does not provide the
-equivalent functionality.  It can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.bouncycastle.txt (MIT License)
-  * HOMEPAGE:
-    * http://www.bouncycastle.org/
-
-This product optionally depends on 'SLF4J', a simple logging facade for Java,
-which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.slf4j.txt (MIT License)
-  * HOMEPAGE:
-    * http://www.slf4j.org/
-
-This product optionally depends on 'Apache Commons Logging', a logging
-framework, which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.commons-logging.txt (Apache License 2.0)
-  * HOMEPAGE:
-    * http://commons.apache.org/logging/
-
-This product optionally depends on 'Apache Log4J', a logging framework,
-which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.log4j.txt (Apache License 2.0)
-  * HOMEPAGE:
-    * http://logging.apache.org/log4j/
-
-This product optionally depends on 'JBoss Logging', a logging framework,
-which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.jboss-logging.txt (GNU LGPL 2.1)
-  * HOMEPAGE:
-    * http://anonsvn.jboss.org/repos/common/common-logging-spi/
-
-This product optionally depends on 'Apache Felix', an open source OSGi
-framework implementation, which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.felix.txt (Apache License 2.0)
-  * HOMEPAGE:
-    * http://felix.apache.org/
diff --git a/modules/transport-netty4/licenses/netty-codec-http-4.1.13.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http-4.1.13.Final.jar.sha1
new file mode 100644
index 0000000000000..80d18918e1d3f
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-codec-http-4.1.13.Final.jar.sha1
@@ -0,0 +1 @@
+0ee87368766e6b900cf6be8ac9cdce27156e9411
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-codec-http-4.1.5.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http-4.1.5.Final.jar.sha1
deleted file mode 100644
index 93a445416a53c..0000000000000
--- a/modules/transport-netty4/licenses/netty-codec-http-4.1.5.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-087bda1b9ec7e3f75ca721fc87735cbedad2aa1a
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-codec-http-NOTICE.txt b/modules/transport-netty4/licenses/netty-codec-http-NOTICE.txt
deleted file mode 100644
index 5bbf91a14de23..0000000000000
--- a/modules/transport-netty4/licenses/netty-codec-http-NOTICE.txt
+++ /dev/null
@@ -1,116 +0,0 @@
-
-                            The Netty Project
-                            =================
-
-Please visit the Netty web site for more information:
-
-  * http://netty.io/
-
-Copyright 2011 The Netty Project
-
-The Netty Project licenses this file to you under the Apache License,
-version 2.0 (the "License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at:
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-License for the specific language governing permissions and limitations
-under the License.
-
-Also, please refer to each LICENSE.<component>.txt file, which is located in
-the 'license' directory of the distribution file, for the license terms of the
-components that this product depends on.
-
--------------------------------------------------------------------------------
-This product contains the extensions to Java Collections Framework which has
-been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene:
-
-  * LICENSE:
-    * license/LICENSE.jsr166y.txt (Public Domain)
-  * HOMEPAGE:
-    * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/
-    * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/
-
-This product contains a modified version of Robert Harder's Public Domain
-Base64 Encoder and Decoder, which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.base64.txt (Public Domain)
-  * HOMEPAGE:
-    * http://iharder.sourceforge.net/current/java/base64/
-
-This product contains a modified version of 'JZlib', a re-implementation of
-zlib in pure Java, which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.jzlib.txt (BSD Style License)
-  * HOMEPAGE:
-    * http://www.jcraft.com/jzlib/
-
-This product contains a modified version of 'Webbit', a Java event based
-WebSocket and HTTP server:
-
-  * LICENSE:
-    * license/LICENSE.webbit.txt (BSD License)
-  * HOMEPAGE:
-    * https://github.com/joewalnes/webbit
-
-This product optionally depends on 'Protocol Buffers', Google's data
-interchange format, which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.protobuf.txt (New BSD License)
-  * HOMEPAGE:
-    * http://code.google.com/p/protobuf/
-
-This product optionally depends on 'Bouncy Castle Crypto APIs' to generate
-a temporary self-signed X.509 certificate when the JVM does not provide the
-equivalent functionality.  It can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.bouncycastle.txt (MIT License)
-  * HOMEPAGE:
-    * http://www.bouncycastle.org/
-
-This product optionally depends on 'SLF4J', a simple logging facade for Java,
-which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.slf4j.txt (MIT License)
-  * HOMEPAGE:
-    * http://www.slf4j.org/
-
-This product optionally depends on 'Apache Commons Logging', a logging
-framework, which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.commons-logging.txt (Apache License 2.0)
-  * HOMEPAGE:
-    * http://commons.apache.org/logging/
-
-This product optionally depends on 'Apache Log4J', a logging framework,
-which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.log4j.txt (Apache License 2.0)
-  * HOMEPAGE:
-    * http://logging.apache.org/log4j/
-
-This product optionally depends on 'JBoss Logging', a logging framework,
-which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.jboss-logging.txt (GNU LGPL 2.1)
-  * HOMEPAGE:
-    * http://anonsvn.jboss.org/repos/common/common-logging-spi/
-
-This product optionally depends on 'Apache Felix', an open source OSGi
-framework implementation, which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.felix.txt (Apache License 2.0)
-  * HOMEPAGE:
-    * http://felix.apache.org/
diff --git a/modules/transport-netty4/licenses/netty-common-4.1.13.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-common-4.1.13.Final.jar.sha1
new file mode 100644
index 0000000000000..044ec3ef4ed2c
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-common-4.1.13.Final.jar.sha1
@@ -0,0 +1 @@
+f640e8cd8866527150784f8986152d3bba45b712
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-common-4.1.5.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-common-4.1.5.Final.jar.sha1
deleted file mode 100644
index 95da96aaf8044..0000000000000
--- a/modules/transport-netty4/licenses/netty-common-4.1.5.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-607f8433d8782445e72abe34e43a7e57e86a5e6c
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-common-LICENSE.txt b/modules/transport-netty4/licenses/netty-common-LICENSE.txt
deleted file mode 100644
index d645695673349..0000000000000
--- a/modules/transport-netty4/licenses/netty-common-LICENSE.txt
+++ /dev/null
@@ -1,202 +0,0 @@
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
diff --git a/modules/transport-netty4/licenses/netty-common-NOTICE.txt b/modules/transport-netty4/licenses/netty-common-NOTICE.txt
deleted file mode 100644
index 5bbf91a14de23..0000000000000
--- a/modules/transport-netty4/licenses/netty-common-NOTICE.txt
+++ /dev/null
@@ -1,116 +0,0 @@
-
-                            The Netty Project
-                            =================
-
-Please visit the Netty web site for more information:
-
-  * http://netty.io/
-
-Copyright 2011 The Netty Project
-
-The Netty Project licenses this file to you under the Apache License,
-version 2.0 (the "License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at:
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-License for the specific language governing permissions and limitations
-under the License.
-
-Also, please refer to each LICENSE.<component>.txt file, which is located in
-the 'license' directory of the distribution file, for the license terms of the
-components that this product depends on.
-
--------------------------------------------------------------------------------
-This product contains the extensions to Java Collections Framework which has
-been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene:
-
-  * LICENSE:
-    * license/LICENSE.jsr166y.txt (Public Domain)
-  * HOMEPAGE:
-    * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/
-    * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/
-
-This product contains a modified version of Robert Harder's Public Domain
-Base64 Encoder and Decoder, which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.base64.txt (Public Domain)
-  * HOMEPAGE:
-    * http://iharder.sourceforge.net/current/java/base64/
-
-This product contains a modified version of 'JZlib', a re-implementation of
-zlib in pure Java, which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.jzlib.txt (BSD Style License)
-  * HOMEPAGE:
-    * http://www.jcraft.com/jzlib/
-
-This product contains a modified version of 'Webbit', a Java event based
-WebSocket and HTTP server:
-
-  * LICENSE:
-    * license/LICENSE.webbit.txt (BSD License)
-  * HOMEPAGE:
-    * https://github.com/joewalnes/webbit
-
-This product optionally depends on 'Protocol Buffers', Google's data
-interchange format, which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.protobuf.txt (New BSD License)
-  * HOMEPAGE:
-    * http://code.google.com/p/protobuf/
-
-This product optionally depends on 'Bouncy Castle Crypto APIs' to generate
-a temporary self-signed X.509 certificate when the JVM does not provide the
-equivalent functionality.  It can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.bouncycastle.txt (MIT License)
-  * HOMEPAGE:
-    * http://www.bouncycastle.org/
-
-This product optionally depends on 'SLF4J', a simple logging facade for Java,
-which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.slf4j.txt (MIT License)
-  * HOMEPAGE:
-    * http://www.slf4j.org/
-
-This product optionally depends on 'Apache Commons Logging', a logging
-framework, which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.commons-logging.txt (Apache License 2.0)
-  * HOMEPAGE:
-    * http://commons.apache.org/logging/
-
-This product optionally depends on 'Apache Log4J', a logging framework,
-which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.log4j.txt (Apache License 2.0)
-  * HOMEPAGE:
-    * http://logging.apache.org/log4j/
-
-This product optionally depends on 'JBoss Logging', a logging framework,
-which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.jboss-logging.txt (GNU LGPL 2.1)
-  * HOMEPAGE:
-    * http://anonsvn.jboss.org/repos/common/common-logging-spi/
-
-This product optionally depends on 'Apache Felix', an open source OSGi
-framework implementation, which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.felix.txt (Apache License 2.0)
-  * HOMEPAGE:
-    * http://felix.apache.org/
diff --git a/modules/transport-netty4/licenses/netty-handler-4.1.13.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-handler-4.1.13.Final.jar.sha1
new file mode 100644
index 0000000000000..862f16a32a3ec
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-handler-4.1.13.Final.jar.sha1
@@ -0,0 +1 @@
+85847aa81a98d29948731befb4784d141046fa0e
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-handler-4.1.5.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-handler-4.1.5.Final.jar.sha1
deleted file mode 100644
index 7aadd85058e78..0000000000000
--- a/modules/transport-netty4/licenses/netty-handler-4.1.5.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-6262900ee9487e62560030a136160df953b1cd6b
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-handler-LICENSE.txt b/modules/transport-netty4/licenses/netty-handler-LICENSE.txt
deleted file mode 100644
index d645695673349..0000000000000
--- a/modules/transport-netty4/licenses/netty-handler-LICENSE.txt
+++ /dev/null
@@ -1,202 +0,0 @@
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
diff --git a/modules/transport-netty4/licenses/netty-handler-NOTICE.txt b/modules/transport-netty4/licenses/netty-handler-NOTICE.txt
deleted file mode 100644
index 5bbf91a14de23..0000000000000
--- a/modules/transport-netty4/licenses/netty-handler-NOTICE.txt
+++ /dev/null
@@ -1,116 +0,0 @@
-
-                            The Netty Project
-                            =================
-
-Please visit the Netty web site for more information:
-
-  * http://netty.io/
-
-Copyright 2011 The Netty Project
-
-The Netty Project licenses this file to you under the Apache License,
-version 2.0 (the "License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at:
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-License for the specific language governing permissions and limitations
-under the License.
-
-Also, please refer to each LICENSE.<component>.txt file, which is located in
-the 'license' directory of the distribution file, for the license terms of the
-components that this product depends on.
-
--------------------------------------------------------------------------------
-This product contains the extensions to Java Collections Framework which has
-been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene:
-
-  * LICENSE:
-    * license/LICENSE.jsr166y.txt (Public Domain)
-  * HOMEPAGE:
-    * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/
-    * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/
-
-This product contains a modified version of Robert Harder's Public Domain
-Base64 Encoder and Decoder, which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.base64.txt (Public Domain)
-  * HOMEPAGE:
-    * http://iharder.sourceforge.net/current/java/base64/
-
-This product contains a modified version of 'JZlib', a re-implementation of
-zlib in pure Java, which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.jzlib.txt (BSD Style License)
-  * HOMEPAGE:
-    * http://www.jcraft.com/jzlib/
-
-This product contains a modified version of 'Webbit', a Java event based
-WebSocket and HTTP server:
-
-  * LICENSE:
-    * license/LICENSE.webbit.txt (BSD License)
-  * HOMEPAGE:
-    * https://github.com/joewalnes/webbit
-
-This product optionally depends on 'Protocol Buffers', Google's data
-interchange format, which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.protobuf.txt (New BSD License)
-  * HOMEPAGE:
-    * http://code.google.com/p/protobuf/
-
-This product optionally depends on 'Bouncy Castle Crypto APIs' to generate
-a temporary self-signed X.509 certificate when the JVM does not provide the
-equivalent functionality.  It can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.bouncycastle.txt (MIT License)
-  * HOMEPAGE:
-    * http://www.bouncycastle.org/
-
-This product optionally depends on 'SLF4J', a simple logging facade for Java,
-which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.slf4j.txt (MIT License)
-  * HOMEPAGE:
-    * http://www.slf4j.org/
-
-This product optionally depends on 'Apache Commons Logging', a logging
-framework, which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.commons-logging.txt (Apache License 2.0)
-  * HOMEPAGE:
-    * http://commons.apache.org/logging/
-
-This product optionally depends on 'Apache Log4J', a logging framework,
-which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.log4j.txt (Apache License 2.0)
-  * HOMEPAGE:
-    * http://logging.apache.org/log4j/
-
-This product optionally depends on 'JBoss Logging', a logging framework,
-which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.jboss-logging.txt (GNU LGPL 2.1)
-  * HOMEPAGE:
-    * http://anonsvn.jboss.org/repos/common/common-logging-spi/
-
-This product optionally depends on 'Apache Felix', an open source OSGi
-framework implementation, which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.felix.txt (Apache License 2.0)
-  * HOMEPAGE:
-    * http://felix.apache.org/
diff --git a/modules/transport-netty4/licenses/netty-resolver-4.1.13.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-resolver-4.1.13.Final.jar.sha1
new file mode 100644
index 0000000000000..7857ddac89cb9
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-resolver-4.1.13.Final.jar.sha1
@@ -0,0 +1 @@
+d33ce420bd22c8a53246296ceb6e1ff08d31f8e1
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-resolver-4.1.5.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-resolver-4.1.5.Final.jar.sha1
deleted file mode 100644
index 060655014a326..0000000000000
--- a/modules/transport-netty4/licenses/netty-resolver-4.1.5.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-5f367bedcdc185a727fda3296b9a18014cdc22c4
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-resolver-LICENSE.txt b/modules/transport-netty4/licenses/netty-resolver-LICENSE.txt
deleted file mode 100644
index d645695673349..0000000000000
--- a/modules/transport-netty4/licenses/netty-resolver-LICENSE.txt
+++ /dev/null
@@ -1,202 +0,0 @@
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
diff --git a/modules/transport-netty4/licenses/netty-resolver-NOTICE.txt b/modules/transport-netty4/licenses/netty-resolver-NOTICE.txt
deleted file mode 100644
index 5bbf91a14de23..0000000000000
--- a/modules/transport-netty4/licenses/netty-resolver-NOTICE.txt
+++ /dev/null
@@ -1,116 +0,0 @@
-
-                            The Netty Project
-                            =================
-
-Please visit the Netty web site for more information:
-
-  * http://netty.io/
-
-Copyright 2011 The Netty Project
-
-The Netty Project licenses this file to you under the Apache License,
-version 2.0 (the "License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at:
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-License for the specific language governing permissions and limitations
-under the License.
-
-Also, please refer to each LICENSE.<component>.txt file, which is located in
-the 'license' directory of the distribution file, for the license terms of the
-components that this product depends on.
-
--------------------------------------------------------------------------------
-This product contains the extensions to Java Collections Framework which has
-been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene:
-
-  * LICENSE:
-    * license/LICENSE.jsr166y.txt (Public Domain)
-  * HOMEPAGE:
-    * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/
-    * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/
-
-This product contains a modified version of Robert Harder's Public Domain
-Base64 Encoder and Decoder, which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.base64.txt (Public Domain)
-  * HOMEPAGE:
-    * http://iharder.sourceforge.net/current/java/base64/
-
-This product contains a modified version of 'JZlib', a re-implementation of
-zlib in pure Java, which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.jzlib.txt (BSD Style License)
-  * HOMEPAGE:
-    * http://www.jcraft.com/jzlib/
-
-This product contains a modified version of 'Webbit', a Java event based
-WebSocket and HTTP server:
-
-  * LICENSE:
-    * license/LICENSE.webbit.txt (BSD License)
-  * HOMEPAGE:
-    * https://github.com/joewalnes/webbit
-
-This product optionally depends on 'Protocol Buffers', Google's data
-interchange format, which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.protobuf.txt (New BSD License)
-  * HOMEPAGE:
-    * http://code.google.com/p/protobuf/
-
-This product optionally depends on 'Bouncy Castle Crypto APIs' to generate
-a temporary self-signed X.509 certificate when the JVM does not provide the
-equivalent functionality.  It can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.bouncycastle.txt (MIT License)
-  * HOMEPAGE:
-    * http://www.bouncycastle.org/
-
-This product optionally depends on 'SLF4J', a simple logging facade for Java,
-which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.slf4j.txt (MIT License)
-  * HOMEPAGE:
-    * http://www.slf4j.org/
-
-This product optionally depends on 'Apache Commons Logging', a logging
-framework, which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.commons-logging.txt (Apache License 2.0)
-  * HOMEPAGE:
-    * http://commons.apache.org/logging/
-
-This product optionally depends on 'Apache Log4J', a logging framework,
-which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.log4j.txt (Apache License 2.0)
-  * HOMEPAGE:
-    * http://logging.apache.org/log4j/
-
-This product optionally depends on 'JBoss Logging', a logging framework,
-which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.jboss-logging.txt (GNU LGPL 2.1)
-  * HOMEPAGE:
-    * http://anonsvn.jboss.org/repos/common/common-logging-spi/
-
-This product optionally depends on 'Apache Felix', an open source OSGi
-framework implementation, which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.felix.txt (Apache License 2.0)
-  * HOMEPAGE:
-    * http://felix.apache.org/
diff --git a/modules/transport-netty4/licenses/netty-transport-4.1.13.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-4.1.13.Final.jar.sha1
new file mode 100644
index 0000000000000..dc86ce66f4357
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-transport-4.1.13.Final.jar.sha1
@@ -0,0 +1 @@
+5008406221a849a350ad2a8885f14ac330e038f3
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-transport-4.1.5.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-4.1.5.Final.jar.sha1
deleted file mode 100644
index 0e7dc27daaf0d..0000000000000
--- a/modules/transport-netty4/licenses/netty-transport-4.1.5.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-37126b370722ff9631ee13c91139aacec0a71d1d
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-transport-LICENSE.txt b/modules/transport-netty4/licenses/netty-transport-LICENSE.txt
deleted file mode 100644
index d645695673349..0000000000000
--- a/modules/transport-netty4/licenses/netty-transport-LICENSE.txt
+++ /dev/null
@@ -1,202 +0,0 @@
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
diff --git a/modules/transport-netty4/licenses/netty-transport-NOTICE.txt b/modules/transport-netty4/licenses/netty-transport-NOTICE.txt
deleted file mode 100644
index 5bbf91a14de23..0000000000000
--- a/modules/transport-netty4/licenses/netty-transport-NOTICE.txt
+++ /dev/null
@@ -1,116 +0,0 @@
-
-                            The Netty Project
-                            =================
-
-Please visit the Netty web site for more information:
-
-  * http://netty.io/
-
-Copyright 2011 The Netty Project
-
-The Netty Project licenses this file to you under the Apache License,
-version 2.0 (the "License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at:
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-License for the specific language governing permissions and limitations
-under the License.
-
-Also, please refer to each LICENSE.<component>.txt file, which is located in
-the 'license' directory of the distribution file, for the license terms of the
-components that this product depends on.
-
--------------------------------------------------------------------------------
-This product contains the extensions to Java Collections Framework which has
-been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene:
-
-  * LICENSE:
-    * license/LICENSE.jsr166y.txt (Public Domain)
-  * HOMEPAGE:
-    * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/
-    * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/
-
-This product contains a modified version of Robert Harder's Public Domain
-Base64 Encoder and Decoder, which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.base64.txt (Public Domain)
-  * HOMEPAGE:
-    * http://iharder.sourceforge.net/current/java/base64/
-
-This product contains a modified version of 'JZlib', a re-implementation of
-zlib in pure Java, which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.jzlib.txt (BSD Style License)
-  * HOMEPAGE:
-    * http://www.jcraft.com/jzlib/
-
-This product contains a modified version of 'Webbit', a Java event based
-WebSocket and HTTP server:
-
-  * LICENSE:
-    * license/LICENSE.webbit.txt (BSD License)
-  * HOMEPAGE:
-    * https://github.com/joewalnes/webbit
-
-This product optionally depends on 'Protocol Buffers', Google's data
-interchange format, which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.protobuf.txt (New BSD License)
-  * HOMEPAGE:
-    * http://code.google.com/p/protobuf/
-
-This product optionally depends on 'Bouncy Castle Crypto APIs' to generate
-a temporary self-signed X.509 certificate when the JVM does not provide the
-equivalent functionality.  It can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.bouncycastle.txt (MIT License)
-  * HOMEPAGE:
-    * http://www.bouncycastle.org/
-
-This product optionally depends on 'SLF4J', a simple logging facade for Java,
-which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.slf4j.txt (MIT License)
-  * HOMEPAGE:
-    * http://www.slf4j.org/
-
-This product optionally depends on 'Apache Commons Logging', a logging
-framework, which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.commons-logging.txt (Apache License 2.0)
-  * HOMEPAGE:
-    * http://commons.apache.org/logging/
-
-This product optionally depends on 'Apache Log4J', a logging framework,
-which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.log4j.txt (Apache License 2.0)
-  * HOMEPAGE:
-    * http://logging.apache.org/log4j/
-
-This product optionally depends on 'JBoss Logging', a logging framework,
-which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.jboss-logging.txt (GNU LGPL 2.1)
-  * HOMEPAGE:
-    * http://anonsvn.jboss.org/repos/common/common-logging-spi/
-
-This product optionally depends on 'Apache Felix', an open source OSGi
-framework implementation, which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.felix.txt (Apache License 2.0)
-  * HOMEPAGE:
-    * http://felix.apache.org/
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpChannel.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpChannel.java
index 62443dc541e5b..07e91ec50e44b 100644
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpChannel.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpChannel.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.http.netty4;
 
 import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
 import io.netty.channel.Channel;
 import io.netty.channel.ChannelFutureListener;
 import io.netty.channel.ChannelPromise;
@@ -29,6 +30,7 @@
 import io.netty.handler.codec.http.HttpHeaderNames;
 import io.netty.handler.codec.http.HttpHeaderValues;
 import io.netty.handler.codec.http.HttpHeaders;
+import io.netty.handler.codec.http.HttpMethod;
 import io.netty.handler.codec.http.HttpResponse;
 import io.netty.handler.codec.http.HttpResponseStatus;
 import io.netty.handler.codec.http.HttpVersion;
@@ -83,17 +85,21 @@ final class Netty4HttpChannel extends AbstractRestChannel {
     }
 
     @Override
-    public BytesStreamOutput newBytesOutput() {
+    protected BytesStreamOutput newBytesOutput() {
         return new ReleasableBytesStreamOutput(transport.bigArrays);
     }
 
-
     @Override
     public void sendResponse(RestResponse response) {
         // if the response object was created upstream, then use it;
         // otherwise, create a new one
         ByteBuf buffer = Netty4Utils.toByteBuf(response.content());
-        FullHttpResponse resp = newResponse(buffer);
+        final FullHttpResponse resp;
+        if (HttpMethod.HEAD.equals(nettyRequest.method())) {
+            resp = newResponse(Unpooled.EMPTY_BUFFER);
+        } else {
+            resp = newResponse(buffer);
+        }
         resp.setStatus(getStatus(response.status()));
 
         Netty4CorsHandler.setCorsResponseHeaders(nettyRequest, resp, transport.getCorsConfig());
@@ -108,7 +114,8 @@ public void sendResponse(RestResponse response) {
         addCustomHeaders(resp, threadContext.getResponseHeaders());
 
         BytesReference content = response.content();
-        boolean release = content instanceof Releasable;
+        boolean releaseContent = content instanceof Releasable;
+        boolean releaseBytesStreamOutput = bytesOutputOrNull() instanceof ReleasableBytesStreamOutput;
         try {
             // If our response doesn't specify a content-type header, set one
             setHeaderField(resp, HttpHeaderNames.CONTENT_TYPE.toString(), response.contentType(), false);
@@ -119,25 +126,34 @@ public void sendResponse(RestResponse response) {
 
             final ChannelPromise promise = channel.newPromise();
 
-            if (release) {
+            if (releaseContent) {
                 promise.addListener(f -> ((Releasable)content).close());
-                release = false;
+            }
+
+            if (releaseBytesStreamOutput) {
+                promise.addListener(f -> bytesOutputOrNull().close());
             }
 
             if (isCloseConnection()) {
                 promise.addListener(ChannelFutureListener.CLOSE);
             }
 
+            final Object msg;
             if (pipelinedRequest != null) {
-                channel.writeAndFlush(pipelinedRequest.createHttpResponse(resp, promise));
+                msg = pipelinedRequest.createHttpResponse(resp, promise);
             } else {
-                channel.writeAndFlush(resp, promise);
+                msg = resp;
             }
-
+            channel.writeAndFlush(msg, promise);
+            releaseContent = false;
+            releaseBytesStreamOutput = false;
         } finally {
-            if (release) {
+            if (releaseContent) {
                 ((Releasable) content).close();
             }
+            if (releaseBytesStreamOutput) {
+                bytesOutputOrNull().close();
+            }
             if (pipelinedRequest != null) {
                 pipelinedRequest.release();
             }
@@ -185,8 +201,8 @@ private boolean isHttp10() {
     // Determine if the request connection should be closed on completion.
     private boolean isCloseConnection() {
         final boolean http10 = isHttp10();
-        return HttpHeaderValues.CLOSE.equals(nettyRequest.headers().get(HttpHeaderNames.CONNECTION)) ||
-            (http10 && HttpHeaderValues.KEEP_ALIVE.equals(nettyRequest.headers().get(HttpHeaderNames.CONNECTION)) == false);
+        return HttpHeaderValues.CLOSE.contentEqualsIgnoreCase(nettyRequest.headers().get(HttpHeaderNames.CONNECTION)) ||
+            (http10 && !HttpHeaderValues.KEEP_ALIVE.contentEqualsIgnoreCase(nettyRequest.headers().get(HttpHeaderNames.CONNECTION)));
     }
 
     // Create a new {@link HttpResponse} to transmit the response for the netty request.
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequest.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequest.java
index 2e511d1562279..61f0e21cb646d 100644
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequest.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequest.java
@@ -19,20 +19,25 @@
 
 package org.elasticsearch.http.netty4;
 
+import io.netty.channel.Channel;
 import io.netty.handler.codec.http.FullHttpRequest;
-import io.netty.handler.codec.http.HttpRequest;
+import io.netty.handler.codec.http.HttpHeaders;
+import io.netty.handler.codec.http.HttpMethod;
+
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.transport.netty4.Netty4Utils;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.RestUtils;
-
-import io.netty.channel.Channel;
-import io.netty.handler.codec.http.HttpMethod;
+import org.elasticsearch.transport.netty4.Netty4Utils;
 
 import java.net.SocketAddress;
-import java.util.HashMap;
+import java.util.AbstractMap;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
 
 public class Netty4HttpRequest extends RestRequest {
 
@@ -40,8 +45,8 @@ public class Netty4HttpRequest extends RestRequest {
     private final Channel channel;
     private final BytesReference content;
 
-    Netty4HttpRequest(FullHttpRequest request, Channel channel) {
-        super(request.uri());
+    Netty4HttpRequest(NamedXContentRegistry xContentRegistry, FullHttpRequest request, Channel channel) {
+        super(xContentRegistry, request.uri(), new HttpHeadersMap(request.headers()));
         this.request = request;
         this.channel = channel;
         if (request.content().isReadable()) {
@@ -122,14 +127,81 @@ public Channel getChannel() {
         return channel;
     }
 
-    @Override
-    public String header(String name) {
-        return request.headers().get(name);
-    }
+    /**
+     * A wrapper of {@link HttpHeaders} that implements a map to prevent copying unnecessarily. This class does not support modifications
+     * and due to the underlying implementation, it performs case insensitive lookups of key to values.
+     *
+     * It is important to note that this implementation does have some downsides in that each invocation of the
+     * {@link #values()} and {@link #entrySet()} methods will perform a copy of the values in the HttpHeaders rather than returning a
+     * view of the underlying values.
+     */
+    private static class HttpHeadersMap implements Map<String, List<String>> {
 
-    @Override
-    public Iterable<Map.Entry<String, String>> headers() {
-        return request.headers().entries();
-    }
+        private final HttpHeaders httpHeaders;
+
+        private HttpHeadersMap(HttpHeaders httpHeaders) {
+            this.httpHeaders = httpHeaders;
+        }
+
+        @Override
+        public int size() {
+            return httpHeaders.size();
+        }
+
+        @Override
+        public boolean isEmpty() {
+            return httpHeaders.isEmpty();
+        }
 
+        @Override
+        public boolean containsKey(Object key) {
+            return key instanceof String && httpHeaders.contains((String) key);
+        }
+
+        @Override
+        public boolean containsValue(Object value) {
+            return value instanceof List && httpHeaders.names().stream().map(httpHeaders::getAll).anyMatch(value::equals);
+        }
+
+        @Override
+        public List<String> get(Object key) {
+            return key instanceof String ? httpHeaders.getAll((String) key) : null;
+        }
+
+        @Override
+        public List<String> put(String key, List<String> value) {
+            throw new UnsupportedOperationException("modifications are not supported");
+        }
+
+        @Override
+        public List<String> remove(Object key) {
+            throw new UnsupportedOperationException("modifications are not supported");
+        }
+
+        @Override
+        public void putAll(Map<? extends String, ? extends List<String>> m) {
+            throw new UnsupportedOperationException("modifications are not supported");
+        }
+
+        @Override
+        public void clear() {
+            throw new UnsupportedOperationException("modifications are not supported");
+        }
+
+        @Override
+        public Set<String> keySet() {
+            return httpHeaders.names();
+        }
+
+        @Override
+        public Collection<List<String>> values() {
+            return httpHeaders.names().stream().map(k -> Collections.unmodifiableList(httpHeaders.getAll(k))).collect(Collectors.toList());
+        }
+
+        @Override
+        public Set<Entry<String, List<String>>> entrySet() {
+            return httpHeaders.names().stream()
+                .map(k -> new AbstractMap.SimpleImmutableEntry<>(k, httpHeaders.getAll(k))).collect(Collectors.toSet());
+        }
+    }
 }
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequestHandler.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequestHandler.java
index a4402d4816fc3..b31c412920aa1 100644
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequestHandler.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequestHandler.java
@@ -27,6 +27,7 @@
 import io.netty.handler.codec.http.FullHttpRequest;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.http.netty4.pipelining.HttpPipelinedRequest;
+import org.elasticsearch.transport.netty4.Netty4Utils;
 
 @ChannelHandler.Sharable
 class Netty4HttpRequestHandler extends SimpleChannelInboundHandler<Object> {
@@ -63,15 +64,21 @@ protected void channelRead0(ChannelHandlerContext ctx, Object msg) throws Except
                         Unpooled.copiedBuffer(request.content()),
                         request.headers(),
                         request.trailingHeaders());
+        final Netty4HttpRequest httpRequest = new Netty4HttpRequest(serverTransport.xContentRegistry, copy, ctx.channel());
+        final Netty4HttpChannel channel =
+                new Netty4HttpChannel(serverTransport, httpRequest, pipelinedRequest, detailedErrorsEnabled, threadContext);
 
-        final Netty4HttpRequest httpRequest = new Netty4HttpRequest(copy, ctx.channel());
-        serverTransport.dispatchRequest(
-            httpRequest,
-            new Netty4HttpChannel(serverTransport, httpRequest, pipelinedRequest, detailedErrorsEnabled, threadContext));
+        if (request.decoderResult().isSuccess()) {
+            serverTransport.dispatchRequest(httpRequest, channel);
+        } else {
+            assert request.decoderResult().isFailure();
+            serverTransport.dispatchBadRequest(httpRequest, channel, request.decoderResult().cause());
+        }
     }
 
     @Override
     public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
+        Netty4Utils.maybeDie(cause);
         serverTransport.exceptionCaught(ctx, cause);
     }
 
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java
index 96795cec3e0cb..a517c032ec908 100644
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java
@@ -47,7 +47,6 @@
 import org.apache.logging.log4j.util.Supplier;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Setting;
@@ -63,16 +62,16 @@
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.http.BindHttpException;
 import org.elasticsearch.http.HttpInfo;
-import org.elasticsearch.http.HttpServerAdapter;
 import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.http.HttpStats;
 import org.elasticsearch.http.netty4.cors.Netty4CorsConfig;
 import org.elasticsearch.http.netty4.cors.Netty4CorsConfigBuilder;
 import org.elasticsearch.http.netty4.cors.Netty4CorsHandler;
 import org.elasticsearch.http.netty4.pipelining.HttpPipeliningHandler;
-import org.elasticsearch.monitor.jvm.JvmInfo;
 import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestUtils;
@@ -148,21 +147,11 @@ public class Netty4HttpServerTransport extends AbstractLifecycleComponent implem
         Setting.byteSizeSetting("http.tcp.receive_buffer_size", NetworkService.TcpSettings.TCP_RECEIVE_BUFFER_SIZE,
             Property.NodeScope, Property.Shared);
     public static final Setting<ByteSizeValue> SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_SIZE =
-        Setting.byteSizeSetting("transport.netty.receive_predictor_size",
-            settings -> {
-                long defaultReceiverPredictor = 512 * 1024;
-                if (JvmInfo.jvmInfo().getMem().getDirectMemoryMax().bytes() > 0) {
-                    // we can guess a better default...
-                    long l = (long) ((0.3 * JvmInfo.jvmInfo().getMem().getDirectMemoryMax().bytes()) / SETTING_HTTP_WORKER_COUNT.get
-                        (settings));
-                    defaultReceiverPredictor = Math.min(defaultReceiverPredictor, Math.max(l, 64 * 1024));
-                }
-                return new ByteSizeValue(defaultReceiverPredictor).toString();
-            }, Property.NodeScope);
+        Setting.byteSizeSetting("http.netty.receive_predictor_size", new ByteSizeValue(64, ByteSizeUnit.KB), Property.NodeScope);
     public static final Setting<ByteSizeValue> SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_MIN =
-        byteSizeSetting("http.netty.receive_predictor_min", SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_SIZE, Property.NodeScope);
+        byteSizeSetting("http.netty.receive_predictor_min", SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_SIZE, Property.NodeScope, Property.Shared);
     public static final Setting<ByteSizeValue> SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_MAX =
-        byteSizeSetting("http.netty.receive_predictor_max", SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_SIZE, Property.NodeScope);
+        byteSizeSetting("http.netty.receive_predictor_max", SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_SIZE, Property.NodeScope, Property.Shared);
 
 
     protected final NetworkService networkService;
@@ -195,6 +184,10 @@ public class Netty4HttpServerTransport extends AbstractLifecycleComponent implem
 
     protected final boolean detailedErrorsEnabled;
     protected final ThreadPool threadPool;
+    /**
+     * The registry used to construct parsers so they support {@link XContentParser#namedObject(Class, String, Object)}.
+     */
+    protected final NamedXContentRegistry xContentRegistry;
 
     protected final boolean tcpNoDelay;
     protected final boolean tcpKeepAlive;
@@ -206,6 +199,7 @@ public class Netty4HttpServerTransport extends AbstractLifecycleComponent implem
 
     protected final ByteSizeValue maxCumulationBufferCapacity;
     protected final int maxCompositeBufferComponents;
+    private final Dispatcher dispatcher;
 
     protected volatile ServerBootstrap serverBootstrap;
 
@@ -216,16 +210,18 @@ public class Netty4HttpServerTransport extends AbstractLifecycleComponent implem
     // package private for testing
     Netty4OpenChannelsHandler serverOpenChannels;
 
-    protected volatile HttpServerAdapter httpServerAdapter;
 
     private final Netty4CorsConfig corsConfig;
 
-    @Inject
-    public Netty4HttpServerTransport(Settings settings, NetworkService networkService, BigArrays bigArrays, ThreadPool threadPool) {
+    public Netty4HttpServerTransport(Settings settings, NetworkService networkService, BigArrays bigArrays, ThreadPool threadPool,
+                                     NamedXContentRegistry xContentRegistry, Dispatcher dispatcher) {
         super(settings);
+        Netty4Utils.setAvailableProcessors(EsExecutors.PROCESSORS_SETTING.get(settings));
         this.networkService = networkService;
         this.bigArrays = bigArrays;
         this.threadPool = threadPool;
+        this.xContentRegistry = xContentRegistry;
+        this.dispatcher = dispatcher;
 
         ByteSizeValue maxContentLength = SETTING_HTTP_MAX_CONTENT_LENGTH.get(settings);
         this.maxChunkSize = SETTING_HTTP_MAX_CHUNK_SIZE.get(settings);
@@ -249,13 +245,13 @@ public Netty4HttpServerTransport(Settings settings, NetworkService networkServic
         // See AdaptiveReceiveBufferSizePredictor#DEFAULT_XXX for default values in netty..., we can use higher ones for us, even fixed one
         ByteSizeValue receivePredictorMin = SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_MIN.get(settings);
         ByteSizeValue receivePredictorMax = SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_MAX.get(settings);
-        if (receivePredictorMax.bytes() == receivePredictorMin.bytes()) {
-            recvByteBufAllocator = new FixedRecvByteBufAllocator(Math.toIntExact(receivePredictorMax.bytes()));
+        if (receivePredictorMax.getBytes() == receivePredictorMin.getBytes()) {
+            recvByteBufAllocator = new FixedRecvByteBufAllocator(Math.toIntExact(receivePredictorMax.getBytes()));
         } else {
             recvByteBufAllocator = new AdaptiveRecvByteBufAllocator(
-                Math.toIntExact(receivePredictorMin.bytes()),
-                Math.toIntExact(receivePredictorMin.bytes()),
-                Math.toIntExact(receivePredictorMax.bytes()));
+                Math.toIntExact(receivePredictorMin.getBytes()),
+                Math.toIntExact(receivePredictorMin.getBytes()),
+                Math.toIntExact(receivePredictorMax.getBytes()));
         }
 
         this.compression = SETTING_HTTP_COMPRESSION.get(settings);
@@ -265,7 +261,7 @@ public Netty4HttpServerTransport(Settings settings, NetworkService networkServic
         this.corsConfig = buildCorsConfig(settings);
 
         // validate max content length
-        if (maxContentLength.bytes() > Integer.MAX_VALUE) {
+        if (maxContentLength.getBytes() > Integer.MAX_VALUE) {
             logger.warn("maxContentLength[{}] set to high value, resetting it to [100mb]", maxContentLength);
             maxContentLength = new ByteSizeValue(100, ByteSizeUnit.MB);
         }
@@ -281,47 +277,55 @@ public Settings settings() {
         return this.settings;
     }
 
-    @Override
-    public void httpServerAdapter(HttpServerAdapter httpServerAdapter) {
-        this.httpServerAdapter = httpServerAdapter;
-    }
-
     @Override
     protected void doStart() {
-        this.serverOpenChannels = new Netty4OpenChannelsHandler(logger);
+        boolean success = false;
+        try {
+            this.serverOpenChannels = new Netty4OpenChannelsHandler(logger);
 
-        serverBootstrap = new ServerBootstrap();
-        if (blockingServer) {
-            serverBootstrap.group(new OioEventLoopGroup(workerCount, daemonThreadFactory(settings, HTTP_SERVER_WORKER_THREAD_NAME_PREFIX)));
-            serverBootstrap.channel(OioServerSocketChannel.class);
-        } else {
-            serverBootstrap.group(new NioEventLoopGroup(workerCount, daemonThreadFactory(settings, HTTP_SERVER_WORKER_THREAD_NAME_PREFIX)));
-            serverBootstrap.channel(NioServerSocketChannel.class);
-        }
+            serverBootstrap = new ServerBootstrap();
+            if (blockingServer) {
+                serverBootstrap.group(new OioEventLoopGroup(workerCount, daemonThreadFactory(settings,
+                    HTTP_SERVER_WORKER_THREAD_NAME_PREFIX)));
+                serverBootstrap.channel(OioServerSocketChannel.class);
+            } else {
+                serverBootstrap.group(new NioEventLoopGroup(workerCount, daemonThreadFactory(settings,
+                    HTTP_SERVER_WORKER_THREAD_NAME_PREFIX)));
+                serverBootstrap.channel(NioServerSocketChannel.class);
+            }
 
-        serverBootstrap.childHandler(configureServerChannelHandler());
+            serverBootstrap.childHandler(configureServerChannelHandler());
 
-        serverBootstrap.childOption(ChannelOption.TCP_NODELAY, SETTING_HTTP_TCP_NO_DELAY.get(settings));
-        serverBootstrap.childOption(ChannelOption.SO_KEEPALIVE, SETTING_HTTP_TCP_KEEP_ALIVE.get(settings));
+            serverBootstrap.childOption(ChannelOption.TCP_NODELAY, SETTING_HTTP_TCP_NO_DELAY.get(settings));
+            serverBootstrap.childOption(ChannelOption.SO_KEEPALIVE, SETTING_HTTP_TCP_KEEP_ALIVE.get(settings));
 
-        final ByteSizeValue tcpSendBufferSize = SETTING_HTTP_TCP_SEND_BUFFER_SIZE.get(settings);
-        if (tcpSendBufferSize.bytes() > 0) {
-            serverBootstrap.childOption(ChannelOption.SO_SNDBUF, Math.toIntExact(tcpSendBufferSize.bytes()));
-        }
+            final ByteSizeValue tcpSendBufferSize = SETTING_HTTP_TCP_SEND_BUFFER_SIZE.get(settings);
+            if (tcpSendBufferSize.getBytes() > 0) {
+                serverBootstrap.childOption(ChannelOption.SO_SNDBUF, Math.toIntExact(tcpSendBufferSize.getBytes()));
+            }
 
-        final ByteSizeValue tcpReceiveBufferSize = SETTING_HTTP_TCP_RECEIVE_BUFFER_SIZE.get(settings);
-        if (tcpReceiveBufferSize.bytes() > 0) {
-            serverBootstrap.childOption(ChannelOption.SO_RCVBUF, Math.toIntExact(tcpReceiveBufferSize.bytes()));
-        }
+            final ByteSizeValue tcpReceiveBufferSize = SETTING_HTTP_TCP_RECEIVE_BUFFER_SIZE.get(settings);
+            if (tcpReceiveBufferSize.getBytes() > 0) {
+                serverBootstrap.childOption(ChannelOption.SO_RCVBUF, Math.toIntExact(tcpReceiveBufferSize.getBytes()));
+            }
 
-        serverBootstrap.option(ChannelOption.RCVBUF_ALLOCATOR, recvByteBufAllocator);
-        serverBootstrap.childOption(ChannelOption.RCVBUF_ALLOCATOR, recvByteBufAllocator);
+            serverBootstrap.option(ChannelOption.RCVBUF_ALLOCATOR, recvByteBufAllocator);
+            serverBootstrap.childOption(ChannelOption.RCVBUF_ALLOCATOR, recvByteBufAllocator);
 
-        final boolean reuseAddress = SETTING_HTTP_TCP_REUSE_ADDRESS.get(settings);
-        serverBootstrap.option(ChannelOption.SO_REUSEADDR, reuseAddress);
-        serverBootstrap.childOption(ChannelOption.SO_REUSEADDR, reuseAddress);
+            final boolean reuseAddress = SETTING_HTTP_TCP_REUSE_ADDRESS.get(settings);
+            serverBootstrap.option(ChannelOption.SO_REUSEADDR, reuseAddress);
+            serverBootstrap.childOption(ChannelOption.SO_REUSEADDR, reuseAddress);
 
-        this.boundAddress = createBoundHttpAddress();
+            this.boundAddress = createBoundHttpAddress();
+            if (logger.isInfoEnabled()) {
+                logger.info("{}", boundAddress);
+            }
+            success = true;
+        } finally {
+            if (success == false) {
+                doStop(); // otherwise we leak threads since we never moved to started
+            }
+        }
     }
 
     private BoundTransportAddress createBoundHttpAddress() {
@@ -405,14 +409,14 @@ static Netty4CorsConfig buildCorsConfig(Settings settings) {
         if (SETTING_CORS_ALLOW_CREDENTIALS.get(settings)) {
             builder.allowCredentials();
         }
-        String[] strMethods = Strings.splitStringByCommaToArray(SETTING_CORS_ALLOW_METHODS.get(settings));
+        String[] strMethods = Strings.tokenizeToStringArray(SETTING_CORS_ALLOW_METHODS.get(settings), ",");
         HttpMethod[] methods = Arrays.asList(strMethods)
             .stream()
             .map(HttpMethod::valueOf)
             .toArray(size -> new HttpMethod[size]);
         return builder.allowedRequestMethods(methods)
             .maxAge(SETTING_CORS_MAX_AGE.get(settings))
-            .allowedRequestHeaders(Strings.splitStringByCommaToArray(SETTING_CORS_ALLOW_HEADERS.get(settings)))
+            .allowedRequestHeaders(Strings.tokenizeToStringArray(SETTING_CORS_ALLOW_HEADERS.get(settings), ","))
             .shortCircuit()
             .build();
     }
@@ -420,24 +424,21 @@ static Netty4CorsConfig buildCorsConfig(Settings settings) {
     private InetSocketTransportAddress bindAddress(final InetAddress hostAddress) {
         final AtomicReference<Exception> lastException = new AtomicReference<>();
         final AtomicReference<InetSocketAddress> boundSocket = new AtomicReference<>();
-        boolean success = port.iterate(new PortsRange.PortCallback() {
-            @Override
-            public boolean onPortNumber(int portNumber) {
-                try {
-                    synchronized (serverChannels) {
-                        ChannelFuture future = serverBootstrap.bind(new InetSocketAddress(hostAddress, portNumber)).sync();
-                        serverChannels.add(future.channel());
-                        boundSocket.set((InetSocketAddress) future.channel().localAddress());
-                    }
-                } catch (Exception e) {
-                    lastException.set(e);
-                    return false;
+        boolean success = port.iterate(portNumber -> {
+            try {
+                synchronized (serverChannels) {
+                    ChannelFuture future = serverBootstrap.bind(new InetSocketAddress(hostAddress, portNumber)).sync();
+                    serverChannels.add(future.channel());
+                    boundSocket.set((InetSocketAddress) future.channel().localAddress());
                 }
-                return true;
+            } catch (Exception e) {
+                lastException.set(e);
+                return false;
             }
+            return true;
         });
         if (!success) {
-            throw new BindHttpException("Failed to bind to [" + port + "]", lastException.get());
+            throw new BindHttpException("Failed to bind to [" + port.getPortRangeString() + "]", lastException.get());
         }
 
         if (logger.isDebugEnabled()) {
@@ -485,7 +486,7 @@ public HttpInfo info() {
         if (boundTransportAddress == null) {
             return null;
         }
-        return new HttpInfo(boundTransportAddress, maxContentLength.bytes());
+        return new HttpInfo(boundTransportAddress, maxContentLength.getBytes());
     }
 
     @Override
@@ -498,8 +499,18 @@ public Netty4CorsConfig getCorsConfig() {
         return corsConfig;
     }
 
-    protected void dispatchRequest(RestRequest request, RestChannel channel) {
-        httpServerAdapter.dispatchRequest(request, channel, threadPool.getThreadContext());
+    void dispatchRequest(final RestRequest request, final RestChannel channel) {
+        final ThreadContext threadContext = threadPool.getThreadContext();
+        try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
+            dispatcher.dispatchRequest(request, channel, threadContext);
+        }
+    }
+
+    void dispatchBadRequest(final RestRequest request, final RestChannel channel, final Throwable cause) {
+        final ThreadContext threadContext = threadPool.getThreadContext();
+        try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
+            dispatcher.dispatchBadRequest(request, channel, threadContext, cause);
+        }
     }
 
     protected void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
@@ -539,9 +550,9 @@ protected static class HttpChannelHandler extends ChannelInitializer<Channel> {
         private final Netty4HttpRequestHandler requestHandler;
 
         protected HttpChannelHandler(
-            final Netty4HttpServerTransport transport,
-            final boolean detailedErrorsEnabled,
-            final ThreadContext threadContext) {
+                final Netty4HttpServerTransport transport,
+                final boolean detailedErrorsEnabled,
+                final ThreadContext threadContext) {
             this.transport = transport;
             this.requestHandler = new Netty4HttpRequestHandler(transport, detailedErrorsEnabled, threadContext);
         }
@@ -550,14 +561,14 @@ protected HttpChannelHandler(
         protected void initChannel(Channel ch) throws Exception {
             ch.pipeline().addLast("openChannels", transport.serverOpenChannels);
             final HttpRequestDecoder decoder = new HttpRequestDecoder(
-                Math.toIntExact(transport.maxInitialLineLength.bytes()),
-                Math.toIntExact(transport.maxHeaderSize.bytes()),
-                Math.toIntExact(transport.maxChunkSize.bytes()));
+                Math.toIntExact(transport.maxInitialLineLength.getBytes()),
+                Math.toIntExact(transport.maxHeaderSize.getBytes()),
+                Math.toIntExact(transport.maxChunkSize.getBytes()));
             decoder.setCumulator(ByteToMessageDecoder.COMPOSITE_CUMULATOR);
             ch.pipeline().addLast("decoder", decoder);
             ch.pipeline().addLast("decoder_compress", new HttpContentDecompressor());
             ch.pipeline().addLast("encoder", new HttpResponseEncoder());
-            final HttpObjectAggregator aggregator = new HttpObjectAggregator(Math.toIntExact(transport.maxContentLength.bytes()));
+            final HttpObjectAggregator aggregator = new HttpObjectAggregator(Math.toIntExact(transport.maxContentLength.getBytes()));
             if (transport.maxCompositeBufferComponents != -1) {
                 aggregator.setMaxCumulationBufferComponents(transport.maxCompositeBufferComponents);
             }
@@ -569,11 +580,17 @@ protected void initChannel(Channel ch) throws Exception {
                 ch.pipeline().addLast("cors", new Netty4CorsHandler(transport.getCorsConfig()));
             }
             if (transport.pipelining) {
-                ch.pipeline().addLast("pipelining", new HttpPipeliningHandler(transport.pipeliningMaxEvents));
+                ch.pipeline().addLast("pipelining", new HttpPipeliningHandler(transport.logger, transport.pipeliningMaxEvents));
             }
             ch.pipeline().addLast("handler", requestHandler);
         }
 
+        @Override
+        public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
+            Netty4Utils.maybeDie(cause);
+            super.exceptionCaught(ctx, cause);
+        }
+
     }
 
 }
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/pipelining/HttpPipelinedRequest.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/pipelining/HttpPipelinedRequest.java
index 792397a3c3dd4..be1669c60c297 100644
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/pipelining/HttpPipelinedRequest.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/pipelining/HttpPipelinedRequest.java
@@ -20,9 +20,7 @@
 package org.elasticsearch.http.netty4.pipelining;
 
 import io.netty.channel.ChannelPromise;
-import io.netty.handler.codec.http.FullHttpRequest;
 import io.netty.handler.codec.http.FullHttpResponse;
-import io.netty.handler.codec.http.HttpResponse;
 import io.netty.handler.codec.http.LastHttpContent;
 import io.netty.util.ReferenceCounted;
 
@@ -35,8 +33,7 @@ public class HttpPipelinedRequest implements ReferenceCounted {
     private final LastHttpContent last;
     private final int sequence;
 
-
-    HttpPipelinedRequest(final LastHttpContent last, final int sequence) {
+    public HttpPipelinedRequest(final LastHttpContent last, final int sequence) {
         this.last = last;
         this.sequence = sequence;
     }
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/pipelining/HttpPipelinedResponse.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/pipelining/HttpPipelinedResponse.java
index 21659d5fbdfd1..6b6db94d69a59 100644
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/pipelining/HttpPipelinedResponse.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/pipelining/HttpPipelinedResponse.java
@@ -21,7 +21,6 @@
 
 import io.netty.channel.ChannelPromise;
 import io.netty.handler.codec.http.FullHttpResponse;
-import io.netty.handler.codec.http.HttpResponse;
 import io.netty.util.ReferenceCounted;
 
 class HttpPipelinedResponse implements Comparable<HttpPipelinedResponse>, ReferenceCounted {
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/pipelining/HttpPipeliningHandler.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/pipelining/HttpPipeliningHandler.java
index b96b7f5b32276..a90027c81482b 100644
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/pipelining/HttpPipeliningHandler.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/pipelining/HttpPipeliningHandler.java
@@ -1,4 +1,3 @@
-
 /*
  * Licensed to Elasticsearch under one or more contributor
  * license agreements. See the NOTICE file distributed with
@@ -24,44 +23,47 @@
 import io.netty.channel.ChannelHandlerContext;
 import io.netty.channel.ChannelPromise;
 import io.netty.handler.codec.http.LastHttpContent;
-import io.netty.util.ReferenceCountUtil;
-import org.elasticsearch.action.termvectors.TermVectorsFilter;
-import org.elasticsearch.common.SuppressForbidden;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.transport.netty4.Netty4Utils;
 
+import java.nio.channels.ClosedChannelException;
 import java.util.Collections;
 import java.util.PriorityQueue;
-import java.util.Queue;
 
 /**
- * Implements HTTP pipelining ordering, ensuring that responses are completely served in the same order as their
- * corresponding requests. NOTE: A side effect of using this handler is that upstream HttpRequest objects will
- * cause the original message event to be effectively transformed into an OrderedUpstreamMessageEvent. Conversely
- * OrderedDownstreamChannelEvent objects are expected to be received for the correlating response objects.
+ * Implements HTTP pipelining ordering, ensuring that responses are completely served in the same order as their corresponding requests.
  */
 public class HttpPipeliningHandler extends ChannelDuplexHandler {
 
-    private static final int INITIAL_EVENTS_HELD = 3;
+    // we use a priority queue so that responses are ordered by their sequence number
+    private final PriorityQueue<HttpPipelinedResponse> holdingQueue;
 
+    private final Logger logger;
     private final int maxEventsHeld;
 
+    /*
+     * The current read and write sequence numbers. Read sequence numbers are attached to requests in the order they are read from the
+     * channel, and then transferred to responses. A response is not written to the channel context until its sequence number matches the
+     * current write sequence, implying that all preceding messages have been written.
+     */
     private int readSequence;
     private int writeSequence;
 
-    private final Queue<HttpPipelinedResponse> holdingQueue;
-
     /**
-     * @param maxEventsHeld the maximum number of channel events that will be retained prior to aborting the channel
-     *                      connection. This is required as events cannot queue up indefinitely; we would run out of
-     *                      memory if this was the case.
+     * Construct a new pipelining handler; this handler should be used downstream of HTTP decoding/aggregation.
+     *
+     * @param logger for logging unexpected errors
+     * @param maxEventsHeld the maximum number of channel events that will be retained prior to aborting the channel connection; this is
+     *                      required as events cannot queue up indefinitely
      */
-    public HttpPipeliningHandler(final int maxEventsHeld) {
+    public HttpPipeliningHandler(Logger logger, final int maxEventsHeld) {
+        this.logger = logger;
         this.maxEventsHeld = maxEventsHeld;
-        this.holdingQueue = new PriorityQueue<>(INITIAL_EVENTS_HELD);
+        this.holdingQueue = new PriorityQueue<>(1);
     }
 
     @Override
-    public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
+    public void channelRead(final ChannelHandlerContext ctx, final Object msg) throws Exception {
         if (msg instanceof LastHttpContent) {
             ctx.fireChannelRead(new HttpPipelinedRequest(((LastHttpContent) msg).retain(), readSequence++));
         } else {
@@ -70,21 +72,39 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception
     }
 
     @Override
-    public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) throws Exception {
+    public void write(final ChannelHandlerContext ctx, final Object msg, final ChannelPromise promise) throws Exception {
         if (msg instanceof HttpPipelinedResponse) {
+            final HttpPipelinedResponse current = (HttpPipelinedResponse) msg;
+            /*
+             * We attach the promise to the response. When we invoke a write on the channel with the response, we must ensure that we invoke
+             * the write methods that accept the same promise that we have attached to the response otherwise as the response proceeds
+             * through the handler pipeline a different promise will be used until reaching this handler. Therefore, we assert here that the
+             * attached promise is identical to the provided promise as a safety mechanism that we are respecting this.
+             */
+            assert current.promise() == promise;
+
             boolean channelShouldClose = false;
 
             synchronized (holdingQueue) {
                 if (holdingQueue.size() < maxEventsHeld) {
-                    holdingQueue.add((HttpPipelinedResponse) msg);
+                    holdingQueue.add(current);
 
                     while (!holdingQueue.isEmpty()) {
-                        final HttpPipelinedResponse response = holdingQueue.peek();
-                        if (response.sequence() != writeSequence) {
+                        /*
+                         * Since the response with the lowest sequence number is the top of the priority queue, we know if its sequence
+                         * number does not match the current write sequence number then we have not processed all preceding responses yet.
+                         */
+                        final HttpPipelinedResponse top = holdingQueue.peek();
+                        if (top.sequence() != writeSequence) {
                             break;
                         }
                         holdingQueue.remove();
-                        ctx.write(response.response(), response.promise());
+                        /*
+                         * We must use the promise attached to the response; this is necessary since are going to hold a response until all
+                         * responses that precede it in the pipeline are written first. Note that the promise from the method invocation is
+                         * not ignored, it will already be attached to an existing response and consumed when that response is drained.
+                         */
+                        ctx.write(top.response(), top.promise());
                         writeSequence++;
                     }
                 } else {
@@ -96,7 +116,7 @@ public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise)
                 try {
                     Netty4Utils.closeChannels(Collections.singletonList(ctx.channel()));
                 } finally {
-                    ((HttpPipelinedResponse) msg).release();
+                    current.release();
                     promise.setSuccess();
                 }
             }
@@ -105,4 +125,20 @@ public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise)
         }
     }
 
+    @Override
+    public void close(ChannelHandlerContext ctx, ChannelPromise promise) throws Exception {
+        if (holdingQueue.isEmpty() == false) {
+            ClosedChannelException closedChannelException = new ClosedChannelException();
+            HttpPipelinedResponse pipelinedResponse;
+            while ((pipelinedResponse = holdingQueue.poll()) != null) {
+                try {
+                    pipelinedResponse.release();
+                    pipelinedResponse.promise().setFailure(closedChannelException);
+                } catch (Exception e) {
+                    logger.error("unexpected error while releasing pipelined http responses", e);
+                }
+            }
+        }
+        ctx.close(promise);
+    }
 }
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/Netty4Plugin.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/Netty4Plugin.java
index 8f449b95ecd4c..2ee603be11142 100644
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/Netty4Plugin.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/Netty4Plugin.java
@@ -19,17 +19,33 @@
 
 package org.elasticsearch.transport;
 
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.network.NetworkModule;
+import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.http.netty4.Netty4HttpServerTransport;
+import org.elasticsearch.indices.breaker.CircuitBreakerService;
+import org.elasticsearch.plugins.NetworkPlugin;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.netty4.Netty4Transport;
+import org.elasticsearch.transport.netty4.Netty4Utils;
 
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
+import java.util.Map;
+import java.util.function.Supplier;
 
-public class Netty4Plugin extends Plugin {
+public class Netty4Plugin extends Plugin implements NetworkPlugin {
+
+    static {
+        Netty4Utils.setup();
+    }
 
     public static final String NETTY_TRANSPORT_NAME = "netty4";
     public static final String NETTY_HTTP_TRANSPORT_NAME = "netty4";
@@ -46,6 +62,9 @@ public List<Setting<?>> getSettings() {
             Netty4HttpServerTransport.SETTING_HTTP_TCP_REUSE_ADDRESS,
             Netty4HttpServerTransport.SETTING_HTTP_TCP_SEND_BUFFER_SIZE,
             Netty4HttpServerTransport.SETTING_HTTP_TCP_RECEIVE_BUFFER_SIZE,
+            Netty4HttpServerTransport.SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_SIZE,
+            Netty4HttpServerTransport.SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_MIN,
+            Netty4HttpServerTransport.SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_MAX,
             Netty4Transport.WORKER_COUNT,
             Netty4Transport.NETTY_MAX_CUMULATION_BUFFER_CAPACITY,
             Netty4Transport.NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS,
@@ -66,11 +85,23 @@ public Settings additionalSettings() {
                 .build();
     }
 
-    public void onModule(NetworkModule networkModule) {
-        if (networkModule.canRegisterHttpExtensions()) {
-            networkModule.registerHttpTransport(NETTY_HTTP_TRANSPORT_NAME, Netty4HttpServerTransport.class);
-        }
-        networkModule.registerTransport(NETTY_TRANSPORT_NAME, Netty4Transport.class);
+    @Override
+    public Map<String, Supplier<Transport>> getTransports(Settings settings, ThreadPool threadPool, BigArrays bigArrays,
+                                                          CircuitBreakerService circuitBreakerService,
+                                                          NamedWriteableRegistry namedWriteableRegistry,
+                                                          NetworkService networkService) {
+        return Collections.singletonMap(NETTY_TRANSPORT_NAME, () -> new Netty4Transport(settings, threadPool, networkService, bigArrays,
+            namedWriteableRegistry, circuitBreakerService));
     }
 
+    @Override
+    public Map<String, Supplier<HttpServerTransport>> getHttpTransports(Settings settings, ThreadPool threadPool, BigArrays bigArrays,
+                                                                        CircuitBreakerService circuitBreakerService,
+                                                                        NamedWriteableRegistry namedWriteableRegistry,
+                                                                        NamedXContentRegistry xContentRegistry,
+                                                                        NetworkService networkService,
+                                                                        HttpServerTransport.Dispatcher dispatcher) {
+        return Collections.singletonMap(NETTY_HTTP_TRANSPORT_NAME,
+            () -> new Netty4HttpServerTransport(settings, networkService, bigArrays, threadPool, xContentRegistry, dispatcher));
+    }
 }
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/ByteBufStreamInput.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/ByteBufStreamInput.java
index 2219ce31ff62d..f29ca9c10ff2d 100644
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/ByteBufStreamInput.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/ByteBufStreamInput.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 
+import java.io.EOFException;
 import java.io.IOException;
 
 /**
@@ -35,7 +36,7 @@ class ByteBufStreamInput extends StreamInput {
     private final int startIndex;
     private final int endIndex;
 
-    public ByteBufStreamInput(ByteBuf buffer, int length) {
+    ByteBufStreamInput(ByteBuf buffer, int length) {
         if (length > buffer.readableBytes()) {
             throw new IndexOutOfBoundsException();
         }
@@ -67,6 +68,14 @@ public int available() throws IOException {
         return endIndex - buffer.readerIndex();
     }
 
+    @Override
+    protected void ensureCanReadBytes(int length) throws EOFException {
+        int bytesAvailable = endIndex - buffer.readerIndex();
+        if (bytesAvailable < length) {
+            throw new EOFException("tried to read: " + length + " bytes but only " + bytesAvailable + " remaining");
+        }
+    }
+
     @Override
     public void mark(int readlimit) {
         buffer.markReaderIndex();
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/ESLoggingHandler.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/ESLoggingHandler.java
new file mode 100644
index 0000000000000..47a31f268a6a8
--- /dev/null
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/ESLoggingHandler.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.transport.netty4;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.logging.LogLevel;
+import io.netty.handler.logging.LoggingHandler;
+import io.netty.util.internal.StringUtil;
+import org.elasticsearch.Version;
+import org.elasticsearch.common.compress.Compressor;
+import org.elasticsearch.common.compress.CompressorFactory;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.transport.TcpHeader;
+import org.elasticsearch.transport.TcpTransport;
+import org.elasticsearch.transport.TransportStatus;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+
+final class ESLoggingHandler extends LoggingHandler {
+
+    ESLoggingHandler() {
+        super(LogLevel.TRACE);
+    }
+
+    @Override
+    protected String format(final ChannelHandlerContext ctx, final String eventName, final Object arg) {
+        if (arg instanceof ByteBuf) {
+            try {
+                return format(ctx, eventName, (ByteBuf) arg);
+            } catch (final Exception e) {
+                // we really do not want to allow a bug in the formatting handling to escape
+                logger.trace("an exception occurred formatting a trace message", e);
+                // we are going to let this be formatted via the default formatting
+                return super.format(ctx, eventName, arg);
+            }
+        } else {
+            return super.format(ctx, eventName, arg);
+        }
+    }
+
+    private static final int MESSAGE_LENGTH_OFFSET = TcpHeader.MARKER_BYTES_SIZE;
+    private static final int REQUEST_ID_OFFSET = MESSAGE_LENGTH_OFFSET + TcpHeader.MESSAGE_LENGTH_SIZE;
+    private static final int STATUS_OFFSET = REQUEST_ID_OFFSET + TcpHeader.REQUEST_ID_SIZE;
+    private static final int VERSION_ID_OFFSET = STATUS_OFFSET + TcpHeader.STATUS_SIZE;
+    private static final int ACTION_OFFSET = VERSION_ID_OFFSET + TcpHeader.VERSION_ID_SIZE;
+
+    private String format(final ChannelHandlerContext ctx, final String eventName, final ByteBuf arg) throws IOException {
+        final int readableBytes = arg.readableBytes();
+        if (readableBytes == 0) {
+            return super.format(ctx, eventName, arg);
+        } else if (readableBytes >= 2) {
+            final StringBuilder sb = new StringBuilder();
+            sb.append(ctx.channel().toString());
+            final int offset = arg.readerIndex();
+            // this might be an ES message, check the header
+            if (arg.getByte(offset) == (byte) 'E' && arg.getByte(offset + 1) == (byte) 'S') {
+                if (readableBytes == TcpHeader.MARKER_BYTES_SIZE + TcpHeader.MESSAGE_LENGTH_SIZE) {
+                    final int length = arg.getInt(offset + MESSAGE_LENGTH_OFFSET);
+                    if (length == TcpTransport.PING_DATA_SIZE) {
+                        sb.append(" [ping]").append(' ').append(eventName).append(": ").append(readableBytes).append('B');
+                        return sb.toString();
+                    }
+                }
+                else if (readableBytes >= TcpHeader.HEADER_SIZE) {
+                    // we are going to try to decode this as an ES message
+                    final int length = arg.getInt(offset + MESSAGE_LENGTH_OFFSET);
+                    final long requestId = arg.getLong(offset + REQUEST_ID_OFFSET);
+                    final byte status = arg.getByte(offset + STATUS_OFFSET);
+                    final boolean isRequest = TransportStatus.isRequest(status);
+                    final String type = isRequest ? "request" : "response";
+                    final String version = Version.fromId(arg.getInt(offset + VERSION_ID_OFFSET)).toString();
+                    sb.append(" [length: ").append(length);
+                    sb.append(", request id: ").append(requestId);
+                    sb.append(", type: ").append(type);
+                    sb.append(", version: ").append(version);
+                    if (isRequest) {
+                        // it looks like an ES request, try to decode the action
+                        final int remaining = readableBytes - ACTION_OFFSET;
+                        final ByteBuf slice = arg.slice(offset + ACTION_OFFSET, remaining);
+                        // the stream might be compressed
+                        try (StreamInput in = in(status, slice, remaining)) {
+                            // the first bytes in the message is the context headers
+                            try (ThreadContext context = new ThreadContext(Settings.EMPTY)) {
+                                context.readHeaders(in);
+                            }
+                            // now we can decode the action name
+                            sb.append(", action: ").append(in.readString());
+                        }
+                    }
+                    sb.append(']');
+                    sb.append(' ').append(eventName).append(": ").append(readableBytes).append('B');
+                    return sb.toString();
+                }
+            }
+        }
+        // we could not decode this as an ES message, use the default formatting
+        return super.format(ctx, eventName, arg);
+    }
+
+    private StreamInput in(final Byte status, final ByteBuf slice, final int remaining) throws IOException {
+        final ByteBufStreamInput in = new ByteBufStreamInput(slice, remaining);
+        if (TransportStatus.isCompress(status)) {
+            final Compressor compressor = CompressorFactory.compressor(Netty4Utils.toBytesReference(slice));
+            return compressor.streamInput(in);
+        } else {
+            return in;
+        }
+    }
+
+}
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4InternalESLogger.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4InternalESLogger.java
index aaa277e34b35f..91bbe1c1a9b3c 100644
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4InternalESLogger.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4InternalESLogger.java
@@ -161,7 +161,7 @@ public boolean isErrorEnabled() {
 
     @Override
     public void error(String msg) {
-        logger.equals(msg);
+        logger.error(msg);
     }
 
     @Override
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4MessageChannelHandler.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4MessageChannelHandler.java
index cddbdac3e761b..e83cfc62fda08 100644
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4MessageChannelHandler.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4MessageChannelHandler.java
@@ -23,9 +23,9 @@
 import io.netty.channel.ChannelDuplexHandler;
 import io.netty.channel.ChannelHandlerContext;
 import io.netty.channel.ChannelPromise;
-import io.netty.util.ReferenceCountUtil;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.transport.TcpHeader;
+import org.elasticsearch.transport.TcpTransport;
 import org.elasticsearch.transport.TransportServiceAdapter;
 import org.elasticsearch.transport.Transports;
 
@@ -66,10 +66,10 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception
         final ByteBuf buffer = (ByteBuf) msg;
         final int remainingMessageSize = buffer.getInt(buffer.readerIndex() - TcpHeader.MESSAGE_LENGTH_SIZE);
         final int expectedReaderIndex = buffer.readerIndex() + remainingMessageSize;
-        InetSocketAddress remoteAddress = (InetSocketAddress) ctx.channel().remoteAddress();
         try {
+            InetSocketAddress remoteAddress = (InetSocketAddress) ctx.channel().remoteAddress();
             // netty always copies a buffer, either in NioWorker in its read handler, where it copies to a fresh
-            // buffer, or in the cumulation buffer, which is cleaned each time so it could be bigger than the actual size
+            // buffer, or in the cumulative buffer, which is cleaned each time so it could be bigger than the actual size
             BytesReference reference = Netty4Utils.toBytesReference(buffer, remainingMessageSize);
             transport.messageReceived(reference, ctx.channel(), profileName, remoteAddress, remainingMessageSize);
         } finally {
@@ -80,6 +80,7 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception
 
     @Override
     public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
+        Netty4Utils.maybeDie(cause);
         transport.exceptionCaught(ctx, cause);
     }
 
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java
index b06e5bb48f72b..85b49ea022e15 100644
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java
@@ -42,11 +42,11 @@
 import org.apache.logging.log4j.util.Supplier;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.network.NetworkService;
@@ -55,15 +55,18 @@
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.FutureUtils;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
-import org.elasticsearch.monitor.jvm.JvmInfo;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.ConnectTransportException;
+import org.elasticsearch.transport.ConnectionProfile;
 import org.elasticsearch.transport.TcpTransport;
+import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportServiceAdapter;
 import org.elasticsearch.transport.TransportSettings;
 
@@ -76,6 +79,7 @@
 import java.util.Map;
 import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
 
 import static org.elasticsearch.common.settings.Setting.byteSizeSetting;
 import static org.elasticsearch.common.settings.Setting.intSetting;
@@ -108,20 +112,8 @@ public class Netty4Transport extends TcpTransport<Channel> {
     public static final Setting<Integer> NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS =
         Setting.intSetting("transport.netty.max_composite_buffer_components", -1, -1, Property.NodeScope, Property.Shared);
 
-    // See AdaptiveReceiveBufferSizePredictor#DEFAULT_XXX for default values in netty..., we can use higher ones for us, even fixed one
     public static final Setting<ByteSizeValue> NETTY_RECEIVE_PREDICTOR_SIZE = Setting.byteSizeSetting(
-            "transport.netty.receive_predictor_size",
-            settings -> {
-                long defaultReceiverPredictor = 512 * 1024;
-                if (JvmInfo.jvmInfo().getMem().getDirectMemoryMax().bytes() > 0) {
-                    // we can guess a better default...
-                    long l = (long) ((0.3 * JvmInfo.jvmInfo().getMem().getDirectMemoryMax().bytes()) / WORKER_COUNT.get(settings));
-                    defaultReceiverPredictor = Math.min(defaultReceiverPredictor, Math.max(l, 64 * 1024));
-                }
-                return new ByteSizeValue(defaultReceiverPredictor).toString();
-            },
-            Property.NodeScope,
-            Property.Shared);
+            "transport.netty.receive_predictor_size", new ByteSizeValue(64, ByteSizeUnit.KB), Property.NodeScope, Property.Shared);
     public static final Setting<ByteSizeValue> NETTY_RECEIVE_PREDICTOR_MIN =
         byteSizeSetting("transport.netty.receive_predictor_min", NETTY_RECEIVE_PREDICTOR_SIZE, Property.NodeScope, Property.Shared);
     public static final Setting<ByteSizeValue> NETTY_RECEIVE_PREDICTOR_MAX =
@@ -141,10 +133,10 @@ public class Netty4Transport extends TcpTransport<Channel> {
     protected volatile Bootstrap bootstrap;
     protected final Map<String, ServerBootstrap> serverBootstraps = newConcurrentMap();
 
-    @Inject
     public Netty4Transport(Settings settings, ThreadPool threadPool, NetworkService networkService, BigArrays bigArrays,
                           NamedWriteableRegistry namedWriteableRegistry, CircuitBreakerService circuitBreakerService) {
         super("netty", settings, threadPool, bigArrays, circuitBreakerService, namedWriteableRegistry, networkService);
+        Netty4Utils.setAvailableProcessors(EsExecutors.PROCESSORS_SETTING.get(settings));
         this.workerCount = WORKER_COUNT.get(settings);
         this.maxCumulationBufferCapacity = NETTY_MAX_CUMULATION_BUFFER_CAPACITY.get(settings);
         this.maxCompositeBufferComponents = NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS.get(settings);
@@ -152,11 +144,11 @@ public Netty4Transport(Settings settings, ThreadPool threadPool, NetworkService
         // See AdaptiveReceiveBufferSizePredictor#DEFAULT_XXX for default values in netty..., we can use higher ones for us, even fixed one
         this.receivePredictorMin = NETTY_RECEIVE_PREDICTOR_MIN.get(settings);
         this.receivePredictorMax = NETTY_RECEIVE_PREDICTOR_MAX.get(settings);
-        if (receivePredictorMax.bytes() == receivePredictorMin.bytes()) {
-            recvByteBufAllocator = new FixedRecvByteBufAllocator((int) receivePredictorMax.bytes());
+        if (receivePredictorMax.getBytes() == receivePredictorMin.getBytes()) {
+            recvByteBufAllocator = new FixedRecvByteBufAllocator((int) receivePredictorMax.getBytes());
         } else {
-            recvByteBufAllocator = new AdaptiveRecvByteBufAllocator((int) receivePredictorMin.bytes(),
-                (int) receivePredictorMin.bytes(), (int) receivePredictorMax.bytes());
+            recvByteBufAllocator = new AdaptiveRecvByteBufAllocator((int) receivePredictorMin.getBytes(),
+                (int) receivePredictorMin.getBytes(), (int) receivePredictorMax.getBytes());
         }
     }
 
@@ -203,18 +195,18 @@ private Bootstrap createBootstrap() {
 
         bootstrap.handler(getClientChannelInitializer());
 
-        bootstrap.option(ChannelOption.CONNECT_TIMEOUT_MILLIS, Math.toIntExact(connectTimeout.millis()));
+        bootstrap.option(ChannelOption.CONNECT_TIMEOUT_MILLIS, Math.toIntExact(defaultConnectionProfile.getConnectTimeout().millis()));
         bootstrap.option(ChannelOption.TCP_NODELAY, TCP_NO_DELAY.get(settings));
         bootstrap.option(ChannelOption.SO_KEEPALIVE, TCP_KEEP_ALIVE.get(settings));
 
         final ByteSizeValue tcpSendBufferSize = TCP_SEND_BUFFER_SIZE.get(settings);
-        if (tcpSendBufferSize.bytes() > 0) {
-            bootstrap.option(ChannelOption.SO_SNDBUF, Math.toIntExact(tcpSendBufferSize.bytes()));
+        if (tcpSendBufferSize.getBytes() > 0) {
+            bootstrap.option(ChannelOption.SO_SNDBUF, Math.toIntExact(tcpSendBufferSize.getBytes()));
         }
 
         final ByteSizeValue tcpReceiveBufferSize = TCP_RECEIVE_BUFFER_SIZE.get(settings);
-        if (tcpReceiveBufferSize.bytes() > 0) {
-            bootstrap.option(ChannelOption.SO_RCVBUF, Math.toIntExact(tcpReceiveBufferSize.bytes()));
+        if (tcpReceiveBufferSize.getBytes() > 0) {
+            bootstrap.option(ChannelOption.SO_RCVBUF, Math.toIntExact(tcpReceiveBufferSize.getBytes()));
         }
 
         bootstrap.option(ChannelOption.RCVBUF_ALLOCATOR, recvByteBufAllocator);
@@ -251,13 +243,13 @@ private Settings createFallbackSettings() {
 
         ByteSizeValue fallbackTcpSendBufferSize = settings.getAsBytesSize("transport.netty.tcp_send_buffer_size",
             TCP_SEND_BUFFER_SIZE.get(settings));
-        if (fallbackTcpSendBufferSize.bytes() >= 0) {
+        if (fallbackTcpSendBufferSize.getBytes() >= 0) {
             fallbackSettingsBuilder.put("tcp_send_buffer_size", fallbackTcpSendBufferSize);
         }
 
         ByteSizeValue fallbackTcpBufferSize = settings.getAsBytesSize("transport.netty.tcp_receive_buffer_size",
             TCP_RECEIVE_BUFFER_SIZE.get(settings));
-        if (fallbackTcpBufferSize.bytes() >= 0) {
+        if (fallbackTcpBufferSize.getBytes() >= 0) {
             fallbackSettingsBuilder.put("tcp_receive_buffer_size", fallbackTcpBufferSize);
         }
 
@@ -269,8 +261,13 @@ private void createServerBootstrap(String name, Settings settings) {
             logger.debug("using profile[{}], worker_count[{}], port[{}], bind_host[{}], publish_host[{}], compress[{}], "
                     + "connect_timeout[{}], connections_per_node[{}/{}/{}/{}/{}], receive_predictor[{}->{}]",
                 name, workerCount, settings.get("port"), settings.get("bind_host"), settings.get("publish_host"), compress,
-                connectTimeout, connectionsPerNodeRecovery, connectionsPerNodeBulk, connectionsPerNodeReg, connectionsPerNodeState,
-                connectionsPerNodePing, receivePredictorMin, receivePredictorMax);
+                defaultConnectionProfile.getConnectTimeout(),
+                defaultConnectionProfile.getNumConnectionsPerType(TransportRequestOptions.Type.RECOVERY),
+                defaultConnectionProfile.getNumConnectionsPerType(TransportRequestOptions.Type.BULK),
+                defaultConnectionProfile.getNumConnectionsPerType(TransportRequestOptions.Type.REG),
+                defaultConnectionProfile.getNumConnectionsPerType(TransportRequestOptions.Type.STATE),
+                defaultConnectionProfile.getNumConnectionsPerType(TransportRequestOptions.Type.PING),
+                receivePredictorMin, receivePredictorMax);
         }
 
         final ThreadFactory workerFactory = daemonThreadFactory(this.settings, TRANSPORT_SERVER_WORKER_THREAD_NAME_PREFIX, name);
@@ -291,12 +288,12 @@ private void createServerBootstrap(String name, Settings settings) {
         serverBootstrap.childOption(ChannelOption.SO_KEEPALIVE, TCP_KEEP_ALIVE.get(settings));
 
         final ByteSizeValue tcpSendBufferSize = TCP_SEND_BUFFER_SIZE.getDefault(settings);
-        if (tcpSendBufferSize != null && tcpSendBufferSize.bytes() > 0) {
-            serverBootstrap.childOption(ChannelOption.SO_SNDBUF, Math.toIntExact(tcpSendBufferSize.bytes()));
+        if (tcpSendBufferSize != null && tcpSendBufferSize.getBytes() > 0) {
+            serverBootstrap.childOption(ChannelOption.SO_SNDBUF, Math.toIntExact(tcpSendBufferSize.getBytes()));
         }
 
         final ByteSizeValue tcpReceiveBufferSize = TCP_RECEIVE_BUFFER_SIZE.getDefault(settings);
-        if (tcpReceiveBufferSize != null && tcpReceiveBufferSize.bytes() > 0) {
+        if (tcpReceiveBufferSize != null && tcpReceiveBufferSize.getBytes() > 0) {
             serverBootstrap.childOption(ChannelOption.SO_RCVBUF, Math.toIntExact(tcpReceiveBufferSize.bytesAsInt()));
         }
 
@@ -332,66 +329,42 @@ public long serverOpen() {
         return channels == null ? 0 : channels.numberOfOpenChannels();
     }
 
-    protected NodeChannels connectToChannelsLight(DiscoveryNode node) {
-        InetSocketAddress address = ((InetSocketTransportAddress) node.getAddress()).address();
-        ChannelFuture connect = bootstrap.connect(address);
-        connect.awaitUninterruptibly((long) (connectTimeout.millis() * 1.5));
-        if (!connect.isSuccess()) {
-            throw new ConnectTransportException(node, "connect_timeout[" + connectTimeout + "]", connect.cause());
-        }
-        Channel[] channels = new Channel[1];
-        channels[0] = connect.channel();
-        channels[0].closeFuture().addListener(new ChannelCloseListener(node));
-        NodeChannels nodeChannels = new NodeChannels(channels, channels, channels, channels, channels);
-        onAfterChannelsConnected(nodeChannels);
-        return nodeChannels;
-    }
-
-    protected NodeChannels connectToChannels(DiscoveryNode node) {
-        final NodeChannels nodeChannels =
-            new NodeChannels(
-                new Channel[connectionsPerNodeRecovery],
-                new Channel[connectionsPerNodeBulk],
-                new Channel[connectionsPerNodeReg],
-                new Channel[connectionsPerNodeState],
-                new Channel[connectionsPerNodePing]);
+    @Override
+    protected NodeChannels connectToChannels(DiscoveryNode node, ConnectionProfile profile, Consumer<Channel> onChannelClose) {
+        final Channel[] channels = new Channel[profile.getNumConnections()];
+        final NodeChannels nodeChannels = new NodeChannels(node, channels, profile);
         boolean success = false;
         try {
-            int numConnections =
-                connectionsPerNodeRecovery +
-                    connectionsPerNodeBulk +
-                    connectionsPerNodeReg +
-                    connectionsPerNodeState +
-                    connectionsPerNodeRecovery;
-            final ArrayList<ChannelFuture> connections = new ArrayList<>(numConnections);
+            final TimeValue connectTimeout;
+            final Bootstrap bootstrap;
+            final TimeValue defaultConnectTimeout = defaultConnectionProfile.getConnectTimeout();
+            if (profile.getConnectTimeout() != null && profile.getConnectTimeout().equals(defaultConnectTimeout) == false) {
+                bootstrap = this.bootstrap.clone(this.bootstrap.config().group());
+                bootstrap.option(ChannelOption.CONNECT_TIMEOUT_MILLIS, Math.toIntExact(profile.getConnectTimeout().millis()));
+                connectTimeout = profile.getConnectTimeout();
+            } else {
+                connectTimeout = defaultConnectTimeout;
+                bootstrap = this.bootstrap;
+            }
+            final ArrayList<ChannelFuture> connections = new ArrayList<>(channels.length);
             final InetSocketAddress address = ((InetSocketTransportAddress) node.getAddress()).address();
-            for (int i = 0; i < numConnections; i++) {
+            for (int i = 0; i < channels.length; i++) {
                 connections.add(bootstrap.connect(address));
             }
             final Iterator<ChannelFuture> iterator = connections.iterator();
+            final ChannelFutureListener closeListener = future -> onChannelClose.accept(future.channel());
             try {
-                for (Channel[] channels : nodeChannels.getChannelArrays()) {
-                    for (int i = 0; i < channels.length; i++) {
-                        assert iterator.hasNext();
-                        ChannelFuture future = iterator.next();
-                        future.awaitUninterruptibly((long) (connectTimeout.millis() * 1.5));
-                        if (!future.isSuccess()) {
-                            throw new ConnectTransportException(node, "connect_timeout[" + connectTimeout + "]", future.cause());
-                        }
-                        channels[i] = future.channel();
-                        channels[i].closeFuture().addListener(new ChannelCloseListener(node));
-                    }
-                }
-                if (nodeChannels.recovery.length == 0) {
-                    if (nodeChannels.bulk.length > 0) {
-                        nodeChannels.recovery = nodeChannels.bulk;
-                    } else {
-                        nodeChannels.recovery = nodeChannels.reg;
+                for (int i = 0; i < channels.length; i++) {
+                    assert iterator.hasNext();
+                    ChannelFuture future = iterator.next();
+                    future.awaitUninterruptibly((long) (connectTimeout.millis() * 1.5));
+                    if (!future.isSuccess()) {
+                        throw new ConnectTransportException(node, "connect_timeout[" + connectTimeout + "]", future.cause());
                     }
+                    channels[i] = future.channel();
+                    channels[i].closeFuture().addListener(closeListener);
                 }
-                if (nodeChannels.bulk.length == 0) {
-                    nodeChannels.bulk = nodeChannels.reg;
-                }
+                assert iterator.hasNext() == false : "not all created connection have been consumed";
             } catch (final RuntimeException e) {
                 for (final ChannelFuture future : Collections.unmodifiableList(connections)) {
                     FutureUtils.cancel(future);
@@ -405,7 +378,6 @@ protected NodeChannels connectToChannels(DiscoveryNode node) {
                 }
                 throw e;
             }
-            onAfterChannelsConnected(nodeChannels);
             success = true;
         } finally {
             if (success == false) {
@@ -419,52 +391,41 @@ protected NodeChannels connectToChannels(DiscoveryNode node) {
         return nodeChannels;
     }
 
-    /**
-     * Allows for logic to be executed after a connection has been made on all channels. While this method is being executed, the node is
-     * not listed as being connected to.
-     * @param nodeChannels the {@link NodeChannels} that have been connected
-     */
-    protected void onAfterChannelsConnected(NodeChannels nodeChannels) {
-    }
-
-    private class ChannelCloseListener implements ChannelFutureListener {
-
-        private final DiscoveryNode node;
-
-        private ChannelCloseListener(DiscoveryNode node) {
-            this.node = node;
-        }
-
-        @Override
-        public void operationComplete(final ChannelFuture future) throws Exception {
-            NodeChannels nodeChannels = connectedNodes.get(node);
-            if (nodeChannels != null && nodeChannels.hasChannel(future.channel())) {
-                threadPool.generic().execute(() -> disconnectFromNode(node, future.channel(), "channel closed event"));
+    @Override
+    protected void sendMessage(Channel channel, BytesReference reference, ActionListener<Channel> listener) {
+        final ChannelFuture future = channel.writeAndFlush(Netty4Utils.toByteBuf(reference));
+        future.addListener(f -> {
+            if (f.isSuccess()) {
+                listener.onResponse(channel);
+            } else {
+                final Throwable cause = f.cause();
+                Netty4Utils.maybeDie(cause);
+                logger.warn((Supplier<?>) () ->
+                    new ParameterizedMessage("write and flush on the network layer failed (channel: {})", channel), cause);
+                assert cause instanceof Exception;
+                listener.onFailure((Exception) cause);
             }
-        }
+        });
     }
 
     @Override
-    protected void sendMessage(Channel channel, BytesReference reference, Runnable sendListener, boolean close) {
-        final ChannelFuture future = channel.writeAndFlush(Netty4Utils.toByteBuf(reference));
-        if (close) {
-            future.addListener(f -> {
-                try {
-                    sendListener.run();
-                } finally {
-                    future.channel().close();
-                }
-            });
+    protected void closeChannels(final List<Channel> channels, boolean blocking) throws IOException {
+        if (blocking) {
+            Netty4Utils.closeChannels(channels);
         } else {
-            future.addListener(f -> sendListener.run());
+            for (Channel channel : channels) {
+                if (channel != null && channel.isOpen()) {
+                    ChannelFuture closeFuture = channel.close();
+                    closeFuture.addListener((f) -> {
+                        if (f.isSuccess() == false) {
+                            logger.warn("failed to close channel", f.cause());
+                        }
+                    });
+                }
+            }
         }
     }
 
-    @Override
-    protected void closeChannels(final List<Channel> channels) throws IOException {
-        Netty4Utils.closeChannels(channels);
-    }
-
     @Override
     protected InetSocketAddress getLocalAddress(Channel channel) {
         return (InetSocketAddress) channel.localAddress();
@@ -514,11 +475,18 @@ protected class ClientChannelInitializer extends ChannelInitializer<Channel> {
 
         @Override
         protected void initChannel(Channel ch) throws Exception {
+            ch.pipeline().addLast("logging", new ESLoggingHandler());
             ch.pipeline().addLast("size", new Netty4SizeHeaderFrameDecoder());
             // using a dot as a prefix means this cannot come from any settings parsed
             ch.pipeline().addLast("dispatcher", new Netty4MessageChannelHandler(Netty4Transport.this, ".client"));
         }
 
+        @Override
+        public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
+            Netty4Utils.maybeDie(cause);
+            super.exceptionCaught(ctx, cause);
+        }
+
     }
 
     protected class ServerChannelInitializer extends ChannelInitializer<Channel> {
@@ -533,10 +501,18 @@ protected ServerChannelInitializer(String name, Settings settings) {
 
         @Override
         protected void initChannel(Channel ch) throws Exception {
+            ch.pipeline().addLast("logging", new ESLoggingHandler());
             ch.pipeline().addLast("open_channels", Netty4Transport.this.serverOpenChannels);
             ch.pipeline().addLast("size", new Netty4SizeHeaderFrameDecoder());
             ch.pipeline().addLast("dispatcher", new Netty4MessageChannelHandler(Netty4Transport.this, name));
         }
+
+        @Override
+        public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
+            Netty4Utils.maybeDie(cause);
+            super.exceptionCaught(ctx, cause);
+        }
+
     }
 
 }
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Utils.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Utils.java
index 53cf1b329aa06..bb67a15e8a160 100644
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Utils.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Utils.java
@@ -16,6 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+
 package org.elasticsearch.transport.netty4;
 
 import io.netty.buffer.ByteBuf;
@@ -23,16 +24,28 @@
 import io.netty.buffer.Unpooled;
 import io.netty.channel.Channel;
 import io.netty.channel.ChannelFuture;
+import io.netty.util.NettyRuntime;
 import io.netty.util.internal.logging.InternalLogger;
 import io.netty.util.internal.logging.InternalLoggerFactory;
+import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefIterator;
+import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.logging.ESLoggerFactory;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedList;
 import java.util.List;
+import java.util.Locale;
+import java.util.Optional;
+import java.util.Queue;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
 
 public class Netty4Utils {
 
@@ -41,7 +54,7 @@ public class Netty4Utils {
 
             @Override
             public InternalLogger newInstance(final String name) {
-                return new Netty4InternalESLogger(name.replace("io.netty.", "netty."));
+                return new Netty4InternalESLogger(name);
             }
 
         });
@@ -51,6 +64,41 @@ public static void setup() {
 
     }
 
+    private static AtomicBoolean isAvailableProcessorsSet = new AtomicBoolean();
+
+    /**
+     * Set the number of available processors that Netty uses for sizing various resources (e.g., thread pools).
+     *
+     * @param availableProcessors the number of available processors
+     * @throws IllegalStateException if available processors was set previously and the specified value does not match the already-set value
+     */
+    public static void setAvailableProcessors(final int availableProcessors) {
+        // we set this to false in tests to avoid tests that randomly set processors from stepping on each other
+        final boolean set = Booleans.parseBoolean(System.getProperty("es.set.netty.runtime.available.processors"), true);
+        if (!set) {
+            return;
+        }
+
+        /*
+         * This can be invoked twice, once from Netty4Transport and another time from Netty4HttpServerTransport; however,
+         * Netty4Runtime#availableProcessors forbids settings the number of processors twice so we prevent double invocation here.
+         */
+        if (isAvailableProcessorsSet.compareAndSet(false, true)) {
+            NettyRuntime.setAvailableProcessors(availableProcessors);
+        } else if (availableProcessors != NettyRuntime.availableProcessors()) {
+            /*
+             * We have previously set the available processors yet either we are trying to set it to a different value now or there is a bug
+             * in Netty and our previous value did not take, bail.
+             */
+            final String message = String.format(
+                    Locale.ROOT,
+                    "available processors value [%d] did not match current value [%d]",
+                    availableProcessors,
+                    NettyRuntime.availableProcessors());
+            throw new IllegalStateException(message);
+        }
+    }
+
     /**
      * Turns the given BytesReference into a ByteBuf. Note: the returned ByteBuf will reference the internal
      * pages of the BytesReference. Don't free the bytes of reference before the ByteBuf goes out of scope.
@@ -63,8 +111,7 @@ public static ByteBuf toByteBuf(final BytesReference reference) {
             return ((ByteBufBytesReference) reference).toByteBuf();
         } else {
             final BytesRefIterator iterator = reference.iterator();
-            // usually we have one, two, or three components
-            // from the header, the message, and a buffer
+            // usually we have one, two, or three components from the header, the message, and a buffer
             final List<ByteBuf> buffers = new ArrayList<>(3);
             try {
                 BytesRef slice;
@@ -118,4 +165,71 @@ public static void closeChannels(final Collection<Channel> channels) throws IOEx
         }
     }
 
+    /**
+     * If the specified cause is an unrecoverable error, this method will rethrow the cause on a separate thread so that it can not be
+     * caught and bubbles up to the uncaught exception handler.
+     *
+     * @param cause the throwable to test
+     */
+    public static void maybeDie(final Throwable cause) {
+        final Optional<Error> maybeError = maybeError(cause);
+        if (maybeError.isPresent()) {
+            /*
+             * Here be dragons. We want to rethrow this so that it bubbles up to the uncaught exception handler. Yet, Netty wraps too many
+             * invocations of user-code in try/catch blocks that swallow all throwables. This means that a rethrow here will not bubble up
+             * to where we want it to. So, we fork a thread and throw the exception from there where Netty can not get to it. We do not wrap
+             * the exception so as to not lose the original cause during exit.
+             */
+            try {
+                // try to log the current stack trace
+                final StackTraceElement[] stackTrace = Thread.currentThread().getStackTrace();
+                final String formatted = Arrays.stream(stackTrace).skip(1).map(e -> "\tat " + e).collect(Collectors.joining("\n"));
+                final Logger logger = ESLoggerFactory.getLogger(Netty4Utils.class);
+                logger.error("fatal error on the network layer\n{}", formatted);
+            } finally {
+                new Thread(
+                        () -> {
+                            throw maybeError.get();
+                        })
+                        .start();
+            }
+        }
+    }
+
+    static final int MAX_ITERATIONS = 1024;
+
+    /**
+     * Unwrap the specified throwable looking for any suppressed errors or errors as a root cause of the specified throwable.
+     *
+     * @param cause the root throwable
+     *
+     * @return an optional error if one is found suppressed or a root cause in the tree rooted at the specified throwable
+     */
+    static Optional<Error> maybeError(final Throwable cause) {
+        // early terminate if the cause is already an error
+        if (cause instanceof Error) {
+            return Optional.of((Error) cause);
+        }
+
+        final Queue<Throwable> queue = new LinkedList<>();
+        queue.add(cause);
+        int iterations = 0;
+        while (!queue.isEmpty()) {
+            iterations++;
+            if (iterations > MAX_ITERATIONS) {
+                ESLoggerFactory.getLogger(Netty4Utils.class).warn("giving up looking for fatal errors on the network layer", cause);
+                break;
+            }
+            final Throwable current = queue.remove();
+            if (current instanceof Error) {
+                return Optional.of((Error) current);
+            }
+            Collections.addAll(queue, current.getSuppressed());
+            if (current.getCause() != null) {
+                queue.add(current.getCause());
+            }
+        }
+        return Optional.empty();
+    }
+
 }
diff --git a/modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy b/modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy
index a8cd1a7fffd5f..1c8e52be7c283 100644
--- a/modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy
+++ b/modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy
@@ -17,8 +17,13 @@
  * under the License.
  */
 
-grant codeBase "${codebase.netty-transport-4.1.5.Final.jar}" {
+grant codeBase "${codebase.netty-common-4.1.13.Final.jar}" {
+   // for reading the system-wide configuration for the backlog of established sockets
+   permission java.io.FilePermission "/proc/sys/net/core/somaxconn", "read";
+};
+
+grant codeBase "${codebase.netty-transport-4.1.13.Final.jar}" {
    // Netty NioEventLoop wants to change this, because of https://bugs.openjdk.java.net/browse/JDK-6427854
    // the bug says it only happened rarely, and that its fixed, but apparently it still happens rarely!
    permission java.util.PropertyPermission "sun.nio.ch.bugLevel", "write";
-};
\ No newline at end of file
+};
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4ClientYamlTestSuiteIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4ClientYamlTestSuiteIT.java
index 8f7483e279109..d6628c69b695c 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4ClientYamlTestSuiteIT.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4ClientYamlTestSuiteIT.java
@@ -26,9 +26,6 @@
 import org.apache.lucene.util.TimeUnits;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-
-import java.io.IOException;
 
 //TODO: This is a *temporary* workaround to ensure a timeout does not mask other problems
 @TimeoutSuite(millis = 30 * TimeUnits.MINUTE)
@@ -39,8 +36,8 @@ public Netty4ClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCan
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
-        return ESClientYamlSuiteTestCase.createParameters(0, 1);
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
     }
 
 }
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpChannelTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpChannelTests.java
index 1185419d0dd10..7d8101df10ea2 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpChannelTests.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpChannelTests.java
@@ -30,22 +30,37 @@
 import io.netty.channel.ChannelProgressivePromise;
 import io.netty.channel.ChannelPromise;
 import io.netty.channel.EventLoop;
+import io.netty.channel.embedded.EmbeddedChannel;
 import io.netty.handler.codec.http.DefaultFullHttpRequest;
 import io.netty.handler.codec.http.FullHttpRequest;
 import io.netty.handler.codec.http.FullHttpResponse;
 import io.netty.handler.codec.http.HttpHeaderNames;
+import io.netty.handler.codec.http.HttpHeaderValues;
 import io.netty.handler.codec.http.HttpMethod;
 import io.netty.handler.codec.http.HttpResponse;
 import io.netty.handler.codec.http.HttpVersion;
 import io.netty.util.Attribute;
 import io.netty.util.AttributeKey;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.bytes.ReleasablePagedBytesReference;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput;
+import org.elasticsearch.common.lease.Releasable;
+import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.ByteArray;
 import org.elasticsearch.common.util.MockBigArrays;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.http.HttpTransportSettings;
+import org.elasticsearch.http.NullDispatcher;
 import org.elasticsearch.http.netty4.cors.Netty4CorsHandler;
+import org.elasticsearch.http.netty4.pipelining.HttpPipelinedRequest;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
+import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESTestCase;
@@ -55,6 +70,8 @@
 import org.junit.After;
 import org.junit.Before;
 
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
 import java.net.SocketAddress;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
@@ -66,7 +83,9 @@
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_ORIGIN;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ENABLED;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
@@ -185,12 +204,13 @@ public void testThatAnyOriginWorks() {
     public void testHeadersSet() {
         Settings settings = Settings.builder().build();
         try (Netty4HttpServerTransport httpServerTransport =
-                     new Netty4HttpServerTransport(settings, networkService, bigArrays, threadPool)) {
+                     new Netty4HttpServerTransport(settings, networkService, bigArrays, threadPool, xContentRegistry(),
+                         new NullDispatcher())) {
             httpServerTransport.start();
             final FullHttpRequest httpRequest = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/");
             httpRequest.headers().add(HttpHeaderNames.ORIGIN, "remote");
             final WriteCapturingChannel writeCapturingChannel = new WriteCapturingChannel();
-            Netty4HttpRequest request = new Netty4HttpRequest(httpRequest, writeCapturingChannel);
+            Netty4HttpRequest request = new Netty4HttpRequest(xContentRegistry(), httpRequest, writeCapturingChannel);
 
             // send a response
             Netty4HttpChannel channel =
@@ -212,6 +232,87 @@ public void testHeadersSet() {
         }
     }
 
+    public void testReleaseOnSendToClosedChannel() {
+        final Settings settings = Settings.builder().build();
+        final NamedXContentRegistry registry = xContentRegistry();
+        try (Netty4HttpServerTransport httpServerTransport =
+                     new Netty4HttpServerTransport(settings, networkService, bigArrays, threadPool, registry, new NullDispatcher())) {
+            final FullHttpRequest httpRequest = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/");
+            final EmbeddedChannel embeddedChannel = new EmbeddedChannel();
+            final Netty4HttpRequest request = new Netty4HttpRequest(registry, httpRequest, embeddedChannel);
+            final HttpPipelinedRequest pipelinedRequest = randomBoolean() ? new HttpPipelinedRequest(request.request(), 1) : null;
+            final Netty4HttpChannel channel =
+                    new Netty4HttpChannel(httpServerTransport, request, pipelinedRequest, randomBoolean(), threadPool.getThreadContext());
+            final TestResponse response = new TestResponse(bigArrays);
+            assertThat(response.content(), instanceOf(Releasable.class));
+            embeddedChannel.close();
+            channel.sendResponse(response);
+            // ESTestCase#after will invoke ensureAllArraysAreReleased which will fail if the response content was not released
+        }
+    }
+
+    public void testReleaseOnSendToChannelAfterException() throws IOException {
+        final Settings settings = Settings.builder().build();
+        final NamedXContentRegistry registry = xContentRegistry();
+        try (Netty4HttpServerTransport httpServerTransport =
+                 new Netty4HttpServerTransport(settings, networkService, bigArrays, threadPool, registry, new NullDispatcher())) {
+            final FullHttpRequest httpRequest = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/");
+            final EmbeddedChannel embeddedChannel = new EmbeddedChannel();
+            final Netty4HttpRequest request = new Netty4HttpRequest(registry, httpRequest, embeddedChannel);
+            final HttpPipelinedRequest pipelinedRequest = randomBoolean() ? new HttpPipelinedRequest(request.request(), 1) : null;
+            final Netty4HttpChannel channel =
+                new Netty4HttpChannel(httpServerTransport, request, pipelinedRequest, randomBoolean(), threadPool.getThreadContext());
+            final BytesRestResponse response = new BytesRestResponse(RestStatus.INTERNAL_SERVER_ERROR,
+                JsonXContent.contentBuilder().startObject().endObject());
+            assertThat(response.content(), not(instanceOf(Releasable.class)));
+
+            // ensure we have reserved bytes
+            if (randomBoolean()) {
+                BytesStreamOutput out = channel.bytesOutput();
+                assertThat(out, instanceOf(ReleasableBytesStreamOutput.class));
+            } else {
+                try (XContentBuilder builder = channel.newBuilder()) {
+                    // do something builder
+                    builder.startObject().endObject();
+                }
+            }
+
+            channel.sendResponse(response);
+            // ESTestCase#after will invoke ensureAllArraysAreReleased which will fail if the response content was not released
+        }
+    }
+
+    public void testConnectionClose() throws Exception {
+        final Settings settings = Settings.builder().build();
+        try (Netty4HttpServerTransport httpServerTransport =
+                 new Netty4HttpServerTransport(settings, networkService, bigArrays, threadPool, xContentRegistry(), new NullDispatcher())) {
+            httpServerTransport.start();
+            final FullHttpRequest httpRequest;
+            final boolean close = randomBoolean();
+            if (randomBoolean()) {
+                httpRequest = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/");
+                if (close) {
+                    httpRequest.headers().add(HttpHeaderNames.CONNECTION, HttpHeaderValues.CLOSE);
+                }
+            } else {
+                httpRequest = new DefaultFullHttpRequest(HttpVersion.HTTP_1_0, HttpMethod.GET, "/");
+                if (!close) {
+                    httpRequest.headers().add(HttpHeaderNames.CONNECTION, HttpHeaderValues.KEEP_ALIVE);
+                }
+            }
+            final EmbeddedChannel embeddedChannel = new EmbeddedChannel();
+            final Netty4HttpRequest request = new Netty4HttpRequest(xContentRegistry(), httpRequest, embeddedChannel);
+
+            // send a response, the channel close status should match
+            assertTrue(embeddedChannel.isOpen());
+            final Netty4HttpChannel channel =
+                new Netty4HttpChannel(httpServerTransport, request, null, randomBoolean(), threadPool.getThreadContext());
+            final TestResponse resp = new TestResponse();
+            channel.sendResponse(resp);
+            assertThat(embeddedChannel.isOpen(), equalTo(!close));
+        }
+    }
+
     private FullHttpResponse executeRequest(final Settings settings, final String host) {
         return executeRequest(settings, null, host);
     }
@@ -219,7 +320,8 @@ private FullHttpResponse executeRequest(final Settings settings, final String ho
     private FullHttpResponse executeRequest(final Settings settings, final String originValue, final String host) {
         // construct request and send it over the transport layer
         try (Netty4HttpServerTransport httpServerTransport =
-                     new Netty4HttpServerTransport(settings, networkService, bigArrays, threadPool)) {
+                     new Netty4HttpServerTransport(settings, networkService, bigArrays, threadPool, xContentRegistry(),
+                             new NullDispatcher())) {
             httpServerTransport.start();
             final FullHttpRequest httpRequest = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/");
             if (originValue != null) {
@@ -227,7 +329,7 @@ private FullHttpResponse executeRequest(final Settings settings, final String or
             }
             httpRequest.headers().add(HttpHeaderNames.HOST, host);
             final WriteCapturingChannel writeCapturingChannel = new WriteCapturingChannel();
-            final Netty4HttpRequest request = new Netty4HttpRequest(httpRequest, writeCapturingChannel);
+            final Netty4HttpRequest request = new Netty4HttpRequest(xContentRegistry(), httpRequest, writeCapturingChannel);
 
             Netty4HttpChannel channel =
                     new Netty4HttpChannel(httpServerTransport, request, null, randomBoolean(), threadPool.getThreadContext());
@@ -471,6 +573,24 @@ List<Object> getWrittenObjects() {
 
     private static class TestResponse extends RestResponse {
 
+        private final BytesReference reference;
+
+        TestResponse() {
+            reference = Netty4Utils.toBytesReference(Unpooled.copiedBuffer("content", StandardCharsets.UTF_8));
+        }
+
+        TestResponse(final BigArrays bigArrays) {
+            final byte[] bytes;
+            try {
+                bytes = "content".getBytes("UTF-8");
+            } catch (final UnsupportedEncodingException e) {
+                throw new AssertionError(e);
+            }
+            final ByteArray bigArray = bigArrays.newByteArray(bytes.length);
+            bigArray.set(0, bytes, 0, bytes.length);
+            reference = new ReleasablePagedBytesReference(bigArrays, bigArray, bytes.length, Releasables.releaseOnce(bigArray));
+        }
+
         @Override
         public String contentType() {
             return "text";
@@ -478,7 +598,7 @@ public String contentType() {
 
         @Override
         public BytesReference content() {
-            return Netty4Utils.toBytesReference(Unpooled.copiedBuffer("content", StandardCharsets.UTF_8));
+            return reference;
         }
 
         @Override
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpClient.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpClient.java
index f6d3e1fe64f52..7c4471e249102 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpClient.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpClient.java
@@ -16,6 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+
 package org.elasticsearch.http.netty4;
 
 import io.netty.bootstrap.Bootstrap;
@@ -121,6 +122,7 @@ private Collection<FullHttpResponse> processRequestsWithBody(HttpMethod method,
             HttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, method, uriAndBody.v1(), content);
             request.headers().add(HttpHeaderNames.HOST, "localhost");
             request.headers().add(HttpHeaderNames.CONTENT_LENGTH, content.readableBytes());
+            request.headers().add(HttpHeaderNames.CONTENT_TYPE, "application/json");
             requests.add(request);
         }
         return sendRequests(remoteAddress, requests);
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerPipeliningTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerPipeliningTests.java
index 63e35a786c23c..9967a1ae9d86e 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerPipeliningTests.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerPipeliningTests.java
@@ -24,6 +24,7 @@
 import io.netty.channel.Channel;
 import io.netty.channel.ChannelHandler;
 import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelPromise;
 import io.netty.channel.SimpleChannelInboundHandler;
 import io.netty.handler.codec.http.DefaultFullHttpResponse;
 import io.netty.handler.codec.http.FullHttpRequest;
@@ -37,6 +38,7 @@
 import org.elasticsearch.common.util.MockBigArrays;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.http.HttpServerTransport;
+import org.elasticsearch.http.NullDispatcher;
 import org.elasticsearch.http.netty4.pipelining.HttpPipelinedRequest;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.test.ESTestCase;
@@ -87,7 +89,7 @@ public void testThatHttpPipeliningWorksWhenEnabled() throws Exception {
             .put("http.pipelining", true)
             .put("http.port", "0")
             .build();
-        try (final HttpServerTransport httpServerTransport = new CustomNettyHttpServerTransport(settings)) {
+        try (HttpServerTransport httpServerTransport = new CustomNettyHttpServerTransport(settings)) {
             httpServerTransport.start();
             final InetSocketTransportAddress transportAddress =
                 (InetSocketTransportAddress) randomFrom(httpServerTransport.boundAddress().boundAddresses());
@@ -115,7 +117,7 @@ public void testThatHttpPipeliningCanBeDisabled() throws Exception {
             .put("http.pipelining", false)
             .put("http.port", "0")
             .build();
-        try (final HttpServerTransport httpServerTransport = new CustomNettyHttpServerTransport(settings)) {
+        try (HttpServerTransport httpServerTransport = new CustomNettyHttpServerTransport(settings)) {
             httpServerTransport.start();
             final InetSocketTransportAddress transportAddress =
                 (InetSocketTransportAddress) randomFrom(httpServerTransport.boundAddress().boundAddresses());
@@ -123,12 +125,10 @@ public void testThatHttpPipeliningCanBeDisabled() throws Exception {
             final int numberOfRequests = randomIntBetween(4, 16);
             final Set<Integer> slowIds = new HashSet<>();
             final List<String> requests = new ArrayList<>(numberOfRequests);
-            int numberOfSlowRequests = 0;
             for (int i = 0; i < numberOfRequests; i++) {
                 if (rarely()) {
                     requests.add("/slow/" + i);
                     slowIds.add(i);
-                    numberOfSlowRequests++;
                 } else {
                     requests.add("/" + i);
                 }
@@ -137,16 +137,15 @@ public void testThatHttpPipeliningCanBeDisabled() throws Exception {
             try (Netty4HttpClient nettyHttpClient = new Netty4HttpClient()) {
                 Collection<FullHttpResponse> responses = nettyHttpClient.get(transportAddress.address(), requests.toArray(new String[]{}));
                 List<String> responseBodies = new ArrayList<>(Netty4HttpClient.returnHttpResponseBodies(responses));
-                // we can not be sure about the order of the responses, but the slow ones should
-                // come last
+                // we can not be sure about the order of the responses, but the slow ones should come last
                 assertThat(responseBodies, hasSize(numberOfRequests));
-                for (int i = 0; i < numberOfRequests - numberOfSlowRequests; i++) {
+                for (int i = 0; i < numberOfRequests - slowIds.size(); i++) {
                     assertThat(responseBodies.get(i), matches("/\\d+"));
                 }
 
                 final Set<Integer> ids = new HashSet<>();
-                for (int i = 0; i < numberOfSlowRequests; i++) {
-                    final String response = responseBodies.get(numberOfRequests - numberOfSlowRequests + i);
+                for (int i = 0; i < slowIds.size(); i++) {
+                    final String response = responseBodies.get(numberOfRequests - slowIds.size() + i);
                     assertThat(response, matches("/slow/\\d+" ));
                     assertTrue(ids.add(Integer.parseInt(response.split("/")[2])));
                 }
@@ -164,7 +163,8 @@ class CustomNettyHttpServerTransport extends Netty4HttpServerTransport {
             super(settings,
                 Netty4HttpServerPipeliningTests.this.networkService,
                 Netty4HttpServerPipeliningTests.this.bigArrays,
-                Netty4HttpServerPipeliningTests.this.threadPool);
+                Netty4HttpServerPipeliningTests.this.threadPool,
+                xContentRegistry(), new NullDispatcher());
         }
 
         @Override
@@ -258,11 +258,14 @@ public void run() {
                 assert uri.matches("/\\d+");
             }
 
+            final ChannelPromise promise = ctx.newPromise();
+            final Object msg;
             if (pipelinedRequest != null) {
-                ctx.writeAndFlush(pipelinedRequest.createHttpResponse(httpResponse, ctx.channel().newPromise()));
+                msg = pipelinedRequest.createHttpResponse(httpResponse, promise);
             } else {
-                ctx.writeAndFlush(httpResponse);
+                msg = httpResponse;
             }
+            ctx.writeAndFlush(msg, promise);
         }
 
     }
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java
index faea5c8ec185b..20cc1a65881ef 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java
@@ -21,42 +21,64 @@
 
 import io.netty.buffer.ByteBufUtil;
 import io.netty.buffer.Unpooled;
+import io.netty.handler.codec.TooLongFrameException;
 import io.netty.handler.codec.http.DefaultFullHttpRequest;
 import io.netty.handler.codec.http.FullHttpRequest;
 import io.netty.handler.codec.http.FullHttpResponse;
+import io.netty.handler.codec.http.HttpHeaderNames;
+import io.netty.handler.codec.http.HttpHeaderValues;
 import io.netty.handler.codec.http.HttpMethod;
 import io.netty.handler.codec.http.HttpResponseStatus;
 import io.netty.handler.codec.http.HttpUtil;
 import io.netty.handler.codec.http.HttpVersion;
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.network.NetworkService;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.MockBigArrays;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.http.BindHttpException;
+import org.elasticsearch.http.HttpServerTransport;
+import org.elasticsearch.http.HttpTransportSettings;
+import org.elasticsearch.http.NullDispatcher;
 import org.elasticsearch.http.netty4.cors.Netty4CorsConfig;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.rest.BytesRestResponse;
+import org.elasticsearch.rest.RestChannel;
+import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.junit.After;
 import org.junit.Before;
 
+import java.io.IOException;
+import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.common.Strings.collectionToDelimitedString;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_CREDENTIALS;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_HEADERS;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_METHODS;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_ORIGIN;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ENABLED;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_MAX_AGE;
+import static org.elasticsearch.rest.RestStatus.BAD_REQUEST;
 import static org.elasticsearch.rest.RestStatus.OK;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 
 /**
@@ -88,11 +110,12 @@ public void shutdown() throws Exception {
     public void testCorsConfig() {
         final Set<String> methods = new HashSet<>(Arrays.asList("get", "options", "post"));
         final Set<String> headers = new HashSet<>(Arrays.asList("Content-Type", "Content-Length"));
+        final String prefix = randomBoolean() ? " " : ""; // sometimes have a leading whitespace between comma delimited elements
         final Settings settings = Settings.builder()
                                       .put(SETTING_CORS_ENABLED.getKey(), true)
                                       .put(SETTING_CORS_ALLOW_ORIGIN.getKey(), "*")
-                                      .put(SETTING_CORS_ALLOW_METHODS.getKey(), Strings.collectionToCommaDelimitedString(methods))
-                                      .put(SETTING_CORS_ALLOW_HEADERS.getKey(), Strings.collectionToCommaDelimitedString(headers))
+                                      .put(SETTING_CORS_ALLOW_METHODS.getKey(), collectionToDelimitedString(methods, ",", prefix, ""))
+                                      .put(SETTING_CORS_ALLOW_HEADERS.getKey(), collectionToDelimitedString(headers, ",", prefix, ""))
                                       .put(SETTING_CORS_ALLOW_CREDENTIALS.getKey(), true)
                                       .build();
         final Netty4CorsConfig corsConfig = Netty4HttpServerTransport.buildCorsConfig(settings);
@@ -117,27 +140,180 @@ public void testCorsConfigWithDefaults() {
 
     /**
      * Test that {@link Netty4HttpServerTransport} supports the "Expect: 100-continue" HTTP header
+     * @throws InterruptedException if the client communication with the server is interrupted
      */
-    public void testExpectContinueHeader() throws Exception {
-        try (Netty4HttpServerTransport transport = new Netty4HttpServerTransport(Settings.EMPTY, networkService, bigArrays, threadPool)) {
-            transport.httpServerAdapter((request, channel, context) ->
-                    channel.sendResponse(new BytesRestResponse(OK, BytesRestResponse.TEXT_CONTENT_TYPE, new BytesArray("done"))));
+    public void testExpectContinueHeader() throws InterruptedException {
+        final Settings settings = Settings.EMPTY;
+        final int contentLength = randomIntBetween(1, HttpTransportSettings.SETTING_HTTP_MAX_CONTENT_LENGTH.get(settings).bytesAsInt());
+        runExpectHeaderTest(settings, HttpHeaderValues.CONTINUE.toString(), contentLength, HttpResponseStatus.CONTINUE);
+    }
+
+    /**
+     * Test that {@link Netty4HttpServerTransport} responds to a
+     * 100-continue expectation with too large a content-length
+     * with a 413 status.
+     * @throws InterruptedException if the client communication with the server is interrupted
+     */
+    public void testExpectContinueHeaderContentLengthTooLong() throws InterruptedException {
+        final String key = HttpTransportSettings.SETTING_HTTP_MAX_CONTENT_LENGTH.getKey();
+        final int maxContentLength = randomIntBetween(1, 104857600);
+        final Settings settings = Settings.builder().put(key, maxContentLength + "b").build();
+        final int contentLength = randomIntBetween(maxContentLength + 1, Integer.MAX_VALUE);
+        runExpectHeaderTest(
+                settings, HttpHeaderValues.CONTINUE.toString(), contentLength, HttpResponseStatus.REQUEST_ENTITY_TOO_LARGE);
+    }
+
+    /**
+     * Test that {@link Netty4HttpServerTransport} responds to an unsupported expectation with a 417 status.
+     * @throws InterruptedException if the client communication with the server is interrupted
+     */
+    public void testExpectUnsupportedExpectation() throws InterruptedException {
+        runExpectHeaderTest(Settings.EMPTY, "chocolate=yummy", 0, HttpResponseStatus.EXPECTATION_FAILED);
+    }
+
+    private void runExpectHeaderTest(
+            final Settings settings,
+            final String expectation,
+            final int contentLength,
+            final HttpResponseStatus expectedStatus) throws InterruptedException {
+        final HttpServerTransport.Dispatcher dispatcher = new HttpServerTransport.Dispatcher() {
+            @Override
+            public void dispatchRequest(RestRequest request, RestChannel channel, ThreadContext threadContext) {
+                channel.sendResponse(new BytesRestResponse(OK, BytesRestResponse.TEXT_CONTENT_TYPE, new BytesArray("done")));
+            }
+
+            @Override
+            public void dispatchBadRequest(RestRequest request, RestChannel channel, ThreadContext threadContext, Throwable cause) {
+                throw new AssertionError();
+            }
+        };
+        try (Netty4HttpServerTransport transport = new Netty4HttpServerTransport(settings, networkService, bigArrays, threadPool,
+                xContentRegistry(), dispatcher)) {
+            transport.start();
+            final InetSocketTransportAddress remoteAddress =
+                    (InetSocketTransportAddress) randomFrom(transport.boundAddress().boundAddresses());
+            try (Netty4HttpClient client = new Netty4HttpClient()) {
+                final FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, "/");
+                request.headers().set(HttpHeaderNames.EXPECT, expectation);
+                HttpUtil.setContentLength(request, contentLength);
+
+                final FullHttpResponse response = client.post(remoteAddress.address(), request);
+                assertThat(response.status(), equalTo(expectedStatus));
+                if (expectedStatus.equals(HttpResponseStatus.CONTINUE)) {
+                    final FullHttpRequest continuationRequest =
+                        new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, "/", Unpooled.EMPTY_BUFFER);
+                    final FullHttpResponse continuationResponse = client.post(remoteAddress.address(), continuationRequest);
+
+                    assertThat(continuationResponse.status(), is(HttpResponseStatus.OK));
+                    assertThat(new String(ByteBufUtil.getBytes(continuationResponse.content()), StandardCharsets.UTF_8), is("done"));
+                }
+            }
+        }
+    }
+
+    public void testBindUnavailableAddress() {
+        try (Netty4HttpServerTransport transport = new Netty4HttpServerTransport(Settings.EMPTY, networkService, bigArrays, threadPool,
+                xContentRegistry(), new NullDispatcher())) {
+            transport.start();
+            TransportAddress remoteAddress = randomFrom(transport.boundAddress().boundAddresses());
+            Settings settings = Settings.builder().put("http.port", remoteAddress.getPort()).build();
+            try (Netty4HttpServerTransport otherTransport = new Netty4HttpServerTransport(settings, networkService, bigArrays, threadPool,
+                    xContentRegistry(), new NullDispatcher())) {
+                BindHttpException bindHttpException = expectThrows(BindHttpException.class, () -> otherTransport.start());
+                assertEquals("Failed to bind to [" + remoteAddress.getPort() + "]", bindHttpException.getMessage());
+            }
+        }
+    }
+
+    public void testBadRequest() throws InterruptedException {
+        final AtomicReference<Throwable> causeReference = new AtomicReference<>();
+        final HttpServerTransport.Dispatcher dispatcher = new HttpServerTransport.Dispatcher() {
+
+            @Override
+            public void dispatchRequest(final RestRequest request, final RestChannel channel, final ThreadContext threadContext) {
+                throw new AssertionError();
+            }
+
+            @Override
+            public void dispatchBadRequest(final RestRequest request,
+                                           final RestChannel channel,
+                                           final ThreadContext threadContext,
+                                           final Throwable cause) {
+                causeReference.set(cause);
+                try {
+                    final ElasticsearchException e = new ElasticsearchException("you sent a bad request and you should feel bad");
+                    channel.sendResponse(new BytesRestResponse(channel, BAD_REQUEST, e));
+                } catch (final IOException e) {
+                    throw new AssertionError(e);
+                }
+            }
+
+        };
+
+        final Settings settings;
+        final int maxInitialLineLength;
+        final Setting<ByteSizeValue> httpMaxInitialLineLengthSetting = HttpTransportSettings.SETTING_HTTP_MAX_INITIAL_LINE_LENGTH;
+        if (randomBoolean()) {
+            maxInitialLineLength = httpMaxInitialLineLengthSetting.getDefault(Settings.EMPTY).bytesAsInt();
+            settings = Settings.EMPTY;
+        } else {
+            maxInitialLineLength = randomIntBetween(1, 8192);
+            settings = Settings.builder().put(httpMaxInitialLineLengthSetting.getKey(), maxInitialLineLength + "b").build();
+        }
+
+        try (Netty4HttpServerTransport transport =
+                     new Netty4HttpServerTransport(settings, networkService, bigArrays, threadPool, xContentRegistry(), dispatcher)) {
             transport.start();
-            InetSocketTransportAddress remoteAddress = (InetSocketTransportAddress) randomFrom(transport.boundAddress().boundAddresses());
+            final InetSocketTransportAddress remoteAddress =
+                    (InetSocketTransportAddress) randomFrom(transport.boundAddress.boundAddresses());
 
             try (Netty4HttpClient client = new Netty4HttpClient()) {
-                FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, "/");
-                HttpUtil.set100ContinueExpected(request, true);
-                HttpUtil.setContentLength(request, 10);
+                final String url = "/" + new String(new byte[maxInitialLineLength], Charset.forName("UTF-8"));
+                final FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, url);
 
-                FullHttpResponse response = client.post(remoteAddress.address(), request);
-                assertThat(response.status(), is(HttpResponseStatus.CONTINUE));
+                final FullHttpResponse response = client.post(remoteAddress.address(), request);
+                assertThat(response.status(), equalTo(HttpResponseStatus.BAD_REQUEST));
+                assertThat(
+                        new String(response.content().array(), Charset.forName("UTF-8")),
+                        containsString("you sent a bad request and you should feel bad"));
+            }
+        }
+
+        assertNotNull(causeReference.get());
+        assertThat(causeReference.get(), instanceOf(TooLongFrameException.class));
+    }
+
+    public void testDispatchDoesNotModifyThreadContext() throws InterruptedException {
+        final HttpServerTransport.Dispatcher dispatcher = new HttpServerTransport.Dispatcher() {
+
+            @Override
+            public void dispatchRequest(final RestRequest request, final RestChannel channel, final ThreadContext threadContext) {
+                threadContext.putHeader("foo", "bar");
+                threadContext.putTransient("bar", "baz");
+            }
 
-                request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, "/", Unpooled.EMPTY_BUFFER);
-                response = client.post(remoteAddress.address(), request);
-                assertThat(response.status(), is(HttpResponseStatus.OK));
-                assertThat(new String(ByteBufUtil.getBytes(response.content()), StandardCharsets.UTF_8), is("done"));
+            @Override
+            public void dispatchBadRequest(final RestRequest request,
+                                           final RestChannel channel,
+                                           final ThreadContext threadContext,
+                                           final Throwable cause) {
+                threadContext.putHeader("foo_bad", "bar");
+                threadContext.putTransient("bar_bad", "baz");
             }
+
+        };
+
+        try (Netty4HttpServerTransport transport =
+                 new Netty4HttpServerTransport(Settings.EMPTY, networkService, bigArrays, threadPool, xContentRegistry(), dispatcher)) {
+            transport.start();
+
+            transport.dispatchRequest(null, null);
+            assertNull(threadPool.getThreadContext().getHeader("foo"));
+            assertNull(threadPool.getThreadContext().getTransient("bar"));
+
+            transport.dispatchBadRequest(null, null, null);
+            assertNull(threadPool.getThreadContext().getHeader("foo_bad"));
+            assertNull(threadPool.getThreadContext().getTransient("bar_bad"));
         }
     }
 }
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/pipelining/Netty4HttpPipeliningHandlerTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/pipelining/Netty4HttpPipeliningHandlerTests.java
index 1feb92223a352..ffb6c8fb3569d 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/pipelining/Netty4HttpPipeliningHandlerTests.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/pipelining/Netty4HttpPipeliningHandlerTests.java
@@ -37,10 +37,10 @@
 import io.netty.handler.codec.http.LastHttpContent;
 import io.netty.handler.codec.http.QueryStringDecoder;
 import org.elasticsearch.common.Randomness;
-import org.elasticsearch.common.util.concurrent.CountDown;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.After;
 
+import java.nio.channels.ClosedChannelException;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.List;
@@ -62,9 +62,9 @@
 
 public class Netty4HttpPipeliningHandlerTests extends ESTestCase {
 
-    private ExecutorService executorService = Executors.newFixedThreadPool(randomIntBetween(4, 8));
-    private Map<String, CountDownLatch> waitingRequests = new ConcurrentHashMap<>();
-    private Map<String, CountDownLatch> finishingRequests = new ConcurrentHashMap<>();
+    private final ExecutorService executorService = Executors.newFixedThreadPool(randomIntBetween(4, 8));
+    private final Map<String, CountDownLatch> waitingRequests = new ConcurrentHashMap<>();
+    private final Map<String, CountDownLatch> finishingRequests = new ConcurrentHashMap<>();
 
     @After
     public void tearDown() throws Exception {
@@ -87,7 +87,8 @@ private void shutdownExecutorService() throws InterruptedException {
 
     public void testThatPipeliningWorksWithFastSerializedRequests() throws InterruptedException {
         final int numberOfRequests = randomIntBetween(2, 128);
-        final EmbeddedChannel embeddedChannel = new EmbeddedChannel(new HttpPipeliningHandler(numberOfRequests), new WorkEmulatorHandler());
+        final EmbeddedChannel embeddedChannel = new EmbeddedChannel(new HttpPipeliningHandler(logger, numberOfRequests),
+            new WorkEmulatorHandler());
 
         for (int i = 0; i < numberOfRequests; i++) {
             embeddedChannel.writeInbound(createHttpRequest("/" + String.valueOf(i)));
@@ -113,7 +114,8 @@ public void testThatPipeliningWorksWithFastSerializedRequests() throws Interrupt
 
     public void testThatPipeliningWorksWhenSlowRequestsInDifferentOrder() throws InterruptedException {
         final int numberOfRequests = randomIntBetween(2, 128);
-        final EmbeddedChannel embeddedChannel = new EmbeddedChannel(new HttpPipeliningHandler(numberOfRequests), new WorkEmulatorHandler());
+        final EmbeddedChannel embeddedChannel = new EmbeddedChannel(new HttpPipeliningHandler(logger, numberOfRequests),
+            new WorkEmulatorHandler());
 
         for (int i = 0; i < numberOfRequests; i++) {
             embeddedChannel.writeInbound(createHttpRequest("/" + String.valueOf(i)));
@@ -145,7 +147,7 @@ public void testThatPipeliningWorksWithChunkedRequests() throws InterruptedExcep
         final EmbeddedChannel embeddedChannel =
             new EmbeddedChannel(
                 new AggregateUrisAndHeadersHandler(),
-                new HttpPipeliningHandler(numberOfRequests),
+                new HttpPipeliningHandler(logger, numberOfRequests),
                 new WorkEmulatorHandler());
 
         for (int i = 0; i < numberOfRequests; i++) {
@@ -174,7 +176,8 @@ public void testThatPipeliningWorksWithChunkedRequests() throws InterruptedExcep
 
     public void testThatPipeliningClosesConnectionWithTooManyEvents() throws InterruptedException {
         final int numberOfRequests = randomIntBetween(2, 128);
-        final EmbeddedChannel embeddedChannel = new EmbeddedChannel(new HttpPipeliningHandler(numberOfRequests), new WorkEmulatorHandler());
+        final EmbeddedChannel embeddedChannel = new EmbeddedChannel(new HttpPipeliningHandler(logger, numberOfRequests),
+            new WorkEmulatorHandler());
 
         for (int i = 0; i < 1 + numberOfRequests + 1; i++) {
             embeddedChannel.writeInbound(createHttpRequest("/" + Integer.toString(i)));
@@ -199,6 +202,40 @@ public void testThatPipeliningClosesConnectionWithTooManyEvents() throws Interru
         assertFalse(embeddedChannel.isOpen());
     }
 
+    public void testPipeliningRequestsAreReleased() throws InterruptedException {
+        final int numberOfRequests = 10;
+        final EmbeddedChannel embeddedChannel =
+            new EmbeddedChannel(new HttpPipeliningHandler(logger, numberOfRequests + 1));
+
+        for (int i = 0; i < numberOfRequests; i++) {
+            embeddedChannel.writeInbound(createHttpRequest("/" + i));
+        }
+
+        HttpPipelinedRequest inbound;
+        ArrayList<HttpPipelinedRequest> requests = new ArrayList<>();
+        while ((inbound = embeddedChannel.readInbound()) != null) {
+            requests.add(inbound);
+        }
+
+        ArrayList<ChannelPromise> promises = new ArrayList<>();
+        for (int i = 1; i < requests.size(); ++i) {
+            final DefaultFullHttpResponse httpResponse = new DefaultFullHttpResponse(HTTP_1_1, OK);
+            ChannelPromise promise = embeddedChannel.newPromise();
+            promises.add(promise);
+            HttpPipelinedResponse response = requests.get(i).createHttpResponse(httpResponse, promise);
+            embeddedChannel.writeAndFlush(response, promise);
+        }
+
+        for (ChannelPromise promise : promises) {
+            assertFalse(promise.isDone());
+        }
+        embeddedChannel.close().syncUninterruptibly();
+        for (ChannelPromise promise : promises) {
+            assertTrue(promise.isDone());
+            assertTrue(promise.cause() instanceof ClosedChannelException);
+        }
+    }
+
 
     private void assertReadHttpMessageHasContent(EmbeddedChannel embeddedChannel, String expectedContent) {
         FullHttpResponse response = (FullHttpResponse) embeddedChannel.outboundMessages().poll();
@@ -256,7 +293,5 @@ protected void channelRead0(final ChannelHandlerContext ctx, final HttpPipelined
                 }
             });
         }
-
     }
-
 }
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/rest/Netty4BadRequestIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/rest/Netty4BadRequestIT.java
new file mode 100644
index 0000000000000..bcbe96a0c4d24
--- /dev/null
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/rest/Netty4BadRequestIT.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.rest;
+
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.http.HttpTransportSettings;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.test.rest.yaml.ObjectPath;
+
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.Collections;
+import java.util.Map;
+
+import static org.elasticsearch.rest.RestStatus.BAD_REQUEST;
+import static org.elasticsearch.test.hamcrest.RegexMatcher.matches;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasToString;
+
+public class Netty4BadRequestIT extends ESRestTestCase {
+
+    public void testBadRequest() throws IOException {
+        final Response response = client().performRequest("GET", "/_nodes/settings", Collections.emptyMap());
+        final ObjectPath objectPath = ObjectPath.createFromResponse(response);
+        @SuppressWarnings("unchecked") final Map<String, Object> map = (Map<String, Object>) objectPath.evaluate("nodes");
+        int maxMaxInitialLineLength = Integer.MIN_VALUE;
+        final Setting<ByteSizeValue> httpMaxInitialLineLength = HttpTransportSettings.SETTING_HTTP_MAX_INITIAL_LINE_LENGTH;
+        final String key = httpMaxInitialLineLength.getKey().substring("http.".length());
+        for (Map.Entry<String, Object> entry : map.entrySet()) {
+            @SuppressWarnings("unchecked") final Map<String, Object> settings =
+                    (Map<String, Object>)((Map<String, Object>)entry.getValue()).get("settings");
+                    final int maxIntialLineLength;
+            if (settings.containsKey("http")) {
+                @SuppressWarnings("unchecked") final Map<String, Object> httpSettings = (Map<String, Object>)settings.get("http");
+                if (httpSettings.containsKey(key)) {
+                    maxIntialLineLength = ByteSizeValue.parseBytesSizeValue((String)httpSettings.get(key), key).bytesAsInt();
+                } else {
+                    maxIntialLineLength = httpMaxInitialLineLength.getDefault(Settings.EMPTY).bytesAsInt();
+                }
+            } else {
+                maxIntialLineLength = httpMaxInitialLineLength.getDefault(Settings.EMPTY).bytesAsInt();
+            }
+            maxMaxInitialLineLength = Math.max(maxMaxInitialLineLength, maxIntialLineLength);
+        }
+
+        final String path = "/" + new String(new byte[maxMaxInitialLineLength], Charset.forName("UTF-8")).replace('\0', 'a');
+        final ResponseException e =
+                expectThrows(
+                        ResponseException.class,
+                        () -> client().performRequest(randomFrom("GET", "POST", "PUT"), path, Collections.emptyMap()));
+        assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(BAD_REQUEST.getStatus()));
+        assertThat(e, hasToString(containsString("too_long_frame_exception")));
+        assertThat(e, hasToString(matches("An HTTP line is larger than \\d+ bytes")));
+    }
+}
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java
new file mode 100644
index 0000000000000..c865c4bb2bed2
--- /dev/null
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.rest;
+
+public class Netty4HeadBodyIsEmptyIT extends HeadBodyIsEmptyIntegTestCase {
+
+}
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/ByteBufBytesReferenceTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/ByteBufBytesReferenceTests.java
index bce875e85167b..afe6bbbc90f14 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/ByteBufBytesReferenceTests.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/ByteBufBytesReferenceTests.java
@@ -28,8 +28,14 @@
 import java.io.IOException;
 
 public class ByteBufBytesReferenceTests extends AbstractBytesReferenceTestCase {
+
     @Override
     protected BytesReference newBytesReference(int length) throws IOException {
+        return newBytesReferenceWithOffsetOfZero(length);
+    }
+
+    @Override
+    protected BytesReference newBytesReferenceWithOffsetOfZero(int length) throws IOException {
         ReleasableBytesStreamOutput out = new ReleasableBytesStreamOutput(length, bigarrays);
         for (int i = 0; i < length; i++) {
             out.writeByte((byte) random().nextInt(1 << 8));
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/ESLoggingHandlerIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/ESLoggingHandlerIT.java
new file mode 100644
index 0000000000000..acd71749e2333
--- /dev/null
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/ESLoggingHandlerIT.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.transport.netty4;
+
+import org.apache.logging.log4j.Level;
+import org.elasticsearch.ESNetty4IntegTestCase;
+import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsRequest;
+import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.MockLogAppender;
+import org.elasticsearch.test.junit.annotations.TestLogging;
+
+@ESIntegTestCase.ClusterScope(numDataNodes = 2)
+@TestLogging(value = "org.elasticsearch.transport.netty4.ESLoggingHandler:trace")
+public class ESLoggingHandlerIT extends ESNetty4IntegTestCase {
+
+    private MockLogAppender appender;
+
+    public void setUp() throws Exception {
+        super.setUp();
+        appender = new MockLogAppender();
+        Loggers.addAppender(Loggers.getLogger(ESLoggingHandler.class), appender);
+        appender.start();
+    }
+
+    public void tearDown() throws Exception {
+        Loggers.removeAppender(Loggers.getLogger(ESLoggingHandler.class), appender);
+        appender.stop();
+        super.tearDown();
+    }
+
+    public void testLoggingHandler() throws IllegalAccessException {
+        final String writePattern =
+                ".*\\[length: \\d+" +
+                        ", request id: \\d+" +
+                        ", type: request" +
+                        ", version: .*" +
+                        ", action: cluster:monitor/nodes/hot_threads\\[n\\]\\]" +
+                        " WRITE: \\d+B";
+        final MockLogAppender.LoggingExpectation writeExpectation =
+                new MockLogAppender.PatternSeenEventExcpectation(
+                        "hot threads request", ESLoggingHandler.class.getCanonicalName(), Level.TRACE, writePattern);
+
+        final MockLogAppender.LoggingExpectation flushExpectation =
+                new MockLogAppender.SeenEventExpectation("flush", ESLoggingHandler.class.getCanonicalName(), Level.TRACE, "*FLUSH*");
+
+        final String readPattern =
+                ".*\\[length: \\d+" +
+                        ", request id: \\d+" +
+                        ", type: request" +
+                        ", version: .*" +
+                        ", action: cluster:monitor/nodes/hot_threads\\[n\\]\\]" +
+                        " READ: \\d+B";
+
+        final MockLogAppender.LoggingExpectation readExpectation =
+                new MockLogAppender.PatternSeenEventExcpectation(
+                        "hot threads request", ESLoggingHandler.class.getCanonicalName(), Level.TRACE, readPattern);
+
+        appender.addExpectation(writeExpectation);
+        appender.addExpectation(flushExpectation);
+        appender.addExpectation(readExpectation);
+        client().admin().cluster().nodesHotThreads(new NodesHotThreadsRequest()).actionGet();
+        appender.assertAllExpectationsMatched();
+    }
+
+}
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4ScheduledPingTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4ScheduledPingTests.java
index 03f7e5fdab785..df60f84246dda 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4ScheduledPingTests.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4ScheduledPingTests.java
@@ -18,12 +18,12 @@
  */
 package org.elasticsearch.transport.netty4;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
@@ -31,6 +31,7 @@
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.ConnectionProfile;
 import org.elasticsearch.transport.TcpTransport;
 import org.elasticsearch.transport.TransportChannel;
 import org.elasticsearch.transport.TransportException;
@@ -40,13 +41,12 @@
 import org.elasticsearch.transport.TransportResponse;
 import org.elasticsearch.transport.TransportResponseHandler;
 import org.elasticsearch.transport.TransportResponseOptions;
+import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.transport.TransportSettings;
 
 import java.io.IOException;
 import java.util.Collections;
 
-import static java.util.Collections.emptyMap;
-import static java.util.Collections.emptySet;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 
@@ -65,31 +65,43 @@ public void testScheduledPing() throws Exception {
         NamedWriteableRegistry registry = new NamedWriteableRegistry(Collections.emptyList());
         final Netty4Transport nettyA = new Netty4Transport(settings, threadPool, new NetworkService(settings, Collections.emptyList()),
             BigArrays.NON_RECYCLING_INSTANCE, registry, circuitBreakerService);
-        MockTransportService serviceA = new MockTransportService(settings, nettyA, threadPool);
+        MockTransportService serviceA = new MockTransportService(settings, nettyA, threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR,
+                null);
         serviceA.start();
         serviceA.acceptIncomingRequests();
 
         final Netty4Transport nettyB = new Netty4Transport(settings, threadPool, new NetworkService(settings, Collections.emptyList()),
             BigArrays.NON_RECYCLING_INSTANCE, registry, circuitBreakerService);
-        MockTransportService serviceB = new MockTransportService(settings, nettyB, threadPool);
+        MockTransportService serviceB = new MockTransportService(settings, nettyB, threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR,
+                null);
 
         serviceB.start();
         serviceB.acceptIncomingRequests();
 
-        DiscoveryNode nodeA =
-            new DiscoveryNode("TS_A", "TS_A", serviceA.boundAddress().publishAddress(), emptyMap(), emptySet(), Version.CURRENT);
-        DiscoveryNode nodeB =
-            new DiscoveryNode("TS_B", "TS_B", serviceB.boundAddress().publishAddress(), emptyMap(), emptySet(), Version.CURRENT);
-
-        serviceA.connectToNode(nodeB);
-        serviceB.connectToNode(nodeA);
+        DiscoveryNode nodeA = serviceA.getLocalDiscoNode();
+        DiscoveryNode nodeB = serviceB.getLocalDiscoNode();
+
+        if (randomBoolean()) {
+            // use connection profile with different connect timeout
+            final ConnectionProfile connectionProfile = new ConnectionProfile.Builder()
+                .addConnections(1,
+                    TransportRequestOptions.Type.BULK,
+                    TransportRequestOptions.Type.PING,
+                    TransportRequestOptions.Type.RECOVERY,
+                    TransportRequestOptions.Type.REG,
+                    TransportRequestOptions.Type.STATE)
+                .setConnectTimeout(TimeValue.timeValueSeconds(42))
+                .build();
+            serviceA.connectToNode(nodeB, connectionProfile);
+            serviceB.connectToNode(nodeA, connectionProfile);
+        } else {
+            serviceA.connectToNode(nodeB);
+            serviceB.connectToNode(nodeA);
+        }
 
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                assertThat(nettyA.getPing().getSuccessfulPings(), greaterThan(100L));
-                assertThat(nettyB.getPing().getSuccessfulPings(), greaterThan(100L));
-            }
+        assertBusy(() -> {
+            assertThat(nettyA.getPing().getSuccessfulPings(), greaterThan(100L));
+            assertThat(nettyB.getPing().getSuccessfulPings(), greaterThan(100L));
         });
         assertThat(nettyA.getPing().getFailedPings(), equalTo(0L));
         assertThat(nettyB.getPing().getFailedPings(), equalTo(0L));
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportIT.java
index b913fad933fda..7e54b53de491f 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportIT.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportIT.java
@@ -25,7 +25,6 @@
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.network.NetworkModule;
@@ -33,10 +32,12 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
+import org.elasticsearch.plugins.NetworkPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportSettings;
 
 import java.io.IOException;
@@ -45,6 +46,8 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
+import java.util.function.Supplier;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.is;
@@ -83,13 +86,19 @@ public void testThatConnectionFailsAsIntended() throws Exception {
 
     public static final class ExceptionThrowingNetty4Transport extends Netty4Transport {
 
-        public static class TestPlugin extends Plugin {
-            public void onModule(NetworkModule module) {
-                module.registerTransport("exception-throwing", ExceptionThrowingNetty4Transport.class);
+        public static class TestPlugin extends Plugin implements NetworkPlugin {
+
+            @Override
+            public Map<String, Supplier<Transport>> getTransports(Settings settings, ThreadPool threadPool, BigArrays bigArrays,
+                                                                  CircuitBreakerService circuitBreakerService,
+                                                                  NamedWriteableRegistry namedWriteableRegistry,
+                                                                  NetworkService networkService) {
+                return Collections.singletonMap("exception-throwing",
+                    () -> new ExceptionThrowingNetty4Transport(settings, threadPool, networkService, bigArrays,
+                    namedWriteableRegistry, circuitBreakerService));
             }
         }
 
-        @Inject
         public ExceptionThrowingNetty4Transport(
                 Settings settings,
                 ThreadPool threadPool,
@@ -102,9 +111,9 @@ public ExceptionThrowingNetty4Transport(
 
         protected String handleRequest(Channel channel, String profileName,
                                        StreamInput stream, long requestId, int messageLengthBytes, Version version,
-                                       InetSocketAddress remoteAddress) throws IOException {
+                                       InetSocketAddress remoteAddress, byte status) throws IOException {
             String action = super.handleRequest(channel, profileName, stream, requestId, messageLengthBytes, version,
-                    remoteAddress);
+                    remoteAddress, status);
             channelProfileName = TransportSettings.DEFAULT_PROFILE;
             return action;
         }
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportMultiPortIntegrationIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportMultiPortIntegrationIT.java
index 32268d1f5d894..62980be850488 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportMultiPortIntegrationIT.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportMultiPortIntegrationIT.java
@@ -58,7 +58,7 @@ public class Netty4TransportMultiPortIntegrationIT extends ESNetty4IntegTestCase
     protected Settings nodeSettings(int nodeOrdinal) {
         if (randomPort == -1) {
             randomPort = randomIntBetween(49152, 65525);
-            randomPortRange = String.format(Locale.ROOT, "%s-%s", randomPort, randomPort+10);
+            randomPortRange = String.format(Locale.ROOT, "%s-%s", randomPort, randomPort + 10);
         }
         Settings.Builder builder = Settings.builder()
             .put(super.nodeSettings(nodeOrdinal))
@@ -76,8 +76,12 @@ public void testThatTransportClientCanConnect() throws Exception {
             .put(NetworkModule.TRANSPORT_TYPE_KEY, Netty4Plugin.NETTY_TRANSPORT_NAME)
             .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
             .build();
+        // we have to test all the ports that the data node might be bound to
         try (TransportClient transportClient = new MockTransportClient(settings, Netty4Plugin.class)) {
             transportClient.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName("127.0.0.1"), randomPort));
+            for (int i = 0; i <= 10; i++) {
+                transportClient.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName("127.0.0.1"), randomPort + i));
+            }
             ClusterHealthResponse response = transportClient.admin().cluster().prepareHealth().get();
             assertThat(response.getStatus(), is(ClusterHealthStatus.GREEN));
         }
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4UtilsTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4UtilsTests.java
index 8372a8540b8be..43be6f0efdda0 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4UtilsTests.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4UtilsTests.java
@@ -22,6 +22,7 @@
 import io.netty.buffer.ByteBuf;
 import io.netty.buffer.CompositeByteBuf;
 import io.netty.buffer.Unpooled;
+import io.netty.handler.codec.DecoderException;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.bytes.AbstractBytesReferenceTestCase;
 import org.elasticsearch.common.bytes.BytesArray;
@@ -32,6 +33,9 @@
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
+import java.util.Optional;
+
+import static org.hamcrest.CoreMatchers.equalTo;
 
 public class Netty4UtilsTests extends ESTestCase {
 
@@ -75,6 +79,60 @@ public void testToChannelBuffer() throws IOException {
         assertArrayEquals(BytesReference.toBytes(ref), BytesReference.toBytes(bytesReference));
     }
 
+    public void testMaybeError() {
+        final Error outOfMemoryError = new OutOfMemoryError();
+        assertError(outOfMemoryError, outOfMemoryError);
+
+        final DecoderException decoderException = new DecoderException(outOfMemoryError);
+        assertError(decoderException, outOfMemoryError);
+
+        final Exception e = new Exception();
+        e.addSuppressed(decoderException);
+        assertError(e, outOfMemoryError);
+
+        final int depth = randomIntBetween(1, 16);
+        Throwable cause = new Exception();
+        boolean fatal = false;
+        Error error = null;
+        for (int i = 0; i < depth; i++) {
+            final int length = randomIntBetween(1, 4);
+            for (int j = 0; j < length; j++) {
+                if (!fatal && rarely()) {
+                    error = new Error();
+                    cause.addSuppressed(error);
+                    fatal = true;
+                } else {
+                    cause.addSuppressed(new Exception());
+                }
+            }
+            if (!fatal && rarely()) {
+                cause = error = new Error(cause);
+                fatal = true;
+            } else {
+                cause = new Exception(cause);
+            }
+        }
+        if (fatal) {
+            assertError(cause, error);
+        } else {
+            assertFalse(Netty4Utils.maybeError(cause).isPresent());
+        }
+
+        assertFalse(Netty4Utils.maybeError(new Exception(new DecoderException())).isPresent());
+
+        Throwable chain = outOfMemoryError;
+        for (int i = 0; i < Netty4Utils.MAX_ITERATIONS; i++) {
+            chain = new Exception(chain);
+        }
+        assertFalse(Netty4Utils.maybeError(chain).isPresent());
+    }
+
+    private void assertError(final Throwable cause, final Error error) {
+        final Optional<Error> maybeError = Netty4Utils.maybeError(cause);
+        assertTrue(maybeError.isPresent());
+        assertThat(maybeError.get(), equalTo(error));
+    }
+
     private BytesReference getRandomizedBytesReference(int length) throws IOException {
         // we know bytes stream output always creates a paged bytes reference, we use it to create randomized content
         ReleasableBytesStreamOutput out = new ReleasableBytesStreamOutput(length, bigarrays);
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SimpleNetty4TransportTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SimpleNetty4TransportTests.java
index 8902d6c109585..faee61bd919b7 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SimpleNetty4TransportTests.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SimpleNetty4TransportTests.java
@@ -19,21 +19,29 @@
 
 package org.elasticsearch.transport.netty4;
 
+import io.netty.channel.Channel;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.network.NetworkService;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
+import org.elasticsearch.node.Node;
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.AbstractSimpleTransportTestCase;
+import org.elasticsearch.transport.BindTransportException;
 import org.elasticsearch.transport.ConnectTransportException;
+import org.elasticsearch.transport.TcpTransport;
 import org.elasticsearch.transport.Transport;
+import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.transport.TransportSettings;
 
+import java.io.IOException;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
 import java.util.Collections;
@@ -44,28 +52,48 @@
 
 public class SimpleNetty4TransportTests extends AbstractSimpleTransportTestCase {
 
-    public static MockTransportService nettyFromThreadPool(
-        Settings settings,
-        ThreadPool threadPool, final Version version) {
+    public static MockTransportService nettyFromThreadPool(Settings settings, ThreadPool threadPool, final Version version,
+                                                           ClusterSettings clusterSettings, boolean doHandshake) {
         NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(Collections.emptyList());
         Transport transport = new Netty4Transport(settings, threadPool, new NetworkService(settings, Collections.emptyList()),
             BigArrays.NON_RECYCLING_INSTANCE, namedWriteableRegistry, new NoneCircuitBreakerService()) {
+
+            @Override
+            protected Version executeHandshake(DiscoveryNode node, Channel channel, TimeValue timeout) throws IOException,
+                InterruptedException {
+                if (doHandshake) {
+                    return super.executeHandshake(node, channel, timeout);
+                } else {
+                    return version.minimumCompatibilityVersion();
+                }
+            }
+
             @Override
             protected Version getCurrentVersion() {
                 return version;
             }
         };
-        return new MockTransportService(Settings.EMPTY, transport, threadPool);
+        MockTransportService mockTransportService =
+            MockTransportService.createNewService(Settings.EMPTY, transport, version, threadPool, clusterSettings);
+        mockTransportService.start();
+        return mockTransportService;
     }
 
     @Override
-    protected MockTransportService build(Settings settings, Version version) {
+    protected MockTransportService build(Settings settings, Version version, ClusterSettings clusterSettings, boolean doHandshake) {
         settings = Settings.builder().put(settings).put(TransportSettings.PORT.getKey(), "0").build();
-        MockTransportService transportService = nettyFromThreadPool(settings, threadPool, version);
+        MockTransportService transportService = nettyFromThreadPool(settings, threadPool, version, clusterSettings, doHandshake);
         transportService.start();
         return transportService;
     }
 
+    @Override
+    protected void closeConnectionChannel(Transport transport, Transport.Connection connection) throws IOException {
+        final Netty4Transport t = (Netty4Transport) transport;
+        @SuppressWarnings("unchecked") final TcpTransport<Channel>.NodeChannels channels = (TcpTransport<Channel>.NodeChannels) connection;
+        t.closeChannels(channels.getChannels().subList(0, randomIntBetween(1, channels.getChannels().size())), true);
+    }
+
     public void testConnectException() throws UnknownHostException {
         try {
             serviceA.connectToNode(new DiscoveryNode("C", new InetSocketTransportAddress(InetAddress.getByName("localhost"), 9876),
@@ -77,4 +105,27 @@ public void testConnectException() throws UnknownHostException {
         }
     }
 
+    public void testBindUnavailableAddress() {
+        // this is on a lower level since it needs access to the TransportService before it's started
+        int port = serviceA.boundAddress().publishAddress().getPort();
+        Settings settings = Settings.builder()
+            .put(Node.NODE_NAME_SETTING.getKey(), "foobar")
+            .put(TransportService.TRACE_LOG_INCLUDE_SETTING.getKey(), "")
+            .put(TransportService.TRACE_LOG_EXCLUDE_SETTING.getKey(), "NOTHING")
+            .put("transport.tcp.port", port)
+            .build();
+        ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        BindTransportException bindTransportException = expectThrows(BindTransportException.class, () -> {
+            MockTransportService transportService =
+                    nettyFromThreadPool(settings, threadPool, Version.CURRENT, clusterSettings, true);
+            try {
+                transportService.start();
+            } finally {
+                transportService.stop();
+                transportService.close();
+            }
+        });
+        assertEquals("Failed to bind to ["+ port + "]", bindTransportException.getMessage());
+    }
+
 }
diff --git a/plugins/analysis-icu/build.gradle b/plugins/analysis-icu/build.gradle
index 9ed155b5fc4b8..53f2747c0a277 100644
--- a/plugins/analysis-icu/build.gradle
+++ b/plugins/analysis-icu/build.gradle
@@ -29,4 +29,4 @@ dependencies {
 
 dependencyLicenses {
   mapping from: /lucene-.*/, to: 'lucene'
-}
+}
\ No newline at end of file
diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.2.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.2.0.jar.sha1
deleted file mode 100644
index 2a734f79a3f83..0000000000000
--- a/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.2.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-68de5f298090b92aa9a803eb4f5aed0c9104e685
\ No newline at end of file
diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.6.1.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.6.1.jar.sha1
new file mode 100644
index 0000000000000..076513195dcb3
--- /dev/null
+++ b/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.6.1.jar.sha1
@@ -0,0 +1 @@
+cac1539380dc1dd5c276c531860b219360534a52
\ No newline at end of file
diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuFoldingTokenFilterFactory.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuFoldingTokenFilterFactory.java
index 5fd3199e99a07..60ab831e6f1f4 100644
--- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuFoldingTokenFilterFactory.java
+++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuFoldingTokenFilterFactory.java
@@ -19,9 +19,8 @@
 
 package org.elasticsearch.index.analysis;
 
-import com.ibm.icu.text.FilteredNormalizer2;
 import com.ibm.icu.text.Normalizer2;
-import com.ibm.icu.text.UnicodeSet;
+
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.icu.ICUFoldingFilter;
 import org.elasticsearch.common.settings.Settings;
@@ -41,31 +40,20 @@
  * @author kimchy (shay.banon)
  */
 public class IcuFoldingTokenFilterFactory extends AbstractTokenFilterFactory implements MultiTermAwareComponent {
-    private final String unicodeSetFilter;
+    /** Store here the same Normalizer used by the lucene ICUFoldingFilter */
+    private static final Normalizer2 ICU_FOLDING_NORMALIZER = Normalizer2.getInstance(
+            ICUFoldingFilter.class.getResourceAsStream("utr30.nrm"), "utr30", Normalizer2.Mode.COMPOSE);
+
+    private final Normalizer2 normalizer;
 
     public IcuFoldingTokenFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
         super(indexSettings, name, settings);
-        this.unicodeSetFilter = settings.get("unicodeSetFilter");
+        this.normalizer = IcuNormalizerTokenFilterFactory.wrapWithUnicodeSetFilter(ICU_FOLDING_NORMALIZER, settings);
     }
 
     @Override
     public TokenStream create(TokenStream tokenStream) {
-
-        // The ICUFoldingFilter is in fact implemented as a ICUNormalizer2Filter.
-        // ICUFoldingFilter lacks a constructor for adding filtering so we implemement it here
-        if (unicodeSetFilter != null) {
-            Normalizer2 base = Normalizer2.getInstance(
-                    ICUFoldingFilter.class.getResourceAsStream("utr30.nrm"),
-                    "utr30", Normalizer2.Mode.COMPOSE);
-            UnicodeSet unicodeSet = new UnicodeSet(unicodeSetFilter);
-
-            unicodeSet.freeze();
-            Normalizer2 filtered = new FilteredNormalizer2(base, unicodeSet);
-            return new org.apache.lucene.analysis.icu.ICUNormalizer2Filter(tokenStream, filtered);
-        }
-        else {
-            return new ICUFoldingFilter(tokenStream);
-        }
+        return new org.apache.lucene.analysis.icu.ICUNormalizer2Filter(tokenStream, normalizer);
     }
 
     @Override
diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuNormalizerCharFilterFactory.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuNormalizerCharFilterFactory.java
index 72bc45a0232f0..3046d6839b9e6 100644
--- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuNormalizerCharFilterFactory.java
+++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuNormalizerCharFilterFactory.java
@@ -21,6 +21,7 @@
 
 
 import com.ibm.icu.text.Normalizer2;
+
 import org.apache.lucene.analysis.icu.ICUNormalizer2CharFilter;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
@@ -33,22 +34,22 @@
  * Uses the {@link org.apache.lucene.analysis.icu.ICUNormalizer2CharFilter} to normalize character.
  * <p>The <tt>name</tt> can be used to provide the type of normalization to perform.</p>
  * <p>The <tt>mode</tt> can be used to provide 'compose' or 'decompose'. Default is compose.</p>
+ * <p>The <tt>unicodeSetFilter</tt> attribute can be used to provide the UniCodeSet for filtering.</p>
  */
 public class IcuNormalizerCharFilterFactory extends AbstractCharFilterFactory implements MultiTermAwareComponent {
 
-    private final String name;
-
     private final Normalizer2 normalizer;
 
     public IcuNormalizerCharFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
         super(indexSettings, name);
-        this.name = settings.get("name", "nfkc_cf");
+        String method = settings.get("name", "nfkc_cf");
         String mode = settings.get("mode");
         if (!"compose".equals(mode) && !"decompose".equals(mode)) {
             mode = "compose";
         }
-        this.normalizer = Normalizer2.getInstance(
-            null, this.name, "compose".equals(mode) ? Normalizer2.Mode.COMPOSE : Normalizer2.Mode.DECOMPOSE);
+        Normalizer2 normalizer = Normalizer2.getInstance(
+            null, method, "compose".equals(mode) ? Normalizer2.Mode.COMPOSE : Normalizer2.Mode.DECOMPOSE);
+        this.normalizer = IcuNormalizerTokenFilterFactory.wrapWithUnicodeSetFilter(normalizer, settings);
     }
 
     @Override
diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuNormalizerTokenFilterFactory.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuNormalizerTokenFilterFactory.java
index 2632958d2036f..4e8d5d702205d 100644
--- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuNormalizerTokenFilterFactory.java
+++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuNormalizerTokenFilterFactory.java
@@ -19,7 +19,10 @@
 
 package org.elasticsearch.index.analysis;
 
+import com.ibm.icu.text.FilteredNormalizer2;
 import com.ibm.icu.text.Normalizer2;
+import com.ibm.icu.text.UnicodeSet;
+
 import org.apache.lucene.analysis.TokenStream;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
@@ -28,26 +31,40 @@
 
 /**
  * Uses the {@link org.apache.lucene.analysis.icu.ICUNormalizer2Filter} to normalize tokens.
- * <p>The <tt>name</tt> can be used to provide the type of normalization to perform.
+ * <p>The <tt>name</tt> can be used to provide the type of normalization to perform.</p>
+ * <p>The <tt>unicodeSetFilter</tt> attribute can be used to provide the UniCodeSet for filtering.</p>
  *
  *
  */
 public class IcuNormalizerTokenFilterFactory extends AbstractTokenFilterFactory implements MultiTermAwareComponent {
 
-    private final String name;
+    private final Normalizer2 normalizer;
 
     public IcuNormalizerTokenFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
         super(indexSettings, name, settings);
-        this.name = settings.get("name", "nfkc_cf");
+        String method = settings.get("name", "nfkc_cf");
+        Normalizer2 normalizer = Normalizer2.getInstance(null, method, Normalizer2.Mode.COMPOSE);
+        this.normalizer = wrapWithUnicodeSetFilter(normalizer, settings);
     }
 
     @Override
     public TokenStream create(TokenStream tokenStream) {
-        return new org.apache.lucene.analysis.icu.ICUNormalizer2Filter(tokenStream, Normalizer2.getInstance(null, name, Normalizer2.Mode.COMPOSE));
+        return new org.apache.lucene.analysis.icu.ICUNormalizer2Filter(tokenStream, normalizer);
     }
 
     @Override
     public Object getMultiTermComponent() {
         return this;
     }
+
+    static Normalizer2 wrapWithUnicodeSetFilter(final Normalizer2 normalizer, Settings settings) {
+        String unicodeSetFilter = settings.get("unicodeSetFilter");
+        if (unicodeSetFilter != null) {
+            UnicodeSet unicodeSet = new UnicodeSet(unicodeSetFilter);
+
+            unicodeSet.freeze();
+            return new FilteredNormalizer2(normalizer, unicodeSet);
+        }
+        return normalizer;
+    }
 }
diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java
new file mode 100644
index 0000000000000..ff4921ee9be66
--- /dev/null
+++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java
@@ -0,0 +1,755 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.mapper;
+
+import com.ibm.icu.text.Collator;
+import com.ibm.icu.text.RawCollationKey;
+import com.ibm.icu.text.RuleBasedCollator;
+import com.ibm.icu.util.ULocale;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.SortedSetDocValuesField;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.search.MultiTermQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.lucene.Lucene;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.Fuzziness;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.index.analysis.IndexableBinaryStringTools;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.search.DocValueFormat;
+import org.joda.time.DateTimeZone;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.BiFunction;
+import java.util.function.LongSupplier;
+
+public class ICUCollationKeywordFieldMapper extends FieldMapper {
+
+    public static final String CONTENT_TYPE = "icu_collation_keyword";
+
+    public static class Defaults {
+        public static final MappedFieldType FIELD_TYPE = new CollationFieldType();
+
+        static {
+            FIELD_TYPE.setTokenized(false);
+            FIELD_TYPE.setOmitNorms(true);
+            FIELD_TYPE.setIndexOptions(IndexOptions.DOCS);
+            FIELD_TYPE.freeze();
+        }
+
+        public static final String NULL_VALUE = null;
+    }
+
+    public static final class CollationFieldType extends StringFieldType {
+        private Collator collator = null;
+
+        public CollationFieldType() {
+            setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
+            setSearchAnalyzer(Lucene.KEYWORD_ANALYZER);
+        }
+
+        protected CollationFieldType(CollationFieldType ref) {
+            super(ref);
+            this.collator = ref.collator;
+        }
+
+        public CollationFieldType clone() {
+            return new CollationFieldType(this);
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            return super.equals(o) && Objects.equals(collator, ((CollationFieldType) o).collator);
+        }
+
+        @Override
+        public void checkCompatibility(MappedFieldType otherFT, List<String> conflicts, boolean strict) {
+            super.checkCompatibility(otherFT, conflicts, strict);
+            CollationFieldType other = (CollationFieldType) otherFT;
+            if (!Objects.equals(collator, other.collator)) {
+                conflicts.add("mapper [" + name() + "] has different [collator]");
+            }
+        }
+
+        @Override
+        public int hashCode() {
+            return 31 * super.hashCode() + Objects.hashCode(collator);
+        }
+
+        @Override
+        public String typeName() {
+            return CONTENT_TYPE;
+        }
+
+        public Collator collator() {
+            return collator;
+        }
+
+        public void setCollator(Collator collator) {
+            checkIfFrozen();
+            this.collator = collator.isFrozen() ? collator : collator.freeze();
+        }
+
+        @Override
+        public Query nullValueQuery() {
+            if (nullValue() == null) {
+                return null;
+            }
+            return termQuery(nullValue(), null);
+        }
+
+        @Override
+        public IndexFieldData.Builder fielddataBuilder() {
+            failIfNoDocValues();
+            return new DocValuesIndexFieldData.Builder();
+        }
+
+        @Override
+        protected BytesRef indexedValueForSearch(Object value) {
+            if (value == null) {
+                return null;
+            }
+            if (value instanceof BytesRef) {
+                value = ((BytesRef) value).utf8ToString();
+            }
+
+            if (collator != null) {
+                RawCollationKey key = collator.getRawCollationKey(value.toString(), null);
+                return new BytesRef(key.bytes, 0, key.size);
+            } else {
+                throw new IllegalStateException("collator is null");
+            }
+        }
+
+        @Override
+        public Query fuzzyQuery(Object value, Fuzziness fuzziness, int prefixLength, int maxExpansions,
+                                boolean transpositions) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, QueryShardContext context) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public Query regexpQuery(String value, int flags, int maxDeterminizedStates,
+                                 MultiTermQuery.RewriteMethod method, QueryShardContext context) {
+            throw new UnsupportedOperationException();
+        }
+
+        public static DocValueFormat COLLATE_FORMAT = new DocValueFormat() {
+            @Override
+            public String getWriteableName() {
+                return "collate";
+            }
+
+            @Override
+            public void writeTo(StreamOutput out) throws IOException {
+            }
+
+            @Override
+            public String format(long value) {
+                throw new UnsupportedOperationException();
+            }
+
+            @Override
+            public String format(double value) {
+                throw new UnsupportedOperationException();
+            }
+
+            @Override
+            public String format(BytesRef value) {
+                int encodedLength = IndexableBinaryStringTools.getEncodedLength(value.bytes, value.offset, value.length);
+                char[] encoded = new char[encodedLength];
+                IndexableBinaryStringTools.encode(value.bytes, value.offset, value.length, encoded, 0, encodedLength);
+                return new String(encoded, 0, encodedLength);
+            }
+
+            @Override
+            public long parseLong(String value, boolean roundUp, LongSupplier now) {
+                throw new UnsupportedOperationException();
+            }
+
+            @Override
+            public double parseDouble(String value, boolean roundUp, LongSupplier now) {
+                throw new UnsupportedOperationException();
+            }
+
+            @Override
+            public BytesRef parseBytesRef(String value) {
+                char[] encoded = value.toCharArray();
+                int decodedLength = IndexableBinaryStringTools.getDecodedLength(encoded, 0, encoded.length);
+                byte[] decoded = new byte[decodedLength];
+                IndexableBinaryStringTools.decode(encoded, 0, encoded.length, decoded, 0, decodedLength);
+                return new BytesRef(decoded);
+            }
+        };
+
+        @Override
+        public DocValueFormat docValueFormat(final String format, final DateTimeZone timeZone) {
+            return COLLATE_FORMAT;
+        }
+    }
+
+    public static class Builder extends FieldMapper.Builder<Builder, ICUCollationKeywordFieldMapper> {
+        private String rules = null;
+        private String language = null;
+        private String country = null;
+        private String variant = null;
+        private String strength = null;
+        private String decomposition = null;
+        private String alternate = null;
+        private boolean caseLevel = false;
+        private String caseFirst = null;
+        private boolean numeric = false;
+        private String variableTop = null;
+        private boolean hiraganaQuaternaryMode = false;
+        private String nullValue = Defaults.NULL_VALUE;
+
+        public Builder(String name) {
+            super(name, Defaults.FIELD_TYPE, Defaults.FIELD_TYPE);
+            builder = this;
+        }
+
+        @Override
+        public CollationFieldType fieldType() {
+            return (CollationFieldType) super.fieldType();
+        }
+
+        @Override
+        public Builder indexOptions(IndexOptions indexOptions) {
+            if (indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS) > 0) {
+                throw new IllegalArgumentException("The [" + CONTENT_TYPE + "] field does not support positions, got [index_options]="
+                    + indexOptionToString(indexOptions));
+            }
+
+            return super.indexOptions(indexOptions);
+        }
+
+        public String rules() {
+            return rules;
+        }
+
+        public Builder rules(final String rules) {
+            this.rules = rules;
+            return this;
+        }
+
+        public String language() {
+            return language;
+        }
+
+        public Builder language(final String language) {
+            this.language = language;
+            return this;
+        }
+
+        public String country() {
+            return country;
+        }
+
+        public Builder country(final String country) {
+            this.country = country;
+            return this;
+        }
+
+        public String variant() {
+            return variant;
+        }
+
+        public Builder variant(final String variant) {
+            this.variant = variant;
+            return this;
+        }
+
+        public String strength() {
+            return strength;
+        }
+
+        public Builder strength(final String strength) {
+            this.strength = strength;
+            return this;
+        }
+
+        public String decomposition() {
+            return decomposition;
+        }
+
+        public Builder decomposition(final String decomposition) {
+            this.decomposition = decomposition;
+            return this;
+        }
+
+        public String alternate() {
+            return alternate;
+        }
+
+        public Builder alternate(final String alternate) {
+            this.alternate = alternate;
+            return this;
+        }
+
+        public boolean caseLevel() {
+            return caseLevel;
+        }
+
+        public Builder caseLevel(final boolean caseLevel) {
+            this.caseLevel = caseLevel;
+            return this;
+        }
+
+        public String caseFirst() {
+            return caseFirst;
+        }
+
+        public Builder caseFirst(final String caseFirst) {
+            this.caseFirst = caseFirst;
+            return this;
+        }
+
+        public boolean numeric() {
+            return numeric;
+        }
+
+        public Builder numeric(final boolean numeric) {
+            this.numeric = numeric;
+            return this;
+        }
+
+        public String variableTop() {
+            return variableTop;
+        }
+
+        public Builder variableTop(final String variableTop) {
+            this.variableTop = variableTop;
+            return this;
+        }
+
+        public boolean hiraganaQuaternaryMode() {
+            return hiraganaQuaternaryMode;
+        }
+
+        public Builder hiraganaQuaternaryMode(final boolean hiraganaQuaternaryMode) {
+            this.hiraganaQuaternaryMode = hiraganaQuaternaryMode;
+            return this;
+        }
+
+        public Collator buildCollator() {
+            Collator collator;
+            if (rules != null) {
+                try {
+                    collator = new RuleBasedCollator(rules);
+                } catch (Exception e) {
+                    throw new IllegalArgumentException("Failed to parse collation rules", e);
+                }
+            } else {
+                if (language != null) {
+                    ULocale locale;
+                    if (country != null) {
+                        if (variant != null) {
+                            locale = new ULocale(language, country, variant);
+                        } else {
+                            locale = new ULocale(language, country);
+                        }
+                    } else {
+                        locale = new ULocale(language);
+                    }
+                    collator = Collator.getInstance(locale);
+                } else {
+                    collator = Collator.getInstance();
+                }
+            }
+
+            // set the strength flag, otherwise it will be the default.
+            if (strength != null) {
+                if (strength.equalsIgnoreCase("primary")) {
+                    collator.setStrength(Collator.PRIMARY);
+                } else if (strength.equalsIgnoreCase("secondary")) {
+                    collator.setStrength(Collator.SECONDARY);
+                } else if (strength.equalsIgnoreCase("tertiary")) {
+                    collator.setStrength(Collator.TERTIARY);
+                } else if (strength.equalsIgnoreCase("quaternary")) {
+                    collator.setStrength(Collator.QUATERNARY);
+                } else if (strength.equalsIgnoreCase("identical")) {
+                    collator.setStrength(Collator.IDENTICAL);
+                } else {
+                    throw new IllegalArgumentException("Invalid strength: " + strength);
+                }
+            }
+
+            // set the decomposition flag, otherwise it will be the default.
+            if (decomposition != null) {
+                if (decomposition.equalsIgnoreCase("no")) {
+                    collator.setDecomposition(Collator.NO_DECOMPOSITION);
+                } else if (decomposition.equalsIgnoreCase("canonical")) {
+                    collator.setDecomposition(Collator.CANONICAL_DECOMPOSITION);
+                } else {
+                    throw new IllegalArgumentException("Invalid decomposition: " + decomposition);
+                }
+            }
+
+            // expert options: concrete subclasses are always a RuleBasedCollator
+            RuleBasedCollator rbc = (RuleBasedCollator) collator;
+            if (alternate != null) {
+                if (alternate.equalsIgnoreCase("shifted")) {
+                    rbc.setAlternateHandlingShifted(true);
+                } else if (alternate.equalsIgnoreCase("non-ignorable")) {
+                    rbc.setAlternateHandlingShifted(false);
+                } else {
+                    throw new IllegalArgumentException("Invalid alternate: " + alternate);
+                }
+            }
+
+            if (caseLevel) {
+                rbc.setCaseLevel(true);
+            }
+
+            if (caseFirst != null) {
+                if (caseFirst.equalsIgnoreCase("lower")) {
+                    rbc.setLowerCaseFirst(true);
+                } else if (caseFirst.equalsIgnoreCase("upper")) {
+                    rbc.setUpperCaseFirst(true);
+                } else {
+                    throw new IllegalArgumentException("Invalid caseFirst: " + caseFirst);
+                }
+            }
+
+            if (numeric) {
+                rbc.setNumericCollation(true);
+            }
+
+            if (variableTop != null) {
+                rbc.setVariableTop(variableTop);
+            }
+
+            if (hiraganaQuaternaryMode) {
+                rbc.setHiraganaQuaternary(true);
+            }
+
+            // freeze so thread-safe
+            return collator.freeze();
+        }
+
+        @Override
+        public ICUCollationKeywordFieldMapper build(BuilderContext context) {
+            final Collator collator = buildCollator();
+            fieldType().setCollator(collator);
+            setupFieldType(context);
+            return new ICUCollationKeywordFieldMapper(name, fieldType, defaultFieldType, context.indexSettings(),
+                multiFieldsBuilder.build(this, context), copyTo, rules, language, country, variant, strength, decomposition,
+                alternate, caseLevel, caseFirst, numeric, variableTop, hiraganaQuaternaryMode, collator);
+        }
+    }
+
+    public static class TypeParser implements Mapper.TypeParser {
+        @Override
+        public Mapper.Builder<?, ?> parse(String name, Map<String, Object> node, ParserContext parserContext)
+            throws MapperParsingException {
+            Builder builder = new Builder(name);
+            TypeParsers.parseField(builder, name, node, parserContext);
+            for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext(); ) {
+                Map.Entry<String, Object> entry = iterator.next();
+                String fieldName = entry.getKey();
+                Object fieldNode = entry.getValue();
+                switch (fieldName) {
+                    case "null_value":
+                        if (fieldNode == null) {
+                            throw new MapperParsingException("Property [null_value] cannot be null.");
+                        }
+                        builder.nullValue(fieldNode.toString());
+                        iterator.remove();
+                        break;
+                    case "norms":
+                        builder.omitNorms(!XContentMapValues.nodeBooleanValue(fieldNode));
+                        iterator.remove();
+                        break;
+                    case "rules":
+                        builder.rules(XContentMapValues.nodeStringValue(fieldNode, null));
+                        iterator.remove();
+                        break;
+                    case "language":
+                        builder.language(XContentMapValues.nodeStringValue(fieldNode, null));
+                        iterator.remove();
+                        break;
+                    case "country":
+                        builder.country(XContentMapValues.nodeStringValue(fieldNode, null));
+                        iterator.remove();
+                        break;
+                    case "variant":
+                        builder.variant(XContentMapValues.nodeStringValue(fieldNode, null));
+                        iterator.remove();
+                        break;
+                    case "strength":
+                        builder.strength(XContentMapValues.nodeStringValue(fieldNode, null));
+                        iterator.remove();
+                        break;
+                    case "decomposition":
+                        builder.decomposition(XContentMapValues.nodeStringValue(fieldNode, null));
+                        iterator.remove();
+                        break;
+                    case "alternate":
+                        builder.alternate(XContentMapValues.nodeStringValue(fieldNode, null));
+                        iterator.remove();
+                        break;
+                    case "case_level":
+                        builder.caseLevel(XContentMapValues.nodeBooleanValue(fieldNode));
+                        iterator.remove();
+                        break;
+                    case "case_first":
+                        builder.caseFirst(XContentMapValues.nodeStringValue(fieldNode, null));
+                        iterator.remove();
+                        break;
+                    case "numeric":
+                        builder.numeric(XContentMapValues.nodeBooleanValue(fieldNode));
+                        iterator.remove();
+                        break;
+                    case "variable_top":
+                        builder.variableTop(XContentMapValues.nodeStringValue(fieldNode, null));
+                        iterator.remove();
+                        break;
+                    case "hiragana_quaternary_mode":
+                        builder.hiraganaQuaternaryMode(XContentMapValues.nodeBooleanValue(fieldNode));
+                        iterator.remove();
+                        break;
+                    default:
+                        break;
+                }
+            }
+
+            return builder;
+        }
+    }
+
+    private final String rules;
+    private final String language;
+    private final String country;
+    private final String variant;
+    private final String strength;
+    private final String decomposition;
+    private final String alternate;
+    private final boolean caseLevel;
+    private final String caseFirst;
+    private final boolean numeric;
+    private final String variableTop;
+    private final boolean hiraganaQuaternaryMode;
+    private final Collator collator;
+    private final BiFunction<String, BytesRef, Field> getDVField;
+
+    protected ICUCollationKeywordFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
+                                             Settings indexSettings, MultiFields multiFields, CopyTo copyTo, String rules, String language,
+                                             String country, String variant,
+                                             String strength, String decomposition, String alternate, boolean caseLevel, String caseFirst,
+                                             boolean numeric, String variableTop, boolean hiraganaQuaternaryMode, Collator collator) {
+        super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo);
+        assert collator.isFrozen();
+        this.rules = rules;
+        this.language = language;
+        this.country = country;
+        this.variant = variant;
+        this.strength = strength;
+        this.decomposition = decomposition;
+        this.alternate = alternate;
+        this.caseLevel = caseLevel;
+        this.caseFirst = caseFirst;
+        this.numeric = numeric;
+        this.variableTop = variableTop;
+        this.hiraganaQuaternaryMode = hiraganaQuaternaryMode;
+        this.collator = collator;
+        if (indexCreatedVersion.onOrAfter(Version.V_5_6_0)) {
+            getDVField = SortedSetDocValuesField::new;
+        } else {
+            getDVField = SortedDocValuesField::new;
+        }
+    }
+
+    @Override
+    public CollationFieldType fieldType() {
+        return (CollationFieldType) super.fieldType();
+    }
+
+    @Override
+    protected String contentType() {
+        return CONTENT_TYPE;
+    }
+
+    @Override
+    protected void doMerge(Mapper mergeWith, boolean updateAllTypes) {
+        super.doMerge(mergeWith, updateAllTypes);
+
+        List<String> conflicts = new ArrayList<>();
+        ICUCollationKeywordFieldMapper icuMergeWith = (ICUCollationKeywordFieldMapper) mergeWith;
+
+        if (!Objects.equals(rules, icuMergeWith.rules)) {
+            conflicts.add("Cannot update rules setting for [" + CONTENT_TYPE + "]");
+        }
+
+        if (!Objects.equals(language, icuMergeWith.language)) {
+            conflicts.add("Cannot update language setting for [" + CONTENT_TYPE + "]");
+        }
+
+        if (!Objects.equals(country, icuMergeWith.country)) {
+            conflicts.add("Cannot update country setting for [" + CONTENT_TYPE + "]");
+        }
+
+        if (!Objects.equals(variant, icuMergeWith.variant)) {
+            conflicts.add("Cannot update variant setting for [" + CONTENT_TYPE + "]");
+        }
+
+        if (!Objects.equals(strength, icuMergeWith.strength)) {
+            conflicts.add("Cannot update strength setting for [" + CONTENT_TYPE + "]");
+        }
+
+        if (!Objects.equals(decomposition, icuMergeWith.decomposition)) {
+            conflicts.add("Cannot update decomposition setting for [" + CONTENT_TYPE + "]");
+        }
+
+        if (!Objects.equals(alternate, icuMergeWith.alternate)) {
+            conflicts.add("Cannot update alternate setting for [" + CONTENT_TYPE + "]");
+        }
+
+        if (caseLevel != icuMergeWith.caseLevel) {
+            conflicts.add("Cannot update case_level setting for [" + CONTENT_TYPE + "]");
+        }
+
+        if (!Objects.equals(caseFirst, icuMergeWith.caseFirst)) {
+            conflicts.add("Cannot update case_first setting for [" + CONTENT_TYPE + "]");
+        }
+
+        if (numeric != icuMergeWith.numeric) {
+            conflicts.add("Cannot update numeric setting for [" + CONTENT_TYPE + "]");
+        }
+
+        if (!Objects.equals(variableTop, icuMergeWith.variableTop)) {
+            conflicts.add("Cannot update variable_top setting for [" + CONTENT_TYPE + "]");
+        }
+
+        if (hiraganaQuaternaryMode != icuMergeWith.hiraganaQuaternaryMode) {
+            conflicts.add("Cannot update hiragana_quaternary_mode setting for [" + CONTENT_TYPE + "]");
+        }
+
+        if (!conflicts.isEmpty()) {
+            throw new IllegalArgumentException("Can't merge because of conflicts: " + conflicts);
+        }
+    }
+
+    @Override
+    protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException {
+        super.doXContentBody(builder, includeDefaults, params);
+
+        if (includeDefaults || fieldType().nullValue() != null) {
+            builder.field("null_value", fieldType().nullValue());
+        }
+
+        if (includeDefaults || rules != null) {
+            builder.field("rules", rules);
+        }
+
+        if (includeDefaults || language != null) {
+            builder.field("language", language);
+        }
+
+        if (includeDefaults || country != null) {
+            builder.field("country", country);
+        }
+
+        if (includeDefaults || variant != null) {
+            builder.field("variant", variant);
+        }
+
+        if (includeDefaults || strength != null) {
+            builder.field("strength", strength);
+        }
+
+        if (includeDefaults || decomposition != null) {
+            builder.field("decomposition", decomposition);
+        }
+
+        if (includeDefaults || alternate != null) {
+            builder.field("alternate", alternate);
+        }
+
+        if (includeDefaults || caseLevel) {
+            builder.field("case_level", caseLevel);
+        }
+
+        if (includeDefaults || caseFirst != null) {
+            builder.field("case_first", caseFirst);
+        }
+
+        if (includeDefaults || numeric) {
+            builder.field("numeric", numeric);
+        }
+
+        if (includeDefaults || variableTop != null) {
+            builder.field("variable_top", variableTop);
+        }
+
+        if (includeDefaults || hiraganaQuaternaryMode) {
+            builder.field("hiragana_quaternary_mode", hiraganaQuaternaryMode);
+        }
+    }
+
+    @Override
+    protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
+        final String value;
+        if (context.externalValueSet()) {
+            value = context.externalValue().toString();
+        } else {
+            XContentParser parser = context.parser();
+            if (parser.currentToken() == XContentParser.Token.VALUE_NULL) {
+                value = fieldType().nullValueAsString();
+            } else {
+                value = parser.textOrNull();
+            }
+        }
+
+        if (value == null) {
+            return;
+        }
+
+        RawCollationKey key = collator.getRawCollationKey(value, null);
+        final BytesRef binaryValue = new BytesRef(key.bytes, 0, key.size);
+
+        if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
+            Field field = new Field(fieldType().name(), binaryValue, fieldType());
+            fields.add(field);
+        }
+
+        if (fieldType().hasDocValues()) {
+            fields.add(getDVField.apply(fieldType().name(), binaryValue));
+        }
+    }
+}
diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/AnalysisICUPlugin.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/AnalysisICUPlugin.java
index 059dabb4f4625..58ebdc8e2a801 100644
--- a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/AnalysisICUPlugin.java
+++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/AnalysisICUPlugin.java
@@ -19,6 +19,9 @@
 
 package org.elasticsearch.plugin.analysis.icu;
 
+import static java.util.Collections.singletonMap;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.index.analysis.CharFilterFactory;
 import org.elasticsearch.index.analysis.IcuCollationTokenFilterFactory;
 import org.elasticsearch.index.analysis.IcuFoldingTokenFilterFactory;
@@ -28,16 +31,20 @@
 import org.elasticsearch.index.analysis.IcuTransformTokenFilterFactory;
 import org.elasticsearch.index.analysis.TokenFilterFactory;
 import org.elasticsearch.index.analysis.TokenizerFactory;
+import org.elasticsearch.index.mapper.ICUCollationKeywordFieldMapper;
+import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider;
 import org.elasticsearch.plugins.AnalysisPlugin;
+import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.search.DocValueFormat;
 
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
-import static java.util.Collections.singletonMap;
-
-public class AnalysisICUPlugin extends Plugin implements AnalysisPlugin {
+public class AnalysisICUPlugin extends Plugin implements AnalysisPlugin, MapperPlugin {
     @Override
     public Map<String, AnalysisProvider<CharFilterFactory>> getCharFilters() {
         return singletonMap("icu_normalizer", IcuNormalizerCharFilterFactory::new);
@@ -57,4 +64,20 @@ public Map<String, AnalysisProvider<TokenFilterFactory>> getTokenFilters() {
     public Map<String, AnalysisProvider<TokenizerFactory>> getTokenizers() {
         return singletonMap("icu_tokenizer", IcuTokenizerFactory::new);
     }
+
+    @Override
+    public Map<String, Mapper.TypeParser> getMappers() {
+        return Collections.singletonMap(ICUCollationKeywordFieldMapper.CONTENT_TYPE, new ICUCollationKeywordFieldMapper.TypeParser());
+    }
+
+    @Override
+    public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
+        return Collections.singletonList(
+            new NamedWriteableRegistry.Entry(
+                DocValueFormat.class,
+                ICUCollationKeywordFieldMapper.CollationFieldType.COLLATE_FORMAT.getWriteableName(),
+                in -> ICUCollationKeywordFieldMapper.CollationFieldType.COLLATE_FORMAT
+            )
+        );
+    }
 }
diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuClientYamlTestSuiteIT.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuClientYamlTestSuiteIT.java
index 472248360377c..0d06f41972415 100644
--- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuClientYamlTestSuiteIT.java
+++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuClientYamlTestSuiteIT.java
@@ -24,9 +24,6 @@
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-
-import java.io.IOException;
 
 public class IcuClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
@@ -35,8 +32,8 @@ public IcuClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandid
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
-        return ESClientYamlSuiteTestCase.createParameters(0, 1);
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
     }
 }
 
diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuTokenizerFactoryTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuTokenizerFactoryTests.java
index 180c4268612c2..716d07385b80a 100644
--- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuTokenizerFactoryTests.java
+++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuTokenizerFactoryTests.java
@@ -42,9 +42,9 @@
 public class IcuTokenizerFactoryTests extends ESTestCase {
 
     public void testSimpleIcuTokenizer() throws IOException {
-        AnalysisService analysisService = createAnalysisService();
+        TestAnalysis analysis = createTestAnalysis();
 
-        TokenizerFactory tokenizerFactory = analysisService.tokenizer("icu_tokenizer");
+        TokenizerFactory tokenizerFactory = analysis.tokenizer.get("icu_tokenizer");
         ICUTokenizer tokenizer = (ICUTokenizer) tokenizerFactory.create();
 
         Reader reader = new StringReader("向日葵, one-two");
@@ -53,10 +53,10 @@ public void testSimpleIcuTokenizer() throws IOException {
     }
 
     public void testIcuCustomizeRuleFile() throws IOException {
-        AnalysisService analysisService = createAnalysisService();
+        TestAnalysis analysis = createTestAnalysis();
 
         // test the tokenizer with single rule file
-        TokenizerFactory tokenizerFactory = analysisService.tokenizer("user_rule_tokenizer");
+        TokenizerFactory tokenizerFactory = analysis.tokenizer.get("user_rule_tokenizer");
         ICUTokenizer tokenizer = (ICUTokenizer) tokenizerFactory.create();
         Reader reader = new StringReader
             ("One-two punch.  Brang-, not brung-it.  This one--not that one--is the right one, -ish.");
@@ -68,10 +68,10 @@ public void testIcuCustomizeRuleFile() throws IOException {
     }
 
     public void testMultipleIcuCustomizeRuleFiles() throws IOException {
-        AnalysisService analysisService = createAnalysisService();
+        TestAnalysis analysis = createTestAnalysis();
 
         // test the tokenizer with two rule files
-        TokenizerFactory tokenizerFactory = analysisService.tokenizer("multi_rule_tokenizer");
+        TokenizerFactory tokenizerFactory = analysis.tokenizer.get("multi_rule_tokenizer");
         ICUTokenizer tokenizer = (ICUTokenizer) tokenizerFactory.create();
         StringReader reader = new StringReader
             ("Some English.  Немного русский.  ข้อความภาษาไทยเล็ก ๆ น้อย ๆ  More English.");
@@ -84,7 +84,7 @@ public void testMultipleIcuCustomizeRuleFiles() throws IOException {
     }
 
 
-    private static AnalysisService createAnalysisService() throws IOException {
+    private static TestAnalysis createTestAnalysis() throws IOException {
         InputStream keywords = IcuTokenizerFactoryTests.class.getResourceAsStream("KeywordTokenizer.rbbi");
         InputStream latin = IcuTokenizerFactoryTests.class.getResourceAsStream("Latin-dont-break-on-hyphens.rbbi");
 
@@ -102,6 +102,6 @@ private static AnalysisService createAnalysisService() throws IOException {
             .build();
         Settings nodeSettings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), home).build();
 
-        return createAnalysisService(new Index("test", "_na_"), nodeSettings, settings, new AnalysisICUPlugin());
+        return createTestAnalysis(new Index("test", "_na_"), nodeSettings, settings, new AnalysisICUPlugin());
     }
 }
diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuAnalysisTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuAnalysisTests.java
index 9255a250f1609..86338e0670da5 100644
--- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuAnalysisTests.java
+++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuAnalysisTests.java
@@ -31,24 +31,24 @@
  */
 public class SimpleIcuAnalysisTests extends ESTestCase {
     public void testDefaultsIcuAnalysis() throws IOException {
-        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), Settings.EMPTY, new AnalysisICUPlugin());
+        TestAnalysis analysis = createTestAnalysis(new Index("test", "_na_"), Settings.EMPTY, new AnalysisICUPlugin());
 
-        TokenizerFactory tokenizerFactory = analysisService.tokenizer("icu_tokenizer");
+        TokenizerFactory tokenizerFactory = analysis.tokenizer.get("icu_tokenizer");
         assertThat(tokenizerFactory, instanceOf(IcuTokenizerFactory.class));
 
-        TokenFilterFactory filterFactory = analysisService.tokenFilter("icu_normalizer");
+        TokenFilterFactory filterFactory = analysis.tokenFilter.get("icu_normalizer");
         assertThat(filterFactory, instanceOf(IcuNormalizerTokenFilterFactory.class));
 
-        filterFactory = analysisService.tokenFilter("icu_folding");
+        filterFactory = analysis.tokenFilter.get("icu_folding");
         assertThat(filterFactory, instanceOf(IcuFoldingTokenFilterFactory.class));
 
-        filterFactory = analysisService.tokenFilter("icu_collation");
+        filterFactory = analysis.tokenFilter.get("icu_collation");
         assertThat(filterFactory, instanceOf(IcuCollationTokenFilterFactory.class));
 
-        filterFactory = analysisService.tokenFilter("icu_transform");
+        filterFactory = analysis.tokenFilter.get("icu_transform");
         assertThat(filterFactory, instanceOf(IcuTransformTokenFilterFactory.class));
 
-        CharFilterFactory charFilterFactory = analysisService.charFilter("icu_normalizer");
+        CharFilterFactory charFilterFactory = analysis.charFilter.get("icu_normalizer");
         assertThat(charFilterFactory, instanceOf(IcuNormalizerCharFilterFactory.class));
     }
 }
diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuCollationTokenFilterTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuCollationTokenFilterTests.java
index 62e9c9db145f8..8f9a38dc8f18c 100644
--- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuCollationTokenFilterTests.java
+++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuCollationTokenFilterTests.java
@@ -50,9 +50,9 @@ public void testBasicUsage() throws Exception {
                 .put("index.analysis.filter.myCollator.language", "tr")
                 .put("index.analysis.filter.myCollator.strength", "primary")
                 .build();
-        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin());
+        TestAnalysis analysis = createTestAnalysis(new Index("test", "_na_"), settings, new AnalysisICUPlugin());
 
-        TokenFilterFactory filterFactory = analysisService.tokenFilter("myCollator");
+        TokenFilterFactory filterFactory = analysis.tokenFilter.get("myCollator");
         assertCollatesToSame(filterFactory, "I WİLL USE TURKİSH CASING", "ı will use turkish casıng");
     }
 
@@ -66,9 +66,9 @@ public void testNormalization() throws IOException {
                 .put("index.analysis.filter.myCollator.strength", "primary")
                 .put("index.analysis.filter.myCollator.decomposition", "canonical")
                 .build();
-        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin());
+        TestAnalysis analysis = createTestAnalysis(new Index("test", "_na_"), settings, new AnalysisICUPlugin());
 
-        TokenFilterFactory filterFactory = analysisService.tokenFilter("myCollator");
+        TokenFilterFactory filterFactory = analysis.tokenFilter.get("myCollator");
         assertCollatesToSame(filterFactory, "I W\u0049\u0307LL USE TURKİSH CASING", "ı will use turkish casıng");
     }
 
@@ -82,9 +82,9 @@ public void testSecondaryStrength() throws IOException {
                 .put("index.analysis.filter.myCollator.strength", "secondary")
                 .put("index.analysis.filter.myCollator.decomposition", "no")
                 .build();
-        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin());
+        TestAnalysis analysis = createTestAnalysis(new Index("test", "_na_"), settings, new AnalysisICUPlugin());
 
-        TokenFilterFactory filterFactory = analysisService.tokenFilter("myCollator");
+        TokenFilterFactory filterFactory = analysis.tokenFilter.get("myCollator");
         assertCollatesToSame(filterFactory, "TESTING", "testing");
     }
 
@@ -99,9 +99,9 @@ public void testIgnorePunctuation() throws IOException {
                 .put("index.analysis.filter.myCollator.strength", "primary")
                 .put("index.analysis.filter.myCollator.alternate", "shifted")
                 .build();
-        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin());
+        TestAnalysis analysis = createTestAnalysis(new Index("test", "_na_"), settings, new AnalysisICUPlugin());
 
-        TokenFilterFactory filterFactory = analysisService.tokenFilter("myCollator");
+        TokenFilterFactory filterFactory = analysis.tokenFilter.get("myCollator");
         assertCollatesToSame(filterFactory, "foo-bar", "foo bar");
     }
 
@@ -117,9 +117,9 @@ public void testIgnoreWhitespace() throws IOException {
                 .put("index.analysis.filter.myCollator.alternate", "shifted")
                 .put("index.analysis.filter.myCollator.variableTop", " ")
                 .build();
-        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin());
+        TestAnalysis analysis = createTestAnalysis(new Index("test", "_na_"), settings, new AnalysisICUPlugin());
 
-        TokenFilterFactory filterFactory = analysisService.tokenFilter("myCollator");
+        TokenFilterFactory filterFactory = analysis.tokenFilter.get("myCollator");
         assertCollatesToSame(filterFactory, "foo bar", "foobar");
         // now assert that punctuation still matters: foo-bar < foo bar
         assertCollation(filterFactory, "foo-bar", "foo bar", -1);
@@ -135,9 +135,9 @@ public void testNumerics() throws IOException {
                 .put("index.analysis.filter.myCollator.language", "en")
                 .put("index.analysis.filter.myCollator.numeric", "true")
                 .build();
-        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin());
+        TestAnalysis analysis = createTestAnalysis(new Index("test", "_na_"), settings, new AnalysisICUPlugin());
 
-        TokenFilterFactory filterFactory = analysisService.tokenFilter("myCollator");
+        TokenFilterFactory filterFactory = analysis.tokenFilter.get("myCollator");
         assertCollation(filterFactory, "foobar-9", "foobar-10", -1);
     }
 
@@ -152,9 +152,9 @@ public void testIgnoreAccentsButNotCase() throws IOException {
                 .put("index.analysis.filter.myCollator.strength", "primary")
                 .put("index.analysis.filter.myCollator.caseLevel", "true")
                 .build();
-        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin());
+        TestAnalysis analysis = createTestAnalysis(new Index("test", "_na_"), settings, new AnalysisICUPlugin());
 
-        TokenFilterFactory filterFactory = analysisService.tokenFilter("myCollator");
+        TokenFilterFactory filterFactory = analysis.tokenFilter.get("myCollator");
         assertCollatesToSame(filterFactory, "résumé", "resume");
         assertCollatesToSame(filterFactory, "Résumé", "Resume");
         // now assert that case still matters: resume < Resume
@@ -172,9 +172,9 @@ public void testUpperCaseFirst() throws IOException {
                 .put("index.analysis.filter.myCollator.strength", "tertiary")
                 .put("index.analysis.filter.myCollator.caseFirst", "upper")
                 .build();
-        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin());
+        TestAnalysis analysis = createTestAnalysis(new Index("test", "_na_"), settings, new AnalysisICUPlugin());
 
-        TokenFilterFactory filterFactory = analysisService.tokenFilter("myCollator");
+        TokenFilterFactory filterFactory = analysis.tokenFilter.get("myCollator");
         assertCollation(filterFactory, "Resume", "resume", -1);
     }
 
@@ -200,9 +200,9 @@ public void testCustomRules() throws Exception {
                 .put("index.analysis.filter.myCollator.rules", tailoredRules)
                 .put("index.analysis.filter.myCollator.strength", "primary")
                 .build();
-        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin());
+        TestAnalysis analysis = createTestAnalysis(new Index("test", "_na_"), settings, new AnalysisICUPlugin());
 
-        TokenFilterFactory filterFactory = analysisService.tokenFilter("myCollator");
+        TokenFilterFactory filterFactory = analysis.tokenFilter.get("myCollator");
         assertCollatesToSame(filterFactory, "Töne", "Toene");
     }
 
diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuNormalizerCharFilterTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuNormalizerCharFilterTests.java
index b82accf0cf84d..32cbabfc9afd9 100644
--- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuNormalizerCharFilterTests.java
+++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuNormalizerCharFilterTests.java
@@ -37,8 +37,8 @@ public void testDefaultSetting() throws Exception {
         Settings settings = Settings.builder()
             .put("index.analysis.char_filter.myNormalizerChar.type", "icu_normalizer")
             .build();
-        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin());
-        CharFilterFactory charFilterFactory = analysisService.charFilter("myNormalizerChar");
+        TestAnalysis analysis = createTestAnalysis(new Index("test", "_na_"), settings, new AnalysisICUPlugin());
+        CharFilterFactory charFilterFactory = analysis.charFilter.get("myNormalizerChar");
 
         String input = "ʰ㌰゙5℃№㈱㌘，バッファーの正規化のテスト．㋐㋑㋒㋓㋔ｶｷｸｹｺｻﾞｼﾞｽﾞｾﾞｿﾞg̈각/각நிเกषिchkʷक्षि";
         Normalizer2 normalizer = Normalizer2.getInstance(null, "nfkc_cf", Normalizer2.Mode.COMPOSE);
@@ -61,8 +61,8 @@ public void testNameAndModeSetting() throws Exception {
             .put("index.analysis.char_filter.myNormalizerChar.name", "nfkc")
             .put("index.analysis.char_filter.myNormalizerChar.mode", "decompose")
             .build();
-        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin());
-        CharFilterFactory charFilterFactory = analysisService.charFilter("myNormalizerChar");
+        TestAnalysis analysis = createTestAnalysis(new Index("test", "_na_"), settings, new AnalysisICUPlugin());
+        CharFilterFactory charFilterFactory = analysis.charFilter.get("myNormalizerChar");
 
         String input = "ʰ㌰゙5℃№㈱㌘，バッファーの正規化のテスト．㋐㋑㋒㋓㋔ｶｷｸｹｺｻﾞｼﾞｽﾞｾﾞｿﾞg̈각/각நிเกषिchkʷक्षि";
         Normalizer2 normalizer = Normalizer2.getInstance(null, "nfkc", Normalizer2.Mode.DECOMPOSE);
diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/CollationFieldTypeTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/CollationFieldTypeTests.java
new file mode 100644
index 0000000000000..94634fc79c893
--- /dev/null
+++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/CollationFieldTypeTests.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.index.mapper;
+
+import com.carrotsearch.randomizedtesting.generators.RandomStrings;
+import com.ibm.icu.text.Collator;
+import com.ibm.icu.text.RawCollationKey;
+import com.ibm.icu.util.ULocale;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.TermInSetQuery;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TermRangeQuery;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.unit.Fuzziness;
+import org.elasticsearch.index.mapper.ICUCollationKeywordFieldMapper.CollationFieldType;
+import org.elasticsearch.index.mapper.MappedFieldType.Relation;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+public class CollationFieldTypeTests extends FieldTypeTestCase {
+    @Override
+    protected MappedFieldType createDefaultFieldType() {
+        return new CollationFieldType();
+    }
+
+    public void testIsFieldWithinQuery() throws IOException {
+        CollationFieldType ft = new CollationFieldType();
+        // current impl ignores args and shourd always return INTERSECTS
+        assertEquals(Relation.INTERSECTS, ft.isFieldWithinQuery(null,
+            RandomStrings.randomAsciiOfLengthBetween(random(), 0, 5),
+            RandomStrings.randomAsciiOfLengthBetween(random(), 0, 5),
+            randomBoolean(), randomBoolean(), null, null, null));
+    }
+
+    public void testTermQuery() {
+        MappedFieldType ft = createDefaultFieldType();
+        ft.setName("field");
+        ft.setIndexOptions(IndexOptions.DOCS);
+
+        Collator collator = Collator.getInstance(new ULocale("tr"));
+        collator.setStrength(Collator.PRIMARY);
+        collator.freeze();
+        ((CollationFieldType) ft).setCollator(collator);
+
+        RawCollationKey key = collator.getRawCollationKey("ı will use turkish casıng", null);
+        BytesRef expected = new BytesRef(key.bytes, 0, key.size);
+
+        assertEquals(new TermQuery(new Term("field", expected)), ft.termQuery("I WİLL USE TURKİSH CASING", null));
+
+        ft.setIndexOptions(IndexOptions.NONE);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> ft.termQuery("bar", null));
+        assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage());
+    }
+
+    public void testTermsQuery() {
+        MappedFieldType ft = createDefaultFieldType();
+        ft.setName("field");
+        ft.setIndexOptions(IndexOptions.DOCS);
+
+        Collator collator = Collator.getInstance().freeze();
+        ((CollationFieldType) ft).setCollator(collator);
+
+        RawCollationKey fooKey = collator.getRawCollationKey("foo", null);
+        RawCollationKey barKey = collator.getRawCollationKey("bar", null);
+
+        List<BytesRef> terms = new ArrayList<>();
+        terms.add(new BytesRef(fooKey.bytes, 0, fooKey.size));
+        terms.add(new BytesRef(barKey.bytes, 0, barKey.size));
+
+        assertEquals(new TermInSetQuery("field", terms),
+            ft.termsQuery(Arrays.asList("foo", "bar"), null));
+
+        ft.setIndexOptions(IndexOptions.NONE);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> ft.termsQuery(Arrays.asList("foo", "bar"), null));
+        assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage());
+    }
+
+    public void testRegexpQuery() {
+        MappedFieldType ft = createDefaultFieldType();
+        ft.setName("field");
+        ft.setIndexOptions(IndexOptions.DOCS);
+        expectThrows(UnsupportedOperationException.class,
+            () -> ft.regexpQuery("foo.*", 0, 10, null, null));
+    }
+
+    public void testFuzzyQuery() {
+        MappedFieldType ft = createDefaultFieldType();
+        ft.setName("field");
+        ft.setIndexOptions(IndexOptions.DOCS);
+        expectThrows(UnsupportedOperationException.class,
+            () -> ft.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true));
+    }
+
+    public void testPrefixQuery() {
+        MappedFieldType ft = createDefaultFieldType();
+        ft.setName("field");
+        ft.setIndexOptions(IndexOptions.DOCS);
+        expectThrows(UnsupportedOperationException.class,
+            () -> ft.prefixQuery("prefix", null, null));
+    }
+
+    public void testRangeQuery() {
+        MappedFieldType ft = createDefaultFieldType();
+        ft.setName("field");
+        ft.setIndexOptions(IndexOptions.DOCS);
+
+        Collator collator = Collator.getInstance().freeze();
+        ((CollationFieldType) ft).setCollator(collator);
+
+        RawCollationKey aKey = collator.getRawCollationKey("a", null);
+        RawCollationKey bKey = collator.getRawCollationKey("b", null);
+
+        TermRangeQuery expected = new TermRangeQuery("field", new BytesRef(aKey.bytes, 0, aKey.size),
+            new BytesRef(bKey.bytes, 0, bKey.size), false, false);
+
+        assertEquals(expected, ft.rangeQuery("a", "b", false, false, null));
+
+        ft.setIndexOptions(IndexOptions.NONE);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> ft.rangeQuery("a", "b", false, false, null));
+        assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage());
+    }
+}
diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperIT.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperIT.java
new file mode 100644
index 0000000000000..5220d44dca308
--- /dev/null
+++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperIT.java
@@ -0,0 +1,503 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.index.mapper;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertOrderedSearchHits;
+
+import com.ibm.icu.text.Collator;
+import com.ibm.icu.text.RuleBasedCollator;
+import com.ibm.icu.util.ULocale;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.plugin.analysis.icu.AnalysisICUPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.sort.SortBuilders;
+import org.elasticsearch.search.sort.SortMode;
+import org.elasticsearch.search.sort.SortOrder;
+import org.elasticsearch.test.ESIntegTestCase;
+
+import java.util.Collection;
+import java.util.Collections;
+
+public class ICUCollationKeywordFieldMapperIT extends ESIntegTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Collections.singletonList(AnalysisICUPlugin.class);
+    }
+
+    /*
+    * Turkish has some funny casing.
+    * This test shows how you can solve this kind of thing easily with collation.
+    * Instead of using LowerCaseFilter, use a turkish collator with primary strength.
+    * Then things will sort and match correctly.
+    */
+    public void testBasicUsage() throws Exception {
+        String index = "foo";
+        String type = "mytype";
+
+        String[] equilavent = {"I WİLL USE TURKİSH CASING", "ı will use turkish casıng"};
+
+        XContentBuilder builder = jsonBuilder()
+            .startObject().startObject("properties")
+            .startObject("collate")
+            .field("type", "icu_collation_keyword")
+            .field("language", "tr")
+            .field("strength", "primary")
+            .endObject()
+            .endObject().endObject();
+
+        assertAcked(client().admin().indices().prepareCreate(index).addMapping(type, builder));
+
+        // both values should collate to same value
+        indexRandom(true,
+            client().prepareIndex(index, type, "1").setSource("{\"collate\":\"" + equilavent[0] + "\"}", XContentType.JSON),
+            client().prepareIndex(index, type, "2").setSource("{\"collate\":\"" + equilavent[1] + "\"}", XContentType.JSON)
+        );
+
+        // searching for either of the terms should return both results since they collate to the same value
+        SearchRequest request = new SearchRequest()
+            .indices(index)
+            .types(type)
+            .source(new SearchSourceBuilder()
+                .fetchSource(false)
+                .query(QueryBuilders.termQuery("collate", randomBoolean() ? equilavent[0] : equilavent[1]))
+                .sort("collate")
+                .sort("_uid", SortOrder.DESC) // secondary sort should kick in because both will collate to same value
+            );
+
+        SearchResponse response = client().search(request).actionGet();
+        assertNoFailures(response);
+        assertHitCount(response, 2L);
+        assertOrderedSearchHits(response, "2", "1");
+    }
+
+    public void testMultipleValues() throws Exception {
+        String index = "foo";
+        String type = "mytype";
+
+        String[] equilavent = {"a", "C", "a", "B"};
+
+        XContentBuilder builder = jsonBuilder()
+            .startObject().startObject("properties")
+            .startObject("collate")
+            .field("type", "icu_collation_keyword")
+            .field("language", "en")
+            .endObject()
+            .endObject().endObject();
+
+        assertAcked(client().admin().indices().prepareCreate(index).addMapping(type, builder));
+
+        // everything should be indexed fine, no exceptions
+        indexRandom(true,
+            client().prepareIndex(index, type, "1").setSource("{\"collate\":[\"" + equilavent[0] + "\", \""
+                + equilavent[1] + "\"]}", XContentType.JSON),
+            client().prepareIndex(index, type, "2").setSource("{\"collate\":\"" + equilavent[2] + "\"}", XContentType.JSON)
+        );
+
+        // using sort mode = max, values B and C will be used for the sort
+        SearchRequest request = new SearchRequest()
+            .indices(index)
+            .types(type)
+            .source(new SearchSourceBuilder()
+                .fetchSource(false)
+                .query(QueryBuilders.termQuery("collate", "a"))
+                // if mode max we use c and b as sort values, if max we use "a" for both
+                .sort(SortBuilders.fieldSort("collate").sortMode(SortMode.MAX).order(SortOrder.DESC))
+                .sort("_uid", SortOrder.DESC) // will be ignored
+            );
+
+        SearchResponse response = client().search(request).actionGet();
+        assertNoFailures(response);
+        assertHitCount(response, 2L);
+        assertOrderedSearchHits(response, "1", "2");
+
+        // same thing, using different sort mode that will use a for both docs
+        request = new SearchRequest()
+            .indices(index)
+            .types(type)
+            .source(new SearchSourceBuilder()
+                .fetchSource(false)
+                .query(QueryBuilders.termQuery("collate", "a"))
+                // if mode max we use c and b as sort values, if max we use "a" for both
+                .sort(SortBuilders.fieldSort("collate").sortMode(SortMode.MIN).order(SortOrder.DESC))
+                .sort("_uid", SortOrder.DESC) // will NOT be ignored and will determine order
+            );
+
+        response = client().search(request).actionGet();
+        assertNoFailures(response);
+        assertHitCount(response, 2L);
+        assertOrderedSearchHits(response, "2", "1");
+    }
+
+    /*
+    * Test usage of the decomposition option for unicode normalization.
+    */
+    public void testNormalization() throws Exception {
+        String index = "foo";
+        String type = "mytype";
+
+        String[] equilavent = {"I W\u0049\u0307LL USE TURKİSH CASING", "ı will use turkish casıng"};
+
+        XContentBuilder builder = jsonBuilder()
+            .startObject().startObject("properties")
+            .startObject("collate")
+            .field("type", "icu_collation_keyword")
+            .field("language", "tr")
+            .field("strength", "primary")
+            .field("decomposition", "canonical")
+            .endObject()
+            .endObject().endObject();
+
+        assertAcked(client().admin().indices().prepareCreate(index).addMapping(type, builder));
+
+        indexRandom(true,
+            client().prepareIndex(index, type, "1").setSource("{\"collate\":\"" + equilavent[0] + "\"}", XContentType.JSON),
+            client().prepareIndex(index, type, "2").setSource("{\"collate\":\"" + equilavent[1] + "\"}", XContentType.JSON)
+        );
+
+        // searching for either of the terms should return both results since they collate to the same value
+        SearchRequest request = new SearchRequest()
+            .indices(index)
+            .types(type)
+            .source(new SearchSourceBuilder()
+                .fetchSource(false)
+                .query(QueryBuilders.termQuery("collate", randomBoolean() ? equilavent[0] : equilavent[1]))
+                .sort("collate")
+                .sort("_uid", SortOrder.DESC) // secondary sort should kick in because both will collate to same value
+            );
+
+        SearchResponse response = client().search(request).actionGet();
+        assertNoFailures(response);
+        assertHitCount(response, 2L);
+        assertOrderedSearchHits(response, "2", "1");
+    }
+
+    /*
+    * Test secondary strength, for english case is not significant.
+    */
+    public void testSecondaryStrength() throws Exception {
+        String index = "foo";
+        String type = "mytype";
+
+        String[] equilavent = {"TESTING", "testing"};
+
+        XContentBuilder builder = jsonBuilder()
+            .startObject().startObject("properties")
+            .startObject("collate")
+            .field("type", "icu_collation_keyword")
+            .field("language", "en")
+            .field("strength", "secondary")
+            .field("decomposition", "no")
+            .endObject()
+            .endObject().endObject();
+
+        assertAcked(client().admin().indices().prepareCreate(index).addMapping(type, builder));
+
+        indexRandom(true,
+            client().prepareIndex(index, type, "1").setSource("{\"collate\":\"" + equilavent[0] + "\"}", XContentType.JSON),
+            client().prepareIndex(index, type, "2").setSource("{\"collate\":\"" + equilavent[1] + "\"}", XContentType.JSON)
+        );
+
+        SearchRequest request = new SearchRequest()
+            .indices(index)
+            .types(type)
+            .source(new SearchSourceBuilder()
+                .fetchSource(false)
+                .query(QueryBuilders.termQuery("collate", randomBoolean() ? equilavent[0] : equilavent[1]))
+                .sort("collate")
+                .sort("_uid", SortOrder.DESC) // secondary sort should kick in because both will collate to same value
+            );
+
+        SearchResponse response = client().search(request).actionGet();
+        assertNoFailures(response);
+        assertHitCount(response, 2L);
+        assertOrderedSearchHits(response, "2", "1");
+    }
+
+    /*
+    * Setting alternate=shifted to shift whitespace, punctuation and symbols
+    * to quaternary level
+    */
+    public void testIgnorePunctuation() throws Exception {
+        String index = "foo";
+        String type = "mytype";
+
+        String[] equilavent = {"foo-bar", "foo bar"};
+
+        XContentBuilder builder = jsonBuilder()
+            .startObject().startObject("properties")
+            .startObject("collate")
+            .field("type", "icu_collation_keyword")
+            .field("language", "en")
+            .field("strength", "primary")
+            .field("alternate", "shifted")
+            .endObject()
+            .endObject().endObject();
+
+        assertAcked(client().admin().indices().prepareCreate(index).addMapping(type, builder));
+
+        indexRandom(true,
+            client().prepareIndex(index, type, "1").setSource("{\"collate\":\"" + equilavent[0] + "\"}", XContentType.JSON),
+            client().prepareIndex(index, type, "2").setSource("{\"collate\":\"" + equilavent[1] + "\"}", XContentType.JSON)
+        );
+
+        SearchRequest request = new SearchRequest()
+            .indices(index)
+            .types(type)
+            .source(new SearchSourceBuilder()
+                .fetchSource(false)
+                .query(QueryBuilders.termQuery("collate", randomBoolean() ? equilavent[0] : equilavent[1]))
+                .sort("collate")
+                .sort("_uid", SortOrder.DESC) // secondary sort should kick in because both will collate to same value
+            );
+
+        SearchResponse response = client().search(request).actionGet();
+        assertNoFailures(response);
+        assertHitCount(response, 2L);
+        assertOrderedSearchHits(response, "2", "1");
+    }
+
+    /*
+    * Setting alternate=shifted and variableTop to shift whitespace, but not
+    * punctuation or symbols, to quaternary level
+    */
+    public void testIgnoreWhitespace() throws Exception {
+        String index = "foo";
+        String type = "mytype";
+
+        XContentBuilder builder = jsonBuilder()
+            .startObject().startObject("properties")
+            .startObject("collate")
+            .field("type", "icu_collation_keyword")
+            .field("language", "en")
+            .field("strength", "primary")
+            .field("alternate", "shifted")
+            .field("variable_top", " ")
+            .field("index", false)
+            .endObject()
+            .endObject().endObject();
+
+        assertAcked(client().admin().indices().prepareCreate(index).addMapping(type, builder));
+
+        indexRandom(true,
+            client().prepareIndex(index, type, "1").setSource("{\"collate\":\"foo bar\"}", XContentType.JSON),
+            client().prepareIndex(index, type, "2").setSource("{\"collate\":\"foobar\"}", XContentType.JSON),
+            client().prepareIndex(index, type, "3").setSource("{\"collate\":\"foo-bar\"}", XContentType.JSON)
+        );
+
+        SearchRequest request = new SearchRequest()
+            .indices(index)
+            .types(type)
+            .source(new SearchSourceBuilder()
+                .fetchSource(false)
+                .sort("collate", SortOrder.ASC)
+                .sort("_uid", SortOrder.ASC) // secondary sort should kick in on docs 1 and 3 because same value collate value
+            );
+
+        SearchResponse response = client().search(request).actionGet();
+        assertNoFailures(response);
+        assertHitCount(response, 3L);
+        assertOrderedSearchHits(response, "3", "1", "2");
+    }
+
+    /*
+     * Setting numeric to encode digits with numeric value, so that
+     * foobar-9 sorts before foobar-10
+     */
+    public void testNumerics() throws Exception {
+        String index = "foo";
+        String type = "mytype";
+
+        XContentBuilder builder = jsonBuilder()
+            .startObject().startObject("properties")
+            .startObject("collate")
+            .field("type", "icu_collation_keyword")
+            .field("language", "en")
+            .field("numeric", true)
+            .field("index", false)
+            .endObject()
+            .endObject().endObject();
+
+        assertAcked(client().admin().indices().prepareCreate(index).addMapping(type, builder));
+
+        indexRandom(true,
+            client().prepareIndex(index, type, "1").setSource("{\"collate\":\"foobar-10\"}", XContentType.JSON),
+            client().prepareIndex(index, type, "2").setSource("{\"collate\":\"foobar-9\"}", XContentType.JSON)
+        );
+
+        SearchRequest request = new SearchRequest()
+            .indices(index)
+            .types(type)
+            .source(new SearchSourceBuilder()
+                .fetchSource(false)
+                .sort("collate", SortOrder.ASC)
+            );
+
+        SearchResponse response = client().search(request).actionGet();
+        assertNoFailures(response);
+        assertHitCount(response, 2L);
+        assertOrderedSearchHits(response, "2", "1");
+    }
+
+    /*
+    * Setting caseLevel=true to create an additional case level between
+    * secondary and tertiary
+    */
+    public void testIgnoreAccentsButNotCase() throws Exception {
+        String index = "foo";
+        String type = "mytype";
+
+        XContentBuilder builder = jsonBuilder()
+            .startObject().startObject("properties")
+            .startObject("collate")
+            .field("type", "icu_collation_keyword")
+            .field("language", "en")
+            .field("strength", "primary")
+            .field("case_level", true)
+            .field("index", false)
+            .endObject()
+            .endObject().endObject();
+
+        assertAcked(client().admin().indices().prepareCreate(index).addMapping(type, builder));
+
+        indexRandom(true,
+            client().prepareIndex(index, type, "1").setSource("{\"collate\":\"résumé\"}", XContentType.JSON),
+            client().prepareIndex(index, type, "2").setSource("{\"collate\":\"Resume\"}", XContentType.JSON),
+            client().prepareIndex(index, type, "3").setSource("{\"collate\":\"resume\"}", XContentType.JSON),
+            client().prepareIndex(index, type, "4").setSource("{\"collate\":\"Résumé\"}", XContentType.JSON)
+        );
+
+        SearchRequest request = new SearchRequest()
+            .indices(index)
+            .types(type)
+            .source(new SearchSourceBuilder()
+                .fetchSource(false)
+                .sort("collate", SortOrder.ASC)
+                .sort("_uid", SortOrder.DESC)
+            );
+
+        SearchResponse response = client().search(request).actionGet();
+        assertNoFailures(response);
+        assertHitCount(response, 4L);
+        assertOrderedSearchHits(response, "3", "1", "4", "2");
+    }
+
+    /*
+    * Setting caseFirst=upper to cause uppercase strings to sort
+    * before lowercase ones.
+    */
+    public void testUpperCaseFirst() throws Exception {
+        String index = "foo";
+        String type = "mytype";
+
+        XContentBuilder builder = jsonBuilder()
+            .startObject().startObject("properties")
+            .startObject("collate")
+            .field("type", "icu_collation_keyword")
+            .field("language", "en")
+            .field("strength", "tertiary")
+            .field("case_first", "upper")
+            .field("index", false)
+            .endObject()
+            .endObject().endObject();
+
+        assertAcked(client().admin().indices().prepareCreate(index).addMapping(type, builder));
+
+        indexRandom(true,
+            client().prepareIndex(index, type, "1").setSource("{\"collate\":\"resume\"}", XContentType.JSON),
+            client().prepareIndex(index, type, "2").setSource("{\"collate\":\"Resume\"}", XContentType.JSON)
+        );
+
+        SearchRequest request = new SearchRequest()
+            .indices(index)
+            .types(type)
+            .source(new SearchSourceBuilder()
+                .fetchSource(false)
+                .sort("collate", SortOrder.ASC)
+            );
+
+        SearchResponse response = client().search(request).actionGet();
+        assertNoFailures(response);
+        assertHitCount(response, 2L);
+        assertOrderedSearchHits(response, "2", "1");
+    }
+
+    /*
+    * For german, you might want oe to sort and match with o umlaut.
+    * This is not the default, but you can make a customized ruleset to do this.
+    *
+    * The default is DIN 5007-1, this shows how to tailor a collator to get DIN 5007-2 behavior.
+    *  http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4423383
+    */
+    public void testCustomRules() throws Exception {
+        String index = "foo";
+        String type = "mytype";
+
+        RuleBasedCollator baseCollator = (RuleBasedCollator) Collator.getInstance(new ULocale("de_DE"));
+        String DIN5007_2_tailorings =
+            "& ae , a\u0308 & AE , A\u0308" +
+                "& oe , o\u0308 & OE , O\u0308" +
+                "& ue , u\u0308 & UE , u\u0308";
+
+        RuleBasedCollator tailoredCollator = new RuleBasedCollator(baseCollator.getRules() + DIN5007_2_tailorings);
+        String tailoredRules = tailoredCollator.getRules();
+
+        String[] equilavent = {"Töne", "Toene"};
+
+        XContentBuilder builder = jsonBuilder()
+            .startObject().startObject("properties")
+            .startObject("collate")
+            .field("type", "icu_collation_keyword")
+            .field("rules", tailoredRules)
+            .field("strength", "primary")
+            .endObject()
+            .endObject().endObject();
+
+        assertAcked(client().admin().indices().prepareCreate(index).addMapping(type, builder));
+
+        indexRandom(true,
+            client().prepareIndex(index, type, "1").setSource("{\"collate\":\"" + equilavent[0] + "\"}", XContentType.JSON),
+            client().prepareIndex(index, type, "2").setSource("{\"collate\":\"" + equilavent[1] + "\"}", XContentType.JSON)
+        );
+
+        SearchRequest request = new SearchRequest()
+            .indices(index)
+            .types(type)
+            .source(new SearchSourceBuilder()
+                .fetchSource(false)
+                .query(QueryBuilders.termQuery("collate", randomBoolean() ? equilavent[0] : equilavent[1]))
+                .sort("collate", SortOrder.ASC)
+                .sort("_uid", SortOrder.DESC) // secondary sort should kick in because both will collate to same value
+            );
+
+        SearchResponse response = client().search(request).actionGet();
+        assertNoFailures(response);
+        assertHitCount(response, 2L);
+        assertOrderedSearchHits(response, "2", "1");
+    }
+}
diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java
new file mode 100644
index 0000000000000..be9dff4bdec28
--- /dev/null
+++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java
@@ -0,0 +1,451 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.index.mapper;
+
+import static org.hamcrest.Matchers.equalTo;
+
+import com.ibm.icu.text.Collator;
+import com.ibm.icu.text.RawCollationKey;
+import com.ibm.icu.util.ULocale;
+import org.apache.lucene.index.DocValuesType;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.IndexableFieldType;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.Version;
+import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.mapper.MapperService.MergeReason;
+import org.elasticsearch.plugin.analysis.icu.AnalysisICUPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.elasticsearch.test.InternalSettingsPlugin;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+
+public class ICUCollationKeywordFieldMapperTests extends ESSingleNodeTestCase {
+
+    private static final String FIELD_TYPE = "icu_collation_keyword";
+
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return Arrays.asList(AnalysisICUPlugin.class, InternalSettingsPlugin.class);
+    }
+
+    IndexService indexService;
+    DocumentMapperParser parser;
+
+    @Before
+    public void setup() {
+        indexService = createIndex("test");
+        parser = indexService.mapperService().documentMapperParser();
+    }
+
+    public void testDefaults() throws Exception {
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startObject("properties").startObject("field").field("type", FIELD_TYPE).endObject().endObject()
+            .endObject().endObject().string();
+
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+
+        assertEquals(mapping, mapper.mappingSource().toString());
+
+        ParsedDocument doc = mapper.parse(SourceToParse.source("test", "type", "1", XContentFactory.jsonBuilder()
+                .startObject()
+                .field("field", "1234")
+                .endObject()
+                .bytes(),
+            XContentType.JSON));
+
+        IndexableField[] fields = doc.rootDoc().getFields("field");
+        assertEquals(2, fields.length);
+
+        Collator collator = Collator.getInstance();
+        RawCollationKey key = collator.getRawCollationKey("1234", null);
+        BytesRef expected = new BytesRef(key.bytes, 0, key.size);
+
+        assertEquals(expected, fields[0].binaryValue());
+        IndexableFieldType fieldType = fields[0].fieldType();
+        assertThat(fieldType.omitNorms(), equalTo(true));
+        assertFalse(fieldType.tokenized());
+        assertFalse(fieldType.stored());
+        assertThat(fieldType.indexOptions(), equalTo(IndexOptions.DOCS));
+        assertThat(fieldType.storeTermVectors(), equalTo(false));
+        assertThat(fieldType.storeTermVectorOffsets(), equalTo(false));
+        assertThat(fieldType.storeTermVectorPositions(), equalTo(false));
+        assertThat(fieldType.storeTermVectorPayloads(), equalTo(false));
+        assertEquals(DocValuesType.NONE, fieldType.docValuesType());
+
+        assertEquals(expected, fields[1].binaryValue());
+        fieldType = fields[1].fieldType();
+        assertThat(fieldType.indexOptions(), equalTo(IndexOptions.NONE));
+        assertEquals(DocValuesType.SORTED_SET, fieldType.docValuesType());
+    }
+
+    public void testBackCompat() throws Exception {
+        indexService = createIndex("oldindex", Settings.builder().put("index.version.created", Version.V_5_5_0).build());
+        parser = indexService.mapperService().documentMapperParser();
+
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startObject("_all").field("enabled", false).endObject()
+            .startObject("properties").startObject("field").field("type", FIELD_TYPE).endObject().endObject()
+            .endObject().endObject().string();
+
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+
+        assertEquals(mapping, mapper.mappingSource().toString());
+
+        ParsedDocument doc = mapper.parse(SourceToParse.source("oldindex", "type", "1", XContentFactory.jsonBuilder()
+                .startObject()
+                .field("field", "1234")
+                .endObject()
+                .bytes(),
+            XContentType.JSON));
+
+        IndexableField[] fields = doc.rootDoc().getFields("field");
+        assertEquals(2, fields.length);
+
+        Collator collator = Collator.getInstance();
+        RawCollationKey key = collator.getRawCollationKey("1234", null);
+        BytesRef expected = new BytesRef(key.bytes, 0, key.size);
+
+        assertEquals(expected, fields[0].binaryValue());
+        IndexableFieldType fieldType = fields[0].fieldType();
+        assertThat(fieldType.omitNorms(), equalTo(true));
+        assertFalse(fieldType.tokenized());
+        assertFalse(fieldType.stored());
+        assertThat(fieldType.indexOptions(), equalTo(IndexOptions.DOCS));
+        assertThat(fieldType.storeTermVectors(), equalTo(false));
+        assertThat(fieldType.storeTermVectorOffsets(), equalTo(false));
+        assertThat(fieldType.storeTermVectorPositions(), equalTo(false));
+        assertThat(fieldType.storeTermVectorPayloads(), equalTo(false));
+        assertEquals(DocValuesType.NONE, fieldType.docValuesType());
+
+        assertEquals(expected, fields[1].binaryValue());
+        fieldType = fields[1].fieldType();
+        assertThat(fieldType.indexOptions(), equalTo(IndexOptions.NONE));
+        assertEquals(DocValuesType.SORTED, fieldType.docValuesType());
+    }
+
+    public void testNullValue() throws IOException {
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startObject("properties").startObject("field").field("type", FIELD_TYPE).endObject().endObject()
+            .endObject().endObject().string();
+
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+        assertEquals(mapping, mapper.mappingSource().toString());
+
+        ParsedDocument doc = mapper.parse(SourceToParse.source("test", "type", "1", XContentFactory.jsonBuilder()
+                .startObject()
+                .nullField("field")
+                .endObject()
+                .bytes(),
+            XContentType.JSON));
+        assertArrayEquals(new IndexableField[0], doc.rootDoc().getFields("field"));
+
+        mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startObject("properties").startObject("field").field("type", FIELD_TYPE)
+            .field("null_value", "1234").endObject().endObject()
+            .endObject().endObject().string();
+
+        mapper = parser.parse("type", new CompressedXContent(mapping));
+
+        assertEquals(mapping, mapper.mappingSource().toString());
+
+        doc = mapper.parse(SourceToParse.source("test", "type", "1", XContentFactory.jsonBuilder()
+                .startObject()
+                .endObject()
+                .bytes(),
+            XContentType.JSON));
+
+        IndexableField[] fields = doc.rootDoc().getFields("field");
+        assertEquals(0, fields.length);
+
+        doc = mapper.parse(SourceToParse.source("test", "type", "1", XContentFactory.jsonBuilder()
+                .startObject()
+                .nullField("field")
+                .endObject()
+                .bytes(),
+            XContentType.JSON));
+
+        Collator collator = Collator.getInstance();
+        RawCollationKey key = collator.getRawCollationKey("1234", null);
+        BytesRef expected = new BytesRef(key.bytes, 0, key.size);
+
+        fields = doc.rootDoc().getFields("field");
+        assertEquals(2, fields.length);
+        assertEquals(expected, fields[0].binaryValue());
+    }
+
+    public void testEnableStore() throws IOException {
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startObject("properties").startObject("field").field("type", FIELD_TYPE)
+            .field("store", true).endObject().endObject()
+            .endObject().endObject().string();
+
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+
+        assertEquals(mapping, mapper.mappingSource().toString());
+
+        ParsedDocument doc = mapper.parse(SourceToParse.source("test", "type", "1", XContentFactory.jsonBuilder()
+                .startObject()
+                .field("field", "1234")
+                .endObject()
+                .bytes(),
+            XContentType.JSON));
+
+        IndexableField[] fields = doc.rootDoc().getFields("field");
+        assertEquals(2, fields.length);
+        assertTrue(fields[0].fieldType().stored());
+    }
+
+    public void testDisableIndex() throws IOException {
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startObject("properties").startObject("field").field("type", FIELD_TYPE)
+            .field("index", false).endObject().endObject()
+            .endObject().endObject().string();
+
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+
+        assertEquals(mapping, mapper.mappingSource().toString());
+
+        ParsedDocument doc = mapper.parse(SourceToParse.source("test", "type", "1", XContentFactory.jsonBuilder()
+                .startObject()
+                .field("field", "1234")
+                .endObject()
+                .bytes(),
+            XContentType.JSON));
+
+        IndexableField[] fields = doc.rootDoc().getFields("field");
+        assertEquals(1, fields.length);
+        assertEquals(IndexOptions.NONE, fields[0].fieldType().indexOptions());
+        assertEquals(DocValuesType.SORTED_SET, fields[0].fieldType().docValuesType());
+    }
+
+    public void testDisableDocValues() throws IOException {
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startObject("properties").startObject("field").field("type", FIELD_TYPE)
+            .field("doc_values", false).endObject().endObject()
+            .endObject().endObject().string();
+
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+
+        assertEquals(mapping, mapper.mappingSource().toString());
+
+        ParsedDocument doc = mapper.parse(SourceToParse.source("test", "type", "1", XContentFactory.jsonBuilder()
+                .startObject()
+                .field("field", "1234")
+                .endObject()
+                .bytes(),
+            XContentType.JSON));
+
+        IndexableField[] fields = doc.rootDoc().getFields("field");
+        assertEquals(1, fields.length);
+        assertEquals(DocValuesType.NONE, fields[0].fieldType().docValuesType());
+    }
+
+    public void testMultipleValues() throws IOException {
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startObject("properties").startObject("field").field("type", FIELD_TYPE).endObject().endObject()
+            .endObject().endObject().string();
+
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+
+        assertEquals(mapping, mapper.mappingSource().toString());
+
+        ParsedDocument doc = mapper.parse(SourceToParse.source("test", "type", "1", XContentFactory.jsonBuilder()
+                .startObject()
+                .field("field", Arrays.asList("1234", "5678"))
+                .endObject()
+                .bytes(),
+            XContentType.JSON));
+
+        IndexableField[] fields = doc.rootDoc().getFields("field");
+        assertEquals(4, fields.length);
+
+        Collator collator = Collator.getInstance();
+        RawCollationKey key = collator.getRawCollationKey("1234", null);
+        BytesRef expected = new BytesRef(key.bytes, 0, key.size);
+
+        assertEquals(expected, fields[0].binaryValue());
+        IndexableFieldType fieldType = fields[0].fieldType();
+        assertThat(fieldType.omitNorms(), equalTo(true));
+        assertFalse(fieldType.tokenized());
+        assertFalse(fieldType.stored());
+        assertThat(fieldType.indexOptions(), equalTo(IndexOptions.DOCS));
+        assertThat(fieldType.storeTermVectors(), equalTo(false));
+        assertThat(fieldType.storeTermVectorOffsets(), equalTo(false));
+        assertThat(fieldType.storeTermVectorPositions(), equalTo(false));
+        assertThat(fieldType.storeTermVectorPayloads(), equalTo(false));
+        assertEquals(DocValuesType.NONE, fieldType.docValuesType());
+
+        assertEquals(expected, fields[1].binaryValue());
+        fieldType = fields[1].fieldType();
+        assertThat(fieldType.indexOptions(), equalTo(IndexOptions.NONE));
+        assertEquals(DocValuesType.SORTED_SET, fieldType.docValuesType());
+
+        collator = Collator.getInstance();
+        key = collator.getRawCollationKey("5678", null);
+        expected = new BytesRef(key.bytes, 0, key.size);
+
+        assertEquals(expected, fields[2].binaryValue());
+        fieldType = fields[2].fieldType();
+        assertThat(fieldType.omitNorms(), equalTo(true));
+        assertFalse(fieldType.tokenized());
+        assertFalse(fieldType.stored());
+        assertThat(fieldType.indexOptions(), equalTo(IndexOptions.DOCS));
+        assertThat(fieldType.storeTermVectors(), equalTo(false));
+        assertThat(fieldType.storeTermVectorOffsets(), equalTo(false));
+        assertThat(fieldType.storeTermVectorPositions(), equalTo(false));
+        assertThat(fieldType.storeTermVectorPayloads(), equalTo(false));
+        assertEquals(DocValuesType.NONE, fieldType.docValuesType());
+
+        assertEquals(expected, fields[3].binaryValue());
+        fieldType = fields[3].fieldType();
+        assertThat(fieldType.indexOptions(), equalTo(IndexOptions.NONE));
+        assertEquals(DocValuesType.SORTED_SET, fieldType.docValuesType());
+    }
+
+    public void testIndexOptions() throws IOException {
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startObject("properties").startObject("field").field("type", FIELD_TYPE)
+            .field("index_options", "freqs").endObject().endObject()
+            .endObject().endObject().string();
+
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+
+        assertEquals(mapping, mapper.mappingSource().toString());
+
+        ParsedDocument doc = mapper.parse(SourceToParse.source("test", "type", "1", XContentFactory.jsonBuilder()
+                .startObject()
+                .field("field", "1234")
+                .endObject()
+                .bytes(),
+            XContentType.JSON));
+
+        IndexableField[] fields = doc.rootDoc().getFields("field");
+        assertEquals(2, fields.length);
+        assertEquals(IndexOptions.DOCS_AND_FREQS, fields[0].fieldType().indexOptions());
+
+        for (String indexOptions : Arrays.asList("positions", "offsets")) {
+            final String mapping2 = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties").startObject("field").field("type", FIELD_TYPE)
+                .field("index_options", indexOptions).endObject().endObject()
+                .endObject().endObject().string();
+            IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> parser.parse("type", new CompressedXContent(mapping2)));
+            assertEquals("The [" + FIELD_TYPE + "] field does not support positions, got [index_options]=" + indexOptions,
+                e.getMessage());
+        }
+    }
+
+    public void testEnableNorms() throws IOException {
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startObject("properties").startObject("field").field("type", FIELD_TYPE)
+            .field("norms", true).endObject().endObject()
+            .endObject().endObject().string();
+
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+
+        assertEquals(mapping, mapper.mappingSource().toString());
+
+        ParsedDocument doc = mapper.parse(SourceToParse.source("test", "type", "1", XContentFactory.jsonBuilder()
+                .startObject()
+                .field("field", "1234")
+                .endObject()
+                .bytes(),
+            XContentType.JSON));
+
+        IndexableField[] fields = doc.rootDoc().getFields("field");
+        assertEquals(2, fields.length);
+        assertFalse(fields[0].fieldType().omitNorms());
+    }
+
+    public void testCollator() throws IOException {
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startObject("properties").startObject("field")
+            .field("type", FIELD_TYPE)
+            .field("language", "tr")
+            .field("strength", "primary")
+            .endObject().endObject().endObject().endObject().string();
+
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+
+        assertEquals(mapping, mapper.mappingSource().toString());
+
+        ParsedDocument doc = mapper.parse(SourceToParse.source("test", "type", "1", XContentFactory.jsonBuilder()
+                .startObject()
+                .field("field", "I WİLL USE TURKİSH CASING")
+                .endObject()
+                .bytes(),
+            XContentType.JSON));
+
+        Collator collator = Collator.getInstance(new ULocale("tr"));
+        collator.setStrength(Collator.PRIMARY);
+        RawCollationKey key = collator.getRawCollationKey("ı will use turkish casıng", null); // should collate to same value
+        BytesRef expected = new BytesRef(key.bytes, 0, key.size);
+
+        IndexableField[] fields = doc.rootDoc().getFields("field");
+        assertEquals(2, fields.length);
+
+        assertEquals(expected, fields[0].binaryValue());
+        IndexableFieldType fieldType = fields[0].fieldType();
+        assertThat(fieldType.omitNorms(), equalTo(true));
+        assertFalse(fieldType.tokenized());
+        assertFalse(fieldType.stored());
+        assertThat(fieldType.indexOptions(), equalTo(IndexOptions.DOCS));
+        assertThat(fieldType.storeTermVectors(), equalTo(false));
+        assertThat(fieldType.storeTermVectorOffsets(), equalTo(false));
+        assertThat(fieldType.storeTermVectorPositions(), equalTo(false));
+        assertThat(fieldType.storeTermVectorPayloads(), equalTo(false));
+        assertEquals(DocValuesType.NONE, fieldType.docValuesType());
+
+        assertEquals(expected, fields[1].binaryValue());
+        fieldType = fields[1].fieldType();
+        assertThat(fieldType.indexOptions(), equalTo(IndexOptions.NONE));
+        assertEquals(DocValuesType.SORTED_SET, fieldType.docValuesType());
+    }
+
+    public void testUpdateCollator() throws IOException {
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startObject("properties").startObject("field")
+            .field("type", FIELD_TYPE)
+            .field("language", "tr")
+            .field("strength", "primary")
+            .endObject().endObject().endObject().endObject().string();
+        indexService.mapperService().merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE, randomBoolean());
+
+        String mapping2 = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startObject("properties").startObject("field")
+            .field("type", FIELD_TYPE)
+            .field("language", "en")
+            .endObject().endObject().endObject().endObject().string();
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> indexService.mapperService().merge("type",
+                new CompressedXContent(mapping2), MergeReason.MAPPING_UPDATE, randomBoolean()));
+        assertEquals("Can't merge because of conflicts: [Cannot update language setting for [" + FIELD_TYPE
+            + "], Cannot update strength setting for [" + FIELD_TYPE + "]]", e.getMessage());
+    }
+}
diff --git a/plugins/analysis-icu/src/test/resources/rest-api-spec/test/analysis_icu/10_basic.yaml b/plugins/analysis-icu/src/test/resources/rest-api-spec/test/analysis_icu/10_basic.yaml
index 64fbbcadf7dd5..7a055135b5328 100644
--- a/plugins/analysis-icu/src/test/resources/rest-api-spec/test/analysis_icu/10_basic.yaml
+++ b/plugins/analysis-icu/src/test/resources/rest-api-spec/test/analysis_icu/10_basic.yaml
@@ -1,7 +1,12 @@
 # Integration tests for ICU analysis components
 #
 "Tokenizer":
+    - skip:
+        features: warnings
     - do:
+        warnings:
+          - text request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
+          - tokenizer request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
         indices.analyze:
           text:         Foo Bar
           tokenizer:    icu_tokenizer
@@ -10,7 +15,13 @@
     - match:  { tokens.1.token: Bar }
 ---
 "Normalization filter":
+    - skip:
+        features: warnings
     - do:
+        warnings:
+          - filter request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
+          - text request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
+          - tokenizer request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
         indices.analyze:
           filter:      icu_normalizer
           text:         Foo Bar Ruß
@@ -19,7 +30,13 @@
     - match:  { tokens.0.token: foo bar russ }
 ---
 "Normalization charfilter":
+    - skip:
+        features: warnings
     - do:
+        warnings:
+          - char_filter request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
+          - text request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
+          - tokenizer request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
         indices.analyze:
           char_filter: icu_normalizer
           text:         Foo Bar Ruß
@@ -28,10 +45,63 @@
     - match:  { tokens.0.token: foo bar russ }
 ---
 "Folding filter":
+    - skip:
+        features: warnings
     - do:
+        warnings:
+          - filter request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
+          - text request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
+          - tokenizer request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
         indices.analyze:
           filter:      icu_folding
           text:         Foo Bar résumé
           tokenizer:    keyword
     - length: { tokens: 1 }
     - match:  { tokens.0.token: foo bar resume }
+---
+"Normalization with a UnicodeSet Filter":
+    - do:
+        indices.create:
+            index:  test
+            body:
+                settings:
+                    index:
+                        analysis:
+                            char_filter:
+                                charfilter_icu_normalizer:
+                                    type: icu_normalizer
+                                    unicodeSetFilter: "[^ß]"
+                            filter:
+                                tokenfilter_icu_normalizer:
+                                    type: icu_normalizer
+                                    unicodeSetFilter: "[^ßB]"
+                                tokenfilter_icu_folding:
+                                    type: icu_folding
+                                    unicodeSetFilter: "[^â]"
+    - do:
+        indices.analyze:
+          index:    test
+          body:
+            char_filter: ["charfilter_icu_normalizer"]
+            tokenizer:  keyword
+            text:     charfilter Föo Bâr Ruß
+    - length: { tokens: 1 }
+    - match:  { tokens.0.token: charfilter föo bâr ruß }
+    - do:
+        indices.analyze:
+          index:    test
+          body:
+            tokenizer:  keyword
+            filter: ["tokenfilter_icu_normalizer"]
+            text:     tokenfilter Föo Bâr Ruß
+    - length: { tokens: 1 }
+    - match:  { tokens.0.token: tokenfilter föo Bâr ruß }
+    - do:
+        indices.analyze:
+          index:    test
+          body:
+            tokenizer:  keyword
+            filter: ["tokenfilter_icu_folding"]
+            text:     icufolding Föo Bâr Ruß
+    - length: { tokens: 1 }
+    - match: { tokens.0.token: icufolding foo bâr russ }
diff --git a/plugins/analysis-icu/src/test/resources/rest-api-spec/test/analysis_icu/20_search.yaml b/plugins/analysis-icu/src/test/resources/rest-api-spec/test/analysis_icu/20_search.yaml
index dce852f07ef6b..67ff1dab98483 100644
--- a/plugins/analysis-icu/src/test/resources/rest-api-spec/test/analysis_icu/20_search.yaml
+++ b/plugins/analysis-icu/src/test/resources/rest-api-spec/test/analysis_icu/20_search.yaml
@@ -26,13 +26,13 @@
                                 analyzer: my_analyzer
 
     - do:
-      index:
-          index:  test
-          type:   type
-          id:     1
-          body:   { "text": "Bâton enflammé" }
+        index:
+            index:  test
+            type:   type
+            id:     1
+            body:   { "text": "Bâton enflammé" }
     - do:
-      indices.refresh: {}
+        indices.refresh: {}
 
     - do:
         search:
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.2.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.2.0.jar.sha1
deleted file mode 100644
index 749cb8ecde8ab..0000000000000
--- a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.2.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-17ee76df332c0342a172790472b777086487a299
\ No newline at end of file
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.6.1.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.6.1.jar.sha1
new file mode 100644
index 0000000000000..33c53356958e5
--- /dev/null
+++ b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.6.1.jar.sha1
@@ -0,0 +1 @@
+2acac0d409e41f2649bb7d7cd6c12c42ea92f2d8
\ No newline at end of file
diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiPartOfSpeechFilterFactory.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiPartOfSpeechFilterFactory.java
index 4598f6293cea0..af376fc0de1e5 100644
--- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiPartOfSpeechFilterFactory.java
+++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiPartOfSpeechFilterFactory.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.index.analysis;
 
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.ja.JapaneseAnalyzer;
 import org.apache.lucene.analysis.ja.JapanesePartOfSpeechStopFilter;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
@@ -38,6 +39,8 @@ public KuromojiPartOfSpeechFilterFactory(IndexSettings indexSettings, Environmen
         List<String> wordList = Analysis.getWordList(env, settings, "stoptags");
         if (wordList != null) {
             stopTags.addAll(wordList);
+        } else {
+            stopTags.addAll(JapaneseAnalyzer.getDefaultStopTags());
         }
     }
 
diff --git a/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/KuromojiAnalysisTests.java b/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/KuromojiAnalysisTests.java
index 53196ac7462a9..823286f8371f9 100644
--- a/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/KuromojiAnalysisTests.java
+++ b/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/KuromojiAnalysisTests.java
@@ -48,44 +48,45 @@
  */
 public class KuromojiAnalysisTests extends ESTestCase {
     public void testDefaultsKuromojiAnalysis() throws IOException {
-        AnalysisService analysisService = createAnalysisService();
+        TestAnalysis analysis = createTestAnalysis();
 
-        TokenizerFactory tokenizerFactory = analysisService.tokenizer("kuromoji_tokenizer");
+        TokenizerFactory tokenizerFactory = analysis.tokenizer.get("kuromoji_tokenizer");
         assertThat(tokenizerFactory, instanceOf(KuromojiTokenizerFactory.class));
 
-        TokenFilterFactory filterFactory = analysisService.tokenFilter("kuromoji_part_of_speech");
+        TokenFilterFactory filterFactory = analysis.tokenFilter.get("kuromoji_part_of_speech");
         assertThat(filterFactory, instanceOf(KuromojiPartOfSpeechFilterFactory.class));
 
-        filterFactory = analysisService.tokenFilter("kuromoji_readingform");
+        filterFactory = analysis.tokenFilter.get("kuromoji_readingform");
         assertThat(filterFactory, instanceOf(KuromojiReadingFormFilterFactory.class));
 
-        filterFactory = analysisService.tokenFilter("kuromoji_baseform");
+        filterFactory = analysis.tokenFilter.get("kuromoji_baseform");
         assertThat(filterFactory, instanceOf(KuromojiBaseFormFilterFactory.class));
 
-        filterFactory = analysisService.tokenFilter("kuromoji_stemmer");
+        filterFactory = analysis.tokenFilter.get("kuromoji_stemmer");
         assertThat(filterFactory, instanceOf(KuromojiKatakanaStemmerFactory.class));
 
-        filterFactory = analysisService.tokenFilter("ja_stop");
+        filterFactory = analysis.tokenFilter.get("ja_stop");
         assertThat(filterFactory, instanceOf(JapaneseStopTokenFilterFactory.class));
 
-        filterFactory = analysisService.tokenFilter("kuromoji_number");
+        filterFactory = analysis.tokenFilter.get("kuromoji_number");
         assertThat(filterFactory, instanceOf(KuromojiNumberFilterFactory.class));
 
-        NamedAnalyzer analyzer = analysisService.analyzer("kuromoji");
+        IndexAnalyzers indexAnalyzers = analysis.indexAnalyzers;
+        NamedAnalyzer analyzer = indexAnalyzers.get("kuromoji");
         assertThat(analyzer.analyzer(), instanceOf(JapaneseAnalyzer.class));
 
-        analyzer = analysisService.analyzer("my_analyzer");
+        analyzer = indexAnalyzers.get("my_analyzer");
         assertThat(analyzer.analyzer(), instanceOf(CustomAnalyzer.class));
         assertThat(analyzer.analyzer().tokenStream(null, new StringReader("")), instanceOf(JapaneseTokenizer.class));
 
-        CharFilterFactory  charFilterFactory = analysisService.charFilter("kuromoji_iteration_mark");
+        CharFilterFactory  charFilterFactory = analysis.charFilter.get("kuromoji_iteration_mark");
         assertThat(charFilterFactory, instanceOf(KuromojiIterationMarkCharFilterFactory.class));
 
     }
 
     public void testBaseFormFilterFactory() throws IOException {
-        AnalysisService analysisService = createAnalysisService();
-        TokenFilterFactory tokenFilter = analysisService.tokenFilter("kuromoji_pos");
+        TestAnalysis analysis = createTestAnalysis();
+        TokenFilterFactory tokenFilter = analysis.tokenFilter.get("kuromoji_pos");
         assertThat(tokenFilter, instanceOf(KuromojiPartOfSpeechFilterFactory.class));
         String source = "私は制限スピードを超える。";
         String[] expected = new String[]{"私", "は", "制限", "スピード", "を"};
@@ -94,9 +95,24 @@ public void testBaseFormFilterFactory() throws IOException {
         assertSimpleTSOutput(tokenFilter.create(tokenizer), expected);
     }
 
+    public void testPartOfSpeechFilter() throws IOException {
+        TestAnalysis analysis = createTestAnalysis();
+        TokenFilterFactory tokenFilter = analysis.tokenFilter.get("kuromoji_part_of_speech");
+
+        assertThat(tokenFilter, instanceOf(KuromojiPartOfSpeechFilterFactory.class));
+
+        String source = "寿司がおいしいね";
+        String[] expected_tokens = new String[]{"寿司", "おいしい"};
+
+        Tokenizer tokenizer = new JapaneseTokenizer(null, true, JapaneseTokenizer.Mode.SEARCH);
+        tokenizer.setReader(new StringReader(source));
+
+        assertSimpleTSOutput(tokenFilter.create(tokenizer), expected_tokens);
+    }
+
     public void testReadingFormFilterFactory() throws IOException {
-        AnalysisService analysisService = createAnalysisService();
-        TokenFilterFactory tokenFilter = analysisService.tokenFilter("kuromoji_rf");
+        TestAnalysis analysis = createTestAnalysis();
+        TokenFilterFactory tokenFilter = analysis.tokenFilter.get("kuromoji_rf");
         assertThat(tokenFilter, instanceOf(KuromojiReadingFormFilterFactory.class));
         String source = "今夜はロバート先生と話した";
         String[] expected_tokens_romaji = new String[]{"kon'ya", "ha", "robato", "sensei", "to", "hanashi", "ta"};
@@ -109,14 +125,14 @@ public void testReadingFormFilterFactory() throws IOException {
         tokenizer = new JapaneseTokenizer(null, true, JapaneseTokenizer.Mode.SEARCH);
         tokenizer.setReader(new StringReader(source));
         String[] expected_tokens_katakana = new String[]{"コンヤ", "ハ", "ロバート", "センセイ", "ト", "ハナシ", "タ"};
-        tokenFilter = analysisService.tokenFilter("kuromoji_readingform");
+        tokenFilter = analysis.tokenFilter.get("kuromoji_readingform");
         assertThat(tokenFilter, instanceOf(KuromojiReadingFormFilterFactory.class));
         assertSimpleTSOutput(tokenFilter.create(tokenizer), expected_tokens_katakana);
     }
 
     public void testKatakanaStemFilter() throws IOException {
-        AnalysisService analysisService = createAnalysisService();
-        TokenFilterFactory tokenFilter = analysisService.tokenFilter("kuromoji_stemmer");
+        TestAnalysis analysis = createTestAnalysis();
+        TokenFilterFactory tokenFilter = analysis.tokenFilter.get("kuromoji_stemmer");
         assertThat(tokenFilter, instanceOf(KuromojiKatakanaStemmerFactory.class));
         String source = "明後日パーティーに行く予定がある。図書館で資料をコピーしました。";
 
@@ -128,7 +144,7 @@ public void testKatakanaStemFilter() throws IOException {
         String[] expected_tokens_katakana = new String[]{"明後日", "パーティ", "に", "行く", "予定", "が", "ある", "図書館", "で", "資料", "を", "コピー", "し", "まし", "た"};
         assertSimpleTSOutput(tokenFilter.create(tokenizer), expected_tokens_katakana);
 
-        tokenFilter = analysisService.tokenFilter("kuromoji_ks");
+        tokenFilter = analysis.tokenFilter.get("kuromoji_ks");
         assertThat(tokenFilter, instanceOf(KuromojiKatakanaStemmerFactory.class));
         tokenizer = new JapaneseTokenizer(null, true, JapaneseTokenizer.Mode.SEARCH);
         tokenizer.setReader(new StringReader(source));
@@ -140,9 +156,9 @@ public void testKatakanaStemFilter() throws IOException {
     }
 
     public void testIterationMarkCharFilter() throws IOException {
-        AnalysisService analysisService = createAnalysisService();
+        TestAnalysis analysis = createTestAnalysis();
         // test only kanji
-        CharFilterFactory charFilterFactory = analysisService.charFilter("kuromoji_im_only_kanji");
+        CharFilterFactory charFilterFactory = analysis.charFilter.get("kuromoji_im_only_kanji");
         assertNotNull(charFilterFactory);
         assertThat(charFilterFactory, instanceOf(KuromojiIterationMarkCharFilterFactory.class));
 
@@ -153,7 +169,7 @@ public void testIterationMarkCharFilter() throws IOException {
 
         // test only kana
 
-        charFilterFactory = analysisService.charFilter("kuromoji_im_only_kana");
+        charFilterFactory = analysis.charFilter.get("kuromoji_im_only_kana");
         assertNotNull(charFilterFactory);
         assertThat(charFilterFactory, instanceOf(KuromojiIterationMarkCharFilterFactory.class));
 
@@ -163,7 +179,7 @@ public void testIterationMarkCharFilter() throws IOException {
 
         // test default
 
-        charFilterFactory = analysisService.charFilter("kuromoji_im_default");
+        charFilterFactory = analysis.charFilter.get("kuromoji_im_default");
         assertNotNull(charFilterFactory);
         assertThat(charFilterFactory, instanceOf(KuromojiIterationMarkCharFilterFactory.class));
 
@@ -173,8 +189,8 @@ public void testIterationMarkCharFilter() throws IOException {
     }
 
     public void testJapaneseStopFilterFactory() throws IOException {
-        AnalysisService analysisService = createAnalysisService();
-        TokenFilterFactory tokenFilter = analysisService.tokenFilter("ja_stop");
+        TestAnalysis analysis = createTestAnalysis();
+        TokenFilterFactory tokenFilter = analysis.tokenFilter.get("ja_stop");
         assertThat(tokenFilter, instanceOf(JapaneseStopTokenFilterFactory.class));
         String source = "私は制限スピードを超える。";
         String[] expected = new String[]{"私", "制限", "超える"};
@@ -183,7 +199,7 @@ public void testJapaneseStopFilterFactory() throws IOException {
         assertSimpleTSOutput(tokenFilter.create(tokenizer), expected);
     }
 
-    private static AnalysisService createAnalysisService() throws IOException {
+    private static TestAnalysis createTestAnalysis() throws IOException {
         InputStream empty_dict = KuromojiAnalysisTests.class.getResourceAsStream("empty_user_dict.txt");
         InputStream dict = KuromojiAnalysisTests.class.getResourceAsStream("user_dict.txt");
         Path home = createTempDir();
@@ -198,7 +214,7 @@ private static AnalysisService createAnalysisService() throws IOException {
             .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
             .build();
         Settings nodeSettings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), home).build();
-        return createAnalysisService(new Index("test", "_na_"), nodeSettings, settings, new AnalysisKuromojiPlugin());
+        return createTestAnalysis(new Index("test", "_na_"), nodeSettings, settings, new AnalysisKuromojiPlugin());
     }
 
     public static void assertSimpleTSOutput(TokenStream stream,
@@ -209,7 +225,7 @@ public static void assertSimpleTSOutput(TokenStream stream,
         int i = 0;
         while (stream.incrementToken()) {
             assertThat(expected.length, greaterThan(i));
-            assertThat( "expected different term at index " + i, expected[i++], equalTo(termAttr.toString()));
+            assertThat("expected different term at index " + i, termAttr.toString(), equalTo(expected[i++]));
         }
         assertThat("not all tokens produced", i, equalTo(expected.length));
     }
@@ -230,8 +246,8 @@ private String readFully(Reader reader) throws IOException {
     }
 
     public void testKuromojiUserDict() throws IOException {
-        AnalysisService analysisService = createAnalysisService();
-        TokenizerFactory tokenizerFactory = analysisService.tokenizer("kuromoji_user_dict");
+        TestAnalysis analysis = createTestAnalysis();
+        TokenizerFactory tokenizerFactory = analysis.tokenizer.get("kuromoji_user_dict");
         String source = "私は制限スピードを超える。";
         String[] expected = new String[]{"私", "は", "制限スピード", "を", "超える"};
 
@@ -242,14 +258,14 @@ public void testKuromojiUserDict() throws IOException {
 
     // fix #59
     public void testKuromojiEmptyUserDict() throws IOException {
-        AnalysisService analysisService = createAnalysisService();
-        TokenizerFactory tokenizerFactory = analysisService.tokenizer("kuromoji_empty_user_dict");
+        TestAnalysis analysis = createTestAnalysis();
+        TokenizerFactory tokenizerFactory = analysis.tokenizer.get("kuromoji_empty_user_dict");
         assertThat(tokenizerFactory, instanceOf(KuromojiTokenizerFactory.class));
     }
 
     public void testNbestCost() throws IOException {
-        AnalysisService analysisService = createAnalysisService();
-        TokenizerFactory tokenizerFactory = analysisService.tokenizer("kuromoji_nbest_cost");
+        TestAnalysis analysis = createTestAnalysis();
+        TokenizerFactory tokenizerFactory = analysis.tokenizer.get("kuromoji_nbest_cost");
         String source = "鳩山積み";
         String[] expected = new String[] {"鳩", "鳩山", "山積み", "積み"};
 
@@ -259,8 +275,8 @@ public void testNbestCost() throws IOException {
     }
 
     public void testNbestExample() throws IOException {
-        AnalysisService analysisService = createAnalysisService();
-        TokenizerFactory tokenizerFactory = analysisService.tokenizer("kuromoji_nbest_examples");
+        TestAnalysis analysis = createTestAnalysis();
+        TokenizerFactory tokenizerFactory = analysis.tokenizer.get("kuromoji_nbest_examples");
         String source = "鳩山積み";
         String[] expected = new String[] {"鳩", "鳩山", "山積み", "積み"};
 
@@ -270,8 +286,8 @@ public void testNbestExample() throws IOException {
     }
 
     public void testNbestBothOptions() throws IOException {
-        AnalysisService analysisService = createAnalysisService();
-        TokenizerFactory tokenizerFactory = analysisService.tokenizer("kuromoji_nbest_both");
+        TestAnalysis analysis = createTestAnalysis();
+        TokenizerFactory tokenizerFactory = analysis.tokenizer.get("kuromoji_nbest_both");
         String source = "鳩山積み";
         String[] expected = new String[] {"鳩", "鳩山", "山積み", "積み"};
 
@@ -282,8 +298,8 @@ public void testNbestBothOptions() throws IOException {
     }
 
     public void testNumberFilterFactory() throws Exception {
-        AnalysisService analysisService = createAnalysisService();
-        TokenFilterFactory tokenFilter = analysisService.tokenFilter("kuromoji_number");
+        TestAnalysis analysis = createTestAnalysis();
+        TokenFilterFactory tokenFilter = analysis.tokenFilter.get("kuromoji_number");
         assertThat(tokenFilter, instanceOf(KuromojiNumberFilterFactory.class));
         String source = "本日十万二千五百円のワインを買った";
         String[] expected = new String[]{"本日", "102500", "円", "の", "ワイン", "を", "買っ", "た"};
diff --git a/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/KuromojiClientYamlTestSuiteIT.java b/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/KuromojiClientYamlTestSuiteIT.java
index e99c5c2bacfed..b3083a2416a19 100644
--- a/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/KuromojiClientYamlTestSuiteIT.java
+++ b/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/KuromojiClientYamlTestSuiteIT.java
@@ -24,9 +24,6 @@
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-
-import java.io.IOException;
 
 public class KuromojiClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
@@ -35,8 +32,8 @@ public KuromojiClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testC
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
-        return ESClientYamlSuiteTestCase.createParameters(0, 1);
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
     }
 }
 
diff --git a/plugins/analysis-kuromoji/src/test/resources/rest-api-spec/test/analysis_kuromoji/10_basic.yaml b/plugins/analysis-kuromoji/src/test/resources/rest-api-spec/test/analysis_kuromoji/10_basic.yaml
index 42df558567d23..d6f65c5160447 100644
--- a/plugins/analysis-kuromoji/src/test/resources/rest-api-spec/test/analysis_kuromoji/10_basic.yaml
+++ b/plugins/analysis-kuromoji/src/test/resources/rest-api-spec/test/analysis_kuromoji/10_basic.yaml
@@ -2,7 +2,12 @@
 #
 ---
 "Analyzer":
+    - skip:
+        features: warnings
     - do:
+        warnings:
+          - text request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
+          - analyzer request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
         indices.analyze:
           text:         JR新宿駅の近くにビールを飲みに行こうか
           analyzer:     kuromoji
@@ -16,7 +21,12 @@
     - match:  { tokens.6.token: 行く }
 ---
 "Tokenizer":
+    - skip:
+        features: warnings
     - do:
+        warnings:
+          - text request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
+          - tokenizer request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
         indices.analyze:
           text:         関西国際空港
           tokenizer:    kuromoji_tokenizer
@@ -27,7 +37,13 @@
     - match:  { tokens.3.token: 空港 }
 ---
 "Baseform filter":
+    - skip:
+        features: warnings
     - do:
+        warnings:
+          - text request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
+          - tokenizer request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
+          - filter request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
         indices.analyze:
           text:         飲み
           tokenizer:    kuromoji_tokenizer
@@ -36,7 +52,13 @@
     - match:  { tokens.0.token: 飲む }
 ---
 "Reading filter":
+    - skip:
+        features: warnings
     - do:
+        warnings:
+          - text request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
+          - tokenizer request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
+          - filter request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
         indices.analyze:
           text:         寿司
           tokenizer:    kuromoji_tokenizer
@@ -45,7 +67,13 @@
     - match:  { tokens.0.token: スシ }
 ---
 "Stemming filter":
+    - skip:
+        features: warnings
     - do:
+        warnings:
+          - text request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
+          - tokenizer request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
+          - filter request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
         indices.analyze:
           text:         サーバー
           tokenizer:    kuromoji_tokenizer
diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.2.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.2.0.jar.sha1
deleted file mode 100644
index 359173e008404..0000000000000
--- a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.2.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-8d2a6b8679563d9f044eb1cee580282b20d8e149
\ No newline at end of file
diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.6.1.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.6.1.jar.sha1
new file mode 100644
index 0000000000000..5e53d22ccc75b
--- /dev/null
+++ b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.6.1.jar.sha1
@@ -0,0 +1 @@
+3f6a4d5ed865da2b53abc18fbf8448ddfdb8907a
\ No newline at end of file
diff --git a/plugins/analysis-phonetic/src/main/java/org/elasticsearch/index/analysis/phonetic/KoelnerPhonetik.java b/plugins/analysis-phonetic/src/main/java/org/elasticsearch/index/analysis/phonetic/KoelnerPhonetik.java
index 3c658191524db..c7b39a86a2bd7 100644
--- a/plugins/analysis-phonetic/src/main/java/org/elasticsearch/index/analysis/phonetic/KoelnerPhonetik.java
+++ b/plugins/analysis-phonetic/src/main/java/org/elasticsearch/index/analysis/phonetic/KoelnerPhonetik.java
@@ -142,8 +142,7 @@ private List<String> partition(String str) {
         List<String> parts = new ArrayList<>();
         parts.add(primaryForm.replaceAll("[^\\p{L}\\p{N}]", ""));
         if (!primary) {
-            List<String> tmpParts = new ArrayList<>();
-            tmpParts.addAll((Arrays.asList(str.split("[\\p{Z}\\p{C}\\p{P}]"))));
+            List<String> tmpParts = new ArrayList<>(Arrays.asList(str.split("[\\p{Z}\\p{C}\\p{P}]")));
             int numberOfParts = tmpParts.size();
             while (tmpParts.size() > 0) {
                 StringBuilder part = new StringBuilder();
diff --git a/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/PhoneticClientYamlTestSuiteIT.java b/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/PhoneticClientYamlTestSuiteIT.java
index 975b84f1574e8..12736fa5fd871 100644
--- a/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/PhoneticClientYamlTestSuiteIT.java
+++ b/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/PhoneticClientYamlTestSuiteIT.java
@@ -24,9 +24,6 @@
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-
-import java.io.IOException;
 
 public class PhoneticClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
@@ -35,8 +32,8 @@ public PhoneticClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testC
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
-        return ESClientYamlSuiteTestCase.createParameters(0, 1);
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
     }
 }
 
diff --git a/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/SimplePhoneticAnalysisTests.java b/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/SimplePhoneticAnalysisTests.java
index 3dcfadce78173..b0c23e29abd57 100644
--- a/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/SimplePhoneticAnalysisTests.java
+++ b/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/SimplePhoneticAnalysisTests.java
@@ -39,8 +39,8 @@ public void testPhoneticTokenFilterFactory() throws IOException {
         Settings settings = Settings.builder().loadFromStream(yaml, getClass().getResourceAsStream(yaml))
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
                 .build();
-        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisPhoneticPlugin());
-        TokenFilterFactory filterFactory = analysisService.tokenFilter("phonetic");
+        TestAnalysis analysis = createTestAnalysis(new Index("test", "_na_"), settings, new AnalysisPhoneticPlugin());
+        TokenFilterFactory filterFactory = analysis.tokenFilter.get("phonetic");
         MatcherAssert.assertThat(filterFactory, instanceOf(PhoneticTokenFilterFactory.class));
     }
 }
diff --git a/plugins/analysis-phonetic/src/test/resources/rest-api-spec/test/analysis_phonetic/10_metaphone.yaml b/plugins/analysis-phonetic/src/test/resources/rest-api-spec/test/analysis_phonetic/10_metaphone.yaml
index 02d4b315b6eee..71fca8768b5e3 100644
--- a/plugins/analysis-phonetic/src/test/resources/rest-api-spec/test/analysis_phonetic/10_metaphone.yaml
+++ b/plugins/analysis-phonetic/src/test/resources/rest-api-spec/test/analysis_phonetic/10_metaphone.yaml
@@ -1,8 +1,8 @@
 # Integration tests for Phonetic analysis components
 #
-
-
 "Metaphone":
+    - skip:
+        features: warnings
     - do:
         indices.create:
             index:  phonetic_sample
@@ -20,6 +20,9 @@
                                     encoder: metaphone
                                     replace: false
     - do:
+        warnings:
+          - analyzer request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
+          - text request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
         indices.analyze:
             index: phonetic_sample
             analyzer: my_analyzer
@@ -30,4 +33,3 @@
     - match:  { tokens.1.token: joe  }
     - match:  { tokens.2.token: BLKS  }
     - match:  { tokens.3.token: bloggs  }
-
diff --git a/plugins/analysis-phonetic/src/test/resources/rest-api-spec/test/analysis_phonetic/20_double_metaphone.yaml b/plugins/analysis-phonetic/src/test/resources/rest-api-spec/test/analysis_phonetic/20_double_metaphone.yaml
index 675847e557e3b..764a0b774ec2d 100644
--- a/plugins/analysis-phonetic/src/test/resources/rest-api-spec/test/analysis_phonetic/20_double_metaphone.yaml
+++ b/plugins/analysis-phonetic/src/test/resources/rest-api-spec/test/analysis_phonetic/20_double_metaphone.yaml
@@ -1,8 +1,8 @@
 # Integration tests for Phonetic analysis components
 #
-
-
 "Double Metaphone":
+    - skip:
+        features: warnings
     - do:
         indices.create:
             index:  phonetic_sample
@@ -20,6 +20,9 @@
                                     encoder: double_metaphone
                                     max_code_len: 6
     - do:
+        warnings:
+          - analyzer request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
+          - text request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
         indices.analyze:
             index: phonetic_sample
             analyzer: my_analyzer
@@ -27,4 +30,3 @@
 
     - length: { tokens: 1 }
     - match:  { tokens.0.token: SPRKLF }
-
diff --git a/plugins/analysis-phonetic/src/test/resources/rest-api-spec/test/analysis_phonetic/30_beider_morse.yaml b/plugins/analysis-phonetic/src/test/resources/rest-api-spec/test/analysis_phonetic/30_beider_morse.yaml
index 015610af17241..5d25dfe143c5a 100644
--- a/plugins/analysis-phonetic/src/test/resources/rest-api-spec/test/analysis_phonetic/30_beider_morse.yaml
+++ b/plugins/analysis-phonetic/src/test/resources/rest-api-spec/test/analysis_phonetic/30_beider_morse.yaml
@@ -1,8 +1,8 @@
 # Integration tests for Phonetic analysis components
 #
-
-
 "Double Metaphone":
+    - skip:
+        features: warnings
     - do:
         indices.create:
             index:  phonetic_sample
@@ -22,6 +22,9 @@
                                     name_type: ashkenazi
                                     languageset: polish
     - do:
+        warnings:
+          - analyzer request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
+          - text request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
         indices.analyze:
             index: phonetic_sample
             analyzer: my_analyzer
@@ -29,4 +32,3 @@
 
     - length: { tokens: 1 }
     - match:  { tokens.0.token: Svarts }
-
diff --git a/plugins/analysis-phonetic/src/test/resources/rest-api-spec/test/analysis_phonetic/40_search.yaml b/plugins/analysis-phonetic/src/test/resources/rest-api-spec/test/analysis_phonetic/40_search.yaml
index 933abaafc3684..75c672172391c 100644
--- a/plugins/analysis-phonetic/src/test/resources/rest-api-spec/test/analysis_phonetic/40_search.yaml
+++ b/plugins/analysis-phonetic/src/test/resources/rest-api-spec/test/analysis_phonetic/40_search.yaml
@@ -26,13 +26,13 @@
                                 analyzer: my_analyzer
 
     - do:
-      index:
-          index:  phonetic_sample
-          type:   type
-          id:     1
-          body:   { "text": "hello world" }
+        index:
+            index:  phonetic_sample
+            type:   type
+            id:     1
+            body:   { "text": "hello world" }
     - do:
-      indices.refresh: {}
+        indices.refresh: {}
 
     - do:
         search:
diff --git a/plugins/analysis-phonetic/src/test/resources/rest-api-spec/test/analysis_phonetic/50_daitch_mokotoff.yaml b/plugins/analysis-phonetic/src/test/resources/rest-api-spec/test/analysis_phonetic/50_daitch_mokotoff.yaml
index 5125ae3d684ef..8772f9e315f7d 100644
--- a/plugins/analysis-phonetic/src/test/resources/rest-api-spec/test/analysis_phonetic/50_daitch_mokotoff.yaml
+++ b/plugins/analysis-phonetic/src/test/resources/rest-api-spec/test/analysis_phonetic/50_daitch_mokotoff.yaml
@@ -1,8 +1,8 @@
 # Integration tests for Phonetic analysis components
 #
-
-
 "Daitch Mokotoff":
+    - skip:
+        features: warnings
     - do:
         indices.create:
             index:  phonetic_sample
@@ -19,6 +19,9 @@
                                     type: phonetic
                                     encoder: daitch_mokotoff
     - do:
+        warnings:
+          - analyzer request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
+          - text request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
         indices.analyze:
             index: phonetic_sample
             analyzer: my_analyzer
@@ -26,4 +29,3 @@
 
     - length: { tokens: 1 }
     - match:  { tokens.0.token: "645740" }
-
diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.2.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.2.0.jar.sha1
deleted file mode 100644
index 66e339bfa2f8a..0000000000000
--- a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.2.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-ba3fd99d1cf47d31b82817accdb199fc7a8d838d
\ No newline at end of file
diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.6.1.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.6.1.jar.sha1
new file mode 100644
index 0000000000000..cb511164e4dce
--- /dev/null
+++ b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.6.1.jar.sha1
@@ -0,0 +1 @@
+831f945a8163aa92b25d58dd67cc7d255a0768ee
\ No newline at end of file
diff --git a/plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/SimpleSmartChineseAnalysisTests.java b/plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/SimpleSmartChineseAnalysisTests.java
index 08aebdee2bb90..e2d6f6db51376 100644
--- a/plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/SimpleSmartChineseAnalysisTests.java
+++ b/plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/SimpleSmartChineseAnalysisTests.java
@@ -31,9 +31,9 @@
 
 public class SimpleSmartChineseAnalysisTests extends ESTestCase {
     public void testDefaultsIcuAnalysis() throws IOException {
-        final AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), Settings.EMPTY,
+        final TestAnalysis analysis = createTestAnalysis(new Index("test", "_na_"), Settings.EMPTY,
                 new AnalysisSmartChinesePlugin());
-        TokenizerFactory tokenizerFactory = analysisService.tokenizer("smartcn_tokenizer");
+        TokenizerFactory tokenizerFactory = analysis.tokenizer.get("smartcn_tokenizer");
         MatcherAssert.assertThat(tokenizerFactory, instanceOf(SmartChineseTokenizerTokenizerFactory.class));
     }
 }
diff --git a/plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/SmartCNClientYamlTestSuiteIT.java b/plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/SmartCNClientYamlTestSuiteIT.java
index 6415dc436eb62..de67e4c4d6cce 100644
--- a/plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/SmartCNClientYamlTestSuiteIT.java
+++ b/plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/SmartCNClientYamlTestSuiteIT.java
@@ -24,9 +24,6 @@
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-
-import java.io.IOException;
 
 public class SmartCNClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
@@ -35,8 +32,8 @@ public SmartCNClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCa
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
-        return ESClientYamlSuiteTestCase.createParameters(0, 1);
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
     }
 }
 
diff --git a/plugins/analysis-smartcn/src/test/resources/rest-api-spec/test/analysis_smartcn/10_basic.yaml b/plugins/analysis-smartcn/src/test/resources/rest-api-spec/test/analysis_smartcn/10_basic.yaml
index 2549f774f81c0..155d577cc4eac 100644
--- a/plugins/analysis-smartcn/src/test/resources/rest-api-spec/test/analysis_smartcn/10_basic.yaml
+++ b/plugins/analysis-smartcn/src/test/resources/rest-api-spec/test/analysis_smartcn/10_basic.yaml
@@ -1,7 +1,12 @@
 # Integration tests for Smart Chinese analysis components
 #
 "Tokenizer":
+    - skip:
+        features: warnings
     - do:
+        warnings:
+          - text request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
+          - tokenizer request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
         indices.analyze:
           text:         我购买了道具和服装。
           tokenizer:    smartcn_tokenizer
@@ -15,7 +20,12 @@
     - match:  { tokens.6.token: "," }
 ---
 "Analyzer":
+    - skip:
+        features: warnings
     - do:
+        warnings:
+          - text request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
+          - analyzer request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
         indices.analyze:
           text:         我购买了道具和服装。
           analyzer:     smartcn
diff --git a/plugins/analysis-smartcn/src/test/resources/rest-api-spec/test/analysis_smartcn/20_search.yaml b/plugins/analysis-smartcn/src/test/resources/rest-api-spec/test/analysis_smartcn/20_search.yaml
index 4101bb1646c4e..87261cd4030f1 100644
--- a/plugins/analysis-smartcn/src/test/resources/rest-api-spec/test/analysis_smartcn/20_search.yaml
+++ b/plugins/analysis-smartcn/src/test/resources/rest-api-spec/test/analysis_smartcn/20_search.yaml
@@ -14,13 +14,13 @@
                                 analyzer: smartcn
 
     - do:
-      index:
-          index:  test
-          type:   type
-          id:     1
-          body:   { "text": "我购买了道具和服装" }
+        index:
+            index:  test
+            type:   type
+            id:     1
+            body:   { "text": "我购买了道具和服装" }
     - do:
-      indices.refresh: {}
+        indices.refresh: {}
 
     - do:
         search:
diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.2.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.2.0.jar.sha1
deleted file mode 100644
index 5cfb071f3a397..0000000000000
--- a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.2.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-09d2a759a765f73e2e7becbc560411469c464cfa
\ No newline at end of file
diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.6.1.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.6.1.jar.sha1
new file mode 100644
index 0000000000000..491f7cfb9b345
--- /dev/null
+++ b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.6.1.jar.sha1
@@ -0,0 +1 @@
+65531a895bbcca12065566d90b8436201f955e09
\ No newline at end of file
diff --git a/plugins/analysis-stempel/src/main/java/org/elasticsearch/index/analysis/pl/PolishStemTokenFilterFactory.java b/plugins/analysis-stempel/src/main/java/org/elasticsearch/index/analysis/pl/PolishStemTokenFilterFactory.java
index afc7d527a6c8b..aa3194c5831b7 100644
--- a/plugins/analysis-stempel/src/main/java/org/elasticsearch/index/analysis/pl/PolishStemTokenFilterFactory.java
+++ b/plugins/analysis-stempel/src/main/java/org/elasticsearch/index/analysis/pl/PolishStemTokenFilterFactory.java
@@ -35,20 +35,11 @@
 
 public class PolishStemTokenFilterFactory extends AbstractTokenFilterFactory {
 
-    private final StempelStemmer stemmer;
-
     public PolishStemTokenFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
         super(indexSettings, name, settings);
-        Trie tire;
-        try {
-            tire = StempelStemmer.load(PolishAnalyzer.class.getResourceAsStream(PolishAnalyzer.DEFAULT_STEMMER_FILE));
-        } catch (IOException ex) {
-            throw new RuntimeException("Unable to load default stemming tables", ex);
-        }
-        stemmer = new StempelStemmer(tire);
     }
 
     @Override public TokenStream create(TokenStream tokenStream) {
-        return new StempelFilter(tokenStream, stemmer);
+        return new StempelFilter(tokenStream, new StempelStemmer(PolishAnalyzer.getDefaultTable()));
     }
 }
diff --git a/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/AnalysisPolishFactoryTests.java b/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/AnalysisPolishFactoryTests.java
index abf739d010a12..e68cb260b0bf0 100644
--- a/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/AnalysisPolishFactoryTests.java
+++ b/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/AnalysisPolishFactoryTests.java
@@ -19,12 +19,24 @@
 
 package org.elasticsearch.index.analysis;
 
-import org.elasticsearch.AnalysisFactoryTestCase;
-import org.elasticsearch.index.analysis.pl.PolishStemTokenFilterFactory;
-
+import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.Tokenizer;
+import org.elasticsearch.AnalysisFactoryTestCase;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.analysis.pl.PolishStemTokenFilterFactory;
+
 public class AnalysisPolishFactoryTests extends AnalysisFactoryTestCase {
 
     @Override
@@ -34,4 +46,31 @@ protected Map<String, Class<?>> getTokenFilters() {
         return filters;
     }
 
+    public void testThreadSafety() throws IOException {
+        // TODO: is this the right boilerplate?  I forked this out of TransportAnalyzeAction.java:
+        Settings settings = Settings.builder()
+            // for _na_
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+            .build();
+        Environment environment = new Environment(settings);
+        IndexMetaData metaData = IndexMetaData.builder(IndexMetaData.INDEX_UUID_NA_VALUE).settings(settings).build();
+        IndexSettings indexSettings = new IndexSettings(metaData, Settings.EMPTY);
+        testThreadSafety(new PolishStemTokenFilterFactory(indexSettings, environment, "stempelpolishstem", settings));
+    }
+
+    // TODO: move to AnalysisFactoryTestCase so we can more easily test thread safety for all factories
+    private void testThreadSafety(TokenFilterFactory factory) throws IOException {
+        final Analyzer analyzer = new Analyzer() {
+            @Override
+            protected TokenStreamComponents createComponents(String fieldName) {
+                Tokenizer tokenizer = new MockTokenizer();
+                return new TokenStreamComponents(tokenizer, factory.create(tokenizer));
+            }
+        };
+        BaseTokenStreamTestCase.checkRandomData(random(), analyzer, 100);
+    }
 }
diff --git a/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/PolishAnalysisTests.java b/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/PolishAnalysisTests.java
index 4f7ee642ebd43..d0b81f01d01c7 100644
--- a/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/PolishAnalysisTests.java
+++ b/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/PolishAnalysisTests.java
@@ -36,12 +36,12 @@
  */
 public class PolishAnalysisTests extends ESTestCase {
     public void testDefaultsPolishAnalysis() throws IOException {
-        final AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), Settings.EMPTY,
+        final TestAnalysis analysis = createTestAnalysis(new Index("test", "_na_"), Settings.EMPTY,
                 new AnalysisStempelPlugin());
-        TokenFilterFactory tokenizerFactory = analysisService.tokenFilter("polish_stem");
+        TokenFilterFactory tokenizerFactory = analysis.tokenFilter.get("polish_stem");
         MatcherAssert.assertThat(tokenizerFactory, instanceOf(PolishStemTokenFilterFactory.class));
 
-        Analyzer analyzer = analysisService.analyzer("polish").analyzer();
+        Analyzer analyzer = analysis.indexAnalyzers.get("polish").analyzer();
         MatcherAssert.assertThat(analyzer, instanceOf(PolishAnalyzer.class));
     }
 }
diff --git a/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/SimplePolishTokenFilterTests.java b/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/SimplePolishTokenFilterTests.java
index 3fc12ccdfedec..26f02c9df4030 100644
--- a/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/SimplePolishTokenFilterTests.java
+++ b/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/SimplePolishTokenFilterTests.java
@@ -49,9 +49,9 @@ private void testToken(String source, String expected) throws IOException {
         Settings settings = Settings.builder()
                 .put("index.analysis.filter.myStemmer.type", "polish_stem")
                 .build();
-        AnalysisService analysisService = createAnalysisService(index, settings, new AnalysisStempelPlugin());
+        TestAnalysis analysis = createTestAnalysis(index, settings, new AnalysisStempelPlugin());
 
-        TokenFilterFactory filterFactory = analysisService.tokenFilter("myStemmer");
+        TokenFilterFactory filterFactory = analysis.tokenFilter.get("myStemmer");
 
         Tokenizer tokenizer = new KeywordTokenizer();
         tokenizer.setReader(new StringReader(source));
@@ -65,9 +65,9 @@ private void testToken(String source, String expected) throws IOException {
     }
 
     private void testAnalyzer(String source, String... expected_terms) throws IOException {
-        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), Settings.EMPTY, new AnalysisStempelPlugin());
+        TestAnalysis analysis = createTestAnalysis(new Index("test", "_na_"), Settings.EMPTY, new AnalysisStempelPlugin());
 
-        Analyzer analyzer = analysisService.analyzer("polish").analyzer();
+        Analyzer analyzer = analysis.indexAnalyzers.get("polish").analyzer();
 
         TokenStream ts = analyzer.tokenStream("test", source);
 
diff --git a/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/StempelClientYamlTestSuiteIT.java b/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/StempelClientYamlTestSuiteIT.java
index 34d264122eff1..5b061c95775c3 100644
--- a/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/StempelClientYamlTestSuiteIT.java
+++ b/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/StempelClientYamlTestSuiteIT.java
@@ -24,9 +24,6 @@
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-
-import java.io.IOException;
 
 public class StempelClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
@@ -35,8 +32,8 @@ public StempelClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCa
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
-        return ESClientYamlSuiteTestCase.createParameters(0, 1);
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
     }
 }
 
diff --git a/plugins/analysis-stempel/src/test/resources/rest-api-spec/test/analysis_stempel/10_basic.yaml b/plugins/analysis-stempel/src/test/resources/rest-api-spec/test/analysis_stempel/10_basic.yaml
index f87f00b7922b0..c8cb86fc412dc 100644
--- a/plugins/analysis-stempel/src/test/resources/rest-api-spec/test/analysis_stempel/10_basic.yaml
+++ b/plugins/analysis-stempel/src/test/resources/rest-api-spec/test/analysis_stempel/10_basic.yaml
@@ -1,7 +1,13 @@
 # Integration tests for Polish analysis components
 #
 "Stemmer":
+    - skip:
+        features: warnings
     - do:
+        warnings:
+          - text request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
+          - tokenizer request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
+          - filter request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
         indices.analyze:
           text:         studenci
           tokenizer:    keyword
@@ -10,7 +16,12 @@
     - match:  { tokens.0.token: student  }
 ---
 "Analyzer":
+    - skip:
+        features: warnings
     - do:
+        warnings:
+          - text request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
+          - analyzer request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
         indices.analyze:
           text:         studenta był
           analyzer:     polish
diff --git a/plugins/analysis-stempel/src/test/resources/rest-api-spec/test/analysis_stempel/20_search.yaml b/plugins/analysis-stempel/src/test/resources/rest-api-spec/test/analysis_stempel/20_search.yaml
index d23a7e368b50e..d22639ab93b88 100644
--- a/plugins/analysis-stempel/src/test/resources/rest-api-spec/test/analysis_stempel/20_search.yaml
+++ b/plugins/analysis-stempel/src/test/resources/rest-api-spec/test/analysis_stempel/20_search.yaml
@@ -14,13 +14,13 @@
                                 analyzer: polish
 
     - do:
-      index:
-          index:  test
-          type:   type
-          id:     1
-          body:   { "text": "studenta był" }
+        index:
+            index:  test
+            type:   type
+            id:     1
+            body:   { "text": "studenta był" }
     - do:
-      indices.refresh: {}
+        indices.refresh: {}
 
     - do:
         search:
diff --git a/plugins/analysis-ukrainian/build.gradle b/plugins/analysis-ukrainian/build.gradle
new file mode 100644
index 0000000000000..61f67021a3dcf
--- /dev/null
+++ b/plugins/analysis-ukrainian/build.gradle
@@ -0,0 +1,40 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+esplugin {
+  description 'The Ukrainian Analysis plugin integrates the Lucene UkrainianMorfologikAnalyzer into elasticsearch.'
+  classname 'org.elasticsearch.plugin.analysis.ukrainian.AnalysisUkrainianPlugin'
+}
+
+dependencies {
+  compile "org.apache.lucene:lucene-analyzers-morfologik:${versions.lucene}"
+  compile "org.carrot2:morfologik-stemming:2.1.1"
+  compile "org.carrot2:morfologik-fsa:2.1.1"
+  compile "ua.net.nlp:morfologik-ukrainian-search:3.7.5"
+}
+
+dependencyLicenses {
+  mapping from: /lucene-.*/, to: 'lucene'
+  mapping from: /morfologik-.*/, to: 'lucene'
+}
+
+thirdPartyAudit.excludes = [
+  // we don't use the morfologik-fsa polish stemmer
+  'morfologik.stemming.polish.PolishStemmer'
+]
diff --git a/plugins/analysis-ukrainian/licenses/lucene-LICENSE.txt b/plugins/analysis-ukrainian/licenses/lucene-LICENSE.txt
new file mode 100644
index 0000000000000..28b134f5f8e4d
--- /dev/null
+++ b/plugins/analysis-ukrainian/licenses/lucene-LICENSE.txt
@@ -0,0 +1,475 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+
+
+Some code in core/src/java/org/apache/lucene/util/UnicodeUtil.java was
+derived from unicode conversion examples available at
+http://www.unicode.org/Public/PROGRAMS/CVTUTF.  Here is the copyright
+from those sources:
+
+/*
+ * Copyright 2001-2004 Unicode, Inc.
+ * 
+ * Disclaimer
+ * 
+ * This source code is provided as is by Unicode, Inc. No claims are
+ * made as to fitness for any particular purpose. No warranties of any
+ * kind are expressed or implied. The recipient agrees to determine
+ * applicability of information provided. If this file has been
+ * purchased on magnetic or optical media from Unicode, Inc., the
+ * sole remedy for any claim will be exchange of defective media
+ * within 90 days of receipt.
+ * 
+ * Limitations on Rights to Redistribute This Code
+ * 
+ * Unicode, Inc. hereby grants the right to freely use the information
+ * supplied in this file in the creation of products supporting the
+ * Unicode Standard, and to make copies of this file in any form
+ * for internal or external distribution as long as this notice
+ * remains attached.
+ */
+
+
+Some code in core/src/java/org/apache/lucene/util/ArrayUtil.java was
+derived from Python 2.4.2 sources available at
+http://www.python.org. Full license is here:
+
+  http://www.python.org/download/releases/2.4.2/license/
+
+Some code in core/src/java/org/apache/lucene/util/UnicodeUtil.java was
+derived from Python 3.1.2 sources available at
+http://www.python.org. Full license is here:
+
+  http://www.python.org/download/releases/3.1.2/license/
+
+Some code in core/src/java/org/apache/lucene/util/automaton was
+derived from Brics automaton sources available at
+www.brics.dk/automaton/. Here is the copyright from those sources:
+
+/*
+ * Copyright (c) 2001-2009 Anders Moeller
+ * All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+ 
+The levenshtein automata tables in core/src/java/org/apache/lucene/util/automaton 
+were automatically generated with the moman/finenight FSA package.
+Here is the copyright for those sources:
+
+# Copyright (c) 2010, Jean-Philippe Barrette-LaPierre, <jpb@rrette.com>
+#
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this software and associated documentation
+# files (the "Software"), to deal in the Software without
+# restriction, including without limitation the rights to use,
+# copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following
+# conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+
+Some code in core/src/java/org/apache/lucene/util/UnicodeUtil.java was
+derived from ICU (http://www.icu-project.org)
+The full license is available here: 
+  http://source.icu-project.org/repos/icu/icu/trunk/license.html
+
+/*
+ * Copyright (C) 1999-2010, International Business Machines
+ * Corporation and others.  All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy 
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights 
+ * to use, copy, modify, merge, publish, distribute, and/or sell copies of the 
+ * Software, and to permit persons to whom the Software is furnished to do so, 
+ * provided that the above copyright notice(s) and this permission notice appear 
+ * in all copies of the Software and that both the above copyright notice(s) and
+ * this permission notice appear in supporting documentation.
+ * 
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. 
+ * IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE BE 
+ * LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR 
+ * ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER 
+ * IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT 
+ * OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ *
+ * Except as contained in this notice, the name of a copyright holder shall not 
+ * be used in advertising or otherwise to promote the sale, use or other 
+ * dealings in this Software without prior written authorization of the 
+ * copyright holder.
+ */
+ 
+The following license applies to the Snowball stemmers:
+
+Copyright (c) 2001, Dr Martin Porter
+Copyright (c) 2002, Richard Boulton
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice,
+    * this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+    * notice, this list of conditions and the following disclaimer in the
+    * documentation and/or other materials provided with the distribution.
+    * Neither the name of the copyright holders nor the names of its contributors
+    * may be used to endorse or promote products derived from this software
+    * without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+The following license applies to the KStemmer:
+
+Copyright © 2003,
+Center for Intelligent Information Retrieval,
+University of Massachusetts, Amherst.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+3. The names "Center for Intelligent Information Retrieval" and
+"University of Massachusetts" must not be used to endorse or promote products
+derived from this software without prior written permission. To obtain
+permission, contact info@ciir.cs.umass.edu.
+
+THIS SOFTWARE IS PROVIDED BY UNIVERSITY OF MASSACHUSETTS AND OTHER CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGE.
+
+The following license applies to the Morfologik project:
+
+Copyright (c) 2006 Dawid Weiss
+Copyright (c) 2007-2011 Dawid Weiss, Marcin Miłkowski
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, 
+are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice, 
+    this list of conditions and the following disclaimer.
+    
+    * Redistributions in binary form must reproduce the above copyright notice, 
+    this list of conditions and the following disclaimer in the documentation 
+    and/or other materials provided with the distribution.
+    
+    * Neither the name of Morfologik nor the names of its contributors 
+    may be used to endorse or promote products derived from this software 
+    without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR 
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+---
+
+The dictionary comes from Morfologik project. Morfologik uses data from 
+Polish ispell/myspell dictionary hosted at http://www.sjp.pl/slownik/en/ and 
+is licenced on the terms of (inter alia) LGPL and Creative Commons 
+ShareAlike. The part-of-speech tags were added in Morfologik project and
+are not found in the data from sjp.pl. The tagset is similar to IPI PAN
+tagset.
+
+---
+
+The following license applies to the Morfeusz project,
+used by org.apache.lucene.analysis.morfologik.
+
+BSD-licensed dictionary of Polish (SGJP)
+http://sgjp.pl/morfeusz/
+
+Copyright © 2011 Zygmunt Saloni, Włodzimierz Gruszczyński, 
+             Marcin Woliński, Robert Wołosz
+
+All rights reserved.
+
+Redistribution and  use in  source and binary  forms, with  or without
+modification, are permitted provided that the following conditions are
+met:
+
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the
+   distribution.
+
+THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDERS “AS IS” AND ANY EXPRESS
+OR  IMPLIED WARRANTIES,  INCLUDING, BUT  NOT LIMITED  TO,  THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED.  IN NO EVENT  SHALL COPYRIGHT  HOLDERS OR  CONTRIBUTORS BE
+LIABLE FOR  ANY DIRECT,  INDIRECT, INCIDENTAL, SPECIAL,  EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES  (INCLUDING, BUT NOT LIMITED  TO, PROCUREMENT OF
+SUBSTITUTE  GOODS OR  SERVICES;  LOSS  OF USE,  DATA,  OR PROFITS;  OR
+BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF LIABILITY,
+WHETHER IN  CONTRACT, STRICT LIABILITY, OR  TORT (INCLUDING NEGLIGENCE
+OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/plugins/analysis-ukrainian/licenses/lucene-NOTICE.txt b/plugins/analysis-ukrainian/licenses/lucene-NOTICE.txt
new file mode 100644
index 0000000000000..ecf08201a5ee6
--- /dev/null
+++ b/plugins/analysis-ukrainian/licenses/lucene-NOTICE.txt
@@ -0,0 +1,191 @@
+Apache Lucene
+Copyright 2014 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+Includes software from other Apache Software Foundation projects,
+including, but not limited to:
+ - Apache Ant
+ - Apache Jakarta Regexp
+ - Apache Commons
+ - Apache Xerces
+
+ICU4J, (under analysis/icu) is licensed under an MIT styles license
+and Copyright (c) 1995-2008 International Business Machines Corporation and others
+
+Some data files (under analysis/icu/src/data) are derived from Unicode data such
+as the Unicode Character Database. See http://unicode.org/copyright.html for more
+details.
+
+Brics Automaton (under core/src/java/org/apache/lucene/util/automaton) is 
+BSD-licensed, created by Anders Møller. See http://www.brics.dk/automaton/
+
+The levenshtein automata tables (under core/src/java/org/apache/lucene/util/automaton) were
+automatically generated with the moman/finenight FSA library, created by
+Jean-Philippe Barrette-LaPierre. This library is available under an MIT license,
+see http://sites.google.com/site/rrettesite/moman and 
+http://bitbucket.org/jpbarrette/moman/overview/
+
+The class org.apache.lucene.util.WeakIdentityMap was derived from
+the Apache CXF project and is Apache License 2.0.
+
+The Google Code Prettify is Apache License 2.0.
+See http://code.google.com/p/google-code-prettify/
+
+JUnit (junit-4.10) is licensed under the Common Public License v. 1.0
+See http://junit.sourceforge.net/cpl-v10.html
+
+This product includes code (JaspellTernarySearchTrie) from Java Spelling Checkin
+g Package (jaspell): http://jaspell.sourceforge.net/
+License: The BSD License (http://www.opensource.org/licenses/bsd-license.php)
+
+The snowball stemmers in
+  analysis/common/src/java/net/sf/snowball
+were developed by Martin Porter and Richard Boulton.
+The snowball stopword lists in
+  analysis/common/src/resources/org/apache/lucene/analysis/snowball
+were developed by Martin Porter and Richard Boulton.
+The full snowball package is available from
+  http://snowball.tartarus.org/
+
+The KStem stemmer in
+  analysis/common/src/org/apache/lucene/analysis/en
+was developed by Bob Krovetz and Sergio Guzman-Lara (CIIR-UMass Amherst)
+under the BSD-license.
+
+The Arabic,Persian,Romanian,Bulgarian, and Hindi analyzers (common) come with a default
+stopword list that is BSD-licensed created by Jacques Savoy.  These files reside in:
+analysis/common/src/resources/org/apache/lucene/analysis/ar/stopwords.txt,
+analysis/common/src/resources/org/apache/lucene/analysis/fa/stopwords.txt,
+analysis/common/src/resources/org/apache/lucene/analysis/ro/stopwords.txt,
+analysis/common/src/resources/org/apache/lucene/analysis/bg/stopwords.txt,
+analysis/common/src/resources/org/apache/lucene/analysis/hi/stopwords.txt
+See http://members.unine.ch/jacques.savoy/clef/index.html.
+
+The German,Spanish,Finnish,French,Hungarian,Italian,Portuguese,Russian and Swedish light stemmers
+(common) are based on BSD-licensed reference implementations created by Jacques Savoy and
+Ljiljana Dolamic. These files reside in:
+analysis/common/src/java/org/apache/lucene/analysis/de/GermanLightStemmer.java
+analysis/common/src/java/org/apache/lucene/analysis/de/GermanMinimalStemmer.java
+analysis/common/src/java/org/apache/lucene/analysis/es/SpanishLightStemmer.java
+analysis/common/src/java/org/apache/lucene/analysis/fi/FinnishLightStemmer.java
+analysis/common/src/java/org/apache/lucene/analysis/fr/FrenchLightStemmer.java
+analysis/common/src/java/org/apache/lucene/analysis/fr/FrenchMinimalStemmer.java
+analysis/common/src/java/org/apache/lucene/analysis/hu/HungarianLightStemmer.java
+analysis/common/src/java/org/apache/lucene/analysis/it/ItalianLightStemmer.java
+analysis/common/src/java/org/apache/lucene/analysis/pt/PortugueseLightStemmer.java
+analysis/common/src/java/org/apache/lucene/analysis/ru/RussianLightStemmer.java
+analysis/common/src/java/org/apache/lucene/analysis/sv/SwedishLightStemmer.java
+
+The Stempel analyzer (stempel) includes BSD-licensed software developed 
+by the Egothor project http://egothor.sf.net/, created by Leo Galambos, Martin Kvapil,
+and Edmond Nolan.
+
+The Polish analyzer (stempel) comes with a default
+stopword list that is BSD-licensed created by the Carrot2 project. The file resides
+in stempel/src/resources/org/apache/lucene/analysis/pl/stopwords.txt.
+See http://project.carrot2.org/license.html.
+
+The SmartChineseAnalyzer source code (smartcn) was
+provided by Xiaoping Gao and copyright 2009 by www.imdict.net.
+
+WordBreakTestUnicode_*.java (under modules/analysis/common/src/test/) 
+is derived from Unicode data such as the Unicode Character Database. 
+See http://unicode.org/copyright.html for more details.
+
+The Morfologik analyzer (morfologik) includes BSD-licensed software
+developed by Dawid Weiss and Marcin Miłkowski (http://morfologik.blogspot.com/).
+
+Morfologik uses data from Polish ispell/myspell dictionary
+(http://www.sjp.pl/slownik/en/) licenced on the terms of (inter alia)
+LGPL and Creative Commons ShareAlike.
+
+Morfologic includes data from BSD-licensed dictionary of Polish (SGJP)
+(http://sgjp.pl/morfeusz/)
+
+Servlet-api.jar and javax.servlet-*.jar are under the CDDL license, the original
+source code for this can be found at http://www.eclipse.org/jetty/downloads.php
+
+===========================================================================
+Kuromoji Japanese Morphological Analyzer - Apache Lucene Integration
+===========================================================================
+
+This software includes a binary and/or source version of data from
+
+  mecab-ipadic-2.7.0-20070801
+
+which can be obtained from
+
+  http://atilika.com/releases/mecab-ipadic/mecab-ipadic-2.7.0-20070801.tar.gz
+
+or
+
+  http://jaist.dl.sourceforge.net/project/mecab/mecab-ipadic/2.7.0-20070801/mecab-ipadic-2.7.0-20070801.tar.gz
+
+===========================================================================
+mecab-ipadic-2.7.0-20070801 Notice
+===========================================================================
+
+Nara Institute of Science and Technology (NAIST),
+the copyright holders, disclaims all warranties with regard to this
+software, including all implied warranties of merchantability and
+fitness, in no event shall NAIST be liable for
+any special, indirect or consequential damages or any damages
+whatsoever resulting from loss of use, data or profits, whether in an
+action of contract, negligence or other tortuous action, arising out
+of or in connection with the use or performance of this software.
+
+A large portion of the dictionary entries
+originate from ICOT Free Software.  The following conditions for ICOT
+Free Software applies to the current dictionary as well.
+
+Each User may also freely distribute the Program, whether in its
+original form or modified, to any third party or parties, PROVIDED
+that the provisions of Section 3 ("NO WARRANTY") will ALWAYS appear
+on, or be attached to, the Program, which is distributed substantially
+in the same form as set out herein and that such intended
+distribution, if actually made, will neither violate or otherwise
+contravene any of the laws and regulations of the countries having
+jurisdiction over the User or the intended distribution itself.
+
+NO WARRANTY
+
+The program was produced on an experimental basis in the course of the
+research and development conducted during the project and is provided
+to users as so produced on an experimental basis.  Accordingly, the
+program is provided without any warranty whatsoever, whether express,
+implied, statutory or otherwise.  The term "warranty" used herein
+includes, but is not limited to, any warranty of the quality,
+performance, merchantability and fitness for a particular purpose of
+the program and the nonexistence of any infringement or violation of
+any right of any third party.
+
+Each user of the program will agree and understand, and be deemed to
+have agreed and understood, that there is no warranty whatsoever for
+the program and, accordingly, the entire risk arising from or
+otherwise connected with the program is assumed by the user.
+
+Therefore, neither ICOT, the copyright holder, or any other
+organization that participated in or was otherwise related to the
+development of the program and their respective officials, directors,
+officers and other employees shall be held liable for any and all
+damages, including, without limitation, general, special, incidental
+and consequential damages, arising out of or otherwise in connection
+with the use or inability to use the program or any product, material
+or result produced or otherwise obtained by using the program,
+regardless of whether they have been advised of, or otherwise had
+knowledge of, the possibility of such damages at any time during the
+project or thereafter.  Each user will be deemed to have agreed to the
+foregoing by his or her commencement of use of the program.  The term
+"use" as used herein includes, but is not limited to, the use,
+modification, copying and distribution of the program and the
+production of secondary products from the program.
+
+In the case where the program, whether in its original form or
+modified, was distributed or delivered to or received by a user from
+any person, organization or entity other than ICOT, unless it makes or
+grants independently of ICOT any specific warranty to the user in
+writing, such person, organization or entity, will also be exempted
+from and not be held liable to the user for any such damages as noted
+above as far as the program is concerned.
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-6.6.1.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-6.6.1.jar.sha1
new file mode 100644
index 0000000000000..54d0a3c8ec45f
--- /dev/null
+++ b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-6.6.1.jar.sha1
@@ -0,0 +1 @@
+6e9573664cadd0d5588c1a3146f5b94a928ea140
\ No newline at end of file
diff --git a/plugins/analysis-ukrainian/licenses/morfologik-fsa-2.1.1.jar.sha1 b/plugins/analysis-ukrainian/licenses/morfologik-fsa-2.1.1.jar.sha1
new file mode 100644
index 0000000000000..07d523ec0c82b
--- /dev/null
+++ b/plugins/analysis-ukrainian/licenses/morfologik-fsa-2.1.1.jar.sha1
@@ -0,0 +1 @@
+87866deba6aa5d19956fbe3406d8ddb5f19f5352
\ No newline at end of file
diff --git a/plugins/analysis-ukrainian/licenses/morfologik-stemming-2.1.1.jar.sha1 b/plugins/analysis-ukrainian/licenses/morfologik-stemming-2.1.1.jar.sha1
new file mode 100644
index 0000000000000..22af41d2b6b1b
--- /dev/null
+++ b/plugins/analysis-ukrainian/licenses/morfologik-stemming-2.1.1.jar.sha1
@@ -0,0 +1 @@
+5c169bab2e7dd04f5cb03d179a73a4339cc1d0a2
\ No newline at end of file
diff --git a/plugins/analysis-ukrainian/licenses/morfologik-ukrainian-search-3.7.5.jar.sha1 b/plugins/analysis-ukrainian/licenses/morfologik-ukrainian-search-3.7.5.jar.sha1
new file mode 100644
index 0000000000000..446e7a91161a8
--- /dev/null
+++ b/plugins/analysis-ukrainian/licenses/morfologik-ukrainian-search-3.7.5.jar.sha1
@@ -0,0 +1 @@
+2b8c8fbd740164d220ca7d18605b8b2092e163e9
\ No newline at end of file
diff --git a/plugins/analysis-ukrainian/src/main/java/org/elasticsearch/index/analysis/UkrainianAnalyzerProvider.java b/plugins/analysis-ukrainian/src/main/java/org/elasticsearch/index/analysis/UkrainianAnalyzerProvider.java
new file mode 100644
index 0000000000000..45bf27b954bff
--- /dev/null
+++ b/plugins/analysis-ukrainian/src/main/java/org/elasticsearch/index/analysis/UkrainianAnalyzerProvider.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.analysis;
+
+import org.apache.lucene.analysis.CharArraySet;
+import org.apache.lucene.analysis.uk.UkrainianMorfologikAnalyzer;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.analysis.AbstractIndexAnalyzerProvider;
+import org.elasticsearch.index.analysis.Analysis;
+
+public class UkrainianAnalyzerProvider extends AbstractIndexAnalyzerProvider<UkrainianMorfologikAnalyzer> {
+
+    private final UkrainianMorfologikAnalyzer analyzer;
+
+    public UkrainianAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) {
+        super(indexSettings, name, settings);
+        analyzer = new UkrainianMorfologikAnalyzer(Analysis.parseStopWords(env, settings, UkrainianMorfologikAnalyzer.getDefaultStopSet()),
+                Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET));
+        analyzer.setVersion(version);
+    }
+
+    @Override
+    public UkrainianMorfologikAnalyzer get() {
+        return this.analyzer;
+    }
+
+
+}
diff --git a/plugins/analysis-ukrainian/src/main/java/org/elasticsearch/plugin/analysis/ukrainian/AnalysisUkrainianPlugin.java b/plugins/analysis-ukrainian/src/main/java/org/elasticsearch/plugin/analysis/ukrainian/AnalysisUkrainianPlugin.java
new file mode 100644
index 0000000000000..ff8425e201cba
--- /dev/null
+++ b/plugins/analysis-ukrainian/src/main/java/org/elasticsearch/plugin/analysis/ukrainian/AnalysisUkrainianPlugin.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.plugin.analysis.ukrainian;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.elasticsearch.index.analysis.AnalyzerProvider;
+import org.elasticsearch.index.analysis.UkrainianAnalyzerProvider;
+import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider;
+import org.elasticsearch.plugins.AnalysisPlugin;
+import org.elasticsearch.plugins.Plugin;
+
+import java.util.Map;
+
+import static java.util.Collections.singletonMap;
+
+public class AnalysisUkrainianPlugin extends Plugin implements AnalysisPlugin {
+
+    @Override
+    public Map<String, AnalysisProvider<AnalyzerProvider<? extends Analyzer>>> getAnalyzers() {
+        return singletonMap("ukrainian", UkrainianAnalyzerProvider::new);
+    }
+}
diff --git a/plugins/analysis-ukrainian/src/test/java/org/elasticsearch/index/analysis/SimpleUkrainianAnalyzerTests.java b/plugins/analysis-ukrainian/src/test/java/org/elasticsearch/index/analysis/SimpleUkrainianAnalyzerTests.java
new file mode 100644
index 0000000000000..6dbc37ea4ab0d
--- /dev/null
+++ b/plugins/analysis-ukrainian/src/test/java/org/elasticsearch/index/analysis/SimpleUkrainianAnalyzerTests.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.analysis;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.plugin.analysis.ukrainian.AnalysisUkrainianPlugin;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class SimpleUkrainianAnalyzerTests extends ESTestCase {
+
+    public void testBasicUsage() throws Exception {
+        testAnalyzer("чергу", "черга");
+        testAnalyzer("рухається", "рухатися");
+        testAnalyzer("колу", "кола", "коло", "кіл");
+        testAnalyzer("Ця п'єса у свою чергу рухається по колу.", "п'єса", "черга", "рухатися", "кола", "коло", "кіл");
+    }
+
+    private static void testAnalyzer(String source, String... expected_terms) throws IOException {
+        TestAnalysis analysis = createTestAnalysis(new Index("test", "_na_"), Settings.EMPTY, new AnalysisUkrainianPlugin());
+        Analyzer analyzer = analysis.indexAnalyzers.get("ukrainian").analyzer();
+        TokenStream ts = analyzer.tokenStream("test", source);
+        CharTermAttribute term1 = ts.addAttribute(CharTermAttribute.class);
+        ts.reset();
+        for (String expected : expected_terms) {
+            assertThat(ts.incrementToken(), equalTo(true));
+            assertThat(term1.toString(), equalTo(expected));
+        }
+        assertThat(ts.incrementToken(), equalTo(false));
+    }
+
+}
diff --git a/plugins/analysis-ukrainian/src/test/java/org/elasticsearch/index/analysis/UkrainianAnalysisTests.java b/plugins/analysis-ukrainian/src/test/java/org/elasticsearch/index/analysis/UkrainianAnalysisTests.java
new file mode 100644
index 0000000000000..a45549c22bd97
--- /dev/null
+++ b/plugins/analysis-ukrainian/src/test/java/org/elasticsearch/index/analysis/UkrainianAnalysisTests.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.analysis;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.uk.UkrainianMorfologikAnalyzer;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.plugin.analysis.ukrainian.AnalysisUkrainianPlugin;
+import org.elasticsearch.test.ESTestCase;
+import org.hamcrest.MatcherAssert;
+
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.instanceOf;
+
+public class UkrainianAnalysisTests extends ESTestCase {
+
+    public void testDefaultsUkranianAnalysis() throws IOException {
+        final TestAnalysis analysis = createTestAnalysis(new Index("test", "_na_"), Settings.EMPTY,
+                new AnalysisUkrainianPlugin());
+
+        Analyzer analyzer = analysis.indexAnalyzers.get("ukrainian").analyzer();
+        MatcherAssert.assertThat(analyzer, instanceOf(UkrainianMorfologikAnalyzer.class));
+    }
+}
diff --git a/plugins/analysis-ukrainian/src/test/java/org/elasticsearch/index/analysis/UkrainianClientYamlTestSuiteIT.java b/plugins/analysis-ukrainian/src/test/java/org/elasticsearch/index/analysis/UkrainianClientYamlTestSuiteIT.java
new file mode 100644
index 0000000000000..e03b1b1b4cbc4
--- /dev/null
+++ b/plugins/analysis-ukrainian/src/test/java/org/elasticsearch/index/analysis/UkrainianClientYamlTestSuiteIT.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.analysis;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
+import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+
+public class UkrainianClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
+
+    public UkrainianClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
+    }
+}
+
diff --git a/plugins/analysis-ukrainian/src/test/resources/rest-api-spec/test/analysis_ukrainian/10_basic.yaml b/plugins/analysis-ukrainian/src/test/resources/rest-api-spec/test/analysis_ukrainian/10_basic.yaml
new file mode 100644
index 0000000000000..48d513c140cd4
--- /dev/null
+++ b/plugins/analysis-ukrainian/src/test/resources/rest-api-spec/test/analysis_ukrainian/10_basic.yaml
@@ -0,0 +1,18 @@
+# Integration tests for Ukrainian analyzer
+---
+"Analyzer":
+    - do:
+        indices.analyze:
+          body:
+            text: колу
+            analyzer: ukrainian
+    - length: { tokens: 3 }
+    - match:  { tokens.0.token: кола  }
+    - match:  { tokens.0.start_offset: 0  }
+    - match:  { tokens.0.end_offset: 4  }
+    - match:  { tokens.1.token: коло  }
+    - match:  { tokens.1.start_offset: 0  }
+    - match:  { tokens.1.end_offset: 4  }
+    - match:  { tokens.2.token: кіл  }
+    - match:  { tokens.2.start_offset: 0  }
+    - match:  { tokens.2.end_offset: 4  }
diff --git a/plugins/analysis-ukrainian/src/test/resources/rest-api-spec/test/analysis_ukrainian/20_search.yaml b/plugins/analysis-ukrainian/src/test/resources/rest-api-spec/test/analysis_ukrainian/20_search.yaml
new file mode 100644
index 0000000000000..c07be73e44334
--- /dev/null
+++ b/plugins/analysis-ukrainian/src/test/resources/rest-api-spec/test/analysis_ukrainian/20_search.yaml
@@ -0,0 +1,32 @@
+# Integration tests for Stempel analysis component
+#
+---
+"Index Stempel content":
+    - do:
+        indices.create:
+            index:  test
+            body:
+                mappings:
+                    type:
+                        properties:
+                            text:
+                                type:     text
+                                analyzer: ukrainian
+
+    - do:
+        index:
+            index:  test
+            type:   type
+            id:     1
+            body:   { "text": "Ця п'єса у свою чергу рухається по колу." }
+    - do:
+        indices.refresh: {}
+
+    - do:
+        search:
+            index: test
+            body:
+                query:
+                    match:
+                        text: кола
+    - match: { hits.total: 1 }
diff --git a/plugins/build.gradle b/plugins/build.gradle
index e49b08c6015c3..27655abf534f6 100644
--- a/plugins/build.gradle
+++ b/plugins/build.gradle
@@ -22,10 +22,12 @@ configure(subprojects.findAll { it.parent.path == project.path }) {
   group = 'org.elasticsearch.plugin'
 
   apply plugin: 'elasticsearch.esplugin'
-  apply plugin: 'com.bmuschko.nexus'
 
   esplugin {
     // for local ES plugins, the name of the plugin is the same as the directory
     name project.name
+    
+    licenseFile rootProject.file('LICENSE.txt')
+    noticeFile rootProject.file('NOTICE.txt')
   }
 }
diff --git a/plugins/discovery-azure-classic/build.gradle b/plugins/discovery-azure-classic/build.gradle
index 88874968b2125..e5ba37d9cb0d6 100644
--- a/plugins/discovery-azure-classic/build.gradle
+++ b/plugins/discovery-azure-classic/build.gradle
@@ -67,7 +67,7 @@ task createKey(type: LoggedExec) {
     project.delete(keystore.parentFile)
     keystore.parentFile.mkdirs()
   }
-  executable = 'keytool'
+  executable = new File(project.javaHome, 'bin/keytool')
   standardInput = new ByteArrayInputStream('FirstName LastName\nUnit\nOrganization\nCity\nState\nNL\nyes\n\n'.getBytes('UTF-8'))
   args '-genkey',
           '-alias', 'test-node',
@@ -125,107 +125,123 @@ thirdPartyAudit.excludes = [
   'org.osgi.framework.SynchronousBundleListener',
   'com.sun.xml.fastinfoset.stax.StAXDocumentParser', 
   'com.sun.xml.fastinfoset.stax.StAXDocumentSerializer',
-
-  // jarhell with jdk (intentionally, because jaxb was removed from default modules in java 9)
-  'javax.xml.bind.Binder',
-  'javax.xml.bind.ContextFinder$1',
-  'javax.xml.bind.ContextFinder',
-  'javax.xml.bind.DataBindingException',
-  'javax.xml.bind.DatatypeConverter',
-  'javax.xml.bind.DatatypeConverterImpl$CalendarFormatter',
-  'javax.xml.bind.DatatypeConverterImpl',
-  'javax.xml.bind.DatatypeConverterInterface',
-  'javax.xml.bind.Element',
-  'javax.xml.bind.GetPropertyAction',
-  'javax.xml.bind.JAXB$Cache',
-  'javax.xml.bind.JAXB',
-  'javax.xml.bind.JAXBContext',
-  'javax.xml.bind.JAXBElement$GlobalScope',
-  'javax.xml.bind.JAXBElement',
-  'javax.xml.bind.JAXBException',
-  'javax.xml.bind.JAXBIntrospector',
-  'javax.xml.bind.JAXBPermission',
-  'javax.xml.bind.MarshalException',
-  'javax.xml.bind.Marshaller$Listener',
-  'javax.xml.bind.Marshaller',
-  'javax.xml.bind.Messages',
-  'javax.xml.bind.NotIdentifiableEvent',
-  'javax.xml.bind.ParseConversionEvent',
-  'javax.xml.bind.PrintConversionEvent',
-  'javax.xml.bind.PropertyException',
-  'javax.xml.bind.SchemaOutputResolver',
-  'javax.xml.bind.TypeConstraintException',
-  'javax.xml.bind.UnmarshalException',
-  'javax.xml.bind.Unmarshaller$Listener',
-  'javax.xml.bind.Unmarshaller',
-  'javax.xml.bind.UnmarshallerHandler',
-  'javax.xml.bind.ValidationEvent',
-  'javax.xml.bind.ValidationEventHandler',
-  'javax.xml.bind.ValidationEventLocator',
-  'javax.xml.bind.ValidationException',
-  'javax.xml.bind.Validator',
-  'javax.xml.bind.WhiteSpaceProcessor',
-  'javax.xml.bind.annotation.DomHandler',
-  'javax.xml.bind.annotation.W3CDomHandler',
-  'javax.xml.bind.annotation.XmlAccessOrder',
-  'javax.xml.bind.annotation.XmlAccessType',
-  'javax.xml.bind.annotation.XmlAccessorOrder',
-  'javax.xml.bind.annotation.XmlAccessorType',
-  'javax.xml.bind.annotation.XmlAnyAttribute',
-  'javax.xml.bind.annotation.XmlAnyElement',
-  'javax.xml.bind.annotation.XmlAttachmentRef',
-  'javax.xml.bind.annotation.XmlAttribute',
-  'javax.xml.bind.annotation.XmlElement$DEFAULT',
-  'javax.xml.bind.annotation.XmlElement',
-  'javax.xml.bind.annotation.XmlElementDecl$GLOBAL',
-  'javax.xml.bind.annotation.XmlElementDecl',
-  'javax.xml.bind.annotation.XmlElementRef$DEFAULT',
-  'javax.xml.bind.annotation.XmlElementRef',
-  'javax.xml.bind.annotation.XmlElementRefs',
-  'javax.xml.bind.annotation.XmlElementWrapper',
-  'javax.xml.bind.annotation.XmlElements',
-  'javax.xml.bind.annotation.XmlEnum',
-  'javax.xml.bind.annotation.XmlEnumValue',
-  'javax.xml.bind.annotation.XmlID',
-  'javax.xml.bind.annotation.XmlIDREF',
-  'javax.xml.bind.annotation.XmlInlineBinaryData',
-  'javax.xml.bind.annotation.XmlList',
-  'javax.xml.bind.annotation.XmlMimeType',
-  'javax.xml.bind.annotation.XmlMixed',
-  'javax.xml.bind.annotation.XmlNs',
-  'javax.xml.bind.annotation.XmlNsForm',
-  'javax.xml.bind.annotation.XmlRegistry',
-  'javax.xml.bind.annotation.XmlRootElement',
-  'javax.xml.bind.annotation.XmlSchema',
-  'javax.xml.bind.annotation.XmlSchemaType$DEFAULT',
-  'javax.xml.bind.annotation.XmlSchemaType',
-  'javax.xml.bind.annotation.XmlSchemaTypes',
-  'javax.xml.bind.annotation.XmlSeeAlso',
-  'javax.xml.bind.annotation.XmlTransient',
-  'javax.xml.bind.annotation.XmlType$DEFAULT',
-  'javax.xml.bind.annotation.XmlType',
-  'javax.xml.bind.annotation.XmlValue',
-  'javax.xml.bind.annotation.adapters.CollapsedStringAdapter',
-  'javax.xml.bind.annotation.adapters.HexBinaryAdapter',
-  'javax.xml.bind.annotation.adapters.NormalizedStringAdapter',
-  'javax.xml.bind.annotation.adapters.XmlAdapter',
-  'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter$DEFAULT',
-  'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter',
-  'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapters',
-  'javax.xml.bind.attachment.AttachmentMarshaller',
-  'javax.xml.bind.attachment.AttachmentUnmarshaller',
-  'javax.xml.bind.helpers.AbstractMarshallerImpl',
-  'javax.xml.bind.helpers.AbstractUnmarshallerImpl',
-  'javax.xml.bind.helpers.DefaultValidationEventHandler',
-  'javax.xml.bind.helpers.Messages',
-  'javax.xml.bind.helpers.NotIdentifiableEventImpl',
-  'javax.xml.bind.helpers.ParseConversionEventImpl',
-  'javax.xml.bind.helpers.PrintConversionEventImpl',
-  'javax.xml.bind.helpers.ValidationEventImpl',
-  'javax.xml.bind.helpers.ValidationEventLocatorImpl',
-  'javax.xml.bind.util.JAXBResult',
-  'javax.xml.bind.util.JAXBSource$1',
-  'javax.xml.bind.util.JAXBSource',
-  'javax.xml.bind.util.Messages',
-  'javax.xml.bind.util.ValidationEventCollector'
 ]
+
+// jarhell with jdk (intentionally, because jaxb was removed from default modules in java 9)
+if (JavaVersion.current() <= JavaVersion.VERSION_1_8) {
+  thirdPartyAudit.excludes += [
+    'javax.xml.bind.Binder',
+    'javax.xml.bind.ContextFinder$1',
+    'javax.xml.bind.ContextFinder',
+    'javax.xml.bind.DataBindingException',
+    'javax.xml.bind.DatatypeConverter',
+    'javax.xml.bind.DatatypeConverterImpl$CalendarFormatter',
+    'javax.xml.bind.DatatypeConverterImpl',
+    'javax.xml.bind.DatatypeConverterInterface',
+    'javax.xml.bind.Element',
+    'javax.xml.bind.GetPropertyAction',
+    'javax.xml.bind.JAXB$Cache',
+    'javax.xml.bind.JAXB',
+    'javax.xml.bind.JAXBContext',
+    'javax.xml.bind.JAXBElement$GlobalScope',
+    'javax.xml.bind.JAXBElement',
+    'javax.xml.bind.JAXBException',
+    'javax.xml.bind.JAXBIntrospector',
+    'javax.xml.bind.JAXBPermission',
+    'javax.xml.bind.MarshalException',
+    'javax.xml.bind.Marshaller$Listener',
+    'javax.xml.bind.Marshaller',
+    'javax.xml.bind.Messages',
+    'javax.xml.bind.NotIdentifiableEvent',
+    'javax.xml.bind.ParseConversionEvent',
+    'javax.xml.bind.PrintConversionEvent',
+    'javax.xml.bind.PropertyException',
+    'javax.xml.bind.SchemaOutputResolver',
+    'javax.xml.bind.TypeConstraintException',
+    'javax.xml.bind.UnmarshalException',
+    'javax.xml.bind.Unmarshaller$Listener',
+    'javax.xml.bind.Unmarshaller',
+    'javax.xml.bind.UnmarshallerHandler',
+    'javax.xml.bind.ValidationEvent',
+    'javax.xml.bind.ValidationEventHandler',
+    'javax.xml.bind.ValidationEventLocator',
+    'javax.xml.bind.ValidationException',
+    'javax.xml.bind.Validator',
+    'javax.xml.bind.WhiteSpaceProcessor',
+    'javax.xml.bind.annotation.DomHandler',
+    'javax.xml.bind.annotation.W3CDomHandler',
+    'javax.xml.bind.annotation.XmlAccessOrder',
+    'javax.xml.bind.annotation.XmlAccessType',
+    'javax.xml.bind.annotation.XmlAccessorOrder',
+    'javax.xml.bind.annotation.XmlAccessorType',
+    'javax.xml.bind.annotation.XmlAnyAttribute',
+    'javax.xml.bind.annotation.XmlAnyElement',
+    'javax.xml.bind.annotation.XmlAttachmentRef',
+    'javax.xml.bind.annotation.XmlAttribute',
+    'javax.xml.bind.annotation.XmlElement$DEFAULT',
+    'javax.xml.bind.annotation.XmlElement',
+    'javax.xml.bind.annotation.XmlElementDecl$GLOBAL',
+    'javax.xml.bind.annotation.XmlElementDecl',
+    'javax.xml.bind.annotation.XmlElementRef$DEFAULT',
+    'javax.xml.bind.annotation.XmlElementRef',
+    'javax.xml.bind.annotation.XmlElementRefs',
+    'javax.xml.bind.annotation.XmlElementWrapper',
+    'javax.xml.bind.annotation.XmlElements',
+    'javax.xml.bind.annotation.XmlEnum',
+    'javax.xml.bind.annotation.XmlEnumValue',
+    'javax.xml.bind.annotation.XmlID',
+    'javax.xml.bind.annotation.XmlIDREF',
+    'javax.xml.bind.annotation.XmlInlineBinaryData',
+    'javax.xml.bind.annotation.XmlList',
+    'javax.xml.bind.annotation.XmlMimeType',
+    'javax.xml.bind.annotation.XmlMixed',
+    'javax.xml.bind.annotation.XmlNs',
+    'javax.xml.bind.annotation.XmlNsForm',
+    'javax.xml.bind.annotation.XmlRegistry',
+    'javax.xml.bind.annotation.XmlRootElement',
+    'javax.xml.bind.annotation.XmlSchema',
+    'javax.xml.bind.annotation.XmlSchemaType$DEFAULT',
+    'javax.xml.bind.annotation.XmlSchemaType',
+    'javax.xml.bind.annotation.XmlSchemaTypes',
+    'javax.xml.bind.annotation.XmlSeeAlso',
+    'javax.xml.bind.annotation.XmlTransient',
+    'javax.xml.bind.annotation.XmlType$DEFAULT',
+    'javax.xml.bind.annotation.XmlType',
+    'javax.xml.bind.annotation.XmlValue',
+    'javax.xml.bind.annotation.adapters.CollapsedStringAdapter',
+    'javax.xml.bind.annotation.adapters.HexBinaryAdapter',
+    'javax.xml.bind.annotation.adapters.NormalizedStringAdapter',
+    'javax.xml.bind.annotation.adapters.XmlAdapter',
+    'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter$DEFAULT',
+    'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter',
+    'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapters',
+    'javax.xml.bind.attachment.AttachmentMarshaller',
+    'javax.xml.bind.attachment.AttachmentUnmarshaller',
+    'javax.xml.bind.helpers.AbstractMarshallerImpl',
+    'javax.xml.bind.helpers.AbstractUnmarshallerImpl',
+    'javax.xml.bind.helpers.DefaultValidationEventHandler',
+    'javax.xml.bind.helpers.Messages',
+    'javax.xml.bind.helpers.NotIdentifiableEventImpl',
+    'javax.xml.bind.helpers.ParseConversionEventImpl',
+    'javax.xml.bind.helpers.PrintConversionEventImpl',
+    'javax.xml.bind.helpers.ValidationEventImpl',
+    'javax.xml.bind.helpers.ValidationEventLocatorImpl',
+    'javax.xml.bind.util.JAXBResult',
+    'javax.xml.bind.util.JAXBSource$1',
+    'javax.xml.bind.util.JAXBSource',
+    'javax.xml.bind.util.Messages',
+    'javax.xml.bind.util.ValidationEventCollector'
+  ]
+} else {
+  // jarhell with jdk (intentionally, because we still expect to run again JDK 8)
+  thirdPartyAudit.excludes += [
+    'javax.activation.ActivationDataFlavor',
+    'javax.activation.DataContentHandler',
+    'javax.activation.DataHandler',
+    'javax.activation.DataSource',
+    'javax.activation.FileDataSource',
+    'javax.activation.FileTypeMap',
+    'javax.activation.MimeType',
+    'javax.activation.MimeTypeParseException',
+  ]
+}
diff --git a/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/AzureDiscoveryModule.java b/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/AzureDiscoveryModule.java
deleted file mode 100644
index 7ee62dd8776ae..0000000000000
--- a/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/AzureDiscoveryModule.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.cloud.azure.classic;
-
-import org.apache.logging.log4j.Logger;
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.cloud.azure.classic.management.AzureComputeService;
-import org.elasticsearch.cloud.azure.classic.management.AzureComputeServiceImpl;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.AbstractModule;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.logging.Loggers;
-import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.discovery.DiscoveryModule;
-import org.elasticsearch.plugin.discovery.azure.classic.AzureDiscoveryPlugin;
-
-/**
- * Azure Module
- *
- * <ul>
- * <li>If needed this module will bind azure discovery service by default
- * to AzureComputeServiceImpl.</li>
- * </ul>
- *
- * @see AzureComputeServiceImpl
- */
-public class AzureDiscoveryModule extends AbstractModule {
-    protected final Logger logger;
-    private Settings settings;
-
-    // pkg private so it is settable by tests
-    Class<? extends AzureComputeService> computeServiceImpl = AzureComputeServiceImpl.class;
-
-    @Inject
-    public AzureDiscoveryModule(Settings settings) {
-        this.settings = settings;
-        this.logger = Loggers.getLogger(getClass(), settings);
-    }
-
-    @Override
-    protected void configure() {
-        logger.debug("starting azure services");
-        // If we have set discovery to azure, let's start the azure compute service
-        if (isDiscoveryReady(settings, logger)) {
-            logger.debug("starting azure discovery service");
-            bind(AzureComputeService.class).to(computeServiceImpl).asEagerSingleton();
-        }
-    }
-
-    /**
-     * Check if discovery is meant to start
-     * @return true if we can start discovery features
-     */
-    public static boolean isDiscoveryReady(Settings settings, Logger logger) {
-        // User set discovery.type: azure
-        if (!AzureDiscoveryPlugin.AZURE.equalsIgnoreCase(DiscoveryModule.DISCOVERY_TYPE_SETTING.get(settings))) {
-            logger.trace("discovery.type not set to {}", AzureDiscoveryPlugin.AZURE);
-            return false;
-        }
-
-        if (isDefined(settings, AzureComputeService.Management.SUBSCRIPTION_ID_SETTING) &&
-            isDefined(settings, AzureComputeService.Management.SERVICE_NAME_SETTING) &&
-            isDefined(settings, AzureComputeService.Management.KEYSTORE_PATH_SETTING) &&
-            isDefined(settings, AzureComputeService.Management.KEYSTORE_PASSWORD_SETTING)) {
-            logger.trace("All required properties for Azure discovery are set!");
-            return true;
-        } else {
-            logger.debug("One or more Azure discovery settings are missing. " +
-                            "Check elasticsearch.yml file. Should have [{}], [{}], [{}] and [{}].",
-                    AzureComputeService.Management.SUBSCRIPTION_ID_SETTING.getKey(),
-                    AzureComputeService.Management.SERVICE_NAME_SETTING.getKey(),
-                    AzureComputeService.Management.KEYSTORE_PATH_SETTING.getKey(),
-                    AzureComputeService.Management.KEYSTORE_PASSWORD_SETTING.getKey());
-            return false;
-        }
-    }
-
-    private static boolean isDefined(Settings settings, Setting<String> property) throws ElasticsearchException {
-        return (property.exists(settings) && Strings.hasText(property.get(settings)));
-    }
-}
diff --git a/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/management/AzureComputeServiceImpl.java b/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/management/AzureComputeServiceImpl.java
index 2375db2502bf1..09519b1449901 100644
--- a/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/management/AzureComputeServiceImpl.java
+++ b/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/management/AzureComputeServiceImpl.java
@@ -19,6 +19,9 @@
 
 package org.elasticsearch.cloud.azure.classic.management;
 
+import java.io.IOException;
+import java.util.ServiceLoader;
+
 import com.microsoft.windowsazure.Configuration;
 import com.microsoft.windowsazure.core.Builder;
 import com.microsoft.windowsazure.core.DefaultBuilder;
@@ -29,30 +32,24 @@
 import com.microsoft.windowsazure.management.configuration.ManagementConfiguration;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.cloud.azure.classic.AzureServiceRemoteException;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 
-import java.io.IOException;
-import java.util.ServiceLoader;
-
-/**
- *
- */
 public class AzureComputeServiceImpl extends AbstractLifecycleComponent
     implements AzureComputeService {
 
     private final ComputeManagementClient client;
     private final String serviceName;
 
-    @Inject
     public AzureComputeServiceImpl(Settings settings) {
         super(settings);
-        String subscriptionId = Management.SUBSCRIPTION_ID_SETTING.get(settings);
+        String subscriptionId = getRequiredSetting(settings, Management.SUBSCRIPTION_ID_SETTING);
 
-        serviceName = Management.SERVICE_NAME_SETTING.get(settings);
-        String keystorePath = Management.KEYSTORE_PATH_SETTING.get(settings);
-        String keystorePassword = Management.KEYSTORE_PASSWORD_SETTING.get(settings);
+        serviceName = getRequiredSetting(settings, Management.SERVICE_NAME_SETTING);
+        String keystorePath = getRequiredSetting(settings, Management.KEYSTORE_PATH_SETTING);
+        String keystorePassword = getRequiredSetting(settings, Management.KEYSTORE_PASSWORD_SETTING);
         KeyStoreType keystoreType = Management.KEYSTORE_TYPE_SETTING.get(settings);
 
         logger.trace("creating new Azure client for [{}], [{}]", subscriptionId, serviceName);
@@ -82,6 +79,14 @@ public AzureComputeServiceImpl(Settings settings) {
         }
     }
 
+    private static String getRequiredSetting(Settings settings, Setting<String> setting) {
+        String value = setting.get(settings);
+        if (value == null || Strings.hasLength(value) == false) {
+            throw new IllegalArgumentException("Missing required setting " + setting.getKey() + " for azure");
+        }
+        return value;
+    }
+
     @Override
     public HostedServiceGetDetailedResponse getServiceDetails() {
         try {
diff --git a/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/discovery/azure/classic/AzureUnicastHostsProvider.java b/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/discovery/azure/classic/AzureUnicastHostsProvider.java
index ed327a3a727b4..b21d0194071ee 100644
--- a/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/discovery/azure/classic/AzureUnicastHostsProvider.java
+++ b/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/discovery/azure/classic/AzureUnicastHostsProvider.java
@@ -39,7 +39,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.discovery.zen.ping.unicast.UnicastHostsProvider;
+import org.elasticsearch.discovery.zen.UnicastHostsProvider;
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
@@ -110,10 +110,8 @@ public static Deployment fromString(String string) {
     private final String deploymentName;
     private final DeploymentSlot deploymentSlot;
 
-    @Inject
     public AzureUnicastHostsProvider(Settings settings, AzureComputeService azureComputeService,
-                                   TransportService transportService,
-                                   NetworkService networkService) {
+                                     TransportService transportService, NetworkService networkService) {
         super(settings);
         this.azureComputeService = azureComputeService;
         this.transportService = transportService;
diff --git a/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/plugin/discovery/azure/classic/AzureDiscoveryPlugin.java b/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/plugin/discovery/azure/classic/AzureDiscoveryPlugin.java
index db5c1cc5c42f8..38e29e0458344 100644
--- a/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/plugin/discovery/azure/classic/AzureDiscoveryPlugin.java
+++ b/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/plugin/discovery/azure/classic/AzureDiscoveryPlugin.java
@@ -19,47 +19,67 @@
 
 package org.elasticsearch.plugin.discovery.azure.classic;
 
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Supplier;
+
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.cloud.azure.classic.AzureDiscoveryModule;
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.cloud.azure.classic.management.AzureComputeService;
-import org.elasticsearch.common.inject.Module;
+import org.elasticsearch.cloud.azure.classic.management.AzureComputeServiceImpl;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.discovery.Discovery;
 import org.elasticsearch.discovery.DiscoveryModule;
 import org.elasticsearch.discovery.azure.classic.AzureUnicastHostsProvider;
+import org.elasticsearch.discovery.zen.UnicastHostsProvider;
 import org.elasticsearch.discovery.zen.ZenDiscovery;
+import org.elasticsearch.discovery.zen.ZenPing;
+import org.elasticsearch.plugins.DiscoveryPlugin;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
 
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-
-public class AzureDiscoveryPlugin extends Plugin {
+public class AzureDiscoveryPlugin extends Plugin implements DiscoveryPlugin {
 
     public static final String AZURE = "azure";
-    private final Settings settings;
-    protected final Logger logger = Loggers.getLogger(AzureDiscoveryPlugin.class);
+    protected final Settings settings;
+    private static final Logger logger = Loggers.getLogger(AzureDiscoveryPlugin.class);
+    private static final DeprecationLogger deprecationLogger = new DeprecationLogger(logger);
 
     public AzureDiscoveryPlugin(Settings settings) {
         this.settings = settings;
-        DeprecationLogger deprecationLogger = new DeprecationLogger(logger);
         deprecationLogger.deprecated("azure classic discovery plugin is deprecated. Use azure arm discovery plugin instead");
         logger.trace("starting azure classic discovery plugin...");
     }
 
+    // overrideable for tests
+    protected AzureComputeService createComputeService() {
+        return new AzureComputeServiceImpl(settings);
+    }
+
     @Override
-    public Collection<Module> createGuiceModules() {
-        return Collections.singletonList((Module) new AzureDiscoveryModule(settings));
+    public Map<String, Supplier<UnicastHostsProvider>> getZenHostsProviders(TransportService transportService,
+                                                                            NetworkService networkService) {
+        return Collections.singletonMap(AZURE,
+            () -> new AzureUnicastHostsProvider(settings, createComputeService(), transportService, networkService));
     }
 
-    public void onModule(DiscoveryModule discoveryModule) {
-        if (AzureDiscoveryModule.isDiscoveryReady(settings, logger)) {
-            discoveryModule.addDiscoveryType(AZURE, ZenDiscovery.class);
-            discoveryModule.addUnicastHostProvider(AZURE, AzureUnicastHostsProvider.class);
-        }
+    @Override
+    public Map<String, Supplier<Discovery>> getDiscoveryTypes(ThreadPool threadPool, TransportService transportService,
+                                                              NamedWriteableRegistry namedWriteableRegistry,
+                                                              ClusterService clusterService, UnicastHostsProvider hostsProvider) {
+        // this is for backcompat with pre 5.1, where users would set discovery.type to use ec2 hosts provider
+        return Collections.singletonMap(AZURE, () ->
+            new ZenDiscovery(settings, threadPool, transportService, namedWriteableRegistry, clusterService, hostsProvider));
     }
 
     @Override
@@ -76,4 +96,19 @@ public List<Setting<?>> getSettings() {
                             AzureComputeService.Discovery.ENDPOINT_NAME_SETTING);
     }
 
+    @Override
+    public Settings additionalSettings() {
+        // For 5.0, the hosts provider was "zen", but this was before the discovery.zen.hosts_provider
+        // setting existed. This check looks for the legacy setting, and sets hosts provider if set
+        String discoveryType = DiscoveryModule.DISCOVERY_TYPE_SETTING.get(settings);
+        if (discoveryType.equals(AZURE)) {
+            deprecationLogger.deprecated("using [" + DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey() +
+                "] to set hosts provider is deprecated; " +
+                "set \"" + DiscoveryModule.DISCOVERY_HOSTS_PROVIDER_SETTING.getKey() + ": " + AZURE + "\" instead");
+            if (DiscoveryModule.DISCOVERY_HOSTS_PROVIDER_SETTING.exists(settings) == false) {
+                return Settings.builder().put(DiscoveryModule.DISCOVERY_HOSTS_PROVIDER_SETTING.getKey(), AZURE).build();
+            }
+        }
+        return Settings.EMPTY;
+    }
 }
diff --git a/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/cloud/azure/classic/AbstractAzureComputeServiceTestCase.java b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/cloud/azure/classic/AbstractAzureComputeServiceTestCase.java
index d7c3b92078074..c9496b1ead4f8 100644
--- a/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/cloud/azure/classic/AbstractAzureComputeServiceTestCase.java
+++ b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/cloud/azure/classic/AbstractAzureComputeServiceTestCase.java
@@ -57,7 +57,7 @@ protected Settings nodeSettings(int nodeOrdinal) {
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Arrays.asList(AzureDiscoveryPlugin.class, mockPlugin);
+        return Arrays.asList(mockPlugin);
     }
 
     protected void checkNumberOfNodes(int expected) {
diff --git a/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/cloud/azure/classic/AzureComputeServiceSimpleMock.java b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/cloud/azure/classic/AzureComputeServiceSimpleMock.java
index 66e853b59538f..2fbb4d7c4d72b 100644
--- a/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/cloud/azure/classic/AzureComputeServiceSimpleMock.java
+++ b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/cloud/azure/classic/AzureComputeServiceSimpleMock.java
@@ -19,32 +19,35 @@
 
 package org.elasticsearch.cloud.azure.classic;
 
+import java.net.InetAddress;
+
 import com.microsoft.windowsazure.management.compute.models.DeploymentSlot;
 import com.microsoft.windowsazure.management.compute.models.DeploymentStatus;
 import com.microsoft.windowsazure.management.compute.models.HostedServiceGetDetailedResponse;
 import com.microsoft.windowsazure.management.compute.models.InstanceEndpoint;
 import com.microsoft.windowsazure.management.compute.models.RoleInstance;
+import org.elasticsearch.cloud.azure.classic.management.AzureComputeService;
 import org.elasticsearch.cloud.azure.classic.management.AzureComputeServiceAbstractMock;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.CollectionUtils;
-import org.elasticsearch.plugins.Plugin;
-
-import java.net.InetAddress;
+import org.elasticsearch.plugin.discovery.azure.classic.AzureDiscoveryPlugin;
 
 /**
  * Mock Azure API with a single started node
  */
 public class AzureComputeServiceSimpleMock extends AzureComputeServiceAbstractMock {
 
-    public static class TestPlugin extends Plugin {
-        public void onModule(AzureDiscoveryModule azureDiscoveryModule) {
-            azureDiscoveryModule.computeServiceImpl = AzureComputeServiceSimpleMock.class;
+    public static class TestPlugin extends AzureDiscoveryPlugin {
+        public TestPlugin(Settings settings) {
+            super(settings);
+        }
+        @Override
+        protected AzureComputeService createComputeService() {
+            return new AzureComputeServiceSimpleMock(settings);
         }
     }
 
-    @Inject
-    public AzureComputeServiceSimpleMock(Settings settings) {
+    private AzureComputeServiceSimpleMock(Settings settings) {
         super(settings);
     }
 
diff --git a/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/cloud/azure/classic/AzureComputeServiceTwoNodesMock.java b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/cloud/azure/classic/AzureComputeServiceTwoNodesMock.java
index d75ce22d55c9f..5b787e1ce4448 100644
--- a/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/cloud/azure/classic/AzureComputeServiceTwoNodesMock.java
+++ b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/cloud/azure/classic/AzureComputeServiceTwoNodesMock.java
@@ -19,20 +19,19 @@
 
 package org.elasticsearch.cloud.azure.classic;
 
+import java.net.InetAddress;
+import java.util.ArrayList;
+import java.util.Arrays;
+
 import com.microsoft.windowsazure.management.compute.models.DeploymentSlot;
 import com.microsoft.windowsazure.management.compute.models.DeploymentStatus;
 import com.microsoft.windowsazure.management.compute.models.HostedServiceGetDetailedResponse;
 import com.microsoft.windowsazure.management.compute.models.InstanceEndpoint;
 import com.microsoft.windowsazure.management.compute.models.RoleInstance;
+import org.elasticsearch.cloud.azure.classic.management.AzureComputeService;
 import org.elasticsearch.cloud.azure.classic.management.AzureComputeServiceAbstractMock;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.plugins.Plugin;
-
-import java.net.InetAddress;
-import java.util.ArrayList;
-import java.util.Arrays;
+import org.elasticsearch.plugin.discovery.azure.classic.AzureDiscoveryPlugin;
 
 import static org.elasticsearch.common.util.CollectionUtils.newSingletonArrayList;
 
@@ -41,18 +40,19 @@
  * Mock Azure API with two started nodes
  */
 public class AzureComputeServiceTwoNodesMock extends AzureComputeServiceAbstractMock {
-    public static class TestPlugin extends Plugin {
-        public void onModule(AzureDiscoveryModule azureDiscoveryModule) {
-            azureDiscoveryModule.computeServiceImpl = AzureComputeServiceTwoNodesMock.class;
+
+    public static class TestPlugin extends AzureDiscoveryPlugin {
+        public TestPlugin(Settings settings) {
+            super(settings);
+        }
+        @Override
+        protected AzureComputeService createComputeService() {
+            return new AzureComputeServiceTwoNodesMock(settings);
         }
     }
 
-    NetworkService networkService;
-
-    @Inject
-    protected AzureComputeServiceTwoNodesMock(Settings settings, NetworkService networkService) {
+    private AzureComputeServiceTwoNodesMock(Settings settings) {
         super(settings);
-        this.networkService = networkService;
     }
 
     @Override
diff --git a/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureDiscoveryClusterFormationTests.java b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureDiscoveryClusterFormationTests.java
index b3f1afb35a1c6..6e2871f7c762c 100644
--- a/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureDiscoveryClusterFormationTests.java
+++ b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureDiscoveryClusterFormationTests.java
@@ -64,8 +64,6 @@
 import java.util.UUID;
 import java.util.concurrent.ExecutionException;
 
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoTimeout;
-
 @ESIntegTestCase.SuppressLocalMode
 @ESIntegTestCase.ClusterScope(numDataNodes = 2, numClientNodes = 0)
 @SuppressForbidden(reason = "use http server")
@@ -111,7 +109,7 @@ protected Settings nodeSettings(int nodeOrdinal) {
             .put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), AzureDiscoveryPlugin.AZURE)
             .put(Environment.PATH_LOGS_SETTING.getKey(), resolve)
             .put(TransportSettings.PORT.getKey(), 0)
-            .put(Node.WRITE_PORTS_FIELD_SETTING.getKey(), "true")
+            .put(Node.WRITE_PORTS_FILE_SETTING.getKey(), "true")
             .put(AzureComputeService.Management.ENDPOINT_SETTING.getKey(), "https://" + InetAddress.getLoopbackAddress().getHostAddress() +
                 ":" + httpsServer.getAddress().getPort())
             .put(Environment.PATH_CONF_SETTING.getKey(), keyStoreFile.getParent().toAbsolutePath())
@@ -270,7 +268,7 @@ public void testJoin() throws ExecutionException, InterruptedException {
         // only wait for the cluster to form
         ensureClusterSizeConsistency();
         // add one more node and wait for it to join
-        internalCluster().startDataOnlyNodeAsync().get();
+        internalCluster().startDataOnlyNode();
         ensureClusterSizeConsistency();
     }
 }
diff --git a/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureMinimumMasterNodesTests.java b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureMinimumMasterNodesTests.java
index 72e1f2da7910f..f8b105884bfe7 100644
--- a/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureMinimumMasterNodesTests.java
+++ b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureMinimumMasterNodesTests.java
@@ -39,7 +39,8 @@
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE,
         numDataNodes = 0,
         transportClientRatio = 0.0,
-        numClientNodes = 0)
+        numClientNodes = 0,
+        autoMinMasterNodes = false)
 @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch-cloud-azure/issues/89")
 public class AzureMinimumMasterNodesTests extends AbstractAzureComputeServiceTestCase {
 
diff --git a/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/DiscoveryAzureClassicClientYamlTestSuiteIT.java b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/DiscoveryAzureClassicClientYamlTestSuiteIT.java
index 30276c16c8917..4ad70a98c8c53 100644
--- a/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/DiscoveryAzureClassicClientYamlTestSuiteIT.java
+++ b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/DiscoveryAzureClassicClientYamlTestSuiteIT.java
@@ -24,9 +24,6 @@
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-
-import java.io.IOException;
 
 public class DiscoveryAzureClassicClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
@@ -35,8 +32,8 @@ public DiscoveryAzureClassicClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCa
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
-        return ESClientYamlSuiteTestCase.createParameters(0, 1);
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
     }
 }
 
diff --git a/plugins/discovery-ec2/build.gradle b/plugins/discovery-ec2/build.gradle
index 9ad2f8d02e29f..815e50f61c12c 100644
--- a/plugins/discovery-ec2/build.gradle
+++ b/plugins/discovery-ec2/build.gradle
@@ -19,7 +19,7 @@
 
 esplugin {
   description 'The EC2 discovery plugin allows to use AWS API for the unicast discovery mechanism.'
-  classname 'org.elasticsearch.plugin.discovery.ec2.Ec2DiscoveryPlugin'
+  classname 'org.elasticsearch.discovery.ec2.Ec2DiscoveryPlugin'
 }
 
 versions << [
@@ -61,3 +61,7 @@ thirdPartyAudit.excludes = [
   'org.apache.log.Hierarchy', 
   'org.apache.log.Logger',
 ]
+
+if (JavaVersion.current() > JavaVersion.VERSION_1_8) {
+  thirdPartyAudit.excludes += ['javax.xml.bind.DatatypeConverter']
+}
diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java
deleted file mode 100644
index 139b78226cf9a..0000000000000
--- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java
+++ /dev/null
@@ -1,204 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.cloud.aws;
-
-import com.amazonaws.Protocol;
-import com.amazonaws.services.ec2.AmazonEC2;
-import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Setting.Property;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.TimeValue;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Locale;
-import java.util.function.Function;
-
-public interface AwsEc2Service {
-    Setting<Boolean> AUTO_ATTRIBUTE_SETTING = Setting.boolSetting("cloud.node.auto_attributes", false, Property.NodeScope);
-
-    // Global AWS settings (shared between discovery-ec2 and repository-s3)
-    // Each setting starting with `cloud.aws` also exists in repository-s3 project. Don't forget to update
-    // the code there if you change anything here.
-    /**
-     * cloud.aws.access_key: AWS Access key. Shared with repository-s3 plugin
-     */
-    Setting<String> KEY_SETTING =
-        Setting.simpleString("cloud.aws.access_key", Property.NodeScope, Property.Filtered, Property.Shared);
-    /**
-     * cloud.aws.secret_key: AWS Secret key. Shared with repository-s3 plugin
-     */
-    Setting<String> SECRET_SETTING =
-        Setting.simpleString("cloud.aws.secret_key", Property.NodeScope, Property.Filtered, Property.Shared);
-    /**
-     * cloud.aws.protocol: Protocol for AWS API: http or https. Defaults to https. Shared with repository-s3 plugin
-     */
-    Setting<Protocol> PROTOCOL_SETTING = new Setting<>("cloud.aws.protocol", "https", s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)),
-        Property.NodeScope, Property.Shared);
-    /**
-     * cloud.aws.proxy.host: In case of proxy, define its hostname/IP. Shared with repository-s3 plugin
-     */
-    Setting<String> PROXY_HOST_SETTING = Setting.simpleString("cloud.aws.proxy.host", Property.NodeScope, Property.Shared);
-    /**
-     * cloud.aws.proxy.port: In case of proxy, define its port. Defaults to 80. Shared with repository-s3 plugin
-     */
-    Setting<Integer> PROXY_PORT_SETTING = Setting.intSetting("cloud.aws.proxy.port", 80, 0, 1<<16, Property.NodeScope,
-        Property.Shared);
-    /**
-     * cloud.aws.proxy.username: In case of proxy with auth, define the username. Shared with repository-s3 plugin
-     */
-    Setting<String> PROXY_USERNAME_SETTING = Setting.simpleString("cloud.aws.proxy.username", Property.NodeScope, Property.Shared);
-    /**
-     * cloud.aws.proxy.password: In case of proxy with auth, define the password. Shared with repository-s3 plugin
-     */
-    Setting<String> PROXY_PASSWORD_SETTING =
-        Setting.simpleString("cloud.aws.proxy.password", Property.NodeScope, Property.Filtered, Property.Shared);
-    /**
-     * cloud.aws.signer: If you are using an old AWS API version, you can define a Signer. Shared with repository-s3 plugin
-     */
-    Setting<String> SIGNER_SETTING = Setting.simpleString("cloud.aws.signer", Property.NodeScope, Property.Shared);
-    /**
-     * cloud.aws.region: Region. Shared with repository-s3 plugin
-     */
-    Setting<String> REGION_SETTING =
-        new Setting<>("cloud.aws.region", "", s -> s.toLowerCase(Locale.ROOT), Property.NodeScope, Property.Shared);
-
-    /**
-     * Defines specific ec2 settings starting with cloud.aws.ec2.
-     */
-    interface CLOUD_EC2 {
-        /**
-         * cloud.aws.ec2.access_key: AWS Access key specific for EC2 API calls. Defaults to cloud.aws.access_key.
-         * @see AwsEc2Service#KEY_SETTING
-         */
-        Setting<String> KEY_SETTING = new Setting<>("cloud.aws.ec2.access_key", AwsEc2Service.KEY_SETTING, Function.identity(),
-            Property.NodeScope, Property.Filtered);
-        /**
-         * cloud.aws.ec2.secret_key: AWS Secret key specific for EC2 API calls. Defaults to cloud.aws.secret_key.
-         * @see AwsEc2Service#SECRET_SETTING
-         */
-        Setting<String> SECRET_SETTING = new Setting<>("cloud.aws.ec2.secret_key", AwsEc2Service.SECRET_SETTING, Function.identity(),
-            Property.NodeScope, Property.Filtered);
-        /**
-         * cloud.aws.ec2.protocol: Protocol for AWS API specific for EC2 API calls: http or https.  Defaults to cloud.aws.protocol.
-         * @see AwsEc2Service#PROTOCOL_SETTING
-         */
-        Setting<Protocol> PROTOCOL_SETTING = new Setting<>("cloud.aws.ec2.protocol", AwsEc2Service.PROTOCOL_SETTING,
-            s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)), Property.NodeScope);
-        /**
-         * cloud.aws.ec2.proxy.host: In case of proxy, define its hostname/IP specific for EC2 API calls. Defaults to cloud.aws.proxy.host.
-         * @see AwsEc2Service#PROXY_HOST_SETTING
-         */
-        Setting<String> PROXY_HOST_SETTING = new Setting<>("cloud.aws.ec2.proxy.host", AwsEc2Service.PROXY_HOST_SETTING,
-            Function.identity(), Property.NodeScope);
-        /**
-         * cloud.aws.ec2.proxy.port: In case of proxy, define its port specific for EC2 API calls.  Defaults to cloud.aws.proxy.port.
-         * @see AwsEc2Service#PROXY_PORT_SETTING
-         */
-        Setting<Integer> PROXY_PORT_SETTING = new Setting<>("cloud.aws.ec2.proxy.port", AwsEc2Service.PROXY_PORT_SETTING,
-            s -> Setting.parseInt(s, 0, 1<<16, "cloud.aws.ec2.proxy.port"), Property.NodeScope);
-        /**
-         * cloud.aws.ec2.proxy.username: In case of proxy with auth, define the username specific for EC2 API calls.
-         * Defaults to cloud.aws.proxy.username.
-         * @see AwsEc2Service#PROXY_USERNAME_SETTING
-         */
-        Setting<String> PROXY_USERNAME_SETTING = new Setting<>("cloud.aws.ec2.proxy.username", AwsEc2Service.PROXY_USERNAME_SETTING,
-            Function.identity(), Property.NodeScope);
-        /**
-         * cloud.aws.ec2.proxy.password: In case of proxy with auth, define the password specific for EC2 API calls.
-         * Defaults to cloud.aws.proxy.password.
-         * @see AwsEc2Service#PROXY_PASSWORD_SETTING
-         */
-        Setting<String> PROXY_PASSWORD_SETTING = new Setting<>("cloud.aws.ec2.proxy.password", AwsEc2Service.PROXY_PASSWORD_SETTING,
-            Function.identity(), Property.NodeScope, Property.Filtered);
-        /**
-         * cloud.aws.ec2.signer: If you are using an old AWS API version, you can define a Signer. Specific for EC2 API calls.
-         * Defaults to cloud.aws.signer.
-         * @see AwsEc2Service#SIGNER_SETTING
-         */
-        Setting<String> SIGNER_SETTING = new Setting<>("cloud.aws.ec2.signer", AwsEc2Service.SIGNER_SETTING, Function.identity(),
-            Property.NodeScope);
-        /**
-         * cloud.aws.ec2.region: Region specific for EC2 API calls. Defaults to cloud.aws.region.
-         * @see AwsEc2Service#REGION_SETTING
-         */
-        Setting<String> REGION_SETTING = new Setting<>("cloud.aws.ec2.region", AwsEc2Service.REGION_SETTING,
-            s -> s.toLowerCase(Locale.ROOT), Property.NodeScope);
-        /**
-         * cloud.aws.ec2.endpoint: Endpoint. If not set, endpoint will be guessed based on region setting.
-         */
-        Setting<String> ENDPOINT_SETTING = Setting.simpleString("cloud.aws.ec2.endpoint", Property.NodeScope);
-    }
-
-    /**
-     * Defines discovery settings for ec2. Starting with discovery.ec2.
-     */
-    interface DISCOVERY_EC2 {
-        enum HostType {
-            PRIVATE_IP,
-            PUBLIC_IP,
-            PRIVATE_DNS,
-            PUBLIC_DNS
-        }
-
-        /**
-         * discovery.ec2.host_type: The type of host type to use to communicate with other instances.
-         * Can be one of private_ip, public_ip, private_dns, public_dns. Defaults to private_ip.
-         */
-        Setting<HostType> HOST_TYPE_SETTING =
-            new Setting<>("discovery.ec2.host_type", HostType.PRIVATE_IP.name(), s -> HostType.valueOf(s.toUpperCase(Locale.ROOT)),
-                Property.NodeScope);
-        /**
-         * discovery.ec2.any_group: If set to false, will require all security groups to be present for the instance to be used for the
-         * discovery. Defaults to true.
-         */
-        Setting<Boolean> ANY_GROUP_SETTING =
-            Setting.boolSetting("discovery.ec2.any_group", true, Property.NodeScope);
-        /**
-         * discovery.ec2.groups: Either a comma separated list or array based list of (security) groups. Only instances with the provided
-         * security groups will be used in the cluster discovery. (NOTE: You could provide either group NAME or group ID.)
-         */
-        Setting<List<String>> GROUPS_SETTING =
-            Setting.listSetting("discovery.ec2.groups", new ArrayList<>(), s -> s.toString(), Property.NodeScope);
-        /**
-         * discovery.ec2.availability_zones: Either a comma separated list or array based list of availability zones. Only instances within
-         * the provided availability zones will be used in the cluster discovery.
-         */
-        Setting<List<String>> AVAILABILITY_ZONES_SETTING =
-            Setting.listSetting("discovery.ec2.availability_zones", Collections.emptyList(), s -> s.toString(),
-                Property.NodeScope);
-        /**
-         * discovery.ec2.node_cache_time: How long the list of hosts is cached to prevent further requests to the AWS API. Defaults to 10s.
-         */
-        Setting<TimeValue> NODE_CACHE_TIME_SETTING =
-            Setting.timeSetting("discovery.ec2.node_cache_time", TimeValue.timeValueSeconds(10), Property.NodeScope);
-
-        /**
-         * discovery.ec2.tag.*: The ec2 discovery can filter machines to include in the cluster based on tags (and not just groups).
-         * The settings to use include the discovery.ec2.tag. prefix. For example, setting discovery.ec2.tag.stage to dev will only filter
-         * instances with a tag key set to stage, and a value of dev. Several tags set will require all of those tags to be set for the
-         * instance to be included.
-         */
-        Setting<Settings> TAG_SETTING = Setting.groupSetting("discovery.ec2.tag.", Property.NodeScope);
-    }
-
-    AmazonEC2 client();
-}
diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImpl.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImpl.java
deleted file mode 100644
index 1a4bf278f3a4c..0000000000000
--- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImpl.java
+++ /dev/null
@@ -1,217 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.cloud.aws;
-
-import com.amazonaws.AmazonClientException;
-import com.amazonaws.AmazonWebServiceRequest;
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.BasicAWSCredentials;
-import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
-import com.amazonaws.http.IdleConnectionReaper;
-import com.amazonaws.internal.StaticCredentialsProvider;
-import com.amazonaws.retry.RetryPolicy;
-import com.amazonaws.services.ec2.AmazonEC2;
-import com.amazonaws.services.ec2.AmazonEC2Client;
-import org.apache.logging.log4j.Logger;
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.common.Randomness;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.component.AbstractLifecycleComponent;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.settings.Settings;
-
-import java.util.Random;
-
-/**
- *
- */
-public class AwsEc2ServiceImpl extends AbstractLifecycleComponent implements AwsEc2Service {
-
-    public static final String EC2_METADATA_URL = "http://169.254.169.254/latest/meta-data/";
-
-    private AmazonEC2Client client;
-
-    @Inject
-    public AwsEc2ServiceImpl(Settings settings) {
-        super(settings);
-    }
-
-    @Override
-    public synchronized AmazonEC2 client() {
-        if (client != null) {
-            return client;
-        }
-
-        this.client = new AmazonEC2Client(buildCredentials(logger, settings), buildConfiguration(logger, settings));
-        String endpoint = findEndpoint(logger, settings);
-        if (endpoint != null) {
-            client.setEndpoint(endpoint);
-        }
-
-        return this.client;
-    }
-
-    protected static AWSCredentialsProvider buildCredentials(Logger logger, Settings settings) {
-        AWSCredentialsProvider credentials;
-
-        String key = CLOUD_EC2.KEY_SETTING.get(settings);
-        String secret = CLOUD_EC2.SECRET_SETTING.get(settings);
-        if (key.isEmpty() && secret.isEmpty()) {
-            logger.debug("Using either environment variables, system properties or instance profile credentials");
-            credentials = new DefaultAWSCredentialsProviderChain();
-        } else {
-            logger.debug("Using basic key/secret credentials");
-            credentials = new StaticCredentialsProvider(new BasicAWSCredentials(key, secret));
-        }
-
-        return credentials;
-    }
-
-    protected static ClientConfiguration buildConfiguration(Logger logger, Settings settings) {
-        ClientConfiguration clientConfiguration = new ClientConfiguration();
-        // the response metadata cache is only there for diagnostics purposes,
-        // but can force objects from every response to the old generation.
-        clientConfiguration.setResponseMetadataCacheSize(0);
-        clientConfiguration.setProtocol(CLOUD_EC2.PROTOCOL_SETTING.get(settings));
-
-        if (PROXY_HOST_SETTING.exists(settings) || CLOUD_EC2.PROXY_HOST_SETTING.exists(settings)) {
-            String proxyHost = CLOUD_EC2.PROXY_HOST_SETTING.get(settings);
-            Integer proxyPort = CLOUD_EC2.PROXY_PORT_SETTING.get(settings);
-            String proxyUsername = CLOUD_EC2.PROXY_USERNAME_SETTING.get(settings);
-            String proxyPassword = CLOUD_EC2.PROXY_PASSWORD_SETTING.get(settings);
-
-            clientConfiguration
-                .withProxyHost(proxyHost)
-                .withProxyPort(proxyPort)
-                .withProxyUsername(proxyUsername)
-                .withProxyPassword(proxyPassword);
-        }
-
-        // #155: we might have 3rd party users using older EC2 API version
-        String awsSigner = CLOUD_EC2.SIGNER_SETTING.get(settings);
-        if (Strings.hasText(awsSigner)) {
-            logger.debug("using AWS API signer [{}]", awsSigner);
-            AwsSigner.configureSigner(awsSigner, clientConfiguration);
-        }
-
-        // Increase the number of retries in case of 5xx API responses
-        final Random rand = Randomness.get();
-        RetryPolicy retryPolicy = new RetryPolicy(
-            RetryPolicy.RetryCondition.NO_RETRY_CONDITION,
-            new RetryPolicy.BackoffStrategy() {
-                @Override
-                public long delayBeforeNextRetry(AmazonWebServiceRequest originalRequest,
-                                                 AmazonClientException exception,
-                                                 int retriesAttempted) {
-                    // with 10 retries the max delay time is 320s/320000ms (10 * 2^5 * 1 * 1000)
-                    logger.warn("EC2 API request failed, retry again. Reason was:", exception);
-                    return 1000L * (long) (10d * Math.pow(2, retriesAttempted / 2.0d) * (1.0d + rand.nextDouble()));
-                }
-            },
-            10,
-            false);
-        clientConfiguration.setRetryPolicy(retryPolicy);
-
-        return clientConfiguration;
-    }
-
-    protected static String findEndpoint(Logger logger, Settings settings) {
-        String endpoint = null;
-        if (CLOUD_EC2.ENDPOINT_SETTING.exists(settings)) {
-            endpoint = CLOUD_EC2.ENDPOINT_SETTING.get(settings);
-            logger.debug("using explicit ec2 endpoint [{}]", endpoint);
-        } else if (REGION_SETTING.exists(settings) || CLOUD_EC2.REGION_SETTING.exists(settings)) {
-            final String region = CLOUD_EC2.REGION_SETTING.get(settings);
-            switch (region) {
-                case "us-east-1":
-                case "us-east":
-                    endpoint = "ec2.us-east-1.amazonaws.com";
-                    break;
-                case "us-west":
-                case "us-west-1":
-                    endpoint = "ec2.us-west-1.amazonaws.com";
-                    break;
-                case "us-west-2":
-                    endpoint = "ec2.us-west-2.amazonaws.com";
-                    break;
-                case "ap-southeast":
-                case "ap-southeast-1":
-                    endpoint = "ec2.ap-southeast-1.amazonaws.com";
-                    break;
-                case "ap-south-1":
-                    endpoint = "ec2.ap-south-1.amazonaws.com";
-                    break;
-                case "us-gov-west":
-                case "us-gov-west-1":
-                    endpoint = "ec2.us-gov-west-1.amazonaws.com";
-                    break;
-                case "ap-southeast-2":
-                    endpoint = "ec2.ap-southeast-2.amazonaws.com";
-                    break;
-                case "ap-northeast":
-                case "ap-northeast-1":
-                    endpoint = "ec2.ap-northeast-1.amazonaws.com";
-                    break;
-                case "ap-northeast-2":
-                    endpoint = "ec2.ap-northeast-2.amazonaws.com";
-                    break;
-                case "eu-west":
-                case "eu-west-1":
-                    endpoint = "ec2.eu-west-1.amazonaws.com";
-                    break;
-                case "eu-central":
-                case "eu-central-1":
-                    endpoint = "ec2.eu-central-1.amazonaws.com";
-                    break;
-                case "sa-east":
-                case "sa-east-1":
-                    endpoint = "ec2.sa-east-1.amazonaws.com";
-                    break;
-                case "cn-north":
-                case "cn-north-1":
-                    endpoint = "ec2.cn-north-1.amazonaws.com.cn";
-                    break;
-                default:
-                    throw new IllegalArgumentException("No automatic endpoint could be derived from region [" + region + "]");
-            }
-            logger.debug("using ec2 region [{}], with endpoint [{}]", region, endpoint);
-        }
-        return endpoint;
-    }
-
-    @Override
-    protected void doStart() throws ElasticsearchException {
-    }
-
-    @Override
-    protected void doStop() throws ElasticsearchException {
-    }
-
-    @Override
-    protected void doClose() throws ElasticsearchException {
-        if (client != null) {
-            client.shutdown();
-        }
-
-        // Ensure that IdleConnectionReaper is shutdown
-        IdleConnectionReaper.shutdown();
-    }
-}
diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsSigner.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsSigner.java
deleted file mode 100644
index 11732725e9dfe..0000000000000
--- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsSigner.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.cloud.aws;
-
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.auth.SignerFactory;
-import org.apache.logging.log4j.Logger;
-import org.elasticsearch.common.logging.Loggers;
-
-public class AwsSigner {
-
-    private static final Logger logger = Loggers.getLogger(AwsSigner.class);
-
-    private AwsSigner() {
-
-    }
-
-    protected static void validateSignerType(String signer) throws IllegalArgumentException {
-        if (signer == null) {
-            throw new IllegalArgumentException("[null] signer set");
-        }
-
-        try {
-            // We check this signer actually exists in AWS SDK
-            // It throws a IllegalArgumentException if not found
-            SignerFactory.getSignerByTypeAndService(signer, null);
-        } catch (IllegalArgumentException e) {
-            throw new IllegalArgumentException("wrong signer set [" + signer + "]");
-        }
-    }
-
-    /**
-     * Add a AWS API Signer.
-     * @param signer Signer to use
-     * @param configuration AWS Client configuration
-     */
-    public static void configureSigner(String signer, ClientConfiguration configuration) {
-        try {
-            validateSignerType(signer);
-        } catch (IllegalArgumentException e) {
-            logger.warn("{}", e.getMessage());
-        }
-
-        configuration.setSignerOverride(signer);
-    }
-}
diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/Ec2Module.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/Ec2Module.java
deleted file mode 100644
index b7cc7a0938610..0000000000000
--- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/Ec2Module.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.cloud.aws;
-
-import org.elasticsearch.common.inject.AbstractModule;
-
-public class Ec2Module extends AbstractModule {
-
-    @Override
-    protected void configure() {
-        bind(AwsEc2Service.class).to(AwsEc2ServiceImpl.class).asEagerSingleton();
-    }
-
-}
diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2Service.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2Service.java
new file mode 100644
index 0000000000000..0b1fdca257d34
--- /dev/null
+++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2Service.java
@@ -0,0 +1,257 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.discovery.ec2;
+
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.Protocol;
+import com.amazonaws.services.ec2.AmazonEC2;
+import org.elasticsearch.common.settings.SecureSetting;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Locale;
+import java.util.function.Function;
+
+interface AwsEc2Service {
+    Setting<Boolean> AUTO_ATTRIBUTE_SETTING = Setting.boolSetting("cloud.node.auto_attributes", false, Property.NodeScope);
+
+    // Global AWS settings (shared between discovery-ec2 and repository-s3)
+    // Each setting starting with `cloud.aws` also exists in repository-s3 project. Don't forget to update
+    // the code there if you change anything here.
+    /**
+     * cloud.aws.access_key: AWS Access key. Shared with repository-s3 plugin
+     */
+    Setting<SecureString> KEY_SETTING = new Setting<>("cloud.aws.access_key", "", SecureString::new,
+            Property.NodeScope, Property.Filtered, Property.Shared, Property.Deprecated);
+    /**
+     * cloud.aws.secret_key: AWS Secret key. Shared with repository-s3 plugin
+     */
+    Setting<SecureString> SECRET_SETTING = new Setting<>("cloud.aws.secret_key", "", SecureString::new,
+        Property.NodeScope, Property.Filtered, Property.Shared, Property.Deprecated);
+    /**
+     * cloud.aws.protocol: Protocol for AWS API: http or https. Defaults to https. Shared with repository-s3 plugin
+     */
+    Setting<Protocol> PROTOCOL_SETTING = new Setting<>("cloud.aws.protocol", "https", s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)),
+        Property.NodeScope, Property.Shared, Property.Deprecated);
+    /**
+     * cloud.aws.proxy.host: In case of proxy, define its hostname/IP. Shared with repository-s3 plugin
+     */
+    Setting<String> PROXY_HOST_SETTING = Setting.simpleString("cloud.aws.proxy.host",
+        Property.NodeScope, Property.Shared, Property.Deprecated);
+    /**
+     * cloud.aws.proxy.port: In case of proxy, define its port. Defaults to 80. Shared with repository-s3 plugin
+     */
+    Setting<Integer> PROXY_PORT_SETTING = Setting.intSetting("cloud.aws.proxy.port", 80, 0, 1<<16,
+        Property.NodeScope, Property.Shared, Property.Deprecated);
+    /**
+     * cloud.aws.proxy.username: In case of proxy with auth, define the username. Shared with repository-s3 plugin
+     */
+    Setting<SecureString> PROXY_USERNAME_SETTING = new Setting<>("cloud.aws.proxy.username", "", SecureString::new,
+        Property.NodeScope, Property.Filtered, Property.Shared, Property.Deprecated);
+    /**
+     * cloud.aws.proxy.password: In case of proxy with auth, define the password. Shared with repository-s3 plugin
+     */
+    Setting<SecureString> PROXY_PASSWORD_SETTING = new Setting<>("cloud.aws.proxy.password", "", SecureString::new,
+        Property.NodeScope, Property.Filtered, Property.Shared, Property.Deprecated);
+    /**
+     * cloud.aws.signer: If you are using an old AWS API version, you can define a Signer. Shared with repository-s3 plugin
+     */
+    Setting<String> SIGNER_SETTING = Setting.simpleString("cloud.aws.signer", Property.NodeScope, Property.Shared, Property.Deprecated);
+    /**
+     * cloud.aws.region: Region. Shared with repository-s3 plugin
+     */
+    Setting<String> REGION_SETTING =
+        new Setting<>("cloud.aws.region", "", s -> s.toLowerCase(Locale.ROOT), Property.NodeScope, Property.Shared, Property.Deprecated);
+    /**
+     * cloud.aws.read_timeout: Socket read timeout. Shared with repository-s3 plugin
+     */
+    Setting<TimeValue> READ_TIMEOUT = Setting.timeSetting("cloud.aws.read_timeout",
+        TimeValue.timeValueMillis(ClientConfiguration.DEFAULT_SOCKET_TIMEOUT), Property.NodeScope, Property.Shared, Property.Deprecated);
+
+    /**
+     * Defines specific ec2 settings starting with cloud.aws.ec2.
+     */
+    interface CLOUD_EC2 {
+        /**
+         * cloud.aws.ec2.access_key: AWS Access key specific for EC2 API calls. Defaults to cloud.aws.access_key.
+         * @see AwsEc2Service#KEY_SETTING
+         */
+        Setting<SecureString> KEY_SETTING = new Setting<>("cloud.aws.ec2.access_key", AwsEc2Service.KEY_SETTING,
+            SecureString::new, Property.NodeScope, Property.Filtered, Property.Deprecated);
+
+        /**
+         * cloud.aws.ec2.secret_key: AWS Secret key specific for EC2 API calls. Defaults to cloud.aws.secret_key.
+         * @see AwsEc2Service#SECRET_SETTING
+         */
+        Setting<SecureString> SECRET_SETTING = new Setting<>("cloud.aws.ec2.secret_key", AwsEc2Service.SECRET_SETTING,
+            SecureString::new, Property.NodeScope, Property.Filtered, Property.Deprecated);
+        /**
+         * cloud.aws.ec2.protocol: Protocol for AWS API specific for EC2 API calls: http or https.  Defaults to cloud.aws.protocol.
+         * @see AwsEc2Service#PROTOCOL_SETTING
+         */
+        Setting<Protocol> PROTOCOL_SETTING = new Setting<>("cloud.aws.ec2.protocol", AwsEc2Service.PROTOCOL_SETTING,
+            s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)), Property.NodeScope, Property.Deprecated);
+        /**
+         * cloud.aws.ec2.proxy.host: In case of proxy, define its hostname/IP specific for EC2 API calls. Defaults to cloud.aws.proxy.host.
+         * @see AwsEc2Service#PROXY_HOST_SETTING
+         */
+        Setting<String> PROXY_HOST_SETTING = new Setting<>("cloud.aws.ec2.proxy.host", AwsEc2Service.PROXY_HOST_SETTING,
+            Function.identity(), Property.NodeScope, Property.Deprecated);
+        /**
+         * cloud.aws.ec2.proxy.port: In case of proxy, define its port specific for EC2 API calls.  Defaults to cloud.aws.proxy.port.
+         * @see AwsEc2Service#PROXY_PORT_SETTING
+         */
+        Setting<Integer> PROXY_PORT_SETTING = new Setting<>("cloud.aws.ec2.proxy.port", AwsEc2Service.PROXY_PORT_SETTING,
+            s -> Setting.parseInt(s, 0, 1<<16, "cloud.aws.ec2.proxy.port"), Property.NodeScope, Property.Deprecated);
+        /**
+         * cloud.aws.ec2.proxy.username: In case of proxy with auth, define the username specific for EC2 API calls.
+         * Defaults to cloud.aws.proxy.username.
+         * @see AwsEc2Service#PROXY_USERNAME_SETTING
+         */
+        Setting<SecureString> PROXY_USERNAME_SETTING = new Setting<>("cloud.aws.ec2.proxy.username", AwsEc2Service.PROXY_USERNAME_SETTING,
+            SecureString::new, Property.NodeScope, Property.Filtered, Property.Deprecated);
+        /**
+         * cloud.aws.ec2.proxy.password: In case of proxy with auth, define the password specific for EC2 API calls.
+         * Defaults to cloud.aws.proxy.password.
+         * @see AwsEc2Service#PROXY_PASSWORD_SETTING
+         */
+        Setting<SecureString> PROXY_PASSWORD_SETTING = new Setting<>("cloud.aws.ec2.proxy.password", AwsEc2Service.PROXY_PASSWORD_SETTING,
+            SecureString::new, Property.NodeScope, Property.Filtered, Property.Deprecated);
+        /**
+         * cloud.aws.ec2.signer: If you are using an old AWS API version, you can define a Signer. Specific for EC2 API calls.
+         * Defaults to cloud.aws.signer.
+         * @see AwsEc2Service#SIGNER_SETTING
+         */
+        Setting<String> SIGNER_SETTING = new Setting<>("cloud.aws.ec2.signer", AwsEc2Service.SIGNER_SETTING, Function.identity(),
+            Property.NodeScope, Property.Deprecated);
+        /**
+         * cloud.aws.ec2.region: Region specific for EC2 API calls. Defaults to cloud.aws.region.
+         * @see AwsEc2Service#REGION_SETTING
+         */
+        Setting<String> REGION_SETTING = new Setting<>("cloud.aws.ec2.region", AwsEc2Service.REGION_SETTING,
+            s -> s.toLowerCase(Locale.ROOT), Property.NodeScope, Property.Deprecated);
+        /**
+         * cloud.aws.ec2.endpoint: Endpoint. If not set, endpoint will be guessed based on region setting.
+         */
+        Setting<String> ENDPOINT_SETTING = Setting.simpleString("cloud.aws.ec2.endpoint", Property.NodeScope, Property.Deprecated);
+        /**
+         * cloud.aws.ec2.read_timeout: Socket read timeout. Defaults to cloud.aws.read_timeout
+         * @see AwsEc2Service#READ_TIMEOUT
+         */
+        Setting<TimeValue> READ_TIMEOUT =
+            Setting.timeSetting("cloud.aws.ec2.read_timeout", AwsEc2Service.READ_TIMEOUT, Property.NodeScope, Property.Deprecated);
+    }
+
+    /**
+     * Defines discovery settings for ec2. Starting with discovery.ec2.
+     */
+    interface DISCOVERY_EC2 {
+        class HostType {
+            public static final String PRIVATE_IP = "private_ip";
+            public static final String PUBLIC_IP = "public_ip";
+            public static final String PRIVATE_DNS = "private_dns";
+            public static final String PUBLIC_DNS = "public_dns";
+            public static final String TAG_PREFIX = "tag:";
+        }
+
+        /** The access key (ie login id) for connecting to ec2. */
+        Setting<SecureString> ACCESS_KEY_SETTING = SecureSetting.secureString("discovery.ec2.access_key", CLOUD_EC2.KEY_SETTING);
+
+        /** The secret key (ie password) for connecting to ec2. */
+        Setting<SecureString> SECRET_KEY_SETTING = SecureSetting.secureString("discovery.ec2.secret_key", CLOUD_EC2.SECRET_SETTING);
+
+        /** An override for the ec2 endpoint to connect to. */
+        Setting<String> ENDPOINT_SETTING = new Setting<>("discovery.ec2.endpoint", CLOUD_EC2.ENDPOINT_SETTING,
+            s -> s.toLowerCase(Locale.ROOT), Setting.Property.NodeScope);
+
+        /** The protocol to use to connect to to ec2. */
+        Setting<Protocol> PROTOCOL_SETTING = new Setting<>("discovery.ec2.protocol", CLOUD_EC2.PROTOCOL_SETTING,
+            s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)), Setting.Property.NodeScope);
+
+        /** The host name of a proxy to connect to ec2 through. */
+        Setting<String> PROXY_HOST_SETTING = new Setting<>("discovery.ec2.proxy.host", CLOUD_EC2.PROXY_HOST_SETTING,
+            Function.identity(), Setting.Property.NodeScope);
+
+        /** The port of a proxy to connect to ec2 through. */
+        Setting<Integer> PROXY_PORT_SETTING = Setting.intSetting("discovery.ec2.proxy.port", CLOUD_EC2.PROXY_PORT_SETTING,
+            0, Setting.Property.NodeScope);
+
+        /** The username of a proxy to connect to s3 through. */
+        Setting<SecureString> PROXY_USERNAME_SETTING = SecureSetting.secureString("discovery.ec2.proxy.username",
+            CLOUD_EC2.PROXY_USERNAME_SETTING);
+
+        /** The password of a proxy to connect to s3 through. */
+        Setting<SecureString> PROXY_PASSWORD_SETTING =  SecureSetting.secureString("discovery.ec2.proxy.password",
+            CLOUD_EC2.PROXY_PASSWORD_SETTING);
+
+        /** The socket timeout for connecting to s3. */
+        Setting<TimeValue> READ_TIMEOUT_SETTING = Setting.timeSetting("discovery.ec2.read_timeout",
+            CLOUD_EC2.READ_TIMEOUT, Setting.Property.NodeScope);
+
+        /**
+         * discovery.ec2.host_type: The type of host type to use to communicate with other instances.
+         * Can be one of private_ip, public_ip, private_dns, public_dns or tag:XXXX where
+         * XXXX refers to a name of a tag configured for all EC2 instances. Instances which don't
+         * have this tag set will be ignored by the discovery process. Defaults to private_ip.
+         */
+        Setting<String> HOST_TYPE_SETTING =
+            new Setting<>("discovery.ec2.host_type", HostType.PRIVATE_IP, Function.identity(), Property.NodeScope);
+        /**
+         * discovery.ec2.any_group: If set to false, will require all security groups to be present for the instance to be used for the
+         * discovery. Defaults to true.
+         */
+        Setting<Boolean> ANY_GROUP_SETTING =
+            Setting.boolSetting("discovery.ec2.any_group", true, Property.NodeScope);
+        /**
+         * discovery.ec2.groups: Either a comma separated list or array based list of (security) groups. Only instances with the provided
+         * security groups will be used in the cluster discovery. (NOTE: You could provide either group NAME or group ID.)
+         */
+        Setting<List<String>> GROUPS_SETTING =
+            Setting.listSetting("discovery.ec2.groups", new ArrayList<>(), s -> s.toString(), Property.NodeScope);
+        /**
+         * discovery.ec2.availability_zones: Either a comma separated list or array based list of availability zones. Only instances within
+         * the provided availability zones will be used in the cluster discovery.
+         */
+        Setting<List<String>> AVAILABILITY_ZONES_SETTING =
+            Setting.listSetting("discovery.ec2.availability_zones", Collections.emptyList(), s -> s.toString(),
+                Property.NodeScope);
+        /**
+         * discovery.ec2.node_cache_time: How long the list of hosts is cached to prevent further requests to the AWS API. Defaults to 10s.
+         */
+        Setting<TimeValue> NODE_CACHE_TIME_SETTING =
+            Setting.timeSetting("discovery.ec2.node_cache_time", TimeValue.timeValueSeconds(10), Property.NodeScope);
+
+        /**
+         * discovery.ec2.tag.*: The ec2 discovery can filter machines to include in the cluster based on tags (and not just groups).
+         * The settings to use include the discovery.ec2.tag. prefix. For example, setting discovery.ec2.tag.stage to dev will only filter
+         * instances with a tag key set to stage, and a value of dev. Several tags set will require all of those tags to be set for the
+         * instance to be included.
+         */
+        Setting<Settings> TAG_SETTING = Setting.groupSetting("discovery.ec2.tag.", Property.NodeScope);
+    }
+
+    AmazonEC2 client();
+}
diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImpl.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImpl.java
new file mode 100644
index 0000000000000..8131b5be9badd
--- /dev/null
+++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImpl.java
@@ -0,0 +1,220 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.discovery.ec2;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Random;
+
+import com.amazonaws.AmazonClientException;
+import com.amazonaws.AmazonWebServiceRequest;
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
+import com.amazonaws.http.IdleConnectionReaper;
+import com.amazonaws.internal.StaticCredentialsProvider;
+import com.amazonaws.retry.RetryPolicy;
+import com.amazonaws.services.ec2.AmazonEC2;
+import com.amazonaws.services.ec2.AmazonEC2Client;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.Randomness;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.component.AbstractComponent;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+
+class AwsEc2ServiceImpl extends AbstractComponent implements AwsEc2Service, Closeable {
+
+    public static final String EC2_METADATA_URL = "http://169.254.169.254/latest/meta-data/";
+
+    private AmazonEC2Client client;
+
+    AwsEc2ServiceImpl(Settings settings) {
+        super(settings);
+    }
+
+    @Override
+    public synchronized AmazonEC2 client() {
+        if (client != null) {
+            return client;
+        }
+
+        this.client = new AmazonEC2Client(buildCredentials(logger, settings), buildConfiguration(logger, settings));
+        String endpoint = findEndpoint(logger, settings);
+        if (endpoint != null) {
+            client.setEndpoint(endpoint);
+        }
+
+        return this.client;
+    }
+
+    protected static AWSCredentialsProvider buildCredentials(Logger logger, Settings settings) {
+        AWSCredentialsProvider credentials;
+
+        try (SecureString key = DISCOVERY_EC2.ACCESS_KEY_SETTING.get(settings);
+             SecureString secret = DISCOVERY_EC2.SECRET_KEY_SETTING.get(settings)) {
+            if (key.length() == 0 && secret.length() == 0) {
+                logger.debug("Using either environment variables, system properties or instance profile credentials");
+                credentials = new DefaultAWSCredentialsProviderChain();
+            } else {
+                logger.debug("Using basic key/secret credentials");
+                credentials = new StaticCredentialsProvider(new BasicAWSCredentials(key.toString(), secret.toString()));
+            }
+        }
+
+        return credentials;
+    }
+
+    protected static ClientConfiguration buildConfiguration(Logger logger, Settings settings) {
+        ClientConfiguration clientConfiguration = new ClientConfiguration();
+        // the response metadata cache is only there for diagnostics purposes,
+        // but can force objects from every response to the old generation.
+        clientConfiguration.setResponseMetadataCacheSize(0);
+        clientConfiguration.setProtocol(DISCOVERY_EC2.PROTOCOL_SETTING.get(settings));
+
+        if (PROXY_HOST_SETTING.exists(settings) || DISCOVERY_EC2.PROXY_HOST_SETTING.exists(settings)) {
+            String proxyHost = DISCOVERY_EC2.PROXY_HOST_SETTING.get(settings);
+            Integer proxyPort = DISCOVERY_EC2.PROXY_PORT_SETTING.get(settings);
+            try (SecureString proxyUsername = DISCOVERY_EC2.PROXY_USERNAME_SETTING.get(settings);
+                 SecureString proxyPassword = DISCOVERY_EC2.PROXY_PASSWORD_SETTING.get(settings)) {
+
+                clientConfiguration
+                    .withProxyHost(proxyHost)
+                    .withProxyPort(proxyPort)
+                    .withProxyUsername(proxyUsername.toString())
+                    .withProxyPassword(proxyPassword.toString());
+            }
+        }
+
+        // #155: we might have 3rd party users using older EC2 API version
+        String awsSigner = CLOUD_EC2.SIGNER_SETTING.get(settings);
+        if (Strings.hasText(awsSigner)) {
+            logger.debug("using AWS API signer [{}]", awsSigner);
+            AwsSigner.configureSigner(awsSigner, clientConfiguration);
+        }
+
+        // Increase the number of retries in case of 5xx API responses
+        final Random rand = Randomness.get();
+        RetryPolicy retryPolicy = new RetryPolicy(
+            RetryPolicy.RetryCondition.NO_RETRY_CONDITION,
+            new RetryPolicy.BackoffStrategy() {
+                @Override
+                public long delayBeforeNextRetry(AmazonWebServiceRequest originalRequest,
+                                                 AmazonClientException exception,
+                                                 int retriesAttempted) {
+                    // with 10 retries the max delay time is 320s/320000ms (10 * 2^5 * 1 * 1000)
+                    logger.warn("EC2 API request failed, retry again. Reason was:", exception);
+                    return 1000L * (long) (10d * Math.pow(2, retriesAttempted / 2.0d) * (1.0d + rand.nextDouble()));
+                }
+            },
+            10,
+            false);
+        clientConfiguration.setRetryPolicy(retryPolicy);
+        clientConfiguration.setSocketTimeout((int) DISCOVERY_EC2.READ_TIMEOUT_SETTING.get(settings).millis());
+
+        return clientConfiguration;
+    }
+
+    protected static String findEndpoint(Logger logger, Settings settings) {
+        String endpoint = null;
+        if (DISCOVERY_EC2.ENDPOINT_SETTING.exists(settings) || CLOUD_EC2.ENDPOINT_SETTING.exists(settings)) {
+            endpoint = DISCOVERY_EC2.ENDPOINT_SETTING.get(settings);
+            logger.debug("using explicit ec2 endpoint [{}]", endpoint);
+        } else if (REGION_SETTING.exists(settings) || CLOUD_EC2.REGION_SETTING.exists(settings)) {
+            final String region = CLOUD_EC2.REGION_SETTING.get(settings);
+            switch (region) {
+                case "us-east-1":
+                case "us-east":
+                    endpoint = "ec2.us-east-1.amazonaws.com";
+                    break;
+                case "us-east-2":
+                    endpoint = "ec2.us-east-2.amazonaws.com";
+                    break;
+                case "us-west":
+                case "us-west-1":
+                    endpoint = "ec2.us-west-1.amazonaws.com";
+                    break;
+                case "us-west-2":
+                    endpoint = "ec2.us-west-2.amazonaws.com";
+                    break;
+                case "ap-southeast":
+                case "ap-southeast-1":
+                    endpoint = "ec2.ap-southeast-1.amazonaws.com";
+                    break;
+                case "ap-south":
+                case "ap-south-1":
+                    endpoint = "ec2.ap-south-1.amazonaws.com";
+                    break;
+                case "us-gov-west":
+                case "us-gov-west-1":
+                    endpoint = "ec2.us-gov-west-1.amazonaws.com";
+                    break;
+                case "ap-southeast-2":
+                    endpoint = "ec2.ap-southeast-2.amazonaws.com";
+                    break;
+                case "ap-northeast":
+                case "ap-northeast-1":
+                    endpoint = "ec2.ap-northeast-1.amazonaws.com";
+                    break;
+                case "ap-northeast-2":
+                    endpoint = "ec2.ap-northeast-2.amazonaws.com";
+                    break;
+                case "eu-west":
+                case "eu-west-1":
+                    endpoint = "ec2.eu-west-1.amazonaws.com";
+                    break;
+                case "eu-west-2":
+                    endpoint = "ec2.eu-west-2.amazonaws.com";
+                    break;
+                case "eu-central":
+                case "eu-central-1":
+                    endpoint = "ec2.eu-central-1.amazonaws.com";
+                    break;
+                case "sa-east":
+                case "sa-east-1":
+                    endpoint = "ec2.sa-east-1.amazonaws.com";
+                    break;
+                case "cn-north":
+                case "cn-north-1":
+                    endpoint = "ec2.cn-north-1.amazonaws.com.cn";
+                    break;
+                case "ca-central":
+                case "ca-central-1":
+                    endpoint = "ec2.ca-central-1.amazonaws.com";
+                    break;
+                default:
+                    throw new IllegalArgumentException("No automatic endpoint could be derived from region [" + region + "]");
+            }
+            logger.debug("using ec2 region [{}], with endpoint [{}]", region, endpoint);
+        }
+        return endpoint;
+    }
+
+    @Override
+    public void close() throws IOException {
+        if (client != null) {
+            client.shutdown();
+        }
+
+        // Ensure that IdleConnectionReaper is shutdown
+        IdleConnectionReaper.shutdown();
+    }
+}
diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2UnicastHostsProvider.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2UnicastHostsProvider.java
index 6d4fcdc4c8da6..bb621f2ccf776 100644
--- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2UnicastHostsProvider.java
+++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2UnicastHostsProvider.java
@@ -19,6 +19,12 @@
 
 package org.elasticsearch.discovery.ec2;
 
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
 import com.amazonaws.AmazonClientException;
 import com.amazonaws.services.ec2.AmazonEC2;
 import com.amazonaws.services.ec2.model.DescribeInstancesRequest;
@@ -27,35 +33,30 @@
 import com.amazonaws.services.ec2.model.GroupIdentifier;
 import com.amazonaws.services.ec2.model.Instance;
 import com.amazonaws.services.ec2.model.Reservation;
+import com.amazonaws.services.ec2.model.Tag;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
 import org.elasticsearch.Version;
-import org.elasticsearch.cloud.aws.AwsEc2Service;
-import org.elasticsearch.cloud.aws.AwsEc2Service.DISCOVERY_EC2;
+import org.elasticsearch.discovery.ec2.AwsEc2Service.DISCOVERY_EC2;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.SingleObjectCache;
-import org.elasticsearch.discovery.zen.ping.unicast.UnicastHostsProvider;
+import org.elasticsearch.discovery.zen.UnicastHostsProvider;
 import org.elasticsearch.transport.TransportService;
 
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
 import static java.util.Collections.disjoint;
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
+import static org.elasticsearch.discovery.ec2.AwsEc2Service.DISCOVERY_EC2.HostType.TAG_PREFIX;
+import static org.elasticsearch.discovery.ec2.AwsEc2Service.DISCOVERY_EC2.HostType.PRIVATE_DNS;
+import static org.elasticsearch.discovery.ec2.AwsEc2Service.DISCOVERY_EC2.HostType.PRIVATE_IP;
+import static org.elasticsearch.discovery.ec2.AwsEc2Service.DISCOVERY_EC2.HostType.PUBLIC_DNS;
+import static org.elasticsearch.discovery.ec2.AwsEc2Service.DISCOVERY_EC2.HostType.PUBLIC_IP;
 
-/**
- *
- */
-public class AwsEc2UnicastHostsProvider extends AbstractComponent implements UnicastHostsProvider {
+class AwsEc2UnicastHostsProvider extends AbstractComponent implements UnicastHostsProvider {
 
     private final TransportService transportService;
 
@@ -69,12 +70,11 @@ public class AwsEc2UnicastHostsProvider extends AbstractComponent implements Uni
 
     private final Set<String> availabilityZones;
 
-    private final DISCOVERY_EC2.HostType hostType;
+    private final String hostType;
 
     private final DiscoNodesCache discoNodes;
 
-    @Inject
-    public AwsEc2UnicastHostsProvider(Settings settings, TransportService transportService, AwsEc2Service awsEc2Service) {
+    AwsEc2UnicastHostsProvider(Settings settings, TransportService transportService, AwsEc2Service awsEc2Service) {
         super(settings);
         this.transportService = transportService;
         this.client = awsEc2Service.client();
@@ -126,8 +126,8 @@ protected List<DiscoveryNode> fetchDynamicNodes() {
                 // lets see if we can filter based on groups
                 if (!groups.isEmpty()) {
                     List<GroupIdentifier> instanceSecurityGroups = instance.getSecurityGroups();
-                    ArrayList<String> securityGroupNames = new ArrayList<String>();
-                    ArrayList<String> securityGroupIds = new ArrayList<String>();
+                    List<String> securityGroupNames = new ArrayList<>(instanceSecurityGroups.size());
+                    List<String> securityGroupIds = new ArrayList<>(instanceSecurityGroups.size());
                     for (GroupIdentifier sg : instanceSecurityGroups) {
                         securityGroupNames.add(sg.getGroupName());
                         securityGroupIds.add(sg.getGroupId());
@@ -153,19 +153,27 @@ && disjoint(securityGroupIds, groups)) {
                 }
 
                 String address = null;
-                switch (hostType) {
-                    case PRIVATE_DNS:
-                        address = instance.getPrivateDnsName();
-                        break;
-                    case PRIVATE_IP:
-                        address = instance.getPrivateIpAddress();
-                        break;
-                    case PUBLIC_DNS:
-                        address = instance.getPublicDnsName();
-                        break;
-                    case PUBLIC_IP:
-                        address = instance.getPublicIpAddress();
-                        break;
+                if (hostType.equals(PRIVATE_DNS)) {
+                    address = instance.getPrivateDnsName();
+                } else if (hostType.equals(PRIVATE_IP)) {
+                    address = instance.getPrivateIpAddress();
+                } else if (hostType.equals(PUBLIC_DNS)) {
+                    address = instance.getPublicDnsName();
+                } else if (hostType.equals(PUBLIC_IP)) {
+                    address = instance.getPublicIpAddress();
+                } else if (hostType.startsWith(TAG_PREFIX)) {
+                    // Reading the node host from its metadata
+                    String tagName = hostType.substring(TAG_PREFIX.length());
+                    logger.debug("reading hostname from [{}] instance tag", tagName);
+                    List<Tag> tags = instance.getTags();
+                    for (Tag tag : tags) {
+                        if (tag.getKey().equals(tagName)) {
+                            address = tag.getValue();
+                            logger.debug("using [{}] as the instance address", address);
+                        }
+                    }
+                } else {
+                    throw new IllegalArgumentException(hostType + " is unknown for discovery.ec2.host_type");
                 }
                 if (address != null) {
                     try {
@@ -173,8 +181,8 @@ && disjoint(securityGroupIds, groups)) {
                         TransportAddress[] addresses = transportService.addressesFromString(address, 1);
                         for (int i = 0; i < addresses.length; i++) {
                             logger.trace("adding {}, address {}, transport_address {}", instance.getInstanceId(), address, addresses[i]);
-                            discoNodes.add(new DiscoveryNode("#cloud-" + instance.getInstanceId() + "-" + i, addresses[i],
-                                    emptyMap(), emptySet(), Version.CURRENT.minimumCompatibilityVersion()));
+                            discoNodes.add(new DiscoveryNode(instance.getInstanceId(), "#cloud-" + instance.getInstanceId() + "-" + i,
+                                addresses[i], emptyMap(), emptySet(), Version.CURRENT.minimumCompatibilityVersion()));
                         }
                     } catch (Exception e) {
                         final String finalAddress = address;
diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsSigner.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsSigner.java
new file mode 100644
index 0000000000000..394559f12ef60
--- /dev/null
+++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsSigner.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.discovery.ec2;
+
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.auth.SignerFactory;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.logging.Loggers;
+
+class AwsSigner {
+
+    private static final Logger logger = Loggers.getLogger(AwsSigner.class);
+
+    private AwsSigner() {
+
+    }
+
+    protected static void validateSignerType(String signer) throws IllegalArgumentException {
+        if (signer == null) {
+            throw new IllegalArgumentException("[null] signer set");
+        }
+
+        try {
+            // We check this signer actually exists in AWS SDK
+            // It throws a IllegalArgumentException if not found
+            SignerFactory.getSignerByTypeAndService(signer, null);
+        } catch (IllegalArgumentException e) {
+            throw new IllegalArgumentException("wrong signer set [" + signer + "]");
+        }
+    }
+
+    /**
+     * Add a AWS API Signer.
+     * @param signer Signer to use
+     * @param configuration AWS Client configuration
+     */
+    public static void configureSigner(String signer, ClientConfiguration configuration) {
+        try {
+            validateSignerType(signer);
+        } catch (IllegalArgumentException e) {
+            logger.warn("{}", e.getMessage());
+        }
+
+        configuration.setSignerOverride(signer);
+    }
+}
diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPlugin.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPlugin.java
new file mode 100644
index 0000000000000..18718483e7d38
--- /dev/null
+++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPlugin.java
@@ -0,0 +1,230 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.discovery.ec2;
+
+import com.amazonaws.util.json.Jackson;
+import java.io.BufferedReader;
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.UncheckedIOException;
+import java.net.URL;
+import java.net.URLConnection;
+import java.nio.charset.StandardCharsets;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Supplier;
+
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.SpecialPermission;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.network.NetworkService;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.discovery.Discovery;
+import org.elasticsearch.discovery.DiscoveryModule;
+import org.elasticsearch.discovery.zen.UnicastHostsProvider;
+import org.elasticsearch.discovery.zen.ZenDiscovery;
+import org.elasticsearch.discovery.zen.ZenPing;
+import org.elasticsearch.node.Node;
+import org.elasticsearch.plugins.DiscoveryPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+
+public class Ec2DiscoveryPlugin extends Plugin implements DiscoveryPlugin, Closeable {
+
+    private static Logger logger = Loggers.getLogger(Ec2DiscoveryPlugin.class);
+    private static final DeprecationLogger deprecationLogger = new DeprecationLogger(logger);
+
+    public static final String EC2 = "ec2";
+
+    static {
+        SecurityManager sm = System.getSecurityManager();
+        if (sm != null) {
+            sm.checkPermission(new SpecialPermission());
+        }
+        AccessController.doPrivileged(new PrivilegedAction<Void>() {
+            @Override
+            public Void run() {
+                try {
+                    // kick jackson to do some static caching of declared members info
+                    Jackson.jsonNodeOf("{}");
+                    // ClientConfiguration clinit has some classloader problems
+                    // TODO: fix that
+                    Class.forName("com.amazonaws.ClientConfiguration");
+                } catch (ClassNotFoundException e) {
+                    throw new RuntimeException(e);
+                }
+                return null;
+            }
+        });
+    }
+
+    private Settings settings;
+    // stashed when created in order to properly close
+    private final SetOnce<AwsEc2ServiceImpl> ec2Service = new SetOnce<>();
+
+    public Ec2DiscoveryPlugin(Settings settings) {
+        this.settings = settings;
+    }
+
+    @Override
+    public Map<String, Supplier<Discovery>> getDiscoveryTypes(ThreadPool threadPool, TransportService transportService,
+                                                              NamedWriteableRegistry namedWriteableRegistry,
+                                                              ClusterService clusterService, UnicastHostsProvider hostsProvider) {
+        // this is for backcompat with pre 5.1, where users would set discovery.type to use ec2 hosts provider
+        return Collections.singletonMap(EC2, () ->
+            new ZenDiscovery(settings, threadPool, transportService, namedWriteableRegistry, clusterService, hostsProvider));
+    }
+
+    @Override
+    public NetworkService.CustomNameResolver getCustomNameResolver(Settings settings) {
+        logger.debug("Register _ec2_, _ec2:xxx_ network names");
+        return new Ec2NameResolver(settings);
+    }
+
+    @Override
+    public Map<String, Supplier<UnicastHostsProvider>> getZenHostsProviders(TransportService transportService,
+                                                                            NetworkService networkService) {
+        return Collections.singletonMap(EC2, () -> {
+            ec2Service.set(new AwsEc2ServiceImpl(settings));
+            return new AwsEc2UnicastHostsProvider(settings, transportService, ec2Service.get());
+        });
+    }
+
+    @Override
+    public List<Setting<?>> getSettings() {
+        return Arrays.asList(
+        // Register global cloud aws settings: cloud.aws (might have been registered in ec2 plugin)
+        AwsEc2Service.KEY_SETTING,
+        AwsEc2Service.SECRET_SETTING,
+        AwsEc2Service.PROTOCOL_SETTING,
+        AwsEc2Service.PROXY_HOST_SETTING,
+        AwsEc2Service.PROXY_PORT_SETTING,
+        AwsEc2Service.PROXY_USERNAME_SETTING,
+        AwsEc2Service.PROXY_PASSWORD_SETTING,
+        AwsEc2Service.SIGNER_SETTING,
+        AwsEc2Service.REGION_SETTING,
+        AwsEc2Service.READ_TIMEOUT,
+        // Register EC2 specific settings: cloud.aws.ec2
+        AwsEc2Service.CLOUD_EC2.KEY_SETTING,
+        AwsEc2Service.CLOUD_EC2.SECRET_SETTING,
+        AwsEc2Service.CLOUD_EC2.PROTOCOL_SETTING,
+        AwsEc2Service.CLOUD_EC2.PROXY_HOST_SETTING,
+        AwsEc2Service.CLOUD_EC2.PROXY_PORT_SETTING,
+        AwsEc2Service.CLOUD_EC2.PROXY_USERNAME_SETTING,
+        AwsEc2Service.CLOUD_EC2.PROXY_PASSWORD_SETTING,
+        AwsEc2Service.CLOUD_EC2.SIGNER_SETTING,
+        AwsEc2Service.CLOUD_EC2.REGION_SETTING,
+        AwsEc2Service.CLOUD_EC2.ENDPOINT_SETTING,
+        AwsEc2Service.CLOUD_EC2.READ_TIMEOUT,
+        // Register EC2 discovery settings: discovery.ec2
+        AwsEc2Service.DISCOVERY_EC2.ACCESS_KEY_SETTING,
+        AwsEc2Service.DISCOVERY_EC2.SECRET_KEY_SETTING,
+        AwsEc2Service.DISCOVERY_EC2.ENDPOINT_SETTING,
+        AwsEc2Service.DISCOVERY_EC2.PROTOCOL_SETTING,
+        AwsEc2Service.DISCOVERY_EC2.PROXY_HOST_SETTING,
+        AwsEc2Service.DISCOVERY_EC2.PROXY_PORT_SETTING,
+        AwsEc2Service.DISCOVERY_EC2.PROXY_USERNAME_SETTING,
+        AwsEc2Service.DISCOVERY_EC2.PROXY_PASSWORD_SETTING,
+        AwsEc2Service.DISCOVERY_EC2.READ_TIMEOUT_SETTING,
+        AwsEc2Service.DISCOVERY_EC2.HOST_TYPE_SETTING,
+        AwsEc2Service.DISCOVERY_EC2.ANY_GROUP_SETTING,
+        AwsEc2Service.DISCOVERY_EC2.GROUPS_SETTING,
+        AwsEc2Service.DISCOVERY_EC2.AVAILABILITY_ZONES_SETTING,
+        AwsEc2Service.DISCOVERY_EC2.NODE_CACHE_TIME_SETTING,
+        AwsEc2Service.DISCOVERY_EC2.TAG_SETTING,
+        // Register cloud node settings: cloud.node
+        AwsEc2Service.AUTO_ATTRIBUTE_SETTING);
+    }
+
+    @Override
+    public Settings additionalSettings() {
+        Settings.Builder builder = Settings.builder();
+        // For 5.0, discovery.type was used prior to the new discovery.zen.hosts_provider
+        // setting existed. This check looks for the legacy setting, and sets hosts provider if set
+        String discoveryType = DiscoveryModule.DISCOVERY_TYPE_SETTING.get(settings);
+        if (discoveryType.equals(EC2)) {
+            deprecationLogger.deprecated("using [" + DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey() +
+                "] setting to set hosts provider is deprecated; " +
+                "set [" + DiscoveryModule.DISCOVERY_HOSTS_PROVIDER_SETTING.getKey() + ": " + EC2 + "] instead");
+            if (DiscoveryModule.DISCOVERY_HOSTS_PROVIDER_SETTING.exists(settings) == false) {
+                builder.put(DiscoveryModule.DISCOVERY_HOSTS_PROVIDER_SETTING.getKey(), EC2).build();
+            }
+        }
+
+        // Adds a node attribute for the ec2 availability zone
+        String azMetadataUrl = AwsEc2ServiceImpl.EC2_METADATA_URL + "placement/availability-zone";
+        builder.put(getAvailabilityZoneNodeAttributes(settings, azMetadataUrl));
+        return builder.build();
+    }
+
+    // pkg private for testing
+    static Settings getAvailabilityZoneNodeAttributes(Settings settings, String azMetadataUrl) {
+        if (AwsEc2Service.AUTO_ATTRIBUTE_SETTING.get(settings) == false) {
+            return Settings.EMPTY;
+        }
+        Settings.Builder attrs = Settings.builder();
+
+        final URL url;
+        final URLConnection urlConnection;
+        try {
+            url = new URL(azMetadataUrl);
+            logger.debug("obtaining ec2 [placement/availability-zone] from ec2 meta-data url {}", url);
+            urlConnection = url.openConnection();
+            urlConnection.setConnectTimeout(2000);
+        } catch (IOException e) {
+            // should not happen, we know the url is not malformed, and openConnection does not actually hit network
+            throw new UncheckedIOException(e);
+        }
+
+        try (InputStream in = urlConnection.getInputStream();
+             BufferedReader urlReader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8))) {
+
+            String metadataResult = urlReader.readLine();
+            if (metadataResult == null || metadataResult.length() == 0) {
+                throw new IllegalStateException("no ec2 metadata returned from " + url);
+            } else {
+                attrs.put(Node.NODE_ATTRIBUTES.getKey() + "aws_availability_zone", metadataResult);
+            }
+        } catch (IOException e) {
+            // this is lenient so the plugin does not fail when installed outside of ec2
+            logger.error("failed to get metadata for [placement/availability-zone]", e);
+        }
+
+        return attrs.build();
+    }
+
+    @Override
+    public void close() throws IOException {
+        IOUtils.close(ec2Service.get());
+    }
+}
diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/network/Ec2NameResolver.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2NameResolver.java
similarity index 93%
rename from plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/network/Ec2NameResolver.java
rename to plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2NameResolver.java
index 3788f8237dcdf..748e0a12f7ba0 100755
--- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/network/Ec2NameResolver.java
+++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2NameResolver.java
@@ -17,10 +17,9 @@
  * under the License.
  */
 
-package org.elasticsearch.cloud.aws.network;
+package org.elasticsearch.discovery.ec2;
 
 import org.apache.lucene.util.IOUtils;
-import org.elasticsearch.cloud.aws.AwsEc2ServiceImpl;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.network.NetworkService.CustomNameResolver;
 import org.elasticsearch.common.settings.Settings;
@@ -51,14 +50,14 @@
  *
  * @author Paul_Loy (keteracel)
  */
-public class Ec2NameResolver extends AbstractComponent implements CustomNameResolver {
+class Ec2NameResolver extends AbstractComponent implements CustomNameResolver {
 
     /**
      * enum that can be added to over time with more meta-data types (such as ipv6 when this is available)
      *
      * @author Paul_Loy
      */
-    private static enum Ec2HostnameType {
+    private enum Ec2HostnameType {
 
         PRIVATE_IPv4("ec2:privateIpv4", "local-ipv4"),
         PRIVATE_DNS("ec2:privateDns", "local-hostname"),
@@ -73,7 +72,7 @@ private static enum Ec2HostnameType {
         final String configName;
         final String ec2Name;
 
-        private Ec2HostnameType(String configName, String ec2Name) {
+        Ec2HostnameType(String configName, String ec2Name) {
             this.configName = configName;
             this.ec2Name = ec2Name;
         }
@@ -82,7 +81,7 @@ private Ec2HostnameType(String configName, String ec2Name) {
     /**
      * Construct a {@link CustomNameResolver}.
      */
-    public Ec2NameResolver(Settings settings) {
+    Ec2NameResolver(Settings settings) {
         super(settings);
     }
 
diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java
deleted file mode 100644
index 7f8e983e52b12..0000000000000
--- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java
+++ /dev/null
@@ -1,193 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.plugin.discovery.ec2;
-
-import org.apache.logging.log4j.Logger;
-import org.elasticsearch.SpecialPermission;
-import org.elasticsearch.cloud.aws.AwsEc2Service;
-import org.elasticsearch.cloud.aws.AwsEc2ServiceImpl;
-import org.elasticsearch.cloud.aws.Ec2Module;
-import org.elasticsearch.cloud.aws.network.Ec2NameResolver;
-import org.elasticsearch.common.component.LifecycleComponent;
-import org.elasticsearch.common.inject.Module;
-import org.elasticsearch.common.logging.Loggers;
-import org.elasticsearch.common.network.NetworkService;
-import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.discovery.DiscoveryModule;
-import org.elasticsearch.discovery.ec2.AwsEc2UnicastHostsProvider;
-import org.elasticsearch.discovery.zen.ZenDiscovery;
-import org.elasticsearch.node.Node;
-import org.elasticsearch.plugins.DiscoveryPlugin;
-import org.elasticsearch.plugins.Plugin;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.UncheckedIOException;
-import java.net.URL;
-import java.net.URLConnection;
-import java.nio.charset.StandardCharsets;
-import java.security.AccessController;
-import java.security.PrivilegedAction;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-
-/**
- *
- */
-public class Ec2DiscoveryPlugin extends Plugin implements DiscoveryPlugin {
-
-    private static Logger logger = Loggers.getLogger(Ec2DiscoveryPlugin.class);
-
-    public static final String EC2 = "ec2";
-
-    // ClientConfiguration clinit has some classloader problems
-    // TODO: fix that
-    static {
-        SecurityManager sm = System.getSecurityManager();
-        if (sm != null) {
-            sm.checkPermission(new SpecialPermission());
-        }
-        AccessController.doPrivileged(new PrivilegedAction<Void>() {
-            @Override
-            public Void run() {
-                try {
-                    Class.forName("com.amazonaws.ClientConfiguration");
-                } catch (ClassNotFoundException e) {
-                    throw new RuntimeException(e);
-                }
-                return null;
-            }
-        });
-    }
-
-    private Settings settings;
-
-    public Ec2DiscoveryPlugin(Settings settings) {
-        this.settings = settings;
-    }
-
-    @Override
-    public Collection<Module> createGuiceModules() {
-        Collection<Module> modules = new ArrayList<>();
-        modules.add(new Ec2Module());
-        return modules;
-    }
-
-    @Override
-    @SuppressWarnings("rawtypes") // Supertype uses rawtype
-    public Collection<Class<? extends LifecycleComponent>> getGuiceServiceClasses() {
-        Collection<Class<? extends LifecycleComponent>> services = new ArrayList<>();
-        services.add(AwsEc2ServiceImpl.class);
-        return services;
-    }
-
-    public void onModule(DiscoveryModule discoveryModule) {
-        discoveryModule.addDiscoveryType(EC2, ZenDiscovery.class);
-        discoveryModule.addUnicastHostProvider(EC2, AwsEc2UnicastHostsProvider.class);
-    }
-
-    @Override
-    public NetworkService.CustomNameResolver getCustomNameResolver(Settings settings) {
-        logger.debug("Register _ec2_, _ec2:xxx_ network names");
-        return new Ec2NameResolver(settings);
-    }
-
-    @Override
-    public List<Setting<?>> getSettings() {
-        return Arrays.asList(
-        // Register global cloud aws settings: cloud.aws (might have been registered in ec2 plugin)
-        AwsEc2Service.KEY_SETTING,
-        AwsEc2Service.SECRET_SETTING,
-        AwsEc2Service.PROTOCOL_SETTING,
-        AwsEc2Service.PROXY_HOST_SETTING,
-        AwsEc2Service.PROXY_PORT_SETTING,
-        AwsEc2Service.PROXY_USERNAME_SETTING,
-        AwsEc2Service.PROXY_PASSWORD_SETTING,
-        AwsEc2Service.SIGNER_SETTING,
-        AwsEc2Service.REGION_SETTING,
-        // Register EC2 specific settings: cloud.aws.ec2
-        AwsEc2Service.CLOUD_EC2.KEY_SETTING,
-        AwsEc2Service.CLOUD_EC2.SECRET_SETTING,
-        AwsEc2Service.CLOUD_EC2.PROTOCOL_SETTING,
-        AwsEc2Service.CLOUD_EC2.PROXY_HOST_SETTING,
-        AwsEc2Service.CLOUD_EC2.PROXY_PORT_SETTING,
-        AwsEc2Service.CLOUD_EC2.PROXY_USERNAME_SETTING,
-        AwsEc2Service.CLOUD_EC2.PROXY_PASSWORD_SETTING,
-        AwsEc2Service.CLOUD_EC2.SIGNER_SETTING,
-        AwsEc2Service.CLOUD_EC2.REGION_SETTING,
-        AwsEc2Service.CLOUD_EC2.ENDPOINT_SETTING,
-        // Register EC2 discovery settings: discovery.ec2
-        AwsEc2Service.DISCOVERY_EC2.HOST_TYPE_SETTING,
-        AwsEc2Service.DISCOVERY_EC2.ANY_GROUP_SETTING,
-        AwsEc2Service.DISCOVERY_EC2.GROUPS_SETTING,
-        AwsEc2Service.DISCOVERY_EC2.AVAILABILITY_ZONES_SETTING,
-        AwsEc2Service.DISCOVERY_EC2.NODE_CACHE_TIME_SETTING,
-        AwsEc2Service.DISCOVERY_EC2.TAG_SETTING,
-        // Register cloud node settings: cloud.node
-        AwsEc2Service.AUTO_ATTRIBUTE_SETTING);
-    }
-
-    /** Adds a node attribute for the ec2 availability zone. */
-    @Override
-    public Settings additionalSettings() {
-        return getAvailabilityZoneNodeAttributes(settings, AwsEc2ServiceImpl.EC2_METADATA_URL + "placement/availability-zone");
-    }
-
-    // pkg private for testing
-    static Settings getAvailabilityZoneNodeAttributes(Settings settings, String azMetadataUrl) {
-        if (AwsEc2Service.AUTO_ATTRIBUTE_SETTING.get(settings) == false) {
-            return Settings.EMPTY;
-        }
-        Settings.Builder attrs = Settings.builder();
-
-        final URL url;
-        final URLConnection urlConnection;
-        try {
-            url = new URL(azMetadataUrl);
-            logger.debug("obtaining ec2 [placement/availability-zone] from ec2 meta-data url {}", url);
-            urlConnection = url.openConnection();
-            urlConnection.setConnectTimeout(2000);
-        } catch (IOException e) {
-            // should not happen, we know the url is not malformed, and openConnection does not actually hit network
-            throw new UncheckedIOException(e);
-        }
-
-        try (InputStream in = urlConnection.getInputStream();
-             BufferedReader urlReader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8))) {
-
-            String metadataResult = urlReader.readLine();
-            if (metadataResult == null || metadataResult.length() == 0) {
-                throw new IllegalStateException("no ec2 metadata returned from " + url);
-            } else {
-                attrs.put(Node.NODE_ATTRIBUTES.getKey() + "aws_availability_zone", metadataResult);
-            }
-        } catch (IOException e) {
-            // this is lenient so the plugin does not fail when installed outside of ec2
-            logger.error("failed to get metadata for [placement/availability-zone]", e);
-        }
-
-        return attrs.build();
-    }
-}
diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/AWSSignersTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/AWSSignersTests.java
deleted file mode 100644
index 555e9f5c10a73..0000000000000
--- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/AWSSignersTests.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.cloud.aws;
-
-import com.amazonaws.ClientConfiguration;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.plugin.discovery.ec2.Ec2DiscoveryPlugin;
-import org.elasticsearch.test.ESTestCase;
-import org.junit.BeforeClass;
-
-import static org.hamcrest.CoreMatchers.is;
-
-public class AWSSignersTests extends ESTestCase {
-
-    /**
-     * Starts Ec2DiscoveryPlugin. It's a workaround when you run test from IntelliJ. Otherwise it generates
-     * java.security.AccessControlException: access denied ("java.lang.RuntimePermission" "accessDeclaredMembers")
-     */
-    @BeforeClass
-    public static void instantiatePlugin() {
-        new Ec2DiscoveryPlugin(Settings.EMPTY);
-    }
-
-    public void testSigners() {
-        assertThat(signerTester(null), is(false));
-        assertThat(signerTester("QueryStringSignerType"), is(true));
-        assertThat(signerTester("AWS3SignerType"), is(true));
-        assertThat(signerTester("AWS4SignerType"), is(true));
-        assertThat(signerTester("NoOpSignerType"), is(true));
-        assertThat(signerTester("UndefinedSigner"), is(false));
-
-        assertThat(signerTester("S3SignerType"), is(false));
-        assertThat(signerTester("AWSS3V4SignerType"), is(false));
-
-        ClientConfiguration configuration = new ClientConfiguration();
-        AwsSigner.configureSigner("AWS4SignerType", configuration);
-        assertEquals(configuration.getSignerOverride(), "AWS4SignerType");
-        AwsSigner.configureSigner("AWS3SignerType", configuration);
-        assertEquals(configuration.getSignerOverride(), "AWS3SignerType");
-    }
-
-    /**
-     * Test a signer configuration
-     * @param signer signer name
-     * @return true if successful, false otherwise
-     */
-    private boolean signerTester(String signer) {
-        try {
-            AwsSigner.validateSignerType(signer);
-            return true;
-        } catch (IllegalArgumentException e) {
-            return false;
-        }
-    }
-}
diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTestCase.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTestCase.java
deleted file mode 100644
index a9e16c30dc2a0..0000000000000
--- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTestCase.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.cloud.aws;
-
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.io.PathUtils;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsException;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.plugin.discovery.ec2.Ec2DiscoveryPlugin;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.ESIntegTestCase.ThirdParty;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collection;
-
-/**
- * Base class for AWS tests that require credentials.
- * <p>
- * You must specify {@code -Dtests.thirdparty=true -Dtests.config=/path/to/config}
- * in order to run these tests.
- */
-@ThirdParty
-public abstract class AbstractAwsTestCase extends ESIntegTestCase {
-
-    @Override
-    protected Settings nodeSettings(int nodeOrdinal) {
-                Settings.Builder settings = Settings.builder()
-                .put(super.nodeSettings(nodeOrdinal))
-                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir());
-
-        // if explicit, just load it and don't load from env
-        try {
-            if (Strings.hasText(System.getProperty("tests.config"))) {
-                try {
-                    settings.loadFromPath(PathUtils.get(System.getProperty("tests.config")));
-                } catch (IOException e) {
-                    throw new IllegalArgumentException("could not load aws tests config", e);
-                }
-            } else {
-                throw new IllegalStateException("to run integration tests, you need to set -Dtests.thirdparty=true and -Dtests.config=/path/to/elasticsearch.yml");
-            }
-        } catch (SettingsException exception) {
-            throw new IllegalStateException("your test configuration file is incorrect: " + System.getProperty("tests.config"), exception);
-        }
-        return settings.build();
-    }
-
-    @Override
-    protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Arrays.asList(Ec2DiscoveryPlugin.class);
-    }
-}
diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImplTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImplTests.java
deleted file mode 100644
index 42216df51ea37..0000000000000
--- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImplTests.java
+++ /dev/null
@@ -1,167 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.cloud.aws;
-
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.Protocol;
-import com.amazonaws.auth.AWSCredentials;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.test.ESTestCase;
-
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.nullValue;
-
-public class AwsEc2ServiceImplTests extends ESTestCase {
-
-    public void testAWSCredentialsWithSystemProviders() {
-        AWSCredentialsProvider credentialsProvider = AwsEc2ServiceImpl.buildCredentials(logger, Settings.EMPTY);
-        assertThat(credentialsProvider, instanceOf(DefaultAWSCredentialsProviderChain.class));
-    }
-
-    public void testAWSCredentialsWithElasticsearchAwsSettings() {
-        Settings settings = Settings.builder()
-            .put(AwsEc2Service.KEY_SETTING.getKey(), "aws_key")
-            .put(AwsEc2Service.SECRET_SETTING.getKey(), "aws_secret")
-            .build();
-        launchAWSCredentialsWithElasticsearchSettingsTest(settings, "aws_key", "aws_secret");
-    }
-
-    public void testAWSCredentialsWithElasticsearchEc2Settings() {
-        Settings settings = Settings.builder()
-            .put(AwsEc2Service.CLOUD_EC2.KEY_SETTING.getKey(), "ec2_key")
-            .put(AwsEc2Service.CLOUD_EC2.SECRET_SETTING.getKey(), "ec2_secret")
-            .build();
-        launchAWSCredentialsWithElasticsearchSettingsTest(settings, "ec2_key", "ec2_secret");
-    }
-
-    public void testAWSCredentialsWithElasticsearchAwsAndEc2Settings() {
-        Settings settings = Settings.builder()
-            .put(AwsEc2Service.KEY_SETTING.getKey(), "aws_key")
-            .put(AwsEc2Service.SECRET_SETTING.getKey(), "aws_secret")
-            .put(AwsEc2Service.CLOUD_EC2.KEY_SETTING.getKey(), "ec2_key")
-            .put(AwsEc2Service.CLOUD_EC2.SECRET_SETTING.getKey(), "ec2_secret")
-            .build();
-        launchAWSCredentialsWithElasticsearchSettingsTest(settings, "ec2_key", "ec2_secret");
-    }
-
-    protected void launchAWSCredentialsWithElasticsearchSettingsTest(Settings settings, String expectedKey, String expectedSecret) {
-        AWSCredentials credentials = AwsEc2ServiceImpl.buildCredentials(logger, settings).getCredentials();
-        assertThat(credentials.getAWSAccessKeyId(), is(expectedKey));
-        assertThat(credentials.getAWSSecretKey(), is(expectedSecret));
-    }
-
-    public void testAWSDefaultConfiguration() {
-        launchAWSConfigurationTest(Settings.EMPTY, Protocol.HTTPS, null, -1, null, null, null);
-    }
-
-    public void testAWSConfigurationWithAwsSettings() {
-        Settings settings = Settings.builder()
-            .put(AwsEc2Service.PROTOCOL_SETTING.getKey(), "http")
-            .put(AwsEc2Service.PROXY_HOST_SETTING.getKey(), "aws_proxy_host")
-            .put(AwsEc2Service.PROXY_PORT_SETTING.getKey(), 8080)
-            .put(AwsEc2Service.PROXY_USERNAME_SETTING.getKey(), "aws_proxy_username")
-            .put(AwsEc2Service.PROXY_PASSWORD_SETTING.getKey(), "aws_proxy_password")
-            .put(AwsEc2Service.SIGNER_SETTING.getKey(), "AWS3SignerType")
-            .build();
-        launchAWSConfigurationTest(settings, Protocol.HTTP, "aws_proxy_host", 8080, "aws_proxy_username", "aws_proxy_password",
-            "AWS3SignerType");
-    }
-
-    public void testAWSConfigurationWithAwsAndEc2Settings() {
-        Settings settings = Settings.builder()
-            .put(AwsEc2Service.PROTOCOL_SETTING.getKey(), "http")
-            .put(AwsEc2Service.PROXY_HOST_SETTING.getKey(), "aws_proxy_host")
-            .put(AwsEc2Service.PROXY_PORT_SETTING.getKey(), 8080)
-            .put(AwsEc2Service.PROXY_USERNAME_SETTING.getKey(), "aws_proxy_username")
-            .put(AwsEc2Service.PROXY_PASSWORD_SETTING.getKey(), "aws_proxy_password")
-            .put(AwsEc2Service.SIGNER_SETTING.getKey(), "AWS3SignerType")
-            .put(AwsEc2Service.CLOUD_EC2.PROTOCOL_SETTING.getKey(), "https")
-            .put(AwsEc2Service.CLOUD_EC2.PROXY_HOST_SETTING.getKey(), "ec2_proxy_host")
-            .put(AwsEc2Service.CLOUD_EC2.PROXY_PORT_SETTING.getKey(), 8081)
-            .put(AwsEc2Service.CLOUD_EC2.PROXY_USERNAME_SETTING.getKey(), "ec2_proxy_username")
-            .put(AwsEc2Service.CLOUD_EC2.PROXY_PASSWORD_SETTING.getKey(), "ec2_proxy_password")
-            .put(AwsEc2Service.CLOUD_EC2.SIGNER_SETTING.getKey(), "NoOpSignerType")
-            .build();
-        launchAWSConfigurationTest(settings, Protocol.HTTPS, "ec2_proxy_host", 8081, "ec2_proxy_username", "ec2_proxy_password",
-            "NoOpSignerType");
-    }
-
-    protected void launchAWSConfigurationTest(Settings settings,
-                                              Protocol expectedProtocol,
-                                              String expectedProxyHost,
-                                              int expectedProxyPort,
-                                              String expectedProxyUsername,
-                                              String expectedProxyPassword,
-                                              String expectedSigner) {
-        ClientConfiguration configuration = AwsEc2ServiceImpl.buildConfiguration(logger, settings);
-
-        assertThat(configuration.getResponseMetadataCacheSize(), is(0));
-        assertThat(configuration.getProtocol(), is(expectedProtocol));
-        assertThat(configuration.getProxyHost(), is(expectedProxyHost));
-        assertThat(configuration.getProxyPort(), is(expectedProxyPort));
-        assertThat(configuration.getProxyUsername(), is(expectedProxyUsername));
-        assertThat(configuration.getProxyPassword(), is(expectedProxyPassword));
-        assertThat(configuration.getSignerOverride(), is(expectedSigner));
-    }
-
-    public void testDefaultEndpoint() {
-        String endpoint = AwsEc2ServiceImpl.findEndpoint(logger, Settings.EMPTY);
-        assertThat(endpoint, nullValue());
-    }
-
-    public void testSpecificEndpoint() {
-        Settings settings = Settings.builder()
-            .put(AwsEc2Service.CLOUD_EC2.ENDPOINT_SETTING.getKey(), "ec2.endpoint")
-            .build();
-        String endpoint = AwsEc2ServiceImpl.findEndpoint(logger, settings);
-        assertThat(endpoint, is("ec2.endpoint"));
-    }
-
-    public void testRegionWithAwsSettings() {
-        Settings settings = Settings.builder()
-            .put(AwsEc2Service.REGION_SETTING.getKey(), randomFrom("eu-west", "eu-west-1"))
-            .build();
-        String endpoint = AwsEc2ServiceImpl.findEndpoint(logger, settings);
-        assertThat(endpoint, is("ec2.eu-west-1.amazonaws.com"));
-    }
-
-    public void testRegionWithAwsAndEc2Settings() {
-        Settings settings = Settings.builder()
-            .put(AwsEc2Service.REGION_SETTING.getKey(), randomFrom("eu-west", "eu-west-1"))
-            .put(AwsEc2Service.CLOUD_EC2.REGION_SETTING.getKey(), randomFrom("us-west", "us-west-1"))
-            .build();
-        String endpoint = AwsEc2ServiceImpl.findEndpoint(logger, settings);
-        assertThat(endpoint, is("ec2.us-west-1.amazonaws.com"));
-    }
-
-    public void testInvalidRegion() {
-        Settings settings = Settings.builder()
-            .put(AwsEc2Service.REGION_SETTING.getKey(), "does-not-exist")
-            .build();
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {
-            AwsEc2ServiceImpl.findEndpoint(logger, settings);
-        });
-        assertThat(e.getMessage(), containsString("No automatic endpoint could be derived from region"));
-    }
-}
diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AWSSignersTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AWSSignersTests.java
new file mode 100644
index 0000000000000..d9d3599b59b36
--- /dev/null
+++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AWSSignersTests.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.discovery.ec2;
+
+import com.amazonaws.ClientConfiguration;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.discovery.ec2.AwsSigner;
+import org.elasticsearch.discovery.ec2.Ec2DiscoveryPlugin;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.BeforeClass;
+
+import static org.hamcrest.CoreMatchers.is;
+
+public class AWSSignersTests extends ESTestCase {
+
+    /**
+     * Starts Ec2DiscoveryPlugin. It's a workaround when you run test from IntelliJ. Otherwise it generates
+     * java.security.AccessControlException: access denied ("java.lang.RuntimePermission" "accessDeclaredMembers")
+     */
+    @BeforeClass
+    public static void instantiatePlugin() {
+        new Ec2DiscoveryPlugin(Settings.EMPTY);
+    }
+
+    public void testSigners() {
+        assertThat(signerTester(null), is(false));
+        assertThat(signerTester("QueryStringSignerType"), is(true));
+        assertThat(signerTester("AWS3SignerType"), is(true));
+        assertThat(signerTester("AWS4SignerType"), is(true));
+        assertThat(signerTester("NoOpSignerType"), is(true));
+        assertThat(signerTester("UndefinedSigner"), is(false));
+
+        assertThat(signerTester("S3SignerType"), is(false));
+        assertThat(signerTester("AWSS3V4SignerType"), is(false));
+
+        ClientConfiguration configuration = new ClientConfiguration();
+        AwsSigner.configureSigner("AWS4SignerType", configuration);
+        assertEquals(configuration.getSignerOverride(), "AWS4SignerType");
+        AwsSigner.configureSigner("AWS3SignerType", configuration);
+        assertEquals(configuration.getSignerOverride(), "AWS3SignerType");
+    }
+
+    /**
+     * Test a signer configuration
+     * @param signer signer name
+     * @return true if successful, false otherwise
+     */
+    private boolean signerTester(String signer) {
+        try {
+            AwsSigner.validateSignerType(signer);
+            return true;
+        } catch (IllegalArgumentException e) {
+            return false;
+        }
+    }
+}
diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AbstractAwsTestCase.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AbstractAwsTestCase.java
new file mode 100644
index 0000000000000..43706e0d6f7e7
--- /dev/null
+++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AbstractAwsTestCase.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.discovery.ec2;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.PathUtils;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsException;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.discovery.ec2.Ec2DiscoveryPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.ESIntegTestCase.ThirdParty;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+
+/**
+ * Base class for AWS tests that require credentials.
+ * <p>
+ * You must specify {@code -Dtests.thirdparty=true -Dtests.config=/path/to/config}
+ * in order to run these tests.
+ */
+@ThirdParty
+public abstract class AbstractAwsTestCase extends ESIntegTestCase {
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+                Settings.Builder settings = Settings.builder()
+                .put(super.nodeSettings(nodeOrdinal))
+                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir());
+
+        // if explicit, just load it and don't load from env
+        try {
+            if (Strings.hasText(System.getProperty("tests.config"))) {
+                try {
+                    settings.loadFromPath(PathUtils.get(System.getProperty("tests.config")));
+                } catch (IOException e) {
+                    throw new IllegalArgumentException("could not load aws tests config", e);
+                }
+            } else {
+                throw new IllegalStateException("to run integration tests, you need to set -Dtests.thirdparty=true and -Dtests.config=/path/to/elasticsearch.yml");
+            }
+        } catch (SettingsException exception) {
+            throw new IllegalStateException("your test configuration file is incorrect: " + System.getProperty("tests.config"), exception);
+        }
+        return settings.build();
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Arrays.asList(Ec2DiscoveryPlugin.class);
+    }
+}
diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImplTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImplTests.java
new file mode 100644
index 0000000000000..0e82d50c17cda
--- /dev/null
+++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImplTests.java
@@ -0,0 +1,260 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.discovery.ec2;
+
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.Protocol;
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
+import org.elasticsearch.common.settings.MockSecureSettings;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.discovery.ec2.AwsEc2Service;
+import org.elasticsearch.discovery.ec2.AwsEc2ServiceImpl;
+import org.elasticsearch.test.ESTestCase;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.nullValue;
+
+public class AwsEc2ServiceImplTests extends ESTestCase {
+
+    public void testAWSCredentialsWithSystemProviders() {
+        AWSCredentialsProvider credentialsProvider = AwsEc2ServiceImpl.buildCredentials(logger, Settings.EMPTY);
+        assertThat(credentialsProvider, instanceOf(DefaultAWSCredentialsProviderChain.class));
+    }
+
+    public void testAWSCredentialsWithElasticsearchAwsSettings() {
+        MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("discovery.ec2.access_key", "aws_key");
+        secureSettings.setString("discovery.ec2.secret_key", "aws_secret");
+        Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
+        launchAWSCredentialsWithElasticsearchSettingsTest(settings, "aws_key", "aws_secret");
+    }
+
+    public void testAWSCredentialsWithElasticsearchAwsSettingsBackcompat() {
+        Settings settings = Settings.builder()
+            .put(AwsEc2Service.KEY_SETTING.getKey(), "aws_key")
+            .put(AwsEc2Service.SECRET_SETTING.getKey(), "aws_secret")
+            .build();
+        launchAWSCredentialsWithElasticsearchSettingsTest(settings, "aws_key", "aws_secret");
+        assertSettingDeprecationsAndWarnings(new Setting<?>[] {
+            AwsEc2Service.KEY_SETTING,
+            AwsEc2Service.SECRET_SETTING
+        });
+    }
+
+    public void testAWSCredentialsWithElasticsearchEc2SettingsBackcompat() {
+        Settings settings = Settings.builder()
+            .put(AwsEc2Service.CLOUD_EC2.KEY_SETTING.getKey(), "ec2_key")
+            .put(AwsEc2Service.CLOUD_EC2.SECRET_SETTING.getKey(), "ec2_secret")
+            .build();
+        launchAWSCredentialsWithElasticsearchSettingsTest(settings, "ec2_key", "ec2_secret");
+        assertSettingDeprecationsAndWarnings(new Setting<?>[] {
+            AwsEc2Service.CLOUD_EC2.KEY_SETTING,
+            AwsEc2Service.CLOUD_EC2.SECRET_SETTING
+        });
+    }
+
+    public void testAWSCredentialsWithElasticsearchAwsAndEc2Settings() {
+        Settings settings = Settings.builder()
+            .put(AwsEc2Service.KEY_SETTING.getKey(), "aws_key")
+            .put(AwsEc2Service.SECRET_SETTING.getKey(), "aws_secret")
+            .put(AwsEc2Service.CLOUD_EC2.KEY_SETTING.getKey(), "ec2_key")
+            .put(AwsEc2Service.CLOUD_EC2.SECRET_SETTING.getKey(), "ec2_secret")
+            .build();
+        launchAWSCredentialsWithElasticsearchSettingsTest(settings, "ec2_key", "ec2_secret");
+        assertSettingDeprecationsAndWarnings(new Setting<?>[] {
+            AwsEc2Service.KEY_SETTING,
+            AwsEc2Service.SECRET_SETTING,
+            AwsEc2Service.CLOUD_EC2.KEY_SETTING,
+            AwsEc2Service.CLOUD_EC2.SECRET_SETTING
+        });
+    }
+
+    protected void launchAWSCredentialsWithElasticsearchSettingsTest(Settings settings, String expectedKey, String expectedSecret) {
+        AWSCredentials credentials = AwsEc2ServiceImpl.buildCredentials(logger, settings).getCredentials();
+        assertThat(credentials.getAWSAccessKeyId(), is(expectedKey));
+        assertThat(credentials.getAWSSecretKey(), is(expectedSecret));
+    }
+
+    public void testAWSDefaultConfiguration() {
+        launchAWSConfigurationTest(Settings.EMPTY, Protocol.HTTPS, null, -1, null, null, null,
+            ClientConfiguration.DEFAULT_SOCKET_TIMEOUT);
+    }
+
+    public void testAWSConfigurationWithAwsSettings() {
+        MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("discovery.ec2.proxy.username", "aws_proxy_username");
+        secureSettings.setString("discovery.ec2.proxy.password", "aws_proxy_password");
+        Settings settings = Settings.builder()
+            .put("discovery.ec2.protocol", "http")
+            .put("discovery.ec2.proxy.host", "aws_proxy_host")
+            .put("discovery.ec2.proxy.port", 8080)
+            .put("discovery.ec2.read_timeout", "10s")
+            .setSecureSettings(secureSettings)
+            .build();
+        launchAWSConfigurationTest(settings, Protocol.HTTP, "aws_proxy_host", 8080, "aws_proxy_username", "aws_proxy_password",
+            null, 10000);
+    }
+
+    public void testAWSConfigurationWithAwsSettingsBackcompat() {
+        Settings settings = Settings.builder()
+            .put(AwsEc2Service.PROTOCOL_SETTING.getKey(), "http")
+            .put(AwsEc2Service.PROXY_HOST_SETTING.getKey(), "aws_proxy_host")
+            .put(AwsEc2Service.PROXY_PORT_SETTING.getKey(), 8080)
+            .put(AwsEc2Service.PROXY_USERNAME_SETTING.getKey(), "aws_proxy_username")
+            .put(AwsEc2Service.PROXY_PASSWORD_SETTING.getKey(), "aws_proxy_password")
+            .put(AwsEc2Service.SIGNER_SETTING.getKey(), "AWS3SignerType")
+            .put(AwsEc2Service.READ_TIMEOUT.getKey(), "10s")
+            .build();
+        launchAWSConfigurationTest(settings, Protocol.HTTP, "aws_proxy_host", 8080, "aws_proxy_username", "aws_proxy_password",
+            "AWS3SignerType", 10000);
+        assertSettingDeprecationsAndWarnings(new Setting<?>[] {
+            AwsEc2Service.PROTOCOL_SETTING,
+            AwsEc2Service.PROXY_HOST_SETTING,
+            AwsEc2Service.PROXY_PORT_SETTING,
+            AwsEc2Service.PROXY_USERNAME_SETTING,
+            AwsEc2Service.PROXY_PASSWORD_SETTING,
+            AwsEc2Service.SIGNER_SETTING,
+            AwsEc2Service.READ_TIMEOUT
+        });
+    }
+
+    public void testAWSConfigurationWithAwsAndEc2Settings() {
+        Settings settings = Settings.builder()
+            .put(AwsEc2Service.PROTOCOL_SETTING.getKey(), "http")
+            .put(AwsEc2Service.PROXY_HOST_SETTING.getKey(), "aws_proxy_host")
+            .put(AwsEc2Service.PROXY_PORT_SETTING.getKey(), 8080)
+            .put(AwsEc2Service.PROXY_USERNAME_SETTING.getKey(), "aws_proxy_username")
+            .put(AwsEc2Service.PROXY_PASSWORD_SETTING.getKey(), "aws_proxy_password")
+            .put(AwsEc2Service.SIGNER_SETTING.getKey(), "AWS3SignerType")
+            .put(AwsEc2Service.READ_TIMEOUT.getKey(), "20s")
+            .put(AwsEc2Service.CLOUD_EC2.PROTOCOL_SETTING.getKey(), "https")
+            .put(AwsEc2Service.CLOUD_EC2.PROXY_HOST_SETTING.getKey(), "ec2_proxy_host")
+            .put(AwsEc2Service.CLOUD_EC2.PROXY_PORT_SETTING.getKey(), 8081)
+            .put(AwsEc2Service.CLOUD_EC2.PROXY_USERNAME_SETTING.getKey(), "ec2_proxy_username")
+            .put(AwsEc2Service.CLOUD_EC2.PROXY_PASSWORD_SETTING.getKey(), "ec2_proxy_password")
+            .put(AwsEc2Service.CLOUD_EC2.SIGNER_SETTING.getKey(), "NoOpSignerType")
+            .put(AwsEc2Service.CLOUD_EC2.READ_TIMEOUT.getKey(), "10s")
+            .build();
+        launchAWSConfigurationTest(settings, Protocol.HTTPS, "ec2_proxy_host", 8081, "ec2_proxy_username", "ec2_proxy_password",
+            "NoOpSignerType", 10000);
+        assertSettingDeprecationsAndWarnings(new Setting<?>[] {
+            AwsEc2Service.PROTOCOL_SETTING,
+            AwsEc2Service.PROXY_HOST_SETTING,
+            AwsEc2Service.PROXY_PORT_SETTING,
+            AwsEc2Service.PROXY_USERNAME_SETTING,
+            AwsEc2Service.PROXY_PASSWORD_SETTING,
+            AwsEc2Service.SIGNER_SETTING,
+            AwsEc2Service.READ_TIMEOUT,
+            AwsEc2Service.CLOUD_EC2.PROTOCOL_SETTING,
+            AwsEc2Service.CLOUD_EC2.PROXY_HOST_SETTING,
+            AwsEc2Service.CLOUD_EC2.PROXY_PORT_SETTING,
+            AwsEc2Service.CLOUD_EC2.PROXY_USERNAME_SETTING,
+            AwsEc2Service.CLOUD_EC2.PROXY_PASSWORD_SETTING,
+            AwsEc2Service.CLOUD_EC2.SIGNER_SETTING,
+            AwsEc2Service.CLOUD_EC2.READ_TIMEOUT
+        });
+    }
+
+    protected void launchAWSConfigurationTest(Settings settings,
+                                              Protocol expectedProtocol,
+                                              String expectedProxyHost,
+                                              int expectedProxyPort,
+                                              String expectedProxyUsername,
+                                              String expectedProxyPassword,
+                                              String expectedSigner,
+                                              int expectedReadTimeout) {
+        ClientConfiguration configuration = AwsEc2ServiceImpl.buildConfiguration(logger, settings);
+
+        assertThat(configuration.getResponseMetadataCacheSize(), is(0));
+        assertThat(configuration.getProtocol(), is(expectedProtocol));
+        assertThat(configuration.getProxyHost(), is(expectedProxyHost));
+        assertThat(configuration.getProxyPort(), is(expectedProxyPort));
+        assertThat(configuration.getProxyUsername(), is(expectedProxyUsername));
+        assertThat(configuration.getProxyPassword(), is(expectedProxyPassword));
+        assertThat(configuration.getSignerOverride(), is(expectedSigner));
+        assertThat(configuration.getSocketTimeout(), is(expectedReadTimeout));
+    }
+
+    public void testDefaultEndpoint() {
+        String endpoint = AwsEc2ServiceImpl.findEndpoint(logger, Settings.EMPTY);
+        assertThat(endpoint, nullValue());
+    }
+
+    public void testSpecificEndpoint() {
+        Settings settings = Settings.builder()
+            .put(AwsEc2Service.DISCOVERY_EC2.ENDPOINT_SETTING.getKey(), "ec2.endpoint")
+            .build();
+        String endpoint = AwsEc2ServiceImpl.findEndpoint(logger, settings);
+        assertThat(endpoint, is("ec2.endpoint"));
+    }
+
+    public void testSpecificEndpointBackcompat() {
+        Settings settings = Settings.builder()
+            .put(AwsEc2Service.CLOUD_EC2.ENDPOINT_SETTING.getKey(), "ec2.endpoint")
+            .build();
+        String endpoint = AwsEc2ServiceImpl.findEndpoint(logger, settings);
+        assertThat(endpoint, is("ec2.endpoint"));
+        assertSettingDeprecationsAndWarnings(new Setting<?>[] {
+            AwsEc2Service.CLOUD_EC2.ENDPOINT_SETTING
+        });
+    }
+
+    public void testRegionWithAwsSettings() {
+        Settings settings = Settings.builder()
+            .put(AwsEc2Service.REGION_SETTING.getKey(), randomFrom("eu-west", "eu-west-1"))
+            .build();
+        String endpoint = AwsEc2ServiceImpl.findEndpoint(logger, settings);
+        assertThat(endpoint, is("ec2.eu-west-1.amazonaws.com"));
+        assertSettingDeprecationsAndWarnings(new Setting<?>[] {
+            AwsEc2Service.REGION_SETTING
+        });
+    }
+
+    public void testRegionWithAwsAndEc2Settings() {
+        Settings settings = Settings.builder()
+            .put(AwsEc2Service.REGION_SETTING.getKey(), randomFrom("eu-west", "eu-west-1"))
+            .put(AwsEc2Service.CLOUD_EC2.REGION_SETTING.getKey(), randomFrom("us-west", "us-west-1"))
+            .build();
+        String endpoint = AwsEc2ServiceImpl.findEndpoint(logger, settings);
+        assertThat(endpoint, is("ec2.us-west-1.amazonaws.com"));
+        assertSettingDeprecationsAndWarnings(new Setting<?>[] {
+            AwsEc2Service.REGION_SETTING,
+            AwsEc2Service.CLOUD_EC2.REGION_SETTING
+        });
+    }
+
+    public void testInvalidRegion() {
+        Settings settings = Settings.builder()
+            .put(AwsEc2Service.REGION_SETTING.getKey(), "does-not-exist")
+            .build();
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {
+            AwsEc2ServiceImpl.findEndpoint(logger, settings);
+        });
+        assertThat(e.getMessage(), containsString("No automatic endpoint could be derived from region"));
+        assertSettingDeprecationsAndWarnings(new Setting<?>[] {
+            AwsEc2Service.REGION_SETTING
+        });
+    }
+}
diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceMock.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceMock.java
index 19a5c8d3b9614..e29821efda223 100644
--- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceMock.java
+++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceMock.java
@@ -21,7 +21,6 @@
 
 import com.amazonaws.services.ec2.AmazonEC2;
 import com.amazonaws.services.ec2.model.Tag;
-import org.elasticsearch.cloud.aws.AwsEc2Service;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.settings.Settings;
 
diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/CloudAwsClientYamlTestSuiteIT.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/CloudAwsClientYamlTestSuiteIT.java
similarity index 80%
rename from plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/CloudAwsClientYamlTestSuiteIT.java
rename to plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/CloudAwsClientYamlTestSuiteIT.java
index f5f49c148338b..39bc468da89b3 100644
--- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/CloudAwsClientYamlTestSuiteIT.java
+++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/CloudAwsClientYamlTestSuiteIT.java
@@ -17,16 +17,13 @@
  * under the License.
  */
 
-package org.elasticsearch.cloud.aws;
+package org.elasticsearch.discovery.ec2;
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-
-import java.io.IOException;
 
 public class CloudAwsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
@@ -35,8 +32,8 @@ public CloudAwsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testC
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
-        return ESClientYamlSuiteTestCase.createParameters(0, 1);
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
     }
 }
 
diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryClusterFormationTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryClusterFormationTests.java
index bd69953eb1e43..964b01179acdd 100644
--- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryClusterFormationTests.java
+++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryClusterFormationTests.java
@@ -24,12 +24,10 @@
 import com.sun.net.httpserver.HttpServer;
 import org.apache.http.NameValuePair;
 import org.apache.http.client.utils.URLEncodedUtils;
-import org.elasticsearch.cloud.aws.AwsEc2Service;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.plugin.discovery.ec2.Ec2DiscoveryPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.junit.AfterClass;
@@ -244,7 +242,7 @@ public void testJoin() throws ExecutionException, InterruptedException {
         // only wait for the cluster to form
         assertNoTimeout(client().admin().cluster().prepareHealth().setWaitForNodes(Integer.toString(2)).get());
         // add one more node and wait for it to join
-        internalCluster().startDataOnlyNodeAsync().get();
+        internalCluster().startDataOnlyNode();
         assertNoTimeout(client().admin().cluster().prepareHealth().setWaitForNodes(Integer.toString(3)).get());
     }
 }
diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPluginTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPluginTests.java
similarity index 95%
rename from plugins/discovery-ec2/src/test/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPluginTests.java
rename to plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPluginTests.java
index 14ba0d2339a9d..9bb75c0b09f97 100644
--- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPluginTests.java
+++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPluginTests.java
@@ -17,15 +17,16 @@
  * under the License.
  */
 
-package org.elasticsearch.plugin.discovery.ec2;
+package org.elasticsearch.discovery.ec2;
 
 import java.io.UncheckedIOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Arrays;
 
-import org.elasticsearch.cloud.aws.AwsEc2Service;
+import org.elasticsearch.discovery.ec2.AwsEc2Service;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.discovery.ec2.Ec2DiscoveryPlugin;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.test.ESTestCase;
 
diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoverySettingsTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoverySettingsTests.java
index 1fef33fdca357..d1ac8cfa0cf59 100644
--- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoverySettingsTests.java
+++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoverySettingsTests.java
@@ -20,7 +20,7 @@
 package org.elasticsearch.discovery.ec2;
 
 import com.amazonaws.Protocol;
-import org.elasticsearch.cloud.aws.AwsEc2Service;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
 
@@ -69,6 +69,17 @@ public void testRepositorySettingsGlobalOnly() {
         assertThat(AwsEc2Service.CLOUD_EC2.SIGNER_SETTING.get(nodeSettings), is("global-signer"));
         assertThat(AwsEc2Service.CLOUD_EC2.REGION_SETTING.get(nodeSettings), is("global-region"));
         assertThat(AwsEc2Service.CLOUD_EC2.ENDPOINT_SETTING.get(nodeSettings), isEmptyString());
+        assertSettingDeprecationsAndWarnings(new Setting<?>[] {
+                AwsEc2Service.KEY_SETTING,
+                AwsEc2Service.SECRET_SETTING,
+                AwsEc2Service.PROTOCOL_SETTING,
+                AwsEc2Service.PROXY_HOST_SETTING,
+                AwsEc2Service.PROXY_PORT_SETTING,
+                AwsEc2Service.PROXY_USERNAME_SETTING,
+                AwsEc2Service.PROXY_PASSWORD_SETTING,
+                AwsEc2Service.SIGNER_SETTING,
+                AwsEc2Service.REGION_SETTING
+        });
     }
 
     /**
@@ -86,6 +97,27 @@ public void testRepositorySettingsGlobalOverloadedByEC2() {
         assertThat(AwsEc2Service.CLOUD_EC2.SIGNER_SETTING.get(nodeSettings), is("ec2-signer"));
         assertThat(AwsEc2Service.CLOUD_EC2.REGION_SETTING.get(nodeSettings), is("ec2-region"));
         assertThat(AwsEc2Service.CLOUD_EC2.ENDPOINT_SETTING.get(nodeSettings), is("ec2-endpoint"));
+        assertSettingDeprecationsAndWarnings(new Setting<?>[] {
+            AwsEc2Service.KEY_SETTING,
+            AwsEc2Service.SECRET_SETTING,
+            AwsEc2Service.PROTOCOL_SETTING,
+            AwsEc2Service.PROXY_HOST_SETTING,
+            AwsEc2Service.PROXY_PORT_SETTING,
+            AwsEc2Service.PROXY_USERNAME_SETTING,
+            AwsEc2Service.PROXY_PASSWORD_SETTING,
+            AwsEc2Service.SIGNER_SETTING,
+            AwsEc2Service.REGION_SETTING,
+            AwsEc2Service.CLOUD_EC2.KEY_SETTING,
+            AwsEc2Service.CLOUD_EC2.SECRET_SETTING,
+            AwsEc2Service.CLOUD_EC2.PROTOCOL_SETTING,
+            AwsEc2Service.CLOUD_EC2.PROXY_HOST_SETTING,
+            AwsEc2Service.CLOUD_EC2.PROXY_PORT_SETTING,
+            AwsEc2Service.CLOUD_EC2.PROXY_USERNAME_SETTING,
+            AwsEc2Service.CLOUD_EC2.PROXY_PASSWORD_SETTING,
+            AwsEc2Service.CLOUD_EC2.SIGNER_SETTING,
+            AwsEc2Service.CLOUD_EC2.REGION_SETTING,
+            AwsEc2Service.CLOUD_EC2.ENDPOINT_SETTING
+        });
     }
 
     private Settings buildSettings(Settings... global) {
diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java
index edb062d9e303f..e64f62298fcef 100644
--- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java
+++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java
@@ -21,24 +21,34 @@
 
 import com.amazonaws.services.ec2.model.Tag;
 import org.elasticsearch.Version;
-import org.elasticsearch.cloud.aws.AwsEc2Service;
-import org.elasticsearch.cloud.aws.AwsEc2Service.DISCOVERY_EC2;
-import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.discovery.ec2.AwsEc2Service.DISCOVERY_EC2;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.LocalTransportAddress;
+import org.elasticsearch.common.transport.InetSocketTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.MockTcpTransport;
+import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportService;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
 
+import java.net.InetAddress;
+import java.net.UnknownHostException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.hasSize;
@@ -51,6 +61,25 @@ public class Ec2DiscoveryTests extends ESTestCase {
 
     protected static ThreadPool threadPool;
     protected MockTransportService transportService;
+    private Map<String, TransportAddress> poorMansDNS = new ConcurrentHashMap<>();
+
+    private static final AtomicInteger portGenerator = new AtomicInteger();
+    private static final InetAddress META_ADDRESS;
+    static {
+        try {
+            META_ADDRESS = InetAddress.getByName("0.0.0.0");
+        } catch (UnknownHostException e) {
+            throw new AssertionError(e);
+        }
+    }
+
+    /**
+     * Generates a new transport address using 0.0.0.0 with an incrementing port number.
+     * The port number starts at 0 and is reset after each test suite run.
+     */
+    public static TransportAddress buildNewFakeTransportAddress() {
+        return new InetSocketTransportAddress(META_ADDRESS, portGenerator.incrementAndGet());
+    }
 
     @BeforeClass
     public static void createThreadPool() {
@@ -67,7 +96,18 @@ public static void stopThreadPool() throws InterruptedException {
 
     @Before
     public void createTransportService() {
-        transportService = MockTransportService.local(Settings.EMPTY, Version.CURRENT, threadPool);
+        NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(Collections.emptyList());
+        final Transport transport = new MockTcpTransport(Settings.EMPTY, threadPool, BigArrays.NON_RECYCLING_INSTANCE,
+            new NoneCircuitBreakerService(), namedWriteableRegistry, new NetworkService(Settings.EMPTY, Collections.emptyList()),
+            Version.CURRENT) {
+            @Override
+            public TransportAddress[] addressesFromString(String address, int perAddressLimit) {
+                // we just need to ensure we don't resolve DNS here
+                return new TransportAddress[] {poorMansDNS.getOrDefault(address, buildNewFakeTransportAddress())};
+            }
+        };
+        transportService = new MockTransportService(Settings.EMPTY, transport, threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR,
+                null);
     }
 
     protected List<DiscoveryNode> buildDynamicNodes(Settings nodeSettings, int nodes) {
@@ -92,41 +132,51 @@ public void testDefaultSettings() throws InterruptedException {
 
     public void testPrivateIp() throws InterruptedException {
         int nodes = randomInt(10);
+        for (int node = 1; node < nodes + 1; node++) {
+            poorMansDNS.put(AmazonEC2Mock.PREFIX_PRIVATE_IP + node, buildNewFakeTransportAddress());
+        }
         Settings nodeSettings = Settings.builder()
-                .put(DISCOVERY_EC2.HOST_TYPE_SETTING.getKey(), "private_ip")
-                .build();
+            .put(DISCOVERY_EC2.HOST_TYPE_SETTING.getKey(), "private_ip")
+            .build();
         List<DiscoveryNode> discoveryNodes = buildDynamicNodes(nodeSettings, nodes);
         assertThat(discoveryNodes, hasSize(nodes));
         // We check that we are using here expected address
         int node = 1;
         for (DiscoveryNode discoveryNode : discoveryNodes) {
             TransportAddress address = discoveryNode.getAddress();
-            TransportAddress expected = new LocalTransportAddress(AmazonEC2Mock.PREFIX_PRIVATE_IP + node++);
-            assertThat(address.sameHost(expected), is(true));
+            TransportAddress expected = poorMansDNS.get(AmazonEC2Mock.PREFIX_PRIVATE_IP + node++);
+            assertEquals(address, expected);
         }
     }
 
     public void testPublicIp() throws InterruptedException {
         int nodes = randomInt(10);
+        for (int node = 1; node < nodes + 1; node++) {
+            poorMansDNS.put(AmazonEC2Mock.PREFIX_PUBLIC_IP + node, buildNewFakeTransportAddress());
+        }
         Settings nodeSettings = Settings.builder()
-                .put(DISCOVERY_EC2.HOST_TYPE_SETTING.getKey(), "public_ip")
-                .build();
+            .put(DISCOVERY_EC2.HOST_TYPE_SETTING.getKey(), "public_ip")
+            .build();
         List<DiscoveryNode> discoveryNodes = buildDynamicNodes(nodeSettings, nodes);
         assertThat(discoveryNodes, hasSize(nodes));
         // We check that we are using here expected address
         int node = 1;
         for (DiscoveryNode discoveryNode : discoveryNodes) {
             TransportAddress address = discoveryNode.getAddress();
-            TransportAddress expected = new LocalTransportAddress(AmazonEC2Mock.PREFIX_PUBLIC_IP + node++);
-            assertThat(address.sameHost(expected), is(true));
+            TransportAddress expected = poorMansDNS.get(AmazonEC2Mock.PREFIX_PUBLIC_IP + node++);
+            assertEquals(address, expected);
         }
     }
 
     public void testPrivateDns() throws InterruptedException {
         int nodes = randomInt(10);
+        for (int node = 1; node < nodes + 1; node++) {
+            poorMansDNS.put(AmazonEC2Mock.PREFIX_PRIVATE_DNS + "node" + node +
+                AmazonEC2Mock.SUFFIX_PRIVATE_DNS, buildNewFakeTransportAddress());
+        }
         Settings nodeSettings = Settings.builder()
-                .put(DISCOVERY_EC2.HOST_TYPE_SETTING.getKey(), "private_dns")
-                .build();
+            .put(DISCOVERY_EC2.HOST_TYPE_SETTING.getKey(), "private_dns")
+            .build();
         List<DiscoveryNode> discoveryNodes = buildDynamicNodes(nodeSettings, nodes);
         assertThat(discoveryNodes, hasSize(nodes));
         // We check that we are using here expected address
@@ -134,17 +184,21 @@ public void testPrivateDns() throws InterruptedException {
         for (DiscoveryNode discoveryNode : discoveryNodes) {
             String instanceId = "node" + node++;
             TransportAddress address = discoveryNode.getAddress();
-            TransportAddress expected = new LocalTransportAddress(
-                    AmazonEC2Mock.PREFIX_PRIVATE_DNS + instanceId + AmazonEC2Mock.SUFFIX_PRIVATE_DNS);
-            assertThat(address.sameHost(expected), is(true));
+            TransportAddress expected = poorMansDNS.get(
+                AmazonEC2Mock.PREFIX_PRIVATE_DNS + instanceId + AmazonEC2Mock.SUFFIX_PRIVATE_DNS);
+            assertEquals(address, expected);
         }
     }
 
     public void testPublicDns() throws InterruptedException {
         int nodes = randomInt(10);
+        for (int node = 1; node < nodes + 1; node++) {
+            poorMansDNS.put(AmazonEC2Mock.PREFIX_PUBLIC_DNS + "node" + node
+                + AmazonEC2Mock.SUFFIX_PUBLIC_DNS, buildNewFakeTransportAddress());
+        }
         Settings nodeSettings = Settings.builder()
-                .put(DISCOVERY_EC2.HOST_TYPE_SETTING.getKey(), "public_dns")
-                .build();
+            .put(DISCOVERY_EC2.HOST_TYPE_SETTING.getKey(), "public_dns")
+            .build();
         List<DiscoveryNode> discoveryNodes = buildDynamicNodes(nodeSettings, nodes);
         assertThat(discoveryNodes, hasSize(nodes));
         // We check that we are using here expected address
@@ -152,9 +206,9 @@ public void testPublicDns() throws InterruptedException {
         for (DiscoveryNode discoveryNode : discoveryNodes) {
             String instanceId = "node" + node++;
             TransportAddress address = discoveryNode.getAddress();
-            TransportAddress expected = new LocalTransportAddress(
-                    AmazonEC2Mock.PREFIX_PUBLIC_DNS + instanceId + AmazonEC2Mock.SUFFIX_PUBLIC_DNS);
-            assertThat(address.sameHost(expected), is(true));
+            TransportAddress expected = poorMansDNS.get(
+                AmazonEC2Mock.PREFIX_PUBLIC_DNS + instanceId + AmazonEC2Mock.SUFFIX_PUBLIC_DNS);
+            assertEquals(address, expected);
         }
     }
 
@@ -162,12 +216,11 @@ public void testInvalidHostType() throws InterruptedException {
         Settings nodeSettings = Settings.builder()
                 .put(DISCOVERY_EC2.HOST_TYPE_SETTING.getKey(), "does_not_exist")
                 .build();
-        try {
+
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> {
             buildDynamicNodes(nodeSettings, 1);
-            fail("Expected IllegalArgumentException");
-        } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), containsString("No enum constant"));
-        }
+        });
+        assertThat(exception.getMessage(), containsString("does_not_exist is unknown for discovery.ec2.host_type"));
     }
 
     public void testFilterByTags() throws InterruptedException {
@@ -226,9 +279,39 @@ public void testFilterByMultipleTags() throws InterruptedException {
         assertThat(discoveryNodes, hasSize(prodInstances));
     }
 
+    public void testReadHostFromTag() throws InterruptedException, UnknownHostException {
+        int nodes = randomIntBetween(5, 10);
+
+        for (int node = 1; node < nodes + 1; node++) {
+            poorMansDNS.put("node" + node, new InetSocketTransportAddress(InetAddress.getByName("192.168.0." + node), 9300));
+        }
+
+        Settings nodeSettings = Settings.builder()
+            .put(DISCOVERY_EC2.HOST_TYPE_SETTING.getKey(), "tag:foo")
+            .build();
+
+        List<List<Tag>> tagsList = new ArrayList<>();
+
+        for (int node = 1; node < nodes + 1; node++) {
+            List<Tag> tags = new ArrayList<>();
+            tags.add(new Tag("foo", "node" + node));
+            tagsList.add(tags);
+        }
+
+        logger.info("started [{}] instances", nodes);
+        List<DiscoveryNode> discoveryNodes = buildDynamicNodes(nodeSettings, nodes, tagsList);
+        assertThat(discoveryNodes, hasSize(nodes));
+        for (DiscoveryNode discoveryNode : discoveryNodes) {
+            TransportAddress address = discoveryNode.getAddress();
+            TransportAddress expected = poorMansDNS.get(discoveryNode.getName());
+            assertEquals(address, expected);
+        }
+    }
+
+
     abstract class DummyEc2HostProvider extends AwsEc2UnicastHostsProvider {
         public int fetchCount = 0;
-        public DummyEc2HostProvider(Settings settings, TransportService transportService, AwsEc2Service service) {
+        DummyEc2HostProvider(Settings settings, TransportService transportService, AwsEc2Service service) {
             super(settings, transportService, service);
         }
     }
diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryUpdateSettingsTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryUpdateSettingsTests.java
index cdb45bfe0875b..fca70e2b436e8 100644
--- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryUpdateSettingsTests.java
+++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryUpdateSettingsTests.java
@@ -21,7 +21,6 @@
 
 
 import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse;
-import org.elasticsearch.cloud.aws.AbstractAwsTestCase;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
@@ -33,7 +32,7 @@
  * starting.
  * This test requires AWS to run.
  */
-@ClusterScope(scope = Scope.TEST, numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0)
+@ClusterScope(scope = Scope.TEST, numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0, autoMinMasterNodes = false)
 public class Ec2DiscoveryUpdateSettingsTests extends AbstractAwsTestCase {
     public void testMinimumMasterNodesStart() {
         Settings nodeSettings = Settings.builder()
diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2NetworkTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2NetworkTests.java
index 11ba53ca59e94..ece9b206396f8 100644
--- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2NetworkTests.java
+++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2NetworkTests.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.discovery.ec2;
 
-import org.elasticsearch.cloud.aws.network.Ec2NameResolver;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
diff --git a/plugins/discovery-file/build.gradle b/plugins/discovery-file/build.gradle
new file mode 100644
index 0000000000000..91457924351bf
--- /dev/null
+++ b/plugins/discovery-file/build.gradle
@@ -0,0 +1,59 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.elasticsearch.gradle.test.ClusterConfiguration
+import org.elasticsearch.gradle.test.ClusterFormationTasks
+import org.elasticsearch.gradle.test.NodeInfo
+
+esplugin {
+    description 'Discovery file plugin enables unicast discovery from hosts stored in a file.'
+    classname 'org.elasticsearch.discovery.file.FileBasedDiscoveryPlugin'
+}
+
+bundlePlugin {
+    from('config/discovery-file') {
+        into 'config'
+    }
+}
+
+task setupSeedNodeAndUnicastHostsFile(type: DefaultTask) {
+    mustRunAfter(precommit)
+}
+// setup the initial cluster with one node that will serve as the seed node
+// for unicast discovery
+ClusterConfiguration config = new ClusterConfiguration(project)
+config.clusterName = 'discovery-file-test-cluster'
+List<NodeInfo> nodes = ClusterFormationTasks.setup(project, 'initialCluster', setupSeedNodeAndUnicastHostsFile, config)
+File srcUnicastHostsFile = file('build/cluster/unicast_hosts.txt')
+
+// write the unicast_hosts.txt file to a temporary location to be used by the second cluster
+setupSeedNodeAndUnicastHostsFile.doLast {
+    // write the unicast_hosts.txt file to a temp file in the build directory
+    srcUnicastHostsFile.setText(nodes.get(0).transportUri(), 'UTF-8')
+}
+
+// second cluster, which will connect to the first via the unicast_hosts.txt file
+integTestCluster {
+    dependsOn setupSeedNodeAndUnicastHostsFile
+    clusterName = 'discovery-file-test-cluster'
+    extraConfigFile 'discovery-file/unicast_hosts.txt', srcUnicastHostsFile
+}
+
+integTestRunner.finalizedBy ':plugins:discovery-file:initialCluster#stop'
+
diff --git a/plugins/discovery-file/config/discovery-file/unicast_hosts.txt b/plugins/discovery-file/config/discovery-file/unicast_hosts.txt
new file mode 100644
index 0000000000000..5e265e0f295a9
--- /dev/null
+++ b/plugins/discovery-file/config/discovery-file/unicast_hosts.txt
@@ -0,0 +1,20 @@
+# The unicast_hosts.txt file contains the list of unicast hosts to connect to
+# for pinging during the discovery process, when using the file-based discovery
+# mechanism.  This file should contain one entry per line, where an entry is a
+# host/port combination.  The host and port should be separated by a `:`.  If
+# the port is left off, a default port of 9300 is assumed.  For example, if the
+# cluster has three nodes that participate in the discovery process:
+# (1) 66.77.88.99 running on port 9300 (2) 66.77.88.100 running on port 9305
+# and (3) 66.77.88.99 running on port 10005, then this file should contain the
+# following text:
+#
+#10.10.10.5
+#10.10.10.6:9305
+#10.10.10.5:10005
+#
+# For IPv6 addresses, make sure to put a bracket around the host part of the address,
+# for example: [2001:cdba:0000:0000:0000:0000:3257:9652]:9301 (where 9301 is the port).
+#
+# NOTE: all lines starting with a `#` are comments, and comments must exist
+# on lines of their own (i.e. comments cannot begin in the middle of a line)
+#
\ No newline at end of file
diff --git a/plugins/discovery-file/src/main/java/org/elasticsearch/discovery/file/FileBasedDiscoveryPlugin.java b/plugins/discovery-file/src/main/java/org/elasticsearch/discovery/file/FileBasedDiscoveryPlugin.java
new file mode 100644
index 0000000000000..d809fd3fa885e
--- /dev/null
+++ b/plugins/discovery-file/src/main/java/org/elasticsearch/discovery/file/FileBasedDiscoveryPlugin.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.discovery.file;
+
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.network.NetworkService;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.discovery.DiscoveryModule;
+import org.elasticsearch.discovery.zen.UnicastHostsProvider;
+import org.elasticsearch.discovery.zen.UnicastZenPing;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.plugins.DiscoveryPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.watcher.ResourceWatcherService;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+
+/**
+ * Plugin for providing file-based unicast hosts discovery. The list of unicast hosts
+ * is obtained by reading the {@link FileBasedUnicastHostsProvider#UNICAST_HOSTS_FILE} in
+ * the {@link Environment#configFile()}/discovery-file directory.
+ */
+public class FileBasedDiscoveryPlugin extends Plugin implements DiscoveryPlugin {
+
+    private static final Logger logger = Loggers.getLogger(FileBasedDiscoveryPlugin.class);
+    private static final DeprecationLogger deprecationLogger = new DeprecationLogger(logger);
+
+    private final Settings settings;
+    private ExecutorService fileBasedDiscoveryExecutorService;
+
+    public FileBasedDiscoveryPlugin(Settings settings) {
+        this.settings = settings;
+    }
+
+    @Override
+    public Collection<Object> createComponents(
+            Client client,
+            ClusterService clusterService,
+            ThreadPool threadPool,
+            ResourceWatcherService resourceWatcherService,
+            ScriptService scriptService,
+            NamedXContentRegistry xContentRegistry) {
+        final int concurrentConnects = UnicastZenPing.DISCOVERY_ZEN_PING_UNICAST_CONCURRENT_CONNECTS_SETTING.get(settings);
+        final ThreadFactory threadFactory = EsExecutors.daemonThreadFactory(settings, "[file_based_discovery_resolve]");
+        fileBasedDiscoveryExecutorService = EsExecutors.newScaling(
+            "file_based_discovery_resolve",
+            0,
+            concurrentConnects,
+            60,
+            TimeUnit.SECONDS,
+            threadFactory,
+            threadPool.getThreadContext());
+
+        return Collections.emptyList();
+    }
+
+    @Override
+    public void close() throws IOException {
+        ThreadPool.terminate(fileBasedDiscoveryExecutorService, 0, TimeUnit.SECONDS);
+    }
+
+    @Override
+    public Map<String, Supplier<UnicastHostsProvider>> getZenHostsProviders(TransportService transportService,
+                                                                            NetworkService networkService) {
+        return Collections.singletonMap(
+            "file",
+            () -> new FileBasedUnicastHostsProvider(settings, transportService, fileBasedDiscoveryExecutorService));
+    }
+
+    @Override
+    public Settings additionalSettings() {
+        // For 5.0, the hosts provider was "zen", but this was before the discovery.zen.hosts_provider
+        // setting existed. This check looks for the legacy zen, and sets the file hosts provider if not set
+        String discoveryType = DiscoveryModule.DISCOVERY_TYPE_SETTING.get(settings);
+        if (discoveryType.equals("zen")) {
+            deprecationLogger.deprecated("Using " + DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey() +
+                " setting to set hosts provider is deprecated. " +
+                "Set \"" + DiscoveryModule.DISCOVERY_HOSTS_PROVIDER_SETTING.getKey() + ": file\" instead");
+            if (DiscoveryModule.DISCOVERY_HOSTS_PROVIDER_SETTING.exists(settings) == false) {
+                return Settings.builder().put(DiscoveryModule.DISCOVERY_HOSTS_PROVIDER_SETTING.getKey(), "file").build();
+            }
+        }
+        return Settings.EMPTY;
+    }
+}
diff --git a/plugins/discovery-file/src/main/java/org/elasticsearch/discovery/file/FileBasedUnicastHostsProvider.java b/plugins/discovery-file/src/main/java/org/elasticsearch/discovery/file/FileBasedUnicastHostsProvider.java
new file mode 100644
index 0000000000000..196e98d658217
--- /dev/null
+++ b/plugins/discovery-file/src/main/java/org/elasticsearch/discovery/file/FileBasedUnicastHostsProvider.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.discovery.file;
+
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.apache.logging.log4j.util.Supplier;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.component.AbstractComponent;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.discovery.zen.UnicastHostsProvider;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.transport.TransportService;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.elasticsearch.discovery.zen.UnicastZenPing.DISCOVERY_ZEN_PING_UNICAST_HOSTS_RESOLVE_TIMEOUT;
+import static org.elasticsearch.discovery.zen.UnicastZenPing.resolveHostsLists;
+
+/**
+ * An implementation of {@link UnicastHostsProvider} that reads hosts/ports
+ * from {@link #UNICAST_HOSTS_FILE}.
+ *
+ * Each unicast host/port that is part of the discovery process must be listed on
+ * a separate line.  If the port is left off an entry, a default port of 9300 is
+ * assumed.  An example unicast hosts file could read:
+ *
+ * 67.81.244.10
+ * 67.81.244.11:9305
+ * 67.81.244.15:9400
+ */
+class FileBasedUnicastHostsProvider extends AbstractComponent implements UnicastHostsProvider {
+
+    static final String UNICAST_HOSTS_FILE = "unicast_hosts.txt";
+    static final String UNICAST_HOST_PREFIX = "#zen_file_unicast_host_";
+
+    private final TransportService transportService;
+    private final ExecutorService executorService;
+
+    private final Path unicastHostsFilePath;
+
+    private final AtomicLong nodeIdGenerator = new AtomicLong(); // generates unique ids for the node
+
+    private final TimeValue resolveTimeout;
+
+    FileBasedUnicastHostsProvider(Settings settings, TransportService transportService, ExecutorService executorService) {
+        super(settings);
+        this.transportService = transportService;
+        this.executorService = executorService;
+        this.unicastHostsFilePath = new Environment(settings).configFile().resolve("discovery-file").resolve(UNICAST_HOSTS_FILE);
+        this.resolveTimeout = DISCOVERY_ZEN_PING_UNICAST_HOSTS_RESOLVE_TIMEOUT.get(settings);
+    }
+
+    @Override
+    public List<DiscoveryNode> buildDynamicNodes() {
+        List<String> hostsList;
+        try (Stream<String> lines = Files.lines(unicastHostsFilePath)) {
+            hostsList = lines.filter(line -> line.startsWith("#") == false) // lines starting with `#` are comments
+                             .collect(Collectors.toList());
+        } catch (FileNotFoundException | NoSuchFileException e) {
+            logger.warn((Supplier<?>) () -> new ParameterizedMessage("[discovery-file] Failed to find unicast hosts file [{}]",
+                                                                        unicastHostsFilePath), e);
+            hostsList = Collections.emptyList();
+        } catch (IOException e) {
+            logger.warn((Supplier<?>) () -> new ParameterizedMessage("[discovery-file] Error reading unicast hosts file [{}]",
+                                                                        unicastHostsFilePath), e);
+            hostsList = Collections.emptyList();
+        }
+
+        final List<DiscoveryNode> discoNodes = new ArrayList<>();
+        try {
+            discoNodes.addAll(resolveHostsLists(
+                executorService,
+                logger,
+                hostsList,
+                1,
+                transportService,
+                UNICAST_HOST_PREFIX,
+                resolveTimeout));
+        } catch (InterruptedException e) {
+            throw new RuntimeException(e);
+        }
+
+        logger.debug("[discovery-file] Using dynamic discovery nodes {}", discoNodes);
+
+        return discoNodes;
+    }
+
+}
diff --git a/plugins/discovery-file/src/test/java/org/elasticsearch/discovery/file/FileBasedDiscoveryClientYamlTestSuiteIT.java b/plugins/discovery-file/src/test/java/org/elasticsearch/discovery/file/FileBasedDiscoveryClientYamlTestSuiteIT.java
new file mode 100644
index 0000000000000..d2ac2095bdfc9
--- /dev/null
+++ b/plugins/discovery-file/src/test/java/org/elasticsearch/discovery/file/FileBasedDiscoveryClientYamlTestSuiteIT.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.discovery.file;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
+import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+
+/**
+ * Integration tests to make sure the file-based discovery plugin works in a cluster.
+ */
+public class FileBasedDiscoveryClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
+
+    public FileBasedDiscoveryClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
+    }
+}
diff --git a/plugins/discovery-file/src/test/java/org/elasticsearch/discovery/file/FileBasedDiscoveryPluginTests.java b/plugins/discovery-file/src/test/java/org/elasticsearch/discovery/file/FileBasedDiscoveryPluginTests.java
new file mode 100644
index 0000000000000..7a7ee9dbd037e
--- /dev/null
+++ b/plugins/discovery-file/src/test/java/org/elasticsearch/discovery/file/FileBasedDiscoveryPluginTests.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.discovery.file;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.discovery.DiscoveryModule;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+public class FileBasedDiscoveryPluginTests extends ESTestCase {
+
+    public void testHostsProviderBwc() {
+        FileBasedDiscoveryPlugin plugin = new FileBasedDiscoveryPlugin(Settings.EMPTY);
+        Settings additionalSettings = plugin.additionalSettings();
+        assertEquals("file", additionalSettings.get(DiscoveryModule.DISCOVERY_HOSTS_PROVIDER_SETTING.getKey()));
+        assertWarnings("Using discovery.type setting to set hosts provider is deprecated. " +
+                "Set \"discovery.zen.hosts_provider: file\" instead");
+    }
+
+    public void testHostsProviderExplicit() {
+        Settings settings = Settings.builder().put(DiscoveryModule.DISCOVERY_HOSTS_PROVIDER_SETTING.getKey(), "foo").build();
+        FileBasedDiscoveryPlugin plugin = new FileBasedDiscoveryPlugin(settings);
+        assertEquals(Settings.EMPTY, plugin.additionalSettings());
+        assertWarnings("Using discovery.type setting to set hosts provider is deprecated. " +
+                "Set \"discovery.zen.hosts_provider: file\" instead");
+    }
+}
diff --git a/plugins/discovery-file/src/test/java/org/elasticsearch/discovery/file/FileBasedUnicastHostsProviderTests.java b/plugins/discovery-file/src/test/java/org/elasticsearch/discovery/file/FileBasedUnicastHostsProviderTests.java
new file mode 100644
index 0000000000000..dba8b8cf6ea46
--- /dev/null
+++ b/plugins/discovery-file/src/test/java/org/elasticsearch/discovery/file/FileBasedUnicastHostsProviderTests.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.discovery.file;
+
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.network.NetworkService;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.BoundTransportAddress;
+import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.transport.MockTransportService;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.MockTcpTransport;
+import org.elasticsearch.transport.TransportService;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import static org.elasticsearch.discovery.file.FileBasedUnicastHostsProvider.UNICAST_HOSTS_FILE;
+import static org.elasticsearch.discovery.file.FileBasedUnicastHostsProvider.UNICAST_HOST_PREFIX;
+
+/**
+ * Tests for {@link FileBasedUnicastHostsProvider}.
+ */
+public class FileBasedUnicastHostsProviderTests extends ESTestCase {
+
+    private ThreadPool threadPool;
+    private ExecutorService executorService;
+    private MockTransportService transportService;
+
+    @Before
+    public void setUp() throws Exception {
+        super.setUp();
+        threadPool = new TestThreadPool(FileBasedUnicastHostsProviderTests.class.getName());
+        executorService = Executors.newSingleThreadExecutor();
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        try {
+            terminate(executorService);
+        } finally {
+            try {
+                terminate(threadPool);
+            } finally {
+                super.tearDown();
+            }
+        }
+    }
+
+    @Before
+    public void createTransportSvc() {
+        MockTcpTransport transport =
+            new MockTcpTransport(Settings.EMPTY,
+                                    threadPool,
+                                    BigArrays.NON_RECYCLING_INSTANCE,
+                                    new NoneCircuitBreakerService(),
+                                    new NamedWriteableRegistry(Collections.emptyList()),
+                                    new NetworkService(Settings.EMPTY, Collections.emptyList())) {
+                @Override
+                public BoundTransportAddress boundAddress() {
+                    return new BoundTransportAddress(
+                        new TransportAddress[]{new InetSocketTransportAddress(InetAddress.getLoopbackAddress(), 9300)},
+                        new InetSocketTransportAddress(InetAddress.getLoopbackAddress(), 9300)
+                    );
+                }
+            };
+        transportService = new MockTransportService(Settings.EMPTY, transport, threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR,
+                null);
+    }
+
+    public void testBuildDynamicNodes() throws Exception {
+        final List<String> hostEntries = Arrays.asList("#comment, should be ignored", "192.168.0.1", "192.168.0.2:9305", "255.255.23.15");
+        final List<DiscoveryNode> nodes = setupAndRunHostProvider(hostEntries);
+        assertEquals(hostEntries.size() - 1, nodes.size()); // minus 1 because we are ignoring the first line that's a comment
+        assertEquals("192.168.0.1", nodes.get(0).getAddress().getHost());
+        assertEquals(9300, nodes.get(0).getAddress().getPort());
+        assertEquals(UNICAST_HOST_PREFIX + "192.168.0.1_0#", nodes.get(0).getId());
+        assertEquals("192.168.0.2", nodes.get(1).getAddress().getAddress());
+        assertEquals(9305, nodes.get(1).getAddress().getPort());
+        assertEquals(UNICAST_HOST_PREFIX + "192.168.0.2:9305_0#", nodes.get(1).getId());
+        assertEquals("255.255.23.15", nodes.get(2).getAddress().getAddress());
+        assertEquals(9300, nodes.get(2).getAddress().getPort());
+        assertEquals(UNICAST_HOST_PREFIX + "255.255.23.15_0#", nodes.get(2).getId());
+    }
+
+    public void testEmptyUnicastHostsFile() throws Exception {
+        final List<String> hostEntries = Collections.emptyList();
+        final List<DiscoveryNode> nodes = setupAndRunHostProvider(hostEntries);
+        assertEquals(0, nodes.size());
+    }
+
+    public void testUnicastHostsDoesNotExist() throws Exception {
+        final Settings settings = Settings.builder()
+                                      .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
+                                      .build();
+        final FileBasedUnicastHostsProvider provider = new FileBasedUnicastHostsProvider(settings, transportService, executorService);
+        final List<DiscoveryNode> nodes = provider.buildDynamicNodes();
+        assertEquals(0, nodes.size());
+    }
+
+    public void testInvalidHostEntries() throws Exception {
+        List<String> hostEntries = Arrays.asList("192.168.0.1:9300:9300");
+        List<DiscoveryNode> nodes = setupAndRunHostProvider(hostEntries);
+        assertEquals(0, nodes.size());
+    }
+
+    public void testSomeInvalidHostEntries() throws Exception {
+        List<String> hostEntries = Arrays.asList("192.168.0.1:9300:9300", "192.168.0.1:9301");
+        List<DiscoveryNode> nodes = setupAndRunHostProvider(hostEntries);
+        assertEquals(1, nodes.size()); // only one of the two is valid and will be used
+        assertEquals("192.168.0.1", nodes.get(0).getAddress().getHost());
+        assertEquals(9301, nodes.get(0).getAddress().getPort());
+    }
+
+    // sets up the config dir, writes to the unicast hosts file in the config dir,
+    // and then runs the file-based unicast host provider to get the list of discovery nodes
+    private List<DiscoveryNode> setupAndRunHostProvider(final List<String> hostEntries) throws IOException {
+        final Path homeDir = createTempDir();
+        final Settings settings = Settings.builder()
+                                      .put(Environment.PATH_HOME_SETTING.getKey(), homeDir)
+                                      .build();
+        final Path configDir = homeDir.resolve("config").resolve("discovery-file");
+        Files.createDirectories(configDir);
+        final Path unicastHostsPath = configDir.resolve(UNICAST_HOSTS_FILE);
+        try (BufferedWriter writer = Files.newBufferedWriter(unicastHostsPath)) {
+            writer.write(String.join("\n", hostEntries));
+        }
+
+        return new FileBasedUnicastHostsProvider(settings, transportService, executorService).buildDynamicNodes();
+    }
+}
diff --git a/plugins/discovery-file/src/test/resources/rest-api-spec/test/discovery_file/10_basic.yaml b/plugins/discovery-file/src/test/resources/rest-api-spec/test/discovery_file/10_basic.yaml
new file mode 100644
index 0000000000000..138115da1123f
--- /dev/null
+++ b/plugins/discovery-file/src/test/resources/rest-api-spec/test/discovery_file/10_basic.yaml
@@ -0,0 +1,13 @@
+# Integration tests for file-based discovery
+#
+"Ensure cluster formed successfully with discovery file":
+    # make sure both nodes joined the cluster
+    - do:
+        cluster.health:
+            wait_for_nodes: 2
+
+    # make sure the cluster was formed with the correct name
+    - do:
+        cluster.state: {}
+
+    - match: { cluster_name: 'discovery-file-test-cluster' } # correct cluster name, we formed the cluster we expected to
diff --git a/plugins/discovery-gce/build.gradle b/plugins/discovery-gce/build.gradle
index bbd2221d8e0b7..ede168e1f9ded 100644
--- a/plugins/discovery-gce/build.gradle
+++ b/plugins/discovery-gce/build.gradle
@@ -35,7 +35,7 @@ task createKey(type: LoggedExec) {
     project.delete(keystore.parentFile)
     keystore.parentFile.mkdirs()
   }
-  executable = 'keytool'
+  executable = new File(project.javaHome, 'bin/keytool')
   standardInput = new ByteArrayInputStream('FirstName LastName\nUnit\nOrganization\nCity\nState\nNL\nyes\n\n'.getBytes('UTF-8'))
   args '-genkey',
           '-alias', 'test-node',
diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceInstancesService.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceInstancesService.java
index 39db86c672ad9..6f7313051fea2 100644
--- a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceInstancesService.java
+++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceInstancesService.java
@@ -25,12 +25,13 @@
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.unit.TimeValue;
 
+import java.io.Closeable;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.function.Function;
 
-public interface GceInstancesService extends LifecycleComponent {
+public interface GceInstancesService {
 
     /**
      * GCE API Version: Elasticsearch/GceCloud/1.0
diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceInstancesServiceImpl.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceInstancesServiceImpl.java
index c6c7b9a0aefc4..bda68780c7974 100644
--- a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceInstancesServiceImpl.java
+++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceInstancesServiceImpl.java
@@ -19,6 +19,18 @@
 
 package org.elasticsearch.cloud.gce;
 
+import java.io.Closeable;
+import java.io.IOException;
+import java.security.AccessController;
+import java.security.GeneralSecurityException;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.function.Function;
+
 import com.google.api.client.googleapis.compute.ComputeCredential;
 import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
 import com.google.api.client.http.HttpTransport;
@@ -30,28 +42,15 @@
 import com.google.api.services.compute.model.InstanceList;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
-import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.SpecialPermission;
-import org.elasticsearch.common.component.AbstractLifecycleComponent;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.discovery.gce.RetryHttpInitializerWrapper;
 
-import java.io.IOException;
-import java.security.AccessController;
-import java.security.GeneralSecurityException;
-import java.security.PrivilegedActionException;
-import java.security.PrivilegedExceptionAction;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.function.Function;
-
-public class GceInstancesServiceImpl extends AbstractLifecycleComponent implements GceInstancesService {
+public class GceInstancesServiceImpl extends AbstractComponent implements GceInstancesService, Closeable {
 
     // all settings just used for testing - not registered by default
     public static final Setting<Boolean> GCE_VALIDATE_CERTIFICATES =
@@ -113,7 +112,6 @@ public InstanceList run() throws Exception {
 
     private final boolean validateCerts;
 
-    @Inject
     public GceInstancesServiceImpl(Settings settings) {
         super(settings);
         this.project = PROJECT_SETTING.get(settings);
@@ -204,22 +202,9 @@ public Void run() throws IOException {
     }
 
     @Override
-    protected void doStart() throws ElasticsearchException {
-    }
-
-    @Override
-    protected void doStop() throws ElasticsearchException {
+    public void close() throws IOException {
         if (gceHttpTransport != null) {
-            try {
-                gceHttpTransport.shutdown();
-            } catch (IOException e) {
-                logger.warn("unable to shutdown GCE Http Transport", e);
-            }
-            gceHttpTransport = null;
+            gceHttpTransport.shutdown();
         }
     }
-
-    @Override
-    protected void doClose() throws ElasticsearchException {
-    }
 }
diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceMetadataService.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceMetadataService.java
index 25b7cf72c924c..5145dc2a51c2e 100644
--- a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceMetadataService.java
+++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceMetadataService.java
@@ -19,28 +19,25 @@
 
 package org.elasticsearch.cloud.gce;
 
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.security.AccessController;
+import java.security.GeneralSecurityException;
+import java.security.PrivilegedAction;
+import java.security.PrivilegedExceptionAction;
+import java.util.function.Function;
+
 import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
 import com.google.api.client.http.GenericUrl;
 import com.google.api.client.http.HttpHeaders;
 import com.google.api.client.http.HttpResponse;
 import com.google.api.client.http.HttpTransport;
 import org.elasticsearch.SpecialPermission;
-import org.elasticsearch.cloud.gce.network.GceNameResolver;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.security.AccessController;
-import java.security.GeneralSecurityException;
-import java.security.PrivilegedAction;
-import java.security.PrivilegedExceptionAction;
-import java.util.function.Function;
-
 public class GceMetadataService extends AbstractLifecycleComponent {
 
     // Forcing Google Token API URL as set in GCE SDK to
@@ -53,7 +50,6 @@ public class GceMetadataService extends AbstractLifecycleComponent {
     /** Global instance of the HTTP transport. */
     private HttpTransport gceHttpTransport;
 
-    @Inject
     public GceMetadataService(Settings settings) {
         super(settings);
     }
diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java
index 3426e74d4a409..7661b96416516 100644
--- a/plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java
+++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java
@@ -19,6 +19,13 @@
 
 package org.elasticsearch.discovery.gce;
 
+import java.io.IOException;
+import java.net.InetAddress;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.function.Function;
+
 import com.google.api.services.compute.model.AccessConfig;
 import com.google.api.services.compute.model.Instance;
 import com.google.api.services.compute.model.NetworkInterface;
@@ -29,7 +36,6 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Setting;
@@ -37,16 +43,9 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.discovery.zen.ping.unicast.UnicastHostsProvider;
+import org.elasticsearch.discovery.zen.UnicastHostsProvider;
 import org.elasticsearch.transport.TransportService;
 
-import java.io.IOException;
-import java.net.InetAddress;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.function.Function;
-
 import static java.util.Collections.emptyList;
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
@@ -78,7 +77,6 @@ static final class Status {
     private long lastRefresh;
     private List<DiscoveryNode> cachedDiscoNodes;
 
-    @Inject
     public GceUnicastHostsProvider(Settings settings, GceInstancesService gceInstancesService,
             TransportService transportService,
             NetworkService networkService) {
diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/plugin/discovery/gce/GceDiscoveryPlugin.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/plugin/discovery/gce/GceDiscoveryPlugin.java
index aeec99118245a..41ba2f76a9be2 100644
--- a/plugins/discovery-gce/src/main/java/org/elasticsearch/plugin/discovery/gce/GceDiscoveryPlugin.java
+++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/plugin/discovery/gce/GceDiscoveryPlugin.java
@@ -22,23 +22,36 @@
 import com.google.api.client.http.HttpHeaders;
 import com.google.api.client.util.ClassInfo;
 import org.apache.logging.log4j.Logger;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.cloud.gce.GceInstancesService;
+import org.elasticsearch.cloud.gce.GceInstancesServiceImpl;
 import org.elasticsearch.cloud.gce.GceMetadataService;
 import org.elasticsearch.cloud.gce.GceModule;
 import org.elasticsearch.cloud.gce.network.GceNameResolver;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.component.LifecycleComponent;
 import org.elasticsearch.common.inject.Module;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.discovery.Discovery;
 import org.elasticsearch.discovery.DiscoveryModule;
 import org.elasticsearch.discovery.gce.GceUnicastHostsProvider;
+import org.elasticsearch.discovery.zen.UnicastHostsProvider;
 import org.elasticsearch.discovery.zen.ZenDiscovery;
+import org.elasticsearch.discovery.zen.ZenPing;
 import org.elasticsearch.plugins.DiscoveryPlugin;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
 
+import java.io.Closeable;
+import java.io.IOException;
 import java.security.AccessController;
 import java.security.PrivilegedAction;
 import java.util.ArrayList;
@@ -46,12 +59,17 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
+import java.util.function.Supplier;
 
-public class GceDiscoveryPlugin extends Plugin implements DiscoveryPlugin {
+public class GceDiscoveryPlugin extends Plugin implements DiscoveryPlugin, Closeable {
 
     public static final String GCE = "gce";
     private final Settings settings;
-    protected final Logger logger = Loggers.getLogger(GceDiscoveryPlugin.class);
+    private static final Logger logger = Loggers.getLogger(GceDiscoveryPlugin.class);
+    private static final DeprecationLogger deprecationLogger = new DeprecationLogger(logger);
+    // stashed when created in order to properly close
+    private final SetOnce<GceInstancesServiceImpl> gceInstancesService = new SetOnce<>();
 
     static {
         /*
@@ -81,23 +99,21 @@ public GceDiscoveryPlugin(Settings settings) {
     }
 
     @Override
-    public Collection<Module> createGuiceModules() {
-        return Collections.singletonList(new GceModule(settings));
+    public Map<String, Supplier<Discovery>> getDiscoveryTypes(ThreadPool threadPool, TransportService transportService,
+                                                              NamedWriteableRegistry namedWriteableRegistry,
+                                                              ClusterService clusterService, UnicastHostsProvider hostsProvider) {
+        // this is for backcompat with pre 5.1, where users would set discovery.type to use ec2 hosts provider
+        return Collections.singletonMap(GCE, () ->
+            new ZenDiscovery(settings, threadPool, transportService, namedWriteableRegistry, clusterService, hostsProvider));
     }
 
     @Override
-    @SuppressWarnings("rawtypes") // Supertype uses raw type
-    public Collection<Class<? extends LifecycleComponent>> getGuiceServiceClasses() {
-        logger.debug("Register gce compute service");
-        Collection<Class<? extends LifecycleComponent>> services = new ArrayList<>();
-        services.add(GceModule.getComputeServiceImpl());
-        return services;
-    }
-
-    public void onModule(DiscoveryModule discoveryModule) {
-        logger.debug("Register gce discovery type and gce unicast provider");
-        discoveryModule.addDiscoveryType(GCE, ZenDiscovery.class);
-        discoveryModule.addUnicastHostProvider(GCE, GceUnicastHostsProvider.class);
+    public Map<String, Supplier<UnicastHostsProvider>> getZenHostsProviders(TransportService transportService,
+                                                                            NetworkService networkService) {
+        return Collections.singletonMap(GCE, () -> {
+            gceInstancesService.set(new GceInstancesServiceImpl(settings));
+            return new GceUnicastHostsProvider(settings, gceInstancesService.get(), transportService, networkService);
+        });
     }
 
     @Override
@@ -117,4 +133,25 @@ public List<Setting<?>> getSettings() {
             GceInstancesService.RETRY_SETTING,
             GceInstancesService.MAX_WAIT_SETTING);
     }
+
+    @Override
+    public Settings additionalSettings() {
+        // For 5.0, the hosts provider was "zen", but this was before the discovery.zen.hosts_provider
+        // setting existed. This check looks for the legacy setting, and sets hosts provider if set
+        String discoveryType = DiscoveryModule.DISCOVERY_TYPE_SETTING.get(settings);
+        if (discoveryType.equals(GCE)) {
+            deprecationLogger.deprecated("Using " + DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey() +
+                " setting to set hosts provider is deprecated. " +
+                "Set \"" + DiscoveryModule.DISCOVERY_HOSTS_PROVIDER_SETTING.getKey() + ": " + GCE + "\" instead");
+            if (DiscoveryModule.DISCOVERY_HOSTS_PROVIDER_SETTING.exists(settings) == false) {
+                return Settings.builder().put(DiscoveryModule.DISCOVERY_HOSTS_PROVIDER_SETTING.getKey(), GCE).build();
+            }
+        }
+        return Settings.EMPTY;
+    }
+
+    @Override
+    public void close() throws IOException {
+        IOUtils.close(gceInstancesService.get());
+    }
 }
diff --git a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/DiscoveryGceClientYamlTestSuiteIT.java b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/DiscoveryGceClientYamlTestSuiteIT.java
index 8ce17ff9fa530..fe1e0cf5d77b0 100644
--- a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/DiscoveryGceClientYamlTestSuiteIT.java
+++ b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/DiscoveryGceClientYamlTestSuiteIT.java
@@ -24,9 +24,6 @@
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-
-import java.io.IOException;
 
 public class DiscoveryGceClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
@@ -35,8 +32,8 @@ public DiscoveryGceClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate t
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
-        return ESClientYamlSuiteTestCase.createParameters(0, 1);
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
     }
 }
 
diff --git a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoverTests.java b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoverTests.java
index 98f6fd0dc1b5e..26ebc5f6cc9cc 100644
--- a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoverTests.java
+++ b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoverTests.java
@@ -40,7 +40,6 @@
 import javax.net.ssl.KeyManagerFactory;
 import javax.net.ssl.SSLContext;
 import javax.net.ssl.TrustManagerFactory;
-
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -97,7 +96,6 @@ protected Settings nodeSettings(int nodeOrdinal) {
             .put("node.portsfile", "true")
             .put("cloud.gce.project_id", "testproject")
             .put("cloud.gce.zone", "primaryzone")
-            .put("discovery.initial_state_timeout", "1s")
             .put("cloud.gce.host", "http://" + httpServer.getAddress().getHostName() + ":" + httpServer.getAddress().getPort())
             .put("cloud.gce.root_url", "https://" + httpsServer.getAddress().getHostName() +
                 ":" + httpsServer.getAddress().getPort())
@@ -201,7 +199,7 @@ public void testJoin() throws ExecutionException, InterruptedException {
         // only wait for the cluster to form
         assertNoTimeout(client().admin().cluster().prepareHealth().setWaitForNodes(Integer.toString(2)).get());
         // add one more node and wait for it to join
-        internalCluster().startDataOnlyNodeAsync().get();
+        internalCluster().startDataOnlyNode();
         assertNoTimeout(client().admin().cluster().prepareHealth().setWaitForNodes(Integer.toString(3)).get());
     }
 }
diff --git a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoveryTests.java b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoveryTests.java
index 08e7ee963d375..9796276246147 100644
--- a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoveryTests.java
+++ b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoveryTests.java
@@ -33,6 +33,7 @@
 import org.junit.Before;
 import org.junit.BeforeClass;
 
+import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
@@ -94,13 +95,13 @@ public void setProjectName() {
 
     @Before
     public void createTransportService() {
-        transportService = MockTransportService.local(Settings.EMPTY, Version.CURRENT, threadPool);
+        transportService = MockTransportService.local(Settings.EMPTY, Version.CURRENT, threadPool, null);
     }
 
     @After
-    public void stopGceComputeService() {
+    public void stopGceComputeService() throws IOException {
         if (mock != null) {
-            mock.stop();
+            mock.close();
         }
     }
 
diff --git a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceMockUtils.java b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceMockUtils.java
index 01009554535d0..aea5acbc178a9 100644
--- a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceMockUtils.java
+++ b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceMockUtils.java
@@ -30,7 +30,6 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.logging.Loggers;
-import org.elasticsearch.common.util.Callback;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -84,12 +83,7 @@ private static String readJsonResponse(String url, String urlRoot) throws IOExce
         }
         try (InputStream is = resource.openStream()) {
             final StringBuilder sb = new StringBuilder();
-            Streams.readAllLines(is, new Callback<String>() {
-                @Override
-                public void handle(String s) {
-                    sb.append(s);
-                }
-            });
+            Streams.readAllLines(is, sb::append);
             String response = sb.toString();
             return response;
         }
diff --git a/plugins/examples/build.gradle b/plugins/examples/build.gradle
new file mode 100644
index 0000000000000..47db55b3b33ca
--- /dev/null
+++ b/plugins/examples/build.gradle
@@ -0,0 +1,10 @@
+// Subprojects aren't published so do not assemble
+gradle.projectsEvaluated {
+  subprojects {
+    Task assemble = project.tasks.findByName('assemble')
+    if (assemble) {
+      project.tasks.remove(assemble)
+      project.build.dependsOn.remove('assemble')
+    }
+  }
+}
diff --git a/plugins/examples/script-expert-scoring/build.gradle b/plugins/examples/script-expert-scoring/build.gradle
new file mode 100644
index 0000000000000..7c602d9bc027d
--- /dev/null
+++ b/plugins/examples/script-expert-scoring/build.gradle
@@ -0,0 +1,28 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+apply plugin: 'elasticsearch.esplugin'
+
+esplugin {
+  name 'script-expert-scoring'
+  description 'An example script engine to use low level Lucene internals for expert scoring'
+  classname 'org.elasticsearch.example.expertscript.ExpertScriptPlugin'
+}
+
+test.enabled = false
diff --git a/plugins/examples/script-expert-scoring/src/main/java/org/elasticsearch/example/expertscript/ExpertScriptPlugin.java b/plugins/examples/script-expert-scoring/src/main/java/org/elasticsearch/example/expertscript/ExpertScriptPlugin.java
new file mode 100644
index 0000000000000..c0187684bd474
--- /dev/null
+++ b/plugins/examples/script-expert-scoring/src/main/java/org/elasticsearch/example/expertscript/ExpertScriptPlugin.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.example.expertscript;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Map;
+import java.util.function.Function;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.PostingsEnum;
+import org.apache.lucene.index.Term;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.ScriptPlugin;
+import org.elasticsearch.script.CompiledScript;
+import org.elasticsearch.script.ExecutableScript;
+import org.elasticsearch.script.LeafSearchScript;
+import org.elasticsearch.script.ScriptEngineService;
+import org.elasticsearch.script.SearchScript;
+import org.elasticsearch.search.lookup.SearchLookup;
+
+/**
+ * An example script plugin that adds a {@link ScriptEngineService} implementing expert scoring.
+ */
+public class ExpertScriptPlugin extends Plugin implements ScriptPlugin {
+
+    @Override
+    public ScriptEngineService getScriptEngineService(Settings settings) {
+        return new MyExpertScriptEngine();
+    }
+
+    /** An example {@link ScriptEngineService} that uses Lucene segment details to implement pure document frequency scoring. */
+    // tag::expert_engine
+    private static class MyExpertScriptEngine implements ScriptEngineService {
+        @Override
+        public String getType() {
+            return "expert_scripts";
+        }
+
+        @Override
+        public Function<Map<String,Object>,SearchScript> compile(String scriptName, String scriptSource, Map<String, String> params) {
+            // we use the script "source" as the script identifier
+            if ("pure_df".equals(scriptSource)) {
+                return p -> new SearchScript() {
+                    final String field;
+                    final String term;
+                    {
+                        if (p.containsKey("field") == false) {
+                            throw new IllegalArgumentException("Missing parameter [field]");
+                        }
+                        if (p.containsKey("term") == false) {
+                            throw new IllegalArgumentException("Missing parameter [term]");
+                        }
+                        field = p.get("field").toString();
+                        term = p.get("term").toString();
+                    }
+
+                    @Override
+                    public LeafSearchScript getLeafSearchScript(LeafReaderContext context) throws IOException {
+                        PostingsEnum postings = context.reader().postings(new Term(field, term));
+                        if (postings == null) {
+                            // the field and/or term don't exist in this segment, so always return 0
+                            return () -> 0.0d;
+                        }
+                        return new LeafSearchScript() {
+                            int currentDocid = -1;
+                            @Override
+                            public void setDocument(int docid) {
+                                // advance has undefined behavior calling with a docid <= its current docid
+                                if (postings.docID() < docid) {
+                                    try {
+                                        postings.advance(docid);
+                                    } catch (IOException e) {
+                                        throw new UncheckedIOException(e);
+                                    }
+                                }
+                                currentDocid = docid;
+                            }
+                            @Override
+                            public double runAsDouble() {
+                                if (postings.docID() != currentDocid) {
+                                    // advance moved past the current doc, so this doc has no occurrences of the term
+                                    return 0.0d;
+                                }
+                                try {
+                                    return postings.freq();
+                                } catch (IOException e) {
+                                    throw new UncheckedIOException(e);
+                                }
+                            }
+                        };
+                    }
+
+                    @Override
+                    public boolean needsScores() {
+                        return false;
+                    }
+                };
+            }
+            throw new IllegalArgumentException("Unknown script name " + scriptSource);
+        }
+
+        @Override
+        @SuppressWarnings("unchecked")
+        public SearchScript search(CompiledScript compiledScript, SearchLookup lookup, @Nullable Map<String, Object> params) {
+          Function<Map<String,Object>,SearchScript> scriptFactory = (Function<Map<String,Object>,SearchScript>) compiledScript.compiled();
+          return scriptFactory.apply(params);
+        }
+
+        @Override
+        public ExecutableScript executable(CompiledScript compiledScript, @Nullable Map<String, Object> params) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public boolean isInlineScriptEnabled() {
+            return true;
+        }
+
+        @Override
+        public void close() {}
+    }
+    // end::expert_engine
+}
diff --git a/plugins/examples/script-expert-scoring/src/test/java/org/elasticsearch/example/expertscript/ExpertScriptClientYamlTestSuiteIT.java b/plugins/examples/script-expert-scoring/src/test/java/org/elasticsearch/example/expertscript/ExpertScriptClientYamlTestSuiteIT.java
new file mode 100644
index 0000000000000..977ca9cf843a9
--- /dev/null
+++ b/plugins/examples/script-expert-scoring/src/test/java/org/elasticsearch/example/expertscript/ExpertScriptClientYamlTestSuiteIT.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.example.expertscript;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
+import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+
+public class ExpertScriptClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
+
+    public ExpertScriptClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
+    }
+}
+
diff --git a/plugins/examples/script-expert-scoring/src/test/resources/rest-api-spec/test/script_expert_scoring/10_basic.yaml b/plugins/examples/script-expert-scoring/src/test/resources/rest-api-spec/test/script_expert_scoring/10_basic.yaml
new file mode 100644
index 0000000000000..b4fafd69dd4ab
--- /dev/null
+++ b/plugins/examples/script-expert-scoring/src/test/resources/rest-api-spec/test/script_expert_scoring/10_basic.yaml
@@ -0,0 +1,13 @@
+# Integration tests for the expert scoring script example plugin
+#
+"Plugin loaded":
+    - do:
+        cluster.state: {}
+
+    # Get master node id
+    - set: { master_node: master }
+
+    - do:
+        nodes.info: {}
+
+    - match:  { nodes.$master.plugins.0.name: script-expert-scoring  }
diff --git a/plugins/examples/script-expert-scoring/src/test/resources/rest-api-spec/test/script_expert_scoring/20_score.yaml b/plugins/examples/script-expert-scoring/src/test/resources/rest-api-spec/test/script_expert_scoring/20_score.yaml
new file mode 100644
index 0000000000000..9347cb343a9ee
--- /dev/null
+++ b/plugins/examples/script-expert-scoring/src/test/resources/rest-api-spec/test/script_expert_scoring/20_score.yaml
@@ -0,0 +1,53 @@
+# Integration tests for the expert scoring script example plugin
+#
+---
+setup:
+  - do:
+      indices.create:
+          index:  test
+
+  - do:
+      index:
+        index:  test
+        type:   test
+        id:     1
+        body:   { "important_field": "foo" }
+  - do:
+        index:
+          index:  test
+          type:   test
+          id:     2
+          body:   { "important_field": "foo foo foo" }
+  - do:
+        index:
+          index:  test
+          type:   test
+          id:     3
+          body:   { "important_field": "foo foo" }
+
+  - do:
+      indices.refresh: {}
+---
+"document scoring":
+  - do:
+      search:
+        index: test
+        body:
+          query:
+            function_score:
+              query:
+                match:
+                  important_field: "foo"
+              functions:
+                - script_score:
+                    script:
+                      source: "pure_df"
+                      lang: "expert_scripts"
+                      params:
+                        field: "important_field"
+                        term: "foo"
+
+  - length: { hits.hits: 3 }
+  - match: {hits.hits.0._id: "2" }
+  - match: {hits.hits.1._id: "3" }
+  - match: {hits.hits.2._id: "1" }
diff --git a/plugins/ingest-attachment/build.gradle b/plugins/ingest-attachment/build.gradle
index 3d9545d7aed04..bdc01c0151e9e 100644
--- a/plugins/ingest-attachment/build.gradle
+++ b/plugins/ingest-attachment/build.gradle
@@ -23,10 +23,11 @@ esplugin {
 }
 
 versions << [
-  'tika': '1.13',
-  'pdfbox': '2.0.1',
-  'bouncycastle': '1.54',
-  'poi': '3.15-beta1'
+  'tika': '1.14',
+  'pdfbox': '2.0.3',
+  'bouncycastle': '1.55',
+  'poi': '3.15',
+  'mime4j': '0.7.2'
 ]
 
 dependencies {
@@ -59,15 +60,25 @@ dependencies {
   compile "org.apache.poi:poi-scratchpad:${versions.poi}"
   // Apple iWork
   compile 'org.apache.commons:commons-compress:1.10'
+  // Outlook documents
+  compile "org.apache.james:apache-mime4j-core:${versions.mime4j}"
+  compile "org.apache.james:apache-mime4j-dom:${versions.mime4j}"
 }
 
 // TODO: stop using LanguageIdentifier...
 compileJava.options.compilerArgs << "-Xlint:-deprecation"
 
+
+dependencyLicenses {
+  mapping from: /apache-mime4j-.*/, to: 'apache-mime4j'
+}
+
 forbiddenPatterns {
+  exclude '**/*.doc'
   exclude '**/*.docx'
   exclude '**/*.pdf'
   exclude '**/*.epub'
+  exclude '**/*.vsdx'
 }
 
 thirdPartyAudit.excludes = [
@@ -216,7 +227,6 @@ thirdPartyAudit.excludes = [
   'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1VTEditList',
   'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1ValList',
   'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1ValidIdsList',
-  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1VisibleList',
   'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1WidthMinList',
   'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2Accel2List',
   'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2AccelList',
@@ -506,6 +516,8 @@ thirdPartyAudit.excludes = [
   'opennlp.tools.namefind.TokenNameFinderModel',
   'opennlp.tools.util.Span',
   'org.apache.avalon.framework.logger.Logger',
+  'org.apache.commons.collections4.ListValuedMap',
+  'org.apache.commons.collections4.multimap.ArrayListValuedHashMap',
   'org.apache.commons.csv.CSVFormat',
   'org.apache.commons.csv.CSVParser',
   'org.apache.commons.csv.CSVRecord',
@@ -525,27 +537,10 @@ thirdPartyAudit.excludes = [
   'org.apache.http.StatusLine',
   'org.apache.http.client.HttpClient',
   'org.apache.http.client.methods.HttpGet',
+  'org.apache.http.client.methods.HttpPost',
   'org.apache.http.client.utils.URIBuilder',
+  'org.apache.http.entity.ByteArrayEntity',
   'org.apache.http.impl.client.DefaultHttpClient',
-  'org.apache.james.mime4j.MimeException',
-  'org.apache.james.mime4j.codec.DecodeMonitor',
-  'org.apache.james.mime4j.codec.DecoderUtil',
-  'org.apache.james.mime4j.dom.FieldParser',
-  'org.apache.james.mime4j.dom.address.Address',
-  'org.apache.james.mime4j.dom.address.AddressList',
-  'org.apache.james.mime4j.dom.address.Mailbox',
-  'org.apache.james.mime4j.dom.address.MailboxList',
-  'org.apache.james.mime4j.dom.field.AddressListField',
-  'org.apache.james.mime4j.dom.field.DateTimeField',
-  'org.apache.james.mime4j.dom.field.MailboxListField',
-  'org.apache.james.mime4j.dom.field.ParsedField',
-  'org.apache.james.mime4j.dom.field.UnstructuredField',
-  'org.apache.james.mime4j.field.LenientFieldParser',
-  'org.apache.james.mime4j.parser.ContentHandler',
-  'org.apache.james.mime4j.parser.MimeStreamParser',
-  'org.apache.james.mime4j.stream.BodyDescriptor',
-  'org.apache.james.mime4j.stream.Field',
-  'org.apache.james.mime4j.stream.MimeConfig',
   'org.apache.jcp.xml.dsig.internal.dom.DOMDigestMethod',
   'org.apache.jcp.xml.dsig.internal.dom.DOMKeyInfo',
   'org.apache.jcp.xml.dsig.internal.dom.DOMReference',
@@ -630,8 +625,6 @@ thirdPartyAudit.excludes = [
   'org.etsi.uri.x01903.v13.impl.UnsignedSignaturePropertiesTypeImpl$1RevocationValuesList',
   'org.etsi.uri.x01903.v13.impl.UnsignedSignaturePropertiesTypeImpl$1SigAndRefsTimeStampList',
   'org.etsi.uri.x01903.v13.impl.UnsignedSignaturePropertiesTypeImpl$1SignatureTimeStampList',
-  'org.etsi.uri.x01903.v14.ValidationDataType$Factory',
-  'org.etsi.uri.x01903.v14.ValidationDataType',
   'org.json.JSONArray',
   'org.json.JSONObject',
   'org.json.XML',
@@ -785,17 +778,16 @@ thirdPartyAudit.excludes = [
   'org.openxmlformats.schemas.drawingml.x2006.main.CTGlowEffect',
   'org.openxmlformats.schemas.drawingml.x2006.main.CTGrayscaleEffect',
   'org.openxmlformats.schemas.drawingml.x2006.main.CTGrayscaleTransform',
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTGroupFillProperties',
   'org.openxmlformats.schemas.drawingml.x2006.main.CTGroupLocking',
   'org.openxmlformats.schemas.drawingml.x2006.main.CTHSLEffect',
   'org.openxmlformats.schemas.drawingml.x2006.main.CTInnerShadowEffect',
   'org.openxmlformats.schemas.drawingml.x2006.main.CTInverseGammaTransform',
   'org.openxmlformats.schemas.drawingml.x2006.main.CTInverseTransform',
   'org.openxmlformats.schemas.drawingml.x2006.main.CTLineJoinBevel',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTLineJoinMiterProperties',
   'org.openxmlformats.schemas.drawingml.x2006.main.CTLuminanceEffect',
   'org.openxmlformats.schemas.drawingml.x2006.main.CTObjectStyleDefaults',
   'org.openxmlformats.schemas.drawingml.x2006.main.CTPath2DArcTo',
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTPatternFillProperties',
   'org.openxmlformats.schemas.drawingml.x2006.main.CTPolarAdjustHandle',
   'org.openxmlformats.schemas.drawingml.x2006.main.CTPositiveFixedAngle',
   'org.openxmlformats.schemas.drawingml.x2006.main.CTPresetShadowEffect',
@@ -808,7 +800,7 @@ thirdPartyAudit.excludes = [
   'org.openxmlformats.schemas.drawingml.x2006.main.CTSoftEdgesEffect',
   'org.openxmlformats.schemas.drawingml.x2006.main.CTSupplementalFont',
   'org.openxmlformats.schemas.drawingml.x2006.main.CTTableBackgroundStyle',
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTTablePartStyle',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTTableCellBorderStyle',
   'org.openxmlformats.schemas.drawingml.x2006.main.CTTextBlipBullet',
   'org.openxmlformats.schemas.drawingml.x2006.main.CTTextBulletColorFollowText',
   'org.openxmlformats.schemas.drawingml.x2006.main.CTTextBulletSizeFollowText',
@@ -823,9 +815,10 @@ thirdPartyAudit.excludes = [
   'org.openxmlformats.schemas.drawingml.x2006.main.STBlackWhiteMode',
   'org.openxmlformats.schemas.drawingml.x2006.main.STBlipCompression',
   'org.openxmlformats.schemas.drawingml.x2006.main.STFixedAngle',
-  'org.openxmlformats.schemas.drawingml.x2006.main.STGuid',
   'org.openxmlformats.schemas.drawingml.x2006.main.STPanose',
   'org.openxmlformats.schemas.drawingml.x2006.main.STPathFillMode',
+  'org.openxmlformats.schemas.drawingml.x2006.main.STPresetPatternVal',
+  'org.openxmlformats.schemas.drawingml.x2006.main.STPresetPatternVal$Enum',
   'org.openxmlformats.schemas.drawingml.x2006.main.STRectAlignment',
   'org.openxmlformats.schemas.drawingml.x2006.main.STTextColumnCount',
   'org.openxmlformats.schemas.drawingml.x2006.main.STTextNonNegativePoint',
@@ -933,6 +926,34 @@ thirdPartyAudit.excludes = [
   'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1SatOffList',
   'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1ShadeList',
   'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1TintList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1AlphaList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1AlphaModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1AlphaOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1BlueList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1BlueModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1BlueOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1CompList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1GammaList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1GrayList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1GreenList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1GreenModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1GreenOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1HueList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1HueModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1HueOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1InvGammaList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1InvList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1LumList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1LumModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1LumOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1RedList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1RedModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1RedOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1SatList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1SatModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1SatOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1ShadeList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1TintList',
   'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1AlphaList',
   'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1AlphaModList',
   'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1AlphaOffList',
@@ -1184,6 +1205,8 @@ thirdPartyAudit.excludes = [
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTOleObjects',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTOleSize',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTPCDKPIs',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTPageField',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTPageFields',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTPhoneticRun',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTPivotFilters',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTPivotHierarchies',
@@ -1269,7 +1292,6 @@ thirdPartyAudit.excludes = [
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTMergeCellsImpl$1MergeCellList',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTNumFmtsImpl$1NumFmtList',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTPageBreakImpl$1BrkList',
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTPageFieldsImpl$1PageFieldList',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTPivotCacheRecordsImpl$1RList',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTPivotCachesImpl$1PivotCacheList',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTPivotFieldsImpl$1PivotFieldList',
@@ -2049,3 +2071,21 @@ thirdPartyAudit.excludes = [
   'ucar.nc2.Variable',
   'ucar.nc2.dataset.NetcdfDataset'
 ]
+
+if (JavaVersion.current() > JavaVersion.VERSION_1_8) {
+  thirdPartyAudit.excludes += [
+    'javax.activation.ActivationDataFlavor',
+    'javax.activation.CommandMap',
+    'javax.activation.DataContentHandler',
+    'javax.activation.DataHandler',
+    'javax.activation.DataSource',
+    'javax.activation.FileDataSource',
+    'javax.activation.MailcapCommandMap',
+    'javax.xml.bind.DatatypeConverter',
+    'javax.xml.bind.JAXBContext',
+    'javax.xml.bind.JAXBElement',
+    'javax.xml.bind.Marshaller',
+    'javax.xml.bind.Unmarshaller',
+    'javax.xml.bind.helpers.DefaultValidationEventHandler',
+  ]
+}
diff --git a/plugins/ingest-attachment/licenses/apache-mime4j-LICENSE.txt b/plugins/ingest-attachment/licenses/apache-mime4j-LICENSE.txt
new file mode 100644
index 0000000000000..63151ee14587c
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/apache-mime4j-LICENSE.txt
@@ -0,0 +1,361 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+
+
+
+   THIS PRODUCT ALSO INCLUDES THIRD PARTY SOFTWARE REDISTRIBUTED UNDER THE
+   FOLLOWING LICENSES:
+
+	Apache Commons Logging,
+	   The Apache Software License, Version 1.1 (commons-logging-1.1.1.jar)
+
+		 The Apache Software License, Version 1.1
+
+		 Redistribution and use in source and binary forms, with or without
+		 modification, are permitted provided that the following conditions
+		 are met:
+
+		 1. Redistributions of source code must retain the above copyright
+		    notice, this list of conditions and the following disclaimer.
+
+		 2. Redistributions in binary form must reproduce the above copyright
+		    notice, this list of conditions and the following disclaimer in
+		    the documentation and/or other materials provided with the
+		    distribution.
+
+		 3. The end-user documentation included with the redistribution,
+		    if any, must include the following acknowledgment:
+		       "This product includes software developed by the
+		        Apache Software Foundation (http://www.apache.org/)."
+		    Alternately, this acknowledgment may appear in the software itself,
+		    if and wherever such third-party acknowledgments normally appear.
+
+		 4. The names "Apache" and "Apache Software Foundation" must
+		    not be used to endorse or promote products derived from this
+		    software without prior written permission. For written
+		    permission, please contact apache@apache.org.
+
+		 5. Products derived from this software may not be called "Apache",
+		    nor may "Apache" appear in their name, without prior written
+		    permission of the Apache Software Foundation.
+
+		 THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
+		 WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+		 OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+		 DISCLAIMED.  IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
+		 ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+		 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+		 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+		 USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+		 ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+		 OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+		 OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+		 SUCH DAMAGE.
+
+
+	Test messages from the Perl-MIME-Tools project,
+
+					 The "Artistic License"
+
+						Preamble
+
+		The intent of this document is to state the conditions under which a
+		Package may be copied, such that the Copyright Holder maintains some
+		semblance of artistic control over the development of the package,
+		while giving the users of the package the right to use and distribute
+		the Package in a more-or-less customary fashion, plus the right to make
+		reasonable modifications.
+
+		Definitions:
+
+			"Package" refers to the collection of files distributed by the
+			Copyright Holder, and derivatives of that collection of files
+			created through textual modification.
+
+			"Standard Version" refers to such a Package if it has not been
+			modified, or has been modified in accordance with the wishes
+			of the Copyright Holder as specified below.
+
+			"Copyright Holder" is whoever is named in the copyright or
+			copyrights for the package.
+
+			"You" is you, if you're thinking about copying or distributing
+			this Package.
+
+			"Reasonable copying fee" is whatever you can justify on the
+			basis of media cost, duplication charges, time of people involved,
+			and so on.  (You will not be required to justify it to the
+			Copyright Holder, but only to the computing community at large
+			as a market that must bear the fee.)
+
+			"Freely Available" means that no fee is charged for the item
+			itself, though there may be fees involved in handling the item.
+			It also means that recipients of the item may redistribute it
+			under the same conditions they received it.
+
+		1. You may make and give away verbatim copies of the source form of the
+		Standard Version of this Package without restriction, provided that you
+		duplicate all of the original copyright notices and associated disclaimers.
+
+		2. You may apply bug fixes, portability fixes and other modifications
+		derived from the Public Domain or from the Copyright Holder.  A Package
+		modified in such a way shall still be considered the Standard Version.
+
+		3. You may otherwise modify your copy of this Package in any way, provided
+		that you insert a prominent notice in each changed file stating how and
+		when you changed that file, and provided that you do at least ONE of the
+		following:
+
+		    a) place your modifications in the Public Domain or otherwise make them
+		    Freely Available, such as by posting said modifications to Usenet or
+		    an equivalent medium, or placing the modifications on a major archive
+		    site such as uunet.uu.net, or by allowing the Copyright Holder to include
+		    your modifications in the Standard Version of the Package.
+
+		    b) use the modified Package only within your corporation or organization.
+
+		    c) rename any non-standard executables so the names do not conflict
+		    with standard executables, which must also be provided, and provide
+		    a separate manual page for each non-standard executable that clearly
+		    documents how it differs from the Standard Version.
+
+		    d) make other distribution arrangements with the Copyright Holder.
+
+		4. You may distribute the programs of this Package in object code or
+		executable form, provided that you do at least ONE of the following:
+
+		    a) distribute a Standard Version of the executables and library files,
+		    together with instructions (in the manual page or equivalent) on where
+		    to get the Standard Version.
+
+		    b) accompany the distribution with the machine-readable source of
+		    the Package with your modifications.
+
+		    c) give non-standard executables non-standard names, and clearly
+		    document the differences in manual pages (or equivalent), together
+		    with instructions on where to get the Standard Version.
+
+		    d) make other distribution arrangements with the Copyright Holder.
+
+		5. You may charge a reasonable copying fee for any distribution of this
+		Package.  You may charge any fee you choose for support of this
+		Package.  You may not charge a fee for this Package itself.  However,
+		you may distribute this Package in aggregate with other (possibly
+		commercial) programs as part of a larger (possibly commercial) software
+		distribution provided that you do not advertise this Package as a
+		product of your own.  You may embed this Package's interpreter within
+		an executable of yours (by linking); this shall be construed as a mere
+		form of aggregation, provided that the complete Standard Version of the
+		interpreter is so embedded.
+
+		6. The scripts and library files supplied as input to or produced as
+		output from the programs of this Package do not automatically fall
+		under the copyright of this Package, but belong to whoever generated
+		them, and may be sold commercially, and may be aggregated with this
+		Package.  If such scripts or library files are aggregated with this
+		Package via the so-called "undump" or "unexec" methods of producing a
+		binary executable image, then distribution of such an image shall
+		neither be construed as a distribution of this Package nor shall it
+		fall under the restrictions of Paragraphs 3 and 4, provided that you do
+		not represent such an executable image as a Standard Version of this
+		Package.
+
+		7. C subroutines (or comparably compiled subroutines in other
+		languages) supplied by you and linked into this Package in order to
+		emulate subroutines and variables of the language defined by this
+		Package shall not be considered part of this Package, but are the
+		equivalent of input as in Paragraph 6, provided these subroutines do
+		not change the language in any way that would cause it to fail the
+		regression tests for the language.
+
+		8. Aggregation of this Package with a commercial distribution is always
+		permitted provided that the use of this Package is embedded; that is,
+		when no overt attempt is made to make this Package's interfaces visible
+		to the end user of the commercial distribution.  Such use shall not be
+		construed as a distribution of this Package.
+
+		9. The name of the Copyright Holder may not be used to endorse or promote
+		products derived from this software without specific prior written permission.
+
+		10. THIS PACKAGE IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR
+		IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
+		WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
+
+						The End
+
+
diff --git a/plugins/ingest-attachment/licenses/apache-mime4j-NOTICE.txt b/plugins/ingest-attachment/licenses/apache-mime4j-NOTICE.txt
new file mode 100644
index 0000000000000..61523975eb2f7
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/apache-mime4j-NOTICE.txt
@@ -0,0 +1,13 @@
+   =========================================================================
+   ==      NOTICE file for use with the Apache License, Version 2.0,      ==
+   =========================================================================
+   
+   Apache JAMES Mime4j
+   Copyright 2004-2010 The Apache Software Foundation
+   
+   This product includes software developed at
+   The Apache Software Foundation (http://www.apache.org/).
+
+   This product test suite includes data (mimetools-testmsgs folder) developed 
+   by Eryq and ZeeGee Software Inc as part of the "MIME-tools" Perl5 toolkit
+   and licensed under the Artistic License
diff --git a/plugins/ingest-attachment/licenses/apache-mime4j-core-0.7.2.jar.sha1 b/plugins/ingest-attachment/licenses/apache-mime4j-core-0.7.2.jar.sha1
new file mode 100644
index 0000000000000..8210fc7fc162c
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/apache-mime4j-core-0.7.2.jar.sha1
@@ -0,0 +1 @@
+a81264fe0265ebe8fd1d8128aad06dc320de6eef
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/apache-mime4j-dom-0.7.2.jar.sha1 b/plugins/ingest-attachment/licenses/apache-mime4j-dom-0.7.2.jar.sha1
new file mode 100644
index 0000000000000..3ede85a019108
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/apache-mime4j-dom-0.7.2.jar.sha1
@@ -0,0 +1 @@
+1c289aa264548a0a1f1b43685a9cb2ab23f67287
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/bcmail-jdk15on-1.54.jar.sha1 b/plugins/ingest-attachment/licenses/bcmail-jdk15on-1.54.jar.sha1
deleted file mode 100644
index 79da45c5c422e..0000000000000
--- a/plugins/ingest-attachment/licenses/bcmail-jdk15on-1.54.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-9d9b5432b4b29ef4a853223bc6e19379ef116cca
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/bcmail-jdk15on-1.55.jar.sha1 b/plugins/ingest-attachment/licenses/bcmail-jdk15on-1.55.jar.sha1
new file mode 100644
index 0000000000000..8fdfb8f580780
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/bcmail-jdk15on-1.55.jar.sha1
@@ -0,0 +1 @@
+5cea2dada69b98698ea975a5c1dd3c91ac8ffbb6
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/bcpkix-jdk15on-1.54.jar.sha1 b/plugins/ingest-attachment/licenses/bcpkix-jdk15on-1.54.jar.sha1
deleted file mode 100644
index 2d0c3cf4e277a..0000000000000
--- a/plugins/ingest-attachment/licenses/bcpkix-jdk15on-1.54.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b11bfee99bb11eea344de6e4a07fe89212c55c02
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/bcpkix-jdk15on-1.55.jar.sha1 b/plugins/ingest-attachment/licenses/bcpkix-jdk15on-1.55.jar.sha1
new file mode 100644
index 0000000000000..a4d546be04fc2
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/bcpkix-jdk15on-1.55.jar.sha1
@@ -0,0 +1 @@
+6392d8cba22b722c6570d660ca0b3921ff1bae4f
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/bcprov-jdk15on-1.54.jar.sha1 b/plugins/ingest-attachment/licenses/bcprov-jdk15on-1.54.jar.sha1
deleted file mode 100644
index fcda646b42a8d..0000000000000
--- a/plugins/ingest-attachment/licenses/bcprov-jdk15on-1.54.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-1acdedeb89f1d950d67b73d481eb7736df65eedb
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/bcprov-jdk15on-1.55.jar.sha1 b/plugins/ingest-attachment/licenses/bcprov-jdk15on-1.55.jar.sha1
new file mode 100644
index 0000000000000..1c507e17b88e0
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/bcprov-jdk15on-1.55.jar.sha1
@@ -0,0 +1 @@
+935f2e57a00ec2c489cbd2ad830d4a399708f979
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/fontbox-2.0.1.jar.sha1 b/plugins/ingest-attachment/licenses/fontbox-2.0.1.jar.sha1
deleted file mode 100644
index 0668199b242fd..0000000000000
--- a/plugins/ingest-attachment/licenses/fontbox-2.0.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b9d4f0993e015f3f1ce0be9e7300cf62dd7a7f15
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/fontbox-2.0.3.jar.sha1 b/plugins/ingest-attachment/licenses/fontbox-2.0.3.jar.sha1
new file mode 100644
index 0000000000000..e3ff3d39459f3
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/fontbox-2.0.3.jar.sha1
@@ -0,0 +1 @@
+448ee588d0136121cf5c4dd397384cccb9db1ad7
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/pdfbox-2.0.1.jar.sha1 b/plugins/ingest-attachment/licenses/pdfbox-2.0.1.jar.sha1
deleted file mode 100644
index 1014db340445f..0000000000000
--- a/plugins/ingest-attachment/licenses/pdfbox-2.0.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-dbc69649118b7eff278f228c070a40ee559e1f62
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/pdfbox-2.0.3.jar.sha1 b/plugins/ingest-attachment/licenses/pdfbox-2.0.3.jar.sha1
new file mode 100644
index 0000000000000..807e2482ac295
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/pdfbox-2.0.3.jar.sha1
@@ -0,0 +1 @@
+be7b09de93f7c7795c57f4fbf14db60ab93806b4
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/poi-3.15-beta1.jar.sha1 b/plugins/ingest-attachment/licenses/poi-3.15-beta1.jar.sha1
deleted file mode 100644
index 6049604dd9751..0000000000000
--- a/plugins/ingest-attachment/licenses/poi-3.15-beta1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-048bb8326b81323631d9ceb4236cfbd382e56da2
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/poi-3.15.jar.sha1 b/plugins/ingest-attachment/licenses/poi-3.15.jar.sha1
new file mode 100644
index 0000000000000..5405d32cd0108
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/poi-3.15.jar.sha1
@@ -0,0 +1 @@
+965bba8899988008bb2341e300347de62aad5391
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/poi-ooxml-3.15-beta1.jar.sha1 b/plugins/ingest-attachment/licenses/poi-ooxml-3.15-beta1.jar.sha1
deleted file mode 100644
index c3cf49d9246d0..0000000000000
--- a/plugins/ingest-attachment/licenses/poi-ooxml-3.15-beta1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-81085a47fdf0d74d473d605c6b3784e26731842e
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/poi-ooxml-3.15.jar.sha1 b/plugins/ingest-attachment/licenses/poi-ooxml-3.15.jar.sha1
new file mode 100644
index 0000000000000..4362223dac4a4
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/poi-ooxml-3.15.jar.sha1
@@ -0,0 +1 @@
+e2800856735b07b8edd417aee07685470216a00f
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/poi-ooxml-schemas-3.15-beta1.jar.sha1 b/plugins/ingest-attachment/licenses/poi-ooxml-schemas-3.15-beta1.jar.sha1
deleted file mode 100644
index afd3b676d08e8..0000000000000
--- a/plugins/ingest-attachment/licenses/poi-ooxml-schemas-3.15-beta1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f8bc979ad79908a99483337f1ca2edf78558ac20
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/poi-ooxml-schemas-3.15.jar.sha1 b/plugins/ingest-attachment/licenses/poi-ooxml-schemas-3.15.jar.sha1
new file mode 100644
index 0000000000000..393e6f885d9f2
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/poi-ooxml-schemas-3.15.jar.sha1
@@ -0,0 +1 @@
+de4a50ca39de48a19606b35644ecadb2f733c479
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/poi-scratchpad-3.15-beta1.jar.sha1 b/plugins/ingest-attachment/licenses/poi-scratchpad-3.15-beta1.jar.sha1
deleted file mode 100644
index 7056a9fa49ed0..0000000000000
--- a/plugins/ingest-attachment/licenses/poi-scratchpad-3.15-beta1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f4e276aaf97a60a1156388c9e38069122b7ea914
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/poi-scratchpad-3.15.jar.sha1 b/plugins/ingest-attachment/licenses/poi-scratchpad-3.15.jar.sha1
new file mode 100644
index 0000000000000..d08f475a3f60a
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/poi-scratchpad-3.15.jar.sha1
@@ -0,0 +1 @@
+f1db76ae4a9389fa4339dc3b7f8208aa82c72b04
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/tika-core-1.13.jar.sha1 b/plugins/ingest-attachment/licenses/tika-core-1.13.jar.sha1
deleted file mode 100644
index cfc36a450bd77..0000000000000
--- a/plugins/ingest-attachment/licenses/tika-core-1.13.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-1305c798d41d1d7bbf12cb7c0ca184c98eed25ad
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/tika-core-1.14.jar.sha1 b/plugins/ingest-attachment/licenses/tika-core-1.14.jar.sha1
new file mode 100644
index 0000000000000..2da2c07d9b27f
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/tika-core-1.14.jar.sha1
@@ -0,0 +1 @@
+afff8f1774994aa973ef90bc8d38ddf089b9d6d9
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/tika-parsers-1.13.jar.sha1 b/plugins/ingest-attachment/licenses/tika-parsers-1.13.jar.sha1
deleted file mode 100644
index 7fb2755d545e0..0000000000000
--- a/plugins/ingest-attachment/licenses/tika-parsers-1.13.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-374fde67b9d35f785534b0e6c4953533c31bab5f
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/tika-parsers-1.14.jar.sha1 b/plugins/ingest-attachment/licenses/tika-parsers-1.14.jar.sha1
new file mode 100644
index 0000000000000..b96b3cdefdd1a
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/tika-parsers-1.14.jar.sha1
@@ -0,0 +1 @@
+d26c10a9e7d116366562aa260013a30a55ff4e8f
\ No newline at end of file
diff --git a/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java b/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java
index 0ba79ecc9fbea..f7f474711be7a 100644
--- a/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java
+++ b/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java
@@ -38,6 +38,7 @@
 import java.util.Set;
 
 import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException;
+import static org.elasticsearch.ingest.ConfigurationUtils.readBooleanProperty;
 import static org.elasticsearch.ingest.ConfigurationUtils.readIntProperty;
 import static org.elasticsearch.ingest.ConfigurationUtils.readOptionalList;
 import static org.elasticsearch.ingest.ConfigurationUtils.readStringProperty;
@@ -52,23 +53,36 @@ public final class AttachmentProcessor extends AbstractProcessor {
     private final String targetField;
     private final Set<Property> properties;
     private final int indexedChars;
+    private final boolean ignoreMissing;
 
     AttachmentProcessor(String tag, String field, String targetField, Set<Property> properties,
-                        int indexedChars) throws IOException {
+                        int indexedChars, boolean ignoreMissing) throws IOException {
         super(tag);
         this.field = field;
         this.targetField = targetField;
         this.properties = properties;
         this.indexedChars = indexedChars;
+        this.ignoreMissing = ignoreMissing;
+    }
+
+    boolean isIgnoreMissing() {
+        return ignoreMissing;
     }
 
     @Override
     public void execute(IngestDocument ingestDocument) {
         Map<String, Object> additionalFields = new HashMap<>();
 
+        byte[] input = ingestDocument.getFieldValueAsBytes(field, ignoreMissing);
+
+        if (input == null && ignoreMissing) {
+            return;
+        } else if (input == null) {
+            throw new IllegalArgumentException("field [" + field + "] is null, cannot parse.");
+        }
+
         try {
             Metadata metadata = new Metadata();
-            byte[] input = ingestDocument.getFieldValueAsBytes(field);
             String parsedContent = TikaImpl.parse(input, metadata, indexedChars);
 
             if (properties.contains(Property.CONTENT) && Strings.hasLength(parsedContent)) {
@@ -166,6 +180,7 @@ public AttachmentProcessor create(Map<String, Processor.Factory> registry, Strin
             String targetField = readStringProperty(TYPE, processorTag, config, "target_field", "attachment");
             List<String> properyNames = readOptionalList(TYPE, processorTag, config, "properties");
             int indexedChars = readIntProperty(TYPE, processorTag, config, "indexed_chars", NUMBER_OF_CHARS_INDEXED);
+            boolean ignoreMissing = readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
 
             final Set<Property> properties;
             if (properyNames != null) {
@@ -182,7 +197,7 @@ public AttachmentProcessor create(Map<String, Processor.Factory> registry, Strin
                 properties = DEFAULT_PROPERTIES;
             }
 
-            return new AttachmentProcessor(processorTag, field, targetField, properties, indexedChars);
+            return new AttachmentProcessor(processorTag, field, targetField, properties, indexedChars, ignoreMissing);
         }
     }
 
diff --git a/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/TikaImpl.java b/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/TikaImpl.java
index 27a4cfebbcd2f..e1ce6cf3d78cf 100644
--- a/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/TikaImpl.java
+++ b/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/TikaImpl.java
@@ -22,8 +22,10 @@
 import org.apache.tika.Tika;
 import org.apache.tika.exception.TikaException;
 import org.apache.tika.metadata.Metadata;
+import org.apache.tika.mime.MediaType;
 import org.apache.tika.parser.AutoDetectParser;
 import org.apache.tika.parser.Parser;
+import org.apache.tika.parser.ParserDecorator;
 import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.bootstrap.JarHell;
 import org.elasticsearch.common.SuppressForbidden;
@@ -45,7 +47,11 @@
 import java.security.PrivilegedExceptionAction;
 import java.security.ProtectionDomain;
 import java.security.SecurityPermission;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashSet;
 import java.util.PropertyPermission;
+import java.util.Set;
 
 /**
  * Runs tika with limited parsers and limited permissions.
@@ -54,6 +60,9 @@
  */
 final class TikaImpl {
 
+    /** Exclude some formats */
+    private static final Set<MediaType> EXCLUDES = Collections.singleton(MediaType.application("x-tika-ooxml"));
+
     /** subset of parsers for types we support */
     private static final Parser PARSERS[] = new Parser[] {
         // documents
@@ -63,7 +72,7 @@ final class TikaImpl {
         new org.apache.tika.parser.txt.TXTParser(),
         new org.apache.tika.parser.microsoft.OfficeParser(),
         new org.apache.tika.parser.microsoft.OldExcelParser(),
-        new org.apache.tika.parser.microsoft.ooxml.OOXMLParser(),
+        ParserDecorator.withoutTypes(new org.apache.tika.parser.microsoft.ooxml.OOXMLParser(), EXCLUDES),
         new org.apache.tika.parser.odf.OpenDocumentParser(),
         new org.apache.tika.parser.iwork.IWorkPackageParser(),
         new org.apache.tika.parser.xml.DcXMLParser(),
@@ -128,7 +137,12 @@ static PermissionCollection getRestrictedPermissions() {
         addReadPermissions(perms, JarHell.parseClassPath());
         // plugin jars
         if (TikaImpl.class.getClassLoader() instanceof URLClassLoader) {
-            addReadPermissions(perms, ((URLClassLoader)TikaImpl.class.getClassLoader()).getURLs());
+            URL[] urls = ((URLClassLoader)TikaImpl.class.getClassLoader()).getURLs();
+            Set<URL> set = new LinkedHashSet<>(Arrays.asList(urls));
+            if (set.size() != urls.length) {
+                throw new AssertionError("duplicate jars: " + Arrays.toString(urls));
+            }
+            addReadPermissions(perms, set);
         }
         // jvm's java.io.tmpdir (needs read/write)
         perms.add(new FilePermission(System.getProperty("java.io.tmpdir") + System.getProperty("file.separator") + "-",
@@ -145,7 +159,7 @@ static PermissionCollection getRestrictedPermissions() {
 
     // add resources to (what is typically) a jar, but might not be (e.g. in tests/IDE)
     @SuppressForbidden(reason = "adds access to jar resources")
-    static void addReadPermissions(Permissions perms, URL resources[]) {
+    static void addReadPermissions(Permissions perms, Set<URL> resources) {
         try {
             for (URL url : resources) {
                 Path path = PathUtils.get(url.toURI());
diff --git a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorFactoryTests.java b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorFactoryTests.java
index 2848d6c3c64ca..c5f2b0d7fbd35 100644
--- a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorFactoryTests.java
+++ b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorFactoryTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.ingest.attachment;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.ArrayList;
@@ -45,13 +44,14 @@ public void testBuildDefaults() throws Exception {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
 
-        String processorTag = randomAsciiOfLength(10);
+        String processorTag = randomAlphaOfLength(10);
 
         AttachmentProcessor processor = factory.create(null, processorTag, config);
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getTargetField(), equalTo("attachment"));
         assertThat(processor.getProperties(), sameInstance(AttachmentProcessor.Factory.DEFAULT_PROPERTIES));
+        assertFalse(processor.isIgnoreMissing());
     }
 
     public void testConfigureIndexedChars() throws Exception {
@@ -60,10 +60,11 @@ public void testConfigureIndexedChars() throws Exception {
         config.put("field", "_field");
         config.put("indexed_chars", indexedChars);
 
-        String processorTag = randomAsciiOfLength(10);
+        String processorTag = randomAlphaOfLength(10);
         AttachmentProcessor processor = factory.create(null, processorTag, config);
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.getIndexedChars(), is(indexedChars));
+        assertFalse(processor.isIgnoreMissing());
     }
 
     public void testBuildTargetField() throws Exception {
@@ -73,6 +74,7 @@ public void testBuildTargetField() throws Exception {
         AttachmentProcessor processor = factory.create(null, null, config);
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getTargetField(), equalTo("_field"));
+        assertFalse(processor.isIgnoreMissing());
     }
 
     public void testBuildFields() throws Exception {
@@ -90,6 +92,7 @@ public void testBuildFields() throws Exception {
         AttachmentProcessor processor = factory.create(null, null, config);
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getProperties(), equalTo(properties));
+        assertFalse(processor.isIgnoreMissing());
     }
 
     public void testBuildIllegalFieldOption() throws Exception {
@@ -117,4 +120,19 @@ public void testBuildIllegalFieldOption() throws Exception {
             assertThat(e.getMessage(), equalTo("[properties] property isn't a list, but of type [java.lang.String]"));
         }
     }
+
+    public void testIgnoreMissing() throws Exception {
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", "_field");
+        config.put("ignore_missing", true);
+
+        String processorTag = randomAlphaOfLength(10);
+
+        AttachmentProcessor processor = factory.create(null, processorTag, config);
+        assertThat(processor.getTag(), equalTo(processorTag));
+        assertThat(processor.getField(), equalTo("_field"));
+        assertThat(processor.getTargetField(), equalTo("attachment"));
+        assertThat(processor.getProperties(), sameInstance(AttachmentProcessor.Factory.DEFAULT_PROPERTIES));
+        assertTrue(processor.isIgnoreMissing());
+    }
 }
diff --git a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorTests.java b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorTests.java
index 4e051d057248c..07e369985321a 100644
--- a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorTests.java
+++ b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorTests.java
@@ -21,8 +21,9 @@
 
 import org.apache.commons.io.IOUtils;
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
+import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
@@ -30,18 +31,22 @@
 import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.Base64;
+import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 
+import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.core.IsCollectionContaining.hasItem;
 
 public class AttachmentProcessorTests extends ESTestCase {
@@ -50,8 +55,8 @@ public class AttachmentProcessorTests extends ESTestCase {
 
     @Before
     public void createStandardProcessor() throws IOException {
-        processor = new AttachmentProcessor(randomAsciiOfLength(10), "source_field",
-            "target_field", EnumSet.allOf(AttachmentProcessor.Property.class), 10000);
+        processor = new AttachmentProcessor(randomAlphaOfLength(10), "source_field",
+            "target_field", EnumSet.allOf(AttachmentProcessor.Property.class), 10000, false);
     }
 
     public void testEnglishTextDocument() throws Exception {
@@ -83,8 +88,8 @@ public void testHtmlDocumentWithRandomFields() throws Exception {
         if (randomBoolean()) {
             selectedProperties.add(AttachmentProcessor.Property.DATE);
         }
-        processor = new AttachmentProcessor(randomAsciiOfLength(10), "source_field",
-            "target_field", selectedProperties, 10000);
+        processor = new AttachmentProcessor(randomAlphaOfLength(10), "source_field",
+            "target_field", selectedProperties, 10000, false);
 
         Map<String, Object> attachmentData = parseDocument("htmlWithEmptyDateMeta.html", processor);
         assertThat(attachmentData.keySet(), hasSize(selectedFieldNames.length));
@@ -125,12 +130,52 @@ public void testWordDocument() throws Exception {
             is("application/vnd.openxmlformats-officedocument.wordprocessingml.document"));
     }
 
+    public void testWordDocumentWithVisioSchema() throws Exception {
+        Map<String, Object> attachmentData = parseDocument("issue-22077.docx", processor);
+
+        assertThat(attachmentData.keySet(), containsInAnyOrder("content", "language", "date", "author", "content_type",
+            "content_length"));
+        assertThat(attachmentData.get("content").toString(), containsString("Table of Contents"));
+        assertThat(attachmentData.get("language"), is("en"));
+        assertThat(attachmentData.get("date"), is("2015-01-06T18:07:00Z"));
+        assertThat(attachmentData.get("author"), is(notNullValue()));
+        assertThat(attachmentData.get("content_length"), is(notNullValue()));
+        assertThat(attachmentData.get("content_type").toString(),
+            is("application/vnd.openxmlformats-officedocument.wordprocessingml.document"));
+    }
+
+    public void testLegacyWordDocumentWithVisioSchema() throws Exception {
+        Map<String, Object> attachmentData = parseDocument("issue-22077.doc", processor);
+
+        assertThat(attachmentData.keySet(), containsInAnyOrder("content", "language", "date", "author", "content_type",
+            "content_length"));
+        assertThat(attachmentData.get("content").toString(), containsString("Table of Contents"));
+        assertThat(attachmentData.get("language"), is("en"));
+        assertThat(attachmentData.get("date"), is("2016-12-16T15:04:00Z"));
+        assertThat(attachmentData.get("author"), is(notNullValue()));
+        assertThat(attachmentData.get("content_length"), is(notNullValue()));
+        assertThat(attachmentData.get("content_type").toString(),
+            is("application/msword"));
+    }
+
+    public void testPdf() throws Exception {
+        Map<String, Object> attachmentData = parseDocument("test.pdf", processor);
+        assertThat(attachmentData.get("content"),
+                is("This is a test, with umlauts, from München\n\nAlso contains newlines for testing.\n\nAnd one more."));
+        assertThat(attachmentData.get("content_type").toString(), is("application/pdf"));
+        assertThat(attachmentData.get("content_length"), is(notNullValue()));
+    }
+
+    public void testVisioIsExcluded() throws Exception {
+        Map<String, Object> attachmentData = parseDocument("issue-22077.vsdx", processor);
+        assertThat(attachmentData.get("content"), nullValue());
+        assertThat(attachmentData.get("content_type"), is("application/vnd.ms-visio.drawing"));
+        assertThat(attachmentData.get("content_length"), is(0L));
+    }
+
     public void testEncryptedPdf() throws Exception {
-        try {
-            parseDocument("encrypted.pdf", processor);
-        } catch (ElasticsearchParseException e) {
-            assertThat(e.getDetailedMessage(), containsString("document is encrypted"));
-        }
+        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> parseDocument("encrypted.pdf", processor));
+        assertThat(e.getDetailedMessage(), containsString("document is encrypted"));
     }
 
     public void testHtmlDocument() throws Exception {
@@ -193,6 +238,40 @@ public void testParseAsBytesArray() throws Exception {
         assertThat(attachmentData.get("content_length"), is(notNullValue()));
     }
 
+    public void testNullValueWithIgnoreMissing() throws Exception {
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(),
+            Collections.singletonMap("source_field", null));
+        IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
+        Processor processor = new AttachmentProcessor(randomAlphaOfLength(10), "source_field", "randomTarget", null, 10, true);
+        processor.execute(ingestDocument);
+        assertIngestDocument(originalIngestDocument, ingestDocument);
+    }
+
+    public void testNonExistentWithIgnoreMissing() throws Exception {
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
+        IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
+        Processor processor = new AttachmentProcessor(randomAlphaOfLength(10), "source_field", "randomTarget", null, 10, true);
+        processor.execute(ingestDocument);
+        assertIngestDocument(originalIngestDocument, ingestDocument);
+    }
+
+    public void testNullWithoutIgnoreMissing() throws Exception {
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(),
+            Collections.singletonMap("source_field", null));
+        IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
+        Processor processor = new AttachmentProcessor(randomAlphaOfLength(10), "source_field", "randomTarget", null, 10, false);
+        Exception exception = expectThrows(Exception.class, () -> processor.execute(ingestDocument));
+        assertThat(exception.getMessage(), equalTo("field [source_field] is null, cannot parse."));
+    }
+
+    public void testNonExistentWithoutIgnoreMissing() throws Exception {
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
+        IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
+        Processor processor = new AttachmentProcessor(randomAlphaOfLength(10), "source_field", "randomTarget", null, 10, false);
+        Exception exception = expectThrows(Exception.class, () -> processor.execute(ingestDocument));
+        assertThat(exception.getMessage(), equalTo("field [source_field] not present as part of path [source_field]"));
+    }
+
     private Map<String, Object> parseDocument(String file, AttachmentProcessor processor) throws Exception {
         Map<String, Object> document = new HashMap<>();
         document.put("source_field", getAsBase64(file));
diff --git a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/IngestAttachmentClientYamlTestSuiteIT.java b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/IngestAttachmentClientYamlTestSuiteIT.java
index d720a4abf284e..679e950c328b8 100644
--- a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/IngestAttachmentClientYamlTestSuiteIT.java
+++ b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/IngestAttachmentClientYamlTestSuiteIT.java
@@ -24,9 +24,6 @@
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-
-import java.io.IOException;
 
 public class IngestAttachmentClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
@@ -35,8 +32,8 @@ public IngestAttachmentClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandida
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
-        return ESClientYamlSuiteTestCase.createParameters(0, 1);
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
     }
 }
 
diff --git a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/TikaDocTests.java b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/TikaDocTests.java
index 4b9a40dd8a9ab..b4445d16b7769 100644
--- a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/TikaDocTests.java
+++ b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/TikaDocTests.java
@@ -22,6 +22,7 @@
 import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems;
 import org.apache.lucene.util.TestUtil;
 import org.apache.tika.metadata.Metadata;
+import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.test.ESTestCase;
 
 import java.nio.file.DirectoryStream;
@@ -37,11 +38,17 @@
 public class TikaDocTests extends ESTestCase {
 
     /** some test files from tika test suite, zipped up */
-    static final String TIKA_FILES = "/org/elasticsearch/ingest/attachment/test/tika-files.zip";
+    static final String TIKA_FILES = "/org/elasticsearch/ingest/attachment/test/tika-files/";
 
     public void testFiles() throws Exception {
         Path tmp = createTempDir();
-        TestUtil.unzip(getClass().getResourceAsStream(TIKA_FILES), tmp);
+        logger.debug("unzipping all tika sample files");
+        try (DirectoryStream<Path> stream = Files.newDirectoryStream(PathUtils.get(getClass().getResource(TIKA_FILES).toURI()))) {
+            for (Path doc : stream) {
+                String filename = doc.getFileName().toString();
+                TestUtil.unzip(getClass().getResourceAsStream(TIKA_FILES + filename), tmp);
+            }
+        }
 
         try (DirectoryStream<Path> stream = Files.newDirectoryStream(tmp)) {
             for (Path doc : stream) {
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/sample-files/issue-22077.doc b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/sample-files/issue-22077.doc
new file mode 100644
index 0000000000000..10badd5809be6
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/sample-files/issue-22077.doc differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/sample-files/issue-22077.docx b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/sample-files/issue-22077.docx
new file mode 100644
index 0000000000000..bab550607a915
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/sample-files/issue-22077.docx differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/sample-files/issue-22077.vsdx b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/sample-files/issue-22077.vsdx
new file mode 100644
index 0000000000000..fb9cde51b4b5b
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/sample-files/issue-22077.vsdx differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/sample-files/test.pdf b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/sample-files/test.pdf
new file mode 100644
index 0000000000000..c6e47d74e660f
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/sample-files/test.pdf differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files.zip
deleted file mode 100644
index 10f5d50767733..0000000000000
Binary files a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files.zip and /dev/null differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/Doc1_ole.doc.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/Doc1_ole.doc.zip
new file mode 100644
index 0000000000000..f9bd7c46b2bf1
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/Doc1_ole.doc.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/EmbeddedDocument.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/EmbeddedDocument.docx.zip
new file mode 100644
index 0000000000000..bb4e8e1ef3e4b
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/EmbeddedDocument.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/EmbeddedOutlook.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/EmbeddedOutlook.docx.zip
new file mode 100644
index 0000000000000..67de787f017e6
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/EmbeddedOutlook.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/EmbeddedPDF.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/EmbeddedPDF.docx.zip
new file mode 100644
index 0000000000000..7e02f55c03dec
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/EmbeddedPDF.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/NullHeader.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/NullHeader.docx.zip
new file mode 100644
index 0000000000000..0516c35b0710c
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/NullHeader.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/big-preamble.html.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/big-preamble.html.zip
new file mode 100644
index 0000000000000..fc851923abbd5
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/big-preamble.html.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/boilerplate-whitespace.html.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/boilerplate-whitespace.html.zip
new file mode 100644
index 0000000000000..2d1f8e601ce88
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/boilerplate-whitespace.html.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/boilerplate.html.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/boilerplate.html.zip
new file mode 100644
index 0000000000000..c728a6303b4ab
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/boilerplate.html.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/footnotes.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/footnotes.docx.zip
new file mode 100644
index 0000000000000..21376b6917945
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/footnotes.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/headerPic.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/headerPic.docx.zip
new file mode 100644
index 0000000000000..6bb9b3f463d29
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/headerPic.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/headers.mbox.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/headers.mbox.zip
new file mode 100644
index 0000000000000..f36e6b8de4724
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/headers.mbox.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/pictures.ppt.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/pictures.ppt.zip
new file mode 100644
index 0000000000000..55dde5eada7b6
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/pictures.ppt.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/protect.xlsx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/protect.xlsx.zip
new file mode 100644
index 0000000000000..9823a8debf8be
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/protect.xlsx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/protectedFile.xlsx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/protectedFile.xlsx.zip
new file mode 100644
index 0000000000000..b4de09b0fdc45
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/protectedFile.xlsx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/protectedSheets.xlsx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/protectedSheets.xlsx.zip
new file mode 100644
index 0000000000000..8dde8a493a20b
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/protectedSheets.xlsx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/resume.html.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/resume.html.zip
new file mode 100644
index 0000000000000..0c53eae3b3aec
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/resume.html.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test-outlook.msg.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test-outlook.msg.zip
new file mode 100644
index 0000000000000..e9a0bfbed00c9
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test-outlook.msg.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test-outlook2003.msg.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test-outlook2003.msg.zip
new file mode 100644
index 0000000000000..bce3da6e05b81
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test-outlook2003.msg.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test.doc.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test.doc.zip
new file mode 100644
index 0000000000000..4bcf68dcebc95
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test.doc.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testAnnotations.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testAnnotations.pdf.zip
new file mode 100644
index 0000000000000..3268c952a3330
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testAnnotations.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testBinControlWord.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testBinControlWord.rtf.zip
new file mode 100644
index 0000000000000..4a7efa19f0a0e
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testBinControlWord.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testBulletPoints.key.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testBulletPoints.key.zip
new file mode 100644
index 0000000000000..41b896525ca02
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testBulletPoints.key.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.doc.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.doc.zip
new file mode 100644
index 0000000000000..15a056cb3a3e4
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.doc.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.docx.zip
new file mode 100644
index 0000000000000..30b4f6877a8c7
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.pdf.zip
new file mode 100644
index 0000000000000..bb21a9d71039f
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.ppt.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.ppt.zip
new file mode 100644
index 0000000000000..e0122d4004bf5
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.ppt.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.pptx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.pptx.zip
new file mode 100644
index 0000000000000..77967db80c69a
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.pptx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.rtf.zip
new file mode 100644
index 0000000000000..b7b6bdf5c1316
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.xls.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.xls.zip
new file mode 100644
index 0000000000000..7250235b89c5a
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.xls.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.xlsx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.xlsx.zip
new file mode 100644
index 0000000000000..7fe71de731d53
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.xlsx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testDOCX_Thumbnail.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testDOCX_Thumbnail.docx.zip
new file mode 100644
index 0000000000000..e0fd82103fe0c
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testDOCX_Thumbnail.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL-charts.xls.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL-charts.xls.zip
new file mode 100644
index 0000000000000..fd5d8472d4964
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL-charts.xls.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL-formats.xls.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL-formats.xls.zip
new file mode 100644
index 0000000000000..f1015248b63c6
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL-formats.xls.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL-formats.xlsx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL-formats.xlsx.zip
new file mode 100644
index 0000000000000..b62c8813e9679
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL-formats.xlsx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL.xls.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL.xls.zip
new file mode 100644
index 0000000000000..f0dc3b00f9685
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL.xls.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL.xlsx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL.xlsx.zip
new file mode 100644
index 0000000000000..45b3dae6f4476
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL.xlsx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_1img.xls.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_1img.xls.zip
new file mode 100644
index 0000000000000..09cb216169797
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_1img.xls.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_1img.xlsx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_1img.xlsx.zip
new file mode 100644
index 0000000000000..81228661820bc
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_1img.xlsx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_4.xls.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_4.xls.zip
new file mode 100644
index 0000000000000..46da553af950c
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_4.xls.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_5.xls.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_5.xls.zip
new file mode 100644
index 0000000000000..ca3e08c4701f0
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_5.xls.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_95.xls.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_95.xls.zip
new file mode 100644
index 0000000000000..00e3d373d118d
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_95.xls.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_custom_props.xls.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_custom_props.xls.zip
new file mode 100644
index 0000000000000..2fa4d6b8b91b3
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_custom_props.xls.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_custom_props.xlsx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_custom_props.xlsx.zip
new file mode 100644
index 0000000000000..9cfd16cec258d
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_custom_props.xlsx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_embeded.xls.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_embeded.xls.zip
new file mode 100644
index 0000000000000..b7291cafc9877
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_embeded.xls.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_embeded.xlsx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_embeded.xlsx.zip
new file mode 100644
index 0000000000000..755b7395e83c9
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_embeded.xlsx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_headers_footers.xls.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_headers_footers.xls.zip
new file mode 100644
index 0000000000000..6cb18625ca7b7
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_headers_footers.xls.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_headers_footers.xlsx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_headers_footers.xlsx.zip
new file mode 100644
index 0000000000000..924ad5eece245
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_headers_footers.xlsx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_textbox.xlsx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_textbox.xlsx.zip
new file mode 100644
index 0000000000000..43046dff52614
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_textbox.xlsx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testExtraSpaces.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testExtraSpaces.pdf.zip
new file mode 100644
index 0000000000000..8f6ed7a19f224
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testExtraSpaces.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testFontAfterBufferedText.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testFontAfterBufferedText.rtf.zip
new file mode 100644
index 0000000000000..77a5ac60dc4e0
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testFontAfterBufferedText.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testFooter.ods.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testFooter.ods.zip
new file mode 100644
index 0000000000000..8659af1e691dc
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testFooter.ods.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testFooter.odt.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testFooter.odt.zip
new file mode 100644
index 0000000000000..85ba5501448ab
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testFooter.odt.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testHTML.html.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testHTML.html.zip
new file mode 100644
index 0000000000000..58cfd638254b9
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testHTML.html.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testHTMLNoisyMetaEncoding_1.html.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testHTMLNoisyMetaEncoding_1.html.zip
new file mode 100644
index 0000000000000..a12735a5205e0
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testHTMLNoisyMetaEncoding_1.html.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testHTMLNoisyMetaEncoding_2.html.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testHTMLNoisyMetaEncoding_2.html.zip
new file mode 100644
index 0000000000000..89a3f9d7aaadb
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testHTMLNoisyMetaEncoding_2.html.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testHTMLNoisyMetaEncoding_3.html.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testHTMLNoisyMetaEncoding_3.html.zip
new file mode 100644
index 0000000000000..da23b13ee4432
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testHTMLNoisyMetaEncoding_3.html.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testHTMLNoisyMetaEncoding_4.html.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testHTMLNoisyMetaEncoding_4.html.zip
new file mode 100644
index 0000000000000..15850474d98cd
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testHTMLNoisyMetaEncoding_4.html.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testHTML_utf8.html.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testHTML_utf8.html.zip
new file mode 100644
index 0000000000000..bdb5e58f373c0
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testHTML_utf8.html.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testKeynote.key.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testKeynote.key.zip
new file mode 100644
index 0000000000000..68693369d10cf
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testKeynote.key.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testMasterFooter.odp.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testMasterFooter.odp.zip
new file mode 100644
index 0000000000000..42467aa951bf3
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testMasterFooter.odp.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testMasterSlideTable.key.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testMasterSlideTable.key.zip
new file mode 100644
index 0000000000000..8d7803d396c8a
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testMasterSlideTable.key.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testNPEOpenDocument.odt.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testNPEOpenDocument.odt.zip
new file mode 100644
index 0000000000000..4c41bc7a466a2
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testNPEOpenDocument.odt.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testODFwithOOo3.odt.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testODFwithOOo3.odt.zip
new file mode 100644
index 0000000000000..9202fe44e446c
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testODFwithOOo3.odt.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOpenOffice2.odt.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOpenOffice2.odt.zip
new file mode 100644
index 0000000000000..1993365b5d1da
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOpenOffice2.odt.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOptionalHyphen.doc.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOptionalHyphen.doc.zip
new file mode 100644
index 0000000000000..b9f515941e315
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOptionalHyphen.doc.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOptionalHyphen.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOptionalHyphen.docx.zip
new file mode 100644
index 0000000000000..033871556d11a
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOptionalHyphen.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOptionalHyphen.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOptionalHyphen.pdf.zip
new file mode 100644
index 0000000000000..37fab9390588b
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOptionalHyphen.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOptionalHyphen.ppt.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOptionalHyphen.ppt.zip
new file mode 100644
index 0000000000000..e50261f432419
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOptionalHyphen.ppt.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOptionalHyphen.pptx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOptionalHyphen.pptx.zip
new file mode 100644
index 0000000000000..19d6e78e6dfa7
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOptionalHyphen.pptx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOptionalHyphen.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOptionalHyphen.rtf.zip
new file mode 100644
index 0000000000000..d86048798c24e
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOptionalHyphen.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOverlappingText.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOverlappingText.pdf.zip
new file mode 100644
index 0000000000000..c7f8b78704858
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOverlappingText.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF-custommetadata.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF-custommetadata.pdf.zip
new file mode 100644
index 0000000000000..44865eca76c7d
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF-custommetadata.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF.pdf.zip
new file mode 100644
index 0000000000000..fd5195b053867
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDFEmbeddingAndEmbedded.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDFEmbeddingAndEmbedded.docx.zip
new file mode 100644
index 0000000000000..a7ceed20665ad
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDFEmbeddingAndEmbedded.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDFFileEmbInAnnotation.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDFFileEmbInAnnotation.pdf.zip
new file mode 100644
index 0000000000000..4b6e333895039
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDFFileEmbInAnnotation.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDFPackage.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDFPackage.pdf.zip
new file mode 100644
index 0000000000000..8bbb9166bb368
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDFPackage.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDFTripleLangTitle.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDFTripleLangTitle.pdf.zip
new file mode 100644
index 0000000000000..2d1ebdbdb2f2f
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDFTripleLangTitle.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDFTwoTextBoxes.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDFTwoTextBoxes.pdf.zip
new file mode 100644
index 0000000000000..d05907a3da109
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDFTwoTextBoxes.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDFVarious.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDFVarious.pdf.zip
new file mode 100644
index 0000000000000..cf86cbae79e7e
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDFVarious.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_PDFEncodedStringInXMP.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_PDFEncodedStringInXMP.pdf.zip
new file mode 100644
index 0000000000000..e8377846e79d7
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_PDFEncodedStringInXMP.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.10.x.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.10.x.pdf.zip
new file mode 100644
index 0000000000000..4fffed77ad888
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.10.x.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.11.x.PDFA-1b.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.11.x.PDFA-1b.pdf.zip
new file mode 100644
index 0000000000000..16098515ce2d3
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.11.x.PDFA-1b.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.4.x.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.4.x.pdf.zip
new file mode 100644
index 0000000000000..cdf93347572b9
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.4.x.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.5.x.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.5.x.pdf.zip
new file mode 100644
index 0000000000000..e5ef413b8bed6
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.5.x.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.6.x.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.6.x.pdf.zip
new file mode 100644
index 0000000000000..b57b4dc4efbf8
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.6.x.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.7.x.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.7.x.pdf.zip
new file mode 100644
index 0000000000000..d69bbb50feede
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.7.x.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.8.x.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.8.x.pdf.zip
new file mode 100644
index 0000000000000..e077af2074323
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.8.x.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.9.x.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.9.x.pdf.zip
new file mode 100644
index 0000000000000..c9b6857c1526e
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.9.x.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_acroform3.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_acroform3.pdf.zip
new file mode 100644
index 0000000000000..267d41df314a2
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_acroform3.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_bom.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_bom.pdf.zip
new file mode 100644
index 0000000000000..c4ea31f96f43d
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_bom.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_bookmarks.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_bookmarks.pdf.zip
new file mode 100644
index 0000000000000..88a5db5e63eff
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_bookmarks.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_multiFormatEmbFiles.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_multiFormatEmbFiles.pdf.zip
new file mode 100644
index 0000000000000..e61f2bf11124d
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_multiFormatEmbFiles.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_no_extract_no_accessibility_owner_empty.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_no_extract_no_accessibility_owner_empty.pdf.zip
new file mode 100644
index 0000000000000..16920b49e3a4a
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_no_extract_no_accessibility_owner_empty.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_no_extract_yes_accessibility_owner_empty.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_no_extract_yes_accessibility_owner_empty.pdf.zip
new file mode 100644
index 0000000000000..59887f7e5d20d
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_no_extract_yes_accessibility_owner_empty.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_twoAuthors.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_twoAuthors.pdf.zip
new file mode 100644
index 0000000000000..d240c51d8221c
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_twoAuthors.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT.ppsm.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT.ppsm.zip
new file mode 100644
index 0000000000000..47f652d14af7f
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT.ppsm.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT.ppsx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT.ppsx.zip
new file mode 100644
index 0000000000000..0f21f13be0f93
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT.ppsx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT.ppt.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT.ppt.zip
new file mode 100644
index 0000000000000..00c5c88c6ee3b
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT.ppt.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT.pptm.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT.pptm.zip
new file mode 100644
index 0000000000000..edd3e444fd78c
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT.pptm.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT.pptx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT.pptx.zip
new file mode 100644
index 0000000000000..68a9d662756d2
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT.pptx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPTX_Thumbnail.pptx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPTX_Thumbnail.pptx.zip
new file mode 100644
index 0000000000000..23dba29f32453
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPTX_Thumbnail.pptx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_2imgs.ppt.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_2imgs.ppt.zip
new file mode 100644
index 0000000000000..317e1bbe3521a
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_2imgs.ppt.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_2imgs.pptx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_2imgs.pptx.zip
new file mode 100644
index 0000000000000..62096338a496b
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_2imgs.pptx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_autodate.ppt.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_autodate.ppt.zip
new file mode 100644
index 0000000000000..5bd71fbd2e4af
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_autodate.ppt.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_autodate.pptx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_autodate.pptx.zip
new file mode 100644
index 0000000000000..811ab85cc26cc
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_autodate.pptx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_comment.ppt.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_comment.ppt.zip
new file mode 100644
index 0000000000000..3e94ba4cce1dd
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_comment.ppt.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_comment.pptx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_comment.pptx.zip
new file mode 100644
index 0000000000000..68d2135d4122b
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_comment.pptx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_custom_props.ppt.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_custom_props.ppt.zip
new file mode 100644
index 0000000000000..80673e625d78a
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_custom_props.ppt.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_custom_props.pptx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_custom_props.pptx.zip
new file mode 100644
index 0000000000000..d27c98d2c2d42
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_custom_props.pptx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_embedded2.ppt.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_embedded2.ppt.zip
new file mode 100644
index 0000000000000..017f9f500c741
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_embedded2.ppt.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_embedded_two_slides.pptx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_embedded_two_slides.pptx.zip
new file mode 100644
index 0000000000000..dfa459cccbdab
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_embedded_two_slides.pptx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_embeded.ppt.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_embeded.ppt.zip
new file mode 100644
index 0000000000000..fb8a3caf75501
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_embeded.ppt.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_embeded.pptx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_embeded.pptx.zip
new file mode 100644
index 0000000000000..f33f463c3356f
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_embeded.pptx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_masterFooter.ppt.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_masterFooter.ppt.zip
new file mode 100644
index 0000000000000..6a7278893942e
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_masterFooter.ppt.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_masterFooter.pptx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_masterFooter.pptx.zip
new file mode 100644
index 0000000000000..992dad41c5346
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_masterFooter.pptx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_masterText.ppt.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_masterText.ppt.zip
new file mode 100644
index 0000000000000..e31c4719f4761
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_masterText.ppt.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_masterText.pptx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_masterText.pptx.zip
new file mode 100644
index 0000000000000..1fa4f48748767
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_masterText.pptx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_masterText2.ppt.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_masterText2.ppt.zip
new file mode 100644
index 0000000000000..f58055ecfcbe3
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_masterText2.ppt.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_masterText2.pptx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_masterText2.pptx.zip
new file mode 100644
index 0000000000000..f8ff31052d129
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_masterText2.pptx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_various.ppt.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_various.ppt.zip
new file mode 100644
index 0000000000000..5d13efb8d4ad4
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_various.ppt.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_various.pptx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_various.pptx.zip
new file mode 100644
index 0000000000000..5d146a1de0bfc
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_various.pptx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPageNumber.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPageNumber.pdf.zip
new file mode 100644
index 0000000000000..4f9db82c2997e
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPageNumber.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPages.pages.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPages.pages.zip
new file mode 100644
index 0000000000000..3db77c0873320
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPages.pages.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPagesComments.pages.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPagesComments.pages.zip
new file mode 100644
index 0000000000000..55004e812d22c
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPagesComments.pages.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPagesHeadersFootersAlphaLower.pages.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPagesHeadersFootersAlphaLower.pages.zip
new file mode 100644
index 0000000000000..bfea0c7825acb
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPagesHeadersFootersAlphaLower.pages.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPagesHeadersFootersAlphaUpper.pages.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPagesHeadersFootersAlphaUpper.pages.zip
new file mode 100644
index 0000000000000..3298d7eaa8fe3
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPagesHeadersFootersAlphaUpper.pages.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPagesHeadersFootersFootnotes.pages.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPagesHeadersFootersFootnotes.pages.zip
new file mode 100644
index 0000000000000..34a4fb83230fe
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPagesHeadersFootersFootnotes.pages.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPagesHeadersFootersRomanLower.pages.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPagesHeadersFootersRomanLower.pages.zip
new file mode 100644
index 0000000000000..b48ebe4458d0a
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPagesHeadersFootersRomanLower.pages.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPagesHeadersFootersRomanUpper.pages.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPagesHeadersFootersRomanUpper.pages.zip
new file mode 100644
index 0000000000000..967e139595f08
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPagesHeadersFootersRomanUpper.pages.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPagesLayout.pages.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPagesLayout.pages.zip
new file mode 100644
index 0000000000000..c6f0e04a53653
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPagesLayout.pages.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPopupAnnotation.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPopupAnnotation.pdf.zip
new file mode 100644
index 0000000000000..4eb78653562d4
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPopupAnnotation.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTF-ms932.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTF-ms932.rtf.zip
new file mode 100644
index 0000000000000..bd80f2b061aba
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTF-ms932.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTF.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTF.rtf.zip
new file mode 100644
index 0000000000000..72ef8ebfc56fe
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTF.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFBoldItalic.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFBoldItalic.rtf.zip
new file mode 100644
index 0000000000000..935c249176b6b
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFBoldItalic.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFControls.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFControls.rtf.zip
new file mode 100644
index 0000000000000..0c55479cc89c0
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFControls.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFCorruptListOverride.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFCorruptListOverride.rtf.zip
new file mode 100644
index 0000000000000..e6ea82d476001
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFCorruptListOverride.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFEmbeddedLink.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFEmbeddedLink.rtf.zip
new file mode 100644
index 0000000000000..857c589d9eb84
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFEmbeddedLink.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFHexEscapeInsideWord.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFHexEscapeInsideWord.rtf.zip
new file mode 100644
index 0000000000000..f10b768d5a2d0
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFHexEscapeInsideWord.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFHyperlink.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFHyperlink.rtf.zip
new file mode 100644
index 0000000000000..658eb87784f23
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFHyperlink.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFIgnoredControlWord.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFIgnoredControlWord.rtf.zip
new file mode 100644
index 0000000000000..15a74d726b6f5
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFIgnoredControlWord.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFInvalidUnicode.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFInvalidUnicode.rtf.zip
new file mode 100644
index 0000000000000..f24a5ab9c0869
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFInvalidUnicode.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFJapanese.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFJapanese.rtf.zip
new file mode 100644
index 0000000000000..8bf4f6e7165b3
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFJapanese.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFListLibreOffice.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFListLibreOffice.rtf.zip
new file mode 100644
index 0000000000000..e2a406d99a652
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFListLibreOffice.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFListMicrosoftWord.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFListMicrosoftWord.rtf.zip
new file mode 100644
index 0000000000000..155718ac8b381
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFListMicrosoftWord.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFListOverride.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFListOverride.rtf.zip
new file mode 100644
index 0000000000000..c9b7f3fdbee65
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFListOverride.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFNewlines.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFNewlines.rtf.zip
new file mode 100644
index 0000000000000..8b0dffe13512e
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFNewlines.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFRegularImages.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFRegularImages.rtf.zip
new file mode 100644
index 0000000000000..2bf8d9b530183
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFRegularImages.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFTableCellSeparation.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFTableCellSeparation.rtf.zip
new file mode 100644
index 0000000000000..5e110de516907
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFTableCellSeparation.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFTableCellSeparation2.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFTableCellSeparation2.rtf.zip
new file mode 100644
index 0000000000000..599e7f22edb5b
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFTableCellSeparation2.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFUmlautSpaces.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFUmlautSpaces.rtf.zip
new file mode 100644
index 0000000000000..f3cbec9e51d57
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFUmlautSpaces.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFUmlautSpaces2.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFUmlautSpaces2.rtf.zip
new file mode 100644
index 0000000000000..2c905405ca48b
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFUmlautSpaces2.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFUnicodeGothic.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFUnicodeGothic.rtf.zip
new file mode 100644
index 0000000000000..40263351a04ba
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFUnicodeGothic.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFUnicodeUCNControlWordCharacterDoubling.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFUnicodeUCNControlWordCharacterDoubling.rtf.zip
new file mode 100644
index 0000000000000..c8f40b549c3b0
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFUnicodeUCNControlWordCharacterDoubling.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFVarious.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFVarious.rtf.zip
new file mode 100644
index 0000000000000..60281e7f9ae11
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFVarious.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFWindowsCodepage1250.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFWindowsCodepage1250.rtf.zip
new file mode 100644
index 0000000000000..4a7852c97d3d8
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFWindowsCodepage1250.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFWithCurlyBraces.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFWithCurlyBraces.rtf.zip
new file mode 100644
index 0000000000000..48268a457c090
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFWithCurlyBraces.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFWord2010CzechCharacters.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFWord2010CzechCharacters.rtf.zip
new file mode 100644
index 0000000000000..be845e4f925e0
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFWord2010CzechCharacters.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFWordPadCzechCharacters.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFWordPadCzechCharacters.rtf.zip
new file mode 100644
index 0000000000000..508801675de7a
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFWordPadCzechCharacters.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testStyles.odt.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testStyles.odt.zip
new file mode 100644
index 0000000000000..22f6c76b9a802
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testStyles.odt.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testTXTNonASCIIUTF8.txt.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testTXTNonASCIIUTF8.txt.zip
new file mode 100644
index 0000000000000..7326a28ad7695
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testTXTNonASCIIUTF8.txt.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testTables.key.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testTables.key.zip
new file mode 100644
index 0000000000000..3c963dcc0770e
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testTables.key.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testTextBoxes.key.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testTextBoxes.key.zip
new file mode 100644
index 0000000000000..aea9b1cf66c5f
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testTextBoxes.key.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD.doc.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD.doc.zip
new file mode 100644
index 0000000000000..38b2954a4ff29
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD.doc.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD.docx.zip
new file mode 100644
index 0000000000000..2ed0aae33f439
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD6.doc.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD6.doc.zip
new file mode 100644
index 0000000000000..b8dbbd8322e96
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD6.doc.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_1img.doc.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_1img.doc.zip
new file mode 100644
index 0000000000000..1baa330e5ce8e
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_1img.doc.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_1img.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_1img.docx.zip
new file mode 100644
index 0000000000000..8920521bd2d38
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_1img.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_3imgs.doc.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_3imgs.doc.zip
new file mode 100644
index 0000000000000..b60716b93d2c0
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_3imgs.doc.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_3imgs.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_3imgs.docx.zip
new file mode 100644
index 0000000000000..062713d931818
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_3imgs.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_bold_character_runs.doc.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_bold_character_runs.doc.zip
new file mode 100644
index 0000000000000..b0a6af23d34da
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_bold_character_runs.doc.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_bold_character_runs.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_bold_character_runs.docx.zip
new file mode 100644
index 0000000000000..546f98bf093c2
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_bold_character_runs.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_bold_character_runs2.doc.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_bold_character_runs2.doc.zip
new file mode 100644
index 0000000000000..5dbedec5e8630
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_bold_character_runs2.doc.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_bold_character_runs2.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_bold_character_runs2.docx.zip
new file mode 100644
index 0000000000000..8edde24f9937a
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_bold_character_runs2.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_closingSmartQInHyperLink.doc.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_closingSmartQInHyperLink.doc.zip
new file mode 100644
index 0000000000000..4484a6b005241
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_closingSmartQInHyperLink.doc.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_custom_props.doc.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_custom_props.doc.zip
new file mode 100644
index 0000000000000..72a94832d3e8c
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_custom_props.doc.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_custom_props.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_custom_props.docx.zip
new file mode 100644
index 0000000000000..30aaef7d81794
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_custom_props.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_embedded_pdf.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_embedded_pdf.docx.zip
new file mode 100644
index 0000000000000..2b74f67dc9f67
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_embedded_pdf.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_embedded_rtf.doc.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_embedded_rtf.doc.zip
new file mode 100644
index 0000000000000..e456721065c21
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_embedded_rtf.doc.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_embeded.doc.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_embeded.doc.zip
new file mode 100644
index 0000000000000..699a13b652ab2
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_embeded.doc.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_embeded.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_embeded.docx.zip
new file mode 100644
index 0000000000000..b53088ddb716c
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_embeded.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_header_hyperlink.doc.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_header_hyperlink.doc.zip
new file mode 100644
index 0000000000000..867dc1808f647
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_header_hyperlink.doc.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_missing_ooxml_bean1.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_missing_ooxml_bean1.docx.zip
new file mode 100644
index 0000000000000..04172a65382f7
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_missing_ooxml_bean1.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_missing_text.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_missing_text.docx.zip
new file mode 100644
index 0000000000000..325ecce927917
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_missing_text.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_no_format.doc.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_no_format.doc.zip
new file mode 100644
index 0000000000000..38eecb941d4bc
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_no_format.doc.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_no_format.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_no_format.docx.zip
new file mode 100644
index 0000000000000..1c19cf3e98224
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_no_format.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_null_style.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_null_style.docx.zip
new file mode 100644
index 0000000000000..605eeb250232f
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_null_style.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_numbered_list.doc.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_numbered_list.doc.zip
new file mode 100644
index 0000000000000..7f8d3b9ad2014
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_numbered_list.doc.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_numbered_list.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_numbered_list.docx.zip
new file mode 100644
index 0000000000000..42b4ed17c4d0e
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_numbered_list.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_override_list_numbering.doc.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_override_list_numbering.doc.zip
new file mode 100644
index 0000000000000..39c954b2d19bd
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_override_list_numbering.doc.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_override_list_numbering.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_override_list_numbering.docx.zip
new file mode 100644
index 0000000000000..77bb9dbda68d2
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_override_list_numbering.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_tabular_symbol.doc.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_tabular_symbol.doc.zip
new file mode 100644
index 0000000000000..0c63f9825677c
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_tabular_symbol.doc.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_text_box.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_text_box.docx.zip
new file mode 100644
index 0000000000000..504c56728c97a
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_text_box.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_various.doc.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_various.doc.zip
new file mode 100644
index 0000000000000..9d7c24e052068
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_various.doc.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_various.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_various.docx.zip
new file mode 100644
index 0000000000000..393aca11ab7df
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_various.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWordArt.pptx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWordArt.pptx.zip
new file mode 100644
index 0000000000000..75234fde98d72
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWordArt.pptx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testXHTML.html.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testXHTML.html.zip
new file mode 100644
index 0000000000000..45c4dd0479adb
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testXHTML.html.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testXLSX_Thumbnail.xlsx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testXLSX_Thumbnail.xlsx.zip
new file mode 100644
index 0000000000000..919cf31e75b4e
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testXLSX_Thumbnail.xlsx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testXML.xml.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testXML.xml.zip
new file mode 100644
index 0000000000000..a6520a8a811fe
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testXML.xml.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testXML2.xml.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testXML2.xml.zip
new file mode 100644
index 0000000000000..5b89863685a13
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testXML2.xml.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testXML3.xml.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testXML3.xml.zip
new file mode 100644
index 0000000000000..08a066d9c1e36
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testXML3.xml.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test_TIKA-1251.doc.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test_TIKA-1251.doc.zip
new file mode 100644
index 0000000000000..4d3071290216b
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test_TIKA-1251.doc.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test_embedded_package.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test_embedded_package.rtf.zip
new file mode 100644
index 0000000000000..58d7f15b23b3b
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test_embedded_package.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test_embedded_zip.pptx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test_embedded_zip.pptx.zip
new file mode 100644
index 0000000000000..e9d802cfd4e3e
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test_embedded_zip.pptx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test_list_override.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test_list_override.rtf.zip
new file mode 100644
index 0000000000000..57d5553829448
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test_list_override.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test_recursive_embedded.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test_recursive_embedded.docx.zip
new file mode 100644
index 0000000000000..edd97967e95e4
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test_recursive_embedded.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test_recursive_embedded_npe.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test_recursive_embedded_npe.docx.zip
new file mode 100644
index 0000000000000..84da48f0250f2
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test_recursive_embedded_npe.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/tika434.html.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/tika434.html.zip
new file mode 100644
index 0000000000000..41972bc8447c8
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/tika434.html.zip differ
diff --git a/plugins/ingest-geoip/build.gradle b/plugins/ingest-geoip/build.gradle
index d9e90a61d406b..e74a27844db75 100644
--- a/plugins/ingest-geoip/build.gradle
+++ b/plugins/ingest-geoip/build.gradle
@@ -23,10 +23,10 @@ esplugin {
 }
 
 dependencies {
-  compile ('com.maxmind.geoip2:geoip2:2.7.0')
+  compile ('com.maxmind.geoip2:geoip2:2.8.0')
   // geoip2 dependencies:
-  compile('com.fasterxml.jackson.core:jackson-annotations:2.7.1')
-  compile('com.fasterxml.jackson.core:jackson-databind:2.7.1')
+  compile('com.fasterxml.jackson.core:jackson-annotations:2.8.2')
+  compile('com.fasterxml.jackson.core:jackson-databind:2.8.2')
   compile('com.maxmind.db:maxmind-db:1.2.1')
 
   testCompile 'org.elasticsearch:geolite2-databases:20160608'
@@ -50,14 +50,19 @@ bundlePlugin {
 }
 
 thirdPartyAudit.excludes = [
-  // geoip WebServiceClient needs Google http client, but we're not using WebServiceClient:
-  'com.google.api.client.http.HttpTransport',
-  'com.google.api.client.http.GenericUrl',
-  'com.google.api.client.http.HttpResponse',
-  'com.google.api.client.http.HttpRequestFactory',
-  'com.google.api.client.http.HttpRequest',
-  'com.google.api.client.http.HttpHeaders',
-  'com.google.api.client.http.HttpResponseException',
-  'com.google.api.client.http.javanet.NetHttpTransport',
-  'com.google.api.client.http.javanet.NetHttpTransport',
+  // geoip WebServiceClient needs apache http client, but we're not using WebServiceClient:
+  'org.apache.http.HttpEntity',
+  'org.apache.http.HttpHost',
+  'org.apache.http.HttpResponse',
+  'org.apache.http.StatusLine',
+  'org.apache.http.auth.UsernamePasswordCredentials',
+  'org.apache.http.client.config.RequestConfig$Builder',
+  'org.apache.http.client.config.RequestConfig',
+  'org.apache.http.client.methods.CloseableHttpResponse',
+  'org.apache.http.client.methods.HttpGet',
+  'org.apache.http.client.utils.URIBuilder',
+  'org.apache.http.impl.auth.BasicScheme',
+  'org.apache.http.impl.client.CloseableHttpClient',
+  'org.apache.http.impl.client.HttpClientBuilder',
+  'org.apache.http.util.EntityUtils'
 ]
diff --git a/plugins/ingest-geoip/licenses/geoip2-2.7.0.jar.sha1 b/plugins/ingest-geoip/licenses/geoip2-2.7.0.jar.sha1
deleted file mode 100644
index 2015e311d6013..0000000000000
--- a/plugins/ingest-geoip/licenses/geoip2-2.7.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-2010d922191f5801939b462a5703ab79a7829626
\ No newline at end of file
diff --git a/plugins/ingest-geoip/licenses/geoip2-2.8.0.jar.sha1 b/plugins/ingest-geoip/licenses/geoip2-2.8.0.jar.sha1
new file mode 100644
index 0000000000000..c603668660169
--- /dev/null
+++ b/plugins/ingest-geoip/licenses/geoip2-2.8.0.jar.sha1
@@ -0,0 +1 @@
+46226778ec32b776e80f282c5bf65b88d36cc0a0
\ No newline at end of file
diff --git a/plugins/ingest-geoip/licenses/jackson-annotations-2.7.1.jar.sha1 b/plugins/ingest-geoip/licenses/jackson-annotations-2.7.1.jar.sha1
deleted file mode 100644
index 69b45742d8439..0000000000000
--- a/plugins/ingest-geoip/licenses/jackson-annotations-2.7.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-8b93f301823b79033fcbe873779b3d84f9730fc1
\ No newline at end of file
diff --git a/plugins/ingest-geoip/licenses/jackson-annotations-2.8.2.jar.sha1 b/plugins/ingest-geoip/licenses/jackson-annotations-2.8.2.jar.sha1
new file mode 100644
index 0000000000000..c3b701dbb8681
--- /dev/null
+++ b/plugins/ingest-geoip/licenses/jackson-annotations-2.8.2.jar.sha1
@@ -0,0 +1 @@
+a38d544583e90cf163b2e45e4a57f5c54de670d3
\ No newline at end of file
diff --git a/plugins/ingest-geoip/licenses/jackson-databind-2.7.1.jar.sha1 b/plugins/ingest-geoip/licenses/jackson-databind-2.7.1.jar.sha1
deleted file mode 100644
index d9b4ca6a79bd2..0000000000000
--- a/plugins/ingest-geoip/licenses/jackson-databind-2.7.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-14d88822bca655de7aa6ed3e4c498d115505710a
\ No newline at end of file
diff --git a/plugins/ingest-geoip/licenses/jackson-databind-2.8.2.jar.sha1 b/plugins/ingest-geoip/licenses/jackson-databind-2.8.2.jar.sha1
new file mode 100644
index 0000000000000..7e90b5f8e978d
--- /dev/null
+++ b/plugins/ingest-geoip/licenses/jackson-databind-2.8.2.jar.sha1
@@ -0,0 +1 @@
+1f12816593c1422be957471c98a80bfbace60fa2
\ No newline at end of file
diff --git a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseReaderLazyLoader.java b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseReaderLazyLoader.java
new file mode 100644
index 0000000000000..f73d2ca13c14a
--- /dev/null
+++ b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseReaderLazyLoader.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.ingest.geoip;
+
+import com.maxmind.geoip2.DatabaseReader;
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.common.CheckedSupplier;
+import org.elasticsearch.common.logging.Loggers;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+/**
+ * Facilitates lazy loading of the database reader, so that when the geoip plugin is installed, but not used,
+ * no memory is being wasted on the database reader.
+ */
+final class DatabaseReaderLazyLoader implements Closeable {
+
+    private static final Logger LOGGER = Loggers.getLogger(DatabaseReaderLazyLoader.class);
+
+    private final String databaseFileName;
+    private final CheckedSupplier<DatabaseReader, IOException> loader;
+    // package protected for testing only:
+    final SetOnce<DatabaseReader> databaseReader;
+
+    DatabaseReaderLazyLoader(String databaseFileName, CheckedSupplier<DatabaseReader, IOException> loader) {
+        this.databaseFileName = databaseFileName;
+        this.loader = loader;
+        this.databaseReader = new SetOnce<>();
+    }
+
+    synchronized DatabaseReader get() throws IOException {
+        if (databaseReader.get() == null) {
+            databaseReader.set(loader.get());
+            LOGGER.debug("Loaded [{}] geoip database", databaseFileName);
+        }
+        return databaseReader.get();
+    }
+
+    @Override
+    public synchronized void close() throws IOException {
+        IOUtils.close(databaseReader.get());
+    }
+}
diff --git a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpCache.java b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpCache.java
new file mode 100644
index 0000000000000..83a3374b504dd
--- /dev/null
+++ b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpCache.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.ingest.geoip;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.maxmind.db.NodeCache;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.common.cache.Cache;
+import org.elasticsearch.common.cache.CacheBuilder;
+
+import java.io.IOException;
+import java.util.concurrent.ExecutionException;
+
+final class GeoIpCache implements NodeCache {
+    private final Cache<Integer, JsonNode> cache;
+
+    GeoIpCache(long maxSize) {
+        this.cache = CacheBuilder.<Integer, JsonNode>builder().setMaximumWeight(maxSize).build();
+    }
+
+    @Override
+    public JsonNode get(int key, Loader loader) throws IOException {
+        try {
+            return cache.computeIfAbsent(key, loader::load);
+        } catch (ExecutionException e) {
+            Throwable cause = e.getCause() != null ? e.getCause() : e;
+            throw new ElasticsearchException(cause);
+        }
+    }
+}
diff --git a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
index 5923e3b690ec1..96902f852a162 100644
--- a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
+++ b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
@@ -19,19 +19,6 @@
 
 package org.elasticsearch.ingest.geoip;
 
-import java.io.IOException;
-import java.net.InetAddress;
-import java.security.AccessController;
-import java.security.PrivilegedAction;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
-
 import com.maxmind.geoip2.DatabaseReader;
 import com.maxmind.geoip2.exception.AddressNotFoundException;
 import com.maxmind.geoip2.model.CityResponse;
@@ -49,55 +36,84 @@
 import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.ingest.Processor;
 
+import java.io.IOException;
+import java.net.InetAddress;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+
 import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException;
+import static org.elasticsearch.ingest.ConfigurationUtils.readBooleanProperty;
 import static org.elasticsearch.ingest.ConfigurationUtils.readOptionalList;
 import static org.elasticsearch.ingest.ConfigurationUtils.readStringProperty;
 
 public final class GeoIpProcessor extends AbstractProcessor {
 
     public static final String TYPE = "geoip";
-    private static final String CITY_DB_TYPE = "GeoLite2-City";
-    private static final String COUNTRY_DB_TYPE = "GeoLite2-Country";
+    private static final String CITY_DB_SUFFIX = "-City";
+    private static final String COUNTRY_DB_SUFFIX = "-Country";
 
     private final String field;
     private final String targetField;
     private final DatabaseReader dbReader;
     private final Set<Property> properties;
+    private final boolean ignoreMissing;
 
-    GeoIpProcessor(String tag, String field, DatabaseReader dbReader, String targetField, Set<Property> properties) throws IOException {
+    GeoIpProcessor(String tag, String field, DatabaseReader dbReader, String targetField, Set<Property> properties,
+                   boolean ignoreMissing) throws IOException {
         super(tag);
         this.field = field;
         this.targetField = targetField;
         this.dbReader = dbReader;
         this.properties = properties;
+        this.ignoreMissing = ignoreMissing;
+    }
+
+    boolean isIgnoreMissing() {
+        return ignoreMissing;
     }
 
     @Override
     public void execute(IngestDocument ingestDocument) {
-        String ip = ingestDocument.getFieldValue(field, String.class);
+        String ip = ingestDocument.getFieldValue(field, String.class, ignoreMissing);
+
+        if (ip == null && ignoreMissing) {
+            return;
+        } else if (ip == null) {
+            throw new IllegalArgumentException("field [" + field + "] is null, cannot extract geoip information.");
+        }
+
         final InetAddress ipAddress = InetAddresses.forString(ip);
 
         Map<String, Object> geoData;
-        switch (dbReader.getMetadata().getDatabaseType()) {
-            case CITY_DB_TYPE:
-                try {
-                    geoData = retrieveCityGeoData(ipAddress);
-                } catch (AddressNotFoundRuntimeException e) {
-                    geoData = Collections.emptyMap();
-                }
-                break;
-            case COUNTRY_DB_TYPE:
-                try {
-                    geoData = retrieveCountryGeoData(ipAddress);
-                } catch (AddressNotFoundRuntimeException e) {
-                    geoData = Collections.emptyMap();
-                }
-                break;
-            default:
-                throw new ElasticsearchParseException("Unsupported database type [" + dbReader.getMetadata().getDatabaseType()
-                        + "]", new IllegalStateException());
+        String databaseType = dbReader.getMetadata().getDatabaseType();
+
+        if (databaseType.endsWith(CITY_DB_SUFFIX)) {
+            try {
+                geoData = retrieveCityGeoData(ipAddress);
+            } catch (AddressNotFoundRuntimeException e) {
+                geoData = Collections.emptyMap();
+            }
+        } else if (databaseType.endsWith(COUNTRY_DB_SUFFIX)) {
+            try {
+                geoData = retrieveCountryGeoData(ipAddress);
+            } catch (AddressNotFoundRuntimeException e) {
+                geoData = Collections.emptyMap();
+            }
+        } else {
+            throw new ElasticsearchParseException("Unsupported database type [" + dbReader.getMetadata().getDatabaseType()
+                    + "]", new IllegalStateException());
+        }
+        if (geoData.isEmpty() == false) {
+            ingestDocument.setFieldValue(targetField, geoData);
         }
-        ingestDocument.setFieldValue(targetField, geoData);
     }
 
     @Override
@@ -149,28 +165,50 @@ private Map<String, Object> retrieveCityGeoData(InetAddress ipAddress) {
                     geoData.put("ip", NetworkAddress.format(ipAddress));
                     break;
                 case COUNTRY_ISO_CODE:
-                    geoData.put("country_iso_code", country.getIsoCode());
+                    String countryIsoCode = country.getIsoCode();
+                    if (countryIsoCode != null) {
+                        geoData.put("country_iso_code", countryIsoCode);
+                    }
                     break;
                 case COUNTRY_NAME:
-                    geoData.put("country_name", country.getName());
+                    String countryName = country.getName();
+                    if (countryName != null) {
+                        geoData.put("country_name", countryName);
+                    }
                     break;
                 case CONTINENT_NAME:
-                    geoData.put("continent_name", continent.getName());
+                    String continentName = continent.getName();
+                    if (continentName != null) {
+                        geoData.put("continent_name", continentName);
+                    }
                     break;
                 case REGION_NAME:
-                    geoData.put("region_name", subdivision.getName());
+                    String subdivisionName = subdivision.getName();
+                    if (subdivisionName != null) {
+                        geoData.put("region_name", subdivisionName);
+                    }
                     break;
                 case CITY_NAME:
-                    geoData.put("city_name", city.getName());
+                    String cityName = city.getName();
+                    if (cityName != null) {
+                        geoData.put("city_name", cityName);
+                    }
                     break;
                 case TIMEZONE:
-                    geoData.put("timezone", location.getTimeZone());
+                    String locationTimeZone = location.getTimeZone();
+                    if (locationTimeZone != null) {
+                        geoData.put("timezone", locationTimeZone);
+                    }
                     break;
                 case LOCATION:
-                    Map<String, Object> locationObject = new HashMap<>();
-                    locationObject.put("lat", location.getLatitude());
-                    locationObject.put("lon", location.getLongitude());
-                    geoData.put("location", locationObject);
+                    Double latitude = location.getLatitude();
+                    Double longitude = location.getLongitude();
+                    if (latitude != null && longitude != null) {
+                        Map<String, Object> locationObject = new HashMap<>();
+                        locationObject.put("lat", latitude);
+                        locationObject.put("lon", longitude);
+                        geoData.put("location", locationObject);
+                    }
                     break;
             }
         }
@@ -202,13 +240,22 @@ private Map<String, Object> retrieveCountryGeoData(InetAddress ipAddress) {
                     geoData.put("ip", NetworkAddress.format(ipAddress));
                     break;
                 case COUNTRY_ISO_CODE:
-                    geoData.put("country_iso_code", country.getIsoCode());
+                    String countryIsoCode = country.getIsoCode();
+                    if (countryIsoCode != null) {
+                        geoData.put("country_iso_code", countryIsoCode);
+                    }
                     break;
                 case COUNTRY_NAME:
-                    geoData.put("country_name", country.getName());
+                    String countryName = country.getName();
+                    if (countryName != null) {
+                        geoData.put("country_name", countryName);
+                    }
                     break;
                 case CONTINENT_NAME:
-                    geoData.put("continent_name", continent.getName());
+                    String continentName = continent.getName();
+                    if (continentName != null) {
+                        geoData.put("continent_name", continentName);
+                    }
                     break;
             }
         }
@@ -222,9 +269,9 @@ public static final class Factory implements Processor.Factory {
         );
         static final Set<Property> DEFAULT_COUNTRY_PROPERTIES = EnumSet.of(Property.CONTINENT_NAME, Property.COUNTRY_ISO_CODE);
 
-        private final Map<String, DatabaseReader> databaseReaders;
+        private final Map<String, DatabaseReaderLazyLoader> databaseReaders;
 
-        public Factory(Map<String, DatabaseReader> databaseReaders) {
+        public Factory(Map<String, DatabaseReaderLazyLoader> databaseReaders) {
             this.databaseReaders = databaseReaders;
         }
 
@@ -235,13 +282,15 @@ public GeoIpProcessor create(Map<String, Processor.Factory> registry, String pro
             String targetField = readStringProperty(TYPE, processorTag, config, "target_field", "geoip");
             String databaseFile = readStringProperty(TYPE, processorTag, config, "database_file", "GeoLite2-City.mmdb.gz");
             List<String> propertyNames = readOptionalList(TYPE, processorTag, config, "properties");
+            boolean ignoreMissing = readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
 
-            DatabaseReader databaseReader = databaseReaders.get(databaseFile);
-            if (databaseReader == null) {
+            DatabaseReaderLazyLoader lazyLoader = databaseReaders.get(databaseFile);
+            if (lazyLoader == null) {
                 throw newConfigurationException(TYPE, processorTag,
                     "database_file", "database file [" + databaseFile + "] doesn't exist");
             }
 
+            DatabaseReader databaseReader = lazyLoader.get();
             String databaseType = databaseReader.getMetadata().getDatabaseType();
 
             final Set<Property> properties;
@@ -255,9 +304,9 @@ public GeoIpProcessor create(Map<String, Processor.Factory> registry, String pro
                     }
                 }
             } else {
-                if (CITY_DB_TYPE.equals(databaseType)) {
+                if (databaseType.endsWith(CITY_DB_SUFFIX)) {
                     properties = DEFAULT_CITY_PROPERTIES;
-                } else if (COUNTRY_DB_TYPE.equals(databaseType)) {
+                } else if (databaseType.endsWith(COUNTRY_DB_SUFFIX)) {
                     properties = DEFAULT_COUNTRY_PROPERTIES;
                 } else {
                     throw newConfigurationException(TYPE, processorTag, "database_file", "Unsupported database type ["
@@ -265,7 +314,7 @@ public GeoIpProcessor create(Map<String, Processor.Factory> registry, String pro
                 }
             }
 
-            return new GeoIpProcessor(processorTag, ipField, databaseReader, targetField, properties);
+            return new GeoIpProcessor(processorTag, ipField, databaseReader, targetField, properties, ignoreMissing);
         }
     }
 
@@ -274,7 +323,7 @@ public GeoIpProcessor create(Map<String, Processor.Factory> registry, String pro
     // it with an unchecked exception.
     private static final class AddressNotFoundRuntimeException extends RuntimeException {
 
-        public AddressNotFoundRuntimeException(Throwable cause) {
+        AddressNotFoundRuntimeException(Throwable cause) {
             super(cause);
         }
     }
@@ -296,9 +345,9 @@ enum Property {
 
         public static Property parseProperty(String databaseType, String value) {
             Set<Property> validProperties = EnumSet.noneOf(Property.class);
-            if (CITY_DB_TYPE.equals(databaseType)) {
+            if (databaseType.endsWith(CITY_DB_SUFFIX)) {
                 validProperties = ALL_CITY_PROPERTIES;
-            } else if (COUNTRY_DB_TYPE.equals(databaseType)) {
+            } else if (databaseType.endsWith(COUNTRY_DB_SUFFIX)) {
                 validProperties = ALL_COUNTRY_PROPERTIES;
             }
 
diff --git a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java
index 2190036c7fcdf..1571bc99ea4a4 100644
--- a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java
+++ b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java
@@ -19,6 +19,15 @@
 
 package org.elasticsearch.ingest.geoip;
 
+import com.maxmind.db.NoCache;
+import com.maxmind.db.NodeCache;
+import com.maxmind.geoip2.DatabaseReader;
+import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.ingest.Processor;
+import org.elasticsearch.plugins.IngestPlugin;
+import org.elasticsearch.plugins.Plugin;
+
 import java.io.Closeable;
 import java.io.IOException;
 import java.io.InputStream;
@@ -26,43 +35,53 @@
 import java.nio.file.Path;
 import java.nio.file.PathMatcher;
 import java.nio.file.StandardOpenOption;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 import java.util.stream.Stream;
 import java.util.zip.GZIPInputStream;
 
-import com.maxmind.geoip2.DatabaseReader;
-import org.apache.lucene.util.IOUtils;
-import org.elasticsearch.ingest.Processor;
-import org.elasticsearch.plugins.IngestPlugin;
-import org.elasticsearch.plugins.Plugin;
-
 public class IngestGeoIpPlugin extends Plugin implements IngestPlugin, Closeable {
+    public static final Setting<Long> CACHE_SIZE =
+        Setting.longSetting("ingest.geoip.cache_size", 1000, 0, Setting.Property.NodeScope);
+
+    private Map<String, DatabaseReaderLazyLoader> databaseReaders;
 
-    private Map<String, DatabaseReader> databaseReaders;
+    @Override
+    public List<Setting<?>> getSettings() {
+        return Arrays.asList(CACHE_SIZE);
+    }
 
     @Override
     public Map<String, Processor.Factory> getProcessors(Processor.Parameters parameters) {
         if (databaseReaders != null) {
-            throw new IllegalStateException("called onModule twice for geoip plugin!!");
+            throw new IllegalStateException("getProcessors called twice for geoip plugin!!");
         }
         Path geoIpConfigDirectory = parameters.env.configFile().resolve("ingest-geoip");
+        NodeCache cache;
+        long cacheSize = CACHE_SIZE.get(parameters.env.settings());
+        if (cacheSize > 0) {
+            cache = new GeoIpCache(cacheSize);
+        } else {
+            cache = NoCache.getInstance();
+        }
         try {
-            databaseReaders = loadDatabaseReaders(geoIpConfigDirectory);
+            databaseReaders = loadDatabaseReaders(geoIpConfigDirectory, cache);
         } catch (IOException e) {
             throw new RuntimeException(e);
         }
         return Collections.singletonMap(GeoIpProcessor.TYPE, new GeoIpProcessor.Factory(databaseReaders));
     }
 
-    static Map<String, DatabaseReader> loadDatabaseReaders(Path geoIpConfigDirectory) throws IOException {
+    static Map<String, DatabaseReaderLazyLoader> loadDatabaseReaders(Path geoIpConfigDirectory, NodeCache cache) throws IOException {
         if (Files.exists(geoIpConfigDirectory) == false && Files.isDirectory(geoIpConfigDirectory)) {
             throw new IllegalStateException("the geoip directory [" + geoIpConfigDirectory  + "] containing databases doesn't exist");
         }
 
-        Map<String, DatabaseReader> databaseReaders = new HashMap<>();
+        Map<String, DatabaseReaderLazyLoader> databaseReaders = new HashMap<>();
         try (Stream<Path> databaseFiles = Files.list(geoIpConfigDirectory)) {
             PathMatcher pathMatcher = geoIpConfigDirectory.getFileSystem().getPathMatcher("glob:**.mmdb.gz");
             // Use iterator instead of forEach otherwise IOException needs to be caught twice...
@@ -70,9 +89,13 @@ static Map<String, DatabaseReader> loadDatabaseReaders(Path geoIpConfigDirectory
             while (iterator.hasNext()) {
                 Path databasePath = iterator.next();
                 if (Files.isRegularFile(databasePath) && pathMatcher.matches(databasePath)) {
-                    try (InputStream inputStream = new GZIPInputStream(Files.newInputStream(databasePath, StandardOpenOption.READ))) {
-                        databaseReaders.put(databasePath.getFileName().toString(), new DatabaseReader.Builder(inputStream).build());
-                    }
+                    String databaseFileName = databasePath.getFileName().toString();
+                    DatabaseReaderLazyLoader holder = new DatabaseReaderLazyLoader(databaseFileName, () -> {
+                        try (InputStream inputStream = new GZIPInputStream(Files.newInputStream(databasePath, StandardOpenOption.READ))) {
+                            return new DatabaseReader.Builder(inputStream).withCache(cache).build();
+                        }
+                    });
+                    databaseReaders.put(databaseFileName, holder);
                 }
             }
         }
@@ -85,4 +108,5 @@ public void close() throws IOException {
             IOUtils.close(databaseReaders.values());
         }
     }
+
 }
diff --git a/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpCacheTests.java b/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpCacheTests.java
new file mode 100644
index 0000000000000..71cab99115fc7
--- /dev/null
+++ b/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpCacheTests.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.ingest.geoip;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.IntNode;
+import com.maxmind.db.NodeCache;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.test.ESTestCase;
+
+public class GeoIpCacheTests extends ESTestCase {
+    public void testCachesAndEvictsResults() throws Exception {
+        GeoIpCache cache = new GeoIpCache(1);
+        final NodeCache.Loader loader = key -> new IntNode(key);
+
+        JsonNode jsonNode1 = cache.get(1, loader);
+        assertSame(jsonNode1, cache.get(1, loader));
+
+        // evict old key by adding another value
+        cache.get(2, loader);
+
+        assertNotSame(jsonNode1, cache.get(1, loader));
+    }
+
+    public void testThrowsElasticsearchException() throws Exception {
+        GeoIpCache cache = new GeoIpCache(1);
+        NodeCache.Loader loader = (int key) -> {
+            throw new IllegalArgumentException("Illegal key");
+        };
+        ElasticsearchException ex = expectThrows(ElasticsearchException.class, () -> cache.get(1, loader));
+        assertTrue("Expected cause to be of type IllegalArgumentException but was [" + ex.getCause().getClass() + "]",
+            ex.getCause() instanceof IllegalArgumentException);
+        assertEquals("Illegal key", ex.getCause().getMessage());
+    }
+}
diff --git a/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java b/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java
index ec4db09cd960f..3904b043a5255 100644
--- a/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java
+++ b/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java
@@ -20,7 +20,8 @@
 package org.elasticsearch.ingest.geoip;
 
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
-import com.maxmind.geoip2.DatabaseReader;
+import com.maxmind.db.NoCache;
+import com.maxmind.db.NodeCache;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.test.ESTestCase;
@@ -46,7 +47,7 @@
 
 public class GeoIpProcessorFactoryTests extends ESTestCase {
 
-    private static Map<String, DatabaseReader> databaseReaders;
+    private static Map<String, DatabaseReaderLazyLoader> databaseReaders;
 
     @BeforeClass
     public static void loadDatabaseReaders() throws IOException {
@@ -57,12 +58,14 @@ public static void loadDatabaseReaders() throws IOException {
                 geoIpConfigDir.resolve("GeoLite2-City.mmdb.gz"));
         Files.copy(new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/GeoLite2-Country.mmdb.gz")),
                 geoIpConfigDir.resolve("GeoLite2-Country.mmdb.gz"));
-        databaseReaders = IngestGeoIpPlugin.loadDatabaseReaders(geoIpConfigDir);
+
+        NodeCache cache = randomFrom(NoCache.getInstance(), new GeoIpCache(randomNonNegativeLong()));
+        databaseReaders = IngestGeoIpPlugin.loadDatabaseReaders(geoIpConfigDir, cache);
     }
 
     @AfterClass
     public static void closeDatabaseReaders() throws IOException {
-        for (DatabaseReader reader : databaseReaders.values()) {
+        for (DatabaseReaderLazyLoader reader : databaseReaders.values()) {
             reader.close();
         }
         databaseReaders = null;
@@ -73,7 +76,24 @@ public void testBuildDefaults() throws Exception {
 
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
-        String processorTag = randomAsciiOfLength(10);
+        String processorTag = randomAlphaOfLength(10);
+
+        GeoIpProcessor processor = factory.create(null, processorTag, config);
+        assertThat(processor.getTag(), equalTo(processorTag));
+        assertThat(processor.getField(), equalTo("_field"));
+        assertThat(processor.getTargetField(), equalTo("geoip"));
+        assertThat(processor.getDbReader().getMetadata().getDatabaseType(), equalTo("GeoLite2-City"));
+        assertThat(processor.getProperties(), sameInstance(GeoIpProcessor.Factory.DEFAULT_CITY_PROPERTIES));
+        assertFalse(processor.isIgnoreMissing());
+    }
+
+    public void testSetIgnoreMissing() throws Exception {
+        GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders);
+
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", "_field");
+        config.put("ignore_missing", true);
+        String processorTag = randomAlphaOfLength(10);
 
         GeoIpProcessor processor = factory.create(null, processorTag, config);
         assertThat(processor.getTag(), equalTo(processorTag));
@@ -81,6 +101,7 @@ public void testBuildDefaults() throws Exception {
         assertThat(processor.getTargetField(), equalTo("geoip"));
         assertThat(processor.getDbReader().getMetadata().getDatabaseType(), equalTo("GeoLite2-City"));
         assertThat(processor.getProperties(), sameInstance(GeoIpProcessor.Factory.DEFAULT_CITY_PROPERTIES));
+        assertTrue(processor.isIgnoreMissing());
     }
 
     public void testCountryBuildDefaults() throws Exception {
@@ -89,7 +110,7 @@ public void testCountryBuildDefaults() throws Exception {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
         config.put("database_file", "GeoLite2-Country.mmdb.gz");
-        String processorTag = randomAsciiOfLength(10);
+        String processorTag = randomAlphaOfLength(10);
 
         GeoIpProcessor processor = factory.create(null, processorTag, config);
 
@@ -98,6 +119,7 @@ public void testCountryBuildDefaults() throws Exception {
         assertThat(processor.getTargetField(), equalTo("geoip"));
         assertThat(processor.getDbReader().getMetadata().getDatabaseType(), equalTo("GeoLite2-Country"));
         assertThat(processor.getProperties(), sameInstance(GeoIpProcessor.Factory.DEFAULT_COUNTRY_PROPERTIES));
+        assertFalse(processor.isIgnoreMissing());
     }
 
     public void testBuildTargetField() throws Exception {
@@ -108,6 +130,7 @@ public void testBuildTargetField() throws Exception {
         GeoIpProcessor processor = factory.create(null, null, config);
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getTargetField(), equalTo("_field"));
+        assertFalse(processor.isIgnoreMissing());
     }
 
     public void testBuildDbFile() throws Exception {
@@ -120,6 +143,7 @@ public void testBuildDbFile() throws Exception {
         assertThat(processor.getTargetField(), equalTo("geoip"));
         assertThat(processor.getDbReader().getMetadata().getDatabaseType(), equalTo("GeoLite2-Country"));
         assertThat(processor.getProperties(), sameInstance(GeoIpProcessor.Factory.DEFAULT_COUNTRY_PROPERTIES));
+        assertFalse(processor.isIgnoreMissing());
     }
 
     public void testBuildWithCountryDbAndCityFields() throws Exception {
@@ -170,6 +194,7 @@ public void testBuildFields() throws Exception {
         GeoIpProcessor processor = factory.create(null, null, config);
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getProperties(), equalTo(properties));
+        assertFalse(processor.isIgnoreMissing());
     }
 
     public void testBuildIllegalFieldOption() throws Exception {
@@ -196,4 +221,37 @@ public void testBuildIllegalFieldOption() throws Exception {
             assertThat(e.getMessage(), equalTo("[properties] property isn't a list, but of type [java.lang.String]"));
         }
     }
+
+    public void testLazyLoading() throws Exception {
+        Path configDir = createTempDir();
+        Path geoIpConfigDir = configDir.resolve("ingest-geoip");
+        Files.createDirectories(geoIpConfigDir);
+        Files.copy(new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/GeoLite2-City.mmdb.gz")),
+            geoIpConfigDir.resolve("GeoLite2-City.mmdb.gz"));
+        Files.copy(new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/GeoLite2-Country.mmdb.gz")),
+            geoIpConfigDir.resolve("GeoLite2-Country.mmdb.gz"));
+
+        // Loading another database reader instances, because otherwise we can't test lazy loading as the the
+        // database readers used at class level are reused between tests. (we want to keep that otherwise running this
+        // test will take roughly 4 times more time)
+        Map<String, DatabaseReaderLazyLoader> databaseReaders =
+            IngestGeoIpPlugin.loadDatabaseReaders(geoIpConfigDir, NoCache.getInstance());
+        GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders);
+        for (DatabaseReaderLazyLoader lazyLoader : databaseReaders.values()) {
+            assertNull(lazyLoader.databaseReader.get());
+        }
+
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", "_field");
+        config.put("database_file", "GeoLite2-City.mmdb.gz");
+        factory.create(null, "_tag", config);
+        config = new HashMap<>();
+        config.put("field", "_field");
+        config.put("database_file", "GeoLite2-Country.mmdb.gz");
+        factory.create(null, "_tag", config);
+
+        for (DatabaseReaderLazyLoader lazyLoader : databaseReaders.values()) {
+            assertNotNull(lazyLoader.databaseReader.get());
+        }
+    }
 }
diff --git a/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java b/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java
index f3141c735d282..54c504ca5c579 100644
--- a/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java
+++ b/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java
@@ -26,20 +26,23 @@
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.zip.GZIPInputStream;
 
+import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
 
 public class GeoIpProcessorTests extends ESTestCase {
 
     public void testCity() throws Exception {
         InputStream database = getDatabaseFileInputStream("/GeoLite2-City.mmdb.gz");
-        GeoIpProcessor processor = new GeoIpProcessor(randomAsciiOfLength(10), "source_field",
-                new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class));
+        GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field",
+                new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false);
 
         Map<String, Object> document = new HashMap<>();
         document.put("source_field", "8.8.8.8");
@@ -63,10 +66,97 @@ public void testCity() throws Exception {
         assertThat(geoData.get("location"), equalTo(location));
     }
 
+    public void testNullValueWithIgnoreMissing() throws Exception {
+        InputStream database = getDatabaseFileInputStream("/GeoLite2-City.mmdb.gz");
+        GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field",
+            new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), true);
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(),
+            Collections.singletonMap("source_field", null));
+        IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
+        processor.execute(ingestDocument);
+        assertIngestDocument(originalIngestDocument, ingestDocument);
+    }
+
+    public void testNonExistentWithIgnoreMissing() throws Exception {
+        InputStream database = getDatabaseFileInputStream("/GeoLite2-City.mmdb.gz");
+        GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field",
+            new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), true);
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
+        IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
+        processor.execute(ingestDocument);
+        assertIngestDocument(originalIngestDocument, ingestDocument);
+    }
+
+    public void testNullWithoutIgnoreMissing() throws Exception {
+        InputStream database = getDatabaseFileInputStream("/GeoLite2-City.mmdb.gz");
+        GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field",
+            new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false);
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(),
+            Collections.singletonMap("source_field", null));
+        IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
+        Exception exception = expectThrows(Exception.class, () -> processor.execute(ingestDocument));
+        assertThat(exception.getMessage(), equalTo("field [source_field] is null, cannot extract geoip information."));
+    }
+
+    public void testNonExistentWithoutIgnoreMissing() throws Exception {
+        InputStream database = getDatabaseFileInputStream("/GeoLite2-City.mmdb.gz");
+        GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field",
+            new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false);
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
+        IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
+        Exception exception = expectThrows(Exception.class, () -> processor.execute(ingestDocument));
+        assertThat(exception.getMessage(), equalTo("field [source_field] not present as part of path [source_field]"));
+    }
+
+    public void testCity_withIpV6() throws Exception {
+        InputStream database = getDatabaseFileInputStream("/GeoLite2-City.mmdb.gz");
+        GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field",
+                new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false);
+
+        String address = "2602:306:33d3:8000::3257:9652";
+        Map<String, Object> document = new HashMap<>();
+        document.put("source_field", address);
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
+        processor.execute(ingestDocument);
+
+        assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(address));
+        @SuppressWarnings("unchecked")
+        Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
+        assertThat(geoData.size(), equalTo(8));
+        assertThat(geoData.get("ip"), equalTo(address));
+        assertThat(geoData.get("country_iso_code"), equalTo("US"));
+        assertThat(geoData.get("country_name"), equalTo("United States"));
+        assertThat(geoData.get("continent_name"), equalTo("North America"));
+        assertThat(geoData.get("region_name"), equalTo("Florida"));
+        assertThat(geoData.get("city_name"), equalTo("Hollywood"));
+        assertThat(geoData.get("timezone"), equalTo("America/New_York"));
+        Map<String, Object> location = new HashMap<>();
+        location.put("lat", 26.0252d);
+        location.put("lon", -80.296d);
+        assertThat(geoData.get("location"), equalTo(location));
+    }
+
+    public void testCityWithMissingLocation() throws Exception {
+        InputStream database = getDatabaseFileInputStream("/GeoLite2-City.mmdb.gz");
+        GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field",
+            new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false);
+
+        Map<String, Object> document = new HashMap<>();
+        document.put("source_field", "93.114.45.13");
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
+        processor.execute(ingestDocument);
+
+        assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo("93.114.45.13"));
+        @SuppressWarnings("unchecked")
+        Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
+        assertThat(geoData.size(), equalTo(1));
+        assertThat(geoData.get("ip"), equalTo("93.114.45.13"));
+    }
+
     public void testCountry() throws Exception {
         InputStream database = getDatabaseFileInputStream("/GeoLite2-Country.mmdb.gz");
-        GeoIpProcessor processor = new GeoIpProcessor(randomAsciiOfLength(10), "source_field",
-                new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class));
+        GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field",
+                new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false);
 
         Map<String, Object> document = new HashMap<>();
         document.put("source_field", "82.170.213.79");
@@ -83,25 +173,40 @@ public void testCountry() throws Exception {
         assertThat(geoData.get("continent_name"), equalTo("Europe"));
     }
 
+    public void testCountryWithMissingLocation() throws Exception {
+        InputStream database = getDatabaseFileInputStream("/GeoLite2-Country.mmdb.gz");
+        GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field",
+            new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false);
+
+        Map<String, Object> document = new HashMap<>();
+        document.put("source_field", "93.114.45.13");
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
+        processor.execute(ingestDocument);
+
+        assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo("93.114.45.13"));
+        @SuppressWarnings("unchecked")
+        Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
+        assertThat(geoData.size(), equalTo(1));
+        assertThat(geoData.get("ip"), equalTo("93.114.45.13"));
+    }
+
     public void testAddressIsNotInTheDatabase() throws Exception {
         InputStream database = getDatabaseFileInputStream("/GeoLite2-City.mmdb.gz");
-        GeoIpProcessor processor = new GeoIpProcessor(randomAsciiOfLength(10), "source_field",
-                new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class));
+        GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field",
+                new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false);
 
         Map<String, Object> document = new HashMap<>();
         document.put("source_field", "127.0.0.1");
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
         processor.execute(ingestDocument);
-        @SuppressWarnings("unchecked")
-        Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
-        assertThat(geoData.size(), equalTo(0));
+        assertThat(ingestDocument.getSourceAndMetadata().containsKey("target_field"), is(false));
     }
 
     /** Don't silently do DNS lookups or anything trappy on bogus data */
     public void testInvalid() throws Exception {
         InputStream database = getDatabaseFileInputStream("/GeoLite2-City.mmdb.gz");
-        GeoIpProcessor processor = new GeoIpProcessor(randomAsciiOfLength(10), "source_field",
-                new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class));
+        GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field",
+                new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false);
 
         Map<String, Object> document = new HashMap<>();
         document.put("source_field", "www.google.com");
@@ -115,7 +220,7 @@ public void testInvalid() throws Exception {
         }
     }
 
-    static InputStream getDatabaseFileInputStream(String path) throws IOException {
+    private static InputStream getDatabaseFileInputStream(String path) throws IOException {
         return new GZIPInputStream(GeoIpProcessor.class.getResourceAsStream(path));
     }
 
diff --git a/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java b/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java
index 26838b600dab3..9d9bdb9c7d003 100644
--- a/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java
+++ b/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java
@@ -24,9 +24,6 @@
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-
-import java.io.IOException;
 
 public class IngestGeoIpClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
@@ -35,8 +32,8 @@ public IngestGeoIpClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate te
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
-        return ESClientYamlSuiteTestCase.createParameters(0, 1);
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
     }
 }
 
diff --git a/plugins/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yaml b/plugins/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yaml
index 33e9ec1ca46ed..f662f34ab57c3 100644
--- a/plugins/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yaml
+++ b/plugins/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yaml
@@ -122,3 +122,76 @@
   - length: { _source.geoip: 2 }
   - match: { _source.geoip.country_iso_code: "US" }
   - match: { _source.geoip.continent_name: "North America" }
+
+---
+"Test geoip processor with geopoint mapping (both missing and including location)":
+  - do:
+      indices.create:
+        index: test
+        body:  >
+          {
+            "mappings" : {
+              "test" : {
+                "properties" : {
+                  "geoip.location" : {
+                    "type": "geo_point"
+                  }
+                }
+              }
+            }
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body:  >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "geoip" : {
+                  "field" : "field1"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: test
+        type: test
+        id: 1
+        pipeline: "my_pipeline"
+        body: { field1: "93.114.45.13" }
+
+  - do:
+      get:
+        index: test
+        type: test
+        id: 1
+  - match: { _source.field1: "93.114.45.13" }
+  - is_false: _source.geoip
+
+  - do:
+      index:
+        index: test
+        type: test
+        id: 2
+        pipeline: "my_pipeline"
+        body: { field1: "128.101.101.101" }
+
+  - do:
+      get:
+        index: test
+        type: test
+        id: 2
+  - match: { _source.field1: "128.101.101.101" }
+  - length: { _source.geoip: 5 }
+  - match: { _source.geoip.city_name: "Minneapolis" }
+  - match: { _source.geoip.country_iso_code: "US" }
+  - match: { _source.geoip.location.lon: -93.2166 }
+  - match: { _source.geoip.location.lat: 44.9759 }
+  - match: { _source.geoip.region_name: "Minnesota" }
+  - match: { _source.geoip.continent_name: "North America" }
diff --git a/plugins/ingest-user-agent/build.gradle b/plugins/ingest-user-agent/build.gradle
index ec599874d155b..2b2669a40042b 100644
--- a/plugins/ingest-user-agent/build.gradle
+++ b/plugins/ingest-user-agent/build.gradle
@@ -22,8 +22,6 @@ esplugin {
   classname 'org.elasticsearch.ingest.useragent.IngestUserAgentPlugin'
 }
 
-integTest {
-  cluster {
-    extraConfigFile 'ingest-user-agent/test-regexes.yaml', 'test/test-regexes.yaml'
-  }
-}
\ No newline at end of file
+integTestCluster {
+  extraConfigFile 'ingest-user-agent/test-regexes.yaml', 'test/test-regexes.yaml'
+}
diff --git a/plugins/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java b/plugins/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java
index af764d5baf204..0d9660e9fb5ea 100644
--- a/plugins/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java
+++ b/plugins/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.ingest.useragent;
 
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
@@ -39,11 +40,11 @@ final class UserAgentParser {
     private final List<UserAgentSubpattern> osPatterns = new ArrayList<>();
     private final List<UserAgentSubpattern> devicePatterns = new ArrayList<>();
     private final String name;
-    
-    public UserAgentParser(String name, InputStream regexStream, UserAgentCache cache) {
+
+    UserAgentParser(String name, InputStream regexStream, UserAgentCache cache) {
         this.name = name;
         this.cache = cache;
-        
+
         try {
             init(regexStream);
         } catch (IOException e) {
@@ -52,17 +53,18 @@ public UserAgentParser(String name, InputStream regexStream, UserAgentCache cach
     }
 
     private void init(InputStream regexStream) throws IOException {
-        XContentParser yamlParser = XContentFactory.xContent(XContentType.YAML).createParser(regexStream);
-        
+        // EMPTY is safe here because we don't use namedObject
+        XContentParser yamlParser = XContentFactory.xContent(XContentType.YAML).createParser(NamedXContentRegistry.EMPTY, regexStream);
+
         XContentParser.Token token = yamlParser.nextToken();
-        
+
         if (token == XContentParser.Token.START_OBJECT) {
             token = yamlParser.nextToken();
-            
+
             for (; token != null; token = yamlParser.nextToken()) {
                 if (token == XContentParser.Token.FIELD_NAME && yamlParser.currentName().equals("user_agent_parsers")) {
                     List<Map<String, String>> parserConfigurations = readParserConfigurations(yamlParser);
-                    
+
                     for (Map<String, String> map : parserConfigurations) {
                         uaPatterns.add(new UserAgentSubpattern(compilePattern(map.get("regex"), map.get("regex_flag")),
                                 map.get("family_replacement"), map.get("v1_replacement"), map.get("v2_replacement"),
@@ -71,7 +73,7 @@ private void init(InputStream regexStream) throws IOException {
                 }
                 else if (token == XContentParser.Token.FIELD_NAME && yamlParser.currentName().equals("os_parsers")) {
                     List<Map<String, String>> parserConfigurations = readParserConfigurations(yamlParser);
-                    
+
                     for (Map<String, String> map : parserConfigurations) {
                         osPatterns.add(new UserAgentSubpattern(compilePattern(map.get("regex"), map.get("regex_flag")),
                                 map.get("os_replacement"), map.get("os_v1_replacement"), map.get("os_v2_replacement"),
@@ -80,7 +82,7 @@ else if (token == XContentParser.Token.FIELD_NAME && yamlParser.currentName().eq
                 }
                 else if (token == XContentParser.Token.FIELD_NAME && yamlParser.currentName().equals("device_parsers")) {
                     List<Map<String, String>> parserConfigurations = readParserConfigurations(yamlParser);
-                    
+
                     for (Map<String, String> map : parserConfigurations) {
                         devicePatterns.add(new UserAgentSubpattern(compilePattern(map.get("regex"), map.get("regex_flag")),
                                 map.get("device_replacement"), null, null, null, null));
@@ -88,7 +90,7 @@ else if (token == XContentParser.Token.FIELD_NAME && yamlParser.currentName().eq
                 }
             }
         }
-        
+
         if (uaPatterns.isEmpty() && osPatterns.isEmpty() && devicePatterns.isEmpty()) {
             throw new ElasticsearchParseException("not a valid regular expression file");
         }
@@ -105,38 +107,38 @@ private Pattern compilePattern(String regex, String regex_flag) {
 
     private List<Map<String, String>> readParserConfigurations(XContentParser yamlParser) throws IOException {
         List <Map<String, String>> patternList = new ArrayList<>();
-        
+
         XContentParser.Token token = yamlParser.nextToken();
         if (token != XContentParser.Token.START_ARRAY) {
             throw new ElasticsearchParseException("malformed regular expression file, should continue with 'array' after 'object'");
         }
-        
+
         token = yamlParser.nextToken();
         if (token != XContentParser.Token.START_OBJECT) {
             throw new ElasticsearchParseException("malformed regular expression file, expecting 'object'");
         }
-        
+
         while (token == XContentParser.Token.START_OBJECT) {
             token = yamlParser.nextToken();
-            
+
             if (token != XContentParser.Token.FIELD_NAME) {
                 throw new ElasticsearchParseException("malformed regular expression file, should continue with 'field_name' after 'array'");
             }
-            
+
             Map<String, String> regexMap = new HashMap<>();
             for (; token == XContentParser.Token.FIELD_NAME; token = yamlParser.nextToken()) {
                 String fieldName = yamlParser.currentName();
-                
+
                 token = yamlParser.nextToken();
                 String fieldValue = yamlParser.text();
                 regexMap.put(fieldName, fieldValue);
             }
 
             patternList.add(regexMap);
-            
+
             token = yamlParser.nextToken();
         }
-        
+
         return patternList;
     }
 
@@ -151,21 +153,21 @@ List<UserAgentSubpattern> getOsPatterns() {
     List<UserAgentSubpattern> getDevicePatterns() {
         return devicePatterns;
     }
-    
+
     String getName() {
         return name;
     }
 
     public Details parse(String agentString) {
         Details details = cache.get(name, agentString);;
-        
+
         if (details == null) {
             VersionedName userAgent = findMatch(uaPatterns, agentString);
             VersionedName operatingSystem = findMatch(osPatterns, agentString);
             VersionedName device = findMatch(devicePatterns, agentString);
-            
+
             details = new Details(userAgent, operatingSystem, device);
-            
+
             cache.put(name, agentString, details);
         }
 
@@ -181,7 +183,7 @@ private VersionedName findMatch(List<UserAgentSubpattern> possiblePatterns, Stri
                 return name;
             }
         }
-        
+
         return null;
     }
 
@@ -189,22 +191,22 @@ static final class Details {
         public final VersionedName userAgent;
         public final VersionedName operatingSystem;
         public final VersionedName device;
-        
-        public Details(VersionedName userAgent, VersionedName operatingSystem, VersionedName device) {
+
+        Details(VersionedName userAgent, VersionedName operatingSystem, VersionedName device) {
             this.userAgent = userAgent;
             this.operatingSystem = operatingSystem;
             this.device = device;
         }
     }
-    
+
     static final class VersionedName {
         public final String name;
         public final String major;
         public final String minor;
         public final String patch;
         public final String build;
-        
-        public VersionedName(String name, String major, String minor, String patch, String build) {
+
+        VersionedName(String name, String major, String minor, String patch, String build) {
             this.name = name;
             this.major = major;
             this.minor = minor;
@@ -212,7 +214,7 @@ public VersionedName(String name, String major, String minor, String patch, Stri
             this.build = build;
         }
     }
-    
+
     /**
      * One of: user agent, operating system, device
      */
@@ -220,7 +222,7 @@ static final class UserAgentSubpattern {
         private final Pattern pattern;
         private final String nameReplacement, v1Replacement, v2Replacement, v3Replacement, v4Replacement;
 
-        public UserAgentSubpattern(Pattern pattern, String nameReplacement,
+        UserAgentSubpattern(Pattern pattern, String nameReplacement,
                 String v1Replacement, String v2Replacement, String v3Replacement, String v4Replacement) {
           this.pattern = pattern;
           this.nameReplacement = nameReplacement;
@@ -261,19 +263,19 @@ public VersionedName match(String agentString) {
           } else if (groupCount >= 3) {
             minor = matcher.group(3);
           }
-          
+
           if (v3Replacement != null) {
               patch = v3Replacement;
           } else if (groupCount >= 4) {
               patch = matcher.group(4);
           }
-          
+
           if (v4Replacement != null) {
               build = v4Replacement;
           } else if (groupCount >= 5) {
               build = matcher.group(5);
           }
-          
+
           return name == null ? null : new VersionedName(name, major, minor, patch, build);
         }
       }
diff --git a/plugins/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java b/plugins/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java
index ec18126457b4a..93f210c427b51 100644
--- a/plugins/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java
+++ b/plugins/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java
@@ -34,6 +34,7 @@
 import java.util.Set;
 
 import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException;
+import static org.elasticsearch.ingest.ConfigurationUtils.readBooleanProperty;
 import static org.elasticsearch.ingest.ConfigurationUtils.readOptionalList;
 import static org.elasticsearch.ingest.ConfigurationUtils.readStringProperty;
 
@@ -44,20 +45,32 @@ public class UserAgentProcessor extends AbstractProcessor {
     private final String field;
     private final String targetField;
     private final Set<Property> properties;
-    
     private final UserAgentParser parser;
+    private final boolean ignoreMissing;
 
-    public UserAgentProcessor(String tag, String field, String targetField, UserAgentParser parser, Set<Property> properties) {
+    public UserAgentProcessor(String tag, String field, String targetField, UserAgentParser parser, Set<Property> properties,
+                              boolean ignoreMissing) {
         super(tag);
         this.field = field;
         this.targetField = targetField;
         this.parser = parser;
         this.properties = properties;
+        this.ignoreMissing = ignoreMissing;
+    }
+
+    boolean isIgnoreMissing() {
+        return ignoreMissing;
     }
 
     @Override
     public void execute(IngestDocument ingestDocument) throws Exception {
-        String userAgent = ingestDocument.getFieldValue(field, String.class);
+        String userAgent = ingestDocument.getFieldValue(field, String.class, ignoreMissing);
+
+        if (userAgent == null && ignoreMissing) {
+            return;
+        } else if (userAgent == null) {
+            throw new IllegalArgumentException("field [" + field + "] is null, cannot parse user-agent.");
+        }
 
         Details uaClient = parser.parse(userAgent);
 
@@ -99,7 +112,7 @@ public void execute(IngestDocument ingestDocument) throws Exception {
                     else {
                         uaDetails.put("os", "Other");
                     }
-                    
+
                     break;
                 case OS_NAME:
                     if (uaClient.operatingSystem != null && uaClient.operatingSystem.name != null) {
@@ -168,7 +181,7 @@ private String buildFullOSName(VersionedName operatingSystem) {
     public String getType() {
         return TYPE;
     }
-    
+
     String getField() {
         return field;
     }
@@ -180,7 +193,7 @@ String getTargetField() {
     Set<Property> getProperties() {
         return properties;
     }
-    
+
     UserAgentParser getUaParser() {
         return parser;
     }
@@ -188,7 +201,7 @@ UserAgentParser getUaParser() {
     public static final class Factory implements Processor.Factory {
 
         private final Map<String, UserAgentParser> userAgentParsers;
-        
+
         public Factory(Map<String, UserAgentParser> userAgentParsers) {
             this.userAgentParsers = userAgentParsers;
         }
@@ -200,13 +213,14 @@ public UserAgentProcessor create(Map<String, Processor.Factory> factories, Strin
             String targetField = readStringProperty(TYPE, processorTag, config, "target_field", "user_agent");
             String regexFilename = readStringProperty(TYPE, processorTag, config, "regex_file", IngestUserAgentPlugin.DEFAULT_PARSER_NAME);
             List<String> propertyNames = readOptionalList(TYPE, processorTag, config, "properties");
+            boolean ignoreMissing = readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
 
             UserAgentParser parser = userAgentParsers.get(regexFilename);
             if (parser == null) {
                 throw newConfigurationException(TYPE, processorTag,
                         "regex_file", "regex file [" + regexFilename + "] doesn't exist (has to exist at node startup)");
             }
-            
+
             final Set<Property> properties;
             if (propertyNames != null) {
                 properties = EnumSet.noneOf(Property.class);
@@ -221,7 +235,7 @@ public UserAgentProcessor create(Map<String, Processor.Factory> factories, Strin
                 properties = EnumSet.allOf(Property.class);
             }
 
-            return new UserAgentProcessor(processorTag, field, targetField, parser, properties);
+            return new UserAgentProcessor(processorTag, field, targetField, parser, properties, ignoreMissing);
         }
     }
 
diff --git a/plugins/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/IngestUserAgentClientYamlTestSuiteIT.java b/plugins/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/IngestUserAgentClientYamlTestSuiteIT.java
index b0aa115a1a26c..9587b4e514c59 100644
--- a/plugins/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/IngestUserAgentClientYamlTestSuiteIT.java
+++ b/plugins/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/IngestUserAgentClientYamlTestSuiteIT.java
@@ -24,9 +24,6 @@
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-
-import java.io.IOException;
 
 public class IngestUserAgentClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
@@ -35,7 +32,7 @@ public IngestUserAgentClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidat
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
-        return ESClientYamlSuiteTestCase.createParameters(0, 1);
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/plugins/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java b/plugins/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java
index 10c6ccd794133..4e0d0fb3695c8 100644
--- a/plugins/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java
+++ b/plugins/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java
@@ -79,7 +79,7 @@ public void testBuildDefaults() throws Exception {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
 
-        String processorTag = randomAsciiOfLength(10);
+        String processorTag = randomAlphaOfLength(10);
 
         UserAgentProcessor processor = factory.create(null, processorTag, config);
         assertThat(processor.getTag(), equalTo(processorTag));
@@ -89,6 +89,27 @@ public void testBuildDefaults() throws Exception {
         assertThat(processor.getUaParser().getOsPatterns().size(), greaterThan(0));
         assertThat(processor.getUaParser().getDevicePatterns().size(), greaterThan(0));
         assertThat(processor.getProperties(), equalTo(EnumSet.allOf(UserAgentProcessor.Property.class)));
+        assertFalse(processor.isIgnoreMissing());
+    }
+
+    public void testBuildWithIgnoreMissing() throws Exception {
+        UserAgentProcessor.Factory factory = new UserAgentProcessor.Factory(userAgentParsers);
+
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", "_field");
+        config.put("ignore_missing", true);
+
+        String processorTag = randomAlphaOfLength(10);
+
+        UserAgentProcessor processor = factory.create(null, processorTag, config);
+        assertThat(processor.getTag(), equalTo(processorTag));
+        assertThat(processor.getField(), equalTo("_field"));
+        assertThat(processor.getTargetField(), equalTo("user_agent"));
+        assertThat(processor.getUaParser().getUaPatterns().size(), greaterThan(0));
+        assertThat(processor.getUaParser().getOsPatterns().size(), greaterThan(0));
+        assertThat(processor.getUaParser().getDevicePatterns().size(), greaterThan(0));
+        assertThat(processor.getProperties(), equalTo(EnumSet.allOf(UserAgentProcessor.Property.class)));
+        assertTrue(processor.isIgnoreMissing());
     }
 
     public void testBuildTargetField() throws Exception {
diff --git a/plugins/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java b/plugins/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java
index d9b5eed059a25..92e94434ccdbe 100644
--- a/plugins/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java
+++ b/plugins/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java
@@ -21,61 +21,101 @@
 
 import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.ingest.IngestDocument;
-import org.elasticsearch.ingest.useragent.UserAgentProcessor;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.BeforeClass;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.Map;
 
+import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.is;
 
 public class UserAgentProcessorTests extends ESTestCase {
 
     private static UserAgentProcessor processor;
-    
+
     @BeforeClass
     public static void setupProcessor() throws IOException {
         InputStream regexStream = UserAgentProcessor.class.getResourceAsStream("/regexes.yaml");
         assertNotNull(regexStream);
-        
-        UserAgentParser parser = new UserAgentParser(randomAsciiOfLength(10), regexStream, new UserAgentCache(1000));
-        
-        processor = new UserAgentProcessor(randomAsciiOfLength(10), "source_field", "target_field", parser,
-                EnumSet.allOf(UserAgentProcessor.Property.class));
+
+        UserAgentParser parser = new UserAgentParser(randomAlphaOfLength(10), regexStream, new UserAgentCache(1000));
+
+        processor = new UserAgentProcessor(randomAlphaOfLength(10), "source_field", "target_field", parser,
+                EnumSet.allOf(UserAgentProcessor.Property.class), false);
+    }
+
+    public void testNullValueWithIgnoreMissing() throws Exception {
+        UserAgentProcessor processor = new UserAgentProcessor(randomAlphaOfLength(10), "source_field", "target_field", null,
+            EnumSet.allOf(UserAgentProcessor.Property.class), true);
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(),
+            Collections.singletonMap("source_field", null));
+        IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
+        processor.execute(ingestDocument);
+        assertIngestDocument(originalIngestDocument, ingestDocument);
+    }
+
+    public void testNonExistentWithIgnoreMissing() throws Exception {
+        UserAgentProcessor processor = new UserAgentProcessor(randomAlphaOfLength(10), "source_field", "target_field", null,
+            EnumSet.allOf(UserAgentProcessor.Property.class), true);
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
+        IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
+        processor.execute(ingestDocument);
+        assertIngestDocument(originalIngestDocument, ingestDocument);
     }
-    
+
+    public void testNullWithoutIgnoreMissing() throws Exception {
+        UserAgentProcessor processor = new UserAgentProcessor(randomAlphaOfLength(10), "source_field", "target_field", null,
+            EnumSet.allOf(UserAgentProcessor.Property.class), false);
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(),
+            Collections.singletonMap("source_field", null));
+        IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
+        Exception exception = expectThrows(Exception.class, () -> processor.execute(ingestDocument));
+        assertThat(exception.getMessage(), equalTo("field [source_field] is null, cannot parse user-agent."));
+    }
+
+    public void testNonExistentWithoutIgnoreMissing() throws Exception {
+        UserAgentProcessor processor = new UserAgentProcessor(randomAlphaOfLength(10), "source_field", "target_field", null,
+            EnumSet.allOf(UserAgentProcessor.Property.class), false);
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
+        IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
+        Exception exception = expectThrows(Exception.class, () -> processor.execute(ingestDocument));
+        assertThat(exception.getMessage(), equalTo("field [source_field] not present as part of path [source_field]"));
+    }
+
     @SuppressWarnings("unchecked")
     public void testCommonBrowser() throws Exception {
         Map<String, Object> document = new HashMap<>();
         document.put("source_field",
             "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36");
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
-        
+
         processor.execute(ingestDocument);
         Map<String, Object> data = ingestDocument.getSourceAndMetadata();
 
         assertThat(data, hasKey("target_field"));
         Map<String, Object> target = (Map<String, Object>) data.get("target_field");
-        
+
         assertThat(target.get("name"), is("Chrome"));
         assertThat(target.get("major"), is("33"));
         assertThat(target.get("minor"), is("0"));
         assertThat(target.get("patch"), is("1750"));
         assertNull(target.get("build"));
-        
+
         assertThat(target.get("os"), is("Mac OS X 10.9.2"));
         assertThat(target.get("os_name"), is("Mac OS X"));
         assertThat(target.get("os_major"), is("10"));
         assertThat(target.get("os_minor"), is("9"));
-        
+
         assertThat(target.get("device"), is("Other"));
     }
-    
+
     @SuppressWarnings("unchecked")
     public void testUncommonDevice() throws Exception {
         Map<String, Object> document = new HashMap<>();
@@ -83,78 +123,78 @@ public void testUncommonDevice() throws Exception {
                 "Mozilla/5.0 (Linux; U; Android 3.0; en-us; Xoom Build/HRI39) AppleWebKit/525.10+ "
                 + "(KHTML, like Gecko) Version/3.0.4 Mobile Safari/523.12.2");
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
-        
+
         processor.execute(ingestDocument);
         Map<String, Object> data = ingestDocument.getSourceAndMetadata();
 
         assertThat(data, hasKey("target_field"));
         Map<String, Object> target = (Map<String, Object>) data.get("target_field");
-        
+
         assertThat(target.get("name"), is("Android"));
         assertThat(target.get("major"), is("3"));
         assertThat(target.get("minor"), is("0"));
         assertNull(target.get("patch"));
         assertNull(target.get("build"));
-        
+
         assertThat(target.get("os"), is("Android 3.0"));
         assertThat(target.get("os_name"), is("Android"));
         assertThat(target.get("os_major"), is("3"));
         assertThat(target.get("os_minor"), is("0"));
-        
+
         assertThat(target.get("device"), is("Motorola Xoom"));
     }
-    
+
     @SuppressWarnings("unchecked")
     public void testSpider() throws Exception {
         Map<String, Object> document = new HashMap<>();
         document.put("source_field",
             "Mozilla/5.0 (compatible; EasouSpider; +http://www.easou.com/search/spider.html)");
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
-        
+
         processor.execute(ingestDocument);
         Map<String, Object> data = ingestDocument.getSourceAndMetadata();
 
         assertThat(data, hasKey("target_field"));
         Map<String, Object> target = (Map<String, Object>) data.get("target_field");
-        
+
         assertThat(target.get("name"), is("EasouSpider"));
         assertNull(target.get("major"));
         assertNull(target.get("minor"));
         assertNull(target.get("patch"));
         assertNull(target.get("build"));
-        
+
         assertThat(target.get("os"), is("Other"));
         assertThat(target.get("os_name"), is("Other"));
         assertNull(target.get("os_major"));
         assertNull(target.get("os_minor"));
-        
+
         assertThat(target.get("device"), is("Spider"));
     }
-    
+
     @SuppressWarnings("unchecked")
     public void testUnknown() throws Exception {
         Map<String, Object> document = new HashMap<>();
         document.put("source_field",
             "Something I made up v42.0.1");
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
-        
+
         processor.execute(ingestDocument);
         Map<String, Object> data = ingestDocument.getSourceAndMetadata();
 
         assertThat(data, hasKey("target_field"));
         Map<String, Object> target = (Map<String, Object>) data.get("target_field");
-        
+
         assertThat(target.get("name"), is("Other"));
         assertNull(target.get("major"));
         assertNull(target.get("minor"));
         assertNull(target.get("patch"));
         assertNull(target.get("build"));
-        
+
         assertThat(target.get("os"), is("Other"));
         assertThat(target.get("os_name"), is("Other"));
         assertNull(target.get("os_major"));
         assertNull(target.get("os_minor"));
-        
+
         assertThat(target.get("device"), is("Other"));
     }
 }
diff --git a/plugins/jvm-example/build.gradle b/plugins/jvm-example/build.gradle
index c5828d9c86df5..78e54d8bc817e 100644
--- a/plugins/jvm-example/build.gradle
+++ b/plugins/jvm-example/build.gradle
@@ -21,6 +21,9 @@ esplugin {
   description 'Demonstrates all the pluggable Java entry points in Elasticsearch'
   classname 'org.elasticsearch.plugin.example.JvmExamplePlugin'
 }
+// Not published so no need to assemble
+tasks.remove(assemble)
+build.dependsOn.remove('assemble')
 
 // no unit tests
 test.enabled = false
@@ -33,7 +36,7 @@ dependencies {
   exampleFixture project(':test:fixtures:example-fixture')
 }
 
-task exampleFixture(type: org.elasticsearch.gradle.test.Fixture) {
+task exampleFixture(type: org.elasticsearch.gradle.test.AntFixture) {
   dependsOn project.configurations.exampleFixture
   executable = new File(project.javaHome, 'bin/java')
   args '-cp', "${ -> project.configurations.exampleFixture.asPath }",
@@ -41,8 +44,9 @@ task exampleFixture(type: org.elasticsearch.gradle.test.Fixture) {
        baseDir
 }
 
-integTest {
+integTestCluster {
   dependsOn exampleFixture
+}
+integTestRunner {
   systemProperty 'external.address', "${ -> exampleFixture.addressAndPort }"
 }
-
diff --git a/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/ExampleCatAction.java b/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/ExampleCatAction.java
index 68ec25f2205ad..d7f3f94c81854 100644
--- a/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/ExampleCatAction.java
+++ b/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/ExampleCatAction.java
@@ -20,10 +20,8 @@
 
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Table;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.cat.AbstractCatAction;
@@ -37,7 +35,6 @@
 public class ExampleCatAction extends AbstractCatAction {
     private final ExamplePluginConfiguration config;
 
-    @Inject
     public ExampleCatAction(Settings settings, RestController controller, ExamplePluginConfiguration config) {
         super(settings);
         this.config = config;
@@ -45,21 +42,18 @@ public ExampleCatAction(Settings settings, RestController controller, ExamplePlu
     }
 
     @Override
-    protected void doRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
+    protected RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client) {
         Table table = getTableWithHeader(request);
         table.startRow();
         table.addCell(config.getTestConfig());
         table.endRow();
-        try {
-            channel.sendResponse(RestTable.buildResponse(table, channel));
-        } catch (Exception e) {
+        return channel -> {
             try {
+                channel.sendResponse(RestTable.buildResponse(table, channel));
+            } catch (final Exception e) {
                 channel.sendResponse(new BytesRestResponse(channel, e));
-            } catch (Exception inner) {
-                inner.addSuppressed(e);
-                logger.error("failed to send failure response", inner);
             }
-        }
+        };
     }
 
     @Override
diff --git a/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/ExamplePluginConfiguration.java b/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/ExamplePluginConfiguration.java
index 21582ccde5283..de8cb09115662 100644
--- a/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/ExamplePluginConfiguration.java
+++ b/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/ExamplePluginConfiguration.java
@@ -19,31 +19,31 @@
 
 package org.elasticsearch.plugin.example;
 
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 
 import java.io.IOException;
 import java.nio.file.Path;
-import java.util.Locale;
 
 /**
  * Example configuration.
  */
 public class ExamplePluginConfiguration {
-
     private final Settings customSettings;
 
     public static final Setting<String> TEST_SETTING =
       new Setting<String>("test", "default_value",
       (value) -> value, Setting.Property.Dynamic);
 
-    @Inject
-    public ExamplePluginConfiguration(Environment env) throws IOException {
+    public ExamplePluginConfiguration(Environment env) {
         // The directory part of the location matches the artifactId of this plugin
         Path path = env.configFile().resolve("jvm-example/example.yaml");
-        customSettings = Settings.builder().loadFromPath(path).build();
+        try {
+            customSettings = Settings.builder().loadFromPath(path).build();
+        } catch (IOException e) {
+            throw new RuntimeException("Failed to load settings, giving up", e);
+        }
 
         // asserts for tests
         assert customSettings != null;
diff --git a/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/JvmExamplePlugin.java b/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/JvmExamplePlugin.java
index ef3302fabcb20..431d481879120 100644
--- a/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/JvmExamplePlugin.java
+++ b/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/JvmExamplePlugin.java
@@ -19,40 +19,32 @@
 
 package org.elasticsearch.plugin.example;
 
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-
-import org.elasticsearch.common.component.LifecycleComponent;
-import org.elasticsearch.common.inject.AbstractModule;
-import org.elasticsearch.common.inject.Module;
-import org.elasticsearch.common.inject.multibindings.Multibinder;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsFilter;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.repositories.RepositoriesModule;
-import org.elasticsearch.rest.action.cat.AbstractCatAction;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestHandler;
+
+import java.util.List;
+import java.util.function.Supplier;
+
+import static java.util.Collections.singletonList;
 
 /**
  * Example of a plugin.
  */
-public class JvmExamplePlugin extends Plugin {
-
-    private final Settings settings;
+public class JvmExamplePlugin extends Plugin implements ActionPlugin {
+    private final ExamplePluginConfiguration config;
 
     public JvmExamplePlugin(Settings settings) {
-        this.settings = settings;
-    }
-
-    @Override
-    public Collection<Module> createGuiceModules() {
-        return Collections.<Module>singletonList(new ConfiguredExampleModule());
-    }
-
-    @Override
-    @SuppressWarnings("rawtypes") // Plugin use a rawtype
-    public Collection<Class<? extends LifecycleComponent>> getGuiceServiceClasses() {
-        Collection<Class<? extends LifecycleComponent>> services = new ArrayList<>();
-        return services;
+        Environment environment = new Environment(settings);
+        config = new ExamplePluginConfiguration(environment);
     }
 
     @Override
@@ -60,19 +52,10 @@ public Settings additionalSettings() {
         return Settings.EMPTY;
     }
 
-    public void onModule(RepositoriesModule repositoriesModule) {
-    }
-
-    /**
-     * Module declaring some example configuration and a _cat action that uses
-     * it.
-     */
-    public static class ConfiguredExampleModule extends AbstractModule {
-        @Override
-        protected void configure() {
-          bind(ExamplePluginConfiguration.class).asEagerSingleton();
-          Multibinder<AbstractCatAction> catActionMultibinder = Multibinder.newSetBinder(binder(), AbstractCatAction.class);
-          catActionMultibinder.addBinding().to(ExampleCatAction.class).asEagerSingleton();
-        }
+    @Override
+    public List<RestHandler> getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings,
+            IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver,
+            Supplier<DiscoveryNodes> nodesInCluster) {
+        return singletonList(new ExampleCatAction(settings, restController, config));
     }
 }
diff --git a/plugins/jvm-example/src/test/java/org/elasticsearch/plugin/example/JvmExampleClientYamlTestSuiteIT.java b/plugins/jvm-example/src/test/java/org/elasticsearch/plugin/example/JvmExampleClientYamlTestSuiteIT.java
index 0ef413d9595e6..a43cca718263d 100644
--- a/plugins/jvm-example/src/test/java/org/elasticsearch/plugin/example/JvmExampleClientYamlTestSuiteIT.java
+++ b/plugins/jvm-example/src/test/java/org/elasticsearch/plugin/example/JvmExampleClientYamlTestSuiteIT.java
@@ -24,9 +24,6 @@
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-
-import java.io.IOException;
 
 public class JvmExampleClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
@@ -35,8 +32,8 @@ public JvmExampleClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate tes
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
-        return ESClientYamlSuiteTestCase.createParameters(0, 1);
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
     }
 }
 
diff --git a/plugins/lang-javascript/build.gradle b/plugins/lang-javascript/build.gradle
index 13204e1d5d7f3..f112850843c85 100644
--- a/plugins/lang-javascript/build.gradle
+++ b/plugins/lang-javascript/build.gradle
@@ -26,10 +26,8 @@ dependencies {
   compile 'org.mozilla:rhino:1.7.7'
 }
 
-integTest {
-  cluster {
-    setting 'script.inline', 'true'
-    setting 'script.stored', 'true'
-    setting 'script.max_compilations_per_minute', '1000'
-  }
+integTestCluster {
+  setting 'script.inline', 'true'
+  setting 'script.stored', 'true'
+  setting 'script.max_compilations_per_minute', '1000'
 }
diff --git a/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineService.java b/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineService.java
index da3d14cd02b98..ad15ee6fb8472 100644
--- a/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineService.java
+++ b/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineService.java
@@ -25,7 +25,6 @@
 import org.elasticsearch.bootstrap.BootstrapInfo;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.script.ClassPermission;
 import org.elasticsearch.script.CompiledScript;
diff --git a/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineTests.java b/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineTests.java
index 84bc97abfbef3..f4c1d75bd1aa5 100644
--- a/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineTests.java
+++ b/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineTests.java
@@ -19,23 +19,23 @@
 
 package org.elasticsearch.script.javascript;
 
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static java.util.Collections.emptyMap;
-
 import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.ExecutableScript;
-import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.After;
 import org.junit.Before;
 
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static java.util.Collections.emptyMap;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 
@@ -55,48 +55,52 @@ public void close() {
         se.close();
     }
 
-    public void testSimpleEquation() {
+    public void testSimpleEquation() throws IOException {
         Map<String, Object> vars = new HashMap<String, Object>();
-        Object o = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testSimpleEquation", "js", se.compile(null, "1 + 2", Collections.emptyMap())), vars).run();
+        Object o = se.executable(new CompiledScript(ScriptType.INLINE, "testSimpleEquation", "js", se.compile(null, "1 + 2", Collections.emptyMap())), vars).run();
         assertThat(((Number) o).intValue(), equalTo(3));
+        assertWarnings("[javascript] scripts are deprecated, use [painless] scripts instead");
     }
 
-    public void testNullVars() {
-        CompiledScript script = new CompiledScript(ScriptService.ScriptType.INLINE, "testSimpleEquation", "js",
+    public void testNullVars() throws IOException {
+        CompiledScript script = new CompiledScript(ScriptType.INLINE, "testSimpleEquation", "js",
                 se.compile(null, "1 + 2", emptyMap()));
         Object o = se.executable(script, null).run();
         assertThat(((Number) o).intValue(), equalTo(3));
+        assertWarnings("[javascript] scripts are deprecated, use [painless] scripts instead");
     }
 
     @SuppressWarnings("unchecked")
-    public void testMapAccess() {
+    public void testMapAccess() throws IOException {
         Map<String, Object> vars = new HashMap<String, Object>();
 
         Map<String, Object> obj2 = MapBuilder.<String, Object>newMapBuilder().put("prop2", "value2").map();
         Map<String, Object> obj1 = MapBuilder.<String, Object>newMapBuilder().put("prop1", "value1").put("obj2", obj2).put("l", Arrays.asList("2", "1")).map();
         vars.put("obj1", obj1);
-        Object o = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testMapAccess", "js", se.compile(null, "obj1", Collections.emptyMap())), vars).run();
+        Object o = se.executable(new CompiledScript(ScriptType.INLINE, "testMapAccess", "js", se.compile(null, "obj1", Collections.emptyMap())), vars).run();
         assertThat(o, instanceOf(Map.class));
         obj1 = (Map<String, Object>) o;
         assertThat((String) obj1.get("prop1"), equalTo("value1"));
         assertThat((String) ((Map<String, Object>) obj1.get("obj2")).get("prop2"), equalTo("value2"));
 
-        o = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testMapAccess", "js", se.compile(null, "obj1.l[0]", Collections.emptyMap())), vars).run();
+        o = se.executable(new CompiledScript(ScriptType.INLINE, "testMapAccess", "js", se.compile(null, "obj1.l[0]", Collections.emptyMap())), vars).run();
         assertThat(((String) o), equalTo("2"));
+        assertWarnings("[javascript] scripts are deprecated, use [painless] scripts instead");
     }
 
     @SuppressWarnings("unchecked")
-    public void testJavaScriptObjectToMap() {
+    public void testJavaScriptObjectToMap() throws IOException {
         Map<String, Object> vars = new HashMap<String, Object>();
-        Object o = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testJavaScriptObjectToMap", "js",
+        Object o = se.executable(new CompiledScript(ScriptType.INLINE, "testJavaScriptObjectToMap", "js",
                 se.compile(null, "var obj1 = {}; obj1.prop1 = 'value1'; obj1.obj2 = {}; obj1.obj2.prop2 = 'value2'; obj1", Collections.emptyMap())), vars).run();
         Map<String, Object> obj1 = (Map<String, Object>) o;
         assertThat((String) obj1.get("prop1"), equalTo("value1"));
         assertThat((String) ((Map<String, Object>) obj1.get("obj2")).get("prop2"), equalTo("value2"));
+        assertWarnings("[javascript] scripts are deprecated, use [painless] scripts instead");
     }
 
     @SuppressWarnings("unchecked")
-    public void testJavaScriptObjectMapInter() {
+    public void testJavaScriptObjectMapInter() throws IOException {
         Map<String, Object> vars = new HashMap<String, Object>();
         Map<String, Object> ctx = new HashMap<String, Object>();
         Map<String, Object> obj1 = new HashMap<String, Object>();
@@ -104,7 +108,7 @@ public void testJavaScriptObjectMapInter() {
         ctx.put("obj1", obj1);
         vars.put("ctx", ctx);
 
-        ExecutableScript executable = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testJavaScriptObjectMapInter", "js",
+        ExecutableScript executable = se.executable(new CompiledScript(ScriptType.INLINE, "testJavaScriptObjectMapInter", "js",
                 se.compile(null, "ctx.obj2 = {}; ctx.obj2.prop2 = 'value2'; ctx.obj1.prop1 = 'uvalue1'", Collections.emptyMap())), vars);
         executable.run();
         ctx = (Map<String, Object>) executable.unwrap(vars.get("ctx"));
@@ -112,16 +116,17 @@ public void testJavaScriptObjectMapInter() {
         assertThat((String) ((Map<String, Object>) ctx.get("obj1")).get("prop1"), equalTo("uvalue1"));
         assertThat(ctx.containsKey("obj2"), equalTo(true));
         assertThat((String) ((Map<String, Object>) ctx.get("obj2")).get("prop2"), equalTo("value2"));
+        assertWarnings("[javascript] scripts are deprecated, use [painless] scripts instead");
     }
 
     @SuppressWarnings("unchecked")
-    public void testJavaScriptInnerArrayCreation() {
+    public void testJavaScriptInnerArrayCreation() throws IOException {
         Map<String, Object> ctx = new HashMap<String, Object>();
         Map<String, Object> doc = new HashMap<String, Object>();
         ctx.put("doc", doc);
 
         Object compiled = se.compile(null, "ctx.doc.field1 = ['value1', 'value2']", Collections.emptyMap());
-        ExecutableScript script = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testJavaScriptInnerArrayCreation", "js",
+        ExecutableScript script = se.executable(new CompiledScript(ScriptType.INLINE, "testJavaScriptInnerArrayCreation", "js",
                 compiled), new HashMap<String, Object>());
         script.setNextVar("ctx", ctx);
         script.run();
@@ -129,41 +134,43 @@ public void testJavaScriptInnerArrayCreation() {
         Map<String, Object> unwrap = (Map<String, Object>) script.unwrap(ctx);
 
         assertThat(((Map<String, Object>) unwrap.get("doc")).get("field1"), instanceOf(List.class));
+        assertWarnings("[javascript] scripts are deprecated, use [painless] scripts instead");
     }
 
     @SuppressWarnings("unchecked")
-    public void testAccessListInScript() {
+    public void testAccessListInScript() throws IOException {
         Map<String, Object> vars = new HashMap<String, Object>();
         Map<String, Object> obj2 = MapBuilder.<String, Object>newMapBuilder().put("prop2", "value2").map();
         Map<String, Object> obj1 = MapBuilder.<String, Object>newMapBuilder().put("prop1", "value1").put("obj2", obj2).map();
         vars.put("l", Arrays.asList("1", "2", "3", obj1));
 
-        Object o = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testAccessInScript", "js",
+        Object o = se.executable(new CompiledScript(ScriptType.INLINE, "testAccessInScript", "js",
                 se.compile(null, "l.length", Collections.emptyMap())), vars).run();
         assertThat(((Number) o).intValue(), equalTo(4));
 
-        o = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testAccessInScript", "js",
+        o = se.executable(new CompiledScript(ScriptType.INLINE, "testAccessInScript", "js",
                 se.compile(null, "l[0]", Collections.emptyMap())), vars).run();
         assertThat(((String) o), equalTo("1"));
 
-        o = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testAccessInScript", "js",
+        o = se.executable(new CompiledScript(ScriptType.INLINE, "testAccessInScript", "js",
                 se.compile(null, "l[3]", Collections.emptyMap())), vars).run();
         obj1 = (Map<String, Object>) o;
         assertThat((String) obj1.get("prop1"), equalTo("value1"));
         assertThat((String) ((Map<String, Object>) obj1.get("obj2")).get("prop2"), equalTo("value2"));
 
-        o = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testAccessInScript", "js",
+        o = se.executable(new CompiledScript(ScriptType.INLINE, "testAccessInScript", "js",
                 se.compile(null, "l[3].prop1", Collections.emptyMap())), vars).run();
         assertThat(((String) o), equalTo("value1"));
+        assertWarnings("[javascript] scripts are deprecated, use [painless] scripts instead");
     }
 
-    public void testChangingVarsCrossExecution1() {
+    public void testChangingVarsCrossExecution1() throws IOException {
         Map<String, Object> vars = new HashMap<String, Object>();
         Map<String, Object> ctx = new HashMap<String, Object>();
         vars.put("ctx", ctx);
         Object compiledScript = se.compile(null, "ctx.value", Collections.emptyMap());
 
-        ExecutableScript script = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testChangingVarsCrossExecution1", "js",
+        ExecutableScript script = se.executable(new CompiledScript(ScriptType.INLINE, "testChangingVarsCrossExecution1", "js",
                 compiledScript), vars);
         ctx.put("value", 1);
         Object o = script.run();
@@ -172,13 +179,14 @@ public void testChangingVarsCrossExecution1() {
         ctx.put("value", 2);
         o = script.run();
         assertThat(((Number) o).intValue(), equalTo(2));
+        assertWarnings("[javascript] scripts are deprecated, use [painless] scripts instead");
     }
 
-    public void testChangingVarsCrossExecution2() {
+    public void testChangingVarsCrossExecution2() throws IOException {
         Map<String, Object> vars = new HashMap<String, Object>();
         Object compiledScript = se.compile(null, "value", Collections.emptyMap());
 
-        ExecutableScript script = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testChangingVarsCrossExecution2", "js",
+        ExecutableScript script = se.executable(new CompiledScript(ScriptType.INLINE, "testChangingVarsCrossExecution2", "js",
                 compiledScript), vars);
         script.setNextVar("value", 1);
         Object o = script.run();
@@ -187,5 +195,6 @@ public void testChangingVarsCrossExecution2() {
         script.setNextVar("value", 2);
         o = script.run();
         assertThat(((Number) o).intValue(), equalTo(2));
+        assertWarnings("[javascript] scripts are deprecated, use [painless] scripts instead");
     }
 }
diff --git a/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptScriptMultiThreadedTests.java b/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptScriptMultiThreadedTests.java
index c3614952ecff2..77f69806cbfdd 100644
--- a/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptScriptMultiThreadedTests.java
+++ b/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptScriptMultiThreadedTests.java
@@ -23,7 +23,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.ExecutableScript;
-import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.Collections;
@@ -56,7 +56,7 @@ public void run() {
                         Map<String, Object> vars = new HashMap<String, Object>();
                         vars.put("x", x);
                         vars.put("y", y);
-                        ExecutableScript script = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testExecutableNoRuntimeParams", "js", compiled), vars);
+                        ExecutableScript script = se.executable(new CompiledScript(ScriptType.INLINE, "testExecutableNoRuntimeParams", "js", compiled), vars);
                         for (int i = 0; i < between(100, 1000); i++) {
                             long result = ((Number) script.run()).longValue();
                             assertThat(result, equalTo(addition));
@@ -76,6 +76,7 @@ public void run() {
         barrier.await();
         latch.await();
         assertThat(failed.get(), equalTo(false));
+        assertWarnings("[javascript] scripts are deprecated, use [painless] scripts instead");
     }
 
     public void testExecutableWithRuntimeParams() throws Exception {
@@ -95,7 +96,7 @@ public void run() {
                         long x = Randomness.get().nextInt();
                         Map<String, Object> vars = new HashMap<String, Object>();
                         vars.put("x", x);
-                        ExecutableScript script = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testExecutableNoRuntimeParams", "js", compiled), vars);
+                        ExecutableScript script = se.executable(new CompiledScript(ScriptType.INLINE, "testExecutableNoRuntimeParams", "js", compiled), vars);
                         for (int i = 0; i < between(100, 1000); i++) {
                             long y = Randomness.get().nextInt();
                             long addition = x + y;
@@ -118,6 +119,7 @@ public void run() {
         barrier.await();
         latch.await();
         assertThat(failed.get(), equalTo(false));
+        assertWarnings("[javascript] scripts are deprecated, use [painless] scripts instead");
     }
 
     public void testExecute() throws Exception {
@@ -141,7 +143,7 @@ public void run() {
                             long addition = x + y;
                             runtimeVars.put("x", x);
                             runtimeVars.put("y", y);
-                            long result = ((Number) se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testExecutableNoRuntimeParams", "js", compiled), runtimeVars).run()).longValue();
+                            long result = ((Number) se.executable(new CompiledScript(ScriptType.INLINE, "testExecutableNoRuntimeParams", "js", compiled), runtimeVars).run()).longValue();
                             assertThat(result, equalTo(addition));
                         }
                     } catch (Exception e) {
@@ -159,5 +161,6 @@ public void run() {
         barrier.await();
         latch.await();
         assertThat(failed.get(), equalTo(false));
+        assertWarnings("[javascript] scripts are deprecated, use [painless] scripts instead");
     }
 }
diff --git a/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptSecurityTests.java b/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptSecurityTests.java
index c1eb77d4e2087..d29eafd32b9ac 100644
--- a/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptSecurityTests.java
+++ b/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptSecurityTests.java
@@ -21,11 +21,12 @@
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.script.CompiledScript;
-import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.test.ESTestCase;
 import org.mozilla.javascript.EcmaError;
 import org.mozilla.javascript.WrappedException;
 
+import java.io.IOException;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
@@ -54,7 +55,7 @@ public void tearDown() throws Exception {
     /** runs a script */
     private void doTest(String script) {
         Map<String, Object> vars = new HashMap<String, Object>();
-        se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "test", "js", se.compile(null, script, Collections.emptyMap())), vars).run();
+        se.executable(new CompiledScript(ScriptType.INLINE, "test", "js", se.compile(null, script, Collections.emptyMap())), vars).run();
     }
 
     /** asserts that a script runs without exception */
@@ -81,10 +82,11 @@ private void assertFailure(String script, Class<? extends Throwable> exceptionCl
     }
 
     /** Test some javascripts that are ok */
-    public void testOK() {
+    public void testOK() throws IOException {
         assertSuccess("1 + 2");
         assertSuccess("Math.cos(Math.PI)");
         assertSuccess("Array.apply(null, Array(100)).map(function (_, i) {return i;}).map(function (i) {return i+1;})");
+        assertWarnings("[javascript] scripts are deprecated, use [painless] scripts instead");
     }
 
     /** Test some javascripts that should hit security exception */
@@ -96,14 +98,16 @@ public void testNotOK() throws Exception {
         assertFailure("new java.net.Socket(\"localhost\", 1024)", EcmaError.class);
         // no files
         assertFailure("java.io.File.createTempFile(\"test\", \"tmp\")", EcmaError.class);
+        assertWarnings("[javascript] scripts are deprecated, use [painless] scripts instead");
     }
 
-    public void testDefinitelyNotOK() {
+    public void testDefinitelyNotOK() throws IOException {
         // no mucking with security controller
         assertFailure("var ctx = org.mozilla.javascript.Context.getCurrentContext(); " +
                       "ctx.setSecurityController(new org.mozilla.javascript.PolicySecurityController());", EcmaError.class);
         // no compiling scripts from scripts
         assertFailure("var ctx = org.mozilla.javascript.Context.getCurrentContext(); " +
                       "ctx.compileString(\"1 + 1\", \"foobar\", 1, null); ", EcmaError.class);
+        assertWarnings("[javascript] scripts are deprecated, use [painless] scripts instead");
     }
 }
diff --git a/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/LangJavascriptClientYamlTestSuiteIT.java b/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/LangJavascriptClientYamlTestSuiteIT.java
index e89372c8b3693..f3d8450aa948b 100644
--- a/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/LangJavascriptClientYamlTestSuiteIT.java
+++ b/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/LangJavascriptClientYamlTestSuiteIT.java
@@ -24,9 +24,6 @@
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-
-import java.io.IOException;
 
 public class LangJavascriptClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
@@ -35,8 +32,8 @@ public LangJavascriptClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
-        return ESClientYamlSuiteTestCase.createParameters(0, 1);
+    public static Iterable<Object[]> parameters() throws Exception {
+        return createParameters();
     }
 }
 
diff --git a/plugins/lang-javascript/src/test/resources/rest-api-spec/test/lang_javascript/10_basic.yaml b/plugins/lang-javascript/src/test/resources/rest-api-spec/test/lang_javascript/10_basic.yaml
index 021e17e4a8325..862a2a5e0b6a9 100644
--- a/plugins/lang-javascript/src/test/resources/rest-api-spec/test/lang_javascript/10_basic.yaml
+++ b/plugins/lang-javascript/src/test/resources/rest-api-spec/test/lang_javascript/10_basic.yaml
@@ -13,13 +13,18 @@ setup:
 ---
 
 "Lang JavaScript":
+    - skip:
+        features:
+          - warnings
     - do:
+        warnings:
+          - '[javascript] scripts are deprecated, use [painless] scripts instead'
         search:
             body:
                 script_fields:
                     bar:
                         script: 
-                            inline: "doc['foo.keyword'].value + x"
+                            source: "doc['foo.keyword'].value + x"
                             lang: javascript
                             params:
                                 x: "bbb"
diff --git a/plugins/lang-javascript/src/test/resources/rest-api-spec/test/lang_javascript/20_search.yaml b/plugins/lang-javascript/src/test/resources/rest-api-spec/test/lang_javascript/20_search.yaml
index 043e9690303b5..905195539777b 100644
--- a/plugins/lang-javascript/src/test/resources/rest-api-spec/test/lang_javascript/20_search.yaml
+++ b/plugins/lang-javascript/src/test/resources/rest-api-spec/test/lang_javascript/20_search.yaml
@@ -1,6 +1,9 @@
 # Integration tests for Lang JavaScript components
 #
 "JavaScript Query":
+    - skip:
+        features:
+          - warnings
     - do:
         index:
             index: test
@@ -23,18 +26,20 @@
         indices.refresh: {}
 
     - do:
+        warnings:
+          - '[javascript] scripts are deprecated, use [painless] scripts instead'
         index: test
         search:
             body:
                 query:
                     script:
                         script:
-                            inline: "doc['num1'].value > 1"
+                            source: "doc['num1'].value > 1"
                             lang: javascript
                 script_fields:
                     sNum1:
                         script: 
-                            inline: "doc['num1'].value"
+                            source: "doc['num1'].value"
                             lang: javascript
                 sort:
                     num1:
@@ -45,13 +50,15 @@
     - match: { hits.hits.1.fields.sNum1.0: 3.0 }
 
     - do:
+        warnings:
+          - '[javascript] scripts are deprecated, use [painless] scripts instead'
         index: test
         search:
             body:
                 query:
                     script:
                         script:
-                            inline: "doc['num1'].value > param1"
+                            source: "doc['num1'].value > param1"
                             lang: javascript
                             params:
                                 param1: 1
@@ -59,7 +66,7 @@
                 script_fields:
                     sNum1:
                         script:
-                            inline: "doc['num1'].value"
+                            source: "doc['num1'].value"
                             lang: javascript
                 sort:
                     num1:
@@ -70,13 +77,15 @@
     - match: { hits.hits.1.fields.sNum1.0: 3.0 }
 
     - do:
+        warnings:
+          - '[javascript] scripts are deprecated, use [painless] scripts instead'
         index: test
         search:
             body:
                 query:
                     script:
                         script:
-                            inline: "doc['num1'].value > param1"
+                            source: "doc['num1'].value > param1"
                             lang: javascript
                             params:
                                 param1: -1
@@ -84,7 +93,7 @@
                 script_fields:
                     sNum1:
                         script: 
-                            inline: "doc['num1'].value"
+                            source: "doc['num1'].value"
                             lang: javascript
                 sort:
                     num1:
@@ -97,8 +106,10 @@
 
 
 ---
-
 "JavaScript Script Field Using Source":
+    - skip:
+        features:
+          - warnings
     - do:
         index:
             index: test
@@ -116,25 +127,27 @@
         indices.refresh: {}
 
     - do:
+        warnings:
+          - '[javascript] scripts are deprecated, use [painless] scripts instead'
         index: test
         search:
             body:
                 script_fields:
                     s_obj1:
                         script: 
-                            inline: "_source.obj1"
+                            source: "_source.obj1"
                             lang: javascript
                     s_obj1_test:
                         script: 
-                            inline: "_source.obj1.test"
+                            source: "_source.obj1.test"
                             lang: javascript
                     s_obj2:
                         script: 
-                            inline: "_source.obj2"
+                            source: "_source.obj2"
                             lang: javascript
                     s_obj2_arr2:
                         script: 
-                            inline: "_source.obj2.arr2"
+                            source: "_source.obj2.arr2"
                             lang: javascript
 
     - match: { hits.total: 1 }
@@ -146,8 +159,10 @@
     - match: { hits.hits.0.fields.s_obj2_arr2.1: arr_value2 }
 
 ---
-
 "JavaScript Custom Script Boost":
+    - skip:
+        features:
+          - warnings
     - do:
         index:
             index: test
@@ -164,6 +179,8 @@
         indices.refresh: {}
 
     - do:
+        warnings:
+          - '[javascript] scripts are deprecated, use [painless] scripts instead'
         index: test
         search:
             body:
@@ -176,7 +193,7 @@
                             "script_score": {
                                 "script": {
                                     "lang": "javascript",
-                                    "inline": "doc['num1'].value"
+                                    "source": "doc['num1'].value"
                                 }
                             }
                         }]
@@ -186,6 +203,8 @@
     - match: { hits.hits.1._id: "1" }
 
     - do:
+        warnings:
+          - '[javascript] scripts are deprecated, use [painless] scripts instead'
         index: test
         search:
             body:
@@ -198,7 +217,7 @@
                             "script_score": {
                                 "script": {
                                     "lang": "javascript",
-                                    "inline": "-doc['num1'].value"
+                                    "source": "-doc['num1'].value"
                                 }
                             }
                         }]
@@ -208,6 +227,8 @@
     - match: { hits.hits.1._id: "2" }
 
     - do:
+        warnings:
+          - '[javascript] scripts are deprecated, use [painless] scripts instead'
         index: test
         search:
             body:
@@ -220,7 +241,7 @@
                             "script_score": {
                                 "script": {
                                     "lang": "javascript",
-                                    "inline": "Math.pow(doc['num1'].value, 2)"
+                                    "source": "Math.pow(doc['num1'].value, 2)"
                                 }
                             }
                         }]
@@ -230,6 +251,8 @@
     - match: { hits.hits.1._id: "1" }
 
     - do:
+        warnings:
+          - '[javascript] scripts are deprecated, use [painless] scripts instead'
         index: test
         search:
             body:
@@ -242,7 +265,7 @@
                             "script_score": {
                                 "script": {
                                     "lang": "javascript",
-                                    "inline": "Math.max(doc['num1'].value, 1)"
+                                    "source": "Math.max(doc['num1'].value, 1)"
                                 }
                             }
                         }]
@@ -252,6 +275,8 @@
     - match: { hits.hits.1._id: "1" }
 
     - do:
+        warnings:
+          - '[javascript] scripts are deprecated, use [painless] scripts instead'
         index: test
         search:
             body:
@@ -264,7 +289,7 @@
                             "script_score": {
                                 "script": {
                                     "lang": "javascript",
-                                    "inline": "doc['num1'].value * _score"
+                                    "source": "doc['num1'].value * _score"
                                 }
                             }
                         }]
@@ -274,6 +299,8 @@
     - match: { hits.hits.1._id: "1" }
 
     - do:
+        warnings:
+          - '[javascript] scripts are deprecated, use [painless] scripts instead'
         index: test
         search:
             body:
@@ -286,7 +313,7 @@
                             "script_score": {
                                 "script": {
                                     "lang": "javascript",
-                                    "inline": "param1 * param2 * _score",
+                                    "source": "param1 * param2 * _score",
                                     "params": {
                                         "param1": 2,
                                         "param2": 2
@@ -299,8 +326,10 @@
     - match: { hits.total: 2 }
 
 ---
-
 "JavaScript Scores Nested":
+    - skip:
+        features:
+          - warnings
     - do:
         index:
             index: test
@@ -311,6 +340,8 @@
         indices.refresh: {}
 
     - do:
+        warnings:
+          - '[javascript] scripts are deprecated, use [painless] scripts instead'
         index: test
         search:
             body:
@@ -323,14 +354,14 @@
                                         "script_score": {
                                             "script": {
                                                 "lang": "javascript",
-                                                "inline": "1"
+                                                "source": "1"
                                             }
                                         }
                                     }, {
                                         "script_score": {
                                             "script": {
                                                 "lang": "javascript",
-                                                "inline": "_score.doubleValue()"
+                                                "source": "_score.doubleValue()"
                                             }
                                         }
                                     }
@@ -339,7 +370,7 @@
                             "script_score": {
                                 "script": {
                                     "lang": "javascript",
-                                    "inline": "_score.doubleValue()"
+                                    "source": "_score.doubleValue()"
                                 }
                             }
                         }]
@@ -349,8 +380,10 @@
 
 
 ---
-
 "JavaScript Scores With Agg":
+    - skip:
+        features:
+          - warnings
     - do:
         index:
             index: test
@@ -360,8 +393,9 @@
     - do:
         indices.refresh: {}
 
-
     - do:
+        warnings:
+          - '[javascript] scripts are deprecated, use [painless] scripts instead'
         index: test
         search:
             body:
@@ -371,7 +405,7 @@
                             "script_score": {
                                 "script": {
                                     "lang": "javascript",
-                                    "inline": "_score.doubleValue()"
+                                    "source": "_score.doubleValue()"
                                 }
                             }
                         }]
@@ -380,7 +414,7 @@
                         terms:
                             script:
                                 lang: javascript
-                                inline: "_score.doubleValue()"
+                                source: "_score.doubleValue()"
 
     - match: { hits.total: 1 }
     - match: { hits.hits.0._score: 1.0 }
@@ -388,8 +422,10 @@
     - match: { aggregations.score_agg.buckets.0.doc_count: 1 }
 
 ---
-
 "JavaScript Use List Length In Scripts":
+    - skip:
+        features:
+          - warnings
     - do:
         index:
             index: test
@@ -401,13 +437,15 @@
 
 
     - do:
+        warnings:
+          - '[javascript] scripts are deprecated, use [painless] scripts instead'
         index: test
         search:
             body:
                 script_fields:
                     foobar:
                         script: 
-                            inline: "doc['f'].values.length"
+                            source: "doc['f'].values.length"
                             lang: javascript
 
 
diff --git a/plugins/lang-python/build.gradle b/plugins/lang-python/build.gradle
index 3deda3be5f477..e365fad04dc1c 100644
--- a/plugins/lang-python/build.gradle
+++ b/plugins/lang-python/build.gradle
@@ -26,12 +26,10 @@ dependencies {
   compile 'org.python:jython-standalone:2.7.0'
 }
 
-integTest {
-  cluster {
-    setting 'script.inline', 'true'
-    setting 'script.stored', 'true'
-    setting 'script.max_compilations_per_minute', '1000'
-  }
+integTestCluster {
+  setting 'script.inline', 'true'
+  setting 'script.stored', 'true'
+  setting 'script.max_compilations_per_minute', '1000'
 }
 
 thirdPartyAudit.excludes = [
diff --git a/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java b/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java
index d31e691b994fb..f3a70a963a75f 100644
--- a/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java
+++ b/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java
@@ -62,7 +62,7 @@ public class PythonScriptEngineService extends AbstractComponent implements Scri
     public PythonScriptEngineService(Settings settings) {
         super(settings);
 
-        deprecationLogger.deprecated("[python] scripts are deprecated, use [painless] scripts instead.");
+        deprecationLogger.deprecated("[python] scripts are deprecated, use [painless] scripts instead");
 
         // classloader created here
         final SecurityManager sm = System.getSecurityManager();
diff --git a/plugins/lang-python/src/test/java/org/elasticsearch/script/python/LangPythonClientYamlTestSuiteIT.java b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/LangPythonClientYamlTestSuiteIT.java
index 618ea6b20e540..264bc87eb9d64 100644
--- a/plugins/lang-python/src/test/java/org/elasticsearch/script/python/LangPythonClientYamlTestSuiteIT.java
+++ b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/LangPythonClientYamlTestSuiteIT.java
@@ -24,9 +24,6 @@
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-
-import java.io.IOException;
 
 public class LangPythonClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
@@ -35,8 +32,8 @@ public LangPythonClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate tes
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
-        return ESClientYamlSuiteTestCase.createParameters(0, 1);
+    public static Iterable<Object[]> parameters() throws Exception {
+        return createParameters();
     }
 }
 
diff --git a/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java
index b96071e3848e2..7f8d235dd53c7 100644
--- a/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java
+++ b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java
@@ -19,20 +19,21 @@
 
 package org.elasticsearch.script.python;
 
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
 import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.ExecutableScript;
-import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.After;
 import org.junit.Before;
 
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 
@@ -52,31 +53,33 @@ public void close() {
         se.close();
     }
 
-    public void testSimpleEquation() {
+    public void testSimpleEquation() throws IOException {
         Map<String, Object> vars = new HashMap<String, Object>();
-        Object o = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testSimpleEquation", "python", se.compile(null, "1 + 2", Collections.emptyMap())), vars).run();
+        Object o = se.executable(new CompiledScript(ScriptType.INLINE, "testSimpleEquation", "python", se.compile(null, "1 + 2", Collections.emptyMap())), vars).run();
         assertThat(((Number) o).intValue(), equalTo(3));
+        assertWarnings("[python] scripts are deprecated, use [painless] scripts instead");
     }
 
     @SuppressWarnings("unchecked")
-    public void testMapAccess() {
+    public void testMapAccess() throws IOException {
         Map<String, Object> vars = new HashMap<String, Object>();
 
         Map<String, Object> obj2 = MapBuilder.<String, Object>newMapBuilder().put("prop2", "value2").map();
         Map<String, Object> obj1 = MapBuilder.<String, Object>newMapBuilder().put("prop1", "value1").put("obj2", obj2).put("l", Arrays.asList("2", "1")).map();
         vars.put("obj1", obj1);
-        Object o = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testMapAccess", "python", se.compile(null, "obj1", Collections.emptyMap())), vars).run();
+        Object o = se.executable(new CompiledScript(ScriptType.INLINE, "testMapAccess", "python", se.compile(null, "obj1", Collections.emptyMap())), vars).run();
         assertThat(o, instanceOf(Map.class));
         obj1 = (Map<String, Object>) o;
         assertThat((String) obj1.get("prop1"), equalTo("value1"));
         assertThat((String) ((Map<String, Object>) obj1.get("obj2")).get("prop2"), equalTo("value2"));
 
-        o = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testMapAccess", "python", se.compile(null, "obj1['l'][0]", Collections.emptyMap())), vars).run();
+        o = se.executable(new CompiledScript(ScriptType.INLINE, "testMapAccess", "python", se.compile(null, "obj1['l'][0]", Collections.emptyMap())), vars).run();
         assertThat(((String) o), equalTo("2"));
+        assertWarnings("[python] scripts are deprecated, use [painless] scripts instead");
     }
 
     @SuppressWarnings("unchecked")
-    public void testObjectMapInter() {
+    public void testObjectMapInter() throws IOException {
         Map<String, Object> vars = new HashMap<String, Object>();
         Map<String, Object> ctx = new HashMap<String, Object>();
         Map<String, Object> obj1 = new HashMap<String, Object>();
@@ -84,7 +87,7 @@ public void testObjectMapInter() {
         ctx.put("obj1", obj1);
         vars.put("ctx", ctx);
 
-        ExecutableScript executable = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testObjectInterMap", "python",
+        ExecutableScript executable = se.executable(new CompiledScript(ScriptType.INLINE, "testObjectInterMap", "python",
                 se.compile(null, "ctx['obj2'] = { 'prop2' : 'value2' }; ctx['obj1']['prop1'] = 'uvalue1'", Collections.emptyMap())), vars);
         executable.run();
         ctx = (Map<String, Object>) executable.unwrap(vars.get("ctx"));
@@ -92,10 +95,11 @@ public void testObjectMapInter() {
         assertThat((String) ((Map<String, Object>) ctx.get("obj1")).get("prop1"), equalTo("uvalue1"));
         assertThat(ctx.containsKey("obj2"), equalTo(true));
         assertThat((String) ((Map<String, Object>) ctx.get("obj2")).get("prop2"), equalTo("value2"));
+        assertWarnings("[python] scripts are deprecated, use [painless] scripts instead");
     }
 
     @SuppressWarnings("unchecked")
-    public void testAccessListInScript() {
+    public void testAccessListInScript() throws IOException {
         Map<String, Object> vars = new HashMap<String, Object>();
         Map<String, Object> obj2 = MapBuilder.<String, Object>newMapBuilder().put("prop2", "value2").map();
         Map<String, Object> obj1 = MapBuilder.<String, Object>newMapBuilder().put("prop1", "value1").put("obj2", obj2).map();
@@ -104,25 +108,26 @@ public void testAccessListInScript() {
 //        Object o = se.execute(se.compile("l.length"), vars);
 //        assertThat(((Number) o).intValue(), equalTo(4));
 
-        Object o = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testAccessListInScript", "python", se.compile(null, "l[0]", Collections.emptyMap())), vars).run();
+        Object o = se.executable(new CompiledScript(ScriptType.INLINE, "testAccessListInScript", "python", se.compile(null, "l[0]", Collections.emptyMap())), vars).run();
         assertThat(((String) o), equalTo("1"));
 
-        o = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testAccessListInScript", "python", se.compile(null, "l[3]", Collections.emptyMap())), vars).run();
+        o = se.executable(new CompiledScript(ScriptType.INLINE, "testAccessListInScript", "python", se.compile(null, "l[3]", Collections.emptyMap())), vars).run();
         obj1 = (Map<String, Object>) o;
         assertThat((String) obj1.get("prop1"), equalTo("value1"));
         assertThat((String) ((Map<String, Object>) obj1.get("obj2")).get("prop2"), equalTo("value2"));
 
-        o = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testAccessListInScript", "python", se.compile(null, "l[3]['prop1']", Collections.emptyMap())), vars).run();
+        o = se.executable(new CompiledScript(ScriptType.INLINE, "testAccessListInScript", "python", se.compile(null, "l[3]['prop1']", Collections.emptyMap())), vars).run();
         assertThat(((String) o), equalTo("value1"));
+        assertWarnings("[python] scripts are deprecated, use [painless] scripts instead");
     }
 
-    public void testChangingVarsCrossExecution1() {
+    public void testChangingVarsCrossExecution1() throws IOException {
         Map<String, Object> vars = new HashMap<String, Object>();
         Map<String, Object> ctx = new HashMap<String, Object>();
         vars.put("ctx", ctx);
         Object compiledScript = se.compile(null, "ctx['value']", Collections.emptyMap());
 
-        ExecutableScript script = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testChangingVarsCrossExecution1", "python", compiledScript), vars);
+        ExecutableScript script = se.executable(new CompiledScript(ScriptType.INLINE, "testChangingVarsCrossExecution1", "python", compiledScript), vars);
         ctx.put("value", 1);
         Object o = script.run();
         assertThat(((Number) o).intValue(), equalTo(1));
@@ -130,14 +135,15 @@ public void testChangingVarsCrossExecution1() {
         ctx.put("value", 2);
         o = script.run();
         assertThat(((Number) o).intValue(), equalTo(2));
+        assertWarnings("[python] scripts are deprecated, use [painless] scripts instead");
     }
 
-    public void testChangingVarsCrossExecution2() {
+    public void testChangingVarsCrossExecution2() throws IOException {
         Map<String, Object> vars = new HashMap<String, Object>();
         Map<String, Object> ctx = new HashMap<String, Object>();
         Object compiledScript = se.compile(null, "value", Collections.emptyMap());
 
-        ExecutableScript script = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testChangingVarsCrossExecution2", "python", compiledScript), vars);
+        ExecutableScript script = se.executable(new CompiledScript(ScriptType.INLINE, "testChangingVarsCrossExecution2", "python", compiledScript), vars);
         script.setNextVar("value", 1);
         Object o = script.run();
         assertThat(((Number) o).intValue(), equalTo(1));
@@ -145,5 +151,6 @@ public void testChangingVarsCrossExecution2() {
         script.setNextVar("value", 2);
         o = script.run();
         assertThat(((Number) o).intValue(), equalTo(2));
+        assertWarnings("[python] scripts are deprecated, use [painless] scripts instead");
     }
 }
diff --git a/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTests.java b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTests.java
index 0a887bc9a7e6f..7d7e81d27a1c8 100644
--- a/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTests.java
+++ b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTests.java
@@ -23,7 +23,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.ExecutableScript;
-import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.Collections;
@@ -43,7 +43,7 @@ public class PythonScriptMultiThreadedTests extends ESTestCase {
     public void testExecutableNoRuntimeParams() throws Exception {
         final PythonScriptEngineService se = new PythonScriptEngineService(Settings.Builder.EMPTY_SETTINGS);
         final Object compiled = se.compile(null, "x + y", Collections.emptyMap());
-        final CompiledScript compiledScript = new CompiledScript(ScriptService.ScriptType.INLINE, "testExecutableNoRuntimeParams", "python", compiled);
+        final CompiledScript compiledScript = new CompiledScript(ScriptType.INLINE, "testExecutableNoRuntimeParams", "python", compiled);
         final AtomicBoolean failed = new AtomicBoolean();
 
         Thread[] threads = new Thread[4];
@@ -81,6 +81,7 @@ public void run() {
         barrier.await();
         latch.await();
         assertThat(failed.get(), equalTo(false));
+        assertWarnings("[python] scripts are deprecated, use [painless] scripts instead");
     }
 
 
@@ -129,7 +130,7 @@ public void run() {
     public void testExecute() throws Exception {
         final PythonScriptEngineService se = new PythonScriptEngineService(Settings.Builder.EMPTY_SETTINGS);
         final Object compiled = se.compile(null, "x + y", Collections.emptyMap());
-        final CompiledScript compiledScript = new CompiledScript(ScriptService.ScriptType.INLINE, "testExecute", "python", compiled);
+        final CompiledScript compiledScript = new CompiledScript(ScriptType.INLINE, "testExecute", "python", compiled);
         final AtomicBoolean failed = new AtomicBoolean();
 
         Thread[] threads = new Thread[4];
@@ -166,5 +167,6 @@ public void run() {
         barrier.await();
         latch.await();
         assertThat(failed.get(), equalTo(false));
+        assertWarnings("[python] scripts are deprecated, use [painless] scripts instead");
     }
 }
diff --git a/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonSecurityTests.java b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonSecurityTests.java
index 7ad13b31986af..a68a142796956 100644
--- a/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonSecurityTests.java
+++ b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonSecurityTests.java
@@ -21,10 +21,11 @@
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.script.CompiledScript;
-import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.test.ESTestCase;
 import org.python.core.PyException;
 
+import java.io.IOException;
 import java.text.DecimalFormatSymbols;
 import java.util.Collections;
 import java.util.HashMap;
@@ -55,7 +56,7 @@ public void tearDown() throws Exception {
     /** runs a script */
     private void doTest(String script) {
         Map<String, Object> vars = new HashMap<String, Object>();
-        se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "test", "python", se.compile(null, script, Collections.emptyMap())), vars).run();
+        se.executable(new CompiledScript(ScriptType.INLINE, "test", "python", se.compile(null, script, Collections.emptyMap())), vars).run();
     }
 
     /** asserts that a script runs without exception */
@@ -79,14 +80,15 @@ private void assertFailure(String script) {
     }
 
     /** Test some py scripts that are ok */
-    public void testOK() {
+    public void testOK() throws IOException {
         assertSuccess("1 + 2");
         assertSuccess("from java.lang import Math\nMath.cos(0)");
         assertSuccess("map(lambda x: x + 1, range(100))");
+        assertWarnings("[python] scripts are deprecated, use [painless] scripts instead");
     }
 
     /** Test some py scripts that should hit security exception */
-    public void testNotOK() {
+    public void testNotOK() throws IOException {
         // sanity check :)
         assertFailure("from java.lang import Runtime\nRuntime.getRuntime().halt(0)");
         // check a few things more restrictive than the ordinary policy
@@ -94,6 +96,7 @@ public void testNotOK() {
         assertFailure("from java.net import Socket\nSocket(\"localhost\", 1024)");
         // no files
         assertFailure("from java.io import File\nFile.createTempFile(\"test\", \"tmp\")");
+        assertWarnings("[python] scripts are deprecated, use [painless] scripts instead");
     }
 
     /** Test again from a new thread, python has complex threadlocal configuration */
diff --git a/plugins/lang-python/src/test/resources/rest-api-spec/test/lang_python/10_basic.yaml b/plugins/lang-python/src/test/resources/rest-api-spec/test/lang_python/10_basic.yaml
index c915411f1d497..a22d806ac6796 100644
--- a/plugins/lang-python/src/test/resources/rest-api-spec/test/lang_python/10_basic.yaml
+++ b/plugins/lang-python/src/test/resources/rest-api-spec/test/lang_python/10_basic.yaml
@@ -11,15 +11,20 @@ setup:
         indices.refresh: {}
 
 ---
-
 "Lang Python":
+    - skip:
+        features:
+          - warnings
     - do:
+        warnings:
+          - '[python] scripts are deprecated, use [painless] scripts instead'
         search:
+
             body:
                 script_fields:
                     bar:
                         script: 
-                            inline: "doc['foo.keyword'].value + x"
+                            source: "doc['foo.keyword'].value + x"
                             lang: python
                             params:
                                 x: "bbb"
diff --git a/plugins/lang-python/src/test/resources/rest-api-spec/test/lang_python/20_search.yaml b/plugins/lang-python/src/test/resources/rest-api-spec/test/lang_python/20_search.yaml
index b0f18e1774872..68321fb5b3152 100644
--- a/plugins/lang-python/src/test/resources/rest-api-spec/test/lang_python/20_search.yaml
+++ b/plugins/lang-python/src/test/resources/rest-api-spec/test/lang_python/20_search.yaml
@@ -1,6 +1,9 @@
 # Integration tests for Lang Python components
 #
 "Python Query":
+    - skip:
+        features:
+          - warnings
     - do:
         index:
             index: test
@@ -23,18 +26,20 @@
         indices.refresh: {}
 
     - do:
+        warnings:
+          - '[python] scripts are deprecated, use [painless] scripts instead'
         index: test
         search:
             body:
                 query:
                     script:
                         script:
-                            inline: "doc['num1'].value > 1"
+                            source: "doc['num1'].value > 1"
                             lang: python
                 script_fields:
                     sNum1:
                         script: 
-                            inline: "doc['num1'].value"
+                            source: "doc['num1'].value"
                             lang: python
                 sort:
                     num1:
@@ -45,13 +50,15 @@
     - match: { hits.hits.1.fields.sNum1.0: 3.0 }
 
     - do:
+        warnings:
+          - '[python] scripts are deprecated, use [painless] scripts instead'
         index: test
         search:
             body:
                 query:
                     script:
                         script:
-                            inline: "doc['num1'].value > param1"
+                            source: "doc['num1'].value > param1"
                             lang: python
                             params:
                                 param1: 1
@@ -59,7 +66,7 @@
                 script_fields:
                     sNum1:
                         script: 
-                            inline: "doc['num1'].value"
+                            source: "doc['num1'].value"
                             lang: python
                 sort:
                     num1:
@@ -70,13 +77,15 @@
     - match: { hits.hits.1.fields.sNum1.0: 3.0 }
 
     - do:
+        warnings:
+          - '[python] scripts are deprecated, use [painless] scripts instead'
         index: test
         search:
             body:
                 query:
                     script:
                         script:
-                            inline: "doc['num1'].value > param1"
+                            source: "doc['num1'].value > param1"
                             lang: python
                             params:
                                 param1: -1
@@ -84,7 +93,7 @@
                 script_fields:
                     sNum1:
                         script: 
-                            inline: "doc['num1'].value"
+                            source: "doc['num1'].value"
                             lang: python
                 sort:
                     num1:
@@ -97,8 +106,10 @@
 
 
 ---
-
 "Python Script Field Using Source":
+    - skip:
+        features:
+          - warnings
     - do:
         index:
             index: test
@@ -116,25 +127,27 @@
         indices.refresh: {}
 
     - do:
+        warnings:
+          - '[python] scripts are deprecated, use [painless] scripts instead'
         index: test
         search:
             body:
                 script_fields:
                     s_obj1:
                         script: 
-                            inline: "_source['obj1']"
+                            source: "_source['obj1']"
                             lang: python
                     s_obj1_test:
                         script: 
-                            inline: "_source['obj1']['test']"
+                            source: "_source['obj1']['test']"
                             lang: python
                     s_obj2:
                         script: 
-                            inline: "_source['obj2']"
+                            source: "_source['obj2']"
                             lang: python
                     s_obj2_arr2:
                         script: 
-                            inline: "_source['obj2']['arr2']"
+                            source: "_source['obj2']['arr2']"
                             lang: python
 
     - match: { hits.total: 1 }
@@ -146,8 +159,10 @@
     - match: { hits.hits.0.fields.s_obj2_arr2.1: arr_value2 }
 
 ---
-
 "Python Custom Script Boost":
+    - skip:
+        features:
+          - warnings
     - do:
         index:
             index: test
@@ -164,6 +179,8 @@
         indices.refresh: {}
 
     - do:
+        warnings:
+          - '[python] scripts are deprecated, use [painless] scripts instead'
         index: test
         search:
             body:
@@ -176,7 +193,7 @@
                             "script_score": {
                                 "script": {
                                     "lang": "python",
-                                    "inline": "doc['num1'].value"
+                                    "source": "doc['num1'].value"
                                 }
                             }
                         }]
@@ -186,6 +203,8 @@
     - match: { hits.hits.1._id: "1" }
 
     - do:
+        warnings:
+          - '[python] scripts are deprecated, use [painless] scripts instead'
         index: test
         search:
             body:
@@ -198,7 +217,7 @@
                             "script_score": {
                                 "script": {
                                     "lang": "python",
-                                    "inline": "-doc['num1'].value"
+                                    "source": "-doc['num1'].value"
                                 }
                             }
                         }]
@@ -208,6 +227,8 @@
     - match: { hits.hits.1._id: "2" }
 
     - do:
+        warnings:
+          - '[python] scripts are deprecated, use [painless] scripts instead'
         index: test
         search:
             body:
@@ -220,7 +241,7 @@
                             "script_score": {
                                 "script": {
                                     "lang": "python",
-                                    "inline": "doc['num1'].value * _score.doubleValue()"
+                                    "source": "doc['num1'].value * _score.doubleValue()"
                                 }
                             }
                         }]
@@ -230,6 +251,8 @@
     - match: { hits.hits.1._id: "1" }
 
     - do:
+        warnings:
+          - '[python] scripts are deprecated, use [painless] scripts instead'
         index: test
         search:
             body:
@@ -242,7 +265,7 @@
                             "script_score": {
                                 "script": {
                                     "lang": "python",
-                                    "inline": "param1 * param2 * _score.doubleValue()",
+                                    "source": "param1 * param2 * _score.doubleValue()",
                                     "params": {
                                         "param1": 2,
                                         "param2": 2
@@ -255,8 +278,10 @@
     - match: { hits.total: 2 }
 
 ---
-
 "Python Scores Nested":
+    - skip:
+        features:
+          - warnings
     - do:
         index:
             index: test
@@ -267,6 +292,8 @@
         indices.refresh: {}
 
     - do:
+        warnings:
+          - '[python] scripts are deprecated, use [painless] scripts instead'
         index: test
         search:
             body:
@@ -279,14 +306,14 @@
                                         "script_score": {
                                             "script": {
                                                 "lang": "python",
-                                                "inline": "1"
+                                                "source": "1"
                                             }
                                         }
                                     }, {
                                         "script_score": {
                                             "script": {
                                                 "lang": "python",
-                                                "inline": "_score.doubleValue()"
+                                                "source": "_score.doubleValue()"
                                             }
                                         }
                                     }
@@ -295,7 +322,7 @@
                             "script_score": {
                                 "script": {
                                     "lang": "python",
-                                    "inline": "_score.doubleValue()"
+                                    "source": "_score.doubleValue()"
                                 }
                             }
                         }]
@@ -305,8 +332,10 @@
 
 
 ---
-
 "Python Scores With Agg":
+    - skip:
+        features:
+          - warnings
     - do:
         index:
             index: test
@@ -318,6 +347,8 @@
 
 
     - do:
+        warnings:
+          - '[python] scripts are deprecated, use [painless] scripts instead'
         index: test
         search:
             body:
@@ -327,7 +358,7 @@
                             "script_score": {
                                 "script": {
                                     "lang": "python",
-                                    "inline": "_score.doubleValue()"
+                                    "source": "_score.doubleValue()"
                                 }
                             }
                         }]
@@ -336,7 +367,7 @@
                         terms:
                             script:
                                 lang: python
-                                inline: "_score.doubleValue()"
+                                source: "_score.doubleValue()"
 
     - match: { hits.total: 1 }
     - match: { hits.hits.0._score: 1.0 }
diff --git a/plugins/lang-python/src/test/resources/rest-api-spec/test/lang_python/30_update.yaml b/plugins/lang-python/src/test/resources/rest-api-spec/test/lang_python/30_update.yaml
index fc5dcba8bcb55..c0c3e0cb25af5 100644
--- a/plugins/lang-python/src/test/resources/rest-api-spec/test/lang_python/30_update.yaml
+++ b/plugins/lang-python/src/test/resources/rest-api-spec/test/lang_python/30_update.yaml
@@ -5,6 +5,9 @@
 # Update request that uses python script with no parameters fails with NullPointerException
 #
 "Python Update With Empty Parameters":
+    - skip:
+        features:
+          - warnings
     - do:
         index:
             index: test
@@ -12,13 +15,15 @@
             id: 1
             body: { "myfield": "foo" }
     - do:
+        warnings:
+          - '[python] scripts are deprecated, use [painless] scripts instead'
         update:
             index: test
             type: test
             id: 1
             body:
                 script:
-                    inline: "ctx[\"_source\"][\"myfield\"]=\"bar\""
+                    source: "ctx[\"_source\"][\"myfield\"]=\"bar\""
                     lang: python
     - do:
         get:
@@ -34,6 +39,9 @@
 # Multi-line or multi-statement Python scripts raise NullPointerException
 #
 "Python Update With Multi Lines":
+    - skip:
+        features:
+          - warnings
     - do:
         index:
             index: test
@@ -41,13 +49,15 @@
             id: 1
             body: { "myfield": "foo" }
     - do:
+        warnings:
+          - '[python] scripts are deprecated, use [painless] scripts instead'
         update:
             index: test
             type: test
             id: 1
             body:
                 script:
-                    inline: "a=42; ctx[\"_source\"][\"myfield\"]=\"bar\""
+                    source: "a=42; ctx[\"_source\"][\"myfield\"]=\"bar\""
                     lang: python
     - do:
         get:
diff --git a/plugins/mapper-attachments/build.gradle b/plugins/mapper-attachments/build.gradle
index b00f61867e956..a99fbd4d4e60e 100644
--- a/plugins/mapper-attachments/build.gradle
+++ b/plugins/mapper-attachments/build.gradle
@@ -23,10 +23,11 @@ esplugin {
 }
 
 versions << [
-  'tika': '1.13',
-  'pdfbox': '2.0.1',
-  'bouncycastle': '1.54',
-  'poi': '3.15-beta1'
+        'tika': '1.14',
+        'pdfbox': '2.0.3',
+        'bouncycastle': '1.55',
+        'poi': '3.15',
+        'mime4j': '0.7.2'
 ]
 
 dependencies {
@@ -59,15 +60,25 @@ dependencies {
   compile "org.apache.poi:poi-scratchpad:${versions.poi}"
   // Apple iWork
   compile 'org.apache.commons:commons-compress:1.10'
+  // Outlook documents
+  compile "org.apache.james:apache-mime4j-core:${versions.mime4j}"
+  compile "org.apache.james:apache-mime4j-dom:${versions.mime4j}"
 }
 
 // TODO: stop using LanguageIdentifier...
 compileJava.options.compilerArgs << "-Xlint:-deprecation"
 
+
+dependencyLicenses {
+  mapping from: /apache-mime4j-.*/, to: 'apache-mime4j'
+}
+
 forbiddenPatterns {
+  exclude '**/*.doc'
   exclude '**/*.docx'
   exclude '**/*.pdf'
   exclude '**/*.epub'
+  exclude '**/*.vsdx'
 }
 
 thirdPartyAudit.excludes = [
@@ -216,7 +227,6 @@ thirdPartyAudit.excludes = [
   'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1VTEditList',
   'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1ValList',
   'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1ValidIdsList',
-  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1VisibleList',
   'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1WidthMinList',
   'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2Accel2List',
   'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2AccelList',
@@ -506,6 +516,8 @@ thirdPartyAudit.excludes = [
   'opennlp.tools.namefind.TokenNameFinderModel',
   'opennlp.tools.util.Span',
   'org.apache.avalon.framework.logger.Logger',
+  'org.apache.commons.collections4.ListValuedMap',
+  'org.apache.commons.collections4.multimap.ArrayListValuedHashMap',
   'org.apache.commons.csv.CSVFormat',
   'org.apache.commons.csv.CSVParser',
   'org.apache.commons.csv.CSVRecord',
@@ -525,27 +537,10 @@ thirdPartyAudit.excludes = [
   'org.apache.http.StatusLine',
   'org.apache.http.client.HttpClient',
   'org.apache.http.client.methods.HttpGet',
+  'org.apache.http.client.methods.HttpPost',
   'org.apache.http.client.utils.URIBuilder',
+  'org.apache.http.entity.ByteArrayEntity',
   'org.apache.http.impl.client.DefaultHttpClient',
-  'org.apache.james.mime4j.MimeException',
-  'org.apache.james.mime4j.codec.DecodeMonitor',
-  'org.apache.james.mime4j.codec.DecoderUtil',
-  'org.apache.james.mime4j.dom.FieldParser',
-  'org.apache.james.mime4j.dom.address.Address',
-  'org.apache.james.mime4j.dom.address.AddressList',
-  'org.apache.james.mime4j.dom.address.Mailbox',
-  'org.apache.james.mime4j.dom.address.MailboxList',
-  'org.apache.james.mime4j.dom.field.AddressListField',
-  'org.apache.james.mime4j.dom.field.DateTimeField',
-  'org.apache.james.mime4j.dom.field.MailboxListField',
-  'org.apache.james.mime4j.dom.field.ParsedField',
-  'org.apache.james.mime4j.dom.field.UnstructuredField',
-  'org.apache.james.mime4j.field.LenientFieldParser',
-  'org.apache.james.mime4j.parser.ContentHandler',
-  'org.apache.james.mime4j.parser.MimeStreamParser',
-  'org.apache.james.mime4j.stream.BodyDescriptor',
-  'org.apache.james.mime4j.stream.Field',
-  'org.apache.james.mime4j.stream.MimeConfig',
   'org.apache.jcp.xml.dsig.internal.dom.DOMDigestMethod',
   'org.apache.jcp.xml.dsig.internal.dom.DOMKeyInfo',
   'org.apache.jcp.xml.dsig.internal.dom.DOMReference',
@@ -630,8 +625,6 @@ thirdPartyAudit.excludes = [
   'org.etsi.uri.x01903.v13.impl.UnsignedSignaturePropertiesTypeImpl$1RevocationValuesList',
   'org.etsi.uri.x01903.v13.impl.UnsignedSignaturePropertiesTypeImpl$1SigAndRefsTimeStampList',
   'org.etsi.uri.x01903.v13.impl.UnsignedSignaturePropertiesTypeImpl$1SignatureTimeStampList',
-  'org.etsi.uri.x01903.v14.ValidationDataType$Factory',
-  'org.etsi.uri.x01903.v14.ValidationDataType',
   'org.json.JSONArray',
   'org.json.JSONObject',
   'org.json.XML',
@@ -785,17 +778,16 @@ thirdPartyAudit.excludes = [
   'org.openxmlformats.schemas.drawingml.x2006.main.CTGlowEffect',
   'org.openxmlformats.schemas.drawingml.x2006.main.CTGrayscaleEffect',
   'org.openxmlformats.schemas.drawingml.x2006.main.CTGrayscaleTransform',
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTGroupFillProperties',
   'org.openxmlformats.schemas.drawingml.x2006.main.CTGroupLocking',
   'org.openxmlformats.schemas.drawingml.x2006.main.CTHSLEffect',
   'org.openxmlformats.schemas.drawingml.x2006.main.CTInnerShadowEffect',
   'org.openxmlformats.schemas.drawingml.x2006.main.CTInverseGammaTransform',
   'org.openxmlformats.schemas.drawingml.x2006.main.CTInverseTransform',
   'org.openxmlformats.schemas.drawingml.x2006.main.CTLineJoinBevel',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTLineJoinMiterProperties',
   'org.openxmlformats.schemas.drawingml.x2006.main.CTLuminanceEffect',
   'org.openxmlformats.schemas.drawingml.x2006.main.CTObjectStyleDefaults',
   'org.openxmlformats.schemas.drawingml.x2006.main.CTPath2DArcTo',
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTPatternFillProperties',
   'org.openxmlformats.schemas.drawingml.x2006.main.CTPolarAdjustHandle',
   'org.openxmlformats.schemas.drawingml.x2006.main.CTPositiveFixedAngle',
   'org.openxmlformats.schemas.drawingml.x2006.main.CTPresetShadowEffect',
@@ -808,7 +800,7 @@ thirdPartyAudit.excludes = [
   'org.openxmlformats.schemas.drawingml.x2006.main.CTSoftEdgesEffect',
   'org.openxmlformats.schemas.drawingml.x2006.main.CTSupplementalFont',
   'org.openxmlformats.schemas.drawingml.x2006.main.CTTableBackgroundStyle',
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTTablePartStyle',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTTableCellBorderStyle',
   'org.openxmlformats.schemas.drawingml.x2006.main.CTTextBlipBullet',
   'org.openxmlformats.schemas.drawingml.x2006.main.CTTextBulletColorFollowText',
   'org.openxmlformats.schemas.drawingml.x2006.main.CTTextBulletSizeFollowText',
@@ -823,9 +815,10 @@ thirdPartyAudit.excludes = [
   'org.openxmlformats.schemas.drawingml.x2006.main.STBlackWhiteMode',
   'org.openxmlformats.schemas.drawingml.x2006.main.STBlipCompression',
   'org.openxmlformats.schemas.drawingml.x2006.main.STFixedAngle',
-  'org.openxmlformats.schemas.drawingml.x2006.main.STGuid',
   'org.openxmlformats.schemas.drawingml.x2006.main.STPanose',
   'org.openxmlformats.schemas.drawingml.x2006.main.STPathFillMode',
+  'org.openxmlformats.schemas.drawingml.x2006.main.STPresetPatternVal',
+  'org.openxmlformats.schemas.drawingml.x2006.main.STPresetPatternVal$Enum',
   'org.openxmlformats.schemas.drawingml.x2006.main.STRectAlignment',
   'org.openxmlformats.schemas.drawingml.x2006.main.STTextColumnCount',
   'org.openxmlformats.schemas.drawingml.x2006.main.STTextNonNegativePoint',
@@ -933,6 +926,34 @@ thirdPartyAudit.excludes = [
   'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1SatOffList',
   'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1ShadeList',
   'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1TintList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1AlphaList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1AlphaModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1AlphaOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1BlueList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1BlueModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1BlueOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1CompList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1GammaList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1GrayList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1GreenList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1GreenModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1GreenOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1HueList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1HueModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1HueOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1InvGammaList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1InvList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1LumList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1LumModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1LumOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1RedList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1RedModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1RedOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1SatList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1SatModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1SatOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1ShadeList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1TintList',
   'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1AlphaList',
   'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1AlphaModList',
   'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1AlphaOffList',
@@ -1184,6 +1205,8 @@ thirdPartyAudit.excludes = [
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTOleObjects',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTOleSize',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTPCDKPIs',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTPageField',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTPageFields',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTPhoneticRun',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTPivotFilters',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTPivotHierarchies',
@@ -1269,7 +1292,6 @@ thirdPartyAudit.excludes = [
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTMergeCellsImpl$1MergeCellList',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTNumFmtsImpl$1NumFmtList',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTPageBreakImpl$1BrkList',
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTPageFieldsImpl$1PageFieldList',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTPivotCacheRecordsImpl$1RList',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTPivotCachesImpl$1PivotCacheList',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTPivotFieldsImpl$1PivotFieldList',
diff --git a/plugins/mapper-attachments/licenses/apache-mime4j-LICENSE.txt b/plugins/mapper-attachments/licenses/apache-mime4j-LICENSE.txt
new file mode 100644
index 0000000000000..63151ee14587c
--- /dev/null
+++ b/plugins/mapper-attachments/licenses/apache-mime4j-LICENSE.txt
@@ -0,0 +1,361 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+
+
+
+   THIS PRODUCT ALSO INCLUDES THIRD PARTY SOFTWARE REDISTRIBUTED UNDER THE
+   FOLLOWING LICENSES:
+
+	Apache Commons Logging,
+	   The Apache Software License, Version 1.1 (commons-logging-1.1.1.jar)
+
+		 The Apache Software License, Version 1.1
+
+		 Redistribution and use in source and binary forms, with or without
+		 modification, are permitted provided that the following conditions
+		 are met:
+
+		 1. Redistributions of source code must retain the above copyright
+		    notice, this list of conditions and the following disclaimer.
+
+		 2. Redistributions in binary form must reproduce the above copyright
+		    notice, this list of conditions and the following disclaimer in
+		    the documentation and/or other materials provided with the
+		    distribution.
+
+		 3. The end-user documentation included with the redistribution,
+		    if any, must include the following acknowledgment:
+		       "This product includes software developed by the
+		        Apache Software Foundation (http://www.apache.org/)."
+		    Alternately, this acknowledgment may appear in the software itself,
+		    if and wherever such third-party acknowledgments normally appear.
+
+		 4. The names "Apache" and "Apache Software Foundation" must
+		    not be used to endorse or promote products derived from this
+		    software without prior written permission. For written
+		    permission, please contact apache@apache.org.
+
+		 5. Products derived from this software may not be called "Apache",
+		    nor may "Apache" appear in their name, without prior written
+		    permission of the Apache Software Foundation.
+
+		 THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
+		 WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+		 OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+		 DISCLAIMED.  IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
+		 ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+		 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+		 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+		 USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+		 ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+		 OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+		 OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+		 SUCH DAMAGE.
+
+
+	Test messages from the Perl-MIME-Tools project,
+
+					 The "Artistic License"
+
+						Preamble
+
+		The intent of this document is to state the conditions under which a
+		Package may be copied, such that the Copyright Holder maintains some
+		semblance of artistic control over the development of the package,
+		while giving the users of the package the right to use and distribute
+		the Package in a more-or-less customary fashion, plus the right to make
+		reasonable modifications.
+
+		Definitions:
+
+			"Package" refers to the collection of files distributed by the
+			Copyright Holder, and derivatives of that collection of files
+			created through textual modification.
+
+			"Standard Version" refers to such a Package if it has not been
+			modified, or has been modified in accordance with the wishes
+			of the Copyright Holder as specified below.
+
+			"Copyright Holder" is whoever is named in the copyright or
+			copyrights for the package.
+
+			"You" is you, if you're thinking about copying or distributing
+			this Package.
+
+			"Reasonable copying fee" is whatever you can justify on the
+			basis of media cost, duplication charges, time of people involved,
+			and so on.  (You will not be required to justify it to the
+			Copyright Holder, but only to the computing community at large
+			as a market that must bear the fee.)
+
+			"Freely Available" means that no fee is charged for the item
+			itself, though there may be fees involved in handling the item.
+			It also means that recipients of the item may redistribute it
+			under the same conditions they received it.
+
+		1. You may make and give away verbatim copies of the source form of the
+		Standard Version of this Package without restriction, provided that you
+		duplicate all of the original copyright notices and associated disclaimers.
+
+		2. You may apply bug fixes, portability fixes and other modifications
+		derived from the Public Domain or from the Copyright Holder.  A Package
+		modified in such a way shall still be considered the Standard Version.
+
+		3. You may otherwise modify your copy of this Package in any way, provided
+		that you insert a prominent notice in each changed file stating how and
+		when you changed that file, and provided that you do at least ONE of the
+		following:
+
+		    a) place your modifications in the Public Domain or otherwise make them
+		    Freely Available, such as by posting said modifications to Usenet or
+		    an equivalent medium, or placing the modifications on a major archive
+		    site such as uunet.uu.net, or by allowing the Copyright Holder to include
+		    your modifications in the Standard Version of the Package.
+
+		    b) use the modified Package only within your corporation or organization.
+
+		    c) rename any non-standard executables so the names do not conflict
+		    with standard executables, which must also be provided, and provide
+		    a separate manual page for each non-standard executable that clearly
+		    documents how it differs from the Standard Version.
+
+		    d) make other distribution arrangements with the Copyright Holder.
+
+		4. You may distribute the programs of this Package in object code or
+		executable form, provided that you do at least ONE of the following:
+
+		    a) distribute a Standard Version of the executables and library files,
+		    together with instructions (in the manual page or equivalent) on where
+		    to get the Standard Version.
+
+		    b) accompany the distribution with the machine-readable source of
+		    the Package with your modifications.
+
+		    c) give non-standard executables non-standard names, and clearly
+		    document the differences in manual pages (or equivalent), together
+		    with instructions on where to get the Standard Version.
+
+		    d) make other distribution arrangements with the Copyright Holder.
+
+		5. You may charge a reasonable copying fee for any distribution of this
+		Package.  You may charge any fee you choose for support of this
+		Package.  You may not charge a fee for this Package itself.  However,
+		you may distribute this Package in aggregate with other (possibly
+		commercial) programs as part of a larger (possibly commercial) software
+		distribution provided that you do not advertise this Package as a
+		product of your own.  You may embed this Package's interpreter within
+		an executable of yours (by linking); this shall be construed as a mere
+		form of aggregation, provided that the complete Standard Version of the
+		interpreter is so embedded.
+
+		6. The scripts and library files supplied as input to or produced as
+		output from the programs of this Package do not automatically fall
+		under the copyright of this Package, but belong to whoever generated
+		them, and may be sold commercially, and may be aggregated with this
+		Package.  If such scripts or library files are aggregated with this
+		Package via the so-called "undump" or "unexec" methods of producing a
+		binary executable image, then distribution of such an image shall
+		neither be construed as a distribution of this Package nor shall it
+		fall under the restrictions of Paragraphs 3 and 4, provided that you do
+		not represent such an executable image as a Standard Version of this
+		Package.
+
+		7. C subroutines (or comparably compiled subroutines in other
+		languages) supplied by you and linked into this Package in order to
+		emulate subroutines and variables of the language defined by this
+		Package shall not be considered part of this Package, but are the
+		equivalent of input as in Paragraph 6, provided these subroutines do
+		not change the language in any way that would cause it to fail the
+		regression tests for the language.
+
+		8. Aggregation of this Package with a commercial distribution is always
+		permitted provided that the use of this Package is embedded; that is,
+		when no overt attempt is made to make this Package's interfaces visible
+		to the end user of the commercial distribution.  Such use shall not be
+		construed as a distribution of this Package.
+
+		9. The name of the Copyright Holder may not be used to endorse or promote
+		products derived from this software without specific prior written permission.
+
+		10. THIS PACKAGE IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR
+		IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
+		WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
+
+						The End
+
+
diff --git a/plugins/mapper-attachments/licenses/apache-mime4j-NOTICE.txt b/plugins/mapper-attachments/licenses/apache-mime4j-NOTICE.txt
new file mode 100644
index 0000000000000..61523975eb2f7
--- /dev/null
+++ b/plugins/mapper-attachments/licenses/apache-mime4j-NOTICE.txt
@@ -0,0 +1,13 @@
+   =========================================================================
+   ==      NOTICE file for use with the Apache License, Version 2.0,      ==
+   =========================================================================
+   
+   Apache JAMES Mime4j
+   Copyright 2004-2010 The Apache Software Foundation
+   
+   This product includes software developed at
+   The Apache Software Foundation (http://www.apache.org/).
+
+   This product test suite includes data (mimetools-testmsgs folder) developed 
+   by Eryq and ZeeGee Software Inc as part of the "MIME-tools" Perl5 toolkit
+   and licensed under the Artistic License
diff --git a/plugins/mapper-attachments/licenses/apache-mime4j-core-0.7.2.jar.sha1 b/plugins/mapper-attachments/licenses/apache-mime4j-core-0.7.2.jar.sha1
new file mode 100644
index 0000000000000..8210fc7fc162c
--- /dev/null
+++ b/plugins/mapper-attachments/licenses/apache-mime4j-core-0.7.2.jar.sha1
@@ -0,0 +1 @@
+a81264fe0265ebe8fd1d8128aad06dc320de6eef
\ No newline at end of file
diff --git a/plugins/mapper-attachments/licenses/apache-mime4j-dom-0.7.2.jar.sha1 b/plugins/mapper-attachments/licenses/apache-mime4j-dom-0.7.2.jar.sha1
new file mode 100644
index 0000000000000..3ede85a019108
--- /dev/null
+++ b/plugins/mapper-attachments/licenses/apache-mime4j-dom-0.7.2.jar.sha1
@@ -0,0 +1 @@
+1c289aa264548a0a1f1b43685a9cb2ab23f67287
\ No newline at end of file
diff --git a/plugins/mapper-attachments/licenses/bcmail-jdk15on-1.54.jar.sha1 b/plugins/mapper-attachments/licenses/bcmail-jdk15on-1.54.jar.sha1
deleted file mode 100644
index 79da45c5c422e..0000000000000
--- a/plugins/mapper-attachments/licenses/bcmail-jdk15on-1.54.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-9d9b5432b4b29ef4a853223bc6e19379ef116cca
\ No newline at end of file
diff --git a/plugins/mapper-attachments/licenses/bcmail-jdk15on-1.55.jar.sha1 b/plugins/mapper-attachments/licenses/bcmail-jdk15on-1.55.jar.sha1
new file mode 100644
index 0000000000000..8fdfb8f580780
--- /dev/null
+++ b/plugins/mapper-attachments/licenses/bcmail-jdk15on-1.55.jar.sha1
@@ -0,0 +1 @@
+5cea2dada69b98698ea975a5c1dd3c91ac8ffbb6
\ No newline at end of file
diff --git a/plugins/mapper-attachments/licenses/bcpkix-jdk15on-1.54.jar.sha1 b/plugins/mapper-attachments/licenses/bcpkix-jdk15on-1.54.jar.sha1
deleted file mode 100644
index 2d0c3cf4e277a..0000000000000
--- a/plugins/mapper-attachments/licenses/bcpkix-jdk15on-1.54.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b11bfee99bb11eea344de6e4a07fe89212c55c02
\ No newline at end of file
diff --git a/plugins/mapper-attachments/licenses/bcpkix-jdk15on-1.55.jar.sha1 b/plugins/mapper-attachments/licenses/bcpkix-jdk15on-1.55.jar.sha1
new file mode 100644
index 0000000000000..a4d546be04fc2
--- /dev/null
+++ b/plugins/mapper-attachments/licenses/bcpkix-jdk15on-1.55.jar.sha1
@@ -0,0 +1 @@
+6392d8cba22b722c6570d660ca0b3921ff1bae4f
\ No newline at end of file
diff --git a/plugins/mapper-attachments/licenses/bcprov-jdk15on-1.54.jar.sha1 b/plugins/mapper-attachments/licenses/bcprov-jdk15on-1.54.jar.sha1
deleted file mode 100644
index fcda646b42a8d..0000000000000
--- a/plugins/mapper-attachments/licenses/bcprov-jdk15on-1.54.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-1acdedeb89f1d950d67b73d481eb7736df65eedb
\ No newline at end of file
diff --git a/plugins/mapper-attachments/licenses/bcprov-jdk15on-1.55.jar.sha1 b/plugins/mapper-attachments/licenses/bcprov-jdk15on-1.55.jar.sha1
new file mode 100644
index 0000000000000..1c507e17b88e0
--- /dev/null
+++ b/plugins/mapper-attachments/licenses/bcprov-jdk15on-1.55.jar.sha1
@@ -0,0 +1 @@
+935f2e57a00ec2c489cbd2ad830d4a399708f979
\ No newline at end of file
diff --git a/plugins/mapper-attachments/licenses/fontbox-2.0.1.jar.sha1 b/plugins/mapper-attachments/licenses/fontbox-2.0.1.jar.sha1
deleted file mode 100644
index 0668199b242fd..0000000000000
--- a/plugins/mapper-attachments/licenses/fontbox-2.0.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b9d4f0993e015f3f1ce0be9e7300cf62dd7a7f15
\ No newline at end of file
diff --git a/plugins/mapper-attachments/licenses/fontbox-2.0.3.jar.sha1 b/plugins/mapper-attachments/licenses/fontbox-2.0.3.jar.sha1
new file mode 100644
index 0000000000000..e3ff3d39459f3
--- /dev/null
+++ b/plugins/mapper-attachments/licenses/fontbox-2.0.3.jar.sha1
@@ -0,0 +1 @@
+448ee588d0136121cf5c4dd397384cccb9db1ad7
\ No newline at end of file
diff --git a/plugins/mapper-attachments/licenses/pdfbox-2.0.1.jar.sha1 b/plugins/mapper-attachments/licenses/pdfbox-2.0.1.jar.sha1
deleted file mode 100644
index 1014db340445f..0000000000000
--- a/plugins/mapper-attachments/licenses/pdfbox-2.0.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-dbc69649118b7eff278f228c070a40ee559e1f62
\ No newline at end of file
diff --git a/plugins/mapper-attachments/licenses/pdfbox-2.0.3.jar.sha1 b/plugins/mapper-attachments/licenses/pdfbox-2.0.3.jar.sha1
new file mode 100644
index 0000000000000..807e2482ac295
--- /dev/null
+++ b/plugins/mapper-attachments/licenses/pdfbox-2.0.3.jar.sha1
@@ -0,0 +1 @@
+be7b09de93f7c7795c57f4fbf14db60ab93806b4
\ No newline at end of file
diff --git a/plugins/mapper-attachments/licenses/poi-3.15-beta1.jar.sha1 b/plugins/mapper-attachments/licenses/poi-3.15-beta1.jar.sha1
deleted file mode 100644
index 6049604dd9751..0000000000000
--- a/plugins/mapper-attachments/licenses/poi-3.15-beta1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-048bb8326b81323631d9ceb4236cfbd382e56da2
\ No newline at end of file
diff --git a/plugins/mapper-attachments/licenses/poi-3.15.jar.sha1 b/plugins/mapper-attachments/licenses/poi-3.15.jar.sha1
new file mode 100644
index 0000000000000..5405d32cd0108
--- /dev/null
+++ b/plugins/mapper-attachments/licenses/poi-3.15.jar.sha1
@@ -0,0 +1 @@
+965bba8899988008bb2341e300347de62aad5391
\ No newline at end of file
diff --git a/plugins/mapper-attachments/licenses/poi-ooxml-3.15-beta1.jar.sha1 b/plugins/mapper-attachments/licenses/poi-ooxml-3.15-beta1.jar.sha1
deleted file mode 100644
index c3cf49d9246d0..0000000000000
--- a/plugins/mapper-attachments/licenses/poi-ooxml-3.15-beta1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-81085a47fdf0d74d473d605c6b3784e26731842e
\ No newline at end of file
diff --git a/plugins/mapper-attachments/licenses/poi-ooxml-3.15.jar.sha1 b/plugins/mapper-attachments/licenses/poi-ooxml-3.15.jar.sha1
new file mode 100644
index 0000000000000..4362223dac4a4
--- /dev/null
+++ b/plugins/mapper-attachments/licenses/poi-ooxml-3.15.jar.sha1
@@ -0,0 +1 @@
+e2800856735b07b8edd417aee07685470216a00f
\ No newline at end of file
diff --git a/plugins/mapper-attachments/licenses/poi-ooxml-schemas-3.15-beta1.jar.sha1 b/plugins/mapper-attachments/licenses/poi-ooxml-schemas-3.15-beta1.jar.sha1
deleted file mode 100644
index afd3b676d08e8..0000000000000
--- a/plugins/mapper-attachments/licenses/poi-ooxml-schemas-3.15-beta1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f8bc979ad79908a99483337f1ca2edf78558ac20
\ No newline at end of file
diff --git a/plugins/mapper-attachments/licenses/poi-ooxml-schemas-3.15.jar.sha1 b/plugins/mapper-attachments/licenses/poi-ooxml-schemas-3.15.jar.sha1
new file mode 100644
index 0000000000000..393e6f885d9f2
--- /dev/null
+++ b/plugins/mapper-attachments/licenses/poi-ooxml-schemas-3.15.jar.sha1
@@ -0,0 +1 @@
+de4a50ca39de48a19606b35644ecadb2f733c479
\ No newline at end of file
diff --git a/plugins/mapper-attachments/licenses/poi-scratchpad-3.15-beta1.jar.sha1 b/plugins/mapper-attachments/licenses/poi-scratchpad-3.15-beta1.jar.sha1
deleted file mode 100644
index 7056a9fa49ed0..0000000000000
--- a/plugins/mapper-attachments/licenses/poi-scratchpad-3.15-beta1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f4e276aaf97a60a1156388c9e38069122b7ea914
\ No newline at end of file
diff --git a/plugins/mapper-attachments/licenses/poi-scratchpad-3.15.jar.sha1 b/plugins/mapper-attachments/licenses/poi-scratchpad-3.15.jar.sha1
new file mode 100644
index 0000000000000..d08f475a3f60a
--- /dev/null
+++ b/plugins/mapper-attachments/licenses/poi-scratchpad-3.15.jar.sha1
@@ -0,0 +1 @@
+f1db76ae4a9389fa4339dc3b7f8208aa82c72b04
\ No newline at end of file
diff --git a/plugins/mapper-attachments/licenses/tika-core-1.13.jar.sha1 b/plugins/mapper-attachments/licenses/tika-core-1.13.jar.sha1
deleted file mode 100644
index cfc36a450bd77..0000000000000
--- a/plugins/mapper-attachments/licenses/tika-core-1.13.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-1305c798d41d1d7bbf12cb7c0ca184c98eed25ad
\ No newline at end of file
diff --git a/plugins/mapper-attachments/licenses/tika-core-1.14.jar.sha1 b/plugins/mapper-attachments/licenses/tika-core-1.14.jar.sha1
new file mode 100644
index 0000000000000..2da2c07d9b27f
--- /dev/null
+++ b/plugins/mapper-attachments/licenses/tika-core-1.14.jar.sha1
@@ -0,0 +1 @@
+afff8f1774994aa973ef90bc8d38ddf089b9d6d9
\ No newline at end of file
diff --git a/plugins/mapper-attachments/licenses/tika-parsers-1.13.jar.sha1 b/plugins/mapper-attachments/licenses/tika-parsers-1.13.jar.sha1
deleted file mode 100644
index 7fb2755d545e0..0000000000000
--- a/plugins/mapper-attachments/licenses/tika-parsers-1.13.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-374fde67b9d35f785534b0e6c4953533c31bab5f
\ No newline at end of file
diff --git a/plugins/mapper-attachments/licenses/tika-parsers-1.14.jar.sha1 b/plugins/mapper-attachments/licenses/tika-parsers-1.14.jar.sha1
new file mode 100644
index 0000000000000..b96b3cdefdd1a
--- /dev/null
+++ b/plugins/mapper-attachments/licenses/tika-parsers-1.14.jar.sha1
@@ -0,0 +1 @@
+d26c10a9e7d116366562aa260013a30a55ff4e8f
\ No newline at end of file
diff --git a/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/AttachmentMapper.java b/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/AttachmentMapper.java
index 1031f0d103434..a0a0096eafe44 100644
--- a/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/AttachmentMapper.java
+++ b/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/AttachmentMapper.java
@@ -20,8 +20,8 @@
 package org.elasticsearch.mapper.attachments;
 
 import org.apache.logging.log4j.Logger;
-import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.search.Query;
 import org.apache.tika.language.LanguageIdentifier;
 import org.apache.tika.metadata.Metadata;
@@ -106,7 +106,7 @@ public static class FieldNames {
     }
 
     static final class AttachmentFieldType extends MappedFieldType {
-        public AttachmentFieldType() {}
+        AttachmentFieldType() {}
 
         protected AttachmentFieldType(AttachmentMapper.AttachmentFieldType ref) {
             super(ref);
@@ -305,6 +305,9 @@ public static class TypeParser implements Mapper.TypeParser {
                 type = "text";
             }
             Mapper.TypeParser typeParser = parserContext.typeParser(type);
+            if (typeParser == null) {
+                throw new MapperParsingException("Type [" + type + "] is not supported. Check your [" + propName + "] field.");
+            }
             Mapper.Builder<?, ?> mapperBuilder = typeParser.parse(propName, propNode, parserContext);
 
             return mapperBuilder;
@@ -601,7 +604,7 @@ public Mapper parse(ParseContext context) throws IOException {
     }
 
     @Override
-    protected void parseCreateField(ParseContext parseContext, List<Field> fields) throws IOException {
+    protected void parseCreateField(ParseContext parseContext, List<IndexableField> fields) throws IOException {
 
     }
 
diff --git a/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/TikaImpl.java b/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/TikaImpl.java
index 2babda8ad00ff..0033b48376b08 100644
--- a/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/TikaImpl.java
+++ b/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/TikaImpl.java
@@ -22,8 +22,10 @@
 import org.apache.tika.Tika;
 import org.apache.tika.exception.TikaException;
 import org.apache.tika.metadata.Metadata;
+import org.apache.tika.mime.MediaType;
 import org.apache.tika.parser.AutoDetectParser;
 import org.apache.tika.parser.Parser;
+import org.apache.tika.parser.ParserDecorator;
 import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.bootstrap.JarHell;
 import org.elasticsearch.common.SuppressForbidden;
@@ -45,7 +47,11 @@
 import java.security.PrivilegedExceptionAction;
 import java.security.ProtectionDomain;
 import java.security.SecurityPermission;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashSet;
 import java.util.PropertyPermission;
+import java.util.Set;
 
 /**
  * Runs tika with limited parsers and limited permissions.
@@ -54,6 +60,9 @@
  */
 final class TikaImpl {
 
+    /** Exclude some formats */
+    private static final Set<MediaType> EXCLUDES = Collections.singleton(MediaType.application("x-tika-ooxml"));
+
     /** subset of parsers for types we support */
     private static final Parser PARSERS[] = new Parser[] {
         // documents
@@ -63,7 +72,7 @@ final class TikaImpl {
         new org.apache.tika.parser.txt.TXTParser(),
         new org.apache.tika.parser.microsoft.OfficeParser(),
         new org.apache.tika.parser.microsoft.OldExcelParser(),
-        new org.apache.tika.parser.microsoft.ooxml.OOXMLParser(),
+        ParserDecorator.withoutTypes(new org.apache.tika.parser.microsoft.ooxml.OOXMLParser(), EXCLUDES),
         new org.apache.tika.parser.odf.OpenDocumentParser(),
         new org.apache.tika.parser.iwork.IWorkPackageParser(),
         new org.apache.tika.parser.xml.DcXMLParser(),
@@ -128,7 +137,12 @@ static PermissionCollection getRestrictedPermissions() {
         addReadPermissions(perms, JarHell.parseClassPath());
         // plugin jars
         if (TikaImpl.class.getClassLoader() instanceof URLClassLoader) {
-            addReadPermissions(perms, ((URLClassLoader)TikaImpl.class.getClassLoader()).getURLs());
+            URL[] urls = ((URLClassLoader)TikaImpl.class.getClassLoader()).getURLs();
+            Set<URL> set = new LinkedHashSet<>(Arrays.asList(urls));
+            if (set.size() != urls.length) {
+                throw new AssertionError("duplicate jars: " + Arrays.toString(urls));
+            }
+            addReadPermissions(perms, set);
         }
         // jvm's java.io.tmpdir (needs read/write)
         perms.add(new FilePermission(System.getProperty("java.io.tmpdir") + System.getProperty("file.separator") + "-",
@@ -145,7 +159,7 @@ static PermissionCollection getRestrictedPermissions() {
 
     // add resources to (what is typically) a jar, but might not be (e.g. in tests/IDE)
     @SuppressForbidden(reason = "adds access to jar resources")
-    static void addReadPermissions(Permissions perms, URL resources[]) {
+    static void addReadPermissions(Permissions perms, Set<URL> resources) {
         try {
             for (URL url : resources) {
                 Path path = PathUtils.get(url.toURI());
diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/DateAttachmentMapperTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/DateAttachmentMapperTests.java
index 635c9bee859ca..4165e659e1678 100644
--- a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/DateAttachmentMapperTests.java
+++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/DateAttachmentMapperTests.java
@@ -39,7 +39,9 @@ public class DateAttachmentMapperTests extends AttachmentUnitTestCase {
 
     @Before
     public void setupMapperParser() throws Exception {
-        mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser();
+        mapperParser = MapperTestUtils
+                .newMapperService(xContentRegistry(), createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper())
+                .documentMapperParser();
     }
 
     public void testSimpleMappings() throws Exception {
diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/EncryptedDocMapperTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/EncryptedDocMapperTests.java
index a10910cf62a30..e9e66839c78d7 100644
--- a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/EncryptedDocMapperTests.java
+++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/EncryptedDocMapperTests.java
@@ -47,7 +47,7 @@
 public class EncryptedDocMapperTests extends AttachmentUnitTestCase {
 
     public void testMultipleDocsEncryptedLast() throws IOException {
-        DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser();
+        DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser();
 
         String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/encrypted/test-mapping.json");
         DocumentMapper docMapper = mapperParser.parse("person", new CompressedXContent(mapping));
@@ -77,7 +77,7 @@ public void testMultipleDocsEncryptedLast() throws IOException {
     }
 
     public void testMultipleDocsEncryptedFirst() throws IOException {
-        DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser();
+        DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser();
         String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/encrypted/test-mapping.json");
         DocumentMapper docMapper = mapperParser.parse("person", new CompressedXContent(mapping));
         byte[] html = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/htmlWithValidDateMeta.html");
@@ -107,7 +107,7 @@ public void testMultipleDocsEncryptedFirst() throws IOException {
 
     public void testMultipleDocsEncryptedNotIgnoringErrors() throws IOException {
         try {
-            DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(),
+            DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(),
                 Settings.builder().put("index.mapping.attachment.ignore_errors", false).build(),
                 getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser();
 
diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/LanguageDetectionAttachmentMapperTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/LanguageDetectionAttachmentMapperTests.java
index 8c09c78740aba..e3c0cd0d48d09 100644
--- a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/LanguageDetectionAttachmentMapperTests.java
+++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/LanguageDetectionAttachmentMapperTests.java
@@ -51,7 +51,7 @@ public void setupMapperParser() throws IOException {
     }
 
     public void setupMapperParser(boolean langDetect) throws IOException {
-        DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(),
+        DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(),
             Settings.builder().put("index.mapping.attachment.detect_language", langDetect).build(),
             getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser();
         String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/language/language-mapping.json");
diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MapperAttachmentsClientYamlTestSuiteIT.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MapperAttachmentsClientYamlTestSuiteIT.java
index 53a5c5c2813d5..0846b3968cc25 100644
--- a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MapperAttachmentsClientYamlTestSuiteIT.java
+++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MapperAttachmentsClientYamlTestSuiteIT.java
@@ -24,9 +24,6 @@
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-
-import java.io.IOException;
 
 public class MapperAttachmentsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
@@ -35,8 +32,13 @@ public MapperAttachmentsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandid
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
-        return createParameters(0, 1);
+    public static Iterable<Object[]> parameters() throws Exception {
+        return createParameters();
+    }
+
+    @Override
+    protected boolean randomizeContentType() {
+        return false;
     }
 }
 
diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MetadataMapperTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MetadataMapperTests.java
index cb8f6746a8487..9013c9a161449 100644
--- a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MetadataMapperTests.java
+++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MetadataMapperTests.java
@@ -50,7 +50,9 @@ protected void checkMeta(String filename, Settings otherSettings, Long expectedD
                                              .put(this.testSettings)
                                              .put(otherSettings)
                                              .build();
-        DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), settings, getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser();
+        DocumentMapperParser mapperParser = MapperTestUtils
+                .newMapperService(xContentRegistry(), createTempDir(), settings, getIndicesModuleWithRegisteredAttachmentMapper())
+                .documentMapperParser();
 
         String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/metadata/test-mapping.json");
         DocumentMapper docMapper = mapperParser.parse("person", new CompressedXContent(mapping));
diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MultifieldAttachmentMapperTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MultifieldAttachmentMapperTests.java
index 84284aa81280c..371dea8f55c67 100644
--- a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MultifieldAttachmentMapperTests.java
+++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MultifieldAttachmentMapperTests.java
@@ -49,7 +49,9 @@ public class MultifieldAttachmentMapperTests extends AttachmentUnitTestCase {
 
     @Before
     public void setupMapperParser() throws Exception {
-        mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser();
+        mapperParser = MapperTestUtils
+                .newMapperService(xContentRegistry(), createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper())
+                .documentMapperParser();
 
     }
 
@@ -86,7 +88,8 @@ public void testExternalValues() throws Exception {
 
         String bytes = Base64.getEncoder().encodeToString(originalText.getBytes(StandardCharsets.ISO_8859_1));
 
-        MapperService mapperService = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper());
+        MapperService mapperService = MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), Settings.EMPTY,
+                getIndicesModuleWithRegisteredAttachmentMapper());
 
         String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/multifield/multifield-mapping.json");
 
@@ -146,13 +149,13 @@ public void testExternalValues() throws Exception {
 
     public void testAllExternalValues() throws Exception {
         String originalText = "This is an elasticsearch mapper attachment test.";
-        String forcedName = randomAsciiOfLength(20);
-        String forcedLanguage = randomAsciiOfLength(20);
-        String forcedContentType = randomAsciiOfLength(20);
+        String forcedName = randomAlphaOfLength(20);
+        String forcedLanguage = randomAlphaOfLength(20);
+        String forcedContentType = randomAlphaOfLength(20);
 
         String bytes = Base64.getEncoder().encodeToString(originalText.getBytes(StandardCharsets.ISO_8859_1));
 
-        MapperService mapperService = MapperTestUtils.newMapperService(createTempDir(),
+        MapperService mapperService = MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(),
             Settings.builder().put(AttachmentMapper.INDEX_ATTACHMENT_DETECT_LANGUAGE_SETTING.getKey(), true).build(),
             getIndicesModuleWithRegisteredAttachmentMapper());
 
diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/SimpleAttachmentMapperTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/SimpleAttachmentMapperTests.java
index bb0f2aca4d568..67d65e8111c83 100644
--- a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/SimpleAttachmentMapperTests.java
+++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/SimpleAttachmentMapperTests.java
@@ -46,7 +46,9 @@
 public class SimpleAttachmentMapperTests extends AttachmentUnitTestCase {
 
     public void testSimpleMappings() throws Exception {
-        DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser();
+        DocumentMapperParser mapperParser = MapperTestUtils
+                .newMapperService(xContentRegistry(), createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper())
+                .documentMapperParser();
         String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/simple/test-mapping.json");
         DocumentMapper docMapper = mapperParser.parse("person", new CompressedXContent(mapping));
         byte[] html = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/testXHTML.html");
@@ -75,7 +77,9 @@ public void testSimpleMappings() throws Exception {
      * test for https://github.com/elastic/elasticsearch-mapper-attachments/issues/179
      */
     public void testSimpleMappingsWithAllFields() throws Exception {
-        DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser();
+        DocumentMapperParser mapperParser = MapperTestUtils
+                .newMapperService(xContentRegistry(), createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper())
+                .documentMapperParser();
         String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/simple/test-mapping-all-fields.json");
         DocumentMapper docMapper = mapperParser.parse("person", new CompressedXContent(mapping));
         byte[] html = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/testXHTML.html");
@@ -121,7 +125,8 @@ public void testMapperErrorWithDotTwoLevels169() throws Exception {
                 .endObject();
 
         byte[] mapping = BytesReference.toBytes(mappingBuilder.bytes());
-        MapperService mapperService = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper());
+        MapperService mapperService = MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), Settings.EMPTY,
+                getIndicesModuleWithRegisteredAttachmentMapper());
         DocumentMapper docMapper = mapperService.parse("mail", new CompressedXContent(mapping), true);
         // this should not throw an exception
         mapperService.parse("mail", new CompressedXContent(docMapper.mapping().toString()), true);
@@ -141,7 +146,8 @@ public void testEmptyName() throws Exception {
             .endObject()
             .endObject();
         String mapping = mappingBuilder.string();
-        MapperService mapperService = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper());
+        MapperService mapperService = MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), Settings.EMPTY,
+                getIndicesModuleWithRegisteredAttachmentMapper());
         DocumentMapperParser parser = mapperService.documentMapperParser();
 
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
@@ -152,7 +158,8 @@ public void testEmptyName() throws Exception {
         // before 5.x
         Version oldVersion = VersionUtils.randomVersionBetween(getRandom(), Version.V_2_0_0, Version.V_2_3_5);
         Settings oldIndexSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, oldVersion).build();
-        MapperService mapperService2x = MapperTestUtils.newMapperService(createTempDir(), oldIndexSettings, getIndicesModuleWithRegisteredAttachmentMapper());
+        MapperService mapperService2x = MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), oldIndexSettings,
+                getIndicesModuleWithRegisteredAttachmentMapper());
         DocumentMapperParser parser2x = mapperService2x.documentMapperParser();
 
         DocumentMapper defaultMapper = parser2x.parse("mail", new CompressedXContent(mapping));
diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/TikaDocTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/TikaDocTests.java
index b32a6ab79a090..43ff2bd4e04d2 100644
--- a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/TikaDocTests.java
+++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/TikaDocTests.java
@@ -22,6 +22,7 @@
 import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems;
 import org.apache.lucene.util.TestUtil;
 import org.apache.tika.metadata.Metadata;
+import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.test.ESTestCase;
 
 import java.nio.file.DirectoryStream;
@@ -37,16 +38,22 @@
 public class TikaDocTests extends ESTestCase {
 
     /** some test files from tika test suite, zipped up */
-    static final String TIKA_FILES = "/org/elasticsearch/index/mapper/attachment/test/tika-files.zip";
+    static final String TIKA_FILES = "/org/elasticsearch/index/mapper/attachment/test/tika-files/";
 
     public void testFiles() throws Exception {
         Path tmp = createTempDir();
-        TestUtil.unzip(getClass().getResourceAsStream(TIKA_FILES), tmp);
+        logger.debug("unzipping all tika sample files");
+        try (DirectoryStream<Path> stream = Files.newDirectoryStream(PathUtils.get(getClass().getResource(TIKA_FILES).toURI()))) {
+            for (Path doc : stream) {
+                String filename = doc.getFileName().toString();
+                TestUtil.unzip(getClass().getResourceAsStream(TIKA_FILES + filename), tmp);
+            }
+        }
 
         try (DirectoryStream<Path> stream = Files.newDirectoryStream(tmp)) {
             for (Path doc : stream) {
-              logger.debug("parsing: {}", doc);
-              assertParseable(doc);
+                logger.debug("parsing: {}", doc);
+                assertParseable(doc);
             }
         }
     }
diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/TikaImplTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/TikaImplTests.java
index f42110c1e62fe..1c90935ab293c 100644
--- a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/TikaImplTests.java
+++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/TikaImplTests.java
@@ -26,5 +26,4 @@ public class TikaImplTests extends ESTestCase {
   public void testTikaLoads() throws Exception {
     Class.forName("org.elasticsearch.mapper.attachments.TikaImpl");
   }
-
 }
diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/VariousDocTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/VariousDocTests.java
index 17e4ec6818a17..1fcbb86aeb01e 100644
--- a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/VariousDocTests.java
+++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/VariousDocTests.java
@@ -44,7 +44,9 @@
 import static org.elasticsearch.mapper.attachments.AttachmentMapper.FieldNames.TITLE;
 import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath;
 import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath;
+import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.isEmptyOrNullString;
+import static org.hamcrest.Matchers.isEmptyString;
 import static org.hamcrest.Matchers.not;
 
 /**
@@ -56,7 +58,9 @@ public class VariousDocTests extends AttachmentUnitTestCase {
 
     @Before
     public void createMapper() throws IOException {
-        DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser();
+        DocumentMapperParser mapperParser = MapperTestUtils
+                .newMapperService(xContentRegistry(), createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper())
+                .documentMapperParser();
 
         String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/various-doc/test-mapping.json");
         docMapper = mapperParser.parse("person", new CompressedXContent(mapping));
@@ -119,6 +123,40 @@ public void testAsciidocDocument() throws Exception {
         testMapper("asciidoc.asciidoc", false);
     }
 
+    public void testWordDocumentWithVisioSchema() throws Exception {
+        assertParseable("issue-22077.docx");
+        testMapper("issue-22077.docx", false);
+    }
+
+    public void testLegacyWordDocumentWithVisioSchema() throws Exception {
+        assertParseable("issue-22077.doc");
+        testMapper("issue-22077.doc", false);
+    }
+
+    public void testVisioIsExcluded() throws Exception {
+        String filename = "issue-22077.vsdx";
+        try (InputStream is = VariousDocTests.class.getResourceAsStream("/org/elasticsearch/index/mapper/attachment/test/sample-files/" +
+            filename)) {
+            byte bytes[] = IOUtils.toByteArray(is);
+            String parsedContent = TikaImpl.parse(bytes, new Metadata(), -1);
+            assertThat(parsedContent, isEmptyString());
+        }
+
+        byte[] html = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/" + filename);
+        BytesReference json = jsonBuilder()
+                .startObject()
+                    .startObject("file")
+                        .field("_name", filename)
+                        .field("_content", html)
+                    .endObject()
+                .endObject().bytes();
+
+        ParseContext.Document doc =  docMapper.parse("person", "person", "1", json).rootDoc();
+        assertThat(doc.get(docMapper.mappers().getMapper("file.content").fieldType().name()), isEmptyString());
+        assertThat(doc.get(docMapper.mappers().getMapper("file.content_type").fieldType().name()), is("application/vnd.ms-visio.drawing"));
+        assertThat(doc.get(docMapper.mappers().getMapper("file.content_length").fieldType().name()), is("210451"));
+    }
+
     void assertException(String filename, String expectedMessage) throws Exception {
         try (InputStream is = VariousDocTests.class.getResourceAsStream("/org/elasticsearch/index/mapper/attachment/test/sample-files/" + filename)) {
             byte bytes[] = IOUtils.toByteArray(is);
diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/WrongAttachmentMapperTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/WrongAttachmentMapperTests.java
new file mode 100644
index 0000000000000..8105d77151ac1
--- /dev/null
+++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/WrongAttachmentMapperTests.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.mapper.attachments;
+
+import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.MapperTestUtils;
+import org.elasticsearch.index.mapper.DocumentMapperParser;
+import org.elasticsearch.index.mapper.MapperParsingException;
+import org.junit.Before;
+
+import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath;
+import static org.hamcrest.Matchers.is;
+
+/**
+ *
+ */
+public class WrongAttachmentMapperTests extends AttachmentUnitTestCase {
+
+    private DocumentMapperParser mapperParser;
+
+    @Before
+    public void setupMapperParser() throws Exception {
+        mapperParser = MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), Settings.EMPTY,
+            getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser();
+
+    }
+
+    public void testSimpleMappings() throws Exception {
+        String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/wrong-mapping/wrong-mapping.json");
+
+        try {
+            mapperParser.parse("person", new CompressedXContent(mapping));
+            fail("We should have raised a MapperParsingException");
+        } catch (MapperParsingException e) {
+            assertThat(e.getMessage(), is("Type [nonexistingtype] is not supported. Check your [content] field."));
+        }
+    }
+}
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/issue-22077.doc b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/issue-22077.doc
new file mode 100644
index 0000000000000..10badd5809be6
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/issue-22077.doc differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/issue-22077.docx b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/issue-22077.docx
new file mode 100644
index 0000000000000..bab550607a915
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/issue-22077.docx differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/issue-22077.vsdx b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/issue-22077.vsdx
new file mode 100644
index 0000000000000..fb9cde51b4b5b
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/issue-22077.vsdx differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files.zip
deleted file mode 100644
index 10f5d50767733..0000000000000
Binary files a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files.zip and /dev/null differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/Doc1_ole.doc.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/Doc1_ole.doc.zip
new file mode 100644
index 0000000000000..f9bd7c46b2bf1
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/Doc1_ole.doc.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/EmbeddedDocument.docx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/EmbeddedDocument.docx.zip
new file mode 100644
index 0000000000000..bb4e8e1ef3e4b
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/EmbeddedDocument.docx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/EmbeddedOutlook.docx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/EmbeddedOutlook.docx.zip
new file mode 100644
index 0000000000000..67de787f017e6
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/EmbeddedOutlook.docx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/EmbeddedPDF.docx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/EmbeddedPDF.docx.zip
new file mode 100644
index 0000000000000..7e02f55c03dec
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/EmbeddedPDF.docx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/NullHeader.docx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/NullHeader.docx.zip
new file mode 100644
index 0000000000000..0516c35b0710c
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/NullHeader.docx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/big-preamble.html.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/big-preamble.html.zip
new file mode 100644
index 0000000000000..fc851923abbd5
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/big-preamble.html.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/boilerplate-whitespace.html.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/boilerplate-whitespace.html.zip
new file mode 100644
index 0000000000000..2d1f8e601ce88
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/boilerplate-whitespace.html.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/boilerplate.html.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/boilerplate.html.zip
new file mode 100644
index 0000000000000..c728a6303b4ab
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/boilerplate.html.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/footnotes.docx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/footnotes.docx.zip
new file mode 100644
index 0000000000000..21376b6917945
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/footnotes.docx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/headerPic.docx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/headerPic.docx.zip
new file mode 100644
index 0000000000000..6bb9b3f463d29
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/headerPic.docx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/headers.mbox.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/headers.mbox.zip
new file mode 100644
index 0000000000000..f36e6b8de4724
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/headers.mbox.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/pictures.ppt.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/pictures.ppt.zip
new file mode 100644
index 0000000000000..55dde5eada7b6
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/pictures.ppt.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/protect.xlsx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/protect.xlsx.zip
new file mode 100644
index 0000000000000..9823a8debf8be
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/protect.xlsx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/protectedFile.xlsx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/protectedFile.xlsx.zip
new file mode 100644
index 0000000000000..b4de09b0fdc45
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/protectedFile.xlsx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/protectedSheets.xlsx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/protectedSheets.xlsx.zip
new file mode 100644
index 0000000000000..8dde8a493a20b
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/protectedSheets.xlsx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/resume.html.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/resume.html.zip
new file mode 100644
index 0000000000000..0c53eae3b3aec
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/resume.html.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/test-outlook.msg.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/test-outlook.msg.zip
new file mode 100644
index 0000000000000..e9a0bfbed00c9
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/test-outlook.msg.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/test-outlook2003.msg.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/test-outlook2003.msg.zip
new file mode 100644
index 0000000000000..bce3da6e05b81
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/test-outlook2003.msg.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/test.doc.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/test.doc.zip
new file mode 100644
index 0000000000000..4bcf68dcebc95
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/test.doc.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testAnnotations.pdf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testAnnotations.pdf.zip
new file mode 100644
index 0000000000000..3268c952a3330
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testAnnotations.pdf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testBinControlWord.rtf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testBinControlWord.rtf.zip
new file mode 100644
index 0000000000000..4a7efa19f0a0e
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testBinControlWord.rtf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testBulletPoints.key.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testBulletPoints.key.zip
new file mode 100644
index 0000000000000..41b896525ca02
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testBulletPoints.key.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testComment.doc.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testComment.doc.zip
new file mode 100644
index 0000000000000..15a056cb3a3e4
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testComment.doc.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testComment.docx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testComment.docx.zip
new file mode 100644
index 0000000000000..30b4f6877a8c7
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testComment.docx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testComment.pdf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testComment.pdf.zip
new file mode 100644
index 0000000000000..bb21a9d71039f
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testComment.pdf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testComment.ppt.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testComment.ppt.zip
new file mode 100644
index 0000000000000..e0122d4004bf5
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testComment.ppt.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testComment.pptx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testComment.pptx.zip
new file mode 100644
index 0000000000000..77967db80c69a
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testComment.pptx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testComment.rtf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testComment.rtf.zip
new file mode 100644
index 0000000000000..b7b6bdf5c1316
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testComment.rtf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testComment.xls.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testComment.xls.zip
new file mode 100644
index 0000000000000..7250235b89c5a
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testComment.xls.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testComment.xlsx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testComment.xlsx.zip
new file mode 100644
index 0000000000000..7fe71de731d53
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testComment.xlsx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testDOCX_Thumbnail.docx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testDOCX_Thumbnail.docx.zip
new file mode 100644
index 0000000000000..e0fd82103fe0c
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testDOCX_Thumbnail.docx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL-charts.xls.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL-charts.xls.zip
new file mode 100644
index 0000000000000..fd5d8472d4964
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL-charts.xls.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL-formats.xls.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL-formats.xls.zip
new file mode 100644
index 0000000000000..f1015248b63c6
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL-formats.xls.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL-formats.xlsx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL-formats.xlsx.zip
new file mode 100644
index 0000000000000..b62c8813e9679
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL-formats.xlsx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL.xls.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL.xls.zip
new file mode 100644
index 0000000000000..f0dc3b00f9685
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL.xls.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL.xlsx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL.xlsx.zip
new file mode 100644
index 0000000000000..45b3dae6f4476
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL.xlsx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL_1img.xls.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL_1img.xls.zip
new file mode 100644
index 0000000000000..09cb216169797
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL_1img.xls.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL_1img.xlsx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL_1img.xlsx.zip
new file mode 100644
index 0000000000000..81228661820bc
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL_1img.xlsx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL_4.xls.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL_4.xls.zip
new file mode 100644
index 0000000000000..46da553af950c
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL_4.xls.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL_5.xls.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL_5.xls.zip
new file mode 100644
index 0000000000000..ca3e08c4701f0
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL_5.xls.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL_95.xls.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL_95.xls.zip
new file mode 100644
index 0000000000000..00e3d373d118d
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL_95.xls.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL_custom_props.xls.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL_custom_props.xls.zip
new file mode 100644
index 0000000000000..2fa4d6b8b91b3
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL_custom_props.xls.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL_custom_props.xlsx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL_custom_props.xlsx.zip
new file mode 100644
index 0000000000000..9cfd16cec258d
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL_custom_props.xlsx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL_embeded.xls.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL_embeded.xls.zip
new file mode 100644
index 0000000000000..b7291cafc9877
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL_embeded.xls.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL_embeded.xlsx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL_embeded.xlsx.zip
new file mode 100644
index 0000000000000..755b7395e83c9
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL_embeded.xlsx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL_headers_footers.xls.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL_headers_footers.xls.zip
new file mode 100644
index 0000000000000..6cb18625ca7b7
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL_headers_footers.xls.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL_headers_footers.xlsx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL_headers_footers.xlsx.zip
new file mode 100644
index 0000000000000..924ad5eece245
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL_headers_footers.xlsx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL_textbox.xlsx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL_textbox.xlsx.zip
new file mode 100644
index 0000000000000..43046dff52614
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testEXCEL_textbox.xlsx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testExtraSpaces.pdf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testExtraSpaces.pdf.zip
new file mode 100644
index 0000000000000..8f6ed7a19f224
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testExtraSpaces.pdf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testFontAfterBufferedText.rtf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testFontAfterBufferedText.rtf.zip
new file mode 100644
index 0000000000000..77a5ac60dc4e0
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testFontAfterBufferedText.rtf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testFooter.ods.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testFooter.ods.zip
new file mode 100644
index 0000000000000..8659af1e691dc
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testFooter.ods.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testFooter.odt.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testFooter.odt.zip
new file mode 100644
index 0000000000000..85ba5501448ab
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testFooter.odt.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testHTML.html.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testHTML.html.zip
new file mode 100644
index 0000000000000..58cfd638254b9
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testHTML.html.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testHTMLNoisyMetaEncoding_1.html.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testHTMLNoisyMetaEncoding_1.html.zip
new file mode 100644
index 0000000000000..a12735a5205e0
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testHTMLNoisyMetaEncoding_1.html.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testHTMLNoisyMetaEncoding_2.html.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testHTMLNoisyMetaEncoding_2.html.zip
new file mode 100644
index 0000000000000..89a3f9d7aaadb
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testHTMLNoisyMetaEncoding_2.html.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testHTMLNoisyMetaEncoding_3.html.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testHTMLNoisyMetaEncoding_3.html.zip
new file mode 100644
index 0000000000000..da23b13ee4432
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testHTMLNoisyMetaEncoding_3.html.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testHTMLNoisyMetaEncoding_4.html.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testHTMLNoisyMetaEncoding_4.html.zip
new file mode 100644
index 0000000000000..15850474d98cd
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testHTMLNoisyMetaEncoding_4.html.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testHTML_utf8.html.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testHTML_utf8.html.zip
new file mode 100644
index 0000000000000..bdb5e58f373c0
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testHTML_utf8.html.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testKeynote.key.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testKeynote.key.zip
new file mode 100644
index 0000000000000..68693369d10cf
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testKeynote.key.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testMasterFooter.odp.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testMasterFooter.odp.zip
new file mode 100644
index 0000000000000..42467aa951bf3
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testMasterFooter.odp.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testMasterSlideTable.key.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testMasterSlideTable.key.zip
new file mode 100644
index 0000000000000..8d7803d396c8a
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testMasterSlideTable.key.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testNPEOpenDocument.odt.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testNPEOpenDocument.odt.zip
new file mode 100644
index 0000000000000..4c41bc7a466a2
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testNPEOpenDocument.odt.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testODFwithOOo3.odt.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testODFwithOOo3.odt.zip
new file mode 100644
index 0000000000000..9202fe44e446c
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testODFwithOOo3.odt.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testOpenOffice2.odt.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testOpenOffice2.odt.zip
new file mode 100644
index 0000000000000..1993365b5d1da
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testOpenOffice2.odt.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testOptionalHyphen.doc.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testOptionalHyphen.doc.zip
new file mode 100644
index 0000000000000..b9f515941e315
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testOptionalHyphen.doc.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testOptionalHyphen.docx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testOptionalHyphen.docx.zip
new file mode 100644
index 0000000000000..033871556d11a
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testOptionalHyphen.docx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testOptionalHyphen.pdf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testOptionalHyphen.pdf.zip
new file mode 100644
index 0000000000000..37fab9390588b
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testOptionalHyphen.pdf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testOptionalHyphen.ppt.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testOptionalHyphen.ppt.zip
new file mode 100644
index 0000000000000..e50261f432419
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testOptionalHyphen.ppt.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testOptionalHyphen.pptx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testOptionalHyphen.pptx.zip
new file mode 100644
index 0000000000000..19d6e78e6dfa7
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testOptionalHyphen.pptx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testOptionalHyphen.rtf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testOptionalHyphen.rtf.zip
new file mode 100644
index 0000000000000..d86048798c24e
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testOptionalHyphen.rtf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testOverlappingText.pdf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testOverlappingText.pdf.zip
new file mode 100644
index 0000000000000..c7f8b78704858
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testOverlappingText.pdf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF-custommetadata.pdf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF-custommetadata.pdf.zip
new file mode 100644
index 0000000000000..44865eca76c7d
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF-custommetadata.pdf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF.pdf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF.pdf.zip
new file mode 100644
index 0000000000000..fd5195b053867
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF.pdf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDFEmbeddingAndEmbedded.docx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDFEmbeddingAndEmbedded.docx.zip
new file mode 100644
index 0000000000000..a7ceed20665ad
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDFEmbeddingAndEmbedded.docx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDFFileEmbInAnnotation.pdf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDFFileEmbInAnnotation.pdf.zip
new file mode 100644
index 0000000000000..4b6e333895039
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDFFileEmbInAnnotation.pdf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDFPackage.pdf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDFPackage.pdf.zip
new file mode 100644
index 0000000000000..8bbb9166bb368
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDFPackage.pdf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDFTripleLangTitle.pdf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDFTripleLangTitle.pdf.zip
new file mode 100644
index 0000000000000..2d1ebdbdb2f2f
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDFTripleLangTitle.pdf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDFTwoTextBoxes.pdf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDFTwoTextBoxes.pdf.zip
new file mode 100644
index 0000000000000..d05907a3da109
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDFTwoTextBoxes.pdf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDFVarious.pdf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDFVarious.pdf.zip
new file mode 100644
index 0000000000000..cf86cbae79e7e
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDFVarious.pdf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_PDFEncodedStringInXMP.pdf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_PDFEncodedStringInXMP.pdf.zip
new file mode 100644
index 0000000000000..e8377846e79d7
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_PDFEncodedStringInXMP.pdf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_Version.10.x.pdf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_Version.10.x.pdf.zip
new file mode 100644
index 0000000000000..4fffed77ad888
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_Version.10.x.pdf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_Version.11.x.PDFA-1b.pdf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_Version.11.x.PDFA-1b.pdf.zip
new file mode 100644
index 0000000000000..16098515ce2d3
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_Version.11.x.PDFA-1b.pdf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_Version.4.x.pdf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_Version.4.x.pdf.zip
new file mode 100644
index 0000000000000..cdf93347572b9
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_Version.4.x.pdf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_Version.5.x.pdf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_Version.5.x.pdf.zip
new file mode 100644
index 0000000000000..e5ef413b8bed6
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_Version.5.x.pdf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_Version.6.x.pdf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_Version.6.x.pdf.zip
new file mode 100644
index 0000000000000..b57b4dc4efbf8
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_Version.6.x.pdf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_Version.7.x.pdf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_Version.7.x.pdf.zip
new file mode 100644
index 0000000000000..d69bbb50feede
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_Version.7.x.pdf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_Version.8.x.pdf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_Version.8.x.pdf.zip
new file mode 100644
index 0000000000000..e077af2074323
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_Version.8.x.pdf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_Version.9.x.pdf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_Version.9.x.pdf.zip
new file mode 100644
index 0000000000000..c9b6857c1526e
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_Version.9.x.pdf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_acroform3.pdf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_acroform3.pdf.zip
new file mode 100644
index 0000000000000..267d41df314a2
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_acroform3.pdf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_bom.pdf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_bom.pdf.zip
new file mode 100644
index 0000000000000..c4ea31f96f43d
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_bom.pdf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_bookmarks.pdf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_bookmarks.pdf.zip
new file mode 100644
index 0000000000000..88a5db5e63eff
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_bookmarks.pdf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_multiFormatEmbFiles.pdf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_multiFormatEmbFiles.pdf.zip
new file mode 100644
index 0000000000000..e61f2bf11124d
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_multiFormatEmbFiles.pdf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_no_extract_no_accessibility_owner_empty.pdf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_no_extract_no_accessibility_owner_empty.pdf.zip
new file mode 100644
index 0000000000000..16920b49e3a4a
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_no_extract_no_accessibility_owner_empty.pdf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_no_extract_yes_accessibility_owner_empty.pdf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_no_extract_yes_accessibility_owner_empty.pdf.zip
new file mode 100644
index 0000000000000..59887f7e5d20d
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_no_extract_yes_accessibility_owner_empty.pdf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_twoAuthors.pdf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_twoAuthors.pdf.zip
new file mode 100644
index 0000000000000..d240c51d8221c
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPDF_twoAuthors.pdf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT.ppsm.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT.ppsm.zip
new file mode 100644
index 0000000000000..47f652d14af7f
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT.ppsm.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT.ppsx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT.ppsx.zip
new file mode 100644
index 0000000000000..0f21f13be0f93
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT.ppsx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT.ppt.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT.ppt.zip
new file mode 100644
index 0000000000000..00c5c88c6ee3b
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT.ppt.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT.pptm.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT.pptm.zip
new file mode 100644
index 0000000000000..edd3e444fd78c
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT.pptm.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT.pptx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT.pptx.zip
new file mode 100644
index 0000000000000..68a9d662756d2
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT.pptx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPTX_Thumbnail.pptx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPTX_Thumbnail.pptx.zip
new file mode 100644
index 0000000000000..23dba29f32453
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPTX_Thumbnail.pptx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_2imgs.ppt.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_2imgs.ppt.zip
new file mode 100644
index 0000000000000..317e1bbe3521a
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_2imgs.ppt.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_2imgs.pptx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_2imgs.pptx.zip
new file mode 100644
index 0000000000000..62096338a496b
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_2imgs.pptx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_autodate.ppt.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_autodate.ppt.zip
new file mode 100644
index 0000000000000..5bd71fbd2e4af
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_autodate.ppt.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_autodate.pptx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_autodate.pptx.zip
new file mode 100644
index 0000000000000..811ab85cc26cc
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_autodate.pptx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_comment.ppt.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_comment.ppt.zip
new file mode 100644
index 0000000000000..3e94ba4cce1dd
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_comment.ppt.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_comment.pptx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_comment.pptx.zip
new file mode 100644
index 0000000000000..68d2135d4122b
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_comment.pptx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_custom_props.ppt.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_custom_props.ppt.zip
new file mode 100644
index 0000000000000..80673e625d78a
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_custom_props.ppt.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_custom_props.pptx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_custom_props.pptx.zip
new file mode 100644
index 0000000000000..d27c98d2c2d42
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_custom_props.pptx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_embedded2.ppt.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_embedded2.ppt.zip
new file mode 100644
index 0000000000000..017f9f500c741
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_embedded2.ppt.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_embedded_two_slides.pptx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_embedded_two_slides.pptx.zip
new file mode 100644
index 0000000000000..dfa459cccbdab
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_embedded_two_slides.pptx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_embeded.ppt.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_embeded.ppt.zip
new file mode 100644
index 0000000000000..fb8a3caf75501
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_embeded.ppt.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_embeded.pptx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_embeded.pptx.zip
new file mode 100644
index 0000000000000..f33f463c3356f
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_embeded.pptx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_masterFooter.ppt.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_masterFooter.ppt.zip
new file mode 100644
index 0000000000000..6a7278893942e
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_masterFooter.ppt.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_masterFooter.pptx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_masterFooter.pptx.zip
new file mode 100644
index 0000000000000..992dad41c5346
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_masterFooter.pptx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_masterText.ppt.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_masterText.ppt.zip
new file mode 100644
index 0000000000000..e31c4719f4761
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_masterText.ppt.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_masterText.pptx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_masterText.pptx.zip
new file mode 100644
index 0000000000000..1fa4f48748767
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_masterText.pptx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_masterText2.ppt.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_masterText2.ppt.zip
new file mode 100644
index 0000000000000..f58055ecfcbe3
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_masterText2.ppt.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_masterText2.pptx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_masterText2.pptx.zip
new file mode 100644
index 0000000000000..f8ff31052d129
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_masterText2.pptx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_various.ppt.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_various.ppt.zip
new file mode 100644
index 0000000000000..5d13efb8d4ad4
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_various.ppt.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_various.pptx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_various.pptx.zip
new file mode 100644
index 0000000000000..5d146a1de0bfc
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPPT_various.pptx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPageNumber.pdf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPageNumber.pdf.zip
new file mode 100644
index 0000000000000..4f9db82c2997e
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPageNumber.pdf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPages.pages.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPages.pages.zip
new file mode 100644
index 0000000000000..3db77c0873320
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPages.pages.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPagesComments.pages.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPagesComments.pages.zip
new file mode 100644
index 0000000000000..55004e812d22c
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPagesComments.pages.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPagesHeadersFootersAlphaLower.pages.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPagesHeadersFootersAlphaLower.pages.zip
new file mode 100644
index 0000000000000..bfea0c7825acb
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPagesHeadersFootersAlphaLower.pages.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPagesHeadersFootersAlphaUpper.pages.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPagesHeadersFootersAlphaUpper.pages.zip
new file mode 100644
index 0000000000000..3298d7eaa8fe3
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPagesHeadersFootersAlphaUpper.pages.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPagesHeadersFootersFootnotes.pages.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPagesHeadersFootersFootnotes.pages.zip
new file mode 100644
index 0000000000000..34a4fb83230fe
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPagesHeadersFootersFootnotes.pages.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPagesHeadersFootersRomanLower.pages.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPagesHeadersFootersRomanLower.pages.zip
new file mode 100644
index 0000000000000..b48ebe4458d0a
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPagesHeadersFootersRomanLower.pages.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPagesHeadersFootersRomanUpper.pages.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPagesHeadersFootersRomanUpper.pages.zip
new file mode 100644
index 0000000000000..967e139595f08
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPagesHeadersFootersRomanUpper.pages.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPagesLayout.pages.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPagesLayout.pages.zip
new file mode 100644
index 0000000000000..c6f0e04a53653
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPagesLayout.pages.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPopupAnnotation.pdf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPopupAnnotation.pdf.zip
new file mode 100644
index 0000000000000..4eb78653562d4
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testPopupAnnotation.pdf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTF-ms932.rtf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTF-ms932.rtf.zip
new file mode 100644
index 0000000000000..bd80f2b061aba
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTF-ms932.rtf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTF.rtf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTF.rtf.zip
new file mode 100644
index 0000000000000..72ef8ebfc56fe
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTF.rtf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFBoldItalic.rtf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFBoldItalic.rtf.zip
new file mode 100644
index 0000000000000..935c249176b6b
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFBoldItalic.rtf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFControls.rtf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFControls.rtf.zip
new file mode 100644
index 0000000000000..0c55479cc89c0
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFControls.rtf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFCorruptListOverride.rtf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFCorruptListOverride.rtf.zip
new file mode 100644
index 0000000000000..e6ea82d476001
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFCorruptListOverride.rtf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFEmbeddedLink.rtf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFEmbeddedLink.rtf.zip
new file mode 100644
index 0000000000000..857c589d9eb84
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFEmbeddedLink.rtf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFHexEscapeInsideWord.rtf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFHexEscapeInsideWord.rtf.zip
new file mode 100644
index 0000000000000..f10b768d5a2d0
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFHexEscapeInsideWord.rtf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFHyperlink.rtf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFHyperlink.rtf.zip
new file mode 100644
index 0000000000000..658eb87784f23
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFHyperlink.rtf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFIgnoredControlWord.rtf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFIgnoredControlWord.rtf.zip
new file mode 100644
index 0000000000000..15a74d726b6f5
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFIgnoredControlWord.rtf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFInvalidUnicode.rtf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFInvalidUnicode.rtf.zip
new file mode 100644
index 0000000000000..f24a5ab9c0869
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFInvalidUnicode.rtf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFJapanese.rtf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFJapanese.rtf.zip
new file mode 100644
index 0000000000000..8bf4f6e7165b3
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFJapanese.rtf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFListLibreOffice.rtf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFListLibreOffice.rtf.zip
new file mode 100644
index 0000000000000..e2a406d99a652
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFListLibreOffice.rtf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFListMicrosoftWord.rtf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFListMicrosoftWord.rtf.zip
new file mode 100644
index 0000000000000..155718ac8b381
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFListMicrosoftWord.rtf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFListOverride.rtf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFListOverride.rtf.zip
new file mode 100644
index 0000000000000..c9b7f3fdbee65
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFListOverride.rtf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFNewlines.rtf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFNewlines.rtf.zip
new file mode 100644
index 0000000000000..8b0dffe13512e
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFNewlines.rtf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFRegularImages.rtf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFRegularImages.rtf.zip
new file mode 100644
index 0000000000000..2bf8d9b530183
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFRegularImages.rtf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFTableCellSeparation.rtf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFTableCellSeparation.rtf.zip
new file mode 100644
index 0000000000000..5e110de516907
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFTableCellSeparation.rtf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFTableCellSeparation2.rtf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFTableCellSeparation2.rtf.zip
new file mode 100644
index 0000000000000..599e7f22edb5b
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFTableCellSeparation2.rtf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFUmlautSpaces.rtf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFUmlautSpaces.rtf.zip
new file mode 100644
index 0000000000000..f3cbec9e51d57
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFUmlautSpaces.rtf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFUmlautSpaces2.rtf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFUmlautSpaces2.rtf.zip
new file mode 100644
index 0000000000000..2c905405ca48b
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFUmlautSpaces2.rtf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFUnicodeGothic.rtf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFUnicodeGothic.rtf.zip
new file mode 100644
index 0000000000000..40263351a04ba
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFUnicodeGothic.rtf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFUnicodeUCNControlWordCharacterDoubling.rtf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFUnicodeUCNControlWordCharacterDoubling.rtf.zip
new file mode 100644
index 0000000000000..c8f40b549c3b0
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFUnicodeUCNControlWordCharacterDoubling.rtf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFVarious.rtf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFVarious.rtf.zip
new file mode 100644
index 0000000000000..60281e7f9ae11
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFVarious.rtf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFWindowsCodepage1250.rtf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFWindowsCodepage1250.rtf.zip
new file mode 100644
index 0000000000000..4a7852c97d3d8
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFWindowsCodepage1250.rtf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFWithCurlyBraces.rtf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFWithCurlyBraces.rtf.zip
new file mode 100644
index 0000000000000..48268a457c090
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFWithCurlyBraces.rtf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFWord2010CzechCharacters.rtf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFWord2010CzechCharacters.rtf.zip
new file mode 100644
index 0000000000000..be845e4f925e0
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFWord2010CzechCharacters.rtf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFWordPadCzechCharacters.rtf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFWordPadCzechCharacters.rtf.zip
new file mode 100644
index 0000000000000..508801675de7a
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testRTFWordPadCzechCharacters.rtf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testStyles.odt.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testStyles.odt.zip
new file mode 100644
index 0000000000000..22f6c76b9a802
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testStyles.odt.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testTXTNonASCIIUTF8.txt.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testTXTNonASCIIUTF8.txt.zip
new file mode 100644
index 0000000000000..7326a28ad7695
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testTXTNonASCIIUTF8.txt.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testTables.key.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testTables.key.zip
new file mode 100644
index 0000000000000..3c963dcc0770e
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testTables.key.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testTextBoxes.key.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testTextBoxes.key.zip
new file mode 100644
index 0000000000000..aea9b1cf66c5f
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testTextBoxes.key.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD.doc.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD.doc.zip
new file mode 100644
index 0000000000000..38b2954a4ff29
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD.doc.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD.docx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD.docx.zip
new file mode 100644
index 0000000000000..2ed0aae33f439
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD.docx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD6.doc.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD6.doc.zip
new file mode 100644
index 0000000000000..b8dbbd8322e96
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD6.doc.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_1img.doc.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_1img.doc.zip
new file mode 100644
index 0000000000000..1baa330e5ce8e
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_1img.doc.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_1img.docx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_1img.docx.zip
new file mode 100644
index 0000000000000..8920521bd2d38
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_1img.docx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_3imgs.doc.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_3imgs.doc.zip
new file mode 100644
index 0000000000000..b60716b93d2c0
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_3imgs.doc.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_3imgs.docx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_3imgs.docx.zip
new file mode 100644
index 0000000000000..062713d931818
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_3imgs.docx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_bold_character_runs.doc.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_bold_character_runs.doc.zip
new file mode 100644
index 0000000000000..b0a6af23d34da
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_bold_character_runs.doc.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_bold_character_runs.docx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_bold_character_runs.docx.zip
new file mode 100644
index 0000000000000..546f98bf093c2
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_bold_character_runs.docx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_bold_character_runs2.doc.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_bold_character_runs2.doc.zip
new file mode 100644
index 0000000000000..5dbedec5e8630
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_bold_character_runs2.doc.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_bold_character_runs2.docx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_bold_character_runs2.docx.zip
new file mode 100644
index 0000000000000..8edde24f9937a
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_bold_character_runs2.docx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_closingSmartQInHyperLink.doc.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_closingSmartQInHyperLink.doc.zip
new file mode 100644
index 0000000000000..4484a6b005241
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_closingSmartQInHyperLink.doc.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_custom_props.doc.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_custom_props.doc.zip
new file mode 100644
index 0000000000000..72a94832d3e8c
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_custom_props.doc.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_custom_props.docx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_custom_props.docx.zip
new file mode 100644
index 0000000000000..30aaef7d81794
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_custom_props.docx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_embedded_pdf.docx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_embedded_pdf.docx.zip
new file mode 100644
index 0000000000000..2b74f67dc9f67
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_embedded_pdf.docx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_embedded_rtf.doc.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_embedded_rtf.doc.zip
new file mode 100644
index 0000000000000..e456721065c21
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_embedded_rtf.doc.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_embeded.doc.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_embeded.doc.zip
new file mode 100644
index 0000000000000..699a13b652ab2
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_embeded.doc.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_embeded.docx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_embeded.docx.zip
new file mode 100644
index 0000000000000..b53088ddb716c
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_embeded.docx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_header_hyperlink.doc.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_header_hyperlink.doc.zip
new file mode 100644
index 0000000000000..867dc1808f647
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_header_hyperlink.doc.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_missing_ooxml_bean1.docx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_missing_ooxml_bean1.docx.zip
new file mode 100644
index 0000000000000..04172a65382f7
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_missing_ooxml_bean1.docx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_missing_text.docx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_missing_text.docx.zip
new file mode 100644
index 0000000000000..325ecce927917
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_missing_text.docx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_no_format.doc.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_no_format.doc.zip
new file mode 100644
index 0000000000000..38eecb941d4bc
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_no_format.doc.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_no_format.docx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_no_format.docx.zip
new file mode 100644
index 0000000000000..1c19cf3e98224
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_no_format.docx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_null_style.docx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_null_style.docx.zip
new file mode 100644
index 0000000000000..605eeb250232f
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_null_style.docx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_numbered_list.doc.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_numbered_list.doc.zip
new file mode 100644
index 0000000000000..7f8d3b9ad2014
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_numbered_list.doc.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_numbered_list.docx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_numbered_list.docx.zip
new file mode 100644
index 0000000000000..42b4ed17c4d0e
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_numbered_list.docx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_override_list_numbering.doc.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_override_list_numbering.doc.zip
new file mode 100644
index 0000000000000..39c954b2d19bd
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_override_list_numbering.doc.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_override_list_numbering.docx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_override_list_numbering.docx.zip
new file mode 100644
index 0000000000000..77bb9dbda68d2
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_override_list_numbering.docx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_tabular_symbol.doc.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_tabular_symbol.doc.zip
new file mode 100644
index 0000000000000..0c63f9825677c
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_tabular_symbol.doc.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_text_box.docx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_text_box.docx.zip
new file mode 100644
index 0000000000000..504c56728c97a
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_text_box.docx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_various.doc.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_various.doc.zip
new file mode 100644
index 0000000000000..9d7c24e052068
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_various.doc.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_various.docx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_various.docx.zip
new file mode 100644
index 0000000000000..393aca11ab7df
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWORD_various.docx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWordArt.pptx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWordArt.pptx.zip
new file mode 100644
index 0000000000000..75234fde98d72
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testWordArt.pptx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testXHTML.html.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testXHTML.html.zip
new file mode 100644
index 0000000000000..45c4dd0479adb
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testXHTML.html.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testXLSX_Thumbnail.xlsx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testXLSX_Thumbnail.xlsx.zip
new file mode 100644
index 0000000000000..919cf31e75b4e
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testXLSX_Thumbnail.xlsx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testXML.xml.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testXML.xml.zip
new file mode 100644
index 0000000000000..a6520a8a811fe
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testXML.xml.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testXML2.xml.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testXML2.xml.zip
new file mode 100644
index 0000000000000..5b89863685a13
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testXML2.xml.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testXML3.xml.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testXML3.xml.zip
new file mode 100644
index 0000000000000..08a066d9c1e36
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/testXML3.xml.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/test_TIKA-1251.doc.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/test_TIKA-1251.doc.zip
new file mode 100644
index 0000000000000..4d3071290216b
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/test_TIKA-1251.doc.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/test_embedded_package.rtf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/test_embedded_package.rtf.zip
new file mode 100644
index 0000000000000..58d7f15b23b3b
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/test_embedded_package.rtf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/test_embedded_zip.pptx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/test_embedded_zip.pptx.zip
new file mode 100644
index 0000000000000..e9d802cfd4e3e
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/test_embedded_zip.pptx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/test_list_override.rtf.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/test_list_override.rtf.zip
new file mode 100644
index 0000000000000..57d5553829448
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/test_list_override.rtf.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/test_recursive_embedded.docx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/test_recursive_embedded.docx.zip
new file mode 100644
index 0000000000000..edd97967e95e4
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/test_recursive_embedded.docx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/test_recursive_embedded_npe.docx.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/test_recursive_embedded_npe.docx.zip
new file mode 100644
index 0000000000000..84da48f0250f2
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/test_recursive_embedded_npe.docx.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/tika434.html.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/tika434.html.zip
new file mode 100644
index 0000000000000..41972bc8447c8
Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files/tika434.html.zip differ
diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/wrong-mapping/wrong-mapping.json b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/wrong-mapping/wrong-mapping.json
new file mode 100644
index 0000000000000..eecab63753d62
--- /dev/null
+++ b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/wrong-mapping/wrong-mapping.json
@@ -0,0 +1,14 @@
+{
+    "person": {
+        "properties": {
+            "file": {
+                "type": "attachment",
+                "fields": {
+                    "content": {
+                        "type": "nonexistingtype"
+                    }
+                }
+            }
+        }
+    }
+}
diff --git a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java
index 418d81fcf1ebf..1f5055f1429a7 100644
--- a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java
+++ b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java
@@ -23,10 +23,10 @@
 import java.util.List;
 import java.util.Map;
 
-import org.apache.lucene.document.Field;
 import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.document.StoredField;
 import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.Version;
@@ -144,7 +144,7 @@ protected String contentType() {
     }
 
     @Override
-    protected void parseCreateField(ParseContext context, List<Field> fields)
+    protected void parseCreateField(ParseContext context, List<IndexableField> fields)
             throws IOException {
         final Object value;
         if (context.externalValueSet()) {
diff --git a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/MapperMurmur3ClientYamlTestSuiteIT.java b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/MapperMurmur3ClientYamlTestSuiteIT.java
index 204f6c07a99a9..5ef64cd502708 100644
--- a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/MapperMurmur3ClientYamlTestSuiteIT.java
+++ b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/MapperMurmur3ClientYamlTestSuiteIT.java
@@ -24,9 +24,6 @@
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-
-import java.io.IOException;
 
 public class MapperMurmur3ClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
@@ -35,8 +32,8 @@ public MapperMurmur3ClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
-        return createParameters(0, 1);
+    public static Iterable<Object[]> parameters() throws Exception {
+        return createParameters();
     }
 }
 
diff --git a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java
index d1a6c0463fb08..325ac72671391 100644
--- a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java
+++ b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java
@@ -32,6 +32,7 @@
 import org.elasticsearch.index.mapper.DocumentMapperParser;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.ParsedDocument;
+import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.indices.mapper.MapperRegistry;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
@@ -42,6 +43,7 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.function.Supplier;
 
 import static com.carrotsearch.randomizedtesting.RandomizedTest.getRandom;
 import static org.hamcrest.Matchers.containsString;
@@ -53,13 +55,16 @@ public class Murmur3FieldMapperTests extends ESSingleNodeTestCase {
     DocumentMapperParser parser;
 
     @Before
-    public void before() {
+    public void setup() {
         indexService = createIndex("test");
         mapperRegistry = new MapperRegistry(
                 Collections.singletonMap(Murmur3FieldMapper.CONTENT_TYPE, new Murmur3FieldMapper.TypeParser()),
                 Collections.emptyMap());
-        parser = new DocumentMapperParser(indexService.getIndexSettings(), indexService.mapperService(),
-        indexService.analysisService(), indexService.similarityService(), mapperRegistry, indexService::newQueryShardContext);
+        Supplier<QueryShardContext> queryShardContext = () -> {
+            return indexService.newQueryShardContext(0, null, () -> { throw new UnsupportedOperationException(); });
+        };
+        parser = new DocumentMapperParser(indexService.getIndexSettings(), indexService.mapperService(), indexService.getIndexAnalyzers(),
+                indexService.xContentRegistry(), indexService.similarityService(), mapperRegistry, queryShardContext);
     }
 
     @Override
@@ -152,8 +157,12 @@ public void testEmptyName() throws Exception {
         Settings oldIndexSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, oldVersion).build();
         IndexService indexService2x = createIndex("test_old", oldIndexSettings);
 
-        DocumentMapperParser parser = new DocumentMapperParser(indexService2x.getIndexSettings(), indexService2x.mapperService(), indexService2x.analysisService(),
-            indexService2x.similarityService(), mapperRegistry, indexService2x::newQueryShardContext);
+        Supplier<QueryShardContext> queryShardContext = () -> {
+            return indexService2x.newQueryShardContext(0, null, () -> { throw new UnsupportedOperationException(); });
+        };
+        DocumentMapperParser parser = new DocumentMapperParser(indexService2x.getIndexSettings(), indexService2x.mapperService(),
+                indexService2x.getIndexAnalyzers(), indexService2x.xContentRegistry(), indexService2x.similarityService(), mapperRegistry,
+                queryShardContext);
 
         DocumentMapper defaultMapper = parser.parse("type", new CompressedXContent(mapping));
         assertEquals(mapping, defaultMapper.mappingSource().string());
diff --git a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperUpgradeTests.java b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperUpgradeTests.java
index c632e139955e0..9552866feeb3b 100644
--- a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperUpgradeTests.java
+++ b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperUpgradeTests.java
@@ -30,7 +30,6 @@
 import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.search.aggregations.metrics.cardinality.Cardinality;
 import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.InternalTestCluster;
 import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
 
 import java.io.IOException;
@@ -53,7 +52,7 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
     public void testUpgradeOldMapping() throws IOException, ExecutionException, InterruptedException {
         final String indexName = "index-mapper-murmur3-2.0.0";
         final String indexUUID = "1VzJds59TTK7lRu17W0mcg";
-        InternalTestCluster.Async<String> master = internalCluster().startNodeAsync();
+        internalCluster().startNode();
         Path unzipDir = createTempDir();
         Path unzipDataDir = unzipDir.resolve("data");
         Path backwardsIndex = getBwcIndicesPath().resolve(indexName + ".zip");
@@ -75,7 +74,6 @@ public void testUpgradeOldMapping() throws IOException, ExecutionException, Inte
         Files.move(src, dataPath);
         Files.move(dataPath.resolve(indexName), dataPath.resolve(indexUUID));
 
-        master.get();
         // force reloading dangling indices with a cluster state republish
         client().admin().cluster().prepareReroute().get();
         ensureGreen(indexName);
diff --git a/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java b/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java
index a9fb9049b04ad..1ce0090be769d 100644
--- a/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java
+++ b/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java
@@ -19,8 +19,8 @@
 
 package org.elasticsearch.index.mapper.size;
 
-import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexableField;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Settings;
@@ -33,14 +33,13 @@
 import org.elasticsearch.index.mapper.MetadataFieldMapper;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
 import org.elasticsearch.index.mapper.ParseContext;
+import org.elasticsearch.index.mapper.TypeParsers;
 
 import java.io.IOException;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
-import static org.elasticsearch.common.xcontent.support.XContentMapValues.lenientNodeBooleanValue;
-
 public class SizeFieldMapper extends MetadataFieldMapper {
     public static final String NAME = "_size";
 
@@ -123,8 +122,8 @@ public static class TypeParser implements MetadataFieldMapper.TypeParser {
                 String fieldName = entry.getKey();
                 Object fieldNode = entry.getValue();
                 if (fieldName.equals("enabled")) {
-                    builder.enabled(lenientNodeBooleanValue(fieldNode) ?
-                        EnabledAttributeMapper.ENABLED : EnabledAttributeMapper.DISABLED);
+                    boolean enabled = TypeParsers.nodeBooleanValue(name, "enabled", fieldNode);
+                    builder.enabled(enabled ? EnabledAttributeMapper.ENABLED : EnabledAttributeMapper.DISABLED);
                     iterator.remove();
                 }
             }
@@ -132,7 +131,8 @@ public static class TypeParser implements MetadataFieldMapper.TypeParser {
         }
 
         @Override
-        public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName) {
+        public MetadataFieldMapper getDefault(MappedFieldType fieldType, ParserContext context) {
+            final Settings indexSettings = context.mapperService().getIndexSettings().getSettings();
             return new SizeFieldMapper(indexSettings, fieldType);
         }
     }
@@ -176,7 +176,7 @@ public Mapper parse(ParseContext context) throws IOException {
     }
 
     @Override
-    protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
+    protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
         if (!enabledState.enabled) {
             return;
         }
diff --git a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/MapperSizeClientYamlTestSuiteIT.java b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/MapperSizeClientYamlTestSuiteIT.java
index 44f26c4ec51df..0d4cb4372b2e6 100644
--- a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/MapperSizeClientYamlTestSuiteIT.java
+++ b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/MapperSizeClientYamlTestSuiteIT.java
@@ -24,9 +24,6 @@
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-
-import java.io.IOException;
 
 public class MapperSizeClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
@@ -35,8 +32,8 @@ public MapperSizeClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate tes
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
-        return createParameters(0, 1);
+    public static Iterable<Object[]> parameters() throws Exception {
+        return createParameters();
     }
 }
 
diff --git a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeFieldMapperUpgradeTests.java b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeFieldMapperUpgradeTests.java
index 7cbce102c57b2..563258a9e43a2 100644
--- a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeFieldMapperUpgradeTests.java
+++ b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeFieldMapperUpgradeTests.java
@@ -30,7 +30,6 @@
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHitField;
 import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.InternalTestCluster;
 import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
 
 import java.io.IOException;
@@ -54,7 +53,7 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
     public void testUpgradeOldMapping() throws IOException, ExecutionException, InterruptedException {
         final String indexName = "index-mapper-size-2.0.0";
         final String indexUUID = "ENCw7sG0SWuTPcH60bHheg";
-        InternalTestCluster.Async<String> master = internalCluster().startNodeAsync();
+        internalCluster().startNode();
         Path unzipDir = createTempDir();
         Path unzipDataDir = unzipDir.resolve("data");
         Path backwardsIndex = getBwcIndicesPath().resolve(indexName + ".zip");
@@ -76,7 +75,6 @@ public void testUpgradeOldMapping() throws IOException, ExecutionException, Inte
         Path src = unzipDataDir.resolve(indexName + "/nodes/0/indices");
         Files.move(src, dataPath);
         Files.move(dataPath.resolve(indexName), dataPath.resolve(indexUUID));
-        master.get();
         // force reloading dangling indices with a cluster state republish
         client().admin().cluster().prepareReroute().get();
         ensureGreen(indexName);
diff --git a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java
index 9e3799a5d5ae1..92cbb2e6e397b 100644
--- a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java
+++ b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.plugin.mapper.MapperSizePlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -107,8 +108,8 @@ public void testBasic() throws Exception {
         assertAcked(prepareCreate("test").addMapping("type", "_size", "enabled=true"));
         final String source = "{\"f\":10}";
         indexRandom(true,
-                client().prepareIndex("test", "type", "1").setSource(source));
-        GetResponse getResponse = client().prepareGet("test", "type", "1").setFields("_size").get();
+                client().prepareIndex("test", "type", "1").setSource(source, XContentType.JSON));
+        GetResponse getResponse = client().prepareGet("test", "type", "1").setStoredFields("_size").get();
         assertNotNull(getResponse.getField("_size"));
         assertEquals(source.length(), getResponse.getField("_size").getValue());
     }
diff --git a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java
index 5d832aa1518dd..334a93f2c05eb 100644
--- a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java
+++ b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
@@ -61,7 +62,7 @@ public void testSizeEnabled() throws Exception {
             .field("field", "value")
             .endObject()
             .bytes();
-        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", source));
+        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", source, XContentType.JSON));
 
         boolean stored = false;
         boolean points = false;
@@ -82,7 +83,7 @@ public void testSizeDisabled() throws Exception {
             .field("field", "value")
             .endObject()
             .bytes();
-        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", source));
+        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", source, XContentType.JSON));
 
         assertThat(doc.rootDoc().getField("_size"), nullValue());
     }
@@ -96,7 +97,7 @@ public void testSizeNotSet() throws Exception {
             .field("field", "value")
             .endObject()
             .bytes();
-        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", source));
+        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", source, XContentType.JSON));
 
         assertThat(doc.rootDoc().getField("_size"), nullValue());
     }
diff --git a/plugins/mapper-size/src/test/resources/rest-api-spec/test/mapper_size/10_basic.yaml b/plugins/mapper-size/src/test/resources/rest-api-spec/test/mapper_size/10_basic.yaml
index 9e76645e26ebd..be08c46034ad5 100644
--- a/plugins/mapper-size/src/test/resources/rest-api-spec/test/mapper_size/10_basic.yaml
+++ b/plugins/mapper-size/src/test/resources/rest-api-spec/test/mapper_size/10_basic.yaml
@@ -26,7 +26,7 @@
             index: test
             type: type1
             id: 1
-            fields: "_size"
+            stored_fields: "_size"
 
-    - match: { _size: 13 }
+    - gt: { _size: 0 }
 
diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle
index 125b7bdd6c5c3..14a0c37a20028 100644
--- a/plugins/repository-azure/build.gradle
+++ b/plugins/repository-azure/build.gradle
@@ -23,8 +23,9 @@ esplugin {
 }
 
 dependencies {
-  compile 'com.microsoft.azure:azure-storage:4.0.0'
-  compile 'org.apache.commons:commons-lang3:3.3.2'
+  compile 'com.microsoft.azure:azure-storage:5.0.0'
+  compile 'com.microsoft.azure:azure-keyvault-core:0.8.0'
+  compile 'org.apache.commons:commons-lang3:3.4'
 }
 
 dependencyLicenses {
@@ -41,10 +42,8 @@ thirdPartyAudit.excludes = [
   'org.slf4j.LoggerFactory',
 ]
 
-integTest {
-  cluster {
-    setting 'cloud.azure.storage.my_account_test.account', 'cloudazureresource'
-    setting 'cloud.azure.storage.my_account_test.key', 'abcdefgh'
-    setting 'script.stored', 'true'
-  }
-}
\ No newline at end of file
+integTestCluster {
+  setting 'cloud.azure.storage.my_account_test.account', 'cloudazureresource'
+  setting 'cloud.azure.storage.my_account_test.key', 'abcdefgh'
+  setting 'script.stored', 'true'
+}
diff --git a/plugins/repository-azure/licenses/azure-keyvault-core-0.8.0.jar.sha1 b/plugins/repository-azure/licenses/azure-keyvault-core-0.8.0.jar.sha1
new file mode 100644
index 0000000000000..b86c58db8423a
--- /dev/null
+++ b/plugins/repository-azure/licenses/azure-keyvault-core-0.8.0.jar.sha1
@@ -0,0 +1 @@
+35f7ac687462f491d0f8b0d96733dfe347493d70
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/azure-storage-4.0.0.jar.sha1 b/plugins/repository-azure/licenses/azure-storage-4.0.0.jar.sha1
deleted file mode 100644
index 9ef89531d6d81..0000000000000
--- a/plugins/repository-azure/licenses/azure-storage-4.0.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b31504f0fb3f9c4458ad053b426357a9b0df6e08
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/azure-storage-5.0.0.jar.sha1 b/plugins/repository-azure/licenses/azure-storage-5.0.0.jar.sha1
new file mode 100644
index 0000000000000..9882cb80204ff
--- /dev/null
+++ b/plugins/repository-azure/licenses/azure-storage-5.0.0.jar.sha1
@@ -0,0 +1 @@
+ba8f04bfeac08016c0f88423a202d0f3aac03aed
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/commons-lang3-3.3.2.jar.sha1 b/plugins/repository-azure/licenses/commons-lang3-3.3.2.jar.sha1
deleted file mode 100644
index bdd913cf2358d..0000000000000
--- a/plugins/repository-azure/licenses/commons-lang3-3.3.2.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-90a3822c38ec8c996e84c16a3477ef632cbc87a3
diff --git a/plugins/repository-azure/licenses/commons-lang3-3.4.jar.sha1 b/plugins/repository-azure/licenses/commons-lang3-3.4.jar.sha1
new file mode 100644
index 0000000000000..fdd7040377b8f
--- /dev/null
+++ b/plugins/repository-azure/licenses/commons-lang3-3.4.jar.sha1
@@ -0,0 +1 @@
+5fe28b9518e58819180a43a850fbc0dd24b7c050
\ No newline at end of file
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java
index 05bb911476a9d..3c97d490c1d64 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java
@@ -19,19 +19,17 @@
 
 package org.elasticsearch.cloud.azure.blobstore;
 
+import com.microsoft.azure.storage.LocationMode;
 import com.microsoft.azure.storage.StorageException;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.blobstore.BlobMetaData;
 import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.support.AbstractBlobContainer;
-import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.logging.Loggers;
-import org.elasticsearch.repositories.RepositoryException;
 
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.OutputStream;
 import java.net.HttpURLConnection;
 import java.net.URISyntaxException;
 import java.nio.file.FileAlreadyExistsException;
@@ -71,6 +69,16 @@ public boolean blobExists(String blobName) {
     public InputStream readBlob(String blobName) throws IOException {
         logger.trace("readBlob({})", blobName);
 
+        if (blobStore.getLocationMode() == LocationMode.SECONDARY_ONLY && !blobExists(blobName)) {
+            // On Azure, if the location path is a secondary location, and the blob does not
+            // exist, instead of returning immediately from the getInputStream call below
+            // with a 404 StorageException, Azure keeps trying and trying for a long timeout
+            // before throwing a storage exception.  This can cause long delays in retrieving
+            // snapshots, so we first check if the blob exists before trying to open an input
+            // stream to it.
+            throw new NoSuchFileException("Blob [" + blobName + "] does not exist");
+        }
+
         try {
             return blobStore.getInputStream(blobStore.container(), buildKey(blobName));
         } catch (StorageException e) {
@@ -88,24 +96,11 @@ public void writeBlob(String blobName, InputStream inputStream, long blobSize) t
         if (blobExists(blobName)) {
             throw new FileAlreadyExistsException("blob [" + blobName + "] already exists, cannot overwrite");
         }
-        logger.trace("writeBlob({}, stream, {})", blobName, blobSize);
-        try (OutputStream stream = createOutput(blobName)) {
-            Streams.copy(inputStream, stream);
-        }
-    }
-
-    private OutputStream createOutput(String blobName) throws IOException {
+        logger.trace("writeBlob({}, stream, {})", buildKey(blobName), blobSize);
         try {
-            return new AzureOutputStream(blobStore.getOutputStream(blobStore.container(), buildKey(blobName)));
-        } catch (StorageException e) {
-            if (e.getHttpStatusCode() == HttpURLConnection.HTTP_NOT_FOUND) {
-                throw new NoSuchFileException(e.getMessage());
-            }
-            throw new IOException(e);
-        } catch (URISyntaxException e) {
-            throw new IOException(e);
-        } catch (IllegalArgumentException e) {
-            throw new RepositoryException(repositoryName, e.getMessage());
+            blobStore.writeBlob(buildKey(blobName), inputStream, blobSize);
+        } catch (URISyntaxException|StorageException e) {
+            throw new IOException("Can not write blob " + blobName, e);
         }
     }
 
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java
index b77cebfa2f26c..9efd3830eea45 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java
@@ -22,7 +22,6 @@
 import com.microsoft.azure.storage.LocationMode;
 import com.microsoft.azure.storage.StorageException;
 import org.elasticsearch.cloud.azure.storage.AzureStorageService;
-import org.elasticsearch.cloud.azure.storage.AzureStorageService.Storage;
 import org.elasticsearch.cluster.metadata.RepositoryMetaData;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.blobstore.BlobContainer;
@@ -34,11 +33,11 @@
 
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.OutputStream;
 import java.net.URISyntaxException;
 import java.util.Locale;
 import java.util.Map;
 
+import static org.elasticsearch.cloud.azure.storage.AzureStorageService.Storage;
 import static org.elasticsearch.cloud.azure.storage.AzureStorageSettings.getValue;
 import static org.elasticsearch.repositories.azure.AzureRepository.Repository;
 
@@ -76,6 +75,13 @@ public String container() {
         return container;
     }
 
+    /**
+     * Gets the configured {@link LocationMode} for the Azure storage requests.
+     */
+    public LocationMode getLocationMode() {
+        return locMode;
+    }
+
     @Override
     public BlobContainer blobContainer(BlobPath path) {
         return new AzureBlobContainer(repositoryName, path, this);
@@ -130,11 +136,6 @@ public InputStream getInputStream(String container, String blob) throws URISynta
         return this.client.getInputStream(this.accountName, this.locMode, container, blob);
     }
 
-    public OutputStream getOutputStream(String container, String blob) throws URISyntaxException, StorageException
-    {
-        return this.client.getOutputStream(this.accountName, this.locMode, container, blob);
-    }
-
     public Map<String,BlobMetaData> listBlobsByPrefix(String container, String keyPath, String prefix) throws URISyntaxException, StorageException
     {
         return this.client.listBlobsByPrefix(this.accountName, this.locMode, container, keyPath, prefix);
@@ -144,4 +145,9 @@ public void moveBlob(String container, String sourceBlob, String targetBlob) thr
     {
         this.client.moveBlob(this.accountName, this.locMode, container, sourceBlob, targetBlob);
     }
+
+    public void writeBlob(String blobName, InputStream inputStream, long blobSize)
+        throws URISyntaxException, StorageException, IOException {
+        this.client.writeBlob(this.accountName, this.locMode, container, blobName, inputStream, blobSize);
+    }
 }
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureOutputStream.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureOutputStream.java
deleted file mode 100644
index 6a95eeba7789c..0000000000000
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureOutputStream.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.cloud.azure.blobstore;
-
-import java.io.IOException;
-import java.io.OutputStream;
-
-public class AzureOutputStream extends OutputStream {
-
-    private final OutputStream blobOutputStream;
-
-    public AzureOutputStream(OutputStream blobOutputStream) {
-        this.blobOutputStream = blobOutputStream;
-    }
-
-    @Override
-    public void write(int b) throws IOException {
-        blobOutputStream.write(b);
-    }
-
-    @Override
-    public void close() throws IOException {
-        try {
-            blobOutputStream.close();
-        } catch (IOException e) {
-            // Azure is sending a "java.io.IOException: Stream is already closed."
-        }
-    }
-}
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java
index 3e854ab9c709e..675f52fdb1d00 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java
@@ -26,12 +26,12 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.OutputStream;
 import java.net.URISyntaxException;
 import java.util.Map;
 import java.util.function.Function;
@@ -42,6 +42,9 @@
  */
 public interface AzureStorageService {
 
+    ByteSizeValue MIN_CHUNK_SIZE = new ByteSizeValue(1, ByteSizeUnit.BYTES);
+    ByteSizeValue MAX_CHUNK_SIZE = new ByteSizeValue(64, ByteSizeUnit.MB);
+
     final class Storage {
         public static final String PREFIX = "cloud.azure.storage.";
 
@@ -49,18 +52,26 @@ final class Storage {
 
         public static final Setting<TimeValue> TIMEOUT_SETTING =
             Setting.timeSetting("cloud.azure.storage.timeout", TimeValue.timeValueMinutes(-1), Property.NodeScope);
+        @Deprecated
         public static final Setting<String> ACCOUNT_SETTING =
-            Setting.simpleString("repositories.azure.account", Property.NodeScope, Property.Filtered);
+            Setting.simpleString("repositories.azure.account", Property.NodeScope, Property.Filtered, Property.Deprecated);
+        @Deprecated
         public static final Setting<String> CONTAINER_SETTING =
-            new Setting<>("repositories.azure.container", "elasticsearch-snapshots", Function.identity(), Property.NodeScope);
+            new Setting<>("repositories.azure.container", "elasticsearch-snapshots", Function.identity(), Property.NodeScope,
+                Property.Deprecated);
+        @Deprecated
         public static final Setting<String> BASE_PATH_SETTING =
-            Setting.simpleString("repositories.azure.base_path", Property.NodeScope);
+            Setting.simpleString("repositories.azure.base_path", Property.NodeScope, Property.Deprecated);
+        @Deprecated
         public static final Setting<String> LOCATION_MODE_SETTING =
-            Setting.simpleString("repositories.azure.location_mode", Property.NodeScope);
+            Setting.simpleString("repositories.azure.location_mode", Property.NodeScope, Property.Deprecated);
+        @Deprecated
         public static final Setting<ByteSizeValue> CHUNK_SIZE_SETTING =
-            Setting.byteSizeSetting("repositories.azure.chunk_size", new ByteSizeValue(-1), Property.NodeScope);
+            Setting.byteSizeSetting("repositories.azure.chunk_size", MAX_CHUNK_SIZE, MIN_CHUNK_SIZE, MAX_CHUNK_SIZE, Property.NodeScope,
+                Property.Deprecated);
+        @Deprecated
         public static final Setting<Boolean> COMPRESS_SETTING =
-            Setting.boolSetting("repositories.azure.compress", false, Property.NodeScope);
+            Setting.boolSetting("repositories.azure.compress", false, Property.NodeScope, Property.Deprecated);
     }
 
     boolean doesContainerExist(String account, LocationMode mode, String container);
@@ -78,12 +89,12 @@ final class Storage {
     InputStream getInputStream(String account, LocationMode mode, String container, String blob)
         throws URISyntaxException, StorageException, IOException;
 
-    OutputStream getOutputStream(String account, LocationMode mode, String container, String blob)
-        throws URISyntaxException, StorageException;
-
     Map<String,BlobMetaData> listBlobsByPrefix(String account, LocationMode mode, String container, String keyPath, String prefix)
         throws URISyntaxException, StorageException;
 
     void moveBlob(String account, LocationMode mode, String container, String sourceBlob, String targetBlob)
         throws URISyntaxException, StorageException;
+
+    void writeBlob(String account, LocationMode mode, String container, String blobName, InputStream inputStream, long blobSize) throws
+        URISyntaxException, StorageException, IOException;
 }
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java
index cd201e7ff5617..d1eff5f8b31e3 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java
@@ -21,6 +21,8 @@
 
 import com.microsoft.azure.storage.CloudStorageAccount;
 import com.microsoft.azure.storage.LocationMode;
+import com.microsoft.azure.storage.RetryExponentialRetry;
+import com.microsoft.azure.storage.RetryPolicy;
 import com.microsoft.azure.storage.StorageException;
 import com.microsoft.azure.storage.blob.BlobProperties;
 import com.microsoft.azure.storage.blob.CloudBlobClient;
@@ -38,8 +40,8 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.repositories.RepositoryException;
 
+import java.io.IOException;
 import java.io.InputStream;
-import java.io.OutputStream;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.HashMap;
@@ -143,6 +145,11 @@ CloudBlobClient getSelectedClient(String account, LocationMode mode) {
                     "]. It can not be longer than 2,147,483,647ms.");
             }
         }
+
+        // We define a default exponential retry policy
+        client.getDefaultRequestOptions().setRetryPolicyFactory(
+            new RetryExponentialRetry(RetryPolicy.DEFAULT_CLIENT_BACKOFF, azureStorageSettings.getMaxRetries()));
+
         return client;
     }
 
@@ -175,7 +182,7 @@ public void createContainer(String account, LocationMode mode, String container)
             blobContainer.createIfNotExists();
         } catch (IllegalArgumentException e) {
             logger.trace((Supplier<?>) () -> new ParameterizedMessage("fails creating container [{}]", container), e);
-            throw new RepositoryException(container, e.getMessage());
+            throw new RepositoryException(container, e.getMessage(), e);
         }
     }
 
@@ -250,13 +257,6 @@ public InputStream getInputStream(String account, LocationMode mode, String cont
         return client.getContainerReference(container).getBlockBlobReference(blob).openInputStream();
     }
 
-    @Override
-    public OutputStream getOutputStream(String account, LocationMode mode, String container, String blob) throws URISyntaxException, StorageException {
-        logger.trace("writing container [{}], blob [{}]", container, blob);
-        CloudBlobClient client = this.getSelectedClient(account, mode);
-        return client.getContainerReference(container).getBlockBlobReference(blob).openOutputStream();
-    }
-
     @Override
     public Map<String, BlobMetaData> listBlobsByPrefix(String account, LocationMode mode, String container, String keyPath, String prefix) throws URISyntaxException, StorageException {
         // NOTE: this should be here: if (prefix == null) prefix = "";
@@ -307,4 +307,15 @@ public void moveBlob(String account, LocationMode mode, String container, String
             logger.debug("moveBlob container [{}], sourceBlob [{}], targetBlob [{}] -> done", container, sourceBlob, targetBlob);
         }
     }
+
+    @Override
+    public void writeBlob(String account, LocationMode mode, String container, String blobName, InputStream inputStream, long blobSize)
+        throws URISyntaxException, StorageException, IOException {
+        logger.trace("writeBlob({}, stream, {})", blobName, blobSize);
+        CloudBlobClient client = this.getSelectedClient(account, mode);
+        CloudBlobContainer blobContainer = client.getContainerReference(container);
+        CloudBlockBlob blob = blobContainer.getBlockBlobReference(blobName);
+        blob.upload(inputStream, blobSize);
+        logger.trace("writeBlob({}, stream, {}) - done", blobName, blobSize);
+    }
 }
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettings.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettings.java
index 6d1ed0c10495c..6fe663a4cd2bb 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettings.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettings.java
@@ -19,12 +19,14 @@
 
 package org.elasticsearch.cloud.azure.storage;
 
+import com.microsoft.azure.storage.RetryPolicy;
 import org.elasticsearch.cloud.azure.storage.AzureStorageService.Storage;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsException;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.node.Node;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -34,38 +36,35 @@
 import java.util.function.Function;
 
 public final class AzureStorageSettings {
-    private static final String TIMEOUT_SUFFIX = "timeout";
-    private static final String ACCOUNT_SUFFIX = "account";
-    private static final String KEY_SUFFIX = "key";
-    private static final String DEFAULT_SUFFIX = "default";
-
-    private static final Setting.AffixKey TIMEOUT_KEY = Setting.AffixKey.withAdfix(Storage.PREFIX, TIMEOUT_SUFFIX);
-
-    private static final Setting<TimeValue> TIMEOUT_SETTING = Setting.affixKeySetting(
-        TIMEOUT_KEY,
-        (s) -> Storage.TIMEOUT_SETTING.get(s).toString(),
-        (s) -> Setting.parseTimeValue(s, TimeValue.timeValueSeconds(-1), TIMEOUT_KEY.toString()),
-        Setting.Property.NodeScope);
+    private static final Setting<TimeValue> TIMEOUT_SETTING = Setting.affixKeySetting(Storage.PREFIX, "timeout",
+        (key) -> Setting.timeSetting(key, Storage.TIMEOUT_SETTING, Setting.Property.NodeScope));
     private static final Setting<String> ACCOUNT_SETTING =
-        Setting.adfixKeySetting(Storage.PREFIX, ACCOUNT_SUFFIX, "", Function.identity(), Setting.Property.NodeScope);
+        Setting.affixKeySetting(Storage.PREFIX, "account", (key) -> Setting.simpleString(key, Setting.Property.NodeScope));
     private static final Setting<String> KEY_SETTING =
-        Setting.adfixKeySetting(Storage.PREFIX, KEY_SUFFIX, "", Function.identity(), Setting.Property.NodeScope);
+        Setting.affixKeySetting(Storage.PREFIX, "key", (key) -> Setting.simpleString(key, Setting.Property.NodeScope));
     private static final Setting<Boolean> DEFAULT_SETTING =
-        Setting.adfixKeySetting(Storage.PREFIX, DEFAULT_SUFFIX, "false", Boolean::valueOf, Setting.Property.NodeScope);
-
+        Setting.affixKeySetting(Storage.PREFIX, "default", (key) -> Setting.boolSetting(key, false, Setting.Property.NodeScope));
+    /**
+     * max_retries: Number of retries in case of Azure errors. Defaults to 3 (RetryPolicy.DEFAULT_CLIENT_RETRY_COUNT).
+     */
+    private static final Setting<Integer> MAX_RETRIES_SETTING =
+        Setting.affixKeySetting(Storage.PREFIX, "max_retries",
+            (key) -> Setting.intSetting(key, RetryPolicy.DEFAULT_CLIENT_RETRY_COUNT, Setting.Property.NodeScope));
 
     private final String name;
     private final String account;
     private final String key;
     private final TimeValue timeout;
     private final boolean activeByDefault;
+    private final int maxRetries;
 
-    public AzureStorageSettings(String name, String account, String key, TimeValue timeout, boolean activeByDefault) {
+    public AzureStorageSettings(String name, String account, String key, TimeValue timeout, boolean activeByDefault, int maxRetries) {
         this.name = name;
         this.account = account;
         this.key = key;
         this.timeout = timeout;
         this.activeByDefault = activeByDefault;
+        this.maxRetries = maxRetries;
     }
 
     public String getName() {
@@ -88,6 +87,10 @@ public boolean isActiveByDefault() {
         return activeByDefault;
     }
 
+    public int getMaxRetries() {
+        return maxRetries;
+    }
+
     @Override
     public String toString() {
         final StringBuilder sb = new StringBuilder("AzureStorageSettings{");
@@ -96,6 +99,7 @@ public String toString() {
         sb.append(", key='").append(key).append('\'');
         sb.append(", activeByDefault='").append(activeByDefault).append('\'');
         sb.append(", timeout=").append(timeout);
+        sb.append(", maxRetries=").append(maxRetries);
         sb.append('}');
         return sb.toString();
     }
@@ -121,7 +125,8 @@ private static List<AzureStorageSettings> createStorageSettings(Settings setting
                     getValue(settings, groupName, ACCOUNT_SETTING),
                     getValue(settings, groupName, KEY_SETTING),
                     getValue(settings, groupName, TIMEOUT_SETTING),
-                    getValue(settings, groupName, DEFAULT_SETTING))
+                    getValue(settings, groupName, DEFAULT_SETTING),
+                    getValue(settings, groupName, MAX_RETRIES_SETTING))
             );
         }
         return storageSettings;
@@ -139,7 +144,8 @@ private static AzureStorageSettings getPrimary(List<AzureStorageSettings> settin
         } else if (settings.size() == 1) {
             // the only storage settings belong (implicitly) to the default primary storage
             AzureStorageSettings storage = settings.get(0);
-            return new AzureStorageSettings(storage.getName(), storage.getAccount(), storage.getKey(), storage.getTimeout(), true);
+            return new AzureStorageSettings(storage.getName(), storage.getAccount(), storage.getKey(), storage.getTimeout(), true,
+                storage.getMaxRetries());
         } else {
             AzureStorageSettings primary = null;
             for (AzureStorageSettings setting : settings) {
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/plugin/repository/azure/AzureRepositoryPlugin.java b/plugins/repository-azure/src/main/java/org/elasticsearch/plugin/repository/azure/AzureRepositoryPlugin.java
index 5b938fce188d4..f895383307469 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/plugin/repository/azure/AzureRepositoryPlugin.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/plugin/repository/azure/AzureRepositoryPlugin.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.cloud.azure.storage.AzureStorageServiceImpl;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.RepositoryPlugin;
@@ -45,9 +46,9 @@ protected AzureStorageService createStorageService(Settings settings) {
     }
 
     @Override
-    public Map<String, Repository.Factory> getRepositories(Environment env) {
+    public Map<String, Repository.Factory> getRepositories(Environment env, NamedXContentRegistry namedXContentRegistry) {
         return Collections.singletonMap(AzureRepository.TYPE,
-            (metadata) -> new AzureRepository(metadata, env, createStorageService(env.settings())));
+            (metadata) -> new AzureRepository(metadata, env, namedXContentRegistry, createStorageService(env.settings())));
     }
 
     @Override
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
index 1cf75780ce1cd..dd757449a26cf 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
@@ -32,6 +32,7 @@
 import org.elasticsearch.cloud.azure.storage.AzureStorageService.Storage;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.metadata.RepositoryMetaData;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.repositories.IndexId;
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.common.Strings;
@@ -39,15 +40,14 @@
 import org.elasticsearch.common.blobstore.BlobStore;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
-import org.elasticsearch.common.settings.SettingsException;
-import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.repositories.RepositoryVerificationException;
 import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
 import org.elasticsearch.snapshots.SnapshotCreationException;
 
-import static org.elasticsearch.cloud.azure.storage.AzureStorageSettings.getEffectiveSetting;
+import static org.elasticsearch.cloud.azure.storage.AzureStorageService.MAX_CHUNK_SIZE;
+import static org.elasticsearch.cloud.azure.storage.AzureStorageService.MIN_CHUNK_SIZE;
 import static org.elasticsearch.cloud.azure.storage.AzureStorageSettings.getValue;
 
 /**
@@ -63,8 +63,6 @@
  */
 public class AzureRepository extends BlobStoreRepository {
 
-    private static final ByteSizeValue MAX_CHUNK_SIZE = new ByteSizeValue(64, ByteSizeUnit.MB);
-
     public static final String TYPE = "azure";
 
     public static final class Repository {
@@ -74,7 +72,7 @@ public static final class Repository {
         public static final Setting<String> BASE_PATH_SETTING = Setting.simpleString("base_path", Property.NodeScope);
         public static final Setting<String> LOCATION_MODE_SETTING = Setting.simpleString("location_mode", Property.NodeScope);
         public static final Setting<ByteSizeValue> CHUNK_SIZE_SETTING =
-            Setting.byteSizeSetting("chunk_size", MAX_CHUNK_SIZE, Property.NodeScope);
+            Setting.byteSizeSetting("chunk_size", MAX_CHUNK_SIZE, MIN_CHUNK_SIZE, MAX_CHUNK_SIZE, Property.NodeScope);
         public static final Setting<Boolean> COMPRESS_SETTING = Setting.boolSetting("compress", false, Property.NodeScope);
     }
 
@@ -84,27 +82,28 @@ public static final class Repository {
     private final boolean compress;
     private final boolean readonly;
 
-    public AzureRepository(RepositoryMetaData metadata, Environment environment, AzureStorageService storageService)
+    public AzureRepository(RepositoryMetaData metadata, Environment environment,
+                           NamedXContentRegistry namedXContentRegistry, AzureStorageService storageService)
         throws IOException, URISyntaxException, StorageException {
-        super(metadata, environment.settings());
+        super(metadata, environment.settings(), namedXContentRegistry);
 
         blobStore = new AzureBlobStore(metadata, environment.settings(), storageService);
         String container = getValue(metadata.settings(), settings, Repository.CONTAINER_SETTING, Storage.CONTAINER_SETTING);
-        ByteSizeValue configuredChunkSize = getValue(metadata.settings(), settings, Repository.CHUNK_SIZE_SETTING, Storage.CHUNK_SIZE_SETTING);
-        if (configuredChunkSize.getMb() > MAX_CHUNK_SIZE.getMb()) {
-            Setting<ByteSizeValue> setting = getEffectiveSetting(metadata.settings(), Repository.CHUNK_SIZE_SETTING, Storage.CHUNK_SIZE_SETTING);
-            throw new SettingsException("["  + setting.getKey() + "] must not exceed [" + MAX_CHUNK_SIZE + "] but is set to [" + configuredChunkSize + "].");
-        } else {
-            this.chunkSize = configuredChunkSize;
-        }
-
+        this.chunkSize = getValue(metadata.settings(), settings, Repository.CHUNK_SIZE_SETTING, Storage.CHUNK_SIZE_SETTING);
         this.compress = getValue(metadata.settings(), settings, Repository.COMPRESS_SETTING, Storage.COMPRESS_SETTING);
         String modeStr = getValue(metadata.settings(), settings, Repository.LOCATION_MODE_SETTING, Storage.LOCATION_MODE_SETTING);
-        if (Strings.hasLength(modeStr)) {
-            LocationMode locationMode = LocationMode.valueOf(modeStr.toUpperCase(Locale.ROOT));
-            readonly = locationMode == LocationMode.SECONDARY_ONLY;
+        Boolean forcedReadonly = metadata.settings().getAsBoolean("readonly", null);
+        // If the user explicitly did not define a readonly value, we set it by ourselves depending on the location mode setting.
+        // For secondary_only setting, the repository should be read only
+        if (forcedReadonly == null) {
+            if (Strings.hasLength(modeStr)) {
+                LocationMode locationMode = LocationMode.valueOf(modeStr.toUpperCase(Locale.ROOT));
+                this.readonly = locationMode == LocationMode.SECONDARY_ONLY;
+            } else {
+                this.readonly = false;
+            }
         } else {
-            readonly = false;
+            readonly = forcedReadonly;
         }
 
         String basePath = getValue(metadata.settings(), settings, Repository.BASE_PATH_SETTING, Storage.BASE_PATH_SETTING);
@@ -158,6 +157,8 @@ public void initializeSnapshot(SnapshotId snapshotId, List<IndexId> indices, Met
         try {
             if (!blobStore.doesContainerExist(blobStore.container())) {
                 logger.debug("container [{}] does not exist. Creating...", blobStore.container());
+                deprecationLogger.deprecated("Auto creation of the container for an azure backed repository is deprecated" +
+                    " and will be removed in 6.0.");
                 blobStore.createContainer(blobStore.container());
             }
             super.initializeSnapshot(snapshotId, indices, clusterMetadata);
@@ -173,6 +174,8 @@ public String startVerification() {
             try {
                 if (!blobStore.doesContainerExist(blobStore.container())) {
                     logger.debug("container [{}] does not exist. Creating...", blobStore.container());
+                    deprecationLogger.deprecated("Auto creation of the container for an azure backed repository is deprecated" +
+                        " and will be removed in 6.0.");
                     blobStore.createContainer(blobStore.container());
                 }
             } catch (StorageException | URISyntaxException e) {
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureIntegTestCase.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureIntegTestCase.java
deleted file mode 100644
index 82c5e6c188b9a..0000000000000
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureIntegTestCase.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.cloud.azure;
-
-import org.elasticsearch.plugin.repository.azure.AzureRepositoryPlugin;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.test.ESIntegTestCase;
-
-import java.util.Arrays;
-import java.util.Collection;
-
-/**
- * Base class for Azure tests.
- */
-public abstract class AbstractAzureIntegTestCase extends ESIntegTestCase {
-
-    @Override
-    protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Arrays.asList(AzureRepositoryPlugin.class);
-    }
-}
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureRepositoryServiceIntegTestCase.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureRepositoryServiceIntegTestCase.java
deleted file mode 100644
index e42dcfcd2988b..0000000000000
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureRepositoryServiceIntegTestCase.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.cloud.azure;
-
-import com.microsoft.azure.storage.LocationMode;
-import com.microsoft.azure.storage.StorageException;
-
-import org.elasticsearch.cloud.azure.storage.AzureStorageService;
-import org.elasticsearch.cloud.azure.storage.AzureStorageService.Storage;
-import org.elasticsearch.cloud.azure.storage.AzureStorageServiceMock;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.plugin.repository.azure.AzureRepositoryPlugin;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.repositories.RepositoryMissingException;
-import org.elasticsearch.test.store.MockFSDirectoryService;
-import org.elasticsearch.test.store.MockFSIndexStore;
-import org.junit.After;
-import org.junit.Before;
-
-import java.net.URISyntaxException;
-import java.util.Arrays;
-import java.util.Collection;
-
-public abstract class AbstractAzureRepositoryServiceIntegTestCase extends AbstractAzureIntegTestCase {
-
-    private static final AzureStorageService storageService = new AzureStorageServiceMock();
-
-    public static class TestPlugin extends AzureRepositoryPlugin {
-        @Override
-        protected AzureStorageService createStorageService(Settings settings) {
-            return storageService;
-        }
-    }
-
-    protected String basePath;
-
-    public AbstractAzureRepositoryServiceIntegTestCase(String basePath) {
-        this.basePath = basePath;
-    }
-
-    /**
-     * Deletes repositories, supports wildcard notation.
-     */
-    public static void wipeRepositories(String... repositories) {
-        // if nothing is provided, delete all
-        if (repositories.length == 0) {
-            repositories = new String[]{"*"};
-        }
-        for (String repository : repositories) {
-            try {
-                client().admin().cluster().prepareDeleteRepository(repository).get();
-            } catch (RepositoryMissingException ex) {
-                // ignore
-            }
-        }
-    }
-
-    @Override
-    protected Settings nodeSettings(int nodeOrdinal) {
-        Settings.Builder builder = Settings.builder()
-                .put(Storage.CONTAINER_SETTING.getKey(), "snapshots");
-        return builder.build();
-    }
-
-    @Override
-    protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Arrays.asList(TestPlugin.class, MockFSIndexStore.TestPlugin.class);
-    }
-
-    @Override
-    public Settings indexSettings() {
-        // During restore we frequently restore index to exactly the same state it was before, that might cause the same
-        // checksum file to be written twice during restore operation
-        return Settings.builder().put(super.indexSettings())
-                .put(MockFSDirectoryService.RANDOM_PREVENT_DOUBLE_WRITE_SETTING.getKey(), false)
-                .put(MockFSDirectoryService.RANDOM_NO_DELETE_OPEN_FILE_SETTING.getKey(), false)
-                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
-                .build();
-    }
-
-    @Before @After
-    public final void wipe() throws StorageException, URISyntaxException {
-        wipeRepositories();
-        cleanRepositoryFiles(basePath);
-    }
-
-    /**
-     * Purge the test container
-     */
-    public void cleanRepositoryFiles(String path) throws StorageException, URISyntaxException {
-        String container = internalCluster().getInstance(Settings.class).get("repositories.azure.container");
-        logger.info("--> remove blobs in container [{}]", container);
-        storageService.deleteFiles(null, LocationMode.PRIMARY_ONLY, container, path);
-    }
-}
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureWithThirdPartyIntegTestCase.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureWithThirdPartyIntegTestCase.java
deleted file mode 100644
index 35afc77f75242..0000000000000
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureWithThirdPartyIntegTestCase.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.cloud.azure;
-
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.plugin.repository.azure.AzureRepositoryPlugin;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.test.ESIntegTestCase.ThirdParty;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collection;
-
-import static org.elasticsearch.cloud.azure.AzureTestUtils.readSettingsFromFile;
-
-/**
- * Base class for Azure tests that require credentials.
- * <p>
- * You must specify {@code -Dtests.thirdparty=true -Dtests.config=/path/to/config}
- * in order to run these tests.
- */
-@ThirdParty
-public abstract class AbstractAzureWithThirdPartyIntegTestCase extends AbstractAzureIntegTestCase {
-
-    @Override
-    protected Settings nodeSettings(int nodeOrdinal) {
-        return Settings.builder()
-                .put(super.nodeSettings(nodeOrdinal))
-                .put(readSettingsFromFile())
-                .build();
-    }
-
-    @Override
-    protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Arrays.asList(AzureRepositoryPlugin.class);
-    }
-
-}
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AzureTestUtils.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AzureTestUtils.java
index 097f519db0363..9b2a160b89616 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AzureTestUtils.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AzureTestUtils.java
@@ -20,36 +20,26 @@
 package org.elasticsearch.cloud.azure;
 
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsException;
-
-import java.io.IOException;
 
 public class AzureTestUtils {
     /**
-     * Read settings from file when running integration tests with ThirdParty annotation.
-     * elasticsearch.yml file path has to be set with -Dtests.config=/path/to/elasticsearch.yml.
-     * @return Settings from elasticsearch.yml integration test file (for 3rd party tests)
+     * Mock settings from sysprops when running integration tests with ThirdParty annotation.
+     * Start the tests with {@code -Dtests.azure.account=AzureStorageAccount and -Dtests.azure.key=AzureStorageKey}
+     * @return Mock Settings from sysprops
      */
-    public static Settings readSettingsFromFile() {
+    public static Settings generateMockSecureSettings() {
         Settings.Builder settings = Settings.builder();
 
-        // if explicit, just load it and don't load from env
-        try {
-            if (Strings.hasText(System.getProperty("tests.config"))) {
-                try {
-                    settings.loadFromPath(PathUtils.get((System.getProperty("tests.config"))));
-                } catch (IOException e) {
-                    throw new IllegalArgumentException("could not load azure tests config", e);
-                }
-            } else {
-                throw new IllegalStateException("to run integration tests, you need to set -Dtests.thirdparty=true and " +
-                    "-Dtests.config=/path/to/elasticsearch.yml");
-            }
-        } catch (SettingsException exception) {
-            throw new IllegalStateException("your test configuration file is incorrect: " + System.getProperty("tests.config"), exception);
+        if (Strings.isEmpty(System.getProperty("tests.azure.account")) ||
+            Strings.isEmpty(System.getProperty("tests.azure.key"))) {
+            throw new IllegalStateException("to run integration tests, you need to set -Dtests.thirdparty=true and " +
+                "-Dtests.azure.account=azure-account -Dtests.azure.key=azure-key");
         }
+
+        settings.put("cloud.azure.storage.default.account", System.getProperty("tests.azure.account"));
+        settings.put("cloud.azure.storage.default.key", System.getProperty("tests.azure.key"));
+
         return settings.build();
     }
 }
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceMock.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceMock.java
index 51b5eae57ae05..7aa37bd5e303c 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceMock.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceMock.java
@@ -25,13 +25,13 @@
 import org.elasticsearch.common.blobstore.support.PlainBlobMetaData;
 import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.component.AbstractComponent;
+import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.settings.Settings;
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.OutputStream;
 import java.net.URISyntaxException;
 import java.nio.file.NoSuchFileException;
 import java.util.Locale;
@@ -43,7 +43,7 @@
  */
 public class AzureStorageServiceMock extends AbstractComponent implements AzureStorageService {
 
-    protected Map<String, ByteArrayOutputStream> blobs = new ConcurrentHashMap<>();
+    protected final Map<String, ByteArrayOutputStream> blobs = new ConcurrentHashMap<>();
 
     public AzureStorageServiceMock() {
         super(Settings.EMPTY);
@@ -84,17 +84,10 @@ public InputStream getInputStream(String account, LocationMode mode, String cont
         return new ByteArrayInputStream(blobs.get(blob).toByteArray());
     }
 
-    @Override
-    public OutputStream getOutputStream(String account, LocationMode mode, String container, String blob) throws URISyntaxException, StorageException {
-        ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
-        blobs.put(blob, outputStream);
-        return outputStream;
-    }
-
     @Override
     public Map<String, BlobMetaData> listBlobsByPrefix(String account, LocationMode mode, String container, String keyPath, String prefix) {
         MapBuilder<String, BlobMetaData> blobsBuilder = MapBuilder.newMapBuilder();
-        for (String blobName : blobs.keySet()) {
+        blobs.forEach((String blobName, ByteArrayOutputStream bos) -> {
             final String checkBlob;
             if (keyPath != null && !keyPath.isEmpty()) {
                 // strip off key path from the beginning of the blob name
@@ -103,9 +96,9 @@ public Map<String, BlobMetaData> listBlobsByPrefix(String account, LocationMode
                 checkBlob = blobName;
             }
             if (prefix == null || startsWithIgnoreCase(checkBlob, prefix)) {
-                blobsBuilder.put(blobName, new PlainBlobMetaData(checkBlob, blobs.get(blobName).size()));
+                blobsBuilder.put(blobName, new PlainBlobMetaData(checkBlob, bos.size()));
             }
-        }
+        });
         return blobsBuilder.immutableMap();
     }
 
@@ -120,6 +113,17 @@ public void moveBlob(String account, LocationMode mode, String container, String
         }
     }
 
+    @Override
+    public void writeBlob(String account, LocationMode mode, String container, String blobName, InputStream inputStream, long blobSize)
+        throws URISyntaxException, StorageException {
+        try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) {
+            blobs.put(blobName, outputStream);
+            Streams.copy(inputStream, outputStream);
+        } catch (IOException e) {
+            throw new StorageException("MOCK", "Error while writing mock stream", e);
+        }
+    }
+
     /**
      * Test if the given String starts with the specified prefix,
      * ignoring upper/lower case.
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTests.java
index cf7f0cd1abb2b..0452380ceb98f 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTests.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.cloud.azure.storage;
 
 import com.microsoft.azure.storage.LocationMode;
+import com.microsoft.azure.storage.RetryExponentialRetry;
 import com.microsoft.azure.storage.blob.CloudBlobClient;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
@@ -28,7 +29,9 @@
 import java.net.URISyntaxException;
 
 import static org.elasticsearch.cloud.azure.storage.AzureStorageServiceImpl.blobNameFromUri;
+import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
 public class AzureStorageServiceTests extends ESTestCase {
@@ -143,11 +146,36 @@ public void testGetSelectedClientNoTimeout() {
         assertThat(client1.getDefaultRequestOptions().getTimeoutIntervalInMs(), is(nullValue()));
     }
 
+    public void testGetSelectedClientBackoffPolicy() {
+        Settings timeoutSettings = Settings.builder()
+            .put("cloud.azure.storage.azure.account", "myaccount")
+            .put("cloud.azure.storage.azure.key", "mykey")
+            .build();
+
+        AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(timeoutSettings);
+        CloudBlobClient client1 = azureStorageService.getSelectedClient("azure", LocationMode.PRIMARY_ONLY);
+        assertThat(client1.getDefaultRequestOptions().getRetryPolicyFactory(), is(notNullValue()));
+        assertThat(client1.getDefaultRequestOptions().getRetryPolicyFactory(), instanceOf(RetryExponentialRetry.class));
+    }
+
+    public void testGetSelectedClientBackoffPolicyNbRetries() {
+        Settings timeoutSettings = Settings.builder()
+            .put("cloud.azure.storage.azure.account", "myaccount")
+            .put("cloud.azure.storage.azure.key", "mykey")
+            .put("cloud.azure.storage.azure.max_retries", 7)
+            .build();
+
+        AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(timeoutSettings);
+        CloudBlobClient client1 = azureStorageService.getSelectedClient("azure", LocationMode.PRIMARY_ONLY);
+        assertThat(client1.getDefaultRequestOptions().getRetryPolicyFactory(), is(notNullValue()));
+        assertThat(client1.getDefaultRequestOptions().getRetryPolicyFactory(), instanceOf(RetryExponentialRetry.class));
+    }
+
     /**
      * This internal class just overload createClient method which is called by AzureStorageServiceImpl.doStart()
      */
     class AzureStorageServiceMock extends AzureStorageServiceImpl {
-        public AzureStorageServiceMock(Settings settings) {
+        AzureStorageServiceMock(Settings settings) {
             super(settings);
         }
 
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilterTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilterTests.java
index e82ed3a18755d..b67073d7d389f 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilterTests.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilterTests.java
@@ -62,7 +62,7 @@ public void testSettingsFiltering() throws IOException {
         settings.toXContent(xContentBuilder, request);
         xContentBuilder.endObject();
         String filteredSettingsString = xContentBuilder.string();
-        filteredSettings = Settings.builder().loadFromSource(filteredSettingsString).build();
+        filteredSettings = Settings.builder().loadFromSource(filteredSettingsString, xContentBuilder.contentType()).build();
         assertThat(filteredSettings.getAsMap().keySet(), contains("cloud.azure.storage.azure1.default"));
     }
 
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreTests.java
index ccd9bf8cf8159..31b6c904a9764 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreTests.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreTests.java
@@ -27,23 +27,24 @@
 import org.elasticsearch.common.blobstore.BlobStore;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.repositories.ESBlobStoreTestCase;
-import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.ESIntegTestCase.ThirdParty;
 
 import java.io.IOException;
 import java.net.URISyntaxException;
 
-import static org.elasticsearch.cloud.azure.AzureTestUtils.readSettingsFromFile;
+import static org.elasticsearch.cloud.azure.AzureTestUtils.generateMockSecureSettings;
 
 /**
- * You must specify {@code -Dtests.thirdparty=true -Dtests.config=/path/to/elasticsearch.yml}
- * in order to run these tests.
+ * Those integration tests need an Azure access and must be run with
+ * {@code -Dtests.thirdparty=true -Dtests.azure.account=AzureStorageAccount -Dtests.azure.key=AzureStorageKey}
+ * options
  */
-@ESIntegTestCase.ThirdParty
+@ThirdParty
 public class AzureBlobStoreTests extends ESBlobStoreTestCase {
     @Override
     protected BlobStore newBlobStore() throws IOException {
         try {
-            Settings settings = readSettingsFromFile();
+            Settings settings = generateMockSecureSettings();
             RepositoryMetaData metadata = new RepositoryMetaData("ittest", "azure", Settings.EMPTY);
             AzureStorageService storageService = new AzureStorageServiceImpl(settings);
             AzureBlobStore blobStore = new AzureBlobStore(metadata, settings, storageService);
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java
new file mode 100644
index 0000000000000..568cdd60a2ab2
--- /dev/null
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.repositories.azure;
+
+import com.microsoft.azure.storage.LocationMode;
+import com.microsoft.azure.storage.StorageException;
+import org.apache.logging.log4j.core.net.Protocol;
+import org.elasticsearch.cloud.azure.storage.AzureStorageService;
+import org.elasticsearch.cluster.metadata.RepositoryMetaData;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+
+import static org.hamcrest.Matchers.is;
+
+public class AzureRepositorySettingsTests extends ESTestCase {
+
+    private AzureRepository azureRepository(Settings settings) throws StorageException, IOException, URISyntaxException {
+        Settings internalSettings = Settings.builder()
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toAbsolutePath())
+            .putArray(Environment.PATH_DATA_SETTING.getKey(), tmpPaths())
+            .put(settings)
+            .build();
+        return new AzureRepository(new RepositoryMetaData("foo", "azure", internalSettings), new Environment(internalSettings),
+            NamedXContentRegistry.EMPTY, null);
+    }
+
+
+    public void testReadonlyDefault() throws StorageException, IOException, URISyntaxException {
+        assertThat(azureRepository(Settings.EMPTY).isReadOnly(), is(false));
+    }
+
+    public void testReadonlyDefaultAndReadonlyOn() throws StorageException, IOException, URISyntaxException {
+        assertThat(azureRepository(Settings.builder()
+            .put("readonly", true)
+            .build()).isReadOnly(), is(true));
+    }
+
+    public void testReadonlyWithPrimaryOnly() throws StorageException, IOException, URISyntaxException {
+        assertThat(azureRepository(Settings.builder()
+            .put(AzureRepository.Repository.LOCATION_MODE_SETTING.getKey(), LocationMode.PRIMARY_ONLY.name())
+            .build()).isReadOnly(), is(false));
+    }
+
+    public void testReadonlyWithPrimaryOnlyAndReadonlyOn() throws StorageException, IOException, URISyntaxException {
+        assertThat(azureRepository(Settings.builder()
+            .put(AzureRepository.Repository.LOCATION_MODE_SETTING.getKey(), LocationMode.PRIMARY_ONLY.name())
+            .put("readonly", true)
+            .build()).isReadOnly(), is(true));
+    }
+
+    public void testReadonlyWithSecondaryOnlyAndReadonlyOn() throws StorageException, IOException, URISyntaxException {
+        assertThat(azureRepository(Settings.builder()
+            .put(AzureRepository.Repository.LOCATION_MODE_SETTING.getKey(), LocationMode.SECONDARY_ONLY.name())
+            .put("readonly", true)
+            .build()).isReadOnly(), is(true));
+    }
+
+    public void testReadonlyWithSecondaryOnlyAndReadonlyOff() throws StorageException, IOException, URISyntaxException {
+        assertThat(azureRepository(Settings.builder()
+            .put(AzureRepository.Repository.LOCATION_MODE_SETTING.getKey(), LocationMode.SECONDARY_ONLY.name())
+            .put("readonly", false)
+            .build()).isReadOnly(), is(false));
+    }
+
+    public void testReadonlyWithPrimaryAndSecondaryOnlyAndReadonlyOn() throws StorageException, IOException, URISyntaxException {
+        assertThat(azureRepository(Settings.builder()
+            .put(AzureRepository.Repository.LOCATION_MODE_SETTING.getKey(), LocationMode.PRIMARY_THEN_SECONDARY.name())
+            .put("readonly", true)
+            .build()).isReadOnly(), is(true));
+    }
+
+    public void testReadonlyWithPrimaryAndSecondaryOnlyAndReadonlyOff() throws StorageException, IOException, URISyntaxException {
+        assertThat(azureRepository(Settings.builder()
+            .put(AzureRepository.Repository.LOCATION_MODE_SETTING.getKey(), LocationMode.PRIMARY_THEN_SECONDARY.name())
+            .put("readonly", false)
+            .build()).isReadOnly(), is(false));
+    }
+
+    public void testChunkSize() throws StorageException, IOException, URISyntaxException {
+        // default chunk size
+        AzureRepository azureRepository = azureRepository(Settings.EMPTY);
+        assertEquals(AzureStorageService.MAX_CHUNK_SIZE, azureRepository.chunkSize());
+
+        // chunk size in settings
+        int size = randomIntBetween(1, 64);
+        azureRepository = azureRepository(Settings.builder().put("chunk_size", size + "mb").build());
+        assertEquals(new ByteSizeValue(size, ByteSizeUnit.MB), azureRepository.chunkSize());
+
+        // zero bytes is not allowed
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () ->
+            azureRepository(Settings.builder().put("chunk_size", "0").build()));
+        assertEquals("Failed to parse value [0] for setting [chunk_size] must be >= 1b", e.getMessage());
+
+        // negative bytes not allowed
+        e = expectThrows(IllegalArgumentException.class, () ->
+            azureRepository(Settings.builder().put("chunk_size", "-1").build()));
+        assertEquals("Failed to parse value [-1] for setting [chunk_size] must be >= 1b", e.getMessage());
+
+        // greater than max chunk size not allowed
+        e = expectThrows(IllegalArgumentException.class, () ->
+            azureRepository(Settings.builder().put("chunk_size", "65mb").build()));
+        assertEquals("Failed to parse value [65mb] for setting [chunk_size] must be <= 64mb", e.getMessage());
+    }
+
+    public void testDeprecatedRepositoriesSettings() throws StorageException, IOException, URISyntaxException {
+        Settings settings = Settings.builder()
+            .put(AzureStorageService.Storage.ACCOUNT_SETTING.getKey(), "account")
+            .put(AzureStorageService.Storage.BASE_PATH_SETTING.getKey(), "path")
+            .put(AzureStorageService.Storage.CHUNK_SIZE_SETTING.getKey(), AzureStorageService.MAX_CHUNK_SIZE)
+            .put(AzureStorageService.Storage.COMPRESS_SETTING.getKey(), false)
+            .put(AzureStorageService.Storage.CONTAINER_SETTING.getKey(), "container")
+            .put(AzureStorageService.Storage.LOCATION_MODE_SETTING.getKey(), "primary_only")
+            .build();
+        azureRepository(settings);
+        assertSettingDeprecationsAndWarnings(new Setting<?>[]{
+                AzureStorageService.Storage.ACCOUNT_SETTING,
+                AzureStorageService.Storage.BASE_PATH_SETTING,
+                AzureStorageService.Storage.CHUNK_SIZE_SETTING,
+                AzureStorageService.Storage.COMPRESS_SETTING,
+                AzureStorageService.Storage.CONTAINER_SETTING,
+                AzureStorageService.Storage.LOCATION_MODE_SETTING});
+    }
+
+}
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreServiceIntegTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreServiceIntegTests.java
deleted file mode 100644
index 9ac15cae5a2b5..0000000000000
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreServiceIntegTests.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.repositories.azure;
-
-
-import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryResponse;
-import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
-import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.cloud.azure.AbstractAzureRepositoryServiceIntegTestCase;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.ByteSizeUnit;
-import org.elasticsearch.snapshots.SnapshotState;
-import org.elasticsearch.test.ESIntegTestCase;
-
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.greaterThan;
-
-@ESIntegTestCase.ClusterScope(
-        scope = ESIntegTestCase.Scope.SUITE,
-        supportsDedicatedMasters = false,
-        numDataNodes = 1,
-        numClientNodes = 0,
-        transportClientRatio = 0.0)
-public class AzureSnapshotRestoreServiceIntegTests extends AbstractAzureRepositoryServiceIntegTestCase {
-    public AzureSnapshotRestoreServiceIntegTests() {
-        super("/snapshot-test/repo-" + randomInt());
-    }
-
-    public void testSimpleWorkflow() {
-        Client client = client();
-        logger.info("-->  creating azure repository with path [{}]", basePath);
-        PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo")
-                .setType("azure").setSettings(Settings.builder()
-                        .put("base_path", basePath)
-                        .put("chunk_size", randomIntBetween(1000, 10000), ByteSizeUnit.BYTES)
-                ).get();
-        assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
-
-        createIndex("test-idx-1", "test-idx-2", "test-idx-3");
-        ensureGreen();
-
-        logger.info("--> indexing some data");
-        for (int i = 0; i < 100; i++) {
-            index("test-idx-1", "doc", Integer.toString(i), "foo", "bar" + i);
-            index("test-idx-2", "doc", Integer.toString(i), "foo", "baz" + i);
-            index("test-idx-3", "doc", Integer.toString(i), "foo", "baz" + i);
-        }
-        refresh();
-        assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().totalHits(), equalTo(100L));
-        assertThat(client.prepareSearch("test-idx-2").setSize(0).get().getHits().totalHits(), equalTo(100L));
-        assertThat(client.prepareSearch("test-idx-3").setSize(0).get().getHits().totalHits(), equalTo(100L));
-
-        logger.info("--> snapshot");
-        CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap")
-            .setWaitForCompletion(true).setIndices("test-idx-*", "-test-idx-3").get();
-        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
-        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(),
-            equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
-
-        assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get().getSnapshots().get(0).state(),
-            equalTo(SnapshotState.SUCCESS));
-
-        logger.info("--> delete some data");
-        for (int i = 0; i < 50; i++) {
-            client.prepareDelete("test-idx-1", "doc", Integer.toString(i)).get();
-        }
-        for (int i = 50; i < 100; i++) {
-            client.prepareDelete("test-idx-2", "doc", Integer.toString(i)).get();
-        }
-        for (int i = 0; i < 100; i += 2) {
-            client.prepareDelete("test-idx-3", "doc", Integer.toString(i)).get();
-        }
-        refresh();
-        assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().totalHits(), equalTo(50L));
-        assertThat(client.prepareSearch("test-idx-2").setSize(0).get().getHits().totalHits(), equalTo(50L));
-        assertThat(client.prepareSearch("test-idx-3").setSize(0).get().getHits().totalHits(), equalTo(50L));
-
-        logger.info("--> close indices");
-        client.admin().indices().prepareClose("test-idx-1", "test-idx-2").get();
-
-        logger.info("--> restore all indices from the snapshot");
-        RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap")
-            .setWaitForCompletion(true).get();
-        assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
-
-        ensureGreen();
-        assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().totalHits(), equalTo(100L));
-        assertThat(client.prepareSearch("test-idx-2").setSize(0).get().getHits().totalHits(), equalTo(100L));
-        assertThat(client.prepareSearch("test-idx-3").setSize(0).get().getHits().totalHits(), equalTo(50L));
-
-        // Test restore after index deletion
-        logger.info("--> delete indices");
-        cluster().wipeIndices("test-idx-1", "test-idx-2");
-        logger.info("--> restore one index after deletion");
-        restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true)
-            .setIndices("test-idx-*", "-test-idx-2").get();
-        assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
-        ensureGreen();
-        assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().totalHits(), equalTo(100L));
-        ClusterState clusterState = client.admin().cluster().prepareState().get().getState();
-        assertThat(clusterState.getMetaData().hasIndex("test-idx-1"), equalTo(true));
-        assertThat(clusterState.getMetaData().hasIndex("test-idx-2"), equalTo(false));
-    }
-
-}
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreTests.java
index 014014b432cdb..d75c70a71f2a9 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreTests.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreTests.java
@@ -28,221 +28,138 @@
 import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.ClusterAdminClient;
-import org.elasticsearch.cloud.azure.AbstractAzureWithThirdPartyIntegTestCase;
 import org.elasticsearch.cloud.azure.storage.AzureStorageService;
 import org.elasticsearch.cloud.azure.storage.AzureStorageServiceImpl;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
-import org.elasticsearch.discovery.DiscoveryModule;
+import org.elasticsearch.plugin.repository.azure.AzureRepositoryPlugin;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.repositories.RepositoryMissingException;
 import org.elasticsearch.repositories.RepositoryVerificationException;
 import org.elasticsearch.repositories.azure.AzureRepository.Repository;
+import org.elasticsearch.repositories.blobstore.ESBlobStoreRepositoryIntegTestCase;
 import org.elasticsearch.snapshots.SnapshotMissingException;
+import org.elasticsearch.snapshots.SnapshotRestoreException;
 import org.elasticsearch.snapshots.SnapshotState;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
+import org.elasticsearch.test.ESIntegTestCase.ThirdParty;
 import org.elasticsearch.test.store.MockFSDirectoryService;
+import org.elasticsearch.test.store.MockFSIndexStore;
 import org.junit.After;
-import org.junit.Before;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
 
 import java.net.URISyntaxException;
+import java.util.Arrays;
+import java.util.Collection;
 import java.util.Locale;
 import java.util.concurrent.TimeUnit;
 
-import static org.elasticsearch.cloud.azure.AzureTestUtils.readSettingsFromFile;
+import static org.elasticsearch.cloud.azure.AzureTestUtils.generateMockSecureSettings;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
 
 /**
- * This test needs Azure to run and -Dtests.thirdparty=true to be set
- * and -Dtests.config=/path/to/elasticsearch.yml
- * @see AbstractAzureWithThirdPartyIntegTestCase
+ * Those integration tests need an Azure access and must be run with
+ * {@code -Dtests.thirdparty=true -Dtests.azure.account=AzureStorageAccount -Dtests.azure.key=AzureStorageKey}
+ * options
  */
 @ClusterScope(
         scope = ESIntegTestCase.Scope.SUITE,
         supportsDedicatedMasters = false, numDataNodes = 1,
         transportClientRatio = 0.0)
-public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyIntegTestCase {
-    private String getRepositoryPath() {
-        String testName = "it-" + getTestName();
-        return testName.contains(" ") ? Strings.split(testName, " ")[0] : testName;
-    }
+@ThirdParty
+public class AzureSnapshotRestoreTests extends ESBlobStoreRepositoryIntegTestCase {
 
-    private static String getContainerName() {
-        String testName = "snapshot-itest-".concat(RandomizedTest.getContext().getRunnerSeedAsString().toLowerCase(Locale.ROOT));
-        return testName.contains(" ") ? Strings.split(testName, " ")[0] : testName;
+    private static AzureStorageService getAzureStorageService() {
+        return new AzureStorageServiceImpl(generateMockSecureSettings());
     }
 
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {
-        return Settings.builder().put(super.nodeSettings(nodeOrdinal))
-                // In snapshot tests, we explicitly disable cloud discovery
-                .put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), "local")
-                .build();
+        return Settings.builder()
+            .put(generateMockSecureSettings())
+            .put(super.nodeSettings(nodeOrdinal))
+            .build();
     }
 
-    @Override
-    public Settings indexSettings() {
-        // During restore we frequently restore index to exactly the same state it was before, that might cause the same
-        // checksum file to be written twice during restore operation
-        return Settings.builder().put(super.indexSettings())
-                .put(MockFSDirectoryService.RANDOM_PREVENT_DOUBLE_WRITE_SETTING.getKey(), false)
-                .put(MockFSDirectoryService.RANDOM_NO_DELETE_OPEN_FILE_SETTING.getKey(), false)
-                .build();
+    private static String getContainerName() {
+        /* Have a different name per test so that there is no possible race condition. As the long can be negative,
+         * there mustn't be a hyphen between the 2 concatenated numbers
+         * (can't have 2 consecutives hyphens on Azure containers)
+         */
+        String testName = "snapshot-itest-"
+            .concat(RandomizedTest.getContext().getRunnerSeedAsString().toLowerCase(Locale.ROOT));
+        return testName.contains(" ") ? Strings.split(testName, " ")[0] : testName;
     }
 
-    @Before @After
-    public final void wipeAzureRepositories() throws StorageException, URISyntaxException {
-        wipeRepositories();
-        cleanRepositoryFiles(
-            getContainerName(),
-            getContainerName().concat("-1"),
-            getContainerName().concat("-2"));
+    @BeforeClass
+    public static void createTestContainers() throws Exception {
+        createTestContainer(getContainerName());
+        // This is needed for testMultipleRepositories() test case
+        createTestContainer(getContainerName() + "-1");
+        createTestContainer(getContainerName() + "-2");
     }
 
-    public void testSimpleWorkflow() {
-        Client client = client();
-        logger.info("-->  creating azure repository with path [{}]", getRepositoryPath());
-        PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo")
-                .setType("azure").setSettings(Settings.builder()
-                        .put(Repository.CONTAINER_SETTING.getKey(), getContainerName())
-                        .put(Repository.BASE_PATH_SETTING.getKey(), getRepositoryPath())
-                        .put(Repository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(1000, 10000), ByteSizeUnit.BYTES)
-                ).get();
-        assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
-
-        createIndex("test-idx-1", "test-idx-2", "test-idx-3");
-        ensureGreen();
-
-        logger.info("--> indexing some data");
-        for (int i = 0; i < 100; i++) {
-            index("test-idx-1", "doc", Integer.toString(i), "foo", "bar" + i);
-            index("test-idx-2", "doc", Integer.toString(i), "foo", "baz" + i);
-            index("test-idx-3", "doc", Integer.toString(i), "foo", "baz" + i);
-        }
-        refresh();
-        assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().totalHits(), equalTo(100L));
-        assertThat(client.prepareSearch("test-idx-2").setSize(0).get().getHits().totalHits(), equalTo(100L));
-        assertThat(client.prepareSearch("test-idx-3").setSize(0).get().getHits().totalHits(), equalTo(100L));
-
-        logger.info("--> snapshot");
-        CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap")
-            .setWaitForCompletion(true).setIndices("test-idx-*", "-test-idx-3").get();
-        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
-        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(),
-            equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
-
-        assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get().getSnapshots()
-            .get(0).state(), equalTo(SnapshotState.SUCCESS));
-
-        logger.info("--> delete some data");
-        for (int i = 0; i < 50; i++) {
-            client.prepareDelete("test-idx-1", "doc", Integer.toString(i)).get();
-        }
-        for (int i = 50; i < 100; i++) {
-            client.prepareDelete("test-idx-2", "doc", Integer.toString(i)).get();
-        }
-        for (int i = 0; i < 100; i += 2) {
-            client.prepareDelete("test-idx-3", "doc", Integer.toString(i)).get();
-        }
-        refresh();
-        assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().totalHits(), equalTo(50L));
-        assertThat(client.prepareSearch("test-idx-2").setSize(0).get().getHits().totalHits(), equalTo(50L));
-        assertThat(client.prepareSearch("test-idx-3").setSize(0).get().getHits().totalHits(), equalTo(50L));
-
-        logger.info("--> close indices");
-        client.admin().indices().prepareClose("test-idx-1", "test-idx-2").get();
-
-        logger.info("--> restore all indices from the snapshot");
-        RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap")
-            .setWaitForCompletion(true).get();
-        assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
-
-        ensureGreen();
-        assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().totalHits(), equalTo(100L));
-        assertThat(client.prepareSearch("test-idx-2").setSize(0).get().getHits().totalHits(), equalTo(100L));
-        assertThat(client.prepareSearch("test-idx-3").setSize(0).get().getHits().totalHits(), equalTo(50L));
-
-        // Test restore after index deletion
-        logger.info("--> delete indices");
-        cluster().wipeIndices("test-idx-1", "test-idx-2");
-        logger.info("--> restore one index after deletion");
-        restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true)
-            .setIndices("test-idx-*", "-test-idx-2").get();
-        assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
-        ensureGreen();
-        assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().totalHits(), equalTo(100L));
-        ClusterState clusterState = client.admin().cluster().prepareState().get().getState();
-        assertThat(clusterState.getMetaData().hasIndex("test-idx-1"), equalTo(true));
-        assertThat(clusterState.getMetaData().hasIndex("test-idx-2"), equalTo(false));
+    @AfterClass
+    public static void removeContainer() throws Exception {
+        removeTestContainer(getContainerName());
+        // This is needed for testMultipleRepositories() test case
+        removeTestContainer(getContainerName() + "-1");
+        removeTestContainer(getContainerName() + "-2");
     }
 
     /**
-     * For issue #51: https://github.com/elastic/elasticsearch-cloud-azure/issues/51
+     * Create a test container in Azure
+     * @param containerName container name to use
      */
-    public void testMultipleSnapshots() throws URISyntaxException, StorageException {
-        final String indexName = "test-idx-1";
-        final String typeName = "doc";
-        final String repositoryName = "test-repo";
-        final String snapshot1Name = "test-snap-1";
-        final String snapshot2Name = "test-snap-2";
-
-        Client client = client();
-
-        logger.info("creating index [{}]", indexName);
-        createIndex(indexName);
-        ensureGreen();
-
-        logger.info("indexing first document");
-        index(indexName, typeName, Integer.toString(1), "foo", "bar " + Integer.toString(1));
-        refresh();
-        assertThat(client.prepareSearch(indexName).setSize(0).get().getHits().totalHits(), equalTo(1L));
-
-        logger.info("creating Azure repository with path [{}]", getRepositoryPath());
-        PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository(repositoryName)
-                .setType("azure").setSettings(Settings.builder()
-                                .put(Repository.CONTAINER_SETTING.getKey(), getContainerName())
-                                .put(Repository.BASE_PATH_SETTING.getKey(), getRepositoryPath())
-                                .put(Repository.BASE_PATH_SETTING.getKey(), randomIntBetween(1000, 10000), ByteSizeUnit.BYTES)
-                ).get();
-        assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
-
-        logger.info("creating snapshot [{}]", snapshot1Name);
-        CreateSnapshotResponse createSnapshotResponse1 = client.admin().cluster().prepareCreateSnapshot(repositoryName, snapshot1Name)
-            .setWaitForCompletion(true).setIndices(indexName).get();
-        assertThat(createSnapshotResponse1.getSnapshotInfo().successfulShards(), greaterThan(0));
-        assertThat(createSnapshotResponse1.getSnapshotInfo().successfulShards(),
-            equalTo(createSnapshotResponse1.getSnapshotInfo().totalShards()));
-
-        assertThat(client.admin().cluster().prepareGetSnapshots(repositoryName).setSnapshots(snapshot1Name).get().getSnapshots()
-            .get(0).state(), equalTo(SnapshotState.SUCCESS));
+    private static void createTestContainer(String containerName) throws Exception {
+        // It could happen that we run this test really close to a previous one
+        // so we might need some time to be able to create the container
+        assertBusy(() -> {
+            getAzureStorageService().createContainer("default", LocationMode.PRIMARY_ONLY, containerName);
+        }, 30, TimeUnit.SECONDS);
+    }
 
-        logger.info("indexing second document");
-        index(indexName, typeName, Integer.toString(2), "foo", "bar " + Integer.toString(2));
-        refresh();
-        assertThat(client.prepareSearch(indexName).setSize(0).get().getHits().totalHits(), equalTo(2L));
+    /**
+     * Remove a test container in Azure
+     * @param containerName container name to use
+     */
+    private static void removeTestContainer(String containerName) throws URISyntaxException, StorageException {
+        getAzureStorageService().removeContainer("default", LocationMode.PRIMARY_ONLY, containerName);
+    }
 
-        logger.info("creating snapshot [{}]", snapshot2Name);
-        CreateSnapshotResponse createSnapshotResponse2 = client.admin().cluster().prepareCreateSnapshot(repositoryName, snapshot2Name)
-            .setWaitForCompletion(true).setIndices(indexName).get();
-        assertThat(createSnapshotResponse2.getSnapshotInfo().successfulShards(), greaterThan(0));
-        assertThat(createSnapshotResponse2.getSnapshotInfo().successfulShards(),
-            equalTo(createSnapshotResponse2.getSnapshotInfo().totalShards()));
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Arrays.asList(AzureRepositoryPlugin.class, MockFSIndexStore.TestPlugin.class);
+    }
 
-        assertThat(client.admin().cluster().prepareGetSnapshots(repositoryName).setSnapshots(snapshot2Name).get().getSnapshots()
-            .get(0).state(), equalTo(SnapshotState.SUCCESS));
+    private String getRepositoryPath() {
+        String testName = "it-" + getTestName();
+        return testName.contains(" ") ? Strings.split(testName, " ")[0] : testName;
+    }
 
-        logger.info("closing index [{}]", indexName);
-        client.admin().indices().prepareClose(indexName).get();
+    @Override
+    public Settings indexSettings() {
+        // During restore we frequently restore index to exactly the same state it was before, that might cause the same
+        // checksum file to be written twice during restore operation
+        return Settings.builder().put(super.indexSettings())
+                .put(MockFSDirectoryService.RANDOM_PREVENT_DOUBLE_WRITE_SETTING.getKey(), false)
+                .put(MockFSDirectoryService.RANDOM_NO_DELETE_OPEN_FILE_SETTING.getKey(), false)
+                .build();
+    }
 
-        logger.info("attempting restore from snapshot [{}]", snapshot1Name);
-        RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot(repositoryName, snapshot1Name)
-            .setWaitForCompletion(true).get();
-        assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
-        ensureGreen();
-        assertThat(client.prepareSearch(indexName).setSize(0).get().getHits().totalHits(), equalTo(1L));
+    @After
+    public final void wipeAzureRepositories() {
+        try {
+            client().admin().cluster().prepareDeleteRepository("*").get();
+        } catch (RepositoryMissingException ignored) {
+        }
     }
 
     public void testMultipleRepositories() {
@@ -374,8 +291,6 @@ public void testListBlobs_26() throws StorageException, URISyntaxException {
 
         // Get all snapshots - should have one
         assertThat(client.prepareGetSnapshots("test-repo").get().getSnapshots().size(), equalTo(1));
-
-
     }
 
     /**
@@ -405,56 +320,6 @@ public void testGetDeleteNonExistingSnapshot_28() throws StorageException, URISy
         }
     }
 
-    /**
-     * For issue #21: https://github.com/elastic/elasticsearch-cloud-azure/issues/21
-     */
-    public void testForbiddenContainerName() throws Exception {
-        checkContainerName("", false);
-        checkContainerName("es", false);
-        checkContainerName("-elasticsearch", false);
-        checkContainerName("elasticsearch--integration", false);
-        checkContainerName("elasticsearch_integration", false);
-        checkContainerName("ElAsTicsearch_integration", false);
-        checkContainerName("123456789-123456789-123456789-123456789-123456789-123456789-1234", false);
-        checkContainerName("123456789-123456789-123456789-123456789-123456789-123456789-123", true);
-        checkContainerName("elasticsearch-integration", true);
-        checkContainerName("elasticsearch-integration-007", true);
-    }
-
-    /**
-     * Create repository with wrong or correct container name
-     * @param container Container name we want to create
-     * @param correct Is this container name correct
-     */
-    private void checkContainerName(final String container, final boolean correct) throws Exception {
-        logger.info("-->  creating azure repository with container name [{}]", container);
-        // It could happen that we just removed from a previous test the same container so
-        // we can not create it yet.
-        assertBusy(() -> {
-            try {
-                PutRepositoryResponse putRepositoryResponse = client().admin().cluster().preparePutRepository("test-repo")
-                        .setType("azure").setSettings(Settings.builder()
-                                        .put(Repository.CONTAINER_SETTING.getKey(), container)
-                                        .put(Repository.BASE_PATH_SETTING.getKey(), getRepositoryPath())
-                                        .put(Repository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(1000, 10000), ByteSizeUnit.BYTES)
-                        ).get();
-                client().admin().cluster().prepareDeleteRepository("test-repo").get();
-                try {
-                    logger.info("--> remove container [{}]", container);
-                    cleanRepositoryFiles(container);
-                } catch (StorageException | URISyntaxException e) {
-                    // We can ignore that as we just try to clean after the test
-                }
-                assertTrue(putRepositoryResponse.isAcknowledged() == correct);
-            } catch (RepositoryVerificationException e) {
-                if (correct) {
-                    logger.debug(" -> container is being removed. Let's wait a bit...");
-                    fail();
-                }
-            }
-        }, 5, TimeUnit.MINUTES);
-    }
-
     /**
      * Test case for issue #23: https://github.com/elastic/elasticsearch-cloud-azure/issues/23
      */
@@ -473,7 +338,7 @@ public void testNonExistingRepo_23() {
         try {
             client.admin().cluster().prepareRestoreSnapshot("test-repo", "no-existing-snapshot").setWaitForCompletion(true).get();
             fail("Shouldn't be here");
-        } catch (SnapshotMissingException ex) {
+        } catch (SnapshotRestoreException ex) {
             // Expected
         }
     }
@@ -483,24 +348,9 @@ public void testNonExistingRepo_23() {
      */
     public void testRemoveAndCreateContainer() throws Exception {
         final String container = getContainerName().concat("-testremove");
-        final AzureStorageService storageService = new AzureStorageServiceImpl(internalCluster().getDefaultSettings());
 
-        // It could happen that we run this test really close to a previous one
-        // so we might need some time to be able to create the container
-        assertBusy(() -> {
-            try {
-                storageService.createContainer(null, LocationMode.PRIMARY_ONLY, container);
-                logger.debug(" -> container created...");
-            } catch (URISyntaxException e) {
-                // Incorrect URL. This should never happen.
-                fail();
-            } catch (StorageException e) {
-                // It could happen. Let's wait for a while.
-                logger.debug(" -> container is being removed. Let's wait a bit...");
-                fail();
-            }
-        }, 30, TimeUnit.SECONDS);
-        storageService.removeContainer(null, LocationMode.PRIMARY_ONLY, container);
+        createTestContainer(container);
+        removeTestContainer(container);
 
         ClusterAdminClient client = client().admin().cluster();
         logger.info("-->  creating azure repository while container is being removed");
@@ -516,30 +366,52 @@ public void testRemoveAndCreateContainer() throws Exception {
     }
 
     /**
-     * Deletes repositories, supports wildcard notation.
+     * Test that you can snapshot on the primary repository and list the available snapshots
+     * from the secondary repository.
+     *
+     * Note that this test requires an Azure storage account which must be a Read-access geo-redundant
+     * storage (RA-GRS) account type.
+     * @throws Exception If anything goes wrong
      */
-    public static void wipeRepositories(String... repositories) {
-        // if nothing is provided, delete all
-        if (repositories.length == 0) {
-            repositories = new String[]{"*"};
-        }
-        for (String repository : repositories) {
-            try {
-                client().admin().cluster().prepareDeleteRepository(repository).get();
-            } catch (RepositoryMissingException ex) {
-                // ignore
-            }
-        }
+    public void testGeoRedundantStorage() throws Exception {
+        Client client = client();
+        logger.info("-->  creating azure primary repository");
+        PutRepositoryResponse putRepositoryResponsePrimary = client.admin().cluster().preparePutRepository("primary")
+            .setType("azure").setSettings(Settings.builder()
+                .put(Repository.CONTAINER_SETTING.getKey(), getContainerName())
+            ).get();
+        assertThat(putRepositoryResponsePrimary.isAcknowledged(), equalTo(true));
+
+        logger.info("--> start get snapshots on primary");
+        long startWait = System.currentTimeMillis();
+        client.admin().cluster().prepareGetSnapshots("primary").get();
+        long endWait = System.currentTimeMillis();
+        // definitely should be done in 30s, and if its not working as expected, it takes over 1m
+        assertThat(endWait - startWait, lessThanOrEqualTo(30000L));
+
+        logger.info("-->  creating azure secondary repository");
+        PutRepositoryResponse putRepositoryResponseSecondary = client.admin().cluster().preparePutRepository("secondary")
+            .setType("azure").setSettings(Settings.builder()
+                .put(Repository.CONTAINER_SETTING.getKey(), getContainerName())
+                .put(Repository.LOCATION_MODE_SETTING.getKey(), "secondary_only")
+            ).get();
+        assertThat(putRepositoryResponseSecondary.isAcknowledged(), equalTo(true));
+
+        logger.info("--> start get snapshots on secondary");
+        startWait = System.currentTimeMillis();
+        client.admin().cluster().prepareGetSnapshots("secondary").get();
+        endWait = System.currentTimeMillis();
+        logger.info("--> end of get snapshots on secondary. Took {} ms", endWait - startWait);
+        assertThat(endWait - startWait, lessThanOrEqualTo(30000L));
     }
 
-    /**
-     * Purge the test containers
-     */
-    public void cleanRepositoryFiles(String... containers) throws StorageException, URISyntaxException {
-        Settings settings = readSettingsFromFile();
-        AzureStorageService client = new AzureStorageServiceImpl(settings);
-        for (String container : containers) {
-            client.removeContainer(null, LocationMode.PRIMARY_ONLY, container);
-        }
+    @Override
+    protected void createTestRepository(String name) {
+        assertAcked(client().admin().cluster().preparePutRepository(name)
+            .setType(AzureRepository.TYPE)
+            .setSettings(Settings.builder()
+                .put(Repository.CONTAINER_SETTING.getKey(), getContainerName())
+                .put(Repository.BASE_PATH_SETTING.getKey(), getRepositoryPath())
+                .put(Repository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES)));
     }
 }
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/RepositoryAzureClientYamlTestSuiteIT.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/RepositoryAzureClientYamlTestSuiteIT.java
index c7eeff27401fa..551599b0918e0 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/RepositoryAzureClientYamlTestSuiteIT.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/RepositoryAzureClientYamlTestSuiteIT.java
@@ -24,9 +24,6 @@
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-
-import java.io.IOException;
 
 public class RepositoryAzureClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
@@ -35,8 +32,8 @@ public RepositoryAzureClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidat
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
-        return ESClientYamlSuiteTestCase.createParameters(0, 1);
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
     }
 }
 
diff --git a/plugins/repository-gcs/build.gradle b/plugins/repository-gcs/build.gradle
index 9968d4408e4d5..84316827a2091 100644
--- a/plugins/repository-gcs/build.gradle
+++ b/plugins/repository-gcs/build.gradle
@@ -19,7 +19,7 @@
 
 esplugin {
   description 'The GCS repository plugin adds Google Cloud Storage support for repositories.'
-  classname 'org.elasticsearch.plugin.repository.gcs.GoogleCloudStoragePlugin'
+  classname 'org.elasticsearch.repositories.gcs.GoogleCloudStoragePlugin'
 }
 
 versions << [
diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/common/blobstore/gcs/GoogleCloudStorageBlobContainer.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainer.java
similarity index 95%
rename from plugins/repository-gcs/src/main/java/org/elasticsearch/common/blobstore/gcs/GoogleCloudStorageBlobContainer.java
rename to plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainer.java
index 92ade4be4f91a..331e2dadca2da 100644
--- a/plugins/repository-gcs/src/main/java/org/elasticsearch/common/blobstore/gcs/GoogleCloudStorageBlobContainer.java
+++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainer.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.common.blobstore.gcs;
+package org.elasticsearch.repositories.gcs;
 
 import org.elasticsearch.common.blobstore.BlobMetaData;
 import org.elasticsearch.common.blobstore.BlobPath;
@@ -29,7 +29,7 @@
 import java.nio.file.FileAlreadyExistsException;
 import java.util.Map;
 
-public class GoogleCloudStorageBlobContainer extends AbstractBlobContainer {
+class GoogleCloudStorageBlobContainer extends AbstractBlobContainer {
 
     private final GoogleCloudStorageBlobStore blobStore;
     private final String path;
diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/common/blobstore/gcs/GoogleCloudStorageBlobStore.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStore.java
similarity index 98%
rename from plugins/repository-gcs/src/main/java/org/elasticsearch/common/blobstore/gcs/GoogleCloudStorageBlobStore.java
rename to plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStore.java
index b63d03487b2a5..6948c8a513f6a 100644
--- a/plugins/repository-gcs/src/main/java/org/elasticsearch/common/blobstore/gcs/GoogleCloudStorageBlobStore.java
+++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStore.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.common.blobstore.gcs;
+package org.elasticsearch.repositories.gcs;
 
 import com.google.api.client.googleapis.batch.BatchRequest;
 import com.google.api.client.googleapis.batch.json.JsonBatchCallback;
@@ -61,7 +61,7 @@
 
 import static java.net.HttpURLConnection.HTTP_NOT_FOUND;
 
-public class GoogleCloudStorageBlobStore extends AbstractComponent implements BlobStore {
+class GoogleCloudStorageBlobStore extends AbstractComponent implements BlobStore {
 
     /**
      * Google Cloud Storage batch requests are limited to 1000 operations
@@ -71,7 +71,7 @@ public class GoogleCloudStorageBlobStore extends AbstractComponent implements Bl
     private final Storage client;
     private final String bucket;
 
-    public GoogleCloudStorageBlobStore(Settings settings, String bucket, Storage storageClient) {
+    GoogleCloudStorageBlobStore(Settings settings, String bucket, Storage storageClient) {
         super(settings);
         this.bucket = bucket;
         this.client = storageClient;
diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/plugin/repository/gcs/GoogleCloudStoragePlugin.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStoragePlugin.java
similarity index 86%
rename from plugins/repository-gcs/src/main/java/org/elasticsearch/plugin/repository/gcs/GoogleCloudStoragePlugin.java
rename to plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStoragePlugin.java
index c0fa38e8b57ad..2c139ca94abdd 100644
--- a/plugins/repository-gcs/src/main/java/org/elasticsearch/plugin/repository/gcs/GoogleCloudStoragePlugin.java
+++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStoragePlugin.java
@@ -17,16 +17,18 @@
  * under the License.
  */
 
-package org.elasticsearch.plugin.repository.gcs;
+package org.elasticsearch.repositories.gcs;
 
 import java.security.AccessController;
 import java.security.PrivilegedAction;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 
 import com.google.api.client.auth.oauth2.TokenRequest;
 import com.google.api.client.auth.oauth2.TokenResponse;
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
 import com.google.api.client.googleapis.json.GoogleJsonError;
 import com.google.api.client.http.GenericUrl;
 import com.google.api.client.http.HttpHeaders;
@@ -40,15 +42,13 @@
 import com.google.api.services.storage.model.Objects;
 import com.google.api.services.storage.model.StorageObject;
 import org.elasticsearch.SpecialPermission;
-import org.elasticsearch.common.inject.Module;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.RepositoryPlugin;
-import org.elasticsearch.repositories.RepositoriesModule;
 import org.elasticsearch.repositories.Repository;
-import org.elasticsearch.repositories.gcs.GoogleCloudStorageRepository;
-import org.elasticsearch.repositories.gcs.GoogleCloudStorageService;
 
 public class GoogleCloudStoragePlugin extends Plugin implements RepositoryPlugin {
 
@@ -114,14 +114,25 @@ public class GoogleCloudStoragePlugin extends Plugin implements RepositoryPlugin
         });
     }
 
+    private final Map<String, GoogleCredential> credentials;
+
+    public GoogleCloudStoragePlugin(Settings settings) {
+        credentials = GoogleCloudStorageService.loadClientCredentials(settings);
+    }
+
     // overridable for tests
     protected GoogleCloudStorageService createStorageService(Environment environment) {
-        return new GoogleCloudStorageService.InternalGoogleCloudStorageService(environment);
+        return new GoogleCloudStorageService.InternalGoogleCloudStorageService(environment, credentials);
     }
 
     @Override
-    public Map<String, Repository.Factory> getRepositories(Environment env) {
+    public Map<String, Repository.Factory> getRepositories(Environment env, NamedXContentRegistry namedXContentRegistry) {
         return Collections.singletonMap(GoogleCloudStorageRepository.TYPE,
-            (metadata) -> new GoogleCloudStorageRepository(metadata, env, createStorageService(env)));
+            (metadata) -> new GoogleCloudStorageRepository(metadata, env, namedXContentRegistry, createStorageService(env)));
+    }
+
+    @Override
+    public List<Setting<?>> getSettings() {
+        return Collections.singletonList(GoogleCloudStorageService.CREDENTIALS_FILE_SETTING);
     }
 }
diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java
index f7b74d5a4f8f7..500306687f65d 100644
--- a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java
+++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java
@@ -24,13 +24,12 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.BlobStore;
-import org.elasticsearch.common.blobstore.gcs.GoogleCloudStorageBlobStore;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.Environment;
-import org.elasticsearch.plugin.repository.gcs.GoogleCloudStoragePlugin;
 import org.elasticsearch.repositories.RepositoryException;
 import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
 
@@ -43,27 +42,32 @@
 import static org.elasticsearch.common.settings.Setting.timeSetting;
 import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
 
-public class GoogleCloudStorageRepository extends BlobStoreRepository {
+class GoogleCloudStorageRepository extends BlobStoreRepository {
 
-    public static final String TYPE = "gcs";
+    // package private for testing
+    static final ByteSizeValue MIN_CHUNK_SIZE = new ByteSizeValue(1, ByteSizeUnit.BYTES);
+    static final ByteSizeValue MAX_CHUNK_SIZE = new ByteSizeValue(100, ByteSizeUnit.MB);
 
-    public static final TimeValue NO_TIMEOUT = timeValueMillis(-1);
+    static final String TYPE = "gcs";
 
-    public static final Setting<String> BUCKET =
+    static final TimeValue NO_TIMEOUT = timeValueMillis(-1);
+
+    static final Setting<String> BUCKET =
             simpleString("bucket", Property.NodeScope, Property.Dynamic);
-    public static final Setting<String> BASE_PATH =
+    static final Setting<String> BASE_PATH =
             simpleString("base_path", Property.NodeScope, Property.Dynamic);
-    public static final Setting<Boolean> COMPRESS =
+    static final Setting<Boolean> COMPRESS =
             boolSetting("compress", false, Property.NodeScope, Property.Dynamic);
-    public static final Setting<ByteSizeValue> CHUNK_SIZE =
-            byteSizeSetting("chunk_size", new ByteSizeValue(100, ByteSizeUnit.MB), Property.NodeScope, Property.Dynamic);
-    public static final Setting<String> APPLICATION_NAME =
+    static final Setting<ByteSizeValue> CHUNK_SIZE =
+            byteSizeSetting("chunk_size", MAX_CHUNK_SIZE, MIN_CHUNK_SIZE, MAX_CHUNK_SIZE, Property.NodeScope, Property.Dynamic);
+    static final Setting<String> APPLICATION_NAME =
             new Setting<>("application_name", GoogleCloudStoragePlugin.NAME, Function.identity(), Property.NodeScope, Property.Dynamic);
-    public static final Setting<String> SERVICE_ACCOUNT =
-            simpleString("service_account", Property.NodeScope, Property.Dynamic);
-    public static final Setting<TimeValue> HTTP_READ_TIMEOUT =
+    static final Setting<String> SERVICE_ACCOUNT =
+            new Setting<>("service_account", "_default_", Function.identity(), Property.NodeScope, Property.Dynamic, Property.Deprecated);
+    static final Setting<String> CLIENT_NAME = new Setting<>("client", "default", Function.identity());
+    static final Setting<TimeValue> HTTP_READ_TIMEOUT =
             timeSetting("http.read_timeout", NO_TIMEOUT, Property.NodeScope, Property.Dynamic);
-    public static final Setting<TimeValue> HTTP_CONNECT_TIMEOUT =
+    static final Setting<TimeValue> HTTP_CONNECT_TIMEOUT =
             timeSetting("http.connect_timeout", NO_TIMEOUT, Property.NodeScope, Property.Dynamic);
 
     private final ByteSizeValue chunkSize;
@@ -71,13 +75,15 @@ public class GoogleCloudStorageRepository extends BlobStoreRepository {
     private final BlobPath basePath;
     private final GoogleCloudStorageBlobStore blobStore;
 
-    public GoogleCloudStorageRepository(RepositoryMetaData metadata, Environment environment,
+    GoogleCloudStorageRepository(RepositoryMetaData metadata, Environment environment,
+                                        NamedXContentRegistry namedXContentRegistry,
                                         GoogleCloudStorageService storageService) throws Exception {
-        super(metadata, environment.settings());
+        super(metadata, environment.settings(), namedXContentRegistry);
 
-        String bucket = get(BUCKET, metadata);
-        String application = get(APPLICATION_NAME, metadata);
-        String serviceAccount = get(SERVICE_ACCOUNT, metadata);
+        String bucket = getSetting(BUCKET, metadata);
+        String application = getSetting(APPLICATION_NAME, metadata);
+        String serviceAccount = SERVICE_ACCOUNT.get(metadata.settings());
+        String clientName = CLIENT_NAME.get(metadata.settings());
 
         String basePath = BASE_PATH.get(metadata.settings());
         if (Strings.hasLength(basePath)) {
@@ -103,13 +109,13 @@ public GoogleCloudStorageRepository(RepositoryMetaData metadata, Environment env
             readTimeout = timeout;
         }
 
-        this.compress = get(COMPRESS, metadata);
-        this.chunkSize = get(CHUNK_SIZE, metadata);
+        this.compress = getSetting(COMPRESS, metadata);
+        this.chunkSize = getSetting(CHUNK_SIZE, metadata);
 
         logger.debug("using bucket [{}], base_path [{}], chunk_size [{}], compress [{}], application [{}]",
                 bucket, basePath, chunkSize, compress, application);
 
-        Storage client = storageService.createClient(serviceAccount, application, connectTimeout, readTimeout);
+        Storage client = storageService.createClient(serviceAccount, clientName, application, connectTimeout, readTimeout);
         this.blobStore = new GoogleCloudStorageBlobStore(settings, bucket, client);
     }
 
@@ -137,7 +143,7 @@ protected ByteSizeValue chunkSize() {
     /**
      * Get a given setting from the repository settings, throwing a {@link RepositoryException} if the setting does not exist or is empty.
      */
-    static <T> T get(Setting<T> setting, RepositoryMetaData metadata) {
+    static <T> T getSetting(Setting<T> setting, RepositoryMetaData metadata) {
         T value = setting.get(metadata.settings());
         if (value == null) {
             throw new RepositoryException(metadata.name(), "Setting [" + setting.getKey() + "] is not defined for repository");
diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java
index 55e7813de4234..8da54a42ffd1b 100644
--- a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java
+++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java
@@ -35,28 +35,43 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.SecureSetting;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.env.Environment;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.UncheckedIOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
 
-public interface GoogleCloudStorageService {
+interface GoogleCloudStorageService {
+
+    String SETTINGS_PREFIX = "gcs.client.";
+
+    /** A json credentials file loaded from secure settings. */
+    Setting.AffixSetting<InputStream> CREDENTIALS_FILE_SETTING = Setting.affixKeySetting(SETTINGS_PREFIX, "credentials_file",
+        key -> SecureSetting.secureFile(key, null));
 
     /**
      * Creates a client that can be used to manage Google Cloud Storage objects.
      *
      * @param serviceAccount path to service account file
+     * @param clientName     name of client settings to use from secure settings
      * @param application    name of the application
      * @param connectTimeout connection timeout for HTTP requests
      * @param readTimeout    read timeout for HTTP requests
      * @return a Client instance that can be used to manage objects
      */
-    Storage createClient(String serviceAccount, String application, TimeValue connectTimeout, TimeValue readTimeout) throws Exception;
+    Storage createClient(String serviceAccount, String clientName, String application,
+                         TimeValue connectTimeout, TimeValue readTimeout) throws Exception;
 
     /**
      * Default implementation
@@ -67,58 +82,60 @@ class InternalGoogleCloudStorageService extends AbstractComponent implements Goo
 
         private final Environment environment;
 
-        public InternalGoogleCloudStorageService(Environment environment) {
+        /** Credentials identified by client name. */
+        private final Map<String, GoogleCredential> credentials;
+
+        InternalGoogleCloudStorageService(Environment environment, Map<String, GoogleCredential> credentials) {
             super(environment.settings());
             this.environment = environment;
+            this.credentials = credentials;
         }
 
         @Override
-        public Storage createClient(String serviceAccount, String application, TimeValue connectTimeout, TimeValue readTimeout)
-                throws Exception {
+        public Storage createClient(String serviceAccountFile, String clientName, String application,
+                                    TimeValue connectTimeout, TimeValue readTimeout) throws Exception {
             try {
-                GoogleCredential credentials = (DEFAULT.equalsIgnoreCase(serviceAccount)) ? loadDefault() : loadCredentials(serviceAccount);
+                GoogleCredential credential = getCredential(serviceAccountFile, clientName);
                 NetHttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport();
 
                 Storage.Builder storage = new Storage.Builder(httpTransport, JacksonFactory.getDefaultInstance(),
-                        new DefaultHttpRequestInitializer(credentials, connectTimeout, readTimeout));
+                        new DefaultHttpRequestInitializer(credential, connectTimeout, readTimeout));
                 storage.setApplicationName(application);
 
                 logger.debug("initializing client with service account [{}/{}]",
-                        credentials.getServiceAccountId(), credentials.getServiceAccountUser());
+                        credential.getServiceAccountId(), credential.getServiceAccountUser());
                 return storage.build();
             } catch (IOException e) {
                 throw new ElasticsearchException("Error when loading Google Cloud Storage credentials file", e);
             }
         }
 
-        /**
-         * HTTP request initializer that loads credentials from the service account file
-         * and manages authentication for HTTP requests
-         */
-        private GoogleCredential loadCredentials(String serviceAccount) throws IOException {
-            if (serviceAccount == null) {
-                throw new ElasticsearchException("Cannot load Google Cloud Storage service account file from a null path");
-            }
-
-            Path account = environment.configFile().resolve(serviceAccount);
-            if (Files.exists(account) == false) {
-                throw new ElasticsearchException("Unable to find service account file [" + serviceAccount
+        // pkg private for tests
+        GoogleCredential getCredential(String serviceAccountFile, String clientName) throws IOException {
+            if (DEFAULT.equalsIgnoreCase(serviceAccountFile) == false) {
+                deprecationLogger.deprecated("Using GCS service account file from disk is deprecated. " +
+                    "Move the file into the elasticsearch keystore.");
+                Path account = environment.configFile().resolve(serviceAccountFile);
+                if (Files.exists(account) == false) {
+                    throw new IllegalArgumentException("Unable to find service account file [" + serviceAccountFile
                         + "] defined for repository");
-            }
+                }
 
-            try (InputStream is = Files.newInputStream(account)) {
-                GoogleCredential credential = GoogleCredential.fromStream(is);
-                if (credential.createScopedRequired()) {
-                    credential = credential.createScoped(Collections.singleton(StorageScopes.DEVSTORAGE_FULL_CONTROL));
+                try (InputStream is = Files.newInputStream(account)) {
+                    GoogleCredential credential = GoogleCredential.fromStream(is);
+                    if (credential.createScopedRequired()) {
+                        credential = credential.createScoped(Collections.singleton(StorageScopes.DEVSTORAGE_FULL_CONTROL));
+                    }
+                    return credential;
                 }
-                return credential;
+            } else if (credentials.containsKey(clientName)) {
+                return credentials.get(clientName);
             }
+            return getDefaultCredential();
         }
 
-        /**
-         * HTTP request initializer that loads default credentials when running on Compute Engine
-         */
-        private GoogleCredential loadDefault() throws IOException {
+        // pkg private for tests
+        GoogleCredential getDefaultCredential() throws IOException {
             return GoogleCredential.getApplicationDefault();
         }
 
@@ -172,4 +189,23 @@ private ExponentialBackOff newBackOff() {
             }
         }
     }
+
+    /** Load all secure credentials from the settings. */
+    static Map<String, GoogleCredential> loadClientCredentials(Settings settings) {
+        Set<String> clientNames = settings.getGroups(SETTINGS_PREFIX).keySet();
+        Map<String, GoogleCredential> credentials = new HashMap<>();
+        for (String clientName : clientNames) {
+            Setting<InputStream> concreteSetting = CREDENTIALS_FILE_SETTING.getConcreteSettingForNamespace(clientName);
+            try (InputStream credStream = concreteSetting.get(settings)) {
+                GoogleCredential credential = GoogleCredential.fromStream(credStream);
+                if (credential.createScopedRequired()) {
+                    credential = credential.createScoped(Collections.singleton(StorageScopes.DEVSTORAGE_FULL_CONTROL));
+                }
+                credentials.put(clientName, credential);
+            } catch (IOException e) {
+                throw new UncheckedIOException(e);
+            }
+        }
+        return credentials;
+    }
 }
diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/common/blobstore/gcs/GoogleCloudStorageBlobStoreContainerTests.java b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreContainerTests.java
similarity index 92%
rename from plugins/repository-gcs/src/test/java/org/elasticsearch/common/blobstore/gcs/GoogleCloudStorageBlobStoreContainerTests.java
rename to plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreContainerTests.java
index 4fe8c7183453a..f3fada5a46319 100644
--- a/plugins/repository-gcs/src/test/java/org/elasticsearch/common/blobstore/gcs/GoogleCloudStorageBlobStoreContainerTests.java
+++ b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreContainerTests.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.common.blobstore.gcs;
+package org.elasticsearch.repositories.gcs;
 
 import org.elasticsearch.common.blobstore.BlobStore;
 import org.elasticsearch.common.settings.Settings;
@@ -27,10 +27,9 @@
 import java.util.Locale;
 
 public class GoogleCloudStorageBlobStoreContainerTests extends ESBlobStoreContainerTestCase {
-
     @Override
     protected BlobStore newBlobStore() throws IOException {
-        String bucket = randomAsciiOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT);
+        String bucket = randomAlphaOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT);
         return new GoogleCloudStorageBlobStore(Settings.EMPTY, bucket, MockHttpTransport.newStorage(bucket, getTestName()));
     }
 }
diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java
index 15cd647dc5cc3..ceb8f8909e2b4 100644
--- a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java
+++ b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java
@@ -20,12 +20,12 @@
 package org.elasticsearch.repositories.gcs;
 
 import com.google.api.services.storage.Storage;
-import org.elasticsearch.common.blobstore.gcs.MockHttpTransport;
+import org.elasticsearch.cluster.metadata.RepositoryMetaData;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.env.Environment;
-import org.elasticsearch.plugin.repository.gcs.GoogleCloudStoragePlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.repositories.blobstore.ESBlobStoreRepositoryIntegTestCase;
 import org.junit.BeforeClass;
@@ -67,6 +67,9 @@ public static void setUpStorage() {
     }
 
     public static class MockGoogleCloudStoragePlugin extends GoogleCloudStoragePlugin {
+        public MockGoogleCloudStoragePlugin() {
+            super(Settings.EMPTY);
+        }
         @Override
         protected GoogleCloudStorageService createStorageService(Environment environment) {
             return new MockGoogleCloudStorageService();
@@ -75,9 +78,48 @@ protected GoogleCloudStorageService createStorageService(Environment environment
 
     public static class MockGoogleCloudStorageService implements GoogleCloudStorageService {
         @Override
-        public Storage createClient(String serviceAccount, String application, TimeValue connectTimeout, TimeValue readTimeout) throws
+        public Storage createClient(String serviceAccount, String accountName, String application,
+                                    TimeValue connectTimeout, TimeValue readTimeout) throws
                 Exception {
             return storage.get();
         }
     }
+
+    public void testChunkSize() {
+        // default chunk size
+        RepositoryMetaData repositoryMetaData = new RepositoryMetaData("repo", GoogleCloudStorageRepository.TYPE, Settings.EMPTY);
+        ByteSizeValue chunkSize = GoogleCloudStorageRepository.getSetting(GoogleCloudStorageRepository.CHUNK_SIZE, repositoryMetaData);
+        assertEquals(GoogleCloudStorageRepository.MAX_CHUNK_SIZE, chunkSize);
+
+        // chunk size in settings
+        int size = randomIntBetween(1, 100);
+        repositoryMetaData = new RepositoryMetaData("repo", GoogleCloudStorageRepository.TYPE,
+                                                       Settings.builder().put("chunk_size", size + "mb").build());
+        chunkSize = GoogleCloudStorageRepository.getSetting(GoogleCloudStorageRepository.CHUNK_SIZE, repositoryMetaData);
+        assertEquals(new ByteSizeValue(size, ByteSizeUnit.MB), chunkSize);
+
+        // zero bytes is not allowed
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {
+            RepositoryMetaData repoMetaData = new RepositoryMetaData("repo", GoogleCloudStorageRepository.TYPE,
+                                                                        Settings.builder().put("chunk_size", "0").build());
+            GoogleCloudStorageRepository.getSetting(GoogleCloudStorageRepository.CHUNK_SIZE, repoMetaData);
+        });
+        assertEquals("Failed to parse value [0] for setting [chunk_size] must be >= 1b", e.getMessage());
+
+        // negative bytes not allowed
+        e = expectThrows(IllegalArgumentException.class, () -> {
+            RepositoryMetaData repoMetaData = new RepositoryMetaData("repo", GoogleCloudStorageRepository.TYPE,
+                                                                        Settings.builder().put("chunk_size", "-1").build());
+            GoogleCloudStorageRepository.getSetting(GoogleCloudStorageRepository.CHUNK_SIZE, repoMetaData);
+        });
+        assertEquals("Failed to parse value [-1] for setting [chunk_size] must be >= 1b", e.getMessage());
+
+        // greater than max chunk size not allowed
+        e = expectThrows(IllegalArgumentException.class, () -> {
+            RepositoryMetaData repoMetaData = new RepositoryMetaData("repo", GoogleCloudStorageRepository.TYPE,
+                                                                        Settings.builder().put("chunk_size", "101mb").build());
+            GoogleCloudStorageRepository.getSetting(GoogleCloudStorageRepository.CHUNK_SIZE, repoMetaData);
+        });
+        assertEquals("Failed to parse value [101mb] for setting [chunk_size] must be <= 100mb", e.getMessage());
+    }
 }
diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/common/blobstore/gcs/GoogleCloudStorageBlobStoreTests.java b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreTests.java
similarity index 92%
rename from plugins/repository-gcs/src/test/java/org/elasticsearch/common/blobstore/gcs/GoogleCloudStorageBlobStoreTests.java
rename to plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreTests.java
index b5489466b5136..fe94237f6c976 100644
--- a/plugins/repository-gcs/src/test/java/org/elasticsearch/common/blobstore/gcs/GoogleCloudStorageBlobStoreTests.java
+++ b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreTests.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.common.blobstore.gcs;
+package org.elasticsearch.repositories.gcs;
 
 import org.elasticsearch.common.blobstore.BlobStore;
 import org.elasticsearch.common.settings.Settings;
@@ -30,7 +30,7 @@ public class GoogleCloudStorageBlobStoreTests extends ESBlobStoreTestCase {
 
     @Override
     protected BlobStore newBlobStore() throws IOException {
-        String bucket = randomAsciiOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT);
+        String bucket = randomAlphaOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT);
         return new GoogleCloudStorageBlobStore(Settings.EMPTY, bucket, MockHttpTransport.newStorage(bucket, getTestName()));
     }
 }
diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageServiceTests.java b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageServiceTests.java
new file mode 100644
index 0000000000000..59f17fcd1c23d
--- /dev/null
+++ b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageServiceTests.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.repositories.gcs;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.Map;
+
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.repositories.gcs.GoogleCloudStorageService.InternalGoogleCloudStorageService;
+import org.elasticsearch.test.ESTestCase;
+
+import static org.hamcrest.Matchers.containsString;
+
+public class GoogleCloudStorageServiceTests extends ESTestCase {
+
+    private InputStream getDummyCredentialStream() throws IOException {
+        return GoogleCloudStorageServiceTests.class.getResourceAsStream("/dummy-account.json");
+    }
+
+    public void testDefaultCredential() throws Exception {
+        Environment env = new Environment(Settings.builder().put("path.home", createTempDir()).build());
+        GoogleCredential cred = GoogleCredential.fromStream(getDummyCredentialStream());
+        InternalGoogleCloudStorageService service = new InternalGoogleCloudStorageService(env, Collections.emptyMap()) {
+            @Override
+            GoogleCredential getDefaultCredential() throws IOException {
+                return cred;
+            }
+        };
+        assertSame(cred, service.getCredential("_default_", "default"));
+    }
+
+    public void testFileCredentialBackcompat() throws Exception {
+        Path home = createTempDir();
+        Path config = home.resolve("config");
+        Files.createDirectories(config);
+        Settings settings = Settings.builder()
+            .put("path.home", home).build();
+        Environment env = new Environment(settings);
+        Files.copy(getDummyCredentialStream(), config.resolve("test-cred.json"));
+        InternalGoogleCloudStorageService service = new InternalGoogleCloudStorageService(env, Collections.emptyMap());
+        GoogleCredential cred = service.getCredential("test-cred.json", "default");
+        assertEquals("some-project-name@appspot.gserviceaccount.com", cred.getServiceAccountId());
+        assertWarnings("Using GCS service account file from disk is deprecated. Move the file into the elasticsearch keystore.");
+    }
+
+    public void testFileCredentialMissing() throws Exception {
+        Environment env = new Environment(Settings.builder().put("path.home", createTempDir()).build());
+        InternalGoogleCloudStorageService service = new InternalGoogleCloudStorageService(env, Collections.emptyMap());
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () ->
+            service.getCredential("test-cred.json", "default"));
+        assertThat(e.getMessage(), containsString("Unable to find service account file"));
+        assertWarnings("Using GCS service account file from disk is deprecated. Move the file into the elasticsearch keystore.");
+    }
+
+    public void testClientCredential() throws Exception {
+        GoogleCredential cred = GoogleCredential.fromStream(getDummyCredentialStream());
+        Map<String, GoogleCredential> credentials = Collections.singletonMap("clientname", cred);
+        Environment env = new Environment(Settings.builder().put("path.home", createTempDir()).build());
+        InternalGoogleCloudStorageService service = new InternalGoogleCloudStorageService(env, credentials);
+        assertSame(cred, service.getCredential("_default_", "clientname"));
+    }
+}
diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/common/blobstore/gcs/MockHttpTransport.java b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/MockHttpTransport.java
similarity index 96%
rename from plugins/repository-gcs/src/test/java/org/elasticsearch/common/blobstore/gcs/MockHttpTransport.java
rename to plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/MockHttpTransport.java
index ffb9c8a910fb3..df9c6c5da825f 100644
--- a/plugins/repository-gcs/src/test/java/org/elasticsearch/common/blobstore/gcs/MockHttpTransport.java
+++ b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/MockHttpTransport.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.common.blobstore.gcs;
+package org.elasticsearch.repositories.gcs;
 
 import com.google.api.client.http.HttpTransport;
 import com.google.api.client.http.LowLevelHttpRequest;
@@ -30,7 +30,6 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.path.PathTrie;
-import org.elasticsearch.common.util.Callback;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.RestStatus;
@@ -291,16 +290,13 @@ public MockHttpTransport(String bucket) {
             try (ByteArrayOutputStream os = new ByteArrayOutputStream((int) req.getContentLength())) {
                 req.getStreamingContent().writeTo(os);
 
-                Streams.readAllLines(new ByteArrayInputStream(os.toByteArray()), new Callback<String>() {
-                    @Override
-                    public void handle(String line) {
-                        Handler handler = handlers.retrieve(line, params);
-                        if (handler != null) {
-                            try {
-                                responses.add(handler.execute(line, params, req));
-                            } catch (IOException e) {
-                                responses.add(newMockError(RestStatus.INTERNAL_SERVER_ERROR, e.getMessage()));
-                            }
+                Streams.readAllLines(new ByteArrayInputStream(os.toByteArray()), line -> {
+                    Handler handler = handlers.retrieve(line, params);
+                    if (handler != null) {
+                        try {
+                            responses.add(handler.execute(line, params, req));
+                        } catch (IOException e) {
+                            responses.add(newMockError(RestStatus.INTERNAL_SERVER_ERROR, e.getMessage()));
                         }
                     }
                 });
diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/RepositoryGcsClientYamlTestSuiteIT.java b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/RepositoryGcsClientYamlTestSuiteIT.java
index 52145bf87e23b..5d87e67153302 100644
--- a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/RepositoryGcsClientYamlTestSuiteIT.java
+++ b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/RepositoryGcsClientYamlTestSuiteIT.java
@@ -24,9 +24,6 @@
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-
-import java.io.IOException;
 
 public class RepositoryGcsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
@@ -35,8 +32,8 @@ public RepositoryGcsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
-        return createParameters(0, 1);
+    public static Iterable<Object[]> parameters() throws Exception {
+        return createParameters();
     }
 }
 
diff --git a/plugins/repository-gcs/src/test/resources/dummy-account.json b/plugins/repository-gcs/src/test/resources/dummy-account.json
new file mode 100644
index 0000000000000..e282b6db0e3b0
--- /dev/null
+++ b/plugins/repository-gcs/src/test/resources/dummy-account.json
@@ -0,0 +1,12 @@
+{
+  "type": "service_account",
+  "project_id": "some-project-name",
+  "private_key_id": "c7cefcb7c72a2880ecce49cb9d1095de5a61aff0",
+  "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDa+7r0RE1YykXC\n+d+DXlN3Dg3aL1YOfYuhy5PF/Vi0FFQHyXuPtAvkVHZD2NxMDZq2DxTu3AVLh1UE\nt2hMrWjDQDuArPl8FezpyYQwde04Qlx1YpQ1xUjTaFWd0hrOZEfsxY00h3ilxR3G\nJsofR3PZBKYI11VGruNemCgjiJg5hcoJDxLXUgcfpKJaiPeHutczCeZ1RANQwQF1\n/dPXhqbtWiaS/iu5so64P54TsrVX5DcXmbGr6hQAReIcI6cjA8QhSu6QBtdvEPhv\n27uTuSu4XRtTh3djVGFzFV9pamGZeGELkTiHVSDI8IkQ32s8yuP5Zys/4bFJk7nn\niqJpe0/DAgMBAAECggEAEexQnPWKLx4/H3o8JRBvXGs2DwmYzY7RAukaqzXVMMgJ\nKKoBBv4Biyquk1cIkOD8LLKHUBWKCWiGOOCaFMyMqo5zUFDYCqPwxCHOQ/ki9VvZ\nHXJ4Fv6Su1rqxwQPVZ03ldWFfSspYMgFa9Z47J54iOasgES/og1mZrOldWMUsoBu\nCKf0fH+vIsxWPwmRtyxKCMwqenqdc22nGGLhmpm8tuw1eQp6XtTXagqkPtAVMMga\nmgC0EGqhZA/IklGW1JuGWELjXVMgS/tLIPq+hYsmY14y6Ie032YoSMWkz6Z5p7i0\n/JwCzVZNO1mD0MwVj7nDmokXOpoyM7Qcbx8r1E4Q4QKBgQDxqAZ6D+A671mCNU0J\n6Qzc3cOZq7MBj4y7M/2qPXHC7i/DdbmnM7PPPriaBBch2nX7jZRlRmVDBsmrC4OG\n3m5+HAx7YPVbefwe6h5ki5O9wg1pLcgYY9uvgLSlD85lVZKAzO7QK2W5zfM19kPD\nSckIa+U7DKFbwKhtCsxcP6ARJwKBgQDn+zAPHepGP2Zf78pOLlVXcQqVA1uOu+bW\nrG4G+lPrytB0C4QdwWdBV3Lcqmnw/ae5PkQBs0dCbtWG8+MT8gA6k5kleflaZrAY\ngdUJIUP6J7ocWYxVTfqGFyFF1n5VT8/jbVucaT7izBZfZvlGyf7Vz7ewQzgWQWlK\nCQ0qstV2BQKBgQCajAQAYlDcQCC1dlMbqHDye91RVQ65S84MF1b+Xid4LA5d6dde\nyGERhKJY1Y7ZtrZHt6cVEe1G7XtiKY3nXi+59URCT6L66svEFaR0VxOYgxdCkeXr\nO0nPNvfQrIgqJIz6VJXSij6XktAdTa7OoUyxVxeWKSC05kSQ4BwMTyCWdwKBgQCW\noqlmZ4qE6w5TJaY8diG8kg7JDFEbsjAHHhikN1DfP+d0MzYrDDc8WsifOZlpf4y1\n4RTP9dZD8Sx+YUgG35H+d3FuwHGGnj+i6kunjg5SFhHn7s4NZoFTKRnV+541T4oy\nqARg4IaRRu0QLhGYQfpUZHlm339AFGGGTbJbE51A8QKBgQDTEN5O+3bRG3Fa1J6z\nU9PMrjjs6l8xhXFso10YEYG5KRnfhzCFujyWNiLE6WrlUL8invVBaCxsZr51GDgA\nhyEEdm4kXCRrv4JyhOvIuGxNcAIiQK/e91UQEM6u1t6hUI1rE7ZOyJQzBxj9hFlV\n7OvhBlHXQUtAOdq0XLHr9GzdSA==\n-----END PRIVATE KEY-----\n",
+  "client_email": "some-project-name@appspot.gserviceaccount.com",
+  "client_id": "123456789101112130594",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "https://accounts.google.com/o/oauth2/token",
+  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/some-project-name%40appspot.gserviceaccount.com"
+}
diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle
index 8cfb5043b6c64..3c016026ada25 100644
--- a/plugins/repository-hdfs/build.gradle
+++ b/plugins/repository-hdfs/build.gradle
@@ -18,6 +18,8 @@
  */
 
 import org.apache.tools.ant.taskdefs.condition.Os
+import org.elasticsearch.gradle.test.RestIntegTestTask
+
 import java.nio.file.Files
 import java.nio.file.Path
 import java.nio.file.Paths
@@ -27,8 +29,10 @@ esplugin {
   classname 'org.elasticsearch.repositories.hdfs.HdfsPlugin'
 }
 
+apply plugin: 'elasticsearch.vagrantsupport'
+
 versions << [
-  'hadoop2': '2.7.1'
+  'hadoop2': '2.8.1'
 ]
 
 configurations {
@@ -41,8 +45,9 @@ dependencies {
   compile "org.apache.hadoop:hadoop-annotations:${versions.hadoop2}"
   compile "org.apache.hadoop:hadoop-auth:${versions.hadoop2}"
   compile "org.apache.hadoop:hadoop-hdfs:${versions.hadoop2}"
-  compile 'org.apache.htrace:htrace-core:3.1.0-incubating'
-  compile 'com.google.guava:guava:16.0.1'
+  compile "org.apache.hadoop:hadoop-hdfs-client:${versions.hadoop2}"
+  compile 'org.apache.htrace:htrace-core4:4.0.1-incubating'
+  compile 'com.google.guava:guava:11.0.2'
   compile 'com.google.protobuf:protobuf-java:2.5.0'
   compile 'commons-logging:commons-logging:1.1.3'
   compile 'commons-cli:commons-cli:1.2'
@@ -52,6 +57,7 @@ dependencies {
   compile 'commons-lang:commons-lang:2.6'
   compile 'javax.servlet:servlet-api:2.5'
   compile "org.slf4j:slf4j-api:${versions.slf4j}"
+  compile "org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}"
 
   hdfsFixture project(':test:fixtures:hdfs-fixture')
 }
@@ -60,7 +66,7 @@ dependencyLicenses {
   mapping from: /hadoop-.*/, to: 'hadoop'
 }
 
-task hdfsFixture(type: org.elasticsearch.gradle.test.Fixture) {
+task hdfsFixture(type: org.elasticsearch.gradle.test.AntFixture) {
   dependsOn project.configurations.hdfsFixture
   executable = new File(project.javaHome, 'bin/java')
   env 'CLASSPATH', "${ -> project.configurations.hdfsFixture.asPath }"
@@ -68,44 +74,152 @@ task hdfsFixture(type: org.elasticsearch.gradle.test.Fixture) {
        baseDir
 }
 
-integTest {
-  boolean fixtureSupported = false;
-  if (Os.isFamily(Os.FAMILY_WINDOWS)) {
-    // hdfs fixture will not start without hadoop native libraries on windows
-    String nativePath = System.getenv("HADOOP_HOME")
-    if (nativePath != null) {
-      Path path = Paths.get(nativePath);
-      if (Files.isDirectory(path) &&
-          Files.exists(path.resolve("bin").resolve("winutils.exe")) &&
-          Files.exists(path.resolve("bin").resolve("hadoop.dll")) &&
-          Files.exists(path.resolve("bin").resolve("hdfs.dll"))) {
-        fixtureSupported = true
-      } else {
-        throw new IllegalStateException("HADOOP_HOME: " + path.toString() + " is invalid, does not contain hadoop native libraries in $HADOOP_HOME/bin");
-      }
+// MIT Kerberos Vagrant Testing Fixture
+String box = "krb5kdc"
+Map<String,String> vagrantEnvVars = [
+        'VAGRANT_CWD'           : "${project(':test:fixtures:krb5kdc-fixture').projectDir}",
+        'VAGRANT_VAGRANTFILE'   : 'Vagrantfile',
+        'VAGRANT_PROJECT_DIR'   : "${project(':test:fixtures:krb5kdc-fixture').projectDir}"
+]
+
+task krb5kdcUpdate(type: org.elasticsearch.gradle.vagrant.VagrantCommandTask) {
+  command 'box'
+  subcommand 'update'
+  boxName box
+  environmentVars vagrantEnvVars
+  dependsOn "vagrantCheckVersion", "virtualboxCheckVersion"
+}
+
+task krb5kdcFixture(type: org.elasticsearch.gradle.test.VagrantFixture) {
+  command 'up'
+  args '--provision', '--provider', 'virtualbox'
+  boxName box
+  environmentVars vagrantEnvVars
+  dependsOn krb5kdcUpdate
+}
+
+task krb5AddPrincipals {
+  dependsOn krb5kdcFixture
+}
+
+List<String> principals = [ "elasticsearch", "hdfs/hdfs.build.elastic.co" ]
+String realm = "BUILD.ELASTIC.CO"
+
+for (String principal : principals) {
+  Task create = project.tasks.create("addPrincipal#${principal}", org.elasticsearch.gradle.vagrant.VagrantCommandTask) {
+    command 'ssh'
+    args '--command', "sudo bash /vagrant/src/main/resources/provision/addprinc.sh $principal"
+    boxName box
+    environmentVars vagrantEnvVars
+    dependsOn krb5kdcFixture
+  }
+  krb5AddPrincipals.dependsOn(create)
+}
+
+task secureHdfsFixture(type: org.elasticsearch.gradle.test.AntFixture) {
+  dependsOn project.configurations.hdfsFixture, krb5kdcFixture, krb5AddPrincipals
+  executable = new File(project.javaHome, 'bin/java')
+  env 'CLASSPATH', "${ -> project.configurations.hdfsFixture.asPath }"
+
+  Path keytabPath = project(':test:fixtures:krb5kdc-fixture').buildDir.toPath().resolve("keytabs").resolve("hdfs_hdfs.build.elastic.co.keytab")
+  Path krb5Config = project(':test:fixtures:krb5kdc-fixture').buildDir.toPath().resolve("conf").resolve("krb5.conf")
+
+  final List<String> miniHDFSArgs = ["-Djava.security.krb5.conf=${krb5Config}"]
+
+  if (project.rootProject.ext.javaVersion == JavaVersion.VERSION_1_9) {
+    miniHDFSArgs.add('--add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED')
+  }
+
+  miniHDFSArgs.add('hdfs.MiniHDFS')
+  miniHDFSArgs.add(baseDir)
+  miniHDFSArgs.add("hdfs/hdfs.build.elastic.co@${realm}")
+  miniHDFSArgs.add("${keytabPath}")
+
+  args miniHDFSArgs.toArray()
+}
+
+boolean fixtureSupported = false
+if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+  // hdfs fixture will not start without hadoop native libraries on windows
+  String nativePath = System.getenv("HADOOP_HOME")
+  if (nativePath != null) {
+    Path path = Paths.get(nativePath);
+    if (Files.isDirectory(path) &&
+        Files.exists(path.resolve("bin").resolve("winutils.exe")) &&
+        Files.exists(path.resolve("bin").resolve("hadoop.dll")) &&
+        Files.exists(path.resolve("bin").resolve("hdfs.dll"))) {
+      fixtureSupported = true
+    } else {
+      throw new IllegalStateException("HADOOP_HOME: ${path} is invalid, does not contain hadoop native libraries in \$HADOOP_HOME/bin");
     }
-  } else {
-    fixtureSupported = true
   }
+} else {
+  fixtureSupported = true
+}
+
+if (fixtureSupported) {
+  integTestCluster.dependsOn hdfsFixture
+  integTestRunner.systemProperty 'tests.rest.suite', 'hdfs_repository'
+} else {
+  logger.warn("hdfsFixture unsupported, please set HADOOP_HOME and put HADOOP_HOME\\bin in PATH")
+  // just tests that the plugin loads
+  integTestRunner.systemProperty 'tests.rest.suite', 'hdfs_repository/10_basic'
+}
+
+boolean secureFixtureSupported = false
+if (fixtureSupported) {
+  secureFixtureSupported = project.rootProject.vagrantSupported
+}
+
+// Create a Integration Test suite just for security based tests
+// This must execute before the afterEvaluate block from integTestSecure
+project.afterEvaluate {
+  Path elasticsearchKT = project(':test:fixtures:krb5kdc-fixture').buildDir.toPath().resolve("keytabs").resolve("elasticsearch.keytab").toAbsolutePath()
+  Path krb5conf = project(':test:fixtures:krb5kdc-fixture').buildDir.toPath().resolve("conf").resolve("krb5.conf").toAbsolutePath()
 
-  if (fixtureSupported) {
-    dependsOn hdfsFixture
-  } else {
-    logger.warn("hdfsFixture unsupported, please set HADOOP_HOME and put HADOOP_HOME\\bin in PATH")
-    // just tests that the plugin loads
-    systemProperty 'tests.rest.suite', 'hdfs_repository/10_basic'
+    project.integTestSecureCluster.dependsOn(project.bundlePlugin)
+    project.integTestSecure.clusterConfig.plugin(project.path)
+    project.integTestSecure.clusterConfig.extraConfigFile("repository-hdfs/krb5.keytab", "${elasticsearchKT}")
+    final String baseJvmArgs = "-Xms" + System.getProperty('tests.heap.size', '512m') +
+            " " + "-Xmx" + System.getProperty('tests.heap.size', '512m') +
+            " " + "-Djava.security.krb5.conf=${krb5conf}" +
+            " " + System.getProperty('tests.jvm.argline', '')
+  final String jvmArgs
+    if (project.rootProject.ext.javaVersion == JavaVersion.VERSION_1_9) {
+      jvmArgs = baseJvmArgs + " " + '--add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED'
+    } else {
+      jvmArgs = baseJvmArgs
+    }
+
+    project.integTestSecure.clusterConfig.jvmArgs = jvmArgs
   }
+
+RestIntegTestTask integTestSecure = project.tasks.create('integTestSecure', RestIntegTestTask.class) {
+  description = "Runs rest tests against an elasticsearch cluster with HDFS secured by MIT Kerberos."
+}
+
+if (secureFixtureSupported) {
+  project.check.dependsOn(integTestSecure)
+
+  // Fixture dependencies
+  integTestSecureCluster.dependsOn secureHdfsFixture, krb5kdcFixture
+  integTestSecureRunner.systemProperty 'tests.rest.suite', 'secure_hdfs_repository'
+} else {
+  integTestSecure.enabled = false
 }
 
 thirdPartyAudit.excludes = [
   // classes are missing, because we added hadoop jars one by one until tests pass.
   'com.google.gson.stream.JsonReader', 
-  'com.google.gson.stream.JsonWriter', 
-  'com.jcraft.jsch.ChannelExec', 
-  'com.jcraft.jsch.JSch', 
-  'com.jcraft.jsch.Logger', 
-  'com.jcraft.jsch.Session', 
-  'com.sun.jersey.api.ParamException', 
+  'com.google.gson.stream.JsonWriter',
+  'com.jcraft.jsch.ChannelExec',
+  'com.jcraft.jsch.ChannelSftp',
+  'com.jcraft.jsch.ChannelSftp$LsEntry',
+  'com.jcraft.jsch.JSch',
+  'com.jcraft.jsch.Logger',
+  'com.jcraft.jsch.Session',
+  'com.jcraft.jsch.SftpATTRS',
+  'com.sun.jersey.api.ParamException',
   'com.sun.jersey.api.core.HttpContext', 
   'com.sun.jersey.core.spi.component.ComponentContext', 
   'com.sun.jersey.core.spi.component.ComponentScope', 
@@ -129,6 +243,7 @@ thirdPartyAudit.excludes = [
   'io.netty.channel.ChannelHandlerContext',
   'io.netty.channel.ChannelInboundHandlerAdapter',
   'io.netty.channel.ChannelInitializer',
+  'io.netty.channel.ChannelOption',
   'io.netty.channel.ChannelPipeline',
   'io.netty.channel.EventLoopGroup',
   'io.netty.channel.SimpleChannelInboundHandler',
@@ -157,7 +272,8 @@ thirdPartyAudit.excludes = [
   'io.netty.handler.stream.ChunkedStream',
   'io.netty.handler.stream.ChunkedWriteHandler',
   'io.netty.util.concurrent.GlobalEventExecutor',
-  'javax.ws.rs.core.Context', 
+  'io.netty.util.ReferenceCountUtil',
+  'javax.ws.rs.core.Context',
   'javax.ws.rs.core.MediaType', 
   'javax.ws.rs.core.MultivaluedMap', 
   'javax.ws.rs.core.Response$ResponseBuilder', 
@@ -207,8 +323,7 @@ thirdPartyAudit.excludes = [
   'org.apache.commons.digester.substitution.MultiVariableExpander', 
   'org.apache.commons.digester.substitution.VariableSubstitutor', 
   'org.apache.commons.digester.xmlrules.DigesterLoader', 
-  'org.apache.commons.httpclient.util.URIUtil', 
-  'org.apache.commons.jxpath.JXPathContext', 
+  'org.apache.commons.jxpath.JXPathContext',
   'org.apache.commons.jxpath.ri.JXPathContextReferenceImpl', 
   'org.apache.commons.jxpath.ri.QName', 
   'org.apache.commons.jxpath.ri.compiler.NodeNameTest', 
@@ -258,7 +373,8 @@ thirdPartyAudit.excludes = [
   'org.apache.curator.utils.EnsurePath', 
   'org.apache.curator.utils.PathUtils', 
   'org.apache.curator.utils.ThreadUtils', 
-  'org.apache.curator.utils.ZKPaths', 
+  'org.apache.curator.utils.ZKPaths',
+  'org.apache.directory.shared.kerberos.components.EncryptionKey',
   'org.apache.directory.server.kerberos.shared.keytab.Keytab', 
   'org.apache.directory.server.kerberos.shared.keytab.KeytabEntry', 
   'org.apache.http.NameValuePair', 
@@ -292,33 +408,32 @@ thirdPartyAudit.excludes = [
   'org.codehaus.jackson.JsonFactory', 
   'org.codehaus.jackson.JsonGenerator', 
   'org.codehaus.jackson.JsonGenerator$Feature', 
-  'org.codehaus.jackson.JsonNode', 
-  'org.codehaus.jackson.map.MappingJsonFactory', 
+  'org.codehaus.jackson.map.MappingJsonFactory',
   'org.codehaus.jackson.map.ObjectMapper', 
   'org.codehaus.jackson.map.ObjectReader', 
   'org.codehaus.jackson.map.ObjectWriter', 
   'org.codehaus.jackson.node.ContainerNode', 
-  'org.codehaus.jackson.type.TypeReference', 
-  'org.codehaus.jackson.util.MinimalPrettyPrinter', 
+  'org.codehaus.jackson.util.MinimalPrettyPrinter',
   'org.fusesource.leveldbjni.JniDBFactory', 
   'org.iq80.leveldb.DB', 
   'org.iq80.leveldb.Options', 
-  'org.iq80.leveldb.WriteBatch', 
-  'org.mortbay.jetty.Connector', 
-  'org.mortbay.jetty.Handler', 
-  'org.mortbay.jetty.InclusiveByteRange', 
-  'org.mortbay.jetty.MimeTypes', 
-  'org.mortbay.jetty.NCSARequestLog', 
-  'org.mortbay.jetty.RequestLog', 
-  'org.mortbay.jetty.Server', 
-  'org.mortbay.jetty.handler.ContextHandler$SContext', 
-  'org.mortbay.jetty.handler.ContextHandler', 
-  'org.mortbay.jetty.handler.ContextHandlerCollection', 
-  'org.mortbay.jetty.handler.HandlerCollection', 
-  'org.mortbay.jetty.handler.RequestLogHandler', 
-  'org.mortbay.jetty.nio.SelectChannelConnector', 
-  'org.mortbay.jetty.security.SslSocketConnector', 
-  'org.mortbay.jetty.servlet.AbstractSessionManager', 
+  'org.iq80.leveldb.WriteBatch',
+  'org.mortbay.jetty.Connector',
+  'org.mortbay.jetty.Handler',
+  'org.mortbay.jetty.InclusiveByteRange',
+  'org.mortbay.jetty.MimeTypes',
+  'org.mortbay.jetty.NCSARequestLog',
+  'org.mortbay.jetty.RequestLog',
+  'org.mortbay.jetty.Server',
+  'org.mortbay.jetty.handler.ContextHandler$SContext',
+  'org.mortbay.jetty.handler.ContextHandler',
+  'org.mortbay.jetty.handler.ContextHandlerCollection',
+  'org.mortbay.jetty.handler.HandlerCollection',
+  'org.mortbay.jetty.handler.RequestLogHandler',
+  'org.mortbay.jetty.nio.SelectChannelConnector',
+  'org.mortbay.jetty.security.SslSelectChannelConnector',
+  'org.mortbay.jetty.security.SslSocketConnector',
+  'org.mortbay.jetty.servlet.AbstractSessionManager',
   'org.mortbay.jetty.servlet.Context', 
   'org.mortbay.jetty.servlet.DefaultServlet', 
   'org.mortbay.jetty.servlet.FilterHolder', 
@@ -327,8 +442,7 @@ thirdPartyAudit.excludes = [
   'org.mortbay.jetty.servlet.ServletHolder', 
   'org.mortbay.jetty.servlet.SessionHandler', 
   'org.mortbay.jetty.webapp.WebAppContext', 
-  'org.mortbay.log.Log', 
-  'org.mortbay.thread.QueuedThreadPool', 
+  'org.mortbay.thread.QueuedThreadPool',
   'org.mortbay.util.MultiException', 
   'org.mortbay.util.ajax.JSON$Convertible', 
   'org.mortbay.util.ajax.JSON$Output', 
@@ -340,9 +454,6 @@ thirdPartyAudit.excludes = [
   'org.apache.hadoop.security.SecurityUtil$QualifiedHostResolver',
 
   // internal java api: sun.misc.Unsafe
-  'com.google.common.cache.Striped64',
-  'com.google.common.cache.Striped64$1',
-  'com.google.common.cache.Striped64$Cell',
   'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator', 
   'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1',
   'org.apache.hadoop.io.FastByteComparisons$LexicographicalComparerHolder$UnsafeComparer',
@@ -359,12 +470,28 @@ thirdPartyAudit.excludes = [
   // internal java api: sun.misc.SignalHandler
   'org.apache.hadoop.util.SignalLogger$Handler',
 
-  // optional dependencies of slf4j-api
-  'org.slf4j.impl.StaticMDCBinder',
-  'org.slf4j.impl.StaticMarkerBinder',
+  // we are not pulling in slf4j-ext, this is okay, Log4j will fallback gracefully
+  'org.slf4j.ext.EventData',
 
   'org.apache.log4j.AppenderSkeleton',
   'org.apache.log4j.AsyncAppender',
   'org.apache.log4j.helpers.ISO8601DateFormat',
-  'org.apache.log4j.spi.ThrowableInformation'
+  'org.apache.log4j.spi.ThrowableInformation',
+
+  // New optional dependencies in 2.8
+  'com.nimbusds.jose.JWSObject$State',
+  'com.nimbusds.jose.crypto.RSASSAVerifier',
+  'com.nimbusds.jwt.ReadOnlyJWTClaimsSet',
+  'com.nimbusds.jwt.SignedJWT',
+  'com.squareup.okhttp.Call',
+  'com.squareup.okhttp.MediaType',
+  'com.squareup.okhttp.OkHttpClient',
+  'com.squareup.okhttp.Request$Builder',
+  'com.squareup.okhttp.RequestBody',
+  'com.squareup.okhttp.Response',
+  'com.squareup.okhttp.ResponseBody'
 ]
+
+if (JavaVersion.current() > JavaVersion.VERSION_1_8) {
+  thirdPartyAudit.excludes += ['javax.xml.bind.annotation.adapters.HexBinaryAdapter']
+}
diff --git a/plugins/repository-hdfs/licenses/guava-11.0.2.jar.sha1 b/plugins/repository-hdfs/licenses/guava-11.0.2.jar.sha1
new file mode 100644
index 0000000000000..0588c7ee5eaf7
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/guava-11.0.2.jar.sha1
@@ -0,0 +1 @@
+35a3c69e19d72743cac83778aecbee68680f63eb
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/guava-16.0.1.jar.sha1 b/plugins/repository-hdfs/licenses/guava-16.0.1.jar.sha1
deleted file mode 100644
index 68f2b233a000d..0000000000000
--- a/plugins/repository-hdfs/licenses/guava-16.0.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-5fa98cd1a63c99a44dd8d3b77e4762b066a5d0c5
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/hadoop-annotations-2.7.1.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-annotations-2.7.1.jar.sha1
deleted file mode 100644
index 660467a4c6e57..0000000000000
--- a/plugins/repository-hdfs/licenses/hadoop-annotations-2.7.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-2a77fe74ee056bf45598cf7e20cd624e8388e627
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/hadoop-annotations-2.8.1.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-annotations-2.8.1.jar.sha1
new file mode 100644
index 0000000000000..bb000d4ddb025
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/hadoop-annotations-2.8.1.jar.sha1
@@ -0,0 +1 @@
+335a867cf42bf789919bfc3229ff26747124e8f1
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/hadoop-auth-2.7.1.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-auth-2.7.1.jar.sha1
deleted file mode 100644
index 0161301ead213..0000000000000
--- a/plugins/repository-hdfs/licenses/hadoop-auth-2.7.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-2515f339f97f1d7ba850485e06e395a58586bc2e
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/hadoop-auth-2.8.1.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-auth-2.8.1.jar.sha1
new file mode 100644
index 0000000000000..67a1c4f3fa546
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/hadoop-auth-2.8.1.jar.sha1
@@ -0,0 +1 @@
+688ccccc0e0739d8737a93b0039a4a661e52084b
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/hadoop-client-2.7.1.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-client-2.7.1.jar.sha1
deleted file mode 100644
index 4c6dca8af4989..0000000000000
--- a/plugins/repository-hdfs/licenses/hadoop-client-2.7.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-dbc2faacd210e6a1e3eb7def6e42065c7457d960
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/hadoop-client-2.8.1.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-client-2.8.1.jar.sha1
new file mode 100644
index 0000000000000..e7be8d468ddfc
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/hadoop-client-2.8.1.jar.sha1
@@ -0,0 +1 @@
+4812f251f8100fd4722c3cec5d7353f71f69cda9
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/hadoop-common-2.7.1.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-common-2.7.1.jar.sha1
deleted file mode 100644
index 64ff368db60bf..0000000000000
--- a/plugins/repository-hdfs/licenses/hadoop-common-2.7.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-50580f5ebab60b1b318ad157f668d8e40a1cc0da
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/hadoop-common-2.8.1.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-common-2.8.1.jar.sha1
new file mode 100644
index 0000000000000..6e6c625b7d6c2
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/hadoop-common-2.8.1.jar.sha1
@@ -0,0 +1 @@
+a4df18b79e4d0349ce4b58a52d314e7ae1d6be99
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/hadoop-hdfs-2.7.1.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-hdfs-2.7.1.jar.sha1
deleted file mode 100644
index 2d4954befaaf7..0000000000000
--- a/plugins/repository-hdfs/licenses/hadoop-hdfs-2.7.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-11681de93a4cd76c841e352b7094f839b072a21f
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/hadoop-hdfs-2.8.1.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-hdfs-2.8.1.jar.sha1
new file mode 100644
index 0000000000000..e419a3124f66b
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/hadoop-hdfs-2.8.1.jar.sha1
@@ -0,0 +1 @@
+a378f4bc8e6cd779d779c9f512e0e31edd771633
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/hadoop-hdfs-client-2.8.1.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-hdfs-client-2.8.1.jar.sha1
new file mode 100644
index 0000000000000..20c7963cd3d1b
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/hadoop-hdfs-client-2.8.1.jar.sha1
@@ -0,0 +1 @@
+6b0100e4f58ecf7ce75817fce1ffdfbec947337a
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/htrace-core-3.1.0-incubating.jar.sha1 b/plugins/repository-hdfs/licenses/htrace-core-3.1.0-incubating.jar.sha1
deleted file mode 100644
index c742d8397cf98..0000000000000
--- a/plugins/repository-hdfs/licenses/htrace-core-3.1.0-incubating.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f73606e7c9ede5802335c290bf47490ad6d51df3
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/htrace-core-LICENSE.txt b/plugins/repository-hdfs/licenses/htrace-core-LICENSE.txt
deleted file mode 100644
index 0befae88d4f56..0000000000000
--- a/plugins/repository-hdfs/licenses/htrace-core-LICENSE.txt
+++ /dev/null
@@ -1,242 +0,0 @@
-Apache HTrace (incubating) is Apache 2.0 Licensed. See below for licensing
-of dependencies that are NOT Apache Licensed.
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-
-The HTrace Owl logo is from http://www.clker.com/clipart-13653.html.  It is
-public domain.
-
-D3, a javascript library for manipulating data, used by htrace-hbase
-is Copyright 2010-2014, Michael Bostock and BSD licensed:
-https://github.com/mbostock/d3/blob/master/LICENSE
-
-Bootstrap, an html, css, and javascript framework, is
-Copyright (c) 2011-2015 Twitter, Inc and MIT licensed:
-https://github.com/twbs/bootstrap/blob/master/LICENSE
-
-underscore, a javascript library of functional programming helpers, is
-(c) 2009-2014 Jeremy Ashkenas, DocumentCloud and Investigative Reporters
-& Editors and an MIT license:
-https://github.com/jashkenas/underscore/blob/master/LICENSE
-
-jquery, a javascript library, is Copyright jQuery Foundation and other
-contributors, https://jquery.org/. The software consists of
-voluntary contributions made by many individuals. For exact
-contribution history, see the revision history
-available at https://github.com/jquery/jquery
-It is MIT licensed:
-https://github.com/jquery/jquery/blob/master/LICENSE.txt
-
-backbone, is a javascript library, that is Copyright (c) 2010-2014
-Jeremy Ashkenas, DocumentCloud. It is MIT licensed:
-https://github.com/jashkenas/backbone/blob/master/LICENSE
-
-moment.js is a front end time conversion project.
-It is (c) 2011-2014 Tim Wood, Iskren Chernev, Moment.js contributors
-and shared under the MIT license:
-https://github.com/moment/moment/blob/develop/LICENSE
-
-CMP is an implementation of the MessagePack serialization format in
-C.  It is licensed under the MIT license:
-https://github.com/camgunz/cmp/blob/master/LICENSE
-See ./htrace-c/src/util/cmp.c and ./htrace-c/src/util/cmp.h.
diff --git a/plugins/repository-hdfs/licenses/htrace-core-NOTICE.txt b/plugins/repository-hdfs/licenses/htrace-core-NOTICE.txt
deleted file mode 100644
index 845b6965e0441..0000000000000
--- a/plugins/repository-hdfs/licenses/htrace-core-NOTICE.txt
+++ /dev/null
@@ -1,13 +0,0 @@
-Apache HTrace
-Copyright 2015 The Apache Software Foundation
-
-This product includes software developed at The Apache Software
-Foundation (http://www.apache.org/).
-
-In addition, this product includes software dependencies. See
-the accompanying LICENSE.txt for a listing of dependencies
-that are NOT Apache licensed (with pointers to their licensing)
-
-Apache HTrace includes an Apache Thrift connector to Zipkin. Zipkin
-is a distributed tracing system that is Apache 2.0 Licensed.
-Copyright 2012 Twitter, Inc.
diff --git a/plugins/repository-hdfs/licenses/htrace-core4-4.0.1-incubating.jar.sha1 b/plugins/repository-hdfs/licenses/htrace-core4-4.0.1-incubating.jar.sha1
new file mode 100644
index 0000000000000..a0a562b2a4eb4
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/htrace-core4-4.0.1-incubating.jar.sha1
@@ -0,0 +1 @@
+f4ef727cb4675788ac66f48e217020acc1690960
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/htrace-core4-LICENSE.txt b/plugins/repository-hdfs/licenses/htrace-core4-LICENSE.txt
new file mode 100644
index 0000000000000..5e4a6e6fa77e5
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/htrace-core4-LICENSE.txt
@@ -0,0 +1,245 @@
+Apache HTrace (incubating) is Apache 2.0 Licensed. See below for licensing
+of dependencies that are NOT Apache Licensed.
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+Units, a string formatting go library, is Copyright (c) 2014 Alec Thomas
+and MIT licensed: https://github.com/alecthomas/units/blob/master/COPYING
+
+D3, a javascript library for manipulating data, used by htrace-hbase
+is Copyright 2010-2014, Michael Bostock and BSD licensed:
+https://github.com/mbostock/d3/blob/master/LICENSE
+
+Bootstrap, an html, css, and javascript framework, is
+Copyright (c) 2011-2015 Twitter, Inc and MIT licensed:
+https://github.com/twbs/bootstrap/blob/master/LICENSE
+
+underscore, a javascript library of functional programming helpers, is
+(c) 2009-2014 Jeremy Ashkenas, DocumentCloud and Investigative Reporters
+& Editors and an MIT license:
+https://github.com/jashkenas/underscore/blob/master/LICENSE
+
+jquery, a javascript library, is Copyright jQuery Foundation and other
+contributors, https://jquery.org/. The software consists of
+voluntary contributions made by many individuals. For exact
+contribution history, see the revision history
+available at https://github.com/jquery/jquery
+It is MIT licensed:
+https://github.com/jquery/jquery/blob/master/LICENSE.txt
+
+backbone, is a javascript library, that is Copyright (c) 2010-2014
+Jeremy Ashkenas, DocumentCloud. It is MIT licensed:
+https://github.com/jashkenas/backbone/blob/master/LICENSE
+
+moment.js is a front end time conversion project.
+It is (c) 2011-2014 Tim Wood, Iskren Chernev, Moment.js contributors
+and shared under the MIT license:
+https://github.com/moment/moment/blob/develop/LICENSE
+
+CMP is an implementation of the MessagePack serialization format in
+C.  It is licensed under the MIT license:
+https://github.com/camgunz/cmp/blob/master/LICENSE
+
+go-codec is an implementation of several serialization and deserialization
+codecs in Go.  It is licensed under the MIT license:
+https://github.com/ugorji/go/blob/master/LICENSE
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/htrace-core4-NOTICE.txt b/plugins/repository-hdfs/licenses/htrace-core4-NOTICE.txt
new file mode 100644
index 0000000000000..c96631aab6721
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/htrace-core4-NOTICE.txt
@@ -0,0 +1,16 @@
+Apache HTrace
+Copyright 2015 The Apache Software Foundation
+
+This product includes software developed at The Apache Software
+Foundation (http://www.apache.org/).
+
+In addition, this product includes software dependencies. See
+the accompanying LICENSE.txt for a listing of dependencies
+that are NOT Apache licensed (with pointers to their licensing)
+
+Apache HTrace includes an Apache Thrift connector to Zipkin. Zipkin
+is a distributed tracing system that is Apache 2.0 Licensed.
+Copyright 2012 Twitter, Inc.
+
+Our Owl logo we took from http://www.clker.com/clipart-13653.html.
+It is public domain/free.
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/log4j-slf4j-impl-2.9.1.jar.sha1 b/plugins/repository-hdfs/licenses/log4j-slf4j-impl-2.9.1.jar.sha1
new file mode 100644
index 0000000000000..66119e87e211f
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/log4j-slf4j-impl-2.9.1.jar.sha1
@@ -0,0 +1 @@
+0a97a849b18b3798c4af1a2ca5b10c66cef17e3a
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/log4j-slf4j-impl-LICENSE.txt b/plugins/repository-hdfs/licenses/log4j-slf4j-impl-LICENSE.txt
new file mode 100644
index 0000000000000..6279e5206de13
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/log4j-slf4j-impl-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 1999-2005 The Apache Software Foundation
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/plugins/repository-hdfs/licenses/log4j-slf4j-impl-NOTICE.txt b/plugins/repository-hdfs/licenses/log4j-slf4j-impl-NOTICE.txt
new file mode 100644
index 0000000000000..0375732360047
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/log4j-slf4j-impl-NOTICE.txt
@@ -0,0 +1,5 @@
+Apache log4j
+Copyright 2007 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
\ No newline at end of file
diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java
index e485cf56b4c35..290de5b873ef8 100644
--- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java
+++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java
@@ -25,6 +25,7 @@
 import org.apache.hadoop.fs.Options.CreateOpts;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.PathFilter;
+import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.blobstore.BlobMetaData;
 import org.elasticsearch.common.blobstore.BlobPath;
@@ -32,10 +33,14 @@
 import org.elasticsearch.common.blobstore.support.PlainBlobMetaData;
 import org.elasticsearch.repositories.hdfs.HdfsBlobStore.Operation;
 
+import java.io.FilterInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.FileAlreadyExistsException;
 import java.nio.file.NoSuchFileException;
+import java.security.AccessController;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.LinkedHashMap;
@@ -43,12 +48,14 @@
 
 final class HdfsBlobContainer extends AbstractBlobContainer {
     private final HdfsBlobStore store;
+    private final HdfsSecurityContext securityContext;
     private final Path path;
     private final int bufferSize;
 
-    HdfsBlobContainer(BlobPath blobPath, HdfsBlobStore store, Path path, int bufferSize) {
+    HdfsBlobContainer(BlobPath blobPath, HdfsBlobStore store, Path path, int bufferSize, HdfsSecurityContext hdfsSecurityContext) {
         super(blobPath);
         this.store = store;
+        this.securityContext = hdfsSecurityContext;
         this.path = path;
         this.bufferSize = bufferSize;
     }
@@ -101,7 +108,10 @@ public InputStream readBlob(String blobName) throws IOException {
         return store.execute(new Operation<InputStream>() {
             @Override
             public InputStream run(FileContext fileContext) throws IOException {
-                return fileContext.open(new Path(path, blobName), bufferSize);
+                // FSDataInputStream can open connections on read() or skip() so we wrap in
+                // HDFSPrivilegedInputSteam which will ensure that underlying methods will
+                // be called with the proper privileges.
+                return new HDFSPrivilegedInputSteam(fileContext.open(new Path(path, blobName), bufferSize), securityContext);
             }
         });
     }
@@ -161,4 +171,59 @@ public boolean accept(Path path) {
     public Map<String, BlobMetaData> listBlobs() throws IOException {
         return listBlobsByPrefix(null);
     }
+
+    /**
+     * Exists to wrap underlying InputStream methods that might need to make connections or
+     * perform actions within doPrivileged blocks. The HDFS Client performs a lot underneath
+     * the FSInputStream, including making connections and executing reflection based RPC calls.
+     */
+    private static class HDFSPrivilegedInputSteam extends FilterInputStream {
+
+        private final HdfsSecurityContext securityContext;
+
+        HDFSPrivilegedInputSteam(InputStream in, HdfsSecurityContext hdfsSecurityContext) {
+            super(in);
+            this.securityContext = hdfsSecurityContext;
+        }
+
+        public int read() throws IOException {
+            return doPrivilegedOrThrow(in::read);
+        }
+
+        public int read(byte b[]) throws IOException {
+            return doPrivilegedOrThrow(() -> in.read(b));
+        }
+
+        public int read(byte b[], int off, int len) throws IOException {
+            return doPrivilegedOrThrow(() -> in.read(b, off, len));
+        }
+
+        public long skip(long n) throws IOException {
+            return doPrivilegedOrThrow(() -> in.skip(n));
+        }
+
+        public int available() throws IOException {
+            return doPrivilegedOrThrow(() -> in.available());
+        }
+
+        public synchronized void reset() throws IOException {
+            doPrivilegedOrThrow(() -> {
+                in.reset();
+                return null;
+            });
+        }
+
+        private <T> T doPrivilegedOrThrow(PrivilegedExceptionAction<T> action) throws IOException {
+            SecurityManager sm = System.getSecurityManager();
+            if (sm != null) {
+                // unprivileged code such as scripts do not have SpecialPermission
+                sm.checkPermission(new SpecialPermission());
+            }
+            try {
+                return AccessController.doPrivileged(action, null, securityContext.getRestrictedExecutionPermissions());
+            } catch (PrivilegedActionException e) {
+                throw (IOException) e.getCause();
+            }
+        }
+    }
 }
diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobStore.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobStore.java
index 23404a7c3609b..0556c17d08513 100644
--- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobStore.java
+++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobStore.java
@@ -29,22 +29,21 @@
 import org.elasticsearch.common.blobstore.BlobStore;
 
 import java.io.IOException;
-import java.lang.reflect.ReflectPermission;
 import java.security.AccessController;
 import java.security.PrivilegedActionException;
 import java.security.PrivilegedExceptionAction;
 
-import javax.security.auth.AuthPermission;
-
 final class HdfsBlobStore implements BlobStore {
 
     private final Path root;
     private final FileContext fileContext;
+    private final HdfsSecurityContext securityContext;
     private final int bufferSize;
     private volatile boolean closed;
 
     HdfsBlobStore(FileContext fileContext, String path, int bufferSize) throws IOException {
         this.fileContext = fileContext;
+        this.securityContext = new HdfsSecurityContext(fileContext.getUgi());
         this.bufferSize = bufferSize;
         this.root = execute(new Operation<Path>() {
           @Override
@@ -87,7 +86,7 @@ public String toString() {
 
     @Override
     public BlobContainer blobContainer(BlobPath path) {
-        return new HdfsBlobContainer(path, this, buildHdfsPath(path), bufferSize);
+        return new HdfsBlobContainer(path, this, buildHdfsPath(path), bufferSize, this.securityContext);
     }
 
     private Path buildHdfsPath(BlobPath blobPath) {
@@ -109,7 +108,7 @@ private Path translateToHdfsPath(BlobPath blobPath) {
         }
         return path;
     }
-    
+
     interface Operation<V> {
         V run(FileContext fileContext) throws IOException;
     }
@@ -117,9 +116,6 @@ interface Operation<V> {
     /**
      * Executes the provided operation against this store
      */
-    // we can do FS ops with only two elevated permissions:
-    // 1) hadoop dynamic proxy is messy with access rules
-    // 2) allow hadoop to add credentials to our Subject
     <V> V execute(Operation<V> operation) throws IOException {
         SecurityManager sm = System.getSecurityManager();
         if (sm != null) {
@@ -130,13 +126,13 @@ <V> V execute(Operation<V> operation) throws IOException {
             throw new AlreadyClosedException("HdfsBlobStore is closed: " + this);
         }
         try {
-            return AccessController.doPrivileged(new PrivilegedExceptionAction<V>() {
-                @Override
-                public V run() throws IOException {
+            return AccessController.doPrivileged((PrivilegedExceptionAction<V>)
+                () -> {
+                    securityContext.ensureLogin();
                     return operation.run(fileContext);
-                }
-            }, null, new ReflectPermission("suppressAccessChecks"),
-                     new AuthPermission("modifyPrivateCredentials"));
+                },
+                null,
+                securityContext.getRestrictedExecutionPermissions());
         } catch (PrivilegedActionException pae) {
             throw (IOException) pae.getException();
         }
@@ -146,4 +142,4 @@ public V run() throws IOException {
     public void close() {
         closed = true;
     }
-}
\ No newline at end of file
+}
diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsPlugin.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsPlugin.java
index d4af26c3bbc95..ae0935910f741 100644
--- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsPlugin.java
+++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsPlugin.java
@@ -26,12 +26,15 @@
 import java.util.Collections;
 import java.util.Map;
 
+import org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolPB;
+import org.apache.hadoop.security.KerberosInfo;
+import org.apache.hadoop.security.SecurityUtil;
 import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.RepositoryPlugin;
-import org.elasticsearch.repositories.RepositoriesModule;
 import org.elasticsearch.repositories.Repository;
 
 public final class HdfsPlugin extends Plugin implements RepositoryPlugin {
@@ -42,12 +45,8 @@ public final class HdfsPlugin extends Plugin implements RepositoryPlugin {
         if (sm != null) {
             sm.checkPermission(new SpecialPermission());
         }
-        AccessController.doPrivileged(new PrivilegedAction<Void>() {
-            @Override
-            public Void run() {
-                return evilHadoopInit();
-            }
-        });
+        AccessController.doPrivileged((PrivilegedAction<Void>) HdfsPlugin::evilHadoopInit);
+        AccessController.doPrivileged((PrivilegedAction<Void>) HdfsPlugin::eagerInit);
     }
 
     @SuppressForbidden(reason = "Needs a security hack for hadoop on windows, until HADOOP-XXXX is fixed")
@@ -66,8 +65,6 @@ private static Void evilHadoopInit() {
             Class.forName("org.apache.hadoop.util.StringUtils");
             Class.forName("org.apache.hadoop.util.ShutdownHookManager");
             Class.forName("org.apache.hadoop.conf.Configuration");
-            Class.forName("org.apache.hadoop.hdfs.protocol.HdfsConstants");
-            Class.forName("org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck");
         } catch (ClassNotFoundException | IOException e) {
             throw new RuntimeException(e);
         } finally {
@@ -87,8 +84,36 @@ private static Void evilHadoopInit() {
         return null;
     }
 
+    private static Void eagerInit() {
+        /*
+         * Hadoop RPC wire serialization uses ProtocolBuffers. All proto classes for Hadoop
+         * come annotated with configurations that denote information about if they support
+         * certain security options like Kerberos, and how to send information with the
+         * message to support that authentication method. SecurityUtil creates a service loader
+         * in a static field during its clinit. This loader provides the implementations that
+         * pull the security information for each proto class. The service loader sources its
+         * services from the current thread's context class loader, which must contain the Hadoop
+         * jars. Since plugins don't execute with their class loaders installed as the thread's
+         * context class loader, we need to install the loader briefly, allow the util to be
+         * initialized, then restore the old loader since we don't actually own this thread.
+         */
+        ClassLoader oldCCL = Thread.currentThread().getContextClassLoader();
+        try {
+            Thread.currentThread().setContextClassLoader(HdfsRepository.class.getClassLoader());
+            KerberosInfo info = SecurityUtil.getKerberosInfo(ClientNamenodeProtocolPB.class, null);
+            // Make sure that the correct class loader was installed.
+            if (info == null) {
+                throw new RuntimeException("Could not initialize SecurityUtil: " +
+                    "Unable to find services for [org.apache.hadoop.security.SecurityInfo]");
+            }
+        } finally {
+            Thread.currentThread().setContextClassLoader(oldCCL);
+        }
+        return null;
+    }
+
     @Override
-    public Map<String, Repository.Factory> getRepositories(Environment env) {
-        return Collections.singletonMap("hdfs", (metadata) -> new HdfsRepository(metadata, env));
+    public Map<String, Repository.Factory> getRepositories(Environment env, NamedXContentRegistry namedXContentRegistry) {
+        return Collections.singletonMap("hdfs", (metadata) -> new HdfsRepository(metadata, env, namedXContentRegistry));
     }
 }
diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java
index b111a5d0d0a51..9fd64b79d29a7 100644
--- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java
+++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java
@@ -19,40 +19,48 @@
 package org.elasticsearch.repositories.hdfs;
 
 import java.io.IOException;
-import java.lang.reflect.Constructor;
+import java.io.UncheckedIOException;
+import java.net.InetAddress;
 import java.net.URI;
+import java.net.UnknownHostException;
 import java.security.AccessController;
-import java.security.Principal;
 import java.security.PrivilegedAction;
-import java.util.Collections;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Map.Entry;
 
-import javax.security.auth.Subject;
-
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.AbstractFileSystem;
 import org.apache.hadoop.fs.FileContext;
 import org.apache.hadoop.fs.UnsupportedFileSystemException;
-import org.elasticsearch.ElasticsearchGenerationException;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.cluster.metadata.RepositoryMetaData;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.BlobStore;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
 
 public final class HdfsRepository extends BlobStoreRepository {
 
-    private final BlobPath basePath = BlobPath.cleanPath();
+    private static final Logger LOGGER = Loggers.getLogger(HdfsRepository.class);
+
+    private static final String CONF_SECURITY_PRINCIPAL = "security.principal";
+
+    private final Environment environment;
     private final ByteSizeValue chunkSize;
     private final boolean compress;
+    private final BlobPath basePath = BlobPath.cleanPath();
 
     private HdfsBlobStore blobStore;
 
@@ -60,9 +68,11 @@ public final class HdfsRepository extends BlobStoreRepository {
     // TODO: why 100KB?
     private static final ByteSizeValue DEFAULT_BUFFER_SIZE = new ByteSizeValue(100, ByteSizeUnit.KB);
 
-    public HdfsRepository(RepositoryMetaData metadata, Environment environment) throws IOException {
-        super(metadata, environment.settings());
+    public HdfsRepository(RepositoryMetaData metadata, Environment environment,
+                          NamedXContentRegistry namedXContentRegistry) throws IOException {
+        super(metadata, environment.settings(), namedXContentRegistry);
 
+        this.environment = environment;
         this.chunkSize = metadata.settings().getAsBytesSize("chunk_size", null);
         this.compress = metadata.settings().getAsBoolean("compress", false);
     }
@@ -88,7 +98,7 @@ protected void doStart() {
         if (pathSetting == null) {
             throw new IllegalArgumentException("No 'path' defined for hdfs snapshot/restore");
         }
-        
+
         int bufferSize = getMetadata().settings().getAsBytesSize("buffer_size", DEFAULT_BUFFER_SIZE).bytesAsInt();
 
         try {
@@ -106,50 +116,114 @@ public FileContext run() {
             blobStore = new HdfsBlobStore(fileContext, pathSetting, bufferSize);
             logger.debug("Using file-system [{}] for URI [{}], path [{}]", fileContext.getDefaultFileSystem(), fileContext.getDefaultFileSystem().getUri(), pathSetting);
         } catch (IOException e) {
-            throw new ElasticsearchGenerationException(String.format(Locale.ROOT, "Cannot create HDFS repository for uri [%s]", uri), e);
+            throw new UncheckedIOException(String.format(Locale.ROOT, "Cannot create HDFS repository for uri [%s]", uri), e);
         }
         super.doStart();
     }
-    
+
     // create hadoop filecontext
-    @SuppressForbidden(reason = "lesser of two evils (the other being a bunch of JNI/classloader nightmares)")
-    private static FileContext createContext(URI uri, Settings repositorySettings)  {
-        Configuration cfg = new Configuration(repositorySettings.getAsBoolean("load_defaults", true));
-        cfg.setClassLoader(HdfsRepository.class.getClassLoader());
-        cfg.reloadConfiguration();
+    private FileContext createContext(URI uri, Settings repositorySettings)  {
+        Configuration hadoopConfiguration = new Configuration(repositorySettings.getAsBoolean("load_defaults", true));
+        hadoopConfiguration.setClassLoader(HdfsRepository.class.getClassLoader());
+        hadoopConfiguration.reloadConfiguration();
 
         Map<String, String> map = repositorySettings.getByPrefix("conf.").getAsMap();
         for (Entry<String, String> entry : map.entrySet()) {
-            cfg.set(entry.getKey(), entry.getValue());
+            hadoopConfiguration.set(entry.getKey(), entry.getValue());
+        }
+
+        // Create a hadoop user
+        UserGroupInformation ugi = login(hadoopConfiguration, repositorySettings);
+
+        // Disable FS cache
+        hadoopConfiguration.setBoolean("fs.hdfs.impl.disable.cache", true);
+
+        // Create the filecontext with our user information
+        // This will correctly configure the filecontext to have our UGI as its internal user.
+        return ugi.doAs((PrivilegedAction<FileContext>) () -> {
+            try {
+                AbstractFileSystem fs = AbstractFileSystem.get(uri, hadoopConfiguration);
+                return FileContext.getFileContext(fs, hadoopConfiguration);
+            } catch (UnsupportedFileSystemException e) {
+                throw new UncheckedIOException(e);
+            }
+        });
+    }
+
+    private UserGroupInformation login(Configuration hadoopConfiguration, Settings repositorySettings) {
+        // Validate the authentication method:
+        AuthenticationMethod authMethod = SecurityUtil.getAuthenticationMethod(hadoopConfiguration);
+        if (authMethod.equals(AuthenticationMethod.SIMPLE) == false
+            && authMethod.equals(AuthenticationMethod.KERBEROS) == false) {
+            throw new RuntimeException("Unsupported authorization mode ["+authMethod+"]");
         }
 
-        // create a hadoop user. if we want some auth, it must be done different anyway, and tested.
-        Subject subject;
+        // Check if the user added a principal to use, and that there is a keytab file provided
+        String kerberosPrincipal = repositorySettings.get(CONF_SECURITY_PRINCIPAL);
+
+        // Check to see if the authentication method is compatible
+        if (kerberosPrincipal != null && authMethod.equals(AuthenticationMethod.SIMPLE)) {
+            LOGGER.warn("Hadoop authentication method is set to [SIMPLE], but a Kerberos principal is " +
+                "specified. Continuing with [KERBEROS] authentication.");
+            SecurityUtil.setAuthenticationMethod(AuthenticationMethod.KERBEROS, hadoopConfiguration);
+        } else if (kerberosPrincipal == null && authMethod.equals(AuthenticationMethod.KERBEROS)) {
+            throw new RuntimeException("HDFS Repository does not support [KERBEROS] authentication without " +
+                "a valid Kerberos principal and keytab. Please specify a principal in the repository settings with [" +
+                CONF_SECURITY_PRINCIPAL + "].");
+        }
+
+        // Now we can initialize the UGI with the configuration.
+        UserGroupInformation.setConfiguration(hadoopConfiguration);
+
+        // Debugging
+        LOGGER.debug("Hadoop security enabled: [{}]", UserGroupInformation.isSecurityEnabled());
+        LOGGER.debug("Using Hadoop authentication method: [{}]", SecurityUtil.getAuthenticationMethod(hadoopConfiguration));
+
+        // UserGroupInformation (UGI) instance is just a Hadoop specific wrapper around a Java Subject
         try {
-            Class<?> clazz = Class.forName("org.apache.hadoop.security.User");
-            Constructor<?> ctor = clazz.getConstructor(String.class);
-            ctor.setAccessible(true);
-            Principal principal = (Principal) ctor.newInstance(System.getProperty("user.name"));
-            subject = new Subject(false, Collections.singleton(principal), Collections.emptySet(), Collections.emptySet());
-        } catch (ReflectiveOperationException e) {
-            throw new RuntimeException(e);
+            if (UserGroupInformation.isSecurityEnabled()) {
+                String principal = preparePrincipal(kerberosPrincipal);
+                String keytab = HdfsSecurityContext.locateKeytabFile(environment).toString();
+                LOGGER.debug("Using kerberos principal [{}] and keytab located at [{}]", principal, keytab);
+                return UserGroupInformation.loginUserFromKeytabAndReturnUGI(principal, keytab);
+            }
+            return UserGroupInformation.getCurrentUser();
+        } catch (IOException e) {
+            throw new UncheckedIOException("Could not retrieve the current user information", e);
         }
+    }
 
-        // disable FS cache
-        cfg.setBoolean("fs.hdfs.impl.disable.cache", true);
-
-        // create the filecontext with our user
-        return Subject.doAs(subject, new PrivilegedAction<FileContext>() {
-            @Override
-            public FileContext run() {
-                try {
-                    AbstractFileSystem fs = AbstractFileSystem.get(uri, cfg);
-                    return FileContext.getFileContext(fs, cfg);
-                } catch (UnsupportedFileSystemException e) {
-                    throw new RuntimeException(e);
-                }
+    // Convert principals of the format 'service/_HOST@REALM' by subbing in the local address for '_HOST'.
+    private static String preparePrincipal(String originalPrincipal) {
+        String finalPrincipal = originalPrincipal;
+        // Don't worry about host name resolution if they don't have the _HOST pattern in the name.
+        if (originalPrincipal.contains("_HOST")) {
+            try {
+                finalPrincipal = SecurityUtil.getServerPrincipal(originalPrincipal, getHostName());
+            } catch (IOException e) {
+                throw new UncheckedIOException(e);
             }
-        });
+
+            if (originalPrincipal.equals(finalPrincipal) == false) {
+                LOGGER.debug("Found service principal. Converted original principal name [{}] to server principal [{}]",
+                    originalPrincipal, finalPrincipal);
+            }
+        }
+        return finalPrincipal;
+    }
+
+    @SuppressForbidden(reason = "InetAddress.getLocalHost(); Needed for filling in hostname for a kerberos principal name pattern.")
+    private static String getHostName() {
+        try {
+            /*
+             * This should not block since it should already be resolved via Log4J and Netty. The
+             * host information is cached by the JVM and the TTL for the cache entry is infinite
+             * when the SecurityManager is activated.
+             */
+            return InetAddress.getLocalHost().getCanonicalHostName();
+        } catch (UnknownHostException e) {
+            throw new RuntimeException("Could not locate host information", e);
+        }
     }
 
     @Override
diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsSecurityContext.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsSecurityContext.java
new file mode 100644
index 0000000000000..2a63df6c8bc43
--- /dev/null
+++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsSecurityContext.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.repositories.hdfs;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.lang.reflect.ReflectPermission;
+import java.net.SocketPermission;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.Permission;
+import java.util.Arrays;
+import javax.security.auth.AuthPermission;
+import javax.security.auth.PrivateCredentialPermission;
+import javax.security.auth.kerberos.ServicePermission;
+
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.env.Environment;
+
+/**
+ * Oversees all the security specific logic for the HDFS Repository plugin.
+ *
+ * Keeps track of the current user for a given repository, as well as which
+ * permissions to grant to privileged methods inside the BlobStore.
+ */
+class HdfsSecurityContext {
+
+    private static final Logger LOGGER = Loggers.getLogger(HdfsSecurityContext.class);
+
+    private static final Permission[] SIMPLE_AUTH_PERMISSIONS;
+    private static final Permission[] KERBEROS_AUTH_PERMISSIONS;
+    static {
+        // We can do FS ops with only a few elevated permissions:
+        SIMPLE_AUTH_PERMISSIONS = new Permission[]{
+            new SocketPermission("*", "connect"),
+            // 1) hadoop dynamic proxy is messy with access rules
+            new ReflectPermission("suppressAccessChecks"),
+            // 2) allow hadoop to add credentials to our Subject
+            new AuthPermission("modifyPrivateCredentials"),
+            // 3) RPC Engine requires this for re-establishing pooled connections over the lifetime of the client
+            new PrivateCredentialPermission("org.apache.hadoop.security.Credentials * \"*\"", "read")
+        };
+
+        // If Security is enabled, we need all the following elevated permissions:
+        KERBEROS_AUTH_PERMISSIONS = new Permission[] {
+            new SocketPermission("*", "connect"),
+            // 1) hadoop dynamic proxy is messy with access rules
+            new ReflectPermission("suppressAccessChecks"),
+            // 2) allow hadoop to add credentials to our Subject
+            new AuthPermission("modifyPrivateCredentials"),
+            // 3) allow hadoop to act as the logged in Subject
+            new AuthPermission("doAs"),
+            // 4) Listen and resolve permissions for kerberos server principals
+            new SocketPermission("localhost:0", "listen,resolve"),
+            // We add the following since hadoop requires the client to re-login when the kerberos ticket expires:
+            // 5) All the permissions needed for UGI to do its weird JAAS hack
+            new RuntimePermission("getClassLoader"),
+            new RuntimePermission("setContextClassLoader"),
+            // 6) Additional permissions for the login modules
+            new AuthPermission("modifyPrincipals"),
+            new PrivateCredentialPermission("org.apache.hadoop.security.Credentials * \"*\"", "read"),
+            new PrivateCredentialPermission("javax.security.auth.kerberos.KerberosTicket * \"*\"", "read"),
+            new PrivateCredentialPermission("javax.security.auth.kerberos.KeyTab * \"*\"", "read")
+            // Included later:
+            // 7) allow code to initiate kerberos connections as the logged in user
+            // Still far and away fewer permissions than the original full plugin policy
+        };
+    }
+
+    /**
+     * Locates the keytab file in the environment and verifies that it exists.
+     * Expects keytab file to exist at {@code $CONFIG_DIR$/repository-hdfs/krb5.keytab}
+     */
+    static Path locateKeytabFile(Environment environment) {
+        Path keytabPath = environment.configFile().resolve("repository-hdfs").resolve("krb5.keytab");
+        try {
+            if (Files.exists(keytabPath) == false) {
+                throw new RuntimeException("Could not locate keytab at [" + keytabPath + "].");
+            }
+        } catch (SecurityException se) {
+            throw new RuntimeException("Could not locate keytab at [" + keytabPath + "]", se);
+        }
+        return keytabPath;
+    }
+
+    private final UserGroupInformation ugi;
+    private final Permission[] restrictedExecutionPermissions;
+
+    HdfsSecurityContext(UserGroupInformation ugi) {
+        this.ugi = ugi;
+        this.restrictedExecutionPermissions = renderPermissions(ugi);
+    }
+
+    private Permission[] renderPermissions(UserGroupInformation ugi) {
+        Permission[] permissions;
+        if (ugi.isFromKeytab()) {
+            // KERBEROS
+            // Leave room to append one extra permission based on the logged in user's info.
+            int permlen = KERBEROS_AUTH_PERMISSIONS.length + 1;
+            permissions = new Permission[permlen];
+
+            System.arraycopy(KERBEROS_AUTH_PERMISSIONS, 0, permissions, 0, KERBEROS_AUTH_PERMISSIONS.length);
+
+            // Append a kerberos.ServicePermission to only allow initiating kerberos connections
+            // as the logged in user.
+            permissions[permissions.length - 1] = new ServicePermission(ugi.getUserName(), "initiate");
+        } else {
+            // SIMPLE
+            permissions = Arrays.copyOf(SIMPLE_AUTH_PERMISSIONS, SIMPLE_AUTH_PERMISSIONS.length);
+        }
+        return permissions;
+    }
+
+    Permission[] getRestrictedExecutionPermissions() {
+        return restrictedExecutionPermissions;
+    }
+
+    void ensureLogin() {
+        if (ugi.isFromKeytab()) {
+            try {
+                ugi.checkTGTAndReloginFromKeytab();
+            } catch (IOException ioe) {
+                throw new UncheckedIOException("Could not re-authenticate", ioe);
+            }
+        }
+    }
+}
diff --git a/plugins/repository-hdfs/src/main/plugin-metadata/plugin-security.policy b/plugins/repository-hdfs/src/main/plugin-metadata/plugin-security.policy
index 85447245c96f2..f6476f290bc34 100644
--- a/plugins/repository-hdfs/src/main/plugin-metadata/plugin-security.policy
+++ b/plugins/repository-hdfs/src/main/plugin-metadata/plugin-security.policy
@@ -25,14 +25,60 @@ grant {
   permission java.lang.RuntimePermission "accessDeclaredMembers";
   permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
 
+  // Needed so that Hadoop can load the correct classes for SPI and JAAS
+  // org.apache.hadoop.security.SecurityUtil clinit
+  // org.apache.hadoop.security.UserGroupInformation.newLoginContext()
+  permission java.lang.RuntimePermission "setContextClassLoader";
+
   // org.apache.hadoop.util.StringUtils clinit
   permission java.util.PropertyPermission "*", "read,write";
 
   // org.apache.hadoop.util.ShutdownHookManager clinit
   permission java.lang.RuntimePermission "shutdownHooks";
   
-  // JAAS is used always, we use a fake subject, hurts nobody
+  // JAAS is used by Hadoop for authentication purposes
+  // The Hadoop Login JAAS module modifies a Subject's private credentials and principals
+  // The Hadoop RPC Layer must be able to read these credentials, and initiate Kerberos connections
+
+  // org.apache.hadoop.security.UserGroupInformation.getCurrentUser()
   permission javax.security.auth.AuthPermission "getSubject";
+
+  // org.apache.hadoop.security.UserGroupInformation.doAs()
   permission javax.security.auth.AuthPermission "doAs";
+
+  // org.apache.hadoop.security.UserGroupInformation.getCredentialsInternal()
+  permission javax.security.auth.PrivateCredentialPermission "org.apache.hadoop.security.Credentials * \"*\"", "read";
+
+  // Hadoop depends on the Kerberos login module for kerberos authentication
+  // com.sun.security.auth.module.Krb5LoginModule.login()
+  permission java.lang.RuntimePermission "accessClassInPackage.sun.security.krb5";
+
+  // com.sun.security.auth.module.Krb5LoginModule.commit()
   permission javax.security.auth.AuthPermission "modifyPrivateCredentials";
+  permission javax.security.auth.AuthPermission "modifyPrincipals";
+  permission javax.security.auth.PrivateCredentialPermission "javax.security.auth.kerberos.KeyTab * \"*\"", "read";
+  permission javax.security.auth.PrivateCredentialPermission "javax.security.auth.kerberos.KerberosTicket * \"*\"", "read";
+
+  // Hadoop depends on OS level user information for simple authentication
+  // Unix: UnixLoginModule: com.sun.security.auth.module.UnixSystem.UnixSystem init
+  permission java.lang.RuntimePermission "loadLibrary.jaas_unix";
+  // Windows: NTLoginModule: com.sun.security.auth.module.NTSystem.loadNative
+  permission java.lang.RuntimePermission "loadLibrary.jaas_nt";
+  permission javax.security.auth.AuthPermission "modifyPublicCredentials";
+
+  // org.apache.hadoop.security.SaslRpcServer.init()
+  permission java.security.SecurityPermission "putProviderProperty.SaslPlainServer";
+
+  // org.apache.hadoop.security.SaslPlainServer.SecurityProvider.SecurityProvider init
+  permission java.security.SecurityPermission "insertProvider.SaslPlainServer";
+
+  // org.apache.hadoop.security.SaslRpcClient.getServerPrincipal -> KerberosPrincipal init
+  permission javax.security.auth.kerberos.ServicePermission "*", "initiate";
+
+  // hdfs client opens socket connections for to access repository
+  permission java.net.SocketPermission "*", "connect";
+
+  // client binds to the address returned from the host name of any principal set up as a service principal
+  // org.apache.hadoop.ipc.Client.Connection.setupConnection
+  permission java.net.SocketPermission "localhost:0", "listen,resolve";
 };
diff --git a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java
index cdc6dd968039d..59f3940dbf1e3 100644
--- a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java
+++ b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java
@@ -19,15 +19,6 @@
 
 package org.elasticsearch.repositories.hdfs;
 
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.AbstractFileSystem;
-import org.apache.hadoop.fs.FileContext;
-import org.apache.hadoop.fs.UnsupportedFileSystemException;
-import org.elasticsearch.common.SuppressForbidden;
-import org.elasticsearch.common.blobstore.BlobStore;
-import org.elasticsearch.repositories.ESBlobStoreContainerTestCase;
-
-import javax.security.auth.Subject;
 import java.io.IOException;
 import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
@@ -37,7 +28,18 @@
 import java.security.Principal;
 import java.security.PrivilegedAction;
 import java.util.Collections;
+import javax.security.auth.Subject;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.AbstractFileSystem;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.UnsupportedFileSystemException;
+import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.common.blobstore.BlobStore;
+import org.elasticsearch.repositories.ESBlobStoreContainerTestCase;
 
+@ThreadLeakFilters(filters = {HdfsClientThreadLeakFilter.class})
 public class HdfsBlobStoreContainerTests extends ESBlobStoreContainerTestCase {
 
     @Override
diff --git a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsClientThreadLeakFilter.java b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsClientThreadLeakFilter.java
new file mode 100644
index 0000000000000..5db9635aa8a3a
--- /dev/null
+++ b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsClientThreadLeakFilter.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.repositories.hdfs;
+
+import com.carrotsearch.randomizedtesting.ThreadFilter;
+
+/**
+ * In Hadoop 2.8.0, there is a thread that is started by the filesystem to clean up old execution stats.
+ * This thread ignores all interrupts, catching InterruptedException, logging it, and continuing on
+ * with its work. The thread is a daemon, so it thankfully does not stop the JVM from closing, and it
+ * is started only once in a class's static initialization. This currently breaks our testing as this
+ * thread leaks out of the client and is picked up by the test framework. This thread filter is meant
+ * to ignore the offending thread until a version of Hadoop is released that addresses the incorrect
+ * interrupt handling.
+ *
+ * @see <a href="https://issues.apache.org/jira/browse/HADOOP-12829">https://issues.apache.org/jira/browse/HADOOP-12829</a>
+ * @see "org.apache.hadoop.fs.FileSystem.Statistics.StatisticsDataReferenceCleaner"
+ * @see "org.apache.hadoop.fs.FileSystem.Statistics"
+ */
+public final class HdfsClientThreadLeakFilter implements ThreadFilter {
+
+    private static final String OFFENDING_THREAD_NAME =
+        "org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner";
+
+    @Override
+    public boolean reject(Thread t) {
+        return t.getName().equals(OFFENDING_THREAD_NAME);
+    }
+}
diff --git a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsTests.java b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsTests.java
index a6ed0858474a1..177dbf60c82f1 100644
--- a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsTests.java
+++ b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsTests.java
@@ -23,6 +23,7 @@
 
 import java.util.Collection;
 
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
 import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryResponse;
 import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
 import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
@@ -35,6 +36,7 @@
 import org.elasticsearch.snapshots.SnapshotState;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
+@ThreadLeakFilters(filters = {HdfsClientThreadLeakFilter.class})
 public class HdfsTests extends ESSingleNodeTestCase {
 
     @Override
diff --git a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/RepositoryHdfsClientYamlTestSuiteIT.java b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/RepositoryHdfsClientYamlTestSuiteIT.java
index 1dfbb3c51b786..7293449cc22d9 100644
--- a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/RepositoryHdfsClientYamlTestSuiteIT.java
+++ b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/RepositoryHdfsClientYamlTestSuiteIT.java
@@ -18,14 +18,11 @@
  */
 package org.elasticsearch.repositories.hdfs;
 
-import java.io.IOException;
-
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 public class RepositoryHdfsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
@@ -34,7 +31,7 @@ public RepositoryHdfsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
-        return ESClientYamlSuiteTestCase.createParameters(0, 1);
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/30_snapshot_readonly.yaml b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/30_snapshot_readonly.yaml
new file mode 100644
index 0000000000000..c2a37964e70a7
--- /dev/null
+++ b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/30_snapshot_readonly.yaml
@@ -0,0 +1,29 @@
+# Integration tests for HDFS Repository plugin
+#
+# Tests retrieving information about snapshot
+#
+---
+"Get a snapshot - readonly":
+  # Create repository
+  - do:
+      snapshot.create_repository:
+        repository: test_snapshot_repository_ro
+        body:
+          type: hdfs
+          settings:
+            uri: "hdfs://localhost:9999"
+            path: "/user/elasticsearch/existing/readonly-repository"
+            readonly: true
+
+  # List snapshot info
+  - do:
+      snapshot.get:
+        repository: test_snapshot_repository_ro
+        snapshot: "_all"
+
+  - length: { snapshots: 1 }
+
+  # Remove our repository
+  - do:
+     snapshot.delete_repository:
+       repository: test_snapshot_repository_ro
diff --git a/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/10_basic.yaml b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/10_basic.yaml
new file mode 100644
index 0000000000000..6fbbfc82e872d
--- /dev/null
+++ b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/10_basic.yaml
@@ -0,0 +1,30 @@
+# Integration tests for HDFS Repository plugin
+#
+# Check plugin is installed
+#
+"Plugin loaded":
+    - do:
+        cluster.state: {}
+
+    # Get master node id
+    - set: { master_node: master }
+
+    - do:
+        nodes.info: {}
+
+    - match:  { nodes.$master.plugins.0.name: repository-hdfs  }
+---
+#
+# Check that we can't use file:// repositories or anything like that
+# We only test this plugin against hdfs://
+#
+"HDFS only":
+    - do:
+        catch: /Invalid scheme/
+        snapshot.create_repository:
+          repository: misconfigured_repository
+          body:
+            type: hdfs
+            settings:
+              uri: "file://bogus"
+              path: "foo/bar"
diff --git a/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/20_repository_create.yaml b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/20_repository_create.yaml
new file mode 100644
index 0000000000000..46cc9e92479c5
--- /dev/null
+++ b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/20_repository_create.yaml
@@ -0,0 +1,29 @@
+# Integration tests for HDFS Repository plugin
+#
+# Tests creating a repository
+#
+"HDFS Repository Creation":
+    # Create repository
+    - do:
+        snapshot.create_repository:
+          repository: test_repository_create
+          body:
+            type: hdfs
+            settings:
+              uri: "hdfs://localhost:9998"
+              path: "/user/elasticsearch/test/repository_create"
+              security:
+                principal: "elasticsearch@BUILD.ELASTIC.CO"
+
+    # Get repository
+    - do:
+        snapshot.get_repository:
+          repository: test_repository_create
+
+    - is_true: test_repository_create
+    - match: {test_repository_create.settings.path : "/user/elasticsearch/test/repository_create"}
+
+    # Remove our repository
+    - do:
+       snapshot.delete_repository:
+         repository: test_repository_create
diff --git a/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/20_repository_delete.yaml b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/20_repository_delete.yaml
new file mode 100644
index 0000000000000..7bff67d3d01b0
--- /dev/null
+++ b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/20_repository_delete.yaml
@@ -0,0 +1,54 @@
+# Integration tests for HDFS Repository plugin
+#
+# Tests creating a repository, then deleting it and creating it again.
+#
+"HDFS Delete Repository":
+    # Create repository
+    - do:
+        snapshot.create_repository:
+          repository: test_repo_hdfs_1
+          body:
+            type: hdfs
+            settings:
+              uri: "hdfs://localhost:9998"
+              path: "/user/elasticsearch/foo/bar"
+              security:
+                principal: "elasticsearch@BUILD.ELASTIC.CO"
+
+    # Get repository
+    - do:
+        snapshot.get_repository:
+          repository: test_repo_hdfs_1
+
+    - is_true: test_repo_hdfs_1
+    - match: {test_repo_hdfs_1.settings.path : "/user/elasticsearch/foo/bar"}
+
+    # Delete repository
+    - do:
+        snapshot.delete_repository:
+          repository: test_repo_hdfs_1
+
+    # Get repository: It should be gone
+    - do:
+        catch:    /repository_missing_exception/
+        snapshot.get_repository:
+          repository: test_repo_hdfs_1
+
+    # Create it again
+    - do:
+        snapshot.create_repository:
+          repository: test_repo_hdfs_1
+          body:
+            type: hdfs
+            settings:
+              uri: "hdfs://localhost:9998"
+              path: "/user/elasticsearch/foo/bar"
+              security:
+                principal: "elasticsearch@BUILD.ELASTIC.CO"
+
+    # Get repository again
+    - do:
+        snapshot.get_repository:
+          repository: test_repo_hdfs_1
+
+    - is_true: test_repo_hdfs_1
diff --git a/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/20_repository_verify.yaml b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/20_repository_verify.yaml
new file mode 100644
index 0000000000000..4593c13caee23
--- /dev/null
+++ b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/20_repository_verify.yaml
@@ -0,0 +1,25 @@
+# Integration tests for HDFS Repository plugin
+#
+# Tests explicit verify
+#
+"HDFS Repository Verify":
+    - do:
+        snapshot.create_repository:
+          repository: test_repository_verify
+          body:
+            type: hdfs
+            settings:
+              uri: "hdfs://localhost:9998"
+              path: "/user/elasticsearch/test/repository_verify"
+              security:
+                principal: "elasticsearch@BUILD.ELASTIC.CO"
+
+    # Verify repository
+    - do:
+        snapshot.verify_repository:
+          repository: test_repository_verify
+
+    # Remove our repository
+    - do:
+       snapshot.delete_repository:
+         repository: test_repository_verify
diff --git a/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/30_snapshot.yaml b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/30_snapshot.yaml
new file mode 100644
index 0000000000000..e232782084e16
--- /dev/null
+++ b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/30_snapshot.yaml
@@ -0,0 +1,50 @@
+# Integration tests for HDFS Repository plugin
+#
+# Actually perform a snapshot to hdfs
+#
+---
+"take snapshot":
+  # Create repository
+  - do:
+      snapshot.create_repository:
+        repository: test_snapshot_repository
+        body:
+          type: hdfs
+          settings:
+            uri: "hdfs://localhost:9998"
+            path: "/user/elasticsearch/test/snapshot"
+            security:
+              principal: "elasticsearch@BUILD.ELASTIC.CO"
+
+  # Create index
+  - do:
+      indices.create:
+        index: test_index
+        body:
+          settings:
+            number_of_shards:   1
+            number_of_replicas: 1
+
+  # Create snapshot
+  - do:
+      snapshot.create:
+        repository: test_snapshot_repository
+        snapshot: test_snapshot
+        wait_for_completion: true
+
+  - match: { snapshot.snapshot: test_snapshot }
+  - match: { snapshot.state : SUCCESS }
+  - match: { snapshot.shards.successful: 1 }
+  - match: { snapshot.shards.failed : 0 }
+
+  # Remove our snapshot
+  - do: 
+      snapshot.delete:
+        repository: test_snapshot_repository
+        snapshot: test_snapshot
+
+  # Remove our repository
+  - do:
+     snapshot.delete_repository:
+       repository: test_snapshot_repository
+
diff --git a/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/30_snapshot_get.yaml b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/30_snapshot_get.yaml
new file mode 100644
index 0000000000000..20d988884113f
--- /dev/null
+++ b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/30_snapshot_get.yaml
@@ -0,0 +1,72 @@
+# Integration tests for HDFS Repository plugin
+#
+# Tests retrieving information about snapshot
+#
+---
+"Get a snapshot":
+  # Create repository
+  - do:
+      snapshot.create_repository:
+        repository: test_snapshot_get_repository
+        body:
+          type: hdfs
+          settings:
+            uri: "hdfs://localhost:9998"
+            path: "/user/elasticsearch/test/snapshot_get"
+            security:
+              principal: "elasticsearch@BUILD.ELASTIC.CO"
+
+  # Create index
+  - do:
+      indices.create:
+        index: test_index
+        body:
+          settings:
+            number_of_shards:   1
+            number_of_replicas: 0
+
+  # Wait for green
+  - do:
+      cluster.health:
+        wait_for_status: green
+
+  # Create snapshot
+  - do:
+      snapshot.create:
+        repository: test_snapshot_get_repository
+        snapshot: test_snapshot_get
+        wait_for_completion: true
+
+  - match: { snapshot.snapshot: test_snapshot_get }
+  - match: { snapshot.state : SUCCESS }
+  - match: { snapshot.shards.successful: 1 }
+  - match: { snapshot.shards.failed : 0 }
+
+  # Get snapshot info
+  - do:
+      snapshot.get:
+        repository: test_snapshot_get_repository
+        snapshot: test_snapshot_get
+
+  - length: { snapshots: 1 }
+  - match: { snapshots.0.snapshot : test_snapshot_get }
+
+  # List snapshot info
+  - do:
+      snapshot.get:
+        repository: test_snapshot_get_repository
+        snapshot: "*"
+
+  - length: { snapshots: 1 }
+  - match: { snapshots.0.snapshot : test_snapshot_get }
+
+  # Remove our snapshot
+  - do: 
+      snapshot.delete:
+        repository: test_snapshot_get_repository
+        snapshot: test_snapshot_get
+
+  # Remove our repository
+  - do:
+     snapshot.delete_repository:
+       repository: test_snapshot_get_repository
diff --git a/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/30_snapshot_readonly.yaml b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/30_snapshot_readonly.yaml
new file mode 100644
index 0000000000000..8c4c0347a156a
--- /dev/null
+++ b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/30_snapshot_readonly.yaml
@@ -0,0 +1,31 @@
+# Integration tests for HDFS Repository plugin
+#
+# Tests retrieving information about snapshot
+#
+---
+"Get a snapshot - readonly":
+  # Create repository
+  - do:
+      snapshot.create_repository:
+        repository: test_snapshot_repository_ro
+        body:
+          type: hdfs
+          settings:
+            uri: "hdfs://localhost:9998"
+            path: "/user/elasticsearch/existing/readonly-repository"
+            security:
+              principal: "elasticsearch@BUILD.ELASTIC.CO"
+            readonly: true
+
+  # List snapshot info
+  - do:
+      snapshot.get:
+        repository: test_snapshot_repository_ro
+        snapshot: "_all"
+
+  - length: { snapshots: 1 }
+
+  # Remove our repository
+  - do:
+     snapshot.delete_repository:
+       repository: test_snapshot_repository_ro
diff --git a/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/40_restore.yaml b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/40_restore.yaml
new file mode 100644
index 0000000000000..9807a0b46b10e
--- /dev/null
+++ b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/40_restore.yaml
@@ -0,0 +1,81 @@
+# Integration tests for HDFS Repository plugin
+#
+# Actually perform a snapshot to hdfs, then restore it
+#
+---
+"Create a snapshot and then restore it":
+
+  # Create repository
+  - do:
+      snapshot.create_repository:
+        repository: test_restore_repository
+        body:
+          type: hdfs
+          settings:
+            uri: "hdfs://localhost:9998"
+            path: "/user/elasticsearch/test/restore"
+            security:
+              principal: "elasticsearch@BUILD.ELASTIC.CO"
+
+  # Create index
+  - do:
+      indices.create:
+        index: test_index
+        body:
+          settings:
+            number_of_shards:   1
+            number_of_replicas: 0
+
+  # Wait for green
+  - do:
+      cluster.health:
+        wait_for_status: green
+
+  # Take snapshot
+  - do:
+      snapshot.create:
+        repository: test_restore_repository
+        snapshot: test_restore
+        wait_for_completion: true
+
+  - match: { snapshot.snapshot: test_restore }
+  - match: { snapshot.state : SUCCESS }
+  - match: { snapshot.shards.successful: 1 }
+  - match: { snapshot.shards.failed : 0 }
+  - is_true: snapshot.version
+  - gt: { snapshot.version_id: 0}
+
+  # Close index
+  - do:
+      indices.close:
+        index : test_index
+
+  # Restore index
+  - do:
+      snapshot.restore:
+        repository: test_restore_repository
+        snapshot: test_restore
+        wait_for_completion: true
+
+  # Check recovery stats
+  - do:
+      indices.recovery:
+        index: test_index
+
+  - match: { test_index.shards.0.type: SNAPSHOT }
+  - match: { test_index.shards.0.stage: DONE }
+  - match: { test_index.shards.0.index.files.recovered: 0}
+  - match: { test_index.shards.0.index.size.recovered_in_bytes: 0}
+  - match: { test_index.shards.0.index.files.reused: 1}
+  - gt: { test_index.shards.0.index.size.reused_in_bytes: 0}
+
+  # Remove our snapshot
+  - do:
+      snapshot.delete:
+        repository: test_restore_repository
+        snapshot: test_restore
+
+  # Remove our repository
+  - do:
+     snapshot.delete_repository:
+       repository: test_restore_repository
diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle
index b136990867076..debd582f2d6c1 100644
--- a/plugins/repository-s3/build.gradle
+++ b/plugins/repository-s3/build.gradle
@@ -19,7 +19,7 @@
 
 esplugin {
   description 'The S3 repository plugin adds S3 repositories'
-  classname 'org.elasticsearch.plugin.repository.s3.S3RepositoryPlugin'
+  classname 'org.elasticsearch.repositories.s3.S3RepositoryPlugin'
 }
 
 versions << [
@@ -54,9 +54,26 @@ bundlePlugin {
   }
 }
 
+additionalTest('testEnvCreds'){
+  include '**/EnvironmentCredentialsTests.class'
+  environment 'AWS_ACCESS_KEY_ID', 'env_access'
+  environment 'AWS_SECRET_ACCESS_KEY', 'env_secret'
+}
+
+additionalTest('testSyspropCreds'){
+  include '**/SyspropCredentialsTests.class'
+  systemProperty 'aws.accessKeyId', 'sysprop_access'
+  systemProperty 'aws.secretKey', 'sysprop_secret'
+}
+
 test {
-  // this is needed for insecure plugins, remove if possible!
-  systemProperty 'tests.artifact', project.name
+  // these are tested explicitly in separate test tasks
+  exclude '**/*CredentialsTests.class'
+}
+
+integTestCluster {
+  keystoreSetting 's3.client.default.access_key', 'myaccesskey'
+  keystoreSetting 's3.client.default.secret_key', 'mysecretkey'
 }
 
 thirdPartyAudit.excludes = [
@@ -66,111 +83,117 @@ thirdPartyAudit.excludes = [
   'org.apache.avalon.framework.logger.Logger', 
   'org.apache.log.Hierarchy', 
   'org.apache.log.Logger',
-
-  // jarhell with jdk (intentionally, because jaxb was removed from default modules in java 9)
-  'javax.xml.bind.Binder',
-  'javax.xml.bind.ContextFinder$1',
-  'javax.xml.bind.ContextFinder',
-  'javax.xml.bind.DataBindingException',
-  'javax.xml.bind.DatatypeConverter',
-  'javax.xml.bind.DatatypeConverterImpl$CalendarFormatter',
-  'javax.xml.bind.DatatypeConverterImpl',
-  'javax.xml.bind.DatatypeConverterInterface',
-  'javax.xml.bind.Element',
-  'javax.xml.bind.GetPropertyAction',
-  'javax.xml.bind.JAXB$Cache',
-  'javax.xml.bind.JAXB',
-  'javax.xml.bind.JAXBContext',
-  'javax.xml.bind.JAXBElement$GlobalScope',
-  'javax.xml.bind.JAXBElement',
-  'javax.xml.bind.JAXBException',
-  'javax.xml.bind.JAXBIntrospector',
-  'javax.xml.bind.JAXBPermission',
-  'javax.xml.bind.MarshalException',
-  'javax.xml.bind.Marshaller$Listener',
-  'javax.xml.bind.Marshaller',
-  'javax.xml.bind.Messages',
-  'javax.xml.bind.NotIdentifiableEvent',
-  'javax.xml.bind.ParseConversionEvent',
-  'javax.xml.bind.PrintConversionEvent',
-  'javax.xml.bind.PropertyException',
-  'javax.xml.bind.SchemaOutputResolver',
-  'javax.xml.bind.TypeConstraintException',
-  'javax.xml.bind.UnmarshalException',
-  'javax.xml.bind.Unmarshaller$Listener',
-  'javax.xml.bind.Unmarshaller',
-  'javax.xml.bind.UnmarshallerHandler',
-  'javax.xml.bind.ValidationEvent',
-  'javax.xml.bind.ValidationEventHandler',
-  'javax.xml.bind.ValidationEventLocator',
-  'javax.xml.bind.ValidationException',
-  'javax.xml.bind.Validator',
-  'javax.xml.bind.WhiteSpaceProcessor',
-  'javax.xml.bind.annotation.DomHandler',
-  'javax.xml.bind.annotation.W3CDomHandler',
-  'javax.xml.bind.annotation.XmlAccessOrder',
-  'javax.xml.bind.annotation.XmlAccessType',
-  'javax.xml.bind.annotation.XmlAccessorOrder',
-  'javax.xml.bind.annotation.XmlAccessorType',
-  'javax.xml.bind.annotation.XmlAnyAttribute',
-  'javax.xml.bind.annotation.XmlAnyElement',
-  'javax.xml.bind.annotation.XmlAttachmentRef',
-  'javax.xml.bind.annotation.XmlAttribute',
-  'javax.xml.bind.annotation.XmlElement$DEFAULT',
-  'javax.xml.bind.annotation.XmlElement',
-  'javax.xml.bind.annotation.XmlElementDecl$GLOBAL',
-  'javax.xml.bind.annotation.XmlElementDecl',
-  'javax.xml.bind.annotation.XmlElementRef$DEFAULT',
-  'javax.xml.bind.annotation.XmlElementRef',
-  'javax.xml.bind.annotation.XmlElementRefs',
-  'javax.xml.bind.annotation.XmlElementWrapper',
-  'javax.xml.bind.annotation.XmlElements',
-  'javax.xml.bind.annotation.XmlEnum',
-  'javax.xml.bind.annotation.XmlEnumValue',
-  'javax.xml.bind.annotation.XmlID',
-  'javax.xml.bind.annotation.XmlIDREF',
-  'javax.xml.bind.annotation.XmlInlineBinaryData',
-  'javax.xml.bind.annotation.XmlList',
-  'javax.xml.bind.annotation.XmlMimeType',
-  'javax.xml.bind.annotation.XmlMixed',
-  'javax.xml.bind.annotation.XmlNs',
-  'javax.xml.bind.annotation.XmlNsForm',
-  'javax.xml.bind.annotation.XmlRegistry',
-  'javax.xml.bind.annotation.XmlRootElement',
-  'javax.xml.bind.annotation.XmlSchema',
-  'javax.xml.bind.annotation.XmlSchemaType$DEFAULT',
-  'javax.xml.bind.annotation.XmlSchemaType',
-  'javax.xml.bind.annotation.XmlSchemaTypes',
-  'javax.xml.bind.annotation.XmlSeeAlso',
-  'javax.xml.bind.annotation.XmlTransient',
-  'javax.xml.bind.annotation.XmlType$DEFAULT',
-  'javax.xml.bind.annotation.XmlType',
-  'javax.xml.bind.annotation.XmlValue',
-  'javax.xml.bind.annotation.adapters.CollapsedStringAdapter',
-  'javax.xml.bind.annotation.adapters.HexBinaryAdapter',
-  'javax.xml.bind.annotation.adapters.NormalizedStringAdapter',
-  'javax.xml.bind.annotation.adapters.XmlAdapter',
-  'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter$DEFAULT',
-  'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter',
-  'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapters',
-  'javax.xml.bind.attachment.AttachmentMarshaller',
-  'javax.xml.bind.attachment.AttachmentUnmarshaller',
-  'javax.xml.bind.helpers.AbstractMarshallerImpl',
-  'javax.xml.bind.helpers.AbstractUnmarshallerImpl',
-  'javax.xml.bind.helpers.DefaultValidationEventHandler',
-  'javax.xml.bind.helpers.Messages',
-  'javax.xml.bind.helpers.NotIdentifiableEventImpl',
-  'javax.xml.bind.helpers.ParseConversionEventImpl',
-  'javax.xml.bind.helpers.PrintConversionEventImpl',
-  'javax.xml.bind.helpers.ValidationEventImpl',
-  'javax.xml.bind.helpers.ValidationEventLocatorImpl',
-  'javax.xml.bind.util.JAXBResult',
-  'javax.xml.bind.util.JAXBSource$1',
-  'javax.xml.bind.util.JAXBSource',
-  'javax.xml.bind.util.Messages',
-  'javax.xml.bind.util.ValidationEventCollector'
 ]
 
+// jarhell with jdk (intentionally, because jaxb was removed from default modules in java 9)
+if (JavaVersion.current() <= JavaVersion.VERSION_1_8) {
+  thirdPartyAudit.excludes += [
+    'javax.xml.bind.Binder',
+    'javax.xml.bind.ContextFinder$1',
+    'javax.xml.bind.ContextFinder',
+    'javax.xml.bind.DataBindingException',
+    'javax.xml.bind.DatatypeConverter',
+    'javax.xml.bind.DatatypeConverterImpl$CalendarFormatter',
+    'javax.xml.bind.DatatypeConverterImpl',
+    'javax.xml.bind.DatatypeConverterInterface',
+    'javax.xml.bind.Element',
+    'javax.xml.bind.GetPropertyAction',
+    'javax.xml.bind.JAXB$Cache',
+    'javax.xml.bind.JAXB',
+    'javax.xml.bind.JAXBContext',
+    'javax.xml.bind.JAXBElement$GlobalScope',
+    'javax.xml.bind.JAXBElement',
+    'javax.xml.bind.JAXBException',
+    'javax.xml.bind.JAXBIntrospector',
+    'javax.xml.bind.JAXBPermission',
+    'javax.xml.bind.MarshalException',
+    'javax.xml.bind.Marshaller$Listener',
+    'javax.xml.bind.Marshaller',
+    'javax.xml.bind.Messages',
+    'javax.xml.bind.NotIdentifiableEvent',
+    'javax.xml.bind.ParseConversionEvent',
+    'javax.xml.bind.PrintConversionEvent',
+    'javax.xml.bind.PropertyException',
+    'javax.xml.bind.SchemaOutputResolver',
+    'javax.xml.bind.TypeConstraintException',
+    'javax.xml.bind.UnmarshalException',
+    'javax.xml.bind.Unmarshaller$Listener',
+    'javax.xml.bind.Unmarshaller',
+    'javax.xml.bind.UnmarshallerHandler',
+    'javax.xml.bind.ValidationEvent',
+    'javax.xml.bind.ValidationEventHandler',
+    'javax.xml.bind.ValidationEventLocator',
+    'javax.xml.bind.ValidationException',
+    'javax.xml.bind.Validator',
+    'javax.xml.bind.WhiteSpaceProcessor',
+    'javax.xml.bind.annotation.DomHandler',
+    'javax.xml.bind.annotation.W3CDomHandler',
+    'javax.xml.bind.annotation.XmlAccessOrder',
+    'javax.xml.bind.annotation.XmlAccessType',
+    'javax.xml.bind.annotation.XmlAccessorOrder',
+    'javax.xml.bind.annotation.XmlAccessorType',
+    'javax.xml.bind.annotation.XmlAnyAttribute',
+    'javax.xml.bind.annotation.XmlAnyElement',
+    'javax.xml.bind.annotation.XmlAttachmentRef',
+    'javax.xml.bind.annotation.XmlAttribute',
+    'javax.xml.bind.annotation.XmlElement$DEFAULT',
+    'javax.xml.bind.annotation.XmlElement',
+    'javax.xml.bind.annotation.XmlElementDecl$GLOBAL',
+    'javax.xml.bind.annotation.XmlElementDecl',
+    'javax.xml.bind.annotation.XmlElementRef$DEFAULT',
+    'javax.xml.bind.annotation.XmlElementRef',
+    'javax.xml.bind.annotation.XmlElementRefs',
+    'javax.xml.bind.annotation.XmlElementWrapper',
+    'javax.xml.bind.annotation.XmlElements',
+    'javax.xml.bind.annotation.XmlEnum',
+    'javax.xml.bind.annotation.XmlEnumValue',
+    'javax.xml.bind.annotation.XmlID',
+    'javax.xml.bind.annotation.XmlIDREF',
+    'javax.xml.bind.annotation.XmlInlineBinaryData',
+    'javax.xml.bind.annotation.XmlList',
+    'javax.xml.bind.annotation.XmlMimeType',
+    'javax.xml.bind.annotation.XmlMixed',
+    'javax.xml.bind.annotation.XmlNs',
+    'javax.xml.bind.annotation.XmlNsForm',
+    'javax.xml.bind.annotation.XmlRegistry',
+    'javax.xml.bind.annotation.XmlRootElement',
+    'javax.xml.bind.annotation.XmlSchema',
+    'javax.xml.bind.annotation.XmlSchemaType$DEFAULT',
+    'javax.xml.bind.annotation.XmlSchemaType',
+    'javax.xml.bind.annotation.XmlSchemaTypes',
+    'javax.xml.bind.annotation.XmlSeeAlso',
+    'javax.xml.bind.annotation.XmlTransient',
+    'javax.xml.bind.annotation.XmlType$DEFAULT',
+    'javax.xml.bind.annotation.XmlType',
+    'javax.xml.bind.annotation.XmlValue',
+    'javax.xml.bind.annotation.adapters.CollapsedStringAdapter',
+    'javax.xml.bind.annotation.adapters.HexBinaryAdapter',
+    'javax.xml.bind.annotation.adapters.NormalizedStringAdapter',
+    'javax.xml.bind.annotation.adapters.XmlAdapter',
+    'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter$DEFAULT',
+    'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter',
+    'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapters',
+    'javax.xml.bind.attachment.AttachmentMarshaller',
+    'javax.xml.bind.attachment.AttachmentUnmarshaller',
+    'javax.xml.bind.helpers.AbstractMarshallerImpl',
+    'javax.xml.bind.helpers.AbstractUnmarshallerImpl',
+    'javax.xml.bind.helpers.DefaultValidationEventHandler',
+    'javax.xml.bind.helpers.Messages',
+    'javax.xml.bind.helpers.NotIdentifiableEventImpl',
+    'javax.xml.bind.helpers.ParseConversionEventImpl',
+    'javax.xml.bind.helpers.PrintConversionEventImpl',
+    'javax.xml.bind.helpers.ValidationEventImpl',
+    'javax.xml.bind.helpers.ValidationEventLocatorImpl',
+    'javax.xml.bind.util.JAXBResult',
+    'javax.xml.bind.util.JAXBSource$1',
+    'javax.xml.bind.util.JAXBSource',
+    'javax.xml.bind.util.Messages',
+    'javax.xml.bind.util.ValidationEventCollector'
+  ]
+} else {
+  thirdPartyAudit.excludes += ['javax.activation.DataHandler']
+}
+
 // AWS SDK is exposing some deprecated methods which we call using a delegate:
 // * setObjectRedirectLocation(String bucketName, String key, String newRedirectLocation)
 // * changeObjectStorageClass(String bucketName, String key, StorageClass newStorageClass)
diff --git a/plugins/repository-s3/config/repository-s3/log4j2.properties b/plugins/repository-s3/config/repository-s3/log4j2.properties
index 3fee57ce3e2c7..aa52f0232e009 100644
--- a/plugins/repository-s3/config/repository-s3/log4j2.properties
+++ b/plugins/repository-s3/config/repository-s3/log4j2.properties
@@ -4,5 +4,5 @@ logger.com_amazonaws.level = warn
 logger.com_amazonaws_jmx_SdkMBeanRegistrySupport.name = com.amazonaws.jmx.SdkMBeanRegistrySupport
 logger.com_amazonaws_jmx_SdkMBeanRegistrySupport.level = error
 
-logger_com_amazonaws_metrics_AwsSdkMetrics.name = com.amazonaws.metrics.AwsSdkMetrics
-logger_com_amazonaws_metrics_AwsSdkMetrics.level = error
+logger.com_amazonaws_metrics_AwsSdkMetrics.name = com.amazonaws.metrics.AwsSdkMetrics
+logger.com_amazonaws_metrics_AwsSdkMetrics.level = error
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java
deleted file mode 100644
index 59c3d3445a241..0000000000000
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.cloud.aws;
-
-import com.amazonaws.Protocol;
-import com.amazonaws.services.s3.AmazonS3;
-import org.elasticsearch.common.component.LifecycleComponent;
-import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Setting.Property;
-import org.elasticsearch.common.settings.Settings;
-
-import java.util.Locale;
-import java.util.function.Function;
-
-/**
- *
- */
-public interface AwsS3Service extends LifecycleComponent {
-
-    // Global AWS settings (shared between discovery-ec2 and repository-s3)
-    // Each setting starting with `cloud.aws` also exists in discovery-ec2 project. Don't forget to update
-    // the code there if you change anything here.
-    /**
-     * cloud.aws.access_key: AWS Access key. Shared with discovery-ec2 plugin
-     */
-    Setting<String> KEY_SETTING =
-        Setting.simpleString("cloud.aws.access_key", Property.NodeScope, Property.Filtered, Property.Shared);
-    /**
-     * cloud.aws.secret_key: AWS Secret key. Shared with discovery-ec2 plugin
-     */
-    Setting<String> SECRET_SETTING =
-        Setting.simpleString("cloud.aws.secret_key", Property.NodeScope, Property.Filtered, Property.Shared);
-    /**
-     * cloud.aws.protocol: Protocol for AWS API: http or https. Defaults to https. Shared with discovery-ec2 plugin
-     */
-    Setting<Protocol> PROTOCOL_SETTING = new Setting<>("cloud.aws.protocol", "https", s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)),
-        Property.NodeScope, Property.Shared);
-    /**
-     * cloud.aws.proxy.host: In case of proxy, define its hostname/IP. Shared with discovery-ec2 plugin
-     */
-    Setting<String> PROXY_HOST_SETTING = Setting.simpleString("cloud.aws.proxy.host", Property.NodeScope, Property.Shared);
-    /**
-     * cloud.aws.proxy.port: In case of proxy, define its port. Defaults to 80. Shared with discovery-ec2 plugin
-     */
-    Setting<Integer> PROXY_PORT_SETTING = Setting.intSetting("cloud.aws.proxy.port", 80, 0, 1<<16, Property.NodeScope,
-        Property.Shared);
-    /**
-     * cloud.aws.proxy.username: In case of proxy with auth, define the username. Shared with discovery-ec2 plugin
-     */
-    Setting<String> PROXY_USERNAME_SETTING = Setting.simpleString("cloud.aws.proxy.username", Property.NodeScope, Property.Shared);
-    /**
-     * cloud.aws.proxy.password: In case of proxy with auth, define the password. Shared with discovery-ec2 plugin
-     */
-    Setting<String> PROXY_PASSWORD_SETTING =
-        Setting.simpleString("cloud.aws.proxy.password", Property.NodeScope, Property.Filtered, Property.Shared);
-    /**
-     * cloud.aws.signer: If you are using an old AWS API version, you can define a Signer. Shared with discovery-ec2 plugin
-     */
-    Setting<String> SIGNER_SETTING = Setting.simpleString("cloud.aws.signer", Property.NodeScope, Property.Shared);
-    /**
-     * cloud.aws.region: Region. Shared with discovery-ec2 plugin
-     */
-    Setting<String> REGION_SETTING =
-        new Setting<>("cloud.aws.region", "", s -> s.toLowerCase(Locale.ROOT), Property.NodeScope, Property.Shared);
-
-    /**
-     * Defines specific s3 settings starting with cloud.aws.s3.
-     */
-    interface CLOUD_S3 {
-        /**
-         * cloud.aws.s3.access_key: AWS Access key specific for S3 API calls. Defaults to cloud.aws.access_key.
-         * @see AwsS3Service#KEY_SETTING
-         */
-        Setting<String> KEY_SETTING =
-            new Setting<>("cloud.aws.s3.access_key", AwsS3Service.KEY_SETTING, Function.identity(),
-                Property.NodeScope, Property.Filtered);
-        /**
-         * cloud.aws.s3.secret_key: AWS Secret key specific for S3 API calls. Defaults to cloud.aws.secret_key.
-         * @see AwsS3Service#SECRET_SETTING
-         */
-        Setting<String> SECRET_SETTING =
-            new Setting<>("cloud.aws.s3.secret_key", AwsS3Service.SECRET_SETTING, Function.identity(),
-                Property.NodeScope, Property.Filtered);
-        /**
-         * cloud.aws.s3.protocol: Protocol for AWS API specific for S3 API calls: http or https. Defaults to cloud.aws.protocol.
-         * @see AwsS3Service#PROTOCOL_SETTING
-         */
-        Setting<Protocol> PROTOCOL_SETTING =
-            new Setting<>("cloud.aws.s3.protocol", AwsS3Service.PROTOCOL_SETTING, s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)),
-                Property.NodeScope);
-        /**
-         * cloud.aws.s3.proxy.host: In case of proxy, define its hostname/IP specific for S3 API calls. Defaults to cloud.aws.proxy.host.
-         * @see AwsS3Service#PROXY_HOST_SETTING
-         */
-        Setting<String> PROXY_HOST_SETTING =
-            new Setting<>("cloud.aws.s3.proxy.host", AwsS3Service.PROXY_HOST_SETTING, Function.identity(),
-                Property.NodeScope);
-        /**
-         * cloud.aws.s3.proxy.port: In case of proxy, define its port specific for S3 API calls.  Defaults to cloud.aws.proxy.port.
-         * @see AwsS3Service#PROXY_PORT_SETTING
-         */
-        Setting<Integer> PROXY_PORT_SETTING =
-            new Setting<>("cloud.aws.s3.proxy.port", AwsS3Service.PROXY_PORT_SETTING,
-                s -> Setting.parseInt(s, 0, 1<<16, "cloud.aws.s3.proxy.port"), Property.NodeScope);
-        /**
-         * cloud.aws.s3.proxy.username: In case of proxy with auth, define the username specific for S3 API calls.
-         * Defaults to cloud.aws.proxy.username.
-         * @see AwsS3Service#PROXY_USERNAME_SETTING
-         */
-        Setting<String> PROXY_USERNAME_SETTING =
-            new Setting<>("cloud.aws.s3.proxy.username", AwsS3Service.PROXY_USERNAME_SETTING, Function.identity(),
-                Property.NodeScope);
-        /**
-         * cloud.aws.s3.proxy.password: In case of proxy with auth, define the password specific for S3 API calls.
-         * Defaults to cloud.aws.proxy.password.
-         * @see AwsS3Service#PROXY_PASSWORD_SETTING
-         */
-        Setting<String> PROXY_PASSWORD_SETTING =
-            new Setting<>("cloud.aws.s3.proxy.password", AwsS3Service.PROXY_PASSWORD_SETTING, Function.identity(),
-                Property.NodeScope, Property.Filtered);
-        /**
-         * cloud.aws.s3.signer: If you are using an old AWS API version, you can define a Signer. Specific for S3 API calls.
-         * Defaults to cloud.aws.signer.
-         * @see AwsS3Service#SIGNER_SETTING
-         */
-        Setting<String> SIGNER_SETTING =
-            new Setting<>("cloud.aws.s3.signer", AwsS3Service.SIGNER_SETTING, Function.identity(), Property.NodeScope);
-        /**
-         * cloud.aws.s3.region: Region specific for S3 API calls. Defaults to cloud.aws.region.
-         * @see AwsS3Service#REGION_SETTING
-         */
-        Setting<String> REGION_SETTING =
-            new Setting<>("cloud.aws.s3.region", AwsS3Service.REGION_SETTING, s -> s.toLowerCase(Locale.ROOT),
-                Property.NodeScope);
-        /**
-         * cloud.aws.s3.endpoint: Endpoint. If not set, endpoint will be guessed based on region setting.
-         */
-        Setting<String> ENDPOINT_SETTING = Setting.simpleString("cloud.aws.s3.endpoint", Property.NodeScope);
-    }
-
-    AmazonS3 client(Settings repositorySettings, String endpoint, Protocol protocol, String region, Integer maxRetries,
-                    boolean useThrottleRetries, Boolean pathStyleAccess);
-}
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsSigner.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsSigner.java
deleted file mode 100644
index c1c36031b5efb..0000000000000
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsSigner.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.cloud.aws;
-
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.auth.SignerFactory;
-import org.apache.logging.log4j.Logger;
-import org.elasticsearch.common.logging.Loggers;
-
-public class AwsSigner {
-
-    private static final Logger logger = Loggers.getLogger(AwsSigner.class);
-
-    private AwsSigner() {
-
-    }
-
-    protected static void validateSignerType(String signer, String endpoint) {
-        if (signer == null) {
-            throw new IllegalArgumentException("[null] signer set");
-        }
-
-        // do not block user to any signerType
-        switch (signer) {
-            case "S3SignerType":
-                if (endpoint.equals("s3.cn-north-1.amazonaws.com.cn") || endpoint.equals("s3.eu-central-1.amazonaws.com")) {
-                    throw new IllegalArgumentException("[S3SignerType] may not be supported in aws Beijing and Frankfurt region");
-                }
-                break;
-            case "AWSS3V4SignerType":
-                break;
-            default:
-                try {
-                    SignerFactory.getSignerByTypeAndService(signer, null);
-                } catch (IllegalArgumentException e) {
-                    throw new IllegalArgumentException("[" + signer + "] may not be supported");
-                }
-        }
-    }
-
-    /**
-     * Add a AWS API Signer.
-     * @param signer Signer to use
-     * @param configuration AWS Client configuration
-     */
-    public static void configureSigner(String signer, ClientConfiguration configuration, String endpoint) {
-        try {
-            validateSignerType(signer, endpoint);
-        } catch (IllegalArgumentException e) {
-            logger.warn("{}", e.getMessage());
-        }
-
-        configuration.setSignerOverride(signer);
-    }
-
-}
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java
deleted file mode 100644
index a9091788f285c..0000000000000
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java
+++ /dev/null
@@ -1,216 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.cloud.aws;
-
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.Protocol;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.BasicAWSCredentials;
-import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
-import com.amazonaws.http.IdleConnectionReaper;
-import com.amazonaws.internal.StaticCredentialsProvider;
-import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.AmazonS3Client;
-import com.amazonaws.services.s3.S3ClientOptions;
-import org.apache.logging.log4j.Logger;
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.component.AbstractLifecycleComponent;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.repositories.s3.S3Repository;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.elasticsearch.repositories.s3.S3Repository.getValue;
-
-/**
- *
- */
-public class InternalAwsS3Service extends AbstractLifecycleComponent implements AwsS3Service {
-
-    /**
-     * (acceskey, endpoint) -&gt; client
-     */
-    private Map<Tuple<String, String>, AmazonS3Client> clients = new HashMap<>();
-
-    public InternalAwsS3Service(Settings settings) {
-        super(settings);
-    }
-
-    @Override
-    public synchronized AmazonS3 client(Settings repositorySettings, String endpoint, Protocol protocol, String region, Integer maxRetries,
-                                        boolean useThrottleRetries, Boolean pathStyleAccess) {
-        String foundEndpoint = findEndpoint(logger, settings, endpoint, region);
-
-        AWSCredentialsProvider credentials = buildCredentials(logger, settings, repositorySettings);
-
-        Tuple<String, String> clientDescriptor = new Tuple<>(foundEndpoint, credentials.getCredentials().getAWSAccessKeyId());
-        AmazonS3Client client = clients.get(clientDescriptor);
-        if (client != null) {
-            return client;
-        }
-
-        client = new AmazonS3Client(
-            credentials,
-            buildConfiguration(logger, settings, protocol, maxRetries, foundEndpoint, useThrottleRetries));
-
-        if (pathStyleAccess != null) {
-            client.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(pathStyleAccess));
-        }
-
-        if (!foundEndpoint.isEmpty()) {
-            client.setEndpoint(foundEndpoint);
-        }
-
-        clients.put(clientDescriptor, client);
-        return client;
-    }
-
-    public static ClientConfiguration buildConfiguration(Logger logger, Settings settings, Protocol protocol, Integer maxRetries,
-                                                         String endpoint, boolean useThrottleRetries) {
-        ClientConfiguration clientConfiguration = new ClientConfiguration();
-        // the response metadata cache is only there for diagnostics purposes,
-        // but can force objects from every response to the old generation.
-        clientConfiguration.setResponseMetadataCacheSize(0);
-        clientConfiguration.setProtocol(protocol);
-
-        String proxyHost = CLOUD_S3.PROXY_HOST_SETTING.get(settings);
-        if (Strings.hasText(proxyHost)) {
-            Integer proxyPort = CLOUD_S3.PROXY_PORT_SETTING.get(settings);
-            String proxyUsername = CLOUD_S3.PROXY_USERNAME_SETTING.get(settings);
-            String proxyPassword = CLOUD_S3.PROXY_PASSWORD_SETTING.get(settings);
-
-            clientConfiguration
-                .withProxyHost(proxyHost)
-                .withProxyPort(proxyPort)
-                .withProxyUsername(proxyUsername)
-                .withProxyPassword(proxyPassword);
-        }
-
-        if (maxRetries != null) {
-            // If not explicitly set, default to 3 with exponential backoff policy
-            clientConfiguration.setMaxErrorRetry(maxRetries);
-        }
-        clientConfiguration.setUseThrottleRetries(useThrottleRetries);
-
-        // #155: we might have 3rd party users using older S3 API version
-        String awsSigner = CLOUD_S3.SIGNER_SETTING.get(settings);
-        if (Strings.hasText(awsSigner)) {
-            logger.debug("using AWS API signer [{}]", awsSigner);
-            AwsSigner.configureSigner(awsSigner, clientConfiguration, endpoint);
-        }
-
-        return clientConfiguration;
-    }
-
-    public static AWSCredentialsProvider buildCredentials(Logger logger, Settings settings, Settings repositorySettings) {
-        AWSCredentialsProvider credentials;
-        String key = getValue(repositorySettings, settings,
-            S3Repository.Repository.KEY_SETTING, S3Repository.Repositories.KEY_SETTING);
-        String secret = getValue(repositorySettings, settings,
-            S3Repository.Repository.SECRET_SETTING, S3Repository.Repositories.SECRET_SETTING);
-
-        if (key.isEmpty() && secret.isEmpty()) {
-            logger.debug("Using either environment variables, system properties or instance profile credentials");
-            credentials = new DefaultAWSCredentialsProviderChain();
-        } else {
-            logger.debug("Using basic key/secret credentials");
-            credentials = new StaticCredentialsProvider(new BasicAWSCredentials(key, secret));
-        }
-
-        return credentials;
-    }
-
-    protected static String findEndpoint(Logger logger, Settings settings, String endpoint, String region) {
-        if (Strings.isNullOrEmpty(endpoint)) {
-            logger.debug("no repository level endpoint has been defined. Trying to guess from repository region [{}]", region);
-            if (!region.isEmpty()) {
-                endpoint = getEndpoint(region);
-                logger.debug("using s3 region [{}], with endpoint [{}]", region, endpoint);
-            } else {
-                // No region has been set so we will use the default endpoint
-                if (CLOUD_S3.ENDPOINT_SETTING.exists(settings)) {
-                    endpoint = CLOUD_S3.ENDPOINT_SETTING.get(settings);
-                    logger.debug("using explicit s3 endpoint [{}]", endpoint);
-                } else if (REGION_SETTING.exists(settings) || CLOUD_S3.REGION_SETTING.exists(settings)) {
-                    region = CLOUD_S3.REGION_SETTING.get(settings);
-                    endpoint = getEndpoint(region);
-                    logger.debug("using s3 region [{}], with endpoint [{}]", region, endpoint);
-                }
-            }
-        } else {
-            logger.debug("using repository level endpoint [{}]", endpoint);
-        }
-
-        return endpoint;
-    }
-
-    private static String getEndpoint(String region) {
-        if ("us-east".equals(region) || "us-east-1".equals(region)) {
-            return "s3.amazonaws.com";
-        } else if ("us-west".equals(region) || "us-west-1".equals(region)) {
-            return "s3-us-west-1.amazonaws.com";
-        } else if ("us-west-2".equals(region)) {
-            return "s3-us-west-2.amazonaws.com";
-        } else if (region.equals("ap-south-1")) {
-            return "s3-ap-south-1.amazonaws.com";
-        } else if ("ap-southeast".equals(region) || "ap-southeast-1".equals(region)) {
-            return "s3-ap-southeast-1.amazonaws.com";
-        } else if ("ap-southeast-2".equals(region)) {
-            return "s3-ap-southeast-2.amazonaws.com";
-        } else if ("ap-northeast".equals(region) || "ap-northeast-1".equals(region)) {
-            return "s3-ap-northeast-1.amazonaws.com";
-        } else if ("ap-northeast-2".equals(region)) {
-            return "s3-ap-northeast-2.amazonaws.com";
-        } else if ("eu-west".equals(region) || "eu-west-1".equals(region)) {
-            return "s3-eu-west-1.amazonaws.com";
-        } else if ("eu-central".equals(region) || "eu-central-1".equals(region)) {
-            return "s3.eu-central-1.amazonaws.com";
-        } else if ("sa-east".equals(region) || "sa-east-1".equals(region)) {
-            return "s3-sa-east-1.amazonaws.com";
-        } else if ("cn-north".equals(region) || "cn-north-1".equals(region)) {
-            return "s3.cn-north-1.amazonaws.com.cn";
-        } else if ("us-gov-west".equals(region) || "us-gov-west-1".equals(region)) {
-            return "s3-us-gov-west-1.amazonaws.com";
-        } else {
-            throw new IllegalArgumentException("No automatic endpoint could be derived from region [" + region + "]");
-        }
-    }
-
-    @Override
-    protected void doStart() throws ElasticsearchException {
-    }
-
-    @Override
-    protected void doStop() throws ElasticsearchException {
-    }
-
-    @Override
-    protected void doClose() throws ElasticsearchException {
-        for (AmazonS3Client client : clients.values()) {
-            client.shutdown();
-        }
-
-        // Ensure that IdleConnectionReaper is shutdown
-        IdleConnectionReaper.shutdown();
-    }
-}
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/DefaultS3OutputStream.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/DefaultS3OutputStream.java
deleted file mode 100644
index ef9b25b2d1e25..0000000000000
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/DefaultS3OutputStream.java
+++ /dev/null
@@ -1,270 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.cloud.aws.blobstore;
-
-import com.amazonaws.AmazonClientException;
-import com.amazonaws.services.s3.model.AbortMultipartUploadRequest;
-import com.amazonaws.services.s3.model.AmazonS3Exception;
-import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
-import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
-import com.amazonaws.services.s3.model.ObjectMetadata;
-import com.amazonaws.services.s3.model.PartETag;
-import com.amazonaws.services.s3.model.PutObjectRequest;
-import com.amazonaws.services.s3.model.PutObjectResult;
-import com.amazonaws.services.s3.model.UploadPartRequest;
-import com.amazonaws.services.s3.model.UploadPartResult;
-import com.amazonaws.util.Base64;
-import org.apache.logging.log4j.Logger;
-import org.elasticsearch.common.logging.Loggers;
-import org.elasticsearch.common.unit.ByteSizeUnit;
-import org.elasticsearch.common.unit.ByteSizeValue;
-
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.security.DigestInputStream;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * DefaultS3OutputStream uploads data to the AWS S3 service using 2 modes: single and multi part.
- * <p>
- * When the length of the chunk is lower than buffer_size, the chunk is uploaded with a single request.
- * Otherwise multiple requests are made, each of buffer_size (except the last one which can be lower than buffer_size).
- * <p>
- * Quick facts about S3:
- * <p>
- * Maximum object size:                 5 TB
- * Maximum number of parts per upload:  10,000
- * Part numbers:                        1 to 10,000 (inclusive)
- * Part size:                           5 MB to 5 GB, last part can be &lt; 5 MB
- * <p>
- * See http://docs.aws.amazon.com/AmazonS3/latest/dev/qfacts.html
- * See http://docs.aws.amazon.com/AmazonS3/latest/dev/uploadobjusingmpu.html
- */
-public class DefaultS3OutputStream extends S3OutputStream {
-
-    private static final ByteSizeValue MULTIPART_MAX_SIZE = new ByteSizeValue(5, ByteSizeUnit.GB);
-    private static final Logger logger = Loggers.getLogger("cloud.aws");
-    /**
-     * Multipart Upload API data
-     */
-    private String multipartId;
-    private int multipartChunks;
-    private List<PartETag> multiparts;
-
-    public DefaultS3OutputStream(S3BlobStore blobStore, String bucketName, String blobName, int bufferSizeInBytes, int numberOfRetries, boolean serverSideEncryption) {
-        super(blobStore, bucketName, blobName, bufferSizeInBytes, numberOfRetries, serverSideEncryption);
-    }
-
-    @Override
-    public void flush(byte[] bytes, int off, int len, boolean closing) throws IOException {
-        if (len > MULTIPART_MAX_SIZE.getBytes()) {
-            throw new IOException("Unable to upload files larger than " + MULTIPART_MAX_SIZE + " to Amazon S3");
-        }
-
-        if (!closing) {
-            if (len < getBufferSize()) {
-                upload(bytes, off, len);
-            } else {
-                if (getFlushCount() == 0) {
-                    initializeMultipart();
-                }
-                uploadMultipart(bytes, off, len, false);
-            }
-        } else {
-            if (multipartId != null) {
-                uploadMultipart(bytes, off, len, true);
-                completeMultipart();
-            } else {
-                upload(bytes, off, len);
-            }
-        }
-    }
-
-    /**
-     * Upload data using a single request.
-     */
-    private void upload(byte[] bytes, int off, int len) throws IOException {
-        try (ByteArrayInputStream is = new ByteArrayInputStream(bytes, off, len)) {
-            int retry = 0;
-            while (retry <= getNumberOfRetries()) {
-                try {
-                    doUpload(getBlobStore(), getBucketName(), getBlobName(), is, len, isServerSideEncryption());
-                    break;
-                } catch (AmazonClientException e) {
-                    if (getBlobStore().shouldRetry(e) && retry < getNumberOfRetries()) {
-                        is.reset();
-                        retry++;
-                    } else {
-                        throw new IOException("Unable to upload object " + getBlobName(), e);
-                    }
-                }
-            }
-        }
-    }
-
-    protected void doUpload(S3BlobStore blobStore, String bucketName, String blobName, InputStream is, int length,
-            boolean serverSideEncryption) throws AmazonS3Exception {
-        ObjectMetadata md = new ObjectMetadata();
-        if (serverSideEncryption) {
-            md.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
-        }
-        md.setContentLength(length);
-
-        InputStream inputStream = is;
-
-        // We try to compute a MD5 while reading it
-        MessageDigest messageDigest;
-        try {
-            messageDigest = MessageDigest.getInstance("MD5");
-            inputStream = new DigestInputStream(is, messageDigest);
-        } catch (NoSuchAlgorithmException impossible) {
-            // Every implementation of the Java platform is required to support MD5 (see MessageDigest)
-            throw new RuntimeException(impossible);
-        }
-
-        PutObjectRequest putRequest = new PutObjectRequest(bucketName, blobName, inputStream, md)
-                .withStorageClass(blobStore.getStorageClass())
-                .withCannedAcl(blobStore.getCannedACL());
-        PutObjectResult putObjectResult = blobStore.client().putObject(putRequest);
-
-        String localMd5 = Base64.encodeAsString(messageDigest.digest());
-        String remoteMd5 = putObjectResult.getContentMd5();
-        if (!localMd5.equals(remoteMd5)) {
-            logger.debug("MD5 local [{}], remote [{}] are not equal...", localMd5, remoteMd5);
-            throw new AmazonS3Exception("MD5 local [" + localMd5 +
-                    "], remote [" + remoteMd5 +
-                    "] are not equal...");
-        }
-    }
-
-    private void initializeMultipart() {
-        int retry = 0;
-        while ((retry <= getNumberOfRetries()) && (multipartId == null)) {
-            try {
-                multipartId = doInitialize(getBlobStore(), getBucketName(), getBlobName(), isServerSideEncryption());
-                if (multipartId != null) {
-                    multipartChunks = 1;
-                    multiparts = new ArrayList<>();
-                }
-            } catch (AmazonClientException e) {
-                if (getBlobStore().shouldRetry(e) && retry < getNumberOfRetries()) {
-                    retry++;
-                } else {
-                    throw e;
-                }
-            }
-        }
-    }
-
-    protected String doInitialize(S3BlobStore blobStore, String bucketName, String blobName, boolean serverSideEncryption) {
-        InitiateMultipartUploadRequest request = new InitiateMultipartUploadRequest(bucketName, blobName)
-                .withCannedACL(blobStore.getCannedACL())
-                .withStorageClass(blobStore.getStorageClass());
-
-        if (serverSideEncryption) {
-            ObjectMetadata md = new ObjectMetadata();
-            md.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
-            request.setObjectMetadata(md);
-        }
-
-        return blobStore.client().initiateMultipartUpload(request).getUploadId();
-    }
-
-    private void uploadMultipart(byte[] bytes, int off, int len, boolean lastPart) throws IOException {
-        try (ByteArrayInputStream is = new ByteArrayInputStream(bytes, off, len)) {
-            int retry = 0;
-            while (retry <= getNumberOfRetries()) {
-                try {
-                    PartETag partETag = doUploadMultipart(getBlobStore(), getBucketName(), getBlobName(), multipartId, is, len, lastPart);
-                    multiparts.add(partETag);
-                    multipartChunks++;
-                    return;
-                } catch (AmazonClientException e) {
-                    if (getBlobStore().shouldRetry(e) && retry < getNumberOfRetries()) {
-                        is.reset();
-                        retry++;
-                    } else {
-                        abortMultipart();
-                        throw e;
-                    }
-                }
-            }
-        }
-    }
-
-    protected PartETag doUploadMultipart(S3BlobStore blobStore, String bucketName, String blobName, String uploadId, InputStream is,
-            int length, boolean lastPart) throws AmazonS3Exception {
-        UploadPartRequest request = new UploadPartRequest()
-        .withBucketName(bucketName)
-        .withKey(blobName)
-        .withUploadId(uploadId)
-        .withPartNumber(multipartChunks)
-        .withInputStream(is)
-        .withPartSize(length)
-        .withLastPart(lastPart);
-
-        UploadPartResult response = blobStore.client().uploadPart(request);
-        return response.getPartETag();
-
-    }
-
-    private void completeMultipart() {
-        int retry = 0;
-        while (retry <= getNumberOfRetries()) {
-            try {
-                doCompleteMultipart(getBlobStore(), getBucketName(), getBlobName(), multipartId, multiparts);
-                multipartId = null;
-                return;
-            } catch (AmazonClientException e) {
-                if (getBlobStore().shouldRetry(e) && retry < getNumberOfRetries()) {
-                    retry++;
-                } else {
-                    abortMultipart();
-                    throw e;
-                }
-            }
-        }
-    }
-
-    protected void doCompleteMultipart(S3BlobStore blobStore, String bucketName, String blobName, String uploadId, List<PartETag> parts)
-            throws AmazonS3Exception {
-        CompleteMultipartUploadRequest request = new CompleteMultipartUploadRequest(bucketName, blobName, uploadId, parts);
-        blobStore.client().completeMultipartUpload(request);
-    }
-
-    private void abortMultipart() {
-        if (multipartId != null) {
-            try {
-                doAbortMultipart(getBlobStore(), getBucketName(), getBlobName(), multipartId);
-            } finally {
-                multipartId = null;
-            }
-        }
-    }
-
-    protected void doAbortMultipart(S3BlobStore blobStore, String bucketName, String blobName, String uploadId)
-            throws AmazonS3Exception {
-        blobStore.client().abortMultipartUpload(new AbortMultipartUploadRequest(bucketName, blobName, uploadId));
-    }
-}
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java b/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java
deleted file mode 100644
index f549b5c39493f..0000000000000
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.plugin.repository.s3;
-
-import java.security.AccessController;
-import java.security.PrivilegedAction;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
-import org.elasticsearch.SpecialPermission;
-import org.elasticsearch.cloud.aws.AwsS3Service;
-import org.elasticsearch.cloud.aws.InternalAwsS3Service;
-import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.plugins.RepositoryPlugin;
-import org.elasticsearch.repositories.Repository;
-import org.elasticsearch.repositories.s3.S3Repository;
-
-/**
- * A plugin to add a repository type that writes to and from the AWS S3.
- */
-public class S3RepositoryPlugin extends Plugin implements RepositoryPlugin {
-
-    // ClientConfiguration clinit has some classloader problems
-    // TODO: fix that
-    static {
-        SecurityManager sm = System.getSecurityManager();
-        if (sm != null) {
-            sm.checkPermission(new SpecialPermission());
-        }
-        AccessController.doPrivileged(new PrivilegedAction<Void>() {
-            @Override
-            public Void run() {
-                try {
-                    Class.forName("com.amazonaws.ClientConfiguration");
-                } catch (ClassNotFoundException e) {
-                    throw new RuntimeException(e);
-                }
-                return null;
-            }
-        });
-    }
-
-    // overridable for tests
-    protected AwsS3Service createStorageService(Settings settings) {
-        return new InternalAwsS3Service(settings);
-    }
-
-    @Override
-    public Map<String, Repository.Factory> getRepositories(Environment env) {
-        return Collections.singletonMap(S3Repository.TYPE,
-            (metadata) -> new S3Repository(metadata, env.settings(), createStorageService(env.settings())));
-    }
-
-    @Override
-    public List<String> getSettingsFilter() {
-        return Arrays.asList(
-            S3Repository.Repository.KEY_SETTING.getKey(),
-            S3Repository.Repository.SECRET_SETTING.getKey());
-    }
-
-    @Override
-    public List<Setting<?>> getSettings() {
-        return Arrays.asList(
-        // Register global cloud aws settings: cloud.aws (might have been registered in ec2 plugin)
-        AwsS3Service.KEY_SETTING,
-        AwsS3Service.SECRET_SETTING,
-        AwsS3Service.PROTOCOL_SETTING,
-        AwsS3Service.PROXY_HOST_SETTING,
-        AwsS3Service.PROXY_PORT_SETTING,
-        AwsS3Service.PROXY_USERNAME_SETTING,
-        AwsS3Service.PROXY_PASSWORD_SETTING,
-        AwsS3Service.SIGNER_SETTING,
-        AwsS3Service.REGION_SETTING,
-
-        // Register S3 specific settings: cloud.aws.s3
-        AwsS3Service.CLOUD_S3.KEY_SETTING,
-        AwsS3Service.CLOUD_S3.SECRET_SETTING,
-        AwsS3Service.CLOUD_S3.PROTOCOL_SETTING,
-        AwsS3Service.CLOUD_S3.PROXY_HOST_SETTING,
-        AwsS3Service.CLOUD_S3.PROXY_PORT_SETTING,
-        AwsS3Service.CLOUD_S3.PROXY_USERNAME_SETTING,
-        AwsS3Service.CLOUD_S3.PROXY_PASSWORD_SETTING,
-        AwsS3Service.CLOUD_S3.SIGNER_SETTING,
-        AwsS3Service.CLOUD_S3.REGION_SETTING,
-        AwsS3Service.CLOUD_S3.ENDPOINT_SETTING,
-
-        // Register S3 repositories settings: repositories.s3
-        S3Repository.Repositories.KEY_SETTING,
-        S3Repository.Repositories.SECRET_SETTING,
-        S3Repository.Repositories.BUCKET_SETTING,
-        S3Repository.Repositories.REGION_SETTING,
-        S3Repository.Repositories.ENDPOINT_SETTING,
-        S3Repository.Repositories.PROTOCOL_SETTING,
-        S3Repository.Repositories.SERVER_SIDE_ENCRYPTION_SETTING,
-        S3Repository.Repositories.BUFFER_SIZE_SETTING,
-        S3Repository.Repositories.MAX_RETRIES_SETTING,
-        S3Repository.Repositories.CHUNK_SIZE_SETTING,
-        S3Repository.Repositories.COMPRESS_SETTING,
-        S3Repository.Repositories.STORAGE_CLASS_SETTING,
-        S3Repository.Repositories.CANNED_ACL_SETTING,
-        S3Repository.Repositories.BASE_PATH_SETTING,
-        S3Repository.Repositories.USE_THROTTLE_RETRIES_SETTING,
-        S3Repository.Repositories.PATH_STYLE_ACCESS_SETTING);
-    }
-}
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/AwsS3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/AwsS3Service.java
new file mode 100644
index 0000000000000..bfffc76a52e01
--- /dev/null
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/AwsS3Service.java
@@ -0,0 +1,179 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.repositories.s3;
+
+import java.util.Locale;
+import java.util.function.Function;
+
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.Protocol;
+import com.amazonaws.services.s3.AmazonS3;
+import org.elasticsearch.cluster.metadata.RepositoryMetaData;
+import org.elasticsearch.common.component.LifecycleComponent;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+
+interface AwsS3Service extends LifecycleComponent {
+
+    // Legacy global AWS settings (shared between discovery-ec2 and repository-s3)
+    // Each setting starting with `cloud.aws` also exists in discovery-ec2 project. Don't forget to update
+    // the code there if you change anything here.
+    /**
+     * cloud.aws.access_key: AWS Access key. Shared with discovery-ec2 plugin
+     */
+    Setting<SecureString> KEY_SETTING = new Setting<>("cloud.aws.access_key", "", SecureString::new,
+        Property.NodeScope, Property.Filtered, Property.Deprecated, Property.Shared);
+    /**
+     * cloud.aws.secret_key: AWS Secret key. Shared with discovery-ec2 plugin
+     */
+    Setting<SecureString> SECRET_SETTING = new Setting<>("cloud.aws.secret_key", "", SecureString::new,
+        Property.NodeScope, Property.Filtered, Property.Deprecated, Property.Shared);
+    /**
+     * cloud.aws.protocol: Protocol for AWS API: http or https. Defaults to https. Shared with discovery-ec2 plugin
+     */
+    Setting<Protocol> PROTOCOL_SETTING = new Setting<>("cloud.aws.protocol", "https",
+        s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)), Property.NodeScope, Property.Deprecated, Property.Shared);
+    /**
+     * cloud.aws.proxy.host: In case of proxy, define its hostname/IP. Shared with discovery-ec2 plugin
+     */
+    Setting<String> PROXY_HOST_SETTING = Setting.simpleString("cloud.aws.proxy.host",
+        Property.NodeScope, Property.Deprecated, Property.Shared);
+    /**
+     * cloud.aws.proxy.port: In case of proxy, define its port. Defaults to 80. Shared with discovery-ec2 plugin
+     */
+    Setting<Integer> PROXY_PORT_SETTING = Setting.intSetting("cloud.aws.proxy.port", 80, 0, 1<<16,
+        Property.NodeScope, Property.Deprecated, Property.Shared);
+    /**
+     * cloud.aws.proxy.username: In case of proxy with auth, define the username. Shared with discovery-ec2 plugin
+     */
+    Setting<SecureString> PROXY_USERNAME_SETTING = new Setting<>("cloud.aws.proxy.username", "", SecureString::new,
+        Property.NodeScope, Property.Deprecated, Property.Shared);
+    /**
+     * cloud.aws.proxy.password: In case of proxy with auth, define the password. Shared with discovery-ec2 plugin
+     */
+    Setting<SecureString> PROXY_PASSWORD_SETTING = new Setting<>("cloud.aws.proxy.password", "", SecureString::new,
+        Property.NodeScope, Property.Filtered, Property.Deprecated, Property.Shared);
+    /**
+     * cloud.aws.signer: If you are using an old AWS API version, you can define a Signer. Shared with discovery-ec2 plugin
+     */
+    Setting<String> SIGNER_SETTING = Setting.simpleString("cloud.aws.signer",
+        Property.NodeScope, Property.Deprecated, Property.Shared);
+    /**
+     * cloud.aws.region: Region. Shared with discovery-ec2 plugin
+     */
+    Setting<String> REGION_SETTING = new Setting<>("cloud.aws.region", "", s -> s.toLowerCase(Locale.ROOT),
+        Property.NodeScope, Property.Deprecated, Property.Shared);
+    /**
+     * cloud.aws.read_timeout: Socket read timeout. Shared with discovery-ec2 plugin
+     */
+    Setting<TimeValue> READ_TIMEOUT = Setting.timeSetting("cloud.aws.read_timeout",
+        TimeValue.timeValueMillis(ClientConfiguration.DEFAULT_SOCKET_TIMEOUT), Property.NodeScope, Property.Deprecated, Property.Shared);
+
+    /**
+     * Defines specific s3 settings starting with cloud.aws.s3.
+     * NOTE: These are legacy settings. Use the named client configs in {@link org.elasticsearch.repositories.s3.S3Repository}.
+     */
+    interface CLOUD_S3 {
+        /**
+         * cloud.aws.s3.access_key: AWS Access key specific for S3 API calls. Defaults to cloud.aws.access_key.
+         * @see AwsS3Service#KEY_SETTING
+         */
+        Setting<SecureString> KEY_SETTING =
+            new Setting<>("cloud.aws.s3.access_key", AwsS3Service.KEY_SETTING, SecureString::new,
+                Property.NodeScope, Property.Filtered, Property.Deprecated);
+        /**
+         * cloud.aws.s3.secret_key: AWS Secret key specific for S3 API calls. Defaults to cloud.aws.secret_key.
+         * @see AwsS3Service#SECRET_SETTING
+         */
+        Setting<SecureString> SECRET_SETTING =
+            new Setting<>("cloud.aws.s3.secret_key", AwsS3Service.SECRET_SETTING, SecureString::new,
+                Property.NodeScope, Property.Filtered, Property.Deprecated);
+        /**
+         * cloud.aws.s3.protocol: Protocol for AWS API specific for S3 API calls: http or https. Defaults to cloud.aws.protocol.
+         * @see AwsS3Service#PROTOCOL_SETTING
+         */
+        Setting<Protocol> PROTOCOL_SETTING =
+            new Setting<>("cloud.aws.s3.protocol", AwsS3Service.PROTOCOL_SETTING, s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)),
+                Property.NodeScope, Property.Deprecated);
+        /**
+         * cloud.aws.s3.proxy.host: In case of proxy, define its hostname/IP specific for S3 API calls. Defaults to cloud.aws.proxy.host.
+         * @see AwsS3Service#PROXY_HOST_SETTING
+         */
+        Setting<String> PROXY_HOST_SETTING =
+            new Setting<>("cloud.aws.s3.proxy.host", AwsS3Service.PROXY_HOST_SETTING, Function.identity(),
+                Property.NodeScope, Property.Deprecated);
+        /**
+         * cloud.aws.s3.proxy.port: In case of proxy, define its port specific for S3 API calls.  Defaults to cloud.aws.proxy.port.
+         * @see AwsS3Service#PROXY_PORT_SETTING
+         */
+        Setting<Integer> PROXY_PORT_SETTING =
+            new Setting<>("cloud.aws.s3.proxy.port", AwsS3Service.PROXY_PORT_SETTING,
+                s -> Setting.parseInt(s, 0, 1<<16, "cloud.aws.s3.proxy.port"), Property.NodeScope, Property.Deprecated);
+        /**
+         * cloud.aws.s3.proxy.username: In case of proxy with auth, define the username specific for S3 API calls.
+         * Defaults to cloud.aws.proxy.username.
+         * @see AwsS3Service#PROXY_USERNAME_SETTING
+         */
+        Setting<SecureString> PROXY_USERNAME_SETTING =
+            new Setting<>("cloud.aws.s3.proxy.username", AwsS3Service.PROXY_USERNAME_SETTING, SecureString::new,
+                Property.NodeScope, Property.Deprecated);
+        /**
+         * cloud.aws.s3.proxy.password: In case of proxy with auth, define the password specific for S3 API calls.
+         * Defaults to cloud.aws.proxy.password.
+         * @see AwsS3Service#PROXY_PASSWORD_SETTING
+         */
+        Setting<SecureString> PROXY_PASSWORD_SETTING =
+            new Setting<>("cloud.aws.s3.proxy.password", AwsS3Service.PROXY_PASSWORD_SETTING, SecureString::new,
+                Property.NodeScope, Property.Filtered, Property.Deprecated);
+        /**
+         * cloud.aws.s3.signer: If you are using an old AWS API version, you can define a Signer. Specific for S3 API calls.
+         * Defaults to cloud.aws.signer.
+         * @see AwsS3Service#SIGNER_SETTING
+         */
+        Setting<String> SIGNER_SETTING =
+            new Setting<>("cloud.aws.s3.signer", AwsS3Service.SIGNER_SETTING, Function.identity(),
+                Property.NodeScope, Property.Deprecated);
+        /**
+         * cloud.aws.s3.region: Region specific for S3 API calls. Defaults to cloud.aws.region.
+         * @see AwsS3Service#REGION_SETTING
+         */
+        Setting<String> REGION_SETTING =
+            new Setting<>("cloud.aws.s3.region", AwsS3Service.REGION_SETTING, s -> s.toLowerCase(Locale.ROOT),
+                Property.NodeScope, Property.Deprecated);
+        /**
+         * cloud.aws.s3.endpoint: Endpoint. If not set, endpoint will be guessed based on region setting.
+         */
+        Setting<String> ENDPOINT_SETTING = Setting.simpleString("cloud.aws.s3.endpoint", Property.NodeScope);
+        /**
+         * cloud.aws.s3.read_timeout: Socket read timeout. Defaults to cloud.aws.read_timeout
+         * @see AwsS3Service#READ_TIMEOUT
+         */
+        Setting<TimeValue> READ_TIMEOUT =
+            Setting.timeSetting("cloud.aws.s3.read_timeout", AwsS3Service.READ_TIMEOUT, Property.NodeScope, Property.Deprecated);
+    }
+
+    /**
+     * Creates an {@code AmazonS3} client from the given repository metadata and node settings.
+     */
+    AmazonS3 client(Settings repositorySettings);
+}
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/AwsSigner.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/AwsSigner.java
new file mode 100644
index 0000000000000..61cdb2a7b43f7
--- /dev/null
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/AwsSigner.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.repositories.s3;
+
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.auth.SignerFactory;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.logging.Loggers;
+
+class AwsSigner {
+
+    private static final Logger logger = Loggers.getLogger(AwsSigner.class);
+
+    private AwsSigner() {
+
+    }
+
+    protected static void validateSignerType(String signer, String endpoint) {
+        if (signer == null) {
+            throw new IllegalArgumentException("[null] signer set");
+        }
+
+        // do not block user to any signerType
+        switch (signer) {
+            case "S3SignerType":
+                if (endpoint.equals("s3.cn-north-1.amazonaws.com.cn") || endpoint.equals("s3.eu-central-1.amazonaws.com")) {
+                    throw new IllegalArgumentException("[S3SignerType] may not be supported in aws Beijing and Frankfurt region");
+                }
+                break;
+            case "AWSS3V4SignerType":
+                break;
+            default:
+                try {
+                    SignerFactory.getSignerByTypeAndService(signer, null);
+                } catch (IllegalArgumentException e) {
+                    throw new IllegalArgumentException("[" + signer + "] may not be supported");
+                }
+        }
+    }
+
+    /**
+     * Add a AWS API Signer.
+     * @param signer Signer to use
+     * @param configuration AWS Client configuration
+     */
+    public static void configureSigner(String signer, ClientConfiguration configuration, String endpoint) {
+        try {
+            validateSignerType(signer, endpoint);
+        } catch (IllegalArgumentException e) {
+            logger.warn("{}", e.getMessage());
+        }
+
+        configuration.setSignerOverride(signer);
+    }
+
+}
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/DefaultS3OutputStream.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/DefaultS3OutputStream.java
new file mode 100644
index 0000000000000..d6bba2eea81a5
--- /dev/null
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/DefaultS3OutputStream.java
@@ -0,0 +1,216 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.repositories.s3;
+
+import com.amazonaws.AmazonClientException;
+import com.amazonaws.services.s3.model.AbortMultipartUploadRequest;
+import com.amazonaws.services.s3.model.AmazonS3Exception;
+import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
+import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
+import com.amazonaws.services.s3.model.ObjectMetadata;
+import com.amazonaws.services.s3.model.PartETag;
+import com.amazonaws.services.s3.model.PutObjectRequest;
+import com.amazonaws.services.s3.model.PutObjectResult;
+import com.amazonaws.services.s3.model.UploadPartRequest;
+import com.amazonaws.services.s3.model.UploadPartResult;
+import com.amazonaws.util.Base64;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.security.DigestInputStream;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * DefaultS3OutputStream uploads data to the AWS S3 service using 2 modes: single and multi part.
+ * <p>
+ * When the length of the chunk is lower than buffer_size, the chunk is uploaded with a single request.
+ * Otherwise multiple requests are made, each of buffer_size (except the last one which can be lower than buffer_size).
+ * <p>
+ * Quick facts about S3:
+ * <p>
+ * Maximum object size:                 5 TB
+ * Maximum number of parts per upload:  10,000
+ * Part numbers:                        1 to 10,000 (inclusive)
+ * Part size:                           5 MB to 5 GB, last part can be &lt; 5 MB
+ * <p>
+ * See http://docs.aws.amazon.com/AmazonS3/latest/dev/qfacts.html
+ * See http://docs.aws.amazon.com/AmazonS3/latest/dev/uploadobjusingmpu.html
+ */
+class DefaultS3OutputStream extends S3OutputStream {
+
+    private static final ByteSizeValue MULTIPART_MAX_SIZE = new ByteSizeValue(5, ByteSizeUnit.GB);
+    private static final Logger logger = Loggers.getLogger("cloud.aws");
+    /**
+     * Multipart Upload API data
+     */
+    private String multipartId;
+    private int multipartChunks;
+    private List<PartETag> multiparts;
+
+    DefaultS3OutputStream(S3BlobStore blobStore, String bucketName, String blobName, int bufferSizeInBytes, boolean serverSideEncryption) {
+        super(blobStore, bucketName, blobName, bufferSizeInBytes, serverSideEncryption);
+    }
+
+    @Override
+    public void flush(byte[] bytes, int off, int len, boolean closing) throws IOException {
+        if (len > MULTIPART_MAX_SIZE.getBytes()) {
+            throw new IOException("Unable to upload files larger than " + MULTIPART_MAX_SIZE + " to Amazon S3");
+        }
+
+        if (!closing) {
+            if (len < getBufferSize()) {
+                upload(bytes, off, len);
+            } else {
+                if (getFlushCount() == 0) {
+                    initializeMultipart();
+                }
+                uploadMultipart(bytes, off, len, false);
+            }
+        } else {
+            if (multipartId != null) {
+                uploadMultipart(bytes, off, len, true);
+                completeMultipart();
+            } else {
+                upload(bytes, off, len);
+            }
+        }
+    }
+
+    /**
+     * Upload data using a single request.
+     */
+    private void upload(byte[] bytes, int off, int len) throws IOException {
+        try (ByteArrayInputStream is = new ByteArrayInputStream(bytes, off, len)) {
+            try {
+                doUpload(getBlobStore(), getBucketName(), getBlobName(), is, len, isServerSideEncryption());
+            } catch (AmazonClientException e) {
+                throw new IOException("Unable to upload object " + getBlobName(), e);
+            }
+        }
+    }
+
+    protected void doUpload(S3BlobStore blobStore, String bucketName, String blobName, InputStream is, int length,
+            boolean serverSideEncryption) throws AmazonS3Exception {
+        ObjectMetadata md = new ObjectMetadata();
+        if (serverSideEncryption) {
+            md.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
+        }
+        md.setContentLength(length);
+
+        PutObjectRequest putRequest = new PutObjectRequest(bucketName, blobName, is, md)
+                .withStorageClass(blobStore.getStorageClass())
+                .withCannedAcl(blobStore.getCannedACL());
+        blobStore.client().putObject(putRequest);
+
+    }
+
+    private void initializeMultipart() {
+        while (multipartId == null) {
+            multipartId = doInitialize(getBlobStore(), getBucketName(), getBlobName(), isServerSideEncryption());
+            if (multipartId != null) {
+                multipartChunks = 1;
+                multiparts = new ArrayList<>();
+            }
+        }
+    }
+
+    protected String doInitialize(S3BlobStore blobStore, String bucketName, String blobName, boolean serverSideEncryption) {
+        InitiateMultipartUploadRequest request = new InitiateMultipartUploadRequest(bucketName, blobName)
+                .withCannedACL(blobStore.getCannedACL())
+                .withStorageClass(blobStore.getStorageClass());
+
+        if (serverSideEncryption) {
+            ObjectMetadata md = new ObjectMetadata();
+            md.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
+            request.setObjectMetadata(md);
+        }
+
+        return blobStore.client().initiateMultipartUpload(request).getUploadId();
+    }
+
+    private void uploadMultipart(byte[] bytes, int off, int len, boolean lastPart) throws IOException {
+        try (ByteArrayInputStream is = new ByteArrayInputStream(bytes, off, len)) {
+            try {
+                PartETag partETag = doUploadMultipart(getBlobStore(), getBucketName(), getBlobName(), multipartId, is, len, lastPart);
+                multiparts.add(partETag);
+                multipartChunks++;
+            } catch (AmazonClientException e) {
+                abortMultipart();
+                throw e;
+            }
+        }
+    }
+
+    protected PartETag doUploadMultipart(S3BlobStore blobStore, String bucketName, String blobName, String uploadId, InputStream is,
+            int length, boolean lastPart) throws AmazonS3Exception {
+        UploadPartRequest request = new UploadPartRequest()
+        .withBucketName(bucketName)
+        .withKey(blobName)
+        .withUploadId(uploadId)
+        .withPartNumber(multipartChunks)
+        .withInputStream(is)
+        .withPartSize(length)
+        .withLastPart(lastPart);
+
+        UploadPartResult response = blobStore.client().uploadPart(request);
+        return response.getPartETag();
+
+    }
+
+    private void completeMultipart() {
+        try {
+            doCompleteMultipart(getBlobStore(), getBucketName(), getBlobName(), multipartId, multiparts);
+            multipartId = null;
+            return;
+        } catch (AmazonClientException e) {
+            abortMultipart();
+            throw e;
+        }
+    }
+
+    protected void doCompleteMultipart(S3BlobStore blobStore, String bucketName, String blobName, String uploadId, List<PartETag> parts)
+            throws AmazonS3Exception {
+        CompleteMultipartUploadRequest request = new CompleteMultipartUploadRequest(bucketName, blobName, uploadId, parts);
+        blobStore.client().completeMultipartUpload(request);
+    }
+
+    private void abortMultipart() {
+        if (multipartId != null) {
+            try {
+                doAbortMultipart(getBlobStore(), getBucketName(), getBlobName(), multipartId);
+            } finally {
+                multipartId = null;
+            }
+        }
+    }
+
+    protected void doAbortMultipart(S3BlobStore blobStore, String bucketName, String blobName, String uploadId)
+            throws AmazonS3Exception {
+        blobStore.client().abortMultipartUpload(new AbortMultipartUploadRequest(bucketName, blobName, uploadId));
+    }
+}
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java
new file mode 100644
index 0000000000000..327b318e24e58
--- /dev/null
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java
@@ -0,0 +1,328 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.repositories.s3;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.Function;
+
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.Protocol;
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.auth.EnvironmentVariableCredentialsProvider;
+import com.amazonaws.auth.InstanceProfileCredentialsProvider;
+import com.amazonaws.auth.SystemPropertiesCredentialsProvider;
+import com.amazonaws.http.IdleConnectionReaper;
+import com.amazonaws.internal.StaticCredentialsProvider;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.S3ClientOptions;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.component.AbstractLifecycleComponent;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+
+import static org.elasticsearch.repositories.s3.S3Repository.getValue;
+
+class InternalAwsS3Service extends AbstractLifecycleComponent implements AwsS3Service {
+
+    // pkg private for tests
+    static final Setting<String> CLIENT_NAME = new Setting<>("client", "default", Function.identity());
+
+    private final Map<String, S3ClientSettings> clientsSettings;
+
+    private final Map<String, AmazonS3Client> clientsCache = new HashMap<>();
+
+    InternalAwsS3Service(Settings settings, Map<String, S3ClientSettings> clientsSettings) {
+        super(settings);
+        this.clientsSettings = clientsSettings;
+    }
+
+    @Override
+    public synchronized AmazonS3 client(Settings repositorySettings) {
+        String clientName = CLIENT_NAME.get(repositorySettings);
+        AmazonS3Client client = clientsCache.get(clientName);
+        if (client != null) {
+            return client;
+        }
+
+        S3ClientSettings clientSettings = clientsSettings.get(clientName);
+        if (clientSettings == null) {
+            throw new IllegalArgumentException("Unknown s3 client name [" + clientName + "]. " +
+                "Existing client configs: " +
+                Strings.collectionToDelimitedString(clientsSettings.keySet(), ","));
+        }
+
+        String endpoint = findEndpoint(logger, deprecationLogger, clientSettings, repositorySettings);
+        // If the user defined a path style access setting, we rely on it,
+        // otherwise we use the default value set by the SDK
+        Boolean pathStyleAccess = null;
+        if (S3Repository.Repository.PATH_STYLE_ACCESS_SETTING.exists(repositorySettings) ||
+            S3Repository.Repositories.PATH_STYLE_ACCESS_SETTING.exists(settings)) {
+            pathStyleAccess = getValue(repositorySettings, settings,
+                S3Repository.Repository.PATH_STYLE_ACCESS_SETTING,
+                S3Repository.Repositories.PATH_STYLE_ACCESS_SETTING);
+        }
+
+        logger.debug("creating S3 client with client_name [{}], endpoint [{}], path_style_access [{}]",
+            clientName, endpoint, pathStyleAccess);
+
+        AWSCredentialsProvider credentials = buildCredentials(logger, deprecationLogger, clientSettings, repositorySettings);
+        ClientConfiguration configuration = buildConfiguration(logger, clientSettings, repositorySettings, endpoint);
+
+        client = new AmazonS3Client(credentials, configuration);
+
+        if (pathStyleAccess != null) {
+            client.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(pathStyleAccess));
+        }
+
+        if (Strings.hasText(endpoint)) {
+            client.setEndpoint(endpoint);
+        }
+
+        clientsCache.put(clientName, client);
+        return client;
+    }
+
+    // pkg private for tests
+    static ClientConfiguration buildConfiguration(Logger logger, S3ClientSettings clientSettings,
+                                                  Settings repositorySettings, String endpoint) {
+        ClientConfiguration clientConfiguration = new ClientConfiguration();
+        // the response metadata cache is only there for diagnostics purposes,
+        // but can force objects from every response to the old generation.
+        clientConfiguration.setResponseMetadataCacheSize(0);
+        Protocol protocol = getRepoValue(repositorySettings, S3Repository.Repository.PROTOCOL_SETTING, clientSettings.protocol);
+        clientConfiguration.setProtocol(protocol);
+
+        if (Strings.hasText(clientSettings.proxyHost)) {
+            // TODO: remove this leniency, these settings should exist together and be validated
+            clientConfiguration.setProxyHost(clientSettings.proxyHost);
+            clientConfiguration.setProxyPort(clientSettings.proxyPort);
+            clientConfiguration.setProxyUsername(clientSettings.proxyUsername);
+            clientConfiguration.setProxyPassword(clientSettings.proxyPassword);
+        }
+
+        Integer maxRetries = getRepoValue(repositorySettings, S3Repository.Repository.MAX_RETRIES_SETTING, clientSettings.maxRetries);
+        if (maxRetries != null) {
+            // If not explicitly set, default to 3 with exponential backoff policy
+            clientConfiguration.setMaxErrorRetry(maxRetries);
+        }
+        boolean useThrottleRetries = getRepoValue(repositorySettings,
+            S3Repository.Repository.USE_THROTTLE_RETRIES_SETTING, clientSettings.throttleRetries);
+        clientConfiguration.setUseThrottleRetries(useThrottleRetries);
+
+        // #155: we might have 3rd party users using older S3 API version
+        if (Strings.hasText(clientSettings.awsSigner)) {
+            logger.debug("using AWS API signer [{}]", clientSettings.awsSigner);
+            AwsSigner.configureSigner(clientSettings.awsSigner, clientConfiguration, endpoint);
+        }
+        clientConfiguration.setSocketTimeout(clientSettings.readTimeoutMillis);
+
+        return clientConfiguration;
+    }
+
+    /** Return credentials from the given provider, or null if full credentials are not available */
+    private static AWSCredentials getDeprecatedCredentials(Logger logger, DeprecationLogger deprecationLogger,
+                                                           AWSCredentialsProvider provider, String description) {
+        try {
+            AWSCredentials credentials = provider.getCredentials();
+            if (credentials.getAWSAccessKeyId() != null && credentials.getAWSSecretKey() != null) {
+                logger.debug("Using " + description + " credentials");
+                deprecationLogger.deprecated("Supplying S3 credentials through " + description + " is deprecated. " +
+                    "See the breaking changes lists in the documentation for details.");
+                return credentials;
+            }
+        } catch (Exception e) {
+            logger.debug("Failed to get aws credentials from " + description, e);
+        }
+        return null;
+    }
+
+    // pkg private for tests
+    static AWSCredentialsProvider buildCredentials(Logger logger, DeprecationLogger deprecationLogger,
+                                                   S3ClientSettings clientSettings, Settings repositorySettings) {
+        BasicAWSCredentials credentials = clientSettings.credentials;
+        if (S3Repository.Repository.KEY_SETTING.exists(repositorySettings)) {
+            if (S3Repository.Repository.SECRET_SETTING.exists(repositorySettings) == false) {
+                throw new IllegalArgumentException("Repository setting [" + S3Repository.Repository.KEY_SETTING +
+                    " must be accompanied by setting [" + S3Repository.Repository.SECRET_SETTING + "]");
+            }
+            // backcompat for reading keys out of repository settings
+            deprecationLogger.deprecated("Using s3 access/secret key from repository settings. Instead " +
+                "store these in named clients and the elasticsearch keystore for secure settings.");
+            try (SecureString key = S3Repository.Repository.KEY_SETTING.get(repositorySettings);
+                 SecureString secret = S3Repository.Repository.SECRET_SETTING.get(repositorySettings)) {
+                credentials = new BasicAWSCredentials(key.toString(), secret.toString());
+            }
+        } else if (S3Repository.Repository.SECRET_SETTING.exists(repositorySettings)) {
+            throw new IllegalArgumentException("Repository setting [" + S3Repository.Repository.SECRET_SETTING +
+                " must be accompanied by setting [" + S3Repository.Repository.KEY_SETTING + "]");
+        }
+        if (credentials == null) {
+            // create a "manual" chain of providers here, so we can log deprecation of unsupported methods
+            AWSCredentials envCredentials = getDeprecatedCredentials(logger, deprecationLogger,
+                new EnvironmentVariableCredentialsProvider(), "environment variables");
+            if (envCredentials != null) {
+                return new StaticCredentialsProvider(envCredentials);
+            } else {
+                AWSCredentials syspropCredentials = getDeprecatedCredentials(logger, deprecationLogger,
+                    new SystemPropertiesCredentialsProvider(), "system properties");
+                if (syspropCredentials != null) {
+                    return new StaticCredentialsProvider(syspropCredentials);
+                } else {
+                    logger.debug("Using instance profile credentials");
+                    return new InstanceProfileCredentialsProvider();
+                }
+            }
+        } else {
+            logger.debug("Using basic key/secret credentials");
+            return new StaticCredentialsProvider(credentials);
+        }
+    }
+
+    // pkg private for tests
+    /** Returns the endpoint the client should use, based on the available endpoint settings found. */
+    static String findEndpoint(Logger logger, DeprecationLogger deprecationLogger,
+                               S3ClientSettings clientSettings, Settings repositorySettings) {
+        String region = getRepoValue(repositorySettings, S3Repository.Repository.REGION_SETTING, clientSettings.region);
+        String endpoint = getRepoValue(repositorySettings, S3Repository.Repository.ENDPOINT_SETTING, clientSettings.endpoint);
+        if (Strings.isNullOrEmpty(endpoint)) {
+            logger.debug("no repository level endpoint has been defined. Trying to guess from repository region [{}]", region);
+            if (!region.isEmpty()) {
+                deprecationLogger.deprecated("Specifying region for an s3 repository is deprecated. Use endpoint " +
+                                             "to specify the region endpoint if the default behavior is not sufficient.");
+                endpoint = getEndpoint(region);
+                logger.debug("using s3 region [{}], with endpoint [{}]", region, endpoint);
+            }
+        } else {
+            logger.debug("using repository level endpoint [{}]", endpoint);
+        }
+        return endpoint;
+    }
+
+    /**
+     * Return the region configured, or empty string.
+     * TODO: remove after https://github.com/elastic/elasticsearch/issues/22761 */
+    public static String getRegion(Settings repositorySettings, Settings settings) {
+        return S3Repository.getValue(repositorySettings, settings,
+            S3Repository.Repository.REGION_SETTING, S3Repository.Repositories.REGION_SETTING);
+    }
+
+    private static String getEndpoint(String region) {
+        final String endpoint;
+        switch (region) {
+            case "us-east":
+            case "us-east-1":
+                endpoint = "s3.amazonaws.com";
+                break;
+            case "us-east-2":
+                endpoint = "s3.us-east-2.amazonaws.com";
+                break;
+            case "us-west":
+            case "us-west-1":
+                endpoint = "s3-us-west-1.amazonaws.com";
+                break;
+            case "us-west-2":
+                endpoint = "s3-us-west-2.amazonaws.com";
+                break;
+            case "ap-south":
+            case "ap-south-1":
+                endpoint = "s3-ap-south-1.amazonaws.com";
+                break;
+            case "ap-southeast":
+            case "ap-southeast-1":
+                endpoint = "s3-ap-southeast-1.amazonaws.com";
+                break;
+            case "ap-southeast-2":
+                endpoint = "s3-ap-southeast-2.amazonaws.com";
+                break;
+            case "ap-northeast":
+            case "ap-northeast-1":
+                endpoint = "s3-ap-northeast-1.amazonaws.com";
+                break;
+            case "ap-northeast-2":
+                endpoint = "s3-ap-northeast-2.amazonaws.com";
+                break;
+            case "eu-west":
+            case "eu-west-1":
+                endpoint = "s3-eu-west-1.amazonaws.com";
+                break;
+            case "eu-west-2":
+                endpoint = "s3-eu-west-2.amazonaws.com";
+                break;
+            case "eu-central":
+            case "eu-central-1":
+                endpoint = "s3.eu-central-1.amazonaws.com";
+                break;
+            case "sa-east":
+            case "sa-east-1":
+                endpoint = "s3-sa-east-1.amazonaws.com";
+                break;
+            case "cn-north":
+            case "cn-north-1":
+                endpoint = "s3.cn-north-1.amazonaws.com.cn";
+                break;
+            case "us-gov-west":
+            case "us-gov-west-1":
+                endpoint = "s3-us-gov-west-1.amazonaws.com";
+                break;
+            case "ca-central":
+            case "ca-central-1":
+                endpoint = "s3.ca-central-1.amazonaws.com";
+                break;
+            default:
+                throw new IllegalArgumentException("No automatic endpoint could be derived from region [" + region + "]");
+        }
+
+        return endpoint;
+    }
+
+    /** Returns the value for a given setting from the repository, or returns the fallback value. */
+    private static <T> T getRepoValue(Settings repositorySettings, Setting<T> repositorySetting, T fallback) {
+        if (repositorySetting.exists(repositorySettings)) {
+            return repositorySetting.get(repositorySettings);
+        }
+        return fallback;
+    }
+
+    @Override
+    protected void doStart() throws ElasticsearchException {
+    }
+
+    @Override
+    protected void doStop() throws ElasticsearchException {
+    }
+
+    @Override
+    protected void doClose() throws ElasticsearchException {
+        for (AmazonS3Client client : clientsCache.values()) {
+            client.shutdown();
+        }
+
+        // Ensure that IdleConnectionReaper is shutdown
+        IdleConnectionReaper.shutdown();
+    }
+}
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java
similarity index 84%
rename from plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java
rename to plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java
index e73a3909d8dcd..3a10df4318810 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.cloud.aws.blobstore;
+package org.elasticsearch.repositories.s3;
 
 import com.amazonaws.AmazonClientException;
 import com.amazonaws.services.s3.model.AmazonS3Exception;
@@ -46,13 +46,13 @@
 import java.security.PrivilegedExceptionAction;
 import java.util.Map;
 
-public class S3BlobContainer extends AbstractBlobContainer {
+class S3BlobContainer extends AbstractBlobContainer {
 
     protected final S3BlobStore blobStore;
 
     protected final String keyPath;
 
-    public S3BlobContainer(BlobPath path, S3BlobStore blobStore) {
+    S3BlobContainer(BlobPath path, S3BlobStore blobStore) {
         super(path);
         this.blobStore = blobStore;
         this.keyPath = path.buildAsString();
@@ -78,25 +78,17 @@ public boolean blobExists(String blobName) {
 
     @Override
     public InputStream readBlob(String blobName) throws IOException {
-        int retry = 0;
-        while (retry <= blobStore.numberOfRetries()) {
-            try {
-                S3Object s3Object = blobStore.client().getObject(blobStore.bucket(), buildKey(blobName));
-                return s3Object.getObjectContent();
-            } catch (AmazonClientException e) {
-                if (blobStore.shouldRetry(e) && (retry < blobStore.numberOfRetries())) {
-                    retry++;
-                } else {
-                    if (e instanceof AmazonS3Exception) {
-                        if (404 == ((AmazonS3Exception) e).getStatusCode()) {
-                            throw new NoSuchFileException("Blob object [" + blobName + "] not found: " + e.getMessage());
-                        }
-                    }
-                    throw e;
+        try {
+            S3Object s3Object = blobStore.client().getObject(blobStore.bucket(), buildKey(blobName));
+            return s3Object.getObjectContent();
+        } catch (AmazonClientException e) {
+            if (e instanceof AmazonS3Exception) {
+                if (404 == ((AmazonS3Exception) e).getStatusCode()) {
+                    throw new NoSuchFileException("Blob object [" + blobName + "] not found: " + e.getMessage());
                 }
             }
+            throw e;
         }
-        throw new BlobStoreException("retries exhausted while attempting to access blob object [name:" + blobName + ", bucket:" + blobStore.bucket() +"]");
     }
 
     @Override
@@ -125,7 +117,7 @@ public void deleteBlob(String blobName) throws IOException {
     private OutputStream createOutput(final String blobName) throws IOException {
         // UploadS3OutputStream does buffering & retry logic internally
         return new DefaultS3OutputStream(blobStore, blobStore.bucket(), buildKey(blobName),
-                blobStore.bufferSizeInBytes(), blobStore.numberOfRetries(), blobStore.serverSideEncryption());
+            blobStore.bufferSizeInBytes(), blobStore.serverSideEncryption());
     }
 
     @Override
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java
similarity index 79%
rename from plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java
rename to plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java
index 991e5f9707f9a..75eea6d7ca50f 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.cloud.aws.blobstore;
+package org.elasticsearch.repositories.s3;
 
 import com.amazonaws.AmazonClientException;
 import com.amazonaws.services.s3.AmazonS3;
@@ -41,10 +41,7 @@
 import java.util.ArrayList;
 import java.util.Locale;
 
-/**
- *
- */
-public class S3BlobStore extends AbstractComponent implements BlobStore {
+class S3BlobStore extends AbstractComponent implements BlobStore {
 
     private final AmazonS3 client;
 
@@ -56,14 +53,12 @@ public class S3BlobStore extends AbstractComponent implements BlobStore {
 
     private final boolean serverSideEncryption;
 
-    private final int numberOfRetries;
-
     private final CannedAccessControlList cannedACL;
 
     private final StorageClass storageClass;
 
-    public S3BlobStore(Settings settings, AmazonS3 client, String bucket, @Nullable String region, boolean serverSideEncryption,
-                       ByteSizeValue bufferSize, int maxRetries, String cannedACL, String storageClass) {
+    S3BlobStore(Settings settings, AmazonS3 client, String bucket, @Nullable String region, boolean serverSideEncryption,
+                       ByteSizeValue bufferSize, String cannedACL, String storageClass) {
         super(settings);
         this.client = client;
         this.bucket = bucket;
@@ -71,7 +66,6 @@ public S3BlobStore(Settings settings, AmazonS3 client, String bucket, @Nullable
         this.serverSideEncryption = serverSideEncryption;
         this.bufferSize = bufferSize;
         this.cannedACL = initCannedACL(cannedACL);
-        this.numberOfRetries = maxRetries;
         this.storageClass = initStorageClass(storageClass);
 
         // Note: the method client.doesBucketExist() may return 'true' is the bucket exists
@@ -79,28 +73,17 @@ public S3BlobStore(Settings settings, AmazonS3 client, String bucket, @Nullable
         // Also, if invalid security credentials are used to execute this method, the
         // client is not able to distinguish between bucket permission errors and
         // invalid credential errors, and this method could return an incorrect result.
-        int retry = 0;
-        while (retry <= maxRetries) {
-            try {
-                if (!client.doesBucketExist(bucket)) {
-                    CreateBucketRequest request = null;
-                    if (region != null) {
-                        request = new CreateBucketRequest(bucket, region);
-                    } else {
-                        request = new CreateBucketRequest(bucket);
-                    }
-                    request.setCannedAcl(this.cannedACL);
-                    client.createBucket(request);
-                }
-                break;
-            } catch (AmazonClientException e) {
-                if (shouldRetry(e) && retry < maxRetries) {
-                    retry++;
-                } else {
-                    logger.debug("S3 client create bucket failed");
-                    throw e;
-                }
+        if (!client.doesBucketExist(bucket)) {
+            deprecationLogger.deprecated("Auto creation of the bucket for an s3 backed " +
+                                         "repository is deprecated and will be removed in 6.0.");
+            final CreateBucketRequest request;
+            if (region != null) {
+                request = new CreateBucketRequest(bucket, region);
+            } else {
+                request = new CreateBucketRequest(bucket);
             }
+            request.setCannedAcl(this.cannedACL);
+            client.createBucket(request);
         }
     }
 
@@ -123,10 +106,6 @@ public int bufferSizeInBytes() {
         return bufferSize.bytesAsInt();
     }
 
-    public int numberOfRetries() {
-        return numberOfRetries;
-    }
-
     @Override
     public BlobContainer blobContainer(BlobPath path) {
         return new S3BlobContainer(path, this);
@@ -170,16 +149,6 @@ public void delete(BlobPath path) {
         }
     }
 
-    protected boolean shouldRetry(AmazonClientException e) {
-        if (e instanceof AmazonS3Exception) {
-            AmazonS3Exception s3e = (AmazonS3Exception)e;
-            if (s3e.getStatusCode() == 400 && "RequestTimeout".equals(s3e.getErrorCode())) {
-                return true;
-            }
-        }
-        return e.isRetryable();
-    }
-
     @Override
     public void close() {
     }
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java
new file mode 100644
index 0000000000000..bf3803adcbd48
--- /dev/null
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java
@@ -0,0 +1,211 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.repositories.s3;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.Protocol;
+import com.amazonaws.auth.BasicAWSCredentials;
+import org.elasticsearch.common.settings.SecureSetting;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.repositories.s3.AwsS3Service.CLOUD_S3;
+
+/**
+ * A container for settings used to create an S3 client.
+ */
+class S3ClientSettings {
+
+    // prefix for s3 client settings
+    private static final String PREFIX = "s3.client.";
+
+    /** The access key (ie login id) for connecting to s3. */
+    static final Setting.AffixSetting<SecureString> ACCESS_KEY_SETTING = Setting.affixKeySetting(PREFIX, "access_key",
+        key -> SecureSetting.secureString(key, S3Repository.Repositories.KEY_SETTING));
+
+    /** The secret key (ie password) for connecting to s3. */
+    static final Setting.AffixSetting<SecureString> SECRET_KEY_SETTING = Setting.affixKeySetting(PREFIX, "secret_key",
+        key -> SecureSetting.secureString(key, S3Repository.Repositories.SECRET_SETTING));
+
+    /** An override for the s3 endpoint to connect to. */
+    static final Setting.AffixSetting<String> ENDPOINT_SETTING = Setting.affixKeySetting(PREFIX, "endpoint",
+        key -> new Setting<>(key, S3Repository.Repositories.ENDPOINT_SETTING, s -> s.toLowerCase(Locale.ROOT),
+            Setting.Property.NodeScope));
+
+    /** The protocol to use to connect to s3. */
+    static final Setting.AffixSetting<Protocol> PROTOCOL_SETTING = Setting.affixKeySetting(PREFIX, "protocol",
+        key -> new Setting<>(key, "https", s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)), Property.NodeScope));
+
+    /** The host name of a proxy to connect to s3 through. */
+    static final Setting.AffixSetting<String> PROXY_HOST_SETTING = Setting.affixKeySetting(PREFIX, "proxy.host",
+        key -> Setting.simpleString(key, Property.NodeScope));
+
+    /** The port of a proxy to connect to s3 through. */
+    static final Setting.AffixSetting<Integer> PROXY_PORT_SETTING = Setting.affixKeySetting(PREFIX, "proxy.port",
+        key -> Setting.intSetting(key, 80, 0, 1<<16, Property.NodeScope));
+
+    /** The username of a proxy to connect to s3 through. */
+    static final Setting.AffixSetting<SecureString> PROXY_USERNAME_SETTING = Setting.affixKeySetting(PREFIX, "proxy.username",
+        key -> SecureSetting.secureString(key, AwsS3Service.PROXY_USERNAME_SETTING));
+
+    /** The password of a proxy to connect to s3 through. */
+    static final Setting.AffixSetting<SecureString> PROXY_PASSWORD_SETTING = Setting.affixKeySetting(PREFIX, "proxy.password",
+        key -> SecureSetting.secureString(key, AwsS3Service.PROXY_PASSWORD_SETTING));
+
+    /** The socket timeout for connecting to s3. */
+    static final Setting.AffixSetting<TimeValue> READ_TIMEOUT_SETTING = Setting.affixKeySetting(PREFIX, "read_timeout",
+        key -> Setting.timeSetting(key, TimeValue.timeValueMillis(ClientConfiguration.DEFAULT_SOCKET_TIMEOUT), Property.NodeScope));
+
+    /** The number of retries to use when an s3 request fails. */
+    static final Setting.AffixSetting<Integer> MAX_RETRIES_SETTING = Setting.affixKeySetting(PREFIX, "max_retries",
+        key -> Setting.intSetting(key, S3Repository.Repositories.MAX_RETRIES_SETTING, 0, Property.NodeScope));
+
+    /** Whether retries should be throttled (ie use backoff). */
+    static final Setting.AffixSetting<Boolean> USE_THROTTLE_RETRIES_SETTING = Setting.affixKeySetting(PREFIX, "use_throttle_retries",
+        key -> Setting.boolSetting(key, S3Repository.Repositories.USE_THROTTLE_RETRIES_SETTING, Property.NodeScope));
+
+    /** Credentials to authenticate with s3. */
+    final BasicAWSCredentials credentials;
+
+    /** The s3 endpoint the client should talk to, or empty string to use the default. */
+    final String endpoint;
+
+    /** The s3 region the client should connect to, or empty string to find it automatically for the configured bucket. */
+    final String region;
+
+    /** The protocol to use to talk to s3. Defaults to https. */
+    final Protocol protocol;
+
+    /** An optional proxy host that requests to s3 should be made through. */
+    final String proxyHost;
+
+    /** The port number the proxy host should be connected on. */
+    final int proxyPort;
+
+    // these should be "secure" yet the api for the s3 client only takes String, so storing them
+    // as SecureString here won't really help with anything
+    /** An optional username for the proxy host, for basic authentication. */
+    final String proxyUsername;
+
+    /** An optional password for the proxy host, for basic authentication. */
+    final String proxyPassword;
+
+    /** The signing algorithm to use for the s3 client. */
+    final String awsSigner;
+
+    /** The read timeout for the s3 client. */
+    final int readTimeoutMillis;
+
+    /** The number of retries to use for the s3 client. */
+    final int maxRetries;
+
+    /** Whether the s3 client should use an exponential backoff retry policy. */
+    final boolean throttleRetries;
+
+    private S3ClientSettings(BasicAWSCredentials credentials, String endpoint, String region, Protocol protocol,
+                             String proxyHost, int proxyPort, String proxyUsername,
+                             String proxyPassword, String awsSigner, int readTimeoutMillis, int maxRetries, boolean throttleRetries) {
+        this.credentials = credentials;
+        this.region = region;
+        this.endpoint = endpoint;
+        this.protocol = protocol;
+        this.proxyHost = proxyHost;
+        this.proxyPort = proxyPort;
+        this.proxyUsername = proxyUsername;
+        this.proxyPassword = proxyPassword;
+        this.awsSigner = awsSigner;
+        this.readTimeoutMillis = readTimeoutMillis;
+        this.maxRetries = maxRetries;
+        this.throttleRetries = throttleRetries;
+    }
+
+    /**
+     * Load all client settings from the given settings.
+     *
+     * Note this will always at least return a client named "default".
+     */
+    static Map<String, S3ClientSettings> load(Settings settings) {
+        Set<String> clientNames = settings.getGroups(PREFIX).keySet();
+        Map<String, S3ClientSettings> clients = new HashMap<>();
+        for (String clientName : clientNames) {
+            clients.put(clientName, getClientSettings(settings, clientName));
+        }
+        if (clients.containsKey("default") == false) {
+            // this won't find any settings under the default client,
+            // but it will pull all the fallback static settings
+            clients.put("default", getClientSettings(settings, "default"));
+        }
+        return Collections.unmodifiableMap(clients);
+    }
+
+    // pkg private for tests
+    /** Parse settings for a single client. */
+    static S3ClientSettings getClientSettings(Settings settings, String clientName) {
+        try (SecureString accessKey = getConfigValue(settings, clientName, ACCESS_KEY_SETTING, S3Repository.Repositories.KEY_SETTING);
+             SecureString secretKey = getConfigValue(settings, clientName, SECRET_KEY_SETTING, S3Repository.Repositories.SECRET_SETTING);
+             SecureString proxyUsername = getConfigValue(settings, clientName, PROXY_USERNAME_SETTING, CLOUD_S3.PROXY_USERNAME_SETTING);
+             SecureString proxyPassword = getConfigValue(settings, clientName, PROXY_PASSWORD_SETTING, CLOUD_S3.PROXY_PASSWORD_SETTING)) {
+            BasicAWSCredentials credentials = null;
+            if (accessKey.length() != 0) {
+                if (secretKey.length() != 0) {
+                    credentials = new BasicAWSCredentials(accessKey.toString(), secretKey.toString());
+                } else {
+                    throw new IllegalArgumentException("Missing secret key for s3 client [" + clientName + "]");
+                }
+            } else if (secretKey.length() != 0) {
+                throw new IllegalArgumentException("Missing access key for s3 client [" + clientName + "]");
+            }
+            return new S3ClientSettings(
+                credentials,
+                getConfigValue(settings, clientName, ENDPOINT_SETTING, S3Repository.Repositories.ENDPOINT_SETTING),
+                S3Repository.Repositories.REGION_SETTING.get(settings),
+                getConfigValue(settings, clientName, PROTOCOL_SETTING, S3Repository.Repositories.PROTOCOL_SETTING),
+                getConfigValue(settings, clientName, PROXY_HOST_SETTING, AwsS3Service.CLOUD_S3.PROXY_HOST_SETTING),
+                getConfigValue(settings, clientName, PROXY_PORT_SETTING, AwsS3Service.CLOUD_S3.PROXY_PORT_SETTING),
+                proxyUsername.toString(),
+                proxyPassword.toString(),
+                CLOUD_S3.SIGNER_SETTING.get(settings),
+                (int)getConfigValue(settings, clientName, READ_TIMEOUT_SETTING, AwsS3Service.CLOUD_S3.READ_TIMEOUT).millis(),
+                getConfigValue(settings, clientName, MAX_RETRIES_SETTING, S3Repository.Repositories.MAX_RETRIES_SETTING),
+                getConfigValue(settings, clientName, USE_THROTTLE_RETRIES_SETTING, S3Repository.Repositories.USE_THROTTLE_RETRIES_SETTING)
+            );
+        }
+    }
+
+    private static <T> T getConfigValue(Settings settings, String clientName,
+                                        Setting.AffixSetting<T> clientSetting,
+                                        Setting<T> globalSetting) {
+        Setting<T> concreteSetting = clientSetting.getConcreteSettingForNamespace(clientName);
+        if (concreteSetting.exists(settings)) {
+            return concreteSetting.get(settings);
+        } else {
+            return globalSetting.get(settings);
+        }
+    }
+
+}
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3OutputStream.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3OutputStream.java
similarity index 88%
rename from plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3OutputStream.java
rename to plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3OutputStream.java
index a1b66ad4e9b22..46c9108f1b585 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3OutputStream.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3OutputStream.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.cloud.aws.blobstore;
+package org.elasticsearch.repositories.s3;
 
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
@@ -28,7 +28,7 @@
 /**
  * S3OutputStream buffers data before flushing it to an underlying S3OutputStream.
  */
-public abstract class S3OutputStream extends OutputStream {
+abstract class S3OutputStream extends OutputStream {
 
     /**
      * Limit of upload allowed by AWS S3.
@@ -39,7 +39,6 @@ public abstract class S3OutputStream extends OutputStream {
     private S3BlobStore blobStore;
     private String bucketName;
     private String blobName;
-    private int numberOfRetries;
     private boolean serverSideEncryption;
 
     private byte[] buffer;
@@ -48,11 +47,10 @@ public abstract class S3OutputStream extends OutputStream {
 
     private int flushCount = 0;
 
-    public S3OutputStream(S3BlobStore blobStore, String bucketName, String blobName, int bufferSizeInBytes, int numberOfRetries, boolean serverSideEncryption) {
+    S3OutputStream(S3BlobStore blobStore, String bucketName, String blobName, int bufferSizeInBytes, boolean serverSideEncryption) {
         this.blobStore = blobStore;
         this.bucketName = bucketName;
         this.blobName = blobName;
-        this.numberOfRetries = numberOfRetries;
         this.serverSideEncryption = serverSideEncryption;
 
         if (bufferSizeInBytes < MULTIPART_MIN_SIZE.getBytes()) {
@@ -107,10 +105,6 @@ public int getBufferSize() {
         return buffer.length;
     }
 
-    public int getNumberOfRetries() {
-        return numberOfRetries;
-    }
-
     public boolean isServerSideEncryption() {
         return serverSideEncryption;
     }
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
index b5abb361be802..8be769060b366 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
@@ -19,27 +19,30 @@
 
 package org.elasticsearch.repositories.s3;
 
-import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.ClientConfiguration;
 import com.amazonaws.Protocol;
-import org.elasticsearch.cloud.aws.AwsS3Service;
-import org.elasticsearch.cloud.aws.AwsS3Service.CLOUD_S3;
-import org.elasticsearch.cloud.aws.blobstore.S3BlobStore;
+import com.amazonaws.services.s3.AmazonS3;
+import org.elasticsearch.repositories.s3.AwsS3Service.CLOUD_S3;
 import org.elasticsearch.cluster.metadata.RepositoryMetaData;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.BlobStore;
+import org.elasticsearch.common.settings.SecureSetting;
+import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.AffixSetting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.monitor.jvm.JvmInfo;
 import org.elasticsearch.repositories.RepositoryException;
 import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
 
 import java.io.IOException;
 import java.util.Locale;
-import java.util.function.Function;
 
 /**
  * Shared file system implementation of the BlobStoreRepository
@@ -54,97 +57,114 @@
  * <dt>{@code compress}</dt><dd>If set to true metadata files will be stored compressed. Defaults to false.</dd>
  * </dl>
  */
-public class S3Repository extends BlobStoreRepository {
+class S3Repository extends BlobStoreRepository {
 
     public static final String TYPE = "s3";
 
     /**
      * Global S3 repositories settings. Starting with: repositories.s3
+     * NOTE: These are legacy settings. Use the named client config settings above.
      */
+    @Deprecated
     public interface Repositories {
         /**
          * repositories.s3.access_key: AWS Access key specific for all S3 Repositories API calls. Defaults to cloud.aws.s3.access_key.
          * @see CLOUD_S3#KEY_SETTING
          */
-        Setting<String> KEY_SETTING =
-            new Setting<>("repositories.s3.access_key", CLOUD_S3.KEY_SETTING, Function.identity(), Property.NodeScope, Property.Filtered);
+        Setting<SecureString> KEY_SETTING = new Setting<>("repositories.s3.access_key", CLOUD_S3.KEY_SETTING, SecureString::new,
+            Property.NodeScope, Property.Filtered, Property.Deprecated);
+
         /**
          * repositories.s3.secret_key: AWS Secret key specific for all S3 Repositories API calls. Defaults to cloud.aws.s3.secret_key.
          * @see CLOUD_S3#SECRET_SETTING
          */
-        Setting<String> SECRET_SETTING =
-            new Setting<>("repositories.s3.secret_key", CLOUD_S3.SECRET_SETTING, Function.identity(), Property.NodeScope, Property.Filtered);
+        Setting<SecureString> SECRET_SETTING = new Setting<>("repositories.s3.secret_key", CLOUD_S3.SECRET_SETTING, SecureString::new,
+            Property.NodeScope, Property.Filtered, Property.Deprecated);
+
         /**
          * repositories.s3.region: Region specific for all S3 Repositories API calls. Defaults to cloud.aws.s3.region.
          * @see CLOUD_S3#REGION_SETTING
          */
-        Setting<String> REGION_SETTING =
-            new Setting<>("repositories.s3.region", CLOUD_S3.REGION_SETTING, s -> s.toLowerCase(Locale.ROOT), Property.NodeScope);
+        Setting<String> REGION_SETTING = new Setting<>("repositories.s3.region", CLOUD_S3.REGION_SETTING,
+            s -> s.toLowerCase(Locale.ROOT), Property.NodeScope, Property.Deprecated);
         /**
          * repositories.s3.endpoint: Endpoint specific for all S3 Repositories API calls. Defaults to cloud.aws.s3.endpoint.
          * @see CLOUD_S3#ENDPOINT_SETTING
          */
-        Setting<String> ENDPOINT_SETTING =
-            new Setting<>("repositories.s3.endpoint", CLOUD_S3.ENDPOINT_SETTING, s -> s.toLowerCase(Locale.ROOT), Property.NodeScope);
+        Setting<String> ENDPOINT_SETTING = new Setting<>("repositories.s3.endpoint", CLOUD_S3.ENDPOINT_SETTING,
+            s -> s.toLowerCase(Locale.ROOT), Property.NodeScope, Property.Deprecated);
         /**
          * repositories.s3.protocol: Protocol specific for all S3 Repositories API calls. Defaults to cloud.aws.s3.protocol.
          * @see CLOUD_S3#PROTOCOL_SETTING
          */
-        Setting<Protocol> PROTOCOL_SETTING =
-            new Setting<>("repositories.s3.protocol", CLOUD_S3.PROTOCOL_SETTING, s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)), Property.NodeScope);
+        Setting<Protocol> PROTOCOL_SETTING = new Setting<>("repositories.s3.protocol", CLOUD_S3.PROTOCOL_SETTING,
+            s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)), Property.NodeScope, Property.Deprecated);
         /**
          * repositories.s3.bucket: The name of the bucket to be used for snapshots.
          */
-        Setting<String> BUCKET_SETTING = Setting.simpleString("repositories.s3.bucket", Property.NodeScope);
+        Setting<String> BUCKET_SETTING = Setting.simpleString("repositories.s3.bucket", Property.NodeScope, Property.Deprecated);
         /**
          * repositories.s3.server_side_encryption: When set to true files are encrypted on server side using AES256 algorithm.
          * Defaults to false.
          */
         Setting<Boolean> SERVER_SIDE_ENCRYPTION_SETTING =
-            Setting.boolSetting("repositories.s3.server_side_encryption", false, Property.NodeScope);
+            Setting.boolSetting("repositories.s3.server_side_encryption", false, Property.NodeScope, Property.Deprecated);
+
+        /**
+         * Default is to use 100MB (S3 defaults) for heaps above 2GB and 5% of
+         * the available memory for smaller heaps.
+         */
+        ByteSizeValue DEFAULT_BUFFER_SIZE = new ByteSizeValue(
+                Math.max(
+                        ByteSizeUnit.MB.toBytes(5), // minimum value
+                        Math.min(
+                                ByteSizeUnit.MB.toBytes(100),
+                                JvmInfo.jvmInfo().getMem().getHeapMax().getBytes() / 20)),
+                ByteSizeUnit.BYTES);
+
         /**
          * repositories.s3.buffer_size: Minimum threshold below which the chunk is uploaded using a single request. Beyond this threshold,
          * the S3 repository will use the AWS Multipart Upload API to split the chunk into several parts, each of buffer_size length, and
          * to upload each part in its own request. Note that setting a buffer size lower than 5mb is not allowed since it will prevents the
-         * use of the Multipart API and may result in upload errors. Defaults to 100m.
+         * use of the Multipart API and may result in upload errors. Defaults to the minimum between 100MB and 5% of the heap size.
          */
         Setting<ByteSizeValue> BUFFER_SIZE_SETTING =
-            Setting.byteSizeSetting("repositories.s3.buffer_size", new ByteSizeValue(100, ByteSizeUnit.MB),
-                new ByteSizeValue(5, ByteSizeUnit.MB), new ByteSizeValue(5, ByteSizeUnit.TB), Property.NodeScope);
+            Setting.byteSizeSetting("repositories.s3.buffer_size", DEFAULT_BUFFER_SIZE,
+                new ByteSizeValue(5, ByteSizeUnit.MB), new ByteSizeValue(5, ByteSizeUnit.TB), Property.NodeScope, Property.Deprecated);
         /**
          * repositories.s3.max_retries: Number of retries in case of S3 errors. Defaults to 3.
          */
-        Setting<Integer> MAX_RETRIES_SETTING = Setting.intSetting("repositories.s3.max_retries", 3, Property.NodeScope);
+        Setting<Integer> MAX_RETRIES_SETTING = Setting.intSetting("repositories.s3.max_retries", 3, Property.NodeScope, Property.Deprecated);
         /**
          * repositories.s3.use_throttle_retries: Set to `true` if you want to throttle retries. Defaults to AWS SDK default value (`false`).
          */
         Setting<Boolean> USE_THROTTLE_RETRIES_SETTING = Setting.boolSetting("repositories.s3.use_throttle_retries",
-            ClientConfiguration.DEFAULT_THROTTLE_RETRIES, Property.NodeScope);
+            ClientConfiguration.DEFAULT_THROTTLE_RETRIES, Property.NodeScope, Property.Deprecated);
         /**
          * repositories.s3.chunk_size: Big files can be broken down into chunks during snapshotting if needed. Defaults to 1g.
          */
         Setting<ByteSizeValue> CHUNK_SIZE_SETTING =
             Setting.byteSizeSetting("repositories.s3.chunk_size", new ByteSizeValue(1, ByteSizeUnit.GB),
-                new ByteSizeValue(5, ByteSizeUnit.MB), new ByteSizeValue(5, ByteSizeUnit.TB), Property.NodeScope);
+                new ByteSizeValue(5, ByteSizeUnit.MB), new ByteSizeValue(5, ByteSizeUnit.TB), Property.NodeScope, Property.Deprecated);
         /**
          * repositories.s3.compress: When set to true metadata files are stored in compressed format. This setting doesn’t affect index
          * files that are already compressed by default. Defaults to false.
          */
-        Setting<Boolean> COMPRESS_SETTING = Setting.boolSetting("repositories.s3.compress", false, Property.NodeScope);
+        Setting<Boolean> COMPRESS_SETTING = Setting.boolSetting("repositories.s3.compress", false, Property.NodeScope, Property.Deprecated);
         /**
          * repositories.s3.storage_class: Sets the S3 storage class type for the backup files. Values may be standard, reduced_redundancy,
          * standard_ia. Defaults to standard.
          */
-        Setting<String> STORAGE_CLASS_SETTING = Setting.simpleString("repositories.s3.storage_class", Property.NodeScope);
+        Setting<String> STORAGE_CLASS_SETTING = Setting.simpleString("repositories.s3.storage_class", Property.NodeScope, Property.Deprecated);
         /**
          * repositories.s3.canned_acl: The S3 repository supports all S3 canned ACLs : private, public-read, public-read-write,
          * authenticated-read, log-delivery-write, bucket-owner-read, bucket-owner-full-control. Defaults to private.
          */
-        Setting<String> CANNED_ACL_SETTING = Setting.simpleString("repositories.s3.canned_acl", Property.NodeScope);
+        Setting<String> CANNED_ACL_SETTING = Setting.simpleString("repositories.s3.canned_acl", Property.NodeScope, Property.Deprecated);
         /**
          * repositories.s3.base_path: Specifies the path within bucket to repository data. Defaults to root directory.
          */
-        Setting<String> BASE_PATH_SETTING = Setting.simpleString("repositories.s3.base_path", Property.NodeScope);
+        Setting<String> BASE_PATH_SETTING = Setting.simpleString("repositories.s3.base_path", Property.NodeScope, Property.Deprecated);
         /**
          * repositories.s3.path_style_access: When set to true configures the client to use path-style access for all requests.
          Amazon S3 supports virtual-hosted-style and path-style access in all Regions. The path-style syntax, however,
@@ -153,7 +173,8 @@ The default behaviour is to detect which access style to use based on the config
          in path-style access) and the bucket being accessed (some buckets are not valid DNS names). Setting this flag
          will result in path-style access being used for all requests.
          */
-        Setting<Boolean> PATH_STYLE_ACCESS_SETTING = Setting.boolSetting("repositories.s3.path_style_access", false, Property.NodeScope);
+        Setting<Boolean> PATH_STYLE_ACCESS_SETTING = Setting.boolSetting("repositories.s3.path_style_access", false,
+            Property.NodeScope, Property.Deprecated);
     }
 
     /**
@@ -161,59 +182,54 @@ The default behaviour is to detect which access style to use based on the config
      * If undefined, they use the repositories.s3.xxx equivalent setting.
      */
     public interface Repository {
-        /**
-         * access_key
-         * @see  Repositories#KEY_SETTING
-         */
-        Setting<String> KEY_SETTING = Setting.simpleString("access_key");
-        /**
-         * secret_key
-         * @see  Repositories#SECRET_SETTING
-         */
-        Setting<String> SECRET_SETTING = Setting.simpleString("secret_key");
-        /**
-         * bucket
-         * @see  Repositories#BUCKET_SETTING
-         */
+        Setting<SecureString> KEY_SETTING = new Setting<>("access_key", "", SecureString::new,
+            Property.Filtered, Property.Deprecated);
+
+
+        Setting<SecureString> SECRET_SETTING = new Setting<>("secret_key", "", SecureString::new,
+            Property.Filtered, Property.Deprecated);
+
         Setting<String> BUCKET_SETTING = Setting.simpleString("bucket");
         /**
          * endpoint
          * @see  Repositories#ENDPOINT_SETTING
          */
-        Setting<String> ENDPOINT_SETTING = Setting.simpleString("endpoint");
+        Setting<String> ENDPOINT_SETTING = Setting.simpleString("endpoint", Property.Deprecated);
         /**
          * protocol
          * @see  Repositories#PROTOCOL_SETTING
          */
-        Setting<Protocol> PROTOCOL_SETTING = new Setting<>("protocol", "https", s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)));
+        Setting<Protocol> PROTOCOL_SETTING = new Setting<>("protocol", "https", s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)),
+            Property.Deprecated);
         /**
          * region
          * @see  Repositories#REGION_SETTING
          */
-        Setting<String> REGION_SETTING = new Setting<>("region", "", s -> s.toLowerCase(Locale.ROOT));
+        Setting<String> REGION_SETTING = new Setting<>("region", "", s -> s.toLowerCase(Locale.ROOT), Property.Deprecated);
         /**
          * server_side_encryption
          * @see  Repositories#SERVER_SIDE_ENCRYPTION_SETTING
          */
         Setting<Boolean> SERVER_SIDE_ENCRYPTION_SETTING = Setting.boolSetting("server_side_encryption", false);
+
         /**
          * buffer_size
          * @see  Repositories#BUFFER_SIZE_SETTING
          */
         Setting<ByteSizeValue> BUFFER_SIZE_SETTING =
-            Setting.byteSizeSetting("buffer_size", new ByteSizeValue(100, ByteSizeUnit.MB),
+            Setting.byteSizeSetting("buffer_size", Repositories.DEFAULT_BUFFER_SIZE,
                 new ByteSizeValue(5, ByteSizeUnit.MB), new ByteSizeValue(5, ByteSizeUnit.TB));
         /**
          * max_retries
          * @see  Repositories#MAX_RETRIES_SETTING
          */
-        Setting<Integer> MAX_RETRIES_SETTING = Setting.intSetting("max_retries", 3);
+        Setting<Integer> MAX_RETRIES_SETTING = Setting.intSetting("max_retries", 3, Property.Deprecated);
         /**
          * use_throttle_retries
          * @see  Repositories#USE_THROTTLE_RETRIES_SETTING
          */
         Setting<Boolean> USE_THROTTLE_RETRIES_SETTING = Setting.boolSetting("use_throttle_retries",
-            ClientConfiguration.DEFAULT_THROTTLE_RETRIES);
+            ClientConfiguration.DEFAULT_THROTTLE_RETRIES, Property.Deprecated);
         /**
          * chunk_size
          * @see  Repositories#CHUNK_SIZE_SETTING
@@ -245,7 +261,7 @@ public interface Repository {
          * path_style_access
          * @see  Repositories#PATH_STYLE_ACCESS_SETTING
          */
-        Setting<Boolean> PATH_STYLE_ACCESS_SETTING = Setting.boolSetting("path_style_access", false);
+        Setting<Boolean> PATH_STYLE_ACCESS_SETTING = Setting.boolSetting("path_style_access", false, Property.Deprecated);
     }
 
     private final S3BlobStore blobStore;
@@ -259,22 +275,17 @@ public interface Repository {
     /**
      * Constructs an s3 backed repository
      */
-    public S3Repository(RepositoryMetaData metadata, Settings settings, AwsS3Service s3Service) throws IOException {
-        super(metadata, settings);
+    S3Repository(RepositoryMetaData metadata, Settings settings,
+                        NamedXContentRegistry namedXContentRegistry, AwsS3Service s3Service) throws IOException {
+        super(metadata, settings, namedXContentRegistry);
 
         String bucket = getValue(metadata.settings(), settings, Repository.BUCKET_SETTING, Repositories.BUCKET_SETTING);
         if (bucket == null) {
             throw new RepositoryException(metadata.name(), "No bucket defined for s3 gateway");
         }
 
-        String endpoint = getValue(metadata.settings(), settings, Repository.ENDPOINT_SETTING, Repositories.ENDPOINT_SETTING);
-        Protocol protocol = getValue(metadata.settings(), settings, Repository.PROTOCOL_SETTING, Repositories.PROTOCOL_SETTING);
-        String region = getValue(metadata.settings(), settings, Repository.REGION_SETTING, Repositories.REGION_SETTING);
-
         boolean serverSideEncryption = getValue(metadata.settings(), settings, Repository.SERVER_SIDE_ENCRYPTION_SETTING, Repositories.SERVER_SIDE_ENCRYPTION_SETTING);
         ByteSizeValue bufferSize = getValue(metadata.settings(), settings, Repository.BUFFER_SIZE_SETTING, Repositories.BUFFER_SIZE_SETTING);
-        Integer maxRetries = getValue(metadata.settings(), settings, Repository.MAX_RETRIES_SETTING, Repositories.MAX_RETRIES_SETTING);
-        boolean useThrottleRetries = getValue(metadata.settings(), settings, Repository.USE_THROTTLE_RETRIES_SETTING, Repositories.USE_THROTTLE_RETRIES_SETTING);
         this.chunkSize = getValue(metadata.settings(), settings, Repository.CHUNK_SIZE_SETTING, Repositories.CHUNK_SIZE_SETTING);
         this.compress = getValue(metadata.settings(), settings, Repository.COMPRESS_SETTING, Repositories.COMPRESS_SETTING);
 
@@ -288,30 +299,23 @@ public S3Repository(RepositoryMetaData metadata, Settings settings, AwsS3Service
         String storageClass = getValue(metadata.settings(), settings, Repository.STORAGE_CLASS_SETTING, Repositories.STORAGE_CLASS_SETTING);
         String cannedACL = getValue(metadata.settings(), settings, Repository.CANNED_ACL_SETTING, Repositories.CANNED_ACL_SETTING);
 
-        // If the user defined a path style access setting, we rely on it otherwise we use the default
-        // value set by the SDK
-        Boolean pathStyleAccess = null;
-        if (Repository.PATH_STYLE_ACCESS_SETTING.exists(metadata.settings()) ||
-            Repositories.PATH_STYLE_ACCESS_SETTING.exists(settings)) {
-            pathStyleAccess = getValue(metadata.settings(), settings, Repository.PATH_STYLE_ACCESS_SETTING, Repositories.PATH_STYLE_ACCESS_SETTING);
-        }
-
-        logger.debug("using bucket [{}], region [{}], endpoint [{}], protocol [{}], chunk_size [{}], server_side_encryption [{}], " +
-            "buffer_size [{}], max_retries [{}], use_throttle_retries [{}], cannedACL [{}], storageClass [{}], path_style_access [{}]",
-            bucket, region, endpoint, protocol, chunkSize, serverSideEncryption, bufferSize, maxRetries, useThrottleRetries, cannedACL,
-            storageClass, pathStyleAccess);
+        logger.debug("using bucket [{}], chunk_size [{}], server_side_encryption [{}], " +
+            "buffer_size [{}], cannedACL [{}], storageClass [{}]",
+            bucket, chunkSize, serverSideEncryption, bufferSize, cannedACL, storageClass);
 
-        blobStore = new S3BlobStore(settings,
-            s3Service.client(metadata.settings(), endpoint, protocol, region, maxRetries, useThrottleRetries, pathStyleAccess),
-                bucket, region, serverSideEncryption, bufferSize, maxRetries, cannedACL, storageClass);
+        AmazonS3 client = s3Service.client(metadata.settings());
+        String region = InternalAwsS3Service.getRegion(metadata.settings(), settings);
+        blobStore = new S3BlobStore(settings, client,
+                bucket, region, serverSideEncryption, bufferSize, cannedACL, storageClass);
 
         String basePath = getValue(metadata.settings(), settings, Repository.BASE_PATH_SETTING, Repositories.BASE_PATH_SETTING);
         if (Strings.hasLength(basePath)) {
-            BlobPath path = new BlobPath();
-            for(String elem : basePath.split("/")) {
-                path = path.add(elem);
+            if (basePath.startsWith("/")) {
+                basePath = basePath.substring(1);
+                deprecationLogger.deprecated("S3 repository base_path trimming the leading `/`, and " +
+                                                 "leading `/` will not be supported for the S3 repository in future releases");
             }
-            this.basePath = path;
+            this.basePath = new BlobPath().add(basePath);
         } else {
             this.basePath = BlobPath.cleanPath();
         }
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java
new file mode 100644
index 0000000000000..abdc97949826e
--- /dev/null
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.repositories.s3;
+
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import com.amazonaws.util.json.Jackson;
+import org.elasticsearch.SpecialPermission;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.RepositoryPlugin;
+import org.elasticsearch.repositories.Repository;
+
+/**
+ * A plugin to add a repository type that writes to and from the AWS S3.
+ */
+public class S3RepositoryPlugin extends Plugin implements RepositoryPlugin {
+
+    static {
+        SecurityManager sm = System.getSecurityManager();
+        if (sm != null) {
+            sm.checkPermission(new SpecialPermission());
+        }
+        AccessController.doPrivileged(new PrivilegedAction<Void>() {
+            @Override
+            public Void run() {
+                try {
+                    // kick jackson to do some static caching of declared members info
+                    Jackson.jsonNodeOf("{}");
+                    // ClientConfiguration clinit has some classloader problems
+                    // TODO: fix that
+                    Class.forName("com.amazonaws.ClientConfiguration");
+                } catch (ClassNotFoundException e) {
+                    throw new RuntimeException(e);
+                }
+                return null;
+            }
+        });
+    }
+
+    private final Map<String, S3ClientSettings> clientsSettings;
+
+    public S3RepositoryPlugin(Settings settings) {
+        // eagerly load client settings so that secure settings are read
+        clientsSettings = S3ClientSettings.load(settings);
+        assert clientsSettings.isEmpty() == false : "always at least have 'default'";
+    }
+
+    // overridable for tests
+    protected AwsS3Service createStorageService(Settings settings) {
+        return new InternalAwsS3Service(settings, clientsSettings);
+    }
+
+    @Override
+    public Map<String, Repository.Factory> getRepositories(Environment env, NamedXContentRegistry namedXContentRegistry) {
+        return Collections.singletonMap(S3Repository.TYPE,
+            (metadata) -> new S3Repository(metadata, env.settings(), namedXContentRegistry, createStorageService(env.settings())));
+    }
+
+    @Override
+    public List<String> getSettingsFilter() {
+        return Arrays.asList(
+            S3Repository.Repository.KEY_SETTING.getKey(),
+            S3Repository.Repository.SECRET_SETTING.getKey());
+    }
+
+    @Override
+    public List<Setting<?>> getSettings() {
+        return Arrays.asList(
+            // named s3 client configuration settings
+            S3ClientSettings.ACCESS_KEY_SETTING,
+            S3ClientSettings.SECRET_KEY_SETTING,
+            S3ClientSettings.ENDPOINT_SETTING,
+            S3ClientSettings.PROTOCOL_SETTING,
+            S3ClientSettings.PROXY_HOST_SETTING,
+            S3ClientSettings.PROXY_PORT_SETTING,
+            S3ClientSettings.PROXY_USERNAME_SETTING,
+            S3ClientSettings.PROXY_PASSWORD_SETTING,
+            S3ClientSettings.READ_TIMEOUT_SETTING,
+            S3ClientSettings.MAX_RETRIES_SETTING,
+            S3ClientSettings.USE_THROTTLE_RETRIES_SETTING,
+
+            // Register global cloud aws settings: cloud.aws (might have been registered in ec2 plugin)
+            AwsS3Service.KEY_SETTING,
+            AwsS3Service.SECRET_SETTING,
+            AwsS3Service.PROTOCOL_SETTING,
+            AwsS3Service.PROXY_HOST_SETTING,
+            AwsS3Service.PROXY_PORT_SETTING,
+            AwsS3Service.PROXY_USERNAME_SETTING,
+            AwsS3Service.PROXY_PASSWORD_SETTING,
+            AwsS3Service.SIGNER_SETTING,
+            AwsS3Service.REGION_SETTING,
+            AwsS3Service.READ_TIMEOUT,
+
+            // Register S3 specific settings: cloud.aws.s3
+            AwsS3Service.CLOUD_S3.KEY_SETTING,
+            AwsS3Service.CLOUD_S3.SECRET_SETTING,
+            AwsS3Service.CLOUD_S3.PROTOCOL_SETTING,
+            AwsS3Service.CLOUD_S3.PROXY_HOST_SETTING,
+            AwsS3Service.CLOUD_S3.PROXY_PORT_SETTING,
+            AwsS3Service.CLOUD_S3.PROXY_USERNAME_SETTING,
+            AwsS3Service.CLOUD_S3.PROXY_PASSWORD_SETTING,
+            AwsS3Service.CLOUD_S3.SIGNER_SETTING,
+            AwsS3Service.CLOUD_S3.REGION_SETTING,
+            AwsS3Service.CLOUD_S3.ENDPOINT_SETTING,
+            AwsS3Service.CLOUD_S3.READ_TIMEOUT,
+
+            // Register S3 repositories settings: repositories.s3
+            S3Repository.Repositories.KEY_SETTING,
+            S3Repository.Repositories.SECRET_SETTING,
+            S3Repository.Repositories.BUCKET_SETTING,
+            S3Repository.Repositories.REGION_SETTING,
+            S3Repository.Repositories.ENDPOINT_SETTING,
+            S3Repository.Repositories.PROTOCOL_SETTING,
+            S3Repository.Repositories.SERVER_SIDE_ENCRYPTION_SETTING,
+            S3Repository.Repositories.BUFFER_SIZE_SETTING,
+            S3Repository.Repositories.MAX_RETRIES_SETTING,
+            S3Repository.Repositories.CHUNK_SIZE_SETTING,
+            S3Repository.Repositories.COMPRESS_SETTING,
+            S3Repository.Repositories.STORAGE_CLASS_SETTING,
+            S3Repository.Repositories.CANNED_ACL_SETTING,
+            S3Repository.Repositories.BASE_PATH_SETTING,
+            S3Repository.Repositories.USE_THROTTLE_RETRIES_SETTING,
+            S3Repository.Repositories.PATH_STYLE_ACCESS_SETTING);
+    }
+}
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AWSSignersTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AWSSignersTests.java
deleted file mode 100644
index 2e13e04f3c7e0..0000000000000
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AWSSignersTests.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.cloud.aws;
-
-import com.amazonaws.ClientConfiguration;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.plugin.repository.s3.S3RepositoryPlugin;
-import org.elasticsearch.test.ESTestCase;
-import org.junit.BeforeClass;
-
-import static org.hamcrest.CoreMatchers.is;
-
-public class AWSSignersTests extends ESTestCase {
-
-    /**
-     * Starts S3RepositoryPlugin. It's a workaround when you run test from IntelliJ. Otherwise it generates
-     * java.security.AccessControlException: access denied ("java.lang.RuntimePermission" "accessDeclaredMembers")
-     */
-    @BeforeClass
-    public static void instantiatePlugin() {
-        new S3RepositoryPlugin();
-    }
-
-    public void testSigners() {
-        assertThat(signerTester(null), is(false));
-        assertThat(signerTester("QueryStringSignerType"), is(true));
-        assertThat(signerTester("AWS3SignerType"), is(true));
-        assertThat(signerTester("AWS4SignerType"), is(true));
-        assertThat(signerTester("NoOpSignerType"), is(true));
-        assertThat(signerTester("UndefinedSigner"), is(false));
-        assertThat(signerTester("S3SignerType"), is(true));
-        assertThat(signerTester("AWSS3V4SignerType"), is(true));
-
-        ClientConfiguration configuration = new ClientConfiguration();
-        AwsSigner.configureSigner("AWS4SignerType", configuration, "any");
-        assertEquals(configuration.getSignerOverride(), "AWS4SignerType");
-        AwsSigner.configureSigner("S3SignerType", configuration, "any");
-        assertEquals(configuration.getSignerOverride(), "S3SignerType");
-    }
-
-    public void testV2InInvalidRegion() {
-        try {
-            AwsSigner.validateSignerType("S3SignerType", "s3.cn-north-1.amazonaws.com.cn");
-            fail("S3SignerType should not be available for China region");
-        } catch (IllegalArgumentException e) {
-            assertEquals("[S3SignerType] may not be supported in aws Beijing and Frankfurt region", e.getMessage());
-        }
-
-        try {
-            AwsSigner.validateSignerType("S3SignerType", "s3.eu-central-1.amazonaws.com");
-            fail("S3SignerType should not be available for Frankfurt region");
-        } catch (IllegalArgumentException e) {
-            assertEquals("[S3SignerType] may not be supported in aws Beijing and Frankfurt region", e.getMessage());
-        }
-    }
-
-
-    /**
-     * Test a signer configuration
-     * @param signer signer name
-     * @return true if successful, false otherwise
-     */
-    private boolean signerTester(String signer) {
-        try {
-            AwsSigner.validateSignerType(signer, "s3.amazonaws.com");
-            return true;
-        } catch (IllegalArgumentException e) {
-            return false;
-        }
-    }
-}
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTestCase.java b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTestCase.java
deleted file mode 100644
index 51b6296e49e2b..0000000000000
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTestCase.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.cloud.aws;
-
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.io.PathUtils;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsException;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.plugin.repository.s3.S3RepositoryPlugin;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.ESIntegTestCase.ThirdParty;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collection;
-
-/**
- * Base class for AWS tests that require credentials.
- * <p>
- * You must specify {@code -Dtests.thirdparty=true -Dtests.config=/path/to/config}
- * in order to run these tests.
- */
-@ThirdParty
-public abstract class AbstractAwsTestCase extends ESIntegTestCase {
-
-    @Override
-    protected Settings nodeSettings(int nodeOrdinal) {
-        Settings.Builder settings = Settings.builder()
-                .put(super.nodeSettings(nodeOrdinal))
-                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
-                .put("cloud.aws.test.random", randomInt())
-                .put("cloud.aws.test.write_failures", 0.1)
-                .put("cloud.aws.test.read_failures", 0.1);
-
-        // if explicit, just load it and don't load from env
-        try {
-            if (Strings.hasText(System.getProperty("tests.config"))) {
-                try {
-                    settings.loadFromPath(PathUtils.get(System.getProperty("tests.config")));
-                } catch (IOException e) {
-                    throw new IllegalArgumentException("could not load aws tests config", e);
-                }
-            } else {
-                throw new IllegalStateException("to run integration tests, you need to set -Dtests.thirdparty=true and -Dtests.config=/path/to/elasticsearch.yml");
-            }
-        } catch (SettingsException exception) {
-            throw new IllegalStateException("your test configuration file is incorrect: " + System.getProperty("tests.config"), exception);
-        }
-        return settings.build();
-    }
-
-    @Override
-    protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Arrays.asList(TestAwsS3Service.TestPlugin.class);
-    }
-}
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AwsS3ServiceImplTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AwsS3ServiceImplTests.java
deleted file mode 100644
index 3f92a511190e5..0000000000000
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AwsS3ServiceImplTests.java
+++ /dev/null
@@ -1,306 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.cloud.aws;
-
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.Protocol;
-import com.amazonaws.auth.AWSCredentials;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.repositories.s3.S3Repository;
-import org.elasticsearch.test.ESTestCase;
-
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.is;
-
-public class AwsS3ServiceImplTests extends ESTestCase {
-
-    public void testAWSCredentialsWithSystemProviders() {
-        AWSCredentialsProvider credentialsProvider = InternalAwsS3Service.buildCredentials(logger, Settings.EMPTY, Settings.EMPTY);
-        assertThat(credentialsProvider, instanceOf(DefaultAWSCredentialsProviderChain.class));
-    }
-
-    public void testAWSCredentialsWithElasticsearchAwsSettings() {
-        Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
-        Settings settings = Settings.builder()
-            .put(AwsS3Service.KEY_SETTING.getKey(), "aws_key")
-            .put(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret")
-            .build();
-        launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "aws_key", "aws_secret");
-    }
-
-    public void testAWSCredentialsWithElasticsearchS3Settings() {
-        Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
-        Settings settings = Settings.builder()
-            .put(AwsS3Service.CLOUD_S3.KEY_SETTING.getKey(), "s3_key")
-            .put(AwsS3Service.CLOUD_S3.SECRET_SETTING.getKey(), "s3_secret")
-            .build();
-        launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "s3_key", "s3_secret");
-    }
-
-    public void testAWSCredentialsWithElasticsearchAwsAndS3Settings() {
-        Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
-        Settings settings = Settings.builder()
-            .put(AwsS3Service.KEY_SETTING.getKey(), "aws_key")
-            .put(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret")
-            .put(AwsS3Service.CLOUD_S3.KEY_SETTING.getKey(), "s3_key")
-            .put(AwsS3Service.CLOUD_S3.SECRET_SETTING.getKey(), "s3_secret")
-            .build();
-        launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "s3_key", "s3_secret");
-    }
-
-    public void testAWSCredentialsWithElasticsearchRepositoriesSettings() {
-        Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
-        Settings settings = Settings.builder()
-            .put(S3Repository.Repositories.KEY_SETTING.getKey(), "repositories_key")
-            .put(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret")
-            .build();
-        launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repositories_key", "repositories_secret");
-    }
-
-    public void testAWSCredentialsWithElasticsearchAwsAndRepositoriesSettings() {
-        Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
-        Settings settings = Settings.builder()
-            .put(AwsS3Service.KEY_SETTING.getKey(), "aws_key")
-            .put(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret")
-            .put(S3Repository.Repositories.KEY_SETTING.getKey(), "repositories_key")
-            .put(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret")
-            .build();
-        launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repositories_key", "repositories_secret");
-    }
-
-    public void testAWSCredentialsWithElasticsearchAwsAndS3AndRepositoriesSettings() {
-        Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
-        Settings settings = Settings.builder()
-            .put(AwsS3Service.KEY_SETTING.getKey(), "aws_key")
-            .put(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret")
-            .put(AwsS3Service.CLOUD_S3.KEY_SETTING.getKey(), "s3_key")
-            .put(AwsS3Service.CLOUD_S3.SECRET_SETTING.getKey(), "s3_secret")
-            .put(S3Repository.Repositories.KEY_SETTING.getKey(), "repositories_key")
-            .put(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret")
-            .build();
-        launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repositories_key", "repositories_secret");
-    }
-
-    public void testAWSCredentialsWithElasticsearchRepositoriesSettingsAndRepositorySettings() {
-        Settings repositorySettings = generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null);
-        Settings settings = Settings.builder()
-            .put(S3Repository.Repositories.KEY_SETTING.getKey(), "repositories_key")
-            .put(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret")
-            .build();
-        launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repository_key", "repository_secret");
-    }
-
-    public void testAWSCredentialsWithElasticsearchAwsAndRepositoriesSettingsAndRepositorySettings() {
-        Settings repositorySettings = generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null);
-        Settings settings = Settings.builder()
-            .put(AwsS3Service.KEY_SETTING.getKey(), "aws_key")
-            .put(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret")
-            .put(S3Repository.Repositories.KEY_SETTING.getKey(), "repositories_key")
-            .put(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret")
-            .build();
-        launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repository_key", "repository_secret");
-    }
-
-    public void testAWSCredentialsWithElasticsearchAwsAndS3AndRepositoriesSettingsAndRepositorySettings() {
-        Settings repositorySettings = generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null);
-        Settings settings = Settings.builder()
-            .put(AwsS3Service.KEY_SETTING.getKey(), "aws_key")
-            .put(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret")
-            .put(AwsS3Service.CLOUD_S3.KEY_SETTING.getKey(), "s3_key")
-            .put(AwsS3Service.CLOUD_S3.SECRET_SETTING.getKey(), "s3_secret")
-            .put(S3Repository.Repositories.KEY_SETTING.getKey(), "repositories_key")
-            .put(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret")
-            .build();
-        launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repository_key", "repository_secret");
-    }
-
-    protected void launchAWSCredentialsWithElasticsearchSettingsTest(Settings singleRepositorySettings, Settings settings,
-                                                                     String expectedKey, String expectedSecret) {
-        AWSCredentials credentials = InternalAwsS3Service.buildCredentials(logger, settings, singleRepositorySettings).getCredentials();
-        assertThat(credentials.getAWSAccessKeyId(), is(expectedKey));
-        assertThat(credentials.getAWSSecretKey(), is(expectedSecret));
-    }
-
-    public void testAWSDefaultConfiguration() {
-        Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
-        launchAWSConfigurationTest(Settings.EMPTY, repositorySettings, Protocol.HTTPS, null, -1, null, null, null, 3, false);
-    }
-
-    public void testAWSConfigurationWithAwsSettings() {
-        Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
-        Settings settings = Settings.builder()
-            .put(AwsS3Service.PROTOCOL_SETTING.getKey(), "http")
-            .put(AwsS3Service.PROXY_HOST_SETTING.getKey(), "aws_proxy_host")
-            .put(AwsS3Service.PROXY_PORT_SETTING.getKey(), 8080)
-            .put(AwsS3Service.PROXY_USERNAME_SETTING.getKey(), "aws_proxy_username")
-            .put(AwsS3Service.PROXY_PASSWORD_SETTING.getKey(), "aws_proxy_password")
-            .put(AwsS3Service.SIGNER_SETTING.getKey(), "AWS3SignerType")
-            .build();
-        launchAWSConfigurationTest(settings, repositorySettings, Protocol.HTTP, "aws_proxy_host", 8080, "aws_proxy_username",
-            "aws_proxy_password", "AWS3SignerType", 3, false);
-    }
-
-    public void testAWSConfigurationWithAwsAndS3Settings() {
-        Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
-        Settings settings = Settings.builder()
-            .put(AwsS3Service.PROTOCOL_SETTING.getKey(), "http")
-            .put(AwsS3Service.PROXY_HOST_SETTING.getKey(), "aws_proxy_host")
-            .put(AwsS3Service.PROXY_PORT_SETTING.getKey(), 8080)
-            .put(AwsS3Service.PROXY_USERNAME_SETTING.getKey(), "aws_proxy_username")
-            .put(AwsS3Service.PROXY_PASSWORD_SETTING.getKey(), "aws_proxy_password")
-            .put(AwsS3Service.SIGNER_SETTING.getKey(), "AWS3SignerType")
-            .put(AwsS3Service.CLOUD_S3.PROTOCOL_SETTING.getKey(), "https")
-            .put(AwsS3Service.CLOUD_S3.PROXY_HOST_SETTING.getKey(), "s3_proxy_host")
-            .put(AwsS3Service.CLOUD_S3.PROXY_PORT_SETTING.getKey(), 8081)
-            .put(AwsS3Service.CLOUD_S3.PROXY_USERNAME_SETTING.getKey(), "s3_proxy_username")
-            .put(AwsS3Service.CLOUD_S3.PROXY_PASSWORD_SETTING.getKey(), "s3_proxy_password")
-            .put(AwsS3Service.CLOUD_S3.SIGNER_SETTING.getKey(), "NoOpSignerType")
-            .build();
-        launchAWSConfigurationTest(settings, repositorySettings, Protocol.HTTPS, "s3_proxy_host", 8081, "s3_proxy_username",
-            "s3_proxy_password", "NoOpSignerType", 3, false);
-    }
-
-    protected void launchAWSConfigurationTest(Settings settings,
-                                              Settings singleRepositorySettings,
-                                              Protocol expectedProtocol,
-                                              String expectedProxyHost,
-                                              int expectedProxyPort,
-                                              String expectedProxyUsername,
-                                              String expectedProxyPassword,
-                                              String expectedSigner,
-                                              Integer expectedMaxRetries,
-                                              boolean expectedUseThrottleRetries) {
-        Protocol protocol = S3Repository.getValue(singleRepositorySettings, settings,
-            S3Repository.Repository.PROTOCOL_SETTING, S3Repository.Repositories.PROTOCOL_SETTING);
-        Integer maxRetries = S3Repository.getValue(singleRepositorySettings, settings,
-            S3Repository.Repository.MAX_RETRIES_SETTING, S3Repository.Repositories.MAX_RETRIES_SETTING);
-        Boolean useThrottleRetries = S3Repository.getValue(singleRepositorySettings, settings,
-            S3Repository.Repository.USE_THROTTLE_RETRIES_SETTING, S3Repository.Repositories.USE_THROTTLE_RETRIES_SETTING);
-
-        ClientConfiguration configuration = InternalAwsS3Service.buildConfiguration(logger, settings, protocol, maxRetries, null,
-            useThrottleRetries);
-
-        assertThat(configuration.getResponseMetadataCacheSize(), is(0));
-        assertThat(configuration.getProtocol(), is(expectedProtocol));
-        assertThat(configuration.getProxyHost(), is(expectedProxyHost));
-        assertThat(configuration.getProxyPort(), is(expectedProxyPort));
-        assertThat(configuration.getProxyUsername(), is(expectedProxyUsername));
-        assertThat(configuration.getProxyPassword(), is(expectedProxyPassword));
-        assertThat(configuration.getSignerOverride(), is(expectedSigner));
-        assertThat(configuration.getMaxErrorRetry(), is(expectedMaxRetries));
-        assertThat(configuration.useThrottledRetries(), is(expectedUseThrottleRetries));
-    }
-
-    private static Settings generateRepositorySettings(String key, String secret, String region, String endpoint, Integer maxRetries) {
-        Settings.Builder builder = Settings.builder();
-        if (region != null) {
-            builder.put(S3Repository.Repository.REGION_SETTING.getKey(), region);
-        }
-        if (endpoint != null) {
-            builder.put(S3Repository.Repository.ENDPOINT_SETTING.getKey(), endpoint);
-        }
-        if (key != null) {
-            builder.put(S3Repository.Repository.KEY_SETTING.getKey(), key);
-        }
-        if (secret != null) {
-            builder.put(S3Repository.Repository.SECRET_SETTING.getKey(), secret);
-        }
-        if (maxRetries != null) {
-            builder.put(S3Repository.Repository.MAX_RETRIES_SETTING.getKey(), maxRetries);
-        }
-        return builder.build();
-    }
-
-    public void testDefaultEndpoint() {
-        launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, null, null), Settings.EMPTY, "");
-        launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null), Settings.EMPTY,
-            "s3.eu-central-1.amazonaws.com");
-        launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, "repository.endpoint", null),
-            Settings.EMPTY, "repository.endpoint");
-    }
-
-    public void testSpecificEndpoint() {
-        Settings settings = Settings.builder()
-            .put(InternalAwsS3Service.CLOUD_S3.ENDPOINT_SETTING.getKey(), "ec2.endpoint")
-            .build();
-        launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings,
-            "ec2.endpoint");
-        // Endpoint has precedence on region. Whatever region we set, we won't use it
-        launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null), settings,
-            "ec2.endpoint");
-        launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, "repository.endpoint", null),
-            settings, "repository.endpoint");
-    }
-
-    public void testRegionWithAwsSettings() {
-        Settings settings = Settings.builder()
-            .put(InternalAwsS3Service.REGION_SETTING.getKey(), randomFrom("eu-west", "eu-west-1"))
-            .build();
-        launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings,
-            "s3-eu-west-1.amazonaws.com");
-        launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null), settings,
-            "s3.eu-central-1.amazonaws.com");
-        launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, "repository.endpoint", null),
-            settings, "repository.endpoint");
-    }
-
-    public void testRegionWithAwsAndS3Settings() {
-        Settings settings = Settings.builder()
-            .put(InternalAwsS3Service.REGION_SETTING.getKey(), randomFrom("eu-west", "eu-west-1"))
-            .put(InternalAwsS3Service.CLOUD_S3.REGION_SETTING.getKey(), randomFrom("us-west", "us-west-1"))
-            .build();
-        launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings,
-            "s3-us-west-1.amazonaws.com");
-        launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null), settings,
-            "s3.eu-central-1.amazonaws.com");
-        launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, "repository.endpoint", null),
-            settings, "repository.endpoint");
-    }
-
-    public void testInvalidRegion() {
-        Settings settings = Settings.builder()
-            .put(InternalAwsS3Service.REGION_SETTING.getKey(), "does-not-exist")
-            .build();
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {
-            launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings, null);
-        });
-        assertThat(e.getMessage(), containsString("No automatic endpoint could be derived from region"));
-
-        launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null), settings,
-            "s3.eu-central-1.amazonaws.com");
-        launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, "repository.endpoint", null),
-            settings, "repository.endpoint");
-    }
-
-    protected void launchAWSEndpointTest(Settings singleRepositorySettings, Settings settings,
-                                         String expectedEndpoint) {
-        String region = S3Repository.getValue(singleRepositorySettings, settings,
-            S3Repository.Repository.REGION_SETTING, S3Repository.Repositories.REGION_SETTING);
-        String endpoint = S3Repository.getValue(singleRepositorySettings, settings,
-            S3Repository.Repository.ENDPOINT_SETTING, S3Repository.Repositories.ENDPOINT_SETTING);
-
-        String foundEndpoint = InternalAwsS3Service.findEndpoint(logger, settings, endpoint, region);
-        assertThat(foundEndpoint, is(expectedEndpoint));
-    }
-
-}
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AWSSignersTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AWSSignersTests.java
new file mode 100644
index 0000000000000..cc33fcc243eed
--- /dev/null
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AWSSignersTests.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.repositories.s3;
+
+import com.amazonaws.ClientConfiguration;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.repositories.s3.AwsSigner;
+import org.elasticsearch.repositories.s3.S3RepositoryPlugin;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.BeforeClass;
+
+import static org.hamcrest.CoreMatchers.is;
+
+public class AWSSignersTests extends ESTestCase {
+
+    /**
+     * Starts S3RepositoryPlugin. It's a workaround when you run test from IntelliJ. Otherwise it generates
+     * java.security.AccessControlException: access denied ("java.lang.RuntimePermission" "accessDeclaredMembers")
+     */
+    @BeforeClass
+    public static void instantiatePlugin() {
+        new S3RepositoryPlugin(Settings.EMPTY);
+    }
+
+    public void testSigners() {
+        assertThat(signerTester(null), is(false));
+        assertThat(signerTester("QueryStringSignerType"), is(true));
+        assertThat(signerTester("AWS3SignerType"), is(true));
+        assertThat(signerTester("AWS4SignerType"), is(true));
+        assertThat(signerTester("NoOpSignerType"), is(true));
+        assertThat(signerTester("UndefinedSigner"), is(false));
+        assertThat(signerTester("S3SignerType"), is(true));
+        assertThat(signerTester("AWSS3V4SignerType"), is(true));
+
+        ClientConfiguration configuration = new ClientConfiguration();
+        AwsSigner.configureSigner("AWS4SignerType", configuration, "any");
+        assertEquals(configuration.getSignerOverride(), "AWS4SignerType");
+        AwsSigner.configureSigner("S3SignerType", configuration, "any");
+        assertEquals(configuration.getSignerOverride(), "S3SignerType");
+    }
+
+    public void testV2InInvalidRegion() {
+        try {
+            AwsSigner.validateSignerType("S3SignerType", "s3.cn-north-1.amazonaws.com.cn");
+            fail("S3SignerType should not be available for China region");
+        } catch (IllegalArgumentException e) {
+            assertEquals("[S3SignerType] may not be supported in aws Beijing and Frankfurt region", e.getMessage());
+        }
+
+        try {
+            AwsSigner.validateSignerType("S3SignerType", "s3.eu-central-1.amazonaws.com");
+            fail("S3SignerType should not be available for Frankfurt region");
+        } catch (IllegalArgumentException e) {
+            assertEquals("[S3SignerType] may not be supported in aws Beijing and Frankfurt region", e.getMessage());
+        }
+    }
+
+
+    /**
+     * Test a signer configuration
+     * @param signer signer name
+     * @return true if successful, false otherwise
+     */
+    private boolean signerTester(String signer) {
+        try {
+            AwsSigner.validateSignerType(signer, "s3.amazonaws.com");
+            return true;
+        } catch (IllegalArgumentException e) {
+            return false;
+        }
+    }
+}
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractAwsTestCase.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractAwsTestCase.java
new file mode 100644
index 0000000000000..f004145e7ff58
--- /dev/null
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractAwsTestCase.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.repositories.s3;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.PathUtils;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsException;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.ESIntegTestCase.ThirdParty;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+
+/**
+ * Base class for AWS tests that require credentials.
+ * <p>
+ * You must specify {@code -Dtests.thirdparty=true -Dtests.config=/path/to/config}
+ * in order to run these tests.
+ */
+@ThirdParty
+public abstract class AbstractAwsTestCase extends ESIntegTestCase {
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        Settings.Builder settings = Settings.builder()
+                .put(super.nodeSettings(nodeOrdinal))
+                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
+                .put("cloud.aws.test.random", randomInt())
+                .put("cloud.aws.test.write_failures", 0.1)
+                .put("cloud.aws.test.read_failures", 0.1);
+
+        // if explicit, just load it and don't load from env
+        try {
+            if (Strings.hasText(System.getProperty("tests.config"))) {
+                try {
+                    settings.loadFromPath(PathUtils.get(System.getProperty("tests.config")));
+                } catch (IOException e) {
+                    throw new IllegalArgumentException("could not load aws tests config", e);
+                }
+            } else {
+                throw new IllegalStateException("to run integration tests, you need to set -Dtests.thirdparty=true and -Dtests.config=/path/to/elasticsearch.yml");
+            }
+        } catch (SettingsException exception) {
+            throw new IllegalStateException("your test configuration file is incorrect: " + System.getProperty("tests.config"), exception);
+        }
+        return settings.build();
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Arrays.asList(TestAwsS3Service.TestPlugin.class);
+    }
+}
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java
index 4d7f30ed9db21..90c47cb3be227 100644
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java
@@ -31,9 +31,8 @@
 import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.ClusterAdminClient;
-import org.elasticsearch.cloud.aws.AbstractAwsTestCase;
-import org.elasticsearch.cloud.aws.AwsS3Service;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.RepositoryMetaData;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.repositories.RepositoryMissingException;
 import org.elasticsearch.repositories.RepositoryVerificationException;
@@ -168,6 +167,7 @@ public void testEncryption() {
             .put(S3Repository.Repository.BASE_PATH_SETTING.getKey(), basePath)
             .put(S3Repository.Repository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(1000, 10000))
             .put(S3Repository.Repository.SERVER_SIDE_ENCRYPTION_SETTING.getKey(), true)
+            .put(S3Repository.Repository.USE_THROTTLE_RETRIES_SETTING.getKey(), randomBoolean())
             .build();
 
         PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo")
@@ -197,12 +197,8 @@ public void testEncryption() {
 
         Settings settings = internalCluster().getInstance(Settings.class);
         Settings bucket = settings.getByPrefix("repositories.s3.");
-        AmazonS3 s3Client = internalCluster().getInstance(AwsS3Service.class).client(
-            repositorySettings,
-            null,
-            S3Repository.Repositories.PROTOCOL_SETTING.get(settings),
-            S3Repository.Repositories.REGION_SETTING.get(settings),
-            null, randomBoolean(), null);
+        RepositoryMetaData metadata = new RepositoryMetaData("test-repo", "fs", Settings.EMPTY);
+        AmazonS3 s3Client = internalCluster().getInstance(AwsS3Service.class).client(repositorySettings);
 
         String bucketName = bucket.get("bucket");
         logger.info("--> verify encryption for bucket [{}], prefix [{}]", bucketName, basePath);
@@ -467,16 +463,13 @@ public void cleanRepositoryFiles(String basePath) {
                 settings.getByPrefix("repositories.s3.external-bucket.")
         };
         for (Settings bucket : buckets) {
-            String endpoint = bucket.get("endpoint", S3Repository.Repositories.ENDPOINT_SETTING.get(settings));
-            Protocol protocol = S3Repository.Repositories.PROTOCOL_SETTING.get(settings);
-            String region = bucket.get("region", S3Repository.Repositories.REGION_SETTING.get(settings));
             String bucketName = bucket.get("bucket");
 
             // We check that settings has been set in elasticsearch.yml integration test file
             // as described in README
-            assertThat("Your settings in elasticsearch.yml are incorrects. Check README file.", bucketName, notNullValue());
-            AmazonS3 client = internalCluster().getInstance(AwsS3Service.class).client(Settings.EMPTY, endpoint, protocol, region, null,
-                randomBoolean(), null);
+            assertThat("Your settings in elasticsearch.yml are incorrect. Check README file.", bucketName, notNullValue());
+            AmazonS3 client = internalCluster().getInstance(AwsS3Service.class).client(
+                Settings.builder().put(S3Repository.Repository.USE_THROTTLE_RETRIES_SETTING.getKey(), randomBoolean()).build());
             try {
                 ObjectListing prevListing = null;
                 //From http://docs.amazonwebservices.com/AmazonS3/latest/dev/DeletingMultipleObjectsUsingJava.html
@@ -513,7 +506,7 @@ public void cleanRepositoryFiles(String basePath) {
                     client.deleteObjects(multiObjectDeleteRequest);
                 }
             } catch (Exception ex) {
-                logger.warn((Supplier<?>) () -> new ParameterizedMessage("Failed to delete S3 repository [{}] in [{}]", bucketName, region), ex);
+                logger.warn((Supplier<?>) () -> new ParameterizedMessage("Failed to delete S3 repository [{}]", bucketName), ex);
             }
         }
     }
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AmazonS3Wrapper.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Wrapper.java
similarity index 99%
rename from plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AmazonS3Wrapper.java
rename to plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Wrapper.java
index 7cda9ee094791..40dc80fb492b2 100644
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AmazonS3Wrapper.java
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Wrapper.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.cloud.aws;
+package org.elasticsearch.repositories.s3;
 
 import com.amazonaws.AmazonClientException;
 import com.amazonaws.AmazonServiceException;
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AwsS3ServiceImplTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AwsS3ServiceImplTests.java
new file mode 100644
index 0000000000000..774d261cef470
--- /dev/null
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AwsS3ServiceImplTests.java
@@ -0,0 +1,497 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.repositories.s3;
+
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.Protocol;
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.auth.InstanceProfileCredentialsProvider;
+import org.elasticsearch.common.settings.MockSecureSettings;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ESTestCase;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
+
+public class AwsS3ServiceImplTests extends ESTestCase {
+
+    public void testAWSCredentialsWithSystemProviders() {
+        S3ClientSettings clientSettings = S3ClientSettings.getClientSettings(Settings.EMPTY, "default");
+        AWSCredentialsProvider credentialsProvider = InternalAwsS3Service.buildCredentials(logger, deprecationLogger, clientSettings, Settings.EMPTY);
+        assertThat(credentialsProvider, instanceOf(InstanceProfileCredentialsProvider.class));
+    }
+
+    public void testAwsCredsDefaultSettings() {
+        Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
+        MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("s3.client.default.access_key", "aws_key");
+        secureSettings.setString("s3.client.default.secret_key", "aws_secret");
+        Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
+        launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "aws_key", "aws_secret");
+    }
+
+    public void testAwsCredsExplicitConfigSettings() {
+        Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
+        repositorySettings = Settings.builder().put(repositorySettings)
+            .put(InternalAwsS3Service.CLIENT_NAME.getKey(), "myconfig").build();
+        MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("s3.client.myconfig.access_key", "aws_key");
+        secureSettings.setString("s3.client.myconfig.secret_key", "aws_secret");
+        secureSettings.setString("s3.client.default.access_key", "wrong_key");
+        secureSettings.setString("s3.client.default.secret_key", "wrong_secret");
+        Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
+        launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "aws_key", "aws_secret");
+    }
+
+    public void testAWSCredentialsWithElasticsearchAwsSettingsBackcompat() {
+        Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
+        Settings settings = Settings.builder()
+            .put(AwsS3Service.KEY_SETTING.getKey(), "aws_key")
+            .put(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret")
+            .build();
+        launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "aws_key", "aws_secret");
+         assertSettingDeprecationsAndWarnings(new Setting<?>[]{AwsS3Service.KEY_SETTING, AwsS3Service.SECRET_SETTING});
+    }
+
+    public void testAWSCredentialsWithElasticsearchS3SettingsBackcompat() {
+        Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
+        Settings settings = Settings.builder()
+            .put(AwsS3Service.CLOUD_S3.KEY_SETTING.getKey(), "s3_key")
+            .put(AwsS3Service.CLOUD_S3.SECRET_SETTING.getKey(), "s3_secret")
+            .build();
+        launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "s3_key", "s3_secret");
+         assertSettingDeprecationsAndWarnings(new Setting<?>[]{AwsS3Service.CLOUD_S3.KEY_SETTING, AwsS3Service.CLOUD_S3.SECRET_SETTING});
+    }
+
+    public void testAWSCredentialsWithElasticsearchAwsAndS3SettingsBackcompat() {
+        Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
+        Settings settings = Settings.builder()
+            .put(AwsS3Service.KEY_SETTING.getKey(), "aws_key")
+            .put(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret")
+            .put(AwsS3Service.CLOUD_S3.KEY_SETTING.getKey(), "s3_key")
+            .put(AwsS3Service.CLOUD_S3.SECRET_SETTING.getKey(), "s3_secret")
+            .build();
+        launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "s3_key", "s3_secret");
+         assertSettingDeprecationsAndWarnings(new Setting<?>[]{
+                AwsS3Service.KEY_SETTING,
+                AwsS3Service.SECRET_SETTING,
+                AwsS3Service.CLOUD_S3.KEY_SETTING,
+                AwsS3Service.CLOUD_S3.SECRET_SETTING});
+    }
+
+    public void testAWSCredentialsWithElasticsearchRepositoriesSettingsBackcompat() {
+        Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
+        Settings settings = Settings.builder()
+            .put(S3Repository.Repositories.KEY_SETTING.getKey(), "repositories_key")
+            .put(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret")
+            .build();
+        launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repositories_key", "repositories_secret");
+         assertSettingDeprecationsAndWarnings(
+                 new Setting<?>[]{S3Repository.Repositories.KEY_SETTING, S3Repository.Repositories.SECRET_SETTING});
+    }
+
+    public void testAWSCredentialsWithElasticsearchAwsAndRepositoriesSettingsBackcompat() {
+        Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
+        Settings settings = Settings.builder()
+            .put(AwsS3Service.KEY_SETTING.getKey(), "aws_key")
+            .put(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret")
+            .put(S3Repository.Repositories.KEY_SETTING.getKey(), "repositories_key")
+            .put(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret")
+            .build();
+        launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repositories_key", "repositories_secret");
+         assertSettingDeprecationsAndWarnings(new Setting<?>[] {
+                AwsS3Service.KEY_SETTING,
+                AwsS3Service.SECRET_SETTING,
+                S3Repository.Repositories.KEY_SETTING,
+                S3Repository.Repositories.SECRET_SETTING});
+    }
+
+    public void testAWSCredentialsWithElasticsearchAwsAndS3AndRepositoriesSettingsBackcompat() {
+        Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
+        Settings settings = Settings.builder()
+            .put(AwsS3Service.KEY_SETTING.getKey(), "aws_key")
+            .put(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret")
+            .put(AwsS3Service.CLOUD_S3.KEY_SETTING.getKey(), "s3_key")
+            .put(AwsS3Service.CLOUD_S3.SECRET_SETTING.getKey(), "s3_secret")
+            .put(S3Repository.Repositories.KEY_SETTING.getKey(), "repositories_key")
+            .put(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret")
+            .build();
+        launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repositories_key", "repositories_secret");
+         assertSettingDeprecationsAndWarnings(new Setting<?>[] {
+                AwsS3Service.KEY_SETTING,
+                AwsS3Service.SECRET_SETTING,
+                AwsS3Service.CLOUD_S3.KEY_SETTING,
+                AwsS3Service.CLOUD_S3.SECRET_SETTING,
+                S3Repository.Repositories.KEY_SETTING,
+                S3Repository.Repositories.SECRET_SETTING});
+    }
+
+    public void testAWSCredentialsWithElasticsearchRepositoriesSettingsAndRepositorySettingsBackcompat() {
+        Settings repositorySettings = generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null);
+        Settings settings = Settings.builder()
+            .put(S3Repository.Repositories.KEY_SETTING.getKey(), "repositories_key")
+            .put(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret")
+            .build();
+        launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repository_key", "repository_secret");
+        assertSettingDeprecationsAndWarnings(new Setting<?>[]{
+            S3Repository.Repositories.KEY_SETTING,
+            S3Repository.Repositories.SECRET_SETTING,
+            S3Repository.Repository.KEY_SETTING,
+            S3Repository.Repository.SECRET_SETTING},
+            "Using s3 access/secret key from repository settings. Instead store these in named clients and the elasticsearch keystore for secure settings.");
+    }
+
+    public void testAWSCredentialsWithElasticsearchAwsAndRepositoriesSettingsAndRepositorySettingsBackcompat() {
+        Settings repositorySettings = generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null);
+        Settings settings = Settings.builder()
+            .put(AwsS3Service.KEY_SETTING.getKey(), "aws_key")
+            .put(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret")
+            .put(S3Repository.Repositories.KEY_SETTING.getKey(), "repositories_key")
+            .put(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret")
+            .build();
+        launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repository_key", "repository_secret");
+        assertSettingDeprecationsAndWarnings(new Setting<?>[]{
+            AwsS3Service.KEY_SETTING,
+            AwsS3Service.SECRET_SETTING,
+            S3Repository.Repositories.KEY_SETTING,
+            S3Repository.Repositories.SECRET_SETTING,
+            S3Repository.Repository.KEY_SETTING,
+            S3Repository.Repository.SECRET_SETTING},
+            "Using s3 access/secret key from repository settings. Instead store these in named clients and the elasticsearch keystore for secure settings.");
+    }
+
+    public void testAWSCredentialsWithElasticsearchAwsAndS3AndRepositoriesSettingsAndRepositorySettingsBackcompat() {
+        Settings repositorySettings = generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null);
+        Settings settings = Settings.builder()
+            .put(AwsS3Service.KEY_SETTING.getKey(), "aws_key")
+            .put(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret")
+            .put(AwsS3Service.CLOUD_S3.KEY_SETTING.getKey(), "s3_key")
+            .put(AwsS3Service.CLOUD_S3.SECRET_SETTING.getKey(), "s3_secret")
+            .put(S3Repository.Repositories.KEY_SETTING.getKey(), "repositories_key")
+            .put(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret")
+            .build();
+        launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repository_key", "repository_secret");
+        assertSettingDeprecationsAndWarnings(new Setting<?>[]{
+            AwsS3Service.KEY_SETTING,
+            AwsS3Service.SECRET_SETTING,
+            AwsS3Service.CLOUD_S3.KEY_SETTING,
+            AwsS3Service.CLOUD_S3.SECRET_SETTING,
+            S3Repository.Repositories.KEY_SETTING,
+            S3Repository.Repositories.SECRET_SETTING,
+            S3Repository.Repository.KEY_SETTING,
+            S3Repository.Repository.SECRET_SETTING},
+            "Using s3 access/secret key from repository settings. Instead store these in named clients and the elasticsearch keystore for secure settings.");
+    }
+
+    protected void launchAWSCredentialsWithElasticsearchSettingsTest(Settings singleRepositorySettings, Settings settings,
+                                                                     String expectedKey, String expectedSecret) {
+        String configName = InternalAwsS3Service.CLIENT_NAME.get(singleRepositorySettings);
+        S3ClientSettings clientSettings = S3ClientSettings.getClientSettings(settings, configName);
+        AWSCredentials credentials = InternalAwsS3Service
+            .buildCredentials(logger, deprecationLogger, clientSettings, singleRepositorySettings)
+            .getCredentials();
+        assertThat(credentials.getAWSAccessKeyId(), is(expectedKey));
+        assertThat(credentials.getAWSSecretKey(), is(expectedSecret));
+    }
+
+    public void testAWSDefaultConfiguration() {
+        Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
+        launchAWSConfigurationTest(Settings.EMPTY, repositorySettings, Protocol.HTTPS, null, -1, null, null, null, 3, false,
+            ClientConfiguration.DEFAULT_SOCKET_TIMEOUT);
+    }
+
+    public void testAWSConfigurationWithAwsSettings() {
+        Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
+        MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("s3.client.default.proxy.username", "aws_proxy_username");
+        secureSettings.setString("s3.client.default.proxy.password", "aws_proxy_password");
+        Settings settings = Settings.builder()
+            .setSecureSettings(secureSettings)
+            .put("s3.client.default.protocol", "http")
+            .put("s3.client.default.proxy.host", "aws_proxy_host")
+            .put("s3.client.default.proxy.port", 8080)
+            .put("s3.client.default.read_timeout", "10s")
+            .build();
+        launchAWSConfigurationTest(settings, repositorySettings, Protocol.HTTP, "aws_proxy_host", 8080, "aws_proxy_username",
+            "aws_proxy_password", null, 3, false, 10000);
+    }
+
+    public void testAWSConfigurationWithAwsSettingsBackcompat() {
+        Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
+        Settings settings = Settings.builder()
+            .put(AwsS3Service.PROTOCOL_SETTING.getKey(), "http")
+            .put(AwsS3Service.PROXY_HOST_SETTING.getKey(), "aws_proxy_host")
+            .put(AwsS3Service.PROXY_PORT_SETTING.getKey(), 8080)
+            .put(AwsS3Service.PROXY_USERNAME_SETTING.getKey(), "aws_proxy_username")
+            .put(AwsS3Service.PROXY_PASSWORD_SETTING.getKey(), "aws_proxy_password")
+            .put(AwsS3Service.SIGNER_SETTING.getKey(), "AWS3SignerType")
+            .put(AwsS3Service.READ_TIMEOUT.getKey(), "10s")
+            .build();
+        launchAWSConfigurationTest(settings, repositorySettings, Protocol.HTTP, "aws_proxy_host", 8080, "aws_proxy_username",
+            "aws_proxy_password", "AWS3SignerType", 3, false, 10000);
+         assertSettingDeprecationsAndWarnings(new Setting<?>[]{
+                AwsS3Service.PROXY_USERNAME_SETTING,
+                AwsS3Service.PROXY_PASSWORD_SETTING,
+                AwsS3Service.PROTOCOL_SETTING,
+                AwsS3Service.PROXY_HOST_SETTING,
+                AwsS3Service.PROXY_PORT_SETTING,
+                AwsS3Service.SIGNER_SETTING,
+                AwsS3Service.READ_TIMEOUT});
+    }
+
+    public void testAWSConfigurationWithAwsAndS3SettingsBackcompat() {
+        Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
+        Settings settings = Settings.builder()
+            .put(AwsS3Service.PROTOCOL_SETTING.getKey(), "http")
+            .put(AwsS3Service.PROXY_HOST_SETTING.getKey(), "aws_proxy_host")
+            .put(AwsS3Service.PROXY_PORT_SETTING.getKey(), 8080)
+            .put(AwsS3Service.PROXY_USERNAME_SETTING.getKey(), "aws_proxy_username")
+            .put(AwsS3Service.PROXY_PASSWORD_SETTING.getKey(), "aws_proxy_password")
+            .put(AwsS3Service.SIGNER_SETTING.getKey(), "AWS3SignerType")
+            .put(AwsS3Service.READ_TIMEOUT.getKey(), "5s")
+            .put(AwsS3Service.CLOUD_S3.PROTOCOL_SETTING.getKey(), "https")
+            .put(AwsS3Service.CLOUD_S3.PROXY_HOST_SETTING.getKey(), "s3_proxy_host")
+            .put(AwsS3Service.CLOUD_S3.PROXY_PORT_SETTING.getKey(), 8081)
+            .put(AwsS3Service.CLOUD_S3.PROXY_USERNAME_SETTING.getKey(), "s3_proxy_username")
+            .put(AwsS3Service.CLOUD_S3.PROXY_PASSWORD_SETTING.getKey(), "s3_proxy_password")
+            .put(AwsS3Service.CLOUD_S3.SIGNER_SETTING.getKey(), "NoOpSignerType")
+            .put(AwsS3Service.CLOUD_S3.READ_TIMEOUT.getKey(), "10s")
+            .build();
+        launchAWSConfigurationTest(settings, repositorySettings, Protocol.HTTPS, "s3_proxy_host", 8081, "s3_proxy_username",
+            "s3_proxy_password", "NoOpSignerType", 3, false, 10000);
+         assertSettingDeprecationsAndWarnings(new Setting<?>[] {
+                AwsS3Service.PROXY_USERNAME_SETTING,
+                AwsS3Service.PROXY_PASSWORD_SETTING,
+                AwsS3Service.PROTOCOL_SETTING,
+                AwsS3Service.PROXY_HOST_SETTING,
+                AwsS3Service.PROXY_PORT_SETTING,
+                AwsS3Service.SIGNER_SETTING,
+                AwsS3Service.READ_TIMEOUT,
+                AwsS3Service.CLOUD_S3.PROXY_USERNAME_SETTING,
+                AwsS3Service.CLOUD_S3.PROXY_PASSWORD_SETTING,
+                AwsS3Service.CLOUD_S3.PROTOCOL_SETTING,
+                AwsS3Service.CLOUD_S3.PROXY_HOST_SETTING,
+                AwsS3Service.CLOUD_S3.PROXY_PORT_SETTING,
+                AwsS3Service.CLOUD_S3.SIGNER_SETTING,
+                AwsS3Service.CLOUD_S3.READ_TIMEOUT});
+    }
+
+    public void testGlobalMaxRetriesBackcompat() {
+        Settings settings = Settings.builder()
+            .put(S3Repository.Repositories.MAX_RETRIES_SETTING.getKey(), 10)
+            .build();
+        launchAWSConfigurationTest(settings, Settings.EMPTY, Protocol.HTTPS, null, -1, null, null,
+            null, 10, false, 50000);
+        assertSettingDeprecationsAndWarnings(new Setting<?>[]{
+            S3Repository.Repositories.MAX_RETRIES_SETTING
+        });
+    }
+
+    public void testRepositoryMaxRetries() {
+        Settings settings = Settings.builder()
+            .put("s3.client.default.max_retries", 5)
+            .build();
+        launchAWSConfigurationTest(settings, Settings.EMPTY, Protocol.HTTPS, null, -1, null, null,
+            null, 5, false, 50000);
+    }
+
+    public void testRepositoryMaxRetriesBackcompat() {
+        Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, 20);
+        Settings settings = Settings.builder()
+            .put(S3Repository.Repositories.MAX_RETRIES_SETTING.getKey(), 10)
+            .build();
+        launchAWSConfigurationTest(settings, repositorySettings, Protocol.HTTPS, null, -1, null,
+            null, null, 20, false, 50000);
+        assertSettingDeprecationsAndWarnings(new Setting<?>[]{
+            S3Repository.Repositories.MAX_RETRIES_SETTING,
+            S3Repository.Repository.MAX_RETRIES_SETTING
+        });
+    }
+
+    public void testGlobalThrottleRetriesBackcompat() {
+        Settings settings = Settings.builder()
+            .put(S3Repository.Repositories.USE_THROTTLE_RETRIES_SETTING.getKey(), true)
+            .build();
+        launchAWSConfigurationTest(settings, Settings.EMPTY, Protocol.HTTPS, null, -1, null, null,
+            null, 3, true, 50000);
+        assertSettingDeprecationsAndWarnings(new Setting<?>[]{
+            S3Repository.Repositories.USE_THROTTLE_RETRIES_SETTING
+        });
+    }
+
+    public void testRepositoryThrottleRetries() {
+        Settings settings = Settings.builder()
+            .put("s3.client.default.use_throttle_retries", true)
+            .build();
+        launchAWSConfigurationTest(settings, Settings.EMPTY, Protocol.HTTPS, null, -1, null, null,
+            null, 3, true, 50000);
+    }
+
+    public void testRepositoryThrottleRetriesBackcompat() {
+        Settings repositorySettings = Settings.builder()
+            .put(S3Repository.Repository.USE_THROTTLE_RETRIES_SETTING.getKey(), true).build();
+        Settings settings = Settings.builder()
+            .put(S3Repository.Repositories.USE_THROTTLE_RETRIES_SETTING.getKey(), false)
+            .build();
+        launchAWSConfigurationTest(settings, repositorySettings, Protocol.HTTPS, null, -1, null, null,
+            null, 3, true, 50000);
+        assertSettingDeprecationsAndWarnings(new Setting<?>[]{
+            S3Repository.Repositories.USE_THROTTLE_RETRIES_SETTING,
+            S3Repository.Repository.USE_THROTTLE_RETRIES_SETTING
+        });
+    }
+
+    protected void launchAWSConfigurationTest(Settings settings,
+                                              Settings singleRepositorySettings,
+                                              Protocol expectedProtocol,
+                                              String expectedProxyHost,
+                                              int expectedProxyPort,
+                                              String expectedProxyUsername,
+                                              String expectedProxyPassword,
+                                              String expectedSigner,
+                                              Integer expectedMaxRetries,
+                                              boolean expectedUseThrottleRetries,
+                                              int expectedReadTimeout) {
+
+        S3ClientSettings clientSettings = S3ClientSettings.getClientSettings(settings, "default");
+        ClientConfiguration configuration = InternalAwsS3Service.buildConfiguration(logger, clientSettings, singleRepositorySettings, null);
+
+        assertThat(configuration.getResponseMetadataCacheSize(), is(0));
+        assertThat(configuration.getProtocol(), is(expectedProtocol));
+        assertThat(configuration.getProxyHost(), is(expectedProxyHost));
+        assertThat(configuration.getProxyPort(), is(expectedProxyPort));
+        assertThat(configuration.getProxyUsername(), is(expectedProxyUsername));
+        assertThat(configuration.getProxyPassword(), is(expectedProxyPassword));
+        assertThat(configuration.getSignerOverride(), is(expectedSigner));
+        assertThat(configuration.getMaxErrorRetry(), is(expectedMaxRetries));
+        assertThat(configuration.useThrottledRetries(), is(expectedUseThrottleRetries));
+        assertThat(configuration.getSocketTimeout(), is(expectedReadTimeout));
+    }
+
+    private static Settings generateRepositorySettings(String key, String secret, String region, String endpoint, Integer maxRetries) {
+        Settings.Builder builder = Settings.builder();
+        if (region != null) {
+            builder.put(S3Repository.Repository.REGION_SETTING.getKey(), region);
+        }
+        if (endpoint != null) {
+            builder.put(S3Repository.Repository.ENDPOINT_SETTING.getKey(), endpoint);
+        }
+        if (key != null) {
+            builder.put(S3Repository.Repository.KEY_SETTING.getKey(), key);
+        }
+        if (secret != null) {
+            builder.put(S3Repository.Repository.SECRET_SETTING.getKey(), secret);
+        }
+        if (maxRetries != null) {
+            builder.put(S3Repository.Repository.MAX_RETRIES_SETTING.getKey(), maxRetries);
+        }
+        return builder.build();
+    }
+
+    public void testDefaultEndpoint() {
+        assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, null, null), Settings.EMPTY, "");
+    }
+
+    public void testEndpointSetting() {
+        Settings settings = Settings.builder()
+            .put("s3.client.default.endpoint", "s3.endpoint")
+            .build();
+        assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings, "s3.endpoint");
+    }
+
+    public void testEndpointSettingBackcompat() {
+        assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, "repository.endpoint", null),
+            Settings.EMPTY, "repository.endpoint");
+         assertSettingDeprecationsAndWarnings(new Setting<?>[]{S3Repository.Repository.ENDPOINT_SETTING});
+        Settings settings = Settings.builder()
+            .put(S3Repository.Repositories.ENDPOINT_SETTING.getKey(), "repositories.endpoint")
+            .build();
+        assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings,
+            "repositories.endpoint");
+         assertSettingDeprecationsAndWarnings(new Setting<?>[]{S3Repository.Repositories.ENDPOINT_SETTING});
+    }
+
+    public void testRegionSettingBackcompat() {
+        Settings settings = Settings.builder()
+            .put(InternalAwsS3Service.REGION_SETTING.getKey(), randomFrom("eu-west", "eu-west-1"))
+            .build();
+        assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings,
+            "s3-eu-west-1.amazonaws.com");
+        assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null), settings,
+            "s3.eu-central-1.amazonaws.com");
+        assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, "repository.endpoint", null),
+            settings, "repository.endpoint");
+         assertSettingDeprecationsAndWarnings(new Setting<?>[]{
+                        InternalAwsS3Service.REGION_SETTING,
+                        S3Repository.Repository.REGION_SETTING,
+                        S3Repository.Repository.ENDPOINT_SETTING},
+                "Specifying region for an s3 repository is deprecated. Use endpoint " +
+                        "to specify the region endpoint if the default behavior is not sufficient.");
+    }
+
+    public void testRegionAndEndpointSettingBackcompatPrecedence() {
+        Settings settings = Settings.builder()
+            .put(InternalAwsS3Service.REGION_SETTING.getKey(), randomFrom("eu-west", "eu-west-1"))
+            .put(InternalAwsS3Service.CLOUD_S3.REGION_SETTING.getKey(), randomFrom("us-west", "us-west-1"))
+            .build();
+        assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings,
+            "s3-us-west-1.amazonaws.com");
+        assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null), settings,
+            "s3.eu-central-1.amazonaws.com");
+        assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, "repository.endpoint", null),
+            settings, "repository.endpoint");
+         assertSettingDeprecationsAndWarnings(new Setting<?>[]{
+                        InternalAwsS3Service.REGION_SETTING,
+                        S3Repository.Repository.REGION_SETTING,
+                        InternalAwsS3Service.CLOUD_S3.REGION_SETTING,
+                        S3Repository.Repository.ENDPOINT_SETTING},
+                "Specifying region for an s3 repository is deprecated. Use endpoint " +
+                        "to specify the region endpoint if the default behavior is not sufficient.");
+    }
+
+    public void testInvalidRegion() {
+        Settings settings = Settings.builder()
+            .put(S3Repository.Repositories.REGION_SETTING.getKey(), "does-not-exist")
+            .build();
+        Settings repositorySettings = generateRepositorySettings("repository_key", "repository_secret", null, null, null);
+        String configName = InternalAwsS3Service.CLIENT_NAME.get(repositorySettings);
+        S3ClientSettings clientSettings = S3ClientSettings.getClientSettings(settings, configName);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () ->
+            InternalAwsS3Service.findEndpoint(logger, deprecationLogger, clientSettings, repositorySettings)
+        );
+        assertThat(e.getMessage(), containsString("No automatic endpoint could be derived from region"));
+        assertSettingDeprecationsAndWarnings(
+                new Setting<?>[]{S3Repository.Repositories.REGION_SETTING},
+                "Specifying region for an s3 repository is deprecated. Use endpoint " +
+                        "to specify the region endpoint if the default behavior is not sufficient.");
+    }
+
+    private void assertEndpoint(Settings repositorySettings, Settings settings,
+                                  String expectedEndpoint) {
+        String configName = InternalAwsS3Service.CLIENT_NAME.get(repositorySettings);
+        S3ClientSettings clientSettings = S3ClientSettings.getClientSettings(settings, configName);
+        String foundEndpoint = InternalAwsS3Service.findEndpoint(logger, deprecationLogger, clientSettings, repositorySettings);
+        assertThat(foundEndpoint, is(expectedEndpoint));
+    }
+
+}
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/EnvironmentCredentialsTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/EnvironmentCredentialsTests.java
new file mode 100644
index 0000000000000..e597391d50ba7
--- /dev/null
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/EnvironmentCredentialsTests.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.repositories.s3;
+
+import com.amazonaws.auth.AWSCredentialsProvider;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ESTestCase;
+
+public class EnvironmentCredentialsTests extends ESTestCase {
+
+    public void test() {
+        S3ClientSettings clientSettings = S3ClientSettings.getClientSettings(Settings.EMPTY, "default");
+        AWSCredentialsProvider provider = InternalAwsS3Service.buildCredentials(logger, deprecationLogger, clientSettings, Settings.EMPTY);
+        // NOTE: env vars are setup by the test runner in gradle
+        assertEquals("env_access", provider.getCredentials().getAWSAccessKeyId());
+        assertEquals("env_secret", provider.getCredentials().getAWSSecretKey());
+        assertWarnings("Supplying S3 credentials through environment variables is deprecated. "
+                + "See the breaking changes lists in the documentation for details.");
+    }
+}
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/MockAmazonS3.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/MockAmazonS3.java
similarity index 94%
rename from plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/MockAmazonS3.java
rename to plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/MockAmazonS3.java
index 8124f693943a5..64304879d0fdb 100644
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/MockAmazonS3.java
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/MockAmazonS3.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.cloud.aws.blobstore;
+package org.elasticsearch.repositories.s3;
 
 import com.amazonaws.AmazonClientException;
 import com.amazonaws.AmazonServiceException;
@@ -76,20 +76,13 @@ public ObjectMetadata getObjectMetadata(
     public PutObjectResult putObject(PutObjectRequest putObjectRequest)
             throws AmazonClientException, AmazonServiceException {
         String blobName = putObjectRequest.getKey();
-        DigestInputStream stream = (DigestInputStream) putObjectRequest.getInputStream();
 
         if (blobs.containsKey(blobName)) {
             throw new AmazonS3Exception("[" + blobName + "] already exists.");
         }
 
-        blobs.put(blobName, stream);
-
-        // input and output md5 hashes need to match to avoid an exception
-        String md5 = Base64.encodeAsString(stream.getMessageDigest().digest());
-        PutObjectResult result = new PutObjectResult();
-        result.setContentMd5(md5);
-
-        return result;
+        blobs.put(blobName, putObjectRequest.getInputStream());
+        return new PutObjectResult();
     }
 
     @Override
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/MockDefaultS3OutputStream.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/MockDefaultS3OutputStream.java
similarity index 95%
rename from plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/MockDefaultS3OutputStream.java
rename to plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/MockDefaultS3OutputStream.java
index cd2450f0e9d63..3a48b70e307c3 100644
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/MockDefaultS3OutputStream.java
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/MockDefaultS3OutputStream.java
@@ -17,12 +17,14 @@
  * under the License.
  */
 
-package org.elasticsearch.cloud.aws.blobstore;
+package org.elasticsearch.repositories.s3;
 
 import com.amazonaws.services.s3.model.AmazonS3Exception;
 import com.amazonaws.services.s3.model.PartETag;
 import com.carrotsearch.randomizedtesting.RandomizedTest;
 import org.elasticsearch.common.io.Streams;
+import org.elasticsearch.repositories.s3.DefaultS3OutputStream;
+import org.elasticsearch.repositories.s3.S3BlobStore;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
@@ -40,7 +42,7 @@ public class MockDefaultS3OutputStream extends DefaultS3OutputStream {
     private int numberOfUploadRequests = 0;
 
     public MockDefaultS3OutputStream(int bufferSizeInBytes) {
-        super(null, "test-bucket", "test-blobname", bufferSizeInBytes, 3, false);
+        super(null, "test-bucket", "test-blobname", bufferSizeInBytes, false);
     }
 
     @Override
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java
index 04c4f6fc0f14f..06f97e6ad17a2 100644
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java
@@ -24,9 +24,6 @@
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-
-import java.io.IOException;
 
 public class RepositoryS3ClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
@@ -35,8 +32,8 @@ public RepositoryS3ClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate t
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
-        return ESClientYamlSuiteTestCase.createParameters(0, 1);
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
     }
 }
 
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStoreContainerTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreContainerTests.java
similarity index 87%
rename from plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStoreContainerTests.java
rename to plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreContainerTests.java
index bca1c1d8a1816..7f81e871756ca 100644
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStoreContainerTests.java
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreContainerTests.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.cloud.aws.blobstore;
+package org.elasticsearch.repositories.s3;
 
 import org.elasticsearch.common.blobstore.BlobStore;
 import org.elasticsearch.common.settings.Settings;
@@ -31,9 +31,9 @@
 public class S3BlobStoreContainerTests extends ESBlobStoreContainerTestCase {
     protected BlobStore newBlobStore() throws IOException {
         MockAmazonS3 client = new MockAmazonS3();
-        String bucket = randomAsciiOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT);
+        String bucket = randomAlphaOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT);
 
         return new S3BlobStore(Settings.EMPTY, client, bucket, null, false,
-            new ByteSizeValue(10, ByteSizeUnit.MB), 5, "public-read-write", "standard");
+            new ByteSizeValue(10, ByteSizeUnit.MB), "public-read-write", "standard");
     }
 }
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStoreTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreTests.java
similarity index 97%
rename from plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStoreTests.java
rename to plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreTests.java
index 31e7b2cfb309e..17bea5239fe7e 100644
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStoreTests.java
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreTests.java
@@ -17,11 +17,12 @@
  * under the License.
  */
 
-package org.elasticsearch.cloud.aws.blobstore;
+package org.elasticsearch.repositories.s3;
 
 import com.amazonaws.services.s3.model.CannedAccessControlList;
 import com.amazonaws.services.s3.model.StorageClass;
 import org.elasticsearch.common.blobstore.BlobStoreException;
+import org.elasticsearch.repositories.s3.S3BlobStore;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/S3OutputStreamTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3OutputStreamTests.java
similarity index 99%
rename from plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/S3OutputStreamTests.java
rename to plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3OutputStreamTests.java
index 0518d657e23cb..8f4c7daea7edf 100644
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/S3OutputStreamTests.java
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3OutputStreamTests.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.cloud.aws.blobstore;
+package org.elasticsearch.repositories.s3;
 
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java
index f8940c6158c8f..b529225d5f340 100644
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java
@@ -19,19 +19,21 @@
 
 package org.elasticsearch.repositories.s3;
 
-import java.io.IOException;
-
-import com.amazonaws.Protocol;
 import com.amazonaws.services.s3.AbstractAmazonS3;
 import com.amazonaws.services.s3.AmazonS3;
-import org.elasticsearch.cloud.aws.AwsS3Service;
 import org.elasticsearch.cluster.metadata.RepositoryMetaData;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.repositories.RepositoryException;
 import org.elasticsearch.test.ESTestCase;
+import org.hamcrest.Matchers;
+
+import java.io.IOException;
 
 import static org.elasticsearch.repositories.s3.S3Repository.Repositories;
 import static org.elasticsearch.repositories.s3.S3Repository.Repository;
@@ -48,7 +50,7 @@ public boolean doesBucketExist(String bucketName) {
     }
 
     private static class DummyS3Service extends AbstractLifecycleComponent implements AwsS3Service {
-        public DummyS3Service() {
+        DummyS3Service() {
             super(Settings.EMPTY);
         }
         @Override
@@ -58,8 +60,7 @@ protected void doStop() {}
         @Override
         protected void doClose() {}
         @Override
-        public AmazonS3 client(Settings repositorySettings, String endpoint, Protocol protocol, String region, Integer maxRetries,
-                               boolean useThrottleRetries, Boolean pathStyleAccess) {
+        public AmazonS3 client(Settings settings) {
             return new DummyS3Client();
         }
     }
@@ -68,10 +69,15 @@ public void testSettingsResolution() throws Exception {
         Settings localSettings = Settings.builder().put(Repository.KEY_SETTING.getKey(), "key1").build();
         Settings globalSettings = Settings.builder().put(Repositories.KEY_SETTING.getKey(), "key2").build();
 
-        assertEquals("key1", getValue(localSettings, globalSettings, Repository.KEY_SETTING, Repositories.KEY_SETTING));
-        assertEquals("key1", getValue(localSettings, Settings.EMPTY, Repository.KEY_SETTING, Repositories.KEY_SETTING));
-        assertEquals("key2", getValue(Settings.EMPTY, globalSettings, Repository.KEY_SETTING, Repositories.KEY_SETTING));
-        assertEquals("", getValue(Settings.EMPTY, Settings.EMPTY, Repository.KEY_SETTING, Repositories.KEY_SETTING));
+        assertEquals(new SecureString("key1".toCharArray()),
+                     getValue(localSettings, globalSettings, Repository.KEY_SETTING, Repositories.KEY_SETTING));
+        assertEquals(new SecureString("key1".toCharArray()),
+                     getValue(localSettings, Settings.EMPTY, Repository.KEY_SETTING, Repositories.KEY_SETTING));
+        assertEquals(new SecureString("key2".toCharArray()),
+                     getValue(Settings.EMPTY, globalSettings, Repository.KEY_SETTING, Repositories.KEY_SETTING));
+        assertEquals(new SecureString("".toCharArray()),
+                     getValue(Settings.EMPTY, Settings.EMPTY, Repository.KEY_SETTING, Repositories.KEY_SETTING));
+        assertSettingDeprecationsAndWarnings(new Setting<?>[]{Repository.KEY_SETTING, Repositories.KEY_SETTING});
     }
 
     public void testInvalidChunkBufferSizeSettings() throws IOException {
@@ -93,7 +99,7 @@ private void assertValidBuffer(long bufferMB, long chunkMB) throws IOException {
         RepositoryMetaData metadata = new RepositoryMetaData("dummy-repo", "mock", Settings.builder()
             .put(Repository.BUFFER_SIZE_SETTING.getKey(), new ByteSizeValue(bufferMB, ByteSizeUnit.MB))
             .put(Repository.CHUNK_SIZE_SETTING.getKey(), new ByteSizeValue(chunkMB, ByteSizeUnit.MB)).build());
-        new S3Repository(metadata, Settings.EMPTY, new DummyS3Service());
+        new S3Repository(metadata, Settings.EMPTY, NamedXContentRegistry.EMPTY, new DummyS3Service());
     }
 
     private void assertInvalidBuffer(int bufferMB, int chunkMB, Class<? extends Exception> clazz, String msg) throws IOException {
@@ -101,7 +107,33 @@ private void assertInvalidBuffer(int bufferMB, int chunkMB, Class<? extends Exce
             .put(Repository.BUFFER_SIZE_SETTING.getKey(), new ByteSizeValue(bufferMB, ByteSizeUnit.MB))
             .put(Repository.CHUNK_SIZE_SETTING.getKey(), new ByteSizeValue(chunkMB, ByteSizeUnit.MB)).build());
 
-        Exception e = expectThrows(clazz, () -> new S3Repository(metadata, Settings.EMPTY, new DummyS3Service()));
+        Exception e = expectThrows(clazz, () -> new S3Repository(metadata, Settings.EMPTY, NamedXContentRegistry.EMPTY,
+            new DummyS3Service()));
         assertThat(e.getMessage(), containsString(msg));
     }
+
+    public void testBasePathSetting() throws IOException {
+        RepositoryMetaData metadata = new RepositoryMetaData("dummy-repo", "mock", Settings.builder()
+            .put(Repository.BASE_PATH_SETTING.getKey(), "/foo/bar").build());
+        S3Repository s3repo = new S3Repository(metadata, Settings.EMPTY, NamedXContentRegistry.EMPTY, new DummyS3Service());
+        assertEquals("foo/bar/", s3repo.basePath().buildAsString()); // make sure leading `/` is removed and trailing is added
+        assertWarnings("S3 repository base_path" +
+                " trimming the leading `/`, and leading `/` will not be supported for the S3 repository in future releases");
+        metadata = new RepositoryMetaData("dummy-repo", "mock", Settings.EMPTY);
+        Settings settings = Settings.builder().put(Repositories.BASE_PATH_SETTING.getKey(), "/foo/bar").build();
+        s3repo = new S3Repository(metadata, settings, NamedXContentRegistry.EMPTY, new DummyS3Service());
+        assertEquals("foo/bar/", s3repo.basePath().buildAsString()); // make sure leading `/` is removed and trailing is added
+        assertSettingDeprecationsAndWarnings(new Setting<?>[]{ Repositories.BASE_PATH_SETTING },
+            "S3 repository base_path trimming the leading `/`, and leading `/` will not be supported for the S3 repository in " +
+                "future releases");
+    }
+
+    public void testDefaultBufferSize() {
+        ByteSizeValue defaultBufferSize = S3Repository.Repository.BUFFER_SIZE_SETTING.get(Settings.EMPTY);
+        assertThat(defaultBufferSize, Matchers.lessThanOrEqualTo(new ByteSizeValue(100, ByteSizeUnit.MB)));
+        assertThat(defaultBufferSize, Matchers.greaterThanOrEqualTo(new ByteSizeValue(5, ByteSizeUnit.MB)));
+
+        ByteSizeValue defaultNodeBufferSize = S3Repository.Repositories.BUFFER_SIZE_SETTING.get(Settings.EMPTY);
+        assertEquals(defaultBufferSize, defaultNodeBufferSize);
+    }
 }
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/SyspropCredentialsTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/SyspropCredentialsTests.java
new file mode 100644
index 0000000000000..0f99bbe0f7cdb
--- /dev/null
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/SyspropCredentialsTests.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.repositories.s3;
+
+import com.amazonaws.auth.AWSCredentialsProvider;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ESTestCase;
+
+public class SyspropCredentialsTests extends ESTestCase {
+    public void test() {
+        S3ClientSettings clientSettings = S3ClientSettings.getClientSettings(Settings.EMPTY, "default");
+        AWSCredentialsProvider provider = InternalAwsS3Service.buildCredentials(logger, deprecationLogger, clientSettings, Settings.EMPTY);
+        // NOTE: sys props are setup by the test runner in gradle
+        assertEquals("sysprop_access", provider.getCredentials().getAWSAccessKeyId());
+        assertEquals("sysprop_secret", provider.getCredentials().getAWSSecretKey());
+        assertWarnings("Supplying S3 credentials through system properties is deprecated. " +
+                "See the breaking changes lists in the documentation for details.");
+    }
+}
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAmazonS3.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/TestAmazonS3.java
similarity index 99%
rename from plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAmazonS3.java
rename to plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/TestAmazonS3.java
index 37087db386b05..111e1773012ca 100644
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAmazonS3.java
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/TestAmazonS3.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.cloud.aws;
+package org.elasticsearch.repositories.s3;
 
 import com.amazonaws.AmazonClientException;
 import com.amazonaws.AmazonServiceException;
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/TestAwsS3Service.java
similarity index 78%
rename from plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java
rename to plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/TestAwsS3Service.java
index e7a57958a8953..522ca06614c00 100644
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/TestAwsS3Service.java
@@ -16,35 +16,36 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.cloud.aws;
 
-import com.amazonaws.Protocol;
+package org.elasticsearch.repositories.s3;
+
+import java.util.IdentityHashMap;
+
 import com.amazonaws.services.s3.AmazonS3;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.cluster.metadata.RepositoryMetaData;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.plugin.repository.s3.S3RepositoryPlugin;
-
-import java.util.IdentityHashMap;
 
 public class TestAwsS3Service extends InternalAwsS3Service {
     public static class TestPlugin extends S3RepositoryPlugin {
+        public TestPlugin(Settings settings) {
+            super(settings);
+        }
         @Override
         protected AwsS3Service createStorageService(Settings settings) {
             return new TestAwsS3Service(settings);
         }
     }
 
-    IdentityHashMap<AmazonS3, TestAmazonS3> clients = new IdentityHashMap<AmazonS3, TestAmazonS3>();
+    IdentityHashMap<AmazonS3, TestAmazonS3> clients = new IdentityHashMap<>();
 
     public TestAwsS3Service(Settings settings) {
-        super(settings);
+        super(settings, S3ClientSettings.load(settings));
     }
 
-
     @Override
-    public synchronized AmazonS3 client(Settings repositorySettings, String endpoint, Protocol protocol, String region, Integer maxRetries,
-                                        boolean useThrottleRetries, Boolean pathStyleAccess) {
-        return cachedWrapper(super.client(repositorySettings, endpoint, protocol, region, maxRetries, useThrottleRetries, pathStyleAccess));
+    public synchronized AmazonS3 client(Settings repositorySettings) {
+        return cachedWrapper(super.client(repositorySettings));
     }
 
     private AmazonS3 cachedWrapper(AmazonS3 client) {
diff --git a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository.yaml b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository.yaml
index 17294986d2ddd..74cab3edcb705 100644
--- a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository.yaml
+++ b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository.yaml
@@ -9,8 +9,6 @@
             type: s3
             settings:
               bucket: "my_bucket_name"
-              access_key: "AKVAIQBF2RECL7FJWGJQ"
-              secret_key: "vExyMThREXeRMm/b/LRzEB8jWwvzQeXgjqMX+6br"
               canned_acl: "public-read"
               storage_class: "standard"
 
diff --git a/plugins/store-smb/src/main/java/org/elasticsearch/index/store/SmbDirectoryWrapper.java b/plugins/store-smb/src/main/java/org/elasticsearch/index/store/SmbDirectoryWrapper.java
index fe2c32723e2ce..19499f56e7952 100644
--- a/plugins/store-smb/src/main/java/org/elasticsearch/index/store/SmbDirectoryWrapper.java
+++ b/plugins/store-smb/src/main/java/org/elasticsearch/index/store/SmbDirectoryWrapper.java
@@ -59,7 +59,7 @@ final class SmbFSIndexOutput extends OutputStreamIndexOutput {
          */
         static final int CHUNK_SIZE = 8192;
 
-        public SmbFSIndexOutput(String name) throws IOException {
+        SmbFSIndexOutput(String name) throws IOException {
             super("SmbFSIndexOutput(path=\"" + fsDirectory.getDirectory().resolve(name) + "\")", name, new FilterOutputStream(Channels.newOutputStream(Files.newByteChannel(fsDirectory.getDirectory().resolve(name), StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.READ, StandardOpenOption.WRITE))) {
                 // This implementation ensures, that we never write more than CHUNK_SIZE bytes:
                 @Override
diff --git a/plugins/store-smb/src/test/java/org/elasticsearch/index/store/StoreSmbClientYamlTestSuiteIT.java b/plugins/store-smb/src/test/java/org/elasticsearch/index/store/StoreSmbClientYamlTestSuiteIT.java
index 72394b7969340..2a35bd1062d9a 100644
--- a/plugins/store-smb/src/test/java/org/elasticsearch/index/store/StoreSmbClientYamlTestSuiteIT.java
+++ b/plugins/store-smb/src/test/java/org/elasticsearch/index/store/StoreSmbClientYamlTestSuiteIT.java
@@ -24,9 +24,6 @@
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-
-import java.io.IOException;
 
 public class StoreSmbClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
@@ -35,8 +32,8 @@ public StoreSmbClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testC
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
-        return ESClientYamlSuiteTestCase.createParameters(0, 1);
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
     }
 }
 
diff --git a/qa/backwards-5.0/build.gradle b/qa/backwards-5.0/build.gradle
deleted file mode 100644
index 18b7ae3c9e7c2..0000000000000
--- a/qa/backwards-5.0/build.gradle
+++ /dev/null
@@ -1,23 +0,0 @@
-apply plugin: 'elasticsearch.rest-test'
-
-/* This project runs the core REST tests against a 2 node cluster where one of the nodes has a different minor.
- * Since we don't have a version to test against we currently use the hardcoded snapshot for to bascially run
- * against ourself. To test that useing a different version works got into distribution/zip and execute:
- *   gradle clean install -Dbuild.snapshot=false
- *
- * This installs the release-build into a local .m2 repository, then change this version here to:
- *   bwcVersion = "5.0.0"
- *
- * now you can run the bwc tests with:
- *   gradle check -Drepos.mavenlocal=true
- *
- * (-Drepos.mavenlocal=true will force gradle to look for the zip distribuiton in the local .m2 repository)
- */
-integTest {
-  includePackaged = true
-  cluster {
-    numNodes = 2
-    numBwcNodes = 1
-    bwcVersion = "5.0.0-alpha6-SNAPSHOT" // this is the same as the current version until we released the first RC
-  }
-}
diff --git a/qa/backwards-5.0/src/test/java/org/elasticsearch/backwards/Backwards50ClientYamlTestSuiteIT.java b/qa/backwards-5.0/src/test/java/org/elasticsearch/backwards/Backwards50ClientYamlTestSuiteIT.java
deleted file mode 100644
index e3ab68b3477b4..0000000000000
--- a/qa/backwards-5.0/src/test/java/org/elasticsearch/backwards/Backwards50ClientYamlTestSuiteIT.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.backwards;
-
-import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
-import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
-import org.apache.lucene.util.TimeUnits;
-import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
-import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-
-import java.io.IOException;
-
-@TimeoutSuite(millis = 40 * TimeUnits.MINUTE) // some of the windows test VMs are slow as hell
-public class Backwards50ClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
-
-    public Backwards50ClientYamlTestSuiteIT(ClientYamlTestCandidate testCandidate) {
-        super(testCandidate);
-    }
-
-    @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
-        return createParameters(0, 1);
-    }
-}
-
diff --git a/qa/evil-tests/build.gradle b/qa/evil-tests/build.gradle
index cba9334fbcaaf..472fc87261602 100644
--- a/qa/evil-tests/build.gradle
+++ b/qa/evil-tests/build.gradle
@@ -42,7 +42,7 @@ thirdPartyAudit.excludes = [
   'com.google.common.cache.Striped64$Cell',
   'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator',
   'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1',
-  
+
   // missing class
   'com.ibm.icu.lang.UCharacter',
 ]
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/ESPolicyUnitTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/ESPolicyUnitTests.java
index 255dfb3ec5c63..ed71934c72ac4 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/ESPolicyUnitTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/ESPolicyUnitTests.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.FilePermission;
+import java.net.SocketPermission;
 import java.security.AllPermission;
 import java.security.CodeSource;
 import java.security.Permission;
@@ -36,7 +37,7 @@
  * we don't allow messing with the policy
  */
 public class ESPolicyUnitTests extends ESTestCase {
-    /** 
+    /**
      * Test policy with null codesource.
      * <p>
      * This can happen when restricting privileges with doPrivileged,
@@ -55,7 +56,7 @@ public void testNullCodeSource() throws Exception {
         assertFalse(policy.implies(new ProtectionDomain(null, noPermissions), new FilePermission("foo", "read")));
     }
 
-    /** 
+    /**
      * test with null location
      * <p>
      * its unclear when/if this happens, see https://bugs.openjdk.java.net/browse/JDK-8129972
@@ -67,4 +68,15 @@ public void testNullLocation() throws Exception {
         assertFalse(policy.implies(new ProtectionDomain(new CodeSource(null, (Certificate[]) null), noPermissions),
                 new FilePermission("foo", "read")));
     }
+
+    public void testListen() {
+        assumeTrue("test cannot run with security manager", System.getSecurityManager() == null);
+        final PermissionCollection noPermissions = new Permissions();
+        final ESPolicy policy = new ESPolicy(noPermissions, Collections.emptyMap(), true);
+        assertFalse(
+            policy.implies(
+                new ProtectionDomain(ESPolicyUnitTests.class.getProtectionDomain().getCodeSource(), noPermissions),
+                new SocketPermission("localhost:" + randomFrom(0, randomIntBetween(49152, 65535)), "listen")));
+    }
+
 }
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilBootstrapChecksTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilBootstrapChecksTests.java
new file mode 100644
index 0000000000000..8e346bf7d9cb9
--- /dev/null
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilBootstrapChecksTests.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.bootstrap;
+
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.node.NodeValidationException;
+import org.elasticsearch.test.ESTestCase;
+import org.hamcrest.Matcher;
+import org.junit.After;
+import org.junit.Before;
+
+import java.util.Collections;
+import java.util.List;
+
+import static java.util.Collections.emptyList;
+import static org.elasticsearch.bootstrap.BootstrapChecks.ES_ENFORCE_BOOTSTRAP_CHECKS;
+import static org.hamcrest.Matchers.allOf;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.hasToString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+
+public class EvilBootstrapChecksTests extends ESTestCase {
+
+    private String esEnforceBootstrapChecks = System.getProperty(ES_ENFORCE_BOOTSTRAP_CHECKS);
+
+    @Override
+    @Before
+    public void setUp() throws Exception {
+        super.setUp();
+    }
+
+    @Override
+    @After
+    public void tearDown() throws Exception {
+        setEsEnforceBootstrapChecks(esEnforceBootstrapChecks);
+        super.tearDown();
+    }
+
+    public void testEnforceBootstrapChecks() throws NodeValidationException {
+        setEsEnforceBootstrapChecks("true");
+        final List<BootstrapCheck> checks = Collections.singletonList(
+                new BootstrapCheck() {
+                    @Override
+                    public boolean check() {
+                        return true;
+                    }
+
+                    @Override
+                    public String errorMessage() {
+                        return "error";
+                    }
+                }
+        );
+        final Logger logger = mock(Logger.class);
+
+        final NodeValidationException e = expectThrows(
+                NodeValidationException.class,
+                () -> BootstrapChecks.check(false, checks, logger));
+        final Matcher<String> allOf =
+                allOf(containsString("bootstrap checks failed"), containsString("error"));
+        assertThat(e, hasToString(allOf));
+        verify(logger).info("explicitly enforcing bootstrap checks");
+        verifyNoMoreInteractions(logger);
+    }
+
+    public void testNonEnforcedBootstrapChecks() throws NodeValidationException {
+        setEsEnforceBootstrapChecks(null);
+        final Logger logger = mock(Logger.class);
+        // nothing should happen
+        BootstrapChecks.check(false, emptyList(), logger);
+        verifyNoMoreInteractions(logger);
+    }
+
+    public void testInvalidValue() {
+        final String value = randomAlphaOfLength(8);
+        setEsEnforceBootstrapChecks(value);
+        final boolean enforceLimits = randomBoolean();
+        final IllegalArgumentException e = expectThrows(
+                IllegalArgumentException.class,
+                () -> BootstrapChecks.check(enforceLimits, emptyList(), "testInvalidValue"));
+        final Matcher<String> matcher = containsString(
+                "[es.enforce.bootstrap.checks] must be [true] but was [" + value + "]");
+        assertThat(e, hasToString(matcher));
+    }
+
+    @SuppressForbidden(reason = "set or clear system property es.enforce.bootstrap.checks")
+    public void setEsEnforceBootstrapChecks(final String value) {
+        if (value == null) {
+            System.clearProperty(ES_ENFORCE_BOOTSTRAP_CHECKS);
+        } else {
+            System.setProperty(ES_ENFORCE_BOOTSTRAP_CHECKS, value);
+        }
+    }
+
+}
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilElasticsearchCliTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilElasticsearchCliTests.java
index 8bd2451da577f..02caeca808944 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilElasticsearchCliTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilElasticsearchCliTests.java
@@ -19,39 +19,45 @@
 
 package org.elasticsearch.bootstrap;
 
+import java.util.Map;
+
 import org.elasticsearch.cli.ExitCodes;
 import org.elasticsearch.common.SuppressForbidden;
-import org.elasticsearch.test.ESTestCase;
 
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.hasEntry;
+import static org.hamcrest.Matchers.hasKey;
 
 public class EvilElasticsearchCliTests extends ESElasticsearchCliTestCase {
 
     @SuppressForbidden(reason = "manipulates system properties for testing")
     public void testPathHome() throws Exception {
         final String pathHome = System.getProperty("es.path.home");
-        final String value = randomAsciiOfLength(16);
+        final String value = randomAlphaOfLength(16);
         System.setProperty("es.path.home", value);
 
         runTest(
                 ExitCodes.OK,
                 true,
                 output -> {},
-                (foreground, pidFile, esSettings) -> {
-                    assertThat(esSettings.size(), equalTo(1));
-                    assertThat(esSettings, hasEntry("path.home", value));
+                (foreground, pidFile, quiet, esSettings) -> {
+                    Map<String, String> settings = esSettings.settings().getAsMap();
+                    assertThat(settings.size(), equalTo(2));
+                    assertThat(settings, hasEntry("path.home", value));
+                    assertThat(settings, hasKey("path.logs")); // added by env initialization
                 });
 
         System.clearProperty("es.path.home");
-        final String commandLineValue = randomAsciiOfLength(16);
+        final String commandLineValue = randomAlphaOfLength(16);
         runTest(
                 ExitCodes.OK,
                 true,
                 output -> {},
-                (foreground, pidFile, esSettings) -> {
-                    assertThat(esSettings.size(), equalTo(1));
-                    assertThat(esSettings, hasEntry("path.home", commandLineValue));
+                (foreground, pidFile, quiet, esSettings) -> {
+                    Map<String, String> settings = esSettings.settings().getAsMap();
+                    assertThat(settings.size(), equalTo(2));
+                    assertThat(settings, hasEntry("path.home", commandLineValue));
+                    assertThat(settings, hasKey("path.logs")); // added by env initialization
                 },
                 "-Epath.home=" + commandLineValue);
 
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilJNANativesTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilJNANativesTests.java
index 069f7e1aeffe6..3ab564c24005b 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilJNANativesTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilJNANativesTests.java
@@ -36,14 +36,15 @@ public class EvilJNANativesTests extends ESTestCase {
     public void testSetMaximumNumberOfThreads() throws IOException {
         if (Constants.LINUX) {
             final List<String> lines = Files.readAllLines(PathUtils.get("/proc/self/limits"));
-            if (!lines.isEmpty()) {
-                for (String line : lines) {
-                    if (line != null && line.startsWith("Max processes")) {
-                        final String[] fields = line.split("\\s+");
-                        final long limit = "unlimited".equals(fields[2]) ? JNACLibrary.RLIM_INFINITY : Long.parseLong(fields[2]);
-                        assertThat(JNANatives.MAX_NUMBER_OF_THREADS, equalTo(limit));
-                        return;
-                    }
+            for (final String line : lines) {
+                if (line != null && line.startsWith("Max processes")) {
+                    final String[] fields = line.split("\\s+");
+                    final long limit =
+                            "unlimited".equals(fields[2])
+                                    ? JNACLibrary.RLIM_INFINITY
+                                    : Long.parseLong(fields[2]);
+                    assertThat(JNANatives.MAX_NUMBER_OF_THREADS, equalTo(limit));
+                    return;
                 }
             }
             fail("should have read max processes from /proc/self/limits");
@@ -55,22 +56,47 @@ public void testSetMaximumNumberOfThreads() throws IOException {
     public void testSetMaxSizeVirtualMemory() throws IOException {
         if (Constants.LINUX) {
             final List<String> lines = Files.readAllLines(PathUtils.get("/proc/self/limits"));
-            if (!lines.isEmpty()) {
-                for (String line : lines) {
-                    if (line != null && line.startsWith("Max address space")) {
-                        final String[] fields = line.split("\\s+");
-                        final String limit = fields[3];
-                        assertEquals(JNANatives.rlimitToString(JNANatives.MAX_SIZE_VIRTUAL_MEMORY), limit);
-                        return;
-                    }
+            for (final String line : lines) {
+                if (line != null && line.startsWith("Max address space")) {
+                    final String[] fields = line.split("\\s+");
+                    final String limit = fields[3];
+                    assertThat(
+                            JNANatives.rlimitToString(JNANatives.MAX_SIZE_VIRTUAL_MEMORY),
+                            equalTo(limit));
+                    return;
                 }
             }
             fail("should have read max size virtual memory from /proc/self/limits");
         } else if (Constants.MAC_OS_X) {
-            assertThat(JNANatives.MAX_SIZE_VIRTUAL_MEMORY, anyOf(equalTo(Long.MIN_VALUE), greaterThanOrEqualTo(0L)));
+            assertThat(
+                    JNANatives.MAX_SIZE_VIRTUAL_MEMORY,
+                    anyOf(equalTo(Long.MIN_VALUE), greaterThanOrEqualTo(0L)));
         } else {
             assertThat(JNANatives.MAX_SIZE_VIRTUAL_MEMORY, equalTo(Long.MIN_VALUE));
         }
     }
 
+    public void testSetMaxFileSize() throws IOException {
+        if (Constants.LINUX) {
+            final List<String> lines = Files.readAllLines(PathUtils.get("/proc/self/limits"));
+            for (final String line : lines) {
+                if (line != null && line.startsWith("Max file size")) {
+                    final String[] fields = line.split("\\s+");
+                    final String limit = fields[3];
+                    assertThat(
+                            JNANatives.rlimitToString(JNANatives.MAX_FILE_SIZE),
+                            equalTo(limit));
+                    return;
+                }
+            }
+            fail("should have read max file size from /proc/self/limits");
+        } else if (Constants.MAC_OS_X) {
+            assertThat(
+                    JNANatives.MAX_FILE_SIZE,
+                    anyOf(equalTo(Long.MIN_VALUE), greaterThanOrEqualTo(0L)));
+        } else {
+            assertThat(JNANatives.MAX_FILE_SIZE, equalTo(Long.MIN_VALUE));
+        }
+    }
+
 }
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilSecurityTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilSecurityTests.java
index 3bf238916a5d7..072a015962e95 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilSecurityTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilSecurityTests.java
@@ -34,6 +34,9 @@
 import java.security.Permissions;
 import java.util.Set;
 
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.hasToString;
+
 @SuppressForbidden(reason = "modifies system properties and attempts to create symbolic links intentionally")
 public class EvilSecurityTests extends ESTestCase {
 
@@ -77,7 +80,6 @@ public void testEnvironmentPaths() throws Exception {
         Settings.Builder settingsBuilder = Settings.builder();
         settingsBuilder.put(Environment.PATH_HOME_SETTING.getKey(), esHome.resolve("home").toString());
         settingsBuilder.put(Environment.PATH_CONF_SETTING.getKey(), esHome.resolve("conf").toString());
-        settingsBuilder.put(Environment.PATH_SCRIPTS_SETTING.getKey(), esHome.resolve("scripts").toString());
         settingsBuilder.putArray(Environment.PATH_DATA_SETTING.getKey(), esHome.resolve("data1").toString(),
                 esHome.resolve("data2").toString());
         settingsBuilder.put(Environment.PATH_SHARED_DATA_SETTING.getKey(), esHome.resolve("custom").toString());
@@ -137,6 +139,30 @@ public void testEnvironmentPaths() throws Exception {
         assertExactPermissions(new FilePermission(environment.pidFile().toString(), "delete"), permissions);
     }
 
+    public void testDuplicateDataPaths() throws IOException {
+        final Path path = createTempDir();
+        final Path home = path.resolve("home");
+        final Path data = path.resolve("data");
+        final Path duplicate;
+        if (randomBoolean()) {
+            duplicate = data;
+        } else {
+            duplicate = createTempDir().toAbsolutePath().resolve("link");
+            Files.createSymbolicLink(duplicate, data);
+        }
+
+        final Settings settings =
+                Settings
+                        .builder()
+                        .put(Environment.PATH_HOME_SETTING.getKey(), home.toString())
+                        .putArray(Environment.PATH_DATA_SETTING.getKey(), data.toString(), duplicate.toString())
+                        .build();
+
+        final Environment environment = new Environment(settings);
+        final IllegalStateException e = expectThrows(IllegalStateException.class, () -> Security.createPermissions(environment));
+        assertThat(e, hasToString(containsString("path [" + duplicate.toRealPath() + "] is duplicated by [" + duplicate + "]")));
+    }
+
     public void testEnsureSymlink() throws IOException {
         Path p = createTempDir();
 
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/SeccompTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/SeccompTests.java
deleted file mode 100644
index d028dfd573a17..0000000000000
--- a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/SeccompTests.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.bootstrap;
-
-import org.apache.lucene.util.Constants;
-import org.elasticsearch.test.ESTestCase;
-
-/** Simple tests seccomp filter is working. */
-public class SeccompTests extends ESTestCase {
-    
-    /** command to try to run in tests */
-    static final String EXECUTABLE = Constants.WINDOWS ? "calc" : "ls";
-
-    @Override
-    public void setUp() throws Exception {
-        super.setUp();
-        assumeTrue("requires seccomp filter installation", Natives.isSeccompInstalled());
-        // otherwise security manager will block the execution, no fun
-        assumeTrue("cannot test with security manager enabled", System.getSecurityManager() == null);
-        // otherwise, since we don't have TSYNC support, rules are not applied to the test thread
-        // (randomizedrunner class initialization happens in its own thread, after the test thread is created)
-        // instead we just forcefully run it for the test thread here.
-        if (!JNANatives.LOCAL_SECCOMP_ALL) {
-            try {
-                Seccomp.init(createTempDir());
-            } catch (Exception e) {
-                throw new RuntimeException("unable to forcefully apply seccomp to test thread", e);
-            }
-        }
-    }
-    
-    public void testNoExecution() throws Exception {
-        try {
-            Runtime.getRuntime().exec(EXECUTABLE);
-            fail("should not have been able to execute!");
-        } catch (Exception expected) {
-            // we can't guarantee how its converted, currently its an IOException, like this:
-            /*
-            java.io.IOException: Cannot run program "ls": error=13, Permission denied
-                    at __randomizedtesting.SeedInfo.seed([65E6C4BED11899E:FC6E1CA6AA2DB634]:0)
-                    at java.lang.ProcessBuilder.start(ProcessBuilder.java:1048)
-                    at java.lang.Runtime.exec(Runtime.java:620)
-                    ...
-                  Caused by: java.io.IOException: error=13, Permission denied
-                    at java.lang.UNIXProcess.forkAndExec(Native Method)
-                    at java.lang.UNIXProcess.<init>(UNIXProcess.java:248)
-                    at java.lang.ProcessImpl.start(ProcessImpl.java:134)
-                    at java.lang.ProcessBuilder.start(ProcessBuilder.java:1029)
-                    ... 
-            */
-        }
-    }
-    
-    // make sure thread inherits this too (its documented that way)
-    public void testNoExecutionFromThread() throws Exception {
-        Thread t = new Thread() {
-            @Override
-            public void run() {
-                try {
-                    Runtime.getRuntime().exec(EXECUTABLE);
-                    fail("should not have been able to execute!");
-                } catch (Exception expected) {
-                    // ok
-                }
-            }
-        };
-        t.start();
-        t.join();
-    }
-}
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/SystemCallFilterTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/SystemCallFilterTests.java
new file mode 100644
index 0000000000000..244f98356d976
--- /dev/null
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/SystemCallFilterTests.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.bootstrap;
+
+import org.apache.lucene.util.Constants;
+import org.elasticsearch.test.ESTestCase;
+
+/** Simple tests system call filter is working. */
+public class SystemCallFilterTests extends ESTestCase {
+
+    /** command to try to run in tests */
+    static final String EXECUTABLE = Constants.WINDOWS ? "calc" : "ls";
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        assumeTrue("requires system call filter installation", Natives.isSystemCallFilterInstalled());
+        // otherwise security manager will block the execution, no fun
+        assumeTrue("cannot test with security manager enabled", System.getSecurityManager() == null);
+        // otherwise, since we don't have TSYNC support, rules are not applied to the test thread
+        // (randomizedrunner class initialization happens in its own thread, after the test thread is created)
+        // instead we just forcefully run it for the test thread here.
+        if (!JNANatives.LOCAL_SYSTEM_CALL_FILTER_ALL) {
+            try {
+                SystemCallFilter.init(createTempDir());
+            } catch (Exception e) {
+                throw new RuntimeException("unable to forcefully apply system call filter to test thread", e);
+            }
+        }
+    }
+
+    public void testNoExecution() throws Exception {
+        try {
+            Runtime.getRuntime().exec(EXECUTABLE);
+            fail("should not have been able to execute!");
+        } catch (Exception expected) {
+            // we can't guarantee how its converted, currently its an IOException, like this:
+            /*
+            java.io.IOException: Cannot run program "ls": error=13, Permission denied
+                    at __randomizedtesting.SeedInfo.seed([65E6C4BED11899E:FC6E1CA6AA2DB634]:0)
+                    at java.lang.ProcessBuilder.start(ProcessBuilder.java:1048)
+                    at java.lang.Runtime.exec(Runtime.java:620)
+                    ...
+                  Caused by: java.io.IOException: error=13, Permission denied
+                    at java.lang.UNIXProcess.forkAndExec(Native Method)
+                    at java.lang.UNIXProcess.<init>(UNIXProcess.java:248)
+                    at java.lang.ProcessImpl.start(ProcessImpl.java:134)
+                    at java.lang.ProcessBuilder.start(ProcessBuilder.java:1029)
+                    ...
+            */
+        }
+    }
+
+    // make sure thread inherits this too (its documented that way)
+    public void testNoExecutionFromThread() throws Exception {
+        Thread t = new Thread() {
+            @Override
+            public void run() {
+                try {
+                    Runtime.getRuntime().exec(EXECUTABLE);
+                    fail("should not have been able to execute!");
+                } catch (Exception expected) {
+                    // ok
+                }
+            }
+        };
+        t.start();
+        t.join();
+    }
+}
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/cli/EvilCommandTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/cli/EvilCommandTests.java
new file mode 100644
index 0000000000000..33d9bbe7782d0
--- /dev/null
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/cli/EvilCommandTests.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cli;
+
+import joptsimple.OptionSet;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.Matchers.isEmptyString;
+
+public class EvilCommandTests extends ESTestCase {
+
+    public void testCommandShutdownHook() throws Exception {
+        final AtomicBoolean closed = new AtomicBoolean();
+        final boolean shouldThrow = randomBoolean();
+        final Command command = new Command("test-command-shutdown-hook") {
+            @Override
+            protected void execute(Terminal terminal, OptionSet options) throws Exception {
+
+            }
+
+            @Override
+            public void close() throws IOException {
+                closed.set(true);
+                if (shouldThrow) {
+                    throw new IOException("fail");
+                }
+            }
+        };
+        final MockTerminal terminal = new MockTerminal();
+        command.main(new String[0], terminal);
+        assertNotNull(command.shutdownHookThread.get());
+        // successful removal here asserts that the runtime hook was installed in Command#main
+        assertTrue(Runtime.getRuntime().removeShutdownHook(command.shutdownHookThread.get()));
+        command.shutdownHookThread.get().run();
+        command.shutdownHookThread.get().join();
+        assertTrue(closed.get());
+        final String output = terminal.getOutput();
+        if (shouldThrow) {
+            // ensure that we dump the exception
+            assertThat(output, containsString("java.io.IOException: fail"));
+            // ensure that we dump the stack trace too
+            assertThat(output, containsString("\tat org.elasticsearch.cli.EvilCommandTests$1.close"));
+        } else {
+            assertThat(output, isEmptyString());
+        }
+    }
+
+}
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/cluster/metadata/EvilSystemPropertyTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/cluster/metadata/EvilSystemPropertyTests.java
new file mode 100644
index 0000000000000..5e44fdbefad0a
--- /dev/null
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/cluster/metadata/EvilSystemPropertyTests.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.cluster.metadata;
+
+import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ESTestCase;
+
+public class EvilSystemPropertyTests extends ESTestCase {
+
+    @SuppressForbidden(reason = "manipulates system properties for testing")
+    public void testMaxNumShards() {
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () ->
+            IndexMetaData.buildNumberOfShardsSetting()
+                .get(Settings.builder().put("index.number_of_shards", 1025).build()));
+        assertEquals("Failed to parse value [1025] for setting [index.number_of_shards] must be <= 1024", exception.getMessage());
+
+        Integer numShards = IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(Settings.builder().put("index.number_of_shards", 100).build());
+        assertEquals(100, numShards.intValue());
+        int limit = randomIntBetween(1, 10);
+        System.setProperty("es.index.max_number_of_shards", Integer.toString(limit));
+        try {
+            IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () ->
+                IndexMetaData.buildNumberOfShardsSetting()
+                    .get(Settings.builder().put("index.number_of_shards", 11).build()));
+            assertEquals("Failed to parse value [11] for setting [index.number_of_shards] must be <= " + limit, e.getMessage());
+        } finally {
+            System.clearProperty("es.index.max_number_of_shards");
+        }
+    }
+}
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerConfigurationTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerConfigurationTests.java
index cfa54b8c0013a..8dd1fb061369e 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerConfigurationTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerConfigurationTests.java
@@ -27,18 +27,29 @@
 import org.apache.logging.log4j.core.config.Configuration;
 import org.apache.logging.log4j.core.config.Configurator;
 import org.apache.logging.log4j.core.config.LoggerConfig;
+import org.elasticsearch.cli.UserException;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
 import java.nio.file.Path;
+import java.util.Map;
 
+import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.hasKey;
+import static org.hamcrest.Matchers.hasToString;
 import static org.hamcrest.Matchers.notNullValue;
 
 public class EvilLoggerConfigurationTests extends ESTestCase {
 
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        LogConfigurator.registerErrorListener();
+    }
+
     @Override
     public void tearDown() throws Exception {
         LoggerContext context = (LoggerContext) LogManager.getContext(false);
@@ -55,7 +66,7 @@ public void testResolveMultipleConfigs() throws Exception {
                 .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
                 .build();
             final Environment environment = new Environment(settings);
-            LogConfigurator.configure(environment, true);
+            LogConfigurator.configure(environment);
 
             {
                 final LoggerContext ctx = (LoggerContext) LogManager.getContext(false);
@@ -85,18 +96,18 @@ public void testResolveMultipleConfigs() throws Exception {
         }
     }
 
-    public void testDefaults() throws IOException {
+    public void testDefaults() throws IOException, UserException {
         final Path configDir = getDataPath("config");
-        final String level = randomFrom(Level.values()).toString();
+        final String level = randomFrom(Level.TRACE, Level.DEBUG, Level.INFO, Level.WARN, Level.ERROR).toString();
         final Settings settings = Settings.builder()
             .put(Environment.PATH_CONF_SETTING.getKey(), configDir.toAbsolutePath())
             .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
             .put("logger.level", level)
             .build();
         final Environment environment = new Environment(settings);
-        LogConfigurator.configure(environment, true);
+        LogConfigurator.configure(environment);
 
-        final String loggerName = Loggers.commonPrefix + "test";
+        final String loggerName = "test";
         final Logger logger = ESLoggerFactory.getLogger(loggerName);
         assertThat(logger.getLevel().toString(), equalTo(level));
     }
@@ -110,12 +121,71 @@ public void testResolveOrder() throws Exception {
             .put("logger.test_resolve_order", "TRACE")
             .build();
         final Environment environment = new Environment(settings);
-        LogConfigurator.configure(environment, true);
+        LogConfigurator.configure(environment);
 
         // args should overwrite whatever is in the config
-        final String loggerName = Loggers.commonPrefix + "test_resolve_order";
+        final String loggerName = "test_resolve_order";
         final Logger logger = ESLoggerFactory.getLogger(loggerName);
         assertTrue(logger.isTraceEnabled());
     }
 
+    public void testHierarchy() throws Exception {
+        final Path configDir = getDataPath("hierarchy");
+        final Settings settings = Settings.builder()
+                .put(Environment.PATH_CONF_SETTING.getKey(), configDir.toAbsolutePath())
+                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+                .build();
+        final Environment environment = new Environment(settings);
+        LogConfigurator.configure(environment);
+
+        assertThat(ESLoggerFactory.getLogger("x").getLevel(), equalTo(Level.TRACE));
+        assertThat(ESLoggerFactory.getLogger("x.y").getLevel(), equalTo(Level.DEBUG));
+
+        final Level level = randomFrom(Level.TRACE, Level.DEBUG, Level.INFO, Level.WARN, Level.ERROR);
+        Loggers.setLevel(ESLoggerFactory.getLogger("x"), level);
+
+        assertThat(ESLoggerFactory.getLogger("x").getLevel(), equalTo(level));
+        assertThat(ESLoggerFactory.getLogger("x.y").getLevel(), equalTo(level));
+    }
+
+    public void testMissingConfigFile() {
+        final Path configDir = getDataPath("does_not_exist");
+        final Settings settings = Settings.builder()
+            .put(Environment.PATH_CONF_SETTING.getKey(), configDir.toAbsolutePath())
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+            .build();
+        final Environment environment = new Environment(settings);
+        UserException e = expectThrows(UserException.class, () -> LogConfigurator.configure(environment));
+        assertThat(e, hasToString(containsString("no log4j2.properties found; tried")));
+    }
+
+    public void testLoggingLevelsFromSettings() throws IOException, UserException {
+        final Level rootLevel = randomFrom(Level.TRACE, Level.DEBUG, Level.INFO, Level.WARN, Level.ERROR);
+        final Level fooLevel = randomFrom(Level.TRACE, Level.DEBUG, Level.INFO, Level.WARN, Level.ERROR);
+        final Level barLevel = randomFrom(Level.TRACE, Level.DEBUG, Level.INFO, Level.WARN, Level.ERROR);
+        final Path configDir = getDataPath("minimal");
+        final Settings settings = Settings.builder()
+            .put(Environment.PATH_CONF_SETTING.getKey(), configDir.toAbsolutePath())
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+            .put("logger.level", rootLevel.name())
+            .put("logger.foo", fooLevel.name())
+            .put("logger.bar", barLevel.name())
+            .build();
+        final Environment environment = new Environment(settings);
+        LogConfigurator.configure(environment);
+
+        final LoggerContext ctx = (LoggerContext) LogManager.getContext(false);
+        final Configuration config = ctx.getConfiguration();
+        final Map<String, LoggerConfig> loggerConfigs = config.getLoggers();
+        assertThat(loggerConfigs.size(), equalTo(3));
+        assertThat(loggerConfigs, hasKey(""));
+        assertThat(loggerConfigs.get("").getLevel(), equalTo(rootLevel));
+        assertThat(loggerConfigs, hasKey("foo"));
+        assertThat(loggerConfigs.get("foo").getLevel(), equalTo(fooLevel));
+        assertThat(loggerConfigs, hasKey("bar"));
+        assertThat(loggerConfigs.get("bar").getLevel(), equalTo(barLevel));
+
+        assertThat(ctx.getLogger(randomAlphaOfLength(16)).getLevel(), equalTo(rootLevel));
+    }
+
 }
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerTests.java
index 23eb074458054..6991ff81e7a46 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerTests.java
@@ -22,44 +22,52 @@
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.core.Appender;
 import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.appender.ConsoleAppender;
+import org.apache.logging.log4j.core.appender.CountingNoOpAppender;
 import org.apache.logging.log4j.core.config.Configurator;
-import org.elasticsearch.Version;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.apache.lucene.util.Constants;
+import org.elasticsearch.cli.UserException;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.io.PathUtils;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.node.Node;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.hamcrest.RegexMatcher;
 
 import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.List;
+import java.util.Set;
+import java.util.concurrent.BrokenBarrierException;
+import java.util.concurrent.CyclicBarrier;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.lessThan;
+import static org.hamcrest.Matchers.startsWith;
 
 public class EvilLoggerTests extends ESTestCase {
 
-    private Logger testLogger;
-    private DeprecationLogger deprecationLogger;
-
     @Override
     public void setUp() throws Exception {
         super.setUp();
-
-        final Path configDir = getDataPath("config");
-        // need to set custom path.conf so we can use a custom log4j2.properties file for the test
-        final Settings settings = Settings.builder()
-            .put(Environment.PATH_CONF_SETTING.getKey(), configDir.toAbsolutePath())
-            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
-            .build();
-        final Environment environment = new Environment(settings);
-        LogConfigurator.configure(environment, true);
-
-        testLogger = ESLoggerFactory.getLogger("test");
-        deprecationLogger = ESLoggerFactory.getDeprecationLogger("test");
+        LogConfigurator.registerErrorListener();
     }
 
     @Override
@@ -69,55 +77,325 @@ public void tearDown() throws Exception {
         super.tearDown();
     }
 
-    public void testLocationInfoTest() throws IOException {
+    public void testLocationInfoTest() throws IOException, UserException {
+        setupLogging("location_info");
+
+        final Logger testLogger = ESLoggerFactory.getLogger("test");
+
         testLogger.error("This is an error message");
         testLogger.warn("This is a warning message");
         testLogger.info("This is an info message");
         testLogger.debug("This is a debug message");
         testLogger.trace("This is a trace message");
-        final String path = System.getProperty("es.logs") + ".log";
+        final String path =
+            System.getProperty("es.logs.base_path") +
+                System.getProperty("file.separator") +
+                System.getProperty("es.logs.cluster_name") +
+                ".log";
         final List<String> events = Files.readAllLines(PathUtils.get(path));
-        assertThat(events.size(), equalTo(6)); // the five messages me log plus a warning for unsupported configuration files
+        assertThat(events.size(), equalTo(5));
         final String location = "org.elasticsearch.common.logging.EvilLoggerTests.testLocationInfoTest";
         // the first message is a warning for unsupported configuration files
-        assertLogLine(events.get(1), Level.ERROR, location, "This is an error message");
-        assertLogLine(events.get(2), Level.WARN, location, "This is a warning message");
-        assertLogLine(events.get(3), Level.INFO, location, "This is an info message");
-        assertLogLine(events.get(4), Level.DEBUG, location, "This is a debug message");
-        assertLogLine(events.get(5), Level.TRACE, location, "This is a trace message");
+        assertLogLine(events.get(0), Level.ERROR, location, "This is an error message");
+        assertLogLine(events.get(1), Level.WARN, location, "This is a warning message");
+        assertLogLine(events.get(2), Level.INFO, location, "This is an info message");
+        assertLogLine(events.get(3), Level.DEBUG, location, "This is a debug message");
+        assertLogLine(events.get(4), Level.TRACE, location, "This is a trace message");
     }
 
-    private void assertLogLine(final String logLine, final Level level, final String location, final String message) {
-        final Matcher matcher = Pattern.compile("\\[(.*)\\]\\[(.*)\\(.*\\)\\] (.*)").matcher(logLine);
-        assertTrue(logLine, matcher.matches());
-        assertThat(matcher.group(1), equalTo(level.toString()));
-        assertThat(matcher.group(2), RegexMatcher.matches(location));
-        assertThat(matcher.group(3), RegexMatcher.matches(message));
+    public void testDeprecationLogger() throws IOException, UserException {
+        setupLogging("deprecation");
+
+        final DeprecationLogger deprecationLogger = new DeprecationLogger(ESLoggerFactory.getLogger("deprecation"));
+
+        final int deprecatedIterations = randomIntBetween(0, 256);
+        for (int i = 0; i < deprecatedIterations; i++) {
+            deprecationLogger.deprecated("This is a deprecation message");
+            assertWarnings("This is a deprecation message");
+        }
+
+        final String deprecationPath =
+            System.getProperty("es.logs.base_path") +
+                System.getProperty("file.separator") +
+                System.getProperty("es.logs.cluster_name") +
+                "_deprecation.log";
+        final List<String> deprecationEvents = Files.readAllLines(PathUtils.get(deprecationPath));
+        assertThat(deprecationEvents.size(), equalTo(deprecatedIterations));
+        for (int i = 0; i < deprecatedIterations; i++) {
+            assertLogLine(
+                    deprecationEvents.get(i),
+                    Level.WARN,
+                    "org.elasticsearch.common.logging.DeprecationLogger.deprecated",
+                    "This is a deprecation message");
+        }
     }
 
-    public void testDeprecationLogger() throws IOException {
-        deprecationLogger.deprecated("This is a deprecation message");
-        final String deprecationPath = System.getProperty("es.logs") + "_deprecation.log";
+    public void testConcurrentDeprecationLogger() throws IOException, UserException, BrokenBarrierException, InterruptedException {
+        setupLogging("deprecation");
+
+        final DeprecationLogger deprecationLogger = new DeprecationLogger(ESLoggerFactory.getLogger("deprecation"));
+
+        final int numberOfThreads = randomIntBetween(2, 4);
+        final CyclicBarrier barrier = new CyclicBarrier(1 + numberOfThreads);
+        final List<Thread> threads = new ArrayList<>();
+        final int iterations = randomIntBetween(1, 4);
+        for (int i = 0; i < numberOfThreads; i++) {
+            final Thread thread = new Thread(() -> {
+                final List<Integer> ids = IntStream.range(0, 128).boxed().collect(Collectors.toList());
+                Randomness.shuffle(ids);
+                final ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
+                DeprecationLogger.setThreadContext(threadContext);
+                try {
+                    barrier.await();
+                } catch (final BrokenBarrierException | InterruptedException e) {
+                    throw new RuntimeException(e);
+                }
+                for (int j = 0; j < iterations; j++) {
+                    for (final Integer id : ids) {
+                        deprecationLogger.deprecatedAndMaybeLog(Integer.toString(id), "This is a maybe logged deprecation message" + id);
+                    }
+                }
+
+                /*
+                 * We have to manually check that each thread has the right warning headers in the thread context because the act of doing
+                 * this through the test framework on one thread would otherwise clear the thread context and we would be unable to assert
+                 * on the other threads.
+                 */
+                final List<String> warnings = threadContext.getResponseHeaders().get("Warning");
+                final Set<String> actualWarningValues =
+                        warnings.stream().map(DeprecationLogger::extractWarningValueFromWarningHeader).collect(Collectors.toSet());
+                for (int j = 0; j < 128; j++) {
+                    assertThat(actualWarningValues, hasItem(DeprecationLogger.escape("This is a maybe logged deprecation message" + j)));
+                }
+
+                try {
+                    barrier.await();
+                } catch (final BrokenBarrierException | InterruptedException e) {
+                    throw new RuntimeException(e);
+                }
+            });
+            threads.add(thread);
+            thread.start();
+        }
+
+        // synchronize the start of all threads
+        barrier.await();
+
+        // wait for all threads to complete their iterations
+        barrier.await();
+
+        final String deprecationPath =
+                System.getProperty("es.logs.base_path") +
+                        System.getProperty("file.separator") +
+                        System.getProperty("es.logs.cluster_name") +
+                        "_deprecation.log";
         final List<String> deprecationEvents = Files.readAllLines(PathUtils.get(deprecationPath));
-        assertThat(deprecationEvents.size(), equalTo(1));
+        // we appended an integer to each log message, use that for sorting
+        deprecationEvents.sort(Comparator.comparingInt(s -> Integer.parseInt(s.split("message")[1])));
+        assertThat(deprecationEvents.size(), equalTo(128));
+        for (int i = 0; i < 128; i++) {
+            assertLogLine(
+                    deprecationEvents.get(i),
+                    Level.WARN,
+                    "org.elasticsearch.common.logging.DeprecationLogger.deprecated",
+                    "This is a maybe logged deprecation message" + i);
+        }
+
+        for (final Thread thread : threads) {
+            thread.join();
+        }
+
+    }
+
+    public void testDeprecationLoggerMaybeLog() throws IOException, UserException {
+        setupLogging("deprecation");
+
+        final DeprecationLogger deprecationLogger = new DeprecationLogger(ESLoggerFactory.getLogger("deprecation"));
+
+        final int iterations = randomIntBetween(1, 16);
+
+        for (int i = 0; i < iterations; i++) {
+            deprecationLogger.deprecatedAndMaybeLog("key", "This is a maybe logged deprecation message");
+            assertWarnings("This is a maybe logged deprecation message");
+        }
+        for (int k = 0; k < 128; k++) {
+            for (int i = 0; i < iterations; i++) {
+                deprecationLogger.deprecatedAndMaybeLog("key" + k, "This is a maybe logged deprecation message" + k);
+                assertWarnings("This is a maybe logged deprecation message" + k);
+            }
+        }
+        for (int i = 0; i < iterations; i++) {
+            deprecationLogger.deprecatedAndMaybeLog("key", "This is a maybe logged deprecation message");
+            assertWarnings("This is a maybe logged deprecation message");
+        }
+
+        final String deprecationPath =
+                System.getProperty("es.logs.base_path") +
+                        System.getProperty("file.separator") +
+                        System.getProperty("es.logs.cluster_name") +
+                        "_deprecation.log";
+        final List<String> deprecationEvents = Files.readAllLines(PathUtils.get(deprecationPath));
+        assertThat(deprecationEvents.size(), equalTo(1 + 128 + 1));
         assertLogLine(
-            deprecationEvents.get(0),
-            Level.WARN,
-            "org.elasticsearch.common.logging.DeprecationLogger.deprecated",
-            "This is a deprecation message");
+                deprecationEvents.get(0),
+                Level.WARN,
+                "org.elasticsearch.common.logging.DeprecationLogger.deprecated",
+                "This is a maybe logged deprecation message");
+        for (int k = 0; k < 128; k++) {
+            assertLogLine(
+                    deprecationEvents.get(1 + k),
+                    Level.WARN,
+                    "org.elasticsearch.common.logging.DeprecationLogger.deprecated",
+                    "This is a maybe logged deprecation message" + k);
+        }
+    }
+
+    public void testDeprecatedSettings() throws IOException, UserException {
+        setupLogging("settings");
+
+        final Setting<Boolean> setting = Setting.boolSetting("deprecated.foo", false, Setting.Property.Deprecated);
+        final Settings settings = Settings.builder().put("deprecated.foo", true).build();
+
+        final int iterations = randomIntBetween(0, 128);
+        for (int i = 0; i < iterations; i++) {
+            setting.get(settings);
+            assertSettingDeprecationsAndWarnings(new Setting<?>[]{setting});
+        }
+
+        final String deprecationPath =
+                System.getProperty("es.logs.base_path") +
+                        System.getProperty("file.separator") +
+                        System.getProperty("es.logs.cluster_name") +
+                        "_deprecation.log";
+        final List<String> deprecationEvents = Files.readAllLines(PathUtils.get(deprecationPath));
+        if (iterations > 0) {
+            assertThat(deprecationEvents.size(), equalTo(1));
+            assertLogLine(
+                    deprecationEvents.get(0),
+                    Level.WARN,
+                    "org.elasticsearch.common.logging.DeprecationLogger.deprecated",
+                    "\\[deprecated.foo\\] setting was deprecated in Elasticsearch and will be removed in a future release! " +
+                            "See the breaking changes documentation for the next major version.");
+        }
     }
 
-    public void testUnsupportedLoggingConfigurationFiles() throws IOException {
-        // TODO: the warning for unsupported logging configurations can be removed in 6.0.0
-        assert Version.CURRENT.major < 6;
+    public void testUnsupportedLoggingConfigurationFiles() throws IOException, UserException {
+        assumeTrue(
+                "will be fixed when https://github.com/apache/logging-log4j2/pull/109 is integrated into to Log4j and we bump versions",
+                !Constants.JRE_IS_MINIMUM_JAVA9);
+        setupLogging("unsupported");
+
         final String path = System.getProperty("es.logs") + ".log";
         final List<String> events = Files.readAllLines(PathUtils.get(path));
         assertThat(events.size(), equalTo(1));
         assertLogLine(
-            events.get(0),
-            Level.WARN,
-            "org\\.elasticsearch\\.common\\.logging\\.LogConfigurator.*",
-            "ignoring unsupported logging configuration file \\[.*\\], logging is configured via \\[.*\\]");
+                events.get(0),
+                Level.WARN,
+                "org\\.elasticsearch\\.common\\.logging\\.LogConfigurator.*",
+                "ignoring unsupported logging configuration file \\[.*\\], logging is configured via \\[.*\\]");
+    }
+
+    public void testFindAppender() throws IOException, UserException {
+        setupLogging("find_appender");
+
+        final Logger hasConsoleAppender = ESLoggerFactory.getLogger("has_console_appender");
+
+        final Appender testLoggerConsoleAppender = Loggers.findAppender(hasConsoleAppender, ConsoleAppender.class);
+        assertNotNull(testLoggerConsoleAppender);
+        assertThat(testLoggerConsoleAppender.getName(), equalTo("console"));
+        final Logger hasCountingNoOpAppender = ESLoggerFactory.getLogger("has_counting_no_op_appender");
+        assertNull(Loggers.findAppender(hasCountingNoOpAppender, ConsoleAppender.class));
+        final Appender countingNoOpAppender = Loggers.findAppender(hasCountingNoOpAppender, CountingNoOpAppender.class);
+        assertThat(countingNoOpAppender.getName(), equalTo("counting_no_op"));
+    }
+
+    public void testPrefixLogger() throws IOException, IllegalAccessException, UserException {
+        setupLogging("prefix");
+
+        final String prefix = randomBoolean() ? null : randomAlphaOfLength(16);
+        final Logger logger = Loggers.getLogger("prefix", prefix);
+        logger.info("test");
+        logger.info("{}", "test");
+        final Exception e = new Exception("exception");
+        logger.info(new ParameterizedMessage("{}", "test"), e);
+
+        final String path =
+            System.getProperty("es.logs.base_path") +
+                System.getProperty("file.separator") +
+                System.getProperty("es.logs.cluster_name") +
+                ".log";
+        final List<String> events = Files.readAllLines(PathUtils.get(path));
+
+        final StringWriter sw = new StringWriter();
+        final PrintWriter pw = new PrintWriter(sw);
+        e.printStackTrace(pw);
+        final int stackTraceLength = sw.toString().split(System.getProperty("line.separator")).length;
+        final int expectedLogLines = 3;
+        assertThat(events.size(), equalTo(expectedLogLines + stackTraceLength));
+        for (int i = 0; i < expectedLogLines; i++) {
+            if (prefix == null) {
+                assertThat(events.get(i), startsWith("test"));
+            } else {
+                assertThat(events.get(i), startsWith("[" + prefix + "] test"));
+            }
+        }
+    }
+
+    public void testPrefixLoggerMarkersCanBeCollected() throws IOException, UserException {
+        setupLogging("prefix");
+
+        final int prefixes = 1 << 19; // to ensure enough markers that the GC should collect some when we force a GC below
+        for (int i = 0; i < prefixes; i++) {
+            Loggers.getLogger("prefix" + i, "prefix" + i); // this has the side effect of caching a marker with this prefix
+
+        }
+
+        System.gc(); // this will free the weakly referenced keys in the marker cache
+        assertThat(PrefixLogger.markersSize(), lessThan(prefixes));
+    }
+
+    public void testProperties() throws IOException, UserException {
+        final Settings.Builder builder = Settings.builder().put("cluster.name", randomAlphaOfLength(16));
+        if (randomBoolean()) {
+            builder.put("node.name", randomAlphaOfLength(16));
+        }
+        final Settings settings = builder.build();
+        setupLogging("minimal", settings);
+
+        assertNotNull(System.getProperty("es.logs.base_path"));
+
+        assertThat(System.getProperty("es.logs.cluster_name"), equalTo(ClusterName.CLUSTER_NAME_SETTING.get(settings).value()));
+        if (Node.NODE_NAME_SETTING.exists(settings)) {
+            assertThat(System.getProperty("es.logs.node_name"), equalTo(Node.NODE_NAME_SETTING.get(settings)));
+        } else {
+            assertNull(System.getProperty("es.logs.node_name"));
+        }
+    }
+
+    private void setupLogging(final String config) throws IOException, UserException {
+        setupLogging(config, Settings.EMPTY);
+    }
+
+    private void setupLogging(final String config, final Settings settings) throws IOException, UserException {
+        assert !Environment.PATH_CONF_SETTING.exists(settings);
+        assert !Environment.PATH_HOME_SETTING.exists(settings);
+        final Path configDir = getDataPath(config);
+        // need to set custom path.conf so we can use a custom log4j2.properties file for the test
+        final Settings mergedSettings = Settings.builder()
+            .put(settings)
+            .put(Environment.PATH_CONF_SETTING.getKey(), configDir.toAbsolutePath())
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+            .build();
+        final Environment environment = new Environment(mergedSettings);
+        LogConfigurator.configure(environment);
+    }
+
+    private void assertLogLine(final String logLine, final Level level, final String location, final String message) {
+        final Matcher matcher = Pattern.compile("\\[(.*)\\]\\[(.*)\\(.*\\)\\] (.*)").matcher(logLine);
+        assertTrue(logLine, matcher.matches());
+        assertThat(matcher.group(1), equalTo(level.toString()));
+        assertThat(matcher.group(2), RegexMatcher.matches(location));
+        assertThat(matcher.group(3), RegexMatcher.matches(message));
     }
 
 }
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/env/NodeEnvironmentEvilTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/env/NodeEnvironmentEvilTests.java
index ccf5e1b105e35..3eebf4a2f6481 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/env/NodeEnvironmentEvilTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/env/NodeEnvironmentEvilTests.java
@@ -30,7 +30,6 @@
 import java.nio.file.attribute.PosixFileAttributeView;
 import java.nio.file.attribute.PosixFilePermission;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.HashSet;
 
 public class NodeEnvironmentEvilTests extends ESTestCase {
@@ -75,7 +74,7 @@ public void testMissingWritePermissionOnIndex() throws IOException {
             IOException ioException = expectThrows(IOException.class, () -> {
                 new NodeEnvironment(build, new Environment(build));
             });
-            assertTrue(ioException.getMessage(), ioException.getMessage().startsWith("failed to write in data directory"));
+            assertTrue(ioException.getMessage(), ioException.getMessage().startsWith("failed to test writes in data directory"));
         }
     }
 
@@ -100,7 +99,7 @@ public void testMissingWritePermissionOnShard() throws IOException {
             IOException ioException = expectThrows(IOException.class, () -> {
                 new NodeEnvironment(build, new Environment(build));
             });
-            assertTrue(ioException.getMessage(), ioException.getMessage().startsWith("failed to write in data directory"));
+            assertTrue(ioException.getMessage(), ioException.getMessage().startsWith("failed to test writes in data directory"));
         }
     }
 }
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/node/EvilNodeTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/node/EvilNodeTests.java
new file mode 100644
index 0000000000000..341d1227926e8
--- /dev/null
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/node/EvilNodeTests.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.node;
+
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.util.Constants;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+
+public class EvilNodeTests extends ESTestCase {
+
+    public void testDefaultPathDataIncludedInPathData() throws IOException {
+        final Path zero = createTempDir().toAbsolutePath();
+        final Path one = createTempDir().toAbsolutePath();
+        // creating hard links to directories is okay on macOS so we exercise it here
+        final int random;
+        if (Constants.MAC_OS_X) {
+            random = randomFrom(0, 1, 2);
+        } else {
+            random = randomFrom(0, 1);
+        }
+        final Path defaultPathData;
+        final Path choice = randomFrom(zero, one);
+        switch (random) {
+            case 0:
+                defaultPathData = choice;
+                break;
+            case 1:
+                defaultPathData = createTempDir().toAbsolutePath().resolve("link");
+                Files.createSymbolicLink(defaultPathData, choice);
+                break;
+            case 2:
+                defaultPathData = createTempDir().toAbsolutePath().resolve("link");
+                Files.createLink(defaultPathData, choice);
+                break;
+            default:
+                throw new AssertionError(Integer.toString(random));
+        }
+        final Settings settings = Settings.builder()
+                .put("path.home", createTempDir().toAbsolutePath())
+                .put("path.data.0", zero)
+                .put("path.data.1", one)
+                .put("default.path.data", defaultPathData)
+                .build();
+        try (NodeEnvironment nodeEnv = new NodeEnvironment(settings, new Environment(settings))) {
+            final Path defaultPathDataWithNodesAndId = defaultPathData.resolve("nodes/0");
+            Files.createDirectories(defaultPathDataWithNodesAndId);
+            final NodeEnvironment.NodePath defaultNodePath = new NodeEnvironment.NodePath(defaultPathDataWithNodesAndId);
+            Files.createDirectories(defaultNodePath.indicesPath.resolve(UUIDs.randomBase64UUID()));
+            final Logger mock = mock(Logger.class);
+            // nothing should happen here
+            Node.checkForIndexDataInDefaultPathData(settings, nodeEnv, mock);
+            verifyNoMoreInteractions(mock);
+        }
+    }
+
+}
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java
deleted file mode 100644
index af1f311dd23d6..0000000000000
--- a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java
+++ /dev/null
@@ -1,633 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.plugins;
-
-import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
-import com.google.common.jimfs.Configuration;
-import com.google.common.jimfs.Jimfs;
-import org.apache.lucene.util.LuceneTestCase;
-import org.elasticsearch.Version;
-import org.elasticsearch.cli.MockTerminal;
-import org.elasticsearch.cli.Terminal;
-import org.elasticsearch.cli.UserException;
-import org.elasticsearch.common.SuppressForbidden;
-import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.io.PathUtils;
-import org.elasticsearch.common.io.PathUtilsForTesting;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.PosixPermissionsResetter;
-import org.junit.After;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.StringReader;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.DirectoryStream;
-import java.nio.file.FileAlreadyExistsException;
-import java.nio.file.FileSystem;
-import java.nio.file.FileVisitResult;
-import java.nio.file.Files;
-import java.nio.file.NoSuchFileException;
-import java.nio.file.Path;
-import java.nio.file.SimpleFileVisitor;
-import java.nio.file.StandardCopyOption;
-import java.nio.file.attribute.BasicFileAttributes;
-import java.nio.file.attribute.PosixFileAttributeView;
-import java.nio.file.attribute.PosixFileAttributes;
-import java.nio.file.attribute.PosixFilePermission;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipOutputStream;
-
-import static org.hamcrest.CoreMatchers.containsString;
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.Matchers.containsInAnyOrder;
-import static org.hamcrest.Matchers.not;
-
-@LuceneTestCase.SuppressFileSystems("*")
-public class InstallPluginCommandTests extends ESTestCase {
-
-    private final Function<String, Path> temp;
-
-    private final FileSystem fs;
-    private final boolean isPosix;
-    private final boolean isReal;
-    private final String javaIoTmpdir;
-
-    @SuppressForbidden(reason = "sets java.io.tmpdir")
-    public InstallPluginCommandTests(FileSystem fs, Function<String, Path> temp) {
-        this.fs = fs;
-        this.temp = temp;
-        this.isPosix = fs.supportedFileAttributeViews().contains("posix");
-        this.isReal = fs == PathUtils.getDefaultFileSystem();
-        PathUtilsForTesting.installMock(fs);
-        javaIoTmpdir = System.getProperty("java.io.tmpdir");
-        System.setProperty("java.io.tmpdir", temp.apply("tmpdir").toString());
-    }
-
-    @After
-    @SuppressForbidden(reason = "resets java.io.tmpdir")
-    public void tearDown() throws Exception {
-        System.setProperty("java.io.tmpdir", javaIoTmpdir);
-        PathUtilsForTesting.teardown();
-        super.tearDown();
-    }
-
-    @ParametersFactory
-    public static Iterable<Object[]> parameters() {
-        class Parameter {
-            private final FileSystem fileSystem;
-            private final Function<String, Path> temp;
-
-            public Parameter(FileSystem fileSystem, String root) {
-                this(fileSystem, s -> {
-                    try {
-                        return Files.createTempDirectory(fileSystem.getPath(root), s);
-                    } catch (IOException e) {
-                        throw new RuntimeException(e);
-                    }
-                });
-            }
-
-            public Parameter(FileSystem fileSystem, Function<String, Path> temp) {
-                this.fileSystem = fileSystem;
-                this.temp = temp;
-            }
-        }
-        List<Parameter> parameters = new ArrayList<>();
-        parameters.add(new Parameter(Jimfs.newFileSystem(Configuration.windows()), "c:\\"));
-        parameters.add(new Parameter(Jimfs.newFileSystem(toPosix(Configuration.osX())), "/"));
-        parameters.add(new Parameter(Jimfs.newFileSystem(toPosix(Configuration.unix())), "/"));
-        parameters.add(new Parameter(PathUtils.getDefaultFileSystem(), LuceneTestCase::createTempDir ));
-        return parameters.stream().map(p -> new Object[] { p.fileSystem, p.temp }).collect(Collectors.toList());
-    }
-
-    private static Configuration toPosix(Configuration configuration) {
-        return configuration.toBuilder().setAttributeViews("basic", "owner", "posix", "unix").build();
-    }
-
-    /** Creates a test environment with bin, config and plugins directories. */
-    static Tuple<Path, Environment> createEnv(FileSystem fs, Function<String, Path> temp) throws IOException {
-        Path home = temp.apply("install-plugin-command-tests");
-        Files.createDirectories(home.resolve("bin"));
-        Files.createFile(home.resolve("bin").resolve("elasticsearch"));
-        Files.createDirectories(home.resolve("config"));
-        Files.createFile(home.resolve("config").resolve("elasticsearch.yml"));
-        Path plugins = Files.createDirectories(home.resolve("plugins"));
-        assertTrue(Files.exists(plugins));
-        Settings settings = Settings.builder()
-            .put("path.home", home)
-            .build();
-        return Tuple.tuple(home, new Environment(settings));
-    }
-
-    static Path createPluginDir(Function<String, Path> temp) throws IOException {
-        return temp.apply("pluginDir");
-    }
-
-    /** creates a fake jar file with empty class files */
-    static void writeJar(Path jar, String... classes) throws IOException {
-        try (ZipOutputStream stream = new ZipOutputStream(Files.newOutputStream(jar))) {
-            for (String clazz : classes) {
-                stream.putNextEntry(new ZipEntry(clazz + ".class")); // no package names, just support simple classes
-            }
-        }
-    }
-
-    static String writeZip(Path structure, String prefix) throws IOException {
-        Path zip = createTempDir().resolve(structure.getFileName() + ".zip");
-        try (ZipOutputStream stream = new ZipOutputStream(Files.newOutputStream(zip))) {
-            Files.walkFileTree(structure, new SimpleFileVisitor<Path>() {
-                @Override
-                public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
-                    String target = (prefix == null ? "" : prefix + "/") + structure.relativize(file).toString();
-                    stream.putNextEntry(new ZipEntry(target));
-                    Files.copy(file, stream);
-                    return FileVisitResult.CONTINUE;
-                }
-            });
-        }
-        return zip.toUri().toURL().toString();
-    }
-
-    /** creates a plugin .zip and returns the url for testing */
-    static String createPlugin(String name, Path structure) throws IOException {
-        return createPlugin(name, structure, false);
-    }
-
-    static String createPlugin(String name, Path structure, boolean createSecurityPolicyFile) throws IOException {
-        PluginTestUtil.writeProperties(structure,
-            "description", "fake desc",
-            "name", name,
-            "version", "1.0",
-            "elasticsearch.version", Version.CURRENT.toString(),
-            "java.version", System.getProperty("java.specification.version"),
-            "classname", "FakePlugin");
-        if (createSecurityPolicyFile) {
-            String securityPolicyContent = "grant {\n  permission java.lang.RuntimePermission \"setFactory\";\n};\n";
-            Files.write(structure.resolve("plugin-security.policy"), securityPolicyContent.getBytes(StandardCharsets.UTF_8));
-        }
-        writeJar(structure.resolve("plugin.jar"), "FakePlugin");
-        return writeZip(structure, "elasticsearch");
-    }
-
-    static MockTerminal installPlugin(String pluginUrl, Path home) throws Exception {
-        return installPlugin(pluginUrl, home, false);
-    }
-
-    static MockTerminal installPlugin(String pluginUrl, Path home, boolean jarHellCheck) throws Exception {
-        Map<String, String> settings = new HashMap<>();
-        settings.put("path.home", home.toString());
-        MockTerminal terminal = new MockTerminal();
-        new InstallPluginCommand() {
-            @Override
-            void jarHellCheck(Path candidate, Path pluginsDir) throws Exception {
-                if (jarHellCheck) {
-                    super.jarHellCheck(candidate, pluginsDir);
-                }
-            }
-        }.execute(terminal, pluginUrl, true, settings);
-        return terminal;
-    }
-
-    void assertPlugin(String name, Path original, Environment env) throws IOException {
-        Path got = env.pluginsFile().resolve(name);
-        assertTrue("dir " + name + " exists", Files.exists(got));
-
-        if (isPosix) {
-            Set<PosixFilePermission> perms = Files.getPosixFilePermissions(got);
-            assertThat(
-                perms,
-                containsInAnyOrder(
-                    PosixFilePermission.OWNER_READ,
-                    PosixFilePermission.OWNER_WRITE,
-                    PosixFilePermission.OWNER_EXECUTE,
-                    PosixFilePermission.GROUP_READ,
-                    PosixFilePermission.GROUP_EXECUTE,
-                    PosixFilePermission.OTHERS_READ,
-                    PosixFilePermission.OTHERS_EXECUTE));
-        }
-
-        assertTrue("jar was copied", Files.exists(got.resolve("plugin.jar")));
-        assertFalse("bin was not copied", Files.exists(got.resolve("bin")));
-        assertFalse("config was not copied", Files.exists(got.resolve("config")));
-        if (Files.exists(original.resolve("bin"))) {
-            Path binDir = env.binFile().resolve(name);
-            assertTrue("bin dir exists", Files.exists(binDir));
-            assertTrue("bin is a dir", Files.isDirectory(binDir));
-            PosixFileAttributes binAttributes = null;
-            if (isPosix) {
-                binAttributes = Files.readAttributes(env.binFile(), PosixFileAttributes.class);
-            }
-            try (DirectoryStream<Path> stream = Files.newDirectoryStream(binDir)) {
-                for (Path file : stream) {
-                    assertFalse("not a dir", Files.isDirectory(file));
-                    if (isPosix) {
-                        PosixFileAttributes attributes = Files.readAttributes(file, PosixFileAttributes.class);
-                        assertEquals(InstallPluginCommand.DIR_AND_EXECUTABLE_PERMS, attributes.permissions());
-                    }
-                }
-            }
-        }
-        if (Files.exists(original.resolve("config"))) {
-            Path configDir = env.configFile().resolve(name);
-            assertTrue("config dir exists", Files.exists(configDir));
-            assertTrue("config is a dir", Files.isDirectory(configDir));
-
-            if (isPosix) {
-                Path configRoot = env.configFile();
-                PosixFileAttributes configAttributes =
-                    Files.getFileAttributeView(configRoot, PosixFileAttributeView.class).readAttributes();
-                PosixFileAttributes attributes = Files.getFileAttributeView(configDir, PosixFileAttributeView.class).readAttributes();
-                assertThat(attributes.owner(), equalTo(configAttributes.owner()));
-                assertThat(attributes.group(), equalTo(configAttributes.group()));
-            }
-
-            try (DirectoryStream<Path> stream = Files.newDirectoryStream(configDir)) {
-                for (Path file : stream) {
-                    assertFalse("not a dir", Files.isDirectory(file));
-                }
-            }
-        }
-        assertInstallCleaned(env);
-    }
-
-    void assertInstallCleaned(Environment env) throws IOException {
-        try (DirectoryStream<Path> stream = Files.newDirectoryStream(env.pluginsFile())) {
-            for (Path file : stream) {
-                if (file.getFileName().toString().startsWith(".installing")) {
-                    fail("Installation dir still exists, " + file);
-                }
-            }
-        }
-    }
-
-    public void testSomethingWorks() throws Exception {
-        Tuple<Path, Environment> env = createEnv(fs, temp);
-        Path pluginDir = createPluginDir(temp);
-        String pluginZip = createPlugin("fake", pluginDir);
-        installPlugin(pluginZip, env.v1());
-        assertPlugin("fake", pluginDir, env.v2());
-    }
-
-    public void testSpaceInUrl() throws Exception {
-        Tuple<Path, Environment> env = createEnv(fs, temp);
-        Path pluginDir = createPluginDir(temp);
-        String pluginZip = createPlugin("fake", pluginDir);
-        Path pluginZipWithSpaces = createTempFile("foo bar", ".zip");
-        try (InputStream in = new URL(pluginZip).openStream()) {
-            Files.copy(in, pluginZipWithSpaces, StandardCopyOption.REPLACE_EXISTING);
-        }
-        installPlugin(pluginZipWithSpaces.toUri().toURL().toString(), env.v1());
-        assertPlugin("fake", pluginDir, env.v2());
-    }
-
-    public void testMalformedUrlNotMaven() throws Exception {
-        Tuple<Path, Environment> env = createEnv(fs, temp);
-        // has two colons, so it appears similar to maven coordinates
-        MalformedURLException e = expectThrows(MalformedURLException.class, () -> installPlugin("://host:1234", env.v1()));
-        assertTrue(e.getMessage(), e.getMessage().contains("no protocol"));
-    }
-
-    public void testUnknownPlugin() throws Exception {
-        Tuple<Path, Environment> env = createEnv(fs, temp);
-        UserException e = expectThrows(UserException.class, () -> installPlugin("foo", env.v1()));
-        assertTrue(e.getMessage(), e.getMessage().contains("Unknown plugin foo"));
-    }
-
-    public void testPluginsDirMissing() throws Exception {
-        Tuple<Path, Environment> env = createEnv(fs, temp);
-        Files.delete(env.v2().pluginsFile());
-        Path pluginDir = createPluginDir(temp);
-        String pluginZip = createPlugin("fake", pluginDir);
-        installPlugin(pluginZip, env.v1());
-        assertPlugin("fake", pluginDir, env.v2());
-    }
-
-    public void testPluginsDirReadOnly() throws Exception {
-        assumeTrue("posix and filesystem", isPosix && isReal);
-        Tuple<Path, Environment> env = createEnv(fs, temp);
-        Path pluginDir = createPluginDir(temp);
-        try (PosixPermissionsResetter pluginsAttrs = new PosixPermissionsResetter(env.v2().pluginsFile())) {
-            pluginsAttrs.setPermissions(new HashSet<>());
-            String pluginZip = createPlugin("fake", pluginDir);
-            IOException e = expectThrows(IOException.class, () -> installPlugin(pluginZip, env.v1()));
-            assertTrue(e.getMessage(), e.getMessage().contains(env.v2().pluginsFile().toString()));
-        }
-        assertInstallCleaned(env.v2());
-    }
-
-    public void testBuiltinModule() throws Exception {
-        Tuple<Path, Environment> env = createEnv(fs, temp);
-        Path pluginDir = createPluginDir(temp);
-        String pluginZip = createPlugin("lang-groovy", pluginDir);
-        UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1()));
-        assertTrue(e.getMessage(), e.getMessage().contains("is a system module"));
-        assertInstallCleaned(env.v2());
-    }
-
-    public void testJarHell() throws Exception {
-        // jar hell test needs a real filesystem
-        assumeTrue("real filesystem", isReal);
-        Tuple<Path, Environment> environment = createEnv(fs, temp);
-        Path pluginDirectory = createPluginDir(temp);
-        writeJar(pluginDirectory.resolve("other.jar"), "FakePlugin");
-        String pluginZip = createPlugin("fake", pluginDirectory); // adds plugin.jar with FakePlugin
-        IllegalStateException e = expectThrows(IllegalStateException.class, () -> installPlugin(pluginZip, environment.v1(), true));
-        assertTrue(e.getMessage(), e.getMessage().contains("jar hell"));
-        assertInstallCleaned(environment.v2());
-    }
-
-    public void testIsolatedPlugins() throws Exception {
-        Tuple<Path, Environment> env = createEnv(fs, temp);
-        // these both share the same FakePlugin class
-        Path pluginDir1 = createPluginDir(temp);
-        String pluginZip1 = createPlugin("fake1", pluginDir1);
-        installPlugin(pluginZip1, env.v1());
-        Path pluginDir2 = createPluginDir(temp);
-        String pluginZip2 = createPlugin("fake2", pluginDir2);
-        installPlugin(pluginZip2, env.v1());
-        assertPlugin("fake1", pluginDir1, env.v2());
-        assertPlugin("fake2", pluginDir2, env.v2());
-    }
-
-    public void testExistingPlugin() throws Exception {
-        Tuple<Path, Environment> env = createEnv(fs, temp);
-        Path pluginDir = createPluginDir(temp);
-        String pluginZip = createPlugin("fake", pluginDir);
-        installPlugin(pluginZip, env.v1());
-        UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1()));
-        assertTrue(e.getMessage(), e.getMessage().contains("already exists"));
-        assertInstallCleaned(env.v2());
-    }
-
-    public void testBin() throws Exception {
-        Tuple<Path, Environment> env = createEnv(fs, temp);
-        Path pluginDir = createPluginDir(temp);
-        Path binDir = pluginDir.resolve("bin");
-        Files.createDirectory(binDir);
-        Files.createFile(binDir.resolve("somescript"));
-        String pluginZip = createPlugin("fake", pluginDir);
-        installPlugin(pluginZip, env.v1());
-        assertPlugin("fake", pluginDir, env.v2());
-    }
-
-    public void testBinNotDir() throws Exception {
-        Tuple<Path, Environment> env = createEnv(fs, temp);
-        Path pluginDir = createPluginDir(temp);
-        Path binDir = pluginDir.resolve("bin");
-        Files.createFile(binDir);
-        String pluginZip = createPlugin("fake", pluginDir);
-        UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1()));
-        assertTrue(e.getMessage(), e.getMessage().contains("not a directory"));
-        assertInstallCleaned(env.v2());
-    }
-
-    public void testBinContainsDir() throws Exception {
-        Tuple<Path, Environment> env = createEnv(fs, temp);
-        Path pluginDir = createPluginDir(temp);
-        Path dirInBinDir = pluginDir.resolve("bin").resolve("foo");
-        Files.createDirectories(dirInBinDir);
-        Files.createFile(dirInBinDir.resolve("somescript"));
-        String pluginZip = createPlugin("fake", pluginDir);
-        UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1()));
-        assertTrue(e.getMessage(), e.getMessage().contains("Directories not allowed in bin dir for plugin"));
-        assertInstallCleaned(env.v2());
-    }
-
-    public void testBinConflict() throws Exception {
-        Tuple<Path, Environment> env = createEnv(fs, temp);
-        Path pluginDir = createPluginDir(temp);
-        Path binDir = pluginDir.resolve("bin");
-        Files.createDirectory(binDir);
-        Files.createFile(binDir.resolve("somescript"));
-        String pluginZip = createPlugin("elasticsearch", pluginDir);
-        FileAlreadyExistsException e = expectThrows(FileAlreadyExistsException.class, () -> installPlugin(pluginZip, env.v1()));
-        assertTrue(e.getMessage(), e.getMessage().contains(env.v2().binFile().resolve("elasticsearch").toString()));
-        assertInstallCleaned(env.v2());
-    }
-
-    public void testBinPermissions() throws Exception {
-        assumeTrue("posix filesystem", isPosix);
-        Tuple<Path, Environment> env = createEnv(fs, temp);
-        Path pluginDir = createPluginDir(temp);
-        Path binDir = pluginDir.resolve("bin");
-        Files.createDirectory(binDir);
-        Files.createFile(binDir.resolve("somescript"));
-        String pluginZip = createPlugin("fake", pluginDir);
-        try (PosixPermissionsResetter binAttrs = new PosixPermissionsResetter(env.v2().binFile())) {
-            Set<PosixFilePermission> perms = binAttrs.getCopyPermissions();
-            // make sure at least one execute perm is missing, so we know we forced it during installation
-            perms.remove(PosixFilePermission.GROUP_EXECUTE);
-            binAttrs.setPermissions(perms);
-            installPlugin(pluginZip, env.v1());
-            assertPlugin("fake", pluginDir, env.v2());
-        }
-    }
-
-    public void testConfig() throws Exception {
-        Tuple<Path, Environment> env = createEnv(fs, temp);
-        Path pluginDir = createPluginDir(temp);
-        Path configDir = pluginDir.resolve("config");
-        Files.createDirectory(configDir);
-        Files.createFile(configDir.resolve("custom.yaml"));
-        String pluginZip = createPlugin("fake", pluginDir);
-        installPlugin(pluginZip, env.v1());
-        assertPlugin("fake", pluginDir, env.v2());
-    }
-
-    public void testExistingConfig() throws Exception {
-        Tuple<Path, Environment> env = createEnv(fs, temp);
-        Path envConfigDir = env.v2().configFile().resolve("fake");
-        Files.createDirectories(envConfigDir);
-        Files.write(envConfigDir.resolve("custom.yaml"), "existing config".getBytes(StandardCharsets.UTF_8));
-        Path pluginDir = createPluginDir(temp);
-        Path configDir = pluginDir.resolve("config");
-        Files.createDirectory(configDir);
-        Files.write(configDir.resolve("custom.yaml"), "new config".getBytes(StandardCharsets.UTF_8));
-        Files.createFile(configDir.resolve("other.yaml"));
-        String pluginZip = createPlugin("fake", pluginDir);
-        installPlugin(pluginZip, env.v1());
-        assertPlugin("fake", pluginDir, env.v2());
-        List<String> configLines = Files.readAllLines(envConfigDir.resolve("custom.yaml"), StandardCharsets.UTF_8);
-        assertEquals(1, configLines.size());
-        assertEquals("existing config", configLines.get(0));
-        assertTrue(Files.exists(envConfigDir.resolve("other.yaml")));
-    }
-
-    public void testConfigNotDir() throws Exception {
-        Tuple<Path, Environment> env = createEnv(fs, temp);
-        Path pluginDir = createPluginDir(temp);
-        Path configDir = pluginDir.resolve("config");
-        Files.createFile(configDir);
-        String pluginZip = createPlugin("fake", pluginDir);
-        UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1()));
-        assertTrue(e.getMessage(), e.getMessage().contains("not a directory"));
-        assertInstallCleaned(env.v2());
-    }
-
-    public void testConfigContainsDir() throws Exception {
-        Tuple<Path, Environment> env = createEnv(fs, temp);
-        Path pluginDir = createPluginDir(temp);
-        Path dirInConfigDir = pluginDir.resolve("config").resolve("foo");
-        Files.createDirectories(dirInConfigDir);
-        Files.createFile(dirInConfigDir.resolve("myconfig.yml"));
-        String pluginZip = createPlugin("fake", pluginDir);
-        UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1()));
-        assertTrue(e.getMessage(), e.getMessage().contains("Directories not allowed in config dir for plugin"));
-        assertInstallCleaned(env.v2());
-    }
-
-    public void testConfigConflict() throws Exception {
-        Tuple<Path, Environment> env = createEnv(fs, temp);
-        Path pluginDir = createPluginDir(temp);
-        Path configDir = pluginDir.resolve("config");
-        Files.createDirectory(configDir);
-        Files.createFile(configDir.resolve("myconfig.yml"));
-        String pluginZip = createPlugin("elasticsearch.yml", pluginDir);
-        FileAlreadyExistsException e = expectThrows(FileAlreadyExistsException.class, () -> installPlugin(pluginZip, env.v1()));
-        assertTrue(e.getMessage(), e.getMessage().contains(env.v2().configFile().resolve("elasticsearch.yml").toString()));
-        assertInstallCleaned(env.v2());
-    }
-
-    public void testMissingDescriptor() throws Exception {
-        Tuple<Path, Environment> env = createEnv(fs, temp);
-        Path pluginDir = createPluginDir(temp);
-        Files.createFile(pluginDir.resolve("fake.yml"));
-        String pluginZip = writeZip(pluginDir, "elasticsearch");
-        NoSuchFileException e = expectThrows(NoSuchFileException.class, () -> installPlugin(pluginZip, env.v1()));
-        assertTrue(e.getMessage(), e.getMessage().contains("plugin-descriptor.properties"));
-        assertInstallCleaned(env.v2());
-    }
-
-    public void testMissingDirectory() throws Exception {
-        Tuple<Path, Environment> env = createEnv(fs, temp);
-        Path pluginDir = createPluginDir(temp);
-        Files.createFile(pluginDir.resolve(PluginInfo.ES_PLUGIN_PROPERTIES));
-        String pluginZip = writeZip(pluginDir, null);
-        UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1()));
-        assertTrue(e.getMessage(), e.getMessage().contains("`elasticsearch` directory is missing in the plugin zip"));
-        assertInstallCleaned(env.v2());
-    }
-
-    public void testZipRelativeOutsideEntryName() throws Exception {
-        Tuple<Path, Environment> env = createEnv(fs, temp);
-        Path zip = createTempDir().resolve("broken.zip");
-        try (ZipOutputStream stream = new ZipOutputStream(Files.newOutputStream(zip))) {
-            stream.putNextEntry(new ZipEntry("elasticsearch/../blah"));
-        }
-        String pluginZip = zip.toUri().toURL().toString();
-        IOException e = expectThrows(IOException.class, () -> installPlugin(pluginZip, env.v1()));
-        assertTrue(e.getMessage(), e.getMessage().contains("resolving outside of plugin directory"));
-    }
-
-    public void testOfficialPluginsHelpSorted() throws Exception {
-        MockTerminal terminal = new MockTerminal();
-        new InstallPluginCommand().main(new String[] { "--help" }, terminal);
-        try (BufferedReader reader = new BufferedReader(new StringReader(terminal.getOutput()))) {
-            String line = reader.readLine();
-
-            // first find the beginning of our list of official plugins
-            while (line.endsWith("may be installed by name:") == false) {
-                line = reader.readLine();
-            }
-
-            // now check each line compares greater than the last, until we reach an empty line
-            String prev = reader.readLine();
-            line = reader.readLine();
-            while (line != null && line.trim().isEmpty() == false) {
-                assertTrue(prev + " < " + line, prev.compareTo(line) < 0);
-                prev = line;
-                line = reader.readLine();
-            }
-        }
-    }
-
-    public void testOfficialPluginsIncludesXpack() throws Exception {
-        MockTerminal terminal = new MockTerminal();
-        new InstallPluginCommand().main(new String[] { "--help" }, terminal);
-        assertTrue(terminal.getOutput(), terminal.getOutput().contains("x-pack"));
-    }
-
-    public void testInstallMisspelledOfficialPlugins() throws Exception {
-        Tuple<Path, Environment> env = createEnv(fs, temp);
-        UserException e = expectThrows(UserException.class, () -> installPlugin("xpack", env.v1()));
-        assertThat(e.getMessage(), containsString("Unknown plugin xpack, did you mean [x-pack]?"));
-
-        e = expectThrows(UserException.class, () -> installPlugin("analysis-smartnc", env.v1()));
-        assertThat(e.getMessage(), containsString("Unknown plugin analysis-smartnc, did you mean [analysis-smartcn]?"));
-
-        e = expectThrows(UserException.class, () -> installPlugin("repository", env.v1()));
-        assertThat(e.getMessage(), containsString("Unknown plugin repository, did you mean any of [repository-s3, repository-gcs]?"));
-
-        e = expectThrows(UserException.class, () -> installPlugin("unknown_plugin", env.v1()));
-        assertThat(e.getMessage(), containsString("Unknown plugin unknown_plugin"));
-    }
-
-    public void testBatchFlag() throws Exception {
-        MockTerminal terminal = new MockTerminal();
-        installPlugin(terminal, true);
-        assertThat(terminal.getOutput(), containsString("WARNING: plugin requires additional permissions"));
-    }
-
-    public void testQuietFlagDisabled() throws Exception {
-        MockTerminal terminal = new MockTerminal();
-        terminal.setVerbosity(randomFrom(Terminal.Verbosity.NORMAL, Terminal.Verbosity.VERBOSE));
-        installPlugin(terminal, false);
-        assertThat(terminal.getOutput(), containsString("100%"));
-    }
-
-    public void testQuietFlagEnabled() throws Exception {
-        MockTerminal terminal = new MockTerminal();
-        terminal.setVerbosity(Terminal.Verbosity.SILENT);
-        installPlugin(terminal, false);
-        assertThat(terminal.getOutput(), not(containsString("100%")));
-    }
-
-    private void installPlugin(MockTerminal terminal, boolean isBatch) throws Exception {
-        Tuple<Path, Environment> env = createEnv(fs, temp);
-        Path pluginDir = createPluginDir(temp);
-        // if batch is enabled, we also want to add a security policy
-        String pluginZip = createPlugin("fake", pluginDir, isBatch);
-
-        Map<String, String> settings = new HashMap<>();
-        settings.put("path.home", env.v1().toString());
-        new InstallPluginCommand() {
-            @Override
-            void jarHellCheck(Path candidate, Path pluginsDir) throws Exception {
-            }
-        }.execute(terminal, pluginZip, isBatch, settings);
-    }
-
-    // TODO: test checksum (need maven/official below)
-    // TODO: test maven, official, and staging install...need tests with fixtures...
-}
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/ListPluginsCommandTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/ListPluginsCommandTests.java
deleted file mode 100644
index 1422280165cc7..0000000000000
--- a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/ListPluginsCommandTests.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.plugins;
-
-
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.NoSuchFileException;
-import java.nio.file.Path;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-import org.apache.lucene.util.LuceneTestCase;
-import org.elasticsearch.cli.ExitCodes;
-import org.elasticsearch.cli.MockTerminal;
-import org.elasticsearch.common.inject.spi.HasDependencies;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.Version;
-import org.junit.Before;
-
-@LuceneTestCase.SuppressFileSystems("*")
-public class ListPluginsCommandTests extends ESTestCase {
-
-    private Path home;
-    private Environment env;
-
-    @Before
-    public void setUp() throws Exception {
-        super.setUp();
-        home = createTempDir();
-        Files.createDirectories(home.resolve("plugins"));
-        Settings settings = Settings.builder()
-                .put("path.home", home)
-                .build();
-        env = new Environment(settings);
-    }
-
-    static MockTerminal listPlugins(Path home) throws Exception {
-        return listPlugins(home, new String[0]);
-    }
-    
-    static MockTerminal listPlugins(Path home, String[] args) throws Exception {
-        String[] argsAndHome = new String[args.length + 1];
-        System.arraycopy(args, 0, argsAndHome, 0, args.length);
-        argsAndHome[args.length] = "-Epath.home=" + home;
-        MockTerminal terminal = new MockTerminal();
-        int status = new ListPluginsCommand().main(argsAndHome, terminal);
-        assertEquals(ExitCodes.OK, status);
-        return terminal;
-    }
-    
-    static String buildMultiline(String... args){
-        return Arrays.asList(args).stream().collect(Collectors.joining("\n", "", "\n"));
-    }
-    
-    static void buildFakePlugin(Environment env, String description, String name, String classname) throws IOException {
-        PluginTestUtil.writeProperties(env.pluginsFile().resolve(name),
-                "description", description,
-                "name", name,
-                "version", "1.0",
-                "elasticsearch.version", Version.CURRENT.toString(),
-                "java.version", System.getProperty("java.specification.version"),
-                "classname", classname);
-    }
-
-
-    public void testPluginsDirMissing() throws Exception {
-        Files.delete(env.pluginsFile());
-        IOException e = expectThrows(IOException.class, () -> listPlugins(home));
-        assertEquals(e.getMessage(), "Plugins directory missing: " + env.pluginsFile());
-    }
-
-    public void testNoPlugins() throws Exception {
-        MockTerminal terminal = listPlugins(home);
-        assertTrue(terminal.getOutput(), terminal.getOutput().isEmpty());
-    }
-
-    public void testOnePlugin() throws Exception {
-        buildFakePlugin(env, "fake desc", "fake", "org.fake");
-        MockTerminal terminal = listPlugins(home);
-        assertEquals(terminal.getOutput(), buildMultiline("fake"));
-    }
-
-    public void testTwoPlugins() throws Exception {
-        buildFakePlugin(env, "fake desc", "fake1", "org.fake");
-        buildFakePlugin(env, "fake desc 2", "fake2", "org.fake");
-        MockTerminal terminal = listPlugins(home);
-        assertEquals(terminal.getOutput(), buildMultiline("fake1", "fake2"));
-    }
-    
-    public void testPluginWithVerbose() throws Exception {
-        buildFakePlugin(env, "fake desc", "fake_plugin", "org.fake");
-        String[] params = { "-v" };
-        MockTerminal terminal = listPlugins(home, params);
-        assertEquals(terminal.getOutput(), buildMultiline("Plugins directory: " + env.pluginsFile(), "fake_plugin",
-                "- Plugin information:", "Name: fake_plugin", "Description: fake desc", "Version: 1.0", " * Classname: org.fake"));
-    }
-    
-    public void testPluginWithVerboseMultiplePlugins() throws Exception {
-        buildFakePlugin(env, "fake desc 1", "fake_plugin1", "org.fake");
-        buildFakePlugin(env, "fake desc 2", "fake_plugin2", "org.fake2");
-        String[] params = { "-v" };
-        MockTerminal terminal = listPlugins(home, params);
-        assertEquals(terminal.getOutput(), buildMultiline("Plugins directory: " + env.pluginsFile(),
-                "fake_plugin1", "- Plugin information:", "Name: fake_plugin1", "Description: fake desc 1", "Version: 1.0",
-                " * Classname: org.fake", "fake_plugin2", "- Plugin information:", "Name: fake_plugin2",
-                "Description: fake desc 2", "Version: 1.0", " * Classname: org.fake2"));
-    }
-        
-    public void testPluginWithoutVerboseMultiplePlugins() throws Exception {
-        buildFakePlugin(env, "fake desc 1", "fake_plugin1", "org.fake");
-        buildFakePlugin(env, "fake desc 2", "fake_plugin2", "org.fake2");
-        MockTerminal terminal = listPlugins(home, new String[0]);
-        String output = terminal.getOutput();
-        assertEquals(output, buildMultiline("fake_plugin1", "fake_plugin2"));
-    }
-    
-    public void testPluginWithoutDescriptorFile() throws Exception{
-        Files.createDirectories(env.pluginsFile().resolve("fake1"));
-        NoSuchFileException e = expectThrows(NoSuchFileException.class, () -> listPlugins(home));
-        assertEquals(e.getFile(), env.pluginsFile().resolve("fake1").resolve(PluginInfo.ES_PLUGIN_PROPERTIES).toString());
-    }
-    
-    public void testPluginWithWrongDescriptorFile() throws Exception{
-        PluginTestUtil.writeProperties(env.pluginsFile().resolve("fake1"),
-                "description", "fake desc");
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> listPlugins(home));
-        assertEquals(e.getMessage(), "Property [name] is missing in [" +
-                env.pluginsFile().resolve("fake1").resolve(PluginInfo.ES_PLUGIN_PROPERTIES).toString() + "]");
-    }
-    
-}
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/PluginSecurityTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/PluginSecurityTests.java
index 466f7d05cd1d2..77ecd12f786fb 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/PluginSecurityTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/PluginSecurityTests.java
@@ -19,60 +19,132 @@
 
 package org.elasticsearch.plugins;
 
+import org.apache.lucene.util.LuceneTestCase;
+import org.elasticsearch.Version;
+import org.elasticsearch.cli.MockTerminal;
 import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.test.ESTestCase;
 
+import java.io.IOException;
 import java.nio.file.Path;
 import java.security.Permission;
 import java.security.PermissionCollection;
 import java.security.Permissions;
 import java.util.Collections;
 import java.util.List;
+import java.util.function.Supplier;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.hasToString;
+import static org.hamcrest.Matchers.not;
 
 /** Tests plugin manager security check */
 public class PluginSecurityTests extends ESTestCase {
 
+    private final Supplier<Path> tmpFile = LuceneTestCase::createTempDir;
+
+    public void testHasNativeController() throws IOException {
+        assumeTrue(
+                "test cannot run with security manager enabled",
+                System.getSecurityManager() == null);
+        final PluginInfo info =
+                new PluginInfo("fake", "fake", Version.CURRENT.toString(), "Fake", true);
+        final MockTerminal terminal = new MockTerminal();
+        terminal.addTextInput("y");
+        terminal.addTextInput("y");
+        final Path policyFile = this.getDataPath("security/simple-plugin-security.policy");
+        PluginSecurity.readPolicy(info, policyFile, terminal, tmpFile, false);
+        final String output = terminal.getOutput();
+        assertThat(output, containsString("plugin forks a native controller"));
+    }
+
+    public void testDeclineNativeController() throws IOException {
+        assumeTrue(
+                "test cannot run with security manager enabled",
+                System.getSecurityManager() == null);
+        final PluginInfo info =
+                new PluginInfo("fake", "fake", Version.CURRENT.toString(), "Fake", true);
+        final MockTerminal terminal = new MockTerminal();
+        terminal.addTextInput("y");
+        terminal.addTextInput("n");
+        final Path policyFile = this.getDataPath("security/simple-plugin-security.policy");
+        RuntimeException e = expectThrows(
+                RuntimeException.class,
+                () -> PluginSecurity.readPolicy(info, policyFile, terminal, tmpFile, false));
+        assertThat(e, hasToString(containsString("installation aborted by user")));
+    }
+
+    public void testDoesNotHaveNativeController() throws IOException {
+        assumeTrue(
+                "test cannot run with security manager enabled",
+                System.getSecurityManager() == null);
+        final PluginInfo info =
+                new PluginInfo("fake", "fake", Version.CURRENT.toString(), "Fake", false);
+        final MockTerminal terminal = new MockTerminal();
+        terminal.addTextInput("y");
+        final Path policyFile = this.getDataPath("security/simple-plugin-security.policy");
+        PluginSecurity.readPolicy(info, policyFile, terminal, tmpFile, false);
+        final String output = terminal.getOutput();
+        assertThat(output, not(containsString("plugin forks a native controller")));
+    }
+
     /** Test that we can parse the set of permissions correctly for a simple policy */
     public void testParsePermissions() throws Exception {
-        assumeTrue("test cannot run with security manager enabled", System.getSecurityManager() == null);
+        assumeTrue(
+                "test cannot run with security manager enabled",
+                System.getSecurityManager() == null);
         Path scratch = createTempDir();
         Path testFile = this.getDataPath("security/simple-plugin-security.policy");
         Permissions expected = new Permissions();
         expected.add(new RuntimePermission("queuePrintJob"));
-        PermissionCollection actual = PluginSecurity.parsePermissions(Terminal.DEFAULT, testFile, scratch);
+        PermissionCollection actual =
+                PluginSecurity.parsePermissions(Terminal.DEFAULT, testFile, scratch);
         assertEquals(expected, actual);
     }
 
     /** Test that we can parse the set of permissions correctly for a complex policy */
     public void testParseTwoPermissions() throws Exception {
-        assumeTrue("test cannot run with security manager enabled", System.getSecurityManager() == null);
+        assumeTrue(
+                "test cannot run with security manager enabled",
+                System.getSecurityManager() == null);
         Path scratch = createTempDir();
         Path testFile = this.getDataPath("security/complex-plugin-security.policy");
         Permissions expected = new Permissions();
         expected.add(new RuntimePermission("getClassLoader"));
         expected.add(new RuntimePermission("closeClassLoader"));
-        PermissionCollection actual = PluginSecurity.parsePermissions(Terminal.DEFAULT, testFile, scratch);
+        PermissionCollection actual =
+                PluginSecurity.parsePermissions(Terminal.DEFAULT, testFile, scratch);
         assertEquals(expected, actual);
     }
 
     /** Test that we can format some simple permissions properly */
     public void testFormatSimplePermission() throws Exception {
-        assertEquals("java.lang.RuntimePermission queuePrintJob", PluginSecurity.formatPermission(new RuntimePermission("queuePrintJob")));
+        assertEquals(
+                "java.lang.RuntimePermission queuePrintJob",
+                PluginSecurity.formatPermission(new RuntimePermission("queuePrintJob")));
     }
 
     /** Test that we can format an unresolved permission properly */
     public void testFormatUnresolvedPermission() throws Exception {
-        assumeTrue("test cannot run with security manager enabled", System.getSecurityManager() == null);
+        assumeTrue(
+                "test cannot run with security manager enabled",
+                System.getSecurityManager() == null);
         Path scratch = createTempDir();
         Path testFile = this.getDataPath("security/unresolved-plugin-security.policy");
-        PermissionCollection actual = PluginSecurity.parsePermissions(Terminal.DEFAULT, testFile, scratch);
+        PermissionCollection actual =
+                PluginSecurity.parsePermissions(Terminal.DEFAULT, testFile, scratch);
         List<Permission> permissions = Collections.list(actual.elements());
         assertEquals(1, permissions.size());
-        assertEquals("org.fake.FakePermission fakeName", PluginSecurity.formatPermission(permissions.get(0)));
+        assertEquals(
+                "org.fake.FakePermission fakeName",
+                PluginSecurity.formatPermission(permissions.get(0)));
     }
 
     /** no guaranteed equals on these classes, we assert they contain the same set */
     private void assertEquals(PermissionCollection expected, PermissionCollection actual) {
-        assertEquals(asSet(Collections.list(expected.elements())), asSet(Collections.list(actual.elements())));
+        assertEquals(
+                asSet(Collections.list(expected.elements())),
+                asSet(Collections.list(actual.elements())));
     }
+
 }
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java
deleted file mode 100644
index ab4f00492b0bc..0000000000000
--- a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.plugins;
-
-import org.apache.lucene.util.LuceneTestCase;
-import org.elasticsearch.cli.MockTerminal;
-import org.elasticsearch.cli.UserException;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.test.ESTestCase;
-import org.junit.Before;
-
-import java.io.IOException;
-import java.nio.file.DirectoryStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.hamcrest.CoreMatchers.containsString;
-import static org.hamcrest.CoreMatchers.not;
-
-@LuceneTestCase.SuppressFileSystems("*")
-public class RemovePluginCommandTests extends ESTestCase {
-
-    private Path home;
-    private Environment env;
-
-    @Before
-    public void setUp() throws Exception {
-        super.setUp();
-        home = createTempDir();
-        Files.createDirectories(home.resolve("bin"));
-        Files.createFile(home.resolve("bin").resolve("elasticsearch"));
-        Files.createDirectories(home.resolve("plugins"));
-        Settings settings = Settings.builder()
-                .put("path.home", home)
-                .build();
-        env = new Environment(settings);
-    }
-
-    static MockTerminal removePlugin(String name, Path home) throws Exception {
-        Map<String, String> settings = new HashMap<>();
-        settings.put("path.home", home.toString());
-        MockTerminal terminal = new MockTerminal();
-        new RemovePluginCommand().execute(terminal, name, settings);
-        return terminal;
-    }
-
-    static void assertRemoveCleaned(Environment env) throws IOException {
-        try (DirectoryStream<Path> stream = Files.newDirectoryStream(env.pluginsFile())) {
-            for (Path file : stream) {
-                if (file.getFileName().toString().startsWith(".removing")) {
-                    fail("Removal dir still exists, " + file);
-                }
-            }
-        }
-    }
-
-    public void testMissing() throws Exception {
-        UserException e = expectThrows(UserException.class, () -> removePlugin("dne", home));
-        assertTrue(e.getMessage(), e.getMessage().contains("plugin dne not found"));
-        assertRemoveCleaned(env);
-    }
-
-    public void testBasic() throws Exception {
-        Files.createDirectory(env.pluginsFile().resolve("fake"));
-        Files.createFile(env.pluginsFile().resolve("fake").resolve("plugin.jar"));
-        Files.createDirectory(env.pluginsFile().resolve("fake").resolve("subdir"));
-        Files.createDirectory(env.pluginsFile().resolve("other"));
-        removePlugin("fake", home);
-        assertFalse(Files.exists(env.pluginsFile().resolve("fake")));
-        assertTrue(Files.exists(env.pluginsFile().resolve("other")));
-        assertRemoveCleaned(env);
-    }
-
-    public void testBin() throws Exception {
-        Files.createDirectories(env.pluginsFile().resolve("fake"));
-        Path binDir = env.binFile().resolve("fake");
-        Files.createDirectories(binDir);
-        Files.createFile(binDir.resolve("somescript"));
-        removePlugin("fake", home);
-        assertFalse(Files.exists(env.pluginsFile().resolve("fake")));
-        assertTrue(Files.exists(env.binFile().resolve("elasticsearch")));
-        assertFalse(Files.exists(binDir));
-        assertRemoveCleaned(env);
-    }
-
-    public void testBinNotDir() throws Exception {
-        Files.createDirectories(env.pluginsFile().resolve("elasticsearch"));
-        UserException e = expectThrows(UserException.class, () -> removePlugin("elasticsearch", home));
-        assertTrue(e.getMessage(), e.getMessage().contains("not a directory"));
-        assertTrue(Files.exists(env.pluginsFile().resolve("elasticsearch"))); // did not remove
-        assertTrue(Files.exists(env.binFile().resolve("elasticsearch")));
-        assertRemoveCleaned(env);
-    }
-
-    public void testConfigDirPreserved() throws Exception {
-        Files.createDirectories(env.pluginsFile().resolve("fake"));
-        final Path configDir = env.configFile().resolve("fake");
-        Files.createDirectories(configDir);
-        Files.createFile(configDir.resolve("fake.yml"));
-        final MockTerminal terminal = removePlugin("fake", home);
-        assertTrue(Files.exists(env.configFile().resolve("fake")));
-        assertThat(terminal.getOutput(), containsString(expectedConfigDirPreservedMessage(configDir)));
-        assertRemoveCleaned(env);
-    }
-
-    public void testNoConfigDirPreserved() throws Exception {
-        Files.createDirectories(env.pluginsFile().resolve("fake"));
-        final Path configDir = env.configFile().resolve("fake");
-        final MockTerminal terminal = removePlugin("fake", home);
-        assertThat(terminal.getOutput(), not(containsString(expectedConfigDirPreservedMessage(configDir))));
-    }
-
-    private String expectedConfigDirPreservedMessage(final Path configDir) {
-        return "-> Preserving plugin config files [" + configDir + "] in case of upgrade, delete manually if not needed";
-    }
-
-}
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/tribe/TribeUnitTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/tribe/TribeUnitTests.java
index 247caa42210c2..643691ed437e4 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/tribe/TribeUnitTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/tribe/TribeUnitTests.java
@@ -19,6 +19,12 @@
 
 package org.elasticsearch.tribe;
 
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterState;
@@ -26,19 +32,20 @@
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.discovery.DiscoveryModule;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.node.MockNode;
 import org.elasticsearch.node.Node;
+import org.elasticsearch.node.NodeValidationException;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.InternalTestCluster;
+import org.elasticsearch.test.discovery.MockZenPing;
+import org.elasticsearch.test.discovery.TestZenDiscovery;
+import org.elasticsearch.transport.MockTcpTransportPlugin;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
-import java.io.IOException;
-import java.nio.file.Path;
-
 import static org.hamcrest.CoreMatchers.either;
 import static org.hamcrest.CoreMatchers.equalTo;
 
@@ -55,29 +62,29 @@ public class TribeUnitTests extends ESTestCase {
 
 
     @BeforeClass
-    public static void createTribes() {
+    public static void createTribes() throws NodeValidationException {
         Settings baseSettings = Settings.builder()
             .put(NetworkModule.HTTP_ENABLED.getKey(), false)
             .put("transport.type", "local")
-            .put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), "local")
             .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
             .put(NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.getKey(), 2)
             .build();
 
-        tribe1 = new TribeClientNode(
+        final List<Class<? extends Plugin>> mockPlugins = Arrays.asList(TestZenDiscovery.TestPlugin.class);
+        tribe1 = new MockNode(
             Settings.builder()
                 .put(baseSettings)
                 .put("cluster.name", "tribe1")
                 .put("node.name", "tribe1_node")
                     .put(NodeEnvironment.NODE_ID_SEED_SETTING.getKey(), random().nextLong())
-                .build()).start();
-        tribe2 = new TribeClientNode(
+                .build(), mockPlugins).start();
+        tribe2 = new MockNode(
             Settings.builder()
                 .put(baseSettings)
                 .put("cluster.name", "tribe2")
                 .put("node.name", "tribe2_node")
                     .put(NodeEnvironment.NODE_ID_SEED_SETTING.getKey(), random().nextLong())
-                .build()).start();
+                .build(), mockPlugins).start();
     }
 
     @AfterClass
@@ -94,6 +101,7 @@ public void testThatTribeClientsIgnoreGlobalConfig() throws Exception {
             .put(Environment.PATH_CONF_SETTING.getKey(), pathConf)
             .build();
         assertTribeNodeSuccessfullyCreated(settings);
+        assertWarnings("tribe nodes are deprecated in favor of cross-cluster search and will be removed in Elasticsearch 7.0.0");
     }
 
     private static void assertTribeNodeSuccessfullyCreated(Settings extraSettings) throws Exception {
@@ -101,11 +109,10 @@ private static void assertTribeNodeSuccessfullyCreated(Settings extraSettings) t
         Settings settings = Settings.builder().put(NetworkModule.HTTP_ENABLED.getKey(), false).put("node.name", "tribe_node")
                 .put("transport.type", "local").put("discovery.type", "local")
                 .put("tribe.t1.transport.type", "local").put("tribe.t2.transport.type", "local")
-                .put("tribe.t1.discovery.type", "local").put("tribe.t2.discovery.type", "local")
                 .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
                 .put(extraSettings).build();
 
-        try (Node node = new Node(settings).start()) {
+        try (Node node = new MockNode(settings, Collections.singletonList(TestZenDiscovery.TestPlugin.class)).start()) {
             try (Client client = node.client()) {
                 assertBusy(() -> {
                     ClusterState state = client.admin().cluster().prepareState().clear().setNodes(true).get().getState();
diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/config/log4j2.properties b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/config/log4j2.properties
index 42fd3f3535954..8b956421458d0 100644
--- a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/config/log4j2.properties
+++ b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/config/log4j2.properties
@@ -1,15 +1,13 @@
-status = error
-
 appender.console.type = Console
 appender.console.name = console
 appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %m%n
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%m%n
 
 appender.file.type = File
 appender.file.name = file
-appender.file.fileName = ${sys:es.logs}.log
+appender.file.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log
 appender.file.layout.type = PatternLayout
-appender.file.layout.pattern = [%p][%l] %m%n
+appender.file.layout.pattern = [%p][%l] %marker%m%n
 
 rootLogger.level = info
 rootLogger.appenderRef.console.ref = console
@@ -23,11 +21,12 @@ logger.test.additivity = false
 
 appender.deprecation_file.type = File
 appender.deprecation_file.name = deprecation_file
-appender.deprecation_file.fileName = ${sys:es.logs}_deprecation.log
+appender.deprecation_file.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.log
 appender.deprecation_file.layout.type = PatternLayout
-appender.deprecation_file.layout.pattern = [%p][%l] %m%n
+appender.deprecation_file.layout.pattern = [%p][%l] %marker%m%n
 
 logger.deprecation.name = deprecation
 logger.deprecation.level = warn
 logger.deprecation.appenderRef.deprecation_file.ref = deprecation_file
 logger.deprecation.additivity = false
+
diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/config/second/log4j2.properties b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/config/second/log4j2.properties
index 2ade4c896c318..9d59e79c08aa7 100644
--- a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/config/second/log4j2.properties
+++ b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/config/second/log4j2.properties
@@ -1,7 +1,7 @@
 appender.console2.type = Console
 appender.console2.name = console2
 appender.console2.layout.type = PatternLayout
-appender.console2.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %m%n
+appender.console2.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marke%m%n
 
 logger.second.name = second
 logger.second.level = debug
diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/config/third/log4j2.properties b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/config/third/log4j2.properties
index 8699f574d5370..ed794cb7c3b59 100644
--- a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/config/third/log4j2.properties
+++ b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/config/third/log4j2.properties
@@ -1,7 +1,7 @@
 appender.console3.type = Console
 appender.console3.name = console3
 appender.console3.layout.type = PatternLayout
-appender.console3.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %m%n
+appender.console3.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%m%n
 
 logger.third.name = third
 logger.third.level = debug
diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/deprecation/log4j2.properties b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/deprecation/log4j2.properties
new file mode 100644
index 0000000000000..fd7af2ce73136
--- /dev/null
+++ b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/deprecation/log4j2.properties
@@ -0,0 +1,25 @@
+appender.console.type = Console
+appender.console.name = console
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%m%n
+
+appender.file.type = File
+appender.file.name = file
+appender.file.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log
+appender.file.layout.type = PatternLayout
+appender.file.layout.pattern = [%p][%l] %marker%m%n
+
+rootLogger.level = info
+rootLogger.appenderRef.console.ref = console
+rootLogger.appenderRef.file.ref = file
+
+appender.deprecation_file.type = File
+appender.deprecation_file.name = deprecation_file
+appender.deprecation_file.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.log
+appender.deprecation_file.layout.type = PatternLayout
+appender.deprecation_file.layout.pattern = [%p][%l] %marker%m%n
+
+logger.deprecation.name = deprecation
+logger.deprecation.level = warn
+logger.deprecation.appenderRef.deprecation_file.ref = deprecation_file
+logger.deprecation.additivity = false
diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/does_not_exist/nothing_to_see_here b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/does_not_exist/nothing_to_see_here
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/find_appender/log4j2.properties b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/find_appender/log4j2.properties
new file mode 100644
index 0000000000000..8553ec5e7915f
--- /dev/null
+++ b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/find_appender/log4j2.properties
@@ -0,0 +1,15 @@
+appender.console.type = Console
+appender.console.name = console
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%m%n
+
+logger.has_console_appender.name = has_console_appender
+logger.has_console_appender.level = trace
+logger.has_console_appender.appenderRef.console.ref = console
+logger.has_console_appender.additivity = false
+
+appender.counting_no_op.type = CountingNoOp
+appender.counting_no_op.name = counting_no_op
+
+logger.has_counting_no_op_appender.name = has_counting_no_op_appender
+logger.has_counting_no_op_appender.appenderRef.counting_no_op.ref = counting_no_op
diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/hierarchy/log4j2.properties b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/hierarchy/log4j2.properties
new file mode 100644
index 0000000000000..fbd35a4c684fc
--- /dev/null
+++ b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/hierarchy/log4j2.properties
@@ -0,0 +1,17 @@
+appender.console.type = Console
+appender.console.name = console
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %m%n
+
+rootLogger.level = info
+rootLogger.appenderRef.console.ref = console
+
+logger.x.name = x
+logger.x.level = trace
+logger.x.appenderRef.console.ref = console
+logger.x.additivity = false
+
+logger.x_y.name = x.y
+logger.x_y.level = debug
+logger.x_y.appenderRef.console.ref = console
+logger.x_y.additivity = false
diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/location_info/log4j2.properties b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/location_info/log4j2.properties
new file mode 100644
index 0000000000000..5fa85b5d156b3
--- /dev/null
+++ b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/location_info/log4j2.properties
@@ -0,0 +1,20 @@
+appender.console.type = Console
+appender.console.name = console
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%m%n
+
+appender.file.type = File
+appender.file.name = file
+appender.file.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log
+appender.file.layout.type = PatternLayout
+appender.file.layout.pattern = [%p][%l] %marker%m%n
+
+rootLogger.level = info
+rootLogger.appenderRef.console.ref = console
+rootLogger.appenderRef.file.ref = file
+
+logger.test.name = test
+logger.test.level = trace
+logger.test.appenderRef.console.ref = console
+logger.test.appenderRef.file.ref = file
+logger.test.additivity = false
diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/minimal/log4j2.properties b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/minimal/log4j2.properties
new file mode 100644
index 0000000000000..f245dde979cbd
--- /dev/null
+++ b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/minimal/log4j2.properties
@@ -0,0 +1,7 @@
+appender.console.type = Console
+appender.console.name = console
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %m%n
+
+rootLogger.level = info
+rootLogger.appenderRef.console.ref = console
diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/prefix/log4j2.properties b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/prefix/log4j2.properties
new file mode 100644
index 0000000000000..5dfa369a3bd25
--- /dev/null
+++ b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/prefix/log4j2.properties
@@ -0,0 +1,20 @@
+appender.console.type = Console
+appender.console.name = console
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%m%n
+
+appender.file.type = File
+appender.file.name = file
+appender.file.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log
+appender.file.layout.type = PatternLayout
+appender.file.layout.pattern = %marker%m%n
+
+rootLogger.level = info
+rootLogger.appenderRef.console.ref = console
+rootLogger.appenderRef.file.ref = file
+
+logger.prefix.name = prefix
+logger.prefix.level = info
+logger.prefix.appenderRef.console.ref = console
+logger.prefix.appenderRef.file.ref = file
+logger.prefix.additivity = false
diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/settings/log4j2.properties b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/settings/log4j2.properties
new file mode 100644
index 0000000000000..2c5b8f8f5ab90
--- /dev/null
+++ b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/settings/log4j2.properties
@@ -0,0 +1,25 @@
+appender.console.type = Console
+appender.console.name = console
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%m%n
+
+appender.file.type = File
+appender.file.name = file
+appender.file.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log
+appender.file.layout.type = PatternLayout
+appender.file.layout.pattern = [%p][%l] %marker%m%n
+
+rootLogger.level = info
+rootLogger.appenderRef.console.ref = console
+rootLogger.appenderRef.file.ref = file
+
+appender.deprecation_file.type = File
+appender.deprecation_file.name = deprecation_file
+appender.deprecation_file.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.log
+appender.deprecation_file.layout.type = PatternLayout
+appender.deprecation_file.layout.pattern = [%p][%l] %marker%m%n
+
+logger.deprecation.name = org.elasticsearch.deprecation.common.settings
+logger.deprecation.level = warn
+logger.deprecation.appenderRef.deprecation_file.ref = deprecation_file
+logger.deprecation.additivity = false
diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/unsupported/log4j2.properties b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/unsupported/log4j2.properties
new file mode 100644
index 0000000000000..b424ce76c188f
--- /dev/null
+++ b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/unsupported/log4j2.properties
@@ -0,0 +1,16 @@
+status = error
+
+appender.console.type = Console
+appender.console.name = console
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %m%n
+
+appender.file.type = File
+appender.file.name = file
+appender.file.fileName = ${sys:es.logs}.log
+appender.file.layout.type = PatternLayout
+appender.file.layout.pattern = [%p][%l] %m%n
+
+rootLogger.level = info
+rootLogger.appenderRef.console.ref = console
+rootLogger.appenderRef.file.ref = file
diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/config/logging.yml b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/unsupported/logging.yml
similarity index 100%
rename from qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/config/logging.yml
rename to qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/unsupported/logging.yml
diff --git a/qa/mixed-cluster/build.gradle b/qa/mixed-cluster/build.gradle
new file mode 100644
index 0000000000000..66185325931d8
--- /dev/null
+++ b/qa/mixed-cluster/build.gradle
@@ -0,0 +1,68 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.elasticsearch.gradle.test.RestIntegTestTask
+import org.elasticsearch.gradle.Version
+
+apply plugin: 'elasticsearch.standalone-test'
+
+// This is a top level task which we will add dependencies to below.
+// It is a single task that can be used to backcompat tests against all versions.
+task bwcTest {
+  description = 'Runs backwards compatibility tests.'
+  group = 'verification'
+}
+
+for (Version version : wireCompatVersions) {
+  String baseName = "v${version}"
+
+  Task mixedClusterTest = tasks.create(name: "${baseName}#mixedClusterTest", type: RestIntegTestTask) {
+    mustRunAfter(precommit)
+    includePackaged = true
+  }
+
+  /* This project runs the core REST tests against a 2 node cluster where one of
+     the nodes has a different minor.  */
+  Object extension = extensions.findByName("${baseName}#mixedClusterTestCluster")
+  configure(extensions.findByName("${baseName}#mixedClusterTestCluster")) {
+    distribution = 'zip'
+    numNodes = 4
+    numBwcNodes = 2
+    bwcVersion = project.wireCompatVersions[-1]
+  }
+
+  Task versionBwcTest = tasks.create(name: "${baseName}#bwcTest") {
+    dependsOn = [mixedClusterTest]
+  }
+
+  if (project.bwc_tests_enabled) {
+    bwcTest.dependsOn(versionBwcTest)
+  }
+}
+
+test.enabled = false // no unit tests for rolling upgrades, only the rest integration test
+
+// basic integ tests includes testing bwc against the most recent version
+task integTest {
+  if (project.bwc_tests_enabled) {
+    dependsOn = ["v${wireCompatVersions[-1]}#bwcTest"]
+  }
+}
+
+check.dependsOn(integTest)
diff --git a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/MixedClusterClientYamlTestSuiteIT.java b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/MixedClusterClientYamlTestSuiteIT.java
new file mode 100644
index 0000000000000..59f350ea5ca28
--- /dev/null
+++ b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/MixedClusterClientYamlTestSuiteIT.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.backwards;
+
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
+import org.apache.lucene.util.TimeUnits;
+import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
+import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+
+@TimeoutSuite(millis = 40 * TimeUnits.MINUTE) // some of the windows test VMs are slow as hell
+public class MixedClusterClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
+
+    public MixedClusterClientYamlTestSuiteIT(ClientYamlTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return createParameters();
+    }
+
+    @Override
+    protected boolean randomizeContentType() {
+        return false;
+    }
+}
+
diff --git a/qa/multi-cluster-search/build.gradle b/qa/multi-cluster-search/build.gradle
new file mode 100644
index 0000000000000..024e306d817d0
--- /dev/null
+++ b/qa/multi-cluster-search/build.gradle
@@ -0,0 +1,60 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.elasticsearch.gradle.test.RestIntegTestTask
+
+apply plugin: 'elasticsearch.standalone-test'
+
+task remoteClusterTest(type: RestIntegTestTask) {
+  mustRunAfter(precommit)
+}
+
+remoteClusterTestCluster {
+  distribution = 'zip'
+  numNodes = 2
+  clusterName = 'remote-cluster'
+  setting 'search.remote.connect', false
+}
+
+remoteClusterTestRunner {
+  systemProperty 'tests.rest.suite', 'remote_cluster'
+}
+
+task mixedClusterTest(type: RestIntegTestTask) {}
+
+mixedClusterTestCluster {
+  dependsOn remoteClusterTestRunner
+  distribution = 'zip'
+  setting 'search.remote.my_remote_cluster.seeds', "\"${-> remoteClusterTest.nodes.get(0).transportUri()}\""
+  setting 'search.remote.connections_per_cluster', 1
+  setting 'search.remote.connect', true
+}
+
+mixedClusterTestRunner {
+  systemProperty 'tests.rest.suite', 'multi_cluster'
+  finalizedBy 'remoteClusterTestCluster#node0.stop','remoteClusterTestCluster#node1.stop'
+}
+
+task integTest {
+  dependsOn = [mixedClusterTest]
+}
+
+test.enabled = false // no unit tests for multi-cluster-search, only the rest integration test
+
+check.dependsOn(integTest)
diff --git a/qa/multi-cluster-search/src/test/java/org/elasticsearch/upgrades/MultiClusterSearchYamlTestSuiteIT.java b/qa/multi-cluster-search/src/test/java/org/elasticsearch/upgrades/MultiClusterSearchYamlTestSuiteIT.java
new file mode 100644
index 0000000000000..fe3a909883181
--- /dev/null
+++ b/qa/multi-cluster-search/src/test/java/org/elasticsearch/upgrades/MultiClusterSearchYamlTestSuiteIT.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.upgrades;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
+import org.apache.lucene.util.TimeUnits;
+import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
+import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+
+@TimeoutSuite(millis = 5 * TimeUnits.MINUTE) // to account for slow as hell VMs
+public class MultiClusterSearchYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
+
+    @Override
+    protected boolean preserveIndicesUponCompletion() {
+        return true;
+    }
+
+    public MultiClusterSearchYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return createParameters();
+    }
+
+}
diff --git a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/10_basic.yaml b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/10_basic.yaml
new file mode 100644
index 0000000000000..7726a1df0b10d
--- /dev/null
+++ b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/10_basic.yaml
@@ -0,0 +1,178 @@
+---
+"Index data and search on the mixed cluster":
+
+  - do:
+      indices.create:
+        index: test_index
+        body:
+          settings:
+            index:
+              number_of_shards: 2
+              number_of_replicas: 0
+
+  - do:
+      bulk:
+        refresh: true
+        body:
+            - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+            - '{"f1": "local_cluster", "filter_field": 0}'
+            - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+            - '{"f1": "local_cluster", "filter_field": 1}'
+            - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+            - '{"f1": "local_cluster", "filter_field": 0}'
+            - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+            - '{"f1": "local_cluster", "filter_field": 1}'
+            - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+            - '{"f1": "local_cluster", "filter_field": 0}'
+
+  - do:
+      search:
+        index: test_index,my_remote_cluster:test_index
+        body:
+          aggs:
+            cluster:
+              terms:
+                field: f1.keyword
+
+  - match: { _shards.total: 5 }
+  - match: { hits.total: 11 }
+  - length: { aggregations.cluster.buckets: 2 }
+  - match: { aggregations.cluster.buckets.0.key: "remote_cluster" }
+  - match: { aggregations.cluster.buckets.0.doc_count: 6 }
+  - match: { aggregations.cluster.buckets.1.key: "local_cluster" }
+  - match: { aggregations.cluster.buckets.1.doc_count: 5 }
+
+  - do:
+      search:
+        index: test_index,my_remote_cluster:test_index
+        body:
+          query:
+            term:
+              f1: remote_cluster
+          aggs:
+            cluster:
+              terms:
+                field: f1.keyword
+
+  - match: { _shards.total: 5 }
+  - match: { hits.total: 6}
+  - match: { hits.hits.0._index: "my_remote_cluster:test_index"}
+  - length: { aggregations.cluster.buckets: 1 }
+  - match: { aggregations.cluster.buckets.0.key: "remote_cluster" }
+  - match: { aggregations.cluster.buckets.0.doc_count: 6 }
+
+  - do:
+      search:
+        index: my_remote_cluster:test_index
+        body:
+          aggs:
+            cluster:
+              terms:
+                field: f1.keyword
+
+  - match: { _shards.total: 3 }
+  - match: { hits.total: 6}
+  - match: { hits.hits.0._index: "my_remote_cluster:test_index"}
+  - length: { aggregations.cluster.buckets: 1 }
+  - match: { aggregations.cluster.buckets.0.key: "remote_cluster" }
+  - match: { aggregations.cluster.buckets.0.doc_count: 6 }
+
+  - do:
+      search:
+        index: test_index
+        body:
+          aggs:
+            cluster:
+              terms:
+                field: f1.keyword
+
+  - match: { _shards.total: 2 }
+  - match: { hits.total: 5}
+  - match: { hits.hits.0._index: "test_index"}
+  - length: { aggregations.cluster.buckets: 1 }
+  - match: { aggregations.cluster.buckets.0.key: "local_cluster" }
+  - match: { aggregations.cluster.buckets.0.doc_count: 5 }
+
+---
+"Add transient remote cluster based on the preset cluster":
+  - do:
+      cluster.get_settings:
+        include_defaults: true
+
+  - set: { defaults.search.remote.my_remote_cluster.seeds.0: remote_ip }
+
+  - do:
+      cluster.put_settings:
+        flat_settings: true
+        body:
+          transient:
+            search.remote.test_remote_cluster.seeds: $remote_ip
+
+  - match: {transient: {search.remote.test_remote_cluster.seeds: $remote_ip}}
+
+  - do:
+      search:
+        index: test_remote_cluster:test_index
+
+  - match: { _shards.total: 3 }
+  - match: { hits.total: 6 }
+  - match: { hits.hits.0._index: "test_remote_cluster:test_index" }
+
+---
+"Test wildcard search":
+  - do:
+      cluster.get_settings:
+        include_defaults: true
+
+  - set: { defaults.search.remote.my_remote_cluster.seeds.0: remote_ip }
+
+  - do:
+      cluster.put_settings:
+        flat_settings: true
+        body:
+          transient:
+            search.remote.test_remote_cluster.seeds: $remote_ip
+
+  - match: {transient: {search.remote.test_remote_cluster.seeds: $remote_ip}}
+
+  - do:
+      search:
+        index: "*:test_index"
+
+  - match: { _shards.total: 6 }
+  - match: { hits.total: 12 }
+
+---
+"Search an filtered alias on the remote cluster":
+
+  - do:
+      search:
+        index: my_remote_cluster:aliased_test_index
+
+  - match: { _shards.total: 3 }
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0._source.filter_field: 1 }
+  - match: { hits.hits.0._index: "my_remote_cluster:test_index" }
+
+---
+"Search an filtered alias and empty index on the remote cluster":
+
+  - do:
+      search:
+        index: my_remote_cluster:aliased_test_index,my_remote_cluster:field_caps_index_1
+
+  - match: { _shards.total: 8 }
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0._source.filter_field: 1 }
+  - match: { hits.hits.0._index: "my_remote_cluster:test_index" }
+
+---
+"Single shard search gets properly proxied":
+
+  - do:
+      search:
+        index: "my_remote_cluster:single_doc_index"
+
+  - match: { _shards.total: 1 }
+  - match: { hits.total: 1 }
+  - match: { hits.hits.0._index: "my_remote_cluster:single_doc_index"}
diff --git a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/20_info.yaml b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/20_info.yaml
new file mode 100644
index 0000000000000..7843e30561ac4
--- /dev/null
+++ b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/20_info.yaml
@@ -0,0 +1,58 @@
+---
+"Fetch remote cluster info for existing cluster":
+
+  - do:
+      remote.info: {}
+  - match: { my_remote_cluster.connected: true }
+  - match: { my_remote_cluster.num_nodes_connected: 1}
+  - match: { my_remote_cluster.max_connections_per_cluster: 1}
+  - match: { my_remote_cluster.initial_connect_timeout: "30s" }
+  - is_true: my_remote_cluster.http_addresses.0
+
+---
+"Add transient remote cluster based on the preset cluster and check remote info":
+  - do:
+      cluster.get_settings:
+        include_defaults: true
+
+  - set: { defaults.search.remote.my_remote_cluster.seeds.0: remote_ip }
+
+  - do:
+      cluster.put_settings:
+        flat_settings: true
+        body:
+          transient:
+            search.remote.test_remote_cluster.seeds: $remote_ip
+
+  - match: {transient: {search.remote.test_remote_cluster.seeds: $remote_ip}}
+
+  # we do another search here since this will enforce the connection to be established
+  # otherwise the cluster might not have been connected yet.
+  - do:
+      search:
+        index: test_remote_cluster:test_index
+
+  - match: { _shards.total: 3 }
+  - match: { hits.total: 6 }
+  - match: { hits.hits.0._index: "test_remote_cluster:test_index" }
+
+  - do:
+      remote.info: {}
+  - set: { my_remote_cluster.http_addresses.0: remote_http }
+  - match: { test_remote_cluster.http_addresses.0: $remote_http }
+
+  - match: { test_remote_cluster.connected: true }
+  - match: { my_remote_cluster.connected: true }
+
+  - match: { test_remote_cluster.seeds.0: $remote_ip }
+  - match: { my_remote_cluster.seeds.0: $remote_ip }
+
+  - match: { my_remote_cluster.num_nodes_connected: 1}
+  - match: { test_remote_cluster.num_nodes_connected: 1}
+
+  - match: { my_remote_cluster.max_connections_per_cluster: 1}
+  - match: { test_remote_cluster.max_connections_per_cluster: 1}
+
+  - match: { my_remote_cluster.initial_connect_timeout: "30s" }
+  - match: { test_remote_cluster.initial_connect_timeout: "30s" }
+
diff --git a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/30_field_caps.yaml b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/30_field_caps.yaml
new file mode 100644
index 0000000000000..e4463eb17a020
--- /dev/null
+++ b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/30_field_caps.yaml
@@ -0,0 +1,79 @@
+---
+"Get simple field caps from remote cluster":
+  - skip:
+      version: " - 5.4.99"
+      reason:  this uses a new API functionality that has been added in 5.5.0
+
+  - do:
+        indices.create:
+          index: field_caps_index_2
+          body:
+              mappings:
+                t:
+                  properties:
+                    text:
+                      type:     text
+                    keyword:
+                      type:     keyword
+                    number:
+                      type:     double
+                    geo:
+                      type:     geo_point
+                    object:
+                      type: object
+                      properties:
+                        nested1 :
+                          type : text
+                          index: true
+                        nested2:
+                          type: float
+                          doc_values: true
+
+  - do:
+      field_caps:
+        index: 'field_caps_index_2,my_remote_cluster:field_*'
+        fields: [text, keyword, number, geo]
+
+  - match: {fields.text.text.searchable:                true}
+  - match: {fields.text.text.aggregatable:              false}
+  - is_false: fields.text.text.indices
+  - is_false: fields.text.text.non_searchable_indices
+  - is_false: fields.text.text.non_aggregatable_indices
+  - match: {fields.keyword.keyword.searchable:          true}
+  - match: {fields.keyword.keyword.aggregatable:        true}
+  - is_false: fields.text.keyword.indices
+  - is_false: fields.text.keyword.non_searchable_indices
+  - is_false: fields.text.keyword.non_aggregatable_indices
+  - match: {fields.number.double.searchable:            true}
+  - match: {fields.number.double.aggregatable:          true}
+  - match: {fields.number.double.indices:               ["field_caps_index_2", "my_remote_cluster:field_caps_index_1"]}
+  - is_false: fields.number.double.non_searchable_indices
+  - is_false: fields.number.double.non_aggregatable_indices
+  - match: {fields.number.long.searchable:              true}
+  - match: {fields.number.long.aggregatable:            true}
+  - match: {fields.number.long.indices:                 ["my_remote_cluster:field_caps_index_3"]}
+  - is_false: fields.number.long.non_searchable_indices
+  - is_false: fields.number.long.non_aggregatable_indices
+  - match: {fields.geo.geo_point.searchable:            true}
+  - match: {fields.geo.geo_point.aggregatable:          true}
+  - match: {fields.geo.geo_point.indices:               ["field_caps_index_2", "my_remote_cluster:field_caps_index_1"]}
+  - is_false: fields.geo.geo_point.non_searchable_indices
+  - is_false: fields.geo.geo_point.non_aggregatable_indices
+  - match: {fields.geo.keyword.searchable:              true}
+  - match: {fields.geo.keyword.aggregatable:            true}
+  - match: {fields.geo.keyword.indices:                 ["my_remote_cluster:field_caps_index_3"]}
+  - is_false: fields.geo.keyword.non_searchable_indices
+  - is_false: fields.geo.keyword.on_aggregatable_indices
+
+  - do:
+      field_caps:
+        index: 'my_remote_cluster:some_index_that_doesnt_exist'
+        fields: [number]
+  - match: { 'fields': {} } # empty response - this index doesn't exists
+
+  - do:
+      field_caps:
+        index: 'my_remote_cluster:field_caps_index_1'
+        fields: [number]
+  - match: {fields.number.double.searchable:            true}
+  - match: {fields.number.double.aggregatable:          true}
diff --git a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/40_scroll.yml b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/40_scroll.yml
new file mode 100644
index 0000000000000..aac5c438c323c
--- /dev/null
+++ b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/40_scroll.yml
@@ -0,0 +1,40 @@
+---
+"Scroll on the mixed cluster":
+
+  - do:
+      search:
+        index: my_remote_cluster:test_index
+        size: 4
+        scroll: 1m
+        sort: filter_field
+        body:
+          query:
+            match_all: {}
+
+  - set: {_scroll_id: scroll_id}
+  - match: {hits.total:      6    }
+  - length: {hits.hits:      4    }
+  - match: {hits.hits.0._source.filter_field: 0 }
+  - match: {hits.hits.1._source.filter_field: 0 }
+  - match: {hits.hits.2._source.filter_field: 0 }
+  - match: {hits.hits.3._source.filter_field: 0 }
+
+  - do:
+      scroll:
+        body: { "scroll_id": "$scroll_id", "scroll": "1m"}
+
+  - match: {hits.total:      6    }
+  - length: {hits.hits:      2    }
+  - match: {hits.hits.0._source.filter_field: 1 }
+  - match: {hits.hits.1._source.filter_field: 1 }
+  - do:
+      scroll:
+        scroll_id: $scroll_id
+        scroll: 1m
+
+  - match: {hits.total:      6    }
+  - length: {hits.hits:      0    }
+
+  - do:
+      clear_scroll:
+        scroll_id: $scroll_id
diff --git a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/50_missing.yml b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/50_missing.yml
new file mode 100644
index 0000000000000..c40e7be1c643e
--- /dev/null
+++ b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/50_missing.yml
@@ -0,0 +1,44 @@
+---
+"Search with missing remote index pattern":
+  - do:
+      catch: "request"
+      search:
+        index: "my_remote_cluster:foo"
+
+  - do:
+      search:
+        index: "my_remote_cluster:fooo*"
+  - match: { _shards.total: 0 }
+  - match: { hits.total: 0 }
+
+  - do:
+      search:
+        index: "*:foo*"
+
+  - match: { _shards.total: 0 }
+  - match: { hits.total: 0 }
+
+  - do:
+      search:
+        index: "my_remote_cluster:test_index,my_remote_cluster:foo*"
+        body:
+          aggs:
+            cluster:
+              terms:
+                field: f1.keyword
+
+  - match: { _shards.total: 3 }
+  - match: { hits.total: 6 }
+  - length: { aggregations.cluster.buckets: 1 }
+  - match: { aggregations.cluster.buckets.0.key: "remote_cluster" }
+  - match: { aggregations.cluster.buckets.0.doc_count: 6 }
+
+  - do:
+      catch: "request"
+      search:
+        index: "my_remote_cluster:test_index,my_remote_cluster:foo"
+        body:
+          aggs:
+            cluster:
+              terms:
+                field: f1.keyword
diff --git a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/60_tophits.yml b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/60_tophits.yml
new file mode 100644
index 0000000000000..478305c926f12
--- /dev/null
+++ b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/60_tophits.yml
@@ -0,0 +1,39 @@
+---
+"Test that remote index names are preserved in top hits":
+
+  - do:
+      indices.create:
+        index: single_doc_index
+        body:
+          settings:
+            index:
+              number_of_shards: 1
+              number_of_replicas: 0
+
+  - do:
+      bulk:
+        refresh: true
+        body:
+            - '{"index": {"_index": "single_doc_index", "_type": "test_type"}}'
+            - '{"f1": "local_cluster", "sort_field": 0}'
+  - do:
+      search:
+        index: "single_doc_index,my_remote_cluster:single_doc_index"
+        body:
+          sort: "sort_field"
+          aggs:
+            cluster:
+              top_hits:
+                size: 2
+                sort: "sort_field"
+
+  - match: { _shards.total: 2 }
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0._index: "single_doc_index"}
+  - match: { hits.hits.1._index: "my_remote_cluster:single_doc_index"}
+
+  - length: { aggregations.cluster.hits.hits: 2 }
+  - match: { aggregations.cluster.hits.hits.0._index: "single_doc_index" }
+  - match: { aggregations.cluster.hits.hits.0._source.f1: "local_cluster" }
+  - match: { aggregations.cluster.hits.hits.1._index: "my_remote_cluster:single_doc_index" }
+  - match: { aggregations.cluster.hits.hits.1._source.f1: "remote_cluster" }
diff --git a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/70_skip_shards.yml b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/70_skip_shards.yml
new file mode 100644
index 0000000000000..e50b32c5e4b3c
--- /dev/null
+++ b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/70_skip_shards.yml
@@ -0,0 +1,55 @@
+---
+"Test that remote indices are subject to shard skipping":
+
+  - do:
+      indices.create:
+        index: skip_shards_index
+        body:
+          settings:
+            index:
+              number_of_shards: 1
+              number_of_replicas: 0
+          mappings:
+            test_type:
+              properties:
+                created_at:
+                   type: date
+                   format: "yyyy-MM-dd"
+
+  - do:
+      bulk:
+        refresh: true
+        body:
+            - '{"index": {"_index": "skip_shards_index", "_type": "test_type"}}'
+            - '{"f1": "local_cluster", "sort_field": 0, "created_at" : "2017-01-01"}'
+
+  # check that we skip the remote shard
+  - do:
+     search:
+       index: "skip_shards_index,my_remote_cluster:single_doc_index"
+       pre_filter_shard_size: 1
+       body: { "size" : 10, "query" : { "range" : { "created_at" : { "gte" : "2016-02-01", "lt": "2018-02-01"} } } }
+
+  - match: { hits.total: 1 }
+  - match: { hits.hits.0._index: "skip_shards_index"}
+  - match: { _shards.total: 2 }
+  - match: { _shards.successful: 2 }
+  - match: { _shards.skipped : 1}
+  - match: { _shards.failed: 0 }
+  - match: { hits.total: 1 }
+
+  # check that we skip the local shard
+  - do:
+     search:
+       index: "skip_shards_index,my_remote_cluster:single_doc_index"
+       pre_filter_shard_size: 1
+       body: { "size" : 10, "query" : { "range" : { "created_at" : { "gte" : "2015-02-01", "lt": "2016-02-01"} } } }
+
+  - match: { hits.total: 1 }
+  - match: { hits.hits.0._index: "my_remote_cluster:single_doc_index"}
+  - match: { _shards.total: 2 }
+  - match: { _shards.successful: 2 }
+  - match: { _shards.skipped : 1}
+  - match: { _shards.failed: 0 }
+  - match: { hits.total: 1 }
+
diff --git a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/remote_cluster/10_basic.yaml b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/remote_cluster/10_basic.yaml
new file mode 100644
index 0000000000000..d37bb5a182586
--- /dev/null
+++ b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/remote_cluster/10_basic.yaml
@@ -0,0 +1,129 @@
+---
+"Index data and search on the old cluster":
+
+  - do:
+      indices.create:
+        index: single_doc_index
+        body:
+          settings:
+            index:
+              number_of_shards: 1
+              number_of_replicas: 0
+          mappings:
+            test_type:
+              properties:
+                created_at:
+                   type: date
+                   format: "yyyy-MM-dd"
+
+  - do:
+      bulk:
+        refresh: true
+        body:
+            - '{"index": {"_index": "single_doc_index", "_type": "test_type"}}'
+            - '{"f1": "remote_cluster", "sort_field": 1, "created_at" : "2016-01-01"}'
+
+  - do:
+        indices.create:
+          index: field_caps_index_1
+          body:
+              mappings:
+                t:
+                  properties:
+                    text:
+                      type:     text
+                    keyword:
+                      type:     keyword
+                    number:
+                      type:     double
+                    geo:
+                      type:     geo_point
+                    object:
+                      type: object
+                      properties:
+                        nested1 :
+                          type : text
+                          index: false
+                        nested2:
+                          type: float
+                          doc_values: false
+  - do:
+        indices.create:
+          index: field_caps_index_3
+          body:
+              mappings:
+                t:
+                  properties:
+                    text:
+                      type:     text
+                    keyword:
+                      type:     keyword
+                    number:
+                      type:     long
+                    geo:
+                      type:     keyword
+                    object:
+                      type: object
+                      properties:
+                        nested1 :
+                          type : long
+                          index: false
+                        nested2:
+                          type: keyword
+                          doc_values: false
+  - do:
+      indices.create:
+        index: test_index
+        body:
+          settings:
+            index:
+              number_of_shards: 3
+              number_of_replicas: 0
+          aliases:
+            aliased_test_index: # we use this alias in the multi cluster test to very filtered aliases work
+              filter:
+                term:
+                  filter_field : 1
+
+  - do:
+      bulk:
+         refresh: true
+         body:
+           - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+           - '{"f1": "remote_cluster", "filter_field": 0}'
+           - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+           - '{"f1": "remote_cluster", "filter_field": 1}'
+           - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+           - '{"f1": "remote_cluster", "filter_field": 0}'
+           - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+           - '{"f1": "remote_cluster", "filter_field": 1}'
+           - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+           - '{"f1": "remote_cluster", "filter_field": 0}'
+           - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+           - '{"f1": "remote_cluster", "filter_field": 0}'
+
+  - do:
+      search:
+        index: test_index
+        body:
+          aggs:
+            cluster:
+              terms:
+                field: f1.keyword
+
+  - match: { _shards.total: 3 }
+  - match: { hits.total: 6 }
+  - length: { aggregations.cluster.buckets: 1 }
+  - match: { aggregations.cluster.buckets.0.key: "remote_cluster" }
+  - match: { aggregations.cluster.buckets.0.doc_count: 6 }
+
+  - do:
+      search:
+        index: aliased_test_index
+
+  - match: { _shards.total: 3 }
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0._source.filter_field: 1 }
+  - match: { hits.hits.0._index: "test_index" }
+
+
diff --git a/qa/no-bootstrap-tests/build.gradle b/qa/no-bootstrap-tests/build.gradle
new file mode 100644
index 0000000000000..25731a99deedc
--- /dev/null
+++ b/qa/no-bootstrap-tests/build.gradle
@@ -0,0 +1,25 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Tests that need to run without the Elasticsearch bootstrap, for
+ * example to test process spawning.
+ */
+
+apply plugin: 'elasticsearch.standalone-test'
diff --git a/qa/no-bootstrap-tests/src/test/java/org/elasticsearch/bootstrap/SpawnerNoBootstrapTests.java b/qa/no-bootstrap-tests/src/test/java/org/elasticsearch/bootstrap/SpawnerNoBootstrapTests.java
new file mode 100644
index 0000000000000..8da6d8d3afe23
--- /dev/null
+++ b/qa/no-bootstrap-tests/src/test/java/org/elasticsearch/bootstrap/SpawnerNoBootstrapTests.java
@@ -0,0 +1,211 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.bootstrap;
+
+import org.apache.lucene.util.Constants;
+import org.apache.lucene.util.LuceneTestCase;
+import org.elasticsearch.Version;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.plugins.PluginTestUtil;
+import org.elasticsearch.plugins.Platforms;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.attribute.PosixFileAttributeView;
+import java.nio.file.attribute.PosixFilePermission;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+
+/**
+ * Create a simple "daemon controller", put it in the right place and check that it runs.
+ *
+ * Extends LuceneTestCase rather than ESTestCase as ESTestCase installs a system call filter, and
+ * that prevents the Spawner class from doing its job. Also needs to run in a separate JVM to other
+ * tests that extend ESTestCase for the same reason.
+ */
+public class SpawnerNoBootstrapTests extends LuceneTestCase {
+
+    private static final String CONTROLLER_SOURCE = "#!/bin/bash\n"
+            + "\n"
+            + "echo I am alive\n"
+            + "\n"
+            + "read SOMETHING\n";
+
+    /**
+     * Simplest case: a plugin with no controller daemon.
+     */
+    public void testNoControllerSpawn() throws IOException, InterruptedException {
+        Path esHome = createTempDir().resolve("esHome");
+        Settings.Builder settingsBuilder = Settings.builder();
+        settingsBuilder.put(Environment.PATH_HOME_SETTING.getKey(), esHome.toString());
+        Settings settings = settingsBuilder.build();
+
+        Environment environment = new Environment(settings);
+
+        // This plugin will NOT have a controller daemon
+        Path plugin = environment.pluginsFile().resolve("a_plugin");
+        Files.createDirectories(plugin);
+        PluginTestUtil.writeProperties(
+                plugin,
+                "description", "a_plugin",
+                "version", Version.CURRENT.toString(),
+                "elasticsearch.version", Version.CURRENT.toString(),
+                "name", "a_plugin",
+                "java.version", "1.8",
+                "classname", "APlugin",
+                "has.native.controller", "false");
+
+        try (Spawner spawner = new Spawner()) {
+            spawner.spawnNativePluginControllers(environment);
+            assertThat(spawner.getProcesses(), hasSize(0));
+        }
+    }
+
+    /**
+     * Two plugins - one with a controller daemon and one without.
+     */
+    public void testControllerSpawn() throws IOException, InterruptedException {
+        /*
+         * On Windows you can not directly run a batch file - you have to run cmd.exe with the batch
+         * file as an argument and that's out of the remit of the controller daemon process spawner.
+         */
+        assumeFalse("This test does not work on Windows", Constants.WINDOWS);
+
+        Path esHome = createTempDir().resolve("esHome");
+        Settings.Builder settingsBuilder = Settings.builder();
+        settingsBuilder.put(Environment.PATH_HOME_SETTING.getKey(), esHome.toString());
+        Settings settings = settingsBuilder.build();
+
+        Environment environment = new Environment(settings);
+
+        // this plugin will have a controller daemon
+        Path plugin = environment.pluginsFile().resolve("test_plugin");
+        Files.createDirectories(plugin);
+        PluginTestUtil.writeProperties(
+                plugin,
+                "description", "test_plugin",
+                "version", Version.CURRENT.toString(),
+                "elasticsearch.version", Version.CURRENT.toString(),
+                "name", "test_plugin",
+                "java.version", "1.8",
+                "classname", "TestPlugin",
+                "has.native.controller", "true");
+        Path controllerProgram = Platforms.nativeControllerPath(plugin);
+        createControllerProgram(controllerProgram);
+
+        // this plugin will not have a controller daemon
+        Path otherPlugin = environment.pluginsFile().resolve("other_plugin");
+        Files.createDirectories(otherPlugin);
+        PluginTestUtil.writeProperties(
+                otherPlugin,
+                "description", "other_plugin",
+                "version", Version.CURRENT.toString(),
+                "elasticsearch.version", Version.CURRENT.toString(),
+                "name", "other_plugin",
+                "java.version", "1.8",
+                "classname", "OtherPlugin",
+                "has.native.controller", "false");
+
+        Spawner spawner = new Spawner();
+        spawner.spawnNativePluginControllers(environment);
+
+        List<Process> processes = spawner.getProcesses();
+        /*
+         * As there should only be a reference in the list for the plugin that had the controller
+         * daemon, we expect one here.
+         */
+        assertThat(processes, hasSize(1));
+        Process process = processes.get(0);
+        final InputStreamReader in =
+                new InputStreamReader(process.getInputStream(), StandardCharsets.UTF_8);
+        try (BufferedReader stdoutReader = new BufferedReader(in)) {
+            String line = stdoutReader.readLine();
+            assertEquals("I am alive", line);
+            spawner.close();
+            /*
+             * Fail if the process does not die within one second; usually it will be even quicker
+             * but it depends on OS scheduling.
+             */
+            assertTrue(process.waitFor(1, TimeUnit.SECONDS));
+        }
+    }
+
+    public void testControllerSpawnWithIncorrectDescriptor() throws IOException {
+        // this plugin will have a controller daemon
+        Path esHome = createTempDir().resolve("esHome");
+        Settings.Builder settingsBuilder = Settings.builder();
+        settingsBuilder.put(Environment.PATH_HOME_SETTING.getKey(), esHome.toString());
+        Settings settings = settingsBuilder.build();
+
+        Environment environment = new Environment(settings);
+
+        Path plugin = environment.pluginsFile().resolve("test_plugin");
+        Files.createDirectories(plugin);
+        PluginTestUtil.writeProperties(
+                plugin,
+                "description", "test_plugin",
+                "version", Version.CURRENT.toString(),
+                "elasticsearch.version", Version.CURRENT.toString(),
+                "name", "test_plugin",
+                "java.version", "1.8",
+                "classname", "TestPlugin",
+                "has.native.controller", "false");
+        Path controllerProgram = Platforms.nativeControllerPath(plugin);
+        createControllerProgram(controllerProgram);
+
+        Spawner spawner = new Spawner();
+        IllegalArgumentException e = expectThrows(
+                IllegalArgumentException.class,
+                () -> spawner.spawnNativePluginControllers(environment));
+        assertThat(
+                e.getMessage(),
+                equalTo("plugin [test_plugin] does not have permission to fork native controller"));
+    }
+
+    private void createControllerProgram(final Path outputFile) throws IOException {
+        final Path outputDir = outputFile.getParent();
+        Files.createDirectories(outputDir);
+        Files.write(outputFile, CONTROLLER_SOURCE.getBytes(StandardCharsets.UTF_8));
+        final PosixFileAttributeView view =
+                Files.getFileAttributeView(outputFile, PosixFileAttributeView.class);
+        if (view != null) {
+            final Set<PosixFilePermission> perms = new HashSet<>();
+            perms.add(PosixFilePermission.OWNER_READ);
+            perms.add(PosixFilePermission.OWNER_WRITE);
+            perms.add(PosixFilePermission.OWNER_EXECUTE);
+            perms.add(PosixFilePermission.GROUP_READ);
+            perms.add(PosixFilePermission.GROUP_EXECUTE);
+            perms.add(PosixFilePermission.OTHERS_READ);
+            perms.add(PosixFilePermission.OTHERS_EXECUTE);
+            Files.setPosixFilePermissions(outputFile, perms);
+        }
+    }
+
+}
diff --git a/qa/reindex-from-old/build.gradle b/qa/reindex-from-old/build.gradle
new file mode 100644
index 0000000000000..72927842e037b
--- /dev/null
+++ b/qa/reindex-from-old/build.gradle
@@ -0,0 +1,91 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+description = """\
+Tests reindex-from-remote against some specific versions of
+Elasticsearch prior to 5.0. Versions of Elasticsearch >= 5.0
+should be able to use the standard launching mechanism which
+is more flexible and reliable.
+"""
+
+import org.apache.tools.ant.taskdefs.condition.Os
+
+apply plugin: 'elasticsearch.standalone-rest-test'
+apply plugin: 'elasticsearch.rest-test'
+
+integTestCluster {
+  // Whitelist reindexing from the local node so we can test it.
+  setting 'reindex.remote.whitelist', '127.0.0.1:*'
+}
+
+configurations {
+  oldesFixture
+  es2
+  es1
+  es090
+}
+
+dependencies {
+  oldesFixture project(':test:fixtures:old-elasticsearch')
+  /* Right now we just test against the latest version of each major we expect
+   * reindex-from-remote to work against. We could randomize the versions but
+   * that doesn't seem worth it at this point. */
+  es2 'org.elasticsearch.distribution.zip:elasticsearch:2.4.5@zip'
+  es1 'org.elasticsearch:elasticsearch:1.7.6@zip'
+  es090 'org.elasticsearch:elasticsearch:0.90.13@zip'
+}
+
+if (project.javaVersion == JavaVersion.VERSION_1_9 || Os.isFamily(Os.FAMILY_WINDOWS)) {
+  /* We can't run the dependencies with Java 9 so for now we'll skip the whole
+   * thing. We can't get the pid files in windows so we skip that as well.... */
+  integTest.enabled = false
+} else {
+  /* Set up tasks to unzip and run the old versions of ES before running the
+   * integration tests. */
+  for (String version : ['2', '1', '090']) {
+    Task unzip = task("unzipEs${version}", type: Sync) {
+      Configuration oldEsDependency = configurations['es' + version]
+      dependsOn oldEsDependency
+      /* Use a closure here to delay resolution of the dependency until we need
+       * it */
+      from {
+        oldEsDependency.collect { zipTree(it) }
+      }
+      into temporaryDir
+    }
+    Task fixture = task("oldEs${version}Fixture",
+          type: org.elasticsearch.gradle.test.AntFixture) {
+      dependsOn project.configurations.oldesFixture
+      dependsOn unzip
+      executable = new File(project.javaHome, 'bin/java')
+      env 'CLASSPATH', "${ -> project.configurations.oldesFixture.asPath }"
+      args 'oldes.OldElasticsearch',
+           baseDir,
+           unzip.temporaryDir,
+           version == '090'
+    }
+    integTest.dependsOn fixture
+    integTestRunner {
+      /* Use a closure on the string to delay evaluation until right before we
+       * run the integration tests so that we can be sure that the file is
+       * ready. */
+      systemProperty "es${version}.port", "${ -> fixture.addressAndPort }"
+    }
+  }
+}
diff --git a/qa/reindex-from-old/src/test/java/org/elasticsearch/smoketest/ReindexFromOldRemoteIT.java b/qa/reindex-from-old/src/test/java/org/elasticsearch/smoketest/ReindexFromOldRemoteIT.java
new file mode 100644
index 0000000000000..162e68e402730
--- /dev/null
+++ b/qa/reindex-from-old/src/test/java/org/elasticsearch/smoketest/ReindexFromOldRemoteIT.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.smoketest;
+
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpHost;
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.util.EntityUtils;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.test.rest.ESRestTestCase;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.TreeMap;
+
+import static java.util.Collections.singletonMap;
+import static org.hamcrest.Matchers.containsString;
+
+public class ReindexFromOldRemoteIT extends ESRestTestCase {
+    private void oldEsTestCase(String portPropertyName, String requestsPerSecond) throws IOException {
+        int oldEsPort = Integer.parseInt(System.getProperty(portPropertyName));
+        try (RestClient oldEs = RestClient.builder(new HttpHost("127.0.0.1", oldEsPort)).build()) {
+            try {
+                HttpEntity entity = new StringEntity("{\"settings\":{\"number_of_shards\": 1}}", ContentType.APPLICATION_JSON);
+                oldEs.performRequest("PUT", "/test", singletonMap("refresh", "true"), entity);
+
+                entity = new StringEntity("{\"test\":\"test\"}", ContentType.APPLICATION_JSON);
+                oldEs.performRequest("PUT", "/test/doc/testdoc1", singletonMap("refresh", "true"), entity);
+                oldEs.performRequest("PUT", "/test/doc/testdoc2", singletonMap("refresh", "true"), entity);
+                oldEs.performRequest("PUT", "/test/doc/testdoc3", singletonMap("refresh", "true"), entity);
+                oldEs.performRequest("PUT", "/test/doc/testdoc4", singletonMap("refresh", "true"), entity);
+                oldEs.performRequest("PUT", "/test/doc/testdoc5", singletonMap("refresh", "true"), entity);
+
+                entity = new StringEntity(
+                        "{\n"
+                      + "  \"source\":{\n"
+                      + "    \"index\": \"test\",\n"
+                      + "    \"size\": 1,\n"
+                      + "    \"remote\": {\n"
+                      + "      \"host\": \"http://127.0.0.1:" + oldEsPort + "\"\n"
+                      + "    }\n"
+                      + "  },\n"
+                      + "  \"dest\": {\n"
+                      + "    \"index\": \"test\"\n"
+                      + "  }\n"
+                      + "}",
+                      ContentType.APPLICATION_JSON);
+                Map<String, String> params = new TreeMap<>();
+                params.put("refresh", "true");
+                params.put("pretty", "true");
+                if (requestsPerSecond != null) {
+                    params.put("requests_per_second", requestsPerSecond);
+                }
+                client().performRequest("POST", "/_reindex", params, entity);
+
+                Response response = client().performRequest("POST", "test/_search", singletonMap("pretty", "true"));
+                String result = EntityUtils.toString(response.getEntity());
+                assertThat(result, containsString("\"_id\" : \"testdoc1\""));
+            } finally {
+                oldEs.performRequest("DELETE", "/test");
+            }
+        }
+    }
+
+    public void testEs2() throws IOException {
+        oldEsTestCase("es2.port", null);
+    }
+
+    public void testEs1() throws IOException {
+        oldEsTestCase("es1.port", null);
+    }
+
+    public void testEs090() throws IOException {
+        oldEsTestCase("es090.port", null);
+    }
+
+    public void testEs2WithFunnyThrottle() throws IOException {
+        oldEsTestCase("es2.port", "11"); // 11 requests per second should give us a nice "funny" number on the scroll timeout
+    }
+
+    public void testEs1WithFunnyThrottle() throws IOException {
+        oldEsTestCase("es1.port", "11"); // 11 requests per second should give us a nice "funny" number on the scroll timeout
+    }
+
+    public void testEs090WithFunnyThrottle() throws IOException {
+        oldEsTestCase("es090.port", "11"); // 11 requests per second should give us a nice "funny" number on the scroll timeout
+    }
+
+}
diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle
new file mode 100644
index 0000000000000..e515b1ad2efbc
--- /dev/null
+++ b/qa/rolling-upgrade/build.gradle
@@ -0,0 +1,111 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.elasticsearch.gradle.test.RestIntegTestTask
+import org.elasticsearch.gradle.Version
+
+apply plugin: 'elasticsearch.standalone-test'
+
+// dummy task for all bwc tests
+task bwcTest {
+  description = 'Runs backwards compatibility tests.'
+  group = 'verification'
+}
+
+for (Version version : wireCompatVersions) {
+  String baseName = "v${version}"
+
+  Task oldClusterTest = tasks.create(name: "${baseName}#oldClusterTest", type: RestIntegTestTask) {
+    mustRunAfter(precommit)
+  }
+
+  Object extension = extensions.findByName("${baseName}#oldClusterTestCluster")
+  configure(extensions.findByName("${baseName}#oldClusterTestCluster")) {
+    distribution = 'zip'
+    bwcVersion = version
+    numBwcNodes = 2
+    numNodes = 2
+    clusterName = 'rolling-upgrade'
+    setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
+    if (version.onOrAfter('5.3.0')) {
+      setting 'http.content_type.required', 'true'
+    }
+  }
+
+  Task oldClusterTestRunner = tasks.getByName("${baseName}#oldClusterTestRunner")
+  oldClusterTestRunner.configure {
+    systemProperty 'tests.rest.suite', 'old_cluster'
+  }
+
+  Task mixedClusterTest = tasks.create(name: "${baseName}#mixedClusterTest", type: RestIntegTestTask)
+
+  configure(extensions.findByName("${baseName}#mixedClusterTestCluster")) {
+    dependsOn oldClusterTestRunner, "${baseName}#oldClusterTestCluster#node1.stop"
+    distribution = 'zip'
+    clusterName = 'rolling-upgrade'
+    unicastTransportUri = { seedNode, node, ant -> oldClusterTest.nodes.get(0).transportUri() }
+    dataDir = "${-> oldClusterTest.nodes[1].dataDir}"
+    setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
+  }
+
+  Task mixedClusterTestRunner = tasks.getByName("${baseName}#mixedClusterTestRunner")
+  mixedClusterTestRunner.configure {
+    systemProperty 'tests.rest.suite', 'mixed_cluster'
+    finalizedBy "${baseName}#oldClusterTestCluster#node0.stop"
+  }
+
+  Task upgradedClusterTest = tasks.create(name: "${baseName}#upgradedClusterTest", type: RestIntegTestTask) {
+    dependsOn(mixedClusterTestRunner, "${baseName}#oldClusterTestCluster#node0.stop")
+  }
+
+  configure(extensions.findByName("${baseName}#upgradedClusterTestCluster")) {
+    distribution = 'zip'
+    clusterName = 'rolling-upgrade'
+    unicastTransportUri = { seedNode, node, ant -> mixedClusterTest.nodes.get(0).transportUri() }
+    dataDir = "${-> oldClusterTest.nodes[0].dataDir}"
+    setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
+  }
+
+  Task upgradedClusterTestRunner = tasks.getByName("${baseName}#upgradedClusterTestRunner")
+  upgradedClusterTestRunner.configure {
+    systemProperty 'tests.rest.suite', 'upgraded_cluster'
+    // only need to kill the mixed cluster tests node here because we explicitly told it to not stop nodes upon completion
+    finalizedBy "${baseName}#mixedClusterTestCluster#stop"
+  }
+
+  Task versionBwcTest = tasks.create(name: "${baseName}#bwcTest") {
+    enabled = project.bwc_tests_enabled
+    dependsOn = [upgradedClusterTest]
+  }
+
+  if (project.bwc_tests_enabled) {
+    bwcTest.dependsOn(versionBwcTest)
+  }
+}
+
+test.enabled = false // no unit tests for rolling upgrades, only the rest integration test
+
+// basic integ tests includes testing bwc against the most recent version
+task integTest {
+  if (project.bwc_tests_enabled) {
+    dependsOn = ["v${wireCompatVersions[-1]}#bwcTest"]
+  }
+}
+
+check.dependsOn(integTest)
diff --git a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java
new file mode 100644
index 0000000000000..c34db91f52efb
--- /dev/null
+++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.upgrades;
+
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
+import org.apache.lucene.util.TimeUnits;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
+import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+
+@TimeoutSuite(millis = 5 * TimeUnits.MINUTE) // to account for slow as hell VMs
+public class UpgradeClusterClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
+
+    @Override
+    protected boolean preserveIndicesUponCompletion() {
+        return true;
+    }
+
+    @Override
+    protected boolean preserveReposUponCompletion() {
+        return true;
+    }
+
+    public UpgradeClusterClientYamlTestSuiteIT(ClientYamlTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return createParameters();
+    }
+
+    @Override
+    protected Settings restClientSettings() {
+        return Settings.builder().put(super.restClientSettings())
+            // increase the timeout so that we can actually see the result of failed cluster health
+            // calls that have a default timeout of 30s
+            .put(ESRestTestCase.CLIENT_RETRY_TIMEOUT, "40s")
+            .put(ESRestTestCase.CLIENT_SOCKET_TIMEOUT, "40s")
+            .build();
+    }
+
+    @Override
+    protected boolean randomizeContentType() {
+        return false;
+    }
+}
diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yaml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yaml
new file mode 100644
index 0000000000000..5e34aa576191b
--- /dev/null
+++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yaml
@@ -0,0 +1,66 @@
+---
+"Index data and search on the mixed cluster":
+ - do:
+     cluster.health:
+        wait_for_status: yellow
+        wait_for_nodes: 2
+
+ - do:
+     search:
+        index: test_index
+
+ - match: { hits.total: 5 } # no new indexed data, so expect the original 5 documents from the old cluster
+
+ - do:
+     search:
+       index: index_with_replicas
+
+ - match: { hits.total: 5 } # just check we recovered fine
+
+ - do:
+     bulk:
+        refresh: true
+        body:
+          - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+          - '{"f1": "v1_mixed", "f2": 5}'
+          - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+          - '{"f1": "v2_mixed", "f2": 6}'
+          - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+          - '{"f1": "v3_mixed", "f2": 7}'
+          - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+          - '{"f1": "v4_mixed", "f2": 8}'
+          - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+          - '{"f1": "v5_mixed", "f2": 9}'
+
+ - do:
+     indices.flush:
+        index: test_index
+
+ - do:
+     search:
+        index: test_index
+
+ - match: { hits.total: 10 } # 5 docs from old cluster, 5 docs from mixed cluster
+
+---
+"Verify that we can still find things with the template":
+  - do:
+      search_template:
+        body:
+          id: test_search_template
+          params:
+            f1: v5_old
+  - match: { hits.total: 1 }
+
+---
+"Verify custom cluster metadata still exists during upgrade":
+  - do:
+      snapshot.get_repository:
+        repository: my_repo
+  - is_true: my_repo
+
+  - do:
+      ingest.get_pipeline:
+        id: "my_pipeline"
+  - match: { my_pipeline.description: "_description" }
+
diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/20_stored_scripts.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/20_stored_scripts.yml
new file mode 100644
index 0000000000000..c6bdccb45e763
--- /dev/null
+++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/20_stored_scripts.yml
@@ -0,0 +1,93 @@
+---
+"Verify put old format scripts":
+  - skip:
+      features: warnings
+
+  - do:
+      cluster.health:
+        wait_for_status: green
+        wait_for_nodes: 2
+
+  - do:
+      search:
+        index: stored_index
+        body: {
+          "query": {
+            "match_all": {
+            }
+          }
+        }
+  - match: { hits.total: 3 }
+
+  - do:
+      warnings:
+        - 'specifying lang [painless] as part of the url path is deprecated'
+      get_script:
+        id: "greater"
+        lang: "painless"
+  - match: { "found": true }
+  - match: { "_id": "greater" }
+  - match: { "lang": "painless"}
+  - match: { "script": "doc['num'].value > 1.0" }
+
+  - do:
+      warnings:
+        - 'specifying lang [painless] as part of the url path is deprecated'
+      get_script:
+        id: "value"
+        lang: "painless"
+  - match: { "found": true }
+  - match: { "_id": "value" }
+  - match: { "lang": "painless"}
+  - match: { "script": "doc['num'].value" }
+
+  - do:
+      warnings:
+        - 'specifying lang [expression] as part of the url path is deprecated'
+      get_script:
+        id: "value"
+        lang: "expression"
+  - match: { "found": true }
+  - match: { "_id": "value" }
+  - match: { "lang": "expression"}
+  - match: { "script": "doc['num'].value" }
+
+  - do:
+      warnings:
+        - 'specifying the field [lang] for executing stored scripts is deprecated; use only the field [stored] to specify an <id>'
+      search:
+        index: stored_index
+        body: {
+          "query": {
+            "script": {
+              "script": {
+                "stored": "greater",
+                "lang": "painless"
+              }
+            }
+          },
+          "script_fields": {
+            "script_painless": {
+              "script": {
+                "stored": "value",
+                "lang": "painless"
+              }
+            },
+            "script_expressions": {
+              "script": {
+                "stored": "value",
+                "lang": "expression"
+              }
+            }
+          },
+          "sort": {
+            "num": {
+              "order": "asc"
+            }
+          }
+        }
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0.fields.script_painless.0: 2.0 }
+  - match: { hits.hits.1.fields.script_painless.0: 3.0 }
+  - match: { hits.hits.0.fields.script_expressions.0: 2.0 }
+  - match: { hits.hits.1.fields.script_expressions.0: 3.0 }
diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yaml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yaml
new file mode 100644
index 0000000000000..1afff401ab1a2
--- /dev/null
+++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yaml
@@ -0,0 +1,213 @@
+---
+"Index data, search, and create things in the cluster state that we'll validate are there after the ugprade 5.6.0 and beyond":
+  - skip:
+      version: " - 5.5.99"
+      reason: stored search template API is deprecated in 5.6.0
+      features: "warnings"
+
+  - do:
+      indices.create:
+        index: test_index
+        body:
+          settings:
+            index:
+              number_of_replicas: 0
+
+  - do:
+      indices.create:
+        index: index_with_replicas # dummy index to ensure we can recover indices with replicas just fine
+        body:
+          # if the node with the replica is the first to be restarted, then delayed
+          # allocation will kick in, and the cluster health won't return to GREEN
+          # before timing out
+          index.unassigned.node_left.delayed_timeout: "100ms"
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - '{"index": {"_index": "test_index", "_type": "doc"}}'
+          - '{"f1": "v1_old", "f2": 0}'
+          - '{"index": {"_index": "test_index", "_type": "doc"}}'
+          - '{"f1": "v2_old", "f2": 1}'
+          - '{"index": {"_index": "test_index", "_type": "doc"}}'
+          - '{"f1": "v3_old", "f2": 2}'
+          - '{"index": {"_index": "test_index", "_type": "doc"}}'
+          - '{"f1": "v4_old", "f2": 3}'
+          - '{"index": {"_index": "test_index", "_type": "doc"}}'
+          - '{"f1": "v5_old", "f2": 4}'
+
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - '{"index": {"_index": "index_with_replicas", "_type": "doc"}}'
+          - '{"f1": "d_old"}'
+          - '{"index": {"_index": "index_with_replicas", "_type": "doc"}}'
+          - '{"f1": "d_old"}'
+          - '{"index": {"_index": "index_with_replicas", "_type": "doc"}}'
+          - '{"f1": "d_old"}'
+          - '{"index": {"_index": "index_with_replicas", "_type": "doc"}}'
+          - '{"f1": "d_old"}'
+          - '{"index": {"_index": "index_with_replicas", "_type": "doc"}}'
+          - '{"f1": "d_old"}'
+
+  - do:
+      indices.flush:
+        index: test_index,index_with_replicas
+
+  - do:
+      search:
+        index: test_index
+
+  - match: { hits.total: 5 }
+
+  - do:
+      search:
+        index: index_with_replicas
+
+  - match: { hits.total: 5 }
+
+  - do:
+      snapshot.create_repository:
+        repository: my_repo
+        body:
+          type: url
+          settings:
+            url: "http://snapshot.test"
+  - match: { "acknowledged": true }
+
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body:  >
+          {
+            "description": "_description",
+            "processors": [
+            ]
+          }
+  - match: { "acknowledged": true }
+
+  - do:
+      warnings:
+        - "The stored search template API is deprecated. Use stored scripts instead."
+      put_template:
+        id: test_search_template
+        body:
+          query:
+            match:
+              f1: "{{f1}}"
+  - match: { acknowledged: true }
+
+  - do:
+      search_template:
+        body:
+          id: test_search_template
+          params:
+            f1: v5_old
+  - match: { hits.total: 1 }
+
+---
+"Index data, search, and create things in the cluster state that we'll validate are there after the ugprade before 5.6.0":
+  - skip:
+      version: "5.6.0 - "
+      reason: stored search template API is deprecated in 5.6.0
+
+  - do:
+      indices.create:
+        index: test_index
+        body:
+          settings:
+            index:
+              number_of_replicas: 0
+
+  - do:
+      indices.create:
+        index: index_with_replicas # dummy index to ensure we can recover indices with replicas just fine
+        body:
+          # if the node with the replica is the first to be restarted, then delayed
+          # allocation will kick in, and the cluster health won't return to GREEN
+          # before timing out
+          index.unassigned.node_left.delayed_timeout: "100ms"
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - '{"index": {"_index": "test_index", "_type": "doc"}}'
+          - '{"f1": "v1_old", "f2": 0}'
+          - '{"index": {"_index": "test_index", "_type": "doc"}}'
+          - '{"f1": "v2_old", "f2": 1}'
+          - '{"index": {"_index": "test_index", "_type": "doc"}}'
+          - '{"f1": "v3_old", "f2": 2}'
+          - '{"index": {"_index": "test_index", "_type": "doc"}}'
+          - '{"f1": "v4_old", "f2": 3}'
+          - '{"index": {"_index": "test_index", "_type": "doc"}}'
+          - '{"f1": "v5_old", "f2": 4}'
+
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - '{"index": {"_index": "index_with_replicas", "_type": "doc"}}'
+          - '{"f1": "d_old"}'
+          - '{"index": {"_index": "index_with_replicas", "_type": "doc"}}'
+          - '{"f1": "d_old"}'
+          - '{"index": {"_index": "index_with_replicas", "_type": "doc"}}'
+          - '{"f1": "d_old"}'
+          - '{"index": {"_index": "index_with_replicas", "_type": "doc"}}'
+          - '{"f1": "d_old"}'
+          - '{"index": {"_index": "index_with_replicas", "_type": "doc"}}'
+          - '{"f1": "d_old"}'
+
+  - do:
+      indices.flush:
+        index: test_index,index_with_replicas
+
+  - do:
+      search:
+        index: test_index
+
+  - match: { hits.total: 5 }
+
+  - do:
+      search:
+        index: index_with_replicas
+
+  - match: { hits.total: 5 }
+
+  - do:
+      snapshot.create_repository:
+        repository: my_repo
+        body:
+          type: url
+          settings:
+            url: "http://snapshot.test"
+  - match: { "acknowledged": true }
+
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body:  >
+          {
+            "description": "_description",
+            "processors": [
+            ]
+          }
+  - match: { "acknowledged": true }
+
+  - do:
+      put_template:
+        id: test_search_template
+        body:
+          query:
+            match:
+              f1: "{{f1}}"
+  - match: { acknowledged: true }
+
+  - do:
+      search_template:
+        body:
+          id: test_search_template
+          params:
+            f1: v5_old
+  - match: { hits.total: 1 }
+
diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/20_stored_scripts.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/20_stored_scripts.yml
new file mode 100644
index 0000000000000..440a363f8499e
--- /dev/null
+++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/20_stored_scripts.yml
@@ -0,0 +1,51 @@
+---
+"Add stored scripts using legacy format":
+  - do:
+     indices.create:
+        index: stored_index
+        body:
+          settings:
+            index:
+              number_of_replicas: 0
+
+  - do:
+     bulk:
+        refresh: true
+        body:
+          - '{"index": {"_index": "stored_index", "_type": "test"}}'
+          - '{"value": "value1", "num": 1.0}'
+          - '{"index": {"_index": "stored_index", "_type": "test"}}'
+          - '{"value": "value2", "num": 2.0}'
+          - '{"index": {"_index": "stored_index", "_type": "test"}}'
+          - '{"value": "value3", "num": 3.0}'
+
+  - do:
+     indices.flush:
+        index: stored_index
+
+  - do:
+      put_script:
+        id: "greater"
+        lang: "painless"
+        body: {
+          "script": "doc['num'].value > 1.0"
+        }
+  - match: { acknowledged: true }
+
+  - do:
+      put_script:
+        id: "value"
+        lang: "painless"
+        body: {
+          "script": "doc['num'].value"
+        }
+  - match: { acknowledged: true }
+
+  - do:
+      put_script:
+        id: "value"
+        lang: "expression"
+        body: {
+          "script": "doc['num'].value"
+        }
+  - match: { acknowledged: true }
diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yaml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yaml
new file mode 100644
index 0000000000000..101e230289132
--- /dev/null
+++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yaml
@@ -0,0 +1,66 @@
+---
+"Index data and search on the upgraded cluster":
+ - do:
+     cluster.health:
+        wait_for_status: green
+        wait_for_nodes: 2
+
+ - do:
+     search:
+       index: test_index
+
+ - match: { hits.total: 10 } # no new indexed data, so expect the original 10 documents from the old and mixed clusters
+
+ - do:
+     search:
+       index: index_with_replicas
+
+ - match: { hits.total: 5 } # just check we recovered fine
+
+ - do:
+     bulk:
+        refresh: true
+        body:
+          - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+          - '{"f1": "v1_upgraded", "f2": 10}'
+          - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+          - '{"f1": "v2_upgraded", "f2": 11}'
+          - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+          - '{"f1": "v3_upgraded", "f2": 12}'
+          - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+          - '{"f1": "v4_upgraded", "f2": 13}'
+          - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+          - '{"f1": "v5_upgraded", "f2": 14}'
+
+ - do:
+     indices.flush:
+        index: test_index
+
+ - do:
+     search:
+        index: test_index
+
+ - match: { hits.total: 15 } # 10 docs from previous clusters plus 5 new docs
+
+---
+"Verify that we can still find things with the template":
+  - do:
+      search_template:
+        body:
+          id: test_search_template
+          params:
+            f1: v5_old
+  - match: { hits.total: 1 }
+
+---
+"Verify custom cluster metadata still exists after rolling upgrade":
+  - do:
+      snapshot.get_repository:
+        repository: my_repo
+  - is_true: my_repo
+
+  - do:
+      ingest.get_pipeline:
+        id: "my_pipeline"
+  - match: { my_pipeline.description: "_description" }
+
diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/20_stored_scripts.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/20_stored_scripts.yml
new file mode 100644
index 0000000000000..811a58b41f2d2
--- /dev/null
+++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/20_stored_scripts.yml
@@ -0,0 +1,423 @@
+---
+"Test old and new scripts work":
+  - skip:
+      features: warnings
+
+  - do:
+      cluster.health:
+        wait_for_status: green
+        wait_for_nodes: 2
+
+  - do:
+      search:
+        index: stored_index
+        body: {
+          "query": {
+            "match_all": {
+            }
+          }
+        }
+  - match: { hits.total: 3 }
+
+  - do:
+      warnings:
+        - 'specifying lang [painless] as part of the url path is deprecated'
+      get_script:
+        id: "greater"
+        lang: "painless"
+  - match: { "found": true }
+  - match: { "_id": "greater" }
+  - match: { "lang": "painless"}
+  - match: { "script": "doc['num'].value > 1.0" }
+
+  - do:
+      warnings:
+        - 'specifying lang [painless] as part of the url path is deprecated'
+      get_script:
+        id: "value"
+        lang: "painless"
+  - match: { "found": true }
+  - match: { "_id": "value" }
+  - match: { "lang": "painless"}
+  - match: { "script": "doc['num'].value" }
+
+  - do:
+      warnings:
+        - 'specifying lang [expression] as part of the url path is deprecated'
+      get_script:
+        id: "value"
+        lang: "expression"
+  - match: { "found": true }
+  - match: { "_id": "value" }
+  - match: { "lang": "expression"}
+  - match: { "script": "doc['num'].value" }
+
+  - do:
+      warnings:
+        - 'specifying the field [lang] for executing stored scripts is deprecated; use only the field [stored] to specify an <id>'
+      search:
+        index: stored_index
+        body: {
+          "query": {
+            "script": {
+              "script": {
+                "stored": "greater",
+                "lang": "painless"
+              }
+            }
+          },
+          "script_fields": {
+            "script_painless": {
+              "script": {
+                "stored": "value",
+                "lang": "painless"
+              }
+            },
+            "script_expressions": {
+              "script": {
+                "stored": "value",
+                "lang": "expression"
+              }
+            }
+          },
+          "sort": {
+            "num": {
+              "order": "asc"
+            }
+          }
+        }
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0.fields.script_painless.0: 2.0 }
+  - match: { hits.hits.1.fields.script_painless.0: 3.0 }
+  - match: { hits.hits.0.fields.script_expressions.0: 2.0 }
+  - match: { hits.hits.1.fields.script_expressions.0: 3.0 }
+
+  - do:
+      warnings:
+        - 'specifying lang [painless] as part of the url path is deprecated, use request content instead'
+      put_script:
+        id: "greater"
+        lang: "painless"
+        body: {
+          "script": "doc['num'].value > 1.0"
+        }
+  - match: { acknowledged: true }
+
+  - do:
+      warnings:
+          - 'specifying lang [painless] as part of the url path is deprecated, use request content instead'
+      put_script:
+        id: "value"
+        lang: "painless"
+        body: {
+          "script": "doc['num'].value"
+        }
+  - match: { acknowledged: true }
+
+  - do:
+      warnings:
+          - 'specifying lang [expression] as part of the url path is deprecated, use request content instead'
+          - 'stored script [value] already exists using a different lang [painless], the new namespace for stored scripts will only use (id) instead of (lang, id)'
+      put_script:
+        id: "value"
+        lang: "expression"
+        body: {
+          "script": "doc['num'].value"
+        }
+  - match: { acknowledged: true }
+
+  - do:
+      warnings:
+        - 'specifying lang [painless] as part of the url path is deprecated'
+      get_script:
+        id: "greater"
+        lang: "painless"
+  - match: { "found": true }
+  - match: { "_id": "greater" }
+  - match: { "lang": "painless"}
+  - match: { "script": "doc['num'].value > 1.0" }
+
+  - do:
+      warnings:
+        - 'specifying lang [painless] as part of the url path is deprecated'
+      get_script:
+        id: "value"
+        lang: "painless"
+  - match: { "found": true }
+  - match: { "_id": "value" }
+  - match: { "lang": "painless"}
+  - match: { "script": "doc['num'].value" }
+
+  - do:
+      warnings:
+        - 'specifying lang [expression] as part of the url path is deprecated'
+      get_script:
+        id: "value"
+        lang: "expression"
+  - match: { "found": true }
+  - match: { "_id": "value" }
+  - match: { "lang": "expression"}
+  - match: { "script": "doc['num'].value" }
+
+  - do:
+      warnings:
+        - 'specifying the field [lang] for executing stored scripts is deprecated; use only the field [stored] to specify an <id>'
+      search:
+        index: stored_index
+        body: {
+          "query": {
+            "script": {
+              "script": {
+                "stored": "greater",
+                "lang": "painless"
+              }
+            }
+          },
+          "script_fields": {
+            "script_painless": {
+              "script": {
+                "stored": "value"
+              }
+            },
+            "script_expressions": {
+              "script": {
+                "stored": "value",
+                "lang": "expression"
+              }
+            }
+          },
+          "sort": {
+            "num": {
+              "order": "asc"
+            }
+          }
+        }
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0.fields.script_painless.0: 2.0 }
+  - match: { hits.hits.1.fields.script_painless.0: 3.0 }
+  - match: { hits.hits.0.fields.script_expressions.0: 2.0 }
+  - match: { hits.hits.1.fields.script_expressions.0: 3.0 }
+
+  - do:
+      warnings:
+        - 'specifying lang [painless] as part of the url path is deprecated'
+      delete_script:
+        id: "value"
+        lang: "painless"
+  - match: { acknowledged: true }
+
+  - do:
+      get_script:
+        lang: "value"
+  - match: { found: true }
+  - match: { _id: "value" }
+  - match: { script.lang: "expression"}
+  - match: { script.code: "doc['num'].value" }
+
+  - do:
+      warnings:
+        - 'specifying lang [expression] as part of the url path is deprecated'
+      get_script:
+        id: "value"
+        lang: "expression"
+  - match: { found: true }
+  - match: { _id: "value" }
+  - match: { lang: "expression"}
+  - match: { script: "doc['num'].value" }
+
+  - do:
+      warnings:
+        - 'specifying lang [painless] as part of the url path is deprecated'
+      catch: missing
+      get_script:
+        id: "value"
+        lang: "painless"
+  - match: { found: false }
+  - match: { _id: "value" }
+
+  - do:
+      warnings:
+        - 'stored script [value] already exists using a different lang [expression], the new namespace for stored scripts will only use (id) instead of (lang, id)'
+      put_script:
+        lang: "value"
+        body: {
+          "script": {
+            "code": "doc['num'].value",
+            "lang": "painless"
+          }
+        }
+  - match: { acknowledged: true }
+
+  - do:
+      get_script:
+        lang: "value"
+  - match: { found: true }
+  - match: { _id: "value" }
+  - match: { script.lang: "painless"}
+  - match: { script.code: "doc['num'].value" }
+
+  - do:
+      warnings:
+        - 'specifying lang [expression] as part of the url path is deprecated'
+      get_script:
+        id: "value"
+        lang: "expression"
+  - match: { found: true }
+  - match: { _id: "value" }
+  - match: { lang: "expression"}
+  - match: { script: "doc['num'].value" }
+
+  - do:
+      warnings:
+        - 'specifying lang [painless] as part of the url path is deprecated'
+      get_script:
+        id: "value"
+        lang: "painless"
+  - match: { found: true }
+  - match: { _id: "value" }
+  - match: { lang: "painless"}
+  - match: { script: "doc['num'].value" }
+
+  - do:
+      warnings:
+        - 'stored script [value] already exists using a different lang [painless], the new namespace for stored scripts will only use (id) instead of (lang, id)'
+      put_script:
+        lang: "value"
+        body: {
+          "script": {
+            "code": "doc['num'].value",
+            "lang": "expression"
+          }
+        }
+  - match: { acknowledged: true }
+
+  - do:
+      get_script:
+        lang: "value"
+  - match: { found: true }
+  - match: { _id: "value" }
+  - match: { script.lang: "expression"}
+  - match: { script.code: "doc['num'].value" }
+
+  - do:
+      warnings:
+        - 'specifying lang [expression] as part of the url path is deprecated'
+      get_script:
+        id: "value"
+        lang: "expression"
+  - match: { found: true }
+  - match: { _id: "value" }
+  - match: { lang: "expression"}
+  - match: { script: "doc['num'].value" }
+
+  - do:
+      warnings:
+        - 'specifying lang [painless] as part of the url path is deprecated'
+      get_script:
+        id: "value"
+        lang: "painless"
+  - match: { found: true }
+  - match: { _id: "value" }
+  - match: { lang: "painless"}
+  - match: { script: "doc['num'].value" }
+
+  - do:
+      delete_script:
+        lang: "value"
+  - match: { acknowledged: true }
+
+  - do:
+      catch: missing
+      get_script:
+        lang: "value"
+  - match: { found: false }
+  - match: { _id: "value" }
+
+  - do:
+      warnings:
+        - 'specifying lang [expression] as part of the url path is deprecated'
+      catch: missing
+      get_script:
+        id: "value"
+        lang: "expression"
+  - match: { found: false }
+  - match: { _id: "value" }
+
+  - do:
+      warnings:
+        - 'specifying lang [painless] as part of the url path is deprecated'
+      get_script:
+        id: "value"
+        lang: "painless"
+  - match: { found: true }
+  - match: { _id: "value" }
+  - match: { lang: "painless"}
+  - match: { script: "doc['num'].value" }
+
+  - do:
+      put_script:
+        lang: "value"
+        body: {
+          "script": {
+            "code": "doc['num'].value",
+            "lang": "painless"
+          }
+        }
+  - match: { acknowledged: true }
+
+  - do:
+      get_script:
+        lang: "value"
+  - match: { found: true }
+  - match: { _id: "value" }
+  - match: { script.lang: "painless"}
+  - match: { script.code: "doc['num'].value" }
+
+  - do:
+      warnings:
+        - 'specifying lang [expression] as part of the url path is deprecated'
+      catch: missing
+      get_script:
+        id: "value"
+        lang: "expression"
+  - match: { found: false }
+  - match: { _id: "value" }
+
+  - do:
+      warnings:
+        - 'specifying lang [painless] as part of the url path is deprecated'
+      get_script:
+        id: "value"
+        lang: "painless"
+  - match: { found: true }
+  - match: { _id: "value" }
+  - match: { lang: "painless"}
+  - match: { script: "doc['num'].value" }
+
+  - do:
+      search:
+        index: stored_index
+        body: {
+          "query": {
+            "script": {
+              "script": {
+                "stored": "greater"
+              }
+            }
+          },
+          "script_fields": {
+            "script_painless": {
+              "script": {
+                "stored": "value"
+              }
+            }
+          },
+          "sort": {
+            "num": {
+              "order": "asc"
+            }
+          }
+        }
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0.fields.script_painless.0: 2.0 }
+  - match: { hits.hits.1.fields.script_painless.0: 3.0 }
diff --git a/qa/smoke-test-client/build.gradle b/qa/smoke-test-client/build.gradle
index 260516a5bf68f..a575a131d8759 100644
--- a/qa/smoke-test-client/build.gradle
+++ b/qa/smoke-test-client/build.gradle
@@ -1,3 +1,5 @@
+import org.elasticsearch.gradle.test.RestIntegTestTask
+
 /*
  * Licensed to Elasticsearch under one or more contributor
  * license agreements. See the NOTICE file distributed with
@@ -17,10 +19,24 @@
  * under the License.
  */
 
+apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 // TODO: this test works, but it isn't really a rest test...should we have another plugin for "non rest test that just needs N clusters?"
 
 dependencies {
     testCompile project(path: ':client:transport', configuration: 'runtime') // randomly swapped in as a transport
-}
\ No newline at end of file
+}
+
+task singleNodeIntegTest(type: RestIntegTestTask) {
+    mustRunAfter(precommit)
+}
+
+singleNodeIntegTestCluster {
+    numNodes = 1
+    setting 'discovery.type', 'single-node'
+}
+
+integTestCluster.dependsOn(singleNodeIntegTestRunner, 'singleNodeIntegTestCluster#stop')
+
+check.dependsOn(integTest)
diff --git a/qa/smoke-test-client/src/test/java/org/elasticsearch/smoketest/SmokeTestClientIT.java b/qa/smoke-test-client/src/test/java/org/elasticsearch/smoketest/SmokeTestClientIT.java
index 6380ed90e18c9..2e86ce82221d2 100644
--- a/qa/smoke-test-client/src/test/java/org/elasticsearch/smoketest/SmokeTestClientIT.java
+++ b/qa/smoke-test-client/src/test/java/org/elasticsearch/smoketest/SmokeTestClientIT.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.smoketest;
 
-import org.apache.lucene.util.Constants;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.client.Client;
@@ -29,35 +28,29 @@
 
 public class SmokeTestClientIT extends ESSmokeClientTestCase {
 
-    // needed to avoid the test suite from failing for having no tests
-    // TODO: remove when Netty 4.1.5 is upgraded to Netty 4.1.6 including https://github.com/netty/netty/pull/5778
-    public void testSoThatTestsDoNotFail() {
-
-    }
-
     /**
      * Check that we are connected to a cluster named "elasticsearch".
      */
     public void testSimpleClient() {
-        // TODO: remove when Netty 4.1.5 is upgraded to Netty 4.1.6 including https://github.com/netty/netty/pull/5778
-        assumeFalse("JDK is JDK 9", Constants.JRE_IS_MINIMUM_JAVA9);
-        Client client = getClient();
+        final Client client = getClient();
 
         // START SNIPPET: java-doc-admin-cluster-health
-        ClusterHealthResponse health = client.admin().cluster().prepareHealth().setWaitForYellowStatus().get();
-        String clusterName = health.getClusterName();
-        int numberOfNodes = health.getNumberOfNodes();
+        final ClusterHealthResponse health =
+                client.admin().cluster().prepareHealth().setWaitForYellowStatus().get();
+        final String clusterName = health.getClusterName();
+        final int numberOfNodes = health.getNumberOfNodes();
         // END SNIPPET: java-doc-admin-cluster-health
-        assertThat("cluster [" + clusterName + "] should have at least 1 node", numberOfNodes, greaterThan(0));
+        assertThat(
+                "cluster [" + clusterName + "] should have at least 1 node",
+                numberOfNodes,
+                greaterThan(0));
     }
 
     /**
      * Create an index and index some docs
      */
     public void testPutDocument() {
-        // TODO: remove when Netty 4.1.5 is upgraded to Netty 4.1.6 including https://github.com/netty/netty/pull/5778
-        assumeFalse("JDK is JDK 9", Constants.JRE_IS_MINIMUM_JAVA9);
-        Client client = getClient();
+        final Client client = getClient();
 
         // START SNIPPET: java-doc-index-doc-simple
         client.prepareIndex(index, "doc", "1")  // Index, Type, Id
@@ -71,7 +64,7 @@ public void testPutDocument() {
         // END SNIPPET: java-doc-admin-indices-refresh
 
         // START SNIPPET: java-doc-search-simple
-        SearchResponse searchResponse = client.prepareSearch(index).get();
+        final SearchResponse searchResponse = client.prepareSearch(index).get();
         assertThat(searchResponse.getHits().getTotalHits(), is(1L));
         // END SNIPPET: java-doc-search-simple
     }
diff --git a/qa/smoke-test-http/build.gradle b/qa/smoke-test-http/build.gradle
index 038115c2d248a..131b52ae09a96 100644
--- a/qa/smoke-test-http/build.gradle
+++ b/qa/smoke-test-http/build.gradle
@@ -17,9 +17,18 @@
  * under the License.
  */
 
+apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
     testCompile project(path: ':modules:transport-netty3', configuration: 'runtime') // for http
     testCompile project(path: ':modules:transport-netty4', configuration: 'runtime') // for http
-}
\ No newline at end of file
+}
+
+integTestRunner {
+    /*
+     * We have to disable setting the number of available processors as tests in the same JVM randomize processors and will step on each
+     * other if we allow them to set the number of available processors as it's set-once in Netty.
+     */
+    systemProperty 'es.set.netty.runtime.available.processors', 'false'
+}
diff --git a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/ContextAndHeaderTransportIT.java b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/ContextAndHeaderTransportIT.java
index 1249f1deadf0a..24fbfd90b5578 100644
--- a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/ContextAndHeaderTransportIT.java
+++ b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/ContextAndHeaderTransportIT.java
@@ -22,7 +22,6 @@
 import org.apache.http.message.BasicHeader;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
-import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.index.IndexRequest;
@@ -76,9 +75,9 @@
 public class ContextAndHeaderTransportIT extends HttpSmokeTestCase {
     private static final List<RequestAndHeaders> requests =  new CopyOnWriteArrayList<>();
     private static final String CUSTOM_HEADER = "SomeCustomHeader";
-    private String randomHeaderValue = randomAsciiOfLength(20);
-    private String queryIndex = "query-" + randomAsciiOfLength(10).toLowerCase(Locale.ROOT);
-    private String lookupIndex = "lookup-" + randomAsciiOfLength(10).toLowerCase(Locale.ROOT);
+    private String randomHeaderValue = randomAlphaOfLength(20);
+    private String queryIndex = "query-" + randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
+    private String lookupIndex = "lookup-" + randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
 
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {
@@ -322,15 +321,10 @@ public int order() {
         }
 
         @Override
-        protected boolean apply(String action, ActionRequest request, ActionListener listener) {
+        protected boolean apply(String action, ActionRequest request, ActionListener<?> listener) {
             requests.add(new RequestAndHeaders(threadPool.getThreadContext().getHeaders(), request));
             return true;
         }
-
-        @Override
-        protected boolean apply(String action, ActionResponse response, ActionListener listener) {
-            return true;
-        }
     }
 
     private static class RequestAndHeaders {
diff --git a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DeprecationHttpIT.java b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DeprecationHttpIT.java
index 778daa7714ec4..948f573a05c8a 100644
--- a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DeprecationHttpIT.java
+++ b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DeprecationHttpIT.java
@@ -23,6 +23,7 @@
 import org.apache.http.entity.ContentType;
 import org.apache.http.entity.StringEntity;
 import org.elasticsearch.client.Response;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.logging.LoggerMessageFormat;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
@@ -39,10 +40,12 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
-import static org.elasticsearch.rest.RestStatus.OK;
+import static org.elasticsearch.common.logging.DeprecationLogger.WARNING_HEADER_PATTERN;
 import static org.elasticsearch.http.TestDeprecationHeaderRestAction.TEST_DEPRECATED_SETTING_TRUE1;
 import static org.elasticsearch.http.TestDeprecationHeaderRestAction.TEST_DEPRECATED_SETTING_TRUE2;
 import static org.elasticsearch.http.TestDeprecationHeaderRestAction.TEST_NOT_DEPRECATED_SETTING;
+import static org.elasticsearch.rest.RestStatus.OK;
+import static org.elasticsearch.test.hamcrest.RegexMatcher.matches;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasItem;
@@ -167,13 +170,20 @@ private void doTestDeprecationWarningsAppearInHeaders() throws IOException {
         }
         for (Setting<?> setting : settings) {
             if (setting.isDeprecated()) {
-                headerMatchers.add(containsString(LoggerMessageFormat.format("[{}] setting was deprecated", (Object)setting.getKey())));
+                headerMatchers.add(equalTo(
+                        "[" + setting.getKey() + "] setting was deprecated in Elasticsearch and will be removed in a future release! " +
+                        "See the breaking changes documentation for the next major version."));
             }
         }
 
         assertThat(deprecatedWarnings, hasSize(headerMatchers.size()));
+        for (final String deprecatedWarning : deprecatedWarnings) {
+            assertThat(deprecatedWarning, matches(WARNING_HEADER_PATTERN.pattern()));
+        }
+        final List<String> actualWarningValues =
+                deprecatedWarnings.stream().map(DeprecationLogger::extractWarningValueFromWarningHeader).collect(Collectors.toList());
         for (Matcher<String> headerMatcher : headerMatchers) {
-            assertThat(deprecatedWarnings, hasItem(headerMatcher));
+            assertThat(actualWarningValues, hasItem(headerMatcher));
         }
     }
 
diff --git a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DetailedErrorsDisabledIT.java b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DetailedErrorsDisabledIT.java
index 0fd4d6e6eddd9..b4158c2a2f32d 100644
--- a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DetailedErrorsDisabledIT.java
+++ b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DetailedErrorsDisabledIT.java
@@ -19,18 +19,18 @@
 
 package org.elasticsearch.http;
 
+import java.io.IOException;
+import java.util.Collections;
+
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
 
-import java.io.IOException;
-import java.util.Collections;
-
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.is;
 
 /**
@@ -54,14 +54,13 @@ protected boolean ignoreExternalCluster() {
     }
 
     public void testThatErrorTraceParamReturns400() throws IOException {
-        try {
-            getRestClient().performRequest("DELETE", "/", Collections.singletonMap("error_trace", "true"));
-            fail("request should have failed");
-        } catch(ResponseException e) {
-            Response response = e.getResponse();
-            assertThat(response.getHeader("Content-Type"), is("application/json"));
-            assertThat(EntityUtils.toString(e.getResponse().getEntity()), is("{\"error\":\"error traces in responses are disabled.\"}"));
-            assertThat(response.getStatusLine().getStatusCode(), is(400));
-        }
+        ResponseException e = expectThrows(ResponseException.class, () ->
+            getRestClient().performRequest("DELETE", "/", Collections.singletonMap("error_trace", "true")));
+
+        Response response = e.getResponse();
+        assertThat(response.getHeader("Content-Type"), is("application/json; charset=UTF-8"));
+        assertThat(EntityUtils.toString(e.getResponse().getEntity()),
+            containsString("{\"error\":\"error traces in responses are disabled.\",\"status\":400}"));
+        assertThat(response.getStatusLine().getStatusCode(), is(400));
     }
 }
diff --git a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DetailedErrorsEnabledIT.java b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DetailedErrorsEnabledIT.java
index cd3cf1a38d8b2..d0b80595a26ee 100644
--- a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DetailedErrorsEnabledIT.java
+++ b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DetailedErrorsEnabledIT.java
@@ -22,7 +22,6 @@
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
-import org.elasticsearch.test.ESIntegTestCase;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -52,7 +51,7 @@ public void testThatErrorTraceWorksByDefault() throws IOException {
             fail("request should have failed");
         } catch(ResponseException e) {
             Response response = e.getResponse();
-            assertThat(response.getHeader("Content-Type"), containsString("application/json"));
+            assertThat(response.getHeader("Content-Type"), containsString("application/json; charset=UTF-8"));
             assertThat(EntityUtils.toString(response.getEntity()),
                     not(containsString("\"stack_trace\":\"[Validation Failed: 1: index / indices is missing;]; "
                     + "nested: ActionRequestValidationException[Validation Failed: 1:")));
diff --git a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/TestDeprecationHeaderRestAction.java b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/TestDeprecationHeaderRestAction.java
index cff7b55f9de9c..bc60b7aa57439 100644
--- a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/TestDeprecationHeaderRestAction.java
+++ b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/TestDeprecationHeaderRestAction.java
@@ -19,19 +19,17 @@
 package org.elasticsearch.http;
 
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestStatus;
 
+import java.io.IOException;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -70,7 +68,6 @@ public class TestDeprecationHeaderRestAction extends BaseRestHandler {
     public static final String DEPRECATED_ENDPOINT = "[/_test_cluster/deprecated_settings] exists for deprecated tests";
     public static final String DEPRECATED_USAGE = "[deprecated_settings] usage is deprecated. use [settings] instead";
 
-    @Inject
     public TestDeprecationHeaderRestAction(Settings settings, RestController controller) {
         super(settings);
 
@@ -80,10 +77,10 @@ public TestDeprecationHeaderRestAction(Settings settings, RestController control
 
     @SuppressWarnings("unchecked") // List<String> casts
     @Override
-    public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
+    public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
         final List<String> settings;
 
-        try (XContentParser parser = XContentFactory.xContent(request.content()).createParser(request.content())) {
+        try (XContentParser parser = request.contentParser()) {
             final Map<String, Object> source = parser.map();
 
             if (source.containsKey("deprecated_settings")) {
@@ -95,14 +92,15 @@ public void handleRequest(RestRequest request, RestChannel channel, NodeClient c
             }
         }
 
-        final XContentBuilder builder = channel.newBuilder();
+        return channel -> {
+            final XContentBuilder builder = channel.newBuilder();
 
-        builder.startObject().startArray("settings");
-        for (String setting : settings) {
-            builder.startObject().field(setting, SETTINGS.get(setting).getRaw(this.settings)).endObject();
-        }
-        builder.endArray().endObject();
-
-        channel.sendResponse(new BytesRestResponse(RestStatus.OK, builder));
+            builder.startObject().startArray("settings");
+            for (String setting : settings) {
+                builder.startObject().field(setting, SETTINGS.get(setting).getRaw(this.settings)).endObject();
+            }
+            builder.endArray().endObject();
+            channel.sendResponse(new BytesRestResponse(RestStatus.OK, builder));
+        };
     }
 }
diff --git a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/TestDeprecationPlugin.java b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/TestDeprecationPlugin.java
index d047713f0c7f4..a264227999eda 100644
--- a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/TestDeprecationPlugin.java
+++ b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/TestDeprecationPlugin.java
@@ -18,15 +18,23 @@
  */
 package org.elasticsearch.http;
 
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.SearchPlugin;
+import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestHandler;
 
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.function.Supplier;
 
 import static java.util.Collections.singletonList;
 
@@ -36,8 +44,10 @@
 public class TestDeprecationPlugin extends Plugin implements ActionPlugin, SearchPlugin {
 
     @Override
-    public List<Class<? extends RestHandler>> getRestHandlers() {
-        return Collections.singletonList(TestDeprecationHeaderRestAction.class);
+    public List<RestHandler> getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings,
+            IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver,
+            Supplier<DiscoveryNodes> nodesInCluster) {
+        return Collections.singletonList(new TestDeprecationHeaderRestAction(settings, restController));
     }
 
     @Override
diff --git a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/TestResponseHeaderPlugin.java b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/TestResponseHeaderPlugin.java
index 398f990a5cf5e..421c6d9bade2d 100644
--- a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/TestResponseHeaderPlugin.java
+++ b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/TestResponseHeaderPlugin.java
@@ -19,17 +19,27 @@
 
 package org.elasticsearch.http;
 
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.IndexScopedSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestHandler;
 
 import java.util.List;
+import java.util.function.Supplier;
 
 import static java.util.Collections.singletonList;
 
 public class TestResponseHeaderPlugin extends Plugin implements ActionPlugin {
     @Override
-    public List<Class<? extends RestHandler>> getRestHandlers() {
-        return singletonList(TestResponseHeaderRestAction.class);
+    public List<RestHandler> getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings,
+            IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver,
+            Supplier<DiscoveryNodes> nodesInCluster) {
+        return singletonList(new TestResponseHeaderRestAction(settings, restController));
     }
 }
diff --git a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/TestResponseHeaderRestAction.java b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/TestResponseHeaderRestAction.java
index a321fc0a457db..d99dd5d524525 100644
--- a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/TestResponseHeaderRestAction.java
+++ b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/TestResponseHeaderRestAction.java
@@ -19,34 +19,32 @@
 package org.elasticsearch.http;
 
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.RestStatus;
 
-public class TestResponseHeaderRestAction extends BaseRestHandler {
+import java.io.IOException;
 
-    @Inject
+public class TestResponseHeaderRestAction extends BaseRestHandler {
     public TestResponseHeaderRestAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(RestRequest.Method.GET, "/_protected", this);
     }
 
     @Override
-    public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) {
+    public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
         if ("password".equals(request.header("Secret"))) {
             RestResponse response = new BytesRestResponse(RestStatus.OK, "Access granted");
             response.addHeader("Secret", "granted");
-            channel.sendResponse(response);
+            return channel -> channel.sendResponse(response);
         } else {
             RestResponse response = new BytesRestResponse(RestStatus.UNAUTHORIZED, "Access denied");
             response.addHeader("Secret", "required");
-            channel.sendResponse(response);
+            return channel -> channel.sendResponse(response);
         }
     }
 }
diff --git a/qa/smoke-test-ingest-disabled/build.gradle b/qa/smoke-test-ingest-disabled/build.gradle
index 08dfbf8ae7a73..cf3ca9a713f96 100644
--- a/qa/smoke-test-ingest-disabled/build.gradle
+++ b/qa/smoke-test-ingest-disabled/build.gradle
@@ -17,14 +17,13 @@
  * under the License.
  */
 
+apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
     testCompile project(path: ':modules:ingest-common', configuration: 'runtime')
 }
 
-integTest {
-    cluster {
-        setting 'node.ingest', 'false'
-    }
+integTestCluster {
+    setting 'node.ingest', 'false'
 }
diff --git a/qa/smoke-test-ingest-disabled/src/test/java/org/elasticsearch/smoketest/SmokeTestIngestDisabledClientYamlTestSuiteIT.java b/qa/smoke-test-ingest-disabled/src/test/java/org/elasticsearch/smoketest/SmokeTestIngestDisabledClientYamlTestSuiteIT.java
index c8d506424c0eb..2cefd0f3335aa 100644
--- a/qa/smoke-test-ingest-disabled/src/test/java/org/elasticsearch/smoketest/SmokeTestIngestDisabledClientYamlTestSuiteIT.java
+++ b/qa/smoke-test-ingest-disabled/src/test/java/org/elasticsearch/smoketest/SmokeTestIngestDisabledClientYamlTestSuiteIT.java
@@ -24,9 +24,6 @@
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-
-import java.io.IOException;
 
 public class SmokeTestIngestDisabledClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
@@ -35,8 +32,8 @@ public SmokeTestIngestDisabledClientYamlTestSuiteIT(@Name("yaml") ClientYamlTest
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
-        return ESClientYamlSuiteTestCase.createParameters(0, 1);
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
     }
 
 }
diff --git a/qa/smoke-test-ingest-with-all-dependencies/build.gradle b/qa/smoke-test-ingest-with-all-dependencies/build.gradle
index df90bf5b9820a..73f80f29e2a5e 100644
--- a/qa/smoke-test-ingest-with-all-dependencies/build.gradle
+++ b/qa/smoke-test-ingest-with-all-dependencies/build.gradle
@@ -17,6 +17,7 @@
  * under the License.
  */
 
+apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
@@ -27,10 +28,9 @@ dependencies {
     testCompile project(path: ':modules:reindex', configuration: 'runtime')
 }
 
-integTest {
-    cluster {
-        plugin ':plugins:ingest-geoip'
-        setting 'script.inline', 'true'
-        setting 'path.scripts', "${project.buildDir}/resources/test/scripts"
-    }
+integTestCluster {
+    plugin ':plugins:ingest-geoip'
+    setting 'script.inline', 'true'
+    setting 'path.scripts', "${project.buildDir}/resources/test/scripts"
+    setting 'script.max_compilations_per_minute', '1000'
 }
diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/AbstractScriptTestCase.java b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/AbstractScriptTestCase.java
index 1c60a32e88317..1ca4c0252c39a 100644
--- a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/AbstractScriptTestCase.java
+++ b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/AbstractScriptTestCase.java
@@ -28,7 +28,6 @@
 import org.elasticsearch.script.mustache.MustacheScriptEngineService;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
-import org.mockito.internal.util.collections.Sets;
 
 import java.util.Arrays;
 import java.util.Collections;
@@ -41,9 +40,8 @@ public abstract class AbstractScriptTestCase extends ESTestCase {
     public void init() throws Exception {
         Settings settings = Settings.builder()
             .put("path.home", createTempDir())
-            .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), false)
             .build();
-        ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Arrays.asList(new MustacheScriptEngineService(settings)));
+        ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Arrays.asList(new MustacheScriptEngineService()));
         ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList());
         ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
 
diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/smoketest/SmokeTestIngestWithAllDepsClientYamlTestSuiteIT.java b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/smoketest/SmokeTestIngestWithAllDepsClientYamlTestSuiteIT.java
index b3b84dfc55e8d..aca15564e90dd 100644
--- a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/smoketest/SmokeTestIngestWithAllDepsClientYamlTestSuiteIT.java
+++ b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/smoketest/SmokeTestIngestWithAllDepsClientYamlTestSuiteIT.java
@@ -24,9 +24,6 @@
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-
-import java.io.IOException;
 
 public class SmokeTestIngestWithAllDepsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
@@ -35,8 +32,8 @@ public SmokeTestIngestWithAllDepsClientYamlTestSuiteIT(@Name("yaml") ClientYamlT
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
-        return ESClientYamlSuiteTestCase.createParameters(0, 1);
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
     }
 
 }
diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/50_script_processor_using_painless.yaml b/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/50_script_processor_using_painless.yaml
index 8a80dec1c0972..b59cd7c1c6037 100644
--- a/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/50_script_processor_using_painless.yaml
+++ b/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/50_script_processor_using_painless.yaml
@@ -9,8 +9,7 @@
             "processors": [
               {
                 "script" : {
-                  "lang" : "painless",
-                  "inline": "ctx.bytes_total = (ctx.bytes_in + ctx.bytes_out) * params.factor",
+                  "source": "ctx.bytes_total = (ctx.bytes_in + ctx.bytes_out) * params.factor",
                   "params": {
                      "factor": 10
                   }
@@ -48,7 +47,6 @@
             "processors": [
               {
                 "script" : {
-                  "lang" : "painless",
                   "file": "master"
                 }
               }
@@ -75,13 +73,18 @@
 
 ---
 "Test script processor with stored script":
+  - skip:
+      features: warnings
+
   - do:
       put_script:
-        id: "sum_bytes"
-        lang: "painless"
+        lang: "sum_bytes"
         body: >
           {
-            "script" : "ctx.bytes_total = ctx.bytes_in + ctx.bytes_out"
+            "script": {
+              "lang": "painless",
+              "source" : "ctx.bytes_total = ctx.bytes_in + ctx.bytes_out"
+            }
           }
   - match: { acknowledged: true }
 
@@ -94,7 +97,6 @@
             "processors": [
               {
                 "script" : {
-                  "lang" : "painless",
                   "id" : "sum_bytes"
                 }
               }
@@ -118,3 +120,26 @@
   - match: { _source.bytes_in: 1234 }
   - match: { _source.bytes_out: 4321 }
   - match: { _source.bytes_total: 5555 }
+
+---
+"Test script processor with syntax error in inline script":
+  - do:
+      catch: request
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body:  >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "script" : {
+                  "source": "invalid painless, hear me roar!"
+                }
+              }
+            ]
+          }
+  - match: { error.header.processor_type: "script" }
+  - match: { error.header.property_name: "source" }
+  - match: { error.type: "script_exception" }
+  - match: { error.reason: "compile error" }
+
diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/60_pipeline_timestamp_date_mapping.yaml b/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/60_pipeline_timestamp_date_mapping.yaml
new file mode 100644
index 0000000000000..475e342c5764a
--- /dev/null
+++ b/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/60_pipeline_timestamp_date_mapping.yaml
@@ -0,0 +1,121 @@
+---
+"Test timestamp templating does not match date-mapping defaults":
+  - do:
+      cluster.health:
+          wait_for_status: green
+
+  - do:
+      indices.create:
+        index: timetest
+        body:
+          mappings:
+            test: { "properties": { "my_time": {"type": "date"}}}
+
+  - do:
+      ingest.put_pipeline:
+        id: "my_timely_pipeline"
+        body:  >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "set" : {
+                  "field": "my_time",
+                  "value": "{{ _ingest.timestamp }}"
+                }
+              },
+              {
+                "grok" : {
+                  "field" : "my_time",
+                  "patterns": ["%{DAY:day} %{MONTH:month} %{MONTHDAY:monthday} %{TIME:time} %{WORD} %{YEAR:year}"]
+                }
+              },
+              {
+                "set": {
+                  "field": "my_time",
+                  "value": "{{day}} {{month}} {{monthday}} {{time}} {{year}}"
+                }
+              },
+              {
+                "date" : {
+                  "field" : "my_time",
+                  "target_field": "my_time",
+                  "formats": ["EEE MMM dd HH:mm:ss yyyy"]
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: timetest
+        type: test
+        id: 1
+        pipeline: "my_timely_pipeline"
+        body: {}
+
+---
+"Test timestamp templating matches date-mapping defaults with ingest.new_date_format":
+  - skip:
+      version: " - 5.3.99"
+      reason: deprecated in 5.4.0
+      features: "warnings"
+
+  - do:
+      cluster.health:
+          wait_for_status: green
+
+  - do:
+      indices.create:
+        index: timetest_newdateformat
+        body:
+          mappings:
+            test: { "properties": { "my_time": {"type": "date"}}}
+
+  - do:
+      cluster.put_settings:
+        body:
+          transient:
+            ingest.new_date_format: true
+      warnings:
+        - "[ingest.new_date_format] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version."
+        - "[script.max_compilations_per_minute] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version."
+
+  - match: {transient: {ingest: {new_date_format: "true"}}}
+
+  - do:
+      ingest.put_pipeline:
+        id: "my_timely_pipeline_with_new_date_format"
+        body:  >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "set" : {
+                  "field": "my_time",
+                  "value": "{{ _ingest.timestamp }}"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: timetest
+        type: test
+        id: 1
+        pipeline: "my_timely_pipeline_with_new_date_format"
+        body: {}
+
+  - do:
+      cluster.put_settings:
+        body:
+          transient:
+            ingest.new_date_format: false
+      warnings:
+        - "[ingest.new_date_format] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version."
+        - "[script.max_compilations_per_minute] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version."
+
+  - match: {transient: {ingest: {new_date_format: "false"}}}
diff --git a/qa/smoke-test-multinode/build.gradle b/qa/smoke-test-multinode/build.gradle
index a30f1d31f738f..50bdd0df2265b 100644
--- a/qa/smoke-test-multinode/build.gradle
+++ b/qa/smoke-test-multinode/build.gradle
@@ -1,8 +1,25 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
 
+apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
-integTest {
-  cluster {
-    numNodes = 2
-  }
+integTestCluster {
+  numNodes = 2
 }
diff --git a/qa/smoke-test-multinode/src/test/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java b/qa/smoke-test-multinode/src/test/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java
index 456387e6c1943..ed33c5b05f3df 100644
--- a/qa/smoke-test-multinode/src/test/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java
+++ b/qa/smoke-test-multinode/src/test/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java
@@ -22,12 +22,12 @@
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
+import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
+import org.apache.lucene.util.TimeUnits;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-
-import java.io.IOException;
 
+@TimeoutSuite(millis = 40 * TimeUnits.MINUTE) // some of the windows test VMs are slow as hell
 public class SmokeTestMultiNodeClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
     public SmokeTestMultiNodeClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
@@ -35,8 +35,8 @@ public SmokeTestMultiNodeClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandi
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
-        return ESClientYamlSuiteTestCase.createParameters(0, 1);
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
     }
 }
 
diff --git a/qa/smoke-test-plugins/build.gradle b/qa/smoke-test-plugins/build.gradle
index ab69b02fc8c13..d60216dad194f 100644
--- a/qa/smoke-test-plugins/build.gradle
+++ b/qa/smoke-test-plugins/build.gradle
@@ -19,14 +19,13 @@
 
 import org.elasticsearch.gradle.MavenFilteringHack
 
+apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 ext.pluginsCount = 0
 project.rootProject.subprojects.findAll { it.parent.path == ':plugins' }.each { subproj ->
-  integTest {
-    cluster {
-      plugin subproj.path
-    }
+  integTestCluster {
+    plugin subproj.path
   }
   pluginsCount += 1
 }
@@ -40,4 +39,3 @@ processTestResources {
   inputs.properties(expansions)
   MavenFilteringHack.filter(it, expansions)
 }
-
diff --git a/qa/smoke-test-plugins/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsClientYamlTestSuiteIT.java b/qa/smoke-test-plugins/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsClientYamlTestSuiteIT.java
index 05021f3c2bbb6..d1f9e6b73703e 100644
--- a/qa/smoke-test-plugins/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsClientYamlTestSuiteIT.java
+++ b/qa/smoke-test-plugins/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsClientYamlTestSuiteIT.java
@@ -24,9 +24,6 @@
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-
-import java.io.IOException;
 
 public class SmokeTestPluginsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
@@ -35,8 +32,8 @@ public SmokeTestPluginsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandida
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
-        return ESClientYamlSuiteTestCase.createParameters(0, 1);
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
     }
 }
 
diff --git a/qa/smoke-test-reindex-with-all-modules/build.gradle b/qa/smoke-test-reindex-with-all-modules/build.gradle
new file mode 100644
index 0000000000000..cab01cb9412ec
--- /dev/null
+++ b/qa/smoke-test-reindex-with-all-modules/build.gradle
@@ -0,0 +1,27 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+apply plugin: 'elasticsearch.standalone-rest-test'
+apply plugin: 'elasticsearch.rest-test'
+
+integTestCluster {
+  setting 'script.max_compilations_per_minute', '1000'
+  // Whitelist reindexing from the local node so we can test it.
+  setting 'reindex.remote.whitelist', '127.0.0.1:*'
+}
\ No newline at end of file
diff --git a/qa/smoke-test-reindex-with-painless/src/test/java/org/elasticsearch/smoketest/SmokeTestReindexWithPainlessClientYamlTestSuiteIT.java b/qa/smoke-test-reindex-with-all-modules/src/test/java/org/elasticsearch/smoketest/SmokeTestReindexWithPainlessClientYamlTestSuiteIT.java
similarity index 83%
rename from qa/smoke-test-reindex-with-painless/src/test/java/org/elasticsearch/smoketest/SmokeTestReindexWithPainlessClientYamlTestSuiteIT.java
rename to qa/smoke-test-reindex-with-all-modules/src/test/java/org/elasticsearch/smoketest/SmokeTestReindexWithPainlessClientYamlTestSuiteIT.java
index db01ce2dfe9ab..db1e62a6b15e6 100644
--- a/qa/smoke-test-reindex-with-painless/src/test/java/org/elasticsearch/smoketest/SmokeTestReindexWithPainlessClientYamlTestSuiteIT.java
+++ b/qa/smoke-test-reindex-with-all-modules/src/test/java/org/elasticsearch/smoketest/SmokeTestReindexWithPainlessClientYamlTestSuiteIT.java
@@ -24,9 +24,6 @@
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-
-import java.io.IOException;
 
 public class SmokeTestReindexWithPainlessClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     public SmokeTestReindexWithPainlessClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
@@ -34,7 +31,7 @@ public SmokeTestReindexWithPainlessClientYamlTestSuiteIT(@Name("yaml") ClientYam
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
-        return ESClientYamlSuiteTestCase.createParameters(0, 1);
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/reindex/10_script.yaml b/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/reindex/10_script.yaml
new file mode 100644
index 0000000000000..c0c50351e388a
--- /dev/null
+++ b/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/reindex/10_script.yaml
@@ -0,0 +1,542 @@
+---
+"Modify a document":
+  - do:
+      index:
+        index:  twitter
+        type:   tweet
+        id:     1
+        body:   { "user": "kimchy" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      reindex:
+        refresh: true
+        body:
+          source:
+            index: twitter
+          dest:
+            index: new_twitter
+          script:
+            lang: painless
+            source: ctx._source.user = "other" + ctx._source.user
+  - match: {created: 1}
+  - match: {noops: 0}
+
+  - do:
+      search:
+        index: new_twitter
+        body:
+          query:
+            match:
+              user: otherkimchy
+  - match: { hits.total: 1 }
+
+---
+"Modify a document based on id":
+  - do:
+      index:
+        index:  twitter
+        type:   tweet
+        id:     1
+        body:   { "user": "kimchy" }
+  - do:
+      index:
+        index:  twitter
+        type:   tweet
+        id:     2
+        body:   { "user": "blort" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      reindex:
+        refresh: true
+        body:
+          source:
+            index: twitter
+          dest:
+            index: new_twitter
+          script:
+            lang: painless
+            source: if (ctx._id == "1") {ctx._source.user = "other" + ctx._source.user}
+  - match: {created: 2}
+  - match: {noops: 0}
+
+  - do:
+      search:
+        index: new_twitter
+        body:
+          query:
+            match:
+              user: otherkimchy
+  - match: { hits.total: 1 }
+
+  - do:
+      search:
+        index: new_twitter
+        body:
+          query:
+            match:
+              user: blort
+  - match: { hits.total: 1 }
+
+---
+"Add new parent":
+  - do:
+      indices.create:
+          index: new_twitter
+          body:
+            mappings:
+              tweet:
+                _parent: { type: "user" }
+
+  - do:
+      index:
+        index:  twitter
+        type:   tweet
+        id:     1
+        body:   { "user": "kimchy" }
+  - do:
+      index:
+        index:  new_twitter
+        type:   user
+        id:     kimchy
+        body:   { "name": "kimchy" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      reindex:
+        refresh: true
+        body:
+          source:
+            index: twitter
+          dest:
+            index: new_twitter
+          script:
+            lang: painless
+            source: ctx._parent = ctx._source.user
+  - match: {created: 1}
+  - match: {noops: 0}
+
+  - do:
+      search:
+        index: new_twitter
+        body:
+          query:
+            has_parent:
+              parent_type: user
+              query:
+                match:
+                  name: kimchy
+  - match: { hits.total: 1 }
+  - match: { hits.hits.0._source.user: kimchy }
+
+---
+"Add routing":
+  - do:
+      index:
+        index:  twitter
+        type:   tweet
+        id:     1
+        body:   { "user": "kimchy" }
+  - do:
+      index:
+        index:  twitter
+        type:   tweet
+        id:     2
+        body:   { "user": "foo" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      reindex:
+        refresh: true
+        body:
+          source:
+            index: twitter
+          dest:
+            index: new_twitter
+          script:
+            lang: painless
+            source: ctx._routing = ctx._source.user
+  - match: {created: 2}
+  - match: {noops: 0}
+
+  - do:
+      get:
+        index:   new_twitter
+        type:    tweet
+        id:      1
+        routing: kimchy
+  - match: { _routing: kimchy }
+
+  - do:
+      get:
+        index:   new_twitter
+        type:    tweet
+        id:      2
+        routing: foo
+  - match: { _routing: foo }
+
+---
+"Add routing and parent":
+  - do:
+      indices.create:
+          index: new_twitter
+          body:
+            mappings:
+              tweet:
+                _parent: { type: "user" }
+
+  - do:
+      index:
+        index:   twitter
+        type:    tweet
+        id:      1
+        body:    { "user": "kimchy" }
+  - do:
+      index:
+        index:  new_twitter
+        type:   user
+        id:     kimchy
+        body:   { "name": "kimchy" }
+        routing: cat
+  - do:
+      indices.refresh: {}
+
+  - do:
+      reindex:
+        refresh: true
+        body:
+          source:
+            index: twitter
+          dest:
+            index: new_twitter
+          script:
+            lang: painless
+            source: ctx._parent = ctx._source.user; ctx._routing = "cat"
+  - match: {created: 1}
+  - match: {noops: 0}
+
+  - do:
+      search:
+        index:   new_twitter
+        routing: cat
+        body:
+          query:
+            has_parent:
+              parent_type: user
+              query:
+                match:
+                  name: kimchy
+  - match: { hits.total: 1 }
+  - match: { hits.hits.0._source.user: kimchy }
+  - match: { hits.hits.0._routing: cat }
+
+---
+"Noop one doc":
+  - do:
+      index:
+        index:  twitter
+        type:   tweet
+        id:     1
+        body:   { "user": "kimchy" }
+  - do:
+      index:
+        index:  twitter
+        type:   tweet
+        id:     2
+        body:   { "user": "foo" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      reindex:
+        refresh: true
+        body:
+          source:
+            index: twitter
+          dest:
+            index: new_twitter
+          script:
+            lang: painless
+            source: if (ctx._source.user == "kimchy") {ctx._source.user = "not" + ctx._source.user} else {ctx.op = "noop"}
+  - match: {created: 1}
+  - match: {noops: 1}
+
+  - do:
+      search:
+        index: new_twitter
+        body:
+          query:
+            match:
+              user: notkimchy
+  - match: { hits.total: 1 }
+
+  - do:
+      search:
+        index: twitter
+        body:
+          query:
+            match:
+              user: notfoo
+  - match: { hits.total: 0 }
+
+  - do:
+      count:
+        index: new_twitter
+  - match: {count: 1}
+
+---
+"Noop all docs":
+  - do:
+      index:
+        index:  twitter
+        type:   tweet
+        id:     1
+        body:   { "user": "kimchy" }
+  - do:
+      index:
+        index:  twitter
+        type:   tweet
+        id:     2
+        body:   { "user": "foo" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      reindex:
+        body:
+          source:
+            index: twitter
+          dest:
+            index: new_twitter
+          script:
+            lang: painless
+            source: ctx.op = "noop"
+  - match: {updated: 0}
+  - match: {noops: 2}
+
+  - do:
+      indices.exists:
+        index: new_twitter
+  - is_false: ''
+
+---
+"Set version to null to force an update":
+  - do:
+      index:
+        index:        twitter
+        type:         tweet
+        id:           1
+        version:      1
+        version_type: external
+        body:         { "user": "kimchy" }
+  - do:
+      index:
+        index:        new_twitter
+        type:         tweet
+        id:           1
+        version:      1
+        version_type: external
+        body:         { "user": "kimchy" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      reindex:
+        refresh: true
+        body:
+          source:
+            index: twitter
+          dest:
+            index: new_twitter
+            version_type: external
+          script:
+            lang: painless
+            source: ctx._source.user = "other" + ctx._source.user; ctx._version = null
+  - match: {updated: 1}
+  - match: {noops: 0}
+
+  - do:
+      search:
+        index: new_twitter
+        body:
+          query:
+            match:
+              user: otherkimchy
+  - match: { hits.total: 1 }
+
+---
+"Set id to null to get an automatic id":
+  - do:
+      index:
+        index:        twitter
+        type:         tweet
+        id:           1
+        body:         { "user": "kimchy" }
+  - do:
+      index:
+        index:        new_twitter
+        type:         tweet
+        id:           1
+        body:         { "user": "kimchy" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      reindex:
+        refresh: true
+        body:
+          source:
+            index: twitter
+          dest:
+            index: new_twitter
+          script:
+            lang: painless
+            source: ctx._source.user = "other" + ctx._source.user; ctx._id = null
+  - match: {created: 1}
+  - match: {noops: 0}
+
+  - do:
+      search:
+        index: new_twitter
+        body:
+          query:
+            match:
+              user: otherkimchy
+  - match: { hits.total: 1 }
+
+---
+"Change index to write to a different index":
+  - do:
+      index:
+        index:        twitter
+        type:         tweet
+        id:           1
+        body:         { "user": "kimchy" }
+  - do:
+      index:
+        index:        twitter
+        type:         tweet
+        id:           2
+        body:         { "user": "another" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      reindex:
+        refresh: true
+        body:
+          source:
+            index: twitter
+          dest:
+            index: new_twitter
+          script:
+            lang: painless
+            source: if (ctx._source.user == "kimchy") {ctx._index = 'other_new_twitter'}
+  - match: {created: 2}
+  - match: {noops: 0}
+
+  - do:
+      search:
+        index: other_new_twitter
+        body:
+          query:
+            match:
+              user: kimchy
+  - match: { hits.total: 1 }
+
+  - do:
+      search:
+        index: new_twitter
+        body:
+          query:
+            match:
+              user: another
+  - match: { hits.total: 1 }
+
+---
+"Reindex all docs with one doc deletion":
+  # Source index
+  - do:
+      index:
+        index:  index1
+        type:   type1
+        id:     1
+        body:   { "lang": "en", "id": 123 }
+  - do:
+      index:
+        index:  index1
+        type:   type1
+        id:     2
+        body:   { "lang": "en", "id": 456 }
+  - do:
+      index:
+        index:  index1
+        type:   type1
+        id:     3
+        body:   { "lang": "fr", "id": 789 }
+  # Destination index
+  - do:
+      index:
+        index:  index2
+        type:   type2
+        id:     fr_789
+        body:   { "lang": "fr", "id": 789 }
+  - do:
+      index:
+        index:  index2
+        type:   type2
+        id:     en_123
+        body:   { "lang": "en", "id": 123 }
+  - do:
+      indices.refresh: {}
+
+  # Reindex all documents from "index1" into "index2", changing their type
+  # to "type2" and their id to the concatened lang+id fields,
+  # trashing all non-english pre existing ones
+  - do:
+      reindex:
+        refresh: true
+        body:
+          source:
+            index: index1
+          dest:
+            index: index2
+            type: type2
+          script:
+            lang: painless
+            source: "ctx._id = ctx._source.lang + '_' + ctx._source.id;
+                     if (ctx._source.lang != \"en\" ) {ctx.op = 'delete'}"
+  - match: {created: 1}
+  - match: {noops: 0}
+  - match: {updated: 1}
+  - match: {deleted: 1}
+
+  - do:
+      mget:
+        body:
+          docs:
+            - { _index: index2, _type: type2, _id: en_123}
+            - { _index: index2, _type: type2, _id: en_456}
+            - { _index: index2, _type: type2, _id: fr_789}
+
+  - is_true: docs.0.found
+  - match:  { docs.0._index:     index2      }
+  - match:  { docs.0._type:      type2       }
+  - match:  { docs.0._id:        en_123      }
+  - match:  { docs.0._version:   2           }
+
+  - is_true: docs.1.found
+  - match:  { docs.1._index:     index2      }
+  - match:  { docs.1._type:      type2       }
+  - match:  { docs.1._id:        en_456      }
+  - match:  { docs.1._version:   1           }
+
+  - is_false: docs.2.found
+  - match:  { docs.2._index:     index2      }
+  - match:  { docs.2._type:      type2       }
+  - match:  { docs.2._id:        fr_789      }
diff --git a/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/reindex/20_broken.yaml b/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/reindex/20_broken.yaml
new file mode 100644
index 0000000000000..fa2c0124ea2a0
--- /dev/null
+++ b/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/reindex/20_broken.yaml
@@ -0,0 +1,24 @@
+---
+"Totally broken scripts report the error properly":
+  - do:
+      index:
+        index:  twitter
+        type:   tweet
+        id:     1
+        body:   { "user": "kimchy" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      catch: request
+      reindex:
+        refresh: true
+        body:
+          source:
+            index: twitter
+          dest:
+            index: new_twitter
+          script:
+            lang: painless
+            source: syntax errors are fun!
+  - match: {error.reason: 'compile error'}
diff --git a/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/reindex/30_timeout.yaml b/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/reindex/30_timeout.yaml
similarity index 100%
rename from qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/reindex/30_timeout.yaml
rename to qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/reindex/30_timeout.yaml
diff --git a/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/reindex/40_search_failures.yaml b/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/reindex/40_search_failures.yaml
similarity index 93%
rename from qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/reindex/40_search_failures.yaml
rename to qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/reindex/40_search_failures.yaml
index a982609e3a1cc..9cf022f2de04a 100644
--- a/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/reindex/40_search_failures.yaml
+++ b/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/reindex/40_search_failures.yaml
@@ -19,7 +19,7 @@
               script:
                 script:
                   lang: painless
-                  inline: throw new IllegalArgumentException("Cats!")
+                  source: throw new IllegalArgumentException("Cats!")
           dest:
             index:   dest
   - match: {created: 0}
diff --git a/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/reindex/50_reindex_with_parentchild.yaml b/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/reindex/50_reindex_with_parentchild.yaml
new file mode 100644
index 0000000000000..81e142c9195c1
--- /dev/null
+++ b/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/reindex/50_reindex_with_parentchild.yaml
@@ -0,0 +1,79 @@
+---
+"Reindex from remote with parent/child":
+  - do:
+      indices.create:
+          index: source
+          body:
+              settings:
+                mapping.single_type: false
+              mappings:
+                foo: {}
+                bar:
+                  _parent:
+                    type:  foo
+  - do:
+      indices.create:
+          index: dest
+          body:
+              settings:
+                mapping.single_type: false
+              mappings:
+                foo: {}
+                bar:
+                  _parent:
+                    type:  foo
+  - do:
+      index:
+        index:   source
+        type:    foo
+        id:      1
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   source
+        type:    bar
+        id:      1
+        parent:  1
+        body:    { "text": "test2" }
+  - do:
+      indices.refresh: {}
+
+  # Fetch the http host. We use the host of the master because we know there will always be a master.
+  - do:
+      cluster.state: {}
+  - set: { master_node: master }
+  - do:
+      nodes.info:
+        metric: [ http ]
+  - is_true: nodes.$master.http.publish_address
+  - set: {nodes.$master.http.publish_address: host}
+  - do:
+      reindex:
+        refresh: true
+        body:
+          source:
+            remote:
+              host: http://${host}
+            index: source
+          dest:
+            index: dest
+  - match: {created: 2}
+
+  - do:
+      search:
+        index: dest
+        body:
+          query:
+            has_parent:
+              parent_type: foo
+              query:
+                match:
+                  text: test
+  - match: {hits.total: 1}
+
+  # Make sure reindex closed all the scroll contexts
+  - do:
+      indices.stats:
+        index: source
+        metric: search
+  - match: {indices.source.total.search.open_contexts: 0}
diff --git a/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/update_by_query/10_script.yaml b/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/update_by_query/10_script.yaml
new file mode 100644
index 0000000000000..c4414229d7efa
--- /dev/null
+++ b/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/update_by_query/10_script.yaml
@@ -0,0 +1,314 @@
+---
+"Update a document using update_by_query":
+  - do:
+      index:
+        index:  twitter
+        type:   tweet
+        id:     1
+        body:   { "user": "kimchy" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      update_by_query:
+        index:   twitter
+        refresh: true
+        body:
+          script:
+            lang: painless
+            source: ctx._source.user = "not" + ctx._source.user
+  - match: {updated: 1}
+  - match: {noops: 0}
+
+  - do:
+      search:
+        index: twitter
+        body:
+          query:
+            match:
+              user: notkimchy
+  - match: { hits.total: 1 }
+
+---
+"Noop one doc":
+  - do:
+      index:
+        index:  twitter
+        type:   tweet
+        id:     1
+        body:   { "user": "kimchy" }
+  - do:
+      index:
+        index:  twitter
+        type:   tweet
+        id:     2
+        body:   { "user": "foo" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      update_by_query:
+        refresh: true
+        index:   twitter
+        body:
+          script:
+            lang: painless
+            source: if (ctx._source.user == "kimchy") {ctx._source.user = "not" + ctx._source.user} else {ctx.op = "noop"}
+  - match: {updated: 1}
+  - match: {noops: 1}
+
+  - do:
+      search:
+        index: twitter
+        body:
+          query:
+            match:
+              user: notkimchy
+  - match: { hits.total: 1 }
+
+  - do:
+      search:
+        index: twitter
+        body:
+          query:
+            match:
+              user: notfoo
+  - match: { hits.total: 0 }
+
+---
+"Noop all docs":
+  - do:
+      index:
+        index:  twitter
+        type:   tweet
+        id:     1
+        body:   { "user": "kimchy" }
+  - do:
+      index:
+        index:  twitter
+        type:   tweet
+        id:     2
+        body:   { "user": "foo" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      update_by_query:
+        refresh: true
+        index:   twitter
+        body:
+          script:
+            lang: painless
+            source: ctx.op = "noop"
+  - match: {updated: 0}
+  - match: {noops: 2}
+  - match: {batches: 1}
+
+---
+"Setting bogus context is an error":
+  - do:
+      index:
+        index:  twitter
+        type:   tweet
+        id:     1
+        body:   { "user": "kimchy" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      catch: /Invalid fields added to context \[junk\]/
+      update_by_query:
+        index: twitter
+        body:
+          script:
+            lang: painless
+            source: ctx.junk = "stuff"
+
+---
+"Can't change _id":
+  - do:
+      index:
+        index:  twitter
+        type:   tweet
+        id:     1
+        body:   { "user": "kimchy" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      catch: /Modifying \[_id\] not allowed/
+      update_by_query:
+        index: twitter
+        body:
+          script:
+            lang: painless
+            source: ctx._id = "stuff"
+
+---
+"Update all docs with one doc deletion":
+  - do:
+      index:
+        index:  twitter
+        type:   tweet
+        id:     1
+        body:   { "level": 9, "last_updated": "2016-01-01T12:10:30Z" }
+  - do:
+      index:
+        index:  twitter
+        type:   tweet
+        id:     2
+        body:   { "level": 10, "last_updated": "2016-01-01T12:10:30Z" }
+  - do:
+      index:
+        index:  twitter
+        type:   tweet
+        id:     3
+        body:   { "level": 11, "last_updated": "2016-01-01T12:10:30Z" }
+  - do:
+      index:
+        index:  twitter
+        type:   tweet
+        id:     4
+        body:   { "level": 12, "last_updated": "2016-01-01T12:10:30Z" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      update_by_query:
+        refresh: true
+        index:   twitter
+        body:
+          script:
+            lang: painless
+            source: if (ctx._source.level != 11) {ctx._source.last_updated = "2016-01-02T00:00:00Z"} else {ctx.op = "delete"}
+  - match: {updated: 3}
+  - match: {deleted: 1}
+  - match: {noops: 0}
+
+  - do:
+      search:
+        index: twitter
+        body:
+          query:
+            match:
+              last_updated: "2016-01-02T00:00:00Z"
+  - match: { hits.total: 3 }
+
+  - do:
+      search:
+        index: twitter
+        body:
+          query:
+            term:
+              level: 11
+  - match: { hits.total: 0 }
+
+  - do:
+      count:
+        index: twitter
+  - match: {count: 3}
+
+---
+"Update all docs with one deletion and one noop":
+  - do:
+      index:
+        index:  twitter
+        type:   tweet
+        id:     1
+        body:   { "level": 9, "last_updated": "2016-01-01T12:10:30Z" }
+  - do:
+      index:
+        index:  twitter
+        type:   tweet
+        id:     2
+        body:   { "level": 10, "last_updated": "2016-01-01T12:10:30Z" }
+  - do:
+      index:
+        index:  twitter
+        type:   tweet
+        id:     3
+        body:   { "level": 11, "last_updated": "2016-01-01T12:10:30Z" }
+  - do:
+      index:
+        index:  twitter
+        type:   tweet
+        id:     4
+        body:   { "level": 12, "last_updated": "2016-01-01T12:10:30Z" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      update_by_query:
+        refresh: true
+        index:   twitter
+        body:
+          script:
+            lang: painless
+            source: |
+              int choice = ctx._source.level % 3;
+              if (choice == 0) {
+                ctx._source.last_updated = "2016-01-02T00:00:00Z";
+              } else if (choice == 1) {
+                ctx.op = "noop";
+              } else {
+                ctx.op = "delete";
+              }
+  - match: {updated: 2}
+  - match: {deleted: 1}
+  - match: {noops: 1}
+
+  - do:
+      search:
+        index: twitter
+        body:
+          query:
+            match:
+              last_updated: "2016-01-02T00:00:00Z"
+  - match: { hits.total: 2 }
+
+  - do:
+      search:
+        index: twitter
+        body:
+          query:
+            match:
+              last_updated: "2016-01-01T12:10:30Z"
+  - match: { hits.total: 1 }
+
+  - do:
+      search:
+        index: twitter
+        body:
+          query:
+            term:
+              level: 11
+  - match: { hits.total: 0 }
+
+---
+"Set unsupported operation type":
+  - do:
+      index:
+        index:  twitter
+        type:   tweet
+        id:     1
+        body:   { "user": "kimchy" }
+  - do:
+      index:
+        index:  twitter
+        type:   tweet
+        id:     2
+        body:   { "user": "foo" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      catch: request
+      update_by_query:
+        refresh: true
+        index:   twitter
+        body:
+          script:
+            lang: painless
+            source: if (ctx._source.user == "kimchy") {ctx.op = "update"} else {ctx.op = "junk"}
+
+  - match: { error.reason: 'Operation type [junk] not allowed, only [noop, index, delete] are allowed' }
diff --git a/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/update_by_query/20_broken.yaml b/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/update_by_query/20_broken.yaml
new file mode 100644
index 0000000000000..8863db5d2aef7
--- /dev/null
+++ b/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/update_by_query/20_broken.yaml
@@ -0,0 +1,21 @@
+---
+"Totally broken scripts report the error properly":
+  - do:
+      index:
+        index:  twitter
+        type:   tweet
+        id:     1
+        body:   { "user": "kimchy" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      catch: request
+      update_by_query:
+        index:   twitter
+        refresh: true
+        body:
+          script:
+            lang: painless
+            source: syntax errors are fun!
+  - match: {error.reason: 'compile error'}
diff --git a/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/update_by_query/30_timeout.yaml b/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/update_by_query/30_timeout.yaml
similarity index 100%
rename from qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/update_by_query/30_timeout.yaml
rename to qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/update_by_query/30_timeout.yaml
diff --git a/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/update_by_query/40_search_failure.yaml b/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/update_by_query/40_search_failure.yaml
similarity index 92%
rename from qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/update_by_query/40_search_failure.yaml
rename to qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/update_by_query/40_search_failure.yaml
index 9960e2cd8ccaa..12801dd3e2ccc 100644
--- a/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/update_by_query/40_search_failure.yaml
+++ b/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/update_by_query/40_search_failure.yaml
@@ -18,7 +18,7 @@
             script:
               script:
                 lang: painless
-                inline: throw new IllegalArgumentException("Cats!")
+                source: throw new IllegalArgumentException("Cats!")
   - match: {updated: 0}
   - match: {version_conflicts: 0}
   - match: {batches: 0}
diff --git a/qa/smoke-test-reindex-with-painless/build.gradle b/qa/smoke-test-reindex-with-painless/build.gradle
deleted file mode 100644
index c857db85bfa49..0000000000000
--- a/qa/smoke-test-reindex-with-painless/build.gradle
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-apply plugin: 'elasticsearch.rest-test'
-
-integTest {
-  cluster {
-    setting 'script.max_compilations_per_minute', '1000'
-  }
-}
diff --git a/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/reindex/10_script.yaml b/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/reindex/10_script.yaml
deleted file mode 100644
index a56eb036c6d0f..0000000000000
--- a/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/reindex/10_script.yaml
+++ /dev/null
@@ -1,542 +0,0 @@
----
-"Modify a document":
-  - do:
-      index:
-        index:  twitter
-        type:   tweet
-        id:     1
-        body:   { "user": "kimchy" }
-  - do:
-      indices.refresh: {}
-
-  - do:
-      reindex:
-        refresh: true
-        body:
-          source:
-            index: twitter
-          dest:
-            index: new_twitter
-          script:
-            lang: painless
-            inline: ctx._source.user = "other" + ctx._source.user
-  - match: {created: 1}
-  - match: {noops: 0}
-
-  - do:
-      search:
-        index: new_twitter
-        body:
-          query:
-            match:
-              user: otherkimchy
-  - match: { hits.total: 1 }
-
----
-"Modify a document based on id":
-  - do:
-      index:
-        index:  twitter
-        type:   tweet
-        id:     1
-        body:   { "user": "kimchy" }
-  - do:
-      index:
-        index:  twitter
-        type:   tweet
-        id:     2
-        body:   { "user": "blort" }
-  - do:
-      indices.refresh: {}
-
-  - do:
-      reindex:
-        refresh: true
-        body:
-          source:
-            index: twitter
-          dest:
-            index: new_twitter
-          script:
-            lang: painless
-            inline: if (ctx._id == "1") {ctx._source.user = "other" + ctx._source.user}
-  - match: {created: 2}
-  - match: {noops: 0}
-
-  - do:
-      search:
-        index: new_twitter
-        body:
-          query:
-            match:
-              user: otherkimchy
-  - match: { hits.total: 1 }
-
-  - do:
-      search:
-        index: new_twitter
-        body:
-          query:
-            match:
-              user: blort
-  - match: { hits.total: 1 }
-
----
-"Add new parent":
-  - do:
-      indices.create:
-          index: new_twitter
-          body:
-            mappings:
-              tweet:
-                _parent: { type: "user" }
-
-  - do:
-      index:
-        index:  twitter
-        type:   tweet
-        id:     1
-        body:   { "user": "kimchy" }
-  - do:
-      index:
-        index:  new_twitter
-        type:   user
-        id:     kimchy
-        body:   { "name": "kimchy" }
-  - do:
-      indices.refresh: {}
-
-  - do:
-      reindex:
-        refresh: true
-        body:
-          source:
-            index: twitter
-          dest:
-            index: new_twitter
-          script:
-            lang: painless
-            inline: ctx._parent = ctx._source.user
-  - match: {created: 1}
-  - match: {noops: 0}
-
-  - do:
-      search:
-        index: new_twitter
-        body:
-          query:
-            has_parent:
-              parent_type: user
-              query:
-                match:
-                  name: kimchy
-  - match: { hits.total: 1 }
-  - match: { hits.hits.0._source.user: kimchy }
-
----
-"Add routing":
-  - do:
-      index:
-        index:  twitter
-        type:   tweet
-        id:     1
-        body:   { "user": "kimchy" }
-  - do:
-      index:
-        index:  twitter
-        type:   tweet
-        id:     2
-        body:   { "user": "foo" }
-  - do:
-      indices.refresh: {}
-
-  - do:
-      reindex:
-        refresh: true
-        body:
-          source:
-            index: twitter
-          dest:
-            index: new_twitter
-          script:
-            lang: painless
-            inline: ctx._routing = ctx._source.user
-  - match: {created: 2}
-  - match: {noops: 0}
-
-  - do:
-      get:
-        index:   new_twitter
-        type:    tweet
-        id:      1
-        routing: kimchy
-  - match: { _routing: kimchy }
-
-  - do:
-      get:
-        index:   new_twitter
-        type:    tweet
-        id:      2
-        routing: foo
-  - match: { _routing: foo }
-
----
-"Add routing and parent":
-  - do:
-      indices.create:
-          index: new_twitter
-          body:
-            mappings:
-              tweet:
-                _parent: { type: "user" }
-
-  - do:
-      index:
-        index:   twitter
-        type:    tweet
-        id:      1
-        body:    { "user": "kimchy" }
-  - do:
-      index:
-        index:  new_twitter
-        type:   user
-        id:     kimchy
-        body:   { "name": "kimchy" }
-        routing: cat
-  - do:
-      indices.refresh: {}
-
-  - do:
-      reindex:
-        refresh: true
-        body:
-          source:
-            index: twitter
-          dest:
-            index: new_twitter
-          script:
-            lang: painless
-            inline: ctx._parent = ctx._source.user; ctx._routing = "cat"
-  - match: {created: 1}
-  - match: {noops: 0}
-
-  - do:
-      search:
-        index:   new_twitter
-        routing: cat
-        body:
-          query:
-            has_parent:
-              parent_type: user
-              query:
-                match:
-                  name: kimchy
-  - match: { hits.total: 1 }
-  - match: { hits.hits.0._source.user: kimchy }
-  - match: { hits.hits.0._routing: cat }
-
----
-"Noop one doc":
-  - do:
-      index:
-        index:  twitter
-        type:   tweet
-        id:     1
-        body:   { "user": "kimchy" }
-  - do:
-      index:
-        index:  twitter
-        type:   tweet
-        id:     2
-        body:   { "user": "foo" }
-  - do:
-      indices.refresh: {}
-
-  - do:
-      reindex:
-        refresh: true
-        body:
-          source:
-            index: twitter
-          dest:
-            index: new_twitter
-          script:
-            lang: painless
-            inline: if (ctx._source.user == "kimchy") {ctx._source.user = "not" + ctx._source.user} else {ctx.op = "noop"}
-  - match: {created: 1}
-  - match: {noops: 1}
-
-  - do:
-      search:
-        index: new_twitter
-        body:
-          query:
-            match:
-              user: notkimchy
-  - match: { hits.total: 1 }
-
-  - do:
-      search:
-        index: twitter
-        body:
-          query:
-            match:
-              user: notfoo
-  - match: { hits.total: 0 }
-
-  - do:
-      count:
-        index: new_twitter
-  - match: {count: 1}
-
----
-"Noop all docs":
-  - do:
-      index:
-        index:  twitter
-        type:   tweet
-        id:     1
-        body:   { "user": "kimchy" }
-  - do:
-      index:
-        index:  twitter
-        type:   tweet
-        id:     2
-        body:   { "user": "foo" }
-  - do:
-      indices.refresh: {}
-
-  - do:
-      reindex:
-        body:
-          source:
-            index: twitter
-          dest:
-            index: new_twitter
-          script:
-            lang: painless
-            inline: ctx.op = "noop"
-  - match: {updated: 0}
-  - match: {noops: 2}
-
-  - do:
-      indices.exists:
-        index: new_twitter
-  - is_false: ''
-
----
-"Set version to null to force an update":
-  - do:
-      index:
-        index:        twitter
-        type:         tweet
-        id:           1
-        version:      1
-        version_type: external
-        body:         { "user": "kimchy" }
-  - do:
-      index:
-        index:        new_twitter
-        type:         tweet
-        id:           1
-        version:      1
-        version_type: external
-        body:         { "user": "kimchy" }
-  - do:
-      indices.refresh: {}
-
-  - do:
-      reindex:
-        refresh: true
-        body:
-          source:
-            index: twitter
-          dest:
-            index: new_twitter
-            version_type: external
-          script:
-            lang: painless
-            inline: ctx._source.user = "other" + ctx._source.user; ctx._version = null
-  - match: {updated: 1}
-  - match: {noops: 0}
-
-  - do:
-      search:
-        index: new_twitter
-        body:
-          query:
-            match:
-              user: otherkimchy
-  - match: { hits.total: 1 }
-
----
-"Set id to null to get an automatic id":
-  - do:
-      index:
-        index:        twitter
-        type:         tweet
-        id:           1
-        body:         { "user": "kimchy" }
-  - do:
-      index:
-        index:        new_twitter
-        type:         tweet
-        id:           1
-        body:         { "user": "kimchy" }
-  - do:
-      indices.refresh: {}
-
-  - do:
-      reindex:
-        refresh: true
-        body:
-          source:
-            index: twitter
-          dest:
-            index: new_twitter
-          script:
-            lang: painless
-            inline: ctx._source.user = "other" + ctx._source.user; ctx._id = null
-  - match: {created: 1}
-  - match: {noops: 0}
-
-  - do:
-      search:
-        index: new_twitter
-        body:
-          query:
-            match:
-              user: otherkimchy
-  - match: { hits.total: 1 }
-
----
-"Change index to write to a different index":
-  - do:
-      index:
-        index:        twitter
-        type:         tweet
-        id:           1
-        body:         { "user": "kimchy" }
-  - do:
-      index:
-        index:        twitter
-        type:         tweet
-        id:           2
-        body:         { "user": "another" }
-  - do:
-      indices.refresh: {}
-
-  - do:
-      reindex:
-        refresh: true
-        body:
-          source:
-            index: twitter
-          dest:
-            index: new_twitter
-          script:
-            lang: painless
-            inline: if (ctx._source.user == "kimchy") {ctx._index = 'other_new_twitter'}
-  - match: {created: 2}
-  - match: {noops: 0}
-
-  - do:
-      search:
-        index: other_new_twitter
-        body:
-          query:
-            match:
-              user: kimchy
-  - match: { hits.total: 1 }
-
-  - do:
-      search:
-        index: new_twitter
-        body:
-          query:
-            match:
-              user: another
-  - match: { hits.total: 1 }
-
----
-"Reindex all docs with one doc deletion":
-  # Source index
-  - do:
-      index:
-        index:  index1
-        type:   type1
-        id:     1
-        body:   { "lang": "en", "id": 123 }
-  - do:
-      index:
-        index:  index1
-        type:   type1
-        id:     2
-        body:   { "lang": "en", "id": 456 }
-  - do:
-      index:
-        index:  index1
-        type:   type1
-        id:     3
-        body:   { "lang": "fr", "id": 789 }
-  # Destination index
-  - do:
-      index:
-        index:  index2
-        type:   type2
-        id:     fr_789
-        body:   { "lang": "fr", "id": 789 }
-  - do:
-      index:
-        index:  index2
-        type:   type2
-        id:     en_123
-        body:   { "lang": "en", "id": 123 }
-  - do:
-      indices.refresh: {}
-
-  # Reindex all documents from "index1" into "index2", changing their type
-  # to "type2" and their id to the concatened lang+id fields,
-  # trashing all non-english pre existing ones
-  - do:
-      reindex:
-        refresh: true
-        body:
-          source:
-            index: index1
-          dest:
-            index: index2
-            type: type2
-          script:
-            lang: painless
-            inline: "ctx._id = ctx._source.lang + '_' + ctx._source.id;
-                     if (ctx._source.lang != \"en\" ) {ctx.op = 'delete'}"
-  - match: {created: 1}
-  - match: {noops: 0}
-  - match: {updated: 1}
-  - match: {deleted: 1}
-
-  - do:
-      mget:
-        body:
-          docs:
-            - { _index: index2, _type: type2, _id: en_123}
-            - { _index: index2, _type: type2, _id: en_456}
-            - { _index: index2, _type: type2, _id: fr_789}
-
-  - is_true: docs.0.found
-  - match:  { docs.0._index:     index2      }
-  - match:  { docs.0._type:      type2       }
-  - match:  { docs.0._id:        en_123      }
-  - match:  { docs.0._version:   2           }
-
-  - is_true: docs.1.found
-  - match:  { docs.1._index:     index2      }
-  - match:  { docs.1._type:      type2       }
-  - match:  { docs.1._id:        en_456      }
-  - match:  { docs.1._version:   1           }
-
-  - is_false: docs.2.found
-  - match:  { docs.2._index:     index2      }
-  - match:  { docs.2._type:      type2       }
-  - match:  { docs.2._id:        fr_789      }
diff --git a/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/reindex/20_broken.yaml b/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/reindex/20_broken.yaml
deleted file mode 100644
index 5ec35b4c9dc71..0000000000000
--- a/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/reindex/20_broken.yaml
+++ /dev/null
@@ -1,24 +0,0 @@
----
-"Totally broken scripts report the error properly":
-  - do:
-      index:
-        index:  twitter
-        type:   tweet
-        id:     1
-        body:   { "user": "kimchy" }
-  - do:
-      indices.refresh: {}
-
-  - do:
-      catch: request
-      reindex:
-        refresh: true
-        body:
-          source:
-            index: twitter
-          dest:
-            index: new_twitter
-          script:
-            lang: painless
-            inline: syntax errors are fun!
-  - match: {error.reason: 'compile error'}
diff --git a/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/update_by_query/10_script.yaml b/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/update_by_query/10_script.yaml
deleted file mode 100644
index a4580b9733449..0000000000000
--- a/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/update_by_query/10_script.yaml
+++ /dev/null
@@ -1,314 +0,0 @@
----
-"Update a document using update_by_query":
-  - do:
-      index:
-        index:  twitter
-        type:   tweet
-        id:     1
-        body:   { "user": "kimchy" }
-  - do:
-      indices.refresh: {}
-
-  - do:
-      update_by_query:
-        index:   twitter
-        refresh: true
-        body:
-          script:
-            lang: painless
-            inline: ctx._source.user = "not" + ctx._source.user
-  - match: {updated: 1}
-  - match: {noops: 0}
-
-  - do:
-      search:
-        index: twitter
-        body:
-          query:
-            match:
-              user: notkimchy
-  - match: { hits.total: 1 }
-
----
-"Noop one doc":
-  - do:
-      index:
-        index:  twitter
-        type:   tweet
-        id:     1
-        body:   { "user": "kimchy" }
-  - do:
-      index:
-        index:  twitter
-        type:   tweet
-        id:     2
-        body:   { "user": "foo" }
-  - do:
-      indices.refresh: {}
-
-  - do:
-      update_by_query:
-        refresh: true
-        index:   twitter
-        body:
-          script:
-            lang: painless
-            inline: if (ctx._source.user == "kimchy") {ctx._source.user = "not" + ctx._source.user} else {ctx.op = "noop"}
-  - match: {updated: 1}
-  - match: {noops: 1}
-
-  - do:
-      search:
-        index: twitter
-        body:
-          query:
-            match:
-              user: notkimchy
-  - match: { hits.total: 1 }
-
-  - do:
-      search:
-        index: twitter
-        body:
-          query:
-            match:
-              user: notfoo
-  - match: { hits.total: 0 }
-
----
-"Noop all docs":
-  - do:
-      index:
-        index:  twitter
-        type:   tweet
-        id:     1
-        body:   { "user": "kimchy" }
-  - do:
-      index:
-        index:  twitter
-        type:   tweet
-        id:     2
-        body:   { "user": "foo" }
-  - do:
-      indices.refresh: {}
-
-  - do:
-      update_by_query:
-        refresh: true
-        index:   twitter
-        body:
-          script:
-            lang: painless
-            inline: ctx.op = "noop"
-  - match: {updated: 0}
-  - match: {noops: 2}
-  - match: {batches: 1}
-
----
-"Setting bogus context is an error":
-  - do:
-      index:
-        index:  twitter
-        type:   tweet
-        id:     1
-        body:   { "user": "kimchy" }
-  - do:
-      indices.refresh: {}
-
-  - do:
-      catch: /Invalid fields added to context \[junk\]/
-      update_by_query:
-        index: twitter
-        body:
-          script:
-            lang: painless
-            inline: ctx.junk = "stuff"
-
----
-"Can't change _id":
-  - do:
-      index:
-        index:  twitter
-        type:   tweet
-        id:     1
-        body:   { "user": "kimchy" }
-  - do:
-      indices.refresh: {}
-
-  - do:
-      catch: /Modifying \[_id\] not allowed/
-      update_by_query:
-        index: twitter
-        body:
-          script:
-            lang: painless
-            inline: ctx._id = "stuff"
-
----
-"Update all docs with one doc deletion":
-  - do:
-      index:
-        index:  twitter
-        type:   tweet
-        id:     1
-        body:   { "level": 9, "last_updated": "2016-01-01T12:10:30Z" }
-  - do:
-      index:
-        index:  twitter
-        type:   tweet
-        id:     2
-        body:   { "level": 10, "last_updated": "2016-01-01T12:10:30Z" }
-  - do:
-      index:
-        index:  twitter
-        type:   tweet
-        id:     3
-        body:   { "level": 11, "last_updated": "2016-01-01T12:10:30Z" }
-  - do:
-      index:
-        index:  twitter
-        type:   tweet
-        id:     4
-        body:   { "level": 12, "last_updated": "2016-01-01T12:10:30Z" }
-  - do:
-      indices.refresh: {}
-
-  - do:
-      update_by_query:
-        refresh: true
-        index:   twitter
-        body:
-          script:
-            lang: painless
-            inline: if (ctx._source.level != 11) {ctx._source.last_updated = "2016-01-02T00:00:00Z"} else {ctx.op = "delete"}
-  - match: {updated: 3}
-  - match: {deleted: 1}
-  - match: {noops: 0}
-
-  - do:
-      search:
-        index: twitter
-        body:
-          query:
-            match:
-              last_updated: "2016-01-02T00:00:00Z"
-  - match: { hits.total: 3 }
-
-  - do:
-      search:
-        index: twitter
-        body:
-          query:
-            term:
-              level: 11
-  - match: { hits.total: 0 }
-
-  - do:
-      count:
-        index: twitter
-  - match: {count: 3}
-
----
-"Update all docs with one deletion and one noop":
-  - do:
-      index:
-        index:  twitter
-        type:   tweet
-        id:     1
-        body:   { "level": 9, "last_updated": "2016-01-01T12:10:30Z" }
-  - do:
-      index:
-        index:  twitter
-        type:   tweet
-        id:     2
-        body:   { "level": 10, "last_updated": "2016-01-01T12:10:30Z" }
-  - do:
-      index:
-        index:  twitter
-        type:   tweet
-        id:     3
-        body:   { "level": 11, "last_updated": "2016-01-01T12:10:30Z" }
-  - do:
-      index:
-        index:  twitter
-        type:   tweet
-        id:     4
-        body:   { "level": 12, "last_updated": "2016-01-01T12:10:30Z" }
-  - do:
-      indices.refresh: {}
-
-  - do:
-      update_by_query:
-        refresh: true
-        index:   twitter
-        body:
-          script:
-            lang: painless
-            inline: |
-              int choice = ctx._source.level % 3;
-              if (choice == 0) {
-                ctx._source.last_updated = "2016-01-02T00:00:00Z";
-              } else if (choice == 1) {
-                ctx.op = "noop";
-              } else {
-                ctx.op = "delete";
-              }
-  - match: {updated: 2}
-  - match: {deleted: 1}
-  - match: {noops: 1}
-
-  - do:
-      search:
-        index: twitter
-        body:
-          query:
-            match:
-              last_updated: "2016-01-02T00:00:00Z"
-  - match: { hits.total: 2 }
-
-  - do:
-      search:
-        index: twitter
-        body:
-          query:
-            match:
-              last_updated: "2016-01-01T12:10:30Z"
-  - match: { hits.total: 1 }
-
-  - do:
-      search:
-        index: twitter
-        body:
-          query:
-            term:
-              level: 11
-  - match: { hits.total: 0 }
-
----
-"Set unsupported operation type":
-  - do:
-      index:
-        index:  twitter
-        type:   tweet
-        id:     1
-        body:   { "user": "kimchy" }
-  - do:
-      index:
-        index:  twitter
-        type:   tweet
-        id:     2
-        body:   { "user": "foo" }
-  - do:
-      indices.refresh: {}
-
-  - do:
-      catch: request
-      update_by_query:
-        refresh: true
-        index:   twitter
-        body:
-          script:
-            lang: painless
-            inline: if (ctx._source.user == "kimchy") {ctx.op = "update"} else {ctx.op = "junk"}
-
-  - match: { error.reason: 'Operation type [junk] not allowed, only [noop, index, delete] are allowed' }
diff --git a/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/update_by_query/20_broken.yaml b/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/update_by_query/20_broken.yaml
deleted file mode 100644
index aa497bff77e90..0000000000000
--- a/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/update_by_query/20_broken.yaml
+++ /dev/null
@@ -1,21 +0,0 @@
----
-"Totally broken scripts report the error properly":
-  - do:
-      index:
-        index:  twitter
-        type:   tweet
-        id:     1
-        body:   { "user": "kimchy" }
-  - do:
-      indices.refresh: {}
-
-  - do:
-      catch: request
-      update_by_query:
-        index:   twitter
-        refresh: true
-        body:
-          script:
-            lang: painless
-            inline: syntax errors are fun!
-  - match: {error.reason: 'compile error'}
diff --git a/qa/smoke-test-tribe-node/build.gradle b/qa/smoke-test-tribe-node/build.gradle
new file mode 100644
index 0000000000000..009298adc6aa0
--- /dev/null
+++ b/qa/smoke-test-tribe-node/build.gradle
@@ -0,0 +1,65 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.elasticsearch.gradle.test.ClusterConfiguration
+import org.elasticsearch.gradle.test.ClusterFormationTasks
+import org.elasticsearch.gradle.test.NodeInfo
+
+apply plugin: 'elasticsearch.standalone-rest-test'
+apply plugin: 'elasticsearch.rest-test'
+
+
+ClusterConfiguration configOne = new ClusterConfiguration(project)
+configOne.clusterName = 'one'
+configOne.setting('node.name', 'one')
+List<NodeInfo> oneNodes = ClusterFormationTasks.setup(project, 'clusterOne', integTestRunner, configOne)
+
+ClusterConfiguration configTwo = new ClusterConfiguration(project)
+configTwo.clusterName = 'two'
+configTwo.setting('node.name', 'two')
+List<NodeInfo> twoNodes = ClusterFormationTasks.setup(project, 'clusterTwo', integTestRunner, configTwo)
+
+integTestCluster {
+  // tribe nodes had a bug where if explicit ports was specified for the tribe node, the dynamic socket permissions that were applied
+  // would not account for the fact that the internal node client needed to bind to sockets too; thus, we use explicit port ranges to
+  // ensure that the code that fixes this bug is exercised
+  setting 'http.port', '40200-40249'
+  setting 'transport.tcp.port', '40300-40349'
+  setting 'node.name', 'quest'
+  setting 'tribe.one.cluster.name', 'one'
+  setting 'tribe.one.discovery.zen.ping.unicast.hosts', "'${-> oneNodes.get(0).transportUri()}'"
+  setting 'tribe.one.http.enabled', 'true'
+  setting 'tribe.one.http.port', '40250-40299'
+  setting 'tribe.one.transport.tcp.port', '40350-40399'
+  setting 'tribe.two.cluster.name', 'two'
+  setting 'tribe.two.discovery.zen.ping.unicast.hosts', "'${-> twoNodes.get(0).transportUri()}'"
+  setting 'tribe.two.http.enabled', 'true'
+  setting 'tribe.two.http.port', '40250-40299'
+  setting 'tribe.two.transport.tcp.port', '40250-40399'
+
+  waitCondition = { node, ant ->
+    File tmpFile = new File(node.cwd, 'wait.success')
+    // 5 nodes: tribe + clusterOne (1 node + tribe internal node) + clusterTwo (1 node + tribe internal node)
+    ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=5&wait_for_status=yellow",
+            dest: tmpFile.toString(),
+            ignoreerrors: true,
+            retries: 10)
+    return tmpFile.exists()
+  }
+}
diff --git a/qa/smoke-test-tribe-node/src/test/java/org/elasticsearch/tribe/TribeClientYamlTestSuiteIT.java b/qa/smoke-test-tribe-node/src/test/java/org/elasticsearch/tribe/TribeClientYamlTestSuiteIT.java
new file mode 100644
index 0000000000000..1a8e7867dd8b6
--- /dev/null
+++ b/qa/smoke-test-tribe-node/src/test/java/org/elasticsearch/tribe/TribeClientYamlTestSuiteIT.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.tribe;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
+import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+
+public class TribeClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
+
+    // tribe nodes can not handle delete indices requests
+    @Override
+    protected boolean preserveIndicesUponCompletion() {
+        return true;
+    }
+
+    // tribe nodes can not handle delete template requests
+    @Override
+    protected boolean preserveTemplatesUponCompletion() {
+        return true;
+    }
+
+    public TribeClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return createParameters();
+    }
+}
+
diff --git a/qa/smoke-test-tribe-node/src/test/resources/rest-api-spec/test/tribe/10_basic.yaml b/qa/smoke-test-tribe-node/src/test/resources/rest-api-spec/test/tribe/10_basic.yaml
new file mode 100644
index 0000000000000..d70a355ac6274
--- /dev/null
+++ b/qa/smoke-test-tribe-node/src/test/resources/rest-api-spec/test/tribe/10_basic.yaml
@@ -0,0 +1,16 @@
+---
+"Tribe node test":
+  - do:
+      cat.nodes:
+          h: name
+          s: name
+          v: true
+
+  - match:
+      $body: |
+          /^ name\n
+             one\n
+             quest\n
+             quest/one\n
+             quest/two\n
+             two\n      $/
diff --git a/qa/vagrant/build.gradle b/qa/vagrant/build.gradle
index 32c74a357546b..40fc3998185c8 100644
--- a/qa/vagrant/build.gradle
+++ b/qa/vagrant/build.gradle
@@ -17,257 +17,25 @@
  * under the License.
  */
 
-import org.elasticsearch.gradle.FileContentsTask
-import org.elasticsearch.gradle.vagrant.BatsOverVagrantTask
-import org.elasticsearch.gradle.vagrant.VagrantCommandTask
-
-String testScripts = '*.bats'
-String testCommand = "cd \$TESTROOT && sudo bats --tap \$BATS/$testScripts"
-String smokeTestCommand = 'echo I work'
-
-// the images we allow testing with
-List<String> availableBoxes = [
-    'centos-6',
-    'centos-7',
-    'debian-8',
-    'fedora-24',
-    'oel-6',
-    'oel-7',
-    'opensuse-13',
-    'sles-12',
-    'ubuntu-1204',
-    'ubuntu-1404',
-    'ubuntu-1504'
-]
-
-String vagrantBoxes = getProperties().get('vagrant.boxes', 'sample')
-List<String> boxes = []
-for (String box : vagrantBoxes.split(',')) {
-  if (box == 'sample') {
-    boxes.add('centos-7')
-    boxes.add('ubuntu-1404')
-  } else if (box == 'all') {
-    boxes = availableBoxes
-    break
-  } else {
-    if (availableBoxes.contains(box) == false) {
-      throw new IllegalArgumentException("Unknown vagrant box '${box}'")
+apply plugin: 'elasticsearch.vagrantsupport'
+apply plugin: 'elasticsearch.vagrant'
+
+List<String> plugins = []
+for (Project subproj : project.rootProject.subprojects) {
+  if (subproj.path.startsWith(':plugins:')) {
+    // add plugin as a dep
+    dependencies {
+      bats project(path: "${subproj.path}", configuration: 'zip')
     }
-    boxes.add(box)
+    plugins.add(subproj.name)
   }
 }
+plugins = plugins.toSorted()
 
-long seed
-String formattedSeed = null
-String[] upgradeFromVersions
-String upgradeFromVersion
-
-String maybeTestsSeed = System.getProperty("tests.seed", null);
-if (maybeTestsSeed != null) {
-  List<String> seeds = maybeTestsSeed.tokenize(':')
-  if (seeds.size() != 0) {
-    String masterSeed = seeds.get(0)
-    seed = new BigInteger(masterSeed, 16).longValue()
-    formattedSeed = maybeTestsSeed
-  }
-}
-if (formattedSeed == null) {
-  seed = new Random().nextLong()
-  formattedSeed = String.format("%016X", seed)
-}
-
-String maybeUpdradeFromVersions = System.getProperty("tests.packaging.upgrade.from.versions", null)
-if (maybeUpdradeFromVersions != null) {
-  upgradeFromVersions = maybeUpdradeFromVersions.split(",")
-} else {
-  upgradeFromVersions = new File(project.projectDir, 'versions')
-}
-
-upgradeFromVersion = upgradeFromVersions[new Random(seed).nextInt(upgradeFromVersions.length)]
-
-configurations {
-  test
-}
-
-repositories {
-  mavenCentral()
-}
-
-dependencies {
-  test project(path: ':distribution:tar', configuration: 'archives')
-  test project(path: ':distribution:rpm', configuration: 'archives')
-  test project(path: ':distribution:deb', configuration: 'archives')
-
-  // Collect all the plugins
-  for (Project subproj : project.rootProject.subprojects) {
-    if (subproj.path.startsWith(':plugins:')) {
-      test project(path: "${subproj.path}", configuration: 'zip')
-    }
-  }
-
-  // The version of elasticsearch that we upgrade *from*
-  test "org.elasticsearch.distribution.deb:elasticsearch:$upgradeFromVersion@deb"
-  test "org.elasticsearch.distribution.rpm:elasticsearch:$upgradeFromVersion@rpm"
-}
-
-task clean(type: Delete) {
-  group 'Build'
-  delete buildDir
-}
-
-task stop {
-  group 'Verification'
-  description 'Stop any tasks from tests that still may be running'
-}
-
-Set<String> getVersions() {
-  Node xml
-  new URL('http://repo1.maven.org/maven2/org/elasticsearch/elasticsearch/maven-metadata.xml').openStream().withStream { s ->
-    xml = new XmlParser().parse(s)
-  }
-  return new TreeSet<>(xml.versioning.versions.version.collect { it.text() }.findAll { it ==~ /2\.\d\.\d/ })
-}
-
-task updatePackagingTestUpgradeFromVersions {
-  group 'Verification'
-  description 'Update file containing options for the\n    "starting" version in the "upgrade from" packaging tests.'
-  doLast {
-    Set<String> versions = getVersions()
-    new File(project.projectDir, 'versions').text = versions.join('\n') + '\n'
-  }
-}
-
-task verifyPackagingTestUpgradeFromVersions {
-  doLast {
-    String maybeUpdateFromVersions = System.getProperty("tests.packaging.upgrade.from.versions", null)
-    if (maybeUpdateFromVersions == null) {
-      Set<String> versions = getVersions()
-      Set<String> actualVersions = new HashSet<>(Arrays.asList(upgradeFromVersions))
-      if (!versions.equals(actualVersions)) {
-        throw new GradleException("out-of-date versions [" + actualVersions + "], expected [" + versions + "]; run gradle updatePackagingTestUpgradeFromVersions")
-      }
-    }
-  }
-}
-
-File testRoot = new File("$buildDir/testroot")
-task createTestRoot {
-  dependsOn verifyPackagingTestUpgradeFromVersions
-  outputs.dir testRoot
-  doLast {
-    testRoot.mkdirs()
-  }
-}
-
-task createVersionFile(type: FileContentsTask) {
-  dependsOn createTestRoot
-  file "${testRoot}/version"
-  contents = version
-}
-
-task createUpgradeFromFile(type: FileContentsTask) {
-  dependsOn createTestRoot
-  file "${testRoot}/upgrade_from_version"
-  contents = upgradeFromVersion
-}
-
-task prepareTestRoot(type: Copy) {
-  description 'Dump bats test dependencies into the $TESTROOT'
-  into testRoot
-  from configurations.test
-
-  dependsOn createVersionFile, createUpgradeFromFile
+setupBats {
   doFirst {
-    gradle.addBuildListener new BuildAdapter() {
-      @Override
-      void buildFinished(BuildResult result) {
-        if (result.failure) {
-          println "Reproduce with: gradle packagingTest -Pvagrant.boxes=${vagrantBoxes} -Dtests.seed=${formattedSeed} -Dtests.packaging.upgrade.from.versions=${upgradeFromVersions.join(",")}"
-        }
-      }
-    }
-  }
-}
-
-task checkVagrantVersion(type: Exec) {
-  commandLine 'vagrant', '--version'
-  standardOutput = new ByteArrayOutputStream()
-  doLast {
-    String version = standardOutput.toString().trim()
-    if ((version ==~ /Vagrant 1\.[789]\..+/) == false) {
-      throw new InvalidUserDataException(
-        "Illegal version of vagrant [${version}]. Need [Vagrant 1.7+]")
-    }
-  }
-}
-
-task vagrantSmokeTest {
-  group 'Verification'
-  description 'Smoke test the specified vagrant boxes'
-}
-
-task packagingTest {
-  group 'Verification'
-  description "Tests yum/apt packages using vagrant and bats.\n" +
-              "    Specify the vagrant boxes to test using the gradle property 'vagrant.boxes'.\n" +
-              "    'sample' can be used to test a single yum and apt box. 'all' can be used to\n" +
-              "    test all available boxes. The available boxes are: \n" +
-              "    ${availableBoxes}"
-}
-
-// Each box gets it own set of tasks
-for (String box : availableBoxes) {
-  String boxTask = box.capitalize().replace('-', '')
-
-  // always add a halt task for all boxes, so clean makes sure they are all shutdown
-  Task halt = tasks.create("vagrant${boxTask}#halt", VagrantCommandTask) {
-    boxName box
-    args 'halt', box
-  }
-  stop.dependsOn(halt)
-  if (boxes.contains(box) == false) {
-    // we only need a halt task if this box was not specified
-    continue;
-  }
-
-  Task update = tasks.create("vagrant${boxTask}#update", VagrantCommandTask) {
-    boxName box
-    args 'box', 'update', box
-    dependsOn checkVagrantVersion
-  }
-
-  Task up = tasks.create("vagrant${boxTask}#up", VagrantCommandTask) {
-    boxName box
-    /* Its important that we try to reprovision the box even if it already
-      exists. That way updates to the vagrant configuration take automatically.
-      That isn't to say that the updates will always be compatible. Its ok to
-      just destroy the boxes if they get busted but that is a manual step
-      because its slow-ish. */
-    /* We lock the provider to virtualbox because the Vagrantfile specifies
-      lots of boxes that only work properly in virtualbox. Virtualbox is
-      vagrant's default but its possible to change that default and folks do.
-      But the boxes that we use are unlikely to work properly with other
-      virtualization providers. Thus the lock. */
-    args 'up', box, '--provision', '--provider', 'virtualbox'
-    /* It'd be possible to check if the box is already up here and output
-      SKIPPED but that would require running vagrant status which is slow! */
-    dependsOn update
-  }
-
-  Task smoke = tasks.create("vagrant${boxTask}#smoketest", Exec) {
-    dependsOn up
-    finalizedBy halt
-    commandLine 'vagrant', 'ssh', box, '--command',
-      "set -o pipefail && ${smokeTestCommand} | sed -ue 's/^/    ${box}: /'"
-  }
-  vagrantSmokeTest.dependsOn(smoke)
-
-  Task packaging = tasks.create("packagingTest${boxTask}", BatsOverVagrantTask) {
-    dependsOn up
-    finalizedBy halt
-    boxName box
-    command testCommand
-    dependsOn prepareTestRoot
+    File expectedPlugins = file('build/plugins/expected')
+    expectedPlugins.parentFile.mkdirs()
+    expectedPlugins.setText(plugins.join('\n'), 'UTF-8')
   }
-  packagingTest.dependsOn(packaging)
 }
diff --git a/qa/vagrant/src/test/resources/packaging/scripts/20_tar_package.bats b/qa/vagrant/src/test/resources/packaging/scripts/20_tar_package.bats
deleted file mode 100644
index eca0c6509010c..0000000000000
--- a/qa/vagrant/src/test/resources/packaging/scripts/20_tar_package.bats
+++ /dev/null
@@ -1,138 +0,0 @@
-#!/usr/bin/env bats
-
-# This file is used to test the tar gz package.
-
-# WARNING: This testing file must be executed as root and can
-# dramatically change your system. It removes the 'elasticsearch'
-# user/group and also many directories. Do not execute this file
-# unless you know exactly what you are doing.
-
-# The test case can be executed with the Bash Automated
-# Testing System tool available at https://github.com/sstephenson/bats
-# Thanks to Sam Stephenson!
-
-# Licensed to Elasticsearch under one or more contributor
-# license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright
-# ownership. Elasticsearch licenses this file to you under
-# the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Load test utilities
-load packaging_test_utils
-load tar
-load plugins
-
-setup() {
-    skip_not_tar_gz
-    export ESHOME=/tmp/elasticsearch
-    export_elasticsearch_paths
-}
-
-##################################
-# Install TAR GZ package
-##################################
-@test "[TAR] tar command is available" {
-    # Cleans everything for the 1st execution
-    clean_before_test
-    run tar --version
-    [ "$status" -eq 0 ]
-}
-
-@test "[TAR] archive is available" {
-    count=$(find . -type f -name 'elasticsearch*.tar.gz' | wc -l)
-    [ "$count" -eq 1 ]
-}
-
-@test "[TAR] archive is not installed" {
-    count=$(find /tmp -type d -name 'elasticsearch*' | wc -l)
-    [ "$count" -eq 0 ]
-}
-
-@test "[TAR] install archive" {
-    # Install the archive
-    install_archive
-
-    count=$(find /tmp -type d -name 'elasticsearch*' | wc -l)
-    [ "$count" -eq 1 ]
-
-    # Its simpler to check that the install was correct in this test rather
-    # than in another test because install_archive sets a number of path
-    # variables that verify_archive_installation reads. To separate this into
-    # another test you'd have to recreate the variables.
-    verify_archive_installation
-}
-
-@test "[TAR] elasticsearch fails if java executable is not found" {
-  local JAVA=$(which java)
-
-  sudo chmod -x $JAVA
-  run "$ESHOME/bin/elasticsearch"
-  sudo chmod +x $JAVA
-
-  [ "$status" -eq 1 ]
-  [[ "$output" == *"Could not find any executable java binary. Please install java in your PATH or set JAVA_HOME"* ]]
-}
-
-##################################
-# Check that Elasticsearch is working
-##################################
-@test "[TAR] test elasticsearch" {
-    # Install scripts used to test script filters and search templates before
-    # starting Elasticsearch so we don't have to wait for elasticsearch to scan for
-    # them.
-    install_elasticsearch_test_scripts
-    start_elasticsearch_service
-    run_elasticsearch_tests
-    stop_elasticsearch_service
-}
-
-@test "[TAR] start Elasticsearch with custom JVM options" {
-    local es_java_opts=$ES_JAVA_OPTS
-    local es_jvm_options=$ES_JVM_OPTIONS
-    local temp=`mktemp -d`
-    touch "$temp/jvm.options"
-    chown -R elasticsearch:elasticsearch "$temp"
-    echo "-Xms264m" >> "$temp/jvm.options"
-    echo "-Xmx264m" >> "$temp/jvm.options"
-    export ES_JVM_OPTIONS="$temp/jvm.options"
-    export ES_JAVA_OPTS="-XX:-UseCompressedOops"
-    start_elasticsearch_service
-    curl -s -XGET localhost:9200/_nodes | fgrep '"heap_init_in_bytes":276824064'
-    curl -s -XGET localhost:9200/_nodes | fgrep '"using_compressed_ordinary_object_pointers":"false"'
-    stop_elasticsearch_service
-    export ES_JVM_OPTIONS=$es_jvm_options
-    export ES_JAVA_OPTS=$es_java_opts
-}
-
-@test "[TAR] start Elasticsearch with unquoted JSON option" {
-    local es_java_opts=$ES_JAVA_OPTS
-    local es_jvm_options=$ES_JVM_OPTIONS
-    local temp=`mktemp -d`
-    touch "$temp/jvm.options"
-    chown -R elasticsearch:elasticsearch "$temp"
-    echo "-Delasticsearch.json.allow_unquoted_field_names=true" >> "$temp/jvm.options"
-    export ES_JVM_OPTIONS="$temp/jvm.options"
-    start_elasticsearch_service
-    # unquoted field name
-    curl -s -XPOST localhost:9200/i/d/1 -d'{foo: "bar"}'
-    [ "$?" -eq 0 ]
-    curl -s -XDELETE localhost:9200/i
-    stop_elasticsearch_service
-    export ES_JVM_OPTIONS=$es_jvm_options
-    export ES_JAVA_OPTS=$es_java_opts
-}
-
-@test "[TAR] remove tar" {
-    rm -rf "/tmp/elasticsearch"
-}
diff --git a/qa/vagrant/src/test/resources/packaging/scripts/60_systemd.bats b/qa/vagrant/src/test/resources/packaging/scripts/60_systemd.bats
deleted file mode 100644
index da7b6a180f1a8..0000000000000
--- a/qa/vagrant/src/test/resources/packaging/scripts/60_systemd.bats
+++ /dev/null
@@ -1,147 +0,0 @@
-#!/usr/bin/env bats
-
-# This file is used to test the elasticsearch Systemd setup.
-
-# WARNING: This testing file must be executed as root and can
-# dramatically change your system. It removes the 'elasticsearch'
-# user/group and also many directories. Do not execute this file
-# unless you know exactly what you are doing.
-
-# The test case can be executed with the Bash Automated
-# Testing System tool available at https://github.com/sstephenson/bats
-# Thanks to Sam Stephenson!
-
-# Licensed to Elasticsearch under one or more contributor
-# license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright
-# ownership. Elasticsearch licenses this file to you under
-# the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Load test utilities
-load packaging_test_utils
-load os_package
-load plugins
-
-# Cleans everything for the 1st execution
-setup() {
-    skip_not_systemd
-    skip_not_dpkg_or_rpm
-    export_elasticsearch_paths
-}
-
-@test "[SYSTEMD] install elasticsearch" {
-    clean_before_test
-    install_package
-}
-
-@test "[SYSTEMD] daemon reload after install" {
-    systemctl daemon-reload
-}
-
-@test "[SYSTEMD] daemon isn't enabled on restart" {
-    # Rather than restart the VM we just ask systemd if it plans on starting
-    # elasticsearch on restart. Not as strong as a restart but much much
-    # faster.
-    run systemctl is-enabled elasticsearch.service
-    [ "$output" = "disabled" ]
-}
-
-@test "[SYSTEMD] enable" {
-    systemctl enable elasticsearch.service
-
-    systemctl is-enabled elasticsearch.service
-}
-
-@test "[SYSTEMD] start" {
-    # Install scripts used to test script filters and search templates before
-    # starting Elasticsearch so we don't have to wait for elasticsearch to scan for
-    # them.
-    install_elasticsearch_test_scripts
-    systemctl start elasticsearch.service
-    wait_for_elasticsearch_status
-    assert_file_exist "/var/run/elasticsearch/elasticsearch.pid"
-}
-
-@test "[SYSTEMD] start (running)" {
-    systemctl start elasticsearch.service
-}
-
-@test "[SYSTEMD] is active (running)" {
-    run systemctl is-active elasticsearch.service
-    [ "$status" -eq 0 ]
-    [ "$output" = "active" ]
-}
-
-@test "[SYSTEMD] status (running)" {
-    systemctl status elasticsearch.service
-}
-
-##################################
-# Check that Elasticsearch is working
-##################################
-@test "[SYSTEMD] test elasticsearch" {
-    run_elasticsearch_tests
-}
-
-@test "[SYSTEMD] restart" {
-    systemctl restart elasticsearch.service
-
-    wait_for_elasticsearch_status
-
-    service elasticsearch status
-}
-
-@test "[SYSTEMD] stop (running)" {
-    systemctl stop elasticsearch.service
-}
-
-@test "[SYSTEMD] status (stopping)" {
-    run systemctl status elasticsearch.service
-    # I'm not sure why suse exits 0 here, but it does
-    if [ ! -e /etc/SuSE-release ]; then
-        [ "$status" -eq 3 ] || "Expected exit code 3 meaning stopped but got $status"
-    fi
-    echo "$output" | grep "Active:" | grep "inactive"
-}
-
-@test "[SYSTEMD] stop (stopped)" {
-    systemctl stop elasticsearch.service
-}
-
-@test "[SYSTEMD] status (stopped)" {
-    run systemctl status elasticsearch.service
-    # I'm not sure why suse exits 0 here, but it does
-    if [ ! -e /etc/SuSE-release ]; then
-        [ "$status" -eq 3 ] || "Expected exit code 3 meaning stopped but got $status"
-    fi
-    echo "$output" | grep "Active:" | grep "inactive"
-}
-
-# Simulates the behavior of a system restart:
-# the PID directory is deleted by the operating system
-# but it should not block ES from starting
-# see https://github.com/elastic/elasticsearch/issues/11594
-@test "[SYSTEMD] delete PID_DIR and restart" {
-    rm -rf /var/run/elasticsearch
-
-    systemd-tmpfiles --create
-
-    systemctl start elasticsearch.service
-
-    wait_for_elasticsearch_status
-
-    assert_file_exist "/var/run/elasticsearch/elasticsearch.pid"
-
-    systemctl stop elasticsearch.service
-}
diff --git a/qa/vagrant/src/test/resources/packaging/scripts/70_sysv_initd.bats b/qa/vagrant/src/test/resources/packaging/scripts/70_sysv_initd.bats
deleted file mode 100644
index fa96882f914c5..0000000000000
--- a/qa/vagrant/src/test/resources/packaging/scripts/70_sysv_initd.bats
+++ /dev/null
@@ -1,151 +0,0 @@
-#!/usr/bin/env bats
-
-# This file is used to test the elasticsearch init.d scripts.
-
-# WARNING: This testing file must be executed as root and can
-# dramatically change your system. It removes the 'elasticsearch'
-# user/group and also many directories. Do not execute this file
-# unless you know exactly what you are doing.
-
-# The test case can be executed with the Bash Automated
-# Testing System tool available at https://github.com/sstephenson/bats
-# Thanks to Sam Stephenson!
-
-# Licensed to Elasticsearch under one or more contributor
-# license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright
-# ownership. Elasticsearch licenses this file to you under
-# the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Load test utilities
-load packaging_test_utils
-load os_package
-load plugins
-
-# Cleans everything for the 1st execution
-setup() {
-    skip_not_sysvinit
-    skip_not_dpkg_or_rpm
-    export_elasticsearch_paths
-}
-
-@test "[INIT.D] remove any leftover configuration to start elasticsearch on restart" {
-    # This configuration can be added with a command like:
-    # $ sudo update-rc.d elasticsearch defaults 95 10
-    # but we want to test that the RPM and deb _don't_ add it on its own.
-    # Note that it'd be incorrect to use:
-    # $ sudo update-rc.d elasticsearch disable
-    # here because that'd prevent elasticsearch from installing the symlinks
-    # that cause it to be started on restart.
-    sudo update-rc.d -f elasticsearch remove || true
-    sudo chkconfig elasticsearch off || true
-}
-
-@test "[INIT.D] install elasticsearch" {
-    clean_before_test
-    install_package
-}
-
-@test "[INIT.D] elasticsearch fails if startup script is not executable" {
-    local INIT="/etc/init.d/elasticsearch"
-    local DAEMON="$ESHOME/bin/elasticsearch"
-    
-    sudo chmod -x "$DAEMON"
-    run "$INIT"
-    sudo chmod +x "$DAEMON"
-    
-    [ "$status" -eq 1 ]
-    [[ "$output" == *"The elasticsearch startup script does not exists or it is not executable, tried: $DAEMON"* ]]
-}
-
-@test "[INIT.D] daemon isn't enabled on restart" {
-    # Rather than restart the VM which would be slow we check for the symlinks
-    # that init.d uses to restart the application on startup.
-    ! find /etc/rc[0123456].d | grep elasticsearch
-    # Note that we don't use -iname above because that'd have to look like:
-    # [ $(find /etc/rc[0123456].d -iname "elasticsearch*" | wc -l) -eq 0 ]
-    # Which isn't really clearer than what we do use.
-}
-
-@test "[INIT.D] start" {
-    # Install scripts used to test script filters and search templates before
-    # starting Elasticsearch so we don't have to wait for elasticsearch to scan for
-    # them.
-    install_elasticsearch_test_scripts
-    service elasticsearch start
-    wait_for_elasticsearch_status
-    assert_file_exist "/var/run/elasticsearch/elasticsearch.pid"
-}
-
-@test "[INIT.D] status (running)" {
-    service elasticsearch status
-}
-
-##################################
-# Check that Elasticsearch is working
-##################################
-@test "[INIT.D] test elasticsearch" {
-    run_elasticsearch_tests
-}
-
-@test "[INIT.D] restart" {
-    service elasticsearch restart
-
-    wait_for_elasticsearch_status
-
-    service elasticsearch status
-}
-
-@test "[INIT.D] stop (running)" {
-    service elasticsearch stop
-}
-
-@test "[INIT.D] status (stopped)" {
-    run service elasticsearch status
-    # precise returns 4, trusty 3
-    [ "$status" -eq 3 ] || [ "$status" -eq 4 ]
-}
-
-@test "[INIT.D] don't mkdir when it contains a comma" {
-    # Remove these just in case they exist beforehand
-    rm -rf /tmp/aoeu,/tmp/asdf
-    rm -rf /tmp/aoeu,
-    # set DATA_DIR to DATA_DIR=/tmp/aoeu,/tmp/asdf
-    sed -i 's/DATA_DIR=.*/DATA_DIR=\/tmp\/aoeu,\/tmp\/asdf/' /etc/init.d/elasticsearch
-    cat /etc/init.d/elasticsearch | grep "DATA_DIR"
-    service elasticsearch start
-    wait_for_elasticsearch_status
-    assert_file_not_exist /tmp/aoeu,/tmp/asdf
-    assert_file_not_exist /tmp/aoeu,
-    service elasticsearch stop
-    run service elasticsearch status
-    # precise returns 4, trusty 3
-    [ "$status" -eq 3 ] || [ "$status" -eq 4 ]
-}
-
-# Simulates the behavior of a system restart:
-# the PID directory is deleted by the operating system
-# but it should not block ES from starting
-# see https://github.com/elastic/elasticsearch/issues/11594
-@test "[INIT.D] delete PID_DIR and restart" {
-    rm -rf /var/run/elasticsearch
-
-    service elasticsearch start
-
-    wait_for_elasticsearch_status
-
-    assert_file_exist "/var/run/elasticsearch/elasticsearch.pid"
-
-    service elasticsearch stop
-}
diff --git a/qa/vagrant/src/test/resources/packaging/scripts/os_package.bash b/qa/vagrant/src/test/resources/packaging/scripts/os_package.bash
deleted file mode 100644
index d46d7fe9a5d2e..0000000000000
--- a/qa/vagrant/src/test/resources/packaging/scripts/os_package.bash
+++ /dev/null
@@ -1,113 +0,0 @@
-#!/bin/bash
-
-# This file contains some utilities to test the elasticsearch scripts with
-# the .deb/.rpm packages.
-
-# WARNING: This testing file must be executed as root and can
-# dramatically change your system. It removes the 'elasticsearch'
-# user/group and also many directories. Do not execute this file
-# unless you know exactly what you are doing.
-
-# Licensed to Elasticsearch under one or more contributor
-# license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright
-# ownership. Elasticsearch licenses this file to you under
-# the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-
-# Export some useful paths.
-export_elasticsearch_paths() {
-    export ESHOME="/usr/share/elasticsearch"
-    export ESPLUGINS="$ESHOME/plugins"
-    export ESMODULES="$ESHOME/modules"
-    export ESCONFIG="/etc/elasticsearch"
-    export ESSCRIPTS="$ESCONFIG/scripts"
-    export ESDATA="/var/lib/elasticsearch"
-    export ESLOG="/var/log/elasticsearch"
-    export ESPIDDIR="/var/run/elasticsearch"
-}
-
-# Install the rpm or deb package.
-# -u upgrade rather than install. This only matters for rpm.
-# -v the version to upgrade to. Defaults to the version under test.
-install_package() {
-    local version=$(cat version)
-    local rpmCommand='-i'
-    while getopts ":uv:" opt; do
-        case $opt in
-            u)
-                rpmCommand='-U'
-                dpkgCommand='--force-confnew'
-                ;;
-            v)
-                version=$OPTARG
-                ;;
-            \?)
-                echo "Invalid option: -$OPTARG" >&2
-                ;;
-        esac
-    done
-    if is_rpm; then
-        rpm $rpmCommand elasticsearch-$version.rpm
-    elif is_dpkg; then
-        dpkg $dpkgCommand -i elasticsearch-$version.deb
-    else
-        skip "Only rpm or deb supported"
-    fi
-}
-
-# Checks that all directories & files are correctly installed after a deb or
-# rpm install.
-verify_package_installation() {
-    id elasticsearch
-
-    getent group elasticsearch
-
-    assert_file "$ESHOME" d root root 755
-    assert_file "$ESHOME/bin" d root root 755
-    assert_file "$ESHOME/lib" d root root 755
-    assert_file "$ESCONFIG" d root elasticsearch 750
-    assert_file "$ESCONFIG/elasticsearch.yml" f root elasticsearch 750
-    assert_file "$ESCONFIG/log4j2.properties" f root elasticsearch 750
-    assert_file "$ESSCRIPTS" d root elasticsearch 750
-    assert_file "$ESDATA" d elasticsearch elasticsearch 755
-    assert_file "$ESLOG" d elasticsearch elasticsearch 755
-    assert_file "$ESPLUGINS" d root root 755
-    assert_file "$ESMODULES" d root root 755
-    assert_file "$ESPIDDIR" d elasticsearch elasticsearch 755
-    assert_file "$ESHOME/NOTICE.txt" f root root 644
-    assert_file "$ESHOME/README.textile" f root root 644
-
-    if is_dpkg; then
-        # Env file
-        assert_file "/etc/default/elasticsearch" f root root 644
-
-        # Doc files
-        assert_file "/usr/share/doc/elasticsearch" d root root 755
-        assert_file "/usr/share/doc/elasticsearch/copyright" f root root 644
-    fi
-
-    if is_rpm; then
-        # Env file
-        assert_file "/etc/sysconfig/elasticsearch" f root root 644
-        # License file
-        assert_file "/usr/share/elasticsearch/LICENSE.txt" f root root 644
-    fi
-
-    if is_systemd; then
-        assert_file "/usr/lib/systemd/system/elasticsearch.service" f root root 644
-        assert_file "/usr/lib/tmpfiles.d/elasticsearch.conf" f root root 644
-        assert_file "/usr/lib/sysctl.d/elasticsearch.conf" f root root 644
-    fi
-}
diff --git a/qa/vagrant/src/test/resources/packaging/scripts/packaging_test_utils.bash b/qa/vagrant/src/test/resources/packaging/scripts/packaging_test_utils.bash
deleted file mode 100644
index fbda05d5f30f8..0000000000000
--- a/qa/vagrant/src/test/resources/packaging/scripts/packaging_test_utils.bash
+++ /dev/null
@@ -1,529 +0,0 @@
-#!/bin/bash
-
-# This file contains some utilities to test the elasticsearch scripts,
-# the .deb/.rpm packages and the SysV/Systemd scripts.
-
-# WARNING: This testing file must be executed as root and can
-# dramatically change your system. It removes the 'elasticsearch'
-# user/group and also many directories. Do not execute this file
-# unless you know exactly what you are doing.
-
-# Licensed to Elasticsearch under one or more contributor
-# license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright
-# ownership. Elasticsearch licenses this file to you under
-# the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Checks if necessary commands are available to run the tests
-
-if [ ! -x /usr/bin/which ]; then
-    echo "'which' command is mandatory to run the tests"
-    exit 1
-fi
-
-if [ ! -x "`which wget 2>/dev/null`" ]; then
-    echo "'wget' command is mandatory to run the tests"
-    exit 1
-fi
-
-if [ ! -x "`which curl 2>/dev/null`" ]; then
-    echo "'curl' command is mandatory to run the tests"
-    exit 1
-fi
-
-if [ ! -x "`which pgrep 2>/dev/null`" ]; then
-    echo "'pgrep' command is mandatory to run the tests"
-    exit 1
-fi
-
-if [ ! -x "`which unzip 2>/dev/null`" ]; then
-    echo "'unzip' command is mandatory to run the tests"
-    exit 1
-fi
-
-if [ ! -x "`which tar 2>/dev/null`" ]; then
-    echo "'tar' command is mandatory to run the tests"
-    exit 1
-fi
-
-if [ ! -x "`which unzip 2>/dev/null`" ]; then
-    echo "'unzip' command is mandatory to run the tests"
-    exit 1
-fi
-
-if [ ! -x "`which java 2>/dev/null`" ]; then
-    echo "'java' command is mandatory to run the tests"
-    exit 1
-fi
-
-# Returns 0 if the 'dpkg' command is available
-is_dpkg() {
-    [ -x "`which dpkg 2>/dev/null`" ]
-}
-
-# Returns 0 if the 'rpm' command is available
-is_rpm() {
-    [ -x "`which rpm 2>/dev/null`" ]
-}
-
-# Skip test if the 'dpkg' command is not supported
-skip_not_dpkg() {
-    is_dpkg || skip "dpkg is not supported"
-}
-
-# Skip test if the 'rpm' command is not supported
-skip_not_rpm() {
-    is_rpm || skip "rpm is not supported"
-}
-
-skip_not_dpkg_or_rpm() {
-    is_dpkg || is_rpm || skip "only dpkg or rpm systems are supported"
-}
-
-# Returns 0 if the system supports Systemd
-is_systemd() {
-    [ -x /bin/systemctl ]
-}
-
-# Skip test if Systemd is not supported
-skip_not_systemd() {
-    if [ ! -x /bin/systemctl ]; then
-        skip "systemd is not supported"
-    fi
-}
-
-# Returns 0 if the system supports SysV
-is_sysvinit() {
-    [ -x "`which service 2>/dev/null`" ]
-}
-
-# Skip test if SysV is not supported
-skip_not_sysvinit() {
-    if [ -x "`which service 2>/dev/null`" ] && is_systemd; then
-        skip "sysvinit is supported, but systemd too"
-    fi
-    if [ ! -x "`which service 2>/dev/null`" ]; then
-        skip "sysvinit is not supported"
-    fi
-}
-
-# Skip if tar is not supported
-skip_not_tar_gz() {
-    if [ ! -x "`which tar 2>/dev/null`" ]; then
-        skip "tar is not supported"
-    fi
-}
-
-# Skip if unzip is not supported
-skip_not_zip() {
-    if [ ! -x "`which unzip 2>/dev/null`" ]; then
-        skip "unzip is not supported"
-    fi
-}
-
-assert_file_exist() {
-    local file="$1"
-    if [ ! -e "$file" ]; then
-        echo "Should exist: ${file} but does not"
-    fi
-    local file=$(readlink -m "${file}")
-    [ -e "$file" ]
-}
-
-assert_file_not_exist() {
-    local file="$1"
-    if [ -e "$file" ]; then
-        echo "Should not exist: ${file} but does"
-    fi
-    local file=$(readlink -m "${file}")
-    [ ! -e "$file" ]
-}
-
-assert_file() {
-    local file="$1"
-    local type=$2
-    local user=$3
-    local group=$4
-    local privileges=$5
-
-    assert_file_exist "$file"
-
-    if [ "$type" = "d" ]; then
-        if [ ! -d "$file" ]; then
-            echo "[$file] should be a directory but is not"
-        fi
-        [ -d "$file" ]
-    else
-        if [ ! -f "$file" ]; then
-            echo "[$file] should be a regular file but is not"
-        fi
-        [ -f "$file" ]
-    fi
-
-    if [ "x$user" != "x" ]; then
-        realuser=$(find "$file" -maxdepth 0 -printf "%u")
-        if [ "$realuser" != "$user" ]; then
-            echo "Expected user: $user, found $realuser [$file]"
-        fi
-        [ "$realuser" = "$user" ]
-    fi
-
-    if [ "x$group" != "x" ]; then
-        realgroup=$(find "$file" -maxdepth 0 -printf "%g")
-        if [ "$realgroup" != "$group" ]; then
-            echo "Expected group: $group, found $realgroup [$file]"
-        fi
-        [ "$realgroup" = "$group" ]
-    fi
-
-    if [ "x$privileges" != "x" ]; then
-        realprivileges=$(find "$file" -maxdepth 0 -printf "%m")
-        if [ "$realprivileges" != "$privileges" ]; then
-            echo "Expected privileges: $privileges, found $realprivileges [$file]"
-        fi
-        [ "$realprivileges" = "$privileges" ]
-    fi
-}
-
-assert_module_or_plugin_directory() {
-    local directory=$1
-    shift
-
-    #owner group and permissions vary depending on how es was installed
-    #just make sure that everything is the same as $CONFIG_DIR, which was properly set up during install
-    config_user=$(find "$ESHOME" -maxdepth 0 -printf "%u")
-    config_owner=$(find "$ESHOME" -maxdepth 0 -printf "%g")
-
-    assert_file $directory d $config_user $config_owner 755
-}
-
-assert_module_or_plugin_file() {
-    local file=$1
-    shift
-
-    assert_file_exist "$(readlink -m $file)"
-    assert_file $file f $config_user $config_owner 644
-}
-
-assert_output() {
-    echo "$output" | grep -E "$1"
-}
-
-assert_recursive_ownership() {
-    local directory=$1
-    local user=$2
-    local group=$3
-
-    realuser=$(find $directory -printf "%u\n" | sort | uniq)
-    [ "$realuser" = "$user" ]
-    realgroup=$(find $directory -printf "%g\n" | sort | uniq)
-    [ "$realgroup" = "$group" ]
-}
-
-# Deletes everything before running a test file
-clean_before_test() {
-
-    # List of files to be deleted
-    ELASTICSEARCH_TEST_FILES=("/usr/share/elasticsearch" \
-                            "/etc/elasticsearch" \
-                            "/var/lib/elasticsearch" \
-                            "/var/log/elasticsearch" \
-                            "/tmp/elasticsearch" \
-                            "/etc/default/elasticsearch" \
-                            "/etc/sysconfig/elasticsearch"  \
-                            "/var/run/elasticsearch"  \
-                            "/usr/share/doc/elasticsearch" \
-                            "/tmp/elasticsearch" \
-                            "/usr/lib/systemd/system/elasticsearch.conf" \
-                            "/usr/lib/tmpfiles.d/elasticsearch.conf" \
-                            "/usr/lib/sysctl.d/elasticsearch.conf")
-
-    # Kills all processes of user elasticsearch
-    if id elasticsearch > /dev/null 2>&1; then
-        pkill -u elasticsearch 2>/dev/null || true
-    fi
-
-    # Kills all running Elasticsearch processes
-    ps aux | grep -i "org.elasticsearch.bootstrap.Elasticsearch" | awk {'print $2'} | xargs kill -9 > /dev/null 2>&1 || true
-
-    purge_elasticsearch
-
-    # Removes user & group
-    userdel elasticsearch > /dev/null 2>&1 || true
-    groupdel elasticsearch > /dev/null 2>&1 || true
-
-
-    # Removes all files
-    for d in "${ELASTICSEARCH_TEST_FILES[@]}"; do
-        if [ -e "$d" ]; then
-            rm -rf "$d"
-        fi
-    done
-}
-
-purge_elasticsearch() {
-    # Removes RPM package
-    if is_rpm; then
-        rpm --quiet -e elasticsearch > /dev/null 2>&1 || true
-    fi
-
-    if [ -x "`which yum 2>/dev/null`" ]; then
-        yum remove -y elasticsearch > /dev/null 2>&1 || true
-    fi
-
-    # Removes DEB package
-    if is_dpkg; then
-        dpkg --purge elasticsearch > /dev/null 2>&1 || true
-    fi
-
-    if [ -x "`which apt-get 2>/dev/null`" ]; then
-        apt-get --quiet --yes purge elasticsearch > /dev/null 2>&1 || true
-    fi
-}
-
-# Start elasticsearch and wait for it to come up with a status.
-# $1 - expected status - defaults to green
-start_elasticsearch_service() {
-    local desiredStatus=${1:-green}
-    local index=$2
-
-    run_elasticsearch_service 0
-
-    wait_for_elasticsearch_status $desiredStatus $index
-
-    if [ -r "/tmp/elasticsearch/elasticsearch.pid" ]; then
-        pid=$(cat /tmp/elasticsearch/elasticsearch.pid)
-        [ "x$pid" != "x" ] && [ "$pid" -gt 0 ]
-        echo "Looking for elasticsearch pid...."
-        ps $pid
-    elif is_systemd; then
-        run systemctl is-active elasticsearch.service
-        [ "$status" -eq 0 ]
-
-        run systemctl status elasticsearch.service
-        [ "$status" -eq 0 ]
-
-    elif is_sysvinit; then
-        run service elasticsearch status
-        [ "$status" -eq 0 ]
-    fi
-}
-
-# Start elasticsearch
-# $1 expected status code
-# $2 additional command line args
-run_elasticsearch_service() {
-    local expectedStatus=$1
-    local commandLineArgs=$2
-    # Set the CONF_DIR setting in case we start as a service
-    if [ ! -z "$CONF_DIR" ] ; then
-        if is_dpkg ; then
-            echo "CONF_DIR=$CONF_DIR" >> /etc/default/elasticsearch;
-        elif is_rpm; then
-            echo "CONF_DIR=$CONF_DIR" >> /etc/sysconfig/elasticsearch;
-        fi
-    fi
-
-    if [ -f "/tmp/elasticsearch/bin/elasticsearch" ]; then
-        if [ -z "$CONF_DIR" ]; then
-            local CONF_DIR=""
-            local ES_PATH_CONF=""
-        else
-            local ES_PATH_CONF="-Epath.conf=$CONF_DIR"
-        fi
-        # we must capture the exit code to compare so we don't want to start as background process in case we expect something other than 0
-        local background=""
-        local timeoutCommand=""
-        if [ "$expectedStatus" = 0 ]; then
-            background="-d"
-        else
-            timeoutCommand="timeout 60s "
-        fi
-        # su and the Elasticsearch init script work together to break bats.
-        # sudo isolates bats enough from the init script so everything continues
-        # to tick along
-        run sudo -u elasticsearch bash <<BASH
-# If jayatana is installed then we try to use it. Elasticsearch should ignore it even when we try.
-# If it doesn't ignore it then Elasticsearch will fail to start because of security errors.
-# This line is attempting to emulate the on login behavior of /usr/share/upstart/sessions/jayatana.conf
-[ -f /usr/share/java/jayatanaag.jar ] && export JAVA_TOOL_OPTIONS="-javaagent:/usr/share/java/jayatanaag.jar"
-# And now we can start Elasticsearch normally, in the background (-d) and with a pidfile (-p).
-export ES_JVM_OPTIONS=$ES_JVM_OPTIONS
-export ES_JAVA_OPTS=$ES_JAVA_OPTS
-$timeoutCommand/tmp/elasticsearch/bin/elasticsearch $background -p /tmp/elasticsearch/elasticsearch.pid $ES_PATH_CONF $commandLineArgs
-BASH
-        [ "$status" -eq "$expectedStatus" ]
-    elif is_systemd; then
-        run systemctl daemon-reload
-        [ "$status" -eq 0 ]
-
-        run systemctl enable elasticsearch.service
-        [ "$status" -eq 0 ]
-
-        run systemctl is-enabled elasticsearch.service
-        [ "$status" -eq 0 ]
-
-        run systemctl start elasticsearch.service
-        [ "$status" -eq "$expectedStatus" ]
-
-    elif is_sysvinit; then
-        run service elasticsearch start
-        [ "$status" -eq "$expectedStatus" ]
-    fi
-}
-
-stop_elasticsearch_service() {
-    if [ -r "/tmp/elasticsearch/elasticsearch.pid" ]; then
-        pid=$(cat /tmp/elasticsearch/elasticsearch.pid)
-        [ "x$pid" != "x" ] && [ "$pid" -gt 0 ]
-
-        kill -SIGTERM $pid
-    elif is_systemd; then
-        run systemctl stop elasticsearch.service
-        [ "$status" -eq 0 ]
-
-        run systemctl is-active elasticsearch.service
-        [ "$status" -eq 3 ]
-
-        echo "$output" | grep -E 'inactive|failed'
-
-    elif is_sysvinit; then
-        run service elasticsearch stop
-        [ "$status" -eq 0 ]
-
-        run service elasticsearch status
-        [ "$status" -ne 0 ]
-    fi
-}
-
-# Waits for Elasticsearch to reach some status.
-# $1 - expected status - defaults to green
-wait_for_elasticsearch_status() {
-    local desiredStatus=${1:-green}
-    local index=$2
-
-    echo "Making sure elasticsearch is up..."
-    wget -O - --retry-connrefused --waitretry=1 --timeout=60 --tries 60 http://localhost:9200/_cluster/health || {
-          echo "Looks like elasticsearch never started. Here is its log:"
-          if [ -e "$ESLOG/elasticsearch.log" ]; then
-              cat "$ESLOG/elasticsearch.log"
-          else
-              echo "The elasticsearch log doesn't exist. Maybe /var/log/messages has something:"
-              tail -n20 /var/log/messages
-          fi
-          false
-    }
-
-    if [ -z "index" ]; then
-      echo "Tring to connect to elasticsearch and wait for expected status $desiredStatus..."
-      curl -sS "http://localhost:9200/_cluster/health?wait_for_status=$desiredStatus&timeout=60s&pretty"
-    else
-      echo "Trying to connect to elasticsearch and wait for expected status $desiredStatus for index $index"
-      curl -sS "http://localhost:9200/_cluster/health/$index?wait_for_status=$desiredStatus&timeout=60s&pretty"
-    fi
-    if [ $? -eq 0 ]; then
-        echo "Connected"
-    else
-        echo "Unable to connect to Elastisearch"
-        false
-    fi
-
-    echo "Checking that the cluster health matches the waited for status..."
-    run curl -sS -XGET 'http://localhost:9200/_cat/health?h=status&v=false'
-    if [ "$status" -ne 0 ]; then
-        echo "error when checking cluster health. code=$status output="
-        echo $output
-        false
-    fi
-    echo $output | grep $desiredStatus || {
-        echo "unexpected status:  '$output' wanted '$desiredStatus'"
-        false
-    }
-}
-
-install_elasticsearch_test_scripts() {
-    install_script is_guide.groovy
-    install_script is_guide.mustache
-}
-
-# Executes some basic Elasticsearch tests
-run_elasticsearch_tests() {
-    # TODO this assertion is the same the one made when waiting for
-    # elasticsearch to start
-    run curl -XGET 'http://localhost:9200/_cat/health?h=status&v=false'
-    [ "$status" -eq 0 ]
-    echo "$output" | grep -w "green"
-
-    curl -s -XPOST 'http://localhost:9200/library/book/1?refresh=true&pretty' -d '{
-      "title": "Elasticsearch - The Definitive Guide"
-    }'
-
-    curl -s -XGET 'http://localhost:9200/_count?pretty' |
-      grep \"count\"\ :\ 1
-
-    curl -s -XPOST 'http://localhost:9200/library/book/_count?pretty' -d '{
-      "query": {
-        "script": {
-          "script": {
-            "file": "is_guide",
-            "lang": "groovy"
-          }
-        }
-      }
-    }' | grep \"count\"\ :\ 1
-
-    curl -s -XGET 'http://localhost:9200/library/book/_search/template?pretty' -d '{
-      "file": "is_guide"
-    }' | grep \"total\"\ :\ 1
-
-    curl -s -XDELETE 'http://localhost:9200/_all'
-}
-
-# Move the config directory to another directory and properly chown it.
-move_config() {
-    local oldConfig="$ESCONFIG"
-    export ESCONFIG="${1:-$(mktemp -d -t 'config.XXXX')}"
-    echo "Moving configuration directory from $oldConfig to $ESCONFIG"
-
-    # Move configuration files to the new configuration directory
-    mv "$oldConfig"/* "$ESCONFIG"
-    chown -R elasticsearch:elasticsearch "$ESCONFIG"
-    assert_file_exist "$ESCONFIG/elasticsearch.yml"
-    assert_file_exist "$ESCONFIG/log4j2.properties"
-}
-
-# Copies a script into the Elasticsearch install.
-install_script() {
-    local name=$1
-    mkdir -p $ESSCRIPTS
-    local script="$BATS_TEST_DIRNAME/example/scripts/$name"
-    echo "Installing $script to $ESSCRIPTS"
-    cp $script $ESSCRIPTS
-}
-
-# permissions from the user umask with the executable bit set
-executable_privileges_for_user_from_umask() {
-    local user=$1
-    shift
-
-    echo $((0777 & ~$(sudo -E -u $user sh -c umask) | 0111))
-}
-
-# permissions from the user umask without the executable bit set
-file_privileges_for_user_from_umask() {
-    local user=$1
-    shift
-
-    echo $((0777 & ~$(sudo -E -u $user sh -c umask) & ~0111))
-}
diff --git a/qa/vagrant/src/test/resources/packaging/scripts/plugins.bash b/qa/vagrant/src/test/resources/packaging/scripts/plugins.bash
deleted file mode 100644
index afae743905712..0000000000000
--- a/qa/vagrant/src/test/resources/packaging/scripts/plugins.bash
+++ /dev/null
@@ -1,152 +0,0 @@
-#!/bin/bash
-
-# This file contains some utilities to test the elasticsearch scripts,
-# the .deb/.rpm packages and the SysV/Systemd scripts.
-
-# WARNING: This testing file must be executed as root and can
-# dramatically change your system. It removes the 'elasticsearch'
-# user/group and also many directories. Do not execute this file
-# unless you know exactly what you are doing.
-
-# Licensed to Elasticsearch under one or more contributor
-# license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright
-# ownership. Elasticsearch licenses this file to you under
-# the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Install a plugin an run all the common post installation tests.
-install_plugin() {
-    local name=$1
-    local path="$2"
-
-    assert_file_exist "$path"
-
-    sudo -E -u $ESPLUGIN_COMMAND_USER "$ESHOME/bin/elasticsearch-plugin" install -batch "file://$path"
-
-    assert_file_exist "$ESPLUGINS/$name"
-    assert_file_exist "$ESPLUGINS/$name/plugin-descriptor.properties"
-    #check we did not accidentially create a log file as root as /usr/share/elasticsearch
-    assert_file_not_exist "/usr/share/elasticsearch/logs"
-
-    # At some point installing or removing plugins caused elasticsearch's logs
-    # to be owned by root. This is bad so we want to make sure it doesn't
-    # happen.
-    if [ -e "$ESLOG" ] && [ $(stat "$ESLOG" --format "%U") == "root" ]; then
-        echo "$ESLOG is now owned by root! That'll break logging when elasticsearch tries to start."
-        false
-    fi
-}
-
-install_jvm_plugin() {
-    local name=$1
-    local path="$2"
-    install_plugin $name "$path"
-    assert_file_exist "$ESPLUGINS/$name/$name"*".jar"
-}
-
-# Remove a plugin and make sure its plugin directory is removed.
-remove_plugin() {
-    local name=$1
-
-    echo "Removing $name...."
-    sudo -E -u $ESPLUGIN_COMMAND_USER "$ESHOME/bin/elasticsearch-plugin" remove $name
-
-    assert_file_not_exist "$ESPLUGINS/$name"
-
-    # At some point installing or removing plugins caused elasticsearch's logs
-    # to be owned by root. This is bad so we want to make sure it doesn't
-    # happen.
-    if [ -e "$ESLOG" ] && [ $(stat "$ESLOG" --format "%U") == "root" ]; then
-        echo "$ESLOG is now owned by root! That'll break logging when elasticsearch tries to start."
-        false
-    fi
-}
-
-# Install the jvm-example plugin which fully exercises the special case file
-# placements for non-site plugins.
-install_jvm_example() {
-    local relativePath=${1:-$(readlink -m jvm-example-*.zip)}
-    install_jvm_plugin jvm-example "$relativePath"
-
-    #owner group and permissions vary depending on how es was installed
-    #just make sure that everything is the same as the parent bin dir, which was properly set up during install
-    bin_user=$(find "$ESHOME/bin" -maxdepth 0 -printf "%u")
-    bin_owner=$(find "$ESHOME/bin" -maxdepth 0 -printf "%g")
-    bin_privileges=$(find "$ESHOME/bin" -maxdepth 0 -printf "%m")
-    assert_file "$ESHOME/bin/jvm-example" d $bin_user $bin_owner 755
-    assert_file "$ESHOME/bin/jvm-example/test" f $bin_user $bin_owner 755
-
-    #owner group and permissions vary depending on how es was installed
-    #just make sure that everything is the same as $CONFIG_DIR, which was properly set up during install
-    config_user=$(find "$ESCONFIG" -maxdepth 0 -printf "%u")
-    config_owner=$(find "$ESCONFIG" -maxdepth 0 -printf "%g")
-    # directories should user the user file-creation mask
-    assert_file "$ESCONFIG/jvm-example" d $config_user $config_owner 755
-    assert_file "$ESCONFIG/jvm-example/example.yaml" f $config_user $config_owner 644
-
-    echo "Running jvm-example's bin script...."
-    "$ESHOME/bin/jvm-example/test" | grep test
-}
-
-# Remove the jvm-example plugin which fully exercises the special cases of
-# removing bin and not removing config.
-remove_jvm_example() {
-    remove_plugin jvm-example
-
-    assert_file_not_exist "$ESHOME/bin/jvm-example"
-    assert_file_exist "$ESCONFIG/jvm-example"
-    assert_file_exist "$ESCONFIG/jvm-example/example.yaml"
-}
-
-# Install a plugin with a special prefix. For the most part prefixes are just
-# useful for grouping but the "analysis" prefix is special because all
-# analysis plugins come with a corresponding lucene-analyzers jar.
-# $1 - the prefix
-# $2 - the plugin name
-# $@ - all remaining arguments are jars that must exist in the plugin's
-#      installation directory
-install_and_check_plugin() {
-    local prefix=$1
-    shift
-    local name=$1
-    shift
-
-    if [ "$prefix" == "-" ]; then
-        local fullName="$name"
-    else
-        local fullName="$prefix-$name"
-    fi
-
-    install_jvm_plugin $fullName "$(readlink -m $fullName-*.zip)"
-
-    assert_module_or_plugin_directory "$ESPLUGINS/$fullName"
-
-    if [ $prefix == 'analysis' ]; then
-        assert_module_or_plugin_file "$ESPLUGINS/$fullName/lucene-analyzers-$name-*.jar"
-    fi
-    for file in "$@"; do
-        assert_module_or_plugin_file "$ESPLUGINS/$fullName/$file"
-    done
-}
-
-# Compare a list of plugin names to the plugins in the plugins pom and see if they are the same
-# $1 the file containing the list of plugins we want to compare to
-# $2 description of the source of the plugin list
-compare_plugins_list() {
-    cat $1 | sort > /tmp/plugins
-    ls /elasticsearch/plugins/*/build.gradle | cut -d '/' -f 4 |
-        sort > /tmp/expected
-    echo "Checking plugins from $2 (<) against expected plugins (>):"
-    diff /tmp/expected /tmp/plugins
-}
diff --git a/qa/vagrant/src/test/resources/packaging/scripts/tar.bash b/qa/vagrant/src/test/resources/packaging/scripts/tar.bash
deleted file mode 100644
index 0ea86ddcc6ebc..0000000000000
--- a/qa/vagrant/src/test/resources/packaging/scripts/tar.bash
+++ /dev/null
@@ -1,94 +0,0 @@
-#!/bin/bash
-
-# This file contains some utilities to test the elasticsearch scripts,
-# the .deb/.rpm packages and the SysV/Systemd scripts.
-
-# WARNING: This testing file must be executed as root and can
-# dramatically change your system. It removes the 'elasticsearch'
-# user/group and also many directories. Do not execute this file
-# unless you know exactly what you are doing.
-
-# Licensed to Elasticsearch under one or more contributor
-# license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright
-# ownership. Elasticsearch licenses this file to you under
-# the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-
-# Install the tar.gz archive
-install_archive() {
-    export ESHOME=${1:-/tmp/elasticsearch}
-
-    echo "Unpacking tarball to $ESHOME"
-    rm -rf /tmp/untar
-    mkdir -p /tmp/untar
-    tar -xzpf elasticsearch*.tar.gz -C /tmp/untar
-
-    find /tmp/untar -depth -type d -name 'elasticsearch*' -exec mv {} "$ESHOME" \; > /dev/null
-
-    # ES cannot run as root so create elasticsearch user & group if needed
-    if ! getent group "elasticsearch" > /dev/null 2>&1 ; then
-        if is_dpkg; then
-            addgroup --system "elasticsearch"
-        else
-            groupadd -r "elasticsearch"
-        fi
-    fi
-    if ! id "elasticsearch" > /dev/null 2>&1 ; then
-        if is_dpkg; then
-            adduser --quiet --system --no-create-home --ingroup "elasticsearch" --disabled-password --shell /bin/false "elasticsearch"
-        else
-            useradd --system -M --gid "elasticsearch" --shell /sbin/nologin --comment "elasticsearch user" "elasticsearch"
-        fi
-    fi
-
-    chown -R elasticsearch:elasticsearch "$ESHOME"
-    export_elasticsearch_paths
-}
-
-# Move the unzipped tarball to another location.
-move_elasticsearch() {
-    local oldhome="$ESHOME"
-    export ESHOME="$1"
-    rm -rf "$ESHOME"
-    mv "$oldhome" "$ESHOME"
-    export_elasticsearch_paths
-}
-
-# Export some useful paths.
-export_elasticsearch_paths() {
-    export ESMODULES="$ESHOME/modules"
-    export ESPLUGINS="$ESHOME/plugins"
-    export ESCONFIG="$ESHOME/config"
-    export ESSCRIPTS="$ESCONFIG/scripts"
-    export ESDATA="$ESHOME/data"
-    export ESLOG="$ESHOME/logs"
-}
-
-# Checks that all directories & files are correctly installed
-# after a archive (tar.gz/zip) install
-verify_archive_installation() {
-    assert_file "$ESHOME" d
-    assert_file "$ESHOME/bin" d
-    assert_file "$ESHOME/bin/elasticsearch" f
-    assert_file "$ESHOME/bin/elasticsearch.in.sh" f
-    assert_file "$ESHOME/bin/elasticsearch-plugin" f
-    assert_file "$ESCONFIG" d
-    assert_file "$ESCONFIG/elasticsearch.yml" f
-    assert_file "$ESCONFIG/log4j2.properties" f
-    assert_file "$ESHOME/lib" d
-    assert_file "$ESHOME/NOTICE.txt" f
-    assert_file "$ESHOME/LICENSE.txt" f
-    assert_file "$ESHOME/README.textile" f
-}
diff --git a/qa/vagrant/src/test/resources/packaging/tests/20_tar_package.bats b/qa/vagrant/src/test/resources/packaging/tests/20_tar_package.bats
new file mode 100644
index 0000000000000..6bb4738072093
--- /dev/null
+++ b/qa/vagrant/src/test/resources/packaging/tests/20_tar_package.bats
@@ -0,0 +1,162 @@
+#!/usr/bin/env bats
+
+# This file is used to test the tar gz package.
+
+# WARNING: This testing file must be executed as root and can
+# dramatically change your system. It should only be executed
+# in a throw-away VM like those made by the Vagrantfile at
+# the root of the Elasticsearch source code. This should
+# cause the script to fail if it is executed any other way:
+[ -f /etc/is_vagrant_vm ] || {
+  >&2 echo "must be run on a vagrant VM"
+  exit 1
+}
+
+# The test case can be executed with the Bash Automated
+# Testing System tool available at https://github.com/sstephenson/bats
+# Thanks to Sam Stephenson!
+
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Load test utilities
+load $BATS_UTILS/utils.bash
+load $BATS_UTILS/tar.bash
+load $BATS_UTILS/plugins.bash
+
+setup() {
+    skip_not_tar_gz
+    export ESHOME=/tmp/elasticsearch
+    export_elasticsearch_paths
+}
+
+##################################
+# Install TAR GZ package
+##################################
+@test "[TAR] tar command is available" {
+    # Cleans everything for the 1st execution
+    clean_before_test
+    run tar --version
+    [ "$status" -eq 0 ]
+}
+
+@test "[TAR] archive is available" {
+    count=$(find . -type f -name 'elasticsearch*.tar.gz' | wc -l)
+    [ "$count" -eq 1 ]
+}
+
+@test "[TAR] archive is not installed" {
+    count=$(find /tmp -type d -name 'elasticsearch*' | wc -l)
+    [ "$count" -eq 0 ]
+}
+
+@test "[TAR] install archive" {
+    # Install the archive
+    install_archive
+
+    count=$(find /tmp -type d -name 'elasticsearch*' | wc -l)
+    [ "$count" -eq 1 ]
+
+    # Its simpler to check that the install was correct in this test rather
+    # than in another test because install_archive sets a number of path
+    # variables that verify_archive_installation reads. To separate this into
+    # another test you'd have to recreate the variables.
+    verify_archive_installation
+}
+
+@test "[TAR] verify elasticsearch-plugin list runs without any plugins installed" {
+    # previously this would fail because the archive installations did
+    # not create an empty plugins directory
+    local plugins_list=`$ESHOME/bin/elasticsearch-plugin list`
+    [[ -z $plugins_list ]]
+}
+
+@test "[TAR] elasticsearch fails if java executable is not found" {
+  local JAVA=$(which java)
+
+  sudo chmod -x $JAVA
+  run "$ESHOME/bin/elasticsearch"
+  sudo chmod +x $JAVA
+
+  [ "$status" -eq 1 ]
+  local expected="Could not find any executable java binary. Please install java in your PATH or set JAVA_HOME"
+  [[ "$output" == *"$expected"* ]] || {
+    echo "Expected error message [$expected] but found: $output"
+    false
+  }
+}
+
+##################################
+# Check that Elasticsearch is working
+##################################
+@test "[TAR] test elasticsearch" {
+    # Install scripts used to test script filters and search templates before
+    # starting Elasticsearch so we don't have to wait for elasticsearch to scan for
+    # them.
+    install_elasticsearch_test_scripts
+    start_elasticsearch_service
+    run_elasticsearch_tests
+    stop_elasticsearch_service
+}
+
+@test "[TAR] start Elasticsearch with custom JVM options" {
+    local es_java_opts=$ES_JAVA_OPTS
+    local es_jvm_options=$ES_JVM_OPTIONS
+    local temp=`mktemp -d`
+    touch "$temp/jvm.options"
+    chown -R elasticsearch:elasticsearch "$temp"
+    echo "-Xms512m" >> "$temp/jvm.options"
+    echo "-Xmx512m" >> "$temp/jvm.options"
+    # we have to disable Log4j from using JMX lest it will hit a security
+    # manager exception before we have configured logging; this will fail
+    # startup since we detect usages of logging before it is configured
+    echo "-Dlog4j2.disable.jmx=true" >> "$temp/jvm.options"
+    export ES_JVM_OPTIONS="$temp/jvm.options"
+    export ES_JAVA_OPTS="-XX:-UseCompressedOops"
+    start_elasticsearch_service
+    curl -s -XGET localhost:9200/_nodes | fgrep '"heap_init_in_bytes":536870912'
+    curl -s -XGET localhost:9200/_nodes | fgrep '"using_compressed_ordinary_object_pointers":"false"'
+    stop_elasticsearch_service
+    export ES_JVM_OPTIONS=$es_jvm_options
+    export ES_JAVA_OPTS=$es_java_opts
+}
+
+@test "[TAR] start Elasticsearch with unquoted JSON option" {
+    local es_java_opts=$ES_JAVA_OPTS
+    local es_jvm_options=$ES_JVM_OPTIONS
+    local temp=`mktemp -d`
+    touch "$temp/jvm.options"
+    chown -R elasticsearch:elasticsearch "$temp"
+    echo "-Delasticsearch.json.allow_unquoted_field_names=true" >> "$temp/jvm.options"
+    # we have to disable Log4j from using JMX lest it will hit a security
+    # manager exception before we have configured logging; this will fail
+    # startup since we detect usages of logging before it is configured
+    echo "-Dlog4j2.disable.jmx=true" >> "$temp/jvm.options"
+    export ES_JVM_OPTIONS="$temp/jvm.options"
+    start_elasticsearch_service
+    # unquoted field name
+    curl -s -XPOST localhost:9200/i/d/1 -d'{foo: "bar"}'
+    [ "$?" -eq 0 ]
+    curl -s -XDELETE localhost:9200/i
+    stop_elasticsearch_service
+    export ES_JVM_OPTIONS=$es_jvm_options
+    export ES_JAVA_OPTS=$es_java_opts
+}
+
+@test "[TAR] remove tar" {
+    rm -rf "/tmp/elasticsearch"
+}
diff --git a/qa/vagrant/src/test/resources/packaging/scripts/25_tar_plugins.bats b/qa/vagrant/src/test/resources/packaging/tests/25_tar_plugins.bats
similarity index 100%
rename from qa/vagrant/src/test/resources/packaging/scripts/25_tar_plugins.bats
rename to qa/vagrant/src/test/resources/packaging/tests/25_tar_plugins.bats
diff --git a/qa/vagrant/src/test/resources/packaging/scripts/30_deb_package.bats b/qa/vagrant/src/test/resources/packaging/tests/30_deb_package.bats
similarity index 82%
rename from qa/vagrant/src/test/resources/packaging/scripts/30_deb_package.bats
rename to qa/vagrant/src/test/resources/packaging/tests/30_deb_package.bats
index 2fdeed4d13f0a..b07825d7f2fde 100644
--- a/qa/vagrant/src/test/resources/packaging/scripts/30_deb_package.bats
+++ b/qa/vagrant/src/test/resources/packaging/tests/30_deb_package.bats
@@ -4,9 +4,14 @@
 # of a Debian package.
 
 # WARNING: This testing file must be executed as root and can
-# dramatically change your system. It removes the 'elasticsearch'
-# user/group and also many directories. Do not execute this file
-# unless you know exactly what you are doing.
+# dramatically change your system. It should only be executed
+# in a throw-away VM like those made by the Vagrantfile at
+# the root of the Elasticsearch source code. This should
+# cause the script to fail if it is executed any other way:
+[ -f /etc/is_vagrant_vm ] || {
+  >&2 echo "must be run on a vagrant VM"
+  exit 1
+}
 
 # The test case can be executed with the Bash Automated
 # Testing System tool available at https://github.com/sstephenson/bats
@@ -30,9 +35,9 @@
 # under the License.
 
 # Load test utilities
-load packaging_test_utils
-load os_package
-load plugins
+load $BATS_UTILS/utils.bash
+load $BATS_UTILS/packages.bash
+load $BATS_UTILS/plugins.bash
 
 # Cleans everything for the 1st execution
 setup() {
@@ -74,6 +79,11 @@ setup() {
     verify_package_installation
 }
 
+@test "[DEB] verify elasticsearch-plugin list runs without any plugins installed" {
+    local plugins_list=`$ESHOME/bin/elasticsearch-plugin list`
+    [[ -z $plugins_list ]]
+}
+
 @test "[DEB] elasticsearch isn't started by package install" {
     # Wait a second to give Elasticsearch a change to start if it is going to.
     # This isn't perfect by any means but its something.
@@ -94,6 +104,11 @@ setup() {
     run_elasticsearch_tests
 }
 
+@test "[DEB] verify package installation after start" {
+    # Checks that the startup scripts didn't change the permissions
+    verify_package_installation
+}
+
 ##################################
 # Uninstall DEB package
 ##################################
@@ -127,10 +142,18 @@ setup() {
     # see postrm file
     assert_file_not_exist "/var/log/elasticsearch"
     assert_file_not_exist "/usr/share/elasticsearch/plugins"
+    assert_file_not_exist "/usr/share/elasticsearch/modules"
     assert_file_not_exist "/var/run/elasticsearch"
 
+    # Those directories are removed by the package manager
+    assert_file_not_exist "/usr/share/elasticsearch/bin"
+    assert_file_not_exist "/usr/share/elasticsearch/lib"
+    assert_file_not_exist "/usr/share/elasticsearch/modules"
+    assert_file_not_exist "/usr/share/elasticsearch/modules/lang-painless"
+
     # The configuration files are still here
     assert_file_exist "/etc/elasticsearch"
+    assert_file_exist "/etc/elasticsearch/scripts"
     assert_file_exist "/etc/elasticsearch/elasticsearch.yml"
     assert_file_exist "/etc/elasticsearch/jvm.options"
     assert_file_exist "/etc/elasticsearch/log4j2.properties"
@@ -152,6 +175,7 @@ setup() {
 @test "[DEB] verify package purge" {
     # all remaining files are deleted by the purge
     assert_file_not_exist "/etc/elasticsearch"
+    assert_file_not_exist "/etc/elasticsearch/scripts"
     assert_file_not_exist "/etc/elasticsearch/elasticsearch.yml"
     assert_file_not_exist "/etc/elasticsearch/jvm.options"
     assert_file_not_exist "/etc/elasticsearch/log4j2.properties"
diff --git a/qa/vagrant/src/test/resources/packaging/scripts/40_rpm_package.bats b/qa/vagrant/src/test/resources/packaging/tests/40_rpm_package.bats
similarity index 80%
rename from qa/vagrant/src/test/resources/packaging/scripts/40_rpm_package.bats
rename to qa/vagrant/src/test/resources/packaging/tests/40_rpm_package.bats
index a7aa860370c83..b92b692401e56 100644
--- a/qa/vagrant/src/test/resources/packaging/scripts/40_rpm_package.bats
+++ b/qa/vagrant/src/test/resources/packaging/tests/40_rpm_package.bats
@@ -3,9 +3,14 @@
 # This file is used to test the installation of a RPM package.
 
 # WARNING: This testing file must be executed as root and can
-# dramatically change your system. It removes the 'elasticsearch'
-# user/group and also many directories. Do not execute this file
-# unless you know exactly what you are doing.
+# dramatically change your system. It should only be executed
+# in a throw-away VM like those made by the Vagrantfile at
+# the root of the Elasticsearch source code. This should
+# cause the script to fail if it is executed any other way:
+[ -f /etc/is_vagrant_vm ] || {
+  >&2 echo "must be run on a vagrant VM"
+  exit 1
+}
 
 # The test case can be executed with the Bash Automated
 # Testing System tool available at https://github.com/sstephenson/bats
@@ -29,9 +34,9 @@
 # under the License.
 
 # Load test utilities
-load packaging_test_utils
-load os_package
-load plugins
+load $BATS_UTILS/utils.bash
+load $BATS_UTILS/packages.bash
+load $BATS_UTILS/plugins.bash
 
 # Cleans everything for the 1st execution
 setup() {
@@ -73,6 +78,11 @@ setup() {
     verify_package_installation
 }
 
+@test "[RPM] verify elasticsearch-plugin list runs without any plugins installed" {
+    local plugins_list=`$ESHOME/bin/elasticsearch-plugin list`
+    [[ -z $plugins_list ]]
+}
+
 @test "[RPM] elasticsearch isn't started by package install" {
     # Wait a second to give Elasticsearch a change to start if it is going to.
     # This isn't perfect by any means but its something.
@@ -89,6 +99,11 @@ setup() {
     run_elasticsearch_tests
 }
 
+@test "[RPM] verify package installation after start" {
+    # Checks that the startup scripts didn't change the permissions
+    verify_package_installation
+}
+
 @test "[RPM] remove package" {
     # User installed scripts aren't removed so we'll just get them ourselves
     rm -rf $ESSCRIPTS
@@ -118,10 +133,12 @@ setup() {
     assert_file_not_exist "/usr/share/elasticsearch/plugins"
     assert_file_not_exist "/var/run/elasticsearch"
 
+    # Those directories are removed by the package manager
+    assert_file_not_exist "/usr/share/elasticsearch/bin"
+    assert_file_not_exist "/usr/share/elasticsearch/lib"
+    assert_file_not_exist "/usr/share/elasticsearch/modules"
+
     assert_file_not_exist "/etc/elasticsearch"
-    assert_file_not_exist "/etc/elasticsearch/elasticsearch.yml"
-    assert_file_not_exist "/etc/elasticsearch/jvm.options"
-    assert_file_not_exist "/etc/elasticsearch/log4j2.properties"
 
     assert_file_not_exist "/etc/init.d/elasticsearch"
     assert_file_not_exist "/usr/lib/systemd/system/elasticsearch.service"
@@ -137,6 +154,10 @@ setup() {
     rpm -qe 'elasticsearch'
 }
 
+@test "[RPM] verify package reinstallation" {
+    verify_package_installation
+}
+
 @test "[RPM] reremove package" {
     echo "# ping" >> "/etc/elasticsearch/elasticsearch.yml"
     echo "# ping" >> "/etc/elasticsearch/jvm.options"
@@ -157,8 +178,14 @@ setup() {
     # see postrm file
     assert_file_not_exist "/var/log/elasticsearch"
     assert_file_not_exist "/usr/share/elasticsearch/plugins"
+    assert_file_not_exist "/usr/share/elasticsearch/modules"
     assert_file_not_exist "/var/run/elasticsearch"
 
+    assert_file_not_exist "/usr/share/elasticsearch/bin"
+    assert_file_not_exist "/usr/share/elasticsearch/lib"
+    assert_file_not_exist "/usr/share/elasticsearch/modules"
+    assert_file_not_exist "/usr/share/elasticsearch/modules/lang-painless"
+
     assert_file_not_exist "/etc/elasticsearch/elasticsearch.yml"
     assert_file_exist "/etc/elasticsearch/elasticsearch.yml.rpmsave"
     assert_file_not_exist "/etc/elasticsearch/jvm.options"
diff --git a/qa/vagrant/src/test/resources/packaging/scripts/50_modules_and_plugins.bats b/qa/vagrant/src/test/resources/packaging/tests/50_modules_and_plugins.bats
similarity index 100%
rename from qa/vagrant/src/test/resources/packaging/scripts/50_modules_and_plugins.bats
rename to qa/vagrant/src/test/resources/packaging/tests/50_modules_and_plugins.bats
diff --git a/qa/vagrant/src/test/resources/packaging/tests/60_systemd.bats b/qa/vagrant/src/test/resources/packaging/tests/60_systemd.bats
new file mode 100644
index 0000000000000..b2f3ef06d4258
--- /dev/null
+++ b/qa/vagrant/src/test/resources/packaging/tests/60_systemd.bats
@@ -0,0 +1,220 @@
+#!/usr/bin/env bats
+
+# This file is used to test the elasticsearch Systemd setup.
+
+# WARNING: This testing file must be executed as root and can
+# dramatically change your system. It should only be executed
+# in a throw-away VM like those made by the Vagrantfile at
+# the root of the Elasticsearch source code. This should
+# cause the script to fail if it is executed any other way:
+[ -f /etc/is_vagrant_vm ] || {
+  >&2 echo "must be run on a vagrant VM"
+  exit 1
+}
+
+# The test case can be executed with the Bash Automated
+# Testing System tool available at https://github.com/sstephenson/bats
+# Thanks to Sam Stephenson!
+
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Load test utilities
+load $BATS_UTILS/utils.bash
+load $BATS_UTILS/packages.bash
+load $BATS_UTILS/plugins.bash
+
+# Cleans everything for the 1st execution
+setup() {
+    skip_not_systemd
+    skip_not_dpkg_or_rpm
+    export_elasticsearch_paths
+}
+
+@test "[SYSTEMD] install elasticsearch" {
+    clean_before_test
+    install_package
+}
+
+@test "[SYSTEMD] daemon reload after install" {
+    systemctl daemon-reload
+}
+
+@test "[SYSTEMD] daemon isn't enabled on restart" {
+    # Rather than restart the VM we just ask systemd if it plans on starting
+    # elasticsearch on restart. Not as strong as a restart but much much
+    # faster.
+    run systemctl is-enabled elasticsearch.service
+    [ "$output" = "disabled" ]
+}
+
+@test "[SYSTEMD] enable" {
+    systemctl enable elasticsearch.service
+
+    systemctl is-enabled elasticsearch.service
+}
+
+@test "[SYSTEMD] start" {
+    # Install scripts used to test script filters and search templates before
+    # starting Elasticsearch so we don't have to wait for elasticsearch to scan for
+    # them.
+    install_elasticsearch_test_scripts
+
+    # Capture the current epoch in millis
+    run date +%s
+    epoch="$output"
+
+    # The OpenJDK packaged for CentOS and OEL both override the default value (false) for the JVM option "AssumeMP".
+    #
+    # Because it is forced to "true" by default for these packages, the following warning message is printed to the
+    # standard output when the Vagrant box has only 1 CPU:
+    #       OpenJDK 64-Bit Server VM warning: If the number of processors is expected to increase from one, then you should configure
+    #       the number of parallel GC threads appropriately using -XX:ParallelGCThreads=N
+    #
+    # This message will then fail the next test where we check if no entries have been added to the journal.
+    #
+    # This message appears since with java-1.8.0-openjdk-1.8.0.111-1.b15.el7_2.x86_64 because of the commit:
+    #       2016-10-10  - Andrew Hughes <gnu.andrew@redhat.com> - 1:1.8.0.111-1.b15 - Turn debug builds on for all JIT architectures.
+    #                     Always AssumeMP on RHEL.
+    #                   - Resolves: rhbz#1381990
+    #
+    if [ -x "$(command -v lsb_release)" ]; then
+        # Here we set the "-XX:-AssumeMP" option to false again:
+        lsb_release=$(lsb_release -i)
+        if [[ "$lsb_release" =~ "CentOS" ]] || [[ "$lsb_release" =~ "OracleServer" ]]; then
+            echo "-XX:-AssumeMP" >> $ESCONFIG/jvm.options
+        fi
+    fi
+
+    systemctl start elasticsearch.service
+    wait_for_elasticsearch_status
+    assert_file_exist "/var/run/elasticsearch/elasticsearch.pid"
+    assert_file_exist "/var/log/elasticsearch/elasticsearch.log"
+
+    # Converts the epoch back in a human readable format
+    run date --date=@$epoch "+%Y-%m-%d %H:%M:%S"
+    since="$output"
+
+    # Verifies that no new entries in journald have been added
+    # since the last start
+    result="$(journalctl _SYSTEMD_UNIT=elasticsearch.service --since "$since" --output cat | wc -l)"
+    [ "$result" -eq "0" ] || {
+            echo "Expected no entries in journalctl for the Elasticsearch service but found:"
+            journalctl _SYSTEMD_UNIT=elasticsearch.service --since "$since"
+            false
+        }
+}
+
+@test "[SYSTEMD] start (running)" {
+    systemctl start elasticsearch.service
+}
+
+@test "[SYSTEMD] is active (running)" {
+    run systemctl is-active elasticsearch.service
+    [ "$status" -eq 0 ]
+    [ "$output" = "active" ]
+}
+
+@test "[SYSTEMD] status (running)" {
+    systemctl status elasticsearch.service
+}
+
+##################################
+# Check that Elasticsearch is working
+##################################
+@test "[SYSTEMD] test elasticsearch" {
+    run_elasticsearch_tests
+}
+
+@test "[SYSTEMD] restart" {
+    systemctl restart elasticsearch.service
+
+    wait_for_elasticsearch_status
+
+    service elasticsearch status
+}
+
+@test "[SYSTEMD] stop (running)" {
+    systemctl stop elasticsearch.service
+}
+
+@test "[SYSTEMD] status (stopping)" {
+    run systemctl status elasticsearch.service
+    # I'm not sure why suse exits 0 here, but it does
+    if [ ! -e /etc/SuSE-release ]; then
+        [ "$status" -eq 3 ] || "Expected exit code 3 meaning stopped but got $status"
+    fi
+    echo "$output" | grep "Active:" | grep "inactive"
+}
+
+@test "[SYSTEMD] stop (stopped)" {
+    systemctl stop elasticsearch.service
+}
+
+@test "[SYSTEMD] status (stopped)" {
+    run systemctl status elasticsearch.service
+    # I'm not sure why suse exits 0 here, but it does
+    if [ ! -e /etc/SuSE-release ]; then
+        [ "$status" -eq 3 ] || "Expected exit code 3 meaning stopped but got $status"
+    fi
+    echo "$output" | grep "Active:" | grep "inactive"
+}
+
+# Simulates the behavior of a system restart:
+# the PID directory is deleted by the operating system
+# but it should not block ES from starting
+# see https://github.com/elastic/elasticsearch/issues/11594
+@test "[SYSTEMD] delete PID_DIR and restart" {
+    rm -rf /var/run/elasticsearch
+
+    systemd-tmpfiles --create
+
+    systemctl start elasticsearch.service
+
+    wait_for_elasticsearch_status
+
+    assert_file_exist "/var/run/elasticsearch/elasticsearch.pid"
+
+    systemctl stop elasticsearch.service
+}
+
+@test "[SYSTEMD] masking systemd-sysctl" {
+    clean_before_test
+
+    systemctl mask systemd-sysctl.service
+    install_package
+
+    systemctl unmask systemd-sysctl.service
+}
+
+@test "[SYSTEMD] service file sets limits" {
+    clean_before_test
+    install_package
+    systemctl start elasticsearch.service
+    wait_for_elasticsearch_status
+    local pid=$(cat /var/run/elasticsearch/elasticsearch.pid)
+    local max_file_size=$(cat /proc/$pid/limits | grep "Max file size" | awk '{ print $4 }')
+    [ "$max_file_size" == "unlimited" ]
+    local max_processes=$(cat /proc/$pid/limits | grep "Max processes" | awk '{ print $3 }')
+    [ "$max_processes" == "2048" ]
+    local max_open_files=$(cat /proc/$pid/limits | grep "Max open files" | awk '{ print $4 }')
+    [ "$max_open_files" == "65536" ]
+    local max_address_space=$(cat /proc/$pid/limits | grep "Max address space" | awk '{ print $4 }')
+    [ "$max_address_space" == "unlimited" ]
+    systemctl stop elasticsearch.service
+}
+
diff --git a/qa/vagrant/src/test/resources/packaging/tests/70_sysv_initd.bats b/qa/vagrant/src/test/resources/packaging/tests/70_sysv_initd.bats
new file mode 100644
index 0000000000000..fa6f91498ca02
--- /dev/null
+++ b/qa/vagrant/src/test/resources/packaging/tests/70_sysv_initd.bats
@@ -0,0 +1,184 @@
+#!/usr/bin/env bats
+
+# This file is used to test the elasticsearch init.d scripts.
+
+# WARNING: This testing file must be executed as root and can
+# dramatically change your system. It should only be executed
+# in a throw-away VM like those made by the Vagrantfile at
+# the root of the Elasticsearch source code. This should
+# cause the script to fail if it is executed any other way:
+[ -f /etc/is_vagrant_vm ] || {
+  >&2 echo "must be run on a vagrant VM"
+  exit 1
+}
+
+# The test case can be executed with the Bash Automated
+# Testing System tool available at https://github.com/sstephenson/bats
+# Thanks to Sam Stephenson!
+
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Load test utilities
+load $BATS_UTILS/utils.bash
+load $BATS_UTILS/packages.bash
+load $BATS_UTILS/plugins.bash
+
+# Cleans everything for the 1st execution
+setup() {
+    skip_not_sysvinit
+    skip_not_dpkg_or_rpm
+    export_elasticsearch_paths
+}
+
+@test "[INIT.D] remove any leftover configuration to start elasticsearch on restart" {
+    # This configuration can be added with a command like:
+    # $ sudo update-rc.d elasticsearch defaults 95 10
+    # but we want to test that the RPM and deb _don't_ add it on its own.
+    # Note that it'd be incorrect to use:
+    # $ sudo update-rc.d elasticsearch disable
+    # here because that'd prevent elasticsearch from installing the symlinks
+    # that cause it to be started on restart.
+    sudo update-rc.d -f elasticsearch remove || true
+    sudo chkconfig elasticsearch off || true
+}
+
+@test "[INIT.D] install elasticsearch" {
+    clean_before_test
+    install_package
+}
+
+@test "[INIT.D] elasticsearch fails if startup script is not executable" {
+    local INIT="/etc/init.d/elasticsearch"
+    local DAEMON="$ESHOME/bin/elasticsearch"
+
+    sudo chmod -x "$DAEMON"
+    run "$INIT"
+    sudo chmod +x "$DAEMON"
+
+    [ "$status" -eq 1 ]
+    [[ "$output" == *"The elasticsearch startup script does not exists or it is not executable, tried: $DAEMON"* ]]
+}
+
+@test "[INIT.D] daemon isn't enabled on restart" {
+    # Rather than restart the VM which would be slow we check for the symlinks
+    # that init.d uses to restart the application on startup.
+    ! find /etc/rc[0123456].d | grep elasticsearch
+    # Note that we don't use -iname above because that'd have to look like:
+    # [ $(find /etc/rc[0123456].d -iname "elasticsearch*" | wc -l) -eq 0 ]
+    # Which isn't really clearer than what we do use.
+}
+
+@test "[INIT.D] start" {
+    # Install scripts used to test script filters and search templates before
+    # starting Elasticsearch so we don't have to wait for elasticsearch to scan for
+    # them.
+    install_elasticsearch_test_scripts
+    service elasticsearch start
+    wait_for_elasticsearch_status
+    assert_file_exist "/var/run/elasticsearch/elasticsearch.pid"
+}
+
+@test "[INIT.D] status (running)" {
+    service elasticsearch status
+}
+
+##################################
+# Check that Elasticsearch is working
+##################################
+@test "[INIT.D] test elasticsearch" {
+    run_elasticsearch_tests
+}
+
+@test "[INIT.D] restart" {
+    service elasticsearch restart
+
+    wait_for_elasticsearch_status
+
+    service elasticsearch status
+}
+
+@test "[INIT.D] stop (running)" {
+    service elasticsearch stop
+}
+
+@test "[INIT.D] status (stopped)" {
+    run service elasticsearch status
+    # precise returns 4, trusty 3
+    [ "$status" -eq 3 ] || [ "$status" -eq 4 ]
+}
+
+@test "[INIT.D] don't mkdir when it contains a comma" {
+    # Remove these just in case they exist beforehand
+    rm -rf /tmp/aoeu,/tmp/asdf
+    rm -rf /tmp/aoeu,
+    # set DATA_DIR to DATA_DIR=/tmp/aoeu,/tmp/asdf
+    sed -i 's/DATA_DIR=.*/DATA_DIR=\/tmp\/aoeu,\/tmp\/asdf/' /etc/init.d/elasticsearch
+    cat /etc/init.d/elasticsearch | grep "DATA_DIR"
+    run service elasticsearch start
+    if [ "$status" -ne 0 ]; then
+      cat /var/log/elasticsearch/*
+      fail
+    fi
+    wait_for_elasticsearch_status
+    assert_file_not_exist /tmp/aoeu,/tmp/asdf
+    assert_file_not_exist /tmp/aoeu,
+    service elasticsearch stop
+    run service elasticsearch status
+    # precise returns 4, trusty 3
+    [ "$status" -eq 3 ] || [ "$status" -eq 4 ]
+}
+
+@test "[INIT.D] start Elasticsearch with custom JVM options" {
+    assert_file_exist $ESENVFILE
+    local es_java_opts=$ES_JAVA_OPTS
+    local es_jvm_options=$ES_JVM_OPTIONS
+    local temp=`mktemp -d`
+    touch "$temp/jvm.options"
+    chown -R elasticsearch:elasticsearch "$temp"
+    echo "-Xms512m" >> "$temp/jvm.options"
+    echo "-Xmx512m" >> "$temp/jvm.options"
+    # we have to disable Log4j from using JMX lest it will hit a security
+    # manager exception before we have configured logging; this will fail
+    # startup since we detect usages of logging before it is configured
+    echo "-Dlog4j2.disable.jmx=true" >> "$temp/jvm.options"
+    cp $ESENVFILE "$temp/elasticsearch"
+    echo "ES_JVM_OPTIONS=\"$temp/jvm.options\"" >> $ESENVFILE
+    echo "ES_JAVA_OPTS=\"-XX:-UseCompressedOops\"" >> $ESENVFILE
+    service elasticsearch start
+    wait_for_elasticsearch_status
+    curl -s -XGET localhost:9200/_nodes | fgrep '"heap_init_in_bytes":536870912'
+    curl -s -XGET localhost:9200/_nodes | fgrep '"using_compressed_ordinary_object_pointers":"false"'
+    service elasticsearch stop
+    cp "$temp/elasticsearch" $ESENVFILE
+}
+
+# Simulates the behavior of a system restart:
+# the PID directory is deleted by the operating system
+# but it should not block ES from starting
+# see https://github.com/elastic/elasticsearch/issues/11594
+@test "[INIT.D] delete PID_DIR and restart" {
+    rm -rf /var/run/elasticsearch
+
+    service elasticsearch start
+
+    wait_for_elasticsearch_status
+
+    assert_file_exist "/var/run/elasticsearch/elasticsearch.pid"
+
+    service elasticsearch stop
+}
diff --git a/qa/vagrant/src/test/resources/packaging/tests/75_bad_data_paths.bats b/qa/vagrant/src/test/resources/packaging/tests/75_bad_data_paths.bats
new file mode 100644
index 0000000000000..59747bd68375f
--- /dev/null
+++ b/qa/vagrant/src/test/resources/packaging/tests/75_bad_data_paths.bats
@@ -0,0 +1,82 @@
+#!/usr/bin/env bats
+
+# Tests data.path settings which in the past have misbehaving, leaking the
+# default.data.path setting into the data.path even when it doesn't belong.
+
+# WARNING: This testing file must be executed as root and can
+# dramatically change your system. It should only be executed
+# in a throw-away VM like those made by the Vagrantfile at
+# the root of the Elasticsearch source code. This should
+# cause the script to fail if it is executed any other way:
+[ -f /etc/is_vagrant_vm ] || {
+  >&2 echo "must be run on a vagrant VM"
+  exit 1
+}
+
+# The test case can be executed with the Bash Automated
+# Testing System tool available at https://github.com/sstephenson/bats
+# Thanks to Sam Stephenson!
+
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Load test utilities
+load $BATS_UTILS/packages.bash
+load $BATS_UTILS/tar.bash
+load $BATS_UTILS/utils.bash
+
+@test "[BAD data.path] install package" {
+    clean_before_test
+    skip_not_dpkg_or_rpm
+    install_package
+}
+
+@test "[BAD data.path] setup funny path.data in package install" {
+    skip_not_dpkg_or_rpm
+    local temp=`mktemp -d`
+    chown elasticsearch:elasticsearch "$temp"
+    echo "path.data: [$temp]" > "/etc/elasticsearch/elasticsearch.yml"
+}
+
+@test "[BAD data.path] start installed from package" {
+    skip_not_dpkg_or_rpm
+    start_elasticsearch_service green
+}
+
+@test "[BAD data.path] check for bad dir after starting from package" {
+    skip_not_dpkg_or_rpm
+    assert_file_not_exist /var/lib/elasticsearch/nodes
+}
+
+@test "[BAD data.path] install tar" {
+    clean_before_test
+    install_archive
+}
+
+@test "[BAD data.path] setup funny path.data in tar install" {
+    local temp=`mktemp -d`
+    chown elasticsearch:elasticsearch "$temp"
+    echo "path.data: [$temp]" > "/tmp/elasticsearch/config/elasticsearch.yml"
+}
+
+@test "[BAD data.path] start installed from tar" {
+    start_elasticsearch_service green "" "-Edefault.path.data=/tmp/elasticsearch/data"
+}
+
+@test "[BAD data.path] check for bad dir after starting from tar" {
+    assert_file_not_exist "/tmp/elasticsearch/data/nodes"
+}
diff --git a/qa/vagrant/src/test/resources/packaging/scripts/80_upgrade.bats b/qa/vagrant/src/test/resources/packaging/tests/80_upgrade.bats
similarity index 88%
rename from qa/vagrant/src/test/resources/packaging/scripts/80_upgrade.bats
rename to qa/vagrant/src/test/resources/packaging/tests/80_upgrade.bats
index 48bf2aca4e383..99a5eb8f76731 100644
--- a/qa/vagrant/src/test/resources/packaging/scripts/80_upgrade.bats
+++ b/qa/vagrant/src/test/resources/packaging/tests/80_upgrade.bats
@@ -5,9 +5,14 @@
 # fancy rolling restarts.
 
 # WARNING: This testing file must be executed as root and can
-# dramatically change your system. It removes the 'elasticsearch'
-# user/group and also many directories. Do not execute this file
-# unless you know exactly what you are doing.
+# dramatically change your system. It should only be executed
+# in a throw-away VM like those made by the Vagrantfile at
+# the root of the Elasticsearch source code. This should
+# cause the script to fail if it is executed any other way:
+[ -f /etc/is_vagrant_vm ] || {
+  >&2 echo "must be run on a vagrant VM"
+  exit 1
+}
 
 # The test case can be executed with the Bash Automated
 # Testing System tool available at https://github.com/sstephenson/bats
@@ -31,8 +36,8 @@
 # under the License.
 
 # Load test utilities
-load packaging_test_utils
-load os_package
+load $BATS_UTILS/utils.bash
+load $BATS_UTILS/packages.bash
 
 # Cleans everything for the 1st execution
 setup() {
@@ -49,6 +54,7 @@ setup() {
 }
 
 @test "[UPGRADE] check elasticsearch version is old version" {
+    echo "Expecting version \"$(cat upgrade_from_version)\""
     curl -s localhost:9200 | grep \"number\"\ :\ \"$(cat upgrade_from_version)\" || {
         echo "Installed an unexpected version:"
         curl -s localhost:9200
diff --git a/qa/vagrant/src/test/resources/packaging/scripts/90_reinstall.bats b/qa/vagrant/src/test/resources/packaging/tests/90_reinstall.bats
similarity index 85%
rename from qa/vagrant/src/test/resources/packaging/scripts/90_reinstall.bats
rename to qa/vagrant/src/test/resources/packaging/tests/90_reinstall.bats
index 3c2f7be7330c3..816247f95f93b 100644
--- a/qa/vagrant/src/test/resources/packaging/scripts/90_reinstall.bats
+++ b/qa/vagrant/src/test/resources/packaging/tests/90_reinstall.bats
@@ -5,9 +5,14 @@
 # fancy rolling restarts.
 
 # WARNING: This testing file must be executed as root and can
-# dramatically change your system. It removes the 'elasticsearch'
-# user/group and also many directories. Do not execute this file
-# unless you know exactly what you are doing.
+# dramatically change your system. It should only be executed
+# in a throw-away VM like those made by the Vagrantfile at
+# the root of the Elasticsearch source code. This should
+# cause the script to fail if it is executed any other way:
+[ -f /etc/is_vagrant_vm ] || {
+  >&2 echo "must be run on a vagrant VM"
+  exit 1
+}
 
 # The test case can be executed with the Bash Automated
 # Testing System tool available at https://github.com/sstephenson/bats
@@ -31,8 +36,8 @@
 # under the License.
 
 # Load test utilities
-load packaging_test_utils
-load os_package
+load $BATS_UTILS/utils.bash
+load $BATS_UTILS/packages.bash
 
 # Cleans everything for the 1st execution
 setup() {
diff --git a/qa/vagrant/src/test/resources/packaging/scripts/example/scripts/is_guide.groovy b/qa/vagrant/src/test/resources/packaging/tests/example/scripts/is_guide.groovy
similarity index 100%
rename from qa/vagrant/src/test/resources/packaging/scripts/example/scripts/is_guide.groovy
rename to qa/vagrant/src/test/resources/packaging/tests/example/scripts/is_guide.groovy
diff --git a/qa/vagrant/src/test/resources/packaging/scripts/example/scripts/is_guide.mustache b/qa/vagrant/src/test/resources/packaging/tests/example/scripts/is_guide.mustache
similarity index 100%
rename from qa/vagrant/src/test/resources/packaging/scripts/example/scripts/is_guide.mustache
rename to qa/vagrant/src/test/resources/packaging/tests/example/scripts/is_guide.mustache
diff --git a/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash b/qa/vagrant/src/test/resources/packaging/tests/module_and_plugin_test_cases.bash
similarity index 86%
rename from qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
rename to qa/vagrant/src/test/resources/packaging/tests/module_and_plugin_test_cases.bash
index b44e5885ff89e..b4be8f7d6c983 100644
--- a/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
+++ b/qa/vagrant/src/test/resources/packaging/tests/module_and_plugin_test_cases.bash
@@ -5,9 +5,14 @@
 # rpm, and deb.
 
 # WARNING: This testing file must be executed as root and can
-# dramatically change your system. It removes the 'elasticsearch'
-# user/group and also many directories. Do not execute this file
-# unless you know exactly what you are doing.
+# dramatically change your system. It should only be executed
+# in a throw-away VM like those made by the Vagrantfile at
+# the root of the Elasticsearch source code. This should
+# cause the script to fail if it is executed any other way:
+[ -f /etc/is_vagrant_vm ] || {
+  >&2 echo "must be run on a vagrant VM"
+  exit 1
+}
 
 # The test case can be executed with the Bash Automated
 # Testing System tool available at https://github.com/sstephenson/bats
@@ -39,9 +44,9 @@
 # system uses.
 
 # Load test utilities
-load packaging_test_utils
-load modules
-load plugins
+load $BATS_UTILS/utils.bash
+load $BATS_UTILS/modules.bash
+load $BATS_UTILS/plugins.bash
 
 setup() {
     # The rules on when we should clean an reinstall are complex - all the
@@ -60,7 +65,7 @@ setup() {
 }
 
 if [[ "$BATS_TEST_FILENAME" =~ 25_tar_plugins.bats$ ]]; then
-    load tar
+    load $BATS_UTILS/tar.bash
     GROUP='TAR PLUGINS'
     install() {
         install_archive
@@ -70,7 +75,7 @@ if [[ "$BATS_TEST_FILENAME" =~ 25_tar_plugins.bats$ ]]; then
     export_elasticsearch_paths
     export ESPLUGIN_COMMAND_USER=elasticsearch
 else
-    load os_package
+    load $BATS_UTILS/packages.bash
     if is_rpm; then
         GROUP='RPM PLUGINS'
     elif is_dpkg; then
@@ -142,7 +147,7 @@ fi
     move_config
 
     CONF_DIR="$ESCONFIG" install_jvm_example
-    CONF_DIR="$ESCONFIG" start_elasticsearch_service
+    CONF_DIR="$ESCONFIG" ES_JVM_OPTIONS="$ESCONFIG/jvm.options" start_elasticsearch_service
     diff  <(curl -s localhost:9200/_cat/configured_example | sed 's/ //g') <(echo "foo")
     stop_elasticsearch_service
     CONF_DIR="$ESCONFIG" remove_jvm_example
@@ -176,7 +181,11 @@ fi
   sudo chmod +x $JAVA
 
   [ "$status" -eq 1 ]
-  [[ "$output" == *"Could not find any executable java binary. Please install java in your PATH or set JAVA_HOME"* ]]
+  local expected="Could not find any executable java binary. Please install java in your PATH or set JAVA_HOME"
+  [[ "$output" == *"$expected"* ]] || {
+    echo "Expected error message [$expected] but found: $output"
+    false
+  }
 }
 
 # Note that all of the tests from here to the end of the file expect to be run
@@ -205,6 +214,10 @@ fi
     install_and_check_plugin analysis stempel
 }
 
+@test "[$GROUP] install ukrainian plugin" {
+    install_and_check_plugin analysis ukrainian morfologik-fsa-*.jar morfologik-stemming-*.jar
+}
+
 @test "[$GROUP] install gce plugin" {
     install_and_check_plugin discovery gce google-api-client-*.jar
 }
@@ -217,11 +230,15 @@ fi
     install_and_check_plugin discovery ec2 aws-java-sdk-core-*.jar
 }
 
+@test "[$GROUP] install discovery-file plugin" {
+    install_and_check_plugin discovery file
+}
+
 @test "[$GROUP] install ingest-attachment plugin" {
-    # we specify the version on the poi-3.15-beta1.jar so that the test does
+    # we specify the version on the poi-3.15.jar so that the test does
     # not spuriously pass if the jar is missing but the other poi jars
     # are present
-    install_and_check_plugin ingest attachment bcprov-jdk15on-*.jar tika-core-*.jar pdfbox-*.jar poi-3.15-beta1.jar poi-ooxml-3.15-beta1.jar poi-ooxml-schemas-*.jar poi-scratchpad-*.jar
+    install_and_check_plugin ingest attachment bcprov-jdk15on-*.jar tika-core-*.jar pdfbox-*.jar poi-3.15.jar poi-ooxml-3.15.jar poi-ooxml-schemas-*.jar poi-scratchpad-*.jar
 }
 
 @test "[$GROUP] install ingest-geoip plugin" {
@@ -276,7 +293,7 @@ fi
 }
 
 @test "[$GROUP] install repository-hdfs plugin" {
-    install_and_check_plugin repository hdfs hadoop-client-*.jar hadoop-common-*.jar hadoop-annotations-*.jar hadoop-auth-*.jar hadoop-hdfs-*.jar htrace-core-*.jar guava-*.jar protobuf-java-*.jar commons-logging-*.jar commons-cli-*.jar commons-collections-*.jar commons-configuration-*.jar commons-io-*.jar commons-lang-*.jar servlet-api-*.jar slf4j-api-*.jar
+    install_and_check_plugin repository hdfs hadoop-client-*.jar hadoop-common-*.jar hadoop-annotations-*.jar hadoop-auth-*.jar hadoop-hdfs-client-*.jar htrace-core4-*.jar guava-*.jar protobuf-java-*.jar commons-logging-*.jar commons-cli-*.jar commons-collections-*.jar commons-configuration-*.jar commons-io-*.jar commons-lang-*.jar servlet-api-*.jar slf4j-api-*.jar
 }
 
 @test "[$GROUP] install size mapper plugin" {
@@ -300,7 +317,7 @@ fi
 }
 
 @test "[$GROUP] check the installed plugins can be listed with 'plugins list' and result matches the list of plugins in plugins pom" {
-    "$ESHOME/bin/elasticsearch-plugin" list > /tmp/installed
+    "$ESHOME/bin/elasticsearch-plugin" list | cut -d'@' -f1 > /tmp/installed
     compare_plugins_list "/tmp/installed" "'plugins list'"
 }
 
@@ -341,6 +358,10 @@ fi
     remove_plugin analysis-stempel
 }
 
+@test "[$GROUP] remove ukrainian plugin" {
+    remove_plugin analysis-ukrainian
+}
+
 @test "[$GROUP] remove gce plugin" {
     remove_plugin discovery-gce
 }
@@ -353,6 +374,10 @@ fi
     remove_plugin discovery-ec2
 }
 
+@test "[$GROUP] remove discovery-file plugin" {
+    remove_plugin discovery-file
+}
+
 @test "[$GROUP] remove ingest-attachment plugin" {
     remove_plugin ingest-attachment
 }
@@ -428,38 +453,21 @@ fi
     sudo -E -u $ESPLUGIN_COMMAND_USER "$ESHOME/bin/elasticsearch-plugin" install "file://$relativePath" > /tmp/plugin-cli-output
     # exclude progress line
     local loglines=$(cat /tmp/plugin-cli-output | grep -v "^[[:cntrl:]]" | wc -l)
-    if [ "$GROUP" == "TAR PLUGINS" ]; then
-    # tar extraction does not create the plugins directory so the plugin tool will print an additional line that the directory will be created
-        [ "$loglines" -eq "3" ] || {
-            echo "Expected 3 lines excluding progress bar but the output had $loglines lines and was:"
-            cat /tmp/plugin-cli-output
-            false
-        }
-    else
-        [ "$loglines" -eq "2" ] || {
-            echo "Expected 2 lines excluding progress bar but the output had $loglines lines and was:"
-            cat /tmp/plugin-cli-output
-            false
-        }
-    fi
+    [ "$loglines" -eq "2" ] || {
+        echo "Expected 2 lines excluding progress bar but the output had $loglines lines and was:"
+        cat /tmp/plugin-cli-output
+        false
+    }
     remove_jvm_example
 
     local relativePath=${1:-$(readlink -m jvm-example-*.zip)}
     sudo -E -u $ESPLUGIN_COMMAND_USER ES_JAVA_OPTS="-Des.logger.level=DEBUG" "$ESHOME/bin/elasticsearch-plugin" install "file://$relativePath" > /tmp/plugin-cli-output
     local loglines=$(cat /tmp/plugin-cli-output | grep -v "^[[:cntrl:]]" | wc -l)
-    if [ "$GROUP" == "TAR PLUGINS" ]; then
-        [ "$loglines" -gt "3" ] || {
-            echo "Expected more than 3 lines excluding progress bar but the output had $loglines lines and was:"
-            cat /tmp/plugin-cli-output
-            false
-        }
-    else
-        [ "$loglines" -gt "2" ] || {
-            echo "Expected more than 2 lines excluding progress bar but the output had $loglines lines and was:"
-            cat /tmp/plugin-cli-output
-            false
-        }
-    fi
+    [ "$loglines" -gt "2" ] || {
+        echo "Expected more than 2 lines excluding progress bar but the output had $loglines lines and was:"
+        cat /tmp/plugin-cli-output
+        false
+    }
     remove_jvm_example
 }
 
@@ -495,3 +503,7 @@ fi
     # restore ES_JAVA_OPTS
     export ES_JAVA_OPTS=$es_java_opts
 }
+
+@test "[$GROUP] test umask" {
+    install_jvm_example $(readlink -m jvm-example-*.zip) 0077
+}
diff --git a/qa/vagrant/src/test/resources/packaging/scripts/modules.bash b/qa/vagrant/src/test/resources/packaging/utils/modules.bash
similarity index 81%
rename from qa/vagrant/src/test/resources/packaging/scripts/modules.bash
rename to qa/vagrant/src/test/resources/packaging/utils/modules.bash
index 2e80fd648f3be..9691fd7470199 100644
--- a/qa/vagrant/src/test/resources/packaging/scripts/modules.bash
+++ b/qa/vagrant/src/test/resources/packaging/utils/modules.bash
@@ -4,9 +4,14 @@
 # the .deb/.rpm packages and the SysV/Systemd scripts.
 
 # WARNING: This testing file must be executed as root and can
-# dramatically change your system. It removes the 'elasticsearch'
-# user/group and also many directories. Do not execute this file
-# unless you know exactly what you are doing.
+# dramatically change your system. It should only be executed
+# in a throw-away VM like those made by the Vagrantfile at
+# the root of the Elasticsearch source code. This should
+# cause the script to fail if it is executed any other way:
+[ -f /etc/is_vagrant_vm ] || {
+  >&2 echo "must be run on a vagrant VM"
+  exit 1
+}
 
 # Licensed to Elasticsearch under one or more contributor
 # license agreements. See the NOTICE file distributed with
@@ -41,4 +46,4 @@ check_module() {
 
 check_secure_module() {
     check_module "$@" plugin-security.policy
-}
\ No newline at end of file
+}
diff --git a/qa/vagrant/src/test/resources/packaging/utils/packages.bash b/qa/vagrant/src/test/resources/packaging/utils/packages.bash
new file mode 100644
index 0000000000000..8fc81013a7004
--- /dev/null
+++ b/qa/vagrant/src/test/resources/packaging/utils/packages.bash
@@ -0,0 +1,144 @@
+#!/bin/bash
+
+# This file contains some utilities to test the elasticsearch scripts with
+# the .deb/.rpm packages.
+
+# WARNING: This testing file must be executed as root and can
+# dramatically change your system. It should only be executed
+# in a throw-away VM like those made by the Vagrantfile at
+# the root of the Elasticsearch source code. This should
+# cause the script to fail if it is executed any other way:
+[ -f /etc/is_vagrant_vm ] || {
+  >&2 echo "must be run on a vagrant VM"
+  exit 1
+}
+
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+# Export some useful paths.
+export_elasticsearch_paths() {
+    export ESHOME="/usr/share/elasticsearch"
+    export ESPLUGINS="$ESHOME/plugins"
+    export ESMODULES="$ESHOME/modules"
+    export ESCONFIG="/etc/elasticsearch"
+    export ESSCRIPTS="$ESCONFIG/scripts"
+    export ESDATA="/var/lib/elasticsearch"
+    export ESLOG="/var/log/elasticsearch"
+    export ESPIDDIR="/var/run/elasticsearch"
+    if is_dpkg; then
+        export ESENVFILE="/etc/default/elasticsearch"
+    fi
+    if is_rpm; then
+        export ESENVFILE="/etc/sysconfig/elasticsearch"
+    fi
+}
+
+# Install the rpm or deb package.
+# -u upgrade rather than install. This only matters for rpm.
+# -v the version to upgrade to. Defaults to the version under test.
+install_package() {
+    local version=$(cat version)
+    local rpmCommand='-i'
+    while getopts ":uv:" opt; do
+        case $opt in
+            u)
+                rpmCommand='-U'
+                dpkgCommand='--force-confnew'
+                ;;
+            v)
+                version=$OPTARG
+                ;;
+            \?)
+                echo "Invalid option: -$OPTARG" >&2
+                ;;
+        esac
+    done
+    if is_rpm; then
+        rpm $rpmCommand elasticsearch-$version.rpm
+    elif is_dpkg; then
+        dpkg $dpkgCommand -i elasticsearch-$version.deb
+    else
+        skip "Only rpm or deb supported"
+    fi
+}
+
+# Checks that all directories & files are correctly installed after a deb or
+# rpm install.
+verify_package_installation() {
+    id elasticsearch
+
+    getent group elasticsearch
+
+    assert_file "$ESHOME" d root root 755
+    assert_file "$ESHOME/bin" d root root 755
+    assert_file "$ESHOME/bin/elasticsearch" f root root 755
+    assert_file "$ESHOME/bin/elasticsearch-plugin" f root root 755
+    assert_file "$ESHOME/bin/elasticsearch-translog" f root root 755
+    assert_file "$ESHOME/lib" d root root 755
+    assert_file "$ESCONFIG" d root elasticsearch 2750
+    assert_file "$ESCONFIG/elasticsearch.yml" f root elasticsearch 660
+    assert_file "$ESCONFIG/jvm.options" f root elasticsearch 660
+    assert_file "$ESCONFIG/log4j2.properties" f root elasticsearch 660
+    assert_file "$ESSCRIPTS" d root elasticsearch 750
+    assert_file "$ESDATA" d elasticsearch elasticsearch 750
+    assert_file "$ESLOG" d elasticsearch elasticsearch 750
+    assert_file "$ESPLUGINS" d root root 755
+    assert_file "$ESMODULES" d root root 755
+    assert_file "$ESPIDDIR" d elasticsearch elasticsearch 755
+    assert_file "$ESHOME/NOTICE.txt" f root root 644
+    assert_file "$ESHOME/README.textile" f root root 644
+
+    if is_dpkg; then
+        # Env file
+        assert_file "/etc/default/elasticsearch" f root root 660
+
+        # Doc files
+        assert_file "/usr/share/doc/elasticsearch" d root root 755
+        assert_file "/usr/share/doc/elasticsearch/copyright" f root root 644
+    fi
+
+    if is_rpm; then
+        # Env file
+        assert_file "/etc/sysconfig/elasticsearch" f root root 660
+        # License file
+        assert_file "/usr/share/elasticsearch/LICENSE.txt" f root root 644
+    fi
+
+    if is_systemd; then
+        assert_file "/usr/lib/systemd/system/elasticsearch.service" f root root 644
+        assert_file "/usr/lib/tmpfiles.d/elasticsearch.conf" f root root 644
+        assert_file "/usr/lib/sysctl.d/elasticsearch.conf" f root root 644
+        if is_rpm; then
+            [[ $(/usr/sbin/sysctl vm.max_map_count) =~ "vm.max_map_count = 262144" ]]
+        else
+            [[ $(/sbin/sysctl vm.max_map_count) =~ "vm.max_map_count = 262144" ]]
+        fi
+    fi
+
+    if is_sysvinit; then
+        assert_file "/etc/init.d/elasticsearch" f root root 750
+    fi
+
+    run sudo -E -u vagrant LANG="en_US.UTF-8" cat "$ESCONFIG/elasticsearch.yml"
+    [ $status = 1 ]
+    [[ "$output" == *"Permission denied"* ]] || {
+        echo "Expected permission denied but found $output:"
+        false
+    }
+}
diff --git a/qa/vagrant/src/test/resources/packaging/utils/plugins.bash b/qa/vagrant/src/test/resources/packaging/utils/plugins.bash
new file mode 100644
index 0000000000000..14650b7130b12
--- /dev/null
+++ b/qa/vagrant/src/test/resources/packaging/utils/plugins.bash
@@ -0,0 +1,178 @@
+#!/bin/bash
+
+# This file contains some utilities to test the elasticsearch
+# plugin installation and uninstallation process.
+
+# WARNING: This testing file must be executed as root and can
+# dramatically change your system. It should only be executed
+# in a throw-away VM like those made by the Vagrantfile at
+# the root of the Elasticsearch source code. This should
+# cause the script to fail if it is executed any other way:
+[ -f /etc/is_vagrant_vm ] || {
+  >&2 echo "must be run on a vagrant VM"
+  exit 1
+}
+
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Install a plugin an run all the common post installation tests.
+install_plugin() {
+    local name=$1
+    local path="$2"
+    local umask="$3"
+
+    assert_file_exist "$path"
+
+    if [ -z "$umask" ]; then
+      sudo -E -u $ESPLUGIN_COMMAND_USER "$ESHOME/bin/elasticsearch-plugin" install -batch "file://$path"
+    else
+      sudo -E -u $ESPLUGIN_COMMAND_USER bash -c "umask $umask && \"$ESHOME/bin/elasticsearch-plugin\" install -batch \"file://$path\""
+    fi
+
+    assert_file_exist "$ESPLUGINS/$name"
+    assert_file_exist "$ESPLUGINS/$name/plugin-descriptor.properties"
+    #check we did not accidentially create a log file as root as /usr/share/elasticsearch
+    assert_file_not_exist "/usr/share/elasticsearch/logs"
+
+    # At some point installing or removing plugins caused elasticsearch's logs
+    # to be owned by root. This is bad so we want to make sure it doesn't
+    # happen.
+    if [ -e "$ESLOG" ] && [ $(stat "$ESLOG" --format "%U") == "root" ]; then
+        echo "$ESLOG is now owned by root! That'll break logging when elasticsearch tries to start."
+        false
+    fi
+}
+
+install_jvm_plugin() {
+    local name=$1
+    local path="$2"
+    install_plugin $name "$path" $3
+    assert_file_exist "$ESPLUGINS/$name/$name"*".jar"
+}
+
+# Remove a plugin and make sure its plugin directory is removed.
+remove_plugin() {
+    local name=$1
+
+    echo "Removing $name...."
+    sudo -E -u $ESPLUGIN_COMMAND_USER "$ESHOME/bin/elasticsearch-plugin" remove $name
+
+    assert_file_not_exist "$ESPLUGINS/$name"
+
+    # At some point installing or removing plugins caused elasticsearch's logs
+    # to be owned by root. This is bad so we want to make sure it doesn't
+    # happen.
+    if [ -e "$ESLOG" ] && [ $(stat "$ESLOG" --format "%U") == "root" ]; then
+        echo "$ESLOG is now owned by root! That'll break logging when elasticsearch tries to start."
+        false
+    fi
+}
+
+# Install the jvm-example plugin which fully exercises the special case file
+# placements for non-site plugins.
+install_jvm_example() {
+    local relativePath=${1:-$(readlink -m jvm-example-*.zip)}
+    install_jvm_plugin jvm-example "$relativePath" $2
+
+    bin_user=$(find "$ESHOME/bin" -maxdepth 0 -printf "%u")
+    bin_owner=$(find "$ESHOME/bin" -maxdepth 0 -printf "%g")
+
+    assert_file "$ESHOME/plugins/jvm-example" d $bin_user $bin_owner 755
+    assert_file "$ESHOME/plugins/jvm-example/jvm-example-$(cat version).jar" f $bin_user $bin_owner 644
+
+    #owner group and permissions vary depending on how es was installed
+    #just make sure that everything is the same as the parent bin dir, which was properly set up during install
+    assert_file "$ESHOME/bin/jvm-example" d $bin_user $bin_owner 755
+    assert_file "$ESHOME/bin/jvm-example/test" f $bin_user $bin_owner 755
+
+    #owner group and permissions vary depending on how es was installed
+    #just make sure that everything is the same as $CONFIG_DIR, which was properly set up during install
+    config_user=$(find "$ESCONFIG" -maxdepth 0 -printf "%u")
+    config_owner=$(find "$ESCONFIG" -maxdepth 0 -printf "%g")
+    # directories should user the user file-creation mask
+    assert_file "$ESCONFIG/jvm-example" d $config_user $config_owner 750
+    assert_file "$ESCONFIG/jvm-example/example.yaml" f $config_user $config_owner 660
+
+    run sudo -E -u vagrant LANG="en_US.UTF-8" cat "$ESCONFIG/jvm-example/example.yaml"
+    [ $status = 1 ]
+    [[ "$output" == *"Permission denied"* ]] || {
+        echo "Expected permission denied but found $output:"
+        false
+    }
+
+    echo "Running jvm-example's bin script...."
+    "$ESHOME/bin/jvm-example/test" | grep test
+}
+
+# Remove the jvm-example plugin which fully exercises the special cases of
+# removing bin and not removing config.
+remove_jvm_example() {
+    remove_plugin jvm-example
+
+    assert_file_not_exist "$ESHOME/bin/jvm-example"
+    assert_file_exist "$ESCONFIG/jvm-example"
+    assert_file_exist "$ESCONFIG/jvm-example/example.yaml"
+}
+
+# Install a plugin with a special prefix. For the most part prefixes are just
+# useful for grouping but the "analysis" prefix is special because all
+# analysis plugins come with a corresponding lucene-analyzers jar.
+# $1 - the prefix
+# $2 - the plugin name
+# $@ - all remaining arguments are jars that must exist in the plugin's
+#      installation directory
+install_and_check_plugin() {
+    local prefix=$1
+    shift
+    local name=$1
+    shift
+
+    if [ "$prefix" == "-" ]; then
+        local full_name="$name"
+    else
+        local full_name="$prefix-$name"
+    fi
+
+    install_jvm_plugin $full_name "$(readlink -m $full_name-*.zip)"
+
+    assert_module_or_plugin_directory "$ESPLUGINS/$full_name"
+
+    # analysis plugins have a corresponding analyzers jar
+    if [ $prefix == 'analysis' ]; then
+        local analyzer_jar_suffix=$name
+        # the name of the analyzer jar for the ukrainian plugin does
+        # not match the name of the plugin, so we have to make an
+        # exception
+        if [ $name == 'ukrainian' ]; then
+             analyzer_jar_suffix='morfologik'
+        fi
+        assert_module_or_plugin_file "$ESPLUGINS/$full_name/lucene-analyzers-$analyzer_jar_suffix-*.jar"
+    fi
+    for file in "$@"; do
+        assert_module_or_plugin_file "$ESPLUGINS/$full_name/$file"
+    done
+}
+
+# Compare a list of plugin names to the plugins in the plugins pom and see if they are the same
+# $1 the file containing the list of plugins we want to compare to
+# $2 description of the source of the plugin list
+compare_plugins_list() {
+    cat $1 | sort > /tmp/plugins
+    echo "Checking plugins from $2 (<) against expected plugins (>):"
+    diff -w /elasticsearch/qa/vagrant/build/plugins/expected /tmp/plugins
+}
diff --git a/qa/vagrant/src/test/resources/packaging/utils/tar.bash b/qa/vagrant/src/test/resources/packaging/utils/tar.bash
new file mode 100644
index 0000000000000..5a34cb5b9fd8d
--- /dev/null
+++ b/qa/vagrant/src/test/resources/packaging/utils/tar.bash
@@ -0,0 +1,102 @@
+#!/bin/bash
+
+# This file contains some utilities to test the elasticsearch
+# tar distribution.
+
+# WARNING: This testing file must be executed as root and can
+# dramatically change your system. It should only be executed
+# in a throw-away VM like those made by the Vagrantfile at
+# the root of the Elasticsearch source code. This should
+# cause the script to fail if it is executed any other way:
+[ -f /etc/is_vagrant_vm ] || {
+  >&2 echo "must be run on a vagrant VM"
+  exit 1
+}
+
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+# Install the tar.gz archive
+install_archive() {
+    export ESHOME=${1:-/tmp/elasticsearch}
+
+    echo "Unpacking tarball to $ESHOME"
+    rm -rf /tmp/untar
+    mkdir -p /tmp/untar
+    tar -xzpf elasticsearch*.tar.gz -C /tmp/untar
+
+    find /tmp/untar -depth -type d -name 'elasticsearch*' -exec mv {} "$ESHOME" \; > /dev/null
+
+    # ES cannot run as root so create elasticsearch user & group if needed
+    if ! getent group "elasticsearch" > /dev/null 2>&1 ; then
+        if is_dpkg; then
+            addgroup --system "elasticsearch"
+        else
+            groupadd -r "elasticsearch"
+        fi
+    fi
+    if ! id "elasticsearch" > /dev/null 2>&1 ; then
+        if is_dpkg; then
+            adduser --quiet --system --no-create-home --ingroup "elasticsearch" --disabled-password --shell /bin/false "elasticsearch"
+        else
+            useradd --system -M --gid "elasticsearch" --shell /sbin/nologin --comment "elasticsearch user" "elasticsearch"
+        fi
+    fi
+
+    chown -R elasticsearch:elasticsearch "$ESHOME"
+    export_elasticsearch_paths
+}
+
+# Move the unzipped tarball to another location.
+move_elasticsearch() {
+    local oldhome="$ESHOME"
+    export ESHOME="$1"
+    rm -rf "$ESHOME"
+    mv "$oldhome" "$ESHOME"
+    export_elasticsearch_paths
+}
+
+# Export some useful paths.
+export_elasticsearch_paths() {
+    export ESMODULES="$ESHOME/modules"
+    export ESPLUGINS="$ESHOME/plugins"
+    export ESCONFIG="$ESHOME/config"
+    export ESSCRIPTS="$ESCONFIG/scripts"
+    export ESDATA="$ESHOME/data"
+    export ESLOG="$ESHOME/logs"
+}
+
+# Checks that all directories & files are correctly installed
+# after a archive (tar.gz/zip) install
+verify_archive_installation() {
+    assert_file "$ESHOME" d elasticsearch elasticsearch 755
+    assert_file "$ESHOME/bin" d elasticsearch elasticsearch 755
+    assert_file "$ESHOME/bin/elasticsearch" f elasticsearch elasticsearch 755
+    assert_file "$ESHOME/bin/elasticsearch.in.sh" f elasticsearch elasticsearch 755
+    assert_file "$ESHOME/bin/elasticsearch-plugin" f elasticsearch elasticsearch 755
+    assert_file "$ESHOME/bin/elasticsearch-translog" f elasticsearch elasticsearch 755
+    assert_file "$ESCONFIG" d elasticsearch elasticsearch 755
+    assert_file "$ESCONFIG/elasticsearch.yml" f elasticsearch elasticsearch 660
+    assert_file "$ESCONFIG/jvm.options" f elasticsearch elasticsearch 660
+    assert_file "$ESCONFIG/log4j2.properties" f elasticsearch elasticsearch 660
+    assert_file "$ESPLUGINS" d elasticsearch elasticsearch 755
+    assert_file "$ESHOME/lib" d elasticsearch elasticsearch 755
+    assert_file "$ESHOME/NOTICE.txt" f elasticsearch elasticsearch 644
+    assert_file "$ESHOME/LICENSE.txt" f elasticsearch elasticsearch 644
+    assert_file "$ESHOME/README.textile" f elasticsearch elasticsearch 644
+}
diff --git a/qa/vagrant/src/test/resources/packaging/utils/utils.bash b/qa/vagrant/src/test/resources/packaging/utils/utils.bash
new file mode 100644
index 0000000000000..7cebee253bb63
--- /dev/null
+++ b/qa/vagrant/src/test/resources/packaging/utils/utils.bash
@@ -0,0 +1,541 @@
+#!/bin/bash
+
+# This file contains some utilities to test the elasticsearch scripts,
+# the .deb/.rpm packages and the SysV/Systemd scripts.
+
+# WARNING: This testing file must be executed as root and can
+# dramatically change your system. It should only be executed
+# in a throw-away VM like those made by the Vagrantfile at
+# the root of the Elasticsearch source code. This should
+# cause the script to fail if it is executed any other way:
+[ -f /etc/is_vagrant_vm ] || {
+  >&2 echo "must be run on a vagrant VM"
+  exit 1
+}
+
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Checks if necessary commands are available to run the tests
+
+if [ ! -x /usr/bin/which ]; then
+    echo "'which' command is mandatory to run the tests"
+    exit 1
+fi
+
+if [ ! -x "`which wget 2>/dev/null`" ]; then
+    echo "'wget' command is mandatory to run the tests"
+    exit 1
+fi
+
+if [ ! -x "`which curl 2>/dev/null`" ]; then
+    echo "'curl' command is mandatory to run the tests"
+    exit 1
+fi
+
+if [ ! -x "`which pgrep 2>/dev/null`" ]; then
+    echo "'pgrep' command is mandatory to run the tests"
+    exit 1
+fi
+
+if [ ! -x "`which unzip 2>/dev/null`" ]; then
+    echo "'unzip' command is mandatory to run the tests"
+    exit 1
+fi
+
+if [ ! -x "`which tar 2>/dev/null`" ]; then
+    echo "'tar' command is mandatory to run the tests"
+    exit 1
+fi
+
+if [ ! -x "`which unzip 2>/dev/null`" ]; then
+    echo "'unzip' command is mandatory to run the tests"
+    exit 1
+fi
+
+if [ ! -x "`which java 2>/dev/null`" ]; then
+    echo "'java' command is mandatory to run the tests"
+    exit 1
+fi
+
+# Returns 0 if the 'dpkg' command is available
+is_dpkg() {
+    [ -x "`which dpkg 2>/dev/null`" ]
+}
+
+# Returns 0 if the 'rpm' command is available
+is_rpm() {
+    [ -x "`which rpm 2>/dev/null`" ]
+}
+
+# Skip test if the 'dpkg' command is not supported
+skip_not_dpkg() {
+    is_dpkg || skip "dpkg is not supported"
+}
+
+# Skip test if the 'rpm' command is not supported
+skip_not_rpm() {
+    is_rpm || skip "rpm is not supported"
+}
+
+skip_not_dpkg_or_rpm() {
+    is_dpkg || is_rpm || skip "only dpkg or rpm systems are supported"
+}
+
+# Returns 0 if the system supports Systemd
+is_systemd() {
+    [ -x /bin/systemctl ]
+}
+
+# Skip test if Systemd is not supported
+skip_not_systemd() {
+    if [ ! -x /bin/systemctl ]; then
+        skip "systemd is not supported"
+    fi
+}
+
+# Returns 0 if the system supports SysV
+is_sysvinit() {
+    [ -x "`which service 2>/dev/null`" ]
+}
+
+# Skip test if SysV is not supported
+skip_not_sysvinit() {
+    if [ -x "`which service 2>/dev/null`" ] && is_systemd; then
+        skip "sysvinit is supported, but systemd too"
+    fi
+    if [ ! -x "`which service 2>/dev/null`" ]; then
+        skip "sysvinit is not supported"
+    fi
+}
+
+# Skip if tar is not supported
+skip_not_tar_gz() {
+    if [ ! -x "`which tar 2>/dev/null`" ]; then
+        skip "tar is not supported"
+    fi
+}
+
+# Skip if unzip is not supported
+skip_not_zip() {
+    if [ ! -x "`which unzip 2>/dev/null`" ]; then
+        skip "unzip is not supported"
+    fi
+}
+
+assert_file_exist() {
+    local file="$1"
+    if [ ! -e "$file" ]; then
+        echo "Should exist: ${file} but does not"
+    fi
+    local file=$(readlink -m "${file}")
+    [ -e "$file" ]
+}
+
+assert_file_not_exist() {
+    local file="$1"
+    if [ -e "$file" ]; then
+        echo "Should not exist: ${file} but does"
+    fi
+    local file=$(readlink -m "${file}")
+    [ ! -e "$file" ]
+}
+
+assert_file() {
+    local file="$1"
+    local type=$2
+    local user=$3
+    local group=$4
+    local privileges=$5
+
+    assert_file_exist "$file"
+
+    if [ "$type" = "d" ]; then
+        if [ ! -d "$file" ]; then
+            echo "[$file] should be a directory but is not"
+        fi
+        [ -d "$file" ]
+    else
+        if [ ! -f "$file" ]; then
+            echo "[$file] should be a regular file but is not"
+        fi
+        [ -f "$file" ]
+    fi
+
+    if [ "x$user" != "x" ]; then
+        realuser=$(find "$file" -maxdepth 0 -printf "%u")
+        if [ "$realuser" != "$user" ]; then
+            echo "Expected user: $user, found $realuser [$file]"
+        fi
+        [ "$realuser" = "$user" ]
+    fi
+
+    if [ "x$group" != "x" ]; then
+        realgroup=$(find "$file" -maxdepth 0 -printf "%g")
+        if [ "$realgroup" != "$group" ]; then
+            echo "Expected group: $group, found $realgroup [$file]"
+        fi
+        [ "$realgroup" = "$group" ]
+    fi
+
+    if [ "x$privileges" != "x" ]; then
+        realprivileges=$(find "$file" -maxdepth 0 -printf "%m")
+        if [ "$realprivileges" != "$privileges" ]; then
+            echo "Expected privileges: $privileges, found $realprivileges [$file]"
+        fi
+        [ "$realprivileges" = "$privileges" ]
+    fi
+}
+
+assert_module_or_plugin_directory() {
+    local directory=$1
+    shift
+
+    #owner group and permissions vary depending on how es was installed
+    #just make sure that everything is the same as $CONFIG_DIR, which was properly set up during install
+    config_user=$(find "$ESHOME" -maxdepth 0 -printf "%u")
+    config_owner=$(find "$ESHOME" -maxdepth 0 -printf "%g")
+
+    assert_file $directory d $config_user $config_owner 755
+}
+
+assert_module_or_plugin_file() {
+    local file=$1
+    shift
+
+    assert_file_exist "$(readlink -m $file)"
+    assert_file $file f $config_user $config_owner 644
+}
+
+assert_output() {
+    echo "$output" | grep -E "$1"
+}
+
+assert_recursive_ownership() {
+    local directory=$1
+    local user=$2
+    local group=$3
+
+    realuser=$(find $directory -printf "%u\n" | sort | uniq)
+    [ "$realuser" = "$user" ]
+    realgroup=$(find $directory -printf "%g\n" | sort | uniq)
+    [ "$realgroup" = "$group" ]
+}
+
+# Deletes everything before running a test file
+clean_before_test() {
+
+    # List of files to be deleted
+    ELASTICSEARCH_TEST_FILES=("/usr/share/elasticsearch" \
+                            "/etc/elasticsearch" \
+                            "/var/lib/elasticsearch" \
+                            "/var/log/elasticsearch" \
+                            "/tmp/elasticsearch" \
+                            "/etc/default/elasticsearch" \
+                            "/etc/sysconfig/elasticsearch"  \
+                            "/var/run/elasticsearch"  \
+                            "/usr/share/doc/elasticsearch" \
+                            "/tmp/elasticsearch" \
+                            "/usr/lib/systemd/system/elasticsearch.conf" \
+                            "/usr/lib/tmpfiles.d/elasticsearch.conf" \
+                            "/usr/lib/sysctl.d/elasticsearch.conf")
+
+    # Kills all processes of user elasticsearch
+    if id elasticsearch > /dev/null 2>&1; then
+        pkill -u elasticsearch 2>/dev/null || true
+    fi
+
+    # Kills all running Elasticsearch processes
+    ps aux | grep -i "org.elasticsearch.bootstrap.Elasticsearch" | awk {'print $2'} | xargs kill -9 > /dev/null 2>&1 || true
+
+    purge_elasticsearch
+
+    # Removes user & group
+    userdel elasticsearch > /dev/null 2>&1 || true
+    groupdel elasticsearch > /dev/null 2>&1 || true
+
+    # Removes all files
+    for d in "${ELASTICSEARCH_TEST_FILES[@]}"; do
+        if [ -e "$d" ]; then
+            rm -rf "$d"
+        fi
+    done
+
+    if is_systemd; then
+        systemctl unmask systemd-sysctl.service
+    fi
+}
+
+purge_elasticsearch() {
+    # Removes RPM package
+    if is_rpm; then
+        rpm --quiet -e elasticsearch > /dev/null 2>&1 || true
+    fi
+
+    if [ -x "`which yum 2>/dev/null`" ]; then
+        yum remove -y elasticsearch > /dev/null 2>&1 || true
+    fi
+
+    # Removes DEB package
+    if is_dpkg; then
+        dpkg --purge elasticsearch > /dev/null 2>&1 || true
+    fi
+
+    if [ -x "`which apt-get 2>/dev/null`" ]; then
+        apt-get --quiet --yes purge elasticsearch > /dev/null 2>&1 || true
+    fi
+}
+
+# Start elasticsearch and wait for it to come up with a status.
+# $1 - expected status - defaults to green
+start_elasticsearch_service() {
+    local desiredStatus=${1:-green}
+    local index=$2
+    local commandLineArgs=$3
+
+    run_elasticsearch_service 0 $commandLineArgs
+
+    wait_for_elasticsearch_status $desiredStatus $index
+
+    if [ -r "/tmp/elasticsearch/elasticsearch.pid" ]; then
+        pid=$(cat /tmp/elasticsearch/elasticsearch.pid)
+        [ "x$pid" != "x" ] && [ "$pid" -gt 0 ]
+        echo "Looking for elasticsearch pid...."
+        ps $pid
+    elif is_systemd; then
+        run systemctl is-active elasticsearch.service
+        [ "$status" -eq 0 ]
+
+        run systemctl status elasticsearch.service
+        [ "$status" -eq 0 ]
+
+    elif is_sysvinit; then
+        run service elasticsearch status
+        [ "$status" -eq 0 ]
+    fi
+}
+
+# Start elasticsearch
+# $1 expected status code
+# $2 additional command line args
+run_elasticsearch_service() {
+    local expectedStatus=$1
+    local commandLineArgs=$2
+    # Set the CONF_DIR setting in case we start as a service
+    if [ ! -z "$CONF_DIR" ] ; then
+        if is_dpkg ; then
+            echo "CONF_DIR=$CONF_DIR" >> /etc/default/elasticsearch;
+            echo "ES_JVM_OPTIONS=$ES_JVM_OPTIONS" >> /etc/default/elasticsearch;
+        elif is_rpm; then
+            echo "CONF_DIR=$CONF_DIR" >> /etc/sysconfig/elasticsearch;
+            echo "ES_JVM_OPTIONS=$ES_JVM_OPTIONS" >> /etc/sysconfig/elasticsearch
+        fi
+    fi
+
+    if [ -f "/tmp/elasticsearch/bin/elasticsearch" ]; then
+        if [ -z "$CONF_DIR" ]; then
+            local CONF_DIR=""
+            local ES_PATH_CONF=""
+        else
+            local ES_PATH_CONF="-Epath.conf=$CONF_DIR"
+        fi
+        # we must capture the exit code to compare so we don't want to start as background process in case we expect something other than 0
+        local background=""
+        local timeoutCommand=""
+        if [ "$expectedStatus" = 0 ]; then
+            background="-d"
+        else
+            timeoutCommand="timeout 60s "
+        fi
+        # su and the Elasticsearch init script work together to break bats.
+        # sudo isolates bats enough from the init script so everything continues
+        # to tick along
+        run sudo -u elasticsearch bash <<BASH
+# If jayatana is installed then we try to use it. Elasticsearch should ignore it even when we try.
+# If it doesn't ignore it then Elasticsearch will fail to start because of security errors.
+# This line is attempting to emulate the on login behavior of /usr/share/upstart/sessions/jayatana.conf
+[ -f /usr/share/java/jayatanaag.jar ] && export JAVA_TOOL_OPTIONS="-javaagent:/usr/share/java/jayatanaag.jar"
+# And now we can start Elasticsearch normally, in the background (-d) and with a pidfile (-p).
+export ES_JVM_OPTIONS=$ES_JVM_OPTIONS
+export ES_JAVA_OPTS=$ES_JAVA_OPTS
+$timeoutCommand/tmp/elasticsearch/bin/elasticsearch $background -p /tmp/elasticsearch/elasticsearch.pid $ES_PATH_CONF $commandLineArgs
+BASH
+        [ "$status" -eq "$expectedStatus" ]
+    elif is_systemd; then
+        run systemctl daemon-reload
+        [ "$status" -eq 0 ]
+
+        run systemctl enable elasticsearch.service
+        [ "$status" -eq 0 ]
+
+        run systemctl is-enabled elasticsearch.service
+        [ "$status" -eq 0 ]
+
+        run systemctl start elasticsearch.service
+        [ "$status" -eq "$expectedStatus" ]
+
+    elif is_sysvinit; then
+        run service elasticsearch start
+        [ "$status" -eq "$expectedStatus" ]
+    fi
+}
+
+stop_elasticsearch_service() {
+    if [ -r "/tmp/elasticsearch/elasticsearch.pid" ]; then
+        pid=$(cat /tmp/elasticsearch/elasticsearch.pid)
+        [ "x$pid" != "x" ] && [ "$pid" -gt 0 ]
+
+        kill -SIGTERM $pid
+    elif is_systemd; then
+        run systemctl stop elasticsearch.service
+        [ "$status" -eq 0 ]
+
+        run systemctl is-active elasticsearch.service
+        [ "$status" -eq 3 ]
+
+        echo "$output" | grep -E 'inactive|failed'
+
+    elif is_sysvinit; then
+        run service elasticsearch stop
+        [ "$status" -eq 0 ]
+
+        run service elasticsearch status
+        [ "$status" -ne 0 ]
+    fi
+}
+
+# Waits for Elasticsearch to reach some status.
+# $1 - expected status - defaults to green
+wait_for_elasticsearch_status() {
+    local desiredStatus=${1:-green}
+    local index=$2
+
+    echo "Making sure elasticsearch is up..."
+    wget -O - --retry-connrefused --waitretry=1 --timeout=60 --tries 60 http://localhost:9200/_cluster/health || {
+          echo "Looks like elasticsearch never started. Here is its log:"
+          if [ -e "$ESLOG/elasticsearch.log" ]; then
+              cat "$ESLOG/elasticsearch.log"
+          else
+              echo "The elasticsearch log doesn't exist. Maybe /var/log/messages has something:"
+              tail -n20 /var/log/messages
+          fi
+          false
+    }
+
+    if [ -z "index" ]; then
+      echo "Tring to connect to elasticsearch and wait for expected status $desiredStatus..."
+      curl -sS "http://localhost:9200/_cluster/health?wait_for_status=$desiredStatus&timeout=60s&pretty"
+    else
+      echo "Trying to connect to elasticsearch and wait for expected status $desiredStatus for index $index"
+      curl -sS "http://localhost:9200/_cluster/health/$index?wait_for_status=$desiredStatus&timeout=60s&pretty"
+    fi
+    if [ $? -eq 0 ]; then
+        echo "Connected"
+    else
+        echo "Unable to connect to Elasticsearch"
+        false
+    fi
+
+    echo "Checking that the cluster health matches the waited for status..."
+    run curl -sS -XGET 'http://localhost:9200/_cat/health?h=status&v=false'
+    if [ "$status" -ne 0 ]; then
+        echo "error when checking cluster health. code=$status output="
+        echo $output
+        false
+    fi
+    echo $output | grep $desiredStatus || {
+        echo "unexpected status:  '$output' wanted '$desiredStatus'"
+        false
+    }
+}
+
+install_elasticsearch_test_scripts() {
+    install_script is_guide.groovy
+    install_script is_guide.mustache
+}
+
+# Executes some basic Elasticsearch tests
+run_elasticsearch_tests() {
+    # TODO this assertion is the same the one made when waiting for
+    # elasticsearch to start
+    run curl -XGET 'http://localhost:9200/_cat/health?h=status&v=false'
+    [ "$status" -eq 0 ]
+    echo "$output" | grep -w "green"
+
+    curl -s -XPOST 'http://localhost:9200/library/book/1?refresh=true&pretty' -d '{
+      "title": "Elasticsearch - The Definitive Guide"
+    }'
+
+    curl -s -XGET 'http://localhost:9200/_count?pretty' |
+      grep \"count\"\ :\ 1
+
+    curl -s -XPOST 'http://localhost:9200/library/book/_count?pretty' -d '{
+      "query": {
+        "script": {
+          "script": {
+            "file": "is_guide",
+            "lang": "groovy"
+          }
+        }
+      }
+    }' | grep \"count\"\ :\ 1
+
+    curl -s -XGET 'http://localhost:9200/library/book/_search/template?pretty' -d '{
+      "file": "is_guide"
+    }' | grep \"total\"\ :\ 1
+
+    curl -s -XDELETE 'http://localhost:9200/_all'
+}
+
+# Move the config directory to another directory and properly chown it.
+move_config() {
+    local oldConfig="$ESCONFIG"
+    export ESCONFIG="${1:-$(mktemp -d -t 'config.XXXX')}"
+    echo "Moving configuration directory from $oldConfig to $ESCONFIG"
+
+    # Move configuration files to the new configuration directory
+    mv "$oldConfig"/* "$ESCONFIG"
+    chown -R elasticsearch:elasticsearch "$ESCONFIG"
+    assert_file_exist "$ESCONFIG/elasticsearch.yml"
+    assert_file_exist "$ESCONFIG/jvm.options"
+    assert_file_exist "$ESCONFIG/log4j2.properties"
+}
+
+# Copies a script into the Elasticsearch install.
+install_script() {
+    local name=$1
+    mkdir -p $ESSCRIPTS
+    local script="$BATS_TEST_DIRNAME/example/scripts/$name"
+    echo "Installing $script to $ESSCRIPTS"
+    cp $script $ESSCRIPTS
+}
+
+# permissions from the user umask with the executable bit set
+executable_privileges_for_user_from_umask() {
+    local user=$1
+    shift
+
+    echo $((0777 & ~$(sudo -E -u $user sh -c umask) | 0111))
+}
+
+# permissions from the user umask without the executable bit set
+file_privileges_for_user_from_umask() {
+    local user=$1
+    shift
+
+    echo $((0777 & ~$(sudo -E -u $user sh -c umask) & ~0111))
+}
diff --git a/qa/vagrant/versions b/qa/vagrant/versions
deleted file mode 100644
index c7aef6cb99a2a..0000000000000
--- a/qa/vagrant/versions
+++ /dev/null
@@ -1,16 +0,0 @@
-2.0.0
-2.0.1
-2.0.2
-2.1.0
-2.1.1
-2.1.2
-2.2.0
-2.2.1
-2.2.2
-2.3.0
-2.3.1
-2.3.2
-2.3.3
-2.3.4
-2.3.5
-2.4.0
diff --git a/qa/verify-version-constants/build.gradle b/qa/verify-version-constants/build.gradle
new file mode 100644
index 0000000000000..db14cafba4728
--- /dev/null
+++ b/qa/verify-version-constants/build.gradle
@@ -0,0 +1,91 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.Locale
+import org.elasticsearch.gradle.Version
+import org.elasticsearch.gradle.VersionProperties
+import org.elasticsearch.gradle.test.RestIntegTestTask
+
+apply plugin: 'elasticsearch.standalone-test'
+
+// This is a top level task which we will add dependencies to below.
+// It is a single task that can be used to backcompat tests against all versions.
+task bwcTest {
+    description = 'Runs backwards compatibility tests.'
+    group = 'verification'
+}
+
+for (Version version : indexCompatVersions) {
+
+    // we do not have infrastructure for starting pre-5.0.0 nodes so we skip them
+    if (version.before("5.0.0")) continue
+
+    String baseName = "v${version}"
+    Task oldClusterTest = tasks.create(name: "${baseName}#oldClusterTest", type: RestIntegTestTask) {
+        mustRunAfter(precommit)
+    }
+    configure(extensions.findByName("${baseName}#oldClusterTestCluster")) {
+        distribution = 'zip'
+        bwcVersion = version
+        numBwcNodes = 1
+        numNodes = 1
+        clusterName = 'verify-version-constants'
+        if (version.onOrAfter('5.3.0')) {
+            setting 'http.content_type.required', 'true'
+        }
+    }
+
+    Task versionBwcTest = tasks.create(name: "${baseName}#bwcTest") {
+        dependsOn = [oldClusterTest]
+    }
+
+    bwcTest.dependsOn(versionBwcTest)
+}
+
+test.enabled = false
+
+task integTest {
+    dependsOn = ["v${indexCompatVersions[-1]}#bwcTest"]
+}
+
+task verifyDocsLuceneVersion {
+  doFirst {
+    File docsVersionsFile = rootProject.file('docs/Versions.asciidoc')
+    List<String> versionLines = docsVersionsFile.readLines('UTF-8')
+    String docsLuceneVersion = null
+    for (String line : versionLines) {
+      if (line.startsWith(':lucene_version:')) {
+        docsLuceneVersion = line.split()[1]
+      }
+    }
+    if (docsLuceneVersion == null) {
+      throw new GradleException('Could not find lucene version in docs version file')
+    }
+    String expectedLuceneVersion = VersionProperties.lucene
+    if (expectedLuceneVersion.contains('-snapshot-')) {
+      expectedLuceneVersion = expectedLuceneVersion.substring(0, expectedLuceneVersion.lastIndexOf('-'))
+      expectedLuceneVersion = expectedLuceneVersion.toUpperCase(Locale.ROOT)
+    }
+    if (docsLuceneVersion != expectedLuceneVersion) {
+      throw new GradleException("Lucene version in docs [${docsLuceneVersion}] does not match version.properties [${expectedLuceneVersion}]")
+    }
+  }
+}
+
+check.dependsOn integTest, verifyDocsLuceneVersion
diff --git a/qa/verify-version-constants/src/test/java/org/elasticsearch/qa/verify_version_constants/VerifyVersionConstantsIT.java b/qa/verify-version-constants/src/test/java/org/elasticsearch/qa/verify_version_constants/VerifyVersionConstantsIT.java
new file mode 100644
index 0000000000000..a26237962bf4b
--- /dev/null
+++ b/qa/verify-version-constants/src/test/java/org/elasticsearch/qa/verify_version_constants/VerifyVersionConstantsIT.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.qa.verify_version_constants;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.test.rest.yaml.ObjectPath;
+
+import java.io.IOException;
+import java.text.ParseException;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+
+public class VerifyVersionConstantsIT extends ESRestTestCase {
+
+    public void testLuceneVersionConstant() throws IOException, ParseException {
+        final Response response = client().performRequest("GET", "/");
+        assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
+        final ObjectPath objectPath = ObjectPath.createFromResponse(response);
+        final String elasticsearchVersionString = objectPath.evaluate("version.number").toString();
+        final Version elasticsearchVersion = Version.fromString(elasticsearchVersionString);
+        final String luceneVersionString = objectPath.evaluate("version.lucene_version").toString();
+        final org.apache.lucene.util.Version luceneVersion = org.apache.lucene.util.Version.parse(luceneVersionString);
+        assertThat(elasticsearchVersion.luceneVersion, equalTo(luceneVersion));
+    }
+
+}
diff --git a/qa/wildfly/build.gradle b/qa/wildfly/build.gradle
new file mode 100644
index 0000000000000..06a2a6ca45275
--- /dev/null
+++ b/qa/wildfly/build.gradle
@@ -0,0 +1,197 @@
+import org.elasticsearch.gradle.LoggedExec
+import org.elasticsearch.gradle.VersionProperties
+import org.apache.tools.ant.taskdefs.condition.Os
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.util.stream.Stream
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+apply plugin: 'war'
+apply plugin: 'elasticsearch.build'
+apply plugin: 'elasticsearch.rest-test'
+
+final String wildflyVersion = '10.0.0.Final'
+final String wildflyDir = "${buildDir}/wildfly"
+final String wildflyInstall = "${buildDir}/wildfly/wildfly-${wildflyVersion}"
+// TODO: use ephemeral ports
+final int portOffset = 30000
+final int managementPort = 9990 + portOffset
+// we skip these tests on Windows so we do not need to worry about compatibility here
+final String stopWildflyCommand = "${wildflyInstall}/bin/jboss-cli.sh --controller=localhost:${managementPort} --connect command=shutdown"
+
+repositories {
+    mavenCentral()
+    // the Wildfly distribution is not available via a repository, so we fake an Ivy repository on top of the download site
+    ivy {
+        url "http://download.jboss.org"
+        layout 'pattern', {
+            artifact 'wildfly/[revision]/[module]-[revision].[ext]'
+        }
+    }
+}
+
+configurations {
+    wildfly
+}
+
+dependencies {
+    providedCompile 'javax.enterprise:cdi-api:1.2'
+    providedCompile 'org.jboss.spec.javax.annotation:jboss-annotations-api_1.2_spec:1.0.0.Final'
+    providedCompile 'org.jboss.spec.javax.ws.rs:jboss-jaxrs-api_2.0_spec:1.0.0.Final'
+    compile ('org.jboss.resteasy:resteasy-jackson2-provider:3.0.19.Final') {
+        exclude module: 'jackson-annotations'
+        exclude module: 'jackson-core'
+        exclude module: 'jackson-databind'
+        exclude module: 'jackson-jaxrs-json-provider'
+    }
+    compile "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
+    compile "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
+    compile "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}"
+    compile "com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:${versions.jackson}"
+    compile "com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:${versions.jackson}"
+    compile "com.fasterxml.jackson.module:jackson-module-jaxb-annotations:${versions.jackson}"
+    compile "org.apache.logging.log4j:log4j-api:${versions.log4j}"
+    compile "org.apache.logging.log4j:log4j-core:${versions.log4j}"
+    compile project(path: ':client:transport', configuration: 'runtime')
+    wildfly "org.jboss:wildfly:${wildflyVersion}@zip"
+    testCompile "org.elasticsearch.test:framework:${VersionProperties.elasticsearch}"
+}
+
+task unzipWildfly(type: Sync) {
+    into wildflyDir
+    from { zipTree(configurations.wildfly.singleFile) }
+}
+
+task deploy(type: Copy) {
+    dependsOn unzipWildfly, war
+    from war
+    into "${wildflyInstall}/standalone/deployments"
+}
+
+task writeElasticsearchProperties {
+    onlyIf { !Os.isFamily(Os.FAMILY_WINDOWS) && project.rootProject.ext.javaVersion == JavaVersion.VERSION_1_8 }
+    dependsOn 'integTestCluster#wait', deploy
+    doLast {
+        final File elasticsearchProperties = file("${wildflyInstall}/standalone/configuration/elasticsearch.properties")
+        elasticsearchProperties.write(
+                [
+                        "transport.uri=${-> integTest.getNodes().get(0).transportUri()}",
+                        "cluster.name=${-> integTest.getNodes().get(0).clusterName}"
+                ].join("\n"))
+    }
+}
+
+// the default configuration ships with IPv6 disabled but our cluster could be bound to IPv6 if the host supports it
+task enableIPv6 {
+    dependsOn unzipWildfly
+    doLast {
+        final File standaloneConf = file("${wildflyInstall}/bin/standalone.conf")
+        final List<String> lines =
+                Files.readAllLines(standaloneConf.toPath())
+                        .collect { line -> line.replace("-Djava.net.preferIPv4Stack=true", "-Djava.net.preferIPv4Stack=false") }
+        standaloneConf.write(lines.join("\n"))
+    }
+}
+
+task startWildfly {
+    dependsOn enableIPv6, writeElasticsearchProperties
+    doFirst {
+        // we skip these tests on Windows so we do no need to worry about compatibility here
+        final File script = new File(project.buildDir, "wildfly/wildfly.killer.sh")
+        script.setText(
+                ["function shutdown {",
+                 "  ${stopWildflyCommand}",
+                 "}",
+                 "trap shutdown EXIT",
+                 // will wait indefinitely for input, but we never pass input, and the pipe is only closed when the build dies
+                 "read line"].join('\n'), 'UTF-8')
+        final ProcessBuilder pb = new ProcessBuilder("bash", script.absolutePath)
+        pb.start()
+    }
+    doLast {
+        // we skip these tests on Windows so we do no need to worry about compatibility here
+        final ProcessBuilder pb =
+                new ProcessBuilder("${wildflyInstall}/bin/standalone.sh", "-Djboss.socket.binding.port-offset=${portOffset}")
+        final Process process = pb.start()
+        new BufferedReader(new InputStreamReader(process.getInputStream())).withReader { br ->
+            String line
+            while ((line = br.readLine()) != null) {
+                if (line.matches(".*WildFly Full \\d+\\.\\d+\\.\\d+\\.Final \\(WildFly Core \\d+\\.\\d+\\.\\d+\\.Final\\) started.*")) {
+                    break
+                }
+            }
+        }
+    }
+}
+
+task configureTransportClient(type: LoggedExec) {
+    dependsOn startWildfly
+    // we skip these tests on Windows so we do not need to worry about compatibility here
+    commandLine "${wildflyInstall}/bin/jboss-cli.sh",
+            "--controller=localhost:${managementPort}",
+            "--connect",
+            "--command=/system-property=elasticsearch.properties:add(value=\${jboss.server.config.dir}/elasticsearch.properties)"
+}
+
+task stopWildfly(type: LoggedExec) {
+    commandLine stopWildflyCommand.split(' ')
+}
+
+if (!Os.isFamily(Os.FAMILY_WINDOWS) && project.rootProject.ext.javaVersion == JavaVersion.VERSION_1_8) {
+    integTestRunner.dependsOn(configureTransportClient)
+    final TaskExecutionAdapter logDumpListener = new TaskExecutionAdapter() {
+        @Override
+        void afterExecute(final Task task, final TaskState state) {
+            if (state.failure != null) {
+                final File logFile = new File(wildflyInstall, "standalone/log/server.log")
+                println("\nWildfly server log (from ${logFile}):")
+                println('-----------------------------------------')
+                final Stream<String> stream = Files.lines(logFile.toPath(), StandardCharsets.UTF_8)
+                try {
+                    for (String line : stream) {
+                        println(line)
+                    }
+                } finally {
+                    stream.close()
+                }
+                println('=========================================')
+            }
+        }
+    }
+    integTestRunner.doFirst {
+        project.gradle.addListener(logDumpListener)
+    }
+    integTestRunner.doLast {
+        project.gradle.removeListener(logDumpListener)
+    }
+    integTestRunner.finalizedBy(stopWildfly)
+} else {
+    integTest.enabled = false
+}
+
+check.dependsOn(integTest)
+
+test.enabled = false
+
+dependencyLicenses.enabled = false
+
+thirdPartyAudit.enabled = false
diff --git a/qa/wildfly/src/main/java/org/elasticsearch/wildfly/model/Employee.java b/qa/wildfly/src/main/java/org/elasticsearch/wildfly/model/Employee.java
new file mode 100644
index 0000000000000..43f2e5faf5e92
--- /dev/null
+++ b/qa/wildfly/src/main/java/org/elasticsearch/wildfly/model/Employee.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.wildfly.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import javax.ws.rs.Consumes;
+import javax.ws.rs.core.MediaType;
+
+import java.util.List;
+
+@Consumes(MediaType.APPLICATION_JSON)
+public class Employee {
+
+    @JsonProperty(value = "first_name")
+    private String firstName;
+
+    public String getFirstName() {
+        return firstName;
+    }
+
+    public void setFirstName(String firstName) {
+        this.firstName = firstName;
+    }
+
+    @JsonProperty(value = "last_name")
+    private String lastName;
+
+    public String getLastName() {
+        return lastName;
+    }
+
+    public void setLastName(String lastName) {
+        this.lastName = lastName;
+    }
+
+    @JsonProperty(value = "age")
+    private int age;
+
+    public int getAge() {
+        return age;
+    }
+
+    public void setAge(int age) {
+        this.age = age;
+    }
+
+    @JsonProperty(value = "about")
+    private String about;
+
+    public String getAbout() {
+        return about;
+    }
+
+    public void setAbout(String about) {
+        this.about = about;
+    }
+
+    @JsonProperty(value = "interests")
+    private List<String> interests;
+
+    public List<String> getInterests() {
+        return interests;
+    }
+
+    public void setInterests(List<String> interests) {
+        this.interests = interests;
+    }
+
+}
diff --git a/qa/wildfly/src/main/java/org/elasticsearch/wildfly/transport/TransportClientActivator.java b/qa/wildfly/src/main/java/org/elasticsearch/wildfly/transport/TransportClientActivator.java
new file mode 100644
index 0000000000000..881b263f35b97
--- /dev/null
+++ b/qa/wildfly/src/main/java/org/elasticsearch/wildfly/transport/TransportClientActivator.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.wildfly.transport;
+
+import javax.ws.rs.ApplicationPath;
+import javax.ws.rs.core.Application;
+
+import java.util.Collections;
+import java.util.Set;
+
+@ApplicationPath("/transport")
+public class TransportClientActivator extends Application {
+
+    @Override
+    public Set<Class<?>> getClasses() {
+        return Collections.singleton(TransportClientEmployeeResource.class);
+    }
+
+}
diff --git a/qa/wildfly/src/main/java/org/elasticsearch/wildfly/transport/TransportClientEmployeeResource.java b/qa/wildfly/src/main/java/org/elasticsearch/wildfly/transport/TransportClientEmployeeResource.java
new file mode 100644
index 0000000000000..4008bf8801a55
--- /dev/null
+++ b/qa/wildfly/src/main/java/org/elasticsearch/wildfly/transport/TransportClientEmployeeResource.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.wildfly.transport;
+
+import org.elasticsearch.action.get.GetResponse;
+import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.wildfly.model.Employee;
+
+import javax.inject.Inject;
+import javax.ws.rs.GET;
+import javax.ws.rs.PUT;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+
+@Path("/employees")
+public class TransportClientEmployeeResource {
+
+    @Inject
+    private TransportClient client;
+
+    @GET
+    @Path("/{id}")
+    @Produces(MediaType.APPLICATION_JSON)
+    public Response getEmployeeById(final @PathParam("id") Long id) {
+        Objects.requireNonNull(id);
+        final GetResponse response = client.prepareGet("megacorp", "employee", Long.toString(id)).get();
+        if (response.isExists()) {
+            final Map<String, Object> source = response.getSource();
+            final Employee employee = new Employee();
+            employee.setFirstName((String) source.get("first_name"));
+            employee.setLastName((String) source.get("last_name"));
+            employee.setAge((Integer) source.get("age"));
+            employee.setAbout((String) source.get("about"));
+            @SuppressWarnings("unchecked") final List<String> interests = (List<String>) source.get("interests");
+            employee.setInterests(interests);
+            return Response.ok(employee).build();
+        } else {
+            return Response.status(Response.Status.NOT_FOUND).build();
+        }
+    }
+
+    @PUT
+    @Path("/{id}")
+    @Produces(MediaType.APPLICATION_JSON)
+    public Response putEmployeeById(final @PathParam("id") Long id, final Employee employee) throws URISyntaxException, IOException {
+        Objects.requireNonNull(id);
+        Objects.requireNonNull(employee);
+        try (XContentBuilder builder = jsonBuilder()) {
+            builder.startObject();
+            {
+                builder.field("first_name", employee.getFirstName());
+                builder.field("last_name", employee.getLastName());
+                builder.field("age", employee.getAge());
+                builder.field("about", employee.getAbout());
+                if (employee.getInterests() != null) {
+                    builder.startArray("interests");
+                    {
+                        for (final String interest : employee.getInterests()) {
+                            builder.value(interest);
+                        }
+                    }
+                    builder.endArray();
+                }
+            }
+            builder.endObject();
+            final IndexResponse response = client.prepareIndex("megacorp", "employee", Long.toString(id)).setSource(builder).get();
+            if (response.status().getStatus() == 201) {
+                return Response.created(new URI("/employees/" + id)).build();
+            } else {
+                return Response.ok().build();
+            }
+        }
+    }
+
+}
diff --git a/qa/wildfly/src/main/java/org/elasticsearch/wildfly/transport/TransportClientProducer.java b/qa/wildfly/src/main/java/org/elasticsearch/wildfly/transport/TransportClientProducer.java
new file mode 100644
index 0000000000000..a4e65cb355921
--- /dev/null
+++ b/qa/wildfly/src/main/java/org/elasticsearch/wildfly/transport/TransportClientProducer.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.wildfly.transport;
+
+import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.common.io.PathUtils;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.transport.client.PreBuiltTransportClient;
+
+import javax.enterprise.inject.Produces;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.InetAddress;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.Properties;
+
+@SuppressWarnings("unused")
+public final class TransportClientProducer {
+
+    @Produces
+    public TransportClient createTransportClient() throws IOException {
+        final String elasticsearchProperties = System.getProperty("elasticsearch.properties");
+        final Properties properties = new Properties();
+
+        final String transportUri;
+        final String clusterName;
+        try (InputStream is = Files.newInputStream(getPath(elasticsearchProperties))) {
+            properties.load(is);
+            transportUri = properties.getProperty("transport.uri");
+            clusterName = properties.getProperty("cluster.name");
+        }
+
+        final int lastColon = transportUri.lastIndexOf(':');
+        final String host = transportUri.substring(0, lastColon);
+        final int port = Integer.parseInt(transportUri.substring(lastColon + 1));
+        final Settings settings = Settings.builder().put("cluster.name", clusterName).build();
+        final TransportClient transportClient = new PreBuiltTransportClient(settings, Collections.emptyList());
+        transportClient.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(host), port));
+        return transportClient;
+    }
+
+    @SuppressForbidden(reason = "get path not configured in environment")
+    private Path getPath(final String elasticsearchProperties) {
+        return PathUtils.get(elasticsearchProperties);
+    }
+
+}
diff --git a/qa/wildfly/src/main/java/org/elasticsearch/wildfly/transport/TransportJacksonJsonProvider.java b/qa/wildfly/src/main/java/org/elasticsearch/wildfly/transport/TransportJacksonJsonProvider.java
new file mode 100644
index 0000000000000..07585780c0665
--- /dev/null
+++ b/qa/wildfly/src/main/java/org/elasticsearch/wildfly/transport/TransportJacksonJsonProvider.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.wildfly.transport;
+
+import org.jboss.resteasy.plugins.providers.jackson.ResteasyJackson2Provider;
+
+import javax.ws.rs.ext.Provider;
+
+@Provider
+public class TransportJacksonJsonProvider extends ResteasyJackson2Provider {
+}
diff --git a/qa/wildfly/src/main/resources/log4j2.properties b/qa/wildfly/src/main/resources/log4j2.properties
new file mode 100644
index 0000000000000..46877d0de32a0
--- /dev/null
+++ b/qa/wildfly/src/main/resources/log4j2.properties
@@ -0,0 +1,9 @@
+status = error
+
+appender.console.type = Console
+appender.console.name = console
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n
+
+rootLogger.level = info
+rootLogger.appenderRef.console.ref = console
diff --git a/qa/wildfly/src/main/webapp/WEB-INF/beans.xml b/qa/wildfly/src/main/webapp/WEB-INF/beans.xml
new file mode 100644
index 0000000000000..50582333a4220
--- /dev/null
+++ b/qa/wildfly/src/main/webapp/WEB-INF/beans.xml
@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to Elasticsearch under one or more contributor
+  license agreements. See the NOTICE file distributed with
+  this work for additional information regarding copyright
+  ownership. Elasticsearch licenses this file to you under
+  the Apache License, Version 2.0 (the "License"); you may
+  not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+<!-- Marker file indicating CDI should be enabled -->
+<beans xmlns="http://xmlns.jcp.org/xml/ns/javaee" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xsi:schemaLocation="
+      http://xmlns.jcp.org/xml/ns/javaee
+      http://xmlns.jcp.org/xml/ns/javaee/beans_1_1.xsd"
+       bean-discovery-mode="all">
+</beans>
diff --git a/qa/wildfly/src/main/webapp/WEB-INF/jboss-deployment-structure.xml b/qa/wildfly/src/main/webapp/WEB-INF/jboss-deployment-structure.xml
new file mode 100644
index 0000000000000..7191bfe1268aa
--- /dev/null
+++ b/qa/wildfly/src/main/webapp/WEB-INF/jboss-deployment-structure.xml
@@ -0,0 +1,10 @@
+<jboss-deployment-structure>
+    <deployment>
+        <exclusions>
+            <module name="com.fasterxml.jackson.core.jackson-core" />
+            <module name="com.fasterxml.jackson.core.jackson-databind" />
+            <module name="com.fasterxml.jackson.jaxrs.jackson-jaxrs-json-provider" />
+            <module name="org.jboss.resteasy.resteasy-jackson2-provider" />
+        </exclusions>
+    </deployment>
+</jboss-deployment-structure>
diff --git a/qa/wildfly/src/test/java/org/elasticsearch/wildfly/WildflyIT.java b/qa/wildfly/src/test/java/org/elasticsearch/wildfly/WildflyIT.java
new file mode 100644
index 0000000000000..90090c35fa9ee
--- /dev/null
+++ b/qa/wildfly/src/test/java/org/elasticsearch/wildfly/WildflyIT.java
@@ -0,0 +1,101 @@
+package org.elasticsearch.wildfly;/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPut;
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.lucene.util.LuceneTestCase;
+import org.elasticsearch.Build;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterModule;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+
+public class WildflyIT extends LuceneTestCase {
+
+    public void testTransportClient() throws URISyntaxException, IOException {
+        try (CloseableHttpClient client = HttpClientBuilder.create().build()) {
+            final String str = String.format(
+                    Locale.ROOT,
+                    "http://localhost:38080/wildfly-%s%s/transport/employees/1",
+                    Version.CURRENT,
+                    Build.CURRENT.isSnapshot() ? "-SNAPSHOT" : "");
+            final HttpPut put = new HttpPut(new URI(str));
+            final String body;
+            try (XContentBuilder builder = jsonBuilder()) {
+                builder.startObject();
+                {
+                    builder.field("first_name", "John");
+                    builder.field("last_name", "Smith");
+                    builder.field("age", 25);
+                    builder.field("about", "I love to go rock climbing");
+                    builder.startArray("interests");
+                    {
+                        builder.value("sports");
+                        builder.value("music");
+                    }
+                    builder.endArray();
+                }
+                builder.endObject();
+                body = builder.string();
+            }
+            put.setEntity(new StringEntity(body, ContentType.APPLICATION_JSON));
+            try (CloseableHttpResponse response = client.execute(put)) {
+                assertThat(response.getStatusLine().getStatusCode(), equalTo(201));
+            }
+
+            final HttpGet get = new HttpGet(new URI(str));
+            try (
+                    CloseableHttpResponse response = client.execute(get);
+                    XContentParser parser =
+                            JsonXContent.jsonXContent.createParser(
+                                    new NamedXContentRegistry(ClusterModule.getNamedXWriteables()),
+                                    response.getEntity().getContent())) {
+                final Map<String, Object> map = parser.map();
+                assertThat(map.get("first_name"), equalTo("John"));
+                assertThat(map.get("last_name"), equalTo("Smith"));
+                assertThat(map.get("age"), equalTo(25));
+                assertThat(map.get("about"), equalTo("I love to go rock climbing"));
+                final Object interests = map.get("interests");
+                assertThat(interests, instanceOf(List.class));
+                @SuppressWarnings("unchecked") final List<String> interestsAsList = (List<String>) interests;
+                assertThat(interestsAsList, containsInAnyOrder("sports", "music"));
+            }
+        }
+    }
+
+}
diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle
index 2a7de2c726da1..cd49b9f192599 100644
--- a/rest-api-spec/build.gradle
+++ b/rest-api-spec/build.gradle
@@ -1,8 +1,7 @@
-apply plugin: 'java'
-apply plugin: 'com.bmuschko.nexus'
+apply plugin: 'elasticsearch.build'
+apply plugin: 'nebula.maven-base-publish'
+apply plugin: 'nebula.maven-scm'
 
-extraArchive {
-  sources = false
-  javadoc = false
-  tests = false
-}
+test.enabled = false
+jarHell.enabled = false
+licenseHeaders.enabled = false
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/_common.json b/rest-api-spec/src/main/resources/rest-api-spec/api/_common.json
new file mode 100644
index 0000000000000..6edd6d80320af
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/_common.json
@@ -0,0 +1,29 @@
+{
+  "description": "Parameters that are accepted by all API endpoints.",
+  "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/common-options.html",
+  "params": {
+    "pretty": {
+      "type": "boolean",
+      "description": "Pretty format the returned JSON response.",
+      "default": false
+    },
+    "human": {
+      "type": "boolean",
+      "description": "Return human readable values for statistics.",
+      "default": true
+    },
+    "error_trace": {
+      "type": "boolean",
+      "description": "Include the stack trace of returned errors.",
+      "default": false
+    },
+    "source": {
+      "type": "string",
+      "description": "The URL-encoded request definition. Useful for libraries that do not accept a request body for non-POST requests."
+    },
+    "filter_path": {
+      "type": "list",
+      "description": "A comma-separated list of filters used to reduce the respone."
+    }
+  }
+}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/bulk.json b/rest-api-spec/src/main/resources/rest-api-spec/api/bulk.json
index 27d951fa41b34..5d4bce6590eca 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/bulk.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/bulk.json
@@ -1,6 +1,6 @@
 {
   "bulk": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/docs-bulk.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/docs-bulk.html",
     "methods": ["POST", "PUT"],
     "url": {
       "path": "/_bulk",
@@ -39,7 +39,19 @@
         },
         "fields": {
           "type": "list",
-          "description" : "Default comma-separated list of fields to return in the response for updates"
+          "description" : "Default comma-separated list of fields to return in the response for updates, can be overridden on each sub-request"
+        },
+        "_source": {
+          "type" : "list",
+          "description" : "True or false to return the _source field or not, or default list of fields to return, can be overridden on each sub-request"
+        },
+        "_source_exclude": {
+          "type" : "list",
+          "description" : "Default list of fields to exclude from the returned _source field, can be overridden on each sub-request"
+        },
+        "_source_include": {
+          "type" : "list",
+          "description" : "Default list of fields to extract and return from the _source field, can be overridden on each sub-request"
         },
         "pipeline" : {
           "type" : "string",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.aliases.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.aliases.json
index 4bc93460b75ce..b4095db1c9503 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.aliases.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.aliases.json
@@ -1,6 +1,6 @@
 {
   "cat.aliases": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-alias.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/cat-alias.html",
     "methods": ["GET"],
     "url": {
       "path": "/_cat/aliases",
@@ -33,6 +33,10 @@
           "description": "Return help information",
           "default": false
         },
+        "s": {
+          "type": "list",
+          "description" : "Comma-separated list of column names or column aliases to sort by"
+        },
         "v": {
           "type": "boolean",
           "description": "Verbose mode. Display column headers",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.allocation.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.allocation.json
index 6799a67bae876..dba04757efe58 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.allocation.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.allocation.json
@@ -1,6 +1,6 @@
 {
   "cat.allocation": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-allocation.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/cat-allocation.html",
     "methods": ["GET"],
     "url": {
       "path": "/_cat/allocation",
@@ -38,6 +38,10 @@
           "description": "Return help information",
           "default": false
         },
+        "s": {
+          "type": "list",
+          "description" : "Comma-separated list of column names or column aliases to sort by"
+        },
         "v": {
           "type": "boolean",
           "description": "Verbose mode. Display column headers",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.count.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.count.json
index 73803bac86754..a02e44bf74c07 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.count.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.count.json
@@ -1,6 +1,6 @@
 {
   "cat.count": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-count.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/cat-count.html",
     "methods": ["GET"],
     "url": {
       "path": "/_cat/count",
@@ -33,6 +33,10 @@
           "description": "Return help information",
           "default": false
         },
+        "s": {
+          "type": "list",
+          "description" : "Comma-separated list of column names or column aliases to sort by"
+        },
         "v": {
           "type": "boolean",
           "description": "Verbose mode. Display column headers",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.fielddata.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.fielddata.json
index b9776ab588126..85d0f8a2d8d09 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.fielddata.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.fielddata.json
@@ -1,6 +1,6 @@
 {
   "cat.fielddata": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-fielddata.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/cat-fielddata.html",
     "methods": ["GET"],
     "url": {
       "path": "/_cat/fielddata",
@@ -38,6 +38,10 @@
           "description": "Return help information",
           "default": false
         },
+        "s": {
+          "type": "list",
+          "description" : "Comma-separated list of column names or column aliases to sort by"
+        },
         "v": {
           "type": "boolean",
           "description": "Verbose mode. Display column headers",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.health.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.health.json
index 349bf37aa0645..db165d71a4234 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.health.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.health.json
@@ -1,6 +1,6 @@
 {
   "cat.health": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-health.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/cat-health.html",
     "methods": ["GET"],
     "url": {
       "path": "/_cat/health",
@@ -29,6 +29,10 @@
           "description": "Return help information",
           "default": false
         },
+        "s": {
+          "type": "list",
+          "description" : "Comma-separated list of column names or column aliases to sort by"
+        },
         "ts": {
           "type": "boolean",
           "description": "Set to false to disable timestamping",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.help.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.help.json
index f21e6485d586d..27db3cd3a8980 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.help.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.help.json
@@ -1,6 +1,6 @@
 {
   "cat.help": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/cat.html",
     "methods": ["GET"],
     "url": {
       "path": "/_cat",
@@ -12,6 +12,10 @@
           "type": "boolean",
           "description": "Return help information",
           "default": false
+        },
+        "s": {
+          "type": "list",
+          "description" : "Comma-separated list of column names or column aliases to sort by"
         }
       }
     },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.indices.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.indices.json
index 7858fc2ee9ec5..6f2bff72ccca3 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.indices.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.indices.json
@@ -1,6 +1,6 @@
 {
   "cat.indices": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-indices.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/cat-indices.html",
     "methods": ["GET"],
     "url": {
       "path": "/_cat/indices",
@@ -33,6 +33,12 @@
             "type": "list",
             "description" : "Comma-separated list of column names to display"
         },
+        "health": {
+          "type" : "enum",
+          "options" : ["green","yellow","red"],
+          "default" : null,
+          "description" : "A health status (\"green\", \"yellow\", or \"red\" to filter only indices matching the specified health status"
+        },
         "help": {
           "type": "boolean",
           "description": "Return help information",
@@ -43,6 +49,10 @@
           "description": "Set to true to return stats only for primary shards",
           "default": false
         },
+        "s": {
+          "type": "list",
+          "description" : "Comma-separated list of column names or column aliases to sort by"
+        },
         "v": {
           "type": "boolean",
           "description": "Verbose mode. Display column headers",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.master.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.master.json
index bdf474c7a022b..2207e8a4b9858 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.master.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.master.json
@@ -1,6 +1,6 @@
 {
   "cat.master": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-master.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/cat-master.html",
     "methods": ["GET"],
     "url": {
       "path": "/_cat/master",
@@ -29,6 +29,10 @@
           "description": "Return help information",
           "default": false
         },
+        "s": {
+          "type": "list",
+          "description" : "Comma-separated list of column names or column aliases to sort by"
+        },
         "v": {
           "type": "boolean",
           "description": "Verbose mode. Display column headers",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.nodeattrs.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.nodeattrs.json
index 4e44eb820f157..3cd40fa103ca6 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.nodeattrs.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.nodeattrs.json
@@ -1,6 +1,6 @@
 {
   "cat.nodeattrs": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-nodeattrs.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/cat-nodeattrs.html",
     "methods": ["GET"],
     "url": {
       "path": "/_cat/nodeattrs",
@@ -29,6 +29,10 @@
           "description": "Return help information",
           "default": false
         },
+        "s": {
+          "type": "list",
+          "description" : "Comma-separated list of column names or column aliases to sort by"
+        },
         "v": {
           "type": "boolean",
           "description": "Verbose mode. Display column headers",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.nodes.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.nodes.json
index f0fcf3901027d..6551b1394b1e0 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.nodes.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.nodes.json
@@ -1,6 +1,6 @@
 {
   "cat.nodes": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-nodes.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/cat-nodes.html",
     "methods": ["GET"],
     "url": {
       "path": "/_cat/nodes",
@@ -12,6 +12,10 @@
           "type" : "string",
           "description" : "a short version of the Accept header, e.g. json, yaml"
         },
+        "full_id": {
+          "type" : "boolean",
+          "description" : "Return the full node ID instead of the shortened version (default: false)"
+        },
         "local": {
           "type" : "boolean",
           "description" : "Return local information, do not retrieve the state from master node (default: false)"
@@ -29,6 +33,10 @@
           "description": "Return help information",
           "default": false
         },
+        "s": {
+          "type": "list",
+          "description" : "Comma-separated list of column names or column aliases to sort by"
+        },
         "v": {
           "type": "boolean",
           "description": "Verbose mode. Display column headers",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.pending_tasks.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.pending_tasks.json
index 74fc54b9604c8..d3c2202f0ac75 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.pending_tasks.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.pending_tasks.json
@@ -1,6 +1,6 @@
 {
   "cat.pending_tasks": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-pending-tasks.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/cat-pending-tasks.html",
     "methods": ["GET"],
     "url": {
       "path": "/_cat/pending_tasks",
@@ -29,6 +29,10 @@
           "description": "Return help information",
           "default": false
         },
+        "s": {
+          "type": "list",
+          "description" : "Comma-separated list of column names or column aliases to sort by"
+        },
         "v": {
           "type": "boolean",
           "description": "Verbose mode. Display column headers",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.plugins.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.plugins.json
index 5822f64c8e26e..216a8f02f4f89 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.plugins.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.plugins.json
@@ -1,6 +1,6 @@
 {
   "cat.plugins": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-plugins.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/cat-plugins.html",
     "methods": ["GET"],
     "url": {
       "path": "/_cat/plugins",
@@ -27,6 +27,10 @@
           "description": "Return help information",
           "default": false
         },
+        "s": {
+          "type": "list",
+          "description" : "Comma-separated list of column names or column aliases to sort by"
+        },
         "v": {
           "type": "boolean",
           "description": "Verbose mode. Display column headers",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.recovery.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.recovery.json
index b220a5eda746e..c1aa9f857ace6 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.recovery.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.recovery.json
@@ -1,6 +1,6 @@
 {
   "cat.recovery": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-recovery.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/cat-recovery.html",
     "methods": ["GET"],
     "url": {
       "path": "/_cat/recovery",
@@ -34,6 +34,10 @@
           "description": "Return help information",
           "default": false
         },
+        "s": {
+          "type": "list",
+          "description" : "Comma-separated list of column names or column aliases to sort by"
+        },
         "v": {
           "type": "boolean",
           "description": "Verbose mode. Display column headers",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.repositories.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.repositories.json
index ace0a1ea3c991..26e3beae41645 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.repositories.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.repositories.json
@@ -1,6 +1,6 @@
 {
   "cat.repositories": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-repositories.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/cat-repositories.html",
     "methods": ["GET"],
     "url": {
       "path": "/_cat/repositories",
@@ -30,6 +30,10 @@
           "description": "Return help information",
           "default": false
         },
+        "s": {
+          "type": "list",
+          "description" : "Comma-separated list of column names or column aliases to sort by"
+        },
         "v": {
           "type": "boolean",
           "description": "Verbose mode. Display column headers",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segments.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segments.json
index eed4a6277364b..1f454cbb6bac9 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segments.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segments.json
@@ -1,6 +1,6 @@
 {
   "cat.segments": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-segments.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/cat-segments.html",
     "methods": ["GET"],
     "url": {
       "path": "/_cat/segments",
@@ -16,6 +16,11 @@
           "type" : "string",
           "description" : "a short version of the Accept header, e.g. json, yaml"
         },
+        "bytes": {
+          "type": "enum",
+          "description" : "The unit in which to display byte values",
+          "options": [ "b", "k", "kb", "m", "mb", "g", "gb", "t", "tb", "p", "pb" ]
+        },
         "h": {
             "type": "list",
             "description" : "Comma-separated list of column names to display"
@@ -25,6 +30,10 @@
           "description": "Return help information",
           "default": false
         },
+        "s": {
+          "type": "list",
+          "description" : "Comma-separated list of column names or column aliases to sort by"
+        },
         "v": {
           "type": "boolean",
           "description": "Verbose mode. Display column headers",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.shards.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.shards.json
index a9a1a30770f3a..d74c18219f988 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.shards.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.shards.json
@@ -1,6 +1,6 @@
 {
   "cat.shards": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-shards.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/cat-shards.html",
     "methods": ["GET"],
     "url": {
       "path": "/_cat/shards",
@@ -16,6 +16,11 @@
           "type" : "string",
           "description" : "a short version of the Accept header, e.g. json, yaml"
         },
+        "bytes": {
+          "type": "enum",
+          "description" : "The unit in which to display byte values",
+          "options": [ "b", "k", "kb", "m", "mb", "g", "gb", "t", "tb", "p", "pb" ]
+        },
         "local": {
           "type" : "boolean",
           "description" : "Return local information, do not retrieve the state from master node (default: false)"
@@ -33,6 +38,10 @@
           "description": "Return help information",
           "default": false
         },
+        "s": {
+          "type": "list",
+          "description" : "Comma-separated list of column names or column aliases to sort by"
+        },
         "v": {
           "type": "boolean",
           "description": "Verbose mode. Display column headers",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.snapshots.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.snapshots.json
index 68468dd542a9f..d842388d03c40 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.snapshots.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.snapshots.json
@@ -1,10 +1,12 @@
 {
   "cat.snapshots": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-snapshots.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/cat-snapshots.html",
     "methods": ["GET"],
     "url": {
-      "path": "/_cat/snapshots/{repository}",
-      "paths": ["/_cat/snapshots/{repository}"],
+      "path": "/_cat/snapshots",
+      "paths": [
+        "/_cat/snapshots",
+        "/_cat/snapshots/{repository}"],
       "parts": {
         "repository": {
           "type" : "list",
@@ -35,6 +37,10 @@
           "description": "Return help information",
           "default": false
         },
+        "s": {
+          "type": "list",
+          "description" : "Comma-separated list of column names or column aliases to sort by"
+        },
         "v": {
           "type": "boolean",
           "description": "Verbose mode. Display column headers",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.tasks.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.tasks.json
index 73f1281ff5f60..e28a0572f9a59 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.tasks.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.tasks.json
@@ -1,6 +1,6 @@
 {
   "cat.tasks": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/tasks.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/tasks.html",
     "methods": ["GET"],
     "url": {
       "path": "/_cat/tasks",
@@ -41,6 +41,10 @@
           "description": "Return help information",
           "default": false
         },
+        "s": {
+          "type": "list",
+          "description" : "Comma-separated list of column names or column aliases to sort by"
+        },
         "v": {
           "type": "boolean",
           "description": "Verbose mode. Display column headers",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.templates.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.templates.json
new file mode 100644
index 0000000000000..4bcea5278c8a8
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.templates.json
@@ -0,0 +1,49 @@
+{
+  "cat.templates": {
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/cat-templates.html",
+    "methods": ["GET"],
+    "url": {
+      "path": "/_cat/templates",
+      "paths": ["/_cat/templates", "/_cat/templates/{name}"],
+      "parts": {
+        "name": {
+          "type" : "string",
+          "description" : "A pattern that returned template names must match"
+        }
+      },
+      "params": {
+        "format": {
+          "type" : "string",
+          "description" : "a short version of the Accept header, e.g. json, yaml"
+        },
+        "local": {
+          "type" : "boolean",
+          "description" : "Return local information, do not retrieve the state from master node (default: false)"
+        },
+        "master_timeout": {
+          "type" : "time",
+          "description" : "Explicit operation timeout for connection to master node"
+        },
+        "h": {
+          "type": "list",
+          "description" : "Comma-separated list of column names to display"
+        },
+        "help": {
+          "type": "boolean",
+          "description": "Return help information",
+          "default": false
+        },
+        "s": {
+          "type": "list",
+          "description" : "Comma-separated list of column names or column aliases to sort by"
+        },
+        "v": {
+          "type": "boolean",
+          "description": "Verbose mode. Display column headers",
+          "default": false
+        }
+      }
+    },
+    "body": null
+  }
+}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.thread_pool.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.thread_pool.json
index d6ba5ebc6b19a..8d3e7e0a972ec 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.thread_pool.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.thread_pool.json
@@ -1,11 +1,15 @@
 {
   "cat.thread_pool": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-thread-pool.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/cat-thread-pool.html",
     "methods": ["GET"],
     "url": {
       "path": "/_cat/thread_pool",
-      "paths": ["/_cat/thread_pool","/_cat/thread_pool/{thread_pools}"],
+      "paths": ["/_cat/thread_pool","/_cat/thread_pool/{thread_pool_patterns}"],
       "parts": {
+        "thread_pool_patterns": {
+          "type": "list",
+          "description": "A comma-separated list of regular-expressions to filter the thread pools in the output"
+        }
       },
       "params": {
         "format": {
@@ -34,14 +38,14 @@
           "description": "Return help information",
           "default": false
         },
+        "s": {
+          "type": "list",
+          "description" : "Comma-separated list of column names or column aliases to sort by"
+        },
         "v": {
           "type": "boolean",
           "description": "Verbose mode. Display column headers",
           "default": false
-        },
-        "thread_pool_patterns": {
-          "type": "list",
-          "description": "A comma-separated list of regular-expressions to filter the thread pools in the output"
         }
       }
     },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/clear_scroll.json b/rest-api-spec/src/main/resources/rest-api-spec/api/clear_scroll.json
index 6b82d1dc4742a..ecdfba142538d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/clear_scroll.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/clear_scroll.json
@@ -1,6 +1,6 @@
 {
   "clear_scroll": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/search-request-scroll.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/search-request-scroll.html",
     "methods": ["DELETE"],
     "url": {
       "path": "/_search/scroll/{scroll_id}",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.allocation_explain.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.allocation_explain.json
index 26b24cfb69760..bd19a8c01eb4c 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.allocation_explain.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.allocation_explain.json
@@ -1,6 +1,6 @@
 {
   "cluster.allocation_explain": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-allocation-explain.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/cluster-allocation-explain.html",
     "methods": ["GET", "POST"],
     "url": {
       "path": "/_cluster/allocation/explain",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.get_settings.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.get_settings.json
index baba4e3436abc..bacb4300c8210 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.get_settings.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.get_settings.json
@@ -1,6 +1,6 @@
 {
   "cluster.get_settings": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-update-settings.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/cluster-update-settings.html",
     "methods": ["GET"],
     "url": {
       "path": "/_cluster/settings",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.health.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.health.json
index f1a6a98217c1d..60219e9e53953 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.health.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.health.json
@@ -1,6 +1,6 @@
 {
   "cluster.health": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-health.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/cluster-health.html",
     "methods": ["GET"],
     "url": {
       "path": "/_cluster/health",
@@ -41,7 +41,7 @@
         "wait_for_events": {
           "type" : "enum",
           "options" : ["immediate", "urgent", "high", "normal", "low", "languid"],
-          "description" : "Wait until all currently queued events with the given priorty are processed"
+          "description" : "Wait until all currently queued events with the given priority are processed"
         },
         "wait_for_no_relocating_shards": {
           "type" : "boolean",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.pending_tasks.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.pending_tasks.json
index fb5e1609ff39d..be67c0d425c44 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.pending_tasks.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.pending_tasks.json
@@ -1,6 +1,6 @@
 {
   "cluster.pending_tasks": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-pending.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/cluster-pending.html",
     "methods": ["GET"],
     "url": {
       "path": "/_cluster/pending_tasks",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.put_settings.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.put_settings.json
index 393d1350dd3e2..fdac1be1128de 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.put_settings.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.put_settings.json
@@ -1,6 +1,6 @@
 {
   "cluster.put_settings": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-update-settings.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/cluster-update-settings.html",
     "methods": ["PUT"],
     "url": {
       "path": "/_cluster/settings",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.reroute.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.reroute.json
index 8bb85ca087ae8..70fd3e450f8fc 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.reroute.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.reroute.json
@@ -1,6 +1,6 @@
 {
   "cluster.reroute": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-reroute.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/cluster-reroute.html",
     "methods": ["POST"],
     "url": {
       "path": "/_cluster/reroute",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.state.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.state.json
index 0f4bf6f21ad4a..de5e88d42d875 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.state.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.state.json
@@ -1,6 +1,6 @@
 {
   "cluster.state": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-state.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/cluster-state.html",
     "methods": ["GET"],
     "url": {
       "path": "/_cluster/state",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.stats.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.stats.json
index 2bccb20f36e35..1f990cae2c8d7 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.stats.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.stats.json
@@ -1,6 +1,6 @@
 {
   "cluster.stats": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-stats.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/cluster-stats.html",
     "methods": ["GET"],
     "url": {
       "path": "/_cluster/stats",
@@ -16,11 +16,6 @@
           "type": "boolean",
           "description": "Return settings in flat format (default: false)"
         },
-        "human": {
-            "type": "boolean",
-            "description": "Whether to return time and byte values in human-readable format.",
-            "default": false
-        },
         "timeout": {
           "type" : "time",
           "description" : "Explicit operation timeout"
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/count.json b/rest-api-spec/src/main/resources/rest-api-spec/api/count.json
index 9048f982712b0..57d21cb06a2fa 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/count.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/count.json
@@ -1,6 +1,6 @@
 {
   "count": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/search-count.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/search-count.html",
     "methods": ["POST", "GET"],
     "url": {
       "path": "/_count",
@@ -68,9 +68,9 @@
           "type" : "boolean",
           "description" : "Specify whether format-based query failures (such as providing text to a numeric field) should be ignored"
         },
-        "lowercase_expanded_terms": {
-          "type" : "boolean",
-          "description" : "Specify whether query terms should be lowercased"
+        "terminate_after" : {
+          "type" : "number",
+          "description" : "The maximum count for each shard, upon reaching which the query execution will terminate early"
         }
       }
     },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/count_percolate.json b/rest-api-spec/src/main/resources/rest-api-spec/api/count_percolate.json
index 584f33685d30b..1f317f4302260 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/count_percolate.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/count_percolate.json
@@ -1,6 +1,6 @@
 {
   "count_percolate": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/search-percolate.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/search-percolate.html",
     "methods": ["GET", "POST"],
     "url": {
       "path": "/{index}/{type}/_percolate/count",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/create.json b/rest-api-spec/src/main/resources/rest-api-spec/api/create.json
new file mode 100644
index 0000000000000..cb72671a9b5c8
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/create.json
@@ -0,0 +1,75 @@
+{
+  "create": {
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/docs-index_.html",
+    "methods": ["PUT","POST"],
+    "url": {
+      "path": "/{index}/{type}/{id}/_create",
+      "paths": ["/{index}/{type}/{id}/_create"],
+      "parts": {
+        "id": {
+          "type" : "string",
+          "required" : true,
+          "description" : "Document ID"
+        },
+        "index": {
+          "type" : "string",
+          "required" : true,
+          "description" : "The name of the index"
+        },
+        "type": {
+          "type" : "string",
+          "required" : true,
+          "description" : "The type of the document"
+        }
+      },
+      "params": {
+        "wait_for_active_shards": {
+          "type" : "string",
+          "description" : "Sets the number of shard copies that must be active before proceeding with the index operation. Defaults to 1, meaning the primary shard only. Set to `all` for all shard copies, otherwise set to any non-negative value less than or equal to the total number of copies for the shard (number of replicas + 1)"
+        },
+        "parent": {
+          "type" : "string",
+          "description" : "ID of the parent document"
+        },
+        "refresh": {
+          "type" : "enum",
+          "options": ["true", "false", "wait_for"],
+          "description" : "If `true` then refresh the affected shards to make this operation visible to search, if `wait_for` then wait for a refresh to make this operation visible to search, if `false` (the default) then do nothing with refreshes."
+        },
+        "routing": {
+          "type" : "string",
+          "description" : "Specific routing value"
+        },
+        "timeout": {
+          "type" : "time",
+          "description" : "Explicit operation timeout"
+        },
+        "timestamp": {
+          "type" : "time",
+          "description" : "Explicit timestamp for the document"
+        },
+        "ttl": {
+          "type" : "time",
+          "description" : "Expiration time for the document"
+        },
+        "version" : {
+          "type" : "number",
+          "description" : "Explicit version number for concurrency control"
+        },
+        "version_type": {
+          "type" : "enum",
+          "options" : ["internal", "external", "external_gte", "force"],
+          "description" : "Specific version type"
+        },
+        "pipeline" : {
+          "type" : "string",
+          "description" : "The pipeline id to preprocess incoming documents with"
+        }
+      }
+    },
+    "body": {
+      "description" : "The document",
+      "required" : true
+    }
+  }
+}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/delete.json b/rest-api-spec/src/main/resources/rest-api-spec/api/delete.json
index 389d00c670622..a2ee6de8cbc56 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/delete.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/delete.json
@@ -1,6 +1,6 @@
 {
   "delete": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/docs-delete.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/docs-delete.html",
     "methods": ["DELETE"],
     "url": {
       "path": "/{index}/{type}/{id}",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query.json b/rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query.json
index f8743e1d1ba11..e47ca6e4381e3 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query.json
@@ -1,6 +1,6 @@
 {
   "delete_by_query": {
-    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/master/docs-delete-by-query.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/docs-delete-by-query.html",
     "methods": ["POST"],
     "url": {
       "path": "/{index}/_delete_by_query",
@@ -36,18 +36,6 @@
           "type" : "string",
           "description" : "The field to use as default where no field prefix is given in the query string"
         },
-        "explain": {
-          "type" : "boolean",
-          "description" : "Specify whether to return detailed information about score computation as part of a hit"
-        },
-        "fields": {
-          "type" : "list",
-          "description" : "A comma-separated list of fields to return as part of a hit"
-        },
-        "fielddata_fields": {
-          "type" : "list",
-          "description" : "A comma-separated list of fields to return as the field data representation of a field for each hit"
-        },
         "from": {
           "type" : "number",
           "description" : "Starting offset (default: 0)"
@@ -77,10 +65,6 @@
           "type" : "boolean",
           "description" : "Specify whether format-based query failures (such as providing text to a numeric field) should be ignored"
         },
-        "lowercase_expanded_terms": {
-          "type" : "boolean",
-          "description" : "Specify whether query terms should be lowercased"
-        },
         "preference": {
           "type" : "string",
           "description" : "Specify the node or shard the operation should be performed on (default: random)"
@@ -94,7 +78,7 @@
           "description" : "A comma-separated list of specific routing values"
         },
         "scroll": {
-          "type" : "duration",
+          "type" : "time",
           "description" : "Specify how long a consistent view of the index should be maintained for scrolled search"
         },
         "search_type": {
@@ -134,32 +118,6 @@
           "type" : "list",
           "description" : "Specific 'tag' of the request for logging and statistical purposes"
         },
-        "suggest_field": {
-          "type" : "string",
-          "description" : "Specify which field to use for suggestions"
-        },
-        "suggest_mode": {
-          "type" : "enum",
-          "options" : ["missing", "popular", "always"],
-          "default" : "missing",
-          "description" : "Specify suggest mode"
-        },
-        "suggest_size": {
-          "type" : "number",
-          "description" : "How many suggestions to return in response"
-        },
-        "suggest_text": {
-          "type" : "text",
-          "description" : "The source text for which the suggestions should be returned"
-        },
-        "timeout": {
-          "type" : "time",
-          "description" : "Explicit operation timeout"
-        },
-        "track_scores": {
-          "type" : "boolean",
-          "description": "Whether to calculate and return scores even if they are not used for sorting"
-        },
         "version": {
           "type" : "boolean",
           "description" : "Specify whether to return document version as part of a hit"
@@ -182,19 +140,24 @@
           "description" : "Sets the number of shard copies that must be active before proceeding with the delete by query operation. Defaults to 1, meaning the primary shard only. Set to `all` for all shard copies, otherwise set to any non-negative value less than or equal to the total number of copies for the shard (number of replicas + 1)"
         },
         "scroll_size": {
-          "type": "integer",
+          "type": "number",
           "defaut_value": 100,
           "description": "Size on the scroll request powering the update_by_query"
         },
         "wait_for_completion": {
            "type" : "boolean",
-           "default": false,
+           "default": true,
            "description" : "Should the request should block until the delete-by-query is complete."
         },
         "requests_per_second": {
-          "type": "float",
+          "type": "number",
           "default": 0,
-          "description": "The throttle for this request in sub-requests per second. -1 means set no throttle."
+          "description": "The throttle for this request in sub-requests per second. -1 means no throttle."
+        },
+        "slices": {
+          "type": "number",
+          "default": 1,
+          "description": "The number of slices this task should be divided into. Defaults to 1 meaning the task isn't sliced into subtasks."
         }
       }
     },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/delete_script.json b/rest-api-spec/src/main/resources/rest-api-spec/api/delete_script.json
index 2c9742057fe86..ada9c14214be0 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/delete_script.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/delete_script.json
@@ -1,10 +1,10 @@
 {
   "delete_script": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/modules-scripting.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/modules-scripting.html",
     "methods": ["DELETE"],
     "url": {
-      "path": "/_scripts/{lang}/{id}",
-      "paths": [ "/_scripts/{lang}/{id}" ],
+      "path": "/_scripts/{lang}",
+      "paths": [ "/_scripts/{lang}", "/_scripts/{lang}/{id}" ],
       "parts": {
         "id": {
           "type" : "string",
@@ -18,6 +18,14 @@
         }
       },
       "params" : {
+        "timeout": {
+          "type" : "time",
+          "description" : "Explicit operation timeout"
+        },
+        "master_timeout": {
+          "type" : "time",
+          "description" : "Specify timeout for connection to master"
+        }
       }
     },
     "body": null
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/delete_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/delete_template.json
index e1e40bd95b63e..7e2201e3ac272 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/delete_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/delete_template.json
@@ -1,6 +1,6 @@
 {
   "delete_template": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/search-template.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/search-template.html",
     "methods": ["DELETE"],
     "url": {
       "path": "/_search/template/{id}",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/exists.json b/rest-api-spec/src/main/resources/rest-api-spec/api/exists.json
index 2e64a2da7cdec..7ea969a2b4455 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/exists.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/exists.json
@@ -1,6 +1,6 @@
 {
   "exists": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/docs-get.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/docs-get.html",
     "methods": ["HEAD"],
     "url": {
       "path": "/{index}/{type}/{id}",
@@ -23,6 +23,10 @@
         }
       },
       "params": {
+        "stored_fields": {
+          "type": "list",
+          "description" : "A comma-separated list of stored fields to return in the response"
+        },
         "parent": {
           "type" : "string",
           "description" : "The ID of the parent document"
@@ -42,6 +46,27 @@
         "routing": {
           "type" : "string",
           "description" : "Specific routing value"
+        },
+        "_source": {
+          "type" : "list",
+          "description" : "True or false to return the _source field or not, or a list of fields to return"
+        },
+        "_source_exclude": {
+          "type" : "list",
+          "description" : "A list of fields to exclude from the returned _source field"
+        },
+        "_source_include": {
+          "type" : "list",
+          "description" : "A list of fields to extract and return from the _source field"
+        },
+        "version" : {
+          "type" : "number",
+          "description" : "Explicit version number for concurrency control"
+        },
+        "version_type": {
+          "type" : "enum",
+          "options" : ["internal", "external", "external_gte", "force"],
+          "description" : "Specific version type"
         }
       }
     },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/exists_source.json b/rest-api-spec/src/main/resources/rest-api-spec/api/exists_source.json
new file mode 100644
index 0000000000000..3e561a21146e6
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/exists_source.json
@@ -0,0 +1,71 @@
+{
+  "exists_source": {
+    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/docs-get.html",
+    "methods": ["HEAD"],
+    "url": {
+      "path": "/{index}/{type}/{id}/_source",
+      "paths": ["/{index}/{type}/{id}/_source"],
+      "parts": {
+        "id": {
+          "type" : "string",
+          "required" : true,
+          "description" : "The document ID"
+        },
+        "index": {
+          "type" : "string",
+          "required" : true,
+          "description" : "The name of the index"
+        },
+        "type": {
+          "type" : "string",
+          "required" : true,
+          "description" : "The type of the document; use `_all` to fetch the first document matching the ID across all types"
+        }
+      },
+      "params": {
+        "parent": {
+          "type" : "string",
+          "description" : "The ID of the parent document"
+        },
+        "preference": {
+          "type" : "string",
+          "description" : "Specify the node or shard the operation should be performed on (default: random)"
+        },
+        "realtime": {
+          "type" : "boolean",
+          "description" : "Specify whether to perform the operation in realtime or search mode"
+        },
+        "refresh": {
+          "type" : "boolean",
+          "description" : "Refresh the shard containing the document before performing the operation"
+        },
+        "routing": {
+          "type" : "string",
+          "description" : "Specific routing value"
+        },
+        "_source": {
+          "type" : "list",
+          "description" : "True or false to return the _source field or not, or a list of fields to return"
+        },
+        "_source_exclude": {
+          "type" : "list",
+          "description" : "A list of fields to exclude from the returned _source field"
+        },
+        "_source_include": {
+          "type" : "list",
+          "description" : "A list of fields to extract and return from the _source field"
+        },
+        "version" : {
+          "type" : "number",
+          "description" : "Explicit version number for concurrency control"
+        },
+        "version_type": {
+          "type" : "enum",
+          "options" : ["internal", "external", "external_gte", "force"],
+          "description" : "Specific version type"
+        }
+      }
+    },
+    "body": null
+  }
+}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/explain.json b/rest-api-spec/src/main/resources/rest-api-spec/api/explain.json
index 30b5deff1d3d1..8b6c8ac089924 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/explain.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/explain.json
@@ -1,6 +1,6 @@
 {
   "explain": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/search-explain.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/search-explain.html",
     "methods": ["GET", "POST"],
     "url": {
       "path": "/{index}/{type}/{id}/_explain",
@@ -41,18 +41,14 @@
           "type" : "string",
           "description" : "The default field for query string query (default: _all)"
         },
-        "fields": {
+        "stored_fields": {
           "type": "list",
-          "description" : "A comma-separated list of fields to return in the response"
+          "description" : "A comma-separated list of stored fields to return in the response"
         },
         "lenient": {
           "type" : "boolean",
           "description" : "Specify whether format-based query failures (such as providing text to a numeric field) should be ignored"
         },
-        "lowercase_expanded_terms": {
-          "type" : "boolean",
-          "description" : "Specify whether query terms should be lowercased"
-        },
         "parent": {
           "type" : "string",
           "description" : "The ID of the parent document"
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/field_caps.json b/rest-api-spec/src/main/resources/rest-api-spec/api/field_caps.json
new file mode 100644
index 0000000000000..43a7ddfd6eb5a
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/field_caps.json
@@ -0,0 +1,43 @@
+{
+  "field_caps": {
+    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/5.x/search-field-caps.html",
+    "methods": ["GET", "POST"],
+    "url": {
+      "path": "/_field_caps",
+      "paths": [
+        "/_field_caps",
+        "/{index}/_field_caps"
+      ],
+      "parts": {
+        "index": {
+          "type" : "list",
+          "description" : "A comma-separated list of index names; use `_all` or empty string to perform the operation on all indices"
+        }
+      },
+      "params": {
+        "fields": {
+          "type" : "list",
+          "description" : "A comma-separated list of field names"
+        },
+        "ignore_unavailable": {
+          "type" : "boolean",
+          "description" : "Whether specified concrete indices should be ignored when unavailable (missing or closed)"
+        },
+        "allow_no_indices": {
+          "type" : "boolean",
+          "description" : "Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified)"
+        },
+        "expand_wildcards": {
+          "type" : "enum",
+          "options" : ["open","closed","none","all"],
+          "default" : "open",
+          "description" : "Whether to expand wildcard expression to concrete indices that are open, closed or both."
+        }
+      }
+    },
+    "body": {
+      "description": "Field json objects containing an array of field names",
+      "required": false
+    }
+  }
+}
\ No newline at end of file
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/field_stats.json b/rest-api-spec/src/main/resources/rest-api-spec/api/field_stats.json
index c7e95dd683ee9..a21c77eb550b3 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/field_stats.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/field_stats.json
@@ -1,6 +1,6 @@
 {
   "field_stats": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/search-field-stats.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/search-field-stats.html",
     "methods": ["GET", "POST"],
     "url": {
       "path": "/_field_stats",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/get.json b/rest-api-spec/src/main/resources/rest-api-spec/api/get.json
index e0e5170f9c3ac..d5eb0129fb9df 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/get.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/get.json
@@ -1,6 +1,6 @@
 {
   "get": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/docs-get.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/docs-get.html",
     "methods": ["GET"],
     "url": {
       "path": "/{index}/{type}/{id}",
@@ -23,9 +23,9 @@
         }
       },
       "params": {
-        "fields": {
+        "stored_fields": {
           "type": "list",
-          "description" : "A comma-separated list of fields to return in the response"
+          "description" : "A comma-separated list of stored fields to return in the response"
         },
         "parent": {
           "type" : "string",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/get_script.json b/rest-api-spec/src/main/resources/rest-api-spec/api/get_script.json
index 163f147aa4298..49b8e8b2faed9 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/get_script.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/get_script.json
@@ -1,10 +1,10 @@
 {
   "get_script": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/modules-scripting.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/modules-scripting.html",
     "methods": ["GET"],
     "url": {
-      "path": "/_scripts/{lang}/{id}",
-      "paths": [ "/_scripts/{lang}/{id}" ],
+      "path": "/_scripts/{lang}",
+      "paths": [ "/_scripts/{lang}", "/_scripts/{lang}/{id}" ],
       "parts": {
         "id": {
           "type" : "string",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/get_source.json b/rest-api-spec/src/main/resources/rest-api-spec/api/get_source.json
index 36065a13f6e99..ce076821ddad1 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/get_source.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/get_source.json
@@ -1,6 +1,6 @@
 {
   "get_source": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/docs-get.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/docs-get.html",
     "methods": ["GET"],
     "url": {
       "path": "/{index}/{type}/{id}/_source",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/get_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/get_template.json
index 487b81c5350ec..6a5eb20c6cc7d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/get_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/get_template.json
@@ -1,6 +1,6 @@
 {
   "get_template": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/search-template.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/search-template.html",
     "methods": ["GET"],
     "url": {
       "path": "/_search/template/{id}",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/index.json b/rest-api-spec/src/main/resources/rest-api-spec/api/index.json
index b46e3fcf116c8..4de9a9e5c824f 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/index.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/index.json
@@ -1,6 +1,6 @@
 {
   "index": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/docs-index_.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/docs-index_.html",
     "methods": ["POST", "PUT"],
     "url": {
       "path": "/{index}/{type}",
@@ -39,7 +39,7 @@
         "refresh": {
           "type" : "enum",
           "options": ["true", "false", "wait_for"],
-          "description" : "If `true` then refresh the effected shards to make this operation visible to search, if `wait_for` then wait for a refresh to make this operation visible to search, if `false` (the default) then do nothing with refreshes."
+          "description" : "If `true` then refresh the affected shards to make this operation visible to search, if `wait_for` then wait for a refresh to make this operation visible to search, if `false` (the default) then do nothing with refreshes."
         },
         "routing": {
           "type" : "string",
@@ -54,7 +54,7 @@
           "description" : "Explicit timestamp for the document"
         },
         "ttl": {
-          "type" : "duration",
+          "type" : "time",
           "description" : "Expiration time for the document"
         },
         "version" : {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.analyze.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.analyze.json
index 881382ffa005f..9002c1fd48d83 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.analyze.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.analyze.json
@@ -1,6 +1,6 @@
 {
   "indices.analyze": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-analyze.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/indices-analyze.html",
     "methods": ["GET", "POST"],
     "url": {
       "path": "/_analyze",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.clear_cache.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.clear_cache.json
index 31a50c4a8c420..e25d2f4706bab 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.clear_cache.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.clear_cache.json
@@ -1,6 +1,6 @@
 {
   "indices.clear_cache": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-clearcache.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/indices-clearcache.html",
     "methods": ["POST", "GET"],
     "url": {
       "path": "/_cache/clear",
@@ -50,6 +50,10 @@
           "type" : "boolean",
           "description" : "Clear the recycler cache"
         },
+        "request_cache": {
+          "type" : "boolean",
+          "description" : "Clear request cache"
+        },
         "request": {
           "type" : "boolean",
           "description" : "Clear request cache"
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.close.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.close.json
index 4eaa93030ee7b..f4fada96dad68 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.close.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.close.json
@@ -1,6 +1,6 @@
 {
   "indices.close": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-open-close.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/indices-open-close.html",
     "methods": ["POST"],
     "url": {
       "path": "/{index}/_close",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.create.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.create.json
index 1433c893e251e..80bfa5e02bff9 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.create.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.create.json
@@ -1,6 +1,6 @@
 {
   "indices.create": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-create-index.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/indices-create-index.html",
     "methods": ["PUT"],
     "url": {
       "path": "/{index}",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete.json
index 829e7a2ded1ee..203fc6dd8c7df 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete.json
@@ -1,6 +1,6 @@
 {
   "indices.delete": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-delete-index.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/indices-delete-index.html",
     "methods": ["DELETE"],
     "url": {
       "path": "/{index}",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_alias.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_alias.json
index 30a32fbfef871..83578a75bebc1 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_alias.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_alias.json
@@ -1,6 +1,6 @@
 {
   "indices.delete_alias": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-aliases.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/indices-aliases.html",
     "methods": ["DELETE"],
     "url": {
       "path": "/{index}/_alias/{name}",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_template.json
index b311c9bbda82a..3c5c2ecc9b2a3 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_template.json
@@ -1,6 +1,6 @@
 {
   "indices.delete_template": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-templates.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/indices-templates.html",
     "methods": ["DELETE"],
     "url": {
       "path": "/_template/{name}",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists.json
index ebfd5cc3a7335..889a54c0a16e4 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists.json
@@ -1,35 +1,44 @@
 {
   "indices.exists": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-exists.html",
-    "methods": ["HEAD"],
+    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/5.x/indices-exists.html",
+    "methods": [ "HEAD" ],
     "url": {
       "path": "/{index}",
-      "paths": ["/{index}"],
+      "paths": [ "/{index}" ],
       "parts": {
         "index": {
-          "type" : "list",
-          "required" : true,
-          "description" : "A comma-separated list of indices to check"
+          "type": "list",
+          "required": true,
+          "description": "A comma-separated list of index names"
         }
       },
       "params": {
+        "local": {
+          "type": "boolean",
+          "description": "Return local information, do not retrieve the state from master node (default: false)"
+        },
         "ignore_unavailable": {
-            "type" : "boolean",
-            "description" : "Whether specified concrete indices should be ignored when unavailable (missing or closed)"
+          "type": "boolean",
+          "description": "Ignore unavailable indexes (default: false)"
         },
         "allow_no_indices": {
-            "type" : "boolean",
-            "description" : "Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified)"
+          "type": "boolean",
+          "description": "Ignore if a wildcard expression resolves to no concrete indices (default: false)"
         },
         "expand_wildcards": {
-            "type" : "enum",
-            "options" : ["open","closed","none","all"],
-            "default" : "open",
-            "description" : "Whether to expand wildcard expression to concrete indices that are open, closed or both."
+          "type": "enum",
+          "options": [ "open", "closed", "none", "all" ],
+          "default": "open",
+          "description": "Whether wildcard expressions should get expanded to open or closed indices (default: open)"
         },
-        "local": {
-            "type": "boolean",
-            "description": "Return local information, do not retrieve the state from master node (default: false)"
+        "flat_settings": {
+          "type": "boolean",
+          "description": "Return settings in flat format (default: false)"
+        },
+        "include_defaults": {
+          "type": "boolean",
+          "description": "Whether to return all default setting for each of the indices.",
+          "default": false
         }
       }
     },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_alias.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_alias.json
index 8862481c18fbc..3d696b9a46eea 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_alias.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_alias.json
@@ -1,10 +1,10 @@
 {
   "indices.exists_alias": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-aliases.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/indices-aliases.html",
     "methods": ["HEAD"],
     "url": {
       "path": "/_alias/{name}",
-      "paths": ["/_alias/{name}", "/{index}/_alias/{name}", "/{index}/_alias"],
+      "paths": ["/_alias/{name}", "/{index}/_alias/{name}"],
       "parts": {
         "index": {
           "type" : "list",
@@ -17,22 +17,22 @@
       },
       "params": {
         "ignore_unavailable": {
-            "type" : "boolean",
-            "description" : "Whether specified concrete indices should be ignored when unavailable (missing or closed)"
+          "type" : "boolean",
+          "description" : "Whether specified concrete indices should be ignored when unavailable (missing or closed)"
         },
         "allow_no_indices": {
-           "type" : "boolean",
-           "description" : "Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified)"
+          "type" : "boolean",
+          "description" : "Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified)"
         },
         "expand_wildcards": {
-           "type" : "enum",
-           "options" : ["open","closed","none","all"],
-           "default" : ["open", "closed"],
-           "description" : "Whether to expand wildcard expression to concrete indices that are open, closed or both."
+          "type" : "enum",
+          "options" : ["open","closed","none","all"],
+          "default" : "all",
+          "description" : "Whether to expand wildcard expression to concrete indices that are open, closed or both."
         },
         "local": {
-           "type": "boolean",
-           "description": "Return local information, do not retrieve the state from master node (default: false)"
+          "type": "boolean",
+          "description": "Return local information, do not retrieve the state from master node (default: false)"
         }
       }
     },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_template.json
index 18684bc940856..3269dcf68d9a2 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_template.json
@@ -1,18 +1,22 @@
 {
   "indices.exists_template": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-templates.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/indices-templates.html",
     "methods": ["HEAD"],
     "url": {
       "path": "/_template/{name}",
-      "paths": ["/_template/{name}"],
+      "paths": [ "/_template/{name}" ],
       "parts": {
         "name": {
-          "type": "string",
-          "required": true,
-          "description": "The name of the template"
+          "type": "list",
+          "required": false,
+          "description": "The comma separated names of the index templates"
         }
       },
       "params": {
+        "flat_settings": {
+          "type": "boolean",
+          "description": "Return settings in flat format (default: false)"
+        },
         "master_timeout": {
           "type": "time",
           "description": "Explicit operation timeout for connection to master node"
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_type.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_type.json
index d793199bc2776..fcd0c7b5d61e0 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_type.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_type.json
@@ -1,6 +1,6 @@
 {
   "indices.exists_type": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-types-exists.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/indices-types-exists.html",
     "methods": ["HEAD"],
     "url": {
       "path": "/{index}/_mapping/{type}",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.flush.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.flush.json
index d47619c73a163..44f8f9c303d1d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.flush.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.flush.json
@@ -1,6 +1,6 @@
 {
   "indices.flush": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-flush.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/indices-flush.html",
     "methods": ["POST", "GET"],
     "url": {
       "path": "/_flush",
@@ -18,7 +18,7 @@
         },
         "wait_if_ongoing": {
           "type" : "boolean",
-          "description" : "If set to true the flush operation will block until the flush can be executed if another flush operation is already executing. The default is false and will cause an exception to be thrown on the shard level if another flush operation is already running."
+          "description" : "If set to true the flush operation will block until the flush can be executed if another flush operation is already executing. The default is true. If set to false the flush will be skipped iff if another flush operation is already running."
         },
         "ignore_unavailable": {
             "type" : "boolean",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.flush_synced.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.flush_synced.json
index 08488eae8648b..984c48307ab3d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.flush_synced.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.flush_synced.json
@@ -1,6 +1,6 @@
 {
   "indices.flush_synced": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-synced-flush.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/indices-synced-flush.html",
     "methods": ["POST", "GET"],
     "url": {
       "path": "/_flush/synced",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.forcemerge.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.forcemerge.json
index c4170c1962aed..08e12a2ade440 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.forcemerge.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.forcemerge.json
@@ -1,6 +1,6 @@
 {
   "indices.forcemerge": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-forcemerge.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/indices-forcemerge.html",
     "methods": ["POST"],
     "url": {
       "path": "/_forcemerge",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get.json
index ed96200687c43..bd07c89581f59 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get.json
@@ -1,6 +1,6 @@
 {
   "indices.get":{
-    "documentation":"http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-get-index.html",
+    "documentation":"https://www.elastic.co/guide/en/elasticsearch/reference/5.x/indices-get-index.html",
     "methods":[ "GET" ],
     "url":{
       "path":"/{index}",
@@ -40,11 +40,6 @@
           "type": "boolean",
           "description": "Return settings in flat format (default: false)"
         },
-        "human": {
-          "type": "boolean",
-          "description": "Whether to return version and creation date values in human-readable format.",
-          "default": false
-        },
         "include_defaults": {
           "type": "boolean",
           "description": "Whether to return all default setting for each of the indices.",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_alias.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_alias.json
index b68d7c527fcdc..01fab07bab58a 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_alias.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_alias.json
@@ -1,6 +1,6 @@
 {
   "indices.get_alias": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-aliases.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/indices-aliases.html",
     "methods": ["GET"],
     "url": {
       "path": "/_alias/",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_field_mapping.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_field_mapping.json
index 3d5a629eff08e..ae4c58c7f01e7 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_field_mapping.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_field_mapping.json
@@ -1,6 +1,6 @@
 {
   "indices.get_field_mapping": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-get-field-mapping.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/indices-get-field-mapping.html",
     "methods": ["GET"],
     "url": {
       "path": "/_mapping/field/{fields}",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_mapping.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_mapping.json
index c3c0622844bb1..15b734c1c31e2 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_mapping.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_mapping.json
@@ -1,6 +1,6 @@
 {
   "indices.get_mapping": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-get-mapping.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/indices-get-mapping.html",
     "methods": ["GET"],
     "url": {
       "path": "/_mapping",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_settings.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_settings.json
index 17a1cd0311849..971c3e97bf9c6 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_settings.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_settings.json
@@ -1,6 +1,6 @@
 {
   "indices.get_settings": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-get-settings.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/indices-get-settings.html",
     "methods": ["GET"],
     "url": {
       "path": "/_settings",
@@ -38,11 +38,6 @@
           "type": "boolean",
           "description": "Return local information, do not retrieve the state from master node (default: false)"
         },
-        "human": {
-          "type": "boolean",
-          "description": "Whether to return version and creation date values in human-readable format.",
-          "default": false
-        },
         "include_defaults": {
           "type": "boolean",
           "description": "Whether to return all default setting for each of the indices.",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_template.json
index e3a97ee5c012a..aa5c26182a757 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_template.json
@@ -1,6 +1,6 @@
 {
   "indices.get_template": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-templates.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/indices-templates.html",
     "methods": ["GET"],
     "url": {
       "path": "/_template/{name}",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_upgrade.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_upgrade.json
index 63d9bbf418bab..db33ee9dbe72e 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_upgrade.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_upgrade.json
@@ -1,6 +1,6 @@
 {
   "indices.get_upgrade": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-upgrade.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/indices-upgrade.html",
     "methods": ["GET"],
     "url": {
       "path": "/_upgrade",
@@ -25,11 +25,6 @@
             "options" : ["open","closed","none","all"],
             "default" : "open",
             "description" : "Whether to expand wildcard expression to concrete indices that are open, closed or both."
-        },
-        "human": {
-            "type": "boolean",
-            "description": "Whether to return time and byte values in human-readable format.",
-            "default": false
         }
       }
     },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.open.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.open.json
index 879ce5a2b9d9a..df357867117c3 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.open.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.open.json
@@ -1,6 +1,6 @@
 {
   "indices.open": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-open-close.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/indices-open-close.html",
     "methods": ["POST"],
     "url": {
       "path": "/{index}/_open",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_alias.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_alias.json
index da53d5ac441e1..6f98ba2444b41 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_alias.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_alias.json
@@ -1,6 +1,6 @@
 {
   "indices.put_alias": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-aliases.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/indices-aliases.html",
     "methods": ["PUT", "POST"],
     "url": {
       "path": "/{index}/_alias/{name}",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_mapping.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_mapping.json
index 5fce0bcefc89a..1e98fd3f5c734 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_mapping.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_mapping.json
@@ -1,6 +1,6 @@
 {
   "indices.put_mapping": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-put-mapping.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/indices-put-mapping.html",
     "methods": ["PUT", "POST"],
     "url": {
       "path": "/{index}/{type}/_mapping",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_settings.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_settings.json
index 7c9cf627530ef..c1440fd9d449b 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_settings.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_settings.json
@@ -1,6 +1,6 @@
 {
   "indices.put_settings": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-update-settings.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/indices-update-settings.html",
     "methods": ["PUT"],
     "url": {
       "path": "/_settings",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_template.json
index 5bcb2f8a24346..134884fc715f3 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_template.json
@@ -1,6 +1,6 @@
 {
   "indices.put_template": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-templates.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/indices-templates.html",
     "methods": ["PUT", "POST"],
     "url": {
       "path": "/_template/{name}",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.recovery.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.recovery.json
index 2b94f1ad3d692..c51c3021a3a33 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.recovery.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.recovery.json
@@ -1,6 +1,6 @@
 {
     "indices.recovery" : {
-        "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-recovery.html",
+        "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/indices-recovery.html",
         "methods": ["GET"],
         "url": {
             "path": "/_recovery",
@@ -21,11 +21,6 @@
                     "type": "boolean",
                     "description": "Display only those recoveries that are currently on-going",
                     "default": false
-                },
-                "human": {
-                    "type": "boolean",
-                    "description": "Whether to return time and byte values in human-readable format.",
-                    "default": false
                 }
             }
         },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.refresh.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.refresh.json
index 703fa73f8d388..9efc272d40c20 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.refresh.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.refresh.json
@@ -1,6 +1,6 @@
 {
   "indices.refresh": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-refresh.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/indices-refresh.html",
     "methods": ["POST", "GET"],
     "url": {
       "path": "/_refresh",
@@ -25,14 +25,6 @@
             "options" : ["open","closed","none","all"],
             "default" : "open",
             "description" : "Whether to expand wildcard expression to concrete indices that are open, closed or both."
-        },
-        "force": {
-            "type" : "boolean",
-            "description" : "Force a refresh even if not required",
-            "default": false
-        },
-        "operation_threading": {
-          "description" : "TODO: ?"
         }
       }
     },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json
index 97580182ea1bd..313eefa11cc06 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json
@@ -1,6 +1,6 @@
 {
   "indices.rollover": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-rollover-index.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/indices-rollover-index.html",
     "methods": ["POST"],
     "url": {
       "path": "/{alias}/_rollover",
@@ -22,6 +22,10 @@
           "type" : "time",
           "description" : "Explicit operation timeout"
         },
+        "dry_run": {
+          "type" : "boolean",
+          "description" : "If set to true the rollover action will only be validated but not actually performed even if a condition matches. The default is false"
+        },
         "master_timeout": {
           "type" : "time",
           "description" : "Specify timeout for connection to master"
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.segments.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.segments.json
index cc51bdab2c3a3..01e0ffca795ed 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.segments.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.segments.json
@@ -1,6 +1,6 @@
 {
   "indices.segments": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-segments.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/indices-segments.html",
     "methods": ["GET"],
     "url": {
       "path": "/_segments",
@@ -26,11 +26,6 @@
             "default" : "open",
             "description" : "Whether to expand wildcard expression to concrete indices that are open, closed or both."
         },
-        "human": {
-            "type": "boolean",
-            "description": "Whether to return time and byte values in human-readable format.",
-            "default": false
-        },
         "operation_threading": {
           "description" : "TODO: ?"
         },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.shard_stores.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.shard_stores.json
index f3d644d5e344f..750011ef6c2e4 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.shard_stores.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.shard_stores.json
@@ -1,6 +1,6 @@
 {
   "indices.shard_stores": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-shards-stores.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/indices-shards-stores.html",
     "methods": ["GET"],
     "url": {
       "path": "/_shard_stores",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.shrink.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.shrink.json
index 5ef943eacba6c..6820d1439c1a2 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.shrink.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.shrink.json
@@ -1,6 +1,6 @@
 {
   "indices.shrink": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-shrink-index.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/indices-shrink-index.html",
     "methods": ["PUT", "POST"],
     "url": {
       "path": "/{index}/_shrink/{target}",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.stats.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.stats.json
index 7099f3e2fd2c3..ba66f2e2d9e3f 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.stats.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.stats.json
@@ -1,6 +1,6 @@
 {
   "indices.stats": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-stats.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/indices-stats.html",
     "methods": ["GET"],
     "url": {
       "path": "/_stats",
@@ -38,11 +38,6 @@
           "type" : "list",
           "description" : "A comma-separated list of search groups for `search` index metric"
         },
-        "human": {
-            "type": "boolean",
-            "description": "Whether to return time and byte values in human-readable format.",
-            "default": false
-        },
         "level": {
           "type" : "enum",
           "description": "Return stats aggregated at cluster, index or shard level",
@@ -52,6 +47,11 @@
         "types" : {
           "type" : "list",
           "description" : "A comma-separated list of document types for the `indexing` index metric"
+        },
+        "include_segment_file_sizes": {
+          "type": "boolean",
+          "description": "Whether to report the aggregated disk usage of each one of the Lucene index files (only applies if segment stats are requested)",
+          "default": false
         }
       }
     },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.update_aliases.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.update_aliases.json
index 30c369e410a25..8a16558cc6f6b 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.update_aliases.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.update_aliases.json
@@ -1,6 +1,6 @@
 {
   "indices.update_aliases": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-aliases.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/indices-aliases.html",
     "methods": ["POST"],
     "url": {
       "path": "/_aliases",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.upgrade.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.upgrade.json
index 0e5e4ffd244c6..6d821cb01070e 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.upgrade.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.upgrade.json
@@ -1,6 +1,6 @@
 {
   "indices.upgrade": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-upgrade.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/indices-upgrade.html",
     "methods": ["POST"],
     "url": {
       "path": "/_upgrade",
@@ -13,8 +13,8 @@
       },
       "params": {
         "allow_no_indices": {
-            "type" : "boolean",
-            "description" : "Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified)"
+          "type" : "boolean",
+          "description" : "Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified)"
         },
         "expand_wildcards": {
             "type" : "enum",
@@ -29,7 +29,7 @@
         "wait_for_completion": {
            "type" : "boolean",
            "description" : "Specify whether the request should block until the all segments are upgraded (default: false)"
-	},
+        },
         "only_ancient_segments": {
           "type" : "boolean",
           "description" : "If true, only ancient (an older Lucene major release) segments will be upgraded"
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.validate_query.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.validate_query.json
index 98af689833a06..85be317a34714 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.validate_query.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.validate_query.json
@@ -1,6 +1,6 @@
 {
   "indices.validate_query": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/search-validate.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/search-validate.html",
     "methods": ["GET", "POST"],
     "url": {
       "path": "/_validate/query",
@@ -63,13 +63,13 @@
           "type" : "boolean",
           "description" : "Specify whether format-based query failures (such as providing text to a numeric field) should be ignored"
         },
-        "lowercase_expanded_terms": {
-          "type" : "boolean",
-          "description" : "Specify whether query terms should be lowercased"
-        },
         "rewrite": {
           "type": "boolean",
           "description": "Provide a more detailed explanation showing the actual Lucene query that will be executed."
+        },
+        "all_shards": {
+          "type": "boolean",
+          "description": "Execute validation on all shards instead of one random shard per index"
         }
       }
     },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/info.json b/rest-api-spec/src/main/resources/rest-api-spec/api/info.json
index 63754eb7f7c3c..c4c18b05253e0 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/info.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/info.json
@@ -1,6 +1,6 @@
 {
   "info": {
-    "documentation": "http://www.elastic.co/guide/",
+    "documentation": "https://www.elastic.co/guide/",
     "methods": ["GET"],
     "url": {
       "path": "/",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.delete_pipeline.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.delete_pipeline.json
index 1c515e4509581..8688731de1c06 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.delete_pipeline.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.delete_pipeline.json
@@ -1,6 +1,6 @@
 {
   "ingest.delete_pipeline": {
-    "documentation": "https://www.elastic.co/guide/en/elasticsearch/plugins/master/ingest.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/plugins/5.x/ingest.html",
     "methods": [ "DELETE" ],
     "url": {
       "path": "/_ingest/pipeline/{id}",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.get_pipeline.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.get_pipeline.json
index 317250874233b..793251fea763b 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.get_pipeline.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.get_pipeline.json
@@ -1,6 +1,6 @@
 {
   "ingest.get_pipeline": {
-    "documentation": "https://www.elastic.co/guide/en/elasticsearch/plugins/master/ingest.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/plugins/5.x/ingest.html",
     "methods": [ "GET" ],
     "url": {
       "path": "/_ingest/pipeline/{id}",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.processor.grok.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.processor.grok.json
new file mode 100644
index 0000000000000..1bd0c3ceec460
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.processor.grok.json
@@ -0,0 +1,15 @@
+{
+  "ingest.processor.grok": {
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/plugins/master/ingest.html",
+    "methods": [ "GET" ],
+    "url": {
+      "path": "/_ingest/processor/grok",
+      "paths": ["/_ingest/processor/grok"],
+      "parts": {
+      },
+      "params": {
+      }
+    },
+    "body": null
+  }
+}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.put_pipeline.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.put_pipeline.json
index e4c3c2eb3f9a3..100ef4fc98fb9 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.put_pipeline.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.put_pipeline.json
@@ -1,6 +1,6 @@
 {
   "ingest.put_pipeline": {
-    "documentation": "https://www.elastic.co/guide/en/elasticsearch/plugins/master/ingest.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/plugins/5.x/ingest.html",
     "methods": [ "PUT" ],
     "url": {
       "path": "/_ingest/pipeline/{id}",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.simulate.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.simulate.json
index d02f97d81dd70..97efd8eabe62e 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.simulate.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.simulate.json
@@ -1,6 +1,6 @@
 {
   "ingest.simulate": {
-    "documentation": "https://www.elastic.co/guide/en/elasticsearch/plugins/master/ingest.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/plugins/5.x/ingest.html",
     "methods": [ "GET", "POST" ],
     "url": {
       "path": "/_ingest/pipeline/_simulate",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/mget.json b/rest-api-spec/src/main/resources/rest-api-spec/api/mget.json
index 1639f3619b39d..e87a67b000e4f 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/mget.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/mget.json
@@ -1,6 +1,6 @@
 {
   "mget": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/docs-multi-get.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/docs-multi-get.html",
     "methods": ["GET", "POST"],
     "url": {
       "path": "/_mget",
@@ -16,9 +16,9 @@
         }
       },
       "params": {
-        "fields": {
+        "stored_fields": {
           "type": "list",
-          "description" : "A comma-separated list of fields to return in the response"
+          "description" : "A comma-separated list of stored fields to return in the response"
         },
         "preference": {
           "type" : "string",
@@ -32,6 +32,10 @@
           "type" : "boolean",
           "description" : "Refresh the shard containing the document before performing the operation"
         },
+        "routing": {
+          "type" : "string",
+          "description" : "Specific routing value"
+        },
         "_source": {
           "type" : "list",
           "description" : "True or false to return the _source field or not, or a list of fields to return"
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/mpercolate.json b/rest-api-spec/src/main/resources/rest-api-spec/api/mpercolate.json
index 7cbf4f61e434d..2455c416e7638 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/mpercolate.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/mpercolate.json
@@ -1,6 +1,6 @@
 {
   "mpercolate": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/search-percolate.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/search-percolate.html",
     "methods": ["GET", "POST"],
     "url": {
       "path": "/_mpercolate",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/msearch.json b/rest-api-spec/src/main/resources/rest-api-spec/api/msearch.json
index 0344702ecfea9..ea015b104a42b 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/msearch.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/msearch.json
@@ -1,6 +1,6 @@
 {
   "msearch": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/search-multi-search.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/search-multi-search.html",
     "methods": ["GET", "POST"],
     "url": {
       "path": "/_msearch",
@@ -24,6 +24,15 @@
         "max_concurrent_searches" : {
           "type" : "number",
           "description" : "Controls the maximum number of concurrent searches the multi search api will execute"
+        },
+        "typed_keys": {
+          "type" : "boolean",
+          "description" : "Specify whether aggregation and suggester names should be prefixed by their respective types in the response"
+        },
+        "pre_filter_shard_size" : {
+          "type" : "number",
+          "description" : "A threshold that enforces a pre-filter roundtrip to prefilter search shards based on query rewriting if the number of shards the search request expands to exceeds the threshold. This filter roundtrip can limit the number of shards significantly if for instance a shard can not match any documents based on it's rewrite method ie. if date filters are mandatory to match but the shard bounds and the query are disjoint.",
+          "default" : 128
         }
       }
     },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/msearch_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/msearch_template.json
index 39aa53b2572eb..838151362329a 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/msearch_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/msearch_template.json
@@ -1,6 +1,6 @@
 {
   "msearch_template": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/search-template.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/search-template.html",
     "methods": ["GET", "POST"],
     "url": {
       "path": "/_msearch/template",
@@ -20,6 +20,14 @@
           "type" : "enum",
           "options" : ["query_then_fetch", "query_and_fetch", "dfs_query_then_fetch", "dfs_query_and_fetch"],
           "description" : "Search operation type"
+        },
+        "typed_keys": {
+          "type" : "boolean",
+          "description" : "Specify whether aggregation and suggester names should be prefixed by their respective types in the response"
+        },
+        "max_concurrent_searches" : {
+          "type" : "number",
+          "description" : "Controls the maximum number of concurrent searches the multi search api will execute"
         }
       }
     },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/mtermvectors.json b/rest-api-spec/src/main/resources/rest-api-spec/api/mtermvectors.json
index 58978b7d190fa..473f62364abfe 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/mtermvectors.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/mtermvectors.json
@@ -1,6 +1,6 @@
 {
   "mtermvectors" : {
-    "documentation" : "http://www.elastic.co/guide/en/elasticsearch/reference/master/docs-multi-termvectors.html",
+    "documentation" : "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/docs-multi-termvectors.html",
     "methods" : ["GET", "POST"],
     "url" : {
       "path" : "/_mtermvectors",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.hot_threads.json b/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.hot_threads.json
index 854cde1a9e731..75743cf661ad0 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.hot_threads.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.hot_threads.json
@@ -1,6 +1,6 @@
 {
   "nodes.hot_threads": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-nodes-hot-threads.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/cluster-nodes-hot-threads.html",
     "methods": ["GET"],
     "url": {
       "path": "/_nodes/hot_threads",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.info.json b/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.info.json
index 12f0d11c5fcb3..4ae11ec0e43dd 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.info.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.info.json
@@ -1,6 +1,6 @@
 {
   "nodes.info": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-nodes-info.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/cluster-nodes-info.html",
     "methods": ["GET"],
     "url": {
       "path": "/_nodes",
@@ -21,11 +21,6 @@
           "type": "boolean",
           "description": "Return settings in flat format (default: false)"
         },
-        "human": {
-            "type": "boolean",
-            "description": "Whether to return time and byte values in human-readable format.",
-            "default": false
-        },
         "timeout": {
           "type" : "time",
           "description" : "Explicit operation timeout"
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.stats.json b/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.stats.json
index fb9ef094f0b16..64b0715df2a8b 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.stats.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.stats.json
@@ -1,6 +1,6 @@
 {
   "nodes.stats": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-nodes-stats.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/cluster-nodes-stats.html",
     "methods": ["GET"],
     "url": {
       "path": "/_nodes/stats",
@@ -45,15 +45,10 @@
           "type" : "boolean",
           "description" : "A comma-separated list of search groups for `search` index metric"
         },
-        "human": {
-            "type": "boolean",
-            "description": "Whether to return time and byte values in human-readable format.",
-            "default": false
-        },
         "level": {
           "type" : "enum",
-          "description": "Return indices stats aggregated at node, index or shard level",
-          "options" : ["node", "indices", "shards"],
+          "description": "Return indices stats aggregated at index, node or shard level",
+          "options" : ["indices", "node", "shards"],
           "default" : "node"
         },
         "types" : {
@@ -63,6 +58,11 @@
         "timeout": {
           "type" : "time",
           "description" : "Explicit operation timeout"
+        },
+        "include_segment_file_sizes": {
+          "type": "boolean",
+          "description": "Whether to report the aggregated disk usage of each one of the Lucene index files (only applies if segment stats are requested)",
+          "default": false
         }
       }
     },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/percolate.json b/rest-api-spec/src/main/resources/rest-api-spec/api/percolate.json
index e58655dea5a27..a524542a3636f 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/percolate.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/percolate.json
@@ -1,6 +1,6 @@
 {
   "percolate": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/search-percolate.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/search-percolate.html",
     "methods": ["GET", "POST"],
     "url": {
       "path": "/{index}/{type}/_percolate",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ping.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ping.json
index 41310bb30d494..d6498014417d9 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/ping.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ping.json
@@ -1,6 +1,6 @@
 {
   "ping": {
-    "documentation": "http://www.elastic.co/guide/",
+    "documentation": "https://www.elastic.co/guide/",
     "methods": ["HEAD"],
     "url": {
       "path": "/",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/put_script.json b/rest-api-spec/src/main/resources/rest-api-spec/api/put_script.json
index c5676392625e7..8d3ee5e5dc3fc 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/put_script.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/put_script.json
@@ -1,10 +1,10 @@
 {
   "put_script": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/modules-scripting.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/modules-scripting.html",
     "methods": ["PUT", "POST"],
     "url": {
-      "path": "/_scripts/{lang}/{id}",
-      "paths": [ "/_scripts/{lang}/{id}" ],
+      "path": "/_scripts/{lang}",
+      "paths": [ "/_scripts/{lang}", "/_scripts/{lang}/{id}" ],
       "parts": {
         "id": {
           "type" : "string",
@@ -18,6 +18,14 @@
         }
       },
       "params" : {
+        "timeout": {
+          "type" : "time",
+          "description" : "Explicit operation timeout"
+        },
+        "master_timeout": {
+          "type" : "time",
+          "description" : "Specify timeout for connection to master"
+        }
       }
     },
     "body": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/put_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/put_template.json
index 294ed32cfff7b..b7530810f1de8 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/put_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/put_template.json
@@ -1,6 +1,6 @@
 {
   "put_template": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/search-template.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/search-template.html",
     "methods": ["PUT", "POST"],
     "url": {
       "path": "/_search/template/{id}",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/reindex.json b/rest-api-spec/src/main/resources/rest-api-spec/api/reindex.json
index 5fb4fe58db37f..37e07eda681ef 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/reindex.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/reindex.json
@@ -1,6 +1,6 @@
 {
   "reindex": {
-    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/master/docs-reindex.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/docs-reindex.html",
     "methods": ["POST"],
     "url": {
       "path": "/_reindex",
@@ -22,13 +22,18 @@
         },
         "wait_for_completion": {
           "type" : "boolean",
-          "default": false,
+          "default": true,
           "description" : "Should the request should block until the reindex is complete."
         },
         "requests_per_second": {
-          "type": "float",
+          "type": "number",
           "default": 0,
-          "description": "The throttle to set on this request in sub-requests per second. -1 means set no throttle as does \"unlimited\" which is the only non-float this accepts."
+          "description": "The throttle to set on this request in sub-requests per second. -1 means no throttle."
+        },
+        "slices": {
+          "type": "number",
+          "default": 1,
+          "description": "The number of slices this task should be divided into. Defaults to 1 meaning the task isn't sliced into subtasks."
         }
       }
     },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/reindex_rethrottle.json b/rest-api-spec/src/main/resources/rest-api-spec/api/reindex_rethrottle.json
index 5be7ea27407f4..ede15552a4dea 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/reindex_rethrottle.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/reindex_rethrottle.json
@@ -1,6 +1,6 @@
 {
   "reindex_rethrottle": {
-    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/master/docs-reindex.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/docs-reindex.html",
     "methods": ["POST"],
     "url": {
       "path": "/_reindex/{task_id}/_rethrottle",
@@ -13,7 +13,7 @@
       },
       "params": {
         "requests_per_second": {
-          "type": "float",
+          "type": "number",
           "required": true,
           "description": "The throttle to set on this request in floating sub-requests per second. -1 means set no throttle."
         }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/remote.info.json b/rest-api-spec/src/main/resources/rest-api-spec/api/remote.info.json
new file mode 100644
index 0000000000000..b3ea5e0966ce1
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/remote.info.json
@@ -0,0 +1,12 @@
+{
+  "remote.info": {
+    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/5.x/cluster-remote-info.html",
+    "methods": ["GET"],
+    "url": {
+      "path": "/_remote/info",
+      "paths": ["/_remote/info"],
+      "params": {}
+    },
+    "body": null
+  }
+}
\ No newline at end of file
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/render_search_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/render_search_template.json
index 8f27c12b819da..002453a723e62 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/render_search_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/render_search_template.json
@@ -1,6 +1,6 @@
 {
   "render_search_template": {
-    "documentation": "http://www.elasticsearch.org/guide/en/elasticsearch/reference/master/search-template.html",
+    "documentation": "http://www.elasticsearch.org/guide/en/elasticsearch/reference/5.x/search-template.html",
     "methods": ["GET", "POST"],
     "url": {
       "path": "/_render/template",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/scroll.json b/rest-api-spec/src/main/resources/rest-api-spec/api/scroll.json
index 885b746d09533..52b450a921425 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/scroll.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/scroll.json
@@ -1,6 +1,6 @@
 {
   "scroll": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/search-request-scroll.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/search-request-scroll.html",
     "methods": ["GET", "POST"],
     "url": {
       "path": "/_search/scroll",
@@ -13,7 +13,7 @@
       },
       "params": {
         "scroll": {
-          "type" : "duration",
+          "type" : "time",
           "description" : "Specify how long a consistent view of the index should be maintained for scrolled search"
         },
         "scroll_id": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json
index 21fda8dc805e1..195f1957641db 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json
@@ -1,6 +1,6 @@
 {
   "search": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/search-search.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/search-search.html",
     "methods": ["GET", "POST"],
     "url": {
       "path": "/_search",
@@ -72,10 +72,6 @@
           "type" : "boolean",
           "description" : "Specify whether format-based query failures (such as providing text to a numeric field) should be ignored"
         },
-        "lowercase_expanded_terms": {
-          "type" : "boolean",
-          "description" : "Specify whether query terms should be lowercased"
-        },
         "preference": {
           "type" : "string",
           "description" : "Specify the node or shard the operation should be performed on (default: random)"
@@ -89,7 +85,7 @@
           "description" : "A comma-separated list of specific routing values"
         },
         "scroll": {
-          "type" : "duration",
+          "type" : "time",
           "description" : "Specify how long a consistent view of the index should be maintained for scrolled search"
         },
         "search_type": {
@@ -140,7 +136,7 @@
           "description" : "How many suggestions to return in response"
         },
         "suggest_text": {
-          "type" : "text",
+          "type" : "string",
           "description" : "The source text for which the suggestions should be returned"
         },
         "timeout": {
@@ -151,6 +147,10 @@
           "type" : "boolean",
           "description": "Whether to calculate and return scores even if they are not used for sorting"
         },
+        "typed_keys": {
+          "type" : "boolean",
+          "description" : "Specify whether aggregation and suggester names should be prefixed by their respective types in the response"
+        },
         "version": {
           "type" : "boolean",
           "description" : "Specify whether to return document version as part of a hit"
@@ -158,6 +158,21 @@
         "request_cache": {
           "type" : "boolean",
           "description" : "Specify if request cache should be used for this request or not, defaults to index level setting"
+        },
+        "batched_reduce_size" : {
+          "type" : "number",
+          "description" : "The number of shard results that should be reduced at once on the coordinating node. This value should be used as a protection mechanism to reduce the memory overhead per search request if the potential number of shards in the request can be large.",
+          "default" : 512
+        },
+        "max_concurrent_shard_requests" : {
+          "type" : "number",
+          "description" : "The number of concurrent shard requests this search executes concurrently. This value should be used to limit the impact of the search on the cluster in order to limit the number of concurrent shard requests",
+          "default" : "The default grows with the number of nodes in the cluster but is at most 256."
+        },
+        "pre_filter_shard_size" : {
+          "type" : "number",
+          "description" : "A threshold that enforces a pre-filter roundtrip to prefilter search shards based on query rewriting if the number of shards the search request expands to exceeds the threshold. This filter roundtrip can limit the number of shards significantly if for instance a shard can not match any documents based on it's rewrite method ie. if date filters are mandatory to match but the shard bounds and the query are disjoint.",
+          "default" : 128
         }
       }
     },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search_shards.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search_shards.json
index e03ed8fef2fec..6195dd9a280de 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/search_shards.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search_shards.json
@@ -1,6 +1,6 @@
 {
   "search_shards": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/search-shards.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/search-shards.html",
     "methods": ["GET", "POST"],
     "url": {
       "path": "/{index}/{type}/_search_shards",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json
index 67cbcf44a9ea2..c6020659212be 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json
@@ -1,6 +1,6 @@
 {
   "search_template": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/search-template.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/search-template.html",
     "methods": ["GET", "POST"],
     "url": {
       "path": "/_search/template",
@@ -39,13 +39,25 @@
           "description" : "A comma-separated list of specific routing values"
         },
         "scroll": {
-          "type" : "duration",
+          "type" : "time",
           "description" : "Specify how long a consistent view of the index should be maintained for scrolled search"
         },
         "search_type": {
           "type" : "enum",
           "options" : ["query_then_fetch", "query_and_fetch", "dfs_query_then_fetch", "dfs_query_and_fetch"],
           "description" : "Search operation type"
+        },
+        "explain": {
+          "type" : "boolean",
+          "description" : "Specify whether to return detailed information about score computation as part of a hit"
+        },
+        "profile": {
+          "type" : "boolean",
+          "description" : "Specify whether to profile the query execution"
+        },
+        "typed_keys": {
+          "type" : "boolean",
+          "description" : "Specify whether aggregation and suggester names should be prefixed by their respective types in the response"
         }
       }
     },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.create.json b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.create.json
index 29ae2206c8568..2d2a20e997aa4 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.create.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.create.json
@@ -1,6 +1,6 @@
 {
   "snapshot.create": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/modules-snapshots.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/modules-snapshots.html",
     "methods": ["PUT", "POST"],
     "url": {
       "path": "/_snapshot/{repository}/{snapshot}",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.create_repository.json b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.create_repository.json
index 7a73f6abc3c28..22d92670b327d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.create_repository.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.create_repository.json
@@ -1,6 +1,6 @@
 {
   "snapshot.create_repository": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/modules-snapshots.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/modules-snapshots.html",
     "methods": ["PUT", "POST"],
     "url": {
       "path": "/_snapshot/{repository}",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.delete.json b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.delete.json
index 6668289f442a6..004fc7b61a43d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.delete.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.delete.json
@@ -1,6 +1,6 @@
 {
   "snapshot.delete": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/modules-snapshots.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/modules-snapshots.html",
     "methods": ["DELETE"],
     "url": {
       "path": "/_snapshot/{repository}/{snapshot}",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.delete_repository.json b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.delete_repository.json
index 4f0c43b8f0317..a6c9b9f31ed2f 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.delete_repository.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.delete_repository.json
@@ -1,6 +1,6 @@
 {
   "snapshot.delete_repository": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/modules-snapshots.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/modules-snapshots.html",
     "methods": ["DELETE"],
     "url": {
       "path": "/_snapshot/{repository}",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.get.json b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.get.json
index 760809cdf9baa..6549e4bbedcdf 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.get.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.get.json
@@ -1,6 +1,6 @@
 {
   "snapshot.get": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/modules-snapshots.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/modules-snapshots.html",
     "methods": ["GET"],
     "url": {
       "path": "/_snapshot/{repository}/{snapshot}",
@@ -25,6 +25,10 @@
         "ignore_unavailable": {
           "type": "boolean",
           "description": "Whether to ignore unavailable snapshots, defaults to false which means a SnapshotMissingException is thrown"
+        },
+        "verbose": {
+          "type": "boolean",
+          "description": "Whether to show verbose snapshot info or only show the basic info found in the repository index blob"
         }
       }
     },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.get_repository.json b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.get_repository.json
index bf6a660146173..5a0a951f09d3c 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.get_repository.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.get_repository.json
@@ -1,6 +1,6 @@
 {
   "snapshot.get_repository": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/modules-snapshots.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/modules-snapshots.html",
     "methods": ["GET"],
     "url": {
       "path": "/_snapshot",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.restore.json b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.restore.json
index bdd265799ce4b..ac734e5897c71 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.restore.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.restore.json
@@ -1,6 +1,6 @@
 {
   "snapshot.restore": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/modules-snapshots.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/modules-snapshots.html",
     "methods": ["POST"],
     "url": {
       "path": "/_snapshot/{repository}/{snapshot}/_restore",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.status.json b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.status.json
index cba488de793bc..dec666d8818d0 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.status.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.status.json
@@ -1,6 +1,6 @@
 {
   "snapshot.status": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/modules-snapshots.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/modules-snapshots.html",
     "methods": ["GET"],
     "url": {
       "path": "/_snapshot/_status",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.verify_repository.json b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.verify_repository.json
index 0d9a97c10877b..67021281db3d8 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.verify_repository.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.verify_repository.json
@@ -1,6 +1,6 @@
 {
     "snapshot.verify_repository": {
-        "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/modules-snapshots.html",
+        "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/modules-snapshots.html",
         "methods": ["POST"],
         "url": {
             "path": "/_snapshot/{repository}/_verify",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/suggest.json b/rest-api-spec/src/main/resources/rest-api-spec/api/suggest.json
index ca0ae8b4f3c8a..9345b93278727 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/suggest.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/suggest.json
@@ -1,7 +1,7 @@
 {
   "suggest": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/search-suggesters.html",
-    "methods": ["POST", "GET"],
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/search-suggesters.html",
+    "methods": ["POST"],
     "url": {
       "path": "/_suggest",
       "paths": ["/_suggest", "/{index}/_suggest"],
@@ -13,18 +13,18 @@
       },
       "params": {
         "ignore_unavailable": {
-            "type" : "boolean",
-            "description" : "Whether specified concrete indices should be ignored when unavailable (missing or closed)"
+          "type" : "boolean",
+          "description" : "Whether specified concrete indices should be ignored when unavailable (missing or closed)"
         },
         "allow_no_indices": {
-            "type" : "boolean",
-            "description" : "Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified)"
+          "type" : "boolean",
+          "description" : "Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified)"
         },
         "expand_wildcards": {
-            "type" : "enum",
-            "options" : ["open","closed","none","all"],
-            "default" : "open",
-            "description" : "Whether to expand wildcard expression to concrete indices that are open, closed or both."
+          "type" : "enum",
+          "options" : ["open","closed","none","all"],
+          "default" : "open",
+          "description" : "Whether to expand wildcard expression to concrete indices that are open, closed or both."
         },
         "preference": {
           "type" : "string",
@@ -41,4 +41,4 @@
       "required" : true
     }
   }
-}
+}
\ No newline at end of file
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.cancel.json b/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.cancel.json
index 69d21f4ec1def..1bb359d963aa5 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.cancel.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.cancel.json
@@ -1,6 +1,6 @@
 {
   "tasks.cancel": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/tasks.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/tasks.html",
     "methods": ["POST"],
     "url": {
       "path": "/_tasks",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.get.json b/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.get.json
index f97206cd16f72..4a0ff4c93ff2f 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.get.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.get.json
@@ -1,6 +1,6 @@
 {
   "tasks.get": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/tasks.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/tasks.html",
     "methods": ["GET"],
     "url": {
       "path": "/_tasks/{task_id}",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.list.json b/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.list.json
index a966cb0e50716..852d989826049 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.list.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.list.json
@@ -1,6 +1,6 @@
 {
   "tasks.list": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/tasks.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/tasks.html",
     "methods": ["GET"],
     "url": {
       "path": "/_tasks",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/termvectors.json b/rest-api-spec/src/main/resources/rest-api-spec/api/termvectors.json
index f5d8b6bd08dfd..c6ea269ffc250 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/termvectors.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/termvectors.json
@@ -1,6 +1,6 @@
 {
   "termvectors" : {
-    "documentation" : "http://www.elastic.co/guide/en/elasticsearch/reference/master/docs-termvectors.html",
+    "documentation" : "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/docs-termvectors.html",
     "methods" : ["GET", "POST"],
     "url" : {
       "path" : "/{index}/{type}/_termvectors",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/update.json b/rest-api-spec/src/main/resources/rest-api-spec/api/update.json
index a18d081f9b7f9..ae8e2be2731f9 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/update.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/update.json
@@ -1,6 +1,6 @@
 {
   "update": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/docs-update.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/docs-update.html",
     "methods": ["POST"],
     "url": {
       "path": "/{index}/{type}/{id}/_update",
@@ -31,9 +31,21 @@
           "type": "list",
           "description": "A comma-separated list of fields to return in the response"
         },
+        "_source": {
+          "type" : "list",
+          "description" : "True or false to return the _source field or not, or a list of fields to return"
+        },
+        "_source_exclude": {
+          "type" : "list",
+          "description" : "A list of fields to exclude from the returned _source field"
+        },
+        "_source_include": {
+          "type" : "list",
+          "description" : "A list of fields to extract and return from the _source field"
+        },
         "lang": {
           "type": "string",
-          "description": "The script language (default: groovy)"
+          "description": "The script language (default: painless)"
         },
         "parent": {
           "type": "string",
@@ -61,7 +73,7 @@
           "description": "Explicit timestamp for the document"
         },
         "ttl": {
-          "type": "duration",
+          "type": "time",
           "description": "Expiration time for the document"
         },
         "version": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/update_by_query.json b/rest-api-spec/src/main/resources/rest-api-spec/api/update_by_query.json
index 72149adc66338..d68efdab22ef8 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/update_by_query.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/update_by_query.json
@@ -1,6 +1,6 @@
 {
   "update_by_query": {
-    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/master/docs-update-by-query.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/docs-update-by-query.html",
     "methods": ["POST"],
     "url": {
       "path": "/{index}/_update_by_query",
@@ -36,18 +36,6 @@
           "type" : "string",
           "description" : "The field to use as default where no field prefix is given in the query string"
         },
-        "explain": {
-          "type" : "boolean",
-          "description" : "Specify whether to return detailed information about score computation as part of a hit"
-        },
-        "fields": {
-          "type" : "list",
-          "description" : "A comma-separated list of fields to return as part of a hit"
-        },
-        "fielddata_fields": {
-          "type" : "list",
-          "description" : "A comma-separated list of fields to return as the field data representation of a field for each hit"
-        },
         "from": {
           "type" : "number",
           "description" : "Starting offset (default: 0)"
@@ -65,7 +53,7 @@
             "type" : "enum",
             "options": ["abort", "proceed"],
             "default": "abort",
-            "description" : "What to do when the reindex hits version conflicts?"
+            "description" : "What to do when the update by query hits version conflicts?"
         },
         "expand_wildcards": {
             "type" : "enum",
@@ -77,10 +65,6 @@
           "type" : "boolean",
           "description" : "Specify whether format-based query failures (such as providing text to a numeric field) should be ignored"
         },
-        "lowercase_expanded_terms": {
-          "type" : "boolean",
-          "description" : "Specify whether query terms should be lowercased"
-        },
         "pipeline": {
           "type" : "string",
           "description" : "Ingest pipeline to set on index requests made by this action. (default: none)"
@@ -98,7 +82,7 @@
           "description" : "A comma-separated list of specific routing values"
         },
         "scroll": {
-          "type" : "duration",
+          "type" : "time",
           "description" : "Specify how long a consistent view of the index should be maintained for scrolled search"
         },
         "search_type": {
@@ -138,32 +122,6 @@
           "type" : "list",
           "description" : "Specific 'tag' of the request for logging and statistical purposes"
         },
-        "suggest_field": {
-          "type" : "string",
-          "description" : "Specify which field to use for suggestions"
-        },
-        "suggest_mode": {
-          "type" : "enum",
-          "options" : ["missing", "popular", "always"],
-          "default" : "missing",
-          "description" : "Specify suggest mode"
-        },
-        "suggest_size": {
-          "type" : "number",
-          "description" : "How many suggestions to return in response"
-        },
-        "suggest_text": {
-          "type" : "text",
-          "description" : "The source text for which the suggestions should be returned"
-        },
-        "timeout": {
-          "type" : "time",
-          "description" : "Explicit operation timeout"
-        },
-        "track_scores": {
-          "type" : "boolean",
-          "description": "Whether to calculate and return scores even if they are not used for sorting"
-        },
         "version": {
           "type" : "boolean",
           "description" : "Specify whether to return document version as part of a hit"
@@ -190,19 +148,24 @@
           "description" : "Sets the number of shard copies that must be active before proceeding with the update by query operation. Defaults to 1, meaning the primary shard only. Set to `all` for all shard copies, otherwise set to any non-negative value less than or equal to the total number of copies for the shard (number of replicas + 1)"
         },
         "scroll_size": {
-          "type": "integer",
+          "type": "number",
           "defaut_value": 100,
           "description": "Size on the scroll request powering the update_by_query"
         },
         "wait_for_completion": {
            "type" : "boolean",
-           "default": false,
-           "description" : "Should the request should block until the reindex is complete."
+           "default": true,
+           "description" : "Should the request should block until the update by query operation is complete."
         },
         "requests_per_second": {
-          "type": "float",
+          "type": "number",
           "default": 0,
-          "description": "The throttle to set on this request in sub-requests per second. -1 means set no throttle as does \"unlimited\" which is the only non-float this accepts."
+          "description": "The throttle to set on this request in sub-requests per second. -1 means no throttle."
+        },
+        "slices": {
+          "type": "number",
+          "default": 1,
+          "description": "The number of slices this task should be divided into. Defaults to 1 meaning the task isn't sliced into subtasks."
         }
       }
     },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc b/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc
index 16b607f4a22e1..c822b665aebfc 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc
@@ -10,7 +10,7 @@ Elasticsearch as follows:
 
 [source,sh]
 ---------------------
-bin/elasticsearch --script.inline true --node.testattr test --path.repo /tmp --repositories.url.allowed_urls 'http://snapshot.*'
+bin/elasticsearch -Enode.attr.testattr=test -Epath.repo=/tmp -Erepositories.url.allowed_urls='http://snapshot.*'
 ---------------------
 
 =======================================
@@ -163,12 +163,17 @@ be caught and tested.  For instance:
 The argument to `catch` can be any of:
 
 [horizontal]
-`missing`::     a 404 response from ES
-`conflict`::    a 409 response from ES
-`request`::     a generic error response from ES
-`param`::       a client-side error indicating an unknown parameter has been passed
-                to the method
-`/foo bar/`::   the text of the error message matches this regular expression
+`unauthorized`::    a 401 response from ES
+`forbidden`::       a 403 response from ES
+`missing`::         a 404 response from ES
+`request_timeout`:: a 408 response from ES
+`conflict`::        a 409 response from ES
+`request`::         a 4xx-5xx error response from ES, not equal to any named response
+                    above
+`unavailable`::     a 503 response from ES
+`param`::           a client-side error indicating an unknown parameter has been passed
+                    to the method
+`/foo bar/`::       the text of the error message matches this regular expression
 
 If `catch` is specified, then the `response` var must be cleared, and the test
 should fail if no error is thrown.
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/10_basic.yaml
index f8617116dc140..776bcea399c53 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/10_basic.yaml
@@ -23,3 +23,56 @@
 
   - match: {count: 2}
 
+---
+"Empty _id":
+  - skip:
+      version: " - 5.3.0"
+      reason: empty IDs were not rejected until 5.3.1
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - index:
+              _index: test
+              _type: type
+              _id: ''
+          - f: 1
+          - index:
+              _index: test
+              _type: type
+              _id: id
+          - f: 2
+          - index:
+              _index: test
+              _type: type
+          - f: 3
+  - match: { errors: true }
+  - match: { items.0.index.status: 400 }
+  - match: { items.0.index.error.type: illegal_argument_exception }
+  - match: { items.0.index.error.reason: if _id is specified it must not be empty }
+  - match: { items.1.index.created: true }
+  - match: { items.2.index.created: true }
+
+  - do:
+      count:
+        index: test
+
+  - match: { count: 2 }
+
+---
+"empty action":
+
+  - skip:
+      version: " - 5.99.99"
+      reason: confusing exception messaged caused by empty object fixed in 6.0.0
+      features: ["headers"]
+
+  - do:
+      catch: /Malformed action\/metadata line \[3\], expected FIELD_NAME but found \[END_OBJECT\]/
+      headers:
+        Content-Type: application/json
+      bulk:
+        body: |
+           {"index": {"_index": "test_index", "_type": "test_type", "_id": "test_id"}}
+           {"f1": "v1", "f2": 42}
+           {}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/20_list_of_strings.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/20_list_of_strings.yaml
index def91f4280722..e25626cf3ae28 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/20_list_of_strings.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/20_list_of_strings.yaml
@@ -11,6 +11,8 @@
 
   - do:
       count:
+        # we count through the primary in case there is a replica that has not yet fully recovered
+        preference: _primary
         index: test_index
 
   - match: {count: 2}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/40_fields.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/40_fields.yaml
deleted file mode 100644
index 3aa9d522633c0..0000000000000
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/40_fields.yaml
+++ /dev/null
@@ -1,49 +0,0 @@
----
-"Fields":
-  - do:
-      index:
-        refresh: true
-        index: test_index
-        type: test_type
-        id: test_id_1
-        body: { "foo": "bar" }
-
-  - do:
-      index:
-        refresh: true
-        index: test_index
-        type: test_type
-        id: test_id_2
-        body: { "foo": "qux" }
-
-  - do:
-      index:
-        refresh: true
-        index: test_index
-        type: test_type
-        id: test_id_3
-        body: { "foo": "corge" }
-
-
-  - do:
-      bulk:
-        refresh: true
-        body: |
-          { "update": { "_index": "test_index", "_type": "test_type", "_id": "test_id_1", "fields": ["_source"] } }
-          { "doc": { "foo": "baz" } }
-          { "update": { "_index": "test_index", "_type": "test_type", "_id": "test_id_2" } }
-          { "fields": ["_source"], "doc": { "foo": "quux" } }
-
-  - match: { items.0.update.get._source.foo: baz }
-  - match: { items.1.update.get._source.foo: quux }
-
-  - do:
-      bulk:
-        index: test_index
-        type: test_type
-        fields: _source
-        body: |
-          { "update": { "_id": "test_id_3" } }
-          { "doc": { "foo": "garply" } }
-
-  - match: { items.0.update.get._source.foo: garply }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/40_source.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/40_source.yaml
new file mode 100644
index 0000000000000..c852c376cc06f
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/40_source.yaml
@@ -0,0 +1,76 @@
+---
+"Source filtering":
+  - do:
+      index:
+        refresh: true
+        index: test_index
+        type: test_type
+        id: test_id_1
+        body: { "foo": "bar", "bar": "foo" }
+
+  - do:
+      index:
+        refresh: true
+        index: test_index
+        type: test_type
+        id: test_id_2
+        body: { "foo": "qux", "bar": "pux" }
+
+  - do:
+      index:
+        refresh: true
+        index: test_index
+        type: test_type
+        id: test_id_3
+        body: { "foo": "corge", "bar": "forge" }
+
+
+  - do:
+      bulk:
+        refresh: true
+        body: |
+          { "update": { "_index": "test_index", "_type": "test_type", "_id": "test_id_1", "_source": true } }
+          { "doc": { "foo": "baz" } }
+          { "update": { "_index": "test_index", "_type": "test_type", "_id": "test_id_2" } }
+          { "_source": true, "doc": { "foo": "quux" } }
+
+  - match: { items.0.update.get._source.foo: baz }
+  - match: { items.1.update.get._source.foo: quux }
+
+  - do:
+      bulk:
+        index: test_index
+        type: test_type
+        _source: true
+        body: |
+          { "update": { "_id": "test_id_3" } }
+          { "doc": { "foo": "garply" } }
+
+  - match: { items.0.update.get._source.foo: garply }
+
+  - do:
+      bulk:
+        refresh: true
+        body: |
+          { "update": { "_index": "test_index", "_type": "test_type", "_id": "test_id_1", "_source": {"includes": "bar"} } }
+          { "doc": { "foo": "baz" } }
+          { "update": { "_index": "test_index", "_type": "test_type", "_id": "test_id_2" } }
+          { "_source": {"includes": "foo"}, "doc": { "foo": "quux" } }
+
+  - match: { items.0.update.get._source.bar: foo }
+  - is_false: items.0.update.get._source.foo
+  - match: { items.1.update.get._source.foo: quux }
+  - is_false: items.1.update.get._source.bar
+
+  - do:
+      bulk:
+        index: test_index
+        type: test_type
+        _source_include: foo
+        body: |
+          { "update": { "_id": "test_id_3" } }
+          { "doc": { "foo": "garply" } }
+
+  - match: { items.0.update.get._source.foo: garply }
+  - is_false: items.0.update.get._source.bar
+
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/10_basic.yaml
index 770c5c8b441b0..12879fa412a1d 100755
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/10_basic.yaml
@@ -126,6 +126,52 @@
   - match:
       $body: / (^|\n)test_2 .+ \n/
 
+---
+"Multiple alias names":
+
+  - skip:
+      version: " - 5.99.99"
+      reason:  multiple aliases are supported only from 6.0.0 on
+
+  - do:
+        indices.create:
+            index: test
+
+  - do:
+        indices.create:
+            index: test2
+  - do:
+        indices.create:
+            index: test3
+
+  - do:
+        indices.put_alias:
+            index: test
+            name: foo
+
+  - do:
+        indices.put_alias:
+            index: test2
+            name: bar
+  - do:
+        indices.put_alias:
+            index: test3
+            name: baz
+
+  - do:
+        cat.aliases:
+            name: foo,bar
+            v: true
+            h: [alias, index]
+            s: [index]
+
+  - match:
+      $body: |
+               /^  alias \s+ index \n
+                   foo   \s+ test  \n
+                   bar   \s+ test2
+               $/
+
 ---
 "Column headers":
 
@@ -216,3 +262,50 @@
                     -                   \s+
                 $/
 
+---
+"Alias sorting":
+
+  - skip:
+      version: " - 5.0.99"
+      reason:  sorting was introduced in 5.1.0
+
+  - do:
+      indices.create:
+        index: test_index
+        body:
+          aliases:
+            test_alias: {}
+            my_alias: {}
+
+  - do:
+      indices.create:
+        index: other_index
+        body:
+          aliases:
+            other_alias: {}
+
+  - do:
+      cat.aliases:
+        h:      [alias, index]
+        s:      [index, alias]
+
+  - match:
+      $body: |
+            /^
+                other_alias \s+ other_index\n
+                my_alias \s+ test_index\n
+                test_alias \s+ test_index\n
+            $/
+
+  - do:
+      cat.aliases:
+        h:      [alias, index]
+        s:      [index, "a:desc"]
+
+  - match:
+      $body: |
+            /^
+                other_alias \s+ other_index\n
+                test_alias \s+ test_index\n
+                my_alias \s+ test_index\n
+            $/
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.allocation/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.allocation/10_basic.yaml
index 5f0646139b528..bdb328fad79ca 100755
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.allocation/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.allocation/10_basic.yaml
@@ -54,7 +54,7 @@
             /^
               ( \s*                          #allow leading spaces to account for right-justified text
                 \d+                    \s+
-                \d+(\.\d+)?b           \s+
+                \d+(\.\d+)?[kmgt]?b    \s+
                 \d+(\.\d+)?[kmgt]?b    \s+
                 (\d+(\.\d+)?[kmgt]b   \s+)  #always should return value since we filter out non data nodes by default
                 (\d+(\.\d+)?[kmgt]b   \s+)  #always should return value since we filter out non data nodes by default
@@ -83,7 +83,7 @@
       $body: |
             /^
               ( 0                      \s+
-                0b                     \s+
+                \d+(\.\d+)?[kmgt]?b    \s+
                 \d+(\.\d+)?[kmgt]?b    \s+
                 (\d+(\.\d+)?[kmgt]b   \s+)?  #no value from client nodes
                 (\d+(\.\d+)?[kmgt]b   \s+)?  #no value from client nodes
@@ -117,7 +117,7 @@
              /^
                ( \s*                          #allow leading spaces to account for right-justified text
                  \d+                    \s+
-                 \d+(\.\d+)?b           \s+
+                 \d+(\.\d+)?[kmgt]?b    \s+
                  \d+(\.\d+)?[kmgt]?b    \s+
                  (\d+(\.\d+)?[kmgt]b   \s+)?  #no value from client nodes
                  (\d+(\.\d+)?[kmgt]b   \s+)?  #no value from client nodes
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.health/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.health/10_basic.yaml
index 9225ad69ea4dc..380caec931899 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.health/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.health/10_basic.yaml
@@ -57,7 +57,7 @@
 
   - do:
       cat.health:
-        ts: 0
+        ts: false
 
   - match:
       $body: |
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.indices/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.indices/10_basic.yaml
index b07b5dadd9870..3e9001322733b 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.indices/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.indices/10_basic.yaml
@@ -70,6 +70,47 @@
                 )
                 $/
 
+---
+"Test cat indices using health status":
+
+  - do:
+      cluster.health: {}
+
+  - set: { number_of_data_nodes: count }
+
+  - do:
+      indices.create:
+        index: foo
+        body:
+          settings:
+            number_of_shards: "1"
+            number_of_replicas: "0"
+  - do:
+      indices.create:
+        index: bar
+        body:
+          settings:
+            number_of_shards: "1"
+            number_of_replicas: $count
+
+  - do:
+      cat.indices:
+        health: green
+        h: index
+
+  - match:
+      $body: |
+                /^(foo)$/
+
+  - do:
+      cat.indices:
+        health: yellow
+        h: index
+
+  - match:
+      $body: |
+               /^(bar)$/
+
 ---
 "Test cat indices using wildcards":
 
@@ -116,3 +157,61 @@
   - match:
       $body: |
                /^(ba(r|z) \n?){2}$/
+
+---
+"Test cat indices sort":
+  - skip:
+      version: " - 5.0.99"
+      reason:  sorting was introduced in 5.1.0
+
+  - do:
+      indices.create:
+        index: foo
+        body:
+          settings:
+            number_of_shards: "1"
+            number_of_replicas: "0"
+
+  - do:
+      indices.create:
+        index: bar
+        body:
+          settings:
+            number_of_shards: "1"
+            number_of_replicas: "0"
+
+  - do:
+      indices.create:
+        index: baz
+        body:
+          settings:
+            number_of_shards: "1"
+            number_of_replicas: "0"
+
+  - do:
+      indices.close:
+        index: bar
+
+  - do:
+      cat.indices:
+        h: [status, index]
+        s: [status, index]
+
+  - match:
+      $body: |
+            /^ close \s+ bar\n
+               open \s+ baz\n
+               open \s+ foo\n
+            $/
+
+  - do:
+      cat.indices:
+        h: [status, index]
+        s: [status, "index:desc"]
+
+  - match:
+      $body: |
+            /^ close \s+ bar\n
+               open \s+ foo\n
+               open \s+ baz\n
+            $/
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.nodes/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.nodes/10_basic.yaml
index 391a7c1e6d18c..e1225ef5da681 100755
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.nodes/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.nodes/10_basic.yaml
@@ -57,3 +57,60 @@
   - match:
       $body: |
                /^      http \n ((\d{1,3}\.){3}\d{1,3}:\d{1,5}\n)+  $/
+
+---
+"Additional disk information":
+  - skip:
+      version: " - 5.5.99"
+      reason:  additional disk info added in 5.6.0
+
+  - do:
+      cat.nodes:
+          h: diskAvail,diskTotal,diskUsed,diskUsedPercent
+          v: true
+
+  - match:
+      # leading whitespace on columns and optional whitespace on values is necessary
+      # because `diskAvail` is right aligned and text representation of disk size might be
+      # longer so it's padded with leading whitespace
+      $body: |
+               /^  \s* diskAvail            \s+ diskTotal            \s+ diskUsed            \s+ diskUsedPercent            \n
+                  (\s* \d+(\.\d+)?[ptgmk]?b \s+ \d+(\.\d+)?[ptgmk]?b \s+ \d+(\.\d+)?[ptgmk]?b\s+ (100\.00 | \d{1,2}\.\d{2}) \n)+  $/
+
+  - do:
+      cat.nodes:
+          h: disk,dt,du,dup
+          v: true
+
+  - match:
+      # leading whitespace on columns and optional whitespace on values is necessary
+      # because `disk` is right aligned and text representation of disk size might be
+      # longer so it's padded with leading whitespace
+      $body: |
+               /^  \s* disk                 \s+ dt                   \s+ du                  \s+ dup                        \n
+                  (\s* \d+(\.\d+)?[ptgmk]?b \s+ \d+(\.\d+)?[ptgmk]?b \s+ \d+(\.\d+)?[ptgmk]?b\s+ (100\.00 | \d{1,2}\.\d{2}) \n)+  $/
+
+---
+"Test cat nodes output with full_id set":
+  - skip:
+      version: " - 5.0.0"
+      reason:  The full_id setting was rejected in 5.0.0 see #21266
+
+
+  - do:
+      cat.nodes:
+          h: id
+  # check for a 4 char non-whitespace character string
+  - match:
+      $body: |
+               /^(\S{4}\n)+$/
+
+  - do:
+      cat.nodes:
+          h: id
+          full_id: true
+  # check for a 5+ char non-whitespace character string
+  - match:
+      $body: |
+               /^(\S{5,}\n)+$/
+
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.repositories/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.repositories/10_basic.yaml
index c7eb9c1f9307f..6d83274726e4e 100755
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.repositories/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.repositories/10_basic.yaml
@@ -43,3 +43,44 @@
                /^   test_cat_repo_1\s+ fs\s*\n
                     test_cat_repo_2\s+ fs\s*\n
                $/
+
+---
+"Test cat repositories sort":
+  - skip:
+      version: " - 5.0.99"
+      reason:  sorting was introduced in 5.1.0
+  - do:
+      snapshot.create_repository:
+        repository: test_cat_repo_1
+        body:
+          type: fs
+          settings:
+            location: "test_cat_repo_1_loc"
+
+  - do:
+      snapshot.create_repository:
+        repository: test_cat_repo_2
+        body:
+          type: fs
+          settings:
+            location: "test_cat_repo_2_loc"
+
+  - do:
+      cat.repositories:
+        s: [type, id]
+
+  - match:
+      $body: |
+             /^ test_cat_repo_1 \s+ fs \n
+                test_cat_repo_2 \s+ fs \n
+             $/
+
+  - do:
+      cat.repositories:
+        s: [type, "id:desc"]
+
+  - match:
+      $body: |
+             /^ test_cat_repo_2 \s+ fs \n
+                test_cat_repo_1 \s+ fs \n
+             $/
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yaml
index a95314c57d5ca..9991d4a8c751e 100755
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yaml
@@ -1,5 +1,8 @@
 ---
 "Help":
+  - skip:
+      version: " - 5.1.99"
+      reason:  refresh listeners is new in 5.2.0
   - do:
       cat.shards:
         help: true
@@ -51,6 +54,7 @@
                     merges.total_time                .+   \n
                     refresh.total                    .+   \n
                     refresh.time                     .+   \n
+                    refresh.listeners                .+   \n
                     search.fetch_current             .+   \n
                     search.fetch_time                .+   \n
                     search.fetch_total               .+   \n
@@ -161,6 +165,13 @@
       $body: |
                /^(index2 \s+ \d \s+ (p|r) \s+ ((STARTED|INITIALIZING|RELOCATING) \s+ (\d \s+ (\d+|\d+[.]\d+)(kb|b) \s+)? \d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} \s+ .+|UNASSIGNED \s+) \n?){5}$/
 
+---
+"Test cat shards with shadow replicas":
+  - skip:
+      version: " - 5.2.99"
+      reason: deprecation was added in 5.2.0 and its format changed in 5.3.0
+      features: "warnings"
+
   - do:
       indices.create:
         index: index3
@@ -170,6 +181,9 @@
             number_of_replicas: "1"
             shadow_replicas: true
             shared_filesystem: false
+      warnings:
+        - "[index.shadow_replicas] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version."
+        - "[index.shared_filesystem] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version."
 
   - do:
       cat.shards:
@@ -224,3 +238,44 @@
   - match:
       $body: |
                /^(ba(r|z) \n?){2}$/
+
+---
+"Test cat shards sort":
+  - skip:
+      version: " - 5.0.99"
+      reason:  sorting was introduced in 5.1.0
+
+  - do:
+      indices.create:
+        index: foo
+        body:
+          settings:
+            number_of_shards: "1"
+            number_of_replicas: "0"
+
+  - do:
+      indices.create:
+        index: bar
+        body:
+          settings:
+            number_of_shards: "1"
+            number_of_replicas: "0"
+
+  - do:
+      index:
+        index: bar
+        type: type
+        body: { test: bar }
+        refresh: true
+
+  - do:
+      cat.shards:
+        h: [index, docs]
+        s: [docs]
+
+# don't use the store here it's cached and might be stale
+  - match:
+      $body: |
+            /^ foo \s+ 0\n
+               bar \s+ 1\n
+            $/
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.snapshots/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.snapshots/10_basic.yaml
index c1a1fc97011c0..6e03ceb98c716 100755
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.snapshots/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.snapshots/10_basic.yaml
@@ -2,7 +2,6 @@
 "Help":
   - do:
       cat.snapshots:
-        repository: test_cat_snapshots_1
         help: true
 
   - match:
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.templates/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.templates/10_basic.yaml
new file mode 100644
index 0000000000000..09ef7f18ecd4d
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.templates/10_basic.yaml
@@ -0,0 +1,239 @@
+---
+"Help":
+    - skip:
+        version: " - 5.0.99"
+        reason:  templates were introduced in 5.1.0
+    - do:
+        cat.templates:
+          help: true
+
+    - match:
+        $body: |
+                   /^  name             .+   \n
+                       template         .+   \n
+                       order            .+   \n
+                       version          .+   \n
+                   $/
+
+---
+"No templates":
+    - skip:
+        version: " - 5.0.99"
+        reason:  templates were introduced in 5.1.0
+    - do:
+        cat.templates: {}
+
+    - match:
+        $body: |
+               /^
+               $/
+
+---
+"Normal templates":
+    - skip:
+        version: " - 5.0.99"
+        reason:  templates were introduced in 5.1.0
+    - do:
+        indices.put_template:
+          name: test
+          body:
+            order: 0
+            version: 1
+            template: test-*
+            settings:
+              number_of_shards: 1
+              number_of_replicas: 0
+
+    - do:
+        indices.put_template:
+          name: test_2
+          body:
+            order: 1
+            version: 2
+            template: test-2*
+            settings:
+              number_of_shards: 1
+              number_of_replicas: 0
+
+    - do:
+        cat.templates: {}
+
+    - match:
+        $body: >
+               /
+                 (^|\n)test     \s+
+                 test-\*        \s+
+                 0              \s+
+                 1
+                 (\n|$)
+               /
+
+    - match:
+        $body: >
+               /
+                (^|\n)test_2    \s+
+                test-2\*        \s+
+                1               \s+
+                2
+                (\n|$)
+               /
+
+---
+"Filtered templates":
+    - skip:
+        version: " - 5.0.99"
+        reason:  templates were introduced in 5.1.0
+    - do:
+        indices.put_template:
+          name: test
+          body:
+            order: 0
+            version: 1
+            template: t*
+            settings:
+              number_of_shards: 1
+              number_of_replicas: 0
+
+    - do:
+        indices.put_template:
+          name: nomatch
+          body:
+            order: 2
+            version: 1
+            template: tea*
+            settings:
+              number_of_shards: 1
+              number_of_replicas: 0
+
+    - do:
+        cat.templates:
+            name: test*
+
+    - match:
+        $body: |
+                /^
+                    test    \s+
+                    t\*     \s+
+                    0       \s+
+                    1
+                    \n
+                $/
+
+---
+"Column headers":
+    - skip:
+        version: " - 5.0.99"
+        reason:  templates were introduced in 5.1.0
+    - do:
+        indices.put_template:
+          name: test
+          body:
+            order: 0
+            version: 1
+            template: t*
+            settings:
+              number_of_shards: 1
+              number_of_replicas: 0
+
+    - do:
+        cat.templates:
+            v: true
+            name: test
+
+    - match:
+        $body: |
+                 /^
+                    name        \s+
+                    template    \s+
+                    order       \s+
+                    version
+                    \n
+                    test        \s+
+                    t\*         \s+
+                    0           \s+
+                    1
+                    \n
+                 $/
+
+---
+"Select columns":
+    - skip:
+        version: " - 5.0.99"
+        reason:  templates were introduced in 5.1.0
+    - do:
+        indices.put_template:
+          name: test
+          body:
+            order: 0
+            version: 1
+            template: t*
+            settings:
+              number_of_shards: 1
+              number_of_replicas: 0
+
+    - do:
+        cat.templates:
+            h:  [name, template]
+            v:  true
+            name: test*
+
+    - match:
+        $body: |
+                /^
+                    name        \s+
+                    template
+                    \n
+                    test        \s+
+                    t\*
+                    \n
+                $/
+
+---
+"Sort templates":
+    - skip:
+        version: " - 5.0.99"
+        reason:  templates were introduced in 5.1.0
+    - do:
+        indices.put_template:
+          name: test
+          body:
+            order: 0
+            template: t*
+            settings:
+              number_of_shards: 1
+              number_of_replicas: 0
+
+    - do:
+        indices.put_template:
+          name: test_1
+          body:
+            order: 0
+            version: 1
+            template: te*
+            settings:
+              number_of_shards: 1
+              number_of_replicas: 0
+
+    - do:
+        cat.templates:
+            h: [name, template, version]
+            s: [version]
+
+    - match:
+        $body: |
+              /^
+                  test \s+ t\* \s+\n
+                  test_1 \s+ te\* \s+ 1\n
+              $/
+
+    - do:
+        cat.templates:
+            h: [name, template, version]
+            s: ["version:desc"]
+
+    - match:
+        $body: |
+              /^
+                  test_1 \s+ te\* \s+ 1\n
+                  test \s+ t\* \s+\n
+              $/
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.allocation_explain/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.allocation_explain/10_basic.yaml
index f0195788e0783..63724be133176 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.allocation_explain/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.allocation_explain/10_basic.yaml
@@ -1,45 +1,49 @@
----
-"cluster shard allocation explanation test":
+"bad cluster shard allocation explanation request":
+  - skip:
+      version: " - 5.5.99"
+      reason: response status on bad request was changed starting in 5.6.0
+
   - do:
       # there aren't any unassigned shards to explain
-      catch: /unable to find any shards to explain/
+      catch: /illegal_argument_exception/
       cluster.allocation_explain: {}
 
+---
+"cluster shard allocation explanation test":
+  - skip:
+      version: " - 5.1.99"
+      reason: explain API response output was changed starting in 5.2.0
+
   - do:
       indices.create:
         index: test
 
+  - match: { acknowledged: true }
+
   - do:
       cluster.state:
         metric: [ master_node ]
 
-  - set: {master_node: node_id}
-
-    # This relies on there only being a single node in the test cluster, which
-    # is currently true, but if this changes in the future this test will need
-    # to be changed
   - do:
       cluster.allocation_explain:
         body: { "index": "test", "shard": 0, "primary": true }
 
-  - match: { assigned: true }
-  # - match: { assigned_node_id: $node_id }
-  - is_true: assigned_node_id
-  - match: { shard.index: "test" }
-  - match: { shard.id: 0 }
-  - match: { shard.primary: true }
-  # unfortunately can't test these because they break with multi-node backwords
-  # compat REST tests
-  # - is_true: nodes.$node_id.node_name
-  # - match: { nodes.$node_id.node_attributes.testattr: "test" }
-  # - match: { nodes.$node_id.node_attributes.portsfile: "true" }
-  # - match: { nodes.$node_id.final_decision: "CURRENTLY_ASSIGNED" }
-  # - match: { nodes.$node_id.weight: 0.0 }
-  # - match: { nodes.$node_id.decisions.0.decider: "same_shard" }
-  # - match: { nodes.$node_id.decisions.0.decision: "NO" }
+  - match: { current_state: "started" }
+  - is_true: current_node.id
+  - match: { index: "test" }
+  - match: { shard: 0 }
+  - match: { primary: true }
+  - is_true: can_remain_on_current_node
+  - is_true: can_rebalance_cluster
+  - is_true: can_rebalance_to_other_node
+  - is_true: rebalance_explanation
 
 ---
 "cluster shard allocation explanation test with empty request":
+  - skip:
+      version: " - 5.1.99"
+      reason: explain API response output was changed starting in 5.2.0
+
   - do:
       indices.create:
         index: test
@@ -49,22 +53,15 @@
       cluster.state:
         metric: [ master_node ]
 
-  - set: {master_node: node_id}
-
   - do:
       cluster.allocation_explain:
         include_disk_info: true
 
-  - match: { assigned: false }
+  - match: { current_state: "unassigned" }
   - match: { unassigned_info.reason: "INDEX_CREATED" }
   - is_true: unassigned_info.at
-  - match: { shard.index: "test" }
-  - match: { shard.id: 0 }
-  - match: { shard.primary: false }
+  - match: { index: "test" }
+  - match: { shard: 0 }
+  - match: { primary: false }
   - is_true: cluster_info
-  # - is_true: nodes.$node_id.node_name
-  # - match: { nodes.$node_id.node_attributes.testattr: "test" }
-  # - match: { nodes.$node_id.node_attributes.portsfile: "true" }
-  # - match: { nodes.$node_id.final_decision: "NO" }
-  # - match: { nodes.$node_id.decisions.0.decider: "same_shard" }
-  # - match: { nodes.$node_id.decisions.0.decision: "NO" }
+  - is_true: can_allocate
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yaml
index 41552f217beec..2b98600de42e0 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yaml
@@ -61,3 +61,16 @@
 
   - match: {persistent: {}}
 
+---
+"Test get a default settings":
+
+# this can't be bumped to 5.0.2 until snapshots are published
+ - skip:
+        version: " - 5.99.99"
+        reason:  Fetching default group setting was buggy until 5.0.3
+
+ - do:
+      cluster.get_settings:
+        include_defaults: true
+
+ - match: {defaults.node.attr.testattr: "test"}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.stats/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.stats/10_basic.yaml
index ccbfc199cec84..29f048068b4a0 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.stats/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.stats/10_basic.yaml
@@ -22,8 +22,8 @@
   - is_true: nodes.os.mem.total_in_bytes
   - is_true: nodes.os.mem.free_in_bytes
   - is_true: nodes.os.mem.used_in_bytes
-  - is_true: nodes.os.mem.free_percent
-  - is_true: nodes.os.mem.used_percent
+  - gte: { nodes.os.mem.free_percent: 0 }
+  - gte: { nodes.os.mem.used_percent: 0 }
   - is_true: nodes.process
   - is_true: nodes.jvm
   - is_true: nodes.fs
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/count/20_query_string.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/count/20_query_string.yaml
index 933033761e9c1..70f402691a379 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/count/20_query_string.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/count/20_query_string.yaml
@@ -58,15 +58,6 @@
       count:
         index: test
         q: field:BA*
-        lowercase_expanded_terms: false
-
-  - match: {count : 0}
-
-  - do:
-      count:
-        index: test
-        q: field:BA*
-        analyze_wildcard: true
 
   - match: {count : 1}
 
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/create/15_without_id.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/create/15_without_id.yaml
index 1126a3d085c0d..ab9932819381f 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/create/15_without_id.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/create/15_without_id.yaml
@@ -1,7 +1,7 @@
 ---
 "Create without ID":
  - do:
-      catch:            /Validation|Invalid/
+      catch: param
       create:
           index:  test_1
           type:   test
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/create/30_internal_version.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/create/30_internal_version.yaml
index fcdb2332a4644..e220d98816161 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/create/30_internal_version.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/create/30_internal_version.yaml
@@ -18,3 +18,22 @@
           id:     1
           body:   { foo: bar }
 
+---
+"Internal versioning with explicit version":
+
+ - skip:
+      version: " - 5.1.1"
+      reason: validation logic only fixed from 5.1.2 onwards
+
+ - do:
+      catch: request
+      create:
+          index:          test
+          type:           test
+          id:             3
+          body:           { foo: bar }
+          version:        5
+
+ - match: { status: 400 }
+ - match: { error.type: action_request_validation_exception }
+ - match: { error.reason: "Validation Failed: 1: create operations do not support explicit versions. use index instead;" }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/create/35_external_version.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/create/35_external_version.yaml
new file mode 100644
index 0000000000000..e29690fe8d03b
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/create/35_external_version.yaml
@@ -0,0 +1,34 @@
+---
+"External version":
+
+ - skip:
+      version: " - 5.1.1"
+      reason: validation logic only fixed from 5.1.2 onwards
+
+ - do:
+      catch:              request
+      create:
+          index:          test
+          type:           test
+          id:             1
+          body:           { foo: bar }
+          version_type:   external
+          version:        0
+
+ - match: { status: 400 }
+ - match: { error.type: action_request_validation_exception }
+ - match: { error.reason: "Validation Failed: 1: create operations only support internal versioning. use index instead;" }
+
+ - do:
+      catch:              request
+      create:
+          index:          test
+          type:           test
+          id:             2
+          body:           { foo: bar }
+          version_type:   external
+          version:        5
+
+ - match: { status: 400 }
+ - match: { error.type: action_request_validation_exception }
+ - match: { error.reason: "Validation Failed: 1: create operations only support internal versioning. use index instead;" }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/create/40_routing.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/create/40_routing.yaml
index 02f011d75b04a..44447ebee8ce7 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/create/40_routing.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/create/40_routing.yaml
@@ -28,7 +28,7 @@
           type:    test
           id:      1
           routing: 5
-          fields:  [_routing]
+          stored_fields:  [_routing]
 
  - match:   { _id:              "1"}
  - match:   { _routing:  "5"}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/create/50_parent.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/create/50_parent.yaml
index cac5387370d89..cb0dfcfe78cf7 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/create/50_parent.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/create/50_parent.yaml
@@ -31,7 +31,7 @@
           type:    test
           id:      1
           parent:  5
-          fields:  [_parent, _routing]
+          stored_fields:  [_parent, _routing]
 
  - match:   { _id:      "1"}
  - match:   { _parent: "5"}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/create/55_parent_with_routing.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/create/55_parent_with_routing.yaml
index 55b840526e58e..5f352ac90c3c7 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/create/55_parent_with_routing.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/create/55_parent_with_routing.yaml
@@ -32,7 +32,7 @@
           id:      1
           parent:  5
           routing: 4
-          fields:  [_parent, _routing]
+          stored_fields:  [_parent, _routing]
 
  - match:   { _id:      "1"}
  - match:   { _parent: "5"}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/delete/27_force_version.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/delete/27_force_version.yaml
new file mode 100644
index 0000000000000..cc259fed8b479
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/delete/27_force_version.yaml
@@ -0,0 +1,57 @@
+---
+"Force Version":
+
+ - skip:
+    features: "warnings"
+    version: " - 5.2.99"
+    reason: warning headers were introduced in 5.0.1 and their format changed in 5.3.0
+
+ - do:
+      warnings:
+          - version type FORCE is deprecated and will be removed in the next major version
+      index:
+          index:          test_1
+          type:           test
+          id:             1
+          body:           { foo: bar }
+          version_type:   force
+          version:        5
+
+ - match:   { _version: 5}
+
+ - do:
+      warnings:
+          - version type FORCE is deprecated and will be removed in the next major version
+      delete:
+          index:          test_1
+          type:           test
+          id:             1
+          version_type:   force
+          version:        4
+
+ - match:   { _version: 4}
+
+ - do:
+      warnings:
+          - version type FORCE is deprecated and will be removed in the next major version
+      index:
+          index:          test_1
+          type:           test
+          id:             1
+          body:           { foo: bar }
+          version_type:   force
+          version:        6
+
+ - match:   { _version: 6}
+
+ - do:
+      warnings:
+          - version type FORCE is deprecated and will be removed in the next major version
+      delete:
+          index:          test_1
+          type:           test
+          id:             1
+          version_type:   force
+          version:        6
+
+ - match:   { _version: 6}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/exists/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/exists/10_basic.yaml
index ee6d73f859968..cba43b6c02503 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/exists/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/exists/10_basic.yaml
@@ -25,3 +25,27 @@
         id:    1
 
   - is_true: ''
+
+---
+"test version parameter is recognized":
+  - skip:
+      version: " - 5.3.99"
+      reason:  This parameter was added in 5.4.0
+
+  - do:
+      index:
+        index: test_1
+        type:  test
+        id:    1
+        body:  { "foo": "bar" }
+
+  - is_true: ''
+
+  - do:
+      exists:
+        index: test_1
+        type: test
+        id: 1
+        version: 1
+
+  - is_true: ''
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/exists/60_realtime_refresh.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/exists/60_realtime_refresh.yaml
index aa45376b52f7b..df8c697e4a1fb 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/exists/60_realtime_refresh.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/exists/60_realtime_refresh.yaml
@@ -26,7 +26,7 @@
           index:    test_1
           type:     test
           id:       1
-          realtime: 0
+          realtime: false
 
  - is_false: ''
  
@@ -35,7 +35,7 @@
           index:    test_1
           type:     test
           id:       1
-          realtime: 1
+          realtime: true
 
  - is_true: ''
 
@@ -44,7 +44,7 @@
           index:    test_1
           type:     test
           id:       1
-          realtime: 0
-          refresh:  1
+          realtime: false
+          refresh:  true
 
  - is_true: ''
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/explain/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/explain/10_basic.yaml
index 1923377ba8310..b5a9212d36b52 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/explain/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/explain/10_basic.yaml
@@ -4,7 +4,9 @@ setup:
           index: test_1
           body:
               aliases:
-                alias_1: {}
+                alias_1:
+                  "filter" : { "term" : { "foo" : "bar"} }
+
   - do:
       index:
           index:  test_1
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/explain/30_query_string.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/explain/30_query_string.yaml
index 30fe6cc55b621..78ef8c4bc8942 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/explain/30_query_string.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/explain/30_query_string.yaml
@@ -68,17 +68,6 @@
         type:   test
         id:     1
         q: field:BA*
-        lowercase_expanded_terms: false
-
-  - is_false: matched
-
-  - do:
-      explain:
-        index:  test
-        type:   test
-        id:     1
-        q: field:BA*
-        analyze_wildcard: true
 
   - is_true: matched
 
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/field_caps/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/field_caps/10_basic.yaml
new file mode 100644
index 0000000000000..f22afb91169dd
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/field_caps/10_basic.yaml
@@ -0,0 +1,209 @@
+---
+setup:
+  - do:
+        indices.create:
+          index: test1
+          body:
+              mappings:
+                t:
+                  properties:
+                    text:
+                      type:     text
+                    keyword:
+                      type:     keyword
+                    number:
+                      type:     double
+                    geo:
+                      type:     geo_point
+                    object:
+                      type: object
+                      properties:
+                        nested1 :
+                          type : text
+                          index: false
+                        nested2:
+                          type: float
+                          doc_values: false
+  - do:
+        indices.create:
+          index: test2
+          body:
+              mappings:
+                t:
+                  properties:
+                    text:
+                      type:     text
+                    keyword:
+                      type:     keyword
+                    number:
+                      type:     double
+                    geo:
+                      type:     geo_point
+                    object:
+                      type: object
+                      properties:
+                        nested1 :
+                          type : text
+                          index: true
+                        nested2:
+                          type: float
+                          doc_values: true
+  - do:
+        indices.create:
+          index: test3
+          body:
+              mappings:
+                t:
+                  properties:
+                    text:
+                      type:     text
+                    keyword:
+                      type:     keyword
+                    number:
+                      type:     long
+                    geo:
+                      type:     keyword
+                    object:
+                      type: object
+                      properties:
+                        nested1 :
+                          type : long
+                          index: false
+                        nested2:
+                          type: keyword
+                          doc_values: false
+
+---
+"Get simple field caps":
+  - skip:
+      version: " - 5.3.99"
+      reason:  this uses a new API that has been added in 5.4.0
+
+  - do:
+      field_caps:
+        index: 'test1,test2,test3'
+        fields: [text, keyword, number, geo]
+
+  - match: {fields.text.text.searchable:                true}
+  - match: {fields.text.text.aggregatable:              false}
+  - is_false: fields.text.text.indices
+  - is_false: fields.text.text.non_searchable_indices
+  - is_false: fields.text.text.non_aggregatable_indices
+  - match: {fields.keyword.keyword.searchable:          true}
+  - match: {fields.keyword.keyword.aggregatable:        true}
+  - is_false: fields.text.keyword.indices
+  - is_false: fields.text.keyword.non_searchable_indices
+  - is_false: fields.text.keyword.non_aggregatable_indices
+  - match: {fields.number.double.searchable:            true}
+  - match: {fields.number.double.aggregatable:          true}
+  - match: {fields.number.double.indices:               ["test1", "test2"]}
+  - is_false: fields.number.double.non_searchable_indices
+  - is_false: fields.number.double.non_aggregatable_indices
+  - match: {fields.number.long.searchable:              true}
+  - match: {fields.number.long.aggregatable:            true}
+  - match: {fields.number.long.indices:                 ["test3"]}
+  - is_false: fields.number.long.non_searchable_indices
+  - is_false: fields.number.long.non_aggregatable_indices
+  - match: {fields.geo.geo_point.searchable:            true}
+  - match: {fields.geo.geo_point.aggregatable:          true}
+  - match: {fields.geo.geo_point.indices:               ["test1", "test2"]}
+  - is_false: fields.geo.geo_point.non_searchable_indices
+  - is_false: fields.geo.geo_point.non_aggregatable_indices
+  - match: {fields.geo.keyword.searchable:              true}
+  - match: {fields.geo.keyword.aggregatable:            true}
+  - match: {fields.geo.keyword.indices:                 ["test3"]}
+  - is_false: fields.geo.keyword.non_searchable_indices
+  - is_false: fields.geo.keyword.on_aggregatable_indices
+---
+"Get nested field caps":
+  - skip:
+      version: " - 5.3.99"
+      reason:  this uses a new API that has been added in 5.4.0
+
+  - do:
+      field_caps:
+        index: 'test1,test2,test3'
+        fields: object*
+
+  - match: {fields.object\.nested1.long.searchable:                       false}
+  - match: {fields.object\.nested1.long.aggregatable:                     true}
+  - match: {fields.object\.nested1.long.indices:                          ["test3"]}
+  - is_false: fields.object\.nested1.long.non_searchable_indices
+  - is_false: fields.object\.nested1.long.non_aggregatable_indices
+  - match: {fields.object\.nested1.text.searchable:                       false}
+  - match: {fields.object\.nested1.text.aggregatable:                     false}
+  - match: {fields.object\.nested1.text.indices:                          ["test1", "test2"]}
+  - match: {fields.object\.nested1.text.non_searchable_indices:           ["test1"]}
+  - is_false: fields.object\.nested1.text.non_aggregatable_indices
+  - match: {fields.object\.nested2.float.searchable:                      true}
+  - match: {fields.object\.nested2.float.aggregatable:                    false}
+  - match: {fields.object\.nested2.float.indices:                         ["test1", "test2"]}
+  - match: {fields.object\.nested2.float.non_aggregatable_indices:        ["test1"]}
+  - is_false: fields.object\.nested2.float.non_searchable_indices
+  - match: {fields.object\.nested2.keyword.searchable:                    true}
+  - match: {fields.object\.nested2.keyword.aggregatable:                  false}
+  - match: {fields.object\.nested2.keyword.indices:                       ["test3"]}
+  - is_false: fields.object\.nested2.keyword.non_aggregatable_indices
+  - is_false: fields.object\.nested2.keyword.non_searchable_indices
+---
+"Get prefix field caps":
+  - skip:
+      version: " - 5.3.99"
+      reason:  this uses a new API that has been added in 5.4.0
+
+  - do:
+      field_caps:
+        index: _all
+        fields: "n*"
+  - match: {fields.number.double.searchable:            true}
+  - match: {fields.number.double.aggregatable:          true}
+  - match: {fields.number.double.indices:               ["test1", "test2"]}
+  - is_false: fields.number.double.non_searchable_indices
+  - is_false: fields.number.double.non_aggregatable_indices
+  - match: {fields.number.long.searchable:              true}
+  - match: {fields.number.long.aggregatable:            true}
+  - match: {fields.number.long.indices:                 ["test3"]}
+  - is_false: fields.number.long.non_searchable_indices
+  - is_false: fields.number.long.non_aggregatable_indices
+
+---
+"Mix in non-existing field field caps":
+  - skip:
+      version: " - 5.4.0"
+      reason:  "#24504 fixed a bug in this API in 5.4.1"
+
+  - do:
+      field_caps:
+        index: 'test1,test2,test3'
+        fields: [text, keyword, no_such_field, number, geo]
+
+  - match: {fields.text.text.searchable:                true}
+  - match: {fields.text.text.aggregatable:              false}
+  - is_false: fields.text.text.indices
+  - is_false: fields.text.text.non_searchable_indices
+  - is_false: fields.text.text.non_aggregatable_indices
+  - match: {fields.keyword.keyword.searchable:          true}
+  - match: {fields.keyword.keyword.aggregatable:        true}
+  - is_false: fields.text.keyword.indices
+  - is_false: fields.text.keyword.non_searchable_indices
+  - is_false: fields.text.keyword.non_aggregatable_indices
+  - match: {fields.number.double.searchable:            true}
+  - match: {fields.number.double.aggregatable:          true}
+  - match: {fields.number.double.indices:               ["test1", "test2"]}
+  - is_false: fields.number.double.non_searchable_indices
+  - is_false: fields.number.double.non_aggregatable_indices
+  - match: {fields.number.long.searchable:              true}
+  - match: {fields.number.long.aggregatable:            true}
+  - match: {fields.number.long.indices:                 ["test3"]}
+  - is_false: fields.number.long.non_searchable_indices
+  - is_false: fields.number.long.non_aggregatable_indices
+  - match: {fields.geo.geo_point.searchable:            true}
+  - match: {fields.geo.geo_point.aggregatable:          true}
+  - match: {fields.geo.geo_point.indices:               ["test1", "test2"]}
+  - is_false: fields.geo.geo_point.non_searchable_indices
+  - is_false: fields.geo.geo_point.non_aggregatable_indices
+  - match: {fields.geo.keyword.searchable:              true}
+  - match: {fields.geo.keyword.aggregatable:            true}
+  - match: {fields.geo.keyword.indices:                 ["test3"]}
+  - is_false: fields.geo.keyword.non_searchable_indices
+  - is_false: fields.geo.keyword.on_aggregatable_indices
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/field_stats/10_basics.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/field_stats/10_basics.yaml
index 35cc19224ec4e..6053d685e2e55 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/field_stats/10_basics.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/field_stats/10_basics.yaml
@@ -12,6 +12,12 @@ setup:
                     type: long
                   bar:
                     type: long
+                  geo:
+                    type: geo_point
+                  geo_shape:
+                    type: geo_shape
+                    tree: quadtree
+                    precision: 1m
 
   - do:
       indices.create:
@@ -26,27 +32,41 @@ setup:
                     type: long
                   bar:
                     type: text
+                  geo:
+                    type: geo_point
+                    index: no
+                  geo_shape:
+                    type: geo_shape
+                    tree: quadtree
+                    precision: 1m
 
   - do:
       index:
           index:  test_1
           type:   test
           id:     id_1
-          body:   { foo: "bar", number: 123, bar: 123 }
+          body:   { foo: "bar", number: 123, bar: 123, geo: { lat: 48.858093, lon: 2.294694} }
 
   - do:
       index:
           index:  test_2
           type:   test
           id:     id_10
-          body: { foo: "babar", number: 456, bar: "123" }
+          body: { foo: "babar", number: 456, bar: "123", geo: { lat: 48.858093, lon: 2.294694},  geo_shape:  {type: "linestring", coordinates : [[-77.03653, 38.897676], [-77.009051, 38.889939]] } }
 
   - do:
       indices.refresh: {}
 
 ---
 "Basic field stats":
+  - skip:
+      version: " - 5.3.99"
+      reason:  Deprecation was added in 5.4.0
+      features: warnings
+
   - do:
+      warnings:
+          - "[_field_stats] endpoint is deprecated! Use [_field_caps] instead or run a min/max aggregations on the desired fields."
       field_stats:
           fields:  [foo, number]
 
@@ -68,11 +88,52 @@ setup:
   - match: { indices._all.fields.number.max_value: 456 }
   - match: { indices._all.fields.number.max_value_as_string: "456" }
   - match: { indices._all.fields.number.type: "integer" }
+
+  - is_false: conflicts
+
+---
+"Geo field stats":
+  - skip:
+      version: " - 5.3.99"
+      reason:  Deprecation was added in 5.4.0
+      features: warnings
+
+  - do:
+      warnings:
+          - "[_field_stats] endpoint is deprecated! Use [_field_caps] instead or run a min/max aggregations on the desired fields."
+      field_stats:
+          fields:  [geo, geo_shape]
+
+  - match: { indices._all.fields.geo.type: "geo_point" }
+  - match: { indices._all.fields.geo.max_doc: 2 }
+  - match: { indices._all.fields.geo.doc_count: -1 }
+  - match: { indices._all.fields.geo.searchable: true }
+  - match: { indices._all.fields.geo.aggregatable: true }
+  - is_false: indices._all.fields.geo.min_value
+  - is_false: indices._all.fields.geo.max_value
+  - is_false: indices._all.fields.geo.min_value_as_string
+  - is_false: indices._all.fields.geo.max_value_as_string
+  - match: { indices._all.fields.geo_shape.type: "string" }
+  - match: { indices._all.fields.geo_shape.max_doc: 1 }
+  - match: { indices._all.fields.geo_shape.doc_count: -1 }
+  - match: { indices._all.fields.geo_shape.searchable: true }
+  - match: { indices._all.fields.geo_shape.aggregatable: false }
+  - is_false: indices._all.fields.geo_shape.min_value
+  - is_false: indices._all.fields.geo_shape.max_value
+  - is_false: indices._all.fields.geo_shape.min_value_as_string
+  - is_false: indices._all.fields.geo_shape.max_value_as_string
   - is_false: conflicts
 
 ---
 "Basic field stats with level set to indices":
+  - skip:
+      version: " - 5.3.99"
+      reason:  Deprecation was added in 5.4.0
+      features: warnings
+
   - do:
+      warnings:
+          - "[_field_stats] endpoint is deprecated! Use [_field_caps] instead or run a min/max aggregations on the desired fields."
       field_stats:
           fields:  [foo, number]
           level: indices
@@ -116,10 +177,76 @@ setup:
   - match: { indices.test_2.fields.number.type: "integer" }
   - is_false: conflicts
 
+
+---
+"Geo field stats with level set to indices":
+  - skip:
+      version: " - 5.3.99"
+      reason:  Deprecation was added in 5.4.0
+      features: warnings
+
+  - do:
+      warnings:
+          - "[_field_stats] endpoint is deprecated! Use [_field_caps] instead or run a min/max aggregations on the desired fields."
+      field_stats:
+          fields:  [geo, geo_shape]
+          level: indices
+
+  - match: { indices.test_1.fields.geo.max_doc: 1 }
+  - match: { indices.test_1.fields.geo.doc_count: 1 }
+  - is_true: indices.test_1.fields.geo.min_value
+  - is_true: indices.test_1.fields.geo.max_value
+  - is_true: indices.test_1.fields.geo.min_value_as_string
+  - is_true: indices.test_1.fields.geo.max_value_as_string
+  - match: { indices.test_1.fields.geo.searchable: true }
+  - match: { indices.test_1.fields.geo.aggregatable: true }
+  - match: { indices.test_1.fields.geo.type: "geo_point" }
+  - is_true: indices.test_2.fields.geo
+  - match: { indices.test_2.fields.geo_shape.max_doc: 1 }
+  - match: { indices.test_2.fields.geo_shape.doc_count: -1 }
+  - is_false: indices.test_2.fields.geo_shape.min_value
+  - is_false: indices.test_2.fields.geo_shape.max_value
+  - is_false: indices.test_2.fields.geo_shape.min_value_as_string
+  - is_false: indices.test_2.fields.geo_shape.max_value_as_string
+  - match: { indices.test_2.fields.geo_shape.searchable: true }
+  - match: { indices.test_2.fields.geo_shape.aggregatable: false }
+  - match: { indices.test_2.fields.geo_shape.type: "string" }
+  - is_false: conflicts
+
+---
+"Geopoint field stats":
+  - skip:
+      version: " - 5.3.99"
+      reason:  Deprecation was added in 5.4.0
+      features: warnings
+
+  - do:
+      warnings:
+          - "[_field_stats] endpoint is deprecated! Use [_field_caps] instead or run a min/max aggregations on the desired fields."
+      field_stats:
+          fields:  [geo]
+          level: indices
+
+  - match: { indices.test_2.fields.geo.max_doc: 1 }
+  - match: { indices.test_2.fields.geo.doc_count: -1 }
+  - is_false: indices.test_2.fields.geo.min_value
+  - is_false: indices.test_2.fields.geo.max_value
+  - match: { indices.test_2.fields.geo.searchable: false }
+  - match: { indices.test_2.fields.geo.aggregatable: true }
+  - match: { indices.test_2.fields.geo.type: "geo_point" }
+  - is_true: indices.test_2.fields.geo
+  - is_false: conflicts
+
 ---
 "Field stats with filtering":
+  - skip:
+      version: " - 5.3.99"
+      reason:  Deprecation was added in 5.4.0
+      features: warnings
 
   - do:
+      warnings:
+          - "[_field_stats] endpoint is deprecated! Use [_field_caps] instead or run a min/max aggregations on the desired fields."
       field_stats:
           level:  indices
           index:  test_1
@@ -136,6 +263,8 @@ setup:
   - is_false: conflicts
 
   - do:
+      warnings:
+          - "[_field_stats] endpoint is deprecated! Use [_field_caps] instead or run a min/max aggregations on the desired fields."
       field_stats:
           level:  indices
           index:  test_1
@@ -145,6 +274,11 @@ setup:
 
 ---
 "Field stats both source and fields":
+  - skip:
+      version: " - 5.3.99"
+      reason:  Deprecation was added in 5.4.0
+      features: warnings
+
   - do:
       catch: request
       field_stats:
@@ -154,7 +288,14 @@ setup:
 
 ---
 "Field stats with conflicts":
+  - skip:
+      version: " - 5.3.99"
+      reason:  Deprecation was added in 5.4.0
+      features: warnings
+
   - do:
+      warnings:
+          - "[_field_stats] endpoint is deprecated! Use [_field_caps] instead or run a min/max aggregations on the desired fields."
       field_stats:
           fields:  [foo, number, bar]
 
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/get/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/get/10_basic.yaml
index 0689f714d6416..288c032edc147 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/get/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/get/10_basic.yaml
@@ -29,3 +29,68 @@
   - match: { _type:    test   }
   - match: { _id:      中文      }
   - match: { _source:  { foo: "Hello: 中文" } }
+
+
+---
+"Single type":
+
+  - skip:
+      version: " - 5.4.99"
+      reason: single_type only available as of 5.5.0
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            mapping.single_type: true
+
+  - do:
+      index:
+        index: test
+        type:  doc
+        id:    abc123
+        body:  { "foo": "bar" }
+
+  - do:
+      get:
+        index: test
+        type:  doc
+        id:    abc123
+
+  - match: { _index:   test }
+  - match: { _type:    doc }
+  - match: { _id:      abc123 }
+  - match: { _source:  { foo: "bar" } }
+
+  - do:
+      get:
+        index: test
+        type:  _all
+        id:    abc123
+
+  - match: { _index:   test }
+  - match: { _type:    doc }
+  - match: { _id:      abc123 }
+  - match: { _source:  { foo: "bar" } }
+
+  - do:
+      catch: missing
+      get:
+        index: test
+        type:  other_type
+        id:    abc123
+
+  - do:
+      delete:
+        index: test
+        type:  doc
+        id:    abc123
+
+  - do:
+      catch: missing
+      get:
+        index: test
+        type:  doc
+        id:    abc123
+
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/get/20_fields.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/get/20_fields.yaml
deleted file mode 100644
index 3de8ab1d91d9b..0000000000000
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/get/20_fields.yaml
+++ /dev/null
@@ -1,59 +0,0 @@
----
-"Fields":
-
-  - do:
-      indices.create:
-        index: test_1
-        body:
-          mappings:
-            test:
-              properties:
-                foo:
-                  type: keyword
-                  store: true
-                count:
-                  type: integer
-                  store: true
-
-  - do:
-      index:
-        index:  test_1
-        type:   test
-        id:     1
-        body:   { "foo": "bar", "count": 1 }
-  - do:
-      get:
-        index:  test_1
-        type:   test
-        id:     1
-        fields: foo
-
-  - match:   { _index:   test_1  }
-  - match:   { _type:    test    }
-  - match:   { _id:      '1'     }
-  - match:   { fields.foo:  [bar] }
-  - is_false:  _source
-
-  - do:
-      get:
-        index:  test_1
-        type:   test
-        id:     1
-        fields: [foo, count]
-
-  - match:   { fields.foo:  [bar] }
-  - match:   { fields.count:  [1] }
-  - is_false:  _source
-
-  - do:
-      get:
-        index:  test_1
-        type:   test
-        id:     1
-        fields: [foo, count, _source]
-
-  - match: { fields.foo:  [bar] }
-  - match: { fields.count:  [1] }
-  - match: { _source.foo: bar }
-
-
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/get/20_stored_fields.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/get/20_stored_fields.yaml
new file mode 100644
index 0000000000000..fbffb9e0ea8ce
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/get/20_stored_fields.yaml
@@ -0,0 +1,59 @@
+---
+"Stored fields":
+
+  - do:
+      indices.create:
+        index: test_1
+        body:
+          mappings:
+            test:
+              properties:
+                foo:
+                  type: keyword
+                  store: true
+                count:
+                  type: integer
+                  store: true
+
+  - do:
+      index:
+        index:  test_1
+        type:   test
+        id:     1
+        body:   { "foo": "bar", "count": 1 }
+  - do:
+      get:
+        index:  test_1
+        type:   test
+        id:     1
+        stored_fields: foo
+
+  - match:   { _index:   test_1  }
+  - match:   { _type:    test    }
+  - match:   { _id:      '1'     }
+  - match:   { fields.foo:  [bar] }
+  - is_false:  _source
+
+  - do:
+      get:
+        index:  test_1
+        type:   test
+        id:     1
+        stored_fields: [foo, count]
+
+  - match:   { fields.foo:  [bar] }
+  - match:   { fields.count:  [1] }
+  - is_false:  _source
+
+  - do:
+      get:
+        index:  test_1
+        type:   test
+        id:     1
+        stored_fields: [foo, count, _source]
+
+  - match: { fields.foo:  [bar] }
+  - match: { fields.count:  [1] }
+  - match: { _source.foo: bar }
+
+
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/get/30_parent.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/get/30_parent.yaml
index 4a842cee2e039..353dce8fab7da 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/get/30_parent.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/get/30_parent.yaml
@@ -25,7 +25,7 @@ setup:
           type:    test
           id:      1
           parent:  中文
-          fields:  [_parent, _routing]
+          stored_fields:  [_parent, _routing]
 
  - match:   { _id:      "1"}
  - match:   { _parent: 中文 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/get/40_routing.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/get/40_routing.yaml
index 11fe04884b30c..94a40c0437a4d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/get/40_routing.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/get/40_routing.yaml
@@ -28,7 +28,7 @@
           type:    test
           id:      1
           routing: 5
-          fields:  [_routing]
+          stored_fields:  [_routing]
 
  - match:   { _id:              "1"}
  - match:   { _routing:  "5"}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/get/55_parent_with_routing.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/get/55_parent_with_routing.yaml
index a7ed2df0dd7c6..db71d16423009 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/get/55_parent_with_routing.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/get/55_parent_with_routing.yaml
@@ -32,7 +32,7 @@
           id:      1
           parent:  5
           routing: 4
-          fields:  [_parent, _routing]
+          stored_fields:  [_parent, _routing]
 
  - match:   { _id:      "1"}
  - match:   { _parent: "5"}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/get/60_realtime_refresh.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/get/60_realtime_refresh.yaml
index 0c705de5e7e40..7d02b4667efe7 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/get/60_realtime_refresh.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/get/60_realtime_refresh.yaml
@@ -27,14 +27,14 @@
           index:    test_1
           type:     test
           id:       1
-          realtime: 0
+          realtime: false
 
  - do:
       get:
           index:    test_1
           type:     test
           id:       1
-          realtime: 1
+          realtime: true
 
  - is_true: found
 
@@ -43,7 +43,7 @@
           index:    test_1
           type:     test
           id:       1
-          realtime: 0
-          refresh:  1
+          realtime: false
+          refresh:  true
 
  - is_true: found
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/get/70_source_filtering.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/get/70_source_filtering.yaml
index 2c85a2d54ab0a..c858886ca3df3 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/get/70_source_filtering.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/get/70_source_filtering.yaml
@@ -58,7 +58,7 @@
         index:  test_1
         type:   test
         id:     1
-        fields: count
+        stored_fields: count
         _source: true
 
   - match:   { _index:   test_1  }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/get/90_versions.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/get/90_versions.yaml
index e255ce510ed17..c6631b83b1867 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/get/90_versions.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/get/90_versions.yaml
@@ -86,3 +86,4 @@
           id:     1
           version: 1
           version_type: external_gte
+
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/get/95_force_versions.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/get/95_force_versions.yaml
new file mode 100644
index 0000000000000..afbc82e3b6d00
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/get/95_force_versions.yaml
@@ -0,0 +1,54 @@
+---
+"Force Versions":
+ - skip:
+    features: "warnings"
+    version: " - 5.2.99"
+    reason: warning headers were introduced in 5.0.1 and their format changed in 5.3.0
+ - do:
+      index:
+          index:  test_1
+          type:   test
+          id:     1
+          body:   { foo: bar }
+ - match:   { _version: 1}
+
+ - do:
+      index:
+          index:  test_1
+          type:   test
+          id:     1
+          body:   { foo: bar }
+ - match:   { _version: 2}
+
+ - do:
+      warnings:
+          - version type FORCE is deprecated and will be removed in the next major version
+      get:
+          index:  test_1
+          type:   test
+          id:     1
+          version: 2
+          version_type: force
+ - match:   { _id: "1" }
+
+ - do:
+      warnings:
+          - version type FORCE is deprecated and will be removed in the next major version
+      get:
+          index:  test_1
+          type:   test
+          id:     1
+          version: 10
+          version_type: force
+ - match:   { _id: "1" }
+
+ - do:
+      warnings:
+          - version type FORCE is deprecated and will be removed in the next major version
+      get:
+          index:  test_1
+          type:   test
+          id:     1
+          version: 1
+          version_type: force
+ - match:   { _id: "1" }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/60_realtime_refresh.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/60_realtime_refresh.yaml
index 20009c87b289b..fbf3948d7b1e8 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/60_realtime_refresh.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/60_realtime_refresh.yaml
@@ -27,14 +27,14 @@
           index:    test_1
           type:     test
           id:       1
-          realtime: 0
+          realtime: false
 
  - do:
       get_source:
           index:    test_1
           type:     test
           id:       1
-          realtime: 1
+          realtime: true
 
  - match:   { '': {foo: bar}}
 
@@ -43,8 +43,8 @@
           index:    test_1
           type:     test
           id:       1
-          realtime: 0
-          refresh:  1
+          realtime: false
+          refresh:  true
 
  - match:   { '': {foo: bar}}
 
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/37_force_version.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/37_force_version.yaml
new file mode 100644
index 0000000000000..4d0d38bfc1ab7
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/37_force_version.yaml
@@ -0,0 +1,59 @@
+---
+"Force Version":
+
+ - skip:
+    features: "warnings"
+    version: " - 5.2.99"
+    reason:  warning headers were introduced in 5.0.1 and their format changed in 5.3.0
+
+ - do:
+      warnings:
+          - version type FORCE is deprecated and will be removed in the next major version
+      index:
+          index:          test_1
+          type:           test
+          id:             1
+          body:           { foo: bar }
+          version_type:   force
+          version:        5
+
+ - match:   { _version: 5}
+
+ - do:
+      warnings:
+          - version type FORCE is deprecated and will be removed in the next major version
+      index:
+          index:          test_1
+          type:           test
+          id:             1
+          body:           { foo: bar }
+          version_type:   force
+          version:        4
+
+ - match:   { _version: 4}
+
+ - do:
+      warnings:
+          - version type FORCE is deprecated and will be removed in the next major version
+      index:
+          index:          test_1
+          type:           test
+          id:             1
+          body:           { foo: bar2 }
+          version_type:   force
+          version:        5
+
+ - match:   { _version: 5}
+
+ - do:
+      warnings:
+          - version type FORCE is deprecated and will be removed in the next major version
+      index:
+          index:          test_1
+          type:           test
+          id:             1
+          body:           { foo: bar3 }
+          version_type:   force
+          version:        5
+
+ - match:   { _version: 5}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/40_routing.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/40_routing.yaml
index bb248a458ae12..7b3c21df4e065 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/index/40_routing.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/40_routing.yaml
@@ -28,7 +28,7 @@
           type:    test
           id:      1
           routing: 5
-          fields:  [_routing]
+          stored_fields:  [_routing]
 
  - match:   { _id:       "1"}
  - match:   { _routing:  "5"}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/50_parent.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/50_parent.yaml
index 916e1ac3520c1..89077dac4ae8e 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/index/50_parent.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/50_parent.yaml
@@ -31,7 +31,7 @@
           type:    test
           id:      1
           parent:  5
-          fields:  [_parent, _routing]
+          stored_fields:  [_parent, _routing]
 
  - match:   { _id:      "1"}
  - match:   { _parent: "5"}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/55_parent_with_routing.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/55_parent_with_routing.yaml
index a7ed2df0dd7c6..db71d16423009 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/index/55_parent_with_routing.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/55_parent_with_routing.yaml
@@ -32,7 +32,7 @@
           id:      1
           parent:  5
           routing: 4
-          fields:  [_parent, _routing]
+          stored_fields:  [_parent, _routing]
 
  - match:   { _id:      "1"}
  - match:   { _parent: "5"}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.analyze/10_analyze.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.analyze/10_analyze.yaml
index 35d4a2b52223b..ffda03c5753cf 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.analyze/10_analyze.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.analyze/10_analyze.yaml
@@ -6,7 +6,13 @@ setup:
 
 ---
 "Basic test":
+    - skip:
+        features: "warnings"
+        version: " - 5.2.99"
+        reason:  text was deprecated in 5.1.0 and the deprecation format changed in 5.3.0
     - do:
+        warnings:
+          - text request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
         indices.analyze:
           text: Foo Bar
     - length: { tokens: 2 }
@@ -15,7 +21,15 @@ setup:
 
 ---
 "Tokenizer and filter":
+    - skip:
+        features: "warnings"
+        version: " - 5.2.99"
+        reason:  text was deprecated in 5.1.0 and the deprecation format changed in 5.3.0
     - do:
+        warnings:
+          - filter request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
+          - text request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
+          - tokenizer request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
         indices.analyze:
           filter:      lowercase
           text:         Foo Bar
@@ -25,6 +39,10 @@ setup:
 
 ---
 "Index and field":
+    - skip:
+        features: "warnings"
+        version: " - 5.2.99"
+        reason:  text was deprecated in 5.1.0 and the deprecation format changed in 5.3.0
     - do:
         indices.create:
           index: test
@@ -37,6 +55,9 @@ setup:
                     analyzer: whitespace
 
     - do:
+        warnings:
+          - field request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
+          - text request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
         indices.analyze:
           field: text
           index: test
@@ -53,7 +74,13 @@ setup:
     - match:     { tokens.0.token: foo bar }
 ---
 "Body params override query string":
+    - skip:
+        features: "warnings"
+        version: " - 5.2.99"
+        reason:  text was deprecated in 5.1.0 and the deprecation format changed in 5.3.0
     - do:
+        warnings:
+          - text request parameter is deprecated and will be removed in the next major release. Please use the JSON in the request body instead request param
         indices.analyze:
           text: Foo Bar
           body: { "text": "Bar Foo", "filter": ["lowercase"], "tokenizer": keyword }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.clear_cache/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.clear_cache/10_basic.yaml
index 3388d06ba1748..d8db152e979b0 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.clear_cache/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.clear_cache/10_basic.yaml
@@ -2,3 +2,26 @@
 "clear_cache test":
   - do:
       indices.clear_cache: {}
+
+---
+"clear_cache with request set to false":
+  - skip:
+      version: " - 5.3.99"
+      reason: this name was added in 5.4
+
+  - do:
+      indices.clear_cache:
+        request: false
+
+---
+"clear_cache with request_cache set to false":
+  - skip:
+      version: " - 5.3.99"
+      reason: request_cache was deprecated in 5.4.0
+      features: "warnings"
+
+  - do:
+      warnings:
+        - 'Deprecated field [request_cache] used, expected [request] instead'
+      indices.clear_cache:
+        request_cache: false
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yaml
index 38ca46570f90b..2df641f9e85eb 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yaml
@@ -30,6 +30,19 @@
 
   - match: { test_index.settings.index.number_of_replicas: "0"}
 
+---
+"Create index":
+  - skip:
+      version: " - 5.5.99"
+      reason: create index response contains index name since 5.6.0
+
+  - do:
+      indices.create:
+        index: test_index
+
+  - match: { acknowledged: true }
+  - match: { index: "test_index"}
+
 ---
 "Create index with too large wait_for_active_shards":
 
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/20_warnings.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/20_warnings.yaml
new file mode 100644
index 0000000000000..c5bcfcc819090
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/20_warnings.yaml
@@ -0,0 +1,28 @@
+---
+"Create index with deprecated settings":
+
+  - skip:
+      version: " - 5.3.0"
+      reason: fixed in 5.3.1
+      features: "warnings"
+  - do:
+      indices.create:
+        index: test_index
+        body:
+          settings:
+            number_of_shards: 1
+            shadow_replicas: true
+            shared_filesystem: false
+          mappings:
+            type:
+              properties:
+                field:
+                  type: "string"
+                field2:
+                  type: "long"
+                  store : "no"
+      warnings:
+        - "[index.shadow_replicas] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version."
+        - "[index.shared_filesystem] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version."
+        - "The [string] field is deprecated, please use [text] or [keyword] instead on [field]"
+        - "Expected a boolean [true/false] for property [field2.store] but got [no]"
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.delete_alias/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.delete_alias/10_basic.yaml
index 87f61efc112bc..e9645e69a534d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.delete_alias/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.delete_alias/10_basic.yaml
@@ -1,5 +1,8 @@
 ---
 "Basic test for delete alias":
+  - skip:
+      version: " - 5.4.99"
+      reason: Previous versions did not 404 on missing aliases
 
   - do:
       indices.create:
@@ -25,8 +28,10 @@
         name: testali
 
   - do:
+      catch: missing
       indices.get_alias:
         index: testind
         name:  testali
   
-  - match: { '': {}}
+  - match: { 'status': 404 }
+  - match: { 'error': 'alias [testali] missing' }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get/10_basic.yaml
index 54728644f72d2..a4e3f26c2c0df 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get/10_basic.yaml
@@ -79,7 +79,14 @@ setup:
 ---
 "Get index infos should work for wildcards":
 
+  - skip:
+      features: "warnings"
+      version: " - 5.4.99"
+      reason: deprecation logging for multiple features was introduced in 5.5.0
+
   - do:
+      warnings:
+        - 'Requesting comma-separated features is deprecated and will be removed in 6.0+, retrieve all features instead.'
       indices.get:
         index: test_*
         feature: _mapping,_settings
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_alias/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_alias/10_basic.yaml
index 3cbac46aa5960..6678cbebbd503 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_alias/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_alias/10_basic.yaml
@@ -78,7 +78,6 @@ setup:
 
 ---
 "Get aliases via /{index}/_alias/prefix*":
-
   - do:
       indices.get_alias:
         index: test_index
@@ -166,25 +165,51 @@ setup:
 
 
 ---
-"Non-existent alias on an existing index returns an empty body":
+"Non-existent alias on an existing index returns 404":
+  - skip:
+      version: " - 5.4.99"
+      reason: Previous versions did not 404 on missing aliases
 
   - do:
+      catch: missing
       indices.get_alias:
         index: test_index
         name: non-existent
 
-  - match: { '': {}}
+  - match: { 'status': 404}
+  - match: { 'error': 'alias [non-existent] missing' }
 
 ---
-"Existent and non-existent alias returns just the existing":
+"Existent and non-existent alias returns 404 and the existing alias":
+  - skip:
+      version: " - 5.4.99"
+      reason: Previous versions did not 404 on missing aliases
 
   - do:
+      catch: missing
       indices.get_alias:
         index: test_index
         name: test_alias,non-existent
 
-  - match: {test_index.aliases.test_alias: {}}
-  - is_false: test_index.aliases.non-existent
+  - match: { 'status': 404 }
+  - match: { 'error': 'alias [non-existent] missing' }
+  - match: { test_index.aliases.test_alias: { } }
+
+---
+"Existent and non-existent aliases returns 404 and the existing alias":
+  - skip:
+      version: " - 5.4.99"
+      reason: Previous versions did not 404 on missing aliases
+
+  - do:
+      catch: missing
+      indices.get_alias:
+        index: test_index
+        name: test_alias,non-existent,another-non-existent
+
+  - match: { 'status': 404 }
+  - match: { 'error': 'aliases [another-non-existent,non-existent] missing' }
+  - match: { test_index.aliases.test_alias: { } }
 
 ---
 "Getting alias on an non-existent index should return 404":
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_mapping/20_missing_type.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_mapping/20_missing_type.yaml
index 8efe2a74c9bd0..dab4754dcd894 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_mapping/20_missing_type.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_mapping/20_missing_type.yaml
@@ -1,5 +1,8 @@
 ---
-"Return empty response when type doesn't exist":
+"Non-existent type returns 404":
+  - skip:
+      version: " - 5.4.99"
+      reason: Previous versions did not 404 on missing types
   - do:
         indices.create:
           index: test_index
@@ -12,8 +15,99 @@
                       analyzer: whitespace
 
   - do:
+      catch: missing
       indices.get_mapping:
         index: test_index
         type: not_test_type
+
+  - match: { status: 404 }
+  - match: { error.reason: 'type[[not_test_type]] missing' }
+
+---
+"No type matching pattern returns 404":
+  - skip:
+      version: " - 5.4.99"
+      reason: Previous versions did not 404 on missing types
+  - do:
+        indices.create:
+          index: test_index
+          body:
+              mappings:
+                test_type:
+                  properties:
+                    text:
+                      type:     text
+                      analyzer: whitespace
+
+  - do:
+      catch: missing
+      indices.get_mapping:
+        index: test_index
+        type: test*,not*
+
+  - match: { status: 404 }
+  - match: { error: 'type [not*] missing' }
+  - is_true: test_index.mappings.test_type
+
+---
+"Existent and non-existent type returns 404 and the existing type":
+  - skip:
+      version: " - 5.4.99"
+      reason: Previous versions did not 404 on missing types
+  - do:
+        indices.create:
+          index: test_index
+          body:
+              mappings:
+                test_type:
+                  properties:
+                    text:
+                      type:     text
+                      analyzer: whitespace
+
+  - do:
+      catch: missing
+      indices.get_mapping:
+        index: test_index
+        type: test_type,not_test_type
+
+  - match: { status: 404 }
+  - match: { error: 'type [not_test_type] missing' }
+  - is_true: test_index.mappings.test_type
+
+---
+"Existent and non-existent types returns 404 and the existing type":
+  - skip:
+      version: " - 5.4.99"
+      reason: Previous versions did not 404 on missing types
+  - do:
+        indices.create:
+          index: test_index
+          body:
+              mappings:
+                test_type:
+                  properties:
+                    text:
+                      type:     text
+                      analyzer: whitespace
+
+  - do:
+      catch: missing
+      indices.get_mapping:
+        index: test_index
+        type: test_type,not_test_type,another_not_test_type
+
+  - match: { status: 404 }
+  - match: { error: 'types [another_not_test_type,not_test_type] missing' }
+  - is_true: test_index.mappings.test_type
+
+---
+"Type missing when no types exist":
+  - skip:
+      version: " - 5.0.2"
+      reason: there was a bug prior to 5.0.2
+  - do:
+      catch: missing
+      indices.get_mapping:
+        type: not_test_type
  
-  - match: { '': {}}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_mapping/30_missing_index.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_mapping/30_missing_index.yaml
index 6aadb3743753a..cf4e5b56e786e 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_mapping/30_missing_index.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_mapping/30_missing_index.yaml
@@ -6,4 +6,10 @@
         index: test_index
         type: not_test_type
   
+---
+"Index missing, no indexes, no types":
+  - do:
+      catch: missing
+      indices.get_mapping:
+        index: test_index
 
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_alias/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_alias/10_basic.yaml
index 98fd6b3a9842d..5527c023b13a3 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_alias/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_alias/10_basic.yaml
@@ -27,3 +27,35 @@
         name: test_alias
 
   - match: {test_index.aliases.test_alias: {}}
+
+---
+"Can't create alias with invalid characters":
+  - skip:
+      version: " - 5.0.99"
+      reason:  alias name validation was introduced in 5.1.0
+
+  - do:
+      indices.create:
+        index: test_index
+
+  - do:
+      catch: request
+      indices.put_alias:
+        index: test_index
+        name: test_*
+
+---
+"Can't create alias with the same name as an index":
+
+  - do:
+      indices.create:
+        index: test_index
+  - do:
+      indices.create:
+        index: foo
+
+  - do:
+      catch: request
+      indices.put_alias:
+        index: test_index
+        name: foo
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_settings/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_settings/10_basic.yaml
index cc9fb95b78414..043800916d19a 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_settings/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_settings/10_basic.yaml
@@ -36,7 +36,7 @@ setup:
         ignore_unavailable: true
         index: test-index, non-existing
         body:
-          number_of_replicas: 1
+          index.number_of_replicas: 1
 
   - do:
       indices.get_settings: {}
@@ -81,7 +81,6 @@ setup:
       indices.get_settings:
         flat_settings: false
 
-
   - match:
       test-index.settings.index.number_of_replicas: "0"
   - match:
@@ -96,8 +95,9 @@ setup:
         preserve_existing: true
         index: test-index
         body:
-          index.translog.durability: "request"
-          index.query_string.lenient: "true"
+          settings:
+            index.translog.durability: "request"
+            index.query_string.lenient: "true"
 
   - do:
       indices.get_settings:
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_settings/11_reset.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_settings/11_reset.yaml
index 6c93dabeec7d9..bc2dace0e1871 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_settings/11_reset.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_settings/11_reset.yaml
@@ -22,8 +22,7 @@ Test reset index settings:
   - do:
       indices.get_settings:
         flat_settings: false
-  - is_false:
-      test-index.settings.index\.refresh_interval
+  - is_false: test-index.settings.index\.refresh_interval
   - do:
       indices.get_settings:
         include_defaults: true
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_template/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_template/10_basic.yaml
index e599ae43b9215..3d70e930a040a 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_template/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_template/10_basic.yaml
@@ -70,3 +70,99 @@
           settings:
             number_of_shards:   1
             number_of_replicas: 0
+
+---
+"Test Put Versioned Template":
+  - do:
+      indices.put_template:
+        name: "my_template"
+        body: >
+          {
+            "version": 10,
+            "template": "*",
+            "settings": { "number_of_shards": 1 }
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      indices.get_template:
+        name: "my_template"
+  - match: { my_template.version: 10 }
+
+  # Lower version
+  - do:
+      indices.put_template:
+        name: "my_template"
+        body: >
+          {
+            "version": 9,
+            "template": "*",
+            "settings": { "number_of_shards": 1 }
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      indices.get_template:
+        name: "my_template"
+  - match: { my_template.version: 9 }
+
+  # Higher version
+  - do:
+      indices.put_template:
+        name: "my_template"
+        body: >
+          {
+            "version": 6789,
+            "template": "*",
+            "settings": { "number_of_shards": 1 }
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      indices.get_template:
+        name: "my_template"
+  - match: { my_template.version: 6789 }
+
+  # No version
+  - do:
+      indices.put_template:
+        name: "my_template"
+        body: >
+          {
+            "template": "*",
+            "settings": { "number_of_shards": 1 }
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      indices.get_template:
+        name: "my_template"
+  - is_false: my_template.version
+
+  # Coming back with a version
+  - do:
+      indices.put_template:
+        name: "my_template"
+        body: >
+          {
+            "version": 5385,
+            "template": "*",
+            "settings": { "number_of_shards": 1 }
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      indices.get_template:
+        name: "my_template"
+  - match: { my_template.version: 5385 }
+
+  # Able to delete the versioned template
+  - do:
+      indices.delete_template:
+        name: "my_template"
+  - match: { acknowledged: true }
+
+  - do:
+      catch: missing
+      indices.get_template:
+        name: "my_template"
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/10_basic.yaml
index 8c962407b30b8..e798888a5e4fe 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/10_basic.yaml
@@ -16,6 +16,7 @@
         index: logs-1
         type:  test
         id:    "1"
+        refresh: true # make this doc visible in index stats
         body:  { "foo": "hello world" }
 
   - do:
@@ -71,3 +72,87 @@
   - match: { hits.total: 1 }
   - match: { hits.hits.0._index: "logs-000002"}
 
+---
+"Rollover no condition matched":
+  - skip:
+        version: " - 5.0.0"
+        reason:  bug fixed in 5.0.1
+
+  # create index with alias
+  - do:
+      indices.create:
+        index: logs-1
+        wait_for_active_shards: 1
+        body:
+          aliases:
+            logs_index: {}
+            logs_search: {}
+
+  # run again and verify results without rolling over
+  - do:
+      indices.rollover:
+        alias: "logs_search"
+        wait_for_active_shards: 1
+        body:
+          conditions:
+            max_docs: 1
+
+  - match: { old_index: logs-1 }
+  - match: { new_index: logs-000002 }
+  - match: { rolled_over: false }
+  - match: { dry_run: false }
+  - match: { conditions: { "[max_docs: 1]": false } }
+
+---
+"Rollover with dry-run but target index exists":
+
+  - skip:
+        version: " - 5.0.0"
+        reason:  bug fixed in 5.0.1 - dry run was returning just fine even if the index exists
+
+  # create index with alias
+  - do:
+      indices.create:
+        index: logs-1
+        wait_for_active_shards: 1
+        body:
+          aliases:
+            logs_index: {}
+            logs_search: {}
+
+  - do:
+      indices.create:
+        index: logs-000002
+
+  - do:
+      catch: /index_already_exists_exception/
+      indices.rollover:
+        dry_run: true
+        alias: "logs_search"
+        wait_for_active_shards: 1
+        body:
+          conditions:
+            max_docs: 1
+
+  # also do it without dry_run
+  - do:
+      catch: /index_already_exists_exception/
+      indices.rollover:
+        dry_run: false
+        alias: "logs_search"
+        wait_for_active_shards: 1
+        body:
+          conditions:
+            max_docs: 1
+
+  - do:
+      catch: /invalid_index_name_exception/
+      indices.rollover:
+        new_index: invalid|index|name
+        dry_run: true
+        alias: "logs_search"
+        wait_for_active_shards: 1
+        body:
+          conditions:
+            max_docs: 1
+
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/20_max_doc_condition.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/20_max_doc_condition.yml
new file mode 100644
index 0000000000000..ec9fabe02595d
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/20_max_doc_condition.yml
@@ -0,0 +1,57 @@
+---
+"Max docs rollover conditions matches only primary shards":
+  - skip:
+        version: "- 5.4.1"
+        reason:  "matching docs changed from all shards to primary shards"
+
+  # create index with alias and replica
+  - do:
+      indices.create:
+        index: logs-1
+        wait_for_active_shards: 1
+        body:
+          aliases:
+            logs_search: {}
+
+  # index first document and wait for refresh
+  - do:
+      index:
+        index: logs-1
+        type:  test
+        id:    "1"
+        body:  { "foo": "hello world" }
+        refresh: true
+
+  # perform alias rollover with no result
+  - do:
+      indices.rollover:
+        alias: "logs_search"
+        wait_for_active_shards: 1
+        body:
+          conditions:
+            max_docs: 2
+
+  - match: { conditions: { "[max_docs: 2]": false } }
+  - match: { rolled_over: false }
+
+  # index second document and wait for refresh
+  - do:
+      index:
+        index: logs-1
+        type:  test
+        id:    "2"
+        body:  { "foo": "hello world" }
+        refresh: true
+
+  # perform alias rollover
+  - do:
+      indices.rollover:
+        alias: "logs_search"
+        wait_for_active_shards: 1
+        body:
+          conditions:
+            max_docs: 2
+
+  - match: { conditions: { "[max_docs: 2]": true } }
+  - match: { rolled_over: true }
+
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/10_basic.yaml
index 9569728ce7da9..f53c88bcfca2e 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/10_basic.yaml
@@ -1,15 +1,26 @@
 ---
 "Shrink index via API":
-  # creates an index with one document.
-  # relocates all it's shards to one node
-  # shrinks it into a new index with a single shard
+  # creates an index with one document solely allocated on the master node
+  # and shrinks it into a new index with a single shard
+  # we don't do the relocation to a single node after the index is created
+  # here since in a mixed version cluster we can't identify
+  # which node is the one with the highest version and that is the only one that can safely
+  # be used to shrink the index.
+  - do:
+      cluster.state: {}
+      # Get master node id
+
+  - set: { master_node: master }
+
   - do:
       indices.create:
         index: source
         wait_for_active_shards: 1
         body:
           settings:
-            number_of_replicas: "0"
+            # ensure everything is allocated on a single node
+            index.routing.allocation.include._id: $master
+            number_of_replicas: 0
   - do:
       index:
         index: source
@@ -28,18 +39,11 @@
   - match: { _id:      "1"     }
   - match: { _source:  { foo: "hello world" } }
 
-  - do:
-      cluster.state: {}
-
-  # Get master node id
-  - set: { master_node: master }
-
-  # relocate everything to the master node and make it read-only
+  # make it read-only
   - do:
       indices.put_settings:
         index: source
         body:
-          index.routing.allocation.include._id: $master
           index.blocks.write: true
           index.number_of_replicas: 0
 
@@ -47,8 +51,6 @@
       cluster.health:
         wait_for_status: green
         index: source
-        wait_for_no_relocating_shards: true
-        wait_for_events: "languid"
 
   # now we do the actual shrink
   - do:
@@ -56,6 +58,7 @@
         index: "source"
         target: "target"
         wait_for_active_shards: 1
+        master_timeout: 10s
         body:
           settings:
             index.number_of_replicas: 0
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/20_source_mapping.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/20_source_mapping.yml
new file mode 100644
index 0000000000000..6dcef57b3e1f4
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/20_source_mapping.yml
@@ -0,0 +1,77 @@
+---
+"Shrink index ignores target template mapping":
+    - skip:
+        version: " - 5.5.99"
+        reason: bug fixed in 5.6.0
+
+    - do:
+        cluster.state: {}
+        # Get master node id
+
+    - set: { master_node: master }
+
+  # create index
+    - do:
+        indices.create:
+          index: source
+          wait_for_active_shards: 1
+          body:
+            settings:
+              # ensure everything is allocated on a single node
+              index.routing.allocation.include._id: $master
+              number_of_replicas: 0
+            mappings:
+              test:
+                properties:
+                  count:
+                    type: text
+
+    # index document
+    - do:
+        index:
+          index: source
+          type:  test
+          id:    "1"
+          body:  { "count": "1" }
+
+    # create template matching shrink tagret
+    - do:
+        indices.put_template:
+          name: tpl1
+          body:
+            index_patterns: targ*
+            mappings:
+              test:
+                properties:
+                  count:
+                    type: integer
+
+    # make it read-only
+    - do:
+        indices.put_settings:
+          index: source
+          body:
+            index.blocks.write: true
+            index.number_of_replicas: 0
+
+    - do:
+        cluster.health:
+          wait_for_status: green
+          index: source
+
+    # now we do the actual shrink
+    - do:
+        indices.shrink:
+          index: "source"
+          target: "target"
+          wait_for_active_shards: 1
+          master_timeout: 10s
+          body:
+            settings:
+              index.number_of_replicas: 0
+
+    - do:
+        cluster.health:
+          wait_for_status: green
+
+
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/10_index.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/10_index.yaml
index 0710b63e5bbd1..a67e6ffd10552 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/10_index.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/10_index.yaml
@@ -100,3 +100,28 @@ setup:
   - is_false: indices.test1
   - is_true: indices.test2
 
+---
+"Indices stats unrecognized parameter":
+  - skip:
+      version: " - 5.0.99"
+      reason: strict stats handling does not exist in 5.0
+  - do:
+      catch: request
+      indices.stats:
+        metric: [ fieldata ]
+
+  - match: { status: 400 }
+  - match: { error.type: illegal_argument_exception }
+  - match: { error.reason: "request [/_stats/fieldata] contains unrecognized metric: [fieldata] -> did you mean [fielddata]?" }
+
+---
+"Indices stats warns on percolate":
+  - skip:
+      features: "warnings"
+      version: " - 5.2.99"
+      reason:  The warning was introduced in 5.1 and its format changed in 5.3.0
+  - do:
+      warnings:
+          - 'percolate stats are no longer available and requests for percolate stats will fail starting in 6.0.0'
+      indices.stats:
+        metric: [ percolate ]
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/11_metric.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/11_metric.yaml
index 7b88ac570806f..e030a27165d75 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/11_metric.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/11_metric.yaml
@@ -121,3 +121,52 @@ setup:
   - is_false:  _all.total.translog
   - is_true:   _all.total.recovery
 
+---
+"Metric - _all include_segment_file_sizes":
+  - skip:
+        version: " - 5.1.1"
+        reason: including segment file sizes triggered an unrecognized parameter in <= 5.1.1
+  - do:
+      indices.stats: { metric: _all, include_segment_file_sizes: true }
+
+  - is_true:   _all.total.docs
+  - is_true:   _all.total.store
+  - is_true:   _all.total.indexing
+  - is_true:   _all.total.get
+  - is_true:   _all.total.search
+  - is_true:   _all.total.merges
+  - is_true:   _all.total.refresh
+  - is_true:   _all.total.flush
+  - is_true:   _all.total.warmer
+  - is_true:   _all.total.query_cache
+  - is_true:   _all.total.fielddata
+  - is_true:   _all.total.completion
+  - is_true:   _all.total.segments
+  - is_true:   _all.total.translog
+  - is_true:   _all.total.recovery
+  - is_true:   _all.total.segments.file_sizes
+
+---
+"Metric - segments include_segment_file_sizes":
+  - skip:
+      version: " - 5.1.1"
+      reason: including segment file sizes triggered an unrecognized parameter in <= 5.1.1
+  - do:
+      indices.stats: { metric: segments, include_segment_file_sizes: true }
+
+  - is_false:  _all.total.docs
+  - is_false:  _all.total.store
+  - is_false:  _all.total.indexing
+  - is_false:  _all.total.get
+  - is_false:  _all.total.search
+  - is_false:  _all.total.merges
+  - is_false:  _all.total.refresh
+  - is_false:  _all.total.flush
+  - is_false:  _all.total.warmer
+  - is_false:  _all.total.query_cache
+  - is_false:  _all.total.fielddata
+  - is_false:  _all.total.completion
+  - is_true:   _all.total.segments
+  - is_false:  _all.total.translog
+  - is_false:  _all.total.recovery
+  - is_true:   _all.total.segments.file_sizes
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/13_fields.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/13_fields.yaml
index 7f9396b85657a..7be97cda1fe0c 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/13_fields.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/13_fields.yaml
@@ -38,22 +38,24 @@ setup:
       indices.refresh: {}
 
   - do:
-      suggest:
+      search:
         index: test1
         body:
-          result:
-            text: "b"
-            completion:
-              field: bar.completion
+          suggest:
+            result:
+              text: "b"
+              completion:
+                field: bar.completion
 
   - do:
-      suggest:
+      search:
         index: test1
         body:
-          result:
-            text: "b"
-            completion:
-              field: baz.completion
+          suggest:
+            result:
+              text: "b"
+              completion:
+                field: baz.completion
 
   - do:
       indices.refresh: {}
@@ -291,4 +293,3 @@ setup:
   - gt:       { _all.total.completion.fields.bar\.completion.size_in_bytes: 0 }
   - is_false:   _all.total.completion.fields.baz\.completion
   - is_false:   _all.total.fielddata.fields
-
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.update_aliases/30_remove_index_and_replace_with_alias.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.update_aliases/30_remove_index_and_replace_with_alias.yaml
index 14e258a6bb4d4..ff1c77a87db8b 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.update_aliases/30_remove_index_and_replace_with_alias.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.update_aliases/30_remove_index_and_replace_with_alias.yaml
@@ -1,5 +1,5 @@
 ---
-"Remove and index and replace it with an alias":
+"Remove an index and replace it with an alias":
 
   - do:
       indices.create:
@@ -31,4 +31,5 @@
   - do:
       indices.get_mapping:
         index: test
-  - is_true: test_2 # the name of the index that the alias points to, would be `test` if the index were still there
+    # the name of the index that the alias points to, would be `test` if the index were still there
+  - is_true: test_2
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.upgrade/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.upgrade/10_basic.yaml
index e696a5600bc2b..abb30eb970e59 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.upgrade/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.upgrade/10_basic.yaml
@@ -9,14 +9,80 @@
             index:
               number_of_replicas: 0
 
+  - do:
+      indices.upgrade:
+        index: test_index
+
+  - match: {upgraded_indices.test_index.oldest_lucene_segment_version: '/(\d\.)+\d/'}
+  - is_true: upgraded_indices.test_index.upgrade_version
+
+---
+"Upgrade indices ignore unavailable":
+  - skip:
+      version: " - 5.0.0"
+      reason: ignore_unavailable was added as a bugfix in 5.0.1 see #21281
 
   - do:
-      cluster.health:
-        wait_for_status: green
+      indices.create:
+        index: test_index
+        body:
+          settings:
+            index:
+              number_of_shards: 1
+              number_of_replicas: 0
 
   - do:
       indices.upgrade:
-        index: test_index
+        index: ["does_not_exist", "test_index"]
+        ignore_unavailable: true
 
-  - match: {upgraded_indices.test_index.oldest_lucene_segment_version: '/(\d\.)+\d/'}
+  - match: {_shards.total: 1}
   - is_true: upgraded_indices.test_index.upgrade_version
+  - is_false: upgraded_indices.does_not_exist
+
+---
+"Upgrade indices allow no indices":
+
+  - skip:
+      version: " - 5.0.0"
+      reason: ignore_unavailable was added as a bugfix in 5.0.1 see #21281
+
+  - do:
+      indices.upgrade:
+        index: test_index
+        ignore_unavailable: true
+        allow_no_indices: true
+
+  - match: {_shards.total: 0}
+
+---
+"Upgrade indices disallow no indices":
+
+  - skip:
+      version: " - 5.0.0"
+      reason: ignore_unavailable was added as a bugfix in 5.0.1 see #21281
+
+  - do:
+      catch: missing
+      indices.upgrade:
+        index: test_index
+        ignore_unavailable: true
+        allow_no_indices: false
+
+---
+"Upgrade indices disallow unavailable":
+
+  - skip:
+      version: " - 5.0.0"
+      reason: ignore_unavailable was added as a bugfix in 5.0.1 see #21281
+
+  - do:
+      indices.create:
+        index: test_index
+
+  - do:
+      catch: missing
+      indices.upgrade:
+        index: ["test_index", "does_not_exist"]
+        ignore_unavailable: false
+
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.validate_query/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.validate_query/10_basic.yaml
index a1f9aa876361c..29b1e664d61c0 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.validate_query/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.validate_query/10_basic.yaml
@@ -5,12 +5,24 @@ setup:
         body:
           settings:
             number_of_replicas: 0
+          aliases:
+            alias_1:
+              "filter" : { "match_all" : {} }
+
 
 ---
 "Validate query api":
   - do:
       indices.validate_query:
         q: query string
+        index: testing
+
+  - is_true: valid
+
+  - do:
+      indices.validate_query:
+        q: query string
+        index: alias_1
 
   - is_true: valid
 
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.validate_query/20_query_string.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.validate_query/20_query_string.yaml
index c6dd323aa6c98..3f96009c12a6c 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.validate_query/20_query_string.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.validate_query/20_query_string.yaml
@@ -49,15 +49,6 @@
       indices.validate_query:
         index: test
         q: field:BA*
-        lowercase_expanded_terms: false
-
-  - is_true: valid
-
-  - do:
-      indices.validate_query:
-        index: test
-        q: field:BA*
-        analyze_wildcard: true
 
   - is_true: valid
 
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/info/10_info.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/info/10_info.yaml
index d46ec7ee2ab71..d0c99ee0a7c5b 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/info/10_info.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/info/10_info.yaml
@@ -3,6 +3,7 @@
     - do:         {info: {}}
     - is_true:    name
     - is_true:    cluster_name
+    - is_true:    cluster_uuid
     - is_true:    tagline
     - is_true:    version
     - is_true:    version.number
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/12_non_existent_index.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/12_non_existent_index.yaml
index fd0100b0d635d..0623464225072 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/12_non_existent_index.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/12_non_existent_index.yaml
@@ -21,7 +21,6 @@
   - do:
       mget:
         body:
-          index:  test_2
           docs:
             - { _index: test_1, _type: test, _id: 1}
 
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/14_alias_to_multiple_indices.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/14_alias_to_multiple_indices.yaml
new file mode 100644
index 0000000000000..3a0fec04738b6
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/14_alias_to_multiple_indices.yaml
@@ -0,0 +1,42 @@
+---
+"Multi Get with alias that resolves to multiple indices":
+
+  - do:
+      bulk:
+        refresh: true
+        body: |
+          {"index": {"_index": "test_1", "_type": "test", "_id": 1}}
+          { "foo": "bar" }
+          {"index": {"_index": "test_2", "_type": "test", "_id": 2}}
+          { "foo": "bar" }
+          {"index": {"_index": "test_3", "_type": "test", "_id": 3}}
+          { "foo": "bar" }
+
+  - do:
+      indices.put_alias:
+        index: test_2
+        name:  test_two_and_three
+
+  - do:
+      indices.put_alias:
+        index: test_3
+        name:  test_two_and_three
+
+  - do:
+      mget:
+        body:
+          docs:
+            - { _index: test_1, _type: test, _id: 1}
+            - { _index: test_two_and_three, _type: test, _id: 2}
+
+  - is_true: docs.0.found
+  - match: { docs.0._index:     test_1      }
+  - match: { docs.0._type:      test        }
+  - match: { docs.0._id:        "1"         }
+
+  - is_false: docs.1.found
+  - match: { docs.1._index:     test_two_and_three      }
+  - match: { docs.1._type:      test                    }
+  - match: { docs.1._id:        "2"                     }
+  - match: { docs.1.error.root_cause.0.type: "illegal_argument_exception" }
+  - match: { docs.1.error.root_cause.0.reason: "/Alias.\\[test_two_and_three\\].has.more.than.one.indices.associated.with.it.\\[\\[test_[23]{1},.test_[23]{1}\\]\\],.can't.execute.a.single.index.op/" }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/20_fields.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/20_fields.yaml
deleted file mode 100644
index 2216c6540b848..0000000000000
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/20_fields.yaml
+++ /dev/null
@@ -1,119 +0,0 @@
----
-"Fields":
-
-  - do:
-      indices.create:
-        index: test_1
-        body:
-          mappings:
-            test:
-              properties:
-                foo:
-                  type: keyword
-                  store: true
-                count:
-                  type: integer
-                  store: true
-
-  - do:
-      index:
-          index:  test_1
-          type:   test
-          id:     1
-          body:   { foo: bar }
-
-  - do:
-      mget:
-        index: test_1
-        type:  test
-        body:
-          docs:
-            - { _id: 1                         }
-            - { _id: 1, fields: foo            }
-            - { _id: 1, fields: [foo]          }
-            - { _id: 1, fields: [foo, _source] }
-
-  - is_false: docs.0.fields
-  - match:  { docs.0._source: { foo: bar }}
-
-  - match:  { docs.1.fields.foo: [bar] }
-  - is_false: docs.1._source
-
-  - match:  { docs.2.fields.foo: [bar] }
-  - is_false: docs.2._source
-
-  - match:  { docs.3.fields.foo: [bar] }
-  - match:  { docs.3._source: { foo: bar }}
-
-  - do:
-      mget:
-        index: test_1
-        type:  test
-        fields: foo
-        body:
-          docs:
-            - { _id: 1                         }
-            - { _id: 1, fields: foo            }
-            - { _id: 1, fields: [foo]          }
-            - { _id: 1, fields: [foo, _source] }
-
-  - match:  { docs.0.fields.foo: [bar] }
-  - is_false: docs.0._source
-
-  - match:  { docs.1.fields.foo: [bar] }
-  - is_false: docs.1._source
-
-  - match:  { docs.2.fields.foo: [bar] }
-  - is_false: docs.2._source
-
-  - match:  { docs.3.fields.foo: [bar] }
-  - match:  { docs.3._source: { foo: bar }}
-
-  - do:
-      mget:
-        index:  test_1
-        type:   test
-        fields: [foo]
-        body:
-          docs:
-            - { _id: 1                         }
-            - { _id: 1, fields: foo            }
-            - { _id: 1, fields: [foo]          }
-            - { _id: 1, fields: [foo, _source] }
-
-  - match:  { docs.0.fields.foo: [bar] }
-  - is_false: docs.0._source
-
-  - match:  { docs.1.fields.foo: [bar] }
-  - is_false: docs.1._source
-
-  - match:  { docs.2.fields.foo: [bar] }
-  - is_false: docs.2._source
-
-  - match:  { docs.3.fields.foo: [bar] }
-  - match:  { docs.3._source: { foo: bar }}
-
-  - do:
-      mget:
-        index:  test_1
-        type:   test
-        fields: [foo, _source]
-        body:
-          docs:
-            - { _id: 1                         }
-            - { _id: 1, fields: foo            }
-            - { _id: 1, fields: [foo]          }
-            - { _id: 1, fields: [foo, _source] }
-
-  - match:  { docs.0.fields.foo: [bar] }
-  - match:  { docs.0._source: { foo: bar }}
-
-  - match:  { docs.1.fields.foo: [bar] }
-  - is_false: docs.1._source
-
-  - match:  { docs.2.fields.foo: [bar] }
-  - is_false: docs.2._source
-
-  - match:  { docs.3.fields.foo: [bar] }
-  - match:  { docs.3._source: { foo: bar }}
-
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/20_stored_fields.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/20_stored_fields.yaml
new file mode 100644
index 0000000000000..33cab111a8b3a
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/20_stored_fields.yaml
@@ -0,0 +1,119 @@
+---
+"Stored fields":
+
+  - do:
+      indices.create:
+        index: test_1
+        body:
+          mappings:
+            test:
+              properties:
+                foo:
+                  type: keyword
+                  store: true
+                count:
+                  type: integer
+                  store: true
+
+  - do:
+      index:
+          index:  test_1
+          type:   test
+          id:     1
+          body:   { foo: bar }
+
+  - do:
+      mget:
+        index: test_1
+        type:  test
+        body:
+          docs:
+            - { _id: 1                         }
+            - { _id: 1, stored_fields: foo            }
+            - { _id: 1, stored_fields: [foo]          }
+            - { _id: 1, stored_fields: [foo, _source] }
+
+  - is_false: docs.0.fields
+  - match:  { docs.0._source: { foo: bar }}
+
+  - match:  { docs.1.fields.foo: [bar] }
+  - is_false: docs.1._source
+
+  - match:  { docs.2.fields.foo: [bar] }
+  - is_false: docs.2._source
+
+  - match:  { docs.3.fields.foo: [bar] }
+  - match:  { docs.3._source: { foo: bar }}
+
+  - do:
+      mget:
+        index: test_1
+        type:  test
+        stored_fields: foo
+        body:
+          docs:
+            - { _id: 1                         }
+            - { _id: 1, stored_fields: foo            }
+            - { _id: 1, stored_fields: [foo]          }
+            - { _id: 1, stored_fields: [foo, _source] }
+
+  - match:  { docs.0.fields.foo: [bar] }
+  - is_false: docs.0._source
+
+  - match:  { docs.1.fields.foo: [bar] }
+  - is_false: docs.1._source
+
+  - match:  { docs.2.fields.foo: [bar] }
+  - is_false: docs.2._source
+
+  - match:  { docs.3.fields.foo: [bar] }
+  - match:  { docs.3._source: { foo: bar }}
+
+  - do:
+      mget:
+        index:  test_1
+        type:   test
+        stored_fields: [foo]
+        body:
+          docs:
+            - { _id: 1                         }
+            - { _id: 1, stored_fields: foo            }
+            - { _id: 1, stored_fields: [foo]          }
+            - { _id: 1, stored_fields: [foo, _source] }
+
+  - match:  { docs.0.fields.foo: [bar] }
+  - is_false: docs.0._source
+
+  - match:  { docs.1.fields.foo: [bar] }
+  - is_false: docs.1._source
+
+  - match:  { docs.2.fields.foo: [bar] }
+  - is_false: docs.2._source
+
+  - match:  { docs.3.fields.foo: [bar] }
+  - match:  { docs.3._source: { foo: bar }}
+
+  - do:
+      mget:
+        index:  test_1
+        type:   test
+        stored_fields: [foo, _source]
+        body:
+          docs:
+            - { _id: 1                         }
+            - { _id: 1, stored_fields: foo            }
+            - { _id: 1, stored_fields: [foo]          }
+            - { _id: 1, stored_fields: [foo, _source] }
+
+  - match:  { docs.0.fields.foo: [bar] }
+  - match:  { docs.0._source: { foo: bar }}
+
+  - match:  { docs.1.fields.foo: [bar] }
+  - is_false: docs.1._source
+
+  - match:  { docs.2.fields.foo: [bar] }
+  - is_false: docs.2._source
+
+  - match:  { docs.3.fields.foo: [bar] }
+  - match:  { docs.3._source: { foo: bar }}
+
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/30_parent.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/30_parent.yaml
index e1cf8df4fc34b..ad064df69ce55 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/30_parent.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/30_parent.yaml
@@ -34,9 +34,9 @@
         body:
           docs:
             - { _id: 1 }
-            - { _id: 1, parent: 5, fields: [ _parent, _routing ] }
-            - { _id: 1, parent: 4, fields: [ _parent, _routing ] }
-            - { _id: 2, parent: 5, fields: [ _parent, _routing ] }
+            - { _id: 1, parent: 5, stored_fields: [ _parent, _routing ] }
+            - { _id: 1, parent: 4, stored_fields: [ _parent, _routing ] }
+            - { _id: 2, parent: 5, stored_fields: [ _parent, _routing ] }
 
  - is_false:  docs.0.found
  - is_false:  docs.1.found
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/40_routing.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/40_routing.yaml
index 71ac0feabbeaa..7196412ebf3f6 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/40_routing.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/40_routing.yaml
@@ -26,7 +26,7 @@
       mget:
           index:   test_1
           type:    test
-          fields:  [_routing]
+          stored_fields:  [_routing]
           body:
             docs:
               - { _id: 1 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/55_parent_with_routing.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/55_parent_with_routing.yaml
index 19b597675cfe5..1fa1ce2cddb9a 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/55_parent_with_routing.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/55_parent_with_routing.yaml
@@ -29,7 +29,7 @@
       mget:
         index:  test_1
         type:   test
-        fields:  [ _routing , _parent]
+        stored_fields:  [ _routing , _parent]
         body:
           docs:
             - { _id: 1 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/60_realtime_refresh.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/60_realtime_refresh.yaml
index c688b4b0e93a2..0cb7b71cf4368 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/60_realtime_refresh.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/60_realtime_refresh.yaml
@@ -25,7 +25,7 @@
       mget:
           index:    test_1
           type:     test
-          realtime: 0
+          realtime: false
           body:
             ids:    [1]
 
@@ -35,7 +35,7 @@
       mget:
           index:    test_1
           type:     test
-          realtime: 1
+          realtime: true
           body:
             ids:    [1]
 
@@ -45,8 +45,8 @@
       mget:
           index:    test_1
           type:     test
-          realtime: 0
-          refresh:  1
+          realtime: false
+          refresh:  true
           body:
             ids:    [1]
 
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/10_basic.yaml
index 6e6b0cd3b4f81..536e2bfaf9495 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/10_basic.yaml
@@ -61,4 +61,3 @@ setup:
   - match:  { responses.3.error.root_cause.0.reason: "/no.such.index/" }
   - match:  { responses.3.error.root_cause.0.index: index_3 }
   - match:  { responses.4.hits.total:     4  }
-
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/20_typed_keys.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/20_typed_keys.yaml
new file mode 100644
index 0000000000000..1efb65d0f613f
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/20_typed_keys.yaml
@@ -0,0 +1,96 @@
+---
+setup:
+  - skip:
+      version: " - 5.3.99"
+      reason: typed_keys parameter was added in 5.4.0
+
+  - do:
+      indices.put_template:
+        name: index_template
+        body:
+          template: test-*
+          settings:
+            number_of_replicas: 0
+          mappings:
+            user:
+              properties:
+                index_start_at:
+                   type: integer
+                integer:
+                   type: integer
+                float:
+                   type: float
+                name:
+                   type: keyword
+                title:
+                   type: completion
+  - do:
+     bulk:
+        refresh: true
+        body:
+          - '{"index": {"_index": "test-0", "_type": "user"}}'
+          - '{"row": 1, "index_start_at": 56, "integer": 38, "float": 12.5713, "name": "Ruth", "bool": true, "title": "doctor"}'
+          - '{"index": {"_index": "test-0", "_type": "user"}}'
+          - '{"row": 2, "index_start_at": 57, "integer": 42, "float": 15.3393, "name": "Jackie", "surname": "Bowling", "bool": false}'
+          - '{"index": {"_index": "test-1", "_type": "user"}}'
+          - '{"row": 3, "index_start_at": 58, "integer": 29, "float": 19.0517, "name": "Stephanie", "bool": true}'
+          - '{"index": {"_index": "test-1", "_type": "user"}}'
+          - '{"row": 4, "index_start_at": 59, "integer": 19, "float": 19.3717, "surname": "Hamilton", "bool": true, "title": "commandant"}'
+          - '{"index": {"_index": "test-2", "_type": "user"}}'
+          - '{"row": 5, "index_start_at": 60, "integer": 0, "float": 17.3349, "name": "Natalie", "bool": false}'
+
+---
+"Multisearch test with typed_keys parameter":
+  - do:
+      msearch:
+        typed_keys: true
+        body:
+          # Testing aggegrations
+          - index: test-*
+          - {query: {match: {bool: true} }, size: 0, aggs: {test_filter: {filter: {range: {integer: {gte: 20} } } } } }
+          - index: test-1
+          - {query: {match_all: {} }, size: 0, aggs: {test_range: {range: {field: float, ranges: [ {to: 19.2499999}, {from: 19.25} ] } } } }
+          - index: test-*
+          - {query: {bool: {filter: {range: {row: {lt: 5}}} } }, size: 0, aggs: {test_percentiles: {percentiles: {field: float} } } }
+          # Testing suggesters
+          - index: test-*
+          - {query: {match_all: {} }, size: 0, suggest: {term_suggester: {text: Natalie, term: {field: name } } } }
+          - index: test-*
+          - {query: {match_all: {} }, size: 0, suggest: {completion_suggester: {prefix: doc, completion: {field: title } } } }
+          - index: test-*
+          - {query: {match_all: {} }, size: 0, suggest: {phrase_suggester: {text: Ruht, phrase: {field: name } } } }
+
+  - match:    { responses.0.hits.total: 3 }
+  - match:    { responses.0.aggregations.filter#test_filter.doc_count : 2 }
+  - match:    { responses.1.hits.total: 2 }
+  - match:    { responses.1.aggregations.range#test_range.buckets.0.key : "*-19.2499999" }
+  - match:    { responses.1.aggregations.range#test_range.buckets.0.doc_count : 1 }
+  - match:    { responses.1.aggregations.range#test_range.buckets.1.key : "19.25-*" }
+  - match:    { responses.1.aggregations.range#test_range.buckets.1.doc_count : 1 }
+  - match:    { responses.2.hits.total: 4 }
+  - is_true:  responses.2.aggregations.tdigest_percentiles#test_percentiles.values
+  - is_true:  responses.3.suggest.term#term_suggester
+  - is_true:  responses.4.suggest.completion#completion_suggester
+  - is_true:  responses.5.suggest.phrase#phrase_suggester
+
+---
+"Multisearch test with typed_keys parameter for sampler and significant terms":
+  - do:
+      msearch:
+        typed_keys: true
+        body:
+          - index: test-*
+          - {query: {match_all: {} }, size: 0, aggs: {test_sampler: {sampler: {shard_size: 200}, aggs: {test_significant_terms: {significant_terms: {field:  name} } } } } }
+          - index: test-*
+          - {query: {match_all: {} }, size: 0, aggs: {test_umterms: {terms: {field: surname} } } }
+          - index: test-*
+          - {query: {match_all: {} }, size: 0, aggs: {test_sterms: {terms: {field: name}, aggs: {test_umsignificant_terms: {significant_terms: {field:  surname} } } } } }
+
+  - match:    { responses.0.hits.total: 5 }
+  - match:    { responses.0.aggregations.sampler#test_sampler.doc_count : 5 }
+  - match:    { responses.0.aggregations.sampler#test_sampler.sigsterms#test_significant_terms.doc_count : 5 }
+  - match:    { responses.1.hits.total: 1 }
+  - match:    { responses.1.aggregations.sterms#test_umterms.doc_count_error_upper_bound : 0 }
+  - match:    { responses.2.hits.total: 1 }
+  - match:    { responses.2.aggregations.sterms#test_sterms.doc_count_error_upper_bound : 0 }
+  - is_true:  responses.2.aggregations.sterms#test_sterms.buckets.0.sigsterms#test_umsignificant_terms
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/nodes.stats/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/nodes.stats/10_basic.yaml
index 977d002748a6d..dcc1138e539b1 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/nodes.stats/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/nodes.stats/10_basic.yaml
@@ -6,3 +6,44 @@
 
   - is_true: cluster_name
   - is_true: nodes
+
+---
+"Nodes stats level":
+  - do:
+      cluster.state: {}
+
+  - set: { master_node: master }
+
+  - do:
+      nodes.stats:
+        metric: [ indices ]
+        level: "indices"
+
+  - is_true: nodes.$master.indices.indices
+
+---
+"Nodes stats unrecognized parameter":
+  - skip:
+      version: " - 5.0.99"
+      reason: strict stats handling does not exist in 5.0
+  - do:
+      catch: request
+      nodes.stats:
+        metric: [ transprot ]
+
+  - match: { status: 400 }
+  - match: { error.type: illegal_argument_exception }
+  - match: { error.reason: "request [/_nodes/stats/transprot] contains unrecognized metric: [transprot] -> did you mean [transport]?" }
+
+---
+"Node stats warns on percolate":
+  - skip:
+      features: "warnings"
+      version: " - 5.2.99"
+      reason:  The warning was introduced in 5.1 and its format changed in 5.3.0
+  - do:
+      warnings:
+          - 'percolate stats are no longer available and requests for percolate stats will fail starting in 6.0.0'
+      nodes.stats:
+        metric: [ indices ]
+        index_metric: [ percolate ]
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yaml
new file mode 100644
index 0000000000000..ab6e7da2345c7
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yaml
@@ -0,0 +1,216 @@
+---
+"Metric - blank":
+  - do:
+      cluster.state: {}
+
+  - set: { master_node: master }
+
+  - do:
+      nodes.stats: {}
+
+  - is_true:  nodes.$master.indices.docs
+  - is_true:  nodes.$master.indices.store
+  - is_true:  nodes.$master.indices.indexing
+  - is_true:  nodes.$master.indices.get
+  - is_true:  nodes.$master.indices.search
+  - is_true:  nodes.$master.indices.merges
+  - is_true:  nodes.$master.indices.refresh
+  - is_true:  nodes.$master.indices.flush
+  - is_true:  nodes.$master.indices.warmer
+  - is_true:  nodes.$master.indices.query_cache
+  - is_true:  nodes.$master.indices.fielddata
+  - is_true:  nodes.$master.indices.completion
+  - is_true:  nodes.$master.indices.segments
+  - is_true:  nodes.$master.indices.translog
+  - is_true:  nodes.$master.indices.recovery
+
+---
+"Metric - _all":
+  - do:
+      cluster.state: {}
+
+  - set: { master_node: master }
+
+  - do:
+      nodes.stats: { metric: _all }
+
+  - is_true:  nodes.$master.indices.docs
+  - is_true:  nodes.$master.indices.store
+  - is_true:  nodes.$master.indices.indexing
+  - is_true:  nodes.$master.indices.get
+  - is_true:  nodes.$master.indices.search
+  - is_true:  nodes.$master.indices.merges
+  - is_true:  nodes.$master.indices.refresh
+  - is_true:  nodes.$master.indices.flush
+  - is_true:  nodes.$master.indices.warmer
+  - is_true:  nodes.$master.indices.query_cache
+  - is_true:  nodes.$master.indices.fielddata
+  - is_true:  nodes.$master.indices.completion
+  - is_true:  nodes.$master.indices.segments
+  - is_true:  nodes.$master.indices.translog
+  - is_true:  nodes.$master.indices.recovery
+
+---
+"Metric - indices _all":
+  - do:
+      cluster.state: {}
+
+  - set: { master_node: master }
+
+  - do:
+      nodes.stats: { metric: indices, index_metric: _all }
+
+  - is_true:  nodes.$master.indices.docs
+  - is_true:  nodes.$master.indices.store
+  - is_true:  nodes.$master.indices.indexing
+  - is_true:  nodes.$master.indices.get
+  - is_true:  nodes.$master.indices.search
+  - is_true:  nodes.$master.indices.merges
+  - is_true:  nodes.$master.indices.refresh
+  - is_true:  nodes.$master.indices.flush
+  - is_true:  nodes.$master.indices.warmer
+  - is_true:  nodes.$master.indices.query_cache
+  - is_true:  nodes.$master.indices.fielddata
+  - is_true:  nodes.$master.indices.completion
+  - is_true:  nodes.$master.indices.segments
+  - is_true:  nodes.$master.indices.translog
+  - is_true:  nodes.$master.indices.recovery
+
+---
+"Metric - one":
+  - do:
+      cluster.state: {}
+
+  - set: { master_node: master }
+
+  - do:
+      nodes.stats: { metric: indices, index_metric: docs }
+
+  - is_true:   nodes.$master.indices.docs
+  - is_false:  nodes.$master.indices.store
+  - is_false:  nodes.$master.indices.indexing
+  - is_false:  nodes.$master.indices.get
+  - is_false:  nodes.$master.indices.search
+  - is_false:  nodes.$master.indices.merges
+  - is_false:  nodes.$master.indices.refresh
+  - is_false:  nodes.$master.indices.flush
+  - is_false:  nodes.$master.indices.warmer
+  - is_false:  nodes.$master.indices.query_cache
+  - is_false:  nodes.$master.indices.fielddata
+  - is_false:  nodes.$master.indices.completion
+  - is_false:  nodes.$master.indices.segments
+  - is_false:  nodes.$master.indices.translog
+  - is_false:  nodes.$master.indices.recovery
+
+---
+"Metric - multi":
+  - do:
+      cluster.state: {}
+
+  - set: { master_node: master }
+
+  - do:
+      nodes.stats: { metric: indices, index_metric: [ store, get, merge ] }
+
+  - is_false:  nodes.$master.indices.docs
+  - is_true:   nodes.$master.indices.store
+  - is_false:  nodes.$master.indices.indexing
+  - is_true:   nodes.$master.indices.get
+  - is_false:  nodes.$master.indices.search
+  - is_true:   nodes.$master.indices.merges
+  - is_false:  nodes.$master.indices.refresh
+  - is_false:  nodes.$master.indices.flush
+  - is_false:  nodes.$master.indices.warmer
+  - is_false:  nodes.$master.indices.query_cache
+  - is_false:  nodes.$master.indices.fielddata
+  - is_false:  nodes.$master.indices.completion
+  - is_false:  nodes.$master.indices.segments
+  - is_false:  nodes.$master.indices.translog
+  - is_false:  nodes.$master.indices.recovery
+
+
+---
+"Metric - recovery":
+  - do:
+      cluster.state: {}
+
+  - set: { master_node: master }
+
+  - do:
+      nodes.stats: { metric: indices, index_metric: [ recovery ] }
+
+  - is_false:  nodes.$master.indices.docs
+  - is_false:  nodes.$master.indices.store
+  - is_false:  nodes.$master.indices.indexing
+  - is_false:  nodes.$master.indices.get
+  - is_false:  nodes.$master.indices.search
+  - is_false:  nodes.$master.indices.merges
+  - is_false:  nodes.$master.indices.refresh
+  - is_false:  nodes.$master.indices.flush
+  - is_false:  nodes.$master.indices.warmer
+  - is_false:  nodes.$master.indices.query_cache
+  - is_false:  nodes.$master.indices.fielddata
+  - is_false:  nodes.$master.indices.completion
+  - is_false:  nodes.$master.indices.segments
+  - is_false:  nodes.$master.indices.translog
+  - is_true:   nodes.$master.indices.recovery
+
+---
+"Metric - _all include_segment_file_sizes":
+  - skip:
+      version: " - 5.1.1"
+      reason: including segment file sizes triggered an unrecognized parameter in <= 5.1.1
+  - do:
+      cluster.state: {}
+
+  - set: { master_node: master }
+
+  - do:
+      nodes.stats: { metric: indices, index_metric: _all, include_segment_file_sizes: true }
+
+  - is_true:   nodes.$master.indices.docs
+  - is_true:   nodes.$master.indices.store
+  - is_true:   nodes.$master.indices.indexing
+  - is_true:   nodes.$master.indices.get
+  - is_true:   nodes.$master.indices.search
+  - is_true:   nodes.$master.indices.merges
+  - is_true:   nodes.$master.indices.refresh
+  - is_true:   nodes.$master.indices.flush
+  - is_true:   nodes.$master.indices.warmer
+  - is_true:   nodes.$master.indices.query_cache
+  - is_true:   nodes.$master.indices.fielddata
+  - is_true:   nodes.$master.indices.completion
+  - is_true:   nodes.$master.indices.segments
+  - is_true:   nodes.$master.indices.translog
+  - is_true:   nodes.$master.indices.recovery
+  - is_true:   nodes.$master.indices.segments.file_sizes
+
+---
+"Metric - segments include_segment_file_sizes":
+  - skip:
+      version: " - 5.1.1"
+      reason: including segment file sizes triggered an unrecognized parameter in <= 5.1.1
+  - do:
+      cluster.state: {}
+
+  - set: { master_node: master }
+
+  - do:
+      nodes.stats: { metric: indices, index_metric: segments, include_segment_file_sizes: true }
+
+  - is_false:  nodes.$master.indices.docs
+  - is_false:  nodes.$master.indices.store
+  - is_false:  nodes.$master.indices.indexing
+  - is_false:  nodes.$master.indices.get
+  - is_false:  nodes.$master.indices.search
+  - is_false:  nodes.$master.indices.merges
+  - is_false:  nodes.$master.indices.refresh
+  - is_false:  nodes.$master.indices.flush
+  - is_false:  nodes.$master.indices.warmer
+  - is_false:  nodes.$master.indices.query_cache
+  - is_false:  nodes.$master.indices.fielddata
+  - is_false:  nodes.$master.indices.completion
+  - is_true:   nodes.$master.indices.segments
+  - is_false:  nodes.$master.indices.translog
+  - is_false:  nodes.$master.indices.recovery
+  - is_true:   nodes.$master.indices.segments.file_sizes
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/remote.info/10_info.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/remote.info/10_info.yaml
new file mode 100644
index 0000000000000..1c6ed4811d870
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/remote.info/10_info.yaml
@@ -0,0 +1,9 @@
+---
+"Get an empty remote info":
+  - skip:
+      version: " - 5.3.99"
+      reason: this API doesn't exist in 5.3.x yet
+  - do:
+      remote.info: {}
+  - is_true: ''
+
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/10_basic.yaml
index 4013f315d4633..0f037b890fc4b 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/10_basic.yaml
@@ -65,6 +65,78 @@
       clear_scroll:
         scroll_id: $scroll_id
 
+---
+"Basic scroll with 1 shard":
+  - do:
+      indices.create:
+        index: test_scroll
+        body:
+          settings:
+            index:
+              number_of_shards: 1
+
+  - do:
+      index:
+        index:  test_scroll
+        type:   test
+        id:     42
+        body:   { foo: 1 }
+
+  - do:
+      index:
+        index:  test_scroll
+        type:   test
+        id:     43
+        body:   { foo: 2 }
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        index: test_scroll
+        size: 1
+        scroll: 1m
+        sort: foo
+        body:
+          query:
+            match_all: {}
+
+  - set: {_scroll_id: scroll_id}
+  - match: {hits.total:      2    }
+  - length: {hits.hits:      1    }
+  - match: {hits.hits.0._id: "42" }
+
+  - do:
+      index:
+        index:  test_scroll
+        type:   test
+        id:     44
+        body:   { foo: 3 }
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      scroll:
+        body: { "scroll_id": "$scroll_id", "scroll": "1m"}
+
+  - match: {hits.total:      2    }
+  - length: {hits.hits:      1    }
+  - match: {hits.hits.0._id: "43" }
+
+  - do:
+      scroll:
+        scroll_id: $scroll_id
+        scroll: 1m
+
+  - match: {hits.total:      2    }
+  - length: {hits.hits:      0    }
+
+  - do:
+      clear_scroll:
+        scroll_id: $scroll_id
+
 ---
 "Body params override query string":
   - do:
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/11_clear.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/11_clear.yaml
index aa4885825d2ad..c2a026df1d744 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/11_clear.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/11_clear.yaml
@@ -38,7 +38,7 @@
           scroll_id: $scroll_id1
 
 ---
-"Body params override query string":
+"Body params with array param override query string":
   - do:
       indices.create:
           index:  test_scroll
@@ -76,3 +76,47 @@
         catch: missing
         clear_scroll:
           scroll_id: $scroll_id1
+
+---
+"Body params with string param scroll id override query string":
+  - skip:
+      version: " - 5.99.99"
+      reason:  this uses a new API that has been added in 6.0
+
+  - do:
+      indices.create:
+          index:  test_scroll
+  - do:
+      index:
+          index:  test_scroll
+          type:   test
+          id:     42
+          body:   { foo: bar }
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        index: test_scroll
+        scroll: 1m
+        body:
+          query:
+            match_all: {}
+
+  - set: {_scroll_id: scroll_id1}
+
+  - do:
+      clear_scroll:
+        scroll_id: "invalid_scroll_id"
+        body: { "scroll_id": "$scroll_id1" }
+
+  - do:
+      catch: missing
+      scroll:
+        scroll_id: $scroll_id1
+
+  - do:
+        catch: missing
+        clear_scroll:
+          scroll_id: $scroll_id1
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/12_slices.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/12_slices.yaml
index 5443059135add..ac66af0095e2d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/12_slices.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/12_slices.yaml
@@ -1,5 +1,5 @@
 ---
-"Sliced scroll":
+setup:
   - do:
       indices.create:
         index: test_sliced_scroll
@@ -8,32 +8,101 @@
       index:
         index:  test_sliced_scroll
         type:   test
-        id:     42
+        id:     1
         body:   { foo: 1 }
 
+  - do:
+      index:
+        index:  test_sliced_scroll
+        type:   test
+        id:     2
+        body:   { foo: 2 }
+
+  - do:
+      index:
+        index:  test_sliced_scroll
+        type:   test
+        id:     3
+        body:   { foo: 3 }
+
+  - do:
+      index:
+        index:  test_sliced_scroll
+        type:   test
+        id:     4
+        body:   { foo: 4 }
+
   - do:
       indices.refresh: {}
 
+---
+"Sliced scroll":
+  - skip:
+      version: " - 5.3.0"
+      reason: Prior version uses a random seed per node to compute the hash of the keys.
+
   - do:
       search:
         index: test_sliced_scroll
-        size: 1
         scroll: 1m
         sort: foo
         body:
-          slice: {
-            id: 0,
-            max: 3
-          }
+          slice:
+            id: 0
+            max: 2
           query:
             match_all: {}
 
   - set: {_scroll_id: scroll_id}
+  - match: {hits.total:      3    }
+  - length: {hits.hits:      3    }
+  - match: {hits.hits.0._id: "2" }
+  - match: {hits.hits.1._id: "3" }
+  - match: {hits.hits.2._id: "4" }
+
+  - do:
+      scroll:
+        scroll_id: $scroll_id
+        scroll: 1m
+
+  - match: {hits.total:      3    }
+  - length: {hits.hits:      0    }
+
+  - do:
+      clear_scroll:
+        scroll_id: $scroll_id
+
+  - do:
+      search:
+        index: test_sliced_scroll
+        scroll: 1m
+        sort: foo
+        body:
+          slice:
+            id: 1
+            max: 2
+          query:
+            match_all: {}
+
+  - set: {_scroll_id: scroll_id}
+  - match: {hits.total:      1    }
+  - length: {hits.hits:      1    }
+  - match: {hits.hits.0._id: "1" }
+
+  - do:
+      scroll:
+        scroll_id: $scroll_id
+        scroll: 1m
+
+  - match: {hits.total:      1    }
+  - length: {hits.hits:      0    }
 
   - do:
         clear_scroll:
           scroll_id: $scroll_id
 
+---
+"Sliced scroll with invalid arguments":
   - do:
         catch:  /query_phase_execution_exception.*The number of slices.*index.max_slices_per_scroll/
         search:
@@ -41,10 +110,9 @@
           size: 1
           scroll: 1m
           body:
-            slice: {
-              id: 0,
+            slice:
+              id: 0
               max: 1025
-            }
             query:
               match_all: {}
 
@@ -60,10 +128,9 @@
           size: 1
           scroll: 1m
           body:
-            slice: {
-              id: 0,
+            slice:
+              id: 0
               max: 1025
-            }
             query:
               match_all: {}
 
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/20_terms.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/20_terms.yaml
index c35e79e6cfe38..55d8809d7911b 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/20_terms.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/20_terms.yaml
@@ -18,6 +18,9 @@ setup:
                     type: long
                   double:
                     type: double
+                  scaled_float:
+                    type: scaled_float
+                    scaling_factor: 100
                   date:
                     type: date
 
@@ -47,30 +50,30 @@ setup:
         type: test
         id: 3
         body: { "str": "bcd" }
-      
+
   - do:
       indices.refresh: {}
-    
+
   - do:
       search:
         body: { "size" : 0, "aggs" : { "str_terms" : { "terms" : { "field" : "str" } } } }
 
   - match: { hits.total: 3 }
-  
+
   - length: { aggregations.str_terms.buckets: 2 }
-  
+
   - match: { aggregations.str_terms.buckets.0.key: "abc" }
-  
+
   - is_false: aggregations.str_terms.buckets.0.key_as_string
-  
+
   - match: { aggregations.str_terms.buckets.0.doc_count: 2 }
-  
+
   - match: { aggregations.str_terms.buckets.1.key: "bcd" }
-  
+
   - is_false: aggregations.str_terms.buckets.1.key_as_string
-  
+
   - match: { aggregations.str_terms.buckets.1.doc_count: 1 }
-  
+
 ---
 "IP test":
   - do:
@@ -112,9 +115,9 @@ setup:
   - match: { aggregations.ip_terms.buckets.0.doc_count: 2 }
 
   - match: { aggregations.ip_terms.buckets.1.key: "127.0.0.1" }
-  
+
   - is_false: aggregations.ip_terms.buckets.1.key_as_string
-  
+
   - match: { aggregations.ip_terms.buckets.1.doc_count: 1 }
 
   - do:
@@ -142,7 +145,7 @@ setup:
       search:
         body: { "size" : 0, "aggs" : { "ip_terms" : { "terms" : { "field" : "ip", "exclude" :  "127.*"  } } } }
 
-  
+
 
 ---
 "Boolean test":
@@ -282,6 +285,56 @@ setup:
 
   - match: { aggregations.double_terms.buckets.1.doc_count: 1 }
 
+---
+"Scaled float test":
+  - skip:
+      version: " - 5.2.0"
+      reason:  scaled_float were considered as longs in aggregations, this was fixed in 5.2.0
+
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 1
+        body: { "scaled_float": 9.99 }
+
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 2
+        body: { "scaled_float": 9.994 }
+
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 3
+        body: { "scaled_float": 8.99 }
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        body: { "size" : 0, "aggs" : { "scaled_float_terms" : { "terms" : { "field" : "scaled_float" } } } }
+
+  - match: { hits.total: 3 }
+
+  - length: { aggregations.scaled_float_terms.buckets: 2 }
+
+  - match: { aggregations.scaled_float_terms.buckets.0.key: 9.99 }
+
+  - is_false: aggregations.scaled_float_terms.buckets.0.key_as_string
+
+  - match: { aggregations.scaled_float_terms.buckets.0.doc_count: 2 }
+
+  - match: { aggregations.scaled_float_terms.buckets.1.key: 8.99 }
+
+  - is_false: aggregations.scaled_float_terms.buckets.1.key_as_string
+
+  - match: { aggregations.scaled_float_terms.buckets.1.doc_count: 1 }
+
 ---
 "Date test":
   - do:
@@ -327,7 +380,7 @@ setup:
   - match: { aggregations.date_terms.buckets.1.key_as_string: "2014-09-01T00:00:00.000Z" }
 
   - match: { aggregations.date_terms.buckets.1.doc_count: 1 }
-  
+
   - do:
       search:
         body: { "size" : 0, "aggs" : { "date_terms" : { "terms" : { "field" : "date", "include" : [ "2016-05-03" ] } } } }
@@ -335,11 +388,11 @@ setup:
   - match: { hits.total: 3 }
 
   - length: { aggregations.date_terms.buckets: 1 }
-  
+
   - match: { aggregations.date_terms.buckets.0.key_as_string: "2016-05-03T00:00:00.000Z" }
-  
-  - match: { aggregations.date_terms.buckets.0.doc_count: 2 }  
-  
+
+  - match: { aggregations.date_terms.buckets.0.doc_count: 2 }
+
   - do:
       search:
         body: { "size" : 0, "aggs" : { "date_terms" : { "terms" : { "field" : "date", "exclude" : [ "2016-05-03" ] } } } }
@@ -347,7 +400,318 @@ setup:
   - match: { hits.total: 3 }
 
   - length: { aggregations.date_terms.buckets: 1 }
-  
+
   - match: { aggregations.date_terms.buckets.0.key_as_string: "2014-09-01T00:00:00.000Z" }
 
-  - match: { aggregations.date_terms.buckets.0.doc_count: 1 }  
+  - match: { aggregations.date_terms.buckets.0.doc_count: 1 }
+
+---
+"Partitioned string test":
+
+  - skip:
+      version: " - 5.1.99"
+      reason:  Partitioning is a 5.2.0 feature
+
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 1
+        body: { "str" : "abc" }
+
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 2
+        body: { "str": "abc" }
+
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 3
+        body: { "str": "bcd" }
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        body: { "size" : 0, "aggs" : { "str_terms" : { "terms" : { "field" : "str", "include" : {"partition": 0, "num_partitions": 2 } } } } }
+
+  - match: { hits.total : 3 }
+
+  - length: { aggregations.str_terms.buckets: 1 }
+
+  - match: { aggregations.str_terms.buckets.0.key: "bcd" }
+
+  - is_false: aggregations.str_terms.buckets.0.key_as_string
+
+  - match: { aggregations.str_terms.buckets.0.doc_count: 1 }
+
+  - do:
+      search:
+        body: { "size" : 0, "aggs" : { "str_terms" : { "terms" : { "field" : "str", "include" : {"partition": 1, "num_partitions": 2 } } } } }
+
+  - match: { hits.total: 3 }
+
+  - length: { aggregations.str_terms.buckets: 1 }
+
+  - match: { aggregations.str_terms.buckets.0.key: "abc" }
+
+  - is_false: aggregations.str_terms.buckets.0.key_as_string
+
+  - match: { aggregations.str_terms.buckets.0.doc_count: 2 }
+
+---
+"Partitioned integer test":
+
+  - skip:
+      version: " - 5.1.99"
+      reason:  Partitioning is a 5.2.0 feature
+
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 1
+        body: { "integer": 1234 }
+
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 2
+        body: { "integer": 5678 }
+
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 3
+        body: { "integer": 1234 }
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        body: { "size" : 0, "aggs" : { "str_terms" : { "terms" : { "field" : "integer", "include" : {"partition": 0, "num_partitions": 2 } } } } }
+
+  - match: { hits.total: 3 }
+
+  - length: { aggregations.str_terms.buckets: 1 }
+
+  - match: { aggregations.str_terms.buckets.0.key: 5678 }
+
+  - match: { aggregations.str_terms.buckets.0.doc_count: 1 }
+
+  - do:
+      search:
+        body: { "size" : 0, "aggs" : { "str_terms" : { "terms" : { "field" : "integer", "include" : {"partition": 1, "num_partitions": 2 } } } } }
+
+  - match: { hits.total: 3 }
+
+  - length: { aggregations.str_terms.buckets: 1 }
+
+  - match: { aggregations.str_terms.buckets.0.key: 1234 }
+
+  - match: { aggregations.str_terms.buckets.0.doc_count: 2 }
+
+---
+"Unmapped strings":
+
+  - skip:
+      version: " - 5.1.99"
+      reason: Unmapped fields handling with value_type was added in 5.2
+
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 1
+        body: {}
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        body: { "size" : 0, "aggs" : { "string_terms" : { "terms" : { "field" : "unmapped_string", "value_type" : "string", "missing": "abc" } } } }
+
+  - match: { hits.total: 1 }
+
+  - length: { aggregations.string_terms.buckets: 1 }
+
+  - match: { aggregations.string_terms.buckets.0.key: "abc" }
+
+  - match: { aggregations.string_terms.buckets.0.doc_count: 1 }
+
+---
+"Unmapped booleans":
+
+  - skip:
+      version: " - 5.1.99"
+      reason: Unmapped fields handling with value_type was added in 5.2
+
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 1
+        body: {}
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        body: { "size" : 0, "aggs" : { "boolean_terms" : { "terms" : { "field" : "unmapped_boolean", "value_type" : "boolean", "missing": true } } } }
+
+  - match: { hits.total: 1 }
+
+  - length: { aggregations.boolean_terms.buckets: 1 }
+
+  - match: { aggregations.boolean_terms.buckets.0.key: 1 }
+
+  - match: { aggregations.boolean_terms.buckets.0.key_as_string: "true" }
+
+  - match: { aggregations.boolean_terms.buckets.0.doc_count: 1 }
+
+---
+"Unmapped dates":
+
+  - skip:
+      version: " - 5.1.99"
+      reason: Unmapped fields handling with value_type was added in 5.2
+
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 1
+        body: {}
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        body: { "size" : 0, "aggs" : { "date_terms" : { "terms" : { "field" : "unmapped_date", "value_type" : "date", "missing": "2016-05-11" } } } }
+
+  - match: { hits.total: 1 }
+
+  - length: { aggregations.date_terms.buckets: 1 }
+
+  - match: { aggregations.date_terms.buckets.0.key: 1462924800000 }
+
+  - match: { aggregations.date_terms.buckets.0.key_as_string: "2016-05-11T00:00:00.000Z" }
+
+  - match: { aggregations.date_terms.buckets.0.doc_count: 1 }
+
+---
+"Unmapped longs":
+
+  - skip:
+      version: " - 5.1.99"
+      reason: Unmapped fields handling with value_type was added in 5.2
+
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 1
+        body: {}
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        body: { "size" : 0, "aggs" : { "long_terms" : { "terms" : { "field" : "unmapped_long", "value_type" : "long", "missing": 3 } } } }
+
+  - match: { hits.total: 1 }
+
+  - length: { aggregations.long_terms.buckets: 1 }
+
+  - match: { aggregations.long_terms.buckets.0.key: 3 }
+
+  - match: { aggregations.long_terms.buckets.0.doc_count: 1 }
+
+---
+"Unmapped doubles":
+
+  - skip:
+      version: " - 5.1.99"
+      reason: Unmapped fields handling with value_type was added in 5.2
+
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 1
+        body: {}
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        body: { "size" : 0, "aggs" : { "double_terms" : { "terms" : { "field" : "unmapped_double", "value_type" : "double", "missing": 3.5 } } } }
+
+  - match: { hits.total: 1 }
+
+  - length: { aggregations.double_terms.buckets: 1 }
+
+  - match: { aggregations.double_terms.buckets.0.key: 3.5 }
+
+  - match: { aggregations.double_terms.buckets.0.doc_count: 1 }
+
+---
+"Pattern include/exclude":
+
+  - skip:
+      features: "warnings"
+      version: " - 5.2.99"
+      reason:  text was deprecated in 5.3.0
+
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 1
+        body: { "str" : "abc" }
+
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 2
+        body: { "str": "bcd" }
+
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 3
+        body: { "str": "cde" }
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      warnings:
+        - "Deprecated field [pattern] used, replaced by [Put patterns directly under the [include] or [exclude]]"
+      search:
+        body: { "size" : 0, "aggs" : { "str_terms" : { "terms" : { "field" : "str", "include" : {"pattern": ".*d.*"}, "exclude": { "pattern": ".*e.*" } } } } }
+
+  - match: { hits.total: 3 }
+
+  - length: { aggregations.str_terms.buckets: 1 }
+
+  - match: { aggregations.str_terms.buckets.0.key: "bcd" }
+
+  - is_false: aggregations.str_terms.buckets.0.key_as_string
+
+  - match: { aggregations.str_terms.buckets.0.doc_count: 1 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/30_sig_terms.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/30_sig_terms.yaml
index a708ff19d7e34..6301d668a2379 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/30_sig_terms.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/30_sig_terms.yaml
@@ -145,4 +145,70 @@
       search:
         body: { "size" : 0, "aggs" : { "ip_terms" : { "significant_terms" : { "field" : "ip", "exclude" :  "127.*"  } } } }
   
- 
+---
+"Pattern include/exclude":
+
+  - skip:
+      features: "warnings"
+      version: " - 5.2.99"
+      reason:  include/pattern was deprecated in 5.3.0
+
+  - do:
+      indices.create:
+          index:  test_1
+          body:
+            settings:
+                number_of_shards: "1"
+            mappings:
+                test:
+                    properties:
+                        str:
+                            type: keyword
+
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 1
+        body: { "str" : "abc" }
+
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 2
+        body: { "str": "bcd" }
+
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 3
+        body: { "str": "cde" }
+
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 4
+        body: { }
+
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      warnings:
+        - "Deprecated field [pattern] used, replaced by [Put patterns directly under the [include] or [exclude]]"
+      search:
+        body: { "size" : 0, "query" : { "exists" : { "field" : "str" } }, "aggs" : { "str_terms" : { "significant_terms" : { "field" : "str", "min_doc_count": 1, "include" : {"pattern": ".*d.*"}, "exclude": { "pattern": ".*e.*" } } } } }
+
+  - match: { hits.total: 3 }
+
+  - length: { aggregations.str_terms.buckets: 1 }
+
+  - match: { aggregations.str_terms.buckets.0.key: "bcd" }
+
+  - is_false: aggregations.str_terms.buckets.0.key_as_string
+
+  - match: { aggregations.str_terms.buckets.0.doc_count: 1 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/40_range.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/40_range.yaml
new file mode 100644
index 0000000000000..c058721124fc3
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/40_range.yaml
@@ -0,0 +1,227 @@
+setup:
+  - do:
+      indices.create:
+          index: test
+          body:
+            settings:
+              number_of_replicas: 0
+            mappings:
+              test:
+                properties:
+                  ip:
+                     type: ip
+                  double:
+                    type: double
+                  date:
+                    type: date
+
+  - do:
+      cluster.health:
+        wait_for_status: green
+
+---
+"Double range":
+  - do:
+      index:
+        index: test
+        type: test
+        id: 1
+        body: { "double" : 42 }
+
+  - do:
+      index:
+        index: test
+        type: test
+        id: 2
+        body: { "double" : 100 }
+
+  - do:
+      index:
+        index: test
+        type: test
+        id: 3
+        body: { "double" : 50 }
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        body: { "size" : 0, "aggs" : { "double_range" : { "range" : { "field" : "double", "ranges": [ { "to": 50 }, { "from": 50, "to": 150 }, { "from": 150 } ] } } } }
+
+  - match: { hits.total: 3 }
+
+  - length: { aggregations.double_range.buckets: 3 }
+
+  - match: { aggregations.double_range.buckets.0.key: "*-50.0" }
+
+  - is_false: aggregations.double_range.buckets.0.from
+
+  - match: { aggregations.double_range.buckets.0.to: 50.0 }
+
+  - match: { aggregations.double_range.buckets.0.doc_count: 1 }
+
+  - match: { aggregations.double_range.buckets.1.key: "50.0-150.0" }
+
+  - match: { aggregations.double_range.buckets.1.from: 50.0 }
+
+  - match: { aggregations.double_range.buckets.1.to: 150.0 }
+
+  - match: { aggregations.double_range.buckets.1.doc_count: 2 }
+
+  - match: { aggregations.double_range.buckets.2.key: "150.0-*" }
+
+  - match: { aggregations.double_range.buckets.2.from: 150.0 }
+
+  - is_false:  aggregations.double_range.buckets.2.to
+
+  - match: { aggregations.double_range.buckets.2.doc_count: 0 }
+
+  - do:
+      search:
+        body: { "size" : 0, "aggs" : { "double_range" : { "range" : { "field" : "double", "ranges": [ { "from": null, "to": 50 }, { "from": 50, "to": 150 }, { "from": 150, "to": null } ] } } } }
+
+  - match: { hits.total: 3 }
+
+  - length: { aggregations.double_range.buckets: 3 }
+
+  - match: { aggregations.double_range.buckets.0.key: "*-50.0" }
+
+  - is_false: aggregations.double_range.buckets.0.from
+
+  - match: { aggregations.double_range.buckets.0.to: 50.0 }
+
+  - match: { aggregations.double_range.buckets.0.doc_count: 1 }
+
+  - match: { aggregations.double_range.buckets.1.key: "50.0-150.0" }
+
+  - match: { aggregations.double_range.buckets.1.from: 50.0 }
+
+  - match: { aggregations.double_range.buckets.1.to: 150.0 }
+
+  - match: { aggregations.double_range.buckets.1.doc_count: 2 }
+
+  - match: { aggregations.double_range.buckets.2.key: "150.0-*" }
+
+  - match: { aggregations.double_range.buckets.2.from: 150.0 }
+
+  - is_false:  aggregations.double_range.buckets.2.to
+
+  - match: { aggregations.double_range.buckets.2.doc_count: 0 }
+
+---
+"IP range":
+  - skip:
+      version: " - 5.1.1"
+      reason: IP range queries had an exclusive range bug prior to 5.1.2
+  - do:
+      index:
+        index: test
+        type: test
+        id: 1
+        body: { "ip" : "::1" }
+
+  - do:
+      index:
+        index: test
+        type: test
+        id: 2
+        body: { "ip" : "192.168.0.1" }
+
+  - do:
+      index:
+        index: test
+        type: test
+        id: 3
+        body: { "ip" : "192.168.0.7" }
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        body: { "size" : 0, "aggs" : { "ip_range" : { "ip_range" : { "field" : "ip", "ranges": [ { "to": "192.168.0.0" }, { "from": "192.168.0.0", "to": "192.169.0.0" }, { "from": "192.169.0.0" } ] } } } }
+
+  - match: { hits.total: 3 }
+
+  - length: { aggregations.ip_range.buckets: 3 }
+
+# ip_range does not automatically add keys to buckets, see #21045
+#  - match: { aggregations.ip_range.buckets.0.key: "*-192.168.0.0" }
+
+  - is_false: aggregations.ip_range.buckets.0.from
+
+  - match: { aggregations.ip_range.buckets.0.to: "192.168.0.0" }
+
+  - match: { aggregations.ip_range.buckets.0.doc_count: 1 }
+
+#  - match: { aggregations.ip_range.buckets.1.key: "192.168.0.0-192.169.0.0" }
+
+  - match: { aggregations.ip_range.buckets.1.from: "192.168.0.0" }
+
+  - match: { aggregations.ip_range.buckets.1.to: "192.169.0.0" }
+
+  - match: { aggregations.ip_range.buckets.1.doc_count: 2 }
+
+#  - match: { aggregations.ip_range.buckets.2.key: "192.169.0.0-*" }
+
+  - match: { aggregations.ip_range.buckets.2.from: "192.169.0.0" }
+
+  - is_false:  aggregations.ip_range.buckets.2.to
+
+  - match: { aggregations.ip_range.buckets.2.doc_count: 0 }
+
+  - do:
+      search:
+        body: { "size" : 0, "aggs" : { "ip_range" : { "ip_range" : { "field" : "ip", "ranges": [ { "from": null, "to": "192.168.0.0" }, { "from": "192.168.0.0", "to": "192.169.0.0" }, { "from": "192.169.0.0", "to": null } ] } } } }
+
+  - match: { hits.total: 3 }
+
+  - length: { aggregations.ip_range.buckets: 3 }
+
+#  - match: { aggregations.ip_range.buckets.0.key: "*-192.168.0.0" }
+
+  - is_false: aggregations.ip_range.buckets.0.from
+
+  - match: { aggregations.ip_range.buckets.0.to: "192.168.0.0" }
+
+  - match: { aggregations.ip_range.buckets.0.doc_count: 1 }
+
+#  - match: { aggregations.ip_range.buckets.1.key: "192.168.0.0-192.169.0.0" }
+
+  - match: { aggregations.ip_range.buckets.1.from: "192.168.0.0" }
+
+  - match: { aggregations.ip_range.buckets.1.to: "192.169.0.0" }
+
+  - match: { aggregations.ip_range.buckets.1.doc_count: 2 }
+
+#  - match: { aggregations.ip_range.buckets.2.key: "192.169.0.0-*" }
+
+  - match: { aggregations.ip_range.buckets.2.from: "192.169.0.0" }
+
+  - is_false:  aggregations.ip_range.buckets.2.to
+
+  - match: { aggregations.ip_range.buckets.2.doc_count: 0 }
+
+  - do:
+      search:
+        body: { "size" : 0, "aggs" : { "ip_range" : { "ip_range" : { "field" : "ip", "ranges": [ { "mask": "::/24" }, { "mask": "192.168.0.0/16" } ] } } } }
+
+  - match: { hits.total: 3 }
+
+  - length: { aggregations.ip_range.buckets: 2 } 
+
+  - match: { aggregations.ip_range.buckets.0.key: "::/24" }
+
+  - match: { aggregations.ip_range.buckets.0.to: "0:100::" }
+
+  - match: { aggregations.ip_range.buckets.0.doc_count: 3 } 
+
+  - match: { aggregations.ip_range.buckets.1.key: "192.168.0.0/16" }
+
+  - match: { aggregations.ip_range.buckets.1.from: "192.168.0.0" }
+
+  - match: { aggregations.ip_range.buckets.1.to: "192.169.0.0" }
+
+  - match: { aggregations.ip_range.buckets.1.doc_count: 2 } 
+
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/50_filter.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/50_filter.yaml
new file mode 100644
index 0000000000000..63cca00509cd7
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/50_filter.yaml
@@ -0,0 +1,80 @@
+setup:
+  - do:
+      indices.create:
+          index: test
+          body:
+            settings:
+              number_of_shards: 1
+              number_of_replicas: 0
+            mappings:
+              post:
+                properties:
+                  mentions:
+                    type: keyword
+              user:
+                properties:
+                  notifications:
+                    type: keyword
+
+  - do:
+        index:
+          index: test
+          type: test
+          id: foo|bar|baz0
+          body: { "notifications" : ["abc"] }
+
+  - do:
+      index:
+        index: test
+        type: test
+        id: foo|bar|baz1
+        body: { "mentions" : ["abc"] }
+
+  - do:
+      indices.refresh: {}
+
+---
+"Filter aggs with terms lookup ensure not cached":
+  - skip:
+      version: " - 5.0.0"
+      reason:  This using filter aggs that needs rewriting, this was fixed in 5.0.1
+
+  - do:
+      search:
+        size: 0
+        request_cache: true
+        body: {"aggs": { "itemsNotify": { "filter": { "terms": { "mentions": { "index": "test", "type": "test", "id": "foo|bar|baz0", "path": "notifications"}}}, "aggs": { "mentions" : {"terms" : { "field" : "mentions" }}}}}}
+
+  # validate result
+  - match: { hits.total: 2 }
+  - match: { aggregations.itemsNotify.doc_count: 1 }
+  - length: { aggregations.itemsNotify.mentions.buckets: 1 }
+  - match: { aggregations.itemsNotify.mentions.buckets.0.key: "abc" }
+  # we are using a lookup - this should not cache
+  - do:
+      indices.stats: { index: test, metric: request_cache}
+  - match: { _shards.total: 1 }
+  - match: { _all.total.request_cache.hit_count: 0 }
+  - match: { _all.total.request_cache.miss_count: 0 }
+  - is_true: indices.test
+
+---
+"Filter aggs no lookup and ensure it's cached":
+  # now run without lookup and ensure we get cached or at least do the lookup
+  - do:
+      search:
+        size: 0
+        request_cache: true
+        body: {"aggs": { "itemsNotify": { "filter": { "terms": { "mentions": ["abc"]}}, "aggs": { "mentions" : {"terms" : { "field" : "mentions" }}}}}}
+
+  - match: { hits.total: 2 }
+  - match: { aggregations.itemsNotify.doc_count: 1 }
+  - length: { aggregations.itemsNotify.mentions.buckets: 1 }
+  - match: { aggregations.itemsNotify.mentions.buckets.0.key: "abc" }
+  - do:
+      indices.stats: { index: test, metric: request_cache}
+  - match: { _shards.total: 1 }
+  - match: { _all.total.request_cache.hit_count: 0 }
+  - match: { _all.total.request_cache.miss_count: 1 }
+  - is_true: indices.test
+
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/70_adjacency_matrix.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/70_adjacency_matrix.yaml
new file mode 100644
index 0000000000000..143ff9e3d1300
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/70_adjacency_matrix.yaml
@@ -0,0 +1,66 @@
+setup:
+  - do:
+      indices.create:
+          index: test
+          body:
+            settings:
+              number_of_shards: 1
+              number_of_replicas: 0
+            mappings:
+              post:
+                properties:
+                  num:
+                    type: integer
+
+  - do:
+        index:
+          index: test
+          type: test
+          id: 1
+          body: { "num": [1, 2] }
+
+  - do:
+      index:
+        index: test
+        type: test
+        id: 2
+        body: { "num": [2, 3] }
+
+  - do:
+      index:
+        index: test
+        type: test
+        id: 3
+        body: { "num": [3, 4] }
+
+  - do:
+      indices.refresh: {}
+
+
+---
+"Filters intersections":
+
+  - skip:
+      version: " - 5.2.99"
+      reason:  Adjacency Matrix is a 5.3.0 feature
+      
+  - do:
+      search:
+        body: { "size": 0, "aggs": { "conns": { "adjacency_matrix": {  "filters": { "1": { "term": { "num": 1 } }, "2": { "term": { "num": 2 } }, "4": { "term": { "num": 4 } } } } } } }
+
+  - match: { hits.total: 3 }
+
+  - length: { aggregations.conns.buckets: 4 }
+
+  - match: { aggregations.conns.buckets.0.doc_count: 1 }
+  - match: { aggregations.conns.buckets.0.key: "1" }
+
+  - match: { aggregations.conns.buckets.1.doc_count: 1 }
+  - match: { aggregations.conns.buckets.1.key: "1&2" }
+
+  - match: { aggregations.conns.buckets.2.doc_count: 2 }
+  - match: { aggregations.conns.buckets.2.key: "2" }
+
+  - match: { aggregations.conns.buckets.3.doc_count: 1 }
+  - match: { aggregations.conns.buckets.3.key: "4" }
+
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/80_typed_keys.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/80_typed_keys.yaml
new file mode 100644
index 0000000000000..9ea856c856e93
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/80_typed_keys.yaml
@@ -0,0 +1,226 @@
+setup:
+  - skip:
+      version: " - 5.3.99"
+      reason: typed_keys parameter was added in 5.4.0
+
+  - do:
+      indices.create:
+          index: test
+          body:
+            settings:
+              number_of_replicas: 0
+            mappings:
+              test:
+                properties:
+                  name:
+                     type: keyword
+                  num:
+                     type: integer
+                  created:
+                    type: date
+
+  - do:
+     bulk:
+        refresh: true
+        index: test
+        type: test
+        body:
+          - '{"index": {}}'
+          - '{"name": "one", "num": 1, "created": "2010-03-12T01:07:45"}'
+          - '{"index": {}}'
+          - '{"name": "two", "num": 2, "created": "2010-03-12T04:11:00"}'
+          - '{"index": {}}'
+          - '{"name": "three", "num": 3, "created": "2010-04-27T03:43:34"}'
+
+---
+"Test typed keys parameter for avg aggregation":
+  - do:
+      search:
+        typed_keys: true
+        body:
+          size: 0
+          aggregations:
+            test_avg:
+              avg:
+                field: num
+  - is_true: aggregations.avg#test_avg
+
+---
+"Test typed keys parameter for cardinality aggregation":
+  - do:
+      search:
+        typed_keys: true
+        body:
+          size: 0
+          aggregations:
+            test_cardinality:
+              cardinality:
+                field: name
+  - is_true: aggregations.cardinality#test_cardinality
+
+---
+"Test typed keys parameter for extended_stats aggregation":
+  - do:
+      search:
+        typed_keys: true
+        body:
+          size: 0
+          aggregations:
+            test_extended_stats:
+              extended_stats:
+                field: num
+  - is_true: aggregations.extended_stats#test_extended_stats
+
+---
+"Test typed keys parameter for max aggregation":
+  - do:
+      search:
+        typed_keys: true
+        body:
+          size: 0
+          aggregations:
+            test_max:
+              max:
+                field: num
+  - is_true: aggregations.max#test_max
+
+---
+"Test typed keys parameter for min aggregation":
+  - do:
+      search:
+        typed_keys: true
+        body:
+          size: 0
+          aggregations:
+            test_min:
+              min:
+                field: num
+  - is_true: aggregations.min#test_min
+
+---
+"Test typed keys parameter for percentiles aggregation":
+  - do:
+      search:
+        typed_keys: true
+        body:
+          size: 0
+          aggregations:
+            test_percentiles:
+              percentiles:
+                field: num
+  - is_true: aggregations.tdigest_percentiles#test_percentiles
+
+---
+"Test typed keys parameter for percentile_ranks aggregation":
+  - do:
+      search:
+        typed_keys: true
+        body:
+          size: 0
+          aggregations:
+            test_percentile_ranks:
+              percentile_ranks:
+                field: num
+                values: [0,10]
+  - is_true: aggregations.tdigest_percentile_ranks#test_percentile_ranks
+
+---
+"Test typed keys parameter for stats aggregation":
+  - do:
+      search:
+        typed_keys: true
+        body:
+          size: 0
+          aggregations:
+            test_stats:
+              stats:
+                field: num
+  - is_true: aggregations.stats#test_stats
+
+---
+"Test typed keys parameter for sum aggregation":
+  - do:
+      search:
+        typed_keys: true
+        body:
+          size: 0
+          aggregations:
+            test_sum:
+              sum:
+                field: num
+  - is_true: aggregations.sum#test_sum
+
+---
+"Test typed keys parameter for terms and top_hits aggregation":
+  - do:
+      search:
+        typed_keys: true
+        body:
+          size: 0
+          aggregations:
+            test_terms:
+              terms:
+                field: name
+              aggs:
+                test_top_hits:
+                  top_hits:
+                    sort: num
+  - is_true: aggregations.sterms#test_terms
+  - is_true: aggregations.sterms#test_terms.buckets.0.top_hits#test_top_hits
+  - is_true: aggregations.sterms#test_terms.buckets.1.top_hits#test_top_hits
+  - is_true: aggregations.sterms#test_terms.buckets.2.top_hits#test_top_hits
+
+---
+"Test typed keys parameter for terms aggregation":
+  - do:
+      search:
+        typed_keys: true
+        body:
+          size: 0
+          aggregations:
+            test_terms:
+              terms:
+                field: num
+  - is_true: aggregations.lterms#test_terms
+
+---
+"Test typed keys parameter for value_count aggregation":
+  - do:
+      search:
+        typed_keys: true
+        body:
+          size: 0
+          aggregations:
+            test_value_count:
+              value_count:
+                field: num
+  - is_true: aggregations.value_count#test_value_count
+
+---
+"Test typed keys parameter for date_histogram aggregation and max_bucket pipeline aggregation":
+  - do:
+      search:
+        typed_keys: true
+        body:
+          size: 0
+          aggregations:
+            test_created_histogram:
+              date_histogram:
+                field: created
+                interval: month
+              aggregations:
+                test_sum:
+                  sum:
+                    field: num
+                test_moving_avg:
+                  moving_avg:
+                    buckets_path: "test_sum"
+            test_max_bucket:
+              max_bucket:
+                buckets_path: "test_created_histogram>test_sum"
+
+  - is_true: aggregations.date_histogram#test_created_histogram
+  - is_true: aggregations.date_histogram#test_created_histogram.buckets.0.sum#test_sum
+  - is_true: aggregations.date_histogram#test_created_histogram.buckets.1.sum#test_sum
+  - is_true: aggregations.date_histogram#test_created_histogram.buckets.1.simple_value#test_moving_avg
+  - is_true: aggregations.bucket_metric_value#test_max_bucket
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/10_unified.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/10_unified.yaml
new file mode 100644
index 0000000000000..72f782e68d124
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/10_unified.yaml
@@ -0,0 +1,39 @@
+setup:
+  - do:
+      indices.create:
+          index: test
+          body:
+            mappings:
+              unified:
+                "properties":
+                  "text":
+                     "type": "text"
+                     "fields":
+                        "fvh":
+                          "type": "text"
+                          "term_vector": "with_positions_offsets"
+                        "postings":
+                          "type": "text"
+                          "index_options": "offsets"
+  - do:
+      index:
+        index: test
+        type:  unified
+        id:    1
+        body:
+            "text" : "The quick brown fox is brown."
+  - do:
+      indices.refresh: {}
+
+---
+"Basic":
+  - skip:
+      version: " - 5.2.99"
+      reason:  this uses a new highlighter that has been added in 5.3
+  - do:
+      search:
+        body: { "query" : {"multi_match" : { "query" : "quick brown fox", "fields" : [ "text*"] } }, "highlight" : { "type" : "unified", "fields" : { "*" : {} } } }
+
+  - match: {hits.hits.0.highlight.text.0: "The <em>quick</em> <em>brown</em> <em>fox</em> is <em>brown</em>."}
+  - match: {hits.hits.0.highlight.text\.fvh.0: "The <em>quick</em> <em>brown</em> <em>fox</em> is <em>brown</em>."}
+  - match: {hits.hits.0.highlight.text\.postings.0: "The <em>quick</em> <em>brown</em> <em>fox</em> is <em>brown</em>."}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/20_postings.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/20_postings.yaml
new file mode 100644
index 0000000000000..57714aa3723e6
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/20_postings.yaml
@@ -0,0 +1,53 @@
+setup:
+  - do:
+      indices.create:
+          index: test
+          body:
+            mappings:
+              doc:
+                "properties":
+                  "text":
+                     "type": "text"
+                     "index_options": "offsets"
+  - do:
+      index:
+        index: test
+        type:  doc
+        id:    1
+        body:
+            "text" : "The quick brown fox is brown."
+
+  - do:
+      index:
+        index: test
+        type:  doc
+        id:    2
+        body:
+            "text" : "The quick brown fox is brown."
+
+  - do:
+      indices.refresh: {}
+
+---
+"Postings highlighter should have deprecation warning":
+  - skip:
+      features: 'warnings'
+      version: ' - 5.6.99'
+      reason: deprecated in 5.6 (set to 5.7 to disable test , see https://github.com/elastic/elasticsearch/issues/25169)
+  - do:
+      warnings:
+        - "[postings] highlighter is deprecated, please use [unified] instead"
+      search:
+        body: { "query" : {"match" : { "text" : "quick brown fox" } }, "highlight" : { "fields" : { "*" : {} } } }
+
+  - match: {hits.hits.0.highlight.text.0: "The <em>quick</em> <em>brown</em> <em>fox</em> is <em>brown</em>."}
+  - match: {hits.hits.1.highlight.text.0: "The <em>quick</em> <em>brown</em> <em>fox</em> is <em>brown</em>."}
+
+  - do:
+      warnings:
+        - "[postings] highlighter is deprecated, please use [unified] instead"
+      search:
+        body: { "query" : {"match" : { "text" : "quick brown fox" } }, "highlight" : { "type" : "postings", "fields" : { "*" : {} } } }
+
+  - match: {hits.hits.0.highlight.text.0: "The <em>quick</em> <em>brown</em> <em>fox</em> is <em>brown</em>."}
+  - match: {hits.hits.1.highlight.text.0: "The <em>quick</em> <em>brown</em> <em>fox</em> is <em>brown</em>."}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/30_fvh.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/30_fvh.yml
new file mode 100644
index 0000000000000..d4cb980a05cc9
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/30_fvh.yml
@@ -0,0 +1,49 @@
+setup:
+  - do:
+      indices.create:
+          index: test
+          body:
+            mappings:
+              doc:
+                "properties":
+                  "title":
+                     "type": "text"
+                     "term_vector": "with_positions_offsets"
+                  "description":
+                     "type": "text"
+                     "term_vector": "with_positions_offsets"
+  - do:
+      index:
+        index: test
+        type:  doc
+        id:    1
+        body:
+            "title" : "The quick brown fox is brown"
+            "description" : "The quick pink panther is pink"
+  - do:
+      indices.refresh: {}
+
+---
+"Highlight query":
+  - skip:
+      version: " - 5.5.99"
+      reason:  bug fixed in 5.6
+  - do:
+      search:
+        body:
+         highlight:
+          type: fvh
+          fields:
+            description:
+              type: fvh
+              highlight_query:
+                prefix:
+                  description: br
+            title:
+              type: fvh
+              highlight_query:
+                prefix:
+                  title: br
+
+  - match: {hits.hits.0.highlight.title.0: "The quick <em>brown</em> fox is <em>brown</em>"}
+  - is_false: hits.hits.0.highlight.description
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yaml
index 98e61dd9fa9a9..034a13ae72255 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yaml
@@ -5,19 +5,10 @@ setup:
         index: test
         body:
             mappings:
-              type_1: {
-                properties: {
-                  nested_field : {
+              type_1:
+                properties:
+                  nested_field:
                     type: nested
-                  }
-                }
-              }
-              type_2: {}
-              type_3: {
-                _parent: {
-                  type: type_2
-                }
-              }
 
 ---
 "Nested inner hits":
@@ -26,13 +17,8 @@ setup:
           index: test
           type:  type_1
           id:    1
-          body:  {
-            "nested_field" : [
-              {
-                "foo": "bar"
-              }
-            ]
-          }
+          body:
+            "nested_field" : [ { "foo": "bar" } ]
 
     - do:
         indices.refresh: {}
@@ -51,34 +37,3 @@ setup:
     - match: { hits.hits.0.inner_hits.nested_field.hits.hits.0._nested.offset: 0 }
     - is_false: hits.hits.0.inner_hits.nested_field.hits.hits.0._nested.child
 
----
-"Parent/child inner hits":
-    - do:
-        index:
-          index: test
-          type:  type_2
-          id:    1
-          body:  {"foo": "bar"}
-
-    - do:
-        index:
-          index: test
-          type:  type_3
-          id:    1
-          parent: 1
-          body:  {"bar": "baz"}
-
-    - do:
-        indices.refresh: {}
-
-    - do:
-        search:
-          body: { "query" : { "has_child" : { "type" : "type_3", "query" : { "match_all" : {} }, "inner_hits" : {} } } }
-    - match: { hits.total: 1 }
-    - match: { hits.hits.0._index: "test" }
-    - match: { hits.hits.0._type: "type_2" }
-    - match: { hits.hits.0._id: "1" }
-    - is_false: hits.hits.0.inner_hits.type_3.hits.hits.0._index
-    - match: { hits.hits.0.inner_hits.type_3.hits.hits.0._type: "type_3" }
-    - match: { hits.hits.0.inner_hits.type_3.hits.hits.0._id: "1" }
-    - is_false: hits.hits.0.inner_hits.type_3.hits.hits.0._nested
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yaml
new file mode 100644
index 0000000000000..5ea597436be4f
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yaml
@@ -0,0 +1,326 @@
+setup:
+  - do:
+      indices.create:
+          index:  test
+  - do:
+      index:
+          index:  test
+          type:   test
+          id:     1
+          body:   { numeric_group: 1, sort: 10 }
+  - do:
+      index:
+          index:  test
+          type:   test
+          id:     2
+          body:   { numeric_group: 1, sort: 6 }
+  - do:
+      index:
+          index:  test
+          type:   test
+          id:     3
+          body:   { numeric_group: 1, sort: 24 }
+  - do:
+      index:
+          index:  test
+          type:   test
+          id:     4
+          body:   { numeric_group: 25, sort: 10 }
+  - do:
+      index:
+          index:  test
+          type:   test
+          id:     5
+          body:   { numeric_group: 25, sort: 5 }
+  - do:
+      index:
+          index:  test
+          type:   test
+          id:     6
+          body:   { numeric_group: 3, sort: 36 }
+  - do:
+      indices.refresh:
+        index: test
+
+---
+"field collapsing":
+
+  - skip:
+      version: " - 5.2.99"
+      reason:  this uses a new API that has been added in 5.3
+
+  - do:
+      search:
+        index: test
+        type:  test
+        body:
+          collapse: { field: numeric_group }
+          sort: [{ sort: desc }]
+
+  - match: {hits.total: 6 }
+  - length: {hits.hits: 3 }
+  - match: {hits.hits.0._index: test }
+  - match: {hits.hits.0._type: test }
+  - match: {hits.hits.0.fields.numeric_group: [3] }
+  - match: {hits.hits.0.sort: [36] }
+  - match: {hits.hits.0._id: "6" }
+  - is_false: hits.hits.0.inner_hits
+  - match: {hits.hits.1._index: test }
+  - match: {hits.hits.1._type: test }
+  - match: {hits.hits.1.fields.numeric_group: [1] }
+  - match: {hits.hits.1.sort: [24] }
+  - match: {hits.hits.1._id: "3" }
+  - is_false: hits.hits.1.inner_hits
+  - match: {hits.hits.2._index: test }
+  - match: {hits.hits.2._type: test }
+  - match: {hits.hits.2.fields.numeric_group: [25] }
+  - match: {hits.hits.2.sort: [10] }
+  - match: {hits.hits.2._id: "4" }
+  - is_false: hits.hits.2.inner_hits
+
+---
+"field collapsing and from":
+
+  - skip:
+      version: " - 5.2.99"
+      reason:  this uses a new API that has been added in 5.3
+
+  - do:
+      search:
+        index: test
+        type:  test
+        body:
+          from: 2
+          collapse: { field: numeric_group }
+          sort: [{ sort: desc }]
+
+  - match: {hits.total: 6 }
+  - length: {hits.hits: 1 }
+  - match: {hits.hits.0._index: test }
+  - match: {hits.hits.0._type: test }
+  - match: {hits.hits.0.fields.numeric_group: [25]}
+  - match: {hits.hits.0.sort: [10] }
+  - match: {hits.hits.0._id: "4" }
+  - is_false: hits.hits.0.inner_hits
+
+---
+"field collapsing and inner_hits":
+
+  - skip:
+      version: " - 5.2.99"
+      reason:  this uses a new API that has been added in 5.3
+
+  - do:
+      search:
+        index: test
+        type:  test
+        body:
+          collapse: { field: numeric_group, inner_hits: { name: sub_hits, size: 2, sort: [{ sort: asc }] } }
+          sort: [{ sort: desc }]
+
+  - match: { hits.total: 6 }
+  - length: { hits.hits: 3 }
+  - match: { hits.hits.0._index: test }
+  - match: { hits.hits.0._type: test }
+  - match: { hits.hits.0.fields.numeric_group: [3] }
+  - match: { hits.hits.0.sort: [36] }
+  - match: { hits.hits.0._id: "6" }
+  - match: { hits.hits.0.inner_hits.sub_hits.hits.total: 1 }
+  - length: { hits.hits.0.inner_hits.sub_hits.hits.hits: 1 }
+  - match: { hits.hits.0.inner_hits.sub_hits.hits.hits.0._id: "6" }
+  - match: { hits.hits.1._index: test }
+  - match: { hits.hits.1._type: test }
+  - match: { hits.hits.1.fields.numeric_group: [1] }
+  - match: { hits.hits.1.sort: [24] }
+  - match: { hits.hits.1._id: "3" }
+  - match: { hits.hits.1.inner_hits.sub_hits.hits.total: 3 }
+  - length: { hits.hits.1.inner_hits.sub_hits.hits.hits: 2 }
+  - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.0._id: "2" }
+  - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.1._id: "1" }
+  - match: { hits.hits.2._index: test }
+  - match: { hits.hits.2._type: test }
+  - match: { hits.hits.2.fields.numeric_group: [25] }
+  - match: { hits.hits.2.sort: [10] }
+  - match: { hits.hits.2._id: "4" }
+  - match: { hits.hits.2.inner_hits.sub_hits.hits.total: 2 }
+  - length: { hits.hits.2.inner_hits.sub_hits.hits.hits: 2 }
+  - match: { hits.hits.2.inner_hits.sub_hits.hits.hits.0._id: "5" }
+  - match: { hits.hits.2.inner_hits.sub_hits.hits.hits.1._id: "4" }
+
+---
+"field collapsing, inner_hits and maxConcurrentGroupRequests":
+
+  - skip:
+      version: " - 5.2.99"
+      reason:  this uses a new API that has been added in 5.3
+
+  - do:
+      search:
+        index: test
+        type:  test
+        body:
+          collapse: { field: numeric_group, max_concurrent_group_searches: 10, inner_hits: { name: sub_hits, size: 2, sort: [{ sort: asc }] } }
+          sort: [{ sort: desc }]
+
+  - match: { hits.total: 6 }
+  - length: { hits.hits: 3 }
+  - match: { hits.hits.0._index: test }
+  - match: { hits.hits.0._type: test }
+  - match: { hits.hits.0.fields.numeric_group: [3] }
+  - match: { hits.hits.0.sort: [36] }
+  - match: { hits.hits.0._id: "6" }
+  - match: { hits.hits.0.inner_hits.sub_hits.hits.total: 1 }
+  - length: { hits.hits.0.inner_hits.sub_hits.hits.hits: 1 }
+  - match: { hits.hits.0.inner_hits.sub_hits.hits.hits.0._id: "6" }
+  - match: { hits.hits.1._index: test }
+  - match: { hits.hits.1._type: test }
+  - match: { hits.hits.1.fields.numeric_group: [1] }
+  - match: { hits.hits.1.sort: [24] }
+  - match: { hits.hits.1._id: "3" }
+  - match: { hits.hits.1.inner_hits.sub_hits.hits.total: 3 }
+  - length: { hits.hits.1.inner_hits.sub_hits.hits.hits: 2 }
+  - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.0._id: "2" }
+  - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.1._id: "1" }
+  - match: { hits.hits.2._index: test }
+  - match: { hits.hits.2._type: test }
+  - match: { hits.hits.2.fields.numeric_group: [25] }
+  - match: { hits.hits.2.sort: [10] }
+  - match: { hits.hits.2._id: "4" }
+  - match: { hits.hits.2.inner_hits.sub_hits.hits.total: 2 }
+  - length: { hits.hits.2.inner_hits.sub_hits.hits.hits: 2 }
+  - match: { hits.hits.2.inner_hits.sub_hits.hits.hits.0._id: "5" }
+  - match: { hits.hits.2.inner_hits.sub_hits.hits.hits.1._id: "4" }
+
+
+---
+"field collapsing and scroll":
+
+  - skip:
+      version: " - 5.2.99"
+      reason:  this uses a new API that has been added in 5.3
+
+  - do:
+      catch:      /cannot use \`collapse\` in a scroll context/
+      search:
+        index: test
+        type:  test
+        scroll: 1s
+        body:
+          collapse: { field: numeric_group }
+
+---
+"field collapsing and search_after":
+
+  - skip:
+      version: " - 5.2.99"
+      reason:  this uses a new API that has been added in 5.3
+
+  - do:
+      catch:      /cannot use \`collapse\` in conjunction with \`search_after\`/
+      search:
+        index: test
+        type:  test
+        body:
+          collapse: { field: numeric_group }
+          search_after: [6]
+          sort: [{ sort: desc }]
+
+---
+"field collapsing and rescore":
+
+  - skip:
+      version: " - 5.2.99"
+      reason:  this uses a new API that has been added in 5.3
+
+  - do:
+      catch:      /cannot use \`collapse\` in conjunction with \`rescore\`/
+      search:
+        index: test
+        type:  test
+        body:
+          collapse: { field: numeric_group }
+          rescore:
+            window_size: 20
+            query:
+              rescore_query:
+                match_all: {}
+              query_weight: 1
+              rescore_query_weight: 2
+
+---
+"no hits and inner_hits":
+
+  - skip:
+      version: " - 5.4.0"
+      reason:  "bug fixed in 5.4.1"
+
+  - do:
+      search:
+        index: test
+        type:  test
+        body:
+          size: 0
+          collapse: { field: numeric_group, inner_hits: { name: sub_hits, size: 1} }
+          sort: [{ sort: desc }]
+
+  - match: { hits.total: 6 }
+  - length: { hits.hits: 0 }
+
+---
+"field collapsing and multiple inner_hits":
+
+  - skip:
+      version: " - 5.4.99"
+      reason:  Multiple inner_hits is a new feature added in 5.5
+
+  - do:
+      search:
+        index: test
+        type:  test
+        body:
+          collapse: {
+            field: numeric_group,
+            inner_hits: [
+              { name: sub_hits_asc, size: 2, sort: [{ sort: asc }] },
+              { name: sub_hits_desc, size: 1, sort: [{ sort: desc }] }
+            ]
+          }
+          sort: [{ sort: desc }]
+
+  - match: { hits.total: 6 }
+  - length: { hits.hits: 3 }
+  - match: { hits.hits.0._index: test }
+  - match: { hits.hits.0._type: test }
+  - match: { hits.hits.0.fields.numeric_group: [3] }
+  - match: { hits.hits.0.sort: [36] }
+  - match: { hits.hits.0._id: "6" }
+  - match: { hits.hits.0.inner_hits.sub_hits_asc.hits.total: 1 }
+  - length: { hits.hits.0.inner_hits.sub_hits_asc.hits.hits: 1 }
+  - match: { hits.hits.0.inner_hits.sub_hits_asc.hits.hits.0._id: "6" }
+  - match: { hits.hits.0.inner_hits.sub_hits_desc.hits.total: 1 }
+  - length: { hits.hits.0.inner_hits.sub_hits_desc.hits.hits: 1 }
+  - match: { hits.hits.0.inner_hits.sub_hits_desc.hits.hits.0._id: "6" }
+  - match: { hits.hits.1._index: test }
+  - match: { hits.hits.1._type: test }
+  - match: { hits.hits.1.fields.numeric_group: [1] }
+  - match: { hits.hits.1.sort: [24] }
+  - match: { hits.hits.1._id: "3" }
+  - match: { hits.hits.1.inner_hits.sub_hits_asc.hits.total: 3 }
+  - length: { hits.hits.1.inner_hits.sub_hits_asc.hits.hits: 2 }
+  - match: { hits.hits.1.inner_hits.sub_hits_asc.hits.hits.0._id: "2" }
+  - match: { hits.hits.1.inner_hits.sub_hits_asc.hits.hits.1._id: "1" }
+  - match: { hits.hits.1.inner_hits.sub_hits_desc.hits.total: 3 }
+  - length: { hits.hits.1.inner_hits.sub_hits_desc.hits.hits: 1 }
+  - match: { hits.hits.1.inner_hits.sub_hits_desc.hits.hits.0._id: "3" }
+  - match: { hits.hits.2._index: test }
+  - match: { hits.hits.2._type: test }
+  - match: { hits.hits.2.fields.numeric_group: [25] }
+  - match: { hits.hits.2.sort: [10] }
+  - match: { hits.hits.2._id: "4" }
+  - match: { hits.hits.2.inner_hits.sub_hits_asc.hits.total: 2 }
+  - length: { hits.hits.2.inner_hits.sub_hits_asc.hits.hits: 2 }
+  - match: { hits.hits.2.inner_hits.sub_hits_asc.hits.hits.0._id: "5" }
+  - match: { hits.hits.2.inner_hits.sub_hits_asc.hits.hits.1._id: "4" }
+  - match: { hits.hits.2.inner_hits.sub_hits_desc.hits.total: 2 }
+  - length: { hits.hits.2.inner_hits.sub_hits_desc.hits.hits: 1 }
+  - match: { hits.hits.2.inner_hits.sub_hits_desc.hits.hits.0._id: "4" }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/120_batch_reduce_size.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/120_batch_reduce_size.yaml
new file mode 100644
index 0000000000000..7444b1bba17b4
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/120_batch_reduce_size.yaml
@@ -0,0 +1,70 @@
+setup:
+  - do:
+      indices.create:
+          index: test_1
+          body:
+            settings:
+              number_of_shards: 5
+              number_of_replicas: 0
+            mappings:
+              test:
+                properties:
+                  str:
+                     type: keyword
+
+---
+"batched_reduce_size lower limit":
+  - skip:
+      version: " - 5.3.99"
+      reason:  this was added in 5.4.0
+  - do:
+      catch:      /batchedReduceSize must be >= 2/
+      search:
+        index: test_1
+        batched_reduce_size: 1
+
+
+---
+"batched_reduce_size 2 with 5 shards":
+  - skip:
+      version: " - 5.3.99"
+      reason:  this was added in 5.4.0
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 1
+        body: { "str" : "abc" }
+
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 2
+        body: { "str": "abc" }
+
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 3
+        body: { "str": "bcd" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        batched_reduce_size: 2
+        body: { "size" : 0, "aggs" : { "str_terms" : { "terms" : { "field" : "str" } } } }
+
+  - match: { num_reduce_phases: 4 }
+  - match: { hits.total: 3 }
+  - length: { aggregations.str_terms.buckets: 2 }
+  - match: { aggregations.str_terms.buckets.0.key: "abc" }
+  - is_false: aggregations.str_terms.buckets.0.key_as_string
+  - match: { aggregations.str_terms.buckets.0.doc_count: 2 }
+  - match: { aggregations.str_terms.buckets.1.key: "bcd" }
+  - is_false: aggregations.str_terms.buckets.1.key_as_string
+  - match: { aggregations.str_terms.buckets.1.doc_count: 1 }
+
+
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/140_pre_filter_search_shards.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/140_pre_filter_search_shards.yml
new file mode 100644
index 0000000000000..158102f2014c9
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/140_pre_filter_search_shards.yml
@@ -0,0 +1,163 @@
+setup:
+  - do:
+      indices.create:
+          index: index_1
+          body:
+            settings:
+              number_of_shards: 1
+            mappings:
+              doc:
+                properties:
+                  created_at:
+                     type: date
+                     format: "yyyy-MM-dd"
+  - do:
+      indices.create:
+          index: index_2
+          body:
+            settings:
+              number_of_shards: 1
+            mappings:
+              doc:
+                properties:
+                  created_at:
+                     type: date
+                     format: "yyyy-MM-dd"
+  - do:
+      indices.create:
+          index: index_3
+          body:
+            settings:
+              number_of_shards: 1
+            mappings:
+              doc:
+                properties:
+                  created_at:
+                     type: date
+                     format: "yyyy-MM-dd"
+
+
+---
+"pre_filter_shard_size with invalid parameter":
+  - skip:
+      version: " - 5.5.99"
+      reason:  this was added in 5.6.0
+  - do:
+      catch: /preFilterShardSize must be >= 1/
+      search:
+        index: test_1
+        pre_filter_shard_size: 0
+
+---
+"pre_filter_shard_size with shards that have no hit":
+  - skip:
+      version: " - 5.5.99"
+      reason:  this was added in 5.6.0
+  - do:
+      index:
+        index: index_1
+        type: doc
+        id: 1
+        body: { "created_at": "2016-01-01"}
+  - do:
+      index:
+        index: index_2
+        type: doc
+        id: 2
+        body: { "created_at": "2017-01-01" }
+
+  - do:
+      index:
+        index: index_3
+        type: doc
+        id: 3
+        body: { "created_at": "2018-01-01" }
+  - do:
+      indices.refresh: {}
+
+
+  - do:
+      search:
+        body: { "size" : 0, "query" : { "range" : { "created_at" : { "gte" : "2016-02-01", "lt": "2018-02-01"} } } }
+
+  - match: { _shards.total: 3 }
+  - match: { _shards.successful: 3 }
+  - match: { _shards.skipped: 0 }
+  - match: { _shards.failed: 0 }
+  - match: { hits.total: 2 }
+
+  # this is the case where we have an empty body and don't skip anything since it's match_all
+  - do:
+      search:
+        pre_filter_shard_size: 1
+
+  - match: { _shards.total: 3 }
+  - match: { _shards.successful: 3 }
+  - match: { _shards.skipped: 0 }
+  - match: { _shards.failed: 0 }
+  - match: { hits.total: 3 }
+
+  # this is a case where we can actually skip due to rewrite
+  - do:
+      search:
+        pre_filter_shard_size: 1
+        body: { "size" : 0, "query" : { "range" : { "created_at" : { "gte" : "2016-02-01", "lt": "2018-02-01"} } } }
+
+  - match: { _shards.total: 3 }
+  - match: { _shards.successful: 3 }
+  - match: { _shards.skipped : 1}
+  - match: { _shards.failed: 0 }
+  - match: { hits.total: 2 }
+
+  # this case we skip all except of one since we need a real result
+  - do:
+      search:
+        pre_filter_shard_size: 1
+        body: { "size" : 0, "query" : { "range" : { "created_at" : { "gte" : "2019-02-01", "lt": "2020-02-01"} } } }
+
+  - match: { _shards.total: 3 }
+  - match: { _shards.successful: 3 }
+  - match: { _shards.skipped : 2} # skip 2 and execute one to fetch the actual empty result
+  - match: { _shards.failed: 0 }
+  - match: { hits.total: 0 }
+
+  - do:
+      search:
+        pre_filter_shard_size: 1
+        body: {"size" : 0, "query" : { "range" : { "created_at" : { "gte" : "2016-02-01", "lt": "2018-02-01"} } }, "aggs" : { "some_agg"  : { "global" : {} }}}
+
+  - match: { _shards.total: 3 }
+  - match: { _shards.successful: 3 }
+  - match: { _shards.skipped : 0 }
+  - match: { _shards.failed: 0 }
+  - match: { hits.total: 2 }
+  - match: { aggregations.some_agg.doc_count: 3 }
+
+
+  - do:
+      search:
+        pre_filter_shard_size: 1
+        body: { "size" : 0, "query" : { "range" : { "created_at" : { "gte" : "2016-02-01", "lt": "2018-02-01"}}}, "aggs" : { "idx_terms" : { "terms" : { "field" : "_index", "min_doc_count" : 0 } } } }
+
+  - match: { _shards.total: 3 }
+  - match: { _shards.successful: 3 }
+  - match: { _shards.skipped : 0 }
+  - match: { _shards.failed: 0 }
+  - match: { hits.total: 2 }
+  - length: { aggregations.idx_terms.buckets: 3 }
+
+  - do:
+      search:
+        pre_filter_shard_size: 1
+        body: { "size" : 0, "query" : { "range" : { "created_at" : { "gte" : "2016-02-01", "lt": "2018-02-01"}}}, "aggs" : { "idx_terms" :  { "terms" : { "field" : "_index" } } } }
+
+  - match: { _shards.total: 3 }
+  - match: { _shards.successful: 3 }
+  - match: { _shards.skipped : 1 }
+  - match: { _shards.failed: 0 }
+  - match: { hits.total: 2 }
+  - length: { aggregations.idx_terms.buckets: 2 }
+
+
+
+
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/30_limits.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/30_limits.yaml
index a93cefff699ef..49c9333c489ec 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/30_limits.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/30_limits.yaml
@@ -17,6 +17,14 @@ setup:
         index: test_1
         from: 10000
 
+---
+"Request with negative from value":
+  - do:
+      catch:      /\[from\] parameter cannot be negative/
+      search:
+        index: test_1
+        from: -2
+
 ---
 "Request window limits with scroll":
   - do:
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/40_indices_boost.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/40_indices_boost.yaml
new file mode 100644
index 0000000000000..38db09393a32e
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/40_indices_boost.yaml
@@ -0,0 +1,196 @@
+setup:
+  - do:
+      indices.create:
+          index:  test_1
+  - do:
+      indices.create:
+          index:  test_2
+
+  - do:
+      indices.put_alias:
+        index: test_1
+        name: alias_1
+
+  - do:
+      indices.put_alias:
+        index: test_2
+        name: alias_2
+
+  - do:
+      index:
+          index:  test_1
+          type:   test
+          id:     1
+          body:   { foo: bar }
+
+  - do:
+      index:
+          index:  test_2
+          type:   test
+          id:     1
+          body:   { foo: bar }
+
+  - do:
+      indices.refresh:
+        index: [test_1, test_2]
+
+---
+"Indices boost using object":
+  - skip:
+      version: " - 5.2.99"
+      reason: deprecation was added in 5.2.0 and its format changed in 5.3.0
+      features: "warnings"
+
+  - do:
+      warnings:
+        - 'Object format in indices_boost is deprecated, please use array format instead'
+      search:
+        index: _all
+        body:
+          indices_boost: {test_1: 2.0, test_2: 1.0}
+
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0._index: test_1 }
+  - match: { hits.hits.1._index: test_2 }
+
+  - do:
+      warnings:
+        - 'Object format in indices_boost is deprecated, please use array format instead'
+      search:
+        index: _all
+        body:
+          indices_boost: {test_1: 1.0, test_2: 2.0}
+
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0._index: test_2 }
+  - match: { hits.hits.1._index: test_1 }
+
+---
+"Indices boost using array":
+  - skip:
+      version: " - 5.1.99"
+      reason: array format was added in 5.2.0
+
+  - do:
+      search:
+        index: _all
+        body:
+          indices_boost: [{test_1: 2.0}, {test_2: 1.0}]
+
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0._index: test_1 }
+  - match: { hits.hits.1._index: test_2 }
+
+  - do:
+      search:
+        index: _all
+        body:
+          indices_boost: [{test_1: 1.0}, {test_2: 2.0}]
+
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0._index: test_2 }
+  - match: { hits.hits.1._index: test_1 }
+
+---
+"Indices boost using array with alias":
+  - skip:
+      version: " - 5.1.99"
+      reason: array format was added in 5.2.0
+
+  - do:
+      search:
+        index: _all
+        body:
+          indices_boost: [{alias_1: 2.0}]
+
+  - match: { hits.total: 2}
+  - match: { hits.hits.0._index: test_1 }
+  - match: { hits.hits.1._index: test_2 }
+
+  - do:
+      search:
+        index: _all
+        body:
+          indices_boost: [{alias_2: 2.0}]
+
+  - match: { hits.total: 2}
+  - match: { hits.hits.0._index: test_2 }
+  - match: { hits.hits.1._index: test_1 }
+
+---
+"Indices boost using array with wildcard":
+  - skip:
+      version: " - 5.1.99"
+      reason: array format was added in 5.2.0
+
+  - do:
+      search:
+        index: _all
+        body:
+          indices_boost: [{"*_1": 2.0}]
+
+  - match: { hits.total: 2}
+  - match: { hits.hits.0._index: test_1 }
+  - match: { hits.hits.1._index: test_2 }
+
+  - do:
+      search:
+        index: _all
+        body:
+          indices_boost: [{"*_2": 2.0}]
+
+  - match: { hits.total: 2}
+  - match: { hits.hits.0._index: test_2 }
+  - match: { hits.hits.1._index: test_1 }
+
+---
+"Indices boost using array multiple match":
+  - skip:
+      version: " - 5.1.99"
+      reason: array format was added in 5.2.0
+
+  - do:
+      search:
+        index: _all
+        body:
+          # First match (3.0) is used for test_1
+          indices_boost: [{"*_1": 3.0}, {alias_1: 1.0}, {test_2: 2.0}]
+
+  - match: { hits.total: 2}
+  - match: { hits.hits.0._index: test_1 }
+  - match: { hits.hits.1._index: test_2 }
+
+  - do:
+      search:
+        index: _all
+        body:
+          # First match (1.0) is used for test_1
+          indices_boost: [{"*_1": 1.0}, {test_2: 2.0}, {alias_1: 3.0}]
+
+  - match: { hits.total: 2}
+  - match: { hits.hits.0._index: test_2 }
+  - match: { hits.hits.1._index: test_1 }
+
+---
+"Indices boost for nonexistent index/alias":
+  - skip:
+      version: " - 5.1.99"
+      reason: array format was added in 5.2.0
+
+  - do:
+      catch: /no such index/
+      search:
+        index: _all
+        body:
+          indices_boost: [{nonexistent: 2.0}, {test_1: 1.0}, {test_2: 2.0}]
+
+  - do:
+      search:
+        index: _all
+        ignore_unavailable: true
+        body:
+          indices_boost: [{nonexistent: 2.0}, {test_1: 1.0}, {test_2: 2.0}]
+
+  - match: { hits.total: 2}
+  - match: { hits.hits.0._index: test_2 }
+  - match: { hits.hits.1._index: test_1 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/60_query_string.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/60_query_string.yaml
index 6fb93bb10449a..8533cfd26686d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/60_query_string.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/60_query_string.yaml
@@ -58,15 +58,6 @@
       search:
         index: test
         q: field:BA*
-        lowercase_expanded_terms: false
-
-  - match: {hits.total: 0}
-
-  - do:
-      search:
-        index: test
-        q: field:BA*
-        analyze_wildcard: true
 
   - match: {hits.total: 1}
 
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/70_response_filtering.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/70_response_filtering.yaml
index bff9a169604de..71eb5665bea43 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/70_response_filtering.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/70_response_filtering.yaml
@@ -43,7 +43,7 @@
         filter_path: "took"
         body: "{ \"query\": { \"match_all\": {} } }"
 
-  - is_true:  took
+  - gte: { took: 0 }
   - is_false: _shards.total
   - is_false: hits.total
   - is_false: hits.hits.0._index
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/80_date_math_index_names.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/80_date_math_index_names.yaml
deleted file mode 100644
index 2f2cf6ab98286..0000000000000
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/80_date_math_index_names.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
----
-"Missing index with catch":
-
-  - do:
-      catch:   /logstash-\d{4}\.\d{2}\.\d{2}/
-      search:
-        index: <logstash-{now/M}>
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/80_indices_options.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/80_indices_options.yaml
new file mode 100644
index 0000000000000..d3cb5dae84602
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/80_indices_options.yaml
@@ -0,0 +1,63 @@
+---
+"Missing index date math with catch":
+
+  - do:
+      catch:   /logstash-\d{4}\.\d{2}\.\d{2}/
+      search:
+        index: <logstash-{now/M}>
+
+---
+"Missing index":
+
+  - do:
+      catch:   missing
+      search:
+        index: missing_index
+
+  - do:
+      search:
+        index: missing_index
+        ignore_unavailable: true
+
+  - match: {hits.total: 0}
+
+---
+"Closed index":
+
+  - do:
+      indices.create:
+          index:  index_closed
+
+  - do:
+      indices.close:
+          index:  index_closed
+
+  - do:
+      catch:  /index_closed_exception/
+      search:
+        index: index_closed
+
+  - do:
+      search:
+        index: index_closed
+        ignore_unavailable: true
+
+  - match: {hits.total: 0}
+
+  - do:
+      search:
+        index: index*
+
+  - match: {hits.total: 0}
+
+  - do:
+      catch: missing
+      search:
+        index: index*
+        allow_no_indices: false
+
+  - do:
+      catch:  /index_closed_exception/
+      search:
+        index: index*
+        expand_wildcards: ["open","closed"]
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search_shards/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search_shards/10_basic.yaml
index 4bfe6e953b250..8033f077b0d80 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search_shards/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search_shards/10_basic.yaml
@@ -10,3 +10,119 @@
         routing: foo
 
   - match: { shards.0.0.index: test_1 }
+
+---
+"Search shards with type":
+  - skip:
+      features: "warnings"
+      version: " - 5.2.99"
+      reason: type was deprecated in 5.1.0 and the deprecation format changed in 5.3.0
+
+  - do:
+      indices.create:
+        index: test_1
+
+  - do:
+      warnings:
+        - type [whatever] doesn't have any effect in the search shards api, it should be rather omitted
+      search_shards:
+        index:  test_1
+        type: whatever
+
+  - match: { shards.0.0.index: test_1 }
+
+---
+"Search shards aliases with and without filters":
+  - skip:
+      version: " - 5.4.0"
+      reason: "#24489 fixed a bug that not all aliases where added in 5.4.1 - indices section was added in 5.1.0"
+
+  - do:
+      indices.create:
+        index: test_index
+        body:
+          settings:
+            index:
+              number_of_shards: 1
+              number_of_replicas: 0
+          mappings:
+            type_1:
+              properties:
+                field:
+                  type: text
+          aliases:
+            test_alias_no_filter: {}
+            test_alias_filter_1:
+              filter:
+                term:
+                  field : value1
+            test_alias_filter_2:
+              filter:
+                term:
+                  field : value2
+
+  - do:
+      search_shards:
+        index:  test_alias_no_filter
+
+  - length: { shards: 1 }
+  - match: { shards.0.0.index: test_index }
+  - is_true: indices.test_index
+  - is_false: indices.test_index.filter
+  - match: { indices.test_index.aliases: [test_alias_no_filter]}
+
+  - do:
+      search_shards:
+        index:  test_alias_filter_1
+
+  - length: { shards: 1 }
+  - match: { shards.0.0.index: test_index }
+  - match: { indices.test_index.aliases: [test_alias_filter_1] }
+  - match: { indices.test_index.filter.term.field.value: value1 }
+  - lte: { indices.test_index.filter.term.field.boost: 1.0 }
+  - gte: { indices.test_index.filter.term.field.boost: 1.0 }
+
+  - do:
+      search_shards:
+        index:  ["test_alias_filter_1","test_alias_filter_2"]
+
+  - length: { shards: 1 }
+  - match: { shards.0.0.index: test_index }
+  - match: { indices.test_index.aliases: [test_alias_filter_1, test_alias_filter_2]}
+  - match: { indices.test_index.filter.bool.should.0.term.field.value: value1 }
+  - lte: { indices.test_index.filter.bool.should.0.term.field.boost: 1.0 }
+  - gte: { indices.test_index.filter.bool.should.0.term.field.boost: 1.0 }
+  - match: { indices.test_index.filter.bool.should.1.term.field.value: value2}
+  - lte: { indices.test_index.filter.bool.should.1.term.field.boost: 1.0 }
+  - gte: { indices.test_index.filter.bool.should.1.term.field.boost: 1.0 }
+  - match: { indices.test_index.filter.bool.adjust_pure_negative: true}
+  - match: { indices.test_index.filter.bool.disable_coord: false}
+  - lte: { indices.test_index.filter.bool.boost: 1.0 }
+  - gte: { indices.test_index.filter.bool.boost: 1.0 }
+
+  - do:
+      search_shards:
+        index:  "test*"
+
+  - length: { shards: 1 }
+  - match: { shards.0.0.index: test_index }
+  - match: { indices.test_index.aliases: [test_alias_filter_1, test_alias_filter_2, test_alias_no_filter]}
+  - is_false: indices.test_index.filter
+
+  - do:
+      search_shards:
+        index:  ["test_alias_filter_1","test_alias_no_filter"]
+
+  - length: { shards: 1 }
+  - match: { shards.0.0.index: test_index }
+  - match: { indices.test_index.aliases: [test_alias_filter_1, test_alias_no_filter]}
+  - is_false: indices.test_index.filter
+
+  - do:
+      search_shards:
+        index:  ["test_alias_no_filter"]
+
+  - length: { shards: 1 }
+  - match: { shards.0.0.index: test_index }
+  - match: { indices.test_index.aliases: [test_alias_no_filter]}
+  - is_false: indices.test_index.filter
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.create/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.create/10_basic.yaml
index 43db8e5206fc4..b12b9d09b6fea 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.create/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.create/10_basic.yaml
@@ -30,3 +30,10 @@ setup:
   - match: { snapshot.state : SUCCESS }
   - match: { snapshot.shards.successful: 1 }
   - match: { snapshot.shards.failed : 0 }
+
+  - do:
+      snapshot.delete:
+        repository: test_repo_create_1
+        snapshot: test_snapshot
+
+  - match: { acknowledged: true }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get/10_basic.yaml
index bd609e3e3bfe8..70008415122d3 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get/10_basic.yaml
@@ -9,13 +9,6 @@ setup:
           settings:
             location: "test_repo_get_1_loc"
 
----
-teardown:
-
-  - do:
-      snapshot.delete_repository:
-        repository: test_repo_get_1
-
 ---
 "Get snapshot info":
 
@@ -39,7 +32,13 @@ teardown:
         snapshot: test_snapshot
 
   - is_true: snapshots
-  
+  - is_true: snapshots.0.failures
+
+  - do:
+      snapshot.delete:
+        repository: test_repo_get_1
+        snapshot: test_snapshot
+
 ---
 "Get missing snapshot info throws an exception":
 
@@ -48,7 +47,7 @@ teardown:
       snapshot.get:
         repository: test_repo_get_1
         snapshot: test_nonexistent_snapshot
-   
+
 ---
 "Get missing snapshot info succeeds when ignoreUnavailable is true":
 
@@ -59,3 +58,41 @@ teardown:
         ignore_unavailable: true
 
   - is_true: snapshots
+
+---
+"Get snapshot info when verbose is false":
+  - skip:
+      version: " - 5.99.99"
+      reason: verbose mode was introduced in 6.0
+
+  - do:
+      indices.create:
+        index: test_index
+        body:
+          settings:
+            number_of_shards:   1
+            number_of_replicas: 0
+
+  - do:
+      snapshot.create:
+        repository: test_repo_get_1
+        snapshot: test_snapshot
+        wait_for_completion: true
+
+  - do:
+      snapshot.get:
+        repository: test_repo_get_1
+        snapshot: test_snapshot
+        verbose: false
+
+  - is_true: snapshots
+  - match: { snapshots.0.snapshot: test_snapshot }
+  - match: { snapshots.0.state: SUCCESS }
+  - is_false: snapshots.0.failures
+  - is_false: snapshots.0.shards
+  - is_false: snapshots.0.version
+
+  - do:
+      snapshot.delete:
+        repository: test_repo_get_1
+        snapshot: test_snapshot
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yaml
index d4548553e251a..838c12649748e 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yaml
@@ -9,13 +9,6 @@ setup:
           settings:
             location: "test_repo_status_1_loc"
 
----
-teardown:
-
-  - do:
-      snapshot.delete_repository:
-        repository: test_repo_status_1
-
 ---
 "Get snapshot status":
 
@@ -39,7 +32,7 @@ teardown:
         snapshot: test_snapshot
 
   - is_true: snapshots
-  
+
 ---
 "Get missing snapshot status throws an exception":
 
@@ -48,7 +41,7 @@ teardown:
       snapshot.status:
         repository: test_repo_status_1
         snapshot: test_nonexistent_snapshot
-   
+
 ---
 "Get missing snapshot status succeeds when ignoreUnavailable is true":
 
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/10_basic.yaml
index 211e95ba79c73..0e553d0a47728 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/10_basic.yaml
@@ -13,6 +13,26 @@ setup:
 "Basic tests for suggest API":
 
   - do:
+      search:
+        body:
+          suggest:
+            test_suggestion:
+              text: "The Amsterdma meetpu"
+              term:
+                field: body
+
+  - match: {suggest.test_suggestion.1.options.0.text: amsterdam}
+  - match: {suggest.test_suggestion.2.options.0.text: meetup}
+
+---
+"Suggest API should have deprecation warning":
+  - skip:
+      features: 'warnings'
+      version: ' - 5.2.99'
+      reason: no deprecation logging prior to 5.2, log format changed in 5.3.0
+  - do:
+      warnings:
+        - "[POST /_suggest] is deprecated! Use [POST /_search] instead."
       suggest:
         body:
           test_suggestion:
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/110_completion.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/110_completion.yaml
new file mode 100644
index 0000000000000..9677d919b3001
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/110_completion.yaml
@@ -0,0 +1,325 @@
+# This test creates one huge mapping in the setup
+# Every test should use its own field to make sure it works
+
+setup:
+
+  - do:
+      indices.create:
+          index: test
+          body:
+            mappings:
+              test:
+                "properties":
+                  "suggest_1":
+                     "type" : "completion"
+                  "suggest_2":
+                     "type" : "completion"
+                  "suggest_3":
+                     "type" : "completion"
+                  "suggest_4":
+                     "type" : "completion"
+                  "suggest_5a":
+                     "type" : "completion"
+                  "suggest_5b":
+                     "type" : "completion"
+                  "suggest_6":
+                     "type" : "completion"
+                  title:
+                     type: keyword
+
+---
+"Simple suggestion should work":
+  - skip:
+      features: 'warnings'
+      version: ' - 5.2.99'
+      reason: no deprecation logging prior to 5.2 and log format changed in 5.3.0
+
+  - do:
+      index:
+        index: test
+        type:  test
+        id:    1
+        body:
+          suggest_1: "bar"
+
+  - do:
+      index:
+        index: test
+        type:  test
+        id:    2
+        body:
+          suggest_1: "baz"
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      warnings:
+        - "[POST /_suggest] is deprecated! Use [POST /_search] instead."
+      suggest:
+        body:
+          result:
+            text: "b"
+            completion:
+              field: suggest_1
+
+  - length: { result: 1  }
+  - length: { result.0.options: 2  }
+
+---
+"Simple suggestion array should work":
+  - skip:
+      features: 'warnings'
+      version: ' - 5.2.99'
+      reason: no deprecation logging prior to 5.2, log format changed in 5.3.0
+
+  - do:
+      index:
+        index: test
+        type:  test
+        id:    1
+        body:
+          suggest_2: ["bar", "foo"]
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      warnings:
+        - "[POST /_suggest] is deprecated! Use [POST /_search] instead."
+      suggest:
+        body:
+          result:
+            text: "f"
+            completion:
+              field: suggest_2
+
+  - length: { result: 1  }
+  - length: { result.0.options: 1  }
+  - match:  { result.0.options.0.text: "foo" }
+
+  - do:
+      warnings:
+        - "[POST /_suggest] is deprecated! Use [POST /_search] instead."
+      suggest:
+        body:
+          result:
+            text: "b"
+            completion:
+              field: suggest_2
+
+  - length: { result: 1  }
+  - length: { result.0.options: 1  }
+  - match:  { result.0.options.0.text: "bar" }
+
+---
+"Suggestion entry should work":
+  - skip:
+      features: 'warnings'
+      version: ' - 5.2.99'
+      reason: no deprecation logging prior to 5.2, log format changed in 5.3.0
+
+  - do:
+      index:
+        index: test
+        type:  test
+        id:    1
+        body:
+          suggest_3:
+            input: "bar"
+            weight: 2
+
+  - do:
+      index:
+        index: test
+        type:  test
+        id:    2
+        body:
+          suggest_3:
+            input: "baz"
+            weight: 3
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      warnings:
+        - "[POST /_suggest] is deprecated! Use [POST /_search] instead."
+      suggest:
+        body:
+          result:
+            text: "b"
+            completion:
+              field: suggest_3
+
+  - length: { result: 1  }
+  - length: { result.0.options: 2  }
+  - match:  { result.0.options.0.text: "baz" }
+  - match:  { result.0.options.1.text: "bar" }
+
+---
+"Suggestion entry array should work":
+  - skip:
+      features: 'warnings'
+      version: ' - 5.2.99'
+      reason: no deprecation logging prior to 5.2, log format changed in 5.3.0
+
+  - do:
+      index:
+        index: test
+        type:  test
+        id:    1
+        body:
+          suggest_4:
+            - input: "bar"
+              weight: 3
+            - input: "fo"
+              weight: 3
+
+  - do:
+      index:
+        index: test
+        type:  test
+        id:    2
+        body:
+          suggest_4:
+            - input: "baz"
+              weight: 2
+            - input: "foo"
+              weight: 1
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      warnings:
+        - "[POST /_suggest] is deprecated! Use [POST /_search] instead."
+      suggest:
+        body:
+          result:
+            text: "b"
+            completion:
+              field: suggest_4
+
+  - length: { result: 1  }
+  - length: { result.0.options: 2  }
+  - match:  { result.0.options.0.text: "bar" }
+  - match:  { result.0.options.1.text: "baz" }
+
+  - do:
+      warnings:
+        - "[POST /_suggest] is deprecated! Use [POST /_search] instead."
+      suggest:
+        body:
+          result:
+            text: "f"
+            completion:
+              field: suggest_4
+
+  - length: { result: 1  }
+  - length: { result.0.options: 2  }
+  - match:  { result.0.options.0.text: "fo" }
+  - match:  { result.0.options.1.text: "foo" }
+
+---
+"Multiple Completion fields should work":
+  - skip:
+      features: 'warnings'
+      version: ' - 5.2.99'
+      reason: no deprecation logging prior to 5.2, log format changed in 5.3.0
+
+  - do:
+      index:
+        index: test
+        type:  test
+        id:    1
+        body:
+          suggest_5a: "bar"
+          suggest_5b: "baz"
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      warnings:
+        - "[POST /_suggest] is deprecated! Use [POST /_search] instead."
+      suggest:
+        body:
+          result:
+            text: "b"
+            completion:
+              field: suggest_5a
+
+  - length: { result: 1  }
+  - length: { result.0.options: 1  }
+  - match:  { result.0.options.0.text: "bar" }
+
+  - do:
+      warnings:
+        - "[POST /_suggest] is deprecated! Use [POST /_search] instead."
+      suggest:
+        body:
+          result:
+            text: "b"
+            completion:
+              field: suggest_5b
+
+  - length: { result: 1  }
+  - length: { result.0.options: 1  }
+  - match:  { result.0.options.0.text: "baz" }
+
+---
+"Suggestions with source should work":
+  - skip:
+      features: 'warnings'
+      version: ' - 5.2.99'
+      reason: no deprecation logging prior to 5.2, log format changed in 5.3.0
+  - do:
+      index:
+        index: test
+        type:  test
+        id:    1
+        body:
+          suggest_6:
+            input: "bar"
+            weight: 2
+          title: "title_bar"
+          count: 4
+
+  - do:
+      index:
+        index: test
+        type:  test
+        id:    2
+        body:
+          suggest_6:
+            input: "baz"
+            weight: 3
+          title: "title_baz"
+          count: 3
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      warnings:
+        - "[POST /_suggest] is deprecated! Use [POST /_search] instead."
+      suggest:
+        body:
+          result:
+            text: "b"
+            completion:
+              field: suggest_6
+
+  - length: { result: 1  }
+  - length: { result.0.options: 2  }
+  - match:  { result.0.options.0.text: "baz" }
+  - match:  { result.0.options.0._index: "test" }
+  - match:  { result.0.options.0._type: "test" }
+  - match:  { result.0.options.0._source.title: "title_baz" }
+  - match:  { result.0.options.0._source.count: 3 }
+  - match:  { result.0.options.1.text: "bar" }
+  - match:  { result.0.options.1._index: "test" }
+  - match:  { result.0.options.1._type: "test" }
+  - match:  { result.0.options.1._source.title: "title_bar" }
+  - match:  { result.0.options.1._source.count: 4 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/20_completion.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/20_completion.yaml
index 2c8b08fd9c59f..860c43bbcbf0e 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/20_completion.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/20_completion.yaml
@@ -50,15 +50,16 @@ setup:
       indices.refresh: {}
 
   - do:
-      suggest:
+      search:
         body:
-          result:
-            text: "b"
-            completion:
-              field: suggest_1
+          suggest:
+            result:
+              text: "b"
+              completion:
+                field: suggest_1
 
-  - length: { result: 1  }
-  - length: { result.0.options: 2  }
+  - length: { suggest.result: 1  }
+  - length: { suggest.result.0.options: 2  }
 
 ---
 "Simple suggestion array should work":
@@ -75,28 +76,30 @@ setup:
       indices.refresh: {}
 
   - do:
-      suggest:
+      search:
         body:
-          result:
-            text: "f"
-            completion:
-              field: suggest_2
+          suggest:
+            result:
+              text: "f"
+              completion:
+                field: suggest_2
 
-  - length: { result: 1  }
-  - length: { result.0.options: 1  }
-  - match:  { result.0.options.0.text: "foo" }
+  - length: { suggest.result: 1  }
+  - length: { suggest.result.0.options: 1  }
+  - match:  { suggest.result.0.options.0.text: "foo" }
 
   - do:
-      suggest:
+      search:
         body:
-          result:
-            text: "b"
-            completion:
-              field: suggest_2
+          suggest:
+            result:
+              text: "b"
+              completion:
+                field: suggest_2
 
-  - length: { result: 1  }
-  - length: { result.0.options: 1  }
-  - match:  { result.0.options.0.text: "bar" }
+  - length: { suggest.result: 1  }
+  - length: { suggest.result.0.options: 1  }
+  - match:  { suggest.result.0.options.0.text: "bar" }
 
 ---
 "Suggestion entry should work":
@@ -125,17 +128,18 @@ setup:
       indices.refresh: {}
 
   - do:
-      suggest:
+      search:
         body:
-          result:
-            text: "b"
-            completion:
-              field: suggest_3
+          suggest:
+            result:
+              text: "b"
+              completion:
+                field: suggest_3
 
-  - length: { result: 1  }
-  - length: { result.0.options: 2  }
-  - match:  { result.0.options.0.text: "baz" }
-  - match:  { result.0.options.1.text: "bar" }
+  - length: { suggest.result: 1  }
+  - length: { suggest.result.0.options: 2  }
+  - match:  { suggest.result.0.options.0.text: "baz" }
+  - match:  { suggest.result.0.options.1.text: "bar" }
 
 ---
 "Suggestion entry array should work":
@@ -168,30 +172,32 @@ setup:
       indices.refresh: {}
 
   - do:
-      suggest:
+      search:
         body:
-          result:
-            text: "b"
-            completion:
-              field: suggest_4
+          suggest:
+            result:
+              text: "b"
+              completion:
+                field: suggest_4
 
-  - length: { result: 1  }
-  - length: { result.0.options: 2  }
-  - match:  { result.0.options.0.text: "bar" }
-  - match:  { result.0.options.1.text: "baz" }
+  - length: { suggest.result: 1  }
+  - length: { suggest.result.0.options: 2  }
+  - match:  { suggest.result.0.options.0.text: "bar" }
+  - match:  { suggest.result.0.options.1.text: "baz" }
 
   - do:
-      suggest:
+      search:
         body:
-          result:
-            text: "f"
-            completion:
-              field: suggest_4
+          suggest:
+            result:
+              text: "f"
+              completion:
+                field: suggest_4
 
-  - length: { result: 1  }
-  - length: { result.0.options: 2  }
-  - match:  { result.0.options.0.text: "fo" }
-  - match:  { result.0.options.1.text: "foo" }
+  - length: { suggest.result: 1  }
+  - length: { suggest.result.0.options: 2  }
+  - match:  { suggest.result.0.options.0.text: "fo" }
+  - match:  { suggest.result.0.options.1.text: "foo" }
 
 ---
 "Multiple Completion fields should work":
@@ -209,28 +215,30 @@ setup:
       indices.refresh: {}
 
   - do:
-      suggest:
+      search:
         body:
-          result:
-            text: "b"
-            completion:
-              field: suggest_5a
+          suggest:
+            result:
+              text: "b"
+              completion:
+                field: suggest_5a
 
-  - length: { result: 1  }
-  - length: { result.0.options: 1  }
-  - match:  { result.0.options.0.text: "bar" }
+  - length: { suggest.result: 1  }
+  - length: { suggest.result.0.options: 1  }
+  - match:  { suggest.result.0.options.0.text: "bar" }
 
   - do:
-      suggest:
+      search:
         body:
-          result:
-            text: "b"
-            completion:
-              field: suggest_5b
+          suggest:
+            result:
+              text: "b"
+              completion:
+                field: suggest_5b
 
-  - length: { result: 1  }
-  - length: { result.0.options: 1  }
-  - match:  { result.0.options.0.text: "baz" }
+  - length: { suggest.result: 1  }
+  - length: { suggest.result.0.options: 1  }
+  - match:  { suggest.result.0.options.0.text: "baz" }
 
 ---
 "Suggestions with source should work":
@@ -263,23 +271,23 @@ setup:
       indices.refresh: {}
 
   - do:
-      suggest:
+      search:
         body:
-          result:
-            text: "b"
-            completion:
-              field: suggest_6
-
-  - length: { result: 1  }
-  - length: { result.0.options: 2  }
-  - match:  { result.0.options.0.text: "baz" }
-  - match:  { result.0.options.0._index: "test" }
-  - match:  { result.0.options.0._type: "test" }
-  - match:  { result.0.options.0._source.title: "title_baz" }
-  - match:  { result.0.options.0._source.count: 3 }
-  - match:  { result.0.options.1.text: "bar" }
-  - match:  { result.0.options.1._index: "test" }
-  - match:  { result.0.options.1._type: "test" }
-  - match:  { result.0.options.1._source.title: "title_bar" }
-  - match:  { result.0.options.1._source.count: 4 }
-
+          suggest:
+            result:
+              text: "b"
+              completion:
+                field: suggest_6
+
+  - length: { suggest.result: 1  }
+  - length: { suggest.result.0.options: 2  }
+  - match:  { suggest.result.0.options.0.text: "baz" }
+  - match:  { suggest.result.0.options.0._index: "test" }
+  - match:  { suggest.result.0.options.0._type: "test" }
+  - match:  { suggest.result.0.options.0._source.title: "title_baz" }
+  - match:  { suggest.result.0.options.0._source.count: 3 }
+  - match:  { suggest.result.0.options.1.text: "bar" }
+  - match:  { suggest.result.0.options.1._index: "test" }
+  - match:  { suggest.result.0.options.1._type: "test" }
+  - match:  { suggest.result.0.options.1._source.title: "title_bar" }
+  - match:  { suggest.result.0.options.1._source.count: 4 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/30_context.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/30_context.yaml
index da7af85cf9f43..778094ec90baf 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/30_context.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/30_context.yaml
@@ -15,28 +15,33 @@ setup:
                   "suggest_context":
                      "type" : "completion"
                      "contexts":
-                        - "name" : "color"
+                        -
+                          "name" : "color"
                           "type" : "category"
                   "suggest_context_with_path":
                      "type" : "completion"
                      "contexts":
-                        - "name" : "color"
+                        -
+                          "name" : "color"
                           "type" : "category"
                           "path" : "color"
                   "suggest_geo":
                      "type" : "completion"
                      "contexts":
-                        - "name" : "location"
+                        -
+                          "name" : "location"
                           "type" : "geo"
                           "precision" : "5km"
                   "suggest_multi_contexts":
                      "type" : "completion"
                      "contexts":
-                        - "name" : "location"
+                        -
+                          "name" : "location"
                           "type" : "geo"
                           "precision" : "5km"
                           "path" : "location"
-                        - "name" : "color"
+                        -
+                          "name" : "color"
                           "type" : "category"
                           "path" : "color"
 
@@ -69,18 +74,19 @@ setup:
       indices.refresh: {}
 
   - do:
-      suggest:
+      search:
         body:
-          result:
-            text: "foo"
-            completion:
-              field: suggest_context
-              contexts:
-                color: "red"
-
-  - length: { result: 1  }
-  - length: { result.0.options: 1  }
-  - match:  {  result.0.options.0.text: "foo red" }
+          suggest:
+            result:
+              text: "foo"
+              completion:
+                field: suggest_context
+                contexts:
+                  color: "red"
+
+  - length: { suggest.result: 1  }
+  - length: { suggest.result.0.options: 1  }
+  - match:  { suggest.result.0.options.0.text: "foo red" }
 
 ---
 "Category suggest context from path should work":
@@ -109,45 +115,48 @@ setup:
       indices.refresh: {}
 
   - do:
-      suggest:
+      search:
         body:
-          result:
-            text: "foo"
-            completion:
-              field: suggest_context_with_path
-              contexts:
-                color: "red"
-
-  - length: { result: 1  }
-  - length: { result.0.options: 1  }
-  - match:  { result.0.options.0.text: "Foo red" }
+          suggest:
+            result:
+              text: "foo"
+              completion:
+                field: suggest_context_with_path
+                contexts:
+                  color: "red"
+
+  - length: { suggest.result: 1  }
+  - length: { suggest.result.0.options: 1  }
+  - match:  { suggest.result.0.options.0.text: "Foo red" }
 
   - do:
-      suggest:
+      search:
         body:
-          result:
-            text: "foo"
-            completion:
-              field: suggest_context_with_path
-              contexts:
-                color: "blue"
-
-  - length: { result: 1  }
-  - length: { result.0.options: 1  }
-  - match:  { result.0.options.0.text: "Foo blue" }
+          suggest:
+            result:
+              text: "foo"
+              completion:
+                field: suggest_context_with_path
+                contexts:
+                  color: "blue"
+
+  - length: { suggest.result: 1  }
+  - length: { suggest.result.0.options: 1  }
+  - match:  { suggest.result.0.options.0.text: "Foo blue" }
 
   - do:
-      suggest:
+      search:
         body:
-          result:
-            text: "foo"
-            completion:
-              field: suggest_context_with_path
-              contexts:
-                color: ["blue", "red"]
+          suggest:
+            result:
+              text: "foo"
+              completion:
+                field: suggest_context_with_path
+                contexts:
+                  color: ["blue", "red"]
 
-  - length: { result: 1  }
-  - length: { result.0.options: 2  }
+  - length: { suggest.result: 1  }
+  - length: { suggest.result.0.options: 2  }
 
 ---
 "Geo suggest should work":
@@ -185,21 +194,22 @@ setup:
       indices.get_mapping: {}
 
   - do:
-      suggest:
+      search:
         index: test
         body:
-          result:
-            text: "mar"
-            completion:
-              field: suggest_geo
-              contexts:
-                location:
-                  lat : 52.2263
-                  lon : 4.543
-
-  - length: { result: 1  }
-  - length: { result.0.options: 1  }
-  - match:  { result.0.options.0.text: "Marriot in Amsterdam" }
+          suggest:
+            result:
+              text: "mar"
+              completion:
+                field: suggest_geo
+                contexts:
+                  location:
+                    lat : 52.2263
+                    lon : 4.543
+
+  - length: { suggest.result: 1  }
+  - length: { suggest.result.0.options: 1  }
+  - match:  { suggest.result.0.options.0.text: "Marriot in Amsterdam" }
 
 ---
 "Multi contexts should work":
@@ -235,33 +245,35 @@ setup:
       indices.get_mapping: {}
 
   - do:
-      suggest:
+      search:
         index: test
         body:
-          result:
-            text: "mar"
-            completion:
-              field: suggest_multi_contexts
-              contexts:
-                location:
-                  lat : 52.22
-                  lon : 4.53
-
-  - length: { result: 1  }
-  - length: { result.0.options: 1  }
-  - match:  { result.0.options.0.text: "Marriot in Amsterdam" }
+          suggest:
+            result:
+              text: "mar"
+              completion:
+                field: suggest_multi_contexts
+                contexts:
+                  location:
+                    lat : 52.22
+                    lon : 4.53
+
+  - length: { suggest.result: 1  }
+  - length: { suggest.result.0.options: 1  }
+  - match:  { suggest.result.0.options.0.text: "Marriot in Amsterdam" }
 
   - do:
-      suggest:
+      search:
         index: test
         body:
-          result:
-            text: "mar"
-            completion:
-              field: suggest_multi_contexts
-              contexts:
-                color: "blue"
-
-  - length: { result: 1  }
-  - length: { result.0.options: 1  }
-  - match:  { result.0.options.0.text: "Marriot in Berlin" }
+          suggest:
+            result:
+              text: "mar"
+              completion:
+                field: suggest_multi_contexts
+                contexts:
+                  color: "blue"
+
+  - length: { suggest.result: 1  }
+  - length: { suggest.result.0.options: 1  }
+  - match:  { suggest.result.0.options.0.text: "Marriot in Berlin" }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/40_typed_keys.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/40_typed_keys.yaml
new file mode 100644
index 0000000000000..139c972eea677
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/40_typed_keys.yaml
@@ -0,0 +1,66 @@
+setup:
+  - skip:
+      version: " - 5.3.99"
+      reason: typed_keys parameter was added in 5.4.0
+
+  - do:
+      indices.create:
+          index: test
+          body:
+            settings:
+              number_of_replicas: 0
+            mappings:
+              test:
+                properties:
+                  title:
+                    type: keyword
+                  suggestions:
+                    type: completion
+                    contexts:
+                        -
+                          "name" : "format"
+                          "type" : "category"
+
+  - do:
+     bulk:
+        refresh: true
+        index: test
+        type: test
+        body:
+          - '{"index": {}}'
+          - '{"title": "Elasticsearch in Action", "suggestions": {"input": "ELK in Action", "contexts": {"format": "ebook"}}}'
+          - '{"index": {}}'
+          - '{"title": "Elasticsearch - The Definitive Guide", "suggestions": {"input": ["Elasticsearch in Action"]}}'
+
+---
+"Test typed keys parameter for suggesters":
+
+  - do:
+      search:
+        typed_keys: true
+        body:
+          query:
+            match_all: {}
+          suggest:
+            text: "Elastic"
+            term_suggester:
+              term:
+                field: title
+            completion_suggester:
+              prefix: "Elastic"
+              completion:
+                field: suggestions
+            context_suggester:
+              prefix: "Elastic"
+              completion:
+                field: suggestions
+                contexts:
+                  format: "ebook"
+            phrase_suggester:
+              phrase:
+                field: title
+
+  - is_true: suggest.term#term_suggester
+  - is_true: suggest.completion#completion_suggester
+  - is_true: suggest.completion#context_suggester
+  - is_true: suggest.phrase#phrase_suggester
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/tasks.get/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/tasks.get/10_basic.yaml
index 48d9f46ac73a0..fdb62cdce2e23 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/tasks.get/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/tasks.get/10_basic.yaml
@@ -1,5 +1,8 @@
 ---
 "get task test":
+  - skip:
+      version: " - 5.1.99"
+      reason: massage was changed in 5.2.0
   # Note that this gets much better testing in reindex's tests because it actually saves the task
   - do:
       cluster.state: {}
@@ -8,3 +11,4 @@
       catch: missing
       tasks.get:
         task_id: foo:1
+  - match: {error.reason: "/task.\\[foo:1\\].belongs.to.the.node.\\[foo\\].which.isn't.part.of.the.cluster.and.there.is.no.record.of.the.task/"}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/20_issue7121.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/20_issue7121.yaml
index cc04ea32a6a7b..168c913b75cb3 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/20_issue7121.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/20_issue7121.yaml
@@ -33,7 +33,7 @@
         index: testidx
         type:  doc
         id:    1
-        realtime: 0
+        realtime: false
 
   - match: { _index: "testidx" }
   - match: { _type: "doc" }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/30_realtime.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/30_realtime.yaml
index 12457861c98b8..26f441207ace8 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/30_realtime.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/30_realtime.yaml
@@ -26,7 +26,7 @@
           index:    test_1
           type:     test
           id:       1
-          realtime: 0
+          realtime: false
 
  - is_false: found
 
@@ -35,6 +35,6 @@
           index:    test_1
           type:     test
           id:       1
-          realtime: 1
+          realtime: true
 
  - is_true: found
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/45_force_versions.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/45_force_versions.yaml
new file mode 100644
index 0000000000000..1a41672348579
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/45_force_versions.yaml
@@ -0,0 +1,55 @@
+---
+"Force Version":
+ - skip:
+    features: "warnings"
+    version: " - 5.2.99"
+    reason: warning headers were introduced in 5.0.1 and their format changed in 5.3.0
+
+ - do:
+      index:
+          index:  test_1
+          type:   test
+          id:     1
+          body:   { foo: bar }
+ - match:   { _version: 1}
+
+ - do:
+      index:
+          index:  test_1
+          type:   test
+          id:     1
+          body:   { foo: bar }
+ - match:   { _version: 2}
+
+ - do:
+      warnings:
+          - version type FORCE is deprecated and will be removed in the next major version
+      get:
+          index:  test_1
+          type:   test
+          id:     1
+          version: 2
+          version_type: force
+ - match:   { _id: "1" }
+
+ - do:
+      warnings:
+          - version type FORCE is deprecated and will be removed in the next major version
+      get:
+          index:  test_1
+          type:   test
+          id:     1
+          version: 10
+          version_type: force
+ - match:   { _id: "1" }
+
+ - do:
+      warnings:
+          - version type FORCE is deprecated and will be removed in the next major version
+      get:
+          index:  test_1
+          type:   test
+          id:     1
+          version: 1
+          version_type: force
+ - match:   { _id: "1" }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/update/22_doc_as_upsert.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/update/22_doc_as_upsert.yaml
index af25730fd7aec..7585b9f3e0b94 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/update/22_doc_as_upsert.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/update/22_doc_as_upsert.yaml
@@ -8,7 +8,7 @@
           id:               1
           body:
             doc:            { foo: bar, count: 1 }
-            doc_as_upsert:  1
+            doc_as_upsert:  true
 
   - do:
       get:
@@ -27,7 +27,7 @@
           id:               1
           body:
             doc:            { count: 2 }
-            doc_as_upsert:  1
+            doc_as_upsert:  true
 
   - do:
       get:
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/update/40_routing.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/update/40_routing.yaml
index e96c175489e4f..097f49007e55f 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/update/40_routing.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/update/40_routing.yaml
@@ -30,7 +30,7 @@
           type:    test
           id:      1
           routing: 5
-          fields:  _routing
+          stored_fields:  _routing
 
  - match: { _routing:  "5"}
 
@@ -49,9 +49,9 @@
           type:    test
           id:      1
           routing: 5
-          fields:  foo
+          _source: foo
           body:
             doc:        { foo: baz }
 
- - match: { get.fields.foo: [baz] }
+ - match: { get._source.foo: baz }
 
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/update/50_parent.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/update/50_parent.yaml
index b25662dbf1487..82508f951e04c 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/update/50_parent.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/update/50_parent.yaml
@@ -36,7 +36,7 @@ setup:
           type:    test
           id:      1
           parent:  5
-          fields:  [_parent, _routing]
+          stored_fields:  [_parent, _routing]
 
  - match: { _parent:  "5"}
  - match: { _routing: "5"}
@@ -47,11 +47,11 @@ setup:
           type:    test
           id:      1
           parent:  5
-          fields:  foo
+          _source:  foo
           body:
             doc:        { foo: baz }
 
- - match: { get.fields.foo: [baz] }
+ - match: { get._source.foo: baz }
 
 ---
 "Parent omitted":
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/update/55_parent_with_routing.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/update/55_parent_with_routing.yaml
index 89dc83198c688..e75eddff9a87e 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/update/55_parent_with_routing.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/update/55_parent_with_routing.yaml
@@ -34,7 +34,7 @@
           id:      1
           routing: 4
           parent:  5
-          fields:  [_parent, _routing]
+          stored_fields:  [_parent, _routing]
 
  - match: { _parent:  "5"}
  - match: { _routing: "4"}
@@ -56,9 +56,9 @@
           id:      1
           parent:  5
           routing: 4
-          fields:  foo
+          _source:  foo
           body:
             doc:        { foo: baz }
 
- - match: { get.fields.foo: [baz] }
+ - match: { get._source.foo: baz }
 
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/update/80_fields.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/update/80_fields.yaml
deleted file mode 100644
index 86d6afa069dc9..0000000000000
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/update/80_fields.yaml
+++ /dev/null
@@ -1,20 +0,0 @@
----
-"Fields":
-
-  - do:
-      update:
-          index:  test_1
-          type:   test
-          id:     1
-          fields: foo,bar,_source
-          body:
-            doc:    { foo: baz }
-            upsert: { foo: bar }
-
-  - match:   { get._source.foo: bar }
-  - match:   { get.fields.foo:  [bar] }
-  - is_false:  get.fields.bar
-
-# TODO:
-#
-# - Add _routing
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/update/80_source_filtering.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/update/80_source_filtering.yaml
new file mode 100644
index 0000000000000..4bb22e6b8012e
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/update/80_source_filtering.yaml
@@ -0,0 +1,19 @@
+---
+"Source filtering":
+
+  - do:
+      update:
+          index:  test_1
+          type:   test
+          id:     1
+          _source: [foo, bar]
+          body:
+            doc:    { foo: baz }
+            upsert: { foo: bar }
+
+  - match:   { get._source.foo: bar }
+  - is_false:  get._source.bar
+
+# TODO:
+#
+# - Add _routing
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/update/85_fields_meta.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/update/85_fields_meta.yaml
index 5478d84e2a38b..7a6a58e12c0ff 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/update/85_fields_meta.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/update/85_fields_meta.yaml
@@ -33,6 +33,6 @@
           type:   test
           id:     1
           parent: 5
-          fields: [ _parent, _routing ]
+          stored_fields: [ _parent, _routing ]
 
 
diff --git a/settings.gradle b/settings.gradle
index 904fb69469d9f..aefb33866d992 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -1,4 +1,5 @@
-rootProject.name = 'elasticsearch'
+String dirName = rootProject.projectDir.name
+rootProject.name = dirName
 
 List projects = [
   'build-tools',
@@ -6,6 +7,7 @@ List projects = [
   'core',
   'docs',
   'client:rest',
+  'client:rest-high-level',
   'client:sniffer',
   'client:transport',
   'client:test',
@@ -13,13 +15,18 @@ List projects = [
   'client:benchmark',
   'benchmarks',
   'distribution:integ-test-zip',
+  'distribution:bwc',
   'distribution:zip',
   'distribution:tar',
   'distribution:deb',
   'distribution:rpm',
+  'distribution:tools:java-version-checker',
+  'distribution:tools:plugin-cli',
   'test:framework',
   'test:fixtures:example-fixture',
   'test:fixtures:hdfs-fixture',
+  'test:fixtures:krb5kdc-fixture',
+  'test:fixtures:old-elasticsearch',
   'test:logger-usage',
   'modules:aggs-matrix-stats',
   'modules:ingest-common',
@@ -31,13 +38,16 @@ List projects = [
   'modules:transport-netty4',
   'modules:reindex',
   'modules:percolator',
+  'modules:parent-join',
   'plugins:analysis-icu',
   'plugins:analysis-kuromoji',
   'plugins:analysis-phonetic',
   'plugins:analysis-smartcn',
   'plugins:analysis-stempel',
+  'plugins:analysis-ukrainian',
   'plugins:discovery-azure-classic',
   'plugins:discovery-ec2',
+  'plugins:discovery-file',
   'plugins:discovery-gce',
   'plugins:ingest-geoip',
   'plugins:ingest-attachment',
@@ -53,18 +63,35 @@ List projects = [
   'plugins:repository-s3',
   'plugins:jvm-example',
   'plugins:store-smb',
-  'qa:backwards-5.0',
+  'qa:auto-create-index',
   'qa:evil-tests',
+  'qa:mixed-cluster',
+  'qa:multi-cluster-search',
+  'qa:no-bootstrap-tests',
+  'qa:reindex-from-old',
+  'qa:rolling-upgrade',
   'qa:smoke-test-client',
+  'qa:smoke-test-http',
   'qa:smoke-test-ingest-with-all-dependencies',
   'qa:smoke-test-ingest-disabled',
   'qa:smoke-test-multinode',
   'qa:smoke-test-plugins',
-  'qa:smoke-test-reindex-with-painless',
-  'qa:smoke-test-http',
+  'qa:smoke-test-reindex-with-all-modules',
+  'qa:smoke-test-tribe-node',
   'qa:vagrant',
+  'qa:verify-version-constants',
+  'qa:wildfly'
 ]
 
+File examplePluginsDir = new File(rootProject.projectDir, 'plugins/examples')
+List<String> examplePlugins = []
+for (File example : examplePluginsDir.listFiles()) {
+  if (example.isDirectory() == false) continue;
+  if (example.name.startsWith('build') || example.name.startsWith('.')) continue;
+  projects.add("example-plugins:${example.name}".toString())
+  examplePlugins.add(example.name)
+}
+
 boolean isEclipse = System.getProperty("eclipse.launcher") != null || gradle.startParameter.taskNames.contains('eclipse') || gradle.startParameter.taskNames.contains('cleanEclipse')
 if (isEclipse) {
   // eclipse cannot handle an intermediate dependency between main and test, so we must create separate projects
@@ -75,6 +102,11 @@ if (isEclipse) {
 include projects.toArray(new String[0])
 
 project(':build-tools').projectDir = new File(rootProject.projectDir, 'buildSrc')
+project(':example-plugins').projectDir = new File(rootProject.projectDir, 'plugins/examples')
+
+for (String example : examplePlugins) {
+  project(":example-plugins:${example}").projectDir = new File(rootProject.projectDir, "plugins/examples/${example}")
+}
 
 if (isEclipse) {
   project(":core").projectDir = new File(rootProject.projectDir, 'core/src/main')
@@ -86,7 +118,7 @@ if (isEclipse) {
 /**
   * Iterates over sub directories, looking for build.gradle, and adds a project if found
   * for that dir with the given path prefix. Note that this requires each level
-  * of the dir hiearchy to have a build.gradle. Otherwise we would have to iterate
+  * of the dir hierarchy to have a build.gradle. Otherwise we would have to iterate
   * all files/directories in the source tree to find all projects.
   */
 void addSubProjects(String path, File dir) {
@@ -96,17 +128,18 @@ void addSubProjects(String path, File dir) {
 
   String projectName = "${path}:${dir.name}"
   include projectName
+  if (path.isEmpty()) {
+    project(projectName).projectDir = dir
+  }
   for (File subdir : dir.listFiles()) {
     addSubProjects(projectName, subdir)
   }
 }
 
 // look for extra plugins for elasticsearch
-File xplugins = new File(rootProject.projectDir.parentFile, 'x-plugins')
-if (xplugins.exists()) {
-  include ':x-plugins'
-  project(':x-plugins').projectDir = xplugins
-  for (File extraPluginDir : xplugins.listFiles()) {
-    addSubProjects(':x-plugins', extraPluginDir)
+File extraProjects = new File(rootProject.projectDir.parentFile, "${dirName}-extra")
+if (extraProjects.exists()) {
+  for (File extraProjectDir : extraProjects.listFiles()) {
+    addSubProjects('', extraProjectDir)
   }
 }
diff --git a/test/fixtures/example-fixture/build.gradle b/test/fixtures/example-fixture/build.gradle
index 17a4586a54d36..225a2cf9deba6 100644
--- a/test/fixtures/example-fixture/build.gradle
+++ b/test/fixtures/example-fixture/build.gradle
@@ -19,3 +19,6 @@
 
 apply plugin: 'elasticsearch.build'
 test.enabled = false
+// Not published so no need to assemble
+tasks.remove(assemble)
+build.dependsOn.remove('assemble')
diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle
index 3d63939f66ecb..70cb2b6bed2cb 100644
--- a/test/fixtures/hdfs-fixture/build.gradle
+++ b/test/fixtures/hdfs-fixture/build.gradle
@@ -20,7 +20,7 @@
 apply plugin: 'elasticsearch.build'
 
 versions << [
-  'hadoop2': '2.7.1'
+  'hadoop2': '2.8.1'
 ]
 
 // we create MiniHdfsCluster with the hadoop artifact
diff --git a/test/fixtures/hdfs-fixture/src/main/java/hdfs/MiniHDFS.java b/test/fixtures/hdfs-fixture/src/main/java/hdfs/MiniHDFS.java
index a4bf47f8eae87..73f4e443b0769 100644
--- a/test/fixtures/hdfs-fixture/src/main/java/hdfs/MiniHDFS.java
+++ b/test/fixtures/hdfs-fixture/src/main/java/hdfs/MiniHDFS.java
@@ -19,19 +19,29 @@
 
 package hdfs;
 
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-
-import java.io.IOException;
+import java.io.File;
+import java.lang.management.ManagementFactory;
+import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.StandardCopyOption;
-import java.util.Locale;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
 
-import java.lang.management.ManagementFactory;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.permission.AclEntry;
+import org.apache.hadoop.fs.permission.AclEntryType;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.security.UserGroupInformation;
 
 /**
  * MiniHDFS test fixture. There is a CLI tool, but here we can
@@ -43,9 +53,12 @@ public class MiniHDFS {
     private static String PID_FILE_NAME = "pid";
 
     public static void main(String[] args) throws Exception {
-        if (args.length != 1) {
-           throw new IllegalArgumentException("MiniHDFS <baseDirectory>");
+        if (args.length != 1 && args.length != 3) {
+            throw new IllegalArgumentException("Expected: MiniHDFS <baseDirectory> [<kerberosPrincipal> <kerberosKeytab>], " +
+                "got: " + Arrays.toString(args));
         }
+        boolean secure = args.length == 3;
+
         // configure Paths
         Path baseDir = Paths.get(args[0]);
         // hadoop-home/, so logs will not complain
@@ -57,13 +70,70 @@ public static void main(String[] args) throws Exception {
         // hdfs-data/, where any data is going
         Path hdfsHome = baseDir.resolve("hdfs-data");
 
-        // start cluster
+        // configure cluster
         Configuration cfg = new Configuration();
         cfg.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, hdfsHome.toAbsolutePath().toString());
         // lower default permission: TODO: needed?
         cfg.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_PERMISSION_KEY, "766");
+
+        // optionally configure security
+        if (secure) {
+            String kerberosPrincipal = args[1];
+            String keytabFile = args[2];
+
+            cfg.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
+            cfg.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, "true");
+            cfg.set(DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY, kerberosPrincipal);
+            cfg.set(DFSConfigKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY, kerberosPrincipal);
+            cfg.set(DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY, kerberosPrincipal);
+            cfg.set(DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY, keytabFile);
+            cfg.set(DFSConfigKeys.DFS_DATANODE_KEYTAB_FILE_KEY, keytabFile);
+            cfg.set(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, "true");
+            cfg.set(DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, "true");
+            cfg.set(DFSConfigKeys.IGNORE_SECURE_PORTS_FOR_TESTING_KEY, "true");
+        }
+
+        UserGroupInformation.setConfiguration(cfg);
+
         // TODO: remove hardcoded port!
-        MiniDFSCluster dfs = new MiniDFSCluster.Builder(cfg).nameNodePort(9999).build();
+        MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(cfg);
+        if (secure) {
+            builder.nameNodePort(9998);
+        } else {
+            builder.nameNodePort(9999);
+        }
+        MiniDFSCluster dfs = builder.build();
+
+        // Configure contents of the filesystem
+        org.apache.hadoop.fs.Path esUserPath = new org.apache.hadoop.fs.Path("/user/elasticsearch");
+        try (FileSystem fs = dfs.getFileSystem()) {
+
+            // Set the elasticsearch user directory up
+            fs.mkdirs(esUserPath);
+            if (UserGroupInformation.isSecurityEnabled()) {
+                List<AclEntry> acls = new ArrayList<>();
+                acls.add(new AclEntry.Builder().setType(AclEntryType.USER).setName("elasticsearch").setPermission(FsAction.ALL).build());
+                fs.modifyAclEntries(esUserPath, acls);
+            }
+
+            // Install a pre-existing repository into HDFS
+            String directoryName = "readonly-repository";
+            String archiveName = directoryName + ".tar.gz";
+            URL readOnlyRepositoryArchiveURL = MiniHDFS.class.getClassLoader().getResource(archiveName);
+            if (readOnlyRepositoryArchiveURL != null) {
+                Path tempDirectory = Files.createTempDirectory(MiniHDFS.class.getName());
+                File readOnlyRepositoryArchive = tempDirectory.resolve(archiveName).toFile();
+                FileUtils.copyURLToFile(readOnlyRepositoryArchiveURL, readOnlyRepositoryArchive);
+                FileUtil.unTar(readOnlyRepositoryArchive, tempDirectory.toFile());
+
+                fs.copyFromLocalFile(true, true,
+                    new org.apache.hadoop.fs.Path(tempDirectory.resolve(directoryName).toAbsolutePath().toUri()),
+                    esUserPath.suffix("/existing/" + directoryName)
+                );
+
+                FileUtils.deleteDirectory(tempDirectory.toFile());
+            }
+        }
 
         // write our PID file
         Path tmp = Files.createTempFile(baseDir, null, null);
diff --git a/test/fixtures/hdfs-fixture/src/main/resources/readonly-repository.tar.gz b/test/fixtures/hdfs-fixture/src/main/resources/readonly-repository.tar.gz
new file mode 100644
index 0000000000000..2cdb6d77c07d0
Binary files /dev/null and b/test/fixtures/hdfs-fixture/src/main/resources/readonly-repository.tar.gz differ
diff --git a/test/fixtures/krb5kdc-fixture/Vagrantfile b/test/fixtures/krb5kdc-fixture/Vagrantfile
new file mode 100644
index 0000000000000..72be4dad9cbe5
--- /dev/null
+++ b/test/fixtures/krb5kdc-fixture/Vagrantfile
@@ -0,0 +1,53 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This Vagrantfile exists to define a virtual machine running MIT's Kerberos 5
+# for usage as a testing fixture for the build process.
+#
+# In order to connect to the KDC process on this virtual machine, find and use
+# the rendered krb5.conf file in the build output directory (build/conf).
+#
+# In order to provision principals in the KDC, use the provided addprinc.sh
+# script with vagrant's ssh facility:
+#
+#    vagrant ssh -c /vagrant/src/main/resources/provision/addprinc.sh principal
+#
+# You will find the newly created principal's keytab file in the build output
+# directory (build/keytabs). Principal creation is idempotent, and will recopy
+# existing user keytabs from the KDC if they already exist.
+
+Vagrant.configure("2") do |config|
+
+  config.vm.define "krb5kdc" do |config|
+    config.vm.box = "elastic/ubuntu-14.04-x86_64"
+  end
+
+  config.vm.hostname = "kerberos.build.elastic.co"
+
+  if Vagrant.has_plugin?("vagrant-cachier")
+    config.cache.scope = :box
+  end
+
+  config.vm.network "forwarded_port", guest: 88, host: 60088, protocol: "tcp"
+  config.vm.network "forwarded_port", guest: 88, host: 60088, protocol: "udp"
+
+  config.vm.provision "shell", path: "src/main/resources/provision/installkdc.sh"
+
+end
diff --git a/test/fixtures/krb5kdc-fixture/build.gradle b/test/fixtures/krb5kdc-fixture/build.gradle
new file mode 100644
index 0000000000000..046ecc5fc0416
--- /dev/null
+++ b/test/fixtures/krb5kdc-fixture/build.gradle
@@ -0,0 +1,84 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+apply plugin: 'elasticsearch.build'
+
+Map<String, String> vagrantEnvVars = [
+        'VAGRANT_CWD'           : "${project.projectDir.absolutePath}",
+        'VAGRANT_VAGRANTFILE'   : 'Vagrantfile',
+        'VAGRANT_PROJECT_DIR'   : "${project.projectDir.absolutePath}"
+]
+
+String box = "krb5kdc"
+
+List<String> defaultPrincipals = [ "elasticsearch" ]
+
+task update(type: org.elasticsearch.gradle.vagrant.VagrantCommandTask) {
+    command 'box'
+    subcommand 'update'
+    boxName box
+    environmentVars vagrantEnvVars
+}
+
+task up(type: org.elasticsearch.gradle.vagrant.VagrantCommandTask) {
+    command 'up'
+    args '--provision', '--provider', 'virtualbox'
+    boxName box
+    environmentVars vagrantEnvVars
+    dependsOn update
+}
+
+task addDefaultPrincipals {
+    dependsOn up
+}
+
+for (String principal : defaultPrincipals) {
+    Task addTask = project.tasks.create("addPrincipal#${principal}", org.elasticsearch.gradle.vagrant.VagrantCommandTask) {
+        command 'ssh'
+        args '--command', "sudo bash /vagrant/src/main/resources/provision/addprinc.sh $principal"
+        boxName box
+        environmentVars vagrantEnvVars
+        dependsOn up
+    }
+    addDefaultPrincipals.dependsOn(addTask)
+}
+
+task halt(type: org.elasticsearch.gradle.vagrant.VagrantCommandTask) {
+    command 'halt'
+    boxName box
+    environmentVars vagrantEnvVars
+}
+
+task destroy(type: org.elasticsearch.gradle.vagrant.VagrantCommandTask) {
+    command 'destroy'
+    args '-f'
+    boxName box
+    environmentVars vagrantEnvVars
+    dependsOn halt
+}
+
+thirdPartyAudit.enabled = false
+licenseHeaders.enabled = false
+test.enabled = false
+
+// installKDC uses tabs in it for the Kerberos ACL file.
+// Ignore it for pattern checking.
+forbiddenPatterns {
+    exclude "**/installkdc.sh"
+}
diff --git a/test/fixtures/krb5kdc-fixture/src/main/resources/env.properties b/test/fixtures/krb5kdc-fixture/src/main/resources/env.properties
new file mode 100644
index 0000000000000..21ef7e1225af9
--- /dev/null
+++ b/test/fixtures/krb5kdc-fixture/src/main/resources/env.properties
@@ -0,0 +1,20 @@
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+realm=BUILD.ELASTIC.CO
+kdc=kerberos.build.elastic.co
+zone=build.elastic.co
\ No newline at end of file
diff --git a/test/fixtures/krb5kdc-fixture/src/main/resources/provision/addprinc.sh b/test/fixtures/krb5kdc-fixture/src/main/resources/provision/addprinc.sh
new file mode 100755
index 0000000000000..137135dc2aa4d
--- /dev/null
+++ b/test/fixtures/krb5kdc-fixture/src/main/resources/provision/addprinc.sh
@@ -0,0 +1,58 @@
+#!/bin/bash
+
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+if [[ $# -lt 1 ]]; then
+  echo 'Usage: addprinc.sh <principalNameNoRealm>'
+  exit 1
+fi
+
+PRINC="$1"
+USER=$(echo $PRINC | tr "/" "_")
+
+VDIR=/vagrant
+RESOURCES=$VDIR/src/main/resources
+PROV_DIR=$RESOURCES/provision
+ENVPROP_FILE=$RESOURCES/env.properties
+BUILD_DIR=$VDIR/build
+CONF_DIR=$BUILD_DIR/conf
+KEYTAB_DIR=$BUILD_DIR/keytabs
+LOCALSTATEDIR=/etc
+LOGDIR=/var/log/krb5
+
+mkdir -p $KEYTAB_DIR
+
+REALM=$(cat $ENVPROP_FILE | grep realm= | head -n 1 | cut -d '=' -f 2)
+
+ADMIN_PRIN=admin/admin@$REALM
+ADMIN_KTAB=$LOCALSTATEDIR/admin.keytab
+
+USER_PRIN=$PRINC@$REALM
+USER_KTAB=$LOCALSTATEDIR/$USER.keytab
+
+if [ -f $USER_KTAB ]; then
+  echo "Principal '${PRINC}@${REALM}' already exists. Re-copying keytab..."
+else
+  echo "Provisioning '${PRINC}@${REALM}' principal and keytab..."
+  sudo kadmin -p $ADMIN_PRIN -kt $ADMIN_KTAB -q "addprinc -randkey $USER_PRIN"
+  sudo kadmin -p $ADMIN_PRIN -kt $ADMIN_KTAB -q "ktadd -k $USER_KTAB $USER_PRIN"
+fi
+
+sudo cp $USER_KTAB $KEYTAB_DIR/$USER.keytab
diff --git a/test/fixtures/krb5kdc-fixture/src/main/resources/provision/installkdc.sh b/test/fixtures/krb5kdc-fixture/src/main/resources/provision/installkdc.sh
new file mode 100755
index 0000000000000..2dc8ed92c9462
--- /dev/null
+++ b/test/fixtures/krb5kdc-fixture/src/main/resources/provision/installkdc.sh
@@ -0,0 +1,120 @@
+#!/bin/bash
+
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+# KDC installation steps and considerations based on https://web.mit.edu/kerberos/krb5-latest/doc/admin/install_kdc.html
+# and helpful input from https://help.ubuntu.com/community/Kerberos
+
+VDIR=/vagrant
+RESOURCES=$VDIR/src/main/resources
+PROV_DIR=$RESOURCES/provision
+ENVPROP_FILE=$RESOURCES/env.properties
+BUILD_DIR=$VDIR/build
+CONF_DIR=$BUILD_DIR/conf
+KEYTAB_DIR=$BUILD_DIR/keytabs
+LOCALSTATEDIR=/etc
+LOGDIR=/var/log/krb5
+
+MARKER_FILE=/etc/marker
+
+# Output location for our rendered configuration files and keytabs
+mkdir -p $BUILD_DIR
+rm -rf $BUILD_DIR/*
+mkdir -p $CONF_DIR
+mkdir -p $KEYTAB_DIR
+
+if [ -f $MARKER_FILE ]; then
+  echo "Already provisioned..."
+  echo "Recopying configuration files..."
+  cp $LOCALSTATEDIR/krb5.conf $CONF_DIR/krb5.conf
+  cp $LOCALSTATEDIR/krb5kdc/kdc.conf $CONF_DIR/kdc.conf
+  exit 0;
+fi
+
+# Pull environment information
+REALM_NAME=$(cat $ENVPROP_FILE | grep realm= | cut -d '=' -f 2)
+KDC_NAME=$(cat $ENVPROP_FILE | grep kdc= | cut -d '=' -f 2)
+BUILD_ZONE=$(cat $ENVPROP_FILE | grep zone= | cut -d '=' -f 2)
+ELASTIC_ZONE=$(echo $BUILD_ZONE | cut -d '.' -f 1,2)
+
+# Transfer and interpolate krb5.conf
+cp $PROV_DIR/krb5.conf.template $LOCALSTATEDIR/krb5.conf
+sed -i 's/${REALM_NAME}/'$REALM_NAME'/g' $LOCALSTATEDIR/krb5.conf
+sed -i 's/${KDC_NAME}/'$KDC_NAME'/g' $LOCALSTATEDIR/krb5.conf
+sed -i 's/${BUILD_ZONE}/'$BUILD_ZONE'/g' $LOCALSTATEDIR/krb5.conf
+sed -i 's/${ELASTIC_ZONE}/'$ELASTIC_ZONE'/g' $LOCALSTATEDIR/krb5.conf
+cp $LOCALSTATEDIR/krb5.conf $CONF_DIR/krb5.conf
+
+# Transfer and interpolate the kdc.conf
+mkdir -p $LOCALSTATEDIR/krb5kdc
+cp $PROV_DIR/kdc.conf.template $LOCALSTATEDIR/krb5kdc/kdc.conf
+sed -i 's/${REALM_NAME}/'$REALM_NAME'/g' $LOCALSTATEDIR/krb5kdc/kdc.conf
+sed -i 's/${KDC_NAME}/'$KDC_NAME'/g' $LOCALSTATEDIR/krb5kdc/kdc.conf
+sed -i 's/${BUILD_ZONE}/'$BUILD_ZONE'/g' $LOCALSTATEDIR/krb5kdc/kdc.conf
+sed -i 's/${ELASTIC_ZONE}/'$ELASTIC_ZONE'/g' $LOCALSTATEDIR/krb5.conf
+cp $LOCALSTATEDIR/krb5kdc/kdc.conf $CONF_DIR/kdc.conf
+
+# Touch logging locations
+mkdir -p $LOGDIR
+touch $LOGDIR/kadmin.log
+touch $LOGDIR/krb5kdc.log
+touch $LOGDIR/krb5lib.log
+
+# Update package manager
+apt-get update -qqy
+
+# Installation asks a bunch of questions via debconf. Set the answers ahead of time
+debconf-set-selections <<< "krb5-config krb5-config/read_conf boolean true"
+debconf-set-selections <<< "krb5-config krb5-config/kerberos_servers string $KDC_NAME"
+debconf-set-selections <<< "krb5-config krb5-config/add_servers boolean true"
+debconf-set-selections <<< "krb5-config krb5-config/admin_server string $KDC_NAME"
+debconf-set-selections <<< "krb5-config krb5-config/add_servers_realm string $REALM_NAME"
+debconf-set-selections <<< "krb5-config krb5-config/default_realm string $REALM_NAME"
+debconf-set-selections <<< "krb5-admin-server krb5-admin-server/kadmind boolean true"
+debconf-set-selections <<< "krb5-admin-server krb5-admin-server/newrealm note"
+debconf-set-selections <<< "krb5-kdc krb5-kdc/debconf boolean true"
+debconf-set-selections <<< "krb5-kdc krb5-kdc/purge_data_too boolean false"
+
+# Install krb5 packages
+apt-get install -qqy krb5-{admin-server,kdc}
+
+# /dev/random produces output very slowly on Ubuntu VM's. Install haveged to increase entropy.
+apt-get install -qqy haveged
+haveged
+
+# Create kerberos database with stash file and garbage password
+kdb5_util create -s -r $REALM_NAME -P zyxwvutsrpqonmlk9876
+
+# Set up admin acls
+cat << EOF > /etc/krb5kdc/kadm5.acl
+*/admin@$REALM_NAME	*
+*/*@$REALM_NAME		i
+EOF
+
+# Create admin principal
+kadmin.local -q "addprinc -pw elastic admin/admin@$REALM_NAME"
+kadmin.local -q "ktadd -k /etc/admin.keytab admin/admin@$REALM_NAME"
+
+# Start Kerberos Services
+krb5kdc
+kadmind
+
+# Mark that the vm is already provisioned
+touch $MARKER_FILE
\ No newline at end of file
diff --git a/test/fixtures/krb5kdc-fixture/src/main/resources/provision/kdc.conf.template b/test/fixtures/krb5kdc-fixture/src/main/resources/provision/kdc.conf.template
new file mode 100644
index 0000000000000..22909ddf60013
--- /dev/null
+++ b/test/fixtures/krb5kdc-fixture/src/main/resources/provision/kdc.conf.template
@@ -0,0 +1,35 @@
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[kdcdefaults]
+    kdc_listen = 88
+    kdc_tcp_listen = 88
+
+[realms]
+    ${REALM_NAME} = {
+        kadmind_port = 749
+        max_life = 12h 0m 0s
+        max_renewable_life = 7d 0h 0m 0s
+        master_key_type = aes256-cts
+        # remove aes256-cts:normal since unlimited strength policy needs installed for java to use it.
+        supported_enctypes = aes128-cts:normal des3-hmac-sha1:normal arcfour-hmac:normal des-hmac-sha1:normal des-cbc-md5:normal des-cbc-crc:normal
+    }
+
+[logging]
+    kdc = FILE:/var/log/krb5kdc.log
+    admin_server = FILE:/var/log/kadmin.log
+    default = FILE:/var/log/krb5lib.log
diff --git a/test/fixtures/krb5kdc-fixture/src/main/resources/provision/krb5.conf.template b/test/fixtures/krb5kdc-fixture/src/main/resources/provision/krb5.conf.template
new file mode 100644
index 0000000000000..e572c12e70957
--- /dev/null
+++ b/test/fixtures/krb5kdc-fixture/src/main/resources/provision/krb5.conf.template
@@ -0,0 +1,50 @@
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[libdefaults]
+    default_realm = ${REALM_NAME}
+    dns_canonicalize_hostname = false
+    dns_lookup_kdc = false
+    dns_lookup_realm = false
+    dns_uri_lookup = false
+    forwardable = true
+    ignore_acceptor_hostname = true
+    rdns = false
+    default_tgs_enctypes = rc4-hmac
+    default_tkt_enctypes = rc4-hmac
+    permitted_enctypes = rc4-hmac
+    # udp_preference_limit = 1
+    kdc_timeout = 3000
+
+[realms]
+    ${REALM_NAME} = {
+        kdc = ${KDC_NAME}:88
+        kdc = ${KDC_NAME}:60088
+        kdc = localhost:60088
+        kdc = localhost:88
+        kdc = 127.0.0.1:60088
+        kdc = 127.0.0.1:88
+        admin_server = ${KDC_NAME}:749
+        default_domain = ${BUILD_ZONE}
+    }
+
+[domain_realm]
+    ${BUILD_ZONE} = ${REALM_NAME}
+    .${BUILD_ZONE} = ${REALM_NAME}
+    ${ELASTIC_ZONE} = ${REALM_NAME}
+    .${ELASTIC_ZONE} = ${REALM_NAME}
+
diff --git a/test/fixtures/old-elasticsearch/build.gradle b/test/fixtures/old-elasticsearch/build.gradle
new file mode 100644
index 0000000000000..5cfc02bbba3c6
--- /dev/null
+++ b/test/fixtures/old-elasticsearch/build.gradle
@@ -0,0 +1,32 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+description = """\
+Launches versions of Elasticsearch prior to 5.0 for testing.
+These need special handling because they do not support writing
+a "ports" file with the port on which Elasticsearch is running.
+"""
+
+apply plugin: 'elasticsearch.build'
+test.enabled = false
+
+dependencies {
+  // Just for the constants....
+  compile "org.apache.lucene:lucene-core:${versions.lucene}"
+}
diff --git a/test/fixtures/old-elasticsearch/src/main/java/oldes/OldElasticsearch.java b/test/fixtures/old-elasticsearch/src/main/java/oldes/OldElasticsearch.java
new file mode 100644
index 0000000000000..bd4c3f8ccd105
--- /dev/null
+++ b/test/fixtures/old-elasticsearch/src/main/java/oldes/OldElasticsearch.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package oldes;
+
+import org.apache.lucene.util.Constants;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Starts a version of Elasticsearch that has been unzipped into an empty directory,
+ * instructing it to ask the OS for an unused port, grepping the logs for the port
+ * it actually got, and writing a {@code ports} file with the port. This is only
+ * required for versions of Elasticsearch before 5.0 because they do not support
+ * writing a "ports" file.
+ */
+public class OldElasticsearch {
+    public static void main(String[] args) throws IOException {
+        Path baseDir = Paths.get(args[0]);
+        Path unzipDir = Paths.get(args[1]);
+
+        // 0.90 must be explicitly foregrounded
+        boolean explicitlyForeground;
+        switch (args[2]) {
+        case "true":
+            explicitlyForeground = true;
+            break;
+        case "false":
+            explicitlyForeground = false;
+            break;
+        default:
+            System.err.println("the third argument must be true or false");
+            System.exit(1);
+            return;
+        }
+
+        Iterator<Path> children = Files.list(unzipDir).iterator();
+        if (false == children.hasNext()) {
+            System.err.println("expected the es directory to contain a single child directory but contained none.");
+            System.exit(1);
+        }
+        Path esDir = children.next();
+        if (children.hasNext()) {
+            System.err.println("expected the es directory to contains a single child directory but contained [" + esDir + "] and ["
+                    + children.next() + "].");
+            System.exit(1);
+        }
+        if (false == Files.isDirectory(esDir)) {
+            System.err.println("expected the es directory to contains a single child directory but contained a single child file.");
+            System.exit(1);
+        }
+
+        Path bin = esDir.resolve("bin").resolve("elasticsearch" + (Constants.WINDOWS ? ".bat" : ""));
+        Path config = esDir.resolve("config").resolve("elasticsearch.yml");
+
+        Files.write(config, Arrays.asList("http.port: 0", "transport.tcp.port: 0", "network.host: 127.0.0.1"), StandardCharsets.UTF_8);
+
+        List<String> command = new ArrayList<>();
+        command.add(bin.toString());
+        if (explicitlyForeground) {
+            command.add("-f");
+        }
+        command.add("-p");
+        command.add("../pid");
+        ProcessBuilder subprocess = new ProcessBuilder(command);
+        Process process = subprocess.start();
+        System.out.println("Running " + command);
+
+        int pid = 0;
+        int port = 0;
+
+        Pattern pidPattern = Pattern.compile("pid\\[(\\d+)\\]");
+        Pattern httpPortPattern = Pattern.compile("\\[http\\s+\\].+bound_address.+127\\.0\\.0\\.1:(\\d+)");
+        try (BufferedReader stdout = new BufferedReader(new InputStreamReader(process.getInputStream(), StandardCharsets.UTF_8))) {
+            String line;
+            while ((line = stdout.readLine()) != null && (pid == 0 || port == 0)) {
+                System.out.println(line);
+                Matcher m = pidPattern.matcher(line);
+                if (m.find()) {
+                    pid = Integer.parseInt(m.group(1));
+                    System.out.println("Found pid:  " + pid);
+                    continue;
+                }
+                m = httpPortPattern.matcher(line);
+                if (m.find()) {
+                    port = Integer.parseInt(m.group(1));
+                    System.out.println("Found port:  " + port);
+                    continue;
+                }
+            }
+        }
+
+        if (port == 0) {
+            System.err.println("port not found");
+            System.exit(1);
+        }
+
+        Path tmp = Files.createTempFile(baseDir, null, null);
+        Files.write(tmp, Integer.toString(port).getBytes(StandardCharsets.UTF_8));
+        Files.move(tmp, baseDir.resolve("ports"), StandardCopyOption.ATOMIC_MOVE);
+    }
+}
diff --git a/test/framework/build.gradle b/test/framework/build.gradle
index 8be06574df8ab..ae610b4ec0b29 100644
--- a/test/framework/build.gradle
+++ b/test/framework/build.gradle
@@ -20,13 +20,13 @@
 import org.elasticsearch.gradle.precommit.PrecommitTasks;
 
 dependencies {
+  compile "org.elasticsearch.client:elasticsearch-rest-client:${version}"
   compile "org.elasticsearch:elasticsearch:${version}"
   compile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
   compile "junit:junit:${versions.junit}"
   compile "org.hamcrest:hamcrest-all:${versions.hamcrest}"
   compile "org.apache.lucene:lucene-test-framework:${versions.lucene}"
   compile "org.apache.lucene:lucene-codecs:${versions.lucene}"
-  compile "org.elasticsearch.client:rest:${version}"
   compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
   compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
   compile "commons-logging:commons-logging:${versions.commonslogging}"
@@ -62,3 +62,8 @@ thirdPartyAudit.excludes = [
   'org.easymock.IArgumentMatcher',
   'org.jmock.core.Constraint',
 ]
+
+task namingConventionsMain(type: org.elasticsearch.gradle.precommit.NamingConventionsTask) {
+  checkForTestsInMain = true
+}
+precommit.dependsOn namingConventionsMain
diff --git a/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java b/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java
index c752563b80420..ccdba9f0388a0 100644
--- a/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java
@@ -19,6 +19,13 @@
 
 package org.elasticsearch;
 
+import org.apache.lucene.analysis.en.PorterStemFilterFactory;
+import org.apache.lucene.analysis.miscellaneous.WordDelimiterGraphFilterFactory;
+import org.apache.lucene.analysis.reverse.ReverseStringFilterFactory;
+import org.apache.lucene.analysis.snowball.SnowballPorterFilterFactory;
+import org.apache.lucene.analysis.util.CharFilterFactory;
+import org.apache.lucene.analysis.util.TokenFilterFactory;
+import org.apache.lucene.analysis.util.TokenizerFactory;
 import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.index.analysis.ASCIIFoldingTokenFilterFactory;
 import org.elasticsearch.index.analysis.ApostropheFilterFactory;
@@ -36,6 +43,7 @@
 import org.elasticsearch.index.analysis.EdgeNGramTokenFilterFactory;
 import org.elasticsearch.index.analysis.EdgeNGramTokenizerFactory;
 import org.elasticsearch.index.analysis.ElisionTokenFilterFactory;
+import org.elasticsearch.index.analysis.FlattenGraphTokenFilterFactory;
 import org.elasticsearch.index.analysis.GermanNormalizationFilterFactory;
 import org.elasticsearch.index.analysis.GermanStemTokenFilterFactory;
 import org.elasticsearch.index.analysis.HindiNormalizationFilterFactory;
@@ -76,6 +84,7 @@
 import org.elasticsearch.index.analysis.StemmerOverrideTokenFilterFactory;
 import org.elasticsearch.index.analysis.StemmerTokenFilterFactory;
 import org.elasticsearch.index.analysis.StopTokenFilterFactory;
+import org.elasticsearch.index.analysis.SynonymGraphTokenFilterFactory;
 import org.elasticsearch.index.analysis.SynonymTokenFilterFactory;
 import org.elasticsearch.index.analysis.ThaiTokenizerFactory;
 import org.elasticsearch.index.analysis.TrimTokenFilterFactory;
@@ -86,13 +95,20 @@
 import org.elasticsearch.index.analysis.WordDelimiterTokenFilterFactory;
 import org.elasticsearch.index.analysis.compound.DictionaryCompoundWordTokenFilterFactory;
 import org.elasticsearch.index.analysis.compound.HyphenationCompoundWordTokenFilterFactory;
+import org.elasticsearch.indices.analysis.PreBuiltCharFilters;
+import org.elasticsearch.indices.analysis.PreBuiltTokenFilters;
+import org.elasticsearch.indices.analysis.PreBuiltTokenizers;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.Collection;
+import java.util.EnumMap;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeSet;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 /**
  * Alerts us if new analyzers are added to lucene, so we don't miss them.
@@ -100,7 +116,20 @@
  * If we don't want to expose one for a specific reason, just map it to Void.
  * The deprecated ones can be mapped to Deprecated.class.
  */
-public class AnalysisFactoryTestCase extends ESTestCase {
+public abstract class AnalysisFactoryTestCase extends ESTestCase {
+
+    private static final Pattern UNDERSCORE_THEN_ANYTHING = Pattern.compile("_(.)");
+
+    private static String toCamelCase(String s) {
+        Matcher m = UNDERSCORE_THEN_ANYTHING.matcher(s);
+        StringBuffer sb = new StringBuffer();
+        while (m.find()) {
+            m.appendReplacement(sb, m.group(1).toUpperCase());
+        }
+        m.appendTail(sb);
+        sb.setCharAt(0, Character.toUpperCase(sb.charAt(0)));
+        return sb.toString();
+    }
 
     static final Map<String,Class<?>> KNOWN_TOKENIZERS = new MapBuilder<String,Class<?>>()
         // exposed in ES
@@ -119,8 +148,32 @@ public class AnalysisFactoryTestCase extends ESTestCase {
 
         // this one "seems to mess up offsets". probably shouldn't be a tokenizer...
         .put("wikipedia",     Void.class)
+
+        // TODO: expose these
+        .put("simplepattern",    Void.class)
+        .put("simplepatternsplit",    Void.class)
         .immutableMap();
 
+    static final Map<PreBuiltTokenizers, Class<?>> PREBUILT_TOKENIZERS;
+    static {
+        PREBUILT_TOKENIZERS = new EnumMap<>(PreBuiltTokenizers.class);
+        for (PreBuiltTokenizers tokenizer : PreBuiltTokenizers.values()) {
+            Class<?> luceneFactoryClazz;
+            switch (tokenizer) {
+            case UAX_URL_EMAIL:
+                luceneFactoryClazz = org.apache.lucene.analysis.standard.UAX29URLEmailTokenizerFactory.class;
+                break;
+            case PATH_HIERARCHY:
+                luceneFactoryClazz = Void.class;
+                break;
+            default:
+                luceneFactoryClazz = org.apache.lucene.analysis.util.TokenizerFactory.lookupClass(
+                        toCamelCase(tokenizer.getTokenizerFactory(Version.CURRENT).name()));
+            }
+            PREBUILT_TOKENIZERS.put(tokenizer, luceneFactoryClazz);
+        }
+    }
+
     static final Map<String,Class<?>> KNOWN_TOKENFILTERS = new MapBuilder<String,Class<?>>()
         // exposed in ES
         .put("apostrophe",                ApostropheFilterFactory.class)
@@ -195,12 +248,15 @@ public class AnalysisFactoryTestCase extends ESTestCase {
         .put("stop",                      StopTokenFilterFactory.class)
         .put("swedishlightstem",          StemmerTokenFilterFactory.class)
         .put("synonym",                   SynonymTokenFilterFactory.class)
+        .put("synonymgraph",              SynonymGraphTokenFilterFactory.class)
         .put("trim",                      TrimTokenFilterFactory.class)
         .put("truncate",                  TruncateTokenFilterFactory.class)
         .put("turkishlowercase",          LowerCaseTokenFilterFactory.class)
         .put("type",                      KeepTypesFilterFactory.class)
         .put("uppercase",                 UpperCaseTokenFilterFactory.class)
         .put("worddelimiter",             WordDelimiterTokenFilterFactory.class)
+        .put("worddelimitergraph",        WordDelimiterGraphFilterFactory.class)
+        .put("flattengraph",              FlattenGraphTokenFilterFactory.class)
 
         // TODO: these tokenfilters are not yet exposed: useful?
 
@@ -233,6 +289,41 @@ public class AnalysisFactoryTestCase extends ESTestCase {
 
         .immutableMap();
 
+    static final Map<PreBuiltTokenFilters, Class<?>> PREBUILT_TOKENFILTERS;
+    static {
+        PREBUILT_TOKENFILTERS = new EnumMap<>(PreBuiltTokenFilters.class);
+        for (PreBuiltTokenFilters tokenizer : PreBuiltTokenFilters.values()) {
+            Class<?> luceneFactoryClazz;
+            switch (tokenizer) {
+            case REVERSE:
+                luceneFactoryClazz = ReverseStringFilterFactory.class;
+                break;
+            case UNIQUE:
+                luceneFactoryClazz = Void.class;
+                break;
+            case SNOWBALL:
+            case DUTCH_STEM:
+            case FRENCH_STEM:
+            case RUSSIAN_STEM:
+                luceneFactoryClazz = SnowballPorterFilterFactory.class;
+                break;
+            case STEMMER:
+                luceneFactoryClazz = PorterStemFilterFactory.class;
+                break;
+            case DELIMITED_PAYLOAD_FILTER:
+                luceneFactoryClazz = org.apache.lucene.analysis.payloads.DelimitedPayloadTokenFilterFactory.class;
+                 break;
+            case LIMIT:
+                luceneFactoryClazz = org.apache.lucene.analysis.miscellaneous.LimitTokenCountFilterFactory.class;
+                break;
+            default:
+                luceneFactoryClazz = org.apache.lucene.analysis.util.TokenFilterFactory.lookupClass(
+                        toCamelCase(tokenizer.getTokenFilterFactory(Version.CURRENT).name()));
+            }
+            PREBUILT_TOKENFILTERS.put(tokenizer, luceneFactoryClazz);
+        }
+    }
+
     static final Map<String,Class<?>> KNOWN_CHARFILTERS = new MapBuilder<String,Class<?>>()
         // exposed in ES
         .put("htmlstrip",      HtmlStripCharFilterFactory.class)
@@ -244,6 +335,20 @@ public class AnalysisFactoryTestCase extends ESTestCase {
         .put("persian",        Void.class)
         .immutableMap();
 
+    static final Map<PreBuiltCharFilters, Class<?>> PREBUILT_CHARFILTERS;
+    static {
+        PREBUILT_CHARFILTERS = new EnumMap<>(PreBuiltCharFilters.class);
+        for (PreBuiltCharFilters tokenizer : PreBuiltCharFilters.values()) {
+            Class<?> luceneFactoryClazz;
+            switch (tokenizer) {
+            default:
+                luceneFactoryClazz = org.apache.lucene.analysis.util.CharFilterFactory.lookupClass(
+                        toCamelCase(tokenizer.getCharFilterFactory(Version.CURRENT).name()));
+            }
+            PREBUILT_CHARFILTERS.put(tokenizer, luceneFactoryClazz);
+        }
+    }
+
     protected Map<String, Class<?>> getTokenizers() {
         return KNOWN_TOKENIZERS;
     }
@@ -325,4 +430,62 @@ public void testMultiTermAware() {
                 classesThatShouldNotHaveMultiTermSupport.isEmpty());
     }
 
+    public void testPreBuiltMultiTermAware() {
+        Collection<Object> expected = new HashSet<>();
+        Collection<Object> actual = new HashSet<>();
+
+        for (Map.Entry<PreBuiltTokenizers, Class<?>> entry : PREBUILT_TOKENIZERS.entrySet()) {
+            PreBuiltTokenizers tokenizer = entry.getKey();
+            Class<?> luceneFactory = entry.getValue();
+            if (luceneFactory == Void.class) {
+                continue;
+            }
+            assertTrue(TokenizerFactory.class.isAssignableFrom(luceneFactory));
+            if (tokenizer.getTokenizerFactory(Version.CURRENT) instanceof MultiTermAwareComponent) {
+                actual.add(tokenizer);
+            }
+            if (org.apache.lucene.analysis.util.MultiTermAwareComponent.class.isAssignableFrom(luceneFactory)) {
+                expected.add(tokenizer);
+            }
+        }
+        for (Map.Entry<PreBuiltTokenFilters, Class<?>> entry : PREBUILT_TOKENFILTERS.entrySet()) {
+            PreBuiltTokenFilters tokenFilter = entry.getKey();
+            Class<?> luceneFactory = entry.getValue();
+            if (luceneFactory == Void.class) {
+                continue;
+            }
+            assertTrue(TokenFilterFactory.class.isAssignableFrom(luceneFactory));
+            if (tokenFilter.getTokenFilterFactory(Version.CURRENT) instanceof MultiTermAwareComponent) {
+                actual.add(tokenFilter);
+            }
+            if (org.apache.lucene.analysis.util.MultiTermAwareComponent.class.isAssignableFrom(luceneFactory)) {
+                expected.add(tokenFilter);
+            }
+        }
+        for (Map.Entry<PreBuiltCharFilters, Class<?>> entry : PREBUILT_CHARFILTERS.entrySet()) {
+            PreBuiltCharFilters charFilter = entry.getKey();
+            Class<?> luceneFactory = entry.getValue();
+            if (luceneFactory == Void.class) {
+                continue;
+            }
+            assertTrue(CharFilterFactory.class.isAssignableFrom(luceneFactory));
+            if (charFilter.getCharFilterFactory(Version.CURRENT) instanceof MultiTermAwareComponent) {
+                actual.add(charFilter);
+            }
+            if (org.apache.lucene.analysis.util.MultiTermAwareComponent.class.isAssignableFrom(luceneFactory)) {
+                expected.add(charFilter);
+            }
+        }
+
+        Set<Object> classesMissingMultiTermSupport = new HashSet<>(expected);
+        classesMissingMultiTermSupport.removeAll(actual);
+        assertTrue("Pre-built components are missing multi-term support: " + classesMissingMultiTermSupport,
+                classesMissingMultiTermSupport.isEmpty());
+
+        Set<Object> classesThatShouldNotHaveMultiTermSupport = new HashSet<>(actual);
+        classesThatShouldNotHaveMultiTermSupport.removeAll(expected);
+        assertTrue("Pre-built components should not have multi-term support: " + classesThatShouldNotHaveMultiTermSupport,
+                classesThatShouldNotHaveMultiTermSupport.isEmpty());
+    }
+
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java b/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java
index 96921c3c90dde..209b7dff2c7f1 100644
--- a/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java
+++ b/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java
@@ -22,6 +22,7 @@
 import com.carrotsearch.randomizedtesting.RandomizedRunner;
 import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.SecureSM;
+import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.io.PathUtils;
@@ -115,7 +116,9 @@ public class BootstrapForTesting {
                     perms.add(new FilePermission(System.getProperty("tests.config"), "read,readlink"));
                 }
                 // jacoco coverage output file
-                if (Boolean.getBoolean("tests.coverage")) {
+                final boolean testsCoverage =
+                        Booleans.parseBoolean(System.getProperty("tests.coverage"), false);
+                if (testsCoverage) {
                     Path coverageDir = PathUtils.get(System.getProperty("tests.coverage.dir"));
                     perms.add(new FilePermission(coverageDir.resolve("jacoco.exec").toString(), "read,write"));
                     // in case we get fancy and use the -integration goals later:
@@ -178,7 +181,7 @@ static Map<String,Policy> getPluginPermissions() throws Exception {
         }
 
         // compute classpath minus obvious places, all other jars will get the permission.
-        Set<URL> codebases = new HashSet<>(Arrays.asList(parseClassPathWithSymlinks()));
+        Set<URL> codebases = new HashSet<>(parseClassPathWithSymlinks());
         Set<URL> excluded = new HashSet<>(Arrays.asList(
                 // es core
                 Bootstrap.class.getProtectionDomain().getCodeSource().getLocation(),
@@ -194,9 +197,9 @@ static Map<String,Policy> getPluginPermissions() throws Exception {
         codebases.removeAll(excluded);
 
         // parse each policy file, with codebase substitution from the classpath
-        final List<Policy> policies = new ArrayList<>();
+        final List<Policy> policies = new ArrayList<>(pluginPolicies.size());
         for (URL policyFile : pluginPolicies) {
-            policies.add(Security.readPolicy(policyFile, codebases.toArray(new URL[codebases.size()])));
+            policies.add(Security.readPolicy(policyFile, codebases));
         }
 
         // consult each policy file for those codebases
@@ -223,10 +226,14 @@ public boolean implies(ProtectionDomain domain, Permission permission) {
      * this is for matching the toRealPath() in the code where we have a proper plugin structure
      */
     @SuppressForbidden(reason = "does evil stuff with paths and urls because devs and jenkins do evil stuff with paths and urls")
-    static URL[] parseClassPathWithSymlinks() throws Exception {
-        URL raw[] = JarHell.parseClassPath();
-        for (int i = 0; i < raw.length; i++) {
-            raw[i] = PathUtils.get(raw[i].toURI()).toRealPath().toUri().toURL();
+    static Set<URL> parseClassPathWithSymlinks() throws Exception {
+        Set<URL> raw = JarHell.parseClassPath();
+        Set<URL> cooked = new HashSet<>(raw.size());
+        for (URL url : raw) {
+            boolean added = cooked.add(PathUtils.get(url.toURI()).toRealPath().toUri().toURL());
+            if (added == false) {
+                throw new IllegalStateException("Duplicate in classpath after resolving symlinks: " + url);
+            }
         }
         return raw;
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/bootstrap/ESElasticsearchCliTestCase.java b/test/framework/src/main/java/org/elasticsearch/bootstrap/ESElasticsearchCliTestCase.java
index 2a04a5be97f7f..7b575d0b7a664 100644
--- a/test/framework/src/main/java/org/elasticsearch/bootstrap/ESElasticsearchCliTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/bootstrap/ESElasticsearchCliTestCase.java
@@ -20,6 +20,9 @@
 package org.elasticsearch.bootstrap;
 
 import org.elasticsearch.cli.MockTerminal;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
 import org.elasticsearch.test.ESTestCase;
 
 import java.nio.file.Path;
@@ -32,7 +35,7 @@
 abstract class ESElasticsearchCliTestCase extends ESTestCase {
 
     interface InitConsumer {
-        void accept(final boolean foreground, final Path pidFile, final Map<String, String> esSettings);
+        void accept(boolean foreground, Path pidFile, boolean quiet, Environment initialEnv);
     }
 
     void runTest(
@@ -42,13 +45,26 @@ void runTest(
             final InitConsumer initConsumer,
             String... args) throws Exception {
         final MockTerminal terminal = new MockTerminal();
+        Path home = createTempDir();
         try {
             final AtomicBoolean init = new AtomicBoolean();
             final int status = Elasticsearch.main(args, new Elasticsearch() {
                 @Override
-                void init(final boolean daemonize, final Path pidFile, final Map<String, String> esSettings) {
+                protected Environment createEnv(Terminal terminal, Map<String, String> settings) {
+                    Settings realSettings = Settings.builder()
+                        .put("path.home", home)
+                        .put(settings).build();
+                    return new Environment(realSettings);
+                }
+                @Override
+                void init(final boolean daemonize, final Path pidFile, final boolean quiet, Environment initialEnv) {
                     init.set(true);
-                    initConsumer.accept(!daemonize, pidFile, esSettings);
+                    initConsumer.accept(!daemonize, pidFile, quiet, initialEnv);
+                }
+
+                @Override
+                protected boolean addShutdownHook() {
+                    return false;
                 }
             }, terminal);
             assertThat(status, equalTo(expectedStatus));
diff --git a/test/framework/src/main/java/org/elasticsearch/cli/MockTerminal.java b/test/framework/src/main/java/org/elasticsearch/cli/MockTerminal.java
index bd8bd493ceacb..a547e4033dd9b 100644
--- a/test/framework/src/main/java/org/elasticsearch/cli/MockTerminal.java
+++ b/test/framework/src/main/java/org/elasticsearch/cli/MockTerminal.java
@@ -25,7 +25,9 @@
 import java.io.UnsupportedEncodingException;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayDeque;
+import java.util.ArrayList;
 import java.util.Deque;
+import java.util.List;
 
 /**
  * A terminal for tests which captures all output, and
@@ -35,8 +37,18 @@ public class MockTerminal extends Terminal {
 
     private final ByteArrayOutputStream buffer = new ByteArrayOutputStream();
     private final PrintWriter writer = new PrintWriter(new OutputStreamWriter(buffer, StandardCharsets.UTF_8));
-    private final Deque<String> textInput = new ArrayDeque<>();
-    private final Deque<String> secretInput = new ArrayDeque<>();
+
+    // A deque would be a perfect data structure for the FIFO queue of input values needed here. However,
+    // to support the valid return value of readText being null (defined by Console), we need to be able
+    // to store nulls. However, java the java Deque api does not allow nulls because it uses null as
+    // a special return value from certain methods like peek(). So instead of deque, we use an array list here,
+    // and keep track of the last position which was read. It means that we will hold onto all input
+    // setup for the mock terminal during its lifetime, but this is normally a very small amount of data
+    // so in reality it will not matter.
+    private final List<String> textInput = new ArrayList<>();
+    private int textIndex = 0;
+    private final List<String> secretInput = new ArrayList<>();
+    private int secretIndex = 0;
 
     public MockTerminal() {
         super("\n"); // always *nix newlines for tests
@@ -44,18 +56,18 @@ public MockTerminal() {
 
     @Override
     public String readText(String prompt) {
-        if (textInput.isEmpty()) {
+        if (textIndex >= textInput.size()) {
             throw new IllegalStateException("No text input configured for prompt [" + prompt + "]");
         }
-        return textInput.removeFirst();
+        return textInput.get(textIndex++);
     }
 
     @Override
     public char[] readSecret(String prompt) {
-        if (secretInput.isEmpty()) {
+        if (secretIndex >= secretInput.size()) {
             throw new IllegalStateException("No secret input configured for prompt [" + prompt + "]");
         }
-        return secretInput.removeFirst().toCharArray();
+        return secretInput.get(secretIndex++).toCharArray();
     }
 
     @Override
@@ -65,12 +77,12 @@ public PrintWriter getWriter() {
 
     /** Adds an an input that will be return from {@link #readText(String)}. Values are read in FIFO order. */
     public void addTextInput(String input) {
-        textInput.addLast(input);
+        textInput.add(input);
     }
 
     /** Adds an an input that will be return from {@link #readText(String)}. Values are read in FIFO order. */
     public void addSecretInput(String input) {
-        secretInput.addLast(input);
+        secretInput.add(input);
     }
 
     /** Returns all output written to this terminal. */
diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java b/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java
index a1efa69f77550..c854523a327bb 100644
--- a/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java
@@ -26,9 +26,8 @@
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
-import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation;
+import org.elasticsearch.cluster.routing.allocation.FailedShard;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
-import org.elasticsearch.cluster.routing.allocation.StartedRerouteAllocation;
 import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
 import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocator;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider;
@@ -38,17 +37,14 @@
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.LocalTransportAddress;
-import org.elasticsearch.gateway.AsyncShardFetch;
 import org.elasticsearch.gateway.GatewayAllocator;
-import org.elasticsearch.gateway.ReplicaShardAllocator;
-import org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.gateway.NoopGatewayAllocator;
+import org.elasticsearch.test.gateway.TestGatewayAllocator;
 
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.HashSet;
+import java.util.EnumSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
@@ -79,13 +75,13 @@ public static MockAllocationService createAllocationService(Settings settings, R
     public static MockAllocationService createAllocationService(Settings settings, ClusterSettings clusterSettings, Random random) {
         return new MockAllocationService(settings,
                 randomAllocationDeciders(settings, clusterSettings, random),
-                NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(settings), EmptyClusterInfoService.INSTANCE);
+                new TestGatewayAllocator(), new BalancedShardsAllocator(settings), EmptyClusterInfoService.INSTANCE);
     }
 
     public static MockAllocationService createAllocationService(Settings settings, ClusterInfoService clusterInfoService) {
         return new MockAllocationService(settings,
                 randomAllocationDeciders(settings, EMPTY_CLUSTER_SETTINGS, random()),
-                NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(settings), clusterInfoService);
+            new TestGatewayAllocator(), new BalancedShardsAllocator(settings), clusterInfoService);
     }
 
     public static MockAllocationService createAllocationService(Settings settings, GatewayAllocator gatewayAllocator) {
@@ -102,7 +98,7 @@ public static AllocationDeciders randomAllocationDeciders(Settings settings, Clu
     }
 
     protected static Set<DiscoveryNode.Role> MASTER_DATA_ROLES =
-            Collections.unmodifiableSet(new HashSet<>(Arrays.asList(DiscoveryNode.Role.MASTER, DiscoveryNode.Role.DATA)));
+            Collections.unmodifiableSet(EnumSet.of(DiscoveryNode.Role.MASTER, DiscoveryNode.Role.DATA));
 
     protected static DiscoveryNode newNode(String nodeId) {
         return newNode(nodeId, Version.CURRENT);
@@ -129,15 +125,15 @@ protected  static ClusterState startRandomInitializingShard(ClusterState cluster
         if (initializingShards.isEmpty()) {
             return clusterState;
         }
-        RoutingAllocation.Result routingResult = strategy.applyStartedShards(clusterState,
+        return strategy.applyStartedShards(clusterState,
             arrayAsArrayList(initializingShards.get(randomInt(initializingShards.size() - 1))));
-        return ClusterState.builder(clusterState).routingResult(routingResult).build();
     }
 
     protected static AllocationDeciders yesAllocationDeciders() {
         return new AllocationDeciders(Settings.EMPTY, Arrays.asList(
             new TestAllocateDecision(Decision.YES),
-            new SameShardAllocationDecider(Settings.EMPTY)));
+            new SameShardAllocationDecider(Settings.EMPTY,
+                                           new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS))));
     }
 
     protected static AllocationDeciders noAllocationDeciders() {
@@ -147,16 +143,17 @@ protected static AllocationDeciders noAllocationDeciders() {
     protected static AllocationDeciders throttleAllocationDeciders() {
         return new AllocationDeciders(Settings.EMPTY, Arrays.asList(
             new TestAllocateDecision(Decision.THROTTLE),
-            new SameShardAllocationDecider(Settings.EMPTY)));
+            new SameShardAllocationDecider(Settings.EMPTY,
+                                           new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS))));
     }
 
     protected ClusterState applyStartedShardsUntilNoChange(ClusterState clusterState, AllocationService service) {
-        RoutingAllocation.Result routingResult;
+        ClusterState lastClusterState;
         do {
-            logger.debug("ClusterState: {}", clusterState.getRoutingNodes().prettyPrint());
-            routingResult = service.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
-            clusterState = ClusterState.builder(clusterState).routingResult(routingResult).build();
-        } while (routingResult.changed());
+            lastClusterState = clusterState;
+            logger.debug("ClusterState: {}", clusterState.getRoutingNodes());
+            clusterState = service.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
+        } while (lastClusterState.equals(clusterState) == false);
         return clusterState;
     }
 
@@ -209,24 +206,20 @@ protected long currentNanoTime() {
      * Mocks behavior in ReplicaShardAllocator to remove delayed shards from list of unassigned shards so they don't get reassigned yet.
      */
     protected static class DelayedShardsMockGatewayAllocator extends GatewayAllocator {
-        private final ReplicaShardAllocator replicaShardAllocator = new ReplicaShardAllocator(Settings.EMPTY) {
-            @Override
-            protected AsyncShardFetch.FetchResult<TransportNodesListShardStoreMetaData.NodeStoreFilesMetaData>
-            fetchData(ShardRouting shard, RoutingAllocation allocation) {
-                return new AsyncShardFetch.FetchResult<>(shard.shardId(), null, Collections.emptySet(), Collections.emptySet());
-            }
-        };
-
 
         public DelayedShardsMockGatewayAllocator() {
             super(Settings.EMPTY, null, null);
         }
 
         @Override
-        public void applyStartedShards(StartedRerouteAllocation allocation) {}
+        public void applyStartedShards(RoutingAllocation allocation, List<ShardRouting> startedShards) {
+            // no-op
+        }
 
         @Override
-        public void applyFailedShards(FailedRerouteAllocation allocation) {}
+        public void applyFailedShards(RoutingAllocation allocation, List<FailedShard> failedShards) {
+            // no-op
+        }
 
         @Override
         public void allocateUnassigned(RoutingAllocation allocation) {
@@ -236,7 +229,9 @@ public void allocateUnassigned(RoutingAllocation allocation) {
                 if (shard.primary() || shard.unassignedInfo().getReason() == UnassignedInfo.Reason.INDEX_CREATED) {
                     continue;
                 }
-                replicaShardAllocator.ignoreUnassignedIfDelayed(unassignedIterator, shard, allocation.changes());
+                if (shard.unassignedInfo().isDelayed()) {
+                    unassignedIterator.removeAndIgnore(UnassignedInfo.AllocationStatus.DELAYED_ALLOCATION, allocation.changes());
+                }
             }
         }
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java b/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java
index 45edbd8bcb26d..85fc60c202899 100644
--- a/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java
+++ b/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java
@@ -18,19 +18,20 @@
  */
 package org.elasticsearch.cluster;
 
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.concurrent.CountDownLatch;
+
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
-import org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
-import org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
@@ -38,10 +39,6 @@
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.threadpool.ThreadPool;
 
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.concurrent.CountDownLatch;
-
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
 
@@ -51,11 +48,8 @@
  */
 public class MockInternalClusterInfoService extends InternalClusterInfoService {
 
-    public static class TestPlugin extends Plugin {
-        public void onModule(ClusterModule module) {
-            module.clusterInfoServiceImpl = MockInternalClusterInfoService.class;
-        }
-    }
+    /** This is a marker plugin used to trigger MockNode to use this mock info service. */
+    public static class TestPlugin extends Plugin {}
 
     private final ClusterName clusterName;
     private volatile NodeStats[] stats = new NodeStats[3];
@@ -75,12 +69,8 @@ public static NodeStats makeStats(String nodeName, DiskUsage usage) {
             null, null, null);
     }
 
-    @Inject
-    public MockInternalClusterInfoService(Settings settings, ClusterSettings clusterSettings,
-                                          TransportNodesStatsAction transportNodesStatsAction,
-                                          TransportIndicesStatsAction transportIndicesStatsAction,
-                                          ClusterService clusterService, ThreadPool threadPool) {
-        super(settings, clusterSettings, transportNodesStatsAction, transportIndicesStatsAction, clusterService, threadPool);
+    public MockInternalClusterInfoService(Settings settings, ClusterService clusterService, ThreadPool threadPool, NodeClient client) {
+        super(settings, clusterService, threadPool, client);
         this.clusterName = ClusterName.CLUSTER_NAME_SETTING.get(settings);
         stats[0] = makeStats("node_t1", new DiskUsage("node_t1", "n1", "/dev/null", 100, 100));
         stats[1] = makeStats("node_t2", new DiskUsage("node_t2", "n2", "/dev/null", 100, 100));
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingHelper.java b/test/framework/src/main/java/org/elasticsearch/cluster/routing/ShardRoutingHelper.java
similarity index 100%
rename from core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingHelper.java
rename to test/framework/src/main/java/org/elasticsearch/cluster/routing/ShardRoutingHelper.java
diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java b/test/framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java
index d0d6c02f6360f..a4ac6fad241a5 100644
--- a/test/framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java
+++ b/test/framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java
@@ -27,7 +27,7 @@
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.test.ESTestCase;
 
-import static org.elasticsearch.test.ESTestCase.randomAsciiOfLength;
+import static org.elasticsearch.test.ESTestCase.randomAlphaOfLength;
 
 /**
  * A helper that allows to create shard routing instances within tests, while not requiring to expose
@@ -131,7 +131,7 @@ public static RecoverySource randomRecoverySource() {
             RecoverySource.PeerRecoverySource.INSTANCE,
             RecoverySource.LocalShardsRecoverySource.INSTANCE,
             new RecoverySource.SnapshotRecoverySource(
-                new Snapshot("repo", new SnapshotId(randomAsciiOfLength(8), UUIDs.randomBase64UUID())),
+                new Snapshot("repo", new SnapshotId(randomAlphaOfLength(8), UUIDs.randomBase64UUID())),
                 Version.CURRENT,
                 "some_index"));
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java b/test/framework/src/main/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java
index aae6522da1227..f1c6bd412a502 100644
--- a/test/framework/src/main/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java
@@ -74,9 +74,14 @@ public void testSlice() throws IOException {
         int sliceLength = Math.max(0, length - sliceOffset - 1);
         BytesReference slice = pbr.slice(sliceOffset, sliceLength);
         assertEquals(sliceLength, slice.length());
+        for (int i = 0; i < sliceLength; i++) {
+            assertEquals(pbr.get(i+sliceOffset), slice.get(i));
+        }
         BytesRef singlePageOrNull = getSinglePageOrNull(slice);
         if (singlePageOrNull != null) {
-            assertEquals(sliceOffset, singlePageOrNull.offset);
+            // we can't assert the offset since if the length is smaller than the refercence
+            // the offset can be anywhere
+            assertEquals(sliceLength, singlePageOrNull.length);
         }
     }
 
@@ -427,8 +432,8 @@ public void testArrayOffset() throws IOException {
     }
 
     public void testSliceArrayOffset() throws IOException {
-        int length = randomInt(PAGE_SIZE * randomIntBetween(2, 5));
-        BytesReference pbr = newBytesReference(length);
+        int length = randomIntBetween(1, PAGE_SIZE * randomIntBetween(2, 5));
+        BytesReference pbr = newBytesReferenceWithOffsetOfZero(length);
         int sliceOffset = randomIntBetween(0, pbr.length() - 1); // an offset to the end would be len 0
         int sliceLength = randomIntBetween(1, pbr.length() - sliceOffset);
         BytesReference slice = pbr.slice(sliceOffset, sliceLength);
@@ -461,7 +466,7 @@ public void testToBytesRef() throws IOException {
 
     public void testSliceToBytesRef() throws IOException {
         int length = randomIntBetween(0, PAGE_SIZE);
-        BytesReference pbr = newBytesReference(length);
+        BytesReference pbr = newBytesReferenceWithOffsetOfZero(length);
         // get a BytesRef from a slice
         int sliceOffset = randomIntBetween(0, pbr.length());
         int sliceLength = randomIntBetween(0, pbr.length() - sliceOffset);
@@ -539,6 +544,8 @@ public void testSliceEquals() {
 
     protected abstract BytesReference newBytesReference(int length) throws IOException;
 
+    protected abstract BytesReference newBytesReferenceWithOffsetOfZero(int length) throws IOException;
+
     public void testCompareTo() throws IOException {
         final int iters = randomIntBetween(5, 10);
         for (int i = 0; i < iters; i++) {
diff --git a/test/framework/src/main/java/org/elasticsearch/common/io/FileTestUtils.java b/test/framework/src/main/java/org/elasticsearch/common/io/FileTestUtils.java
index 98155967514bf..4128c9d9efc88 100644
--- a/test/framework/src/main/java/org/elasticsearch/common/io/FileTestUtils.java
+++ b/test/framework/src/main/java/org/elasticsearch/common/io/FileTestUtils.java
@@ -71,7 +71,7 @@ public static void unzip(final Path zip, final Path destDir, @Nullable final Str
         }
         Files.createDirectories(destDir);
 
-        try (final ZipInputStream zipInput = new ZipInputStream(Files.newInputStream(zip))) {
+        try (ZipInputStream zipInput = new ZipInputStream(Files.newInputStream(zip))) {
             ZipEntry entry;
             while ((entry = zipInput.getNextEntry()) != null) {
                 final String entryPath;
diff --git a/test/framework/src/main/java/org/elasticsearch/common/io/PathUtilsForTesting.java b/test/framework/src/main/java/org/elasticsearch/common/io/PathUtilsForTesting.java
index e576d1770664f..5b00f9f2e8654 100644
--- a/test/framework/src/main/java/org/elasticsearch/common/io/PathUtilsForTesting.java
+++ b/test/framework/src/main/java/org/elasticsearch/common/io/PathUtilsForTesting.java
@@ -30,7 +30,7 @@ public class PathUtilsForTesting {
 
     /** Sets a new default filesystem for testing */
     public static void setup() {
-        installMock(LuceneTestCase.getBaseTempDirForTestClass().getFileSystem());
+        installMock(LuceneTestCase.createTempDir().getFileSystem());
     }
 
     /** Installs a mock filesystem for testing */
diff --git a/test/framework/src/main/java/org/elasticsearch/common/logging/TestLoggers.java b/test/framework/src/main/java/org/elasticsearch/common/logging/TestLoggers.java
deleted file mode 100644
index 3b6cf7e2c94d9..0000000000000
--- a/test/framework/src/main/java/org/elasticsearch/common/logging/TestLoggers.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.common.logging;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.core.Appender;
-import org.apache.logging.log4j.core.LoggerContext;
-import org.apache.logging.log4j.core.config.AppenderRef;
-import org.apache.logging.log4j.core.config.Configuration;
-import org.apache.logging.log4j.core.config.LoggerConfig;
-
-public class TestLoggers {
-
-    public static void addAppender(final Logger logger, final Appender appender) {
-        final LoggerContext ctx = (LoggerContext) LogManager.getContext(false);
-        final Configuration config = ctx.getConfiguration();
-        config.addAppender(appender);
-        LoggerConfig loggerConfig = config.getLoggerConfig(logger.getName());
-        if (!logger.getName().equals(loggerConfig.getName())) {
-            loggerConfig = new LoggerConfig(logger.getName(), logger.getLevel(), true);
-            config.addLogger(logger.getName(), loggerConfig);
-        }
-        loggerConfig.addAppender(appender, null, null);
-        ctx.updateLoggers();
-    }
-
-    public static void removeAppender(final Logger logger, final Appender appender) {
-        final LoggerContext ctx = (LoggerContext) LogManager.getContext(false);
-        final Configuration config = ctx.getConfiguration();
-        LoggerConfig loggerConfig = config.getLoggerConfig(logger.getName());
-        if (!logger.getName().equals(loggerConfig.getName())) {
-            loggerConfig = new LoggerConfig(logger.getName(), logger.getLevel(), true);
-            config.addLogger(logger.getName(), loggerConfig);
-        }
-        loggerConfig.removeAppender(appender.getName());
-        ctx.updateLoggers();
-    }
-
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/common/settings/MockSecureSettings.java b/test/framework/src/main/java/org/elasticsearch/common/settings/MockSecureSettings.java
new file mode 100644
index 0000000000000..22496642cb9b7
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/common/settings/MockSecureSettings.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * A mock implementation of secure settings for tests to use.
+ */
+public class MockSecureSettings implements SecureSettings {
+
+    private Map<String, SecureString> secureStrings = new HashMap<>();
+    private Map<String, byte[]> files = new HashMap<>();
+    private Set<String> settingNames = new HashSet<>();
+    private final AtomicBoolean closed = new AtomicBoolean(false);
+
+    @Override
+    public boolean isLoaded() {
+        return true;
+    }
+
+    @Override
+    public Set<String> getSettingNames() {
+        return settingNames;
+    }
+
+    @Override
+    public SecureString getString(String setting) {
+        ensureOpen();
+        return secureStrings.get(setting);
+    }
+
+    @Override
+    public InputStream getFile(String setting) {
+        ensureOpen();
+        return new ByteArrayInputStream(files.get(setting));
+    }
+
+    public void setString(String setting, String value) {
+        ensureOpen();
+        secureStrings.put(setting, new SecureString(value.toCharArray()));
+        settingNames.add(setting);
+    }
+
+    public void setFile(String setting, byte[] value) {
+        ensureOpen();
+        files.put(setting, value);
+        settingNames.add(setting);
+    }
+
+    /** Merge the given secure settings into this one. */
+    public void merge(MockSecureSettings secureSettings) {
+        for (String setting : secureSettings.getSettingNames()) {
+            if (settingNames.contains(setting)) {
+                throw new IllegalArgumentException("Cannot overwrite existing secure setting " + setting);
+            }
+        }
+        settingNames.addAll(secureSettings.settingNames);
+        secureStrings.putAll(secureSettings.secureStrings);
+        files.putAll(secureSettings.files);
+    }
+
+    @Override
+    public void close() throws IOException {
+        closed.set(true);
+    }
+
+    private void ensureOpen() {
+        if (closed.get()) {
+            throw new IllegalStateException("secure settings are already closed");
+        }
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java b/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java
index 22a1e2660b63d..2f75d92d89829 100644
--- a/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java
+++ b/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java
@@ -24,7 +24,6 @@
 import org.apache.lucene.util.Accountable;
 import org.apache.lucene.util.Accountables;
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
@@ -35,11 +34,9 @@
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Random;
-import java.util.Set;
-import java.util.WeakHashMap;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
 
 public class MockBigArrays extends BigArrays {
 
@@ -49,7 +46,6 @@ public class MockBigArrays extends BigArrays {
      */
     private static final boolean TRACK_ALLOCATIONS = false;
 
-    private static final Set<BigArrays> INSTANCES = Collections.synchronizedSet(Collections.newSetFromMap(new WeakHashMap<BigArrays, Boolean>()));
     private static final ConcurrentMap<Object, Object> ACQUIRED_ARRAYS = new ConcurrentHashMap<>();
 
     public static void ensureAllArraysAreReleased() throws Exception {
@@ -89,7 +85,6 @@ private MockBigArrays(PageCacheRecycler recycler, CircuitBreakerService breakerS
             seed = 0;
         }
         random = new Random(seed);
-        INSTANCES.add(this);
     }
 
 
@@ -248,15 +243,13 @@ public <T> ObjectArray<T> resize(ObjectArray<T> array, long size) {
 
     private abstract static class AbstractArrayWrapper {
 
-        final BigArray in;
-        boolean clearOnResize;
-        AtomicBoolean released;
+        final boolean clearOnResize;
+        private final AtomicReference<AssertionError> originalRelease;
 
-        AbstractArrayWrapper(BigArray in, boolean clearOnResize) {
-            ACQUIRED_ARRAYS.put(this, TRACK_ALLOCATIONS ? new RuntimeException() : Boolean.TRUE);
-            this.in = in;
+        AbstractArrayWrapper(boolean clearOnResize) {
             this.clearOnResize = clearOnResize;
-            released = new AtomicBoolean(false);
+            this.originalRelease = new AtomicReference<>();
+            ACQUIRED_ARRAYS.put(this, TRACK_ALLOCATIONS ? new RuntimeException() : Boolean.TRUE);
         }
 
         protected abstract BigArray getDelegate();
@@ -268,12 +261,12 @@ public long size() {
         }
 
         public long ramBytesUsed() {
-            return in.ramBytesUsed();
+            return getDelegate().ramBytesUsed();
         }
 
         public void close() {
-            if (!released.compareAndSet(false, true)) {
-                throw new IllegalStateException("Double release");
+            if (originalRelease.compareAndSet(null, new AssertionError()) == false) {
+                throw new IllegalStateException("Double release. Original release attached as cause", originalRelease.get());
             }
             ACQUIRED_ARRAYS.remove(this);
             randomizeContent(0, size());
@@ -287,7 +280,7 @@ private class ByteArrayWrapper extends AbstractArrayWrapper implements ByteArray
         private final ByteArray in;
 
         ByteArrayWrapper(ByteArray in, boolean clearOnResize) {
-            super(in, clearOnResize);
+            super(clearOnResize);
             this.in = in;
         }
 
@@ -337,7 +330,7 @@ private class IntArrayWrapper extends AbstractArrayWrapper implements IntArray {
         private final IntArray in;
 
         IntArrayWrapper(IntArray in, boolean clearOnResize) {
-            super(in, clearOnResize);
+            super(clearOnResize);
             this.in = in;
         }
 
@@ -382,7 +375,7 @@ private class LongArrayWrapper extends AbstractArrayWrapper implements LongArray
         private final LongArray in;
 
         LongArrayWrapper(LongArray in, boolean clearOnResize) {
-            super(in, clearOnResize);
+            super(clearOnResize);
             this.in = in;
         }
 
@@ -428,7 +421,7 @@ private class FloatArrayWrapper extends AbstractArrayWrapper implements FloatArr
         private final FloatArray in;
 
         FloatArrayWrapper(FloatArray in, boolean clearOnResize) {
-            super(in, clearOnResize);
+            super(clearOnResize);
             this.in = in;
         }
 
@@ -473,7 +466,7 @@ private class DoubleArrayWrapper extends AbstractArrayWrapper implements DoubleA
         private final DoubleArray in;
 
         DoubleArrayWrapper(DoubleArray in, boolean clearOnResize) {
-            super(in, clearOnResize);
+            super(clearOnResize);
             this.in = in;
         }
 
@@ -518,7 +511,7 @@ private class ObjectArrayWrapper<T> extends AbstractArrayWrapper implements Obje
         private final ObjectArray<T> in;
 
         ObjectArrayWrapper(ObjectArray<T> in) {
-            super(in, false);
+            super(false);
             this.in = in;
         }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/http/NullDispatcher.java b/test/framework/src/main/java/org/elasticsearch/http/NullDispatcher.java
new file mode 100644
index 0000000000000..17a5e57fae80f
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/http/NullDispatcher.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.http;
+
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.rest.RestChannel;
+import org.elasticsearch.rest.RestRequest;
+
+public class NullDispatcher implements HttpServerTransport.Dispatcher {
+
+    @Override
+    public void dispatchRequest(RestRequest request, RestChannel channel, ThreadContext threadContext) {
+
+    }
+
+    @Override
+    public void dispatchBadRequest(RestRequest request, RestChannel channel, ThreadContext threadContext, Throwable cause) {
+
+    }
+
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java b/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java
index a897de7073a5e..b01ee7cc5c877 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java
@@ -21,11 +21,10 @@
 
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.Environment;
-import org.elasticsearch.index.analysis.AnalysisRegistry;
-import org.elasticsearch.index.analysis.AnalysisService;
+import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.indices.IndicesModule;
@@ -36,17 +35,19 @@
 import java.nio.file.Path;
 import java.util.Collections;
 
-import static org.elasticsearch.test.ESTestCase.createAnalysisService;
+import static org.elasticsearch.test.ESTestCase.createTestAnalysis;
 
 
 public class MapperTestUtils {
 
-    public static MapperService newMapperService(Path tempDir, Settings indexSettings) throws IOException {
+    public static MapperService newMapperService(NamedXContentRegistry xContentRegistry, Path tempDir, Settings indexSettings)
+            throws IOException {
         IndicesModule indicesModule = new IndicesModule(Collections.emptyList());
-        return newMapperService(tempDir, indexSettings, indicesModule);
+        return newMapperService(xContentRegistry, tempDir, indexSettings, indicesModule);
     }
 
-    public static MapperService newMapperService(Path tempDir, Settings settings, IndicesModule indicesModule) throws IOException {
+    public static MapperService newMapperService(NamedXContentRegistry xContentRegistry, Path tempDir, Settings settings,
+            IndicesModule indicesModule) throws IOException {
         Settings.Builder settingsBuilder = Settings.builder()
             .put(Environment.PATH_HOME_SETTING.getKey(), tempDir)
             .put(settings);
@@ -56,10 +57,11 @@ public static MapperService newMapperService(Path tempDir, Settings settings, In
         Settings finalSettings = settingsBuilder.build();
         MapperRegistry mapperRegistry = indicesModule.getMapperRegistry();
         IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("test", finalSettings);
-        AnalysisService analysisService = createAnalysisService(indexSettings, finalSettings);
+        IndexAnalyzers indexAnalyzers = createTestAnalysis(indexSettings, finalSettings).indexAnalyzers;
         SimilarityService similarityService = new SimilarityService(indexSettings, Collections.emptyMap());
         return new MapperService(indexSettings,
-            analysisService,
+            indexAnalyzers,
+            xContentRegistry,
             similarityService,
             mapperRegistry,
             () -> null);
diff --git a/test/framework/src/main/java/org/elasticsearch/index/MockEngineFactoryPlugin.java b/test/framework/src/main/java/org/elasticsearch/index/MockEngineFactoryPlugin.java
index 7ddd2526fcd37..c6065f7e5831a 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/MockEngineFactoryPlugin.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/MockEngineFactoryPlugin.java
@@ -33,11 +33,13 @@
 import java.util.Collections;
 import java.util.List;
 
-// this must exist in the same package as IndexModule to allow access to setting the impl
+/**
+ * A plugin to use {@link MockEngineFactory}.
+ *
+ * Subclasses may override the reader wrapper used.
+ */
 public class MockEngineFactoryPlugin extends Plugin {
 
-    private Class<? extends FilterDirectoryReader> readerWrapper = AssertingDirectoryReader.class;
-
     @Override
     public List<Setting<?>> getSettings() {
         return Arrays.asList(MockEngineSupport.DISABLE_FLUSH_ON_CLOSE, MockEngineSupport.WRAP_READER_RATIO);
@@ -45,22 +47,10 @@ public List<Setting<?>> getSettings() {
 
     @Override
     public void onIndexModule(IndexModule module) {
-        module.engineFactory.set(new MockEngineFactory(readerWrapper));
-    }
-
-    @Override
-    public Collection<Module> createGuiceModules() {
-        return Collections.singleton(new MockEngineReaderModule());
+        module.engineFactory.set(new MockEngineFactory(getReaderWrapperClass()));
     }
 
-    public class MockEngineReaderModule extends AbstractModule {
-
-        public void setReaderClass(Class<? extends FilterDirectoryReader> readerWrapper) {
-            MockEngineFactoryPlugin.this.readerWrapper = readerWrapper;
-        }
-
-        @Override
-        protected void configure() {
-        }
+    protected Class<? extends FilterDirectoryReader> getReaderWrapperClass() {
+        return AssertingDirectoryReader.class;
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java
similarity index 94%
rename from core/src/test/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java
rename to test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java
index 37d0436c9dbfc..a2ecc0e4253ca 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java
@@ -20,6 +20,7 @@
 
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.analysis.AnalyzerScope;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.similarity.BM25SimilarityProvider;
 import org.elasticsearch.test.ESTestCase;
@@ -68,49 +69,49 @@ public void modify(MappedFieldType ft) {
         new Modifier("analyzer", false) {
             @Override
             public void modify(MappedFieldType ft) {
-                ft.setIndexAnalyzer(new NamedAnalyzer("bar", new StandardAnalyzer()));
+                ft.setIndexAnalyzer(new NamedAnalyzer("bar", AnalyzerScope.INDEX, new StandardAnalyzer()));
             }
         },
         new Modifier("analyzer", false) {
             @Override
             public void modify(MappedFieldType ft) {
-                ft.setIndexAnalyzer(new NamedAnalyzer("bar", new StandardAnalyzer()));
+                ft.setIndexAnalyzer(new NamedAnalyzer("bar", AnalyzerScope.INDEX, new StandardAnalyzer()));
             }
             @Override
             public void normalizeOther(MappedFieldType other) {
-                other.setIndexAnalyzer(new NamedAnalyzer("foo", new StandardAnalyzer()));
+                other.setIndexAnalyzer(new NamedAnalyzer("foo", AnalyzerScope.INDEX, new StandardAnalyzer()));
             }
         },
         new Modifier("search_analyzer", true) {
             @Override
             public void modify(MappedFieldType ft) {
-                ft.setSearchAnalyzer(new NamedAnalyzer("bar", new StandardAnalyzer()));
+                ft.setSearchAnalyzer(new NamedAnalyzer("bar", AnalyzerScope.INDEX, new StandardAnalyzer()));
             }
         },
         new Modifier("search_analyzer", true) {
             @Override
             public void modify(MappedFieldType ft) {
-                ft.setSearchAnalyzer(new NamedAnalyzer("bar", new StandardAnalyzer()));
+                ft.setSearchAnalyzer(new NamedAnalyzer("bar", AnalyzerScope.INDEX, new StandardAnalyzer()));
             }
             @Override
             public void normalizeOther(MappedFieldType other) {
-                other.setSearchAnalyzer(new NamedAnalyzer("foo", new StandardAnalyzer()));
+                other.setSearchAnalyzer(new NamedAnalyzer("foo", AnalyzerScope.INDEX, new StandardAnalyzer()));
             }
         },
         new Modifier("search_quote_analyzer", true) {
             @Override
             public void modify(MappedFieldType ft) {
-                ft.setSearchQuoteAnalyzer(new NamedAnalyzer("bar", new StandardAnalyzer()));
+                ft.setSearchQuoteAnalyzer(new NamedAnalyzer("bar", AnalyzerScope.INDEX, new StandardAnalyzer()));
             }
         },
         new Modifier("search_quote_analyzer", true) {
             @Override
             public void modify(MappedFieldType ft) {
-                ft.setSearchQuoteAnalyzer(new NamedAnalyzer("bar", new StandardAnalyzer()));
+                ft.setSearchQuoteAnalyzer(new NamedAnalyzer("bar", AnalyzerScope.INDEX, new StandardAnalyzer()));
             }
             @Override
             public void normalizeOther(MappedFieldType other) {
-                other.setSearchQuoteAnalyzer(new NamedAnalyzer("foo", new StandardAnalyzer()));
+                other.setSearchQuoteAnalyzer(new NamedAnalyzer("foo", AnalyzerScope.INDEX, new StandardAnalyzer()));
             }
         },
         new Modifier("similarity", false) {
@@ -170,13 +171,15 @@ MappedFieldType createNamedDefaultFieldType() {
     // TODO: remove this once toString is no longer final on FieldType...
     protected void assertFieldTypeEquals(String property, MappedFieldType ft1, MappedFieldType ft2) {
         if (ft1.equals(ft2) == false) {
-            fail("Expected equality, testing property " + property + "\nexpected: " + toString(ft1) + "; \nactual:   " + toString(ft2) + "\n");
+            fail("Expected equality, testing property " + property + "\nexpected: " + toString(ft1) + "; \nactual:   " + toString(ft2)
+                + "\n");
         }
     }
 
     protected void assertFieldTypeNotEquals(String property, MappedFieldType ft1, MappedFieldType ft2) {
         if (ft1.equals(ft2)) {
-            fail("Expected inequality, testing property " + property + "\nfirst:  " + toString(ft1) + "; \nsecond: " + toString(ft2) + "\n");
+            fail("Expected inequality, testing property " + property + "\nfirst:  " + toString(ft1) + "; \nsecond: " + toString(ft2)
+                + "\n");
         }
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionTestCase.java
new file mode 100644
index 0000000000000..079c784342b2b
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionTestCase.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex;
+
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.tasks.TaskId;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.junit.After;
+import org.junit.Before;
+
+public abstract class AbstractAsyncBulkByScrollActionTestCase<
+                Request extends AbstractBulkByScrollRequest<Request>,
+                Response extends BulkByScrollResponse>
+        extends ESTestCase {
+    protected ThreadPool threadPool;
+    protected WorkingBulkByScrollTask task;
+
+    @Before
+    public void setupForTest() {
+        threadPool = new TestThreadPool(getTestName());
+        task = new WorkingBulkByScrollTask(1, "test", "test", "test", TaskId.EMPTY_TASK_ID, null, Float.MAX_VALUE);
+    }
+
+    @After
+    @Override
+    public void tearDown() throws Exception {
+        super.tearDown();
+        threadPool.shutdown();
+    }
+
+    protected abstract Request request();
+
+    protected PlainActionFuture<Response> listener() {
+        return new PlainActionFuture<>();
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java
new file mode 100644
index 0000000000000..59f3f9b8385d4
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java
@@ -0,0 +1,484 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.index.shard;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.IndexNotFoundException;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.admin.indices.flush.FlushRequest;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.RecoverySource;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.ShardRoutingHelper;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.TestShardRouting;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.lucene.uid.Versions;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.LocalTransportAddress;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.MapperTestUtils;
+import org.elasticsearch.index.VersionType;
+import org.elasticsearch.index.cache.IndexCache;
+import org.elasticsearch.index.cache.query.DisabledQueryCache;
+import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.fielddata.IndexFieldDataCache;
+import org.elasticsearch.index.fielddata.IndexFieldDataService;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.SourceToParse;
+import org.elasticsearch.index.mapper.Uid;
+import org.elasticsearch.index.mapper.UidFieldMapper;
+import org.elasticsearch.index.similarity.SimilarityService;
+import org.elasticsearch.index.store.DirectoryService;
+import org.elasticsearch.index.store.Store;
+import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
+import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
+import org.elasticsearch.indices.recovery.PeerRecoveryTargetService;
+import org.elasticsearch.indices.recovery.RecoveryFailedException;
+import org.elasticsearch.indices.recovery.RecoverySourceHandler;
+import org.elasticsearch.indices.recovery.RecoveryState;
+import org.elasticsearch.indices.recovery.RecoveryTarget;
+import org.elasticsearch.indices.recovery.StartRecoveryRequest;
+import org.elasticsearch.test.DummyShardLock;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.threadpool.ThreadPool;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.function.BiFunction;
+
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.hasSize;
+
+/**
+ * A base class for unit tests that need to create and shutdown {@link IndexShard} instances easily,
+ * containing utilities for shard creation and recoveries. See {{@link #newShard(boolean)}} and
+ * {@link #newStartedShard()} for a good starting points
+ */
+public abstract class IndexShardTestCase extends ESTestCase {
+
+    protected static final PeerRecoveryTargetService.RecoveryListener recoveryListener = new PeerRecoveryTargetService.RecoveryListener() {
+        @Override
+        public void onRecoveryDone(RecoveryState state) {
+
+        }
+
+        @Override
+        public void onRecoveryFailure(RecoveryState state, RecoveryFailedException e, boolean sendShardFailure) {
+            throw new AssertionError(e);
+        }
+    };
+
+    protected ThreadPool threadPool;
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        threadPool = new TestThreadPool(getClass().getName());
+    }
+
+    @Override
+    public void tearDown() throws Exception {
+        try {
+            ThreadPool.terminate(threadPool, 30, TimeUnit.SECONDS);
+        } finally {
+            super.tearDown();
+        }
+    }
+
+    private Store createStore(IndexSettings indexSettings, ShardPath shardPath) throws IOException {
+        final ShardId shardId = shardPath.getShardId();
+        final DirectoryService directoryService = new DirectoryService(shardId, indexSettings) {
+            @Override
+            public Directory newDirectory() throws IOException {
+                return newFSDirectory(shardPath.resolveIndex());
+            }
+
+            @Override
+            public long throttleTimeInNanos() {
+                return 0;
+            }
+        };
+        return new Store(shardId, indexSettings, directoryService, new DummyShardLock(shardId));
+    }
+
+    /**
+     * creates a new initializing shard. The shard will have its own unique data path.
+     *
+     * @param primary indicates whether to a primary shard (ready to recover from an empty store) or a replica
+     *                (ready to recover from another shard)
+     */
+    protected IndexShard newShard(boolean primary) throws IOException {
+        ShardRouting shardRouting = TestShardRouting.newShardRouting(new ShardId("index", "_na_", 0), "n1", primary,
+            ShardRoutingState.INITIALIZING,
+            primary ? RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE : RecoverySource.PeerRecoverySource.INSTANCE);
+        return newShard(shardRouting);
+    }
+
+    /**
+     * creates a new initializing shard. The shard will have its own unique data path.
+     *
+     * @param shardRouting the {@link ShardRouting} to use for this shard
+     * @param listeners    an optional set of listeners to add to the shard
+     */
+    protected IndexShard newShard(ShardRouting shardRouting, IndexingOperationListener... listeners) throws IOException {
+        assert shardRouting.initializing() : shardRouting;
+        Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+            .build();
+        IndexMetaData.Builder metaData = IndexMetaData.builder(shardRouting.getIndexName())
+            .settings(settings)
+            .primaryTerm(0, 1);
+        return newShard(shardRouting, metaData.build(), listeners);
+    }
+
+    /**
+     * creates a new initializing shard. The shard will have its own unique data path.
+     *
+     * @param shardId   the shard id to use
+     * @param primary   indicates whether to a primary shard (ready to recover from an empty store) or a replica
+     *                  (ready to recover from another shard)
+     * @param listeners an optional set of listeners to add to the shard
+     */
+    protected IndexShard newShard(ShardId shardId, boolean primary, IndexingOperationListener... listeners) throws IOException {
+        ShardRouting shardRouting = TestShardRouting.newShardRouting(shardId, randomAlphaOfLength(5), primary,
+            ShardRoutingState.INITIALIZING,
+            primary ? RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE : RecoverySource.PeerRecoverySource.INSTANCE);
+        return newShard(shardRouting, listeners);
+    }
+
+    /**
+     * creates a new initializing shard. The shard will will be put in its proper path under the
+     * supplied node id.
+     *
+     * @param shardId the shard id to use
+     * @param primary indicates whether to a primary shard (ready to recover from an empty store) or a replica
+     *                (ready to recover from another shard)
+     */
+    protected IndexShard newShard(ShardId shardId, boolean primary, String nodeId, IndexMetaData indexMetaData,
+                                  @Nullable IndexSearcherWrapper searcherWrapper) throws IOException {
+        ShardRouting shardRouting = TestShardRouting.newShardRouting(shardId, nodeId, primary, ShardRoutingState.INITIALIZING,
+            primary ? RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE : RecoverySource.PeerRecoverySource.INSTANCE);
+        return newShard(shardRouting, indexMetaData, searcherWrapper);
+    }
+
+    /**
+     * creates a new initializing shard. The shard will will be put in its proper path under the
+     * current node id the shard is assigned to.
+     *
+     * @param routing       shard routing to use
+     * @param indexMetaData indexMetaData for the shard, including any mapping
+     * @param listeners     an optional set of listeners to add to the shard
+     */
+    protected IndexShard newShard(ShardRouting routing, IndexMetaData indexMetaData, IndexingOperationListener... listeners)
+        throws IOException {
+        return newShard(routing, indexMetaData, null, listeners);
+    }
+
+    /**
+     * creates a new initializing shard. The shard will will be put in its proper path under the
+     * current node id the shard is assigned to.
+     *
+     * @param routing              shard routing to use
+     * @param indexMetaData        indexMetaData for the shard, including any mapping
+     * @param indexSearcherWrapper an optional wrapper to be used during searchers
+     * @param listeners            an optional set of listeners to add to the shard
+     */
+    protected IndexShard newShard(ShardRouting routing, IndexMetaData indexMetaData,
+                                  @Nullable IndexSearcherWrapper indexSearcherWrapper, IndexingOperationListener... listeners)
+        throws IOException {
+        // add node id as name to settings for proper logging
+        final ShardId shardId = routing.shardId();
+        final NodeEnvironment.NodePath nodePath = new NodeEnvironment.NodePath(createTempDir());
+        ShardPath shardPath = new ShardPath(false, nodePath.resolve(shardId), nodePath.resolve(shardId), shardId);
+        return newShard(routing, shardPath, indexMetaData, indexSearcherWrapper, listeners);
+    }
+
+    /**
+     * creates a new initializing shard.
+     *
+     * @param routing              shard routing to use
+     * @param shardPath            path to use for shard data
+     * @param indexMetaData        indexMetaData for the shard, including any mapping
+     * @param indexSearcherWrapper an optional wrapper to be used during searchers
+     * @param listeners            an optional set of listeners to add to the shard
+     */
+    protected IndexShard newShard(ShardRouting routing, ShardPath shardPath, IndexMetaData indexMetaData,
+                                  @Nullable IndexSearcherWrapper indexSearcherWrapper,
+                                  IndexingOperationListener... listeners) throws IOException {
+        final Settings nodeSettings = Settings.builder().put("node.name", routing.currentNodeId()).build();
+        final IndexSettings indexSettings = new IndexSettings(indexMetaData, nodeSettings);
+        final IndexShard indexShard;
+        final Store store = createStore(indexSettings, shardPath);
+        boolean success = false;
+        try {
+            IndexCache indexCache = new IndexCache(indexSettings, new DisabledQueryCache(indexSettings), null);
+            MapperService mapperService = MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(),
+                    indexSettings.getSettings());
+            mapperService.merge(indexMetaData, MapperService.MergeReason.MAPPING_RECOVERY, true);
+            SimilarityService similarityService = new SimilarityService(indexSettings, Collections.emptyMap());
+            final IndexEventListener indexEventListener = new IndexEventListener() {
+            };
+            final Engine.Warmer warmer = searcher -> {
+            };
+            IndicesFieldDataCache indicesFieldDataCache = new IndicesFieldDataCache(nodeSettings, new IndexFieldDataCache.Listener() {
+            });
+            IndexFieldDataService indexFieldDataService = new IndexFieldDataService(indexSettings, indicesFieldDataCache,
+                new NoneCircuitBreakerService(), mapperService);
+            indexShard = new IndexShard(routing, indexSettings, shardPath, store, indexCache, mapperService, similarityService,
+                indexFieldDataService, null, indexEventListener, indexSearcherWrapper, threadPool, BigArrays.NON_RECYCLING_INSTANCE, warmer,
+                Collections.emptyList(), Arrays.asList(listeners));
+            success = true;
+        } finally {
+            if (success == false) {
+                IOUtils.close(store);
+            }
+        }
+        return indexShard;
+    }
+
+    /**
+     * Takes an existing shard, closes it and and starts a new initialing shard at the same location
+     *
+     * @param listeners new listerns to use for the newly created shard
+     */
+    protected IndexShard reinitShard(IndexShard current, IndexingOperationListener... listeners) throws IOException {
+        final ShardRouting shardRouting = current.routingEntry();
+        return reinitShard(current, ShardRoutingHelper.initWithSameId(shardRouting,
+            shardRouting.primary() ? RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE : RecoverySource.PeerRecoverySource.INSTANCE
+        ), listeners);
+    }
+
+    /**
+     * Takes an existing shard, closes it and and starts a new initialing shard at the same location
+     *
+     * @param routing   the shard routing to use for the newly created shard.
+     * @param listeners new listerns to use for the newly created shard
+     */
+    protected IndexShard reinitShard(IndexShard current, ShardRouting routing, IndexingOperationListener... listeners) throws IOException {
+        closeShards(current);
+        return newShard(routing, current.shardPath(), current.indexSettings().getIndexMetaData(), null, listeners);
+    }
+
+    /**
+     * creates a new empyu shard and starts it. The shard will be either a replica or a primary.
+     */
+    protected IndexShard newStartedShard() throws IOException {
+        return newStartedShard(randomBoolean());
+    }
+
+    /**
+     * creates a new empty shard and starts it.
+     *
+     * @param primary controls whether the shard will be a primary or a replica.
+     */
+    protected IndexShard newStartedShard(boolean primary) throws IOException {
+        IndexShard shard = newShard(primary);
+        if (primary) {
+            recoveryShardFromStore(shard);
+        } else {
+            recoveryEmptyReplica(shard);
+        }
+        return shard;
+    }
+
+    protected void closeShards(IndexShard... shards) throws IOException {
+        closeShards(Arrays.asList(shards));
+    }
+
+    protected void closeShards(Iterable<IndexShard> shards) throws IOException {
+        for (IndexShard shard : shards) {
+            if (shard != null) {
+                try {
+                    shard.close("test", false);
+                } finally {
+                    IOUtils.close(shard.store());
+                }
+            }
+        }
+    }
+
+    protected void recoveryShardFromStore(IndexShard primary) throws IOException {
+        primary.markAsRecovering("store", new RecoveryState(primary.routingEntry(),
+            getFakeDiscoNode(primary.routingEntry().currentNodeId()),
+            null));
+        primary.recoverFromStore();
+        primary.updateRoutingEntry(ShardRoutingHelper.moveToStarted(primary.routingEntry()));
+    }
+
+    protected void recoveryEmptyReplica(IndexShard replica) throws IOException {
+        IndexShard primary = null;
+        try {
+            primary = newStartedShard(true);
+            recoverReplica(replica, primary);
+        } finally {
+            closeShards(primary);
+        }
+    }
+
+    private DiscoveryNode getFakeDiscoNode(String id) {
+        return new DiscoveryNode(id, new LocalTransportAddress("_fake_" + id), Version.CURRENT);
+    }
+
+    /** recovers a replica from the given primary **/
+    protected void recoverReplica(IndexShard replica, IndexShard primary) throws IOException {
+        recoverReplica(replica, primary,
+            (r, sourceNode) -> new RecoveryTarget(r, sourceNode, recoveryListener, version -> {
+            }),
+            true);
+    }
+
+    /**
+     * Recovers a replica from the give primary, allow the user to supply a custom recovery target.
+     * A typical usage of a custome recovery target is to assert things in the various stages of recovery
+     *
+     * @param markAsRecovering set to false if you have already marked the replica as recovering
+     */
+    protected void recoverReplica(IndexShard replica, IndexShard primary,
+                                  BiFunction<IndexShard, DiscoveryNode, RecoveryTarget> targetSupplier,
+                                  boolean markAsRecovering)
+        throws IOException {
+        final DiscoveryNode pNode = getFakeDiscoNode(primary.routingEntry().currentNodeId());
+        final DiscoveryNode rNode = getFakeDiscoNode(replica.routingEntry().currentNodeId());
+        if (markAsRecovering) {
+            replica.markAsRecovering("remote",
+                new RecoveryState(replica.routingEntry(), pNode, rNode));
+        } else {
+            assertEquals(replica.state(), IndexShardState.RECOVERING);
+        }
+        replica.prepareForIndexRecovery();
+        RecoveryTarget recoveryTarget = targetSupplier.apply(replica, pNode);
+        final String targetAllocationId = recoveryTarget.indexShard().routingEntry().allocationId().getId();
+        StartRecoveryRequest request = new StartRecoveryRequest(replica.shardId(), targetAllocationId, pNode, rNode,
+            getMetadataSnapshotOrEmpty(replica), false, 0);
+        RecoverySourceHandler recovery = new RecoverySourceHandler(primary, recoveryTarget, request, () -> 0L, e -> () -> {},
+            (int) ByteSizeUnit.MB.toKB(1), logger);
+        recovery.recoverToTarget();
+        recoveryTarget.markAsDone();
+        replica.updateRoutingEntry(ShardRoutingHelper.moveToStarted(replica.routingEntry()));
+    }
+
+    private Store.MetadataSnapshot getMetadataSnapshotOrEmpty(IndexShard replica) throws IOException {
+        Store.MetadataSnapshot result;
+        try {
+            result = replica.snapshotStoreMetadata();
+        } catch (IndexNotFoundException e) {
+            // OK!
+            result = Store.MetadataSnapshot.EMPTY;
+        } catch (IOException e) {
+            logger.warn("failed read store, treating as empty", e);
+            result = Store.MetadataSnapshot.EMPTY;
+        }
+        return result;
+    }
+
+    protected Set<Uid> getShardDocUIDs(final IndexShard shard) throws IOException {
+        shard.refresh("get_uids");
+        try (Engine.Searcher searcher = shard.acquireSearcher("test")) {
+            Set<Uid> ids = new HashSet<>();
+            for (LeafReaderContext leafContext : searcher.reader().leaves()) {
+                LeafReader reader = leafContext.reader();
+                Bits liveDocs = reader.getLiveDocs();
+                for (int i = 0; i < reader.maxDoc(); i++) {
+                    if (liveDocs == null || liveDocs.get(i)) {
+                        Document uuid = reader.document(i, Collections.singleton(UidFieldMapper.NAME));
+                        ids.add(Uid.createUid(uuid.get(UidFieldMapper.NAME)));
+                    }
+                }
+            }
+            return ids;
+        }
+    }
+
+    protected void assertDocCount(IndexShard shard, int docDount) throws IOException {
+        assertThat(getShardDocUIDs(shard), hasSize(docDount));
+    }
+
+    protected void assertDocs(IndexShard shard, Uid... uids) throws IOException {
+        final Set<Uid> shardDocUIDs = getShardDocUIDs(shard);
+        assertThat(shardDocUIDs, contains(uids));
+        assertThat(shardDocUIDs, hasSize(uids.length));
+    }
+
+
+    protected Engine.Index indexDoc(IndexShard shard, String type, String id) throws IOException {
+        return indexDoc(shard, type, id, "{}");
+    }
+
+    protected Engine.Index indexDoc(IndexShard shard, String type, String id, String source) throws IOException {
+        return indexDoc(shard, type, id, source, XContentType.JSON);
+    }
+
+    protected Engine.Index indexDoc(IndexShard shard, String type, String id, String source, XContentType xContentType) throws IOException {
+        final Engine.Index index;
+        if (shard.routingEntry().primary()) {
+            index = shard.prepareIndexOnPrimary(
+                SourceToParse.source(SourceToParse.Origin.PRIMARY, shard.shardId().getIndexName(), type, id, new BytesArray(source),
+                    xContentType),
+                Versions.MATCH_ANY,
+                VersionType.INTERNAL,
+                IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP,
+                false);
+        } else {
+            index = shard.prepareIndexOnReplica(
+                SourceToParse.source(SourceToParse.Origin.PRIMARY, shard.shardId().getIndexName(), type, id, new BytesArray(source),
+                    xContentType),
+                1, VersionType.EXTERNAL, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false);
+        }
+        shard.index(index);
+        return index;
+    }
+
+    protected Engine.Delete deleteDoc(IndexShard shard, String type, String id) throws IOException {
+        final Engine.Delete delete;
+        if (shard.routingEntry().primary()) {
+            delete = shard.prepareDeleteOnPrimary(type, id, Versions.MATCH_ANY, VersionType.INTERNAL);
+        } else {
+            delete = shard.prepareDeleteOnPrimary(type, id, 1, VersionType.EXTERNAL);
+        }
+        shard.delete(delete);
+        return delete;
+    }
+
+    protected void flushShard(IndexShard shard) {
+        flushShard(shard, false);
+    }
+
+    protected void flushShard(IndexShard shard, boolean force) {
+        shard.flush(new FlushRequest(shard.shardId().getIndexName()).force(force));
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/ingest/IngestDocumentMatcher.java b/test/framework/src/main/java/org/elasticsearch/ingest/IngestDocumentMatcher.java
new file mode 100644
index 0000000000000..bb058c5cfdbb5
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/ingest/IngestDocumentMatcher.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest;
+
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertThat;
+
+public class IngestDocumentMatcher {
+    /**
+     * Helper method to assert the equivalence between two IngestDocuments.
+     *
+     * @param a first object to compare
+     * @param b second object to compare
+     */
+    public static void assertIngestDocument(Object a, Object b) {
+        if (a instanceof Map) {
+            Map<?, ?> mapA = (Map<?, ?>) a;
+            Map<?, ?> mapB = (Map<?, ?>) b;
+            for (Map.Entry<?, ?> entry : mapA.entrySet()) {
+                if (entry.getValue() instanceof List || entry.getValue() instanceof Map) {
+                    assertIngestDocument(entry.getValue(), mapB.get(entry.getKey()));
+                }
+            }
+        } else if (a instanceof List) {
+            List<?> listA = (List<?>) a;
+            List<?> listB = (List<?>) b;
+            for (int i = 0; i < listA.size(); i++) {
+                Object value = listA.get(i);
+                if (value instanceof List || value instanceof Map) {
+                    assertIngestDocument(value, listB.get(i));
+                }
+            }
+        } else if (a instanceof byte[]) {
+            assertArrayEquals((byte[]) a, (byte[])b);
+        } else if (a instanceof IngestDocument) {
+            IngestDocument docA = (IngestDocument) a;
+            IngestDocument docB = (IngestDocument) b;
+            assertIngestDocument(docA.getSourceAndMetadata(), docB.getSourceAndMetadata());
+            assertIngestDocument(docA.getIngestMetadata(), docB.getIngestMetadata());
+        } else {
+            String msg = String.format(Locale.ROOT, "Expected %s class to be equal to %s", a.getClass().getName(), b.getClass().getName());
+            assertThat(msg, a, equalTo(b));
+        }
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/ingest/RandomDocumentPicks.java b/test/framework/src/main/java/org/elasticsearch/ingest/RandomDocumentPicks.java
index 3be4cd3edf4b1..e36b5913ad6d9 100644
--- a/test/framework/src/main/java/org/elasticsearch/ingest/RandomDocumentPicks.java
+++ b/test/framework/src/main/java/org/elasticsearch/ingest/RandomDocumentPicks.java
@@ -19,10 +19,12 @@
 
 package org.elasticsearch.ingest;
 
-import com.carrotsearch.randomizedtesting.generators.RandomInts;
+import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
 import com.carrotsearch.randomizedtesting.generators.RandomStrings;
 
+import org.elasticsearch.common.Strings;
+
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -41,7 +43,7 @@ private RandomDocumentPicks() {
      * path to refer to a field name using the dot notation.
      */
     public static String randomFieldName(Random random) {
-        int numLevels = RandomInts.randomIntBetween(random, 1, 5);
+        int numLevels = RandomNumbers.randomIntBetween(random, 1, 5);
         String fieldName = "";
         for (int i = 0; i < numLevels; i++) {
             if (i > 0) {
@@ -59,7 +61,7 @@ public static String randomLeafFieldName(Random random) {
         String fieldName;
         do {
             fieldName = randomString(random);
-        } while (fieldName.contains("."));
+        } while (fieldName.contains(".") || Strings.hasText(fieldName) == false);
         return fieldName;
     }
 
@@ -169,7 +171,7 @@ public static Object randomFieldValue(Random random) {
     }
 
     private static Object randomFieldValue(Random random, int currentDepth) {
-        switch(RandomInts.randomIntBetween(random, 0, 9)) {
+        switch(RandomNumbers.randomIntBetween(random, 0, 9)) {
             case 0:
                 return randomString(random);
             case 1:
@@ -180,28 +182,28 @@ private static Object randomFieldValue(Random random, int currentDepth) {
                 return random.nextDouble();
             case 4:
                 List<String> stringList = new ArrayList<>();
-                int numStringItems = RandomInts.randomIntBetween(random, 1, 10);
+                int numStringItems = RandomNumbers.randomIntBetween(random, 1, 10);
                 for (int j = 0; j < numStringItems; j++) {
                     stringList.add(randomString(random));
                 }
                 return stringList;
             case 5:
                 List<Integer> intList = new ArrayList<>();
-                int numIntItems = RandomInts.randomIntBetween(random, 1, 10);
+                int numIntItems = RandomNumbers.randomIntBetween(random, 1, 10);
                 for (int j = 0; j < numIntItems; j++) {
                     intList.add(random.nextInt());
                 }
                 return intList;
             case 6:
                 List<Boolean> booleanList = new ArrayList<>();
-                int numBooleanItems = RandomInts.randomIntBetween(random, 1, 10);
+                int numBooleanItems = RandomNumbers.randomIntBetween(random, 1, 10);
                 for (int j = 0; j < numBooleanItems; j++) {
                     booleanList.add(random.nextBoolean());
                 }
                 return booleanList;
             case 7:
                 List<Double> doubleList = new ArrayList<>();
-                int numDoubleItems = RandomInts.randomIntBetween(random, 1, 10);
+                int numDoubleItems = RandomNumbers.randomIntBetween(random, 1, 10);
                 for (int j = 0; j < numDoubleItems; j++) {
                     doubleList.add(random.nextDouble());
                 }
@@ -211,7 +213,7 @@ private static Object randomFieldValue(Random random, int currentDepth) {
                 addRandomFields(random, newNode, ++currentDepth);
                 return newNode;
             case 9:
-                byte[] byteArray = new byte[RandomInts.randomIntBetween(random, 1, 1024)];
+                byte[] byteArray = new byte[RandomNumbers.randomIntBetween(random, 1, 1024)];
                 random.nextBytes(byteArray);
                 return byteArray;
             default:
@@ -230,7 +232,7 @@ private static void addRandomFields(Random random, Map<String, Object> parentNod
         if (currentDepth > 5) {
             return;
         }
-        int numFields = RandomInts.randomIntBetween(random, 1, 10);
+        int numFields = RandomNumbers.randomIntBetween(random, 1, 10);
         for (int i = 0; i < numFields; i++) {
             String fieldName = randomLeafFieldName(random);
             Object fieldValue = randomFieldValue(random, currentDepth);
diff --git a/test/framework/src/main/java/org/elasticsearch/node/MockNode.java b/test/framework/src/main/java/org/elasticsearch/node/MockNode.java
index 37da93b82571d..8774ba5836b6c 100644
--- a/test/framework/src/main/java/org/elasticsearch/node/MockNode.java
+++ b/test/framework/src/main/java/org/elasticsearch/node/MockNode.java
@@ -19,18 +19,32 @@
 
 package org.elasticsearch.node;
 
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.cluster.ClusterInfoService;
+import org.elasticsearch.cluster.MockInternalClusterInfoService;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.MockBigArrays;
+import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
-import org.elasticsearch.node.internal.InternalSettingsPreparer;
+import org.elasticsearch.indices.recovery.RecoverySettings;
 import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.plugins.SearchPlugin;
+import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.MockSearchService;
 import org.elasticsearch.search.SearchService;
+import org.elasticsearch.search.fetch.FetchPhase;
+import org.elasticsearch.test.transport.MockTransportService;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.Transport;
+import org.elasticsearch.transport.TransportInterceptor;
+import org.elasticsearch.transport.TransportService;
 
 import java.util.Collection;
-import java.util.List;
+import java.util.function.Function;
 
 /**
  * A node for testing which allows:
@@ -62,11 +76,53 @@ protected BigArrays createBigArrays(Settings settings, CircuitBreakerService cir
         return new MockBigArrays(settings, circuitBreakerService);
     }
 
+
     @Override
-    protected Class<? extends SearchService> pickSearchServiceImplementation() {
+    protected SearchService newSearchService(ClusterService clusterService, IndicesService indicesService,
+                                             ThreadPool threadPool, ScriptService scriptService, BigArrays bigArrays,
+                                             FetchPhase fetchPhase) {
         if (getPluginsService().filterPlugins(MockSearchService.TestPlugin.class).isEmpty()) {
-            return super.pickSearchServiceImplementation();
+            return super.newSearchService(clusterService, indicesService, threadPool, scriptService, bigArrays, fetchPhase);
+        }
+        return new MockSearchService(clusterService, indicesService, threadPool, scriptService, bigArrays, fetchPhase);
+    }
+
+    @Override
+    protected TransportService newTransportService(Settings settings, Transport transport, ThreadPool threadPool,
+                                                   TransportInterceptor interceptor,
+                                                   Function<BoundTransportAddress, DiscoveryNode> localNodeFactory,
+                                                   ClusterSettings clusterSettings) {
+        // we use the MockTransportService.TestPlugin class as a marker to create a network
+        // module with this MockNetworkService. NetworkService is such an integral part of the systme
+        // we don't allow to plug it in from plugins or anything. this is a test-only override and
+        // can't be done in a production env.
+        if (getPluginsService().filterPlugins(MockTransportService.TestPlugin.class).isEmpty()) {
+            return super.newTransportService(settings, transport, threadPool, interceptor, localNodeFactory, clusterSettings);
+        } else {
+            return new MockTransportService(settings, transport, threadPool, interceptor, localNodeFactory, clusterSettings);
+        }
+    }
+
+    @Override
+    protected Node newTribeClientNode(Settings settings, Collection<Class<? extends Plugin>> classpathPlugins) {
+        return new MockNode(settings, classpathPlugins);
+    }
+
+    @Override
+    protected void processRecoverySettings(ClusterSettings clusterSettings, RecoverySettings recoverySettings) {
+        if (false == getPluginsService().filterPlugins(RecoverySettingsChunkSizePlugin.class).isEmpty()) {
+            clusterSettings.addSettingsUpdateConsumer(RecoverySettingsChunkSizePlugin.CHUNK_SIZE_SETTING, recoverySettings::setChunkSize);
+        }
+    }
+
+    @Override
+    protected ClusterInfoService newClusterInfoService(Settings settings, ClusterService clusterService,
+                                                       ThreadPool threadPool, NodeClient client) {
+        if (getPluginsService().filterPlugins(MockInternalClusterInfoService.TestPlugin.class).isEmpty()) {
+            return super.newClusterInfoService(settings, clusterService, threadPool, client);
+        } else {
+            return new MockInternalClusterInfoService(settings, clusterService, threadPool, client);
         }
-        return MockSearchService.class;
     }
 }
+
diff --git a/test/framework/src/main/java/org/elasticsearch/node/NodeTests.java b/test/framework/src/main/java/org/elasticsearch/node/NodeTests.java
deleted file mode 100644
index db8b6825ec8e2..0000000000000
--- a/test/framework/src/main/java/org/elasticsearch/node/NodeTests.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.node;
-
-import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.common.network.NetworkModule;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.InternalTestCluster;
-
-import java.io.IOException;
-import java.nio.file.Path;
-import java.util.Collections;
-
-import static org.hamcrest.Matchers.equalTo;
-
-public class NodeTests extends ESTestCase {
-
-    public void testNodeName() throws IOException {
-        final Path tempDir = createTempDir();
-        final String name = randomBoolean() ? randomAsciiOfLength(10) : null;
-        Settings.Builder settings = Settings.builder()
-            .put(ClusterName.CLUSTER_NAME_SETTING.getKey(), InternalTestCluster.clusterName("single-node-cluster", randomLong()))
-            .put(Environment.PATH_HOME_SETTING.getKey(), tempDir)
-            .put(NetworkModule.HTTP_ENABLED.getKey(), false)
-            .put("discovery.type", "local")
-            .put("transport.type", "local")
-            .put(Node.NODE_DATA_SETTING.getKey(), true);
-        if (name != null) {
-            settings.put(Node.NODE_NAME_SETTING.getKey(), name);
-        }
-        try (Node node = new MockNode(settings.build(), Collections.emptyList())) {
-            final Settings nodeSettings = randomBoolean() ? node.settings() : node.getEnvironment().settings();
-            if (name == null) {
-                assertThat(Node.NODE_NAME_SETTING.get(nodeSettings), equalTo(node.getNodeEnvironment().nodeId().substring(0, 7)));
-            } else {
-                assertThat(Node.NODE_NAME_SETTING.get(nodeSettings), equalTo(name));
-            }
-        }
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/node/RecoverySettingsChunkSizePlugin.java b/test/framework/src/main/java/org/elasticsearch/node/RecoverySettingsChunkSizePlugin.java
new file mode 100644
index 0000000000000..14616adc528da
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/node/RecoverySettingsChunkSizePlugin.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.node;
+
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.indices.recovery.RecoverySettings;
+import org.elasticsearch.plugins.Plugin;
+
+import java.util.List;
+
+import static java.util.Collections.singletonList;
+
+/**
+ * Marker plugin that will trigger {@link MockNode} making {@link #CHUNK_SIZE_SETTING} dynamic.
+ */
+public class RecoverySettingsChunkSizePlugin extends Plugin {
+    /**
+     * The chunk size. Only exposed by tests.
+     */
+    public static final Setting<ByteSizeValue> CHUNK_SIZE_SETTING = Setting.byteSizeSetting("indices.recovery.chunk_size",
+            RecoverySettings.DEFAULT_CHUNK_SIZE, Property.Dynamic, Property.NodeScope);
+
+    @Override
+    public List<Setting<?>> getSettings() {
+        return singletonList(CHUNK_SIZE_SETTING);
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreContainerTestCase.java b/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreContainerTestCase.java
index e25c16cb5c0a3..42029b053a4b0 100644
--- a/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreContainerTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreContainerTestCase.java
@@ -46,7 +46,7 @@
 public abstract class ESBlobStoreContainerTestCase extends ESTestCase {
 
     public void testWriteRead() throws IOException {
-        try(final BlobStore store = newBlobStore()) {
+        try(BlobStore store = newBlobStore()) {
             final BlobContainer container = store.blobContainer(new BlobPath());
             byte[] data = randomBytes(randomIntBetween(10, scaledRandomIntBetween(1024, 1 << 16)));
             writeBlob(container, "foobar", new BytesArray(data));
@@ -65,7 +65,7 @@ public void testWriteRead() throws IOException {
     }
 
     public void testMoveAndList() throws IOException {
-        try(final BlobStore store = newBlobStore()) {
+        try(BlobStore store = newBlobStore()) {
             final BlobContainer container = store.blobContainer(new BlobPath());
             assertThat(container.listBlobs().size(), equalTo(0));
             int numberOfFooBlobs = randomIntBetween(0, 10);
@@ -113,7 +113,7 @@ public void testMoveAndList() throws IOException {
     }
 
     public void testDeleteBlob() throws IOException {
-        try (final BlobStore store = newBlobStore()) {
+        try (BlobStore store = newBlobStore()) {
             final String blobName = "foobar";
             final BlobContainer container = store.blobContainer(new BlobPath());
             expectThrows(IOException.class, () -> container.deleteBlob(blobName));
@@ -129,7 +129,7 @@ public void testDeleteBlob() throws IOException {
     }
 
     public void testVerifyOverwriteFails() throws IOException {
-        try (final BlobStore store = newBlobStore()) {
+        try (BlobStore store = newBlobStore()) {
             final String blobName = "foobar";
             final BlobContainer container = store.blobContainer(new BlobPath());
             byte[] data = randomBytes(randomIntBetween(10, scaledRandomIntBetween(1024, 1 << 16)));
diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreTestCase.java b/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreTestCase.java
index 0cec570dbe8cc..e7f8edb1fa208 100644
--- a/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreTestCase.java
@@ -36,7 +36,7 @@
 public abstract class ESBlobStoreTestCase extends ESTestCase {
 
     public void testContainerCreationAndDeletion() throws IOException {
-        try(final BlobStore store = newBlobStore()) {
+        try(BlobStore store = newBlobStore()) {
             final BlobContainer containerFoo = store.blobContainer(new BlobPath().add("foo"));
             final BlobContainer containerBar = store.blobContainer(new BlobPath().add("bar"));
             byte[] data1 = randomBytes(randomIntBetween(10, scaledRandomIntBetween(1024, 1 << 16)));
diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java
index d61155ecd6015..683534306f6ac 100644
--- a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java
@@ -222,7 +222,7 @@ protected void addRandomDocuments(String name, int numDocs) throws ExecutionExce
         IndexRequestBuilder[] indexRequestBuilders = new IndexRequestBuilder[numDocs];
         for (int i = 0; i < numDocs; i++) {
             indexRequestBuilders[i] = client().prepareIndex(name, name, Integer.toString(i))
-                .setRouting(randomAsciiOfLength(randomIntBetween(1, 10))).setSource("field", "value");
+                .setRouting(randomAlphaOfLength(randomIntBetween(1, 10))).setSource("field", "value");
         }
         indexRandom(true, indexRequestBuilders);
     }
@@ -262,6 +262,6 @@ public static void assertSuccessfulRestore(RestoreSnapshotResponse response) {
     }
 
     public static String randomAsciiName() {
-        return randomAsciiOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT);
+        return randomAlphaOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT);
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/rest/HeadBodyIsEmptyIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/rest/HeadBodyIsEmptyIntegTestCase.java
new file mode 100644
index 0000000000000..bd7d6e7e22899
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/rest/HeadBodyIsEmptyIntegTestCase.java
@@ -0,0 +1,197 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.rest;
+
+import org.apache.http.entity.StringEntity;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.hamcrest.Matcher;
+
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.util.Map;
+
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.singletonMap;
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.rest.RestStatus.NOT_FOUND;
+import static org.elasticsearch.rest.RestStatus.OK;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+
+/**
+ * Tests that HTTP HEAD requests don't respond with a body.
+ */
+public abstract class HeadBodyIsEmptyIntegTestCase extends ESRestTestCase {
+
+    public void testHeadRoot() throws IOException {
+        headTestCase("/", emptyMap(), greaterThan(0));
+        headTestCase("/", singletonMap("pretty", ""), greaterThan(0));
+        headTestCase("/", singletonMap("pretty", "true"), greaterThan(0));
+    }
+
+    private void createTestDoc() throws IOException {
+        createTestDoc("test", "test");
+    }
+
+    private void createTestDoc(final String indexName, final String typeName) throws IOException {
+        try (XContentBuilder builder = jsonBuilder()) {
+            builder.startObject();
+            {
+                builder.field("test", "test");
+            }
+            builder.endObject();
+            client().performRequest("PUT", "/" + indexName + "/" + typeName + "/" + "1", emptyMap(), new StringEntity(builder.string()));
+        }
+    }
+
+    public void testDocumentExists() throws IOException {
+        createTestDoc();
+        headTestCase("/test/test/1", emptyMap(), greaterThan(0));
+        headTestCase("/test/test/1", singletonMap("pretty", "true"), greaterThan(0));
+        headTestCase("/test/test/2", emptyMap(), NOT_FOUND.getStatus(), greaterThan(0));
+    }
+
+    public void testIndexExists() throws IOException {
+        createTestDoc();
+        headTestCase("/test", emptyMap(), greaterThan(0));
+        headTestCase("/test", singletonMap("pretty", "true"), greaterThan(0));
+    }
+
+    public void testTypeExists() throws IOException {
+        createTestDoc();
+        headTestCase("/test/_mapping/test", emptyMap(), greaterThan(0));
+        headTestCase("/test/_mapping/test", singletonMap("pretty", "true"), greaterThan(0));
+    }
+
+    public void testTypeDoesNotExist() throws IOException {
+        createTestDoc();
+        headTestCase("/test/_mapping/does-not-exist", emptyMap(), NOT_FOUND.getStatus(), greaterThan(0));
+        headTestCase("/text/_mapping/test,does-not-exist", emptyMap(), NOT_FOUND.getStatus(), greaterThan(0));
+    }
+
+    public void testAliasExists() throws IOException {
+        createTestDoc();
+        try (XContentBuilder builder = jsonBuilder()) {
+            builder.startObject();
+            {
+                builder.startArray("actions");
+                {
+                    builder.startObject();
+                    {
+                        builder.startObject("add");
+                        {
+                            builder.field("index", "test");
+                            builder.field("alias", "test_alias");
+                        }
+                        builder.endObject();
+                    }
+                    builder.endObject();
+                }
+                builder.endArray();
+            }
+            builder.endObject();
+
+            client().performRequest("POST", "_aliases", emptyMap(), new StringEntity(builder.string()));
+            headTestCase("/_alias/test_alias", emptyMap(), greaterThan(0));
+            headTestCase("/test/_alias/test_alias", emptyMap(), greaterThan(0));
+        }
+    }
+
+    public void testAliasDoesNotExist() throws IOException {
+        createTestDoc();
+        headTestCase("/_alias/test_alias", emptyMap(), NOT_FOUND.getStatus(), greaterThan(0));
+        headTestCase("/test/_alias/test_alias", emptyMap(), NOT_FOUND.getStatus(), greaterThan(0));
+    }
+
+    public void testTemplateExists() throws IOException {
+        try (XContentBuilder builder = jsonBuilder()) {
+            builder.startObject();
+            {
+                builder.array("template", "*");
+                builder.startObject("settings");
+                {
+                    builder.field("number_of_replicas", 0);
+                }
+                builder.endObject();
+            }
+            builder.endObject();
+
+            client().performRequest("PUT", "/_template/template", emptyMap(), new StringEntity(builder.string()));
+            headTestCase("/_template/template", emptyMap(), greaterThan(0));
+        }
+    }
+
+    public void testGetSourceAction() throws IOException {
+        createTestDoc();
+        headTestCase("/test/test/1/_source", emptyMap(), greaterThan(0));
+        headTestCase("/test/test/2/_source", emptyMap(), NOT_FOUND.getStatus(), equalTo(0));
+
+        try (XContentBuilder builder = jsonBuilder()) {
+            builder.startObject();
+            {
+                builder.startObject("mappings");
+                {
+                    builder.startObject("test-no-source");
+                    {
+                        builder.startObject("_source");
+                        {
+                            builder.field("enabled", false);
+                        }
+                        builder.endObject();
+                    }
+                    builder.endObject();
+                }
+                builder.endObject();
+            }
+            builder.endObject();
+            client().performRequest("PUT", "/test-no-source", emptyMap(), new StringEntity(builder.string()));
+            createTestDoc("test-no-source", "test-no-source");
+            headTestCase("/test-no-source/test-no-source/1/_source", emptyMap(), NOT_FOUND.getStatus(), equalTo(0));
+        }
+    }
+
+    public void testException() throws IOException {
+        /*
+         * This will throw an index not found exception which will be sent on the channel; previously when handling HEAD requests that would
+         * throw an exception, the content was swallowed and a content length header of zero was returned. Instead of swallowing the content
+         * we now let it rise up to the upstream channel so that it can compute the content length that would be returned. This test case is
+         * a test for this situation.
+         */
+        headTestCase("/index-not-found-exception", emptyMap(), NOT_FOUND.getStatus(), greaterThan(0));
+    }
+
+    private void headTestCase(final String url, final Map<String, String> params, final Matcher<Integer> matcher) throws IOException {
+        headTestCase(url, params, OK.getStatus(), matcher);
+    }
+
+    private void headTestCase(
+            final String url,
+            final Map<String, String> params,
+            final int expectedStatusCode,
+            final Matcher<Integer> matcher) throws IOException {
+        Response response = client().performRequest("HEAD", url, params);
+        assertEquals(expectedStatusCode, response.getStatusLine().getStatusCode());
+        assertThat(Integer.valueOf(response.getHeader("Content-Length")), matcher);
+        assertNull("HEAD requests shouldn't have a response body but " + url + " did", response.getEntity());
+    }
+
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java
index e036676677f9b..62e3d51ad10b4 100644
--- a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java
+++ b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java
@@ -32,6 +32,8 @@
 import java.util.Map;
 import java.util.function.Function;
 
+import static java.util.Collections.emptyMap;
+
 /**
  * A mocked script engine that can be used for testing purpose.
  *
@@ -182,8 +184,7 @@ public MockSearchScript(SearchLookup lookup, Map<String, Object> vars, Function<
         public LeafSearchScript getLeafSearchScript(LeafReaderContext context) throws IOException {
             LeafSearchLookup leafLookup = lookup.getLeafSearchLookup(context);
 
-            Map<String, Object> ctx = new HashMap<>();
-            ctx.putAll(leafLookup.asMap());
+            Map<String, Object> ctx = new HashMap<>(leafLookup.asMap());
             if (vars != null) {
                 ctx.putAll(vars);
             }
@@ -215,4 +216,9 @@ public boolean needsScores() {
             return true;
         }
     }
+
+    public static Script mockInlineScript(final String script) {
+        return new Script(ScriptType.INLINE, "mock", script, emptyMap());
+    }
+
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/search/MockSearchService.java b/test/framework/src/main/java/org/elasticsearch/search/MockSearchService.java
index cae5b2ff95ba9..bf300889cd50d 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/MockSearchService.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/MockSearchService.java
@@ -20,9 +20,6 @@
 package org.elasticsearch.search;
 
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.settings.ClusterSettings;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.node.MockNode;
@@ -69,11 +66,10 @@ static void removeActiveContext(SearchContext context) {
         ACTIVE_SEARCH_CONTEXTS.remove(context);
     }
 
-    @Inject
-    public MockSearchService(Settings settings, ClusterSettings clusterSettings, ClusterService clusterService,
+    public MockSearchService(ClusterService clusterService,
             IndicesService indicesService, ThreadPool threadPool, ScriptService scriptService,
             BigArrays bigArrays, FetchPhase fetchPhase) {
-        super(settings, clusterSettings, clusterService, indicesService, threadPool, scriptService, bigArrays, fetchPhase);
+        super(clusterService, indicesService, threadPool, scriptService, bigArrays, fetchPhase);
     }
 
     @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java b/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java
new file mode 100644
index 0000000000000..4cd46fe5076d3
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java
@@ -0,0 +1,351 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search;
+
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchType;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.common.text.Text;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptType;
+import org.elasticsearch.search.aggregations.AggregationBuilders;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.collapse.CollapseBuilder;
+import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
+import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
+import org.elasticsearch.search.rescore.RescoreBuilder;
+import org.elasticsearch.search.searchafter.SearchAfterBuilder;
+import org.elasticsearch.search.slice.SliceBuilder;
+import org.elasticsearch.search.sort.ScriptSortBuilder;
+import org.elasticsearch.search.sort.SortBuilders;
+import org.elasticsearch.search.sort.SortOrder;
+import org.elasticsearch.search.suggest.SuggestBuilder;
+import org.elasticsearch.test.AbstractQueryTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Supplier;
+
+import static java.util.Collections.emptyMap;
+import static org.elasticsearch.test.ESTestCase.between;
+import static org.elasticsearch.test.ESTestCase.generateRandomStringArray;
+import static org.elasticsearch.test.ESTestCase.mockScript;
+import static org.elasticsearch.test.ESTestCase.randomAlphaOfLengthBetween;
+import static org.elasticsearch.test.ESTestCase.randomBoolean;
+import static org.elasticsearch.test.ESTestCase.randomByte;
+import static org.elasticsearch.test.ESTestCase.randomDouble;
+import static org.elasticsearch.test.ESTestCase.randomFloat;
+import static org.elasticsearch.test.ESTestCase.randomFrom;
+import static org.elasticsearch.test.ESTestCase.randomInt;
+import static org.elasticsearch.test.ESTestCase.randomIntBetween;
+import static org.elasticsearch.test.ESTestCase.randomLong;
+import static org.elasticsearch.test.ESTestCase.randomPositiveTimeValue;
+import static org.elasticsearch.test.ESTestCase.randomShort;
+import static org.elasticsearch.test.ESTestCase.randomTimeValue;
+
+/**
+ * Builds random search requests.
+ */
+public class RandomSearchRequestGenerator {
+    private RandomSearchRequestGenerator() {}
+
+    /**
+     * Build a random search request.
+     *
+     * @param randomSearchSourceBuilder builds a random {@link SearchSourceBuilder}. You can use
+     *        {@link #randomSearchSourceBuilder(Supplier, Supplier, Supplier, Supplier, Supplier)}.
+     */
+    public static SearchRequest randomSearchRequest(Supplier<SearchSourceBuilder> randomSearchSourceBuilder) throws IOException {
+        SearchRequest searchRequest = new SearchRequest();
+        if (randomBoolean()) {
+            searchRequest.indices(generateRandomStringArray(10, 10, false, false));
+        }
+        if (randomBoolean()) {
+            searchRequest.indicesOptions(IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()));
+        }
+        if (randomBoolean()) {
+            searchRequest.types(generateRandomStringArray(10, 10, false, false));
+        }
+        if (randomBoolean()) {
+            searchRequest.preference(randomAlphaOfLengthBetween(3, 10));
+        }
+        if (randomBoolean()) {
+            searchRequest.requestCache(randomBoolean());
+        }
+        if (randomBoolean()) {
+            searchRequest.routing(randomAlphaOfLengthBetween(3, 10));
+        }
+        if (randomBoolean()) {
+            searchRequest.scroll(randomPositiveTimeValue());
+        }
+        if (randomBoolean()) {
+            searchRequest.searchType(randomFrom(SearchType.DFS_QUERY_THEN_FETCH, SearchType.QUERY_THEN_FETCH));
+        }
+        if (randomBoolean()) {
+            searchRequest.source(randomSearchSourceBuilder.get());
+        }
+        return searchRequest;
+    }
+
+    public static SearchSourceBuilder randomSearchSourceBuilder(
+            Supplier<HighlightBuilder> randomHighlightBuilder,
+            Supplier<SuggestBuilder> randomSuggestBuilder,
+            Supplier<RescoreBuilder<?>> randomRescoreBuilder,
+            Supplier<List<SearchExtBuilder>> randomExtBuilders,
+            Supplier<CollapseBuilder> randomCollapseBuilder) {
+        SearchSourceBuilder builder = new SearchSourceBuilder();
+        if (randomBoolean()) {
+            builder.from(randomIntBetween(0, 10000));
+        }
+        if (randomBoolean()) {
+            builder.size(randomIntBetween(0, 10000));
+        }
+        if (randomBoolean()) {
+            builder.explain(randomBoolean());
+        }
+        if (randomBoolean()) {
+            builder.version(randomBoolean());
+        }
+        if (randomBoolean()) {
+            builder.trackScores(randomBoolean());
+        }
+        if (randomBoolean()) {
+            builder.minScore(randomFloat() * 1000);
+        }
+        if (randomBoolean()) {
+            builder.timeout(TimeValue.parseTimeValue(randomTimeValue(), null, "timeout"));
+        }
+        if (randomBoolean()) {
+            builder.terminateAfter(randomIntBetween(1, 100000));
+        }
+
+        switch(randomInt(2)) {
+            case 0:
+                builder.storedFields();
+                break;
+            case 1:
+                builder.storedField("_none_");
+                break;
+            case 2:
+                int fieldsSize = randomInt(25);
+                List<String> fields = new ArrayList<>(fieldsSize);
+                for (int i = 0; i < fieldsSize; i++) {
+                    fields.add(randomAlphaOfLengthBetween(5, 50));
+                }
+                builder.storedFields(fields);
+                break;
+            default:
+                throw new IllegalStateException();
+        }
+
+        if (randomBoolean()) {
+            int scriptFieldsSize = randomInt(25);
+            for (int i = 0; i < scriptFieldsSize; i++) {
+                if (randomBoolean()) {
+                    builder.scriptField(randomAlphaOfLengthBetween(5, 50), mockScript("foo"), randomBoolean());
+                } else {
+                    builder.scriptField(randomAlphaOfLengthBetween(5, 50), mockScript("foo"));
+                }
+            }
+        }
+        if (randomBoolean()) {
+            FetchSourceContext fetchSourceContext;
+            int branch = randomInt(5);
+            String[] includes = new String[randomIntBetween(0, 20)];
+            for (int i = 0; i < includes.length; i++) {
+                includes[i] = randomAlphaOfLengthBetween(5, 20);
+            }
+            String[] excludes = new String[randomIntBetween(0, 20)];
+            for (int i = 0; i < excludes.length; i++) {
+                excludes[i] = randomAlphaOfLengthBetween(5, 20);
+            }
+            switch (branch) {
+                case 0:
+                    fetchSourceContext = new FetchSourceContext(randomBoolean());
+                    break;
+                case 1:
+                    fetchSourceContext = new FetchSourceContext(true, includes, excludes);
+                    break;
+                case 2:
+                    fetchSourceContext = new FetchSourceContext(true, new String[]{randomAlphaOfLengthBetween(5, 20)},
+                        new String[]{randomAlphaOfLengthBetween(5, 20)});
+                    break;
+                case 3:
+                    fetchSourceContext = new FetchSourceContext(true, includes, excludes);
+                    break;
+                case 4:
+                    fetchSourceContext = new FetchSourceContext(true, includes, null);
+                    break;
+                case 5:
+                    fetchSourceContext = new FetchSourceContext(true, new String[] {randomAlphaOfLengthBetween(5, 20)}, null);
+                    break;
+                default:
+                    throw new IllegalStateException();
+            }
+            builder.fetchSource(fetchSourceContext);
+        }
+        if (randomBoolean()) {
+            int size = randomIntBetween(0, 20);
+            List<String> statsGroups = new ArrayList<>(size);
+            for (int i = 0; i < size; i++) {
+                statsGroups.add(randomAlphaOfLengthBetween(5, 20));
+            }
+            builder.stats(statsGroups);
+        }
+        if (randomBoolean()) {
+            int indexBoostSize = randomIntBetween(1, 10);
+            for (int i = 0; i < indexBoostSize; i++) {
+                builder.indexBoost(randomAlphaOfLengthBetween(5, 20), randomFloat() * 10);
+            }
+        }
+        if (randomBoolean()) {
+            builder.query(QueryBuilders.termQuery(randomAlphaOfLengthBetween(5, 20), randomAlphaOfLengthBetween(5, 20)));
+        }
+        if (randomBoolean()) {
+            builder.postFilter(QueryBuilders.termQuery(randomAlphaOfLengthBetween(5, 20), randomAlphaOfLengthBetween(5, 20)));
+        }
+        if (randomBoolean()) {
+            int numSorts = randomIntBetween(1, 5);
+            for (int i = 0; i < numSorts; i++) {
+                int branch = randomInt(5);
+                switch (branch) {
+                    case 0:
+                        builder.sort(SortBuilders.fieldSort(randomAlphaOfLengthBetween(5, 20)).order(randomFrom(SortOrder.values())));
+                        break;
+                    case 1:
+                        builder.sort(SortBuilders.geoDistanceSort(randomAlphaOfLengthBetween(5, 20),
+                                AbstractQueryTestCase.randomGeohash(1, 12)).order(randomFrom(SortOrder.values())));
+                        break;
+                    case 2:
+                        builder.sort(SortBuilders.scoreSort().order(randomFrom(SortOrder.values())));
+                        break;
+                    case 3:
+                        builder.sort(SortBuilders
+                                .scriptSort(
+                                        new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, "foo", emptyMap()),
+                                        ScriptSortBuilder.ScriptSortType.NUMBER)
+                                .order(randomFrom(SortOrder.values())));
+                        break;
+                    case 4:
+                        builder.sort(randomAlphaOfLengthBetween(5, 20));
+                        break;
+                    case 5:
+                        builder.sort(randomAlphaOfLengthBetween(5, 20), randomFrom(SortOrder.values()));
+                        break;
+                }
+            }
+        }
+
+        if (randomBoolean()) {
+            int numSearchFrom = randomIntBetween(1, 5);
+            try {
+                // We build a json version of the search_from first in order to
+                // ensure that every number type remain the same before/after xcontent (de)serialization.
+                // This is not a problem because the final type of each field value is extracted from associated sort field.
+                // This little trick ensure that equals and hashcode are the same when using the xcontent serialization.
+                XContentBuilder jsonBuilder = XContentFactory.jsonBuilder();
+                jsonBuilder.startObject();
+                jsonBuilder.startArray("search_from");
+                for (int i = 0; i < numSearchFrom; i++) {
+                    int branch = randomInt(8);
+                    switch (branch) {
+                        case 0:
+                            jsonBuilder.value(randomInt());
+                            break;
+                        case 1:
+                            jsonBuilder.value(randomFloat());
+                            break;
+                        case 2:
+                            jsonBuilder.value(randomLong());
+                            break;
+                        case 3:
+                            jsonBuilder.value(randomDouble());
+                            break;
+                        case 4:
+                            jsonBuilder.value(randomAlphaOfLengthBetween(5, 20));
+                            break;
+                        case 5:
+                            jsonBuilder.value(randomBoolean());
+                            break;
+                        case 6:
+                            jsonBuilder.value(randomByte());
+                            break;
+                        case 7:
+                            jsonBuilder.value(randomShort());
+                            break;
+                        case 8:
+                            jsonBuilder.value(new Text(randomAlphaOfLengthBetween(5, 20)));
+                            break;
+                    }
+                }
+                jsonBuilder.endArray();
+                jsonBuilder.endObject();
+                XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(NamedXContentRegistry.EMPTY,
+                        jsonBuilder.bytes());
+                parser.nextToken();
+                parser.nextToken();
+                parser.nextToken();
+                builder.searchAfter(SearchAfterBuilder.fromXContent(parser).getSortValues());
+            } catch (IOException e) {
+                throw new RuntimeException("Error building search_from", e);
+            }
+        }
+        if (randomBoolean()) {
+            builder.highlighter(randomHighlightBuilder.get());
+        }
+        if (randomBoolean()) {
+            builder.suggest(randomSuggestBuilder.get());
+        }
+        if (randomBoolean()) {
+            int numRescores = randomIntBetween(1, 5);
+            for (int i = 0; i < numRescores; i++) {
+                builder.addRescorer(randomRescoreBuilder.get());
+            }
+        }
+        if (randomBoolean()) {
+            builder.aggregation(AggregationBuilders.avg(randomAlphaOfLengthBetween(5, 20)));
+        }
+        if (randomBoolean()) {
+            builder.ext(randomExtBuilders.get());
+        }
+        if (randomBoolean()) {
+            String field = randomBoolean() ? null : randomAlphaOfLengthBetween(5, 20);
+            int max = between(2, 1000);
+            int id = randomInt(max-1);
+            if (field == null) {
+                builder.slice(new SliceBuilder(id, max));
+            } else {
+                builder.slice(new SliceBuilder(field, id, max));
+            }
+        }
+        if (randomBoolean()) {
+            builder.collapse(randomCollapseBuilder.get());
+        }
+        return builder;
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
new file mode 100644
index 0000000000000..3f49837166bf4
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
@@ -0,0 +1,297 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations;
+
+import org.apache.lucene.index.CompositeReaderContext;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReaderContext;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.Collector;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.QueryCache;
+import org.apache.lucene.search.QueryCachingPolicy;
+import org.apache.lucene.search.Weight;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.lease.Releasable;
+import org.elasticsearch.common.lease.Releasables;
+import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.MockBigArrays;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
+import org.elasticsearch.index.cache.bitset.BitsetFilterCache.Listener;
+import org.elasticsearch.index.cache.query.DisabledQueryCache;
+import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.fielddata.IndexFieldDataCache;
+import org.elasticsearch.index.fielddata.IndexFieldDataService;
+import org.elasticsearch.index.mapper.ContentPath;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.Mapper.BuilderContext;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.ObjectMapper;
+import org.elasticsearch.index.mapper.ObjectMapper.Nested;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.query.support.NestedScope;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.indices.breaker.CircuitBreakerService;
+import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
+import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
+import org.elasticsearch.mock.orig.Mockito;
+import org.elasticsearch.search.fetch.FetchPhase;
+import org.elasticsearch.search.fetch.subphase.DocValueFieldsFetchSubPhase;
+import org.elasticsearch.search.fetch.subphase.FetchSourceSubPhase;
+import org.elasticsearch.search.internal.ContextIndexSearcher;
+import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.search.lookup.SearchLookup;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.IndexSettingsModule;
+import org.mockito.Matchers;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import static org.mockito.Matchers.anyObject;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Base class for testing {@link Aggregator} implementations.
+ * Provides helpers for constructing and searching an {@link Aggregator} implementation based on a provided
+ * {@link AggregationBuilder} instance.
+ */
+public abstract class AggregatorTestCase extends ESTestCase {
+    private static final String NESTEDFIELD_PREFIX = "nested_";
+    private List<Releasable> releasables = new ArrayList<>();
+
+    protected IndexSettings indexSettings() {
+        return new IndexSettings(
+            IndexMetaData.builder("_index").settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT))
+                .numberOfShards(1)
+                .numberOfReplicas(0)
+                .creationDate(System.currentTimeMillis())
+                .build(),
+            Settings.EMPTY
+        );
+    }
+
+    protected <B extends AggregationBuilder> AggregatorFactory<?> createAggregatorFactory(B aggregationBuilder,
+            IndexSearcher indexSearcher,
+            MappedFieldType... fieldTypes) throws IOException {
+
+        IndexSettings indexSettings = indexSettings();
+        Engine.Searcher searcher = new Engine.Searcher("aggregator_test", indexSearcher);
+        QueryCache queryCache = new DisabledQueryCache(indexSettings);
+        QueryCachingPolicy queryCachingPolicy = new QueryCachingPolicy() {
+            @Override
+            public void onUse(Query query) {
+            }
+
+            @Override
+            public boolean shouldCache(Query query) throws IOException {
+                // never cache a query
+                return false;
+            }
+        };
+        ContextIndexSearcher contextIndexSearcher = new ContextIndexSearcher(searcher, queryCache, queryCachingPolicy);
+
+        CircuitBreakerService circuitBreakerService = new NoneCircuitBreakerService();
+        SearchContext searchContext = mock(SearchContext.class);
+        when(searchContext.numberOfShards()).thenReturn(1);
+        when(searchContext.searcher()).thenReturn(contextIndexSearcher);
+        when(searchContext.bigArrays()).thenReturn(new MockBigArrays(Settings.EMPTY, circuitBreakerService));
+        when(searchContext.fetchPhase())
+            .thenReturn(new FetchPhase(Arrays.asList(new FetchSourceSubPhase(), new DocValueFieldsFetchSubPhase())));
+        when(searchContext.getObjectMapper(anyString())).thenAnswer(invocation -> {
+            String fieldName = (String) invocation.getArguments()[0];
+            if (fieldName.startsWith(NESTEDFIELD_PREFIX)) {
+                BuilderContext context = new BuilderContext(indexSettings.getSettings(), new ContentPath());
+                return new ObjectMapper.Builder<>(fieldName).nested(Nested.newNested(false, false)).build(context);
+            }
+            return null;
+        });
+        when(searchContext.bitsetFilterCache()).thenReturn(new BitsetFilterCache(indexSettings, mock(Listener.class)));
+        doAnswer(invocation -> {
+            /* Store the releasables so we can release them at the end of the test case. This is important because aggregations don't
+             * close their sub-aggregations. This is fairly similar to what the production code does. */
+            releasables.add((Releasable) invocation.getArguments()[0]);
+            return null;
+        }).when(searchContext).addReleasable(anyObject(), anyObject());
+
+        // TODO: now just needed for top_hits, this will need to be revised for other agg unit tests:
+        MapperService mapperService = mapperServiceMock();
+        when(mapperService.getIndexSettings()).thenReturn(indexSettings);
+        when(mapperService.hasNested()).thenReturn(false);
+        when(searchContext.mapperService()).thenReturn(mapperService);
+        IndexFieldDataService ifds = new IndexFieldDataService(IndexSettingsModule.newIndexSettings("test", Settings.EMPTY),
+                new IndicesFieldDataCache(Settings.EMPTY, new IndexFieldDataCache.Listener() {
+                }), circuitBreakerService, mapperService);
+        when(searchContext.fieldData()).thenReturn(ifds);
+
+        SearchLookup searchLookup = new SearchLookup(mapperService, ifds, new String[]{"type"});
+        when(searchContext.lookup()).thenReturn(searchLookup);
+
+        QueryShardContext queryShardContext = queryShardContextMock(mapperService, fieldTypes, circuitBreakerService);
+        when(searchContext.getQueryShardContext()).thenReturn(queryShardContext);
+        return aggregationBuilder.build(searchContext, null);
+    }
+
+    protected <A extends Aggregator, B extends AggregationBuilder> A createAggregator(B aggregationBuilder,
+            IndexSearcher indexSearcher,
+            MappedFieldType... fieldTypes) throws IOException {
+        AggregatorFactory<?> factory = createAggregatorFactory(aggregationBuilder, indexSearcher, fieldTypes);
+        @SuppressWarnings("unchecked")
+        A aggregator = (A) factory.create(null, true);
+        return aggregator;
+    }
+
+    /**
+     * sub-tests that need a more complex mock can overwrite this
+     */
+    protected MapperService mapperServiceMock() {
+        return mock(MapperService.class);
+    }
+
+    /**
+     * sub-tests that need a more complex mock can overwrite this
+     */
+    protected QueryShardContext queryShardContextMock(MapperService mapperService, MappedFieldType[] fieldTypes,
+            CircuitBreakerService circuitBreakerService) {
+        QueryShardContext queryShardContext = mock(QueryShardContext.class);
+        when(queryShardContext.getMapperService()).thenReturn(mapperService);
+        for (MappedFieldType fieldType : fieldTypes) {
+            when(queryShardContext.fieldMapper(fieldType.name())).thenReturn(fieldType);
+            when(queryShardContext.getForField(fieldType)).then(invocation -> fieldType.fielddataBuilder().build(
+                    mapperService.getIndexSettings(), fieldType,
+                    new IndexFieldDataCache.None(), circuitBreakerService, mock(MapperService.class)));
+        }
+        NestedScope nestedScope = new NestedScope();
+        when(queryShardContext.isFilter()).thenCallRealMethod();
+        Mockito.doCallRealMethod().when(queryShardContext).setIsFilter(Matchers.anyBoolean());
+        when(queryShardContext.nestedScope()).thenReturn(nestedScope);
+        return queryShardContext;
+    }
+
+    protected <A extends InternalAggregation, C extends Aggregator> A search(IndexSearcher searcher,
+                                                                             Query query,
+                                                                             AggregationBuilder builder,
+                                                                             MappedFieldType... fieldTypes) throws IOException {
+        C a = createAggregator(builder, searcher, fieldTypes);
+        try {
+            a.preCollection();
+            searcher.search(query, a);
+            a.postCollection();
+            @SuppressWarnings("unchecked")
+            A internalAgg = (A) a.buildAggregation(0L);
+            return internalAgg;
+        } finally {
+            Releasables.close(releasables);
+            releasables.clear();
+        }
+    }
+
+    /**
+     * Divides the provided {@link IndexSearcher} in sub-searcher, one for each segment,
+     * builds an aggregator for each sub-searcher filtered by the provided {@link Query} and
+     * returns the reduced {@link InternalAggregation}.
+     */
+    protected <A extends InternalAggregation, C extends Aggregator> A searchAndReduce(IndexSearcher searcher,
+                                                                                      Query query,
+                                                                                      AggregationBuilder builder,
+                                                                                      MappedFieldType... fieldTypes) throws IOException {
+        final IndexReaderContext ctx = searcher.getTopReaderContext();
+
+        final ShardSearcher[] subSearchers;
+        if (ctx instanceof LeafReaderContext) {
+            subSearchers = new ShardSearcher[1];
+            subSearchers[0] = new ShardSearcher((LeafReaderContext) ctx, ctx);
+        } else {
+            final CompositeReaderContext compCTX = (CompositeReaderContext) ctx;
+            final int size = compCTX.leaves().size();
+            subSearchers = new ShardSearcher[size];
+            for(int searcherIDX=0;searcherIDX<subSearchers.length;searcherIDX++) {
+                final LeafReaderContext leave = compCTX.leaves().get(searcherIDX);
+                subSearchers[searcherIDX] = new ShardSearcher(leave, compCTX);
+            }
+        }
+
+        List<InternalAggregation> aggs = new ArrayList<> ();
+        Query rewritten = searcher.rewrite(query);
+        Weight weight = searcher.createWeight(rewritten, true);
+        C root = createAggregator(builder, searcher, fieldTypes);
+        try {
+            for (ShardSearcher subSearcher : subSearchers) {
+                C a = createAggregator(builder, subSearcher, fieldTypes);
+                a.preCollection();
+                subSearcher.search(weight, a);
+                a.postCollection();
+                aggs.add(a.buildAggregation(0L));
+            }
+            if (aggs.isEmpty()) {
+                return null;
+            } else {
+                if (randomBoolean()) {
+                    // sometimes do an incremental reduce
+                    List<InternalAggregation> internalAggregations = randomSubsetOf(randomIntBetween(1, aggs.size()), aggs);
+                    A internalAgg = (A) aggs.get(0).doReduce(internalAggregations,
+                        new InternalAggregation.ReduceContext(root.context().bigArrays(), null, false));
+                    aggs.removeAll(internalAggregations);
+                    aggs.add(internalAgg);
+                }
+                // now do the final reduce
+                @SuppressWarnings("unchecked")
+                A internalAgg = (A) aggs.get(0).doReduce(aggs, new InternalAggregation.ReduceContext(root.context().bigArrays(), null,
+                    true));
+                return internalAgg;
+            }
+        } finally {
+            Releasables.close(releasables);
+            releasables.clear();
+        }
+    }
+
+    private static class ShardSearcher extends IndexSearcher {
+        private final List<LeafReaderContext> ctx;
+
+        ShardSearcher(LeafReaderContext ctx, IndexReaderContext parent) {
+            super(parent);
+            this.ctx = Collections.singletonList(ctx);
+        }
+
+        public void search(Weight weight, Collector collector) throws IOException {
+            search(ctx, weight, collector);
+        }
+
+        @Override
+        public String toString() {
+            return "ShardSearcher(" + ctx.get(0) + ")";
+        }
+    }
+
+    protected static DirectoryReader wrap(DirectoryReader directoryReader) throws IOException {
+        return ElasticsearchDirectoryReader.wrap(directoryReader, new ShardId(new Index("_index", "_na_"), 0));
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
new file mode 100644
index 0000000000000..e27199918f412
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
@@ -0,0 +1,206 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations;
+
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.indices.IndicesModule;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.PluginsService;
+import org.elasticsearch.plugins.SearchPlugin;
+import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
+import org.elasticsearch.test.AbstractQueryTestCase;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+import static java.util.Collections.emptyList;
+import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
+import static org.hamcrest.Matchers.hasSize;
+
+public abstract class BaseAggregationTestCase<AB extends AbstractAggregationBuilder<AB>> extends ESTestCase {
+
+    protected static final String STRING_FIELD_NAME = "mapped_string";
+    protected static final String INT_FIELD_NAME = "mapped_int";
+    protected static final String DOUBLE_FIELD_NAME = "mapped_double";
+    protected static final String BOOLEAN_FIELD_NAME = "mapped_boolean";
+    protected static final String DATE_FIELD_NAME = "mapped_date";
+    protected static final String IP_FIELD_NAME = "mapped_ip";
+
+    private String[] currentTypes;
+
+    protected String[] getCurrentTypes() {
+        return currentTypes;
+    }
+
+    private NamedWriteableRegistry namedWriteableRegistry;
+    private NamedXContentRegistry xContentRegistry;
+    protected abstract AB createTestAggregatorBuilder();
+
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return Collections.emptyList();
+    }
+
+    /**
+     * Setup for the whole base test class.
+     */
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        Settings settings = Settings.builder()
+            .put("node.name", AbstractQueryTestCase.class.toString())
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
+            .build();
+        IndicesModule indicesModule = new IndicesModule(Collections.emptyList());
+        PluginsService pluginsService = new PluginsService(settings, null, null, getPlugins());
+        SearchModule searchModule = new SearchModule(settings, false, pluginsService.filterPlugins(SearchPlugin.class));
+        List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
+        entries.addAll(indicesModule.getNamedWriteables());
+        entries.addAll(searchModule.getNamedWriteables());
+        namedWriteableRegistry = new NamedWriteableRegistry(entries);
+        xContentRegistry = new NamedXContentRegistry(searchModule.getNamedXContents());
+        //create some random type with some default field, those types will stick around for all of the subclasses
+        currentTypes = new String[randomIntBetween(0, 5)];
+        for (int i = 0; i < currentTypes.length; i++) {
+            String type = randomAlphaOfLengthBetween(1, 10);
+            currentTypes[i] = type;
+        }
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return xContentRegistry;
+    }
+
+    /**
+     * Generic test that creates new AggregatorFactory from the test
+     * AggregatorFactory and checks both for equality and asserts equality on
+     * the two queries.
+     */
+    public void testFromXContent() throws IOException {
+        AB testAgg = createTestAggregatorBuilder();
+        AggregatorFactories.Builder factoriesBuilder = AggregatorFactories.builder().addAggregator(testAgg);
+        XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
+        if (randomBoolean()) {
+            builder.prettyPrint();
+        }
+        factoriesBuilder.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        XContentBuilder shuffled = shuffleXContent(builder);
+        XContentParser parser = createParser(shuffled);
+        AggregationBuilder newAgg = parse(parser);
+        assertNotSame(newAgg, testAgg);
+        assertEquals(testAgg, newAgg);
+        assertEquals(testAgg.hashCode(), newAgg.hashCode());
+    }
+
+    protected AggregationBuilder parse(XContentParser parser) throws IOException {
+        QueryParseContext parseContext = new QueryParseContext(parser);
+        assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
+        AggregatorFactories.Builder parsed = AggregatorFactories.parseAggregators(parseContext);
+        assertThat(parsed.getAggregatorFactories(), hasSize(1));
+        assertThat(parsed.getPipelineAggregatorFactories(), hasSize(0));
+        AggregationBuilder newAgg = parsed.getAggregatorFactories().get(0);
+        assertNull(parser.nextToken());
+        assertNotNull(newAgg);
+        return newAgg;
+    }
+
+    /**
+     * Test serialization and deserialization of the test AggregatorFactory.
+     */
+    public void testSerialization() throws IOException {
+        AB testAgg = createTestAggregatorBuilder();
+        try (BytesStreamOutput output = new BytesStreamOutput()) {
+            output.writeNamedWriteable(testAgg);
+            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
+                AggregationBuilder deserialized = in.readNamedWriteable(AggregationBuilder.class);
+                assertEquals(testAgg, deserialized);
+                assertEquals(testAgg.hashCode(), deserialized.hashCode());
+                assertNotSame(testAgg, deserialized);
+            }
+        }
+    }
+
+    public void testEqualsAndHashcode() throws IOException {
+        // TODO we only change name and boost, we should extend by any sub-test supplying a "mutate" method that randomly changes one
+        // aspect of the object under test
+        checkEqualsAndHashCode(createTestAggregatorBuilder(), this::copyAggregation);
+    }
+
+    // we use the streaming infra to create a copy of the query provided as
+    // argument
+    private AB copyAggregation(AB agg) throws IOException {
+        try (BytesStreamOutput output = new BytesStreamOutput()) {
+            agg.writeTo(output);
+            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
+                @SuppressWarnings("unchecked")
+                AB secondAgg = (AB) namedWriteableRegistry.getReader(AggregationBuilder.class, agg.getWriteableName()).read(in);
+                return secondAgg;
+            }
+        }
+    }
+
+    public String randomNumericField() {
+        int randomInt = randomInt(3);
+        switch (randomInt) {
+            case 0:
+                return DATE_FIELD_NAME;
+            case 1:
+                return DOUBLE_FIELD_NAME;
+            case 2:
+            default:
+                return INT_FIELD_NAME;
+        }
+    }
+
+    protected void randomFieldOrScript(ValuesSourceAggregationBuilder<?, ?> factory, String field) {
+        int choice = randomInt(2);
+        switch (choice) {
+        case 0:
+            factory.field(field);
+            break;
+        case 1:
+            factory.field(field);
+            factory.script(mockScript("_value + 1"));
+            break;
+        case 2:
+            factory.script(mockScript("doc[" + field + "] + 1"));
+            break;
+        default:
+            throw new AssertionError("Unknow random operation [" + choice + "]");
+        }
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/InternalSingleBucketAggregationTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/InternalSingleBucketAggregationTestCase.java
new file mode 100644
index 0000000000000..27be78fea8741
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/InternalSingleBucketAggregationTestCase.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations;
+
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.rest.action.search.RestSearchAction;
+import org.elasticsearch.search.aggregations.bucket.InternalSingleBucketAggregation;
+import org.elasticsearch.search.aggregations.bucket.ParsedSingleBucketAggregation;
+import org.elasticsearch.search.aggregations.metrics.max.InternalMax;
+import org.elasticsearch.search.aggregations.metrics.min.InternalMin;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.test.InternalAggregationTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Supplier;
+
+import static java.util.Collections.emptyList;
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.singletonMap;
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
+
+public abstract class InternalSingleBucketAggregationTestCase<T extends InternalSingleBucketAggregation>
+        extends InternalAggregationTestCase<T> {
+
+    private boolean hasInternalMax;
+    private boolean hasInternalMin;
+
+    public Supplier<InternalAggregations> subAggregationsSupplier;
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        hasInternalMax = randomBoolean();
+        hasInternalMin = randomBoolean();
+        subAggregationsSupplier = () -> {
+            List<InternalAggregation> aggs = new ArrayList<>();
+            if (hasInternalMax) {
+                aggs.add(new InternalMax("max", randomDouble(), randomNumericDocValueFormat(), emptyList(), emptyMap()));
+            }
+            if (hasInternalMin) {
+                aggs.add(new InternalMin("min", randomDouble(), randomNumericDocValueFormat(), emptyList(), emptyMap()));
+            }
+            return new InternalAggregations(aggs);
+        };
+    }
+
+    protected abstract T createTestInstance(String name, long docCount, InternalAggregations aggregations,
+            List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData);
+
+    @Override
+    protected final T createTestInstance(String name, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
+        // we shouldn't use the full long range here since we sum doc count on reduce, and don't want to overflow the long range there
+        long docCount = between(0, Integer.MAX_VALUE);
+        return createTestInstance(name, docCount, subAggregationsSupplier.get(), pipelineAggregators, metaData);
+    }
+
+    @Override
+    protected void assertFromXContent(T aggregation, ParsedAggregation parsedAggregation) throws IOException {
+        assertTrue(parsedAggregation instanceof ParsedSingleBucketAggregation);
+        ParsedSingleBucketAggregation parsed = (ParsedSingleBucketAggregation) parsedAggregation;
+
+        assertEquals(aggregation.getDocCount(), parsed.getDocCount());
+        InternalAggregations aggregations = aggregation.getAggregations();
+        Map<String, Aggregation> expectedAggregations = new HashMap<>();
+        int expectedNumberOfAggregations = 0;
+        for (Aggregation expectedAggregation : aggregations) {
+            // since we shuffle xContent, we cannot rely on the order of the original inner aggregations for comparison
+            assertTrue(expectedAggregation instanceof InternalAggregation);
+            expectedAggregations.put(expectedAggregation.getName(), expectedAggregation);
+            expectedNumberOfAggregations++;
+        }
+        int parsedNumberOfAggregations = 0;
+        for (Aggregation parsedAgg : parsed.getAggregations()) {
+            assertTrue(parsedAgg instanceof ParsedAggregation);
+            assertTrue(expectedAggregations.keySet().contains(parsedAgg.getName()));
+            Aggregation expectedInternalAggregation = expectedAggregations.get(parsedAgg.getName());
+            final XContentType xContentType = randomFrom(XContentType.values());
+            final ToXContent.Params params = new ToXContent.MapParams(singletonMap(RestSearchAction.TYPED_KEYS_PARAM, "true"));
+            BytesReference expectedBytes = toXContent(expectedInternalAggregation, xContentType, params, false);
+            BytesReference actualBytes = toXContent(parsedAgg, xContentType, params, false);
+            assertToXContentEquivalent(expectedBytes, actualBytes, xContentType);
+            parsedNumberOfAggregations++;
+        }
+        assertEquals(expectedNumberOfAggregations, parsedNumberOfAggregations);
+        Class<? extends ParsedSingleBucketAggregation> parsedClass = implementationClass();
+        assertTrue(parsedClass != null && parsedClass.isInstance(parsedAggregation));
+    }
+
+    protected abstract Class<? extends ParsedSingleBucketAggregation> implementationClass();
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractNumericTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractNumericTestCase.java
index 703119a7a141d..164b9f4b4d4cf 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractNumericTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractNumericTestCase.java
@@ -42,7 +42,7 @@ public void setupSuiteScopeCluster() throws Exception {
 
         final int numDocs = 10;
         for (int i = 0; i < numDocs; i++) { // TODO randomize the size and the params in here?
-            builders.add(client().prepareIndex("idx", "type", ""+i).setSource(jsonBuilder()
+            builders.add(client().prepareIndex("idx", "type", String.valueOf(i)).setSource(jsonBuilder()
                     .startObject()
                     .field("value", i+1)
                     .startArray("values").value(i+2).value(i+3).endArray()
@@ -61,7 +61,7 @@ public void setupSuiteScopeCluster() throws Exception {
         prepareCreate("empty_bucket_idx").addMapping("type", "value", "type=integer").execute().actionGet();
         builders = new ArrayList<>();
         for (int i = 0; i < 2; i++) {
-            builders.add(client().prepareIndex("empty_bucket_idx", "type", ""+i).setSource(jsonBuilder()
+            builders.add(client().prepareIndex("empty_bucket_idx", "type", String.valueOf(i)).setSource(jsonBuilder()
                     .startObject()
                     .field("value", i*2)
                     .endObject()));
@@ -99,4 +99,4 @@ public void setupSuiteScopeCluster() throws Exception {
     public abstract void testScriptMultiValuedWithParams() throws Exception;
 
     public abstract void testOrderByEmptyAggregation() throws Exception;
-}
\ No newline at end of file
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
index 85045aa3f80bb..de9c090edff04 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.test;
 
 import com.fasterxml.jackson.core.io.JsonStringEncoder;
+
 import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
@@ -35,31 +36,24 @@
 import org.elasticsearch.action.termvectors.MultiTermVectorsRequest;
 import org.elasticsearch.action.termvectors.MultiTermVectorsResponse;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.compress.CompressedXContent;
-import org.elasticsearch.common.inject.Injector;
-import org.elasticsearch.common.inject.Module;
-import org.elasticsearch.common.inject.ModulesBuilder;
-import org.elasticsearch.common.inject.util.Providers;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.Writeable.Reader;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.common.unit.Fuzziness;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -67,13 +61,15 @@
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.analysis.AnalysisService;
+import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.fielddata.IndexFieldDataService;
+import org.elasticsearch.index.mapper.LatLonPointFieldMapper;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.query.AbstractQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
@@ -85,12 +81,10 @@
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.indices.IndicesModule;
 import org.elasticsearch.indices.analysis.AnalysisModule;
-import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
 import org.elasticsearch.indices.mapper.MapperRegistry;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
-import org.elasticsearch.node.internal.InternalSettingsPreparer;
+import org.elasticsearch.node.InternalSettingsPreparer;
 import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.PluginsService;
@@ -100,7 +94,6 @@
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.search.internal.SearchContext;
-import org.elasticsearch.threadpool.ThreadPool;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.junit.After;
@@ -124,37 +117,42 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
+import java.util.function.Function;
+import java.util.stream.Stream;
 
 import static java.util.Collections.emptyList;
+import static java.util.stream.Collectors.toList;
+import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
+import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.either;
-import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.not;
 
 public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>> extends ESTestCase {
 
     public static final String STRING_FIELD_NAME = "mapped_string";
     protected static final String STRING_FIELD_NAME_2 = "mapped_string_2";
     protected static final String INT_FIELD_NAME = "mapped_int";
+    protected static final String INT_RANGE_FIELD_NAME = "mapped_int_range";
     protected static final String DOUBLE_FIELD_NAME = "mapped_double";
     protected static final String BOOLEAN_FIELD_NAME = "mapped_boolean";
     protected static final String DATE_FIELD_NAME = "mapped_date";
+    protected static final String DATE_RANGE_FIELD_NAME = "mapped_date_range";
     protected static final String OBJECT_FIELD_NAME = "mapped_object";
     protected static final String GEO_POINT_FIELD_NAME = "mapped_geo_point";
-    protected static final String GEO_POINT_FIELD_MAPPING = "type=geo_point,lat_lon=true,geohash=true,geohash_prefix=true";
+    protected static final String LEGACY_GEO_POINT_FIELD_MAPPING = "type=geo_point,lat_lon=true,geohash=true,geohash_prefix=true";
     protected static final String GEO_SHAPE_FIELD_NAME = "mapped_geo_shape";
-    protected static final String[] MAPPED_FIELD_NAMES = new String[]{STRING_FIELD_NAME, INT_FIELD_NAME, DOUBLE_FIELD_NAME,
-            BOOLEAN_FIELD_NAME, DATE_FIELD_NAME, OBJECT_FIELD_NAME, GEO_POINT_FIELD_NAME, GEO_SHAPE_FIELD_NAME};
-    protected static final String[] MAPPED_LEAF_FIELD_NAMES = new String[]{STRING_FIELD_NAME, INT_FIELD_NAME, DOUBLE_FIELD_NAME,
-            BOOLEAN_FIELD_NAME, DATE_FIELD_NAME, GEO_POINT_FIELD_NAME};
+    protected static final String[] MAPPED_FIELD_NAMES = new String[]{STRING_FIELD_NAME, INT_FIELD_NAME, INT_RANGE_FIELD_NAME,
+            DOUBLE_FIELD_NAME, BOOLEAN_FIELD_NAME, DATE_FIELD_NAME, DATE_RANGE_FIELD_NAME, OBJECT_FIELD_NAME, GEO_POINT_FIELD_NAME,
+            GEO_SHAPE_FIELD_NAME};
+    protected static final String[] MAPPED_LEAF_FIELD_NAMES = new String[]{STRING_FIELD_NAME, INT_FIELD_NAME, INT_RANGE_FIELD_NAME,
+            DOUBLE_FIELD_NAME, BOOLEAN_FIELD_NAME, DATE_FIELD_NAME, DATE_RANGE_FIELD_NAME, GEO_POINT_FIELD_NAME, };
     private static final int NUMBER_OF_TESTQUERIES = 20;
 
     private static ServiceHolder serviceHolder;
     private static int queryNameId = 0;
     private static Settings nodeSettings;
-    private static Settings indexSettings;
     private static Index index;
     private static String[] currentTypes;
     private static String[] randomTypes;
@@ -164,7 +162,7 @@ protected static Index getIndex() {
     }
 
     protected static String[] getCurrentTypes() {
-        return currentTypes;
+        return currentTypes == null ? Strings.EMPTY_ARRAY : currentTypes;
     }
 
     protected Collection<Class<? extends Plugin>> getPlugins() {
@@ -176,30 +174,27 @@ protected void initializeAdditionalMappings(MapperService mapperService) throws
 
     @BeforeClass
     public static void beforeClass() {
-        // we have to prefer CURRENT since with the range of versions we support it's rather unlikely to get the current actually.
-        Version indexVersionCreated = randomBoolean() ? Version.CURRENT
-                : VersionUtils.randomVersionBetween(random(), Version.V_2_0_0_beta1, Version.CURRENT);
         nodeSettings = Settings.builder()
                 .put("node.name", AbstractQueryTestCase.class.toString())
                 .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
-                .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), false)
                 .build();
-        indexSettings = Settings.builder()
-                .put(ParseFieldMatcher.PARSE_STRICT, true)
-                .put(IndexMetaData.SETTING_VERSION_CREATED, indexVersionCreated).build();
 
-        index = new Index(randomAsciiOfLengthBetween(1, 10), "_na_");
+        index = new Index(randomAlphaOfLengthBetween(1, 10), "_na_");
 
-        //create some random type with some default field, those types will stick around for all of the subclasses
-        currentTypes = new String[randomIntBetween(0, 5)];
-        for (int i = 0; i < currentTypes.length; i++) {
-            String type = randomAsciiOfLengthBetween(1, 10);
-            currentTypes[i] = type;
-        }
-        //set some random types to be queried as part the search request, before each test
+        // Set a single type in the index
+        currentTypes = new String[] { "doc" };
         randomTypes = getRandomTypes();
     }
 
+    protected Settings indexSettings() {
+        // we have to prefer CURRENT since with the range of versions we support it's rather unlikely to get the current actually.
+        Version indexVersionCreated = randomBoolean() ? Version.CURRENT :
+            VersionUtils.randomVersionBetween(random(), Version.V_5_0_0, Version.CURRENT);
+        return Settings.builder()
+            .put(IndexMetaData.SETTING_VERSION_CREATED, indexVersionCreated)
+            .build();
+    }
+
     @AfterClass
     public static void afterClass() throws Exception {
         IOUtils.close(serviceHolder);
@@ -209,12 +204,12 @@ public static void afterClass() throws Exception {
     @Before
     public void beforeTest() throws IOException {
         if (serviceHolder == null) {
-            serviceHolder = new ServiceHolder(nodeSettings, indexSettings, getPlugins(), this);
+            serviceHolder = new ServiceHolder(nodeSettings, indexSettings(), getPlugins(), this);
         }
         serviceHolder.clientInvocationHandler.delegate = this;
     }
 
-    private static void setSearchContext(String[] types, QueryShardContext context) {
+    private static SearchContext getSearchContext(String[] types, QueryShardContext context) {
         TestSearchContext testSearchContext = new TestSearchContext(context) {
             @Override
             public MapperService mapperService() {
@@ -227,13 +222,12 @@ public IndexFieldDataService fieldData() {
             }
         };
         testSearchContext.getQueryShardContext().setTypes(types);
-        SearchContext.setCurrent(testSearchContext);
+        return testSearchContext;
     }
 
     @After
     public void afterTest() {
         serviceHolder.clientInvocationHandler.delegate = null;
-        SearchContext.removeCurrent();
     }
 
     public final QB createTestQueryBuilder() {
@@ -254,7 +248,7 @@ public final QB createTestQueryBuilder() {
      * make sure query names are unique by suffixing them with increasing counter
      */
     private static String createUniqueRandomName() {
-        String queryName = randomAsciiOfLengthBetween(1, 10) + queryNameId;
+        String queryName = randomAlphaOfLengthBetween(1, 10) + queryNameId;
         queryNameId++;
         return queryName;
     }
@@ -271,12 +265,13 @@ private static String createUniqueRandomName() {
     public void testFromXContent() throws IOException {
         for (int runs = 0; runs < NUMBER_OF_TESTQUERIES; runs++) {
             QB testQuery = createTestQueryBuilder();
-            XContentBuilder builder = toXContent(testQuery, randomFrom(XContentType.values()));
-            XContentBuilder shuffled = shuffleXContent(builder, shuffleProtectedFields());
-            assertParsedQuery(shuffled.bytes(), testQuery);
+            XContentType xContentType = randomFrom(XContentType.values());
+            BytesReference shuffledXContent = toShuffledXContent(testQuery, xContentType, ToXContent.EMPTY_PARAMS, randomBoolean(),
+                    shuffleProtectedFields());
+            assertParsedQuery(createParser(xContentType.xContent(), shuffledXContent), testQuery);
             for (Map.Entry<String, QB> alternateVersion : getAlternateVersions().entrySet()) {
                 String queryAsString = alternateVersion.getKey();
-                assertParsedQuery(new BytesArray(queryAsString), alternateVersion.getValue());
+                assertParsedQuery(createParser(JsonXContent.jsonXContent, queryAsString), alternateVersion.getValue());
             }
         }
     }
@@ -289,15 +284,6 @@ protected String[] shuffleProtectedFields() {
         return Strings.EMPTY_ARRAY;
     }
 
-    protected static XContentBuilder toXContent(QueryBuilder query, XContentType contentType) throws IOException {
-        XContentBuilder builder = XContentFactory.contentBuilder(contentType);
-        if (randomBoolean()) {
-            builder.prettyPrint();
-        }
-        query.toXContent(builder, ToXContent.EMPTY_PARAMS);
-        return builder;
-    }
-
     /**
      * Test that unknown field trigger ParsingException.
      * To find the right position in the root query, we add a marker as `queryName` which
@@ -406,7 +392,7 @@ static List<Tuple<String, Boolean>> alterateQueries(Set<String> queries, Set<Str
                 BytesStreamOutput out = new BytesStreamOutput();
                 try (
                         XContentGenerator generator = XContentType.JSON.xContent().createGenerator(out);
-                        XContentParser parser = XContentHelper.createParser(new BytesArray(query));
+                        XContentParser parser = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, query);
                 ) {
                     int objectIndex = -1;
                     Deque<String> levels = new LinkedList<>();
@@ -477,7 +463,7 @@ protected Set<String> getObjectsHoldingArbitraryContent() {
      * Test that wraps the randomly generated query into an array as follows: { "query_name" : [{}]}
      * This causes unexpected situations in parser code that may not be handled properly.
      */
-    public final void testQueryWrappedInArray() throws IOException {
+    public final void testQueryWrappedInArray() {
         QB queryBuilder = createTestQueryBuilder();
         String queryName = queryBuilder.getName();
         String validQuery = queryBuilder.toString();
@@ -487,7 +473,7 @@ public final void testQueryWrappedInArray() throws IOException {
         }
     }
 
-    private void queryWrappedInArrayTest(String queryName, String validQuery) throws IOException {
+    private void queryWrappedInArrayTest(String queryName, String validQuery) {
         int i = validQuery.indexOf("\"" + queryName + "\"");
         assertThat(i, greaterThan(0));
 
@@ -525,13 +511,8 @@ protected Map<String, QB> getAlternateVersions() {
     /**
      * Parses the query provided as string argument and compares it with the expected result provided as argument as a {@link QueryBuilder}
      */
-    protected static void assertParsedQuery(String queryAsString, QueryBuilder expectedQuery) throws IOException {
-        assertParsedQuery(queryAsString, expectedQuery, ParseFieldMatcher.STRICT);
-    }
-
-    protected static void assertParsedQuery(String queryAsString, QueryBuilder expectedQuery, ParseFieldMatcher matcher)
-            throws IOException {
-        QueryBuilder newQuery = parseQuery(queryAsString, matcher);
+    protected void assertParsedQuery(String queryAsString, QueryBuilder expectedQuery) throws IOException {
+        QueryBuilder newQuery = parseQuery(queryAsString);
         assertNotSame(newQuery, expectedQuery);
         assertEquals(expectedQuery, newQuery);
         assertEquals(expectedQuery.hashCode(), newQuery.hashCode());
@@ -540,44 +521,37 @@ protected static void assertParsedQuery(String queryAsString, QueryBuilder expec
     /**
      * Parses the query provided as bytes argument and compares it with the expected result provided as argument as a {@link QueryBuilder}
      */
-    private static void assertParsedQuery(BytesReference queryAsBytes, QueryBuilder expectedQuery) throws IOException {
-        assertParsedQuery(queryAsBytes, expectedQuery, ParseFieldMatcher.STRICT);
-    }
-
-    private static void assertParsedQuery(BytesReference queryAsBytes, QueryBuilder expectedQuery, ParseFieldMatcher matcher)
-            throws IOException {
-        QueryBuilder newQuery = parseQuery(queryAsBytes, matcher);
+    private static void assertParsedQuery(XContentParser parser, QueryBuilder expectedQuery) throws IOException {
+        QueryBuilder newQuery = parseQuery(parser);
         assertNotSame(newQuery, expectedQuery);
         assertEquals(expectedQuery, newQuery);
         assertEquals(expectedQuery.hashCode(), newQuery.hashCode());
     }
 
-    protected static QueryBuilder parseQuery(String queryAsString) throws IOException {
-        return parseQuery(queryAsString, ParseFieldMatcher.STRICT);
-    }
-
-    protected static QueryBuilder parseQuery(String queryAsString, ParseFieldMatcher matcher) throws IOException {
-        XContentParser parser = XContentFactory.xContent(queryAsString).createParser(queryAsString);
-        return parseQuery(parser, matcher);
+    protected QueryBuilder parseQuery(AbstractQueryBuilder<?> builder) throws IOException {
+        return parseQuery(createParser(JsonXContent.jsonXContent, builder.buildAsBytes(XContentType.JSON)));
     }
 
-    protected static QueryBuilder parseQuery(BytesReference queryAsBytes) throws IOException {
-        return parseQuery(queryAsBytes, ParseFieldMatcher.STRICT);
+    protected QueryBuilder parseQuery(String queryAsString) throws IOException {
+        XContentParser parser = createParser(JsonXContent.jsonXContent, queryAsString);
+        return parseQuery(parser);
     }
 
-    protected static QueryBuilder parseQuery(BytesReference queryAsBytes, ParseFieldMatcher matcher) throws IOException {
-        XContentParser parser = XContentFactory.xContent(queryAsBytes).createParser(queryAsBytes);
-        return parseQuery(parser, matcher);
-    }
-
-    private static QueryBuilder parseQuery(XContentParser parser, ParseFieldMatcher matcher) throws IOException {
-        QueryParseContext context = createParseContext(parser, matcher);
+    protected static QueryBuilder parseQuery(XContentParser parser) throws IOException {
+        QueryParseContext context = createParseContext(parser);
         QueryBuilder parseInnerQueryBuilder = context.parseInnerQueryBuilder()
                 .orElseThrow(() -> new IllegalArgumentException("inner query body cannot be empty"));
         assertNull(parser.nextToken());
         return parseInnerQueryBuilder;
     }
 
+    /**
+     * Whether the queries produced by this builder are expected to be cacheable.
+     */
+    protected boolean builderGeneratesCacheableQueries() {
+        return true;
+    }
+
     /**
      * Test creates the {@link Query} from the {@link QueryBuilder} under test and delegates the
      * assertions being made on the result to the implementing subclass.
@@ -585,15 +559,26 @@ private static QueryBuilder parseQuery(XContentParser parser, ParseFieldMatcher
     public void testToQuery() throws IOException {
         for (int runs = 0; runs < NUMBER_OF_TESTQUERIES; runs++) {
             QueryShardContext context = createShardContext();
+            assert context.isCachable();
             context.setAllowUnmappedFields(true);
             QB firstQuery = createTestQueryBuilder();
             QB controlQuery = copyQuery(firstQuery);
-            setSearchContext(randomTypes, context); // only set search context for toQuery to be more realistic
-            Query firstLuceneQuery = rewriteQuery(firstQuery, context).toQuery(context);
+            SearchContext searchContext = getSearchContext(randomTypes, context);// only set search context for toQuery to be more realistic
+/* we use a private rewrite context here since we want the most realistic way of asserting that we are cachabel or not.
+             * We do it this way in SearchService where
+             * we first rewrite the query with a private context, then reset the context and then build the actual lucene query*/
+            QueryBuilder rewritten = rewriteQuery(firstQuery, new QueryShardContext(context));
+            Query firstLuceneQuery = rewritten.toQuery(context);
+            if (isCachable(firstQuery)) {
+                assertTrue("query was marked as not cacheable in the context but this test indicates it should be cacheable: "
+                        + firstQuery.toString(), context.isCachable());
+            } else {
+                assertFalse("query was marked as cacheable in the context but this test indicates it should not be cacheable: "
+                        + firstQuery.toString(), context.isCachable());
+            }
             assertNotNull("toQuery should not return null", firstLuceneQuery);
-            assertLuceneQuery(firstQuery, firstLuceneQuery, context);
+            assertLuceneQuery(firstQuery, firstLuceneQuery, searchContext);
             //remove after assertLuceneQuery since the assertLuceneQuery impl might access the context as well
-            SearchContext.removeCurrent();
             assertTrue(
                     "query is not equal to its copy after calling toQuery, firstQuery: " + firstQuery + ", secondQuery: " + controlQuery,
                     firstQuery.equals(controlQuery));
@@ -605,23 +590,22 @@ public void testToQuery() throws IOException {
             QB secondQuery = copyQuery(firstQuery);
             // query _name never should affect the result of toQuery, we randomly set it to make sure
             if (randomBoolean()) {
-                secondQuery.queryName(secondQuery.queryName() == null ? randomAsciiOfLengthBetween(1, 30) : secondQuery.queryName()
-                        + randomAsciiOfLengthBetween(1, 10));
+                secondQuery.queryName(secondQuery.queryName() == null ? randomAlphaOfLengthBetween(1, 30) : secondQuery.queryName()
+                        + randomAlphaOfLengthBetween(1, 10));
             }
-            setSearchContext(randomTypes, context);
+            searchContext = getSearchContext(randomTypes, context);
             Query secondLuceneQuery = rewriteQuery(secondQuery, context).toQuery(context);
             assertNotNull("toQuery should not return null", secondLuceneQuery);
-            assertLuceneQuery(secondQuery, secondLuceneQuery, context);
-            SearchContext.removeCurrent();
+            assertLuceneQuery(secondQuery, secondLuceneQuery, searchContext);
 
-            assertEquals("two equivalent query builders lead to different lucene queries",
-                    rewrite(secondLuceneQuery), rewrite(firstLuceneQuery));
+            if (builderGeneratesCacheableQueries()) {
+                assertEquals("two equivalent query builders lead to different lucene queries",
+                        rewrite(secondLuceneQuery), rewrite(firstLuceneQuery));
+            }
 
             if (supportsBoostAndQueryName()) {
                 secondQuery.boost(firstQuery.boost() + 1f + randomFloat());
-                setSearchContext(randomTypes, context);
                 Query thirdLuceneQuery = rewriteQuery(secondQuery, context).toQuery(context);
-                SearchContext.removeCurrent();
                 assertNotEquals("modifying the boost doesn't affect the corresponding lucene query", rewrite(firstLuceneQuery),
                         rewrite(thirdLuceneQuery));
             }
@@ -641,6 +625,10 @@ private QueryBuilder rewriteQuery(QB queryBuilder, QueryRewriteContext rewriteCo
         return rewritten;
     }
 
+    protected boolean isCachable(QB queryBuilder) {
+        return true;
+    }
+
     /**
      * Few queries allow you to set the boost and queryName on the java api, although the corresponding parser
      * doesn't parse them as they are not supported. This method allows to disable boost and queryName related tests for those queries.
@@ -654,11 +642,11 @@ protected boolean supportsBoostAndQueryName() {
     /**
      * Checks the result of {@link QueryBuilder#toQuery(QueryShardContext)} given the original {@link QueryBuilder}
      * and {@link QueryShardContext}. Verifies that named queries and boost are properly handled and delegates to
-     * {@link #doAssertLuceneQuery(AbstractQueryBuilder, Query, QueryShardContext)} for query specific checks.
+     * {@link #doAssertLuceneQuery(AbstractQueryBuilder, Query, SearchContext)} for query specific checks.
      */
-    private void assertLuceneQuery(QB queryBuilder, Query query, QueryShardContext context) throws IOException {
+    private void assertLuceneQuery(QB queryBuilder, Query query, SearchContext context) throws IOException {
         if (queryBuilder.queryName() != null) {
-            Query namedQuery = context.copyNamedQueries().get(queryBuilder.queryName());
+            Query namedQuery = context.getQueryShardContext().copyNamedQueries().get(queryBuilder.queryName());
             assertThat(namedQuery, equalTo(query));
         }
         if (query != null) {
@@ -682,7 +670,7 @@ private void assertLuceneQuery(QB queryBuilder, Query query, QueryShardContext c
      * Checks the result of {@link QueryBuilder#toQuery(QueryShardContext)} given the original {@link QueryBuilder}
      * and {@link QueryShardContext}. Contains the query specific checks to be implemented by subclasses.
      */
-    protected abstract void doAssertLuceneQuery(QB queryBuilder, Query query, QueryShardContext context) throws IOException;
+    protected abstract void doAssertLuceneQuery(QB queryBuilder, Query query, SearchContext context) throws IOException;
 
     protected static void assertTermOrBoostQuery(Query query, String field, String value, float fieldBoost) {
         if (fieldBoost != AbstractQueryBuilder.DEFAULT_BOOST) {
@@ -711,13 +699,19 @@ public void testSerialization() throws IOException {
         }
     }
 
+    protected static QueryBuilder assertSerialization(QueryBuilder testQuery) throws IOException {
+        return assertSerialization(testQuery, Version.CURRENT);
+    }
+
     /**
      * Serialize the given query builder and asserts that both are equal
      */
-    protected static QueryBuilder assertSerialization(QueryBuilder testQuery) throws IOException {
+    protected static QueryBuilder assertSerialization(QueryBuilder testQuery, Version version) throws IOException {
         try (BytesStreamOutput output = new BytesStreamOutput()) {
+            output.setVersion(version);
             output.writeNamedWriteable(testQuery);
             try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), serviceHolder.namedWriteableRegistry)) {
+                in.setVersion(version);
                 QueryBuilder deserializedQuery = in.readNamedWriteable(QueryBuilder.class);
                 assertEquals(testQuery, deserializedQuery);
                 assertEquals(testQuery.hashCode(), deserializedQuery.hashCode());
@@ -727,49 +721,30 @@ protected static QueryBuilder assertSerialization(QueryBuilder testQuery) throws
         }
     }
 
-    public void testEqualsAndHashcode() throws IOException {
+    public void testEqualsAndHashcode() {
         for (int runs = 0; runs < NUMBER_OF_TESTQUERIES; runs++) {
-            QB firstQuery = createTestQueryBuilder();
-            assertFalse("query is equal to null", firstQuery.equals(null));
-            assertFalse("query is equal to incompatible type", firstQuery.equals(""));
-            assertTrue("query is not equal to self", firstQuery.equals(firstQuery));
-            assertThat("same query's hashcode returns different values if called multiple times", firstQuery.hashCode(),
-                    equalTo(firstQuery.hashCode()));
-
-            QB secondQuery = copyQuery(firstQuery);
-            assertTrue("query is not equal to self", secondQuery.equals(secondQuery));
-            assertTrue("query is not equal to its copy", firstQuery.equals(secondQuery));
-            assertTrue("equals is not symmetric", secondQuery.equals(firstQuery));
-            assertThat("query copy's hashcode is different from original hashcode", secondQuery.hashCode(), equalTo(firstQuery.hashCode()));
-
-            QB thirdQuery = copyQuery(secondQuery);
-            assertTrue("query is not equal to self", thirdQuery.equals(thirdQuery));
-            assertTrue("query is not equal to its copy", secondQuery.equals(thirdQuery));
-            assertThat("query copy's hashcode is different from original hashcode", secondQuery.hashCode(), equalTo(thirdQuery.hashCode()));
-            assertTrue("equals is not transitive", firstQuery.equals(thirdQuery));
-            assertThat("query copy's hashcode is different from original hashcode", firstQuery.hashCode(), equalTo(thirdQuery.hashCode()));
-            assertTrue("equals is not symmetric", thirdQuery.equals(secondQuery));
-            assertTrue("equals is not symmetric", thirdQuery.equals(firstQuery));
+            // TODO we only change name and boost, we should extend by any sub-test supplying a "mutate" method that randomly changes one
+            // aspect of the object under test
+            checkEqualsAndHashCode(createTestQueryBuilder(), this::copyQuery, this::changeNameOrBoost);
+        }
+    }
 
-            if (randomBoolean()) {
-                secondQuery.queryName(secondQuery.queryName() == null ? randomAsciiOfLengthBetween(1, 30) : secondQuery.queryName()
-                        + randomAsciiOfLengthBetween(1, 10));
-            } else {
-                secondQuery.boost(firstQuery.boost() + 1f + randomFloat());
-            }
-            assertThat("different queries should not be equal", secondQuery, not(equalTo(firstQuery)));
+    private QB changeNameOrBoost(QB original) throws IOException {
+        QB secondQuery = copyQuery(original);
+        if (randomBoolean()) {
+            secondQuery.queryName(secondQuery.queryName() == null ? randomAlphaOfLengthBetween(1, 30) : secondQuery.queryName()
+                    + randomAlphaOfLengthBetween(1, 10));
+        } else {
+            secondQuery.boost(original.boost() + 1f + randomFloat());
         }
+        return secondQuery;
     }
 
     //we use the streaming infra to create a copy of the query provided as argument
     @SuppressWarnings("unchecked")
     private QB copyQuery(QB query) throws IOException {
-        try (BytesStreamOutput output = new BytesStreamOutput()) {
-            output.writeNamedWriteable(query);
-            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), serviceHolder.namedWriteableRegistry)) {
-                return (QB) in.readNamedWriteable(QueryBuilder.class);
-            }
-        }
+        Reader<QB> reader = (Reader<QB>) serviceHolder.namedWriteableRegistry.getReader(QueryBuilder.class, query.getWriteableName());
+        return copyWriteable(query, serviceHolder.namedWriteableRegistry, reader);
     }
 
     /**
@@ -779,11 +754,8 @@ protected static QueryShardContext createShardContext() {
         return serviceHolder.createShardContext();
     }
 
-    /**
-     * @return a new {@link QueryParseContext} based on the base test index and queryParserService
-     */
-    protected static QueryParseContext createParseContext(XContentParser parser, ParseFieldMatcher matcher) {
-        return new QueryParseContext(serviceHolder.indicesQueriesRegistry, parser, matcher);
+    protected static QueryParseContext createParseContext(XContentParser parser) {
+        return new QueryParseContext(parser);
     }
 
     /**
@@ -800,14 +772,14 @@ protected static Object getRandomValueForFieldName(String fieldName) {
                     JsonStringEncoder encoder = JsonStringEncoder.getInstance();
                     value = new String(encoder.quoteAsString(randomUnicodeOfLength(10)));
                 } else {
-                    value = randomAsciiOfLengthBetween(1, 10);
+                    value = randomAlphaOfLengthBetween(1, 10);
                 }
                 break;
             case INT_FIELD_NAME:
                 value = randomIntBetween(0, 10);
                 break;
             case DOUBLE_FIELD_NAME:
-                value = randomDouble() * 10;
+                value = 1 + randomDouble() * 9;
                 break;
             case BOOLEAN_FIELD_NAME:
                 value = randomBoolean();
@@ -816,7 +788,7 @@ protected static Object getRandomValueForFieldName(String fieldName) {
                 value = new DateTime(System.currentTimeMillis(), DateTimeZone.UTC).toString();
                 break;
             default:
-                value = randomAsciiOfLengthBetween(1, 10);
+                value = randomAlphaOfLengthBetween(1, 10);
         }
         return value;
     }
@@ -825,7 +797,7 @@ protected static String getRandomQueryText() {
         int terms = randomIntBetween(0, 3);
         StringBuilder builder = new StringBuilder();
         for (int i = 0; i < terms; i++) {
-            builder.append(randomAsciiOfLengthBetween(1, 10)).append(" ");
+            builder.append(randomAlphaOfLengthBetween(1, 10)).append(" ");
         }
         return builder.toString().trim();
     }
@@ -836,7 +808,7 @@ protected static String getRandomQueryText() {
     protected static String getRandomFieldName() {
         // if no type is set then return a random field name
         if (currentTypes.length == 0 || randomBoolean()) {
-            return randomAsciiOfLengthBetween(1, 10);
+            return randomAlphaOfLengthBetween(1, 10);
         }
         return randomFrom(MAPPED_LEAF_FIELD_NAMES);
     }
@@ -876,10 +848,6 @@ private static String[] getRandomTypes() {
         return types;
     }
 
-    protected static String getRandomType() {
-        return (currentTypes.length == 0) ? MetaData.ALL : randomFrom(currentTypes);
-    }
-
     protected static Fuzziness randomFuzziness(String fieldName) {
         switch (fieldName) {
             case INT_FIELD_NAME:
@@ -1004,7 +972,6 @@ public void testMustRewrite() throws IOException {
         QueryShardContext context = createShardContext();
         context.setAllowUnmappedFields(true);
         QB queryBuilder = createTestQueryBuilder();
-        setSearchContext(randomTypes, context); // only set search context for toQuery to be more realistic
         queryBuilder.toQuery(context);
     }
 
@@ -1012,31 +979,31 @@ protected Query rewrite(Query query) throws IOException {
         return query;
     }
 
-    private static class ServiceHolder implements Closeable {
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return serviceHolder.xContentRegistry;
+    }
 
-        private final Injector injector;
-        private final IndicesQueriesRegistry indicesQueriesRegistry;
+    private static class ServiceHolder implements Closeable {
         private final IndexFieldDataService indexFieldDataService;
         private final SearchModule searchModule;
         private final NamedWriteableRegistry namedWriteableRegistry;
+        private final NamedXContentRegistry xContentRegistry;
         private final ClientInvocationHandler clientInvocationHandler = new ClientInvocationHandler();
         private final IndexSettings idxSettings;
         private final SimilarityService similarityService;
         private final MapperService mapperService;
         private final BitsetFilterCache bitsetFilterCache;
         private final ScriptService scriptService;
+        private final Client client;
+        private final long nowInMillis = randomNonNegativeLong();
 
         ServiceHolder(Settings nodeSettings, Settings indexSettings,
                       Collection<Class<? extends Plugin>> plugins, AbstractQueryTestCase<?> testCase) throws IOException {
-            final ThreadPool threadPool = new ThreadPool(nodeSettings);
-            ClusterService clusterService = ClusterServiceUtils.createClusterService(threadPool);
-            ClusterServiceUtils.setState(clusterService, new ClusterState.Builder(clusterService.state()).metaData(
-                            new MetaData.Builder().put(new IndexMetaData.Builder(
-                                    index.getName()).settings(indexSettings).numberOfShards(1).numberOfReplicas(0))));
             Environment env = InternalSettingsPreparer.prepareEnvironment(nodeSettings, null);
             PluginsService pluginsService = new PluginsService(nodeSettings, env.modulesFile(), env.pluginsFile(), plugins);
 
-            final Client proxy = (Client) Proxy.newProxyInstance(
+            client = (Client) Proxy.newProxyInstance(
                     Client.class.getClassLoader(),
                     new Class[]{Client.class},
                     clientInvocationHandler);
@@ -1045,53 +1012,28 @@ private static class ServiceHolder implements Closeable {
             scriptSettings.addAll(pluginsService.getPluginSettings());
             scriptSettings.add(InternalSettingsPlugin.VERSION_CREATED);
             SettingsModule settingsModule = new SettingsModule(nodeSettings, scriptSettings, pluginsService.getPluginSettingsFilter());
-            searchModule = new SearchModule(nodeSettings, false, pluginsService.filterPlugins(SearchPlugin.class)) {
-                @Override
-                protected void configureSearch() {
-                    // Skip me
-                }
-            };
-            IndicesModule indicesModule = new IndicesModule(pluginsService.filterPlugins(MapperPlugin.class)) {
-                @Override
-                public void configure() {
-                    // skip services
-                    bindMapperExtension();
-                }
-            };
+            searchModule = new SearchModule(nodeSettings, false, pluginsService.filterPlugins(SearchPlugin.class));
+            IndicesModule indicesModule = new IndicesModule(pluginsService.filterPlugins(MapperPlugin.class));
             List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
             entries.addAll(indicesModule.getNamedWriteables());
             entries.addAll(searchModule.getNamedWriteables());
-            NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(entries);
-            ModulesBuilder modulesBuilder = new ModulesBuilder();
-            for (Module pluginModule : pluginsService.createGuiceModules()) {
-                modulesBuilder.add(pluginModule);
-            }
-            modulesBuilder.add(
-                    b -> {
-                        b.bind(PluginsService.class).toInstance(pluginsService);
-                        b.bind(Environment.class).toInstance(new Environment(nodeSettings));
-                        b.bind(ThreadPool.class).toInstance(threadPool);
-                        b.bind(Client.class).toInstance(proxy);
-                        b.bind(ClusterService.class).toProvider(Providers.of(clusterService));
-                        b.bind(CircuitBreakerService.class).to(NoneCircuitBreakerService.class);
-                        b.bind(NamedWriteableRegistry.class).toInstance(namedWriteableRegistry);
-                    },
-                    settingsModule, indicesModule, searchModule, new IndexSettingsModule(index, indexSettings)
-            );
-            pluginsService.processModules(modulesBuilder);
-            injector = modulesBuilder.createInjector();
-            IndexScopedSettings indexScopedSettings = injector.getInstance(IndexScopedSettings.class);
+            namedWriteableRegistry = new NamedWriteableRegistry(entries);
+            xContentRegistry = new NamedXContentRegistry(Stream.of(
+                    searchModule.getNamedXContents().stream()
+                    ).flatMap(Function.identity()).collect(toList()));
+            IndexScopedSettings indexScopedSettings = settingsModule.getIndexScopedSettings();
             idxSettings = IndexSettingsModule.newIndexSettings(index, indexSettings, indexScopedSettings);
             AnalysisModule analysisModule = new AnalysisModule(new Environment(nodeSettings), emptyList());
-            AnalysisService analysisService = analysisModule.getAnalysisRegistry().build(idxSettings);
+            IndexAnalyzers indexAnalyzers = analysisModule.getAnalysisRegistry().build(idxSettings);
             scriptService = scriptModule.getScriptService();
             similarityService = new SimilarityService(idxSettings, Collections.emptyMap());
-            MapperRegistry mapperRegistry = injector.getInstance(MapperRegistry.class);
-            mapperService = new MapperService(idxSettings, analysisService, similarityService, mapperRegistry, this::createShardContext);
+            MapperRegistry mapperRegistry = indicesModule.getMapperRegistry();
+            mapperService = new MapperService(idxSettings, indexAnalyzers, xContentRegistry, similarityService, mapperRegistry,
+                    this::createShardContext);
             IndicesFieldDataCache indicesFieldDataCache = new IndicesFieldDataCache(nodeSettings, new IndexFieldDataCache.Listener() {
             });
             indexFieldDataService = new IndexFieldDataService(idxSettings, indicesFieldDataCache,
-                    injector.getInstance(CircuitBreakerService.class), mapperService);
+                    new NoneCircuitBreakerService(), mapperService);
             bitsetFilterCache = new BitsetFilterCache(idxSettings, new BitsetFilterCache.Listener() {
                 @Override
                 public void onCache(ShardId shardId, Accountable accountable) {
@@ -1103,18 +1045,22 @@ public void onRemoval(ShardId shardId, Accountable accountable) {
 
                 }
             });
-            indicesQueriesRegistry = injector.getInstance(IndicesQueriesRegistry.class);
+
+            String geoFieldMapping = (idxSettings.getIndexVersionCreated().before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) ?
+                LEGACY_GEO_POINT_FIELD_MAPPING : "type=geo_point";
 
             for (String type : currentTypes) {
                 mapperService.merge(type, new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef(type,
                         STRING_FIELD_NAME, "type=text",
                         STRING_FIELD_NAME_2, "type=keyword",
                         INT_FIELD_NAME, "type=integer",
+                        INT_RANGE_FIELD_NAME, "type=integer_range",
                         DOUBLE_FIELD_NAME, "type=double",
                         BOOLEAN_FIELD_NAME, "type=boolean",
                         DATE_FIELD_NAME, "type=date",
+                        DATE_RANGE_FIELD_NAME, "type=date_range",
                         OBJECT_FIELD_NAME, "type=object",
-                        GEO_POINT_FIELD_NAME, GEO_POINT_FIELD_MAPPING,
+                        GEO_POINT_FIELD_NAME, geoFieldMapping,
                         GEO_SHAPE_FIELD_NAME, "type=geo_shape"
                 ).string()), MapperService.MergeReason.MAPPING_UPDATE, false);
                 // also add mappings for two inner field in the object field
@@ -1122,26 +1068,22 @@ public void onRemoval(ShardId shardId, Accountable accountable) {
                                 + "\"properties\":{\"" + DATE_FIELD_NAME + "\":{\"type\":\"date\"},\"" +
                                 INT_FIELD_NAME + "\":{\"type\":\"integer\"}}}}}"),
                         MapperService.MergeReason.MAPPING_UPDATE, false);
+                if (idxSettings.getIndexVersionCreated().before(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
+                    testCase.assertWarnings("geo_point lat_lon parameter is deprecated and will be removed in the next major release",
+                            "geo_point geohash parameter is deprecated and will be removed in the next major release",
+                            "geo_point geohash_prefix parameter is deprecated and will be removed in the next major release");
+                }
             }
             testCase.initializeAdditionalMappings(mapperService);
-            this.namedWriteableRegistry = injector.getInstance(NamedWriteableRegistry.class);
         }
 
         @Override
         public void close() throws IOException {
-            injector.getInstance(ClusterService.class).close();
-            try {
-                terminate(injector.getInstance(ThreadPool.class));
-            } catch (InterruptedException e) {
-                IOUtils.reThrow(e);
-            }
         }
 
         QueryShardContext createShardContext() {
-            ClusterState state = ClusterState.builder(new ClusterName("_name")).build();
-            Client client = injector.getInstance(Client.class);
-            return new QueryShardContext(idxSettings, bitsetFilterCache, indexFieldDataService, mapperService, similarityService,
-                    scriptService, indicesQueriesRegistry, client, null, state);
+            return new QueryShardContext(0, idxSettings, bitsetFilterCache, indexFieldDataService, mapperService, similarityService,
+                    scriptService, xContentRegistry, this.client, null, () -> nowInMillis);
         }
 
         ScriptModule createScriptModule(List<ScriptPlugin> scriptPlugins) {
@@ -1151,8 +1093,6 @@ ScriptModule createScriptModule(List<ScriptPlugin> scriptPlugins) {
 
             Settings settings = Settings.builder()
                     .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
-                    // no file watching, so we don't need a ResourceWatcherService
-                    .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), false)
                     .build();
             Environment environment = new Environment(settings);
             return ScriptModule.create(settings, environment, null, scriptPlugins);
diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractStreamableTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractStreamableTestCase.java
new file mode 100644
index 0000000000000..bdf9e4badd809
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractStreamableTestCase.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.test;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteable;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.Streamable;
+
+import java.io.IOException;
+import java.util.Collections;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public abstract class AbstractStreamableTestCase<T extends Streamable> extends ESTestCase {
+    protected static final int NUMBER_OF_TEST_RUNS = 20;
+
+    /**
+     * Creates a random test instance to use in the tests. This method will be
+     * called multiple times during test execution and should return a different
+     * random instance each time it is called.
+     */
+    protected abstract T createTestInstance();
+
+    /**
+     * Creates an empty instance to use when deserialising the
+     * {@link Streamable}. This usually returns an instance created using the
+     * zer-arg constructor
+     */
+    protected abstract T createBlankInstance();
+
+    /**
+     * Tests that the equals and hashcode methods are consistent and copied
+     * versions of the instance have are equal.
+     */
+    public void testEqualsAndHashcode() throws IOException {
+        for (int runs = 0; runs < NUMBER_OF_TEST_RUNS; runs++) {
+            T firstInstance = createTestInstance();
+            assertFalse("instance is equal to null", firstInstance.equals(null));
+            assertFalse("instance is equal to incompatible type", firstInstance.equals(""));
+            assertEquals("instance is not equal to self", firstInstance, firstInstance);
+            assertThat("same instance's hashcode returns different values if called multiple times", firstInstance.hashCode(),
+                    equalTo(firstInstance.hashCode()));
+
+            T secondInstance = copyInstance(firstInstance, Version.CURRENT);
+            assertEquals("instance is not equal to self", secondInstance, secondInstance);
+            assertEquals("instance is not equal to its copy", firstInstance, secondInstance);
+            assertEquals("equals is not symmetric", secondInstance, firstInstance);
+            assertThat("instance copy's hashcode is different from original hashcode", secondInstance.hashCode(),
+                    equalTo(firstInstance.hashCode()));
+
+            T thirdInstance = copyInstance(secondInstance, Version.CURRENT);
+            assertEquals("instance is not equal to self", thirdInstance, thirdInstance);
+            assertEquals("instance is not equal to its copy", secondInstance, thirdInstance);
+            assertThat("instance copy's hashcode is different from original hashcode", secondInstance.hashCode(),
+                    equalTo(thirdInstance.hashCode()));
+            assertEquals("equals is not transitive", firstInstance, thirdInstance);
+            assertThat("instance copy's hashcode is different from original hashcode", firstInstance.hashCode(),
+                    equalTo(thirdInstance.hashCode()));
+            assertEquals("equals is not symmetric", thirdInstance, secondInstance);
+            assertEquals("equals is not symmetric", thirdInstance, firstInstance);
+        }
+    }
+
+    /**
+     * Test serialization and deserialization of the test instance.
+     */
+    public void testSerialization() throws IOException {
+        for (int runs = 0; runs < NUMBER_OF_TEST_RUNS; runs++) {
+            T testInstance = createTestInstance();
+            assertSerialization(testInstance);
+        }
+    }
+
+    /**
+     * Serialize the given instance and asserts that both are equal
+     */
+    protected T assertSerialization(T testInstance) throws IOException {
+        T deserializedInstance = copyInstance(testInstance, Version.CURRENT);
+        assertEquals(testInstance, deserializedInstance);
+        assertEquals(testInstance.hashCode(), deserializedInstance.hashCode());
+        assertNotSame(testInstance, deserializedInstance);
+        return deserializedInstance;
+    }
+
+    /**
+     * Round trip {@code instance} through binary serialization, setting the wire compatibility version to {@code version}.
+     */
+    protected T copyInstance(T instance, Version version) throws IOException {
+        try (BytesStreamOutput output = new BytesStreamOutput()) {
+            output.setVersion(version);
+            instance.writeTo(output);
+            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(),
+                    getNamedWriteableRegistry())) {
+                in.setVersion(version);
+                T newInstance = createBlankInstance();
+                newInstance.readFrom(in);
+                return newInstance;
+            }
+        }
+    }
+
+    /**
+     * Get the {@link NamedWriteableRegistry} to use when de-serializing the object.
+     * 
+     * Override this method if you need to register {@link NamedWriteable}s for the test object to de-serialize.
+     * 
+     * By default this will return a {@link NamedWriteableRegistry} with no registered {@link NamedWriteable}s
+     */
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return new NamedWriteableRegistry(Collections.emptyList());
+    }
+
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/BackgroundIndexer.java b/test/framework/src/main/java/org/elasticsearch/test/BackgroundIndexer.java
index 3c5f105e4d123..b739099cff01c 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/BackgroundIndexer.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/BackgroundIndexer.java
@@ -18,7 +18,7 @@
  */
 
 import com.carrotsearch.randomizedtesting.RandomizedTest;
-import com.carrotsearch.randomizedtesting.generators.RandomInts;
+import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
 import com.carrotsearch.randomizedtesting.generators.RandomStrings;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
@@ -208,10 +208,10 @@ public void run() {
     }
 
     private XContentBuilder generateSource(long id, Random random) throws IOException {
-        int contentLength = RandomInts.randomIntBetween(random, minFieldSize, maxFieldSize);
+        int contentLength = RandomNumbers.randomIntBetween(random, minFieldSize, maxFieldSize);
         StringBuilder text = new StringBuilder(contentLength);
         while (text.length() < contentLength) {
-            int tokenLength = RandomInts.randomIntBetween(random, 1, Math.min(contentLength - text.length(), 10));
+            int tokenLength = RandomNumbers.randomIntBetween(random, 1, Math.min(contentLength - text.length(), 10));
             text.append(" ").append(RandomStrings.randomRealisticUnicodeOfCodepointLength(random, tokenLength));
         }
         XContentBuilder builder = XContentFactory.smileBuilder();
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java b/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java
index a6d35930e6b7b..f3d310ab78ac8 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java
@@ -20,9 +20,8 @@
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
-import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.LocalClusterUpdateTask;
 import org.elasticsearch.cluster.NodeConnectionsService;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -30,11 +29,11 @@
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.LocalTransportAddress;
+import org.elasticsearch.discovery.DiscoverySettings;
 import org.elasticsearch.threadpool.ThreadPool;
 
-import java.util.Arrays;
 import java.util.Collections;
-import java.util.HashSet;
+import java.util.EnumSet;
 import java.util.concurrent.CountDownLatch;
 
 import static junit.framework.TestCase.fail;
@@ -42,24 +41,35 @@
 public class ClusterServiceUtils {
 
     public static ClusterService createClusterService(ThreadPool threadPool) {
-        ClusterService clusterService = new ClusterService(Settings.builder().put("cluster.name", "ClusterServiceTests").build(),
-                new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
-                threadPool);
-        clusterService.setLocalNode(new DiscoveryNode("node", LocalTransportAddress.buildUnique(), Collections.emptyMap(),
-                new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())),Version.CURRENT));
+        DiscoveryNode discoveryNode = new DiscoveryNode("node", LocalTransportAddress.buildUnique(), Collections.emptyMap(),
+            EnumSet.allOf(DiscoveryNode.Role.class), Version.CURRENT);
+        return createClusterService(threadPool, discoveryNode);
+    }
+
+    public static ClusterService createClusterService(ThreadPool threadPool, DiscoveryNode localNode) {
+        return createClusterService(Settings.EMPTY, threadPool, localNode);
+    }
+
+    public static ClusterService createClusterService(Settings settings, ThreadPool threadPool, DiscoveryNode localNode) {
+        ClusterService clusterService = new ClusterService(
+            Settings.builder().put("cluster.name", "ClusterServiceTests").put(settings).build(),
+                new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
+                threadPool, () -> localNode);
         clusterService.setNodeConnectionsService(new NodeConnectionsService(Settings.EMPTY, null, null) {
             @Override
-            public void connectToAddedNodes(ClusterChangedEvent event) {
+            public void connectToNodes(DiscoveryNodes discoveryNodes) {
                 // skip
             }
 
             @Override
-            public void disconnectFromRemovedNodes(ClusterChangedEvent event) {
+            public void disconnectFromNodesExcept(DiscoveryNodes nodesToKeep) {
                 // skip
             }
         });
         clusterService.setClusterStatePublisher((event, ackListener) -> {
         });
+        clusterService.setDiscoverySettings(new DiscoverySettings(Settings.EMPTY,
+            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)));
         clusterService.start();
         final DiscoveryNodes.Builder nodes = DiscoveryNodes.builder(clusterService.state().nodes());
         nodes.masterNodeId(clusterService.localNode().getId());
@@ -79,16 +89,11 @@ public static void setState(ClusterService clusterService, ClusterState.Builder
 
     public static void setState(ClusterService clusterService, ClusterState clusterState) {
         CountDownLatch latch = new CountDownLatch(1);
-        clusterService.submitStateUpdateTask("test setting state", new ClusterStateUpdateTask() {
+        clusterService.submitStateUpdateTask("test setting state", new LocalClusterUpdateTask() {
             @Override
-            public ClusterState execute(ClusterState currentState) throws Exception {
+            public ClusterTasksResult<LocalClusterUpdateTask> execute(ClusterState currentState) throws Exception {
                 // make sure we increment versions as listener may depend on it for change
-                return ClusterState.builder(clusterState).version(currentState.version() + 1).build();
-            }
-
-            @Override
-            public boolean runOnlyOnMaster() {
-                return false;
+                return newState(ClusterState.builder(clusterState).version(currentState.version() + 1).build());
             }
 
             @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/test/CompositeTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/CompositeTestCluster.java
index c6470123090d5..8c0f7597023ce 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/CompositeTestCluster.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/CompositeTestCluster.java
@@ -280,7 +280,7 @@ private synchronized Client internalClient() {
 
     private final class ExternalClient extends FilterClient {
 
-        public ExternalClient() {
+        ExternalClient() {
             super(internalClient());
         }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESBackcompatTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESBackcompatTestCase.java
index 16647b04a4738..bc5aa63bf82f6 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESBackcompatTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESBackcompatTestCase.java
@@ -29,9 +29,8 @@
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.discovery.DiscoveryModule;
-import org.elasticsearch.node.Node;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.discovery.TestZenDiscovery;
 import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.test.junit.listeners.LoggingListener;
 
@@ -143,7 +142,7 @@ private Version compatibilityVersion(Class<?> clazz) {
         }
         CompatibilityVersion annotation = clazz.getAnnotation(CompatibilityVersion.class);
         if (annotation != null) {
-            return Version.smallest(Version.fromId(annotation.version()), compatibilityVersion(clazz.getSuperclass()));
+            return Version.min(Version.fromId(annotation.version()), compatibilityVersion(clazz.getSuperclass()));
         }
         return compatibilityVersion(clazz.getSuperclass());
     }
@@ -243,7 +242,7 @@ public void assertAllShardsOnNodes(String index, String pattern) {
     protected Settings commonNodeSettings(int nodeOrdinal) {
         Settings.Builder builder = Settings.builder().put(requiredSettings());
         builder.put(NetworkModule.TRANSPORT_TYPE_KEY, randomBoolean() ? "netty3" : "netty4"); // run same transport  / disco as external
-        builder.put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), "zen");
+        builder.put(TestZenDiscovery.USE_MOCK_PINGS.getKey(), false);
         return builder.build();
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
index f76c9e43a2226..9f8ede29abace 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
@@ -20,7 +20,7 @@
 
 import com.carrotsearch.randomizedtesting.RandomizedContext;
 import com.carrotsearch.randomizedtesting.annotations.TestGroup;
-import com.carrotsearch.randomizedtesting.generators.RandomInts;
+import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
 import org.apache.http.HttpHost;
 import org.apache.lucene.util.IOUtils;
@@ -28,14 +28,8 @@
 import org.apache.lucene.util.TestUtil;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.action.DocWriteResponse;
-import org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings;
-import org.elasticsearch.discovery.DiscoveryModule;
-import org.elasticsearch.client.RestClientBuilder;
-import org.elasticsearch.env.NodeEnvironment;
-import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.transport.MockTcpTransportPlugin;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.ShardOperationFailedException;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
@@ -64,6 +58,8 @@
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestClientBuilder;
+import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
@@ -73,6 +69,7 @@
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
+import org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings;
 import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
@@ -80,6 +77,7 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.regex.Regex;
@@ -92,16 +90,19 @@
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.discovery.Discovery;
+import org.elasticsearch.discovery.zen.ElectMasterService;
 import org.elasticsearch.discovery.zen.ZenDiscovery;
-import org.elasticsearch.discovery.zen.elect.ElectMasterService;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.IndexService;
@@ -119,13 +120,18 @@
 import org.elasticsearch.node.NodeMocksPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.MockSearchService;
 import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchService;
 import org.elasticsearch.test.client.RandomizingClient;
+import org.elasticsearch.test.discovery.TestZenDiscovery;
+import org.elasticsearch.test.disruption.NetworkDisruption;
 import org.elasticsearch.test.disruption.ServiceDisruptionScheme;
 import org.elasticsearch.test.store.MockFSIndexStore;
 import org.elasticsearch.test.transport.AssertingLocalTransport;
 import org.elasticsearch.test.transport.MockTransportService;
+import org.elasticsearch.transport.MockTcpTransportPlugin;
 import org.hamcrest.Matchers;
 import org.junit.After;
 import org.junit.AfterClass;
@@ -150,9 +156,11 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.IdentityHashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Random;
 import java.util.Set;
@@ -165,6 +173,8 @@
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.BooleanSupplier;
 import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import static org.elasticsearch.client.Requests.syncedFlushRequest;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
@@ -176,10 +186,12 @@
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoTimeout;
+import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.emptyArray;
 import static org.hamcrest.Matchers.emptyIterable;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.startsWith;
 
@@ -283,6 +295,7 @@ public abstract class ESIntegTestCase extends ESTestCase {
      */
     public static final String TESTS_ENABLE_MOCK_MODULES = "tests.enable_mock_modules";
 
+    private static final boolean MOCK_MODULES_ENABLED = "true".equals(System.getProperty(TESTS_ENABLE_MOCK_MODULES, "true"));
     /**
      * Threshold at which indexing switches from frequently async to frequently bulk.
      */
@@ -333,6 +346,13 @@ public static void beforeClass() throws Exception {
         initializeSuiteScope();
     }
 
+    @Override
+    protected final boolean enableWarningsCheck() {
+        //In an integ test it doesn't make sense to keep track of warnings: if the cluster is external the warnings are in another jvm,
+        //if the cluster is internal the deprecation logger is shared across all nodes
+        return false;
+    }
+
     protected final void beforeInternal() throws Exception {
         final Scope currentClusterScope = getCurrentClusterScope();
         switch (currentClusterScope) {
@@ -368,14 +388,14 @@ public void randomIndexTemplate() throws IOException {
         // TODO move settings for random directory etc here into the index based randomized settings.
         if (cluster().size() > 0) {
             Settings.Builder randomSettingsBuilder =
-                    setRandomIndexSettings(random(), Settings.builder());
+                setRandomIndexSettings(random(), Settings.builder());
             if (isInternalCluster()) {
                 // this is only used by mock plugins and if the cluster is not internal we just can't set it
                 randomSettingsBuilder.put(INDEX_TEST_SEED_SETTING.getKey(), random().nextLong());
             }
 
             randomSettingsBuilder.put(SETTING_NUMBER_OF_SHARDS, numberOfShards())
-                    .put(SETTING_NUMBER_OF_REPLICAS, numberOfReplicas());
+                .put(SETTING_NUMBER_OF_REPLICAS, numberOfReplicas());
 
             // if the test class is annotated with SuppressCodecs("*"), it means don't use lucene's codec randomization
             // otherwise, use it, it has assertions and so on that can find bugs.
@@ -403,11 +423,14 @@ public void randomIndexTemplate() throws IOException {
             if (randomBoolean()) {
                 randomSettingsBuilder.put(IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING.getKey(), randomBoolean());
             }
+            if (randomBoolean()) {
+                randomSettingsBuilder.put(IndexModule.INDEX_QUERY_CACHE_TERM_QUERIES_SETTING.getKey(), randomBoolean());
+            }
             PutIndexTemplateRequestBuilder putTemplate = client().admin().indices()
-                    .preparePutTemplate("random_index_template")
-                    .setTemplate("*")
-                    .setOrder(0)
-                    .setSettings(randomSettingsBuilder);
+                .preparePutTemplate("random_index_template")
+                .setTemplate("*")
+                .setOrder(0)
+                .setSettings(randomSettingsBuilder);
             if (mappings != null) {
                 logger.info("test using _default_ mappings: [{}]", mappings.bytes().utf8ToString());
                 putTemplate.addMapping("_default_", mappings);
@@ -434,7 +457,7 @@ protected Settings.Builder setRandomIndexSettings(Random random, Settings.Builde
 
         if (randomBoolean()) {
             // keep this low so we don't stall tests
-            builder.put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), RandomInts.randomIntBetween(random, 1, 15) + "ms");
+            builder.put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), RandomNumbers.randomIntBetween(random, 1, 15) + "ms");
         }
 
         return builder;
@@ -443,12 +466,12 @@ protected Settings.Builder setRandomIndexSettings(Random random, Settings.Builde
     private static Settings.Builder setRandomIndexMergeSettings(Random random, Settings.Builder builder) {
         if (random.nextBoolean()) {
             builder.put(MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING.getKey(),
-                    random.nextBoolean() ? random.nextDouble() : random.nextBoolean());
+                random.nextBoolean() ? random.nextDouble() : random.nextBoolean());
         }
         switch (random.nextInt(4)) {
             case 3:
-                final int maxThreadCount = RandomInts.randomIntBetween(random, 1, 4);
-                final int maxMergeCount = RandomInts.randomIntBetween(random, maxThreadCount, maxThreadCount + 4);
+                final int maxThreadCount = RandomNumbers.randomIntBetween(random, 1, 4);
+                final int maxMergeCount = RandomNumbers.randomIntBetween(random, maxThreadCount, maxThreadCount + 4);
                 builder.put(MergeSchedulerConfig.MAX_MERGE_COUNT_SETTING.getKey(), maxMergeCount);
                 builder.put(MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING.getKey(), maxThreadCount);
                 break;
@@ -459,7 +482,7 @@ private static Settings.Builder setRandomIndexMergeSettings(Random random, Setti
 
     private static Settings.Builder setRandomIndexTranslogSettings(Random random, Settings.Builder builder) {
         if (random.nextBoolean()) {
-            builder.put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(RandomInts.randomIntBetween(random, 1, 300), ByteSizeUnit.MB));
+            builder.put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(RandomNumbers.randomIntBetween(random, 1, 300), ByteSizeUnit.MB));
         }
         if (random.nextBoolean()) {
             builder.put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(1, ByteSizeUnit.PB)); // just don't flush
@@ -469,14 +492,14 @@ private static Settings.Builder setRandomIndexTranslogSettings(Random random, Se
         }
 
         if (random.nextBoolean()) {
-            builder.put(IndexSettings.INDEX_TRANSLOG_SYNC_INTERVAL_SETTING.getKey(), RandomInts.randomIntBetween(random, 100, 5000), TimeUnit.MILLISECONDS);
+            builder.put(IndexSettings.INDEX_TRANSLOG_SYNC_INTERVAL_SETTING.getKey(), RandomNumbers.randomIntBetween(random, 100, 5000), TimeUnit.MILLISECONDS);
         }
 
         return builder;
     }
 
     private TestCluster buildWithPrivateContext(final Scope scope, final long seed) throws Exception {
-        return RandomizedContext.current().runWithPrivateRandomness(new com.carrotsearch.randomizedtesting.Randomness(seed), new Callable<TestCluster>() {
+        return RandomizedContext.current().runWithPrivateRandomness(seed, new Callable<TestCluster>() {
             @Override
             public TestCluster call() throws Exception {
                 return buildTestCluster(scope, seed);
@@ -524,10 +547,15 @@ protected final void afterInternal(boolean afterClass) throws Exception {
                 if (cluster() != null) {
                     if (currentClusterScope != Scope.TEST) {
                         MetaData metaData = client().admin().cluster().prepareState().execute().actionGet().getState().getMetaData();
-                        assertThat("test leaves persistent cluster metadata behind: " + metaData.persistentSettings().getAsMap(), metaData
-                                .persistentSettings().getAsMap().size(), equalTo(0));
-                        assertThat("test leaves transient cluster metadata behind: " + metaData.transientSettings().getAsMap(), metaData
-                                .transientSettings().getAsMap().size(), equalTo(0));
+                        final Map<String, String> persistent = metaData.persistentSettings().getAsMap();
+                        assertThat("test leaves persistent cluster metadata behind: " + persistent, persistent.size(), equalTo(0));
+                        final Map<String, String> transientSettings =  new HashMap<>(metaData.transientSettings().getAsMap());
+                        if (isInternalCluster() && internalCluster().getAutoManageMinMasterNode()) {
+                            // this is set by the test infra
+                            transientSettings.remove(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey());
+                        }
+                        assertThat("test leaves transient cluster metadata behind: " + transientSettings,
+                            transientSettings.keySet(), empty());
                     }
                     ensureClusterSizeConsistency();
                     ensureClusterStateConsistency();
@@ -536,12 +564,11 @@ protected final void afterInternal(boolean afterClass) throws Exception {
                         for (Discovery discovery : internalCluster().getInstances(Discovery.class)) {
                             if (discovery instanceof ZenDiscovery) {
                                 final ZenDiscovery zenDiscovery = (ZenDiscovery) discovery;
-                                assertBusy(new Runnable() {
-                                    @Override
-                                    public void run() {
-                                        assertThat("still having pending states: " + Strings.arrayToDelimitedString(zenDiscovery.pendingClusterStates(), "\n"),
-                                                zenDiscovery.pendingClusterStates(), emptyArray());
-                                    }
+                                assertBusy(() -> {
+                                    final ClusterState[] states = zenDiscovery.pendingClusterStates();
+                                    assertThat(zenDiscovery.localNode().getName() + " still having pending states:\n" +
+                                            Stream.of(states).map(ClusterState::toString).collect(Collectors.joining("\n")),
+                                        states, emptyArray());
                                 });
                             }
                         }
@@ -575,7 +602,7 @@ protected Set<String> excludeTemplates() {
         return Collections.emptySet();
     }
 
-    protected void beforeIndexDeletion() {
+    protected void beforeIndexDeletion() throws Exception {
         cluster().beforeIndexDeletion();
     }
 
@@ -680,7 +707,7 @@ public Settings indexSettings() {
         }
         // 30% of the time
         if (randomInt(9) < 3) {
-            final String dataPath = randomAsciiOfLength(10);
+            final String dataPath = randomAlphaOfLength(10);
             logger.info("using custom data_path for index: [{}]", dataPath);
             builder.put(IndexMetaData.SETTING_DATA_PATH, dataPath);
         }
@@ -761,17 +788,14 @@ private Settings.Builder getExcludeSettings(String index, int num, Settings.Buil
      */
     public void waitNoPendingTasksOnAll() throws Exception {
         assertNoTimeout(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).get());
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                for (Client client : clients()) {
-                    ClusterHealthResponse clusterHealth = client.admin().cluster().prepareHealth().setLocal(true).get();
-                    assertThat("client " + client + " still has in flight fetch", clusterHealth.getNumberOfInFlightFetch(), equalTo(0));
-                    PendingClusterTasksResponse pendingTasks = client.admin().cluster().preparePendingClusterTasks().setLocal(true).get();
-                    assertThat("client " + client + " still has pending tasks " + pendingTasks.prettyPrint(), pendingTasks, Matchers.emptyIterable());
-                    clusterHealth = client.admin().cluster().prepareHealth().setLocal(true).get();
-                    assertThat("client " + client + " still has in flight fetch", clusterHealth.getNumberOfInFlightFetch(), equalTo(0));
-                }
+        assertBusy(() -> {
+            for (Client client : clients()) {
+                ClusterHealthResponse clusterHealth = client.admin().cluster().prepareHealth().setLocal(true).get();
+                assertThat("client " + client + " still has in flight fetch", clusterHealth.getNumberOfInFlightFetch(), equalTo(0));
+                PendingClusterTasksResponse pendingTasks = client.admin().cluster().preparePendingClusterTasks().setLocal(true).get();
+                assertThat("client " + client + " still has pending tasks " + pendingTasks, pendingTasks, Matchers.emptyIterable());
+                clusterHealth = client.admin().cluster().prepareHealth().setLocal(true).get();
+                assertThat("client " + client + " still has in flight fetch", clusterHealth.getNumberOfInFlightFetch(), equalTo(0));
             }
         });
         assertNoTimeout(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).get());
@@ -829,7 +853,7 @@ public void assertResultsAndLogOnFailure(long expectedResults, SearchResponse se
             String failMsg = sb.toString();
             for (SearchHit hit : searchResponse.getHits().getHits()) {
                 sb.append("\n-> _index: [").append(hit.getIndex()).append("] type [").append(hit.getType())
-                        .append("] id [").append(hit.id()).append("]");
+                    .append("] id [").append(hit.id()).append("]");
             }
             logger.warn("{}", sb);
             fail(failMsg);
@@ -849,7 +873,7 @@ public void allowNodes(String index, int n) {
             getExcludeSettings(index, n, builder);
         }
         Settings build = builder.build();
-        if (!build.getAsMap().isEmpty()) {
+        if (!build.isEmpty()) {
             logger.debug("allowNodes: updating [{}]'s setting to [{}]", index, build.toDelimitedString(';'));
             client().admin().indices().prepareUpdateSettings(index).setSettings(build).execute().actionGet();
         }
@@ -872,14 +896,45 @@ public ClusterHealthStatus ensureGreen(String... indices) {
      * @param timeout time out value to set on {@link org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest}
      */
     public ClusterHealthStatus ensureGreen(TimeValue timeout, String... indices) {
-        ClusterHealthResponse actionGet = client().admin().cluster()
-                .health(Requests.clusterHealthRequest(indices).timeout(timeout).waitForGreenStatus().waitForEvents(Priority.LANGUID).waitForNoRelocatingShards(true)).actionGet();
+        return ensureColor(ClusterHealthStatus.GREEN, timeout, indices);
+    }
+
+    /**
+     * Ensures the cluster has a yellow state via the cluster health API.
+     */
+    public ClusterHealthStatus ensureYellow(String... indices) {
+        return ensureColor(ClusterHealthStatus.YELLOW, TimeValue.timeValueSeconds(30), indices);
+    }
+
+    private ClusterHealthStatus ensureColor(ClusterHealthStatus clusterHealthStatus, TimeValue timeout, String... indices) {
+        String color = clusterHealthStatus.name().toLowerCase(Locale.ROOT);
+        String method = "ensure" + Strings.capitalize(color);
+
+        ClusterHealthRequest healthRequest = Requests.clusterHealthRequest(indices)
+            .timeout(timeout)
+            .waitForStatus(clusterHealthStatus)
+            .waitForEvents(Priority.LANGUID)
+            .waitForNoRelocatingShards(true)
+            // We currently often use ensureGreen or ensureYellow to check whether the cluster is back in a good state after shutting down
+            // a node. If the node that is stopped is the master node, another node will become master and publish a cluster state where it
+            // is master but where the node that was stopped hasn't been removed yet from the cluster state. It will only subsequently
+            // publish a second state where the old master is removed. If the ensureGreen/ensureYellow is timed just right, it will get to
+            // execute before the second cluster state update removes the old master and the condition ensureGreen / ensureYellow will
+            // trivially hold if it held before the node was shut down. The following "waitForNodes" condition ensures that the node has
+            // been removed by the master so that the health check applies to the set of nodes we expect to be part of the cluster.
+            .waitForNodes(Integer.toString(cluster().size()));
+
+        ClusterHealthResponse actionGet = client().admin().cluster().health(healthRequest).actionGet();
         if (actionGet.isTimedOut()) {
-            logger.info("ensureGreen timed out, cluster state:\n{}\n{}", client().admin().cluster().prepareState().get().getState().prettyPrint(), client().admin().cluster().preparePendingClusterTasks().get().prettyPrint());
-            fail("timed out waiting for green state");
-        }
-        assertThat(actionGet.getStatus(), equalTo(ClusterHealthStatus.GREEN));
-        logger.debug("indices {} are green", indices.length == 0 ? "[_all]" : indices);
+            logger.info("{} timed out, cluster state:\n{}\n{}",
+                method,
+                client().admin().cluster().prepareState().get().getState(),
+                client().admin().cluster().preparePendingClusterTasks().get());
+            fail("timed out waiting for " + color + " state");
+        }
+        assertThat("Expected at least " + clusterHealthStatus + " but got " + actionGet.getStatus(),
+            actionGet.getStatus().value(), lessThanOrEqualTo(clusterHealthStatus.value()));
+        logger.debug("indices {} are {}", indices.length == 0 ? "[_all]" : indices, color);
         return actionGet.getStatus();
     }
 
@@ -900,9 +955,10 @@ public ClusterHealthStatus waitForRelocation(ClusterHealthStatus status) {
             request.waitForStatus(status);
         }
         ClusterHealthResponse actionGet = client().admin().cluster()
-                .health(request).actionGet();
+            .health(request).actionGet();
         if (actionGet.isTimedOut()) {
-            logger.info("waitForRelocation timed out (status={}), cluster state:\n{}\n{}", status, client().admin().cluster().prepareState().get().getState().prettyPrint(), client().admin().cluster().preparePendingClusterTasks().get().prettyPrint());
+            logger.info("waitForRelocation timed out (status={}), cluster state:\n{}\n{}", status,
+                client().admin().cluster().prepareState().get().getState(), client().admin().cluster().preparePendingClusterTasks().get());
             assertThat("timed out waiting for relocation", actionGet.isTimedOut(), equalTo(false));
         }
         if (status != null) {
@@ -945,7 +1001,7 @@ public long waitForDocs(final long numDocs, @Nullable final BackgroundIndexer in
      * @return the actual number of docs seen.
      */
     public long waitForDocs(final long numDocs, int maxWaitTime, TimeUnit maxWaitTimeUnit, @Nullable final BackgroundIndexer indexer)
-            throws InterruptedException {
+        throws InterruptedException {
         final AtomicLong lastKnownCount = new AtomicLong(-1);
         long lastStartCount = -1;
         BooleanSupplier testDocs = () -> {
@@ -988,29 +1044,16 @@ public long waitForDocs(final long numDocs, int maxWaitTime, TimeUnit maxWaitTim
      */
     public void setMinimumMasterNodes(int n) {
         assertTrue(client().admin().cluster().prepareUpdateSettings().setTransientSettings(
-                Settings.builder().put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), n))
-                .get().isAcknowledged());
-    }
-
-    /**
-     * Ensures the cluster has a yellow state via the cluster health API.
-     */
-    public ClusterHealthStatus ensureYellow(String... indices) {
-        ClusterHealthResponse actionGet = client().admin().cluster()
-                .health(Requests.clusterHealthRequest(indices).waitForNoRelocatingShards(true).waitForYellowStatus().waitForEvents(Priority.LANGUID)).actionGet();
-        if (actionGet.isTimedOut()) {
-            logger.info("ensureYellow timed out, cluster state:\n{}\n{}", client().admin().cluster().prepareState().get().getState().prettyPrint(), client().admin().cluster().preparePendingClusterTasks().get().prettyPrint());
-            assertThat("timed out waiting for yellow", actionGet.isTimedOut(), equalTo(false));
-        }
-        logger.debug("indices {} are yellow", indices.length == 0 ? "[_all]" : indices);
-        return actionGet.getStatus();
+            Settings.builder().put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), n))
+            .get().isAcknowledged());
     }
 
     /**
      * Prints the current cluster state as debug logging.
      */
     public void logClusterState() {
-        logger.debug("cluster state:\n{}\n{}", client().admin().cluster().prepareState().get().getState().prettyPrint(), client().admin().cluster().preparePendingClusterTasks().get().prettyPrint());
+        logger.debug("cluster state:\n{}\n{}",
+            client().admin().cluster().prepareState().get().getState(), client().admin().cluster().preparePendingClusterTasks().get());
     }
 
     /**
@@ -1019,7 +1062,7 @@ public void logClusterState() {
     public void logSegmentsState(String... indices) throws Exception {
         IndicesSegmentResponse segsRsp = client().admin().indices().prepareSegments(indices).get();
         logger.debug("segments {} state: \n{}", indices.length == 0 ? "[_all]" : indices,
-                segsRsp.toXContent(JsonXContent.contentBuilder().prettyPrint(), ToXContent.EMPTY_PARAMS).string());
+            segsRsp.toXContent(JsonXContent.contentBuilder().prettyPrint(), ToXContent.EMPTY_PARAMS).string());
     }
 
     /**
@@ -1030,7 +1073,7 @@ public void logMemoryStats() {
     }
 
     protected void ensureClusterSizeConsistency() {
-        if (cluster() != null) { // if static init fails the cluster can be null
+        if (cluster() != null && cluster().size() > 0) { // if static init fails the cluster can be null
             logger.trace("Check consistency for [{}] nodes", cluster().size());
             assertNoTimeout(client().admin().cluster().prepareHealth().setWaitForNodes(Integer.toString(cluster().size())).get());
         }
@@ -1040,11 +1083,19 @@ protected void ensureClusterSizeConsistency() {
      * Verifies that all nodes that have the same version of the cluster state as master have same cluster state
      */
     protected void ensureClusterStateConsistency() throws IOException {
-        if (cluster() != null) {
+        if (cluster() != null && cluster().size() > 0) {
+            final NamedWriteableRegistry namedWriteableRegistry;
+            if (isInternalCluster()) {
+                // If it's internal cluster - using existing registry in case plugin registered custom data
+                namedWriteableRegistry = internalCluster().getInstance(NamedWriteableRegistry.class);
+            } else {
+                // If it's external cluster - fall back to the standard set
+                namedWriteableRegistry = new NamedWriteableRegistry(ClusterModule.getNamedWriteables());
+            }
             ClusterState masterClusterState = client().admin().cluster().prepareState().all().get().getState();
             byte[] masterClusterStateBytes = ClusterState.Builder.toBytes(masterClusterState);
             // remove local node reference
-            masterClusterState = ClusterState.Builder.fromBytes(masterClusterStateBytes, null);
+            masterClusterState = ClusterState.Builder.fromBytes(masterClusterStateBytes, null, namedWriteableRegistry);
             Map<String, Object> masterStateMap = convertToMap(masterClusterState);
             int masterClusterStateSize = ClusterState.Builder.toBytes(masterClusterState).length;
             String masterId = masterClusterState.nodes().getMasterNodeId();
@@ -1052,7 +1103,7 @@ protected void ensureClusterStateConsistency() throws IOException {
                 ClusterState localClusterState = client.admin().cluster().prepareState().all().setLocal(true).get().getState();
                 byte[] localClusterStateBytes = ClusterState.Builder.toBytes(localClusterState);
                 // remove local node reference
-                localClusterState = ClusterState.Builder.fromBytes(localClusterStateBytes, null);
+                localClusterState = ClusterState.Builder.fromBytes(localClusterStateBytes, null, namedWriteableRegistry);
                 final Map<String, Object> localStateMap = convertToMap(localClusterState);
                 final int localClusterStateSize = ClusterState.Builder.toBytes(localClusterState).length;
                 // Check that the non-master node has the same version of the cluster state as the master and
@@ -1102,18 +1153,30 @@ protected void ensureStableCluster(int nodeCount, TimeValue timeValue, boolean l
         }
         logger.debug("ensuring cluster is stable with [{}] nodes. access node: [{}]. timeout: [{}]", nodeCount, viaNode, timeValue);
         ClusterHealthResponse clusterHealthResponse = client(viaNode).admin().cluster().prepareHealth()
-                .setWaitForEvents(Priority.LANGUID)
-                .setWaitForNodes(Integer.toString(nodeCount))
-                .setTimeout(timeValue)
-                .setLocal(local)
-                .setWaitForNoRelocatingShards(true)
-                .get();
+            .setWaitForEvents(Priority.LANGUID)
+            .setWaitForNodes(Integer.toString(nodeCount))
+            .setTimeout(timeValue)
+            .setLocal(local)
+            .setWaitForNoRelocatingShards(true)
+            .get();
         if (clusterHealthResponse.isTimedOut()) {
             ClusterStateResponse stateResponse = client(viaNode).admin().cluster().prepareState().get();
             fail("failed to reach a stable cluster of [" + nodeCount + "] nodes. Tried via [" + viaNode + "]. last cluster state:\n"
-                    + stateResponse.getState().prettyPrint());
+                + stateResponse.getState());
         }
         assertThat(clusterHealthResponse.isTimedOut(), is(false));
+        ensureFullyConnectedCluster();
+    }
+
+    /**
+     * Ensures that all nodes in the cluster are connected to each other.
+     *
+     * Some network disruptions may leave nodes that are not the master disconnected from each other.
+     * {@link org.elasticsearch.cluster.NodeConnectionsService} will eventually reconnect but it's
+     * handy to be able to ensure this happens faster
+     */
+    protected void ensureFullyConnectedCluster() {
+        NetworkDisruption.ensureFullyConnectedCluster(internalCluster());
     }
 
     /**
@@ -1172,10 +1235,10 @@ protected final IndexResponse index(String index, String type, String id, Object
      *   return client().prepareIndex(index, type, id).setSource(source).execute().actionGet();
      * </pre>
      * <p>
-     * where source is a String.
+     * where source is a JSON String.
      */
     protected final IndexResponse index(String index, String type, String id, String source) {
-        return client().prepareIndex(index, type, id).setSource(source).execute().actionGet();
+        return client().prepareIndex(index, type, id).setSource(source, XContentType.JSON).execute().actionGet();
     }
 
     /**
@@ -1204,7 +1267,7 @@ protected final void flushAndRefresh(String... indices) {
      */
     protected final FlushResponse flush(String... indices) {
         waitForRelocation();
-        FlushResponse actionGet = client().admin().indices().prepareFlush(indices).setWaitIfOngoing(true).execute().actionGet();
+        FlushResponse actionGet = client().admin().indices().prepareFlush(indices).execute().actionGet();
         for (ShardOperationFailedException failure : actionGet.getShardFailures()) {
             assertThat("unexpected flush failure " + failure.reason(), failure.status(), equalTo(RestStatus.SERVICE_UNAVAILABLE));
         }
@@ -1234,7 +1297,7 @@ protected boolean indexExists(String index) {
      */
     protected final void enableAllocation(String... indices) {
         client().admin().indices().prepareUpdateSettings(indices).setSettings(Settings.builder().put(
-                EnableAllocationDecider.INDEX_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), "all"
+            EnableAllocationDecider.INDEX_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), "all"
         )).get();
     }
 
@@ -1243,7 +1306,7 @@ protected final void enableAllocation(String... indices) {
      */
     protected final void disableAllocation(String... indices) {
         client().admin().indices().prepareUpdateSettings(indices).setSettings(Settings.builder().put(
-                EnableAllocationDecider.INDEX_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), "none"
+            EnableAllocationDecider.INDEX_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), "none"
         )).get();
     }
 
@@ -1266,9 +1329,6 @@ public void indexRandom(boolean forceRefresh, boolean dummyDocuments, IndexReque
         indexRandom(forceRefresh, dummyDocuments, Arrays.asList(builders));
     }
 
-
-    private static final String RANDOM_BOGUS_TYPE = "RANDOM_BOGUS_TYPE______";
-
     /**
      * Indexes the given {@link IndexRequestBuilder} instances randomly. It shuffles the given builders and either
      * indexes them in a blocking or async fashion. This is very useful to catch problems that relate to internal document
@@ -1316,31 +1376,33 @@ public void indexRandom(boolean forceRefresh, boolean dummyDocuments, List<Index
      * @param builders       the documents to index.
      */
     public void indexRandom(boolean forceRefresh, boolean dummyDocuments, boolean maybeFlush, List<IndexRequestBuilder> builders) throws InterruptedException, ExecutionException {
-
         Random random = random();
-        Set<String> indicesSet = new HashSet<>();
+        Map<String, Set<String>> indicesAndTypes = new HashMap<>();
         for (IndexRequestBuilder builder : builders) {
-            indicesSet.add(builder.request().index());
+            final Set<String> types = indicesAndTypes.computeIfAbsent(builder.request().index(), index -> new HashSet<>());
+            types.add(builder.request().type());
         }
-        Set<Tuple<String, String>> bogusIds = new HashSet<>();
+        Set<List<String>> bogusIds = new HashSet<>(); // (index, type, id)
         if (random.nextBoolean() && !builders.isEmpty() && dummyDocuments) {
             builders = new ArrayList<>(builders);
-            final String[] indices = indicesSet.toArray(new String[indicesSet.size()]);
             // inject some bogus docs
             final int numBogusDocs = scaledRandomIntBetween(1, builders.size() * 2);
             final int unicodeLen = between(1, 10);
             for (int i = 0; i < numBogusDocs; i++) {
-                String id = randomRealisticUnicodeOfLength(unicodeLen) + Integer.toString(dummmyDocIdGenerator.incrementAndGet());
-                String index = RandomPicks.randomFrom(random, indices);
-                bogusIds.add(new Tuple<>(index, id));
-                builders.add(client().prepareIndex(index, RANDOM_BOGUS_TYPE, id).setSource("{}"));
+                String id = "bogus_doc_" + randomRealisticUnicodeOfLength(unicodeLen) + Integer.toString(dummmyDocIdGenerator.incrementAndGet());
+                Map.Entry<String, Set<String>> indexAndTypes = RandomPicks.randomFrom(random, indicesAndTypes.entrySet());
+                String index = indexAndTypes.getKey();
+                String type = RandomPicks.randomFrom(random, indexAndTypes.getValue());
+                bogusIds.add(Arrays.asList(index, type, id));
+                // We configure a routing key in case the mapping requires it
+                builders.add(client().prepareIndex(index, type, id).setSource("{}", XContentType.JSON).setRouting(id));
             }
         }
-        final String[] indices = indicesSet.toArray(new String[indicesSet.size()]);
         Collections.shuffle(builders, random());
         final CopyOnWriteArrayList<Tuple<IndexRequestBuilder, Exception>> errors = new CopyOnWriteArrayList<>();
         List<CountDownLatch> inFlightAsyncOperations = new ArrayList<>();
         // If you are indexing just a few documents then frequently do it one at a time.  If many then frequently in bulk.
+        final String[] indices = indicesAndTypes.keySet().toArray(new String[0]);
         if (builders.size() < FREQUENT_BULK_THRESHOLD ? frequently() : builders.size() < ALWAYS_BULK_THRESHOLD ? rarely() : false) {
             if (frequently()) {
                 logger.info("Index [{}] docs async: [{}] bulk: [{}]", builders.size(), true, false);
@@ -1357,7 +1419,7 @@ public void indexRandom(boolean forceRefresh, boolean dummyDocuments, boolean ma
             }
         } else {
             List<List<IndexRequestBuilder>> partition = eagerPartition(builders, Math.min(MAX_BULK_INDEX_REQUEST_SIZE,
-                    Math.max(1, (int) (builders.size() * randomDouble()))));
+                Math.max(1, (int) (builders.size() * randomDouble()))));
             logger.info("Index [{}] docs async: [{}] bulk: [{}] partitions [{}]", builders.size(), false, true, partition.size());
             for (List<IndexRequestBuilder> segmented : partition) {
                 BulkRequestBuilder bulkBuilder = client().prepareBulk();
@@ -1382,10 +1444,10 @@ public void indexRandom(boolean forceRefresh, boolean dummyDocuments, boolean ma
         assertThat(actualErrors, emptyIterable());
         if (!bogusIds.isEmpty()) {
             // delete the bogus types again - it might trigger merges or at least holes in the segments and enforces deleted docs!
-            for (Tuple<String, String> doc : bogusIds) {
-                assertEquals("failed to delete a dummy doc [" + doc.v1() + "][" + doc.v2() + "]",
+            for (List<String> doc : bogusIds) {
+                assertEquals("failed to delete a dummy doc [" + doc.get(0) + "][" + doc.get(2) + "]",
                     DocWriteResponse.Result.DELETED,
-                    client().prepareDelete(doc.v1(), RANDOM_BOGUS_TYPE, doc.v2()).get().getResult());
+                    client().prepareDelete(doc.get(0), doc.get(1), doc.get(2)).setRouting(doc.get(2)).get().getResult());
             }
         }
         if (forceRefresh) {
@@ -1409,7 +1471,8 @@ public static void enableIndexBlock(String index, String block) {
 
     /** Sets or unsets the cluster read_only mode **/
     public static void setClusterReadOnly(boolean value) {
-        Settings settings = Settings.builder().put(MetaData.SETTING_READ_ONLY_SETTING.getKey(), value).build();
+        Settings settings = value ? Settings.builder().put(MetaData.SETTING_READ_ONLY_SETTING.getKey(), value).build() :
+            Settings.builder().putNull(MetaData.SETTING_READ_ONLY_SETTING.getKey()).build()  ;
         assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(settings).get());
     }
 
@@ -1426,18 +1489,18 @@ private void postIndexAsyncActions(String[] indices, List<CountDownLatch> inFlig
         if (rarely()) {
             if (rarely()) {
                 client().admin().indices().prepareRefresh(indices).setIndicesOptions(IndicesOptions.lenientExpandOpen()).execute(
-                        new LatchedActionListener<>(newLatch(inFlightAsyncOperations)));
+                    new LatchedActionListener<>(newLatch(inFlightAsyncOperations)));
             } else if (maybeFlush && rarely()) {
                 if (randomBoolean()) {
                     client().admin().indices().prepareFlush(indices).setIndicesOptions(IndicesOptions.lenientExpandOpen()).execute(
-                            new LatchedActionListener<>(newLatch(inFlightAsyncOperations)));
+                        new LatchedActionListener<>(newLatch(inFlightAsyncOperations)));
                 } else {
                     client().admin().indices().syncedFlush(syncedFlushRequest(indices).indicesOptions(IndicesOptions.lenientExpandOpen()),
                         new LatchedActionListener<>(newLatch(inFlightAsyncOperations)));
                 }
             } else if (rarely()) {
                 client().admin().indices().prepareForceMerge(indices).setIndicesOptions(IndicesOptions.lenientExpandOpen()).setMaxNumSegments(between(1, 10)).setFlush(maybeFlush && randomBoolean()).execute(
-                        new LatchedActionListener<>(newLatch(inFlightAsyncOperations)));
+                    new LatchedActionListener<>(newLatch(inFlightAsyncOperations)));
             }
         }
         while (inFlightAsyncOperations.size() > MAX_IN_FLIGHT_ASYNC_INDEXES) {
@@ -1500,6 +1563,12 @@ public enum Scope {
          */
         boolean supportsDedicatedMasters() default true;
 
+        /**
+         * The cluster automatically manages the {@link ElectMasterService#DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING} by default
+         * as nodes are started and stopped. Set this to false to manage the setting manually.
+         */
+        boolean autoMinMasterNodes() default true;
+
         /**
          * Returns the number of client nodes in the cluster. Default is {@link InternalTestCluster#DEFAULT_NUM_CLIENT_NODES}, a
          * negative value means that the number of client nodes will be randomized.
@@ -1521,7 +1590,7 @@ public enum Scope {
     private class LatchedActionListener<Response> implements ActionListener<Response> {
         private final CountDownLatch latch;
 
-        public LatchedActionListener(CountDownLatch latch) {
+        LatchedActionListener(CountDownLatch latch) {
             this.latch = latch;
         }
 
@@ -1549,7 +1618,7 @@ private class PayloadLatchedActionListener<Response, T> extends LatchedActionLis
         private final CopyOnWriteArrayList<Tuple<T, Exception>> errors;
         private final T builder;
 
-        public PayloadLatchedActionListener(T builder, CountDownLatch latch, CopyOnWriteArrayList<Tuple<T, Exception>> errors) {
+        PayloadLatchedActionListener(T builder, CountDownLatch latch, CopyOnWriteArrayList<Tuple<T, Exception>> errors) {
             super(latch);
             this.errors = errors;
             this.builder = builder;
@@ -1567,7 +1636,7 @@ protected void addError(Exception e) {
      */
     public void clearScroll(String... scrollIds) {
         ClearScrollResponse clearResponse = client().prepareClearScroll()
-                .setScrollIds(Arrays.asList(scrollIds)).get();
+            .setScrollIds(Arrays.asList(scrollIds)).get();
         assertThat(clearResponse.isSucceeded(), equalTo(true));
     }
 
@@ -1597,6 +1666,11 @@ private boolean getSupportsDedicatedMasters() {
         return annotation == null ? true : annotation.supportsDedicatedMasters();
     }
 
+    private boolean getAutoMinMasterNodes() {
+        ClusterScope annotation = getAnnotation(this.getClass(), ClusterScope.class);
+        return annotation == null ? true : annotation.autoMinMasterNodes();
+    }
+
     private int getNumDataNodes() {
         ClusterScope annotation = getAnnotation(this.getClass(), ClusterScope.class);
         return annotation == null ? -1 : annotation.numDataNodes();
@@ -1631,20 +1705,21 @@ private boolean randomDynamicTemplates() {
      */
     protected Settings nodeSettings(int nodeOrdinal) {
         Settings.Builder builder = Settings.builder()
-                .put(NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.getKey(), Integer.MAX_VALUE)
-                // Default the watermarks to absurdly low to prevent the tests
-                // from failing on nodes without enough disk space
-                .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "1b")
-                .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "1b")
-                .put(ScriptService.SCRIPT_MAX_COMPILATIONS_PER_MINUTE.getKey(), 1000)
-                .put("script.stored", "true")
-                .put("script.inline", "true")
-                // by default we never cache below 10k docs in a segment,
-                // bypass this limit so that caching gets some testing in
-                // integration tests that usually create few documents
-                .put(IndicesQueryCache.INDICES_QUERIES_CACHE_ALL_SEGMENTS_SETTING.getKey(), nodeOrdinal % 2 == 0)
-                // wait short time for other active shards before actually deleting, default 30s not needed in tests
-                .put(IndicesStore.INDICES_STORE_DELETE_SHARD_TIMEOUT.getKey(), new TimeValue(1, TimeUnit.SECONDS));
+            .put(NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.getKey(), Integer.MAX_VALUE)
+            // Default the watermarks to absurdly low to prevent the tests
+            // from failing on nodes without enough disk space
+            .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "1b")
+            .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "1b")
+            .put(ScriptService.SCRIPT_MAX_COMPILATIONS_PER_MINUTE.getKey(), 2048)
+            .put("script.stored", "true")
+            .put("script.inline", "true")
+            // by default we never cache below 10k docs in a segment,
+            // bypass this limit so that caching gets some testing in
+            // integration tests that usually create few documents
+            .put(IndicesQueryCache.INDICES_QUERIES_CACHE_ALL_SEGMENTS_SETTING.getKey(), nodeOrdinal % 2 == 0)
+            // wait short time for other active shards before actually deleting, default 30s not needed in tests
+            .put(IndicesStore.INDICES_STORE_DELETE_SHARD_TIMEOUT.getKey(), new TimeValue(1, TimeUnit.SECONDS))
+            .put(SearchService.LOW_LEVEL_CANCELLATION_SETTING.getKey(), randomBoolean());
         return builder.build();
     }
 
@@ -1738,9 +1813,10 @@ protected TestCluster buildTestCluster(Scope scope, long seed) throws IOExceptio
             }
             mockPlugins = mocks;
         }
-        return new InternalTestCluster(seed, createTempDir(), supportsDedicatedMasters, minNumDataNodes, maxNumDataNodes,
-                InternalTestCluster.clusterName(scope.name(), seed) + "-cluster", nodeConfigurationSource, getNumClientNodes(),
-                InternalTestCluster.DEFAULT_ENABLE_HTTP_PIPELINING, nodePrefix, mockPlugins, getClientWrapper());
+        return new InternalTestCluster(seed, createTempDir(), supportsDedicatedMasters, getAutoMinMasterNodes(),
+            minNumDataNodes, maxNumDataNodes,
+            InternalTestCluster.clusterName(scope.name(), seed) + "-cluster", nodeConfigurationSource, getNumClientNodes(),
+            InternalTestCluster.DEFAULT_ENABLE_HTTP_PIPELINING, nodePrefix, mockPlugins, getClientWrapper());
     }
 
     protected NodeConfigurationSource getNodeConfigSource() {
@@ -1769,10 +1845,8 @@ protected NodeConfigurationSource getNodeConfigSource() {
             public Settings nodeSettings(int nodeOrdinal) {
                 return Settings.builder()
                     .put(NetworkModule.HTTP_ENABLED.getKey(), false)
-                    .put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(),
-                        isNetwork ? DiscoveryModule.DISCOVERY_TYPE_SETTING.getDefault(Settings.EMPTY) : "local")
                     .put(networkSettings.build()).
-                    put(ESIntegTestCase.this.nodeSettings(nodeOrdinal)).build();
+                        put(ESIntegTestCase.this.nodeSettings(nodeOrdinal)).build();
             }
 
             @Override
@@ -1822,7 +1896,7 @@ protected Function<Client,Client> getClientWrapper() {
     /** Return the mock plugins the cluster should use */
     protected Collection<Class<? extends Plugin>> getMockPlugins() {
         final ArrayList<Class<? extends Plugin>> mocks = new ArrayList<>();
-        if (randomBoolean()) { // sometimes run without those completely
+        if (MOCK_MODULES_ENABLED && randomBoolean()) { // sometimes run without those completely
             if (randomBoolean() && addMockTransportService()) {
                 mocks.add(MockTransportService.TestPlugin.class);
             }
@@ -1844,6 +1918,8 @@ protected Collection<Class<? extends Plugin>> getMockPlugins() {
             mocks.add(AssertingLocalTransport.TestPlugin.class);
             mocks.add(MockTcpTransportPlugin.class);
         }
+
+        mocks.add(TestZenDiscovery.TestPlugin.class);
         mocks.add(TestSeedPlugin.class);
         return Collections.unmodifiableList(mocks);
     }
@@ -1905,7 +1981,7 @@ public static Path randomRepoPath(Settings settings) {
         assert repoFiles.length > 0;
         Path path;
         do {
-            path = repoFiles[0].resolve(randomAsciiOfLength(10));
+            path = repoFiles[0].resolve(randomAlphaOfLength(10));
         } while (Files.exists(path));
         return path;
     }
@@ -1958,19 +2034,17 @@ private static boolean runTestScopeLifecycle() {
 
 
     @Before
-    public final void before() throws Exception {
-
+    public final void setupTestCluster() throws Exception {
         if (runTestScopeLifecycle()) {
-            printTestMessage("setup");
+            printTestMessage("setting up");
             beforeInternal();
+            printTestMessage("all set up");
         }
-        printTestMessage("starting");
     }
 
 
     @After
-    public final void after() throws Exception {
-        printTestMessage("finished");
+    public final void cleanUpCluster() throws Exception {
         // Deleting indices is going to clear search contexts implicitly so we
         // need to check that there are no more in-flight search contexts before
         // we remove indices
@@ -1988,6 +2062,7 @@ public static void afterClass() throws Exception {
             try {
                 INSTANCE.printTestMessage("cleaning up after");
                 INSTANCE.afterInternal(true);
+                checkStaticState();
             } finally {
                 INSTANCE = null;
             }
@@ -2071,8 +2146,8 @@ protected Settings prepareBackwardsDataDir(Path backwardsIndex, Object... settin
         assertFalse(Files.exists(src));
         assertTrue(Files.exists(dest));
         Settings.Builder builder = Settings.builder()
-                .put(settings)
-                .put(Environment.PATH_DATA_SETTING.getKey(), dataDir.toAbsolutePath());
+            .put(settings)
+            .put(Environment.PATH_DATA_SETTING.getKey(), dataDir.toAbsolutePath());
 
         Path configDir = indexDir.resolve("config");
         if (Files.exists(configDir)) {
@@ -2081,6 +2156,17 @@ protected Settings prepareBackwardsDataDir(Path backwardsIndex, Object... settin
         return builder.build();
     }
 
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        if (isInternalCluster() && cluster().size() > 0) {
+            // If it's internal cluster - using existing registry in case plugin registered custom data
+            return internalCluster().getInstance(NamedXContentRegistry.class);
+        } else {
+            // If it's external cluster - fall back to the standard set
+            return new NamedXContentRegistry(ClusterModule.getNamedXWriteables());
+        }
+    }
+
     /**
      * Returns an instance of {@link RestClient} pointing to the current test cluster.
      * Creates a new client if the method is invoked for the first time in the context of the current test scope.
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java
index 99c21413619b3..e75ce8420eecc 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java
@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.test;
 
+import com.carrotsearch.randomizedtesting.RandomizedContext;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
@@ -34,24 +35,27 @@
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.node.MockNode;
 import org.elasticsearch.node.Node;
+import org.elasticsearch.node.NodeValidationException;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.test.discovery.TestZenDiscovery;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.junit.After;
 import org.junit.AfterClass;
-import org.junit.Before;
 import org.junit.BeforeClass;
 
 import java.io.IOException;
 import java.nio.file.Path;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
@@ -68,62 +72,56 @@ public abstract class ESSingleNodeTestCase extends ESTestCase {
 
     private static Node NODE = null;
 
-    private void reset() throws IOException {
-        assert NODE != null;
-        stopNode();
-        startNode();
-    }
-
-    protected void startNode() {
+    protected void startNode(long seed) throws Exception {
         assert NODE == null;
-        NODE = newNode();
-        // we must wait for the node to actually be up and running. otherwise the node might have started, elected itself master but might not yet have removed the
+        NODE = RandomizedContext.current().runWithPrivateRandomness(seed, this::newNode);
+        // we must wait for the node to actually be up and running. otherwise the node might have started,
+        // elected itself master but might not yet have removed the
         // SERVICE_UNAVAILABLE/1/state not recovered / initialized block
         ClusterHealthResponse clusterHealthResponse = client().admin().cluster().prepareHealth().setWaitForGreenStatus().get();
         assertFalse(clusterHealthResponse.isTimedOut());
         client().admin().indices()
-            .preparePutTemplate("random_index_template")
+            .preparePutTemplate("one_shard_index_template")
             .setTemplate("*")
             .setOrder(0)
             .setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
             .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)).get();
     }
 
-    protected static void stopNode() throws IOException {
+    private static void stopNode() throws IOException {
         Node node = NODE;
         NODE = null;
         IOUtils.close(node);
     }
 
-    private void cleanup(boolean resetNode) throws IOException {
-        assertAcked(client().admin().indices().prepareDelete("*").get());
-        if (resetNode) {
-            reset();
-        }
-        MetaData metaData = client().admin().cluster().prepareState().get().getState().getMetaData();
-        assertThat("test leaves persistent cluster metadata behind: " + metaData.persistentSettings().getAsMap(),
-                metaData.persistentSettings().getAsMap().size(), equalTo(0));
-        assertThat("test leaves transient cluster metadata behind: " + metaData.transientSettings().getAsMap(),
-                metaData.transientSettings().getAsMap().size(), equalTo(0));
-    }
-
-    @Before
     @Override
     public void setUp() throws Exception {
         super.setUp();
+        //the seed has to be created regardless of whether it will be used or not, for repeatability
+        long seed = random().nextLong();
         // Create the node lazily, on the first test. This is ok because we do not randomize any settings,
         // only the cluster name. This allows us to have overridden properties for plugins and the version to use.
         if (NODE == null) {
-            startNode();
+            startNode(seed);
         }
     }
 
-    @After
     @Override
     public void tearDown() throws Exception {
         logger.info("[{}#{}]: cleaning up after test", getTestClass().getSimpleName(), getTestName());
         super.tearDown();
-        cleanup(resetNodeAfterTest());
+        assertAcked(client().admin().indices().prepareDelete("*").get());
+        MetaData metaData = client().admin().cluster().prepareState().get().getState().getMetaData();
+        assertThat("test leaves persistent cluster metadata behind: " + metaData.persistentSettings().getAsMap(),
+                metaData.persistentSettings().size(), equalTo(0));
+        assertThat("test leaves transient cluster metadata behind: " + metaData.transientSettings().getAsMap(),
+                metaData.transientSettings().size(), equalTo(0));
+        if (resetNodeAfterTest()) {
+            assert NODE != null;
+            stopNode();
+            //the seed can be created within this if as it will either be executed before every test method or will never be.
+            startNode(random().nextLong());
+        }
     }
 
     @BeforeClass
@@ -145,7 +143,6 @@ protected boolean resetNodeAfterTest() {
         return false;
     }
 
-
     /** The plugin classes that should be added to the node. */
     protected Collection<Class<? extends Plugin>> getPlugins() {
         return Collections.emptyList();
@@ -166,25 +163,32 @@ protected Settings nodeSettings() {
     private Node newNode() {
         final Path tempDir = createTempDir();
         Settings settings = Settings.builder()
-            .put(ClusterName.CLUSTER_NAME_SETTING.getKey(), InternalTestCluster.clusterName("single-node-cluster", randomLong()))
+            .put(ClusterName.CLUSTER_NAME_SETTING.getKey(), InternalTestCluster.clusterName("single-node-cluster", random().nextLong()))
             .put(Environment.PATH_HOME_SETTING.getKey(), tempDir)
             .put(Environment.PATH_REPO_SETTING.getKey(), tempDir.resolve("repo"))
             // TODO: use a consistent data path for custom paths
             // This needs to tie into the ESIntegTestCase#indexSettings() method
             .put(Environment.PATH_SHARED_DATA_SETTING.getKey(), createTempDir().getParent())
-            .put("node.name", nodeName())
-            .put("script.inline", "true")
-            .put("script.stored", "true")
+            .put("node.name", "node_s_0")
             .put(ScriptService.SCRIPT_MAX_COMPILATIONS_PER_MINUTE.getKey(), 1000)
             .put(EsExecutors.PROCESSORS_SETTING.getKey(), 1) // limit the number of threads created
             .put(NetworkModule.HTTP_ENABLED.getKey(), false)
-            .put("discovery.type", "local")
             .put("transport.type", "local")
             .put(Node.NODE_DATA_SETTING.getKey(), true)
+            .put(NodeEnvironment.NODE_ID_SEED_SETTING.getKey(), random().nextLong())
             .put(nodeSettings()) // allow test cases to provide their own settings or override these
             .build();
-        Node build = new MockNode(settings, getPlugins());
-        build.start();
+        Collection<Class<? extends Plugin>> plugins = getPlugins();
+        if (plugins.contains(TestZenDiscovery.TestPlugin.class) == false) {
+            plugins = new ArrayList<>(plugins);
+            plugins.add(TestZenDiscovery.TestPlugin.class);
+        }
+        Node build = new MockNode(settings, plugins);
+        try {
+            build.start();
+        } catch (NodeValidationException e) {
+            throw new RuntimeException(e);
+        }
         return build;
     }
 
@@ -195,13 +199,6 @@ public Client client() {
         return NODE.client();
     }
 
-    /**
-     * Returns the single test nodes name.
-     */
-    public String nodeName() {
-        return "node_s_0";
-    }
-
     /**
      * Return a reference to the singleton node.
      */
@@ -257,7 +254,8 @@ protected IndexService createIndex(String index, CreateIndexRequestBuilder creat
         // Wait for the index to be allocated so that cluster state updates don't override
         // changes that would have been done locally
         ClusterHealthResponse health = client().admin().cluster()
-                .health(Requests.clusterHealthRequest(index).waitForYellowStatus().waitForEvents(Priority.LANGUID).waitForNoRelocatingShards(true)).actionGet();
+                .health(Requests.clusterHealthRequest(index).waitForYellowStatus().waitForEvents(Priority.LANGUID)
+                        .waitForNoRelocatingShards(true)).actionGet();
         assertThat(health.getStatus(), lessThanOrEqualTo(ClusterHealthStatus.YELLOW));
         assertThat("Cluster must be a single node cluster", health.getNumberOfDataNodes(), equalTo(1));
         IndicesService instanceFromNode = getInstanceFromNode(IndicesService.class);
@@ -277,8 +275,7 @@ public Index resolveIndex(String index) {
     protected SearchContext createSearchContext(IndexService indexService) {
         BigArrays bigArrays = indexService.getBigArrays();
         ThreadPool threadPool = indexService.getThreadPool();
-        ScriptService scriptService = node().injector().getInstance(ScriptService.class);
-        return new TestSearchContext(threadPool, bigArrays, scriptService, indexService);
+        return new TestSearchContext(threadPool, bigArrays, indexService);
     }
 
     /**
@@ -300,9 +297,11 @@ public ClusterHealthStatus ensureGreen(String... indices) {
      */
     public ClusterHealthStatus ensureGreen(TimeValue timeout, String... indices) {
         ClusterHealthResponse actionGet = client().admin().cluster()
-                .health(Requests.clusterHealthRequest(indices).timeout(timeout).waitForGreenStatus().waitForEvents(Priority.LANGUID).waitForNoRelocatingShards(true)).actionGet();
+                .health(Requests.clusterHealthRequest(indices).timeout(timeout).waitForGreenStatus().waitForEvents(Priority.LANGUID)
+                        .waitForNoRelocatingShards(true)).actionGet();
         if (actionGet.isTimedOut()) {
-            logger.info("ensureGreen timed out, cluster state:\n{}\n{}", client().admin().cluster().prepareState().get().getState().prettyPrint(), client().admin().cluster().preparePendingClusterTasks().get().prettyPrint());
+            logger.info("ensureGreen timed out, cluster state:\n{}\n{}", client().admin().cluster().prepareState().get().getState(),
+                client().admin().cluster().preparePendingClusterTasks().get());
             assertThat("timed out waiting for green state", actionGet.isTimedOut(), equalTo(false));
         }
         assertThat(actionGet.getStatus(), equalTo(ClusterHealthStatus.GREEN));
@@ -310,5 +309,8 @@ public ClusterHealthStatus ensureGreen(TimeValue timeout, String... indices) {
         return actionGet.getStatus();
     }
 
-
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return getInstanceFromNode(NamedXContentRegistry.class);
+    }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
index adc3e1ec2d279..75cb789075a0b 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
@@ -16,6 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+
 package org.elasticsearch.test;
 
 import com.carrotsearch.randomizedtesting.RandomizedTest;
@@ -25,11 +26,18 @@
 import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope.Scope;
 import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
 import com.carrotsearch.randomizedtesting.generators.CodepointSetGenerator;
-import com.carrotsearch.randomizedtesting.generators.RandomInts;
+import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
 import com.carrotsearch.randomizedtesting.generators.RandomStrings;
 import com.carrotsearch.randomizedtesting.rules.TestRuleAdapter;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.config.Configurator;
+import org.apache.logging.log4j.status.StatusConsoleListener;
+import org.apache.logging.log4j.status.StatusData;
+import org.apache.logging.log4j.status.StatusLogger;
 import org.apache.lucene.uninverting.UninvertingReader;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
@@ -39,23 +47,42 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.bootstrap.BootstrapForTesting;
 import org.elasticsearch.client.Requests;
+import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.CheckedRunnable;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.io.PathUtilsForTesting;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteable;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.MockBigArrays;
 import org.elasticsearch.common.util.MockPageCacheRecycler;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.analysis.AnalysisService;
+import org.elasticsearch.index.analysis.AnalysisRegistry;
+import org.elasticsearch.index.analysis.CharFilterFactory;
+import org.elasticsearch.index.analysis.IndexAnalyzers;
+import org.elasticsearch.index.analysis.TokenFilterFactory;
+import org.elasticsearch.index.analysis.TokenizerFactory;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MetadataFieldMapper;
 import org.elasticsearch.indices.IndicesModule;
@@ -64,8 +91,9 @@
 import org.elasticsearch.plugins.AnalysisPlugin;
 import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.script.MockScriptEngine;
+import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptModule;
-import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.MockSearchService;
 import org.elasticsearch.test.junit.listeners.LoggingListener;
 import org.elasticsearch.test.junit.listeners.ReproduceInfoPrinter;
@@ -80,6 +108,7 @@
 import org.junit.rules.RuleChain;
 
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.UncheckedIOException;
 import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
@@ -89,22 +118,27 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
 import java.util.Set;
-import java.util.TreeMap;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.TimeUnit;
 import java.util.function.BooleanSupplier;
 import java.util.function.Consumer;
 import java.util.function.Predicate;
 import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import static java.util.Collections.emptyList;
+import static java.util.Collections.emptyMap;
 import static java.util.Collections.singletonList;
 import static org.elasticsearch.common.util.CollectionUtils.arrayAsArrayList;
+import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasItem;
 
 /**
  * Base testcase for randomized unit testing with Elasticsearch
@@ -129,16 +163,21 @@ public abstract class ESTestCase extends LuceneTestCase {
 
     static {
         System.setProperty("log4j.shutdownHookEnabled", "false");
-        // we can not shutdown logging when tests are running or the next test that runs within the
-        // same JVM will try to initialize logging after a security manager has been installed and
-        // this will fail
-        System.setProperty("es.log4j.shutdownEnabled", "false");
         System.setProperty("log4j2.disable.jmx", "true");
         System.setProperty("log4j.skipJansi", "true"); // jython has this crazy shaded Jansi version that log4j2 tries to load
+
+        // shutdown hook so that when the test JVM exits, logging is shutdown too
+        Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+            LoggerContext context = (LoggerContext) LogManager.getContext(false);
+            Configurator.shutdown(context);
+        }));
+
         BootstrapForTesting.ensureInitialized();
     }
 
     protected final Logger logger = Loggers.getLogger(getClass());
+    protected final DeprecationLogger deprecationLogger = new DeprecationLogger(logger);
+    private ThreadContext threadContext;
 
     // -----------------------------------------------------------------
     // Suite and test case setup/cleanup.
@@ -206,17 +245,155 @@ public static void restoreContentType() {
         Requests.INDEX_CONTENT_TYPE = XContentType.JSON;
     }
 
+    @Before
+    public final void before()  {
+        logger.info("[{}]: before test", getTestName());
+        assertNull("Thread context initialized twice", threadContext);
+        if (enableWarningsCheck()) {
+            this.threadContext = new ThreadContext(Settings.EMPTY);
+            DeprecationLogger.setThreadContext(threadContext);
+        }
+    }
+
+    /**
+     * Whether or not we check after each test whether it has left warnings behind. That happens if any deprecated feature or syntax
+     * was used by the test and the test didn't assert on it using {@link #assertWarnings(String...)}.
+     */
+    protected boolean enableWarningsCheck() {
+        return true;
+    }
+
     @After
-    public final void ensureCleanedUp() throws Exception {
+    public final void after() throws Exception {
+        checkStaticState();
+        // We check threadContext != null rather than enableWarningsCheck()
+        // because after methods are still called in the event that before
+        // methods failed, in which case threadContext might not have been
+        // initialized
+        if (threadContext != null) {
+            ensureNoWarnings();
+            assert threadContext == null;
+        }
+        ensureAllSearchContextsReleased();
+        ensureCheckIndexPassed();
+        logger.info("[{}]: after test", getTestName());
+    }
+
+    private void ensureNoWarnings() throws IOException {
+        //Check that there are no unaccounted warning headers. These should be checked with {@link #assertWarnings(String...)} in the
+        //appropriate test
+        try {
+            final List<String> warnings = threadContext.getResponseHeaders().get("Warning");
+            assertNull("unexpected warning headers", warnings);
+        } finally {
+            resetDeprecationLogger(false);
+        }
+    }
+
+    /**
+     * Convenience method to assert warnings for settings deprecations and general deprecation warnings.
+     *
+     * @param settings the settings that are expected to be deprecated
+     * @param warnings other expected general deprecation warnings
+     */
+    protected final void assertSettingDeprecationsAndWarnings(final Setting<?>[] settings, final String... warnings) {
+        assertWarnings(
+                Stream.concat(
+                        Arrays
+                                .stream(settings)
+                                .map(Setting::getKey)
+                                .map(k -> "[" + k + "] setting was deprecated in Elasticsearch and will be removed in a future release! " +
+                                        "See the breaking changes documentation for the next major version."),
+                        Arrays.stream(warnings))
+                        .toArray(String[]::new));
+    }
+
+    protected final void assertWarnings(String... expectedWarnings) {
+        if (enableWarningsCheck() == false) {
+            throw new IllegalStateException("unable to check warning headers if the test is not set to do so");
+        }
+        try {
+            final List<String> actualWarnings = threadContext.getResponseHeaders().get("Warning");
+            assertNotNull(actualWarnings);
+            final Set<String> actualWarningValues =
+                    actualWarnings.stream().map(DeprecationLogger::extractWarningValueFromWarningHeader).collect(Collectors.toSet());
+            for (String msg : expectedWarnings) {
+                assertThat(actualWarningValues, hasItem(DeprecationLogger.escape(msg)));
+            }
+            assertEquals("Expected " + expectedWarnings.length + " warnings but found " + actualWarnings.size() + "\nExpected: "
+                + Arrays.asList(expectedWarnings) + "\nActual: " + actualWarnings, expectedWarnings.length, actualWarnings.size());
+        } finally {
+            resetDeprecationLogger(true);
+        }
+    }
+
+    /**
+     * Reset the deprecation logger by removing the current thread context, and setting a new thread context if {@code setNewThreadContext}
+     * is set to {@code true} and otherwise clearing the current thread context.
+     *
+     * @param setNewThreadContext whether or not to attach a new thread context to the deprecation logger
+     */
+    private void resetDeprecationLogger(final boolean setNewThreadContext) {
+        // "clear" current warning headers by setting a new ThreadContext
+        DeprecationLogger.removeThreadContext(this.threadContext);
+        try {
+            this.threadContext.close();
+            // catch IOException to avoid that call sites have to deal with it. It is only declared because this class implements Closeable
+            // but it is impossible that this implementation will ever throw an IOException.
+        } catch (IOException ex) {
+            throw new AssertionError("IOException thrown while closing deprecation logger's thread context", ex);
+        }
+        if (setNewThreadContext) {
+            this.threadContext = new ThreadContext(Settings.EMPTY);
+            DeprecationLogger.setThreadContext(this.threadContext);
+        } else {
+            this.threadContext = null;
+        }
+    }
+
+    private static final List<StatusData> statusData = new ArrayList<>();
+    static {
+        // ensure that the status logger is set to the warn level so we do not miss any warnings with our Log4j usage
+        StatusLogger.getLogger().setLevel(Level.WARN);
+        // Log4j will write out status messages indicating problems with the Log4j usage to the status logger; we hook into this logger and
+        // assert that no such messages were written out as these would indicate a problem with our logging configuration
+        StatusLogger.getLogger().registerListener(new StatusConsoleListener(Level.WARN) {
+
+            @Override
+            public void log(StatusData data) {
+                synchronized (statusData) {
+                    statusData.add(data);
+                }
+            }
+
+        });
+    }
+
+    // separate method so that this can be checked again after suite scoped cluster is shut down
+    protected static void checkStaticState() throws Exception {
         MockPageCacheRecycler.ensureAllPagesAreReleased();
         MockBigArrays.ensureAllArraysAreReleased();
         // field cache should NEVER get loaded.
         String[] entries = UninvertingReader.getUninvertedStats();
         assertEquals("fieldcache must never be used, got=" + Arrays.toString(entries), 0, entries.length);
+
+        // ensure no one changed the status logger level on us
+        assertThat(StatusLogger.getLogger().getLevel(), equalTo(Level.WARN));
+        synchronized (statusData) {
+            try {
+                // ensure that there are no status logger messages which would indicate a problem with our Log4j usage; we map the
+                // StatusData instances to Strings as otherwise their toString output is useless
+                assertThat(
+                    statusData.stream().map(status -> status.getMessage().getFormattedMessage()).collect(Collectors.toList()),
+                    empty());
+            } finally {
+                // we clear the list so that status data from other tests do not interfere with tests within the same JVM
+                statusData.clear();
+            }
+        }
     }
 
     // this must be a separate method from other ensure checks above so suite scoped integ tests can call...TODO: fix that
-    @After
     public final void ensureAllSearchContextsReleased() throws Exception {
         assertBusy(() -> MockSearchService.assertNoInFlightContext());
     }
@@ -232,7 +409,6 @@ public final void resetCheckIndexStatus() throws Exception {
         checkIndexFailed = false;
     }
 
-    @After
     public final void ensureCheckIndexPassed() throws Exception {
         assertFalse("at least one shard failed CheckIndex", checkIndexFailed);
     }
@@ -259,7 +435,7 @@ public static int scaledRandomIntBetween(int min, int max) {
      * @see #scaledRandomIntBetween(int, int)
      */
     public static int randomIntBetween(int min, int max) {
-        return RandomInts.randomIntBetween(random(), min, max);
+        return RandomNumbers.randomIntBetween(random(), min, max);
     }
 
     /**
@@ -303,7 +479,7 @@ public static int randomInt() {
         return random().nextInt();
     }
 
-    public static long randomPositiveLong() {
+    public static long randomNonNegativeLong() {
         long randomLong;
         do {
             randomLong = randomLong();
@@ -367,7 +543,6 @@ public static <T> T randomFrom(Random random, T... array) {
         return RandomPicks.randomFrom(random, array);
     }
 
-
     /** Pick a random object from the given list. */
     public static <T> T randomFrom(List<T> list) {
         return RandomPicks.randomFrom(random(), list);
@@ -375,14 +550,19 @@ public static <T> T randomFrom(List<T> list) {
 
     /** Pick a random object from the given collection. */
     public static <T> T randomFrom(Collection<T> collection) {
-        return RandomPicks.randomFrom(random(), collection);
+        return randomFrom(random(), collection);
+    }
+
+    /** Pick a random object from the given collection. */
+    public static <T> T randomFrom(Random random, Collection<T> collection) {
+        return RandomPicks.randomFrom(random, collection);
     }
 
-    public static String randomAsciiOfLengthBetween(int minCodeUnits, int maxCodeUnits) {
+    public static String randomAlphaOfLengthBetween(int minCodeUnits, int maxCodeUnits) {
         return RandomizedTest.randomAsciiOfLengthBetween(minCodeUnits, maxCodeUnits);
     }
 
-    public static String randomAsciiOfLength(int codeUnits) {
+    public static String randomAlphaOfLength(int codeUnits) {
         return RandomizedTest.randomAsciiOfLength(codeUnits);
     }
 
@@ -434,10 +614,14 @@ public static String[] generateRandomStringArray(int maxArraySize, int maxString
         return generateRandomStringArray(maxArraySize, maxStringSize, allowNull, true);
     }
 
-    private static String[] TIME_SUFFIXES = new String[]{"d", "h", "ms", "s", "m"};
+    private static final String[] TIME_SUFFIXES = new String[]{"d", "h", "ms", "s", "m", "micros", "nanos"};
+
+    public static String randomTimeValue(int lower, int upper, String[] suffixes) {
+        return randomIntBetween(lower, upper) + randomFrom(suffixes);
+    }
 
-    private static String randomTimeValue(int lower, int upper) {
-        return randomIntBetween(lower, upper) + randomFrom(TIME_SUFFIXES);
+    public static String randomTimeValue(int lower, int upper) {
+        return randomTimeValue(lower, upper, TIME_SUFFIXES);
     }
 
     public static String randomTimeValue() {
@@ -491,14 +675,14 @@ public static <T> T randomValueOtherThanMany(Predicate<T> input, Supplier<T> ran
     /**
      * Runs the code block for 10 seconds waiting for no assertion to trip.
      */
-    public static void assertBusy(Runnable codeBlock) throws Exception {
+    public static void assertBusy(CheckedRunnable<Exception> codeBlock) throws Exception {
         assertBusy(codeBlock, 10, TimeUnit.SECONDS);
     }
 
     /**
      * Runs the code block for the provided interval, waiting for no assertions to trip.
      */
-    public static void assertBusy(Runnable codeBlock, long maxWaitTime, TimeUnit unit) throws Exception {
+    public static void assertBusy(CheckedRunnable<Exception> codeBlock, long maxWaitTime, TimeUnit unit) throws Exception {
         long maxTimeInMillis = TimeUnit.MILLISECONDS.convert(maxWaitTime, unit);
         long iterations = Math.max(Math.round(Math.log10(maxTimeInMillis) / Math.log10(2)), 1);
         long timeInMillis = 1;
@@ -645,7 +829,7 @@ public static <T> List<T> randomSubsetOf(int size, T... values) {
      * Returns a random subset of values (including a potential empty list)
      */
     public static <T> List<T> randomSubsetOf(Collection<T> collection) {
-        return randomSubsetOf(randomInt(collection.size() - 1), collection);
+        return randomSubsetOf(randomInt(Math.max(collection.size() - 1, 0)), collection);
     }
 
     /**
@@ -653,7 +837,8 @@ public static <T> List<T> randomSubsetOf(Collection<T> collection) {
      */
     public static <T> List<T> randomSubsetOf(int size, Collection<T> collection) {
         if (size > collection.size()) {
-            throw new IllegalArgumentException("Can\'t pick " + size + " random objects from a collection of " + collection.size() + " objects");
+            throw new IllegalArgumentException("Can\'t pick " + size + " random objects from a collection of " +
+                    collection.size() + " objects");
         }
         List<T> tempList = new ArrayList<>(collection);
         Collections.shuffle(tempList, random());
@@ -691,35 +876,61 @@ public GeohashGenerator() {
         }
     }
 
+    /**
+     * Returns the bytes that represent the XContent output of the provided {@link ToXContent} object, using the provided
+     * {@link XContentType}. Wraps the output into a new anonymous object according to the value returned
+     * by the {@link ToXContent#isFragment()} method returns. Shuffles the keys to make sure that parsing never relies on keys ordering.
+     */
+    protected final BytesReference toShuffledXContent(ToXContent toXContent, XContentType xContentType, ToXContent.Params params,
+                                                      boolean humanReadable, String... exceptFieldNames) throws IOException{
+        BytesReference bytes = XContentHelper.toXContent(toXContent, xContentType, params, humanReadable);
+        try (XContentParser parser = createParser(xContentType.xContent(), bytes)) {
+            try (XContentBuilder builder = shuffleXContent(parser, rarely(), exceptFieldNames)) {
+                return builder.bytes();
+            }
+        }
+    }
+
     /**
      * Randomly shuffles the fields inside objects in the {@link XContentBuilder} passed in.
      * Recursively goes through inner objects and also shuffles them. Exceptions for this
      * recursive shuffling behavior can be made by passing in the names of fields which
      * internally should stay untouched.
      */
-    public static XContentBuilder shuffleXContent(XContentBuilder builder, String... exceptFieldNames) throws IOException {
-        BytesReference bytes = builder.bytes();
-        XContentParser parser = XContentFactory.xContent(bytes).createParser(bytes);
-        // use ordered maps for reproducibility
-        Map<String, Object> shuffledMap = shuffleMap(parser.mapOrdered(), new HashSet<>(Arrays.asList(exceptFieldNames)));
-        XContentBuilder xContentBuilder = XContentFactory.contentBuilder(builder.contentType());
-        if (builder.isPrettyPrint()) {
+    protected final XContentBuilder shuffleXContent(XContentBuilder builder, String... exceptFieldNames) throws IOException {
+        try (XContentParser parser = createParser(builder)) {
+            return shuffleXContent(parser, builder.isPrettyPrint(), exceptFieldNames);
+        }
+    }
+
+    /**
+     * Randomly shuffles the fields inside objects parsed using the {@link XContentParser} passed in.
+     * Recursively goes through inner objects and also shuffles them. Exceptions for this
+     * recursive shuffling behavior can be made by passing in the names of fields which
+     * internally should stay untouched.
+     */
+    protected static XContentBuilder shuffleXContent(XContentParser parser, boolean prettyPrint, String... exceptFieldNames)
+            throws IOException {
+        //we need a sorted map for reproducibility, as we are going to shuffle its keys and write XContent back
+        Map<String, Object> shuffledMap = shuffleMap((LinkedHashMap<String, Object>)parser.mapOrdered(),
+                new HashSet<>(Arrays.asList(exceptFieldNames)));
+        XContentBuilder xContentBuilder = XContentFactory.contentBuilder(parser.contentType());
+        if (prettyPrint) {
             xContentBuilder.prettyPrint();
         }
         return xContentBuilder.map(shuffledMap);
     }
 
-    private static Map<String, Object> shuffleMap(Map<String, Object> map, Set<String> exceptFields) {
+    public static LinkedHashMap<String, Object> shuffleMap(LinkedHashMap<String, Object> map, Set<String> exceptFields) {
         List<String> keys = new ArrayList<>(map.keySet());
-
-        // even though we shuffle later, we need this to make tests reproduce on different jvms
-        Collections.sort(keys);
-        Map<String, Object> targetMap = new TreeMap<>();
+        LinkedHashMap<String, Object> targetMap = new LinkedHashMap<>();
         Collections.shuffle(keys, random());
         for (String key : keys) {
             Object value = map.get(key);
             if (value instanceof Map && exceptFields.contains(key) == false) {
-                targetMap.put(key, shuffleMap((Map) value, exceptFields));
+                @SuppressWarnings("unchecked")
+                LinkedHashMap<String, Object> valueMap = (LinkedHashMap<String, Object>) value;
+                targetMap.put(key, shuffleMap(valueMap, exceptFields));
             } else {
                 targetMap.put(key, value);
             }
@@ -728,12 +939,19 @@ private static Map<String, Object> shuffleMap(Map<String, Object> map, Set<Strin
     }
 
     /**
-     * Returns true iff assertions for elasticsearch packages are enabled
+     * Create a copy of an original {@link Writeable} object by running it through a {@link BytesStreamOutput} and
+     * reading it in again using a provided {@link Writeable.Reader}. The stream that is wrapped around the {@link StreamInput}
+     * potentially need to use a {@link NamedWriteableRegistry}, so this needs to be provided too (although it can be
+     * empty if the object that is streamed doesn't contain any {@link NamedWriteable} objects itself.
      */
-    public static boolean assertionsEnabled() {
-        boolean enabled = false;
-        assert (enabled = true);
-        return enabled;
+    public static <T extends Writeable> T copyWriteable(T original, NamedWriteableRegistry namedWritabelRegistry,
+            Writeable.Reader<T> reader) throws IOException {
+        try (BytesStreamOutput output = new BytesStreamOutput()) {
+            original.writeTo(output);
+            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWritabelRegistry)) {
+                return reader.read(in);
+            }
+        }
     }
 
     public void assertAllIndicesRemovedAndDeletionCompleted(Iterable<IndicesService> indicesServices) throws Exception {
@@ -775,6 +993,56 @@ public void assertPathHasBeenCleared(Path path) {
         assertThat(count + " files exist that should have been cleaned:\n" + sb.toString(), count, equalTo(0));
     }
 
+    /**
+     * Create a new {@link XContentParser}.
+     */
+    protected final XContentParser createParser(XContentBuilder builder) throws IOException {
+        return builder.generator().contentType().xContent().createParser(xContentRegistry(), builder.bytes());
+    }
+
+    /**
+     * Create a new {@link XContentParser}.
+     */
+    protected final XContentParser createParser(XContent xContent, String data) throws IOException {
+        return xContent.createParser(xContentRegistry(), data);
+    }
+
+    /**
+     * Create a new {@link XContentParser}.
+     */
+    protected final XContentParser createParser(XContent xContent, InputStream data) throws IOException {
+        return xContent.createParser(xContentRegistry(), data);
+    }
+
+    /**
+     * Create a new {@link XContentParser}.
+     */
+    protected final XContentParser createParser(XContent xContent, byte[] data) throws IOException {
+        return xContent.createParser(xContentRegistry(), data);
+    }
+
+    /**
+     * Create a new {@link XContentParser}.
+     */
+    protected final XContentParser createParser(XContent xContent, BytesReference data) throws IOException {
+        return xContent.createParser(xContentRegistry(), data);
+    }
+
+    /**
+     * The {@link NamedXContentRegistry} to use for this test. Subclasses should override and use liberally.
+     */
+    protected NamedXContentRegistry xContentRegistry() {
+        return new NamedXContentRegistry(ClusterModule.getNamedXWriteables());
+    }
+
+    /**
+     * Create a "mock" script for use either with {@link MockScriptEngine} or anywhere where you need a script but don't really care about
+     * its contents.
+     */
+    public static final Script mockScript(String id) {
+        return new Script(ScriptType.INLINE, MockScriptEngine.NAME, id, emptyMap());
+    }
+
     /** Returns the suite failure marker: internal use only! */
     public static TestRuleMarkFailure getSuiteFailureMarker() {
         return suiteFailureMarker;
@@ -798,9 +1066,15 @@ public static void assertEquals(StackTraceElement expected, StackTraceElement ac
     }
 
     protected static long spinForAtLeastOneMillisecond() {
-        long nanosecondsInMillisecond = TimeUnit.NANOSECONDS.convert(1, TimeUnit.MILLISECONDS);
-        // force at least one millisecond to elapse, but ensure the
-        // clock has enough resolution to observe the passage of time
+        return spinForAtLeastNMilliseconds(1);
+    }
+
+    protected static long spinForAtLeastNMilliseconds(final long ms) {
+        long nanosecondsInMillisecond = TimeUnit.NANOSECONDS.convert(ms, TimeUnit.MILLISECONDS);
+        /*
+         * Force at least ms milliseconds to elapse, but ensure the clock has enough resolution to
+         * observe the passage of time.
+         */
         long start = System.nanoTime();
         long elapsed;
         while ((elapsed = (System.nanoTime() - start)) < nanosecondsInMillisecond) {
@@ -810,42 +1084,42 @@ protected static long spinForAtLeastOneMillisecond() {
     }
 
     /**
-     * Creates an AnalysisService with all the default analyzers configured.
+     * Creates an TestAnalysis with all the default analyzers configured.
      */
-    public static AnalysisService createAnalysisService(Index index, Settings settings, AnalysisPlugin... analysisPlugins)
+    public static TestAnalysis createTestAnalysis(Index index, Settings settings, AnalysisPlugin... analysisPlugins)
             throws IOException {
         Settings nodeSettings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build();
-        return createAnalysisService(index, nodeSettings, settings, analysisPlugins);
+        return createTestAnalysis(index, nodeSettings, settings, analysisPlugins);
     }
 
     /**
-     * Creates an AnalysisService with all the default analyzers configured.
+     * Creates an TestAnalysis with all the default analyzers configured.
      */
-    public static AnalysisService createAnalysisService(Index index, Settings nodeSettings, Settings settings,
-            AnalysisPlugin... analysisPlugins) throws IOException {
+    public static TestAnalysis createTestAnalysis(Index index, Settings nodeSettings, Settings settings,
+                                                  AnalysisPlugin... analysisPlugins) throws IOException {
         Settings indexSettings = Settings.builder().put(settings)
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
                 .build();
-        return createAnalysisService(IndexSettingsModule.newIndexSettings(index, indexSettings), nodeSettings, analysisPlugins);
+        return createTestAnalysis(IndexSettingsModule.newIndexSettings(index, indexSettings), nodeSettings, analysisPlugins);
     }
 
     /**
-     * Creates an AnalysisService with all the default analyzers configured.
+     * Creates an TestAnalysis with all the default analyzers configured.
      */
-    public static AnalysisService createAnalysisService(IndexSettings indexSettings, Settings nodeSettings,
-            AnalysisPlugin... analysisPlugins) throws IOException {
+    public static TestAnalysis createTestAnalysis(IndexSettings indexSettings, Settings nodeSettings,
+                                                  AnalysisPlugin... analysisPlugins) throws IOException {
         Environment env = new Environment(nodeSettings);
         AnalysisModule analysisModule = new AnalysisModule(env, Arrays.asList(analysisPlugins));
-        final AnalysisService analysisService = analysisModule.getAnalysisRegistry()
-                .build(indexSettings);
-        return analysisService;
+        AnalysisRegistry analysisRegistry = analysisModule.getAnalysisRegistry();
+        return new TestAnalysis(analysisRegistry.build(indexSettings),
+            analysisRegistry.buildTokenFilterFactories(indexSettings),
+            analysisRegistry.buildTokenizerFactories(indexSettings),
+            analysisRegistry.buildCharFilterFactories(indexSettings));
     }
 
     public static ScriptModule newTestScriptModule() {
         Settings settings = Settings.builder()
                 .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
-                // no file watching, so we don't need a ResourceWatcherService
-                .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), false)
                 .build();
         Environment environment = new Environment(settings);
         MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, Collections.singletonMap("1", script -> "1"));
@@ -868,4 +1142,28 @@ public Map<String, MetadataFieldMapper.TypeParser> getMetadataMappers() {
             }
         ));
     }
+
+    /**
+     * This cute helper class just holds all analysis building blocks that are used
+     * to build IndexAnalyzers. This is only for testing since in production we only need the
+     * result and we don't even expose it there.
+     */
+    public static final class TestAnalysis {
+
+        public final IndexAnalyzers indexAnalyzers;
+        public final Map<String, TokenFilterFactory> tokenFilter;
+        public final Map<String, TokenizerFactory> tokenizer;
+        public final Map<String, CharFilterFactory> charFilter;
+
+        public TestAnalysis(IndexAnalyzers indexAnalyzers,
+                            Map<String, TokenFilterFactory> tokenFilter,
+                            Map<String, TokenizerFactory> tokenizer,
+                            Map<String, CharFilterFactory> charFilter) {
+            this.indexAnalyzers = indexAnalyzers;
+            this.tokenFilter = tokenFilter;
+            this.tokenizer = tokenizer;
+            this.charFilter = charFilter;
+        }
+    }
+
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/EqualsHashCodeTestUtils.java b/test/framework/src/main/java/org/elasticsearch/test/EqualsHashCodeTestUtils.java
new file mode 100644
index 0000000000000..76cfcce033b11
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/test/EqualsHashCodeTestUtils.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test;
+
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Utility class that encapsulates standard checks and assertions around testing the equals() and hashCode()
+ * methods of objects that implement them.
+ */
+public class EqualsHashCodeTestUtils {
+
+    private static Object[] someObjects = new Object[] { "some string", new Integer(1), new Double(1.0) };
+
+    /**
+     * A function that makes a copy of its input argument
+     */
+    public interface CopyFunction<T> {
+        T copy(T t) throws IOException;
+    };
+
+    /**
+     * A function that creates a copy of its input argument that is different from its
+     * input in exactly one aspect (e.g. one parameter of a class instance should change)
+     */
+    public interface MutateFunction<T> {
+        T mutate(T t) throws IOException;
+    };
+
+    /**
+     * Perform common equality and hashCode checks on the input object
+     * @param original the object under test
+     * @param copyFunction a function that creates a deep copy of the input object
+     */
+    public static <T> void checkEqualsAndHashCode(T original, CopyFunction<T> copyFunction) {
+        checkEqualsAndHashCode(original, copyFunction, null);
+    }
+
+    /**
+     * Perform common equality and hashCode checks on the input object
+     * @param original the object under test
+     * @param copyFunction a function that creates a deep copy of the input object
+     * @param mutationFunction a function that creates a copy of the input object that is different
+     * from the input in one aspect. The output of this call is used to check that it is not equal()
+     * to the input object
+     */
+    public static <T> void checkEqualsAndHashCode(T original, CopyFunction<T> copyFunction,
+            MutateFunction<T> mutationFunction) {
+        try {
+            String objectName = original.getClass().getSimpleName();
+            assertFalse(objectName + " is equal to null", original.equals(null));
+            // TODO not sure how useful the following test is
+            assertFalse(objectName + " is equal to incompatible type", original.equals(ESTestCase.randomFrom(someObjects)));
+            assertTrue(objectName + " is not equal to self", original.equals(original));
+            assertThat(objectName + " hashcode returns different values if called multiple times", original.hashCode(),
+                    equalTo(original.hashCode()));
+            if (mutationFunction != null) {
+                T mutation = mutationFunction.mutate(original);
+                assertThat(objectName + " mutation should not be equal to original", mutation,
+                        not(equalTo(original)));
+            }
+
+            T copy = copyFunction.copy(original);
+            assertTrue(objectName + " copy is not equal to self", copy.equals(copy));
+            assertTrue(objectName + " is not equal to its copy", original.equals(copy));
+            assertTrue("equals is not symmetric", copy.equals(original));
+            assertThat(objectName + " hashcode is different from copies hashcode", copy.hashCode(), equalTo(original.hashCode()));
+
+            T secondCopy = copyFunction.copy(copy);
+            assertTrue("second copy is not equal to self", secondCopy.equals(secondCopy));
+            assertTrue("copy is not equal to its second copy", copy.equals(secondCopy));
+            assertThat("second copy's hashcode is different from original hashcode", copy.hashCode(), equalTo(secondCopy.hashCode()));
+            assertTrue("equals is not transitive", original.equals(secondCopy));
+            assertTrue("equals is not symmetric", secondCopy.equals(copy));
+            assertTrue("equals is not symmetric", secondCopy.equals(original));
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ExternalNode.java b/test/framework/src/main/java/org/elasticsearch/test/ExternalNode.java
index 8725ed815ad76..2e8001bf0f40c 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ExternalNode.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ExternalNode.java
@@ -109,7 +109,6 @@ synchronized void startInternal(Client client, Settings settings, String nodeNam
                 case "path.home":
                 case NetworkModule.TRANSPORT_TYPE_KEY:
                 case "discovery.type":
-                case NetworkModule.TRANSPORT_SERVICE_TYPE_KEY:
                 case "config.ignore_system_properties":
                     continue;
                 default:
diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java
new file mode 100644
index 0000000000000..cb005b3861a4d
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java
@@ -0,0 +1,312 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test;
+
+import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.ContextParser;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParserUtils;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.rest.action.search.RestSearchAction;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.Aggregation;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.ParsedAggregation;
+import org.elasticsearch.search.aggregations.bucket.adjacency.AdjacencyMatrixAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.adjacency.ParsedAdjacencyMatrix;
+import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.filter.ParsedFilter;
+import org.elasticsearch.search.aggregations.bucket.filters.FiltersAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.filters.ParsedFilters;
+import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.geogrid.ParsedGeoHashGrid;
+import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.global.ParsedGlobal;
+import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.histogram.ParsedDateHistogram;
+import org.elasticsearch.search.aggregations.bucket.histogram.ParsedHistogram;
+import org.elasticsearch.search.aggregations.bucket.missing.MissingAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.missing.ParsedMissing;
+import org.elasticsearch.search.aggregations.bucket.nested.NestedAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.nested.ParsedNested;
+import org.elasticsearch.search.aggregations.bucket.nested.ParsedReverseNested;
+import org.elasticsearch.search.aggregations.bucket.nested.ReverseNestedAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.range.ParsedBinaryRange;
+import org.elasticsearch.search.aggregations.bucket.range.ParsedRange;
+import org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.range.date.DateRangeAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.range.date.ParsedDateRange;
+import org.elasticsearch.search.aggregations.bucket.range.geodistance.GeoDistanceAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.range.geodistance.ParsedGeoDistance;
+import org.elasticsearch.search.aggregations.bucket.range.ip.IpRangeAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.sampler.InternalSampler;
+import org.elasticsearch.search.aggregations.bucket.sampler.ParsedSampler;
+import org.elasticsearch.search.aggregations.bucket.significant.ParsedSignificantLongTerms;
+import org.elasticsearch.search.aggregations.bucket.significant.ParsedSignificantStringTerms;
+import org.elasticsearch.search.aggregations.bucket.significant.SignificantLongTerms;
+import org.elasticsearch.search.aggregations.bucket.significant.SignificantStringTerms;
+import org.elasticsearch.search.aggregations.bucket.terms.DoubleTerms;
+import org.elasticsearch.search.aggregations.bucket.terms.LongTerms;
+import org.elasticsearch.search.aggregations.bucket.terms.ParsedDoubleTerms;
+import org.elasticsearch.search.aggregations.bucket.terms.ParsedLongTerms;
+import org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms;
+import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
+import org.elasticsearch.search.aggregations.metrics.avg.AvgAggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.avg.ParsedAvg;
+import org.elasticsearch.search.aggregations.metrics.cardinality.CardinalityAggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.cardinality.ParsedCardinality;
+import org.elasticsearch.search.aggregations.metrics.geobounds.GeoBoundsAggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.geobounds.ParsedGeoBounds;
+import org.elasticsearch.search.aggregations.metrics.geocentroid.GeoCentroidAggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.geocentroid.ParsedGeoCentroid;
+import org.elasticsearch.search.aggregations.metrics.max.MaxAggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.max.ParsedMax;
+import org.elasticsearch.search.aggregations.metrics.min.MinAggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.min.ParsedMin;
+import org.elasticsearch.search.aggregations.metrics.percentiles.hdr.InternalHDRPercentileRanks;
+import org.elasticsearch.search.aggregations.metrics.percentiles.hdr.InternalHDRPercentiles;
+import org.elasticsearch.search.aggregations.metrics.percentiles.hdr.ParsedHDRPercentileRanks;
+import org.elasticsearch.search.aggregations.metrics.percentiles.hdr.ParsedHDRPercentiles;
+import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.InternalTDigestPercentileRanks;
+import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.InternalTDigestPercentiles;
+import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.ParsedTDigestPercentileRanks;
+import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.ParsedTDigestPercentiles;
+import org.elasticsearch.search.aggregations.metrics.scripted.ParsedScriptedMetric;
+import org.elasticsearch.search.aggregations.metrics.scripted.ScriptedMetricAggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.stats.ParsedStats;
+import org.elasticsearch.search.aggregations.metrics.stats.StatsAggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStatsAggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.stats.extended.ParsedExtendedStats;
+import org.elasticsearch.search.aggregations.metrics.sum.ParsedSum;
+import org.elasticsearch.search.aggregations.metrics.sum.SumAggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.tophits.ParsedTopHits;
+import org.elasticsearch.search.aggregations.metrics.tophits.TopHitsAggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.valuecount.ParsedValueCount;
+import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCountAggregationBuilder;
+import org.elasticsearch.search.aggregations.pipeline.InternalSimpleValue;
+import org.elasticsearch.search.aggregations.pipeline.ParsedSimpleValue;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.InternalBucketMetricValue;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.ParsedBucketMetricValue;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.percentile.ParsedPercentilesBucket;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.percentile.PercentilesBucketPipelineAggregationBuilder;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.ParsedStatsBucket;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.StatsBucketPipelineAggregationBuilder;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.extended.ExtendedStatsBucketPipelineAggregationBuilder;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.extended.ParsedExtendedStatsBucket;
+import org.elasticsearch.search.aggregations.pipeline.derivative.DerivativePipelineAggregationBuilder;
+import org.elasticsearch.search.aggregations.pipeline.derivative.ParsedDerivative;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Predicate;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+
+import static java.util.Collections.singletonMap;
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+import static org.elasticsearch.test.XContentTestUtils.insertRandomFields;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
+
+public abstract class InternalAggregationTestCase<T extends InternalAggregation> extends ESTestCase {
+
+    private final NamedXContentRegistry namedXContentRegistry = new NamedXContentRegistry(getNamedXContents());
+
+    private static final List<NamedXContentRegistry.Entry> namedXContents;
+    static {
+        Map<String, ContextParser<Object, ? extends Aggregation>> map = new HashMap<>();
+        map.put(CardinalityAggregationBuilder.NAME, (p, c) -> ParsedCardinality.fromXContent(p, (String) c));
+        map.put(InternalHDRPercentiles.NAME, (p, c) -> ParsedHDRPercentiles.fromXContent(p, (String) c));
+        map.put(InternalHDRPercentileRanks.NAME, (p, c) -> ParsedHDRPercentileRanks.fromXContent(p, (String) c));
+        map.put(InternalTDigestPercentiles.NAME, (p, c) -> ParsedTDigestPercentiles.fromXContent(p, (String) c));
+        map.put(InternalTDigestPercentileRanks.NAME, (p, c) -> ParsedTDigestPercentileRanks.fromXContent(p, (String) c));
+        map.put(PercentilesBucketPipelineAggregationBuilder.NAME, (p, c) -> ParsedPercentilesBucket.fromXContent(p, (String) c));
+        map.put(MinAggregationBuilder.NAME, (p, c) -> ParsedMin.fromXContent(p, (String) c));
+        map.put(MaxAggregationBuilder.NAME, (p, c) -> ParsedMax.fromXContent(p, (String) c));
+        map.put(SumAggregationBuilder.NAME, (p, c) -> ParsedSum.fromXContent(p, (String) c));
+        map.put(AvgAggregationBuilder.NAME, (p, c) -> ParsedAvg.fromXContent(p, (String) c));
+        map.put(ValueCountAggregationBuilder.NAME, (p, c) -> ParsedValueCount.fromXContent(p, (String) c));
+        map.put(InternalSimpleValue.NAME, (p, c) -> ParsedSimpleValue.fromXContent(p, (String) c));
+        map.put(DerivativePipelineAggregationBuilder.NAME, (p, c) -> ParsedDerivative.fromXContent(p, (String) c));
+        map.put(InternalBucketMetricValue.NAME, (p, c) -> ParsedBucketMetricValue.fromXContent(p, (String) c));
+        map.put(StatsAggregationBuilder.NAME, (p, c) -> ParsedStats.fromXContent(p, (String) c));
+        map.put(StatsBucketPipelineAggregationBuilder.NAME, (p, c) -> ParsedStatsBucket.fromXContent(p, (String) c));
+        map.put(ExtendedStatsAggregationBuilder.NAME, (p, c) -> ParsedExtendedStats.fromXContent(p, (String) c));
+        map.put(ExtendedStatsBucketPipelineAggregationBuilder.NAME, (p, c) -> ParsedExtendedStatsBucket.fromXContent(p, (String) c));
+        map.put(GeoBoundsAggregationBuilder.NAME, (p, c) -> ParsedGeoBounds.fromXContent(p, (String) c));
+        map.put(GeoCentroidAggregationBuilder.NAME, (p, c) -> ParsedGeoCentroid.fromXContent(p, (String) c));
+        map.put(HistogramAggregationBuilder.NAME, (p, c) -> ParsedHistogram.fromXContent(p, (String) c));
+        map.put(DateHistogramAggregationBuilder.NAME, (p, c) -> ParsedDateHistogram.fromXContent(p, (String) c));
+        map.put(StringTerms.NAME, (p, c) -> ParsedStringTerms.fromXContent(p, (String) c));
+        map.put(LongTerms.NAME, (p, c) -> ParsedLongTerms.fromXContent(p, (String) c));
+        map.put(DoubleTerms.NAME, (p, c) -> ParsedDoubleTerms.fromXContent(p, (String) c));
+        map.put(MissingAggregationBuilder.NAME, (p, c) -> ParsedMissing.fromXContent(p, (String) c));
+        map.put(NestedAggregationBuilder.NAME, (p, c) -> ParsedNested.fromXContent(p, (String) c));
+        map.put(ReverseNestedAggregationBuilder.NAME, (p, c) -> ParsedReverseNested.fromXContent(p, (String) c));
+        map.put(GlobalAggregationBuilder.NAME, (p, c) -> ParsedGlobal.fromXContent(p, (String) c));
+        map.put(FilterAggregationBuilder.NAME, (p, c) -> ParsedFilter.fromXContent(p, (String) c));
+        map.put(InternalSampler.PARSER_NAME, (p, c) -> ParsedSampler.fromXContent(p, (String) c));
+        map.put(GeoGridAggregationBuilder.NAME, (p, c) -> ParsedGeoHashGrid.fromXContent(p, (String) c));
+        map.put(RangeAggregationBuilder.NAME, (p, c) -> ParsedRange.fromXContent(p, (String) c));
+        map.put(DateRangeAggregationBuilder.NAME, (p, c) -> ParsedDateRange.fromXContent(p, (String) c));
+        map.put(GeoDistanceAggregationBuilder.NAME, (p, c) -> ParsedGeoDistance.fromXContent(p, (String) c));
+        map.put(FiltersAggregationBuilder.NAME, (p, c) -> ParsedFilters.fromXContent(p, (String) c));
+        map.put(AdjacencyMatrixAggregationBuilder.NAME, (p, c) -> ParsedAdjacencyMatrix.fromXContent(p, (String) c));
+        map.put(SignificantLongTerms.NAME, (p, c) -> ParsedSignificantLongTerms.fromXContent(p, (String) c));
+        map.put(SignificantStringTerms.NAME, (p, c) -> ParsedSignificantStringTerms.fromXContent(p, (String) c));
+        map.put(ScriptedMetricAggregationBuilder.NAME, (p, c) -> ParsedScriptedMetric.fromXContent(p, (String) c));
+        map.put(IpRangeAggregationBuilder.NAME, (p, c) -> ParsedBinaryRange.fromXContent(p, (String) c));
+        map.put(TopHitsAggregationBuilder.NAME, (p, c) -> ParsedTopHits.fromXContent(p, (String) c));
+
+        namedXContents = map.entrySet().stream()
+                .map(entry -> new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(entry.getKey()), entry.getValue()))
+                .collect(Collectors.toList());
+    }
+
+    public static List<NamedXContentRegistry.Entry> getDefaultNamedXContents() {
+        return namedXContents;
+    }
+
+    protected List<NamedXContentRegistry.Entry> getNamedXContents() {
+        return namedXContents;
+    }
+
+    protected abstract T createTestInstance(String name, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData);
+
+    public final T createTestInstance() {
+        String name = randomAlphaOfLength(5);
+        List<PipelineAggregator> pipelineAggregators = new ArrayList<>();
+        // TODO populate pipelineAggregators
+        Map<String, Object> metaData = null;
+        if (randomBoolean()) {
+            metaData = new HashMap<>();
+            int metaDataCount = between(0, 10);
+            while (metaData.size() < metaDataCount) {
+                metaData.put(randomAlphaOfLength(5), randomAlphaOfLength(5));
+            }
+        }
+        return createTestInstance(name, pipelineAggregators, metaData);
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return namedXContentRegistry;
+    }
+
+    public final void testFromXContent() throws IOException {
+        final T aggregation = createTestInstance();
+        final Aggregation parsedAggregation = parseAndAssert(aggregation, randomBoolean(), false);
+        assertFromXContent(aggregation, (ParsedAggregation) parsedAggregation);
+    }
+
+    public final void testFromXContentWithRandomFields() throws IOException {
+        final T aggregation = createTestInstance();
+        final Aggregation parsedAggregation = parseAndAssert(aggregation, randomBoolean(), true);
+        assertFromXContent(aggregation, (ParsedAggregation) parsedAggregation);
+    }
+
+    protected abstract void assertFromXContent(T aggregation, ParsedAggregation parsedAggregation) throws IOException;
+
+    @SuppressWarnings("unchecked")
+    protected <P extends ParsedAggregation> P parseAndAssert(final InternalAggregation aggregation,
+                                                             final boolean shuffled, final boolean addRandomFields) throws IOException {
+
+        final ToXContent.Params params = new ToXContent.MapParams(singletonMap(RestSearchAction.TYPED_KEYS_PARAM, "true"));
+        final XContentType xContentType = randomFrom(XContentType.values());
+        final boolean humanReadable = randomBoolean();
+
+        final BytesReference originalBytes;
+        if (shuffled) {
+            originalBytes = toShuffledXContent(aggregation, xContentType, params, humanReadable);
+        } else {
+            originalBytes = toXContent(aggregation, xContentType, params, humanReadable);
+        }
+        BytesReference mutated;
+        if (addRandomFields) {
+            /*
+             * - we don't add to the root object because it should only contain
+             * the named aggregation to test - we don't want to insert into the
+             * "meta" object, because we pass on everything we find there
+             *
+             * - we don't want to directly insert anything random into "buckets"
+             * objects, they are used with "keyed" aggregations and contain
+             * named bucket objects. Any new named object on this level should
+             * also be a bucket and be parsed as such.
+             */
+            Predicate<String> basicExcludes = path -> path.isEmpty() || path.endsWith(Aggregation.CommonFields.META.getPreferredName())
+                    || path.endsWith(Aggregation.CommonFields.BUCKETS.getPreferredName());
+            Predicate<String> excludes = basicExcludes.or(excludePathsFromXContentInsertion());
+            mutated = insertRandomFields(xContentType, originalBytes, excludes, random());
+        } else {
+            mutated = originalBytes;
+        }
+
+        SetOnce<Aggregation> parsedAggregation = new SetOnce<>();
+        try (XContentParser parser = createParser(xContentType.xContent(), mutated)) {
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            XContentParserUtils.parseTypedKeysObject(parser, Aggregation.TYPED_KEYS_DELIMITER, Aggregation.class, parsedAggregation::set);
+
+            assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());
+            assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
+            assertNull(parser.nextToken());
+
+            Aggregation agg = parsedAggregation.get();
+            assertEquals(aggregation.getName(), agg.getName());
+            assertEquals(aggregation.getMetaData(), agg.getMetaData());
+
+            assertTrue(agg instanceof ParsedAggregation);
+            assertEquals(aggregation.getType(), agg.getType());
+
+            BytesReference parsedBytes = toXContent(agg, xContentType, params, humanReadable);
+            assertToXContentEquivalent(originalBytes, parsedBytes, xContentType);
+
+            return (P) agg;
+        }
+
+    }
+
+    /**
+     * Overwrite this in your test if other than the basic xContent paths should be excluded during insertion of random fields
+     */
+    protected Predicate<String> excludePathsFromXContentInsertion() {
+        return path -> false;
+    }
+
+    /**
+     * @return a random {@link DocValueFormat} that can be used in aggregations which
+     * compute numbers.
+     */
+    protected static DocValueFormat randomNumericDocValueFormat() {
+        final List<Supplier<DocValueFormat>> formats = new ArrayList<>(3);
+        formats.add(() -> DocValueFormat.RAW);
+        formats.add(() -> new DocValueFormat.Decimal(randomFrom("###.##", "###,###.##")));
+        return randomFrom(formats).get();
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalSettingsPlugin.java b/test/framework/src/main/java/org/elasticsearch/test/InternalSettingsPlugin.java
index f46069484d6ce..a5b1667a13ff0 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/InternalSettingsPlugin.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/InternalSettingsPlugin.java
@@ -21,7 +21,6 @@
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
-import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.plugins.Plugin;
 
 import java.util.Arrays;
@@ -31,6 +30,8 @@ public final class InternalSettingsPlugin extends Plugin {
 
     public static final Setting<Integer> VERSION_CREATED =
         Setting.intSetting("index.version.created", 0, Property.IndexScope, Property.NodeScope);
+    public static final Setting<String> PROVIDED_NAME_SETTING =
+        Setting.simpleString("index.provided_name",Property.IndexScope, Property.NodeScope);
     public static final Setting<Boolean> MERGE_ENABLED =
         Setting.boolSetting("index.merge.enabled", true, Property.IndexScope, Property.NodeScope);
     public static final Setting<Long> INDEX_CREATION_DATE_SETTING =
@@ -38,6 +39,7 @@ public final class InternalSettingsPlugin extends Plugin {
 
     @Override
     public List<Setting<?>> getSettings() {
-        return Arrays.asList(VERSION_CREATED, MERGE_ENABLED, INDEX_CREATION_DATE_SETTING);
+        return Arrays.asList(VERSION_CREATED, MERGE_ENABLED,
+            INDEX_CREATION_DATE_SETTING, PROVIDED_NAME_SETTING);
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
index 8a3274ec727b5..a204fafb812b4 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
@@ -21,18 +21,18 @@
 import com.carrotsearch.randomizedtesting.RandomizedTest;
 import com.carrotsearch.randomizedtesting.SeedUtils;
 import com.carrotsearch.randomizedtesting.SysGlobals;
-import com.carrotsearch.randomizedtesting.generators.RandomInts;
+import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
 import com.carrotsearch.randomizedtesting.generators.RandomStrings;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.store.StoreRateLimiting;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.Version;
-import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
+import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags.Flag;
+import org.elasticsearch.action.support.replication.ReplicationTask;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.cluster.ClusterName;
@@ -54,6 +54,7 @@
 import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.network.NetworkModule;
+import org.elasticsearch.common.settings.SecureSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.Settings.Builder;
 import org.elasticsearch.common.transport.InetSocketTransportAddress;
@@ -64,7 +65,11 @@
 import org.elasticsearch.common.util.PageCacheRecycler;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.discovery.DiscoverySettings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.discovery.DiscoveryModule;
+import org.elasticsearch.discovery.zen.ElectMasterService;
+import org.elasticsearch.discovery.zen.ZenDiscovery;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.env.ShardLockObtainFailedException;
@@ -83,10 +88,13 @@
 import org.elasticsearch.indices.recovery.RecoverySettings;
 import org.elasticsearch.node.MockNode;
 import org.elasticsearch.node.Node;
-import org.elasticsearch.node.service.NodeService;
+import org.elasticsearch.node.NodeService;
+import org.elasticsearch.node.NodeValidationException;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.SearchService;
+import org.elasticsearch.tasks.TaskInfo;
+import org.elasticsearch.tasks.TaskManager;
 import org.elasticsearch.test.disruption.ServiceDisruptionScheme;
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.transport.MockTransportClient;
@@ -98,6 +106,7 @@
 
 import java.io.Closeable;
 import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.net.InetSocketAddress;
 import java.nio.file.Path;
 import java.util.ArrayList;
@@ -126,8 +135,12 @@
 
 import static org.apache.lucene.util.LuceneTestCase.TEST_NIGHTLY;
 import static org.apache.lucene.util.LuceneTestCase.rarely;
+import static org.elasticsearch.discovery.DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING;
+import static org.elasticsearch.discovery.zen.ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING;
 import static org.elasticsearch.test.ESTestCase.assertBusy;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoTimeout;
+import static org.elasticsearch.test.ESTestCase.awaitBusy;
+import static org.elasticsearch.test.ESTestCase.randomFrom;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
@@ -218,6 +231,8 @@ public final class InternalTestCluster extends TestCluster {
 
     private final ExecutorService executor;
 
+    private final boolean autoManageMinMasterNodes;
+
     private final Collection<Class<? extends Plugin>> mockPlugins;
 
     /**
@@ -231,9 +246,10 @@ public final class InternalTestCluster extends TestCluster {
 
     public InternalTestCluster(long clusterSeed, Path baseDir,
                                boolean randomlyAddDedicatedMasters,
-                               int minNumDataNodes, int maxNumDataNodes, String clusterName, NodeConfigurationSource nodeConfigurationSource, int numClientNodes,
+                               boolean autoManageMinMasterNodes, int minNumDataNodes, int maxNumDataNodes, String clusterName, NodeConfigurationSource nodeConfigurationSource, int numClientNodes,
                                boolean enableHttpPipelining, String nodePrefix, Collection<Class<? extends Plugin>> mockPlugins, Function<Client, Client> clientWrapper) {
         super(clusterSeed);
+        this.autoManageMinMasterNodes = autoManageMinMasterNodes;
         this.clientWrapper = clientWrapper;
         this.baseDir = baseDir;
         this.clusterName = clusterName;
@@ -249,7 +265,7 @@ public InternalTestCluster(long clusterSeed, Path baseDir,
 
         boolean useDedicatedMasterNodes = randomlyAddDedicatedMasters ? random.nextBoolean() : false;
 
-        this.numSharedDataNodes = RandomInts.randomIntBetween(random, minNumDataNodes, maxNumDataNodes);
+        this.numSharedDataNodes = RandomNumbers.randomIntBetween(random, minNumDataNodes, maxNumDataNodes);
         assert this.numSharedDataNodes >= 0;
 
         if (numSharedDataNodes == 0) {
@@ -267,7 +283,7 @@ public InternalTestCluster(long clusterSeed, Path baseDir,
                 this.numSharedDedicatedMasterNodes = 0;
             }
             if (numClientNodes < 0) {
-                this.numSharedCoordOnlyNodes = RandomInts.randomIntBetween(random, DEFAULT_MIN_NUM_CLIENT_NODES, DEFAULT_MAX_NUM_CLIENT_NODES);
+                this.numSharedCoordOnlyNodes = RandomNumbers.randomIntBetween(random, DEFAULT_MIN_NUM_CLIENT_NODES, DEFAULT_MAX_NUM_CLIENT_NODES);
             } else {
                 this.numSharedCoordOnlyNodes = numClientNodes;
             }
@@ -285,9 +301,10 @@ public InternalTestCluster(long clusterSeed, Path baseDir,
         }
 
         logger.info("Setup InternalTestCluster [{}] with seed [{}] using [{}] dedicated masters, " +
-                "[{}] (data) nodes and [{}] coord only nodes",
+                "[{}] (data) nodes and [{}] coord only nodes (min_master_nodes are [{}])",
             clusterName, SeedUtils.formatSeed(clusterSeed),
-            numSharedDedicatedMasterNodes, numSharedDataNodes, numSharedCoordOnlyNodes);
+            numSharedDedicatedMasterNodes, numSharedDataNodes, numSharedCoordOnlyNodes,
+            autoManageMinMasterNodes ? "auto-managed" : "manual");
         this.nodeConfigurationSource = nodeConfigurationSource;
         Builder builder = Settings.builder();
         if (random.nextInt(5) == 0) { // sometimes set this
@@ -321,14 +338,14 @@ public InternalTestCluster(long clusterSeed, Path baseDir,
         // Some tests make use of scripting quite a bit, so increase the limit for integration tests
         builder.put(ScriptService.SCRIPT_MAX_COMPILATIONS_PER_MINUTE.getKey(), 1000);
         if (TEST_NIGHTLY) {
-            builder.put(ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_INCOMING_RECOVERIES_SETTING.getKey(), RandomInts.randomIntBetween(random, 5, 10));
-            builder.put(ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_OUTGOING_RECOVERIES_SETTING.getKey(), RandomInts.randomIntBetween(random, 5, 10));
+            builder.put(ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_INCOMING_RECOVERIES_SETTING.getKey(), RandomNumbers.randomIntBetween(random, 5, 10));
+            builder.put(ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_OUTGOING_RECOVERIES_SETTING.getKey(), RandomNumbers.randomIntBetween(random, 5, 10));
         } else if (random.nextInt(100) <= 90) {
-            builder.put(ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_INCOMING_RECOVERIES_SETTING.getKey(), RandomInts.randomIntBetween(random, 2, 5));
-            builder.put(ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_OUTGOING_RECOVERIES_SETTING.getKey(), RandomInts.randomIntBetween(random, 2, 5));
+            builder.put(ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_INCOMING_RECOVERIES_SETTING.getKey(), RandomNumbers.randomIntBetween(random, 2, 5));
+            builder.put(ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_OUTGOING_RECOVERIES_SETTING.getKey(), RandomNumbers.randomIntBetween(random, 2, 5));
         }
         // always reduce this - it can make tests really slow
-        builder.put(RecoverySettings.INDICES_RECOVERY_RETRY_DELAY_STATE_SYNC_SETTING.getKey(), TimeValue.timeValueMillis(RandomInts.randomIntBetween(random, 20, 50)));
+        builder.put(RecoverySettings.INDICES_RECOVERY_RETRY_DELAY_STATE_SYNC_SETTING.getKey(), TimeValue.timeValueMillis(RandomNumbers.randomIntBetween(random, 20, 50)));
         defaultSettings = builder.build();
         executor = EsExecutors.newScaling("test runner", 0, Integer.MAX_VALUE, 0, TimeUnit.SECONDS, EsExecutors.daemonThreadFactory("test_" + clusterName), new ThreadContext(Settings.EMPTY));
     }
@@ -338,6 +355,11 @@ public String getClusterName() {
         return clusterName;
     }
 
+    /** returns true if the {@link ElectMasterService#DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING} setting is auto managed by this cluster */
+    public boolean getAutoManageMinMasterNode() {
+        return autoManageMinMasterNodes;
+    }
+
     public String[] getNodeNames() {
         return nodes.keySet().toArray(Strings.EMPTY_ARRAY);
     }
@@ -396,7 +418,7 @@ private Settings getRandomNodeSettings(long seed) {
         }
 
         if (random.nextBoolean()) {
-            builder.put(MappingUpdatedAction.INDICES_MAPPING_DYNAMIC_TIMEOUT_SETTING.getKey(), new TimeValue(RandomInts.randomIntBetween(random, 10, 30), TimeUnit.SECONDS));
+            builder.put(MappingUpdatedAction.INDICES_MAPPING_DYNAMIC_TIMEOUT_SETTING.getKey(), new TimeValue(RandomNumbers.randomIntBetween(random, 10, 30), TimeUnit.SECONDS));
         }
 
         if (random.nextInt(10) == 0) {
@@ -406,9 +428,9 @@ private Settings getRandomNodeSettings(long seed) {
 
         if (random.nextBoolean()) {
             if (random.nextInt(10) == 0) { // do something crazy slow here
-                builder.put(IndexStoreConfig.INDICES_STORE_THROTTLE_MAX_BYTES_PER_SEC_SETTING.getKey(), new ByteSizeValue(RandomInts.randomIntBetween(random, 1, 10), ByteSizeUnit.MB));
+                builder.put(IndexStoreConfig.INDICES_STORE_THROTTLE_MAX_BYTES_PER_SEC_SETTING.getKey(), new ByteSizeValue(RandomNumbers.randomIntBetween(random, 1, 10), ByteSizeUnit.MB));
             } else {
-                builder.put(IndexStoreConfig.INDICES_STORE_THROTTLE_MAX_BYTES_PER_SEC_SETTING.getKey(), new ByteSizeValue(RandomInts.randomIntBetween(random, 10, 200), ByteSizeUnit.MB));
+                builder.put(IndexStoreConfig.INDICES_STORE_THROTTLE_MAX_BYTES_PER_SEC_SETTING.getKey(), new ByteSizeValue(RandomNumbers.randomIntBetween(random, 10, 200), ByteSizeUnit.MB));
             }
         }
         if (random.nextBoolean()) {
@@ -417,21 +439,21 @@ private Settings getRandomNodeSettings(long seed) {
 
         if (random.nextBoolean()) {
             if (random.nextInt(10) == 0) { // do something crazy slow here
-                builder.put(RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey(), new ByteSizeValue(RandomInts.randomIntBetween(random, 1, 10), ByteSizeUnit.MB));
+                builder.put(RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey(), new ByteSizeValue(RandomNumbers.randomIntBetween(random, 1, 10), ByteSizeUnit.MB));
             } else {
-                builder.put(RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey(), new ByteSizeValue(RandomInts.randomIntBetween(random, 10, 200), ByteSizeUnit.MB));
+                builder.put(RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey(), new ByteSizeValue(RandomNumbers.randomIntBetween(random, 10, 200), ByteSizeUnit.MB));
             }
         }
 
         if (random.nextBoolean()) {
-            builder.put(TcpTransport.PING_SCHEDULE.getKey(), RandomInts.randomIntBetween(random, 100, 2000) + "ms");
+            builder.put(TcpTransport.PING_SCHEDULE.getKey(), RandomNumbers.randomIntBetween(random, 100, 2000) + "ms");
         }
 
         if (random.nextBoolean()) {
-            builder.put(ScriptService.SCRIPT_CACHE_SIZE_SETTING.getKey(), RandomInts.randomIntBetween(random, 0, 2000));
+            builder.put(ScriptService.SCRIPT_CACHE_SIZE_SETTING.getKey(), RandomNumbers.randomIntBetween(random, 0, 2000));
         }
         if (random.nextBoolean()) {
-            builder.put(ScriptService.SCRIPT_CACHE_EXPIRE_SETTING.getKey(), TimeValue.timeValueMillis(RandomInts.randomIntBetween(random, 750, 10000000)).getStringRep());
+            builder.put(ScriptService.SCRIPT_CACHE_EXPIRE_SETTING.getKey(), TimeValue.timeValueMillis(RandomNumbers.randomIntBetween(random, 750, 10000000)).getStringRep());
         }
 
         return builder.build();
@@ -459,7 +481,7 @@ private synchronized NodeAndClient getOrBuildRandomNode() {
         if (randomNodeAndClient != null) {
             return randomNodeAndClient;
         }
-        NodeAndClient buildNode = buildNode();
+        NodeAndClient buildNode = buildNode(1);
         buildNode.startNode();
         publishNode(buildNode);
         return buildNode;
@@ -489,30 +511,20 @@ private synchronized NodeAndClient getRandomNodeAndClient(Predicate<NodeAndClien
      * if more nodes than <code>n</code> are present this method will not
      * stop any of the running nodes.
      */
-    public void ensureAtLeastNumDataNodes(int n) {
-        final List<Async<String>> asyncs = new ArrayList<>();
-        synchronized (this) {
-            int size = numDataNodes();
-            for (int i = size; i < n; i++) {
-                logger.info("increasing cluster size from {} to {}", size, n);
-                if (numSharedDedicatedMasterNodes > 0) {
-                    asyncs.add(startDataOnlyNodeAsync());
-                } else {
-                    asyncs.add(startNodeAsync());
-                }
-            }
-        }
-        try {
-            for (Async<String> async : asyncs) {
-                async.get();
+    public synchronized void ensureAtLeastNumDataNodes(int n) {
+        boolean added = false;
+        int size = numDataNodes();
+        for (int i = size; i < n; i++) {
+            logger.info("increasing cluster size from {} to {}", size, n);
+            added = true;
+            if (numSharedDedicatedMasterNodes > 0) {
+                startDataOnlyNode(Settings.EMPTY);
+            } else {
+                startNode(Settings.EMPTY);
             }
-        } catch (Exception e) {
-            throw new ElasticsearchException("failed to start nodes", e);
         }
-        if (!asyncs.isEmpty()) {
-            synchronized (this) {
-                assertNoTimeout(client().admin().cluster().prepareHealth().setWaitForNodes(Integer.toString(nodes.size())).get());
-            }
+        if (added) {
+            validateClusterFormed();
         }
     }
 
@@ -537,28 +549,47 @@ public synchronized void ensureAtMostNumDataNodes(int n) throws IOException {
         while (values.hasNext() && numNodesAndClients++ < size - n) {
             NodeAndClient next = values.next();
             nodesToRemove.add(next);
-            removeDisruptionSchemeFromNode(next);
-            next.close();
-        }
-        for (NodeAndClient toRemove : nodesToRemove) {
-            nodes.remove(toRemove.name);
         }
+
+        stopNodesAndClients(nodesToRemove);
         if (!nodesToRemove.isEmpty() && size() > 0) {
-            assertNoTimeout(client().admin().cluster().prepareHealth().setWaitForNodes(Integer.toString(nodes.size())).get());
+            validateClusterFormed();
         }
     }
 
-    private NodeAndClient buildNode(Settings settings) {
+    /**
+     * builds a new node given the settings.
+     *
+     * @param settings              the settings to use
+     * @param defaultMinMasterNodes min_master_nodes value to use if min_master_nodes is auto managed
+     */
+    private NodeAndClient buildNode(Settings settings, int defaultMinMasterNodes) {
         int ord = nextNodeId.getAndIncrement();
-        return buildNode(ord, random.nextLong(), settings, false);
+        return buildNode(ord, random.nextLong(), settings, false, defaultMinMasterNodes);
     }
 
-    private NodeAndClient buildNode() {
+    /**
+     * builds a new node with default settings
+     *
+     * @param defaultMinMasterNodes min_master_nodes value to use if min_master_nodes is auto managed
+     */
+    private NodeAndClient buildNode(int defaultMinMasterNodes) {
         int ord = nextNodeId.getAndIncrement();
-        return buildNode(ord, random.nextLong(), null, false);
+        return buildNode(ord, random.nextLong(), null, false, defaultMinMasterNodes);
     }
 
-    private NodeAndClient buildNode(int nodeId, long seed, Settings settings, boolean reuseExisting) {
+    /**
+     * builds a new node
+     *
+     * @param nodeId                the node internal id (see {@link NodeAndClient#nodeAndClientId()}
+     * @param seed                  the node's random seed
+     * @param settings              the settings to use
+     * @param reuseExisting         if a node with the same name is already part of {@link #nodes}, no new node will be built and
+     *                              the method will return the existing one
+     * @param defaultMinMasterNodes min_master_nodes value to use if min_master_nodes is auto managed
+     */
+    private NodeAndClient buildNode(int nodeId, long seed, Settings settings,
+                                    boolean reuseExisting, int defaultMinMasterNodes) {
         assert Thread.holdsLock(this);
         ensureOpen();
         settings = getSettings(nodeId, seed, settings);
@@ -570,13 +601,33 @@ private NodeAndClient buildNode(int nodeId, long seed, Settings settings, boolea
             assert reuseExisting == true || nodes.containsKey(name) == false :
                 "node name [" + name + "] already exists but not allowed to use it";
         }
-        Settings finalSettings = Settings.builder()
+        Settings.Builder finalSettings = Settings.builder()
             .put(Environment.PATH_HOME_SETTING.getKey(), baseDir) // allow overriding path.home
             .put(settings)
             .put("node.name", name)
-            .put(NodeEnvironment.NODE_ID_SEED_SETTING.getKey(), seed)
-            .build();
-        MockNode node = new MockNode(finalSettings, plugins);
+            .put(NodeEnvironment.NODE_ID_SEED_SETTING.getKey(), seed);
+
+        final boolean usingSingleNodeDiscovery = DiscoveryModule.DISCOVERY_TYPE_SETTING.get(finalSettings.build()).equals("single-node");
+        if (!usingSingleNodeDiscovery && autoManageMinMasterNodes) {
+            assert finalSettings.get(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()) == null :
+                "min master nodes may not be set when auto managed";
+            assert finalSettings.get(INITIAL_STATE_TIMEOUT_SETTING.getKey()) == null :
+                "automatically managing min master nodes require nodes to complete a join cycle" +
+                    " when starting";
+            finalSettings
+                // don't wait too long not to slow down tests
+                .put(ZenDiscovery.MASTER_ELECTION_WAIT_FOR_JOINS_TIMEOUT_SETTING.getKey(), "5s")
+                .put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), defaultMinMasterNodes);
+        } else if (!usingSingleNodeDiscovery && finalSettings.get(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()) == null) {
+            throw new IllegalArgumentException(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey() + " must be configured");
+        }
+        SecureSettings secureSettings = finalSettings.getSecureSettings();
+        MockNode node = new MockNode(finalSettings.build(), plugins);
+        try {
+            IOUtils.close(secureSettings);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
         return new NodeAndClient(name, node, nodeId);
     }
 
@@ -675,10 +726,6 @@ public synchronized String startCoordinatingOnlyNode(Settings settings) {
         ensureOpen(); // currently unused
         Builder builder = Settings.builder().put(settings).put(Node.NODE_MASTER_SETTING.getKey(), false)
             .put(Node.NODE_DATA_SETTING.getKey(), false).put(Node.NODE_INGEST_SETTING.getKey(), false);
-        if (size() == 0) {
-            // if we are the first node - don't wait for a state
-            builder.put(DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING.getKey(), 0);
-        }
         return startNode(builder);
     }
 
@@ -770,6 +817,14 @@ public int nodeAndClientId() {
             return nodeAndClientId;
         }
 
+        public String getName() {
+            return name;
+        }
+
+        public boolean isMasterEligible() {
+            return Node.NODE_MASTER_SETTING.get(node.settings());
+        }
+
         Client client(Random random) {
             if (closed.get()) {
                 throw new RuntimeException("already closed");
@@ -825,7 +880,11 @@ void resetClient() throws IOException {
         }
 
         void startNode() {
-            node.start();
+            try {
+                node.start();
+            } catch (NodeValidationException e) {
+                throw new RuntimeException(e);
+            }
         }
 
         void closeNode() throws IOException {
@@ -833,21 +892,37 @@ void closeNode() throws IOException {
             node.close();
         }
 
-        void restart(RestartCallback callback, boolean clearDataIfNeeded) throws Exception {
-            assert callback != null;
-            resetClient();
+        /**
+         * closes the current node if not already closed, builds a new node object using the current node settings and starts it
+         */
+        void restart(RestartCallback callback, boolean clearDataIfNeeded, int minMasterNodes) throws Exception {
             if (!node.isClosed()) {
                 closeNode();
             }
-            Settings newSettings = callback.onNodeStopped(name);
-            if (newSettings == null) {
-                newSettings = Settings.EMPTY;
+            recreateNodeOnRestart(callback, clearDataIfNeeded, minMasterNodes);
+            startNode();
+        }
+
+        /**
+         * rebuilds a new node object using the current node settings and starts it
+         */
+        void recreateNodeOnRestart(RestartCallback callback, boolean clearDataIfNeeded, int minMasterNodes) throws Exception {
+            assert callback != null;
+            Settings callbackSettings = callback.onNodeStopped(name);
+            Settings.Builder newSettings = Settings.builder();
+            if (callbackSettings != null) {
+                newSettings.put(callbackSettings);
+            }
+            if (minMasterNodes >= 0) {
+                assert DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.exists(newSettings.build()) == false : "min master nodes is auto managed";
+                newSettings.put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), minMasterNodes).build();
             }
             if (clearDataIfNeeded) {
                 clearDataIfNeeded(callback);
             }
-            createNewNode(newSettings);
-            startNode();
+            createNewNode(newSettings.build());
+            // make sure cached client points to new node
+            resetClient();
         }
 
         private void clearDataIfNeeded(RestartCallback callback) throws IOException {
@@ -864,6 +939,10 @@ private void clearDataIfNeeded(RestartCallback callback) throws IOException {
         private void createNewNode(final Settings newSettings) {
             final long newIdSeed = NodeEnvironment.NODE_ID_SEED_SETTING.get(node.settings()) + 1; // use a new seed to make sure we have new node id
             Settings finalSettings = Settings.builder().put(node.settings()).put(newSettings).put(NodeEnvironment.NODE_ID_SEED_SETTING.getKey(), newIdSeed).build();
+            if (DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.exists(finalSettings) == false) {
+                throw new IllegalStateException(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey() +
+                    " is not configured after restart of [" + name + "]");
+            }
             Collection<Class<? extends Plugin>> plugins = node.getClasspathPlugins();
             node = new MockNode(finalSettings, plugins);
             markNodeDataDirsAsNotEligableForWipe(node);
@@ -937,22 +1016,24 @@ private synchronized void reset(boolean wipeData) throws IOException {
             if (wipeData) {
                 wipePendingDataDirectories();
             }
+            if (nodes.size() > 0 && autoManageMinMasterNodes) {
+                updateMinMasterNodes(getMasterNodesCount());
+            }
             logger.debug("Cluster hasn't changed - moving out - nodes: [{}] nextNodeId: [{}] numSharedNodes: [{}]", nodes.keySet(), nextNodeId.get(), newSize);
             return;
         }
         logger.debug("Cluster is NOT consistent - restarting shared nodes - nodes: [{}] nextNodeId: [{}] numSharedNodes: [{}]", nodes.keySet(), nextNodeId.get(), newSize);
 
         // trash all nodes with id >= sharedNodesSeeds.length - they are non shared
-
-
+        final List<NodeAndClient> toClose = new ArrayList<>();
         for (Iterator<NodeAndClient> iterator = nodes.values().iterator(); iterator.hasNext();) {
             NodeAndClient nodeAndClient = iterator.next();
             if (nodeAndClient.nodeAndClientId() >= sharedNodesSeeds.length) {
                 logger.debug("Close Node [{}] not shared", nodeAndClient.name);
-                nodeAndClient.close();
-                iterator.remove();
+                toClose.add(nodeAndClient);
             }
         }
+        stopNodesAndClients(toClose);
 
         // clean up what the nodes left that is unused
         if (wipeData) {
@@ -961,13 +1042,15 @@ private synchronized void reset(boolean wipeData) throws IOException {
 
         // start any missing node
         assert newSize == numSharedDedicatedMasterNodes + numSharedDataNodes + numSharedCoordOnlyNodes;
+        final int numberOfMasterNodes = numSharedDedicatedMasterNodes > 0 ? numSharedDedicatedMasterNodes : numSharedDataNodes;
+        final int defaultMinMasterNodes = (numberOfMasterNodes / 2) + 1;
+        final List<NodeAndClient> toStartAndPublish = new ArrayList<>(); // we want to start nodes in one go due to min master nodes
         for (int i = 0; i < numSharedDedicatedMasterNodes; i++) {
             final Settings.Builder settings = Settings.builder();
-            settings.put(Node.NODE_MASTER_SETTING.getKey(), true).build();
-            settings.put(Node.NODE_DATA_SETTING.getKey(), false).build();
-            NodeAndClient nodeAndClient = buildNode(i, sharedNodesSeeds[i], settings.build(), true);
-            nodeAndClient.startNode();
-            publishNode(nodeAndClient);
+            settings.put(Node.NODE_MASTER_SETTING.getKey(), true);
+            settings.put(Node.NODE_DATA_SETTING.getKey(), false);
+            NodeAndClient nodeAndClient = buildNode(i, sharedNodesSeeds[i], settings.build(), true, defaultMinMasterNodes);
+            toStartAndPublish.add(nodeAndClient);
         }
         for (int i = numSharedDedicatedMasterNodes; i < numSharedDedicatedMasterNodes + numSharedDataNodes; i++) {
             final Settings.Builder settings = Settings.builder();
@@ -976,32 +1059,62 @@ private synchronized void reset(boolean wipeData) throws IOException {
                 settings.put(Node.NODE_MASTER_SETTING.getKey(), false).build();
                 settings.put(Node.NODE_DATA_SETTING.getKey(), true).build();
             }
-            NodeAndClient nodeAndClient = buildNode(i, sharedNodesSeeds[i], settings.build(), true);
-            nodeAndClient.startNode();
-            publishNode(nodeAndClient);
+            NodeAndClient nodeAndClient = buildNode(i, sharedNodesSeeds[i], settings.build(), true, defaultMinMasterNodes);
+            toStartAndPublish.add(nodeAndClient);
         }
         for (int i = numSharedDedicatedMasterNodes + numSharedDataNodes;
              i < numSharedDedicatedMasterNodes + numSharedDataNodes + numSharedCoordOnlyNodes; i++) {
             final Builder settings = Settings.builder().put(Node.NODE_MASTER_SETTING.getKey(), false)
                 .put(Node.NODE_DATA_SETTING.getKey(), false).put(Node.NODE_INGEST_SETTING.getKey(), false);
-            NodeAndClient nodeAndClient = buildNode(i, sharedNodesSeeds[i], settings.build(), true);
-            nodeAndClient.startNode();
-            publishNode(nodeAndClient);
+            NodeAndClient nodeAndClient = buildNode(i, sharedNodesSeeds[i], settings.build(), true, defaultMinMasterNodes);
+            toStartAndPublish.add(nodeAndClient);
         }
 
+        startAndPublishNodesAndClients(toStartAndPublish);
+
         nextNodeId.set(newSize);
         assert size() == newSize;
         if (newSize > 0) {
-            ClusterHealthResponse response = client().admin().cluster().prepareHealth()
-                .setWaitForNodes(Integer.toString(newSize)).get();
-            if (response.isTimedOut()) {
-                logger.warn("failed to wait for a cluster of size [{}], got [{}]", newSize, response);
-                throw new IllegalStateException("cluster failed to reach the expected size of [" + newSize + "]");
-            }
+            validateClusterFormed();
         }
         logger.debug("Cluster is consistent again - nodes: [{}] nextNodeId: [{}] numSharedNodes: [{}]", nodes.keySet(), nextNodeId.get(), newSize);
     }
 
+    /** ensure a cluster is formed with all published nodes. */
+    public synchronized void validateClusterFormed() {
+        String name = randomFrom(random, getNodeNames());
+        validateClusterFormed(name);
+    }
+
+    /** ensure a cluster is formed with all published nodes, but do so by using the client of the specified node */
+    public synchronized void validateClusterFormed(String viaNode) {
+        Set<DiscoveryNode> expectedNodes = new HashSet<>();
+        for (NodeAndClient nodeAndClient : nodes.values()) {
+            expectedNodes.add(getInstanceFromNode(ClusterService.class, nodeAndClient.node()).localNode());
+        }
+        logger.trace("validating cluster formed via [{}], expecting {}", viaNode, expectedNodes);
+        final Client client = client(viaNode);
+        try {
+            if (awaitBusy(() -> {
+                DiscoveryNodes discoveryNodes = client.admin().cluster().prepareState().get().getState().nodes();
+                if (discoveryNodes.getSize() != expectedNodes.size()) {
+                    return false;
+                }
+                for (DiscoveryNode expectedNode : expectedNodes) {
+                    if (discoveryNodes.nodeExists(expectedNode) == false) {
+                        return false;
+                    }
+                }
+                return true;
+            }, 30, TimeUnit.SECONDS) == false) {
+                throw new IllegalStateException("cluster failed to form with expected nodes " + expectedNodes + " and actual nodes " +
+                    client.admin().cluster().prepareState().get().getState().nodes());
+            }
+        } catch (InterruptedException e) {
+            throw new IllegalStateException(e);
+        }
+    }
+
     @Override
     public synchronized void afterTest() throws IOException {
         wipePendingDataDirectories();
@@ -1009,7 +1122,7 @@ public synchronized void afterTest() throws IOException {
     }
 
     @Override
-    public void beforeIndexDeletion() {
+    public void beforeIndexDeletion() throws Exception {
         // Check that the operations counter on index shard has reached 0.
         // The assumption here is that after a test there are no ongoing write operations.
         // test that have ongoing write operations after the test (for example because ttl is used
@@ -1044,16 +1157,35 @@ private void assertSameSyncIdSameDocs() {
         }
     }
 
-    private void assertShardIndexCounter() {
-        final Collection<NodeAndClient> nodesAndClients = nodes.values();
-        for (NodeAndClient nodeAndClient : nodesAndClients) {
-            IndicesService indexServices = getInstance(IndicesService.class, nodeAndClient.name);
-            for (IndexService indexService : indexServices) {
-                for (IndexShard indexShard : indexService) {
-                    assertThat("index shard counter on shard " + indexShard.shardId() + " on node " + nodeAndClient.name + " not 0", indexShard.getActiveOperationsCount(), equalTo(0));
+    private void assertShardIndexCounter() throws Exception {
+        assertBusy(() -> {
+            final Collection<NodeAndClient> nodesAndClients = nodes.values();
+            for (NodeAndClient nodeAndClient : nodesAndClients) {
+                IndicesService indexServices = getInstance(IndicesService.class, nodeAndClient.name);
+                for (IndexService indexService : indexServices) {
+                    for (IndexShard indexShard : indexService) {
+                        int activeOperationsCount = indexShard.getActiveOperationsCount();
+                        if (activeOperationsCount > 0) {
+                            TaskManager taskManager = getInstance(TransportService.class, nodeAndClient.name).getTaskManager();
+                            DiscoveryNode localNode = getInstance(ClusterService.class, nodeAndClient.name).localNode();
+                            List<TaskInfo> taskInfos = taskManager.getTasks().values().stream()
+                                    .filter(task -> task instanceof ReplicationTask)
+                                    .map(task -> task.taskInfo(localNode.getId(), true))
+                                    .collect(Collectors.toList());
+                            ListTasksResponse response = new ListTasksResponse(taskInfos, Collections.emptyList(), Collections.emptyList());
+                            try {
+                                XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint().value(response);
+                                throw new AssertionError("expected index shard counter on shard " + indexShard.shardId() + " on node " +
+                                        nodeAndClient.name + " to be 0 but was " + activeOperationsCount + ". Current replication tasks on node:\n" +
+                                        builder.string());
+                            } catch (IOException e) {
+                                throw new RuntimeException("caught exception while building response [" + response + "]", e);
+                            }
+                        }
+                    }
                 }
             }
-        }
+        });
     }
 
     private void randomlyResetClients() throws IOException {
@@ -1199,9 +1331,7 @@ public synchronized boolean stopRandomDataNode() throws IOException {
         NodeAndClient nodeAndClient = getRandomNodeAndClient(new DataNodePredicate());
         if (nodeAndClient != null) {
             logger.info("Closing random node [{}] ", nodeAndClient.name);
-            removeDisruptionSchemeFromNode(nodeAndClient);
-            nodes.remove(nodeAndClient.name);
-            nodeAndClient.close();
+            stopNodesAndClient(nodeAndClient);
             return true;
         }
         return false;
@@ -1216,9 +1346,7 @@ public synchronized void stopRandomNode(final Predicate<Settings> filter) throws
         NodeAndClient nodeAndClient = getRandomNodeAndClient(nc -> filter.test(nc.node.settings()));
         if (nodeAndClient != null) {
             logger.info("Closing filtered random node [{}] ", nodeAndClient.name);
-            removeDisruptionSchemeFromNode(nodeAndClient);
-            nodes.remove(nodeAndClient.name);
-            nodeAndClient.close();
+            stopNodesAndClient(nodeAndClient);
         }
     }
 
@@ -1231,9 +1359,7 @@ public synchronized void stopCurrentMasterNode() throws IOException {
         String masterNodeName = getMasterName();
         assert nodes.containsKey(masterNodeName);
         logger.info("Closing master node [{}] ", masterNodeName);
-        removeDisruptionSchemeFromNode(nodes.get(masterNodeName));
-        NodeAndClient remove = nodes.remove(masterNodeName);
-        remove.close();
+        stopNodesAndClient(nodes.get(masterNodeName));
     }
 
     /**
@@ -1243,8 +1369,55 @@ public synchronized void stopRandomNonMasterNode() throws IOException {
         NodeAndClient nodeAndClient = getRandomNodeAndClient(new MasterNodePredicate(getMasterName()).negate());
         if (nodeAndClient != null) {
             logger.info("Closing random non master node [{}] current master [{}] ", nodeAndClient.name, getMasterName());
+            stopNodesAndClient(nodeAndClient);
+        }
+    }
+
+    private synchronized void startAndPublishNodesAndClients(List<NodeAndClient> nodeAndClients) {
+        if (nodeAndClients.size() > 0) {
+            final int newMasters = (int) nodeAndClients.stream().filter(NodeAndClient::isMasterEligible)
+                .filter(nac -> nodes.containsKey(nac.name) == false) // filter out old masters
+                .count();
+            final int currentMasters = getMasterNodesCount();
+            if (autoManageMinMasterNodes && currentMasters > 1 && newMasters > 0) {
+                // special case for 1 node master - we can't update the min master nodes before we add more nodes.
+                updateMinMasterNodes(currentMasters + newMasters);
+            }
+            List<Future<?>> futures = nodeAndClients.stream().map(node -> executor.submit(node::startNode)).collect(Collectors.toList());
+            try {
+                for (Future<?> future : futures) {
+                    future.get();
+                }
+            } catch (InterruptedException e) {
+                throw new AssertionError("interrupted while starting nodes", e);
+            } catch (ExecutionException e) {
+                throw new RuntimeException("failed to start nodes", e);
+            }
+            nodeAndClients.forEach(this::publishNode);
+
+            if (autoManageMinMasterNodes && currentMasters == 1 && newMasters > 0) {
+                // update once masters have joined
+                validateClusterFormed();
+                updateMinMasterNodes(currentMasters + newMasters);
+            }
+        }
+    }
+
+    private synchronized void stopNodesAndClient(NodeAndClient nodeAndClient) throws IOException {
+        stopNodesAndClients(Collections.singleton(nodeAndClient));
+    }
+
+    private synchronized void stopNodesAndClients(Collection<NodeAndClient> nodeAndClients) throws IOException {
+        if (autoManageMinMasterNodes && nodeAndClients.size() > 0) {
+            int masters = (int)nodeAndClients.stream().filter(NodeAndClient::isMasterEligible).count();
+            if (masters > 0) {
+                updateMinMasterNodes(getMasterNodesCount() - masters);
+            }
+        }
+        for (NodeAndClient nodeAndClient: nodeAndClients) {
             removeDisruptionSchemeFromNode(nodeAndClient);
-            nodes.remove(nodeAndClient.name);
+            NodeAndClient previous = nodes.remove(nodeAndClient.name);
+            assert previous == nodeAndClient;
             nodeAndClient.close();
         }
     }
@@ -1284,8 +1457,7 @@ private synchronized void restartRandomNode(Predicate<NodeAndClient> predicate,
         ensureOpen();
         NodeAndClient nodeAndClient = getRandomNodeAndClient(predicate);
         if (nodeAndClient != null) {
-            logger.info("Restarting random node [{}] ", nodeAndClient.name);
-            nodeAndClient.restart(callback, true);
+            restartNode(nodeAndClient, callback);
         }
     }
 
@@ -1296,93 +1468,10 @@ public synchronized void restartNode(String nodeName, RestartCallback callback)
         ensureOpen();
         NodeAndClient nodeAndClient = nodes.get(nodeName);
         if (nodeAndClient != null) {
-            logger.info("Restarting node [{}] ", nodeAndClient.name);
-            nodeAndClient.restart(callback, true);
-        }
-    }
-
-    private synchronized void restartAllNodes(boolean rollingRestart, RestartCallback callback) throws Exception {
-        ensureOpen();
-        List<NodeAndClient> toRemove = new ArrayList<>();
-        try {
-            for (NodeAndClient nodeAndClient : nodes.values()) {
-                if (!callback.doRestart(nodeAndClient.name)) {
-                    logger.info("Closing node [{}] during restart", nodeAndClient.name);
-                    toRemove.add(nodeAndClient);
-                    if (activeDisruptionScheme != null) {
-                        activeDisruptionScheme.removeFromNode(nodeAndClient.name, this);
-                    }
-                    nodeAndClient.close();
-                }
-            }
-        } finally {
-            for (NodeAndClient nodeAndClient : toRemove) {
-                nodes.remove(nodeAndClient.name);
-            }
-        }
-        logger.info("Restarting remaining nodes rollingRestart [{}]", rollingRestart);
-        if (rollingRestart) {
-            int numNodesRestarted = 0;
-            for (NodeAndClient nodeAndClient : nodes.values()) {
-                callback.doAfterNodes(numNodesRestarted++, nodeAndClient.nodeClient());
-                logger.info("Restarting node [{}] ", nodeAndClient.name);
-                if (activeDisruptionScheme != null) {
-                    activeDisruptionScheme.removeFromNode(nodeAndClient.name, this);
-                }
-                nodeAndClient.restart(callback, true);
-                if (activeDisruptionScheme != null) {
-                    activeDisruptionScheme.applyToNode(nodeAndClient.name, this);
-                }
-            }
-        } else {
-            int numNodesRestarted = 0;
-            Set[] nodesRoleOrder = new Set[nextNodeId.get()];
-            Map<Set<Role>, List<NodeAndClient>> nodesByRoles = new HashMap<>();
-            for (NodeAndClient nodeAndClient : nodes.values()) {
-                callback.doAfterNodes(numNodesRestarted++, nodeAndClient.nodeClient());
-                logger.info("Stopping node [{}] ", nodeAndClient.name);
-                if (activeDisruptionScheme != null) {
-                    activeDisruptionScheme.removeFromNode(nodeAndClient.name, this);
-                }
-                nodeAndClient.closeNode();
-                // delete data folders now, before we start other nodes that may claim it
-                nodeAndClient.clearDataIfNeeded(callback);
-
-                DiscoveryNode discoveryNode = getInstanceFromNode(ClusterService.class, nodeAndClient.node()).localNode();
-                nodesRoleOrder[nodeAndClient.nodeAndClientId()] = discoveryNode.getRoles();
-                nodesByRoles.computeIfAbsent(discoveryNode.getRoles(), k -> new ArrayList<>()).add(nodeAndClient);
-            }
-
-            assert nodesByRoles.values().stream().collect(Collectors.summingInt(List::size)) == nodes.size();
-
-            // randomize start up order, but making sure that:
-            // 1) A data folder that was assigned to a data node will stay so
-            // 2) Data nodes will get the same node lock ordinal range, so custom index paths (where the ordinal is used)
-            //    will still belong to data nodes
-            for (List<NodeAndClient> sameRoleNodes : nodesByRoles.values()) {
-                Collections.shuffle(sameRoleNodes, random);
-            }
-
-            for (Set roles : nodesRoleOrder) {
-                if (roles == null) {
-                    // if some nodes were stopped, we want have a role for them
-                    continue;
-                }
-                NodeAndClient nodeAndClient = nodesByRoles.get(roles).remove(0);
-                logger.info("Starting node [{}] ", nodeAndClient.name);
-                if (activeDisruptionScheme != null) {
-                    activeDisruptionScheme.removeFromNode(nodeAndClient.name, this);
-                }
-                // we already cleared data folders, before starting nodes up
-                nodeAndClient.restart(callback, false);
-                if (activeDisruptionScheme != null) {
-                    activeDisruptionScheme.applyToNode(nodeAndClient.name, this);
-                }
-            }
+            restartNode(nodeAndClient, callback);
         }
     }
 
-
     public static final RestartCallback EMPTY_CALLBACK = new RestartCallback() {
         @Override
         public Settings onNodeStopped(String node) {
@@ -1407,15 +1496,94 @@ public void rollingRestart() throws Exception {
     /**
      * Restarts all nodes in a rolling restart fashion ie. only restarts on node a time.
      */
-    public void rollingRestart(RestartCallback function) throws Exception {
-        restartAllNodes(true, function);
+    public synchronized void rollingRestart(RestartCallback callback) throws Exception {
+        int numNodesRestarted = 0;
+        for (NodeAndClient nodeAndClient : nodes.values()) {
+            callback.doAfterNodes(numNodesRestarted++, nodeAndClient.nodeClient());
+            restartNode(nodeAndClient, callback);
+        }
+    }
+
+    private void restartNode(NodeAndClient nodeAndClient, RestartCallback callback) throws Exception {
+        logger.info("Restarting node [{}] ", nodeAndClient.name);
+        if (activeDisruptionScheme != null) {
+            activeDisruptionScheme.removeFromNode(nodeAndClient.name, this);
+        }
+        final int masterNodesCount = getMasterNodesCount();
+        // special case to allow stopping one node in a two node cluster and keep it functional
+        final boolean updateMinMaster = nodeAndClient.isMasterEligible() && masterNodesCount == 2 && autoManageMinMasterNodes;
+        if (updateMinMaster) {
+            updateMinMasterNodes(masterNodesCount - 1);
+        }
+        nodeAndClient.restart(callback, true, autoManageMinMasterNodes ? getMinMasterNodes(masterNodesCount) : -1);
+        if (activeDisruptionScheme != null) {
+            activeDisruptionScheme.applyToNode(nodeAndClient.name, this);
+        }
+        if (callback.validateClusterForming() || updateMinMaster) {
+            // we have to validate cluster size if updateMinMaster == true, because we need the
+            // second node to join in order to increment min_master_nodes back to 2.
+            // we also have to do via the node that was just restarted as it may be that the master didn't yet process
+            // the fact it left
+            validateClusterFormed(nodeAndClient.name);
+        }
+        if (updateMinMaster) {
+            updateMinMasterNodes(masterNodesCount);
+        }
     }
 
     /**
      * Restarts all nodes in the cluster. It first stops all nodes and then restarts all the nodes again.
      */
-    public void fullRestart(RestartCallback function) throws Exception {
-        restartAllNodes(false, function);
+    public synchronized void fullRestart(RestartCallback callback) throws Exception {
+        int numNodesRestarted = 0;
+        Map<Set<Role>, List<NodeAndClient>> nodesByRoles = new HashMap<>();
+        Set[] rolesOrderedByOriginalStartupOrder =  new Set[nextNodeId.get()];
+        for (NodeAndClient nodeAndClient : nodes.values()) {
+            callback.doAfterNodes(numNodesRestarted++, nodeAndClient.nodeClient());
+            logger.info("Stopping node [{}] ", nodeAndClient.name);
+            if (activeDisruptionScheme != null) {
+                activeDisruptionScheme.removeFromNode(nodeAndClient.name, this);
+            }
+            nodeAndClient.closeNode();
+            // delete data folders now, before we start other nodes that may claim it
+            nodeAndClient.clearDataIfNeeded(callback);
+            DiscoveryNode discoveryNode = getInstanceFromNode(ClusterService.class, nodeAndClient.node()).localNode();
+            rolesOrderedByOriginalStartupOrder[nodeAndClient.nodeAndClientId] = discoveryNode.getRoles();
+            nodesByRoles.computeIfAbsent(discoveryNode.getRoles(), k -> new ArrayList<>()).add(nodeAndClient);
+        }
+
+        assert nodesByRoles.values().stream().collect(Collectors.summingInt(List::size)) == nodes.size();
+
+        // randomize start up order, but making sure that:
+        // 1) A data folder that was assigned to a data node will stay so
+        // 2) Data nodes will get the same node lock ordinal range, so custom index paths (where the ordinal is used)
+        //    will still belong to data nodes
+        for (List<NodeAndClient> sameRoleNodes : nodesByRoles.values()) {
+            Collections.shuffle(sameRoleNodes, random);
+        }
+        List<NodeAndClient> startUpOrder = new ArrayList<>();
+        for (Set roles : rolesOrderedByOriginalStartupOrder) {
+            if (roles == null) {
+                // if some nodes were stopped, we want have a role for that ordinal
+                continue;
+            }
+            final List<NodeAndClient> nodesByRole = nodesByRoles.get(roles);
+            startUpOrder.add(nodesByRole.remove(0));
+        }
+        assert nodesByRoles.values().stream().collect(Collectors.summingInt(List::size)) == 0;
+
+        // do two rounds to minimize pinging (mock zen pings pings with no delay and can create a lot of logs)
+        for (NodeAndClient nodeAndClient : startUpOrder) {
+            logger.info("resetting node [{}] ", nodeAndClient.name);
+            // we already cleared data folders, before starting nodes up
+            nodeAndClient.recreateNodeOnRestart(callback, false, autoManageMinMasterNodes ? getMinMasterNodes(getMasterNodesCount()) : -1);
+        }
+
+        startAndPublishNodesAndClients(startUpOrder);
+
+        if (callback.validateClusterForming()) {
+            validateClusterFormed();
+        }
     }
 
 
@@ -1499,133 +1667,114 @@ public synchronized String startNode(Settings.Builder settings) {
      * Starts a node with the given settings and returns it's name.
      */
     public synchronized String startNode(Settings settings) {
-        NodeAndClient buildNode = buildNode(settings);
-        buildNode.startNode();
-        publishNode(buildNode);
+        final int defaultMinMasterNodes = getMinMasterNodes(getMasterNodesCount() + (Node.NODE_MASTER_SETTING.get(settings) ? 1 : 0));
+        NodeAndClient buildNode = buildNode(settings, defaultMinMasterNodes);
+        startAndPublishNodesAndClients(Collections.singletonList(buildNode));
         return buildNode.name;
     }
 
-    public synchronized Async<List<String>> startMasterOnlyNodesAsync(int numNodes) {
-        return startMasterOnlyNodesAsync(numNodes, Settings.EMPTY);
-    }
-
-    public synchronized Async<List<String>> startMasterOnlyNodesAsync(int numNodes, Settings settings) {
-        Settings settings1 = Settings.builder().put(settings).put(Node.NODE_MASTER_SETTING.getKey(), true).put(Node.NODE_DATA_SETTING.getKey(), false).build();
-        return startNodesAsync(numNodes, settings1, Version.CURRENT);
+    /**
+     * Starts multiple nodes with default settings and returns their names
+     */
+    public synchronized List<String> startNodes(int numOfNodes) {
+        return startNodes(numOfNodes, Settings.EMPTY);
     }
 
-    public synchronized Async<List<String>> startDataOnlyNodesAsync(int numNodes) {
-        return startDataOnlyNodesAsync(numNodes, Settings.EMPTY);
+    /**
+     * Starts multiple nodes with the given settings and returns their names
+     */
+    public synchronized List<String> startNodes(int numOfNodes, Settings settings) {
+        return startNodes(Collections.nCopies(numOfNodes, settings).stream().toArray(Settings[]::new));
     }
 
-    public synchronized Async<List<String>> startDataOnlyNodesAsync(int numNodes, Settings settings) {
-        Settings settings1 = Settings.builder().put(settings).put(Node.NODE_MASTER_SETTING.getKey(), false).put(Node.NODE_DATA_SETTING.getKey(), true).build();
-        return startNodesAsync(numNodes, settings1, Version.CURRENT);
-    }
+    /**
+     * Starts multiple nodes with the given settings and returns their names
+     */
+    public synchronized List<String> startNodes(Settings... settings) {
+        final int defaultMinMasterNodes;
+        if (autoManageMinMasterNodes) {
+            int mastersDelta = (int) Stream.of(settings).filter(Node.NODE_MASTER_SETTING::get).count();
+            defaultMinMasterNodes = getMinMasterNodes(getMasterNodesCount() + mastersDelta);
+        } else {
+            defaultMinMasterNodes = -1;
+        }
+        List<NodeAndClient> nodes = new ArrayList<>();
+        for (Settings nodeSettings: settings) {
+            nodes.add(buildNode(nodeSettings, defaultMinMasterNodes));
+        }
+        startAndPublishNodesAndClients(nodes);
+        if (autoManageMinMasterNodes) {
+            validateClusterFormed();
+        }
 
-    public synchronized Async<String> startMasterOnlyNodeAsync() {
-        return startMasterOnlyNodeAsync(Settings.EMPTY);
+        return nodes.stream().map(NodeAndClient::getName).collect(Collectors.toList());
     }
 
-    public synchronized Async<String> startMasterOnlyNodeAsync(Settings settings) {
-        Settings settings1 = Settings.builder().put(settings).put(Node.NODE_MASTER_SETTING.getKey(), true).put(Node.NODE_DATA_SETTING.getKey(), false).build();
-        return startNodeAsync(settings1, Version.CURRENT);
+    public synchronized List<String> startMasterOnlyNodes(int numNodes) {
+        return startMasterOnlyNodes(numNodes, Settings.EMPTY);
     }
 
-    public synchronized String startMasterOnlyNode(Settings settings) {
+    public synchronized List<String> startMasterOnlyNodes(int numNodes, Settings settings) {
         Settings settings1 = Settings.builder().put(settings).put(Node.NODE_MASTER_SETTING.getKey(), true).put(Node.NODE_DATA_SETTING.getKey(), false).build();
-        return startNode(settings1);
+        return startNodes(numNodes, settings1);
     }
 
-    public synchronized Async<String> startDataOnlyNodeAsync() {
-        return startDataOnlyNodeAsync(Settings.EMPTY);
-    }
-
-    public synchronized Async<String> startDataOnlyNodeAsync(Settings settings) {
-        Settings settings1 = Settings.builder().put(settings).put(Node.NODE_MASTER_SETTING.getKey(), false).put(Node.NODE_DATA_SETTING.getKey(), true).build();
-        return startNodeAsync(settings1, Version.CURRENT);
+    public synchronized List<String> startDataOnlyNodes(int numNodes) {
+        return startDataOnlyNodes(numNodes, Settings.EMPTY);
     }
 
-    public synchronized String startDataOnlyNode(Settings settings) {
+    public synchronized List<String> startDataOnlyNodes(int numNodes, Settings settings) {
         Settings settings1 = Settings.builder().put(settings).put(Node.NODE_MASTER_SETTING.getKey(), false).put(Node.NODE_DATA_SETTING.getKey(), true).build();
-        return startNode(settings1);
+        return startNodes(numNodes, settings1);
     }
 
     /**
-     * Starts a node in an async manner with the given settings and returns future with its name.
+     * updates the min master nodes setting in the current running cluster.
+     *
+     * @param eligibleMasterNodeCount the number of master eligible nodes to use as basis for the min master node setting
      */
-    public synchronized Async<String> startNodeAsync() {
-        return startNodeAsync(Settings.EMPTY, Version.CURRENT);
+    private int updateMinMasterNodes(int eligibleMasterNodeCount) {
+        assert autoManageMinMasterNodes;
+        final int minMasterNodes = getMinMasterNodes(eligibleMasterNodeCount);
+        if (getMasterNodesCount() > 0) {
+            // there should be at least one master to update
+            logger.debug("updating min_master_nodes to [{}]", minMasterNodes);
+            try {
+                assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(
+                    Settings.builder().put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), minMasterNodes)
+                ));
+            } catch (Exception e) {
+                throw new ElasticsearchException("failed to update minimum master node to [{}] (current masters [{}])", e,
+                    minMasterNodes, getMasterNodesCount());
+            }
+        }
+        return minMasterNodes;
     }
 
-    /**
-     * Starts a node in an async manner with the given settings and returns future with its name.
-     */
-    public synchronized Async<String> startNodeAsync(final Settings settings) {
-        return startNodeAsync(settings, Version.CURRENT);
+    /** calculates a min master nodes value based on the given number of master nodes */
+    private int getMinMasterNodes(int eligibleMasterNodes) {
+        return eligibleMasterNodes / 2 + 1;
     }
 
-    /**
-     * Starts a node in an async manner with the given settings and version and returns future with its name.
-     */
-    public synchronized Async<String> startNodeAsync(final Settings settings, final Version version) {
-        final NodeAndClient buildNode = buildNode(settings);
-        final Future<String> submit = executor.submit(() -> {
-            buildNode.startNode();
-            publishNode(buildNode);
-            return buildNode.name;
-        });
-        return () -> submit.get();
+    private int getMasterNodesCount() {
+        return (int)nodes.values().stream().filter(n -> Node.NODE_MASTER_SETTING.get(n.node().settings())).count();
     }
 
-    /**
-     * Starts multiple nodes in an async manner and returns future with its name.
-     */
-    public synchronized Async<List<String>> startNodesAsync(final int numNodes) {
-        return startNodesAsync(numNodes, Settings.EMPTY, Version.CURRENT);
+    public synchronized String startMasterOnlyNode() {
+        return startMasterOnlyNode(Settings.EMPTY);
     }
 
-    /**
-     * Starts multiple nodes in an async manner with the given settings and returns future with its name.
-     */
-    public synchronized Async<List<String>> startNodesAsync(final int numNodes, final Settings settings) {
-        return startNodesAsync(numNodes, settings, Version.CURRENT);
+    public synchronized String startMasterOnlyNode(Settings settings) {
+        Settings settings1 = Settings.builder().put(settings).put(Node.NODE_MASTER_SETTING.getKey(), true).put(Node.NODE_DATA_SETTING.getKey(), false).build();
+        return startNode(settings1);
     }
 
-    /**
-     * Starts multiple nodes in an async manner with the given settings and version and returns future with its name.
-     */
-    public synchronized Async<List<String>> startNodesAsync(final int numNodes, final Settings settings, final Version version) {
-        final List<Async<String>> asyncs = new ArrayList<>();
-        for (int i = 0; i < numNodes; i++) {
-            asyncs.add(startNodeAsync(settings, version));
-        }
-
-        return () -> {
-            List<String> ids = new ArrayList<>();
-            for (Async<String> async : asyncs) {
-                ids.add(async.get());
-            }
-            return ids;
-        };
+    public synchronized String startDataOnlyNode() {
+        return startDataOnlyNode(Settings.EMPTY);
     }
-
-    /**
-     * Starts multiple nodes (based on the number of settings provided) in an async manner, with explicit settings for each node.
-     * The order of the node names returned matches the order of the settings provided.
-     */
-    public synchronized Async<List<String>> startNodesAsync(final Settings... settings) {
-        List<Async<String>> asyncs = new ArrayList<>();
-        for (Settings setting : settings) {
-            asyncs.add(startNodeAsync(setting, Version.CURRENT));
-        }
-        return () -> {
-            List<String> ids = new ArrayList<>();
-            for (Async<String> async : asyncs) {
-                ids.add(async.get());
-            }
-            return ids;
-        };
+    public synchronized String startDataOnlyNode(Settings settings) {
+        Settings settings1 = Settings.builder().put(settings).put(Node.NODE_MASTER_SETTING.getKey(), false).put(Node.NODE_DATA_SETTING.getKey(), true).build();
+        return startNode(settings1);
     }
 
     private synchronized void publishNode(NodeAndClient nodeAndClient) {
@@ -1648,17 +1797,31 @@ public int numDataAndMasterNodes() {
         return dataAndMasterNodes().size();
     }
 
+    public synchronized int numMasterNodes() {
+      return filterNodes(nodes, NodeAndClient::isMasterEligible).size();
+    }
+
+
     public void setDisruptionScheme(ServiceDisruptionScheme scheme) {
-        clearDisruptionScheme();
+        assert activeDisruptionScheme == null :
+            "there is already and active disruption [" + activeDisruptionScheme + "]. call clearDisruptionScheme first";
         scheme.applyToCluster(this);
         activeDisruptionScheme = scheme;
     }
 
     public void clearDisruptionScheme() {
+        clearDisruptionScheme(true);
+    }
+
+    public void clearDisruptionScheme(boolean ensureHealthyCluster) {
         if (activeDisruptionScheme != null) {
             TimeValue expectedHealingTime = activeDisruptionScheme.expectedTimeToHeal();
             logger.info("Clearing active scheme {}, expected healing time {}", activeDisruptionScheme, expectedHealingTime);
-            activeDisruptionScheme.removeAndEnsureHealthy(this);
+            if (ensureHealthyCluster) {
+                activeDisruptionScheme.removeAndEnsureHealthy(this);
+            } else {
+                activeDisruptionScheme.removeFromCluster(this);
+            }
         }
         activeDisruptionScheme = null;
     }
@@ -1711,7 +1874,7 @@ public boolean test(NodeAndClient nodeAndClient) {
     private static final class MasterNodePredicate implements Predicate<NodeAndClient> {
         private final String masterNodeName;
 
-        public MasterNodePredicate(String masterNodeName) {
+        MasterNodePredicate(String masterNodeName) {
             this.masterNodeName = masterNodeName;
         }
 
@@ -1802,8 +1965,7 @@ public static Predicate<Settings> nameFilter(String... nodeName) {
     private static final class NodeNamePredicate implements Predicate<Settings> {
         private final HashSet<String> nodeNames;
 
-
-        public NodeNamePredicate(HashSet<String> nodeNames) {
+        NodeNamePredicate(HashSet<String> nodeNames) {
             this.nodeNames = nodeNames;
         }
 
@@ -1844,14 +2006,8 @@ public boolean clearData(String nodeName) {
             return false;
         }
 
-
-        /**
-         * If this returns <code>false</code> the node with the given node name will not be restarted. It will be
-         * closed and removed from the cluster. Returns <code>true</code> by default.
-         */
-        public boolean doRestart(String nodeName) {
-            return true;
-        }
+        /** returns true if the restart should also validate the cluster has reformed */
+        public boolean validateClusterForming() { return true; }
     }
 
     public Settings getDefaultSettings() {
@@ -1882,12 +2038,9 @@ public void ensureEstimatedStats() {
                 // in an assertBusy loop, so it will try for 10 seconds and
                 // fail if it never reached 0
                 try {
-                    assertBusy(new Runnable() {
-                        @Override
-                        public void run() {
-                            CircuitBreaker reqBreaker = breakerService.getBreaker(CircuitBreaker.REQUEST);
-                            assertThat("Request breaker not reset to 0 on node: " + name, reqBreaker.getUsed(), equalTo(0L));
-                        }
+                    assertBusy(() -> {
+                        CircuitBreaker reqBreaker = breakerService.getBreaker(CircuitBreaker.REQUEST);
+                        assertThat("Request breaker not reset to 0 on node: " + name, reqBreaker.getUsed(), equalTo(0L));
                     });
                 } catch (Exception e) {
                     fail("Exception during check for request breaker reset to 0: " + e);
@@ -1940,14 +2093,4 @@ private void assertRequestsFinished() {
             }
         }
     }
-
-    /**
-     * Simple interface that allows to wait for an async operation to finish
-     *
-     * @param <T> the result of the async execution
-     */
-    public interface Async<T> {
-        T get() throws ExecutionException, InterruptedException;
-    }
-
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/MockIndexEventListener.java b/test/framework/src/main/java/org/elasticsearch/test/MockIndexEventListener.java
index c4a9515f545ac..13caec4adc834 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/MockIndexEventListener.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/MockIndexEventListener.java
@@ -24,14 +24,15 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.shard.IndexEventListener;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.IndexShardState;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason;
 import org.elasticsearch.plugins.Plugin;
 
 import java.util.Arrays;
@@ -133,13 +134,13 @@ public void beforeIndexShardCreated(ShardId shardId, Settings indexSettings) {
         }
 
         @Override
-        public void beforeIndexClosed(IndexService indexService) {
-            delegate.beforeIndexClosed(indexService);
+        public void beforeIndexRemoved(IndexService indexService, IndexRemovalReason reason) {
+            delegate.beforeIndexRemoved(indexService, reason);
         }
 
         @Override
-        public void afterIndexClosed(Index index, Settings indexSettings) {
-            delegate.afterIndexClosed(index, indexSettings);
+        public void afterIndexRemoved(Index index, IndexSettings indexSettings, IndexRemovalReason reason) {
+            delegate.afterIndexRemoved(index, indexSettings, reason);
         }
 
         @Override
@@ -152,16 +153,6 @@ public void afterIndexShardDeleted(ShardId shardId, Settings indexSettings) {
             delegate.afterIndexShardDeleted(shardId, indexSettings);
         }
 
-        @Override
-        public void afterIndexDeleted(Index index, Settings indexSettings) {
-            delegate.afterIndexDeleted(index, indexSettings);
-        }
-
-        @Override
-        public void beforeIndexDeleted(IndexService indexService) {
-            delegate.beforeIndexDeleted(indexService);
-        }
-
         @Override
         public void beforeIndexAddedToCluster(Index index, Settings indexSettings) {
             delegate.beforeIndexAddedToCluster(index, indexSettings);
diff --git a/core/src/test/java/org/elasticsearch/test/MockLogAppender.java b/test/framework/src/main/java/org/elasticsearch/test/MockLogAppender.java
similarity index 80%
rename from core/src/test/java/org/elasticsearch/test/MockLogAppender.java
rename to test/framework/src/main/java/org/elasticsearch/test/MockLogAppender.java
index 786319e2aeefe..b35dc9563ce5c 100644
--- a/core/src/test/java/org/elasticsearch/test/MockLogAppender.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/MockLogAppender.java
@@ -26,9 +26,11 @@
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.regex.Pattern;
 
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.MatcherAssert.assertThat;
+import static org.junit.Assert.assertTrue;
 
 /**
  * Test appender that can be used to verify that certain events were logged correctly
@@ -40,7 +42,7 @@ public class MockLogAppender extends AbstractAppender {
     private List<LoggingExpectation> expectations;
 
     public MockLogAppender() throws IllegalAccessException {
-        super("mock", RegexFilter.createFilter(".*(\n.*)*", new String[0], true, null, null), null);
+        super("mock", RegexFilter.createFilter(".*(\n.*)*", new String[0], false, null, null), null);
         expectations = new ArrayList<>();
     }
 
@@ -122,6 +124,37 @@ public void assertMatched() {
         }
     }
 
+    public static class PatternSeenEventExcpectation implements LoggingExpectation {
+
+        protected final String name;
+        protected final String logger;
+        protected final Level level;
+        protected final String pattern;
+        volatile boolean saw;
+
+        public PatternSeenEventExcpectation(String name, String logger, Level level, String pattern) {
+            this.name = name;
+            this.logger = logger;
+            this.level = level;
+            this.pattern = pattern;
+        }
+
+        @Override
+        public void match(LogEvent event) {
+            if (event.getLevel().equals(level) && event.getLoggerName().equals(logger)) {
+                if (Pattern.matches(pattern, event.getMessage().getFormattedMessage())) {
+                    saw = true;
+                }
+            }
+        }
+
+        @Override
+        public void assertMatched() {
+            assertThat(name, saw, equalTo(true));
+        }
+
+    }
+
     private static String getLoggerName(String name) {
         if (name.startsWith("org.elasticsearch.")) {
             name = name.substring("org.elasticsearch.".length());
diff --git a/test/framework/src/main/java/org/elasticsearch/test/OldIndexUtils.java b/test/framework/src/main/java/org/elasticsearch/test/OldIndexUtils.java
index fe46251e3eea2..81d8fa84a19a8 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/OldIndexUtils.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/OldIndexUtils.java
@@ -30,11 +30,13 @@
 import org.elasticsearch.action.admin.indices.upgrade.get.IndexUpgradeStatus;
 import org.elasticsearch.action.admin.indices.upgrade.get.UpgradeStatusResponse;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider;
 import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.IndexFolderUpgrader;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.MergePolicyConfig;
 import org.elasticsearch.index.engine.Segment;
@@ -56,7 +58,10 @@
 import static junit.framework.TestCase.assertTrue;
 import static org.elasticsearch.test.ESTestCase.randomInt;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.MatcherAssert.assertThat;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
 
 
 public class OldIndexUtils {
@@ -103,10 +108,37 @@ public static void loadIndex(String indexName, String indexFile, Path unzipDir,
             throw new IllegalStateException("Backwards index must contain exactly one cluster");
         }
 
-        // the bwc scripts packs the indices under this path
-        Path src = list[0].resolve("nodes/0/indices/" + indexName);
-        assertTrue("[" + indexFile + "] missing index dir: " + src.toString(), Files.exists(src));
-        copyIndex(logger, src, indexName, paths);
+        final Path src = getIndexDir(logger, indexName, indexFile, list[0]);
+        copyIndex(logger, src, src.getFileName().toString(), paths);
+    }
+
+    public static Path getIndexDir(
+        final Logger logger,
+        final String indexName,
+        final String indexFile,
+        final Path dataDir) throws IOException {
+        final Version version = Version.fromString(indexName.substring("index-".length()));
+        if (version.before(Version.V_5_0_0_alpha1)) {
+            // the bwc scripts packs the indices under this path
+            Path src = dataDir.resolve("nodes/0/indices/" + indexName);
+            assertTrue("[" + indexFile + "] missing index dir: " + src.toString(), Files.exists(src));
+            return src;
+        } else {
+            final List<Path> indexFolders = new ArrayList<>();
+            try (DirectoryStream<Path> stream = Files.newDirectoryStream(dataDir.resolve("0/indices"),
+                (p) -> p.getFileName().toString().startsWith("extra") == false)) { // extra FS can break this...
+                for (final Path path : stream) {
+                    indexFolders.add(path);
+                }
+            }
+            assertThat(indexFolders.toString(), indexFolders.size(), equalTo(1));
+            final IndexMetaData indexMetaData = IndexMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY,
+                indexFolders.get(0));
+            assertNotNull(indexMetaData);
+            assertThat(indexFolders.get(0).getFileName().toString(), equalTo(indexMetaData.getIndexUUID()));
+            assertThat(indexMetaData.getCreationVersion(), equalTo(version));
+            return indexFolders.get(0);
+        }
     }
 
     public static void assertNotUpgraded(Client client, String... index) throws Exception {
@@ -128,10 +160,10 @@ public static Collection<IndexUpgradeStatus> getUpgradeStatus(Client client, Str
     }
 
     // randomly distribute the files from src over dests paths
-    public static void copyIndex(final Logger logger, final Path src, final String indexName, final Path... dests) throws IOException {
+    public static void copyIndex(final Logger logger, final Path src, final String folderName, final Path... dests) throws IOException {
         Path destinationDataPath = dests[randomInt(dests.length - 1)];
         for (Path dest : dests) {
-            Path indexDir = dest.resolve(indexName);
+            Path indexDir = dest.resolve(folderName);
             assertFalse(Files.exists(indexDir));
             Files.createDirectories(indexDir);
         }
@@ -140,7 +172,7 @@ public static void copyIndex(final Logger logger, final Path src, final String i
             public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
                 Path relativeDir = src.relativize(dir);
                 for (Path dest : dests) {
-                    Path destDir = dest.resolve(indexName).resolve(relativeDir);
+                    Path destDir = dest.resolve(folderName).resolve(relativeDir);
                     Files.createDirectories(destDir);
                 }
                 return FileVisitResult.CONTINUE;
@@ -155,7 +187,7 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO
                 }
 
                 Path relativeFile = src.relativize(file);
-                Path destFile = destinationDataPath.resolve(indexName).resolve(relativeFile);
+                Path destFile = destinationDataPath.resolve(folderName).resolve(relativeFile);
                 logger.trace("--> Moving {} to {}", relativeFile, destFile);
                 Files.move(file, destFile);
                 assertFalse(Files.exists(file));
diff --git a/test/framework/src/main/java/org/elasticsearch/test/PosixPermissionsResetter.java b/test/framework/src/main/java/org/elasticsearch/test/PosixPermissionsResetter.java
index a644205bad96a..19bea3802f07d 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/PosixPermissionsResetter.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/PosixPermissionsResetter.java
@@ -18,6 +18,8 @@
  */
 package org.elasticsearch.test;
 
+import java.util.EnumSet;
+import java.util.Set;
 import org.junit.Assert;
 
 import java.io.IOException;
@@ -25,8 +27,6 @@
 import java.nio.file.Path;
 import java.nio.file.attribute.PosixFileAttributeView;
 import java.nio.file.attribute.PosixFilePermission;
-import java.util.HashSet;
-import java.util.Set;
 
 /** Stores the posix attributes for a path and resets them on close. */
 public class PosixPermissionsResetter implements AutoCloseable {
@@ -46,6 +46,6 @@ public void setPermissions(Set<PosixFilePermission> newPermissions) throws IOExc
     }
 
     public Set<PosixFilePermission> getCopyPermissions() {
-        return new HashSet<>(permissions);
+        return EnumSet.copyOf(permissions);
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/RandomObjects.java b/test/framework/src/main/java/org/elasticsearch/test/RandomObjects.java
new file mode 100644
index 0000000000000..c963103436389
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/test/RandomObjects.java
@@ -0,0 +1,334 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test;
+
+import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+import com.carrotsearch.randomizedtesting.generators.RandomStrings;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.support.replication.ReplicationResponse.ShardInfo;
+import org.elasticsearch.action.support.replication.ReplicationResponse.ShardInfo.Failure;
+import org.elasticsearch.cluster.block.ClusterBlockException;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.discovery.DiscoverySettings;
+import org.elasticsearch.index.shard.IndexShardRecoveringException;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.index.shard.ShardNotFoundException;
+import org.elasticsearch.rest.RestStatus;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Base64;
+import java.util.List;
+import java.util.Random;
+
+import static com.carrotsearch.randomizedtesting.generators.RandomNumbers.randomIntBetween;
+import static com.carrotsearch.randomizedtesting.generators.RandomStrings.randomAsciiOfLength;
+import static com.carrotsearch.randomizedtesting.generators.RandomStrings.randomUnicodeOfLengthBetween;
+import static java.util.Collections.singleton;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.INDEX_UUID_NA_VALUE;
+import static org.elasticsearch.test.ESTestCase.randomFrom;
+
+public final class RandomObjects {
+
+    private RandomObjects() {
+
+    }
+
+    /**
+     * Returns a tuple containing random stored field values and their corresponding expected values once printed out
+     * via {@link org.elasticsearch.common.xcontent.ToXContent#toXContent(XContentBuilder, ToXContent.Params)} and parsed back via
+     * {@link org.elasticsearch.common.xcontent.XContentParser#objectText()}.
+     * Generates values based on what can get printed out. Stored fields values are retrieved from lucene and converted via
+     * {@link org.elasticsearch.index.mapper.MappedFieldType#valueForDisplay(Object)} to either strings, numbers or booleans.
+     *
+     * @param random Random generator
+     * @param xContentType the content type, used to determine what the expected values are for float numbers.
+     */
+    public static Tuple<List<Object>, List<Object>> randomStoredFieldValues(Random random, XContentType xContentType) {
+        int numValues = randomIntBetween(random, 1, 5);
+        List<Object> originalValues = new ArrayList<>();
+        List<Object> expectedParsedValues = new ArrayList<>();
+        int dataType = randomIntBetween(random, 0, 8);
+        for (int i = 0; i < numValues; i++) {
+            switch(dataType) {
+                case 0:
+                    long randomLong = random.nextLong();
+                    originalValues.add(randomLong);
+                    expectedParsedValues.add(randomLong);
+                    break;
+                case 1:
+                    int randomInt = random.nextInt();
+                    originalValues.add(randomInt);
+                    expectedParsedValues.add(randomInt);
+                    break;
+                case 2:
+                    Short randomShort = (short) random.nextInt();
+                    originalValues.add(randomShort);
+                    expectedParsedValues.add(randomShort.intValue());
+                    break;
+                case 3:
+                    Byte randomByte = (byte)random.nextInt();
+                    originalValues.add(randomByte);
+                    expectedParsedValues.add(randomByte.intValue());
+                    break;
+                case 4:
+                    double randomDouble = random.nextDouble();
+                    originalValues.add(randomDouble);
+                    expectedParsedValues.add(randomDouble);
+                    break;
+                case 5:
+                    Float randomFloat = random.nextFloat();
+                    originalValues.add(randomFloat);
+                    if (xContentType == XContentType.CBOR) {
+                        //with CBOR we get back a float
+                        expectedParsedValues.add(randomFloat);
+                    } else if (xContentType == XContentType.SMILE) {
+                        //with SMILE we get back a double
+                        expectedParsedValues.add(randomFloat.doubleValue());
+                    } else {
+                        //with JSON AND YAML we get back a double, but with float precision.
+                        expectedParsedValues.add(Double.parseDouble(randomFloat.toString()));
+                    }
+                    break;
+                case 6:
+                    boolean randomBoolean = random.nextBoolean();
+                    originalValues.add(randomBoolean);
+                    expectedParsedValues.add(randomBoolean);
+                    break;
+                case 7:
+                    String randomString = random.nextBoolean() ? RandomStrings.randomAsciiOfLengthBetween(random, 3, 10 ) :
+                            randomUnicodeOfLengthBetween(random, 3, 10);
+                    originalValues.add(randomString);
+                    expectedParsedValues.add(randomString);
+                    break;
+                case 8:
+                    byte[] randomBytes = RandomStrings.randomUnicodeOfLengthBetween(random, 10, 50).getBytes(StandardCharsets.UTF_8);
+                    BytesArray randomBytesArray = new BytesArray(randomBytes);
+                    originalValues.add(randomBytesArray);
+                    if (xContentType == XContentType.JSON || xContentType == XContentType.YAML) {
+                        //JSON and YAML write the base64 format
+                        expectedParsedValues.add(Base64.getEncoder().encodeToString(randomBytes));
+                    } else {
+                        //SMILE and CBOR write the original bytes as they support binary format
+                        expectedParsedValues.add(randomBytesArray);
+                    }
+                    break;
+                default:
+                    throw new UnsupportedOperationException();
+            }
+        }
+        return Tuple.tuple(originalValues, expectedParsedValues);
+    }
+
+    /**
+     * Returns a random source containing a random number of fields, objects and array, with maximum depth 5.
+     *
+     * @param random Random generator
+     */
+    public static BytesReference randomSource(Random random) {
+        //the source can be stored in any format and eventually converted when retrieved depending on the format of the response
+        return randomSource(random, RandomPicks.randomFrom(random, XContentType.values()));
+    }
+
+    /**
+     * Returns a random source in a given XContentType containing a random number of fields, objects and array, with maximum depth 5.
+     * The minimum number of fields per object is 1.
+     *
+     * @param random Random generator
+     */
+    public static BytesReference randomSource(Random random, XContentType xContentType) {
+        return randomSource(random, xContentType, 1);
+    }
+
+    /**
+     * Returns a random source in a given XContentType containing a random number of fields, objects and array, with maximum depth 5.
+     * The minimum number of fields per object is provided as an argument.
+     *
+     * @param random Random generator
+     */
+    public static BytesReference randomSource(Random random, XContentType xContentType, int minNumFields) {
+        try (XContentBuilder builder = XContentFactory.contentBuilder(xContentType)) {
+            builder.startObject();
+            addFields(random, builder, minNumFields, 0);
+            builder.endObject();
+            return builder.bytes();
+        } catch(IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * Randomly adds fields, objects, or arrays to the provided builder. The maximum depth is 5.
+     */
+    private static void addFields(Random random, XContentBuilder builder, int minNumFields, int currentDepth) throws IOException {
+        int numFields = randomIntBetween(random, minNumFields, 10);
+        for (int i = 0; i < numFields; i++) {
+            if (currentDepth < 5 && random.nextBoolean()) {
+                if (random.nextBoolean()) {
+                    builder.startObject(RandomStrings.randomAsciiOfLengthBetween(random, 4, 10));
+                    addFields(random, builder, minNumFields, currentDepth + 1);
+                    builder.endObject();
+                } else {
+                    builder.startArray(RandomStrings.randomAsciiOfLengthBetween(random, 4, 10));
+                    int numElements = randomIntBetween(random, 1, 5);
+                    boolean object = random.nextBoolean();
+                    int dataType = -1;
+                    if (object == false) {
+                        dataType = randomDataType(random);
+                    }
+                    for (int j = 0; j < numElements; j++) {
+                        if (object) {
+                            builder.startObject();
+                            addFields(random, builder, minNumFields, 5);
+                            builder.endObject();
+                        } else {
+                            builder.value(randomFieldValue(random, dataType));
+                        }
+                    }
+                    builder.endArray();
+                }
+            } else {
+                builder.field(RandomStrings.randomAsciiOfLengthBetween(random, 4, 10),
+                        randomFieldValue(random, randomDataType(random)));
+            }
+        }
+    }
+
+    private static int randomDataType(Random random) {
+        return randomIntBetween(random, 0, 3);
+    }
+
+    private static Object randomFieldValue(Random random, int dataType) {
+        switch(dataType) {
+            case 0:
+                return RandomStrings.randomAsciiOfLengthBetween(random, 3, 10);
+            case 1:
+                return RandomStrings.randomAsciiOfLengthBetween(random, 3, 10);
+            case 2:
+                return random.nextLong();
+            case 3:
+                return random.nextDouble();
+            default:
+                throw new UnsupportedOperationException();
+        }
+    }
+
+    /**
+     * Returns a tuple that contains a randomized {@link ShardInfo} value (left side) and its corresponding
+     * value (right side) after it has been printed out as a {@link ToXContent} and parsed back using a parsing
+     * method like {@link ShardInfo#fromXContent(XContentParser)}. The ShardInfo randomly contains shard failures.
+     *
+     * @param random Random generator
+     */
+    public static Tuple<ShardInfo, ShardInfo> randomShardInfo(Random random) {
+        return randomShardInfo(random, random.nextBoolean());
+    }
+
+    /**
+     * Returns a tuple that contains a randomized {@link ShardInfo} value (left side) and its corresponding
+     * value (right side) after it has been printed out as a {@link ToXContent} and parsed back using a parsing
+     * method like {@link ShardInfo#fromXContent(XContentParser)}. A `withShardFailures` parameter indicates if
+     * the randomized ShardInfo must or must not contain shard failures.
+     *
+     * @param random            Random generator
+     * @param withShardFailures indicates if the generated ShardInfo must contain shard failures
+     */
+    public static Tuple<ShardInfo, ShardInfo> randomShardInfo(Random random, boolean withShardFailures) {
+        int total = randomIntBetween(random, 1, 10);
+        if (withShardFailures == false) {
+            return Tuple.tuple(new ShardInfo(total, total), new ShardInfo(total, total));
+        }
+
+        int successful = randomIntBetween(random, 1, Math.max(1, (total - 1)));
+        int failures = Math.max(1, (total - successful));
+
+        Failure[] actualFailures = new Failure[failures];
+        Failure[] expectedFailures = new Failure[failures];
+
+        for (int i = 0; i < failures; i++) {
+            Tuple<Failure, Failure> failure = randomShardInfoFailure(random);
+            actualFailures[i] = failure.v1();
+            expectedFailures[i] = failure.v2();
+        }
+        return Tuple.tuple(new ShardInfo(total, successful, actualFailures), new ShardInfo(total, successful, expectedFailures));
+    }
+
+    /**
+     * Returns a tuple that contains a randomized {@link Failure} value (left side) and its corresponding
+     * value (right side) after it has been printed out as a {@link ToXContent} and parsed back using a parsing
+     * method like {@link ShardInfo.Failure#fromXContent(XContentParser)}.
+     *
+     * @param random Random generator
+     */
+    private static Tuple<Failure, Failure> randomShardInfoFailure(Random random) {
+        String index = randomAsciiOfLength(random, 5);
+        String indexUuid = randomAsciiOfLength(random, 5);
+        int shardId = randomIntBetween(random, 1, 10);
+        String nodeId = randomAsciiOfLength(random, 5);
+        RestStatus status = randomFrom(random, RestStatus.INTERNAL_SERVER_ERROR, RestStatus.FORBIDDEN, RestStatus.NOT_FOUND);
+        boolean primary = random.nextBoolean();
+        ShardId shard = new ShardId(index, indexUuid, shardId);
+
+        Exception actualException;
+        ElasticsearchException expectedException;
+
+        int type = randomIntBetween(random, 0, 3);
+        switch (type) {
+            case 0:
+                actualException = new ClusterBlockException(singleton(DiscoverySettings.NO_MASTER_BLOCK_WRITES));
+                expectedException = new ElasticsearchException("Elasticsearch exception [type=cluster_block_exception, " +
+                        "reason=blocked by: [SERVICE_UNAVAILABLE/2/no master];]");
+                break;
+            case 1:
+                actualException = new ShardNotFoundException(shard);
+                expectedException = new ElasticsearchException("Elasticsearch exception [type=shard_not_found_exception, " +
+                        "reason=no such shard]");
+                expectedException.setShard(shard);
+                break;
+            case 2:
+                actualException = new IllegalArgumentException("Closed resource", new RuntimeException("Resource"));
+                expectedException = new ElasticsearchException("Elasticsearch exception [type=illegal_argument_exception, " +
+                        "reason=Closed resource]",
+                        new ElasticsearchException("Elasticsearch exception [type=runtime_exception, reason=Resource]"));
+                break;
+            case 3:
+                actualException = new IndexShardRecoveringException(shard);
+                expectedException = new ElasticsearchException("Elasticsearch exception [type=index_shard_recovering_exception, " +
+                        "reason=CurrentState[RECOVERING] Already recovering]");
+                expectedException.setShard(shard);
+                break;
+            default:
+                throw new UnsupportedOperationException("No randomized exceptions generated for type [" + type + "]");
+        }
+
+        Failure actual = new Failure(shard, nodeId, actualException, status, primary);
+        Failure expected = new Failure(new ShardId(index, INDEX_UUID_NA_VALUE, shardId), nodeId, expectedException, status, primary);
+
+        return Tuple.tuple(actual, expected);
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/TestCluster.java
index 124960fe921e5..c2ac65d9980e1 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/TestCluster.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/TestCluster.java
@@ -82,7 +82,7 @@ public void wipe(Set<String> excludeTemplates) {
     /**
      * Assertions that should run before the cluster is wiped should be called in this method
      */
-    public void beforeIndexDeletion() {
+    public void beforeIndexDeletion() throws Exception {
     }
 
     /**
diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestCustomMetaData.java b/test/framework/src/main/java/org/elasticsearch/test/TestCustomMetaData.java
index 92d5b95cfac69..fb709a7fd06d9 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/TestCustomMetaData.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/TestCustomMetaData.java
@@ -21,15 +21,21 @@
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.cluster.AbstractDiffable;
+import org.elasticsearch.cluster.AbstractNamedDiffable;
+import org.elasticsearch.cluster.Diffable;
+import org.elasticsearch.cluster.NamedDiff;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
+import java.util.function.Function;
+import java.util.function.Supplier;
 
-public abstract class TestCustomMetaData extends AbstractDiffable<MetaData.Custom> implements MetaData.Custom {
+public abstract class TestCustomMetaData extends AbstractNamedDiffable<MetaData.Custom> implements MetaData.Custom {
     private final String data;
 
     protected TestCustomMetaData(String data) {
@@ -57,11 +63,12 @@ public int hashCode() {
         return data.hashCode();
     }
 
-    protected abstract TestCustomMetaData newTestCustomMetaData(String data);
+    protected static <T extends TestCustomMetaData> T readFrom(Function<String, T> supplier, StreamInput in) throws IOException {
+        return supplier.apply(in.readString());
+    }
 
-    @Override
-    public MetaData.Custom readFrom(StreamInput in) throws IOException {
-        return newTestCustomMetaData(in.readString());
+    public static NamedDiff<MetaData.Custom> readDiffFrom(String name, StreamInput in)  throws IOException {
+        return readDiffFrom(MetaData.Custom.class, name, in);
     }
 
     @Override
@@ -69,8 +76,9 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeString(getData());
     }
 
-    @Override
-    public MetaData.Custom fromXContent(XContentParser parser) throws IOException {
+    @SuppressWarnings("unchecked")
+    public static <T extends MetaData.Custom> T fromXContent(Function<String, MetaData.Custom> supplier, XContentParser parser)
+        throws IOException {
         XContentParser.Token token;
         String data = null;
         while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
@@ -91,7 +99,7 @@ public MetaData.Custom fromXContent(XContentParser parser) throws IOException {
         if (data == null) {
             throw new ElasticsearchParseException("failed to parse snapshottable metadata, data not found");
         }
-        return newTestCustomMetaData(data);
+        return (T) supplier.apply(data);
     }
 
     @Override
@@ -99,4 +107,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.field("data", getData());
         return builder;
     }
+
+    @Override
+    public String toString() {
+        return "[" + getWriteableName() + "][" + data +"]";
+    }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java
index cc325d2e60f9a..f77414e7d50b3 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java
@@ -22,12 +22,11 @@
 import org.apache.lucene.search.FieldDoc;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.Counter;
+import org.elasticsearch.action.search.SearchTask;
 import org.elasticsearch.action.search.SearchType;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.index.IndexService;
-import org.elasticsearch.index.analysis.AnalysisService;
 import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.fielddata.IndexFieldDataService;
@@ -36,17 +35,17 @@
 import org.elasticsearch.index.mapper.ObjectMapper;
 import org.elasticsearch.index.query.ParsedQuery;
 import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.search.fetch.StoredFieldsContext;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.similarity.SimilarityService;
-import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.search.SearchExtBuilder;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.aggregations.SearchContextAggregations;
+import org.elasticsearch.search.collapse.CollapseContext;
 import org.elasticsearch.search.dfs.DfsSearchResult;
 import org.elasticsearch.search.fetch.FetchPhase;
 import org.elasticsearch.search.fetch.FetchSearchResult;
-import org.elasticsearch.search.fetch.FetchSubPhase;
-import org.elasticsearch.search.fetch.FetchSubPhaseContext;
+import org.elasticsearch.search.fetch.StoredFieldsContext;
+import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 import org.elasticsearch.search.fetch.subphase.ScriptFieldsContext;
 import org.elasticsearch.search.fetch.subphase.highlight.SearchContextHighlight;
@@ -54,7 +53,6 @@
 import org.elasticsearch.search.internal.ScrollContext;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.internal.ShardSearchRequest;
-import org.elasticsearch.search.lookup.SearchLookup;
 import org.elasticsearch.search.profile.Profilers;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.search.rescore.RescoreSearchContext;
@@ -78,11 +76,11 @@ public class TestSearchContext extends SearchContext {
     final Counter timeEstimateCounter = Counter.newCounter();
     final QuerySearchResult queryResult = new QuerySearchResult();
     final QueryShardContext queryShardContext;
-    ScriptService scriptService;
     ParsedQuery originalQuery;
     ParsedQuery postFilter;
     Query query;
     Float minScore;
+    SearchTask task;
 
     ContextIndexSearcher searcher;
     int size;
@@ -90,38 +88,34 @@ public class TestSearchContext extends SearchContext {
     private SearchContextAggregations aggregations;
 
     private final long originNanoTime = System.nanoTime();
-    private final Map<String, FetchSubPhaseContext> subPhaseContexts = new HashMap<>();
+    private final Map<String, SearchExtBuilder> searchExtBuilders = new HashMap<>();
 
-    public TestSearchContext(ThreadPool threadPool, BigArrays bigArrays, ScriptService scriptService, IndexService indexService) {
-        super(ParseFieldMatcher.STRICT);
+    public TestSearchContext(ThreadPool threadPool, BigArrays bigArrays, IndexService indexService) {
         this.bigArrays = bigArrays.withCircuitBreaking();
         this.indexService = indexService;
         this.indexFieldDataService = indexService.fieldData();
         this.fixedBitSetFilterCache = indexService.cache().bitsetFilterCache();
         this.threadPool = threadPool;
         this.indexShard = indexService.getShardOrNull(0);
-        this.scriptService = scriptService;
-        queryShardContext = indexService.newQueryShardContext();
+        queryShardContext = indexService.newQueryShardContext(0, null, () -> 0L);
     }
 
     public TestSearchContext(QueryShardContext queryShardContext) {
-        super(ParseFieldMatcher.STRICT);
         this.bigArrays = null;
         this.indexService = null;
         this.indexFieldDataService = null;
         this.threadPool = null;
         this.fixedBitSetFilterCache = null;
         this.indexShard = null;
-        scriptService = null;
         this.queryShardContext = queryShardContext;
     }
 
     @Override
-    public void preProcess() {
+    public void preProcess(boolean rewrite) {
     }
 
     @Override
-    public Query searchFilter(String[] types) {
+    public Query buildFilteredQuery(Query query) {
         return null;
     }
 
@@ -160,21 +154,11 @@ public float queryBoost() {
         return 0;
     }
 
-    @Override
-    public SearchContext queryBoost(float queryBoost) {
-        return null;
-    }
-
     @Override
     public long getOriginNanoTime() {
         return originNanoTime;
     }
 
-    @Override
-    protected long nowInMillisImpl() {
-        return 0;
-    }
-
     @Override
     public ScrollContext scrollContext() {
         return null;
@@ -197,12 +181,13 @@ public SearchContext aggregations(SearchContextAggregations aggregations) {
     }
 
     @Override
-    public <SubPhaseContext extends FetchSubPhaseContext> SubPhaseContext getFetchSubPhaseContext(FetchSubPhase.ContextFactory<SubPhaseContext> contextFactory) {
-        String subPhaseName = contextFactory.getName();
-        if (subPhaseContexts.get(subPhaseName) == null) {
-            subPhaseContexts.put(subPhaseName, contextFactory.newContextInstance());
-        }
-        return (SubPhaseContext) subPhaseContexts.get(subPhaseName);
+    public void addSearchExt(SearchExtBuilder searchExtBuilder) {
+        searchExtBuilders.put(searchExtBuilder.getWriteableName(), searchExtBuilder);
+    }
+
+    @Override
+    public SearchExtBuilder getSearchExt(String name) {
+        return searchExtBuilders.get(name);
     }
 
     @Override
@@ -262,6 +247,16 @@ public SearchContext fetchSourceContext(FetchSourceContext fetchSourceContext) {
         return null;
     }
 
+    @Override
+    public DocValueFieldsContext docValueFieldsContext() {
+        return null;
+    }
+
+    @Override
+    public SearchContext docValueFieldsContext(DocValueFieldsContext docValueFieldsContext) {
+        return null;
+    }
+
     @Override
     public ContextIndexSearcher searcher() {
         return searcher;
@@ -284,19 +279,11 @@ public MapperService mapperService() {
         return null;
     }
 
-    @Override
-    public AnalysisService analysisService() { return indexService.analysisService();}
-
     @Override
     public SimilarityService similarityService() {
         return null;
     }
 
-    @Override
-    public ScriptService scriptService() {
-        return scriptService;
-    }
-
     @Override
     public BigArrays bigArrays() {
         return bigArrays;
@@ -331,6 +318,11 @@ public void terminateAfter(int terminateAfter) {
         this.terminateAfter = terminateAfter;
     }
 
+    @Override
+    public boolean lowLevelCancellation() {
+        return false;
+    }
+
     @Override
     public SearchContext minimumScore(float minimumScore) {
         this.minScore = minimumScore;
@@ -372,6 +364,16 @@ public FieldDoc searchAfter() {
         return null;
     }
 
+    @Override
+    public SearchContext collapse(CollapseContext collapse) {
+        return null;
+    }
+
+    @Override
+    public CollapseContext collapse() {
+        return null;
+    }
+
     @Override
     public SearchContext parsedPostFilter(ParsedQuery postFilter) {
         this.postFilter = postFilter;
@@ -520,11 +522,6 @@ public long keepAlive() {
     public void keepAlive(long keepAlive) {
     }
 
-    @Override
-    public SearchLookup lookup() {
-        return new SearchLookup(mapperService(), fieldData(), null);
-    }
-
     @Override
     public DfsSearchResult dfsResult() {
         return null;
@@ -583,4 +580,18 @@ public QueryShardContext getQueryShardContext() {
         return queryShardContext;
     }
 
+    @Override
+    public void setTask(SearchTask task) {
+        this.task = task;
+    }
+
+    @Override
+    public SearchTask getTask() {
+        return task;
+    }
+
+    @Override
+    public boolean isCancelled() {
+        return task.isCancelled();
+    }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java b/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java
index d09c763322c49..69ac1e3abc0be 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java
@@ -27,47 +27,94 @@
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Optional;
 import java.util.Random;
 import java.util.Set;
+import java.util.stream.Collectors;
+
+import static java.util.Collections.unmodifiableList;
 
 /** Utilities for selecting versions in tests */
 public class VersionUtils {
 
-    private static final List<Version> SORTED_VERSIONS;
+    private static final List<Version> RELEASED_VERSIONS;
+    private static final List<Version> UNRELEASED_VERSIONS;
+    private static final List<Version> ALL_VERSIONS;
+
     static {
-        Field[] declaredFields = Version.class.getFields();
-        Set<Integer> ids = new HashSet<>();
-        for (Field field : declaredFields) {
+        final Field[] declaredFields = Version.class.getFields();
+        final Set<Integer> releasedIdsSet = new HashSet<>();
+        final Set<Integer> unreleasedIdsSet = new HashSet<>();
+        for (final Field field : declaredFields) {
             final int mod = field.getModifiers();
             if (Modifier.isStatic(mod) && Modifier.isFinal(mod) && Modifier.isPublic(mod)) {
                 if (field.getType() == Version.class) {
+                    final int id;
                     try {
-                        Version object = (Version) field.get(null);
-                        ids.add(object.id);
-                    } catch (IllegalAccessException e) {
+                        id = ((Version) field.get(null)).id;
+                    } catch (final IllegalAccessException e) {
                         throw new RuntimeException(e);
                     }
+                    assert field.getName().matches("(V(_\\d+)+(_(alpha|beta|rc)\\d+)?(_UNRELEASED)?|CURRENT)") : field.getName();
+                    // note that below we remove CURRENT and add it to released; we do it this way because there are two constants that
+                    // correspond to CURRENT, CURRENT itself and the actual version that CURRENT points to
+                    if (field.getName().equals("CURRENT") || field.getName().endsWith("UNRELEASED")) {
+                        unreleasedIdsSet.add(id);
+                    } else {
+                        releasedIdsSet.add(id);
+                    }
                 }
             }
         }
-        List<Integer> idList = new ArrayList<>(ids);
-        Collections.sort(idList);
-        List<Version> version = new ArrayList<>();
-        for (Integer integer : idList) {
-            version.add(Version.fromId(integer));
-        }
-        SORTED_VERSIONS = Collections.unmodifiableList(version);
+
+        // treat CURRENT as released for BWC testing
+        unreleasedIdsSet.remove(Version.CURRENT.id);
+        releasedIdsSet.add(Version.CURRENT.id);
+
+        // unreleasedIdsSet and releasedIdsSet should be disjoint
+        assert unreleasedIdsSet.stream().filter(releasedIdsSet::contains).collect(Collectors.toSet()).isEmpty();
+
+        RELEASED_VERSIONS =
+            Collections.unmodifiableList(releasedIdsSet.stream().sorted().map(Version::fromId).collect(Collectors.toList()));
+        UNRELEASED_VERSIONS =
+            Collections.unmodifiableList(unreleasedIdsSet.stream().sorted().map(Version::fromId).collect(Collectors.toList()));
+
+        List<Version> allVersions = new ArrayList<>(RELEASED_VERSIONS.size() + UNRELEASED_VERSIONS.size());
+        allVersions.addAll(RELEASED_VERSIONS);
+        allVersions.addAll(UNRELEASED_VERSIONS);
+        Collections.sort(allVersions);
+        ALL_VERSIONS = unmodifiableList(allVersions);
+    }
+
+    /**
+     * Returns an immutable, sorted list containing all released versions.
+     *
+     * @return all released versions
+     */
+    public static List<Version> allReleasedVersions() {
+        return RELEASED_VERSIONS;
+    }
+
+    /**
+     * Returns an immutable, sorted list containing all unreleased versions.
+     *
+     * @return all unreleased versions
+     */
+    public static List<Version> allUnreleasedVersions() {
+        return UNRELEASED_VERSIONS;
     }
 
-    /** Returns immutable list of all known versions. */
+    /**
+     * Returns an immutable, sorted list containing all versions, both released and unreleased.
+     */
     public static List<Version> allVersions() {
-        return Collections.unmodifiableList(SORTED_VERSIONS);
+        return ALL_VERSIONS;
     }
 
     public static Version getPreviousVersion(Version version) {
-        int index = SORTED_VERSIONS.indexOf(version);
+        int index = RELEASED_VERSIONS.indexOf(version);
         assert index > 0;
-        return SORTED_VERSIONS.get(index - 1);
+        return RELEASED_VERSIONS.get(index - 1);
     }
 
     /** Returns the {@link Version} before the {@link Version#CURRENT} */
@@ -77,25 +124,41 @@ public static Version getPreviousVersion() {
         return version;
     }
 
+    /** Returns the {@link Version} before the {@link Version#CURRENT} where the minor version is less than the currents minor version. */
+    public static Version getPreviousMinorVersion() {
+        Version version = Version.CURRENT;
+        do {
+            version = getPreviousVersion(version);
+            assert version.before(Version.CURRENT);
+        } while (version.minor == Version.CURRENT.minor);
+        return version;
+    }
+
     /** Returns the oldest {@link Version} */
     public static Version getFirstVersion() {
-        return SORTED_VERSIONS.get(0);
+        return RELEASED_VERSIONS.get(0);
     }
 
     /** Returns a random {@link Version} from all available versions. */
     public static Version randomVersion(Random random) {
-        return SORTED_VERSIONS.get(random.nextInt(SORTED_VERSIONS.size()));
+        return RELEASED_VERSIONS.get(random.nextInt(RELEASED_VERSIONS.size()));
+    }
+
+    /** Returns a random {@link Version} from all available versions, that is compatible with the given version. */
+    public static Version randomCompatibleVersion(Random random, Version version) {
+        final List<Version> compatible = ALL_VERSIONS.stream().filter(version::isCompatible).collect(Collectors.toList());
+        return compatible.get(random.nextInt(compatible.size()));
     }
 
     /** Returns a random {@link Version} between <code>minVersion</code> and <code>maxVersion</code> (inclusive). */
     public static Version randomVersionBetween(Random random, Version minVersion, Version maxVersion) {
         int minVersionIndex = 0;
         if (minVersion != null) {
-            minVersionIndex = SORTED_VERSIONS.indexOf(minVersion);
+            minVersionIndex = RELEASED_VERSIONS.indexOf(minVersion);
         }
-        int maxVersionIndex = SORTED_VERSIONS.size() - 1;
+        int maxVersionIndex = RELEASED_VERSIONS.size() - 1;
         if (maxVersion != null) {
-            maxVersionIndex = SORTED_VERSIONS.indexOf(maxVersion);
+            maxVersionIndex = RELEASED_VERSIONS.indexOf(maxVersion);
         }
         if (minVersionIndex == -1) {
             throw new IllegalArgumentException("minVersion [" + minVersion + "] does not exist.");
@@ -106,7 +169,7 @@ public static Version randomVersionBetween(Random random, Version minVersion, Ve
         } else {
             // minVersionIndex is inclusive so need to add 1 to this index
             int range = maxVersionIndex + 1 - minVersionIndex;
-            return SORTED_VERSIONS.get(minVersionIndex + random.nextInt(range));
+            return RELEASED_VERSIONS.get(minVersionIndex + random.nextInt(range));
         }
     }
 
@@ -116,4 +179,20 @@ public static boolean isSnapshot(Version version) {
         }
         return false;
     }
+
+    /** returns the first future incompatible version */
+    public static Version incompatibleFutureVersion(Version version) {
+        final Optional<Version> opt = ALL_VERSIONS.stream().filter(version::before).filter(v -> v.isCompatible(version) == false).findAny();
+        assert opt.isPresent() : "no future incompatible version for " + version;
+        return opt.get();
+    }
+
+    /** Returns the maximum {@link Version} that is compatible with the given version. */
+    public static Version maxCompatibleVersion(Version version) {
+        final List<Version> compatible = ALL_VERSIONS.stream().filter(version::isCompatible).filter(version::onOrBefore)
+            .collect(Collectors.toList());
+        assert compatible.size() > 0;
+        return compatible.get(compatible.size() - 1);
+    }
+
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java b/test/framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java
index 866a19e0a72f6..fe4c397113a34 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java
@@ -19,17 +19,30 @@
 
 package org.elasticsearch.test;
 
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.rest.yaml.ObjectPath;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.Random;
+import java.util.Stack;
+import java.util.function.Predicate;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
 
+import static com.carrotsearch.randomizedtesting.generators.RandomStrings.randomAsciiOfLength;
 import static org.elasticsearch.common.xcontent.ToXContent.EMPTY_PARAMS;
+import static org.elasticsearch.common.xcontent.XContentHelper.createParser;
 
 public final class XContentTestUtils {
     private XContentTestUtils() {
@@ -41,7 +54,7 @@ public static Map<String, Object> convertToMap(ToXContent part) throws IOExcepti
         builder.startObject();
         part.toXContent(builder, EMPTY_PARAMS);
         builder.endObject();
-        return XContentHelper.convertToMap(builder.bytes(), false).v2();
+        return XContentHelper.convertToMap(builder.bytes(), false, builder.contentType()).v2();
     }
 
 
@@ -123,4 +136,161 @@ private static String differenceBetweenObjectsIgnoringArrayOrder(String path, Ob
         }
     }
 
+    /**
+     * This method takes the input xContent data and adds a random field value, inner object or array into each
+     * json object. This can e.g. be used to test if parsers that handle the resulting xContent can handle the
+     * augmented xContent correctly, for example when testing lenient parsing.
+     *
+     * If the xContent output contains objects that should be skipped of such treatment, an optional filtering
+     * {@link Predicate} can be supplied that checks xContent paths that should be excluded from this treatment.
+     *
+     * This predicate should check the xContent path that we want to insert to and return <tt>true</tt> if the
+     * path should be excluded. Paths are string concatenating field names and array indices, so e.g. in:
+     *
+     * <pre>
+     * {
+     *      "foo1 : {
+     *          "bar" : [
+     *              { ... },
+     *              { ... },
+     *              {
+     *                  "baz" : {
+     *                      // insert here
+     *                  }
+     *              }
+     *          ]
+     *      }
+     * }
+     * </pre>
+     *
+     * "foo1.bar.2.baz" would point to the desired insert location.
+     *
+     * To exclude inserting into the "foo1" object we would user a {@link Predicate} like
+     * <pre>
+     * {@code
+     *      (path) -> path.endsWith("foo1")
+     * }
+     * </pre>
+     *
+     * or if we don't want any random insertions in the "foo1" tree we could use
+     * <pre>
+     * {@code
+     *      (path) -> path.contains("foo1")
+     * }
+     * </pre>
+     */
+    public static BytesReference insertRandomFields(XContentType contentType, BytesReference xContent, Predicate<String> excludeFilter,
+            Random random) throws IOException {
+        List<String> insertPaths;
+
+        // we can use NamedXContentRegistry.EMPTY here because we only traverse the xContent once and don't use it
+        try (XContentParser parser = createParser(NamedXContentRegistry.EMPTY, xContent, contentType)) {
+            parser.nextToken();
+            List<String> possiblePaths = XContentTestUtils.getInsertPaths(parser, new Stack<>());
+            if (excludeFilter == null) {
+                insertPaths = possiblePaths;
+            } else {
+                insertPaths = new ArrayList<>();
+                possiblePaths.stream().filter(excludeFilter.negate()).forEach(insertPaths::add);
+            }
+        }
+
+        Supplier<Object> value = () -> {
+            List<Object> randomValues = RandomObjects.randomStoredFieldValues(random, contentType).v1();
+            if (random.nextBoolean()) {
+                return randomValues.get(0);
+            } else {
+                if (random.nextBoolean()) {
+                    return randomValues.stream().collect(Collectors.toMap(obj -> randomAsciiOfLength(random, 10), obj -> obj));
+                } else {
+                    return randomValues;
+                }
+            }
+        };
+        return XContentTestUtils
+                .insertIntoXContent(contentType.xContent(), xContent, insertPaths, () -> randomAsciiOfLength(random, 10), value).bytes();
+    }
+
+    /**
+     * This utility method takes an XContentParser and walks the xContent structure to find all
+     * possible paths to where a new object or array starts. This can be used in tests that add random
+     * xContent values to test parsing code for errors or to check their robustness against new fields.
+     *
+     * The path uses dot separated fieldnames and numbers for array indices, similar to what we do in
+     * {@link ObjectPath}.
+     *
+     * The {@link Stack} passed in should initially be empty, it gets pushed to by recursive calls
+     *
+     * As an example, the following json xContent:
+     * <pre>
+     *     {
+     *         "foo" : "bar",
+     *         "foo1" : [ 1, { "foo2" : "baz" }, 3, 4]
+     *         "foo3" : {
+     *             "foo4" : {
+     *                  "foo5": "buzz"
+     *             }
+     *         }
+     *     }
+     * </pre>
+     *
+     * Would return the following list:
+     *
+     * <ul>
+     *  <li>"" (the empty string is the path to the root object)</li>
+     *  <li>"foo1.1"</li>
+     *  <li>"foo3</li>
+     *  <li>"foo3.foo4</li>
+     * </ul>
+     */
+    static List<String> getInsertPaths(XContentParser parser, Stack<String> currentPath) throws IOException {
+        assert parser.currentToken() == XContentParser.Token.START_OBJECT || parser.currentToken() == XContentParser.Token.START_ARRAY :
+            "should only be called when new objects or arrays start";
+        List<String> validPaths = new ArrayList<>();
+        // parser.currentName() can be null for root object and unnamed objects in arrays
+        if (parser.currentName() != null) {
+            // dots in randomized field names need to be escaped, we use that character as the path separator
+            currentPath.push(parser.currentName().replaceAll("\\.", "\\\\."));
+        }
+        if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
+            validPaths.add(String.join(".", currentPath.toArray(new String[currentPath.size()])));
+            while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
+                if (parser.currentToken() == XContentParser.Token.START_OBJECT
+                        || parser.currentToken() == XContentParser.Token.START_ARRAY) {
+                    validPaths.addAll(getInsertPaths(parser, currentPath));
+                }
+            }
+        } else if (parser.currentToken() == XContentParser.Token.START_ARRAY) {
+            int itemCount = 0;
+            while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
+                if (parser.currentToken() == XContentParser.Token.START_OBJECT
+                        || parser.currentToken() == XContentParser.Token.START_ARRAY) {
+                    currentPath.push(Integer.toString(itemCount));
+                    validPaths.addAll(getInsertPaths(parser, currentPath));
+                    currentPath.pop();
+                }
+                itemCount++;
+            }
+        }
+        if (parser.currentName() != null) {
+            currentPath.pop();
+        }
+        return validPaths;
+    }
+
+    /**
+     * Inserts key/value pairs into xContent passed in as {@link BytesReference} and returns a new {@link XContentBuilder}
+     * The paths argument uses dot separated fieldnames and numbers for array indices, similar to what we do in
+     * {@link ObjectPath}.
+     * The key/value arguments can suppliers that either return fixed or random values.
+     */
+    static XContentBuilder insertIntoXContent(XContent xContent, BytesReference original, List<String> paths, Supplier<String> key,
+            Supplier<Object> value) throws IOException {
+        ObjectPath object = ObjectPath.createFromXContent(xContent, original);
+        for (String path : paths) {
+            Map<String, Object> insertMap = (Map<String, Object>) object.evaluate(path);
+            insertMap.put(key.get(), value.get());
+        }
+        return object.toXContentBuilder(xContent);
+    }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/client/NoOpClient.java b/test/framework/src/main/java/org/elasticsearch/test/client/NoOpClient.java
index 1d91b0980e43b..c393e19653fca 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/client/NoOpClient.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/client/NoOpClient.java
@@ -32,14 +32,26 @@
 
 import java.util.concurrent.TimeUnit;
 
+/**
+ * Client that always responds with {@code null} to every request. Override this for testing.
+ */
 public class NoOpClient extends AbstractClient {
+    /**
+     * Build with {@link ThreadPool}. This {@linkplain ThreadPool} is terminated on {@link #close()}.
+     */
+    public NoOpClient(ThreadPool threadPool) {
+        super(Settings.EMPTY, threadPool);
+    }
 
+    /**
+     * Create a new {@link TestThreadPool} for this client.
+     */
     public NoOpClient(String testName) {
         super(Settings.EMPTY, new TestThreadPool(testName));
     }
 
     @Override
-    protected <Request extends ActionRequest<Request>,
+    protected <Request extends ActionRequest,
                     Response extends ActionResponse,
                     RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>>
             void doExecute(Action<Request, Response, RequestBuilder> action, Request request, ActionListener<Response> listener) {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/client/RandomizingClient.java b/test/framework/src/main/java/org/elasticsearch/test/client/RandomizingClient.java
index 5814cac1316bd..b144898d643d0 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/client/RandomizingClient.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/client/RandomizingClient.java
@@ -26,16 +26,22 @@
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.FilterClient;
 import org.elasticsearch.cluster.routing.Preference;
+import org.elasticsearch.common.unit.TimeValue;
 
 import java.util.Arrays;
 import java.util.EnumSet;
 import java.util.Random;
+import java.util.concurrent.TimeUnit;
 
 /** A {@link Client} that randomizes request parameters. */
 public class RandomizingClient extends FilterClient {
 
     private final SearchType defaultSearchType;
     private final String defaultPreference;
+    private final int batchedReduceSize;
+    private final int maxConcurrentShardRequests;
+    private final int preFilterShardSize;
+    private final boolean doTimeout;
 
 
     public RandomizingClient(Client client, Random random) {
@@ -53,12 +59,34 @@ public RandomizingClient(Client client, Random random) {
         } else {
             defaultPreference = null;
         }
-
+        this.batchedReduceSize = 2 + random.nextInt(10);
+        if (random.nextBoolean()) {
+            this.maxConcurrentShardRequests = 1 + random.nextInt(1 << random.nextInt(8));
+        } else {
+            this.maxConcurrentShardRequests = -1; // randomly use the default
+        }
+        if (random.nextBoolean()) {
+            preFilterShardSize =  1 + random.nextInt(1 << random.nextInt(7));
+        } else {
+            preFilterShardSize = -1;
+        }
+        doTimeout = random.nextBoolean();
     }
-    
+
     @Override
     public SearchRequestBuilder prepareSearch(String... indices) {
-        return in.prepareSearch(indices).setSearchType(defaultSearchType).setPreference(defaultPreference);
+        SearchRequestBuilder searchRequestBuilder = in.prepareSearch(indices).setSearchType(defaultSearchType)
+            .setPreference(defaultPreference).setBatchedReduceSize(batchedReduceSize);
+        if (maxConcurrentShardRequests != -1) {
+            searchRequestBuilder.setMaxConcurrentShardRequests(maxConcurrentShardRequests);
+        }
+        if (preFilterShardSize != -1) {
+            searchRequestBuilder.setPreFilterShardSize(preFilterShardSize);
+        }
+        if (doTimeout) {
+            searchRequestBuilder.setTimeout(new TimeValue(1, TimeUnit.DAYS));
+        }
+        return searchRequestBuilder;
     }
 
     @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/test/discovery/ClusterDiscoveryConfiguration.java b/test/framework/src/main/java/org/elasticsearch/test/discovery/ClusterDiscoveryConfiguration.java
index b393498ec89f6..3fd2b024a1d9b 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/discovery/ClusterDiscoveryConfiguration.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/discovery/ClusterDiscoveryConfiguration.java
@@ -25,7 +25,6 @@
 import org.elasticsearch.common.network.NetworkUtils;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.CollectionUtils;
-import org.elasticsearch.discovery.DiscoveryModule;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.test.InternalTestCluster;
 import org.elasticsearch.test.NodeConfigurationSource;
@@ -39,7 +38,7 @@
 
 public class ClusterDiscoveryConfiguration extends NodeConfigurationSource {
 
-    static Settings DEFAULT_NODE_SETTINGS = Settings.builder().put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), "zen").build();
+    static Settings DEFAULT_NODE_SETTINGS = Settings.EMPTY;
     private static final String IP_ADDR = "127.0.0.1";
 
     final int numOfNodes;
diff --git a/test/framework/src/main/java/org/elasticsearch/test/discovery/MockZenPing.java b/test/framework/src/main/java/org/elasticsearch/test/discovery/MockZenPing.java
new file mode 100644
index 0000000000000..b03e157b01cdb
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/test/discovery/MockZenPing.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.test.discovery;
+
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.common.component.AbstractComponent;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
+import org.elasticsearch.discovery.zen.PingContextProvider;
+import org.elasticsearch.discovery.zen.ZenPing;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Consumer;
+
+/**
+ * A {@link ZenPing} implementation which returns results based on an static in-memory map. This allows pinging
+ * to be immediate and can be used to speed up tests.
+ */
+public final class MockZenPing extends AbstractComponent implements ZenPing {
+
+    static final Map<ClusterName, Set<MockZenPing>> activeNodesPerCluster = new HashMap<>();
+
+    /** a set of the last discovered pings. used to throttle busy spinning where MockZenPing will keep returning the same results */
+    private Set<MockZenPing> lastDiscoveredPings = null;
+
+    private volatile PingContextProvider contextProvider;
+
+    public MockZenPing(Settings settings) {
+        super(settings);
+    }
+
+    @Override
+    public void start(PingContextProvider contextProvider) {
+        this.contextProvider = contextProvider;
+        assert contextProvider != null;
+        synchronized (activeNodesPerCluster) {
+            boolean added = getActiveNodesForCurrentCluster().add(this);
+            assert added;
+            activeNodesPerCluster.notifyAll();
+        }
+    }
+
+    @Override
+    public void ping(Consumer<PingCollection> resultsConsumer, TimeValue timeout) {
+        logger.info("pinging using mock zen ping");
+        synchronized (activeNodesPerCluster) {
+            Set<MockZenPing> activeNodes = getActiveNodesForCurrentCluster();
+            if (activeNodes.equals(lastDiscoveredPings)) {
+                try {
+                    logger.trace("nothing has changed since the last ping. waiting for a change");
+                    activeNodesPerCluster.wait(timeout.millis());
+                } catch (InterruptedException e) {
+
+                }
+                activeNodes = getActiveNodesForCurrentCluster();
+            }
+            lastDiscoveredPings = activeNodes;
+            PingCollection pingCollection = new PingCollection();
+            activeNodes.stream()
+                .filter(p -> p != this) // remove this as pings are not expected to return the local node
+                .map(MockZenPing::getPingResponse)
+                .forEach(pingCollection::addPing);
+            resultsConsumer.accept(pingCollection);
+        }
+    }
+
+    private ClusterName getClusterName() {
+        return contextProvider.clusterState().getClusterName();
+    }
+
+    private PingResponse getPingResponse() {
+        final ClusterState clusterState = contextProvider.clusterState();
+        return new PingResponse(clusterState.nodes().getLocalNode(), clusterState.nodes().getMasterNode(), clusterState);
+    }
+
+    private Set<MockZenPing> getActiveNodesForCurrentCluster() {
+        assert Thread.holdsLock(activeNodesPerCluster);
+        return activeNodesPerCluster.computeIfAbsent(getClusterName(),
+            clusterName -> ConcurrentCollections.newConcurrentSet());
+    }
+
+    @Override
+    public void close() {
+        synchronized (activeNodesPerCluster) {
+            boolean found = getActiveNodesForCurrentCluster().remove(this);
+            assert found;
+            activeNodesPerCluster.notifyAll();
+        }
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/discovery/TestZenDiscovery.java b/test/framework/src/main/java/org/elasticsearch/test/discovery/TestZenDiscovery.java
new file mode 100644
index 0000000000000..c2a237ef337d5
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/test/discovery/TestZenDiscovery.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.discovery;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Supplier;
+
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.discovery.Discovery;
+import org.elasticsearch.discovery.DiscoveryModule;
+import org.elasticsearch.discovery.zen.UnicastHostsProvider;
+import org.elasticsearch.discovery.zen.ZenDiscovery;
+import org.elasticsearch.discovery.zen.ZenPing;
+import org.elasticsearch.plugins.DiscoveryPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+
+/**
+ * A alternative zen discovery which allows using mocks for things like pings, as well as
+ * giving access to internals.
+ */
+public class TestZenDiscovery extends ZenDiscovery {
+
+    public static final Setting<Boolean> USE_MOCK_PINGS =
+        Setting.boolSetting("discovery.zen.use_mock_pings", true, Setting.Property.NodeScope);
+
+    /** A plugin which installs mock discovery and configures it to be used. */
+    public static class TestPlugin extends Plugin implements DiscoveryPlugin {
+        private Settings settings;
+        public TestPlugin(Settings settings) {
+            this.settings = settings;
+        }
+        @Override
+        public Map<String, Supplier<Discovery>> getDiscoveryTypes(ThreadPool threadPool, TransportService transportService,
+                                                                  NamedWriteableRegistry namedWriteableRegistry,
+                                                                  ClusterService clusterService, UnicastHostsProvider hostsProvider) {
+            return Collections.singletonMap("test-zen",
+                () -> new TestZenDiscovery(settings, threadPool, transportService, namedWriteableRegistry, clusterService, hostsProvider));
+        }
+
+        @Override
+        public List<Setting<?>> getSettings() {
+            return Collections.singletonList(USE_MOCK_PINGS);
+        }
+
+        @Override
+        public Settings additionalSettings() {
+            return Settings.builder().put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), "test-zen").build();
+        }
+    }
+
+    private TestZenDiscovery(Settings settings, ThreadPool threadPool, TransportService transportService,
+                             NamedWriteableRegistry namedWriteableRegistry, ClusterService clusterService,
+                             UnicastHostsProvider hostsProvider) {
+        super(settings, threadPool, transportService, namedWriteableRegistry, clusterService, hostsProvider);
+    }
+
+    @Override
+    protected ZenPing newZenPing(Settings settings, ThreadPool threadPool, TransportService transportService,
+                                 UnicastHostsProvider hostsProvider) {
+        if (USE_MOCK_PINGS.get(settings)) {
+            return new MockZenPing(settings);
+        } else {
+            return super.newZenPing(settings, threadPool, transportService, hostsProvider);
+        }
+    }
+
+    public ZenPing getZenPing() {
+        return zenPing;
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/disruption/BlockClusterStateProcessing.java b/test/framework/src/main/java/org/elasticsearch/test/disruption/BlockClusterStateProcessing.java
index 956088f0fd1cb..8a6be290502c4 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/disruption/BlockClusterStateProcessing.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/disruption/BlockClusterStateProcessing.java
@@ -19,7 +19,7 @@
 package org.elasticsearch.test.disruption;
 
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.LocalClusterUpdateTask;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.unit.TimeValue;
@@ -58,21 +58,16 @@ public void startDisrupting() {
         boolean success = disruptionLatch.compareAndSet(null, new CountDownLatch(1));
         assert success : "startDisrupting called without waiting on stopDisrupting to complete";
         final CountDownLatch started = new CountDownLatch(1);
-        clusterService.submitStateUpdateTask("service_disruption_block", new ClusterStateUpdateTask(Priority.IMMEDIATE) {
+        clusterService.submitStateUpdateTask("service_disruption_block", new LocalClusterUpdateTask(Priority.IMMEDIATE) {
 
             @Override
-            public boolean runOnlyOnMaster() {
-                return false;
-            }
-
-            @Override
-            public ClusterState execute(ClusterState currentState) throws Exception {
+            public ClusterTasksResult<LocalClusterUpdateTask> execute(ClusterState currentState) throws Exception {
                 started.countDown();
                 CountDownLatch latch = disruptionLatch.get();
                 if (latch != null) {
                     latch.await();
                 }
-                return currentState;
+                return unchanged();
             }
 
             @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/test/disruption/IntermittentLongGCDisruption.java b/test/framework/src/main/java/org/elasticsearch/test/disruption/IntermittentLongGCDisruption.java
index caab35e4b4283..9dd7883aab4c6 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/disruption/IntermittentLongGCDisruption.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/disruption/IntermittentLongGCDisruption.java
@@ -39,19 +39,6 @@ public class IntermittentLongGCDisruption extends LongGCDisruption {
     final long delayDurationMax;
 
 
-    public IntermittentLongGCDisruption(Random random) {
-        this(null, random);
-    }
-
-    public IntermittentLongGCDisruption(String disruptedNode, Random random) {
-        this(disruptedNode, random, 100, 200, 300, 20000);
-    }
-
-    public IntermittentLongGCDisruption(String disruptedNode, Random random, long intervalBetweenDelaysMin,
-                                        long intervalBetweenDelaysMax, long delayDurationMin, long delayDurationMax) {
-        this(random, disruptedNode, intervalBetweenDelaysMin, intervalBetweenDelaysMax, delayDurationMin, delayDurationMax);
-    }
-
     public IntermittentLongGCDisruption(Random random, String disruptedNode, long intervalBetweenDelaysMin, long intervalBetweenDelaysMax,
                                         long delayDurationMin, long delayDurationMax) {
         super(random, disruptedNode);
@@ -88,19 +75,15 @@ public void stopDisrupting() {
     }
 
     private void simulateLongGC(final TimeValue duration) throws InterruptedException {
-        final String disruptionNodeCopy = disruptedNode;
-        if (disruptionNodeCopy == null) {
-            return;
-        }
-        logger.info("node [{}] goes into GC for for [{}]", disruptionNodeCopy, duration);
+        logger.info("node [{}] goes into GC for for [{}]", disruptedNode, duration);
         final Set<Thread> nodeThreads = new HashSet<>();
         try {
-            while (stopNodeThreads(disruptionNodeCopy, nodeThreads)) ;
+            while (suspendThreads(nodeThreads)) ;
             if (!nodeThreads.isEmpty()) {
                 Thread.sleep(duration.millis());
             }
         } finally {
-            logger.info("node [{}] resumes from GC", disruptionNodeCopy);
+            logger.info("node [{}] resumes from GC", disruptedNode);
             resumeThreads(nodeThreads);
         }
     }
@@ -109,13 +92,13 @@ class BackgroundWorker implements Runnable {
 
         @Override
         public void run() {
-            while (disrupting && disruptedNode != null) {
+            while (disrupting) {
                 try {
                     TimeValue duration = new TimeValue(delayDurationMin + random.nextInt((int) (delayDurationMax - delayDurationMin)));
                     simulateLongGC(duration);
 
                     duration = new TimeValue(intervalBetweenDelaysMin + random.nextInt((int) (intervalBetweenDelaysMax - intervalBetweenDelaysMin)));
-                    if (disrupting && disruptedNode != null) {
+                    if (disrupting) {
                         Thread.sleep(duration.millis());
                     }
                 } catch (InterruptedException e) {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/disruption/LongGCDisruption.java b/test/framework/src/main/java/org/elasticsearch/test/disruption/LongGCDisruption.java
index 76f68a7c0983d..deebb4f53c124 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/disruption/LongGCDisruption.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/disruption/LongGCDisruption.java
@@ -19,14 +19,22 @@
 
 package org.elasticsearch.test.disruption;
 
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.test.InternalTestCluster;
 
-import java.util.HashSet;
+import java.lang.management.ManagementFactory;
+import java.lang.management.ThreadInfo;
+import java.lang.management.ThreadMXBean;
+import java.util.Arrays;
 import java.util.Random;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
 /**
  * Suspends all threads on the specified node in order to simulate a long gc.
@@ -34,12 +42,19 @@
 public class LongGCDisruption extends SingleNodeDisruption {
 
     private static final Pattern[] unsafeClasses = new Pattern[]{
-            // logging has shared JVM locks - we may suspend a thread and block other nodes from doing their thing
-            Pattern.compile("Logger")
+        // logging has shared JVM locks; we may suspend a thread and block other nodes from doing their thing
+        Pattern.compile("logging\\.log4j"),
+        // security manager is shared across all nodes and it uses synchronized maps internally
+        Pattern.compile("java\\.lang\\.SecurityManager"),
+        // SecureRandom instance from SecureRandomHolder class is shared by all nodes
+        Pattern.compile("java\\.security\\.SecureRandom")
     };
 
+    private static final ThreadMXBean threadBean = ManagementFactory.getThreadMXBean();
+
     protected final String disruptedNode;
     private Set<Thread> suspendedThreads;
+    private Thread blockDetectionThread;
 
     public LongGCDisruption(Random random, String disruptedNode) {
         super(random);
@@ -49,21 +64,140 @@ public LongGCDisruption(Random random, String disruptedNode) {
     @Override
     public synchronized void startDisrupting() {
         if (suspendedThreads == null) {
-            suspendedThreads = new HashSet<>();
-            stopNodeThreads(disruptedNode, suspendedThreads);
+            boolean success = false;
+            try {
+                suspendedThreads = ConcurrentHashMap.newKeySet();
+
+                final String currentThreadName = Thread.currentThread().getName();
+                assert isDisruptedNodeThread(currentThreadName) == false :
+                    "current thread match pattern. thread name: " + currentThreadName + ", node: " + disruptedNode;
+                // we spawn a background thread to protect against deadlock which can happen
+                // if there are shared resources between caller thread and and suspended threads
+                // see unsafeClasses to how to avoid that
+                final AtomicReference<Exception> suspendingError = new AtomicReference<>();
+                final Thread suspendingThread = new Thread(new AbstractRunnable() {
+                    @Override
+                    public void onFailure(Exception e) {
+                        suspendingError.set(e);
+                    }
+
+                    @Override
+                    protected void doRun() throws Exception {
+                        // keep trying to suspend threads, until no new threads are discovered.
+                        while (suspendThreads(suspendedThreads)) {
+                            if (Thread.interrupted()) {
+                                return;
+                            }
+                        }
+                    }
+                });
+                suspendingThread.setName(currentThreadName + "[LongGCDisruption][threadSuspender]");
+                suspendingThread.start();
+                try {
+                    suspendingThread.join(getSuspendingTimeoutInMillis());
+                } catch (InterruptedException e) {
+                    suspendingThread.interrupt(); // best effort to signal suspending
+                    throw new RuntimeException(e);
+                }
+                if (suspendingError.get() != null) {
+                    throw new RuntimeException("unknown error while suspending threads", suspendingError.get());
+                }
+                if (suspendingThread.isAlive()) {
+                    logger.warn("failed to suspend node [{}]'s threads within [{}] millis. Suspending thread stack trace:\n {}"
+                        , disruptedNode, getSuspendingTimeoutInMillis(), stackTrace(suspendingThread.getStackTrace()));
+                    suspendingThread.interrupt(); // best effort;
+                    try {
+                        /*
+                         * We need to join on the suspending thread in case it has suspended a thread that is in a critical section and
+                         * needs to be resumed.
+                         */
+                        suspendingThread.join();
+                    } catch (InterruptedException e) {
+                        throw new RuntimeException(e);
+                    }
+                    throw new RuntimeException("suspending node threads took too long");
+                }
+                // block detection checks if other threads are blocked waiting on an object that is held by one
+                // of the threads that was suspended
+                if (isBlockDetectionSupported()) {
+                    blockDetectionThread = new Thread(new AbstractRunnable() {
+                        @Override
+                        public void onFailure(Exception e) {
+                            if (e instanceof InterruptedException == false) {
+                                throw new AssertionError("unexpected exception in blockDetectionThread", e);
+                            }
+                        }
+
+                        @Override
+                        protected void doRun() throws Exception {
+                            while (Thread.currentThread().isInterrupted() == false) {
+                                ThreadInfo[] threadInfos = threadBean.dumpAllThreads(true, true);
+                                for (ThreadInfo threadInfo : threadInfos) {
+                                    if (isDisruptedNodeThread(threadInfo.getThreadName()) == false &&
+                                        threadInfo.getLockOwnerName() != null &&
+                                        isDisruptedNodeThread(threadInfo.getLockOwnerName())) {
+
+                                        // find ThreadInfo object of the blocking thread (if available)
+                                        ThreadInfo blockingThreadInfo = null;
+                                        for (ThreadInfo otherThreadInfo : threadInfos) {
+                                            if (otherThreadInfo.getThreadId() == threadInfo.getLockOwnerId()) {
+                                                blockingThreadInfo = otherThreadInfo;
+                                                break;
+                                            }
+                                        }
+                                        onBlockDetected(threadInfo, blockingThreadInfo);
+                                    }
+                                }
+                                Thread.sleep(getBlockDetectionIntervalInMillis());
+                            }
+                        }
+                    });
+                    blockDetectionThread.setName(currentThreadName + "[LongGCDisruption][blockDetection]");
+                    blockDetectionThread.start();
+                }
+                success = true;
+            } finally {
+                if (success == false) {
+                    stopBlockDetection();
+                    // resume threads if failed
+                    resumeThreads(suspendedThreads);
+                    suspendedThreads = null;
+                }
+            }
         } else {
             throw new IllegalStateException("can't disrupt twice, call stopDisrupting() first");
         }
     }
 
+    public boolean isDisruptedNodeThread(String threadName) {
+        return threadName.contains("[" + disruptedNode + "]");
+    }
+
+    private String stackTrace(StackTraceElement[] stackTraceElements) {
+        return Arrays.stream(stackTraceElements).map(Object::toString).collect(Collectors.joining("\n"));
+    }
+
     @Override
     public synchronized void stopDisrupting() {
+        stopBlockDetection();
         if (suspendedThreads != null) {
             resumeThreads(suspendedThreads);
             suspendedThreads = null;
         }
     }
 
+    private void stopBlockDetection() {
+        if (blockDetectionThread != null) {
+            try {
+                blockDetectionThread.interrupt(); // best effort
+                blockDetectionThread.join(getSuspendingTimeoutInMillis());
+            } catch (InterruptedException e) {
+                throw new RuntimeException(e);
+            }
+            blockDetectionThread = null;
+        }
+    }
+
     @Override
     public void removeAndEnsureHealthy(InternalTestCluster cluster) {
         removeFromCluster(cluster);
@@ -75,9 +209,16 @@ public TimeValue expectedTimeToHeal() {
         return TimeValue.timeValueMillis(0);
     }
 
-    @SuppressWarnings("deprecation") // stops/resumes threads intentionally
-    @SuppressForbidden(reason = "stops/resumes threads intentionally")
-    protected boolean stopNodeThreads(String node, Set<Thread> nodeThreads) {
+    /**
+     * resolves all threads belonging to given node and suspends them if their current stack trace
+     * is "safe". Threads are added to nodeThreads if suspended.
+     *
+     * returns true if some live threads were found. The caller is expected to call this method
+     * until no more "live" are found.
+     */
+    @SuppressWarnings("deprecation") // suspends/resumes threads intentionally
+    @SuppressForbidden(reason = "suspends/resumes threads intentionally")
+    protected boolean suspendThreads(Set<Thread> nodeThreads) {
         Thread[] allThreads = null;
         while (allThreads == null) {
             allThreads = new Thread[Thread.activeCount()];
@@ -86,43 +227,90 @@ protected boolean stopNodeThreads(String node, Set<Thread> nodeThreads) {
                 allThreads = null;
             }
         }
-        boolean stopped = false;
-        final String nodeThreadNamePart = "[" + node + "]";
+        boolean liveThreadsFound = false;
         for (Thread thread : allThreads) {
             if (thread == null) {
                 continue;
             }
-            String name = thread.getName();
-            if (name.contains(nodeThreadNamePart)) {
+            String threadName = thread.getName();
+            if (isDisruptedNodeThread(threadName)) {
                 if (thread.isAlive() && nodeThreads.add(thread)) {
-                    stopped = true;
-                    logger.trace("stopping thread [{}]", name);
-                    thread.suspend();
-                    // double check the thread is not in a shared resource like logging. If so, let it go and come back..
-                    boolean safe = true;
-                    safe:
-                    for (StackTraceElement stackElement : thread.getStackTrace()) {
-                        String className = stackElement.getClassName();
-                        for (Pattern unsafePattern : unsafeClasses) {
-                            if (unsafePattern.matcher(className).find()) {
-                                safe = false;
-                                break safe;
+                    liveThreadsFound = true;
+                    logger.trace("suspending thread [{}]", threadName);
+                    // we assume it is not safe to suspend the thread
+                    boolean safe = false;
+                    try {
+                        /*
+                         * At the bottom of this try-block we will know whether or not it is safe to suspend the thread; we start by
+                         * assuming that it is safe.
+                         */
+                        boolean definitelySafe = true;
+                        thread.suspend();
+                        // double check the thread is not in a shared resource like logging; if so, let it go and come back
+                        safe:
+                        for (StackTraceElement stackElement : thread.getStackTrace()) {
+                            String className = stackElement.getClassName();
+                            for (Pattern unsafePattern : getUnsafeClasses()) {
+                                if (unsafePattern.matcher(className).find()) {
+                                    // it is definitely not safe to suspend the thread
+                                    definitelySafe = false;
+                                    break safe;
+                                }
                             }
                         }
-                    }
-                    if (!safe) {
-                        logger.trace("resuming thread [{}] as it is in a critical section", name);
-                        thread.resume();
-                        nodeThreads.remove(thread);
+                        safe = definitelySafe;
+                    } finally {
+                        if (!safe) {
+                            /*
+                             * Do not log before resuming as we might be interrupted while logging in which case we will throw an
+                             * interrupted exception and never resume the suspended thread that is in a critical section. Also, logging
+                             * before resuming makes for confusing log messages if we never hit the resume.
+                             */
+                            thread.resume();
+                            logger.trace("resumed thread [{}] as it is in a critical section", threadName);
+                            nodeThreads.remove(thread);
+                        }
                     }
                 }
             }
         }
-        return stopped;
+        return liveThreadsFound;
+    }
+
+    // for testing
+    protected Pattern[] getUnsafeClasses() {
+        return unsafeClasses;
+    }
+
+    // for testing
+    protected long getSuspendingTimeoutInMillis() {
+        return TimeValue.timeValueSeconds(30).getMillis();
+    }
+
+    public boolean isBlockDetectionSupported() {
+        return threadBean.isObjectMonitorUsageSupported() && threadBean.isSynchronizerUsageSupported();
+    }
+
+    // for testing
+    protected long getBlockDetectionIntervalInMillis() {
+        return 3000L;
+    }
+
+    // for testing
+    protected void onBlockDetected(ThreadInfo blockedThread, @Nullable ThreadInfo blockingThread) {
+        String blockedThreadStackTrace = stackTrace(blockedThread.getStackTrace());
+        String blockingThreadStackTrace = blockingThread != null ?
+            stackTrace(blockingThread.getStackTrace()) : "not available";
+        throw new AssertionError("Thread [" + blockedThread.getThreadName() + "] is blocked waiting on the resource [" +
+            blockedThread.getLockInfo() + "] held by the suspended thread [" + blockedThread.getLockOwnerName() +
+            "] of the disrupted node [" + disruptedNode + "].\n" +
+            "Please add this occurrence to the unsafeClasses list in [" + LongGCDisruption.class.getName() + "].\n" +
+            "Stack trace of blocked thread: " + blockedThreadStackTrace + "\n" +
+            "Stack trace of blocking thread: " + blockingThreadStackTrace);
     }
 
-    @SuppressWarnings("deprecation") // stops/resumes threads intentionally
-    @SuppressForbidden(reason = "stops/resumes threads intentionally")
+    @SuppressWarnings("deprecation") // suspends/resumes threads intentionally
+    @SuppressForbidden(reason = "suspends/resumes threads intentionally")
     protected void resumeThreads(Set<Thread> threads) {
         for (Thread thread : threads) {
             thread.resume();
diff --git a/test/framework/src/main/java/org/elasticsearch/test/disruption/NetworkDisruption.java b/test/framework/src/main/java/org/elasticsearch/test/disruption/NetworkDisruption.java
index 4e135c4c2b0bf..5d731b094d1da 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/disruption/NetworkDisruption.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/disruption/NetworkDisruption.java
@@ -21,6 +21,9 @@
 
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.NodeConnectionsService;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.set.Sets;
@@ -58,6 +61,14 @@ public NetworkDisruption(DisruptedLinks disruptedLinks, NetworkLinkDisruptionTyp
         this.networkLinkDisruptionType = networkLinkDisruptionType;
     }
 
+    public DisruptedLinks getDisruptedLinks() {
+        return disruptedLinks;
+    }
+
+    public NetworkLinkDisruptionType getNetworkLinkDisruptionType() {
+        return networkLinkDisruptionType;
+    }
+
     @Override
     public void applyToCluster(InternalTestCluster cluster) {
         this.cluster = cluster;
@@ -71,12 +82,35 @@ public void removeFromCluster(InternalTestCluster cluster) {
     @Override
     public void removeAndEnsureHealthy(InternalTestCluster cluster) {
         removeFromCluster(cluster);
+        ensureHealthy(cluster);
+    }
+
+    /**
+     * ensures the cluster is healthy after the disruption
+     */
+    public void ensureHealthy(InternalTestCluster cluster) {
+        assert activeDisruption == false;
         ensureNodeCount(cluster);
+        ensureFullyConnectedCluster(cluster);
+    }
+
+    /**
+     * Ensures that all nodes in the cluster are connected to each other.
+     *
+     * Some network disruptions may leave nodes that are not the master disconnected from each other.
+     * {@link org.elasticsearch.cluster.NodeConnectionsService} will eventually reconnect but it's
+     * handy to be able to ensure this happens faster
+     */
+    public static void ensureFullyConnectedCluster(InternalTestCluster cluster) {
+        for (String node: cluster.getNodeNames()) {
+            ClusterState stateOnNode = cluster.getInstance(ClusterService.class, node).state();
+            cluster.getInstance(NodeConnectionsService.class, node).connectToNodes(stateOnNode.nodes());
+        }
     }
 
     protected void ensureNodeCount(InternalTestCluster cluster) {
         assertFalse("cluster failed to form after disruption was healed", cluster.client().admin().cluster().prepareHealth()
-            .setWaitForNodes("" + cluster.size())
+            .setWaitForNodes(String.valueOf(cluster.size()))
             .setWaitForNoRelocatingShards(true)
             .get().isTimedOut());
     }
@@ -143,6 +177,11 @@ private MockTransportService transport(String node) {
         return (MockTransportService) cluster.getInstance(TransportService.class, node);
     }
 
+    @Override
+    public String toString() {
+        return "network disruption (disruption type: " + networkLinkDisruptionType + ", disrupted links: " + disruptedLinks + ")";
+    }
+
     /**
      * Represents a set of nodes with connections between nodes that are to be disrupted
      */
@@ -328,6 +367,18 @@ public String toString() {
         }
     }
 
+    public static class IsolateAllNodes extends DisruptedLinks {
+
+        public IsolateAllNodes(Set<String> nodes) {
+            super(nodes);
+        }
+
+        @Override
+        public boolean disrupt(String node1, String node2) {
+            return true;
+        }
+    }
+
     /**
      * Abstract class representing various types of network disruptions. Instances of this class override the {@link #applyDisruption}
      * method to apply their specific disruption type to requests that are send from a source to a target node.
diff --git a/test/framework/src/main/java/org/elasticsearch/test/disruption/NetworkDisruptionIT.java b/test/framework/src/main/java/org/elasticsearch/test/disruption/NetworkDisruptionIT.java
deleted file mode 100644
index 50860fca23d6e..0000000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/disruption/NetworkDisruptionIT.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.test.disruption;
-
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.InternalTestCluster;
-import org.elasticsearch.test.disruption.NetworkDisruption.NetworkUnresponsive;
-import org.elasticsearch.test.disruption.NetworkDisruption.TwoPartitions;
-import org.elasticsearch.test.transport.MockTransportService;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collection;
-
-public class NetworkDisruptionIT extends ESIntegTestCase {
-    @Override
-    protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Arrays.asList(MockTransportService.TestPlugin.class);
-    }
-
-    public void testNetworkPartitionWithNodeShutdown() throws IOException {
-        internalCluster().ensureAtLeastNumDataNodes(2);
-        String[] nodeNames = internalCluster().getNodeNames();
-        NetworkDisruption networkDisruption = new NetworkDisruption(new TwoPartitions(nodeNames[0], nodeNames[1]),
-            new NetworkUnresponsive());
-        internalCluster().setDisruptionScheme(networkDisruption);
-        networkDisruption.startDisrupting();
-        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodeNames[0]));
-        internalCluster().clearDisruptionScheme();
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/disruption/SingleNodeDisruption.java b/test/framework/src/main/java/org/elasticsearch/test/disruption/SingleNodeDisruption.java
index 862e18d7aca81..a22994cfa9bc4 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/disruption/SingleNodeDisruption.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/disruption/SingleNodeDisruption.java
@@ -84,7 +84,7 @@ public synchronized void testClusterClosed() {
 
     protected void ensureNodeCount(InternalTestCluster cluster) {
         assertFalse("cluster failed to form after disruption was healed", cluster.client().admin().cluster().prepareHealth()
-                .setWaitForNodes("" + cluster.size())
+                .setWaitForNodes(String.valueOf(cluster.size()))
                 .setWaitForNoRelocatingShards(true)
                 .get().isTimedOut());
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/disruption/SlowClusterStateProcessing.java b/test/framework/src/main/java/org/elasticsearch/test/disruption/SlowClusterStateProcessing.java
index f69c0a3085d63..61afa4f77f35f 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/disruption/SlowClusterStateProcessing.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/disruption/SlowClusterStateProcessing.java
@@ -19,7 +19,7 @@
 package org.elasticsearch.test.disruption;
 
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.LocalClusterUpdateTask;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.unit.TimeValue;
@@ -102,15 +102,10 @@ private boolean interruptClusterStateProcessing(final TimeValue duration) throws
             return false;
         }
         final AtomicBoolean stopped = new AtomicBoolean(false);
-        clusterService.submitStateUpdateTask("service_disruption_delay", new ClusterStateUpdateTask(Priority.IMMEDIATE) {
+        clusterService.submitStateUpdateTask("service_disruption_delay", new LocalClusterUpdateTask(Priority.IMMEDIATE) {
 
             @Override
-            public boolean runOnlyOnMaster() {
-                return false;
-            }
-
-            @Override
-            public ClusterState execute(ClusterState currentState) throws Exception {
+            public ClusterTasksResult<LocalClusterUpdateTask> execute(ClusterState currentState) throws Exception {
                 long count = duration.millis() / 200;
                 // wait while checking for a stopped
                 for (; count > 0 && !stopped.get(); count--) {
@@ -120,7 +115,7 @@ public ClusterState execute(ClusterState currentState) throws Exception {
                     Thread.sleep(duration.millis() % 200);
                 }
                 countDownLatch.countDown();
-                return currentState;
+                return unchanged();
             }
 
             @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/test/engine/ThrowingLeafReaderWrapper.java b/test/framework/src/main/java/org/elasticsearch/test/engine/ThrowingLeafReaderWrapper.java
index 37ed43b9450d3..9da5d7515c76d 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/engine/ThrowingLeafReaderWrapper.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/engine/ThrowingLeafReaderWrapper.java
@@ -147,7 +147,7 @@ public TermsEnum iterator() throws IOException {
     static class ThrowingTermsEnum extends FilterTermsEnum {
         private final Thrower thrower;
 
-        public ThrowingTermsEnum(TermsEnum in, Thrower thrower) {
+        ThrowingTermsEnum(TermsEnum in, Thrower thrower) {
             super(in);
             this.thrower = thrower;
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/gateway/NoopGatewayAllocator.java b/test/framework/src/main/java/org/elasticsearch/test/gateway/NoopGatewayAllocator.java
index e321a98f371de..b2b41b31461e3 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/gateway/NoopGatewayAllocator.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/gateway/NoopGatewayAllocator.java
@@ -19,12 +19,14 @@
 
 package org.elasticsearch.test.gateway;
 
-import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.allocation.FailedShard;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
-import org.elasticsearch.cluster.routing.allocation.StartedRerouteAllocation;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.gateway.GatewayAllocator;
 
+import java.util.List;
+
 /**
  * An allocator used for tests that doesn't do anything
  */
@@ -37,12 +39,12 @@ protected NoopGatewayAllocator() {
     }
 
     @Override
-    public void applyStartedShards(StartedRerouteAllocation allocation) {
+    public void applyStartedShards(RoutingAllocation allocation, List<ShardRouting> startedShards) {
         // noop
     }
 
     @Override
-    public void applyFailedShards(FailedRerouteAllocation allocation) {
+    public void applyFailedShards(RoutingAllocation allocation, List<FailedShard> failedShards) {
         // noop
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/gateway/TestGatewayAllocator.java b/test/framework/src/main/java/org/elasticsearch/test/gateway/TestGatewayAllocator.java
new file mode 100644
index 0000000000000..ab467b76c9243
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/test/gateway/TestGatewayAllocator.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.gateway;
+
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.allocation.FailedShard;
+import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.gateway.AsyncShardFetch;
+import org.elasticsearch.gateway.GatewayAllocator;
+import org.elasticsearch.gateway.PrimaryShardAllocator;
+import org.elasticsearch.gateway.ReplicaShardAllocator;
+import org.elasticsearch.gateway.TransportNodesListGatewayStartedShards.NodeGatewayStartedShards;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.NodeStoreFilesMetaData;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * A gateway allocator implementation that keeps an in memory list of started shard allocation
+ * that are used as replies to the, normally async, fetch data requests. The in memory list
+ * is adapted when shards are started and failed.
+ *
+ * Nodes leaving and joining the cluster do not change the list of shards the class tracks but
+ * rather serves as a filter to what is returned by fetch data. Concretely - fetch data will
+ * only return shards that were started on nodes that are currently part of the cluster.
+ *
+ * For now only primary shard related data is fetched. Replica request always get an empty response.
+ *
+ *
+ * This class is useful to use in unit tests that require the functionality of {@link GatewayAllocator} but do
+ * not have all the infrastructure required to use it.
+ */
+public class TestGatewayAllocator extends GatewayAllocator {
+
+    Map<String /* node id */, Map<ShardId, ShardRouting>> knownAllocations = new HashMap<>();
+    DiscoveryNodes currentNodes = DiscoveryNodes.EMPTY_NODES;
+
+    PrimaryShardAllocator primaryShardAllocator = new PrimaryShardAllocator(Settings.EMPTY) {
+        @Override
+        protected AsyncShardFetch.FetchResult<NodeGatewayStartedShards> fetchData(ShardRouting shard, RoutingAllocation allocation) {
+            // for now always return immediately what we know
+            final ShardId shardId = shard.shardId();
+            final Set<String> ignoreNodes = allocation.getIgnoreNodes(shardId);
+            Map<DiscoveryNode, NodeGatewayStartedShards> foundShards = knownAllocations.values().stream()
+                .flatMap(shardMap -> shardMap.values().stream())
+                .filter(ks -> ks.shardId().equals(shardId))
+                .filter(ks -> ignoreNodes.contains(ks.currentNodeId()) == false)
+                .filter(ks -> currentNodes.nodeExists(ks.currentNodeId()))
+                .collect(Collectors.toMap(
+                    routing -> currentNodes.get(routing.currentNodeId()),
+                    routing ->
+                        new NodeGatewayStartedShards(
+                            currentNodes.get(routing.currentNodeId()), -1, routing.allocationId().getId(), routing.primary())));
+
+            return new AsyncShardFetch.FetchResult<>(shardId, foundShards, ignoreNodes);
+        }
+    };
+
+    ReplicaShardAllocator replicaShardAllocator = new ReplicaShardAllocator(Settings.EMPTY) {
+        @Override
+        protected AsyncShardFetch.FetchResult<NodeStoreFilesMetaData> fetchData(ShardRouting shard, RoutingAllocation allocation) {
+            // for now, just pretend no node has data
+            final ShardId shardId = shard.shardId();
+            return new AsyncShardFetch.FetchResult<>(shardId, Collections.emptyMap(), allocation.getIgnoreNodes(shardId));
+        }
+
+        @Override
+        protected boolean hasInitiatedFetching(ShardRouting shard) {
+            return true;
+        }
+    };
+
+    public TestGatewayAllocator() {
+        super(Settings.EMPTY, null, null);
+    }
+
+    @Override
+    public void applyStartedShards(RoutingAllocation allocation, List<ShardRouting> startedShards) {
+        currentNodes = allocation.nodes();
+        allocation.routingNodes().shards(ShardRouting::active).forEach(this::addKnownAllocation);
+    }
+
+    @Override
+    public void applyFailedShards(RoutingAllocation allocation, List<FailedShard> failedShards) {
+        currentNodes = allocation.nodes();
+        for (FailedShard failedShard : failedShards) {
+            final ShardRouting failedRouting = failedShard.getRoutingEntry();
+            Map<ShardId, ShardRouting> nodeAllocations = knownAllocations.get(failedRouting.currentNodeId());
+            if (nodeAllocations != null) {
+                nodeAllocations.remove(failedRouting.shardId());
+                if (nodeAllocations.isEmpty()) {
+                    knownAllocations.remove(failedRouting.currentNodeId());
+                }
+            }
+        }
+    }
+
+    @Override
+    public void allocateUnassigned(RoutingAllocation allocation) {
+        currentNodes = allocation.nodes();
+        innerAllocatedUnassigned(allocation, primaryShardAllocator, replicaShardAllocator);
+    }
+
+    /**
+     * manually add a specific shard to the allocations the gateway keeps track of
+     */
+    public void addKnownAllocation(ShardRouting shard) {
+            knownAllocations.computeIfAbsent(shard.currentNodeId(), id -> new HashMap<>())
+                .put(shard.shardId(), shard);
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
index 18d6939dd4d5a..6280de426ce7e 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
@@ -55,6 +55,11 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchModule;
@@ -73,8 +78,10 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
 import java.util.Set;
 
 import static java.util.Collections.emptyList;
@@ -636,7 +643,7 @@ public static void assertVersionSerializable(Version version, Streamable streama
          * a way that sucks less.
          */
         NamedWriteableRegistry registry;
-        if (ESIntegTestCase.isInternalCluster()) {
+        if (ESIntegTestCase.isInternalCluster() && ESIntegTestCase.internalCluster().size() > 0) {
             registry = ESIntegTestCase.internalCluster().getInstance(NamedWriteableRegistry.class);
         } else {
             SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList());
@@ -653,9 +660,15 @@ public static void assertVersionSerializable(Version version, Streamable streama
                 // streamable that comes in.
             }
             if (streamable instanceof ActionRequest) {
-                ((ActionRequest<?>) streamable).validate();
+                ((ActionRequest) streamable).validate();
+            }
+            BytesReference orig;
+            try {
+                orig = serialize(version, streamable);
+            } catch (IllegalArgumentException e) {
+                // Can't serialize with this version so skip this test.
+                return;
             }
-            BytesReference orig = serialize(version, streamable);
             StreamInput input = orig.streamInput();
             if (namedWriteableRegistry != null) {
                 input = new NamedWriteableAwareStreamInput(input, namedWriteableRegistry);
@@ -760,4 +773,76 @@ public static void assertDirectoryExists(Path dir) {
         assertFileExists(dir);
         assertThat("file [" + dir + "] should be a directory.", Files.isDirectory(dir), is(true));
     }
+
+    /**
+     * Asserts that the provided {@link BytesReference}s created through
+     * {@link org.elasticsearch.common.xcontent.ToXContent#toXContent(XContentBuilder, ToXContent.Params)} hold the same content.
+     * The comparison is done by parsing both into a map and comparing those two, so that keys ordering doesn't matter.
+     * Also binary values (byte[]) are properly compared through arrays comparisons.
+     */
+    public static void assertToXContentEquivalent(BytesReference expected, BytesReference actual, XContentType xContentType)
+            throws IOException {
+        //we tried comparing byte per byte, but that didn't fly for a couple of reasons:
+        //1) whenever anything goes through a map while parsing, ordering is not preserved, which is perfectly ok
+        //2) Jackson SMILE parser parses floats as double, which then get printed out as double (with double precision)
+        //Note that byte[] holding binary values need special treatment as they need to be properly compared item per item.
+        try (XContentParser actualParser = xContentType.xContent().createParser(NamedXContentRegistry.EMPTY, actual)) {
+            Map<String, Object> actualMap = actualParser.map();
+            try (XContentParser expectedParser = xContentType.xContent().createParser(NamedXContentRegistry.EMPTY, expected)) {
+                Map<String, Object> expectedMap = expectedParser.map();
+                assertMapEquals(expectedMap, actualMap);
+            }
+        }
+    }
+
+    /**
+     * Compares two maps recursively, using arrays comparisons for byte[] through Arrays.equals(byte[], byte[])
+     */
+    @SuppressWarnings("unchecked")
+    private static void assertMapEquals(Map<String, Object> expected, Map<String, Object> actual) {
+        assertEquals(expected.size(), actual.size());
+        for (Map.Entry<String, Object> expectedEntry : expected.entrySet()) {
+            String expectedKey = expectedEntry.getKey();
+            Object expectedValue = expectedEntry.getValue();
+            if (expectedValue == null) {
+                assertTrue(actual.get(expectedKey) == null && actual.containsKey(expectedKey));
+            } else {
+                Object actualValue = actual.get(expectedKey);
+                assertObjectEquals(expectedValue, actualValue);
+            }
+        }
+    }
+
+    /**
+     * Compares two lists recursively, but using arrays comparisons for byte[] through Arrays.equals(byte[], byte[])
+     */
+    @SuppressWarnings("unchecked")
+    private static void assertListEquals(List<Object> expected, List<Object> actual) {
+        assertEquals(expected.size(), actual.size());
+        Iterator<Object> actualIterator = actual.iterator();
+        for (Object expectedValue : expected) {
+            Object actualValue = actualIterator.next();
+            assertObjectEquals(expectedValue, actualValue);
+        }
+    }
+
+    /**
+     * Compares two objects, recursively walking eventual maps and lists encountered, and using arrays comparisons
+     * for byte[] through Arrays.equals(byte[], byte[])
+     */
+    @SuppressWarnings("unchecked")
+    private static void assertObjectEquals(Object expected, Object actual) {
+        if (expected instanceof Map) {
+            assertThat(actual, instanceOf(Map.class));
+            assertMapEquals((Map<String, Object>) expected, (Map<String, Object>) actual);
+        } else if (expected instanceof List) {
+            assertListEquals((List<Object>) expected, (List<Object>) actual);
+        } else if (expected instanceof byte[]) {
+            //byte[] is really a special case for binary values when comparing SMILE and CBOR, arrays of other types
+            //don't need to be handled. Ordinary arrays get parsed as lists.
+            assertArrayEquals((byte[]) expected, (byte[]) actual);
+        } else {
+            assertEquals(expected, actual);
+        }
+    }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/junit/annotations/TestLogging.java b/test/framework/src/main/java/org/elasticsearch/test/junit/annotations/TestLogging.java
index e09cc7534e6a1..d0e799d63fefb 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/junit/annotations/TestLogging.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/junit/annotations/TestLogging.java
@@ -32,7 +32,6 @@
  * It supports multiple logger:level comma separated key value pairs
  * Use the _root keyword to set the root logger level
  * e.g. @TestLogging("_root:DEBUG,org.elasticsearch.cluster.metadata:TRACE")
- * or just @TestLogging("_root:DEBUG,cluster.metadata:TRACE") since we start the test with -Des.logger.prefix=
  */
 @Retention(RetentionPolicy.RUNTIME)
 @Target({PACKAGE, TYPE, METHOD})
diff --git a/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/LoggingListener.java b/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/LoggingListener.java
index 0009c21d6aad2..332dd0267fd38 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/LoggingListener.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/LoggingListener.java
@@ -16,6 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+
 package org.elasticsearch.test.junit.listeners;
 
 import org.apache.logging.log4j.Logger;
@@ -26,18 +27,18 @@
 import org.junit.runner.Result;
 import org.junit.runner.notification.RunListener;
 
-import java.util.HashMap;
+import java.util.Collections;
 import java.util.Map;
+import java.util.TreeMap;
 
 /**
- * A {@link RunListener} that allows to change the log level for a specific test method.
- * When a test method is annotated with the {@link org.elasticsearch.test.junit.annotations.TestLogging} annotation, the level for the specified loggers
- * will be internally saved before the test method execution and overridden with the specified ones.
- * At the end of the test method execution the original loggers levels will be restored.
+ * A {@link RunListener} that allows changing the log level for a specific test method. When a test method is annotated with the
+ * {@link TestLogging} annotation, the level for the specified loggers will be internally saved before the test method execution and
+ * overridden with the specified ones. At the end of the test method execution the original loggers levels will be restored.
  *
- * Note: This class is not thread-safe. Given the static nature of the logging api, it assumes that tests
- * are never run concurrently in the same jvm. For the very same reason no synchronization has been implemented
- * regarding the save/restore process of the original loggers levels.
+ * This class is not thread-safe. Given the static nature of the logging API, it assumes that tests are never run concurrently in the same
+ * JVM. For the very same reason no synchronization has been implemented regarding the save/restore process of the original loggers
+ * levels.
  */
 public class LoggingListener extends RunListener {
 
@@ -46,29 +47,35 @@ public class LoggingListener extends RunListener {
     private Map<String, String> previousPackageLoggingMap;
 
     @Override
-    public void testRunStarted(Description description) throws Exception {
+    public void testRunStarted(final Description description) throws Exception {
         Package testClassPackage = description.getTestClass().getPackage();
         previousPackageLoggingMap = processTestLogging(testClassPackage != null ? testClassPackage.getAnnotation(TestLogging.class) : null);
         previousClassLoggingMap = processTestLogging(description.getAnnotation(TestLogging.class));
     }
 
     @Override
-    public void testRunFinished(Result result) throws Exception {
+    public void testRunFinished(final Result result) throws Exception {
         previousClassLoggingMap = reset(previousClassLoggingMap);
         previousPackageLoggingMap = reset(previousPackageLoggingMap);
     }
 
     @Override
-    public void testStarted(Description description) throws Exception {
+    public void testStarted(final Description description) throws Exception {
         final TestLogging testLogging = description.getAnnotation(TestLogging.class);
         previousLoggingMap = processTestLogging(testLogging);
     }
 
     @Override
-    public void testFinished(Description description) throws Exception {
+    public void testFinished(final Description description) throws Exception {
         previousLoggingMap = reset(previousLoggingMap);
     }
 
+    /**
+     * Obtain the logger with the given name.
+     *
+     * @param loggerName the logger to obtain
+     * @return the logger
+     */
     private static Logger resolveLogger(String loggerName) {
         if (loggerName.equalsIgnoreCase("_root")) {
             return ESLoggerFactory.getRootLogger();
@@ -76,44 +83,71 @@ private static Logger resolveLogger(String loggerName) {
         return Loggers.getLogger(loggerName);
     }
 
-    private Map<String, String> processTestLogging(TestLogging testLogging) {
-        Map<String, String> map = getLoggersAndLevelsFromAnnotation(testLogging);
+    /**
+     * Applies the test logging annotation and returns the existing logging levels.
+     *
+     * @param testLogging the test logging annotation to apply
+     * @return the existing logging levels
+     */
+    private Map<String, String> processTestLogging(final TestLogging testLogging) {
+        final Map<String, String> map = getLoggersAndLevelsFromAnnotation(testLogging);
+
         if (map == null) {
-            return null;
+            return Collections.emptyMap();
+        }
+
+        // obtain the existing logging levels so that we can restore them at the end of the test; we have to do this separately from setting
+        // the logging levels so that setting foo does not impact the logging level for foo.bar when we check the existing logging level for
+        // for.bar
+        final Map<String, String> existing = new TreeMap<>();
+        for (final Map.Entry<String, String> entry : map.entrySet()) {
+            final Logger logger = resolveLogger(entry.getKey());
+            existing.put(entry.getKey(), logger.getLevel().toString());
         }
-        Map<String, String> previousValues = new HashMap<>();
-        for (Map.Entry<String, String> entry : map.entrySet()) {
-            Logger logger = resolveLogger(entry.getKey());
-            previousValues.put(entry.getKey(), logger.getLevel().toString());
+        for (final Map.Entry<String, String> entry : map.entrySet()) {
+            final Logger logger = resolveLogger(entry.getKey());
             Loggers.setLevel(logger, entry.getValue());
         }
-        return previousValues;
+        return existing;
     }
 
-    public static Map<String, String> getLoggersAndLevelsFromAnnotation(TestLogging testLogging) {
+    /**
+     * Obtain the logging levels from the test logging annotation.
+     *
+     * @param testLogging the test logging annotation
+     * @return a map from logger name to logging level
+     */
+    public static Map<String, String> getLoggersAndLevelsFromAnnotation(final TestLogging testLogging) {
         if (testLogging == null) {
-            return null;
+            return Collections.emptyMap();
         }
-        Map<String, String> map = new HashMap<>();
+        // use a sorted set so that we apply a parent logger before its children thus not overwriting the child setting when processing the
+        // parent setting
+        final Map<String, String> map = new TreeMap<>();
         final String[] loggersAndLevels = testLogging.value().split(",");
-        for (String loggerAndLevel : loggersAndLevels) {
-            String[] loggerAndLevelArray = loggerAndLevel.split(":");
-            if (loggerAndLevelArray.length >=2) {
-                String loggerName = loggerAndLevelArray[0];
-                String level = loggerAndLevelArray[1];
-                map.put(loggerName, level);
+        for (final String loggerAndLevel : loggersAndLevels) {
+            final String[] loggerAndLevelArray = loggerAndLevel.split(":");
+            if (loggerAndLevelArray.length == 2) {
+                map.put(loggerAndLevelArray[0], loggerAndLevelArray[1]);
+            } else {
+                throw new IllegalArgumentException("invalid test logging annotation [" + loggerAndLevel + "]");
             }
         }
         return map;
     }
 
-    private Map<String, String> reset(Map<String, String> map) {
-        if (map != null) {
-            for (Map.Entry<String, String> previousLogger : map.entrySet()) {
-                Logger logger = resolveLogger(previousLogger.getKey());
-                Loggers.setLevel(logger, previousLogger.getValue());
-            }
+    /**
+     * Reset the logging levels to the state provided by the map.
+     *
+     * @param map the logging levels to apply
+     * @return an empty map
+     */
+    private Map<String, String> reset(final Map<String, String> map) {
+        for (final Map.Entry<String, String> previousLogger : map.entrySet()) {
+            final Logger logger = resolveLogger(previousLogger.getKey());
+            Loggers.setLevel(logger, previousLogger.getValue());
         }
-        return null;
+
+        return Collections.emptyMap();
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java b/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java
index 93ad8bb1e9f61..9ddc0dd58220b 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java
@@ -38,7 +38,6 @@
 import static com.carrotsearch.randomizedtesting.SysGlobals.SYSPROP_TESTMETHOD;
 import static org.elasticsearch.test.ESIntegTestCase.TESTS_CLUSTER;
 import static org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase.REST_TESTS_BLACKLIST;
-import static org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase.REST_TESTS_SPEC;
 import static org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase.REST_TESTS_SUITE;
 
 /**
@@ -143,7 +142,7 @@ public ReproduceErrorMessageBuilder appendESProperties() {
                 appendProperties(TESTS_CLUSTER, ESIntegTestCase.TESTS_ENABLE_MOCK_MODULES);
             }
             appendProperties("tests.assertion.disabled", "tests.security.manager", "tests.nightly", "tests.jvms",
-                             "tests.client.ratio", "tests.heap.size", "tests.bwc", "tests.bwc.version");
+                             "tests.client.ratio", "tests.heap.size", "tests.bwc", "tests.bwc.version", "build.snapshot");
             if (System.getProperty("tests.jvm.argline") != null && !System.getProperty("tests.jvm.argline").isEmpty()) {
                 appendOpt("tests.jvm.argline", "\"" + System.getProperty("tests.jvm.argline") + "\"");
             }
@@ -153,7 +152,7 @@ public ReproduceErrorMessageBuilder appendESProperties() {
         }
 
         public ReproduceErrorMessageBuilder appendClientYamlSuiteProperties() {
-            return appendProperties(REST_TESTS_SUITE, REST_TESTS_SPEC, REST_TESTS_BLACKLIST);
+            return appendProperties(REST_TESTS_SUITE, REST_TESTS_BLACKLIST);
         }
 
         protected ReproduceErrorMessageBuilder appendProperties(String... properties) {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
index 573c301105ad9..4f269199bbfae 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
@@ -32,11 +32,14 @@
 import org.elasticsearch.client.RestClientBuilder;
 import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -55,105 +58,182 @@
 
 import javax.net.ssl.SSLContext;
 
+import static java.util.Collections.singletonMap;
 import static java.util.Collections.sort;
 import static java.util.Collections.unmodifiableList;
 
 /**
  * Superclass for tests that interact with an external test cluster using Elasticsearch's {@link RestClient}.
  */
-public class ESRestTestCase extends ESTestCase {
+public abstract class ESRestTestCase extends ESTestCase {
     public static final String TRUSTSTORE_PATH = "truststore.path";
     public static final String TRUSTSTORE_PASSWORD = "truststore.password";
+    public static final String CLIENT_RETRY_TIMEOUT = "client.retry.timeout";
+    public static final String CLIENT_SOCKET_TIMEOUT = "client.socket.timeout";
 
     /**
      * Convert the entity from a {@link Response} into a map of maps.
      */
-    public static Map<String, Object> entityAsMap(Response response) throws IOException {
+    public Map<String, Object> entityAsMap(Response response) throws IOException {
         XContentType xContentType = XContentType.fromMediaTypeOrFormat(response.getEntity().getContentType().getValue());
-        try (XContentParser parser = xContentType.xContent().createParser(response.getEntity().getContent())) {
+        try (XContentParser parser = createParser(xContentType.xContent(), response.getEntity().getContent())) {
             return parser.map();
         }
     }
 
-    private final List<HttpHost> clusterHosts;
+    private static List<HttpHost> clusterHosts;
     /**
-     * A client for the running Elasticsearch cluster. Lazily initialized on first use.
+     * A client for the running Elasticsearch cluster
      */
-    private final RestClient client;
+    private static RestClient client;
     /**
      * A client for the running Elasticsearch cluster configured to take test administrative actions like remove all indexes after the test
-     * completes. Lazily initialized on first use.
+     * completes
      */
-    private final RestClient adminClient;
+    private static RestClient adminClient;
 
-    public ESRestTestCase() {
-        String cluster = System.getProperty("tests.rest.cluster");
-        if (cluster == null) {
-            throw new RuntimeException("Must specify [tests.rest.cluster] system property with a comma delimited list of [host:port] "
-                    + "to which to send REST requests");
-        }
-        String[] stringUrls = cluster.split(",");
-        List<HttpHost> clusterHosts = new ArrayList<>(stringUrls.length);
-        for (String stringUrl : stringUrls) {
-            int portSeparator = stringUrl.lastIndexOf(':');
-            if (portSeparator < 0) {
-                throw new IllegalArgumentException("Illegal cluster url [" + stringUrl + "]");
+    @Before
+    public void initClient() throws IOException {
+        if (client == null) {
+            assert adminClient == null;
+            assert clusterHosts == null;
+            String cluster = System.getProperty("tests.rest.cluster");
+            if (cluster == null) {
+                throw new RuntimeException("Must specify [tests.rest.cluster] system property with a comma delimited list of [host:port] "
+                        + "to which to send REST requests");
             }
-            String host = stringUrl.substring(0, portSeparator);
-            int port = Integer.valueOf(stringUrl.substring(portSeparator + 1));
-            clusterHosts.add(new HttpHost(host, port, getProtocol()));
-        }
-        this.clusterHosts = unmodifiableList(clusterHosts);
-        try {
+            String[] stringUrls = cluster.split(",");
+            List<HttpHost> hosts = new ArrayList<>(stringUrls.length);
+            for (String stringUrl : stringUrls) {
+                int portSeparator = stringUrl.lastIndexOf(':');
+                if (portSeparator < 0) {
+                    throw new IllegalArgumentException("Illegal cluster url [" + stringUrl + "]");
+                }
+                String host = stringUrl.substring(0, portSeparator);
+                int port = Integer.valueOf(stringUrl.substring(portSeparator + 1));
+                hosts.add(new HttpHost(host, port, getProtocol()));
+            }
+            clusterHosts = unmodifiableList(hosts);
+            logger.info("initializing REST clients against {}", clusterHosts);
             client = buildClient(restClientSettings());
             adminClient = buildClient(restAdminSettings());
-        } catch (IOException e) {
-            // Wrap the IOException so children don't have to declare a constructor just to rethrow it.
-            throw new RuntimeException("Error building clients", e);
         }
+        assert client != null;
+        assert adminClient != null;
+        assert clusterHosts != null;
     }
 
     /**
      * Clean up after the test case.
      */
     @After
-    public final void after() throws Exception {
+    public final void cleanUpCluster() throws Exception {
         wipeCluster();
+        waitForClusterStateUpdatesToFinish();
         logIfThereAreRunningTasks();
-        closeClients();
+    }
+
+    @AfterClass
+    public static void closeClients() throws IOException {
+        try {
+            IOUtils.close(client, adminClient);
+        } finally {
+            clusterHosts = null;
+            client = null;
+            adminClient = null;
+        }
     }
 
     /**
-     * Get a client, building it if it hasn't been built for this test.
+     * Get the client used for ordinary api calls while writing a test
      */
-    protected final RestClient client() {
+    protected static RestClient client() {
         return client;
     }
 
     /**
      * Get the client used for test administrative actions. Do not use this while writing a test. Only use it for cleaning up after tests.
      */
-    protected final RestClient adminClient() {
+    protected static RestClient adminClient() {
         return adminClient;
     }
 
+    /**
+     * Returns whether to preserve the indices created during this test on completion of this test.
+     * Defaults to {@code false}. Override this method if indices should be preserved after the test,
+     * with the assumption that some other process or test will clean up the indices afterward.
+     * This is useful if the data directory and indices need to be preserved between test runs
+     * (for example, when testing rolling upgrades).
+     */
+    protected boolean preserveIndicesUponCompletion() {
+        return false;
+    }
+
+    /**
+     * Controls whether or not to preserve templates upon completion of this test. The default implementation is to delete not preserve
+     * templates.
+     *
+     * @return whether or not to preserve templates
+     */
+    protected boolean preserveTemplatesUponCompletion() {
+        return false;
+    }
+
+    /**
+     * Returns whether to preserve the repositories on completion of this test.
+     */
+    protected boolean preserveReposUponCompletion() {
+        return false;
+    }
+
     private void wipeCluster() throws IOException {
-        // wipe indices
-        try {
-            adminClient().performRequest("DELETE", "*");
-        } catch (ResponseException e) {
-            // 404 here just means we had no indexes
-            if (e.getResponse().getStatusLine().getStatusCode() != 404) {
-                throw e;
+        if (preserveIndicesUponCompletion() == false) {
+            // wipe indices
+            try {
+                adminClient().performRequest("DELETE", "*");
+            } catch (ResponseException e) {
+                // 404 here just means we had no indexes
+                if (e.getResponse().getStatusLine().getStatusCode() != 404) {
+                    throw e;
+                }
             }
         }
 
         // wipe index templates
-        adminClient().performRequest("DELETE", "_template/*");
+        if (preserveTemplatesUponCompletion() == false) {
+            adminClient().performRequest("DELETE", "_template/*");
+        }
 
-        // wipe snapshots
-        // Technically this deletes all repositories and leave the snapshots in the repository. OK.
-        adminClient().performRequest("DELETE", "_snapshot/*");
+        wipeSnapshots();
+    }
+
+    /**
+     * Wipe fs snapshots we created one by one and all repositories so that the next test can create the repositories fresh and they'll
+     * start empty. There isn't an API to delete all snapshots. There is an API to delete all snapshot repositories but that leaves all of
+     * the snapshots intact in the repository.
+     */
+    private void wipeSnapshots() throws IOException {
+        for (Map.Entry<String, ?> repo : entityAsMap(adminClient.performRequest("GET", "_snapshot/_all")).entrySet()) {
+            String repoName = repo.getKey();
+            Map<?, ?> repoSpec = (Map<?, ?>) repo.getValue();
+            String repoType = (String) repoSpec.get("type");
+            if (repoType.equals("fs")) {
+                // All other repo types we really don't have a chance of being able to iterate properly, sadly.
+                String url = "_snapshot/" + repoName + "/_all";
+                Map<String, String> params = singletonMap("ignore_unavailable", "true");
+                List<?> snapshots = (List<?>) entityAsMap(adminClient.performRequest("GET", url, params)).get("snapshots");
+                for (Object snapshot : snapshots) {
+                    Map<?, ?> snapshotInfo = (Map<?, ?>) snapshot;
+                    String name = (String) snapshotInfo.get("snapshot");
+                    logger.debug("wiping snapshot [{}/{}]", repoName, name);
+                    adminClient().performRequest("DELETE", "_snapshot/" + repoName + "/" + name);
+                }
+            }
+            if (preserveReposUponCompletion() == false) {
+                logger.debug("wiping snapshot repository [{}]", repoName);
+                adminClient().performRequest("DELETE", "_snapshot/" + repoName);
+            }
+        }
     }
 
     /**
@@ -178,8 +258,26 @@ private void logIfThereAreRunningTasks() throws InterruptedException, IOExceptio
          */
     }
 
-    private void closeClients() throws IOException {
-        IOUtils.close(client, adminClient);
+    /**
+     * Waits for the cluster state updates to have been processed, so that no cluster
+     * state updates are still in-progress when the next test starts.
+     */
+    private void waitForClusterStateUpdatesToFinish() throws Exception {
+        assertBusy(() -> {
+            try {
+                Response response = adminClient().performRequest("GET", "_cluster/pending_tasks");
+                List<?> tasks = (List<?>) entityAsMap(response).get("tasks");
+                if (false == tasks.isEmpty()) {
+                    StringBuilder message = new StringBuilder("there are still running tasks:");
+                    for (Object task: tasks) {
+                        message.append('\n').append(task.toString());
+                    }
+                    fail(message.toString());
+                }
+            } catch (IOException e) {
+                fail("cannot get cluster's pending tasks: " + e.getMessage());
+            }
+        });
     }
 
     /**
@@ -211,8 +309,7 @@ protected String getProtocol() {
     }
 
     private RestClient buildClient(Settings settings) throws IOException {
-        RestClientBuilder builder = RestClient.builder(clusterHosts.toArray(new HttpHost[0])).setMaxRetryTimeoutMillis(30000)
-                .setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder.setSocketTimeout(30000));
+        RestClientBuilder builder = RestClient.builder(clusterHosts.toArray(new HttpHost[clusterHosts.size()]));
         String keystorePath = settings.get(TRUSTSTORE_PATH);
         if (keystorePath != null) {
             final String keystorePass = settings.get(TRUSTSTORE_PASSWORD);
@@ -244,6 +341,17 @@ private RestClient buildClient(Settings settings) throws IOException {
             }
             builder.setDefaultHeaders(defaultHeaders);
         }
+
+        final String requestTimeoutString = settings.get(CLIENT_RETRY_TIMEOUT);
+        if (requestTimeoutString != null) {
+            final TimeValue maxRetryTimeout = TimeValue.parseTimeValue(requestTimeoutString, CLIENT_RETRY_TIMEOUT);
+            builder.setMaxRetryTimeoutMillis(Math.toIntExact(maxRetryTimeout.getMillis()));
+        }
+        final String socketTimeoutString = settings.get(CLIENT_SOCKET_TIMEOUT);
+        if (socketTimeoutString != null) {
+            final TimeValue socketTimeout = TimeValue.parseTimeValue(socketTimeoutString, CLIENT_SOCKET_TIMEOUT);
+            builder.setRequestConfigCallback(conf -> conf.setSocketTimeout(Math.toIntExact(socketTimeout.getMillis())));
+        }
         return builder.build();
     }
 
@@ -262,5 +370,4 @@ private Set<String> runningTasks(Response response) throws IOException {
         }
         return runningTasks;
     }
-
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestChannel.java b/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestChannel.java
index 3d1ce29143208..bca4af7bd6036 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestChannel.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestChannel.java
@@ -19,9 +19,9 @@
 package org.elasticsearch.test.rest;
 
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.rest.AbstractRestChannel;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -53,8 +53,8 @@ public XContentBuilder newErrorBuilder() throws IOException {
     }
 
     @Override
-    public XContentBuilder newBuilder(@Nullable BytesReference autoDetectSource, boolean useFiltering) throws IOException {
-        return super.newBuilder(autoDetectSource, useFiltering);
+    public XContentBuilder newBuilder(@Nullable XContentType requestContentType, boolean useFiltering) throws IOException {
+        return super.newBuilder(requestContentType, useFiltering);
     }
 
     @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestRequest.java b/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestRequest.java
index f291e0bbeec1f..83caf0293e0ab 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestRequest.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestRequest.java
@@ -20,27 +20,32 @@
 package org.elasticsearch.test.rest;
 
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.rest.RestRequest;
 
+import java.net.SocketAddress;
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 public class FakeRestRequest extends RestRequest {
 
-    private final Map<String, String> headers;
     private final BytesReference content;
     private final Method method;
-
+    private final SocketAddress remoteAddress;
 
     public FakeRestRequest() {
-        this(new HashMap<>(), new HashMap<>(), null, Method.GET, "/");
+        this(NamedXContentRegistry.EMPTY, new HashMap<>(), new HashMap<>(), null, Method.GET, "/", null);
     }
 
-    private FakeRestRequest(Map<String, String> headers, Map<String, String> params, BytesReference content, Method method, String path) {
-        super(params, path);
-        this.headers = headers;
+    private FakeRestRequest(NamedXContentRegistry xContentRegistry, Map<String, List<String>> headers, Map<String, String> params,
+                            BytesReference content, Method method, String path, SocketAddress remoteAddress) {
+        super(xContentRegistry, params, path, headers);
         this.content = content;
         this.method = method;
+        this.remoteAddress = remoteAddress;
     }
 
     @Override
@@ -64,18 +69,14 @@ public BytesReference content() {
     }
 
     @Override
-    public String header(String name) {
-        return headers.get(name);
-    }
-
-    @Override
-    public Iterable<Map.Entry<String, String>> headers() {
-        return headers.entrySet();
+    public SocketAddress getRemoteAddress() {
+        return remoteAddress;
     }
 
     public static class Builder {
+        private final NamedXContentRegistry xContentRegistry;
 
-        private Map<String, String> headers = new HashMap<>();
+        private Map<String, List<String>> headers = new HashMap<>();
 
         private Map<String, String> params = new HashMap<>();
 
@@ -85,7 +86,13 @@ public static class Builder {
 
         private Method method = Method.GET;
 
-        public Builder withHeaders(Map<String, String> headers) {
+        private SocketAddress address = null;
+
+        public Builder(NamedXContentRegistry xContentRegistry) {
+            this.xContentRegistry = xContentRegistry;
+        }
+
+        public Builder withHeaders(Map<String, List<String>> headers) {
             this.headers = headers;
             return this;
         }
@@ -95,8 +102,11 @@ public Builder withParams(Map<String, String> params) {
             return this;
         }
 
-        public Builder withContent(BytesReference content) {
+        public Builder withContent(BytesReference content, XContentType xContentType) {
             this.content = content;
+            if (xContentType != null) {
+                headers.put("Content-Type", Collections.singletonList(xContentType.mediaType()));
+            }
             return this;
         }
 
@@ -110,8 +120,13 @@ public Builder withMethod(Method method) {
             return this;
         }
 
+        public Builder withRemoteAddress(SocketAddress address) {
+            this.address = address;
+            return this;
+        }
+
         public FakeRestRequest build() {
-            return new FakeRestRequest(headers, params, content, method, path);
+            return new FakeRestRequest(xContentRegistry, headers, params, content, method, path, address);
         }
 
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlDocsTestClient.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlDocsTestClient.java
new file mode 100644
index 0000000000000..c7308f69b0b8f
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlDocsTestClient.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.rest.yaml;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.client.RestClient;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpHost;
+import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestSpec;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Used to execute REST requests according to the docs snippets that need to be tests. Wraps a
+ * {@link RestClient} instance used to send the REST requests. Holds the {@link ClientYamlSuiteRestSpec} used to translate api calls into
+ * REST calls. Supports raw requests besides the usual api calls based on the rest spec.
+ */
+public final class ClientYamlDocsTestClient extends ClientYamlTestClient {
+
+    public ClientYamlDocsTestClient(ClientYamlSuiteRestSpec restSpec, RestClient restClient, List<HttpHost> hosts,
+                                    Version esVersion, Map<HttpHost, Version> hostVersionMap)
+            throws IOException {
+        super(restSpec, restClient, hosts, esVersion, hostVersionMap);
+    }
+
+    public ClientYamlTestResponse callApi(String apiName, Map<String, String> params, HttpEntity entity, Map<String, String> headers)
+            throws IOException {
+
+        if ("raw".equals(apiName)) {
+            // Raw requests are bit simpler....
+            Map<String, String> queryStringParams = new HashMap<>(params);
+            String method = Objects.requireNonNull(queryStringParams.remove("method"), "Method must be set to use raw request");
+            String path = "/"+ Objects.requireNonNull(queryStringParams.remove("path"), "Path must be set to use raw request");
+            // And everything else is a url parameter!
+            try {
+                Response response = restClient.performRequest(method, path, queryStringParams, entity);
+                return new ClientYamlTestResponse(response, hostVersionMap.get(response.getHost()));
+            } catch(ResponseException e) {
+                throw new ClientYamlTestResponseException(e, hostVersionMap.get(e.getResponse().getHost()));
+            }
+        }
+        return super.callApi(apiName, params, entity, headers);
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java
index 8040c421dcefb..577898d3cca16 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java
@@ -18,21 +18,20 @@
  */
 package org.elasticsearch.test.rest.yaml;
 
-import com.carrotsearch.randomizedtesting.RandomizedTest;
 import org.apache.http.Header;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpHost;
+import org.apache.http.client.methods.HttpGet;
 import org.apache.http.entity.ContentType;
-import org.apache.http.entity.StringEntity;
 import org.apache.http.message.BasicHeader;
+import org.apache.http.util.EntityUtils;
+import com.carrotsearch.randomizedtesting.RandomizedTest;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Version;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.RestClient;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.logging.Loggers;
-import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestApi;
 import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestPath;
 import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestSpec;
@@ -40,13 +39,9 @@
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
 
 /**
  * Used by {@link ESClientYamlSuiteTestCase} to execute REST requests according to the tests written in yaml suite files. Wraps a
@@ -55,45 +50,21 @@
  */
 public class ClientYamlTestClient {
     private static final Logger logger = Loggers.getLogger(ClientYamlTestClient.class);
-    //query_string params that don't need to be declared in the spec, they are supported by default
-    private static final Set<String> ALWAYS_ACCEPTED_QUERY_STRING_PARAMS = Sets.newHashSet("pretty", "source", "filter_path");
+
+    private static final ContentType YAML_CONTENT_TYPE = ContentType.create("application/yaml");
 
     private final ClientYamlSuiteRestSpec restSpec;
-    private final RestClient restClient;
+    protected final RestClient restClient;
     private final Version esVersion;
+    protected final Map<HttpHost, Version> hostVersionMap;
 
-    public ClientYamlTestClient(ClientYamlSuiteRestSpec restSpec, RestClient restClient, List<HttpHost> hosts) throws IOException {
+    public ClientYamlTestClient(ClientYamlSuiteRestSpec restSpec, RestClient restClient, List<HttpHost> hosts,
+                                Version esVersion, Map<HttpHost, Version> hostVersionMap) throws IOException {
         assert hosts.size() > 0;
         this.restSpec = restSpec;
         this.restClient = restClient;
-        this.esVersion = readAndCheckVersion(hosts);
-    }
-
-    private Version readAndCheckVersion(List<HttpHost> hosts) throws IOException {
-        ClientYamlSuiteRestApi restApi = restApi("info");
-        assert restApi.getPaths().size() == 1;
-        assert restApi.getMethods().size() == 1;
-
-        String version = null;
-        for (HttpHost ignored : hosts) {
-            //we don't really use the urls here, we rely on the client doing round-robin to touch all the nodes in the cluster
-            String method = restApi.getMethods().get(0);
-            String endpoint = restApi.getPaths().get(0);
-            Response response = restClient.performRequest(method, endpoint);
-            ClientYamlTestResponse restTestResponse = new ClientYamlTestResponse(response);
-            Object latestVersion = restTestResponse.evaluate("version.number");
-            if (latestVersion == null) {
-                throw new RuntimeException("elasticsearch version not found in the response");
-            }
-            if (version == null) {
-                version = latestVersion.toString();
-            } else {
-                if (!latestVersion.equals(version)) {
-                    throw new IllegalArgumentException("provided nodes addresses run different elasticsearch versions");
-                }
-            }
-        }
-        return Version.fromString(version);
+        this.esVersion = esVersion;
+        this.hostVersionMap = hostVersionMap;
     }
 
     public Version getEsVersion() {
@@ -103,59 +74,20 @@ public Version getEsVersion() {
     /**
      * Calls an api with the provided parameters and body
      */
-    public ClientYamlTestResponse callApi(String apiName, Map<String, String> params, String body, Map<String, String> headers)
+    public ClientYamlTestResponse callApi(String apiName, Map<String, String> params, HttpEntity entity, Map<String, String> headers)
             throws IOException {
 
-        if ("raw".equals(apiName)) {
-            // Raw requests are bit simpler....
-            Map<String, String> queryStringParams = new HashMap<>(params);
-            String method = Objects.requireNonNull(queryStringParams.remove("method"), "Method must be set to use raw request");
-            String path = "/"+ Objects.requireNonNull(queryStringParams.remove("path"), "Path must be set to use raw request");
-            HttpEntity entity = null;
-            if (body != null && body.length() > 0) {
-                entity = new StringEntity(body, ContentType.APPLICATION_JSON);
-            }
-            // And everything else is a url parameter!
-            try {
-                Response response = restClient.performRequest(method, path, queryStringParams, entity);
-                return new ClientYamlTestResponse(response);
-            } catch(ResponseException e) {
-                throw new ClientYamlTestResponseException(e);
-            }
-        }
-
-        List<Integer> ignores = new ArrayList<>();
-        Map<String, String> requestParams;
-        if (params == null) {
-            requestParams = Collections.emptyMap();
-        } else {
-            requestParams = new HashMap<>(params);
-            if (params.isEmpty() == false) {
-                //ignore is a special parameter supported by the clients, shouldn't be sent to es
-                String ignoreString = requestParams.remove("ignore");
-                if (ignoreString != null) {
-                    try {
-                        ignores.add(Integer.valueOf(ignoreString));
-                    } catch (NumberFormatException e) {
-                        throw new IllegalArgumentException("ignore value should be a number, found [" + ignoreString + "] instead");
-                    }
-                }
-            }
-        }
-
-        //create doesn't exist in the spec but is supported in the clients (index with op_type=create)
-        boolean indexCreateApi = "create".equals(apiName);
-        String api = indexCreateApi ? "index" : apiName;
-        ClientYamlSuiteRestApi restApi = restApi(api);
+        ClientYamlSuiteRestApi restApi = restApi(apiName);
 
         //divide params between ones that go within query string and ones that go within path
         Map<String, String> pathParts = new HashMap<>();
         Map<String, String> queryStringParams = new HashMap<>();
-        for (Map.Entry<String, String> entry : requestParams.entrySet()) {
+        for (Map.Entry<String, String> entry : params.entrySet()) {
             if (restApi.getPathParts().contains(entry.getKey())) {
                 pathParts.put(entry.getKey(), entry.getValue());
             } else {
-                if (restApi.getParams().contains(entry.getKey()) || ALWAYS_ACCEPTED_QUERY_STRING_PARAMS.contains(entry.getKey())) {
+                if (restApi.getParams().contains(entry.getKey()) || restSpec.isGlobalParameter(entry.getKey())
+                        || restSpec.isClientParameter(entry.getKey())) {
                     queryStringParams.put(entry.getKey(), entry.getValue());
                 } else {
                     throw new IllegalArgumentException("param [" + entry.getKey() + "] not supported in ["
@@ -164,25 +96,24 @@ public ClientYamlTestResponse callApi(String apiName, Map<String, String> params
             }
         }
 
-        if (indexCreateApi) {
-            queryStringParams.put("op_type", "create");
-        }
-
         List<String> supportedMethods = restApi.getSupportedMethods(pathParts.keySet());
         String requestMethod;
-        StringEntity requestBody = null;
-        if (Strings.hasLength(body)) {
+        if (entity != null) {
             if (!restApi.isBodySupported()) {
                 throw new IllegalArgumentException("body is not supported by [" + restApi.getName() + "] api");
             }
+            String contentType = entity.getContentType().getValue();
             //randomly test the GET with source param instead of GET/POST with body
-            if (supportedMethods.contains("GET") && RandomizedTest.rarely()) {
+            if (sendBodyAsSourceParam(supportedMethods, contentType)) {
                 logger.debug("sending the request body as source param with GET method");
-                queryStringParams.put("source", body);
-                requestMethod = "GET";
+                queryStringParams.put("source", EntityUtils.toString(entity));
+                if (esVersion.onOrAfter(Version.V_5_3_0)) {
+                    queryStringParams.put("source_content_type", contentType);
+                }
+                requestMethod = HttpGet.METHOD_NAME;
+                entity = null;
             } else {
                 requestMethod = RandomizedTest.randomFrom(supportedMethods);
-                requestBody = new StringEntity(body, ContentType.APPLICATION_JSON);
             }
         } else {
             if (restApi.isBodyRequired()) {
@@ -224,14 +155,21 @@ public ClientYamlTestResponse callApi(String apiName, Map<String, String> params
 
         logger.debug("calling api [{}]", apiName);
         try {
-            Response response = restClient.performRequest(requestMethod, requestPath, queryStringParams, requestBody, requestHeaders);
-            return new ClientYamlTestResponse(response);
+            Response response = restClient.performRequest(requestMethod, requestPath, queryStringParams, entity, requestHeaders);
+            return new ClientYamlTestResponse(response, hostVersionMap.get(response.getHost()));
         } catch(ResponseException e) {
-            if (ignores.contains(e.getResponse().getStatusLine().getStatusCode())) {
-                return new ClientYamlTestResponse(e.getResponse());
+            throw new ClientYamlTestResponseException(e, hostVersionMap.get(e.getResponse().getHost()));
+        }
+    }
+
+    private static boolean sendBodyAsSourceParam(List<String> supportedMethods, String contentType) {
+        if (supportedMethods.contains(HttpGet.METHOD_NAME)) {
+            if (contentType.startsWith(ContentType.APPLICATION_JSON.getMimeType()) ||
+                    contentType.startsWith(YAML_CONTENT_TYPE.getMimeType())) {
+                return RandomizedTest.rarely();
             }
-            throw new ClientYamlTestResponseException(e);
         }
+        return false;
     }
 
     private ClientYamlSuiteRestApi restApi(String apiName) {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java
index 2f1e42c12cb79..6e41919d8e003 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java
@@ -18,15 +18,21 @@
  */
 package org.elasticsearch.test.rest.yaml;
 
-import org.apache.http.HttpHost;
+import com.carrotsearch.randomizedtesting.RandomizedTest;
+import org.apache.http.HttpEntity;
+import org.apache.http.entity.ByteArrayEntity;
+import org.apache.http.entity.ContentType;
 import org.apache.logging.log4j.Logger;
+import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.Version;
-import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestSpec;
+import org.elasticsearch.common.xcontent.XContentType;
 
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -41,16 +47,18 @@ public class ClientYamlTestExecutionContext {
 
     private static final Logger logger = Loggers.getLogger(ClientYamlTestExecutionContext.class);
 
-    private final Stash stash = new Stash();
-
-    private final ClientYamlSuiteRestSpec restSpec;
+    private static final XContentType[] STREAMING_CONTENT_TYPES = new XContentType[]{XContentType.JSON, XContentType.SMILE};
 
-    private ClientYamlTestClient restTestClient;
+    private final Stash stash = new Stash();
+    private final ClientYamlTestClient clientYamlTestClient;
 
     private ClientYamlTestResponse response;
 
-    public ClientYamlTestExecutionContext(ClientYamlSuiteRestSpec restSpec) {
-        this.restSpec = restSpec;
+    private final boolean randomizeContentType;
+
+    ClientYamlTestExecutionContext(ClientYamlTestClient clientYamlTestClient, boolean randomizeContentType) {
+        this.clientYamlTestClient = clientYamlTestClient;
+        this.randomizeContentType = randomizeContentType;
     }
 
     /**
@@ -60,51 +68,97 @@ public ClientYamlTestExecutionContext(ClientYamlSuiteRestSpec restSpec) {
     public ClientYamlTestResponse callApi(String apiName, Map<String, String> params, List<Map<String, Object>> bodies,
                                     Map<String, String> headers) throws IOException {
         //makes a copy of the parameters before modifying them for this specific request
-        HashMap<String, String> requestParams = new HashMap<>(params);
+        Map<String, String> requestParams = new HashMap<>(params);
+        if (false == (esVersion().before(Version.V_5_2_0) && apiName.endsWith("put_settings"))) {
+            // But not for APIs for which it is broken....
+            // You should able to remove this branch and keep the statement below after releasing 5.2
+            requestParams.putIfAbsent("error_trace", "true");
+        }
         for (Map.Entry<String, String> entry : requestParams.entrySet()) {
             if (stash.containsStashedValue(entry.getValue())) {
                 entry.setValue(stash.getValue(entry.getValue()).toString());
             }
         }
 
-        String body = actualBody(bodies);
+        //make a copy of the headers before modifying them for this specific request
+        Map<String, String> requestHeaders = new HashMap<>(headers);
+        for (Map.Entry<String, String> entry : requestHeaders.entrySet()) {
+            if (stash.containsStashedValue(entry.getValue())) {
+                entry.setValue(stash.getValue(entry.getValue()).toString());
+            }
+        }
+
+        HttpEntity entity = createEntity(bodies, requestHeaders);
         try {
-            response = callApiInternal(apiName, requestParams, body, headers);
+            response = callApiInternal(apiName, requestParams, entity, requestHeaders);
             return response;
         } catch(ClientYamlTestResponseException e) {
             response = e.getRestTestResponse();
             throw e;
         } finally {
             // if we hit a bad exception the response is null
-            Object repsponseBody = response != null ? response.getBody() : null;
+            Object responseBody = response != null ? response.getBody() : null;
             //we always stash the last response body
-            stash.stashValue("body", repsponseBody);
+            stash.stashValue("body", responseBody);
         }
     }
 
-    private String actualBody(List<Map<String, Object>> bodies) throws IOException {
+    private HttpEntity createEntity(List<Map<String, Object>> bodies, Map<String, String> headers) throws IOException {
         if (bodies.isEmpty()) {
-            return "";
+            return null;
         }
-
         if (bodies.size() == 1) {
-            return bodyAsString(stash.replaceStashedValues(bodies.get(0)));
+            XContentType xContentType = getContentType(headers, XContentType.values());
+            BytesRef bytesRef = bodyAsBytesRef(bodies.get(0), xContentType);
+            return new ByteArrayEntity(bytesRef.bytes, bytesRef.offset, bytesRef.length,
+                    ContentType.create(xContentType.mediaTypeWithoutParameters(), StandardCharsets.UTF_8));
+        } else {
+            XContentType xContentType = getContentType(headers, STREAMING_CONTENT_TYPES);
+            List<BytesRef> bytesRefList = new ArrayList<>(bodies.size());
+            int totalBytesLength = 0;
+            for (Map<String, Object> body : bodies) {
+                BytesRef bytesRef = bodyAsBytesRef(body, xContentType);
+                bytesRefList.add(bytesRef);
+                totalBytesLength += bytesRef.length - bytesRef.offset + 1;
+            }
+            byte[] bytes = new byte[totalBytesLength];
+            int position = 0;
+            for (BytesRef bytesRef : bytesRefList) {
+                for (int i = bytesRef.offset; i < bytesRef.length; i++) {
+                    bytes[position++] = bytesRef.bytes[i];
+                }
+                bytes[position++] = xContentType.xContent().streamSeparator();
+            }
+            return new ByteArrayEntity(bytes, ContentType.create(xContentType.mediaTypeWithoutParameters(), StandardCharsets.UTF_8));
         }
+    }
 
-        StringBuilder bodyBuilder = new StringBuilder();
-        for (Map<String, Object> body : bodies) {
-            bodyBuilder.append(bodyAsString(stash.replaceStashedValues(body))).append("\n");
+    private XContentType getContentType(Map<String, String> headers, XContentType[] supportedContentTypes) {
+        XContentType xContentType = null;
+        String contentType = headers.get("Content-Type");
+        if (contentType != null) {
+            xContentType = XContentType.fromMediaType(contentType);
+        }
+        if (xContentType != null) {
+            return xContentType;
+        }
+        if (randomizeContentType) {
+            return RandomizedTest.randomFrom(supportedContentTypes);
         }
-        return bodyBuilder.toString();
+        return XContentType.JSON;
     }
 
-    private String bodyAsString(Map<String, Object> body) throws IOException {
-        return XContentFactory.jsonBuilder().map(body).string();
+    private BytesRef bodyAsBytesRef(Map<String, Object> bodyAsMap, XContentType xContentType) throws IOException {
+        Map<String, Object> finalBodyAsMap = stash.replaceStashedValues(bodyAsMap);
+        try (XContentBuilder builder = XContentFactory.contentBuilder(xContentType)) {
+            return builder.map(finalBodyAsMap).bytes().toBytesRef();
+        }
     }
 
-    private ClientYamlTestResponse callApiInternal(String apiName, Map<String, String> params, String body, Map<String, String> headers)
-            throws IOException  {
-        return restTestClient.callApi(apiName, params, body, headers);
+    // pkg-private for testing
+    ClientYamlTestResponse callApiInternal(String apiName, Map<String, String> params,
+                                                   HttpEntity entity, Map<String, String> headers) throws IOException  {
+        return clientYamlTestClient.callApi(apiName, params, entity, headers);
     }
 
     /**
@@ -114,13 +168,6 @@ public Object response(String path) throws IOException {
         return response.evaluate(path, stash);
     }
 
-    /**
-     * Creates the embedded REST client when needed. Needs to be called before each test.
-     */
-    public void initClient(RestClient client, List<HttpHost> hosts) throws IOException {
-        restTestClient = new ClientYamlTestClient(restSpec, client, hosts);
-    }
-
     /**
      * Clears the last obtained response and the stashed fields
      */
@@ -138,7 +185,7 @@ public Stash stash() {
      * Returns the current es version as a string
      */
     public Version esVersion() {
-        return restTestClient.getEsVersion();
+        return clientYamlTestClient.getEsVersion();
     }
 
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponse.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponse.java
index 481ae752d05f0..18bf1c393017d 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponse.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponse.java
@@ -21,10 +21,17 @@
 import org.apache.http.Header;
 import org.apache.http.client.methods.HttpHead;
 import org.apache.http.util.EntityUtils;
+import org.elasticsearch.Version;
 import org.elasticsearch.client.Response;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 
 import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.List;
@@ -36,32 +43,32 @@
 public class ClientYamlTestResponse {
 
     private final Response response;
-    private final String body;
+    private final Version nodeVersion;
+    private final byte[] body;
+    private final XContentType bodyContentType;
     private ObjectPath parsedResponse;
+    private String bodyAsString;
 
-    ClientYamlTestResponse(Response response) throws IOException {
+    public ClientYamlTestResponse(Response response, Version version) throws IOException {
         this.response = response;
+        this.nodeVersion = version;
         if (response.getEntity() != null) {
+            String contentType = response.getHeader("Content-Type");
+            this.bodyContentType = XContentType.fromMediaTypeOrFormat(contentType);
             try {
-                this.body = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
+                byte[] bytes = EntityUtils.toByteArray(response.getEntity());
+                //skip parsing if we got text back (e.g. if we called _cat apis)
+                if (bodyContentType != null) {
+                    this.parsedResponse = ObjectPath.createFromXContent(bodyContentType.xContent(), new BytesArray(bytes));
+                }
+                this.body = bytes;
             } catch (IOException e) {
                 EntityUtils.consumeQuietly(response.getEntity());
-                throw new RuntimeException(e);
+                throw e;
             }
         } else {
             this.body = null;
-        }
-        parseResponseBody();
-    }
-
-    private void parseResponseBody() throws IOException {
-        if (body != null) {
-            String contentType = response.getHeader("Content-Type");
-            XContentType xContentType = XContentType.fromMediaTypeOrFormat(contentType);
-            //skip parsing if we got text back (e.g. if we called _cat apis)
-            if (xContentType == XContentType.JSON || xContentType == XContentType.YAML) {
-                this.parsedResponse = ObjectPath.createFromXContent(xContentType.xContent(), body);
-            }
+            this.bodyContentType = null;
         }
     }
 
@@ -80,7 +87,17 @@ public List<String> getWarningHeaders() {
         List<String> warningHeaders = new ArrayList<>();
         for (Header header : response.getHeaders()) {
             if (header.getName().equals("Warning")) {
-                warningHeaders.add(header.getValue());
+                if (nodeVersion.onOrAfter(Version.V_5_3_0) && response.getRequestLine().getMethod().equals("GET")
+                    && response.getRequestLine().getUri().contains("source")
+                    && response.getRequestLine().getUri().contains("source_content_type") == false && header.getValue().contains(
+                        "Deprecated use of the [source] parameter without the [source_content_type] parameter.")) {
+                    // this is because we do not send the source content type header when the node is 5.3.0 or below and the request
+                    // might have been sent to a node with a version > 5.3.0 when running backwards 5.0 tests. The Java RestClient
+                    // has control of the node the request is sent to so we can only detect this after the fact right now
+                } else {
+                    warningHeaders.add(header.getValue());
+                }
+
             }
         }
         return warningHeaders;
@@ -94,14 +111,32 @@ public Object getBody() throws IOException {
         if (parsedResponse != null) {
             return parsedResponse.evaluate("");
         }
-        return body;
+        //we only get here if there is no response body or the body is text
+        assert bodyContentType == null;
+        return getBodyAsString();
     }
 
     /**
      * Returns the body as a string
      */
     public String getBodyAsString() {
-        return body;
+        if (bodyAsString == null && body != null) {
+            //content-type null means that text was returned
+            if (bodyContentType == null || bodyContentType == XContentType.JSON || bodyContentType == XContentType.YAML) {
+                bodyAsString = new String(body, StandardCharsets.UTF_8);
+            } else {
+                //if the body is in a binary format and gets requested as a string (e.g. to log a test failure), we convert it to json
+                try (XContentBuilder jsonBuilder = XContentFactory.jsonBuilder()) {
+                    try (XContentParser parser = bodyContentType.xContent().createParser(NamedXContentRegistry.EMPTY, body)) {
+                        jsonBuilder.copyCurrentStructure(parser);
+                    }
+                    bodyAsString = jsonBuilder.string();
+                } catch (IOException e) {
+                    throw new UncheckedIOException("unable to convert response body to a string format", e);
+                }
+            }
+        }
+        return bodyAsString;
     }
 
     public boolean isError() {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponseException.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponseException.java
index 7d983d480296b..5575274747067 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponseException.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponseException.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.test.rest.yaml;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.client.ResponseException;
 
 import java.io.IOException;
@@ -32,10 +33,10 @@ public class ClientYamlTestResponseException extends IOException {
     private final ClientYamlTestResponse restTestResponse;
     private final ResponseException responseException;
 
-    ClientYamlTestResponseException(ResponseException responseException) throws IOException {
+    public ClientYamlTestResponseException(ResponseException responseException, Version version) throws IOException {
         super(responseException);
         this.responseException = responseException;
-        this.restTestResponse = new ClientYamlTestResponse(responseException.getResponse());
+        this.restTestResponse = new ClientYamlTestResponse(responseException.getResponse(), version);
     }
 
     /**
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java
index 2e29721f06e09..4ede41c9d5aa2 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java
@@ -19,42 +19,36 @@
 
 package org.elasticsearch.test.rest.yaml;
 
-import com.carrotsearch.randomizedtesting.RandomizedTest;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 
-import org.apache.lucene.util.IOUtils;
+import com.carrotsearch.randomizedtesting.RandomizedTest;
+import org.apache.http.HttpHost;
+import org.elasticsearch.Version;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.test.rest.ESRestTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestSuiteParser;
 import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestApi;
 import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestSpec;
+import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection;
 import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSuite;
 import org.elasticsearch.test.rest.yaml.section.DoSection;
 import org.elasticsearch.test.rest.yaml.section.ExecutableSection;
-import org.elasticsearch.test.rest.yaml.section.SkipSection;
-import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection;
 import org.junit.AfterClass;
 import org.junit.Before;
-import org.junit.BeforeClass;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.nio.file.FileSystem;
-import java.nio.file.FileSystems;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.StandardCopyOption;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
 
 /**
  * Runs a suite of yaml tests shared with all the official Elasticsearch clients against against an elasticsearch cluster.
@@ -74,16 +68,10 @@ public abstract class ESClientYamlSuiteTestCase extends ESRestTestCase {
     /**
      * Property that allows to control whether spec validation is enabled or not (default true).
      */
-    public static final String REST_TESTS_VALIDATE_SPEC = "tests.rest.validate_spec";
-    /**
-     * Property that allows to control where the REST spec files need to be loaded from
-     */
-    public static final String REST_TESTS_SPEC = "tests.rest.spec";
-
-    public static final String REST_LOAD_PACKAGED_TESTS = "tests.rest.load_packaged";
+    private static final String REST_TESTS_VALIDATE_SPEC = "tests.rest.validate_spec";
 
-    private static final String DEFAULT_TESTS_PATH = "/rest-api-spec/test";
-    private static final String DEFAULT_SPEC_PATH = "/rest-api-spec/api";
+    private static final String TESTS_PATH = "/rest-api-spec/test";
+    private static final String SPEC_PATH = "/rest-api-spec/api";
 
     /**
      * This separator pattern matches ',' except it is preceded by a '\'.
@@ -96,77 +84,127 @@ public abstract class ESClientYamlSuiteTestCase extends ESRestTestCase {
      */
     private static final String PATHS_SEPARATOR = "(?<!\\\\),";
 
-    private final List<BlacklistedPathPatternMatcher> blacklistPathMatchers = new ArrayList<>();
+    private static List<BlacklistedPathPatternMatcher> blacklistPathMatchers;
     private static ClientYamlTestExecutionContext restTestExecutionContext;
     private static ClientYamlTestExecutionContext adminExecutionContext;
 
     private final ClientYamlTestCandidate testCandidate;
 
-    public ESClientYamlSuiteTestCase(ClientYamlTestCandidate testCandidate) {
+    protected ESClientYamlSuiteTestCase(ClientYamlTestCandidate testCandidate) {
         this.testCandidate = testCandidate;
-        String[] blacklist = resolvePathsProperty(REST_TESTS_BLACKLIST, null);
-        for (String entry : blacklist) {
-            this.blacklistPathMatchers.add(new BlacklistedPathPatternMatcher(entry));
+    }
+
+    @Before
+    public void initAndResetContext() throws Exception {
+        if (restTestExecutionContext == null) {
+            assert adminExecutionContext == null;
+            assert blacklistPathMatchers == null;
+            ClientYamlSuiteRestSpec restSpec = ClientYamlSuiteRestSpec.load(SPEC_PATH);
+            validateSpec(restSpec);
+            List<HttpHost> hosts = getClusterHosts();
+            RestClient restClient = client();
+            Tuple<Version, Map<HttpHost, Version>> versionMapTuple = readVersionsFromInfo(restClient, hosts.size());
+            Version esVersion;
+            try {
+                Tuple<Version, Version> versionVersionTuple = readVersionsFromCatNodes(restClient);
+                esVersion = versionVersionTuple.v1();
+                Version masterVersion = versionVersionTuple.v2();
+                logger.info("initializing yaml client, minimum es version: [{}] master version: [{}] hosts: {}",
+                        esVersion, masterVersion, hosts);
+            } catch (ResponseException ex) {
+                if (ex.getResponse().getStatusLine().getStatusCode() == 403) {
+                    logger.warn("Fallback to simple info '/' request, _cat/nodes is not authorized");
+                    esVersion = versionMapTuple.v1();
+                    logger.info("initializing yaml client, minimum es version: [{}] hosts: {}", esVersion, hosts);
+                } else {
+                    throw ex;
+                }
+            }
+            ClientYamlTestClient clientYamlTestClient = initClientYamlTestClient(restSpec, restClient, hosts,
+                    esVersion, versionMapTuple.v2());
+            restTestExecutionContext = new ClientYamlTestExecutionContext(clientYamlTestClient, randomizeContentType());
+            adminExecutionContext = new ClientYamlTestExecutionContext(clientYamlTestClient, false);
+            String[] blacklist = resolvePathsProperty(REST_TESTS_BLACKLIST, null);
+            blacklistPathMatchers = new ArrayList<>();
+            for (String entry : blacklist) {
+                blacklistPathMatchers.add(new BlacklistedPathPatternMatcher(entry));
+            }
         }
-        
+        assert restTestExecutionContext != null;
+        assert adminExecutionContext != null;
+        assert blacklistPathMatchers != null;
+
+        // admin context must be available for @After always, regardless of whether the test was blacklisted
+        adminExecutionContext.clear();
+
+        restTestExecutionContext.clear();
+    }
+
+    protected ClientYamlTestClient initClientYamlTestClient(ClientYamlSuiteRestSpec restSpec, RestClient restClient, List<HttpHost> hosts,
+                                                            Version esVersion, Map<HttpHost, Version> hostVersionMap) throws IOException {
+        return new ClientYamlTestClient(restSpec, restClient, hosts, esVersion, hostVersionMap);
     }
 
     @Override
     protected void afterIfFailed(List<Throwable> errors) {
-        logger.info("Stash dump on failure [{}]", XContentHelper.toString(restTestExecutionContext.stash()));
+        // Dump the stash on failure. Instead of dumping it in true json we escape `\n`s so stack traces are easier to read
+        logger.info("Stash dump on failure [{}]",
+                XContentHelper.toString(restTestExecutionContext.stash()).replace("\\n", "\n").replace("\\r", "\r").replace("\\t", "\t"));
         super.afterIfFailed(errors);
     }
 
-    public static Iterable<Object[]> createParameters(int id, int count) throws IOException, ClientYamlTestParseException {
-        //parse tests only if rest test group is enabled, otherwise rest tests might not even be available on file system
-        List<ClientYamlTestCandidate> restTestCandidates = collectTestCandidates(id, count);
-        List<Object[]> objects = new ArrayList<>();
-        for (ClientYamlTestCandidate restTestCandidate : restTestCandidates) {
-            objects.add(new Object[]{restTestCandidate});
-        }
-        return objects;
-    }
-
-    private static List<ClientYamlTestCandidate> collectTestCandidates(int id, int count) throws ClientYamlTestParseException, IOException {
-        List<ClientYamlTestCandidate> testCandidates = new ArrayList<>();
-        FileSystem fileSystem = getFileSystem();
-        // don't make a try-with, getFileSystem returns null
-        // ... and you can't close() the default filesystem
-        try {
-            String[] paths = resolvePathsProperty(REST_TESTS_SUITE, DEFAULT_TESTS_PATH);
-            Map<String, Set<Path>> yamlSuites = FileUtils.findYamlSuites(fileSystem, DEFAULT_TESTS_PATH, paths);
-            ClientYamlTestSuiteParser restTestSuiteParser = new ClientYamlTestSuiteParser();
-            //yaml suites are grouped by directory (effectively by api)
-            for (String api : yamlSuites.keySet()) {
-                List<Path> yamlFiles = new ArrayList<>(yamlSuites.get(api));
-                for (Path yamlFile : yamlFiles) {
-                    String key = api + yamlFile.getFileName().toString();
-                    if (mustExecute(key, id, count)) {
-                        ClientYamlTestSuite restTestSuite = restTestSuiteParser.parse(api, yamlFile);
-                        for (ClientYamlTestSection testSection : restTestSuite.getTestSections()) {
-                            testCandidates.add(new ClientYamlTestCandidate(restTestSuite, testSection));
-                        }
-                    }
+    public static Iterable<Object[]> createParameters() throws Exception {
+        String[] paths = resolvePathsProperty(REST_TESTS_SUITE, ""); // default to all tests under the test root
+        List<Object[]> tests = new ArrayList<>();
+        Map<String, Set<Path>> yamlSuites = loadYamlSuites(paths);
+        // yaml suites are grouped by directory (effectively by api)
+        for (String api : yamlSuites.keySet()) {
+            List<Path> yamlFiles = new ArrayList<>(yamlSuites.get(api));
+            for (Path yamlFile : yamlFiles) {
+                ClientYamlTestSuite restTestSuite = ClientYamlTestSuite.parse(api, yamlFile);
+                for (ClientYamlTestSection testSection : restTestSuite.getTestSections()) {
+                    tests.add(new Object[]{ new ClientYamlTestCandidate(restTestSuite, testSection) });
                 }
             }
-        } finally {
-            IOUtils.close(fileSystem);
         }
 
         //sort the candidates so they will always be in the same order before being shuffled, for repeatability
-        Collections.sort(testCandidates, new Comparator<ClientYamlTestCandidate>() {
-            @Override
-            public int compare(ClientYamlTestCandidate o1, ClientYamlTestCandidate o2) {
-                return o1.getTestPath().compareTo(o2.getTestPath());
-            }
-        });
+        Collections.sort(tests,
+            (o1, o2) -> ((ClientYamlTestCandidate)o1[0]).getTestPath().compareTo(((ClientYamlTestCandidate)o2[0]).getTestPath()));
+        return tests;
+    }
 
-        return testCandidates;
+    /** Find all yaml suites that match the given list of paths from the root test path. */
+    // pkg private for tests
+    static Map<String, Set<Path>> loadYamlSuites(String... paths) throws Exception {
+        Map<String, Set<Path>> files = new HashMap<>();
+        Path root = PathUtils.get(ESClientYamlSuiteTestCase.class.getResource(TESTS_PATH).toURI());
+        for (String strPath : paths) {
+            Path path = root.resolve(strPath);
+            if (Files.isDirectory(path)) {
+                Files.walk(path).forEach(file -> {
+                    if (file.toString().endsWith(".yaml")) {
+                        addYamlSuite(root, file, files);
+                    }
+                });
+            } else {
+                path = root.resolve(strPath + ".yaml");
+                assert Files.exists(path);
+                addYamlSuite(root, path, files);
+            }
+        }
+        return files;
     }
 
-    private static boolean mustExecute(String test, int id, int count) {
-        int hash = (int) (Math.abs((long)test.hashCode()) % count);
-        return hash == id;
+    /** Add a single suite file to the set of suites. */
+    private static void addYamlSuite(Path root, Path file, Map<String, Set<Path>> files) {
+        String groupName = root.relativize(file.getParent()).toString();
+        Set<Path> filesSet = files.get(groupName);
+        if (filesSet == null) {
+            filesSet = new HashSet<>();
+            files.put(groupName, filesSet);
+        }
+        filesSet.add(file);
     }
 
     private static String[] resolvePathsProperty(String propertyName, String defaultValue) {
@@ -178,51 +216,6 @@ private static String[] resolvePathsProperty(String propertyName, String default
         }
     }
 
-    /**
-     * Returns a new FileSystem to read REST resources, or null if they
-     * are available from classpath.
-     */
-    @SuppressForbidden(reason = "proper use of URL, hack around a JDK bug")
-    static FileSystem getFileSystem() throws IOException {
-        // REST suite handling is currently complicated, with lots of filtering and so on
-        // For now, to work embedded in a jar, return a ZipFileSystem over the jar contents.
-        URL codeLocation = FileUtils.class.getProtectionDomain().getCodeSource().getLocation();
-        boolean loadPackaged = RandomizedTest.systemPropertyAsBoolean(REST_LOAD_PACKAGED_TESTS, true);
-        if (codeLocation.getFile().endsWith(".jar") && loadPackaged) {
-            try {
-                // hack around a bug in the zipfilesystem implementation before java 9,
-                // its checkWritable was incorrect and it won't work without write permissions.
-                // if we add the permission, it will open jars r/w, which is too scary! so copy to a safe r-w location.
-                Path tmp = Files.createTempFile(null, ".jar");
-                try (InputStream in = codeLocation.openStream()) {
-                    Files.copy(in, tmp, StandardCopyOption.REPLACE_EXISTING);
-                }
-                return FileSystems.newFileSystem(new URI("jar:" + tmp.toUri()), Collections.<String,Object>emptyMap());
-            } catch (URISyntaxException e) {
-                throw new IOException("couldn't open zipfilesystem: ", e);
-            }
-        } else {
-            return null;
-        }
-    }
-
-    @BeforeClass
-    public static void initExecutionContext() throws IOException {
-        String[] specPaths = resolvePathsProperty(REST_TESTS_SPEC, DEFAULT_SPEC_PATH);
-        ClientYamlSuiteRestSpec restSpec = null;
-        FileSystem fileSystem = getFileSystem();
-        // don't make a try-with, getFileSystem returns null
-        // ... and you can't close() the default filesystem
-        try {
-            restSpec = ClientYamlSuiteRestSpec.parseFrom(fileSystem, DEFAULT_SPEC_PATH, specPaths);
-        } finally {
-            IOUtils.close(fileSystem);
-        }
-        validateSpec(restSpec);
-        restTestExecutionContext = new ClientYamlTestExecutionContext(restSpec);
-        adminExecutionContext = new ClientYamlTestExecutionContext(restSpec);
-    }
-
     protected ClientYamlTestExecutionContext getAdminExecutionContext() {
         return adminExecutionContext;
     }
@@ -246,49 +239,77 @@ private static void validateSpec(ClientYamlSuiteRestSpec restSpec) {
 
     @AfterClass
     public static void clearStatic() {
+        blacklistPathMatchers = null;
         restTestExecutionContext = null;
         adminExecutionContext = null;
     }
 
-    @Before
-    public void reset() throws IOException {
-        // admin context must be available for @After always, regardless of whether the test was blacklisted
-        adminExecutionContext.initClient(adminClient(), getClusterHosts());
-        adminExecutionContext.clear();
+    private static Tuple<Version, Version> readVersionsFromCatNodes(RestClient restClient) throws IOException {
+        // we simply go to the _cat/nodes API and parse all versions in the cluster
+        Response response = restClient.performRequest("GET", "/_cat/nodes", Collections.singletonMap("h", "version,master"));
+        ClientYamlTestResponse restTestResponse = new ClientYamlTestResponse(response, Version.CURRENT);
+        String nodesCatResponse = restTestResponse.getBodyAsString();
+        String[] split = nodesCatResponse.split("\n");
+        Version version = null;
+        Version masterVersion = null;
+        for (String perNode : split) {
+            final String[] versionAndMaster = perNode.split("\\s+");
+            assert versionAndMaster.length == 2 : "invalid line: " + perNode + " length: " + versionAndMaster.length;
+            final Version currentVersion = Version.fromString(versionAndMaster[0]);
+            final boolean master = versionAndMaster[1].trim().equals("*");
+            if (master) {
+                assert masterVersion == null;
+                masterVersion = currentVersion;
+            }
+            if (version == null) {
+                version = currentVersion;
+            } else if (version.onOrAfter(currentVersion)) {
+                version = currentVersion;
+            }
+        }
+        return new Tuple<>(version, masterVersion);
+    }
+
+    private static Tuple<Version, Map<HttpHost, Version>> readVersionsFromInfo(RestClient restClient, int numHosts) throws IOException {
+        Version version = null;
+        Map<HttpHost, Version> hostVersionMap = new HashMap<>();
+        for (int i = 0; i < numHosts; i++) {
+            //we don't really use the urls here, we rely on the client doing round-robin to touch all the nodes in the cluster
+            Response response = restClient.performRequest("GET", "/");
+            ClientYamlTestResponse restTestResponse = new ClientYamlTestResponse(response, Version.CURRENT);
+            Object latestVersion = restTestResponse.evaluate("version.number");
+            if (latestVersion == null) {
+                throw new RuntimeException("elasticsearch version not found in the response");
+            }
+            final Version currentVersion = Version.fromString(latestVersion.toString());
+            if (version == null) {
+                version = currentVersion;
+            } else if (version.onOrAfter(currentVersion)) {
+                version = currentVersion;
+            }
+            hostVersionMap.put(response.getHost(), currentVersion);
+        }
+        return new Tuple<>(version, Collections.unmodifiableMap(hostVersionMap));
+    }
 
+    public void test() throws IOException {
         //skip test if it matches one of the blacklist globs
         for (BlacklistedPathPatternMatcher blacklistedPathMatcher : blacklistPathMatchers) {
             String testPath = testCandidate.getSuitePath() + "/" + testCandidate.getTestSection().getName();
             assumeFalse("[" + testCandidate.getTestPath() + "] skipped, reason: blacklisted", blacklistedPathMatcher
-                    .isSuffixMatch(testPath));
+                .isSuffixMatch(testPath));
         }
-        //The client needs non static info to get initialized, therefore it can't be initialized in the before class
-        restTestExecutionContext.initClient(client(), getClusterHosts());
-        restTestExecutionContext.clear();
 
         //skip test if the whole suite (yaml file) is disabled
-        assumeFalse(buildSkipMessage(testCandidate.getSuitePath(), testCandidate.getSetupSection().getSkipSection()),
-                testCandidate.getSetupSection().getSkipSection().skip(restTestExecutionContext.esVersion()));
+        assumeFalse(testCandidate.getSetupSection().getSkipSection().getSkipMessage(testCandidate.getSuitePath()),
+            testCandidate.getSetupSection().getSkipSection().skip(restTestExecutionContext.esVersion()));
         //skip test if the whole suite (yaml file) is disabled
-        assumeFalse(buildSkipMessage(testCandidate.getSuitePath(), testCandidate.getTeardownSection().getSkipSection()),
+        assumeFalse(testCandidate.getTeardownSection().getSkipSection().getSkipMessage(testCandidate.getSuitePath()),
             testCandidate.getTeardownSection().getSkipSection().skip(restTestExecutionContext.esVersion()));
         //skip test if test section is disabled
-        assumeFalse(buildSkipMessage(testCandidate.getTestPath(), testCandidate.getTestSection().getSkipSection()),
-                testCandidate.getTestSection().getSkipSection().skip(restTestExecutionContext.esVersion()));
-    }
-
-    private static String buildSkipMessage(String description, SkipSection skipSection) {
-        StringBuilder messageBuilder = new StringBuilder();
-        if (skipSection.isVersionCheck()) {
-            messageBuilder.append("[").append(description).append("] skipped, reason: [").append(skipSection.getReason()).append("] ");
-        } else {
-            messageBuilder.append("[").append(description).append("] skipped, reason: features ")
-                    .append(skipSection.getFeatures()).append(" not supported");
-        }
-        return messageBuilder.toString();
-    }
+        assumeFalse(testCandidate.getTestSection().getSkipSection().getSkipMessage(testCandidate.getTestPath()),
+            testCandidate.getTestSection().getSkipSection().skip(restTestExecutionContext.esVersion()));
 
-    public void test() throws IOException {
         //let's check that there is something to run, otherwise there might be a problem with the test section
         if (testCandidate.getTestSection().getExecutableSections().size() == 0) {
             throw new IllegalArgumentException("No executable sections loaded for [" + testCandidate.getTestPath() + "]");
@@ -333,4 +354,8 @@ private void executeSection(ExecutableSection executableSection) {
     private String errorMessage(ExecutableSection executableSection, Throwable t) {
         return "Failure at [" + testCandidate.getSuitePath() + ":" + executableSection.getLocation().lineNumber + "]: " + t.getMessage();
     }
+
+    protected boolean randomizeContentType() {
+        return true;
+    }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/Features.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/Features.java
index 6e47220ec083b..68cc525f4f2a5 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/Features.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/Features.java
@@ -40,7 +40,9 @@ public final class Features {
             "embedded_stash_key",
             "groovy_scripting",
             "headers",
+            "stash_in_key",
             "stash_in_path",
+            "stash_path_replace",
             "warnings",
             "yaml"));
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/FileUtils.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/FileUtils.java
deleted file mode 100644
index 4519953819a88..0000000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/FileUtils.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml;
-
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.io.PathUtils;
-
-import java.io.IOException;
-import java.net.URL;
-import java.nio.file.DirectoryStream;
-import java.nio.file.FileSystem;
-import java.nio.file.FileVisitResult;
-import java.nio.file.Files;
-import java.nio.file.NoSuchFileException;
-import java.nio.file.NotDirectoryException;
-import java.nio.file.Path;
-import java.nio.file.SimpleFileVisitor;
-import java.nio.file.attribute.BasicFileAttributes;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-public final class FileUtils {
-
-    private static final String YAML_SUFFIX = ".yaml";
-    private static final String JSON_SUFFIX = ".json";
-
-    private FileUtils() {
-
-    }
-
-    /**
-     * Returns the json files found within the directory provided as argument.
-     * Files are looked up in the classpath, or optionally from {@code fileSystem} if its not null.
-     */
-    public static Set<Path> findJsonSpec(FileSystem fileSystem, String optionalPathPrefix, String path) throws IOException {
-        Path dir = resolveFile(fileSystem, optionalPathPrefix, path, null);
-
-        if (!Files.isDirectory(dir)) {
-            throw new NotDirectoryException(path);
-        }
-
-        Set<Path> jsonFiles = new HashSet<>();
-        try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir)) {
-            for (Path item : stream) {
-                if (item.toString().endsWith(JSON_SUFFIX)) {
-                    jsonFiles.add(item);
-                }
-            }
-        }
-
-        if (jsonFiles.isEmpty()) {
-            throw new NoSuchFileException(path, null, "no json files found");
-        }
-
-        return jsonFiles;
-    }
-
-    /**
-     * Returns the yaml files found within the paths provided.
-     * Each input path can either be a single file (the .yaml suffix is optional) or a directory.
-     * Each path is looked up in the classpath, or optionally from {@code fileSystem} if its not null.
-     */
-    public static Map<String, Set<Path>> findYamlSuites(FileSystem fileSystem, String optionalPathPrefix, final String... paths)
-            throws IOException {
-        Map<String, Set<Path>> yamlSuites = new HashMap<>();
-        for (String path : paths) {
-            collectFiles(resolveFile(fileSystem, optionalPathPrefix, path, YAML_SUFFIX), YAML_SUFFIX, yamlSuites);
-        }
-        return yamlSuites;
-    }
-
-    private static Path resolveFile(FileSystem fileSystem, String optionalPathPrefix, String path, String optionalFileSuffix)
-            throws IOException {
-        if (fileSystem != null) {
-            Path file = findFile(fileSystem, path, optionalFileSuffix);
-            if (!lenientExists(file)) {
-                // try with optional prefix: /rest-api-spec/test (or /rest-api-spec/api) is optional
-                String newPath = optionalPathPrefix + "/" + path;
-                file = findFile(fileSystem, newPath, optionalFileSuffix);
-                if (!lenientExists(file)) {
-                    throw new NoSuchFileException("path prefix: " + optionalPathPrefix + ", path: " + path + ", file suffix: "
-                            + optionalFileSuffix);
-                }
-            }
-            return file;
-        } else {
-            //try within classpath
-            URL resource = findResource(path, optionalFileSuffix);
-            if (resource == null) {
-                //try within classpath with optional prefix: /rest-api-spec/test (or /rest-api-spec/api) is optional
-                String newPath = optionalPathPrefix + "/" + path;
-                resource = findResource(newPath, optionalFileSuffix);
-                if (resource == null) {
-                    throw new NoSuchFileException(path);
-                }
-            }
-            try {
-                return PathUtils.get(resource.toURI());
-            } catch (Exception e) {
-                // some filesystems have REALLY useless exceptions here.
-                // ZipFileSystem I am looking at you.
-                throw new RuntimeException("couldn't retrieve URL: " + resource, e);
-            }
-        }
-    }
-
-    private static URL findResource(String path, String optionalFileSuffix) {
-        URL resource = FileUtils.class.getResource(path);
-        if (resource == null) {
-            //if not found we append the file suffix to the path (as it is optional)
-            if (Strings.hasLength(optionalFileSuffix) && !path.endsWith(optionalFileSuffix)) {
-                resource = FileUtils.class.getResource(path + optionalFileSuffix);
-            }
-        }
-        return resource;
-    }
-    
-    // used because this test "guesses" from like 4 different places from the filesystem!
-    private static boolean lenientExists(Path file) {
-        boolean exists = false; 
-        try {
-            exists = Files.exists(file);
-        } catch (SecurityException ok) {}
-        return exists;
-    }
-
-    private static Path findFile(FileSystem fileSystem, String path, String optionalFileSuffix) {
-        Path file = fileSystem.getPath(path);
-        if (!lenientExists(file)) {
-            file = fileSystem.getPath(path + optionalFileSuffix);
-        }
-        return file;
-    }
-
-    private static void collectFiles(final Path dir, final String fileSuffix, final Map<String, Set<Path>> files) throws IOException {
-        Files.walkFileTree(dir, new SimpleFileVisitor<Path>() {
-            @Override
-            public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
-                if (file.toString().endsWith(fileSuffix)) {
-                    String groupName = dir.relativize(file.getParent()).toString();
-                    Set<Path> filesSet = files.get(groupName);
-                    if (filesSet == null) {
-                        filesSet = new HashSet<>();
-                        files.put(groupName, filesSet);
-                    }
-                    filesSet.add(file);
-                }
-                return FileVisitResult.CONTINUE;
-            }
-        });
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ObjectPath.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ObjectPath.java
index 6311944fdcbb7..18d8e69d32c8d 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ObjectPath.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ObjectPath.java
@@ -18,8 +18,15 @@
  */
 package org.elasticsearch.test.rest.yaml;
 
+import org.apache.http.util.EntityUtils;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -33,8 +40,15 @@ public class ObjectPath {
 
     private final Object object;
 
-    public static ObjectPath createFromXContent(XContent xContent, String input) throws IOException {
-        try (XContentParser parser = xContent.createParser(input)) {
+    public static ObjectPath createFromResponse(Response response) throws IOException {
+        byte[] bytes = EntityUtils.toByteArray(response.getEntity());
+        String contentType = response.getHeader("Content-Type");
+        XContentType xContentType = XContentType.fromMediaTypeOrFormat(contentType);
+        return ObjectPath.createFromXContent(xContentType.xContent(), new BytesArray(bytes));
+    }
+
+    public static ObjectPath createFromXContent(XContent xContent, BytesReference input) throws IOException {
+        try (XContentParser parser = xContent.createParser(NamedXContentRegistry.EMPTY, input)) {
             if (parser.nextToken() == XContentParser.Token.START_ARRAY) {
                 return new ObjectPath(parser.listOrderedMap());
             }
@@ -124,4 +138,20 @@ private String[] parsePath(String path) {
 
         return list.toArray(new String[list.size()]);
     }
+
+    /**
+     * Create a new {@link XContentBuilder} from the xContent object underlying this {@link ObjectPath}.
+     * This only works for {@link ObjectPath} instances created from an xContent object, not from nested
+     * substructures. We throw an {@link UnsupportedOperationException} in those cases.
+     */
+    @SuppressWarnings("unchecked")
+    public XContentBuilder toXContentBuilder(XContent xContent) throws IOException {
+        XContentBuilder builder = XContentBuilder.builder(xContent);
+        if (this.object instanceof Map) {
+            builder.map((Map<String, Object>) this.object);
+        } else {
+            throw new UnsupportedOperationException("Only ObjectPath created from a map supported.");
+        }
+        return builder;
+    }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/Stash.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/Stash.java
index d9a4d957a2590..04d2ffac9f4ec 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/Stash.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/Stash.java
@@ -26,7 +26,9 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.regex.Matcher;
@@ -38,6 +40,7 @@
  */
 public class Stash implements ToXContent {
     private static final Pattern EXTENDED_KEY = Pattern.compile("\\$\\{([^}]+)\\}");
+    private static final Pattern PATH = Pattern.compile("\\$_path");
 
     private static final Logger logger = Loggers.getLogger(Stash.class);
 
@@ -121,35 +124,79 @@ private Object unstash(String key) throws IOException {
      * Goes recursively against each map entry and replaces any string value starting with "$" with its
      * corresponding value retrieved from the stash
      */
+    @SuppressWarnings("unchecked") // Safe because we check that all the map keys are string in unstashObject
     public Map<String, Object> replaceStashedValues(Map<String, Object> map) throws IOException {
-        Map<String, Object> copy = new HashMap<>(map);
-        unstashObject(copy);
-        return copy;
+        return (Map<String, Object>) unstashObject(new ArrayList<>(), map);
     }
 
-    @SuppressWarnings("unchecked")
-    private void unstashObject(Object obj) throws IOException {
+    private Object unstashObject(List<Object> path, Object obj) throws IOException {
         if (obj instanceof List) {
-            List list = (List) obj;
-            for (int i = 0; i < list.size(); i++) {
-                Object o = list.get(i);
+            List<?> list = (List<?>) obj;
+            List<Object> result = new ArrayList<>();
+            int index = 0;
+            for (Object o : list) {
+                path.add(index++);
                 if (containsStashedValue(o)) {
-                    list.set(i, getValue(o.toString()));
+                    result.add(getValue(path, o.toString()));
                 } else {
-                    unstashObject(o);
+                    result.add(unstashObject(path, o));
                 }
+                path.remove(path.size() - 1);
             }
+            return result;
         }
         if (obj instanceof Map) {
-            Map<String, Object> map = (Map) obj;
-            for (Map.Entry<String, Object> entry : map.entrySet()) {
-                if (containsStashedValue(entry.getValue())) {
-                    entry.setValue(getValue(entry.getValue().toString()));
+            Map<?, ?> map = (Map<?, ?>) obj;
+            Map<String, Object> result = new HashMap<>();
+            for (Map.Entry<?, ?> entry : map.entrySet()) {
+                String key = (String) entry.getKey();
+                Object value = entry.getValue();
+                if (containsStashedValue(key)) {
+                    key = getValue(key).toString();
+                }
+                path.add(key);
+                if (containsStashedValue(value)) {
+                    value = getValue(path, value.toString());
                 } else {
-                    unstashObject(entry.getValue());
+                    value = unstashObject(path, value);
+                }
+                path.remove(path.size() - 1);
+                if (null != result.putIfAbsent(key, value)) {
+                    throw new IllegalArgumentException("Unstashing has caused a key conflict! The map is [" + result + "] and the key is ["
+                            + entry.getKey() + "] which unstashes to [" + key + "]");
                 }
             }
+            return result;
+        }
+        return obj;
+    }
+
+    /**
+     * Lookup a value from the stash adding support for a special key ({@code $_path}) which
+     * returns a string that is the location in the path of the of the object currently being
+     * unstashed. This is useful during documentation testing.
+     */
+    private Object getValue(List<Object> path, String key) throws IOException {
+        Matcher matcher = PATH.matcher(key);
+        if (false == matcher.find()) {
+            return getValue(key);
+        }
+        StringBuilder pathBuilder = new StringBuilder();
+        Iterator<Object> element = path.iterator();
+        if (element.hasNext()) {
+            pathBuilder.append(element.next());
+            while (element.hasNext()) {
+                pathBuilder.append('.');
+                pathBuilder.append(element.next());
+            }
         }
+        String builtPath = Matcher.quoteReplacement(pathBuilder.toString());
+        StringBuffer newKey = new StringBuffer(key.length());
+        do {
+            matcher.appendReplacement(newKey, builtPath);
+        } while (matcher.find());
+        matcher.appendTail(newKey);
+        return getValue(newKey.toString());
     }
 
     @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestFragmentParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestFragmentParser.java
deleted file mode 100644
index 390ac1ce366c6..0000000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestFragmentParser.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import java.io.IOException;
-
-/**
- * Base parser for a REST test suite fragment
- * @param <T> the test fragment's type that gets parsed and returned
- */
-public interface ClientYamlTestFragmentParser<T> {
-
-    /**
-     * Parses a test fragment given the current {@link ClientYamlTestSuiteParseContext}
-     */
-    T parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException;
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestParseException.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestParseException.java
deleted file mode 100644
index 594f701c79ab7..0000000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestParseException.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-/**
- * Exception thrown whenever there is a problem parsing any of the REST test suite fragment
- */
-public class ClientYamlTestParseException extends Exception {
-
-    ClientYamlTestParseException(String message) {
-        super(message);
-    }
-
-    ClientYamlTestParseException(String message, Throwable cause) {
-        super(message, cause);
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestSectionParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestSectionParser.java
deleted file mode 100644
index b6e8ad6c0f4a5..0000000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestSectionParser.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection;
-
-import java.io.IOException;
-
-/**
- * Parser for a complete test section
- */
-public class ClientYamlTestSectionParser implements ClientYamlTestFragmentParser<ClientYamlTestSection> {
-
-    @Override
-    public ClientYamlTestSection parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
-        XContentParser parser = parseContext.parser();
-        parseContext.advanceToFieldName();
-        ClientYamlTestSection testSection = new ClientYamlTestSection(parser.currentName());
-        try {
-            parser.nextToken();
-            testSection.setSkipSection(parseContext.parseSkipSection());
-    
-            while ( parser.currentToken() != XContentParser.Token.END_ARRAY) {
-                parseContext.advanceToFieldName();
-                testSection.addExecutableSection(parseContext.parseExecutableSection());
-            }
-    
-            parser.nextToken();
-            assert parser.currentToken() == XContentParser.Token.END_OBJECT;
-            parser.nextToken();
-    
-            return testSection;
-        } catch (Exception e) {
-            throw new ClientYamlTestParseException("Error parsing test named [" + testSection.getName() + "]", e);
-        }
-    }
-
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestSuiteParseContext.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestSuiteParseContext.java
deleted file mode 100644
index e466c1010f6b2..0000000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestSuiteParseContext.java
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
-import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.xcontent.XContentLocation;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.test.rest.yaml.section.DoSection;
-import org.elasticsearch.test.rest.yaml.section.ExecutableSection;
-import org.elasticsearch.test.rest.yaml.section.SetupSection;
-import org.elasticsearch.test.rest.yaml.section.SkipSection;
-import org.elasticsearch.test.rest.yaml.section.TeardownSection;
-import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Context shared across the whole tests parse phase.
- * Provides shared parse methods and holds information needed to parse the test sections (e.g. es version)
- */
-public class ClientYamlTestSuiteParseContext implements ParseFieldMatcherSupplier {
-
-    private static final SetupSectionParser SETUP_SECTION_PARSER = new SetupSectionParser();
-    private static final TeardownSectionParser TEARDOWN_SECTION_PARSER = new TeardownSectionParser();
-    private static final ClientYamlTestSectionParser TEST_SECTION_PARSER = new ClientYamlTestSectionParser();
-    private static final SkipSectionParser SKIP_SECTION_PARSER = new SkipSectionParser();
-    private static final DoSectionParser DO_SECTION_PARSER = new DoSectionParser();
-    private static final Map<String, ClientYamlTestFragmentParser<? extends ExecutableSection>> EXECUTABLE_SECTIONS_PARSERS =
-            new HashMap<>();
-    static {
-        EXECUTABLE_SECTIONS_PARSERS.put("do", DO_SECTION_PARSER);
-        EXECUTABLE_SECTIONS_PARSERS.put("set", new SetSectionParser());
-        EXECUTABLE_SECTIONS_PARSERS.put("match", new MatchParser());
-        EXECUTABLE_SECTIONS_PARSERS.put("is_true", new IsTrueParser());
-        EXECUTABLE_SECTIONS_PARSERS.put("is_false", new IsFalseParser());
-        EXECUTABLE_SECTIONS_PARSERS.put("gt", new GreaterThanParser());
-        EXECUTABLE_SECTIONS_PARSERS.put("gte", new GreaterThanEqualToParser());
-        EXECUTABLE_SECTIONS_PARSERS.put("lt", new LessThanParser());
-        EXECUTABLE_SECTIONS_PARSERS.put("lte", new LessThanOrEqualToParser());
-        EXECUTABLE_SECTIONS_PARSERS.put("length", new LengthParser());
-    }
-
-    private final String api;
-    private final String suiteName;
-    private final XContentParser parser;
-
-    public ClientYamlTestSuiteParseContext(String api, String suiteName, XContentParser parser) {
-        this.api = api;
-        this.suiteName = suiteName;
-        this.parser = parser;
-    }
-
-    public String getApi() {
-        return api;
-    }
-
-    public String getSuiteName() {
-        return suiteName;
-    }
-
-    public XContentParser parser() {
-        return parser;
-    }
-
-    public SetupSection parseSetupSection() throws IOException, ClientYamlTestParseException {
-
-        advanceToFieldName();
-
-        if ("setup".equals(parser.currentName())) {
-            parser.nextToken();
-            SetupSection setupSection = SETUP_SECTION_PARSER.parse(this);
-            parser.nextToken();
-            return setupSection;
-        }
-
-        return SetupSection.EMPTY;
-    }
-
-    public TeardownSection parseTeardownSection() throws IOException, ClientYamlTestParseException {
-        advanceToFieldName();
-
-        if ("teardown".equals(parser.currentName())) {
-            parser.nextToken();
-            TeardownSection teardownSection = TEARDOWN_SECTION_PARSER.parse(this);
-            parser.nextToken();
-            return teardownSection;
-        }
-
-        return TeardownSection.EMPTY;
-    }
-
-    public ClientYamlTestSection parseTestSection() throws IOException, ClientYamlTestParseException {
-        return TEST_SECTION_PARSER.parse(this);
-    }
-
-    public SkipSection parseSkipSection() throws IOException, ClientYamlTestParseException {
-
-        advanceToFieldName();
-
-        if ("skip".equals(parser.currentName())) {
-            SkipSection skipSection = SKIP_SECTION_PARSER.parse(this);
-            parser.nextToken();
-            return skipSection;
-        }
-
-        return SkipSection.EMPTY;
-    }
-
-    public ExecutableSection parseExecutableSection() throws IOException, ClientYamlTestParseException {
-        advanceToFieldName();
-        String section = parser.currentName();
-        ClientYamlTestFragmentParser<? extends ExecutableSection> execSectionParser = EXECUTABLE_SECTIONS_PARSERS.get(section);
-        if (execSectionParser == null) {
-            throw new ClientYamlTestParseException("no parser found for executable section [" + section + "]");
-        }
-        XContentLocation location = parser.getTokenLocation();
-        try {
-            ExecutableSection executableSection = execSectionParser.parse(this);
-            parser.nextToken();
-            return executableSection;
-        } catch (Exception e) {
-            throw new IOException("Error parsing section starting at ["+ location + "]", e);
-        }
-    }
-
-    public DoSection parseDoSection() throws IOException, ClientYamlTestParseException {
-        return DO_SECTION_PARSER.parse(this);
-    }
-
-    public void advanceToFieldName() throws IOException, ClientYamlTestParseException {
-        XContentParser.Token token = parser.currentToken();
-        //we are in the beginning, haven't called nextToken yet
-        if (token == null) {
-            token = parser.nextToken();
-        }
-        if (token == XContentParser.Token.START_ARRAY) {
-            token = parser.nextToken();
-        }
-        if (token == XContentParser.Token.START_OBJECT) {
-            token = parser.nextToken();
-        }
-        if (token != XContentParser.Token.FIELD_NAME) {
-            throw new ClientYamlTestParseException("malformed test section: field name expected but found " + token + " at "
-                    + parser.getTokenLocation());
-        }
-    }
-
-    public String parseField() throws IOException, ClientYamlTestParseException {
-        parser.nextToken();
-        assert parser.currentToken().isValue();
-        String field = parser.text();
-        parser.nextToken();
-        return field;
-    }
-
-    public Tuple<String, Object> parseTuple() throws IOException, ClientYamlTestParseException {
-        parser.nextToken();
-        advanceToFieldName();
-        Map<String,Object> map = parser.map();
-        assert parser.currentToken() == XContentParser.Token.END_OBJECT;
-        parser.nextToken();
-
-        if (map.size() != 1) {
-            throw new ClientYamlTestParseException("expected key value pair but found " + map.size() + " ");
-        }
-
-        Map.Entry<String, Object> entry = map.entrySet().iterator().next();
-        return Tuple.tuple(entry.getKey(), entry.getValue());
-    }
-
-    @Override
-    public ParseFieldMatcher getParseFieldMatcher() {
-        return ParseFieldMatcher.STRICT;
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestSuiteParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestSuiteParser.java
deleted file mode 100644
index 652778884714a..0000000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestSuiteParser.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.yaml.YamlXContent;
-import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSuite;
-import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.channels.FileChannel;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.StandardOpenOption;
-
-/**
- * Parser for a complete test suite (yaml file)
- */
-public class ClientYamlTestSuiteParser implements ClientYamlTestFragmentParser<ClientYamlTestSuite> {
-
-    public ClientYamlTestSuite parse(String api, Path file) throws IOException, ClientYamlTestParseException {
-
-        if (!Files.isRegularFile(file)) {
-            throw new IllegalArgumentException(file.toAbsolutePath() + " is not a file");
-        }
-
-        String filename = file.getFileName().toString();
-        //remove the file extension
-        int i = filename.lastIndexOf('.');
-        if (i > 0) {
-            filename = filename.substring(0, i);
-        }
-
-        //our yaml parser seems to be too tolerant. Each yaml suite must end with \n, otherwise clients tests might break.
-        try (FileChannel channel = FileChannel.open(file, StandardOpenOption.READ)) {
-            ByteBuffer bb = ByteBuffer.wrap(new byte[1]);
-            channel.read(bb, channel.size() - 1);
-            if (bb.get(0) != 10) {
-                throw new ClientYamlTestParseException("test suite [" + api + "/" + filename + "] doesn't end with line feed (\\n)");
-            }
-        }
-
-        try (XContentParser parser = YamlXContent.yamlXContent.createParser(Files.newInputStream(file))) {
-            ClientYamlTestSuiteParseContext testParseContext = new ClientYamlTestSuiteParseContext(api, filename, parser);
-            return parse(testParseContext);
-        } catch(Exception e) {
-            throw new ClientYamlTestParseException("Error parsing " + api + "/" + filename, e);
-        }
-    }
-
-    @Override
-    public ClientYamlTestSuite parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
-        XContentParser parser = parseContext.parser();
-
-        parser.nextToken();
-        assert parser.currentToken() == XContentParser.Token.START_OBJECT : "expected token to be START_OBJECT but was "
-                + parser.currentToken();
-
-        ClientYamlTestSuite restTestSuite = new ClientYamlTestSuite(parseContext.getApi(), parseContext.getSuiteName());
-
-        restTestSuite.setSetupSection(parseContext.parseSetupSection());
-        restTestSuite.setTeardownSection(parseContext.parseTeardownSection());
-
-        while(true) {
-            //the "---" section separator is not understood by the yaml parser. null is returned, same as when the parser is closed
-            //we need to somehow distinguish between a null in the middle of a test ("---")
-            // and a null at the end of the file (at least two consecutive null tokens)
-            if(parser.currentToken() == null) {
-                if (parser.nextToken() == null) {
-                    break;
-                }
-            }
-
-            ClientYamlTestSection testSection = parseContext.parseTestSection();
-            if (!restTestSuite.addTestSection(testSection)) {
-                throw new ClientYamlTestParseException("duplicate test section [" + testSection.getName() + "] found in ["
-                        + restTestSuite.getPath() + "]");
-            }
-        }
-
-        return restTestSuite;
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/DoSectionParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/DoSectionParser.java
deleted file mode 100644
index b89aa821ec94b..0000000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/DoSectionParser.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.test.rest.yaml.section.ApiCallSection;
-import org.elasticsearch.test.rest.yaml.section.DoSection;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static java.util.Collections.unmodifiableList;
-
-/**
- * Parser for do sections
- */
-public class DoSectionParser implements ClientYamlTestFragmentParser<DoSection> {
-
-    @Override
-    public DoSection parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
-
-        XContentParser parser = parseContext.parser();
-
-        String currentFieldName = null;
-        XContentParser.Token token;
-
-        DoSection doSection = new DoSection(parseContext.parser().getTokenLocation());
-        ApiCallSection apiCallSection = null;
-        Map<String, String> headers = new HashMap<>();
-        List<String> expectedWarnings = new ArrayList<>();
-
-        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-            if (token == XContentParser.Token.FIELD_NAME) {
-                currentFieldName = parser.currentName();
-            } else if (token.isValue()) {
-                if ("catch".equals(currentFieldName)) {
-                    doSection.setCatch(parser.text());
-                }
-            } else if (token == XContentParser.Token.START_ARRAY) {
-                if ("warnings".equals(currentFieldName)) {
-                    while ((token = parser.nextToken()) == XContentParser.Token.VALUE_STRING) {
-                        expectedWarnings.add(parser.text());
-                    }
-                    if (token != XContentParser.Token.END_ARRAY) {
-                        throw new ParsingException(parser.getTokenLocation(), "[warnings] must be a string array but saw [" + token + "]");
-                    }
-                } else {
-                    throw new ParsingException(parser.getTokenLocation(), "unknown array [" + currentFieldName + "]");
-                }
-            } else if (token == XContentParser.Token.START_OBJECT) {
-                if ("headers".equals(currentFieldName)) {
-                    String headerName = null;
-                    while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                        if (token == XContentParser.Token.FIELD_NAME) {
-                            headerName = parser.currentName();
-                        } else if (token.isValue()) {
-                            headers.put(headerName, parser.text());
-                        }
-                    }
-                } else if (currentFieldName != null) { // must be part of API call then
-                    apiCallSection = new ApiCallSection(currentFieldName);
-                    String paramName = null;
-                    while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                        if (token == XContentParser.Token.FIELD_NAME) {
-                            paramName = parser.currentName();
-                        } else if (token.isValue()) {
-                            if ("body".equals(paramName)) {
-                                String body = parser.text();
-                                XContentType bodyContentType = XContentFactory.xContentType(body);
-                                XContentParser bodyParser = XContentFactory.xContent(bodyContentType).createParser(body);
-                                //multiple bodies are supported e.g. in case of bulk provided as a whole string
-                                while(bodyParser.nextToken() != null) {
-                                    apiCallSection.addBody(bodyParser.mapOrdered());
-                                }
-                            } else {
-                                apiCallSection.addParam(paramName, parser.text());
-                            }
-                        } else if (token == XContentParser.Token.START_OBJECT) {
-                            if ("body".equals(paramName)) {
-                                apiCallSection.addBody(parser.mapOrdered());
-                            }
-                        }
-                    }
-                }
-            }
-        }
-        try {
-            if (apiCallSection == null) {
-                throw new ClientYamlTestParseException("client call section is mandatory within a do section");
-            }
-            if (headers.isEmpty() == false) {
-                apiCallSection.addHeaders(headers);
-            }
-            doSection.setApiCallSection(apiCallSection);
-            doSection.setExpectedWarningHeaders(unmodifiableList(expectedWarnings));
-        } finally {
-            parser.nextToken();
-        }
-        return doSection;
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/GreaterThanEqualToParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/GreaterThanEqualToParser.java
deleted file mode 100644
index 65a46d139bd06..0000000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/GreaterThanEqualToParser.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.xcontent.XContentLocation;
-import org.elasticsearch.test.rest.yaml.section.GreaterThanEqualToAssertion;
-
-import java.io.IOException;
-
-/**
- * Parser for gte assert sections
- */
-public class GreaterThanEqualToParser implements ClientYamlTestFragmentParser<GreaterThanEqualToAssertion> {
-    @Override
-    public GreaterThanEqualToAssertion parse(ClientYamlTestSuiteParseContext parseContext)
-            throws IOException, ClientYamlTestParseException {
-        XContentLocation location = parseContext.parser().getTokenLocation();
-        Tuple<String,Object> stringObjectTuple = parseContext.parseTuple();
-        if (! (stringObjectTuple.v2() instanceof Comparable) ) {
-            throw new ClientYamlTestParseException("gte section can only be used with objects that support natural ordering, found "
-                    + stringObjectTuple.v2().getClass().getSimpleName());
-        }
-        return new GreaterThanEqualToAssertion(location, stringObjectTuple.v1(), stringObjectTuple.v2());
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/GreaterThanParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/GreaterThanParser.java
deleted file mode 100644
index f6c532720672c..0000000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/GreaterThanParser.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.xcontent.XContentLocation;
-import org.elasticsearch.test.rest.yaml.section.GreaterThanAssertion;
-
-import java.io.IOException;
-
-/**
- * Parser for gt assert sections
- */
-public class GreaterThanParser implements ClientYamlTestFragmentParser<GreaterThanAssertion> {
-
-    @Override
-    public GreaterThanAssertion parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
-        XContentLocation location = parseContext.parser().getTokenLocation();
-        Tuple<String,Object> stringObjectTuple = parseContext.parseTuple();
-        if (! (stringObjectTuple.v2() instanceof Comparable) ) {
-            throw new ClientYamlTestParseException("gt section can only be used with objects that support natural ordering, found "
-                    + stringObjectTuple.v2().getClass().getSimpleName());
-        }
-        return new GreaterThanAssertion(location, stringObjectTuple.v1(), stringObjectTuple.v2());
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/IsFalseParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/IsFalseParser.java
deleted file mode 100644
index 3d9593ce29023..0000000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/IsFalseParser.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.test.rest.yaml.section.IsFalseAssertion;
-
-import java.io.IOException;
-
-/**
- * Parser for is_false assert sections
- */
-public class IsFalseParser implements ClientYamlTestFragmentParser<IsFalseAssertion> {
-
-    @Override
-    public IsFalseAssertion parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
-        return new IsFalseAssertion(parseContext.parser().getTokenLocation(), parseContext.parseField());
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/IsTrueParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/IsTrueParser.java
deleted file mode 100644
index 9dfc3be57e37b..0000000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/IsTrueParser.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.test.rest.yaml.section.IsTrueAssertion;
-
-import java.io.IOException;
-
-/**
- * Parser for is_true assert sections
- */
-public class IsTrueParser implements ClientYamlTestFragmentParser<IsTrueAssertion> {
-
-    @Override
-    public IsTrueAssertion parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
-        return new IsTrueAssertion(parseContext.parser().getTokenLocation(), parseContext.parseField());
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/LengthParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/LengthParser.java
deleted file mode 100644
index 98c030cd6a13c..0000000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/LengthParser.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.xcontent.XContentLocation;
-import org.elasticsearch.test.rest.yaml.section.LengthAssertion;
-
-import java.io.IOException;
-
-/**
- * Parser for length assert sections
- */
-public class LengthParser implements ClientYamlTestFragmentParser<LengthAssertion> {
-
-    @Override
-    public LengthAssertion parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
-        XContentLocation location = parseContext.parser().getTokenLocation();
-        Tuple<String,Object> stringObjectTuple = parseContext.parseTuple();
-        assert stringObjectTuple.v2() != null;
-        int value;
-        if (stringObjectTuple.v2() instanceof  Number) {
-            value = ((Number) stringObjectTuple.v2()).intValue();
-        } else {
-            try {
-                value = Integer.valueOf(stringObjectTuple.v2().toString());
-            } catch(NumberFormatException e) {
-                throw new ClientYamlTestParseException("length is not a valid number", e);
-            }
-
-        }
-        return new LengthAssertion(location, stringObjectTuple.v1(), value);
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/LessThanOrEqualToParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/LessThanOrEqualToParser.java
deleted file mode 100644
index 5dfe4c742f525..0000000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/LessThanOrEqualToParser.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.xcontent.XContentLocation;
-import org.elasticsearch.test.rest.yaml.section.LessThanOrEqualToAssertion;
-
-import java.io.IOException;
-
-/**
- * Parser for lte assert section
- */
-public class LessThanOrEqualToParser implements ClientYamlTestFragmentParser<LessThanOrEqualToAssertion> {
-
-    @Override
-    public LessThanOrEqualToAssertion parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
-        XContentLocation location = parseContext.parser().getTokenLocation();
-        Tuple<String,Object> stringObjectTuple = parseContext.parseTuple();
-        if (! (stringObjectTuple.v2() instanceof Comparable) ) {
-            throw new ClientYamlTestParseException("lte section can only be used with objects that support natural ordering, found "
-                    + stringObjectTuple.v2().getClass().getSimpleName());
-        }
-        return new LessThanOrEqualToAssertion(location, stringObjectTuple.v1(), stringObjectTuple.v2());
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/LessThanParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/LessThanParser.java
deleted file mode 100644
index 0caf12ea91a02..0000000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/LessThanParser.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.xcontent.XContentLocation;
-import org.elasticsearch.test.rest.yaml.section.LessThanAssertion;
-
-import java.io.IOException;
-
-/**
- * Parser for lt assert sections
- */
-public class LessThanParser implements ClientYamlTestFragmentParser<LessThanAssertion> {
-
-    @Override
-    public LessThanAssertion parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
-        XContentLocation location = parseContext.parser().getTokenLocation();
-        Tuple<String,Object> stringObjectTuple = parseContext.parseTuple();
-        if (! (stringObjectTuple.v2() instanceof Comparable) ) {
-            throw new ClientYamlTestParseException("lt section can only be used with objects that support natural ordering, found "
-                    + stringObjectTuple.v2().getClass().getSimpleName());
-        }
-        return new LessThanAssertion(location, stringObjectTuple.v1(), stringObjectTuple.v2());
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/MatchParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/MatchParser.java
deleted file mode 100644
index f456c32d0925d..0000000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/MatchParser.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.xcontent.XContentLocation;
-import org.elasticsearch.test.rest.yaml.section.MatchAssertion;
-
-import java.io.IOException;
-
-/**
- * Parser for match assert sections
- */
-public class MatchParser implements ClientYamlTestFragmentParser<MatchAssertion> {
-
-    @Override
-    public MatchAssertion parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
-        XContentLocation location = parseContext.parser().getTokenLocation();
-        Tuple<String,Object> stringObjectTuple = parseContext.parseTuple();
-        return new MatchAssertion(location, stringObjectTuple.v1(), stringObjectTuple.v2());
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/SetSectionParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/SetSectionParser.java
deleted file mode 100644
index c7797e42d5787..0000000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/SetSectionParser.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.test.rest.yaml.section.SetSection;
-
-import java.io.IOException;
-
-/**
- * Parser for set sections
- */
-public class SetSectionParser implements ClientYamlTestFragmentParser<SetSection> {
-
-    @Override
-    public SetSection parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
-
-        XContentParser parser = parseContext.parser();
-
-        String currentFieldName = null;
-        XContentParser.Token token;
-
-        SetSection setSection = new SetSection(parser.getTokenLocation());
-
-        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-            if (token == XContentParser.Token.FIELD_NAME) {
-                currentFieldName = parser.currentName();
-            } else if (token.isValue()) {
-                setSection.addSet(currentFieldName, parser.text());
-            }
-        }
-
-        parser.nextToken();
-
-        if (setSection.getStash().isEmpty()) {
-            throw new ClientYamlTestParseException("set section must set at least a value");
-        }
-
-        return setSection;
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/SetupSectionParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/SetupSectionParser.java
deleted file mode 100644
index e62b3af5251e0..0000000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/SetupSectionParser.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.test.rest.yaml.section.SetupSection;
-
-import java.io.IOException;
-
-/**
- * Parser for setup sections
- */
-public class SetupSectionParser implements ClientYamlTestFragmentParser<SetupSection> {
-
-    @Override
-    public SetupSection parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
-
-        XContentParser parser = parseContext.parser();
-
-        SetupSection setupSection = new SetupSection();
-        setupSection.setSkipSection(parseContext.parseSkipSection());
-
-        while (parser.currentToken() != XContentParser.Token.END_ARRAY) {
-            parseContext.advanceToFieldName();
-            if (!"do".equals(parser.currentName())) {
-                throw new ClientYamlTestParseException("section [" + parser.currentName() + "] not supported within setup section");
-            }
-
-            parser.nextToken();
-            setupSection.addDoSection(parseContext.parseDoSection());
-            parser.nextToken();
-        }
-
-        parser.nextToken();
-
-        return setupSection;
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/SkipSectionParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/SkipSectionParser.java
deleted file mode 100644
index 31451dee247a6..0000000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/SkipSectionParser.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.test.rest.yaml.section.SkipSection;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Parser for skip sections
- */
-public class SkipSectionParser implements ClientYamlTestFragmentParser<SkipSection> {
-
-    @Override
-    public SkipSection parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
-
-        XContentParser parser = parseContext.parser();
-
-        String currentFieldName = null;
-        XContentParser.Token token;
-        String version = null;
-        String reason = null;
-        List<String> features = new ArrayList<>();
-
-        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-            if (token == XContentParser.Token.FIELD_NAME) {
-                currentFieldName = parser.currentName();
-            } else if (token.isValue()) {
-                if ("version".equals(currentFieldName)) {
-                    version = parser.text();
-                } else if ("reason".equals(currentFieldName)) {
-                    reason = parser.text();
-                } else if ("features".equals(currentFieldName)) {
-                    features.add(parser.text());
-                }
-                else {
-                    throw new ClientYamlTestParseException("field " + currentFieldName + " not supported within skip section");
-                }
-            } else if (token == XContentParser.Token.START_ARRAY) {
-                if ("features".equals(currentFieldName)) {
-                    while(parser.nextToken() != XContentParser.Token.END_ARRAY) {
-                        features.add(parser.text());
-                    }
-                }
-            }
-        }
-
-        parser.nextToken();
-
-        if (!Strings.hasLength(version) && features.isEmpty()) {
-            throw new ClientYamlTestParseException("version or features is mandatory within skip section");
-        }
-        if (Strings.hasLength(version) && !features.isEmpty()) {
-            throw new ClientYamlTestParseException("version or features are mutually exclusive");
-        }
-        if (Strings.hasLength(version) && !Strings.hasLength(reason)) {
-            throw new ClientYamlTestParseException("reason is mandatory within skip version section");
-        }
-
-        return new SkipSection(version, features, reason);
-
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/TeardownSectionParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/TeardownSectionParser.java
deleted file mode 100644
index ed1b42c3a9cc7..0000000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/TeardownSectionParser.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.test.rest.yaml.section.TeardownSection;
-
-import java.io.IOException;
-
-/**
- * Parser for teardown section
- */
-public class TeardownSectionParser implements ClientYamlTestFragmentParser<TeardownSection> {
-
-    @Override
-    public TeardownSection parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
-        XContentParser parser = parseContext.parser();
-
-        TeardownSection teardownSection = new TeardownSection();
-        teardownSection.setSkipSection(parseContext.parseSkipSection());
-
-        while (parser.currentToken() != XContentParser.Token.END_ARRAY) {
-            parseContext.advanceToFieldName();
-            if (!"do".equals(parser.currentName())) {
-                throw new ClientYamlTestParseException("section [" + parser.currentName() + "] not supported within teardown section");
-            }
-
-            parser.nextToken();
-            teardownSection.addDoSection(parseContext.parseDoSection());
-            parser.nextToken();
-        }
-
-        parser.nextToken();
-        return teardownSection;
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParser.java
index 178fb9c5f3423..8abcfc35f27a3 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParser.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParser.java
@@ -42,7 +42,11 @@ public ClientYamlSuiteRestApi parse(String location, XContentParser parser) thro
                 if ("methods".equals(parser.currentName())) {
                     parser.nextToken();
                     while (parser.nextToken() == XContentParser.Token.VALUE_STRING) {
-                        restApi.addMethod(parser.text());
+                        String method = parser.text();
+                        if (restApi.getMethods().contains(method)) {
+                            throw new IllegalArgumentException("Found duplicate method [" + method + "]");
+                        }
+                        restApi.addMethod(method);
                     }
                 }
 
@@ -56,16 +60,24 @@ public ClientYamlSuiteRestApi parse(String location, XContentParser parser) thro
 
                         if (parser.currentToken() == XContentParser.Token.START_ARRAY && "paths".equals(currentFieldName)) {
                             while (parser.nextToken() == XContentParser.Token.VALUE_STRING) {
-                                restApi.addPath(parser.text());
+                                String path = parser.text();
+                                if (restApi.getPaths().contains(path)) {
+                                    throw new IllegalArgumentException("Found duplicate path [" + path + "]");
+                                }
+                                restApi.addPath(path);
                             }
                         }
 
                         if (parser.currentToken() == XContentParser.Token.START_OBJECT && "parts".equals(currentFieldName)) {
                             while (parser.nextToken() == XContentParser.Token.FIELD_NAME) {
-                                restApi.addPathPart(parser.currentName());
+                                String part = parser.currentName();
+                                if (restApi.getPathParts().contains(part)) {
+                                    throw new IllegalArgumentException("Found duplicate part [" + part + "]");
+                                }
+                                restApi.addPathPart(part);
                                 parser.nextToken();
                                 if (parser.currentToken() != XContentParser.Token.START_OBJECT) {
-                                    throw new IOException("Expected parts field in rest api definition to contain an object");
+                                    throw new IllegalArgumentException("Expected parts field in rest api definition to contain an object");
                                 }
                                 parser.skipChildren();
                             }
@@ -73,10 +85,14 @@ public ClientYamlSuiteRestApi parse(String location, XContentParser parser) thro
 
                         if (parser.currentToken() == XContentParser.Token.START_OBJECT && "params".equals(currentFieldName)) {
                             while (parser.nextToken() == XContentParser.Token.FIELD_NAME) {
+                                String param = parser.currentName();
+                                if (restApi.getParams().contains(param)) {
+                                    throw new IllegalArgumentException("Found duplicate param [" + param + "]");
+                                }
                                 restApi.addParam(parser.currentName());
                                 parser.nextToken();
                                 if (parser.currentToken() != XContentParser.Token.START_OBJECT) {
-                                    throw new IOException("Expected params field in rest api definition to contain an object");
+                                    throw new IllegalArgumentException("Expected params field in rest api definition to contain an object");
                                 }
                                 parser.skipChildren();
                             }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestSpec.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestSpec.java
index 9839bb089ae60..dd800f5c9ddce 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestSpec.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestSpec.java
@@ -18,29 +18,33 @@
  */
 package org.elasticsearch.test.rest.yaml.restspec;
 
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.json.JsonXContent;
-import org.elasticsearch.test.rest.yaml.FileUtils;
-
 import java.io.IOException;
 import java.io.InputStream;
-import java.nio.file.FileSystem;
+import java.io.UncheckedIOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Collection;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
+import java.util.stream.Stream;
+
+import org.elasticsearch.common.io.PathUtils;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 
 /**
  * Holds the specification used to turn {@code do} actions in the YAML suite into REST api calls.
  */
 public class ClientYamlSuiteRestSpec {
-    Map<String, ClientYamlSuiteRestApi> restApiMap = new HashMap<>();
+    private final Set<String> globalParameters = new HashSet<>();
+    private final Map<String, ClientYamlSuiteRestApi> restApiMap = new HashMap<>();
 
-    private ClientYamlSuiteRestSpec() {
-    }
+    private ClientYamlSuiteRestSpec() {}
 
-    void addApi(ClientYamlSuiteRestApi restApi) {
+    private void addApi(ClientYamlSuiteRestApi restApi) {
         ClientYamlSuiteRestApi previous = restApiMap.putIfAbsent(restApi.getName(), restApi);
         if (previous != null) {
             throw new IllegalArgumentException("cannot register api [" + restApi.getName() + "] found in [" + restApi.getLocation() + "]. "
@@ -56,30 +60,76 @@ public Collection<ClientYamlSuiteRestApi> getApis() {
         return restApiMap.values();
     }
 
+    /**
+     * Returns whether the provided parameter is one of those parameters that are supported by all Elasticsearch api
+     */
+    public boolean isGlobalParameter(String param) {
+        return globalParameters.contains(param);
+    }
+
+    /**
+     * Returns whether the provided parameter is one of those parameters that are supported by the Elasticsearch language clients, meaning
+     * that they influence the client behaviour and don't get sent to Elasticsearch
+     */
+    public boolean isClientParameter(String name) {
+        return "ignore".equals(name);
+    }
+
     /**
      * Parses the complete set of REST spec available under the provided directories
      */
-    public static ClientYamlSuiteRestSpec parseFrom(FileSystem fileSystem, String optionalPathPrefix, String... paths) throws IOException {
+    public static ClientYamlSuiteRestSpec load(String classpathPrefix) throws Exception {
+        Path dir = PathUtils.get(ClientYamlSuiteRestSpec.class.getResource(classpathPrefix).toURI());
         ClientYamlSuiteRestSpec restSpec = new ClientYamlSuiteRestSpec();
         ClientYamlSuiteRestApiParser restApiParser = new ClientYamlSuiteRestApiParser();
-        for (String path : paths) {
-            for (Path jsonFile : FileUtils.findJsonSpec(fileSystem, optionalPathPrefix, path)) {
-                try (InputStream stream = Files.newInputStream(jsonFile)) {
-                    try (XContentParser parser = JsonXContent.jsonXContent.createParser(stream)) {
-                        ClientYamlSuiteRestApi restApi = restApiParser.parse(jsonFile.toString(), parser);
-                        String filename = jsonFile.getFileName().toString();
-                        String expectedApiName = filename.substring(0, filename.lastIndexOf('.'));
-                        if (restApi.getName().equals(expectedApiName) == false) {
-                            throw new IllegalArgumentException("found api [" + restApi.getName() + "] in [" + jsonFile.toString() + "]. " +
-                                    "Each api is expected to have the same name as the file that defines it.");
+        try (Stream<Path> stream = Files.walk(dir)) {
+            stream.forEach(item -> {
+                if (item.toString().endsWith(".json")) {
+                    parseSpecFile(restApiParser, item, restSpec);
+                }
+            });
+        }
+        return restSpec;
+    }
+
+    private static void parseSpecFile(ClientYamlSuiteRestApiParser restApiParser, Path jsonFile, ClientYamlSuiteRestSpec restSpec) {
+        try (InputStream stream = Files.newInputStream(jsonFile)) {
+            try (XContentParser parser = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, stream)) {
+                String filename = jsonFile.getFileName().toString();
+                if (filename.equals("_common.json")) {
+                    String currentFieldName = null;
+                    while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
+                        if (parser.currentToken() == XContentParser.Token.FIELD_NAME) {
+                            currentFieldName = parser.currentName();
+                        } else if (parser.currentToken() == XContentParser.Token.START_OBJECT
+                            && "params".equals(currentFieldName)) {
+                            while (parser.nextToken() == XContentParser.Token.FIELD_NAME) {
+                                String param = parser.currentName();
+                                if (restSpec.globalParameters.contains(param)) {
+                                    throw new IllegalArgumentException("Found duplicate global param [" + param + "]");
+                                }
+                                restSpec.globalParameters.add(param);
+                                parser.nextToken();
+                                if (parser.currentToken() != XContentParser.Token.START_OBJECT) {
+                                    throw new IllegalArgumentException("Expected params field in rest api definition to " +
+                                        "contain an object");
+                                }
+                                parser.skipChildren();
+                            }
                         }
-                        restSpec.addApi(restApi);
                     }
-                } catch (Exception ex) {
-                    throw new IOException("Can't parse rest spec file: [" + jsonFile + "]", ex);
+                } else {
+                    ClientYamlSuiteRestApi restApi = restApiParser.parse(jsonFile.toString(), parser);
+                    String expectedApiName = filename.substring(0, filename.lastIndexOf('.'));
+                    if (restApi.getName().equals(expectedApiName) == false) {
+                        throw new IllegalArgumentException("found api [" + restApi.getName() + "] in [" + jsonFile.toString() + "]. " +
+                            "Each api is expected to have the same name as the file that defines it.");
+                    }
+                    restSpec.addApi(restApi);
                 }
             }
+        } catch (IOException ex) {
+            throw new UncheckedIOException("Can't parse rest spec file: [" + jsonFile + "]", ex);
         }
-        return restSpec;
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ApiCallSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ApiCallSection.java
index 5d097f872b4e2..4553845458541 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ApiCallSection.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ApiCallSection.java
@@ -61,10 +61,6 @@ public void addHeaders(Map<String, String> otherHeaders) {
         this.headers.putAll(otherHeaders);
     }
 
-    public void addHeader(String key, String value) {
-        this.headers.put(key, value);
-    }
-
     public Map<String, String> getHeaders() {
         return unmodifiableMap(headers);
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java
index 4df126cba1900..321d22ed70aa7 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java
@@ -18,6 +18,11 @@
  */
 package org.elasticsearch.test.rest.yaml.section;
 
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.xcontent.XContentLocation;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -25,15 +30,42 @@
  * Represents a test section, which is composed of a skip section and multiple executable sections.
  */
 public class ClientYamlTestSection implements Comparable<ClientYamlTestSection> {
+    public static ClientYamlTestSection parse(XContentParser parser) throws IOException {
+        ParserUtils.advanceToFieldName(parser);
+        ClientYamlTestSection testSection = new ClientYamlTestSection(parser.getTokenLocation(), parser.currentName());
+        try {
+            parser.nextToken();
+            testSection.setSkipSection(SkipSection.parseIfNext(parser));
+            while (parser.currentToken() != XContentParser.Token.END_ARRAY) {
+                ParserUtils.advanceToFieldName(parser);
+                testSection.addExecutableSection(ExecutableSection.parse(parser));
+            }
+            if (parser.nextToken() != XContentParser.Token.END_OBJECT) {
+                throw new IllegalArgumentException("malformed section [" + testSection.getName() + "] expected ["
+                        + XContentParser.Token.END_OBJECT + "] but was [" + parser.currentToken() + "]");
+            }
+            parser.nextToken();
+            return testSection;
+        } catch (Exception e) {
+            throw new ParsingException(parser.getTokenLocation(), "Error parsing test named [" + testSection.getName() + "]", e);
+        }
+    }
+
+    private final XContentLocation location;
     private final String name;
     private SkipSection skipSection;
     private final List<ExecutableSection> executableSections;
 
-    public ClientYamlTestSection(String name) {
+    public ClientYamlTestSection(XContentLocation location, String name) {
+        this.location = location;
         this.name = name;
         this.executableSections = new ArrayList<>();
     }
 
+    public XContentLocation getLocation() {
+        return location;
+    }
+
     public String getName() {
         return name;
     }
@@ -51,6 +83,15 @@ public List<ExecutableSection> getExecutableSections() {
     }
 
     public void addExecutableSection(ExecutableSection executableSection) {
+        if (executableSection instanceof DoSection) {
+            DoSection doSection = (DoSection) executableSection;
+            if (false == doSection.getExpectedWarningHeaders().isEmpty()
+                    && false == skipSection.getFeatures().contains("warnings")) {
+                throw new IllegalArgumentException("Attempted to add a [do] with a [warnings] section without a corresponding [skip] so "
+                        + "runners that do not support the [warnings] section can skip the test at line ["
+                        + doSection.getLocation().lineNumber + "]");
+            }
+        }
         this.executableSections.add(executableSection);
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java
index af3e1be0528f3..5f97eb4af6a8f 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java
@@ -18,6 +18,16 @@
  */
 package org.elasticsearch.test.rest.yaml.section;
 
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.yaml.YamlXContent;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
@@ -28,6 +38,63 @@
  * Supports a setup section and multiple test sections.
  */
 public class ClientYamlTestSuite {
+    public static ClientYamlTestSuite parse(String api, Path file) throws IOException {
+        if (!Files.isRegularFile(file)) {
+            throw new IllegalArgumentException(file.toAbsolutePath() + " is not a file");
+        }
+
+        String filename = file.getFileName().toString();
+        //remove the file extension
+        int i = filename.lastIndexOf('.');
+        if (i > 0) {
+            filename = filename.substring(0, i);
+        }
+
+        //our yaml parser seems to be too tolerant. Each yaml suite must end with \n, otherwise clients tests might break.
+        try (FileChannel channel = FileChannel.open(file, StandardOpenOption.READ)) {
+            ByteBuffer bb = ByteBuffer.wrap(new byte[1]);
+            channel.read(bb, channel.size() - 1);
+            if (bb.get(0) != 10) {
+                throw new IOException("test suite [" + api + "/" + filename + "] doesn't end with line feed (\\n)");
+            }
+        }
+
+        try (XContentParser parser = YamlXContent.yamlXContent.createParser(ExecutableSection.XCONTENT_REGISTRY,
+                Files.newInputStream(file))) {
+            return parse(api, filename, parser);
+        } catch(Exception e) {
+            throw new IOException("Error parsing " + api + "/" + filename, e);
+        }
+    }
+
+    public static ClientYamlTestSuite parse(String api, String suiteName, XContentParser parser) throws IOException {
+        parser.nextToken();
+        assert parser.currentToken() == XContentParser.Token.START_OBJECT : "expected token to be START_OBJECT but was "
+                + parser.currentToken();
+
+        ClientYamlTestSuite restTestSuite = new ClientYamlTestSuite(api, suiteName);
+
+        restTestSuite.setSetupSection(SetupSection.parseIfNext(parser));
+        restTestSuite.setTeardownSection(TeardownSection.parseIfNext(parser));
+
+        while(true) {
+            //the "---" section separator is not understood by the yaml parser. null is returned, same as when the parser is closed
+            //we need to somehow distinguish between a null in the middle of a test ("---")
+            // and a null at the end of the file (at least two consecutive null tokens)
+            if(parser.currentToken() == null) {
+                if (parser.nextToken() == null) {
+                    break;
+                }
+            }
+
+            ClientYamlTestSection testSection = ClientYamlTestSection.parse(parser);
+            if (!restTestSuite.addTestSection(testSection)) {
+                throw new ParsingException(testSection.getLocation(), "duplicate test section [" + testSection.getName() + "]");
+            }
+        }
+
+        return restTestSuite;
+    }
 
     private final String api;
     private final String name;
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java
index e233e9fab8099..461a25011458b 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java
@@ -16,26 +16,38 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+
 package org.elasticsearch.test.rest.yaml.section;
 
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentLocation;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestResponse;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestResponseException;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.stream.Collectors;
+import java.util.TreeMap;
 
 import static java.util.Collections.emptyList;
+import static java.util.Collections.unmodifiableList;
 import static org.elasticsearch.common.collect.Tuple.tuple;
+import static org.elasticsearch.common.logging.DeprecationLogger.WARNING_HEADER_PATTERN;
 import static org.elasticsearch.test.hamcrest.RegexMatcher.matches;
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.equalTo;
@@ -65,6 +77,87 @@
  *
  */
 public class DoSection implements ExecutableSection {
+    public static DoSection parse(XContentParser parser) throws IOException {
+        String currentFieldName = null;
+        XContentParser.Token token;
+
+        DoSection doSection = new DoSection(parser.getTokenLocation());
+        ApiCallSection apiCallSection = null;
+        Map<String, String> headers = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
+        List<String> expectedWarnings = new ArrayList<>();
+
+        if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
+            throw new IllegalArgumentException("expected [" + XContentParser.Token.START_OBJECT + "], " +
+                    "found [" + parser.currentToken() + "], the do section is not properly indented");
+        }
+
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token.isValue()) {
+                if ("catch".equals(currentFieldName)) {
+                    doSection.setCatch(parser.text());
+                }
+            } else if (token == XContentParser.Token.START_ARRAY) {
+                if ("warnings".equals(currentFieldName)) {
+                    while ((token = parser.nextToken()) == XContentParser.Token.VALUE_STRING) {
+                        expectedWarnings.add(parser.text());
+                    }
+                    if (token != XContentParser.Token.END_ARRAY) {
+                        throw new ParsingException(parser.getTokenLocation(), "[warnings] must be a string array but saw [" + token + "]");
+                    }
+                } else {
+                    throw new ParsingException(parser.getTokenLocation(), "unknown array [" + currentFieldName + "]");
+                }
+            } else if (token == XContentParser.Token.START_OBJECT) {
+                if ("headers".equals(currentFieldName)) {
+                    String headerName = null;
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                        if (token == XContentParser.Token.FIELD_NAME) {
+                            headerName = parser.currentName();
+                        } else if (token.isValue()) {
+                            headers.put(headerName, parser.text());
+                        }
+                    }
+                } else if (currentFieldName != null) { // must be part of API call then
+                    apiCallSection = new ApiCallSection(currentFieldName);
+                    String paramName = null;
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                        if (token == XContentParser.Token.FIELD_NAME) {
+                            paramName = parser.currentName();
+                        } else if (token.isValue()) {
+                            if ("body".equals(paramName)) {
+                                String body = parser.text();
+                                XContentParser bodyParser = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, body);
+                                //multiple bodies are supported e.g. in case of bulk provided as a whole string
+                                while(bodyParser.nextToken() != null) {
+                                    apiCallSection.addBody(bodyParser.mapOrdered());
+                                }
+                            } else {
+                                apiCallSection.addParam(paramName, parser.text());
+                            }
+                        } else if (token == XContentParser.Token.START_OBJECT) {
+                            if ("body".equals(paramName)) {
+                                apiCallSection.addBody(parser.mapOrdered());
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        try {
+            if (apiCallSection == null) {
+                throw new IllegalArgumentException("client call section is mandatory within a do section");
+            }
+            apiCallSection.addHeaders(headers);
+            doSection.setApiCallSection(apiCallSection);
+            doSection.setExpectedWarningHeaders(unmodifiableList(expectedWarnings));
+        } finally {
+            parser.nextToken();
+        }
+        return doSection;
+    }
+
 
     private static final Logger logger = Loggers.getLogger(DoSection.class);
 
@@ -164,33 +257,48 @@ public void execute(ClientYamlTestExecutionContext executionContext) throws IOEx
     /**
      * Check that the response contains only the warning headers that we expect.
      */
-    void checkWarningHeaders(List<String> warningHeaders) {
-        StringBuilder failureMessage = null;
+    void checkWarningHeaders(final List<String> warningHeaders) {
+        final List<String> unexpected = new ArrayList<>();
+        final List<String> unmatched = new ArrayList<>();
+        final List<String> missing = new ArrayList<>();
         // LinkedHashSet so that missing expected warnings come back in a predictable order which is nice for testing
-        Set<String> expected = new LinkedHashSet<>(expectedWarningHeaders);
-        for (String header : warningHeaders) {
-            if (expected.remove(header)) {
-                // Was expected, all good.
-                continue;
-            }
-            if (failureMessage == null) {
-                failureMessage = new StringBuilder("got unexpected warning headers [");
-            }
-            failureMessage.append('\n').append(header);
-        }
-        if (false == expected.isEmpty()) {
-            if (failureMessage == null) {
-                failureMessage = new StringBuilder();
+        final Set<String> expected =
+                new LinkedHashSet<>(expectedWarningHeaders.stream().map(DeprecationLogger::escape).collect(Collectors.toList()));
+        for (final String header : warningHeaders) {
+            final Matcher matcher = WARNING_HEADER_PATTERN.matcher(header);
+            final boolean matches = matcher.matches();
+            if (matches) {
+                final String message = matcher.group(1);
+                if (expected.remove(message) == false) {
+                    unexpected.add(header);
+                }
             } else {
-                failureMessage.append("\n] ");
+                unmatched.add(header);
             }
-            failureMessage.append("didn't get expected warning headers [");
-            for (String header : expected) {
-                failureMessage.append('\n').append(header);
+        }
+        if (expected.isEmpty() == false) {
+            for (final String header : expected) {
+                missing.add(header);
             }
         }
-        if (failureMessage != null) {
-            fail(failureMessage + "\n]");
+
+        if (unexpected.isEmpty() == false || unmatched.isEmpty() == false || missing.isEmpty() == false) {
+            final StringBuilder failureMessage = new StringBuilder();
+            appendBadHeaders(failureMessage, unexpected, "got unexpected warning header" + (unexpected.size() > 1 ? "s" : ""));
+            appendBadHeaders(failureMessage, unmatched, "got unmatched warning header" + (unmatched.size() > 1 ? "s" : ""));
+            appendBadHeaders(failureMessage, missing, "did not get expected warning header" + (missing.size() > 1 ? "s" : ""));
+            fail(failureMessage.toString());
+        }
+
+    }
+
+    private void appendBadHeaders(final StringBuilder sb, final List<String> headers, final String message) {
+        if (headers.isEmpty() == false) {
+            sb.append(message).append(" [\n");
+            for (final String header : headers) {
+                sb.append("\t").append(header).append("\n");
+            }
+            sb.append("]\n");
         }
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ExecutableSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ExecutableSection.java
index 11b8f3e7a0f65..827457f4c2ae2 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ExecutableSection.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ExecutableSection.java
@@ -18,15 +18,47 @@
  */
 package org.elasticsearch.test.rest.yaml.section;
 
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentLocation;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext;
 
 import java.io.IOException;
+import java.util.Arrays;
 
 /**
  * Represents a test fragment that can be executed (e.g. api call, assertion)
  */
 public interface ExecutableSection {
+    /**
+     * {@link NamedXContentRegistry} needed in the {@link XContentParser} before calling {@link ExecutableSection#parse(XContentParser)}.
+     */
+    NamedXContentRegistry XCONTENT_REGISTRY = new NamedXContentRegistry(Arrays.asList(
+            new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("do"), DoSection::parse),
+            new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("set"), SetSection::parse),
+            new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("match"), MatchAssertion::parse),
+            new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("is_true"), IsTrueAssertion::parse),
+            new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("is_false"), IsFalseAssertion::parse),
+            new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("gt"), GreaterThanAssertion::parse),
+            new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("gte"), GreaterThanEqualToAssertion::parse),
+            new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("lt"), LessThanAssertion::parse),
+            new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("lte"), LessThanOrEqualToAssertion::parse),
+            new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("length"), LengthAssertion::parse)));
+
+    static ExecutableSection parse(XContentParser parser) throws IOException {
+        ParserUtils.advanceToFieldName(parser);
+        String section = parser.currentName();
+        XContentLocation location = parser.getTokenLocation();
+        try {
+            ExecutableSection executableSection = parser.namedObject(ExecutableSection.class, section, null);
+            parser.nextToken();
+            return executableSection;
+        } catch (Exception e) {
+            throw new IOException("Error parsing section starting at [" + location + "]", e);
+        }
+    }
+
     /**
      * Get the location in the test that this was defined. 
      */
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/GreaterThanAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/GreaterThanAssertion.java
index b531f180fd6e3..39a1f1d378067 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/GreaterThanAssertion.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/GreaterThanAssertion.java
@@ -19,8 +19,12 @@
 package org.elasticsearch.test.rest.yaml.section;
 
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.XContentLocation;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
 
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.instanceOf;
@@ -33,6 +37,15 @@
  * - gt:    { fields._ttl: 0}
  */
 public class GreaterThanAssertion extends Assertion {
+    public static GreaterThanAssertion parse(XContentParser parser) throws IOException {
+        XContentLocation location = parser.getTokenLocation();
+        Tuple<String,Object> stringObjectTuple = ParserUtils.parseTuple(parser);
+        if (! (stringObjectTuple.v2() instanceof Comparable) ) {
+            throw new IllegalArgumentException("gt section can only be used with objects that support natural ordering, found "
+                    + stringObjectTuple.v2().getClass().getSimpleName());
+        }
+        return new GreaterThanAssertion(location, stringObjectTuple.v1(), stringObjectTuple.v2());
+    }
 
     private static final Logger logger = Loggers.getLogger(GreaterThanAssertion.class);
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/GreaterThanEqualToAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/GreaterThanEqualToAssertion.java
index 14b1a08a87954..3fd9bf7adfd61 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/GreaterThanEqualToAssertion.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/GreaterThanEqualToAssertion.java
@@ -20,8 +20,12 @@
 package org.elasticsearch.test.rest.yaml.section;
 
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.XContentLocation;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
 
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.instanceOf;
@@ -34,6 +38,15 @@
  *   - gte:     { fields._ttl: 0 }
  */
 public class GreaterThanEqualToAssertion extends Assertion {
+    public static GreaterThanEqualToAssertion parse(XContentParser parser) throws IOException {
+        XContentLocation location = parser.getTokenLocation();
+        Tuple<String,Object> stringObjectTuple = ParserUtils.parseTuple(parser);
+        if (! (stringObjectTuple.v2() instanceof Comparable) ) {
+            throw new IllegalArgumentException("gte section can only be used with objects that support natural ordering, found "
+                    + stringObjectTuple.v2().getClass().getSimpleName());
+        }
+        return new GreaterThanEqualToAssertion(location, stringObjectTuple.v1(), stringObjectTuple.v2());
+    }
 
     private static final Logger logger = Loggers.getLogger(GreaterThanEqualToAssertion.class);
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/IsFalseAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/IsFalseAssertion.java
index a356182ab46ac..56ee603c70f75 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/IsFalseAssertion.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/IsFalseAssertion.java
@@ -21,6 +21,9 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.XContentLocation;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
 
 import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.equalTo;
@@ -34,6 +37,9 @@
  *
  */
 public class IsFalseAssertion extends Assertion {
+    public static IsFalseAssertion parse(XContentParser parser) throws IOException {
+        return new IsFalseAssertion(parser.getTokenLocation(), ParserUtils.parseField(parser));
+    }
 
     private static final Logger logger = Loggers.getLogger(IsFalseAssertion.class);
 
@@ -42,7 +48,6 @@ public IsFalseAssertion(XContentLocation location, String field) {
     }
 
     @Override
-    @SuppressWarnings("unchecked")
     protected void doAssert(Object actualValue, Object expectedValue) {
         logger.trace("assert that [{}] doesn't have a true value (field: [{}])", actualValue, getField());
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/IsTrueAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/IsTrueAssertion.java
index 76ca0de70d996..9b3f37e1f52a0 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/IsTrueAssertion.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/IsTrueAssertion.java
@@ -21,6 +21,9 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.XContentLocation;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.equalToIgnoringCase;
@@ -35,6 +38,9 @@
  *
  */
 public class IsTrueAssertion extends Assertion {
+    public static IsTrueAssertion parse(XContentParser parser) throws IOException {
+        return new IsTrueAssertion(parser.getTokenLocation(), ParserUtils.parseField(parser));
+    }
 
     private static final Logger logger = Loggers.getLogger(IsTrueAssertion.class);
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LengthAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LengthAssertion.java
index 062b9ecd87f34..aeecc50b90d41 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LengthAssertion.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LengthAssertion.java
@@ -19,9 +19,12 @@
 package org.elasticsearch.test.rest.yaml.section;
 
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.XContentLocation;
+import org.elasticsearch.common.xcontent.XContentParser;
 
+import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
@@ -35,6 +38,22 @@
  * - length:   { hits.hits: 1  }
  */
 public class LengthAssertion extends Assertion {
+    public static LengthAssertion parse(XContentParser parser) throws IOException {
+        XContentLocation location = parser.getTokenLocation();
+        Tuple<String,Object> stringObjectTuple = ParserUtils.parseTuple(parser);
+        assert stringObjectTuple.v2() != null;
+        int value;
+        if (stringObjectTuple.v2() instanceof Number) {
+            value = ((Number) stringObjectTuple.v2()).intValue();
+        } else {
+            try {
+                value = Integer.valueOf(stringObjectTuple.v2().toString());
+            } catch(NumberFormatException e) {
+                throw new IllegalArgumentException("length is not a valid number", e);
+            }
+        }
+        return new LengthAssertion(location, stringObjectTuple.v1(), value);
+    }
 
     private static final Logger logger = Loggers.getLogger(LengthAssertion.class);
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LessThanAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LessThanAssertion.java
index 591bd83fa624e..75a1edcf81caa 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LessThanAssertion.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LessThanAssertion.java
@@ -19,8 +19,12 @@
 package org.elasticsearch.test.rest.yaml.section;
 
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.XContentLocation;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
 
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.lessThan;
@@ -34,6 +38,15 @@
  *
  */
 public class LessThanAssertion extends Assertion {
+    public static LessThanAssertion parse(XContentParser parser) throws IOException {
+        XContentLocation location = parser.getTokenLocation();
+        Tuple<String,Object> stringObjectTuple = ParserUtils.parseTuple(parser);
+        if (false == stringObjectTuple.v2() instanceof Comparable) {
+            throw new IllegalArgumentException("lt section can only be used with objects that support natural ordering, found "
+                    + stringObjectTuple.v2().getClass().getSimpleName());
+        }
+        return new LessThanAssertion(location, stringObjectTuple.v1(), stringObjectTuple.v2());
+    }
 
     private static final Logger logger = Loggers.getLogger(LessThanAssertion.class);
 
@@ -42,7 +55,6 @@ public LessThanAssertion(XContentLocation location, String field, Object expecte
     }
 
     @Override
-    @SuppressWarnings("unchecked")
     protected void doAssert(Object actualValue, Object expectedValue) {
         logger.trace("assert that [{}] is less than [{}] (field: [{}])", actualValue, expectedValue, getField());
         assertThat("value of [" + getField() + "] is not comparable (got [" + safeClass(actualValue) + "])",
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LessThanOrEqualToAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LessThanOrEqualToAssertion.java
index 7c5710f689da3..23b6a1e4efc41 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LessThanOrEqualToAssertion.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LessThanOrEqualToAssertion.java
@@ -20,8 +20,12 @@
 package org.elasticsearch.test.rest.yaml.section;
 
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.XContentLocation;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
 
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
@@ -34,6 +38,15 @@
  *   - lte:     { fields._ttl: 0 }
  */
 public class LessThanOrEqualToAssertion  extends Assertion {
+    public static LessThanOrEqualToAssertion parse(XContentParser parser) throws IOException {
+        XContentLocation location = parser.getTokenLocation();
+        Tuple<String,Object> stringObjectTuple = ParserUtils.parseTuple(parser);
+        if (false == stringObjectTuple.v2() instanceof Comparable) {
+            throw new IllegalArgumentException("lte section can only be used with objects that support natural ordering, found "
+                    + stringObjectTuple.v2().getClass().getSimpleName());
+        }
+        return new LessThanOrEqualToAssertion(location, stringObjectTuple.v1(), stringObjectTuple.v2());
+    }
 
     private static final Logger logger = Loggers.getLogger(LessThanOrEqualToAssertion.class);
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/MatchAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/MatchAssertion.java
index 2bfb94e6584c5..fb52e8ee695cb 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/MatchAssertion.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/MatchAssertion.java
@@ -20,9 +20,12 @@
 
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.XContentLocation;
+import org.elasticsearch.common.xcontent.XContentParser;
 
+import java.io.IOException;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -43,6 +46,11 @@
  *
  */
 public class MatchAssertion extends Assertion {
+    public static MatchAssertion parse(XContentParser parser) throws IOException {
+        XContentLocation location = parser.getTokenLocation();
+        Tuple<String,Object> stringObjectTuple = ParserUtils.parseTuple(parser);
+        return new MatchAssertion(location, stringObjectTuple.v1(), stringObjectTuple.v2());
+    }
 
     private static final Logger logger = Loggers.getLogger(MatchAssertion.class);
 
@@ -173,6 +181,14 @@ private void compare(String field, @Nullable Object actual, Object expected) {
                 return;
             }
             if (Objects.equals(expected, actual)) {
+                if (expected instanceof String) {
+                    String expectedString = (String) expected;
+                    if (expectedString.length() > 50) {
+                        expectedString = expectedString.substring(0, 50) + "...";
+                    }
+                    field(field, "same [" + expectedString + "]");
+                    return;
+                }
                 field(field, "same [" + expected + "]");
                 return;
             }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ParserUtils.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ParserUtils.java
new file mode 100644
index 0000000000000..b35450c66e61a
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ParserUtils.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.rest.yaml.section;
+
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Utility methods used for parsing test sections.
+ */
+class ParserUtils {
+    private ParserUtils() {
+        // Do not build.
+    }
+
+    public static String parseField(XContentParser parser) throws IOException {
+        parser.nextToken();
+        assert parser.currentToken().isValue();
+        String field = parser.text();
+        parser.nextToken();
+        return field;
+    }
+
+    public static Tuple<String, Object> parseTuple(XContentParser parser) throws IOException {
+        parser.nextToken();
+        advanceToFieldName(parser);
+        Map<String,Object> map = parser.map();
+        assert parser.currentToken() == XContentParser.Token.END_OBJECT;
+        parser.nextToken();
+
+        if (map.size() != 1) {
+            throw new IllegalArgumentException("expected key value pair but found an object with " + map.size() + " fields");
+        }
+
+        Map.Entry<String, Object> entry = map.entrySet().iterator().next();
+        return Tuple.tuple(entry.getKey(), entry.getValue());
+    }
+
+    public static void advanceToFieldName(XContentParser parser) throws IOException {
+        XContentParser.Token token = parser.currentToken();
+        //we are in the beginning, haven't called nextToken yet
+        if (token == null) {
+            token = parser.nextToken();
+        }
+        if (token == XContentParser.Token.START_ARRAY) {
+            token = parser.nextToken();
+        }
+        if (token == XContentParser.Token.START_OBJECT) {
+            token = parser.nextToken();
+        }
+        if (token != XContentParser.Token.FIELD_NAME) {
+            throw new IllegalArgumentException("malformed test section: field name expected but found " + token + " at "
+                    + parser.getTokenLocation());
+        }
+    } 
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetSection.java
index aa39c4d30c8cb..087f667ac9bc3 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetSection.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetSection.java
@@ -18,7 +18,9 @@
  */
 package org.elasticsearch.test.rest.yaml.section;
 
+import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.xcontent.XContentLocation;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext;
 
 import java.io.IOException;
@@ -32,6 +34,28 @@
  *
  */
 public class SetSection implements ExecutableSection {
+    public static SetSection parse(XContentParser parser) throws IOException {
+        String currentFieldName = null;
+        XContentParser.Token token;
+
+        SetSection setSection = new SetSection(parser.getTokenLocation());
+
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token.isValue()) {
+                setSection.addSet(currentFieldName, parser.text());
+            }
+        }
+
+        parser.nextToken();
+
+        if (setSection.getStash().isEmpty()) {
+            throw new ParsingException(setSection.location, "set section must set at least a value");
+        }
+
+        return setSection;
+    }
 
     private final Map<String, String> stash = new HashMap<>();
     private final XContentLocation location;
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetupSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetupSection.java
index c2bcaa3ecd134..d536fa06d363a 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetupSection.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetupSection.java
@@ -18,6 +18,9 @@
  */
 package org.elasticsearch.test.rest.yaml.section;
 
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -25,6 +28,40 @@
  * Represents a setup section. Holds a skip section and multiple do sections.
  */
 public class SetupSection {
+    /**
+     * Parse a {@link SetupSection} if the next field is {@code skip}, otherwise returns {@link SetupSection#EMPTY}.
+     */
+    public static SetupSection parseIfNext(XContentParser parser) throws IOException {
+        ParserUtils.advanceToFieldName(parser);
+
+        if ("setup".equals(parser.currentName())) {
+            parser.nextToken();
+            SetupSection section = parse(parser);
+            parser.nextToken();
+            return section;
+        }
+
+        return EMPTY;
+    }
+
+    public static SetupSection parse(XContentParser parser) throws IOException {
+        SetupSection setupSection = new SetupSection();
+        setupSection.setSkipSection(SkipSection.parseIfNext(parser));
+
+        while (parser.currentToken() != XContentParser.Token.END_ARRAY) {
+            ParserUtils.advanceToFieldName(parser);
+            if (!"do".equals(parser.currentName())) {
+                throw new IllegalArgumentException("section [" + parser.currentName() + "] not supported within setup section");
+            }
+
+            setupSection.addDoSection(DoSection.parse(parser));
+            parser.nextToken();
+        }
+
+        parser.nextToken();
+
+        return setupSection;
+    }
 
     public static final SetupSection EMPTY;
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SkipSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SkipSection.java
index 3ce952b94d3b1..eb1fea4b79aed 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SkipSection.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SkipSection.java
@@ -19,9 +19,13 @@
 package org.elasticsearch.test.rest.yaml.section;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.test.rest.yaml.Features;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -32,6 +36,65 @@
  * - a specific test feature required that might not be implemented yet by the runner
  */
 public class SkipSection {
+    /**
+     * Parse a {@link SkipSection} if the next field is {@code skip}, otherwise returns {@link SkipSection#EMPTY}.
+     */
+    public static SkipSection parseIfNext(XContentParser parser) throws IOException {
+        ParserUtils.advanceToFieldName(parser);
+
+        if ("skip".equals(parser.currentName())) {
+            SkipSection section = parse(parser);
+            parser.nextToken();
+            return section;
+        }
+
+        return EMPTY;
+    }
+
+    public static SkipSection parse(XContentParser parser) throws IOException {
+        if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
+            throw new IllegalArgumentException("Expected [" + XContentParser.Token.START_OBJECT +
+                    ", found [" + parser.currentToken() + "], the skip section is not properly indented");
+        }
+        String currentFieldName = null;
+        XContentParser.Token token;
+        String version = null;
+        String reason = null;
+        List<String> features = new ArrayList<>();
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token.isValue()) {
+                if ("version".equals(currentFieldName)) {
+                    version = parser.text();
+                } else if ("reason".equals(currentFieldName)) {
+                    reason = parser.text();
+                } else if ("features".equals(currentFieldName)) {
+                    features.add(parser.text());
+                }
+                else {
+                    throw new ParsingException(parser.getTokenLocation(),
+                            "field " + currentFieldName + " not supported within skip section");
+                }
+            } else if (token == XContentParser.Token.START_ARRAY) {
+                if ("features".equals(currentFieldName)) {
+                    while(parser.nextToken() != XContentParser.Token.END_ARRAY) {
+                        features.add(parser.text());
+                    }
+                }
+            }
+        }
+
+        parser.nextToken();
+
+        if (!Strings.hasLength(version) && features.isEmpty()) {
+            throw new ParsingException(parser.getTokenLocation(), "version or features is mandatory within skip section");
+        }
+        if (Strings.hasLength(version) && !Strings.hasLength(reason)) {
+            throw new ParsingException(parser.getTokenLocation(), "reason is mandatory within skip version section");
+        }
+        return new SkipSection(version, features, reason);
+    }
 
     public static final SkipSection EMPTY = new SkipSection();
 
@@ -39,7 +102,7 @@ public class SkipSection {
     private final Version upperVersion;
     private final List<String> features;
     private final String reason;
-    
+
     private SkipSection() {
         this.lowerVersion = null;
         this.upperVersion = null;
@@ -49,7 +112,6 @@ private SkipSection() {
 
     public SkipSection(String versionRange, List<String> features, String reason) {
         assert features != null;
-        assert versionRange != null && features.isEmpty() || versionRange == null && features.isEmpty() == false;
         Version[] versions = parseVersionRange(versionRange);
         this.lowerVersion = versions[0];
         this.upperVersion = versions[1];
@@ -60,7 +122,7 @@ public SkipSection(String versionRange, List<String> features, String reason) {
     public Version getLowerVersion() {
         return lowerVersion;
     }
-    
+
     public Version getUpperVersion() {
         return upperVersion;
     }
@@ -77,11 +139,10 @@ public boolean skip(Version currentVersion) {
         if (isEmpty()) {
             return false;
         }
-        if (isVersionCheck()) {
-            return currentVersion.onOrAfter(lowerVersion) && currentVersion.onOrBefore(upperVersion);
-        } else {
-            return Features.areAllSupported(features) == false;
-        }
+        boolean skip = lowerVersion != null && upperVersion != null && currentVersion.onOrAfter(lowerVersion)
+            && currentVersion.onOrBefore(upperVersion);
+        skip |= Features.areAllSupported(features) == false;
+        return skip;
     }
 
     public boolean isVersionCheck() {
@@ -91,7 +152,7 @@ public boolean isVersionCheck() {
     public boolean isEmpty() {
         return EMPTY.equals(this);
     }
-    
+
     private Version[] parseVersionRange(String versionRange) {
         if (versionRange == null) {
             return new Version[] { null, null };
@@ -111,4 +172,16 @@ private Version[] parseVersionRange(String versionRange) {
             upper.isEmpty() ? Version.CURRENT : Version.fromString(upper)
         };
     }
+
+    public String getSkipMessage(String description) {
+        StringBuilder messageBuilder = new StringBuilder();
+        messageBuilder.append("[").append(description).append("] skipped,");
+        if (reason != null) {
+             messageBuilder.append(" reason: [").append(getReason()).append("]");
+        }
+        if (features.isEmpty() == false) {
+            messageBuilder.append(" unsupported features ").append(getFeatures());
+        }
+        return messageBuilder.toString();
+    }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/TeardownSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/TeardownSection.java
index 1e49f8b5037a7..3b272fe673c76 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/TeardownSection.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/TeardownSection.java
@@ -19,10 +19,48 @@
 
 package org.elasticsearch.test.rest.yaml.section;
 
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
 public class TeardownSection {
+    /**
+     * Parse a {@link TeardownSection} if the next field is {@code skip}, otherwise returns {@link TeardownSection#EMPTY}.
+     */
+    public static TeardownSection parseIfNext(XContentParser parser) throws IOException {
+        ParserUtils.advanceToFieldName(parser);
+
+        if ("teardown".equals(parser.currentName())) {
+            parser.nextToken();
+            TeardownSection section = parse(parser);
+            parser.nextToken();
+            return section;
+        }
+
+        return EMPTY;
+    }
+
+    public static TeardownSection parse(XContentParser parser) throws IOException {
+        TeardownSection teardownSection = new TeardownSection();
+        teardownSection.setSkipSection(SkipSection.parseIfNext(parser));
+
+        while (parser.currentToken() != XContentParser.Token.END_ARRAY) {
+            ParserUtils.advanceToFieldName(parser);
+            if (!"do".equals(parser.currentName())) {
+                throw new ParsingException(parser.getTokenLocation(),
+                        "section [" + parser.currentName() + "] not supported within teardown section");
+            }
+
+            teardownSection.addDoSection(DoSection.parse(parser));
+            parser.nextToken();
+        }
+
+        parser.nextToken();
+        return teardownSection;
+    }
 
     public static final TeardownSection EMPTY;
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/store/MockFSDirectoryService.java b/test/framework/src/main/java/org/elasticsearch/test/store/MockFSDirectoryService.java
index 057e7c4845620..0e0ced705cb85 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/store/MockFSDirectoryService.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/store/MockFSDirectoryService.java
@@ -98,7 +98,7 @@ public MockFSDirectoryService(IndexSettings idxSettings, IndexStore indexStore,
             logger.debug("Using MockDirWrapper with seed [{}] throttle: [{}] crashIndex: [{}]", SeedUtils.formatSeed(seed),
                     throttle, crashIndex);
         }
-        delegateService = randomDirectorService(indexStore, path);
+        delegateService = randomDirectoryService(indexStore, path);
     }
 
 
@@ -178,9 +178,16 @@ private Directory wrap(Directory dir) {
         return w;
     }
 
-    private FsDirectoryService randomDirectorService(IndexStore indexStore, ShardPath path) {
+    private FsDirectoryService randomDirectoryService(IndexStore indexStore, ShardPath path) {
         final IndexSettings indexSettings = indexStore.getIndexSettings();
-        final IndexMetaData build = IndexMetaData.builder(indexSettings.getIndexMetaData()).settings(Settings.builder().put(indexSettings.getSettings()).put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), RandomPicks.randomFrom(random, IndexModule.Type.values()).getSettingsKey())).build();
+        final IndexMetaData build = IndexMetaData.builder(indexSettings.getIndexMetaData())
+            .settings(Settings.builder()
+                // don't use the settings from indexSettings#getSettings() they are merged with node settings and might contain
+                // secure settings that should not be copied in here since the new IndexSettings ctor below will barf if we do
+                .put(indexSettings.getIndexMetaData().getSettings())
+                .put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(),
+                    RandomPicks.randomFrom(random, IndexModule.Type.values()).getSettingsKey()))
+            .build();
         final IndexSettings newIndexSettings = new IndexSettings(build, indexSettings.getNodeSettings());
         return new FsDirectoryService(newIndexSettings, indexStore, path);
     }
@@ -206,7 +213,7 @@ final class CloseableDirectory implements Closeable {
         private final BaseDirectoryWrapper dir;
         private final TestRuleMarkFailure failureMarker;
 
-        public CloseableDirectory(BaseDirectoryWrapper dir) {
+        CloseableDirectory(BaseDirectoryWrapper dir) {
             this.dir = dir;
             this.failureMarker = ESTestCase.getSuiteFailureMarker();
         }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManager.java b/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManager.java
index b07e7315b88cf..bccbd537a53b4 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManager.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManager.java
@@ -25,8 +25,8 @@
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskAwareRequest;
 import org.elasticsearch.tasks.TaskManager;
-import org.elasticsearch.transport.TransportRequest;
 
 import java.util.Collection;
 import java.util.concurrent.CopyOnWriteArrayList;
@@ -46,7 +46,7 @@ public MockTaskManager(Settings settings) {
     }
 
     @Override
-    public Task register(String type, String action, TransportRequest request) {
+    public Task register(String type, String action, TaskAwareRequest request) {
         Task task = super.register(type, action, request);
         if (task != null) {
             for (MockTaskManagerListener listener : listeners) {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/AssertingLocalTransport.java b/test/framework/src/main/java/org/elasticsearch/test/transport/AssertingLocalTransport.java
index 2724f53857de3..a3d4804cc9759 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/transport/AssertingLocalTransport.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/transport/AssertingLocalTransport.java
@@ -24,15 +24,19 @@
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.network.NetworkModule;
+import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
+import org.elasticsearch.plugins.NetworkPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportException;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.transport.TransportRequestOptions;
@@ -42,22 +46,31 @@
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.Random;
+import java.util.function.Supplier;
 
 public class AssertingLocalTransport extends LocalTransport {
 
     public static final String ASSERTING_TRANSPORT_NAME = "asserting_local";
 
-    public static class TestPlugin extends Plugin {
-        public void onModule(NetworkModule module) {
-            module.registerTransport(ASSERTING_TRANSPORT_NAME, AssertingLocalTransport.class);
-        }
+    public static class TestPlugin extends Plugin implements NetworkPlugin {
 
         @Override
         public List<Setting<?>> getSettings() {
             return Arrays.asList(ASSERTING_TRANSPORT_MIN_VERSION_KEY, ASSERTING_TRANSPORT_MAX_VERSION_KEY);
         }
+
+        @Override
+        public Map<String, Supplier<Transport>> getTransports(Settings settings, ThreadPool threadPool, BigArrays bigArrays,
+                                                              CircuitBreakerService circuitBreakerService,
+                                                              NamedWriteableRegistry namedWriteableRegistry,
+                                                              NetworkService networkService) {
+            return Collections.singletonMap(ASSERTING_TRANSPORT_NAME,
+                () -> new AssertingLocalTransport(settings, circuitBreakerService, threadPool, namedWriteableRegistry));
+        }
     }
 
     public static final Setting<Version> ASSERTING_TRANSPORT_MIN_VERSION_KEY =
@@ -88,10 +101,10 @@ protected void handleParsedResponse(final TransportResponse response, final Tran
     }
 
     @Override
-    public void sendRequest(final DiscoveryNode node, final long requestId, final String action, final TransportRequest request,
-                            TransportRequestOptions options) throws IOException, TransportException {
+    public void sendRequest(LocalTransport targetTransport, final DiscoveryNode node, final long requestId, final String action,
+                            final TransportRequest request, TransportRequestOptions options) throws IOException, TransportException {
         ElasticsearchAssertions.assertVersionSerializable(VersionUtils.randomVersionBetween(random, minVersion, maxVersion), request,
                 namedWriteableRegistry);
-        super.sendRequest(node, requestId, action, request, options);
+        super.sendRequest(targetTransport, node, requestId, action, request, options);
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java b/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java
index 4ff899aeac205..503503dd68b8f 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java
@@ -21,6 +21,7 @@
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.CheckedBiConsumer;
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.component.Lifecycle;
@@ -30,6 +31,7 @@
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.transport.ConnectTransportException;
+import org.elasticsearch.transport.ConnectionProfile;
 import org.elasticsearch.transport.RemoteTransportException;
 import org.elasticsearch.transport.SendRequestTransportException;
 import org.elasticsearch.transport.Transport;
@@ -40,6 +42,8 @@
 import org.elasticsearch.transport.TransportServiceAdapter;
 
 import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.net.UnknownHostException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -48,6 +52,7 @@
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicLong;
 
 import static org.apache.lucene.util.LuceneTestCase.rarely;
 
@@ -72,6 +77,8 @@ public CapturedRequest(DiscoveryNode node, long requestId, String action, Transp
 
     private ConcurrentMap<Long, Tuple<DiscoveryNode, String>> requests = new ConcurrentHashMap<>();
     private BlockingQueue<CapturedRequest> capturedRequests = ConcurrentCollections.newBlockingQueue();
+    private final AtomicLong requestId = new AtomicLong();
+
 
     /** returns all requests captured so far. Doesn't clear the captured request list. See {@link #clear()} */
     public CapturedRequest[] capturedRequests() {
@@ -185,12 +192,26 @@ public void handleError(final long requestId, final TransportException e) {
     }
 
     @Override
-    public void sendRequest(DiscoveryNode node, long requestId, String action, TransportRequest request, TransportRequestOptions options)
-        throws IOException, TransportException {
-        requests.put(requestId, Tuple.tuple(node, action));
-        capturedRequests.add(new CapturedRequest(node, requestId, action, request));
-    }
+    public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) throws IOException {
+        return new Connection() {
+            @Override
+            public DiscoveryNode getNode() {
+                return node;
+            }
+
+            @Override
+            public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options)
+                throws IOException, TransportException {
+                requests.put(requestId, Tuple.tuple(node, action));
+                capturedRequests.add(new CapturedRequest(node, requestId, action, request));
+            }
 
+            @Override
+            public void close() throws IOException {
+
+            }
+        };
+    }
 
     @Override
     public void transportServiceAdapter(TransportServiceAdapter adapter) {
@@ -208,7 +229,7 @@ public Map<String, BoundTransportAddress> profileBoundAddresses() {
     }
 
     @Override
-    public TransportAddress[] addressesFromString(String address, int perAddressLimit) throws Exception {
+    public TransportAddress[] addressesFromString(String address, int perAddressLimit) throws UnknownHostException {
         return new TransportAddress[0];
     }
 
@@ -223,12 +244,9 @@ public boolean nodeConnected(DiscoveryNode node) {
     }
 
     @Override
-    public void connectToNode(DiscoveryNode node) throws ConnectTransportException {
-
-    }
-
-    @Override
-    public void connectToNodeLight(DiscoveryNode node) throws ConnectTransportException {
+    public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile,
+                              CheckedBiConsumer<Connection, ConnectionProfile, IOException> connectionValidator)
+        throws ConnectTransportException {
 
     }
 
@@ -271,4 +289,16 @@ public List<String> getLocalAddresses() {
         return Collections.emptyList();
     }
 
+    @Override
+    public long newRequestId() {
+        return requestId.incrementAndGet();
+    }
+
+    public Connection getConnection(DiscoveryNode node) {
+        try {
+            return openConnection(node, null);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
index 9c4612b22b376..9cf3953f4df1d 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
@@ -19,47 +19,63 @@
 
 package org.elasticsearch.test.transport;
 
+import com.carrotsearch.randomizedtesting.SysGlobals;
 import org.elasticsearch.Version;
-import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.transport.TransportService;
-
+import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.CheckedBiConsumer;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.component.Lifecycle;
 import org.elasticsearch.common.component.LifecycleListener;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.network.NetworkService;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
+import org.elasticsearch.node.Node;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.tasks.TaskManager;
 import org.elasticsearch.test.tasks.MockTaskManager;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.ConnectTransportException;
+import org.elasticsearch.transport.ConnectionProfile;
+import org.elasticsearch.transport.MockTcpTransport;
 import org.elasticsearch.transport.RequestHandlerRegistry;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportException;
+import org.elasticsearch.transport.TransportInterceptor;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.transport.TransportRequestOptions;
+import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.transport.TransportServiceAdapter;
+import org.elasticsearch.transport.TransportSettings;
 import org.elasticsearch.transport.local.LocalTransport;
 
 import java.io.IOException;
+import java.net.UnknownHostException;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Queue;
 import java.util.Set;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Function;
 
 /**
  * A mock transport service that allows to simulate different network topology failures.
@@ -72,39 +88,78 @@
  * (for example, @see org.elasticsearch.discovery.zen.ping.unicast.UnicastZenPing, which constructs
  * fake DiscoveryNode instances where the publish address is one of the bound addresses).
  */
-public class MockTransportService extends TransportService {
+public final class MockTransportService extends TransportService {
 
-    public static class TestPlugin extends Plugin {
-        public void onModule(NetworkModule module) {
-            module.registerTransportService("mock", MockTransportService.class);
-        }
+    private final Map<DiscoveryNode, List<Transport.Connection>> openConnections = new HashMap<>();
+    private static final int JVM_ORDINAL = Integer.parseInt(System.getProperty(SysGlobals.CHILDVM_SYSPROP_JVM_ID, "0"));
 
+    public static class TestPlugin extends Plugin {
         @Override
         public List<Setting<?>> getSettings() {
             return Arrays.asList(MockTaskManager.USE_MOCK_TASK_MANAGER_SETTING);
         }
-        @Override
-        public Settings additionalSettings() {
-            return Settings.builder().put(NetworkModule.TRANSPORT_SERVICE_TYPE_KEY, "mock").build();
-        }
     }
 
-    public static MockTransportService local(Settings settings, Version version, ThreadPool threadPool) {
-        NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(Collections.emptyList());
+    public static MockTransportService local(Settings settings, Version version, ThreadPool threadPool,
+                                             @Nullable ClusterSettings clusterSettings) {
+        NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(ClusterModule.getNamedWriteables());
         Transport transport = new LocalTransport(settings, threadPool, namedWriteableRegistry, new NoneCircuitBreakerService()) {
             @Override
             protected Version getVersion() {
                 return version;
             }
         };
-        return new MockTransportService(settings, transport, threadPool);
+        return createNewService(settings, transport, version, threadPool, clusterSettings);
+    }
+
+    public static MockTransportService createNewService(Settings settings, Version version, ThreadPool threadPool,
+            @Nullable ClusterSettings clusterSettings) {
+        // some tests use MockTransportService to do network based testing. Yet, we run tests in multiple JVMs that means
+        // concurrent tests could claim port that another JVM just released and if that test tries to simulate a disconnect it might
+        // be smart enough to re-connect depending on what is tested. To reduce the risk, since this is very hard to debug we use
+        // a different default port range per JVM unless the incoming settings override it
+        int basePort = 10300 + (JVM_ORDINAL * 100); // use a non-default port otherwise some cluster in this JVM might reuse a port
+        settings = Settings.builder().put(TransportSettings.PORT.getKey(), basePort + "-" + (basePort+100)).put(settings).build();
+        NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(ClusterModule.getNamedWriteables());
+        final Transport transport = new MockTcpTransport(settings, threadPool, BigArrays.NON_RECYCLING_INSTANCE,
+                new NoneCircuitBreakerService(), namedWriteableRegistry, new NetworkService(settings, Collections.emptyList()), version);
+        return createNewService(settings, transport, version, threadPool, clusterSettings);
+    }
+
+    public static MockTransportService createNewService(Settings settings, Transport transport, Version version, ThreadPool threadPool,
+                                                        @Nullable ClusterSettings clusterSettings) {
+        return new MockTransportService(settings, transport, threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR,
+            boundAddress ->
+            new DiscoveryNode(Node.NODE_NAME_SETTING.get(settings), UUIDs.randomBase64UUID(), boundAddress.publishAddress(),
+                Node.NODE_ATTRIBUTES.get(settings).getAsMap(), DiscoveryNode.getRolesFromSettings(settings), version),
+            clusterSettings);
     }
 
     private final Transport original;
 
-    @Inject
-    public MockTransportService(Settings settings, Transport transport, ThreadPool threadPool) {
-        super(settings, new LookupTestTransport(transport), threadPool);
+    /**
+     * Build the service.
+     *
+     * @param clusterSettings if non null the the {@linkplain TransportService} will register with the {@link ClusterSettings} for settings
+     *        updates for {@link #TRACE_LOG_EXCLUDE_SETTING} and {@link #TRACE_LOG_INCLUDE_SETTING}.
+     */
+    public MockTransportService(Settings settings, Transport transport, ThreadPool threadPool, TransportInterceptor interceptor,
+            @Nullable ClusterSettings clusterSettings) {
+        this(settings, transport, threadPool, interceptor, (boundAddress) ->
+            DiscoveryNode.createLocal(settings, boundAddress.publishAddress(), settings.get(Node.NODE_NAME_SETTING.getKey(),
+                UUIDs.randomBase64UUID())), clusterSettings);
+    }
+
+    /**
+     * Build the service.
+     *
+     * @param clusterSettings if non null the the {@linkplain TransportService} will register with the {@link ClusterSettings} for settings
+     *        updates for {@link #TRACE_LOG_EXCLUDE_SETTING} and {@link #TRACE_LOG_INCLUDE_SETTING}.
+     */
+    public MockTransportService(Settings settings, Transport transport, ThreadPool threadPool, TransportInterceptor interceptor,
+                                Function<BoundTransportAddress, DiscoveryNode> localNodeFactory,
+                                @Nullable ClusterSettings clusterSettings) {
+        super(settings, new LookupTestTransport(transport), threadPool, interceptor, localNodeFactory, clusterSettings);
         this.original = transport;
     }
 
@@ -125,6 +180,17 @@ protected TaskManager createTaskManager() {
         }
     }
 
+    private volatile String executorName;
+
+    public void setExecutorName(final String executorName) {
+        this.executorName = executorName;
+    }
+
+    @Override
+    protected ExecutorService getExecutorService() {
+        return executorName == null ? super.getExecutorService() : getThreadPool().executor(executorName);
+    }
+
     /**
      * Clears all the registered rules.
      */
@@ -145,7 +211,10 @@ public void clearRule(TransportService transportService) {
      * Clears the rule associated with the provided transport address.
      */
     public void clearRule(TransportAddress transportAddress) {
-        transport().transports.remove(transportAddress);
+        Transport transport = transport().transports.remove(transportAddress);
+        if (transport instanceof ClearableTransport) {
+            ((ClearableTransport) transport).clearRule();
+        }
     }
 
     /**
@@ -171,20 +240,28 @@ public void addFailToSendNoConnectRule(TransportService transportService) {
      */
     public void addFailToSendNoConnectRule(TransportAddress transportAddress) {
         addDelegate(transportAddress, new DelegateTransport(original) {
+
             @Override
-            public void connectToNode(DiscoveryNode node) throws ConnectTransportException {
-                throw new ConnectTransportException(node, "DISCONNECT: simulated");
+            public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile,
+                                      CheckedBiConsumer<Connection, ConnectionProfile, IOException> connectionValidator)
+                throws ConnectTransportException {
+                if (original.nodeConnected(node) == false) {
+                    // connecting to an already connected node is a no-op
+                    throw new ConnectTransportException(node, "DISCONNECT: simulated");
+                }
             }
 
             @Override
-            public void connectToNodeLight(DiscoveryNode node) throws ConnectTransportException {
+            public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) throws IOException {
                 throw new ConnectTransportException(node, "DISCONNECT: simulated");
             }
 
             @Override
-            public void sendRequest(DiscoveryNode node, long requestId, String action, TransportRequest request,
-                                    TransportRequestOptions options) throws IOException, TransportException {
-                throw new ConnectTransportException(node, "DISCONNECT: simulated");
+            protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request,
+                                       TransportRequestOptions options) throws IOException {
+                connection.close();
+                // send the request, which will blow up
+                connection.sendRequest(requestId, action, request, options);
             }
         });
     }
@@ -218,24 +295,15 @@ public void addFailToSendNoConnectRule(TransportService transportService, final
     public void addFailToSendNoConnectRule(TransportAddress transportAddress, final Set<String> blockedActions) {
 
         addDelegate(transportAddress, new DelegateTransport(original) {
-            @Override
-            public void connectToNode(DiscoveryNode node) throws ConnectTransportException {
-                original.connectToNode(node);
-            }
 
             @Override
-            public void connectToNodeLight(DiscoveryNode node) throws ConnectTransportException {
-                original.connectToNodeLight(node);
-            }
-
-            @Override
-            public void sendRequest(DiscoveryNode node, long requestId, String action, TransportRequest request,
-                                    TransportRequestOptions options) throws IOException, TransportException {
+            protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request,
+                                       TransportRequestOptions options) throws IOException {
                 if (blockedActions.contains(action)) {
                     logger.info("--> preventing {} request", action);
-                    throw new ConnectTransportException(node, "DISCONNECT: prevented " + action + " request");
+                    connection.close();
                 }
-                original.sendRequest(node, requestId, action, request, options);
+                connection.sendRequest(requestId, action, request, options);
             }
         });
     }
@@ -256,19 +324,25 @@ public void addUnresponsiveRule(TransportService transportService) {
      */
     public void addUnresponsiveRule(TransportAddress transportAddress) {
         addDelegate(transportAddress, new DelegateTransport(original) {
+
             @Override
-            public void connectToNode(DiscoveryNode node) throws ConnectTransportException {
-                throw new ConnectTransportException(node, "UNRESPONSIVE: simulated");
+            public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile,
+                                      CheckedBiConsumer<Connection, ConnectionProfile, IOException> connectionValidator)
+                throws ConnectTransportException {
+                if (original.nodeConnected(node) == false) {
+                    // connecting to an already connected node is a no-op
+                    throw new ConnectTransportException(node, "UNRESPONSIVE: simulated");
+                }
             }
 
             @Override
-            public void connectToNodeLight(DiscoveryNode node) throws ConnectTransportException {
+            public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) throws IOException {
                 throw new ConnectTransportException(node, "UNRESPONSIVE: simulated");
             }
 
             @Override
-            public void sendRequest(DiscoveryNode node, long requestId, String action, TransportRequest request,
-                                    TransportRequestOptions options) throws IOException, TransportException {
+            protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request,
+                                       TransportRequestOptions options) throws IOException {
                 // don't send anything, the receiving node is unresponsive
             }
         });
@@ -295,17 +369,25 @@ public void addUnresponsiveRule(TransportService transportService, final TimeVal
     public void addUnresponsiveRule(TransportAddress transportAddress, final TimeValue duration) {
         final long startTime = System.currentTimeMillis();
 
-        addDelegate(transportAddress, new DelegateTransport(original) {
+        addDelegate(transportAddress, new ClearableTransport(original) {
+            private final Queue<Runnable> requestsToSendWhenCleared = new LinkedBlockingDeque<Runnable>();
+            private boolean cleared = false;
 
             TimeValue getDelay() {
                 return new TimeValue(duration.millis() - (System.currentTimeMillis() - startTime));
             }
 
             @Override
-            public void connectToNode(DiscoveryNode node) throws ConnectTransportException {
+            public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile,
+                                      CheckedBiConsumer<Connection, ConnectionProfile, IOException> connectionValidator)
+                throws ConnectTransportException {
+                if (original.nodeConnected(node)) {
+                    // connecting to an already connected node is a no-op
+                    return;
+                }
                 TimeValue delay = getDelay();
                 if (delay.millis() <= 0) {
-                    original.connectToNode(node);
+                    original.connectToNode(node, connectionProfile, connectionValidator);
                     return;
                 }
 
@@ -314,22 +396,21 @@ public void connectToNode(DiscoveryNode node) throws ConnectTransportException {
                 try {
                     if (delay.millis() < connectingTimeout.millis()) {
                         Thread.sleep(delay.millis());
-                        original.connectToNode(node);
+                        original.connectToNode(node, connectionProfile, connectionValidator);
                     } else {
                         Thread.sleep(connectingTimeout.millis());
                         throw new ConnectTransportException(node, "UNRESPONSIVE: simulated");
                     }
                 } catch (InterruptedException e) {
-                    throw new ConnectTransportException(node, "UNRESPONSIVE: interrupted while sleeping", e);
+                    throw new ConnectTransportException(node, "UNRESPONSIVE: simulated");
                 }
             }
 
             @Override
-            public void connectToNodeLight(DiscoveryNode node) throws ConnectTransportException {
+            public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) throws IOException {
                 TimeValue delay = getDelay();
                 if (delay.millis() <= 0) {
-                    original.connectToNodeLight(node);
-                    return;
+                    return original.openConnection(node, profile);
                 }
 
                 // TODO: Replace with proper setting
@@ -337,24 +418,23 @@ public void connectToNodeLight(DiscoveryNode node) throws ConnectTransportExcept
                 try {
                     if (delay.millis() < connectingTimeout.millis()) {
                         Thread.sleep(delay.millis());
-                        original.connectToNodeLight(node);
+                        return original.openConnection(node, profile);
                     } else {
                         Thread.sleep(connectingTimeout.millis());
                         throw new ConnectTransportException(node, "UNRESPONSIVE: simulated");
                     }
                 } catch (InterruptedException e) {
-                    throw new ConnectTransportException(node, "UNRESPONSIVE: interrupted while sleeping", e);
+                    throw new ConnectTransportException(node, "UNRESPONSIVE: simulated");
                 }
             }
 
             @Override
-            public void sendRequest(final DiscoveryNode node, final long requestId, final String action, TransportRequest request,
-                                    final TransportRequestOptions options) throws IOException, TransportException {
+            protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request,
+                                       TransportRequestOptions options) throws IOException {
                 // delayed sending - even if larger then the request timeout to simulated a potential late response from target node
-
                 TimeValue delay = getDelay();
                 if (delay.millis() <= 0) {
-                    original.sendRequest(node, requestId, action, request, options);
+                    connection.sendRequest(requestId, action, request, options);
                     return;
                 }
 
@@ -365,7 +445,9 @@ public void sendRequest(final DiscoveryNode node, final long requestId, final St
                 final TransportRequest clonedRequest = reg.newRequest();
                 clonedRequest.readFrom(bStream.bytes().streamInput());
 
-                threadPool.schedule(delay, ThreadPool.Names.GENERIC, new AbstractRunnable() {
+                Runnable runnable = new AbstractRunnable() {
+                    AtomicBoolean requestSent = new AtomicBoolean();
+
                     @Override
                     public void onFailure(Exception e) {
                         logger.debug("failed to send delayed request", e);
@@ -373,9 +455,30 @@ public void onFailure(Exception e) {
 
                     @Override
                     protected void doRun() throws IOException {
-                        original.sendRequest(node, requestId, action, clonedRequest, options);
+                        if (requestSent.compareAndSet(false, true)) {
+                            connection.sendRequest(requestId, action, clonedRequest, options);
+                        }
                     }
-                });
+                };
+
+                // store the request to send it once the rule is cleared.
+                synchronized (this) {
+                    if (cleared) {
+                        runnable.run();
+                    } else {
+                        requestsToSendWhenCleared.add(runnable);
+                        threadPool.schedule(delay, ThreadPool.Names.GENERIC, runnable);
+                    }
+                }
+            }
+
+            @Override
+            public void clearRule() {
+                synchronized (this) {
+                    assert cleared == false;
+                    cleared = true;
+                    requestsToSendWhenCleared.forEach(Runnable::run);
+                }
             }
         });
     }
@@ -432,24 +535,25 @@ public boolean nodeConnected(DiscoveryNode node) {
         }
 
         @Override
-        public void connectToNode(DiscoveryNode node) throws ConnectTransportException {
-            getTransport(node).connectToNode(node);
+        public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile,
+                                  CheckedBiConsumer<Connection, ConnectionProfile, IOException> connectionValidator)
+            throws ConnectTransportException {
+            getTransport(node).connectToNode(node, connectionProfile, connectionValidator);
         }
 
         @Override
-        public void connectToNodeLight(DiscoveryNode node) throws ConnectTransportException {
-            getTransport(node).connectToNodeLight(node);
+        public void disconnectFromNode(DiscoveryNode node) {
+            getTransport(node).disconnectFromNode(node);
         }
 
         @Override
-        public void disconnectFromNode(DiscoveryNode node) {
-            getTransport(node).disconnectFromNode(node);
+        public Connection getConnection(DiscoveryNode node) {
+            return getTransport(node).getConnection(node);
         }
 
         @Override
-        public void sendRequest(DiscoveryNode node, long requestId, String action, TransportRequest request,
-                                TransportRequestOptions options) throws IOException, TransportException {
-            getTransport(node).sendRequest(node, requestId, action, request, options);
+        public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) throws IOException {
+            return getTransport(node).openConnection(node, profile);
         }
     }
 
@@ -477,7 +581,7 @@ public BoundTransportAddress boundAddress() {
         }
 
         @Override
-        public TransportAddress[] addressesFromString(String address, int perAddressLimit) throws Exception {
+        public TransportAddress[] addressesFromString(String address, int perAddressLimit) throws UnknownHostException {
             return transport.addressesFromString(address, perAddressLimit);
         }
 
@@ -492,13 +596,10 @@ public boolean nodeConnected(DiscoveryNode node) {
         }
 
         @Override
-        public void connectToNode(DiscoveryNode node) throws ConnectTransportException {
-            transport.connectToNode(node);
-        }
-
-        @Override
-        public void connectToNodeLight(DiscoveryNode node) throws ConnectTransportException {
-            transport.connectToNodeLight(node);
+        public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile,
+                                  CheckedBiConsumer<Connection, ConnectionProfile, IOException> connectionValidator)
+            throws ConnectTransportException {
+            transport.connectToNode(node, connectionProfile, connectionValidator);
         }
 
         @Override
@@ -506,12 +607,6 @@ public void disconnectFromNode(DiscoveryNode node) {
             transport.disconnectFromNode(node);
         }
 
-        @Override
-        public void sendRequest(DiscoveryNode node, long requestId, String action, TransportRequest request,
-                                TransportRequestOptions options) throws IOException, TransportException {
-            transport.sendRequest(node, requestId, action, request, options);
-        }
-
         @Override
         public long serverOpen() {
             return transport.serverOpen();
@@ -522,6 +617,33 @@ public List<String> getLocalAddresses() {
             return transport.getLocalAddresses();
         }
 
+        @Override
+        public long newRequestId() {
+            return transport.newRequestId();
+        }
+
+        @Override
+        public Connection getConnection(DiscoveryNode node) {
+            return new FilteredConnection(transport.getConnection(node)) {
+                @Override
+                public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options)
+                    throws IOException, TransportException {
+                    DelegateTransport.this.sendRequest(connection, requestId, action, request, options);
+                }
+            };
+        }
+
+        @Override
+        public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) throws IOException {
+            return new FilteredConnection(transport.openConnection(node, profile)) {
+                @Override
+                public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options)
+                    throws IOException, TransportException {
+                    DelegateTransport.this.sendRequest(connection, requestId, action, request, options);
+                }
+            };
+        }
+
         @Override
         public Lifecycle.State lifecycleState() {
             return transport.lifecycleState();
@@ -548,14 +670,34 @@ public void stop() {
         }
 
         @Override
-        public void close() {
-            transport.close();
-        }
+        public void close() { transport.close(); }
 
         @Override
         public Map<String, BoundTransportAddress> profileBoundAddresses() {
             return transport.profileBoundAddresses();
         }
+
+        protected void sendRequest(Transport.Connection connection, long requestId, String action, TransportRequest request,
+                                   TransportRequestOptions options) throws IOException {
+            connection.sendRequest(requestId, action, request, options);
+        }
+    }
+
+    /**
+     * The delegate transport instances defined in this class mock various kinds of disruption types. This subclass adds a method
+     * {@link #clearRule()} so that when the disruptions are cleared (see {@link #clearRule(TransportService)}) this gives the
+     * disruption a possibility to run clean-up actions.
+     */
+    public abstract static class ClearableTransport extends DelegateTransport {
+
+        public ClearableTransport(Transport transport) {
+            super(transport);
+        }
+
+        /**
+         * Called by {@link #clearRule(TransportService)}
+         */
+        public abstract void clearRule();
     }
 
 
@@ -643,5 +785,88 @@ protected void traceRequestSent(DiscoveryNode node, long requestId, String actio
         }
     }
 
+    private static class FilteredConnection implements Transport.Connection {
+        protected final Transport.Connection connection;
+
+        private FilteredConnection(Transport.Connection connection) {
+            this.connection = connection;
+        }
+
+        @Override
+        public DiscoveryNode getNode() {
+            return connection.getNode();
+        }
+
+        @Override
+        public Version getVersion() {
+            return connection.getVersion();
+        }
+
+        @Override
+        public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options)
+            throws IOException, TransportException {
+            connection.sendRequest(requestId, action, request, options);
+        }
+
+        @Override
+        public void close() throws IOException {
+            connection.close();
+        }
+
+        @Override
+        public Object getCacheKey() {
+            return connection.getCacheKey();
+        }
+    }
+
+    public Transport getOriginalTransport() {
+        Transport transport = transport();
+        while (transport instanceof DelegateTransport) {
+            transport = ((DelegateTransport) transport).transport;
+        }
+        return transport;
+    }
+
+    @Override
+    public Transport.Connection openConnection(DiscoveryNode node, ConnectionProfile profile) throws IOException {
+        FilteredConnection filteredConnection = new FilteredConnection(super.openConnection(node, profile)) {
+            final AtomicBoolean closed = new AtomicBoolean(false);
+            @Override
+            public void close() throws IOException {
+                try {
+                    super.close();
+                } finally {
+                    if (closed.compareAndSet(false, true)) {
+                        synchronized (openConnections) {
+                            List<Transport.Connection> connections = openConnections.get(node);
+                            boolean remove = connections.remove(this);
+                            assert remove;
+                            if (connections.isEmpty()) {
+                                openConnections.remove(node);
+                            }
+                        }
+                    }
+                }
+
+            }
+        };
+        synchronized (openConnections) {
+            List<Transport.Connection> connections = openConnections.computeIfAbsent(node,
+                (n) -> new CopyOnWriteArrayList<>());
+            connections.add(filteredConnection);
+        }
+        return filteredConnection;
+    }
+
+    @Override
+    protected void doClose() throws IOException {
+        super.doClose();
+        synchronized (openConnections) {
+            assert openConnections.size() == 0 : "still open connections: " + openConnections;
+        }
+    }
 
+    public DiscoveryNode getLocalDiscoNode() {
+        return this.getLocalNode();
+    }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/threadpool/TestThreadPool.java b/test/framework/src/main/java/org/elasticsearch/threadpool/TestThreadPool.java
index 0d525f7f59aa2..f40820cc8771d 100644
--- a/test/framework/src/main/java/org/elasticsearch/threadpool/TestThreadPool.java
+++ b/test/framework/src/main/java/org/elasticsearch/threadpool/TestThreadPool.java
@@ -25,7 +25,11 @@
 public class TestThreadPool extends ThreadPool {
 
     public TestThreadPool(String name) {
-        super(Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), name).build());
+        this(name, Settings.EMPTY);
+    }
+
+    public TestThreadPool(String name, Settings settings) {
+        super(Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), name).put(settings).build());
     }
 
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
index c65e885a9bc09..71da3c66207cf 100644
--- a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
@@ -21,31 +21,53 @@
 
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
+import org.apache.lucene.util.CollectionUtil;
+import org.apache.lucene.util.Constants;
+import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListenerResponseHandler;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.network.NetworkService;
+import org.elasticsearch.common.network.NetworkUtils;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.BoundTransportAddress;
+import org.elasticsearch.common.transport.InetSocketTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
+import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.junit.annotations.TestLogging;
+import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.local.LocalTransport;
 import org.junit.After;
 import org.junit.Before;
 
 import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.net.Inet4Address;
+import java.net.Inet6Address;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.net.Socket;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -56,21 +78,28 @@
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
-import static org.hamcrest.Matchers.endsWith;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasToString;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.startsWith;
 
 public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
 
     protected ThreadPool threadPool;
 
     protected static final Version version0 = Version.CURRENT.minimumCompatibilityVersion();
+
+    private ClusterSettings clusterSettings;
+
     protected volatile DiscoveryNode nodeA;
     protected volatile MockTransportService serviceA;
 
@@ -78,23 +107,21 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
     protected volatile DiscoveryNode nodeB;
     protected volatile MockTransportService serviceB;
 
-    protected abstract MockTransportService build(Settings settings, Version version);
+    protected abstract MockTransportService build(Settings settings, Version version, ClusterSettings clusterSettings, boolean doHandshake);
 
     @Override
     @Before
     public void setUp() throws Exception {
         super.setUp();
         threadPool = new TestThreadPool(getClass().getName());
-        serviceA = buildService("TS_A", version0);
-        nodeA = new DiscoveryNode("TS_A", serviceA.boundAddress().publishAddress(), emptyMap(), emptySet(), version0);
-        // serviceA.setLocalNode(nodeA);
-        serviceB = buildService("TS_B", version1);
-        nodeB = new DiscoveryNode("TS_B", serviceB.boundAddress().publishAddress(), emptyMap(), emptySet(), version1);
-        //serviceB.setLocalNode(nodeB);
+        clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        serviceA = buildService("TS_A", version0, clusterSettings); // this one supports dynamic tracer updates
+        nodeA = serviceA.getLocalNode();
+        serviceB = buildService("TS_B", version1, null); // this one doesn't support dynamic tracer updates
+        nodeB = serviceB.getLocalNode();
         // wait till all nodes are properly connected and the event has been sent, so tests in this class
         // will not get this callback called on the connections done in this setup
-        final boolean useLocalNode = randomBoolean();
-        final CountDownLatch latch = new CountDownLatch(useLocalNode ? 2 : 4);
+        final CountDownLatch latch = new CountDownLatch(2);
         TransportConnectionListener waitForConnection = new TransportConnectionListener() {
             @Override
             public void onNodeConnected(DiscoveryNode node) {
@@ -108,58 +135,78 @@ public void onNodeDisconnected(DiscoveryNode node) {
         };
         serviceA.addConnectionListener(waitForConnection);
         serviceB.addConnectionListener(waitForConnection);
-
-        if (useLocalNode) {
-            logger.info("--> using local node optimization");
-            serviceA.setLocalNode(nodeA);
-            serviceB.setLocalNode(nodeB);
-        } else {
-            logger.info("--> actively connecting to local node");
-            serviceA.connectToNode(nodeA);
-            serviceB.connectToNode(nodeB);
-        }
-
+        int numHandshakes = 1;
         serviceA.connectToNode(nodeB);
         serviceB.connectToNode(nodeA);
+        assertNumHandshakes(numHandshakes, serviceA.getOriginalTransport());
+        assertNumHandshakes(numHandshakes, serviceB.getOriginalTransport());
 
         assertThat("failed to wait for all nodes to connect", latch.await(5, TimeUnit.SECONDS), equalTo(true));
         serviceA.removeConnectionListener(waitForConnection);
         serviceB.removeConnectionListener(waitForConnection);
     }
 
-    private MockTransportService buildService(final String name, final Version version) {
+    private MockTransportService buildService(final String name, final Version version, ClusterSettings clusterSettings,
+                                              Settings settings, boolean acceptRequests, boolean doHandshake) {
         MockTransportService service = build(
-            Settings.builder()
-                .put(Node.NODE_NAME_SETTING.getKey(), name)
-                .put(TransportService.TRACE_LOG_INCLUDE_SETTING.getKey(), "")
-                .put(TransportService.TRACE_LOG_EXCLUDE_SETTING.getKey(), "NOTHING")
-                .build(),
-            version);
-        service.acceptIncomingRequests();
+                Settings.builder()
+                        .put(settings)
+                        .put(Node.NODE_NAME_SETTING.getKey(), name)
+                        .put(TransportService.TRACE_LOG_INCLUDE_SETTING.getKey(), "")
+                        .put(TransportService.TRACE_LOG_EXCLUDE_SETTING.getKey(), "NOTHING")
+                        .build(),
+                version,
+                clusterSettings, doHandshake);
+        if (acceptRequests) {
+            service.acceptIncomingRequests();
+        }
         return service;
     }
 
+    private MockTransportService buildService(final String name, final Version version, ClusterSettings clusterSettings) {
+        return buildService(name, version, clusterSettings, Settings.EMPTY, true, true);
+    }
+
     @Override
     @After
     public void tearDown() throws Exception {
         super.tearDown();
-        serviceA.close();
-        serviceB.close();
-        terminate(threadPool);
+        try {
+            assertNoPendingHandshakes(serviceA.getOriginalTransport());
+            assertNoPendingHandshakes(serviceB.getOriginalTransport());
+        } finally {
+            IOUtils.close(serviceA, serviceB, () -> {
+                try {
+                    terminate(threadPool);
+                } catch (InterruptedException e) {
+                    Thread.currentThread().interrupt();
+                }
+            });
+        }
+    }
+
+    public void assertNumHandshakes(long expected, Transport transport) {
+        if (transport instanceof TcpTransport) {
+            assertEquals(expected, ((TcpTransport) transport).getNumHandshakes());
+        }
+    }
+
+    public void assertNoPendingHandshakes(Transport transport) {
+        if (transport instanceof TcpTransport) {
+            assertEquals(0, ((TcpTransport) transport).getNumPendingHandshakes());
+        }
     }
 
+
     public void testHelloWorld() {
         serviceA.registerRequestHandler("sayHello", StringMessageRequest::new, ThreadPool.Names.GENERIC,
-            new TransportRequestHandler<StringMessageRequest>() {
-                @Override
-                public void messageReceived(StringMessageRequest request, TransportChannel channel) {
-                    assertThat("moshe", equalTo(request.message));
-                    try {
-                        channel.sendResponse(new StringMessageResponse("hello " + request.message));
-                    } catch (IOException e) {
-                        logger.error("Unexpected failure", e);
-                        fail(e.getMessage());
-                    }
+            (request, channel) -> {
+                assertThat("moshe", equalTo(request.message));
+                try {
+                    channel.sendResponse(new StringMessageResponse("hello " + request.message));
+                } catch (IOException e) {
+                    logger.error("Unexpected failure", e);
+                    fail(e.getMessage());
                 }
             });
 
@@ -224,8 +271,6 @@ public void handleException(TransportException exp) {
         } catch (Exception e) {
             assertThat(e.getMessage(), false, equalTo(true));
         }
-
-        serviceA.removeHandler("sayHello");
     }
 
     public void testThreadContext() throws ExecutionException, InterruptedException {
@@ -281,26 +326,19 @@ public void handleException(TransportException exp) {
         assertEquals("ping_user", threadPool.getThreadContext().getHeader("test.ping.user"));
         assertSame(context, threadPool.getThreadContext().getTransient("my_private_context"));
         assertNull("this header is only visible in the handler context", threadPool.getThreadContext().getHeader("some.temp.header"));
-
-        serviceA.removeHandler("sayHello");
     }
 
     public void testLocalNodeConnection() throws InterruptedException {
         assertTrue("serviceA is not connected to nodeA", serviceA.nodeConnected(nodeA));
-        if (((TransportService) serviceA).getLocalNode() != null) {
-            // this should be a noop
-            serviceA.disconnectFromNode(nodeA);
-        }
+        // this should be a noop
+        serviceA.disconnectFromNode(nodeA);
         final AtomicReference<Exception> exception = new AtomicReference<>();
         serviceA.registerRequestHandler("localNode", StringMessageRequest::new, ThreadPool.Names.GENERIC,
-            new TransportRequestHandler<StringMessageRequest>() {
-                @Override
-                public void messageReceived(StringMessageRequest request, TransportChannel channel) {
-                    try {
-                        channel.sendResponse(new StringMessageResponse(request.message));
-                    } catch (IOException e) {
-                        exception.set(e);
-                    }
+            (request, channel) -> {
+                try {
+                    channel.sendResponse(new StringMessageResponse(request.message));
+                } catch (IOException e) {
+                    exception.set(e);
                 }
             });
         final AtomicReference<String> responseString = new AtomicReference<>();
@@ -333,6 +371,113 @@ public String executor() {
         assertThat(responseString.get(), equalTo("test"));
     }
 
+    public void testAdapterSendReceiveCallbacks() throws Exception {
+        final TransportRequestHandler<TransportRequest.Empty> requestHandler = (request, channel) -> {
+            try {
+                if (randomBoolean()) {
+                    channel.sendResponse(TransportResponse.Empty.INSTANCE);
+                } else {
+                    channel.sendResponse(new ElasticsearchException("simulated"));
+                }
+            } catch (IOException e) {
+                logger.error("Unexpected failure", e);
+                fail(e.getMessage());
+            }
+        };
+        final String ACTION = "action";
+        serviceA.registerRequestHandler(ACTION, TransportRequest.Empty::new, ThreadPool.Names.GENERIC,
+            requestHandler);
+        serviceB.registerRequestHandler(ACTION, TransportRequest.Empty::new, ThreadPool.Names.GENERIC,
+            requestHandler);
+
+
+        class CountingTracer extends MockTransportService.Tracer {
+            AtomicInteger requestsReceived = new AtomicInteger();
+            AtomicInteger requestsSent = new AtomicInteger();
+            AtomicInteger responseReceived = new AtomicInteger();
+            AtomicInteger responseSent = new AtomicInteger();
+
+            @Override
+            public void receivedRequest(long requestId, String action) {
+                if (action.equals(ACTION)) {
+                    requestsReceived.incrementAndGet();
+                }
+            }
+
+            @Override
+            public void responseSent(long requestId, String action) {
+                if (action.equals(ACTION)) {
+                    responseSent.incrementAndGet();
+                }
+            }
+
+            @Override
+            public void responseSent(long requestId, String action, Throwable t) {
+                if (action.equals(ACTION)) {
+                    responseSent.incrementAndGet();
+                }
+            }
+
+            @Override
+            public void receivedResponse(long requestId, DiscoveryNode sourceNode, String action) {
+                if (action.equals(ACTION)) {
+                    responseReceived.incrementAndGet();
+                }
+            }
+
+            @Override
+            public void requestSent(DiscoveryNode node, long requestId, String action, TransportRequestOptions options) {
+                if (action.equals(ACTION)) {
+                    requestsSent.incrementAndGet();
+                }
+            }
+        }
+        final CountingTracer tracerA = new CountingTracer();
+        final CountingTracer tracerB = new CountingTracer();
+        serviceA.addTracer(tracerA);
+        serviceB.addTracer(tracerB);
+
+        try {
+            serviceA
+                .submitRequest(nodeB, ACTION, TransportRequest.Empty.INSTANCE, EmptyTransportResponseHandler.INSTANCE_SAME).get();
+        } catch (ExecutionException e) {
+            assertThat(e.getCause(), instanceOf(ElasticsearchException.class));
+            assertThat(ExceptionsHelper.unwrapCause(e.getCause()).getMessage(), equalTo("simulated"));
+        }
+
+        // use assert busy as call backs are sometime called after the response have been sent
+        assertBusy(() -> {
+            assertThat(tracerA.requestsReceived.get(), equalTo(0));
+            assertThat(tracerA.requestsSent.get(), equalTo(1));
+            assertThat(tracerA.responseReceived.get(), equalTo(1));
+            assertThat(tracerA.responseSent.get(), equalTo(0));
+            assertThat(tracerB.requestsReceived.get(), equalTo(1));
+            assertThat(tracerB.requestsSent.get(), equalTo(0));
+            assertThat(tracerB.responseReceived.get(), equalTo(0));
+            assertThat(tracerB.responseSent.get(), equalTo(1));
+        });
+
+        try {
+            serviceA
+                .submitRequest(nodeA, ACTION, TransportRequest.Empty.INSTANCE, EmptyTransportResponseHandler.INSTANCE_SAME).get();
+        } catch (ExecutionException e) {
+            assertThat(e.getCause(), instanceOf(ElasticsearchException.class));
+            assertThat(ExceptionsHelper.unwrapCause(e.getCause()).getMessage(), equalTo("simulated"));
+        }
+
+        // use assert busy as call backs are sometime called after the response have been sent
+        assertBusy(() -> {
+            assertThat(tracerA.requestsReceived.get(), equalTo(1));
+            assertThat(tracerA.requestsSent.get(), equalTo(2));
+            assertThat(tracerA.responseReceived.get(), equalTo(2));
+            assertThat(tracerA.responseSent.get(), equalTo(1));
+            assertThat(tracerB.requestsReceived.get(), equalTo(1));
+            assertThat(tracerB.requestsSent.get(), equalTo(0));
+            assertThat(tracerB.responseReceived.get(), equalTo(0));
+            assertThat(tracerB.responseSent.get(), equalTo(1));
+        });
+    }
+
     public void testVoidMessageCompressed() {
         serviceA.registerRequestHandler("sayHello", TransportRequest.Empty::new, ThreadPool.Names.GENERIC,
             (request, channel) -> {
@@ -375,8 +520,6 @@ public void handleException(TransportException exp) {
         } catch (Exception e) {
             assertThat(e.getMessage(), false, equalTo(true));
         }
-
-        serviceA.removeHandler("sayHello");
     }
 
     public void testHelloWorldCompressed() {
@@ -426,8 +569,6 @@ public void handleException(TransportException exp) {
         } catch (Exception e) {
             assertThat(e.getMessage(), false, equalTo(true));
         }
-
-        serviceA.removeHandler("sayHello");
     }
 
     public void testErrorMessage() {
@@ -469,8 +610,6 @@ public void handleException(TransportException exp) {
         } catch (Exception e) {
             assertThat(e.getCause().getMessage(), equalTo("runtime_exception: bad message !!!"));
         }
-
-        serviceA.removeHandler("sayHelloException");
     }
 
     public void testDisconnectListener() throws Exception {
@@ -566,17 +705,22 @@ protected void doRun() throws Exception {
                         PlainActionFuture<TestResponse> listener = new PlainActionFuture<>();
                         final String info = sender + "_" + iter;
                         final DiscoveryNode node = nodeB; // capture now
-                        serviceA.sendRequest(node, "test", new TestRequest(info),
-                            new ActionListenerResponseHandler<>(listener, TestResponse::new));
                         try {
-                            listener.actionGet();
-                        } catch (ConnectTransportException e) {
-                            // ok!
-                        } catch (Exception e) {
-                            logger.error(
-                                (Supplier<?>) () -> new ParameterizedMessage("caught exception while sending to node {}", node), e);
-                            sendingErrors.add(e);
+                            serviceA.sendRequest(node, "test", new TestRequest(info),
+                                new ActionListenerResponseHandler<>(listener, TestResponse::new));
+                            try {
+                                listener.actionGet();
+                            } catch (ConnectTransportException e) {
+                                // ok!
+                            } catch (Exception e) {
+                                logger.error(
+                                    (Supplier<?>) () -> new ParameterizedMessage("caught exception while sending to node {}", node), e);
+                                sendingErrors.add(e);
+                            }
+                        } catch (NodeNotConnectedException ex) {
+                            // ok
                         }
+
                     }
                 }
 
@@ -591,10 +735,10 @@ public void onAfter() {
             if (i % 3 == 0) {
                 // simulate restart of nodeB
                 serviceB.close();
-                MockTransportService newService = buildService("TS_B_" + i, version1);
+                MockTransportService newService = buildService("TS_B_" + i, version1, null);
                 newService.registerRequestHandler("test", TestRequest::new, ThreadPool.Names.SAME, ignoringRequestHandler);
                 serviceB = newService;
-                nodeB = new DiscoveryNode("TS_B_" + i, "TS_B", serviceB.boundAddress().publishAddress(), emptyMap(), emptySet(), version1);
+                nodeB = newService.getLocalDiscoNode();
                 serviceB.connectToNode(nodeA);
                 serviceA.connectToNode(nodeB);
             } else if (serviceA.nodeConnected(nodeB)) {
@@ -612,11 +756,9 @@ public void onAfter() {
 
     public void testNotifyOnShutdown() throws Exception {
         final CountDownLatch latch2 = new CountDownLatch(1);
-
-        serviceA.registerRequestHandler("foobar", StringMessageRequest::new, ThreadPool.Names.GENERIC,
-            new TransportRequestHandler<StringMessageRequest>() {
-                @Override
-                public void messageReceived(StringMessageRequest request, TransportChannel channel) {
+        try {
+            serviceA.registerRequestHandler("foobar", StringMessageRequest::new, ThreadPool.Names.GENERIC,
+                (request, channel) -> {
                     try {
                         latch2.await();
                         logger.info("Stop ServiceB now");
@@ -624,18 +766,20 @@ public void messageReceived(StringMessageRequest request, TransportChannel chann
                     } catch (Exception e) {
                         fail(e.getMessage());
                     }
-                }
-            });
-        TransportFuture<TransportResponse.Empty> foobar = serviceB.submitRequest(nodeA, "foobar",
-            new StringMessageRequest(""), TransportRequestOptions.EMPTY, EmptyTransportResponseHandler.INSTANCE_SAME);
-        latch2.countDown();
-        try {
-            foobar.txGet();
-            fail("TransportException expected");
-        } catch (TransportException ex) {
+                });
+            TransportFuture<TransportResponse.Empty> foobar = serviceB.submitRequest(nodeA, "foobar",
+                new StringMessageRequest(""), TransportRequestOptions.EMPTY, EmptyTransportResponseHandler.INSTANCE_SAME);
+            latch2.countDown();
+            try {
+                foobar.txGet();
+                fail("TransportException expected");
+            } catch (TransportException ex) {
 
+            }
+        } finally {
+            serviceB.close(); // make sure we are fully closed here otherwise we might run into assertions down the road
+            serviceA.disconnectFromNode(nodeB);
         }
-        serviceA.removeHandler("sayHelloTimeoutDelayedResponse");
     }
 
     public void testTimeoutSendExceptionWithNeverSendingBackResponse() throws Exception {
@@ -678,8 +822,6 @@ public void handleException(TransportException exp) {
         } catch (Exception e) {
             assertThat(e, instanceOf(ReceiveTimeoutTransportException.class));
         }
-
-        serviceA.removeHandler("sayHelloTimeoutNoResponse");
     }
 
     public void testTimeoutSendExceptionWithDelayedResponse() throws Exception {
@@ -785,21 +927,13 @@ public void handleException(TransportException exp) {
         for (Runnable runnable : assertions) {
             runnable.run();
         }
-        serviceA.removeHandler("sayHelloTimeoutDelayedResponse");
         waitForever.countDown();
         doneWaitingForever.await();
         assertTrue(inFlight.tryAcquire(Integer.MAX_VALUE, 10, TimeUnit.SECONDS));
     }
 
-    @TestLogging(value = "test.transport.tracer:TRACE")
     public void testTracerLog() throws InterruptedException {
-        TransportRequestHandler handler = new TransportRequestHandler<StringMessageRequest>() {
-            @Override
-            public void messageReceived(StringMessageRequest request, TransportChannel channel) throws Exception {
-                channel.sendResponse(new StringMessageResponse(""));
-            }
-        };
-
+        TransportRequestHandler handler = (request, channel) -> channel.sendResponse(new StringMessageResponse(""));
         TransportRequestHandler handlerWithError = new TransportRequestHandler<StringMessageRequest>() {
             @Override
             public void messageReceived(StringMessageRequest request, TransportChannel channel) throws Exception {
@@ -840,7 +974,10 @@ public String executor() {
         serviceB.registerRequestHandler("test", StringMessageRequest::new, ThreadPool.Names.SAME, handler);
         serviceB.registerRequestHandler("testError", StringMessageRequest::new, ThreadPool.Names.SAME, handlerWithError);
 
-        final Tracer tracer = new Tracer();
+        final Tracer tracer = new Tracer(new HashSet<>(Arrays.asList("test", "testError")));
+        // the tracer is invoked concurrently after the actual action is executed. that means a Tracer#requestSent can still be in-flight
+        // from a handshake executed on connect in the setup method. this might confuse this test since it expects exact number of
+        // invocations. To prevent any unrelated events messing with this test we filter on the actions we execute in this test.
         serviceA.addTracer(tracer);
         serviceB.addTracer(tracer);
 
@@ -877,9 +1014,7 @@ public String executor() {
             includeSettings = "test";
             excludeSettings = "DOESN'T_MATCH";
         }
-        ClusterSettings service = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-        serviceA.setDynamicSettings(service);
-        service.applySettings(Settings.builder()
+        clusterSettings.applySettings(Settings.builder()
             .put(TransportService.TRACE_LOG_INCLUDE_SETTING.getKey(), includeSettings)
             .put(TransportService.TRACE_LOG_EXCLUDE_SETTING.getKey(), excludeSettings)
             .build());
@@ -906,6 +1041,7 @@ public String executor() {
     }
 
     private static class Tracer extends MockTransportService.Tracer {
+        private final Set<String> actions;
         public volatile boolean sawRequestSent;
         public volatile boolean sawRequestReceived;
         public volatile boolean sawResponseSent;
@@ -914,40 +1050,52 @@ private static class Tracer extends MockTransportService.Tracer {
 
         public AtomicReference<CountDownLatch> expectedEvents = new AtomicReference<>();
 
-
+        Tracer(Set<String> actions) {
+            this.actions = actions;
+        }
         @Override
         public void receivedRequest(long requestId, String action) {
             super.receivedRequest(requestId, action);
-            sawRequestReceived = true;
-            expectedEvents.get().countDown();
+            if (actions.contains(action)) {
+                sawRequestReceived = true;
+                expectedEvents.get().countDown();
+            }
         }
 
         @Override
         public void requestSent(DiscoveryNode node, long requestId, String action, TransportRequestOptions options) {
             super.requestSent(node, requestId, action, options);
-            sawRequestSent = true;
-            expectedEvents.get().countDown();
+            if (actions.contains(action)) {
+                sawRequestSent = true;
+                expectedEvents.get().countDown();
+            }
         }
 
         @Override
         public void responseSent(long requestId, String action) {
             super.responseSent(requestId, action);
-            sawResponseSent = true;
-            expectedEvents.get().countDown();
+            if (actions.contains(action)) {
+                sawResponseSent = true;
+                expectedEvents.get().countDown();
+            }
         }
 
         @Override
         public void responseSent(long requestId, String action, Throwable t) {
             super.responseSent(requestId, action, t);
-            sawErrorSent = true;
-            expectedEvents.get().countDown();
+            if (actions.contains(action)) {
+                sawErrorSent = true;
+                expectedEvents.get().countDown();
+            }
         }
 
         @Override
         public void receivedResponse(long requestId, DiscoveryNode sourceNode, String action) {
             super.receivedResponse(requestId, sourceNode, action);
-            sawResponseReceived = true;
-            expectedEvents.get().countDown();
+            if (actions.contains(action)) {
+                sawResponseReceived = true;
+                expectedEvents.get().countDown();
+            }
         }
 
         public void reset(int expectedCount) {
@@ -1110,6 +1258,7 @@ public void messageReceived(Version1Request request, TransportChannel channel) t
                     response.value1 = 1;
                     response.value2 = 2;
                     channel.sendResponse(response);
+                    assertEquals(version0, channel.getVersion());
                 }
             });
 
@@ -1151,6 +1300,7 @@ public void messageReceived(Version0Request request, TransportChannel channel) t
                     Version0Response response = new Version0Response();
                     response.value1 = 1;
                     channel.sendResponse(response);
+                    assertEquals(version0, channel.getVersion());
                 }
             });
 
@@ -1188,16 +1338,14 @@ public String executor() {
 
     public void testVersionFrom1to1() throws Exception {
         serviceB.registerRequestHandler("/version", Version1Request::new, ThreadPool.Names.SAME,
-            new TransportRequestHandler<Version1Request>() {
-                @Override
-                public void messageReceived(Version1Request request, TransportChannel channel) throws Exception {
-                    assertThat(request.value1, equalTo(1));
-                    assertThat(request.value2, equalTo(2));
-                    Version1Response response = new Version1Response();
-                    response.value1 = 1;
-                    response.value2 = 2;
-                    channel.sendResponse(response);
-                }
+            (request, channel) -> {
+                assertThat(request.value1, equalTo(1));
+                assertThat(request.value2, equalTo(2));
+                Version1Response response = new Version1Response();
+                response.value1 = 1;
+                response.value2 = 2;
+                channel.sendResponse(response);
+                assertEquals(version1, channel.getVersion());
             });
 
         Version1Request version1Request = new Version1Request();
@@ -1239,6 +1387,7 @@ public void testVersionFrom0to0() throws Exception {
                 Version0Response response = new Version0Response();
                 response.value1 = 1;
                 channel.sendResponse(response);
+                assertEquals(version0, channel.getVersion());
             });
 
         Version0Request version0Request = new Version0Request();
@@ -1270,14 +1419,11 @@ public String executor() {
         assertThat(version0Response.value1, equalTo(1));
     }
 
-    public void testMockFailToSendNoConnectRule() {
+    public void testMockFailToSendNoConnectRule() throws Exception {
         serviceA.registerRequestHandler("sayHello", StringMessageRequest::new, ThreadPool.Names.GENERIC,
-            new TransportRequestHandler<StringMessageRequest>() {
-                @Override
-                public void messageReceived(StringMessageRequest request, TransportChannel channel) throws Exception {
-                    assertThat("moshe", equalTo(request.message));
-                    throw new RuntimeException("bad message !!!");
-                }
+            (request, channel) -> {
+                assertThat("moshe", equalTo(request.message));
+                throw new RuntimeException("bad message !!!");
             });
 
         serviceB.addFailToSendNoConnectRule(serviceA);
@@ -1301,7 +1447,9 @@ public void handleResponse(StringMessageResponse response) {
 
                 @Override
                 public void handleException(TransportException exp) {
-                    assertThat(exp.getCause().getMessage(), endsWith("DISCONNECT: simulated"));
+                    Throwable cause = ExceptionsHelper.unwrapCause(exp);
+                    assertThat(cause, instanceOf(ConnectTransportException.class));
+                    assertThat(((ConnectTransportException)cause).node(), equalTo(nodeA));
                 }
             });
 
@@ -1309,34 +1457,30 @@ public void handleException(TransportException exp) {
             res.txGet();
             fail("exception should be thrown");
         } catch (Exception e) {
-            assertThat(e.getCause().getMessage(), endsWith("DISCONNECT: simulated"));
+            Throwable cause = ExceptionsHelper.unwrapCause(e);
+            assertThat(cause, instanceOf(ConnectTransportException.class));
+            assertThat(((ConnectTransportException)cause).node(), equalTo(nodeA));
         }
 
-        try {
-            serviceB.connectToNode(nodeA);
-            fail("exception should be thrown");
-        } catch (ConnectTransportException e) {
-            // all is well
-        }
+        // wait for the transport to process the sending failure and disconnect from node
+        assertBusy(() -> assertFalse(serviceB.nodeConnected(nodeA)));
 
+        // now try to connect again and see that it fails
         try {
-            serviceB.connectToNodeLightAndHandshake(nodeA, 100);
+            serviceB.connectToNode(nodeA);
             fail("exception should be thrown");
         } catch (ConnectTransportException e) {
             // all is well
         }
 
-        serviceA.removeHandler("sayHello");
+        expectThrows(ConnectTransportException.class, () -> serviceB.openConnection(nodeA, MockTcpTransport.LIGHT_PROFILE));
     }
 
-    public void testMockUnresponsiveRule() {
+    public void testMockUnresponsiveRule() throws IOException {
         serviceA.registerRequestHandler("sayHello", StringMessageRequest::new, ThreadPool.Names.GENERIC,
-            new TransportRequestHandler<StringMessageRequest>() {
-                @Override
-                public void messageReceived(StringMessageRequest request, TransportChannel channel) throws Exception {
-                    assertThat("moshe", equalTo(request.message));
-                    throw new RuntimeException("bad message !!!");
-                }
+            (request, channel) -> {
+                assertThat("moshe", equalTo(request.message));
+                throw new RuntimeException("bad message !!!");
             });
 
         serviceB.addUnresponsiveRule(serviceA);
@@ -1373,20 +1517,14 @@ public void handleException(TransportException exp) {
         }
 
         try {
+            serviceB.disconnectFromNode(nodeA);
             serviceB.connectToNode(nodeA);
             fail("exception should be thrown");
         } catch (ConnectTransportException e) {
             // all is well
         }
 
-        try {
-            serviceB.connectToNodeLightAndHandshake(nodeA, 100);
-            fail("exception should be thrown");
-        } catch (ConnectTransportException e) {
-            // all is well
-        }
-
-        serviceA.removeHandler("sayHello");
+        expectThrows(ConnectTransportException.class, () -> serviceB.openConnection(nodeA, MockTcpTransport.LIGHT_PROFILE));
     }
 
 
@@ -1434,55 +1572,52 @@ public String executor() {
     }
 
     public void testBlockingIncomingRequests() throws Exception {
-        TransportService service = build(
-            Settings.builder()
-                .put("name", "TS_TEST")
-                .put(TransportService.TRACE_LOG_INCLUDE_SETTING.getKey(), "")
-                .put(TransportService.TRACE_LOG_EXCLUDE_SETTING.getKey(), "NOTHING")
-                .build(),
-            version0);
-        AtomicBoolean requestProcessed = new AtomicBoolean();
-        service.registerRequestHandler("action", TestRequest::new, ThreadPool.Names.SAME,
-            (request, channel) -> {
-                requestProcessed.set(true);
-                channel.sendResponse(TransportResponse.Empty.INSTANCE);
-            });
-
-        DiscoveryNode node =
-            new DiscoveryNode("TS_TEST", "TS_TEST", service.boundAddress().publishAddress(), emptyMap(), emptySet(), version0);
-        serviceA.connectToNode(node);
-
-        CountDownLatch latch = new CountDownLatch(1);
-        serviceA.sendRequest(node, "action", new TestRequest(), new TransportResponseHandler<TestResponse>() {
-            @Override
-            public TestResponse newInstance() {
-                return new TestResponse();
-            }
+        try (TransportService service = buildService("TS_TEST", version0, null,
+            Settings.EMPTY, false, false)) {
+            AtomicBoolean requestProcessed = new AtomicBoolean(false);
+            service.registerRequestHandler("action", TestRequest::new, ThreadPool.Names.SAME,
+                (request, channel) -> {
+                    requestProcessed.set(true);
+                    channel.sendResponse(TransportResponse.Empty.INSTANCE);
+                });
 
-            @Override
-            public void handleResponse(TestResponse response) {
-                latch.countDown();
-            }
+            DiscoveryNode node = service.getLocalNode();
+            serviceA.close();
+            serviceA = buildService("TS_A", version0, null,
+                Settings.EMPTY, true, false);
+            try (Transport.Connection connection = serviceA.openConnection(node, null)) {
+                CountDownLatch latch = new CountDownLatch(1);
+                serviceA.sendRequest(connection, "action", new TestRequest(), TransportRequestOptions.EMPTY,
+                    new TransportResponseHandler<TestResponse>() {
+                    @Override
+                    public TestResponse newInstance() {
+                        return new TestResponse();
+                    }
 
-            @Override
-            public void handleException(TransportException exp) {
-                latch.countDown();
-            }
+                    @Override
+                    public void handleResponse(TestResponse response) {
+                        latch.countDown();
+                    }
 
-            @Override
-            public String executor() {
-                return ThreadPool.Names.SAME;
-            }
-        });
+                    @Override
+                    public void handleException(TransportException exp) {
+                        latch.countDown();
+                    }
 
-        assertFalse(requestProcessed.get());
+                    @Override
+                    public String executor() {
+                        return ThreadPool.Names.SAME;
+                    }
+                });
 
-        service.acceptIncomingRequests();
-        assertBusy(() -> assertTrue(requestProcessed.get()));
+                assertFalse(requestProcessed.get());
 
-        latch.await();
-        service.close();
+                service.acceptIncomingRequests();
+                assertBusy(() -> assertTrue(requestProcessed.get()));
 
+                latch.await();
+            }
+        }
     }
 
     public static class TestRequest extends TransportRequest {
@@ -1523,10 +1658,10 @@ private static class TestResponse extends TransportResponse {
 
         String info;
 
-        public TestResponse() {
+        TestResponse() {
         }
 
-        public TestResponse(String info) {
+        TestResponse(String info) {
             this.info = info;
         }
 
@@ -1557,12 +1692,12 @@ public void testSendRandomRequests() throws InterruptedException {
                 .put(TransportService.TRACE_LOG_INCLUDE_SETTING.getKey(), "")
                 .put(TransportService.TRACE_LOG_EXCLUDE_SETTING.getKey(), "NOTHING")
                 .build(),
-            version0);
-        DiscoveryNode nodeC =
-            new DiscoveryNode("TS_C", "TS_C", serviceC.boundAddress().publishAddress(), emptyMap(), emptySet(), version0);
+            version0,
+            null, true);
+        DiscoveryNode nodeC = serviceC.getLocalNode();
         serviceC.acceptIncomingRequests();
 
-        final CountDownLatch latch = new CountDownLatch(5);
+        final CountDownLatch latch = new CountDownLatch(4);
         TransportConnectionListener waitForConnection = new TransportConnectionListener() {
             @Override
             public void onNodeConnected(DiscoveryNode node) {
@@ -1582,12 +1717,11 @@ public void onNodeDisconnected(DiscoveryNode node) {
         serviceC.connectToNode(nodeB);
         serviceA.connectToNode(nodeC);
         serviceB.connectToNode(nodeC);
-        serviceC.connectToNode(nodeC);
 
         latch.await();
         serviceA.removeConnectionListener(waitForConnection);
         serviceB.removeConnectionListener(waitForConnection);
-        serviceB.removeConnectionListener(waitForConnection);
+        serviceC.removeConnectionListener(waitForConnection);
 
 
         Map<TransportService, DiscoveryNode> toNodeMap = new HashMap<>();
@@ -1671,7 +1805,7 @@ class TestResponseHandler implements TransportResponseHandler<TestResponse> {
 
             private final int id;
 
-            public TestResponseHandler(int id) {
+            TestResponseHandler(int id) {
                 this.id = id;
             }
 
@@ -1719,6 +1853,426 @@ public String executor() {
         }
         logger.debug("DONE");
         serviceC.close();
+        // when we close C here we have to disconnect the service otherwise assertions mit trip with pending connections in tearDown
+        // since the disconnect will then happen concurrently and that might confuse the assertions since we disconnect due to a
+        // connection reset by peer or other exceptions depending on the implementation
+        serviceB.disconnectFromNode(nodeC);
+        serviceA.disconnectFromNode(nodeC);
+    }
+
+    public void testRegisterHandlerTwice() {
+        serviceB.registerRequestHandler("action1", TestRequest::new, randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC),
+            (request, message) -> {throw new AssertionError("boom");});
+        expectThrows(IllegalArgumentException.class, () ->
+            serviceB.registerRequestHandler("action1", TestRequest::new, randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC),
+                (request, message) -> {throw new AssertionError("boom");})
+        );
+
+        serviceA.registerRequestHandler("action1", TestRequest::new, randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC),
+            (request, message) -> {throw new AssertionError("boom");});
+    }
+
+    public void testTimeoutPerConnection() throws IOException {
+        assumeTrue("Works only on BSD network stacks and apparently windows",
+            Constants.MAC_OS_X || Constants.FREE_BSD || Constants.WINDOWS);
+        assumeFalse("This test does not work for the local transport as it uses different address types",
+            serviceA.original() instanceof LocalTransport || serviceB.original() instanceof LocalTransport);
+        try (ServerSocket socket = new ServerSocket()) {
+            // note - this test uses backlog=1 which is implementation specific ie. it might not work on some TCP/IP stacks
+            // on linux (at least newer ones) the listen(addr, backlog=1) should just ignore new connections if the queue is full which
+            // means that once we received an ACK from the client we just drop the packet on the floor (which is what we want) and we run
+            // into a connection timeout quickly. Yet other implementations can for instance can terminate the connection within the 3 way
+            // handshake which I haven't tested yet.
+            socket.bind(new InetSocketAddress(InetAddress.getLocalHost(), 0), 1);
+            socket.setReuseAddress(true);
+            DiscoveryNode first = new DiscoveryNode("TEST", new InetSocketTransportAddress(socket.getInetAddress(),
+                socket.getLocalPort()), emptyMap(),
+                emptySet(), version0);
+            DiscoveryNode second = new DiscoveryNode("TEST", new InetSocketTransportAddress(socket.getInetAddress(),
+                socket.getLocalPort()), emptyMap(),
+                emptySet(), version0);
+            ConnectionProfile.Builder builder = new ConnectionProfile.Builder();
+            builder.addConnections(1,
+                TransportRequestOptions.Type.BULK,
+                TransportRequestOptions.Type.PING,
+                TransportRequestOptions.Type.RECOVERY,
+                TransportRequestOptions.Type.REG,
+                TransportRequestOptions.Type.STATE);
+            // connection with one connection and a large timeout -- should consume the one spot in the backlog queue
+            try (TransportService service = buildService("TS_TPC", Version.CURRENT, null,
+                Settings.EMPTY, true, false)) {
+                IOUtils.close(service.openConnection(first, builder.build()));
+                builder.setConnectTimeout(TimeValue.timeValueMillis(1));
+                final ConnectionProfile profile = builder.build();
+                // now with the 1ms timeout we got and test that is it's applied
+                long startTime = System.nanoTime();
+                ConnectTransportException ex = expectThrows(ConnectTransportException.class, () -> service.openConnection(second, profile));
+                final long now = System.nanoTime();
+                final long timeTaken = TimeValue.nsecToMSec(now - startTime);
+                assertTrue("test didn't timeout quick enough, time taken: [" + timeTaken + "]",
+                    timeTaken < TimeValue.timeValueSeconds(5).millis());
+                assertEquals(ex.getMessage(), "[][" + second.getAddress() + "] connect_timeout[1ms]");
+            }
+        }
+    }
+
+    public void testHandshakeWithIncompatVersion() {
+        assumeTrue("only tcp transport has a handshake method", serviceA.getOriginalTransport() instanceof TcpTransport);
+        NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(Collections.emptyList());
+        try (MockTcpTransport transport = new MockTcpTransport(Settings.EMPTY, threadPool, BigArrays.NON_RECYCLING_INSTANCE,
+            new NoneCircuitBreakerService(), namedWriteableRegistry, new NetworkService(Settings.EMPTY, Collections.emptyList()),
+            Version.fromString("2.0.0"))) {
+            transport.transportServiceAdapter(serviceA.new Adapter());
+            transport.start();
+            DiscoveryNode node =
+                new DiscoveryNode("TS_TPC", "TS_TPC", transport.boundAddress().publishAddress(), emptyMap(), emptySet(), version0);
+            ConnectionProfile.Builder builder = new ConnectionProfile.Builder();
+            builder.addConnections(1,
+                TransportRequestOptions.Type.BULK,
+                TransportRequestOptions.Type.PING,
+                TransportRequestOptions.Type.RECOVERY,
+                TransportRequestOptions.Type.REG,
+                TransportRequestOptions.Type.STATE);
+            expectThrows(ConnectTransportException.class, () -> serviceA.openConnection(node, builder.build()));
+        }
+    }
+
+    public void testHandshakeUpdatesVersion() throws IOException {
+        assumeTrue("only tcp transport has a handshake method", serviceA.getOriginalTransport() instanceof TcpTransport);
+        NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(Collections.emptyList());
+        Version version = VersionUtils.randomVersionBetween(random(), Version.CURRENT.minimumCompatibilityVersion(), Version.CURRENT);
+        try (MockTcpTransport transport = new MockTcpTransport(Settings.EMPTY, threadPool, BigArrays.NON_RECYCLING_INSTANCE,
+            new NoneCircuitBreakerService(), namedWriteableRegistry, new NetworkService(Settings.EMPTY, Collections.emptyList()),version)) {
+            transport.transportServiceAdapter(serviceA.new Adapter());
+            transport.start();
+            DiscoveryNode node =
+                new DiscoveryNode("TS_TPC", "TS_TPC", transport.boundAddress().publishAddress(), emptyMap(), emptySet(),
+                    Version.fromString("2.0.0"));
+            ConnectionProfile.Builder builder = new ConnectionProfile.Builder();
+            builder.addConnections(1,
+                TransportRequestOptions.Type.BULK,
+                TransportRequestOptions.Type.PING,
+                TransportRequestOptions.Type.RECOVERY,
+                TransportRequestOptions.Type.REG,
+                TransportRequestOptions.Type.STATE);
+            try (Transport.Connection connection = serviceA.openConnection(node, builder.build())) {
+                assertEquals(connection.getVersion(), version);
+            }
+        }
+    }
+
+
+    public void testTcpHandshake() throws IOException, InterruptedException {
+        assumeTrue("only tcp transport has a handshake method", serviceA.getOriginalTransport() instanceof TcpTransport);
+        TcpTransport originalTransport = (TcpTransport) serviceA.getOriginalTransport();
+        NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(Collections.emptyList());
+
+        try (MockTcpTransport transport = new MockTcpTransport(Settings.EMPTY, threadPool, BigArrays.NON_RECYCLING_INSTANCE,
+            new NoneCircuitBreakerService(), namedWriteableRegistry,  new NetworkService(Settings.EMPTY, Collections.emptyList())){
+            @Override
+            protected String handleRequest(MockChannel mockChannel, String profileName, StreamInput stream, long requestId,
+                                           int messageLengthBytes, Version version, InetSocketAddress remoteAddress, byte status)
+                throws IOException {
+                return super.handleRequest(mockChannel, profileName, stream, requestId, messageLengthBytes, version, remoteAddress,
+                    (byte)(status & ~(1<<3))); // we flip the isHandshake bit back and act like the handler is not found
+            }
+        }) {
+            transport.transportServiceAdapter(serviceA.new Adapter());
+            transport.start();
+            // this acts like a node that doesn't have support for handshakes
+            DiscoveryNode node =
+                new DiscoveryNode("TS_TPC", "TS_TPC", transport.boundAddress().publishAddress(), emptyMap(), emptySet(), version0);
+            try(TcpTransport.NodeChannels connection = originalTransport.openConnection(node, null)) {
+                Version version = originalTransport.executeHandshake(node, connection.channel(TransportRequestOptions.Type.PING),
+                    TimeValue.timeValueSeconds(10));
+                assertNull(version);
+            }
+        }
+
+        try (TransportService service = buildService("TS_TPC", Version.CURRENT, null);
+             TcpTransport.NodeChannels connection = originalTransport.openConnection(
+                 new DiscoveryNode("TS_TPC", "TS_TPC", service.boundAddress().publishAddress(), emptyMap(), emptySet(), version0),
+                 null
+             ) ) {
+            Version version = originalTransport.executeHandshake(connection.getNode(),
+                connection.channel(TransportRequestOptions.Type.PING), TimeValue.timeValueSeconds(10));
+            assertEquals(version, Version.CURRENT);
+        }
+    }
+
+    public void testTcpHandshakeTimeout() throws IOException {
+        assumeTrue("only tcp transport does a handshake", serviceA.getOriginalTransport() instanceof TcpTransport);
+        try (ServerSocket socket = new ServerSocket()) {
+            socket.bind(new InetSocketAddress(InetAddress.getLocalHost(), 0), 1);
+            socket.setReuseAddress(true);
+            DiscoveryNode dummy = new DiscoveryNode("TEST", new InetSocketTransportAddress(socket.getInetAddress(),
+                socket.getLocalPort()), emptyMap(),
+                emptySet(), version0);
+            ConnectionProfile.Builder builder = new ConnectionProfile.Builder();
+            builder.addConnections(1,
+                TransportRequestOptions.Type.BULK,
+                TransportRequestOptions.Type.PING,
+                TransportRequestOptions.Type.RECOVERY,
+                TransportRequestOptions.Type.REG,
+                TransportRequestOptions.Type.STATE);
+            builder.setHandshakeTimeout(TimeValue.timeValueMillis(1));
+            ConnectTransportException ex = expectThrows(ConnectTransportException.class,
+                () -> serviceA.connectToNode(dummy, builder.build()));
+            assertEquals("[][" + dummy.getAddress() +"] handshake_timeout[1ms]", ex.getMessage());
+        }
+    }
 
+    public void testTcpHandshakeConnectionReset() throws IOException, InterruptedException {
+        assumeTrue("only tcp transport does a handshake", serviceA.getOriginalTransport() instanceof TcpTransport);
+        try (ServerSocket socket = new ServerSocket()) {
+            socket.bind(new InetSocketAddress(InetAddress.getLocalHost(), 0), 1);
+            socket.setReuseAddress(true);
+            DiscoveryNode dummy = new DiscoveryNode("TEST", new InetSocketTransportAddress(socket.getInetAddress(),
+                socket.getLocalPort()), emptyMap(),
+                emptySet(), version0);
+            Thread t = new Thread() {
+                @Override
+                public void run() {
+                    try (Socket accept = socket.accept()) {
+                        if (randomBoolean()) { // sometimes wait until the other side sends the message
+                            accept.getInputStream().read();
+                        }
+                    } catch (IOException e) {
+                        throw new UncheckedIOException(e);
+                    }
+                }
+            };
+            t.start();
+            ConnectionProfile.Builder builder = new ConnectionProfile.Builder();
+            builder.addConnections(1,
+                TransportRequestOptions.Type.BULK,
+                TransportRequestOptions.Type.PING,
+                TransportRequestOptions.Type.RECOVERY,
+                TransportRequestOptions.Type.REG,
+                TransportRequestOptions.Type.STATE);
+            builder.setHandshakeTimeout(TimeValue.timeValueHours(1));
+            ConnectTransportException ex = expectThrows(ConnectTransportException.class,
+                () -> serviceA.connectToNode(dummy, builder.build()));
+            assertEquals(ex.getMessage(), "[][" + dummy.getAddress() +"] general node connection failure");
+            assertThat(ex.getCause().getMessage(), startsWith("handshake failed"));
+            t.join();
+        }
     }
+
+    public void testResponseHeadersArePreserved() throws InterruptedException {
+        List<String> executors = new ArrayList<>(ThreadPool.THREAD_POOL_TYPES.keySet());
+        CollectionUtil.timSort(executors); // makes sure it's reproducible
+        serviceA.registerRequestHandler("action", TestRequest::new, ThreadPool.Names.SAME,
+            (request, channel) -> {
+
+                threadPool.getThreadContext().putTransient("boom", new Object());
+                threadPool.getThreadContext().addResponseHeader("foo.bar", "baz");
+                if ("fail".equals(request.info)) {
+                    throw new RuntimeException("boom");
+                } else {
+                    channel.sendResponse(TransportResponse.Empty.INSTANCE);
+                }
+            });
+
+        CountDownLatch latch = new CountDownLatch(2);
+
+        TransportResponseHandler<TransportResponse> transportResponseHandler = new TransportResponseHandler<TransportResponse>() {
+            @Override
+            public TransportResponse newInstance() {
+                return TransportResponse.Empty.INSTANCE;
+            }
+
+            @Override
+            public void handleResponse(TransportResponse response) {
+                try {
+                    assertSame(response, TransportResponse.Empty.INSTANCE);
+                    assertTrue(threadPool.getThreadContext().getResponseHeaders().containsKey("foo.bar"));
+                    assertEquals(1, threadPool.getThreadContext().getResponseHeaders().get("foo.bar").size());
+                    assertEquals("baz", threadPool.getThreadContext().getResponseHeaders().get("foo.bar").get(0));
+                    assertNull(threadPool.getThreadContext().getTransient("boom"));
+                } finally {
+                    latch.countDown();
+                }
+
+            }
+
+            @Override
+            public void handleException(TransportException exp) {
+                try {
+                    assertTrue(threadPool.getThreadContext().getResponseHeaders().containsKey("foo.bar"));
+                    assertEquals(1, threadPool.getThreadContext().getResponseHeaders().get("foo.bar").size());
+                    assertEquals("baz", threadPool.getThreadContext().getResponseHeaders().get("foo.bar").get(0));
+                    assertNull(threadPool.getThreadContext().getTransient("boom"));
+                } finally {
+                    latch.countDown();
+                }
+            }
+
+            @Override
+            public String executor() {
+                return randomFrom(executors);
+            }
+        };
+
+        serviceB.sendRequest(nodeA, "action",  new TestRequest(randomFrom("fail", "pass")), transportResponseHandler);
+        serviceA.sendRequest(nodeA, "action",  new TestRequest(randomFrom("fail", "pass")), transportResponseHandler);
+        latch.await();
+    }
+
+    public void testHandlerIsInvokedOnConnectionClose() throws IOException, InterruptedException {
+        List<String> executors = new ArrayList<>(ThreadPool.THREAD_POOL_TYPES.keySet());
+        CollectionUtil.timSort(executors); // makes sure it's reproducible
+        TransportService serviceC = build(Settings.builder().put("name", "TS_TEST").build(), version0, null, true);
+        serviceC.registerRequestHandler("action", TestRequest::new, ThreadPool.Names.SAME,
+            (request, channel) -> {
+                // do nothing
+            });
+        serviceC.start();
+        serviceC.acceptIncomingRequests();
+        CountDownLatch latch = new CountDownLatch(1);
+        TransportResponseHandler<TransportResponse> transportResponseHandler = new TransportResponseHandler<TransportResponse>() {
+            @Override
+            public TransportResponse newInstance() {
+                return TransportResponse.Empty.INSTANCE;
+            }
+
+            @Override
+            public void handleResponse(TransportResponse response) {
+                try {
+                    fail("no response expected");
+                } finally {
+                    latch.countDown();
+                }
+            }
+
+            @Override
+            public void handleException(TransportException exp) {
+                try {
+                    if (exp instanceof SendRequestTransportException) {
+                        assertTrue(exp.getCause().getClass().toString(), exp.getCause() instanceof NodeNotConnectedException);
+                    } else {
+                        // here the concurrent disconnect was faster and invoked the listener first
+                        assertTrue(exp.getClass().toString(), exp instanceof NodeDisconnectedException);
+                    }
+                } finally {
+                    latch.countDown();
+                }
+            }
+
+            @Override
+            public String executor() {
+                return randomFrom(executors);
+            }
+        };
+        ConnectionProfile.Builder builder = new ConnectionProfile.Builder();
+        builder.addConnections(1,
+            TransportRequestOptions.Type.BULK,
+            TransportRequestOptions.Type.PING,
+            TransportRequestOptions.Type.RECOVERY,
+            TransportRequestOptions.Type.REG,
+            TransportRequestOptions.Type.STATE);
+        try (Transport.Connection connection = serviceB.openConnection(serviceC.getLocalNode(), builder.build())) {
+            serviceC.close();
+            serviceB.sendRequest(connection, "action", new TestRequest("boom"), TransportRequestOptions.EMPTY,
+                transportResponseHandler);
+        }
+        latch.await();
+    }
+
+    public void testConcurrentDisconnectOnNonPublishedConnection() throws IOException, InterruptedException {
+        MockTransportService serviceC = build(Settings.builder().put("name", "TS_TEST").build(), version0, null, true);
+        CountDownLatch receivedLatch = new CountDownLatch(1);
+        CountDownLatch sendResponseLatch = new CountDownLatch(1);
+        serviceC.registerRequestHandler("action", TestRequest::new, ThreadPool.Names.SAME,
+            (request, channel) -> {
+                // don't block on a network thread here
+                threadPool.generic().execute(new AbstractRunnable() {
+                    @Override
+                    public void onFailure(Exception e) {
+                        try {
+                            channel.sendResponse(e);
+                        } catch (IOException e1) {
+                            throw new UncheckedIOException(e1);
+                        }
+                    }
+
+                    @Override
+                    protected void doRun() throws Exception {
+                        receivedLatch.countDown();
+                        sendResponseLatch.await();
+                        channel.sendResponse(TransportResponse.Empty.INSTANCE);
+                    }
+                });
+            });
+        serviceC.start();
+        serviceC.acceptIncomingRequests();
+        CountDownLatch responseLatch = new CountDownLatch(1);
+        TransportResponseHandler<TransportResponse> transportResponseHandler = new TransportResponseHandler<TransportResponse>() {
+            @Override
+            public TransportResponse newInstance() {
+                return TransportResponse.Empty.INSTANCE;
+            }
+
+            @Override
+            public void handleResponse(TransportResponse response) {
+                responseLatch.countDown();
+            }
+
+            @Override
+            public void handleException(TransportException exp) {
+                responseLatch.countDown();
+            }
+
+            @Override
+            public String executor() {
+                return ThreadPool.Names.SAME;
+            }
+        };
+
+        ConnectionProfile.Builder builder = new ConnectionProfile.Builder();
+        builder.addConnections(1,
+            TransportRequestOptions.Type.BULK,
+            TransportRequestOptions.Type.PING,
+            TransportRequestOptions.Type.RECOVERY,
+            TransportRequestOptions.Type.REG,
+            TransportRequestOptions.Type.STATE);
+
+        try (Transport.Connection connection = serviceB.openConnection(serviceC.getLocalNode(), builder.build())) {
+            serviceB.sendRequest(connection, "action", new TestRequest("hello world"), TransportRequestOptions.EMPTY,
+                transportResponseHandler);
+            receivedLatch.await();
+            serviceC.close();
+            sendResponseLatch.countDown();
+            responseLatch.await();
+        }
+    }
+
+    public void testChannelCloseWhileConnecting() throws IOException {
+        try (MockTransportService service = build(Settings.builder().put("name", "close").build(), version0, null, true)) {
+            service.setExecutorName(ThreadPool.Names.SAME); // make sure stuff is executed in a blocking fashion
+            service.addConnectionListener(new TransportConnectionListener() {
+                @Override
+                public void onConnectionOpened(final Transport.Connection connection) {
+                    try {
+                        closeConnectionChannel(service.getOriginalTransport(), connection);
+                    } catch (final IOException e) {
+                        throw new AssertionError(e);
+                    }
+                }
+            });
+            final ConnectionProfile.Builder builder = new ConnectionProfile.Builder();
+            builder.addConnections(1,
+                    TransportRequestOptions.Type.BULK,
+                    TransportRequestOptions.Type.PING,
+                    TransportRequestOptions.Type.RECOVERY,
+                    TransportRequestOptions.Type.REG,
+                    TransportRequestOptions.Type.STATE);
+            final ConnectTransportException e =
+                    expectThrows(ConnectTransportException.class, () -> service.openConnection(nodeA, builder.build()));
+            assertThat(e, hasToString(containsString(("a channel closed while connecting"))));
+        }
+    }
+
+    protected abstract void closeConnectionChannel(Transport transport, Transport.Connection connection) throws IOException;
+
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/transport/MockTcpTransport.java b/test/framework/src/main/java/org/elasticsearch/transport/MockTcpTransport.java
index da7fcc5341012..87e455119995e 100644
--- a/test/framework/src/main/java/org/elasticsearch/transport/MockTcpTransport.java
+++ b/test/framework/src/main/java/org/elasticsearch/transport/MockTcpTransport.java
@@ -20,9 +20,9 @@
 
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.InputStreamStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -31,6 +31,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.InetSocketTransportAddress;
 import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.CancellableThreads;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
@@ -47,14 +48,17 @@
 import java.net.ServerSocket;
 import java.net.Socket;
 import java.net.SocketException;
+import java.net.SocketTimeoutException;
+import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.Executor;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
-import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Consumer;
@@ -66,10 +70,28 @@
  */
 public class MockTcpTransport extends TcpTransport<MockTcpTransport.MockChannel> {
 
+    /**
+     * A pre-built light connection profile that shares a single connection across all
+     * types.
+     */
+    public static final ConnectionProfile LIGHT_PROFILE;
+
+    private final Set<MockChannel> openChannels = new HashSet<>();
+
+    static {
+        ConnectionProfile.Builder builder = new ConnectionProfile.Builder();
+        builder.addConnections(1,
+            TransportRequestOptions.Type.BULK,
+            TransportRequestOptions.Type.PING,
+            TransportRequestOptions.Type.RECOVERY,
+            TransportRequestOptions.Type.REG,
+            TransportRequestOptions.Type.STATE);
+        LIGHT_PROFILE = builder.build();
+    }
+
     private final ExecutorService executor;
     private final Version mockVersion;
 
-    @Inject
     public MockTcpTransport(Settings settings, ThreadPool threadPool, BigArrays bigArrays,
                             CircuitBreakerService circuitBreakerService, NamedWriteableRegistry namedWriteableRegistry,
                             NetworkService networkService) {
@@ -96,9 +118,9 @@ protected InetSocketAddress getLocalAddress(MockChannel mockChannel) {
     protected MockChannel bind(final String name, InetSocketAddress address) throws IOException {
         ServerSocket socket = new ServerSocket();
         socket.bind(address);
-        socket.setReuseAddress(TCP_REUSE_ADDRESS.get(settings()));
+        socket.setReuseAddress(TCP_REUSE_ADDRESS.get(settings));
         ByteSizeValue tcpReceiveBufferSize = TCP_RECEIVE_BUFFER_SIZE.get(settings);
-        if (tcpReceiveBufferSize.bytes() > 0) {
+        if (tcpReceiveBufferSize.getBytes() > 0) {
             socket.setReceiveBufferSize(tcpReceiveBufferSize.bytesAsInt());
         }
         MockChannel serverMockChannel = new MockChannel(socket, name);
@@ -106,11 +128,7 @@ protected MockChannel bind(final String name, InetSocketAddress address) throws
         executor.execute(new AbstractRunnable() {
             @Override
             public void onFailure(Exception e) {
-                try {
-                    onException(serverMockChannel, e);
-                } catch (IOException ex) {
-                    logger.warn("failed on handling exception", ex);
-                }
+                onException(serverMockChannel, e);
             }
 
             @Override
@@ -158,43 +176,26 @@ private void readMessage(MockChannel mockChannel, StreamInput input) throws IOEx
     }
 
     @Override
-    protected NodeChannels connectToChannelsLight(DiscoveryNode node) throws IOException {
-        return connectToChannels(node);
-    }
-
-    @Override
-    protected NodeChannels connectToChannels(DiscoveryNode node) throws IOException {
-        final NodeChannels nodeChannels = new NodeChannels(new MockChannel[1],
-            new MockChannel[1],
-            new MockChannel[1],
-            new MockChannel[1],
-            new MockChannel[1]);
+    protected NodeChannels connectToChannels(DiscoveryNode node,
+                                             ConnectionProfile profile,
+                                             Consumer<MockChannel> onChannelClose) throws IOException {
+        final MockChannel[] mockChannels = new MockChannel[1];
+        final NodeChannels nodeChannels = new NodeChannels(node, mockChannels, LIGHT_PROFILE); // we always use light here
         boolean success = false;
         final Socket socket = new Socket();
         try {
-            Consumer<MockChannel> onClose = (channel) -> {
-                final NodeChannels connected = connectedNodes.get(node);
-                if (connected != null && connected.hasChannel(channel)) {
-                    try {
-                        executor.execute(() -> {
-                            disconnectFromNode(node, channel, "channel closed event");
-                        });
-                    } catch (RejectedExecutionException ex) {
-                        logger.debug("failed to run disconnectFromNode - node is shutting down");
-                    }
-                }
-            };
             InetSocketAddress address = ((InetSocketTransportAddress) node.getAddress()).address();
             // we just use a single connections
             configureSocket(socket);
-            socket.connect(address, (int) TCP_CONNECT_TIMEOUT.get(settings).millis());
-            MockChannel channel = new MockChannel(socket, address, "none", onClose);
-            channel.loopRead(executor);
-            for (MockChannel[] channels : nodeChannels.getChannelArrays()) {
-                for (int i = 0; i < channels.length; i++) {
-                    channels[i] = channel;
-                }
+            final TimeValue connectTimeout = profile.getConnectTimeout();
+            try {
+                socket.connect(address, Math.toIntExact(connectTimeout.millis()));
+            } catch (SocketTimeoutException ex) {
+                throw new ConnectTransportException(node, "connect_timeout[" + connectTimeout + "]", ex);
             }
+            MockChannel channel = new MockChannel(socket, address, "none", onChannelClose);
+            channel.loopRead(executor);
+            mockChannels[0] = channel;
             success = true;
         } finally {
             if (success == false) {
@@ -210,14 +211,14 @@ protected NodeChannels connectToChannels(DiscoveryNode node) throws IOException
     private void configureSocket(Socket socket) throws SocketException {
         socket.setTcpNoDelay(TCP_NO_DELAY.get(settings));
         ByteSizeValue tcpSendBufferSize = TCP_SEND_BUFFER_SIZE.get(settings);
-        if (tcpSendBufferSize.bytes() > 0) {
+        if (tcpSendBufferSize.getBytes() > 0) {
             socket.setSendBufferSize(tcpSendBufferSize.bytesAsInt());
         }
         ByteSizeValue tcpReceiveBufferSize = TCP_RECEIVE_BUFFER_SIZE.get(settings);
-        if (tcpReceiveBufferSize.bytes() > 0) {
+        if (tcpReceiveBufferSize.getBytes() > 0) {
             socket.setReceiveBufferSize(tcpReceiveBufferSize.bytesAsInt());
         }
-        socket.setReuseAddress(TCP_REUSE_ADDRESS.get(settings()));
+        socket.setReuseAddress(TCP_REUSE_ADDRESS.get(settings));
     }
 
     @Override
@@ -226,23 +227,23 @@ protected boolean isOpen(MockChannel mockChannel) {
     }
 
     @Override
-    protected void sendMessage(MockChannel mockChannel, BytesReference reference, Runnable sendListener, boolean close) throws IOException {
-        synchronized (mockChannel) {
-            final Socket socket = mockChannel.activeChannel;
-            OutputStream outputStream = new BufferedOutputStream(socket.getOutputStream());
-            reference.writeTo(outputStream);
-            outputStream.flush();
-        }
-        if (sendListener != null) {
-            sendListener.run();
-        }
-        if (close) {
-            IOUtils.closeWhileHandlingException(mockChannel);
+    protected void sendMessage(MockChannel mockChannel, BytesReference reference, ActionListener<MockChannel> listener) {
+        try {
+            synchronized (mockChannel) {
+                final Socket socket = mockChannel.activeChannel;
+                OutputStream outputStream = new BufferedOutputStream(socket.getOutputStream());
+                reference.writeTo(outputStream);
+                outputStream.flush();
+            }
+            listener.onResponse(mockChannel);
+        } catch (IOException e) {
+            listener.onFailure(e);
+            onException(mockChannel, e);
         }
     }
 
     @Override
-    protected void closeChannels(List<MockChannel> channel) throws IOException {
+    protected void closeChannels(List<MockChannel> channel, boolean blocking) throws IOException {
         IOUtils.close(channel);
     }
 
@@ -255,7 +256,7 @@ public final class MockChannel implements Closeable {
         private final AtomicBoolean isOpen = new AtomicBoolean(true);
         private final InetSocketAddress localAddress;
         private final ServerSocket serverSocket;
-        private final ConcurrentHashMap<MockChannel, Boolean> workerChannels = new ConcurrentHashMap<>();
+        private final Set<MockChannel> workerChannels = Collections.newSetFromMap(new ConcurrentHashMap<>());
         private final Socket activeChannel;
         private final String profile;
         private final CancellableThreads cancellableThreads = new CancellableThreads();
@@ -275,6 +276,9 @@ public MockChannel(Socket socket, InetSocketAddress localAddress, String profile
             this.serverSocket = null;
             this.profile = profile;
             this.onClose = () -> onClose.accept(this);
+            synchronized (openChannels) {
+                openChannels.add(this);
+            }
         }
 
         /**
@@ -289,6 +293,9 @@ public MockChannel(ServerSocket serverSocket, String profile) {
             this.profile = profile;
             this.activeChannel = null;
             this.onClose = null;
+            synchronized (openChannels) {
+                openChannels.add(this);
+            }
         }
 
         public void accept(Executor executor) throws IOException {
@@ -297,11 +304,12 @@ public void accept(Executor executor) throws IOException {
                 MockChannel incomingChannel = null;
                 try {
                     configureSocket(incomingSocket);
-                    incomingChannel = new MockChannel(incomingSocket, localAddress, profile, workerChannels::remove);
-                    //establish a happens-before edge between closing and accepting a new connection
                     synchronized (this) {
                         if (isOpen.get()) {
-                            workerChannels.put(incomingChannel, Boolean.TRUE);
+                            incomingChannel = new MockChannel(incomingSocket, localAddress, profile, workerChannels::remove);
+                            //establish a happens-before edge between closing and accepting a new connection
+                            workerChannels.add(incomingChannel);
+
                             // this spawns a new thread immediately, so OK under lock
                             incomingChannel.loopRead(executor);
                             // the channel is properly registered and will be cleared by the close code.
@@ -324,8 +332,9 @@ public void onFailure(Exception e) {
                     if (isOpen.get()) {
                         try {
                             onException(MockChannel.this, e);
-                        } catch (IOException ex) {
+                        } catch (Exception ex) {
                             logger.warn("failed on handling exception", ex);
+                            IOUtils.closeWhileHandlingException(MockChannel.this); // pure paranoia
                         }
                     }
                 }
@@ -342,15 +351,33 @@ protected void doRun() throws Exception {
         }
 
         @Override
-        public void close() throws IOException {
+        public synchronized void close() throws IOException {
+            // establish a happens-before edge between closing and accepting a new connection
+            // we have to sync this entire block to ensure that our openChannels checks work correctly.
+            // The close block below will close all worker channels but if one of the worker channels runs into an exception
+            // for instance due to a disconnect the handling of this exception might be executed concurrently.
+            // now if we are in-turn concurrently call close we might not wait for the actual close to happen and that will, down the road
+            // make the assertion trip that not all channels are closed.
             if (isOpen.compareAndSet(true, false)) {
-                //establish a happens-before edge between closing and accepting a new connection
-                synchronized (this) {
-                    IOUtils.close(serverSocket, activeChannel, () -> IOUtils.close(workerChannels.keySet()),
-                        () -> cancellableThreads.cancel("channel closed"), onClose);
+                final boolean removedChannel;
+                synchronized (openChannels) {
+                    removedChannel = openChannels.remove(this);
                 }
+                IOUtils.close(serverSocket, activeChannel, () -> IOUtils.close(workerChannels),
+                    () -> cancellableThreads.cancel("channel closed"), onClose);
+                assert removedChannel: "Channel was not removed or removed twice?";
             }
         }
+
+        @Override
+        public String toString() {
+            return "MockChannel{" +
+                "profile='" + profile + '\'' +
+                ", isOpen=" + isOpen +
+                ", localAddress=" + localAddress +
+                ", isServerSocket=" + (serverSocket != null) +
+                '}';
+        }
     }
 
 
@@ -378,12 +405,14 @@ protected void doStart() {
     @Override
     protected void stopInternal() {
         ThreadPool.terminate(executor, 10, TimeUnit.SECONDS);
+        synchronized (openChannels) {
+            assert openChannels.isEmpty() : "there are still open channels: " + openChannels;
+        }
     }
 
     @Override
     protected Version getCurrentVersion() {
         return mockVersion;
     }
-
 }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/transport/MockTcpTransportPlugin.java b/test/framework/src/main/java/org/elasticsearch/transport/MockTcpTransportPlugin.java
index 3e17cdcb30aae..d48e4bb7f73f6 100644
--- a/test/framework/src/main/java/org/elasticsearch/transport/MockTcpTransportPlugin.java
+++ b/test/framework/src/main/java/org/elasticsearch/transport/MockTcpTransportPlugin.java
@@ -18,14 +18,30 @@
  */
 package org.elasticsearch.transport;
 
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.network.NetworkModule;
+import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.indices.breaker.CircuitBreakerService;
+import org.elasticsearch.plugins.NetworkPlugin;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.threadpool.ThreadPool;
 
-public class MockTcpTransportPlugin extends Plugin {
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Supplier;
+
+public class MockTcpTransportPlugin extends Plugin implements NetworkPlugin {
     public static final String MOCK_TCP_TRANSPORT_NAME = "mock-socket-network";
 
-    public void onModule(NetworkModule module) {
-        module.registerTransport(MOCK_TCP_TRANSPORT_NAME, MockTcpTransport.class);
+    @Override
+    public Map<String, Supplier<Transport>> getTransports(Settings settings, ThreadPool threadPool, BigArrays bigArrays,
+                                                          CircuitBreakerService circuitBreakerService,
+                                                          NamedWriteableRegistry namedWriteableRegistry,
+                                                          NetworkService networkService) {
+        return Collections.singletonMap(MOCK_TCP_TRANSPORT_NAME,
+            () -> new MockTcpTransport(settings, threadPool, bigArrays, circuitBreakerService, namedWriteableRegistry, networkService));
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/transport/MockTransportClient.java b/test/framework/src/main/java/org/elasticsearch/transport/MockTransportClient.java
index 2b0f551dbb748..69a3240c609e0 100644
--- a/test/framework/src/main/java/org/elasticsearch/transport/MockTransportClient.java
+++ b/test/framework/src/main/java/org/elasticsearch/transport/MockTransportClient.java
@@ -31,11 +31,14 @@ public class MockTransportClient extends TransportClient {
 
 
     public MockTransportClient(Settings settings, Class<? extends Plugin>... plugins) {
-        super(settings, DEFAULT_SETTINGS, Arrays.asList(plugins));
+        this(settings, Arrays.asList(plugins), null);
     }
 
     public MockTransportClient(Settings settings, Collection<Class<? extends Plugin>> plugins) {
-        super(settings, DEFAULT_SETTINGS, plugins);
+        this(settings, plugins, null);
     }
 
+    public MockTransportClient(Settings settings, Collection<Class<? extends Plugin>> plugins, HostFailureListener listener) {
+        super(settings, DEFAULT_SETTINGS, plugins, listener);
+    }
 }
diff --git a/test/framework/src/main/resources/log4j2-test.properties b/test/framework/src/main/resources/log4j2-test.properties
index 9cfe3e326aa80..828555ed52e6f 100644
--- a/test/framework/src/main/resources/log4j2-test.properties
+++ b/test/framework/src/main/resources/log4j2-test.properties
@@ -1,9 +1,7 @@
-status = error
-
 appender.console.type = Console
 appender.console.name = console
 appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %m%n
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n
 
-rootLogger.level = info
+rootLogger.level = ${sys:tests.es.logger.level:-info}
 rootLogger.appenderRef.console.ref = console
diff --git a/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java b/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java
index d6cd3eea5acae..2384edd7b8959 100644
--- a/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.search;
 
 import org.apache.lucene.search.Query;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.lucene.search.Queries;
@@ -32,12 +33,17 @@
 import org.elasticsearch.test.TestSearchContext;
 
 public class MockSearchServiceTests extends ESTestCase {
+    public static final IndexMetaData EMPTY_INDEX_METADATA = IndexMetaData.builder("")
+        .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT))
+        .numberOfShards(1).numberOfReplicas(0).build();
+
     public void testAssertNoInFlightContext() {
-        SearchContext s = new TestSearchContext(new QueryShardContext(new IndexSettings(IndexMetaData.PROTO, Settings.EMPTY), null, null,
-                null, null, null, null, null, null, null)) {
+        final long nowInMillis = randomNonNegativeLong();
+        SearchContext s = new TestSearchContext(new QueryShardContext(0, new IndexSettings(EMPTY_INDEX_METADATA, Settings.EMPTY), null,
+            null, null, null, null, xContentRegistry(), null, null, () -> nowInMillis)) {
             @Override
             public SearchShardTarget shardTarget() {
-                return new SearchShardTarget("node", new Index("idx", "ignored"), 0);
+                return new SearchShardTarget("node", new Index("idx", "ignored"), 0, null);
             }
 
             @Override
diff --git a/test/framework/src/test/java/org/elasticsearch/test/XContentTestUtilsTests.java b/test/framework/src/test/java/org/elasticsearch/test/XContentTestUtilsTests.java
new file mode 100644
index 0000000000000..f3b44f2510490
--- /dev/null
+++ b/test/framework/src/test/java/org/elasticsearch/test/XContentTestUtilsTests.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test;
+
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Stack;
+import java.util.function.Predicate;
+
+import static org.elasticsearch.test.XContentTestUtils.insertRandomFields;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.instanceOf;;
+
+public class XContentTestUtilsTests extends ESTestCase {
+
+    public void testGetInsertPaths() throws IOException {
+        XContentBuilder builder = JsonXContent.contentBuilder();
+        builder.startObject();
+        {
+            builder.field("field1", "value");
+            builder.startArray("list1");
+            {
+                builder.value(0);
+                builder.value(1);
+                builder.startObject();
+                builder.endObject();
+                builder.value(3);
+                builder.startObject();
+                builder.endObject();
+            }
+            builder.endArray();
+            builder.startObject("inner1");
+            {
+                builder.field("inner1field1", "value");
+                builder.startObject("inn.er2");
+                {
+                    builder.field("inner2field1", "value");
+                }
+                builder.endObject();
+            }
+            builder.endObject();
+        }
+        builder.endObject();
+
+        try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, builder.bytes(), builder.contentType())) {
+            parser.nextToken();
+            List<String> insertPaths = XContentTestUtils.getInsertPaths(parser, new Stack<>());
+            assertEquals(5, insertPaths.size());
+            assertThat(insertPaths, hasItem(equalTo("")));
+            assertThat(insertPaths, hasItem(equalTo("list1.2")));
+            assertThat(insertPaths, hasItem(equalTo("list1.4")));
+            assertThat(insertPaths, hasItem(equalTo("inner1")));
+            assertThat(insertPaths, hasItem(equalTo("inner1.inn\\.er2")));
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testInsertIntoXContent() throws IOException {
+        XContentBuilder builder = JsonXContent.contentBuilder();
+        builder.startObject();
+        builder.endObject();
+        builder = XContentTestUtils.insertIntoXContent(XContentType.JSON.xContent(), builder.bytes(), Collections.singletonList(""),
+                () -> "inn.er1", () -> new HashMap<>());
+        builder = XContentTestUtils.insertIntoXContent(XContentType.JSON.xContent(), builder.bytes(), Collections.singletonList(""),
+                () -> "field1", () -> "value1");
+        builder = XContentTestUtils.insertIntoXContent(XContentType.JSON.xContent(), builder.bytes(),
+                Collections.singletonList("inn\\.er1"), () -> "inner2", () -> new HashMap<>());
+        builder = XContentTestUtils.insertIntoXContent(XContentType.JSON.xContent(), builder.bytes(),
+                Collections.singletonList("inn\\.er1"), () -> "field2", () -> "value2");
+        try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, builder.bytes(), builder.contentType())) {
+            Map<String, Object> map = parser.map();
+            assertEquals(2, map.size());
+            assertEquals("value1", map.get("field1"));
+            assertThat(map.get("inn.er1"), instanceOf(Map.class));
+            Map<String, Object> innerMap = (Map<String, Object>) map.get("inn.er1");
+            assertEquals(2, innerMap.size());
+            assertEquals("value2", innerMap.get("field2"));
+            assertThat(innerMap.get("inner2"), instanceOf(Map.class));
+            assertEquals(0, ((Map<String, Object>) innerMap.get("inner2")).size());
+        }
+    }
+
+
+    @SuppressWarnings("unchecked")
+    public void testInsertRandomXContent() throws IOException {
+        XContentBuilder builder = XContentFactory.jsonBuilder();
+        builder.startObject();
+        {
+            builder.startObject("foo");
+            {
+                builder.field("bar", 1);
+            }
+            builder.endObject();
+            builder.startObject("foo1");
+            {
+                builder.startObject("foo2");
+                {
+                    builder.field("buzz", 1);
+                }
+                builder.endObject();
+            }
+            builder.endObject();
+            builder.field("foo3", 2);
+            builder.startArray("foo4");
+            {
+                builder.startObject();
+                {
+                    builder.field("foo5", 1);
+                }
+                builder.endObject();
+            }
+            builder.endArray();
+        }
+        builder.endObject();
+
+        Map<String, Object> resultMap;
+
+        try (XContentParser parser = createParser(XContentType.JSON.xContent(),
+                insertRandomFields(builder.contentType(), builder.bytes(), null, random()))) {
+            resultMap = parser.map();
+        }
+        assertEquals(5, resultMap.keySet().size());
+        assertEquals(2, ((Map<String, Object>) resultMap.get("foo")).keySet().size());
+        Map<String, Object> foo1 = (Map<String, Object>) resultMap.get("foo1");
+        assertEquals(2, foo1.keySet().size());
+        assertEquals(2, ((Map<String, Object>) foo1.get("foo2")).keySet().size());
+        List<Object> foo4List = (List<Object>) resultMap.get("foo4");
+        assertEquals(1, foo4List.size());
+        assertEquals(2, ((Map<String, Object>) foo4List.get(0)).keySet().size());
+
+        Predicate<String> pathsToExclude = path -> path.endsWith("foo1");
+        try (XContentParser parser = createParser(XContentType.JSON.xContent(),
+                insertRandomFields(builder.contentType(), builder.bytes(), pathsToExclude, random()))) {
+            resultMap = parser.map();
+        }
+        assertEquals(5, resultMap.keySet().size());
+        assertEquals(2, ((Map<String, Object>) resultMap.get("foo")).keySet().size());
+        foo1 = (Map<String, Object>) resultMap.get("foo1");
+        assertEquals(1, foo1.keySet().size());
+        assertEquals(2, ((Map<String, Object>) foo1.get("foo2")).keySet().size());
+        foo4List = (List<Object>) resultMap.get("foo4");
+        assertEquals(1, foo4List.size());
+        assertEquals(2, ((Map<String, Object>) foo4List.get(0)).keySet().size());
+
+        pathsToExclude = path -> path.contains("foo1");
+        try (XContentParser parser = createParser(XContentType.JSON.xContent(),
+                insertRandomFields(builder.contentType(), builder.bytes(), pathsToExclude, random()))) {
+            resultMap = parser.map();
+        }
+        assertEquals(5, resultMap.keySet().size());
+        assertEquals(2, ((Map<String, Object>) resultMap.get("foo")).keySet().size());
+        foo1 = (Map<String, Object>) resultMap.get("foo1");
+        assertEquals(1, foo1.keySet().size());
+        assertEquals(1, ((Map<String, Object>) foo1.get("foo2")).keySet().size());
+        foo4List = (List<Object>) resultMap.get("foo4");
+        assertEquals(1, foo4List.size());
+        assertEquals(2, ((Map<String, Object>) foo4List.get(0)).keySet().size());
+    }
+}
diff --git a/test/framework/src/test/java/org/elasticsearch/test/disruption/LongGCDisruptionTests.java b/test/framework/src/test/java/org/elasticsearch/test/disruption/LongGCDisruptionTests.java
new file mode 100644
index 0000000000000..43a5e9c133e4d
--- /dev/null
+++ b/test/framework/src/test/java/org/elasticsearch/test/disruption/LongGCDisruptionTests.java
@@ -0,0 +1,259 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.test.disruption;
+
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.test.ESTestCase;
+
+import java.lang.management.ThreadInfo;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.regex.Pattern;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+
+public class LongGCDisruptionTests extends ESTestCase {
+
+    static class LockedExecutor {
+        ReentrantLock lock = new ReentrantLock();
+
+        public void executeLocked(Runnable r) {
+            lock.lock();
+            try {
+                r.run();
+            } finally {
+                lock.unlock();
+            }
+        }
+    }
+
+    public void testBlockingTimeout() throws Exception {
+        final String nodeName = "test_node";
+        LongGCDisruption disruption = new LongGCDisruption(random(), nodeName) {
+            @Override
+            protected Pattern[] getUnsafeClasses() {
+                return new Pattern[]{
+                    Pattern.compile(LockedExecutor.class.getSimpleName())
+                };
+            }
+
+            @Override
+            protected long getSuspendingTimeoutInMillis() {
+                return 100;
+            }
+        };
+        final AtomicBoolean stop = new AtomicBoolean();
+        final CountDownLatch underLock = new CountDownLatch(1);
+        final CountDownLatch pauseUnderLock = new CountDownLatch(1);
+        final LockedExecutor lockedExecutor = new LockedExecutor();
+        final AtomicLong ops = new AtomicLong();
+        final Thread[] threads = new Thread[10];
+        try {
+            for (int i = 0; i < 10; i++) {
+                // at least one locked and one none lock thread
+                final boolean lockedExec = (i < 9 && randomBoolean()) || i == 0;
+                threads[i] = new Thread(() -> {
+                    while (stop.get() == false) {
+                        if (lockedExec) {
+                            lockedExecutor.executeLocked(() -> {
+                                try {
+                                    underLock.countDown();
+                                    ops.incrementAndGet();
+                                    pauseUnderLock.await();
+                                } catch (InterruptedException e) {
+
+                                }
+                            });
+                        } else {
+                            ops.incrementAndGet();
+                        }
+                    }
+                });
+                threads[i].setName("[" + nodeName + "][" + i + "]");
+                threads[i].start();
+            }
+            // make sure some threads are under lock
+            underLock.await();
+            RuntimeException e = expectThrows(RuntimeException.class, disruption::startDisrupting);
+            assertThat(e.getMessage(), containsString("suspending node threads took too long"));
+        } finally {
+            stop.set(true);
+            pauseUnderLock.countDown();
+            for (final Thread thread : threads) {
+                thread.join();
+            }
+        }
+    }
+
+    /**
+     * Checks that a GC disruption never blocks threads while they are doing something "unsafe"
+     * but does keep retrying until all threads can be safely paused
+     */
+    public void testNotBlockingUnsafeStackTraces() throws Exception {
+        final String nodeName = "test_node";
+        LongGCDisruption disruption = new LongGCDisruption(random(), nodeName) {
+            @Override
+            protected Pattern[] getUnsafeClasses() {
+                return new Pattern[]{
+                    Pattern.compile(LockedExecutor.class.getSimpleName())
+                };
+            }
+        };
+        final AtomicBoolean stop = new AtomicBoolean();
+        final LockedExecutor lockedExecutor = new LockedExecutor();
+        final AtomicLong ops = new AtomicLong();
+        final Thread[] threads = new Thread[5];
+        try {
+            for (int i = 0; i < threads.length; i++) {
+                threads[i] = new Thread(() -> {
+                    for (int iter = 0; stop.get() == false; iter++) {
+                        if (iter % 2 == 0) {
+                            lockedExecutor.executeLocked(() -> {
+                                Thread.yield(); // give some chance to catch this stack trace
+                                ops.incrementAndGet();
+                            });
+                        } else {
+                            Thread.yield(); // give some chance to catch this stack trace
+                            ops.incrementAndGet();
+                        }
+                    }
+                });
+                threads[i].setName("[" + nodeName + "][" + i + "]");
+                threads[i].start();
+            }
+            // make sure some threads are under lock
+            disruption.startDisrupting();
+            long first = ops.get();
+            assertThat(lockedExecutor.lock.isLocked(), equalTo(false)); // no threads should own the lock
+            Thread.sleep(100);
+            assertThat(ops.get(), equalTo(first));
+            disruption.stopDisrupting();
+            assertBusy(() -> assertThat(ops.get(), greaterThan(first)));
+        } finally {
+            stop.set(true);
+            for (final Thread thread : threads) {
+                thread.join();
+            }
+        }
+    }
+
+    public void testBlockDetection() throws Exception {
+        final String disruptedNodeName = "disrupted_node";
+        final String blockedNodeName = "blocked_node";
+        CountDownLatch waitForBlockDetectionResult = new CountDownLatch(1);
+        AtomicReference<ThreadInfo> blockDetectionResult = new AtomicReference<>();
+        LongGCDisruption disruption = new LongGCDisruption(random(), disruptedNodeName) {
+            @Override
+            protected Pattern[] getUnsafeClasses() {
+                return new Pattern[0];
+            }
+
+            @Override
+            protected void onBlockDetected(ThreadInfo blockedThread, @Nullable ThreadInfo blockingThread) {
+                blockDetectionResult.set(blockedThread);
+                waitForBlockDetectionResult.countDown();
+            }
+
+            @Override
+            protected long getBlockDetectionIntervalInMillis() {
+                return 10L;
+            }
+        };
+        if (disruption.isBlockDetectionSupported() == false) {
+            return;
+        }
+        final AtomicBoolean stop = new AtomicBoolean();
+        final CountDownLatch underLock = new CountDownLatch(1);
+        final CountDownLatch pauseUnderLock = new CountDownLatch(1);
+        final LockedExecutor lockedExecutor = new LockedExecutor();
+        final AtomicLong ops = new AtomicLong();
+        final List<Thread> threads = new ArrayList<>();
+        try {
+            for (int i = 0; i < 5; i++) {
+                // at least one locked and one none lock thread
+                final boolean lockedExec = (i < 4 && randomBoolean()) || i == 0;
+                Thread thread = new Thread(() -> {
+                    while (stop.get() == false) {
+                        if (lockedExec) {
+                            lockedExecutor.executeLocked(() -> {
+                                try {
+                                    underLock.countDown();
+                                    ops.incrementAndGet();
+                                    pauseUnderLock.await();
+                                } catch (InterruptedException e) {
+
+                                }
+                            });
+                        } else {
+                            ops.incrementAndGet();
+                        }
+                    }
+                });
+
+                thread.setName("[" + disruptedNodeName + "][" + i + "]");
+                threads.add(thread);
+                thread.start();
+            }
+
+            for (int i = 0; i < 5; i++) {
+                // at least one locked and one none lock thread
+                final boolean lockedExec = (i < 4 && randomBoolean()) || i == 0;
+                Thread thread = new Thread(() -> {
+                    while (stop.get() == false) {
+                        if (lockedExec) {
+                            lockedExecutor.executeLocked(() -> {
+                                ops.incrementAndGet();
+                            });
+                        } else {
+                            ops.incrementAndGet();
+                        }
+                    }
+                });
+                thread.setName("[" + blockedNodeName + "][" + i + "]");
+                threads.add(thread);
+                thread.start();
+            }
+            // make sure some threads of test_node are under lock
+            underLock.await();
+            disruption.startDisrupting();
+            assertTrue(waitForBlockDetectionResult.await(30, TimeUnit.SECONDS));
+            disruption.stopDisrupting();
+
+            ThreadInfo threadInfo = blockDetectionResult.get();
+            assertNotNull(threadInfo);
+            assertThat(threadInfo.getThreadName(), containsString("[" + blockedNodeName + "]"));
+            assertThat(threadInfo.getLockOwnerName(), containsString("[" + disruptedNodeName + "]"));
+            assertThat(threadInfo.getLockInfo().getClassName(), containsString(ReentrantLock.class.getName()));
+        } finally {
+            stop.set(true);
+            pauseUnderLock.countDown();
+            for (final Thread thread : threads) {
+                thread.join();
+            }
+        }
+    }
+}
diff --git a/test/framework/src/test/java/org/elasticsearch/test/disruption/NetworkDisruptionIT.java b/test/framework/src/test/java/org/elasticsearch/test/disruption/NetworkDisruptionIT.java
new file mode 100644
index 0000000000000..da035a2712330
--- /dev/null
+++ b/test/framework/src/test/java/org/elasticsearch/test/disruption/NetworkDisruptionIT.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.disruption;
+
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.InternalTestCluster;
+import org.elasticsearch.test.disruption.NetworkDisruption.NetworkUnresponsive;
+import org.elasticsearch.test.disruption.NetworkDisruption.TwoPartitions;
+import org.elasticsearch.test.transport.MockTransportService;
+import org.elasticsearch.transport.TransportService;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+
+public class NetworkDisruptionIT extends ESIntegTestCase {
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Arrays.asList(MockTransportService.TestPlugin.class);
+    }
+
+    public void testNetworkPartitionWithNodeShutdown() throws IOException {
+        internalCluster().ensureAtLeastNumDataNodes(2);
+        String[] nodeNames = internalCluster().getNodeNames();
+        NetworkDisruption networkDisruption = new NetworkDisruption(new TwoPartitions(nodeNames[0], nodeNames[1]),
+            new NetworkUnresponsive());
+        internalCluster().setDisruptionScheme(networkDisruption);
+        networkDisruption.startDisrupting();
+        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodeNames[0]));
+        internalCluster().clearDisruptionScheme();
+    }
+
+    public void testNetworkPartitionRemovalRestoresConnections() throws IOException {
+        Set<String> nodes = new HashSet<>();
+        nodes.addAll(Arrays.asList(internalCluster().getNodeNames()));
+        nodes.remove(internalCluster().getMasterName());
+        if (nodes.size() <= 2) {
+            internalCluster().ensureAtLeastNumDataNodes(3 - nodes.size());
+            nodes.addAll(Arrays.asList(internalCluster().getNodeNames()));
+            nodes.remove(internalCluster().getMasterName());
+        }
+        Set<String> side1 = new HashSet<>(randomSubsetOf(randomIntBetween(1, nodes.size() - 1), nodes));
+        Set<String> side2 = new HashSet<>(nodes);
+        side2.removeAll(side1);
+        assertThat(side2.size(), greaterThanOrEqualTo(1));
+        NetworkDisruption networkDisruption = new NetworkDisruption(new TwoPartitions(side1, side2),
+            new NetworkDisruption.NetworkDisconnect());
+        internalCluster().setDisruptionScheme(networkDisruption);
+        networkDisruption.startDisrupting();
+        // sends some requests
+        client(randomFrom(side1)).admin().cluster().prepareNodesInfo().get();
+        client(randomFrom(side2)).admin().cluster().prepareNodesInfo().get();
+        internalCluster().clearDisruptionScheme();
+        // check all connections are restore
+        for (String nodeA : side1) {
+            for (String nodeB : side2) {
+                TransportService serviceA = internalCluster().getInstance(TransportService.class, nodeA);
+                TransportService serviceB = internalCluster().getInstance(TransportService.class, nodeB);
+                assertTrue(nodeA + " is not connected to " + nodeB, serviceA.nodeConnected(serviceB.getLocalNode()));
+                assertTrue(nodeB + " is not connected to " + nodeA, serviceB.nodeConnected(serviceA.getLocalNode()));
+            }
+        }
+    }
+
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/disruption/NetworkDisruptionTests.java b/test/framework/src/test/java/org/elasticsearch/test/disruption/NetworkDisruptionTests.java
similarity index 79%
rename from test/framework/src/main/java/org/elasticsearch/test/disruption/NetworkDisruptionTests.java
rename to test/framework/src/test/java/org/elasticsearch/test/disruption/NetworkDisruptionTests.java
index 4d0f1123a1bee..298ca869ae51c 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/disruption/NetworkDisruptionTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/disruption/NetworkDisruptionTests.java
@@ -48,18 +48,33 @@ private void checkTwoPartitions(TwoPartitions topology, Set<String> partition1,
             assertTrue(topology.disrupt(randomFrom(partition2), randomFrom(partition1)));
             assertFalse(topology.disrupt(randomFrom(partition1), randomFrom(partition1)));
             assertFalse(topology.disrupt(randomFrom(partition2), randomFrom(partition2)));
-            assertFalse(topology.disrupt(randomAsciiOfLength(10), randomFrom(partition1)));
-            assertFalse(topology.disrupt(randomAsciiOfLength(10), randomFrom(partition2)));
-            assertFalse(topology.disrupt(randomFrom(partition1), randomAsciiOfLength(10)));
-            assertFalse(topology.disrupt(randomFrom(partition2), randomAsciiOfLength(10)));
+            assertFalse(topology.disrupt(randomAlphaOfLength(10), randomFrom(partition1)));
+            assertFalse(topology.disrupt(randomAlphaOfLength(10), randomFrom(partition2)));
+            assertFalse(topology.disrupt(randomFrom(partition1), randomAlphaOfLength(10)));
+            assertFalse(topology.disrupt(randomFrom(partition2), randomAlphaOfLength(10)));
         }
         assertTrue(topology.getMajoritySide().size() >= topology.getMinoritySide().size());
     }
 
+    public void testIsolateAll() {
+        Set<String> nodes = generateRandomStringSet(1, 10);
+        NetworkDisruption.DisruptedLinks topology = new NetworkDisruption.IsolateAllNodes(nodes);
+        for (int i = 0; i < 10; i++) {
+            final String node1 = randomFrom(nodes);
+            final String node2 = randomFrom(nodes);
+            if (node1.equals(node2)) {
+                continue;
+            }
+            assertTrue(topology.nodes().contains(node1));
+            assertTrue(topology.nodes().contains(node2));
+            assertTrue(topology.disrupt(node1, node2));
+        }
+    }
+
     public void testBridge() {
         Set<String> partition1 = generateRandomStringSet(1, 10);
         Set<String> partition2 = generateRandomStringSet(1, 10);
-        String bridgeNode = randomAsciiOfLength(10);
+        String bridgeNode = randomAlphaOfLength(10);
         Bridge topology = new Bridge(bridgeNode, partition1, partition2);
         checkBridge(topology, bridgeNode, partition1, partition2);
     }
@@ -82,12 +97,12 @@ private void checkBridge(Bridge topology, String bridgeNode, Set<String> partiti
             assertFalse(topology.disrupt(randomFrom(partition2), randomFrom(partition2)));
             assertFalse(topology.disrupt(randomFrom(partition2), bridgeNode));
             assertFalse(topology.disrupt(bridgeNode, randomFrom(partition2)));
-            assertFalse(topology.disrupt(randomAsciiOfLength(10), randomFrom(partition1)));
-            assertFalse(topology.disrupt(randomAsciiOfLength(10), randomFrom(partition2)));
-            assertFalse(topology.disrupt(randomAsciiOfLength(10), bridgeNode));
-            assertFalse(topology.disrupt(randomFrom(partition1), randomAsciiOfLength(10)));
-            assertFalse(topology.disrupt(randomFrom(partition2), randomAsciiOfLength(10)));
-            assertFalse(topology.disrupt(bridgeNode, randomAsciiOfLength(10)));
+            assertFalse(topology.disrupt(randomAlphaOfLength(10), randomFrom(partition1)));
+            assertFalse(topology.disrupt(randomAlphaOfLength(10), randomFrom(partition2)));
+            assertFalse(topology.disrupt(randomAlphaOfLength(10), bridgeNode));
+            assertFalse(topology.disrupt(randomFrom(partition1), randomAlphaOfLength(10)));
+            assertFalse(topology.disrupt(randomFrom(partition2), randomAlphaOfLength(10)));
+            assertFalse(topology.disrupt(bridgeNode, randomAlphaOfLength(10)));
         }
     }
 
@@ -95,7 +110,7 @@ private Set<String> generateRandomStringSet(int minSize, int maxSize) {
         assert maxSize >= minSize;
         Set<String> result = new HashSet<>();
         for (int i = 0; i < minSize + randomInt(maxSize - minSize); i++) {
-            result.add(randomAsciiOfLength(10));
+            result.add(randomAlphaOfLength(10));
         }
         return result;
     }
diff --git a/test/framework/src/test/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertionsTests.java b/test/framework/src/test/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertionsTests.java
new file mode 100644
index 0000000000000..ef6d0265b593e
--- /dev/null
+++ b/test/framework/src/test/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertionsTests.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.hamcrest;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Streamable;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+import static java.util.Collections.emptyList;
+import static org.elasticsearch.test.VersionUtils.randomVersion;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertVersionSerializable;
+
+public class ElasticsearchAssertionsTests extends ESTestCase {
+    public void testAssertVersionSerializableIsOkWithIllegalArgumentException() {
+        Version version = randomVersion(random());
+        NamedWriteableRegistry registry = new NamedWriteableRegistry(emptyList());
+        Streamable testStreamable = new TestStreamable();
+
+        // Should catch the exception and do nothing.
+        assertVersionSerializable(version, testStreamable, registry);
+    }
+
+    public static class TestStreamable implements Streamable {
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            throw new IllegalArgumentException("Not supported.");
+        }
+    }
+}
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContextTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContextTests.java
new file mode 100644
index 0000000000000..cdf79b0dd61c4
--- /dev/null
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContextTests.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.rest.yaml;
+
+import org.apache.http.HttpEntity;
+import org.elasticsearch.Version;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
+
+public class ClientYamlTestExecutionContextTests extends ESTestCase {
+
+    public void testHeadersSupportStashedValueReplacement() throws IOException {
+        final AtomicReference<Map<String, String>> headersRef = new AtomicReference<>();
+        final ClientYamlTestExecutionContext context =
+            new ClientYamlTestExecutionContext(null, randomBoolean()) {
+                @Override
+                ClientYamlTestResponse callApiInternal(String apiName, Map<String, String> params,
+                                                       HttpEntity entity,
+                                                       Map<String, String> headers) {
+                    headersRef.set(headers);
+                    return null;
+                }
+
+                @Override
+                public Version esVersion() {
+                    return Version.CURRENT;
+                }
+            };
+        final Map<String, String> headers = new HashMap<>();
+        headers.put("foo", "$bar");
+        headers.put("foo1", "baz ${c}");
+
+        context.stash().stashValue("bar", "foo2");
+        context.stash().stashValue("c", "bar1");
+
+        assertNull(headersRef.get());
+        context.callApi("test", Collections.emptyMap(), Collections.emptyList(), headers);
+        assertNotNull(headersRef.get());
+        assertNotEquals(headers, headersRef.get());
+
+        assertEquals("foo2", headersRef.get().get("foo"));
+        assertEquals("baz bar1", headersRef.get().get("foo1"));
+    }
+}
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCaseTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCaseTests.java
new file mode 100644
index 0000000000000..ee76ad351a62b
--- /dev/null
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCaseTests.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.test.rest.yaml;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Map;
+import java.util.Set;
+
+import org.elasticsearch.test.ESTestCase;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.notNullValue;
+import static org.hamcrest.Matchers.greaterThan;
+
+public class ESClientYamlSuiteTestCaseTests extends ESTestCase {
+
+    public void testLoadAllYamlSuites() throws Exception {
+        Map<String,Set<Path>> yamlSuites = ESClientYamlSuiteTestCase.loadYamlSuites("");
+        assertEquals(2, yamlSuites.size());
+    }
+
+    public void testLoadSingleYamlSuite() throws Exception {
+        Map<String,Set<Path>> yamlSuites = ESClientYamlSuiteTestCase.loadYamlSuites("suite1/10_basic");
+        assertSingleFile(yamlSuites, "suite1", "10_basic.yaml");
+
+        //extension .yaml is optional
+        yamlSuites = ESClientYamlSuiteTestCase.loadYamlSuites("suite1/10_basic");
+        assertSingleFile(yamlSuites, "suite1", "10_basic.yaml");
+    }
+
+    public void testLoadMultipleYamlSuites() throws Exception {
+        //single directory
+        Map<String,Set<Path>> yamlSuites = ESClientYamlSuiteTestCase.loadYamlSuites("suite1");
+        assertThat(yamlSuites, notNullValue());
+        assertThat(yamlSuites.size(), equalTo(1));
+        assertThat(yamlSuites.containsKey("suite1"), equalTo(true));
+        assertThat(yamlSuites.get("suite1").size(), greaterThan(1));
+
+        //multiple directories
+        yamlSuites = ESClientYamlSuiteTestCase.loadYamlSuites("suite1", "suite2");
+        assertThat(yamlSuites, notNullValue());
+        assertThat(yamlSuites.size(), equalTo(2));
+        assertThat(yamlSuites.containsKey("suite1"), equalTo(true));
+        assertEquals(2, yamlSuites.get("suite1").size());
+        assertThat(yamlSuites.containsKey("suite2"), equalTo(true));
+        assertEquals(2, yamlSuites.get("suite2").size());
+
+        //multiple paths, which can be both directories or yaml test suites (with optional file extension)
+        yamlSuites = ESClientYamlSuiteTestCase.loadYamlSuites("suite2/10_basic", "suite1");
+        assertThat(yamlSuites, notNullValue());
+        assertThat(yamlSuites.size(), equalTo(2));
+        assertThat(yamlSuites.containsKey("suite2"), equalTo(true));
+        assertThat(yamlSuites.get("suite2").size(), equalTo(1));
+        assertSingleFile(yamlSuites.get("suite2"), "suite2", "10_basic.yaml");
+        assertThat(yamlSuites.containsKey("suite1"), equalTo(true));
+        assertThat(yamlSuites.get("suite1").size(), greaterThan(1));
+
+        //files can be loaded from classpath and from file system too
+        Path dir = createTempDir();
+        Path file = dir.resolve("test_loading.yaml");
+        Files.createFile(file);
+    }
+
+    private static void assertSingleFile(Map<String, Set<Path>> yamlSuites, String dirName, String fileName) {
+        assertThat(yamlSuites, notNullValue());
+        assertThat(yamlSuites.size(), equalTo(1));
+        assertThat(yamlSuites.containsKey(dirName), equalTo(true));
+        assertSingleFile(yamlSuites.get(dirName), dirName, fileName);
+    }
+
+    private static void assertSingleFile(Set<Path> files, String dirName, String fileName) {
+        assertThat(files.size(), equalTo(1));
+        Path file = files.iterator().next();
+        assertThat(file.getFileName().toString(), equalTo(fileName));
+        assertThat(file.toAbsolutePath().getParent().getFileName().toString(), equalTo(dirName));
+    }
+}
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/FileUtilsTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/FileUtilsTests.java
deleted file mode 100644
index 457152381ba5c..0000000000000
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/FileUtilsTests.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml;
-
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.rest.yaml.FileUtils;
-
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.Map;
-import java.util.Set;
-
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.CoreMatchers.notNullValue;
-import static org.hamcrest.Matchers.greaterThan;
-
-public class FileUtilsTests extends ESTestCase {
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/20240")
-    public void testLoadSingleYamlSuite() throws Exception {
-        Map<String,Set<Path>> yamlSuites = FileUtils.findYamlSuites(null, "/rest-api-spec/test", "/rest-api-spec/test/suite1/10_basic");
-        assertSingleFile(yamlSuites, "suite1", "10_basic.yaml");
-
-        //the path prefix is optional
-        yamlSuites = FileUtils.findYamlSuites(null, "/rest-api-spec/test", "suite1/10_basic.yaml");
-        assertSingleFile(yamlSuites, "suite1", "10_basic.yaml");
-
-        //extension .yaml is optional
-        yamlSuites = FileUtils.findYamlSuites(null, "/rest-api-spec/test", "suite1/10_basic");
-        assertSingleFile(yamlSuites, "suite1", "10_basic.yaml");
-    }
-
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/20240")
-    public void testLoadMultipleYamlSuites() throws Exception {
-        //single directory
-        Map<String,Set<Path>> yamlSuites = FileUtils.findYamlSuites(null, "/rest-api-spec/test", "suite1");
-        assertThat(yamlSuites, notNullValue());
-        assertThat(yamlSuites.size(), equalTo(1));
-        assertThat(yamlSuites.containsKey("suite1"), equalTo(true));
-        assertThat(yamlSuites.get("suite1").size(), greaterThan(1));
-
-        //multiple directories
-        yamlSuites = FileUtils.findYamlSuites(null, "/rest-api-spec/test", "suite1", "suite2");
-        assertThat(yamlSuites, notNullValue());
-        assertThat(yamlSuites.size(), equalTo(2));
-        assertThat(yamlSuites.containsKey("suite1"), equalTo(true));
-        assertEquals(2, yamlSuites.get("suite1").size());
-        assertThat(yamlSuites.containsKey("suite2"), equalTo(true));
-        assertEquals(2, yamlSuites.get("suite2").size());
-
-        //multiple paths, which can be both directories or yaml test suites (with optional file extension)
-        yamlSuites = FileUtils.findYamlSuites(null, "/rest-api-spec/test", "suite2/10_basic", "suite1");
-        assertThat(yamlSuites, notNullValue());
-        assertThat(yamlSuites.size(), equalTo(2));
-        assertThat(yamlSuites.containsKey("suite2"), equalTo(true));
-        assertThat(yamlSuites.get("suite2").size(), equalTo(1));
-        assertSingleFile(yamlSuites.get("suite2"), "suite2", "10_basic.yaml");
-        assertThat(yamlSuites.containsKey("suite1"), equalTo(true));
-        assertThat(yamlSuites.get("suite1").size(), greaterThan(1));
-
-        //files can be loaded from classpath and from file system too
-        Path dir = createTempDir();
-        Path file = dir.resolve("test_loading.yaml");
-        Files.createFile(file);
-
-        //load from directory outside of the classpath
-        yamlSuites = FileUtils.findYamlSuites(dir.getFileSystem(), "/rest-api-spec/test", dir.toAbsolutePath().toString());
-        assertThat(yamlSuites, notNullValue());
-        assertThat(yamlSuites.size(), equalTo(1));
-        assertThat(yamlSuites.containsKey(dir.getFileName().toString()), equalTo(true));
-        assertSingleFile(yamlSuites.get(dir.getFileName().toString()), dir.getFileName().toString(), file.getFileName().toString());
-
-        //load from external file (optional extension)
-        yamlSuites = FileUtils.findYamlSuites(dir.getFileSystem(), "/rest-api-spec/test",
-                dir.resolve("test_loading").toAbsolutePath().toString());
-        assertThat(yamlSuites, notNullValue());
-        assertThat(yamlSuites.size(), equalTo(1));
-        assertThat(yamlSuites.containsKey(dir.getFileName().toString()), equalTo(true));
-        assertSingleFile(yamlSuites.get(dir.getFileName().toString()), dir.getFileName().toString(), file.getFileName().toString());
-    }
-
-    private static void assertSingleFile(Map<String, Set<Path>> yamlSuites, String dirName, String fileName) {
-        assertThat(yamlSuites, notNullValue());
-        assertThat(yamlSuites.size(), equalTo(1));
-        assertThat(yamlSuites.containsKey(dirName), equalTo(true));
-        assertSingleFile(yamlSuites.get(dirName), dirName, fileName);
-    }
-
-    private static void assertSingleFile(Set<Path> files, String dirName, String fileName) {
-        assertThat(files.size(), equalTo(1));
-        Path file = files.iterator().next();
-        assertThat(file.getFileName().toString(), equalTo(fileName));
-        assertThat(file.toAbsolutePath().getParent().getFileName().toString(), equalTo(dirName));
-    }
-}
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/ObjectPathTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/ObjectPathTests.java
index 5559fd3f385f7..c377b500ccce3 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/ObjectPathTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/ObjectPathTests.java
@@ -22,8 +22,6 @@
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.rest.yaml.ObjectPath;
-import org.elasticsearch.test.rest.yaml.Stash;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -41,8 +39,7 @@
 public class ObjectPathTests extends ESTestCase {
 
     private static XContentBuilder randomXContentBuilder() throws IOException {
-        //only string based formats are supported, no cbor nor smile
-        XContentType xContentType = randomFrom(XContentType.JSON, XContentType.YAML);
+        XContentType xContentType = randomFrom(XContentType.values());
         return XContentBuilder.builder(XContentFactory.xContent(xContentType));
     }
 
@@ -53,7 +50,7 @@ public void testEvaluateObjectPathEscape() throws Exception {
         xContentBuilder.field("field2.field3", "value2");
         xContentBuilder.endObject();
         xContentBuilder.endObject();
-        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.contentType().xContent(), xContentBuilder.string());
+        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.contentType().xContent(), xContentBuilder.bytes());
         Object object = objectPath.evaluate("field1.field2\\.field3");
         assertThat(object, instanceOf(String.class));
         assertThat(object, equalTo("value2"));
@@ -66,7 +63,7 @@ public void testEvaluateObjectPathWithDots() throws Exception {
         xContentBuilder.field("field2", "value2");
         xContentBuilder.endObject();
         xContentBuilder.endObject();
-        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.contentType().xContent(), xContentBuilder.string());
+        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.contentType().xContent(), xContentBuilder.bytes());
         Object object = objectPath.evaluate("field1..field2");
         assertThat(object, instanceOf(String.class));
         assertThat(object, equalTo("value2"));
@@ -85,7 +82,7 @@ public void testEvaluateInteger() throws Exception {
         xContentBuilder.field("field2", 333);
         xContentBuilder.endObject();
         xContentBuilder.endObject();
-        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.contentType().xContent(), xContentBuilder.string());
+        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.contentType().xContent(), xContentBuilder.bytes());
         Object object = objectPath.evaluate("field1.field2");
         assertThat(object, instanceOf(Integer.class));
         assertThat(object, equalTo(333));
@@ -98,7 +95,7 @@ public void testEvaluateDouble() throws Exception {
         xContentBuilder.field("field2", 3.55);
         xContentBuilder.endObject();
         xContentBuilder.endObject();
-        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.contentType().xContent(), xContentBuilder.string());
+        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.contentType().xContent(), xContentBuilder.bytes());
         Object object = objectPath.evaluate("field1.field2");
         assertThat(object, instanceOf(Double.class));
         assertThat(object, equalTo(3.55));
@@ -111,7 +108,7 @@ public void testEvaluateArray() throws Exception {
         xContentBuilder.array("array1", "value1", "value2");
         xContentBuilder.endObject();
         xContentBuilder.endObject();
-        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.contentType().xContent(), xContentBuilder.string());
+        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.contentType().xContent(), xContentBuilder.bytes());
         Object object = objectPath.evaluate("field1.array1");
         assertThat(object, instanceOf(List.class));
         List list = (List) object;
@@ -140,7 +137,7 @@ public void testEvaluateArrayElementObject() throws Exception {
         xContentBuilder.endArray();
         xContentBuilder.endObject();
         xContentBuilder.endObject();
-        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.contentType().xContent(), xContentBuilder.string());
+        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.contentType().xContent(), xContentBuilder.bytes());
         Object object = objectPath.evaluate("field1.array1.1.element");
         assertThat(object, instanceOf(String.class));
         assertThat(object, equalTo("value2"));
@@ -167,7 +164,7 @@ public void testEvaluateObjectKeys() throws Exception {
         xContentBuilder.endObject();
         xContentBuilder.endObject();
         xContentBuilder.endObject();
-        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.contentType().xContent(), xContentBuilder.string());
+        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.contentType().xContent(), xContentBuilder.bytes());
         Object object = objectPath.evaluate("metadata.templates");
         assertThat(object, instanceOf(Map.class));
         Map<String, Object> map = (Map<String, Object>)object;
@@ -185,7 +182,7 @@ public void testEvaluateStashInPropertyName() throws Exception {
         xContentBuilder.endObject();
         xContentBuilder.endObject();
         xContentBuilder.endObject();
-        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.contentType().xContent(), xContentBuilder.string());
+        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.contentType().xContent(), xContentBuilder.bytes());
         try {
             objectPath.evaluate("field1.$placeholder.element1");
             fail("evaluate should have failed due to unresolved placeholder");
@@ -248,7 +245,8 @@ public void testEvaluateArrayAsRoot() throws Exception {
         xContentBuilder.field("index", "test2");
         xContentBuilder.endObject();
         xContentBuilder.endArray();
-        ObjectPath objectPath = ObjectPath.createFromXContent(XContentFactory.xContent(XContentType.YAML), xContentBuilder.string());
+        ObjectPath objectPath = ObjectPath.createFromXContent(
+                XContentFactory.xContent(xContentBuilder.contentType()), xContentBuilder.bytes());
         Object object = objectPath.evaluate("");
         assertThat(object, notNullValue());
         assertThat(object, instanceOf(List.class));
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/StashTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/StashTests.java
index a8d32a316ac9b..8e7e0284249de 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/StashTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/StashTests.java
@@ -20,27 +20,139 @@
 package org.elasticsearch.test.rest.yaml;
 
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.rest.yaml.Stash;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
 
 import static java.util.Collections.singletonMap;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.sameInstance;
 
 public class StashTests extends ESTestCase {
-    public void testReplaceStashedValuesEmbeddedStashKey() throws IOException {
+    public void testReplaceStashedValuesStashKeyInMapValue() throws IOException {
         Stash stash = new Stash();
-        stash.stashValue("stashed", "bar");
-        
+
         Map<String, Object> expected = new HashMap<>();
         expected.put("key", singletonMap("a", "foobar"));
         Map<String, Object> map = new HashMap<>();
         Map<String, Object> map2 = new HashMap<>();
-        map2.put("a", "foo${stashed}");
+        if (randomBoolean()) {
+            stash.stashValue("stashed", "bar");
+            map2.put("a", "foo${stashed}");
+        } else {
+            stash.stashValue("stashed", "foobar");
+            map2.put("a", "$stashed");
+        }
+        map.put("key", map2);
+
+        Map<String, Object> actual = stash.replaceStashedValues(map);
+        assertEquals(expected, actual);
+        assertThat(actual, not(sameInstance(map)));
+    }
+
+    public void testReplaceStashedValuesStashKeyInMapKey() throws IOException {
+        Stash stash = new Stash();
+
+        Map<String, Object> expected = new HashMap<>();
+        expected.put("key", singletonMap("foobar", "a"));
+        Map<String, Object> map = new HashMap<>();
+        Map<String, Object> map2 = new HashMap<>();
+        if (randomBoolean()) {
+            stash.stashValue("stashed", "bar");
+            map2.put("foo${stashed}", "a");
+        } else {
+            stash.stashValue("stashed", "foobar");
+            map2.put("$stashed", "a");
+        }
+        map.put("key", map2);
+
+        Map<String, Object> actual = stash.replaceStashedValues(map);
+        assertEquals(expected, actual);
+        assertThat(actual, not(sameInstance(map)));
+    }
+
+    public void testReplaceStashedValuesStashKeyInMapKeyConflicts() throws IOException {
+        Stash stash = new Stash();
+
+        Map<String, Object> map = new HashMap<>();
+        Map<String, Object> map2 = new HashMap<>();
+        String key;
+        if (randomBoolean()) {
+            stash.stashValue("stashed", "bar");
+            key = "foo${stashed}";
+        } else {
+            stash.stashValue("stashed", "foobar");
+            key = "$stashed";
+        }
+        map2.put(key, "a");
+        map2.put("foobar", "whatever");
         map.put("key", map2);
 
+        Exception e = expectThrows(IllegalArgumentException.class, () -> stash.replaceStashedValues(map));
+        assertEquals(e.getMessage(), "Unstashing has caused a key conflict! The map is [{foobar=whatever}] and the key is ["
+                            + key + "] which unstashes to [foobar]");
+    }
+
+    public void testReplaceStashedValuesStashKeyInList() throws IOException {
+        Stash stash = new Stash();
+        stash.stashValue("stashed", "bar");
+
+        Map<String, Object> expected = new HashMap<>();
+        expected.put("key", Arrays.asList("foot", "foobar", 1));
+        Map<String, Object> map = new HashMap<>();
+        Object value;
+        if (randomBoolean()) {
+            stash.stashValue("stashed", "bar");
+            value = "foo${stashed}";
+        } else {
+            stash.stashValue("stashed", "foobar");
+            value = "$stashed";
+        }
+        map.put("key", Arrays.asList("foot", value, 1));
+
         Map<String, Object> actual = stash.replaceStashedValues(map);
         assertEquals(expected, actual);
+        assertThat(actual, not(sameInstance(map)));
     }
+
+    public void testPathInList() throws IOException {
+        Stash stash = new Stash();
+        stash.stashValue("body", singletonMap("foo", Arrays.asList("a", "b")));
+
+        Map<String, Object> expected;
+        Map<String, Object> map;
+        if (randomBoolean()) {
+            expected = singletonMap("foo", Arrays.asList("test", "boooooh!"));
+            map = singletonMap("foo", Arrays.asList("test", "${body.$_path}oooooh!"));
+        } else {
+            expected = singletonMap("foo", Arrays.asList("test", "b"));
+            map = singletonMap("foo", Arrays.asList("test", "$body.$_path"));
+        }
+
+        Map<String, Object> actual = stash.replaceStashedValues(map);
+        assertEquals(expected, actual);
+        assertThat(actual, not(sameInstance(map)));
+    }
+
+    public void testPathInMapValue() throws IOException {
+        Stash stash = new Stash();
+        stash.stashValue("body", singletonMap("foo", singletonMap("a", "b")));
+
+        Map<String, Object> expected;
+        Map<String, Object> map;
+        if (randomBoolean()) {
+            expected = singletonMap("foo", singletonMap("a", "boooooh!"));
+            map = singletonMap("foo", singletonMap("a", "${body.$_path}oooooh!"));
+        } else {
+            expected = singletonMap("foo", singletonMap("a", "b"));
+            map = singletonMap("foo", singletonMap("a", "$body.$_path"));
+        }
+
+        Map<String, Object> actual = stash.replaceStashedValues(map);
+        assertEquals(expected, actual);
+        assertThat(actual, not(sameInstance(map)));
+    }
+
 }
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/AbstractParserTestCase.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/AbstractParserTestCase.java
deleted file mode 100644
index 7d32389ddba62..0000000000000
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/AbstractParserTestCase.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.test.ESTestCase;
-import org.junit.After;
-
-import static org.hamcrest.Matchers.nullValue;
-
-public abstract class AbstractParserTestCase extends ESTestCase {
-
-    protected XContentParser parser;
-
-    @Override
-    @After
-    public void tearDown() throws Exception {
-        super.tearDown();
-        //this is the way to make sure that we consumed the whole yaml
-        assertThat(parser.currentToken(), nullValue());
-        parser.close();
-    }
-}
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/AssertionParsersTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/AssertionParsersTests.java
deleted file mode 100644
index 4e382fb7b0989..0000000000000
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/AssertionParsersTests.java
+++ /dev/null
@@ -1,173 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.common.xcontent.yaml.YamlXContent;
-import org.elasticsearch.test.rest.yaml.parser.GreaterThanParser;
-import org.elasticsearch.test.rest.yaml.parser.IsFalseParser;
-import org.elasticsearch.test.rest.yaml.parser.IsTrueParser;
-import org.elasticsearch.test.rest.yaml.parser.LengthParser;
-import org.elasticsearch.test.rest.yaml.parser.LessThanParser;
-import org.elasticsearch.test.rest.yaml.parser.MatchParser;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestSuiteParseContext;
-import org.elasticsearch.test.rest.yaml.section.GreaterThanAssertion;
-import org.elasticsearch.test.rest.yaml.section.IsFalseAssertion;
-import org.elasticsearch.test.rest.yaml.section.IsTrueAssertion;
-import org.elasticsearch.test.rest.yaml.section.LengthAssertion;
-import org.elasticsearch.test.rest.yaml.section.LessThanAssertion;
-import org.elasticsearch.test.rest.yaml.section.MatchAssertion;
-
-import java.util.List;
-import java.util.Map;
-
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.notNullValue;
-
-public class AssertionParsersTests extends AbstractParserTestCase {
-    public void testParseIsTrue() throws Exception {
-        parser = YamlXContent.yamlXContent.createParser(
-                "get.fields._timestamp"
-        );
-
-        IsTrueParser isTrueParser = new IsTrueParser();
-        IsTrueAssertion trueAssertion = isTrueParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(trueAssertion, notNullValue());
-        assertThat(trueAssertion.getField(), equalTo("get.fields._timestamp"));
-    }
-
-    public void testParseIsFalse() throws Exception {
-        parser = YamlXContent.yamlXContent.createParser(
-                "docs.1._source"
-        );
-
-        IsFalseParser isFalseParser = new IsFalseParser();
-        IsFalseAssertion falseAssertion = isFalseParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(falseAssertion, notNullValue());
-        assertThat(falseAssertion.getField(), equalTo("docs.1._source"));
-    }
-
-    public void testParseGreaterThan() throws Exception {
-        parser = YamlXContent.yamlXContent.createParser(
-                "{ field: 3}"
-        );
-
-        GreaterThanParser greaterThanParser = new GreaterThanParser();
-        GreaterThanAssertion greaterThanAssertion = greaterThanParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-        assertThat(greaterThanAssertion, notNullValue());
-        assertThat(greaterThanAssertion.getField(), equalTo("field"));
-        assertThat(greaterThanAssertion.getExpectedValue(), instanceOf(Integer.class));
-        assertThat((Integer) greaterThanAssertion.getExpectedValue(), equalTo(3));
-    }
-
-    public void testParseLessThan() throws Exception {
-        parser = YamlXContent.yamlXContent.createParser(
-                "{ field: 3}"
-        );
-
-        LessThanParser lessThanParser = new LessThanParser();
-        LessThanAssertion lessThanAssertion = lessThanParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-        assertThat(lessThanAssertion, notNullValue());
-        assertThat(lessThanAssertion.getField(), equalTo("field"));
-        assertThat(lessThanAssertion.getExpectedValue(), instanceOf(Integer.class));
-        assertThat((Integer) lessThanAssertion.getExpectedValue(), equalTo(3));
-    }
-
-    public void testParseLength() throws Exception {
-        parser = YamlXContent.yamlXContent.createParser(
-                "{ _id: 22}"
-        );
-
-        LengthParser lengthParser = new LengthParser();
-        LengthAssertion lengthAssertion = lengthParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-        assertThat(lengthAssertion, notNullValue());
-        assertThat(lengthAssertion.getField(), equalTo("_id"));
-        assertThat(lengthAssertion.getExpectedValue(), instanceOf(Integer.class));
-        assertThat((Integer) lengthAssertion.getExpectedValue(), equalTo(22));
-    }
-
-    public void testParseMatchSimpleIntegerValue() throws Exception {
-        parser = YamlXContent.yamlXContent.createParser(
-                "{ field: 10 }"
-        );
-
-        MatchParser matchParser = new MatchParser();
-        MatchAssertion matchAssertion = matchParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(matchAssertion, notNullValue());
-        assertThat(matchAssertion.getField(), equalTo("field"));
-        assertThat(matchAssertion.getExpectedValue(), instanceOf(Integer.class));
-        assertThat((Integer) matchAssertion.getExpectedValue(), equalTo(10));
-    }
-
-    public void testParseMatchSimpleStringValue() throws Exception {
-        parser = YamlXContent.yamlXContent.createParser(
-                "{ foo: bar }"
-        );
-
-        MatchParser matchParser = new MatchParser();
-        MatchAssertion matchAssertion = matchParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(matchAssertion, notNullValue());
-        assertThat(matchAssertion.getField(), equalTo("foo"));
-        assertThat(matchAssertion.getExpectedValue(), instanceOf(String.class));
-        assertThat(matchAssertion.getExpectedValue().toString(), equalTo("bar"));
-    }
-
-    public void testParseMatchArray() throws Exception {
-        parser = YamlXContent.yamlXContent.createParser(
-                "{'matches': ['test_percolator_1', 'test_percolator_2']}"
-        );
-
-        MatchParser matchParser = new MatchParser();
-        MatchAssertion matchAssertion = matchParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(matchAssertion, notNullValue());
-        assertThat(matchAssertion.getField(), equalTo("matches"));
-        assertThat(matchAssertion.getExpectedValue(), instanceOf(List.class));
-        List strings = (List) matchAssertion.getExpectedValue();
-        assertThat(strings.size(), equalTo(2));
-        assertThat(strings.get(0).toString(), equalTo("test_percolator_1"));
-        assertThat(strings.get(1).toString(), equalTo("test_percolator_2"));
-    }
-
-    @SuppressWarnings("unchecked")
-    public void testParseMatchSourceValues() throws Exception {
-        parser = YamlXContent.yamlXContent.createParser(
-                "{ _source: { responses.0.hits.total: 3, foo: bar  }}"
-        );
-
-        MatchParser matchParser = new MatchParser();
-        MatchAssertion matchAssertion = matchParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(matchAssertion, notNullValue());
-        assertThat(matchAssertion.getField(), equalTo("_source"));
-        assertThat(matchAssertion.getExpectedValue(), instanceOf(Map.class));
-        Map<String, Object> expectedValue = (Map<String, Object>) matchAssertion.getExpectedValue();
-        assertThat(expectedValue.size(), equalTo(2));
-        Object o = expectedValue.get("responses.0.hits.total");
-        assertThat(o, instanceOf(Integer.class));
-        assertThat((Integer)o, equalTo(3));
-        o = expectedValue.get("foo");
-        assertThat(o, instanceOf(String.class));
-        assertThat(o.toString(), equalTo("bar"));
-    }
-}
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlSuiteTestParserTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlSuiteTestParserTests.java
deleted file mode 100644
index 3b7313ec22177..0000000000000
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlSuiteTestParserTests.java
+++ /dev/null
@@ -1,405 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.yaml.YamlXContent;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestSuiteParseContext;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestSuiteParser;
-import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSuite;
-import org.elasticsearch.test.rest.yaml.section.DoSection;
-import org.elasticsearch.test.rest.yaml.section.IsTrueAssertion;
-import org.elasticsearch.test.rest.yaml.section.MatchAssertion;
-import org.junit.After;
-
-import java.util.Map;
-
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.notNullValue;
-import static org.hamcrest.Matchers.nullValue;
-
-public class ClientYamlSuiteTestParserTests extends ESTestCase {
-    private XContentParser parser;
-
-    @Override
-    @After
-    public void tearDown() throws Exception {
-        super.tearDown();
-        //makes sure that we consumed the whole stream, XContentParser doesn't expose isClosed method
-        //next token can be null even in the middle of the document (e.g. with "---"), but not too many consecutive times
-        assertThat(parser.currentToken(), nullValue());
-        assertThat(parser.nextToken(), nullValue());
-        assertThat(parser.nextToken(), nullValue());
-        parser.close();
-    }
-
-    public void testParseTestSetupTeardownAndSections() throws Exception {
-        final boolean includeSetup = randomBoolean();
-        final boolean includeTeardown = randomBoolean();
-        StringBuilder testSpecBuilder = new StringBuilder();
-        if (includeSetup) {
-            testSpecBuilder
-                .append("---\n" +
-                        "setup:\n" +
-                        "  - do:\n" +
-                        "        indices.create:\n" +
-                        "          index: test_index\n" +
-                        "\n");
-        }
-        if (includeTeardown) {
-            testSpecBuilder
-                .append("---\n" +
-                        "teardown:\n" +
-                        "  - do:\n" +
-                        "      indices.delete:\n" +
-                        "        index: test_index\n" +
-                        "\n");
-        }
-        parser = YamlXContent.yamlXContent.createParser(
-                        testSpecBuilder.toString() +
-                        "---\n" +
-                        "\"Get index mapping\":\n" +
-                        "  - do:\n" +
-                        "      indices.get_mapping:\n" +
-                        "        index: test_index\n" +
-                        "\n" +
-                        "  - match: {test_index.test_type.properties.text.type:     string}\n" +
-                        "  - match: {test_index.test_type.properties.text.analyzer: whitespace}\n" +
-                        "\n" +
-                        "---\n" +
-                        "\"Get type mapping - pre 1.0\":\n" +
-                        "\n" +
-                        "  - skip:\n" +
-                        "      version:     \"2.0.0 - \"\n" +
-                        "      reason:      \"for newer versions the index name is always returned\"\n" +
-                        "\n" +
-                        "  - do:\n" +
-                        "      indices.get_mapping:\n" +
-                        "        index: test_index\n" +
-                        "        type: test_type\n" +
-                        "\n" +
-                        "  - match: {test_type.properties.text.type:     string}\n" +
-                        "  - match: {test_type.properties.text.analyzer: whitespace}\n"
-        );
-
-        ClientYamlTestSuiteParser testParser = new ClientYamlTestSuiteParser();
-        ClientYamlTestSuite restTestSuite = testParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(restTestSuite, notNullValue());
-        assertThat(restTestSuite.getName(), equalTo("suite"));
-        assertThat(restTestSuite.getSetupSection(), notNullValue());
-        if (includeSetup) {
-            assertThat(restTestSuite.getSetupSection().isEmpty(), equalTo(false));
-            assertThat(restTestSuite.getSetupSection().getSkipSection().isEmpty(), equalTo(true));
-            assertThat(restTestSuite.getSetupSection().getDoSections().size(), equalTo(1));
-            assertThat(restTestSuite.getSetupSection().getDoSections().get(0).getApiCallSection().getApi(), equalTo("indices.create"));
-            assertThat(restTestSuite.getSetupSection().getDoSections().get(0).getApiCallSection().getParams().size(), equalTo(1));
-            assertThat(restTestSuite.getSetupSection().getDoSections().get(0).getApiCallSection().getParams().get("index"),
-                    equalTo("test_index"));
-        } else {
-            assertThat(restTestSuite.getSetupSection().isEmpty(), equalTo(true));
-        }
-        
-        assertThat(restTestSuite.getTeardownSection(), notNullValue());
-        if (includeTeardown) {
-            assertThat(restTestSuite.getTeardownSection().isEmpty(), equalTo(false));
-            assertThat(restTestSuite.getTeardownSection().getSkipSection().isEmpty(), equalTo(true));
-            assertThat(restTestSuite.getTeardownSection().getDoSections().size(), equalTo(1));
-            assertThat(restTestSuite.getTeardownSection().getDoSections().get(0).getApiCallSection().getApi(), equalTo("indices.delete"));
-            assertThat(restTestSuite.getTeardownSection().getDoSections().get(0).getApiCallSection().getParams().size(), equalTo(1));
-            assertThat(restTestSuite.getTeardownSection().getDoSections().get(0).getApiCallSection().getParams().get("index"),
-                equalTo("test_index"));
-        } else {
-            assertThat(restTestSuite.getTeardownSection().isEmpty(), equalTo(true));
-        }
-
-        assertThat(restTestSuite.getTestSections().size(), equalTo(2));
-
-        assertThat(restTestSuite.getTestSections().get(0).getName(), equalTo("Get index mapping"));
-        assertThat(restTestSuite.getTestSections().get(0).getSkipSection().isEmpty(), equalTo(true));
-        assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().size(), equalTo(3));
-        assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(0), instanceOf(DoSection.class));
-        DoSection doSection = (DoSection) restTestSuite.getTestSections().get(0).getExecutableSections().get(0);
-        assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_mapping"));
-        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(1));
-        assertThat(doSection.getApiCallSection().getParams().get("index"), equalTo("test_index"));
-        assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(1), instanceOf(MatchAssertion.class));
-        MatchAssertion matchAssertion = (MatchAssertion) restTestSuite.getTestSections().get(0).getExecutableSections().get(1);
-        assertThat(matchAssertion.getField(), equalTo("test_index.test_type.properties.text.type"));
-        assertThat(matchAssertion.getExpectedValue().toString(), equalTo("string"));
-        assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(2), instanceOf(MatchAssertion.class));
-        matchAssertion = (MatchAssertion) restTestSuite.getTestSections().get(0).getExecutableSections().get(2);
-        assertThat(matchAssertion.getField(), equalTo("test_index.test_type.properties.text.analyzer"));
-        assertThat(matchAssertion.getExpectedValue().toString(), equalTo("whitespace"));
-
-        assertThat(restTestSuite.getTestSections().get(1).getName(), equalTo("Get type mapping - pre 1.0"));
-        assertThat(restTestSuite.getTestSections().get(1).getSkipSection().isEmpty(), equalTo(false));
-        assertThat(restTestSuite.getTestSections().get(1).getSkipSection().getReason(),
-                equalTo("for newer versions the index name is always returned"));
-        assertThat(restTestSuite.getTestSections().get(1).getSkipSection().getLowerVersion(), equalTo(Version.V_2_0_0));
-        assertThat(restTestSuite.getTestSections().get(1).getSkipSection().getUpperVersion(), equalTo(Version.CURRENT));
-        assertThat(restTestSuite.getTestSections().get(1).getExecutableSections().size(), equalTo(3));
-        assertThat(restTestSuite.getTestSections().get(1).getExecutableSections().get(0), instanceOf(DoSection.class));
-        doSection = (DoSection) restTestSuite.getTestSections().get(1).getExecutableSections().get(0);
-        assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_mapping"));
-        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2));
-        assertThat(doSection.getApiCallSection().getParams().get("index"), equalTo("test_index"));
-        assertThat(doSection.getApiCallSection().getParams().get("type"), equalTo("test_type"));
-        assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(1), instanceOf(MatchAssertion.class));
-        matchAssertion = (MatchAssertion) restTestSuite.getTestSections().get(1).getExecutableSections().get(1);
-        assertThat(matchAssertion.getField(), equalTo("test_type.properties.text.type"));
-        assertThat(matchAssertion.getExpectedValue().toString(), equalTo("string"));
-        assertThat(restTestSuite.getTestSections().get(1).getExecutableSections().get(2), instanceOf(MatchAssertion.class));
-        matchAssertion = (MatchAssertion) restTestSuite.getTestSections().get(1).getExecutableSections().get(2);
-        assertThat(matchAssertion.getField(), equalTo("test_type.properties.text.analyzer"));
-        assertThat(matchAssertion.getExpectedValue().toString(), equalTo("whitespace"));
-    }
-
-    public void testParseTestSingleTestSection() throws Exception {
-        parser = YamlXContent.yamlXContent.createParser(
-        "---\n" +
-                "\"Index with ID\":\n" +
-                "\n" +
-                " - do:\n" +
-                "      index:\n" +
-                "          index:  test-weird-index-中文\n" +
-                "          type:   weird.type\n" +
-                "          id:     1\n" +
-                "          body:   { foo: bar }\n" +
-                "\n" +
-                " - is_true:   ok\n" +
-                " - match:   { _index:   test-weird-index-中文 }\n" +
-                " - match:   { _type:    weird.type }\n" +
-                " - match:   { _id:      \"1\"}\n" +
-                " - match:   { _version: 1}\n" +
-                "\n" +
-                " - do:\n" +
-                "      get:\n" +
-                "          index:  test-weird-index-中文\n" +
-                "          type:   weird.type\n" +
-                "          id:     1\n" +
-                "\n" +
-                " - match:   { _index:   test-weird-index-中文 }\n" +
-                " - match:   { _type:    weird.type }\n" +
-                " - match:   { _id:      \"1\"}\n" +
-                " - match:   { _version: 1}\n" +
-                " - match:   { _source: { foo: bar }}"
-        );
-
-        ClientYamlTestSuiteParser testParser = new ClientYamlTestSuiteParser();
-        ClientYamlTestSuite restTestSuite = testParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(restTestSuite, notNullValue());
-        assertThat(restTestSuite.getName(), equalTo("suite"));
-
-        assertThat(restTestSuite.getSetupSection().isEmpty(), equalTo(true));
-
-        assertThat(restTestSuite.getTestSections().size(), equalTo(1));
-
-        assertThat(restTestSuite.getTestSections().get(0).getName(), equalTo("Index with ID"));
-        assertThat(restTestSuite.getTestSections().get(0).getSkipSection().isEmpty(), equalTo(true));
-        assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().size(), equalTo(12));
-        assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(0), instanceOf(DoSection.class));
-        DoSection doSection = (DoSection) restTestSuite.getTestSections().get(0).getExecutableSections().get(0);
-        assertThat(doSection.getCatch(), nullValue());
-        assertThat(doSection.getApiCallSection().getApi(), equalTo("index"));
-        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3));
-        assertThat(doSection.getApiCallSection().hasBody(), equalTo(true));
-        assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(1), instanceOf(IsTrueAssertion.class));
-        IsTrueAssertion trueAssertion = (IsTrueAssertion) restTestSuite.getTestSections().get(0).getExecutableSections().get(1);
-        assertThat(trueAssertion.getField(), equalTo("ok"));
-        assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(2), instanceOf(MatchAssertion.class));
-        MatchAssertion matchAssertion = (MatchAssertion) restTestSuite.getTestSections().get(0).getExecutableSections().get(2);
-        assertThat(matchAssertion.getField(), equalTo("_index"));
-        assertThat(matchAssertion.getExpectedValue().toString(), equalTo("test-weird-index-中文"));
-        assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(3), instanceOf(MatchAssertion.class));
-        matchAssertion = (MatchAssertion) restTestSuite.getTestSections().get(0).getExecutableSections().get(3);
-        assertThat(matchAssertion.getField(), equalTo("_type"));
-        assertThat(matchAssertion.getExpectedValue().toString(), equalTo("weird.type"));
-        assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(4), instanceOf(MatchAssertion.class));
-        matchAssertion = (MatchAssertion) restTestSuite.getTestSections().get(0).getExecutableSections().get(4);
-        assertThat(matchAssertion.getField(), equalTo("_id"));
-        assertThat(matchAssertion.getExpectedValue().toString(), equalTo("1"));
-        assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(5), instanceOf(MatchAssertion.class));
-        matchAssertion = (MatchAssertion) restTestSuite.getTestSections().get(0).getExecutableSections().get(5);
-        assertThat(matchAssertion.getField(), equalTo("_version"));
-        assertThat(matchAssertion.getExpectedValue().toString(), equalTo("1"));
-        assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(6), instanceOf(DoSection.class));
-        doSection = (DoSection) restTestSuite.getTestSections().get(0).getExecutableSections().get(6);
-        assertThat(doSection.getCatch(), nullValue());
-        assertThat(doSection.getApiCallSection().getApi(), equalTo("get"));
-        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3));
-        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
-        assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(7), instanceOf(MatchAssertion.class));
-        matchAssertion = (MatchAssertion) restTestSuite.getTestSections().get(0).getExecutableSections().get(7);
-        assertThat(matchAssertion.getField(), equalTo("_index"));
-        assertThat(matchAssertion.getExpectedValue().toString(), equalTo("test-weird-index-中文"));
-        assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(8), instanceOf(MatchAssertion.class));
-        matchAssertion = (MatchAssertion) restTestSuite.getTestSections().get(0).getExecutableSections().get(8);
-        assertThat(matchAssertion.getField(), equalTo("_type"));
-        assertThat(matchAssertion.getExpectedValue().toString(), equalTo("weird.type"));
-        assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(9), instanceOf(MatchAssertion.class));
-        matchAssertion = (MatchAssertion) restTestSuite.getTestSections().get(0).getExecutableSections().get(9);
-        assertThat(matchAssertion.getField(), equalTo("_id"));
-        assertThat(matchAssertion.getExpectedValue().toString(), equalTo("1"));
-        assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(10), instanceOf(MatchAssertion.class));
-        matchAssertion = (MatchAssertion) restTestSuite.getTestSections().get(0).getExecutableSections().get(10);
-        assertThat(matchAssertion.getField(), equalTo("_version"));
-        assertThat(matchAssertion.getExpectedValue().toString(), equalTo("1"));
-        assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(11), instanceOf(MatchAssertion.class));
-        matchAssertion = (MatchAssertion) restTestSuite.getTestSections().get(0).getExecutableSections().get(11);
-        assertThat(matchAssertion.getField(), equalTo("_source"));
-        assertThat(matchAssertion.getExpectedValue(), instanceOf(Map.class));
-        assertThat(((Map) matchAssertion.getExpectedValue()).get("foo").toString(), equalTo("bar"));
-    }
-
-    public void testParseTestMultipleTestSections() throws Exception {
-        parser = YamlXContent.yamlXContent.createParser(
-        "---\n" +
-                "\"Missing document (partial doc)\":\n" +
-                "\n" +
-                "  - do:\n" +
-                "      catch:      missing\n" +
-                "      update:\n" +
-                "          index:  test_1\n" +
-                "          type:   test\n" +
-                "          id:     1\n" +
-                "          body:   { doc: { foo: bar } }\n" +
-                "\n" +
-                "  - do:\n" +
-                "      update:\n" +
-                "          index: test_1\n" +
-                "          type:  test\n" +
-                "          id:    1\n" +
-                "          body:  { doc: { foo: bar } }\n" +
-                "          ignore: 404\n" +
-                "\n" +
-                "---\n" +
-                "\"Missing document (script)\":\n" +
-                "\n" +
-                "\n" +
-                "  - do:\n" +
-                "      catch:      missing\n" +
-                "      update:\n" +
-                "          index:  test_1\n" +
-                "          type:   test\n" +
-                "          id:     1\n" +
-                "          body:\n" +
-                "            script: \"ctx._source.foo = bar\"\n" +
-                "            params: { bar: 'xxx' }\n" +
-                "\n" +
-                "  - do:\n" +
-                "      update:\n" +
-                "          index:  test_1\n" +
-                "          type:   test\n" +
-                "          id:     1\n" +
-                "          ignore: 404\n" +
-                "          body:\n" +
-                "            script:       \"ctx._source.foo = bar\"\n" +
-                "            params:       { bar: 'xxx' }\n"
-        );
-
-        ClientYamlTestSuiteParser testParser = new ClientYamlTestSuiteParser();
-        ClientYamlTestSuite restTestSuite = testParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(restTestSuite, notNullValue());
-        assertThat(restTestSuite.getName(), equalTo("suite"));
-
-        assertThat(restTestSuite.getSetupSection().isEmpty(), equalTo(true));
-
-        assertThat(restTestSuite.getTestSections().size(), equalTo(2));
-
-        assertThat(restTestSuite.getTestSections().get(0).getName(), equalTo("Missing document (partial doc)"));
-        assertThat(restTestSuite.getTestSections().get(0).getSkipSection().isEmpty(), equalTo(true));
-        assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().size(), equalTo(2));
-
-        assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(0), instanceOf(DoSection.class));
-        DoSection doSection = (DoSection) restTestSuite.getTestSections().get(0).getExecutableSections().get(0);
-        assertThat(doSection.getCatch(), equalTo("missing"));
-        assertThat(doSection.getApiCallSection().getApi(), equalTo("update"));
-        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3));
-        assertThat(doSection.getApiCallSection().hasBody(), equalTo(true));
-        assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(1), instanceOf(DoSection.class));
-        doSection = (DoSection) restTestSuite.getTestSections().get(0).getExecutableSections().get(1);
-        assertThat(doSection.getCatch(), nullValue());
-        assertThat(doSection.getApiCallSection().getApi(), equalTo("update"));
-        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(4));
-        assertThat(doSection.getApiCallSection().hasBody(), equalTo(true));
-
-        assertThat(restTestSuite.getTestSections().get(1).getName(), equalTo("Missing document (script)"));
-        assertThat(restTestSuite.getTestSections().get(1).getSkipSection().isEmpty(), equalTo(true));
-        assertThat(restTestSuite.getTestSections().get(1).getExecutableSections().size(), equalTo(2));
-        assertThat(restTestSuite.getTestSections().get(1).getExecutableSections().get(0), instanceOf(DoSection.class));
-        assertThat(restTestSuite.getTestSections().get(1).getExecutableSections().get(1), instanceOf(DoSection.class));
-        doSection = (DoSection) restTestSuite.getTestSections().get(1).getExecutableSections().get(0);
-        assertThat(doSection.getCatch(), equalTo("missing"));
-        assertThat(doSection.getApiCallSection().getApi(), equalTo("update"));
-        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3));
-        assertThat(doSection.getApiCallSection().hasBody(), equalTo(true));
-        assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(1), instanceOf(DoSection.class));
-        doSection = (DoSection) restTestSuite.getTestSections().get(1).getExecutableSections().get(1);
-        assertThat(doSection.getCatch(), nullValue());
-        assertThat(doSection.getApiCallSection().getApi(), equalTo("update"));
-        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(4));
-        assertThat(doSection.getApiCallSection().hasBody(), equalTo(true));
-    }
-
-    public void testParseTestDuplicateTestSections() throws Exception {
-        parser = YamlXContent.yamlXContent.createParser(
-                "---\n" +
-                        "\"Missing document (script)\":\n" +
-                        "\n" +
-                        "  - do:\n" +
-                        "      catch:      missing\n" +
-                        "      update:\n" +
-                        "          index:  test_1\n" +
-                        "          type:   test\n" +
-                        "          id:     1\n" +
-                        "          body:   { doc: { foo: bar } }\n" +
-                        "\n" +
-                        "---\n" +
-                        "\"Missing document (script)\":\n" +
-                        "\n" +
-                        "\n" +
-                        "  - do:\n" +
-                        "      catch:      missing\n" +
-                        "      update:\n" +
-                        "          index:  test_1\n" +
-                        "          type:   test\n" +
-                        "          id:     1\n" +
-                        "          body:\n" +
-                        "            script: \"ctx._source.foo = bar\"\n" +
-                        "            params: { bar: 'xxx' }\n" +
-                        "\n"
-        );
-
-        ClientYamlTestSuiteParser testParser = new ClientYamlTestSuiteParser();
-        try {
-            testParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-            fail("Expected RestTestParseException");
-        } catch (ClientYamlTestParseException e) {
-            assertThat(e.getMessage(), containsString("duplicate test section"));
-        }
-    }
-}
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/DoSectionParserTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/DoSectionParserTests.java
deleted file mode 100644
index 0cecc508bf512..0000000000000
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/DoSectionParserTests.java
+++ /dev/null
@@ -1,452 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.json.JsonXContent;
-import org.elasticsearch.common.xcontent.yaml.YamlXContent;
-import org.elasticsearch.test.rest.yaml.parser.DoSectionParser;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestSuiteParseContext;
-import org.elasticsearch.test.rest.yaml.section.ApiCallSection;
-import org.elasticsearch.test.rest.yaml.section.DoSection;
-import org.hamcrest.MatcherAssert;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Map;
-
-import static java.util.Collections.singletonList;
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.notNullValue;
-import static org.hamcrest.Matchers.nullValue;
-
-public class DoSectionParserTests extends AbstractParserTestCase {
-    public void testParseDoSectionNoBody() throws Exception {
-        parser = YamlXContent.yamlXContent.createParser(
-                "get:\n" +
-                "    index:    test_index\n" +
-                "    type:    test_type\n" +
-                "    id:        1"
-        );
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-        ApiCallSection apiCallSection = doSection.getApiCallSection();
-
-        assertThat(apiCallSection, notNullValue());
-        assertThat(apiCallSection.getApi(), equalTo("get"));
-        assertThat(apiCallSection.getParams().size(), equalTo(3));
-        assertThat(apiCallSection.getParams().get("index"), equalTo("test_index"));
-        assertThat(apiCallSection.getParams().get("type"), equalTo("test_type"));
-        assertThat(apiCallSection.getParams().get("id"), equalTo("1"));
-        assertThat(apiCallSection.hasBody(), equalTo(false));
-    }
-
-    public void testParseDoSectionNoParamsNoBody() throws Exception {
-        parser = YamlXContent.yamlXContent.createParser(
-                "cluster.node_info: {}"
-        );
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-        ApiCallSection apiCallSection = doSection.getApiCallSection();
-
-        assertThat(apiCallSection, notNullValue());
-        assertThat(apiCallSection.getApi(), equalTo("cluster.node_info"));
-        assertThat(apiCallSection.getParams().size(), equalTo(0));
-        assertThat(apiCallSection.hasBody(), equalTo(false));
-    }
-
-    public void testParseDoSectionWithJsonBody() throws Exception {
-        String body = "{ \"include\": { \"field1\": \"v1\", \"field2\": \"v2\" }, \"count\": 1 }";
-        parser = YamlXContent.yamlXContent.createParser(
-                "index:\n" +
-                "    index:  test_1\n" +
-                "    type:   test\n" +
-                "    id:     1\n" +
-                "    body:   " + body
-        );
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-        ApiCallSection apiCallSection = doSection.getApiCallSection();
-
-        assertThat(apiCallSection, notNullValue());
-        assertThat(apiCallSection.getApi(), equalTo("index"));
-        assertThat(apiCallSection.getParams().size(), equalTo(3));
-        assertThat(apiCallSection.getParams().get("index"), equalTo("test_1"));
-        assertThat(apiCallSection.getParams().get("type"), equalTo("test"));
-        assertThat(apiCallSection.getParams().get("id"), equalTo("1"));
-        assertThat(apiCallSection.hasBody(), equalTo(true));
-
-        assertJsonEquals(apiCallSection.getBodies().get(0), body);
-    }
-
-    public void testParseDoSectionWithJsonMultipleBodiesAsLongString() throws Exception {
-        String bodies[] = new String[]{
-                "{ \"index\": { \"_index\":\"test_index\", \"_type\":\"test_type\", \"_id\":\"test_id\" } }\n",
-                "{ \"f1\":\"v1\", \"f2\":42 }\n",
-                "{ \"index\": { \"_index\":\"test_index2\", \"_type\":\"test_type2\", \"_id\":\"test_id2\" } }\n",
-                "{ \"f1\":\"v2\", \"f2\":47 }\n"
-        };
-        parser = YamlXContent.yamlXContent.createParser(
-                "bulk:\n" +
-                "    refresh: true\n" +
-                "    body: |\n" +
-                "        " + bodies[0] +
-                "        " + bodies[1] +
-                "        " + bodies[2] +
-                "        " + bodies[3]
-        );
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-        ApiCallSection apiCallSection = doSection.getApiCallSection();
-
-        assertThat(apiCallSection, notNullValue());
-        assertThat(apiCallSection.getApi(), equalTo("bulk"));
-        assertThat(apiCallSection.getParams().size(), equalTo(1));
-        assertThat(apiCallSection.getParams().get("refresh"), equalTo("true"));
-        assertThat(apiCallSection.hasBody(), equalTo(true));
-        assertThat(apiCallSection.getBodies().size(), equalTo(4));
-    }
-
-    public void testParseDoSectionWithJsonMultipleBodiesRepeatedProperty() throws Exception {
-        String[] bodies = new String[] {
-                "{ \"index\": { \"_index\":\"test_index\", \"_type\":\"test_type\", \"_id\":\"test_id\" } }",
-                "{ \"f1\":\"v1\", \"f2\":42 }",
-        };
-        parser = YamlXContent.yamlXContent.createParser(
-                "bulk:\n" +
-                "    refresh: true\n" +
-                "    body: \n" +
-                "        " + bodies[0] + "\n" +
-                "    body: \n" +
-                "        " + bodies[1]
-        );
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-        ApiCallSection apiCallSection = doSection.getApiCallSection();
-
-        assertThat(apiCallSection, notNullValue());
-        assertThat(apiCallSection.getApi(), equalTo("bulk"));
-        assertThat(apiCallSection.getParams().size(), equalTo(1));
-        assertThat(apiCallSection.getParams().get("refresh"), equalTo("true"));
-        assertThat(apiCallSection.hasBody(), equalTo(true));
-        assertThat(apiCallSection.getBodies().size(), equalTo(bodies.length));
-        for (int i = 0; i < bodies.length; i++) {
-            assertJsonEquals(apiCallSection.getBodies().get(i), bodies[i]);
-        }
-    }
-
-    public void testParseDoSectionWithYamlBody() throws Exception {
-        parser = YamlXContent.yamlXContent.createParser(
-                "search:\n" +
-                "    body:\n" +
-                "        \"_source\": [ include.field1, include.field2 ]\n" +
-                "        \"query\": { \"match_all\": {} }"
-        );
-        String body = "{ \"_source\": [ \"include.field1\", \"include.field2\" ], \"query\": { \"match_all\": {} }}";
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-        ApiCallSection apiCallSection = doSection.getApiCallSection();
-
-        assertThat(apiCallSection, notNullValue());
-        assertThat(apiCallSection.getApi(), equalTo("search"));
-        assertThat(apiCallSection.getParams().size(), equalTo(0));
-        assertThat(apiCallSection.hasBody(), equalTo(true));
-        assertThat(apiCallSection.getBodies().size(), equalTo(1));
-        assertJsonEquals(apiCallSection.getBodies().get(0), body);
-    }
-
-    public void testParseDoSectionWithYamlMultipleBodies() throws Exception {
-        parser = YamlXContent.yamlXContent.createParser(
-                "bulk:\n" +
-                "    refresh: true\n" +
-                "    body:\n" +
-                "        - index:\n" +
-                "            _index: test_index\n" +
-                "            _type:  test_type\n" +
-                "            _id:    test_id\n" +
-                "        - f1: v1\n" +
-                "          f2: 42\n" +
-                "        - index:\n" +
-                "            _index: test_index2\n" +
-                "            _type:  test_type2\n" +
-                "            _id:    test_id2\n" +
-                "        - f1: v2\n" +
-                "          f2: 47"
-        );
-        String[] bodies = new String[4];
-        bodies[0] = "{\"index\": {\"_index\": \"test_index\", \"_type\":  \"test_type\", \"_id\": \"test_id\"}}";
-        bodies[1] = "{ \"f1\":\"v1\", \"f2\": 42 }";
-        bodies[2] = "{\"index\": {\"_index\": \"test_index2\", \"_type\":  \"test_type2\", \"_id\": \"test_id2\"}}";
-        bodies[3] = "{ \"f1\":\"v2\", \"f2\": 47 }";
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-        ApiCallSection apiCallSection = doSection.getApiCallSection();
-
-        assertThat(apiCallSection, notNullValue());
-        assertThat(apiCallSection.getApi(), equalTo("bulk"));
-        assertThat(apiCallSection.getParams().size(), equalTo(1));
-        assertThat(apiCallSection.getParams().get("refresh"), equalTo("true"));
-        assertThat(apiCallSection.hasBody(), equalTo(true));
-        assertThat(apiCallSection.getBodies().size(), equalTo(bodies.length));
-
-        for (int i = 0; i < bodies.length; i++) {
-            assertJsonEquals(apiCallSection.getBodies().get(i), bodies[i]);
-        }
-    }
-
-    public void testParseDoSectionWithYamlMultipleBodiesRepeatedProperty() throws Exception {
-        parser = YamlXContent.yamlXContent.createParser(
-                "bulk:\n" +
-                "    refresh: true\n" +
-                "    body:\n" +
-                "        index:\n" +
-                "            _index: test_index\n" +
-                "            _type:  test_type\n" +
-                "            _id:    test_id\n" +
-                "    body:\n" +
-                "        f1: v1\n" +
-                "        f2: 42\n"
-        );
-        String[] bodies = new String[2];
-        bodies[0] = "{\"index\": {\"_index\": \"test_index\", \"_type\":  \"test_type\", \"_id\": \"test_id\"}}";
-        bodies[1] = "{ \"f1\":\"v1\", \"f2\": 42 }";
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-        ApiCallSection apiCallSection = doSection.getApiCallSection();
-
-        assertThat(apiCallSection, notNullValue());
-        assertThat(apiCallSection.getApi(), equalTo("bulk"));
-        assertThat(apiCallSection.getParams().size(), equalTo(1));
-        assertThat(apiCallSection.getParams().get("refresh"), equalTo("true"));
-        assertThat(apiCallSection.hasBody(), equalTo(true));
-        assertThat(apiCallSection.getBodies().size(), equalTo(bodies.length));
-
-        for (int i = 0; i < bodies.length; i++) {
-            assertJsonEquals(apiCallSection.getBodies().get(i), bodies[i]);
-        }
-    }
-
-    public void testParseDoSectionWithYamlBodyMultiGet() throws Exception {
-        parser = YamlXContent.yamlXContent.createParser(
-                "mget:\n" +
-                "    body:\n" +
-                "        docs:\n" +
-                "            - { _index: test_2, _type: test, _id: 1}\n" +
-                "            - { _index: test_1, _type: none, _id: 1}"
-        );
-        String body = "{ \"docs\": [ " +
-                "{\"_index\": \"test_2\", \"_type\":\"test\", \"_id\":1}, " +
-                "{\"_index\": \"test_1\", \"_type\":\"none\", \"_id\":1} " +
-                "]}";
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-        ApiCallSection apiCallSection = doSection.getApiCallSection();
-
-        assertThat(apiCallSection, notNullValue());
-        assertThat(apiCallSection.getApi(), equalTo("mget"));
-        assertThat(apiCallSection.getParams().size(), equalTo(0));
-        assertThat(apiCallSection.hasBody(), equalTo(true));
-        assertThat(apiCallSection.getBodies().size(), equalTo(1));
-        assertJsonEquals(apiCallSection.getBodies().get(0), body);
-    }
-
-    public void testParseDoSectionWithBodyStringified() throws Exception {
-        parser = YamlXContent.yamlXContent.createParser(
-                "index:\n" +
-                "    index:  test_1\n" +
-                "    type:   test\n" +
-                "    id:     1\n" +
-                "    body:   \"{ \\\"_source\\\": true, \\\"query\\\": { \\\"match_all\\\": {} } }\""
-        );
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-        ApiCallSection apiCallSection = doSection.getApiCallSection();
-
-        assertThat(apiCallSection, notNullValue());
-        assertThat(apiCallSection.getApi(), equalTo("index"));
-        assertThat(apiCallSection.getParams().size(), equalTo(3));
-        assertThat(apiCallSection.getParams().get("index"), equalTo("test_1"));
-        assertThat(apiCallSection.getParams().get("type"), equalTo("test"));
-        assertThat(apiCallSection.getParams().get("id"), equalTo("1"));
-        assertThat(apiCallSection.hasBody(), equalTo(true));
-        assertThat(apiCallSection.getBodies().size(), equalTo(1));
-        //stringified body is taken as is
-        assertJsonEquals(apiCallSection.getBodies().get(0), "{ \"_source\": true, \"query\": { \"match_all\": {} } }");
-    }
-
-    public void testParseDoSectionWithBodiesStringifiedAndNot() throws Exception {
-        parser = YamlXContent.yamlXContent.createParser(
-                "index:\n" +
-                "    body:\n" +
-                "        - \"{ \\\"_source\\\": true, \\\"query\\\": { \\\"match_all\\\": {} } }\"\n" +
-                "        - { size: 100, query: { match_all: {} } }"
-        );
-
-        String body = "{ \"size\": 100, \"query\": { \"match_all\": {} } }";
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-        ApiCallSection apiCallSection = doSection.getApiCallSection();
-
-        assertThat(apiCallSection.getApi(), equalTo("index"));
-        assertThat(apiCallSection.getParams().size(), equalTo(0));
-        assertThat(apiCallSection.hasBody(), equalTo(true));
-        assertThat(apiCallSection.getBodies().size(), equalTo(2));
-        //stringified body is taken as is
-        assertJsonEquals(apiCallSection.getBodies().get(0), "{ \"_source\": true, \"query\": { \"match_all\": {} } }");
-        assertJsonEquals(apiCallSection.getBodies().get(1), body);
-    }
-
-    public void testParseDoSectionWithCatch() throws Exception {
-        parser = YamlXContent.yamlXContent.createParser(
-                "catch: missing\n" +
-                "indices.get_warmer:\n" +
-                "    index: test_index\n" +
-                "    name: test_warmer"
-        );
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(doSection.getCatch(), equalTo("missing"));
-        assertThat(doSection.getApiCallSection(), notNullValue());
-        assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_warmer"));
-        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2));
-        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
-    }
-
-    public void testParseDoSectionWithHeaders() throws Exception {
-        parser = YamlXContent.yamlXContent.createParser(
-                "headers:\n" +
-                "    Authorization: \"thing one\"\n" +
-                "    Content-Type: \"application/json\"\n" +
-                "indices.get_warmer:\n" +
-                "    index: test_index\n" +
-                "    name: test_warmer"
-        );
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(doSection.getApiCallSection(), notNullValue());
-        assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_warmer"));
-        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2));
-        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
-        assertThat(doSection.getApiCallSection().getHeaders(), notNullValue());
-        assertThat(doSection.getApiCallSection().getHeaders().size(), equalTo(2));
-        assertThat(doSection.getApiCallSection().getHeaders().get("Authorization"), equalTo("thing one"));
-        assertThat(doSection.getApiCallSection().getHeaders().get("Content-Type"), equalTo("application/json"));
-    }
-
-    public void testParseDoSectionWithoutClientCallSection() throws Exception {
-        parser = YamlXContent.yamlXContent.createParser(
-                "catch: missing\n"
-        );
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        try {
-            doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-            fail("Expected RestTestParseException");
-        } catch (ClientYamlTestParseException e) {
-            assertThat(e.getMessage(), is("client call section is mandatory within a do section"));
-        }
-    }
-
-    public void testParseDoSectionMultivaluedField() throws Exception {
-        parser = YamlXContent.yamlXContent.createParser(
-                "indices.get_field_mapping:\n" +
-                "        index: test_index\n" +
-                "        type: test_type\n" +
-                "        field: [ text , text1 ]"
-        );
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(doSection.getCatch(), nullValue());
-        assertThat(doSection.getApiCallSection(), notNullValue());
-        assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_field_mapping"));
-        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3));
-        assertThat(doSection.getApiCallSection().getParams().get("index"), equalTo("test_index"));
-        assertThat(doSection.getApiCallSection().getParams().get("type"), equalTo("test_type"));
-        assertThat(doSection.getApiCallSection().getParams().get("field"), equalTo("text,text1"));
-        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
-        assertThat(doSection.getApiCallSection().getBodies().size(), equalTo(0));
-    }
-
-    public void testParseDoSectionExpectedWarnings() throws Exception {
-        parser = YamlXContent.yamlXContent.createParser(
-                "indices.get_field_mapping:\n" +
-                "        index: test_index\n" +
-                "        type: test_type\n" +
-                "warnings:\n" +
-                "    - some test warning they are typically pretty long\n" +
-                "    - some other test warning somtimes they have [in] them"
-        );
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(doSection.getCatch(), nullValue());
-        assertThat(doSection.getApiCallSection(), notNullValue());
-        assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_field_mapping"));
-        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2));
-        assertThat(doSection.getApiCallSection().getParams().get("index"), equalTo("test_index"));
-        assertThat(doSection.getApiCallSection().getParams().get("type"), equalTo("test_type"));
-        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
-        assertThat(doSection.getApiCallSection().getBodies().size(), equalTo(0));
-        assertThat(doSection.getExpectedWarningHeaders(), equalTo(Arrays.asList(
-                "some test warning they are typically pretty long",
-                "some other test warning somtimes they have [in] them")));
-
-        parser = YamlXContent.yamlXContent.createParser(
-                "indices.get_field_mapping:\n" +
-                "        index: test_index\n" +
-                "warnings:\n" +
-                "    - just one entry this time"
-        );
-
-        doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-        assertThat(doSection.getCatch(), nullValue());
-        assertThat(doSection.getApiCallSection(), notNullValue());
-        assertThat(doSection.getExpectedWarningHeaders(), equalTo(singletonList(
-                "just one entry this time")));
-
-    }
-
-    private static void assertJsonEquals(Map<String, Object> actual, String expected) throws IOException {
-        Map<String,Object> expectedMap;
-        try (XContentParser parser = JsonXContent.jsonXContent.createParser(expected)) {
-            expectedMap = parser.mapOrdered();
-        }
-        MatcherAssert.assertThat(actual, equalTo(expectedMap));
-    }
-}
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/SetSectionParserTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/SetSectionParserTests.java
deleted file mode 100644
index 386cdd6deb414..0000000000000
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/SetSectionParserTests.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.common.xcontent.yaml.YamlXContent;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestSuiteParseContext;
-import org.elasticsearch.test.rest.yaml.parser.SetSectionParser;
-import org.elasticsearch.test.rest.yaml.section.SetSection;
-
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.notNullValue;
-
-public class SetSectionParserTests extends AbstractParserTestCase {
-    public void testParseSetSectionSingleValue() throws Exception {
-        parser = YamlXContent.yamlXContent.createParser(
-                        "{ _id: id }"
-        );
-
-        SetSectionParser setSectionParser = new SetSectionParser();
-
-        SetSection setSection = setSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(setSection, notNullValue());
-        assertThat(setSection.getStash(), notNullValue());
-        assertThat(setSection.getStash().size(), equalTo(1));
-        assertThat(setSection.getStash().get("_id"), equalTo("id"));
-    }
-
-    public void testParseSetSectionMultipleValues() throws Exception {
-        parser = YamlXContent.yamlXContent.createParser(
-                "{ _id: id, _type: type, _index: index }"
-        );
-
-        SetSectionParser setSectionParser = new SetSectionParser();
-
-        SetSection setSection = setSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(setSection, notNullValue());
-        assertThat(setSection.getStash(), notNullValue());
-        assertThat(setSection.getStash().size(), equalTo(3));
-        assertThat(setSection.getStash().get("_id"), equalTo("id"));
-        assertThat(setSection.getStash().get("_type"), equalTo("type"));
-        assertThat(setSection.getStash().get("_index"), equalTo("index"));
-    }
-
-    public void testParseSetSectionNoValues() throws Exception {
-        parser = YamlXContent.yamlXContent.createParser(
-                "{ }"
-        );
-
-        SetSectionParser setSectionParser = new SetSectionParser();
-        try {
-            setSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-            fail("Expected RestTestParseException");
-        } catch (ClientYamlTestParseException e) {
-            assertThat(e.getMessage(), is("set section must set at least a value"));
-        }
-    }
-}
\ No newline at end of file
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/SetupSectionParserTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/SetupSectionParserTests.java
deleted file mode 100644
index fa7c34eecde57..0000000000000
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/SetupSectionParserTests.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.common.xcontent.yaml.YamlXContent;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestSuiteParseContext;
-import org.elasticsearch.test.rest.yaml.parser.SetupSectionParser;
-import org.elasticsearch.test.rest.yaml.section.SetupSection;
-
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.notNullValue;
-
-public class SetupSectionParserTests extends AbstractParserTestCase {
-    public void testParseSetupSection() throws Exception {
-        parser = YamlXContent.yamlXContent.createParser(
-                "  - do:\n" +
-                "      index1:\n" +
-                "        index:  test_1\n" +
-                "        type:   test\n" +
-                "        id:     1\n" +
-                "        body:   { \"include\": { \"field1\": \"v1\", \"field2\": \"v2\" }, \"count\": 1 }\n" +
-                "  - do:\n" +
-                "      index2:\n" +
-                "        index:  test_1\n" +
-                "        type:   test\n" +
-                "        id:     2\n" +
-                "        body:   { \"include\": { \"field1\": \"v1\", \"field2\": \"v2\" }, \"count\": 1 }\n"
-        );
-
-        SetupSectionParser setupSectionParser = new SetupSectionParser();
-        SetupSection setupSection = setupSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(setupSection, notNullValue());
-        assertThat(setupSection.getSkipSection().isEmpty(), equalTo(true));
-        assertThat(setupSection.getDoSections().size(), equalTo(2));
-        assertThat(setupSection.getDoSections().get(0).getApiCallSection().getApi(), equalTo("index1"));
-        assertThat(setupSection.getDoSections().get(1).getApiCallSection().getApi(), equalTo("index2"));
-    }
-
-    public void testParseSetupAndSkipSectionNoSkip() throws Exception {
-        parser = YamlXContent.yamlXContent.createParser(
-                "  - skip:\n" +
-                        "      version:  \"2.0.0 - 2.3.0\"\n" +
-                        "      reason:   \"Update doesn't return metadata fields, waiting for #3259\"\n" +
-                        "  - do:\n" +
-                        "      index1:\n" +
-                        "        index:  test_1\n" +
-                        "        type:   test\n" +
-                        "        id:     1\n" +
-                        "        body:   { \"include\": { \"field1\": \"v1\", \"field2\": \"v2\" }, \"count\": 1 }\n" +
-                        "  - do:\n" +
-                        "      index2:\n" +
-                        "        index:  test_1\n" +
-                        "        type:   test\n" +
-                        "        id:     2\n" +
-                        "        body:   { \"include\": { \"field1\": \"v1\", \"field2\": \"v2\" }, \"count\": 1 }\n"
-        );
-
-        SetupSectionParser setupSectionParser = new SetupSectionParser();
-        SetupSection setupSection = setupSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(setupSection, notNullValue());
-        assertThat(setupSection.getSkipSection().isEmpty(), equalTo(false));
-        assertThat(setupSection.getSkipSection(), notNullValue());
-        assertThat(setupSection.getSkipSection().getLowerVersion(), equalTo(Version.V_2_0_0));
-        assertThat(setupSection.getSkipSection().getUpperVersion(), equalTo(Version.V_2_3_0));
-        assertThat(setupSection.getSkipSection().getReason(), equalTo("Update doesn't return metadata fields, waiting for #3259"));
-        assertThat(setupSection.getDoSections().size(), equalTo(2));
-        assertThat(setupSection.getDoSections().get(0).getApiCallSection().getApi(), equalTo("index1"));
-        assertThat(setupSection.getDoSections().get(1).getApiCallSection().getApi(), equalTo("index2"));
-    }
-}
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/SkipSectionParserTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/SkipSectionParserTests.java
deleted file mode 100644
index f5d46cdd3d629..0000000000000
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/SkipSectionParserTests.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.common.xcontent.yaml.YamlXContent;
-import org.elasticsearch.test.VersionUtils;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestSuiteParseContext;
-import org.elasticsearch.test.rest.yaml.parser.SkipSectionParser;
-import org.elasticsearch.test.rest.yaml.section.SkipSection;
-
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.notNullValue;
-import static org.hamcrest.Matchers.nullValue;
-
-public class SkipSectionParserTests extends AbstractParserTestCase {
-    public void testParseSkipSectionVersionNoFeature() throws Exception {
-        parser = YamlXContent.yamlXContent.createParser(
-                "version:     \" - 2.1.0\"\n" +
-                "reason:      Delete ignores the parent param"
-        );
-
-        SkipSectionParser skipSectionParser = new SkipSectionParser();
-
-        SkipSection skipSection = skipSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(skipSection, notNullValue());
-        assertThat(skipSection.getLowerVersion(), equalTo(VersionUtils.getFirstVersion()));
-        assertThat(skipSection.getUpperVersion(), equalTo(Version.V_2_1_0));
-        assertThat(skipSection.getFeatures().size(), equalTo(0));
-        assertThat(skipSection.getReason(), equalTo("Delete ignores the parent param"));
-    }
-
-    public void testParseSkipSectionAllVersions() throws Exception {
-        parser = YamlXContent.yamlXContent.createParser(
-            "version:     \" all \"\n" +
-            "reason:      Delete ignores the parent param"
-        );
-
-        SkipSectionParser skipSectionParser = new SkipSectionParser();
-
-        SkipSection skipSection = skipSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(skipSection, notNullValue());
-        assertThat(skipSection.getLowerVersion(), equalTo(VersionUtils.getFirstVersion()));
-        assertThat(skipSection.getUpperVersion(), equalTo(Version.CURRENT));
-        assertThat(skipSection.getFeatures().size(), equalTo(0));
-        assertThat(skipSection.getReason(), equalTo("Delete ignores the parent param"));
-    }
-
-    public void testParseSkipSectionFeatureNoVersion() throws Exception {
-        parser = YamlXContent.yamlXContent.createParser(
-                "features:     regex"
-        );
-
-        SkipSectionParser skipSectionParser = new SkipSectionParser();
-
-        SkipSection skipSection = skipSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(skipSection, notNullValue());
-        assertThat(skipSection.isVersionCheck(), equalTo(false));
-        assertThat(skipSection.getFeatures().size(), equalTo(1));
-        assertThat(skipSection.getFeatures().get(0), equalTo("regex"));
-        assertThat(skipSection.getReason(), nullValue());
-    }
-
-    public void testParseSkipSectionFeaturesNoVersion() throws Exception {
-        parser = YamlXContent.yamlXContent.createParser(
-                "features:     [regex1,regex2,regex3]"
-        );
-
-        SkipSectionParser skipSectionParser = new SkipSectionParser();
-
-        SkipSection skipSection = skipSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(skipSection, notNullValue());
-        assertThat(skipSection.isVersionCheck(), equalTo(false));
-        assertThat(skipSection.getFeatures().size(), equalTo(3));
-        assertThat(skipSection.getFeatures().get(0), equalTo("regex1"));
-        assertThat(skipSection.getFeatures().get(1), equalTo("regex2"));
-        assertThat(skipSection.getFeatures().get(2), equalTo("regex3"));
-        assertThat(skipSection.getReason(), nullValue());
-    }
-
-    public void testParseSkipSectionBothFeatureAndVersion() throws Exception {
-        parser = YamlXContent.yamlXContent.createParser(
-                "version:     \" - 0.90.2\"\n" +
-                "features:     regex\n" +
-                "reason:      Delete ignores the parent param"
-        );
-
-        SkipSectionParser skipSectionParser = new SkipSectionParser();
-
-        try {
-            skipSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-            fail("Expected RestTestParseException");
-        } catch (ClientYamlTestParseException e) {
-            assertThat(e.getMessage(), is("version or features are mutually exclusive"));
-        }
-    }
-
-    public void testParseSkipSectionNoReason() throws Exception {
-        parser = YamlXContent.yamlXContent.createParser(
-                "version:     \" - 0.90.2\"\n"
-        );
-
-        SkipSectionParser skipSectionParser = new SkipSectionParser();
-        try {
-            skipSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-            fail("Expected RestTestParseException");
-        } catch (ClientYamlTestParseException e) {
-            assertThat(e.getMessage(), is("reason is mandatory within skip version section"));
-        }
-    }
-
-    public void testParseSkipSectionNoVersionNorFeature() throws Exception {
-        parser = YamlXContent.yamlXContent.createParser(
-                "reason:      Delete ignores the parent param\n"
-        );
-
-        SkipSectionParser skipSectionParser = new SkipSectionParser();
-        try {
-            skipSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-            fail("Expected RestTestParseException");
-        } catch (ClientYamlTestParseException e) {
-            assertThat(e.getMessage(), is("version or features is mandatory within skip section"));
-        }
-    }
-}
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/TeardownSectionParserTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/TeardownSectionParserTests.java
deleted file mode 100644
index 4ad47e35e4807..0000000000000
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/TeardownSectionParserTests.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.common.xcontent.yaml.YamlXContent;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestSuiteParseContext;
-import org.elasticsearch.test.rest.yaml.parser.TeardownSectionParser;
-import org.elasticsearch.test.rest.yaml.section.TeardownSection;
-
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.notNullValue;
-
-/**
- * Unit tests for the teardown section parser
- */
-public class TeardownSectionParserTests extends AbstractParserTestCase {
-
-    public void testParseTeardownSection() throws Exception {
-        parser = YamlXContent.yamlXContent.createParser(
-                "  - do:\n" +
-                "      delete:\n" +
-                "        index: foo\n" +
-                "        type: doc\n" +
-                "        id: 1\n" +
-                "        ignore: 404\n" +
-                "  - do:\n" +
-                "      delete2:\n" +
-                "        index: foo\n" +
-                "        type: doc\n" +
-                "        id: 1\n" +
-                "        ignore: 404"
-        );
-
-        TeardownSectionParser teardownSectionParser = new TeardownSectionParser();
-        TeardownSection section = teardownSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(section, notNullValue());
-        assertThat(section.getSkipSection().isEmpty(), equalTo(true));
-        assertThat(section.getDoSections().size(), equalTo(2));
-        assertThat(section.getDoSections().get(0).getApiCallSection().getApi(), equalTo("delete"));
-        assertThat(section.getDoSections().get(1).getApiCallSection().getApi(), equalTo("delete2"));
-    }
-
-    public void testParseWithSkip() throws Exception {
-        parser = YamlXContent.yamlXContent.createParser(
-            "  - skip:\n" +
-                "      version:  \"2.0.0 - 2.3.0\"\n" +
-                "      reason:   \"there is a reason\"\n" +
-                "  - do:\n" +
-                "      delete:\n" +
-                "        index: foo\n" +
-                "        type: doc\n" +
-                "        id: 1\n" +
-                "        ignore: 404\n" +
-                "  - do:\n" +
-                "      delete2:\n" +
-                "        index: foo\n" +
-                "        type: doc\n" +
-                "        id: 1\n" +
-                "        ignore: 404"
-        );
-
-        TeardownSectionParser teardownSectionParser = new TeardownSectionParser();
-        TeardownSection section = teardownSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(section, notNullValue());
-        assertThat(section.getSkipSection().isEmpty(), equalTo(false));
-        assertThat(section.getSkipSection().getLowerVersion(), equalTo(Version.V_2_0_0));
-        assertThat(section.getSkipSection().getUpperVersion(), equalTo(Version.V_2_3_0));
-        assertThat(section.getSkipSection().getReason(), equalTo("there is a reason"));
-        assertThat(section.getDoSections().size(), equalTo(2));
-        assertThat(section.getDoSections().get(0).getApiCallSection().getApi(), equalTo("delete"));
-        assertThat(section.getDoSections().get(1).getApiCallSection().getApi(), equalTo("delete2"));
-    }
-}
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/TestSectionParserTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/TestSectionParserTests.java
deleted file mode 100644
index aa23cabe07b78..0000000000000
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/TestSectionParserTests.java
+++ /dev/null
@@ -1,251 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.common.xcontent.yaml.YamlXContent;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestSectionParser;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestSuiteParseContext;
-import org.elasticsearch.test.rest.yaml.section.DoSection;
-import org.elasticsearch.test.rest.yaml.section.GreaterThanAssertion;
-import org.elasticsearch.test.rest.yaml.section.IsFalseAssertion;
-import org.elasticsearch.test.rest.yaml.section.IsTrueAssertion;
-import org.elasticsearch.test.rest.yaml.section.LengthAssertion;
-import org.elasticsearch.test.rest.yaml.section.LessThanAssertion;
-import org.elasticsearch.test.rest.yaml.section.MatchAssertion;
-import org.elasticsearch.test.rest.yaml.section.SetSection;
-import org.elasticsearch.test.rest.yaml.section.SkipSection;
-import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection;
-
-import java.util.Map;
-
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.notNullValue;
-import static org.hamcrest.Matchers.nullValue;
-
-public class TestSectionParserTests extends AbstractParserTestCase {
-    public void testParseTestSectionWithDoSection() throws Exception {
-        parser = YamlXContent.yamlXContent.createParser(
-                "\"First test section\": \n" +
-                " - do :\n" +
-                "     catch: missing\n" +
-                "     indices.get_warmer:\n" +
-                "         index: test_index\n" +
-                "         name: test_warmer"
-        );
-
-        ClientYamlTestSectionParser testSectionParser = new ClientYamlTestSectionParser();
-        ClientYamlTestSection testSection = testSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(testSection, notNullValue());
-        assertThat(testSection.getName(), equalTo("First test section"));
-        assertThat(testSection.getSkipSection(), equalTo(SkipSection.EMPTY));
-        assertThat(testSection.getExecutableSections().size(), equalTo(1));
-        DoSection doSection = (DoSection)testSection.getExecutableSections().get(0);
-        assertThat(doSection.getCatch(), equalTo("missing"));
-        assertThat(doSection.getApiCallSection(), notNullValue());
-        assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_warmer"));
-        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2));
-        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
-    }
-
-    public void testParseTestSectionWithDoSetAndSkipSectionsNoSkip() throws Exception {
-        String yaml =
-                "\"First test section\": \n" +
-                        "  - skip:\n" +
-                        "      version:  \"2.0.0 - 2.2.0\"\n" +
-                        "      reason:   \"Update doesn't return metadata fields, waiting for #3259\"\n" +
-                        "  - do :\n" +
-                        "      catch: missing\n" +
-                        "      indices.get_warmer:\n" +
-                        "          index: test_index\n" +
-                        "          name: test_warmer\n" +
-                        "  - set: {_scroll_id: scroll_id}";
-
-
-        ClientYamlTestSectionParser testSectionParser = new ClientYamlTestSectionParser();
-        parser = YamlXContent.yamlXContent.createParser(yaml);
-        ClientYamlTestSection testSection = testSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(testSection, notNullValue());
-        assertThat(testSection.getName(), equalTo("First test section"));
-        assertThat(testSection.getSkipSection(), notNullValue());
-        assertThat(testSection.getSkipSection().getLowerVersion(), equalTo(Version.V_2_0_0));
-        assertThat(testSection.getSkipSection().getUpperVersion(), equalTo(Version.V_2_2_0));
-        assertThat(testSection.getSkipSection().getReason(), equalTo("Update doesn't return metadata fields, waiting for #3259"));
-        assertThat(testSection.getExecutableSections().size(), equalTo(2));
-        DoSection doSection = (DoSection)testSection.getExecutableSections().get(0);
-        assertThat(doSection.getCatch(), equalTo("missing"));
-        assertThat(doSection.getApiCallSection(), notNullValue());
-        assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_warmer"));
-        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2));
-        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
-        SetSection setSection = (SetSection) testSection.getExecutableSections().get(1);
-        assertThat(setSection.getStash().size(), equalTo(1));
-        assertThat(setSection.getStash().get("_scroll_id"), equalTo("scroll_id"));
-    }
-
-    public void testParseTestSectionWithMultipleDoSections() throws Exception {
-        parser = YamlXContent.yamlXContent.createParser(
-                "\"Basic\":\n" +
-                        "\n" +
-                        "  - do:\n" +
-                        "      index:\n" +
-                        "        index: test_1\n" +
-                        "        type:  test\n" +
-                        "        id:    中文\n" +
-                        "        body:  { \"foo\": \"Hello: 中文\" }\n" +
-                        "  - do:\n" +
-                        "      get:\n" +
-                        "        index: test_1\n" +
-                        "        type:  test\n" +
-                        "        id:    中文"
-        );
-
-        ClientYamlTestSectionParser testSectionParser = new ClientYamlTestSectionParser();
-        ClientYamlTestSection testSection = testSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(testSection, notNullValue());
-        assertThat(testSection.getName(), equalTo("Basic"));
-        assertThat(testSection.getSkipSection(), equalTo(SkipSection.EMPTY));
-        assertThat(testSection.getExecutableSections().size(), equalTo(2));
-        DoSection doSection = (DoSection)testSection.getExecutableSections().get(0);
-        assertThat(doSection.getCatch(), nullValue());
-        assertThat(doSection.getApiCallSection(), notNullValue());
-        assertThat(doSection.getApiCallSection().getApi(), equalTo("index"));
-        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3));
-        assertThat(doSection.getApiCallSection().hasBody(), equalTo(true));
-        doSection = (DoSection)testSection.getExecutableSections().get(1);
-        assertThat(doSection.getCatch(), nullValue());
-        assertThat(doSection.getApiCallSection(), notNullValue());
-        assertThat(doSection.getApiCallSection().getApi(), equalTo("get"));
-        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3));
-        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
-    }
-
-    public void testParseTestSectionWithDoSectionsAndAssertions() throws Exception {
-        parser = YamlXContent.yamlXContent.createParser(
-                "\"Basic\":\n" +
-                        "\n" +
-                        "  - do:\n" +
-                        "      index:\n" +
-                        "        index: test_1\n" +
-                        "        type:  test\n" +
-                        "        id:    中文\n" +
-                        "        body:  { \"foo\": \"Hello: 中文\" }\n" +
-                        "\n" +
-                        "  - do:\n" +
-                        "      get:\n" +
-                        "        index: test_1\n" +
-                        "        type:  test\n" +
-                        "        id:    中文\n" +
-                        "\n" +
-                        "  - match: { _index:   test_1 }\n" +
-                        "  - is_true: _source\n" +
-                        "  - match: { _source:  { foo: \"Hello: 中文\" } }\n" +
-                        "\n" +
-                        "  - do:\n" +
-                        "      get:\n" +
-                        "        index: test_1\n" +
-                        "        id:    中文\n" +
-                        "\n" +
-                        "  - length: { _index:   6 }\n" +
-                        "  - is_false: whatever\n" +
-                        "  - gt: { size: 5      }\n" +
-                        "  - lt: { size: 10      }"
-        );
-
-        ClientYamlTestSectionParser testSectionParser = new ClientYamlTestSectionParser();
-        ClientYamlTestSection testSection = testSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(testSection, notNullValue());
-        assertThat(testSection.getName(), equalTo("Basic"));
-        assertThat(testSection.getSkipSection(), equalTo(SkipSection.EMPTY));
-        assertThat(testSection.getExecutableSections().size(), equalTo(10));
-
-        DoSection doSection = (DoSection)testSection.getExecutableSections().get(0);
-        assertThat(doSection.getCatch(), nullValue());
-        assertThat(doSection.getApiCallSection(), notNullValue());
-        assertThat(doSection.getApiCallSection().getApi(), equalTo("index"));
-        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3));
-        assertThat(doSection.getApiCallSection().hasBody(), equalTo(true));
-
-        doSection = (DoSection)testSection.getExecutableSections().get(1);
-        assertThat(doSection.getCatch(), nullValue());
-        assertThat(doSection.getApiCallSection(), notNullValue());
-        assertThat(doSection.getApiCallSection().getApi(), equalTo("get"));
-        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3));
-        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
-
-        MatchAssertion matchAssertion = (MatchAssertion)testSection.getExecutableSections().get(2);
-        assertThat(matchAssertion.getField(), equalTo("_index"));
-        assertThat(matchAssertion.getExpectedValue().toString(), equalTo("test_1"));
-
-        IsTrueAssertion trueAssertion = (IsTrueAssertion)testSection.getExecutableSections().get(3);
-        assertThat(trueAssertion.getField(), equalTo("_source"));
-
-        matchAssertion = (MatchAssertion)testSection.getExecutableSections().get(4);
-        assertThat(matchAssertion.getField(), equalTo("_source"));
-        assertThat(matchAssertion.getExpectedValue(), instanceOf(Map.class));
-        Map map = (Map) matchAssertion.getExpectedValue();
-        assertThat(map.size(), equalTo(1));
-        assertThat(map.get("foo").toString(), equalTo("Hello: 中文"));
-
-        doSection = (DoSection)testSection.getExecutableSections().get(5);
-        assertThat(doSection.getCatch(), nullValue());
-        assertThat(doSection.getApiCallSection(), notNullValue());
-        assertThat(doSection.getApiCallSection().getApi(), equalTo("get"));
-        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2));
-        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
-
-        LengthAssertion lengthAssertion = (LengthAssertion) testSection.getExecutableSections().get(6);
-        assertThat(lengthAssertion.getField(), equalTo("_index"));
-        assertThat(lengthAssertion.getExpectedValue(), instanceOf(Integer.class));
-        assertThat((Integer) lengthAssertion.getExpectedValue(), equalTo(6));
-
-        IsFalseAssertion falseAssertion = (IsFalseAssertion)testSection.getExecutableSections().get(7);
-        assertThat(falseAssertion.getField(), equalTo("whatever"));
-
-        GreaterThanAssertion greaterThanAssertion = (GreaterThanAssertion) testSection.getExecutableSections().get(8);
-        assertThat(greaterThanAssertion.getField(), equalTo("size"));
-        assertThat(greaterThanAssertion.getExpectedValue(), instanceOf(Integer.class));
-        assertThat((Integer) greaterThanAssertion.getExpectedValue(), equalTo(5));
-
-        LessThanAssertion lessThanAssertion = (LessThanAssertion) testSection.getExecutableSections().get(9);
-        assertThat(lessThanAssertion.getField(), equalTo("size"));
-        assertThat(lessThanAssertion.getExpectedValue(), instanceOf(Integer.class));
-        assertThat((Integer) lessThanAssertion.getExpectedValue(), equalTo(10));
-    }
-
-    public void testSmallSection() throws Exception {
-        parser = YamlXContent.yamlXContent.createParser(
-                "\"node_info test\":\n" +
-                "  - do:\n" +
-                "      cluster.node_info: {}\n" +
-                "  \n" +
-                "  - is_true: nodes\n" +
-                "  - is_true: cluster_name\n");
-        ClientYamlTestSectionParser testSectionParser = new ClientYamlTestSectionParser();
-        ClientYamlTestSection testSection = testSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-        assertThat(testSection, notNullValue());
-        assertThat(testSection.getName(), equalTo("node_info test"));
-        assertThat(testSection.getExecutableSections().size(), equalTo(3));
-    }
-}
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParserFailingTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParserFailingTests.java
index f2219816462a5..2d3afa91eda5b 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParserFailingTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParserFailingTests.java
@@ -19,11 +19,8 @@
 package org.elasticsearch.test.rest.yaml.restspec;
 
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.common.xcontent.yaml.YamlXContent;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestApiParser;
-
-import java.io.IOException;
 
 import static org.hamcrest.Matchers.containsString;
 
@@ -32,6 +29,109 @@
  * stream
  */
 public class ClientYamlSuiteRestApiParserFailingTests extends ESTestCase {
+
+    public void testDuplicateMethods() throws Exception {
+       parseAndExpectFailure("{\n" +
+               "  \"ping\": {" +
+               "    \"documentation\": \"http://www.elasticsearch.org/guide/\"," +
+               "    \"methods\": [\"PUT\", \"PUT\"]," +
+               "    \"url\": {" +
+               "      \"path\": \"/\"," +
+               "      \"paths\": [\"/\"]," +
+               "      \"parts\": {" +
+               "      }," +
+               "      \"params\": {" +
+               "        \"type\" : \"boolean\",\n" +
+               "        \"description\" : \"Whether specified concrete indices should be ignored when unavailable (missing or closed)\"" +
+               "      }" +
+               "    }," +
+               "    \"body\": null" +
+               "  }" +
+               "}", "Found duplicate method [PUT]");
+    }
+
+    public void testDuplicatePaths() throws Exception {
+        parseAndExpectFailure("{\n" +
+                "  \"ping\": {" +
+                "    \"documentation\": \"http://www.elasticsearch.org/guide/\"," +
+                "    \"methods\": [\"PUT\"]," +
+                "    \"url\": {" +
+                "      \"path\": \"/pingone\"," +
+                "      \"paths\": [\"/pingone\", \"/pingtwo\", \"/pingtwo\"]," +
+                "      \"parts\": {" +
+                "      }," +
+                "      \"params\": {" +
+                "        \"type\" : \"boolean\",\n" +
+                "        \"description\" : \"Whether specified concrete indices should be ignored when unavailable (missing or closed)\"" +
+                "      }" +
+                "    }," +
+                "    \"body\": null" +
+                "  }" +
+                "}", "Found duplicate path [/pingtwo]");
+    }
+
+    public void testDuplicateParts() throws Exception {
+        parseAndExpectFailure("{\n" +
+                "  \"ping\": {" +
+                "    \"documentation\": \"http://www.elasticsearch.org/guide/\"," +
+                "    \"methods\": [\"PUT\"]," +
+                "    \"url\": {" +
+                "      \"path\": \"/\"," +
+                "      \"paths\": [\"/\"]," +
+                "      \"parts\": {" +
+                "        \"index\": {" +
+                "          \"type\" : \"string\",\n" +
+                "          \"description\" : \"index part\"\n" +
+                "        }," +
+                "        \"type\": {" +
+                "          \"type\" : \"string\",\n" +
+                "          \"description\" : \"type part\"\n" +
+                "        }," +
+                "        \"index\": {" +
+                "          \"type\" : \"string\",\n" +
+                "          \"description\" : \"index parameter part\"\n" +
+                "        }" +
+                "      }," +
+                "      \"params\": {" +
+                "        \"type\" : \"boolean\",\n" +
+                "        \"description\" : \"Whether specified concrete indices should be ignored when unavailable (missing or closed)\"" +
+                "      }" +
+                "    }," +
+                "    \"body\": null" +
+                "  }" +
+                "}", "Found duplicate part [index]");
+    }
+
+    public void testDuplicateParams() throws Exception {
+        parseAndExpectFailure("{\n" +
+                "  \"ping\": {" +
+                "    \"documentation\": \"http://www.elasticsearch.org/guide/\"," +
+                "    \"methods\": [\"PUT\"]," +
+                "    \"url\": {" +
+                "      \"path\": \"/\"," +
+                "      \"paths\": [\"/\"]," +
+                "      \"parts\": {" +
+                "      }," +
+                "      \"params\": {" +
+                "        \"timeout\": {" +
+                "          \"type\" : \"string\",\n" +
+                "          \"description\" : \"timeout parameter\"\n" +
+                "        }," +
+                "        \"refresh\": {" +
+                "          \"type\" : \"string\",\n" +
+                "          \"description\" : \"refresh parameter\"\n" +
+                "        }," +
+                "        \"timeout\": {" +
+                "          \"type\" : \"string\",\n" +
+                "          \"description\" : \"timeout parameter again\"\n" +
+                "        }" +
+                "      }" +
+                "    }," +
+                "    \"body\": null" +
+                "  }" +
+                "}", "Found duplicate param [timeout]");
+    }
+
     public void testBrokenSpecShouldThrowUsefulExceptionWhenParsingFailsOnParams() throws Exception {
         parseAndExpectFailure(BROKEN_SPEC_PARAMS, "Expected params field in rest api definition to contain an object");
     }
@@ -41,13 +141,11 @@ public void testBrokenSpecShouldThrowUsefulExceptionWhenParsingFailsOnParts() th
     }
 
     private void parseAndExpectFailure(String brokenJson, String expectedErrorMessage) throws Exception {
-        XContentParser parser = JsonXContent.jsonXContent.createParser(brokenJson);
-        try {
-            new ClientYamlSuiteRestApiParser().parse("location", parser);
-            fail("Expected to fail parsing but did not happen");
-        } catch (IOException e) {
-            assertThat(e.getMessage(), containsString(expectedErrorMessage));
-        }
+        XContentParser parser = createParser(YamlXContent.yamlXContent, brokenJson);
+        ClientYamlSuiteRestApiParser restApiParser = new ClientYamlSuiteRestApiParser();
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> restApiParser.parse("location", parser));
+        assertThat(e.getMessage(), containsString(expectedErrorMessage));
     }
 
     // see params section is broken, an inside param is missing
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParserTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParserTests.java
index 8396e343a566a..6acdd935400f4 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParserTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParserTests.java
@@ -18,18 +18,16 @@
  */
 package org.elasticsearch.test.rest.yaml.restspec;
 
-import org.elasticsearch.common.xcontent.json.JsonXContent;
-import org.elasticsearch.test.rest.yaml.parser.AbstractParserTestCase;
-import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestApi;
-import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestApiParser;
+import org.elasticsearch.common.xcontent.yaml.YamlXContent;
+import org.elasticsearch.test.rest.yaml.section.AbstractClientYamlTestFragmentParserTestCase;
 
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
 
-public class ClientYamlSuiteRestApiParserTests extends AbstractParserTestCase {
+public class ClientYamlSuiteRestApiParserTests extends AbstractClientYamlTestFragmentParserTestCase {
     public void testParseRestSpecIndexApi() throws Exception {
-        parser = JsonXContent.jsonXContent.createParser(REST_SPEC_INDEX_API);
+        parser = createParser(YamlXContent.yamlXContent, REST_SPEC_INDEX_API);
         ClientYamlSuiteRestApi restApi = new ClientYamlSuiteRestApiParser().parse("location", parser);
 
         assertThat(restApi, notNullValue());
@@ -51,7 +49,7 @@ public void testParseRestSpecIndexApi() throws Exception {
     }
 
     public void testParseRestSpecGetTemplateApi() throws Exception {
-        parser = JsonXContent.jsonXContent.createParser(REST_SPEC_GET_TEMPLATE_API);
+        parser = createParser(YamlXContent.yamlXContent, REST_SPEC_GET_TEMPLATE_API);
         ClientYamlSuiteRestApi restApi = new ClientYamlSuiteRestApiParser().parse("location", parser);
         assertThat(restApi, notNullValue());
         assertThat(restApi.getName(), equalTo("indices.get_template"));
@@ -68,7 +66,7 @@ public void testParseRestSpecGetTemplateApi() throws Exception {
     }
 
     public void testParseRestSpecCountApi() throws Exception {
-        parser = JsonXContent.jsonXContent.createParser(REST_SPEC_COUNT_API);
+        parser = createParser(YamlXContent.yamlXContent, REST_SPEC_COUNT_API);
         ClientYamlSuiteRestApi restApi = new ClientYamlSuiteRestApiParser().parse("location", parser);
         assertThat(restApi, notNullValue());
         assertThat(restApi.getName(), equalTo("count"));
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/AbstractClientYamlTestFragmentParserTestCase.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/AbstractClientYamlTestFragmentParserTestCase.java
new file mode 100644
index 0000000000000..b7038a7d7a329
--- /dev/null
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/AbstractClientYamlTestFragmentParserTestCase.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.rest.yaml.section;
+
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.rest.yaml.section.ExecutableSection;
+import org.junit.After;
+
+import static org.hamcrest.Matchers.nullValue;
+
+/**
+ * Superclass for tests that parse parts of the test suite.
+ */
+public abstract class AbstractClientYamlTestFragmentParserTestCase extends ESTestCase {
+    protected XContentParser parser;
+
+    @Override
+    @After
+    public void tearDown() throws Exception {
+        super.tearDown();
+        // test may be skipped so we did not create a parser instance
+        if (parser != null) {
+            //next token can be null even in the middle of the document (e.g. with "---"), but not too many consecutive times
+            assertThat(parser.currentToken(), nullValue());
+            assertThat(parser.nextToken(), nullValue());
+            assertThat(parser.nextToken(), nullValue());
+            parser.close();
+        }
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return ExecutableSection.XCONTENT_REGISTRY;
+    }
+}
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/AssertionTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/AssertionTests.java
new file mode 100644
index 0000000000000..fcef74678359e
--- /dev/null
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/AssertionTests.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.test.rest.yaml.section;
+
+import org.elasticsearch.common.xcontent.yaml.YamlXContent;
+import org.elasticsearch.test.rest.yaml.section.GreaterThanAssertion;
+import org.elasticsearch.test.rest.yaml.section.IsFalseAssertion;
+import org.elasticsearch.test.rest.yaml.section.IsTrueAssertion;
+import org.elasticsearch.test.rest.yaml.section.LengthAssertion;
+import org.elasticsearch.test.rest.yaml.section.LessThanAssertion;
+import org.elasticsearch.test.rest.yaml.section.MatchAssertion;
+
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.notNullValue;
+
+public class AssertionTests extends AbstractClientYamlTestFragmentParserTestCase {
+    public void testParseIsTrue() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "get.fields._timestamp"
+        );
+
+        IsTrueAssertion trueAssertion = IsTrueAssertion.parse(parser);
+
+        assertThat(trueAssertion, notNullValue());
+        assertThat(trueAssertion.getField(), equalTo("get.fields._timestamp"));
+    }
+
+    public void testParseIsFalse() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "docs.1._source"
+        );
+
+        IsFalseAssertion falseAssertion = IsFalseAssertion.parse(parser);
+
+        assertThat(falseAssertion, notNullValue());
+        assertThat(falseAssertion.getField(), equalTo("docs.1._source"));
+    }
+
+    public void testParseGreaterThan() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "{ field: 3}"
+        );
+
+        GreaterThanAssertion greaterThanAssertion = GreaterThanAssertion.parse(parser);
+        assertThat(greaterThanAssertion, notNullValue());
+        assertThat(greaterThanAssertion.getField(), equalTo("field"));
+        assertThat(greaterThanAssertion.getExpectedValue(), instanceOf(Integer.class));
+        assertThat((Integer) greaterThanAssertion.getExpectedValue(), equalTo(3));
+    }
+
+    public void testParseLessThan() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "{ field: 3}"
+        );
+
+        LessThanAssertion lessThanAssertion = LessThanAssertion.parse(parser);
+        assertThat(lessThanAssertion, notNullValue());
+        assertThat(lessThanAssertion.getField(), equalTo("field"));
+        assertThat(lessThanAssertion.getExpectedValue(), instanceOf(Integer.class));
+        assertThat((Integer) lessThanAssertion.getExpectedValue(), equalTo(3));
+    }
+
+    public void testParseLength() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "{ _id: 22}"
+        );
+
+        LengthAssertion lengthAssertion = LengthAssertion.parse(parser);
+        assertThat(lengthAssertion, notNullValue());
+        assertThat(lengthAssertion.getField(), equalTo("_id"));
+        assertThat(lengthAssertion.getExpectedValue(), instanceOf(Integer.class));
+        assertThat((Integer) lengthAssertion.getExpectedValue(), equalTo(22));
+    }
+
+    public void testParseMatchSimpleIntegerValue() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "{ field: 10 }"
+        );
+
+        MatchAssertion matchAssertion = MatchAssertion.parse(parser);
+
+        assertThat(matchAssertion, notNullValue());
+        assertThat(matchAssertion.getField(), equalTo("field"));
+        assertThat(matchAssertion.getExpectedValue(), instanceOf(Integer.class));
+        assertThat((Integer) matchAssertion.getExpectedValue(), equalTo(10));
+    }
+
+    public void testParseMatchSimpleStringValue() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "{ foo: bar }"
+        );
+
+        MatchAssertion matchAssertion = MatchAssertion.parse(parser);
+
+        assertThat(matchAssertion, notNullValue());
+        assertThat(matchAssertion.getField(), equalTo("foo"));
+        assertThat(matchAssertion.getExpectedValue(), instanceOf(String.class));
+        assertThat(matchAssertion.getExpectedValue().toString(), equalTo("bar"));
+    }
+
+    public void testParseMatchArray() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "{'matches': ['test_percolator_1', 'test_percolator_2']}"
+        );
+
+        MatchAssertion matchAssertion = MatchAssertion.parse(parser);
+
+        assertThat(matchAssertion, notNullValue());
+        assertThat(matchAssertion.getField(), equalTo("matches"));
+        assertThat(matchAssertion.getExpectedValue(), instanceOf(List.class));
+        List<?> strings = (List<?>) matchAssertion.getExpectedValue();
+        assertThat(strings.size(), equalTo(2));
+        assertThat(strings.get(0).toString(), equalTo("test_percolator_1"));
+        assertThat(strings.get(1).toString(), equalTo("test_percolator_2"));
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testParseMatchSourceValues() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "{ _source: { responses.0.hits.total: 3, foo: bar  }}"
+        );
+
+        MatchAssertion matchAssertion = MatchAssertion.parse(parser);
+
+        assertThat(matchAssertion, notNullValue());
+        assertThat(matchAssertion.getField(), equalTo("_source"));
+        assertThat(matchAssertion.getExpectedValue(), instanceOf(Map.class));
+        Map<String, Object> expectedValue = (Map<String, Object>) matchAssertion.getExpectedValue();
+        assertThat(expectedValue.size(), equalTo(2));
+        Object o = expectedValue.get("responses.0.hits.total");
+        assertThat(o, instanceOf(Integer.class));
+        assertThat((Integer)o, equalTo(3));
+        o = expectedValue.get("foo");
+        assertThat(o, instanceOf(String.class));
+        assertThat(o.toString(), equalTo("bar"));
+    }
+}
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSectionTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSectionTests.java
new file mode 100644
index 0000000000000..7ad1cc8377fc1
--- /dev/null
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSectionTests.java
@@ -0,0 +1,301 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.rest.yaml.section;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.xcontent.XContentLocation;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.yaml.YamlXContent;
+
+import java.io.IOException;
+import java.util.Map;
+
+import static java.util.Collections.singletonList;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+
+public class ClientYamlTestSectionTests extends AbstractClientYamlTestFragmentParserTestCase {
+    public void testAddingDoWithoutWarningWithoutSkip() {
+        int lineNumber = between(1, 10000);
+        ClientYamlTestSection section = new ClientYamlTestSection(new XContentLocation(0, 0), "test");
+        section.setSkipSection(SkipSection.EMPTY);
+        DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0));
+        section.addExecutableSection(doSection);
+    }
+
+    public void testAddingDoWithWarningWithSkip() {
+        int lineNumber = between(1, 10000);
+        ClientYamlTestSection section = new ClientYamlTestSection(new XContentLocation(0, 0), "test");
+        section.setSkipSection(new SkipSection(null, singletonList("warnings"), null));
+        DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0));
+        doSection.setExpectedWarningHeaders(singletonList("foo"));
+        section.addExecutableSection(doSection);
+    }
+
+    public void testAddingDoWithWarningWithSkipButNotWarnings() {
+        int lineNumber = between(1, 10000);
+        ClientYamlTestSection section = new ClientYamlTestSection(new XContentLocation(0, 0), "test");
+        section.setSkipSection(new SkipSection(null, singletonList("yaml"), null));
+        DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0));
+        doSection.setExpectedWarningHeaders(singletonList("foo"));
+        Exception e = expectThrows(IllegalArgumentException.class, () -> section.addExecutableSection(doSection));
+        assertEquals("Attempted to add a [do] with a [warnings] section without a corresponding [skip] so runners that do not support the"
+                + " [warnings] section can skip the test at line [" + lineNumber + "]", e.getMessage());
+    }
+
+    public void testWrongIndentation() throws Exception {
+        {
+            XContentParser parser = createParser(YamlXContent.yamlXContent,
+                    "\"First test section\": \n" +
+                            "  - skip:\n" +
+                            "    version:  \"2.0.0 - 2.2.0\"\n" +
+                            "    reason:   \"Update doesn't return metadata fields, waiting for #3259\"");
+
+            ParsingException e = expectThrows(ParsingException.class, () -> ClientYamlTestSection.parse(parser));
+            assertEquals("Error parsing test named [First test section]", e.getMessage());
+            assertThat(e.getCause(), instanceOf(IllegalArgumentException.class));
+            assertEquals("Expected [START_OBJECT, found [VALUE_NULL], the skip section is not properly indented",
+                    e.getCause().getMessage());
+        }
+        {
+            XContentParser parser = createParser(YamlXContent.yamlXContent,
+                    "\"First test section\": \n" +
+                            " - do :\n" +
+                            "   catch: missing\n" +
+                            "   indices.get_warmer:\n" +
+                            "       index: test_index\n" +
+                            "       name: test_warmer"
+            );
+            ParsingException e = expectThrows(ParsingException.class, () -> ClientYamlTestSection.parse(parser));
+            assertEquals("Error parsing test named [First test section]", e.getMessage());
+            assertThat(e.getCause(), instanceOf(IOException.class));
+            assertThat(e.getCause().getCause(), instanceOf(IllegalArgumentException.class));
+            assertEquals("expected [START_OBJECT], found [VALUE_NULL], the do section is not properly indented",
+                    e.getCause().getCause().getMessage());
+        }
+    }
+
+    public void testParseTestSectionWithDoSection() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "\"First test section\": \n" +
+                " - do :\n" +
+                "     catch: missing\n" +
+                "     indices.get_warmer:\n" +
+                "         index: test_index\n" +
+                "         name: test_warmer"
+        );
+
+        ClientYamlTestSection testSection = ClientYamlTestSection.parse(parser);
+
+        assertThat(testSection, notNullValue());
+        assertThat(testSection.getName(), equalTo("First test section"));
+        assertThat(testSection.getSkipSection(), equalTo(SkipSection.EMPTY));
+        assertThat(testSection.getExecutableSections().size(), equalTo(1));
+        DoSection doSection = (DoSection)testSection.getExecutableSections().get(0);
+        assertThat(doSection.getCatch(), equalTo("missing"));
+        assertThat(doSection.getApiCallSection(), notNullValue());
+        assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_warmer"));
+        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2));
+        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
+    }
+
+    public void testParseTestSectionWithDoSetAndSkipSectionsNoSkip() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "\"First test section\": \n" +
+                        "  - skip:\n" +
+                        "      version:  \"2.0.0 - 2.2.0\"\n" +
+                        "      reason:   \"Update doesn't return metadata fields, waiting for #3259\"\n" +
+                        "  - do :\n" +
+                        "      catch: missing\n" +
+                        "      indices.get_warmer:\n" +
+                        "          index: test_index\n" +
+                        "          name: test_warmer\n" +
+                        "  - set: {_scroll_id: scroll_id}");
+
+
+        ClientYamlTestSection testSection = ClientYamlTestSection.parse(parser);
+
+        assertThat(testSection, notNullValue());
+        assertThat(testSection.getName(), equalTo("First test section"));
+        assertThat(testSection.getSkipSection(), notNullValue());
+        assertThat(testSection.getSkipSection().getLowerVersion(), equalTo(Version.V_2_0_0));
+        assertThat(testSection.getSkipSection().getUpperVersion(), equalTo(Version.V_2_2_0));
+        assertThat(testSection.getSkipSection().getReason(), equalTo("Update doesn't return metadata fields, waiting for #3259"));
+        assertThat(testSection.getExecutableSections().size(), equalTo(2));
+        DoSection doSection = (DoSection)testSection.getExecutableSections().get(0);
+        assertThat(doSection.getCatch(), equalTo("missing"));
+        assertThat(doSection.getApiCallSection(), notNullValue());
+        assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_warmer"));
+        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2));
+        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
+        SetSection setSection = (SetSection) testSection.getExecutableSections().get(1);
+        assertThat(setSection.getStash().size(), equalTo(1));
+        assertThat(setSection.getStash().get("_scroll_id"), equalTo("scroll_id"));
+    }
+
+    public void testParseTestSectionWithMultipleDoSections() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "\"Basic\":\n" +
+                        "\n" +
+                        "  - do:\n" +
+                        "      index:\n" +
+                        "        index: test_1\n" +
+                        "        type:  test\n" +
+                        "        id:    中文\n" +
+                        "        body:  { \"foo\": \"Hello: 中文\" }\n" +
+                        "  - do:\n" +
+                        "      get:\n" +
+                        "        index: test_1\n" +
+                        "        type:  test\n" +
+                        "        id:    中文"
+        );
+
+        ClientYamlTestSection testSection = ClientYamlTestSection.parse(parser);
+
+        assertThat(testSection, notNullValue());
+        assertThat(testSection.getName(), equalTo("Basic"));
+        assertThat(testSection.getSkipSection(), equalTo(SkipSection.EMPTY));
+        assertThat(testSection.getExecutableSections().size(), equalTo(2));
+        DoSection doSection = (DoSection)testSection.getExecutableSections().get(0);
+        assertThat(doSection.getCatch(), nullValue());
+        assertThat(doSection.getApiCallSection(), notNullValue());
+        assertThat(doSection.getApiCallSection().getApi(), equalTo("index"));
+        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3));
+        assertThat(doSection.getApiCallSection().hasBody(), equalTo(true));
+        doSection = (DoSection)testSection.getExecutableSections().get(1);
+        assertThat(doSection.getCatch(), nullValue());
+        assertThat(doSection.getApiCallSection(), notNullValue());
+        assertThat(doSection.getApiCallSection().getApi(), equalTo("get"));
+        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3));
+        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
+    }
+
+    public void testParseTestSectionWithDoSectionsAndAssertions() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "\"Basic\":\n" +
+                        "\n" +
+                        "  - do:\n" +
+                        "      index:\n" +
+                        "        index: test_1\n" +
+                        "        type:  test\n" +
+                        "        id:    中文\n" +
+                        "        body:  { \"foo\": \"Hello: 中文\" }\n" +
+                        "\n" +
+                        "  - do:\n" +
+                        "      get:\n" +
+                        "        index: test_1\n" +
+                        "        type:  test\n" +
+                        "        id:    中文\n" +
+                        "\n" +
+                        "  - match: { _index:   test_1 }\n" +
+                        "  - is_true: _source\n" +
+                        "  - match: { _source:  { foo: \"Hello: 中文\" } }\n" +
+                        "\n" +
+                        "  - do:\n" +
+                        "      get:\n" +
+                        "        index: test_1\n" +
+                        "        id:    中文\n" +
+                        "\n" +
+                        "  - length: { _index:   6 }\n" +
+                        "  - is_false: whatever\n" +
+                        "  - gt: { size: 5      }\n" +
+                        "  - lt: { size: 10      }"
+        );
+
+        ClientYamlTestSection testSection = ClientYamlTestSection.parse(parser);
+
+        assertThat(testSection, notNullValue());
+        assertThat(testSection.getName(), equalTo("Basic"));
+        assertThat(testSection.getSkipSection(), equalTo(SkipSection.EMPTY));
+        assertThat(testSection.getExecutableSections().size(), equalTo(10));
+
+        DoSection doSection = (DoSection)testSection.getExecutableSections().get(0);
+        assertThat(doSection.getCatch(), nullValue());
+        assertThat(doSection.getApiCallSection(), notNullValue());
+        assertThat(doSection.getApiCallSection().getApi(), equalTo("index"));
+        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3));
+        assertThat(doSection.getApiCallSection().hasBody(), equalTo(true));
+
+        doSection = (DoSection)testSection.getExecutableSections().get(1);
+        assertThat(doSection.getCatch(), nullValue());
+        assertThat(doSection.getApiCallSection(), notNullValue());
+        assertThat(doSection.getApiCallSection().getApi(), equalTo("get"));
+        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3));
+        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
+
+        MatchAssertion matchAssertion = (MatchAssertion)testSection.getExecutableSections().get(2);
+        assertThat(matchAssertion.getField(), equalTo("_index"));
+        assertThat(matchAssertion.getExpectedValue().toString(), equalTo("test_1"));
+
+        IsTrueAssertion trueAssertion = (IsTrueAssertion)testSection.getExecutableSections().get(3);
+        assertThat(trueAssertion.getField(), equalTo("_source"));
+
+        matchAssertion = (MatchAssertion)testSection.getExecutableSections().get(4);
+        assertThat(matchAssertion.getField(), equalTo("_source"));
+        assertThat(matchAssertion.getExpectedValue(), instanceOf(Map.class));
+        Map map = (Map) matchAssertion.getExpectedValue();
+        assertThat(map.size(), equalTo(1));
+        assertThat(map.get("foo").toString(), equalTo("Hello: 中文"));
+
+        doSection = (DoSection)testSection.getExecutableSections().get(5);
+        assertThat(doSection.getCatch(), nullValue());
+        assertThat(doSection.getApiCallSection(), notNullValue());
+        assertThat(doSection.getApiCallSection().getApi(), equalTo("get"));
+        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2));
+        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
+
+        LengthAssertion lengthAssertion = (LengthAssertion) testSection.getExecutableSections().get(6);
+        assertThat(lengthAssertion.getField(), equalTo("_index"));
+        assertThat(lengthAssertion.getExpectedValue(), instanceOf(Integer.class));
+        assertThat((Integer) lengthAssertion.getExpectedValue(), equalTo(6));
+
+        IsFalseAssertion falseAssertion = (IsFalseAssertion)testSection.getExecutableSections().get(7);
+        assertThat(falseAssertion.getField(), equalTo("whatever"));
+
+        GreaterThanAssertion greaterThanAssertion = (GreaterThanAssertion) testSection.getExecutableSections().get(8);
+        assertThat(greaterThanAssertion.getField(), equalTo("size"));
+        assertThat(greaterThanAssertion.getExpectedValue(), instanceOf(Integer.class));
+        assertThat((Integer) greaterThanAssertion.getExpectedValue(), equalTo(5));
+
+        LessThanAssertion lessThanAssertion = (LessThanAssertion) testSection.getExecutableSections().get(9);
+        assertThat(lessThanAssertion.getField(), equalTo("size"));
+        assertThat(lessThanAssertion.getExpectedValue(), instanceOf(Integer.class));
+        assertThat((Integer) lessThanAssertion.getExpectedValue(), equalTo(10));
+    }
+
+    public void testSmallSection() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "\"node_info test\":\n" +
+                "  - do:\n" +
+                "      cluster.node_info: {}\n" +
+                "  \n" +
+                "  - is_true: nodes\n" +
+                "  - is_true: cluster_name\n");
+
+        ClientYamlTestSection testSection = ClientYamlTestSection.parse(parser);
+        assertThat(testSection, notNullValue());
+        assertThat(testSection.getName(), equalTo("node_info test"));
+        assertThat(testSection.getExecutableSections().size(), equalTo(3));
+    }
+
+}
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuiteTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuiteTests.java
new file mode 100644
index 0000000000000..4c96986146b5e
--- /dev/null
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuiteTests.java
@@ -0,0 +1,376 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.rest.yaml.section;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.xcontent.yaml.YamlXContent;
+
+import java.util.Map;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+
+public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentParserTestCase {
+    public void testParseTestSetupTeardownAndSections() throws Exception {
+        final boolean includeSetup = randomBoolean();
+        final boolean includeTeardown = randomBoolean();
+        StringBuilder testSpecBuilder = new StringBuilder();
+        if (includeSetup) {
+            testSpecBuilder
+                .append("---\n" +
+                        "setup:\n" +
+                        "  - do:\n" +
+                        "        indices.create:\n" +
+                        "          index: test_index\n" +
+                        "\n");
+        }
+        if (includeTeardown) {
+            testSpecBuilder
+                .append("---\n" +
+                        "teardown:\n" +
+                        "  - do:\n" +
+                        "      indices.delete:\n" +
+                        "        index: test_index\n" +
+                        "\n");
+        }
+        parser = createParser(YamlXContent.yamlXContent,
+                        testSpecBuilder.toString() +
+                        "---\n" +
+                        "\"Get index mapping\":\n" +
+                        "  - do:\n" +
+                        "      indices.get_mapping:\n" +
+                        "        index: test_index\n" +
+                        "\n" +
+                        "  - match: {test_index.test_type.properties.text.type:     string}\n" +
+                        "  - match: {test_index.test_type.properties.text.analyzer: whitespace}\n" +
+                        "\n" +
+                        "---\n" +
+                        "\"Get type mapping - pre 1.0\":\n" +
+                        "\n" +
+                        "  - skip:\n" +
+                        "      version:     \"2.0.0 - \"\n" +
+                        "      reason:      \"for newer versions the index name is always returned\"\n" +
+                        "\n" +
+                        "  - do:\n" +
+                        "      indices.get_mapping:\n" +
+                        "        index: test_index\n" +
+                        "        type: test_type\n" +
+                        "\n" +
+                        "  - match: {test_type.properties.text.type:     string}\n" +
+                        "  - match: {test_type.properties.text.analyzer: whitespace}\n"
+        );
+
+        ClientYamlTestSuite restTestSuite = ClientYamlTestSuite.parse(getTestClass().getName(), getTestName(), parser);
+
+        assertThat(restTestSuite, notNullValue());
+        assertThat(restTestSuite.getName(), equalTo(getTestName()));
+        assertThat(restTestSuite.getSetupSection(), notNullValue());
+        if (includeSetup) {
+            assertThat(restTestSuite.getSetupSection().isEmpty(), equalTo(false));
+            assertThat(restTestSuite.getSetupSection().getSkipSection().isEmpty(), equalTo(true));
+            assertThat(restTestSuite.getSetupSection().getDoSections().size(), equalTo(1));
+            assertThat(restTestSuite.getSetupSection().getDoSections().get(0).getApiCallSection().getApi(), equalTo("indices.create"));
+            assertThat(restTestSuite.getSetupSection().getDoSections().get(0).getApiCallSection().getParams().size(), equalTo(1));
+            assertThat(restTestSuite.getSetupSection().getDoSections().get(0).getApiCallSection().getParams().get("index"),
+                    equalTo("test_index"));
+        } else {
+            assertThat(restTestSuite.getSetupSection().isEmpty(), equalTo(true));
+        }
+        
+        assertThat(restTestSuite.getTeardownSection(), notNullValue());
+        if (includeTeardown) {
+            assertThat(restTestSuite.getTeardownSection().isEmpty(), equalTo(false));
+            assertThat(restTestSuite.getTeardownSection().getSkipSection().isEmpty(), equalTo(true));
+            assertThat(restTestSuite.getTeardownSection().getDoSections().size(), equalTo(1));
+            assertThat(restTestSuite.getTeardownSection().getDoSections().get(0).getApiCallSection().getApi(), equalTo("indices.delete"));
+            assertThat(restTestSuite.getTeardownSection().getDoSections().get(0).getApiCallSection().getParams().size(), equalTo(1));
+            assertThat(restTestSuite.getTeardownSection().getDoSections().get(0).getApiCallSection().getParams().get("index"),
+                equalTo("test_index"));
+        } else {
+            assertThat(restTestSuite.getTeardownSection().isEmpty(), equalTo(true));
+        }
+
+        assertThat(restTestSuite.getTestSections().size(), equalTo(2));
+
+        assertThat(restTestSuite.getTestSections().get(0).getName(), equalTo("Get index mapping"));
+        assertThat(restTestSuite.getTestSections().get(0).getSkipSection().isEmpty(), equalTo(true));
+        assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().size(), equalTo(3));
+        assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(0), instanceOf(DoSection.class));
+        DoSection doSection = (DoSection) restTestSuite.getTestSections().get(0).getExecutableSections().get(0);
+        assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_mapping"));
+        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(1));
+        assertThat(doSection.getApiCallSection().getParams().get("index"), equalTo("test_index"));
+        assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(1), instanceOf(MatchAssertion.class));
+        MatchAssertion matchAssertion = (MatchAssertion) restTestSuite.getTestSections().get(0).getExecutableSections().get(1);
+        assertThat(matchAssertion.getField(), equalTo("test_index.test_type.properties.text.type"));
+        assertThat(matchAssertion.getExpectedValue().toString(), equalTo("string"));
+        assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(2), instanceOf(MatchAssertion.class));
+        matchAssertion = (MatchAssertion) restTestSuite.getTestSections().get(0).getExecutableSections().get(2);
+        assertThat(matchAssertion.getField(), equalTo("test_index.test_type.properties.text.analyzer"));
+        assertThat(matchAssertion.getExpectedValue().toString(), equalTo("whitespace"));
+
+        assertThat(restTestSuite.getTestSections().get(1).getName(), equalTo("Get type mapping - pre 1.0"));
+        assertThat(restTestSuite.getTestSections().get(1).getSkipSection().isEmpty(), equalTo(false));
+        assertThat(restTestSuite.getTestSections().get(1).getSkipSection().getReason(),
+                equalTo("for newer versions the index name is always returned"));
+        assertThat(restTestSuite.getTestSections().get(1).getSkipSection().getLowerVersion(), equalTo(Version.V_2_0_0));
+        assertThat(restTestSuite.getTestSections().get(1).getSkipSection().getUpperVersion(), equalTo(Version.CURRENT));
+        assertThat(restTestSuite.getTestSections().get(1).getExecutableSections().size(), equalTo(3));
+        assertThat(restTestSuite.getTestSections().get(1).getExecutableSections().get(0), instanceOf(DoSection.class));
+        doSection = (DoSection) restTestSuite.getTestSections().get(1).getExecutableSections().get(0);
+        assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_mapping"));
+        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2));
+        assertThat(doSection.getApiCallSection().getParams().get("index"), equalTo("test_index"));
+        assertThat(doSection.getApiCallSection().getParams().get("type"), equalTo("test_type"));
+        assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(1), instanceOf(MatchAssertion.class));
+        matchAssertion = (MatchAssertion) restTestSuite.getTestSections().get(1).getExecutableSections().get(1);
+        assertThat(matchAssertion.getField(), equalTo("test_type.properties.text.type"));
+        assertThat(matchAssertion.getExpectedValue().toString(), equalTo("string"));
+        assertThat(restTestSuite.getTestSections().get(1).getExecutableSections().get(2), instanceOf(MatchAssertion.class));
+        matchAssertion = (MatchAssertion) restTestSuite.getTestSections().get(1).getExecutableSections().get(2);
+        assertThat(matchAssertion.getField(), equalTo("test_type.properties.text.analyzer"));
+        assertThat(matchAssertion.getExpectedValue().toString(), equalTo("whitespace"));
+    }
+
+    public void testParseTestSingleTestSection() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+        "---\n" +
+                "\"Index with ID\":\n" +
+                "\n" +
+                " - do:\n" +
+                "      index:\n" +
+                "          index:  test-weird-index-中文\n" +
+                "          type:   weird.type\n" +
+                "          id:     1\n" +
+                "          body:   { foo: bar }\n" +
+                "\n" +
+                " - is_true:   ok\n" +
+                " - match:   { _index:   test-weird-index-中文 }\n" +
+                " - match:   { _type:    weird.type }\n" +
+                " - match:   { _id:      \"1\"}\n" +
+                " - match:   { _version: 1}\n" +
+                "\n" +
+                " - do:\n" +
+                "      get:\n" +
+                "          index:  test-weird-index-中文\n" +
+                "          type:   weird.type\n" +
+                "          id:     1\n" +
+                "\n" +
+                " - match:   { _index:   test-weird-index-中文 }\n" +
+                " - match:   { _type:    weird.type }\n" +
+                " - match:   { _id:      \"1\"}\n" +
+                " - match:   { _version: 1}\n" +
+                " - match:   { _source: { foo: bar }}"
+        );
+
+        ClientYamlTestSuite restTestSuite = ClientYamlTestSuite.parse(getTestClass().getName(), getTestName(), parser);
+
+        assertThat(restTestSuite, notNullValue());
+        assertThat(restTestSuite.getName(), equalTo(getTestName()));
+
+        assertThat(restTestSuite.getSetupSection().isEmpty(), equalTo(true));
+
+        assertThat(restTestSuite.getTestSections().size(), equalTo(1));
+
+        assertThat(restTestSuite.getTestSections().get(0).getName(), equalTo("Index with ID"));
+        assertThat(restTestSuite.getTestSections().get(0).getSkipSection().isEmpty(), equalTo(true));
+        assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().size(), equalTo(12));
+        assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(0), instanceOf(DoSection.class));
+        DoSection doSection = (DoSection) restTestSuite.getTestSections().get(0).getExecutableSections().get(0);
+        assertThat(doSection.getCatch(), nullValue());
+        assertThat(doSection.getApiCallSection().getApi(), equalTo("index"));
+        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3));
+        assertThat(doSection.getApiCallSection().hasBody(), equalTo(true));
+        assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(1), instanceOf(IsTrueAssertion.class));
+        IsTrueAssertion trueAssertion = (IsTrueAssertion) restTestSuite.getTestSections().get(0).getExecutableSections().get(1);
+        assertThat(trueAssertion.getField(), equalTo("ok"));
+        assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(2), instanceOf(MatchAssertion.class));
+        MatchAssertion matchAssertion = (MatchAssertion) restTestSuite.getTestSections().get(0).getExecutableSections().get(2);
+        assertThat(matchAssertion.getField(), equalTo("_index"));
+        assertThat(matchAssertion.getExpectedValue().toString(), equalTo("test-weird-index-中文"));
+        assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(3), instanceOf(MatchAssertion.class));
+        matchAssertion = (MatchAssertion) restTestSuite.getTestSections().get(0).getExecutableSections().get(3);
+        assertThat(matchAssertion.getField(), equalTo("_type"));
+        assertThat(matchAssertion.getExpectedValue().toString(), equalTo("weird.type"));
+        assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(4), instanceOf(MatchAssertion.class));
+        matchAssertion = (MatchAssertion) restTestSuite.getTestSections().get(0).getExecutableSections().get(4);
+        assertThat(matchAssertion.getField(), equalTo("_id"));
+        assertThat(matchAssertion.getExpectedValue().toString(), equalTo("1"));
+        assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(5), instanceOf(MatchAssertion.class));
+        matchAssertion = (MatchAssertion) restTestSuite.getTestSections().get(0).getExecutableSections().get(5);
+        assertThat(matchAssertion.getField(), equalTo("_version"));
+        assertThat(matchAssertion.getExpectedValue().toString(), equalTo("1"));
+        assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(6), instanceOf(DoSection.class));
+        doSection = (DoSection) restTestSuite.getTestSections().get(0).getExecutableSections().get(6);
+        assertThat(doSection.getCatch(), nullValue());
+        assertThat(doSection.getApiCallSection().getApi(), equalTo("get"));
+        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3));
+        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
+        assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(7), instanceOf(MatchAssertion.class));
+        matchAssertion = (MatchAssertion) restTestSuite.getTestSections().get(0).getExecutableSections().get(7);
+        assertThat(matchAssertion.getField(), equalTo("_index"));
+        assertThat(matchAssertion.getExpectedValue().toString(), equalTo("test-weird-index-中文"));
+        assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(8), instanceOf(MatchAssertion.class));
+        matchAssertion = (MatchAssertion) restTestSuite.getTestSections().get(0).getExecutableSections().get(8);
+        assertThat(matchAssertion.getField(), equalTo("_type"));
+        assertThat(matchAssertion.getExpectedValue().toString(), equalTo("weird.type"));
+        assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(9), instanceOf(MatchAssertion.class));
+        matchAssertion = (MatchAssertion) restTestSuite.getTestSections().get(0).getExecutableSections().get(9);
+        assertThat(matchAssertion.getField(), equalTo("_id"));
+        assertThat(matchAssertion.getExpectedValue().toString(), equalTo("1"));
+        assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(10), instanceOf(MatchAssertion.class));
+        matchAssertion = (MatchAssertion) restTestSuite.getTestSections().get(0).getExecutableSections().get(10);
+        assertThat(matchAssertion.getField(), equalTo("_version"));
+        assertThat(matchAssertion.getExpectedValue().toString(), equalTo("1"));
+        assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(11), instanceOf(MatchAssertion.class));
+        matchAssertion = (MatchAssertion) restTestSuite.getTestSections().get(0).getExecutableSections().get(11);
+        assertThat(matchAssertion.getField(), equalTo("_source"));
+        assertThat(matchAssertion.getExpectedValue(), instanceOf(Map.class));
+        assertThat(((Map) matchAssertion.getExpectedValue()).get("foo").toString(), equalTo("bar"));
+    }
+
+    public void testParseTestMultipleTestSections() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+        "---\n" +
+                "\"Missing document (partial doc)\":\n" +
+                "\n" +
+                "  - do:\n" +
+                "      catch:      missing\n" +
+                "      update:\n" +
+                "          index:  test_1\n" +
+                "          type:   test\n" +
+                "          id:     1\n" +
+                "          body:   { doc: { foo: bar } }\n" +
+                "\n" +
+                "  - do:\n" +
+                "      update:\n" +
+                "          index: test_1\n" +
+                "          type:  test\n" +
+                "          id:    1\n" +
+                "          body:  { doc: { foo: bar } }\n" +
+                "          ignore: 404\n" +
+                "\n" +
+                "---\n" +
+                "\"Missing document (script)\":\n" +
+                "\n" +
+                "\n" +
+                "  - do:\n" +
+                "      catch:      missing\n" +
+                "      update:\n" +
+                "          index:  test_1\n" +
+                "          type:   test\n" +
+                "          id:     1\n" +
+                "          body:\n" +
+                "            script: \"ctx._source.foo = bar\"\n" +
+                "            params: { bar: 'xxx' }\n" +
+                "\n" +
+                "  - do:\n" +
+                "      update:\n" +
+                "          index:  test_1\n" +
+                "          type:   test\n" +
+                "          id:     1\n" +
+                "          ignore: 404\n" +
+                "          body:\n" +
+                "            script:       \"ctx._source.foo = bar\"\n" +
+                "            params:       { bar: 'xxx' }\n"
+        );
+
+        ClientYamlTestSuite restTestSuite = ClientYamlTestSuite.parse(getTestClass().getName(), getTestName(), parser);
+
+        assertThat(restTestSuite, notNullValue());
+        assertThat(restTestSuite.getName(), equalTo(getTestName()));
+
+        assertThat(restTestSuite.getSetupSection().isEmpty(), equalTo(true));
+
+        assertThat(restTestSuite.getTestSections().size(), equalTo(2));
+
+        assertThat(restTestSuite.getTestSections().get(0).getName(), equalTo("Missing document (partial doc)"));
+        assertThat(restTestSuite.getTestSections().get(0).getSkipSection().isEmpty(), equalTo(true));
+        assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().size(), equalTo(2));
+
+        assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(0), instanceOf(DoSection.class));
+        DoSection doSection = (DoSection) restTestSuite.getTestSections().get(0).getExecutableSections().get(0);
+        assertThat(doSection.getCatch(), equalTo("missing"));
+        assertThat(doSection.getApiCallSection().getApi(), equalTo("update"));
+        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3));
+        assertThat(doSection.getApiCallSection().hasBody(), equalTo(true));
+        assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(1), instanceOf(DoSection.class));
+        doSection = (DoSection) restTestSuite.getTestSections().get(0).getExecutableSections().get(1);
+        assertThat(doSection.getCatch(), nullValue());
+        assertThat(doSection.getApiCallSection().getApi(), equalTo("update"));
+        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(4));
+        assertThat(doSection.getApiCallSection().hasBody(), equalTo(true));
+
+        assertThat(restTestSuite.getTestSections().get(1).getName(), equalTo("Missing document (script)"));
+        assertThat(restTestSuite.getTestSections().get(1).getSkipSection().isEmpty(), equalTo(true));
+        assertThat(restTestSuite.getTestSections().get(1).getExecutableSections().size(), equalTo(2));
+        assertThat(restTestSuite.getTestSections().get(1).getExecutableSections().get(0), instanceOf(DoSection.class));
+        assertThat(restTestSuite.getTestSections().get(1).getExecutableSections().get(1), instanceOf(DoSection.class));
+        doSection = (DoSection) restTestSuite.getTestSections().get(1).getExecutableSections().get(0);
+        assertThat(doSection.getCatch(), equalTo("missing"));
+        assertThat(doSection.getApiCallSection().getApi(), equalTo("update"));
+        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3));
+        assertThat(doSection.getApiCallSection().hasBody(), equalTo(true));
+        assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(1), instanceOf(DoSection.class));
+        doSection = (DoSection) restTestSuite.getTestSections().get(1).getExecutableSections().get(1);
+        assertThat(doSection.getCatch(), nullValue());
+        assertThat(doSection.getApiCallSection().getApi(), equalTo("update"));
+        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(4));
+        assertThat(doSection.getApiCallSection().hasBody(), equalTo(true));
+    }
+
+    public void testParseTestDuplicateTestSections() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "---\n" +
+                        "\"Missing document (script)\":\n" +
+                        "\n" +
+                        "  - do:\n" +
+                        "      catch:      missing\n" +
+                        "      update:\n" +
+                        "          index:  test_1\n" +
+                        "          type:   test\n" +
+                        "          id:     1\n" +
+                        "          body:   { doc: { foo: bar } }\n" +
+                        "\n" +
+                        "---\n" +
+                        "\"Missing document (script)\":\n" +
+                        "\n" +
+                        "\n" +
+                        "  - do:\n" +
+                        "      catch:      missing\n" +
+                        "      update:\n" +
+                        "          index:  test_1\n" +
+                        "          type:   test\n" +
+                        "          id:     1\n" +
+                        "          body:\n" +
+                        "            script: \"ctx._source.foo = bar\"\n" +
+                        "            params: { bar: 'xxx' }\n" +
+                        "\n"
+        );
+
+        Exception e = expectThrows(ParsingException.class, () ->
+            ClientYamlTestSuite.parse(getTestClass().getName(), getTestName(), parser));
+        assertThat(e.getMessage(), containsString("duplicate test section"));
+    }
+}
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java
index e981b2d999c06..8e736119c807c 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java
@@ -19,48 +19,484 @@
 
 package org.elasticsearch.test.rest.yaml.section;
 
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.xcontent.XContentLocation;
-import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.yaml.YamlXContent;
+import org.hamcrest.MatcherAssert;
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Map;
 
 import static java.util.Collections.emptyList;
 import static java.util.Collections.singletonList;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+
+public class DoSectionTests extends AbstractClientYamlTestFragmentParserTestCase {
 
-public class DoSectionTests extends ESTestCase {
     public void testWarningHeaders() throws IOException {
-        DoSection section = new DoSection(new XContentLocation(1, 1));
+        {
+            final DoSection section = new DoSection(new XContentLocation(1, 1));
 
-        // No warning headers doesn't throw an exception
-        section.checkWarningHeaders(emptyList());
+            // No warning headers doesn't throw an exception
+            section.checkWarningHeaders(emptyList());
+        }
 
+        final String testHeader = DeprecationLogger.formatWarning("test");
+        final String anotherHeader = DeprecationLogger.formatWarning("another");
+        final String someMoreHeader = DeprecationLogger.formatWarning("some more");
+        final String catHeader = DeprecationLogger.formatWarning("cat");
         // Any warning headers fail
-        AssertionError e = expectThrows(AssertionError.class, () -> section.checkWarningHeaders(singletonList("test")));
-        assertEquals("got unexpected warning headers [\ntest\n]", e.getMessage());
-        e = expectThrows(AssertionError.class, () -> section.checkWarningHeaders(Arrays.asList("test", "another", "some more")));
-        assertEquals("got unexpected warning headers [\ntest\nanother\nsome more\n]", e.getMessage());
+        {
+            final DoSection section = new DoSection(new XContentLocation(1, 1));
+
+            final AssertionError one = expectThrows(AssertionError.class, () -> section.checkWarningHeaders(singletonList(testHeader)));
+            assertEquals("got unexpected warning header [\n\t" + testHeader + "\n]\n", one.getMessage());
+
+            final AssertionError multiple =
+                    expectThrows(
+                            AssertionError.class,
+                            () -> section.checkWarningHeaders(Arrays.asList(testHeader, anotherHeader, someMoreHeader)));
+            assertEquals(
+                    "got unexpected warning headers [\n\t" +
+                            testHeader + "\n\t" +
+                            anotherHeader + "\n\t" +
+                            someMoreHeader + "\n]\n",
+                    multiple.getMessage());
+        }
 
         // But not when we expect them
-        section.setExpectedWarningHeaders(singletonList("test"));
-        section.checkWarningHeaders(singletonList("test"));
-        section.setExpectedWarningHeaders(Arrays.asList("test", "another", "some more"));
-        section.checkWarningHeaders(Arrays.asList("test", "another", "some more"));
+        {
+            final DoSection section = new DoSection(new XContentLocation(1, 1));
+            section.setExpectedWarningHeaders(singletonList("test"));
+            section.checkWarningHeaders(singletonList(testHeader));
+        }
+        {
+            final DoSection section = new DoSection(new XContentLocation(1, 1));
+            section.setExpectedWarningHeaders(Arrays.asList("test", "another", "some more"));
+            section.checkWarningHeaders(Arrays.asList(testHeader, anotherHeader, someMoreHeader));
+        }
 
         // But if you don't get some that you did expect, that is an error
-        section.setExpectedWarningHeaders(singletonList("test"));
-        e = expectThrows(AssertionError.class, () -> section.checkWarningHeaders(emptyList()));
-        assertEquals("didn't get expected warning headers [\ntest\n]", e.getMessage());
-        section.setExpectedWarningHeaders(Arrays.asList("test", "another", "some more"));
-        e = expectThrows(AssertionError.class, () -> section.checkWarningHeaders(emptyList()));
-        assertEquals("didn't get expected warning headers [\ntest\nanother\nsome more\n]", e.getMessage());
-        e = expectThrows(AssertionError.class, () -> section.checkWarningHeaders(Arrays.asList("test", "some more")));
-        assertEquals("didn't get expected warning headers [\nanother\n]", e.getMessage());
+        {
+            final DoSection section = new DoSection(new XContentLocation(1, 1));
+            section.setExpectedWarningHeaders(singletonList("test"));
+            final AssertionError e = expectThrows(AssertionError.class, () -> section.checkWarningHeaders(emptyList()));
+            assertEquals("did not get expected warning header [\n\ttest\n]\n", e.getMessage());
+        }
+        {
+            final DoSection section = new DoSection(new XContentLocation(1, 1));
+            section.setExpectedWarningHeaders(Arrays.asList("test", "another", "some more"));
+
+            final AssertionError multiple = expectThrows(AssertionError.class, () -> section.checkWarningHeaders(emptyList()));
+            assertEquals("did not get expected warning headers [\n\ttest\n\tanother\n\tsome more\n]\n", multiple.getMessage());
+
+            final AssertionError one =
+                    expectThrows(AssertionError.class, () -> section.checkWarningHeaders(Arrays.asList(testHeader, someMoreHeader)));
+            assertEquals("did not get expected warning header [\n\tanother\n]\n", one.getMessage());
+        }
 
         // It is also an error if you get some warning you want and some you don't want
-        section.setExpectedWarningHeaders(Arrays.asList("test", "another", "some more"));
-        e = expectThrows(AssertionError.class, () -> section.checkWarningHeaders(Arrays.asList("test", "cat")));
-        assertEquals("got unexpected warning headers [\ncat\n] didn't get expected warning headers [\nanother\nsome more\n]",
-                e.getMessage());
+        {
+            final DoSection section = new DoSection(new XContentLocation(1, 1));
+            section.setExpectedWarningHeaders(Arrays.asList("test", "another", "some more"));
+            final AssertionError e =
+                    expectThrows(AssertionError.class, () -> section.checkWarningHeaders(Arrays.asList(testHeader, catHeader)));
+            assertEquals("got unexpected warning header [\n\t" +
+                            catHeader + "\n]\n" +
+                            "did not get expected warning headers [\n\tanother\n\tsome more\n]\n",
+                    e.getMessage());
+        }
+    }
+
+    public void testParseDoSectionNoBody() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "get:\n" +
+                "    index:    test_index\n" +
+                "    type:    test_type\n" +
+                "    id:        1"
+        );
+
+        DoSection doSection = DoSection.parse(parser);
+        ApiCallSection apiCallSection = doSection.getApiCallSection();
+
+        assertThat(apiCallSection, notNullValue());
+        assertThat(apiCallSection.getApi(), equalTo("get"));
+        assertThat(apiCallSection.getParams().size(), equalTo(3));
+        assertThat(apiCallSection.getParams().get("index"), equalTo("test_index"));
+        assertThat(apiCallSection.getParams().get("type"), equalTo("test_type"));
+        assertThat(apiCallSection.getParams().get("id"), equalTo("1"));
+        assertThat(apiCallSection.hasBody(), equalTo(false));
+    }
+
+    public void testParseDoSectionNoParamsNoBody() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "cluster.node_info: {}"
+        );
+
+        DoSection doSection = DoSection.parse(parser);
+        ApiCallSection apiCallSection = doSection.getApiCallSection();
+
+        assertThat(apiCallSection, notNullValue());
+        assertThat(apiCallSection.getApi(), equalTo("cluster.node_info"));
+        assertThat(apiCallSection.getParams().size(), equalTo(0));
+        assertThat(apiCallSection.hasBody(), equalTo(false));
+    }
+
+    public void testParseDoSectionWithJsonBody() throws Exception {
+        String body = "{ \"include\": { \"field1\": \"v1\", \"field2\": \"v2\" }, \"count\": 1 }";
+        parser = createParser(YamlXContent.yamlXContent,
+                "index:\n" +
+                "    index:  test_1\n" +
+                "    type:   test\n" +
+                "    id:     1\n" +
+                "    body:   " + body
+        );
+
+        DoSection doSection = DoSection.parse(parser);
+        ApiCallSection apiCallSection = doSection.getApiCallSection();
+
+        assertThat(apiCallSection, notNullValue());
+        assertThat(apiCallSection.getApi(), equalTo("index"));
+        assertThat(apiCallSection.getParams().size(), equalTo(3));
+        assertThat(apiCallSection.getParams().get("index"), equalTo("test_1"));
+        assertThat(apiCallSection.getParams().get("type"), equalTo("test"));
+        assertThat(apiCallSection.getParams().get("id"), equalTo("1"));
+        assertThat(apiCallSection.hasBody(), equalTo(true));
+
+        assertJsonEquals(apiCallSection.getBodies().get(0), body);
+    }
+
+    public void testParseDoSectionWithJsonMultipleBodiesAsLongString() throws Exception {
+        String bodies[] = new String[]{
+                "{ \"index\": { \"_index\":\"test_index\", \"_type\":\"test_type\", \"_id\":\"test_id\" } }\n",
+                "{ \"f1\":\"v1\", \"f2\":42 }\n",
+                "{ \"index\": { \"_index\":\"test_index2\", \"_type\":\"test_type2\", \"_id\":\"test_id2\" } }\n",
+                "{ \"f1\":\"v2\", \"f2\":47 }\n"
+        };
+        parser = createParser(YamlXContent.yamlXContent,
+                "bulk:\n" +
+                "    refresh: true\n" +
+                "    body: |\n" +
+                "        " + bodies[0] +
+                "        " + bodies[1] +
+                "        " + bodies[2] +
+                "        " + bodies[3]
+        );
+
+        DoSection doSection = DoSection.parse(parser);
+        ApiCallSection apiCallSection = doSection.getApiCallSection();
+
+        assertThat(apiCallSection, notNullValue());
+        assertThat(apiCallSection.getApi(), equalTo("bulk"));
+        assertThat(apiCallSection.getParams().size(), equalTo(1));
+        assertThat(apiCallSection.getParams().get("refresh"), equalTo("true"));
+        assertThat(apiCallSection.hasBody(), equalTo(true));
+        assertThat(apiCallSection.getBodies().size(), equalTo(4));
+    }
+
+    public void testParseDoSectionWithJsonMultipleBodiesRepeatedProperty() throws Exception {
+        String[] bodies = new String[] {
+                "{ \"index\": { \"_index\":\"test_index\", \"_type\":\"test_type\", \"_id\":\"test_id\" } }",
+                "{ \"f1\":\"v1\", \"f2\":42 }",
+        };
+        parser = createParser(YamlXContent.yamlXContent,
+                "bulk:\n" +
+                "    refresh: true\n" +
+                "    body: \n" +
+                "        " + bodies[0] + "\n" +
+                "    body: \n" +
+                "        " + bodies[1]
+        );
+
+        DoSection doSection = DoSection.parse(parser);
+        ApiCallSection apiCallSection = doSection.getApiCallSection();
+
+        assertThat(apiCallSection, notNullValue());
+        assertThat(apiCallSection.getApi(), equalTo("bulk"));
+        assertThat(apiCallSection.getParams().size(), equalTo(1));
+        assertThat(apiCallSection.getParams().get("refresh"), equalTo("true"));
+        assertThat(apiCallSection.hasBody(), equalTo(true));
+        assertThat(apiCallSection.getBodies().size(), equalTo(bodies.length));
+        for (int i = 0; i < bodies.length; i++) {
+            assertJsonEquals(apiCallSection.getBodies().get(i), bodies[i]);
+        }
+    }
+
+    public void testParseDoSectionWithYamlBody() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "search:\n" +
+                "    body:\n" +
+                "        \"_source\": [ include.field1, include.field2 ]\n" +
+                "        \"query\": { \"match_all\": {} }"
+        );
+        String body = "{ \"_source\": [ \"include.field1\", \"include.field2\" ], \"query\": { \"match_all\": {} }}";
+
+        DoSection doSection = DoSection.parse(parser);
+        ApiCallSection apiCallSection = doSection.getApiCallSection();
+
+        assertThat(apiCallSection, notNullValue());
+        assertThat(apiCallSection.getApi(), equalTo("search"));
+        assertThat(apiCallSection.getParams().size(), equalTo(0));
+        assertThat(apiCallSection.hasBody(), equalTo(true));
+        assertThat(apiCallSection.getBodies().size(), equalTo(1));
+        assertJsonEquals(apiCallSection.getBodies().get(0), body);
+    }
+
+    public void testParseDoSectionWithYamlMultipleBodies() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "bulk:\n" +
+                "    refresh: true\n" +
+                "    body:\n" +
+                "        - index:\n" +
+                "            _index: test_index\n" +
+                "            _type:  test_type\n" +
+                "            _id:    test_id\n" +
+                "        - f1: v1\n" +
+                "          f2: 42\n" +
+                "        - index:\n" +
+                "            _index: test_index2\n" +
+                "            _type:  test_type2\n" +
+                "            _id:    test_id2\n" +
+                "        - f1: v2\n" +
+                "          f2: 47"
+        );
+        String[] bodies = new String[4];
+        bodies[0] = "{\"index\": {\"_index\": \"test_index\", \"_type\":  \"test_type\", \"_id\": \"test_id\"}}";
+        bodies[1] = "{ \"f1\":\"v1\", \"f2\": 42 }";
+        bodies[2] = "{\"index\": {\"_index\": \"test_index2\", \"_type\":  \"test_type2\", \"_id\": \"test_id2\"}}";
+        bodies[3] = "{ \"f1\":\"v2\", \"f2\": 47 }";
+
+        DoSection doSection = DoSection.parse(parser);
+        ApiCallSection apiCallSection = doSection.getApiCallSection();
+
+        assertThat(apiCallSection, notNullValue());
+        assertThat(apiCallSection.getApi(), equalTo("bulk"));
+        assertThat(apiCallSection.getParams().size(), equalTo(1));
+        assertThat(apiCallSection.getParams().get("refresh"), equalTo("true"));
+        assertThat(apiCallSection.hasBody(), equalTo(true));
+        assertThat(apiCallSection.getBodies().size(), equalTo(bodies.length));
+
+        for (int i = 0; i < bodies.length; i++) {
+            assertJsonEquals(apiCallSection.getBodies().get(i), bodies[i]);
+        }
+    }
+
+    public void testParseDoSectionWithYamlMultipleBodiesRepeatedProperty() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "bulk:\n" +
+                "    refresh: true\n" +
+                "    body:\n" +
+                "        index:\n" +
+                "            _index: test_index\n" +
+                "            _type:  test_type\n" +
+                "            _id:    test_id\n" +
+                "    body:\n" +
+                "        f1: v1\n" +
+                "        f2: 42\n"
+        );
+        String[] bodies = new String[2];
+        bodies[0] = "{\"index\": {\"_index\": \"test_index\", \"_type\":  \"test_type\", \"_id\": \"test_id\"}}";
+        bodies[1] = "{ \"f1\":\"v1\", \"f2\": 42 }";
+
+        DoSection doSection = DoSection.parse(parser);
+        ApiCallSection apiCallSection = doSection.getApiCallSection();
+
+        assertThat(apiCallSection, notNullValue());
+        assertThat(apiCallSection.getApi(), equalTo("bulk"));
+        assertThat(apiCallSection.getParams().size(), equalTo(1));
+        assertThat(apiCallSection.getParams().get("refresh"), equalTo("true"));
+        assertThat(apiCallSection.hasBody(), equalTo(true));
+        assertThat(apiCallSection.getBodies().size(), equalTo(bodies.length));
+
+        for (int i = 0; i < bodies.length; i++) {
+            assertJsonEquals(apiCallSection.getBodies().get(i), bodies[i]);
+        }
+    }
+
+    public void testParseDoSectionWithYamlBodyMultiGet() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "mget:\n" +
+                "    body:\n" +
+                "        docs:\n" +
+                "            - { _index: test_2, _type: test, _id: 1}\n" +
+                "            - { _index: test_1, _type: none, _id: 1}"
+        );
+        String body = "{ \"docs\": [ " +
+                "{\"_index\": \"test_2\", \"_type\":\"test\", \"_id\":1}, " +
+                "{\"_index\": \"test_1\", \"_type\":\"none\", \"_id\":1} " +
+                "]}";
+
+        DoSection doSection = DoSection.parse(parser);
+        ApiCallSection apiCallSection = doSection.getApiCallSection();
+
+        assertThat(apiCallSection, notNullValue());
+        assertThat(apiCallSection.getApi(), equalTo("mget"));
+        assertThat(apiCallSection.getParams().size(), equalTo(0));
+        assertThat(apiCallSection.hasBody(), equalTo(true));
+        assertThat(apiCallSection.getBodies().size(), equalTo(1));
+        assertJsonEquals(apiCallSection.getBodies().get(0), body);
+    }
+
+    public void testParseDoSectionWithBodyStringified() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "index:\n" +
+                "    index:  test_1\n" +
+                "    type:   test\n" +
+                "    id:     1\n" +
+                "    body:   \"{ \\\"_source\\\": true, \\\"query\\\": { \\\"match_all\\\": {} } }\""
+        );
+
+        DoSection doSection = DoSection.parse(parser);
+        ApiCallSection apiCallSection = doSection.getApiCallSection();
+
+        assertThat(apiCallSection, notNullValue());
+        assertThat(apiCallSection.getApi(), equalTo("index"));
+        assertThat(apiCallSection.getParams().size(), equalTo(3));
+        assertThat(apiCallSection.getParams().get("index"), equalTo("test_1"));
+        assertThat(apiCallSection.getParams().get("type"), equalTo("test"));
+        assertThat(apiCallSection.getParams().get("id"), equalTo("1"));
+        assertThat(apiCallSection.hasBody(), equalTo(true));
+        assertThat(apiCallSection.getBodies().size(), equalTo(1));
+        //stringified body is taken as is
+        assertJsonEquals(apiCallSection.getBodies().get(0), "{ \"_source\": true, \"query\": { \"match_all\": {} } }");
+    }
+
+    public void testParseDoSectionWithBodiesStringifiedAndNot() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "index:\n" +
+                "    body:\n" +
+                "        - \"{ \\\"_source\\\": true, \\\"query\\\": { \\\"match_all\\\": {} } }\"\n" +
+                "        - { size: 100, query: { match_all: {} } }"
+        );
+
+        String body = "{ \"size\": 100, \"query\": { \"match_all\": {} } }";
+
+        DoSection doSection = DoSection.parse(parser);
+        ApiCallSection apiCallSection = doSection.getApiCallSection();
+
+        assertThat(apiCallSection.getApi(), equalTo("index"));
+        assertThat(apiCallSection.getParams().size(), equalTo(0));
+        assertThat(apiCallSection.hasBody(), equalTo(true));
+        assertThat(apiCallSection.getBodies().size(), equalTo(2));
+        //stringified body is taken as is
+        assertJsonEquals(apiCallSection.getBodies().get(0), "{ \"_source\": true, \"query\": { \"match_all\": {} } }");
+        assertJsonEquals(apiCallSection.getBodies().get(1), body);
+    }
+
+    public void testParseDoSectionWithCatch() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "catch: missing\n" +
+                "indices.get_warmer:\n" +
+                "    index: test_index\n" +
+                "    name: test_warmer"
+        );
+
+        DoSection doSection = DoSection.parse(parser);
+        assertThat(doSection.getCatch(), equalTo("missing"));
+        assertThat(doSection.getApiCallSection(), notNullValue());
+        assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_warmer"));
+        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2));
+        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
+    }
+
+    public void testParseDoSectionWithHeaders() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "headers:\n" +
+                "    Authorization: \"thing one\"\n" +
+                "    Content-Type: \"application/json\"\n" +
+                "indices.get_warmer:\n" +
+                "    index: test_index\n" +
+                "    name: test_warmer"
+        );
+
+        DoSection doSection = DoSection.parse(parser);
+        assertThat(doSection.getApiCallSection(), notNullValue());
+        assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_warmer"));
+        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2));
+        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
+        assertThat(doSection.getApiCallSection().getHeaders(), notNullValue());
+        assertThat(doSection.getApiCallSection().getHeaders().size(), equalTo(2));
+        assertThat(doSection.getApiCallSection().getHeaders().get("Authorization"), equalTo("thing one"));
+        assertThat(doSection.getApiCallSection().getHeaders().get("Content-Type"), equalTo("application/json"));
+    }
+
+    public void testParseDoSectionWithoutClientCallSection() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "catch: missing\n"
+        );
+
+        Exception e = expectThrows(IllegalArgumentException.class, () -> DoSection.parse(parser));
+        assertThat(e.getMessage(), is("client call section is mandatory within a do section"));
+    }
+
+    public void testParseDoSectionMultivaluedField() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "indices.get_field_mapping:\n" +
+                "        index: test_index\n" +
+                "        type: test_type\n" +
+                "        field: [ text , text1 ]"
+        );
+
+        DoSection doSection = DoSection.parse(parser);
+        assertThat(doSection.getCatch(), nullValue());
+        assertThat(doSection.getApiCallSection(), notNullValue());
+        assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_field_mapping"));
+        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3));
+        assertThat(doSection.getApiCallSection().getParams().get("index"), equalTo("test_index"));
+        assertThat(doSection.getApiCallSection().getParams().get("type"), equalTo("test_type"));
+        assertThat(doSection.getApiCallSection().getParams().get("field"), equalTo("text,text1"));
+        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
+        assertThat(doSection.getApiCallSection().getBodies().size(), equalTo(0));
+    }
+
+    public void testParseDoSectionExpectedWarnings() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "indices.get_field_mapping:\n" +
+                "        index: test_index\n" +
+                "        type: test_type\n" +
+                "warnings:\n" +
+                "    - some test warning they are typically pretty long\n" +
+                "    - some other test warning somtimes they have [in] them"
+        );
+
+        DoSection doSection = DoSection.parse(parser);
+        assertThat(doSection.getCatch(), nullValue());
+        assertThat(doSection.getApiCallSection(), notNullValue());
+        assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_field_mapping"));
+        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2));
+        assertThat(doSection.getApiCallSection().getParams().get("index"), equalTo("test_index"));
+        assertThat(doSection.getApiCallSection().getParams().get("type"), equalTo("test_type"));
+        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
+        assertThat(doSection.getApiCallSection().getBodies().size(), equalTo(0));
+        assertThat(doSection.getExpectedWarningHeaders(), equalTo(Arrays.asList(
+                "some test warning they are typically pretty long",
+                "some other test warning somtimes they have [in] them")));
+
+        parser = createParser(YamlXContent.yamlXContent,
+                "indices.get_field_mapping:\n" +
+                "        index: test_index\n" +
+                "warnings:\n" +
+                "    - just one entry this time"
+        );
+
+        doSection = DoSection.parse(parser);
+        assertThat(doSection.getCatch(), nullValue());
+        assertThat(doSection.getApiCallSection(), notNullValue());
+        assertThat(doSection.getExpectedWarningHeaders(), equalTo(singletonList(
+                "just one entry this time")));
+
+    }
+
+    private void assertJsonEquals(Map<String, Object> actual, String expected) throws IOException {
+        Map<String,Object> expectedMap;
+        try (XContentParser parser = createParser(YamlXContent.yamlXContent, expected)) {
+            expectedMap = parser.mapOrdered();
+        }
+        MatcherAssert.assertThat(actual, equalTo(expectedMap));
     }
 }
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/SetSectionTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/SetSectionTests.java
new file mode 100644
index 0000000000000..74b3d774bd96f
--- /dev/null
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/SetSectionTests.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.test.rest.yaml.section;
+
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.xcontent.yaml.YamlXContent;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+
+public class SetSectionTests extends AbstractClientYamlTestFragmentParserTestCase {
+    public void testParseSetSectionSingleValue() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                        "{ _id: id }"
+        );
+
+        SetSection setSection = SetSection.parse(parser);
+        assertThat(setSection, notNullValue());
+        assertThat(setSection.getStash(), notNullValue());
+        assertThat(setSection.getStash().size(), equalTo(1));
+        assertThat(setSection.getStash().get("_id"), equalTo("id"));
+    }
+
+    public void testParseSetSectionMultipleValues() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "{ _id: id, _type: type, _index: index }"
+        );
+
+        SetSection setSection = SetSection.parse(parser);
+        assertThat(setSection, notNullValue());
+        assertThat(setSection.getStash(), notNullValue());
+        assertThat(setSection.getStash().size(), equalTo(3));
+        assertThat(setSection.getStash().get("_id"), equalTo("id"));
+        assertThat(setSection.getStash().get("_type"), equalTo("type"));
+        assertThat(setSection.getStash().get("_index"), equalTo("index"));
+    }
+
+    public void testParseSetSectionNoValues() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "{ }"
+        );
+
+        Exception e = expectThrows(ParsingException.class, () -> SetSection.parse(parser));
+        assertThat(e.getMessage(), is("set section must set at least a value"));
+    }
+}
\ No newline at end of file
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/SetupSectionTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/SetupSectionTests.java
new file mode 100644
index 0000000000000..f6174cf0be207
--- /dev/null
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/SetupSectionTests.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.test.rest.yaml.section;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.xcontent.yaml.YamlXContent;
+import org.elasticsearch.test.rest.yaml.section.SetupSection;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.notNullValue;
+
+public class SetupSectionTests extends AbstractClientYamlTestFragmentParserTestCase {
+    public void testParseSetupSection() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "  - do:\n" +
+                "      index1:\n" +
+                "        index:  test_1\n" +
+                "        type:   test\n" +
+                "        id:     1\n" +
+                "        body:   { \"include\": { \"field1\": \"v1\", \"field2\": \"v2\" }, \"count\": 1 }\n" +
+                "  - do:\n" +
+                "      index2:\n" +
+                "        index:  test_1\n" +
+                "        type:   test\n" +
+                "        id:     2\n" +
+                "        body:   { \"include\": { \"field1\": \"v1\", \"field2\": \"v2\" }, \"count\": 1 }\n"
+        );
+
+        SetupSection setupSection = SetupSection.parse(parser);
+
+        assertThat(setupSection, notNullValue());
+        assertThat(setupSection.getSkipSection().isEmpty(), equalTo(true));
+        assertThat(setupSection.getDoSections().size(), equalTo(2));
+        assertThat(setupSection.getDoSections().get(0).getApiCallSection().getApi(), equalTo("index1"));
+        assertThat(setupSection.getDoSections().get(1).getApiCallSection().getApi(), equalTo("index2"));
+    }
+
+    public void testParseSetupAndSkipSectionNoSkip() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "  - skip:\n" +
+                        "      version:  \"2.0.0 - 2.3.0\"\n" +
+                        "      reason:   \"Update doesn't return metadata fields, waiting for #3259\"\n" +
+                        "  - do:\n" +
+                        "      index1:\n" +
+                        "        index:  test_1\n" +
+                        "        type:   test\n" +
+                        "        id:     1\n" +
+                        "        body:   { \"include\": { \"field1\": \"v1\", \"field2\": \"v2\" }, \"count\": 1 }\n" +
+                        "  - do:\n" +
+                        "      index2:\n" +
+                        "        index:  test_1\n" +
+                        "        type:   test\n" +
+                        "        id:     2\n" +
+                        "        body:   { \"include\": { \"field1\": \"v1\", \"field2\": \"v2\" }, \"count\": 1 }\n"
+        );
+
+        SetupSection setupSection = SetupSection.parse(parser);
+
+        assertThat(setupSection, notNullValue());
+        assertThat(setupSection.getSkipSection().isEmpty(), equalTo(false));
+        assertThat(setupSection.getSkipSection(), notNullValue());
+        assertThat(setupSection.getSkipSection().getLowerVersion(), equalTo(Version.V_2_0_0));
+        assertThat(setupSection.getSkipSection().getUpperVersion(), equalTo(Version.V_2_3_0));
+        assertThat(setupSection.getSkipSection().getReason(), equalTo("Update doesn't return metadata fields, waiting for #3259"));
+        assertThat(setupSection.getDoSections().size(), equalTo(2));
+        assertThat(setupSection.getDoSections().get(0).getApiCallSection().getApi(), equalTo("index1"));
+        assertThat(setupSection.getDoSections().get(1).getApiCallSection().getApi(), equalTo("index2"));
+    }
+}
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/SkipSectionTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/SkipSectionTests.java
new file mode 100644
index 0000000000000..85304be1711a7
--- /dev/null
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/SkipSectionTests.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.rest.yaml.section;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.xcontent.yaml.YamlXContent;
+import org.elasticsearch.test.VersionUtils;
+
+import java.util.Collections;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+
+public class SkipSectionTests extends AbstractClientYamlTestFragmentParserTestCase {
+
+    public void testSkip() {
+        SkipSection section = new SkipSection("2.0.0 - 2.1.0",
+                randomBoolean() ? Collections.emptyList() : Collections.singletonList("warnings"), "foobar");
+        assertFalse(section.skip(Version.CURRENT));
+        assertTrue(section.skip(Version.V_2_0_0));
+        section = new SkipSection(randomBoolean() ? null : "2.0.0 - 2.1.0", Collections.singletonList("boom"), "foobar");
+        assertTrue(section.skip(Version.CURRENT));
+    }
+
+    public void testMessage() {
+        SkipSection section = new SkipSection("2.0.0 - 2.1.0", Collections.singletonList("warnings"), "foobar");
+        assertEquals("[FOOBAR] skipped, reason: [foobar] unsupported features [warnings]", section.getSkipMessage("FOOBAR"));
+        section = new SkipSection(null, Collections.singletonList("warnings"), "foobar");
+        assertEquals("[FOOBAR] skipped, reason: [foobar] unsupported features [warnings]", section.getSkipMessage("FOOBAR"));
+        section = new SkipSection(null, Collections.singletonList("warnings"), null);
+        assertEquals("[FOOBAR] skipped, unsupported features [warnings]", section.getSkipMessage("FOOBAR"));
+    }
+
+    public void testParseSkipSectionVersionNoFeature() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "version:     \" - 2.1.0\"\n" +
+                "reason:      Delete ignores the parent param"
+        );
+
+        SkipSection skipSection = SkipSection.parse(parser);
+        assertThat(skipSection, notNullValue());
+        assertThat(skipSection.getLowerVersion(), equalTo(VersionUtils.getFirstVersion()));
+        assertThat(skipSection.getUpperVersion(), equalTo(Version.V_2_1_0));
+        assertThat(skipSection.getFeatures().size(), equalTo(0));
+        assertThat(skipSection.getReason(), equalTo("Delete ignores the parent param"));
+    }
+
+    public void testParseSkipSectionAllVersions() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+            "version:     \" all \"\n" +
+            "reason:      Delete ignores the parent param"
+        );
+
+        SkipSection skipSection = SkipSection.parse(parser);
+        assertThat(skipSection, notNullValue());
+        assertThat(skipSection.getLowerVersion(), equalTo(VersionUtils.getFirstVersion()));
+        assertThat(skipSection.getUpperVersion(), equalTo(Version.CURRENT));
+        assertThat(skipSection.getFeatures().size(), equalTo(0));
+        assertThat(skipSection.getReason(), equalTo("Delete ignores the parent param"));
+    }
+
+    public void testParseSkipSectionFeatureNoVersion() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "features:     regex"
+        );
+
+        SkipSection skipSection = SkipSection.parse(parser);
+        assertThat(skipSection, notNullValue());
+        assertThat(skipSection.isVersionCheck(), equalTo(false));
+        assertThat(skipSection.getFeatures().size(), equalTo(1));
+        assertThat(skipSection.getFeatures().get(0), equalTo("regex"));
+        assertThat(skipSection.getReason(), nullValue());
+    }
+
+    public void testParseSkipSectionFeaturesNoVersion() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "features:     [regex1,regex2,regex3]"
+        );
+
+        SkipSection skipSection = SkipSection.parse(parser);
+        assertThat(skipSection, notNullValue());
+        assertThat(skipSection.isVersionCheck(), equalTo(false));
+        assertThat(skipSection.getFeatures().size(), equalTo(3));
+        assertThat(skipSection.getFeatures().get(0), equalTo("regex1"));
+        assertThat(skipSection.getFeatures().get(1), equalTo("regex2"));
+        assertThat(skipSection.getFeatures().get(2), equalTo("regex3"));
+        assertThat(skipSection.getReason(), nullValue());
+    }
+
+    public void testParseSkipSectionBothFeatureAndVersion() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "version:     \" - 0.90.2\"\n" +
+                "features:     regex\n" +
+                "reason:      Delete ignores the parent param"
+        );
+
+        SkipSection skipSection = SkipSection.parse(parser);
+        assertEquals(VersionUtils.getFirstVersion(), skipSection.getLowerVersion());
+        assertEquals(Version.fromString("0.90.2"), skipSection.getUpperVersion());
+        assertEquals(Collections.singletonList("regex"), skipSection.getFeatures());
+        assertEquals("Delete ignores the parent param", skipSection.getReason());
+    }
+
+    public void testParseSkipSectionNoReason() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "version:     \" - 0.90.2\"\n"
+        );
+
+        Exception e = expectThrows(ParsingException.class, () -> SkipSection.parse(parser));
+        assertThat(e.getMessage(), is("reason is mandatory within skip version section"));
+    }
+
+    public void testParseSkipSectionNoVersionNorFeature() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "reason:      Delete ignores the parent param\n"
+        );
+
+        Exception e = expectThrows(ParsingException.class, () -> SkipSection.parse(parser));
+        assertThat(e.getMessage(), is("version or features is mandatory within skip section"));
+    }
+}
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/TeardownSectionTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/TeardownSectionTests.java
new file mode 100644
index 0000000000000..f057d0d370d63
--- /dev/null
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/TeardownSectionTests.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.rest.yaml.section;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.xcontent.yaml.YamlXContent;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.notNullValue;
+
+/**
+ * Unit tests for the teardown section.
+ */
+public class TeardownSectionTests extends AbstractClientYamlTestFragmentParserTestCase {
+    public void testParseTeardownSection() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "  - do:\n" +
+                "      delete:\n" +
+                "        index: foo\n" +
+                "        type: doc\n" +
+                "        id: 1\n" +
+                "        ignore: 404\n" +
+                "  - do:\n" +
+                "      delete2:\n" +
+                "        index: foo\n" +
+                "        type: doc\n" +
+                "        id: 1\n" +
+                "        ignore: 404"
+        );
+
+        TeardownSection section = TeardownSection.parse(parser);
+        assertThat(section, notNullValue());
+        assertThat(section.getSkipSection().isEmpty(), equalTo(true));
+        assertThat(section.getDoSections().size(), equalTo(2));
+        assertThat(section.getDoSections().get(0).getApiCallSection().getApi(), equalTo("delete"));
+        assertThat(section.getDoSections().get(1).getApiCallSection().getApi(), equalTo("delete2"));
+    }
+
+    public void testParseWithSkip() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+            "  - skip:\n" +
+                "      version:  \"2.0.0 - 2.3.0\"\n" +
+                "      reason:   \"there is a reason\"\n" +
+                "  - do:\n" +
+                "      delete:\n" +
+                "        index: foo\n" +
+                "        type: doc\n" +
+                "        id: 1\n" +
+                "        ignore: 404\n" +
+                "  - do:\n" +
+                "      delete2:\n" +
+                "        index: foo\n" +
+                "        type: doc\n" +
+                "        id: 1\n" +
+                "        ignore: 404"
+        );
+
+        TeardownSection section = TeardownSection.parse(parser);
+        assertThat(section, notNullValue());
+        assertThat(section.getSkipSection().isEmpty(), equalTo(false));
+        assertThat(section.getSkipSection().getLowerVersion(), equalTo(Version.V_2_0_0));
+        assertThat(section.getSkipSection().getUpperVersion(), equalTo(Version.V_2_3_0));
+        assertThat(section.getSkipSection().getReason(), equalTo("there is a reason"));
+        assertThat(section.getDoSections().size(), equalTo(2));
+        assertThat(section.getDoSections().get(0).getApiCallSection().getApi(), equalTo("delete"));
+        assertThat(section.getDoSections().get(1).getApiCallSection().getApi(), equalTo("delete2"));
+    }
+}
diff --git a/test/framework/src/test/java/org/elasticsearch/test/test/ESTestCaseTests.java b/test/framework/src/test/java/org/elasticsearch/test/test/ESTestCaseTests.java
index 005cd64348017..bf4c786c110f1 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/test/ESTestCaseTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/test/ESTestCaseTests.java
@@ -21,17 +21,22 @@
 
 import junit.framework.AssertionFailedError;
 
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.RandomObjects;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.HashMap;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import static org.hamcrest.Matchers.greaterThan;
@@ -65,52 +70,86 @@ public void testExpectThrows() {
         }
     }
 
-    public void testShuffleXContent() throws IOException {
-        Map<String, Object> randomStringObjectMap = randomStringObjectMap(5);
-        XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
-        builder.map(randomStringObjectMap);
-        XContentBuilder shuffleXContent = shuffleXContent(builder);
-        XContentParser parser = XContentFactory.xContent(shuffleXContent.bytes()).createParser(shuffleXContent.bytes());
-        Map<String, Object> resultMap = parser.map();
-        assertEquals("both maps should contain the same mappings", randomStringObjectMap, resultMap);
-        assertNotEquals("Both builders string representations should be different", builder.bytes(), shuffleXContent.bytes());
+    public void testShuffleMap() throws IOException {
+        XContentType xContentType = randomFrom(XContentType.values());
+        BytesReference source = RandomObjects.randomSource(random(), xContentType, 5);
+        try (XContentParser parser = createParser(xContentType.xContent(), source)) {
+            LinkedHashMap<String, Object> initialMap = (LinkedHashMap<String, Object>)parser.mapOrdered();
+
+            Set<List<String>> distinctKeys = new HashSet<>();
+            for (int i = 0; i < 10; i++) {
+                LinkedHashMap<String, Object> shuffledMap = shuffleMap(initialMap, Collections.emptySet());
+                assertEquals("both maps should contain the same mappings", initialMap, shuffledMap);
+                List<String> shuffledKeys = new ArrayList<>(shuffledMap.keySet());
+                distinctKeys.add(shuffledKeys);
+            }
+            //out of 10 shuffling runs we expect to have at least more than 1 distinct output.
+            //This is to make sure that we actually do the shuffling
+            assertThat(distinctKeys.size(), greaterThan(1));
+        }
     }
 
-    private static Map<String, Object> randomStringObjectMap(int depth) {
-        Map<String, Object> result = new HashMap<>();
-        int entries = randomInt(10);
-        for (int i = 0; i < entries; i++) {
-            String key = randomAsciiOfLengthBetween(5, 15);
-            int suprise = randomIntBetween(0, 4);
-            switch (suprise) {
-            case 0:
-                result.put(key, randomUnicodeOfCodepointLength(20));
-                break;
-            case 1:
-                result.put(key, randomInt(100));
-                break;
-            case 2:
-                result.put(key, randomDoubleBetween(-100.0, 100.0, true));
-                break;
-            case 3:
-                result.put(key, randomBoolean());
-                break;
-            case 4:
-                List<String> stringList = new ArrayList<>();
-                int size = randomInt(5);
-                for (int s = 0; s < size; s++) {
-                    stringList.add(randomUnicodeOfCodepointLength(20));
+    public void testShuffleXContentExcludeFields() throws IOException {
+        XContentType xContentType = randomFrom(XContentType.values());
+        try (XContentBuilder builder = XContentBuilder.builder(xContentType.xContent())) {
+            builder.startObject();
+            {
+                builder.field("field1", "value1");
+                builder.field("field2", "value2");
+                {
+                    builder.startObject("object1");
+                    {
+                        builder.field("inner1", "value1");
+                        builder.field("inner2", "value2");
+                        builder.field("inner3", "value3");
+                    }
+                    builder.endObject();
+                }
+                {
+                    builder.startObject("object2");
+                    {
+                        builder.field("inner4", "value4");
+                        builder.field("inner5", "value5");
+                        builder.field("inner6", "value6");
+                    }
+                    builder.endObject();
                 }
-                result.put(key, stringList);
-                break;
-            default:
-                throw new IllegalArgumentException("unexpected random option: " + suprise);
             }
+            builder.endObject();
+            BytesReference bytes = builder.bytes();
+            final LinkedHashMap<String, Object> initialMap;
+            try (XContentParser parser = createParser(xContentType.xContent(), bytes)) {
+                initialMap = (LinkedHashMap<String, Object>)parser.mapOrdered();
+            }
+
+            List<String> expectedInnerKeys1 = Arrays.asList("inner1", "inner2", "inner3");
+            Set<List<String>> distinctTopLevelKeys = new HashSet<>();
+            Set<List<String>> distinctInnerKeys2 = new HashSet<>();
+            for (int i = 0; i < 10; i++) {
+                try (XContentParser parser = createParser(xContentType.xContent(), bytes)) {
+                    try (XContentBuilder shuffledBuilder = shuffleXContent(parser, randomBoolean(), "object1")) {
+                        try (XContentParser shuffledParser = createParser(shuffledBuilder)) {
+                            Map<String, Object> shuffledMap = shuffledParser.mapOrdered();
+                            assertEquals("both maps should contain the same mappings", initialMap, shuffledMap);
+                            List<String> shuffledKeys = new ArrayList<>(shuffledMap.keySet());
+                            distinctTopLevelKeys.add(shuffledKeys);
+                            @SuppressWarnings("unchecked")
+                            Map<String, Object> innerMap1 = (Map<String, Object>)shuffledMap.get("object1");
+                            List<String> actualInnerKeys1 = new ArrayList<>(innerMap1.keySet());
+                            assertEquals("object1 should have been left untouched", expectedInnerKeys1, actualInnerKeys1);
+                            @SuppressWarnings("unchecked")
+                            Map<String, Object> innerMap2 = (Map<String, Object>)shuffledMap.get("object2");
+                            List<String> actualInnerKeys2 = new ArrayList<>(innerMap2.keySet());
+                            distinctInnerKeys2.add(actualInnerKeys2);
+                        }
+                    }
+                }
+            }
+
+            //out of 10 shuffling runs we expect to have at least more than 1 distinct output for both top level keys and inner object2
+            assertThat(distinctTopLevelKeys.size(), greaterThan(1));
+            assertThat(distinctInnerKeys2.size(), greaterThan(1));
         }
-        if (depth > 0) {
-            result.put(randomAsciiOfLengthBetween(5, 15), randomStringObjectMap(depth - 1));
-        }
-        return result;
     }
 
     public void testRandomUniqueNotUnique() {
@@ -123,6 +162,6 @@ public void testRandomUniqueTotallyUnique() {
     }
 
     public void testRandomUniqueNormalUsageAlwayMoreThanOne() {
-        assertThat(randomUnique(() -> randomAsciiOfLengthBetween(1, 20), 10), hasSize(greaterThan(0)));
+        assertThat(randomUnique(() -> randomAlphaOfLengthBetween(1, 20), 10), hasSize(greaterThan(0)));
     }
 }
diff --git a/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java b/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java
index f0b7454fe9d1f..090f1c7e0d65c 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java
@@ -27,17 +27,21 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.discovery.DiscoveryModule;
 import org.elasticsearch.discovery.DiscoverySettings;
+import org.elasticsearch.discovery.zen.ZenDiscovery;
 import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.InternalTestCluster;
 import org.elasticsearch.test.NodeConfigurationSource;
+import org.elasticsearch.test.discovery.TestZenDiscovery;
+import org.elasticsearch.transport.MockTcpTransportPlugin;
 import org.elasticsearch.transport.TransportSettings;
 
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
@@ -53,6 +57,7 @@
 import static org.elasticsearch.cluster.node.DiscoveryNode.Role.DATA;
 import static org.elasticsearch.cluster.node.DiscoveryNode.Role.INGEST;
 import static org.elasticsearch.cluster.node.DiscoveryNode.Role.MASTER;
+import static org.elasticsearch.discovery.zen.ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFileExists;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFileNotExists;
 import static org.hamcrest.Matchers.equalTo;
@@ -66,6 +71,12 @@
 @LuceneTestCase.SuppressFileSystems("ExtrasFS") // doesn't work with potential multi data path from test cluster yet
 public class InternalTestClusterTests extends ESTestCase {
 
+    @Override
+    protected boolean enableWarningsCheck() {
+        //some settings randomization in InternalTestCluster cause deprecation warnings, that's ok
+        return false;
+    }
+
     public void testInitializiationIsConsistent() {
         long clusterSeed = randomLong();
         boolean masterNodes = randomBoolean();
@@ -79,10 +90,10 @@ public void testInitializiationIsConsistent() {
 
         Path baseDir = createTempDir();
         InternalTestCluster cluster0 = new InternalTestCluster(clusterSeed, baseDir, masterNodes,
-            minNumDataNodes, maxNumDataNodes, clusterName, nodeConfigurationSource, numClientNodes,
+            randomBoolean(), minNumDataNodes, maxNumDataNodes, clusterName, nodeConfigurationSource, numClientNodes,
             enableHttpPipelining, nodePrefix, Collections.emptyList(), Function.identity());
         InternalTestCluster cluster1 = new InternalTestCluster(clusterSeed, baseDir, masterNodes,
-            minNumDataNodes, maxNumDataNodes, clusterName, nodeConfigurationSource, numClientNodes,
+            randomBoolean(), minNumDataNodes, maxNumDataNodes, clusterName, nodeConfigurationSource, numClientNodes,
             enableHttpPipelining, nodePrefix, Collections.emptyList(), Function.identity());
         // TODO: this is not ideal - we should have a way to make sure ports are initialized in the same way
         assertClusters(cluster0, cluster1, false);
@@ -114,7 +125,8 @@ public static void assertClusters(InternalTestCluster cluster0, InternalTestClus
     public static void assertSettings(Settings left, Settings right, boolean checkClusterUniqueSettings) {
         Set<Map.Entry<String, String>> entries0 = left.getAsMap().entrySet();
         Map<String, String> entries1 = right.getAsMap();
-        assertThat(entries0.size(), equalTo(entries1.size()));
+        assertThat("--> left:\n" + left.toDelimitedString('\n') +  "\n-->right:\n" + right.toDelimitedString('\n'),
+            entries0.size(), equalTo(entries1.size()));
         for (Map.Entry<String, String> entry : entries0) {
             if (clusterUniqueSettings.contains(entry.getKey()) && checkClusterUniqueSettings == false) {
                 continue;
@@ -123,24 +135,68 @@ public static void assertSettings(Settings left, Settings right, boolean checkCl
         }
     }
 
+    private void assertMMNinNodeSetting(InternalTestCluster cluster, int masterNodes) {
+        for (final String node : cluster.getNodeNames()) {
+            assertMMNinNodeSetting(node, cluster, masterNodes);
+        }
+    }
+
+    private void assertMMNinNodeSetting(String node, InternalTestCluster cluster, int masterNodes) {
+        final int minMasterNodes = masterNodes / 2 + 1;
+        Settings nodeSettings = cluster.client(node).admin().cluster().prepareNodesInfo(node).get().getNodes().get(0).getSettings();
+        assertThat("node setting of node [" + node + "] has the wrong min_master_node setting: ["
+            + nodeSettings.get(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()) + "]",
+            nodeSettings.getAsMap(),
+            hasEntry(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.toString(minMasterNodes)));
+    }
+
+    private void assertMMNinClusterSetting(InternalTestCluster cluster, int masterNodes) {
+        final int minMasterNodes = masterNodes / 2 + 1;
+        for (final String node : cluster.getNodeNames()) {
+            Settings stateSettings = cluster.client(node).admin().cluster().prepareState().setLocal(true)
+                .get().getState().getMetaData().settings();
+
+            assertThat("dynamic setting for node [" + node + "] has the wrong min_master_node setting : ["
+                    + stateSettings.get(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()) + "]",
+                stateSettings.getAsMap(),
+                hasEntry(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.toString(minMasterNodes)));
+        }
+    }
+
     public void testBeforeTest() throws Exception {
+        final boolean autoManageMinMasterNodes = randomBoolean();
         long clusterSeed = randomLong();
-        boolean masterNodes = randomBoolean();
-        int minNumDataNodes = randomIntBetween(0, 3);
-        int maxNumDataNodes = randomIntBetween(minNumDataNodes, 4);
-        int numClientNodes = randomIntBetween(0, 2);
+        final boolean masterNodes;
+        final int minNumDataNodes;
+        final int maxNumDataNodes;
+        if (autoManageMinMasterNodes) {
+            masterNodes = randomBoolean();
+            minNumDataNodes = randomIntBetween(0, 3);
+            maxNumDataNodes = randomIntBetween(minNumDataNodes, 4);
+        } else {
+            // if we manage min master nodes, we need to lock down the number of nodes
+            minNumDataNodes = randomIntBetween(0, 4);
+            maxNumDataNodes = minNumDataNodes;
+            masterNodes = false;
+        }
+        final int numClientNodes = randomIntBetween(0, 2);
         final String clusterName1 = "shared1";
         final String clusterName2 = "shared2";
         NodeConfigurationSource nodeConfigurationSource = new NodeConfigurationSource() {
             @Override
             public Settings nodeSettings(int nodeOrdinal) {
-                return Settings.builder()
+                final Settings.Builder settings = Settings.builder()
                     .put(
                         NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.getKey(),
                         2 * ((masterNodes ? InternalTestCluster.DEFAULT_HIGH_NUM_MASTER_NODES : 0) + maxNumDataNodes + numClientNodes))
                     .put(NetworkModule.HTTP_ENABLED.getKey(), false)
-                    .put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), "local")
-                    .put(NetworkModule.TRANSPORT_TYPE_KEY, "local").build();
+                    .put(NetworkModule.TRANSPORT_TYPE_KEY, "local");
+                if (autoManageMinMasterNodes == false) {
+                    assert minNumDataNodes == maxNumDataNodes;
+                    assert masterNodes == false;
+                    settings.put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), minNumDataNodes / 2 + 1);
+                }
+                return settings.build();
             }
 
             @Override
@@ -154,12 +210,13 @@ public Settings transportClientSettings() {
         String nodePrefix = "foobar";
 
         Path baseDir = createTempDir();
+        final List<Class<? extends Plugin>> mockPlugins = Collections.singletonList( TestZenDiscovery.TestPlugin.class);
         InternalTestCluster cluster0 = new InternalTestCluster(clusterSeed, baseDir, masterNodes,
-            minNumDataNodes, maxNumDataNodes, clusterName1, nodeConfigurationSource, numClientNodes,
-            enableHttpPipelining, nodePrefix, Collections.emptyList(), Function.identity());
+            autoManageMinMasterNodes, minNumDataNodes, maxNumDataNodes, clusterName1, nodeConfigurationSource, numClientNodes,
+            enableHttpPipelining, nodePrefix, mockPlugins, Function.identity());
         InternalTestCluster cluster1 = new InternalTestCluster(clusterSeed, baseDir, masterNodes,
-            minNumDataNodes, maxNumDataNodes, clusterName2, nodeConfigurationSource, numClientNodes,
-            enableHttpPipelining, nodePrefix, Collections.emptyList(), Function.identity());
+            autoManageMinMasterNodes, minNumDataNodes, maxNumDataNodes, clusterName2, nodeConfigurationSource, numClientNodes,
+            enableHttpPipelining, nodePrefix, mockPlugins, Function.identity());
 
         assertClusters(cluster0, cluster1, false);
         long seed = randomLong();
@@ -180,6 +237,8 @@ public Settings transportClientSettings() {
                 assertSettings(client.settings(), other.settings(), false);
             }
             assertArrayEquals(cluster0.getNodeNames(), cluster1.getNodeNames());
+            assertMMNinNodeSetting(cluster0, cluster0.numMasterNodes());
+            assertMMNinNodeSetting(cluster1, cluster0.numMasterNodes());
             cluster0.afterTest();
             cluster1.afterTest();
         } finally {
@@ -203,7 +262,6 @@ public Settings nodeSettings(int nodeOrdinal) {
                         NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.getKey(),
                         2 + (masterNodes ? InternalTestCluster.DEFAULT_HIGH_NUM_MASTER_NODES : 0) + maxNumDataNodes + numClientNodes)
                     .put(NetworkModule.TRANSPORT_TYPE_KEY, "local")
-                    .put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), "local")
                     .build();
             }
             @Override
@@ -216,10 +274,13 @@ public Settings transportClientSettings() {
         String nodePrefix = "test";
         Path baseDir = createTempDir();
         InternalTestCluster cluster = new InternalTestCluster(clusterSeed, baseDir, masterNodes,
-            minNumDataNodes, maxNumDataNodes, clusterName1, nodeConfigurationSource, numClientNodes,
-            enableHttpPipelining, nodePrefix, Collections.emptyList(), Function.identity());
+            true, minNumDataNodes, maxNumDataNodes, clusterName1, nodeConfigurationSource, numClientNodes,
+            enableHttpPipelining, nodePrefix, Collections.singletonList(TestZenDiscovery.TestPlugin.class),
+            Function.identity());
         try {
             cluster.beforeTest(random(), 0.0);
+            final int originalMasterCount = cluster.numMasterNodes();
+            assertMMNinNodeSetting(cluster, originalMasterCount);
             final Map<String,Path[]> shardNodePaths = new HashMap<>();
             for (String name: cluster.getNodeNames()) {
                 shardNodePaths.put(name, getNodePaths(cluster, name));
@@ -228,7 +289,15 @@ public Settings transportClientSettings() {
             Path dataPath = getNodePaths(cluster, poorNode)[0];
             final Path testMarker = dataPath.resolve("testMarker");
             Files.createDirectories(testMarker);
+            int expectedMasterCount = originalMasterCount;
+            if (cluster.getInstance(ClusterService.class, poorNode).localNode().isMasterNode()) {
+                expectedMasterCount--;
+            }
             cluster.stopRandomNode(InternalTestCluster.nameFilter(poorNode));
+            if (expectedMasterCount != originalMasterCount) {
+                // check for updated
+                assertMMNinClusterSetting(cluster, expectedMasterCount);
+            }
             assertFileExists(testMarker); // stopping a node half way shouldn't clean data
 
             final String stableNode = randomFrom(cluster.getNodeNames());
@@ -238,10 +307,17 @@ public Settings transportClientSettings() {
             Files.createDirectories(stableTestMarker);
 
             final String newNode1 =  cluster.startNode();
+            expectedMasterCount++;
             assertThat(getNodePaths(cluster, newNode1)[0], equalTo(dataPath));
             assertFileExists(testMarker); // starting a node should re-use data folders and not clean it
+            if (expectedMasterCount > 1) { // this is the first master, it's in cluster state settings won't be updated
+                assertMMNinClusterSetting(cluster, expectedMasterCount);
+            }
+            assertMMNinNodeSetting(newNode1, cluster, expectedMasterCount);
 
             final String newNode2 =  cluster.startNode();
+            expectedMasterCount++;
+            assertMMNinClusterSetting(cluster, expectedMasterCount);
             final Path newDataPath = getNodePaths(cluster, newNode2)[0];
             final Path newTestMarker = newDataPath.resolve("newTestMarker");
             assertThat(newDataPath, not(dataPath));
@@ -260,6 +336,7 @@ public Settings transportClientSettings() {
                 assertThat("data paths for " + name + " changed", getNodePaths(cluster, name),
                     equalTo(shardNodePaths.get(name)));
             }
+            assertMMNinNodeSetting(cluster, originalMasterCount);
 
         } finally {
             cluster.close();
@@ -278,16 +355,19 @@ private Path[] getNodePaths(InternalTestCluster cluster, String name) {
     public void testDifferentRolesMaintainPathOnRestart() throws Exception {
         final Path baseDir = createTempDir();
         final int numNodes = 5;
-        InternalTestCluster cluster = new InternalTestCluster(randomLong(), baseDir, true, 0, 0, "test",
-            new NodeConfigurationSource() {
+        InternalTestCluster cluster = new InternalTestCluster(randomLong(), baseDir, false,
+            false, 0, 0, "test", new NodeConfigurationSource() {
                 @Override
                 public Settings nodeSettings(int nodeOrdinal) {
                     return Settings.builder()
                         .put(NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.getKey(), numNodes)
                         .put(NetworkModule.HTTP_ENABLED.getKey(), false)
                         .put(NetworkModule.TRANSPORT_TYPE_KEY, "local")
-                        .put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), "local")
-                        .put(DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING.getKey(), 0).build();
+                        .put(DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING.getKey(), 0)
+                        // speedup join timeout as setting initial state timeout to 0 makes split
+                        // elections more likely
+                        .put(ZenDiscovery.JOIN_TIMEOUT_SETTING.getKey(), "3s")
+                        .build();
                 }
 
                 @Override
@@ -295,22 +375,34 @@ public Settings transportClientSettings() {
                     return Settings.builder()
                         .put(NetworkModule.TRANSPORT_TYPE_KEY, "local").build();
                 }
-            }, 0, randomBoolean(), "", Collections.emptyList(), Function.identity());
+            }, 0, randomBoolean(), "", Collections.singletonList(TestZenDiscovery.TestPlugin.class), Function.identity());
         cluster.beforeTest(random(), 0.0);
+        List<DiscoveryNode.Role> roles = new ArrayList<>();
+        for (int i = 0; i < numNodes; i++) {
+            final DiscoveryNode.Role role = i == numNodes - 1 && roles.contains(MASTER) == false ?
+                MASTER : // last node and still no master
+                randomFrom(MASTER, DiscoveryNode.Role.DATA, DiscoveryNode.Role.INGEST);
+            roles.add(role);
+        }
+
+        final Settings minMasterNodes = Settings.builder()
+            .put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(),
+                roles.stream().filter(role -> role == MASTER).count() / 2 + 1
+            ).build();
         try {
             Map<DiscoveryNode.Role, Set<String>> pathsPerRole = new HashMap<>();
             for (int i = 0; i < numNodes; i++) {
-                final DiscoveryNode.Role role = randomFrom(MASTER, DiscoveryNode.Role.DATA, DiscoveryNode.Role.INGEST);
+                final DiscoveryNode.Role role = roles.get(i);
                 final String node;
                 switch (role) {
                     case MASTER:
-                        node = cluster.startMasterOnlyNode(Settings.EMPTY);
+                        node = cluster.startMasterOnlyNode(minMasterNodes);
                         break;
                     case DATA:
-                        node = cluster.startDataOnlyNode(Settings.EMPTY);
+                        node = cluster.startDataOnlyNode(minMasterNodes);
                         break;
                     case INGEST:
-                        node = cluster.startCoordinatingOnlyNode(Settings.EMPTY);
+                        node = cluster.startCoordinatingOnlyNode(minMasterNodes);
                         break;
                     default:
                         throw new IllegalStateException("get your story straight");
@@ -342,6 +434,58 @@ public Settings transportClientSettings() {
         } finally {
             cluster.close();
         }
+    }
+
+    public void testTwoNodeCluster() throws Exception {
+        NodeConfigurationSource nodeConfigurationSource = new NodeConfigurationSource() {
+            @Override
+            public Settings nodeSettings(int nodeOrdinal) {
+                return Settings.builder().put(NetworkModule.HTTP_ENABLED.getKey(), false)
+                    .put(NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.getKey(), 2)
+                    .put(NetworkModule.TRANSPORT_TYPE_KEY, MockTcpTransportPlugin.MOCK_TCP_TRANSPORT_NAME)
+                    .build();
+            }
 
+            @Override
+            public Settings transportClientSettings() {
+                return Settings.builder()
+                    .put(NetworkModule.TRANSPORT_TYPE_KEY, MockTcpTransportPlugin.MOCK_TCP_TRANSPORT_NAME).build();
+            }
+        };
+        boolean enableHttpPipelining = randomBoolean();
+        String nodePrefix = "test";
+        Path baseDir = createTempDir();
+        InternalTestCluster cluster = new InternalTestCluster(randomLong(), baseDir, false, true, 2, 2,
+            "test", nodeConfigurationSource, 0, enableHttpPipelining, nodePrefix,
+            Arrays.asList(MockTcpTransportPlugin.class, TestZenDiscovery.TestPlugin.class), Function.identity());
+        try {
+            cluster.beforeTest(random(), 0.0);
+            assertMMNinNodeSetting(cluster, 2);
+            switch (randomInt(2)) {
+                case 0:
+                    cluster.stopRandomDataNode();
+                    assertMMNinClusterSetting(cluster, 1);
+                    cluster.startNode();
+                    assertMMNinClusterSetting(cluster, 2);
+                    assertMMNinNodeSetting(cluster, 2);
+                    break;
+                case 1:
+                    cluster.rollingRestart(new InternalTestCluster.RestartCallback() {
+                        @Override
+                        public Settings onNodeStopped(String nodeName) throws Exception {
+                            assertMMNinClusterSetting(cluster, 1);
+                            return super.onNodeStopped(nodeName);
+                        }
+                    });
+                    assertMMNinClusterSetting(cluster, 2);
+                    break;
+                case 2:
+                    cluster.fullRestart();
+                    break;
+            }
+            assertMMNinNodeSetting(cluster, 2);
+        } finally {
+            cluster.close();
+        }
     }
 }
diff --git a/test/framework/src/test/java/org/elasticsearch/test/test/LoggingListenerTests.java b/test/framework/src/test/java/org/elasticsearch/test/test/LoggingListenerTests.java
index 2d4282027410e..0845fc2546f49 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/test/LoggingListenerTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/test/LoggingListenerTests.java
@@ -21,6 +21,7 @@
 
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.junit.annotations.TestLogging;
@@ -50,27 +51,28 @@ public void testCustomLevelPerMethod() throws Exception {
         Logger xyzLogger = Loggers.getLogger("xyz");
         Logger abcLogger = Loggers.getLogger("abc");
 
-        assertEquals(Level.INFO, abcLogger.getLevel());
-        assertThat(xyzLogger.getLevel(), equalTo(Level.INFO));
-        assertThat(abcLogger.getLevel(), equalTo(Level.INFO));
+        final Level level = ESLoggerFactory.getRootLogger().getLevel();
+
+        assertThat(xyzLogger.getLevel(), equalTo(level));
+        assertThat(abcLogger.getLevel(), equalTo(level));
         loggingListener.testRunStarted(suiteDescription);
-        assertThat(xyzLogger.getLevel(), equalTo(Level.INFO));
-        assertThat(abcLogger.getLevel(), equalTo(Level.INFO));
+        assertThat(xyzLogger.getLevel(), equalTo(level));
+        assertThat(abcLogger.getLevel(), equalTo(level));
 
         Method method = TestClass.class.getMethod("annotatedTestMethod");
         TestLogging annotation = method.getAnnotation(TestLogging.class);
         Description testDescription = Description.createTestDescription(LoggingListenerTests.class, "annotatedTestMethod", annotation);
         loggingListener.testStarted(testDescription);
         assertThat(xyzLogger.getLevel(), equalTo(Level.TRACE));
-        assertThat(abcLogger.getLevel(), equalTo(Level.INFO));
+        assertThat(abcLogger.getLevel(), equalTo(level));
 
         loggingListener.testFinished(testDescription);
-        assertThat(xyzLogger.getLevel(), equalTo(Level.INFO));
-        assertThat(abcLogger.getLevel(), equalTo(Level.INFO));
+        assertThat(xyzLogger.getLevel(), equalTo(level));
+        assertThat(abcLogger.getLevel(), equalTo(level));
 
         loggingListener.testRunFinished(new Result());
-        assertThat(xyzLogger.getLevel(), equalTo(Level.INFO));
-        assertThat(abcLogger.getLevel(), equalTo(Level.INFO));
+        assertThat(xyzLogger.getLevel(), equalTo(level));
+        assertThat(abcLogger.getLevel(), equalTo(level));
     }
 
     public void testCustomLevelPerClass() throws Exception {
@@ -80,25 +82,42 @@ public void testCustomLevelPerClass() throws Exception {
 
         Logger abcLogger = Loggers.getLogger("abc");
         Logger xyzLogger = Loggers.getLogger("xyz");
-
-        assertThat(xyzLogger.getLevel(), equalTo(Level.INFO));
-        assertThat(abcLogger.getLevel(), equalTo(Level.INFO));
+        // we include foo and foo.bar to maintain that logging levels are applied from the top of the hierarchy down; this ensures that
+        // setting the logging level for a parent logger and a child logger applies the parent level first and then the child as otherwise
+        // setting the parent level would overwrite the child level
+        Logger fooLogger = Loggers.getLogger("foo");
+        Logger fooBarLogger = Loggers.getLogger("foo.bar");
+
+        final Level level = ESLoggerFactory.getRootLogger().getLevel();
+
+        assertThat(xyzLogger.getLevel(), equalTo(level));
+        assertThat(abcLogger.getLevel(), equalTo(level));
+        assertThat(fooLogger.getLevel(), equalTo(level));
+        assertThat(fooBarLogger.getLevel(), equalTo(level));
         loggingListener.testRunStarted(suiteDescription);
-        assertThat(xyzLogger.getLevel(), equalTo(Level.INFO));
+        assertThat(xyzLogger.getLevel(), equalTo(level));
         assertThat(abcLogger.getLevel(), equalTo(Level.WARN));
+        assertThat(fooLogger.getLevel(), equalTo(Level.WARN));
+        assertThat(fooBarLogger.getLevel(), equalTo(Level.ERROR));
 
         Description testDescription = Description.createTestDescription(LoggingListenerTests.class, "test");
         loggingListener.testStarted(testDescription);
-        assertThat(xyzLogger.getLevel(), equalTo(Level.INFO));
+        assertThat(xyzLogger.getLevel(), equalTo(level));
         assertThat(abcLogger.getLevel(), equalTo(Level.WARN));
+        assertThat(fooLogger.getLevel(), equalTo(Level.WARN));
+        assertThat(fooBarLogger.getLevel(), equalTo(Level.ERROR));
 
         loggingListener.testFinished(testDescription);
-        assertThat(xyzLogger.getLevel(), equalTo(Level.INFO));
+        assertThat(xyzLogger.getLevel(), equalTo(level));
         assertThat(abcLogger.getLevel(), equalTo(Level.WARN));
+        assertThat(fooLogger.getLevel(), equalTo(Level.WARN));
+        assertThat(fooBarLogger.getLevel(), equalTo(Level.ERROR));
 
         loggingListener.testRunFinished(new Result());
-        assertThat(xyzLogger.getLevel(), equalTo(Level.INFO));
-        assertThat(abcLogger.getLevel(), equalTo(Level.INFO));
+        assertThat(xyzLogger.getLevel(), equalTo(level));
+        assertThat(abcLogger.getLevel(), equalTo(level));
+        assertThat(fooLogger.getLevel(), equalTo(level));
+        assertThat(fooBarLogger.getLevel(), equalTo(level));
     }
 
     public void testCustomLevelPerClassAndPerMethod() throws Exception {
@@ -109,10 +128,12 @@ public void testCustomLevelPerClassAndPerMethod() throws Exception {
         Logger abcLogger = Loggers.getLogger("abc");
         Logger xyzLogger = Loggers.getLogger("xyz");
 
-        assertThat(xyzLogger.getLevel(), equalTo(Level.INFO));
-        assertThat(abcLogger.getLevel(), equalTo(Level.INFO));
+        final Level level = ESLoggerFactory.getRootLogger().getLevel();
+
+        assertThat(xyzLogger.getLevel(), equalTo(level));
+        assertThat(abcLogger.getLevel(), equalTo(level));
         loggingListener.testRunStarted(suiteDescription);
-        assertThat(xyzLogger.getLevel(), equalTo(Level.INFO));
+        assertThat(xyzLogger.getLevel(), equalTo(level));
         assertThat(abcLogger.getLevel(), equalTo(Level.WARN));
 
         Method method = TestClass.class.getMethod("annotatedTestMethod");
@@ -123,7 +144,7 @@ public void testCustomLevelPerClassAndPerMethod() throws Exception {
         assertThat(abcLogger.getLevel(), equalTo(Level.WARN));
 
         loggingListener.testFinished(testDescription);
-        assertThat(xyzLogger.getLevel(), equalTo(Level.INFO));
+        assertThat(xyzLogger.getLevel(), equalTo(level));
         assertThat(abcLogger.getLevel(), equalTo(Level.WARN));
 
         Method method2 = TestClass.class.getMethod("annotatedTestMethod2");
@@ -134,34 +155,85 @@ public void testCustomLevelPerClassAndPerMethod() throws Exception {
         assertThat(abcLogger.getLevel(), equalTo(Level.TRACE));
 
         loggingListener.testFinished(testDescription2);
-        assertThat(xyzLogger.getLevel(), equalTo(Level.INFO));
+        assertThat(xyzLogger.getLevel(), equalTo(level));
         assertThat(abcLogger.getLevel(), equalTo(Level.WARN));
 
         loggingListener.testRunFinished(new Result());
-        assertThat(xyzLogger.getLevel(), equalTo(Level.INFO));
-        assertThat(abcLogger.getLevel(), equalTo(Level.INFO));
+        assertThat(xyzLogger.getLevel(), equalTo(level));
+        assertThat(abcLogger.getLevel(), equalTo(level));
+    }
+
+    public void testInvalidClassTestLoggingAnnotation() throws Exception {
+        final LoggingListener loggingListener = new LoggingListener();
+
+        final Description suiteDescription = Description.createSuiteDescription(InvalidClass.class);
+
+        final IllegalArgumentException e =
+            expectThrows(IllegalArgumentException.class, () -> loggingListener.testRunStarted(suiteDescription));
+        assertThat(e.getMessage(), equalTo("invalid test logging annotation [abc]"));
+    }
+
+    public void testInvalidMethodTestLoggingAnnotation() throws Exception {
+        final LoggingListener loggingListener = new LoggingListener();
+
+        final Description suiteDescription = Description.createSuiteDescription(InvalidMethod.class);
+
+        loggingListener.testRunStarted(suiteDescription);
+
+        final Method method = InvalidMethod.class.getMethod("invalidMethod");
+        final TestLogging annotation = method.getAnnotation(TestLogging.class);
+        Description testDescription = Description.createTestDescription(InvalidMethod.class, "invalidMethod", annotation);
+        final IllegalArgumentException e =
+            expectThrows(IllegalArgumentException.class, () -> loggingListener.testStarted(testDescription));
+        assertThat(e.getMessage(), equalTo("invalid test logging annotation [abc:INFO:WARN]"));
     }
 
     /**
-     * dummy class used to create a junit suite description that has the @TestLogging annotation
+     * Dummy class used to create a JUnit suite description that has the {@link TestLogging} annotation.
      */
-    @TestLogging("abc:WARN")
+    @TestLogging("abc:WARN,foo:WARN,foo.bar:ERROR")
     public static class AnnotatedTestClass {
 
     }
 
     /**
-     * dummy class used to create a junit suite description that doesn't have the @TestLogging annotation, but its test methods have it
+     * Dummy class used to create a JUnit suite description that doesn't have the {@link TestLogging} annotation, but its test methods have
+     * it.
      */
     public static class TestClass {
 
         @SuppressWarnings("unused")
-        @TestLogging("xyz:TRACE")
-        public void annotatedTestMethod() {}
+        @TestLogging("xyz:TRACE,foo:WARN,foo.bar:ERROR")
+        public void annotatedTestMethod() {
+
+        }
 
         @SuppressWarnings("unused")
         @TestLogging("abc:TRACE,xyz:DEBUG")
-        public void annotatedTestMethod2() {}
+        public void annotatedTestMethod2() {
+
+        }
+
+    }
+
+    /**
+     * Dummy class with an invalid {@link TestLogging} annotation.
+     */
+    @TestLogging("abc")
+    public static class InvalidClass {
+
+    }
+
+    /**
+     * Dummy class with an invalid {@link TestLogging} annotation on a method.
+     */
+    public static class InvalidMethod {
+
+        @SuppressWarnings("unused")
+        @TestLogging("abc:INFO:WARN")
+        public void invalidMethod() {
+
+        }
 
     }
 
diff --git a/test/framework/src/test/java/org/elasticsearch/test/test/VersionUtilsTests.java b/test/framework/src/test/java/org/elasticsearch/test/test/VersionUtilsTests.java
index 97683b620f077..c97188e7016aa 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/test/VersionUtilsTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/test/VersionUtilsTests.java
@@ -27,7 +27,7 @@
 public class VersionUtilsTests extends ESTestCase {
 
     public void testAllVersionsSorted() {
-        List<Version> allVersions = VersionUtils.allVersions();
+        List<Version> allVersions = VersionUtils.allReleasedVersions();
         for (int i = 0, j = 1; j < allVersions.size(); ++i, ++j) {
             assertTrue(allVersions.get(i).before(allVersions.get(j)));
         }
@@ -54,9 +54,9 @@ public void testRandomVersionBetween() {
         got = VersionUtils.randomVersionBetween(random(), null, Version.V_5_0_0_alpha1);
         assertTrue(got.onOrAfter(VersionUtils.getFirstVersion()));
         assertTrue(got.onOrBefore(Version.V_5_0_0_alpha1));
-        got = VersionUtils.randomVersionBetween(random(), null, VersionUtils.allVersions().get(0));
+        got = VersionUtils.randomVersionBetween(random(), null, VersionUtils.allReleasedVersions().get(0));
         assertTrue(got.onOrAfter(VersionUtils.getFirstVersion()));
-        assertTrue(got.onOrBefore(VersionUtils.allVersions().get(0)));
+        assertTrue(got.onOrBefore(VersionUtils.allReleasedVersions().get(0)));
 
         // unbounded upper
         got = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, null);
diff --git a/test/framework/src/test/java/org/elasticsearch/transport/MockTcpTransportTests.java b/test/framework/src/test/java/org/elasticsearch/transport/MockTcpTransportTests.java
index 4ee7b0c7b2770..bb43c5e0c4e69 100644
--- a/test/framework/src/test/java/org/elasticsearch/transport/MockTcpTransportTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/transport/MockTcpTransportTests.java
@@ -19,23 +19,48 @@
 package org.elasticsearch.transport;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.network.NetworkService;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.test.transport.MockTransportService;
 
+import java.io.IOException;
 import java.util.Collections;
 
 public class MockTcpTransportTests extends AbstractSimpleTransportTestCase {
+
     @Override
-    protected MockTransportService build(Settings settings, Version version) {
+    protected MockTransportService build(Settings settings, Version version, ClusterSettings clusterSettings, boolean doHandshake) {
         NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(Collections.emptyList());
         Transport transport = new MockTcpTransport(settings, threadPool, BigArrays.NON_RECYCLING_INSTANCE,
-            new NoneCircuitBreakerService(), namedWriteableRegistry, new NetworkService(settings, Collections.emptyList()), version);
-        MockTransportService mockTransportService = new MockTransportService(Settings.EMPTY, transport, threadPool);
+            new NoneCircuitBreakerService(), namedWriteableRegistry, new NetworkService(settings, Collections.emptyList()), version) {
+            @Override
+            protected Version executeHandshake(DiscoveryNode node, MockChannel mockChannel, TimeValue timeout) throws IOException,
+                InterruptedException {
+                if (doHandshake) {
+                    return super.executeHandshake(node, mockChannel, timeout);
+                } else {
+                    return version.minimumCompatibilityVersion();
+                }
+            }
+        };
+        MockTransportService mockTransportService =
+            MockTransportService.createNewService(Settings.EMPTY, transport, version, threadPool, clusterSettings);
         mockTransportService.start();
         return mockTransportService;
     }
+
+    @Override
+    protected void closeConnectionChannel(Transport transport, Transport.Connection connection) throws IOException {
+        final MockTcpTransport t = (MockTcpTransport) transport;
+        @SuppressWarnings("unchecked") final TcpTransport<MockTcpTransport.MockChannel>.NodeChannels channels =
+                (TcpTransport<MockTcpTransport.MockChannel>.NodeChannels) connection;
+        t.closeChannels(channels.getChannels().subList(0, randomIntBetween(1, channels.getChannels().size())), true);
+    }
+
 }
diff --git a/test/logger-usage/build.gradle b/test/logger-usage/build.gradle
index a0acd58cff4b3..98fe76bfcdc31 100644
--- a/test/logger-usage/build.gradle
+++ b/test/logger-usage/build.gradle
@@ -21,14 +21,27 @@ import org.elasticsearch.gradle.precommit.PrecommitTasks
 
 dependencies {
   compile 'org.ow2.asm:asm-debug-all:5.0.4' // use asm-debug-all as asm-all is broken
+  compile "org.apache.logging.log4j:log4j-api:${versions.log4j}"
   testCompile "org.elasticsearch.test:framework:${version}"
 }
 
-// https://github.com/elastic/elasticsearch/issues/20243
-// loggerUsageCheck.enabled = false
+loggerUsageCheck.enabled = false
 
 forbiddenApisMain.enabled = true // disabled by parent project
 forbiddenApisMain {
   signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')] // does not depend on core, only jdk signatures
 }
-jarHell.enabled = true // disabled by parent project
\ No newline at end of file
+jarHell.enabled = true // disabled by parent project
+
+thirdPartyAudit.excludes = [
+  // log4j
+  'org.osgi.framework.AdaptPermission',
+  'org.osgi.framework.AdminPermission',
+  'org.osgi.framework.Bundle',
+  'org.osgi.framework.BundleActivator',
+  'org.osgi.framework.BundleContext',
+  'org.osgi.framework.BundleEvent',
+  'org.osgi.framework.SynchronousBundleListener',
+  'org.osgi.framework.wiring.BundleWire',
+  'org.osgi.framework.wiring.BundleWiring'
+]
diff --git a/test/logger-usage/src/main/java/org/elasticsearch/test/loggerusage/ESLoggerUsageChecker.java b/test/logger-usage/src/main/java/org/elasticsearch/test/loggerusage/ESLoggerUsageChecker.java
index 041d21cc76282..4ee2e473890e2 100644
--- a/test/logger-usage/src/main/java/org/elasticsearch/test/loggerusage/ESLoggerUsageChecker.java
+++ b/test/logger-usage/src/main/java/org/elasticsearch/test/loggerusage/ESLoggerUsageChecker.java
@@ -19,6 +19,10 @@
 
 package org.elasticsearch.test.loggerusage;
 
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.Marker;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.apache.logging.log4j.util.Supplier;
 import org.objectweb.asm.AnnotationVisitor;
 import org.objectweb.asm.ClassReader;
 import org.objectweb.asm.ClassVisitor;
@@ -52,9 +56,16 @@
 import java.util.function.Predicate;
 
 public class ESLoggerUsageChecker {
-    public static final String LOGGER_CLASS = "org.elasticsearch.common.logging.ESLogger";
-    public static final String THROWABLE_CLASS = "java.lang.Throwable";
-    public static final List<String> LOGGER_METHODS = Arrays.asList("trace", "debug", "info", "warn", "error");
+    public static final Type LOGGER_CLASS = Type.getType(Logger.class);
+    public static final Type THROWABLE_CLASS = Type.getType(Throwable.class);
+    public static final Type STRING_CLASS = Type.getType(String.class);
+    public static final Type STRING_ARRAY_CLASS = Type.getType(String[].class);
+    public static final Type PARAMETERIZED_MESSAGE_CLASS = Type.getType(ParameterizedMessage.class);
+    public static final Type OBJECT_CLASS = Type.getType(Object.class);
+    public static final Type OBJECT_ARRAY_CLASS = Type.getType(Object[].class);
+    public static final Type SUPPLIER_ARRAY_CLASS = Type.getType(Supplier[].class);
+    public static final Type MARKER_CLASS = Type.getType(Marker.class);
+    public static final List<String> LOGGER_METHODS = Arrays.asList("trace", "debug", "info", "warn", "error", "fatal");
     public static final String IGNORE_CHECKS_ANNOTATION = "org.elasticsearch.common.SuppressLoggerChecks";
 
     @SuppressForbidden(reason = "command line tool")
@@ -143,7 +154,7 @@ public String getErrorLines() {
             simpleClassName = simpleClassName + ".java";
             StringBuilder sb = new StringBuilder();
             sb.append("Bad usage of ");
-            sb.append(LOGGER_CLASS).append("#").append(logMethodName);
+            sb.append(LOGGER_CLASS.getClassName()).append("#").append(logMethodName);
             sb.append(": ");
             sb.append(errorMessage);
             sb.append("\n\tat ");
@@ -165,7 +176,7 @@ private static class ClassChecker extends ClassVisitor {
         private final Consumer<WrongLoggerUsage> wrongUsageCallback;
         private final Predicate<String> methodsToCheck;
 
-        public ClassChecker(Consumer<WrongLoggerUsage> wrongUsageCallback, Predicate<String> methodsToCheck) {
+        ClassChecker(Consumer<WrongLoggerUsage> wrongUsageCallback, Predicate<String> methodsToCheck) {
             super(Opcodes.ASM5);
             this.wrongUsageCallback = wrongUsageCallback;
             this.methodsToCheck = methodsToCheck;
@@ -199,7 +210,7 @@ private static class MethodChecker extends MethodVisitor {
         private final Consumer<WrongLoggerUsage> wrongUsageCallback;
         private boolean ignoreChecks;
 
-        public MethodChecker(String className, int access, String name, String desc, Consumer<WrongLoggerUsage> wrongUsageCallback) {
+        MethodChecker(String className, int access, String name, String desc, Consumer<WrongLoggerUsage> wrongUsageCallback) {
             super(Opcodes.ASM5, new MethodNode(access, name, desc, null, null));
             this.className = className;
             this.wrongUsageCallback = wrongUsageCallback;
@@ -230,7 +241,7 @@ public void findBadLoggerUsages(MethodNode methodNode) {
             } catch (AnalyzerException e) {
                 throw new RuntimeException("Internal error: failed in analysis step", e);
             }
-            Frame<BasicValue>[] stringFrames = stringPlaceHolderAnalyzer.getFrames();
+            Frame<BasicValue>[] logMessageFrames = stringPlaceHolderAnalyzer.getFrames();
             Frame<BasicValue>[] arraySizeFrames = arraySizeAnalyzer.getFrames();
             AbstractInsnNode[] insns = methodNode.instructions.toArray();
             int lineNumber = -1;
@@ -240,53 +251,141 @@ public void findBadLoggerUsages(MethodNode methodNode) {
                     LineNumberNode lineNumberNode = (LineNumberNode) insn;
                     lineNumber = lineNumberNode.line;
                 }
-                if (insn.getOpcode() == Opcodes.INVOKEVIRTUAL) {
+                if (insn.getOpcode() == Opcodes.INVOKEINTERFACE) {
                     MethodInsnNode methodInsn = (MethodInsnNode) insn;
-                    if (Type.getObjectType(methodInsn.owner).getClassName().equals(LOGGER_CLASS) == false) {
-                        continue;
-                    }
-                    if (LOGGER_METHODS.contains(methodInsn.name) == false) {
-                        continue;
-                    }
-                    BasicValue varArgsSizeObject = getStackValue(arraySizeFrames[i], 0); // last argument
-                    if (varArgsSizeObject instanceof ArraySizeBasicValue == false) {
-                        wrongUsageCallback.accept(new WrongLoggerUsage(className, methodNode.name, methodInsn.name, lineNumber,
-                            "Could not determine size of varargs array"));
-                        continue;
-                    }
-                    ArraySizeBasicValue varArgsSize = (ArraySizeBasicValue) varArgsSizeObject;
-                    Type[] argumentTypes = Type.getArgumentTypes(methodInsn.desc);
-                    BasicValue logMessageLengthObject = getStackValue(stringFrames[i], argumentTypes.length - 1); // first argument
-                    if (logMessageLengthObject instanceof PlaceHolderStringBasicValue == false) {
-                        if (varArgsSize.minValue > 0) {
-                            wrongUsageCallback.accept(new WrongLoggerUsage(className, methodNode.name, methodInsn.name, lineNumber,
-                                "First argument must be a string constant so that we can statically ensure proper place holder usage"));
+                    if (Type.getObjectType(methodInsn.owner).equals(LOGGER_CLASS)) {
+                        if (LOGGER_METHODS.contains(methodInsn.name) == false) {
                             continue;
+                        }
+
+                        Type[] argumentTypes = Type.getArgumentTypes(methodInsn.desc);
+                        int markerOffset = 0;
+                        if (argumentTypes[0].equals(MARKER_CLASS)) {
+                            markerOffset = 1;
+                        }
+
+                        int lengthWithoutMarker = argumentTypes.length - markerOffset;
+
+                        if (lengthWithoutMarker == 2 &&
+                            argumentTypes[markerOffset + 0].equals(STRING_CLASS) &&
+                            (argumentTypes[markerOffset + 1].equals(OBJECT_ARRAY_CLASS) ||
+                                argumentTypes[markerOffset + 1].equals(SUPPLIER_ARRAY_CLASS))) {
+                            // VARARGS METHOD: debug(Marker?, String, (Object...|Supplier...))
+                            checkArrayArgs(methodNode, logMessageFrames[i], arraySizeFrames[i], lineNumber, methodInsn, markerOffset + 0,
+                                markerOffset + 1);
+                        } else if (lengthWithoutMarker >= 2 &&
+                            argumentTypes[markerOffset + 0].equals(STRING_CLASS) &&
+                            argumentTypes[markerOffset + 1].equals(OBJECT_CLASS)) {
+                            // MULTI-PARAM METHOD: debug(Marker?, String, Object p0, ...)
+                            checkFixedArityArgs(methodNode, logMessageFrames[i], lineNumber, methodInsn, markerOffset + 0,
+                                lengthWithoutMarker - 1);
+                        } else if ((lengthWithoutMarker == 1 || lengthWithoutMarker == 2) &&
+                            lengthWithoutMarker == 2 ? argumentTypes[markerOffset + 1].equals(THROWABLE_CLASS) : true) {
+                            // all the rest: debug(Marker?, (Message|MessageSupplier|CharSequence|Object|String|Supplier), Throwable?)
+                            checkFixedArityArgs(methodNode, logMessageFrames[i], lineNumber, methodInsn, markerOffset + 0, 0);
                         } else {
-                            // don't check logger usage for logger.warn(someObject) as someObject will be fully logged
-                            continue;
+                            throw new IllegalStateException("Method invoked on " + LOGGER_CLASS.getClassName() +
+                                " that is not supported by logger usage checker");
                         }
                     }
-                    PlaceHolderStringBasicValue logMessageLength = (PlaceHolderStringBasicValue) logMessageLengthObject;
-                    if (logMessageLength.minValue != logMessageLength.maxValue) {
-                        wrongUsageCallback.accept(new WrongLoggerUsage(className, methodNode.name, methodInsn.name, lineNumber,
-                            "Multiple log messages with conflicting number of place holders"));
-                        continue;
-                    }
-                    if (varArgsSize.minValue != varArgsSize.maxValue) {
-                        wrongUsageCallback.accept(new WrongLoggerUsage(className, methodNode.name, methodInsn.name, lineNumber,
-                            "Multiple parameter arrays with conflicting sizes"));
-                        continue;
-                    }
-                    assert logMessageLength.minValue == logMessageLength.maxValue && varArgsSize.minValue == varArgsSize.maxValue;
-                    if (logMessageLength.minValue != varArgsSize.minValue) {
-                        wrongUsageCallback.accept(new WrongLoggerUsage(className, methodNode.name, methodInsn.name, lineNumber,
-                            "Expected " + logMessageLength.minValue + " arguments but got " + varArgsSize.minValue));
-                        continue;
+                } else if (insn.getOpcode() == Opcodes.INVOKESPECIAL) { // constructor invocation
+                    MethodInsnNode methodInsn = (MethodInsnNode) insn;
+                    if (Type.getObjectType(methodInsn.owner).equals(PARAMETERIZED_MESSAGE_CLASS)) {
+                        Type[] argumentTypes = Type.getArgumentTypes(methodInsn.desc);
+                        if (argumentTypes.length == 2 &&
+                            argumentTypes[0].equals(STRING_CLASS) &&
+                            argumentTypes[1].equals(OBJECT_ARRAY_CLASS)) {
+                            checkArrayArgs(methodNode, logMessageFrames[i], arraySizeFrames[i], lineNumber, methodInsn, 0, 1);
+                        } else if (argumentTypes.length == 2 &&
+                            argumentTypes[0].equals(STRING_CLASS) &&
+                            argumentTypes[1].equals(OBJECT_CLASS)) {
+                            checkFixedArityArgs(methodNode, logMessageFrames[i], lineNumber, methodInsn, 0, 1);
+                        } else if (argumentTypes.length == 3 &&
+                            argumentTypes[0].equals(STRING_CLASS) &&
+                            argumentTypes[1].equals(OBJECT_CLASS) &&
+                            argumentTypes[2].equals(OBJECT_CLASS)) {
+                            checkFixedArityArgs(methodNode, logMessageFrames[i], lineNumber, methodInsn, 0, 2);
+                        } else if (argumentTypes.length == 3 &&
+                            argumentTypes[0].equals(STRING_CLASS) &&
+                            argumentTypes[1].equals(OBJECT_ARRAY_CLASS) &&
+                            argumentTypes[2].equals(THROWABLE_CLASS)) {
+                            checkArrayArgs(methodNode, logMessageFrames[i], arraySizeFrames[i], lineNumber, methodInsn, 0, 1);
+                        } else if (argumentTypes.length == 3 &&
+                            argumentTypes[0].equals(STRING_CLASS) &&
+                            argumentTypes[1].equals(STRING_ARRAY_CLASS) &&
+                            argumentTypes[2].equals(THROWABLE_CLASS)) {
+                            checkArrayArgs(methodNode, logMessageFrames[i], arraySizeFrames[i], lineNumber, methodInsn, 0, 1);
+                        } else {
+                            throw new IllegalStateException("Constructor invoked on " + PARAMETERIZED_MESSAGE_CLASS.getClassName() +
+                                " that is not supported by logger usage checker");
+                        }
                     }
                 }
             }
         }
+
+        private void checkFixedArityArgs(MethodNode methodNode, Frame<BasicValue> logMessageFrame, int lineNumber,
+                                         MethodInsnNode methodInsn, int messageIndex, int positionalArgsLength) {
+            PlaceHolderStringBasicValue logMessageLength = checkLogMessageConsistency(methodNode, logMessageFrame, lineNumber, methodInsn,
+                messageIndex, positionalArgsLength);
+            if (logMessageLength == null) {
+                return;
+            }
+            if (logMessageLength.minValue != positionalArgsLength) {
+                wrongUsageCallback.accept(new WrongLoggerUsage(className, methodNode.name, methodInsn.name, lineNumber,
+                    "Expected " + logMessageLength.minValue + " arguments but got " + positionalArgsLength));
+                return;
+            }
+        }
+
+        private void checkArrayArgs(MethodNode methodNode, Frame<BasicValue> logMessageFrame, Frame<BasicValue> arraySizeFrame,
+                                    int lineNumber, MethodInsnNode methodInsn, int messageIndex, int arrayIndex) {
+            BasicValue arraySizeObject = getStackValue(arraySizeFrame, methodInsn, arrayIndex);
+            if (arraySizeObject instanceof ArraySizeBasicValue == false) {
+                wrongUsageCallback.accept(new WrongLoggerUsage(className, methodNode.name, methodInsn.name, lineNumber,
+                    "Could not determine size of array"));
+                return;
+            }
+            ArraySizeBasicValue arraySize = (ArraySizeBasicValue) arraySizeObject;
+            PlaceHolderStringBasicValue logMessageLength = checkLogMessageConsistency(methodNode, logMessageFrame, lineNumber, methodInsn,
+                messageIndex, arraySize.minValue);
+            if (logMessageLength == null) {
+                return;
+            }
+            if (arraySize.minValue != arraySize.maxValue) {
+                wrongUsageCallback.accept(new WrongLoggerUsage(className, methodNode.name, methodInsn.name, lineNumber,
+                    "Multiple parameter arrays with conflicting sizes"));
+                return;
+            }
+            assert logMessageLength.minValue == logMessageLength.maxValue && arraySize.minValue == arraySize.maxValue;
+            if (logMessageLength.minValue != arraySize.minValue) {
+                wrongUsageCallback.accept(new WrongLoggerUsage(className, methodNode.name, methodInsn.name, lineNumber,
+                    "Expected " + logMessageLength.minValue + " arguments but got " + arraySize.minValue));
+                return;
+            }
+        }
+
+        private PlaceHolderStringBasicValue checkLogMessageConsistency(MethodNode methodNode, Frame<BasicValue> logMessageFrame,
+                                                                       int lineNumber, MethodInsnNode methodInsn, int messageIndex,
+                                                                       int argsSize) {
+            BasicValue logMessageLengthObject = getStackValue(logMessageFrame, methodInsn, messageIndex);
+            if (logMessageLengthObject instanceof PlaceHolderStringBasicValue == false) {
+                if (argsSize > 0) {
+                    wrongUsageCallback.accept(new WrongLoggerUsage(className, methodNode.name, methodInsn.name, lineNumber,
+                        "First argument must be a string constant so that we can statically ensure proper place holder usage"));
+                } else {
+                    // don't check logger usage for logger.warn(someObject)
+                }
+                return null;
+            }
+            PlaceHolderStringBasicValue logMessageLength = (PlaceHolderStringBasicValue) logMessageLengthObject;
+            if (logMessageLength.minValue != logMessageLength.maxValue) {
+                wrongUsageCallback.accept(new WrongLoggerUsage(className, methodNode.name, methodInsn.name, lineNumber,
+                    "Multiple log messages with conflicting number of place holders"));
+                return null;
+            }
+            return logMessageLength;
+        }
     }
 
     private static int calculateNumberOfPlaceHolders(String message) {
@@ -300,22 +399,23 @@ private static int calculateNumberOfPlaceHolders(String message) {
         return count;
     }
 
-    private static BasicValue getStackValue(Frame<BasicValue> f, int index) {
+    private static BasicValue getStackValue(Frame<BasicValue> f, MethodInsnNode methodInsn, int index) {
+        int relIndex = Type.getArgumentTypes(methodInsn.desc).length - 1 - index;
         int top = f.getStackSize() - 1;
-        return index <= top ? f.getStack(top - index) : null;
+        return relIndex <= top ? f.getStack(top - relIndex) : null;
     }
 
     private static class IntMinMaxTrackingBasicValue extends BasicValue {
         protected final int minValue;
         protected final int maxValue;
 
-        public IntMinMaxTrackingBasicValue(Type type, int value) {
+        IntMinMaxTrackingBasicValue(Type type, int value) {
             super(type);
             this.minValue = value;
             this.maxValue = value;
         }
 
-        public IntMinMaxTrackingBasicValue(Type type, int minValue, int maxValue) {
+        IntMinMaxTrackingBasicValue(Type type, int minValue, int maxValue) {
             super(type);
             this.minValue = minValue;
             this.maxValue = maxValue;
@@ -354,27 +454,27 @@ public String toString() {
     private static final class PlaceHolderStringBasicValue extends IntMinMaxTrackingBasicValue {
         public static final Type STRING_OBJECT_TYPE = Type.getObjectType("java/lang/String");
 
-        public PlaceHolderStringBasicValue(int placeHolders) {
+        PlaceHolderStringBasicValue(int placeHolders) {
             super(STRING_OBJECT_TYPE, placeHolders);
         }
 
-        public PlaceHolderStringBasicValue(int minPlaceHolders, int maxPlaceHolders) {
+        PlaceHolderStringBasicValue(int minPlaceHolders, int maxPlaceHolders) {
             super(STRING_OBJECT_TYPE, minPlaceHolders, maxPlaceHolders);
         }
     }
 
     private static final class ArraySizeBasicValue extends IntMinMaxTrackingBasicValue {
-        public ArraySizeBasicValue(Type type, int minArraySize, int maxArraySize) {
+        ArraySizeBasicValue(Type type, int minArraySize, int maxArraySize) {
             super(type, minArraySize, maxArraySize);
         }
     }
 
     private static final class IntegerConstantBasicValue extends IntMinMaxTrackingBasicValue {
-        public IntegerConstantBasicValue(Type type, int constant) {
+        IntegerConstantBasicValue(Type type, int constant) {
             super(type, constant);
         }
 
-        public IntegerConstantBasicValue(Type type, int minConstant, int maxConstant) {
+        IntegerConstantBasicValue(Type type, int minConstant, int maxConstant) {
             super(type, minConstant, maxConstant);
         }
     }
diff --git a/test/logger-usage/src/test/java/org/elasticsearch/test/loggerusage/ESLoggerUsageTests.java b/test/logger-usage/src/test/java/org/elasticsearch/test/loggerusage/ESLoggerUsageTests.java
index 8d2d772d56778..ea60b0cf5b7ca 100644
--- a/test/logger-usage/src/test/java/org/elasticsearch/test/loggerusage/ESLoggerUsageTests.java
+++ b/test/logger-usage/src/test/java/org/elasticsearch/test/loggerusage/ESLoggerUsageTests.java
@@ -20,31 +20,29 @@
 package org.elasticsearch.test.loggerusage;
 
 import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.Marker;
+import org.apache.logging.log4j.message.Message;
 import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.apache.logging.log4j.util.MessageSupplier;
 import org.apache.logging.log4j.util.Supplier;
 import org.elasticsearch.common.SuppressLoggerChecks;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.loggerusage.ESLoggerUsageChecker.WrongLoggerUsage;
+import org.hamcrest.Matchers;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.lang.reflect.Constructor;
 import java.lang.reflect.Method;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.function.Predicate;
+import java.util.stream.Stream;
 
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.hasItem;
-import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
 
 public class ESLoggerUsageTests extends ESTestCase {
 
-    // needed to avoid the test suite from failing for having no tests
-    public void testSoThatTestsDoNotFail() {
-
-    }
-
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/20243")
     public void testLoggerUsageChecks() throws IOException {
         for (Method method : getClass().getMethods()) {
             if (method.getDeclaringClass().equals(getClass())) {
@@ -52,7 +50,8 @@ public void testLoggerUsageChecks() throws IOException {
                     logger.info("Checking logger usage for method {}", method.getName());
                     InputStream classInputStream = getClass().getResourceAsStream(getClass().getSimpleName() + ".class");
                     List<WrongLoggerUsage> errors = new ArrayList<>();
-                    ESLoggerUsageChecker.check(errors::add, classInputStream, Predicate.isEqual(method.getName()));
+                    ESLoggerUsageChecker.check(errors::add, classInputStream,
+                        m -> m.equals(method.getName()) || m.startsWith("lambda$" + method.getName()));
                     if (method.getName().startsWith("checkFail")) {
                         assertFalse("Expected " + method.getName() + " to have wrong Logger usage", errors.isEmpty());
                     } else {
@@ -65,27 +64,57 @@ public void testLoggerUsageChecks() throws IOException {
         }
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/20243")
-    public void testLoggerUsageCheckerCompatibilityWithESLogger() throws NoSuchMethodException {
-        assertThat(ESLoggerUsageChecker.LOGGER_CLASS, equalTo(Logger.class.getName()));
-        assertThat(ESLoggerUsageChecker.THROWABLE_CLASS, equalTo(Throwable.class.getName()));
-        int varargsMethodCount = 0;
+    public void testLoggerUsageCheckerCompatibilityWithLog4j2Logger() throws NoSuchMethodException {
         for (Method method : Logger.class.getMethods()) {
-            if (method.isVarArgs()) {
-                // check that logger usage checks all varargs methods
-                assertThat(ESLoggerUsageChecker.LOGGER_METHODS, hasItem(method.getName()));
-                varargsMethodCount++;
+            if (ESLoggerUsageChecker.LOGGER_METHODS.contains(method.getName())) {
+                assertThat(method.getParameterTypes().length, greaterThanOrEqualTo(1));
+                int markerOffset = method.getParameterTypes()[0].equals(Marker.class) ? 1 : 0;
+                int paramLength = method.getParameterTypes().length - markerOffset;
+                if (method.isVarArgs()) {
+                    assertEquals(2, paramLength);
+                    assertEquals(String.class, method.getParameterTypes()[markerOffset]);
+                    assertThat(method.getParameterTypes()[markerOffset + 1], Matchers.<Class<?>>isOneOf(Object[].class, Supplier[].class));
+                } else {
+                    assertThat(method.getParameterTypes()[markerOffset], Matchers.<Class<?>>isOneOf(Message.class, MessageSupplier.class,
+                        CharSequence.class, Object.class, String.class, Supplier.class));
+
+                    if (paramLength == 2) {
+                        assertThat(method.getParameterTypes()[markerOffset + 1], Matchers.<Class<?>>isOneOf(Throwable.class, Object.class));
+                        if (method.getParameterTypes()[markerOffset + 1].equals(Object.class)) {
+                            assertEquals(String.class, method.getParameterTypes()[markerOffset]);
+                        }
+                    }
+                    if (paramLength > 2) {
+                        assertEquals(String.class, method.getParameterTypes()[markerOffset]);
+                        assertThat(paramLength, lessThanOrEqualTo(11));
+                        for (int i = 1; i < paramLength; i++) {
+                            assertEquals(Object.class, method.getParameterTypes()[markerOffset + i]);
+                        }
+                    }
+                }
             }
         }
-        // currently we have two overloaded methods for each of debug, info, ...
-        // if that changes, we might want to have another look at the usage checker
-        assertThat(varargsMethodCount, equalTo(ESLoggerUsageChecker.LOGGER_METHODS.size() * 2));
 
-        // check that signature is same as we expect in the usage checker
         for (String methodName : ESLoggerUsageChecker.LOGGER_METHODS) {
-            assertThat(Logger.class.getMethod(methodName, String.class, Object[].class), notNullValue());
-            assertThat(Logger.class.getMethod(methodName, String.class, Throwable.class, Object[].class), notNullValue());
+            assertEquals(48, Stream.of(Logger.class.getMethods()).filter(m -> methodName.equals(m.getName())).count());
+        }
+
+        for (Constructor<?> constructor : ParameterizedMessage.class.getConstructors()) {
+            assertThat(constructor.getParameterTypes().length, greaterThanOrEqualTo(2));
+            assertEquals(String.class, constructor.getParameterTypes()[0]);
+            assertThat(constructor.getParameterTypes()[1], Matchers.<Class<?>>isOneOf(String[].class, Object[].class, Object.class));
+
+            if (constructor.getParameterTypes().length > 2) {
+                assertEquals(3, constructor.getParameterTypes().length);
+                if (constructor.getParameterTypes()[1].equals(Object.class)) {
+                    assertEquals(Object.class, constructor.getParameterTypes()[2]);
+                } else {
+                    assertEquals(Throwable.class, constructor.getParameterTypes()[2]);
+                }
+            }
         }
+
+        assertEquals(5, ParameterizedMessage.class.getConstructors().length);
     }
 
     public void checkNumberOfArguments1() {
@@ -110,7 +139,6 @@ public void checkFailNumberOfArguments2() {
     }
 
     public void checkNumberOfArguments3() {
-        // long argument list (> 5), emits different bytecode
         logger.info("Hello {}, {}, {}, {}, {}, {}, {}", "world", 2, "third argument", 4, 5, 6, new String("last arg"));
     }
 
@@ -118,6 +146,30 @@ public void checkFailNumberOfArguments3() {
         logger.info("Hello {}, {}, {}, {}, {}, {}, {}", "world", 2, "third argument", 4, 5, 6, 7, new String("last arg"));
     }
 
+    public void checkNumberOfArgumentsParameterizedMessage1() {
+        logger.info(new ParameterizedMessage("Hello {}, {}, {}", "world", 2, "third argument"));
+    }
+
+    public void checkFailNumberOfArgumentsParameterizedMessage1() {
+        logger.info(new ParameterizedMessage("Hello {}, {}", "world", 2, "third argument"));
+    }
+
+    public void checkNumberOfArgumentsParameterizedMessage2() {
+        logger.info(new ParameterizedMessage("Hello {}, {}", "world", 2));
+    }
+
+    public void checkFailNumberOfArgumentsParameterizedMessage2() {
+        logger.info(new ParameterizedMessage("Hello {}, {}, {}", "world", 2));
+    }
+
+    public void checkNumberOfArgumentsParameterizedMessage3() {
+        logger.info((Supplier<?>) () -> new ParameterizedMessage("Hello {}, {}, {}", "world", 2, "third argument"));
+    }
+
+    public void checkFailNumberOfArgumentsParameterizedMessage3() {
+        logger.info((Supplier<?>) () -> new ParameterizedMessage("Hello {}, {}", "world", 2, "third argument"));
+    }
+
     public void checkOrderOfExceptionArgument() {
         logger.info("Hello", new Exception());
     }